diff --git a/openssl/cmake/darwin_arm64.cmake b/openssl/cmake/darwin_arm64.cmake index 98b501f43..cc3d4465b 100644 --- a/openssl/cmake/darwin_arm64.cmake +++ b/openssl/cmake/darwin_arm64.cmake @@ -8,9 +8,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_wrap.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/darwin_arm64/aesv8-armx.S - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/darwin_arm64/bsaes-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/darwin_arm64/vpaes-armv8.S - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/armcap.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c @@ -83,11 +81,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -108,7 +101,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -118,6 +110,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_conv.c @@ -144,35 +137,23 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_arm64/armv8-mont.S + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/darwin_arm64/chacha-armv8-sve.S ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/darwin_arm64/chacha-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -198,9 +179,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -241,10 +220,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -254,7 +235,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -290,7 +270,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -324,18 +303,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_oct.c - ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_sm2p256.c - ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_sm2p256_table.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_smpl.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_backend.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/darwin_arm64/ecp_nistz256-armv8.S - ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/darwin_arm64/ecp_sm2p256-armv8.S + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -363,6 +341,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -370,9 +349,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -392,21 +369,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -428,10 +399,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -462,26 +431,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c ${PROJECT_SOURCE_DIR}/src/crypto/gen/darwin_arm64/arm64cpuid.S - ${PROJECT_SOURCE_DIR}/src/crypto/gen/darwin_arm64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c ${PROJECT_SOURCE_DIR}/src/crypto/kdf/kdf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/lhash/lh_stats.c ${PROJECT_SOURCE_DIR}/src/crypto/lhash/lhash.c - ${PROJECT_SOURCE_DIR}/src/crypto/md5/gen/darwin_arm64/md5-aarch64.S ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_dgst.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_one.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_sha1.c @@ -493,7 +453,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/ctr128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cts128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/gcm128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8-unroll8_64.S ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8_64.S ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/darwin_arm64/ghashv8-armx.S ${PROJECT_SOURCE_DIR}/src/crypto/modes/ocb128.c @@ -501,7 +460,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -578,24 +536,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -622,11 +571,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/darwin_arm64/keccak1600-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/darwin_arm64/sha1-armv8.S @@ -638,17 +582,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/gen/darwin_arm64/sm3-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm4/gen/darwin_arm64/sm4-armv8.S - ${PROJECT_SOURCE_DIR}/src/crypto/sm4/gen/darwin_arm64/vpsm4-armv8.S - ${PROJECT_SOURCE_DIR}/src/crypto/sm4/gen/darwin_arm64/vpsm4_ex-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/sm4/sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/sparse_array.c ${PROJECT_SOURCE_DIR}/src/crypto/srp/srp_lib.c @@ -661,17 +603,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -716,16 +651,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -734,12 +664,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -773,6 +700,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -810,9 +738,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -822,18 +747,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -843,25 +756,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -869,6 +774,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -876,14 +783,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -902,24 +805,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -928,9 +826,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -962,67 +859,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1030,6 +881,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1044,6 +896,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1053,4 +906,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DBSAES_ASM -DECP_NISTZ256_ASM -DECP_SM2P256_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_MONT -DOPENSSL_CPUID_OBJ -DOPENSSL_SM3_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSM4_ASM -DSTATIC_LEGACY -DVPAES_ASM -DVPSM4_ASM) +add_definitions(-DECP_NISTZ256_ASM -DKECCAK1600_ASM -DOPENSSL_BN_ASM_MONT -DOPENSSL_CPUID_OBJ -DOPENSSL_SM3_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSM4_ASM -DSTATIC_LEGACY -DVPAES_ASM) diff --git a/openssl/cmake/darwin_x64.cmake b/openssl/cmake/darwin_x64.cmake index 7eb387a31..ca2b3e945 100644 --- a/openssl/cmake/darwin_x64.cmake +++ b/openssl/cmake/darwin_x64.cmake @@ -12,7 +12,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/darwin_x64/aesni-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/darwin_x64/bsaes-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/darwin_x64/vpaes-x86_64.s - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -84,11 +83,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -109,7 +103,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -118,6 +111,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_sock.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/asm/x86_64-gcc.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/asm/x86_64-gcc.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c @@ -145,42 +139,32 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/rsaz-2k-avx512.s - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/rsaz-3k-avx512.s - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/rsaz-4k-avx512.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/rsaz-avx2.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/rsaz-avx512.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/rsaz-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/x86_64-gf2m.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/x86_64-mont.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/darwin_x64/x86_64-mont5.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp_x2.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp_x2.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/gen/darwin_x64/cmll-x86_64.s - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/darwin_x64/chacha-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -206,9 +190,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -249,10 +231,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -262,7 +246,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -298,7 +281,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -332,6 +314,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c @@ -342,6 +325,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/darwin_x64/ecp_nistz256-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/darwin_x64/x25519-x86_64.s + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -369,6 +353,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -376,9 +361,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -398,21 +381,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -434,10 +411,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -467,20 +442,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/darwin_x64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/gen/darwin_x64/x86_64cpuid.s ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -499,7 +466,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/ctr128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cts128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/gcm128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/darwin_x64/aes-gcm-avx512.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/darwin_x64/aesni-gcm-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/darwin_x64/ghash-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/ocb128.c @@ -507,7 +473,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -584,24 +549,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/darwin_x64/rc4-md5-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/darwin_x64/rc4-x86_64.s - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -628,11 +584,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/darwin_x64/keccak1600-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/darwin_x64/sha1-mb-x86_64.s @@ -646,10 +597,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -665,17 +616,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -720,16 +664,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -738,12 +677,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -777,6 +713,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -814,9 +751,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -826,18 +760,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -847,25 +769,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -873,6 +787,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -880,14 +796,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -906,24 +818,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -932,9 +839,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -966,67 +872,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1034,6 +894,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1048,6 +909,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1057,4 +919,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DAES_ASM -DBSAES_ASM -DCMLL_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DRC4_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM -DX25519_ASM) +add_definitions(-DAES_ASM -DBSAES_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM -DX25519_ASM) diff --git a/openssl/cmake/linux_arm.cmake b/openssl/cmake/linux_arm.cmake index b3b4f7a14..208582e4d 100644 --- a/openssl/cmake/linux_arm.cmake +++ b/openssl/cmake/linux_arm.cmake @@ -9,7 +9,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_arm/aes-armv4.S ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_arm/aesv8-armx.S ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_arm/bsaes-armv7.S - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/armcap.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c @@ -82,11 +81,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -107,7 +101,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -117,6 +110,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_conv.c @@ -143,6 +137,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c @@ -150,28 +145,16 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_arm/armv4-gf2m.S ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_arm/armv4-mont.S + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_arm/chacha-armv4.S ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -197,9 +180,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -240,10 +221,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -253,7 +236,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -289,7 +271,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -323,6 +304,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c @@ -332,6 +314,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_arm/ecp_nistz256-armv4.S + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -359,6 +342,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -366,9 +350,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -388,21 +370,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -424,10 +400,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -458,19 +432,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_arm/armv4cpuid.S - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_arm/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -495,7 +461,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -572,24 +537,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -616,11 +572,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_arm/keccak1600-armv4.S ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_arm/sha1-armv4-large.S @@ -632,10 +583,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -651,17 +602,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -706,16 +650,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -724,12 +663,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -763,6 +699,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -800,9 +737,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -812,18 +746,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -833,25 +755,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -859,6 +773,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -866,14 +782,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -892,24 +804,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -918,9 +825,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -952,67 +858,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1020,6 +880,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1034,6 +895,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c diff --git a/openssl/cmake/linux_arm64.cmake b/openssl/cmake/linux_arm64.cmake index 824c9e470..2735778ab 100644 --- a/openssl/cmake/linux_arm64.cmake +++ b/openssl/cmake/linux_arm64.cmake @@ -8,9 +8,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_wrap.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_arm64/aesv8-armx.S - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_arm64/bsaes-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_arm64/vpaes-armv8.S - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/armcap.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c @@ -83,11 +81,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -108,7 +101,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -118,6 +110,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_conv.c @@ -144,35 +137,23 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_arm64/armv8-mont.S + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_arm64/chacha-armv8-sve.S ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_arm64/chacha-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -198,9 +179,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -241,10 +220,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -254,7 +235,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -290,7 +270,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -324,18 +303,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_oct.c - ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_sm2p256.c - ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_sm2p256_table.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_smpl.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_backend.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_arm64/ecp_nistz256-armv8.S - ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_arm64/ecp_sm2p256-armv8.S + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -363,6 +341,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -370,9 +349,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -392,21 +369,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -428,10 +399,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -462,26 +431,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_arm64/arm64cpuid.S - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_arm64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c ${PROJECT_SOURCE_DIR}/src/crypto/kdf/kdf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/lhash/lh_stats.c ${PROJECT_SOURCE_DIR}/src/crypto/lhash/lhash.c - ${PROJECT_SOURCE_DIR}/src/crypto/md5/gen/linux_arm64/md5-aarch64.S ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_dgst.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_one.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_sha1.c @@ -493,7 +453,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/ctr128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cts128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/gcm128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8-unroll8_64.S ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8_64.S ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_arm64/ghashv8-armx.S ${PROJECT_SOURCE_DIR}/src/crypto/modes/ocb128.c @@ -501,7 +460,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -578,24 +536,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -622,11 +571,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_arm64/keccak1600-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_arm64/sha1-armv8.S @@ -638,17 +582,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/gen/linux_arm64/sm3-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm4/gen/linux_arm64/sm4-armv8.S - ${PROJECT_SOURCE_DIR}/src/crypto/sm4/gen/linux_arm64/vpsm4-armv8.S - ${PROJECT_SOURCE_DIR}/src/crypto/sm4/gen/linux_arm64/vpsm4_ex-armv8.S ${PROJECT_SOURCE_DIR}/src/crypto/sm4/sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/sparse_array.c ${PROJECT_SOURCE_DIR}/src/crypto/srp/srp_lib.c @@ -661,17 +603,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -716,16 +651,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -734,12 +664,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -773,6 +700,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -810,9 +738,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -822,18 +747,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -843,25 +756,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -869,6 +774,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -876,14 +783,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -902,24 +805,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -928,9 +826,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -942,7 +839,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/rands/drbg_hash.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/rands/drbg_hmac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/rands/seed_src.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/rands/seeding/rand_cpu_arm64.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/rands/seeding/rand_cpu_x86.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/rands/seeding/rand_tsc.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/rands/seeding/rand_unix.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/rands/seeding/rand_win.c @@ -962,67 +859,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1030,6 +881,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1044,6 +896,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1053,4 +906,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DBSAES_ASM -DECP_NISTZ256_ASM -DECP_SM2P256_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_MONT -DOPENSSL_CPUID_OBJ -DOPENSSL_SM3_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSM4_ASM -DSTATIC_LEGACY -DVPAES_ASM -DVPSM4_ASM) +add_definitions(-DECP_NISTZ256_ASM -DKECCAK1600_ASM -DOPENSSL_BN_ASM_MONT -DOPENSSL_CPUID_OBJ -DOPENSSL_SM3_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSM4_ASM -DSTATIC_LEGACY -DVPAES_ASM) diff --git a/openssl/cmake/linux_ia32.cmake b/openssl/cmake/linux_ia32.cmake index a60a440dc..1e9fb9574 100644 --- a/openssl/cmake/linux_ia32.cmake +++ b/openssl/cmake/linux_ia32.cmake @@ -5,10 +5,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_misc.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_wrap.c - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ia32/aes-586.S - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ia32/aesni-x86.S - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ia32/vpaes-x86.S - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c + ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ia32/aes-586.s + ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ia32/aesni-x86.s + ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ia32/vpaes-x86.s ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -80,11 +79,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/gen/linux_ia32/bf-586.S ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -105,7 +99,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -140,35 +133,26 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ia32/bn-586.S - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ia32/co-586.S - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ia32/x86-gf2m.S - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ia32/x86-mont.S + ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ia32/bn-586.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ia32/co-586.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ia32/x86-gf2m.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ia32/x86-mont.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/gen/linux_ia32/cmll-x86.S - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_ia32/chacha-x86.S + ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_ia32/chacha-x86.s ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -194,9 +178,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -239,8 +221,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c - ${PROJECT_SOURCE_DIR}/src/crypto/des/gen/linux_ia32/crypt586.S - ${PROJECT_SOURCE_DIR}/src/crypto/des/gen/linux_ia32/des-586.S + ${PROJECT_SOURCE_DIR}/src/crypto/des/gen/linux_ia32/crypt586.s + ${PROJECT_SOURCE_DIR}/src/crypto/des/gen/linux_ia32/des-586.s ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -250,7 +232,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -286,7 +267,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -320,6 +300,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c @@ -328,7 +309,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_backend.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_ia32/ecp_nistz256-x86.S + ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_ia32/ecp_nistz256-x86.s + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -356,6 +338,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -363,9 +346,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -385,21 +366,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -421,10 +396,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -454,27 +427,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_ia32/params_idx.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_ia32/x86cpuid.S + ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_ia32/x86cpuid.s ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c ${PROJECT_SOURCE_DIR}/src/crypto/kdf/kdf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/lhash/lh_stats.c ${PROJECT_SOURCE_DIR}/src/crypto/lhash/lhash.c - ${PROJECT_SOURCE_DIR}/src/crypto/md5/gen/linux_ia32/md5-586.S + ${PROJECT_SOURCE_DIR}/src/crypto/md5/gen/linux_ia32/md5-586.s ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_dgst.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_one.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_sha1.c @@ -486,13 +451,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/ctr128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cts128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/gcm128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_ia32/ghash-x86.S + ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_ia32/ghash-x86.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/ocb128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/ofb128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -555,7 +519,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/pkcs7/pk7_mime.c ${PROJECT_SOURCE_DIR}/src/crypto/pkcs7/pk7_smime.c ${PROJECT_SOURCE_DIR}/src/crypto/pkcs7/pkcs7err.c - ${PROJECT_SOURCE_DIR}/src/crypto/poly1305/gen/linux_ia32/poly1305-x86.S + ${PROJECT_SOURCE_DIR}/src/crypto/poly1305/gen/linux_ia32/poly1305-x86.s ${PROJECT_SOURCE_DIR}/src/crypto/poly1305/poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/property/defn_cache.c ${PROJECT_SOURCE_DIR}/src/crypto/property/property.c @@ -569,24 +533,14 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/linux_ia32/rc4-586.S - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/gen/linux_ia32/rmd-586.S - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c + ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/linux_ia32/rc4-586.s ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -613,15 +567,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c - ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_ia32/sha1-586.S - ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_ia32/sha256-586.S - ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_ia32/sha512-586.S + ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_ia32/sha1-586.s + ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_ia32/sha256-586.s + ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_ia32/sha512-586.s ${PROJECT_SOURCE_DIR}/src/crypto/sha/keccak1600.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha1_one.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha1dgst.c @@ -629,10 +578,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -648,17 +597,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -703,16 +645,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -721,12 +658,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -760,6 +694,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -797,9 +732,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -809,18 +741,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -830,25 +750,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -856,6 +768,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -863,14 +777,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -889,24 +799,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -915,9 +820,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -949,67 +853,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1017,6 +875,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1031,6 +890,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1040,4 +900,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DAES_ASM -DCMLL_ASM -DDES_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_PART_WORDS -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DRC4_ASM -DRMD160_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM) +add_definitions(-DAES_ASM -DDES_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_PART_WORDS -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM) diff --git a/openssl/cmake/linux_loong64.cmake b/openssl/cmake/linux_loong64.cmake index d8f79dd26..d387846ab 100644 --- a/openssl/cmake/linux_loong64.cmake +++ b/openssl/cmake/linux_loong64.cmake @@ -7,8 +7,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_misc.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_wrap.c - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_loong64/vpaes-loongarch64.S - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -80,11 +78,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -105,7 +98,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -115,6 +107,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_conv.c @@ -141,33 +134,22 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_loong64/chacha-loongarch64.S + ${PROJECT_SOURCE_DIR}/src/crypto/chacha/chacha_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -193,9 +175,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -236,10 +216,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -249,7 +231,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -285,7 +266,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -319,6 +299,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_oct.c @@ -326,6 +307,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_backend.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -353,6 +335,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -360,9 +343,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -382,21 +363,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -418,10 +393,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -451,32 +424,22 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_loong64/loongarch64cpuid.s - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_loong64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c ${PROJECT_SOURCE_DIR}/src/crypto/kdf/kdf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/lhash/lh_stats.c ${PROJECT_SOURCE_DIR}/src/crypto/lhash/lhash.c - ${PROJECT_SOURCE_DIR}/src/crypto/loongarchcap.c - ${PROJECT_SOURCE_DIR}/src/crypto/md5/gen/linux_loong64/md5-loongarch64.S ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_dgst.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_one.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/mem.c + ${PROJECT_SOURCE_DIR}/src/crypto/mem_clr.c ${PROJECT_SOURCE_DIR}/src/crypto/mem_sec.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cbc128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/ccm128.c @@ -489,7 +452,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -565,24 +527,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -609,11 +562,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/keccak1600.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha1_one.c @@ -622,10 +570,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -641,17 +589,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -696,16 +637,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -714,12 +650,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -753,6 +686,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -790,9 +724,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -802,18 +733,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -823,25 +742,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -849,6 +760,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -856,14 +769,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -882,24 +791,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -908,9 +812,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -942,67 +845,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1010,6 +867,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1024,6 +882,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1033,4 +892,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DMD5_ASM -DOPENSSL_CPUID_OBJ -DSTATIC_LEGACY -DVPAES_ASM) +add_definitions(-DSTATIC_LEGACY) diff --git a/openssl/cmake/linux_mips64.cmake b/openssl/cmake/linux_mips64.cmake index c2dfbf363..012abaa42 100644 --- a/openssl/cmake/linux_mips64.cmake +++ b/openssl/cmake/linux_mips64.cmake @@ -7,7 +7,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_wrap.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_mips64/aes-mips.S - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -79,11 +78,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -104,7 +98,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -139,6 +132,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c @@ -146,28 +140,16 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_mips64/bn-mips.S ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_mips64/mips-mont.S + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/chacha_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -193,9 +175,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -236,10 +216,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -249,7 +231,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -285,7 +266,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -319,6 +299,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_oct.c @@ -326,6 +307,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_backend.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -353,6 +335,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -360,9 +343,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -382,21 +363,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -418,10 +393,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -451,19 +424,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_mips64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -487,7 +452,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -564,24 +528,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -608,11 +563,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_mips64/sha1-mips.S ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_mips64/sha256-mips.S @@ -624,10 +574,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -643,17 +593,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -698,16 +641,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -716,12 +654,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -755,6 +690,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -792,9 +728,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -804,18 +737,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -825,25 +746,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -851,6 +764,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -858,14 +773,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -884,24 +795,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -910,9 +816,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -944,67 +849,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1012,6 +871,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1026,6 +886,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c diff --git a/openssl/cmake/linux_ppc64.cmake b/openssl/cmake/linux_ppc64.cmake index 99365e764..760ec8485 100644 --- a/openssl/cmake/linux_ppc64.cmake +++ b/openssl/cmake/linux_ppc64.cmake @@ -10,7 +10,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ppc64/aes-ppc.s ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ppc64/aesp8-ppc.s ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_ppc64/vpaes-ppc.s - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -82,11 +81,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -107,7 +101,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -137,12 +130,14 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_ppc.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_ppc.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_prime.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_print.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c @@ -151,30 +146,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ppc64/bn-ppc.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ppc64/ppc-mont.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_ppc64/ppc64-mont-fixed.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/chacha_ppc.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_ppc64/chacha-ppc.s - ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_ppc64/chachap10-ppc.s ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -200,9 +182,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -243,10 +223,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -256,7 +238,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -292,7 +273,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -326,6 +306,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c @@ -337,6 +318,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_ppc64/ecp_nistz256-ppc64.s ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_ppc64/x25519-ppc64.s + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -364,6 +346,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -371,9 +354,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -393,21 +374,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -429,10 +404,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -462,20 +435,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_ppc64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_ppc64/ppccpuid.s ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -493,14 +458,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/ctr128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cts128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/gcm128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_ppc64/aes-gcm-ppc.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_ppc64/ghashp8-ppc.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/ocb128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/ofb128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -580,24 +543,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -624,11 +578,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_ppc64/keccak1600-ppc64.s ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_ppc64/sha1-ppc.s @@ -643,10 +592,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha_ppc.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -662,17 +611,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -717,16 +659,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -735,12 +672,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -774,6 +708,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -811,9 +746,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -823,18 +755,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -844,25 +764,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -870,6 +782,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -877,14 +791,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -903,24 +813,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -929,9 +834,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -963,67 +867,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1031,6 +889,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1045,6 +904,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c diff --git a/openssl/cmake/linux_riscv64.cmake b/openssl/cmake/linux_riscv64.cmake index a75b6c643..2fc560840 100644 --- a/openssl/cmake/linux_riscv64.cmake +++ b/openssl/cmake/linux_riscv64.cmake @@ -1,17 +1,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_cbc.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_cfb.c + ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_core.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ecb.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ige.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_misc.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_wrap.c - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zkn.s - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvbb-zvkg-zvkned.s - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvkb-zvkned.s - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvkned.s - ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_riscv64/aes-riscv64.s - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -83,11 +78,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -108,7 +98,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -118,6 +107,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_conv.c @@ -144,35 +134,22 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/chacha_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/chacha/chacha_riscv.c - ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_riscv64/chacha-riscv64-zbb-zvkb.s ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -198,9 +175,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -241,10 +216,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -254,7 +231,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -290,7 +266,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -324,6 +299,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_oct.c @@ -331,6 +307,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_backend.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -358,6 +335,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -365,9 +343,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -387,21 +363,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -423,10 +393,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -456,20 +424,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_riscv64/params_idx.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_riscv64/riscv64cpuid.s ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -480,6 +439,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_one.c ${PROJECT_SOURCE_DIR}/src/crypto/md5/md5_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/mem.c + ${PROJECT_SOURCE_DIR}/src/crypto/mem_clr.c ${PROJECT_SOURCE_DIR}/src/crypto/mem_sec.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cbc128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/ccm128.c @@ -487,16 +447,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/ctr128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cts128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/gcm128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_riscv64/aes-gcm-riscv64-zvkb-zvkg-zvkned.s - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_riscv64/ghash-riscv64-zvkb-zvbc.s - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_riscv64/ghash-riscv64-zvkg.s - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_riscv64/ghash-riscv64.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/ocb128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/ofb128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -572,25 +527,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c - ${PROJECT_SOURCE_DIR}/src/crypto/riscvcap.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -617,32 +562,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c - ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_riscv64/sha256-riscv64-zvkb-zvknha_or_zvknhb.S - ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_riscv64/sha512-riscv64-zvkb-zvknhb.S ${PROJECT_SOURCE_DIR}/src/crypto/sha/keccak1600.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha1_one.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha1dgst.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha256.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c - ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha_riscv.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c - ${PROJECT_SOURCE_DIR}/src/crypto/sm3/gen/linux_riscv64/sm3-riscv64-zvksh.S ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c - ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3_riscv.c - ${PROJECT_SOURCE_DIR}/src/crypto/sm4/gen/linux_riscv64/sm4-riscv64-zvksed.s ${PROJECT_SOURCE_DIR}/src/crypto/sm4/sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/sparse_array.c ${PROJECT_SOURCE_DIR}/src/crypto/srp/srp_lib.c @@ -655,17 +589,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -710,16 +637,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -728,12 +650,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -767,6 +686,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -804,9 +724,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -816,18 +733,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -837,25 +742,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -863,6 +760,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -870,14 +769,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -896,24 +791,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -922,9 +812,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -956,67 +845,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1024,6 +867,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1038,6 +882,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1047,4 +892,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DAES_ASM -DGHASH_ASM -DINCLUDE_C_CHACHA20 -DINCLUDE_C_SHA256 -DINCLUDE_C_SHA512 -DOPENSSL_CPUID_OBJ -DOPENSSL_SM3_ASM -DSHA256_ASM -DSHA512_ASM -DSM4_ASM -DSTATIC_LEGACY) +add_definitions(-DSTATIC_LEGACY) diff --git a/openssl/cmake/linux_x64.cmake b/openssl/cmake/linux_x64.cmake index 9a7b63e66..ad33ed24d 100644 --- a/openssl/cmake/linux_x64.cmake +++ b/openssl/cmake/linux_x64.cmake @@ -12,7 +12,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_x64/aesni-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_x64/bsaes-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/linux_x64/vpaes-x86_64.s - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -84,11 +83,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -109,7 +103,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -118,6 +111,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_sock.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/asm/x86_64-gcc.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/asm/x86_64-gcc.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c @@ -145,42 +139,32 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/rsaz-2k-avx512.s - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/rsaz-3k-avx512.s - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/rsaz-4k-avx512.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/rsaz-avx2.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/rsaz-avx512.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/rsaz-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/x86_64-gf2m.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/x86_64-mont.s ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/linux_x64/x86_64-mont5.s + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp_x2.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp_x2.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/gen/linux_x64/cmll-x86_64.s - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/linux_x64/chacha-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -206,9 +190,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -249,10 +231,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -262,7 +246,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -298,7 +281,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -332,6 +314,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c @@ -342,6 +325,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_x64/ecp_nistz256-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/linux_x64/x25519-x86_64.s + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -369,6 +353,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -376,9 +361,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -398,21 +381,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -434,10 +411,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -467,20 +442,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_x64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/gen/linux_x64/x86_64cpuid.s ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -499,7 +466,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/ctr128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cts128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/gcm128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_x64/aes-gcm-avx512.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_x64/aesni-gcm-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/linux_x64/ghash-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/modes/ocb128.c @@ -507,7 +473,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -584,24 +549,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/linux_x64/rc4-md5-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/linux_x64/rc4-x86_64.s - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -628,11 +584,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_x64/keccak1600-x86_64.s ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/linux_x64/sha1-mb-x86_64.s @@ -646,10 +597,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -665,17 +616,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -720,16 +664,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -738,12 +677,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -777,6 +713,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -814,9 +751,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -826,18 +760,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -847,25 +769,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -873,6 +787,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -880,14 +796,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -906,24 +818,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -932,9 +839,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -966,67 +872,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1034,6 +894,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1048,6 +909,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1057,4 +919,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DAES_ASM -DBSAES_ASM -DCMLL_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DRC4_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM -DX25519_ASM) +add_definitions(-DAES_ASM -DBSAES_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM -DX25519_ASM) diff --git a/openssl/cmake/windows_arm64.cmake b/openssl/cmake/windows_arm64.cmake index cbba42563..a832afdb2 100644 --- a/openssl/cmake/windows_arm64.cmake +++ b/openssl/cmake/windows_arm64.cmake @@ -7,7 +7,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_misc.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/aes/aes_wrap.c - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -79,11 +78,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -104,7 +98,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -114,6 +107,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_conv.c @@ -140,33 +134,22 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/chacha_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -192,9 +175,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -235,10 +216,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -248,7 +231,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -284,7 +266,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -318,6 +299,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_oct.c @@ -325,6 +307,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_backend.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -352,6 +335,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -359,9 +343,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -381,21 +363,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -417,10 +393,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -450,19 +424,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/windows_arm64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -486,7 +452,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -562,24 +527,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/rc4_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -606,11 +562,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/keccak1600.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha1_one.c @@ -619,10 +570,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -638,17 +589,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -693,16 +637,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -711,12 +650,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -750,6 +686,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -787,9 +724,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -799,18 +733,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -820,25 +742,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -846,6 +760,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -853,14 +769,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -879,24 +791,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -905,9 +812,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -932,7 +838,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/signature/sm2_sig.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/file_store.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/file_store_any2obj.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/winstore_store.c ${PROJECT_SOURCE_DIR}/src/providers/legacyprov.c ${PROJECT_SOURCE_DIR}/src/providers/nullprov.c ${PROJECT_SOURCE_DIR}/src/providers/prov_running.c @@ -940,67 +845,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1008,6 +867,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1022,6 +882,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c diff --git a/openssl/cmake/windows_ia32.cmake b/openssl/cmake/windows_ia32.cmake index a9568e269..dd4802ceb 100644 --- a/openssl/cmake/windows_ia32.cmake +++ b/openssl/cmake/windows_ia32.cmake @@ -8,7 +8,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/windows_ia32/aes-586.asm ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/windows_ia32/aesni-x86.asm ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/windows_ia32/vpaes-x86.asm - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -80,11 +79,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/gen/windows_ia32/bf-586.asm ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -105,7 +99,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -140,6 +133,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c @@ -149,26 +143,16 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_ia32/co-586.asm ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_ia32/x86-gf2m.asm ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_ia32/x86-mont.asm + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/gen/windows_ia32/cmll-x86.asm - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/windows_ia32/chacha-x86.asm ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -194,9 +178,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -250,7 +232,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -286,7 +267,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -320,6 +300,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c @@ -329,6 +310,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_key.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/windows_ia32/ecp_nistz256-x86.asm + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -356,6 +338,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -363,9 +346,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -385,21 +366,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -421,10 +396,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -454,20 +427,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/windows_ia32/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/gen/windows_ia32/x86cpuid.asm ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -492,7 +457,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -569,24 +533,14 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/windows_ia32/rc4-586.asm - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/gen/windows_ia32/rmd-586.asm - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -613,11 +567,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/windows_ia32/sha1-586.asm ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/windows_ia32/sha256-586.asm @@ -629,10 +578,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -648,17 +597,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -703,16 +645,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -721,12 +658,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -760,6 +694,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -797,9 +732,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -809,18 +741,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -830,25 +750,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -856,6 +768,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -863,14 +777,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -889,24 +799,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -915,9 +820,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -942,7 +846,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/signature/sm2_sig.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/file_store.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/file_store_any2obj.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/winstore_store.c ${PROJECT_SOURCE_DIR}/src/providers/legacyprov.c ${PROJECT_SOURCE_DIR}/src/providers/nullprov.c ${PROJECT_SOURCE_DIR}/src/providers/prov_running.c @@ -950,67 +853,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1018,6 +875,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1032,6 +890,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1041,4 +900,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DAES_ASM -DCMLL_ASM -DDES_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_PART_WORDS -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DRC4_ASM -DRMD160_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM) +add_definitions(-DAES_ASM -DDES_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_PART_WORDS -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM) diff --git a/openssl/cmake/windows_x64.cmake b/openssl/cmake/windows_x64.cmake index 215776e03..43f868993 100644 --- a/openssl/cmake/windows_x64.cmake +++ b/openssl/cmake/windows_x64.cmake @@ -12,7 +12,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/windows_x64/aesni-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/windows_x64/bsaes-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/aes/gen/windows_x64/vpaes-x86_64.asm - ${PROJECT_SOURCE_DIR}/src/crypto/aria/aria.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_bitstr.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_d2i_fp.c ${PROJECT_SOURCE_DIR}/src/crypto/asn1/a_digest.c @@ -84,11 +83,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/async/async.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_err.c ${PROJECT_SOURCE_DIR}/src/crypto/async/async_wait.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/bf/bf_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_buff.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_lbuf.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bf_nbio.c @@ -109,7 +103,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_conn.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_core.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram.c - ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_dgram_pair.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_fd.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_file.c ${PROJECT_SOURCE_DIR}/src/crypto/bio/bss_log.c @@ -119,6 +112,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bio/ossl_core_bio.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_add.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_asm.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_blind.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_const.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_conv.c @@ -145,42 +139,32 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_recp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_rsa_fips186_4.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_shift.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sm2.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqr.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_sqrt.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_srp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_word.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/bn_x931p.c - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/rsaz-2k-avx512.asm - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/rsaz-3k-avx512.asm - ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/rsaz-4k-avx512.asm ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/rsaz-avx2.asm + ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/rsaz-avx512.asm ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/rsaz-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/x86_64-gf2m.asm ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/x86_64-mont.asm ${PROJECT_SOURCE_DIR}/src/crypto/bn/gen/windows_x64/x86_64-mont5.asm + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsa_sup_mul.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp.c ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp_x2.c + ${PROJECT_SOURCE_DIR}/src/crypto/bn/rsaz_exp_x2.c ${PROJECT_SOURCE_DIR}/src/crypto/bsearch.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buf_err.c ${PROJECT_SOURCE_DIR}/src/crypto/buffer/buffer.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ctr.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_misc.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/cmll_ofb.c - ${PROJECT_SOURCE_DIR}/src/crypto/camellia/gen/windows_x64/cmll-x86_64.asm - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/cast/c_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/chacha/gen/windows_x64/chacha-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/cmac/cmac.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_asn.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_client.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_ctx.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_err.c - ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_genm.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_hdr.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_http.c ${PROJECT_SOURCE_DIR}/src/crypto/cmp/cmp_msg.c @@ -206,9 +190,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_rsa.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_sd.c ${PROJECT_SOURCE_DIR}/src/crypto/cms/cms_smime.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_brotli.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zlib.c - ${PROJECT_SOURCE_DIR}/src/crypto/comp/c_zstd.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_err.c ${PROJECT_SOURCE_DIR}/src/crypto/comp/comp_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/conf/conf_api.c @@ -249,10 +231,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/cfb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/des_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb3_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ecb_enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt.c ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c + ${PROJECT_SOURCE_DIR}/src/crypto/des/fcrypt_b.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64ede.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb64enc.c ${PROJECT_SOURCE_DIR}/src/crypto/des/ofb_enc.c @@ -262,7 +246,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/des/set_key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/str2key.c ${PROJECT_SOURCE_DIR}/src/crypto/des/xcbc_enc.c - ${PROJECT_SOURCE_DIR}/src/crypto/deterministic_nonce.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/dh/dh_backend.c @@ -298,7 +281,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_err.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_openssl.c - ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_vms.c ${PROJECT_SOURCE_DIR}/src/crypto/dso/dso_win32.c ${PROJECT_SOURCE_DIR}/src/crypto/ebcdic.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/curve25519.c @@ -332,6 +314,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecdsa_vrf.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/eck_prn.c + ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_mont.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nist.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecp_nistz256.c @@ -342,6 +325,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ec/ecx_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/windows_x64/ecp_nistz256-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/ec/gen/windows_x64/x25519-x86_64.asm + ${PROJECT_SOURCE_DIR}/src/crypto/eia3/eia3.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_err.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/encode_decode/decoder_meth.c @@ -369,6 +353,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_digest.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_dsa.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_eckey.c + ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_ecpmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_pkmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rand.c ${PROJECT_SOURCE_DIR}/src/crypto/engine/tb_rsa.c @@ -376,9 +361,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_all_legacy.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_blocks.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_mark.c ${PROJECT_SOURCE_DIR}/src/crypto/err/err_prn.c - ${PROJECT_SOURCE_DIR}/src/crypto/err/err_save.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_err.c ${PROJECT_SOURCE_DIR}/src/crypto/ess/ess_lib.c @@ -398,21 +381,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha1.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aes_cbc_hmac_sha256.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_aria.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_bf.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_camellia.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_cast.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_chacha20_poly1305.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_des3.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_idea.c + ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_eea3.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_old.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_rc5.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_sm4.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/e_xcbc_d.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/ec_ctrl.c @@ -434,10 +411,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/evp/kem.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/keymgmt_meth.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_blake2.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_md5_sha1.c - ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_ripemd.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/legacy_sha.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_null.c ${PROJECT_SOURCE_DIR}/src/crypto/evp/m_sigver.c @@ -467,20 +442,12 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_generate.c ${PROJECT_SOURCE_DIR}/src/crypto/ffc/ffc_params_validate.c - ${PROJECT_SOURCE_DIR}/src/crypto/gen/windows_x64/params_idx.c ${PROJECT_SOURCE_DIR}/src/crypto/gen/windows_x64/x86_64cpuid.asm ${PROJECT_SOURCE_DIR}/src/crypto/getenv.c ${PROJECT_SOURCE_DIR}/src/crypto/hmac/hmac.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke.c - ${PROJECT_SOURCE_DIR}/src/crypto/hpke/hpke_util.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_client.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_err.c ${PROJECT_SOURCE_DIR}/src/crypto/http/http_lib.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_ofb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/idea/i_skey.c ${PROJECT_SOURCE_DIR}/src/crypto/info.c ${PROJECT_SOURCE_DIR}/src/crypto/init.c ${PROJECT_SOURCE_DIR}/src/crypto/initthread.c @@ -499,7 +466,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/ctr128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/cts128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/gcm128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/windows_x64/aes-gcm-avx512.asm ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/windows_x64/aesni-gcm-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/modes/gen/windows_x64/ghash-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/modes/ocb128.c @@ -507,7 +473,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/modes/siv128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/wrap128.c ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128.c - ${PROJECT_SOURCE_DIR}/src/crypto/modes/xts128gb.c ${PROJECT_SOURCE_DIR}/src/crypto/o_dir.c ${PROJECT_SOURCE_DIR}/src/crypto/o_fopen.c ${PROJECT_SOURCE_DIR}/src/crypto/o_init.c @@ -584,24 +549,15 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/provider_core.c ${PROJECT_SOURCE_DIR}/src/crypto/provider_predefined.c ${PROJECT_SOURCE_DIR}/src/crypto/punycode.c - ${PROJECT_SOURCE_DIR}/src/crypto/quic_vlint.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/prov_seed.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_deprecated.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_err.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_meth.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_pool.c - ${PROJECT_SOURCE_DIR}/src/crypto/rand/rand_uniform.c ${PROJECT_SOURCE_DIR}/src/crypto/rand/randfile.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2_skey.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2cfb64.c - ${PROJECT_SOURCE_DIR}/src/crypto/rc2/rc2ofb64.c ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/windows_x64/rc4-md5-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/rc4/gen/windows_x64/rc4-x86_64.asm - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_dgst.c - ${PROJECT_SOURCE_DIR}/src/crypto/ripemd/rmd_one.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_ameth.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_backend.c @@ -628,11 +584,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_sp800_56b_gen.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931.c ${PROJECT_SOURCE_DIR}/src/crypto/rsa/rsa_x931g.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cbc.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_cfb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ecb.c - ${PROJECT_SOURCE_DIR}/src/crypto/seed/seed_ofb.c ${PROJECT_SOURCE_DIR}/src/crypto/self_test_core.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/windows_x64/keccak1600-x86_64.asm ${PROJECT_SOURCE_DIR}/src/crypto/sha/gen/windows_x64/sha1-mb-x86_64.asm @@ -646,10 +597,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha3.c ${PROJECT_SOURCE_DIR}/src/crypto/sha/sha512.c ${PROJECT_SOURCE_DIR}/src/crypto/siphash/siphash.c - ${PROJECT_SOURCE_DIR}/src/crypto/sleep.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_crypt.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_err.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_key.c + ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_kmeth.c ${PROJECT_SOURCE_DIR}/src/crypto/sm2/sm2_sign.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/legacy_sm3.c ${PROJECT_SOURCE_DIR}/src/crypto/sm3/sm3.c @@ -665,17 +616,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/store/store_register.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_result.c ${PROJECT_SOURCE_DIR}/src/crypto/store/store_strings.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/api.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_none.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_posix.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/arch/thread_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/thread/internal.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_none.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_pthread.c ${PROJECT_SOURCE_DIR}/src/crypto/threads_win.c - ${PROJECT_SOURCE_DIR}/src/crypto/time.c ${PROJECT_SOURCE_DIR}/src/crypto/trace.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_asn1.c ${PROJECT_SOURCE_DIR}/src/crypto/ts/ts_conf.c @@ -720,16 +664,11 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_enum.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_extku.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_genn.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_group_ac.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ia5.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ind_iss.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_info.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_int.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ist.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_lib.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_ncons.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_ass.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_no_rev_avail.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pci.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcia.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_pcons.c @@ -738,12 +677,9 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_prn.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_purp.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_san.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_single_use.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_skid.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_soa_id.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_sxnet.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_tlsf.c - ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utf8.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3_utl.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/v3err.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x509_att.c @@ -777,6 +713,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_req.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509.c ${PROJECT_SOURCE_DIR}/src/crypto/x509/x_x509a.c + ${PROJECT_SOURCE_DIR}/src/crypto/zuc/zuc.c ${PROJECT_SOURCE_DIR}/src/providers/baseprov.c ${PROJECT_SOURCE_DIR}/src/providers/common/bio_prov.c ${PROJECT_SOURCE_DIR}/src/providers/common/capabilities.c @@ -814,9 +751,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -826,18 +760,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aes_xts_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_aria_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_blowfish_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_camellia_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_cast5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -847,25 +769,17 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_des_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_desx_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_idea_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_null.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc2_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hmac_md5_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_rc4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_seed_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_common.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_default.c @@ -873,6 +787,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_tdes_wrap_hw.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_block.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -880,14 +796,10 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_gcm_hw.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/ciphers/ciphercommon_hw.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2b_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/blake2s_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/digestcommon.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/md5_sha1_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/null_prov.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/ripemd_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha2_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sha3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/digests/sm3_prov.c @@ -906,24 +818,19 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecdh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/ecx_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/kdf_exch.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/argon2.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/exchange/sm2dh_exch.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hkdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/hmacdrbg_kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/kbkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/krb5kdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf1.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pbkdf2_fips.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pkcs12kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/pvkkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/scrypt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sshkdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/sskdf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/tls1_prf.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kdfs/x942kdf.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ec_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/ecx_kem.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/kem_util.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/kem/rsa_kem.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dh_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -932,9 +839,8 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/keymgmt/rsa_kmgmt.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2b_mac.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/blake2s_mac.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/cmac_prov.c + ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/eia3_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/gmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/hmac_prov.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/macs/kmac_prov.c @@ -959,7 +865,6 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/providers/implementations/signature/sm2_sig.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/file_store.c ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/file_store_any2obj.c - ${PROJECT_SOURCE_DIR}/src/providers/implementations/storemgmt/winstore_store.c ${PROJECT_SOURCE_DIR}/src/providers/legacyprov.c ${PROJECT_SOURCE_DIR}/src/providers/nullprov.c ${PROJECT_SOURCE_DIR}/src/providers/prov_running.c @@ -967,67 +872,21 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/d1_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/d1_srtp.c - ${PROJECT_SOURCE_DIR}/src/ssl/event_queue.c ${PROJECT_SOURCE_DIR}/src/ssl/methods.c ${PROJECT_SOURCE_DIR}/src/ssl/pqueue.c - ${PROJECT_SOURCE_DIR}/src/ssl/priority_queue.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/cc_newreno.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/json_enc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/qlog_event_helpers.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_ackm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_cfq.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_channel.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_demux.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_engine.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fc.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_fifd.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_impl.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_lcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_method.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_port.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rcidm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_reactor.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_rx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_shared.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_tx.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_record_util.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_rx_depack.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sf_list.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srt_gen.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_srtm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_sstream.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_statm.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_stream_map.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_thread_assist.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tls.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_trace.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_tserver.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txp.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_txpim.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_types.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/quic_wire_pkt.c - ${PROJECT_SOURCE_DIR}/src/ssl/quic/uint_set.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/dtls_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_cbc.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/ssl3_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls13_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls1_meth.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_common.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_multib.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tls_pad.c - ${PROJECT_SOURCE_DIR}/src/ssl/record/methods/tlsany_meth.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/dtls1_bitmap.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_d1.c ${PROJECT_SOURCE_DIR}/src/ssl/record/rec_layer_s3.c - ${PROJECT_SOURCE_DIR}/src/ssl/rio/poll_immediate.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_buffer.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/ssl3_record_tls13.c + ${PROJECT_SOURCE_DIR}/src/ssl/record/tls_pad.c + ${PROJECT_SOURCE_DIR}/src/ssl/s3_cbc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/s3_msg.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_asn1.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert.c - ${PROJECT_SOURCE_DIR}/src/ssl/ssl_cert_comp.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_ciph.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_conf.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_err.c @@ -1035,6 +894,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/ssl_init.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_lib.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_mcnf.c + ${PROJECT_SOURCE_DIR}/src/ssl/ssl_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_rsa_legacy.c ${PROJECT_SOURCE_DIR}/src/ssl/ssl_sess.c @@ -1049,6 +909,7 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_clnt.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_dtls.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_lib.c + ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_quic.c ${PROJECT_SOURCE_DIR}/src/ssl/statem/statem_srvr.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_enc.c ${PROJECT_SOURCE_DIR}/src/ssl/t1_lib.c @@ -1058,4 +919,4 @@ set(src_list ${PROJECT_SOURCE_DIR}/src/ssl/tls_srp.c ) -add_definitions(-DAES_ASM -DBSAES_ASM -DCMLL_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DRC4_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM -DX25519_ASM) +add_definitions(-DAES_ASM -DBSAES_ASM -DECP_NISTZ256_ASM -DGHASH_ASM -DKECCAK1600_ASM -DMD5_ASM -DOPENSSL_BN_ASM_GF2m -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_CPUID_OBJ -DOPENSSL_IA32_SSE2 -DPADLOCK_ASM -DPOLY1305_ASM -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DSTATIC_LEGACY -DVPAES_ASM -DX25519_ASM) diff --git a/openssl/include/crypto/__DECC_INCLUDE_EPILOGUE.H b/openssl/include/crypto/__DECC_INCLUDE_EPILOGUE.H deleted file mode 100644 index e57c0eab3..000000000 --- a/openssl/include/crypto/__DECC_INCLUDE_EPILOGUE.H +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is only used by HP C on VMS, and is included automatically - * after each header file from this directory - */ - -/* restore state. Must correspond to the save in __decc_include_prologue.h */ -#pragma names restore diff --git a/openssl/include/crypto/__DECC_INCLUDE_PROLOGUE.H b/openssl/include/crypto/__DECC_INCLUDE_PROLOGUE.H deleted file mode 100644 index a01395755..000000000 --- a/openssl/include/crypto/__DECC_INCLUDE_PROLOGUE.H +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is only used by HP C on VMS, and is included automatically - * after each header file from this directory - */ - -/* save state */ -#pragma names save -/* have the compiler shorten symbols larger than 31 chars to 23 chars - * followed by a 8 hex char CRC - */ -#pragma names as_is,shortened diff --git a/openssl/include/crypto/aes_platform.h b/openssl/include/crypto/aes_platform.h index bb0c10ec8..c7af60972 100644 --- a/openssl/include/crypto/aes_platform.h +++ b/openssl/include/crypto/aes_platform.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -60,37 +60,47 @@ void AES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len, # endif /* AES_XTS_ASM */ # if defined(OPENSSL_CPUID_OBJ) -# if (defined(__powerpc__) || defined(__POWERPC__) || defined(_ARCH_PPC)) +# if (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC)) # include "crypto/ppc_arch.h" # ifdef VPAES_ASM # define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC) # endif -# if !defined(OPENSSL_SYS_AIX) && !defined(OPENSSL_SYS_MACOSX) -# define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207) -# define HWAES_set_encrypt_key aes_p8_set_encrypt_key -# define HWAES_set_decrypt_key aes_p8_set_decrypt_key -# define HWAES_encrypt aes_p8_encrypt -# define HWAES_decrypt aes_p8_decrypt -# define HWAES_cbc_encrypt aes_p8_cbc_encrypt -# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks -# define HWAES_xts_encrypt aes_p8_xts_encrypt -# define HWAES_xts_decrypt aes_p8_xts_decrypt -# define PPC_AES_GCM_CAPABLE (OPENSSL_ppccap_P & PPC_MADD300) -# define AES_GCM_ENC_BYTES 128 -# define AES_GCM_DEC_BYTES 128 -size_t ppc_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const void *key, unsigned char ivec[16], - u64 *Xi); -size_t ppc_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, - size_t len, const void *key, unsigned char ivec[16], - u64 *Xi); -# define AES_GCM_ASM_PPC(gctx) ((gctx)->ctr==aes_p8_ctr32_encrypt_blocks && \ - (gctx)->gcm.funcs.ghash==gcm_ghash_p8) -void gcm_ghash_p8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len); -# endif /* OPENSSL_SYS_AIX || OPENSSL_SYS_MACOSX */ +# define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207) +# ifdef HWAES_set_encrypt_key +# undef HWAES_set_encrypt_key +# endif +# define HWAES_set_encrypt_key aes_p8_set_encrypt_key +# ifdef HWAES_set_decrypt_key +# undef HWAES_set_decrypt_key +# endif +# define HWAES_set_decrypt_key aes_p8_set_decrypt_key +# ifdef HWAES_encrypt +# undef HWAES_encrypt +# endif +# define HWAES_encrypt aes_p8_encrypt +# ifdef HWAES_decrypt +# undef HWAES_decrypt +# endif +# define HWAES_decrypt aes_p8_decrypt +# ifdef HWAES_cbc_encrypt +# undef HWAES_cbc_encrypt +# endif +# define HWAES_cbc_encrypt aes_p8_cbc_encrypt +# ifdef HWAES_ctr32_encrypt_blocks +# undef HWAES_ctr32_encrypt_blocks +# endif +# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks +# ifdef HWAES_xts_encrypt +# undef HWAES_xts_encrypt +# endif +# define HWAES_xts_encrypt aes_p8_xts_encrypt +# ifdef HWAES_xts_decrypt +# undef HWAES_xts_decrypt +# endif +# define HWAES_xts_decrypt aes_p8_xts_decrypt # endif /* PPC */ -# if (defined(__arm__) || defined(__arm) || defined(__aarch64__) || defined(_M_ARM64)) +# if (defined(__arm__) || defined(__arm) || defined(__aarch64__)) # include "arm_arch.h" # if __ARM_MAX_ARCH__>=7 # if defined(BSAES_ASM) @@ -100,54 +110,69 @@ void gcm_ghash_p8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len); # define VPAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON) # endif # define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES) +# ifdef HWAES_set_encrypt_key +# undef HWAES_set_encrypt_key +# endif # define HWAES_set_encrypt_key aes_v8_set_encrypt_key +# ifdef HWAES_set_decrypt_key +# undef HWAES_set_decrypt_key +# endif # define HWAES_set_decrypt_key aes_v8_set_decrypt_key +# ifdef HWAES_encrypt +# undef HWAES_encrypt +# endif # define HWAES_encrypt aes_v8_encrypt +# ifdef HWAES_decrypt +# undef HWAES_decrypt +# endif # define HWAES_decrypt aes_v8_decrypt +# ifdef HWAES_cbc_encrypt +# undef HWAES_cbc_encrypt +# endif # define HWAES_cbc_encrypt aes_v8_cbc_encrypt +# ifdef HWAES_ecb_encrypt +# undef HWAES_ecb_encrypt +# endif # define HWAES_ecb_encrypt aes_v8_ecb_encrypt -# if __ARM_MAX_ARCH__>=8 && (defined(__aarch64__) || defined(_M_ARM64)) -# define ARMv8_HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES) +# if __ARM_MAX_ARCH__>=8 && defined(__aarch64__) +# ifdef HWAES_xts_encrypt +# undef HWAES_xts_encrypt +# endif # define HWAES_xts_encrypt aes_v8_xts_encrypt +# ifdef HWAES_xts_decrypt +# undef HWAES_xts_decrypt +# endif # define HWAES_xts_decrypt aes_v8_xts_decrypt # endif +# ifdef HWAES_ctr32_encrypt_blocks +# undef HWAES_ctr32_encrypt_blocks +# endif # define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks -# define HWAES_ctr32_encrypt_blocks_unroll12_eor3 aes_v8_ctr32_encrypt_blocks_unroll12_eor3 # define AES_PMULL_CAPABLE ((OPENSSL_armcap_P & ARMV8_PMULL) && (OPENSSL_armcap_P & ARMV8_AES)) -# define AES_UNROLL12_EOR3_CAPABLE (OPENSSL_armcap_P & ARMV8_UNROLL12_EOR3) # define AES_GCM_ENC_BYTES 512 # define AES_GCM_DEC_BYTES 512 -# if __ARM_MAX_ARCH__>=8 && (defined(__aarch64__) || defined(_M_ARM64)) +# if __ARM_MAX_ARCH__>=8 && defined(__aarch64__) +# ifdef AES_gcm_encrypt +# undef AES_gcm_encrypt +# endif # define AES_gcm_encrypt armv8_aes_gcm_encrypt +# ifdef AES_gcm_decrypt +# undef AES_gcm_decrypt +# endif # define AES_gcm_decrypt armv8_aes_gcm_decrypt -# define AES_GCM_ASM(gctx) (((gctx)->ctr==aes_v8_ctr32_encrypt_blocks_unroll12_eor3 || \ - (gctx)->ctr==aes_v8_ctr32_encrypt_blocks) && \ - (gctx)->gcm.funcs.ghash==gcm_ghash_v8) -/* The [unroll8_eor3_]aes_gcm_(enc|dec)_(128|192|256)_kernel() functions - * take input length in BITS and return number of BYTES processed */ -size_t aes_gcm_enc_128_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext, - uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t aes_gcm_enc_192_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext, +# define AES_GCM_ASM(gctx) ((gctx)->ctr==aes_v8_ctr32_encrypt_blocks && \ + (gctx)->gcm.ghash==gcm_ghash_v8) +size_t aes_gcm_enc_128_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext, uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t aes_gcm_enc_256_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext, +size_t aes_gcm_enc_192_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext, uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t aes_gcm_dec_128_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext, +size_t aes_gcm_enc_256_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext, uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t aes_gcm_dec_192_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext, +size_t aes_gcm_dec_128_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext, uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t aes_gcm_dec_256_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext, +size_t aes_gcm_dec_192_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext, uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t unroll8_eor3_aes_gcm_enc_128_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext, - uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t unroll8_eor3_aes_gcm_enc_192_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext, - uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t unroll8_eor3_aes_gcm_enc_256_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext, - uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t unroll8_eor3_aes_gcm_dec_128_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext, - uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t unroll8_eor3_aes_gcm_dec_192_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext, - uint64_t *Xi, unsigned char ivec[16], const void *key); -size_t unroll8_eor3_aes_gcm_dec_256_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext, +size_t aes_gcm_dec_256_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext, uint64_t *Xi, unsigned char ivec[16], const void *key); size_t armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, const void *key, unsigned char ivec[16], u64 *Xi); @@ -166,13 +191,6 @@ void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len); # define AESNI_CBC_HMAC_SHA_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32))) # endif -# if defined(__loongarch__) || defined(__loongarch64) -# include "loongarch_arch.h" -# if defined(VPAES_ASM) -# define VPAES_CAPABLE (OPENSSL_loongarch_hwcap_P & LOONGARCH_HWCAP_LSX) -# endif -# endif - # if defined(AES_ASM) && !defined(I386_ONLY) && ( \ ((defined(__i386) || defined(__i386__) || \ defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \ @@ -262,84 +280,19 @@ size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len const void *key, unsigned char ivec[16], u64 *Xi); void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in, size_t len); +# ifdef AES_gcm_encrypt +# undef AES_gcm_encrypt +# endif # define AES_gcm_encrypt aesni_gcm_encrypt +# ifdef AES_gcm_decrypt +# undef AES_gcm_decrypt +# endif # define AES_gcm_decrypt aesni_gcm_decrypt # define AES_GCM_ASM(ctx) (ctx->ctr == aesni_ctr32_encrypt_blocks && \ - ctx->gcm.funcs.ghash == gcm_ghash_avx) + ctx->gcm.ghash == gcm_ghash_avx) # endif -# elif defined(AES_ASM) && (defined(__sparc) || defined(__sparc__)) - -/* Fujitsu SPARC64 X support */ -# include "crypto/sparc_arch.h" - -# define SPARC_AES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_AES) -# define HWAES_CAPABLE (OPENSSL_sparcv9cap_P[0] & SPARCV9_FJAESX) -# define HWAES_set_encrypt_key aes_fx_set_encrypt_key -# define HWAES_set_decrypt_key aes_fx_set_decrypt_key -# define HWAES_encrypt aes_fx_encrypt -# define HWAES_decrypt aes_fx_decrypt -# define HWAES_cbc_encrypt aes_fx_cbc_encrypt -# define HWAES_ctr32_encrypt_blocks aes_fx_ctr32_encrypt_blocks - -void aes_t4_set_encrypt_key(const unsigned char *key, int bits, AES_KEY *ks); -void aes_t4_set_decrypt_key(const unsigned char *key, int bits, AES_KEY *ks); -void aes_t4_encrypt(const unsigned char *in, unsigned char *out, - const AES_KEY *key); -void aes_t4_decrypt(const unsigned char *in, unsigned char *out, - const AES_KEY *key); -/* - * Key-length specific subroutines were chosen for following reason. - * Each SPARC T4 core can execute up to 8 threads which share core's - * resources. Loading as much key material to registers allows to - * minimize references to shared memory interface, as well as amount - * of instructions in inner loops [much needed on T4]. But then having - * non-key-length specific routines would require conditional branches - * either in inner loops or on subroutines' entries. Former is hardly - * acceptable, while latter means code size increase to size occupied - * by multiple key-length specific subroutines, so why fight? - */ -void aes128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const AES_KEY *key, - unsigned char *ivec, int /*unused*/); -void aes128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, - size_t len, const AES_KEY *key, - unsigned char *ivec, int /*unused*/); -void aes192_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const AES_KEY *key, - unsigned char *ivec, int /*unused*/); -void aes192_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, - size_t len, const AES_KEY *key, - unsigned char *ivec, int /*unused*/); -void aes256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const AES_KEY *key, - unsigned char *ivec, int /*unused*/); -void aes256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, - size_t len, const AES_KEY *key, - unsigned char *ivec, int /*unused*/); -void aes128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const AES_KEY *key, - unsigned char *ivec); -void aes192_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const AES_KEY *key, - unsigned char *ivec); -void aes256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const AES_KEY *key, - unsigned char *ivec); -void aes128_t4_xts_encrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const AES_KEY *key1, - const AES_KEY *key2, const unsigned char *ivec); -void aes128_t4_xts_decrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const AES_KEY *key1, - const AES_KEY *key2, const unsigned char *ivec); -void aes256_t4_xts_encrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const AES_KEY *key1, - const AES_KEY *key2, const unsigned char *ivec); -void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const AES_KEY *key1, - const AES_KEY *key2, const unsigned char *ivec); - # elif defined(OPENSSL_CPUID_OBJ) && defined(__s390__) /* IBM S390X support */ # include "s390x_arch.h" @@ -435,100 +388,6 @@ void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out, /* Convert key size to function code: [16,24,32] -> [18,19,20]. */ # define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6)) -# elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 -/* RISC-V 64 support */ -# include "riscv_arch.h" - -/* Zkne and Zknd extensions (scalar crypto AES). */ -int rv64i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); -int rv64i_zknd_set_decrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); -void rv64i_zkne_encrypt(const unsigned char *in, unsigned char *out, - const AES_KEY *key); -void rv64i_zknd_decrypt(const unsigned char *in, unsigned char *out, - const AES_KEY *key); -/* Zvkned extension (vector crypto AES). */ -int rv64i_zvkned_set_encrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); -int rv64i_zvkned_set_decrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); -void rv64i_zvkned_encrypt(const unsigned char *in, unsigned char *out, - const AES_KEY *key); -void rv64i_zvkned_decrypt(const unsigned char *in, unsigned char *out, - const AES_KEY *key); - -void rv64i_zvkned_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const AES_KEY *key, - unsigned char *ivec, const int enc); - -void rv64i_zvkned_cbc_decrypt(const unsigned char *in, unsigned char *out, - size_t length, const AES_KEY *key, - unsigned char *ivec, const int enc); - -void rv64i_zvkned_ecb_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const AES_KEY *key, - const int enc); - -void rv64i_zvkned_ecb_decrypt(const unsigned char *in, unsigned char *out, - size_t length, const AES_KEY *key, - const int enc); - -void rv64i_zvkb_zvkned_ctr32_encrypt_blocks(const unsigned char *in, - unsigned char *out, size_t blocks, - const void *key, - const unsigned char ivec[16]); - -size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in, - unsigned char *out, size_t len, - const void *key, - unsigned char ivec[16], u64 *Xi); - -size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in, - unsigned char *out, size_t len, - const void *key, - unsigned char ivec[16], u64 *Xi); - -void rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt(const unsigned char *in, - unsigned char *out, size_t length, - const AES_KEY *key1, - const AES_KEY *key2, - const unsigned char iv[16]); - -void rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt(const unsigned char *in, - unsigned char *out, size_t length, - const AES_KEY *key1, - const AES_KEY *key2, - const unsigned char iv[16]); - -void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16], const u8 *inp, - size_t len); - -#define AES_GCM_ENC_BYTES 64 -#define AES_GCM_DEC_BYTES 64 -#define AES_gcm_encrypt rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt -#define AES_gcm_decrypt rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt -#define AES_GCM_ASM(ctx) \ - (ctx->ctr == rv64i_zvkb_zvkned_ctr32_encrypt_blocks && \ - ctx->gcm.funcs.ghash == gcm_ghash_rv64i_zvkg) - -# elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 -/* RISC-V 32 support */ -# include "riscv_arch.h" - -int rv32i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); -/* set_decrypt_key needs both zknd and zkne */ -int rv32i_zknd_zkne_set_decrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); -int rv32i_zbkb_zkne_set_encrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); -int rv32i_zbkb_zknd_zkne_set_decrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); -void rv32i_zkne_encrypt(const unsigned char *in, unsigned char *out, - const AES_KEY *key); -void rv32i_zknd_decrypt(const unsigned char *in, unsigned char *out, - const AES_KEY *key); # endif # if defined(HWAES_CAPABLE) @@ -549,11 +408,6 @@ void HWAES_ecb_encrypt(const unsigned char *in, unsigned char *out, void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, size_t len, const void *key, const unsigned char ivec[16]); -# if defined(AES_UNROLL12_EOR3_CAPABLE) -void HWAES_ctr32_encrypt_blocks_unroll12_eor3(const unsigned char *in, unsigned char *out, - size_t len, const void *key, - const unsigned char ivec[16]); -# endif void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out, size_t len, const AES_KEY *key1, const AES_KEY *key2, const unsigned char iv[16]); @@ -561,7 +415,7 @@ void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len, const AES_KEY *key1, const AES_KEY *key2, const unsigned char iv[16]); # ifndef OPENSSL_NO_OCB -# ifdef HWAES_ocb_encrypt +# ifdef ASM_HWAES_ocb_encrypt void HWAES_ocb_encrypt(const unsigned char *in, unsigned char *out, size_t blocks, const void *key, size_t start_block_num, @@ -569,9 +423,12 @@ void HWAES_ocb_encrypt(const unsigned char *in, unsigned char *out, const unsigned char L_[][16], unsigned char checksum[16]); # else +# ifdef HWAES_ocb_encrypt +# undef HWAES_ocb_encrypt +# endif # define HWAES_ocb_encrypt ((ocb128_f)NULL) # endif -# ifdef HWAES_ocb_decrypt +# ifdef ASM_HWAES_ocb_decrypt void HWAES_ocb_decrypt(const unsigned char *in, unsigned char *out, size_t blocks, const void *key, size_t start_block_num, @@ -579,6 +436,9 @@ void HWAES_ocb_decrypt(const unsigned char *in, unsigned char *out, const unsigned char L_[][16], unsigned char checksum[16]); # else +# ifdef HWAES_ocb_decrypt +# undef HWAES_ocb_decrypt +# endif # define HWAES_ocb_decrypt ((ocb128_f)NULL) # endif # endif /* OPENSSL_NO_OCB */ diff --git a/openssl/include/crypto/aria.h b/openssl/include/crypto/aria.h deleted file mode 100644 index 21a9932e8..000000000 --- a/openssl/include/crypto/aria.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - /* Copyright (c) 2017 National Security Research Institute. All rights reserved. */ - -#ifndef OSSL_CRYPTO_ARIA_H -# define OSSL_CRYPTO_ARIA_H -# pragma once - -# include - -# ifdef OPENSSL_NO_ARIA -# error ARIA is disabled. -# endif - -# define ARIA_ENCRYPT 1 -# define ARIA_DECRYPT 0 - -# define ARIA_BLOCK_SIZE 16 /* Size of each encryption/decryption block */ -# define ARIA_MAX_KEYS 17 /* Number of keys needed in the worst case */ - -typedef union { - unsigned char c[ARIA_BLOCK_SIZE]; - unsigned int u[ARIA_BLOCK_SIZE / sizeof(unsigned int)]; -} ARIA_u128; - -typedef unsigned char ARIA_c128[ARIA_BLOCK_SIZE]; - -struct aria_key_st { - ARIA_u128 rd_key[ARIA_MAX_KEYS]; - unsigned int rounds; -}; -typedef struct aria_key_st ARIA_KEY; - - -int ossl_aria_set_encrypt_key(const unsigned char *userKey, const int bits, - ARIA_KEY *key); -int ossl_aria_set_decrypt_key(const unsigned char *userKey, const int bits, - ARIA_KEY *key); - -void ossl_aria_encrypt(const unsigned char *in, unsigned char *out, - const ARIA_KEY *key); - -#endif diff --git a/openssl/include/crypto/asn1.h b/openssl/include/crypto/asn1.h index 36af1d768..4bc037f9c 100644 --- a/openssl/include/crypto/asn1.h +++ b/openssl/include/crypto/asn1.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -102,6 +102,7 @@ extern const EVP_PKEY_ASN1_METHOD ossl_sm2_asn1_meth; extern const EVP_PKEY_ASN1_METHOD ossl_rsa_asn1_meths[2]; extern const EVP_PKEY_ASN1_METHOD ossl_rsa_pss_asn1_meth; +extern const EVP_PKEY_ASN1_METHOD ossl_eia3_asn1_meth; /* * These are used internally in the ASN1_OBJECT to keep track of whether the @@ -142,12 +143,8 @@ X509_ALGOR *ossl_x509_algor_mgf1_decode(X509_ALGOR *alg); int ossl_x509_algor_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md); int ossl_asn1_time_print_ex(BIO *bp, const ASN1_TIME *tm, unsigned long flags); -EVP_PKEY *ossl_d2i_PrivateKey_legacy(int keytype, EVP_PKEY **a, - const unsigned char **pp, long length, - OSSL_LIB_CTX *libctx, const char *propq); -X509_ALGOR *ossl_X509_ALGOR_from_nid(int nid, int ptype, void *pval); - -time_t ossl_asn1_string_to_time_t(const char *asn1_string); -void ossl_asn1_string_set_bits_left(ASN1_STRING *str, unsigned int num); +EVP_PKEY * ossl_d2i_PrivateKey_legacy(int keytype, EVP_PKEY **a, + const unsigned char **pp, long length, + OSSL_LIB_CTX *libctx, const char *propq); #endif /* ndef OSSL_CRYPTO_ASN1_H */ diff --git a/openssl/include/crypto/bioerr.h b/openssl/include/crypto/bioerr.h index e38b981ab..a0c06099f 100644 --- a/openssl/include/crypto/bioerr.h +++ b/openssl/include/crypto/bioerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/bn.h b/openssl/include/crypto/bn.h index 9a988a467..cd4565421 100644 --- a/openssl/include/crypto/bn.h +++ b/openssl/include/crypto/bn.h @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -87,14 +87,6 @@ int bn_lshift_fixed_top(BIGNUM *r, const BIGNUM *a, int n); int bn_rshift_fixed_top(BIGNUM *r, const BIGNUM *a, int n); int bn_div_fixed_top(BIGNUM *dv, BIGNUM *rem, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx); -int ossl_bn_mask_bits_fixed_top(BIGNUM *a, int n); -int ossl_bn_is_word_fixed_top(const BIGNUM *a, BN_ULONG w); -int ossl_bn_priv_rand_range_fixed_top(BIGNUM *r, const BIGNUM *range, - unsigned int strength, BN_CTX *ctx); -int ossl_bn_gen_dsa_nonce_fixed_top(BIGNUM *out, const BIGNUM *range, - const BIGNUM *priv, - const unsigned char *message, - size_t message_len, BN_CTX *ctx); #define BN_PRIMETEST_COMPOSITE 0 #define BN_PRIMETEST_COMPOSITE_WITH_FACTOR 1 @@ -103,8 +95,6 @@ int ossl_bn_gen_dsa_nonce_fixed_top(BIGNUM *out, const BIGNUM *range, int ossl_bn_miller_rabin_is_prime(const BIGNUM *w, int iterations, BN_CTX *ctx, BN_GENCB *cb, int enhanced, int *status); -int ossl_bn_check_generated_prime(const BIGNUM *w, int checks, BN_CTX *ctx, - BN_GENCB *cb); const BIGNUM *ossl_bn_get0_small_factors(void); @@ -124,14 +114,10 @@ OSSL_LIB_CTX *ossl_bn_get_libctx(BN_CTX *ctx); extern const BIGNUM ossl_bn_inv_sqrt_2; -#if defined(OPENSSL_SYS_LINUX) && !defined(FIPS_MODULE) && defined (__s390x__) \ - && !defined (OPENSSL_NO_ASM) -# define S390X_MOD_EXP -#endif - -int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, - const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx); -int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q, - const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp); +int ossl_bn_rsa_do_unblind(const BIGNUM *intermediate, + const BN_BLINDING *blinding, + const BIGNUM *possible_arg2, + const BIGNUM *to_mod, BN_CTX *ctx, + unsigned char *buf, int num); #endif diff --git a/openssl/include/crypto/bn_conf.h b/openssl/include/crypto/bn_conf.h index 45259a0cb..d57ee3353 100644 --- a/openssl/include/crypto/bn_conf.h +++ b/openssl/include/crypto/bn_conf.h @@ -22,13 +22,8 @@ /* Should we define BN_DIV2W here? */ /* Only one for the following should be defined */ - -#ifdef _WIN64 -# define SIXTY_FOUR_BIT -#elif defined(__i386__) || defined(_M_IX86) || defined(__arm__) -# define THIRTY_TWO_BIT -#else -# define SIXTY_FOUR_BIT_LONG -#endif +#define SIXTY_FOUR_BIT_LONG +#undef SIXTY_FOUR_BIT +#undef THIRTY_TWO_BIT #endif diff --git a/openssl/include/crypto/chacha.h b/openssl/include/crypto/chacha.h index 2af2c2ecf..f526d3730 100644 --- a/openssl/include/crypto/chacha.h +++ b/openssl/include/crypto/chacha.h @@ -12,6 +12,7 @@ # pragma once #include +#include /* * ChaCha20_ctr32 encrypts |len| bytes from |inp| with the given key and @@ -27,12 +28,6 @@ void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len, const unsigned int key[8], const unsigned int counter[4]); -#ifdef INCLUDE_C_CHACHA20 -/* The fallback implementation for `ChaCha20_ctr32`. */ -void ChaCha20_ctr32_c(unsigned char *out, const unsigned char *inp, size_t len, - const unsigned int key[8], const unsigned int counter[4]); -#endif - /* * You can notice that there is no key setup procedure. Because it's * as trivial as collecting bytes into 32-bit elements, it's reckoned diff --git a/openssl/include/crypto/cmll_platform.h b/openssl/include/crypto/cmll_platform.h index 8cadadf68..f81661cca 100644 --- a/openssl/include/crypto/cmll_platform.h +++ b/openssl/include/crypto/cmll_platform.h @@ -11,41 +11,4 @@ # define OSSL_CMLL_PLATFORM_H # pragma once -# if defined(CMLL_ASM) && (defined(__sparc) || defined(__sparc__)) - -/* Fujitsu SPARC64 X support */ -# include "crypto/sparc_arch.h" - -# ifndef OPENSSL_NO_CAMELLIA -# define SPARC_CMLL_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_CAMELLIA) -# include - -void cmll_t4_set_key(const unsigned char *key, int bits, CAMELLIA_KEY *ks); -void cmll_t4_encrypt(const unsigned char *in, unsigned char *out, - const CAMELLIA_KEY *key); -void cmll_t4_decrypt(const unsigned char *in, unsigned char *out, - const CAMELLIA_KEY *key); - -void cmll128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const CAMELLIA_KEY *key, - unsigned char *ivec, int /*unused*/); -void cmll128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, - size_t len, const CAMELLIA_KEY *key, - unsigned char *ivec, int /*unused*/); -void cmll256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const CAMELLIA_KEY *key, - unsigned char *ivec, int /*unused*/); -void cmll256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out, - size_t len, const CAMELLIA_KEY *key, - unsigned char *ivec, int /*unused*/); -void cmll128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const CAMELLIA_KEY *key, - unsigned char *ivec); -void cmll256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const CAMELLIA_KEY *key, - unsigned char *ivec); -# endif /* OPENSSL_NO_CAMELLIA */ - -# endif /* CMLL_ASM && sparc */ - #endif /* OSSL_CRYPTO_CIPHERMODE_PLATFORM_H */ diff --git a/openssl/include/crypto/cmperr.h b/openssl/include/crypto/cmperr.h index 2bd163738..ac61dd487 100644 --- a/openssl/include/crypto/cmperr.h +++ b/openssl/include/crypto/cmperr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/cmserr.h b/openssl/include/crypto/cmserr.h index a7fcf11fa..1de2f9c7d 100644 --- a/openssl/include/crypto/cmserr.h +++ b/openssl/include/crypto/cmserr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/conferr.h b/openssl/include/crypto/conferr.h index fc9645127..cb367e4f3 100644 --- a/openssl/include/crypto/conferr.h +++ b/openssl/include/crypto/conferr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/context.h b/openssl/include/crypto/context.h deleted file mode 100644 index 7369a730f..000000000 --- a/openssl/include/crypto/context.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include - -void *ossl_provider_store_new(OSSL_LIB_CTX *); -void *ossl_property_string_data_new(OSSL_LIB_CTX *); -void *ossl_stored_namemap_new(OSSL_LIB_CTX *); -void *ossl_property_defns_new(OSSL_LIB_CTX *); -void *ossl_ctx_global_properties_new(OSSL_LIB_CTX *); -void *ossl_rand_ctx_new(OSSL_LIB_CTX *); -void *ossl_prov_conf_ctx_new(OSSL_LIB_CTX *); -void *ossl_bio_core_globals_new(OSSL_LIB_CTX *); -void *ossl_child_prov_ctx_new(OSSL_LIB_CTX *); -void *ossl_prov_drbg_nonce_ctx_new(OSSL_LIB_CTX *); -void *ossl_self_test_set_callback_new(OSSL_LIB_CTX *); -void *ossl_rand_crng_ctx_new(OSSL_LIB_CTX *); -int ossl_thread_register_fips(OSSL_LIB_CTX *); -void *ossl_thread_event_ctx_new(OSSL_LIB_CTX *); -void *ossl_fips_prov_ossl_ctx_new(OSSL_LIB_CTX *); -#if defined(OPENSSL_THREADS) -void *ossl_threads_ctx_new(OSSL_LIB_CTX *); -#endif - -void ossl_provider_store_free(void *); -void ossl_property_string_data_free(void *); -void ossl_stored_namemap_free(void *); -void ossl_property_defns_free(void *); -void ossl_ctx_global_properties_free(void *); -void ossl_rand_ctx_free(void *); -void ossl_prov_conf_ctx_free(void *); -void ossl_bio_core_globals_free(void *); -void ossl_child_prov_ctx_free(void *); -void ossl_prov_drbg_nonce_ctx_free(void *); -void ossl_self_test_set_callback_free(void *); -void ossl_rand_crng_ctx_free(void *); -void ossl_thread_event_ctx_free(void *); -void ossl_fips_prov_ossl_ctx_free(void *); -void ossl_release_default_drbg_ctx(void); -#if defined(OPENSSL_THREADS) -void ossl_threads_ctx_free(void *); -#endif diff --git a/openssl/include/crypto/cryptoerr.h b/openssl/include/crypto/cryptoerr.h index 1b6192e3f..288b87ac8 100644 --- a/openssl/include/crypto/cryptoerr.h +++ b/openssl/include/crypto/cryptoerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/ctype.h b/openssl/include/crypto/ctype.h index 22f692218..9d14314c7 100644 --- a/openssl/include/crypto/ctype.h +++ b/openssl/include/crypto/ctype.h @@ -49,13 +49,8 @@ */ # define CTYPE_MASK_ascii (~0) -# ifdef CHARSET_EBCDIC int ossl_toascii(int c); int ossl_fromascii(int c); -# else -# define ossl_toascii(c) (c) -# define ossl_fromascii(c) (c) -# endif int ossl_ctype_check(int c, unsigned int mask); int ossl_tolower(int c); diff --git a/openssl/include/crypto/decoder.h b/openssl/include/crypto/decoder.h index a0d5de652..cc06ef292 100644 --- a/openssl/include/crypto/decoder.h +++ b/openssl/include/crypto/decoder.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,6 +13,10 @@ # include +OSSL_DECODER *ossl_decoder_fetch_by_number(OSSL_LIB_CTX *libctx, + int id, + const char *properties); + /* * These are specially made for the 'file:' provider-native loader, which * uses this to install a DER to anything decoder, which doesn't do much @@ -25,16 +29,14 @@ void *ossl_decoder_from_algorithm(int id, const OSSL_ALGORITHM *algodef, OSSL_DECODER_INSTANCE * ossl_decoder_instance_new(OSSL_DECODER *decoder, void *decoderctx); void ossl_decoder_instance_free(OSSL_DECODER_INSTANCE *decoder_inst); -OSSL_DECODER_INSTANCE *ossl_decoder_instance_dup(const OSSL_DECODER_INSTANCE *src); int ossl_decoder_ctx_add_decoder_inst(OSSL_DECODER_CTX *ctx, OSSL_DECODER_INSTANCE *di); -int ossl_decoder_get_number(const OSSL_DECODER *encoder); -int ossl_decoder_store_cache_flush(OSSL_LIB_CTX *libctx); -int ossl_decoder_store_remove_all_provided(const OSSL_PROVIDER *prov); +int ossl_decoder_ctx_setup_for_pkey(OSSL_DECODER_CTX *ctx, + EVP_PKEY **pkey, const char *keytype, + OSSL_LIB_CTX *libctx, + const char *propquery); -void *ossl_decoder_cache_new(OSSL_LIB_CTX *ctx); -void ossl_decoder_cache_free(void *vcache); -int ossl_decoder_cache_flush(OSSL_LIB_CTX *libctx); +int ossl_decoder_get_number(const OSSL_DECODER *encoder); #endif diff --git a/openssl/include/crypto/des_platform.h b/openssl/include/crypto/des_platform.h index 07ea1ddd0..c4426d251 100644 --- a/openssl/include/crypto/des_platform.h +++ b/openssl/include/crypto/des_platform.h @@ -11,25 +11,4 @@ # define OSSL_DES_PLATFORM_H # pragma once -# if defined(DES_ASM) && (defined(__sparc) || defined(__sparc__)) - -/* Fujitsu SPARC64 X support */ -# include "crypto/sparc_arch.h" - -# ifndef OPENSSL_NO_DES -# define SPARC_DES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_DES) -# include -void des_t4_key_expand(const void *key, DES_key_schedule *ks); -void des_t4_ede3_cbc_encrypt(const void *inp, void *out, size_t len, - const DES_key_schedule ks[3], unsigned char iv[8]); -void des_t4_ede3_cbc_decrypt(const void *inp, void *out, size_t len, - const DES_key_schedule ks[3], unsigned char iv[8]); -void des_t4_cbc_encrypt(const void *inp, void *out, size_t len, - const DES_key_schedule *ks, unsigned char iv[8]); -void des_t4_cbc_decrypt(const void *inp, void *out, size_t len, - const DES_key_schedule *ks, unsigned char iv[8]); -# endif /* OPENSSL_NO_DES */ - -# endif /* DES_ASM && sparc */ - #endif /* OSSL_CRYPTO_CIPHERMODE_PLATFORM_H */ diff --git a/openssl/include/crypto/dsa.h b/openssl/include/crypto/dsa.h index b08a42c7f..260c30fa4 100644 --- a/openssl/include/crypto/dsa.h +++ b/openssl/include/crypto/dsa.h @@ -15,11 +15,6 @@ # include # include "internal/ffc.h" -/* - * DSA Paramgen types - * Note, adding to this list requires adjustments to various checks - * in dsa_gen range validation checks - */ #define DSA_PARAMGEN_TYPE_FIPS_186_4 0 /* Use FIPS186-4 standard */ #define DSA_PARAMGEN_TYPE_FIPS_186_2 1 /* Use legacy FIPS186-2 standard */ #define DSA_PARAMGEN_TYPE_FIPS_DEFAULT 2 @@ -31,9 +26,7 @@ int ossl_dsa_generate_ffc_parameters(DSA *dsa, int type, int pbits, int qbits, BN_GENCB *cb); int ossl_dsa_sign_int(int type, const unsigned char *dgst, int dlen, - unsigned char *sig, unsigned int *siglen, DSA *dsa, - unsigned int nonce_type, const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq); + unsigned char *sig, unsigned int *siglen, DSA *dsa); FFC_PARAMS *ossl_dsa_get0_params(DSA *dsa); int ossl_dsa_ffc_params_fromdata(DSA *dsa, const OSSL_PARAM params[]); diff --git a/openssl/include/crypto/dsaerr.h b/openssl/include/crypto/dsaerr.h index fde8358fc..9898097d0 100644 --- a/openssl/include/crypto/dsaerr.h +++ b/openssl/include/crypto/dsaerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/dso_conf.h b/openssl/include/crypto/dso_conf.h index 50f4896c5..cb3585b9e 100644 --- a/openssl/include/crypto/dso_conf.h +++ b/openssl/include/crypto/dso_conf.h @@ -13,13 +13,6 @@ # define OSSL_CRYPTO_DSO_CONF_H # pragma once -#ifdef _WIN32 -# define DSO_WIN32 -# define DSO_EXTENSION ".dll" -#else -# define DSO_DLFCN -# define HAVE_DLFCN_H -# define DSO_EXTENSION ".so" -#endif - +# define DSO_NONE +# define DSO_EXTENSION ".dylib" #endif diff --git a/openssl/include/crypto/ec.h b/openssl/include/crypto/ec.h index da85a7bd8..62163b31a 100644 --- a/openssl/include/crypto/ec.h +++ b/openssl/include/crypto/ec.h @@ -95,12 +95,6 @@ char *ossl_ec_pt_format_id2name(int id); char *ossl_ec_check_group_type_id2name(int flags); int ossl_ec_set_check_group_type_from_name(EC_KEY *ec, const char *name); -int ossl_ec_generate_key_dhkem(EC_KEY *eckey, - const unsigned char *ikm, size_t ikmlen); -int ossl_ecdsa_deterministic_sign(const unsigned char *dgst, int dlen, - unsigned char *sig, unsigned int *siglen, - EC_KEY *eckey, unsigned int nonce_type, - const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq); + # endif /* OPENSSL_NO_EC */ #endif diff --git a/openssl/include/crypto/ecerr.h b/openssl/include/crypto/ecerr.h index 782526bf8..4658ae8fb 100644 --- a/openssl/include/crypto/ecerr.h +++ b/openssl/include/crypto/ecerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/ecx.h b/openssl/include/crypto/ecx.h index f35b875fb..48b95fa5b 100644 --- a/openssl/include/crypto/ecx.h +++ b/openssl/include/crypto/ecx.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,7 +15,7 @@ # include -# ifndef OPENSSL_NO_ECX +# ifndef OPENSSL_NO_EC # include # include @@ -72,6 +72,7 @@ struct ecx_key_st { size_t keylen; ECX_KEY_TYPE type; CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; }; size_t ossl_ecx_key_length(ECX_KEY_TYPE type); @@ -82,9 +83,6 @@ unsigned char *ossl_ecx_key_allocate_privkey(ECX_KEY *key); void ossl_ecx_key_free(ECX_KEY *key); int ossl_ecx_key_up_ref(ECX_KEY *key); ECX_KEY *ossl_ecx_key_dup(const ECX_KEY *key, int selection); -int ossl_ecx_compute_key(ECX_KEY *peer, ECX_KEY *priv, size_t keylen, - unsigned char *secret, size_t *secretlen, - size_t outlen); int ossl_x25519(uint8_t out_shared_key[32], const uint8_t private_key[32], const uint8_t peer_public_value[32]); @@ -96,33 +94,27 @@ ossl_ed25519_public_from_private(OSSL_LIB_CTX *ctx, uint8_t out_public_key[32], const uint8_t private_key[32], const char *propq); int -ossl_ed25519_sign(uint8_t *out_sig, const uint8_t *tbs, size_t tbs_len, +ossl_ed25519_sign(uint8_t *out_sig, const uint8_t *message, size_t message_len, const uint8_t public_key[32], const uint8_t private_key[32], - const uint8_t dom2flag, const uint8_t phflag, const uint8_t csflag, - const uint8_t *context, size_t context_len, OSSL_LIB_CTX *libctx, const char *propq); int -ossl_ed25519_verify(const uint8_t *tbs, size_t tbs_len, +ossl_ed25519_verify(const uint8_t *message, size_t message_len, const uint8_t signature[64], const uint8_t public_key[32], - const uint8_t dom2flag, const uint8_t phflag, const uint8_t csflag, - const uint8_t *context, size_t context_len, OSSL_LIB_CTX *libctx, const char *propq); + int ossl_ed448_public_from_private(OSSL_LIB_CTX *ctx, uint8_t out_public_key[57], const uint8_t private_key[57], const char *propq); int -ossl_ed448_sign(OSSL_LIB_CTX *ctx, uint8_t *out_sig, - const uint8_t *message, size_t message_len, - const uint8_t public_key[57], const uint8_t private_key[57], - const uint8_t *context, size_t context_len, - const uint8_t phflag, const char *propq); +ossl_ed448_sign(OSSL_LIB_CTX *ctx, uint8_t *out_sig, const uint8_t *message, + size_t message_len, const uint8_t public_key[57], + const uint8_t private_key[57], const uint8_t *context, + size_t context_len, const char *propq); int -ossl_ed448_verify(OSSL_LIB_CTX *ctx, - const uint8_t *message, size_t message_len, +ossl_ed448_verify(OSSL_LIB_CTX *ctx, const uint8_t *message, size_t message_len, const uint8_t signature[114], const uint8_t public_key[57], - const uint8_t *context, size_t context_len, - const uint8_t phflag, const char *propq); + const uint8_t *context, size_t context_len, const char *propq); int ossl_x448(uint8_t out_shared_key[56], const uint8_t private_key[56], @@ -154,5 +146,5 @@ ECX_KEY *ossl_evp_pkey_get1_X25519(EVP_PKEY *pkey); ECX_KEY *ossl_evp_pkey_get1_X448(EVP_PKEY *pkey); ECX_KEY *ossl_evp_pkey_get1_ED25519(EVP_PKEY *pkey); ECX_KEY *ossl_evp_pkey_get1_ED448(EVP_PKEY *pkey); -# endif /* OPENSSL_NO_ECX */ +# endif /* OPENSSL_NO_EC */ #endif diff --git a/openssl/include/crypto/encoder.h b/openssl/include/crypto/encoder.h index 6240438d6..09d445d21 100644 --- a/openssl/include/crypto/encoder.h +++ b/openssl/include/crypto/encoder.h @@ -7,14 +7,8 @@ * https://www.openssl.org/source/license.html */ -#ifndef OSSL_CRYPTO_ENCODER_H -# define OSSL_CRYPTO_ENCODER_H -# pragma once - -# include +#include +OSSL_ENCODER *ossl_encoder_fetch_by_number(OSSL_LIB_CTX *libctx, int id, + const char *properties); int ossl_encoder_get_number(const OSSL_ENCODER *encoder); -int ossl_encoder_store_cache_flush(OSSL_LIB_CTX *libctx); -int ossl_encoder_store_remove_all_provided(const OSSL_PROVIDER *prov); - -#endif diff --git a/openssl/include/crypto/engineerr.h b/openssl/include/crypto/engineerr.h index 737c841d6..2d4fc5cdf 100644 --- a/openssl/include/crypto/engineerr.h +++ b/openssl/include/crypto/engineerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/evp.h b/openssl/include/crypto/evp.h index 50ad737fc..1267a9ccb 100644 --- a/openssl/include/crypto/evp.h +++ b/openssl/include/crypto/evp.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -16,21 +16,11 @@ # include "internal/refcount.h" # include "crypto/ecx.h" -/* - * Default PKCS5 PBE KDF salt lengths - * In RFC 8018, PBE1 uses 8 bytes (64 bits) for its salt length. - * It also specifies to use at least 8 bytes for PBES2. - * The NIST requirement for PBKDF2 is 128 bits so we use this as the - * default for PBE2 (scrypt and HKDF2) - */ -# define PKCS5_DEFAULT_PBE1_SALT_LEN PKCS5_SALT_LEN -# define PKCS5_DEFAULT_PBE2_SALT_LEN 16 /* * Don't free up md_ctx->pctx in EVP_MD_CTX_reset, use the reserved flag * values in evp.h */ #define EVP_MD_CTX_FLAG_KEEP_PKEY_CTX 0x0400 -#define EVP_MD_CTX_FLAG_FINALISED 0x0800 #define evp_pkey_ctx_is_legacy(ctx) \ ((ctx)->keymgmt == NULL) @@ -213,6 +203,7 @@ struct evp_mac_st { const char *description; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_mac_newctx_fn *newctx; OSSL_FUNC_mac_dupctx_fn *dupctx; @@ -234,6 +225,7 @@ struct evp_kdf_st { char *type_name; const char *description; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_kdf_newctx_fn *newctx; OSSL_FUNC_kdf_dupctx_fn *dupctx; @@ -278,11 +270,11 @@ struct evp_md_st { const char *description; OSSL_PROVIDER *prov; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_digest_newctx_fn *newctx; OSSL_FUNC_digest_init_fn *dinit; OSSL_FUNC_digest_update_fn *dupdate; OSSL_FUNC_digest_final_fn *dfinal; - OSSL_FUNC_digest_squeeze_fn *dsqueeze; OSSL_FUNC_digest_digest_fn *digest; OSSL_FUNC_digest_freectx_fn *freectx; OSSL_FUNC_digest_dupctx_fn *dupctx; @@ -334,6 +326,7 @@ struct evp_cipher_st { const char *description; OSSL_PROVIDER *prov; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_cipher_newctx_fn *newctx; OSSL_FUNC_cipher_encrypt_init_fn *einit; OSSL_FUNC_cipher_decrypt_init_fn *dinit; @@ -372,7 +365,7 @@ static int cname##_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const uns return 1;\ } -#define EVP_MAXCHUNK ((size_t)1 << 30) +#define EVP_MAXCHUNK ((size_t)1<<(sizeof(long)*8-2)) #define BLOCK_CIPHER_func_ofb(cname, cprefix, cbits, kstruct, ksched) \ static int cname##_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl) \ @@ -596,7 +589,6 @@ int evp_cipher_asn1_to_param_ex(EVP_CIPHER_CTX *c, ASN1_TYPE *type, typedef struct { EVP_KEYMGMT *keymgmt; void *keydata; - int selection; } OP_CACHE_ELEM; DEFINE_STACK_OF(OP_CACHE_ELEM) @@ -653,9 +645,7 @@ union legacy_pkey_st { # endif # ifndef OPENSSL_NO_EC struct ec_key_st *ec; /* ECC */ -# ifndef OPENSSL_NO_ECX ECX_KEY *ecx; /* X25519, X448, Ed25519, Ed448 */ -# endif # endif }; @@ -788,14 +778,12 @@ EVP_PKEY *evp_keymgmt_util_make_pkey(EVP_KEYMGMT *keymgmt, void *keydata); int evp_keymgmt_util_export(const EVP_PKEY *pk, int selection, OSSL_CALLBACK *export_cb, void *export_cbarg); -void *evp_keymgmt_util_export_to_provider(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, - int selection); +void *evp_keymgmt_util_export_to_provider(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt); OP_CACHE_ELEM *evp_keymgmt_util_find_operation_cache(EVP_PKEY *pk, - EVP_KEYMGMT *keymgmt, - int selection); -int evp_keymgmt_util_clear_operation_cache(EVP_PKEY *pk); -int evp_keymgmt_util_cache_keydata(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, - void *keydata, int selection); + EVP_KEYMGMT *keymgmt); +int evp_keymgmt_util_clear_operation_cache(EVP_PKEY *pk, int locking); +int evp_keymgmt_util_cache_keydata(EVP_PKEY *pk, + EVP_KEYMGMT *keymgmt, void *keydata); void evp_keymgmt_util_cache_keyinfo(EVP_PKEY *pk); void *evp_keymgmt_util_fromdata(EVP_PKEY *target, EVP_KEYMGMT *keymgmt, int selection, const OSSL_PARAM params[]); @@ -822,7 +810,7 @@ int evp_keymgmt_set_params(const EVP_KEYMGMT *keymgmt, void *evp_keymgmt_gen_init(const EVP_KEYMGMT *keymgmt, int selection, const OSSL_PARAM params[]); int evp_keymgmt_gen_set_template(const EVP_KEYMGMT *keymgmt, void *genctx, - void *templ); + void *template_); int evp_keymgmt_gen_set_params(const EVP_KEYMGMT *keymgmt, void *genctx, const OSSL_PARAM params[]); void *evp_keymgmt_gen(const EVP_KEYMGMT *keymgmt, void *genctx, @@ -850,9 +838,6 @@ const OSSL_PARAM *evp_keymgmt_export_types(const EVP_KEYMGMT *keymgmt, int selection); void *evp_keymgmt_dup(const EVP_KEYMGMT *keymgmt, const void *keydata_from, int selection); -EVP_KEYMGMT *evp_keymgmt_fetch_from_prov(OSSL_PROVIDER *prov, - const char *name, - const char *properties); /* Pulling defines out of C source files */ @@ -901,12 +886,14 @@ EVP_MD_CTX *evp_md_ctx_new_ex(EVP_PKEY *pkey, const ASN1_OCTET_STRING *id, int evp_pkey_name2type(const char *name); const char *evp_pkey_type2name(int type); +int evp_pkey_ctx_set1_id_prov(EVP_PKEY_CTX *ctx, const void *id, int len); +int evp_pkey_ctx_get1_id_prov(EVP_PKEY_CTX *ctx, void *id); +int evp_pkey_ctx_get1_id_len_prov(EVP_PKEY_CTX *ctx, size_t *id_len); + int evp_pkey_ctx_use_cached_data(EVP_PKEY_CTX *ctx); # endif /* !defined(FIPS_MODULE) */ -int evp_method_store_cache_flush(OSSL_LIB_CTX *libctx); -int evp_method_store_remove_all_provided(const OSSL_PROVIDER *prov); - +int evp_method_store_flush(OSSL_LIB_CTX *libctx); int evp_default_properties_enable_fips_int(OSSL_LIB_CTX *libctx, int enable, int loadconfig); int evp_set_default_properties_int(OSSL_LIB_CTX *libctx, const char *propq, @@ -914,8 +901,6 @@ int evp_set_default_properties_int(OSSL_LIB_CTX *libctx, const char *propq, char *evp_get_global_properties_str(OSSL_LIB_CTX *libctx, int loadconfig); void evp_md_ctx_clear_digest(EVP_MD_CTX *ctx, int force, int keep_digest); -/* just free the algctx if set, returns 0 on inconsistent state of ctx */ -int evp_md_ctx_free_algctx(EVP_MD_CTX *ctx); /* Three possible states: */ # define EVP_PKEY_STATE_UNKNOWN 0 @@ -951,22 +936,9 @@ int evp_kdf_get_number(const EVP_KDF *kdf); int evp_kem_get_number(const EVP_KEM *wrap); int evp_keyexch_get_number(const EVP_KEYEXCH *keyexch); int evp_keymgmt_get_number(const EVP_KEYMGMT *keymgmt); -int evp_keymgmt_get_legacy_alg(const EVP_KEYMGMT *keymgmt); int evp_mac_get_number(const EVP_MAC *mac); int evp_md_get_number(const EVP_MD *md); int evp_rand_get_number(const EVP_RAND *rand); -int evp_rand_can_seed(EVP_RAND_CTX *ctx); -size_t evp_rand_get_seed(EVP_RAND_CTX *ctx, - unsigned char **buffer, - int entropy, size_t min_len, size_t max_len, - int prediction_resistance, - const unsigned char *adin, size_t adin_len); -void evp_rand_clear_seed(EVP_RAND_CTX *ctx, - unsigned char *buffer, size_t b_len); int evp_signature_get_number(const EVP_SIGNATURE *signature); -int evp_pkey_decrypt_alloc(EVP_PKEY_CTX *ctx, unsigned char **outp, - size_t *outlenp, size_t expected_outlen, - const unsigned char *in, size_t inlen); - #endif /* OSSL_CRYPTO_EVP_H */ diff --git a/openssl/include/crypto/evperr.h b/openssl/include/crypto/evperr.h index 4a0011079..1aa20554d 100644 --- a/openssl/include/crypto/evperr.h +++ b/openssl/include/crypto/evperr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/httperr.h b/openssl/include/crypto/httperr.h index 827d61a23..969df17b8 100644 --- a/openssl/include/crypto/httperr.h +++ b/openssl/include/crypto/httperr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,10 +19,7 @@ extern "C" { # endif -# ifndef OPENSSL_NO_HTTP - int ossl_err_load_HTTP_strings(void); -# endif # ifdef __cplusplus } diff --git a/openssl/include/crypto/md32_common.h b/openssl/include/crypto/md32_common.h index 46214f323..2c750af36 100644 --- a/openssl/include/crypto/md32_common.h +++ b/openssl/include/crypto/md32_common.h @@ -1,5 +1,5 @@ /* - * Copyright 1999-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2018 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -63,89 +63,63 @@ * #define HASH_BLOCK_DATA_ORDER md5_block_data_order */ -#ifndef OSSL_CRYPTO_MD32_COMMON_H -# define OSSL_CRYPTO_MD32_COMMON_H -# pragma once +#include -# include - -# if !defined(DATA_ORDER_IS_BIG_ENDIAN) && !defined(DATA_ORDER_IS_LITTLE_ENDIAN) -# error "DATA_ORDER must be defined!" -# endif - -# ifndef HASH_CBLOCK -# error "HASH_CBLOCK must be defined!" -# endif -# ifndef HASH_LONG -# error "HASH_LONG must be defined!" -# endif -# ifndef HASH_CTX -# error "HASH_CTX must be defined!" -# endif - -# ifndef HASH_UPDATE -# error "HASH_UPDATE must be defined!" -# endif -# ifndef HASH_TRANSFORM -# error "HASH_TRANSFORM must be defined!" -# endif -# ifndef HASH_FINAL -# error "HASH_FINAL must be defined!" -# endif +#if !defined(DATA_ORDER_IS_BIG_ENDIAN) && !defined(DATA_ORDER_IS_LITTLE_ENDIAN) +# error "DATA_ORDER must be defined!" +#endif -# ifndef HASH_BLOCK_DATA_ORDER -# error "HASH_BLOCK_DATA_ORDER must be defined!" -# endif +#ifndef HASH_CBLOCK +# error "HASH_CBLOCK must be defined!" +#endif +#ifndef HASH_LONG +# error "HASH_LONG must be defined!" +#endif +#ifndef HASH_CTX +# error "HASH_CTX must be defined!" +#endif -# define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n)))) +#ifndef HASH_UPDATE +# error "HASH_UPDATE must be defined!" +#endif +#ifndef HASH_TRANSFORM +# error "HASH_TRANSFORM must be defined!" +#endif +#ifndef HASH_FINAL +# error "HASH_FINAL must be defined!" +#endif -#ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && \ - !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if defined(__riscv_zbb) || defined(__riscv_zbkb) -# if __riscv_xlen == 64 -# undef ROTATE -# define ROTATE(x, n) ({ MD32_REG_T ret; \ - asm ("roriw %0, %1, %2" \ - : "=r"(ret) \ - : "r"(x), "i"(32 - (n))); ret;}) -# endif -# if __riscv_xlen == 32 -# undef ROTATE -# define ROTATE(x, n) ({ MD32_REG_T ret; \ - asm ("rori %0, %1, %2" \ - : "=r"(ret) \ - : "r"(x), "i"(32 - (n))); ret;}) -# endif -# endif -# endif +#ifndef HASH_BLOCK_DATA_ORDER +# error "HASH_BLOCK_DATA_ORDER must be defined!" #endif -# if defined(DATA_ORDER_IS_BIG_ENDIAN) +#define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n)))) -# define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ +#if defined(DATA_ORDER_IS_BIG_ENDIAN) + +# define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ l|=(((unsigned long)(*((c)++)))<<16), \ l|=(((unsigned long)(*((c)++)))<< 8), \ l|=(((unsigned long)(*((c)++))) ) ) -# define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ +# define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ *((c)++)=(unsigned char)(((l)>>16)&0xff), \ *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ *((c)++)=(unsigned char)(((l) )&0xff), \ l) -# elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) +#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) -# define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ +# define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ l|=(((unsigned long)(*((c)++)))<< 8), \ l|=(((unsigned long)(*((c)++)))<<16), \ l|=(((unsigned long)(*((c)++)))<<24) ) -# define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ +# define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ *((c)++)=(unsigned char)(((l)>>16)&0xff), \ *((c)++)=(unsigned char)(((l)>>24)&0xff), \ l) -# endif +#endif /* * Time for some action :-) @@ -230,30 +204,30 @@ int HASH_FINAL(unsigned char *md, HASH_CTX *c) memset(p + n, 0, HASH_CBLOCK - 8 - n); p += HASH_CBLOCK - 8; -# if defined(DATA_ORDER_IS_BIG_ENDIAN) +#if defined(DATA_ORDER_IS_BIG_ENDIAN) (void)HOST_l2c(c->Nh, p); (void)HOST_l2c(c->Nl, p); -# elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) +#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) (void)HOST_l2c(c->Nl, p); (void)HOST_l2c(c->Nh, p); -# endif +#endif p -= HASH_CBLOCK; HASH_BLOCK_DATA_ORDER(c, p, 1); c->num = 0; OPENSSL_cleanse(p, HASH_CBLOCK); -# ifndef HASH_MAKE_STRING -# error "HASH_MAKE_STRING must be defined!" -# else +#ifndef HASH_MAKE_STRING +# error "HASH_MAKE_STRING must be defined!" +#else HASH_MAKE_STRING(c, md); -# endif +#endif return 1; } -# ifndef MD32_REG_T -# if defined(__alpha) || defined(__sparcv9) || defined(__mips) -# define MD32_REG_T long +#ifndef MD32_REG_T +# if defined(__alpha) || defined(__mips) +# define MD32_REG_T long /* * This comment was originally written for MD5, which is why it * discusses A-D. But it basically applies to all 32-bit digests, @@ -270,15 +244,13 @@ int HASH_FINAL(unsigned char *md, HASH_CTX *c) * Well, to be honest it should say that this *prevents* * performance degradation. */ -# else +# else /* * Above is not absolute and there are LP64 compilers that * generate better code if MD32_REG_T is defined int. The above * pre-processor condition reflects the circumstances under which * the conclusion was made and is subject to further extension. */ -# define MD32_REG_T int -# endif +# define MD32_REG_T int # endif - #endif diff --git a/openssl/include/crypto/modes.h b/openssl/include/crypto/modes.h index d03ca83d0..19f9d8595 100644 --- a/openssl/include/crypto/modes.h +++ b/openssl/include/crypto/modes.h @@ -1,5 +1,5 @@ /* - * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2010-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -74,13 +74,6 @@ typedef unsigned char u8; asm ("rev %0,%1" \ : "=r"(ret_) : "r"((u32)(x))); \ ret_; }) -# elif (defined(__riscv_zbb) || defined(__riscv_zbkb)) && __riscv_xlen == 64 -# define BSWAP8(x) ({ u64 ret_=(x); \ - asm ("rev8 %0,%0" \ - : "+r"(ret_)); ret_; }) -# define BSWAP4(x) ({ u32 ret_=(x); \ - asm ("rev8 %0,%0; srli %0,%0,32"\ - : "+&r"(ret_)); ret_; }) # endif # elif defined(_MSC_VER) # if _MSC_VER>=1300 @@ -107,14 +100,14 @@ _asm mov eax, val _asm bswap eax} u64 hi, lo; } u128; -typedef void (*gcm_init_fn)(u128 Htable[16], const u64 H[2]); -typedef void (*gcm_ghash_fn)(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); -typedef void (*gcm_gmult_fn)(u64 Xi[2], const u128 Htable[16]); -struct gcm_funcs_st { - gcm_init_fn ginit; - gcm_ghash_fn ghash; - gcm_gmult_fn gmult; -}; +#ifdef TABLE_BITS +# undef TABLE_BITS +#endif +/* + * Even though permitted values for TABLE_BITS are 8, 4 and 1, it should + * never be set to 8 [or 1]. For further information see gcm128.c. + */ +#define TABLE_BITS 4 struct gcm128_context { /* Following 6 names follow names in GCM specification */ @@ -125,11 +118,17 @@ struct gcm128_context { size_t t[16 / sizeof(size_t)]; } Yi, EKi, EK0, len, Xi, H; /* - * Relative position of Yi, EKi, EK0, len, Xi, H and pre-computed Htable is - * used in some assembler modules, i.e. don't change the order! + * Relative position of Xi, H and pre-computed Htable is used in some + * assembler modules, i.e. don't change the order! */ +#if TABLE_BITS==8 + u128 Htable[256]; +#else u128 Htable[16]; - struct gcm_funcs_st funcs; + void (*gmult) (u64 Xi[2], const u128 Htable[16]); + void (*ghash) (u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); +#endif unsigned int mres, ares; block128_f block; void *key; @@ -138,12 +137,6 @@ struct gcm128_context { #endif }; -/* GHASH functions */ -void ossl_gcm_init_4bit(u128 Htable[16], const u64 H[2]); -void ossl_gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], - const u8 *inp, size_t len); -void ossl_gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16]); - /* * The maximum permitted number of cipher blocks per data unit in XTS mode. * Reference IEEE Std 1619-2018. @@ -155,12 +148,6 @@ struct xts128_context { block128_f block1, block2; }; -/* XTS mode for SM4 algorithm specified by GB/T 17964-2021 */ -int ossl_crypto_xts128gb_encrypt(const XTS128_CONTEXT *ctx, - const unsigned char iv[16], - const unsigned char *inp, unsigned char *out, - size_t len, int enc); - struct ccm128_context { union { u64 u[2]; diff --git a/openssl/include/crypto/pkcs12err.h b/openssl/include/crypto/pkcs12err.h index 114971c60..662f412e9 100644 --- a/openssl/include/crypto/pkcs12err.h +++ b/openssl/include/crypto/pkcs12err.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/poly1305.h b/openssl/include/crypto/poly1305.h index ba54f3bdc..99c9a111a 100644 --- a/openssl/include/crypto/poly1305.h +++ b/openssl/include/crypto/poly1305.h @@ -12,6 +12,7 @@ # pragma once #include +#include #define POLY1305_BLOCK_SIZE 16 #define POLY1305_DIGEST_SIZE 16 diff --git a/openssl/include/crypto/ppc_arch.h b/openssl/include/crypto/ppc_arch.h index d999396a2..3b3ce4bff 100644 --- a/openssl/include/crypto/ppc_arch.h +++ b/openssl/include/crypto/ppc_arch.h @@ -1,5 +1,5 @@ /* - * Copyright 2014-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,6 +24,5 @@ extern unsigned int OPENSSL_ppccap_P; # define PPC_MADD300 (1<<4) # define PPC_MFTB (1<<5) # define PPC_MFSPR268 (1<<6) -# define PPC_BRD31 (1<<7) #endif diff --git a/openssl/include/crypto/punycode.h b/openssl/include/crypto/punycode.h index 2e1c85c1f..133826d87 100644 --- a/openssl/include/crypto/punycode.h +++ b/openssl/include/crypto/punycode.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,8 +11,6 @@ # define OSSL_CRYPTO_PUNYCODE_H # pragma once -# include /* for size_t */ - int ossl_punycode_decode ( const char *pEncoded, const size_t enc_len, @@ -20,6 +18,7 @@ int ossl_punycode_decode ( unsigned int *pout_length ); -int ossl_a2ulabel(const char *in, char *out, size_t outlen); +int ossl_a2ulabel(const char *in, char *out, size_t *outlen); +int ossl_a2ucompare(const char *a, const char *u); #endif diff --git a/openssl/include/crypto/rand.h b/openssl/include/crypto/rand.h index d375c2f93..fa3b5b2b9 100644 --- a/openssl/include/crypto/rand.h +++ b/openssl/include/crypto/rand.h @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -108,26 +108,16 @@ void ossl_random_add_conf_module(void); /* * Get and cleanup random seed material. */ -size_t ossl_rand_get_entropy(OSSL_LIB_CTX *ctx, +size_t ossl_rand_get_entropy(ossl_unused OSSL_CORE_HANDLE *handle, unsigned char **pout, int entropy, size_t min_len, size_t max_len); -size_t ossl_rand_get_user_entropy(OSSL_LIB_CTX *ctx, - unsigned char **pout, int entropy, - size_t min_len, size_t max_len); -void ossl_rand_cleanup_entropy(OSSL_LIB_CTX *ctx, +void ossl_rand_cleanup_entropy(ossl_unused OSSL_CORE_HANDLE *handle, unsigned char *buf, size_t len); -void ossl_rand_cleanup_user_entropy(OSSL_LIB_CTX *ctx, - unsigned char *buf, size_t len); -size_t ossl_rand_get_nonce(OSSL_LIB_CTX *ctx, +size_t ossl_rand_get_nonce(ossl_unused OSSL_CORE_HANDLE *handle, unsigned char **pout, size_t min_len, size_t max_len, const void *salt, size_t salt_len); -size_t ossl_rand_get_user_nonce(OSSL_LIB_CTX *ctx, unsigned char **pout, - size_t min_len, size_t max_len, - const void *salt, size_t salt_len); -void ossl_rand_cleanup_nonce(OSSL_LIB_CTX *ctx, +void ossl_rand_cleanup_nonce(ossl_unused OSSL_CORE_HANDLE *handle, unsigned char *buf, size_t len); -void ossl_rand_cleanup_user_nonce(OSSL_LIB_CTX *ctx, - unsigned char *buf, size_t len); /* * Get seeding material from the operating system sources. @@ -135,20 +125,4 @@ void ossl_rand_cleanup_user_nonce(OSSL_LIB_CTX *ctx, size_t ossl_pool_acquire_entropy(RAND_POOL *pool); int ossl_pool_add_nonce_data(RAND_POOL *pool); -# ifdef FIPS_MODULE -EVP_RAND_CTX *ossl_rand_get0_private_noncreating(OSSL_LIB_CTX *ctx); -# else -EVP_RAND_CTX *ossl_rand_get0_seed_noncreating(OSSL_LIB_CTX *ctx); -# endif - -/* Generate a uniformly distributed random integer in the interval [0, upper) */ -uint32_t ossl_rand_uniform_uint32(OSSL_LIB_CTX *ctx, uint32_t upper, int *err); - -/* - * Generate a uniformly distributed random integer in the interval - * [lower, upper). - */ -uint32_t ossl_rand_range_uint32(OSSL_LIB_CTX *ctx, uint32_t lower, uint32_t upper, - int *err); - #endif diff --git a/openssl/include/crypto/randerr.h b/openssl/include/crypto/randerr.h index 6e2eb0716..832a8b7d3 100644 --- a/openssl/include/crypto/randerr.h +++ b/openssl/include/crypto/randerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/riscv_arch.def b/openssl/include/crypto/riscv_arch.def deleted file mode 100644 index 70b0647ae..000000000 --- a/openssl/include/crypto/riscv_arch.def +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* X Macro Definitions for Specification of RISC-V Arch Capabilities */ - -/* - * Each RISC-V capability ends up encoded as a single set bit in an array of - * words. When specifying a new capability, write a new RISCV_DEFINE_CAP - * statement, with an argument as the extension name in all-caps, - * second argument as the index in the array where the capability will be stored - * and third argument as the index of the bit to be used to encode the - * capability. - * RISCV_DEFINE_CAP(EXTENSION NAME, array index, bit index) */ - -RISCV_DEFINE_CAP(ZBA, 0, 0) -RISCV_DEFINE_CAP(ZBB, 0, 1) -RISCV_DEFINE_CAP(ZBC, 0, 2) -RISCV_DEFINE_CAP(ZBS, 0, 3) -RISCV_DEFINE_CAP(ZBKB, 0, 4) -RISCV_DEFINE_CAP(ZBKC, 0, 5) -RISCV_DEFINE_CAP(ZBKX, 0, 6) -RISCV_DEFINE_CAP(ZKND, 0, 7) -RISCV_DEFINE_CAP(ZKNE, 0, 8) -RISCV_DEFINE_CAP(ZKNH, 0, 9) -RISCV_DEFINE_CAP(ZKSED, 0, 10) -RISCV_DEFINE_CAP(ZKSH, 0, 11) -RISCV_DEFINE_CAP(ZKR, 0, 12) -RISCV_DEFINE_CAP(ZKT, 0, 13) -RISCV_DEFINE_CAP(V, 0, 14) -RISCV_DEFINE_CAP(ZVBB, 0, 15) -RISCV_DEFINE_CAP(ZVBC, 0, 16) -RISCV_DEFINE_CAP(ZVKB, 0, 17) -RISCV_DEFINE_CAP(ZVKG, 0, 18) -RISCV_DEFINE_CAP(ZVKNED, 0, 19) -RISCV_DEFINE_CAP(ZVKNHA, 0, 20) -RISCV_DEFINE_CAP(ZVKNHB, 0, 21) -RISCV_DEFINE_CAP(ZVKSED, 0, 22) -RISCV_DEFINE_CAP(ZVKSH, 0, 23) - -/* - * In the future ... - * RISCV_DEFINE_CAP(ZFOO, 0, 31) - * RISCV_DEFINE_CAP(ZBAR, 1, 0) - * ... and so on. - */ - -#undef RISCV_DEFINE_CAP diff --git a/openssl/include/crypto/riscv_arch.h b/openssl/include/crypto/riscv_arch.h deleted file mode 100644 index 6950137f4..000000000 --- a/openssl/include/crypto/riscv_arch.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_CRYPTO_RISCV_ARCH_H -# define OSSL_CRYPTO_RISCV_ARCH_H - -# include -# include - -# define RISCV_DEFINE_CAP(NAME, INDEX, BIT_INDEX) +1 -extern uint32_t OPENSSL_riscvcap_P[ (( -# include "riscv_arch.def" -) + sizeof(uint32_t) - 1) / sizeof(uint32_t) ]; - -# ifdef OPENSSL_RISCVCAP_IMPL -# define RISCV_DEFINE_CAP(NAME, INDEX, BIT_INDEX) +1 -uint32_t OPENSSL_riscvcap_P[ (( -# include "riscv_arch.def" -) + sizeof(uint32_t) - 1) / sizeof(uint32_t) ]; -# endif - -# define RISCV_DEFINE_CAP(NAME, INDEX, BIT_INDEX) \ - static inline int RISCV_HAS_##NAME(void) \ - { \ - return (OPENSSL_riscvcap_P[INDEX] & (1 << BIT_INDEX)) != 0; \ - } -# include "riscv_arch.def" - -struct RISCV_capability_s { - const char *name; - size_t index; - size_t bit_offset; -}; - -# define RISCV_DEFINE_CAP(NAME, INDEX, BIT_INDEX) +1 -extern const struct RISCV_capability_s RISCV_capabilities[ -# include "riscv_arch.def" -]; - -# ifdef OPENSSL_RISCVCAP_IMPL -# define RISCV_DEFINE_CAP(NAME, INDEX, BIT_INDEX) \ - { #NAME, INDEX, BIT_INDEX }, -const struct RISCV_capability_s RISCV_capabilities[] = { -# include "riscv_arch.def" -}; -# endif - -# define RISCV_DEFINE_CAP(NAME, INDEX, BIT_INDEX) +1 -static const size_t kRISCVNumCaps = -# include "riscv_arch.def" -; - -/* Extension combination tests. */ -#define RISCV_HAS_ZBB_AND_ZBC() (RISCV_HAS_ZBB() && RISCV_HAS_ZBC()) -#define RISCV_HAS_ZBKB_AND_ZKND_AND_ZKNE() (RISCV_HAS_ZBKB() && RISCV_HAS_ZKND() && RISCV_HAS_ZKNE()) -#define RISCV_HAS_ZKND_AND_ZKNE() (RISCV_HAS_ZKND() && RISCV_HAS_ZKNE()) -/* - * The ZVBB is the superset of ZVKB extension. We use macro here to replace the - * `RISCV_HAS_ZVKB()` with `RISCV_HAS_ZVBB() || RISCV_HAS_ZVKB()`. - */ -#define RISCV_HAS_ZVKB() (RISCV_HAS_ZVBB() || RISCV_HAS_ZVKB()) -#define RISCV_HAS_ZVKB_AND_ZVKNHA() (RISCV_HAS_ZVKB() && RISCV_HAS_ZVKNHA()) -#define RISCV_HAS_ZVKB_AND_ZVKNHB() (RISCV_HAS_ZVKB() && RISCV_HAS_ZVKNHB()) -#define RISCV_HAS_ZVKB_AND_ZVKSED() (RISCV_HAS_ZVKB() && RISCV_HAS_ZVKSED()) -#define RISCV_HAS_ZVKB_AND_ZVKSH() (RISCV_HAS_ZVKB() && RISCV_HAS_ZVKSH()) - -/* - * Get the size of a vector register in bits (VLEN). - * If RISCV_HAS_V() is false, then this returns 0. - */ -size_t riscv_vlen(void); - -#endif diff --git a/openssl/include/crypto/rsa.h b/openssl/include/crypto/rsa.h index 592efdb7f..949873d0e 100644 --- a/openssl/include/crypto/rsa.h +++ b/openssl/include/crypto/rsa.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -34,6 +34,8 @@ int ossl_rsa_pss_params_30_copy(RSA_PSS_PARAMS_30 *to, int ossl_rsa_pss_params_30_is_unrestricted(const RSA_PSS_PARAMS_30 *rsa_pss_params); int ossl_rsa_pss_params_30_set_hashalg(RSA_PSS_PARAMS_30 *rsa_pss_params, int hashalg_nid); +int ossl_rsa_pss_params_30_set_maskgenalg(RSA_PSS_PARAMS_30 *rsa_pss_params, + int maskgenalg_nid); int ossl_rsa_pss_params_30_set_maskgenhashalg(RSA_PSS_PARAMS_30 *rsa_pss_params, int maskgenhashalg_nid); int ossl_rsa_pss_params_30_set_saltlen(RSA_PSS_PARAMS_30 *rsa_pss_params, @@ -54,9 +56,9 @@ RSA *ossl_rsa_new_with_ctx(OSSL_LIB_CTX *libctx); OSSL_LIB_CTX *ossl_rsa_get0_libctx(RSA *r); void ossl_rsa_set0_libctx(RSA *r, OSSL_LIB_CTX *libctx); -int ossl_rsa_set0_all_params(RSA *r, STACK_OF(BIGNUM) *primes, - STACK_OF(BIGNUM) *exps, - STACK_OF(BIGNUM) *coeffs); +int ossl_rsa_set0_all_params(RSA *r, const STACK_OF(BIGNUM) *primes, + const STACK_OF(BIGNUM) *exps, + const STACK_OF(BIGNUM) *coeffs); int ossl_rsa_get0_all_params(RSA *r, STACK_OF(BIGNUM_const) *primes, STACK_OF(BIGNUM_const) *exps, STACK_OF(BIGNUM_const) *coeffs); @@ -81,10 +83,6 @@ int ossl_rsa_param_decode(RSA *rsa, const X509_ALGOR *alg); RSA *ossl_rsa_key_from_pkcs8(const PKCS8_PRIV_KEY_INFO *p8inf, OSSL_LIB_CTX *libctx, const char *propq); -int ossl_rsa_padding_check_PKCS1_type_2(OSSL_LIB_CTX *ctx, - unsigned char *to, int tlen, - const unsigned char *from, int flen, - int num, unsigned char *kdk); int ossl_rsa_padding_check_PKCS1_type_2_TLS(OSSL_LIB_CTX *ctx, unsigned char *to, size_t tlen, const unsigned char *from, diff --git a/openssl/include/crypto/sha.h b/openssl/include/crypto/sha.h index 99bcf0ff8..64305d179 100644 --- a/openssl/include/crypto/sha.h +++ b/openssl/include/crypto/sha.h @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -14,7 +14,6 @@ # include -int ossl_sha256_192_init(SHA256_CTX *c); int sha512_224_init(SHA512_CTX *); int sha512_256_init(SHA512_CTX *); int ossl_sha1_ctrl(SHA_CTX *ctx, int cmd, int mslen, void *ms); diff --git a/openssl/include/crypto/sm2.h b/openssl/include/crypto/sm2.h index 9ab6c0b72..d032aa13b 100644 --- a/openssl/include/crypto/sm2.h +++ b/openssl/include/crypto/sm2.h @@ -28,7 +28,7 @@ int ossl_sm2_key_private_check(const EC_KEY *eckey); int ossl_sm2_compute_z_digest(uint8_t *out, const EVP_MD *digest, const uint8_t *id, - const size_t id_len, + size_t id_len, const EC_KEY *key); /* @@ -82,5 +82,13 @@ int ossl_sm2_decrypt(const EC_KEY *key, const unsigned char *ossl_sm2_algorithmidentifier_encoding(int md_nid, size_t *len); + +int SM2_compute_key(void *out, size_t outlen, int initiator, + const uint8_t *peer_id, size_t peer_id_len, + const uint8_t *self_id, size_t self_id_len, + const EC_KEY *peer_ecdhe_key, const EC_KEY *self_ecdhe_key, + const EC_KEY *peer_pub_key, const EC_KEY *self_eckey, + const EVP_MD *md, OSSL_LIB_CTX *libctx, + const char *propq); # endif /* OPENSSL_NO_SM2 */ #endif diff --git a/openssl/include/crypto/sm2err.h b/openssl/include/crypto/sm2err.h index 706f4d69d..294aed20b 100644 --- a/openssl/include/crypto/sm2err.h +++ b/openssl/include/crypto/sm2err.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -39,6 +39,7 @@ int ossl_err_load_SM2_strings(void); # define SM2_R_INVALID_FIELD 105 # define SM2_R_INVALID_PRIVATE_KEY 113 # define SM2_R_NO_PARAMETERS_SET 109 +# define SM2_R_POINT_ARITHMETIC_FAILURE 114 # define SM2_R_USER_ID_TOO_LARGE 106 # endif diff --git a/openssl/include/crypto/sm4_platform.h b/openssl/include/crypto/sm4_platform.h index 928dc17ff..8567aaa3f 100644 --- a/openssl/include/crypto/sm4_platform.h +++ b/openssl/include/crypto/sm4_platform.h @@ -1,5 +1,5 @@ /* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,46 +11,43 @@ # define OSSL_SM4_PLATFORM_H # pragma once +# include + # if defined(OPENSSL_CPUID_OBJ) -# if defined(__aarch64__) || defined (_M_ARM64) +# if (defined(__arm__) || defined(__arm) || defined(__aarch64__)) # include "arm_arch.h" -extern unsigned int OPENSSL_arm_midr; -static inline int vpsm4_capable(void) -{ - return (OPENSSL_armcap_P & ARMV8_CPUID) && - (MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V1) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N1)); -} -static inline int vpsm4_ex_capable(void) -{ - return (OPENSSL_armcap_P & ARMV8_CPUID) && - (MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, HISI_CPU_IMP, HISI_CPU_PART_KP920)); -} -# if defined(VPSM4_ASM) -# define VPSM4_CAPABLE vpsm4_capable() -# define VPSM4_EX_CAPABLE vpsm4_ex_capable() +# if __ARM_MAX_ARCH__>=8 +# define HWSM4_CAPABLE (OPENSSL_armcap_P & ARMV8_SM4) +# ifdef HWSM4_set_encrypt_key +# undef HWSM4_set_encrypt_key +# endif +# define HWSM4_set_encrypt_key sm4_v8_set_encrypt_key +# ifdef HWSM4_set_decrypt_key +# undef HWSM4_set_decrypt_key +# endif +# define HWSM4_set_decrypt_key sm4_v8_set_decrypt_key +# ifdef HWSM4_encrypt +# undef HWSM4_encrypt +# endif +# define HWSM4_encrypt sm4_v8_encrypt +# ifdef HWSM4_decrypt +# undef HWSM4_decrypt +# endif +# define HWSM4_decrypt sm4_v8_decrypt +# ifdef HWSM4_cbc_encrypt +# undef HWSM4_cbc_encrypt +# endif +# define HWSM4_cbc_encrypt sm4_v8_cbc_encrypt +# ifdef HWSM4_ecb_encrypt +# undef HWSM4_ecb_encrypt +# endif +# define HWSM4_ecb_encrypt sm4_v8_ecb_encrypt +# ifdef HWSM4_ctr32_encrypt_blocks +# undef HWSM4_ctr32_encrypt_blocks +# endif +# define HWSM4_ctr32_encrypt_blocks sm4_v8_ctr32_encrypt_blocks # endif -# define HWSM4_CAPABLE (OPENSSL_armcap_P & ARMV8_SM4) -# define HWSM4_set_encrypt_key sm4_v8_set_encrypt_key -# define HWSM4_set_decrypt_key sm4_v8_set_decrypt_key -# define HWSM4_encrypt sm4_v8_encrypt -# define HWSM4_decrypt sm4_v8_decrypt -# define HWSM4_cbc_encrypt sm4_v8_cbc_encrypt -# define HWSM4_ecb_encrypt sm4_v8_ecb_encrypt -# define HWSM4_ctr32_encrypt_blocks sm4_v8_ctr32_encrypt_blocks -# elif defined(__riscv) && __riscv_xlen == 64 -/* RV64 support */ -# include "riscv_arch.h" -/* Zvksed extension (vector crypto SM4). */ -int rv64i_zvksed_sm4_set_encrypt_key(const unsigned char *userKey, - SM4_KEY *key); -int rv64i_zvksed_sm4_set_decrypt_key(const unsigned char *userKey, - SM4_KEY *key); -void rv64i_zvksed_sm4_encrypt(const unsigned char *in, unsigned char *out, - const SM4_KEY *key); -void rv64i_zvksed_sm4_decrypt(const unsigned char *in, unsigned char *out, - const SM4_KEY *key); -# endif /* RV64 */ +# endif # endif /* OPENSSL_CPUID_OBJ */ # if defined(HWSM4_CAPABLE) @@ -71,53 +68,4 @@ void HWSM4_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, const unsigned char ivec[16]); # endif /* HWSM4_CAPABLE */ -# ifdef VPSM4_CAPABLE -int vpsm4_set_encrypt_key(const unsigned char *userKey, SM4_KEY *key); -int vpsm4_set_decrypt_key(const unsigned char *userKey, SM4_KEY *key); -void vpsm4_encrypt(const unsigned char *in, unsigned char *out, - const SM4_KEY *key); -void vpsm4_decrypt(const unsigned char *in, unsigned char *out, - const SM4_KEY *key); -void vpsm4_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const SM4_KEY *key, - unsigned char *ivec, const int enc); -void vpsm4_ecb_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const SM4_KEY *key, - const int enc); -void vpsm4_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, - size_t len, const void *key, - const unsigned char ivec[16]); -void vpsm4_xts_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const SM4_KEY *key1, const SM4_KEY *key2, - const unsigned char ivec[16], const int enc); -void vpsm4_xts_encrypt_gb(const unsigned char *in, unsigned char *out, - size_t len, const SM4_KEY *key1, const SM4_KEY *key2, - const unsigned char ivec[16], const int enc); -# endif /* VPSM4_CAPABLE */ - -# ifdef VPSM4_EX_CAPABLE -int vpsm4_ex_set_encrypt_key(const unsigned char *userKey, SM4_KEY *key); -int vpsm4_ex_set_decrypt_key(const unsigned char *userKey, SM4_KEY *key); -void vpsm4_ex_encrypt(const unsigned char *in, unsigned char *out, - const SM4_KEY *key); -void vpsm4_ex_decrypt(const unsigned char *in, unsigned char *out, - const SM4_KEY *key); -void vpsm4_ex_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const SM4_KEY *key, - unsigned char *ivec, const int enc); -void vpsm4_ex_ecb_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const SM4_KEY *key, - const int enc); -void vpsm4_ex_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, - size_t len, const void *key, - const unsigned char ivec[16]); -void vpsm4_ex_xts_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const SM4_KEY *key1, const SM4_KEY *key2, - const unsigned char ivec[16], const int enc); -void vpsm4_ex_xts_encrypt_gb(const unsigned char *in, unsigned char *out, - size_t len, const SM4_KEY *key1, - const SM4_KEY *key2, const unsigned char ivec[16], - const int enc); -# endif /* VPSM4_EX_CAPABLE */ - #endif /* OSSL_SM4_PLATFORM_H */ diff --git a/openssl/include/crypto/sparc_arch.h b/openssl/include/crypto/sparc_arch.h deleted file mode 100644 index 447e715bf..000000000 --- a/openssl/include/crypto/sparc_arch.h +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2012-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_CRYPTO_SPARC_ARCH_H -# define OSSL_CRYPTO_SPARC_ARCH_H - -# define SPARCV9_TICK_PRIVILEGED (1<<0) -# define SPARCV9_PREFER_FPU (1<<1) -# define SPARCV9_VIS1 (1<<2) -# define SPARCV9_VIS2 (1<<3)/* reserved */ -# define SPARCV9_FMADD (1<<4) -# define SPARCV9_BLK (1<<5)/* VIS1 block copy */ -# define SPARCV9_VIS3 (1<<6) -# define SPARCV9_RANDOM (1<<7) -# define SPARCV9_64BIT_STACK (1<<8) -# define SPARCV9_FJAESX (1<<9)/* Fujitsu SPARC64 X AES */ -# define SPARCV9_FJDESX (1<<10)/* Fujitsu SPARC64 X DES, reserved */ -# define SPARCV9_FJHPCACE (1<<11)/* Fujitsu HPC-ACE, reserved */ -# define SPARCV9_IMA (1<<13)/* reserved */ -# define SPARCV9_VIS4 (1<<14)/* reserved */ - -/* - * OPENSSL_sparcv9cap_P[1] is copy of Compatibility Feature Register, - * %asr26, SPARC-T4 and later. There is no SPARCV9_CFR bit in - * OPENSSL_sparcv9cap_P[0], as %cfr copy is sufficient... - */ -# define CFR_AES 0x00000001/* Supports AES opcodes */ -# define CFR_DES 0x00000002/* Supports DES opcodes */ -# define CFR_KASUMI 0x00000004/* Supports KASUMI opcodes */ -# define CFR_CAMELLIA 0x00000008/* Supports CAMELLIA opcodes */ -# define CFR_MD5 0x00000010/* Supports MD5 opcodes */ -# define CFR_SHA1 0x00000020/* Supports SHA1 opcodes */ -# define CFR_SHA256 0x00000040/* Supports SHA256 opcodes */ -# define CFR_SHA512 0x00000080/* Supports SHA512 opcodes */ -# define CFR_MPMUL 0x00000100/* Supports MPMUL opcodes */ -# define CFR_MONTMUL 0x00000200/* Supports MONTMUL opcodes */ -# define CFR_MONTSQR 0x00000400/* Supports MONTSQR opcodes */ -# define CFR_CRC32C 0x00000800/* Supports CRC32C opcodes */ -# define CFR_XMPMUL 0x00001000/* Supports XMPMUL opcodes */ -# define CFR_XMONTMUL 0x00002000/* Supports XMONTMUL opcodes */ -# define CFR_XMONTSQR 0x00004000/* Supports XMONTSQR opcodes */ - -# if defined(OPENSSL_PIC) && !defined(__PIC__) -# define __PIC__ -# endif - -# if defined(__SUNPRO_C) && defined(__sparcv9) && !defined(__arch64__) -# define __arch64__ -# endif - -# define SPARC_PIC_THUNK(reg) \ - .align 32; \ -.Lpic_thunk: \ - jmp %o7 + 8; \ - add %o7, reg, reg; - -# define SPARC_PIC_THUNK_CALL(reg) \ - sethi %hi(_GLOBAL_OFFSET_TABLE_-4), reg; \ - call .Lpic_thunk; \ - or reg, %lo(_GLOBAL_OFFSET_TABLE_+4), reg; - -# if 1 -# define SPARC_SETUP_GOT_REG(reg) SPARC_PIC_THUNK_CALL(reg) -# else -# define SPARC_SETUP_GOT_REG(reg) \ - sethi %hi(_GLOBAL_OFFSET_TABLE_-4), reg; \ - call .+8; \ - or reg,%lo(_GLOBAL_OFFSET_TABLE_+4), reg; \ - add %o7, reg, reg -# endif - -# if defined(__arch64__) - -# define SPARC_LOAD_ADDRESS(SYM, reg) \ - setx SYM, %o7, reg; -# define LDPTR ldx -# define SIZE_T_CC %xcc -# define STACK_FRAME 192 -# define STACK_BIAS 2047 -# define STACK_7thARG (STACK_BIAS+176) - -# else - -# define SPARC_LOAD_ADDRESS(SYM, reg) \ - set SYM, reg; -# define LDPTR ld -# define SIZE_T_CC %icc -# define STACK_FRAME 112 -# define STACK_BIAS 0 -# define STACK_7thARG 92 -# define SPARC_LOAD_ADDRESS_LEAF(SYM,reg,tmp) SPARC_LOAD_ADDRESS(SYM,reg) - -# endif - -# ifdef __PIC__ -# undef SPARC_LOAD_ADDRESS -# undef SPARC_LOAD_ADDRESS_LEAF -# define SPARC_LOAD_ADDRESS(SYM, reg) \ - SPARC_SETUP_GOT_REG(reg); \ - sethi %hi(SYM), %o7; \ - or %o7, %lo(SYM), %o7; \ - LDPTR [reg + %o7], reg; -# endif - -# ifndef SPARC_LOAD_ADDRESS_LEAF -# define SPARC_LOAD_ADDRESS_LEAF(SYM, reg, tmp) \ - mov %o7, tmp; \ - SPARC_LOAD_ADDRESS(SYM, reg) \ - mov tmp, %o7; -# endif - -# ifndef __ASSEMBLER__ -extern unsigned int OPENSSL_sparcv9cap_P[2]; -# endif - -#endif /* OSSL_CRYPTO_SPARC_ARCH_H */ diff --git a/openssl/include/crypto/store.h b/openssl/include/crypto/store.h index 9b7be71ac..13d2646bb 100644 --- a/openssl/include/crypto/store.h +++ b/openssl/include/crypto/store.h @@ -17,7 +17,5 @@ void ossl_store_cleanup_int(void); int ossl_store_loader_get_number(const OSSL_STORE_LOADER *loader); -int ossl_store_loader_store_cache_flush(OSSL_LIB_CTX *libctx); -int ossl_store_loader_store_remove_all_provided(const OSSL_PROVIDER *prov); #endif diff --git a/openssl/include/crypto/types.h b/openssl/include/crypto/types.h index ad17f052e..0d8140409 100644 --- a/openssl/include/crypto/types.h +++ b/openssl/include/crypto/types.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,9 +20,6 @@ typedef struct rsa_meth_st RSA_METHOD; typedef struct ec_key_st EC_KEY; typedef struct ec_key_method_st EC_KEY_METHOD; # endif -# ifndef OPENSSL_NO_DSA -typedef struct dsa_st DSA; -# endif # endif # ifndef OPENSSL_NO_EC diff --git a/openssl/include/crypto/x509.h b/openssl/include/crypto/x509.h index 18eb2f7c6..6efa98268 100644 --- a/openssl/include/crypto/x509.h +++ b/openssl/include/crypto/x509.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -207,6 +207,26 @@ struct x509_st { char *propq; } /* X509 */ ; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +struct delegated_credential_st{ + OSSL_LIB_CTX *libctx; + char *propq; + + unsigned long valid_time; + unsigned int expected_cert_verify_algorithm; + size_t dc_publickey_raw_len; + unsigned char *dc_publickey_raw; + unsigned int signature_sign_algorithm; + size_t dc_signature_len; + unsigned char *dc_signature; + EVP_PKEY *pkey; + unsigned char *raw_byte; + size_t raw_byte_len; + CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; +} /* DC */; +#endif + /* * This is a used when verifying cert chains. Since the gathering of the * cert chain can take some time (and have to be 'retried', this needs to be @@ -278,8 +298,8 @@ struct x509_store_ctx_st { /* X509_STORE_CTX */ SSL_DANE *dane; /* signed via bare TA public key, rather than CA certificate */ int bare_ta_signed; - /* Raw Public Key */ - EVP_PKEY *rpk; + /* verify options */ + STACK_OF(OPENSSL_STRING) *vfyopts; OSSL_LIB_CTX *libctx; char *propq; @@ -311,7 +331,7 @@ struct x509_object_st { }; int ossl_a2i_ipadd(unsigned char *ipout, const char *ipasc); -int ossl_x509_set1_time(int *modified, ASN1_TIME **ptm, const ASN1_TIME *tm); +int ossl_x509_set1_time(ASN1_TIME **ptm, const ASN1_TIME *tm); int ossl_x509_print_ex_brief(BIO *bio, X509 *cert, unsigned long neg_cflags); int ossl_x509v3_cache_extensions(X509 *x); int ossl_x509_init_sig_info(X509 *x); @@ -336,15 +356,11 @@ int ossl_x509_PUBKEY_get0_libctx(OSSL_LIB_CTX **plibctx, const char **ppropq, ASN1_OCTET_STRING *ossl_x509_pubkey_hash(X509_PUBKEY *pubkey); X509_PUBKEY *ossl_d2i_X509_PUBKEY_INTERNAL(const unsigned char **pp, - long len, OSSL_LIB_CTX *libctx, - const char *propq); + long len, OSSL_LIB_CTX *libctx); void ossl_X509_PUBKEY_INTERNAL_free(X509_PUBKEY *xpub); RSA *ossl_d2i_RSA_PSS_PUBKEY(RSA **a, const unsigned char **pp, long length); int ossl_i2d_RSA_PSS_PUBKEY(const RSA *a, unsigned char **pp); -# ifndef OPENSSL_NO_DSA -DSA *ossl_d2i_DSA_PUBKEY(DSA **a, const unsigned char **pp, long length); -# endif /* OPENSSL_NO_DSA */ # ifndef OPENSSL_NO_DH DH *ossl_d2i_DH_PUBKEY(DH **a, const unsigned char **pp, long length); int ossl_i2d_DH_PUBKEY(const DH *a, unsigned char **pp); @@ -367,25 +383,7 @@ int ossl_i2d_X448_PUBKEY(const ECX_KEY *a, unsigned char **pp); # endif /* OPENSSL_NO_EC */ EVP_PKEY *ossl_d2i_PUBKEY_legacy(EVP_PKEY **a, const unsigned char **pp, long length); -int ossl_x509_check_private_key(const EVP_PKEY *k, const EVP_PKEY *pkey); int x509v3_add_len_value_uchar(const char *name, const unsigned char *value, size_t vallen, STACK_OF(CONF_VALUE) **extlist); -/* Attribute addition functions not checking for duplicate attributes */ -STACK_OF(X509_ATTRIBUTE) *ossl_x509at_add1_attr(STACK_OF(X509_ATTRIBUTE) **x, - X509_ATTRIBUTE *attr); -STACK_OF(X509_ATTRIBUTE) *ossl_x509at_add1_attr_by_OBJ(STACK_OF(X509_ATTRIBUTE) **x, - const ASN1_OBJECT *obj, - int type, - const unsigned char *bytes, - int len); -STACK_OF(X509_ATTRIBUTE) *ossl_x509at_add1_attr_by_NID(STACK_OF(X509_ATTRIBUTE) **x, - int nid, int type, - const unsigned char *bytes, - int len); -STACK_OF(X509_ATTRIBUTE) *ossl_x509at_add1_attr_by_txt(STACK_OF(X509_ATTRIBUTE) **x, - const char *attrname, - int type, - const unsigned char *bytes, - int len); #endif /* OSSL_CRYPTO_X509_H */ diff --git a/openssl/include/crypto/x509err.h b/openssl/include/crypto/x509err.h index c7c7d25e9..53f567d92 100644 --- a/openssl/include/crypto/x509err.h +++ b/openssl/include/crypto/x509err.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/include/crypto/zkpbperr.h b/openssl/include/crypto/zkpbperr.h new file mode 100644 index 000000000..cf25b5430 --- /dev/null +++ b/openssl/include/crypto/zkpbperr.h @@ -0,0 +1,27 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_ZKPBPERR_H +# define OSSL_CRYPTO_ZKPBPERR_H +# pragma once + +# include +# include + +# ifdef __cplusplus +extern "C" { +# endif + +int ossl_err_load_ZKP_BP_strings(void); + +# ifdef __cplusplus +} +# endif +#endif diff --git a/openssl/include/crypto/zkperr.h b/openssl/include/crypto/zkperr.h new file mode 100644 index 000000000..0501d1c6f --- /dev/null +++ b/openssl/include/crypto/zkperr.h @@ -0,0 +1,27 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_ZKPERR_H +# define OSSL_CRYPTO_ZKPERR_H +# pragma once + +# include +# include + +# ifdef __cplusplus +extern "C" { +# endif + +int ossl_err_load_ZKP_strings(void); + +# ifdef __cplusplus +} +# endif +#endif diff --git a/openssl/include/crypto/zkpnizkerr.h b/openssl/include/crypto/zkpnizkerr.h new file mode 100644 index 000000000..26191d9fd --- /dev/null +++ b/openssl/include/crypto/zkpnizkerr.h @@ -0,0 +1,27 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_ZKPNIZKERR_H +# define OSSL_CRYPTO_ZKPNIZKERR_H +# pragma once + +# include +# include + +# ifdef __cplusplus +extern "C" { +# endif + +int ossl_err_load_ZKP_NIZK_strings(void); + +# ifdef __cplusplus +} +# endif +#endif diff --git a/openssl/include/crypto/zuc.h b/openssl/include/crypto/zuc.h new file mode 100644 index 000000000..ec424413b --- /dev/null +++ b/openssl/include/crypto/zuc.h @@ -0,0 +1,60 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_ZUC_H +# define HEADER_ZUC_H + +# include + +# ifdef OPENSSL_NO_ZUC +# error ZUC is disabled. +# endif + +#define EVP_ZUC_KEY_SIZE 16 +#define EIA3_DIGEST_SIZE 4 + +#define ZUC_KEY_SIZE EVP_ZUC_KEY_SIZE +#define ZUC_CTR_SIZE 5 + +typedef struct ZUC_KEY_st { + /* Linear Feedback Shift Register cells */ + uint32_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15; + + /* the outputs of BitReorganization */ + uint32_t X0, X1, X2, X3; + + /* non linear function F cells */ + uint32_t R1, R2; + + const uint8_t *k; + uint8_t iv[16]; + + /* keystream */ + uint8_t keystream[8]; + uint8_t keystream_tail[8]; + uint32_t keystream_len; + int L; + + int inited; +} ZUC_KEY; + +typedef struct eia3_context EIA3_CTX; + +void ZUC_init(ZUC_KEY *zk); +int ZUC_generate_keystream(ZUC_KEY *zk); +void ZUC_destroy_keystream(ZUC_KEY *zk); +int ZUC_keystream_get_word(ZUC_KEY *zk, int i); +int ZUC_keystream_get_byte(ZUC_KEY *zk, int i); + +size_t EIA3_ctx_size(void); +int EIA3_Init(EIA3_CTX *ctx, const unsigned char key[EVP_ZUC_KEY_SIZE], const unsigned char iv[5]); +int EIA3_Update(EIA3_CTX *ctx, const unsigned char *inp, size_t len); +void EIA3_Final(EIA3_CTX *ctx, unsigned char out[EIA3_DIGEST_SIZE]); + +#endif diff --git a/openssl/include/internal/__DECC_INCLUDE_EPILOGUE.H b/openssl/include/internal/__DECC_INCLUDE_EPILOGUE.H deleted file mode 100644 index e57c0eab3..000000000 --- a/openssl/include/internal/__DECC_INCLUDE_EPILOGUE.H +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is only used by HP C on VMS, and is included automatically - * after each header file from this directory - */ - -/* restore state. Must correspond to the save in __decc_include_prologue.h */ -#pragma names restore diff --git a/openssl/include/internal/__DECC_INCLUDE_PROLOGUE.H b/openssl/include/internal/__DECC_INCLUDE_PROLOGUE.H deleted file mode 100644 index a01395755..000000000 --- a/openssl/include/internal/__DECC_INCLUDE_PROLOGUE.H +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is only used by HP C on VMS, and is included automatically - * after each header file from this directory - */ - -/* save state */ -#pragma names save -/* have the compiler shorten symbols larger than 31 chars to 23 chars - * followed by a 8 hex char CRC - */ -#pragma names as_is,shortened diff --git a/openssl/include/internal/asn1.h b/openssl/include/internal/asn1.h index 36dbe0fcd..3143e3405 100644 --- a/openssl/include/internal/asn1.h +++ b/openssl/include/internal/asn1.h @@ -11,8 +11,6 @@ # define OSSL_INTERNAL_ASN1_H # pragma once -# include - int asn1_d2i_read_bio(BIO *in, BUF_MEM **pb); #endif diff --git a/openssl/include/internal/bio.h b/openssl/include/internal/bio.h index 9481f4c98..547a73d02 100644 --- a/openssl/include/internal/bio.h +++ b/openssl/include/internal/bio.h @@ -27,8 +27,6 @@ struct bio_method_st { int (*create) (BIO *); int (*destroy) (BIO *); long (*callback_ctrl) (BIO *, int, BIO_info_cb *); - int (*bsendmmsg) (BIO *, BIO_MSG *, size_t, size_t, uint64_t, size_t *); - int (*brecvmmsg) (BIO *, BIO_MSG *, size_t, size_t, uint64_t, size_t *); }; void bio_free_ex_data(BIO *bio); @@ -43,20 +41,16 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); # define BIO_CTRL_SET_KTLS 72 # define BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG 74 # define BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG 75 -# define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90 /* * This is used with socket BIOs: * BIO_FLAGS_KTLS_TX means we are using ktls with this BIO for sending. * BIO_FLAGS_KTLS_TX_CTRL_MSG means we are about to send a ctrl message next. * BIO_FLAGS_KTLS_RX means we are using ktls with this BIO for receiving. - * BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE means we are using the zerocopy mode with - * this BIO for sending using sendfile. */ # define BIO_FLAGS_KTLS_TX_CTRL_MSG 0x1000 # define BIO_FLAGS_KTLS_RX 0x2000 # define BIO_FLAGS_KTLS_TX 0x4000 -# define BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE 0x8000 /* KTLS related controls and flags */ # define BIO_set_ktls_flag(b, is_tx) \ @@ -69,8 +63,6 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); BIO_test_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG) # define BIO_clear_ktls_ctrl_msg_flag(b) \ BIO_clear_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG) -# define BIO_set_ktls_zerocopy_sendfile_flag(b) \ - BIO_set_flags(b, BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE) # define BIO_set_ktls(b, keyblob, is_tx) \ BIO_ctrl(b, BIO_CTRL_SET_KTLS, is_tx, keyblob) @@ -78,8 +70,6 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG, record_type, NULL) # define BIO_clear_ktls_ctrl_msg(b) \ BIO_ctrl(b, BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG, 0, NULL) -# define BIO_set_ktls_tx_zerocopy_sendfile(b) \ - BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE, 0, NULL) /* Functions to allow the core to offer the CORE_BIO type to providers */ OSSL_CORE_BIO *ossl_core_bio_new_from_bio(BIO *bio); diff --git a/openssl/include/internal/bio_addr.h b/openssl/include/internal/bio_addr.h deleted file mode 100644 index a6449b7eb..000000000 --- a/openssl/include/internal/bio_addr.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_BIO_ADDR_H -# define OSSL_BIO_ADDR_H - -# include "internal/e_os.h" -# include "internal/sockets.h" - -# ifndef OPENSSL_NO_SOCK -union bio_addr_st { - struct sockaddr sa; -# if OPENSSL_USE_IPV6 - struct sockaddr_in6 s_in6; -# endif - struct sockaddr_in s_in; -# ifndef OPENSSL_NO_UNIX_SOCK - struct sockaddr_un s_un; -# endif -}; -# endif - -#endif diff --git a/openssl/include/internal/bio_tfo.h b/openssl/include/internal/bio_tfo.h deleted file mode 100644 index 64c0d4c32..000000000 --- a/openssl/include/internal/bio_tfo.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Contains definitions for simplifying the use of TCP Fast Open - * (RFC7413) in OpenSSL socket BIOs. - */ - -/* If a supported OS is added here, update test/bio_tfo_test.c */ -#if defined(TCP_FASTOPEN) && !defined(OPENSSL_NO_TFO) - -# if defined(OPENSSL_SYS_MACOSX) || defined(__FreeBSD__) -# include -# endif - -/* - * OSSL_TFO_SYSCTL is used to determine if TFO is supported by - * this kernel, and if supported, if it is enabled. This is more of - * a problem on FreeBSD 10.3 ~ 11.4, where TCP_FASTOPEN was defined, - * but not enabled by default in the kernel, and only for the server. - * Linux does not have sysctlbyname(), and the closest equivalent - * is to go into the /proc filesystem, but I'm not sure it's - * worthwhile. - * - * On MacOS and Linux: - * These operating systems use a single parameter to control TFO. - * The OSSL_TFO_CLIENT_FLAG and OSSL_TFO_SERVER_FLAGS are used to - * determine if TFO is enabled for the client and server respectively. - * - * OSSL_TFO_CLIENT_FLAG = 1 = client TFO enabled - * OSSL_TFO_SERVER_FLAG = 2 = server TFO enabled - * - * Such that: - * 0 = TFO disabled - * 3 = server and client TFO enabled - * - * macOS 10.14 and later support TFO. - * Linux kernel 3.6 added support for client TFO. - * Linux kernel 3.7 added support for server TFO. - * Linux kernel 3.13 enabled TFO by default. - * Linux kernel 4.11 added the TCP_FASTOPEN_CONNECT option. - * - * On FreeBSD: - * FreeBSD 10.3 ~ 11.4 uses a single sysctl for server enable. - * FreeBSD 12.0 and later uses separate sysctls for server and - * client enable. - * - * Some options are purposely NOT defined per-platform - * - * OSSL_TFO_SYSCTL - * Defined as a sysctlbyname() option to determine if - * TFO is enabled in the kernel (macOS, FreeBSD) - * - * OSSL_TFO_SERVER_SOCKOPT - * Defined to indicate the socket option used to enable - * TFO on a server socket (all) - * - * OSSL_TFO_SERVER_SOCKOPT_VALUE - * Value to be used with OSSL_TFO_SERVER_SOCKOPT - * - * OSSL_TFO_CONNECTX - * Use the connectx() function to make a client connection - * (macOS) - * - * OSSL_TFO_CLIENT_SOCKOPT - * Defined to indicate the socket option used to enable - * TFO on a client socket (FreeBSD, Linux 4.14 and later) - * - * OSSL_TFO_SENDTO - * Defined to indicate the sendto() message type to - * be used to initiate a TFO connection (FreeBSD, - * Linux pre-4.14) - * - * OSSL_TFO_DO_NOT_CONNECT - * Defined to skip calling connect() when creating a - * client socket (macOS, FreeBSD, Linux pre-4.14) - */ - -# if defined(OPENSSL_SYS_WINDOWS) -/* - * NO WINDOWS SUPPORT - * - * But this is what would be used on the server: - * - * define OSSL_TFO_SERVER_SOCKOPT TCP_FASTOPEN - * define OSSL_TFO_SERVER_SOCKOPT_VALUE 1 - * - * Still have to figure out client support - */ -# undef TCP_FASTOPEN -# endif - -/* NO VMS SUPPORT */ -# if defined(OPENSSL_SYS_VMS) -# undef TCP_FASTOPEN -# endif - -# if defined(OPENSSL_SYS_MACOSX) -# define OSSL_TFO_SYSCTL "net.inet.tcp.fastopen" -# define OSSL_TFO_SERVER_SOCKOPT TCP_FASTOPEN -# define OSSL_TFO_SERVER_SOCKOPT_VALUE 1 -# define OSSL_TFO_CONNECTX 1 -# define OSSL_TFO_DO_NOT_CONNECT 1 -# define OSSL_TFO_CLIENT_FLAG 1 -# define OSSL_TFO_SERVER_FLAG 2 -# endif - -# if defined(__FreeBSD__) -# if defined(TCP_FASTOPEN_PSK_LEN) -/* As of 12.0 these are the SYSCTLs */ -# define OSSL_TFO_SYSCTL_SERVER "net.inet.tcp.fastopen.server_enable" -# define OSSL_TFO_SYSCTL_CLIENT "net.inet.tcp.fastopen.client_enable" -# define OSSL_TFO_SERVER_SOCKOPT TCP_FASTOPEN -# define OSSL_TFO_SERVER_SOCKOPT_VALUE MAX_LISTEN -# define OSSL_TFO_CLIENT_SOCKOPT TCP_FASTOPEN -# define OSSL_TFO_DO_NOT_CONNECT 1 -# define OSSL_TFO_SENDTO 0 -/* These are the same because the sysctl are client/server-specific */ -# define OSSL_TFO_CLIENT_FLAG 1 -# define OSSL_TFO_SERVER_FLAG 1 -# else -/* 10.3 through 11.4 SYSCTL - ONLY SERVER SUPPORT */ -# define OSSL_TFO_SYSCTL "net.inet.tcp.fastopen.enabled" -# define OSSL_TFO_SERVER_SOCKOPT TCP_FASTOPEN -# define OSSL_TFO_SERVER_SOCKOPT_VALUE MAX_LISTEN -# define OSSL_TFO_SERVER_FLAG 1 -# endif -# endif - -# if defined(OPENSSL_SYS_LINUX) -/* OSSL_TFO_PROC not used, but of interest */ -# define OSSL_TFO_PROC "/proc/sys/net/ipv4/tcp_fastopen" -# define OSSL_TFO_SERVER_SOCKOPT TCP_FASTOPEN -# define OSSL_TFO_SERVER_SOCKOPT_VALUE MAX_LISTEN -# if defined(TCP_FASTOPEN_CONNECT) -# define OSSL_TFO_CLIENT_SOCKOPT TCP_FASTOPEN_CONNECT -# else -# define OSSL_TFO_SENDTO MSG_FASTOPEN -# define OSSL_TFO_DO_NOT_CONNECT 1 -# endif -# define OSSL_TFO_CLIENT_FLAG 1 -# define OSSL_TFO_SERVER_FLAG 2 -# endif - -#endif diff --git a/openssl/include/internal/common.h b/openssl/include/internal/common.h deleted file mode 100644 index b176a2749..000000000 --- a/openssl/include/internal/common.h +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_COMMON_H -# define OSSL_INTERNAL_COMMON_H -# pragma once - -# include -# include -# include "openssl/configuration.h" - -# include "internal/e_os.h" /* ossl_inline in many files */ -# include "internal/nelem.h" - -# if defined(__GNUC__) || defined(__clang__) -# define ossl_likely(x) __builtin_expect(!!(x), 1) -# define ossl_unlikely(x) __builtin_expect(!!(x), 0) -# else -# define ossl_likely(x) x -# define ossl_unlikely(x) x -# endif - -# if defined(__GNUC__) || defined(__clang__) -# define ALIGN32 __attribute((aligned(32))) -# define ALIGN64 __attribute((aligned(64))) -# elif defined(_MSC_VER) -# define ALIGN32 __declspec(align(32)) -# define ALIGN64 __declspec(align(64)) -# else -# define ALIGN32 -# define ALIGN64 -# endif - -# ifdef NDEBUG -# define ossl_assert(x) ossl_likely((x) != 0) -# else -__owur static ossl_inline int ossl_assert_int(int expr, const char *exprstr, - const char *file, int line) -{ - if (!expr) - OPENSSL_die(exprstr, file, line); - - return expr; -} - -# define ossl_assert(x) ossl_assert_int((x) != 0, "Assertion failed: "#x, \ - __FILE__, __LINE__) - -# endif - -/* Check if |pre|, which must be a string literal, is a prefix of |str| */ -#define HAS_PREFIX(str, pre) (strncmp(str, pre "", sizeof(pre) - 1) == 0) -/* As before, and if check succeeds, advance |str| past the prefix |pre| */ -#define CHECK_AND_SKIP_PREFIX(str, pre) \ - (HAS_PREFIX(str, pre) ? ((str) += sizeof(pre) - 1, 1) : 0) -/* Check if the string literal |p| is a case-insensitive prefix of |s| */ -#define HAS_CASE_PREFIX(s, p) (OPENSSL_strncasecmp(s, p "", sizeof(p) - 1) == 0) -/* As before, and if check succeeds, advance |str| past the prefix |pre| */ -#define CHECK_AND_SKIP_CASE_PREFIX(str, pre) \ - (HAS_CASE_PREFIX(str, pre) ? ((str) += sizeof(pre) - 1, 1) : 0) -/* Check if the string literal |suffix| is a case-insensitive suffix of |str| */ -#define HAS_CASE_SUFFIX(str, suffix) (strlen(str) < sizeof(suffix) - 1 ? 0 : \ - OPENSSL_strcasecmp(str + strlen(str) - sizeof(suffix) + 1, suffix "") == 0) - -/* - * Use this inside a union with the field that needs to be aligned to a - * reasonable boundary for the platform. The most pessimistic alignment - * of the listed types will be used by the compiler. - */ -# define OSSL_UNION_ALIGN \ - double align; \ - ossl_uintmax_t align_int; \ - void *align_ptr - -# define OPENSSL_CONF "openssl.cnf" - -# ifndef OPENSSL_SYS_VMS -# define X509_CERT_AREA OPENSSLDIR -# define X509_CERT_DIR OPENSSLDIR "/certs" -# define X509_CERT_FILE OPENSSLDIR "/cert.pem" -# define X509_PRIVATE_DIR OPENSSLDIR "/private" -# define CTLOG_FILE OPENSSLDIR "/ct_log_list.cnf" -# else -# define X509_CERT_AREA "OSSL$DATAROOT:[000000]" -# define X509_CERT_DIR "OSSL$DATAROOT:[CERTS]" -# define X509_CERT_FILE "OSSL$DATAROOT:[000000]cert.pem" -# define X509_PRIVATE_DIR "OSSL$DATAROOT:[PRIVATE]" -# define CTLOG_FILE "OSSL$DATAROOT:[000000]ct_log_list.cnf" -# endif - -# define X509_CERT_DIR_EVP "SSL_CERT_DIR" -# define X509_CERT_FILE_EVP "SSL_CERT_FILE" -# define CTLOG_FILE_EVP "CTLOG_FILE" - -/* size of string representations */ -# define DECIMAL_SIZE(type) ((sizeof(type)*8+2)/3+1) -# define HEX_SIZE(type) (sizeof(type)*2) - -# define c2l(c,l) (l = ((unsigned long)(*((c)++))) , \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<<24)) - -/* NOTE - c is not incremented as per c2l */ -# define c2ln(c,l1,l2,n) { \ - c+=n; \ - l1=l2=0; \ - switch (n) { \ - case 8: l2 =((unsigned long)(*(--(c))))<<24; \ - case 7: l2|=((unsigned long)(*(--(c))))<<16; \ - case 6: l2|=((unsigned long)(*(--(c))))<< 8; \ - case 5: l2|=((unsigned long)(*(--(c)))); \ - case 4: l1 =((unsigned long)(*(--(c))))<<24; \ - case 3: l1|=((unsigned long)(*(--(c))))<<16; \ - case 2: l1|=((unsigned long)(*(--(c))))<< 8; \ - case 1: l1|=((unsigned long)(*(--(c)))); \ - } \ - } - -# define l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>>24)&0xff)) - -# define n2l(c,l) (l =((unsigned long)(*((c)++)))<<24, \ - l|=((unsigned long)(*((c)++)))<<16, \ - l|=((unsigned long)(*((c)++)))<< 8, \ - l|=((unsigned long)(*((c)++)))) - -# define n2l8(c,l) (l =((uint64_t)(*((c)++)))<<56, \ - l|=((uint64_t)(*((c)++)))<<48, \ - l|=((uint64_t)(*((c)++)))<<40, \ - l|=((uint64_t)(*((c)++)))<<32, \ - l|=((uint64_t)(*((c)++)))<<24, \ - l|=((uint64_t)(*((c)++)))<<16, \ - l|=((uint64_t)(*((c)++)))<< 8, \ - l|=((uint64_t)(*((c)++)))) - -# define l2n(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff)) - -# define l2n8(l,c) (*((c)++)=(unsigned char)(((l)>>56)&0xff), \ - *((c)++)=(unsigned char)(((l)>>48)&0xff), \ - *((c)++)=(unsigned char)(((l)>>40)&0xff), \ - *((c)++)=(unsigned char)(((l)>>32)&0xff), \ - *((c)++)=(unsigned char)(((l)>>24)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff)) - -/* NOTE - c is not incremented as per l2c */ -# define l2cn(l1,l2,c,n) { \ - c+=n; \ - switch (n) { \ - case 8: *(--(c))=(unsigned char)(((l2)>>24)&0xff); \ - case 7: *(--(c))=(unsigned char)(((l2)>>16)&0xff); \ - case 6: *(--(c))=(unsigned char)(((l2)>> 8)&0xff); \ - case 5: *(--(c))=(unsigned char)(((l2) )&0xff); \ - case 4: *(--(c))=(unsigned char)(((l1)>>24)&0xff); \ - case 3: *(--(c))=(unsigned char)(((l1)>>16)&0xff); \ - case 2: *(--(c))=(unsigned char)(((l1)>> 8)&0xff); \ - case 1: *(--(c))=(unsigned char)(((l1) )&0xff); \ - } \ - } - -# define n2s(c,s) ((s=(((unsigned int)((c)[0]))<< 8)| \ - (((unsigned int)((c)[1])) )),(c)+=2) -# define s2n(s,c) (((c)[0]=(unsigned char)(((s)>> 8)&0xff), \ - (c)[1]=(unsigned char)(((s) )&0xff)),(c)+=2) - -# define n2l3(c,l) ((l =(((unsigned long)((c)[0]))<<16)| \ - (((unsigned long)((c)[1]))<< 8)| \ - (((unsigned long)((c)[2])) )),(c)+=3) - -# define l2n3(l,c) (((c)[0]=(unsigned char)(((l)>>16)&0xff), \ - (c)[1]=(unsigned char)(((l)>> 8)&0xff), \ - (c)[2]=(unsigned char)(((l) )&0xff)),(c)+=3) - -static ossl_inline int ossl_ends_with_dirsep(const char *path) -{ - if (*path != '\0') - path += strlen(path) - 1; -# if defined __VMS - if (*path == ']' || *path == '>' || *path == ':') - return 1; -# elif defined _WIN32 - if (*path == '\\') - return 1; -# endif - return *path == '/'; -} - -static ossl_inline char ossl_determine_dirsep(const char *path) -{ - if (ossl_ends_with_dirsep(path)) - return '\0'; - -# if defined(_WIN32) - return '\\'; -# elif defined(__VMS) - return ':'; -# else - return '/'; -# endif -} - -static ossl_inline int ossl_is_absolute_path(const char *path) -{ -# if defined __VMS - if (strchr(path, ':') != NULL - || ((path[0] == '[' || path[0] == '<') - && path[1] != '.' && path[1] != '-' - && path[1] != ']' && path[1] != '>')) - return 1; -# elif defined _WIN32 - if (path[0] == '\\' - || (path[0] != '\0' && path[1] == ':')) - return 1; -# endif - return path[0] == '/'; -} - -#endif diff --git a/openssl/include/internal/comp.h b/openssl/include/internal/comp.h index c48c29d56..3ad86fc7b 100644 --- a/openssl/include/internal/comp.h +++ b/openssl/include/internal/comp.h @@ -10,5 +10,3 @@ #include void ossl_comp_zlib_cleanup(void); -void ossl_comp_brotli_cleanup(void); -void ossl_comp_zstd_cleanup(void); diff --git a/openssl/include/internal/constant_time.h b/openssl/include/internal/constant_time.h index 2b49afe1e..0ed6f823c 100644 --- a/openssl/include/internal/constant_time.h +++ b/openssl/include/internal/constant_time.h @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -140,29 +140,6 @@ static ossl_inline uint64_t constant_time_lt_64(uint64_t a, uint64_t b) return constant_time_msb_64(a ^ ((a ^ b) | ((a - b) ^ b))); } -#ifdef BN_ULONG -static ossl_inline BN_ULONG constant_time_msb_bn(BN_ULONG a) -{ - return 0 - (a >> (sizeof(a) * 8 - 1)); -} - -static ossl_inline BN_ULONG constant_time_lt_bn(BN_ULONG a, BN_ULONG b) -{ - return constant_time_msb_bn(a ^ ((a ^ b) | ((a - b) ^ b))); -} - -static ossl_inline BN_ULONG constant_time_is_zero_bn(BN_ULONG a) -{ - return constant_time_msb_bn(~a & (a - 1)); -} - -static ossl_inline BN_ULONG constant_time_eq_bn(BN_ULONG a, - BN_ULONG b) -{ - return constant_time_is_zero_bn(a ^ b); -} -#endif - static ossl_inline unsigned int constant_time_ge(unsigned int a, unsigned int b) { diff --git a/openssl/include/internal/core.h b/openssl/include/internal/core.h index 03adb66bd..c43e0d4f1 100644 --- a/openssl/include/internal/core.h +++ b/openssl/include/internal/core.h @@ -30,10 +30,6 @@ typedef struct ossl_method_construct_method_st { /* Get a temporary store */ void *(*get_tmp_store)(void *data); - /* Reserve the appropriate method store */ - int (*lock_store)(void *store, void *data); - /* Unreserve the appropriate method store */ - int (*unlock_store)(void *store, void *data); /* Get an already existing method from a store */ void *(*get)(void *store, const OSSL_PROVIDER **prov, void *data); /* Store a method in a store */ @@ -53,12 +49,10 @@ void *ossl_method_construct(OSSL_LIB_CTX *ctx, int operation_id, void ossl_algorithm_do_all(OSSL_LIB_CTX *libctx, int operation_id, OSSL_PROVIDER *provider, int (*pre)(OSSL_PROVIDER *, int operation_id, - int no_store, void *data, int *result), - int (*reserve_store)(int no_store, void *data), + void *data, int *result), void (*fn)(OSSL_PROVIDER *provider, const OSSL_ALGORITHM *algo, int no_store, void *data), - int (*unreserve_store)(void *data), int (*post)(OSSL_PROVIDER *, int operation_id, int no_store, void *data, int *result), void *data); diff --git a/openssl/include/internal/cryptlib.h b/openssl/include/internal/cryptlib.h index 64851fd8e..c836772cc 100644 --- a/openssl/include/internal/cryptlib.h +++ b/openssl/include/internal/cryptlib.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,6 +11,10 @@ # define OSSL_INTERNAL_CRYPTLIB_H # pragma once +# include +# include +#include + # ifdef OPENSSL_USE_APPLINK # define BIO_FLAGS_UPLINK_INTERNAL 0x8000 # include "ms/uplink.h" @@ -18,19 +22,61 @@ # define BIO_FLAGS_UPLINK_INTERNAL 0 # endif -# include "internal/common.h" - # include # include # include # include # include +# include "internal/nelem.h" + +#ifdef NDEBUG +# define ossl_assert(x) ((x) != 0) +#else +__owur static ossl_inline int ossl_assert_int(int expr, const char *exprstr, + const char *file, int line) +{ + if (!expr) + OPENSSL_die(exprstr, file, line); + + return expr; +} + +# define ossl_assert(x) ossl_assert_int((x) != 0, "Assertion failed: "#x, \ + __FILE__, __LINE__) + +#endif + +/* + * Use this inside a union with the field that needs to be aligned to a + * reasonable boundary for the platform. The most pessimistic alignment + * of the listed types will be used by the compiler. + */ +# define OSSL_UNION_ALIGN \ + double align; \ + ossl_uintmax_t align_int; \ + void *align_ptr typedef struct ex_callback_st EX_CALLBACK; DEFINE_STACK_OF(EX_CALLBACK) typedef struct mem_st MEM; -DEFINE_LHASH_OF_EX(MEM); +DEFINE_LHASH_OF(MEM); + +# define OPENSSL_CONF "openssl.cnf" + +# define X509_CERT_AREA OPENSSLDIR +# define X509_CERT_DIR OPENSSLDIR "/certs" +# define X509_CERT_FILE OPENSSLDIR "/cert.pem" +# define X509_PRIVATE_DIR OPENSSLDIR "/private" +# define CTLOG_FILE OPENSSLDIR "/ct_log_list.cnf" + +# define X509_CERT_DIR_EVP "SSL_CERT_DIR" +# define X509_CERT_FILE_EVP "SSL_CERT_FILE" +# define CTLOG_FILE_EVP "CTLOG_FILE" + +/* size of string representations */ +# define DECIMAL_SIZE(type) ((sizeof(type)*8+2)/3+1) +# define HEX_SIZE(type) (sizeof(type)*2) void OPENSSL_cpuid_setup(void); #if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ @@ -115,20 +161,35 @@ typedef struct ossl_ex_data_global_st { # define OSSL_LIB_CTX_PROVIDER_CONF_INDEX 16 # define OSSL_LIB_CTX_BIO_CORE_INDEX 17 # define OSSL_LIB_CTX_CHILD_PROVIDER_INDEX 18 -# define OSSL_LIB_CTX_THREAD_INDEX 19 -# define OSSL_LIB_CTX_DECODER_CACHE_INDEX 20 -# define OSSL_LIB_CTX_MAX_INDEXES 20 +# define OSSL_LIB_CTX_MAX_INDEXES 19 + +# define OSSL_LIB_CTX_METHOD_LOW_PRIORITY -1 +# define OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY 0 +# define OSSL_LIB_CTX_METHOD_PRIORITY_1 1 +# define OSSL_LIB_CTX_METHOD_PRIORITY_2 2 + +typedef struct ossl_lib_ctx_method { + int priority; + void *(*new_func)(OSSL_LIB_CTX *ctx); + void (*free_func)(void *); +} OSSL_LIB_CTX_METHOD; OSSL_LIB_CTX *ossl_lib_ctx_get_concrete(OSSL_LIB_CTX *ctx); int ossl_lib_ctx_is_default(OSSL_LIB_CTX *ctx); int ossl_lib_ctx_is_global_default(OSSL_LIB_CTX *ctx); /* Functions to retrieve pointers to data by index */ -void *ossl_lib_ctx_get_data(OSSL_LIB_CTX *, int /* index */); +void *ossl_lib_ctx_get_data(OSSL_LIB_CTX *, int /* index */, + const OSSL_LIB_CTX_METHOD * ctx); void ossl_lib_ctx_default_deinit(void); OSSL_EX_DATA_GLOBAL *ossl_lib_ctx_get_ex_data_global(OSSL_LIB_CTX *ctx); +typedef int (ossl_lib_ctx_run_once_fn)(OSSL_LIB_CTX *ctx); +typedef void (ossl_lib_ctx_onfree_fn)(OSSL_LIB_CTX *ctx); +int ossl_lib_ctx_run_once(OSSL_LIB_CTX *ctx, unsigned int idx, + ossl_lib_ctx_run_once_fn run_once_fn); +int ossl_lib_ctx_onfree(OSSL_LIB_CTX *ctx, ossl_lib_ctx_onfree_fn onfreefn); const char *ossl_lib_ctx_get_descriptor(OSSL_LIB_CTX *libctx); OSSL_LIB_CTX *ossl_crypto_ex_data_get_ossl_lib_ctx(const CRYPTO_EX_DATA *ad); @@ -160,4 +221,25 @@ char *ossl_buf2hexstr_sep(const unsigned char *buf, long buflen, char sep); unsigned char *ossl_hexstr2buf_sep(const char *str, long *buflen, const char sep); +static ossl_inline int ossl_ends_with_dirsep(const char *path) +{ + if (*path != '\0') + path += strlen(path) - 1; +# if defined _WIN32 + if (*path == '\\') + return 1; +# endif + return *path == '/'; +} + +static ossl_inline int ossl_is_absolute_path(const char *path) +{ +# if defined _WIN32 + if (path[0] == '\\' + || (path[0] != '\0' && path[1] == ':')) + return 1; +# endif + return path[0] == '/'; +} + #endif diff --git a/openssl/include/internal/der.h b/openssl/include/internal/der.h index 8d6db8f06..a3e56d0dc 100644 --- a/openssl/include/internal/der.h +++ b/openssl/include/internal/der.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,12 +7,8 @@ * https://www.openssl.org/source/license.html */ -#ifndef OSSL_INTERNAL_DER_H -# define OSSL_INTERNAL_DER_H -# pragma once - -# include -# include "internal/packet.h" +#include +#include "internal/packet.h" /* * NOTE: X.690 numbers the identifier octet bits 1 to 8. @@ -26,42 +22,42 @@ * These are only valid for the UNIVERSAL class. With the other classes, * these bits have a different meaning. */ -# define DER_P_EOC 0 /* BER End Of Contents tag */ -# define DER_P_BOOLEAN 1 -# define DER_P_INTEGER 2 -# define DER_P_BIT_STRING 3 -# define DER_P_OCTET_STRING 4 -# define DER_P_NULL 5 -# define DER_P_OBJECT 6 -# define DER_P_OBJECT_DESCRIPTOR 7 -# define DER_P_EXTERNAL 8 -# define DER_P_REAL 9 -# define DER_P_ENUMERATED 10 -# define DER_P_UTF8STRING 12 -# define DER_P_SEQUENCE 16 -# define DER_P_SET 17 -# define DER_P_NUMERICSTRING 18 -# define DER_P_PRINTABLESTRING 19 -# define DER_P_T61STRING 20 -# define DER_P_VIDEOTEXSTRING 21 -# define DER_P_IA5STRING 22 -# define DER_P_UTCTIME 23 -# define DER_P_GENERALIZEDTIME 24 -# define DER_P_GRAPHICSTRING 25 -# define DER_P_ISO64STRING 26 -# define DER_P_GENERALSTRING 27 -# define DER_P_UNIVERSALSTRING 28 -# define DER_P_BMPSTRING 30 +#define DER_P_EOC 0 /* BER End Of Contents tag */ +#define DER_P_BOOLEAN 1 +#define DER_P_INTEGER 2 +#define DER_P_BIT_STRING 3 +#define DER_P_OCTET_STRING 4 +#define DER_P_NULL 5 +#define DER_P_OBJECT 6 +#define DER_P_OBJECT_DESCRIPTOR 7 +#define DER_P_EXTERNAL 8 +#define DER_P_REAL 9 +#define DER_P_ENUMERATED 10 +#define DER_P_UTF8STRING 12 +#define DER_P_SEQUENCE 16 +#define DER_P_SET 17 +#define DER_P_NUMERICSTRING 18 +#define DER_P_PRINTABLESTRING 19 +#define DER_P_T61STRING 20 +#define DER_P_VIDEOTEXSTRING 21 +#define DER_P_IA5STRING 22 +#define DER_P_UTCTIME 23 +#define DER_P_GENERALIZEDTIME 24 +#define DER_P_GRAPHICSTRING 25 +#define DER_P_ISO64STRING 26 +#define DER_P_GENERALSTRING 27 +#define DER_P_UNIVERSALSTRING 28 +#define DER_P_BMPSTRING 30 /* DER Flags, occupying bit 6 in the DER identifier byte */ -# define DER_F_PRIMITIVE 0x00 -# define DER_F_CONSTRUCTED 0x20 +#define DER_F_PRIMITIVE 0x00 +#define DER_F_CONSTRUCTED 0x20 /* DER classes tags, occupying bits 7-8 in the DER identifier byte */ -# define DER_C_UNIVERSAL 0x00 -# define DER_C_APPLICATION 0x40 -# define DER_C_CONTEXT 0x80 -# define DER_C_PRIVATE 0xC0 +#define DER_C_UNIVERSAL 0x00 +#define DER_C_APPLICATION 0x40 +#define DER_C_CONTEXT 0x80 +#define DER_C_PRIVATE 0xC0 /* * Run-time constructors. @@ -71,14 +67,14 @@ */ /* This can be used for all items that don't have a context */ -# define DER_NO_CONTEXT -1 +#define DER_NO_CONTEXT -1 int ossl_DER_w_precompiled(WPACKET *pkt, int tag, const unsigned char *precompiled, size_t precompiled_n); int ossl_DER_w_boolean(WPACKET *pkt, int tag, int b); -int ossl_DER_w_uint32(WPACKET *pkt, int tag, uint32_t v); +int ossl_DER_w_ulong(WPACKET *pkt, int tag, unsigned long v); int ossl_DER_w_bn(WPACKET *pkt, int tag, const BIGNUM *v); int ossl_DER_w_null(WPACKET *pkt, int tag); int ossl_DER_w_octet_string(WPACKET *pkt, int tag, @@ -90,5 +86,3 @@ int ossl_DER_w_octet_string_uint32(WPACKET *pkt, int tag, uint32_t value); */ int ossl_DER_w_begin_sequence(WPACKET *pkt, int tag); int ossl_DER_w_end_sequence(WPACKET *pkt, int tag); - -#endif diff --git a/openssl/include/internal/deterministic_nonce.h b/openssl/include/internal/deterministic_nonce.h deleted file mode 100644 index 5f0313fe3..000000000 --- a/openssl/include/internal/deterministic_nonce.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_DETERMINISTIC_NONCE_H -# define OSSL_INTERNAL_DETERMINISTIC_NONCE_H -# pragma once - -# include - -int ossl_gen_deterministic_nonce_rfc6979(BIGNUM *out, const BIGNUM *q, - const BIGNUM *priv, - const unsigned char *message, - size_t message_len, - const char *digestname, - OSSL_LIB_CTX *libctx, - const char *propq); - -#endif /*OSSL_INTERNAL_DETERMINISTIC_NONCE_H */ diff --git a/openssl/include/internal/e_os.h b/openssl/include/internal/e_os.h index 003d63e17..0f2b9093f 100644 --- a/openssl/include/internal/e_os.h +++ b/openssl/include/internal/e_os.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,7 +15,7 @@ # include # include -# include "internal/numbers.h" /* Ensure the definition of SIZE_MAX */ +# include "internal/nelem.h" /* * contains what we can justify to make visible to the @@ -143,7 +143,7 @@ static __inline unsigned int _strlen31(const char *str) # undef stdin # undef stdout # undef stderr -FILE *__iob_func(void); +FILE *__iob_func(); # define stdin (&__iob_func()[0]) # define stdout (&__iob_func()[1]) # define stderr (&__iob_func()[2]) @@ -188,68 +188,22 @@ FILE *__iob_func(void); # include # endif -# ifdef OPENSSL_SYS_VMS -# define VMS 1 - /* - * some programs don't include stdlib, so exit() and others give implicit - * function warnings - */ -# include -# if defined(__DECC) -# include -# else -# include -# endif -# define LIST_SEPARATOR_CHAR ',' - /* We don't have any well-defined random devices on VMS, yet... */ -# undef DEVRANDOM - /*- - We need to do this since VMS has the following coding on status codes: - - Bits 0-2: status type: 0 = warning, 1 = success, 2 = error, 3 = info ... - The important thing to know is that odd numbers are considered - good, while even ones are considered errors. - Bits 3-15: actual status number - Bits 16-27: facility number. 0 is considered "unknown" - Bits 28-31: control bits. If bit 28 is set, the shell won't try to - output the message (which, for random codes, just looks ugly) - - So, what we do here is to change 0 to 1 to get the default success status, - and everything else is shifted up to fit into the status number field, and - the status is tagged as an error, which is what is wanted here. - - Finally, we add the VMS C facility code 0x35a000, because there are some - programs, such as Perl, that will reinterpret the code back to something - POSIX. 'man perlvms' explains it further. - - NOTE: the perlvms manual wants to turn all codes 2 to 255 into success - codes (status type = 1). I couldn't disagree more. Fortunately, the - status type doesn't seem to bother Perl. - -- Richard Levitte - */ -# define EXIT(n) exit((n) ? (((n) << 3) | 2 | 0x10000000 | 0x35a000) : 1) - -# define DEFAULT_HOME "SYS$LOGIN:" - -# else - /* !defined VMS */ -# include -# include -# ifdef OPENSSL_SYS_WIN32_CYGWIN -# include -# include -# endif - -# define LIST_SEPARATOR_CHAR ':' -# define EXIT(n) exit(n) +# include +# include +# ifdef OPENSSL_SYS_WIN32_CYGWIN +# include +# include # endif +# define LIST_SEPARATOR_CHAR ':' +# define EXIT(n) exit(n) + # endif /***********************************************/ # if defined(OPENSSL_SYS_WINDOWS) -# if defined(_MSC_VER) && (_MSC_VER >= 1310) && !defined(_WIN32_WCE) +# if (_MSC_VER >= 1310) && !defined(_WIN32_WCE) # define open _open # define fdopen _fdopen # define close _close @@ -258,7 +212,6 @@ FILE *__iob_func(void); # endif # define unlink _unlink # define fileno _fileno -# define isatty _isatty # endif # else # include @@ -287,6 +240,54 @@ struct servent *getservbyname(const char *name, const char *proto); # endif /* end vxworks */ +/* system-specific variants defining ossl_sleep() */ +#ifdef OPENSSL_SYS_UNIX +# include +static ossl_inline void ossl_sleep(unsigned long millis) +{ +# ifdef OPENSSL_SYS_VXWORKS + struct timespec ts; + ts.tv_sec = (long int) (millis / 1000); + ts.tv_nsec = (long int) (millis % 1000) * 1000000ul; + nanosleep(&ts, NULL); +# elif defined(__TANDEM) +# if !defined(_REENTRANT) +# include + /* HPNS does not support usleep for non threaded apps */ + PROCESS_DELAY_(millis * 1000); +# elif defined(_SPT_MODEL_) +# include +# include + usleep(millis * 1000); +# else + usleep(millis * 1000); +# endif +# else + usleep(millis * 1000); +# endif +} +#elif defined(_WIN32) +# include +static ossl_inline void ossl_sleep(unsigned long millis) +{ + Sleep(millis); +} +#else +/* Fallback to a busy wait */ +static ossl_inline void ossl_sleep(unsigned long millis) +{ + struct timeval start, now; + unsigned long elapsedms; + + gettimeofday(&start, NULL); + do { + gettimeofday(&now, NULL); + elapsedms = (((now.tv_sec - start.tv_sec) * 1000000) + + now.tv_usec - start.tv_usec) / 1000; + } while (elapsedms < millis); +} +#endif /* defined OPENSSL_SYS_UNIX */ + /* ----------------------------- HP NonStop -------------------------------- */ /* Required to support platform variant without getpid() and pid_t. */ # if defined(__TANDEM) && defined(_GUARDIAN_TARGET) @@ -296,12 +297,12 @@ struct servent *getservbyname(const char *name, const char *proto); # define gethostbyname(name) gethostbyname((char*)name) # define ioctlsocket(a,b,c) ioctl(a,b,c) # ifdef NO_GETPID -inline int nssgetpid(void); +inline int nssgetpid(); # ifndef NSSGETPID_MACRO # define NSSGETPID_MACRO # include # include - inline int nssgetpid(void) + inline int nssgetpid() { short phandle[10]={0}; union pseudo_pid { @@ -353,7 +354,7 @@ inline int nssgetpid(void); /* unistd.h defines _POSIX_VERSION */ # if (defined(OPENSSL_SYS_UNIX) \ && ( (defined(_POSIX_VERSION) && _POSIX_VERSION >= 200112L) \ - || defined(__sun) || defined(__hpux) || defined(__sgi) \ + || defined(__sun) || defined(__sgi) \ || defined(__osf__) )) \ || defined(_WIN32) /* secure memory is implemented */ diff --git a/openssl/include/internal/endian.h b/openssl/include/internal/endian.h index 7d5a73b1b..8b34e03e4 100644 --- a/openssl/include/internal/endian.h +++ b/openssl/include/internal/endian.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,7 @@ # pragma once /* - * IS_LITTLE_ENDIAN and IS_BIG_ENDIAN can be used to detect the endianness + * IS_LITTLE_ENDIAN and IS_BIG_ENDIAN can be used to detect the endiannes * at compile time. To use it, DECLARE_IS_ENDIAN must be used to declare * a variable. * diff --git a/openssl/include/internal/event_queue.h b/openssl/include/internal/event_queue.h deleted file mode 100644 index bda1ee6ad..000000000 --- a/openssl/include/internal/event_queue.h +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_EVENT_QUEUE_H -# define OSSL_INTERNAL_EVENT_QUEUE_H -# pragma once - -# include "internal/priority_queue.h" -# include "internal/time.h" - -/* - * Opaque type holding an event. - */ -typedef struct ossl_event_st OSSL_EVENT; - -DEFINE_PRIORITY_QUEUE_OF(OSSL_EVENT); - -/* - * Public type representing an event queue, the underlying structure being - * opaque. - */ -typedef struct ossl_event_queue_st OSSL_EVENT_QUEUE; - -/* - * Public type representing a event queue entry. - * It is (internally) public so that it can be embedded into other structures, - * it should otherwise be treated as opaque. - */ -struct ossl_event_st { - uint32_t type; /* What type of event this is */ - uint32_t priority; /* What priority this event has */ - OSSL_TIME when; /* When the event is scheduled to happen */ - void *ctx; /* User argument passed to call backs */ - void *payload; /* Event specific data of unknown kind */ - size_t payload_size; /* Length (in bytes) of event specific data */ - - /* These fields are for internal use only */ - PRIORITY_QUEUE_OF(OSSL_EVENT) *queue; /* Queue containing this event */ - size_t ref; /* ID for this event */ - unsigned int flag_dynamic : 1; /* Malloced or not? */ -}; - -/* - * Utility function to populate an event structure and add it to the queue - */ -int ossl_event_queue_add(OSSL_EVENT_QUEUE *queue, OSSL_EVENT *event, - uint32_t type, uint32_t priority, - OSSL_TIME when, void *ctx, - void *payload, size_t payload_size); - -/* - * Utility functions to extract event fields - */ -static ossl_unused ossl_inline -uint32_t ossl_event_get_type(const OSSL_EVENT *event) -{ - return event->type; -} - -static ossl_unused ossl_inline -uint32_t ossl_event_get_priority(const OSSL_EVENT *event) -{ - return event->priority; -} - -static ossl_unused ossl_inline -OSSL_TIME ossl_event_get_when(const OSSL_EVENT *event) -{ - return event->when; -} - -static ossl_unused ossl_inline -void *ossl_event_get0_ctx(const OSSL_EVENT *event) -{ - return event->ctx; -} - -static ossl_unused ossl_inline -void *ossl_event_get0_payload(const OSSL_EVENT *event, size_t *length) -{ - if (length != NULL) - *length = event->payload_size; - return event->payload; -} - -/* - * Create and free a queue. - */ -OSSL_EVENT_QUEUE *ossl_event_queue_new(void); -void ossl_event_queue_free(OSSL_EVENT_QUEUE *queue); - -/* - * Schedule a new event into an event queue. - * - * The event parameters are taken from the function arguments. - * - * The function returns NULL on failure and the added event on success. - */ -OSSL_EVENT *ossl_event_queue_add_new(OSSL_EVENT_QUEUE *queue, - uint32_t type, uint32_t priority, - OSSL_TIME when, void *ctx, - void *payload, size_t payload_size) -; - -/* - * Schedule an event into an event queue. - * - * The event parameters are taken from the function arguments. - * - * The function returns 0 on failure and 1 on success. - */ -int ossl_event_queue_add(OSSL_EVENT_QUEUE *queue, OSSL_EVENT *event, - uint32_t type, uint32_t priority, - OSSL_TIME when, void *ctx, - void *payload, size_t payload_size); - -/* - * Delete an event from the queue. - * This will cause the early deletion function to be called if it is non-NULL. - * A pointer to the event structure is returned. - */ -int ossl_event_queue_remove(OSSL_EVENT_QUEUE *queue, OSSL_EVENT *event); - -/* - * Free a dynamic event. - * Is a NOP for a static event. - */ -void ossl_event_free(OSSL_EVENT *event); - -/* - * Return the time until the next event for the specified event, if the event's - * time is past, zero is returned. Once activated, the event reference becomes - * invalid and this function becomes undefined. - */ -OSSL_TIME ossl_event_time_until(const OSSL_EVENT *event); - -/* - * Return the time until the next event in the queue. - * If the next event is in the past, zero is returned. - */ -OSSL_TIME ossl_event_queue_time_until_next(const OSSL_EVENT_QUEUE *queue); - -/* - * Postpone an event to trigger at the specified time. - * If the event has triggered, this function's behaviour is undefined. - */ -int ossl_event_queue_postpone_until(OSSL_EVENT_QUEUE *queue, - OSSL_EVENT *event, - OSSL_TIME when); - -/* - * Return the next event to process. - */ -int ossl_event_queue_get1_next_event(OSSL_EVENT_QUEUE *queue, - OSSL_EVENT **event); - -#endif diff --git a/openssl/include/internal/ffc.h b/openssl/include/internal/ffc.h index edd8381e8..3e8f65509 100644 --- a/openssl/include/internal/ffc.h +++ b/openssl/include/internal/ffc.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -118,8 +118,6 @@ typedef struct ffc_params_st { */ const char *mdname; const char *mdprops; - /* Default key length for known named groups according to RFC7919 */ - int keylength; } FFC_PARAMS; void ossl_ffc_params_init(FFC_PARAMS *params); @@ -137,7 +135,7 @@ void ossl_ffc_params_set_h(FFC_PARAMS *params, int index); void ossl_ffc_params_set_flags(FFC_PARAMS *params, unsigned int flags); void ossl_ffc_params_enable_flags(FFC_PARAMS *params, unsigned int flags, int enable); -void ossl_ffc_set_digest(FFC_PARAMS *params, const char *alg, const char *props); +int ossl_ffc_set_digest(FFC_PARAMS *params, const char *alg, const char *props); int ossl_ffc_params_set_validate_params(FFC_PARAMS *params, const unsigned char *seed, @@ -213,9 +211,8 @@ const DH_NAMED_GROUP *ossl_ffc_numbers_to_dh_named_group(const BIGNUM *p, int ossl_ffc_named_group_get_uid(const DH_NAMED_GROUP *group); const char *ossl_ffc_named_group_get_name(const DH_NAMED_GROUP *); #ifndef OPENSSL_NO_DH -int ossl_ffc_named_group_get_keylength(const DH_NAMED_GROUP *group); const BIGNUM *ossl_ffc_named_group_get_q(const DH_NAMED_GROUP *group); -int ossl_ffc_named_group_set(FFC_PARAMS *ffc, const DH_NAMED_GROUP *group); +int ossl_ffc_named_group_set_pqg(FFC_PARAMS *ffc, const DH_NAMED_GROUP *group); #endif #endif /* OSSL_INTERNAL_FFC_H */ diff --git a/openssl/include/internal/hpke_util.h b/openssl/include/internal/hpke_util.h deleted file mode 100644 index e1da5e059..000000000 --- a/openssl/include/internal/hpke_util.h +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_HPKE_UTIL_H -# define OSSL_INTERNAL_HPKE_UTIL_H -# pragma once - -/* Constants from RFC 9180 Section 7.1 and 7.3 */ -# define OSSL_HPKE_MAX_SECRET 64 -# define OSSL_HPKE_MAX_PUBLIC 133 -# define OSSL_HPKE_MAX_PRIVATE 66 -# define OSSL_HPKE_MAX_KDF_INPUTLEN 64 - -/* - * max length of a base-nonce (the Nn field from OSSL_HPKE_AEAD_INFO), this - * is used for a local stack array size - */ -# define OSSL_HPKE_MAX_NONCELEN 12 - -/* - * @brief info about a KEM - * Used to store constants from Section 7.1 "Table 2 KEM IDs" - * and the bitmask for EC curves described in Section 7.1.3 DeriveKeyPair - */ -typedef struct { - uint16_t kem_id; /* code point for key encipherment method */ - const char *keytype; /* string form of algtype "EC"/"X25519"/"X448" */ - const char *groupname; /* string form of EC group for NIST curves */ - const char *mdname; /* hash alg name for the HKDF */ - size_t Nsecret; /* size of secrets */ - size_t Nenc; /* length of encapsulated key */ - size_t Npk; /* length of public key */ - size_t Nsk; /* length of raw private key */ - uint8_t bitmask; -} OSSL_HPKE_KEM_INFO; - -/* - * @brief info about a KDF - */ -typedef struct { - uint16_t kdf_id; /* code point for KDF */ - const char *mdname; /* hash alg name for the HKDF */ - size_t Nh; /* length of hash/extract output */ -} OSSL_HPKE_KDF_INFO; - -/* - * @brief info about an AEAD - */ -typedef struct { - uint16_t aead_id; /* code point for aead alg */ - const char *name; /* alg name */ - size_t taglen; /* aead tag len */ - size_t Nk; /* size of a key for this aead */ - size_t Nn; /* length of a nonce for this aead */ -} OSSL_HPKE_AEAD_INFO; - -const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_curve(const char *curve); -const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_id(uint16_t kemid); -const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_random(OSSL_LIB_CTX *ctx); -const OSSL_HPKE_KDF_INFO *ossl_HPKE_KDF_INFO_find_id(uint16_t kdfid); -const OSSL_HPKE_KDF_INFO *ossl_HPKE_KDF_INFO_find_random(OSSL_LIB_CTX *ctx); -const OSSL_HPKE_AEAD_INFO *ossl_HPKE_AEAD_INFO_find_id(uint16_t aeadid); -const OSSL_HPKE_AEAD_INFO *ossl_HPKE_AEAD_INFO_find_random(OSSL_LIB_CTX *ctx); - -int ossl_hpke_kdf_extract(EVP_KDF_CTX *kctx, - unsigned char *prk, size_t prklen, - const unsigned char *salt, size_t saltlen, - const unsigned char *ikm, size_t ikmlen); - -int ossl_hpke_kdf_expand(EVP_KDF_CTX *kctx, - unsigned char *okm, size_t okmlen, - const unsigned char *prk, size_t prklen, - const unsigned char *info, size_t infolen); - -int ossl_hpke_labeled_extract(EVP_KDF_CTX *kctx, - unsigned char *prk, size_t prklen, - const unsigned char *salt, size_t saltlen, - const char *protocol_label, - const unsigned char *suiteid, size_t suiteidlen, - const char *label, - const unsigned char *ikm, size_t ikmlen); -int ossl_hpke_labeled_expand(EVP_KDF_CTX *kctx, - unsigned char *okm, size_t okmlen, - const unsigned char *prk, size_t prklen, - const char *protocol_label, - const unsigned char *suiteid, size_t suiteidlen, - const char *label, - const unsigned char *info, size_t infolen); - -EVP_KDF_CTX *ossl_kdf_ctx_create(const char *kdfname, const char *mdname, - OSSL_LIB_CTX *libctx, const char *propq); - -int ossl_hpke_str2suite(const char *suitestr, OSSL_HPKE_SUITE *suite); -#endif diff --git a/openssl/include/internal/json_enc.h b/openssl/include/internal/json_enc.h deleted file mode 100644 index e7d9a6d92..000000000 --- a/openssl/include/internal/json_enc.h +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_JSON_ENC_H -# define OSSL_JSON_ENC_H - -# include - -/* - * JSON Encoder - * ============ - * - * This JSON encoder is used for qlog. It supports ordinary JSON (RFC 7159), - * JSON-SEQ (RFC 7464) and I-JSON (RFC 7493). It supports only basic ASCII. - */ - -struct json_write_buf { - BIO *bio; - char *buf; - size_t alloc, cur; -}; - -typedef struct ossl_json_enc_st { - uint32_t flags; - /* error: 1 if an error has occurred. */ - /* state: current state. */ - /* stack stores a bitmap. 0=object, 1=array. */ - /* stack cur size: stack_end_byte bytes, stack_end_bit bits. */ - /* stack alloc size: stack_bytes bytes. */ - unsigned char error, stack_end_bit, state, *stack, defer_indent; - unsigned char stack_small[16]; - struct json_write_buf wbuf; - size_t stack_end_byte, stack_bytes; -} OSSL_JSON_ENC; - -/* - * ossl_json_init - * -------------- - * - * Initialises a JSON encoder. - * - * If the flag OSSL_JSON_FLAG_SEQ is passed, the output is in JSON-SEQ. The - * caller should use the encoder as though it is encoding members of a JSON - * array (but without calling ossl_json_array_begin() or ossl_json_array_end()). - * Each top-level JSON item (e.g. JSON object) encoded will be separated - * correctly as per the JSON-SEQ format. - * - * If the flag OSSL_JSON_FLAG_SEQ is not passed, the output is in JSON format. - * Generally the caller should encode only a single output item (e.g. a JSON - * object). - * - * By default, JSON output is maximally compact. If OSSL_JSON_FLAG_PRETTY is - * set, JSON/JSON-SEQ output is spaced for optimal human readability. - * - * If OSSL_JSON_FLAG_IJSON is set, integers outside the range `[-2**53 + 1, - * 2**53 - 1]` are automatically converted to decimal strings before - * serialization. - */ -#define OSSL_JSON_FLAG_NONE 0 -#define OSSL_JSON_FLAG_SEQ (1U << 0) -#define OSSL_JSON_FLAG_PRETTY (1U << 1) -#define OSSL_JSON_FLAG_IJSON (1U << 2) - -int ossl_json_init(OSSL_JSON_ENC *json, BIO *bio, uint32_t flags); - -/* - * ossl_json_cleanup - * ----------------- - * - * Destroys a JSON encoder. - */ -void ossl_json_cleanup(OSSL_JSON_ENC *json); - -/* - * ossl_json_reset - * --------------- - * - * Resets a JSON encoder, as though it has just been initialised, allowing it - * to be used again for new output syntactically unrelated to any previous - * output. This is similar to calling ossl_json_cleanup followed by - * ossl_json_init but may allow internal buffers to be reused. - * - * If the JSON encoder has entered an error state, this function MAY allow - * recovery from this error state, in which case it will return 1. If this - * function returns 0, the JSON encoder is unrecoverable and - * ossl_json_cleanup() must be called. - * - * Automatically calls ossl_json_flush(). - */ -int ossl_json_reset(OSSL_JSON_ENC *json); - -/* - * ossl_json_flush - * --------------- - * - * Flushes the JSON encoder, ensuring that any residual bytes in internal - * buffers are written to the provided sink BIO. Flushing may also happen - * autonomously as buffers are filled, but the caller must use this function - * to guarantee all data has been flushed. - */ -int ossl_json_flush(OSSL_JSON_ENC *json); - -/* - * ossl_json_flush_cleanup - * ----------------------- - * - * Tries to flush as in a call to ossl_json_flush, and then calls - * ossl_json_cleanup regardless of the result. The result of the flush call is - * returned. - */ -int ossl_json_flush_cleanup(OSSL_JSON_ENC *json); - -/* - * ossl_json_set0_sink - * ------------------- - * - * Changes the sink used by the JSON encoder. - */ -int ossl_json_set0_sink(OSSL_JSON_ENC *json, BIO *bio); - -/* - * ossl_json_in_error - * ------------------ - * - * To enhance the ergonomics of the JSON API, the JSON object uses an implicit - * error tracking model. When a JSON API call fails (for example due to caller - * error, such as trying to close an array which was not opened), the JSON - * object enters an error state and all further calls are silently ignored. - * - * The caller can detect this condition after it is finished making builder - * calls to the JSON object by calling this function. This function returns 1 - * if an error occurred. At this point the caller's only recourse is to call - * ossl_json_reset() or ossl_json_cleanup(). - * - * Note that partial (i.e., invalid) output may still have been sent to the BIO - * in this case. Since the amount of output which can potentially be produced - * by a JSON object is unbounded, it is impractical to buffer it all before - * flushing. It is expected that errors will ordinarily be either caller errors - * (programming errors) or BIO errors. - */ -int ossl_json_in_error(OSSL_JSON_ENC *json); - -/* - * JSON Builder Calls - * ================== - * - * These functions are used to build JSON output. The functions which have - * begin and end function pairs must be called in correctly nested sequence. - * When writing an object, ossl_json_key() must be called exactly once before - * each call to write a JSON item. - * - * The JSON library takes responsibility for enforcing correct usage patterns. - * If a call is made that does not correspond to the JSON syntax, the JSON - * object enters the error state and all subsequent calls are ignored. - * - * In JSON-SEQ mode, the caller should act as though the library implicitly - * places all calls between an ossl_json_array_begin() and - * ossl_json_array_end() pair; for example, the normal usage pattern would be - * to call ossl_json_object_begin() followed by ossl_json_object_end(), in - * repeated sequence. - * - * The library does not enforce non-generation of duplicate keys. Avoiding this - * is the caller's responsibility. It is also the caller's responsibility to - * pass valid UTF-8 strings. All other forms of invalid output will cause an - * error. Note that due to the immediate nature of the API, partial output may - * have already been generated in such a case. - */ - -/* Begin a new JSON object. */ -void ossl_json_object_begin(OSSL_JSON_ENC *json); - -/* End a JSON object. Must be matched with a call to ossl_json_object_begin(). */ -void ossl_json_object_end(OSSL_JSON_ENC *json); - -/* Begin a new JSON array. */ -void ossl_json_array_begin(OSSL_JSON_ENC *json); - -/* End a JSON array. Must be matched with a call to ossl_json_array_end(). */ -void ossl_json_array_end(OSSL_JSON_ENC *json); - -/* - * Encode a JSON key within an object. Pass a zero-terminated string, which can - * be freed immediately following the call to this function. - */ -void ossl_json_key(OSSL_JSON_ENC *json, const char *key); - -/* Encode a JSON 'null' value. */ -void ossl_json_null(OSSL_JSON_ENC *json); - -/* Encode a JSON boolean value. */ -void ossl_json_bool(OSSL_JSON_ENC *json, int value); - -/* Encode a JSON integer from a uint64_t. */ -void ossl_json_u64(OSSL_JSON_ENC *json, uint64_t value); - -/* Encode a JSON integer from an int64_t. */ -void ossl_json_i64(OSSL_JSON_ENC *json, int64_t value); - -/* Encode a JSON number from a 64-bit floating point value. */ -void ossl_json_f64(OSSL_JSON_ENC *json, double value); - -/* - * Encode a JSON UTF-8 string from a zero-terminated string. The string passed - * can be freed immediately following the call to this function. - */ -void ossl_json_str(OSSL_JSON_ENC *json, const char *str); - -/* - * Encode a JSON UTF-8 string from a string with the given length. The string - * passed can be freed immediately following the call to this function. - */ -void ossl_json_str_len(OSSL_JSON_ENC *json, const char *str, size_t str_len); - -/* - * Encode binary data as a lowercase hex string. data_len is the data length in - * bytes. - */ -void ossl_json_str_hex(OSSL_JSON_ENC *json, const void *data, size_t data_len); - -#endif diff --git a/openssl/include/internal/ktls.h b/openssl/include/internal/ktls.h index 072653dc5..95492fd06 100644 --- a/openssl/include/internal/ktls.h +++ b/openssl/include/internal/ktls.h @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -40,11 +40,6 @@ # define OPENSSL_KTLS_AES_GCM_128 # define OPENSSL_KTLS_AES_GCM_256 # define OPENSSL_KTLS_TLS13 -# ifdef TLS_CHACHA20_IV_LEN -# ifndef OPENSSL_NO_CHACHA -# define OPENSSL_KTLS_CHACHA20_POLY1305 -# endif -# endif typedef struct tls_enable ktls_crypto_info_t; @@ -80,12 +75,6 @@ static ossl_inline int ktls_start(int fd, ktls_crypto_info_t *tls_en, int is_tx) # endif } -/* Not supported on FreeBSD */ -static ossl_inline int ktls_enable_tx_zerocopy_sendfile(int fd) -{ - return 0; -} - /* * Send a TLS record using the tls_en provided in ktls_start and use * record_type instead of the default SSL3_RT_APPLICATION_DATA. @@ -220,13 +209,6 @@ static ossl_inline ossl_ssize_t ktls_sendfile(int s, int fd, off_t off, # warning "Skipping Compilation of KTLS receive data path" # endif # endif -# if LINUX_VERSION_CODE < KERNEL_VERSION(5, 19, 0) -# define OPENSSL_NO_KTLS_ZC_TX -# ifndef PEDANTIC -# warning "KTLS requires Kernel Headers >= 5.19.0 for zerocopy sendfile" -# warning "Skipping Compilation of KTLS zerocopy sendfile" -# endif -# endif # define OPENSSL_KTLS_AES_GCM_128 # if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0) # define OPENSSL_KTLS_AES_GCM_256 @@ -306,18 +288,6 @@ static ossl_inline int ktls_start(int fd, ktls_crypto_info_t *crypto_info, crypto_info, crypto_info->tls_crypto_info_len) ? 0 : 1; } -static ossl_inline int ktls_enable_tx_zerocopy_sendfile(int fd) -{ -#ifndef OPENSSL_NO_KTLS_ZC_TX - int enable = 1; - - return setsockopt(fd, SOL_TLS, TLS_TX_ZEROCOPY_RO, - &enable, sizeof(enable)) ? 0 : 1; -#else - return 0; -#endif -} - /* * Send a TLS record using the crypto_info provided in ktls_start and use * record_type instead of the default SSL3_RT_APPLICATION_DATA. diff --git a/openssl/include/internal/list.h b/openssl/include/internal/list.h deleted file mode 100644 index 902047641..000000000 --- a/openssl/include/internal/list.h +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_LIST_H -# define OSSL_INTERNAL_LIST_H -# pragma once - -# include -# include - -# ifdef NDEBUG -# define OSSL_LIST_DBG(x) -# else -# define OSSL_LIST_DBG(x) x; -# endif - -# define LIST_FOREACH_FROM(p, name, init) \ - for ((p) = (init); \ - (p) != NULL; \ - (p) = ossl_list_##name##_next(p)) -# define LIST_FOREACH(p, name, l) \ - LIST_FOREACH_FROM(p, name, ossl_list_##name##_head(l)) - -# define LIST_FOREACH_REV_FROM(p, name, init) \ - for ((p) = (init); \ - (p) != NULL; \ - (p) = ossl_list_##name##_prev(p)) -# define LIST_FOREACH_REV(p, name, l) \ - LIST_FOREACH_FROM(p, name, ossl_list_##name##_tail(l)) - -# define LIST_FOREACH_DELSAFE_FROM(p, pn, name, init) \ - for ((p) = (init); \ - (p) != NULL && (((pn) = ossl_list_##name##_next(p)), 1); \ - (p) = (pn)) -#define LIST_FOREACH_DELSAFE(p, pn, name, l) \ - LIST_FOREACH_DELSAFE_FROM(p, pn, name, ossl_list_##name##_head(l)) - -# define LIST_FOREACH_REV_DELSAFE_FROM(p, pn, name, init) \ - for ((p) = (init); \ - (p) != NULL && (((pn) = ossl_list_##name##_prev(p)), 1); \ - (p) = (pn)) -# define LIST_FOREACH_REV_DELSAFE(p, pn, name, l) \ - LIST_FOREACH_REV_DELSAFE_FROM(p, pn, name, ossl_list_##name##_tail(l)) - -/* Define a list structure */ -# define OSSL_LIST(name) OSSL_LIST_ ## name - -/* Define fields to include an element of a list */ -# define OSSL_LIST_MEMBER(name, type) \ - struct { \ - type *next, *prev; \ - OSSL_LIST_DBG(struct ossl_list_st_ ## name *list) \ - } ossl_list_ ## name - -# define DECLARE_LIST_OF(name, type) \ - typedef struct ossl_list_st_ ## name OSSL_LIST(name); \ - struct ossl_list_st_ ## name { \ - type *alpha, *omega; \ - size_t num_elems; \ - } \ - -# define DEFINE_LIST_OF_IMPL(name, type) \ - static ossl_unused ossl_inline void \ - ossl_list_##name##_init(OSSL_LIST(name) *list) \ - { \ - memset(list, 0, sizeof(*list)); \ - } \ - static ossl_unused ossl_inline void \ - ossl_list_##name##_init_elem(type *elem) \ - { \ - memset(&elem->ossl_list_ ## name, 0, \ - sizeof(elem->ossl_list_ ## name)); \ - } \ - static ossl_unused ossl_inline int \ - ossl_list_##name##_is_empty(const OSSL_LIST(name) *list) \ - { \ - return list->num_elems == 0; \ - } \ - static ossl_unused ossl_inline size_t \ - ossl_list_##name##_num(const OSSL_LIST(name) *list) \ - { \ - return list->num_elems; \ - } \ - static ossl_unused ossl_inline type * \ - ossl_list_##name##_head(const OSSL_LIST(name) *list) \ - { \ - assert(list->alpha == NULL \ - || list->alpha->ossl_list_ ## name.list == list); \ - return list->alpha; \ - } \ - static ossl_unused ossl_inline type * \ - ossl_list_##name##_tail(const OSSL_LIST(name) *list) \ - { \ - assert(list->omega == NULL \ - || list->omega->ossl_list_ ## name.list == list); \ - return list->omega; \ - } \ - static ossl_unused ossl_inline type * \ - ossl_list_##name##_next(const type *elem) \ - { \ - assert(elem->ossl_list_ ## name.next == NULL \ - || elem->ossl_list_ ## name.next \ - ->ossl_list_ ## name.prev == elem); \ - return elem->ossl_list_ ## name.next; \ - } \ - static ossl_unused ossl_inline type * \ - ossl_list_##name##_prev(const type *elem) \ - { \ - assert(elem->ossl_list_ ## name.prev == NULL \ - || elem->ossl_list_ ## name.prev \ - ->ossl_list_ ## name.next == elem); \ - return elem->ossl_list_ ## name.prev; \ - } \ - static ossl_unused ossl_inline void \ - ossl_list_##name##_remove(OSSL_LIST(name) *list, type *elem) \ - { \ - assert(elem->ossl_list_ ## name.list == list); \ - OSSL_LIST_DBG(elem->ossl_list_ ## name.list = NULL) \ - if (list->alpha == elem) \ - list->alpha = elem->ossl_list_ ## name.next; \ - if (list->omega == elem) \ - list->omega = elem->ossl_list_ ## name.prev; \ - if (elem->ossl_list_ ## name.prev != NULL) \ - elem->ossl_list_ ## name.prev->ossl_list_ ## name.next = \ - elem->ossl_list_ ## name.next; \ - if (elem->ossl_list_ ## name.next != NULL) \ - elem->ossl_list_ ## name.next->ossl_list_ ## name.prev = \ - elem->ossl_list_ ## name.prev; \ - list->num_elems--; \ - memset(&elem->ossl_list_ ## name, 0, \ - sizeof(elem->ossl_list_ ## name)); \ - } \ - static ossl_unused ossl_inline void \ - ossl_list_##name##_insert_head(OSSL_LIST(name) *list, type *elem) \ - { \ - assert(elem->ossl_list_ ## name.list == NULL); \ - OSSL_LIST_DBG(elem->ossl_list_ ## name.list = list) \ - if (list->alpha != NULL) \ - list->alpha->ossl_list_ ## name.prev = elem; \ - elem->ossl_list_ ## name.next = list->alpha; \ - elem->ossl_list_ ## name.prev = NULL; \ - list->alpha = elem; \ - if (list->omega == NULL) \ - list->omega = elem; \ - list->num_elems++; \ - } \ - static ossl_unused ossl_inline void \ - ossl_list_##name##_insert_tail(OSSL_LIST(name) *list, type *elem) \ - { \ - assert(elem->ossl_list_ ## name.list == NULL); \ - OSSL_LIST_DBG(elem->ossl_list_ ## name.list = list) \ - if (list->omega != NULL) \ - list->omega->ossl_list_ ## name.next = elem; \ - elem->ossl_list_ ## name.prev = list->omega; \ - elem->ossl_list_ ## name.next = NULL; \ - list->omega = elem; \ - if (list->alpha == NULL) \ - list->alpha = elem; \ - list->num_elems++; \ - } \ - static ossl_unused ossl_inline void \ - ossl_list_##name##_insert_before(OSSL_LIST(name) *list, type *e, \ - type *elem) \ - { \ - assert(elem->ossl_list_ ## name.list == NULL); \ - OSSL_LIST_DBG(elem->ossl_list_ ## name.list = list) \ - elem->ossl_list_ ## name.next = e; \ - elem->ossl_list_ ## name.prev = e->ossl_list_ ## name.prev; \ - if (e->ossl_list_ ## name.prev != NULL) \ - e->ossl_list_ ## name.prev->ossl_list_ ## name.next = elem; \ - e->ossl_list_ ## name.prev = elem; \ - if (list->alpha == e) \ - list->alpha = elem; \ - list->num_elems++; \ - } \ - static ossl_unused ossl_inline void \ - ossl_list_##name##_insert_after(OSSL_LIST(name) *list, type *e, \ - type *elem) \ - { \ - assert(elem->ossl_list_ ## name.list == NULL); \ - OSSL_LIST_DBG(elem->ossl_list_ ## name.list = list) \ - elem->ossl_list_ ## name.prev = e; \ - elem->ossl_list_ ## name.next = e->ossl_list_ ## name.next; \ - if (e->ossl_list_ ## name.next != NULL) \ - e->ossl_list_ ## name.next->ossl_list_ ## name.prev = elem; \ - e->ossl_list_ ## name.next = elem; \ - if (list->omega == e) \ - list->omega = elem; \ - list->num_elems++; \ - } \ - struct ossl_list_st_ ## name - -# define DEFINE_LIST_OF(name, type) \ - DECLARE_LIST_OF(name, type); \ - DEFINE_LIST_OF_IMPL(name, type) - -#endif diff --git a/openssl/include/internal/namemap.h b/openssl/include/internal/namemap.h index 6c42a9cd7..a4c60ae69 100644 --- a/openssl/include/internal/namemap.h +++ b/openssl/include/internal/namemap.h @@ -18,6 +18,8 @@ void ossl_namemap_free(OSSL_NAMEMAP *namemap); int ossl_namemap_empty(OSSL_NAMEMAP *namemap); int ossl_namemap_add_name(OSSL_NAMEMAP *namemap, int number, const char *name); +int ossl_namemap_add_name_n(OSSL_NAMEMAP *namemap, int number, + const char *name, size_t name_len); /* * The number<->name relationship is 1<->many diff --git a/openssl/include/internal/numbers.h b/openssl/include/internal/numbers.h index 47fb16770..4f4d3306d 100644 --- a/openssl/include/internal/numbers.h +++ b/openssl/include/internal/numbers.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -61,31 +61,6 @@ # define UINT64_MAX __MAXUINT__(uint64_t) # endif -/* - * 64-bit processor with LP64 ABI - */ -# ifdef SIXTY_FOUR_BIT_LONG -# ifndef UINT32_C -# define UINT32_C(c) (c) -# endif -# ifndef UINT64_C -# define UINT64_C(c) (c##UL) -# endif -# endif - -/* - * 64-bit processor other than LP64 ABI - */ -# ifdef SIXTY_FOUR_BIT -# ifndef UINT32_C -# define UINT32_C(c) (c##UL) -# endif -# ifndef UINT64_C -# define UINT64_C(c) (c##ULL) -# endif -# endif - - # ifndef INT128_MAX # if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16 typedef __int128_t int128_t; diff --git a/openssl/include/internal/packet.h b/openssl/include/internal/packet.h index 7abc6b8b1..170997db6 100644 --- a/openssl/include/internal/packet.h +++ b/openssl/include/internal/packet.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -228,28 +228,6 @@ __owur static ossl_inline int PACKET_peek_net_4(const PACKET *pkt, return 1; } -/* - * Peek ahead at 8 bytes in network order from |pkt| and store the value in - * |*data| - */ -__owur static ossl_inline int PACKET_peek_net_8(const PACKET *pkt, - uint64_t *data) -{ - if (PACKET_remaining(pkt) < 8) - return 0; - - *data = ((uint64_t)(*pkt->curr)) << 56; - *data |= ((uint64_t)(*(pkt->curr + 1))) << 48; - *data |= ((uint64_t)(*(pkt->curr + 2))) << 40; - *data |= ((uint64_t)(*(pkt->curr + 3))) << 32; - *data |= ((uint64_t)(*(pkt->curr + 4))) << 24; - *data |= ((uint64_t)(*(pkt->curr + 5))) << 16; - *data |= ((uint64_t)(*(pkt->curr + 6))) << 8; - *data |= *(pkt->curr + 7); - - return 1; -} - /* Equivalent of n2l */ /* Get 4 bytes in network order from |pkt| and store the value in |*data| */ __owur static ossl_inline int PACKET_get_net_4(PACKET *pkt, unsigned long *data) @@ -274,17 +252,6 @@ __owur static ossl_inline int PACKET_get_net_4_len(PACKET *pkt, size_t *data) return ret; } -/* Get 8 bytes in network order from |pkt| and store the value in |*data| */ -__owur static ossl_inline int PACKET_get_net_8(PACKET *pkt, uint64_t *data) -{ - if (!PACKET_peek_net_8(pkt, data)) - return 0; - - packet_forward(pkt, 8); - - return 1; -} - /* Peek ahead at 1 byte from |pkt| and store the value in |*data| */ __owur static ossl_inline int PACKET_peek_1(const PACKET *pkt, unsigned int *data) @@ -691,8 +658,6 @@ struct wpacket_st { */ #define WPACKET_FLAGS_ABANDON_ON_ZERO_LENGTH 2 -/* QUIC variable-length integer length prefix */ -#define WPACKET_FLAGS_QUIC_VLINT 4 /* * Initialise a WPACKET with the buffer in |buf|. The buffer must exist @@ -868,7 +833,7 @@ int WPACKET_sub_reserve_bytes__(WPACKET *pkt, size_t len, * 1 byte will fail. Don't call this directly. Use the convenience macros below * instead. */ -int WPACKET_put_bytes__(WPACKET *pkt, uint64_t val, size_t bytes); +int WPACKET_put_bytes__(WPACKET *pkt, unsigned int val, size_t bytes); /* * Convenience macros for calling WPACKET_put_bytes with different @@ -882,8 +847,6 @@ int WPACKET_put_bytes__(WPACKET *pkt, uint64_t val, size_t bytes); WPACKET_put_bytes__((pkt), (val), 3) #define WPACKET_put_bytes_u32(pkt, val) \ WPACKET_put_bytes__((pkt), (val), 4) -#define WPACKET_put_bytes_u64(pkt, val) \ - WPACKET_put_bytes__((pkt), (val), 8) /* Set a maximum size that we will not allow the WPACKET to grow beyond */ int WPACKET_set_max_size(WPACKET *pkt, size_t maxsize); diff --git a/openssl/include/internal/packet_quic.h b/openssl/include/internal/packet_quic.h deleted file mode 100644 index 5173b4675..000000000 --- a/openssl/include/internal/packet_quic.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_PACKET_QUIC_H -# define OSSL_INTERNAL_PACKET_QUIC_H -# pragma once - -# include "internal/packet.h" -# include "internal/quic_vlint.h" - -# ifndef OPENSSL_NO_QUIC -/* - * Decodes a QUIC variable-length integer in |pkt| and stores the result in - * |data|. - */ -__owur static ossl_inline int PACKET_get_quic_vlint(PACKET *pkt, - uint64_t *data) -{ - size_t enclen; - - if (PACKET_remaining(pkt) < 1) - return 0; - - enclen = ossl_quic_vlint_decode_len(*pkt->curr); - - if (PACKET_remaining(pkt) < enclen) - return 0; - - *data = ossl_quic_vlint_decode_unchecked(pkt->curr); - packet_forward(pkt, enclen); - return 1; -} - -/* - * Decodes a QUIC variable-length integer in |pkt| and stores the result in - * |data|. Unlike PACKET_get_quic_vlint, this does not advance the current - * position. If was_minimal is non-NULL, *was_minimal is set to 1 if the integer - * was encoded using the minimal possible number of bytes and 0 otherwise. - */ -__owur static ossl_inline int PACKET_peek_quic_vlint_ex(PACKET *pkt, - uint64_t *data, - int *was_minimal) -{ - size_t enclen; - - if (PACKET_remaining(pkt) < 1) - return 0; - - enclen = ossl_quic_vlint_decode_len(*pkt->curr); - - if (PACKET_remaining(pkt) < enclen) - return 0; - - *data = ossl_quic_vlint_decode_unchecked(pkt->curr); - - if (was_minimal != NULL) - *was_minimal = (enclen == ossl_quic_vlint_encode_len(*data)); - - return 1; -} - -__owur static ossl_inline int PACKET_peek_quic_vlint(PACKET *pkt, - uint64_t *data) -{ - return PACKET_peek_quic_vlint_ex(pkt, data, NULL); -} - -/* - * Skips over a QUIC variable-length integer in |pkt| without decoding it. - */ -__owur static ossl_inline int PACKET_skip_quic_vlint(PACKET *pkt) -{ - size_t enclen; - - if (PACKET_remaining(pkt) < 1) - return 0; - - enclen = ossl_quic_vlint_decode_len(*pkt->curr); - - if (PACKET_remaining(pkt) < enclen) - return 0; - - packet_forward(pkt, enclen); - return 1; -} - -/* - * Reads a variable-length vector prefixed with a QUIC variable-length integer - * denoting the length, and stores the contents in |subpkt|. |pkt| can equal - * |subpkt|. Data is not copied: the |subpkt| packet will share its underlying - * buffer with the original |pkt|, so data wrapped by |pkt| must outlive the - * |subpkt|. Upon failure, the original |pkt| and |subpkt| are not modified. - */ -__owur static ossl_inline int PACKET_get_quic_length_prefixed(PACKET *pkt, - PACKET *subpkt) -{ - uint64_t length; - const unsigned char *data; - PACKET tmp = *pkt; - - if (!PACKET_get_quic_vlint(&tmp, &length) || - length > SIZE_MAX || - !PACKET_get_bytes(&tmp, &data, (size_t)length)) { - return 0; - } - - *pkt = tmp; - subpkt->curr = data; - subpkt->remaining = (size_t)length; - - return 1; -} - -/* - * Starts a QUIC sub-packet headed by a QUIC variable-length integer. A 4-byte - * representation is used. - */ -__owur int WPACKET_start_quic_sub_packet(WPACKET *pkt); - -/* - * Starts a QUIC sub-packet headed by a QUIC variable-length integer. max_len - * specifies the upper bound for the sub-packet size at the time the sub-packet - * is closed, which determines the encoding size for the variable-length - * integer header. max_len can be a precise figure or a worst-case bound - * if a precise figure is not available. - */ -__owur int WPACKET_start_quic_sub_packet_bound(WPACKET *pkt, size_t max_len); - -/* - * Allocates a QUIC sub-packet with exactly len bytes of payload, headed by a - * QUIC variable-length integer. The pointer to the payload buffer is output and - * must be filled by the caller. This function assures optimal selection of - * variable-length integer encoding length. - */ -__owur int WPACKET_quic_sub_allocate_bytes(WPACKET *pkt, size_t len, - unsigned char **bytes); - -/* - * Write a QUIC variable-length integer to the packet. - */ -__owur int WPACKET_quic_write_vlint(WPACKET *pkt, uint64_t v); - -# endif /* OPENSSL_NO_QUIC */ -#endif /* OSSL_INTERNAL_PACKET_QUIC_H */ diff --git a/openssl/include/internal/param_build_set.h b/openssl/include/internal/param_build_set.h index 3518f008f..126211b7f 100644 --- a/openssl/include/internal/param_build_set.h +++ b/openssl/include/internal/param_build_set.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -39,11 +39,6 @@ int ossl_param_build_set_bn(OSSL_PARAM_BLD *bld, OSSL_PARAM *p, const char *key, const BIGNUM *bn); int ossl_param_build_set_bn_pad(OSSL_PARAM_BLD *bld, OSSL_PARAM *p, const char *key, const BIGNUM *bn, size_t sz); -int ossl_param_build_set_signed_bn(OSSL_PARAM_BLD *bld, OSSL_PARAM *p, - const char *key, const BIGNUM *bn); -int ossl_param_build_set_signed_bn_pad(OSSL_PARAM_BLD *bld, OSSL_PARAM *p, - const char *key, const BIGNUM *bn, - size_t sz); int ossl_param_build_set_multi_key_bn(OSSL_PARAM_BLD *bld, OSSL_PARAM *p, const char *names[], STACK_OF(BIGNUM_const) *stk); diff --git a/openssl/include/internal/param_names.h b/openssl/include/internal/param_names.h deleted file mode 100644 index 15f4e84f6..000000000 --- a/openssl/include/internal/param_names.h +++ /dev/null @@ -1,377 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/include/internal/param_names.h.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -int ossl_param_find_pidx(const char *s); - -/* Parameter name definitions - generated by util/perl/OpenSSL/paramnames.pm */ -#define NUM_PIDX 291 - -#define PIDX_ALG_PARAM_CIPHER 0 -#define PIDX_ALG_PARAM_DIGEST 1 -#define PIDX_ALG_PARAM_ENGINE 2 -#define PIDX_ALG_PARAM_MAC 3 -#define PIDX_ALG_PARAM_PROPERTIES 4 -#define PIDX_ASYM_CIPHER_PARAM_DIGEST PIDX_PKEY_PARAM_DIGEST -#define PIDX_ASYM_CIPHER_PARAM_ENGINE PIDX_PKEY_PARAM_ENGINE -#define PIDX_ASYM_CIPHER_PARAM_IMPLICIT_REJECTION 5 -#define PIDX_ASYM_CIPHER_PARAM_MGF1_DIGEST PIDX_PKEY_PARAM_MGF1_DIGEST -#define PIDX_ASYM_CIPHER_PARAM_MGF1_DIGEST_PROPS PIDX_PKEY_PARAM_MGF1_PROPERTIES -#define PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST PIDX_ALG_PARAM_DIGEST -#define PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS 6 -#define PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL 7 -#define PIDX_ASYM_CIPHER_PARAM_PAD_MODE PIDX_PKEY_PARAM_PAD_MODE -#define PIDX_ASYM_CIPHER_PARAM_PROPERTIES PIDX_PKEY_PARAM_PROPERTIES -#define PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION 8 -#define PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION 9 -#define PIDX_CAPABILITY_TLS_GROUP_ALG 10 -#define PIDX_CAPABILITY_TLS_GROUP_ID 11 -#define PIDX_CAPABILITY_TLS_GROUP_IS_KEM 12 -#define PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS 13 -#define PIDX_CAPABILITY_TLS_GROUP_MAX_TLS 14 -#define PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS 15 -#define PIDX_CAPABILITY_TLS_GROUP_MIN_TLS 16 -#define PIDX_CAPABILITY_TLS_GROUP_NAME 17 -#define PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL 18 -#define PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS 19 -#define PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT 20 -#define PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME 21 -#define PIDX_CAPABILITY_TLS_SIGALG_HASH_OID 22 -#define PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME 23 -#define PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE 24 -#define PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID 25 -#define PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS 14 -#define PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS 16 -#define PIDX_CAPABILITY_TLS_SIGALG_NAME 26 -#define PIDX_CAPABILITY_TLS_SIGALG_OID 27 -#define PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS 28 -#define PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME 29 -#define PIDX_CAPABILITY_TLS_SIGALG_SIG_OID 30 -#define PIDX_CIPHER_PARAM_AEAD 31 -#define PIDX_CIPHER_PARAM_AEAD_IVLEN PIDX_CIPHER_PARAM_IVLEN -#define PIDX_CIPHER_PARAM_AEAD_MAC_KEY 32 -#define PIDX_CIPHER_PARAM_AEAD_TAG 33 -#define PIDX_CIPHER_PARAM_AEAD_TAGLEN 34 -#define PIDX_CIPHER_PARAM_AEAD_TLS1_AAD 35 -#define PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD 36 -#define PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN 37 -#define PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED 38 -#define PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV 39 -#define PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS 40 -#define PIDX_CIPHER_PARAM_BLOCK_SIZE 41 -#define PIDX_CIPHER_PARAM_CTS 42 -#define PIDX_CIPHER_PARAM_CTS_MODE 43 -#define PIDX_CIPHER_PARAM_CUSTOM_IV 44 -#define PIDX_CIPHER_PARAM_HAS_RAND_KEY 45 -#define PIDX_CIPHER_PARAM_IV 46 -#define PIDX_CIPHER_PARAM_IVLEN 47 -#define PIDX_CIPHER_PARAM_KEYLEN 48 -#define PIDX_CIPHER_PARAM_MODE 49 -#define PIDX_CIPHER_PARAM_NUM 50 -#define PIDX_CIPHER_PARAM_PADDING 51 -#define PIDX_CIPHER_PARAM_RANDOM_KEY 52 -#define PIDX_CIPHER_PARAM_RC2_KEYBITS 53 -#define PIDX_CIPHER_PARAM_ROUNDS 54 -#define PIDX_CIPHER_PARAM_SPEED 55 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK 56 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD 57 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN 58 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC 59 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN 60 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN 61 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE 62 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE 63 -#define PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT 64 -#define PIDX_CIPHER_PARAM_TLS_MAC 65 -#define PIDX_CIPHER_PARAM_TLS_MAC_SIZE 66 -#define PIDX_CIPHER_PARAM_TLS_VERSION 67 -#define PIDX_CIPHER_PARAM_UPDATED_IV 68 -#define PIDX_CIPHER_PARAM_USE_BITS 69 -#define PIDX_CIPHER_PARAM_XTS_STANDARD 70 -#define PIDX_DECODER_PARAM_PROPERTIES PIDX_ALG_PARAM_PROPERTIES -#define PIDX_DIGEST_PARAM_ALGID_ABSENT 71 -#define PIDX_DIGEST_PARAM_BLOCK_SIZE 41 -#define PIDX_DIGEST_PARAM_MICALG 72 -#define PIDX_DIGEST_PARAM_PAD_TYPE 73 -#define PIDX_DIGEST_PARAM_SIZE 74 -#define PIDX_DIGEST_PARAM_SSL3_MS 75 -#define PIDX_DIGEST_PARAM_XOF 76 -#define PIDX_DIGEST_PARAM_XOFLEN 77 -#define PIDX_DRBG_PARAM_CIPHER PIDX_ALG_PARAM_CIPHER -#define PIDX_DRBG_PARAM_DIGEST PIDX_ALG_PARAM_DIGEST -#define PIDX_DRBG_PARAM_ENTROPY_REQUIRED 78 -#define PIDX_DRBG_PARAM_MAC PIDX_ALG_PARAM_MAC -#define PIDX_DRBG_PARAM_MAX_ADINLEN 79 -#define PIDX_DRBG_PARAM_MAX_ENTROPYLEN 80 -#define PIDX_DRBG_PARAM_MAX_LENGTH 81 -#define PIDX_DRBG_PARAM_MAX_NONCELEN 82 -#define PIDX_DRBG_PARAM_MAX_PERSLEN 83 -#define PIDX_DRBG_PARAM_MIN_ENTROPYLEN 84 -#define PIDX_DRBG_PARAM_MIN_LENGTH 85 -#define PIDX_DRBG_PARAM_MIN_NONCELEN 86 -#define PIDX_DRBG_PARAM_PREDICTION_RESISTANCE 87 -#define PIDX_DRBG_PARAM_PROPERTIES PIDX_ALG_PARAM_PROPERTIES -#define PIDX_DRBG_PARAM_RANDOM_DATA 88 -#define PIDX_DRBG_PARAM_RESEED_COUNTER 89 -#define PIDX_DRBG_PARAM_RESEED_REQUESTS 90 -#define PIDX_DRBG_PARAM_RESEED_TIME 91 -#define PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL 92 -#define PIDX_DRBG_PARAM_SIZE 74 -#define PIDX_DRBG_PARAM_USE_DF 93 -#define PIDX_ENCODER_PARAM_CIPHER PIDX_ALG_PARAM_CIPHER -#define PIDX_ENCODER_PARAM_ENCRYPT_LEVEL 94 -#define PIDX_ENCODER_PARAM_PROPERTIES PIDX_ALG_PARAM_PROPERTIES -#define PIDX_ENCODER_PARAM_SAVE_PARAMETERS 95 -#define PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE 96 -#define PIDX_EXCHANGE_PARAM_KDF_DIGEST 97 -#define PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS 98 -#define PIDX_EXCHANGE_PARAM_KDF_OUTLEN 99 -#define PIDX_EXCHANGE_PARAM_KDF_TYPE 100 -#define PIDX_EXCHANGE_PARAM_KDF_UKM 101 -#define PIDX_EXCHANGE_PARAM_PAD 102 -#define PIDX_GEN_PARAM_ITERATION 103 -#define PIDX_GEN_PARAM_POTENTIAL 104 -#define PIDX_KDF_PARAM_ARGON2_AD 105 -#define PIDX_KDF_PARAM_ARGON2_LANES 106 -#define PIDX_KDF_PARAM_ARGON2_MEMCOST 107 -#define PIDX_KDF_PARAM_ARGON2_VERSION 108 -#define PIDX_KDF_PARAM_CEK_ALG 109 -#define PIDX_KDF_PARAM_CIPHER PIDX_ALG_PARAM_CIPHER -#define PIDX_KDF_PARAM_CONSTANT 110 -#define PIDX_KDF_PARAM_DATA 111 -#define PIDX_KDF_PARAM_DIGEST PIDX_ALG_PARAM_DIGEST -#define PIDX_KDF_PARAM_EARLY_CLEAN 112 -#define PIDX_KDF_PARAM_HMACDRBG_ENTROPY 113 -#define PIDX_KDF_PARAM_HMACDRBG_NONCE 114 -#define PIDX_KDF_PARAM_INFO 115 -#define PIDX_KDF_PARAM_ITER 116 -#define PIDX_KDF_PARAM_KBKDF_R 117 -#define PIDX_KDF_PARAM_KBKDF_USE_L 118 -#define PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR 119 -#define PIDX_KDF_PARAM_KEY 120 -#define PIDX_KDF_PARAM_LABEL 121 -#define PIDX_KDF_PARAM_MAC PIDX_ALG_PARAM_MAC -#define PIDX_KDF_PARAM_MAC_SIZE 122 -#define PIDX_KDF_PARAM_MODE 49 -#define PIDX_KDF_PARAM_PASSWORD 123 -#define PIDX_KDF_PARAM_PKCS12_ID 124 -#define PIDX_KDF_PARAM_PKCS5 125 -#define PIDX_KDF_PARAM_PREFIX 126 -#define PIDX_KDF_PARAM_PROPERTIES PIDX_ALG_PARAM_PROPERTIES -#define PIDX_KDF_PARAM_SALT 127 -#define PIDX_KDF_PARAM_SCRYPT_MAXMEM 128 -#define PIDX_KDF_PARAM_SCRYPT_N 129 -#define PIDX_KDF_PARAM_SCRYPT_P 130 -#define PIDX_KDF_PARAM_SCRYPT_R 117 -#define PIDX_KDF_PARAM_SECRET 131 -#define PIDX_KDF_PARAM_SEED 132 -#define PIDX_KDF_PARAM_SIZE 74 -#define PIDX_KDF_PARAM_SSHKDF_SESSION_ID 133 -#define PIDX_KDF_PARAM_SSHKDF_TYPE 134 -#define PIDX_KDF_PARAM_SSHKDF_XCGHASH 135 -#define PIDX_KDF_PARAM_THREADS 136 -#define PIDX_KDF_PARAM_UKM 137 -#define PIDX_KDF_PARAM_X942_ACVPINFO 138 -#define PIDX_KDF_PARAM_X942_PARTYUINFO 139 -#define PIDX_KDF_PARAM_X942_PARTYVINFO 140 -#define PIDX_KDF_PARAM_X942_SUPP_PRIVINFO 141 -#define PIDX_KDF_PARAM_X942_SUPP_PUBINFO 142 -#define PIDX_KDF_PARAM_X942_USE_KEYBITS 143 -#define PIDX_KEM_PARAM_IKME 144 -#define PIDX_KEM_PARAM_OPERATION 145 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING 146 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA 147 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN 148 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE 49 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS 149 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD 150 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC 151 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE 152 -#define PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM 153 -#define PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN 154 -#define PIDX_MAC_PARAM_BLOCK_SIZE 155 -#define PIDX_MAC_PARAM_CIPHER PIDX_ALG_PARAM_CIPHER -#define PIDX_MAC_PARAM_CUSTOM 156 -#define PIDX_MAC_PARAM_C_ROUNDS 157 -#define PIDX_MAC_PARAM_DIGEST PIDX_ALG_PARAM_DIGEST -#define PIDX_MAC_PARAM_DIGEST_NOINIT 158 -#define PIDX_MAC_PARAM_DIGEST_ONESHOT 159 -#define PIDX_MAC_PARAM_D_ROUNDS 160 -#define PIDX_MAC_PARAM_IV 46 -#define PIDX_MAC_PARAM_KEY 120 -#define PIDX_MAC_PARAM_PROPERTIES PIDX_ALG_PARAM_PROPERTIES -#define PIDX_MAC_PARAM_SALT 127 -#define PIDX_MAC_PARAM_SIZE 74 -#define PIDX_MAC_PARAM_TLS_DATA_SIZE 161 -#define PIDX_MAC_PARAM_XOF 76 -#define PIDX_OBJECT_PARAM_DATA 111 -#define PIDX_OBJECT_PARAM_DATA_STRUCTURE 162 -#define PIDX_OBJECT_PARAM_DATA_TYPE 163 -#define PIDX_OBJECT_PARAM_DESC 164 -#define PIDX_OBJECT_PARAM_REFERENCE 165 -#define PIDX_OBJECT_PARAM_TYPE 134 -#define PIDX_PASSPHRASE_PARAM_INFO 115 -#define PIDX_PKEY_PARAM_BITS 166 -#define PIDX_PKEY_PARAM_CIPHER PIDX_ALG_PARAM_CIPHER -#define PIDX_PKEY_PARAM_DEFAULT_DIGEST 167 -#define PIDX_PKEY_PARAM_DHKEM_IKM 168 -#define PIDX_PKEY_PARAM_DH_GENERATOR 169 -#define PIDX_PKEY_PARAM_DH_PRIV_LEN 170 -#define PIDX_PKEY_PARAM_DIGEST PIDX_ALG_PARAM_DIGEST -#define PIDX_PKEY_PARAM_DIGEST_SIZE 171 -#define PIDX_PKEY_PARAM_DIST_ID 172 -#define PIDX_PKEY_PARAM_EC_A 173 -#define PIDX_PKEY_PARAM_EC_B 174 -#define PIDX_PKEY_PARAM_EC_CHAR2_M 175 -#define PIDX_PKEY_PARAM_EC_CHAR2_PP_K1 176 -#define PIDX_PKEY_PARAM_EC_CHAR2_PP_K2 177 -#define PIDX_PKEY_PARAM_EC_CHAR2_PP_K3 178 -#define PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS 179 -#define PIDX_PKEY_PARAM_EC_CHAR2_TYPE 180 -#define PIDX_PKEY_PARAM_EC_COFACTOR 181 -#define PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS 182 -#define PIDX_PKEY_PARAM_EC_ENCODING 183 -#define PIDX_PKEY_PARAM_EC_FIELD_TYPE 184 -#define PIDX_PKEY_PARAM_EC_GENERATOR 185 -#define PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE 186 -#define PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC 187 -#define PIDX_PKEY_PARAM_EC_ORDER 188 -#define PIDX_PKEY_PARAM_EC_P 130 -#define PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT 189 -#define PIDX_PKEY_PARAM_EC_PUB_X 190 -#define PIDX_PKEY_PARAM_EC_PUB_Y 191 -#define PIDX_PKEY_PARAM_EC_SEED 132 -#define PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY 192 -#define PIDX_PKEY_PARAM_ENGINE PIDX_ALG_PARAM_ENGINE -#define PIDX_PKEY_PARAM_FFC_COFACTOR 193 -#define PIDX_PKEY_PARAM_FFC_DIGEST PIDX_PKEY_PARAM_DIGEST -#define PIDX_PKEY_PARAM_FFC_DIGEST_PROPS PIDX_PKEY_PARAM_PROPERTIES -#define PIDX_PKEY_PARAM_FFC_G 194 -#define PIDX_PKEY_PARAM_FFC_GINDEX 195 -#define PIDX_PKEY_PARAM_FFC_H 196 -#define PIDX_PKEY_PARAM_FFC_P 130 -#define PIDX_PKEY_PARAM_FFC_PBITS 197 -#define PIDX_PKEY_PARAM_FFC_PCOUNTER 198 -#define PIDX_PKEY_PARAM_FFC_Q 199 -#define PIDX_PKEY_PARAM_FFC_QBITS 200 -#define PIDX_PKEY_PARAM_FFC_SEED 132 -#define PIDX_PKEY_PARAM_FFC_TYPE 134 -#define PIDX_PKEY_PARAM_FFC_VALIDATE_G 201 -#define PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY 202 -#define PIDX_PKEY_PARAM_FFC_VALIDATE_PQ 203 -#define PIDX_PKEY_PARAM_GROUP_NAME 204 -#define PIDX_PKEY_PARAM_IMPLICIT_REJECTION 5 -#define PIDX_PKEY_PARAM_MANDATORY_DIGEST 205 -#define PIDX_PKEY_PARAM_MASKGENFUNC 206 -#define PIDX_PKEY_PARAM_MAX_SIZE 207 -#define PIDX_PKEY_PARAM_MGF1_DIGEST 208 -#define PIDX_PKEY_PARAM_MGF1_PROPERTIES 209 -#define PIDX_PKEY_PARAM_PAD_MODE 210 -#define PIDX_PKEY_PARAM_PRIV_KEY 211 -#define PIDX_PKEY_PARAM_PROPERTIES PIDX_ALG_PARAM_PROPERTIES -#define PIDX_PKEY_PARAM_PUB_KEY 212 -#define PIDX_PKEY_PARAM_RSA_BITS PIDX_PKEY_PARAM_BITS -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT 213 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT1 214 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT2 215 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT3 216 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT4 217 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT5 218 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT6 219 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT7 220 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT8 221 -#define PIDX_PKEY_PARAM_RSA_COEFFICIENT9 222 -#define PIDX_PKEY_PARAM_RSA_D 223 -#define PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ 224 -#define PIDX_PKEY_PARAM_RSA_DIGEST PIDX_PKEY_PARAM_DIGEST -#define PIDX_PKEY_PARAM_RSA_DIGEST_PROPS PIDX_PKEY_PARAM_PROPERTIES -#define PIDX_PKEY_PARAM_RSA_E 225 -#define PIDX_PKEY_PARAM_RSA_EXPONENT 226 -#define PIDX_PKEY_PARAM_RSA_EXPONENT1 227 -#define PIDX_PKEY_PARAM_RSA_EXPONENT10 228 -#define PIDX_PKEY_PARAM_RSA_EXPONENT2 229 -#define PIDX_PKEY_PARAM_RSA_EXPONENT3 230 -#define PIDX_PKEY_PARAM_RSA_EXPONENT4 231 -#define PIDX_PKEY_PARAM_RSA_EXPONENT5 232 -#define PIDX_PKEY_PARAM_RSA_EXPONENT6 233 -#define PIDX_PKEY_PARAM_RSA_EXPONENT7 234 -#define PIDX_PKEY_PARAM_RSA_EXPONENT8 235 -#define PIDX_PKEY_PARAM_RSA_EXPONENT9 236 -#define PIDX_PKEY_PARAM_RSA_FACTOR 237 -#define PIDX_PKEY_PARAM_RSA_FACTOR1 238 -#define PIDX_PKEY_PARAM_RSA_FACTOR10 239 -#define PIDX_PKEY_PARAM_RSA_FACTOR2 240 -#define PIDX_PKEY_PARAM_RSA_FACTOR3 241 -#define PIDX_PKEY_PARAM_RSA_FACTOR4 242 -#define PIDX_PKEY_PARAM_RSA_FACTOR5 243 -#define PIDX_PKEY_PARAM_RSA_FACTOR6 244 -#define PIDX_PKEY_PARAM_RSA_FACTOR7 245 -#define PIDX_PKEY_PARAM_RSA_FACTOR8 246 -#define PIDX_PKEY_PARAM_RSA_FACTOR9 247 -#define PIDX_PKEY_PARAM_RSA_MASKGENFUNC PIDX_PKEY_PARAM_MASKGENFUNC -#define PIDX_PKEY_PARAM_RSA_MGF1_DIGEST PIDX_PKEY_PARAM_MGF1_DIGEST -#define PIDX_PKEY_PARAM_RSA_N 129 -#define PIDX_PKEY_PARAM_RSA_PRIMES 248 -#define PIDX_PKEY_PARAM_RSA_PSS_SALTLEN 249 -#define PIDX_PKEY_PARAM_RSA_TEST_P1 250 -#define PIDX_PKEY_PARAM_RSA_TEST_P2 251 -#define PIDX_PKEY_PARAM_RSA_TEST_Q1 252 -#define PIDX_PKEY_PARAM_RSA_TEST_Q2 253 -#define PIDX_PKEY_PARAM_RSA_TEST_XP 254 -#define PIDX_PKEY_PARAM_RSA_TEST_XP1 255 -#define PIDX_PKEY_PARAM_RSA_TEST_XP2 256 -#define PIDX_PKEY_PARAM_RSA_TEST_XQ 257 -#define PIDX_PKEY_PARAM_RSA_TEST_XQ1 258 -#define PIDX_PKEY_PARAM_RSA_TEST_XQ2 259 -#define PIDX_PKEY_PARAM_SECURITY_BITS 260 -#define PIDX_PKEY_PARAM_USE_COFACTOR_ECDH PIDX_PKEY_PARAM_USE_COFACTOR_FLAG -#define PIDX_PKEY_PARAM_USE_COFACTOR_FLAG 261 -#define PIDX_PROV_PARAM_BUILDINFO 262 -#define PIDX_PROV_PARAM_CORE_MODULE_FILENAME 263 -#define PIDX_PROV_PARAM_CORE_PROV_NAME 264 -#define PIDX_PROV_PARAM_CORE_VERSION 265 -#define PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST 266 -#define PIDX_PROV_PARAM_NAME 267 -#define PIDX_PROV_PARAM_SECURITY_CHECKS 268 -#define PIDX_PROV_PARAM_SELF_TEST_DESC 269 -#define PIDX_PROV_PARAM_SELF_TEST_PHASE 270 -#define PIDX_PROV_PARAM_SELF_TEST_TYPE 271 -#define PIDX_PROV_PARAM_STATUS 272 -#define PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK 273 -#define PIDX_PROV_PARAM_VERSION 108 -#define PIDX_RAND_PARAM_GENERATE 274 -#define PIDX_RAND_PARAM_MAX_REQUEST 275 -#define PIDX_RAND_PARAM_STATE 276 -#define PIDX_RAND_PARAM_STRENGTH 277 -#define PIDX_RAND_PARAM_TEST_ENTROPY 278 -#define PIDX_RAND_PARAM_TEST_NONCE 279 -#define PIDX_SIGNATURE_PARAM_ALGORITHM_ID 280 -#define PIDX_SIGNATURE_PARAM_CONTEXT_STRING 281 -#define PIDX_SIGNATURE_PARAM_DIGEST PIDX_PKEY_PARAM_DIGEST -#define PIDX_SIGNATURE_PARAM_DIGEST_SIZE PIDX_PKEY_PARAM_DIGEST_SIZE -#define PIDX_SIGNATURE_PARAM_INSTANCE 282 -#define PIDX_SIGNATURE_PARAM_KAT 283 -#define PIDX_SIGNATURE_PARAM_MGF1_DIGEST PIDX_PKEY_PARAM_MGF1_DIGEST -#define PIDX_SIGNATURE_PARAM_MGF1_PROPERTIES PIDX_PKEY_PARAM_MGF1_PROPERTIES -#define PIDX_SIGNATURE_PARAM_NONCE_TYPE 284 -#define PIDX_SIGNATURE_PARAM_PAD_MODE PIDX_PKEY_PARAM_PAD_MODE -#define PIDX_SIGNATURE_PARAM_PROPERTIES PIDX_PKEY_PARAM_PROPERTIES -#define PIDX_SIGNATURE_PARAM_PSS_SALTLEN 249 -#define PIDX_STORE_PARAM_ALIAS 285 -#define PIDX_STORE_PARAM_DIGEST 1 -#define PIDX_STORE_PARAM_EXPECT 286 -#define PIDX_STORE_PARAM_FINGERPRINT 287 -#define PIDX_STORE_PARAM_INPUT_TYPE 288 -#define PIDX_STORE_PARAM_ISSUER 267 -#define PIDX_STORE_PARAM_PROPERTIES 4 -#define PIDX_STORE_PARAM_SERIAL 289 -#define PIDX_STORE_PARAM_SUBJECT 290 diff --git a/openssl/include/internal/param_names.h.in b/openssl/include/internal/param_names.h.in deleted file mode 100644 index f34db2195..000000000 --- a/openssl/include/internal/param_names.h.in +++ /dev/null @@ -1,18 +0,0 @@ -/* - * {- join("\n * ", @autowarntext) -} - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -{- -use OpenSSL::paramnames qw(generate_internal_macros); --} - -int ossl_param_find_pidx(const char *s); - -/* Parameter name definitions - generated by util/perl/OpenSSL/paramnames.pm */ -{- generate_internal_macros(); -} diff --git a/openssl/include/internal/params.h b/openssl/include/internal/params.h deleted file mode 100644 index 3fbd0cf95..000000000 --- a/openssl/include/internal/params.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include - -/* - * Extract the parameter into an allocated buffer. - * Any existing allocation in *out is cleared and freed. - * - * Returns 1 on success, 0 on failure and -1 if there are no matching params. - * - * *out and *out_len are guaranteed to be untouched if this function - * doesn't return success. - */ -int ossl_param_get1_octet_string(const OSSL_PARAM *params, const char *name, - unsigned char **out, size_t *out_len); -/* - * Concatenate all of the matching params together. - * *out will point to an allocated buffer on successful return. - * Any existing allocation in *out is cleared and freed. - * - * Passing 0 for maxsize means unlimited size output. - * - * Returns 1 on success, 0 on failure and -1 if there are no matching params. - * - * *out and *out_len are guaranteed to be untouched if this function - * doesn't return success. - */ -int ossl_param_get1_concat_octet_string(const OSSL_PARAM *params, const char *name, - unsigned char **out, size_t *out_len, - size_t maxsize); diff --git a/openssl/include/internal/priority_queue.h b/openssl/include/internal/priority_queue.h deleted file mode 100644 index 5be03bf13..000000000 --- a/openssl/include/internal/priority_queue.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_PRIORITY_QUEUE_H -# define OSSL_INTERNAL_PRIORITY_QUEUE_H -# pragma once - -# include -# include - -# define PRIORITY_QUEUE_OF(type) OSSL_PRIORITY_QUEUE_ ## type - -# define DEFINE_PRIORITY_QUEUE_OF_INTERNAL(type, ctype) \ - typedef struct ossl_priority_queue_st_ ## type PRIORITY_QUEUE_OF(type); \ - static ossl_unused ossl_inline PRIORITY_QUEUE_OF(type) * \ - ossl_pqueue_##type##_new(int (*compare)(const ctype *, const ctype *)) \ - { \ - return (PRIORITY_QUEUE_OF(type) *)ossl_pqueue_new( \ - (int (*)(const void *, const void *))compare); \ - } \ - static ossl_unused ossl_inline void \ - ossl_pqueue_##type##_free(PRIORITY_QUEUE_OF(type) *pq) \ - { \ - ossl_pqueue_free((OSSL_PQUEUE *)pq); \ - } \ - static ossl_unused ossl_inline void \ - ossl_pqueue_##type##_pop_free(PRIORITY_QUEUE_OF(type) *pq, \ - void (*freefunc)(ctype *)) \ - { \ - ossl_pqueue_pop_free((OSSL_PQUEUE *)pq, (void (*)(void *))freefunc);\ - } \ - static ossl_unused ossl_inline int \ - ossl_pqueue_##type##_reserve(PRIORITY_QUEUE_OF(type) *pq, size_t n) \ - { \ - return ossl_pqueue_reserve((OSSL_PQUEUE *)pq, n); \ - } \ - static ossl_unused ossl_inline size_t \ - ossl_pqueue_##type##_num(const PRIORITY_QUEUE_OF(type) *pq) \ - { \ - return ossl_pqueue_num((OSSL_PQUEUE *)pq); \ - } \ - static ossl_unused ossl_inline int \ - ossl_pqueue_##type##_push(PRIORITY_QUEUE_OF(type) *pq, \ - ctype *data, size_t *elem) \ - { \ - return ossl_pqueue_push((OSSL_PQUEUE *)pq, (void *)data, elem); \ - } \ - static ossl_unused ossl_inline ctype * \ - ossl_pqueue_##type##_peek(const PRIORITY_QUEUE_OF(type) *pq) \ - { \ - return (type *)ossl_pqueue_peek((OSSL_PQUEUE *)pq); \ - } \ - static ossl_unused ossl_inline ctype * \ - ossl_pqueue_##type##_pop(PRIORITY_QUEUE_OF(type) *pq) \ - { \ - return (type *)ossl_pqueue_pop((OSSL_PQUEUE *)pq); \ - } \ - static ossl_unused ossl_inline ctype * \ - ossl_pqueue_##type##_remove(PRIORITY_QUEUE_OF(type) *pq, \ - size_t elem) \ - { \ - return (type *)ossl_pqueue_remove((OSSL_PQUEUE *)pq, elem); \ - } \ - struct ossl_priority_queue_st_ ## type - -# define DEFINE_PRIORITY_QUEUE_OF(type) \ - DEFINE_PRIORITY_QUEUE_OF_INTERNAL(type, type) - -typedef struct ossl_pqueue_st OSSL_PQUEUE; - -OSSL_PQUEUE *ossl_pqueue_new(int (*compare)(const void *, const void *)); -void ossl_pqueue_free(OSSL_PQUEUE *pq); -void ossl_pqueue_pop_free(OSSL_PQUEUE *pq, void (*freefunc)(void *)); -int ossl_pqueue_reserve(OSSL_PQUEUE *pq, size_t n); - -size_t ossl_pqueue_num(const OSSL_PQUEUE *pq); -int ossl_pqueue_push(OSSL_PQUEUE *pq, void *data, size_t *elem); -void *ossl_pqueue_peek(const OSSL_PQUEUE *pq); -void *ossl_pqueue_pop(OSSL_PQUEUE *pq); -void *ossl_pqueue_remove(OSSL_PQUEUE *pq, size_t elem); - -#endif diff --git a/openssl/include/internal/property.h b/openssl/include/internal/property.h index 3adff4994..821197459 100644 --- a/openssl/include/internal/property.h +++ b/openssl/include/internal/property.h @@ -52,10 +52,6 @@ int64_t ossl_property_get_number_value(const OSSL_PROPERTY_DEFINITION *prop); /* Implementation store functions */ OSSL_METHOD_STORE *ossl_method_store_new(OSSL_LIB_CTX *ctx); void ossl_method_store_free(OSSL_METHOD_STORE *store); - -int ossl_method_lock_store(OSSL_METHOD_STORE *store); -int ossl_method_unlock_store(OSSL_METHOD_STORE *store); - int ossl_method_store_add(OSSL_METHOD_STORE *store, const OSSL_PROVIDER *prov, int nid, const char *properties, void *method, int (*method_up_ref)(void *), @@ -68,8 +64,6 @@ void ossl_method_store_do_all(OSSL_METHOD_STORE *store, int ossl_method_store_fetch(OSSL_METHOD_STORE *store, int nid, const char *prop_query, const OSSL_PROVIDER **prov, void **method); -int ossl_method_store_remove_all_provided(OSSL_METHOD_STORE *store, - const OSSL_PROVIDER *prov); /* Get the global properties associate with the specified library context */ OSSL_PROPERTY_LIST **ossl_ctx_global_properties(OSSL_LIB_CTX *ctx, @@ -83,7 +77,7 @@ int ossl_method_store_cache_set(OSSL_METHOD_STORE *store, OSSL_PROVIDER *prov, int (*method_up_ref)(void *), void (*method_destruct)(void *)); -__owur int ossl_method_store_cache_flush_all(OSSL_METHOD_STORE *store); +__owur int ossl_method_store_flush_cache(OSSL_METHOD_STORE *store, int all); /* Merge two property queries together */ OSSL_PROPERTY_LIST *ossl_property_merge(const OSSL_PROPERTY_LIST *a, diff --git a/openssl/include/internal/provider.h b/openssl/include/internal/provider.h index ab41d643d..d09829d05 100644 --- a/openssl/include/internal/provider.h +++ b/openssl/include/internal/provider.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -32,11 +32,12 @@ OSSL_PROVIDER *ossl_provider_find(OSSL_LIB_CTX *libctx, const char *name, int noconfig); OSSL_PROVIDER *ossl_provider_new(OSSL_LIB_CTX *libctx, const char *name, OSSL_provider_init_fn *init_function, - OSSL_PARAM *params, int noconfig); + int noconfig); int ossl_provider_up_ref(OSSL_PROVIDER *prov); void ossl_provider_free(OSSL_PROVIDER *prov); /* Setters */ +int ossl_provider_set_fallback(OSSL_PROVIDER *prov); int ossl_provider_set_module_path(OSSL_PROVIDER *prov, const char *module_path); int ossl_provider_add_parameter(OSSL_PROVIDER *prov, const char *name, const char *value); @@ -94,14 +95,11 @@ void ossl_provider_unquery_operation(const OSSL_PROVIDER *prov, int operation_id, const OSSL_ALGORITHM *algs); -/* - * Cache of bits to see if we already added methods for an operation in - * the "permanent" method store. - * They should never be called for temporary method stores! - */ +/* Cache of bits to see if we already queried an operation */ int ossl_provider_set_operation_bit(OSSL_PROVIDER *provider, size_t bitnum); int ossl_provider_test_operation_bit(OSSL_PROVIDER *provider, size_t bitnum, int *result); +int ossl_provider_clear_all_operation_bits(OSSL_LIB_CTX *libctx); /* Configuration */ void ossl_provider_add_conf_module(void); diff --git a/openssl/include/internal/qlog.h b/openssl/include/internal/qlog.h deleted file mode 100644 index b81bfe7e4..000000000 --- a/openssl/include/internal/qlog.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QLOG_H -# define OSSL_QLOG_H - -# include -# include "internal/quic_types.h" -# include "internal/time.h" - -typedef struct qlog_st QLOG; - -# ifndef OPENSSL_NO_QLOG - -enum { - QLOG_EVENT_TYPE_NONE, - -# define QLOG_EVENT(cat, name) QLOG_EVENT_TYPE_##cat##_##name, -# include "internal/qlog_events.h" -# undef QLOG_EVENT - - QLOG_EVENT_TYPE_NUM -}; - -typedef struct qlog_trace_info_st { - QUIC_CONN_ID odcid; - const char *title, *description, *group_id; - int is_server; - OSSL_TIME (*now_cb)(void *arg); - void *now_cb_arg; - uint64_t override_process_id; - const char *override_impl_name; -} QLOG_TRACE_INFO; - -QLOG *ossl_qlog_new(const QLOG_TRACE_INFO *info); -QLOG *ossl_qlog_new_from_env(const QLOG_TRACE_INFO *info); - -void ossl_qlog_free(QLOG *qlog); - -/* Configuration */ -int ossl_qlog_set_event_type_enabled(QLOG *qlog, uint32_t event_type, - int enable); -int ossl_qlog_set_filter(QLOG *qlog, const char *filter); - -int ossl_qlog_set_sink_bio(QLOG *qlog, BIO *bio); -# ifndef OPENSSL_NO_STDIO -int ossl_qlog_set_sink_file(QLOG *qlog, FILE *file, int close_flag); -# endif -int ossl_qlog_set_sink_filename(QLOG *qlog, const char *filename); - -/* Operations */ -int ossl_qlog_flush(QLOG *qlog); - -/* Queries */ -int ossl_qlog_enabled(QLOG *qlog, uint32_t event_type); - -/* Grouping Functions */ -int ossl_qlog_event_try_begin(QLOG *qlog, uint32_t event_type, - const char *event_cat, const char *event_name, - const char *event_combined_name); -void ossl_qlog_event_end(QLOG *qlog); - -void ossl_qlog_group_begin(QLOG *qlog, const char *name); -void ossl_qlog_group_end(QLOG *qlog); - -void ossl_qlog_array_begin(QLOG *qlog, const char *name); -void ossl_qlog_array_end(QLOG *qlog); - -void ossl_qlog_override_time(QLOG *qlog, OSSL_TIME event_time); - -/* Grouping Macros */ -# define QLOG_EVENT_BEGIN(qlog, cat, name) \ - { \ - QLOG *qlog_instance = (qlog); \ - uint32_t qlog_event_type = QLOG_EVENT_TYPE_##cat##_##name; \ - \ - if (ossl_qlog_event_try_begin(qlog_instance, qlog_event_type, \ - #cat, #name, #cat ":" #name)) { - -# define QLOG_EVENT_END() \ - ossl_qlog_event_end(qlog_instance); \ - } \ - } - -# define QLOG_BEGIN(name) \ - { \ - ossl_qlog_group_begin(qlog_instance, (name)); - -# define QLOG_END() \ - ossl_qlog_group_end(qlog_instance); \ - } - -# define QLOG_BEGIN_ARRAY(name) \ - { \ - ossl_qlog_array_begin(qlog_instance, (name)); - -# define QLOG_END_ARRAY() \ - ossl_qlog_array_end(qlog_instance); \ - } - -/* Field Functions */ -void ossl_qlog_str(QLOG *qlog, const char *name, const char *value); -void ossl_qlog_str_len(QLOG *qlog, const char *name, - const char *value, size_t value_len); -void ossl_qlog_u64(QLOG *qlog, const char *name, uint64_t value); -void ossl_qlog_i64(QLOG *qlog, const char *name, int64_t value); -void ossl_qlog_bool(QLOG *qlog, const char *name, int value); -void ossl_qlog_bin(QLOG *qlog, const char *name, - const void *value, size_t value_len); - -/* Field Macros */ -# define QLOG_STR(name, value) ossl_qlog_str(qlog_instance, (name), (value)) -# define QLOG_STR_LEN(name, value, value_len) \ - ossl_qlog_str_len(qlog_instance, (name), (value), (value_len)) -# define QLOG_I64(name, value) ossl_qlog_i64(qlog_instance, (name), (value)) -# define QLOG_U64(name, value) ossl_qlog_u64(qlog_instance, (name), (value)) -# define QLOG_F64(name, value) ossl_qlog_f64(qlog_instance, (name), (value)) -# define QLOG_BOOL(name, value) ossl_qlog_bool(qlog_instance, (name), (value)) -# define QLOG_BIN(name, value, value_len) \ - ossl_qlog_bin(qlog_instance, (name), (value), (value_len)) -# define QLOG_CID(name, value) QLOG_BIN((name), (value)->id, (value)->id_len) - -# endif - -#endif diff --git a/openssl/include/internal/qlog_event_helpers.h b/openssl/include/internal/qlog_event_helpers.h deleted file mode 100644 index 43d623608..000000000 --- a/openssl/include/internal/qlog_event_helpers.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QLOG_EVENT_HELPERS_H -# define OSSL_QLOG_EVENT_HELPERS_H - -# include -# include "internal/qlog.h" -# include "internal/quic_types.h" -# include "internal/quic_channel.h" -# include "internal/quic_txpim.h" -# include "internal/quic_record_tx.h" -# include "internal/quic_wire_pkt.h" - -/* connectivity:connection_started */ -void ossl_qlog_event_connectivity_connection_started(QLOG *qlog, - const QUIC_CONN_ID *init_dcid); - -/* connectivity:connection_state_updated */ -void ossl_qlog_event_connectivity_connection_state_updated(QLOG *qlog, - uint32_t old_state, - uint32_t new_state, - int handshake_complete, - int handshake_confirmed); - -/* connectivity:connection_closed */ -void ossl_qlog_event_connectivity_connection_closed(QLOG *qlog, - const QUIC_TERMINATE_CAUSE *tcause); - -/* recovery:packet_lost */ -void ossl_qlog_event_recovery_packet_lost(QLOG *qlog, - const QUIC_TXPIM_PKT *tpkt); - -/* transport:packet_sent */ -void ossl_qlog_event_transport_packet_sent(QLOG *qlog, - const QUIC_PKT_HDR *hdr, - QUIC_PN pn, - const OSSL_QTX_IOVEC *iovec, - size_t numn_iovec, - uint64_t datagram_id); - -/* transport:packet_received */ -void ossl_qlog_event_transport_packet_received(QLOG *qlog, - const QUIC_PKT_HDR *hdr, - QUIC_PN pn, - const OSSL_QTX_IOVEC *iovec, - size_t numn_iovec, - uint64_t datagram_id); - -#endif diff --git a/openssl/include/internal/qlog_events.h b/openssl/include/internal/qlog_events.h deleted file mode 100644 index 6dd44bf36..000000000 --- a/openssl/include/internal/qlog_events.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -QLOG_EVENT(connectivity, connection_started) -QLOG_EVENT(connectivity, connection_state_updated) -QLOG_EVENT(connectivity, connection_closed) -QLOG_EVENT(transport, parameters_set) -QLOG_EVENT(transport, packet_sent) -QLOG_EVENT(transport, packet_received) -QLOG_EVENT(recovery, packet_lost) diff --git a/openssl/include/internal/quic_ackm.h b/openssl/include/internal/quic_ackm.h deleted file mode 100644 index 69b862d9c..000000000 --- a/openssl/include/internal/quic_ackm.h +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#ifndef OSSL_QUIC_ACKM_H -# define OSSL_QUIC_ACKM_H - -# include "internal/quic_statm.h" -# include "internal/quic_cc.h" -# include "internal/quic_types.h" -# include "internal/quic_wire.h" -# include "internal/quic_predef.h" -# include "internal/time.h" -# include "internal/list.h" - -# ifndef OPENSSL_NO_QUIC - -OSSL_ACKM *ossl_ackm_new(OSSL_TIME (*now)(void *arg), - void *now_arg, - OSSL_STATM *statm, - const OSSL_CC_METHOD *cc_method, - OSSL_CC_DATA *cc_data); -void ossl_ackm_free(OSSL_ACKM *ackm); - -void ossl_ackm_set_loss_detection_deadline_callback(OSSL_ACKM *ackm, - void (*fn)(OSSL_TIME deadline, - void *arg), - void *arg); - -void ossl_ackm_set_ack_deadline_callback(OSSL_ACKM *ackm, - void (*fn)(OSSL_TIME deadline, - int pkt_space, - void *arg), - void *arg); - -/* - * Configures the RX-side maximum ACK delay. This is the maximum amount of time - * the peer is allowed to delay sending an ACK frame after receiving an - * ACK-eliciting packet. The peer communicates this value via a transport - * parameter and it must be provided to the ACKM. - */ -void ossl_ackm_set_rx_max_ack_delay(OSSL_ACKM *ackm, OSSL_TIME rx_max_ack_delay); - -/* - * Configures the TX-side maximum ACK delay. This is the maximum amount of time - * we are allowed to delay sending an ACK frame after receiving an ACK-eliciting - * packet. Note that this cannot be changed after a connection is established as - * it must be accurately reported in the transport parameters we send to our - * peer. - */ -void ossl_ackm_set_tx_max_ack_delay(OSSL_ACKM *ackm, OSSL_TIME tx_max_ack_delay); - -typedef struct ossl_ackm_tx_pkt_st OSSL_ACKM_TX_PKT; -struct ossl_ackm_tx_pkt_st { - /* The packet number of the transmitted packet. */ - QUIC_PN pkt_num; - - /* The number of bytes in the packet which was sent. */ - size_t num_bytes; - - /* The time at which the packet was sent. */ - OSSL_TIME time; - - /* - * If the packet being described by this structure contains an ACK frame, - * this must be set to the largest PN ACK'd by that frame. - * - * Otherwise, it should be set to QUIC_PN_INVALID. - * - * This is necessary to bound the number of PNs we have to keep track of on - * the RX side (RFC 9000 s. 13.2.4). It allows older PN tracking information - * on the RX side to be discarded. - */ - QUIC_PN largest_acked; - - /* - * One of the QUIC_PN_SPACE_* values. This qualifies the pkt_num field - * into a packet number space. - */ - unsigned int pkt_space :2; - - /* - * 1 if the packet is in flight. A packet is considered 'in flight' if it is - * counted for purposes of congestion control and 'bytes in flight' counts. - * Most packets are considered in flight. The only circumstance where a - * numbered packet is not considered in flight is if it contains only ACK - * frames (not even PADDING frames), as these frames can bypass CC. - */ - unsigned int is_inflight :1; - - /* - * 1 if the packet has one or more ACK-eliciting frames. - * Note that if this is set, is_inflight must be set. - */ - unsigned int is_ack_eliciting :1; - - /* 1 if the packet is a PTO probe. */ - unsigned int is_pto_probe :1; - - /* 1 if the packet is an MTU probe. */ - unsigned int is_mtu_probe :1; - - /* Callback called if frames in this packet are lost. arg is cb_arg. */ - void (*on_lost)(void *arg); - /* Callback called if frames in this packet are acked. arg is cb_arg. */ - void (*on_acked)(void *arg); - /* - * Callback called if frames in this packet are neither acked nor lost. arg - * is cb_arg. - */ - void (*on_discarded)(void *arg); - void *cb_arg; - - /* - * (Internal use fields; must be zero-initialized.) - * - * Keep a TX history list, anext is used to manifest - * a singly-linked list of newly-acknowledged packets, and lnext is used to - * manifest a singly-linked list of newly lost packets. - */ - OSSL_LIST_MEMBER(tx_history, OSSL_ACKM_TX_PKT); - - struct ossl_ackm_tx_pkt_st *anext; - struct ossl_ackm_tx_pkt_st *lnext; -}; - -int ossl_ackm_on_tx_packet(OSSL_ACKM *ackm, OSSL_ACKM_TX_PKT *pkt); -int ossl_ackm_on_rx_datagram(OSSL_ACKM *ackm, size_t num_bytes); - -# define OSSL_ACKM_ECN_NONE 0 -# define OSSL_ACKM_ECN_ECT1 1 -# define OSSL_ACKM_ECN_ECT0 2 -# define OSSL_ACKM_ECN_ECNCE 3 - -typedef struct ossl_ackm_rx_pkt_st { - /* The packet number of the received packet. */ - QUIC_PN pkt_num; - - /* The time at which the packet was received. */ - OSSL_TIME time; - - /* - * One of the QUIC_PN_SPACE_* values. This qualifies the pkt_num field - * into a packet number space. - */ - unsigned int pkt_space :2; - - /* 1 if the packet has one or more ACK-eliciting frames. */ - unsigned int is_ack_eliciting :1; - - /* - * One of the OSSL_ACKM_ECN_* values. This is the ECN labelling applied to - * the received packet. If unknown, use OSSL_ACKM_ECN_NONE. - */ - unsigned int ecn :2; -} OSSL_ACKM_RX_PKT; - -int ossl_ackm_on_rx_packet(OSSL_ACKM *ackm, const OSSL_ACKM_RX_PKT *pkt); - -int ossl_ackm_on_rx_ack_frame(OSSL_ACKM *ackm, const OSSL_QUIC_FRAME_ACK *ack, - int pkt_space, OSSL_TIME rx_time); - -/* - * Discards a PN space. This must be called for a PN space before freeing the - * ACKM if you want in-flight packets to have their discarded callbacks called. - * This should never be called in ordinary QUIC usage for the Application Data - * PN space, but it may be called for the Application Data PN space prior to - * freeing the ACKM to simplify teardown implementations. - */ -int ossl_ackm_on_pkt_space_discarded(OSSL_ACKM *ackm, int pkt_space); - -int ossl_ackm_on_handshake_confirmed(OSSL_ACKM *ackm); -int ossl_ackm_on_timeout(OSSL_ACKM *ackm); - -OSSL_TIME ossl_ackm_get_loss_detection_deadline(OSSL_ACKM *ackm); - -/* - * Generates an ACK frame, regardless of whether the ACK manager thinks - * one should currently be sent. - * - * This clears the flag returned by ossl_ackm_is_ack_desired and the deadline - * returned by ossl_ackm_get_ack_deadline. - */ -const OSSL_QUIC_FRAME_ACK *ossl_ackm_get_ack_frame(OSSL_ACKM *ackm, - int pkt_space); - -/* - * Returns the deadline after which an ACK frame should be generated by calling - * ossl_ackm_get_ack_frame, or OSSL_TIME_INFINITY if no deadline is currently - * applicable. If the deadline has already passed, this function may return that - * deadline, or may return OSSL_TIME_ZERO. - */ -OSSL_TIME ossl_ackm_get_ack_deadline(OSSL_ACKM *ackm, int pkt_space); - -/* - * Returns 1 if the ACK manager thinks an ACK frame ought to be generated and - * sent at this time. ossl_ackm_get_ack_frame will always provide an ACK frame - * whether or not this returns 1, so it is suggested that you call this function - * first to determine whether you need to generate an ACK frame. - * - * The return value of this function can change based on calls to - * ossl_ackm_on_rx_packet and based on the passage of time (see - * ossl_ackm_get_ack_deadline). - */ -int ossl_ackm_is_ack_desired(OSSL_ACKM *ackm, int pkt_space); - -/* - * Returns 1 if the given RX PN is 'processable'. A processable PN is one that - * is not either - * - * - duplicate, meaning that we have already been passed such a PN in a call - * to ossl_ackm_on_rx_packet; or - * - * - written off, meaning that the PN is so old we have stopped tracking state - * for it (meaning that we cannot tell whether it is a duplicate and cannot - * process it safely). - * - * This should be called for a packet before attempting to process its contents. - * Failure to do so may result in processing a duplicated packet in violation of - * the RFC. - * - * The return value of this function transitions from 1 to 0 for a given PN once - * that PN is passed to ossl_ackm_on_rx_packet, thus this function must be used - * before calling ossl_ackm_on_rx_packet. - */ -int ossl_ackm_is_rx_pn_processable(OSSL_ACKM *ackm, QUIC_PN pn, int pkt_space); - -typedef struct ossl_ackm_probe_info_st { - /* - * The following two probe request types are used only for anti-deadlock - * purposes in relation to the anti-amplification logic, by generating - * packets to buy ourselves more anti-amplification credit with the server - * until a client address is verified. Note that like all Initial packets, - * any Initial probes are padded. - * - * Note: The ACKM will only ever increase these by one at a time, - * as only one probe packet should be generated for these cases. - */ - uint32_t anti_deadlock_initial, anti_deadlock_handshake; - - /* - * Send an ACK-eliciting packet for each count here. - * - * Note: The ACKM may increase this by either one or two for each probe - * request, depending on how many probe packets it thinks should be - * generated. - */ - uint32_t pto[QUIC_PN_SPACE_NUM]; -} OSSL_ACKM_PROBE_INFO; - -/* - * Returns a pointer to a structure counting any pending probe requests which - * have been generated by the ACKM. The fields in the structure are incremented - * by one every time the ACKM wants another probe of the given type to be sent. - * If the ACKM thinks two packets should be generated for a probe, it will - * increment the field twice. - * - * It is permissible for the caller to decrement or zero these fields to keep - * track of when it has generated a probe as asked. The returned structure - * has the same lifetime as the ACKM. - * - * This function should be called after calling e.g. ossl_ackm_on_timeout - * to determine if any probe requests have been generated. - */ -OSSL_ACKM_PROBE_INFO *ossl_ackm_get0_probe_request(OSSL_ACKM *ackm); - -int ossl_ackm_get_largest_unacked(OSSL_ACKM *ackm, int pkt_space, QUIC_PN *pn); - -/* - * Forces the ACKM to consider a packet with the given PN in the given PN space - * as having been pseudo-lost. The main reason to use this is during a Retry, to - * force any resources sent in the first Initial packet to be resent. - * - * The lost callback is called for the packet, but the packet is NOT considered - * lost for congestion control purposes. Thus this is not exactly the same as a - * true loss situation. - */ -int ossl_ackm_mark_packet_pseudo_lost(OSSL_ACKM *ackm, - int pkt_space, QUIC_PN pn); - -/* - * Returns the PTO duration as currently calculated. This is a quantity of time. - * This duration is used in various parts of QUIC besides the ACKM. - */ -OSSL_TIME ossl_ackm_get_pto_duration(OSSL_ACKM *ackm); - -/* Returns the largest acked PN in the given PN space. */ -QUIC_PN ossl_ackm_get_largest_acked(OSSL_ACKM *ackm, int pkt_space); - -# endif - -#endif diff --git a/openssl/include/internal/quic_cc.h b/openssl/include/internal/quic_cc.h deleted file mode 100644 index dbd439dd0..000000000 --- a/openssl/include/internal/quic_cc.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#ifndef OSSL_QUIC_CC_H -# define OSSL_QUIC_CC_H - -#include "openssl/params.h" -#include "internal/time.h" -#include "internal/quic_predef.h" - -# ifndef OPENSSL_NO_QUIC - -typedef struct ossl_cc_ack_info_st { - /* The time the packet being acknowledged was originally sent. */ - OSSL_TIME tx_time; - - /* The size in bytes of the packet being acknowledged. */ - size_t tx_size; -} OSSL_CC_ACK_INFO; - -typedef struct ossl_cc_loss_info_st { - /* The time the packet being lost was originally sent. */ - OSSL_TIME tx_time; - - /* The size in bytes of the packet which has been determined lost. */ - size_t tx_size; -} OSSL_CC_LOSS_INFO; - -typedef struct ossl_cc_ecn_info_st { - /* - * The time at which the largest acked PN (in the incoming ACK frame) was - * sent. - */ - OSSL_TIME largest_acked_time; -} OSSL_CC_ECN_INFO; - -/* Parameter (read-write): Maximum datagram payload length in bytes. */ -#define OSSL_CC_OPTION_MAX_DGRAM_PAYLOAD_LEN "max_dgram_payload_len" - -/* Diagnostic (read-only): current congestion window size in bytes. */ -#define OSSL_CC_OPTION_CUR_CWND_SIZE "cur_cwnd_size" - -/* Diagnostic (read-only): minimum congestion window size in bytes. */ -#define OSSL_CC_OPTION_MIN_CWND_SIZE "min_cwnd_size" - -/* Diagnostic (read-only): current net bytes in flight. */ -#define OSSL_CC_OPTION_CUR_BYTES_IN_FLIGHT "bytes_in_flight" - -/* Diagnostic (read-only): method-specific state value. */ -#define OSSL_CC_OPTION_CUR_STATE "cur_state" - -/* - * Congestion control abstract interface. - * - * This interface is broadly based on the design described in RFC 9002. However, - * the demarcation between the ACKM and the congestion controller does not - * exactly match that delineated in the RFC 9002 pseudocode. Where aspects of - * the demarcation involve the congestion controller accessing internal state of - * the ACKM, the interface has been revised where possible to provide the - * information needed by the congestion controller and avoid needing to give the - * congestion controller access to the ACKM's internal data structures. - * - * Particular changes include: - * - * - In our implementation, it is the responsibility of the ACKM to determine - * if a loss event constitutes persistent congestion. - * - * - In our implementation, it is the responsibility of the ACKM to determine - * if the ECN-CE counter has increased. The congestion controller is simply - * informed when an ECN-CE event occurs. - * - * All of these changes are intended to avoid having a congestion controller - * have to access ACKM internal state. - */ -#define OSSL_CC_LOST_FLAG_PERSISTENT_CONGESTION (1U << 0) - -struct ossl_cc_method_st { - /* - * Instantiation. - */ - OSSL_CC_DATA *(*new)(OSSL_TIME (*now_cb)(void *arg), - void *now_cb_arg); - - void (*free)(OSSL_CC_DATA *ccdata); - - /* - * Reset of state. - */ - void (*reset)(OSSL_CC_DATA *ccdata); - - /* - * Escape hatch for option configuration. - * - * params is an array of OSSL_PARAM structures. - * - * Returns 1 on success and 0 on failure. - */ - int (*set_input_params)(OSSL_CC_DATA *ccdata, - const OSSL_PARAM *params); - - /* - * (Re)bind output (diagnostic) information. - * - * params is an array of OSSL_PARAM structures used to output values. The - * storage locations associated with each parameter are stored internally - * and updated whenever the state of the congestion controller is updated; - * thus, the storage locations associated with the OSSL_PARAMs passed in the - * call to this function must remain valid until the congestion controller - * is freed or those parameters are unbound. A given parameter name may be - * bound to only one location at a time. The params structures themselves - * do not need to remain allocated after this call returns. - * - * Returns 1 on success and 0 on failure. - */ - int (*bind_diagnostics)(OSSL_CC_DATA *ccdata, - OSSL_PARAM *params); - - /* - * Unbind diagnostic information. The parameters with the given names are - * unbound, cancelling the effects of a previous call to bind_diagnostic(). - * params is an array of OSSL_PARAMs. The values of the parameters are - * ignored. If a parameter is already unbound, there is no effect for that - * parameter but other parameters are still unbound. - * - * Returns 1 on success or 0 on failure. - */ - int (*unbind_diagnostics)(OSSL_CC_DATA *ccdata, - OSSL_PARAM *params); - - /* - * Returns the amount of additional data (above and beyond the data - * currently in flight) which can be sent in bytes. Returns 0 if no more - * data can be sent at this time. The return value of this method - * can vary as time passes. - */ - uint64_t (*get_tx_allowance)(OSSL_CC_DATA *ccdata); - - /* - * Returns the time at which the return value of get_tx_allowance might be - * higher than its current value. This is not a guarantee and spurious - * wakeups are allowed. Returns ossl_time_infinite() if there is no current - * wakeup deadline. - */ - OSSL_TIME (*get_wakeup_deadline)(OSSL_CC_DATA *ccdata); - - /* - * The On Data Sent event. num_bytes should be the size of the packet in - * bytes (or the aggregate size of multiple packets which have just been - * sent). - */ - int (*on_data_sent)(OSSL_CC_DATA *ccdata, - uint64_t num_bytes); - - /* - * The On Data Acked event. See OSSL_CC_ACK_INFO structure for details - * of the information to be passed. - */ - int (*on_data_acked)(OSSL_CC_DATA *ccdata, - const OSSL_CC_ACK_INFO *info); - - /* - * The On Data Lost event. See OSSL_CC_LOSS_INFO structure for details - * of the information to be passed. - * - * Note: When the ACKM determines that a set of multiple packets has been - * lost, it is useful for a congestion control algorithm to be able to - * process this as a single loss event rather than multiple loss events. - * Thus, calling this function may cause the congestion controller to defer - * state updates under the assumption that subsequent calls to - * on_data_lost() representing further lost packets in the same loss event - * may be forthcoming. Always call on_data_lost_finished() after one or more - * calls to on_data_lost(). - */ - int (*on_data_lost)(OSSL_CC_DATA *ccdata, - const OSSL_CC_LOSS_INFO *info); - - /* - * To be called after a sequence of one or more on_data_lost() calls - * representing multiple packets in a single loss detection incident. - * - * Flags may be 0 or OSSL_CC_LOST_FLAG_PERSISTENT_CONGESTION. - */ - int (*on_data_lost_finished)(OSSL_CC_DATA *ccdata, uint32_t flags); - - /* - * For use when a PN space is invalidated or a packet must otherwise be - * 'undone' for congestion control purposes without acting as a loss signal. - * Only the size of the packet is needed. - */ - int (*on_data_invalidated)(OSSL_CC_DATA *ccdata, - uint64_t num_bytes); - - /* - * Called from the ACKM when detecting an increased ECN-CE value in an ACK - * frame. This indicates congestion. - * - * Note that this differs from the RFC's conceptual segregation of the loss - * detection and congestion controller functions, as in our implementation - * the ACKM is responsible for detecting increases to ECN-CE and simply - * tells the congestion controller when ECN-triggered congestion has - * occurred. This allows a slightly more efficient implementation and - * narrower interface between the ACKM and CC. - */ - int (*on_ecn)(OSSL_CC_DATA *ccdata, - const OSSL_CC_ECN_INFO *info); -}; - -extern const OSSL_CC_METHOD ossl_cc_dummy_method; -extern const OSSL_CC_METHOD ossl_cc_newreno_method; - -# endif - -#endif diff --git a/openssl/include/internal/quic_cfq.h b/openssl/include/internal/quic_cfq.h deleted file mode 100644 index 56ebcb930..000000000 --- a/openssl/include/internal/quic_cfq.h +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_CFQ_H -# define OSSL_QUIC_CFQ_H - -# include -# include "internal/quic_types.h" -# include "internal/quic_predef.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Control Frame Queue Item - * ============================= - * - * The CFQ item structure has a public and a private part. This structure - * documents the public part. - */ -typedef struct quic_cfq_item_st QUIC_CFQ_ITEM; - -struct quic_cfq_item_st { - /* - * These fields are not used by the CFQ, but are a convenience to assist the - * TXPIM in keeping a list of GCR control frames which were sent in a - * packet. They may be used for any purpose. - */ - QUIC_CFQ_ITEM *pkt_prev, *pkt_next; - - /* All other fields are private; use ossl_quic_cfq_item_* accessors. */ -}; - -# define QUIC_CFQ_STATE_NEW 0 -# define QUIC_CFQ_STATE_TX 1 - -/* If set, do not retransmit on loss */ -#define QUIC_CFQ_ITEM_FLAG_UNRELIABLE (1U << 0) - -/* Returns the frame type of a CFQ item. */ -uint64_t ossl_quic_cfq_item_get_frame_type(const QUIC_CFQ_ITEM *item); - -/* Returns a pointer to the encoded buffer of a CFQ item. */ -const unsigned char *ossl_quic_cfq_item_get_encoded(const QUIC_CFQ_ITEM *item); - -/* Returns the length of the encoded buffer in bytes. */ -size_t ossl_quic_cfq_item_get_encoded_len(const QUIC_CFQ_ITEM *item); - -/* Returns the CFQ item state, a QUIC_CFQ_STATE_* value. */ -int ossl_quic_cfq_item_get_state(const QUIC_CFQ_ITEM *item); - -/* Returns the PN space for the CFQ item. */ -uint32_t ossl_quic_cfq_item_get_pn_space(const QUIC_CFQ_ITEM *item); - -/* Returns 1 if this is an unreliable frame. */ -int ossl_quic_cfq_item_is_unreliable(const QUIC_CFQ_ITEM *item); - -/* - * QUIC Control Frame Queue - * ======================== - */ - -QUIC_CFQ *ossl_quic_cfq_new(void); -void ossl_quic_cfq_free(QUIC_CFQ *cfq); - -/* - * Input Side - * ---------- - */ - -/* - * Enqueue a frame to the CFQ. - * - * encoded points to the opaque encoded frame. - * - * free_cb is called by the CFQ when the buffer is no longer needed; - * free_cb_arg is an opaque value passed to free_cb. - * - * priority determines the relative ordering of control frames in a packet. - * Lower numerical values for priority mean that a frame should come earlier in - * a packet. pn_space is a QUIC_PN_SPACE_* value. - * - * On success, returns a QUIC_CFQ_ITEM pointer which acts as a handle to - * the queued frame. On failure, returns NULL. - * - * The frame is initially in the TX state, so there is no need to call - * ossl_quic_cfq_mark_tx() immediately after calling this function. - * - * The frame type is duplicated as the frame_type argument here, even though it - * is also encoded into the buffer. This allows the caller to determine the - * frame type if desired without having to decode the frame. - * - * flags is zero or more QUIC_CFQ_ITEM_FLAG values. - */ -typedef void (cfq_free_cb)(unsigned char *buf, size_t buf_len, void *arg); - -QUIC_CFQ_ITEM *ossl_quic_cfq_add_frame(QUIC_CFQ *cfq, - uint32_t priority, - uint32_t pn_space, - uint64_t frame_type, - uint32_t flags, - const unsigned char *encoded, - size_t encoded_len, - cfq_free_cb *free_cb, - void *free_cb_arg); - -/* - * Effects an immediate transition of the given CFQ item to the TX state. - */ -void ossl_quic_cfq_mark_tx(QUIC_CFQ *cfq, QUIC_CFQ_ITEM *item); - -/* - * Effects an immediate transition of the given CFQ item to the NEW state, - * allowing the frame to be retransmitted. If priority is not UINT32_MAX, - * the priority is changed to the given value. - */ -void ossl_quic_cfq_mark_lost(QUIC_CFQ *cfq, QUIC_CFQ_ITEM *item, - uint32_t priority); - -/* - * Releases a CFQ item. The item may be in either state (NEW or TX) prior to the - * call. The QUIC_CFQ_ITEM pointer must not be used following this call. - */ -void ossl_quic_cfq_release(QUIC_CFQ *cfq, QUIC_CFQ_ITEM *item); - -/* - * Output Side - * ----------- - */ - -/* - * Gets the highest priority CFQ item in the given PN space awaiting - * transmission. If there are none, returns NULL. - */ -QUIC_CFQ_ITEM *ossl_quic_cfq_get_priority_head(const QUIC_CFQ *cfq, - uint32_t pn_space); - -/* - * Given a CFQ item, gets the next CFQ item awaiting transmission in priority - * order in the given PN space. In other words, given the return value of - * ossl_quic_cfq_get_priority_head(), returns the next-lower priority item. - * Returns NULL if the given item is the last item in priority order. - */ -QUIC_CFQ_ITEM *ossl_quic_cfq_item_get_priority_next(const QUIC_CFQ_ITEM *item, - uint32_t pn_space); - -# endif - -#endif diff --git a/openssl/include/internal/quic_channel.h b/openssl/include/internal/quic_channel.h deleted file mode 100644 index 3b373ab68..000000000 --- a/openssl/include/internal/quic_channel.h +++ /dev/null @@ -1,450 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_CHANNEL_H -# define OSSL_QUIC_CHANNEL_H - -# include -# include "internal/quic_types.h" -# include "internal/quic_record_tx.h" -# include "internal/quic_wire.h" -# include "internal/quic_predef.h" -# include "internal/qlog.h" -# include "internal/time.h" -# include "internal/thread.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Channel - * ============ - * - * A QUIC channel (QUIC_CHANNEL) is an object which binds together all of the - * various pieces of QUIC into a single top-level object, and handles connection - * state which is not specific to the client or server roles. In particular, it - * is strictly separated from the libssl front end I/O API personality layer, - * and is not an SSL object. - * - * The name QUIC_CHANNEL is chosen because QUIC_CONNECTION is already in use, - * but functionally these relate to the same thing (a QUIC connection). The use - * of two separate objects ensures clean separation between the API personality - * layer and common code for handling connections, and between the functionality - * which is specific to clients and which is specific to servers, and the - * functionality which is common to both. - * - * The API personality layer provides SSL objects (e.g. a QUIC_CONNECTION) which - * consume a QUIC channel and implement a specific public API. Things which are - * handled by the API personality layer include emulation of blocking semantics, - * handling of SSL object mode flags like non-partial write mode, etc. - * - * Where the QUIC_CHANNEL is used in a server role, there is one QUIC_CHANNEL - * per connection. In the future a QUIC Channel Manager will probably be defined - * to handle ownership of resources which are shared between connections (e.g. - * demuxers). Since we only use server-side functionality for dummy test servers - * for now, which only need to handle one connection at a time, this is not - * currently modelled. - * - * Synchronisation - * --------------- - * - * To support thread assisted mode, QUIC_CHANNEL can be used by multiple - * threads. **It is the caller's responsibility to ensure that the QUIC_CHANNEL - * is only accessed (whether via its methods or via direct access to its state) - * while the channel mutex is held**, except for methods explicitly marked as - * not requiring prior locking. This is an unchecked precondition. - * - * The instantiator of the channel is responsible for providing a suitable - * mutex which then serves as the channel mutex; see QUIC_CHANNEL_ARGS. - */ - -/* - * The function does not acquire the channel mutex and assumes it is already - * held by the calling thread. - * - * Any function tagged with this has the following precondition: - * - * Precondition: must hold channel mutex (unchecked) - */ -# define QUIC_NEEDS_LOCK - -/* - * The function acquires the channel mutex and releases it before returning in - * all circumstances. - * - * Any function tagged with this has the following precondition and - * postcondition: - * - * Precondition: must not hold channel mutex (unchecked) - * Postcondition: channel mutex is not held (by calling thread) - */ -# define QUIC_TAKES_LOCK - -/* - * The function acquires the channel mutex and leaves it acquired - * when returning success. - * - * Any function tagged with this has the following precondition and - * postcondition: - * - * Precondition: must not hold channel mutex (unchecked) - * Postcondition: channel mutex is held by calling thread - * or function returned failure - */ -# define QUIC_ACQUIRES_LOCK - -# define QUIC_TODO_LOCK - -# define QUIC_CHANNEL_STATE_IDLE 0 -# define QUIC_CHANNEL_STATE_ACTIVE 1 -# define QUIC_CHANNEL_STATE_TERMINATING_CLOSING 2 -# define QUIC_CHANNEL_STATE_TERMINATING_DRAINING 3 -# define QUIC_CHANNEL_STATE_TERMINATED 4 - -typedef struct quic_channel_args_st { - /* - * The QUIC_PORT which the channel is to belong to. The lifetime of the - * QUIC_PORT must exceed that of the created channel. - */ - QUIC_PORT *port; - /* LCIDM to register LCIDs with. */ - QUIC_LCIDM *lcidm; - /* SRTM to register SRTs with. */ - QUIC_SRTM *srtm; - - int is_server; - SSL *tls; - - /* Whether to use qlog. */ - int use_qlog; - - /* Title to use for the qlog session, or NULL. */ - const char *qlog_title; -} QUIC_CHANNEL_ARGS; - -/* Represents the cause for a connection's termination. */ -typedef struct quic_terminate_cause_st { - /* - * If we are in a TERMINATING or TERMINATED state, this is the error code - * associated with the error. This field is valid iff we are in the - * TERMINATING or TERMINATED states. - */ - uint64_t error_code; - - /* - * If terminate_app is set and this is nonzero, this is the frame type which - * caused the connection to be terminated. - */ - uint64_t frame_type; - - /* - * Optional reason string. When calling ossl_quic_channel_local_close, if a - * reason string pointer is passed, it is copied and stored inside - * QUIC_CHANNEL for the remainder of the lifetime of the channel object. - * Thus the string pointed to by this value, if non-NULL, is valid for the - * lifetime of the QUIC_CHANNEL object. - */ - const char *reason; - - /* - * Length of reason in bytes. The reason is supposed to contain a UTF-8 - * string but may be arbitrary data if the reason came from the network. - */ - size_t reason_len; - - /* Is this error code in the transport (0) or application (1) space? */ - unsigned int app : 1; - - /* - * If set, the cause of the termination is a received CONNECTION_CLOSE - * frame. Otherwise, we decided to terminate ourselves and sent a - * CONNECTION_CLOSE frame (regardless of whether the peer later also sends - * one). - */ - unsigned int remote : 1; -} QUIC_TERMINATE_CAUSE; - -/* - * Create a new QUIC channel using the given arguments. The argument structure - * does not need to remain allocated. Returns NULL on failure. - * - * Only QUIC_PORT should use this function. - */ -QUIC_CHANNEL *ossl_quic_channel_new(const QUIC_CHANNEL_ARGS *args); - -/* No-op if ch is NULL. */ -void ossl_quic_channel_free(QUIC_CHANNEL *ch); - -/* Set mutator callbacks for test framework support */ -int ossl_quic_channel_set_mutator(QUIC_CHANNEL *ch, - ossl_mutate_packet_cb mutatecb, - ossl_finish_mutate_cb finishmutatecb, - void *mutatearg); - -/* - * Connection Lifecycle Events - * =========================== - * - * Various events that can be raised on the channel by other parts of the QUIC - * implementation. Some of these are suitable for general use by any part of the - * code (e.g. ossl_quic_channel_raise_protocol_error), others are for very - * specific use by particular components only (e.g. - * ossl_quic_channel_on_handshake_confirmed). - */ - -/* - * To be used by a QUIC connection. Starts the channel. For a client-mode - * channel, this starts sending the first handshake layer message, etc. Can only - * be called in the idle state; successive calls are ignored. - */ -int ossl_quic_channel_start(QUIC_CHANNEL *ch); - -/* Start a locally initiated connection shutdown. */ -void ossl_quic_channel_local_close(QUIC_CHANNEL *ch, uint64_t app_error_code, - const char *app_reason); - -/* - * Called when the handshake is confirmed. - */ -int ossl_quic_channel_on_handshake_confirmed(QUIC_CHANNEL *ch); - -/* - * Raises a protocol error. This is intended to be the universal call suitable - * for handling of all peer-triggered protocol violations or errors detected by - * us. We specify a QUIC transport-scope error code and optional frame type - * which was responsible. If a frame type is not applicable, specify zero. The - * reason string is not currently handled, but should be a string of static - * storage duration. If the connection has already terminated due to a previous - * protocol error, this is a no-op; first error wins. - * - * Usually the ossl_quic_channel_raise_protocol_error() function should be used. - * The ossl_quic_channel_raise_protocol_error_loc() function can be used - * directly for passing through existing call site information from an existing - * error. - */ -void ossl_quic_channel_raise_protocol_error_loc(QUIC_CHANNEL *ch, - uint64_t error_code, - uint64_t frame_type, - const char *reason, - ERR_STATE *err_state, - const char *src_file, - int src_line, - const char *src_func); - -#define ossl_quic_channel_raise_protocol_error(ch, error_code, frame_type, reason) \ - ossl_quic_channel_raise_protocol_error_loc((ch), (error_code), \ - (frame_type), \ - (reason), \ - NULL, \ - OPENSSL_FILE, \ - OPENSSL_LINE, \ - OPENSSL_FUNC) - -#define ossl_quic_channel_raise_protocol_error_state(ch, error_code, frame_type, reason, state) \ - ossl_quic_channel_raise_protocol_error_loc((ch), (error_code), \ - (frame_type), \ - (reason), \ - (state), \ - OPENSSL_FILE, \ - OPENSSL_LINE, \ - OPENSSL_FUNC) - - -/* - * Returns 1 if permanent net error was detected on the QUIC_CHANNEL, - * 0 otherwise. - */ -int ossl_quic_channel_net_error(QUIC_CHANNEL *ch); - -/* Restore saved error state (best effort) */ -void ossl_quic_channel_restore_err_state(QUIC_CHANNEL *ch); - -/* For RXDP use. */ -void ossl_quic_channel_on_remote_conn_close(QUIC_CHANNEL *ch, - OSSL_QUIC_FRAME_CONN_CLOSE *f); -void ossl_quic_channel_on_new_conn_id(QUIC_CHANNEL *ch, - OSSL_QUIC_FRAME_NEW_CONN_ID *f); - -/* Temporarily exposed during QUIC_PORT transition. */ -int ossl_quic_channel_on_new_conn(QUIC_CHANNEL *ch, const BIO_ADDR *peer, - const QUIC_CONN_ID *peer_scid, - const QUIC_CONN_ID *peer_dcid); - -/* For use by QUIC_PORT. You should not need to call this directly. */ -void ossl_quic_channel_subtick(QUIC_CHANNEL *ch, QUIC_TICK_RESULT *r, - uint32_t flags); - -/* For use by QUIC_PORT only. */ -void ossl_quic_channel_raise_net_error(QUIC_CHANNEL *ch); - -/* For use by QUIC_PORT only. */ -void ossl_quic_channel_on_stateless_reset(QUIC_CHANNEL *ch); - -void ossl_quic_channel_inject(QUIC_CHANNEL *ch, QUIC_URXE *e); - -/* - * Queries and Accessors - * ===================== - */ - -/* Gets the reactor which can be used to tick/poll on the channel. */ -QUIC_REACTOR *ossl_quic_channel_get_reactor(QUIC_CHANNEL *ch); - -/* Gets the QSM used with the channel. */ -QUIC_STREAM_MAP *ossl_quic_channel_get_qsm(QUIC_CHANNEL *ch); - -/* Gets the statistics manager used with the channel. */ -OSSL_STATM *ossl_quic_channel_get_statm(QUIC_CHANNEL *ch); - -/* - * Gets/sets the current peer address. Generally this should be used before - * starting a channel in client mode. - */ -int ossl_quic_channel_get_peer_addr(QUIC_CHANNEL *ch, BIO_ADDR *peer_addr); -int ossl_quic_channel_set_peer_addr(QUIC_CHANNEL *ch, const BIO_ADDR *peer_addr); - -/* - * Returns an existing stream by stream ID. Returns NULL if the stream does not - * exist. - */ -QUIC_STREAM *ossl_quic_channel_get_stream_by_id(QUIC_CHANNEL *ch, - uint64_t stream_id); - -/* Returns 1 if channel is terminating or terminated. */ -int ossl_quic_channel_is_term_any(const QUIC_CHANNEL *ch); -const QUIC_TERMINATE_CAUSE * -ossl_quic_channel_get_terminate_cause(const QUIC_CHANNEL *ch); -int ossl_quic_channel_is_closing(const QUIC_CHANNEL *ch); -int ossl_quic_channel_is_terminated(const QUIC_CHANNEL *ch); -int ossl_quic_channel_is_active(const QUIC_CHANNEL *ch); -int ossl_quic_channel_is_handshake_complete(const QUIC_CHANNEL *ch); -int ossl_quic_channel_is_handshake_confirmed(const QUIC_CHANNEL *ch); - -QUIC_PORT *ossl_quic_channel_get0_port(QUIC_CHANNEL *ch); -QUIC_ENGINE *ossl_quic_channel_get0_engine(QUIC_CHANNEL *ch); -QUIC_DEMUX *ossl_quic_channel_get0_demux(QUIC_CHANNEL *ch); - -SSL *ossl_quic_channel_get0_ssl(QUIC_CHANNEL *ch); - -/* - * Retrieves a pointer to the channel mutex which was provided at the time the - * channel was instantiated. In order to allow locks to be acquired and released - * with the correct granularity, it is the caller's responsibility to ensure - * this lock is held for write while calling any QUIC_CHANNEL method, except for - * methods explicitly designed otherwise. - * - * This method is thread safe and does not require prior locking. It can also be - * called while the lock is already held. Note that this is simply a convenience - * function to access the mutex which was passed to the channel at instantiation - * time; it does not belong to the channel but rather is presumed to belong to - * the owner of the channel. - */ -CRYPTO_MUTEX *ossl_quic_channel_get_mutex(QUIC_CHANNEL *ch); - -/* - * Creates a new locally-initiated stream in the stream mapper, choosing an - * appropriate stream ID. If is_uni is 1, creates a unidirectional stream, else - * creates a bidirectional stream. Returns NULL on failure. - */ -QUIC_STREAM *ossl_quic_channel_new_stream_local(QUIC_CHANNEL *ch, int is_uni); - -/* - * Creates a new remotely-initiated stream in the stream mapper. The stream ID - * is used to confirm the initiator and determine the stream type. The stream is - * automatically added to the QSM's accept queue. A pointer to the stream is - * also returned. Returns NULL on failure. - */ -QUIC_STREAM *ossl_quic_channel_new_stream_remote(QUIC_CHANNEL *ch, - uint64_t stream_id); - -/* - * Configures incoming stream auto-reject. If enabled, incoming streams have - * both their sending and receiving parts automatically rejected using - * STOP_SENDING and STREAM_RESET frames. aec is the application error - * code to be used for those frames. - */ -void ossl_quic_channel_set_incoming_stream_auto_reject(QUIC_CHANNEL *ch, - int enable, - uint64_t aec); - -/* - * Causes the channel to reject the sending and receiving parts of a stream, - * as though autorejected. Can be used if a stream has already been - * accepted. - */ -void ossl_quic_channel_reject_stream(QUIC_CHANNEL *ch, QUIC_STREAM *qs); - -/* Replace local connection ID in TXP and DEMUX for testing purposes. */ -int ossl_quic_channel_replace_local_cid(QUIC_CHANNEL *ch, - const QUIC_CONN_ID *conn_id); - -/* Setters for the msg_callback and msg_callback_arg */ -void ossl_quic_channel_set_msg_callback(QUIC_CHANNEL *ch, - ossl_msg_cb msg_callback, - SSL *msg_callback_ssl); -void ossl_quic_channel_set_msg_callback_arg(QUIC_CHANNEL *ch, - void *msg_callback_arg); - -/* Testing use only - sets a TXKU threshold packet count override value. */ -void ossl_quic_channel_set_txku_threshold_override(QUIC_CHANNEL *ch, - uint64_t tx_pkt_threshold); - -/* Testing use only - gets current 1-RTT key epochs for QTX and QRX. */ -uint64_t ossl_quic_channel_get_tx_key_epoch(QUIC_CHANNEL *ch); -uint64_t ossl_quic_channel_get_rx_key_epoch(QUIC_CHANNEL *ch); - -/* Artificially trigger a spontaneous TXKU if possible. */ -int ossl_quic_channel_trigger_txku(QUIC_CHANNEL *ch); -int ossl_quic_channel_has_pending(const QUIC_CHANNEL *ch); - -/* Force transmission of an ACK-eliciting packet. */ -int ossl_quic_channel_ping(QUIC_CHANNEL *ch); - -/* - * These queries exist for diagnostic purposes only. They may roll over. - * Do not rely on them for non-testing purposes. - */ -uint16_t ossl_quic_channel_get_diag_num_rx_ack(QUIC_CHANNEL *ch); - -/* - * Diagnostic use only. Gets the current local CID. - */ -void ossl_quic_channel_get_diag_local_cid(QUIC_CHANNEL *ch, QUIC_CONN_ID *cid); - -/* - * Returns 1 if stream count flow control allows us to create a new - * locally-initiated stream. - */ -int ossl_quic_channel_is_new_local_stream_admissible(QUIC_CHANNEL *ch, int is_uni); - -/* - * Returns the number of additional streams that can currently be created based - * on flow control. - */ -uint64_t ossl_quic_channel_get_local_stream_count_avail(const QUIC_CHANNEL *ch, - int is_uni); -uint64_t ossl_quic_channel_get_remote_stream_count_avail(const QUIC_CHANNEL *ch, - int is_uni); - -/* - * Returns 1 if we have generated our local transport parameters yet. - */ -int ossl_quic_channel_have_generated_transport_params(const QUIC_CHANNEL *ch); - -/* Configures the idle timeout to request from peer (milliseconds, 0=no timeout). */ -void ossl_quic_channel_set_max_idle_timeout_request(QUIC_CHANNEL *ch, uint64_t ms); -/* Get the configured idle timeout to request from peer. */ -uint64_t ossl_quic_channel_get_max_idle_timeout_request(const QUIC_CHANNEL *ch); -/* Get the idle timeout requested by the peer. */ -uint64_t ossl_quic_channel_get_max_idle_timeout_peer_request(const QUIC_CHANNEL *ch); -/* Get the idle timeout actually negotiated. */ -uint64_t ossl_quic_channel_get_max_idle_timeout_actual(const QUIC_CHANNEL *ch); - -# endif - -#endif diff --git a/openssl/include/internal/quic_demux.h b/openssl/include/internal/quic_demux.h deleted file mode 100644 index d0781e61c..000000000 --- a/openssl/include/internal/quic_demux.h +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_DEMUX_H -# define OSSL_QUIC_DEMUX_H - -# include -# include "internal/quic_types.h" -# include "internal/quic_predef.h" -# include "internal/bio_addr.h" -# include "internal/time.h" -# include "internal/list.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Demuxer - * ============ - * - * The QUIC connection demuxer is the entity responsible for receiving datagrams - * from the network via a datagram BIO. It parses the headers of the first - * packet in the datagram to determine that packet's DCID and hands off - * processing of the entire datagram to a single callback function which can - * decide how to handle and route the datagram, for example by looking up - * a QRX instance and injecting the URXE into that QRX. - * - * A QRX will typically be instantiated per QUIC connection and contains the - * cryptographic resources needed to decrypt QUIC packets for that connection. - * However, it is up to the callback function to handle routing, for example by - * consulting a LCIDM instance. Thus the demuxer has no specific knowledge of - * any QRX and is not coupled to it. All CID knowledge is also externalised into - * a LCIDM or other CID state tracking object, without the DEMUX being coupled - * to any particular DCID resolution mechanism. - * - * URX Queue - * --------- - * - * Since the demuxer must handle the initial reception of datagrams from the OS, - * RX queue management for new, unprocessed datagrams is also handled by the - * demuxer. - * - * The demuxer maintains a queue of Unprocessed RX Entries (URXEs), which store - * unprocessed (i.e., encrypted, unvalidated) data received from the network. - * The URXE queue is designed to allow multiple datagrams to be received in a - * single call to BIO_recvmmsg, where supported. - * - * One URXE is used per received datagram. Each datagram may contain multiple - * packets, however, this is not the demuxer's concern. QUIC prohibits different - * packets in the same datagram from containing different DCIDs; the demuxer - * only considers the DCID of the first packet in a datagram when deciding how - * to route a received datagram, and it is the responsibility of the QRX to - * enforce this rule. Packets other than the first packet in a datagram are not - * examined by the demuxer, and the demuxer does not perform validation of - * packet headers other than to the minimum extent necessary to extract the - * DCID; further parsing and validation of packet headers is the responsibility - * of the QRX. - * - * Rather than defining an opaque interface, the URXE structure internals - * are exposed. Since the demuxer is only exposed to other parts of the QUIC - * implementation internals, this poses no problem, and has a number of - * advantages: - * - * - Fields in the URXE can be allocated to support requirements in other - * components, like the QRX, which would otherwise have to allocate extra - * memory corresponding to each URXE. - * - * - Other components, like the QRX, can keep the URXE in queues of its own - * when it is not being managed by the demuxer. - * - * URX Queue Structure - * ------------------- - * - * The URXE queue is maintained as a simple doubly-linked list. URXE entries are - * moved between different lists in their lifecycle (for example, from a free - * list to a pending list and vice versa). The buffer into which datagrams are - * received immediately follows this URXE header structure and is part of the - * same allocation. - */ - -/* Maximum number of packets we allow to exist in one datagram. */ -#define QUIC_MAX_PKT_PER_URXE (sizeof(uint64_t) * 8) - -struct quic_urxe_st { - OSSL_LIST_MEMBER(urxe, QUIC_URXE); - - /* - * The URXE data starts after this structure so we don't need a pointer. - * data_len stores the current length (i.e., the length of the received - * datagram) and alloc_len stores the allocation length. The URXE will be - * reallocated if we need a larger allocation than is available, though this - * should not be common as we will have a good idea of worst-case MTUs up - * front. - */ - size_t data_len, alloc_len; - - /* - * Bitfields per packet. processed indicates the packet has been processed - * and must not be processed again, hpr_removed indicates header protection - * has already been removed. Used by QRX only; not used by the demuxer. - */ - uint64_t processed, hpr_removed; - - /* - * This monotonically increases with each datagram received. It is used for - * diagnostic purposes only. - */ - uint64_t datagram_id; - - /* - * Address of peer we received the datagram from, and the local interface - * address we received it on. If local address support is not enabled, local - * is zeroed. - */ - BIO_ADDR peer, local; - - /* - * Time at which datagram was received (or ossl_time_zero()) if a now - * function was not provided). - */ - OSSL_TIME time; - - /* - * Used by the QRX to mark whether a datagram has been deferred. Used by the - * QRX only; not used by the demuxer. - */ - char deferred; - - /* - * Used by the DEMUX to track if a URXE has been handed out. Used primarily - * for debugging purposes. - */ - char demux_state; -}; - -/* Accessors for URXE buffer. */ -static ossl_unused ossl_inline unsigned char * -ossl_quic_urxe_data(const QUIC_URXE *e) -{ - return (unsigned char *)&e[1]; -} - -static ossl_unused ossl_inline unsigned char * -ossl_quic_urxe_data_end(const QUIC_URXE *e) -{ - return ossl_quic_urxe_data(e) + e->data_len; -} - -/* List structure tracking a queue of URXEs. */ -DEFINE_LIST_OF(urxe, QUIC_URXE); -typedef OSSL_LIST(urxe) QUIC_URXE_LIST; - -/* - * List management helpers. These are used by the demuxer but can also be used - * by users of the demuxer to manage URXEs. - */ -void ossl_quic_urxe_remove(QUIC_URXE_LIST *l, QUIC_URXE *e); -void ossl_quic_urxe_insert_head(QUIC_URXE_LIST *l, QUIC_URXE *e); -void ossl_quic_urxe_insert_tail(QUIC_URXE_LIST *l, QUIC_URXE *e); - -/* - * Called when a datagram is received for a given connection ID. - * - * e is a URXE containing the datagram payload. It is permissible for the callee - * to mutate this buffer; once the demuxer calls this callback, it will never - * read the buffer again. - * - * If a DCID was identified for the datagram, dcid is non-NULL; otherwise - * it is NULL. - * - * The callee must arrange for ossl_quic_demux_release_urxe or - * ossl_quic_demux_reinject_urxe to be called on the URXE at some point in the - * future (this need not be before the callback returns). - * - * At the time the callback is made, the URXE will not be in any queue, - * therefore the callee can use the prev and next fields as it wishes. - */ -typedef void (ossl_quic_demux_cb_fn)(QUIC_URXE *e, void *arg, - const QUIC_CONN_ID *dcid); - -/* - * Creates a new demuxer. The given BIO is used to receive datagrams from the - * network using BIO_recvmmsg. short_conn_id_len is the length of destination - * connection IDs used in RX'd packets; it must have the same value for all - * connections used on a socket. default_urxe_alloc_len is the buffer size to - * receive datagrams into; it should be a value large enough to contain any - * received datagram according to local MTUs, etc. - * - * now is an optional function used to determine the time a datagram was - * received. now_arg is an opaque argument passed to the function. If now is - * NULL, ossl_time_zero() is used as the datagram reception time. - */ -QUIC_DEMUX *ossl_quic_demux_new(BIO *net_bio, - size_t short_conn_id_len, - OSSL_TIME (*now)(void *arg), - void *now_arg); - -/* - * Destroy a demuxer. All URXEs must have been released back to the demuxer - * before calling this. No-op if demux is NULL. - */ -void ossl_quic_demux_free(QUIC_DEMUX *demux); - -/* - * Changes the BIO which the demuxer reads from. This also sets the MTU if the - * BIO supports querying the MTU. - */ -void ossl_quic_demux_set_bio(QUIC_DEMUX *demux, BIO *net_bio); - -/* - * Changes the MTU in bytes we use to receive datagrams. - */ -int ossl_quic_demux_set_mtu(QUIC_DEMUX *demux, unsigned int mtu); - -/* - * Set the default packet handler. This is used for incoming packets which don't - * match a registered DCID. This is only needed for servers. If a default packet - * handler is not set, a packet which doesn't match a registered DCID is - * silently dropped. A default packet handler may be unset by passing NULL. - * - * The handler is responsible for ensuring that ossl_quic_demux_reinject_urxe or - * ossl_quic_demux_release_urxe is called on the passed packet at some point in - * the future, which may or may not be before the handler returns. - */ -void ossl_quic_demux_set_default_handler(QUIC_DEMUX *demux, - ossl_quic_demux_cb_fn *cb, - void *cb_arg); - -/* - * Releases a URXE back to the demuxer. No reference must be made to the URXE or - * its buffer after calling this function. The URXE must not be in any queue; - * that is, its prev and next pointers must be NULL. - */ -void ossl_quic_demux_release_urxe(QUIC_DEMUX *demux, - QUIC_URXE *e); - -/* - * Reinjects a URXE which was issued to a registered DCID callback or the - * default packet handler callback back into the pending queue. This is useful - * when a packet has been handled by the default packet handler callback such - * that a DCID has now been registered and can be dispatched normally by DCID. - * Once this has been called, the caller must not touch the URXE anymore and - * must not also call ossl_quic_demux_release_urxe(). - * - * The URXE is reinjected at the head of the queue, so it will be reprocessed - * immediately. - */ -void ossl_quic_demux_reinject_urxe(QUIC_DEMUX *demux, - QUIC_URXE *e); - -/* - * Process any unprocessed RX'd datagrams, by calling registered callbacks by - * connection ID, reading more datagrams from the BIO if necessary. - * - * Returns one of the following values: - * - * QUIC_DEMUX_PUMP_RES_OK - * At least one incoming datagram was processed. - * - * QUIC_DEMUX_PUMP_RES_TRANSIENT_FAIL - * No more incoming datagrams are currently available. - * Call again later. - * - * QUIC_DEMUX_PUMP_RES_PERMANENT_FAIL - * Either the network read BIO has failed in a non-transient fashion, or - * the QUIC implementation has encountered an internal state, assertion - * or allocation error. The caller should tear down the connection - * similarly to in the case of a protocol violation. - * - */ -#define QUIC_DEMUX_PUMP_RES_OK 1 -#define QUIC_DEMUX_PUMP_RES_TRANSIENT_FAIL (-1) -#define QUIC_DEMUX_PUMP_RES_PERMANENT_FAIL (-2) - -int ossl_quic_demux_pump(QUIC_DEMUX *demux); - -/* - * Artificially inject a packet into the demuxer for testing purposes. The - * buffer must not exceed the URXE size being used by the demuxer. - * - * If peer or local are NULL, their respective fields are zeroed in the injected - * URXE. - * - * Returns 1 on success or 0 on failure. - */ -int ossl_quic_demux_inject(QUIC_DEMUX *demux, - const unsigned char *buf, - size_t buf_len, - const BIO_ADDR *peer, - const BIO_ADDR *local); - -/* - * Returns 1 if there are any pending URXEs. - */ -int ossl_quic_demux_has_pending(const QUIC_DEMUX *demux); - -# endif - -#endif diff --git a/openssl/include/internal/quic_engine.h b/openssl/include/internal/quic_engine.h deleted file mode 100644 index 5d06d076b..000000000 --- a/openssl/include/internal/quic_engine.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#ifndef OSSL_QUIC_ENGINE_H -# define OSSL_QUIC_ENGINE_H - -# include - -# include "internal/quic_predef.h" -# include "internal/quic_port.h" -# include "internal/thread_arch.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Engine - * =========== - * - * A QUIC Engine (QUIC_ENGINE) represents an event processing domain for the - * purposes of QUIC and contains zero or more subsidiary QUIC_PORT instances - * (each of which currently represents a UDP socket), each of which in turn - * contains zero or more subsidiary QUIC_CHANNEL instances, each of which - * represents a single QUIC connection. All QUIC_PORT instances must belong - * to a QUIC_ENGINE. - * - * TODO(QUIC SERVER): Currently a QUIC_PORT belongs to a single QUIC_CHANNEL. - * This will cease to be the case once connection migration and/or multipath is - * implemented, so in future a channel might be associated with multiple ports. - * - * A QUIC engine is the root object in a QUIC event domain, and is responsible - * for managing event processing for all QUIC ports and channels (e.g. timeouts, - * clock management, the QUIC_REACTOR instance, etc.). - */ -typedef struct quic_engine_args_st { - OSSL_LIB_CTX *libctx; - const char *propq; - - /* - * This must be a mutex the lifetime of which will exceed that of the engine - * and all ports and channels. The instantiator of the engine is responsible - * for providing a mutex as this makes it easier to handle instantiation and - * teardown of channels in situations potentially requiring locking. - * - * Note that this is a MUTEX not a RWLOCK as it needs to be an OS mutex for - * compatibility with an OS's condition variable wait API, whereas RWLOCK - * may, depending on the build configuration, be implemented using an OS's - * mutex primitive or using its RW mutex primitive. - */ - CRYPTO_MUTEX *mutex; - - OSSL_TIME (*now_cb)(void *arg); - void *now_cb_arg; -} QUIC_ENGINE_ARGS; - -QUIC_ENGINE *ossl_quic_engine_new(const QUIC_ENGINE_ARGS *args); - -void ossl_quic_engine_free(QUIC_ENGINE *qeng); - -/* - * Create a port which is a child of the engine. args->engine shall be NULL. - */ -QUIC_PORT *ossl_quic_engine_create_port(QUIC_ENGINE *qeng, - const QUIC_PORT_ARGS *args); - -/* Gets the mutex used by the engine. */ -CRYPTO_MUTEX *ossl_quic_engine_get0_mutex(QUIC_ENGINE *qeng); - -/* Gets the current time. */ -OSSL_TIME ossl_quic_engine_get_time(QUIC_ENGINE *qeng); - -/* For testing use. While enabled, ticking is not performed. */ -void ossl_quic_engine_set_inhibit_tick(QUIC_ENGINE *qeng, int inhibit); - -/* Gets the reactor which can be used to tick/poll on the port. */ -QUIC_REACTOR *ossl_quic_engine_get0_reactor(QUIC_ENGINE *qeng); - -# endif - -#endif diff --git a/openssl/include/internal/quic_error.h b/openssl/include/internal/quic_error.h deleted file mode 100644 index 86d1c692b..000000000 --- a/openssl/include/internal/quic_error.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_ERROR_H -# define OSSL_QUIC_ERROR_H - -# include -# include - -# ifndef OPENSSL_NO_QUIC - -# define OSSL_QUIC_ERR_CRYPTO_UNEXPECTED_MESSAGE \ - OSSL_QUIC_ERR_CRYPTO_ERR(SSL3_AD_UNEXPECTED_MESSAGE) - -# define OSSL_QUIC_ERR_CRYPTO_MISSING_EXT \ - OSSL_QUIC_ERR_CRYPTO_ERR(TLS13_AD_MISSING_EXTENSION) - -# define OSSL_QUIC_ERR_CRYPTO_NO_APP_PROTO \ - OSSL_QUIC_ERR_CRYPTO_ERR(TLS1_AD_NO_APPLICATION_PROTOCOL) - -const char *ossl_quic_err_to_string(uint64_t error_code); - -# endif - -#endif diff --git a/openssl/include/internal/quic_fc.h b/openssl/include/internal/quic_fc.h deleted file mode 100644 index 923bd43bc..000000000 --- a/openssl/include/internal/quic_fc.h +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_FC_H -# define OSSL_QUIC_FC_H - -# include -# include "internal/time.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * TX Flow Controller (TXFC) - * ========================= - * - * For discussion, see doc/designs/quic-design/quic-fc.md. - */ -typedef struct quic_txfc_st QUIC_TXFC; - -struct quic_txfc_st { - QUIC_TXFC *parent; /* stream-level iff non-NULL */ - uint64_t swm, cwm; - char has_become_blocked; -}; - -/* - * Initialises a TX flow controller. conn_txfc should be non-NULL and point to - * the connection-level flow controller if the TXFC is for stream-level flow - * control, and NULL otherwise. - */ -int ossl_quic_txfc_init(QUIC_TXFC *txfc, QUIC_TXFC *conn_txfc); - -/* - * Gets the parent (i.e., connection-level) TX flow controller. Returns NULL if - * called on a connection-level TX flow controller. - */ -QUIC_TXFC *ossl_quic_txfc_get_parent(QUIC_TXFC *txfc); - -/* - * Bump the credit watermark (CWM) value. This is the 'On TX Window Updated' - * operation. This function is a no-op if it has already been called with an - * equal or higher CWM value. - * - * It returns 1 iff the call resulted in the CWM being bumped and 0 if it was - * not increased because it has already been called with an equal or higher CWM - * value. This is not an error per se but may indicate a local programming error - * or a protocol error in a remote peer. - */ -int ossl_quic_txfc_bump_cwm(QUIC_TXFC *txfc, uint64_t cwm); - -/* - * Get the number of bytes by which we are in credit. This is the number of - * controlled bytes we are allowed to send. (Thus if this function returns 0, we - * are currently blocked.) - * - * If called on a stream-level TXFC, ossl_quic_txfc_get_credit is called on - * the connection-level TXFC as well, and the lesser of the two values is - * returned. The consumed value is the amount already consumed on the connection - * level TXFC. - */ -uint64_t ossl_quic_txfc_get_credit(QUIC_TXFC *txfc, uint64_t consumed); - -/* - * Like ossl_quic_txfc_get_credit(), but when called on a stream-level TXFC, - * retrieves only the stream-level credit value and does not clamp it based on - * connection-level flow control. Any credit value is reduced by the consumed - * amount. - */ -uint64_t ossl_quic_txfc_get_credit_local(QUIC_TXFC *txfc, uint64_t consumed); - -/* - * Consume num_bytes of credit. This is the 'On TX' operation. This should be - * called when we transmit any controlled bytes. Calling this with an argument - * of 0 is a no-op. - * - * We must never transmit more controlled bytes than we are in credit for (see - * the return value of ossl_quic_txfc_get_credit()). If you call this function - * with num_bytes greater than our current credit, this function consumes the - * remainder of the credit and returns 0. This indicates a serious programming - * error on the caller's part. Otherwise, the function returns 1. - * - * If called on a stream-level TXFC, ossl_quic_txfc_consume_credit() is called - * on the connection-level TXFC also. If the call to that function on the - * connection-level TXFC returns zero, this function will also return zero. - */ -int ossl_quic_txfc_consume_credit(QUIC_TXFC *txfc, uint64_t num_bytes); - -/* - * Like ossl_quic_txfc_consume_credit(), but when called on a stream-level TXFC, - * consumes only from the stream-level credit and does not inform the - * connection-level TXFC. - */ -int ossl_quic_txfc_consume_credit_local(QUIC_TXFC *txfc, uint64_t num_bytes); - -/* - * This flag is provided for convenience. A caller is not required to use it. It - * is a boolean flag set whenever our credit drops to zero. If clear is 1, the - * flag is cleared. The old value of the flag is returned. Callers may use this - * to determine if they need to send a DATA_BLOCKED or STREAM_DATA_BLOCKED - * frame, which should contain the value returned by ossl_quic_txfc_get_cwm(). - */ -int ossl_quic_txfc_has_become_blocked(QUIC_TXFC *txfc, int clear); - -/* - * Get the current CWM value. This is mainly only needed when generating a - * DATA_BLOCKED or STREAM_DATA_BLOCKED frame, or for diagnostic purposes. - */ -uint64_t ossl_quic_txfc_get_cwm(QUIC_TXFC *txfc); - -/* - * Get the current spent watermark (SWM) value. This is purely for diagnostic - * use and should not be needed in normal circumstances. - */ -uint64_t ossl_quic_txfc_get_swm(QUIC_TXFC *txfc); - -/* - * RX Flow Controller (RXFC) - * ========================= - */ -typedef struct quic_rxfc_st QUIC_RXFC; - -struct quic_rxfc_st { - /* - * swm is the sent/received watermark, which tracks how much we have - * received from the peer. rwm is the retired watermark, which tracks how - * much has been passed to the application. esrwm is the rwm value at which - * the current auto-tuning epoch started. hwm is the highest stream length - * (STREAM frame offset + payload length) we have seen from a STREAM frame - * yet. - */ - uint64_t cwm, swm, rwm, esrwm, hwm, cur_window_size, max_window_size; - OSSL_TIME epoch_start; - OSSL_TIME (*now)(void *arg); - void *now_arg; - QUIC_RXFC *parent; - unsigned char error_code, has_cwm_changed, is_fin, standalone; -}; - -/* - * Initialises an RX flow controller. conn_rxfc should be non-NULL and point to - * a connection-level RXFC if the RXFC is for stream-level flow control, and - * NULL otherwise. initial_window_size and max_window_size specify the initial - * and absolute maximum window sizes, respectively. Window size values are - * expressed in bytes and determine how much credit the RXFC extends to the peer - * to transmit more data at a time. - */ -int ossl_quic_rxfc_init(QUIC_RXFC *rxfc, QUIC_RXFC *conn_rxfc, - uint64_t initial_window_size, - uint64_t max_window_size, - OSSL_TIME (*now)(void *arg), - void *now_arg); - -/* - * Initialises an RX flow controller which is used by itself and not under a - * connection-level RX flow controller. This can be used for stream count - * enforcement as well as CRYPTO buffer enforcement. - */ -int ossl_quic_rxfc_init_standalone(QUIC_RXFC *rxfc, - uint64_t initial_window_size, - OSSL_TIME (*now)(void *arg), - void *now_arg); - -/* - * Gets the parent (i.e., connection-level) RXFC. Returns NULL if called on a - * connection-level RXFC. - */ -QUIC_RXFC *ossl_quic_rxfc_get_parent(QUIC_RXFC *rxfc); - -/* - * Changes the current maximum window size value. - */ -void ossl_quic_rxfc_set_max_window_size(QUIC_RXFC *rxfc, - size_t max_window_size); - -/* - * To be called whenever a STREAM frame is received. - * - * end is the value (offset + len), where offset is the offset field of the - * STREAM frame and len is the length of the STREAM frame's payload in bytes. - * - * is_fin should be 1 if the STREAM frame had the FIN flag set and 0 otherwise. - * - * This function may be used on a stream-level RXFC only. The connection-level - * RXFC will have its state updated by the stream-level RXFC. - * - * You should check ossl_quic_rxfc_has_error() on both connection-level and - * stream-level RXFCs after calling this function, as an incoming STREAM frame - * may cause flow control limits to be exceeded by an errant peer. This - * function still returns 1 in this case, as this is not a caller error. - * - * Returns 1 on success or 0 on failure. - */ -int ossl_quic_rxfc_on_rx_stream_frame(QUIC_RXFC *rxfc, - uint64_t end, int is_fin); - -/* - * To be called whenever controlled bytes are retired, i.e. when bytes are - * dequeued from a QUIC stream and passed to the application. num_bytes - * is the number of bytes which were passed to the application. - * - * You should call this only on a stream-level RXFC. This function will update - * the connection-level RXFC automatically. - * - * rtt should be the current best understanding of the RTT to the peer, as - * offered by the Statistics Manager. - * - * You should check ossl_quic_rxfc_has_cwm_changed() after calling this - * function, as it may have caused the RXFC to decide to grant more flow control - * credit to the peer. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_rxfc_on_retire(QUIC_RXFC *rxfc, - uint64_t num_bytes, - OSSL_TIME rtt); - -/* - * Returns the current CWM which the RXFC thinks the peer should have. - * - * Note that the RXFC will increase this value in response to events, at which - * time a MAX_DATA or MAX_STREAM_DATA frame must be generated. Use - * ossl_quic_rxfc_has_cwm_changed() to detect this condition. - * - * This value increases monotonically. - */ -uint64_t ossl_quic_rxfc_get_cwm(const QUIC_RXFC *rxfc); - -/* - * Returns the current SWM. This is the total number of bytes the peer has - * transmitted to us. This is intended for diagnostic use only; you should - * not need it. - */ -uint64_t ossl_quic_rxfc_get_swm(const QUIC_RXFC *rxfc); - -/* - * Returns the current RWM. This is the total number of bytes that has been - * retired. This is intended for diagnostic use only; you should not need it. - */ -uint64_t ossl_quic_rxfc_get_rwm(const QUIC_RXFC *rxfc); - -/* - * Returns the current credit. This is the CWM minus the SWM. This is intended - * for diagnostic use only; you should not need it. - */ -uint64_t ossl_quic_rxfc_get_credit(const QUIC_RXFC *rxfc); - -/* - * Returns the CWM changed flag. If clear is 1, the flag is cleared and the old - * value is returned. - */ -int ossl_quic_rxfc_has_cwm_changed(QUIC_RXFC *rxfc, int clear); - -/* - * Returns a QUIC_ERR_* error code if a flow control error has been detected. - * Otherwise, returns QUIC_ERR_NO_ERROR. If clear is 1, the error is cleared - * and the old value is returned. - * - * May return one of the following values: - * - * QUIC_ERR_FLOW_CONTROL_ERROR: - * This indicates a flow control protocol violation by the remote peer; the - * connection should be terminated in this event. - * QUIC_ERR_FINAL_SIZE: - * The peer attempted to change the stream length after ending the stream. - */ -int ossl_quic_rxfc_get_error(QUIC_RXFC *rxfc, int clear); - -/* - * Returns 1 if the RXFC is a stream-level RXFC and the RXFC knows the final - * size for the stream in bytes. If this is the case and final_size is non-NULL, - * writes the final size to *final_size. Otherwise, returns 0. - */ -int ossl_quic_rxfc_get_final_size(const QUIC_RXFC *rxfc, uint64_t *final_size); - -# endif - -#endif diff --git a/openssl/include/internal/quic_fifd.h b/openssl/include/internal/quic_fifd.h deleted file mode 100644 index c1644e4d8..000000000 --- a/openssl/include/internal/quic_fifd.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_FIFD_H -# define OSSL_QUIC_FIFD_H - -# include -# include "internal/quic_types.h" -# include "internal/quic_cfq.h" -# include "internal/quic_ackm.h" -# include "internal/quic_txpim.h" -# include "internal/quic_stream.h" -# include "internal/qlog.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Frame-in-Flight Dispatcher (FIFD) - * ====================================== - */ -struct quic_fifd_st { - /* Internal data; use the ossl_quic_fifd functions. */ - QUIC_CFQ *cfq; - OSSL_ACKM *ackm; - QUIC_TXPIM *txpim; - QUIC_SSTREAM *(*get_sstream_by_id)(uint64_t stream_id, - uint32_t pn_space, - void *arg); - void *get_sstream_by_id_arg; - void (*regen_frame)(uint64_t frame_type, - uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, - void *arg); - void *regen_frame_arg; - void (*confirm_frame)(uint64_t frame_type, - uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, - void *arg); - void *confirm_frame_arg; - void (*sstream_updated)(uint64_t stream_id, - void *arg); - void *sstream_updated_arg; - QLOG *(*get_qlog_cb)(void *arg); - void *get_qlog_cb_arg; -}; - -int ossl_quic_fifd_init(QUIC_FIFD *fifd, - QUIC_CFQ *cfq, - OSSL_ACKM *ackm, - QUIC_TXPIM *txpim, - /* stream_id is UINT64_MAX for the crypto stream */ - QUIC_SSTREAM *(*get_sstream_by_id)(uint64_t stream_id, - uint32_t pn_space, - void *arg), - void *get_sstream_by_id_arg, - /* stream_id is UINT64_MAX if not applicable */ - void (*regen_frame)(uint64_t frame_type, - uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, - void *arg), - void *regen_frame_arg, - void (*confirm_frame)(uint64_t frame_type, - uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, - void *arg), - void *confirm_frame_arg, - void (*sstream_updated)(uint64_t stream_id, - void *arg), - void *sstream_updated_arg, - QLOG *(*get_qlog_cb)(void *arg), - void *get_qlog_cb_arg); - -void ossl_quic_fifd_cleanup(QUIC_FIFD *fifd); /* (no-op) */ - -int ossl_quic_fifd_pkt_commit(QUIC_FIFD *fifd, QUIC_TXPIM_PKT *pkt); - -void ossl_quic_fifd_set_qlog_cb(QUIC_FIFD *fifd, QLOG *(*get_qlog_cb)(void *arg), - void *arg); - -# endif - -#endif diff --git a/openssl/include/internal/quic_lcidm.h b/openssl/include/internal/quic_lcidm.h deleted file mode 100644 index 4911e0423..000000000 --- a/openssl/include/internal/quic_lcidm.h +++ /dev/null @@ -1,257 +0,0 @@ -/* -* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#ifndef OSSL_INTERNAL_QUIC_LCIDM_H -# define OSSL_INTERNAL_QUIC_LCIDM_H -# pragma once - -# include "internal/e_os.h" -# include "internal/time.h" -# include "internal/quic_types.h" -# include "internal/quic_wire.h" -# include "internal/quic_predef.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Local Connection ID Manager - * ================================ - * - * This manages connection IDs for the RX side, which is to say that it issues - * local CIDs (LCIDs) to a peer which that peer can then use to address us via a - * packet DCID. This is as opposed to CID management for the TX side, which - * determines which CIDs we use to transmit based on remote CIDs (RCIDs) the - * peer sent to us. - * - * An opaque pointer can be associated with each LCID. Pointer identity - * (equality) is used to distinguish distinct connections. - * - * LCIDs fall into three categories: - * - * 1. A client's Initial ODCID (1) - * 2. Our local Initial SCID (1) - * 3. A CID issued via a NEW_CONNECTION_ID frame (n) - * 4. A server's Retry SCID (0..1) - * - * (1) is enrolled using ossl_quic_lcidm_enrol_odcid() and retired by the time - * of handshake completion at the latest. It is needed in case the first - * response packet from a server is lost and the client keeps using its Initial - * ODCID. There is never more than one of these, and no sequence number is - * associated with this temporary LCID. - * - * (2) is created by a client when it begins connecting, or by a server when it - * responds to a new connection request. In the latter case, it is generated by - * the server as the preferred DCID for traffic directed towards it. A client - * should switch to using this as a RCID as soon as it receives a valid packet - * from the server. This LCID has a sequence number of 0. - * - * (3) is created when we issue a NEW_CONNECTION_ID frame. Arbitrarily many of - * these can exist. - * - * (4) is a special case. When a server issues a retry it generates a new SCID - * much as it does for (2). However since retries are supposed to be stateless, - * we don't actually register it as an LCID. When the client subsequently - * replies with an Initial packet with token in response to the Retry, the - * server will handle this as a new connection attempt due to not recognising - * the DCID, which is what we want anyway. (The Retry SCID is subsequently - * validated as matching the new Initial ODCID via attestation in the encrypted - * contents of the opaque retry token.) Thus, the LCIDM is not actually involved - * at all here. - * - * Retirement is as follows: - * - * (1) is retired automatically when we know it won't be needed anymore. This is - * when the handshake is completed at the latest, and could potentially be - * earlier. - * - * Both (2) and (3) are retired normally via RETIRE_CONNECTION_ID frames, as it - * has a sequence number of 0. - * - * - * ODCID Peculiarities - * ------------------- - * - * Almost all LCIDs are issued by the receiver responsible for routing them, - * which means that almost all LCIDs will have the same length (specified in - * lcid_len below). The only exception to this is (1); the ODCID is the only - * case where we recognise an LCID we didn't ourselves generate. Since an ODCID - * is chosen by the peer, it can be any length and doesn't necessarily match the - * length we use for LCIDs we generate ourselves. - * - * Since DCID decoding for short-header packets requires an implicitly known - * DCID length, it logically follows that an ODCID can never be used in a 1-RTT - * packet. This is fine as by the time the 1-RTT EL is reached the peer should - * already have switched away from the ODCID to a CID we generated ourselves, - * and if this has not happened we can consider that a protocol violation. - * - * In any case, this means that the LCIDM must necessarily support LCIDs of - * different lengths, even if it always generates LCIDs of a given length. - * - * An ODCID has no sequence number associated with it. It is the only CID to - * lack one. - */ - -/* - * Creates a new LCIDM. lcid_len is the length to use for LCIDs in bytes, which - * may be zero. - * - * Returns NULL on failure. - */ -QUIC_LCIDM *ossl_quic_lcidm_new(OSSL_LIB_CTX *libctx, size_t lcid_len); - -/* Frees a LCIDM. */ -void ossl_quic_lcidm_free(QUIC_LCIDM *lcidm); - -/* Gets the local CID length this LCIDM was configured to use. */ -size_t ossl_quic_lcidm_get_lcid_len(const QUIC_LCIDM *lcidm); - -/* - * Determines the number of active LCIDs (i.e,. LCIDs which can be used for - * reception) currently associated with the given opaque pointer. - */ -size_t ossl_quic_lcidm_get_num_active_lcid(const QUIC_LCIDM *lcidm, - void *opaque); - -/* - * Enrol an Initial ODCID sent by the peer. This is the DCID in the first - * Initial packet sent by a client. When we receive a client's first Initial - * packet, we immediately respond with our own SCID (generated using - * ossl_quic_lcidm_generate_initial) to tell the client to switch to using that, - * so ideally the ODCID will only be used for a single packet. However since - * that response might be lost, we also need to accept additional packets using - * the ODCID and need to make sure they get routed to the same connection and - * not interpreted as another new connection attempt. Thus before the CID - * switchover is confirmed, we also have to handle incoming packets addressed to - * the ODCID. This function is used to temporarily enroll the ODCID for a - * connection. Such a LCID is considered to have a sequence number of - * LCIDM_ODCID_SEQ_NUM internally for our purposes. - * - * Note that this is the *only* circumstance where we recognise an LCID we did - * not generate ourselves, or allow an LCID with a different length to lcid_len. - * - * An ODCID MUST be at least 8 bytes in length (RFC 9000 s. 7.2). - * - * This function may only be called once for a given connection. - * Returns 1 on success or 0 on failure. - */ -int ossl_quic_lcidm_enrol_odcid(QUIC_LCIDM *lcidm, void *opaque, - const QUIC_CONN_ID *initial_odcid); - -/* - * Retire a previously enrolled ODCID for a connection. This is generally done - * when we know the peer won't be using it any more (when the handshake is - * completed at the absolute latest, possibly earlier). - * - * Returns 1 if there was an enrolled ODCID which was retired and 0 if there was - * not or on other failure. - */ -int ossl_quic_lcidm_retire_odcid(QUIC_LCIDM *lcidm, void *opaque); - -/* - * Create the first LCID for a given opaque pointer. The generated LCID is - * written to *initial_lcid and associated with the given opaque pointer. - * - * After this function returns successfully, the caller can for example - * register the new LCID with a DEMUX. - * - * May not be called more than once for a given opaque pointer value. - */ -int ossl_quic_lcidm_generate_initial(QUIC_LCIDM *lcidm, - void *opaque, - QUIC_CONN_ID *initial_lcid); - -/* - * Create a subsequent LCID for a given opaque pointer. The information needed - * for a NEW_CONN_ID frame informing the peer of the new LCID, including the - * LCID itself, is written to *ncid_frame. - * - * ncid_frame->stateless_reset is not initialised and the caller is responsible - * for setting it. - * - * After this function returns successfully, the caller can for example - * register the new LCID with a DEMUX and queue the NEW_CONN_ID frame. - */ -int ossl_quic_lcidm_generate(QUIC_LCIDM *lcidm, - void *opaque, - OSSL_QUIC_FRAME_NEW_CONN_ID *ncid_frame); - -/* - * Retire up to one LCID for a given opaque pointer value. Called repeatedly to - * handle a RETIRE_CONN_ID frame. - * - * If containing_pkt_dcid is non-NULL, this function enforces the requirement - * that a CID not be retired by a packet using that CID as the DCID. If - * containing_pkt_dcid is NULL, this check is skipped. - * - * If a LCID is retired as a result of a call to this function, the LCID which - * was retired is written to *retired_lcid, the sequence number of the LCID is - * written to *retired_seq_num and *did_retire is set to 1. Otherwise, - * *did_retire is set to 0. This enables a caller to e.g. unregister the LCID - * from a DEMUX. A caller should call this function repeatedly until the - * function returns with *did_retire set to 0. - * - * This call is likely to cause the value returned by - * ossl_quic_lcidm_get_num_active_lcid() to go down. A caller may wish to call - * ossl_quic_lcidm_generate() repeatedly to bring the number of active LCIDs - * back up to some threshold in response after calling this function. - * - * Returns 1 on success and 0 on failure. If arguments are valid but zero LCIDs - * are retired, this is considered a success condition. - */ -int ossl_quic_lcidm_retire(QUIC_LCIDM *lcidm, - void *opaque, - uint64_t retire_prior_to, - const QUIC_CONN_ID *containing_pkt_dcid, - QUIC_CONN_ID *retired_lcid, - uint64_t *retired_seq_num, - int *did_retire); - -/* - * Cull all LCIDM state relating to a given opaque pointer value. This is useful - * if connection state is spontaneously freed. The caller is responsible for - * e.g. DEMUX state updates. - */ -int ossl_quic_lcidm_cull(QUIC_LCIDM *lcidm, void *opaque); - -/* - * Lookup a LCID. If the LCID is found, writes the associated opaque pointer to - * *opaque and the associated sequence number to *seq_num. Returns 1 on success - * and 0 if an entry is not found. An output argument may be set to NULL if its - * value is not required. - * - * If the LCID is for an Initial ODCID, *seq_num is set to - * LCIDM_ODCID_SEQ_NUM. - */ -#define LCIDM_ODCID_SEQ_NUM UINT64_MAX - -int ossl_quic_lcidm_lookup(QUIC_LCIDM *lcidm, - const QUIC_CONN_ID *lcid, - uint64_t *seq_num, - void **opaque); - -/* - * Debug call to manually remove a specific LCID. Should not be needed in normal - * usage. Returns 1 if the LCID was successfully found and removed and 0 - * otherwise. - */ -int ossl_quic_lcidm_debug_remove(QUIC_LCIDM *lcidm, - const QUIC_CONN_ID *lcid); - -/* - * Debug call to manually add a numbered LCID with a specific CID value and - * sequence number. Should not be needed in normal usage. Returns 1 on success - * and 0 on failure. - */ -int ossl_quic_lcidm_debug_add(QUIC_LCIDM *lcidm, void *opaque, - const QUIC_CONN_ID *lcid, - uint64_t seq_num); - -# endif - -#endif diff --git a/openssl/include/internal/quic_port.h b/openssl/include/internal/quic_port.h deleted file mode 100644 index bcb578c3f..000000000 --- a/openssl/include/internal/quic_port.h +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#ifndef OSSL_QUIC_PORT_H -# define OSSL_QUIC_PORT_H - -# include -# include "internal/quic_types.h" -# include "internal/quic_reactor.h" -# include "internal/quic_demux.h" -# include "internal/quic_predef.h" -# include "internal/thread_arch.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Port - * ========= - * - * A QUIC Port (QUIC_PORT) represents a single UDP network socket and contains - * zero or more subsidiary QUIC_CHANNEL instances, each of which represents a - * single QUIC connection. All QUIC_CHANNEL instances must belong to a - * QUIC_PORT. - * - * A QUIC port is responsible for managing a set of channels which all use the - * same UDP socket, and (in future) for automatically creating new channels when - * incoming connections are received. - * - * In order to retain compatibility with QUIC_TSERVER, it also supports a point - * of legacy compatibility where a caller can create an incoming (server role) - * channel and that channel will be automatically be bound to the next incoming - * connection. In the future this will go away once QUIC_TSERVER is removed. - * - * All QUIC_PORT instances are created by a QUIC_ENGINE. - */ -typedef struct quic_port_args_st { - /* The engine which the QUIC port is to be a child of. */ - QUIC_ENGINE *engine; - - /* - * This SSL_CTX will be used when constructing the handshake layer object - * inside newly created channels. - */ - SSL_CTX *channel_ctx; - - /* - * If 1, this port is to be used for multiple connections, so - * non-zero-length CIDs should be used. If 0, this port will only be used - * for a single connection, so a zero-length local CID can be used. - */ - int is_multi_conn; -} QUIC_PORT_ARGS; - -/* Only QUIC_ENGINE should use this function. */ -QUIC_PORT *ossl_quic_port_new(const QUIC_PORT_ARGS *args); - -void ossl_quic_port_free(QUIC_PORT *port); - -/* - * Operations - * ========== - */ - -/* Create an outgoing channel using this port. */ -QUIC_CHANNEL *ossl_quic_port_create_outgoing(QUIC_PORT *port, SSL *tls); - -/* - * Create an incoming channel using this port. - * - * TODO(QUIC SERVER): temporary TSERVER use only - will be removed. - */ -QUIC_CHANNEL *ossl_quic_port_create_incoming(QUIC_PORT *port, SSL *tls); - -/* - * Queries and Accessors - * ===================== - */ - -/* Gets/sets the underlying network read and write BIO. */ -BIO *ossl_quic_port_get_net_rbio(QUIC_PORT *port); -BIO *ossl_quic_port_get_net_wbio(QUIC_PORT *port); -int ossl_quic_port_set_net_rbio(QUIC_PORT *port, BIO *net_rbio); -int ossl_quic_port_set_net_wbio(QUIC_PORT *port, BIO *net_wbio); - -/* - * Re-poll the network BIOs already set to determine if their support - * for polling has changed. - */ -int ossl_quic_port_update_poll_descriptors(QUIC_PORT *port); - -/* Gets the engine which this port is a child of. */ -QUIC_ENGINE *ossl_quic_port_get0_engine(QUIC_PORT *port); - -/* Gets the reactor which can be used to tick/poll on the port. */ -QUIC_REACTOR *ossl_quic_port_get0_reactor(QUIC_PORT *port); - -/* Gets the demuxer belonging to the port. */ -QUIC_DEMUX *ossl_quic_port_get0_demux(QUIC_PORT *port); - -/* Gets the mutex used by the port. */ -CRYPTO_MUTEX *ossl_quic_port_get0_mutex(QUIC_PORT *port); - -/* Gets the current time. */ -OSSL_TIME ossl_quic_port_get_time(QUIC_PORT *port); - -int ossl_quic_port_get_rx_short_dcid_len(const QUIC_PORT *port); -int ossl_quic_port_get_tx_init_dcid_len(const QUIC_PORT *port); - -/* Returns 1 if the port is running/healthy, 0 if it has failed. */ -int ossl_quic_port_is_running(const QUIC_PORT *port); - -/* - * Restores port-level error to the error stack. To be called only if - * the port is no longer running. - */ -void ossl_quic_port_restore_err_state(const QUIC_PORT *port); - -/* For use by QUIC_ENGINE. You should not need to call this directly. */ -void ossl_quic_port_subtick(QUIC_PORT *port, QUIC_TICK_RESULT *r, - uint32_t flags); - -/* - * Events - * ====== - */ - -/* - * Called if a permanent network error occurs. Terminates all channels - * immediately. triggering_ch is an optional argument designating - * a channel which encountered the network error. - */ -void ossl_quic_port_raise_net_error(QUIC_PORT *port, - QUIC_CHANNEL *triggering_ch); - -# endif - -#endif diff --git a/openssl/include/internal/quic_predef.h b/openssl/include/internal/quic_predef.h deleted file mode 100644 index 7c7567b9c..000000000 --- a/openssl/include/internal/quic_predef.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_PREDEF_H -# define OSSL_QUIC_PREDEF_H - -# ifndef OPENSSL_NO_QUIC - -typedef struct quic_port_st QUIC_PORT; -typedef struct quic_channel_st QUIC_CHANNEL; -typedef struct quic_tls_st QUIC_TLS; -typedef struct quic_txpim_st QUIC_TXPIM; -typedef struct quic_fifd_st QUIC_FIFD; -typedef struct quic_cfq_st QUIC_CFQ; -typedef struct ossl_quic_tx_packetiser_st OSSL_QUIC_TX_PACKETISER; -typedef struct ossl_ackm_st OSSL_ACKM; -typedef struct quic_srt_elem_st QUIC_SRT_ELEM; -typedef struct ossl_cc_data_st OSSL_CC_DATA; -typedef struct ossl_cc_method_st OSSL_CC_METHOD; -typedef struct quic_stream_map_st QUIC_STREAM_MAP; -typedef struct quic_stream_st QUIC_STREAM; -typedef struct quic_sstream_st QUIC_SSTREAM; -typedef struct quic_rstream_st QUIC_RSTREAM; -typedef struct quic_reactor_st QUIC_REACTOR; -typedef struct ossl_statm_st OSSL_STATM; -typedef struct quic_demux_st QUIC_DEMUX; -typedef struct ossl_qrx_pkt_st OSSL_QRX_PKT; -typedef struct ossl_qtx_pkt_st OSSL_QTX_PKT; -typedef struct quic_tick_result_st QUIC_TICK_RESULT; -typedef struct quic_srtm_st QUIC_SRTM; -typedef struct quic_lcidm_st QUIC_LCIDM; -typedef struct quic_urxe_st QUIC_URXE; -typedef struct quic_engine_st QUIC_ENGINE; - -# endif - -#endif diff --git a/openssl/include/internal/quic_rcidm.h b/openssl/include/internal/quic_rcidm.h deleted file mode 100644 index fd102241b..000000000 --- a/openssl/include/internal/quic_rcidm.h +++ /dev/null @@ -1,185 +0,0 @@ -/* -* Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#ifndef OSSL_INTERNAL_QUIC_RCIDM_H -# define OSSL_INTERNAL_QUIC_RCIDM_H -# pragma once - -# include "internal/e_os.h" -# include "internal/time.h" -# include "internal/quic_types.h" -# include "internal/quic_wire.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Remote Connection ID Manager - * ================================= - * - * This manages connection IDs for the TX side. The RCIDM tracks remote CIDs - * (RCIDs) which a peer has issued to us and which we can use as the DCID of - * packets we transmit. It is entirely separate from the LCIDM, which handles - * routing received packets by their DCIDs. - * - * RCIDs fall into four categories: - * - * 1. A client's Initial ODCID (0..1) - * 2. A peer's Initial SCID (1) - * 3. A server's Retry SCID (0..1) - * 4. A CID issued via a NEW_CONNECTION_ID frame (n) - * - * Unlike a LCIDM, which is per port, a RCIDM is per connection, as there is no - * need for routing of outgoing packets. - */ -typedef struct quic_rcidm_st QUIC_RCIDM; - -/* - * Creates a new RCIDM. Returns NULL on failure. - * - * For a client, initial_odcid is the client's Initial ODCID. - * For a server, initial_odcid is NULL. - */ -QUIC_RCIDM *ossl_quic_rcidm_new(const QUIC_CONN_ID *initial_odcid); - -/* Frees a RCIDM. */ -void ossl_quic_rcidm_free(QUIC_RCIDM *rcidm); - -/* - * CID Events - * ========== - */ - -/* - * To be called by a client when a server responds to the first Initial packet - * sent with its own Initial packet with its own SCID; or to be called by a - * server when we first get an Initial packet from a client with the client's - * supplied SCID. The added RCID implicitly has a sequence number of 0. - * - * We immediately switch to using this SCID as our preferred RCID. This SCID - * must be enrolled using this function. May only be called once. - */ -int ossl_quic_rcidm_add_from_initial(QUIC_RCIDM *rcidm, - const QUIC_CONN_ID *rcid); - -/* - * To be called by a client when a server responds to the first Initial packet - * sent with a Retry packet with its own SCID (the "Retry ODCID"). We - * immediately switch to using this SCID as our preferred RCID when conducting - * the retry. This SCID must be enrolled using this function. May only be called - * once. The added RCID has no sequence number associated with it as it is - * essentially a new ODCID (hereafter a Retry ODCID). - * - * Not for server use. - */ -int ossl_quic_rcidm_add_from_server_retry(QUIC_RCIDM *rcidm, - const QUIC_CONN_ID *retry_odcid); - -/* - * Processes an incoming NEW_CONN_ID frame, recording the new CID as a potential - * RCID. The RCIDM retirement mechanism is ratcheted according to the - * ncid->retire_prior_to field. The stateless_reset field is ignored; the caller - * is responsible for handling it separately. - */ -int ossl_quic_rcidm_add_from_ncid(QUIC_RCIDM *rcidm, - const OSSL_QUIC_FRAME_NEW_CONN_ID *ncid); - -/* - * Other Events - * ============ - */ - -/* - * Notifies the RCIDM that the handshake for a connection is complete. - * Should only be called once; further calls are ignored. - * - * This may influence the RCIDM's RCID change policy. - */ -void ossl_quic_rcidm_on_handshake_complete(QUIC_RCIDM *rcidm); - -/* - * Notifies the RCIDM that one or more packets have been sent. - * - * This may influence the RCIDM's RCID change policy. - */ -void ossl_quic_rcidm_on_packet_sent(QUIC_RCIDM *rcidm, uint64_t num_packets); - -/* - * Manually request switching to a new RCID as soon as possible. - */ -void ossl_quic_rcidm_request_roll(QUIC_RCIDM *rcidm); - -/* - * Queries - * ======= - */ - -/* - * The RCIDM decides when it will never use a given RCID again. When it does - * this, it outputs the sequence number of that RCID using this function, which - * pops from a logical queue of retired RCIDs. The caller is responsible - * for polling this function and generating Retire CID frames from the result. - * - * If nothing needs doing and the queue is empty, this function returns 0. If - * there is an RCID which needs retiring, the sequence number of that RCID is - * written to *seq_num (if seq_num is non-NULL) and this function returns 1. The - * queue entry is popped (and the caller is thus assumed to have taken - * responsibility for transmitting the necessary Retire CID frame). - * - * Note that the caller should not transmit a Retire CID frame immediately as - * packets using the RCID may still be in flight. The caller must determine an - * appropriate delay using knowledge of network conditions (RTT, etc.) which is - * outside the scope of the RCIDM. The caller is responsible for implementing - * this delay based on the last time a packet was transmitted using the RCID - * being retired. - */ -int ossl_quic_rcidm_pop_retire_seq_num(QUIC_RCIDM *rcid, uint64_t *seq_num); - -/* - * Like ossl_quic_rcidm_pop_retire_seq_num, but does not pop the item from the - * queue. If this call succeeds, the next call to - * ossl_quic_rcidm_pop_retire_seq_num is guaranteed to output the same sequence - * number. - */ -int ossl_quic_rcidm_peek_retire_seq_num(QUIC_RCIDM *rcid, uint64_t *seq_num); - -/* - * Writes the DCID preferred for a newly transmitted packet at this time to - * *tx_dcid. This function should be called to determine what DCID to use when - * transmitting a packet to the peer. The RCIDM may implement arbitrary policy - * to decide when to change the preferred RCID. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_rcidm_get_preferred_tx_dcid(QUIC_RCIDM *rcidm, - QUIC_CONN_ID *tx_dcid); - -/* - * Returns 1 if the value output by ossl_quic_rcidm_get_preferred_tx_dcid() has - * changed since the last call to this function with clear set. If clear is set, - * clears the changed flag. Returns the old value of the changed flag. - */ -int ossl_quic_rcidm_get_preferred_tx_dcid_changed(QUIC_RCIDM *rcidm, - int clear); - -/* - * Returns the number of active numbered RCIDs we have. Note that this includes - * RCIDs on the retir*ing* queue accessed via - * ossl_quic_rcidm_pop_retire_seq_num() as these are still active until actually - * retired. - */ -size_t ossl_quic_rcidm_get_num_active(const QUIC_RCIDM *rcidm); - -/* - * Returns the number of retir*ing* numbered RCIDs we have. - */ -size_t ossl_quic_rcidm_get_num_retiring(const QUIC_RCIDM *rcidm); - -# endif - -#endif diff --git a/openssl/include/internal/quic_reactor.h b/openssl/include/internal/quic_reactor.h deleted file mode 100644 index a6fdb7d12..000000000 --- a/openssl/include/internal/quic_reactor.h +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#ifndef OSSL_QUIC_REACTOR_H -# define OSSL_QUIC_REACTOR_H - -# include "internal/time.h" -# include "internal/sockets.h" -# include "internal/quic_predef.h" -# include "internal/thread_arch.h" -# include - -# ifndef OPENSSL_NO_QUIC - -/* - * Core I/O Reactor Framework - * ========================== - * - * Manages use of async network I/O which the QUIC stack is built on. The core - * mechanic looks like this: - * - * - There is a pollable FD for both the read and write side respectively. - * Readability and writeability of these FDs respectively determines when - * network I/O is available. - * - * - The reactor can export these FDs to the user, as well as flags indicating - * whether the user should listen for readability, writeability, or neither. - * - * - The reactor can export a timeout indication to the user, indicating when - * the reactor should be called (via libssl APIs) regardless of whether - * the network socket has become ready. - * - * The reactor is based around a tick callback which is essentially the mutator - * function. The mutator attempts to do whatever it can, attempting to perform - * network I/O to the extent currently feasible. When done, the mutator returns - * information to the reactor indicating when it should be woken up again: - * - * - Should it be woken up when network RX is possible? - * - Should it be woken up when network TX is possible? - * - Should it be woken up no later than some deadline X? - * - * The intention is that ALL I/O-related SSL_* functions with side effects (e.g. - * SSL_read/SSL_write) consist of three phases: - * - * - Optionally mutate the QUIC machine's state. - * - Optionally tick the QUIC reactor. - * - Optionally mutate the QUIC machine's state. - * - * For example, SSL_write is a mutation (appending to a stream buffer) followed - * by an optional tick (generally expected as we may want to send the data - * immediately, though not strictly needed if transmission is being deferred due - * to Nagle's algorithm, etc.). - * - * SSL_read is also a mutation and in principle does not need to tick the - * reactor, but it generally will anyway to ensure that the reactor is regularly - * ticked by an application which is only reading and not writing. - * - * If the SSL object is being used in blocking mode, SSL_read may need to block - * if no data is available yet, and SSL_write may need to block if buffers - * are full. - * - * The internals of the QUIC I/O engine always use asynchronous I/O. If the - * application desires blocking semantics, we handle this by adding a blocking - * adaptation layer on top of our internal asynchronous I/O API as exposed by - * the reactor interface. - */ -struct quic_tick_result_st { - char net_read_desired; - char net_write_desired; - OSSL_TIME tick_deadline; -}; - -static ossl_inline ossl_unused void -ossl_quic_tick_result_merge_into(QUIC_TICK_RESULT *r, - const QUIC_TICK_RESULT *src) -{ - r->net_read_desired = r->net_read_desired || src->net_read_desired; - r->net_write_desired = r->net_write_desired || src->net_write_desired; - r->tick_deadline = ossl_time_min(r->tick_deadline, src->tick_deadline); -} - -struct quic_reactor_st { - /* - * BIO poll descriptors which can be polled. poll_r is a poll descriptor - * which becomes readable when the QUIC state machine can potentially do - * work, and poll_w is a poll descriptor which becomes writable when the - * QUIC state machine can potentially do work. Generally, either of these - * conditions means that SSL_tick() should be called, or another SSL - * function which implicitly calls SSL_tick() (e.g. SSL_read/SSL_write()). - */ - BIO_POLL_DESCRIPTOR poll_r, poll_w; - OSSL_TIME tick_deadline; /* ossl_time_infinite() if none currently applicable */ - - void (*tick_cb)(QUIC_TICK_RESULT *res, void *arg, uint32_t flags); - void *tick_cb_arg; - - /* - * These are true if we would like to know when we can read or write from - * the network respectively. - */ - unsigned int net_read_desired : 1; - unsigned int net_write_desired : 1; - - /* - * Are the read and write poll descriptors we are currently configured with - * things we can actually poll? - */ - unsigned int can_poll_r : 1; - unsigned int can_poll_w : 1; -}; - -void ossl_quic_reactor_init(QUIC_REACTOR *rtor, - void (*tick_cb)(QUIC_TICK_RESULT *res, void *arg, - uint32_t flags), - void *tick_cb_arg, - OSSL_TIME initial_tick_deadline); - -void ossl_quic_reactor_set_poll_r(QUIC_REACTOR *rtor, - const BIO_POLL_DESCRIPTOR *r); - -void ossl_quic_reactor_set_poll_w(QUIC_REACTOR *rtor, - const BIO_POLL_DESCRIPTOR *w); - -const BIO_POLL_DESCRIPTOR *ossl_quic_reactor_get_poll_r(const QUIC_REACTOR *rtor); -const BIO_POLL_DESCRIPTOR *ossl_quic_reactor_get_poll_w(const QUIC_REACTOR *rtor); - -int ossl_quic_reactor_can_poll_r(const QUIC_REACTOR *rtor); -int ossl_quic_reactor_can_poll_w(const QUIC_REACTOR *rtor); - -int ossl_quic_reactor_can_support_poll_descriptor(const QUIC_REACTOR *rtor, - const BIO_POLL_DESCRIPTOR *d); - -int ossl_quic_reactor_net_read_desired(QUIC_REACTOR *rtor); -int ossl_quic_reactor_net_write_desired(QUIC_REACTOR *rtor); - -OSSL_TIME ossl_quic_reactor_get_tick_deadline(QUIC_REACTOR *rtor); - -/* - * Do whatever work can be done, and as much work as can be done. This involves - * e.g. seeing if we can read anything from the network (if we want to), seeing - * if we can write anything to the network (if we want to), etc. - * - * If the CHANNEL_ONLY flag is set, this indicates that we should only - * touch state which is synchronised by the channel mutex. - */ -#define QUIC_REACTOR_TICK_FLAG_CHANNEL_ONLY (1U << 0) - -int ossl_quic_reactor_tick(QUIC_REACTOR *rtor, uint32_t flags); - -/* - * Blocking I/O Adaptation Layer - * ============================= - * - * The blocking I/O adaptation layer implements blocking I/O on top of our - * asynchronous core. - * - * The core mechanism is block_until_pred(), which does not return until pred() - * returns a value other than 0. The blocker uses OS I/O synchronisation - * primitives (e.g. poll(2)) and ticks the reactor until the predicate is - * satisfied. The blocker is not required to call pred() more than once between - * tick calls. - * - * When pred returns a non-zero value, that value is returned by this function. - * This can be used to allow pred() to indicate error conditions and short - * circuit the blocking process. - * - * A return value of -1 is reserved for network polling errors. Therefore this - * return value should not be used by pred() if ambiguity is not desired. Note - * that the predicate function can always arrange its own output mechanism, for - * example by passing a structure of its own as the argument. - * - * If the SKIP_FIRST_TICK flag is set, the first call to reactor_tick() before - * the first call to pred() is skipped. This is useful if it is known that - * ticking the reactor again will not be useful (e.g. because it has already - * been done). - * - * This function assumes a write lock is held for the entire QUIC_CHANNEL. If - * mutex is non-NULL, it must be a lock currently held for write; it will be - * unlocked during any sleep, and then relocked for write afterwards. - * - * Precondition: mutex is NULL or is held for write (unchecked) - * Postcondition: mutex is NULL or is held for write (unless - * CRYPTO_THREAD_write_lock fails) - */ -#define SKIP_FIRST_TICK (1U << 0) - -int ossl_quic_reactor_block_until_pred(QUIC_REACTOR *rtor, - int (*pred)(void *arg), void *pred_arg, - uint32_t flags, - CRYPTO_MUTEX *mutex); - -# endif - -#endif diff --git a/openssl/include/internal/quic_record_rx.h b/openssl/include/internal/quic_record_rx.h deleted file mode 100644 index 001509bd5..000000000 --- a/openssl/include/internal/quic_record_rx.h +++ /dev/null @@ -1,548 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_RECORD_RX_H -# define OSSL_QUIC_RECORD_RX_H - -# include -# include "internal/quic_wire_pkt.h" -# include "internal/quic_types.h" -# include "internal/quic_predef.h" -# include "internal/quic_record_util.h" -# include "internal/quic_demux.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Record Layer - RX - * ====================== - */ -typedef struct ossl_qrx_st OSSL_QRX; - -typedef struct ossl_qrx_args_st { - OSSL_LIB_CTX *libctx; - const char *propq; - - /* Demux which owns the URXEs passed to us. */ - QUIC_DEMUX *demux; - - /* Length of connection IDs used in short-header packets in bytes. */ - size_t short_conn_id_len; - - /* - * Maximum number of deferred datagrams buffered at any one time. - * Suggested value: 32. - */ - size_t max_deferred; - - /* Initial reference PN used for RX. */ - QUIC_PN init_largest_pn[QUIC_PN_SPACE_NUM]; - - /* Initial key phase. For debugging use only; always 0 in real use. */ - unsigned char init_key_phase_bit; -} OSSL_QRX_ARGS; - -/* Instantiates a new QRX. */ -OSSL_QRX *ossl_qrx_new(const OSSL_QRX_ARGS *args); - -/* - * Frees the QRX. All packets obtained using ossl_qrx_read_pkt must already - * have been released by calling ossl_qrx_release_pkt. - * - * You do not need to call ossl_qrx_remove_dst_conn_id first; this function will - * unregister the QRX from the demuxer for all registered destination connection - * IDs (DCIDs) automatically. - */ -void ossl_qrx_free(OSSL_QRX *qrx); - -/* Setters for the msg_callback and msg_callback_arg */ -void ossl_qrx_set_msg_callback(OSSL_QRX *qrx, ossl_msg_cb msg_callback, - SSL *msg_callback_ssl); -void ossl_qrx_set_msg_callback_arg(OSSL_QRX *qrx, - void *msg_callback_arg); - -/* - * Secret Management - * ================= - * - * A QRX has several encryption levels (Initial, Handshake, 0-RTT, 1-RTT) and - * two directions (RX, TX). At any given time, key material is managed for each - * (EL, RX/TX) combination. - * - * Broadly, for a given (EL, RX/TX), the following state machine is applicable: - * - * WAITING_FOR_KEYS --[Provide]--> HAVE_KEYS --[Discard]--> | DISCARDED | - * \-------------------------------------[Discard]--> | | - * - * To transition the RX side of an EL from WAITING_FOR_KEYS to HAVE_KEYS, call - * ossl_qrx_provide_secret (for the INITIAL EL, use of - * ossl_quic_provide_initial_secret is recommended). - * - * Once keys have been provisioned for an EL, you call - * ossl_qrx_discard_enc_level to transition the EL to the DISCARDED state. You - * can also call this function to transition directly to the DISCARDED state - * even before any keys have been provisioned for that EL. - * - * The DISCARDED state is terminal for a given EL; you cannot provide a secret - * again for that EL after reaching it. - * - * Incoming packets cannot be processed and decrypted if they target an EL - * not in the HAVE_KEYS state. However, there is a distinction between - * the WAITING_FOR_KEYS and DISCARDED states: - * - * - In the WAITING_FOR_KEYS state, the QRX assumes keys for the given - * EL will eventually arrive. Therefore, if it receives any packet - * for an EL in this state, it buffers it and tries to process it - * again once the EL reaches HAVE_KEYS. - * - * - In the DISCARDED state, the QRX assumes no keys for the given - * EL will ever arrive again. If it receives any packet for an EL - * in this state, it is simply discarded. - * - * If the user wishes to instantiate a new QRX to replace an old one for - * whatever reason, for example to take over for an already established QUIC - * connection, it is important that all ELs no longer being used (i.e., INITIAL, - * 0-RTT, 1-RTT) are transitioned to the DISCARDED state. Otherwise, the QRX - * will assume that keys for these ELs will arrive in future, and will buffer - * any received packets for those ELs perpetually. This can be done by calling - * ossl_qrx_discard_enc_level for all non-1-RTT ELs immediately after - * instantiating the QRX. - * - * The INITIAL EL is not setup automatically when the QRX is instantiated. This - * allows the caller to instead discard it immediately after instantiation of - * the QRX if it is not needed, for example if the QRX is being instantiated to - * take over handling of an existing connection which has already passed the - * INITIAL phase. This avoids the unnecessary derivation of INITIAL keys where - * they are not needed. In the ordinary case, ossl_quic_provide_initial_secret - * should be called immediately after instantiation. - */ - -/* - * Provides a secret to the QRX, which arises due to an encryption level change. - * enc_level is a QUIC_ENC_LEVEL_* value. To initialise the INITIAL encryption - * level, it is recommended to use ossl_quic_provide_initial_secret instead. - * - * You should seek to call this function for a given EL before packets of that - * EL arrive and are processed by the QRX. However, if packets have already - * arrived for a given EL, the QRX will defer processing of them and perform - * processing of them when this function is eventually called for the EL in - * question. - * - * suite_id is a QRL_SUITE_* value which determines the AEAD function used for - * the QRX. - * - * The secret passed is used directly to derive the "quic key", "quic iv" and - * "quic hp" values. - * - * secret_len is the length of the secret buffer in bytes. The buffer must be - * sized correctly to the chosen suite, else the function fails. - * - * This function can only be called once for a given EL, except for the INITIAL - * EL, which can need rekeying when a connection retry occurs. Subsequent calls - * for non-INITIAL ELs fail, as do calls made after a corresponding call to - * ossl_qrx_discard_enc_level for that EL. The secret for a non-INITIAL EL - * cannot be changed after it is set because QUIC has no facility for - * introducing additional key material after an EL is setup. QUIC key updates - * are managed semi-automatically by the QRX but do require some caller handling - * (see below). - * - * md is for internal use and should be NULL. - * - * Returns 1 on success or 0 on failure. - */ -int ossl_qrx_provide_secret(OSSL_QRX *qrx, - uint32_t enc_level, - uint32_t suite_id, - EVP_MD *md, - const unsigned char *secret, - size_t secret_len); - -/* - * Informs the QRX that it can now discard key material for a given EL. The QRX - * will no longer be able to process incoming packets received at that - * encryption level. This function is idempotent and succeeds if the EL has - * already been discarded. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_qrx_discard_enc_level(OSSL_QRX *qrx, uint32_t enc_level); - -/* - * Packet Reception - * ================ - */ - -/* Information about a received packet. */ -struct ossl_qrx_pkt_st { - /* - * Points to a logical representation of the decoded QUIC packet header. The - * data and len fields point to the decrypted QUIC payload (i.e., to a - * sequence of zero or more (potentially malformed) frames to be decoded). - */ - QUIC_PKT_HDR *hdr; - - /* - * Address the packet was received from. If this is not available for this - * packet, this field is NULL (but this can only occur for manually injected - * packets). - */ - const BIO_ADDR *peer; - - /* - * Local address the packet was sent to. If this is not available for this - * packet, this field is NULL. - */ - const BIO_ADDR *local; - - /* - * This is the length of the datagram which contained this packet. Note that - * the datagram may have contained other packets than this. The intended use - * for this is so that the user can enforce minimum datagram sizes (e.g. for - * datagrams containing INITIAL packets), as required by RFC 9000. - */ - size_t datagram_len; - - /* The PN which was decoded for the packet, if the packet has a PN field. */ - QUIC_PN pn; - - /* - * Time the packet was received, or ossl_time_zero() if the demuxer is not - * using a now() function. - */ - OSSL_TIME time; - - /* The QRX which was used to receive the packet. */ - OSSL_QRX *qrx; - - /* - * The key epoch the packet was received with. Always 0 for non-1-RTT - * packets. - */ - uint64_t key_epoch; - - /* - * This monotonically increases with each datagram received. - * It is for diagnostic use only. - */ - uint64_t datagram_id; -}; - -/* - * Tries to read a new decrypted packet from the QRX. - * - * On success, *pkt points to a OSSL_QRX_PKT structure. The structure should be - * freed when no longer needed by calling ossl_qrx_pkt_release(). The structure - * is refcounted; to gain extra references, call ossl_qrx_pkt_up_ref(). This - * will cause a corresponding number of calls to ossl_qrx_pkt_release() to be - * ignored. - * - * The resources referenced by (*pkt)->hdr, (*pkt)->hdr->data and (*pkt)->peer - * have the same lifetime as *pkt. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_qrx_read_pkt(OSSL_QRX *qrx, OSSL_QRX_PKT **pkt); - -/* - * Decrement the reference count for the given packet and frees it if the - * reference count drops to zero. No-op if pkt is NULL. - */ -void ossl_qrx_pkt_release(OSSL_QRX_PKT *pkt); - -/* Increments the reference count for the given packet. */ -void ossl_qrx_pkt_up_ref(OSSL_QRX_PKT *pkt); - -/* - * Returns 1 if there are any already processed (i.e. decrypted) packets waiting - * to be read from the QRX. - */ -int ossl_qrx_processed_read_pending(OSSL_QRX *qrx); - -/* - * Returns 1 if there are any unprocessed (i.e. not yet decrypted) packets - * waiting to be processed by the QRX. These may or may not result in - * successfully decrypted packets once processed. This indicates whether - * unprocessed data is buffered by the QRX, not whether any data is available in - * a kernel socket buffer. - */ -int ossl_qrx_unprocessed_read_pending(OSSL_QRX *qrx); - -/* - * Returns the number of UDP payload bytes received from the network so far - * since the last time this counter was cleared. If clear is 1, clears the - * counter and returns the old value. - * - * The intended use of this is to allow callers to determine how much credit to - * add to their anti-amplification budgets. This is reported separately instead - * of in the OSSL_QRX_PKT structure so that a caller can apply - * anti-amplification credit as soon as a datagram is received, before it has - * necessarily read all processed packets contained within that datagram from - * the QRX. - */ -uint64_t ossl_qrx_get_bytes_received(OSSL_QRX *qrx, int clear); - -/* - * Sets a callback which is called when a packet is received and being validated - * before being queued in the read queue. This is called after packet body - * decryption and authentication to prevent exposing side channels. pn_space is - * a QUIC_PN_SPACE_* value denoting which PN space the PN belongs to. - * - * If this callback returns 1, processing continues normally. - * If this callback returns 0, the packet is discarded. - * - * Other packets in the same datagram will still be processed where possible. - * - * The callback is optional and can be unset by passing NULL for cb. - * cb_arg is an opaque value passed to cb. - */ -typedef int (ossl_qrx_late_validation_cb)(QUIC_PN pn, int pn_space, - void *arg); - -int ossl_qrx_set_late_validation_cb(OSSL_QRX *qrx, - ossl_qrx_late_validation_cb *cb, - void *cb_arg); - -/* - * Forcibly injects a URXE which has been issued by the DEMUX into the QRX for - * processing. This can be used to pass a received datagram to the QRX if it - * would not be correctly routed to the QRX via standard DCID-based routing; for - * example, when handling an incoming Initial packet which is attempting to - * establish a new connection. - */ -void ossl_qrx_inject_urxe(OSSL_QRX *qrx, QUIC_URXE *e); - -/* - * Decryption of 1-RTT packets must be explicitly enabled by calling this - * function. This is to comply with the requirement that we not process 1-RTT - * packets until the handshake is complete, even if we already have 1-RTT - * secrets. Even if a 1-RTT secret is provisioned for the QRX, incoming 1-RTT - * packets will be handled as though no key is available until this function is - * called. Calling this function will then requeue any such deferred packets for - * processing. - */ -void ossl_qrx_allow_1rtt_processing(OSSL_QRX *qrx); - -/* - * Key Update (RX) - * =============== - * - * Key update on the RX side is a largely but not entirely automatic process. - * - * Key update is initially triggered by receiving a 1-RTT packet with a - * different Key Phase value. This could be caused by an attacker in the network - * flipping random bits, therefore such a key update is tentative until the - * packet payload is successfully decrypted and authenticated by the AEAD with - * the 'next' keys. These 'next' keys then become the 'current' keys and the - * 'current' keys then become the 'previous' keys. The 'previous' keys must be - * kept around temporarily as some packets may still be in flight in the network - * encrypted with the old keys. If the old Key Phase value is X and the new Key - * Phase Value is Y (where obviously X != Y), this creates an ambiguity as any - * new packet received with a KP of X could either be an attempt to initiate yet - * another key update right after the last one, or an old packet encrypted - * before the key update. - * - * RFC 9001 provides some guidance on handling this issue: - * - * Strategy 1: - * Three keys, disambiguation using packet numbers - * - * "A recovered PN that is lower than any PN from the current KP uses the - * previous packet protection keys; a recovered PN that is higher than any - * PN from the current KP requires use of the next packet protection - * keys." - * - * Strategy 2: - * Two keys and a timer - * - * "Alternatively, endpoints can retain only two sets of packet protection - * keys, swapping previous keys for next after enough time has passed to - * allow for reordering in the network. In this case, the KP bit alone can - * be used to select keys." - * - * Strategy 2 is more efficient (we can keep fewer cipher contexts around) and - * should cover all actually possible network conditions. It also allows a delay - * after we make the 'next' keys our 'current' keys before we generate new - * 'next' keys, which allows us to mitigate against malicious peers who try to - * initiate an excessive number of key updates. - * - * We therefore model the following state machine: - * - * - * PROVISIONED - * _______________________________ - * | | - * UNPROVISIONED --|----> NORMAL <----------\ |------> DISCARDED - * | | | | - * | | | | - * | v | | - * | UPDATING | | - * | | | | - * | | | | - * | v | | - * | COOLDOWN | | - * | | | | - * | | | | - * | \---------------| | - * |_______________________________| - * - * - * The RX starts (once a secret has been provisioned) in the NORMAL state. In - * the NORMAL state, the current expected value of the Key Phase bit is - * recorded. When a flipped Key Phase bit is detected, the RX attempts to - * decrypt and authenticate the received packet with the 'next' keys rather than - * the 'current' keys. If (and only if) this authentication is successful, we - * move to the UPDATING state. (An attacker in the network could flip - * the Key Phase bit randomly, so it is essential we do nothing until AEAD - * authentication is complete.) - * - * In the UPDATING state, we know a key update is occurring and record - * the new Key Phase bit value as the newly current value, but we still keep the - * old keys around so that we can still process any packets which were still in - * flight when the key update was initiated. In the UPDATING state, a - * Key Phase bit value different to the current expected value is treated not as - * the initiation of another key update, but a reference to our old keys. - * - * Eventually we will be reasonably sure we are not going to receive any more - * packets with the old keys. At this point, we can transition to the COOLDOWN - * state. This transition occurs automatically after a certain amount of time; - * RFC 9001 recommends it be the PTO interval, which relates to our RTT to the - * peer. The duration also SHOULD NOT exceed three times the PTO to assist with - * maintaining PFS. - * - * In the COOLDOWN phase, the old keys have been securely erased and only one - * set of keys can be used: the current keys. If a packet is received with a Key - * Phase bit value different to the current Key Phase Bit value, this is treated - * as a request for a Key Update, but this request is ignored and the packet is - * treated as malformed. We do this to allow mitigation against malicious peers - * trying to initiate an excessive number of Key Updates. The timeout for the - * transition from UPDATING to COOLDOWN is recommended as adequate for - * this purpose in itself by the RFC, so the normal additional timeout value for - * the transition from COOLDOWN to normal is zero (immediate transition). - * - * A summary of each state: - * - * Epoch Exp KP Uses Keys KS0 KS1 If Non-Expected KP Bit - * ----- ------ --------- ------ ----- ---------------------- - * NORMAL 0 0 Keyset 0 Gen 0 Gen 1 → UPDATING - * UPDATING 1 1 Keyset 1 Gen 0 Gen 1 Use Keyset 0 - * COOLDOWN 1 1 Keyset 1 Erased Gen 1 Ignore Packet (*) - * - * NORMAL 1 1 Keyset 1 Gen 2 Gen 1 → UPDATING - * UPDATING 2 0 Keyset 0 Gen 2 Gen 1 Use Keyset 1 - * COOLDOWN 2 0 Keyset 0 Gen 2 Erased Ignore Packet (*) - * - * (*) Actually implemented by attempting to decrypt the packet with the - * wrong keys (which ultimately has the same outcome), as recommended - * by RFC 9001 to avoid creating timing channels. - * - * Note that the key material for the next key generation ("key epoch") is - * always kept in the NORMAL state (necessary to avoid side-channel attacks). - * This material is derived during the transition from COOLDOWN to NORMAL. - * - * Note that when a peer initiates a Key Update, we MUST also initiate a Key - * Update as per the RFC. The caller is responsible for detecting this condition - * and making the necessary calls to the TX side by detecting changes to the - * return value of ossl_qrx_get_key_epoch(). - * - * The above states (NORMAL, UPDATING, COOLDOWN) can themselves be - * considered substates of the PROVISIONED state. Providing a secret to the QRX - * for an EL transitions from UNPROVISIONED, the initial state, to PROVISIONED - * (NORMAL). Dropping key material for an EL transitions from whatever the - * current substate of the PROVISIONED state is to the DISCARDED state, which is - * the terminal state. - * - * Note that non-1RTT ELs cannot undergo key update, therefore a non-1RTT EL is - * always in the NORMAL substate if it is in the PROVISIONED state. - */ - -/* - * Return the current RX key epoch for the 1-RTT encryption level. This is - * initially zero and is incremented by one for every Key Update successfully - * signalled by the peer. If the 1-RTT EL has not yet been provisioned or has - * been discarded, returns UINT64_MAX. - * - * A necessary implication of this API is that the least significant bit of the - * returned value corresponds to the currently expected Key Phase bit, though - * callers are not anticipated to have any need of this information. - * - * It is not possible for the returned value to overflow, as a QUIC connection - * cannot support more than 2**62 packet numbers, and a connection must be - * terminated if this limit is reached. - * - * The caller should use this function to detect when the key epoch has changed - * and use it to initiate a key update on the TX side. - * - * The value returned by this function increments specifically at the transition - * from the NORMAL to the UPDATING state discussed above. - */ -uint64_t ossl_qrx_get_key_epoch(OSSL_QRX *qrx); - -/* - * Sets an optional callback which will be called when the key epoch changes. - * - * The callback is optional and can be unset by passing NULL for cb. - * cb_arg is an opaque value passed to cb. pn is the PN of the packet. - * Since key update is only supported for 1-RTT packets, the PN is always - * in the Application Data PN space. -*/ -typedef void (ossl_qrx_key_update_cb)(QUIC_PN pn, void *arg); - -int ossl_qrx_set_key_update_cb(OSSL_QRX *qrx, - ossl_qrx_key_update_cb *cb, void *cb_arg); - -/* - * Relates to the 1-RTT encryption level. The caller should call this after the - * UPDATING state is reached, after a timeout to be determined by the caller. - * - * This transitions from the UPDATING state to the COOLDOWN state (if - * still in the UPDATING state). If normal is 1, then transitions from - * the COOLDOWN state to the NORMAL state. Both transitions can be performed at - * once if desired. - * - * If in the normal state, or if in the COOLDOWN state and normal is 0, this is - * a no-op and returns 1. Returns 0 if the 1-RTT EL has not been provisioned or - * has been dropped. - * - * It is essential that the caller call this within a few PTO intervals of a key - * update occurring (as detected by the caller in a call to - * ossl_qrx_key_get_key_epoch()), as otherwise the peer will not be able to - * perform a Key Update ever again. - */ -int ossl_qrx_key_update_timeout(OSSL_QRX *qrx, int normal); - - -/* - * Key Expiration - * ============== - */ - -/* - * Returns the number of seemingly forged packets which have been received by - * the QRX. If this value reaches the value returned by - * ossl_qrx_get_max_epoch_forged_pkt_count() for a given EL, all further - * received encrypted packets for that EL will be discarded without processing. - * - * Note that the forged packet limit is for the connection lifetime, thus it is - * not reset by a key update. It is suggested that the caller terminate the - * connection a reasonable margin before the limit is reached. However, the - * exact limit imposed does vary by EL due to the possibility that different ELs - * use different AEADs. - */ -uint64_t ossl_qrx_get_cur_forged_pkt_count(OSSL_QRX *qrx); - -/* - * Returns the maximum number of forged packets which the record layer will - * permit to be verified using this QRX instance. - */ -uint64_t ossl_qrx_get_max_forged_pkt_count(OSSL_QRX *qrx, - uint32_t enc_level); - -# endif - -#endif diff --git a/openssl/include/internal/quic_record_tx.h b/openssl/include/internal/quic_record_tx.h deleted file mode 100644 index e84523f89..000000000 --- a/openssl/include/internal/quic_record_tx.h +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_RECORD_TX_H -# define OSSL_QUIC_RECORD_TX_H - -# include -# include "internal/quic_wire_pkt.h" -# include "internal/quic_types.h" -# include "internal/quic_predef.h" -# include "internal/quic_record_util.h" -# include "internal/qlog.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Record Layer - TX - * ====================== - */ -typedef struct ossl_qtx_iovec_st { - const unsigned char *buf; - size_t buf_len; -} OSSL_QTX_IOVEC; - -typedef struct ossl_qtx_st OSSL_QTX; - -typedef int (*ossl_mutate_packet_cb)(const QUIC_PKT_HDR *hdrin, - const OSSL_QTX_IOVEC *iovecin, size_t numin, - QUIC_PKT_HDR **hdrout, - const OSSL_QTX_IOVEC **iovecout, - size_t *numout, - void *arg); - -typedef void (*ossl_finish_mutate_cb)(void *arg); - -typedef struct ossl_qtx_args_st { - OSSL_LIB_CTX *libctx; - const char *propq; - - /* BIO to transmit to. */ - BIO *bio; - - /* Maximum datagram payload length (MDPL) for TX purposes. */ - size_t mdpl; - - /* Callback returning QLOG instance to use, or NULL. */ - QLOG *(*get_qlog_cb)(void *arg); - void *get_qlog_cb_arg; -} OSSL_QTX_ARGS; - -/* Instantiates a new QTX. */ -OSSL_QTX *ossl_qtx_new(const OSSL_QTX_ARGS *args); - -/* Frees the QTX. */ -void ossl_qtx_free(OSSL_QTX *qtx); - -/* Set mutator callbacks for test framework support */ -void ossl_qtx_set_mutator(OSSL_QTX *qtx, ossl_mutate_packet_cb mutatecb, - ossl_finish_mutate_cb finishmutatecb, void *mutatearg); - -/* Setters for the msg_callback and the msg_callback_arg */ -void ossl_qtx_set_msg_callback(OSSL_QTX *qtx, ossl_msg_cb msg_callback, - SSL *msg_callback_ssl); -void ossl_qtx_set_msg_callback_arg(OSSL_QTX *qtx, void *msg_callback_arg); - -/* Change QLOG instance retrieval callback in use after instantiation. */ -void ossl_qtx_set_qlog_cb(OSSL_QTX *qtx, QLOG *(*get_qlog_cb)(void *arg), - void *get_qlog_cb_arg); - -/* - * Secret Management - * ----------------- - */ - -/* - * Provides a secret to the QTX, which arises due to an encryption level change. - * enc_level is a QUIC_ENC_LEVEL_* value. - * - * This function can be used to initialise the INITIAL encryption level, but you - * should not do so directly; see the utility function - * ossl_qrl_provide_initial_secret() instead, which can initialise the INITIAL - * encryption level of a QRX and QTX simultaneously without duplicating certain - * key derivation steps. - * - * You must call this function for a given EL before transmitting packets at - * that EL using this QTX, otherwise ossl_qtx_write_pkt will fail. - * - * suite_id is a QRL_SUITE_* value which determines the AEAD function used for - * the QTX. - * - * The secret passed is used directly to derive the "quic key", "quic iv" and - * "quic hp" values. - * - * secret_len is the length of the secret buffer in bytes. The buffer must be - * sized correctly to the chosen suite, else the function fails. - * - * This function can only be called once for a given EL, except for the INITIAL - * EL, as the INITIAL EL can need to be rekeyed if connection retry occurs. - * Subsequent calls for non-INITIAL ELs fail. Calls made after a corresponding - * call to ossl_qtx_discard_enc_level for a given EL also fail, including for - * the INITIAL EL. The secret for a non-INITIAL EL cannot be changed after it is - * set because QUIC has no facility for introducing additional key material - * after an EL is setup. (QUIC key updates generate new keys from existing key - * material and do not introduce new entropy into a connection's key material.) - * - * Returns 1 on success or 0 on failure. - */ -int ossl_qtx_provide_secret(OSSL_QTX *qtx, - uint32_t enc_level, - uint32_t suite_id, - EVP_MD *md, - const unsigned char *secret, - size_t secret_len); - -/* - * Informs the QTX that it can now discard key material for a given EL. The QTX - * will no longer be able to generate packets at that EL. This function is - * idempotent and succeeds if the EL has already been discarded. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_qtx_discard_enc_level(OSSL_QTX *qtx, uint32_t enc_level); - -/* Returns 1 if the given encryption level is provisioned. */ -int ossl_qtx_is_enc_level_provisioned(OSSL_QTX *qtx, uint32_t enc_level); - -/* - * Given the value ciphertext_len representing an encrypted packet payload - * length in bytes, determines how many plaintext bytes it will decrypt to. - * Returns 0 if the specified EL is not provisioned or ciphertext_len is too - * small. The result is written to *plaintext_len. - */ -int ossl_qtx_calculate_plaintext_payload_len(OSSL_QTX *qtx, uint32_t enc_level, - size_t ciphertext_len, - size_t *plaintext_len); - -/* - * Given the value plaintext_len represented a plaintext packet payload length - * in bytes, determines how many ciphertext bytes it will encrypt to. The value - * output does not include packet headers. Returns 0 if the specified EL is not - * provisioned. The result is written to *ciphertext_len. - */ -int ossl_qtx_calculate_ciphertext_payload_len(OSSL_QTX *qtx, uint32_t enc_level, - size_t plaintext_len, - size_t *ciphertext_len); - -uint32_t ossl_qrl_get_suite_cipher_tag_len(uint32_t suite_id); - - -/* - * Packet Transmission - * ------------------- - */ - -struct ossl_qtx_pkt_st { - /* Logical packet header to be serialized. */ - QUIC_PKT_HDR *hdr; - - /* - * iovecs expressing the logical packet payload buffer. Zero-length entries - * are permitted. - */ - const OSSL_QTX_IOVEC *iovec; - size_t num_iovec; - - /* Destination address. Will be passed through to the BIO if non-NULL. */ - const BIO_ADDR *peer; - - /* - * Local address (optional). Specify as non-NULL only if TX BIO - * has local address support enabled. - */ - const BIO_ADDR *local; - - /* - * Logical PN. Used for encryption. This will automatically be encoded to - * hdr->pn, which need not be initialized. - */ - QUIC_PN pn; - - /* Packet flags. Zero or more OSSL_QTX_PKT_FLAG_* values. */ - uint32_t flags; -}; - -/* - * More packets will be written which should be coalesced into a single - * datagram; do not send this packet yet. To use this, set this flag for all - * packets but the final packet in a datagram, then send the final packet - * without this flag set. - * - * This flag is not a guarantee and the QTX may transmit immediately anyway if - * it is not possible to fit any more packets in the current datagram. - * - * If the caller change its mind and needs to cause a packet queued with - * COALESCE after having passed it to this function but without writing another - * packet, it should call ossl_qtx_flush_pkt(). - */ -#define OSSL_QTX_PKT_FLAG_COALESCE (1U << 0) - -/* - * Writes a packet. - * - * *pkt need be valid only for the duration of the call to this function. - * - * pkt->hdr->data and pkt->hdr->len are unused. The payload buffer is specified - * via an array of OSSL_QTX_IOVEC structures. The API is designed to support - * single-copy transmission; data is copied from the iovecs as it is encrypted - * into an internal staging buffer for transmission. - * - * The function may modify and clobber pkt->hdr->data, pkt->hdr->len, - * pkt->hdr->key_phase and pkt->hdr->pn for its own internal use. No other - * fields of pkt or pkt->hdr will be modified. - * - * It is the callers responsibility to determine how long the PN field in the - * encoded packet should be by setting pkt->hdr->pn_len. This function takes - * care of the PN encoding. Set pkt->pn to the desired PN. - * - * Note that 1-RTT packets do not have a DCID Length field, therefore the DCID - * length must be understood contextually. This function assumes the caller - * knows what it is doing and will serialize a DCID of whatever length is given. - * It is the caller's responsibility to ensure it uses a consistent DCID length - * for communication with any given set of remote peers. - * - * The packet is queued regardless of whether it is able to be sent immediately. - * This enables packets to be batched and sent at once on systems which support - * system calls to send multiple datagrams in a single system call (see - * BIO_sendmmsg). To flush queued datagrams to the network, see - * ossl_qtx_flush_net(). - * - * Returns 1 on success or 0 on failure. - */ -int ossl_qtx_write_pkt(OSSL_QTX *qtx, const OSSL_QTX_PKT *pkt); - -/* - * Finish any incomplete datagrams for transmission which were flagged for - * coalescing. If there is no current coalescing datagram, this is a no-op. - */ -void ossl_qtx_finish_dgram(OSSL_QTX *qtx); - -/* - * (Attempt to) flush any datagrams which are queued for transmission. Note that - * this does not cancel coalescing; call ossl_qtx_finish_dgram() first if that - * is desired. The queue is drained into the OS's sockets as much as possible. - * To determine if there is still data to be sent after calling this function, - * use ossl_qtx_get_queue_len_bytes(). - * - * Returns one of the following values: - * - * QTX_FLUSH_NET_RES_OK - * Either no packets are currently queued for transmission, - * or at least one packet was successfully submitted. - * - * QTX_FLUSH_NET_RES_TRANSIENT_FAIL - * The underlying network write BIO indicated a transient error - * (e.g. buffers full). - * - * QTX_FLUSH_NET_RES_PERMANENT_FAIL - * Internal error (e.g. assertion or allocation error) - * or the underlying network write BIO indicated a non-transient - * error. - */ -#define QTX_FLUSH_NET_RES_OK 1 -#define QTX_FLUSH_NET_RES_TRANSIENT_FAIL (-1) -#define QTX_FLUSH_NET_RES_PERMANENT_FAIL (-2) - -int ossl_qtx_flush_net(OSSL_QTX *qtx); - -/* - * Diagnostic function. If there is any datagram pending transmission, pops it - * and writes the details of the datagram as they would have been passed to - * *msg. Returns 1, or 0 if there are no datagrams pending. For test use only. - */ -int ossl_qtx_pop_net(OSSL_QTX *qtx, BIO_MSG *msg); - -/* Returns number of datagrams which are fully-formed but not yet sent. */ -size_t ossl_qtx_get_queue_len_datagrams(OSSL_QTX *qtx); - -/* - * Returns number of payload bytes across all datagrams which are fully-formed - * but not yet sent. Does not count any incomplete coalescing datagram. - */ -size_t ossl_qtx_get_queue_len_bytes(OSSL_QTX *qtx); - -/* - * Returns number of bytes in the current coalescing datagram, or 0 if there is - * no current coalescing datagram. Returns 0 after a call to - * ossl_qtx_finish_dgram(). - */ -size_t ossl_qtx_get_cur_dgram_len_bytes(OSSL_QTX *qtx); - -/* - * Returns number of queued coalesced packets which have not been put into a - * datagram yet. If this is non-zero, ossl_qtx_flush_pkt() needs to be called. - */ -size_t ossl_qtx_get_unflushed_pkt_count(OSSL_QTX *qtx); - -/* - * Change the BIO being used by the QTX. May be NULL if actual transmission is - * not currently required. Does not up-ref the BIO; the caller is responsible - * for ensuring the lifetime of the BIO exceeds the lifetime of the QTX. - */ -void ossl_qtx_set_bio(OSSL_QTX *qtx, BIO *bio); - -/* Changes the MDPL. */ -int ossl_qtx_set_mdpl(OSSL_QTX *qtx, size_t mdpl); - -/* Retrieves the current MDPL. */ -size_t ossl_qtx_get_mdpl(OSSL_QTX *qtx); - - -/* - * Key Update - * ---------- - * - * For additional discussion of key update considerations, see QRX header file. - */ - -/* - * Triggers a key update. The key update will be started by inverting the Key - * Phase bit of the next packet transmitted; no key update occurs until the next - * packet is transmitted. Thus, this function should generally be called - * immediately before queueing the next packet. - * - * There are substantial requirements imposed by RFC 9001 on under what - * circumstances a key update can be initiated. The caller is responsible for - * meeting most of these requirements. For example, this function cannot be - * called too soon after a previous key update has occurred. Key updates also - * cannot be initiated until the 1-RTT encryption level is reached. - * - * As a sanity check, this function will fail and return 0 if the non-1RTT - * encryption levels have not yet been dropped. - * - * The caller may decide itself to initiate a key update, but it also MUST - * initiate a key update where it detects that the peer has initiated a key - * update. The caller is responsible for initiating a TX key update by calling - * this function in this circumstance; thus, the caller is responsible for - * coupling the RX and TX QUIC record layers in this way. - */ -int ossl_qtx_trigger_key_update(OSSL_QTX *qtx); - - -/* - * Key Expiration - * -------------- - */ - -/* - * Returns the number of packets which have been encrypted for transmission with - * the current set of TX keys (the current "TX key epoch"). Reset to zero after - * a key update and incremented for each packet queued. If enc_level is not - * valid or relates to an EL which is not currently available, returns - * UINT64_MAX. - */ -uint64_t ossl_qtx_get_cur_epoch_pkt_count(OSSL_QTX *qtx, uint32_t enc_level); - -/* - * Returns the maximum number of packets which the record layer will permit to - * be encrypted using the current set of TX keys. If this limit is reached (that - * is, if the counter returned by ossl_qrx_tx_get_cur_epoch_pkt_count() reaches - * this value), as a safety measure, the QTX will not permit any further packets - * to be queued. All calls to ossl_qrx_write_pkt that try to send packets of a - * kind which need to be encrypted will fail. It is not possible to recover from - * this condition and the QTX must then be destroyed; therefore, callers should - * ensure they always trigger a key update well in advance of reaching this - * limit. - * - * The value returned by this function is based on the ciphersuite configured - * for the given encryption level. If keys have not been provisioned for the - * specified enc_level or the enc_level argument is invalid, this function - * returns UINT64_MAX, which is not a valid value. Note that it is not possible - * to perform a key update at any encryption level other than 1-RTT, therefore - * if this limit is reached at earlier encryption levels (which should not be - * possible) the connection must be terminated. Since this condition precludes - * the transmission of further packets, the only possible signalling of such an - * error condition to a peer is a Stateless Reset packet. - */ -uint64_t ossl_qtx_get_max_epoch_pkt_count(OSSL_QTX *qtx, uint32_t enc_level); - -/* - * Get the 1-RTT EL key epoch number for the QTX. This is intended for - * diagnostic purposes. Returns 0 if 1-RTT EL is not provisioned yet. - */ -uint64_t ossl_qtx_get_key_epoch(OSSL_QTX *qtx); - -# endif - -#endif diff --git a/openssl/include/internal/quic_record_util.h b/openssl/include/internal/quic_record_util.h deleted file mode 100644 index 97e630d92..000000000 --- a/openssl/include/internal/quic_record_util.h +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_RECORD_UTIL_H -# define OSSL_QUIC_RECORD_UTIL_H - -# include -# include "internal/quic_types.h" - -# ifndef OPENSSL_NO_QUIC - -struct ossl_qrx_st; -struct ossl_qtx_st; - -/* - * QUIC Key Derivation Utilities - * ============================= - */ - -/* HKDF-Extract(salt, IKM) (RFC 5869) */ -int ossl_quic_hkdf_extract(OSSL_LIB_CTX *libctx, - const char *propq, - const EVP_MD *md, - const unsigned char *salt, size_t salt_len, - const unsigned char *ikm, size_t ikm_len, - unsigned char *out, size_t out_len); - -/* - * A QUIC client sends its first INITIAL packet with a random DCID, which - * is used to compute the secrets used for INITIAL packet encryption in both - * directions (both client-to-server and server-to-client). - * - * This function performs the necessary DCID-based key derivation, and then - * provides the derived key material for the INITIAL encryption level to a QRX - * instance, a QTX instance, or both. - * - * This function derives the necessary key material and then: - * - if qrx is non-NULL, provides the appropriate secret to it; - * - if qtx is non-NULL, provides the appropriate secret to it. - * - * If both qrx and qtx are NULL, this is a no-op. This function is equivalent to - * making the appropriate calls to ossl_qrx_provide_secret() and - * ossl_qtx_provide_secret(). - * - * It is possible to use a QRX or QTX without ever calling this, for example if - * there is no desire to handle INITIAL packets (e.g. if a QRX/QTX is - * instantiated to succeed a previous QRX/QTX and handle a connection which is - * already established). However in this case you should make sure you call - * ossl_qrx_discard_enc_level(); see the header for that function for more - * details. Calling ossl_qtx_discard_enc_level() is not essential but could - * protect against programming errors. - * - * Returns 1 on success or 0 on error. - */ -int ossl_quic_provide_initial_secret(OSSL_LIB_CTX *libctx, - const char *propq, - const QUIC_CONN_ID *dst_conn_id, - int is_server, - struct ossl_qrx_st *qrx, - struct ossl_qtx_st *qtx); - -/* - * QUIC Record Layer Ciphersuite Info - * ================================== - */ - -/* Available QUIC Record Layer (QRL) ciphersuites. */ -# define QRL_SUITE_AES128GCM 1 /* SHA256 */ -# define QRL_SUITE_AES256GCM 2 /* SHA384 */ -# define QRL_SUITE_CHACHA20POLY1305 3 /* SHA256 */ - -/* Returns cipher name in bytes or NULL if suite ID is invalid. */ -const char *ossl_qrl_get_suite_cipher_name(uint32_t suite_id); - -/* Returns hash function name in bytes or NULL if suite ID is invalid. */ -const char *ossl_qrl_get_suite_md_name(uint32_t suite_id); - -/* Returns secret length in bytes or 0 if suite ID is invalid. */ -uint32_t ossl_qrl_get_suite_secret_len(uint32_t suite_id); - -/* Returns key length in bytes or 0 if suite ID is invalid. */ -uint32_t ossl_qrl_get_suite_cipher_key_len(uint32_t suite_id); - -/* Returns IV length in bytes or 0 if suite ID is invalid. */ -uint32_t ossl_qrl_get_suite_cipher_iv_len(uint32_t suite_id); - -/* Returns AEAD auth tag length in bytes or 0 if suite ID is invalid. */ -uint32_t ossl_qrl_get_suite_cipher_tag_len(uint32_t suite_id); - -/* Returns a QUIC_HDR_PROT_CIPHER_* value or 0 if suite ID is invalid. */ -uint32_t ossl_qrl_get_suite_hdr_prot_cipher_id(uint32_t suite_id); - -/* Returns header protection key length in bytes or 0 if suite ID is invalid. */ -uint32_t ossl_qrl_get_suite_hdr_prot_key_len(uint32_t suite_id); - -/* - * Returns maximum number of packets which may be safely encrypted with a suite - * or 0 if suite ID is invalid. - */ -uint64_t ossl_qrl_get_suite_max_pkt(uint32_t suite_id); - -/* - * Returns maximum number of RX'd packets which may safely fail AEAD decryption - * for a given suite or 0 if suite ID is invalid. - */ -uint64_t ossl_qrl_get_suite_max_forged_pkt(uint32_t suite_id); - -# endif - -#endif diff --git a/openssl/include/internal/quic_rx_depack.h b/openssl/include/internal/quic_rx_depack.h deleted file mode 100644 index c90964a7c..000000000 --- a/openssl/include/internal/quic_rx_depack.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_RX_DEPACK_H -# define OSSL_QUIC_RX_DEPACK_H - -# include "internal/quic_channel.h" - -# ifndef OPENSSL_NO_QUIC - -int ossl_quic_handle_frames(QUIC_CHANNEL *qc, OSSL_QRX_PKT *qpacket); - -# endif - -#endif diff --git a/openssl/include/internal/quic_sf_list.h b/openssl/include/internal/quic_sf_list.h deleted file mode 100644 index 8ed1dcb13..000000000 --- a/openssl/include/internal/quic_sf_list.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_SF_LIST_H -# define OSSL_QUIC_SF_LIST_H - -#include "internal/common.h" -#include "internal/uint_set.h" -#include "internal/quic_record_rx.h" - -/* - * Stream frame list - * ================= - * - * This data structure supports similar operations as uint64 set but - * it has slightly different invariants and also carries data associated with - * the ranges in the list. - * - * Operations: - * Insert frame (optimized insertion at the beginning and at the end). - * Iterated peek into the frame(s) from the beginning. - * Dropping frames from the beginning up to an offset (exclusive). - * - * Invariant: The frames in the list are sorted by the start and end bounds. - * Invariant: There are no fully overlapping frames or frames that would - * be fully encompassed by another frame in the list. - * Invariant: No frame has start > end. - * Invariant: The range start is inclusive the end is exclusive to be - * able to mark an empty frame. - * Invariant: The offset never points further than into the first frame. - */ -# ifndef OPENSSL_NO_QUIC - -typedef struct stream_frame_st STREAM_FRAME; - -typedef struct sframe_list_st { - STREAM_FRAME *head, *tail; - /* Is the tail frame final. */ - unsigned int fin; - /* Number of stream frames in the list. */ - size_t num_frames; - /* Offset of data not yet dropped */ - uint64_t offset; - /* Is head locked ? */ - int head_locked; - /* Cleanse data on release? */ - int cleanse; -} SFRAME_LIST; - -/* - * Initializes the stream frame list fl. - */ -void ossl_sframe_list_init(SFRAME_LIST *fl); - -/* - * Destroys the stream frame list fl releasing any data - * still present inside it. - */ -void ossl_sframe_list_destroy(SFRAME_LIST *fl); - -/* - * Insert a stream frame data into the list. - * The data covers an offset range (range.start is inclusive, - * range.end is exclusive). - * fin should be set if this is the final frame of the stream. - * Returns an error if a frame cannot be inserted - due to - * STREAM_FRAME allocation error, or in case of erroneous - * fin flag (this is an ossl_assert() check so a caller must - * check it on its own too). - */ -int ossl_sframe_list_insert(SFRAME_LIST *fl, UINT_RANGE *range, - OSSL_QRX_PKT *pkt, - const unsigned char *data, int fin); - -/* - * Iterator to peek at the contiguous frames at the beginning - * of the frame list fl. - * The *data covers an offset range (range.start is inclusive, - * range.end is exclusive). - * *fin is set if this is the final frame of the stream. - * Opaque iterator *iter can be used to peek at the subsequent - * frame if there is any without any gap before it. - * Returns 1 on success. - * Returns 0 if there is no further contiguous frame. In that - * case *fin is set, if the end of the stream is reached. - */ -int ossl_sframe_list_peek(const SFRAME_LIST *fl, void **iter, - UINT_RANGE *range, const unsigned char **data, - int *fin); - -/* - * Drop all frames up to the offset limit. - * Also unlocks the head frame if locked. - * Returns 1 on success. - * Returns 0 when trying to drop frames at offsets that were not - * received yet. (ossl_assert() is used to check, so this is an invalid call.) - */ -int ossl_sframe_list_drop_frames(SFRAME_LIST *fl, uint64_t limit); - -/* - * Locks and returns the head frame of fl if it is readable - read offset is - * at the beginning or middle of the frame. - * range is set to encompass the not yet read part of the head frame, - * data pointer is set to appropriate offset within the frame if the read - * offset points in the middle of the frame, - * fin is set to 1 if the head frame is also the tail frame. - * Returns 1 on success, 0 if there is no readable data or the head - * frame is already locked. - */ -int ossl_sframe_list_lock_head(SFRAME_LIST *fl, UINT_RANGE *range, - const unsigned char **data, - int *fin); - -/* - * Just returns whether the head frame is locked by previous - * ossl_sframe_list_lock_head() call. - */ -int ossl_sframe_list_is_head_locked(SFRAME_LIST *fl); - -/* - * Callback function type to write stream frame data to some - * side storage before the packet containing the frame data - * is released. - * It should return 1 on success or 0 if there is not enough - * space available in the side storage. - */ -typedef int (sframe_list_write_at_cb)(uint64_t logical_offset, - const unsigned char *buf, - size_t buf_len, - void *cb_arg); - -/* - * Move the frame data in all the stream frames in the list fl - * from the packets to the side storage using the write_at_cb - * callback. - * Returns 1 if all the calls to the callback return 1. - * If the callback returns 0, the function stops processing further - * frames and returns 0. - */ -int ossl_sframe_list_move_data(SFRAME_LIST *fl, - sframe_list_write_at_cb *write_at_cb, - void *cb_arg); -# endif - -#endif diff --git a/openssl/include/internal/quic_srt_gen.h b/openssl/include/internal/quic_srt_gen.h deleted file mode 100644 index a25e71aa8..000000000 --- a/openssl/include/internal/quic_srt_gen.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#ifndef OSSL_INTERNAL_QUIC_SRT_GEN_H -# define OSSL_INTERNAL_QUIC_SRT_GEN_H -# pragma once - -# include "internal/e_os.h" -# include "internal/time.h" -# include "internal/quic_types.h" -# include "internal/quic_wire.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Stateless Reset Token Generator - * ==================================== - * - * This generates 16-byte QUIC Stateless Reset Tokens given a secret symmetric - * key and a DCID. Because the output is deterministic with regards to these - * inputs, assuming the same key is used between invocations of a process, we - * are able to generate the same stateless reset token in a subsequent process, - * thereby allowing us to achieve stateless reset of a peer which still thinks - * it is connected to a past process at the same UDP address. - */ -typedef struct quic_srt_gen_st QUIC_SRT_GEN; - -/* - * Create a new stateless reset token generator using the given key as input. - * The key may be of arbitrary length. - * - * The caller is responsible for performing domain separation with regards to - * the key; i.e., the caller is responsible for ensuring the key is never used - * in any other context. - */ -QUIC_SRT_GEN *ossl_quic_srt_gen_new(OSSL_LIB_CTX *libctx, const char *propq, - const unsigned char *key, size_t key_len); - -/* Free the stateless reset token generator. No-op if srt_gen is NULL. */ -void ossl_quic_srt_gen_free(QUIC_SRT_GEN *srt_gen); - -/* - * Calculates a token using the given DCID and writes it to *token. Returns 0 on - * failure. - */ -int ossl_quic_srt_gen_calculate_token(QUIC_SRT_GEN *srt_gen, - const QUIC_CONN_ID *dcid, - QUIC_STATELESS_RESET_TOKEN *token); - -# endif -#endif diff --git a/openssl/include/internal/quic_srtm.h b/openssl/include/internal/quic_srtm.h deleted file mode 100644 index d60c285e2..000000000 --- a/openssl/include/internal/quic_srtm.h +++ /dev/null @@ -1,109 +0,0 @@ -/* -* Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#ifndef OSSL_INTERNAL_QUIC_SRTM_H -# define OSSL_INTERNAL_QUIC_SRTM_H -# pragma once - -# include "internal/e_os.h" -# include "internal/time.h" -# include "internal/quic_types.h" -# include "internal/quic_wire.h" -# include "internal/quic_predef.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Stateless Reset Token Manager - * ================================== - * - * The stateless reset token manager is responsible for mapping stateless reset - * tokens to connections. It is used to identify stateless reset tokens in - * incoming packets. In this regard it can be considered an alternate "routing" - * mechanism for incoming packets, and is somewhat analogous with the LCIDM, - * except that it uses SRTs to route rather than DCIDs. - * - * The SRTM specifically stores a bidirectional mapping of the form - * - * (opaque pointer, sequence number) [1] <-> [0..n] SRT - * - * The (opaque pointer, sequence number) tuple is used to refer to an entry (for - * example for the purposes of removing it later when it is no longer needed). - * Likewise, an entry can be looked up using SRT to get the opaque pointer and - * sequence number. - * - * It is important to note that the same SRT may exist multiple times and map to - * multiple (opaque pointer, sequence number) tuples, for example, if we - * initiate multiple connections to the same peer using the same local QUIC_PORT - * and the peer decides to behave bizarrely and issue the same SRT for both - * connections. It should not do this, but we have to be resilient against - * byzantine peer behaviour. Thus we are capable of storing multiple identical - * SRTs for different (opaque pointer, sequence number) keys. - * - * The SRTM supports arbitrary insertion, arbitrary deletion of specific keys - * identified by a (opaque pointer, sequence number) key, and mass deletion of - * all entries under a specific opaque pointer. It supports lookup by SRT to - * identify zero or more corresponding (opaque pointer, sequence number) tuples. - * - * The opaque pointer may be used for any purpose but is intended to represent a - * connection identity and must therefore be consistent (usefully comparable). - */ - -/* Creates a new empty SRTM instance. */ -QUIC_SRTM *ossl_quic_srtm_new(OSSL_LIB_CTX *libctx, const char *propq); - -/* Frees a SRTM instance. No-op if srtm is NULL. */ -void ossl_quic_srtm_free(QUIC_SRTM *srtm); - -/* - * Add a (opaque, seq_num) -> SRT entry to the SRTM. This operation fails if a - * SRT entry already exists with the same (opaque, seq_num) tuple. The token is - * copied. Returns 1 on success or 0 on failure. - */ -int ossl_quic_srtm_add(QUIC_SRTM *srtm, void *opaque, uint64_t seq_num, - const QUIC_STATELESS_RESET_TOKEN *token); - -/* - * Removes an entry by identifying it via its (opaque, seq_num) tuple. - * Returns 1 if the entry was found and removed, and 0 if it was not found. - */ -int ossl_quic_srtm_remove(QUIC_SRTM *srtm, void *opaque, uint64_t seq_num); - -/* - * Removes all entries (opaque, *) with the given opaque pointer. - * - * Returns 1 on success and 0 on failure. If no entries with the given opaque - * pointer were found, this is considered a success condition. - */ -int ossl_quic_srtm_cull(QUIC_SRTM *strm, void *opaque); - -/* - * Looks up a SRT to find the corresponding opaque pointer and sequence number. - * An output field pointer can be set to NULL if it is not required. - * - * This function is designed to avoid exposing timing channels on token values - * or the contents of the SRT mapping. - * - * If there are several identical SRTs, idx can be used to get the nth entry. - * Call this function with idx set to 0 first, and keep calling it after - * incrementing idx until it returns 0. - * - * Returns 1 if an entry was found and 0 otherwise. - */ -int ossl_quic_srtm_lookup(QUIC_SRTM *srtm, - const QUIC_STATELESS_RESET_TOKEN *token, - size_t idx, - void **opaque, uint64_t *seq_num); - -/* Verify internal invariants and assert if they are not met. */ -void ossl_quic_srtm_check(const QUIC_SRTM *srtm); - -# endif - -#endif diff --git a/openssl/include/internal/quic_ssl.h b/openssl/include/internal/quic_ssl.h deleted file mode 100644 index 4fc7a21a5..000000000 --- a/openssl/include/internal/quic_ssl.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_SSL_H -# define OSSL_QUIC_SSL_H - -# include -# include -# include "internal/quic_record_rx.h" /* OSSL_QRX */ -# include "internal/quic_ackm.h" /* OSSL_ACKM */ -# include "internal/quic_channel.h" /* QUIC_CHANNEL */ - -# ifndef OPENSSL_NO_QUIC - -__owur SSL *ossl_quic_new(SSL_CTX *ctx); -__owur int ossl_quic_init(SSL *s); -void ossl_quic_deinit(SSL *s); -void ossl_quic_free(SSL *s); -int ossl_quic_reset(SSL *s); -int ossl_quic_clear(SSL *s); -__owur int ossl_quic_accept(SSL *s); -__owur int ossl_quic_connect(SSL *s); -__owur int ossl_quic_read(SSL *s, void *buf, size_t len, size_t *readbytes); -__owur int ossl_quic_peek(SSL *s, void *buf, size_t len, size_t *readbytes); -__owur int ossl_quic_write_flags(SSL *s, const void *buf, size_t len, - uint64_t flags, size_t *written); -__owur int ossl_quic_write(SSL *s, const void *buf, size_t len, size_t *written); -__owur long ossl_quic_ctrl(SSL *s, int cmd, long larg, void *parg); -__owur long ossl_quic_ctx_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg); -__owur long ossl_quic_callback_ctrl(SSL *s, int cmd, void (*fp) (void)); -__owur long ossl_quic_ctx_callback_ctrl(SSL_CTX *ctx, int cmd, void (*fp) (void)); -__owur size_t ossl_quic_pending(const SSL *s); -__owur int ossl_quic_key_update(SSL *s, int update_type); -__owur int ossl_quic_get_key_update_type(const SSL *s); -__owur const SSL_CIPHER *ossl_quic_get_cipher_by_char(const unsigned char *p); -__owur int ossl_quic_num_ciphers(void); -__owur const SSL_CIPHER *ossl_quic_get_cipher(unsigned int u); -int ossl_quic_renegotiate_check(SSL *ssl, int initok); - -typedef struct quic_conn_st QUIC_CONNECTION; -typedef struct quic_xso_st QUIC_XSO; - -int ossl_quic_do_handshake(SSL *s); -void ossl_quic_set_connect_state(SSL *s); -void ossl_quic_set_accept_state(SSL *s); - -__owur int ossl_quic_has_pending(const SSL *s); -__owur int ossl_quic_handle_events(SSL *s); -__owur int ossl_quic_get_event_timeout(SSL *s, struct timeval *tv, - int *is_infinite); -OSSL_TIME ossl_quic_get_event_deadline(SSL *s); -__owur int ossl_quic_get_rpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *d); -__owur int ossl_quic_get_wpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *d); -__owur int ossl_quic_get_net_read_desired(SSL *s); -__owur int ossl_quic_get_net_write_desired(SSL *s); -__owur int ossl_quic_get_error(const SSL *s, int i); -__owur int ossl_quic_want(const SSL *s); -__owur int ossl_quic_conn_get_blocking_mode(const SSL *s); -__owur int ossl_quic_conn_set_blocking_mode(SSL *s, int blocking); -__owur int ossl_quic_conn_shutdown(SSL *s, uint64_t flags, - const SSL_SHUTDOWN_EX_ARGS *args, - size_t args_len); -__owur int ossl_quic_conn_stream_conclude(SSL *s); -void ossl_quic_conn_set0_net_rbio(SSL *s, BIO *net_wbio); -void ossl_quic_conn_set0_net_wbio(SSL *s, BIO *net_wbio); -BIO *ossl_quic_conn_get_net_rbio(const SSL *s); -BIO *ossl_quic_conn_get_net_wbio(const SSL *s); -__owur int ossl_quic_conn_set_initial_peer_addr(SSL *s, - const BIO_ADDR *peer_addr); -__owur SSL *ossl_quic_conn_stream_new(SSL *s, uint64_t flags); -__owur SSL *ossl_quic_get0_connection(SSL *s); -__owur int ossl_quic_get_stream_type(SSL *s); -__owur uint64_t ossl_quic_get_stream_id(SSL *s); -__owur int ossl_quic_is_stream_local(SSL *s); -__owur int ossl_quic_set_default_stream_mode(SSL *s, uint32_t mode); -__owur SSL *ossl_quic_detach_stream(SSL *s); -__owur int ossl_quic_attach_stream(SSL *conn, SSL *stream); -__owur int ossl_quic_set_incoming_stream_policy(SSL *s, int policy, - uint64_t aec); -__owur SSL *ossl_quic_accept_stream(SSL *s, uint64_t flags); -__owur size_t ossl_quic_get_accept_stream_queue_len(SSL *s); -__owur int ossl_quic_get_value_uint(SSL *s, uint32_t class_, uint32_t id, - uint64_t *value); -__owur int ossl_quic_set_value_uint(SSL *s, uint32_t class_, uint32_t id, - uint64_t value); - -__owur int ossl_quic_stream_reset(SSL *ssl, - const SSL_STREAM_RESET_ARGS *args, - size_t args_len); - -__owur int ossl_quic_get_stream_read_state(SSL *ssl); -__owur int ossl_quic_get_stream_write_state(SSL *ssl); -__owur int ossl_quic_get_stream_read_error_code(SSL *ssl, - uint64_t *app_error_code); -__owur int ossl_quic_get_stream_write_error_code(SSL *ssl, - uint64_t *app_error_code); -__owur int ossl_quic_get_conn_close_info(SSL *ssl, - SSL_CONN_CLOSE_INFO *info, - size_t info_len); - -uint64_t ossl_quic_set_options(SSL *s, uint64_t opts); -uint64_t ossl_quic_clear_options(SSL *s, uint64_t opts); -uint64_t ossl_quic_get_options(const SSL *s); - -/* Modifies write buffer size for a stream. */ -__owur int ossl_quic_set_write_buffer_size(SSL *s, size_t size); - -/* - * Used to override ossl_time_now() for debug purposes. While this may be - * overridden at any time, expect strange results if you change it after - * connecting. - */ -int ossl_quic_conn_set_override_now_cb(SSL *s, - OSSL_TIME (*now_cb)(void *arg), - void *now_cb_arg); - -/* - * Condvar waiting in the assist thread doesn't support time faking as it relies - * on the OS's notion of time, thus this is used in test code to force a - * spurious wakeup instead. - */ -void ossl_quic_conn_force_assist_thread_wake(SSL *s); - -/* For use by tests only. */ -QUIC_CHANNEL *ossl_quic_conn_get_channel(SSL *s); - -int ossl_quic_has_pending(const SSL *s); -int ossl_quic_get_shutdown(const SSL *s); - -/* - * Set qlog diagnostic title. String is copied internally on success and need - * not remain allocated. Only has any effect if logging has not already begun. - * For use by tests only. Setting this on a context affects any QCSO created - * after this is called but does not affect QCSOs already created from a - * context. - */ -int ossl_quic_set_diag_title(SSL_CTX *ctx, const char *title); - -/* APIs used by the polling infrastructure */ -int ossl_quic_conn_poll_events(SSL *ssl, uint64_t events, int do_tick, - uint64_t *revents); - -# endif - -#endif diff --git a/openssl/include/internal/quic_statm.h b/openssl/include/internal/quic_statm.h deleted file mode 100644 index 2fca69b0d..000000000 --- a/openssl/include/internal/quic_statm.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_STATS_H -# define OSSL_QUIC_STATS_H - -# include -# include "internal/time.h" -# include "internal/quic_predef.h" - -# ifndef OPENSSL_NO_QUIC - -struct ossl_statm_st { - OSSL_TIME smoothed_rtt, latest_rtt, min_rtt, rtt_variance; - char have_first_sample; -}; - -typedef struct ossl_rtt_info_st { - /* As defined in RFC 9002. */ - OSSL_TIME smoothed_rtt, latest_rtt, rtt_variance, min_rtt; -} OSSL_RTT_INFO; - -int ossl_statm_init(OSSL_STATM *statm); - -void ossl_statm_destroy(OSSL_STATM *statm); - -void ossl_statm_get_rtt_info(OSSL_STATM *statm, OSSL_RTT_INFO *rtt_info); - -void ossl_statm_update_rtt(OSSL_STATM *statm, - OSSL_TIME ack_delay, - OSSL_TIME override_latest_rtt); - -# endif - -#endif diff --git a/openssl/include/internal/quic_stream.h b/openssl/include/internal/quic_stream.h deleted file mode 100644 index d446dadc5..000000000 --- a/openssl/include/internal/quic_stream.h +++ /dev/null @@ -1,428 +0,0 @@ -/* -* Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#ifndef OSSL_INTERNAL_QUIC_STREAM_H -# define OSSL_INTERNAL_QUIC_STREAM_H -# pragma once - -#include "internal/e_os.h" -#include "internal/time.h" -#include "internal/quic_types.h" -#include "internal/quic_predef.h" -#include "internal/quic_wire.h" -#include "internal/quic_record_tx.h" -#include "internal/quic_record_rx.h" -#include "internal/quic_fc.h" -#include "internal/quic_statm.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Send Stream - * ================ - * - * The QUIC Send Stream Manager (QUIC_SSTREAM) is responsible for: - * - * - accepting octet strings of stream data; - * - * - generating corresponding STREAM frames; - * - * - receiving notifications of lost frames, in order to generate new STREAM - * frames for the lost data; - * - * - receiving notifications of acknowledged frames, in order to internally - * reuse memory used to store acknowledged stream data; - * - * - informing the caller of how much more stream data it can accept into - * its internal buffers, so as to ensure that the amount of unacknowledged - * data which can be written to a stream is not infinite and to allow the - * caller to manifest backpressure conditions to the user. - * - * The QUIC_SSTREAM is instantiated once for every stream with a send component - * (i.e., for a unidirectional send stream or for the send component of a - * bidirectional stream). - * - * Note: The terms 'TX' and 'RX' are used when referring to frames, packets and - * datagrams. The terms 'send' and 'receive' are used when referring to the - * stream abstraction. Applications send; we transmit. - */ - -/* - * Instantiates a new QUIC_SSTREAM. init_buf_size specifies the initial size of - * the stream data buffer in bytes, which must be positive. - */ -QUIC_SSTREAM *ossl_quic_sstream_new(size_t init_buf_size); - -/* - * Frees a QUIC_SSTREAM and associated stream data storage. - * - * Any iovecs returned by ossl_quic_sstream_get_stream_frame cease to be valid after - * calling this function. - */ -void ossl_quic_sstream_free(QUIC_SSTREAM *qss); - -/* - * (For TX packetizer use.) Retrieves information about application stream data - * which is ready for transmission. - * - * *hdr is filled with the logical offset, maximum possible length of stream - * data which can be transmitted, and a pointer to the stream data to be - * transmitted. is_fin is set to 1 if hdr->offset + hdr->len is the final size - * of the stream and 0 otherwise. hdr->stream_id is not set; the caller must set - * it. - * - * The caller is not obligated to send all of the data. If the caller does not - * send all of the data, the caller must reduce hdr->len before serializing the - * header structure and must ensure that hdr->is_fin is cleared. - * - * hdr->has_explicit_len is always set. It is the caller's responsibility to - * clear this if it wants to use the optimization of omitting the length field, - * as only the caller can know when this optimization can be performed. - * - * *num_iov must be set to the size of the iov array at call time. When this - * function returns successfully, it is updated to the number of iov entries - * which have been written. - * - * The stream data may be split across up to two IOVs due to internal ring - * buffer organisation. The sum of the lengths of the IOVs and the value written - * to hdr->len will always match. If the caller decides to send less than - * hdr->len of stream data, it must adjust the IOVs accordingly. This may be - * done by updating hdr->len and then calling the utility function - * ossl_quic_sstream_adjust_iov(). - * - * After committing one or more bytes returned by ossl_quic_sstream_get_stream_frame to a - * packet, call ossl_quic_sstream_mark_transmitted with the inclusive range of logical - * byte numbers of the transmitted bytes (i.e., hdr->offset, hdr->offset + - * hdr->len - 1). If you do not call ossl_quic_sstream_mark_transmitted, the next call to - * ossl_quic_sstream_get_stream_frame will return the same data (or potentially the same - * and more, if more data has been appended by the application). - * - * It is the caller's responsibility to clamp the length of data which this - * function indicates is available according to other concerns, such as - * stream-level flow control, connection-level flow control, or the applicable - * maximum datagram payload length (MDPL) for a packet under construction. - * - * The skip argument can usually be given as zero. If it is non-zero, this - * function outputs a range which would be output if it were called again after - * calling ossl_quic_sstream_mark_transmitted() with the returned range, repeated 'skip' - * times, and so on. This may be useful for callers which wish to enumerate - * available stream frames and batch their calls to ossl_quic_sstream_mark_transmitted at - * a later time. - * - * On success, this function will never write *num_iov with a value other than - * 0, 1 or 2. A *num_iov value of 0 can only occurs when hdr->is_fin is set (for - * example, when a stream is closed after all existing data has been sent, and - * without sending any more data); otherwise the function returns 0 as there is - * nothing useful to report. - * - * Returns 1 on success and 0 if there is no stream data available for - * transmission, or on other error (such as if the caller provides fewer - * than two IOVs.) - */ -int ossl_quic_sstream_get_stream_frame(QUIC_SSTREAM *qss, - size_t skip, - OSSL_QUIC_FRAME_STREAM *hdr, - OSSL_QTX_IOVEC *iov, - size_t *num_iov); - -/* - * Returns 1 if there is data pending transmission. Equivalent to calling - * ossl_quic_sstream_get_stream_frame and seeing if it succeeds. - */ -int ossl_quic_sstream_has_pending(QUIC_SSTREAM *qss); - -/* - * Returns the current size of the stream; i.e., the number of bytes which have - * been appended to the stream so far. - */ -uint64_t ossl_quic_sstream_get_cur_size(QUIC_SSTREAM *qss); - -/* - * (For TX packetizer use.) Marks a logical range of the send stream as having - * been transmitted. - * - * 0 denotes the first byte ever sent on the stream. The start and end values - * are both inclusive, therefore all calls to this function always mark at least - * one byte as being transmitted; if no bytes have been transmitted, do not call - * this function. - * - * If the STREAM frame sent had the FIN bit set, you must also call - * ossl_quic_sstream_mark_transmitted_fin() after calling this function. - * - * If you sent a zero-length STREAM frame with the FIN bit set, you need only - * call ossl_quic_sstream_mark_transmitted_fin() and must not call this function. - * - * Returns 1 on success and 0 on error (e.g. if end < start). - */ -int ossl_quic_sstream_mark_transmitted(QUIC_SSTREAM *qss, - uint64_t start, - uint64_t end); - -/* - * (For TX packetizer use.) Marks a STREAM frame with the FIN bit set as having - * been transmitted. final_size is the final size of the stream (i.e., the value - * offset + len of the transmitted STREAM frame). - * - * This function fails returning 0 if ossl_quic_sstream_fin() has not been called or if - * final_size is not correct. The final_size argument is not strictly needed by - * the QUIC_SSTREAM but is required as a sanity check. - */ -int ossl_quic_sstream_mark_transmitted_fin(QUIC_SSTREAM *qss, - uint64_t final_size); - -/* - * (RX/ACKM use.) Marks a logical range of the send stream as having been lost. - * The send stream will return the lost data for retransmission on a future call - * to ossl_quic_sstream_get_stream_frame. The start and end values denote logical byte - * numbers and are inclusive. - * - * If the lost frame had the FIN bit set, you must also call - * ossl_quic_sstream_mark_lost_fin() after calling this function. - * - * Returns 1 on success and 0 on error (e.g. if end < start). - */ -int ossl_quic_sstream_mark_lost(QUIC_SSTREAM *qss, - uint64_t start, - uint64_t end); - -/* - * (RX/ACKM use.) Informs the QUIC_SSTREAM that a STREAM frame with the FIN bit - * set was lost. - * - * Returns 1 on success and 0 on error. - */ -int ossl_quic_sstream_mark_lost_fin(QUIC_SSTREAM *qss); - -/* - * (RX/ACKM use.) Marks a logical range of the send stream as having been - * acknowledged, meaning that the storage for the data in that range of the - * stream can be now recycled and neither that logical range of the stream nor - * any subset of it can be retransmitted again. The start and end values are - * inclusive. - * - * If the acknowledged frame had the FIN bit set, you must also call - * ossl_quic_sstream_mark_acked_fin() after calling this function. - * - * Returns 1 on success and 0 on error (e.g. if end < start). - */ -int ossl_quic_sstream_mark_acked(QUIC_SSTREAM *qss, - uint64_t start, - uint64_t end); - -/* - * (RX/ACKM use.) Informs the QUIC_SSTREAM that a STREAM frame with the FIN bit - * set was acknowledged. - * - * Returns 1 on success and 0 on error. - */ -int ossl_quic_sstream_mark_acked_fin(QUIC_SSTREAM *qss); - -/* - * (Front end use.) Appends user data to the stream. The data is copied into the - * stream. The amount of data consumed from buf is written to *consumed on - * success (short writes are possible). The amount of data which can be written - * can be determined in advance by calling the ossl_quic_sstream_get_buffer_avail() - * function; data is copied into an internal ring buffer of finite size. - * - * If the buffer is full, this should be materialised as a backpressure - * condition by the front end. This is not considered a failure condition; - * *consumed is written as 0 and the function returns 1. - * - * Returns 1 on success or 0 on failure. - */ -int ossl_quic_sstream_append(QUIC_SSTREAM *qss, - const unsigned char *buf, - size_t buf_len, - size_t *consumed); - -/* - * Marks a stream as finished. ossl_quic_sstream_append() may not be called anymore - * after calling this. - */ -void ossl_quic_sstream_fin(QUIC_SSTREAM *qss); - -/* - * If the stream has had ossl_quic_sstream_fin() called, returns 1 and writes - * the final size to *final_size. Otherwise, returns 0. - */ -int ossl_quic_sstream_get_final_size(QUIC_SSTREAM *qss, uint64_t *final_size); - -/* - * Returns 1 iff all bytes (and any FIN, if any) which have been appended to the - * QUIC_SSTREAM so far, and any FIN (if any), have been both sent and acked. - */ -int ossl_quic_sstream_is_totally_acked(QUIC_SSTREAM *qss); - -/* - * Resizes the internal ring buffer. All stream data is preserved safely. - * - * This can be used to expand or contract the ring buffer, but not to contract - * the ring buffer below the amount of stream data currently stored in it. - * Returns 1 on success and 0 on failure. - * - * IMPORTANT: Any buffers referenced by iovecs output by - * ossl_quic_sstream_get_stream_frame() cease to be valid after calling this function. - */ -int ossl_quic_sstream_set_buffer_size(QUIC_SSTREAM *qss, size_t num_bytes); - -/* - * Gets the internal ring buffer size in bytes. - */ -size_t ossl_quic_sstream_get_buffer_size(QUIC_SSTREAM *qss); - -/* - * Gets the number of bytes used in the internal ring buffer. - */ -size_t ossl_quic_sstream_get_buffer_used(QUIC_SSTREAM *qss); - -/* - * Gets the number of bytes free in the internal ring buffer. - */ -size_t ossl_quic_sstream_get_buffer_avail(QUIC_SSTREAM *qss); - -/* - * Utility function to ensure the length of an array of iovecs matches the - * length given as len. Trailing iovecs have their length values reduced or set - * to 0 as necessary. - */ -void ossl_quic_sstream_adjust_iov(size_t len, - OSSL_QTX_IOVEC *iov, - size_t num_iov); - -/* - * Sets flag to cleanse the buffered data when it is acked. - */ -void ossl_quic_sstream_set_cleanse(QUIC_SSTREAM *qss, int cleanse); - -/* - * QUIC Receive Stream Manager - * =========================== - * - * The QUIC Receive Stream Manager (QUIC_RSTREAM) is responsible for - * storing the received stream data frames until the application - * is able to read the data. - * - * The QUIC_RSTREAM is instantiated once for every stream that can receive data. - * (i.e., for a unidirectional receiving stream or for the receiving component - * of a bidirectional stream). - */ - -/* - * Create a new instance of QUIC_RSTREAM with pointers to the flow - * controller and statistics module. They can be NULL for unit testing. - * If they are non-NULL, the `rxfc` is called when receive stream data - * is read by application. `statm` is queried for current rtt. - * `rbuf_size` is the initial size of the ring buffer to be used - * when ossl_quic_rstream_move_to_rbuf() is called. - */ -QUIC_RSTREAM *ossl_quic_rstream_new(QUIC_RXFC *rxfc, - OSSL_STATM *statm, size_t rbuf_size); - -/* - * Frees a QUIC_RSTREAM and any associated storage. - */ -void ossl_quic_rstream_free(QUIC_RSTREAM *qrs); - -/* - * Adds received stream frame data to `qrs`. The `pkt_wrap` refcount is - * incremented if the `data` is queued directly without copying. - * It can be NULL for unit-testing purposes, i.e. if `data` is static or - * never released before calling ossl_quic_rstream_free(). - * The `offset` is the absolute offset of the data in the stream. - * `data_len` can be 0 - can be useful for indicating `fin` for empty stream. - * Or to indicate `fin` without any further data added to the stream. - */ - -int ossl_quic_rstream_queue_data(QUIC_RSTREAM *qrs, OSSL_QRX_PKT *pkt, - uint64_t offset, - const unsigned char *data, uint64_t data_len, - int fin); - -/* - * Copies the data from the stream storage to buffer `buf` of size `size`. - * `readbytes` is set to the number of bytes actually copied. - * `fin` is set to 1 if all the data from the stream were read so the - * stream is finished. It is set to 0 otherwise. - */ -int ossl_quic_rstream_read(QUIC_RSTREAM *qrs, unsigned char *buf, size_t size, - size_t *readbytes, int *fin); - -/* - * Peeks at the data in the stream storage. It copies them to buffer `buf` - * of size `size` and sets `readbytes` to the number of bytes actually copied. - * `fin` is set to 1 if the copied data reach end of the stream. - * It is set to 0 otherwise. - */ -int ossl_quic_rstream_peek(QUIC_RSTREAM *qrs, unsigned char *buf, size_t size, - size_t *readbytes, int *fin); - -/* - * Returns the size of the data available for reading. `fin` is set to 1 if - * after reading all the available data the stream will be finished, - * set to 0 otherwise. - */ -int ossl_quic_rstream_available(QUIC_RSTREAM *qrs, size_t *avail, int *fin); - -/* - * Sets *record to the beginning of the first readable stream data chunk and - * *reclen to the size of the chunk. *fin is set to 1 if the end of the - * chunk is the last of the stream data chunks. - * If there is no record available *record is set to NULL and *rec_len to 0; - * ossl_quic_rstream_release_record() should not be called in that case. - * Returns 1 on success (including calls if no record is available, or - * after end of the stream - in that case *fin will be set to 1 and - * *rec_len to 0), 0 on error. - * It is an error to call ossl_quic_rstream_get_record() multiple times - * without calling ossl_quic_rstream_release_record() in between. - */ -int ossl_quic_rstream_get_record(QUIC_RSTREAM *qrs, - const unsigned char **record, size_t *rec_len, - int *fin); - -/* - * Releases (possibly partially) the record returned by - * previous ossl_quic_rstream_get_record() call. - * read_len between previously returned *rec_len and SIZE_MAX indicates - * release of the whole record. Otherwise only part of the record is - * released. The remaining part of the record is unlocked, another - * call to ossl_quic_rstream_get_record() is needed to obtain further - * stream data. - * Returns 1 on success, 0 on error. - * It is an error to call ossl_quic_rstream_release_record() multiple - * times without calling ossl_quic_rstream_get_record() in between. - */ -int ossl_quic_rstream_release_record(QUIC_RSTREAM *qrs, size_t read_len); - -/* - * Moves received frame data from decrypted packets to ring buffer. - * This should be called when there are too many decrypted packets allocated. - * Returns 1 on success, 0 when it was not possible to release all - * referenced packets due to an insufficient size of the ring buffer. - * Exception is the packet from the record returned previously by - * ossl_quic_rstream_get_record() - that one will be always skipped. - */ -int ossl_quic_rstream_move_to_rbuf(QUIC_RSTREAM *qrs); - -/* - * Resizes the internal ring buffer to a new `rbuf_size` size. - * Returns 1 on success, 0 on error. - * Possible error conditions are an allocation failure, trying to resize - * the ring buffer when ossl_quic_rstream_get_record() was called and - * not yet released, or trying to resize the ring buffer to a smaller size - * than currently occupied. - */ -int ossl_quic_rstream_resize_rbuf(QUIC_RSTREAM *qrs, size_t rbuf_size); - -/* - * Sets flag to cleanse the buffered data when user reads it. - */ -void ossl_quic_rstream_set_cleanse(QUIC_RSTREAM *qrs, int cleanse); -# endif - -#endif diff --git a/openssl/include/internal/quic_stream_map.h b/openssl/include/internal/quic_stream_map.h deleted file mode 100644 index 745d9c03d..000000000 --- a/openssl/include/internal/quic_stream_map.h +++ /dev/null @@ -1,916 +0,0 @@ -/* -* Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#ifndef OSSL_INTERNAL_QUIC_STREAM_MAP_H -# define OSSL_INTERNAL_QUIC_STREAM_MAP_H -# pragma once - -# include "internal/e_os.h" -# include "internal/time.h" -# include "internal/common.h" -# include "internal/quic_types.h" -# include "internal/quic_predef.h" -# include "internal/quic_stream.h" -# include "internal/quic_fc.h" -# include - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Stream - * =========== - * - * Logical QUIC stream composing all relevant send and receive components. - */ - -typedef struct quic_stream_list_node_st QUIC_STREAM_LIST_NODE; - -struct quic_stream_list_node_st { - QUIC_STREAM_LIST_NODE *prev, *next; -}; - -/* - * QUIC Send Stream States - * ----------------------- - * - * These correspond to the states defined in RFC 9000 s. 3.1, with the - * exception of the NONE state which represents the absence of a send stream - * part. - * - * Invariants in each state are noted in comments below. In particular, once all - * data has been acknowledged received, or we have reset the stream, we don't - * need to keep the QUIC_SSTREAM and data buffers around. Of course, we also - * don't have a QUIC_SSTREAM on a receive-only stream. - */ -#define QUIC_SSTREAM_STATE_NONE 0 /* --- sstream == NULL */ -#define QUIC_SSTREAM_STATE_READY 1 /* \ */ -#define QUIC_SSTREAM_STATE_SEND 2 /* |-- sstream != NULL */ -#define QUIC_SSTREAM_STATE_DATA_SENT 3 /* / */ -#define QUIC_SSTREAM_STATE_DATA_RECVD 4 /* \ */ -#define QUIC_SSTREAM_STATE_RESET_SENT 5 /* |-- sstream == NULL */ -#define QUIC_SSTREAM_STATE_RESET_RECVD 6 /* / */ - -/* - * QUIC Receive Stream States - * -------------------------- - * - * These correspond to the states defined in RFC 9000 s. 3.2, with the exception - * of the NONE state which represents the absence of a receive stream part. - * - * Invariants in each state are noted in comments below. In particular, once all - * data has been read by the application, we don't need to keep the QUIC_RSTREAM - * and data buffers around. If the receive part is instead reset before it is - * finished, we also don't need to keep the QUIC_RSTREAM around. Finally, we - * don't need a QUIC_RSTREAM on a send-only stream. - */ -#define QUIC_RSTREAM_STATE_NONE 0 /* --- rstream == NULL */ -#define QUIC_RSTREAM_STATE_RECV 1 /* \ */ -#define QUIC_RSTREAM_STATE_SIZE_KNOWN 2 /* |-- rstream != NULL */ -#define QUIC_RSTREAM_STATE_DATA_RECVD 3 /* / */ -#define QUIC_RSTREAM_STATE_DATA_READ 4 /* \ */ -#define QUIC_RSTREAM_STATE_RESET_RECVD 5 /* |-- rstream == NULL */ -#define QUIC_RSTREAM_STATE_RESET_READ 6 /* / */ - -struct quic_stream_st { - QUIC_STREAM_LIST_NODE active_node; /* for use by QUIC_STREAM_MAP */ - QUIC_STREAM_LIST_NODE accept_node; /* accept queue of remotely-created streams */ - QUIC_STREAM_LIST_NODE ready_for_gc_node; /* queue of streams now ready for GC */ - - /* Temporary link used by TXP. */ - QUIC_STREAM *txp_next; - - /* - * QUIC Stream ID. Do not assume that this encodes a type as this is a - * version-specific property and may change between QUIC versions; instead, - * use the type field. - */ - uint64_t id; - - /* - * Application Error Code (AEC) used for STOP_SENDING frame. - * This is only valid if stop_sending is 1. - */ - uint64_t stop_sending_aec; - - /* - * Application Error Code (AEC) used for RESET_STREAM frame. - * This is only valid if reset_stream is 1. - */ - uint64_t reset_stream_aec; - - /* - * Application Error Code (AEC) for incoming STOP_SENDING frame. - * This is only valid if peer_stop_sending is 1. - */ - uint64_t peer_stop_sending_aec; - - /* - * Application Error Code (AEC) for incoming RESET_STREAM frame. - * This is only valid if peer_reset_stream is 1. - */ - uint64_t peer_reset_stream_aec; - - /* Temporary value used by TXP. */ - uint64_t txp_txfc_new_credit_consumed; - - /* - * The final size of the send stream. Although this information can be - * discerned from a QUIC_SSTREAM, it is stored separately as we need to keep - * track of this even if we have thrown away the QUIC_SSTREAM. Use - * ossl_quic_stream_send_get_final_size to determine if this contain a - * valid value or if there is no final size yet for a sending part. - * - * For the receive part, the final size is tracked by the stream-level RXFC; - * use ossl_quic_stream_recv_get_final_size or - * ossl_quic_rxfc_get_final_size. - */ - uint64_t send_final_size; - - /* - * Send stream part and receive stream part buffer management objects. - * - * DO NOT test these pointers (sstream, rstream) for NULL. Determine the - * state of the send or receive stream part first using the appropriate - * function; then the invariant of that state guarantees that sstream or - * rstream either is or is not NULL respectively, therefore there is no - * valid use case for testing these pointers for NULL. In particular, a - * stream with a send part can still have sstream as NULL, and a stream with - * a receive part can still have rstream as NULL. QUIC_SSTREAM and - * QUIC_RSTREAM are stream buffer resource management objects which exist - * only when they need to for buffer management purposes. The existence or - * non-existence of a QUIC_SSTREAM or QUIC_RSTREAM object does not - * correspond with whether a stream's respective send or receive part - * logically exists or not. - */ - QUIC_SSTREAM *sstream; /* NULL if RX-only */ - QUIC_RSTREAM *rstream; /* NULL if TX only */ - - /* Stream-level flow control managers. */ - QUIC_TXFC txfc; /* NULL if RX-only */ - QUIC_RXFC rxfc; /* NULL if TX-only */ - - unsigned int type : 8; /* QUIC_STREAM_INITIATOR_*, QUIC_STREAM_DIR_* */ - - unsigned int send_state : 8; /* QUIC_SSTREAM_STATE_* */ - unsigned int recv_state : 8; /* QUIC_RSTREAM_STATE_* */ - - /* 1 iff this QUIC_STREAM is on the active queue (invariant). */ - unsigned int active : 1; - - /* - * This is a copy of the QUIC connection as_server value, indicating - * whether we are locally operating as a server or not. Having this - * significantly simplifies stream type determination relative to our - * perspective. It never changes after a QUIC_STREAM is created and is the - * same for all QUIC_STREAMS under a QUIC_STREAM_MAP. - */ - unsigned int as_server : 1; - - /* - * Has STOP_SENDING been requested (by us)? Note that this is not the same - * as want_stop_sending below, as a STOP_SENDING frame may already have been - * sent and fully acknowledged. - */ - unsigned int stop_sending : 1; - - /* - * Has RESET_STREAM been requested (by us)? Works identically to - * STOP_SENDING for transmission purposes. - */ - /* Has our peer sent a STOP_SENDING frame? */ - unsigned int peer_stop_sending : 1; - - /* Temporary flags used by TXP. */ - unsigned int txp_sent_fc : 1; - unsigned int txp_sent_stop_sending : 1; - unsigned int txp_sent_reset_stream : 1; - unsigned int txp_drained : 1; - unsigned int txp_blocked : 1; - - /* Frame regeneration flags. */ - unsigned int want_max_stream_data : 1; /* used for regen only */ - unsigned int want_stop_sending : 1; /* used for gen or regen */ - unsigned int want_reset_stream : 1; /* used for gen or regen */ - - /* Flags set when frames *we* sent were acknowledged. */ - unsigned int acked_stop_sending : 1; - - /* - * The stream's XSO has been deleted. Pending GC. - * - * Here is how stream deletion works: - * - * - A QUIC_STREAM cannot be deleted until it is neither in the accept - * queue nor has an associated XSO. This condition occurs when and only - * when deleted is true. - * - * - Once this is the case (i.e., no user-facing API object exposing the - * stream), we can delete the stream once we determine that all of our - * protocol obligations requiring us to keep the QUIC_STREAM around have - * been met. - * - * The following frames relate to the streams layer for a specific - * stream: - * - * STREAM - * - * RX Obligations: - * Ignore for a deleted stream. - * - * (This is different from our obligation for a - * locally-initiated stream ID we have not created yet, - * which we must treat as a protocol error. This can be - * distinguished via a simple monotonic counter.) - * - * TX Obligations: - * None, once we've decided to (someday) delete the stream. - * - * STOP_SENDING - * - * We cannot delete the stream until we have finished informing - * the peer that we are not going to be listening to it - * anymore. - * - * RX Obligations: - * When we delete a stream we must have already had a FIN - * or RESET_STREAM we transmitted acknowledged by the peer. - * Thus we can ignore STOP_SENDING frames for deleted - * streams (if they occur, they are probably just - * retransmissions). - * - * TX Obligations: - * _Acknowledged_ receipt of a STOP_SENDING frame by the - * peer (unless the peer's send part has already FIN'd). - * - * RESET_STREAM - * - * We cannot delete the stream until we have finished informing - * the peer that we are not going to be transmitting on it - * anymore. - * - * RX Obligations: - * This indicates the peer is not going to send any more - * data on the stream. We don't need to care about this - * since once a stream is marked for deletion we don't care - * about any data it does send. We can ignore this for - * deleted streams. The important criterion is that the - * peer has been successfully delivered our STOP_SENDING - * frame. - * - * TX Obligations: - * _Acknowledged_ receipt of a RESET_STREAM frame or FIN by - * the peer. - * - * MAX_STREAM_DATA - * - * RX Obligations: - * Ignore. Since we are not going to be sending any more - * data on a stream once it has been marked for deletion, - * we don't need to care about flow control information. - * - * TX Obligations: - * None. - * - * In other words, our protocol obligation is simply: - * - * - either: - * - the peer has acknowledged receipt of a STOP_SENDING frame sent - * by us; -or- - * - we have received a FIN and all preceding segments from the peer - * - * [NOTE: The actual criterion required here is simply 'we have - * received a FIN from the peer'. However, due to reordering and - * retransmissions we might subsequently receive non-FIN segments - * out of order. The FIN means we know the peer will stop - * transmitting on the stream at *some* point, but by sending - * STOP_SENDING we can avoid these needless retransmissions we - * will just ignore anyway. In actuality we could just handle all - * cases by sending a STOP_SENDING. The strategy we choose is to - * only avoid sending a STOP_SENDING and rely on a received FIN - * when we have received all preceding data, as this makes it - * reasonably certain no benefit would be gained by sending - * STOP_SENDING.] - * - * TODO(QUIC FUTURE): Implement the latter case (currently we - just always do STOP_SENDING). - * - * and; - * - * - we have drained our send stream (for a finished send stream) - * and got acknowledgement all parts of it including the FIN, or - * sent a RESET_STREAM frame and got acknowledgement of that frame. - * - * Once these conditions are met, we can GC the QUIC_STREAM. - * - */ - unsigned int deleted : 1; - /* Set to 1 once the above conditions are actually met. */ - unsigned int ready_for_gc : 1; - /* Set to 1 if this is currently counted in the shutdown flush stream count. */ - unsigned int shutdown_flush : 1; -}; - -#define QUIC_STREAM_INITIATOR_CLIENT 0 -#define QUIC_STREAM_INITIATOR_SERVER 1 -#define QUIC_STREAM_INITIATOR_MASK 1 - -#define QUIC_STREAM_DIR_BIDI 0 -#define QUIC_STREAM_DIR_UNI 2 -#define QUIC_STREAM_DIR_MASK 2 - -void ossl_quic_stream_check(const QUIC_STREAM *s); - -/* - * Returns 1 if the QUIC_STREAM was initiated by the endpoint with the server - * role. - */ -static ossl_inline ossl_unused int ossl_quic_stream_is_server_init(const QUIC_STREAM *s) -{ - return (s->type & QUIC_STREAM_INITIATOR_MASK) == QUIC_STREAM_INITIATOR_SERVER; -} - -/* - * Returns 1 if the QUIC_STREAM is bidirectional and 0 if it is unidirectional. - */ -static ossl_inline ossl_unused int ossl_quic_stream_is_bidi(const QUIC_STREAM *s) -{ - return (s->type & QUIC_STREAM_DIR_MASK) == QUIC_STREAM_DIR_BIDI; -} - -/* Returns 1 if the QUIC_STREAM was locally initiated. */ -static ossl_inline ossl_unused int ossl_quic_stream_is_local_init(const QUIC_STREAM *s) -{ - return ossl_quic_stream_is_server_init(s) == s->as_server; -} - -/* - * Returns 1 if the QUIC_STREAM has a sending part, based on its stream type. - * - * Do NOT use (s->sstream != NULL) to test this; use this function. Note that - * even if this function returns 1, s->sstream might be NULL if the QUIC_SSTREAM - * has been deemed no longer needed, for example due to a RESET_STREAM. - */ -static ossl_inline ossl_unused int ossl_quic_stream_has_send(const QUIC_STREAM *s) -{ - return s->send_state != QUIC_SSTREAM_STATE_NONE; -} - -/* - * Returns 1 if the QUIC_STREAM has a receiving part, based on its stream type. - * - * Do NOT use (s->rstream != NULL) to test this; use this function. Note that - * even if this function returns 1, s->rstream might be NULL if the QUIC_RSTREAM - * has been deemed no longer needed, for example if the receive stream is - * completely finished with. - */ -static ossl_inline ossl_unused int ossl_quic_stream_has_recv(const QUIC_STREAM *s) -{ - return s->recv_state != QUIC_RSTREAM_STATE_NONE; -} - -/* - * Returns 1 if the QUIC_STREAM has a QUIC_SSTREAM send buffer associated with - * it. If this returns 1, s->sstream is guaranteed to be non-NULL. The converse - * is not necessarily true; erasure of a send stream buffer which is no longer - * required is an optimisation which the QSM may, but is not obliged, to - * perform. - * - * This call should be used where it is desired to do something with the send - * stream buffer but there is no more specific send state restriction which is - * applicable. - * - * Note: This does NOT indicate whether it is suitable to allow an application - * to append to the buffer. DATA_SENT indicates all data (including FIN) has - * been *sent*; the absence of DATA_SENT does not mean a FIN has not been queued - * (meaning no more application data can be appended). This is enforced by - * QUIC_SSTREAM. - */ -static ossl_inline ossl_unused int ossl_quic_stream_has_send_buffer(const QUIC_STREAM *s) -{ - switch (s->send_state) { - case QUIC_SSTREAM_STATE_READY: - case QUIC_SSTREAM_STATE_SEND: - case QUIC_SSTREAM_STATE_DATA_SENT: - return 1; - default: - return 0; - } -} - -/* - * Returns 1 if the QUIC_STREAM has a sending part which is in one of the reset - * states. - */ -static ossl_inline ossl_unused int ossl_quic_stream_send_is_reset(const QUIC_STREAM *s) -{ - return s->send_state == QUIC_SSTREAM_STATE_RESET_SENT - || s->send_state == QUIC_SSTREAM_STATE_RESET_RECVD; -} - -/* - * Returns 1 if the QUIC_STREAM has a QUIC_RSTREAM receive buffer associated - * with it. If this returns 1, s->rstream is guaranteed to be non-NULL. The - * converse is not necessarily true; erasure of a receive stream buffer which is - * no longer required is an optimisation which the QSM may, but is not obliged, - * to perform. - * - * This call should be used where it is desired to do something with the receive - * stream buffer but there is no more specific receive state restriction which is - * applicable. - */ -static ossl_inline ossl_unused int ossl_quic_stream_has_recv_buffer(const QUIC_STREAM *s) -{ - switch (s->recv_state) { - case QUIC_RSTREAM_STATE_RECV: - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - case QUIC_RSTREAM_STATE_DATA_RECVD: - return 1; - default: - return 0; - } -} - -/* - * Returns 1 if the QUIC_STREAM has a receiving part which is in one of the - * reset states. - */ -static ossl_inline ossl_unused int ossl_quic_stream_recv_is_reset(const QUIC_STREAM *s) -{ - return s->recv_state == QUIC_RSTREAM_STATE_RESET_RECVD - || s->recv_state == QUIC_RSTREAM_STATE_RESET_READ; -} - -/* - * Returns 1 if the stream has a send part and that part has a final size. - * - * If final_size is non-NULL, *final_size is the final size (on success) or an - * undefined value otherwise. - */ -static ossl_inline ossl_unused int ossl_quic_stream_send_get_final_size(const QUIC_STREAM *s, - uint64_t *final_size) -{ - switch (s->send_state) { - default: - case QUIC_SSTREAM_STATE_NONE: - return 0; - case QUIC_SSTREAM_STATE_SEND: - /* - * SEND may or may not have had a FIN - even if we have a FIN we do not - * move to DATA_SENT until we have actually sent all the data. So - * ask the QUIC_SSTREAM. - */ - return ossl_quic_sstream_get_final_size(s->sstream, final_size); - case QUIC_SSTREAM_STATE_DATA_SENT: - case QUIC_SSTREAM_STATE_DATA_RECVD: - case QUIC_SSTREAM_STATE_RESET_SENT: - case QUIC_SSTREAM_STATE_RESET_RECVD: - if (final_size != NULL) - *final_size = s->send_final_size; - return 1; - } -} - -/* - * Returns 1 if the stream has a receive part and that part has a final size. - * - * If final_size is non-NULL, *final_size is the final size (on success) or an - * undefined value otherwise. - */ -static ossl_inline ossl_unused int ossl_quic_stream_recv_get_final_size(const QUIC_STREAM *s, - uint64_t *final_size) -{ - switch (s->recv_state) { - default: - case QUIC_RSTREAM_STATE_NONE: - case QUIC_RSTREAM_STATE_RECV: - return 0; - - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - case QUIC_RSTREAM_STATE_DATA_RECVD: - case QUIC_RSTREAM_STATE_DATA_READ: - case QUIC_RSTREAM_STATE_RESET_RECVD: - case QUIC_RSTREAM_STATE_RESET_READ: - if (!ossl_assert(ossl_quic_rxfc_get_final_size(&s->rxfc, final_size))) - return 0; - - return 1; - } -} - -/* - * Determines the number of bytes available still to be read, and (if - * include_fin is 1) whether a FIN or reset has yet to be read. - */ -static ossl_inline ossl_unused int ossl_quic_stream_recv_pending(const QUIC_STREAM *s, - int include_fin) -{ - size_t avail; - int fin = 0; - - switch (s->recv_state) { - default: - case QUIC_RSTREAM_STATE_NONE: - return 0; - - case QUIC_RSTREAM_STATE_RECV: - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - case QUIC_RSTREAM_STATE_DATA_RECVD: - if (!ossl_quic_rstream_available(s->rstream, &avail, &fin)) - avail = 0; - - if (avail == 0 && include_fin && fin) - avail = 1; - - return avail; - - case QUIC_RSTREAM_STATE_RESET_RECVD: - return include_fin; - - case QUIC_RSTREAM_STATE_DATA_READ: - case QUIC_RSTREAM_STATE_RESET_READ: - return 0; - } -} - -/* - * QUIC Stream Map - * =============== - * - * The QUIC stream map: - * - * - maps stream IDs to QUIC_STREAM objects; - * - tracks which streams are 'active' (currently have data for transmission); - * - allows iteration over the active streams only. - * - */ -struct quic_stream_map_st { - LHASH_OF(QUIC_STREAM) *map; - QUIC_STREAM_LIST_NODE active_list; - QUIC_STREAM_LIST_NODE accept_list; - QUIC_STREAM_LIST_NODE ready_for_gc_list; - size_t rr_stepping, rr_counter; - size_t num_accept_bidi, num_accept_uni, num_shutdown_flush; - QUIC_STREAM *rr_cur; - uint64_t (*get_stream_limit_cb)(int uni, void *arg); - void *get_stream_limit_cb_arg; - QUIC_RXFC *max_streams_bidi_rxfc; - QUIC_RXFC *max_streams_uni_rxfc; - int is_server; -}; - -/* - * get_stream_limit is a callback which is called to retrieve the current stream - * limit for streams created by us. This mechanism is not used for - * peer-initiated streams. If a stream's stream ID is x, a stream is allowed if - * (x >> 2) < returned limit value; i.e., the returned value is exclusive. - * - * If uni is 1, get the limit for locally-initiated unidirectional streams, else - * get the limit for locally-initiated bidirectional streams. - * - * If the callback is NULL, stream limiting is not applied. - * Stream limiting is used to determine if frames can currently be produced for - * a stream. - */ -int ossl_quic_stream_map_init(QUIC_STREAM_MAP *qsm, - uint64_t (*get_stream_limit_cb)(int uni, void *arg), - void *get_stream_limit_cb_arg, - QUIC_RXFC *max_streams_bidi_rxfc, - QUIC_RXFC *max_streams_uni_rxfc, - int is_server); - -/* - * Any streams still in the map will be released as though - * ossl_quic_stream_map_release was called on them. - */ -void ossl_quic_stream_map_cleanup(QUIC_STREAM_MAP *qsm); - -/* - * Allocate a new stream. type is a combination of one QUIC_STREAM_INITIATOR_* - * value and one QUIC_STREAM_DIR_* value. Note that clients can e.g. allocate - * server-initiated streams as they will need to allocate a QUIC_STREAM - * structure to track any stream created by the server, etc. - * - * stream_id must be a valid value. Returns NULL if a stream already exists - * with the given ID. - */ -QUIC_STREAM *ossl_quic_stream_map_alloc(QUIC_STREAM_MAP *qsm, - uint64_t stream_id, - int type); - -/* - * Releases a stream object. Note that this must only be done once the teardown - * process is entirely complete and the object will never be referenced again. - */ -void ossl_quic_stream_map_release(QUIC_STREAM_MAP *qsm, QUIC_STREAM *stream); - -/* - * Calls visit_cb() for each stream in the map. visit_cb_arg is an opaque - * argument which is passed through. - */ -void ossl_quic_stream_map_visit(QUIC_STREAM_MAP *qsm, - void (*visit_cb)(QUIC_STREAM *stream, void *arg), - void *visit_cb_arg); - -/* - * Retrieves a stream by stream ID. Returns NULL if it does not exist. - */ -QUIC_STREAM *ossl_quic_stream_map_get_by_id(QUIC_STREAM_MAP *qsm, - uint64_t stream_id); - -/* - * Marks the given stream as active or inactive based on its state. Idempotent. - * - * When a stream is marked active, it becomes available in the iteration list, - * and when a stream is marked inactive, it no longer appears in the iteration - * list. - * - * Calling this function invalidates any iterator currently pointing at the - * given stream object, but iterators not currently pointing at the given stream - * object are not invalidated. - */ -void ossl_quic_stream_map_update_state(QUIC_STREAM_MAP *qsm, QUIC_STREAM *s); - -/* - * Sets the RR stepping value, n. The RR rotation will be advanced every n - * packets. The default value is 1. - */ -void ossl_quic_stream_map_set_rr_stepping(QUIC_STREAM_MAP *qsm, size_t stepping); - -/* - * Returns 1 if the stream ordinal given is allowed by the current stream count - * flow control limit, assuming a locally initiated stream of a type described - * by is_uni. - * - * Note that stream_ordinal is a stream ordinal, not a stream ID. - */ -int ossl_quic_stream_map_is_local_allowed_by_stream_limit(QUIC_STREAM_MAP *qsm, - uint64_t stream_ordinal, - int is_uni); - -/* - * Stream Send Part - * ================ - */ - -/* - * Ensures that the sending part has transitioned out of the READY state (i.e., - * to SEND, or a subsequent state). This function is named as it is because, - * while on paper the distinction between READY and SEND is whether we have - * started transmitting application data, in practice the meaningful distinction - * between the two states is whether we have allocated a stream ID to the stream - * or not. QUIC permits us to defer stream ID allocation until first STREAM (or - * STREAM_DATA_BLOCKED) frame transmission for locally-initiated streams. - * - * Our implementation does not currently do this and we allocate stream IDs up - * front, however we may revisit this in the future. Calling this represents a - * demand for a stream ID by the caller and ensures one has been allocated to - * the stream, and causes us to transition to SEND if we are still in the READY - * state. - * - * Returns 0 if there is no send part (caller error) and 1 otherwise. - */ -int ossl_quic_stream_map_ensure_send_part_id(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs); - -/* - * Transitions from SEND to the DATA_SENT state. Note that this is NOT the same - * as the point in time at which the final size of the stream becomes known - * (i.e., the time at which ossl_quic_sstream_fin()) is called as it occurs when - * we have SENT all data on a given stream send part, not merely buffered it. - * Note that this transition is NOT reversed in the event of some of that data - * being lost. - * - * Returns 1 if the state transition was successfully taken. Returns 0 if there - * is no send part (caller error) or if the state transition cannot be taken - * because the send part is not in the SEND state. - */ -int ossl_quic_stream_map_notify_all_data_sent(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs); - -/* - * Transitions from the DATA_SENT to DATA_RECVD state; should be called - * when all transmitted stream data is ACKed by the peer. - * - * Returns 1 if the state transition was successfully taken. Returns 0 if there - * is no send part (caller error) or the state transition cannot be taken - * because the send part is not in the DATA_SENT state. Because - * ossl_quic_stream_map_notify_all_data_sent() should always be called prior to - * this function, the send state must already be in DATA_SENT in order for this - * function to succeed. - */ -int ossl_quic_stream_map_notify_totally_acked(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs); - -/* - * Resets the sending part of a stream. This is a transition from the READY, - * SEND or DATA_SENT send stream states to the RESET_SENT state. - * - * This function returns 1 if the transition is taken (i.e., if the send stream - * part was in one of the states above), or if it is already in the RESET_SENT - * state (idempotent operation), or if it has reached the RESET_RECVD state. - * - * It returns 0 if in the DATA_RECVD state, as a send stream cannot be reset - * in this state. It also returns 0 if there is no send part (caller error). - */ -int ossl_quic_stream_map_reset_stream_send_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs, - uint64_t aec); - -/* - * Transitions from the RESET_SENT to the RESET_RECVD state. This should be - * called when a sent RESET_STREAM frame has been acknowledged by the peer. - * - * This function returns 1 if the transition is taken (i.e., if the send stream - * part was in one of the states above) or if it is already in the RESET_RECVD - * state (idempotent operation). - * - * It returns 0 if not in the RESET_SENT or RESET_RECVD states, as this function - * should only be called after we have already sent a RESET_STREAM frame and - * entered the RESET_SENT state. It also returns 0 if there is no send part - * (caller error). - */ -int ossl_quic_stream_map_notify_reset_stream_acked(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs); - - -/* - * Stream Receive Part - * =================== - */ - -/* - * Transitions from the RECV receive stream state to the SIZE_KNOWN state. This - * should be called once a STREAM frame is received for the stream with the FIN - * bit set. final_size should be the final size of the stream in bytes. - * - * Returns 1 if the transition was taken. - */ -int ossl_quic_stream_map_notify_size_known_recv_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs, - uint64_t final_size); - -/* - * Transitions from the SIZE_KNOWN receive stream state to the DATA_RECVD state. - * This should be called once all data for a receive stream is received. - * - * Returns 1 if the transition was taken. - */ -int ossl_quic_stream_map_notify_totally_received(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs); - -/* - * Transitions from the DATA_RECVD receive stream state to the DATA_READ state. - * This should be called once all data for a receive stream is read by the - * application. - * - * Returns 1 if the transition was taken. - */ -int ossl_quic_stream_map_notify_totally_read(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs); - -/* - * Transitions from the RECV, SIZE_KNOWN or DATA_RECVD receive stream state to - * the RESET_RECVD state. This should be called on RESET_STREAM. - * - * Returns 1 if the transition was taken. - */ -int ossl_quic_stream_map_notify_reset_recv_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs, - uint64_t app_error_code, - uint64_t final_size); - -/* - * Transitions from the RESET_RECVD receive stream state to the RESET_READ - * receive stream state. This should be called when the application is notified - * of a stream reset. - */ -int ossl_quic_stream_map_notify_app_read_reset_recv_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs); - -/* - * Marks the receiving part of a stream for STOP_SENDING. This is orthogonal to - * receive stream state as it does not affect it directly. - * - * Returns 1 if the receiving part of a stream was not already marked for - * STOP_SENDING. - * Returns 0 otherwise, which need not be considered an error. - */ -int ossl_quic_stream_map_stop_sending_recv_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs, - uint64_t aec); - -/* - * Marks the stream as wanting a STOP_SENDING frame transmitted. It is not valid - * to call this if ossl_quic_stream_map_stop_sending_recv_part() has not been - * called. For TXP use. - */ -int ossl_quic_stream_map_schedule_stop_sending(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs); - - -/* - * Accept Queue Management - * ======================= - */ - -/* - * Adds a stream to the accept queue. - */ -void ossl_quic_stream_map_push_accept_queue(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *s); - -/* - * Returns the next item to be popped from the accept queue, or NULL if it is - * empty. - */ -QUIC_STREAM *ossl_quic_stream_map_peek_accept_queue(QUIC_STREAM_MAP *qsm); - -/* - * Removes a stream from the accept queue. rtt is the estimated connection RTT. - * The stream is retired for the purposes of MAX_STREAMS RXFC. - * - * Precondition: s is in the accept queue. - */ -void ossl_quic_stream_map_remove_from_accept_queue(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *s, - OSSL_TIME rtt); - -/* Returns the length of the accept queue for the given stream type. */ -size_t ossl_quic_stream_map_get_accept_queue_len(QUIC_STREAM_MAP *qsm, int is_uni); - -/* Returns the total length of the accept queues for all stream types. */ -size_t ossl_quic_stream_map_get_total_accept_queue_len(QUIC_STREAM_MAP *qsm); - -/* - * Shutdown Flush and GC - * ===================== - */ - -/* - * Delete streams ready for GC. Pointers to those QUIC_STREAM objects become - * invalid. - */ -void ossl_quic_stream_map_gc(QUIC_STREAM_MAP *qsm); - -/* - * Begins shutdown stream flush triage. Analyses all streams, including deleted - * but not yet GC'd streams, to determine if we should wait for that stream to - * be fully flushed before shutdown. After calling this, call - * ossl_quic_stream_map_is_shutdown_flush_finished() to determine if all - * shutdown flush eligible streams have been flushed. - */ -void ossl_quic_stream_map_begin_shutdown_flush(QUIC_STREAM_MAP *qsm); - -/* - * Returns 1 if all shutdown flush eligible streams have finished flushing, - * or if ossl_quic_stream_map_begin_shutdown_flush() has not been called. - */ -int ossl_quic_stream_map_is_shutdown_flush_finished(QUIC_STREAM_MAP *qsm); - -/* - * QUIC Stream Iterator - * ==================== - * - * Allows the current set of active streams to be walked using a RR-based - * algorithm. Each time ossl_quic_stream_iter_init is called, the RR algorithm - * is stepped. The RR algorithm rotates the iteration order such that the next - * active stream is returned first after n calls to ossl_quic_stream_iter_init, - * where n is the stepping value configured via - * ossl_quic_stream_map_set_rr_stepping. - * - * Suppose there are three active streams and the configured stepping is n: - * - * Iteration 0n: [Stream 1] [Stream 2] [Stream 3] - * Iteration 1n: [Stream 2] [Stream 3] [Stream 1] - * Iteration 2n: [Stream 3] [Stream 1] [Stream 2] - * - */ -typedef struct quic_stream_iter_st { - QUIC_STREAM_MAP *qsm; - QUIC_STREAM *first_stream, *stream; -} QUIC_STREAM_ITER; - -/* - * Initialise an iterator, advancing the RR algorithm as necessary (if - * advance_rr is 1). After calling this, it->stream will be the first stream in - * the iteration sequence, or NULL if there are no active streams. - */ -void ossl_quic_stream_iter_init(QUIC_STREAM_ITER *it, QUIC_STREAM_MAP *qsm, - int advance_rr); - -/* - * Advances to next stream in iteration sequence. You do not need to call this - * immediately after calling ossl_quic_stream_iter_init(). If the end of the - * list is reached, it->stream will be NULL after calling this. - */ -void ossl_quic_stream_iter_next(QUIC_STREAM_ITER *it); - -# endif - -#endif diff --git a/openssl/include/internal/quic_thread_assist.h b/openssl/include/internal/quic_thread_assist.h deleted file mode 100644 index 592c2ffab..000000000 --- a/openssl/include/internal/quic_thread_assist.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_THREAD_ASSIST_H -# define OSSL_QUIC_THREAD_ASSIST_H - -# include -# include "internal/thread.h" -# include "internal/time.h" - -# if defined(OPENSSL_NO_QUIC) || defined(OPENSSL_NO_THREAD_POOL) -# define OPENSSL_NO_QUIC_THREAD_ASSIST -# endif - -# ifndef OPENSSL_NO_QUIC_THREAD_ASSIST - -/* - * QUIC Thread Assisted Functionality - * ================================== - * - * Where OS threading support is available, QUIC can optionally support a thread - * assisted mode of operation. The purpose of this mode of operation is to - * ensure that assorted timeout events which QUIC expects to be handled in a - * timely manner can be handled without the application needing to ensure that - * SSL_tick() is called on time. This is not needed if the application always - * has a call blocking to SSL_read() or SSL_write() (or another I/O function) on - * a QUIC SSL object, but if the application goes for long periods of time - * without making any such call to a QUIC SSL object, libssl cannot ordinarily - * guarantee that QUIC timeout events will be serviced in a timely fashion. - * Thread assisted mode is therefore of use to applications which do not always - * have an ongoing call to an I/O function on a QUIC SSL object but also do not - * want to have to arrange periodic ticking. - * - * A consequence of this is that the intrusiveness of thread assisted mode upon - * the general architecture of our QUIC engine is actually fairly limited and - * amounts to an automatic ticking of the QUIC engine when timeouts expire, - * synchronised correctly with an application's own threads using locking. - */ -typedef struct quic_thread_assist_st { - QUIC_CHANNEL *ch; - CRYPTO_CONDVAR *cv; - CRYPTO_THREAD *t; - int teardown, joined; - OSSL_TIME (*now_cb)(void *arg); - void *now_cb_arg; -} QUIC_THREAD_ASSIST; - -/* - * Initialise the thread assist object. The channel must have a valid mutex - * configured on it which will be retrieved automatically. It is assumed that - * the mutex is currently held when this function is called. This function does - * not affect the state of the mutex. - */ -int ossl_quic_thread_assist_init_start(QUIC_THREAD_ASSIST *qta, - QUIC_CHANNEL *ch, - OSSL_TIME (*now_cb)(void *arg), - void *now_cb_arg); - -/* - * Request the thread assist helper to begin stopping the assist thread. This - * returns before the teardown is complete. Idempotent; multiple calls to this - * function are inconsequential. - * - * Precondition: channel mutex must be held (unchecked) - */ -int ossl_quic_thread_assist_stop_async(QUIC_THREAD_ASSIST *qta); - -/* - * Wait until the thread assist helper is torn down. This automatically implies - * the effects of ossl_quic_thread_assist_stop_async(). Returns immediately - * if the teardown has already completed. - * - * Precondition: channel mutex must be held (unchecked) - */ -int ossl_quic_thread_assist_wait_stopped(QUIC_THREAD_ASSIST *qta); - -/* - * Deallocates state associated with the thread assist helper. - * ossl_quic_thread_assist_wait_stopped() must have returned successfully before - * calling this. It does not matter whether the channel mutex is held or not. - * - * Precondition: ossl_quic_thread_assist_wait_stopped() has returned 1 - * (asserted) - */ -int ossl_quic_thread_assist_cleanup(QUIC_THREAD_ASSIST *qta); - -/* - * Must be called to notify the assist thread if the channel deadline changes. - * - * Precondition: channel mutex must be held (unchecked) - */ -int ossl_quic_thread_assist_notify_deadline_changed(QUIC_THREAD_ASSIST *qta); - -# endif - -#endif diff --git a/openssl/include/internal/quic_tls.h b/openssl/include/internal/quic_tls.h deleted file mode 100644 index f9f007a76..000000000 --- a/openssl/include/internal/quic_tls.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_TLS_H -# define OSSL_QUIC_TLS_H - -# include -# include "internal/quic_stream.h" -# include "internal/quic_predef.h" - -# ifndef OPENSSL_NO_QUIC - -typedef struct quic_tls_args_st { - /* - * The "inner" SSL object for the QUIC Connection. Contains an - * SSL_CONNECTION - */ - SSL *s; - - /* - * Called to send data on the crypto stream. We use a callback rather than - * passing the crypto stream QUIC_SSTREAM directly because this lets the CSM - * dynamically select the correct outgoing crypto stream based on the - * current EL. - */ - int (*crypto_send_cb)(const unsigned char *buf, size_t buf_len, - size_t *consumed, void *arg); - void *crypto_send_cb_arg; - - /* - * Call to receive crypto stream data. A pointer to the underlying buffer - * is provided, and subsequently released to avoid unnecessary copying of - * data. - */ - int (*crypto_recv_rcd_cb)(const unsigned char **buf, size_t *bytes_read, - void *arg); - void *crypto_recv_rcd_cb_arg; - int (*crypto_release_rcd_cb)(size_t bytes_read, void *arg); - void *crypto_release_rcd_cb_arg; - - - /* Called when a traffic secret is available for a given encryption level. */ - int (*yield_secret_cb)(uint32_t enc_level, int direction /* 0=RX, 1=TX */, - uint32_t suite_id, EVP_MD *md, - const unsigned char *secret, size_t secret_len, - void *arg); - void *yield_secret_cb_arg; - - /* - * Called when we receive transport parameters from the peer. - * - * Note: These parameters are not authenticated until the handshake is - * marked as completed. - */ - int (*got_transport_params_cb)(const unsigned char *params, - size_t params_len, - void *arg); - void *got_transport_params_cb_arg; - - /* - * Called when the handshake has been completed as far as the handshake - * protocol is concerned, meaning that the connection has been - * authenticated. - */ - int (*handshake_complete_cb)(void *arg); - void *handshake_complete_cb_arg; - - /* - * Called when something has gone wrong with the connection as far as the - * handshake layer is concerned, meaning that it should be immediately torn - * down. Note that this may happen at any time, including after a connection - * has been fully established. - */ - int (*alert_cb)(void *arg, unsigned char alert_code); - void *alert_cb_arg; - - /* Set to 1 if we are running in the server role. */ - int is_server; -} QUIC_TLS_ARGS; - -QUIC_TLS *ossl_quic_tls_new(const QUIC_TLS_ARGS *args); - -void ossl_quic_tls_free(QUIC_TLS *qtls); - -/* Advance the state machine */ -int ossl_quic_tls_tick(QUIC_TLS *qtls); - -int ossl_quic_tls_set_transport_params(QUIC_TLS *qtls, - const unsigned char *transport_params, - size_t transport_params_len); - -int ossl_quic_tls_get_error(QUIC_TLS *qtls, - uint64_t *error_code, - const char **error_msg, - ERR_STATE **error_state); - -int ossl_quic_tls_is_cert_request(QUIC_TLS *qtls); -int ossl_quic_tls_has_bad_max_early_data(QUIC_TLS *qtls); - -# endif - -#endif diff --git a/openssl/include/internal/quic_tserver.h b/openssl/include/internal/quic_tserver.h deleted file mode 100644 index 4f358dd4e..000000000 --- a/openssl/include/internal/quic_tserver.h +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_TSERVER_H -# define OSSL_QUIC_TSERVER_H - -# include -# include -# include "internal/quic_stream.h" -# include "internal/quic_channel.h" -# include "internal/statem.h" -# include "internal/time.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Test Server Module - * ======================= - * - * This implements a QUIC test server. Since full QUIC server support is not yet - * implemented this server is limited in features and scope. It exists to - * provide a target for our QUIC client to talk to for testing purposes. - * - * A given QUIC test server instance supports only one client at a time. - * - * Note that this test server is not suitable for production use because it does - * not implement address verification, anti-amplification or retry logic. - */ -typedef struct quic_tserver_st QUIC_TSERVER; - -typedef struct quic_tserver_args_st { - OSSL_LIB_CTX *libctx; - const char *propq; - SSL_CTX *ctx; - BIO *net_rbio, *net_wbio; - OSSL_TIME (*now_cb)(void *arg); - void *now_cb_arg; - const unsigned char *alpn; - size_t alpnlen; -} QUIC_TSERVER_ARGS; - -QUIC_TSERVER *ossl_quic_tserver_new(const QUIC_TSERVER_ARGS *args, - const char *certfile, const char *keyfile); - -void ossl_quic_tserver_free(QUIC_TSERVER *srv); - -/* Set mutator callbacks for test framework support */ -int ossl_quic_tserver_set_plain_packet_mutator(QUIC_TSERVER *srv, - ossl_mutate_packet_cb mutatecb, - ossl_finish_mutate_cb finishmutatecb, - void *mutatearg); - -int ossl_quic_tserver_set_handshake_mutator(QUIC_TSERVER *srv, - ossl_statem_mutate_handshake_cb mutate_handshake_cb, - ossl_statem_finish_mutate_handshake_cb finish_mutate_handshake_cb, - void *mutatearg); - -/* Advances the state machine. */ -int ossl_quic_tserver_tick(QUIC_TSERVER *srv); - -/* Returns 1 if we have a (non-terminated) client. */ -int ossl_quic_tserver_is_connected(QUIC_TSERVER *srv); - -/* - * Returns 1 if we have finished the TLS handshake - */ -int ossl_quic_tserver_is_handshake_confirmed(const QUIC_TSERVER *srv); - -/* Returns 1 if the server is in any terminating or terminated state */ -int ossl_quic_tserver_is_term_any(const QUIC_TSERVER *srv); - -const QUIC_TERMINATE_CAUSE * -ossl_quic_tserver_get_terminate_cause(const QUIC_TSERVER *srv); - -/* Returns 1 if the server is in a terminated state */ -int ossl_quic_tserver_is_terminated(const QUIC_TSERVER *srv); - -/* - * Attempts to read from stream 0. Writes the number of bytes read to - * *bytes_read and returns 1 on success. If no bytes are available, 0 is written - * to *bytes_read and 1 is returned (this is considered a success case). - * - * Returns 0 if connection is not currently active. If the receive part of - * the stream has reached the end of stream condition, returns 0; call - * ossl_quic_tserver_has_read_ended() to identify this condition. - */ -int ossl_quic_tserver_read(QUIC_TSERVER *srv, - uint64_t stream_id, - unsigned char *buf, - size_t buf_len, - size_t *bytes_read); - -/* - * Returns 1 if the read part of the stream has ended normally. - */ -int ossl_quic_tserver_has_read_ended(QUIC_TSERVER *srv, uint64_t stream_id); - -/* - * Attempts to write to the given stream. Writes the number of bytes consumed to - * *bytes_written and returns 1 on success. If there is no space currently - * available to write any bytes, 0 is written to *consumed and 1 is returned - * (this is considered a success case). - * - * Note that unlike libssl public APIs, this API always works in a 'partial - * write' mode. - * - * Returns 0 if connection is not currently active. - */ -int ossl_quic_tserver_write(QUIC_TSERVER *srv, - uint64_t stream_id, - const unsigned char *buf, - size_t buf_len, - size_t *bytes_written); - -/* - * Signals normal end of the stream. - */ -int ossl_quic_tserver_conclude(QUIC_TSERVER *srv, uint64_t stream_id); - -/* - * Create a server-initiated stream. The stream ID of the newly - * created stream is written to *stream_id. - */ -int ossl_quic_tserver_stream_new(QUIC_TSERVER *srv, - int is_uni, - uint64_t *stream_id); - -BIO *ossl_quic_tserver_get0_rbio(QUIC_TSERVER *srv); - -SSL_CTX *ossl_quic_tserver_get0_ssl_ctx(QUIC_TSERVER *srv); - -/* - * Returns 1 if the peer has sent a STOP_SENDING frame for a stream. - * app_error_code is written if this returns 1. - */ -int ossl_quic_tserver_stream_has_peer_stop_sending(QUIC_TSERVER *srv, - uint64_t stream_id, - uint64_t *app_error_code); - -/* - * Returns 1 if the peer has sent a RESET_STREAM frame for a stream. - * app_error_code is written if this returns 1. - */ -int ossl_quic_tserver_stream_has_peer_reset_stream(QUIC_TSERVER *srv, - uint64_t stream_id, - uint64_t *app_error_code); - -/* - * Replaces existing local connection ID in the underlying QUIC_CHANNEL. - */ -int ossl_quic_tserver_set_new_local_cid(QUIC_TSERVER *srv, - const QUIC_CONN_ID *conn_id); - -/* - * Returns the stream ID of the next incoming stream, or UINT64_MAX if there - * currently is none. - */ -uint64_t ossl_quic_tserver_pop_incoming_stream(QUIC_TSERVER *srv); - -/* - * Returns 1 if all data sent on the given stream_id has been acked by the peer. - */ -int ossl_quic_tserver_is_stream_totally_acked(QUIC_TSERVER *srv, - uint64_t stream_id); - -/* Returns 1 if we are currently interested in reading data from the network */ -int ossl_quic_tserver_get_net_read_desired(QUIC_TSERVER *srv); - -/* Returns 1 if we are currently interested in writing data to the network */ -int ossl_quic_tserver_get_net_write_desired(QUIC_TSERVER *srv); - -/* Returns the next event deadline */ -OSSL_TIME ossl_quic_tserver_get_deadline(QUIC_TSERVER *srv); - -/* - * Shutdown the QUIC connection. Returns 1 if the connection is terminated and - * 0 otherwise. - */ -int ossl_quic_tserver_shutdown(QUIC_TSERVER *srv, uint64_t app_error_code); - -/* Force generation of an ACK-eliciting packet. */ -int ossl_quic_tserver_ping(QUIC_TSERVER *srv); - -/* Set tracing callback on channel. */ -void ossl_quic_tserver_set_msg_callback(QUIC_TSERVER *srv, - void (*f)(int write_p, int version, - int content_type, - const void *buf, size_t len, - SSL *ssl, void *arg), - void *arg); - -/* - * This is similar to ossl_quic_conn_get_channel; it should be used for test - * instrumentation only and not to bypass QUIC_TSERVER for 'normal' operations. - */ -QUIC_CHANNEL *ossl_quic_tserver_get_channel(QUIC_TSERVER *srv); - -/* Send a TLS new session ticket */ -int ossl_quic_tserver_new_ticket(QUIC_TSERVER *srv); - -/* - * Set the max_early_data value to be sent in NewSessionTickets. Only the - * values 0 and 0xffffffff are valid for use in QUIC. - */ -int ossl_quic_tserver_set_max_early_data(QUIC_TSERVER *srv, - uint32_t max_early_data); - -/* Set the find session callback for getting a server PSK */ -void ossl_quic_tserver_set_psk_find_session_cb(QUIC_TSERVER *srv, - SSL_psk_find_session_cb_func cb); - -# endif - -#endif diff --git a/openssl/include/internal/quic_txp.h b/openssl/include/internal/quic_txp.h deleted file mode 100644 index 607cefc01..000000000 --- a/openssl/include/internal/quic_txp.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_TXP_H -# define OSSL_QUIC_TXP_H - -# include -# include "internal/quic_types.h" -# include "internal/quic_predef.h" -# include "internal/quic_record_tx.h" -# include "internal/quic_cfq.h" -# include "internal/quic_txpim.h" -# include "internal/quic_stream.h" -# include "internal/quic_stream_map.h" -# include "internal/quic_fc.h" -# include "internal/bio_addr.h" -# include "internal/time.h" -# include "internal/qlog.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC TX Packetiser - * ================== - */ -typedef struct ossl_quic_tx_packetiser_args_st { - /* Configuration Settings */ - QUIC_CONN_ID cur_scid; /* Current Source Connection ID we use. */ - QUIC_CONN_ID cur_dcid; /* Current Destination Connection ID we use. */ - BIO_ADDR peer; /* Current destination L4 address we use. */ - uint32_t ack_delay_exponent; /* ACK delay exponent used when encoding. */ - - /* Injected Dependencies */ - OSSL_QTX *qtx; /* QUIC Record Layer TX we are using */ - QUIC_TXPIM *txpim; /* QUIC TX'd Packet Information Manager */ - QUIC_CFQ *cfq; /* QUIC Control Frame Queue */ - OSSL_ACKM *ackm; /* QUIC Acknowledgement Manager */ - QUIC_STREAM_MAP *qsm; /* QUIC Streams Map */ - QUIC_TXFC *conn_txfc; /* QUIC Connection-Level TX Flow Controller */ - QUIC_RXFC *conn_rxfc; /* QUIC Connection-Level RX Flow Controller */ - QUIC_RXFC *max_streams_bidi_rxfc; /* QUIC RXFC for MAX_STREAMS generation */ - QUIC_RXFC *max_streams_uni_rxfc; - const OSSL_CC_METHOD *cc_method; /* QUIC Congestion Controller */ - OSSL_CC_DATA *cc_data; /* QUIC Congestion Controller Instance */ - OSSL_TIME (*now)(void *arg); /* Callback to get current time. */ - void *now_arg; - QLOG *(*get_qlog_cb)(void *arg); /* Optional QLOG retrieval func */ - void *get_qlog_cb_arg; - - /* - * Injected dependencies - crypto streams. - * - * Note: There is no crypto stream for the 0-RTT EL. - * crypto[QUIC_PN_SPACE_APP] is the 1-RTT crypto stream. - */ - QUIC_SSTREAM *crypto[QUIC_PN_SPACE_NUM]; - - } OSSL_QUIC_TX_PACKETISER_ARGS; - -OSSL_QUIC_TX_PACKETISER *ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS *args); - -typedef void (ossl_quic_initial_token_free_fn)(const unsigned char *buf, - size_t buf_len, void *arg); - -void ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER *txp); - -/* - * When in the closing state we need to maintain a count of received bytes - * so that we can limit the number of close connection frames we send. - * Refer RFC 9000 s. 10.2.1 Closing Connection State. - */ -void ossl_quic_tx_packetiser_record_received_closing_bytes( - OSSL_QUIC_TX_PACKETISER *txp, size_t n); - -/* - * Generates a datagram by polling the various ELs to determine if they want to - * generate any frames, and generating a datagram which coalesces packets for - * any ELs which do. - * - * Returns 0 on failure (e.g. allocation error or other errors), 1 otherwise. - * - * *status is filled with status information about the generated packet. - * It is always filled even in case of failure. In particular, packets can be - * sent even if failure is later returned. - * See QUIC_TXP_STATUS for details. - */ -typedef struct quic_txp_status_st { - int sent_ack_eliciting; /* Was an ACK-eliciting packet sent? */ - int sent_handshake; /* Was a Handshake packet sent? */ - size_t sent_pkt; /* Number of packets sent (0 if nothing was sent) */ -} QUIC_TXP_STATUS; - -int ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER *txp, - QUIC_TXP_STATUS *status); - -/* - * Returns a deadline after which a call to ossl_quic_tx_packetiser_generate() - * might succeed even if it did not previously. This may return - * ossl_time_infinite() if there is no such deadline currently applicable. It - * returns ossl_time_zero() if there is (potentially) more data to be generated - * immediately. The value returned is liable to change after any call to - * ossl_quic_tx_packetiser_generate() (or after ACKM or CC state changes). Note - * that ossl_quic_tx_packetiser_generate() can also start to succeed for other - * non-chronological reasons, such as changes to send stream buffers, etc. - */ -OSSL_TIME ossl_quic_tx_packetiser_get_deadline(OSSL_QUIC_TX_PACKETISER *txp); - -/* - * Set the token used in Initial packets. The callback is called when the buffer - * is no longer needed; for example, when the TXP is freed or when this function - * is called again with a new buffer. Fails returning 0 if the token is too big - * to ever be reasonably encapsulated in an outgoing packet based on our current - * understanding of our PMTU. - */ -int ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER *txp, - const unsigned char *token, - size_t token_len, - ossl_quic_initial_token_free_fn *free_cb, - void *free_cb_arg); - -/* Change the DCID the TXP uses to send outgoing packets. */ -int ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER *txp, - const QUIC_CONN_ID *dcid); - -/* Change the SCID the TXP uses to send outgoing (long) packets. */ -int ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER *txp, - const QUIC_CONN_ID *scid); - -/* - * Change the destination L4 address the TXP uses to send datagrams. Specify - * NULL (or AF_UNSPEC) to disable use of addressed mode. - */ -int ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER *txp, - const BIO_ADDR *peer); - -/* - * Change the QLOG instance retrieval function in use after instantiation. - */ -void ossl_quic_tx_packetiser_set_qlog_cb(OSSL_QUIC_TX_PACKETISER *txp, - QLOG *(*get_qlog_cb)(void *arg), - void *get_qlog_cb_arg); - -/* - * Inform the TX packetiser that an EL has been discarded. Idempotent. - * - * This does not inform the QTX as well; the caller must also inform the QTX. - * - * The TXP will no longer reference the crypto[enc_level] QUIC_SSTREAM which was - * provided in the TXP arguments. However, it is the callers responsibility to - * free that QUIC_SSTREAM if desired. - */ -int ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t enc_level); - -/* - * Informs the TX packetiser that the handshake is complete. The TX packetiser - * will not send 1-RTT application data until the handshake is complete, - * as the authenticity of the peer is not confirmed until the handshake - * complete event occurs. - */ -void ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER *txp); - -/* Asks the TXP to generate a HANDSHAKE_DONE frame in the next 1-RTT packet. */ -void ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER *txp); - -/* Asks the TXP to ensure the next packet in the given PN space is ACK-eliciting. */ -void ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t pn_space); - -/* - * Asks the TXP to ensure an ACK is put in the next packet in the given PN - * space. - */ -void ossl_quic_tx_packetiser_schedule_ack(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t pn_space); - -/* - * Schedules a connection close. *f and f->reason are copied. This operation is - * irreversible and causes all further packets generated by the TXP to contain a - * CONNECTION_CLOSE frame. This function fails if it has already been called - * successfully; the information in *f cannot be changed after the first - * successful call to this function. - */ -int ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER *txp, - const OSSL_QUIC_FRAME_CONN_CLOSE *f); - -/* Setters for the msg_callback and msg_callback_arg */ -void ossl_quic_tx_packetiser_set_msg_callback(OSSL_QUIC_TX_PACKETISER *txp, - ossl_msg_cb msg_callback, - SSL *msg_callback_ssl); -void ossl_quic_tx_packetiser_set_msg_callback_arg(OSSL_QUIC_TX_PACKETISER *txp, - void *msg_callback_arg); - -/* - * Determines the next PN which will be used for a given PN space. - */ -QUIC_PN ossl_quic_tx_packetiser_get_next_pn(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t pn_space); - -/* - * Sets a callback which is called whenever TXP sends an ACK frame. The callee - * must not modify the ACK frame data. Can be used to snoop on PNs being ACKed. - */ -void ossl_quic_tx_packetiser_set_ack_tx_cb(OSSL_QUIC_TX_PACKETISER *txp, - void (*cb)(const OSSL_QUIC_FRAME_ACK *ack, - uint32_t pn_space, - void *arg), - void *cb_arg); - -# endif - -#endif diff --git a/openssl/include/internal/quic_txpim.h b/openssl/include/internal/quic_txpim.h deleted file mode 100644 index 5df6ad46c..000000000 --- a/openssl/include/internal/quic_txpim.h +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_TXPIM_H -# define OSSL_QUIC_TXPIM_H - -# include -# include "internal/quic_types.h" -# include "internal/quic_predef.h" -# include "internal/quic_cfq.h" -# include "internal/quic_ackm.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Transmitted Packet Information Manager - * =========================================== - */ - -typedef struct quic_txpim_pkt_st { - /* ACKM-specific data. Caller should fill this. */ - OSSL_ACKM_TX_PKT ackm_pkt; - - /* Linked list of CFQ items in this packet. */ - QUIC_CFQ_ITEM *retx_head; - - /* Reserved for FIFD use. */ - QUIC_FIFD *fifd; - - /* QUIC_PKT_TYPE value. For diagnostic use only. */ - unsigned char pkt_type; - - /* Regenerate-strategy frames. */ - unsigned int had_handshake_done_frame : 1; - unsigned int had_max_data_frame : 1; - unsigned int had_max_streams_bidi_frame : 1; - unsigned int had_max_streams_uni_frame : 1; - unsigned int had_ack_frame : 1; - unsigned int had_conn_close : 1; - - /* Private data follows. */ -} QUIC_TXPIM_PKT; - -/* Represents a range of bytes in an application or CRYPTO stream. */ -typedef struct quic_txpim_chunk_st { - /* The stream ID, or UINT64_MAX for the CRYPTO stream. */ - uint64_t stream_id; - /* - * The inclusive range of bytes in the stream. Exceptionally, if end < - * start, designates a frame of zero length (used for FIN-only frames). In - * this case end is the number of the final byte (i.e., one less than the - * final size of the stream). - */ - uint64_t start, end; - /* - * Whether a FIN was sent for this stream in the packet. Not valid for - * CRYPTO stream. - */ - unsigned int has_fin : 1; - /* - * If set, a STOP_SENDING frame was sent for this stream ID. (If no data was - * sent for the stream, set end < start.) - */ - unsigned int has_stop_sending : 1; - /* - * If set, a RESET_STREAM frame was sent for this stream ID. (If no data was - * sent for the stream, set end < start.) - */ - unsigned int has_reset_stream : 1; -} QUIC_TXPIM_CHUNK; - -QUIC_TXPIM *ossl_quic_txpim_new(void); - -/* - * Frees the TXPIM. All QUIC_TXPIM_PKTs which have been handed out by the TXPIM - * must be released via a call to ossl_quic_txpim_pkt_release() before calling - * this function. - */ -void ossl_quic_txpim_free(QUIC_TXPIM *txpim); - -/* - * Allocates a new QUIC_TXPIM_PKT structure from the pool. Returns NULL on - * failure. The returned structure is cleared of all data and is in a fresh - * initial state. - */ -QUIC_TXPIM_PKT *ossl_quic_txpim_pkt_alloc(QUIC_TXPIM *txpim); - -/* - * Releases the TXPIM packet, returning it to the pool. - */ -void ossl_quic_txpim_pkt_release(QUIC_TXPIM *txpim, QUIC_TXPIM_PKT *fpkt); - -/* Clears the chunk list of the packet, removing all entries. */ -void ossl_quic_txpim_pkt_clear_chunks(QUIC_TXPIM_PKT *fpkt); - -/* Appends a chunk to the packet. The structure is copied. */ -int ossl_quic_txpim_pkt_append_chunk(QUIC_TXPIM_PKT *fpkt, - const QUIC_TXPIM_CHUNK *chunk); - -/* Adds a CFQ item to the packet by prepending it to the retx_head list. */ -void ossl_quic_txpim_pkt_add_cfq_item(QUIC_TXPIM_PKT *fpkt, - QUIC_CFQ_ITEM *item); - -/* - * Returns a pointer to an array of stream chunk information structures for the - * given packet. The caller must call ossl_quic_txpim_pkt_get_num_chunks() to - * determine the length of this array. The returned pointer is invalidated - * if the chunk list is mutated, for example via a call to - * ossl_quic_txpim_pkt_append_chunk() or ossl_quic_txpim_pkt_clear_chunks(). - * - * The chunks are sorted by (stream_id, start) in ascending order. - */ -const QUIC_TXPIM_CHUNK *ossl_quic_txpim_pkt_get_chunks(const QUIC_TXPIM_PKT *fpkt); - -/* - * Returns the number of entries in the array returned by - * ossl_quic_txpim_pkt_get_chunks(). - */ -size_t ossl_quic_txpim_pkt_get_num_chunks(const QUIC_TXPIM_PKT *fpkt); - -/* - * Returns the number of QUIC_TXPIM_PKTs allocated by the given TXPIM that have - * yet to be returned to the TXPIM. - */ -size_t ossl_quic_txpim_get_in_use(const QUIC_TXPIM *txpim); - -# endif - -#endif diff --git a/openssl/include/internal/quic_types.h b/openssl/include/internal/quic_types.h deleted file mode 100644 index fa1ac81ca..000000000 --- a/openssl/include/internal/quic_types.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_TYPES_H -# define OSSL_QUIC_TYPES_H - -# include -# include -# include -# include - -# ifndef OPENSSL_NO_QUIC - -/* QUIC encryption levels. */ -enum { - QUIC_ENC_LEVEL_INITIAL = 0, - QUIC_ENC_LEVEL_HANDSHAKE, - QUIC_ENC_LEVEL_0RTT, - QUIC_ENC_LEVEL_1RTT, - QUIC_ENC_LEVEL_NUM /* Must be the ultimate entry */ -}; - -/* QUIC packet number spaces. */ -enum { - QUIC_PN_SPACE_INITIAL = 0, - QUIC_PN_SPACE_HANDSHAKE, - /* New entries must go here, so that QUIC_PN_SPACE_APP is the penultimate */ - QUIC_PN_SPACE_APP, - QUIC_PN_SPACE_NUM /* Must be the ultimate entry */ -}; - -static ossl_unused ossl_inline uint32_t -ossl_quic_enc_level_to_pn_space(uint32_t enc_level) -{ - switch (enc_level) { - case QUIC_ENC_LEVEL_INITIAL: - return QUIC_PN_SPACE_INITIAL; - case QUIC_ENC_LEVEL_HANDSHAKE: - return QUIC_PN_SPACE_HANDSHAKE; - case QUIC_ENC_LEVEL_0RTT: - case QUIC_ENC_LEVEL_1RTT: - return QUIC_PN_SPACE_APP; - default: - assert(0); - return QUIC_PN_SPACE_APP; - } -} - -/* QUIC packet number representation. */ -typedef uint64_t QUIC_PN; -# define QUIC_PN_INVALID UINT64_MAX - -static ossl_unused ossl_inline QUIC_PN ossl_quic_pn_max(QUIC_PN a, QUIC_PN b) -{ - return a > b ? a : b; -} - -static ossl_unused ossl_inline QUIC_PN ossl_quic_pn_min(QUIC_PN a, QUIC_PN b) -{ - return a < b ? a : b; -} - -static ossl_unused ossl_inline int ossl_quic_pn_valid(QUIC_PN pn) -{ - return pn < (((QUIC_PN)1) << 62); -} - -/* QUIC connection ID representation. */ -# define QUIC_MAX_CONN_ID_LEN 20 -# define QUIC_MIN_ODCID_LEN 8 /* RFC 9000 s. 7.2 */ - -typedef struct quic_conn_id_st { - unsigned char id_len, id[QUIC_MAX_CONN_ID_LEN]; -} QUIC_CONN_ID; - -static ossl_unused ossl_inline int ossl_quic_conn_id_eq(const QUIC_CONN_ID *a, - const QUIC_CONN_ID *b) -{ - if (a->id_len != b->id_len || a->id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - return memcmp(a->id, b->id, a->id_len) == 0; -} - -/* - * Generates a random CID of the given length. libctx may be NULL. - * Returns 1 on success or 0 on failure. - */ -int ossl_quic_gen_rand_conn_id(OSSL_LIB_CTX *libctx, size_t len, - QUIC_CONN_ID *cid); - -# define QUIC_MIN_INITIAL_DGRAM_LEN 1200 - -# define QUIC_DEFAULT_ACK_DELAY_EXP 3 -# define QUIC_MAX_ACK_DELAY_EXP 20 - -# define QUIC_DEFAULT_MAX_ACK_DELAY 25 - -# define QUIC_MIN_ACTIVE_CONN_ID_LIMIT 2 - -/* Arbitrary choice of default idle timeout (not an RFC value). */ -# define QUIC_DEFAULT_IDLE_TIMEOUT 30000 - -# define QUIC_STATELESS_RESET_TOKEN_LEN 16 - -typedef struct { - unsigned char token[QUIC_STATELESS_RESET_TOKEN_LEN]; -} QUIC_STATELESS_RESET_TOKEN; - -/* - * An encoded preferred_addr transport parameter cannot be shorter or longer - * than these lengths in bytes. - */ -# define QUIC_MIN_ENCODED_PREFERRED_ADDR_LEN 41 -# define QUIC_MAX_ENCODED_PREFERRED_ADDR_LEN 61 - -# endif - -#endif diff --git a/openssl/include/internal/quic_vlint.h b/openssl/include/internal/quic_vlint.h deleted file mode 100644 index d4b70b229..000000000 --- a/openssl/include/internal/quic_vlint.h +++ /dev/null @@ -1,127 +0,0 @@ -/* -* Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#ifndef OSSL_INTERNAL_QUIC_VLINT_H -# define OSSL_INTERNAL_QUIC_VLINT_H -# pragma once - -# include "internal/e_os.h" - -# ifndef OPENSSL_NO_QUIC - -/* The smallest value requiring a 1, 2, 4, or 8-byte representation. */ -#define OSSL_QUIC_VLINT_1B_MIN 0 -#define OSSL_QUIC_VLINT_2B_MIN 64 -#define OSSL_QUIC_VLINT_4B_MIN 16384 -#define OSSL_QUIC_VLINT_8B_MIN 1073741824 - -/* The largest value representable in a given number of bytes. */ -#define OSSL_QUIC_VLINT_1B_MAX (OSSL_QUIC_VLINT_2B_MIN - 1) -#define OSSL_QUIC_VLINT_2B_MAX (OSSL_QUIC_VLINT_4B_MIN - 1) -#define OSSL_QUIC_VLINT_4B_MAX (OSSL_QUIC_VLINT_8B_MIN - 1) -#define OSSL_QUIC_VLINT_8B_MAX (((uint64_t)1 << 62) - 1) - -/* The largest value representable as a variable-length integer. */ -#define OSSL_QUIC_VLINT_MAX OSSL_QUIC_VLINT_8B_MAX - -/* - * Returns the number of bytes needed to encode v in the QUIC variable-length - * integer encoding. - * - * Returns 0 if v exceeds OSSL_QUIC_VLINT_MAX. - */ -static ossl_unused ossl_inline size_t ossl_quic_vlint_encode_len(uint64_t v) -{ - if (v < OSSL_QUIC_VLINT_2B_MIN) - return 1; - - if (v < OSSL_QUIC_VLINT_4B_MIN) - return 2; - - if (v < OSSL_QUIC_VLINT_8B_MIN) - return 4; - - if (v <= OSSL_QUIC_VLINT_MAX) - return 8; - - return 0; -} - -/* - * This function writes a QUIC varable-length encoded integer to buf. - * The smallest usable representation is used. - * - * It is the caller's responsibility to ensure that the buffer is big enough by - * calling ossl_quic_vlint_encode_len(v) before calling this function. - * - * Precondition: buf is at least ossl_quic_vlint_enc_len(v) bytes in size - * (unchecked) - * Precondition: v does not exceed OSSL_QUIC_VLINT_MAX - * (unchecked) - */ -void ossl_quic_vlint_encode(unsigned char *buf, uint64_t v); - -/* - * This function writes a QUIC variable-length encoded integer to buf. The - * specified number of bytes n are used for the encoding, which means that the - * encoded value may take up more space than necessary. - * - * It is the caller's responsibility to ensure that the buffer is of at least n - * bytes, and that v is representable by a n-byte QUIC variable-length integer. - * The representable ranges are: - * - * 1-byte encoding: [0, 2** 6-1] - * 2-byte encoding: [0, 2**14-1] - * 4-byte encoding: [0, 2**30-1] - * 8-byte encoding: [0, 2**62-1] - * - * Precondition: buf is at least n bytes in size (unchecked) - * Precondition: v does not exceed the representable range - * (ossl_quic_vlint_encode_len(v) <= n) (unchecked) - * Precondition: v does not exceed OSSL_QUIC_VLINT_MAX - * (unchecked) - */ -void ossl_quic_vlint_encode_n(unsigned char *buf, uint64_t v, int n); - -/* - * Given the first byte of an encoded QUIC variable-length integer, returns - * the number of bytes comprising the encoded integer, including the first - * byte. - */ -static ossl_unused ossl_inline size_t ossl_quic_vlint_decode_len(uint8_t first_byte) -{ - return 1U << ((first_byte & 0xC0) >> 6); -} - -/* - * Given a buffer containing an encoded QUIC variable-length integer, returns - * the decoded value. The buffer must be of at least - * ossl_quic_vlint_decode_len(buf[0]) bytes in size, and the caller is responsible - * for checking this. - * - * Precondition: buf is at least ossl_quic_vlint_decode_len(buf[0]) bytes in size - * (unchecked) - */ -uint64_t ossl_quic_vlint_decode_unchecked(const unsigned char *buf); - -/* - * Given a buffer buf of buf_len bytes in length, attempts to decode an encoded - * QUIC variable-length integer at the start of the buffer and writes the result - * to *v. If buf_len is inadequate, suggesting a truncated encoded integer, the - * function fails and 0 is returned. Otherwise, returns the number of bytes - * consumed. - * - * Precondition: buf is at least buf_len bytes in size - * Precondition: v (unchecked) - */ -int ossl_quic_vlint_decode(const unsigned char *buf, size_t buf_len, uint64_t *v); - -# endif - -#endif diff --git a/openssl/include/internal/quic_wire.h b/openssl/include/internal/quic_wire.h deleted file mode 100644 index cd01feb03..000000000 --- a/openssl/include/internal/quic_wire.h +++ /dev/null @@ -1,784 +0,0 @@ -/* -* Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#ifndef OSSL_INTERNAL_QUIC_WIRE_H -# define OSSL_INTERNAL_QUIC_WIRE_H -# pragma once - -# include "internal/e_os.h" -# include "internal/time.h" -# include "internal/quic_types.h" -# include "internal/packet_quic.h" - -# ifndef OPENSSL_NO_QUIC - -# define OSSL_QUIC_FRAME_TYPE_PADDING 0x00 -# define OSSL_QUIC_FRAME_TYPE_PING 0x01 -# define OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN 0x02 -# define OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN 0x03 -# define OSSL_QUIC_FRAME_TYPE_RESET_STREAM 0x04 -# define OSSL_QUIC_FRAME_TYPE_STOP_SENDING 0x05 -# define OSSL_QUIC_FRAME_TYPE_CRYPTO 0x06 -# define OSSL_QUIC_FRAME_TYPE_NEW_TOKEN 0x07 -# define OSSL_QUIC_FRAME_TYPE_MAX_DATA 0x10 -# define OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA 0x11 -# define OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI 0x12 -# define OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI 0x13 -# define OSSL_QUIC_FRAME_TYPE_DATA_BLOCKED 0x14 -# define OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED 0x15 -# define OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI 0x16 -# define OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_UNI 0x17 -# define OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID 0x18 -# define OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID 0x19 -# define OSSL_QUIC_FRAME_TYPE_PATH_CHALLENGE 0x1A -# define OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE 0x1B -# define OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT 0x1C -# define OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_APP 0x1D -# define OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE 0x1E - -# define OSSL_QUIC_FRAME_FLAG_STREAM_FIN 0x01 -# define OSSL_QUIC_FRAME_FLAG_STREAM_LEN 0x02 -# define OSSL_QUIC_FRAME_FLAG_STREAM_OFF 0x04 -# define OSSL_QUIC_FRAME_FLAG_STREAM_MASK ((uint64_t)0x07) - -/* Low 3 bits of the type contain flags */ -# define OSSL_QUIC_FRAME_TYPE_STREAM 0x08 /* base ID */ -# define OSSL_QUIC_FRAME_TYPE_STREAM_FIN \ - (OSSL_QUIC_FRAME_TYPE_STREAM | \ - OSSL_QUIC_FRAME_FLAG_STREAM_FIN) -# define OSSL_QUIC_FRAME_TYPE_STREAM_LEN \ - (OSSL_QUIC_FRAME_TYPE_STREAM | \ - OSSL_QUIC_FRAME_FLAG_STREAM_LEN) -# define OSSL_QUIC_FRAME_TYPE_STREAM_LEN_FIN \ - (OSSL_QUIC_FRAME_TYPE_STREAM | \ - OSSL_QUIC_FRAME_FLAG_STREAM_LEN | \ - OSSL_QUIC_FRAME_FLAG_STREAM_FIN) -# define OSSL_QUIC_FRAME_TYPE_STREAM_OFF \ - (OSSL_QUIC_FRAME_TYPE_STREAM | \ - OSSL_QUIC_FRAME_FLAG_STREAM_OFF) -# define OSSL_QUIC_FRAME_TYPE_STREAM_OFF_FIN \ - (OSSL_QUIC_FRAME_TYPE_STREAM | \ - OSSL_QUIC_FRAME_FLAG_STREAM_OFF | \ - OSSL_QUIC_FRAME_FLAG_STREAM_FIN) -# define OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN \ - (OSSL_QUIC_FRAME_TYPE_STREAM | \ - OSSL_QUIC_FRAME_FLAG_STREAM_OFF | \ - OSSL_QUIC_FRAME_FLAG_STREAM_LEN) -# define OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN_FIN \ - (OSSL_QUIC_FRAME_TYPE_STREAM | \ - OSSL_QUIC_FRAME_FLAG_STREAM_OFF | \ - OSSL_QUIC_FRAME_FLAG_STREAM_LEN | \ - OSSL_QUIC_FRAME_FLAG_STREAM_FIN) - -# define OSSL_QUIC_FRAME_TYPE_IS_STREAM(x) \ - (((x) & ~OSSL_QUIC_FRAME_FLAG_STREAM_MASK) == OSSL_QUIC_FRAME_TYPE_STREAM) -# define OSSL_QUIC_FRAME_TYPE_IS_ACK(x) \ - (((x) & ~(uint64_t)1) == OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN) -# define OSSL_QUIC_FRAME_TYPE_IS_MAX_STREAMS(x) \ - (((x) & ~(uint64_t)1) == OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI) -# define OSSL_QUIC_FRAME_TYPE_IS_STREAMS_BLOCKED(x) \ - (((x) & ~(uint64_t)1) == OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI) -# define OSSL_QUIC_FRAME_TYPE_IS_CONN_CLOSE(x) \ - (((x) & ~(uint64_t)1) == OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT) - -const char *ossl_quic_frame_type_to_string(uint64_t frame_type); - -static ossl_unused ossl_inline int -ossl_quic_frame_type_is_ack_eliciting(uint64_t frame_type) -{ - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_PADDING: - case OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN: - case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN: - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT: - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_APP: - return 0; - default: - return 1; - } -} - -/* QUIC Transport Parameter Types */ -# define QUIC_TPARAM_ORIG_DCID 0x00 -# define QUIC_TPARAM_MAX_IDLE_TIMEOUT 0x01 -# define QUIC_TPARAM_STATELESS_RESET_TOKEN 0x02 -# define QUIC_TPARAM_MAX_UDP_PAYLOAD_SIZE 0x03 -# define QUIC_TPARAM_INITIAL_MAX_DATA 0x04 -# define QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_BIDI_LOCAL 0x05 -# define QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_BIDI_REMOTE 0x06 -# define QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_UNI 0x07 -# define QUIC_TPARAM_INITIAL_MAX_STREAMS_BIDI 0x08 -# define QUIC_TPARAM_INITIAL_MAX_STREAMS_UNI 0x09 -# define QUIC_TPARAM_ACK_DELAY_EXP 0x0A -# define QUIC_TPARAM_MAX_ACK_DELAY 0x0B -# define QUIC_TPARAM_DISABLE_ACTIVE_MIGRATION 0x0C -# define QUIC_TPARAM_PREFERRED_ADDR 0x0D -# define QUIC_TPARAM_ACTIVE_CONN_ID_LIMIT 0x0E -# define QUIC_TPARAM_INITIAL_SCID 0x0F -# define QUIC_TPARAM_RETRY_SCID 0x10 - -/* - * QUIC Frame Logical Representations - * ================================== - */ - -/* QUIC Frame: ACK */ -typedef struct ossl_quic_ack_range_st { - /* - * Represents an inclusive range of packet numbers [start, end]. - * start must be <= end. - */ - QUIC_PN start, end; -} OSSL_QUIC_ACK_RANGE; - -typedef struct ossl_quic_frame_ack_st { - /* - * A sequence of packet number ranges [[start, end]...]. - * - * The ranges must be sorted in descending order, for example: - * [ 95, 100] - * [ 90, 92] - * etc. - * - * As such, ack_ranges[0].end is always the highest packet number - * being acknowledged and ack_ranges[num_ack_ranges-1].start is - * always the lowest packet number being acknowledged. - * - * num_ack_ranges must be greater than zero, as an ACK frame must - * acknowledge at least one packet number. - */ - OSSL_QUIC_ACK_RANGE *ack_ranges; - size_t num_ack_ranges; - - OSSL_TIME delay_time; - uint64_t ect0, ect1, ecnce; - unsigned int ecn_present : 1; -} OSSL_QUIC_FRAME_ACK; - -/* Returns 1 if the given frame contains the given PN. */ -int ossl_quic_frame_ack_contains_pn(const OSSL_QUIC_FRAME_ACK *ack, QUIC_PN pn); - -/* QUIC Frame: STREAM */ -typedef struct ossl_quic_frame_stream_st { - uint64_t stream_id; /* Stream ID */ - uint64_t offset; /* Logical offset in stream */ - uint64_t len; /* Length of data in bytes */ - const unsigned char *data; - - /* - * On encode, this determines whether the len field should be encoded or - * not. If zero, the len field is not encoded and it is assumed the frame - * runs to the end of the packet. - * - * On decode, this determines whether the frame had an explicitly encoded - * length. If not set, the frame runs to the end of the packet and len has - * been set accordingly. - */ - unsigned int has_explicit_len : 1; - - /* 1 if this is the end of the stream */ - unsigned int is_fin : 1; -} OSSL_QUIC_FRAME_STREAM; - -/* QUIC Frame: CRYPTO */ -typedef struct ossl_quic_frame_crypto_st { - uint64_t offset; /* Logical offset in stream */ - uint64_t len; /* Length of the data in bytes */ - const unsigned char *data; -} OSSL_QUIC_FRAME_CRYPTO; - -/* QUIC Frame: RESET_STREAM */ -typedef struct ossl_quic_frame_reset_stream_st { - uint64_t stream_id; - uint64_t app_error_code; - uint64_t final_size; -} OSSL_QUIC_FRAME_RESET_STREAM; - -/* QUIC Frame: STOP_SENDING */ -typedef struct ossl_quic_frame_stop_sending_st { - uint64_t stream_id; - uint64_t app_error_code; -} OSSL_QUIC_FRAME_STOP_SENDING; - -/* QUIC Frame: NEW_CONNECTION_ID */ -typedef struct ossl_quic_frame_new_conn_id_st { - uint64_t seq_num; - uint64_t retire_prior_to; - QUIC_CONN_ID conn_id; - QUIC_STATELESS_RESET_TOKEN stateless_reset; -} OSSL_QUIC_FRAME_NEW_CONN_ID; - -/* QUIC Frame: CONNECTION_CLOSE */ -typedef struct ossl_quic_frame_conn_close_st { - unsigned int is_app : 1; /* 0: transport error, 1: app error */ - uint64_t error_code; /* 62-bit transport or app error code */ - uint64_t frame_type; /* transport errors only */ - char *reason; /* UTF-8 string, not necessarily zero-terminated */ - size_t reason_len; /* Length of reason in bytes */ -} OSSL_QUIC_FRAME_CONN_CLOSE; - -/* - * QUIC Wire Format Encoding - * ========================= - * - * These functions return 1 on success and 0 on failure. - */ - -/* - * Encodes zero or more QUIC PADDING frames to the packet writer. Each PADDING - * frame consumes one byte; num_bytes specifies the number of bytes of padding - * to write. - */ -int ossl_quic_wire_encode_padding(WPACKET *pkt, size_t num_bytes); - -/* - * Encodes a QUIC PING frame to the packet writer. This frame type takes - * no arguments. -*/ -int ossl_quic_wire_encode_frame_ping(WPACKET *pkt); - -/* - * Encodes a QUIC ACK frame to the packet writer, given a logical representation - * of the ACK frame. - * - * The ACK ranges passed must be sorted in descending order. - * - * The logical representation stores a list of packet number ranges. The wire - * encoding is slightly different and stores the first range in the list - * in a different manner. - * - * The ack_delay_exponent argument specifies the index of a power of two by - * which the ack->ack_delay field is be divided. This exponent value must match - * the value used when decoding. - */ -int ossl_quic_wire_encode_frame_ack(WPACKET *pkt, - uint32_t ack_delay_exponent, - const OSSL_QUIC_FRAME_ACK *ack); - -/* - * Encodes a QUIC RESET_STREAM frame to the packet writer, given a logical - * representation of the RESET_STREAM frame. - */ -int ossl_quic_wire_encode_frame_reset_stream(WPACKET *pkt, - const OSSL_QUIC_FRAME_RESET_STREAM *f); - -/* - * Encodes a QUIC STOP_SENDING frame to the packet writer, given a logical - * representation of the STOP_SENDING frame. - */ -int ossl_quic_wire_encode_frame_stop_sending(WPACKET *pkt, - const OSSL_QUIC_FRAME_STOP_SENDING *f); - -/* - * Encodes a QUIC CRYPTO frame header to the packet writer. - * - * To create a well-formed frame, the data written using this function must be - * immediately followed by f->len bytes of data. - */ -int ossl_quic_wire_encode_frame_crypto_hdr(WPACKET *hdr, - const OSSL_QUIC_FRAME_CRYPTO *f); - -/* - * Returns the number of bytes which will be required to encode the given - * CRYPTO frame header. Does not include the payload bytes in the count. - * Returns 0 if input is invalid. - */ -size_t ossl_quic_wire_get_encoded_frame_len_crypto_hdr(const OSSL_QUIC_FRAME_CRYPTO *f); - -/* - * Encodes a QUIC CRYPTO frame to the packet writer. - * - * This function returns a pointer to a buffer of f->len bytes which the caller - * should fill however it wishes. If f->data is non-NULL, it is automatically - * copied to the target buffer, otherwise the caller must fill the returned - * buffer. Returns NULL on failure. - */ -void *ossl_quic_wire_encode_frame_crypto(WPACKET *pkt, - const OSSL_QUIC_FRAME_CRYPTO *f); - -/* - * Encodes a QUIC NEW_TOKEN frame to the packet writer. - */ -int ossl_quic_wire_encode_frame_new_token(WPACKET *pkt, - const unsigned char *token, - size_t token_len); - -/* - * Encodes a QUIC STREAM frame's header to the packet writer. The f->stream_id, - * f->offset and f->len fields are the values for the respective Stream ID, - * Offset and Length fields. - * - * If f->is_fin is non-zero, the frame is marked as the final frame in the - * stream. - * - * If f->has_explicit_len is zerro, the frame is assumed to be the final frame - * in the packet, which the caller is responsible for ensuring; the Length - * field is then omitted. - * - * To create a well-formed frame, the data written using this function must be - * immediately followed by f->len bytes of stream data. - */ -int ossl_quic_wire_encode_frame_stream_hdr(WPACKET *pkt, - const OSSL_QUIC_FRAME_STREAM *f); - -/* - * Returns the number of bytes which will be required to encode the given - * STREAM frame header. Does not include the payload bytes in the count. - * Returns 0 if input is invalid. - */ -size_t ossl_quic_wire_get_encoded_frame_len_stream_hdr(const OSSL_QUIC_FRAME_STREAM *f); - -/* - * Functions similarly to ossl_quic_wire_encode_frame_stream_hdr, but it also - * allocates space for f->len bytes of data after the header, creating a - * well-formed QUIC STREAM frame in one call. - * - * A pointer to the bytes allocated for the framme payload is returned, - * which the caller can fill however it wishes. If f->data is non-NULL, - * it is automatically copied to the target buffer, otherwise the caller - * must fill the returned buffer. Returns NULL on failure. - */ -void *ossl_quic_wire_encode_frame_stream(WPACKET *pkt, - const OSSL_QUIC_FRAME_STREAM *f); - -/* - * Encodes a QUIC MAX_DATA frame to the packet writer. - */ -int ossl_quic_wire_encode_frame_max_data(WPACKET *pkt, - uint64_t max_data); - -/* - * Encodes a QUIC MAX_STREAM_DATA frame to the packet writer. - */ -int ossl_quic_wire_encode_frame_max_stream_data(WPACKET *pkt, - uint64_t stream_id, - uint64_t max_data); - -/* - * Encodes a QUIC MAX_STREAMS frame to the packet writer. - * - * If is_uni is 0, the count specifies the maximum number of - * bidirectional streams; else it specifies the maximum number of unidirectional - * streams. - */ -int ossl_quic_wire_encode_frame_max_streams(WPACKET *pkt, - char is_uni, - uint64_t max_streams); - -/* - * Encodes a QUIC DATA_BLOCKED frame to the packet writer. - */ -int ossl_quic_wire_encode_frame_data_blocked(WPACKET *pkt, - uint64_t max_data); - -/* - * Encodes a QUIC STREAM_DATA_BLOCKED frame to the packet writer. - */ -int ossl_quic_wire_encode_frame_stream_data_blocked(WPACKET *pkt, - uint64_t stream_id, - uint64_t max_stream_data); -/* - * Encodes a QUIC STREAMS_BLOCKED frame to the packet writer. - * - * If is_uni is 0, the count specifies the maximum number of - * bidirectional streams; else it specifies the maximum number of unidirectional - * streams. - */ -int ossl_quic_wire_encode_frame_streams_blocked(WPACKET *pkt, - char is_uni, - uint64_t max_streams); - -/* - * Encodes a QUIC NEW_CONNECTION_ID frame to the packet writer, given a logical - * representation of the NEW_CONNECTION_ID frame. - * - * The buffer pointed to by the conn_id field must be valid for the duration of - * the call. - */ -int ossl_quic_wire_encode_frame_new_conn_id(WPACKET *pkt, - const OSSL_QUIC_FRAME_NEW_CONN_ID *f); - -/* - * Encodes a QUIC RETIRE_CONNECTION_ID frame to the packet writer. - */ -int ossl_quic_wire_encode_frame_retire_conn_id(WPACKET *pkt, - uint64_t seq_num); - -/* - * Encodes a QUIC PATH_CHALLENGE frame to the packet writer. - */ -int ossl_quic_wire_encode_frame_path_challenge(WPACKET *pkt, - uint64_t data); - -/* - * Encodes a QUIC PATH_RESPONSE frame to the packet writer. - */ -int ossl_quic_wire_encode_frame_path_response(WPACKET *pkt, - uint64_t data); - -/* - * Encodes a QUIC CONNECTION_CLOSE frame to the packet writer, given a logical - * representation of the CONNECTION_CLOSE frame. - * - * The reason field may be NULL, in which case no reason is encoded. If the - * reason field is non-NULL, it must point to a valid UTF-8 string and - * reason_len must be set to the length of the reason string in bytes. The - * reason string need not be zero terminated. - */ -int ossl_quic_wire_encode_frame_conn_close(WPACKET *pkt, - const OSSL_QUIC_FRAME_CONN_CLOSE *f); - -/* - * Encodes a QUIC HANDSHAKE_DONE frame to the packet writer. This frame type - * takes no arguiments. - */ -int ossl_quic_wire_encode_frame_handshake_done(WPACKET *pkt); - -/* - * Encodes a QUIC transport parameter TLV with the given ID into the WPACKET. - * The payload is an arbitrary buffer. - * - * If value is non-NULL, the value is copied into the packet. - * If it is NULL, value_len bytes are allocated for the payload and the caller - * should fill the buffer using the returned pointer. - * - * Returns a pointer to the start of the payload on success, or NULL on failure. - */ -unsigned char *ossl_quic_wire_encode_transport_param_bytes(WPACKET *pkt, - uint64_t id, - const unsigned char *value, - size_t value_len); - -/* - * Encodes a QUIC transport parameter TLV with the given ID into the WPACKET. - * The payload is a QUIC variable-length integer with the given value. - */ -int ossl_quic_wire_encode_transport_param_int(WPACKET *pkt, - uint64_t id, - uint64_t value); - -/* - * Encodes a QUIC transport parameter TLV with a given ID into the WPACKET. - * The payload is a QUIC connection ID. - */ -int ossl_quic_wire_encode_transport_param_cid(WPACKET *wpkt, - uint64_t id, - const QUIC_CONN_ID *cid); - -/* - * QUIC Wire Format Decoding - * ========================= - * - * These functions return 1 on success or 0 for failure. Typical reasons - * why these functions may fail include: - * - * - A frame decode function is called but the frame in the PACKET's buffer - * is not of the correct type. - * - * - A variable-length field in the encoded frame appears to exceed the bounds - * of the PACKET's buffer. - * - * These functions should be called with the PACKET pointing to the start of the - * frame (including the initial type field), and consume an entire frame - * including its type field. The expectation is that the caller will have - * already discerned the frame type using ossl_quic_wire_peek_frame_header(). - */ - -/* - * Decodes the type field header of a QUIC frame (without advancing the current - * position). This can be used to determine the frame type and determine which - * frame decoding function to call. - */ -int ossl_quic_wire_peek_frame_header(PACKET *pkt, uint64_t *type, - int *was_minimal); - -/* - * Like ossl_quic_wire_peek_frame_header, but advances the current position - * so that the type field is consumed. For advanced use only. - */ -int ossl_quic_wire_skip_frame_header(PACKET *pkt, uint64_t *type); - -/* - * Determines how many ranges are needed to decode a QUIC ACK frame. - * - * The number of ranges which must be allocated before the call to - * ossl_quic_wire_decode_frame_ack is written to *total_ranges. - * - * The PACKET is not advanced. - */ -int ossl_quic_wire_peek_frame_ack_num_ranges(const PACKET *pkt, - uint64_t *total_ranges); - -/* - * Decodes a QUIC ACK frame. The ack_ranges field of the passed structure should - * point to a preallocated array of ACK ranges and the num_ack_ranges field - * should specify the length of allocation. - * - * *total_ranges is written with the number of ranges in the decoded frame, - * which may be greater than the number of ranges which were decoded (i.e. if - * num_ack_ranges was too small to decode all ranges). - * - * On success, this function modifies the num_ack_ranges field to indicate the - * number of ranges in the decoded frame. This is the number of entries in the - * ACK ranges array written by this function; any additional entries are not - * modified. - * - * If the number of ACK ranges in the decoded frame exceeds that in - * num_ack_ranges, as many ACK ranges as possible are decoded into the range - * array. The caller can use the value written to *total_ranges to detect this - * condition, as *total_ranges will exceed num_ack_ranges. - * - * If ack is NULL, the frame is still decoded, but only *total_ranges is - * written. This can be used to determine the number of ranges which must be - * allocated. - * - * The ack_delay_exponent argument specifies the index of a power of two used to - * decode the ack_delay field. This must match the ack_delay_exponent value used - * to encode the frame. - */ -int ossl_quic_wire_decode_frame_ack(PACKET *pkt, - uint32_t ack_delay_exponent, - OSSL_QUIC_FRAME_ACK *ack, - uint64_t *total_ranges); - -/* - * Decodes a QUIC RESET_STREAM frame. - */ -int ossl_quic_wire_decode_frame_reset_stream(PACKET *pkt, - OSSL_QUIC_FRAME_RESET_STREAM *f); - -/* - * Decodes a QUIC STOP_SENDING frame. - */ -int ossl_quic_wire_decode_frame_stop_sending(PACKET *pkt, - OSSL_QUIC_FRAME_STOP_SENDING *f); - -/* - * Decodes a QUIC CRYPTO frame. - * - * f->data is set to point inside the packet buffer inside the PACKET, therefore - * it is safe to access for as long as the packet buffer exists. If nodata is - * set to 1 then reading the PACKET stops after the frame header and f->data is - * set to NULL. - */ -int ossl_quic_wire_decode_frame_crypto(PACKET *pkt, int nodata, - OSSL_QUIC_FRAME_CRYPTO *f); - -/* - * Decodes a QUIC NEW_TOKEN frame. *token is written with a pointer to the token - * bytes and *token_len is written with the length of the token in bytes. - */ -int ossl_quic_wire_decode_frame_new_token(PACKET *pkt, - const unsigned char **token, - size_t *token_len); - -/* - * Decodes a QUIC STREAM frame. - * - * If nodata is set to 1 then reading the PACKET stops after the frame header - * and f->data is set to NULL. In this case f->len will also be 0 in the event - * that "has_explicit_len" is 0. - * - * If the frame did not contain an offset field, f->offset is set to 0, as the - * absence of an offset field is equivalent to an offset of 0. - * - * If the frame contained a length field, f->has_explicit_len is set to 1 and - * the length of the data is placed in f->len. This function ensures that the - * length does not exceed the packet buffer, thus it is safe to access f->data. - * - * If the frame did not contain a length field, this means that the frame runs - * until the end of the packet. This function sets f->has_explicit_len to zero, - * and f->len to the amount of data remaining in the input buffer. Therefore, - * this function should be used with a PACKET representing a single packet (and - * not e.g. multiple packets). - * - * Note also that this means f->len is always valid after this function returns - * successfully, regardless of the value of f->has_explicit_len. - * - * f->data points inside the packet buffer inside the PACKET, therefore it is - * safe to access for as long as the packet buffer exists. - * - * f->is_fin is set according to whether the frame was marked as ending the - * stream. - */ -int ossl_quic_wire_decode_frame_stream(PACKET *pkt, int nodata, - OSSL_QUIC_FRAME_STREAM *f); - -/* - * Decodes a QUIC MAX_DATA frame. The Maximum Data field is written to - * *max_data. - */ -int ossl_quic_wire_decode_frame_max_data(PACKET *pkt, - uint64_t *max_data); - -/* - * Decodes a QUIC MAX_STREAM_DATA frame. The Stream ID is written to *stream_id - * and Maximum Stream Data field is written to *max_stream_data. - */ -int ossl_quic_wire_decode_frame_max_stream_data(PACKET *pkt, - uint64_t *stream_id, - uint64_t *max_stream_data); -/* - * Decodes a QUIC MAX_STREAMS frame. The Maximum Streams field is written to - * *max_streams. - * - * Whether the limit concerns bidirectional streams or unidirectional streams is - * denoted by the frame type; the caller should examine the frame type to - * determine this. - */ -int ossl_quic_wire_decode_frame_max_streams(PACKET *pkt, - uint64_t *max_streams); - -/* - * Decodes a QUIC DATA_BLOCKED frame. The Maximum Data field is written to - * *max_data. - */ -int ossl_quic_wire_decode_frame_data_blocked(PACKET *pkt, - uint64_t *max_data); - -/* - * Decodes a QUIC STREAM_DATA_BLOCKED frame. The Stream ID and Maximum Stream - * Data fields are written to *stream_id and *max_stream_data respectively. - */ -int ossl_quic_wire_decode_frame_stream_data_blocked(PACKET *pkt, - uint64_t *stream_id, - uint64_t *max_stream_data); - -/* - * Decodes a QUIC STREAMS_BLOCKED frame. The Maximum Streams field is written to - * *max_streams. - * - * Whether the limit concerns bidirectional streams or unidirectional streams is - * denoted by the frame type; the caller should examine the frame type to - * determine this. - */ -int ossl_quic_wire_decode_frame_streams_blocked(PACKET *pkt, - uint64_t *max_streams); - - -/* - * Decodes a QUIC NEW_CONNECTION_ID frame. The logical representation of the - * frame is written to *f. - * - * The conn_id field is set to point to the connection ID string inside the - * packet buffer; it is therefore valid for as long as the PACKET's buffer is - * valid. The conn_id_len field is set to the length of the connection ID string - * in bytes. - */ -int ossl_quic_wire_decode_frame_new_conn_id(PACKET *pkt, - OSSL_QUIC_FRAME_NEW_CONN_ID *f); - -/* - * Decodes a QUIC RETIRE_CONNECTION_ID frame. The Sequence Number field - * is written to *seq_num. - */ -int ossl_quic_wire_decode_frame_retire_conn_id(PACKET *pkt, - uint64_t *seq_num); - -/* - * Decodes a QUIC PATH_CHALLENGE frame. The Data field is written to *data. - */ -int ossl_quic_wire_decode_frame_path_challenge(PACKET *pkt, - uint64_t *data); - -/* - * Decodes a QUIC PATH_CHALLENGE frame. The Data field is written to *data. - */ -int ossl_quic_wire_decode_frame_path_response(PACKET *pkt, - uint64_t *data); - -/* - * Decodes a QUIC CONNECTION_CLOSE frame. The logical representation - * of the frame is written to *f. - * - * The reason field is set to point to the UTF-8 reason string inside - * the packet buffer; it is therefore valid for as long as the PACKET's - * buffer is valid. The reason_len field is set to the length of the - * reason string in bytes. - * - * IMPORTANT: The reason string is not zero-terminated. - * - * Returns 1 on success or 0 on failure. - */ -int ossl_quic_wire_decode_frame_conn_close(PACKET *pkt, - OSSL_QUIC_FRAME_CONN_CLOSE *f); - -/* - * Decodes one or more PADDING frames. PADDING frames have no arguments. - * - * Returns the number of PADDING frames decoded or 0 on error. - */ -size_t ossl_quic_wire_decode_padding(PACKET *pkt); - -/* - * Decodes a PING frame. The frame has no arguments. - */ -int ossl_quic_wire_decode_frame_ping(PACKET *pkt); - -/* - * Decodes a HANDSHAKE_DONE frame. The frame has no arguments. - */ -int ossl_quic_wire_decode_frame_handshake_done(PACKET *pkt); - -/* - * Peeks at the ID of the next QUIC transport parameter TLV in the stream. - * The ID is written to *id. - */ -int ossl_quic_wire_peek_transport_param(PACKET *pkt, uint64_t *id); - -/* - * Decodes a QUIC transport parameter TLV. A pointer to the value buffer is - * returned on success. This points inside the PACKET's buffer and is therefore - * valid as long as the PACKET's buffer is valid. - * - * The transport parameter ID is written to *id (if non-NULL) and the length of - * the payload in bytes is written to *len. - * - * Returns NULL on failure. - */ -const unsigned char *ossl_quic_wire_decode_transport_param_bytes(PACKET *pkt, - uint64_t *id, - size_t *len); - -/* - * Decodes a QUIC transport parameter TLV containing a variable-length integer. - * - * The transport parameter ID is written to *id (if non-NULL) and the value is - * written to *value. - */ -int ossl_quic_wire_decode_transport_param_int(PACKET *pkt, - uint64_t *id, - uint64_t *value); - -/* - * Decodes a QUIC transport parameter TLV containing a connection ID. - * - * The transport parameter ID is written to *id (if non-NULL) and the value is - * written to *value. - */ -int ossl_quic_wire_decode_transport_param_cid(PACKET *pkt, - uint64_t *id, - QUIC_CONN_ID *cid); - -/* - * Decodes a QUIC transport parameter TLV containing a preferred_address. - */ -typedef struct quic_preferred_addr_st { - uint16_t ipv4_port, ipv6_port; - unsigned char ipv4[4], ipv6[16]; - QUIC_STATELESS_RESET_TOKEN stateless_reset; - QUIC_CONN_ID cid; -} QUIC_PREFERRED_ADDR; - -int ossl_quic_wire_decode_transport_param_preferred_addr(PACKET *pkt, - QUIC_PREFERRED_ADDR *p); - -# endif - -#endif diff --git a/openssl/include/internal/quic_wire_pkt.h b/openssl/include/internal/quic_wire_pkt.h deleted file mode 100644 index 18a483fc2..000000000 --- a/openssl/include/internal/quic_wire_pkt.h +++ /dev/null @@ -1,629 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_WIRE_PKT_H -# define OSSL_QUIC_WIRE_PKT_H - -# include -# include "internal/packet_quic.h" -# include "internal/quic_types.h" - -# ifndef OPENSSL_NO_QUIC - -# define QUIC_VERSION_NONE ((uint32_t)0) /* Used for version negotiation */ -# define QUIC_VERSION_1 ((uint32_t)1) /* QUIC v1 */ - -/* QUIC logical packet type. These do not match wire values. */ -# define QUIC_PKT_TYPE_INITIAL 1 -# define QUIC_PKT_TYPE_0RTT 2 -# define QUIC_PKT_TYPE_HANDSHAKE 3 -# define QUIC_PKT_TYPE_RETRY 4 -# define QUIC_PKT_TYPE_1RTT 5 -# define QUIC_PKT_TYPE_VERSION_NEG 6 - -/* - * Determine encryption level from packet type. Returns QUIC_ENC_LEVEL_NUM if - * the packet is not of a type which is encrypted. - */ -static ossl_inline ossl_unused uint32_t -ossl_quic_pkt_type_to_enc_level(uint32_t pkt_type) -{ - switch (pkt_type) { - case QUIC_PKT_TYPE_INITIAL: - return QUIC_ENC_LEVEL_INITIAL; - case QUIC_PKT_TYPE_HANDSHAKE: - return QUIC_ENC_LEVEL_HANDSHAKE; - case QUIC_PKT_TYPE_0RTT: - return QUIC_ENC_LEVEL_0RTT; - case QUIC_PKT_TYPE_1RTT: - return QUIC_ENC_LEVEL_1RTT; - default: - return QUIC_ENC_LEVEL_NUM; - } -} - -static ossl_inline ossl_unused uint32_t -ossl_quic_enc_level_to_pkt_type(uint32_t enc_level) -{ - switch (enc_level) { - case QUIC_ENC_LEVEL_INITIAL: - return QUIC_PKT_TYPE_INITIAL; - case QUIC_ENC_LEVEL_HANDSHAKE: - return QUIC_PKT_TYPE_HANDSHAKE; - case QUIC_ENC_LEVEL_0RTT: - return QUIC_PKT_TYPE_0RTT; - case QUIC_ENC_LEVEL_1RTT: - return QUIC_PKT_TYPE_1RTT; - default: - return UINT32_MAX; - } -} - -/* Determine if a packet type contains an encrypted payload. */ -static ossl_inline ossl_unused int -ossl_quic_pkt_type_is_encrypted(uint32_t pkt_type) -{ - switch (pkt_type) { - case QUIC_PKT_TYPE_RETRY: - case QUIC_PKT_TYPE_VERSION_NEG: - return 0; - default: - return 1; - } -} - -/* Determine if a packet type contains a PN field. */ -static ossl_inline ossl_unused int -ossl_quic_pkt_type_has_pn(uint32_t pkt_type) -{ - /* - * Currently a packet has a PN iff it is encrypted. This could change - * someday. - */ - return ossl_quic_pkt_type_is_encrypted(pkt_type); -} - -/* - * Determine if a packet type can appear with other packets in a datagram. Some - * packet types must be the sole packet in a datagram. - */ -static ossl_inline ossl_unused int -ossl_quic_pkt_type_can_share_dgram(uint32_t pkt_type) -{ - /* - * Currently only the encrypted packet types can share a datagram. This - * could change someday. - */ - return ossl_quic_pkt_type_is_encrypted(pkt_type); -} - -/* - * Determine if the packet type must come at the end of the datagram (due to the - * lack of a length field). - */ -static ossl_inline ossl_unused int -ossl_quic_pkt_type_must_be_last(uint32_t pkt_type) -{ - /* - * Any packet type which cannot share a datagram obviously must come last. - * 1-RTT also must come last as it lacks a length field. - */ - return !ossl_quic_pkt_type_can_share_dgram(pkt_type) - || pkt_type == QUIC_PKT_TYPE_1RTT; -} - -/* - * Determine if the packet type has a version field. - */ -static ossl_inline ossl_unused int -ossl_quic_pkt_type_has_version(uint32_t pkt_type) -{ - return pkt_type != QUIC_PKT_TYPE_1RTT && pkt_type != QUIC_PKT_TYPE_VERSION_NEG; -} - -/* - * Determine if the packet type has a SCID field. - */ -static ossl_inline ossl_unused int -ossl_quic_pkt_type_has_scid(uint32_t pkt_type) -{ - return pkt_type != QUIC_PKT_TYPE_1RTT; -} - -/* - * Smallest possible QUIC packet size as per RFC (aside from version negotiation - * packets). - */ -# define QUIC_MIN_VALID_PKT_LEN_CRYPTO 21 -# define QUIC_MIN_VALID_PKT_LEN_VERSION_NEG 7 -# define QUIC_MIN_VALID_PKT_LEN QUIC_MIN_VALID_PKT_LEN_VERSION_NEG - -typedef struct quic_pkt_hdr_ptrs_st QUIC_PKT_HDR_PTRS; - -/* - * QUIC Packet Header Protection - * ============================= - * - * Functions to apply and remove QUIC packet header protection. A header - * protector is initialised using ossl_quic_hdr_protector_init and must be - * destroyed using ossl_quic_hdr_protector_cleanup when no longer needed. - */ -typedef struct quic_hdr_protector_st { - OSSL_LIB_CTX *libctx; - const char *propq; - EVP_CIPHER_CTX *cipher_ctx; - EVP_CIPHER *cipher; - uint32_t cipher_id; -} QUIC_HDR_PROTECTOR; - -# define QUIC_HDR_PROT_CIPHER_AES_128 1 -# define QUIC_HDR_PROT_CIPHER_AES_256 2 -# define QUIC_HDR_PROT_CIPHER_CHACHA 3 - -/* - * Initialises a header protector. - * - * cipher_id: - * The header protection cipher method to use. One of - * QUIC_HDR_PROT_CIPHER_*. Must be chosen based on negotiated TLS cipher - * suite. - * - * quic_hp_key: - * This must be the "quic hp" key derived from a traffic secret. - * - * The length of the quic_hp_key must correspond to that expected for the - * given cipher ID. - * - * The header protector performs amortisable initialisation in this function, - * therefore a header protector should be used for as long as possible. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_hdr_protector_init(QUIC_HDR_PROTECTOR *hpr, - OSSL_LIB_CTX *libctx, - const char *propq, - uint32_t cipher_id, - const unsigned char *quic_hp_key, - size_t quic_hp_key_len); - -/* - * Destroys a header protector. This is also safe to call on a zero-initialized - * OSSL_QUIC_HDR_PROTECTOR structure which has not been initialized, or which - * has already been destroyed. - */ -void ossl_quic_hdr_protector_cleanup(QUIC_HDR_PROTECTOR *hpr); - -/* - * Removes header protection from a packet. The packet payload must currently be - * encrypted (i.e., you must remove header protection before decrypting packets - * received). The function examines the header buffer to determine which bytes - * of the header need to be decrypted. - * - * If this function fails, no data is modified. - * - * This is implemented as a call to ossl_quic_hdr_protector_decrypt_fields(). - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_hdr_protector_decrypt(QUIC_HDR_PROTECTOR *hpr, - QUIC_PKT_HDR_PTRS *ptrs); - -/* - * Applies header protection to a packet. The packet payload must already have - * been encrypted (i.e., you must apply header protection after encrypting - * a packet). The function examines the header buffer to determine which bytes - * of the header need to be encrypted. - * - * This is implemented as a call to ossl_quic_hdr_protector_encrypt_fields(). - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_hdr_protector_encrypt(QUIC_HDR_PROTECTOR *hpr, - QUIC_PKT_HDR_PTRS *ptrs); - -/* - * Removes header protection from a packet. The packet payload must currently - * be encrypted. This is a low-level function which assumes you have already - * determined which parts of the packet header need to be decrypted. - * - * sample: - * The range of bytes in the packet to be used to generate the header - * protection mask. It is permissible to set sample_len to the size of the - * remainder of the packet; this function will only use as many bytes as - * needed. If not enough sample bytes are provided, this function fails. - * - * first_byte: - * The first byte of the QUIC packet header to be decrypted. - * - * pn: - * Pointer to the start of the PN field. The caller is responsible - * for ensuring at least four bytes follow this pointer. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_hdr_protector_decrypt_fields(QUIC_HDR_PROTECTOR *hpr, - const unsigned char *sample, - size_t sample_len, - unsigned char *first_byte, - unsigned char *pn_bytes); - -/* - * Works analogously to ossl_hdr_protector_decrypt_fields, but applies header - * protection instead of removing it. - */ -int ossl_quic_hdr_protector_encrypt_fields(QUIC_HDR_PROTECTOR *hpr, - const unsigned char *sample, - size_t sample_len, - unsigned char *first_byte, - unsigned char *pn_bytes); - -/* - * QUIC Packet Header - * ================== - * - * This structure provides a logical representation of a QUIC packet header. - * - * QUIC packet formats fall into the following categories: - * - * Long Packets, which is subdivided into five possible packet types: - * Version Negotiation (a special case); - * Initial; - * 0-RTT; - * Handshake; and - * Retry - * - * Short Packets, which comprises only a single packet type (1-RTT). - * - * The packet formats vary and common fields are found in some packets but - * not others. The below table indicates which fields are present in which - * kinds of packet. * indicates header protection is applied. - * - * SLLLLL Legend: 1=1-RTT, i=Initial, 0=0-RTT, h=Handshake - * 1i0hrv r=Retry, v=Version Negotiation - * ------ - * 1i0hrv Header Form (0=Short, 1=Long) - * 1i0hr Fixed Bit (always 1) - * 1 Spin Bit - * 1 * Reserved Bits - * 1 * Key Phase - * 1i0h * Packet Number Length - * i0hr? Long Packet Type - * i0h Type-Specific Bits - * i0hr Version (note: always 0 for Version Negotiation packets) - * 1i0hrv Destination Connection ID - * i0hrv Source Connection ID - * 1i0h * Packet Number - * i Token - * i0h Length - * r Retry Token - * r Retry Integrity Tag - * - * For each field below, the conditions under which the field is valid are - * specified. If a field is not currently valid, it is initialized to a zero or - * NULL value. - */ -typedef struct quic_pkt_hdr_st { - /* [ALL] A QUIC_PKT_TYPE_* value. Always valid. */ - unsigned int type :8; - - /* [S] Value of the spin bit. Valid if (type == 1RTT). */ - unsigned int spin_bit :1; - - /* - * [S] Value of the Key Phase bit in the short packet. - * Valid if (type == 1RTT && !partial). - */ - unsigned int key_phase :1; - - /* - * [1i0h] Length of packet number in bytes. This is the decoded value. - * Valid if ((type == 1RTT || (version && type != RETRY)) && !partial). - */ - unsigned int pn_len :4; - - /* - * [ALL] Set to 1 if this is a partial decode because the packet header - * has not yet been deprotected. pn_len, pn and key_phase are not valid if - * this is set. - */ - unsigned int partial :1; - - /* - * [ALL] Whether the fixed bit was set. Note that only Version Negotiation - * packets are allowed to have this unset, so this will always be 1 for all - * other packet types (decode will fail if it is not set). Ignored when - * encoding unless encoding a Version Negotiation packet. - */ - unsigned int fixed :1; - - /* - * The unused bits in the low 4 bits of a Retry packet header's first byte. - * This is used to ensure that Retry packets have the same bit-for-bit - * representation in their header when decoding and encoding them again. - * This is necessary to validate Retry packet headers. - */ - unsigned int unused :4; - - /* - * The 'Reserved' bits in an Initial, Handshake, 0-RTT or 1-RTT packet - * header's first byte. These are provided so that the caller can validate - * that they are zero, as this must be done after packet protection is - * successfully removed to avoid creating a timing channel. - */ - unsigned int reserved :2; - - /* [L] Version field. Valid if (type != 1RTT). */ - uint32_t version; - - /* [ALL] The destination connection ID. Always valid. */ - QUIC_CONN_ID dst_conn_id; - - /* - * [L] The source connection ID. - * Valid if (type != 1RTT). - */ - QUIC_CONN_ID src_conn_id; - - /* - * [1i0h] Relatively-encoded packet number in raw, encoded form. The correct - * decoding of this value is context-dependent. The number of bytes valid in - * this buffer is determined by pn_len above. If the decode was partial, - * this field is not valid. - * - * Valid if ((type == 1RTT || (version && type != RETRY)) && !partial). - */ - unsigned char pn[4]; - - /* - * [i] Token field in Initial packet. Points to memory inside the decoded - * PACKET, and therefore is valid for as long as the PACKET's buffer is - * valid. token_len is the length of the token in bytes. - * - * Valid if (type == INITIAL). - */ - const unsigned char *token; - size_t token_len; - - /* - * [ALL] Payload length in bytes. - * - * Though 1-RTT, Retry and Version Negotiation packets do not contain an - * explicit length field, this field is always valid and is used by the - * packet header encoding and decoding routines to describe the payload - * length, regardless of whether the packet type encoded or decoded uses an - * explicit length indication. - */ - size_t len; - - /* - * Pointer to start of payload data in the packet. Points to memory inside - * the decoded PACKET, and therefore is valid for as long as the PACKET'S - * buffer is valid. The length of the buffer in bytes is in len above. - * - * For Version Negotiation packets, points to the array of supported - * versions. - * - * For Retry packets, points to the Retry packet payload, which comprises - * the Retry Token followed by a 16-byte Retry Integrity Tag. - * - * Regardless of whether a packet is a Version Negotiation packet (where the - * payload contains a list of supported versions), a Retry packet (where the - * payload contains a Retry Token and Retry Integrity Tag), or any other - * packet type (where the payload contains frames), the payload is not - * validated and the user must parse the payload bearing this in mind. - * - * If the decode was partial (partial is set), this points to the start of - * the packet number field, rather than the protected payload, as the length - * of the packet number field is unknown. The len field reflects this in - * this case (i.e., the len field is the number of payload bytes plus the - * number of bytes comprising the PN). - */ - const unsigned char *data; -} QUIC_PKT_HDR; - -/* - * Extra information which can be output by the packet header decode functions - * for the assistance of the header protector. This avoids the header protector - * needing to partially re-decode the packet header. - */ -struct quic_pkt_hdr_ptrs_st { - unsigned char *raw_start; /* start of packet */ - unsigned char *raw_sample; /* start of sampling range */ - size_t raw_sample_len; /* maximum length of sampling range */ - - /* - * Start of PN field. Guaranteed to be NULL unless at least four bytes are - * available via this pointer. - */ - unsigned char *raw_pn; -}; - -/* - * If partial is 1, reads the unprotected parts of a protected packet header - * from a PACKET, performing a partial decode. - * - * If partial is 0, the input is assumed to have already had header protection - * removed, and all header fields are decoded. - * - * If nodata is 1, the input is assumed to have no payload data in it. Otherwise - * payload data must be present. - * - * On success, the logical decode of the packet header is written to *hdr. - * hdr->partial is set or cleared according to whether a partial decode was - * performed. *ptrs is filled with pointers to various parts of the packet - * buffer. - * - * In order to decode short packets, the connection ID length being used must be - * known contextually, and should be passed as short_conn_id_len. If - * short_conn_id_len is set to an invalid value (a value greater than - * QUIC_MAX_CONN_ID_LEN), this function fails when trying to decode a short - * packet, but succeeds for long packets. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_wire_decode_pkt_hdr(PACKET *pkt, - size_t short_conn_id_len, - int partial, - int nodata, - QUIC_PKT_HDR *hdr, - QUIC_PKT_HDR_PTRS *ptrs); - -/* - * Encodes a packet header. The packet is written to pkt. - * - * The length of the (encrypted) packet payload should be written to hdr->len - * and will be placed in the serialized packet header. The payload data itself - * is not copied; the caller should write hdr->len bytes of encrypted payload to - * the WPACKET immediately after the call to this function. However, - * WPACKET_reserve_bytes is called for the payload size. - * - * This function does not apply header protection. You must apply header - * protection yourself after calling this function. *ptrs is filled with - * pointers which can be passed to a header protector, but this must be - * performed after the encrypted payload is written. - * - * The pointers in *ptrs are direct pointers into the WPACKET buffer. If more - * data is written to the WPACKET buffer, WPACKET buffer reallocations may - * occur, causing these pointers to become invalid. Therefore, you must not call - * any write WPACKET function between this call and the call to - * ossl_quic_hdr_protector_encrypt. This function calls WPACKET_reserve_bytes - * for the payload length, so you may assume hdr->len bytes are already free to - * write at the WPACKET cursor location once this function returns successfully. - * It is recommended that you call this function, write the encrypted payload, - * call ossl_quic_hdr_protector_encrypt, and then call - * WPACKET_allocate_bytes(hdr->len). - * - * Version Negotiation and Retry packets do not use header protection; for these - * header types, the fields in *ptrs are all written as zero. Version - * Negotiation, Retry and 1-RTT packets do not contain a Length field, but - * hdr->len bytes of data are still reserved in the WPACKET. - * - * If serializing a short packet and short_conn_id_len does not match the DCID - * specified in hdr, the function fails. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_wire_encode_pkt_hdr(WPACKET *pkt, - size_t short_conn_id_len, - const QUIC_PKT_HDR *hdr, - QUIC_PKT_HDR_PTRS *ptrs); - -/* - * Retrieves only the DCID from a packet header. This is intended for demuxer - * use. It avoids the need to parse the rest of the packet header twice. - * - * Information on packet length is not decoded, as this only needs to be used on - * the first packet in a datagram, therefore this takes a buffer and not a - * PACKET. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_wire_get_pkt_hdr_dst_conn_id(const unsigned char *buf, - size_t buf_len, - size_t short_conn_id_len, - QUIC_CONN_ID *dst_conn_id); - -/* - * Precisely predicts the encoded length of a packet header structure. - * - * May return 0 if the packet header is not valid, but the fact that this - * function returns non-zero does not guarantee that - * ossl_quic_wire_encode_pkt_hdr() will succeed. - */ -int ossl_quic_wire_get_encoded_pkt_hdr_len(size_t short_conn_id_len, - const QUIC_PKT_HDR *hdr); - -/* - * Packet Number Encoding - * ====================== - */ - -/* - * Decode an encoded packet header QUIC PN. - * - * enc_pn is the raw encoded PN to decode. enc_pn_len is its length in bytes as - * indicated by packet headers. largest_pn is the largest PN successfully - * processed in the relevant PN space. - * - * The resulting PN is written to *res_pn. - * - * Returns 1 on success or 0 on failure. - */ -int ossl_quic_wire_decode_pkt_hdr_pn(const unsigned char *enc_pn, - size_t enc_pn_len, - QUIC_PN largest_pn, - QUIC_PN *res_pn); - -/* - * Determine how many bytes should be used to encode a PN. Returns the number of - * bytes (which will be in range [1, 4]). - */ -int ossl_quic_wire_determine_pn_len(QUIC_PN pn, QUIC_PN largest_acked); - -/* - * Encode a PN for a packet header using the specified number of bytes, which - * should have been determined by calling ossl_quic_wire_determine_pn_len. The - * PN encoding process is done in two parts to allow the caller to override PN - * encoding length if it wishes. - * - * Returns 1 on success and 0 on failure. - */ -int ossl_quic_wire_encode_pkt_hdr_pn(QUIC_PN pn, - unsigned char *enc_pn, - size_t enc_pn_len); - -/* - * Retry Integrity Tags - * ==================== - */ - -# define QUIC_RETRY_INTEGRITY_TAG_LEN 16 - -/* - * Validate a retry integrity tag. Returns 1 if the tag is valid. - * - * Must be called on a hdr with a type of QUIC_PKT_TYPE_RETRY with a valid data - * pointer. - * - * client_initial_dcid must be the original DCID used by the client in its first - * Initial packet, as this is used to calculate the Retry Integrity Tag. - * - * Returns 0 if the tag is invalid, if called on any other type of packet or if - * the body is too short. - */ -int ossl_quic_validate_retry_integrity_tag(OSSL_LIB_CTX *libctx, - const char *propq, - const QUIC_PKT_HDR *hdr, - const QUIC_CONN_ID *client_initial_dcid); - -/* - * Calculates a retry integrity tag. Returns 0 on error, for example if hdr does - * not have a type of QUIC_PKT_TYPE_RETRY. - * - * client_initial_dcid must be the original DCID used by the client in its first - * Initial packet, as this is used to calculate the Retry Integrity Tag. - * - * tag must point to a buffer of QUIC_RETRY_INTEGRITY_TAG_LEN bytes in size. - * - * Note that hdr->data must point to the Retry packet body, and hdr->len must - * include the space for the Retry Integrity Tag. (This means that you can - * easily fill in a tag in a Retry packet you are generating by calling this - * function and passing (hdr->data + hdr->len - QUIC_RETRY_INTEGRITY_TAG_LEN) as - * the tag argument.) This function fails if hdr->len is too short to contain a - * Retry Integrity Tag. - */ -int ossl_quic_calculate_retry_integrity_tag(OSSL_LIB_CTX *libctx, - const char *propq, - const QUIC_PKT_HDR *hdr, - const QUIC_CONN_ID *client_initial_dcid, - unsigned char *tag); - -# endif - -#endif diff --git a/openssl/include/internal/rcu.h b/openssl/include/internal/rcu.h deleted file mode 100644 index 7716a1c7f..000000000 --- a/openssl/include/internal/rcu.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_RCU_H -# define OPENSSL_RCU_H -# pragma once - -typedef void (*rcu_cb_fn)(void *data); - -typedef struct rcu_lock_st CRYPTO_RCU_LOCK; - -CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers); -void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock); -void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock); -void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock); -void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock); -void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock); -void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock); -int ossl_rcu_call(CRYPTO_RCU_LOCK *lock, rcu_cb_fn cb, void *data); -void *ossl_rcu_uptr_deref(void **p); -void ossl_rcu_assign_uptr(void **p, void **v); -#define ossl_rcu_deref(p) ossl_rcu_uptr_deref((void **)p) -#define ossl_rcu_assign_ptr(p,v) ossl_rcu_assign_uptr((void **)p, (void **)v) - -#endif diff --git a/openssl/include/internal/recordmethod.h b/openssl/include/internal/recordmethod.h deleted file mode 100644 index 53bd4ca6d..000000000 --- a/openssl/include/internal/recordmethod.h +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_RECORDMETHOD_H -# define OSSL_INTERNAL_RECORDMETHOD_H -# pragma once - -# include - -/* - * We use the term "record" here to refer to a packet of data. Records are - * typically protected via a cipher and MAC, or an AEAD cipher (although not - * always). This usage of the term record is consistent with the TLS concept. - * In QUIC the term "record" is not used but it is analogous to the QUIC term - * "packet". The interface in this file applies to all protocols that protect - * records/packets of data, i.e. (D)TLS and QUIC. The term record is used to - * refer to both contexts. - */ - -/* - * An OSSL_RECORD_METHOD is a protocol specific method which provides the - * functions for reading and writing records for that protocol. Which - * OSSL_RECORD_METHOD to use for a given protocol is defined by the SSL_METHOD. - */ -typedef struct ossl_record_method_st OSSL_RECORD_METHOD; - -/* - * An OSSL_RECORD_LAYER is just an externally defined opaque pointer created by - * the method - */ -typedef struct ossl_record_layer_st OSSL_RECORD_LAYER; - - -# define OSSL_RECORD_ROLE_CLIENT 0 -# define OSSL_RECORD_ROLE_SERVER 1 - -# define OSSL_RECORD_DIRECTION_READ 0 -# define OSSL_RECORD_DIRECTION_WRITE 1 - -/* - * Protection level. For <= TLSv1.2 only "NONE" and "APPLICATION" are used. - */ -# define OSSL_RECORD_PROTECTION_LEVEL_NONE 0 -# define OSSL_RECORD_PROTECTION_LEVEL_EARLY 1 -# define OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE 2 -# define OSSL_RECORD_PROTECTION_LEVEL_APPLICATION 3 - -# define OSSL_RECORD_RETURN_SUCCESS 1 -# define OSSL_RECORD_RETURN_RETRY 0 -# define OSSL_RECORD_RETURN_NON_FATAL_ERR -1 -# define OSSL_RECORD_RETURN_FATAL -2 -# define OSSL_RECORD_RETURN_EOF -3 - -/* - * Template for creating a record. A record consists of the |type| of data it - * will contain (e.g. alert, handshake, application data, etc) along with a - * buffer of payload data in |buf| of length |buflen|. - */ -struct ossl_record_template_st { - unsigned char type; - unsigned int version; - const unsigned char *buf; - size_t buflen; -}; - -typedef struct ossl_record_template_st OSSL_RECORD_TEMPLATE; - -/* - * Rather than a "method" approach, we could make this fetchable - Should we? - * There could be some complexity in finding suitable record layer implementations - * e.g. we need to find one that matches the negotiated protocol, cipher, - * extensions, etc. The selection_cb approach given above doesn't work so well - * if unknown third party providers with OSSL_RECORD_METHOD implementations are - * loaded. - */ - -/* - * If this becomes public API then we will need functions to create and - * free an OSSL_RECORD_METHOD, as well as functions to get/set the various - * function pointers....unless we make it fetchable. - */ -struct ossl_record_method_st { - /* - * Create a new OSSL_RECORD_LAYER object for handling the protocol version - * set by |vers|. |role| is 0 for client and 1 for server. |direction| - * indicates either read or write. |level| is the protection level as - * described above. |settings| are mandatory settings that will cause the - * new() call to fail if they are not understood (for example to require - * Encrypt-Then-Mac support). |options| are optional settings that will not - * cause the new() call to fail if they are not understood (for example - * whether to use "read ahead" or not). - * - * The BIO in |transport| is the BIO for the underlying transport layer. - * Where the direction is "read", then this BIO will only ever be used for - * reading data. Where the direction is "write", then this BIO will only - * every be used for writing data. - * - * An SSL object will always have at least 2 OSSL_RECORD_LAYER objects in - * force at any one time (one for reading and one for writing). In some - * protocols more than 2 might be used (e.g. in DTLS for retransmitting - * messages from an earlier epoch). - * - * The created OSSL_RECORD_LAYER object is stored in *ret on success (or - * NULL otherwise). The return value will be one of - * OSSL_RECORD_RETURN_SUCCESS, OSSL_RECORD_RETURN_FATAL or - * OSSL_RECORD_RETURN_NON_FATAL. A non-fatal return means that creation of - * the record layer has failed because it is unsuitable, but an alternative - * record layer can be tried instead. - */ - - /* - * If we eventually make this fetchable then we will need to use something - * other than EVP_CIPHER. Also mactype would not be a NID, but a string. For - * now though, this works. - */ - int (*new_record_layer)(OSSL_LIB_CTX *libctx, - const char *propq, int vers, - int role, int direction, - int level, - uint16_t epoch, - unsigned char *secret, - size_t secretlen, - unsigned char *key, - size_t keylen, - unsigned char *iv, - size_t ivlen, - unsigned char *mackey, - size_t mackeylen, - const EVP_CIPHER *ciph, - size_t taglen, - int mactype, - const EVP_MD *md, - COMP_METHOD *comp, - const EVP_MD *kdfdigest, - BIO *prev, - BIO *transport, - BIO *next, - BIO_ADDR *local, - BIO_ADDR *peer, - const OSSL_PARAM *settings, - const OSSL_PARAM *options, - const OSSL_DISPATCH *fns, - void *cbarg, - void *rlarg, - OSSL_RECORD_LAYER **ret); - int (*free)(OSSL_RECORD_LAYER *rl); - - /* Returns 1 if we have unprocessed data buffered or 0 otherwise */ - int (*unprocessed_read_pending)(OSSL_RECORD_LAYER *rl); - - /* - * Returns 1 if we have processed data buffered that can be read or 0 otherwise - * - not necessarily app data - */ - int (*processed_read_pending)(OSSL_RECORD_LAYER *rl); - - /* - * The amount of processed app data that is internally buffered and - * available to read - */ - size_t (*app_data_pending)(OSSL_RECORD_LAYER *rl); - - /* - * Find out the maximum number of records that the record layer is prepared - * to process in a single call to write_records. It is the caller's - * responsibility to ensure that no call to write_records exceeds this - * number of records. |type| is the type of the records that the caller - * wants to write, and |len| is the total amount of data that it wants - * to send. |maxfrag| is the maximum allowed fragment size based on user - * configuration, or TLS parameter negotiation. |*preffrag| contains on - * entry the default fragment size that will actually be used based on user - * configuration. This will always be less than or equal to |maxfrag|. On - * exit the record layer may update this to an alternative fragment size to - * be used. This must always be less than or equal to |maxfrag|. - */ - size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len, - size_t maxfrag, size_t *preffrag); - - /* - * Write |numtempl| records from the array of record templates pointed to - * by |templates|. Each record should be no longer than the value returned - * by get_max_record_len(), and there should be no more records than the - * value returned by get_max_records(). - * Where possible the caller will attempt to ensure that all records are the - * same length, except the last record. This may not always be possible so - * the record method implementation should not rely on this being the case. - * In the event of a retry the caller should call retry_write_records() - * to try again. No more calls to write_records() should be attempted until - * retry_write_records() returns success. - * Buffers allocated for the record templates can be freed immediately after - * write_records() returns - even in the case a retry. - * The record templates represent the plaintext payload. The encrypted - * output is written to the |transport| BIO. - * Returns: - * 1 on success - * 0 on retry - * -1 on failure - */ - int (*write_records)(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, - size_t numtempl); - - /* - * Retry a previous call to write_records. The caller should continue to - * call this until the function returns with success or failure. After - * each retry more of the data may have been incrementally sent. - * Returns: - * 1 on success - * 0 on retry - * -1 on failure - */ - int (*retry_write_records)(OSSL_RECORD_LAYER *rl); - - /* - * Read a record and return the record layer version and record type in - * the |rversion| and |type| parameters. |*data| is set to point to a - * record layer buffer containing the record payload data and |*datalen| - * is filled in with the length of that data. The |epoch| and |seq_num| - * values are only used if DTLS has been negotiated. In that case they are - * filled in with the epoch and sequence number from the record. - * An opaque record layer handle for the record is returned in |*rechandle| - * which is used in a subsequent call to |release_record|. The buffer must - * remain available until all the bytes from record are released via one or - * more release_record calls. - * - * Internally the OSSL_RECORD_METHOD implementation may read/process - * multiple records in one go and buffer them. - */ - int (*read_record)(OSSL_RECORD_LAYER *rl, void **rechandle, int *rversion, - uint8_t *type, const unsigned char **data, size_t *datalen, - uint16_t *epoch, unsigned char *seq_num); - /* - * Release length bytes from a buffer associated with a record previously - * read with read_record. Once all the bytes from a record are released, the - * whole record and its associated buffer is released. Records are - * guaranteed to be released in the order that they are read. - */ - int (*release_record)(OSSL_RECORD_LAYER *rl, void *rechandle, size_t length); - - /* - * In the event that a fatal error is returned from the functions above then - * get_alert_code() can be called to obtain a more details identifier for - * the error. In (D)TLS this is the alert description code. - */ - int (*get_alert_code)(OSSL_RECORD_LAYER *rl); - - /* - * Update the transport BIO from the one originally set in the - * new_record_layer call - */ - int (*set1_bio)(OSSL_RECORD_LAYER *rl, BIO *bio); - - /* Called when protocol negotiation selects a protocol version to use */ - int (*set_protocol_version)(OSSL_RECORD_LAYER *rl, int version); - - /* - * Whether we are allowed to receive unencrypted alerts, even if we might - * otherwise expect encrypted records. Ignored by protocol versions where - * this isn't relevant - */ - void (*set_plain_alerts)(OSSL_RECORD_LAYER *rl, int allow); - - /* - * Called immediately after creation of the record layer if we are in a - * first handshake. Also called at the end of the first handshake - */ - void (*set_first_handshake)(OSSL_RECORD_LAYER *rl, int first); - - /* - * Set the maximum number of pipelines that the record layer should process. - * The default is 1. - */ - void (*set_max_pipelines)(OSSL_RECORD_LAYER *rl, size_t max_pipelines); - - /* - * Called to tell the record layer whether we are currently "in init" or - * not. Default at creation of the record layer is "yes". - */ - void (*set_in_init)(OSSL_RECORD_LAYER *rl, int in_init); - - /* - * Get a short or long human readable description of the record layer state - */ - void (*get_state)(OSSL_RECORD_LAYER *rl, const char **shortstr, - const char **longstr); - - /* - * Set new options or modify ones that were originally specified in the - * new_record_layer call. - */ - int (*set_options)(OSSL_RECORD_LAYER *rl, const OSSL_PARAM *options); - - const COMP_METHOD *(*get_compression)(OSSL_RECORD_LAYER *rl); - - /* - * Set the maximum fragment length to be used for the record layer. This - * will override any previous value supplied for the "max_frag_len" - * setting during construction of the record layer. - */ - void (*set_max_frag_len)(OSSL_RECORD_LAYER *rl, size_t max_frag_len); - - /* - * The maximum expansion in bytes that the record layer might add while - * writing a record - */ - size_t (*get_max_record_overhead)(OSSL_RECORD_LAYER *rl); - - /* - * Increment the record sequence number - */ - int (*increment_sequence_ctr)(OSSL_RECORD_LAYER *rl); - - /* - * Allocate read or write buffers. Does nothing if already allocated. - * Assumes default buffer length and 1 pipeline. - */ - int (*alloc_buffers)(OSSL_RECORD_LAYER *rl); - - /* - * Free read or write buffers. Fails if there is pending read or write - * data. Buffers are automatically reallocated on next read/write. - */ - int (*free_buffers)(OSSL_RECORD_LAYER *rl); -}; - - -/* Standard built-in record methods */ -extern const OSSL_RECORD_METHOD ossl_tls_record_method; -# ifndef OPENSSL_NO_KTLS -extern const OSSL_RECORD_METHOD ossl_ktls_record_method; -# endif -extern const OSSL_RECORD_METHOD ossl_dtls_record_method; - -#endif /* !defined(OSSL_INTERNAL_RECORDMETHOD_H) */ diff --git a/openssl/include/internal/refcount.h b/openssl/include/internal/refcount.h index 0bab06122..7412d62f5 100644 --- a/openssl/include/internal/refcount.h +++ b/openssl/include/internal/refcount.h @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,9 +12,8 @@ # include # include -# include -# if defined(OPENSSL_THREADS) && !defined(OPENSSL_DEV_NO_ATOMICS) +# ifndef OPENSSL_DEV_NO_ATOMICS # if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ && !defined(__STDC_NO_ATOMICS__) # include @@ -26,13 +25,12 @@ # define HAVE_ATOMICS 1 -typedef struct { - _Atomic int val; -} CRYPTO_REF_COUNT; +typedef _Atomic int CRYPTO_REF_COUNT; -static inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static inline int CRYPTO_UP_REF(_Atomic int *val, int *ret, + ossl_unused void *lock) { - *ret = atomic_fetch_add_explicit(&refcnt->val, 1, memory_order_relaxed) + 1; + *ret = atomic_fetch_add_explicit(val, 1, memory_order_relaxed) + 1; return 1; } @@ -46,70 +44,50 @@ static inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) * to mutable members doesn't have to be serialized anymore, which would * otherwise imply an acquire fence. Hence conditional acquire fence... */ -static inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static inline int CRYPTO_DOWN_REF(_Atomic int *val, int *ret, + ossl_unused void *lock) { - *ret = atomic_fetch_sub_explicit(&refcnt->val, 1, memory_order_relaxed) - 1; + *ret = atomic_fetch_sub_explicit(val, 1, memory_order_relaxed) - 1; if (*ret == 0) atomic_thread_fence(memory_order_acquire); return 1; } -static inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) -{ - *ret = atomic_load_explicit(&refcnt->val, memory_order_relaxed); - return 1; -} - # elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0 # define HAVE_ATOMICS 1 -typedef struct { - int val; -} CRYPTO_REF_COUNT; +typedef int CRYPTO_REF_COUNT; -static __inline__ int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static __inline__ int CRYPTO_UP_REF(int *val, int *ret, ossl_unused void *lock) { - *ret = __atomic_fetch_add(&refcnt->val, 1, __ATOMIC_RELAXED) + 1; + *ret = __atomic_fetch_add(val, 1, __ATOMIC_RELAXED) + 1; return 1; } -static __inline__ int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static __inline__ int CRYPTO_DOWN_REF(int *val, int *ret, + ossl_unused void *lock) { - *ret = __atomic_fetch_sub(&refcnt->val, 1, __ATOMIC_RELAXED) - 1; + *ret = __atomic_fetch_sub(val, 1, __ATOMIC_RELAXED) - 1; if (*ret == 0) __atomic_thread_fence(__ATOMIC_ACQUIRE); return 1; } - -static __inline__ int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) -{ - *ret = __atomic_load_n(&refcnt->val, __ATOMIC_RELAXED); - return 1; -} - # elif defined(__ICL) && defined(_WIN32) # define HAVE_ATOMICS 1 +typedef volatile int CRYPTO_REF_COUNT; -typedef struct { - volatile int val; -} CRYPTO_REF_COUNT; - -static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) -{ - *ret = _InterlockedExchangeAdd((void *)&refcnt->val, 1) + 1; - return 1; -} - -static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static __inline int CRYPTO_UP_REF(volatile int *val, int *ret, + ossl_unused void *lock) { - *ret = _InterlockedExchangeAdd((void *)&refcnt->val, -1) - 1; + *ret = _InterlockedExchangeAdd((void *)val, 1) + 1; return 1; } -static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static __inline int CRYPTO_DOWN_REF(volatile int *val, int *ret, + ossl_unused void *lock) { - *ret = _InterlockedOr((void *)&refcnt->val, 0); + *ret = _InterlockedExchangeAdd((void *)val, -1) - 1; return 1; } @@ -117,9 +95,7 @@ static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) # define HAVE_ATOMICS 1 -typedef struct { - volatile int val; -} CRYPTO_REF_COUNT; +typedef volatile int CRYPTO_REF_COUNT; # if (defined(_M_ARM) && _M_ARM>=7 && !defined(_WIN32_WCE)) || defined(_M_ARM64) # include @@ -127,26 +103,21 @@ typedef struct { # define _ARM_BARRIER_ISH _ARM64_BARRIER_ISH # endif -static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static __inline int CRYPTO_UP_REF(volatile int *val, int *ret, + ossl_unused void *lock) { - *ret = _InterlockedExchangeAdd_nf(&refcnt->val, 1) + 1; + *ret = _InterlockedExchangeAdd_nf(val, 1) + 1; return 1; } -static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static __inline int CRYPTO_DOWN_REF(volatile int *val, int *ret, + ossl_unused void *lock) { - *ret = _InterlockedExchangeAdd_nf(&refcnt->val, -1) - 1; + *ret = _InterlockedExchangeAdd_nf(val, -1) - 1; if (*ret == 0) __dmb(_ARM_BARRIER_ISH); return 1; } - -static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) -{ - *ret = _InterlockedOr_nf((void *)&refcnt->val, 0); - return 1; -} - # else # if !defined(_WIN32_WCE) # pragma intrinsic(_InterlockedExchangeAdd) @@ -160,24 +131,19 @@ static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) # endif # endif -static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret) -{ - *ret = _InterlockedExchangeAdd(&refcnt->val, 1) + 1; - return 1; -} - -static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static __inline int CRYPTO_UP_REF(volatile int *val, int *ret, + ossl_unused void *lock) { - *ret = _InterlockedExchangeAdd(&refcnt->val, -1) - 1; + *ret = _InterlockedExchangeAdd(val, 1) + 1; return 1; } -static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) +static __inline int CRYPTO_DOWN_REF(volatile int *val, int *ret, + ossl_unused void *lock) { - *ret = _InterlockedExchangeAdd(&refcnt->val, 0); + *ret = _InterlockedExchangeAdd(val, -1) - 1; return 1; } - # endif # endif @@ -190,92 +156,13 @@ static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) */ # ifndef HAVE_ATOMICS -typedef struct { - int val; -# ifdef OPENSSL_THREADS - CRYPTO_RWLOCK *lock; -# endif -} CRYPTO_REF_COUNT; - -# ifdef OPENSSL_THREADS +typedef int CRYPTO_REF_COUNT; -static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, - int *ret) -{ - return CRYPTO_atomic_add(&refcnt->val, 1, ret, refcnt->lock); -} - -static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, - int *ret) -{ - return CRYPTO_atomic_add(&refcnt->val, -1, ret, refcnt->lock); -} +# define CRYPTO_UP_REF(val, ret, lock) CRYPTO_atomic_add(val, 1, ret, lock) +# define CRYPTO_DOWN_REF(val, ret, lock) CRYPTO_atomic_add(val, -1, ret, lock) -static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, - int *ret) -{ - return CRYPTO_atomic_load_int(&refcnt->val, ret, refcnt->lock); -} - -# define CRYPTO_NEW_FREE_DEFINED 1 -static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n) -{ - refcnt->val = n; - refcnt->lock = CRYPTO_THREAD_lock_new(); - if (refcnt->lock == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); - return 0; - } - return 1; -} - -static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt) \ -{ - if (refcnt != NULL) - CRYPTO_THREAD_lock_free(refcnt->lock); -} - -# else /* OPENSSL_THREADS */ - -static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, - int *ret) -{ - refcnt->val++; - *ret = refcnt->val; - return 1; -} - -static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, - int *ret) -{ - refcnt->val--; - *ret = refcnt->val; - return 1; -} - -static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, - int *ret) -{ - *ret = refcnt->val; - return 1; -} - -# endif /* OPENSSL_THREADS */ # endif -# ifndef CRYPTO_NEW_FREE_DEFINED -static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n) -{ - refcnt->val = n; - return 1; -} - -static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt) \ -{ -} -# endif /* CRYPTO_NEW_FREE_DEFINED */ -#undef CRYPTO_NEW_FREE_DEFINED - # if !defined(NDEBUG) && !defined(OPENSSL_NO_STDIO) # define REF_ASSERT_ISNT(test) \ (void)((test) ? (OPENSSL_die("refcount error", __FILE__, __LINE__), 1) : 0) @@ -286,6 +173,6 @@ static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt) # define REF_PRINT_EX(text, count, object) \ OSSL_TRACE3(REF_COUNT, "%p:%4d:%s\n", (object), (count), (text)); # define REF_PRINT_COUNT(text, object) \ - REF_PRINT_EX(text, object->references.val, (void *)object) + REF_PRINT_EX(text, object->references, (void *)object) #endif diff --git a/openssl/include/internal/ring_buf.h b/openssl/include/internal/ring_buf.h deleted file mode 100644 index 436f1ca14..000000000 --- a/openssl/include/internal/ring_buf.h +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_RING_BUF_H -# define OSSL_INTERNAL_RING_BUF_H -# pragma once - -# include /* For 'ossl_inline' */ -# include "internal/safe_math.h" - -/* - * ================================================================== - * Byte-wise ring buffer which supports pushing and popping blocks of multiple - * bytes at a time. The logical offset of each byte for the purposes of a QUIC - * stream is tracked. Bytes can be popped from the ring buffer in two stages; - * first they are popped, and then they are culled. Bytes which have been popped - * but not yet culled will not be overwritten, and can be restored. - */ -struct ring_buf { - void *start; - size_t alloc; /* size of buffer allocation in bytes */ - - /* - * Logical offset of the head (where we append to). This is the current size - * of the QUIC stream. This increases monotonically. - */ - uint64_t head_offset; - - /* - * Logical offset of the cull tail. Data is no longer needed and is - * deallocated as the cull tail advances, which occurs as data is - * acknowledged. This increases monotonically. - */ - uint64_t ctail_offset; -}; - -OSSL_SAFE_MATH_UNSIGNED(u64, uint64_t) - -#define MAX_OFFSET (((uint64_t)1) << 62) /* QUIC-imposed limit */ - -static ossl_inline int ring_buf_init(struct ring_buf *r) -{ - r->start = NULL; - r->alloc = 0; - r->head_offset = r->ctail_offset = 0; - return 1; -} - -static ossl_inline void ring_buf_destroy(struct ring_buf *r, int cleanse) -{ - if (cleanse) - OPENSSL_clear_free(r->start, r->alloc); - else - OPENSSL_free(r->start); - r->start = NULL; - r->alloc = 0; -} - -static ossl_inline size_t ring_buf_used(struct ring_buf *r) -{ - return (size_t)(r->head_offset - r->ctail_offset); -} - -static ossl_inline size_t ring_buf_avail(struct ring_buf *r) -{ - return r->alloc - ring_buf_used(r); -} - -static ossl_inline int ring_buf_write_at(struct ring_buf *r, - uint64_t logical_offset, - const unsigned char *buf, - size_t buf_len) -{ - size_t avail, idx, l; - unsigned char *start = r->start; - int i, err = 0; - - avail = ring_buf_avail(r); - if (logical_offset < r->ctail_offset - || safe_add_u64(logical_offset, buf_len, &err) - > safe_add_u64(r->head_offset, avail, &err) - || safe_add_u64(r->head_offset, buf_len, &err) - > MAX_OFFSET - || err) - return 0; - - for (i = 0; buf_len > 0 && i < 2; ++i) { - idx = logical_offset % r->alloc; - l = r->alloc - idx; - if (buf_len < l) - l = buf_len; - - memcpy(start + idx, buf, l); - if (r->head_offset < logical_offset + l) - r->head_offset = logical_offset + l; - - logical_offset += l; - buf += l; - buf_len -= l; - } - - assert(buf_len == 0); - - return 1; -} - -static ossl_inline size_t ring_buf_push(struct ring_buf *r, - const unsigned char *buf, - size_t buf_len) -{ - size_t pushed = 0, avail, idx, l; - unsigned char *start = r->start; - - for (;;) { - avail = ring_buf_avail(r); - if (buf_len > avail) - buf_len = avail; - - if (buf_len > MAX_OFFSET - r->head_offset) - buf_len = (size_t)(MAX_OFFSET - r->head_offset); - - if (buf_len == 0) - break; - - idx = r->head_offset % r->alloc; - l = r->alloc - idx; - if (buf_len < l) - l = buf_len; - - memcpy(start + idx, buf, l); - r->head_offset += l; - buf += l; - buf_len -= l; - pushed += l; - } - - return pushed; -} - -static ossl_inline const unsigned char *ring_buf_get_ptr(const struct ring_buf *r, - uint64_t logical_offset, - size_t *max_len) -{ - unsigned char *start = r->start; - size_t idx; - - if (logical_offset >= r->head_offset || logical_offset < r->ctail_offset) - return NULL; - idx = logical_offset % r->alloc; - *max_len = r->alloc - idx; - return start + idx; -} - -/* - * Retrieves data out of the read side of the ring buffer starting at the given - * logical offset. *buf is set to point to a contiguous span of bytes and - * *buf_len is set to the number of contiguous bytes. After this function - * returns, there may or may not be more bytes available at the logical offset - * of (logical_offset + *buf_len) by calling this function again. If the logical - * offset is out of the range retained by the ring buffer, returns 0, else - * returns 1. A logical offset at the end of the range retained by the ring - * buffer is not considered an error and is returned with a *buf_len of 0. - * - * The ring buffer state is not changed. - */ -static ossl_inline int ring_buf_get_buf_at(const struct ring_buf *r, - uint64_t logical_offset, - const unsigned char **buf, - size_t *buf_len) -{ - const unsigned char *start = r->start; - size_t idx, l; - - if (logical_offset > r->head_offset || logical_offset < r->ctail_offset) - return 0; - - if (r->alloc == 0) { - *buf = NULL; - *buf_len = 0; - return 1; - } - - idx = logical_offset % r->alloc; - l = (size_t)(r->head_offset - logical_offset); - if (l > r->alloc - idx) - l = r->alloc - idx; - - *buf = start + idx; - *buf_len = l; - return 1; -} - -static ossl_inline void ring_buf_cpop_range(struct ring_buf *r, - uint64_t start, uint64_t end, - int cleanse) -{ - assert(end >= start); - - if (start > r->ctail_offset || end >= MAX_OFFSET) - return; - - if (cleanse && r->alloc > 0 && end > r->ctail_offset) { - size_t idx = r->ctail_offset % r->alloc; - uint64_t cleanse_end = end + 1; - size_t l; - - if (cleanse_end > r->head_offset) - cleanse_end = r->head_offset; - l = (size_t)(cleanse_end - r->ctail_offset); - if (l > r->alloc - idx) { - OPENSSL_cleanse((unsigned char *)r->start + idx, r->alloc - idx); - l -= r->alloc - idx; - idx = 0; - } - if (l > 0) - OPENSSL_cleanse((unsigned char *)r->start + idx, l); - } - - r->ctail_offset = end + 1; - /* Allow culling unpushed data */ - if (r->head_offset < r->ctail_offset) - r->head_offset = r->ctail_offset; -} - -static ossl_inline int ring_buf_resize(struct ring_buf *r, size_t num_bytes, - int cleanse) -{ - struct ring_buf rnew = {0}; - const unsigned char *src = NULL; - size_t src_len = 0, copied = 0; - - if (num_bytes == r->alloc) - return 1; - - if (num_bytes < ring_buf_used(r)) - return 0; - - rnew.start = OPENSSL_malloc(num_bytes); - if (rnew.start == NULL) - return 0; - - rnew.alloc = num_bytes; - rnew.head_offset = r->head_offset - ring_buf_used(r); - rnew.ctail_offset = rnew.head_offset; - - for (;;) { - if (!ring_buf_get_buf_at(r, r->ctail_offset + copied, &src, &src_len)) { - OPENSSL_free(rnew.start); - return 0; - } - - if (src_len == 0) - break; - - if (ring_buf_push(&rnew, src, src_len) != src_len) { - OPENSSL_free(rnew.start); - return 0; - } - - copied += src_len; - } - - assert(rnew.head_offset == r->head_offset); - rnew.ctail_offset = r->ctail_offset; - - ring_buf_destroy(r, cleanse); - memcpy(r, &rnew, sizeof(*r)); - return 1; -} - -#endif /* OSSL_INTERNAL_RING_BUF_H */ diff --git a/openssl/include/internal/safe_math.h b/openssl/include/internal/safe_math.h deleted file mode 100644 index be37e6ab8..000000000 --- a/openssl/include/internal/safe_math.h +++ /dev/null @@ -1,443 +0,0 @@ -/* - * Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_SAFE_MATH_H -# define OSSL_INTERNAL_SAFE_MATH_H -# pragma once - -# include /* For 'ossl_inline' */ - -# ifndef OPENSSL_NO_BUILTIN_OVERFLOW_CHECKING -# ifdef __has_builtin -# define has(func) __has_builtin(func) -# elif __GNUC__ > 5 -# define has(func) 1 -# endif -# endif /* OPENSSL_NO_BUILTIN_OVERFLOW_CHECKING */ - -# ifndef has -# define has(func) 0 -# endif - -/* - * Safe addition helpers - */ -# if has(__builtin_add_overflow) -# define OSSL_SAFE_MATH_ADDS(type_name, type, min, max) \ - static ossl_inline ossl_unused type safe_add_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - type r; \ - \ - if (!__builtin_add_overflow(a, b, &r)) \ - return r; \ - *err |= 1; \ - return a < 0 ? min : max; \ - } - -# define OSSL_SAFE_MATH_ADDU(type_name, type, max) \ - static ossl_inline ossl_unused type safe_add_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - type r; \ - \ - if (!__builtin_add_overflow(a, b, &r)) \ - return r; \ - *err |= 1; \ - return a + b; \ - } - -# else /* has(__builtin_add_overflow) */ -# define OSSL_SAFE_MATH_ADDS(type_name, type, min, max) \ - static ossl_inline ossl_unused type safe_add_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if ((a < 0) ^ (b < 0) \ - || (a > 0 && b <= max - a) \ - || (a < 0 && b >= min - a) \ - || a == 0) \ - return a + b; \ - *err |= 1; \ - return a < 0 ? min : max; \ - } - -# define OSSL_SAFE_MATH_ADDU(type_name, type, max) \ - static ossl_inline ossl_unused type safe_add_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (b > max - a) \ - *err |= 1; \ - return a + b; \ - } -# endif /* has(__builtin_add_overflow) */ - -/* - * Safe subtraction helpers - */ -# if has(__builtin_sub_overflow) -# define OSSL_SAFE_MATH_SUBS(type_name, type, min, max) \ - static ossl_inline ossl_unused type safe_sub_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - type r; \ - \ - if (!__builtin_sub_overflow(a, b, &r)) \ - return r; \ - *err |= 1; \ - return a < 0 ? min : max; \ - } - -# else /* has(__builtin_sub_overflow) */ -# define OSSL_SAFE_MATH_SUBS(type_name, type, min, max) \ - static ossl_inline ossl_unused type safe_sub_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (!((a < 0) ^ (b < 0)) \ - || (b > 0 && a >= min + b) \ - || (b < 0 && a <= max + b) \ - || b == 0) \ - return a - b; \ - *err |= 1; \ - return a < 0 ? min : max; \ - } - -# endif /* has(__builtin_sub_overflow) */ - -# define OSSL_SAFE_MATH_SUBU(type_name, type) \ - static ossl_inline ossl_unused type safe_sub_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (b > a) \ - *err |= 1; \ - return a - b; \ - } - -/* - * Safe multiplication helpers - */ -# if has(__builtin_mul_overflow) -# define OSSL_SAFE_MATH_MULS(type_name, type, min, max) \ - static ossl_inline ossl_unused type safe_mul_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - type r; \ - \ - if (!__builtin_mul_overflow(a, b, &r)) \ - return r; \ - *err |= 1; \ - return (a < 0) ^ (b < 0) ? min : max; \ - } - -# define OSSL_SAFE_MATH_MULU(type_name, type, max) \ - static ossl_inline ossl_unused type safe_mul_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - type r; \ - \ - if (!__builtin_mul_overflow(a, b, &r)) \ - return r; \ - *err |= 1; \ - return a * b; \ - } - -# else /* has(__builtin_mul_overflow) */ -# define OSSL_SAFE_MATH_MULS(type_name, type, min, max) \ - static ossl_inline ossl_unused type safe_mul_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (a == 0 || b == 0) \ - return 0; \ - if (a == 1) \ - return b; \ - if (b == 1) \ - return a; \ - if (a != min && b != min) { \ - const type x = a < 0 ? -a : a; \ - const type y = b < 0 ? -b : b; \ - \ - if (x <= max / y) \ - return a * b; \ - } \ - *err |= 1; \ - return (a < 0) ^ (b < 0) ? min : max; \ - } - -# define OSSL_SAFE_MATH_MULU(type_name, type, max) \ - static ossl_inline ossl_unused type safe_mul_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (b != 0 && a > max / b) \ - *err |= 1; \ - return a * b; \ - } -# endif /* has(__builtin_mul_overflow) */ - -/* - * Safe division helpers - */ -# define OSSL_SAFE_MATH_DIVS(type_name, type, min, max) \ - static ossl_inline ossl_unused type safe_div_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (b == 0) { \ - *err |= 1; \ - return a < 0 ? min : max; \ - } \ - if (b == -1 && a == min) { \ - *err |= 1; \ - return max; \ - } \ - return a / b; \ - } - -# define OSSL_SAFE_MATH_DIVU(type_name, type, max) \ - static ossl_inline ossl_unused type safe_div_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (b != 0) \ - return a / b; \ - *err |= 1; \ - return max; \ - } - -/* - * Safe modulus helpers - */ -# define OSSL_SAFE_MATH_MODS(type_name, type, min, max) \ - static ossl_inline ossl_unused type safe_mod_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (b == 0) { \ - *err |= 1; \ - return 0; \ - } \ - if (b == -1 && a == min) { \ - *err |= 1; \ - return max; \ - } \ - return a % b; \ - } - -# define OSSL_SAFE_MATH_MODU(type_name, type) \ - static ossl_inline ossl_unused type safe_mod_ ## type_name(type a, \ - type b, \ - int *err) \ - { \ - if (b != 0) \ - return a % b; \ - *err |= 1; \ - return 0; \ - } - -/* - * Safe negation helpers - */ -# define OSSL_SAFE_MATH_NEGS(type_name, type, min) \ - static ossl_inline ossl_unused type safe_neg_ ## type_name(type a, \ - int *err) \ - { \ - if (a != min) \ - return -a; \ - *err |= 1; \ - return min; \ - } - -# define OSSL_SAFE_MATH_NEGU(type_name, type) \ - static ossl_inline ossl_unused type safe_neg_ ## type_name(type a, \ - int *err) \ - { \ - if (a == 0) \ - return a; \ - *err |= 1; \ - return 1 + ~a; \ - } - -/* - * Safe absolute value helpers - */ -# define OSSL_SAFE_MATH_ABSS(type_name, type, min) \ - static ossl_inline ossl_unused type safe_abs_ ## type_name(type a, \ - int *err) \ - { \ - if (a != min) \ - return a < 0 ? -a : a; \ - *err |= 1; \ - return min; \ - } - -# define OSSL_SAFE_MATH_ABSU(type_name, type) \ - static ossl_inline ossl_unused type safe_abs_ ## type_name(type a, \ - int *err) \ - { \ - return a; \ - } - -/* - * Safe fused multiply divide helpers - * - * These are a bit obscure: - * . They begin by checking the denominator for zero and getting rid of this - * corner case. - * - * . Second is an attempt to do the multiplication directly, if it doesn't - * overflow, the quotient is returned (for signed values there is a - * potential problem here which isn't present for unsigned). - * - * . Finally, the multiplication/division is transformed so that the larger - * of the numerators is divided first. This requires a remainder - * correction: - * - * a b / c = (a / c) b + (a mod c) b / c, where a > b - * - * The individual operations need to be overflow checked (again signed - * being more problematic). - * - * The algorithm used is not perfect but it should be "good enough". - */ -# define OSSL_SAFE_MATH_MULDIVS(type_name, type, max) \ - static ossl_inline ossl_unused type safe_muldiv_ ## type_name(type a, \ - type b, \ - type c, \ - int *err) \ - { \ - int e2 = 0; \ - type q, r, x, y; \ - \ - if (c == 0) { \ - *err |= 1; \ - return a == 0 || b == 0 ? 0 : max; \ - } \ - x = safe_mul_ ## type_name(a, b, &e2); \ - if (!e2) \ - return safe_div_ ## type_name(x, c, err); \ - if (b > a) { \ - x = b; \ - b = a; \ - a = x; \ - } \ - q = safe_div_ ## type_name(a, c, err); \ - r = safe_mod_ ## type_name(a, c, err); \ - x = safe_mul_ ## type_name(r, b, err); \ - y = safe_mul_ ## type_name(q, b, err); \ - q = safe_div_ ## type_name(x, c, err); \ - return safe_add_ ## type_name(y, q, err); \ - } - -# define OSSL_SAFE_MATH_MULDIVU(type_name, type, max) \ - static ossl_inline ossl_unused type safe_muldiv_ ## type_name(type a, \ - type b, \ - type c, \ - int *err) \ - { \ - int e2 = 0; \ - type x, y; \ - \ - if (c == 0) { \ - *err |= 1; \ - return a == 0 || b == 0 ? 0 : max; \ - } \ - x = safe_mul_ ## type_name(a, b, &e2); \ - if (!e2) \ - return x / c; \ - if (b > a) { \ - x = b; \ - b = a; \ - a = x; \ - } \ - x = safe_mul_ ## type_name(a % c, b, err); \ - y = safe_mul_ ## type_name(a / c, b, err); \ - return safe_add_ ## type_name(y, x / c, err); \ - } - -/* - * Calculate a / b rounding up: - * i.e. a / b + (a % b != 0) - * Which is usually (less safely) converted to (a + b - 1) / b - * If you *know* that b != 0, then it's safe to ignore err. - */ -#define OSSL_SAFE_MATH_DIV_ROUND_UP(type_name, type, max) \ - static ossl_inline ossl_unused type safe_div_round_up_ ## type_name \ - (type a, type b, int *errp) \ - { \ - type x; \ - int *err, err_local = 0; \ - \ - /* Allow errors to be ignored by callers */ \ - err = errp != NULL ? errp : &err_local; \ - /* Fast path, both positive */ \ - if (b > 0 && a > 0) { \ - /* Faster path: no overflow concerns */ \ - if (a < max - b) \ - return (a + b - 1) / b; \ - return a / b + (a % b != 0); \ - } \ - if (b == 0) { \ - *err |= 1; \ - return a == 0 ? 0 : max; \ - } \ - if (a == 0) \ - return 0; \ - /* Rather slow path because there are negatives involved */ \ - x = safe_mod_ ## type_name(a, b, err); \ - return safe_add_ ## type_name(safe_div_ ## type_name(a, b, err), \ - x != 0, err); \ - } - -/* Calculate ranges of types */ -# define OSSL_SAFE_MATH_MINS(type) ((type)1 << (sizeof(type) * 8 - 1)) -# define OSSL_SAFE_MATH_MAXS(type) (~OSSL_SAFE_MATH_MINS(type)) -# define OSSL_SAFE_MATH_MAXU(type) (~(type)0) - -/* - * Wrapper macros to create all the functions of a given type - */ -# define OSSL_SAFE_MATH_SIGNED(type_name, type) \ - OSSL_SAFE_MATH_ADDS(type_name, type, OSSL_SAFE_MATH_MINS(type), \ - OSSL_SAFE_MATH_MAXS(type)) \ - OSSL_SAFE_MATH_SUBS(type_name, type, OSSL_SAFE_MATH_MINS(type), \ - OSSL_SAFE_MATH_MAXS(type)) \ - OSSL_SAFE_MATH_MULS(type_name, type, OSSL_SAFE_MATH_MINS(type), \ - OSSL_SAFE_MATH_MAXS(type)) \ - OSSL_SAFE_MATH_DIVS(type_name, type, OSSL_SAFE_MATH_MINS(type), \ - OSSL_SAFE_MATH_MAXS(type)) \ - OSSL_SAFE_MATH_MODS(type_name, type, OSSL_SAFE_MATH_MINS(type), \ - OSSL_SAFE_MATH_MAXS(type)) \ - OSSL_SAFE_MATH_DIV_ROUND_UP(type_name, type, \ - OSSL_SAFE_MATH_MAXS(type)) \ - OSSL_SAFE_MATH_MULDIVS(type_name, type, OSSL_SAFE_MATH_MAXS(type)) \ - OSSL_SAFE_MATH_NEGS(type_name, type, OSSL_SAFE_MATH_MINS(type)) \ - OSSL_SAFE_MATH_ABSS(type_name, type, OSSL_SAFE_MATH_MINS(type)) - -# define OSSL_SAFE_MATH_UNSIGNED(type_name, type) \ - OSSL_SAFE_MATH_ADDU(type_name, type, OSSL_SAFE_MATH_MAXU(type)) \ - OSSL_SAFE_MATH_SUBU(type_name, type) \ - OSSL_SAFE_MATH_MULU(type_name, type, OSSL_SAFE_MATH_MAXU(type)) \ - OSSL_SAFE_MATH_DIVU(type_name, type, OSSL_SAFE_MATH_MAXU(type)) \ - OSSL_SAFE_MATH_MODU(type_name, type) \ - OSSL_SAFE_MATH_DIV_ROUND_UP(type_name, type, \ - OSSL_SAFE_MATH_MAXU(type)) \ - OSSL_SAFE_MATH_MULDIVU(type_name, type, OSSL_SAFE_MATH_MAXU(type)) \ - OSSL_SAFE_MATH_NEGU(type_name, type) \ - OSSL_SAFE_MATH_ABSU(type_name, type) - -#endif /* OSSL_INTERNAL_SAFE_MATH_H */ diff --git a/openssl/include/internal/sha3.h b/openssl/include/internal/sha3.h index 332916aa5..80ad86e58 100644 --- a/openssl/include/internal/sha3.h +++ b/openssl/include/internal/sha3.h @@ -22,31 +22,23 @@ typedef struct keccak_st KECCAK1600_CTX; -typedef size_t (sha3_absorb_fn)(void *vctx, const void *in, size_t inlen); -typedef int (sha3_final_fn)(void *vctx, unsigned char *out, size_t outlen); -typedef int (sha3_squeeze_fn)(void *vctx, unsigned char *out, size_t outlen); +typedef size_t (sha3_absorb_fn)(void *vctx, const void *inp, size_t len); +typedef int (sha3_final_fn)(unsigned char *md, void *vctx); typedef struct prov_sha3_meth_st { sha3_absorb_fn *absorb; sha3_final_fn *final; - sha3_squeeze_fn *squeeze; } PROV_SHA3_METHOD; -#define XOF_STATE_INIT 0 -#define XOF_STATE_ABSORB 1 -#define XOF_STATE_FINAL 2 -#define XOF_STATE_SQUEEZE 3 - struct keccak_st { uint64_t A[5][5]; - unsigned char buf[KECCAK1600_WIDTH / 8 - 32]; size_t block_size; /* cached ctx->digest->block_size */ size_t md_size; /* output length, variable in XOF */ size_t bufsz; /* used bytes in below buffer */ + unsigned char buf[KECCAK1600_WIDTH / 8 - 32]; unsigned char pad; PROV_SHA3_METHOD meth; - int xof_state; }; void ossl_sha3_reset(KECCAK1600_CTX *ctx); @@ -54,8 +46,7 @@ int ossl_sha3_init(KECCAK1600_CTX *ctx, unsigned char pad, size_t bitlen); int ossl_keccak_kmac_init(KECCAK1600_CTX *ctx, unsigned char pad, size_t bitlen); int ossl_sha3_update(KECCAK1600_CTX *ctx, const void *_inp, size_t len); -int ossl_sha3_final(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen); -int ossl_sha3_squeeze(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen); +int ossl_sha3_final(unsigned char *md, KECCAK1600_CTX *ctx); size_t SHA3_absorb(uint64_t A[5][5], const unsigned char *inp, size_t len, size_t r); diff --git a/openssl/include/internal/sm3.h b/openssl/include/internal/sm3.h deleted file mode 100644 index db1d61f05..000000000 --- a/openssl/include/internal/sm3.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. - * Copyright 2017 Ribose Inc. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* This header can move into provider when legacy support is removed */ -#ifndef OSSL_INTERNAL_SM3_H -# define OSSL_INTERNAL_SM3_H -# pragma once - -# include - -# ifdef OPENSSL_NO_SM3 -# error SM3 is disabled. -# endif - -# define SM3_DIGEST_LENGTH 32 -# define SM3_WORD unsigned int - -# define SM3_CBLOCK 64 -# define SM3_LBLOCK (SM3_CBLOCK/4) - -typedef struct SM3state_st { - SM3_WORD A, B, C, D, E, F, G, H; - SM3_WORD Nl, Nh; - SM3_WORD data[SM3_LBLOCK]; - unsigned int num; -} SM3_CTX; - -int ossl_sm3_init(SM3_CTX *c); -int ossl_sm3_update(SM3_CTX *c, const void *data, size_t len); -int ossl_sm3_final(unsigned char *md, SM3_CTX *c); - -#endif /* OSSL_INTERNAL_SM3_H */ diff --git a/openssl/include/internal/sockets.h b/openssl/include/internal/sockets.h index f51c1b075..041d53db0 100644 --- a/openssl/include/internal/sockets.h +++ b/openssl/include/internal/sockets.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,17 +19,11 @@ # ifdef WIN32 # define NO_SYS_UN_H # endif -# ifdef OPENSSL_SYS_VMS -# define NO_SYS_PARAM_H -# define NO_SYS_UN_H -# endif # ifdef OPENSSL_NO_SOCK # elif defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_MSDOS) # if defined(__DJGPP__) -# define WATT32 -# define WATT32_NO_OLDIES # include # include # include @@ -61,25 +55,7 @@ struct servent *PASCAL getservbyname(const char *, const char *); # define accept(s,f,l) ((int)accept(s,f,l)) # endif -/* Windows have other names for shutdown() reasons */ -# ifndef SHUT_RD -# define SHUT_RD SD_RECEIVE -# endif -# ifndef SHUT_WR -# define SHUT_WR SD_SEND -# endif -# ifndef SHUT_RDWR -# define SHUT_RDWR SD_BOTH -# endif - # else -# if defined(__APPLE__) - /* - * This must be defined before including to get - * IPV6_RECVPKTINFO - */ -# define __APPLE_USE_RFC_3542 -# endif # ifndef NO_SYS_PARAM_H # include @@ -89,52 +65,26 @@ struct servent *PASCAL getservbyname(const char *, const char *); # endif # include -# if defined(OPENSSL_SYS_VMS) -typedef size_t socklen_t; /* Currently appears to be missing on VMS */ -# endif -# if defined(OPENSSL_SYS_VMS_NODECC) -# include -# include -# include -# else -# include -# if !defined(NO_SYS_UN_H) && defined(AF_UNIX) && !defined(OPENSSL_NO_UNIX_SOCK) -# include -# ifndef UNIX_PATH_MAX -# define UNIX_PATH_MAX sizeof(((struct sockaddr_un *)NULL)->sun_path) -# endif -# endif -# ifdef FILIO_H -# include /* FIONBIO in some SVR4, e.g. unixware, solaris */ + +# include +# ifndef NO_SYS_UN_H +# include +# ifndef UNIX_PATH_MAX +# define UNIX_PATH_MAX sizeof(((struct sockaddr_un *)NULL)->sun_path) # endif -# include -# include -# include # endif +# ifdef FILIO_H +# include /* FIONBIO in some SVR4, e.g. unixware, solaris */ +# endif +# include +# include +# include # ifdef OPENSSL_SYS_AIX # include # endif -# ifdef OPENSSL_SYS_UNIX -# ifndef OPENSSL_SYS_TANDEM -# include -# endif -# include -# endif - -# ifndef VMS -# include -# else -# if !defined(TCPIP_TYPE_SOCKETSHR) && defined(__VMS_VER) && (__VMS_VER > 70000000) - /* ioctl is only in VMS > 7.0 and when socketshr is not used */ -# include -# endif -# include -# if defined(TCPIP_TYPE_SOCKETSHR) -# include -# endif -# endif +# include # ifndef INVALID_SOCKET # define INVALID_SOCKET (-1) @@ -153,45 +103,36 @@ typedef size_t socklen_t; /* Currently appears to be missing on VMS */ # endif # endif -/* - * Some platforms define AF_UNIX, but don't support it - */ -# if !defined(OPENSSL_NO_UNIX_SOCK) -# if !defined(AF_UNIX) || defined(NO_SYS_UN_H) -# define OPENSSL_NO_UNIX_SOCK -# endif -# endif - # define get_last_socket_error() errno # define clear_socket_error() errno=0 -# define get_last_socket_error_is_eintr() (get_last_socket_error() == EINTR) # if defined(OPENSSL_SYS_WINDOWS) # undef get_last_socket_error # undef clear_socket_error -# undef get_last_socket_error_is_eintr # define get_last_socket_error() WSAGetLastError() # define clear_socket_error() WSASetLastError(0) -# define get_last_socket_error_is_eintr() (get_last_socket_error() == WSAEINTR) # define readsocket(s,b,n) recv((s),(b),(n),0) # define writesocket(s,b,n) send((s),(b),(n),0) # elif defined(__DJGPP__) +# define WATT32 +# define WATT32_NO_OLDIES # define closesocket(s) close_s(s) # define readsocket(s,b,n) read_s(s,b,n) # define writesocket(s,b,n) send(s,b,n,0) -# elif defined(OPENSSL_SYS_VMS) -# define ioctlsocket(a,b,c) ioctl(a,b,c) -# define closesocket(s) close(s) -# define readsocket(s,b,n) recv((s),(b),(n),0) -# define writesocket(s,b,n) send((s),(b),(n),0) # elif defined(OPENSSL_SYS_VXWORKS) # define ioctlsocket(a,b,c) ioctl((a),(b),(int)(c)) # define closesocket(s) close(s) # define readsocket(s,b,n) read((s),(b),(n)) # define writesocket(s,b,n) write((s),(char *)(b),(n)) # elif defined(OPENSSL_SYS_TANDEM) -# define readsocket(s,b,n) read((s),(b),(n)) -# define writesocket(s,b,n) write((s),(b),(n)) +# if defined(OPENSSL_TANDEM_FLOSS) +# include +# define readsocket(s,b,n) floss_read((s),(b),(n)) +# define writesocket(s,b,n) floss_write((s),(b),(n)) +# else +# define readsocket(s,b,n) read((s),(b),(n)) +# define writesocket(s,b,n) write((s),(b),(n)) +# endif # define ioctlsocket(a,b,c) ioctl(a,b,c) # define closesocket(s) close(s) # else diff --git a/openssl/include/internal/ssl.h b/openssl/include/internal/ssl.h deleted file mode 100644 index 8a0c79749..000000000 --- a/openssl/include/internal/ssl.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include - -#ifndef OSSL_INTERNAL_SSL_H -# define OSSL_INTERNAL_SSL_H -# pragma once - -typedef void (*ossl_msg_cb)(int write_p, int version, int content_type, - const void *buf, size_t len, SSL *ssl, void *arg); - -int ossl_ssl_get_error(const SSL *s, int i, int check_err); - -/* Set if this is the QUIC handshake layer */ -# define TLS1_FLAGS_QUIC 0x2000 - -#endif diff --git a/openssl/include/internal/ssl3_cbc.h b/openssl/include/internal/ssl3_cbc.h deleted file mode 100644 index 4fb5da190..000000000 --- a/openssl/include/internal/ssl3_cbc.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include - -/* tls_pad.c */ -int ssl3_cbc_remove_padding_and_mac(size_t *reclen, - size_t origreclen, - unsigned char *recdata, - unsigned char **mac, - int *alloced, - size_t block_size, size_t mac_size, - OSSL_LIB_CTX *libctx); - -int tls1_cbc_remove_padding_and_mac(size_t *reclen, - size_t origreclen, - unsigned char *recdata, - unsigned char **mac, - int *alloced, - size_t block_size, size_t mac_size, - int aead, - OSSL_LIB_CTX *libctx); - -/* ssl3_cbc.c */ -__owur char ssl3_cbc_record_digest_supported(const EVP_MD_CTX *ctx); -__owur int ssl3_cbc_digest_record(const EVP_MD *md, - unsigned char *md_out, - size_t *md_out_size, - const unsigned char *header, - const unsigned char *data, - size_t data_size, - size_t data_plus_mac_plus_padding_size, - const unsigned char *mac_secret, - size_t mac_secret_length, char is_sslv3); diff --git a/openssl/include/internal/symhacks.h b/openssl/include/internal/symhacks.h index 33bae51e4..d1f51013c 100644 --- a/openssl/include/internal/symhacks.h +++ b/openssl/include/internal/symhacks.h @@ -13,15 +13,4 @@ # include -# if defined(OPENSSL_SYS_VMS) - -/* ossl_provider_gettable_params vs OSSL_PROVIDER_gettable_params */ -# undef ossl_provider_gettable_params -# define ossl_provider_gettable_params ossl_int_prov_gettable_params -/* ossl_provider_get_params vs OSSL_PROVIDER_get_params */ -# undef ossl_provider_get_params -# define ossl_provider_get_params ossl_int_prov_get_params - -# endif - #endif /* ! defined HEADER_VMS_IDHACKS_H */ diff --git a/openssl/include/internal/thread.h b/openssl/include/internal/thread.h deleted file mode 100644 index 8c5bad776..000000000 --- a/openssl/include/internal/thread.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_INTERNAL_THREAD_H -# define OPENSSL_INTERNAL_THREAD_H -# include -# include -# include -# include -# include -# include "crypto/context.h" - -void *ossl_crypto_thread_start(OSSL_LIB_CTX *ctx, CRYPTO_THREAD_ROUTINE start, - void *data); -int ossl_crypto_thread_join(void *task, CRYPTO_THREAD_RETVAL *retval); -int ossl_crypto_thread_clean(void *vhandle); -uint64_t ossl_get_avail_threads(OSSL_LIB_CTX *ctx); - -# if defined(OPENSSL_THREADS) - -# define OSSL_LIB_CTX_GET_THREADS(CTX) \ - ossl_lib_ctx_get_data(CTX, OSSL_LIB_CTX_THREAD_INDEX); - -typedef struct openssl_threads_st { - uint64_t max_threads; - uint64_t active_threads; - CRYPTO_MUTEX *lock; - CRYPTO_CONDVAR *cond_finished; -} OSSL_LIB_CTX_THREADS; - -# endif /* defined(OPENSSL_THREADS) */ - -#endif /* OPENSSL_INTERNAL_THREAD_H */ diff --git a/openssl/include/internal/thread_arch.h b/openssl/include/internal/thread_arch.h deleted file mode 100644 index aba9362e8..000000000 --- a/openssl/include/internal/thread_arch.h +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_THREAD_ARCH_H -# define OSSL_INTERNAL_THREAD_ARCH_H -# include -# include -# include "internal/time.h" - -# if defined(_WIN32) -# include -# endif - -# if defined(OPENSSL_THREADS) && defined(OPENSSL_SYS_UNIX) -# define OPENSSL_THREADS_POSIX -# elif defined(OPENSSL_THREADS) && defined(OPENSSL_SYS_VMS) -# define OPENSSL_THREADS_POSIX -# elif defined(OPENSSL_THREADS) && defined(OPENSSL_SYS_WINDOWS) && \ - defined(_WIN32_WINNT) -# if _WIN32_WINNT >= 0x0600 -# define OPENSSL_THREADS_WINNT -# elif _WIN32_WINNT >= 0x0501 -# define OPENSSL_THREADS_WINNT -# define OPENSSL_THREADS_WINNT_LEGACY -# else -# define OPENSSL_THREADS_NONE -# endif -# else -# define OPENSSL_THREADS_NONE -# endif - -# include - -typedef struct crypto_mutex_st CRYPTO_MUTEX; -typedef struct crypto_condvar_st CRYPTO_CONDVAR; - -CRYPTO_MUTEX *ossl_crypto_mutex_new(void); -void ossl_crypto_mutex_lock(CRYPTO_MUTEX *mutex); -int ossl_crypto_mutex_try_lock(CRYPTO_MUTEX *mutex); -void ossl_crypto_mutex_unlock(CRYPTO_MUTEX *mutex); -void ossl_crypto_mutex_free(CRYPTO_MUTEX **mutex); - -CRYPTO_CONDVAR *ossl_crypto_condvar_new(void); -void ossl_crypto_condvar_wait(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *mutex); -void ossl_crypto_condvar_wait_timeout(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *mutex, - OSSL_TIME deadline); -void ossl_crypto_condvar_broadcast(CRYPTO_CONDVAR *cv); -void ossl_crypto_condvar_signal(CRYPTO_CONDVAR *cv); -void ossl_crypto_condvar_free(CRYPTO_CONDVAR **cv); - -typedef uint32_t CRYPTO_THREAD_RETVAL; -typedef CRYPTO_THREAD_RETVAL (*CRYPTO_THREAD_ROUTINE)(void *); -typedef CRYPTO_THREAD_RETVAL (*CRYPTO_THREAD_ROUTINE_CB)(void *, - void (**)(void *), - void **); - -# define CRYPTO_THREAD_NO_STATE 0UL -# define CRYPTO_THREAD_FINISHED (1UL << 0) -# define CRYPTO_THREAD_JOIN_AWAIT (1UL << 1) -# define CRYPTO_THREAD_JOINED (1UL << 2) - -# define CRYPTO_THREAD_GET_STATE(THREAD, FLAG) ((THREAD)->state & (FLAG)) -# define CRYPTO_THREAD_GET_ERROR(THREAD, FLAG) (((THREAD)->state >> 16) & (FLAG)) - -typedef struct crypto_thread_st { - uint32_t state; - void *data; - CRYPTO_THREAD_ROUTINE routine; - CRYPTO_THREAD_RETVAL retval; - void *handle; - CRYPTO_MUTEX *lock; - CRYPTO_MUTEX *statelock; - CRYPTO_CONDVAR *condvar; - unsigned long thread_id; - int joinable; - OSSL_LIB_CTX *ctx; -} CRYPTO_THREAD; - -# if defined(OPENSSL_THREADS) - -# define CRYPTO_THREAD_UNSET_STATE(THREAD, FLAG) \ - do { \ - (THREAD)->state &= ~(FLAG); \ - } while ((void)0, 0) - -# define CRYPTO_THREAD_SET_STATE(THREAD, FLAG) \ - do { \ - (THREAD)->state |= (FLAG); \ - } while ((void)0, 0) - -# define CRYPTO_THREAD_SET_ERROR(THREAD, FLAG) \ - do { \ - (THREAD)->state |= ((FLAG) << 16); \ - } while ((void)0, 0) - -# define CRYPTO_THREAD_UNSET_ERROR(THREAD, FLAG) \ - do { \ - (THREAD)->state &= ~((FLAG) << 16); \ - } while ((void)0, 0) - -# else - -# define CRYPTO_THREAD_UNSET_STATE(THREAD, FLAG) -# define CRYPTO_THREAD_SET_STATE(THREAD, FLAG) -# define CRYPTO_THREAD_SET_ERROR(THREAD, FLAG) -# define CRYPTO_THREAD_UNSET_ERROR(THREAD, FLAG) - -# endif /* defined(OPENSSL_THREADS) */ - -CRYPTO_THREAD * ossl_crypto_thread_native_start(CRYPTO_THREAD_ROUTINE routine, - void *data, int joinable); -int ossl_crypto_thread_native_spawn(CRYPTO_THREAD *thread); -int ossl_crypto_thread_native_join(CRYPTO_THREAD *thread, - CRYPTO_THREAD_RETVAL *retval); -int ossl_crypto_thread_native_perform_join(CRYPTO_THREAD *thread, - CRYPTO_THREAD_RETVAL *retval); -int ossl_crypto_thread_native_exit(void); -int ossl_crypto_thread_native_is_self(CRYPTO_THREAD *thread); -int ossl_crypto_thread_native_clean(CRYPTO_THREAD *thread); - -#endif /* OSSL_INTERNAL_THREAD_ARCH_H */ diff --git a/openssl/include/internal/time.h b/openssl/include/internal/time.h deleted file mode 100644 index 14d724abf..000000000 --- a/openssl/include/internal/time.h +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_INTERNAL_TIME_H -# define OSSL_INTERNAL_TIME_H -# pragma once - -# include /* uint64_t */ -# include "internal/e_os.h" /* for struct timeval */ -# include "internal/safe_math.h" - -/* - * Internal type defining a time. - * This should be treated as an opaque structure. - * - * The time datum is Unix's 1970 and at nanosecond precision, this gives - * a range of 584 years roughly. - */ -typedef struct { - uint64_t t; /* Ticks since the epoch */ -} OSSL_TIME; - -/* The precision of times allows this many values per second */ -# define OSSL_TIME_SECOND ((uint64_t)1000000000) - -/* One millisecond. */ -# define OSSL_TIME_MS (OSSL_TIME_SECOND / 1000) - -/* One microsecond. */ -# define OSSL_TIME_US (OSSL_TIME_MS / 1000) - -/* One nanosecond. */ -# define OSSL_TIME_NS (OSSL_TIME_US / 1000) - -#define ossl_seconds2time(s) ossl_ticks2time((s) * OSSL_TIME_SECOND) -#define ossl_time2seconds(t) (ossl_time2ticks(t) / OSSL_TIME_SECOND) -#define ossl_ms2time(ms) ossl_ticks2time((ms) * OSSL_TIME_MS) -#define ossl_time2ms(t) (ossl_time2ticks(t) / OSSL_TIME_MS) -#define ossl_us2time(us) ossl_ticks2time((us) * OSSL_TIME_US) -#define ossl_time2us(t) (ossl_time2ticks(t) / OSSL_TIME_US) - -/* - * Arithmetic operations on times. - * These operations are saturating, in that an overflow or underflow returns - * the largest or smallest value respectively. - */ -OSSL_SAFE_MATH_UNSIGNED(time, uint64_t) - -/* Convert a tick count into a time */ -static ossl_unused ossl_inline -OSSL_TIME ossl_ticks2time(uint64_t ticks) -{ - OSSL_TIME r; - - r.t = ticks; - return r; -} - -/* Convert a time to a tick count */ -static ossl_unused ossl_inline -uint64_t ossl_time2ticks(OSSL_TIME t) -{ - return t.t; -} - -/* Get current time */ -OSSL_TIME ossl_time_now(void); - -/* The beginning and end of the time range */ -static ossl_unused ossl_inline -OSSL_TIME ossl_time_zero(void) -{ - return ossl_ticks2time(0); -} - -static ossl_unused ossl_inline -OSSL_TIME ossl_time_infinite(void) -{ - return ossl_ticks2time(~(uint64_t)0); -} - - -/* Convert time to timeval */ -static ossl_unused ossl_inline -struct timeval ossl_time_to_timeval(OSSL_TIME t) -{ - struct timeval tv; - int err = 0; - - /* - * Round up any nano secs which struct timeval doesn't support. Ensures that - * we never return a zero time if the input time is non zero - */ - t.t = safe_add_time(t.t, OSSL_TIME_US - 1, &err); - if (err) - t = ossl_time_infinite(); - -#ifdef _WIN32 - tv.tv_sec = (long int)(t.t / OSSL_TIME_SECOND); -#else - tv.tv_sec = (time_t)(t.t / OSSL_TIME_SECOND); -#endif - tv.tv_usec = (t.t % OSSL_TIME_SECOND) / OSSL_TIME_US; - return tv; -} - -/* Convert timeval to time */ -static ossl_unused ossl_inline -OSSL_TIME ossl_time_from_timeval(struct timeval tv) -{ - OSSL_TIME t; - -#ifndef __DJGPP__ /* tv_sec is unsigned on djgpp. */ - if (tv.tv_sec < 0) - return ossl_time_zero(); -#endif - t.t = tv.tv_sec * OSSL_TIME_SECOND + tv.tv_usec * OSSL_TIME_US; - return t; -} - -/* Convert OSSL_TIME to time_t */ -static ossl_unused ossl_inline -time_t ossl_time_to_time_t(OSSL_TIME t) -{ - return (time_t)(t.t / OSSL_TIME_SECOND); -} - -/* Convert time_t to OSSL_TIME */ -static ossl_unused ossl_inline -OSSL_TIME ossl_time_from_time_t(time_t t) -{ - OSSL_TIME ot; - - ot.t = t; - ot.t *= OSSL_TIME_SECOND; - return ot; -} - -/* Compare two time values, return -1 if less, 1 if greater and 0 if equal */ -static ossl_unused ossl_inline -int ossl_time_compare(OSSL_TIME a, OSSL_TIME b) -{ - if (a.t > b.t) - return 1; - if (a.t < b.t) - return -1; - return 0; -} - -/* Returns true if an OSSL_TIME is ossl_time_zero(). */ -static ossl_unused ossl_inline -int ossl_time_is_zero(OSSL_TIME t) -{ - return ossl_time_compare(t, ossl_time_zero()) == 0; -} - -/* Returns true if an OSSL_TIME is ossl_time_infinite(). */ -static ossl_unused ossl_inline -int ossl_time_is_infinite(OSSL_TIME t) -{ - return ossl_time_compare(t, ossl_time_infinite()) == 0; -} - -static ossl_unused ossl_inline -OSSL_TIME ossl_time_add(OSSL_TIME a, OSSL_TIME b) -{ - OSSL_TIME r; - int err = 0; - - r.t = safe_add_time(a.t, b.t, &err); - return err ? ossl_time_infinite() : r; -} - -static ossl_unused ossl_inline -OSSL_TIME ossl_time_subtract(OSSL_TIME a, OSSL_TIME b) -{ - OSSL_TIME r; - int err = 0; - - r.t = safe_sub_time(a.t, b.t, &err); - return err ? ossl_time_zero() : r; -} - -/* Returns |a - b|. */ -static ossl_unused ossl_inline -OSSL_TIME ossl_time_abs_difference(OSSL_TIME a, OSSL_TIME b) -{ - return a.t > b.t ? ossl_time_subtract(a, b) - : ossl_time_subtract(b, a); -} - -static ossl_unused ossl_inline -OSSL_TIME ossl_time_multiply(OSSL_TIME a, uint64_t b) -{ - OSSL_TIME r; - int err = 0; - - r.t = safe_mul_time(a.t, b, &err); - return err ? ossl_time_infinite() : r; -} - -static ossl_unused ossl_inline -OSSL_TIME ossl_time_divide(OSSL_TIME a, uint64_t b) -{ - OSSL_TIME r; - int err = 0; - - r.t = safe_div_time(a.t, b, &err); - return err ? ossl_time_zero() : r; -} - -static ossl_unused ossl_inline -OSSL_TIME ossl_time_muldiv(OSSL_TIME a, uint64_t b, uint64_t c) -{ - OSSL_TIME r; - int err = 0; - - r.t = safe_muldiv_time(a.t, b, c, &err); - return err ? ossl_time_zero() : r; -} - -/* Return higher of the two given time values. */ -static ossl_unused ossl_inline -OSSL_TIME ossl_time_max(OSSL_TIME a, OSSL_TIME b) -{ - return a.t > b.t ? a : b; -} - -/* Return the lower of the two given time values. */ -static ossl_unused ossl_inline -OSSL_TIME ossl_time_min(OSSL_TIME a, OSSL_TIME b) -{ - return a.t < b.t ? a : b; -} - -#endif diff --git a/openssl/include/internal/tlsgroups.h b/openssl/include/internal/tlsgroups.h index 73fb53bc5..9c975d4dd 100644 --- a/openssl/include/internal/tlsgroups.h +++ b/openssl/include/internal/tlsgroups.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -41,16 +41,7 @@ # define OSSL_TLS_GROUP_ID_brainpoolP512r1 0x001C # define OSSL_TLS_GROUP_ID_x25519 0x001D # define OSSL_TLS_GROUP_ID_x448 0x001E -# define OSSL_TLS_GROUP_ID_brainpoolP256r1_tls13 0x001F -# define OSSL_TLS_GROUP_ID_brainpoolP384r1_tls13 0x0020 -# define OSSL_TLS_GROUP_ID_brainpoolP512r1_tls13 0x0021 -# define OSSL_TLS_GROUP_ID_gc256A 0x0022 -# define OSSL_TLS_GROUP_ID_gc256B 0x0023 -# define OSSL_TLS_GROUP_ID_gc256C 0x0024 -# define OSSL_TLS_GROUP_ID_gc256D 0x0025 -# define OSSL_TLS_GROUP_ID_gc512A 0x0026 -# define OSSL_TLS_GROUP_ID_gc512B 0x0027 -# define OSSL_TLS_GROUP_ID_gc512C 0x0028 +# define OSSL_TLS_GROUP_ID_sm2 0x0029 # define OSSL_TLS_GROUP_ID_ffdhe2048 0x0100 # define OSSL_TLS_GROUP_ID_ffdhe3072 0x0101 # define OSSL_TLS_GROUP_ID_ffdhe4096 0x0102 diff --git a/openssl/include/internal/tsan_assist.h b/openssl/include/internal/tsan_assist.h index a840df0b2..60ecbd5f0 100644 --- a/openssl/include/internal/tsan_assist.h +++ b/openssl/include/internal/tsan_assist.h @@ -47,37 +47,35 @@ * can use TSAN_QUALIFIER in cast specifically when it has to count. */ -#ifndef OSSL_INTERNAL_TSAN_ASSIST_H -# define OSSL_INTERNAL_TSAN_ASSIST_H -# pragma once - -# if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ && !defined(__STDC_NO_ATOMICS__) -# include +# include -# if defined(ATOMIC_POINTER_LOCK_FREE) \ +# if defined(ATOMIC_POINTER_LOCK_FREE) \ && ATOMIC_POINTER_LOCK_FREE >= 2 -# define TSAN_QUALIFIER _Atomic -# define tsan_load(ptr) atomic_load_explicit((ptr), memory_order_relaxed) -# define tsan_store(ptr, val) atomic_store_explicit((ptr), (val), memory_order_relaxed) -# define tsan_add(ptr, n) atomic_fetch_add_explicit((ptr), (n), memory_order_relaxed) -# define tsan_ld_acq(ptr) atomic_load_explicit((ptr), memory_order_acquire) -# define tsan_st_rel(ptr, val) atomic_store_explicit((ptr), (val), memory_order_release) -# endif +# define TSAN_QUALIFIER _Atomic +# define tsan_load(ptr) atomic_load_explicit((ptr), memory_order_relaxed) +# define tsan_store(ptr, val) atomic_store_explicit((ptr), (val), memory_order_relaxed) +# define tsan_counter(ptr) atomic_fetch_add_explicit((ptr), 1, memory_order_relaxed) +# define tsan_decr(ptr) atomic_fetch_add_explicit((ptr), -1, memory_order_relaxed) +# define tsan_ld_acq(ptr) atomic_load_explicit((ptr), memory_order_acquire) +# define tsan_st_rel(ptr, val) atomic_store_explicit((ptr), (val), memory_order_release) +# endif -# elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) +#elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) -# if defined(__GCC_ATOMIC_POINTER_LOCK_FREE) \ +# if defined(__GCC_ATOMIC_POINTER_LOCK_FREE) \ && __GCC_ATOMIC_POINTER_LOCK_FREE >= 2 -# define TSAN_QUALIFIER volatile -# define tsan_load(ptr) __atomic_load_n((ptr), __ATOMIC_RELAXED) -# define tsan_store(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELAXED) -# define tsan_add(ptr, n) __atomic_fetch_add((ptr), (n), __ATOMIC_RELAXED) -# define tsan_ld_acq(ptr) __atomic_load_n((ptr), __ATOMIC_ACQUIRE) -# define tsan_st_rel(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELEASE) -# endif +# define TSAN_QUALIFIER volatile +# define tsan_load(ptr) __atomic_load_n((ptr), __ATOMIC_RELAXED) +# define tsan_store(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELAXED) +# define tsan_counter(ptr) __atomic_fetch_add((ptr), 1, __ATOMIC_RELAXED) +# define tsan_decr(ptr) __atomic_fetch_add((ptr), -1, __ATOMIC_RELAXED) +# define tsan_ld_acq(ptr) __atomic_load_n((ptr), __ATOMIC_ACQUIRE) +# define tsan_st_rel(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELEASE) +# endif -# elif defined(_MSC_VER) && _MSC_VER>=1200 \ +#elif defined(_MSC_VER) && _MSC_VER>=1200 \ && (defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \ defined(_M_ARM64) || (defined(_M_ARM) && _M_ARM >= 7 && !defined(_WIN32_WCE))) /* @@ -91,63 +89,62 @@ * with additional instructions and penalties, it kind of makes sense to * default to "iso"... */ -# define TSAN_QUALIFIER volatile -# if defined(_M_ARM) || defined(_M_ARM64) -# define _InterlockedExchangeAdd _InterlockedExchangeAdd_nf -# pragma intrinsic(_InterlockedExchangeAdd_nf) -# pragma intrinsic(__iso_volatile_load32, __iso_volatile_store32) -# ifdef _WIN64 -# define _InterlockedExchangeAdd64 _InterlockedExchangeAdd64_nf -# pragma intrinsic(_InterlockedExchangeAdd64_nf) -# pragma intrinsic(__iso_volatile_load64, __iso_volatile_store64) -# define tsan_load(ptr) (sizeof(*(ptr)) == 8 ? __iso_volatile_load64(ptr) \ +# define TSAN_QUALIFIER volatile +# if defined(_M_ARM) || defined(_M_ARM64) +# define _InterlockedExchangeAdd _InterlockedExchangeAdd_nf +# pragma intrinsic(_InterlockedExchangeAdd_nf) +# pragma intrinsic(__iso_volatile_load32, __iso_volatile_store32) +# ifdef _WIN64 +# define _InterlockedExchangeAdd64 _InterlockedExchangeAdd64_nf +# pragma intrinsic(_InterlockedExchangeAdd64_nf) +# pragma intrinsic(__iso_volatile_load64, __iso_volatile_store64) +# define tsan_load(ptr) (sizeof(*(ptr)) == 8 ? __iso_volatile_load64(ptr) \ : __iso_volatile_load32(ptr)) -# define tsan_store(ptr, val) (sizeof(*(ptr)) == 8 ? __iso_volatile_store64((ptr), (val)) \ +# define tsan_store(ptr, val) (sizeof(*(ptr)) == 8 ? __iso_volatile_store64((ptr), (val)) \ : __iso_volatile_store32((ptr), (val))) -# else -# define tsan_load(ptr) __iso_volatile_load32(ptr) -# define tsan_store(ptr, val) __iso_volatile_store32((ptr), (val)) -# endif -# else -# define tsan_load(ptr) (*(ptr)) -# define tsan_store(ptr, val) (*(ptr) = (val)) -# endif -# pragma intrinsic(_InterlockedExchangeAdd) -# ifdef _WIN64 -# pragma intrinsic(_InterlockedExchangeAdd64) -# define tsan_add(ptr, n) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), (n)) \ - : _InterlockedExchangeAdd((ptr), (n))) # else -# define tsan_add(ptr, n) _InterlockedExchangeAdd((ptr), (n)) -# endif -# if !defined(_ISO_VOLATILE) -# define tsan_ld_acq(ptr) (*(ptr)) -# define tsan_st_rel(ptr, val) (*(ptr) = (val)) +# define tsan_load(ptr) __iso_volatile_load32(ptr) +# define tsan_store(ptr, val) __iso_volatile_store32((ptr), (val)) # endif - +# else +# define tsan_load(ptr) (*(ptr)) +# define tsan_store(ptr, val) (*(ptr) = (val)) +# endif +# pragma intrinsic(_InterlockedExchangeAdd) +# ifdef _WIN64 +# pragma intrinsic(_InterlockedExchangeAdd64) +# define tsan_counter(ptr) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), 1) \ + : _InterlockedExchangeAdd((ptr), 1)) +# define tsan_decr(ptr) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), -1) \ + : _InterlockedExchangeAdd((ptr), -1)) +# else +# define tsan_counter(ptr) _InterlockedExchangeAdd((ptr), 1) +# define tsan_decr(ptr) _InterlockedExchangeAdd((ptr), -1) +# endif +# if !defined(_ISO_VOLATILE) +# define tsan_ld_acq(ptr) (*(ptr)) +# define tsan_st_rel(ptr, val) (*(ptr) = (val)) # endif -# ifndef TSAN_QUALIFIER +#endif -# ifdef OPENSSL_THREADS -# define TSAN_QUALIFIER volatile -# define TSAN_REQUIRES_LOCKING -# else /* OPENSSL_THREADS */ -# define TSAN_QUALIFIER -# endif /* OPENSSL_THREADS */ +#ifndef TSAN_QUALIFIER -# define tsan_load(ptr) (*(ptr)) -# define tsan_store(ptr, val) (*(ptr) = (val)) -# define tsan_add(ptr, n) (*(ptr) += (n)) +# ifdef OPENSSL_THREADS +# define TSAN_QUALIFIER volatile +# define TSAN_REQUIRES_LOCKING +# else /* OPENSSL_THREADS */ +# define TSAN_QUALIFIER +# endif /* OPENSSL_THREADS */ + +# define tsan_load(ptr) (*(ptr)) +# define tsan_store(ptr, val) (*(ptr) = (val)) +# define tsan_counter(ptr) ((*(ptr))++) +# define tsan_decr(ptr) ((*(ptr))--) /* * Lack of tsan_ld_acq and tsan_ld_rel means that compiler support is not * sophisticated enough to support them. Code that relies on them should be * protected with #ifdef tsan_ld_acq with locked fallback. */ -# endif - -# define tsan_counter(ptr) tsan_add((ptr), 1) -# define tsan_decr(ptr) tsan_add((ptr), -1) - #endif diff --git a/openssl/include/internal/uint_set.h b/openssl/include/internal/uint_set.h deleted file mode 100644 index dcb29b33f..000000000 --- a/openssl/include/internal/uint_set.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#ifndef OSSL_UINT_SET_H -# define OSSL_UINT_SET_H - -#include "openssl/params.h" -#include "internal/list.h" - -/* - * uint64_t Integer Sets - * ===================== - * - * Utilities for managing a logical set of unsigned 64-bit integers. The - * structure tracks each contiguous range of integers using one allocation and - * is thus optimised for cases where integers tend to appear consecutively. - * Queries are optimised under the assumption that they will generally be made - * on integers near the end of the set. - * - * Discussion of implementation details can be found in uint_set.c. - */ -typedef struct uint_range_st { - uint64_t start, end; -} UINT_RANGE; - -typedef struct uint_set_item_st UINT_SET_ITEM; -struct uint_set_item_st { - OSSL_LIST_MEMBER(uint_set, UINT_SET_ITEM); - UINT_RANGE range; -}; - -DEFINE_LIST_OF(uint_set, UINT_SET_ITEM); - -typedef OSSL_LIST(uint_set) UINT_SET; - -void ossl_uint_set_init(UINT_SET *s); -void ossl_uint_set_destroy(UINT_SET *s); - -/* - * Insert a range into a integer set. Returns 0 on allocation failure, in which - * case the integer set is in a valid but undefined state. Otherwise, returns 1. - * Ranges can overlap existing ranges without limitation. If a range is a subset - * of an existing range in the set, this is a no-op and returns 1. - */ -int ossl_uint_set_insert(UINT_SET *s, const UINT_RANGE *range); - -/* - * Remove a range from the set. Returns 0 on allocation failure, in which case - * the integer set is unchanged. Otherwise, returns 1. Ranges which are not - * already in the set can be removed without issue. If a passed range is not in - * the integer set at all, this is a no-op and returns 1. - */ -int ossl_uint_set_remove(UINT_SET *s, const UINT_RANGE *range); - -/* Returns 1 iff the given integer is in the integer set. */ -int ossl_uint_set_query(const UINT_SET *s, uint64_t v); - -#endif diff --git a/openssl/include/openssl/__DECC_INCLUDE_EPILOGUE.H b/openssl/include/openssl/__DECC_INCLUDE_EPILOGUE.H deleted file mode 100644 index d251d0a03..000000000 --- a/openssl/include/openssl/__DECC_INCLUDE_EPILOGUE.H +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is only used by HP C/C++ on VMS, and is included automatically - * after each header file from this directory - */ - -/* - * The C++ compiler doesn't understand these pragmas, even though it - * understands the corresponding command line qualifier. - */ -#ifndef __cplusplus -/* restore state. Must correspond to the save in __decc_include_prologue.h */ -# pragma names restore -#endif diff --git a/openssl/include/openssl/__DECC_INCLUDE_PROLOGUE.H b/openssl/include/openssl/__DECC_INCLUDE_PROLOGUE.H deleted file mode 100644 index 91ac6b33c..000000000 --- a/openssl/include/openssl/__DECC_INCLUDE_PROLOGUE.H +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is only used by HP C/C++ on VMS, and is included automatically - * after each header file from this directory - */ - -/* - * The C++ compiler doesn't understand these pragmas, even though it - * understands the corresponding command line qualifier. - */ -#ifndef __cplusplus -/* save state */ -# pragma names save -/* have the compiler shorten symbols larger than 31 chars to 23 chars - * followed by a 8 hex char CRC - */ -# pragma names as_is,shortened -#endif diff --git a/openssl/include/openssl/asn1.h b/openssl/include/openssl/asn1.h index a69601c13..859b070b8 100644 --- a/openssl/include/openssl/asn1.h +++ b/openssl/include/openssl/asn1.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/asn1.h.in * - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,9 +21,6 @@ # define HEADER_ASN1_H # endif -# ifndef OPENSSL_NO_STDIO -# include -# endif # include # include # include @@ -158,7 +155,7 @@ SKM_DEFINE_STACK_OF_INTERNAL(X509_ALGOR, X509_ALGOR, X509_ALGOR) -# define ASN1_STRING_FLAG_BITS_LEFT 0x08 /* Set if 0x07 has bits left value */ +# define ASN1_STRING_FLAG_BITS_LEFT 0x08/* Set if 0x07 has bits left value */ /* * This indicates that the ASN1_STRING is not a real value but just a place * holder for the location where indefinite length constructed data should be @@ -351,6 +348,17 @@ typedef struct ASN1_VALUE_st ASN1_VALUE; # define DECLARE_ASN1_PRINT_FUNCTION_fname(stname, fname) \ DECLARE_ASN1_PRINT_FUNCTION_fname_attr(extern, stname, fname) +/* Declare ASN1 functions with static limit: the implement macro in in asn1t.h */ + +# define DECLARE_STATIC_ASN1_FUNCTIONS(type) \ + DECLARE_ASN1_FUNCTIONS_attr(static ossl_unused, type) + +# define DECLARE_STATIC_ASN1_ALLOC_FUNCTIONS(type) \ + DECLARE_ASN1_ALLOC_FUNCTIONS_attr(static ossl_unused, type) + +# define DECLARE_STATIC_ASN1_ENCODE_FUNCTIONS_name(type, name) \ + DECLARE_ASN1_ENCODE_FUNCTIONS_name_attr(static, type, name) + # define D2I_OF(type) type *(*)(type **,const unsigned char **,long) # define I2D_OF(type) int (*)(const type *,unsigned char **) @@ -999,8 +1007,6 @@ int ASN1_TYPE_get_int_octetstring(const ASN1_TYPE *a, long *num, unsigned char *data, int max_len); void *ASN1_item_unpack(const ASN1_STRING *oct, const ASN1_ITEM *it); -void *ASN1_item_unpack_ex(const ASN1_STRING *oct, const ASN1_ITEM *it, - OSSL_LIB_CTX *libctx, const char *propq); ASN1_STRING *ASN1_item_pack(void *obj, const ASN1_ITEM *it, ASN1_OCTET_STRING **oct); diff --git a/openssl/include/openssl/asn1.h.in b/openssl/include/openssl/asn1.h.in index 798b22115..dcd3500f5 100644 --- a/openssl/include/openssl/asn1.h.in +++ b/openssl/include/openssl/asn1.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,9 +22,6 @@ use OpenSSL::stackhash qw(generate_stack_macros); # define HEADER_ASN1_H # endif -# ifndef OPENSSL_NO_STDIO -# include -# endif # include # include # include @@ -135,7 +132,7 @@ extern "C" { -} -# define ASN1_STRING_FLAG_BITS_LEFT 0x08 /* Set if 0x07 has bits left value */ +# define ASN1_STRING_FLAG_BITS_LEFT 0x08/* Set if 0x07 has bits left value */ /* * This indicates that the ASN1_STRING is not a real value but just a place * holder for the location where indefinite length constructed data should be @@ -304,6 +301,17 @@ typedef struct ASN1_VALUE_st ASN1_VALUE; # define DECLARE_ASN1_PRINT_FUNCTION_fname(stname, fname) \ DECLARE_ASN1_PRINT_FUNCTION_fname_attr(extern, stname, fname) +/* Declare ASN1 functions with static limit: the implement macro in in asn1t.h */ + +# define DECLARE_STATIC_ASN1_FUNCTIONS(type) \ + DECLARE_ASN1_FUNCTIONS_attr(static ossl_unused, type) + +# define DECLARE_STATIC_ASN1_ALLOC_FUNCTIONS(type) \ + DECLARE_ASN1_ALLOC_FUNCTIONS_attr(static ossl_unused, type) + +# define DECLARE_STATIC_ASN1_ENCODE_FUNCTIONS_name(type, name) \ + DECLARE_ASN1_ENCODE_FUNCTIONS_name_attr(static, type, name) + # define D2I_OF(type) type *(*)(type **,const unsigned char **,long) # define I2D_OF(type) int (*)(const type *,unsigned char **) @@ -832,8 +840,6 @@ int ASN1_TYPE_get_int_octetstring(const ASN1_TYPE *a, long *num, unsigned char *data, int max_len); void *ASN1_item_unpack(const ASN1_STRING *oct, const ASN1_ITEM *it); -void *ASN1_item_unpack_ex(const ASN1_STRING *oct, const ASN1_ITEM *it, - OSSL_LIB_CTX *libctx, const char *propq); ASN1_STRING *ASN1_item_pack(void *obj, const ASN1_ITEM *it, ASN1_OCTET_STRING **oct); diff --git a/openssl/include/openssl/asn1err.h b/openssl/include/openssl/asn1err.h index 8fd85ed88..d4276220c 100644 --- a/openssl/include/openssl/asn1err.h +++ b/openssl/include/openssl/asn1err.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -47,7 +47,6 @@ # define ASN1_R_EXPLICIT_TAG_NOT_CONSTRUCTED 120 # define ASN1_R_FIELD_MISSING 121 # define ASN1_R_FIRST_NUM_TOO_LARGE 122 -# define ASN1_R_GENERALIZEDTIME_IS_TOO_SHORT 232 # define ASN1_R_HEADER_TOO_LONG 123 # define ASN1_R_ILLEGAL_BITSTRING_FORMAT 175 # define ASN1_R_ILLEGAL_BOOLEAN 176 @@ -134,7 +133,6 @@ # define ASN1_R_UNSUPPORTED_CIPHER 228 # define ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE 167 # define ASN1_R_UNSUPPORTED_TYPE 196 -# define ASN1_R_UTCTIME_IS_TOO_SHORT 233 # define ASN1_R_WRONG_INTEGER_TYPE 225 # define ASN1_R_WRONG_PUBLIC_KEY_TYPE 200 # define ASN1_R_WRONG_TAG 168 diff --git a/openssl/include/openssl/asn1t.h b/openssl/include/openssl/asn1t.h index 7558f4202..faefb3412 100644 --- a/openssl/include/openssl/asn1t.h +++ b/openssl/include/openssl/asn1t.h @@ -940,6 +940,23 @@ int ASN1_item_ex_i2d(const ASN1_VALUE **pval, unsigned char **out, # define IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(stname, itname, fname) \ IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(stname, itname, fname) +# define IMPLEMENT_ASN1_ENCODE_FUNCTIONS_pfname(pre, stname, itname, fname) \ + pre stname *d2i_##fname(stname **a, const unsigned char **in, long len) \ + { \ + return (stname *)ASN1_item_d2i((ASN1_VALUE **)a, in, len, ASN1_ITEM_rptr(itname));\ + } \ + pre int i2d_##fname(const stname *a, unsigned char **out) \ + { \ + return ASN1_item_i2d((const ASN1_VALUE *)a, out, ASN1_ITEM_rptr(itname));\ + } + +# define IMPLEMENT_ASN1_FUNCTIONS_pfname(pre, stname, itname, fname) \ + IMPLEMENT_ASN1_ENCODE_FUNCTIONS_pfname(pre, stname, itname, fname) \ + IMPLEMENT_ASN1_ALLOC_FUNCTIONS_pfname(pre, stname, itname, fname) +# define IMPLEMENT_STATIC_ASN1_FUNCTIONS(stname) IMPLEMENT_ASN1_FUNCTIONS_pfname(static, stname, stname, stname) +# define IMPLEMENT_STATIC_ASN1_ENCODE_FUNCTIONS_fname(stname, itname, fname) \ + IMPLEMENT_ASN1_ENCODE_FUNCTIONS_pfname(static, stname, itname, fname) + #ifdef __cplusplus } #endif diff --git a/openssl/include/openssl/asn1t.h.in b/openssl/include/openssl/asn1t.h.in index b536fe51a..75ffbaeff 100644 --- a/openssl/include/openssl/asn1t.h.in +++ b/openssl/include/openssl/asn1t.h.in @@ -917,6 +917,23 @@ int ASN1_item_ex_i2d(const ASN1_VALUE **pval, unsigned char **out, # define IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(stname, itname, fname) \ IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(stname, itname, fname) +# define IMPLEMENT_ASN1_ENCODE_FUNCTIONS_pfname(pre, stname, itname, fname) \ + pre stname *d2i_##fname(stname **a, const unsigned char **in, long len) \ + { \ + return (stname *)ASN1_item_d2i((ASN1_VALUE **)a, in, len, ASN1_ITEM_rptr(itname));\ + } \ + pre int i2d_##fname(const stname *a, unsigned char **out) \ + { \ + return ASN1_item_i2d((const ASN1_VALUE *)a, out, ASN1_ITEM_rptr(itname));\ + } + +# define IMPLEMENT_ASN1_FUNCTIONS_pfname(pre, stname, itname, fname) \ + IMPLEMENT_ASN1_ENCODE_FUNCTIONS_pfname(pre, stname, itname, fname) \ + IMPLEMENT_ASN1_ALLOC_FUNCTIONS_pfname(pre, stname, itname, fname) +# define IMPLEMENT_STATIC_ASN1_FUNCTIONS(stname) IMPLEMENT_ASN1_FUNCTIONS_pfname(static, stname, stname, stname) +# define IMPLEMENT_STATIC_ASN1_ENCODE_FUNCTIONS_fname(stname, itname, fname) \ + IMPLEMENT_ASN1_ENCODE_FUNCTIONS_pfname(static, stname, itname, fname) + #ifdef __cplusplus } #endif diff --git a/openssl/include/openssl/async.h b/openssl/include/openssl/async.h index 826ffb993..bc27d5db0 100644 --- a/openssl/include/openssl/async.h +++ b/openssl/include/openssl/async.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2018 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -80,14 +80,6 @@ int ASYNC_WAIT_CTX_clear_fd(ASYNC_WAIT_CTX *ctx, const void *key); int ASYNC_is_capable(void); -typedef void *(*ASYNC_stack_alloc_fn)(size_t *num); -typedef void (*ASYNC_stack_free_fn)(void *addr); - -int ASYNC_set_mem_functions(ASYNC_stack_alloc_fn alloc_fn, - ASYNC_stack_free_fn free_fn); -void ASYNC_get_mem_functions(ASYNC_stack_alloc_fn *alloc_fn, - ASYNC_stack_free_fn *free_fn); - int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *ctx, int *ret, int (*func)(void *), void *args, size_t size); int ASYNC_pause_job(void); diff --git a/openssl/include/openssl/bio.h b/openssl/include/openssl/bio.h index 7ec832c80..05e1e1988 100644 --- a/openssl/include/openssl/bio.h +++ b/openssl/include/openssl/bio.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/bio.h.in * - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -67,13 +67,8 @@ extern "C" { # define BIO_TYPE_DGRAM_SCTP (24|BIO_TYPE_SOURCE_SINK|BIO_TYPE_DESCRIPTOR) # endif # define BIO_TYPE_CORE_TO_PROV (25|BIO_TYPE_SOURCE_SINK) -# define BIO_TYPE_DGRAM_PAIR (26|BIO_TYPE_SOURCE_SINK) -# define BIO_TYPE_DGRAM_MEM (27|BIO_TYPE_SOURCE_SINK) -/* Custom type starting index returned by BIO_get_new_index() */ #define BIO_TYPE_START 128 -/* Custom type maximum index that can be returned by BIO_get_new_index() */ -#define BIO_TYPE_MASK 0xFF /* * BIO_FILENAME_READ|BIO_CLOSE to open or close on free. @@ -176,35 +171,11 @@ extern "C" { # define BIO_CTRL_SET_INDENT 80 # define BIO_CTRL_GET_INDENT 81 -# define BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP 82 -# define BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE 83 -# define BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE 84 -# define BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS 85 -# define BIO_CTRL_DGRAM_GET_CAPS 86 -# define BIO_CTRL_DGRAM_SET_CAPS 87 -# define BIO_CTRL_DGRAM_GET_NO_TRUNC 88 -# define BIO_CTRL_DGRAM_SET_NO_TRUNC 89 - -/* - * internal BIO: - * # define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90 - */ - -# define BIO_CTRL_GET_RPOLL_DESCRIPTOR 91 -# define BIO_CTRL_GET_WPOLL_DESCRIPTOR 92 -# define BIO_CTRL_DGRAM_DETECT_PEER_ADDR 93 - -# define BIO_DGRAM_CAP_NONE 0U -# define BIO_DGRAM_CAP_HANDLES_SRC_ADDR (1U << 0) -# define BIO_DGRAM_CAP_HANDLES_DST_ADDR (1U << 1) -# define BIO_DGRAM_CAP_PROVIDES_SRC_ADDR (1U << 2) -# define BIO_DGRAM_CAP_PROVIDES_DST_ADDR (1U << 3) - # ifndef OPENSSL_NO_KTLS # define BIO_get_ktls_send(b) \ - (BIO_ctrl(b, BIO_CTRL_GET_KTLS_SEND, 0, NULL) > 0) + BIO_ctrl(b, BIO_CTRL_GET_KTLS_SEND, 0, NULL) # define BIO_get_ktls_recv(b) \ - (BIO_ctrl(b, BIO_CTRL_GET_KTLS_RECV, 0, NULL) > 0) + BIO_ctrl(b, BIO_CTRL_GET_KTLS_RECV, 0, NULL) # else # define BIO_get_ktls_send(b) (0) # define BIO_get_ktls_recv(b) (0) @@ -237,7 +208,7 @@ extern "C" { # define BIO_FLAGS_NONCLEAR_RST 0x400 # define BIO_FLAGS_IN_EOF 0x800 -/* the BIO FLAGS values 0x1000 to 0x8000 are reserved for internal KTLS flags */ +/* the BIO FLAGS values 0x1000 to 0x4000 are reserved for internal KTLS flags */ typedef union bio_addr_st BIO_ADDR; typedef struct bio_addrinfo_st BIO_ADDRINFO; @@ -284,15 +255,17 @@ void BIO_clear_flags(BIO *b, int flags); /* Returned from the accept BIO when an accept would have blocked */ # define BIO_RR_ACCEPT 0x03 +# ifndef OPENSSL_NO_SESSION_LOOKUP +# define BIO_RR_SSL_SESSION_LOOKUP 0x04 +# endif + /* These are passed by the BIO callback */ -# define BIO_CB_FREE 0x01 -# define BIO_CB_READ 0x02 -# define BIO_CB_WRITE 0x03 -# define BIO_CB_PUTS 0x04 -# define BIO_CB_GETS 0x05 -# define BIO_CB_CTRL 0x06 -# define BIO_CB_RECVMMSG 0x07 -# define BIO_CB_SENDMMSG 0x08 +# define BIO_CB_FREE 0x01 +# define BIO_CB_READ 0x02 +# define BIO_CB_WRITE 0x03 +# define BIO_CB_PUTS 0x04 +# define BIO_CB_GETS 0x05 +# define BIO_CB_CTRL 0x06 /* * The callback is called before and after the underling operation, The @@ -393,36 +366,6 @@ struct bio_dgram_sctp_prinfo { }; # endif -/* BIO_sendmmsg/BIO_recvmmsg-related definitions */ -typedef struct bio_msg_st { - void *data; - size_t data_len; - BIO_ADDR *peer, *local; - uint64_t flags; -} BIO_MSG; - -typedef struct bio_mmsg_cb_args_st { - BIO_MSG *msg; - size_t stride, num_msg; - uint64_t flags; - size_t *msgs_processed; -} BIO_MMSG_CB_ARGS; - -#define BIO_POLL_DESCRIPTOR_TYPE_NONE 0 -#define BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD 1 -#define BIO_POLL_DESCRIPTOR_TYPE_SSL 2 -#define BIO_POLL_DESCRIPTOR_CUSTOM_START 8192 - -typedef struct bio_poll_descriptor_st { - uint32_t type; - union { - int fd; - void *custom; - uintptr_t custom_ui; - SSL *ssl; - } value; -} BIO_POLL_DESCRIPTOR; - /* * #define BIO_CONN_get_param_hostname BIO_ctrl */ @@ -489,17 +432,10 @@ typedef struct bio_poll_descriptor_st { # define BIO_C_SET_CONNECT_MODE 155 -# define BIO_C_SET_TFO 156 /* like BIO_C_SET_NBIO */ - -# define BIO_C_SET_SOCK_TYPE 157 -# define BIO_C_GET_SOCK_TYPE 158 -# define BIO_C_GET_DGRAM_BIO 159 - # define BIO_set_app_data(s,arg) BIO_set_ex_data(s,0,arg) # define BIO_get_app_data(s) BIO_get_ex_data(s,0) -# define BIO_set_nbio(b,n) BIO_ctrl(b,BIO_C_SET_NBIO,(n),NULL) -# define BIO_set_tfo(b,n) BIO_ctrl(b,BIO_C_SET_TFO,(n),NULL) +# define BIO_set_nbio(b,n) BIO_ctrl(b,BIO_C_SET_NBIO,(n),NULL) # ifndef OPENSSL_NO_SOCK /* IP families we support, for BIO_s_connect() and BIO_s_accept() */ @@ -520,11 +456,7 @@ typedef struct bio_poll_descriptor_st { # define BIO_get_conn_port(b) ((const char *)BIO_ptr_ctrl(b,BIO_C_GET_CONNECT,1)) # define BIO_get_conn_address(b) ((const BIO_ADDR *)BIO_ptr_ctrl(b,BIO_C_GET_CONNECT,2)) # define BIO_get_conn_ip_family(b) BIO_ctrl(b,BIO_C_GET_CONNECT,3,NULL) -# define BIO_get_conn_mode(b) BIO_ctrl(b,BIO_C_GET_CONNECT,4,NULL) # define BIO_set_conn_mode(b,n) BIO_ctrl(b,BIO_C_SET_CONNECT_MODE,(n),NULL) -# define BIO_set_sock_type(b,t) BIO_ctrl(b,BIO_C_SET_SOCK_TYPE,(t),NULL) -# define BIO_get_sock_type(b) BIO_ctrl(b,BIO_C_GET_SOCK_TYPE,0,NULL) -# define BIO_get0_dgram_bio(b, p) BIO_ctrl(b,BIO_C_GET_DGRAM_BIO,0,(void *)(BIO **)(p)) /* BIO_s_accept() */ # define BIO_set_accept_name(b,name) BIO_ctrl(b,BIO_C_SET_ACCEPT,0, \ @@ -541,7 +473,6 @@ typedef struct bio_poll_descriptor_st { (char *)(bio)) # define BIO_set_accept_ip_family(b,f) BIO_int_ctrl(b,BIO_C_SET_ACCEPT,4,f) # define BIO_get_accept_ip_family(b) BIO_ctrl(b,BIO_C_GET_ACCEPT,4,NULL) -# define BIO_set_tfo_accept(b,n) BIO_ctrl(b,BIO_C_SET_ACCEPT,5,(n)?(void *)"a":NULL) /* Aliases kept for backward compatibility */ # define BIO_BIND_NORMAL 0 @@ -669,30 +600,8 @@ int BIO_ctrl_reset_read_request(BIO *b); (int)BIO_ctrl(b, BIO_CTRL_DGRAM_GET_PEER, 0, (char *)(peer)) # define BIO_dgram_set_peer(b,peer) \ (int)BIO_ctrl(b, BIO_CTRL_DGRAM_SET_PEER, 0, (char *)(peer)) -# define BIO_dgram_detect_peer_addr(b,peer) \ - (int)BIO_ctrl(b, BIO_CTRL_DGRAM_DETECT_PEER_ADDR, 0, (char *)(peer)) # define BIO_dgram_get_mtu_overhead(b) \ (unsigned int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_MTU_OVERHEAD, 0, NULL) -# define BIO_dgram_get_local_addr_cap(b) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP, 0, NULL) -# define BIO_dgram_get_local_addr_enable(b, penable) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE, 0, (char *)(penable)) -# define BIO_dgram_set_local_addr_enable(b, enable) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE, (enable), NULL) -# define BIO_dgram_get_effective_caps(b) \ - (uint32_t)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS, 0, NULL) -# define BIO_dgram_get_caps(b) \ - (uint32_t)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_CAPS, 0, NULL) -# define BIO_dgram_set_caps(b, caps) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_SET_CAPS, (long)(caps), NULL) -# define BIO_dgram_get_no_trunc(b) \ - (unsigned int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_NO_TRUNC, 0, NULL) -# define BIO_dgram_set_no_trunc(b, enable) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_SET_NO_TRUNC, (enable), NULL) -# define BIO_dgram_get_mtu(b) \ - (unsigned int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_MTU, 0, NULL) -# define BIO_dgram_set_mtu(b, mtu) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_SET_MTU, (mtu), NULL) /* ctrl macros for BIO_f_prefix */ # define BIO_set_prefix(b,p) BIO_ctrl((b), BIO_CTRL_SET_PREFIX, 0, (void *)(p)) @@ -735,18 +644,10 @@ void BIO_vfree(BIO *a); int BIO_up_ref(BIO *a); int BIO_read(BIO *b, void *data, int dlen); int BIO_read_ex(BIO *b, void *data, size_t dlen, size_t *readbytes); -__owur int BIO_recvmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, uint64_t flags, - size_t *msgs_processed); int BIO_gets(BIO *bp, char *buf, int size); int BIO_get_line(BIO *bio, char *buf, int size); int BIO_write(BIO *b, const void *data, int dlen); int BIO_write_ex(BIO *b, const void *data, size_t dlen, size_t *written); -__owur int BIO_sendmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, uint64_t flags, - size_t *msgs_processed); -__owur int BIO_get_rpoll_descriptor(BIO *b, BIO_POLL_DESCRIPTOR *desc); -__owur int BIO_get_wpoll_descriptor(BIO *b, BIO_POLL_DESCRIPTOR *desc); int BIO_puts(BIO *bp, const char *buf); int BIO_indent(BIO *b, int indent, int max); long BIO_ctrl(BIO *bp, int cmd, long larg, void *parg); @@ -770,9 +671,6 @@ int BIO_nwrite0(BIO *bio, char **buf); int BIO_nwrite(BIO *bio, char **buf, int num); const BIO_METHOD *BIO_s_mem(void); -# ifndef OPENSSL_NO_DGRAM -const BIO_METHOD *BIO_s_dgram_mem(void); -# endif const BIO_METHOD *BIO_s_secmem(void); BIO *BIO_new_mem_buf(const void *buf, int len); # ifndef OPENSSL_NO_SOCK @@ -792,7 +690,6 @@ const BIO_METHOD *BIO_f_nbio_test(void); const BIO_METHOD *BIO_f_prefix(void); const BIO_METHOD *BIO_s_core(void); # ifndef OPENSSL_NO_DGRAM -const BIO_METHOD *BIO_s_dgram_pair(void); const BIO_METHOD *BIO_s_datagram(void); int BIO_dgram_non_fatal_error(int error); BIO *BIO_new_dgram(int fd, int close_flag); @@ -811,7 +708,6 @@ int BIO_dgram_sctp_msg_waiting(BIO *b); # ifndef OPENSSL_NO_SOCK int BIO_sock_should_retry(int i); int BIO_sock_non_fatal_error(int error); -int BIO_err_is_non_fatal(unsigned int errcode); int BIO_socket_wait(int fd, int for_read, time_t max_time); # endif int BIO_wait(BIO *bio, time_t max_time, unsigned int nap_milliseconds); @@ -834,8 +730,6 @@ int BIO_hex_string(BIO *out, int indent, int width, const void *data, # ifndef OPENSSL_NO_SOCK BIO_ADDR *BIO_ADDR_new(void); -int BIO_ADDR_copy(BIO_ADDR *dst, const BIO_ADDR *src); -BIO_ADDR *BIO_ADDR_dup(const BIO_ADDR *ap); int BIO_ADDR_rawmake(BIO_ADDR *ap, int family, const void *where, size_t wherelen, unsigned short port); void BIO_ADDR_free(BIO_ADDR *); @@ -898,7 +792,6 @@ int BIO_sock_info(int sock, # define BIO_SOCK_KEEPALIVE 0x04 # define BIO_SOCK_NONBLOCK 0x08 # define BIO_SOCK_NODELAY 0x10 -# define BIO_SOCK_TFO 0x20 int BIO_socket(int domain, int socktype, int protocol, int options); int BIO_connect(int sock, const BIO_ADDR *addr, int options); @@ -916,11 +809,6 @@ BIO *BIO_new_fd(int fd, int close_flag); int BIO_new_bio_pair(BIO **bio1, size_t writebuf1, BIO **bio2, size_t writebuf2); -# ifndef OPENSSL_NO_DGRAM -int BIO_new_bio_dgram_pair(BIO **bio1, size_t writebuf1, - BIO **bio2, size_t writebuf2); -# endif - /* * If successful, returns 1 and in *bio1, *bio2 two BIO pair endpoints. * Otherwise returns 0 and sets *bio1 and *bio2 to NULL. Size 0 uses default @@ -935,7 +823,6 @@ void BIO_copy_next_retry(BIO *b); # define ossl_bio__attr__(x) # if defined(__GNUC__) && defined(__STDC_VERSION__) \ - && !defined(__MINGW32__) && !defined(__MINGW64__) \ && !defined(__APPLE__) /* * Because we support the 'z' modifier, which made its appearance in C99, @@ -972,30 +859,18 @@ int BIO_meth_set_write(BIO_METHOD *biom, int (*write) (BIO *, const char *, int)); int BIO_meth_set_write_ex(BIO_METHOD *biom, int (*bwrite) (BIO *, const char *, size_t, size_t *)); -int BIO_meth_set_sendmmsg(BIO_METHOD *biom, - int (*f) (BIO *, BIO_MSG *, size_t, size_t, - uint64_t, size_t *)); -int (*BIO_meth_get_sendmmsg(const BIO_METHOD *biom))(BIO *, BIO_MSG *, - size_t, size_t, - uint64_t, size_t *); int (*BIO_meth_get_read(const BIO_METHOD *biom)) (BIO *, char *, int); int (*BIO_meth_get_read_ex(const BIO_METHOD *biom)) (BIO *, char *, size_t, size_t *); int BIO_meth_set_read(BIO_METHOD *biom, int (*read) (BIO *, char *, int)); int BIO_meth_set_read_ex(BIO_METHOD *biom, int (*bread) (BIO *, char *, size_t, size_t *)); -int BIO_meth_set_recvmmsg(BIO_METHOD *biom, - int (*f) (BIO *, BIO_MSG *, size_t, size_t, - uint64_t, size_t *)); -int (*BIO_meth_get_recvmmsg(const BIO_METHOD *biom))(BIO *, BIO_MSG *, - size_t, size_t, - uint64_t, size_t *); int (*BIO_meth_get_puts(const BIO_METHOD *biom)) (BIO *, const char *); int BIO_meth_set_puts(BIO_METHOD *biom, int (*puts) (BIO *, const char *)); int (*BIO_meth_get_gets(const BIO_METHOD *biom)) (BIO *, char *, int); int BIO_meth_set_gets(BIO_METHOD *biom, - int (*ossl_gets) (BIO *, char *, int)); + int (*gets) (BIO *, char *, int)); long (*BIO_meth_get_ctrl(const BIO_METHOD *biom)) (BIO *, int, long, void *); int BIO_meth_set_ctrl(BIO_METHOD *biom, long (*ctrl) (BIO *, int, long, void *)); diff --git a/openssl/include/openssl/bio.h.in b/openssl/include/openssl/bio.h.in index 440d838e6..04c5ba302 100644 --- a/openssl/include/openssl/bio.h.in +++ b/openssl/include/openssl/bio.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -68,13 +68,8 @@ extern "C" { # define BIO_TYPE_DGRAM_SCTP (24|BIO_TYPE_SOURCE_SINK|BIO_TYPE_DESCRIPTOR) # endif # define BIO_TYPE_CORE_TO_PROV (25|BIO_TYPE_SOURCE_SINK) -# define BIO_TYPE_DGRAM_PAIR (26|BIO_TYPE_SOURCE_SINK) -# define BIO_TYPE_DGRAM_MEM (27|BIO_TYPE_SOURCE_SINK) -/* Custom type starting index returned by BIO_get_new_index() */ #define BIO_TYPE_START 128 -/* Custom type maximum index that can be returned by BIO_get_new_index() */ -#define BIO_TYPE_MASK 0xFF /* * BIO_FILENAME_READ|BIO_CLOSE to open or close on free. @@ -177,35 +172,11 @@ extern "C" { # define BIO_CTRL_SET_INDENT 80 # define BIO_CTRL_GET_INDENT 81 -# define BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP 82 -# define BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE 83 -# define BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE 84 -# define BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS 85 -# define BIO_CTRL_DGRAM_GET_CAPS 86 -# define BIO_CTRL_DGRAM_SET_CAPS 87 -# define BIO_CTRL_DGRAM_GET_NO_TRUNC 88 -# define BIO_CTRL_DGRAM_SET_NO_TRUNC 89 - -/* - * internal BIO: - * # define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90 - */ - -# define BIO_CTRL_GET_RPOLL_DESCRIPTOR 91 -# define BIO_CTRL_GET_WPOLL_DESCRIPTOR 92 -# define BIO_CTRL_DGRAM_DETECT_PEER_ADDR 93 - -# define BIO_DGRAM_CAP_NONE 0U -# define BIO_DGRAM_CAP_HANDLES_SRC_ADDR (1U << 0) -# define BIO_DGRAM_CAP_HANDLES_DST_ADDR (1U << 1) -# define BIO_DGRAM_CAP_PROVIDES_SRC_ADDR (1U << 2) -# define BIO_DGRAM_CAP_PROVIDES_DST_ADDR (1U << 3) - # ifndef OPENSSL_NO_KTLS # define BIO_get_ktls_send(b) \ - (BIO_ctrl(b, BIO_CTRL_GET_KTLS_SEND, 0, NULL) > 0) + BIO_ctrl(b, BIO_CTRL_GET_KTLS_SEND, 0, NULL) # define BIO_get_ktls_recv(b) \ - (BIO_ctrl(b, BIO_CTRL_GET_KTLS_RECV, 0, NULL) > 0) + BIO_ctrl(b, BIO_CTRL_GET_KTLS_RECV, 0, NULL) # else # define BIO_get_ktls_send(b) (0) # define BIO_get_ktls_recv(b) (0) @@ -238,7 +209,7 @@ extern "C" { # define BIO_FLAGS_NONCLEAR_RST 0x400 # define BIO_FLAGS_IN_EOF 0x800 -/* the BIO FLAGS values 0x1000 to 0x8000 are reserved for internal KTLS flags */ +/* the BIO FLAGS values 0x1000 to 0x4000 are reserved for internal KTLS flags */ typedef union bio_addr_st BIO_ADDR; typedef struct bio_addrinfo_st BIO_ADDRINFO; @@ -285,15 +256,17 @@ void BIO_clear_flags(BIO *b, int flags); /* Returned from the accept BIO when an accept would have blocked */ # define BIO_RR_ACCEPT 0x03 +# ifndef OPENSSL_NO_SESSION_LOOKUP +# define BIO_RR_SSL_SESSION_LOOKUP 0x04 +# endif + /* These are passed by the BIO callback */ -# define BIO_CB_FREE 0x01 -# define BIO_CB_READ 0x02 -# define BIO_CB_WRITE 0x03 -# define BIO_CB_PUTS 0x04 -# define BIO_CB_GETS 0x05 -# define BIO_CB_CTRL 0x06 -# define BIO_CB_RECVMMSG 0x07 -# define BIO_CB_SENDMMSG 0x08 +# define BIO_CB_FREE 0x01 +# define BIO_CB_READ 0x02 +# define BIO_CB_WRITE 0x03 +# define BIO_CB_PUTS 0x04 +# define BIO_CB_GETS 0x05 +# define BIO_CB_CTRL 0x06 /* * The callback is called before and after the underling operation, The @@ -370,36 +343,6 @@ struct bio_dgram_sctp_prinfo { }; # endif -/* BIO_sendmmsg/BIO_recvmmsg-related definitions */ -typedef struct bio_msg_st { - void *data; - size_t data_len; - BIO_ADDR *peer, *local; - uint64_t flags; -} BIO_MSG; - -typedef struct bio_mmsg_cb_args_st { - BIO_MSG *msg; - size_t stride, num_msg; - uint64_t flags; - size_t *msgs_processed; -} BIO_MMSG_CB_ARGS; - -#define BIO_POLL_DESCRIPTOR_TYPE_NONE 0 -#define BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD 1 -#define BIO_POLL_DESCRIPTOR_TYPE_SSL 2 -#define BIO_POLL_DESCRIPTOR_CUSTOM_START 8192 - -typedef struct bio_poll_descriptor_st { - uint32_t type; - union { - int fd; - void *custom; - uintptr_t custom_ui; - SSL *ssl; - } value; -} BIO_POLL_DESCRIPTOR; - /* * #define BIO_CONN_get_param_hostname BIO_ctrl */ @@ -466,17 +409,10 @@ typedef struct bio_poll_descriptor_st { # define BIO_C_SET_CONNECT_MODE 155 -# define BIO_C_SET_TFO 156 /* like BIO_C_SET_NBIO */ - -# define BIO_C_SET_SOCK_TYPE 157 -# define BIO_C_GET_SOCK_TYPE 158 -# define BIO_C_GET_DGRAM_BIO 159 - # define BIO_set_app_data(s,arg) BIO_set_ex_data(s,0,arg) # define BIO_get_app_data(s) BIO_get_ex_data(s,0) -# define BIO_set_nbio(b,n) BIO_ctrl(b,BIO_C_SET_NBIO,(n),NULL) -# define BIO_set_tfo(b,n) BIO_ctrl(b,BIO_C_SET_TFO,(n),NULL) +# define BIO_set_nbio(b,n) BIO_ctrl(b,BIO_C_SET_NBIO,(n),NULL) # ifndef OPENSSL_NO_SOCK /* IP families we support, for BIO_s_connect() and BIO_s_accept() */ @@ -497,11 +433,7 @@ typedef struct bio_poll_descriptor_st { # define BIO_get_conn_port(b) ((const char *)BIO_ptr_ctrl(b,BIO_C_GET_CONNECT,1)) # define BIO_get_conn_address(b) ((const BIO_ADDR *)BIO_ptr_ctrl(b,BIO_C_GET_CONNECT,2)) # define BIO_get_conn_ip_family(b) BIO_ctrl(b,BIO_C_GET_CONNECT,3,NULL) -# define BIO_get_conn_mode(b) BIO_ctrl(b,BIO_C_GET_CONNECT,4,NULL) # define BIO_set_conn_mode(b,n) BIO_ctrl(b,BIO_C_SET_CONNECT_MODE,(n),NULL) -# define BIO_set_sock_type(b,t) BIO_ctrl(b,BIO_C_SET_SOCK_TYPE,(t),NULL) -# define BIO_get_sock_type(b) BIO_ctrl(b,BIO_C_GET_SOCK_TYPE,0,NULL) -# define BIO_get0_dgram_bio(b, p) BIO_ctrl(b,BIO_C_GET_DGRAM_BIO,0,(void *)(BIO **)(p)) /* BIO_s_accept() */ # define BIO_set_accept_name(b,name) BIO_ctrl(b,BIO_C_SET_ACCEPT,0, \ @@ -518,7 +450,6 @@ typedef struct bio_poll_descriptor_st { (char *)(bio)) # define BIO_set_accept_ip_family(b,f) BIO_int_ctrl(b,BIO_C_SET_ACCEPT,4,f) # define BIO_get_accept_ip_family(b) BIO_ctrl(b,BIO_C_GET_ACCEPT,4,NULL) -# define BIO_set_tfo_accept(b,n) BIO_ctrl(b,BIO_C_SET_ACCEPT,5,(n)?(void *)"a":NULL) /* Aliases kept for backward compatibility */ # define BIO_BIND_NORMAL 0 @@ -646,30 +577,8 @@ int BIO_ctrl_reset_read_request(BIO *b); (int)BIO_ctrl(b, BIO_CTRL_DGRAM_GET_PEER, 0, (char *)(peer)) # define BIO_dgram_set_peer(b,peer) \ (int)BIO_ctrl(b, BIO_CTRL_DGRAM_SET_PEER, 0, (char *)(peer)) -# define BIO_dgram_detect_peer_addr(b,peer) \ - (int)BIO_ctrl(b, BIO_CTRL_DGRAM_DETECT_PEER_ADDR, 0, (char *)(peer)) # define BIO_dgram_get_mtu_overhead(b) \ (unsigned int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_MTU_OVERHEAD, 0, NULL) -# define BIO_dgram_get_local_addr_cap(b) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP, 0, NULL) -# define BIO_dgram_get_local_addr_enable(b, penable) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE, 0, (char *)(penable)) -# define BIO_dgram_set_local_addr_enable(b, enable) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE, (enable), NULL) -# define BIO_dgram_get_effective_caps(b) \ - (uint32_t)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS, 0, NULL) -# define BIO_dgram_get_caps(b) \ - (uint32_t)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_CAPS, 0, NULL) -# define BIO_dgram_set_caps(b, caps) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_SET_CAPS, (long)(caps), NULL) -# define BIO_dgram_get_no_trunc(b) \ - (unsigned int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_NO_TRUNC, 0, NULL) -# define BIO_dgram_set_no_trunc(b, enable) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_SET_NO_TRUNC, (enable), NULL) -# define BIO_dgram_get_mtu(b) \ - (unsigned int)BIO_ctrl((b), BIO_CTRL_DGRAM_GET_MTU, 0, NULL) -# define BIO_dgram_set_mtu(b, mtu) \ - (int)BIO_ctrl((b), BIO_CTRL_DGRAM_SET_MTU, (mtu), NULL) /* ctrl macros for BIO_f_prefix */ # define BIO_set_prefix(b,p) BIO_ctrl((b), BIO_CTRL_SET_PREFIX, 0, (void *)(p)) @@ -712,18 +621,10 @@ void BIO_vfree(BIO *a); int BIO_up_ref(BIO *a); int BIO_read(BIO *b, void *data, int dlen); int BIO_read_ex(BIO *b, void *data, size_t dlen, size_t *readbytes); -__owur int BIO_recvmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, uint64_t flags, - size_t *msgs_processed); int BIO_gets(BIO *bp, char *buf, int size); int BIO_get_line(BIO *bio, char *buf, int size); int BIO_write(BIO *b, const void *data, int dlen); int BIO_write_ex(BIO *b, const void *data, size_t dlen, size_t *written); -__owur int BIO_sendmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, uint64_t flags, - size_t *msgs_processed); -__owur int BIO_get_rpoll_descriptor(BIO *b, BIO_POLL_DESCRIPTOR *desc); -__owur int BIO_get_wpoll_descriptor(BIO *b, BIO_POLL_DESCRIPTOR *desc); int BIO_puts(BIO *bp, const char *buf); int BIO_indent(BIO *b, int indent, int max); long BIO_ctrl(BIO *bp, int cmd, long larg, void *parg); @@ -747,9 +648,6 @@ int BIO_nwrite0(BIO *bio, char **buf); int BIO_nwrite(BIO *bio, char **buf, int num); const BIO_METHOD *BIO_s_mem(void); -# ifndef OPENSSL_NO_DGRAM -const BIO_METHOD *BIO_s_dgram_mem(void); -# endif const BIO_METHOD *BIO_s_secmem(void); BIO *BIO_new_mem_buf(const void *buf, int len); # ifndef OPENSSL_NO_SOCK @@ -769,7 +667,6 @@ const BIO_METHOD *BIO_f_nbio_test(void); const BIO_METHOD *BIO_f_prefix(void); const BIO_METHOD *BIO_s_core(void); # ifndef OPENSSL_NO_DGRAM -const BIO_METHOD *BIO_s_dgram_pair(void); const BIO_METHOD *BIO_s_datagram(void); int BIO_dgram_non_fatal_error(int error); BIO *BIO_new_dgram(int fd, int close_flag); @@ -788,7 +685,6 @@ int BIO_dgram_sctp_msg_waiting(BIO *b); # ifndef OPENSSL_NO_SOCK int BIO_sock_should_retry(int i); int BIO_sock_non_fatal_error(int error); -int BIO_err_is_non_fatal(unsigned int errcode); int BIO_socket_wait(int fd, int for_read, time_t max_time); # endif int BIO_wait(BIO *bio, time_t max_time, unsigned int nap_milliseconds); @@ -811,8 +707,6 @@ int BIO_hex_string(BIO *out, int indent, int width, const void *data, # ifndef OPENSSL_NO_SOCK BIO_ADDR *BIO_ADDR_new(void); -int BIO_ADDR_copy(BIO_ADDR *dst, const BIO_ADDR *src); -BIO_ADDR *BIO_ADDR_dup(const BIO_ADDR *ap); int BIO_ADDR_rawmake(BIO_ADDR *ap, int family, const void *where, size_t wherelen, unsigned short port); void BIO_ADDR_free(BIO_ADDR *); @@ -875,7 +769,6 @@ int BIO_sock_info(int sock, # define BIO_SOCK_KEEPALIVE 0x04 # define BIO_SOCK_NONBLOCK 0x08 # define BIO_SOCK_NODELAY 0x10 -# define BIO_SOCK_TFO 0x20 int BIO_socket(int domain, int socktype, int protocol, int options); int BIO_connect(int sock, const BIO_ADDR *addr, int options); @@ -893,11 +786,6 @@ BIO *BIO_new_fd(int fd, int close_flag); int BIO_new_bio_pair(BIO **bio1, size_t writebuf1, BIO **bio2, size_t writebuf2); -# ifndef OPENSSL_NO_DGRAM -int BIO_new_bio_dgram_pair(BIO **bio1, size_t writebuf1, - BIO **bio2, size_t writebuf2); -# endif - /* * If successful, returns 1 and in *bio1, *bio2 two BIO pair endpoints. * Otherwise returns 0 and sets *bio1 and *bio2 to NULL. Size 0 uses default @@ -912,7 +800,6 @@ void BIO_copy_next_retry(BIO *b); # define ossl_bio__attr__(x) # if defined(__GNUC__) && defined(__STDC_VERSION__) \ - && !defined(__MINGW32__) && !defined(__MINGW64__) \ && !defined(__APPLE__) /* * Because we support the 'z' modifier, which made its appearance in C99, @@ -949,30 +836,18 @@ int BIO_meth_set_write(BIO_METHOD *biom, int (*write) (BIO *, const char *, int)); int BIO_meth_set_write_ex(BIO_METHOD *biom, int (*bwrite) (BIO *, const char *, size_t, size_t *)); -int BIO_meth_set_sendmmsg(BIO_METHOD *biom, - int (*f) (BIO *, BIO_MSG *, size_t, size_t, - uint64_t, size_t *)); -int (*BIO_meth_get_sendmmsg(const BIO_METHOD *biom))(BIO *, BIO_MSG *, - size_t, size_t, - uint64_t, size_t *); int (*BIO_meth_get_read(const BIO_METHOD *biom)) (BIO *, char *, int); int (*BIO_meth_get_read_ex(const BIO_METHOD *biom)) (BIO *, char *, size_t, size_t *); int BIO_meth_set_read(BIO_METHOD *biom, int (*read) (BIO *, char *, int)); int BIO_meth_set_read_ex(BIO_METHOD *biom, int (*bread) (BIO *, char *, size_t, size_t *)); -int BIO_meth_set_recvmmsg(BIO_METHOD *biom, - int (*f) (BIO *, BIO_MSG *, size_t, size_t, - uint64_t, size_t *)); -int (*BIO_meth_get_recvmmsg(const BIO_METHOD *biom))(BIO *, BIO_MSG *, - size_t, size_t, - uint64_t, size_t *); int (*BIO_meth_get_puts(const BIO_METHOD *biom)) (BIO *, const char *); int BIO_meth_set_puts(BIO_METHOD *biom, int (*puts) (BIO *, const char *)); int (*BIO_meth_get_gets(const BIO_METHOD *biom)) (BIO *, char *, int); int BIO_meth_set_gets(BIO_METHOD *biom, - int (*ossl_gets) (BIO *, char *, int)); + int (*gets) (BIO *, char *, int)); long (*BIO_meth_get_ctrl(const BIO_METHOD *biom)) (BIO *, int, long, void *); int BIO_meth_set_ctrl(BIO_METHOD *biom, long (*ctrl) (BIO *, int, long, void *)); diff --git a/openssl/include/openssl/bioerr.h b/openssl/include/openssl/bioerr.h index e4fdb6497..787b30afc 100644 --- a/openssl/include/openssl/bioerr.h +++ b/openssl/include/openssl/bioerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -37,18 +37,14 @@ # define BIO_R_IN_USE 123 # define BIO_R_LENGTH_TOO_LONG 102 # define BIO_R_LISTEN_V6_ONLY 136 -# define BIO_R_LOCAL_ADDR_NOT_AVAILABLE 111 # define BIO_R_LOOKUP_RETURNED_NOTHING 142 # define BIO_R_MALFORMED_HOST_OR_SERVICE 130 # define BIO_R_NBIO_CONNECT_ERROR 110 -# define BIO_R_NON_FATAL 112 # define BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED 143 # define BIO_R_NO_HOSTNAME_OR_SERVICE_SPECIFIED 144 # define BIO_R_NO_PORT_DEFINED 113 # define BIO_R_NO_SUCH_FILE 128 # define BIO_R_NULL_PARAMETER 115 /* unused */ -# define BIO_R_TFO_DISABLED 106 -# define BIO_R_TFO_NO_KERNEL_SUPPORT 108 # define BIO_R_TRANSFER_ERROR 104 # define BIO_R_TRANSFER_TIMEOUT 105 # define BIO_R_UNABLE_TO_BIND_SOCKET 117 @@ -57,7 +53,6 @@ # define BIO_R_UNABLE_TO_LISTEN_SOCKET 119 # define BIO_R_UNABLE_TO_NODELAY 138 # define BIO_R_UNABLE_TO_REUSEADDR 139 -# define BIO_R_UNABLE_TO_TFO 109 # define BIO_R_UNAVAILABLE_IP_FAMILY 145 # define BIO_R_UNINITIALIZED 120 # define BIO_R_UNKNOWN_INFO_TYPE 140 @@ -66,7 +61,5 @@ # define BIO_R_UNSUPPORTED_PROTOCOL_FAMILY 131 # define BIO_R_WRITE_TO_READ_ONLY_BIO 126 # define BIO_R_WSASTARTUP 122 -# define BIO_R_PORT_MISMATCH 150 -# define BIO_R_PEER_ADDR_NOT_AVAILABLE 151 #endif diff --git a/openssl/include/openssl/blowfish.h b/openssl/include/openssl/blowfish.h deleted file mode 100644 index 667d64239..000000000 --- a/openssl/include/openssl/blowfish.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_BLOWFISH_H -# define OPENSSL_BLOWFISH_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_BLOWFISH_H -# endif - -# include - -# ifndef OPENSSL_NO_BF -# include -# ifdef __cplusplus -extern "C" { -# endif - -# define BF_BLOCK 8 - -# ifndef OPENSSL_NO_DEPRECATED_3_0 - -# define BF_ENCRYPT 1 -# define BF_DECRYPT 0 - -/*- - * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - * ! BF_LONG has to be at least 32 bits wide. ! - * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - */ -# define BF_LONG unsigned int - -# define BF_ROUNDS 16 - -typedef struct bf_key_st { - BF_LONG P[BF_ROUNDS + 2]; - BF_LONG S[4 * 256]; -} BF_KEY; - -# endif /* OPENSSL_NO_DEPRECATED_3_0 */ -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 void BF_set_key(BF_KEY *key, int len, - const unsigned char *data); -OSSL_DEPRECATEDIN_3_0 void BF_encrypt(BF_LONG *data, const BF_KEY *key); -OSSL_DEPRECATEDIN_3_0 void BF_decrypt(BF_LONG *data, const BF_KEY *key); -OSSL_DEPRECATEDIN_3_0 void BF_ecb_encrypt(const unsigned char *in, - unsigned char *out, const BF_KEY *key, - int enc); -OSSL_DEPRECATEDIN_3_0 void BF_cbc_encrypt(const unsigned char *in, - unsigned char *out, long length, - const BF_KEY *schedule, - unsigned char *ivec, int enc); -OSSL_DEPRECATEDIN_3_0 void BF_cfb64_encrypt(const unsigned char *in, - unsigned char *out, - long length, const BF_KEY *schedule, - unsigned char *ivec, int *num, - int enc); -OSSL_DEPRECATEDIN_3_0 void BF_ofb64_encrypt(const unsigned char *in, - unsigned char *out, - long length, const BF_KEY *schedule, - unsigned char *ivec, int *num); -OSSL_DEPRECATEDIN_3_0 const char *BF_options(void); -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/bn.h b/openssl/include/openssl/bn.h index ea706dca7..6de79fa41 100644 --- a/openssl/include/openssl/bn.h +++ b/openssl/include/openssl/bn.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -211,6 +211,57 @@ BN_CTX *BN_CTX_new(void); BN_CTX *BN_CTX_secure_new_ex(OSSL_LIB_CTX *ctx); BN_CTX *BN_CTX_secure_new(void); void BN_CTX_free(BN_CTX *c); +# ifndef OPENSSL_NO_BN_METHOD +# if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE) +int BN_CTX_set_engine(BN_CTX *ctx, ENGINE *engine); +const ENGINE *BN_CTX_get0_engine(BN_CTX *ctx); +# endif +int BN_CTX_set_method(BN_CTX *ctx, const BN_METHOD *method); +BN_METHOD *BN_METHOD_new(const char *name); +void BN_METHOD_free(BN_METHOD *meth); +int BN_METHOD_copy(BN_METHOD *dst, const BN_METHOD *src); +char *BN_METHOD_name(BN_METHOD *meth); +int (*BN_METHOD_get_add(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx); +void BN_METHOD_set_add(BN_METHOD *meth, + int (*mod_add)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, + const BIGNUM *m, BN_CTX *ctx)); +int (*BN_METHOD_get_sub(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx); +void BN_METHOD_set_sub(BN_METHOD *meth, + int (*mod_sub)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, + const BIGNUM *m, BN_CTX *ctx)); +int (*BN_METHOD_get_mul(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx); +void BN_METHOD_set_mul(BN_METHOD *meth, + int (*mod_mul)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, + const BIGNUM *m, BN_CTX *ctx)); +int (*BN_METHOD_get_exp(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx); +void BN_METHOD_set_exp(BN_METHOD *meth, + int (*mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, + const BIGNUM *m, BN_CTX *ctx)); +int (*BN_METHOD_get_sqr(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx); +void BN_METHOD_set_sqr(BN_METHOD *meth, + int (*mod_sqr)(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, + BN_CTX *ctx)); +BIGNUM *(*BN_METHOD_get_sqrt(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *n, BN_CTX *ctx); +void BN_METHOD_set_sqrt(BN_METHOD *meth, + BIGNUM *(*mod_sqrt)(BIGNUM *r, const BIGNUM *a, + const BIGNUM *n, BN_CTX *ctx)); +BIGNUM *(*BN_METHOD_get_inverse(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *n, BN_CTX *ctx); +void BN_METHOD_set_inverse(BN_METHOD *meth, + BIGNUM *(*mod_inverse)(BIGNUM *r, const BIGNUM *a, + const BIGNUM *n, BN_CTX *ctx)); +int (*BN_METHOD_get_div(BN_METHOD *meth)) + (BIGNUM *dv, BIGNUM *rem, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx); +void BN_METHOD_set_div(BN_METHOD *meth, + int (*div)(BIGNUM *dv, BIGNUM *rem, const BIGNUM *m, + const BIGNUM *d, BN_CTX *ctx)); +# endif void BN_CTX_start(BN_CTX *ctx); BIGNUM *BN_CTX_get(BN_CTX *ctx); void BN_CTX_end(BN_CTX *ctx); @@ -241,18 +292,12 @@ void BN_clear_free(BIGNUM *a); BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b); void BN_swap(BIGNUM *a, BIGNUM *b); BIGNUM *BN_bin2bn(const unsigned char *s, int len, BIGNUM *ret); -BIGNUM *BN_signed_bin2bn(const unsigned char *s, int len, BIGNUM *ret); int BN_bn2bin(const BIGNUM *a, unsigned char *to); int BN_bn2binpad(const BIGNUM *a, unsigned char *to, int tolen); -int BN_signed_bn2bin(const BIGNUM *a, unsigned char *to, int tolen); BIGNUM *BN_lebin2bn(const unsigned char *s, int len, BIGNUM *ret); -BIGNUM *BN_signed_lebin2bn(const unsigned char *s, int len, BIGNUM *ret); int BN_bn2lebinpad(const BIGNUM *a, unsigned char *to, int tolen); -int BN_signed_bn2lebin(const BIGNUM *a, unsigned char *to, int tolen); BIGNUM *BN_native2bn(const unsigned char *s, int len, BIGNUM *ret); -BIGNUM *BN_signed_native2bn(const unsigned char *s, int len, BIGNUM *ret); int BN_bn2nativepad(const BIGNUM *a, unsigned char *to, int tolen); -int BN_signed_bn2native(const BIGNUM *a, unsigned char *to, int tolen); BIGNUM *BN_mpi2bn(const unsigned char *s, int len, BIGNUM *ret); int BN_bn2mpi(const BIGNUM *a, unsigned char *to); int BN_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b); @@ -350,7 +395,6 @@ int BN_gcd(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx); int BN_kronecker(const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx); /* returns * -2 for * error */ -int BN_are_coprime(BIGNUM *a, const BIGNUM *b, BN_CTX *ctx); BIGNUM *BN_mod_inverse(BIGNUM *ret, const BIGNUM *a, const BIGNUM *n, BN_CTX *ctx); BIGNUM *BN_mod_sqrt(BIGNUM *ret, @@ -551,6 +595,15 @@ const BIGNUM *BN_get0_nist_prime_256(void); const BIGNUM *BN_get0_nist_prime_384(void); const BIGNUM *BN_get0_nist_prime_521(void); +/* + * faster mod functions for the sm2 prime, 0 <= a < p^2 + */ +# ifndef OPENSSL_NO_SM2 +int BN_sm2_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx); + +const BIGNUM *BN_get0_sm2_prime_256(void); +# endif + int (*BN_nist_mod_func(const BIGNUM *p)) (BIGNUM *r, const BIGNUM *a, const BIGNUM *field, BN_CTX *ctx); diff --git a/openssl/include/openssl/bnerr.h b/openssl/include/openssl/bnerr.h index 7c3f6ef3d..039637e64 100644 --- a/openssl/include/openssl/bnerr.h +++ b/openssl/include/openssl/bnerr.h @@ -25,6 +25,7 @@ # define BN_R_BAD_RECIPROCAL 101 # define BN_R_BIGNUM_TOO_LONG 114 # define BN_R_BITS_TOO_SMALL 118 +# define BN_R_BN_METHOD_NOT_FOUND 121 # define BN_R_CALLED_WITH_EVEN_MODULUS 102 # define BN_R_DIV_BY_ZERO 103 # define BN_R_ENCODING_ERROR 104 @@ -36,7 +37,6 @@ # define BN_R_NOT_A_SQUARE 111 # define BN_R_NOT_INITIALIZED 107 # define BN_R_NO_INVERSE 108 -# define BN_R_NO_PRIME_CANDIDATE 121 # define BN_R_NO_SOLUTION 116 # define BN_R_NO_SUITABLE_DIGEST 120 # define BN_R_PRIVATE_KEY_TOO_LARGE 117 diff --git a/openssl/include/openssl/bulletproofs.h b/openssl/include/openssl/bulletproofs.h new file mode 100644 index 000000000..c05e3a839 --- /dev/null +++ b/openssl/include/openssl/bulletproofs.h @@ -0,0 +1,396 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_BULLETPROOFS_H +# define HEADER_BULLETPROOFS_H + +# include +# include +# include +# include +# include +# include +# include + +# ifndef OPENSSL_NO_BULLETPROOFS +# ifdef __cplusplus +extern "C" { +# endif + +# define PEM_STRING_BULLETPROOFS_PUB_PARAM "BULLETPROOFS PUBLIC PARAM" +# define PEM_STRING_BULLETPROOFS_WITNESS "BULLETPROOFS WITNESS" +# define PEM_STRING_BULLETPROOFS_RANGE_PROOF "BULLETPROOFS RANGE PROOF" +# define PEM_STRING_BULLETPROOFS_R1CS_PROOF "BULLETPROOFS R1CS PROOF" + +# define BULLET_PROOF_MAX_GENS_CAPACITY 128 +# define BULLET_PROOF_MAX_PARTY_CAPACITY 64 + +typedef struct bp_pub_param_st BP_PUB_PARAM; +typedef struct bp_witness_st BP_WITNESS; +typedef struct bp_variable_st BP_VARIABLE; + +typedef struct bp_range_ctx_st BP_RANGE_CTX; +typedef struct bp_range_proof_st BP_RANGE_PROOF; + +typedef struct bp_r1cs_ctx_st BP_R1CS_CTX; +typedef struct bp_r1cs_proof_st BP_R1CS_PROOF; + +typedef struct bp_r1cs_variable_st BP_R1CS_VARIABLE; +typedef struct bp_r1cs_linear_combination_item_st BP_R1CS_LINEAR_COMBINATION_ITEM; +typedef BP_R1CS_LINEAR_COMBINATION_ITEM BP_R1CS_LC_ITEM; +typedef struct bp_r1cs_linear_combination_st BP_R1CS_LINEAR_COMBINATION; +typedef BP_R1CS_LINEAR_COMBINATION BP_R1CS_LC; + +/********************************************************************/ +/* functions for doing bulletproofs arithmetic */ +/********************************************************************/ + +/** Creates a new BP_PUB_PARAM object + * \param group underlying EC_GROUP object + * \param gens_capacity the number of generators to precompute for each party. + * For range_proof, it is the maximum bitsize of the + * range_proof, maximum value is 64. For r1cs_proof, + * the capacity must be greater than the number of + * multipliers, rounded up to the next power of two. + * \param party_capacity the maximum number of parties that can produce on + * aggregated proof. For r1cs_proof, set to 1. + * \return newly created BP_PUB_PARAM object or NULL in case of an error + */ +BP_PUB_PARAM *BP_PUB_PARAM_new(const EC_GROUP *group, int gens_capacity, + int party_capacity); + +/** Creates a new BP_PUB_PARAM object by curve name + * \param curve_name the elliptic curve name + * \param gens_capacity the number of generators to precompute for each party. + * For range_proof, it is the maximum bitsize of the + * range_proof, maximum value is 64. For r1cs_proof, + * the capacity must be greater than the number of + * multipliers, rounded up to the next power of two. + * \param party_capacity the maximum number of parties that can produce on + * aggregated proof. For r1cs_proof, set to 1. + * \return newly created BP_PUB_PARAM object or NULL in case of an error + */ +BP_PUB_PARAM *BP_PUB_PARAM_new_by_curve_name(const char *curve_name, + int gens_capacity, + int party_capacity); + +/** Creates a new BP_PUB_PARAM object by curve id + * \param curve_id the elliptic curve id + * \param gens_capacity the number of generators to precompute for each party. + * For range_proof, it is the maximum bitsize of the + * range_proof, maximum value is 64. For r1cs_proof, + * the capacity must be greater than the number of + * multipliers, rounded up to the next power of two. + * \param party_capacity the maximum number of parties that can produce on + * aggregated proof. For r1cs_proof, set to 1. + * \return newly created BP_PUB_PARAM object or NULL in case of an error + */ +BP_PUB_PARAM *BP_PUB_PARAM_new_by_curve_id(int curve_id, + int gens_capacity, + int party_capacity); + +/** Frees a BP_PUB_PARAM object + * \param pp BP_PUB_PARAM object to be freed + */ +void BP_PUB_PARAM_free(BP_PUB_PARAM *pp); + +/** Increases the internal reference count of a BP_PUB_PARAM object. + * \param pp BP_PUB_PARAM object + * \return 1 on success and 0 if an error occurred. + */ +int BP_PUB_PARAM_up_ref(BP_PUB_PARAM *pp); + +/** Decreases the internal reference count of a BP_PUB_PARAM object. + * \param pp BP_PUB_PARAM object + * \return 1 on success and 0 if an error occurred. + */ +int BP_PUB_PARAM_down_ref(BP_PUB_PARAM *pp); + +/** Creates a new BP_VARIABLE object + * \param name the bulletproofs variable name, used for indexing. + * \param point EC_POINT object + * \param group EC_GROUP object + * \return newly created BP_WITNESS object or NULL in case of an error + */ +BP_VARIABLE *BP_VARIABLE_new(const char *name, const EC_POINT *point, const EC_GROUP *group); + +/** Frees a BP_VARIABLE object + * \param var BP_VARIABLE object to be freed + */ +void BP_VARIABLE_free(BP_VARIABLE *var); + +/** Creates a new BP_WITNESS object + * \param pp underlying BP_PUB_PARAM object + * \return newly created BP_WITNESS object or NULL in case of an error + */ +BP_WITNESS *BP_WITNESS_new(const BP_PUB_PARAM *pp); + +/** Frees a BP_WITNESS object + * \param witness BP_WITNESS object to be freed + */ +void BP_WITNESS_free(BP_WITNESS *witness); + +/** Increases the internal reference count of a BP_WITNESS object. + * \param witness BP_WITNESS object + * \return 1 on success and 0 if an error occurred. + */ +int BP_WITNESS_up_ref(BP_WITNESS *witness); + +/** Decreases the internal reference count of a BP_WITNESS object. + * \param witness BP_WITNESS object + * \return 1 on success and 0 if an error occurred. + */ +int BP_WITNESS_down_ref(BP_WITNESS *witness); + +/** Commit v to the witness and calculate V=G^r*H^v + * \param witness BP_WITNESS object + * \param name the name used to index the BP_VARIABLE object + * \param v plaintext BIGNUM object + * \return 1 on success and 0 otherwise + */ +int BP_WITNESS_commit(BP_WITNESS *witness, const char *name, const BIGNUM *v); + +/** Get the BP_VARIABLE with the variable name from the witness. + * \param witness BP_WITNESS object + * \param name the name of the BP_VARIABLE object + * \return the BP_VARIABLE object when found by name, otherwise return NULL. + */ +BP_VARIABLE *BP_WITNESS_get_variable(BP_WITNESS *witness, const char *name); + +/** Get the index of the BP_VARIABLE in the stack that corresponds to the variable + * name from the witness. + * \param witness BP_WITNESS object + * \param name the name of the BP_VARIABLE object + * \return the index of the BP_VARIABLE object when found by name, + * otherwise return -1. + */ +int BP_WITNESS_get_variable_index(BP_WITNESS *witness, const char *name); + +/********************************************************************/ +/* functions for doing range proof arithmetic */ +/********************************************************************/ + +/** Creates a new BP_RANGE_CTX object + * \param pp BP_PUB_PARAM object + * \param witness BP_WITNESS object + * \param transcript ZKP_TRANSCRIPT object + * \return newly created BP_RANGE_CTX object or NULL in case of an error + */ +BP_RANGE_CTX *BP_RANGE_CTX_new(BP_PUB_PARAM *pp, BP_WITNESS *witness, + ZKP_TRANSCRIPT *transcript); + +/** Frees a BP_RANGE_CTX object + * \param ctx BP_RANGE_CTX object to be freed + */ +void BP_RANGE_CTX_free(BP_RANGE_CTX *ctx); + +/** Creates a new BP_RANGE_PROOF object + * \param pp BP_PUB_PARAM object + * \return newly created BP_RANGE_PROOF object or NULL in case of an error + */ +BP_RANGE_PROOF *BP_RANGE_PROOF_new(const BP_PUB_PARAM *pp); + +/** Frees a BP_RANGE_PROOF object + * \param proof BP_RANGE_PROOF object to be freed + */ +void BP_RANGE_PROOF_free(BP_RANGE_PROOF *proof); + +/** Increases the internal reference count of a BP_RANGE_PROOF object. + * \param proof BP_RANGE_PROOF object + * \return 1 on success and 0 if an error occurred. + */ +int BP_RANGE_PROOF_up_ref(BP_RANGE_PROOF *proof); + +/** Decreases the internal reference count of a BP_RANGE_PROOF object. + * \param proof BP_RANGE_PROOF object + * \return 1 on success and 0 if an error occurred. + */ +int BP_RANGE_PROOF_down_ref(BP_RANGE_PROOF *proof); + +/** Prove computes the ZK rangeproof. + * \param ctx BP_RANGE_CTX object + * \param proof BP_RANGE_PROOF object + * \return 1 on success and 0 otherwise + */ +int BP_RANGE_PROOF_prove(BP_RANGE_CTX *ctx, BP_RANGE_PROOF *proof); + +/** Prove computes the ZK rangeproof. + * \param ctx BP_RANGE_CTX object + * \return the BP_RANGE_PROOF object on success or NULL in case of an error + */ +BP_RANGE_PROOF *BP_RANGE_PROOF_new_prove(BP_RANGE_CTX *ctx); + +/** Verifies that the supplied proof is a valid proof + * for the supplied secret values using the supplied public parameters. + * \param ctx BP_RANGE_CTX object + * \param proof BP_RANGE_PROOF object + * \return 1 if the proof is valid, 0 if the proof is invalid and -1 on error + */ +int BP_RANGE_PROOF_verify(BP_RANGE_CTX *ctx, const BP_RANGE_PROOF *proof); + +/** Encodes BP_PUB_PARAM to binary + * \param pp BP_PUB_PARAM object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t BP_PUB_PARAM_encode(const BP_PUB_PARAM *pp, unsigned char *out, size_t size); + +/** Encodes BP_WITNESS to binary + * \param witness BP_WITNESS object + * \param out The buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param flag The flag is an indicator for encoding random number 'r' + * and plaintext 'v', with 1 indicating encoding and 0 + * indicating no encoding. + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t BP_WITNESS_encode(const BP_WITNESS *witness, unsigned char *out, + size_t size, int flag); + +/** Decodes binary to BP_WITNESS + * \param in Memory buffer with the encoded BP_WITNESS + * object + * \param size The memory size of the in pointer object + * \param flag The flag is an indicator for decoding random number 'r' + * and plaintext 'v', with 1 indicating decoding and 0 + * \return BP_WITNESS object pointer on success and NULL otherwise + */ +BP_WITNESS *BP_WITNESS_decode(const unsigned char *in, size_t size, int flag); + +/** Decodes binary to BP_PUB_PARAM + * \param in Memory buffer with the encoded BP_PUB_PARAM + * object + * \param size The memory size of the in pointer object + * \return the BP_PUB_PARAM object pointer on success and NULL otherwise + */ +BP_PUB_PARAM *BP_PUB_PARAM_decode(const unsigned char *in, size_t size); + +/** Encodes BP_RANGE_PROOF to binary + * \param proof BP_RANGE_PROOF object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t BP_RANGE_PROOF_encode(const BP_RANGE_PROOF *proof, unsigned char *out, + size_t size); + +/** Decodes binary to BP_RANGE_PROOF + * \param in Memory buffer with the encoded BP_RANGE_PROOF object + * \param size The memory size of the in pointer object + * \return BP_RANGE_PROOF object pointer on success and NULL otherwise + */ +BP_RANGE_PROOF *BP_RANGE_PROOF_decode(const unsigned char *in, size_t size); + +/** Encodes BP_R1CS_PROOF to binary + * \param proof BP_R1CS_PROOF object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t BP_R1CS_PROOF_encode(const BP_R1CS_PROOF *proof, unsigned char *out, + size_t size); + +/** Decodes binary to BP_R1CS_PROOF + * \param in Memory buffer with the encoded BP_R1CS_PROOF object + * \param size The memory size of the in pointer object + * \return BP_R1CS_PROOF object pointer on success and NULL otherwise + */ +BP_R1CS_PROOF *BP_R1CS_PROOF_decode(const unsigned char *in, size_t size); + +/********************************************************************/ +/* functions for doing r1cs arithmetic */ +/********************************************************************/ + +BP_R1CS_LINEAR_COMBINATION *BP_R1CS_LINEAR_COMBINATION_new(void); +BP_R1CS_LINEAR_COMBINATION *BP_R1CS_LINEAR_COMBINATION_dup(const BP_R1CS_LINEAR_COMBINATION *lc); +void BP_R1CS_LINEAR_COMBINATION_free(BP_R1CS_LINEAR_COMBINATION *lc); +int BP_R1CS_LINEAR_COMBINATION_clean(BP_R1CS_LINEAR_COMBINATION *lc); + +int BP_R1CS_LINEAR_COMBINATION_raw_mul(BP_R1CS_LINEAR_COMBINATION **output, + BP_R1CS_LINEAR_COMBINATION **left, + BP_R1CS_LINEAR_COMBINATION **right, + const BIGNUM *l, const BIGNUM *r, + BP_R1CS_CTX *ctx); +int BP_R1CS_LINEAR_COMBINATION_mul(BP_R1CS_LINEAR_COMBINATION *lc, + const BP_R1CS_LINEAR_COMBINATION *other, + BP_R1CS_CTX *ctx); +int BP_R1CS_LINEAR_COMBINATION_add(BP_R1CS_LINEAR_COMBINATION *lc, + const BP_R1CS_LINEAR_COMBINATION *other); +int BP_R1CS_LINEAR_COMBINATION_sub(BP_R1CS_LINEAR_COMBINATION *lc, + const BP_R1CS_LINEAR_COMBINATION *other); +int BP_R1CS_LINEAR_COMBINATION_neg(BP_R1CS_LINEAR_COMBINATION *lc); +int BP_R1CS_LINEAR_COMBINATION_mul_bn(BP_R1CS_LINEAR_COMBINATION *lc, + const BIGNUM *value); +int BP_R1CS_LINEAR_COMBINATION_add_bn(BP_R1CS_LINEAR_COMBINATION *lc, + const BIGNUM *value); +int BP_R1CS_LINEAR_COMBINATION_sub_bn(BP_R1CS_LINEAR_COMBINATION *lc, + const BIGNUM *value); + +BP_R1CS_LINEAR_COMBINATION *BP_WITNESS_r1cs_linear_combination_commit(BP_WITNESS *witness, + const char *name, + BIGNUM *v); +BP_R1CS_LINEAR_COMBINATION *BP_WITNESS_r1cs_linear_combination_get(BP_WITNESS *witness, + const char *name); +int BP_R1CS_LINEAR_COMBINATION_constrain(BP_R1CS_LINEAR_COMBINATION *lc, BP_R1CS_CTX *ctx); +int BP_WITNESS_r1cs_commit(BP_WITNESS *witness, const char *name, BIGNUM *v); +int BP_R1CS_constraint_expression(BP_R1CS_CTX *ctx, const char *constraint, int is_prove); + +BP_R1CS_PROOF *BP_R1CS_PROOF_new(BP_R1CS_CTX *ctx); +void BP_R1CS_PROOF_free(BP_R1CS_PROOF *proof); +BP_R1CS_PROOF *BP_R1CS_PROOF_prove(BP_R1CS_CTX *ctx); +int BP_R1CS_PROOF_verify(BP_R1CS_CTX *ctx, BP_R1CS_PROOF *proof); + +/** Creates a new BP_R1CS_CTX object + * \param pp BP_PUB_PARAM object + * \param witness BP_WITNESS object + * \param transcript ZKP_TRANSCRIPT object + * \return newly created BP_R1CS_CTX object or NULL in case of an error + */ +BP_R1CS_CTX *BP_R1CS_CTX_new(BP_PUB_PARAM *pp, BP_WITNESS *witness, + ZKP_TRANSCRIPT *transcript); + +void BP_R1CS_CTX_free(BP_R1CS_CTX *ctx); + +# ifndef OPENSSL_NO_STDIO +int BP_PUB_PARAM_print_fp(FILE *fp, const BP_PUB_PARAM *pp, int indent); +int BP_WITNESS_print_fp(FILE *fp, const BP_WITNESS *witness, int indent, int flag); +int BP_RANGE_PROOF_print_fp(FILE *fp, const BP_RANGE_PROOF *proof, int indent); +int BP_R1CS_PROOF_print_fp(FILE *fp, const BP_R1CS_PROOF *proof, int indent); +# endif +int BP_PUB_PARAM_print(BIO *bp, const BP_PUB_PARAM *pp, int indent); +int BP_WITNESS_print(BIO *bp, const BP_WITNESS *witness, int indent, int flag); +int BP_RANGE_PROOF_print(BIO *bp, const BP_RANGE_PROOF *proof, int indent); +int BP_R1CS_PROOF_print(BIO *bp, const BP_R1CS_PROOF *proof, int indent); + +/********************************************************************/ +/* functions for doing bulletproofs encoding/decoding */ +/********************************************************************/ + +DECLARE_PEM_rw(BULLETPROOFS_PublicParam, BP_PUB_PARAM) +DECLARE_PEM_rw(BULLETPROOFS_LongWitness, BP_WITNESS) +DECLARE_PEM_rw(BULLETPROOFS_ShortWitness, BP_WITNESS) +DECLARE_PEM_rw(BULLETPROOFS_RangeProof, BP_RANGE_PROOF) +DECLARE_PEM_rw(BULLETPROOFS_R1CSProof, BP_R1CS_PROOF) +DECLARE_ASN1_ENCODE_FUNCTIONS_only(BP_PUB_PARAM, BP_PUB_PARAM) +DECLARE_ASN1_ENCODE_FUNCTIONS_only(BP_WITNESS, long_BP_WITNESS) +DECLARE_ASN1_ENCODE_FUNCTIONS_only(BP_WITNESS, short_BP_WITNESS) +DECLARE_ASN1_ENCODE_FUNCTIONS_only(BP_RANGE_PROOF, BP_RANGE_PROOF) +DECLARE_ASN1_ENCODE_FUNCTIONS_only(BP_R1CS_PROOF, BP_R1CS_PROOF) + +# ifdef __cplusplus +} +# endif +# endif + +#endif diff --git a/openssl/include/openssl/camellia.h b/openssl/include/openssl/camellia.h deleted file mode 100644 index 88c2279e9..000000000 --- a/openssl/include/openssl/camellia.h +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_CAMELLIA_H -# define OPENSSL_CAMELLIA_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_CAMELLIA_H -# endif - -# include - -# ifndef OPENSSL_NO_CAMELLIA -# include -#ifdef __cplusplus -extern "C" { -#endif - -# define CAMELLIA_BLOCK_SIZE 16 - -# ifndef OPENSSL_NO_DEPRECATED_3_0 - -# define CAMELLIA_ENCRYPT 1 -# define CAMELLIA_DECRYPT 0 - -/* - * Because array size can't be a const in C, the following two are macros. - * Both sizes are in bytes. - */ - -/* This should be a hidden type, but EVP requires that the size be known */ - -# define CAMELLIA_TABLE_BYTE_LEN 272 -# define CAMELLIA_TABLE_WORD_LEN (CAMELLIA_TABLE_BYTE_LEN / 4) - -typedef unsigned int KEY_TABLE_TYPE[CAMELLIA_TABLE_WORD_LEN]; /* to match - * with WORD */ - -struct camellia_key_st { - union { - double d; /* ensures 64-bit align */ - KEY_TABLE_TYPE rd_key; - } u; - int grand_rounds; -}; -typedef struct camellia_key_st CAMELLIA_KEY; - -# endif /* OPENSSL_NO_DEPRECATED_3_0 */ -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 int Camellia_set_key(const unsigned char *userKey, - const int bits, - CAMELLIA_KEY *key); -OSSL_DEPRECATEDIN_3_0 void Camellia_encrypt(const unsigned char *in, - unsigned char *out, - const CAMELLIA_KEY *key); -OSSL_DEPRECATEDIN_3_0 void Camellia_decrypt(const unsigned char *in, - unsigned char *out, - const CAMELLIA_KEY *key); -OSSL_DEPRECATEDIN_3_0 void Camellia_ecb_encrypt(const unsigned char *in, - unsigned char *out, - const CAMELLIA_KEY *key, - const int enc); -OSSL_DEPRECATEDIN_3_0 void Camellia_cbc_encrypt(const unsigned char *in, - unsigned char *out, - size_t length, - const CAMELLIA_KEY *key, - unsigned char *ivec, - const int enc); -OSSL_DEPRECATEDIN_3_0 void Camellia_cfb128_encrypt(const unsigned char *in, - unsigned char *out, - size_t length, - const CAMELLIA_KEY *key, - unsigned char *ivec, - int *num, - const int enc); -OSSL_DEPRECATEDIN_3_0 void Camellia_cfb1_encrypt(const unsigned char *in, - unsigned char *out, - size_t length, - const CAMELLIA_KEY *key, - unsigned char *ivec, - int *num, - const int enc); -OSSL_DEPRECATEDIN_3_0 void Camellia_cfb8_encrypt(const unsigned char *in, - unsigned char *out, - size_t length, - const CAMELLIA_KEY *key, - unsigned char *ivec, - int *num, - const int enc); -OSSL_DEPRECATEDIN_3_0 void Camellia_ofb128_encrypt(const unsigned char *in, - unsigned char *out, - size_t length, - const CAMELLIA_KEY *key, - unsigned char *ivec, - int *num); -OSSL_DEPRECATEDIN_3_0 -void Camellia_ctr128_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const CAMELLIA_KEY *key, - unsigned char ivec[CAMELLIA_BLOCK_SIZE], - unsigned char ecount_buf[CAMELLIA_BLOCK_SIZE], - unsigned int *num); -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/cast.h b/openssl/include/openssl/cast.h deleted file mode 100644 index 0bf217bea..000000000 --- a/openssl/include/openssl/cast.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_CAST_H -# define OPENSSL_CAST_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_CAST_H -# endif - -# include - -# ifndef OPENSSL_NO_CAST -# ifdef __cplusplus -extern "C" { -# endif - -# define CAST_BLOCK 8 -# define CAST_KEY_LENGTH 16 - -# ifndef OPENSSL_NO_DEPRECATED_3_0 - -# define CAST_ENCRYPT 1 -# define CAST_DECRYPT 0 - -# define CAST_LONG unsigned int - -typedef struct cast_key_st { - CAST_LONG data[32]; - int short_key; /* Use reduced rounds for short key */ -} CAST_KEY; - -# endif /* OPENSSL_NO_DEPRECATED_3_0 */ -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 -void CAST_set_key(CAST_KEY *key, int len, const unsigned char *data); -OSSL_DEPRECATEDIN_3_0 -void CAST_ecb_encrypt(const unsigned char *in, unsigned char *out, - const CAST_KEY *key, int enc); -OSSL_DEPRECATEDIN_3_0 -void CAST_encrypt(CAST_LONG *data, const CAST_KEY *key); -OSSL_DEPRECATEDIN_3_0 -void CAST_decrypt(CAST_LONG *data, const CAST_KEY *key); -OSSL_DEPRECATEDIN_3_0 -void CAST_cbc_encrypt(const unsigned char *in, unsigned char *out, - long length, const CAST_KEY *ks, unsigned char *iv, - int enc); -OSSL_DEPRECATEDIN_3_0 -void CAST_cfb64_encrypt(const unsigned char *in, unsigned char *out, - long length, const CAST_KEY *schedule, - unsigned char *ivec, int *num, int enc); -OSSL_DEPRECATEDIN_3_0 -void CAST_ofb64_encrypt(const unsigned char *in, unsigned char *out, - long length, const CAST_KEY *schedule, - unsigned char *ivec, int *num); -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/cmp.h b/openssl/include/openssl/cmp.h index ab5bde4d4..a2929b09a 100644 --- a/openssl/include/openssl/cmp.h +++ b/openssl/include/openssl/cmp.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/cmp.h.in * - * Copyright 2007-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -35,9 +35,7 @@ extern "C" { # endif -# define OSSL_CMP_PVNO_2 2 -# define OSSL_CMP_PVNO_3 3 -# define OSSL_CMP_PVNO OSSL_CMP_PVNO_2 /* v2 is the default */ +# define OSSL_CMP_PVNO 2 /*- * PKIFailureInfo ::= BIT STRING { @@ -139,6 +137,7 @@ extern "C" { # if OSSL_CMP_PKIFAILUREINFO_MAX_BIT_PATTERN > INT_MAX # error CMP_PKIFAILUREINFO_MAX bit pattern does not fit in type int # endif + typedef ASN1_BIT_STRING OSSL_CMP_PKIFAILUREINFO; # define OSSL_CMP_CTX_FAILINFO_badAlg (1 << 0) @@ -194,18 +193,15 @@ typedef ASN1_BIT_STRING OSSL_CMP_PKIFAILUREINFO; * -- CertReqMsg * } */ -# define OSSL_CMP_PKISTATUS_request -3 -# define OSSL_CMP_PKISTATUS_trans -2 -# define OSSL_CMP_PKISTATUS_unspecified -1 -# define OSSL_CMP_PKISTATUS_accepted 0 -# define OSSL_CMP_PKISTATUS_grantedWithMods 1 -# define OSSL_CMP_PKISTATUS_rejection 2 -# define OSSL_CMP_PKISTATUS_waiting 3 -# define OSSL_CMP_PKISTATUS_revocationWarning 4 +# define OSSL_CMP_PKISTATUS_accepted 0 +# define OSSL_CMP_PKISTATUS_grantedWithMods 1 +# define OSSL_CMP_PKISTATUS_rejection 2 +# define OSSL_CMP_PKISTATUS_waiting 3 +# define OSSL_CMP_PKISTATUS_revocationWarning 4 # define OSSL_CMP_PKISTATUS_revocationNotification 5 -# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 -typedef ASN1_INTEGER OSSL_CMP_PKISTATUS; +# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 +typedef ASN1_INTEGER OSSL_CMP_PKISTATUS; DECLARE_ASN1_ITEM(OSSL_CMP_PKISTATUS) # define OSSL_CMP_CERTORENCCERT_CERTIFICATE 0 @@ -379,39 +375,18 @@ ASN1_TYPE *OSSL_CMP_ITAV_get0_value(const OSSL_CMP_ITAV *itav); int OSSL_CMP_ITAV_push0_stack_item(STACK_OF(OSSL_CMP_ITAV) **itav_sk_p, OSSL_CMP_ITAV *itav); void OSSL_CMP_ITAV_free(OSSL_CMP_ITAV *itav); - -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new0_certProfile(STACK_OF(ASN1_UTF8STRING) - *certProfile); -int OSSL_CMP_ITAV_get0_certProfile(const OSSL_CMP_ITAV *itav, - STACK_OF(ASN1_UTF8STRING) **out); -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_caCerts(const STACK_OF(X509) *caCerts); -int OSSL_CMP_ITAV_get0_caCerts(const OSSL_CMP_ITAV *itav, STACK_OF(X509) **out); - -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_rootCaCert(const X509 *rootCaCert); -int OSSL_CMP_ITAV_get0_rootCaCert(const OSSL_CMP_ITAV *itav, X509 **out); -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_rootCaKeyUpdate(const X509 *newWithNew, - const X509 *newWithOld, - const X509 *oldWithNew); -int OSSL_CMP_ITAV_get0_rootCaKeyUpdate(const OSSL_CMP_ITAV *itav, - X509 **newWithNew, - X509 **newWithOld, - X509 **oldWithNew); - void OSSL_CMP_MSG_free(OSSL_CMP_MSG *msg); /* from cmp_ctx.c */ OSSL_CMP_CTX *OSSL_CMP_CTX_new(OSSL_LIB_CTX *libctx, const char *propq); void OSSL_CMP_CTX_free(OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_reinit(OSSL_CMP_CTX *ctx); -OSSL_LIB_CTX *OSSL_CMP_CTX_get0_libctx(const OSSL_CMP_CTX *ctx); -const char *OSSL_CMP_CTX_get0_propq(const OSSL_CMP_CTX *ctx); /* CMP general options: */ # define OSSL_CMP_OPT_LOG_VERBOSITY 0 /* CMP transfer options: */ -# define OSSL_CMP_OPT_KEEP_ALIVE 10 -# define OSSL_CMP_OPT_MSG_TIMEOUT 11 +# define OSSL_CMP_OPT_KEEP_ALIVE 10 +# define OSSL_CMP_OPT_MSG_TIMEOUT 11 # define OSSL_CMP_OPT_TOTAL_TIMEOUT 12 -# define OSSL_CMP_OPT_USE_TLS 13 /* CMP request options: */ # define OSSL_CMP_OPT_VALIDITY_DAYS 20 # define OSSL_CMP_OPT_SUBJECTALTNAME_NODEFAULT 21 @@ -429,7 +404,6 @@ const char *OSSL_CMP_CTX_get0_propq(const OSSL_CMP_CTX *ctx); # define OSSL_CMP_OPT_DIGEST_ALGNID 34 # define OSSL_CMP_OPT_IGNORE_KEYUSAGE 35 # define OSSL_CMP_OPT_PERMIT_TA_IN_EXTRACERTS_FOR_IR 36 -# define OSSL_CMP_OPT_NO_CACHE_EXTRACERTS 37 int OSSL_CMP_CTX_set_option(OSSL_CMP_CTX *ctx, int opt, int val); int OSSL_CMP_CTX_get_option(const OSSL_CMP_CTX *ctx, int opt); /* CMP-specific callback for logging and outputting the error queue: */ @@ -443,11 +417,9 @@ int OSSL_CMP_CTX_set1_server(OSSL_CMP_CTX *ctx, const char *address); int OSSL_CMP_CTX_set_serverPort(OSSL_CMP_CTX *ctx, int port); int OSSL_CMP_CTX_set1_proxy(OSSL_CMP_CTX *ctx, const char *name); int OSSL_CMP_CTX_set1_no_proxy(OSSL_CMP_CTX *ctx, const char *names); -# ifndef OPENSSL_NO_HTTP int OSSL_CMP_CTX_set_http_cb(OSSL_CMP_CTX *ctx, OSSL_HTTP_bio_cb_t cb); int OSSL_CMP_CTX_set_http_cb_arg(OSSL_CMP_CTX *ctx, void *arg); void *OSSL_CMP_CTX_get_http_cb_arg(const OSSL_CMP_CTX *ctx); -# endif typedef OSSL_CMP_MSG *(*OSSL_CMP_transfer_cb_t) (OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req); int OSSL_CMP_CTX_set_transfer_cb(OSSL_CMP_CTX *ctx, OSSL_CMP_transfer_cb_t cb); @@ -457,9 +429,7 @@ void *OSSL_CMP_CTX_get_transfer_cb_arg(const OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_set1_srvCert(OSSL_CMP_CTX *ctx, X509 *cert); int OSSL_CMP_CTX_set1_expected_sender(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_set0_trustedStore(OSSL_CMP_CTX *ctx, X509_STORE *store); -# define OSSL_CMP_CTX_set0_trusted OSSL_CMP_CTX_set0_trustedStore X509_STORE *OSSL_CMP_CTX_get0_trustedStore(const OSSL_CMP_CTX *ctx); -# define OSSL_CMP_CTX_get0_trusted OSSL_CMP_CTX_get0_trustedStore int OSSL_CMP_CTX_set1_untrusted(OSSL_CMP_CTX *ctx, STACK_OF(X509) *certs); STACK_OF(X509) *OSSL_CMP_CTX_get0_untrusted(const OSSL_CMP_CTX *ctx); /* client authentication: */ @@ -469,21 +439,17 @@ int OSSL_CMP_CTX_build_cert_chain(OSSL_CMP_CTX *ctx, X509_STORE *own_trusted, int OSSL_CMP_CTX_set1_pkey(OSSL_CMP_CTX *ctx, EVP_PKEY *pkey); int OSSL_CMP_CTX_set1_referenceValue(OSSL_CMP_CTX *ctx, const unsigned char *ref, int len); -int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, - const unsigned char *sec, int len); +int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, const unsigned char *sec, + const int len); /* CMP message header and extra certificates: */ int OSSL_CMP_CTX_set1_recipient(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_push0_geninfo_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav); -int OSSL_CMP_CTX_reset_geninfo_ITAVs(OSSL_CMP_CTX *ctx); -STACK_OF(OSSL_CMP_ITAV) - *OSSL_CMP_CTX_get0_geninfo_ITAVs(const OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_set1_extraCertsOut(OSSL_CMP_CTX *ctx, STACK_OF(X509) *extraCertsOut); /* certificate template: */ int OSSL_CMP_CTX_set0_newPkey(OSSL_CMP_CTX *ctx, int priv, EVP_PKEY *pkey); EVP_PKEY *OSSL_CMP_CTX_get0_newPkey(const OSSL_CMP_CTX *ctx, int priv); int OSSL_CMP_CTX_set1_issuer(OSSL_CMP_CTX *ctx, const X509_NAME *name); -int OSSL_CMP_CTX_set1_serialNumber(OSSL_CMP_CTX *ctx, const ASN1_INTEGER *sn); int OSSL_CMP_CTX_set1_subjectName(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_push1_subjectAltName(OSSL_CMP_CTX *ctx, const GENERAL_NAME *name); @@ -507,7 +473,6 @@ int OSSL_CMP_CTX_get_status(const OSSL_CMP_CTX *ctx); OSSL_CMP_PKIFREETEXT *OSSL_CMP_CTX_get0_statusString(const OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_get_failInfoCode(const OSSL_CMP_CTX *ctx); # define OSSL_CMP_PKISI_BUFLEN 1024 -X509 *OSSL_CMP_CTX_get0_validatedSrvCert(const OSSL_CMP_CTX *ctx); X509 *OSSL_CMP_CTX_get0_newCert(const OSSL_CMP_CTX *ctx); STACK_OF(X509) *OSSL_CMP_CTX_get1_newChain(const OSSL_CMP_CTX *ctx); STACK_OF(X509) *OSSL_CMP_CTX_get1_caPubs(const OSSL_CMP_CTX *ctx); @@ -529,15 +494,11 @@ OSSL_CMP_STATUSINFO_new(int status, int fail_info, const char *text); ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_transactionID(const OSSL_CMP_PKIHEADER *hdr); ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_recipNonce(const OSSL_CMP_PKIHEADER *hdr); -STACK_OF(OSSL_CMP_ITAV) - *OSSL_CMP_HDR_get0_geninfo_ITAVs(const OSSL_CMP_PKIHEADER *hdr); /* from cmp_msg.c */ OSSL_CMP_PKIHEADER *OSSL_CMP_MSG_get0_header(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_get_bodytype(const OSSL_CMP_MSG *msg); -X509_PUBKEY *OSSL_CMP_MSG_get0_certreq_publickey(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_update_transactionID(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); -int OSSL_CMP_MSG_update_recipNonce(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid); OSSL_CMP_MSG *OSSL_CMP_MSG_read(const char *file, OSSL_LIB_CTX *libctx, const char *propq); @@ -551,10 +512,8 @@ int OSSL_CMP_validate_cert_path(const OSSL_CMP_CTX *ctx, X509_STORE *trusted_store, X509 *cert); /* from cmp_http.c */ -# ifndef OPENSSL_NO_HTTP OSSL_CMP_MSG *OSSL_CMP_MSG_http_perform(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req); -# endif /* from cmp_server.c */ typedef struct ossl_cmp_srv_ctx_st OSSL_CMP_SRV_CTX; @@ -597,13 +556,6 @@ int OSSL_CMP_SRV_CTX_init(OSSL_CMP_SRV_CTX *srv_ctx, void *custom_ctx, OSSL_CMP_SRV_error_cb_t process_error, OSSL_CMP_SRV_certConf_cb_t process_certConf, OSSL_CMP_SRV_pollReq_cb_t process_pollReq); -typedef int (*OSSL_CMP_SRV_delayed_delivery_cb_t)(OSSL_CMP_SRV_CTX *srv_ctx, - const OSSL_CMP_MSG *req); -typedef int (*OSSL_CMP_SRV_clean_transaction_cb_t)(OSSL_CMP_SRV_CTX *srv_ctx, - const ASN1_OCTET_STRING *id); -int OSSL_CMP_SRV_CTX_init_trans(OSSL_CMP_SRV_CTX *srv_ctx, - OSSL_CMP_SRV_delayed_delivery_cb_t delay, - OSSL_CMP_SRV_clean_transaction_cb_t clean); OSSL_CMP_CTX *OSSL_CMP_SRV_CTX_get0_cmp_ctx(const OSSL_CMP_SRV_CTX *srv_ctx); void *OSSL_CMP_SRV_CTX_get0_custom_ctx(const OSSL_CMP_SRV_CTX *srv_ctx); int OSSL_CMP_SRV_CTX_set_send_unprotected_errors(OSSL_CMP_SRV_CTX *srv_ctx, @@ -620,8 +572,6 @@ X509 *OSSL_CMP_exec_certreq(OSSL_CMP_CTX *ctx, int req_type, # define OSSL_CMP_CR 2 # define OSSL_CMP_P10CR 4 # define OSSL_CMP_KUR 7 -# define OSSL_CMP_GENM 21 -# define OSSL_CMP_ERROR 23 # define OSSL_CMP_exec_IR_ses(ctx) \ OSSL_CMP_exec_certreq(ctx, OSSL_CMP_IR, NULL) # define OSSL_CMP_exec_CR_ses(ctx) \ @@ -635,12 +585,6 @@ int OSSL_CMP_try_certreq(OSSL_CMP_CTX *ctx, int req_type, int OSSL_CMP_exec_RR_ses(OSSL_CMP_CTX *ctx); STACK_OF(OSSL_CMP_ITAV) *OSSL_CMP_exec_GENM_ses(OSSL_CMP_CTX *ctx); -/* from cmp_genm.c */ -int OSSL_CMP_get1_caCerts(OSSL_CMP_CTX *ctx, STACK_OF(X509) **out); -int OSSL_CMP_get1_rootCaKeyUpdate(OSSL_CMP_CTX *ctx, - const X509 *oldWithOld, X509 **newWithNew, - X509 **newWithOld, X509 **oldWithNew); - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/cmp.h.in b/openssl/include/openssl/cmp.h.in index ad9eb3429..b47344215 100644 --- a/openssl/include/openssl/cmp.h.in +++ b/openssl/include/openssl/cmp.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 2007-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -36,9 +36,7 @@ use OpenSSL::stackhash qw(generate_stack_macros); extern "C" { # endif -# define OSSL_CMP_PVNO_2 2 -# define OSSL_CMP_PVNO_3 3 -# define OSSL_CMP_PVNO OSSL_CMP_PVNO_2 /* v2 is the default */ +# define OSSL_CMP_PVNO 2 /*- * PKIFailureInfo ::= BIT STRING { @@ -140,6 +138,7 @@ extern "C" { # if OSSL_CMP_PKIFAILUREINFO_MAX_BIT_PATTERN > INT_MAX # error CMP_PKIFAILUREINFO_MAX bit pattern does not fit in type int # endif + typedef ASN1_BIT_STRING OSSL_CMP_PKIFAILUREINFO; # define OSSL_CMP_CTX_FAILINFO_badAlg (1 << 0) @@ -195,18 +194,15 @@ typedef ASN1_BIT_STRING OSSL_CMP_PKIFAILUREINFO; * -- CertReqMsg * } */ -# define OSSL_CMP_PKISTATUS_request -3 -# define OSSL_CMP_PKISTATUS_trans -2 -# define OSSL_CMP_PKISTATUS_unspecified -1 -# define OSSL_CMP_PKISTATUS_accepted 0 -# define OSSL_CMP_PKISTATUS_grantedWithMods 1 -# define OSSL_CMP_PKISTATUS_rejection 2 -# define OSSL_CMP_PKISTATUS_waiting 3 -# define OSSL_CMP_PKISTATUS_revocationWarning 4 +# define OSSL_CMP_PKISTATUS_accepted 0 +# define OSSL_CMP_PKISTATUS_grantedWithMods 1 +# define OSSL_CMP_PKISTATUS_rejection 2 +# define OSSL_CMP_PKISTATUS_waiting 3 +# define OSSL_CMP_PKISTATUS_revocationWarning 4 # define OSSL_CMP_PKISTATUS_revocationNotification 5 -# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 -typedef ASN1_INTEGER OSSL_CMP_PKISTATUS; +# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 +typedef ASN1_INTEGER OSSL_CMP_PKISTATUS; DECLARE_ASN1_ITEM(OSSL_CMP_PKISTATUS) # define OSSL_CMP_CERTORENCCERT_CERTIFICATE 0 @@ -260,39 +256,18 @@ ASN1_TYPE *OSSL_CMP_ITAV_get0_value(const OSSL_CMP_ITAV *itav); int OSSL_CMP_ITAV_push0_stack_item(STACK_OF(OSSL_CMP_ITAV) **itav_sk_p, OSSL_CMP_ITAV *itav); void OSSL_CMP_ITAV_free(OSSL_CMP_ITAV *itav); - -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new0_certProfile(STACK_OF(ASN1_UTF8STRING) - *certProfile); -int OSSL_CMP_ITAV_get0_certProfile(const OSSL_CMP_ITAV *itav, - STACK_OF(ASN1_UTF8STRING) **out); -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_caCerts(const STACK_OF(X509) *caCerts); -int OSSL_CMP_ITAV_get0_caCerts(const OSSL_CMP_ITAV *itav, STACK_OF(X509) **out); - -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_rootCaCert(const X509 *rootCaCert); -int OSSL_CMP_ITAV_get0_rootCaCert(const OSSL_CMP_ITAV *itav, X509 **out); -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_rootCaKeyUpdate(const X509 *newWithNew, - const X509 *newWithOld, - const X509 *oldWithNew); -int OSSL_CMP_ITAV_get0_rootCaKeyUpdate(const OSSL_CMP_ITAV *itav, - X509 **newWithNew, - X509 **newWithOld, - X509 **oldWithNew); - void OSSL_CMP_MSG_free(OSSL_CMP_MSG *msg); /* from cmp_ctx.c */ OSSL_CMP_CTX *OSSL_CMP_CTX_new(OSSL_LIB_CTX *libctx, const char *propq); void OSSL_CMP_CTX_free(OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_reinit(OSSL_CMP_CTX *ctx); -OSSL_LIB_CTX *OSSL_CMP_CTX_get0_libctx(const OSSL_CMP_CTX *ctx); -const char *OSSL_CMP_CTX_get0_propq(const OSSL_CMP_CTX *ctx); /* CMP general options: */ # define OSSL_CMP_OPT_LOG_VERBOSITY 0 /* CMP transfer options: */ -# define OSSL_CMP_OPT_KEEP_ALIVE 10 -# define OSSL_CMP_OPT_MSG_TIMEOUT 11 +# define OSSL_CMP_OPT_KEEP_ALIVE 10 +# define OSSL_CMP_OPT_MSG_TIMEOUT 11 # define OSSL_CMP_OPT_TOTAL_TIMEOUT 12 -# define OSSL_CMP_OPT_USE_TLS 13 /* CMP request options: */ # define OSSL_CMP_OPT_VALIDITY_DAYS 20 # define OSSL_CMP_OPT_SUBJECTALTNAME_NODEFAULT 21 @@ -310,7 +285,6 @@ const char *OSSL_CMP_CTX_get0_propq(const OSSL_CMP_CTX *ctx); # define OSSL_CMP_OPT_DIGEST_ALGNID 34 # define OSSL_CMP_OPT_IGNORE_KEYUSAGE 35 # define OSSL_CMP_OPT_PERMIT_TA_IN_EXTRACERTS_FOR_IR 36 -# define OSSL_CMP_OPT_NO_CACHE_EXTRACERTS 37 int OSSL_CMP_CTX_set_option(OSSL_CMP_CTX *ctx, int opt, int val); int OSSL_CMP_CTX_get_option(const OSSL_CMP_CTX *ctx, int opt); /* CMP-specific callback for logging and outputting the error queue: */ @@ -324,11 +298,9 @@ int OSSL_CMP_CTX_set1_server(OSSL_CMP_CTX *ctx, const char *address); int OSSL_CMP_CTX_set_serverPort(OSSL_CMP_CTX *ctx, int port); int OSSL_CMP_CTX_set1_proxy(OSSL_CMP_CTX *ctx, const char *name); int OSSL_CMP_CTX_set1_no_proxy(OSSL_CMP_CTX *ctx, const char *names); -# ifndef OPENSSL_NO_HTTP int OSSL_CMP_CTX_set_http_cb(OSSL_CMP_CTX *ctx, OSSL_HTTP_bio_cb_t cb); int OSSL_CMP_CTX_set_http_cb_arg(OSSL_CMP_CTX *ctx, void *arg); void *OSSL_CMP_CTX_get_http_cb_arg(const OSSL_CMP_CTX *ctx); -# endif typedef OSSL_CMP_MSG *(*OSSL_CMP_transfer_cb_t) (OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req); int OSSL_CMP_CTX_set_transfer_cb(OSSL_CMP_CTX *ctx, OSSL_CMP_transfer_cb_t cb); @@ -338,9 +310,7 @@ void *OSSL_CMP_CTX_get_transfer_cb_arg(const OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_set1_srvCert(OSSL_CMP_CTX *ctx, X509 *cert); int OSSL_CMP_CTX_set1_expected_sender(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_set0_trustedStore(OSSL_CMP_CTX *ctx, X509_STORE *store); -# define OSSL_CMP_CTX_set0_trusted OSSL_CMP_CTX_set0_trustedStore X509_STORE *OSSL_CMP_CTX_get0_trustedStore(const OSSL_CMP_CTX *ctx); -# define OSSL_CMP_CTX_get0_trusted OSSL_CMP_CTX_get0_trustedStore int OSSL_CMP_CTX_set1_untrusted(OSSL_CMP_CTX *ctx, STACK_OF(X509) *certs); STACK_OF(X509) *OSSL_CMP_CTX_get0_untrusted(const OSSL_CMP_CTX *ctx); /* client authentication: */ @@ -350,21 +320,17 @@ int OSSL_CMP_CTX_build_cert_chain(OSSL_CMP_CTX *ctx, X509_STORE *own_trusted, int OSSL_CMP_CTX_set1_pkey(OSSL_CMP_CTX *ctx, EVP_PKEY *pkey); int OSSL_CMP_CTX_set1_referenceValue(OSSL_CMP_CTX *ctx, const unsigned char *ref, int len); -int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, - const unsigned char *sec, int len); +int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, const unsigned char *sec, + const int len); /* CMP message header and extra certificates: */ int OSSL_CMP_CTX_set1_recipient(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_push0_geninfo_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav); -int OSSL_CMP_CTX_reset_geninfo_ITAVs(OSSL_CMP_CTX *ctx); -STACK_OF(OSSL_CMP_ITAV) - *OSSL_CMP_CTX_get0_geninfo_ITAVs(const OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_set1_extraCertsOut(OSSL_CMP_CTX *ctx, STACK_OF(X509) *extraCertsOut); /* certificate template: */ int OSSL_CMP_CTX_set0_newPkey(OSSL_CMP_CTX *ctx, int priv, EVP_PKEY *pkey); EVP_PKEY *OSSL_CMP_CTX_get0_newPkey(const OSSL_CMP_CTX *ctx, int priv); int OSSL_CMP_CTX_set1_issuer(OSSL_CMP_CTX *ctx, const X509_NAME *name); -int OSSL_CMP_CTX_set1_serialNumber(OSSL_CMP_CTX *ctx, const ASN1_INTEGER *sn); int OSSL_CMP_CTX_set1_subjectName(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_push1_subjectAltName(OSSL_CMP_CTX *ctx, const GENERAL_NAME *name); @@ -388,7 +354,6 @@ int OSSL_CMP_CTX_get_status(const OSSL_CMP_CTX *ctx); OSSL_CMP_PKIFREETEXT *OSSL_CMP_CTX_get0_statusString(const OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_get_failInfoCode(const OSSL_CMP_CTX *ctx); # define OSSL_CMP_PKISI_BUFLEN 1024 -X509 *OSSL_CMP_CTX_get0_validatedSrvCert(const OSSL_CMP_CTX *ctx); X509 *OSSL_CMP_CTX_get0_newCert(const OSSL_CMP_CTX *ctx); STACK_OF(X509) *OSSL_CMP_CTX_get1_newChain(const OSSL_CMP_CTX *ctx); STACK_OF(X509) *OSSL_CMP_CTX_get1_caPubs(const OSSL_CMP_CTX *ctx); @@ -410,15 +375,11 @@ OSSL_CMP_STATUSINFO_new(int status, int fail_info, const char *text); ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_transactionID(const OSSL_CMP_PKIHEADER *hdr); ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_recipNonce(const OSSL_CMP_PKIHEADER *hdr); -STACK_OF(OSSL_CMP_ITAV) - *OSSL_CMP_HDR_get0_geninfo_ITAVs(const OSSL_CMP_PKIHEADER *hdr); /* from cmp_msg.c */ OSSL_CMP_PKIHEADER *OSSL_CMP_MSG_get0_header(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_get_bodytype(const OSSL_CMP_MSG *msg); -X509_PUBKEY *OSSL_CMP_MSG_get0_certreq_publickey(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_update_transactionID(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); -int OSSL_CMP_MSG_update_recipNonce(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid); OSSL_CMP_MSG *OSSL_CMP_MSG_read(const char *file, OSSL_LIB_CTX *libctx, const char *propq); @@ -432,10 +393,8 @@ int OSSL_CMP_validate_cert_path(const OSSL_CMP_CTX *ctx, X509_STORE *trusted_store, X509 *cert); /* from cmp_http.c */ -# ifndef OPENSSL_NO_HTTP OSSL_CMP_MSG *OSSL_CMP_MSG_http_perform(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req); -# endif /* from cmp_server.c */ typedef struct ossl_cmp_srv_ctx_st OSSL_CMP_SRV_CTX; @@ -478,13 +437,6 @@ int OSSL_CMP_SRV_CTX_init(OSSL_CMP_SRV_CTX *srv_ctx, void *custom_ctx, OSSL_CMP_SRV_error_cb_t process_error, OSSL_CMP_SRV_certConf_cb_t process_certConf, OSSL_CMP_SRV_pollReq_cb_t process_pollReq); -typedef int (*OSSL_CMP_SRV_delayed_delivery_cb_t)(OSSL_CMP_SRV_CTX *srv_ctx, - const OSSL_CMP_MSG *req); -typedef int (*OSSL_CMP_SRV_clean_transaction_cb_t)(OSSL_CMP_SRV_CTX *srv_ctx, - const ASN1_OCTET_STRING *id); -int OSSL_CMP_SRV_CTX_init_trans(OSSL_CMP_SRV_CTX *srv_ctx, - OSSL_CMP_SRV_delayed_delivery_cb_t delay, - OSSL_CMP_SRV_clean_transaction_cb_t clean); OSSL_CMP_CTX *OSSL_CMP_SRV_CTX_get0_cmp_ctx(const OSSL_CMP_SRV_CTX *srv_ctx); void *OSSL_CMP_SRV_CTX_get0_custom_ctx(const OSSL_CMP_SRV_CTX *srv_ctx); int OSSL_CMP_SRV_CTX_set_send_unprotected_errors(OSSL_CMP_SRV_CTX *srv_ctx, @@ -501,8 +453,6 @@ X509 *OSSL_CMP_exec_certreq(OSSL_CMP_CTX *ctx, int req_type, # define OSSL_CMP_CR 2 # define OSSL_CMP_P10CR 4 # define OSSL_CMP_KUR 7 -# define OSSL_CMP_GENM 21 -# define OSSL_CMP_ERROR 23 # define OSSL_CMP_exec_IR_ses(ctx) \ OSSL_CMP_exec_certreq(ctx, OSSL_CMP_IR, NULL) # define OSSL_CMP_exec_CR_ses(ctx) \ @@ -516,12 +466,6 @@ int OSSL_CMP_try_certreq(OSSL_CMP_CTX *ctx, int req_type, int OSSL_CMP_exec_RR_ses(OSSL_CMP_CTX *ctx); STACK_OF(OSSL_CMP_ITAV) *OSSL_CMP_exec_GENM_ses(OSSL_CMP_CTX *ctx); -/* from cmp_genm.c */ -int OSSL_CMP_get1_caCerts(OSSL_CMP_CTX *ctx, STACK_OF(X509) **out); -int OSSL_CMP_get1_rootCaKeyUpdate(OSSL_CMP_CTX *ctx, - const X509 *oldWithOld, X509 **newWithNew, - X509 **newWithOld, X509 **oldWithNew); - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/cmperr.h b/openssl/include/openssl/cmperr.h index 0d876e501..3a26fd0bd 100644 --- a/openssl/include/openssl/cmperr.h +++ b/openssl/include/openssl/cmperr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -55,29 +55,21 @@ # define CMP_R_ERROR_UNEXPECTED_CERTCONF 160 # define CMP_R_ERROR_VALIDATING_PROTECTION 140 # define CMP_R_ERROR_VALIDATING_SIGNATURE 171 -# define CMP_R_EXPECTED_POLLREQ 104 # define CMP_R_FAILED_BUILDING_OWN_CHAIN 164 # define CMP_R_FAILED_EXTRACTING_PUBKEY 141 # define CMP_R_FAILURE_OBTAINING_RANDOM 110 # define CMP_R_FAIL_INFO_OUT_OF_RANGE 129 -# define CMP_R_GETTING_GENP 192 # define CMP_R_INVALID_ARGS 100 -# define CMP_R_INVALID_GENP 193 # define CMP_R_INVALID_OPTION 174 -# define CMP_R_INVALID_ROOTCAKEYUPDATE 195 # define CMP_R_MISSING_CERTID 165 # define CMP_R_MISSING_KEY_INPUT_FOR_CREATING_PROTECTION 130 # define CMP_R_MISSING_KEY_USAGE_DIGITALSIGNATURE 142 # define CMP_R_MISSING_P10CSR 121 # define CMP_R_MISSING_PBM_SECRET 166 # define CMP_R_MISSING_PRIVATE_KEY 131 -# define CMP_R_MISSING_PRIVATE_KEY_FOR_POPO 190 # define CMP_R_MISSING_PROTECTION 143 -# define CMP_R_MISSING_PUBLIC_KEY 183 # define CMP_R_MISSING_REFERENCE_CERT 168 -# define CMP_R_MISSING_SECRET 178 # define CMP_R_MISSING_SENDER_IDENTIFICATION 111 -# define CMP_R_MISSING_TRUST_ANCHOR 179 # define CMP_R_MISSING_TRUST_STORE 144 # define CMP_R_MULTIPLE_REQUESTS_NOT_SUPPORTED 161 # define CMP_R_MULTIPLE_RESPONSES_NOT_SUPPORTED 170 @@ -98,19 +90,14 @@ # define CMP_R_TOTAL_TIMEOUT 184 # define CMP_R_TRANSACTIONID_UNMATCHED 152 # define CMP_R_TRANSFER_ERROR 159 -# define CMP_R_UNCLEAN_CTX 191 -# define CMP_R_UNEXPECTED_CERTPROFILE 196 # define CMP_R_UNEXPECTED_PKIBODY 133 # define CMP_R_UNEXPECTED_PKISTATUS 185 -# define CMP_R_UNEXPECTED_POLLREQ 105 # define CMP_R_UNEXPECTED_PVNO 153 -# define CMP_R_UNEXPECTED_SENDER 106 # define CMP_R_UNKNOWN_ALGORITHM_ID 134 # define CMP_R_UNKNOWN_CERT_TYPE 135 # define CMP_R_UNKNOWN_PKISTATUS 186 # define CMP_R_UNSUPPORTED_ALGORITHM 136 # define CMP_R_UNSUPPORTED_KEY_TYPE 137 -# define CMP_R_UNSUPPORTED_PKIBODY 101 # define CMP_R_UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC 154 # define CMP_R_VALUE_TOO_LARGE 175 # define CMP_R_VALUE_TOO_SMALL 177 diff --git a/openssl/include/openssl/cms.h b/openssl/include/openssl/cms.h index 2d64912a4..c43a4c106 100644 --- a/openssl/include/openssl/cms.h +++ b/openssl/include/openssl/cms.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/cms.h.in * - * Copyright 2008-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -31,10 +31,8 @@ extern "C" { # endif -typedef struct CMS_EnvelopedData_st CMS_EnvelopedData; typedef struct CMS_ContentInfo_st CMS_ContentInfo; typedef struct CMS_SignerInfo_st CMS_SignerInfo; -typedef struct CMS_SignedData_st CMS_SignedData; typedef struct CMS_CertificateChoices CMS_CertificateChoices; typedef struct CMS_RevocationInfoChoice_st CMS_RevocationInfoChoice; typedef struct CMS_RecipientInfo_st CMS_RecipientInfo; @@ -149,8 +147,6 @@ SKM_DEFINE_STACK_OF_INTERNAL(CMS_RevocationInfoChoice, CMS_RevocationInfoChoice, #define sk_CMS_RevocationInfoChoice_set_cmp_func(sk, cmp) ((sk_CMS_RevocationInfoChoice_compfunc)OPENSSL_sk_set_cmp_func(ossl_check_CMS_RevocationInfoChoice_sk_type(sk), ossl_check_CMS_RevocationInfoChoice_compfunc_type(cmp))) -DECLARE_ASN1_ITEM(CMS_EnvelopedData) -DECLARE_ASN1_ALLOC_FUNCTIONS(CMS_SignedData) DECLARE_ASN1_FUNCTIONS(CMS_ContentInfo) DECLARE_ASN1_FUNCTIONS(CMS_ReceiptRequest) DECLARE_ASN1_PRINT_FUNCTION(CMS_ContentInfo) @@ -221,16 +217,13 @@ int SMIME_write_CMS(BIO *bio, CMS_ContentInfo *cms, BIO *data, int flags); int CMS_final(CMS_ContentInfo *cms, BIO *data, BIO *dcont, unsigned int flags); -int CMS_final_digest(CMS_ContentInfo *cms, - const unsigned char *md, unsigned int mdlen, BIO *dcont, - unsigned int flags); CMS_ContentInfo *CMS_sign(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs, BIO *data, unsigned int flags); CMS_ContentInfo *CMS_sign_ex(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs, BIO *data, - unsigned int flags, OSSL_LIB_CTX *libctx, + unsigned int flags, OSSL_LIB_CTX *ctx, const char *propq); CMS_ContentInfo *CMS_sign_receipt(CMS_SignerInfo *si, @@ -240,26 +233,27 @@ CMS_ContentInfo *CMS_sign_receipt(CMS_SignerInfo *si, int CMS_data(CMS_ContentInfo *cms, BIO *out, unsigned int flags); CMS_ContentInfo *CMS_data_create(BIO *in, unsigned int flags); CMS_ContentInfo *CMS_data_create_ex(BIO *in, unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq); + OSSL_LIB_CTX *ctx, const char *propq); int CMS_digest_verify(CMS_ContentInfo *cms, BIO *dcont, BIO *out, unsigned int flags); CMS_ContentInfo *CMS_digest_create(BIO *in, const EVP_MD *md, unsigned int flags); CMS_ContentInfo *CMS_digest_create_ex(BIO *in, const EVP_MD *md, - unsigned int flags, OSSL_LIB_CTX *libctx, + unsigned int flags, OSSL_LIB_CTX *ctx, const char *propq); int CMS_EncryptedData_decrypt(CMS_ContentInfo *cms, const unsigned char *key, size_t keylen, BIO *dcont, BIO *out, unsigned int flags); + CMS_ContentInfo *CMS_EncryptedData_encrypt(BIO *in, const EVP_CIPHER *cipher, const unsigned char *key, size_t keylen, unsigned int flags); CMS_ContentInfo *CMS_EncryptedData_encrypt_ex(BIO *in, const EVP_CIPHER *cipher, const unsigned char *key, size_t keylen, unsigned int flags, - OSSL_LIB_CTX *libctx, + OSSL_LIB_CTX *ctx, const char *propq); int CMS_EncryptedData_set1_key(CMS_ContentInfo *cms, const EVP_CIPHER *ciph, @@ -278,7 +272,7 @@ CMS_ContentInfo *CMS_encrypt(STACK_OF(X509) *certs, BIO *in, const EVP_CIPHER *cipher, unsigned int flags); CMS_ContentInfo *CMS_encrypt_ex(STACK_OF(X509) *certs, BIO *in, const EVP_CIPHER *cipher, unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq); + OSSL_LIB_CTX *ctx, const char *propq); int CMS_decrypt(CMS_ContentInfo *cms, EVP_PKEY *pkey, X509 *cert, BIO *dcont, BIO *out, unsigned int flags); @@ -297,16 +291,12 @@ int CMS_RecipientInfo_type(CMS_RecipientInfo *ri); EVP_PKEY_CTX *CMS_RecipientInfo_get0_pkey_ctx(CMS_RecipientInfo *ri); CMS_ContentInfo *CMS_AuthEnvelopedData_create(const EVP_CIPHER *cipher); CMS_ContentInfo * -CMS_AuthEnvelopedData_create_ex(const EVP_CIPHER *cipher, OSSL_LIB_CTX *libctx, +CMS_AuthEnvelopedData_create_ex(const EVP_CIPHER *cipher, OSSL_LIB_CTX *ctx, const char *propq); CMS_ContentInfo *CMS_EnvelopedData_create(const EVP_CIPHER *cipher); CMS_ContentInfo *CMS_EnvelopedData_create_ex(const EVP_CIPHER *cipher, - OSSL_LIB_CTX *libctx, + OSSL_LIB_CTX *ctx, const char *propq); -BIO *CMS_EnvelopedData_decrypt(CMS_EnvelopedData *env, BIO *detached_data, - EVP_PKEY *pkey, X509 *cert, - ASN1_OCTET_STRING *secret, unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq); CMS_RecipientInfo *CMS_add1_recipient_cert(CMS_ContentInfo *cms, X509 *recip, unsigned int flags); @@ -395,11 +385,6 @@ ASN1_OCTET_STRING *CMS_SignerInfo_get0_signature(CMS_SignerInfo *si); int CMS_SignerInfo_sign(CMS_SignerInfo *si); int CMS_SignerInfo_verify(CMS_SignerInfo *si); int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain); -BIO *CMS_SignedData_verify(CMS_SignedData *sd, BIO *detached_data, - STACK_OF(X509) *scerts, X509_STORE *store, - STACK_OF(X509) *extra, STACK_OF(X509_CRL) *crls, - unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq); int CMS_add_smimecap(CMS_SignerInfo *si, STACK_OF(X509_ALGOR) *algs); int CMS_add_simple_smimecap(STACK_OF(X509_ALGOR) **algs, @@ -456,7 +441,7 @@ CMS_ReceiptRequest *CMS_ReceiptRequest_create0_ex( unsigned char *id, int idlen, int allorfirst, STACK_OF(GENERAL_NAMES) *receiptList, STACK_OF(GENERAL_NAMES) *receiptsTo, - OSSL_LIB_CTX *libctx); + OSSL_LIB_CTX *ctx); int CMS_add1_ReceiptRequest(CMS_SignerInfo *si, CMS_ReceiptRequest *rr); void CMS_ReceiptRequest_get0_values(CMS_ReceiptRequest *rr, diff --git a/openssl/include/openssl/cms.h.in b/openssl/include/openssl/cms.h.in index 239667700..da20ddf2f 100644 --- a/openssl/include/openssl/cms.h.in +++ b/openssl/include/openssl/cms.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 2008-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -32,10 +32,8 @@ use OpenSSL::stackhash qw(generate_stack_macros); extern "C" { # endif -typedef struct CMS_EnvelopedData_st CMS_EnvelopedData; typedef struct CMS_ContentInfo_st CMS_ContentInfo; typedef struct CMS_SignerInfo_st CMS_SignerInfo; -typedef struct CMS_SignedData_st CMS_SignedData; typedef struct CMS_CertificateChoices CMS_CertificateChoices; typedef struct CMS_RevocationInfoChoice_st CMS_RevocationInfoChoice; typedef struct CMS_RecipientInfo_st CMS_RecipientInfo; @@ -51,8 +49,6 @@ typedef struct CMS_OtherKeyAttribute_st CMS_OtherKeyAttribute; .generate_stack_macros("CMS_RevocationInfoChoice"); -} -DECLARE_ASN1_ITEM(CMS_EnvelopedData) -DECLARE_ASN1_ALLOC_FUNCTIONS(CMS_SignedData) DECLARE_ASN1_FUNCTIONS(CMS_ContentInfo) DECLARE_ASN1_FUNCTIONS(CMS_ReceiptRequest) DECLARE_ASN1_PRINT_FUNCTION(CMS_ContentInfo) @@ -123,16 +119,13 @@ int SMIME_write_CMS(BIO *bio, CMS_ContentInfo *cms, BIO *data, int flags); int CMS_final(CMS_ContentInfo *cms, BIO *data, BIO *dcont, unsigned int flags); -int CMS_final_digest(CMS_ContentInfo *cms, - const unsigned char *md, unsigned int mdlen, BIO *dcont, - unsigned int flags); CMS_ContentInfo *CMS_sign(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs, BIO *data, unsigned int flags); CMS_ContentInfo *CMS_sign_ex(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs, BIO *data, - unsigned int flags, OSSL_LIB_CTX *libctx, + unsigned int flags, OSSL_LIB_CTX *ctx, const char *propq); CMS_ContentInfo *CMS_sign_receipt(CMS_SignerInfo *si, @@ -142,26 +135,27 @@ CMS_ContentInfo *CMS_sign_receipt(CMS_SignerInfo *si, int CMS_data(CMS_ContentInfo *cms, BIO *out, unsigned int flags); CMS_ContentInfo *CMS_data_create(BIO *in, unsigned int flags); CMS_ContentInfo *CMS_data_create_ex(BIO *in, unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq); + OSSL_LIB_CTX *ctx, const char *propq); int CMS_digest_verify(CMS_ContentInfo *cms, BIO *dcont, BIO *out, unsigned int flags); CMS_ContentInfo *CMS_digest_create(BIO *in, const EVP_MD *md, unsigned int flags); CMS_ContentInfo *CMS_digest_create_ex(BIO *in, const EVP_MD *md, - unsigned int flags, OSSL_LIB_CTX *libctx, + unsigned int flags, OSSL_LIB_CTX *ctx, const char *propq); int CMS_EncryptedData_decrypt(CMS_ContentInfo *cms, const unsigned char *key, size_t keylen, BIO *dcont, BIO *out, unsigned int flags); + CMS_ContentInfo *CMS_EncryptedData_encrypt(BIO *in, const EVP_CIPHER *cipher, const unsigned char *key, size_t keylen, unsigned int flags); CMS_ContentInfo *CMS_EncryptedData_encrypt_ex(BIO *in, const EVP_CIPHER *cipher, const unsigned char *key, size_t keylen, unsigned int flags, - OSSL_LIB_CTX *libctx, + OSSL_LIB_CTX *ctx, const char *propq); int CMS_EncryptedData_set1_key(CMS_ContentInfo *cms, const EVP_CIPHER *ciph, @@ -180,7 +174,7 @@ CMS_ContentInfo *CMS_encrypt(STACK_OF(X509) *certs, BIO *in, const EVP_CIPHER *cipher, unsigned int flags); CMS_ContentInfo *CMS_encrypt_ex(STACK_OF(X509) *certs, BIO *in, const EVP_CIPHER *cipher, unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq); + OSSL_LIB_CTX *ctx, const char *propq); int CMS_decrypt(CMS_ContentInfo *cms, EVP_PKEY *pkey, X509 *cert, BIO *dcont, BIO *out, unsigned int flags); @@ -199,16 +193,12 @@ int CMS_RecipientInfo_type(CMS_RecipientInfo *ri); EVP_PKEY_CTX *CMS_RecipientInfo_get0_pkey_ctx(CMS_RecipientInfo *ri); CMS_ContentInfo *CMS_AuthEnvelopedData_create(const EVP_CIPHER *cipher); CMS_ContentInfo * -CMS_AuthEnvelopedData_create_ex(const EVP_CIPHER *cipher, OSSL_LIB_CTX *libctx, +CMS_AuthEnvelopedData_create_ex(const EVP_CIPHER *cipher, OSSL_LIB_CTX *ctx, const char *propq); CMS_ContentInfo *CMS_EnvelopedData_create(const EVP_CIPHER *cipher); CMS_ContentInfo *CMS_EnvelopedData_create_ex(const EVP_CIPHER *cipher, - OSSL_LIB_CTX *libctx, + OSSL_LIB_CTX *ctx, const char *propq); -BIO *CMS_EnvelopedData_decrypt(CMS_EnvelopedData *env, BIO *detached_data, - EVP_PKEY *pkey, X509 *cert, - ASN1_OCTET_STRING *secret, unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq); CMS_RecipientInfo *CMS_add1_recipient_cert(CMS_ContentInfo *cms, X509 *recip, unsigned int flags); @@ -297,11 +287,6 @@ ASN1_OCTET_STRING *CMS_SignerInfo_get0_signature(CMS_SignerInfo *si); int CMS_SignerInfo_sign(CMS_SignerInfo *si); int CMS_SignerInfo_verify(CMS_SignerInfo *si); int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain); -BIO *CMS_SignedData_verify(CMS_SignedData *sd, BIO *detached_data, - STACK_OF(X509) *scerts, X509_STORE *store, - STACK_OF(X509) *extra, STACK_OF(X509_CRL) *crls, - unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq); int CMS_add_smimecap(CMS_SignerInfo *si, STACK_OF(X509_ALGOR) *algs); int CMS_add_simple_smimecap(STACK_OF(X509_ALGOR) **algs, @@ -358,7 +343,7 @@ CMS_ReceiptRequest *CMS_ReceiptRequest_create0_ex( unsigned char *id, int idlen, int allorfirst, STACK_OF(GENERAL_NAMES) *receiptList, STACK_OF(GENERAL_NAMES) *receiptsTo, - OSSL_LIB_CTX *libctx); + OSSL_LIB_CTX *ctx); int CMS_add1_ReceiptRequest(CMS_SignerInfo *si, CMS_ReceiptRequest *rr); void CMS_ReceiptRequest_get0_values(CMS_ReceiptRequest *rr, diff --git a/openssl/include/openssl/cmserr.h b/openssl/include/openssl/cmserr.h index 887035b1b..1c4f4c799 100644 --- a/openssl/include/openssl/cmserr.h +++ b/openssl/include/openssl/cmserr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -86,7 +86,6 @@ # define CMS_R_NO_PUBLIC_KEY 134 # define CMS_R_NO_RECEIPT_REQUEST 168 # define CMS_R_NO_SIGNERS 135 -# define CMS_R_OPERATION_UNSUPPORTED 182 # define CMS_R_PEER_KEY_ERROR 188 # define CMS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE 136 # define CMS_R_RECEIPT_DECODE_ERROR 169 @@ -106,7 +105,6 @@ # define CMS_R_UNKNOWN_DIGEST_ALGORITHM 149 # define CMS_R_UNKNOWN_ID 150 # define CMS_R_UNSUPPORTED_COMPRESSION_ALGORITHM 151 -# define CMS_R_UNSUPPORTED_CONTENT_ENCRYPTION_ALGORITHM 194 # define CMS_R_UNSUPPORTED_CONTENT_TYPE 152 # define CMS_R_UNSUPPORTED_ENCRYPTION_TYPE 192 # define CMS_R_UNSUPPORTED_KEK_ALGORITHM 153 @@ -114,7 +112,6 @@ # define CMS_R_UNSUPPORTED_LABEL_SOURCE 193 # define CMS_R_UNSUPPORTED_RECIPIENTINFO_TYPE 155 # define CMS_R_UNSUPPORTED_RECIPIENT_TYPE 154 -# define CMS_R_UNSUPPORTED_SIGNATURE_ALGORITHM 195 # define CMS_R_UNSUPPORTED_TYPE 156 # define CMS_R_UNWRAP_ERROR 157 # define CMS_R_UNWRAP_FAILURE 180 diff --git a/openssl/include/openssl/comp.h b/openssl/include/openssl/comp.h index f81ba0f39..06ff58100 100644 --- a/openssl/include/openssl/comp.h +++ b/openssl/include/openssl/comp.h @@ -40,20 +40,15 @@ int COMP_expand_block(COMP_CTX *ctx, unsigned char *out, int olen, unsigned char *in, int ilen); COMP_METHOD *COMP_zlib(void); -COMP_METHOD *COMP_zlib_oneshot(void); -COMP_METHOD *COMP_brotli(void); -COMP_METHOD *COMP_brotli_oneshot(void); -COMP_METHOD *COMP_zstd(void); -COMP_METHOD *COMP_zstd_oneshot(void); #ifndef OPENSSL_NO_DEPRECATED_1_1_0 # define COMP_zlib_cleanup() while(0) continue #endif # ifdef OPENSSL_BIO_H +# ifdef ZLIB const BIO_METHOD *BIO_f_zlib(void); -const BIO_METHOD *BIO_f_brotli(void); -const BIO_METHOD *BIO_f_zstd(void); +# endif # endif diff --git a/openssl/include/openssl/comperr.h b/openssl/include/openssl/comperr.h index 1948d37f1..01dd3e6bc 100644 --- a/openssl/include/openssl/comperr.h +++ b/openssl/include/openssl/comperr.h @@ -23,16 +23,9 @@ /* * COMP reason codes. */ -# define COMP_R_BROTLI_DECODE_ERROR 102 -# define COMP_R_BROTLI_ENCODE_ERROR 103 -# define COMP_R_BROTLI_NOT_SUPPORTED 104 # define COMP_R_ZLIB_DEFLATE_ERROR 99 # define COMP_R_ZLIB_INFLATE_ERROR 100 # define COMP_R_ZLIB_NOT_SUPPORTED 101 -# define COMP_R_ZSTD_COMPRESS_ERROR 105 -# define COMP_R_ZSTD_DECODE_ERROR 106 -# define COMP_R_ZSTD_DECOMPRESS_ERROR 107 -# define COMP_R_ZSTD_NOT_SUPPORTED 108 # endif #endif diff --git a/openssl/include/openssl/conf.h b/openssl/include/openssl/conf.h index 00e9f0f45..a11c23a06 100644 --- a/openssl/include/openssl/conf.h +++ b/openssl/include/openssl/conf.h @@ -27,9 +27,6 @@ # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -68,7 +65,7 @@ SKM_DEFINE_STACK_OF_INTERNAL(CONF_VALUE, CONF_VALUE, CONF_VALUE) #define sk_CONF_VALUE_deep_copy(sk, copyfunc, freefunc) ((STACK_OF(CONF_VALUE) *)OPENSSL_sk_deep_copy(ossl_check_const_CONF_VALUE_sk_type(sk), ossl_check_CONF_VALUE_copyfunc_type(copyfunc), ossl_check_CONF_VALUE_freefunc_type(freefunc))) #define sk_CONF_VALUE_set_cmp_func(sk, cmp) ((sk_CONF_VALUE_compfunc)OPENSSL_sk_set_cmp_func(ossl_check_CONF_VALUE_sk_type(sk), ossl_check_CONF_VALUE_compfunc_type(cmp))) DEFINE_LHASH_OF_INTERNAL(CONF_VALUE); -#define lh_CONF_VALUE_new(hfn, cmp) ((LHASH_OF(CONF_VALUE) *)OPENSSL_LH_set_thunks(OPENSSL_LH_new(ossl_check_CONF_VALUE_lh_hashfunc_type(hfn), ossl_check_CONF_VALUE_lh_compfunc_type(cmp)), lh_CONF_VALUE_hash_thunk, lh_CONF_VALUE_comp_thunk, lh_CONF_VALUE_doall_thunk, lh_CONF_VALUE_doall_arg_thunk)) +#define lh_CONF_VALUE_new(hfn, cmp) ((LHASH_OF(CONF_VALUE) *)OPENSSL_LH_new(ossl_check_CONF_VALUE_lh_hashfunc_type(hfn), ossl_check_CONF_VALUE_lh_compfunc_type(cmp))) #define lh_CONF_VALUE_free(lh) OPENSSL_LH_free(ossl_check_CONF_VALUE_lh_type(lh)) #define lh_CONF_VALUE_flush(lh) OPENSSL_LH_flush(ossl_check_CONF_VALUE_lh_type(lh)) #define lh_CONF_VALUE_insert(lh, ptr) ((CONF_VALUE *)OPENSSL_LH_insert(ossl_check_CONF_VALUE_lh_type(lh), ossl_check_CONF_VALUE_lh_plain_type(ptr))) diff --git a/openssl/include/openssl/conf.h.in b/openssl/include/openssl/conf.h.in index 566328708..b0bd579aa 100644 --- a/openssl/include/openssl/conf.h.in +++ b/openssl/include/openssl/conf.h.in @@ -28,9 +28,6 @@ use OpenSSL::stackhash qw(generate_stack_macros generate_lhash_macros); # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { diff --git a/openssl/include/openssl/conferr.h b/openssl/include/openssl/conferr.h index a8798e792..496e2e1ef 100644 --- a/openssl/include/openssl/conferr.h +++ b/openssl/include/openssl/conferr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -38,7 +38,6 @@ # define CONF_R_NUMBER_TOO_LARGE 121 # define CONF_R_OPENSSL_CONF_REFERENCES_MISSING_SECTION 124 # define CONF_R_RECURSIVE_DIRECTORY_INCLUDE 111 -# define CONF_R_RECURSIVE_SECTION_REFERENCE 126 # define CONF_R_RELATIVE_PATH 125 # define CONF_R_SSL_COMMAND_SECTION_EMPTY 117 # define CONF_R_SSL_COMMAND_SECTION_NOT_FOUND 118 diff --git a/openssl/include/openssl/configuration.h b/openssl/include/openssl/configuration.h index dda3d7f54..0d57fd0b8 100644 --- a/openssl/include/openssl/configuration.h +++ b/openssl/include/openssl/configuration.h @@ -1,7 +1,6 @@ /* * WARNING: do not edit! - * Generated by configdata.pm from ../../openssl/Configurations/common0.tmpl, ../../openssl/Configurations/unix-Makefile.tmpl - * via Makefile.in + * Generated by Makefile from ../../openssl/include/openssl/configuration.h.in * * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * @@ -33,7 +32,7 @@ extern "C" { # endif #endif -# define OPENSSL_CONFIGURED_API 30300 +# define OPENSSL_CONFIGURED_API 30000 # ifndef OPENSSL_RAND_SEED_OS # define OPENSSL_RAND_SEED_OS # endif @@ -49,11 +48,14 @@ extern "C" { # ifndef OPENSSL_NO_ASAN # define OPENSSL_NO_ASAN # endif -# ifndef OPENSSL_NO_BROTLI -# define OPENSSL_NO_BROTLI +# ifndef OPENSSL_NO_BN_METHOD +# define OPENSSL_NO_BN_METHOD # endif -# ifndef OPENSSL_NO_BROTLI_DYNAMIC -# define OPENSSL_NO_BROTLI_DYNAMIC +# ifndef OPENSSL_NO_BULLETPROOFS +# define OPENSSL_NO_BULLETPROOFS +# endif +# ifndef OPENSSL_NO_CERT_COMPRESSION +# define OPENSSL_NO_CERT_COMPRESSION # endif # ifndef OPENSSL_NO_CRYPTO_MDEBUG # define OPENSSL_NO_CRYPTO_MDEBUG @@ -61,15 +63,33 @@ extern "C" { # ifndef OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE # define OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE # endif +# ifndef OPENSSL_NO_CRYPTO_MDEBUG_COUNT +# define OPENSSL_NO_CRYPTO_MDEBUG_COUNT +# endif +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +# define OPENSSL_NO_DELEGATED_CREDENTIAL +# endif # ifndef OPENSSL_NO_DEVCRYPTOENG # define OPENSSL_NO_DEVCRYPTOENG # endif +# ifndef OPENSSL_NO_DSO +# define OPENSSL_NO_DSO +# endif +# ifndef OPENSSL_NO_EC_ELGAMAL +# define OPENSSL_NO_EC_ELGAMAL +# endif # ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 # define OPENSSL_NO_EC_NISTP_64_GCC_128 # endif +# ifndef OPENSSL_NO_EC_SM2P_64_GCC_128 +# define OPENSSL_NO_EC_SM2P_64_GCC_128 +# endif # ifndef OPENSSL_NO_EGD # define OPENSSL_NO_EGD # endif +# ifndef OPENSSL_NO_EVP_CIPHER_API_COMPAT +# define OPENSSL_NO_EVP_CIPHER_API_COMPAT +# endif # ifndef OPENSSL_NO_EXTERNAL_TESTS # define OPENSSL_NO_EXTERNAL_TESTS # endif @@ -88,17 +108,20 @@ extern "C" { # ifndef OPENSSL_NO_LOADERENG # define OPENSSL_NO_LOADERENG # endif -# ifndef OPENSSL_NO_MD2 -# define OPENSSL_NO_MD2 +# ifndef OPENSSL_NO_MSAN +# define OPENSSL_NO_MSAN # endif -# ifndef OPENSSL_NO_MD4 -# define OPENSSL_NO_MD4 +# ifndef OPENSSL_NO_NIZK +# define OPENSSL_NO_NIZK # endif -# ifndef OPENSSL_NO_MDC2 -# define OPENSSL_NO_MDC2 +# ifndef OPENSSL_NO_NTLS +# define OPENSSL_NO_NTLS # endif -# ifndef OPENSSL_NO_MSAN -# define OPENSSL_NO_MSAN +# ifndef OPENSSL_NO_OPTIMIZE_CHACHA_CHOOSE +# define OPENSSL_NO_OPTIMIZE_CHACHA_CHOOSE +# endif +# ifndef OPENSSL_NO_PAILLIER +# define OPENSSL_NO_PAILLIER # endif # ifndef OPENSSL_NO_RC5 # define OPENSSL_NO_RC5 @@ -112,12 +135,15 @@ extern "C" { # ifndef OPENSSL_NO_SSL3_METHOD # define OPENSSL_NO_SSL3_METHOD # endif -# ifndef OPENSSL_NO_TFO -# define OPENSSL_NO_TFO +# ifndef OPENSSL_NO_STATUS +# define OPENSSL_NO_STATUS # endif # ifndef OPENSSL_NO_TRACE # define OPENSSL_NO_TRACE # endif +# ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL +# define OPENSSL_NO_TWISTED_EC_ELGAMAL +# endif # ifndef OPENSSL_NO_UBSAN # define OPENSSL_NO_UBSAN # endif @@ -130,23 +156,11 @@ extern "C" { # ifndef OPENSSL_NO_WEAK_SSL_CIPHERS # define OPENSSL_NO_WEAK_SSL_CIPHERS # endif -# ifndef OPENSSL_NO_WHIRLPOOL -# define OPENSSL_NO_WHIRLPOOL -# endif -# ifndef OPENSSL_NO_WINSTORE -# define OPENSSL_NO_WINSTORE -# endif -# ifndef OPENSSL_NO_ZLIB -# define OPENSSL_NO_ZLIB -# endif -# ifndef OPENSSL_NO_ZLIB_DYNAMIC -# define OPENSSL_NO_ZLIB_DYNAMIC +# ifndef OPENSSL_NO_ZKP_GADGET +# define OPENSSL_NO_ZKP_GADGET # endif -# ifndef OPENSSL_NO_ZSTD -# define OPENSSL_NO_ZSTD -# endif -# ifndef OPENSSL_NO_ZSTD_DYNAMIC -# define OPENSSL_NO_ZSTD_DYNAMIC +# ifndef OPENSSL_NO_ZKP_TRANSCRIPT +# define OPENSSL_NO_ZKP_TRANSCRIPT # endif # ifndef OPENSSL_NO_DYNAMIC_ENGINE # define OPENSSL_NO_DYNAMIC_ENGINE @@ -156,7 +170,11 @@ extern "C" { /* Generate 80386 code? */ # undef I386_ONLY +/* + * The following are cipher-specific, but are part of the public API. + */ # if !defined(OPENSSL_SYS_UEFI) +# undef BN_LLONG # ifdef _WIN64 # define SIXTY_FOUR_BIT # elif defined(__i386__) || defined(_M_IX86) || defined(__arm__) @@ -169,12 +187,6 @@ extern "C" { # define RC4_INT unsigned int -# if defined(OPENSSL_NO_COMP) || (defined(OPENSSL_NO_BROTLI) && defined(OPENSSL_NO_ZSTD) && defined(OPENSSL_NO_ZLIB)) -# define OPENSSL_NO_COMP_ALG -# else -# undef OPENSSL_NO_COMP_ALG -# endif - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/configuration.h.in b/openssl/include/openssl/configuration.h.in index 86077d0a6..b84dc1dfe 100644 --- a/openssl/include/openssl/configuration.h.in +++ b/openssl/include/openssl/configuration.h.in @@ -62,12 +62,6 @@ extern "C" { # define RC4_INT {- $config{rc4_int} -} -# if defined(OPENSSL_NO_COMP) || (defined(OPENSSL_NO_BROTLI) && defined(OPENSSL_NO_ZSTD) && defined(OPENSSL_NO_ZLIB)) -# define OPENSSL_NO_COMP_ALG -# else -# undef OPENSSL_NO_COMP_ALG -# endif - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/core.h b/openssl/include/openssl/core.h index 18c199182..0e402e276 100644 --- a/openssl/include/openssl/core.h +++ b/openssl/include/openssl/core.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -42,9 +42,6 @@ struct ossl_dispatch_st { void (*function)(void); }; -# define OSSL_DISPATCH_END \ - { 0, NULL } - /* * Other items, essentially an int<->pointer map element. * @@ -194,14 +191,8 @@ typedef int (OSSL_provider_init_fn)(const OSSL_CORE_HANDLE *handle, const OSSL_DISPATCH *in, const OSSL_DISPATCH **out, void **provctx); -# ifdef __VMS -# pragma names save -# pragma names uppercase,truncated -# endif + OPENSSL_EXPORT OSSL_provider_init_fn OSSL_provider_init; -# ifdef __VMS -# pragma names restore -# endif /* * Generic callback function signature. diff --git a/openssl/include/openssl/core_dispatch.h b/openssl/include/openssl/core_dispatch.h index a5bc2cf75..99fcda002 100644 --- a/openssl/include/openssl/core_dispatch.h +++ b/openssl/include/openssl/core_dispatch.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -176,12 +176,6 @@ OSSL_CORE_MAKE_FUNC(int, BIO_vsnprintf, OSSL_CORE_MAKE_FUNC(int, BIO_ctrl, (OSSL_CORE_BIO *bio, int cmd, long num, void *ptr)) -/* New seeding functions prototypes with the 101-104 series */ -#define OSSL_FUNC_CLEANUP_USER_ENTROPY 96 -#define OSSL_FUNC_CLEANUP_USER_NONCE 97 -#define OSSL_FUNC_GET_USER_ENTROPY 98 -#define OSSL_FUNC_GET_USER_NONCE 99 - #define OSSL_FUNC_SELF_TEST_CB 100 OSSL_CORE_MAKE_FUNC(void, self_test_cb, (OPENSSL_CORE_CTX *ctx, OSSL_CALLBACK **cb, void **cbarg)) @@ -194,25 +188,14 @@ OSSL_CORE_MAKE_FUNC(void, self_test_cb, (OPENSSL_CORE_CTX *ctx, OSSL_CALLBACK ** OSSL_CORE_MAKE_FUNC(size_t, get_entropy, (const OSSL_CORE_HANDLE *handle, unsigned char **pout, int entropy, size_t min_len, size_t max_len)) -OSSL_CORE_MAKE_FUNC(size_t, get_user_entropy, (const OSSL_CORE_HANDLE *handle, - unsigned char **pout, int entropy, - size_t min_len, size_t max_len)) OSSL_CORE_MAKE_FUNC(void, cleanup_entropy, (const OSSL_CORE_HANDLE *handle, unsigned char *buf, size_t len)) -OSSL_CORE_MAKE_FUNC(void, cleanup_user_entropy, (const OSSL_CORE_HANDLE *handle, - unsigned char *buf, size_t len)) OSSL_CORE_MAKE_FUNC(size_t, get_nonce, (const OSSL_CORE_HANDLE *handle, unsigned char **pout, size_t min_len, size_t max_len, const void *salt, size_t salt_len)) -OSSL_CORE_MAKE_FUNC(size_t, get_user_nonce, (const OSSL_CORE_HANDLE *handle, - unsigned char **pout, size_t min_len, - size_t max_len, const void *salt, - size_t salt_len)) OSSL_CORE_MAKE_FUNC(void, cleanup_nonce, (const OSSL_CORE_HANDLE *handle, unsigned char *buf, size_t len)) -OSSL_CORE_MAKE_FUNC(void, cleanup_user_nonce, (const OSSL_CORE_HANDLE *handle, - unsigned char *buf, size_t len)) /* Functions to access the core's providers */ #define OSSL_FUNC_PROVIDER_REGISTER_CHILD_CB 105 @@ -300,7 +283,6 @@ OSSL_CORE_MAKE_FUNC(int, provider_self_test, (void *provctx)) # define OSSL_FUNC_DIGEST_GETTABLE_PARAMS 11 # define OSSL_FUNC_DIGEST_SETTABLE_CTX_PARAMS 12 # define OSSL_FUNC_DIGEST_GETTABLE_CTX_PARAMS 13 -# define OSSL_FUNC_DIGEST_SQUEEZE 14 OSSL_CORE_MAKE_FUNC(void *, digest_newctx, (void *provctx)) OSSL_CORE_MAKE_FUNC(int, digest_init, (void *dctx, const OSSL_PARAM params[])) @@ -309,9 +291,6 @@ OSSL_CORE_MAKE_FUNC(int, digest_update, OSSL_CORE_MAKE_FUNC(int, digest_final, (void *dctx, unsigned char *out, size_t *outl, size_t outsz)) -OSSL_CORE_MAKE_FUNC(int, digest_squeeze, - (void *dctx, - unsigned char *out, size_t *outl, size_t outsz)) OSSL_CORE_MAKE_FUNC(int, digest_digest, (void *provctx, const unsigned char *in, size_t inl, unsigned char *out, size_t *outl, size_t outsz)) @@ -532,7 +511,7 @@ OSSL_CORE_MAKE_FUNC(void,rand_clear_seed, * and key material, etc, essentially everything that manipulates the keys * themselves and their parameters. * - * The key objects are commonly referred to as |keydata|, and it MUST be able + * The key objects are commonly refered to as |keydata|, and it MUST be able * to contain parameters if the key has any, the public key and the private * key. All parts are optional, but their presence determines what can be * done with the key object in terms of encryption, signature, and so on. @@ -665,14 +644,6 @@ OSSL_CORE_MAKE_FUNC(const OSSL_PARAM *, keymgmt_export_types, OSSL_CORE_MAKE_FUNC(void *, keymgmt_dup, (const void *keydata_from, int selection)) -/* Extended import and export functions */ -# define OSSL_FUNC_KEYMGMT_IMPORT_TYPES_EX 45 -# define OSSL_FUNC_KEYMGMT_EXPORT_TYPES_EX 46 -OSSL_CORE_MAKE_FUNC(const OSSL_PARAM *, keymgmt_import_types_ex, - (void *provctx, int selection)) -OSSL_CORE_MAKE_FUNC(const OSSL_PARAM *, keymgmt_export_types_ex, - (void *provctx, int selection)) - /* Key Exchange */ # define OSSL_FUNC_KEYEXCH_NEWCTX 1 @@ -844,24 +815,16 @@ OSSL_CORE_MAKE_FUNC(const OSSL_PARAM *, asym_cipher_settable_ctx_params, # define OSSL_FUNC_KEM_GETTABLE_CTX_PARAMS 9 # define OSSL_FUNC_KEM_SET_CTX_PARAMS 10 # define OSSL_FUNC_KEM_SETTABLE_CTX_PARAMS 11 -# define OSSL_FUNC_KEM_AUTH_ENCAPSULATE_INIT 12 -# define OSSL_FUNC_KEM_AUTH_DECAPSULATE_INIT 13 OSSL_CORE_MAKE_FUNC(void *, kem_newctx, (void *provctx)) OSSL_CORE_MAKE_FUNC(int, kem_encapsulate_init, (void *ctx, void *provkey, const OSSL_PARAM params[])) -OSSL_CORE_MAKE_FUNC(int, kem_auth_encapsulate_init, (void *ctx, void *provkey, - void *authprivkey, - const OSSL_PARAM params[])) OSSL_CORE_MAKE_FUNC(int, kem_encapsulate, (void *ctx, unsigned char *out, size_t *outlen, unsigned char *secret, size_t *secretlen)) OSSL_CORE_MAKE_FUNC(int, kem_decapsulate_init, (void *ctx, void *provkey, const OSSL_PARAM params[])) -OSSL_CORE_MAKE_FUNC(int, kem_auth_decapsulate_init, (void *ctx, void *provkey, - void *authpubkey, - const OSSL_PARAM params[])) OSSL_CORE_MAKE_FUNC(int, kem_decapsulate, (void *ctx, unsigned char *out, size_t *outlen, const unsigned char *in, size_t inlen)) @@ -957,8 +920,6 @@ OSSL_CORE_MAKE_FUNC(int, decoder_export_object, #define OSSL_FUNC_STORE_EOF 6 #define OSSL_FUNC_STORE_CLOSE 7 #define OSSL_FUNC_STORE_EXPORT_OBJECT 8 -#define OSSL_FUNC_STORE_DELETE 9 -#define OSSL_FUNC_STORE_OPEN_EX 10 OSSL_CORE_MAKE_FUNC(void *, store_open, (void *provctx, const char *uri)) OSSL_CORE_MAKE_FUNC(void *, store_attach, (void *provctx, OSSL_CORE_BIO *in)) OSSL_CORE_MAKE_FUNC(const OSSL_PARAM *, store_settable_ctx_params, @@ -974,12 +935,6 @@ OSSL_CORE_MAKE_FUNC(int, store_close, (void *loaderctx)) OSSL_CORE_MAKE_FUNC(int, store_export_object, (void *loaderctx, const void *objref, size_t objref_sz, OSSL_CALLBACK *export_cb, void *export_cbarg)) -OSSL_CORE_MAKE_FUNC(int, store_delete, - (void *provctx, const char *uri, const OSSL_PARAM params[], - OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg)) -OSSL_CORE_MAKE_FUNC(void *, store_open_ex, - (void *provctx, const char *uri, const OSSL_PARAM params[], - OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg)) # ifdef __cplusplus } diff --git a/openssl/include/openssl/core_names.h b/openssl/include/openssl/core_names.h index e07cbd11e..5bbfbc6fa 100644 --- a/openssl/include/openssl/core_names.h +++ b/openssl/include/openssl/core_names.h @@ -1,8 +1,5 @@ /* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/include/openssl/core_names.h.in - * - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,7 +7,6 @@ * https://www.openssl.org/source/license.html */ - #ifndef OPENSSL_CORE_NAMES_H # define OPENSSL_CORE_NAMES_H # pragma once @@ -19,455 +15,546 @@ extern "C" { # endif +/* Well known parameter names that core passes to providers */ +#define OSSL_PROV_PARAM_CORE_VERSION "openssl-version" /* utf8_ptr */ +#define OSSL_PROV_PARAM_CORE_PROV_NAME "provider-name" /* utf8_ptr */ +#define OSSL_PROV_PARAM_CORE_MODULE_FILENAME "module-filename" /* utf8_ptr */ + +/* Well known parameter names that Providers can define */ +#define OSSL_PROV_PARAM_NAME "name" /* utf8_string */ +#define OSSL_PROV_PARAM_VERSION "version" /* utf8_string */ +#define OSSL_PROV_PARAM_BUILDINFO "buildinfo" /* utf8_string */ +#define OSSL_PROV_PARAM_STATUS "status" /* uint */ +#define OSSL_PROV_PARAM_SECURITY_CHECKS "security-checks" /* uint */ + +/* Self test callback parameters */ +#define OSSL_PROV_PARAM_SELF_TEST_PHASE "st-phase" /* utf8_string */ +#define OSSL_PROV_PARAM_SELF_TEST_TYPE "st-type" /* utf8_string */ +#define OSSL_PROV_PARAM_SELF_TEST_DESC "st-desc" /* utf8_string */ + +/*- + * Provider-native object abstractions + * + * These are used when a provider wants to pass object data or an object + * reference back to libcrypto. This is only useful for provider functions + * that take a callback to which an OSSL_PARAM array with these parameters + * can be passed. + * + * This set of parameter names is explained in detail in provider-object(7) + * (doc/man7/provider-object.pod) + */ +#define OSSL_OBJECT_PARAM_TYPE "type" /* INTEGER */ +#define OSSL_OBJECT_PARAM_DATA_TYPE "data-type" /* UTF8_STRING */ +#define OSSL_OBJECT_PARAM_DATA_STRUCTURE "data-structure" /* UTF8_STRING */ +#define OSSL_OBJECT_PARAM_REFERENCE "reference" /* OCTET_STRING */ +#define OSSL_OBJECT_PARAM_DATA "data" /* OCTET_STRING or UTF8_STRING */ +#define OSSL_OBJECT_PARAM_DESC "desc" /* UTF8_STRING */ + +/* + * Algorithm parameters + * If "engine" or "properties" are specified, they should always be paired + * with the algorithm type. + * Note these are common names that are shared by many types (such as kdf, mac, + * and pkey) e.g: see OSSL_MAC_PARAM_DIGEST below. + */ +#define OSSL_ALG_PARAM_DIGEST "digest" /* utf8_string */ +#define OSSL_ALG_PARAM_CIPHER "cipher" /* utf8_string */ +#define OSSL_ALG_PARAM_ENGINE "engine" /* utf8_string */ +#define OSSL_ALG_PARAM_MAC "mac" /* utf8_string */ +#define OSSL_ALG_PARAM_PROPERTIES "properties"/* utf8_string */ + +/* cipher parameters */ +#define OSSL_CIPHER_PARAM_PADDING "padding" /* uint */ +#define OSSL_CIPHER_PARAM_USE_BITS "use-bits" /* uint */ +#define OSSL_CIPHER_PARAM_TLS_VERSION "tls-version" /* uint */ +#define OSSL_CIPHER_PARAM_TLS_MAC "tls-mac" /* octet_ptr */ +#define OSSL_CIPHER_PARAM_TLS_MAC_SIZE "tls-mac-size" /* size_t */ +#define OSSL_CIPHER_PARAM_MODE "mode" /* uint */ +#define OSSL_CIPHER_PARAM_BLOCK_SIZE "blocksize" /* size_t */ +#define OSSL_CIPHER_PARAM_AEAD "aead" /* int, 0 or 1 */ +#define OSSL_CIPHER_PARAM_CUSTOM_IV "custom-iv" /* int, 0 or 1 */ +#define OSSL_CIPHER_PARAM_CTS "cts" /* int, 0 or 1 */ +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK "tls-multi" /* int, 0 or 1 */ +#define OSSL_CIPHER_PARAM_HAS_RAND_KEY "has-randkey" /* int, 0 or 1 */ +#define OSSL_CIPHER_PARAM_KEYLEN "keylen" /* size_t */ +#define OSSL_CIPHER_PARAM_IVLEN "ivlen" /* size_t */ +#define OSSL_CIPHER_PARAM_IV "iv" /* octet_string OR octet_ptr */ +#define OSSL_CIPHER_PARAM_UPDATED_IV "updated-iv" /* octet_string OR octet_ptr */ +#define OSSL_CIPHER_PARAM_NUM "num" /* uint */ +#define OSSL_CIPHER_PARAM_ROUNDS "rounds" /* uint */ +#define OSSL_CIPHER_PARAM_AEAD_TAG "tag" /* octet_string */ +#define OSSL_CIPHER_PARAM_AEAD_TLS1_AAD "tlsaad" /* octet_string */ +#define OSSL_CIPHER_PARAM_AEAD_TLS1_AAD_PAD "tlsaadpad" /* size_t */ +#define OSSL_CIPHER_PARAM_AEAD_TLS1_IV_FIXED "tlsivfixed" /* octet_string */ +#define OSSL_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN "tlsivgen" /* octet_string */ +#define OSSL_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV "tlsivinv" /* octet_string */ +#define OSSL_CIPHER_PARAM_AEAD_IVLEN OSSL_CIPHER_PARAM_IVLEN +#define OSSL_CIPHER_PARAM_AEAD_TAGLEN "taglen" /* size_t */ +#define OSSL_CIPHER_PARAM_AEAD_MAC_KEY "mackey" /* octet_string */ +#define OSSL_CIPHER_PARAM_RANDOM_KEY "randkey" /* octet_string */ +#define OSSL_CIPHER_PARAM_SPEED "speed" /* uint */ +#define OSSL_CIPHER_PARAM_CTS_MODE "cts_mode" /* utf8_string */ +/* For passing the AlgorithmIdentifier parameter in DER form */ +#define OSSL_CIPHER_PARAM_ALGORITHM_ID_PARAMS "alg_id_param" /* octet_string */ + +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT \ + "tls1multi_maxsndfrag" /* uint */ +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE \ + "tls1multi_maxbufsz" /* size_t */ +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE \ + "tls1multi_interleave" /* uint */ +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD \ + "tls1multi_aad" /* octet_string */ +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN \ + "tls1multi_aadpacklen" /* uint */ +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC \ + "tls1multi_enc" /* octet_string */ +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN \ + "tls1multi_encin" /* octet_string */ +#define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN \ + "tls1multi_enclen" /* size_t */ + /* OSSL_CIPHER_PARAM_CTS_MODE Values */ -# define OSSL_CIPHER_CTS_MODE_CS1 "CS1" -# define OSSL_CIPHER_CTS_MODE_CS2 "CS2" -# define OSSL_CIPHER_CTS_MODE_CS3 "CS3" +#define OSSL_CIPHER_CTS_MODE_CS1 "CS1" +#define OSSL_CIPHER_CTS_MODE_CS2 "CS2" +#define OSSL_CIPHER_CTS_MODE_CS3 "CS3" -/* Known CIPHER names (not a complete list) */ -# define OSSL_CIPHER_NAME_AES_128_GCM_SIV "AES-128-GCM-SIV" -# define OSSL_CIPHER_NAME_AES_192_GCM_SIV "AES-192-GCM-SIV" -# define OSSL_CIPHER_NAME_AES_256_GCM_SIV "AES-256-GCM-SIV" +/* digest parameters */ +#define OSSL_DIGEST_PARAM_XOFLEN "xoflen" /* size_t */ +#define OSSL_DIGEST_PARAM_SSL3_MS "ssl3-ms" /* octet string */ +#define OSSL_DIGEST_PARAM_PAD_TYPE "pad-type" /* uint */ +#define OSSL_DIGEST_PARAM_MICALG "micalg" /* utf8 string */ +#define OSSL_DIGEST_PARAM_BLOCK_SIZE "blocksize" /* size_t */ +#define OSSL_DIGEST_PARAM_SIZE "size" /* size_t */ +#define OSSL_DIGEST_PARAM_XOF "xof" /* int, 0 or 1 */ +#define OSSL_DIGEST_PARAM_ALGID_ABSENT "algid-absent" /* int, 0 or 1 */ /* Known DIGEST names (not a complete list) */ -# define OSSL_DIGEST_NAME_MD5 "MD5" -# define OSSL_DIGEST_NAME_MD5_SHA1 "MD5-SHA1" -# define OSSL_DIGEST_NAME_SHA1 "SHA1" -# define OSSL_DIGEST_NAME_SHA2_224 "SHA2-224" -# define OSSL_DIGEST_NAME_SHA2_256 "SHA2-256" -# define OSSL_DIGEST_NAME_SHA2_256_192 "SHA2-256/192" -# define OSSL_DIGEST_NAME_SHA2_384 "SHA2-384" -# define OSSL_DIGEST_NAME_SHA2_512 "SHA2-512" -# define OSSL_DIGEST_NAME_SHA2_512_224 "SHA2-512/224" -# define OSSL_DIGEST_NAME_SHA2_512_256 "SHA2-512/256" -# define OSSL_DIGEST_NAME_MD2 "MD2" -# define OSSL_DIGEST_NAME_MD4 "MD4" -# define OSSL_DIGEST_NAME_MDC2 "MDC2" -# define OSSL_DIGEST_NAME_RIPEMD160 "RIPEMD160" -# define OSSL_DIGEST_NAME_SHA3_224 "SHA3-224" -# define OSSL_DIGEST_NAME_SHA3_256 "SHA3-256" -# define OSSL_DIGEST_NAME_SHA3_384 "SHA3-384" -# define OSSL_DIGEST_NAME_SHA3_512 "SHA3-512" -# define OSSL_DIGEST_NAME_KECCAK_KMAC128 "KECCAK-KMAC-128" -# define OSSL_DIGEST_NAME_KECCAK_KMAC256 "KECCAK-KMAC-256" -# define OSSL_DIGEST_NAME_SM3 "SM3" +#define OSSL_DIGEST_NAME_MD5 "MD5" +#define OSSL_DIGEST_NAME_MD5_SHA1 "MD5-SHA1" +#define OSSL_DIGEST_NAME_SHA1 "SHA1" +#define OSSL_DIGEST_NAME_SHA2_224 "SHA2-224" +#define OSSL_DIGEST_NAME_SHA2_256 "SHA2-256" +#define OSSL_DIGEST_NAME_SHA2_384 "SHA2-384" +#define OSSL_DIGEST_NAME_SHA2_512 "SHA2-512" +#define OSSL_DIGEST_NAME_SHA2_512_224 "SHA2-512/224" +#define OSSL_DIGEST_NAME_SHA2_512_256 "SHA2-512/256" +#define OSSL_DIGEST_NAME_SHA3_224 "SHA3-224" +#define OSSL_DIGEST_NAME_SHA3_256 "SHA3-256" +#define OSSL_DIGEST_NAME_SHA3_384 "SHA3-384" +#define OSSL_DIGEST_NAME_SHA3_512 "SHA3-512" +#define OSSL_DIGEST_NAME_KECCAK_KMAC128 "KECCAK-KMAC-128" +#define OSSL_DIGEST_NAME_KECCAK_KMAC256 "KECCAK-KMAC-256" +#define OSSL_DIGEST_NAME_SM3 "SM3" + +/* MAC parameters */ +#define OSSL_MAC_PARAM_KEY "key" /* octet string */ +#define OSSL_MAC_PARAM_IV "iv" /* octet string */ +#define OSSL_MAC_PARAM_CUSTOM "custom" /* utf8 string */ +#define OSSL_MAC_PARAM_SALT "salt" /* octet string */ +#define OSSL_MAC_PARAM_XOF "xof" /* int, 0 or 1 */ +#define OSSL_MAC_PARAM_DIGEST_NOINIT "digest-noinit" /* int, 0 or 1 */ +#define OSSL_MAC_PARAM_DIGEST_ONESHOT "digest-oneshot" /* int, 0 or 1 */ +#define OSSL_MAC_PARAM_C_ROUNDS "c-rounds" /* unsigned int */ +#define OSSL_MAC_PARAM_D_ROUNDS "d-rounds" /* unsigned int */ + +/* + * If "engine" or "properties" are specified, they should always be paired + * with "cipher" or "digest". + */ +#define OSSL_MAC_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER /* utf8 string */ +#define OSSL_MAC_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST /* utf8 string */ +#define OSSL_MAC_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES /* utf8 string */ +#define OSSL_MAC_PARAM_SIZE "size" /* size_t */ +#define OSSL_MAC_PARAM_BLOCK_SIZE "block-size" /* size_t */ +#define OSSL_MAC_PARAM_TLS_DATA_SIZE "tls-data-size" /* size_t */ /* Known MAC names */ -# define OSSL_MAC_NAME_BLAKE2BMAC "BLAKE2BMAC" -# define OSSL_MAC_NAME_BLAKE2SMAC "BLAKE2SMAC" -# define OSSL_MAC_NAME_CMAC "CMAC" -# define OSSL_MAC_NAME_GMAC "GMAC" -# define OSSL_MAC_NAME_HMAC "HMAC" -# define OSSL_MAC_NAME_KMAC128 "KMAC128" -# define OSSL_MAC_NAME_KMAC256 "KMAC256" -# define OSSL_MAC_NAME_POLY1305 "POLY1305" -# define OSSL_MAC_NAME_SIPHASH "SIPHASH" +#define OSSL_MAC_NAME_CMAC "CMAC" +#define OSSL_MAC_NAME_GMAC "GMAC" +#define OSSL_MAC_NAME_HMAC "HMAC" +#define OSSL_MAC_NAME_KMAC128 "KMAC128" +#define OSSL_MAC_NAME_KMAC256 "KMAC256" +#define OSSL_MAC_NAME_POLY1305 "POLY1305" +#define OSSL_MAC_NAME_SIPHASH "SIPHASH" + +/* KDF / PRF parameters */ +#define OSSL_KDF_PARAM_SECRET "secret" /* octet string */ +#define OSSL_KDF_PARAM_KEY "key" /* octet string */ +#define OSSL_KDF_PARAM_SALT "salt" /* octet string */ +#define OSSL_KDF_PARAM_PASSWORD "pass" /* octet string */ +#define OSSL_KDF_PARAM_PREFIX "prefix" /* octet string */ +#define OSSL_KDF_PARAM_LABEL "label" /* octet string */ +#define OSSL_KDF_PARAM_DATA "data" /* octet string */ +#define OSSL_KDF_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST /* utf8 string */ +#define OSSL_KDF_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER /* utf8 string */ +#define OSSL_KDF_PARAM_MAC OSSL_ALG_PARAM_MAC /* utf8 string */ +#define OSSL_KDF_PARAM_MAC_SIZE "maclen" /* size_t */ +#define OSSL_KDF_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES /* utf8 string */ +#define OSSL_KDF_PARAM_ITER "iter" /* unsigned int */ +#define OSSL_KDF_PARAM_MODE "mode" /* utf8 string or int */ +#define OSSL_KDF_PARAM_PKCS5 "pkcs5" /* int */ +#define OSSL_KDF_PARAM_UKM "ukm" /* octet string */ +#define OSSL_KDF_PARAM_CEK_ALG "cekalg" /* utf8 string */ +#define OSSL_KDF_PARAM_SCRYPT_N "n" /* uint64_t */ +#define OSSL_KDF_PARAM_SCRYPT_R "r" /* uint32_t */ +#define OSSL_KDF_PARAM_SCRYPT_P "p" /* uint32_t */ +#define OSSL_KDF_PARAM_SCRYPT_MAXMEM "maxmem_bytes" /* uint64_t */ +#define OSSL_KDF_PARAM_INFO "info" /* octet string */ +#define OSSL_KDF_PARAM_SEED "seed" /* octet string */ +#define OSSL_KDF_PARAM_SSHKDF_XCGHASH "xcghash" /* octet string */ +#define OSSL_KDF_PARAM_SSHKDF_SESSION_ID "session_id" /* octet string */ +#define OSSL_KDF_PARAM_SSHKDF_TYPE "type" /* int */ +#define OSSL_KDF_PARAM_SIZE "size" /* size_t */ +#define OSSL_KDF_PARAM_CONSTANT "constant" /* octet string */ +#define OSSL_KDF_PARAM_PKCS12_ID "id" /* int */ +#define OSSL_KDF_PARAM_KBKDF_USE_L "use-l" /* int */ +#define OSSL_KDF_PARAM_KBKDF_USE_SEPARATOR "use-separator" /* int */ +#define OSSL_KDF_PARAM_X942_ACVPINFO "acvp-info" +#define OSSL_KDF_PARAM_X942_PARTYUINFO "partyu-info" +#define OSSL_KDF_PARAM_X942_PARTYVINFO "partyv-info" +#define OSSL_KDF_PARAM_X942_SUPP_PUBINFO "supp-pubinfo" +#define OSSL_KDF_PARAM_X942_SUPP_PRIVINFO "supp-privinfo" +#define OSSL_KDF_PARAM_X942_USE_KEYBITS "use-keybits" /* Known KDF names */ -# define OSSL_KDF_NAME_HKDF "HKDF" -# define OSSL_KDF_NAME_TLS1_3_KDF "TLS13-KDF" -# define OSSL_KDF_NAME_PBKDF1 "PBKDF1" -# define OSSL_KDF_NAME_PBKDF2 "PBKDF2" -# define OSSL_KDF_NAME_SCRYPT "SCRYPT" -# define OSSL_KDF_NAME_SSHKDF "SSHKDF" -# define OSSL_KDF_NAME_SSKDF "SSKDF" -# define OSSL_KDF_NAME_TLS1_PRF "TLS1-PRF" -# define OSSL_KDF_NAME_X942KDF_ASN1 "X942KDF-ASN1" -# define OSSL_KDF_NAME_X942KDF_CONCAT "X942KDF-CONCAT" -# define OSSL_KDF_NAME_X963KDF "X963KDF" -# define OSSL_KDF_NAME_KBKDF "KBKDF" -# define OSSL_KDF_NAME_KRB5KDF "KRB5KDF" -# define OSSL_KDF_NAME_HMACDRBGKDF "HMAC-DRBG-KDF" +#define OSSL_KDF_NAME_HKDF "HKDF" +#define OSSL_KDF_NAME_TLS1_3_KDF "TLS13-KDF" +#define OSSL_KDF_NAME_PBKDF1 "PBKDF1" +#define OSSL_KDF_NAME_PBKDF2 "PBKDF2" +#define OSSL_KDF_NAME_SCRYPT "SCRYPT" +#define OSSL_KDF_NAME_SSHKDF "SSHKDF" +#define OSSL_KDF_NAME_SSKDF "SSKDF" +#define OSSL_KDF_NAME_TLS1_PRF "TLS1-PRF" +#define OSSL_KDF_NAME_X942KDF_ASN1 "X942KDF-ASN1" +#define OSSL_KDF_NAME_X942KDF_CONCAT "X942KDF-CONCAT" +#define OSSL_KDF_NAME_X963KDF "X963KDF" +#define OSSL_KDF_NAME_KBKDF "KBKDF" +#define OSSL_KDF_NAME_KRB5KDF "KRB5KDF" + +/* Known RAND names */ +#define OSSL_RAND_PARAM_STATE "state" +#define OSSL_RAND_PARAM_STRENGTH "strength" +#define OSSL_RAND_PARAM_MAX_REQUEST "max_request" +#define OSSL_RAND_PARAM_TEST_ENTROPY "test_entropy" +#define OSSL_RAND_PARAM_TEST_NONCE "test_nonce" + +/* RAND/DRBG names */ +#define OSSL_DRBG_PARAM_RESEED_REQUESTS "reseed_requests" +#define OSSL_DRBG_PARAM_RESEED_TIME_INTERVAL "reseed_time_interval" +#define OSSL_DRBG_PARAM_MIN_ENTROPYLEN "min_entropylen" +#define OSSL_DRBG_PARAM_MAX_ENTROPYLEN "max_entropylen" +#define OSSL_DRBG_PARAM_MIN_NONCELEN "min_noncelen" +#define OSSL_DRBG_PARAM_MAX_NONCELEN "max_noncelen" +#define OSSL_DRBG_PARAM_MAX_PERSLEN "max_perslen" +#define OSSL_DRBG_PARAM_MAX_ADINLEN "max_adinlen" +#define OSSL_DRBG_PARAM_RESEED_COUNTER "reseed_counter" +#define OSSL_DRBG_PARAM_RESEED_TIME "reseed_time" +#define OSSL_DRBG_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES +#define OSSL_DRBG_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST +#define OSSL_DRBG_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER +#define OSSL_DRBG_PARAM_MAC OSSL_ALG_PARAM_MAC +#define OSSL_DRBG_PARAM_USE_DF "use_derivation_function" + +/* DRBG call back parameters */ +#define OSSL_DRBG_PARAM_ENTROPY_REQUIRED "entropy_required" +#define OSSL_DRBG_PARAM_PREDICTION_RESISTANCE "prediction_resistance" +#define OSSL_DRBG_PARAM_MIN_LENGTH "minium_length" +#define OSSL_DRBG_PARAM_MAX_LENGTH "maxium_length" +#define OSSL_DRBG_PARAM_RANDOM_DATA "random_data" +#define OSSL_DRBG_PARAM_SIZE "size" + +/* PKEY parameters */ +/* Common PKEY parameters */ +#define OSSL_PKEY_PARAM_BITS "bits" /* integer */ +#define OSSL_PKEY_PARAM_MAX_SIZE "max-size" /* integer */ +#define OSSL_PKEY_PARAM_SECURITY_BITS "security-bits" /* integer */ +#define OSSL_PKEY_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST +#define OSSL_PKEY_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER /* utf8 string */ +#define OSSL_PKEY_PARAM_ENGINE OSSL_ALG_PARAM_ENGINE /* utf8 string */ +#define OSSL_PKEY_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES +#define OSSL_PKEY_PARAM_DEFAULT_DIGEST "default-digest" /* utf8 string */ +#define OSSL_PKEY_PARAM_MANDATORY_DIGEST "mandatory-digest" /* utf8 string */ +#define OSSL_PKEY_PARAM_PAD_MODE "pad-mode" +#define OSSL_PKEY_PARAM_DIGEST_SIZE "digest-size" +#define OSSL_PKEY_PARAM_MASKGENFUNC "mgf" +#define OSSL_PKEY_PARAM_MGF1_DIGEST "mgf1-digest" +#define OSSL_PKEY_PARAM_MGF1_PROPERTIES "mgf1-properties" +#define OSSL_PKEY_PARAM_ENCODED_PUBLIC_KEY "encoded-pub-key" +#define OSSL_PKEY_PARAM_GROUP_NAME "group" +#define OSSL_PKEY_PARAM_DIST_ID "distid" +#define OSSL_PKEY_PARAM_PUB_KEY "pub" +#define OSSL_PKEY_PARAM_PRIV_KEY "priv" + +/* Diffie-Hellman/DSA Parameters */ +#define OSSL_PKEY_PARAM_FFC_P "p" +#define OSSL_PKEY_PARAM_FFC_G "g" +#define OSSL_PKEY_PARAM_FFC_Q "q" +#define OSSL_PKEY_PARAM_FFC_GINDEX "gindex" +#define OSSL_PKEY_PARAM_FFC_PCOUNTER "pcounter" +#define OSSL_PKEY_PARAM_FFC_SEED "seed" +#define OSSL_PKEY_PARAM_FFC_COFACTOR "j" +#define OSSL_PKEY_PARAM_FFC_H "hindex" +#define OSSL_PKEY_PARAM_FFC_VALIDATE_PQ "validate-pq" +#define OSSL_PKEY_PARAM_FFC_VALIDATE_G "validate-g" +#define OSSL_PKEY_PARAM_FFC_VALIDATE_LEGACY "validate-legacy" + +/* Diffie-Hellman params */ +#define OSSL_PKEY_PARAM_DH_GENERATOR "safeprime-generator" +#define OSSL_PKEY_PARAM_DH_PRIV_LEN "priv_len" + +/* Elliptic Curve Domain Parameters */ +#define OSSL_PKEY_PARAM_EC_PUB_X "qx" +#define OSSL_PKEY_PARAM_EC_PUB_Y "qy" + +/* Elliptic Curve Explicit Domain Parameters */ +#define OSSL_PKEY_PARAM_EC_FIELD_TYPE "field-type" +#define OSSL_PKEY_PARAM_EC_P "p" +#define OSSL_PKEY_PARAM_EC_A "a" +#define OSSL_PKEY_PARAM_EC_B "b" +#define OSSL_PKEY_PARAM_EC_GENERATOR "generator" +#define OSSL_PKEY_PARAM_EC_ORDER "order" +#define OSSL_PKEY_PARAM_EC_COFACTOR "cofactor" +#define OSSL_PKEY_PARAM_EC_SEED "seed" +#define OSSL_PKEY_PARAM_EC_CHAR2_M "m" +#define OSSL_PKEY_PARAM_EC_CHAR2_TYPE "basis-type" +#define OSSL_PKEY_PARAM_EC_CHAR2_TP_BASIS "tp" +#define OSSL_PKEY_PARAM_EC_CHAR2_PP_K1 "k1" +#define OSSL_PKEY_PARAM_EC_CHAR2_PP_K2 "k2" +#define OSSL_PKEY_PARAM_EC_CHAR2_PP_K3 "k3" +#define OSSL_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS "decoded-from-explicit" + +/* Elliptic Curve Key Parameters */ +#define OSSL_PKEY_PARAM_USE_COFACTOR_FLAG "use-cofactor-flag" +#define OSSL_PKEY_PARAM_USE_COFACTOR_ECDH \ + OSSL_PKEY_PARAM_USE_COFACTOR_FLAG + +/* SM2 Specific Parameters */ +#define OSSL_PKEY_PARAM_SM2_ZA "sm2-za" + +/* RSA Keys */ +/* + * n, e, d are the usual public and private key components + * + * rsa-num is the number of factors, including p and q + * rsa-factor is used for each factor: p, q, r_i (i = 3, ...) + * rsa-exponent is used for each exponent: dP, dQ, d_i (i = 3, ...) + * rsa-coefficient is used for each coefficient: qInv, t_i (i = 3, ...) + * + * The number of rsa-factor items must be equal to the number of rsa-exponent + * items, and the number of rsa-coefficients must be one less. + * (the base i for the coefficients is 2, not 1, at least as implied by + * RFC 8017) + */ +#define OSSL_PKEY_PARAM_RSA_N "n" +#define OSSL_PKEY_PARAM_RSA_E "e" +#define OSSL_PKEY_PARAM_RSA_D "d" +#define OSSL_PKEY_PARAM_RSA_FACTOR "rsa-factor" +#define OSSL_PKEY_PARAM_RSA_EXPONENT "rsa-exponent" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT "rsa-coefficient" +#define OSSL_PKEY_PARAM_RSA_FACTOR1 OSSL_PKEY_PARAM_RSA_FACTOR"1" +#define OSSL_PKEY_PARAM_RSA_FACTOR2 OSSL_PKEY_PARAM_RSA_FACTOR"2" +#define OSSL_PKEY_PARAM_RSA_FACTOR3 OSSL_PKEY_PARAM_RSA_FACTOR"3" +#define OSSL_PKEY_PARAM_RSA_FACTOR4 OSSL_PKEY_PARAM_RSA_FACTOR"4" +#define OSSL_PKEY_PARAM_RSA_FACTOR5 OSSL_PKEY_PARAM_RSA_FACTOR"5" +#define OSSL_PKEY_PARAM_RSA_FACTOR6 OSSL_PKEY_PARAM_RSA_FACTOR"6" +#define OSSL_PKEY_PARAM_RSA_FACTOR7 OSSL_PKEY_PARAM_RSA_FACTOR"7" +#define OSSL_PKEY_PARAM_RSA_FACTOR8 OSSL_PKEY_PARAM_RSA_FACTOR"8" +#define OSSL_PKEY_PARAM_RSA_FACTOR9 OSSL_PKEY_PARAM_RSA_FACTOR"9" +#define OSSL_PKEY_PARAM_RSA_FACTOR10 OSSL_PKEY_PARAM_RSA_FACTOR"10" +#define OSSL_PKEY_PARAM_RSA_EXPONENT1 OSSL_PKEY_PARAM_RSA_EXPONENT"1" +#define OSSL_PKEY_PARAM_RSA_EXPONENT2 OSSL_PKEY_PARAM_RSA_EXPONENT"2" +#define OSSL_PKEY_PARAM_RSA_EXPONENT3 OSSL_PKEY_PARAM_RSA_EXPONENT"3" +#define OSSL_PKEY_PARAM_RSA_EXPONENT4 OSSL_PKEY_PARAM_RSA_EXPONENT"4" +#define OSSL_PKEY_PARAM_RSA_EXPONENT5 OSSL_PKEY_PARAM_RSA_EXPONENT"5" +#define OSSL_PKEY_PARAM_RSA_EXPONENT6 OSSL_PKEY_PARAM_RSA_EXPONENT"6" +#define OSSL_PKEY_PARAM_RSA_EXPONENT7 OSSL_PKEY_PARAM_RSA_EXPONENT"7" +#define OSSL_PKEY_PARAM_RSA_EXPONENT8 OSSL_PKEY_PARAM_RSA_EXPONENT"8" +#define OSSL_PKEY_PARAM_RSA_EXPONENT9 OSSL_PKEY_PARAM_RSA_EXPONENT"9" +#define OSSL_PKEY_PARAM_RSA_EXPONENT10 OSSL_PKEY_PARAM_RSA_EXPONENT"10" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT1 OSSL_PKEY_PARAM_RSA_COEFFICIENT"1" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT2 OSSL_PKEY_PARAM_RSA_COEFFICIENT"2" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT3 OSSL_PKEY_PARAM_RSA_COEFFICIENT"3" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT4 OSSL_PKEY_PARAM_RSA_COEFFICIENT"4" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT5 OSSL_PKEY_PARAM_RSA_COEFFICIENT"5" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT6 OSSL_PKEY_PARAM_RSA_COEFFICIENT"6" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT7 OSSL_PKEY_PARAM_RSA_COEFFICIENT"7" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT8 OSSL_PKEY_PARAM_RSA_COEFFICIENT"8" +#define OSSL_PKEY_PARAM_RSA_COEFFICIENT9 OSSL_PKEY_PARAM_RSA_COEFFICIENT"9" /* RSA padding modes */ -# define OSSL_PKEY_RSA_PAD_MODE_NONE "none" -# define OSSL_PKEY_RSA_PAD_MODE_PKCSV15 "pkcs1" -# define OSSL_PKEY_RSA_PAD_MODE_OAEP "oaep" -# define OSSL_PKEY_RSA_PAD_MODE_X931 "x931" -# define OSSL_PKEY_RSA_PAD_MODE_PSS "pss" +#define OSSL_PKEY_RSA_PAD_MODE_NONE "none" +#define OSSL_PKEY_RSA_PAD_MODE_PKCSV15 "pkcs1" +#define OSSL_PKEY_RSA_PAD_MODE_OAEP "oaep" +#define OSSL_PKEY_RSA_PAD_MODE_X931 "x931" +#define OSSL_PKEY_RSA_PAD_MODE_PSS "pss" /* RSA pss padding salt length */ -# define OSSL_PKEY_RSA_PSS_SALT_LEN_DIGEST "digest" -# define OSSL_PKEY_RSA_PSS_SALT_LEN_MAX "max" -# define OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO "auto" -# define OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX "auto-digestmax" +#define OSSL_PKEY_RSA_PSS_SALT_LEN_DIGEST "digest" +#define OSSL_PKEY_RSA_PSS_SALT_LEN_MAX "max" +#define OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO "auto" + +/* Key generation parameters */ +#define OSSL_PKEY_PARAM_RSA_BITS OSSL_PKEY_PARAM_BITS +#define OSSL_PKEY_PARAM_RSA_PRIMES "primes" +#define OSSL_PKEY_PARAM_RSA_DIGEST OSSL_PKEY_PARAM_DIGEST +#define OSSL_PKEY_PARAM_RSA_DIGEST_PROPS OSSL_PKEY_PARAM_PROPERTIES +#define OSSL_PKEY_PARAM_RSA_MASKGENFUNC OSSL_PKEY_PARAM_MASKGENFUNC +#define OSSL_PKEY_PARAM_RSA_MGF1_DIGEST OSSL_PKEY_PARAM_MGF1_DIGEST +#define OSSL_PKEY_PARAM_RSA_PSS_SALTLEN "saltlen" + +/* Key generation parameters */ +#define OSSL_PKEY_PARAM_FFC_TYPE "type" +#define OSSL_PKEY_PARAM_FFC_PBITS "pbits" +#define OSSL_PKEY_PARAM_FFC_QBITS "qbits" +#define OSSL_PKEY_PARAM_FFC_DIGEST OSSL_PKEY_PARAM_DIGEST +#define OSSL_PKEY_PARAM_FFC_DIGEST_PROPS OSSL_PKEY_PARAM_PROPERTIES + +#define OSSL_PKEY_PARAM_EC_ENCODING "encoding" /* utf8_string */ +#define OSSL_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT "point-format" +#define OSSL_PKEY_PARAM_EC_GROUP_CHECK_TYPE "group-check" +#define OSSL_PKEY_PARAM_EC_INCLUDE_PUBLIC "include-public" /* OSSL_PKEY_PARAM_EC_ENCODING values */ -# define OSSL_PKEY_EC_ENCODING_EXPLICIT "explicit" -# define OSSL_PKEY_EC_ENCODING_GROUP "named_curve" +#define OSSL_PKEY_EC_ENCODING_EXPLICIT "explicit" +#define OSSL_PKEY_EC_ENCODING_GROUP "named_curve" + +#define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_UNCOMPRESSED "uncompressed" +#define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_COMPRESSED "compressed" +#define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_HYBRID "hybrid" + +#define OSSL_PKEY_EC_GROUP_CHECK_DEFAULT "default" +#define OSSL_PKEY_EC_GROUP_CHECK_NAMED "named" +#define OSSL_PKEY_EC_GROUP_CHECK_NAMED_NIST "named-nist" + +/* Key Exchange parameters */ +#define OSSL_EXCHANGE_PARAM_PAD "pad" /* uint */ +#define OSSL_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE "ecdh-cofactor-mode" /* int */ +#define OSSL_EXCHANGE_PARAM_KDF_TYPE "kdf-type" /* utf8_string */ +#define OSSL_EXCHANGE_PARAM_KDF_DIGEST "kdf-digest" /* utf8_string */ +#define OSSL_EXCHANGE_PARAM_KDF_DIGEST_PROPS "kdf-digest-props" /* utf8_string */ +#define OSSL_EXCHANGE_PARAM_KDF_OUTLEN "kdf-outlen" /* size_t */ +/* The following parameter is an octet_string on set and an octet_ptr on get */ +#define OSSL_EXCHANGE_PARAM_KDF_UKM "kdf-ukm" + +/* SM2DH Exchange parameters */ +#define OSSL_EXCHANGE_PARAM_INITIATOR "sm2-initiator" +#define OSSL_EXCHANGE_PARAM_SELF_ID "self-id" +#define OSSL_EXCHANGE_PARAM_PEER_ID "peer-id" +#define OSSL_EXCHANGE_PARAM_SELF_ENC_KEY "self-enc-key" +#define OSSL_EXCHANGE_PARAM_PEER_ENC_KEY "peer-enc-key" + +#define OSSL_EXCHANGE_PARAM_DIGEST OSSL_PKEY_PARAM_DIGEST +#define OSSL_EXCHANGE_PARAM_DIGEST_PROPS "digest-props" /* utf8_string */ +#define OSSL_EXCHANGE_PARAM_OUTLEN "outlen" /* size_t */ + +/* Signature parameters */ +#define OSSL_SIGNATURE_PARAM_ALGORITHM_ID "algorithm-id" +#define OSSL_SIGNATURE_PARAM_PAD_MODE OSSL_PKEY_PARAM_PAD_MODE +#define OSSL_SIGNATURE_PARAM_DIGEST OSSL_PKEY_PARAM_DIGEST +#define OSSL_SIGNATURE_PARAM_PROPERTIES OSSL_PKEY_PARAM_PROPERTIES +#define OSSL_SIGNATURE_PARAM_PSS_SALTLEN "saltlen" +#define OSSL_SIGNATURE_PARAM_MGF1_DIGEST OSSL_PKEY_PARAM_MGF1_DIGEST +#define OSSL_SIGNATURE_PARAM_MGF1_PROPERTIES \ + OSSL_PKEY_PARAM_MGF1_PROPERTIES +#define OSSL_SIGNATURE_PARAM_DIGEST_SIZE OSSL_PKEY_PARAM_DIGEST_SIZE + +/* Asym cipher parameters */ +#define OSSL_ASYM_CIPHER_PARAM_DIGEST OSSL_PKEY_PARAM_DIGEST +#define OSSL_ASYM_CIPHER_PARAM_PROPERTIES OSSL_PKEY_PARAM_PROPERTIES +#define OSSL_ASYM_CIPHER_PARAM_ENGINE OSSL_PKEY_PARAM_ENGINE +#define OSSL_ASYM_CIPHER_PARAM_PAD_MODE OSSL_PKEY_PARAM_PAD_MODE +#define OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST \ + OSSL_PKEY_PARAM_MGF1_DIGEST +#define OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST_PROPS \ + OSSL_PKEY_PARAM_MGF1_PROPERTIES +#define OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST OSSL_ALG_PARAM_DIGEST +#define OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS "digest-props" +/* The following parameter is an octet_string on set and an octet_ptr on get */ +#define OSSL_ASYM_CIPHER_PARAM_OAEP_LABEL "oaep-label" +#define OSSL_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION "tls-client-version" +#define OSSL_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION "tls-negotiated-version" + +/* + * Encoder / decoder parameters + */ +#define OSSL_ENCODER_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER +#define OSSL_ENCODER_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES +/* Currently PVK only, but reusable for others as needed */ +#define OSSL_ENCODER_PARAM_ENCRYPT_LEVEL "encrypt-level" +#define OSSL_ENCODER_PARAM_SAVE_PARAMETERS "save-parameters" /* integer */ + +#define OSSL_DECODER_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES + +/* Passphrase callback parameters */ +#define OSSL_PASSPHRASE_PARAM_INFO "info" -# define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_UNCOMPRESSED "uncompressed" -# define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_COMPRESSED "compressed" -# define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_HYBRID "hybrid" +/* Keygen callback parameters, from provider to libcrypto */ +#define OSSL_GEN_PARAM_POTENTIAL "potential" /* integer */ +#define OSSL_GEN_PARAM_ITERATION "iteration" /* integer */ -# define OSSL_PKEY_EC_GROUP_CHECK_DEFAULT "default" -# define OSSL_PKEY_EC_GROUP_CHECK_NAMED "named" -# define OSSL_PKEY_EC_GROUP_CHECK_NAMED_NIST "named-nist" +/* ACVP Test parameters : These should not be used normally */ +#define OSSL_PKEY_PARAM_RSA_TEST_XP1 "xp1" +#define OSSL_PKEY_PARAM_RSA_TEST_XP2 "xp2" +#define OSSL_PKEY_PARAM_RSA_TEST_XP "xp" +#define OSSL_PKEY_PARAM_RSA_TEST_XQ1 "xq1" +#define OSSL_PKEY_PARAM_RSA_TEST_XQ2 "xq2" +#define OSSL_PKEY_PARAM_RSA_TEST_XQ "xq" +#define OSSL_PKEY_PARAM_RSA_TEST_P1 "p1" +#define OSSL_PKEY_PARAM_RSA_TEST_P2 "p2" +#define OSSL_PKEY_PARAM_RSA_TEST_Q1 "q1" +#define OSSL_PKEY_PARAM_RSA_TEST_Q2 "q2" +#define OSSL_SIGNATURE_PARAM_KAT "kat" + +/* KEM parameters */ +#define OSSL_KEM_PARAM_OPERATION "operation" /* OSSL_KEM_PARAM_OPERATION values */ #define OSSL_KEM_PARAM_OPERATION_RSASVE "RSASVE" -#define OSSL_KEM_PARAM_OPERATION_DHKEM "DHKEM" - -/* Parameter name definitions - generated by util/perl/OpenSSL/paramnames.pm */ -# define OSSL_ALG_PARAM_CIPHER "cipher" -# define OSSL_ALG_PARAM_DIGEST "digest" -# define OSSL_ALG_PARAM_ENGINE "engine" -# define OSSL_ALG_PARAM_MAC "mac" -# define OSSL_ALG_PARAM_PROPERTIES "properties" -# define OSSL_ASYM_CIPHER_PARAM_DIGEST OSSL_PKEY_PARAM_DIGEST -# define OSSL_ASYM_CIPHER_PARAM_ENGINE OSSL_PKEY_PARAM_ENGINE -# define OSSL_ASYM_CIPHER_PARAM_IMPLICIT_REJECTION "implicit-rejection" -# define OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST OSSL_PKEY_PARAM_MGF1_DIGEST -# define OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST_PROPS OSSL_PKEY_PARAM_MGF1_PROPERTIES -# define OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST OSSL_ALG_PARAM_DIGEST -# define OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS "digest-props" -# define OSSL_ASYM_CIPHER_PARAM_OAEP_LABEL "oaep-label" -# define OSSL_ASYM_CIPHER_PARAM_PAD_MODE OSSL_PKEY_PARAM_PAD_MODE -# define OSSL_ASYM_CIPHER_PARAM_PROPERTIES OSSL_PKEY_PARAM_PROPERTIES -# define OSSL_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION "tls-client-version" -# define OSSL_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION "tls-negotiated-version" -# define OSSL_CAPABILITY_TLS_GROUP_ALG "tls-group-alg" -# define OSSL_CAPABILITY_TLS_GROUP_ID "tls-group-id" -# define OSSL_CAPABILITY_TLS_GROUP_IS_KEM "tls-group-is-kem" -# define OSSL_CAPABILITY_TLS_GROUP_MAX_DTLS "tls-max-dtls" -# define OSSL_CAPABILITY_TLS_GROUP_MAX_TLS "tls-max-tls" -# define OSSL_CAPABILITY_TLS_GROUP_MIN_DTLS "tls-min-dtls" -# define OSSL_CAPABILITY_TLS_GROUP_MIN_TLS "tls-min-tls" -# define OSSL_CAPABILITY_TLS_GROUP_NAME "tls-group-name" -# define OSSL_CAPABILITY_TLS_GROUP_NAME_INTERNAL "tls-group-name-internal" -# define OSSL_CAPABILITY_TLS_GROUP_SECURITY_BITS "tls-group-sec-bits" -# define OSSL_CAPABILITY_TLS_SIGALG_CODE_POINT "tls-sigalg-code-point" -# define OSSL_CAPABILITY_TLS_SIGALG_HASH_NAME "tls-sigalg-hash-name" -# define OSSL_CAPABILITY_TLS_SIGALG_HASH_OID "tls-sigalg-hash-oid" -# define OSSL_CAPABILITY_TLS_SIGALG_IANA_NAME "tls-sigalg-iana-name" -# define OSSL_CAPABILITY_TLS_SIGALG_KEYTYPE "tls-sigalg-keytype" -# define OSSL_CAPABILITY_TLS_SIGALG_KEYTYPE_OID "tls-sigalg-keytype-oid" -# define OSSL_CAPABILITY_TLS_SIGALG_MAX_TLS "tls-max-tls" -# define OSSL_CAPABILITY_TLS_SIGALG_MIN_TLS "tls-min-tls" -# define OSSL_CAPABILITY_TLS_SIGALG_NAME "tls-sigalg-name" -# define OSSL_CAPABILITY_TLS_SIGALG_OID "tls-sigalg-oid" -# define OSSL_CAPABILITY_TLS_SIGALG_SECURITY_BITS "tls-sigalg-sec-bits" -# define OSSL_CAPABILITY_TLS_SIGALG_SIG_NAME "tls-sigalg-sig-name" -# define OSSL_CAPABILITY_TLS_SIGALG_SIG_OID "tls-sigalg-sig-oid" -# define OSSL_CIPHER_PARAM_AEAD "aead" -# define OSSL_CIPHER_PARAM_AEAD_IVLEN OSSL_CIPHER_PARAM_IVLEN -# define OSSL_CIPHER_PARAM_AEAD_MAC_KEY "mackey" -# define OSSL_CIPHER_PARAM_AEAD_TAG "tag" -# define OSSL_CIPHER_PARAM_AEAD_TAGLEN "taglen" -# define OSSL_CIPHER_PARAM_AEAD_TLS1_AAD "tlsaad" -# define OSSL_CIPHER_PARAM_AEAD_TLS1_AAD_PAD "tlsaadpad" -# define OSSL_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN "tlsivgen" -# define OSSL_CIPHER_PARAM_AEAD_TLS1_IV_FIXED "tlsivfixed" -# define OSSL_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV "tlsivinv" -# define OSSL_CIPHER_PARAM_ALGORITHM_ID_PARAMS "alg_id_param" -# define OSSL_CIPHER_PARAM_BLOCK_SIZE "blocksize" -# define OSSL_CIPHER_PARAM_CTS "cts" -# define OSSL_CIPHER_PARAM_CTS_MODE "cts_mode" -# define OSSL_CIPHER_PARAM_CUSTOM_IV "custom-iv" -# define OSSL_CIPHER_PARAM_HAS_RAND_KEY "has-randkey" -# define OSSL_CIPHER_PARAM_IV "iv" -# define OSSL_CIPHER_PARAM_IVLEN "ivlen" -# define OSSL_CIPHER_PARAM_KEYLEN "keylen" -# define OSSL_CIPHER_PARAM_MODE "mode" -# define OSSL_CIPHER_PARAM_NUM "num" -# define OSSL_CIPHER_PARAM_PADDING "padding" -# define OSSL_CIPHER_PARAM_RANDOM_KEY "randkey" -# define OSSL_CIPHER_PARAM_RC2_KEYBITS "keybits" -# define OSSL_CIPHER_PARAM_ROUNDS "rounds" -# define OSSL_CIPHER_PARAM_SPEED "speed" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK "tls-multi" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD "tls1multi_aad" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN "tls1multi_aadpacklen" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC "tls1multi_enc" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN "tls1multi_encin" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN "tls1multi_enclen" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE "tls1multi_interleave" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE "tls1multi_maxbufsz" -# define OSSL_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT "tls1multi_maxsndfrag" -# define OSSL_CIPHER_PARAM_TLS_MAC "tls-mac" -# define OSSL_CIPHER_PARAM_TLS_MAC_SIZE "tls-mac-size" -# define OSSL_CIPHER_PARAM_TLS_VERSION "tls-version" -# define OSSL_CIPHER_PARAM_UPDATED_IV "updated-iv" -# define OSSL_CIPHER_PARAM_USE_BITS "use-bits" -# define OSSL_CIPHER_PARAM_XTS_STANDARD "xts_standard" -# define OSSL_DECODER_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES -# define OSSL_DIGEST_PARAM_ALGID_ABSENT "algid-absent" -# define OSSL_DIGEST_PARAM_BLOCK_SIZE "blocksize" -# define OSSL_DIGEST_PARAM_MICALG "micalg" -# define OSSL_DIGEST_PARAM_PAD_TYPE "pad-type" -# define OSSL_DIGEST_PARAM_SIZE "size" -# define OSSL_DIGEST_PARAM_SSL3_MS "ssl3-ms" -# define OSSL_DIGEST_PARAM_XOF "xof" -# define OSSL_DIGEST_PARAM_XOFLEN "xoflen" -# define OSSL_DRBG_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER -# define OSSL_DRBG_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST -# define OSSL_DRBG_PARAM_ENTROPY_REQUIRED "entropy_required" -# define OSSL_DRBG_PARAM_MAC OSSL_ALG_PARAM_MAC -# define OSSL_DRBG_PARAM_MAX_ADINLEN "max_adinlen" -# define OSSL_DRBG_PARAM_MAX_ENTROPYLEN "max_entropylen" -# define OSSL_DRBG_PARAM_MAX_LENGTH "maxium_length" -# define OSSL_DRBG_PARAM_MAX_NONCELEN "max_noncelen" -# define OSSL_DRBG_PARAM_MAX_PERSLEN "max_perslen" -# define OSSL_DRBG_PARAM_MIN_ENTROPYLEN "min_entropylen" -# define OSSL_DRBG_PARAM_MIN_LENGTH "minium_length" -# define OSSL_DRBG_PARAM_MIN_NONCELEN "min_noncelen" -# define OSSL_DRBG_PARAM_PREDICTION_RESISTANCE "prediction_resistance" -# define OSSL_DRBG_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES -# define OSSL_DRBG_PARAM_RANDOM_DATA "random_data" -# define OSSL_DRBG_PARAM_RESEED_COUNTER "reseed_counter" -# define OSSL_DRBG_PARAM_RESEED_REQUESTS "reseed_requests" -# define OSSL_DRBG_PARAM_RESEED_TIME "reseed_time" -# define OSSL_DRBG_PARAM_RESEED_TIME_INTERVAL "reseed_time_interval" -# define OSSL_DRBG_PARAM_SIZE "size" -# define OSSL_DRBG_PARAM_USE_DF "use_derivation_function" -# define OSSL_ENCODER_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER -# define OSSL_ENCODER_PARAM_ENCRYPT_LEVEL "encrypt-level" -# define OSSL_ENCODER_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES -# define OSSL_ENCODER_PARAM_SAVE_PARAMETERS "save-parameters" -# define OSSL_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE "ecdh-cofactor-mode" -# define OSSL_EXCHANGE_PARAM_KDF_DIGEST "kdf-digest" -# define OSSL_EXCHANGE_PARAM_KDF_DIGEST_PROPS "kdf-digest-props" -# define OSSL_EXCHANGE_PARAM_KDF_OUTLEN "kdf-outlen" -# define OSSL_EXCHANGE_PARAM_KDF_TYPE "kdf-type" -# define OSSL_EXCHANGE_PARAM_KDF_UKM "kdf-ukm" -# define OSSL_EXCHANGE_PARAM_PAD "pad" -# define OSSL_GEN_PARAM_ITERATION "iteration" -# define OSSL_GEN_PARAM_POTENTIAL "potential" -# define OSSL_KDF_PARAM_ARGON2_AD "ad" -# define OSSL_KDF_PARAM_ARGON2_LANES "lanes" -# define OSSL_KDF_PARAM_ARGON2_MEMCOST "memcost" -# define OSSL_KDF_PARAM_ARGON2_VERSION "version" -# define OSSL_KDF_PARAM_CEK_ALG "cekalg" -# define OSSL_KDF_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER -# define OSSL_KDF_PARAM_CONSTANT "constant" -# define OSSL_KDF_PARAM_DATA "data" -# define OSSL_KDF_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST -# define OSSL_KDF_PARAM_EARLY_CLEAN "early_clean" -# define OSSL_KDF_PARAM_HMACDRBG_ENTROPY "entropy" -# define OSSL_KDF_PARAM_HMACDRBG_NONCE "nonce" -# define OSSL_KDF_PARAM_INFO "info" -# define OSSL_KDF_PARAM_ITER "iter" -# define OSSL_KDF_PARAM_KBKDF_R "r" -# define OSSL_KDF_PARAM_KBKDF_USE_L "use-l" -# define OSSL_KDF_PARAM_KBKDF_USE_SEPARATOR "use-separator" -# define OSSL_KDF_PARAM_KEY "key" -# define OSSL_KDF_PARAM_LABEL "label" -# define OSSL_KDF_PARAM_MAC OSSL_ALG_PARAM_MAC -# define OSSL_KDF_PARAM_MAC_SIZE "maclen" -# define OSSL_KDF_PARAM_MODE "mode" -# define OSSL_KDF_PARAM_PASSWORD "pass" -# define OSSL_KDF_PARAM_PKCS12_ID "id" -# define OSSL_KDF_PARAM_PKCS5 "pkcs5" -# define OSSL_KDF_PARAM_PREFIX "prefix" -# define OSSL_KDF_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES -# define OSSL_KDF_PARAM_SALT "salt" -# define OSSL_KDF_PARAM_SCRYPT_MAXMEM "maxmem_bytes" -# define OSSL_KDF_PARAM_SCRYPT_N "n" -# define OSSL_KDF_PARAM_SCRYPT_P "p" -# define OSSL_KDF_PARAM_SCRYPT_R "r" -# define OSSL_KDF_PARAM_SECRET "secret" -# define OSSL_KDF_PARAM_SEED "seed" -# define OSSL_KDF_PARAM_SIZE "size" -# define OSSL_KDF_PARAM_SSHKDF_SESSION_ID "session_id" -# define OSSL_KDF_PARAM_SSHKDF_TYPE "type" -# define OSSL_KDF_PARAM_SSHKDF_XCGHASH "xcghash" -# define OSSL_KDF_PARAM_THREADS "threads" -# define OSSL_KDF_PARAM_UKM "ukm" -# define OSSL_KDF_PARAM_X942_ACVPINFO "acvp-info" -# define OSSL_KDF_PARAM_X942_PARTYUINFO "partyu-info" -# define OSSL_KDF_PARAM_X942_PARTYVINFO "partyv-info" -# define OSSL_KDF_PARAM_X942_SUPP_PRIVINFO "supp-privinfo" -# define OSSL_KDF_PARAM_X942_SUPP_PUBINFO "supp-pubinfo" -# define OSSL_KDF_PARAM_X942_USE_KEYBITS "use-keybits" -# define OSSL_KEM_PARAM_IKME "ikme" -# define OSSL_KEM_PARAM_OPERATION "operation" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING "block_padding" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA "max_early_data" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN "max_frag_len" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_MODE "mode" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_OPTIONS "options" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD "read_ahead" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC "stream_mac" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_TLSTREE "tlstree" -# define OSSL_LIBSSL_RECORD_LAYER_PARAM_USE_ETM "use_etm" -# define OSSL_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN "read_buffer_len" -# define OSSL_MAC_PARAM_BLOCK_SIZE "block-size" -# define OSSL_MAC_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER -# define OSSL_MAC_PARAM_CUSTOM "custom" -# define OSSL_MAC_PARAM_C_ROUNDS "c-rounds" -# define OSSL_MAC_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST -# define OSSL_MAC_PARAM_DIGEST_NOINIT "digest-noinit" -# define OSSL_MAC_PARAM_DIGEST_ONESHOT "digest-oneshot" -# define OSSL_MAC_PARAM_D_ROUNDS "d-rounds" -# define OSSL_MAC_PARAM_IV "iv" -# define OSSL_MAC_PARAM_KEY "key" -# define OSSL_MAC_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES -# define OSSL_MAC_PARAM_SALT "salt" -# define OSSL_MAC_PARAM_SIZE "size" -# define OSSL_MAC_PARAM_TLS_DATA_SIZE "tls-data-size" -# define OSSL_MAC_PARAM_XOF "xof" -# define OSSL_OBJECT_PARAM_DATA "data" -# define OSSL_OBJECT_PARAM_DATA_STRUCTURE "data-structure" -# define OSSL_OBJECT_PARAM_DATA_TYPE "data-type" -# define OSSL_OBJECT_PARAM_DESC "desc" -# define OSSL_OBJECT_PARAM_REFERENCE "reference" -# define OSSL_OBJECT_PARAM_TYPE "type" -# define OSSL_PASSPHRASE_PARAM_INFO "info" -# define OSSL_PKEY_PARAM_BITS "bits" -# define OSSL_PKEY_PARAM_CIPHER OSSL_ALG_PARAM_CIPHER -# define OSSL_PKEY_PARAM_DEFAULT_DIGEST "default-digest" -# define OSSL_PKEY_PARAM_DHKEM_IKM "dhkem-ikm" -# define OSSL_PKEY_PARAM_DH_GENERATOR "safeprime-generator" -# define OSSL_PKEY_PARAM_DH_PRIV_LEN "priv_len" -# define OSSL_PKEY_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST -# define OSSL_PKEY_PARAM_DIGEST_SIZE "digest-size" -# define OSSL_PKEY_PARAM_DIST_ID "distid" -# define OSSL_PKEY_PARAM_EC_A "a" -# define OSSL_PKEY_PARAM_EC_B "b" -# define OSSL_PKEY_PARAM_EC_CHAR2_M "m" -# define OSSL_PKEY_PARAM_EC_CHAR2_PP_K1 "k1" -# define OSSL_PKEY_PARAM_EC_CHAR2_PP_K2 "k2" -# define OSSL_PKEY_PARAM_EC_CHAR2_PP_K3 "k3" -# define OSSL_PKEY_PARAM_EC_CHAR2_TP_BASIS "tp" -# define OSSL_PKEY_PARAM_EC_CHAR2_TYPE "basis-type" -# define OSSL_PKEY_PARAM_EC_COFACTOR "cofactor" -# define OSSL_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS "decoded-from-explicit" -# define OSSL_PKEY_PARAM_EC_ENCODING "encoding" -# define OSSL_PKEY_PARAM_EC_FIELD_TYPE "field-type" -# define OSSL_PKEY_PARAM_EC_GENERATOR "generator" -# define OSSL_PKEY_PARAM_EC_GROUP_CHECK_TYPE "group-check" -# define OSSL_PKEY_PARAM_EC_INCLUDE_PUBLIC "include-public" -# define OSSL_PKEY_PARAM_EC_ORDER "order" -# define OSSL_PKEY_PARAM_EC_P "p" -# define OSSL_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT "point-format" -# define OSSL_PKEY_PARAM_EC_PUB_X "qx" -# define OSSL_PKEY_PARAM_EC_PUB_Y "qy" -# define OSSL_PKEY_PARAM_EC_SEED "seed" -# define OSSL_PKEY_PARAM_ENCODED_PUBLIC_KEY "encoded-pub-key" -# define OSSL_PKEY_PARAM_ENGINE OSSL_ALG_PARAM_ENGINE -# define OSSL_PKEY_PARAM_FFC_COFACTOR "j" -# define OSSL_PKEY_PARAM_FFC_DIGEST OSSL_PKEY_PARAM_DIGEST -# define OSSL_PKEY_PARAM_FFC_DIGEST_PROPS OSSL_PKEY_PARAM_PROPERTIES -# define OSSL_PKEY_PARAM_FFC_G "g" -# define OSSL_PKEY_PARAM_FFC_GINDEX "gindex" -# define OSSL_PKEY_PARAM_FFC_H "hindex" -# define OSSL_PKEY_PARAM_FFC_P "p" -# define OSSL_PKEY_PARAM_FFC_PBITS "pbits" -# define OSSL_PKEY_PARAM_FFC_PCOUNTER "pcounter" -# define OSSL_PKEY_PARAM_FFC_Q "q" -# define OSSL_PKEY_PARAM_FFC_QBITS "qbits" -# define OSSL_PKEY_PARAM_FFC_SEED "seed" -# define OSSL_PKEY_PARAM_FFC_TYPE "type" -# define OSSL_PKEY_PARAM_FFC_VALIDATE_G "validate-g" -# define OSSL_PKEY_PARAM_FFC_VALIDATE_LEGACY "validate-legacy" -# define OSSL_PKEY_PARAM_FFC_VALIDATE_PQ "validate-pq" -# define OSSL_PKEY_PARAM_GROUP_NAME "group" -# define OSSL_PKEY_PARAM_IMPLICIT_REJECTION "implicit-rejection" -# define OSSL_PKEY_PARAM_MANDATORY_DIGEST "mandatory-digest" -# define OSSL_PKEY_PARAM_MASKGENFUNC "mgf" -# define OSSL_PKEY_PARAM_MAX_SIZE "max-size" -# define OSSL_PKEY_PARAM_MGF1_DIGEST "mgf1-digest" -# define OSSL_PKEY_PARAM_MGF1_PROPERTIES "mgf1-properties" -# define OSSL_PKEY_PARAM_PAD_MODE "pad-mode" -# define OSSL_PKEY_PARAM_PRIV_KEY "priv" -# define OSSL_PKEY_PARAM_PROPERTIES OSSL_ALG_PARAM_PROPERTIES -# define OSSL_PKEY_PARAM_PUB_KEY "pub" -# define OSSL_PKEY_PARAM_RSA_BITS OSSL_PKEY_PARAM_BITS -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT "rsa-coefficient" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT1 "rsa-coefficient1" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT2 "rsa-coefficient2" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT3 "rsa-coefficient3" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT4 "rsa-coefficient4" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT5 "rsa-coefficient5" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT6 "rsa-coefficient6" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT7 "rsa-coefficient7" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT8 "rsa-coefficient8" -# define OSSL_PKEY_PARAM_RSA_COEFFICIENT9 "rsa-coefficient9" -# define OSSL_PKEY_PARAM_RSA_D "d" -# define OSSL_PKEY_PARAM_RSA_DERIVE_FROM_PQ "rsa-derive-from-pq" -# define OSSL_PKEY_PARAM_RSA_DIGEST OSSL_PKEY_PARAM_DIGEST -# define OSSL_PKEY_PARAM_RSA_DIGEST_PROPS OSSL_PKEY_PARAM_PROPERTIES -# define OSSL_PKEY_PARAM_RSA_E "e" -# define OSSL_PKEY_PARAM_RSA_EXPONENT "rsa-exponent" -# define OSSL_PKEY_PARAM_RSA_EXPONENT1 "rsa-exponent1" -# define OSSL_PKEY_PARAM_RSA_EXPONENT10 "rsa-exponent10" -# define OSSL_PKEY_PARAM_RSA_EXPONENT2 "rsa-exponent2" -# define OSSL_PKEY_PARAM_RSA_EXPONENT3 "rsa-exponent3" -# define OSSL_PKEY_PARAM_RSA_EXPONENT4 "rsa-exponent4" -# define OSSL_PKEY_PARAM_RSA_EXPONENT5 "rsa-exponent5" -# define OSSL_PKEY_PARAM_RSA_EXPONENT6 "rsa-exponent6" -# define OSSL_PKEY_PARAM_RSA_EXPONENT7 "rsa-exponent7" -# define OSSL_PKEY_PARAM_RSA_EXPONENT8 "rsa-exponent8" -# define OSSL_PKEY_PARAM_RSA_EXPONENT9 "rsa-exponent9" -# define OSSL_PKEY_PARAM_RSA_FACTOR "rsa-factor" -# define OSSL_PKEY_PARAM_RSA_FACTOR1 "rsa-factor1" -# define OSSL_PKEY_PARAM_RSA_FACTOR10 "rsa-factor10" -# define OSSL_PKEY_PARAM_RSA_FACTOR2 "rsa-factor2" -# define OSSL_PKEY_PARAM_RSA_FACTOR3 "rsa-factor3" -# define OSSL_PKEY_PARAM_RSA_FACTOR4 "rsa-factor4" -# define OSSL_PKEY_PARAM_RSA_FACTOR5 "rsa-factor5" -# define OSSL_PKEY_PARAM_RSA_FACTOR6 "rsa-factor6" -# define OSSL_PKEY_PARAM_RSA_FACTOR7 "rsa-factor7" -# define OSSL_PKEY_PARAM_RSA_FACTOR8 "rsa-factor8" -# define OSSL_PKEY_PARAM_RSA_FACTOR9 "rsa-factor9" -# define OSSL_PKEY_PARAM_RSA_MASKGENFUNC OSSL_PKEY_PARAM_MASKGENFUNC -# define OSSL_PKEY_PARAM_RSA_MGF1_DIGEST OSSL_PKEY_PARAM_MGF1_DIGEST -# define OSSL_PKEY_PARAM_RSA_N "n" -# define OSSL_PKEY_PARAM_RSA_PRIMES "primes" -# define OSSL_PKEY_PARAM_RSA_PSS_SALTLEN "saltlen" -# define OSSL_PKEY_PARAM_RSA_TEST_P1 "p1" -# define OSSL_PKEY_PARAM_RSA_TEST_P2 "p2" -# define OSSL_PKEY_PARAM_RSA_TEST_Q1 "q1" -# define OSSL_PKEY_PARAM_RSA_TEST_Q2 "q2" -# define OSSL_PKEY_PARAM_RSA_TEST_XP "xp" -# define OSSL_PKEY_PARAM_RSA_TEST_XP1 "xp1" -# define OSSL_PKEY_PARAM_RSA_TEST_XP2 "xp2" -# define OSSL_PKEY_PARAM_RSA_TEST_XQ "xq" -# define OSSL_PKEY_PARAM_RSA_TEST_XQ1 "xq1" -# define OSSL_PKEY_PARAM_RSA_TEST_XQ2 "xq2" -# define OSSL_PKEY_PARAM_SECURITY_BITS "security-bits" -# define OSSL_PKEY_PARAM_USE_COFACTOR_ECDH OSSL_PKEY_PARAM_USE_COFACTOR_FLAG -# define OSSL_PKEY_PARAM_USE_COFACTOR_FLAG "use-cofactor-flag" -# define OSSL_PROV_PARAM_BUILDINFO "buildinfo" -# define OSSL_PROV_PARAM_CORE_MODULE_FILENAME "module-filename" -# define OSSL_PROV_PARAM_CORE_PROV_NAME "provider-name" -# define OSSL_PROV_PARAM_CORE_VERSION "openssl-version" -# define OSSL_PROV_PARAM_DRBG_TRUNC_DIGEST "drbg-no-trunc-md" -# define OSSL_PROV_PARAM_NAME "name" -# define OSSL_PROV_PARAM_SECURITY_CHECKS "security-checks" -# define OSSL_PROV_PARAM_SELF_TEST_DESC "st-desc" -# define OSSL_PROV_PARAM_SELF_TEST_PHASE "st-phase" -# define OSSL_PROV_PARAM_SELF_TEST_TYPE "st-type" -# define OSSL_PROV_PARAM_STATUS "status" -# define OSSL_PROV_PARAM_TLS1_PRF_EMS_CHECK "tls1-prf-ems-check" -# define OSSL_PROV_PARAM_VERSION "version" -# define OSSL_RAND_PARAM_GENERATE "generate" -# define OSSL_RAND_PARAM_MAX_REQUEST "max_request" -# define OSSL_RAND_PARAM_STATE "state" -# define OSSL_RAND_PARAM_STRENGTH "strength" -# define OSSL_RAND_PARAM_TEST_ENTROPY "test_entropy" -# define OSSL_RAND_PARAM_TEST_NONCE "test_nonce" -# define OSSL_SIGNATURE_PARAM_ALGORITHM_ID "algorithm-id" -# define OSSL_SIGNATURE_PARAM_CONTEXT_STRING "context-string" -# define OSSL_SIGNATURE_PARAM_DIGEST OSSL_PKEY_PARAM_DIGEST -# define OSSL_SIGNATURE_PARAM_DIGEST_SIZE OSSL_PKEY_PARAM_DIGEST_SIZE -# define OSSL_SIGNATURE_PARAM_INSTANCE "instance" -# define OSSL_SIGNATURE_PARAM_KAT "kat" -# define OSSL_SIGNATURE_PARAM_MGF1_DIGEST OSSL_PKEY_PARAM_MGF1_DIGEST -# define OSSL_SIGNATURE_PARAM_MGF1_PROPERTIES OSSL_PKEY_PARAM_MGF1_PROPERTIES -# define OSSL_SIGNATURE_PARAM_NONCE_TYPE "nonce-type" -# define OSSL_SIGNATURE_PARAM_PAD_MODE OSSL_PKEY_PARAM_PAD_MODE -# define OSSL_SIGNATURE_PARAM_PROPERTIES OSSL_PKEY_PARAM_PROPERTIES -# define OSSL_SIGNATURE_PARAM_PSS_SALTLEN "saltlen" -# define OSSL_STORE_PARAM_ALIAS "alias" -# define OSSL_STORE_PARAM_DIGEST "digest" -# define OSSL_STORE_PARAM_EXPECT "expect" -# define OSSL_STORE_PARAM_FINGERPRINT "fingerprint" -# define OSSL_STORE_PARAM_INPUT_TYPE "input-type" -# define OSSL_STORE_PARAM_ISSUER "name" -# define OSSL_STORE_PARAM_PROPERTIES "properties" -# define OSSL_STORE_PARAM_SERIAL "serial" -# define OSSL_STORE_PARAM_SUBJECT "subject" + +/* Capabilities */ + +/* TLS-GROUP Capability */ +#define OSSL_CAPABILITY_TLS_GROUP_NAME "tls-group-name" +#define OSSL_CAPABILITY_TLS_GROUP_NAME_INTERNAL "tls-group-name-internal" +#define OSSL_CAPABILITY_TLS_GROUP_ID "tls-group-id" +#define OSSL_CAPABILITY_TLS_GROUP_ALG "tls-group-alg" +#define OSSL_CAPABILITY_TLS_GROUP_SECURITY_BITS "tls-group-sec-bits" +#define OSSL_CAPABILITY_TLS_GROUP_IS_KEM "tls-group-is-kem" +#define OSSL_CAPABILITY_TLS_GROUP_MIN_TLS "tls-min-tls" +#define OSSL_CAPABILITY_TLS_GROUP_MAX_TLS "tls-max-tls" +#define OSSL_CAPABILITY_TLS_GROUP_MIN_DTLS "tls-min-dtls" +#define OSSL_CAPABILITY_TLS_GROUP_MAX_DTLS "tls-max-dtls" + +/*- + * storemgmt parameters + */ + +/* + * Used by storemgmt_ctx_set_params(): + * + * - OSSL_STORE_PARAM_EXPECT is an INTEGER, and the value is any of the + * OSSL_STORE_INFO numbers. This is used to set the expected type of + * object loaded. + * + * - OSSL_STORE_PARAM_SUBJECT, OSSL_STORE_PARAM_ISSUER, + * OSSL_STORE_PARAM_SERIAL, OSSL_STORE_PARAM_FINGERPRINT, + * OSSL_STORE_PARAM_DIGEST, OSSL_STORE_PARAM_ALIAS + * are used as search criteria. + * (OSSL_STORE_PARAM_DIGEST is used with OSSL_STORE_PARAM_FINGERPRINT) + */ +#define OSSL_STORE_PARAM_EXPECT "expect" /* INTEGER */ +#define OSSL_STORE_PARAM_SUBJECT "subject" /* DER blob => OCTET_STRING */ +#define OSSL_STORE_PARAM_ISSUER "name" /* DER blob => OCTET_STRING */ +#define OSSL_STORE_PARAM_SERIAL "serial" /* INTEGER */ +#define OSSL_STORE_PARAM_DIGEST "digest" /* UTF8_STRING */ +#define OSSL_STORE_PARAM_FINGERPRINT "fingerprint" /* OCTET_STRING */ +#define OSSL_STORE_PARAM_ALIAS "alias" /* UTF8_STRING */ + +/* You may want to pass properties for the provider implementation to use */ +#define OSSL_STORE_PARAM_PROPERTIES "properties" /* utf8_string */ +/* OSSL_DECODER input type if a decoder is used by the store */ +#define OSSL_STORE_PARAM_INPUT_TYPE "input-type" /* UTF8_STRING */ # ifdef __cplusplus } diff --git a/openssl/include/openssl/core_names.h.in b/openssl/include/openssl/core_names.h.in deleted file mode 100644 index c14520fe2..000000000 --- a/openssl/include/openssl/core_names.h.in +++ /dev/null @@ -1,119 +0,0 @@ -/* - * {- join("\n * ", @autowarntext) -} - * - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -{- -use OpenSSL::paramnames qw(generate_public_macros); --} - -#ifndef OPENSSL_CORE_NAMES_H -# define OPENSSL_CORE_NAMES_H -# pragma once - -# ifdef __cplusplus -extern "C" { -# endif - -/* OSSL_CIPHER_PARAM_CTS_MODE Values */ -# define OSSL_CIPHER_CTS_MODE_CS1 "CS1" -# define OSSL_CIPHER_CTS_MODE_CS2 "CS2" -# define OSSL_CIPHER_CTS_MODE_CS3 "CS3" - -/* Known CIPHER names (not a complete list) */ -# define OSSL_CIPHER_NAME_AES_128_GCM_SIV "AES-128-GCM-SIV" -# define OSSL_CIPHER_NAME_AES_192_GCM_SIV "AES-192-GCM-SIV" -# define OSSL_CIPHER_NAME_AES_256_GCM_SIV "AES-256-GCM-SIV" - -/* Known DIGEST names (not a complete list) */ -# define OSSL_DIGEST_NAME_MD5 "MD5" -# define OSSL_DIGEST_NAME_MD5_SHA1 "MD5-SHA1" -# define OSSL_DIGEST_NAME_SHA1 "SHA1" -# define OSSL_DIGEST_NAME_SHA2_224 "SHA2-224" -# define OSSL_DIGEST_NAME_SHA2_256 "SHA2-256" -# define OSSL_DIGEST_NAME_SHA2_256_192 "SHA2-256/192" -# define OSSL_DIGEST_NAME_SHA2_384 "SHA2-384" -# define OSSL_DIGEST_NAME_SHA2_512 "SHA2-512" -# define OSSL_DIGEST_NAME_SHA2_512_224 "SHA2-512/224" -# define OSSL_DIGEST_NAME_SHA2_512_256 "SHA2-512/256" -# define OSSL_DIGEST_NAME_MD2 "MD2" -# define OSSL_DIGEST_NAME_MD4 "MD4" -# define OSSL_DIGEST_NAME_MDC2 "MDC2" -# define OSSL_DIGEST_NAME_RIPEMD160 "RIPEMD160" -# define OSSL_DIGEST_NAME_SHA3_224 "SHA3-224" -# define OSSL_DIGEST_NAME_SHA3_256 "SHA3-256" -# define OSSL_DIGEST_NAME_SHA3_384 "SHA3-384" -# define OSSL_DIGEST_NAME_SHA3_512 "SHA3-512" -# define OSSL_DIGEST_NAME_KECCAK_KMAC128 "KECCAK-KMAC-128" -# define OSSL_DIGEST_NAME_KECCAK_KMAC256 "KECCAK-KMAC-256" -# define OSSL_DIGEST_NAME_SM3 "SM3" - -/* Known MAC names */ -# define OSSL_MAC_NAME_BLAKE2BMAC "BLAKE2BMAC" -# define OSSL_MAC_NAME_BLAKE2SMAC "BLAKE2SMAC" -# define OSSL_MAC_NAME_CMAC "CMAC" -# define OSSL_MAC_NAME_GMAC "GMAC" -# define OSSL_MAC_NAME_HMAC "HMAC" -# define OSSL_MAC_NAME_KMAC128 "KMAC128" -# define OSSL_MAC_NAME_KMAC256 "KMAC256" -# define OSSL_MAC_NAME_POLY1305 "POLY1305" -# define OSSL_MAC_NAME_SIPHASH "SIPHASH" - -/* Known KDF names */ -# define OSSL_KDF_NAME_HKDF "HKDF" -# define OSSL_KDF_NAME_TLS1_3_KDF "TLS13-KDF" -# define OSSL_KDF_NAME_PBKDF1 "PBKDF1" -# define OSSL_KDF_NAME_PBKDF2 "PBKDF2" -# define OSSL_KDF_NAME_SCRYPT "SCRYPT" -# define OSSL_KDF_NAME_SSHKDF "SSHKDF" -# define OSSL_KDF_NAME_SSKDF "SSKDF" -# define OSSL_KDF_NAME_TLS1_PRF "TLS1-PRF" -# define OSSL_KDF_NAME_X942KDF_ASN1 "X942KDF-ASN1" -# define OSSL_KDF_NAME_X942KDF_CONCAT "X942KDF-CONCAT" -# define OSSL_KDF_NAME_X963KDF "X963KDF" -# define OSSL_KDF_NAME_KBKDF "KBKDF" -# define OSSL_KDF_NAME_KRB5KDF "KRB5KDF" -# define OSSL_KDF_NAME_HMACDRBGKDF "HMAC-DRBG-KDF" - -/* RSA padding modes */ -# define OSSL_PKEY_RSA_PAD_MODE_NONE "none" -# define OSSL_PKEY_RSA_PAD_MODE_PKCSV15 "pkcs1" -# define OSSL_PKEY_RSA_PAD_MODE_OAEP "oaep" -# define OSSL_PKEY_RSA_PAD_MODE_X931 "x931" -# define OSSL_PKEY_RSA_PAD_MODE_PSS "pss" - -/* RSA pss padding salt length */ -# define OSSL_PKEY_RSA_PSS_SALT_LEN_DIGEST "digest" -# define OSSL_PKEY_RSA_PSS_SALT_LEN_MAX "max" -# define OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO "auto" -# define OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX "auto-digestmax" - -/* OSSL_PKEY_PARAM_EC_ENCODING values */ -# define OSSL_PKEY_EC_ENCODING_EXPLICIT "explicit" -# define OSSL_PKEY_EC_ENCODING_GROUP "named_curve" - -# define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_UNCOMPRESSED "uncompressed" -# define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_COMPRESSED "compressed" -# define OSSL_PKEY_EC_POINT_CONVERSION_FORMAT_HYBRID "hybrid" - -# define OSSL_PKEY_EC_GROUP_CHECK_DEFAULT "default" -# define OSSL_PKEY_EC_GROUP_CHECK_NAMED "named" -# define OSSL_PKEY_EC_GROUP_CHECK_NAMED_NIST "named-nist" - -/* OSSL_KEM_PARAM_OPERATION values */ -#define OSSL_KEM_PARAM_OPERATION_RSASVE "RSASVE" -#define OSSL_KEM_PARAM_OPERATION_DHKEM "DHKEM" - -/* Parameter name definitions - generated by util/perl/OpenSSL/paramnames.pm */ -{- generate_public_macros(); -} - -# ifdef __cplusplus -} -# endif - -#endif diff --git a/openssl/include/openssl/crmf.h b/openssl/include/openssl/crmf.h index e04af8ece..b4be4e8ea 100644 --- a/openssl/include/openssl/crmf.h +++ b/openssl/include/openssl/crmf.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/crmf.h.in * - * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -43,8 +43,8 @@ extern "C" { # define OSSL_CRMF_SUBSEQUENTMESSAGE_ENCRCERT 0 # define OSSL_CRMF_SUBSEQUENTMESSAGE_CHALLENGERESP 1 -typedef struct ossl_crmf_encryptedvalue_st OSSL_CRMF_ENCRYPTEDVALUE; +typedef struct ossl_crmf_encryptedvalue_st OSSL_CRMF_ENCRYPTEDVALUE; DECLARE_ASN1_FUNCTIONS(OSSL_CRMF_ENCRYPTEDVALUE) typedef struct ossl_crmf_msg_st OSSL_CRMF_MSG; DECLARE_ASN1_FUNCTIONS(OSSL_CRMF_MSG) @@ -198,14 +198,12 @@ int OSSL_CRMF_MSGS_verify_popo(const OSSL_CRMF_MSGS *reqs, int rid, int acceptRAVerified, OSSL_LIB_CTX *libctx, const char *propq); OSSL_CRMF_CERTTEMPLATE *OSSL_CRMF_MSG_get0_tmpl(const OSSL_CRMF_MSG *crm); -X509_PUBKEY -*OSSL_CRMF_CERTTEMPLATE_get0_publicKey(const OSSL_CRMF_CERTTEMPLATE *tmpl); +const ASN1_INTEGER +*OSSL_CRMF_CERTTEMPLATE_get0_serialNumber(const OSSL_CRMF_CERTTEMPLATE *tmpl); const X509_NAME *OSSL_CRMF_CERTTEMPLATE_get0_subject(const OSSL_CRMF_CERTTEMPLATE *tmpl); const X509_NAME *OSSL_CRMF_CERTTEMPLATE_get0_issuer(const OSSL_CRMF_CERTTEMPLATE *tmpl); -const ASN1_INTEGER -*OSSL_CRMF_CERTTEMPLATE_get0_serialNumber(const OSSL_CRMF_CERTTEMPLATE *tmpl); X509_EXTENSIONS *OSSL_CRMF_CERTTEMPLATE_get0_extensions(const OSSL_CRMF_CERTTEMPLATE *tmpl); const X509_NAME diff --git a/openssl/include/openssl/crmf.h.in b/openssl/include/openssl/crmf.h.in index 43411fa42..4d37ea6d9 100644 --- a/openssl/include/openssl/crmf.h.in +++ b/openssl/include/openssl/crmf.h.in @@ -1,7 +1,7 @@ /*- * {- join("\n * ", @autowarntext) -} * - * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -44,8 +44,8 @@ extern "C" { # define OSSL_CRMF_SUBSEQUENTMESSAGE_ENCRCERT 0 # define OSSL_CRMF_SUBSEQUENTMESSAGE_CHALLENGERESP 1 -typedef struct ossl_crmf_encryptedvalue_st OSSL_CRMF_ENCRYPTEDVALUE; +typedef struct ossl_crmf_encryptedvalue_st OSSL_CRMF_ENCRYPTEDVALUE; DECLARE_ASN1_FUNCTIONS(OSSL_CRMF_ENCRYPTEDVALUE) typedef struct ossl_crmf_msg_st OSSL_CRMF_MSG; DECLARE_ASN1_FUNCTIONS(OSSL_CRMF_MSG) @@ -151,14 +151,12 @@ int OSSL_CRMF_MSGS_verify_popo(const OSSL_CRMF_MSGS *reqs, int rid, int acceptRAVerified, OSSL_LIB_CTX *libctx, const char *propq); OSSL_CRMF_CERTTEMPLATE *OSSL_CRMF_MSG_get0_tmpl(const OSSL_CRMF_MSG *crm); -X509_PUBKEY -*OSSL_CRMF_CERTTEMPLATE_get0_publicKey(const OSSL_CRMF_CERTTEMPLATE *tmpl); +const ASN1_INTEGER +*OSSL_CRMF_CERTTEMPLATE_get0_serialNumber(const OSSL_CRMF_CERTTEMPLATE *tmpl); const X509_NAME *OSSL_CRMF_CERTTEMPLATE_get0_subject(const OSSL_CRMF_CERTTEMPLATE *tmpl); const X509_NAME *OSSL_CRMF_CERTTEMPLATE_get0_issuer(const OSSL_CRMF_CERTTEMPLATE *tmpl); -const ASN1_INTEGER -*OSSL_CRMF_CERTTEMPLATE_get0_serialNumber(const OSSL_CRMF_CERTTEMPLATE *tmpl); X509_EXTENSIONS *OSSL_CRMF_CERTTEMPLATE_get0_extensions(const OSSL_CRMF_CERTTEMPLATE *tmpl); const X509_NAME diff --git a/openssl/include/openssl/crypto.h b/openssl/include/openssl/crypto.h index c85ce67c7..adc65b834 100644 --- a/openssl/include/openssl/crypto.h +++ b/openssl/include/openssl/crypto.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/crypto.h.in * - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -88,7 +88,6 @@ int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock); int CRYPTO_atomic_or(uint64_t *val, uint64_t op, uint64_t *ret, CRYPTO_RWLOCK *lock); int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock); -int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock); /* No longer needed, so this is a no-op */ #define OPENSSL_malloc_init() while(0) continue @@ -150,6 +149,8 @@ const char *OPENSSL_version_pre_release(void); const char *OPENSSL_version_build_metadata(void); unsigned long OpenSSL_version_num(void); +unsigned long Tongsuo_version_num(void); +# define BabaSSL_version_num Tongsuo_version_num const char *OpenSSL_version(int type); # define OPENSSL_VERSION 0 # define OPENSSL_CFLAGS 1 @@ -161,6 +162,10 @@ const char *OpenSSL_version(int type); # define OPENSSL_FULL_VERSION_STRING 7 # define OPENSSL_MODULES_DIR 8 # define OPENSSL_CPU_INFO 9 +# define TONGSUO_VERSION 10 +# ifndef OPENSSL_NO_QUIC +# define OPENSSL_INFO_QUIC 2000 +# endif const char *OPENSSL_info(int type); /* @@ -342,11 +347,11 @@ void CRYPTO_get_mem_functions(CRYPTO_malloc_fn *malloc_fn, CRYPTO_realloc_fn *realloc_fn, CRYPTO_free_fn *free_fn); -OSSL_CRYPTO_ALLOC void *CRYPTO_malloc(size_t num, const char *file, int line); -OSSL_CRYPTO_ALLOC void *CRYPTO_zalloc(size_t num, const char *file, int line); -OSSL_CRYPTO_ALLOC void *CRYPTO_memdup(const void *str, size_t siz, const char *file, int line); -OSSL_CRYPTO_ALLOC char *CRYPTO_strdup(const char *str, const char *file, int line); -OSSL_CRYPTO_ALLOC char *CRYPTO_strndup(const char *str, size_t s, const char *file, int line); +void *CRYPTO_malloc(size_t num, const char *file, int line); +void *CRYPTO_zalloc(size_t num, const char *file, int line); +void *CRYPTO_memdup(const void *str, size_t siz, const char *file, int line); +char *CRYPTO_strdup(const char *str, const char *file, int line); +char *CRYPTO_strndup(const char *str, size_t s, const char *file, int line); void CRYPTO_free(void *ptr, const char *file, int line); void CRYPTO_clear_free(void *ptr, size_t num, const char *file, int line); void *CRYPTO_realloc(void *addr, size_t num, const char *file, int line); @@ -355,8 +360,8 @@ void *CRYPTO_clear_realloc(void *addr, size_t old_num, size_t num, int CRYPTO_secure_malloc_init(size_t sz, size_t minsize); int CRYPTO_secure_malloc_done(void); -OSSL_CRYPTO_ALLOC void *CRYPTO_secure_malloc(size_t num, const char *file, int line); -OSSL_CRYPTO_ALLOC void *CRYPTO_secure_zalloc(size_t num, const char *file, int line); +void *CRYPTO_secure_malloc(size_t num, const char *file, int line); +void *CRYPTO_secure_zalloc(size_t num, const char *file, int line); void CRYPTO_secure_free(void *ptr, const char *file, int line); void CRYPTO_secure_clear_free(void *ptr, size_t num, const char *file, int line); @@ -367,6 +372,10 @@ size_t CRYPTO_secure_used(void); void OPENSSL_cleanse(void *ptr, size_t len); +# ifndef OPENSSL_NO_CRYPTO_MDEBUG_COUNT +void CRYPTO_get_mem_counts(int *count, size_t *size); +# endif + # ifndef OPENSSL_NO_CRYPTO_MDEBUG /* * The following can be used to detect memory leaks in the library. If @@ -553,8 +562,6 @@ void OSSL_LIB_CTX_free(OSSL_LIB_CTX *); OSSL_LIB_CTX *OSSL_LIB_CTX_get0_global_default(void); OSSL_LIB_CTX *OSSL_LIB_CTX_set0_default(OSSL_LIB_CTX *libctx); -void OSSL_sleep(uint64_t millis); - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/crypto.h.in b/openssl/include/openssl/crypto.h.in index b2d691b90..6ffccc19b 100644 --- a/openssl/include/openssl/crypto.h.in +++ b/openssl/include/openssl/crypto.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -89,7 +89,6 @@ int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock); int CRYPTO_atomic_or(uint64_t *val, uint64_t op, uint64_t *ret, CRYPTO_RWLOCK *lock); int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock); -int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock); /* No longer needed, so this is a no-op */ #define OPENSSL_malloc_init() while(0) continue @@ -151,6 +150,8 @@ const char *OPENSSL_version_pre_release(void); const char *OPENSSL_version_build_metadata(void); unsigned long OpenSSL_version_num(void); +unsigned long Tongsuo_version_num(void); +# define BabaSSL_version_num Tongsuo_version_num const char *OpenSSL_version(int type); # define OPENSSL_VERSION 0 # define OPENSSL_CFLAGS 1 @@ -162,6 +163,10 @@ const char *OpenSSL_version(int type); # define OPENSSL_FULL_VERSION_STRING 7 # define OPENSSL_MODULES_DIR 8 # define OPENSSL_CPU_INFO 9 +# define TONGSUO_VERSION 10 +# ifndef OPENSSL_NO_QUIC +# define OPENSSL_INFO_QUIC 2000 +# endif const char *OPENSSL_info(int type); /* @@ -319,11 +324,11 @@ void CRYPTO_get_mem_functions(CRYPTO_malloc_fn *malloc_fn, CRYPTO_realloc_fn *realloc_fn, CRYPTO_free_fn *free_fn); -OSSL_CRYPTO_ALLOC void *CRYPTO_malloc(size_t num, const char *file, int line); -OSSL_CRYPTO_ALLOC void *CRYPTO_zalloc(size_t num, const char *file, int line); -OSSL_CRYPTO_ALLOC void *CRYPTO_memdup(const void *str, size_t siz, const char *file, int line); -OSSL_CRYPTO_ALLOC char *CRYPTO_strdup(const char *str, const char *file, int line); -OSSL_CRYPTO_ALLOC char *CRYPTO_strndup(const char *str, size_t s, const char *file, int line); +void *CRYPTO_malloc(size_t num, const char *file, int line); +void *CRYPTO_zalloc(size_t num, const char *file, int line); +void *CRYPTO_memdup(const void *str, size_t siz, const char *file, int line); +char *CRYPTO_strdup(const char *str, const char *file, int line); +char *CRYPTO_strndup(const char *str, size_t s, const char *file, int line); void CRYPTO_free(void *ptr, const char *file, int line); void CRYPTO_clear_free(void *ptr, size_t num, const char *file, int line); void *CRYPTO_realloc(void *addr, size_t num, const char *file, int line); @@ -332,8 +337,8 @@ void *CRYPTO_clear_realloc(void *addr, size_t old_num, size_t num, int CRYPTO_secure_malloc_init(size_t sz, size_t minsize); int CRYPTO_secure_malloc_done(void); -OSSL_CRYPTO_ALLOC void *CRYPTO_secure_malloc(size_t num, const char *file, int line); -OSSL_CRYPTO_ALLOC void *CRYPTO_secure_zalloc(size_t num, const char *file, int line); +void *CRYPTO_secure_malloc(size_t num, const char *file, int line); +void *CRYPTO_secure_zalloc(size_t num, const char *file, int line); void CRYPTO_secure_free(void *ptr, const char *file, int line); void CRYPTO_secure_clear_free(void *ptr, size_t num, const char *file, int line); @@ -344,6 +349,10 @@ size_t CRYPTO_secure_used(void); void OPENSSL_cleanse(void *ptr, size_t len); +# ifndef OPENSSL_NO_CRYPTO_MDEBUG_COUNT +void CRYPTO_get_mem_counts(int *count, size_t *size); +# endif + # ifndef OPENSSL_NO_CRYPTO_MDEBUG /* * The following can be used to detect memory leaks in the library. If @@ -530,8 +539,6 @@ void OSSL_LIB_CTX_free(OSSL_LIB_CTX *); OSSL_LIB_CTX *OSSL_LIB_CTX_get0_global_default(void); OSSL_LIB_CTX *OSSL_LIB_CTX_set0_default(OSSL_LIB_CTX *libctx); -void OSSL_sleep(uint64_t millis); - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/cryptoerr.h b/openssl/include/openssl/cryptoerr.h index e84b12df6..c6a04d9b9 100644 --- a/openssl/include/openssl/cryptoerr.h +++ b/openssl/include/openssl/cryptoerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -28,19 +28,10 @@ # define CRYPTO_R_INSUFFICIENT_DATA_SPACE 106 # define CRYPTO_R_INSUFFICIENT_PARAM_SIZE 107 # define CRYPTO_R_INSUFFICIENT_SECURE_DATA_SPACE 108 -# define CRYPTO_R_INTEGER_OVERFLOW 127 # define CRYPTO_R_INVALID_NEGATIVE_VALUE 122 # define CRYPTO_R_INVALID_NULL_ARGUMENT 109 # define CRYPTO_R_INVALID_OSSL_PARAM_TYPE 110 -# define CRYPTO_R_NO_PARAMS_TO_MERGE 131 -# define CRYPTO_R_NO_SPACE_FOR_TERMINATING_NULL 128 # define CRYPTO_R_ODD_NUMBER_OF_DIGITS 103 -# define CRYPTO_R_PARAM_CANNOT_BE_REPRESENTED_EXACTLY 123 -# define CRYPTO_R_PARAM_NOT_INTEGER_TYPE 124 -# define CRYPTO_R_PARAM_OF_INCOMPATIBLE_TYPE 129 -# define CRYPTO_R_PARAM_UNSIGNED_INTEGER_NEGATIVE_VALUE_UNSUPPORTED 125 -# define CRYPTO_R_PARAM_UNSUPPORTED_FLOATING_POINT_FORMAT 130 -# define CRYPTO_R_PARAM_VALUE_TOO_LARGE_FOR_DESTINATION 126 # define CRYPTO_R_PROVIDER_ALREADY_EXISTS 104 # define CRYPTO_R_PROVIDER_SECTION_ERROR 105 # define CRYPTO_R_RANDOM_SECTION_ERROR 119 diff --git a/openssl/include/openssl/cryptoerr_legacy.h b/openssl/include/openssl/cryptoerr_legacy.h index ccab33a5d..0a52043e7 100644 --- a/openssl/include/openssl/cryptoerr_legacy.h +++ b/openssl/include/openssl/cryptoerr_legacy.h @@ -810,12 +810,7 @@ OSSL_DEPRECATEDIN_3_0 int ERR_load_X509V3_strings(void); # define EVP_F_AES_WRAP_CIPHER 0 # define EVP_F_AES_XTS_INIT_KEY 0 # define EVP_F_ALG_MODULE_INIT 0 -# define EVP_F_ARIA_CCM_INIT_KEY 0 -# define EVP_F_ARIA_GCM_CTRL 0 -# define EVP_F_ARIA_GCM_INIT_KEY 0 -# define EVP_F_ARIA_INIT_KEY 0 # define EVP_F_B64_NEW 0 -# define EVP_F_CAMELLIA_INIT_KEY 0 # define EVP_F_CHACHA20_POLY1305_CTRL 0 # define EVP_F_CMLL_T4_INIT_KEY 0 # define EVP_F_DES_EDE3_WRAP_CIPHER 0 @@ -897,7 +892,6 @@ OSSL_DEPRECATEDIN_3_0 int ERR_load_X509V3_strings(void); # define EVP_F_PKCS5_V2_PBKDF2_KEYIVGEN 0 # define EVP_F_PKCS5_V2_SCRYPT_KEYIVGEN 0 # define EVP_F_PKEY_SET_TYPE 0 -# define EVP_F_RC2_MAGIC_TO_METH 0 # define EVP_F_RC5_CTRL 0 # define EVP_F_R_32_12_16_INIT_KEY 0 # define EVP_F_S390X_AES_GCM_CTRL 0 diff --git a/openssl/include/openssl/ct.h b/openssl/include/openssl/ct.h index 0c4f95563..798fda303 100644 --- a/openssl/include/openssl/ct.h +++ b/openssl/include/openssl/ct.h @@ -133,7 +133,7 @@ typedef enum { */ CT_POLICY_EVAL_CTX *CT_POLICY_EVAL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq); - + /* * The same as CT_POLICY_EVAL_CTX_new_ex() but the default library * context and property query string is used. diff --git a/openssl/include/openssl/ct.h.in b/openssl/include/openssl/ct.h.in index 3fc2aaa94..16086b33b 100644 --- a/openssl/include/openssl/ct.h.in +++ b/openssl/include/openssl/ct.h.in @@ -85,7 +85,7 @@ typedef enum { */ CT_POLICY_EVAL_CTX *CT_POLICY_EVAL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq); - + /* * The same as CT_POLICY_EVAL_CTX_new_ex() but the default library * context and property query string is used. diff --git a/openssl/include/openssl/dh.h b/openssl/include/openssl/dh.h index 97024929a..cb0c9d0d5 100644 --- a/openssl/include/openssl/dh.h +++ b/openssl/include/openssl/dh.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,11 +25,7 @@ extern "C" { #include -/* - * DH parameter generation types used by EVP_PKEY_CTX_set_dh_paramgen_type() - * Note that additions/changes to this set of values requires corresponding - * adjustments to range checks in dh_gen() - */ +/* DH parameter generation types used by EVP_PKEY_CTX_set_dh_paramgen_type() */ # define DH_PARAMGEN_TYPE_GENERATOR 0 /* Use a safe prime generator */ # define DH_PARAMGEN_TYPE_FIPS_186_2 1 /* Use FIPS186-2 standard */ # define DH_PARAMGEN_TYPE_FIPS_186_4 2 /* Use FIPS186-4 standard */ @@ -83,9 +79,6 @@ int EVP_PKEY_CTX_get0_dh_kdf_ukm(EVP_PKEY_CTX *ctx, unsigned char **ukm); # define EVP_PKEY_DH_KDF_NONE 1 # define EVP_PKEY_DH_KDF_X9_42 2 -# ifndef OPENSSL_NO_STDIO -# include -# endif # ifndef OPENSSL_NO_DH # include # include diff --git a/openssl/include/openssl/dherr.h b/openssl/include/openssl/dherr.h index 2997d7d4c..074a70145 100644 --- a/openssl/include/openssl/dherr.h +++ b/openssl/include/openssl/dherr.h @@ -40,7 +40,6 @@ # define DH_R_INVALID_PARAMETER_NID 114 # define DH_R_INVALID_PUBKEY 102 # define DH_R_INVALID_SECRET 128 -# define DH_R_INVALID_SIZE 129 # define DH_R_KDF_PARAMETER_ERROR 112 # define DH_R_KEYS_NOT_SET 108 # define DH_R_MISSING_PUBKEY 125 diff --git a/openssl/include/openssl/dsa.h b/openssl/include/openssl/dsa.h index 109878e68..5c0e4cddf 100644 --- a/openssl/include/openssl/dsa.h +++ b/openssl/include/openssl/dsa.h @@ -19,27 +19,12 @@ # include # include -# include - -# ifndef OPENSSL_NO_DSA -# include -# include -# include -# include -# include -# ifndef OPENSSL_NO_DEPRECATED_1_1_0 -# include -# endif -# include -# ifndef OPENSSL_NO_STDIO -# include -# endif -# endif - # ifdef __cplusplus extern "C" { # endif +# include + int EVP_PKEY_CTX_set_dsa_paramgen_bits(EVP_PKEY_CTX *ctx, int nbits); int EVP_PKEY_CTX_set_dsa_paramgen_q_bits(EVP_PKEY_CTX *ctx, int qbits); int EVP_PKEY_CTX_set_dsa_paramgen_md_props(EVP_PKEY_CTX *ctx, @@ -57,6 +42,16 @@ int EVP_PKEY_CTX_set_dsa_paramgen_md(EVP_PKEY_CTX *ctx, const EVP_MD *md); # define EVP_PKEY_CTRL_DSA_PARAMGEN_MD (EVP_PKEY_ALG_CTRL + 3) # ifndef OPENSSL_NO_DSA +# include +# include +# include +# include +# include +# ifndef OPENSSL_NO_DEPRECATED_1_1_0 +# include +# endif +# include + # ifndef OPENSSL_DSA_MAX_MODULUS_BITS # define OPENSSL_DSA_MAX_MODULUS_BITS 10000 # endif diff --git a/openssl/include/openssl/dsaerr.h b/openssl/include/openssl/dsaerr.h index 26ada57d8..5f0ca8d12 100644 --- a/openssl/include/openssl/dsaerr.h +++ b/openssl/include/openssl/dsaerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -38,7 +38,6 @@ # define DSA_R_P_NOT_PRIME 115 # define DSA_R_Q_NOT_PRIME 113 # define DSA_R_SEED_LEN_SMALL 110 -# define DSA_R_TOO_MANY_RETRIES 116 # endif #endif diff --git a/openssl/include/openssl/e_os2.h b/openssl/include/openssl/e_os2.h index f1e17958a..9ce07f6a0 100644 --- a/openssl/include/openssl/e_os2.h +++ b/openssl/include/openssl/e_os2.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -89,7 +89,7 @@ extern "C" { /* * DLL settings. This part is a bit tough, because it's up to the - * application implementer how he or she will link the application, so it + * application implementor how he or she will link the application, so it * requires some macro to be used. */ # ifdef OPENSSL_SYS_WINDOWS @@ -101,22 +101,6 @@ extern "C" { # endif # endif -/* ------------------------------- OpenVMS -------------------------------- */ -# if defined(__VMS) || defined(VMS) -# if !defined(OPENSSL_SYS_VMS) -# undef OPENSSL_SYS_UNIX -# define OPENSSL_SYS_VMS -# endif -# if defined(__DECC) -# define OPENSSL_SYS_VMS_DECC -# elif defined(__DECCXX) -# define OPENSSL_SYS_VMS_DECC -# define OPENSSL_SYS_VMS_DECCXX -# else -# define OPENSSL_SYS_VMS_NODECC -# endif -# endif - /* -------------------------------- Unix ---------------------------------- */ # ifdef OPENSSL_SYS_UNIX # if defined(linux) || defined(__linux__) && !defined(OPENSSL_SYS_LINUX) @@ -228,10 +212,8 @@ typedef INT32 int32_t; typedef UINT32 uint32_t; typedef INT64 int64_t; typedef UINT64 uint64_t; -typedef UINTN uintptr_t; # elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - defined(__osf__) || defined(__sgi) || defined(__hpux) || \ - defined(OPENSSL_SYS_VMS) || defined (__OpenBSD__) + defined(__osf__) || defined(__sgi) || defined (__OpenBSD__) # include # undef OPENSSL_NO_INTTYPES_H /* Because the specs say that inttypes.h includes stdint.h if present */ @@ -249,9 +231,6 @@ typedef int int32_t; typedef unsigned int uint32_t; typedef __int64 int64_t; typedef unsigned __int64 uint64_t; -# elif defined(OPENSSL_SYS_TANDEM) -# include -# include # else # include # undef OPENSSL_NO_STDINT_H @@ -288,7 +267,7 @@ typedef uint64_t ossl_uintmax_t; # endif # if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && \ - !defined(__cplusplus) + !defined(__cplusplus) # define ossl_noreturn _Noreturn # elif defined(__GNUC__) && __GNUC__ >= 2 # define ossl_noreturn __attribute__((noreturn)) diff --git a/openssl/include/openssl/e_ostime.h b/openssl/include/openssl/e_ostime.h deleted file mode 100644 index 0e1748750..000000000 --- a/openssl/include/openssl/e_ostime.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_E_OSTIME_H -# define OPENSSL_E_OSTIME_H -# pragma once - -# include -# include -# include - -/* - * This header guarantees that 'struct timeval' will be available. It includes - * the minimum headers needed to facilitate this. This may still be a - * substantial set of headers on some platforms (e.g. on Win32). - */ - -# if defined(OPENSSL_SYS_WINDOWS) -# if !defined(_WINSOCKAPI_) - /* - * winsock2.h defines _WINSOCK2API_ and both winsock2.h and winsock.h define - * _WINSOCKAPI_. Both of these provide struct timeval. Don't include - * winsock2.h if either header has been included to avoid breakage with - * applications that prefer to use over . - */ -# include -# endif -# else -# include -# endif - -#endif diff --git a/openssl/include/openssl/ebcdic.h b/openssl/include/openssl/ebcdic.h index e0ae1aa84..1c8852526 100644 --- a/openssl/include/openssl/ebcdic.h +++ b/openssl/include/openssl/ebcdic.h @@ -23,9 +23,21 @@ extern "C" { #endif /* Avoid name clashes with other applications */ +# ifdef os_toascii +# undef os_toascii +# endif # define os_toascii _openssl_os_toascii +# ifdef os_toebcdic +# undef os_toebcdic +# endif # define os_toebcdic _openssl_os_toebcdic +# ifdef ebcdic2ascii +# undef ebcdic2ascii +# endif # define ebcdic2ascii _openssl_ebcdic2ascii +# ifdef ascii2ebcdic +# undef ascii2ebcdic +# endif # define ascii2ebcdic _openssl_ascii2ebcdic extern const unsigned char os_toascii[256]; diff --git a/openssl/include/openssl/ec.h b/openssl/include/openssl/ec.h index e1cbe9822..156c4a4b6 100644 --- a/openssl/include/openssl/ec.h +++ b/openssl/include/openssl/ec.h @@ -1,5 +1,5 @@ /* - * Copyright 2002-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -19,6 +19,7 @@ # include # include +# include # include @@ -88,9 +89,6 @@ typedef enum { const char *OSSL_EC_curve_nid2name(int nid); -# ifndef OPENSSL_NO_STDIO -# include -# endif # ifndef OPENSSL_NO_EC # include # include @@ -111,6 +109,7 @@ typedef struct ec_group_st EC_GROUP; typedef struct ec_point_st EC_POINT; typedef struct ecpk_parameters_st ECPKPARAMETERS; typedef struct ec_parameters_st ECPARAMETERS; +typedef struct ec_points_st EC_POINTS; /********************************************************************/ /* EC_METHODs for curves over GF(p) */ @@ -150,6 +149,13 @@ OSSL_DEPRECATEDIN_3_0 const EC_METHOD *EC_GFp_nistp256_method(void); OSSL_DEPRECATEDIN_3_0 const EC_METHOD *EC_GFp_nistp521_method(void); # endif /* OPENSSL_NO_EC_NISTP_64_GCC_128 */ +# if !defined(OPENSSL_NO_EC_SM2P_64_GCC_128) && !defined(OPENSSL_NO_SM2) +/** Returns 64-bit optimized methods for sm2p256 + * \return EC_METHOD object + */ +OSSL_DEPRECATEDIN_3_0 const EC_METHOD *EC_GFp_sm2p256_method(void); +# endif /* OPENSSL_NO_EC_SM2P_64_GCC_128 */ + # ifndef OPENSSL_NO_EC2M /********************************************************************/ /* EC_METHOD for curves over GF(2^m) */ @@ -191,6 +197,32 @@ OSSL_DEPRECATEDIN_3_0 const EC_METHOD *EC_GROUP_method_of(const EC_GROUP *group) OSSL_DEPRECATEDIN_3_0 int EC_METHOD_get_field_type(const EC_METHOD *meth); # endif /* OPENSSL_NO_DEPRECATED_3_0 */ +#ifndef FIPS_MODULE +# ifndef OPENSSL_NO_ENGINE + +/** Creates a new EC_GROUP object, and binding engine + * \param meth EC_METHOD to use + * \param engine ENGINE to use + * \return newly created EC_GROUP object or NULL in case of an error. + */ +EC_GROUP *EC_GROUP_new_ex(const EC_METHOD *meth, ENGINE *engine); + +/** Sets the engine of a EC_GROUP object. + * \param meth EC_METHOD to use + * \param engine ENGINE to use + * \return 1 on success and 0 if an error occurred. + */ +int EC_GROUP_set_engine(EC_GROUP *group, ENGINE *engine); + +/** Returns the ENGINE object of a EC_GROUP object + * \param group EC_GROUP object + * \return the ENGINE object (possibly NULL). + */ +const ENGINE *EC_GROUP_get0_engine(const EC_GROUP *group); + +# endif +#endif + /** Frees a EC_GROUP object * \param group EC_GROUP object to be freed. */ @@ -460,22 +492,6 @@ EC_GROUP *EC_GROUP_new_curve_GF2m(const BIGNUM *p, const BIGNUM *a, EC_GROUP *EC_GROUP_new_from_params(const OSSL_PARAM params[], OSSL_LIB_CTX *libctx, const char *propq); -/** - * Creates an OSSL_PARAM array with the parameters describing the given - * EC_GROUP. - * The resulting parameters may contain an explicit or a named curve depending - * on the EC_GROUP. - * \param group pointer to the EC_GROUP object - * \param libctx The associated library context or NULL for the default - * context - * \param propq A property query string - * \param bnctx BN_CTX object (optional) - * \return newly created OSSL_PARAM array with the parameters - * describing the given EC_GROUP or NULL if an error occurred - */ -OSSL_PARAM *EC_GROUP_to_params(const EC_GROUP *group, OSSL_LIB_CTX *libctx, - const char *propq, BN_CTX *bnctx); - /** * Creates a EC_GROUP object with a curve specified by a NID * \param libctx The associated library context or NULL for the default @@ -552,6 +568,64 @@ int EC_curve_nist2nid(const char *name); int EC_GROUP_check_named_curve(const EC_GROUP *group, int nist_only, BN_CTX *ctx); +/********************************************************************/ +/* EC_POINTS functions */ +/********************************************************************/ + +/** Creates a new EC_POINTS object for the specified EC_GROUP and the count of + * EC_POINT + * \param group EC_GROUP the underlying EC_GROUP object + * \param count the count of EC_POINT + * \return newly created EC_POINTS object or NULL if an error occurred + */ +EC_POINTS *EC_POINTS_new(const EC_GROUP *group, int count); + +/** Frees a EC_POINTS object + * \param points EC_POINTS object to be freed + */ +void EC_POINTS_free(EC_POINTS *points); + +/** Clears and frees a EC_POINTS object + * \param points EC_POINTS object to be cleared and freed + */ +void EC_POINTS_clear_free(EC_POINTS *points); + +/** Copies EC_POINTS object + * \param dst destination EC_POINTS object + * \param src source EC_POINTS object + * \return 1 on success and 0 if an error occurred + */ +int EC_POINTS_copy(EC_POINTS *dst, const EC_POINTS *src); + +/** Creates a new EC_POINTS object and copies the content of the supplied + * EC_POINT + * \param src source EC_POINTS object + * \param group underlying the EC_GROUP object + * \return newly created EC_POINTS object or NULL if an error occurred + */ +EC_POINTS *EC_POINTS_dup(const EC_POINTS *src, const EC_GROUP *group); + +/** Returns the i-th EC_POINT object in EC_POINTS object + * \param p EC_POINTS object + * \param i the index number + * \return EC_POINT object or NULL if an error occurred. + */ +EC_POINT *EC_POINTS_get_item(EC_POINTS *p, int i); + +/** Stores EC_POINT object into EC_POINTS object + * \param p EC_POINTS object + * \param i the index number + * \param point EC_POINT object + * \return 1 on success and 0 if an error occurred + */ +int EC_POINTS_set_item(EC_POINTS *p, int i, EC_POINT *point); + +/** Returns the count of EC_POINT object in the EC_POINTS object. + * \param p EC_POINTS object + * \return the count of EC_POINT object in the EC_POINTS object. + */ +int EC_POINTS_count(EC_POINTS *p); + /********************************************************************/ /* EC_POINT functions */ /********************************************************************/ @@ -587,6 +661,19 @@ int EC_POINT_copy(EC_POINT *dst, const EC_POINT *src); */ EC_POINT *EC_POINT_dup(const EC_POINT *src, const EC_GROUP *group); +#ifndef FIPS_MODULE +/* + * Functions for convert string to ec_point on the elliptic curve. + * \param group underlying EC_GROUP object + * \param r EC_POINT object for the result + * \param str string pointer + * \param len length of the string + * \return 1 on success and 0 if an error occurred + */ +int EC_POINT_from_string(const EC_GROUP *group, EC_POINT *r, + const unsigned char *str, size_t len); +#endif + /** Sets a point to infinity (neutral element) * \param group underlying EC_GROUP object * \param point EC_POINT to set to infinity @@ -601,6 +688,12 @@ int EC_POINT_set_to_infinity(const EC_GROUP *group, EC_POINT *point); */ OSSL_DEPRECATEDIN_3_0 const EC_METHOD *EC_POINT_method_of(const EC_POINT *point); +/** Returns the curve name of a EC_POINT object + * \param point EC_POINT object + * \return NID of the curve name OID or 0 if not set. + */ +int EC_POINT_get_curve_name(const EC_POINT *point); + /** Sets the jacobian projective coordinates of a EC_POINT over GFp * \param group underlying EC_GROUP object * \param p EC_POINT object @@ -893,6 +986,172 @@ OSSL_DEPRECATEDIN_3_0 int EC_POINTs_mul(const EC_GROUP *group, EC_POINT *r, int EC_POINT_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *n, const EC_POINT *q, const BIGNUM *m, BN_CTX *ctx); +/* + * Functions for point multiplication: r[i] = points[i] * scalars[i] + * \param group underlying EC_GROUP object + * \param r a pointer to a EC_POINTS object for the result (if *r NULL + * the function allocates a EC_POINTS object and write to *r) + * \param num number of points and scalars + * \param points array of size num of EC_POINT objects + * \param scalars array of size num of BIGNUM objects + * \param ctx BN_CTX object (optional) + * \return 1 on success and 0 if an error occurred + */ +int EC_POINTs_scalars_mul(const EC_GROUP *group, EC_POINTS **r, size_t num, + const EC_POINT *points[], const BIGNUM *scalars[], + BN_CTX *ctx); + +/* + * Functions for point multiplication: r[i] = points[i] * scalar + * \param group underlying EC_GROUP object + * \param r a pointer to a EC_POINTS object for the result (if *r NULL + * the function allocates a EC_POINTS object and write to *r) + * \param num number of points + * \param points array of size num of EC_POINT objects + * \param scalar BIGNUM object for scalar multiply + * \param ctx BN_CTX object (optional) + * \return 1 on success and 0 if an error occurred + */ +int EC_POINTs_scalar_mul(const EC_GROUP *group, EC_POINTS **r, size_t num, + const EC_POINT *points[], const BIGNUM *scalar, + BN_CTX *ctx); + +/* + * Functions for convert some strings to some points on the elliptic curve. + * r[i]->X = hash(strings[i]) + * r[i]->Y = F(hash(strings[i])), the Y coordinate can be calculated by taking + * the X coordinate into the equation + * r[i]->Z = 1 + * \param group underlying EC_GROUP object + * \param r a pointer to a EC_POINTS object for the result (if *r NULL + * the function allocates a EC_POINTS object and write to *r) + * \param num number of strings + * \param strings array of size num of string objects + * \param ctx BN_CTX object (optional) + * \return 1 on success and 0 if an error occurred + */ +int EC_POINTs_from_strings(const EC_GROUP *group, EC_POINTS **r, + size_t num, const unsigned char *strings[], + BN_CTX *ctx); + +/* + * Functions for convert some strings to some points on the elliptic curve, then + * multiply with scalar. + * point[i]->X = hash(strings[i]) + * point[i]->Y = F(hash(strings[i])), the Y coordinate can be calculated by taking + * the X coordinate into the equation + * point[i]->Z = 1 + * r[i] = scalar * point[i] + * \param group underlying EC_GROUP object + * \param r a pointer to a EC_POINTS object for the result (if *r NULL + * the function allocates a EC_POINTS object and write to *r) + * \param num number of strings + * \param strings array of size num of string objects + * \param scalar BIGNUM object for scalar multiply + * \param ctx BN_CTX object (optional) + * \return 1 on success and 0 if an error occurred + */ +int EC_POINTs_from_strings_scalar_mul(const EC_GROUP *group, EC_POINTS **r, + size_t num, const unsigned char *strings[], + const BIGNUM *scalar, BN_CTX *ctx); + +/********************************************************************/ +/* EC_POINT_METHOD constructors, destructors, writers and accessors */ +/********************************************************************/ + +/** Creates a new EC_POINT_METHOD object for the specified curve_id + * \param curve_id the elliptic curve id + * \return newly created EC_POINT_METHOD object or NULL if an error occurred + */ +EC_POINT_METHOD *EC_POINT_METHOD_new(int curve_id); + +/** Frees a EC_POINT_METHOD object + * \param meth EC_POINT_METHOD object to be freed + */ +void EC_POINT_METHOD_free(EC_POINT_METHOD *meth); + +/** Copies EC_POINT_METHOD object + * \param dst destination EC_POINT_METHOD object + * \param src source EC_POINT_METHOD object + * \return 1 on success and 0 if an error occurred + */ +int EC_POINT_METHOD_copy(EC_POINT_METHOD *dst, const EC_POINT_METHOD *src); + +/** Returns the curve_id of a EC_POINT_METHOD object + * \param meth EC_POINT_METHOD object + * \return NID of the curve name OID or 0 if not set. + */ +int EC_POINT_METHOD_curve_id(EC_POINT_METHOD *meth); + +int (*EC_POINT_METHOD_get_add(EC_POINT_METHOD *meth)) + (const EC_GROUP *, EC_POINT *r, const EC_POINT *a, const EC_POINT *b, + BN_CTX *); +void EC_POINT_METHOD_set_add(EC_POINT_METHOD *meth, + int (*add)(const EC_GROUP *, EC_POINT *r, + const EC_POINT *a, const EC_POINT *b, + BN_CTX *)); + +int (*EC_POINT_METHOD_get_dbl(EC_POINT_METHOD *meth)) + (const EC_GROUP *, EC_POINT *r, const EC_POINT *a, BN_CTX *); +void EC_POINT_METHOD_set_dbl(EC_POINT_METHOD *meth, + int (*dbl)(const EC_GROUP *, EC_POINT *r, + const EC_POINT *a, BN_CTX *)); + +int (*EC_POINT_METHOD_get_invert(EC_POINT_METHOD *meth)) + (const EC_GROUP *, EC_POINT *point, BN_CTX *); +void EC_POINT_METHOD_set_invert(EC_POINT_METHOD *meth, + int (*invert)(const EC_GROUP *, EC_POINT *point, + BN_CTX *)); + +int (*EC_POINT_METHOD_get_mul(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, size_t num, + const EC_POINT *points[], const BIGNUM *scalars[], BN_CTX *); +void EC_POINT_METHOD_set_mul(EC_POINT_METHOD *meth, + int (*mul)(const EC_GROUP *group, EC_POINT *r, + const BIGNUM *scalar, size_t num, + const EC_POINT *points[], + const BIGNUM *scalars[], BN_CTX *)); + +int (*EC_POINT_METHOD_get_scalars_mul(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r[], size_t num, const EC_POINT *points[], + const BIGNUM *scalars[], BN_CTX *ctx); +void EC_POINT_METHOD_set_scalars_mul(EC_POINT_METHOD *meth, + int (*scalars_mul)(const EC_GROUP *group, + EC_POINT *r[], size_t num, + const EC_POINT *points[], + const BIGNUM *scalars[], + BN_CTX *ctx)); + +int (*EC_POINT_METHOD_get_scalar_mul(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r[], size_t num, const EC_POINT *points[], + const BIGNUM *scalar, BN_CTX *ctx); +void EC_POINT_METHOD_set_scalar_mul(EC_POINT_METHOD *meth, + int (*scalar_mul)(const EC_GROUP *group, + EC_POINT *r[], size_t num, + const EC_POINT *points[], + const BIGNUM *scalar, + BN_CTX *ctx)); + +int (*EC_POINT_METHOD_get_strings_to_points(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r[], size_t num, const unsigned char *[], + BN_CTX *ctx); +void EC_POINT_METHOD_set_strings_to_points(EC_POINT_METHOD *meth, + int (*func)(const EC_GROUP *, + EC_POINT *[], size_t, + const unsigned char *[], + BN_CTX *)); + +int (*EC_POINT_METHOD_get_strings_to_points_scalar_mul(EC_POINT_METHOD *meth)) + (const EC_GROUP *, EC_POINT *[], size_t, const unsigned char *[], + const BIGNUM *, BN_CTX *); +void EC_POINT_METHOD_set_strings_to_points_scalar_mul(EC_POINT_METHOD *meth, + int (*func)(const EC_GROUP *, + EC_POINT *[], + size_t, + const unsigned char *[], + const BIGNUM *, + BN_CTX *)); + # ifndef OPENSSL_NO_DEPRECATED_3_0 /** Stores multiples of generator for faster point multiplication * \param group EC_GROUP object @@ -1127,7 +1386,7 @@ OSSL_DEPRECATEDIN_3_0 int EC_KEY_check_key(const EC_KEY *key); /** Indicates if an EC_KEY can be used for signing. * \param eckey the EC_KEY object - * \return 1 if can sign and 0 otherwise. + * \return 1 if can can sign and 0 otherwise. */ OSSL_DEPRECATEDIN_3_0 int EC_KEY_can_sign(const EC_KEY *eckey); @@ -1303,7 +1562,7 @@ OSSL_DEPRECATEDIN_3_0 int EC_KEY_set_method(EC_KEY *key, const EC_KEY_METHOD *me OSSL_DEPRECATEDIN_3_0 EC_KEY *EC_KEY_new_method(ENGINE *engine); /** The old name for ecdh_KDF_X9_63 - * The ECDH KDF specification has been mistakenly attributed to ANSI X9.62, + * The ECDH KDF specification has been mistakingly attributed to ANSI X9.62, * it is actually specified in ANSI X9.63. * This identifier is retained for backwards compatibility */ @@ -1581,6 +1840,231 @@ OSSL_DEPRECATEDIN_3_0 void EC_KEY_METHOD_get_verify # endif # endif +# ifndef OPENSSL_NO_EC_ELGAMAL +/********************************************************************/ +/* EC_ELGAMAL for curves over GF(p) */ +/********************************************************************/ +# define EC_ELGAMAL_MAX_BITS 32 +# define EC_ELGAMAL_FLAG_DEFAULT 0x00 +# define EC_ELGAMAL_FLAG_TWISTED 0x01 +# define EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE 0x01 +# define EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE_FIRST 0x02 +# define EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE_ONLY 0x03 + +STACK_OF(EC_KEY); + +typedef struct ec_elgamal_ctx_st EC_ELGAMAL_CTX; +typedef struct ec_elgamal_mr_ctx_st EC_ELGAMAL_MR_CTX; +typedef struct ec_elgamal_ciphertext_st EC_ELGAMAL_CIPHERTEXT; +typedef struct ec_elgamal_mr_ciphertext_st EC_ELGAMAL_MR_CIPHERTEXT; +typedef struct ec_elgamal_decrypt_table_st EC_ELGAMAL_DECRYPT_TABLE; + +/********************************************************************/ +/* EC_ELGAMAL functions */ +/********************************************************************/ + +/** Creates a new EC_ELGAMAL object + * \param key EC_KEY to use + * \param h EC_POINT object pointer + * \param flag flag of ctx + * \return newly created EC_ELGAMAL_CTX object or NULL in case of an error + */ +EC_ELGAMAL_CTX *EC_ELGAMAL_CTX_new(EC_KEY *key, const EC_POINT *h, int32_t flag); +EC_ELGAMAL_CTX *EC_ELGAMAL_CTX_dup(EC_ELGAMAL_CTX *ctx); + +/** Frees a EC_ELGAMAL_CTX object + * \param ctx EC_ELGAMAL_CTX object to be freed + */ +void EC_ELGAMAL_CTX_free(EC_ELGAMAL_CTX *ctx); + +/** Creates a new EC_ELGAMAL_MR_CTX object + * \param key EC_KEY to use + * \param flag flag of ctx + * \return newly created EC_ELGAMAL_MR_CTX object or NULL in case of an error + */ +EC_ELGAMAL_MR_CTX *EC_ELGAMAL_MR_CTX_new(STACK_OF(EC_KEY) *keys, const EC_POINT *h, + int32_t flag); + +/** Frees a EC_ELGAMAL_MR_CTX object + * \param ctx EC_ELGAMAL_MR_CTX object to be freed + */ +void EC_ELGAMAL_MR_CTX_free(EC_ELGAMAL_MR_CTX *ctx); + +/** Creates a new EC_ELGAMAL_DECRYPT_TABLE object + * \param ctx EC_ELGAMAL_CTX object + * \param decrypt_negative Whether negative numbers can be decrypted (1 or 0) + * \return newly created EC_ELGAMAL_DECRYPT_TABLE object or NULL in case of an error + */ +EC_ELGAMAL_DECRYPT_TABLE *EC_ELGAMAL_DECRYPT_TABLE_new(EC_ELGAMAL_CTX *ctx, + int32_t decrypt_negative); + +/** Creates a new EC_ELGAMAL_DECRYPT_TABLE object with some extra paramers + * \param ctx EC_ELGAMAL_CTX object + * \param flag the flag of decrypt table + * \param baby_step_bits baby step exponent/bits + * \param giant_step_bits giant step exponent/bits + * \return newly created EC_ELGAMAL_DECRYPT_TABLE object or NULL in case of an error + */ +EC_ELGAMAL_DECRYPT_TABLE *EC_ELGAMAL_DECRYPT_TABLE_new_ex(EC_ELGAMAL_CTX *ctx, + int32_t flag, + uint32_t baby_step_bits, + uint32_t giant_step_bits); + +/** Frees a EC_ELGAMAL_DECRYPT_TABLE object + * \param table EC_ELGAMAL_DECRYPT_TABLE object to be freed + */ +void EC_ELGAMAL_DECRYPT_TABLE_free(EC_ELGAMAL_DECRYPT_TABLE *table); + +/** Sets a EC_ELGAMAL_DECRYPT_TABLE object for decryption. + * \param ctx EC_ELGAMAL_CTX object + * \param table EC_ELGAMAL_DECRYPT_TABLE object + */ +void EC_ELGAMAL_CTX_set_decrypt_table(EC_ELGAMAL_CTX *ctx, + EC_ELGAMAL_DECRYPT_TABLE *table); + +/** Encrypts an Integer with additadive homomorphic EC-ElGamal + * \param ctx EC_ELGAMAL_CTX object. + * \param r EC_ELGAMAL_CIPHERTEXT object that stores the result of + * the encryption + * \param plaintext The plaintext integer to be encrypted + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_encrypt(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, int32_t plaintext); +int EC_ELGAMAL_bn_encrypt(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + const BIGNUM *plaintext, const BIGNUM *rand); + +/** Encryption with one plaintext for multiple recipients. + * \param ctx EC_ELGAMAL_CTX object. + * \param r EC_ELGAMAL_CIPHERTEXT_MR object that stores the result of + * the encryption + * \param plaintext The plaintext BIGNUM object to be encrypted + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_MR_encrypt(EC_ELGAMAL_MR_CTX *ctx, EC_ELGAMAL_MR_CIPHERTEXT *r, + const BIGNUM *plaintext, BIGNUM *rand); + +/** Decrypts the ciphertext + * \param ctx EC_ELGAMAL_CTX object + * \param r The resulting plaintext integer + * \param cihpertext EC_ELGAMAL_CIPHERTEXT object to be decrypted + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_decrypt(EC_ELGAMAL_CTX *ctx, int32_t *r, + const EC_ELGAMAL_CIPHERTEXT *ciphertext); + +/** Adds two EC-Elgamal ciphertext and stores it in r (r = c1 + c2). + * \param ctx EC_ELGAMAL_CTX object + * \param r The EC_ELGAMAL_CIPHERTEXT object that stores the addition + * result + * \param c1 EC_ELGAMAL_CIPHERTEXT object + * \param c2 EC_ELGAMAL_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_add(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + const EC_ELGAMAL_CIPHERTEXT *c1, + const EC_ELGAMAL_CIPHERTEXT *c2); + +/** Substracts two EC-Elgamal ciphertext and stores it in r (r = c1 - c2). + * \param ctx EC_ELGAMAL_CTX object + * \param r The EC_ELGAMAL_CIPHERTEXT object that stores the + * subtraction result + * \param c1 EC_ELGAMAL_CIPHERTEXT object + * \param c2 EC_ELGAMAL_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_sub(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + const EC_ELGAMAL_CIPHERTEXT *c1, + const EC_ELGAMAL_CIPHERTEXT *c2); + +/** Ciphertext multiplication, computes r = c * m + * \param ctx EC_ELGAMAL_CTX object + * \param r The EC_ELGAMAL_CIPHERTEXT object that stores the + * multiplication result + * \param c1 EC_ELGAMAL_CIPHERTEXT object + * \param c2 EC_ELGAMAL_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_mul(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + const EC_ELGAMAL_CIPHERTEXT *c, int32_t m); + +/** Creates a new EC_ELGAMAL_CIPHERTEXT object for EC-ELGAMAL oparations + * \param ctx EC_ELGAMAL_CTX object + * \return newly created EC_ELGAMAL_CIPHERTEXT object or NULL in case of an error + */ +EC_ELGAMAL_CIPHERTEXT *EC_ELGAMAL_CIPHERTEXT_new(EC_ELGAMAL_CTX *ctx); +EC_ELGAMAL_CIPHERTEXT *EC_ELGAMAL_CIPHERTEXT_dup(const EC_ELGAMAL_CIPHERTEXT *ct, + const EC_GROUP *group); + +/** Frees a EC_ELGAMAL_CIPHERTEXT object + * \param ciphertext EC_ELGAMAL_CIPHERTEXT object to be freed + */ +void EC_ELGAMAL_CIPHERTEXT_free(EC_ELGAMAL_CIPHERTEXT *ciphertext); + +/** Creates a new EC_ELGAMAL_MR_CIPHERTEXT object for EC-ELGAMAL oparations + * \param ctx EC_ELGAMAL_MR_CTX object + * \return newly created EC_ELGAMAL_MR_CIPHERTEXT object or NULL in case of an error + */ +EC_ELGAMAL_MR_CIPHERTEXT *EC_ELGAMAL_MR_CIPHERTEXT_new(EC_ELGAMAL_MR_CTX *ctx); +EC_ELGAMAL_MR_CIPHERTEXT *EC_ELGAMAL_MR_CIPHERTEXT_dup(const EC_ELGAMAL_MR_CIPHERTEXT *ct, + const EC_GROUP *group); + +/** Frees a EC_ELGAMAL_MR_CIPHERTEXT object + * \param ciphertext EC_ELGAMAL_MR_CIPHERTEXT object to be freed + */ +void EC_ELGAMAL_MR_CIPHERTEXT_free(EC_ELGAMAL_MR_CIPHERTEXT *ciphertext); + +/** Encodes EC_ELGAMAL_CIPHERTEXT to binary + * \param ctx EC_ELGAMAL_CTX object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param ciphertext EC_ELGAMAL_CIPHERTEXT object + * \param compressed Whether to compress the encoding (either 0 or 1) + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t EC_ELGAMAL_CIPHERTEXT_encode(EC_ELGAMAL_CTX *ctx, unsigned char *out, + size_t size, + const EC_ELGAMAL_CIPHERTEXT *ciphertext, + int compressed); + +/** Decodes binary to EC_ELGAMAL_CIPHERTEXT + * \param ctx EC_ELGAMAL_CTX object + * \param r the resulting ciphertext + * \param in Memory buffer with the encoded EC_ELGAMAL_CIPHERTEXT + * object + * \param size The memory size of the in pointer object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_CIPHERTEXT_decode(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + unsigned char *in, size_t size); + +/** Encodes EC_ELGAMAL_MR_CIPHERTEXT to binary + * \param ctx EC_ELGAMAL_MR_CTX object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param ciphertext EC_ELGAMAL_MR_CIPHERTEXT object + * \param compressed Whether to compress the encoding (either 0 or 1) + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t EC_ELGAMAL_MR_CIPHERTEXT_encode(EC_ELGAMAL_MR_CTX *ctx, unsigned char *out, + size_t size, + const EC_ELGAMAL_MR_CIPHERTEXT *ciphertext, + int compressed); + +/** Decodes binary to EC_ELGAMAL_MR_CIPHERTEXT + * \param ctx EC_ELGAMAL_MR_CTX object + * \param r the resulting ciphertext + * \param in Memory buffer with the encoded EC_ELGAMAL_MR_CIPHERTEXT + * object + * \param size The memory size of the in pointer object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_MR_CIPHERTEXT_decode(EC_ELGAMAL_MR_CTX *ctx, EC_ELGAMAL_MR_CIPHERTEXT *r, + unsigned char *in, size_t size); + +# endif + # endif # ifdef __cplusplus } diff --git a/openssl/include/openssl/ecerr.h b/openssl/include/openssl/ecerr.h index f15f91f6b..b9db0e405 100644 --- a/openssl/include/openssl/ecerr.h +++ b/openssl/include/openssl/ecerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -35,6 +35,8 @@ # define EC_R_DECODE_ERROR 142 # define EC_R_DISCRIMINANT_IS_ZERO 118 # define EC_R_EC_GROUP_NEW_BY_NAME_FAILURE 119 +# define EC_R_EC_POINT_METHOD_NOT_FOUND 176 +# define EC_R_ELGAMAL_DLOG_FAILED 177 # define EC_R_EXPLICIT_PARAMS_NOT_SUPPORTED 127 # define EC_R_FAILED_MAKING_PUBLIC_KEY 166 # define EC_R_FIELD_TOO_LARGE 143 @@ -90,7 +92,6 @@ # define EC_R_RANDOM_NUMBER_GENERATION_FAILED 158 # define EC_R_SHARED_INFO_ERROR 150 # define EC_R_SLOT_FULL 108 -# define EC_R_TOO_MANY_RETRIES 176 # define EC_R_UNDEFINED_GENERATOR 113 # define EC_R_UNDEFINED_ORDER 128 # define EC_R_UNKNOWN_COFACTOR 164 diff --git a/openssl/include/openssl/engine.h b/openssl/include/openssl/engine.h index 2fbc82c3f..a221e50fd 100644 --- a/openssl/include/openssl/engine.h +++ b/openssl/include/openssl/engine.h @@ -46,6 +46,9 @@ extern "C" { # define ENGINE_METHOD_DSA (unsigned int)0x0002 # define ENGINE_METHOD_DH (unsigned int)0x0004 # define ENGINE_METHOD_RAND (unsigned int)0x0008 +# ifndef OPENSSL_NO_BN_METHOD +# define ENGINE_METHOD_BN (unsigned int)0x0010 +# endif # define ENGINE_METHOD_CIPHERS (unsigned int)0x0040 # define ENGINE_METHOD_DIGESTS (unsigned int)0x0080 # define ENGINE_METHOD_PKEY_METHS (unsigned int)0x0200 @@ -301,6 +304,8 @@ typedef int (*ENGINE_PKEY_METHS_PTR) (ENGINE *, EVP_PKEY_METHOD **, const int **, int); typedef int (*ENGINE_PKEY_ASN1_METHS_PTR) (ENGINE *, EVP_PKEY_ASN1_METHOD **, const int **, int); +typedef int (*ENGINE_ECP_METHS_PTR) (ENGINE *, const EC_POINT_METHOD **, + const int **, int); /* * STRUCTURE functions ... all of these functions deal with pointers to * ENGINE structures where the pointers have a "structural reference". This @@ -404,6 +409,16 @@ OSSL_DEPRECATEDIN_3_0 void ENGINE_unregister_pkey_asn1_meths(ENGINE *e); OSSL_DEPRECATEDIN_3_0 void ENGINE_register_all_pkey_asn1_meths(void); # endif +int ENGINE_register_ecp_meths(ENGINE *e); +void ENGINE_unregister_ecp_meths(ENGINE *e); +void ENGINE_register_all_ecp_meths(void); + +# ifndef OPENSSL_NO_BN_METHOD +int ENGINE_register_bn_meth(ENGINE *e); +void ENGINE_unregister_bn_meth(ENGINE *e); +void ENGINE_register_all_bn_meth(void); +# endif + /* * These functions register all support from the above categories. Note, use * of these functions can result in static linkage of code your application @@ -526,6 +541,11 @@ OSSL_DEPRECATEDIN_3_0 int ENGINE_set_flags(ENGINE *e, int flags); OSSL_DEPRECATEDIN_3_0 int ENGINE_set_cmd_defns(ENGINE *e, const ENGINE_CMD_DEFN *defns); # endif +int ENGINE_set_ecp_meths(ENGINE *e, ENGINE_ECP_METHS_PTR f); +# ifndef OPENSSL_NO_BN_METHOD +int ENGINE_set_bn_meth(ENGINE *e, const BN_METHOD *bn_meth); +# endif + /* These functions allow control over any per-structure ENGINE data. */ # define ENGINE_get_ex_new_index(l, p, newf, dupf, freef) \ CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_ENGINE, l, p, newf, dupf, freef) @@ -598,6 +618,13 @@ const ENGINE_CMD_DEFN *ENGINE_get_cmd_defns(const ENGINE *e); OSSL_DEPRECATEDIN_3_0 int ENGINE_get_flags(const ENGINE *e); # endif +ENGINE_ECP_METHS_PTR ENGINE_get_ecp_meths(ENGINE *e); +const EC_POINT_METHOD *ENGINE_get_ecp_meth(ENGINE *e, int curve_id); + +# ifndef OPENSSL_NO_BN_METHOD +const BN_METHOD *ENGINE_get_bn_meth(ENGINE *e); +# endif + /* * FUNCTIONAL functions. These functions deal with ENGINE structures that * have (or will) be initialised for use. Broadly speaking, the structural @@ -612,7 +639,7 @@ OSSL_DEPRECATEDIN_3_0 int ENGINE_get_flags(const ENGINE *e); */ /* - * Initialise an engine type for use (or up its reference count if it's + * Initialise a engine type for use (or up its reference count if it's * already in use). This will fail if the engine is not currently operational * and cannot initialise. */ @@ -620,7 +647,7 @@ OSSL_DEPRECATEDIN_3_0 int ENGINE_get_flags(const ENGINE *e); OSSL_DEPRECATEDIN_3_0 int ENGINE_init(ENGINE *e); # endif /* - * Free a functional reference to an engine type. This does not require a + * Free a functional reference to a engine type. This does not require a * corresponding call to ENGINE_free as it also releases a structural * reference. */ @@ -674,6 +701,8 @@ OSSL_DEPRECATEDIN_3_0 ENGINE *ENGINE_get_pkey_meth_engine(int nid); OSSL_DEPRECATEDIN_3_0 ENGINE *ENGINE_get_pkey_asn1_meth_engine(int nid); # endif +ENGINE *ENGINE_get_ecp_meth_engine(int curve_id); + /* * This sets a new default ENGINE structure for performing RSA operations. If * the result is non-zero (success) then the ENGINE structure will have had @@ -697,6 +726,13 @@ OSSL_DEPRECATEDIN_3_0 int ENGINE_set_default_pkey_meths(ENGINE *e); OSSL_DEPRECATEDIN_3_0 int ENGINE_set_default_pkey_asn1_meths(ENGINE *e); # endif +int ENGINE_set_default_ecp_meths(ENGINE *e); + +# ifndef OPENSSL_NO_BN_METHOD +int ENGINE_set_default_bn_meth(ENGINE *e); +ENGINE *ENGINE_get_default_bn_meth(void); +# endif + /* * The combination "set" - the flags are bitwise "OR"d from the * ENGINE_METHOD_*** defines above. As with the "ENGINE_register_complete()" diff --git a/openssl/include/openssl/engineerr.h b/openssl/include/openssl/engineerr.h index d439b6827..e7c0cf3a0 100644 --- a/openssl/include/openssl/engineerr.h +++ b/openssl/include/openssl/engineerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -56,6 +56,7 @@ # define ENGINE_R_NO_SUCH_ENGINE 116 # define ENGINE_R_UNIMPLEMENTED_CIPHER 146 # define ENGINE_R_UNIMPLEMENTED_DIGEST 147 +# define ENGINE_R_UNIMPLEMENTED_ECP_METH 107 # define ENGINE_R_UNIMPLEMENTED_PUBLIC_KEY_METHOD 101 # define ENGINE_R_VERSION_INCOMPATIBILITY 145 diff --git a/openssl/include/openssl/err.h b/openssl/include/openssl/err.h index daca18e7b..1ae1d9b1e 100644 --- a/openssl/include/openssl/err.h +++ b/openssl/include/openssl/err.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -120,6 +120,10 @@ struct err_state_st { # define ERR_LIB_OSSL_ENCODER 59 # define ERR_LIB_OSSL_DECODER 60 # define ERR_LIB_HTTP 61 +# define ERR_LIB_PAILLIER 62 +# define ERR_LIB_ZKP 63 +# define ERR_LIB_ZKP_BP 64 +# define ERR_LIB_ZKP_NIZK 65 # define ERR_LIB_USER 128 @@ -167,6 +171,7 @@ struct err_state_st { # define UIerr(f, r) ERR_raise_data(ERR_LIB_UI, (r), NULL) # define X509V3err(f, r) ERR_raise_data(ERR_LIB_X509V3, (r), NULL) # define X509err(f, r) ERR_raise_data(ERR_LIB_X509, (r), NULL) +# define PAILLIERerr(f, r) ERR_raise_data(ERR_LIB_PAILLIER, (r), NULL) # endif /*- @@ -323,27 +328,15 @@ static ossl_unused ossl_inline int ERR_COMMON_ERROR(unsigned long errcode) # define ERR_R_DSA_LIB (ERR_LIB_DSA/* 10 */ | ERR_RFLAG_COMMON) # define ERR_R_X509_LIB (ERR_LIB_X509/* 11 */ | ERR_RFLAG_COMMON) # define ERR_R_ASN1_LIB (ERR_LIB_ASN1/* 13 */ | ERR_RFLAG_COMMON) -# define ERR_R_CONF_LIB (ERR_LIB_CONF/* 14 */ | ERR_RFLAG_COMMON) # define ERR_R_CRYPTO_LIB (ERR_LIB_CRYPTO/* 15 */ | ERR_RFLAG_COMMON) # define ERR_R_EC_LIB (ERR_LIB_EC/* 16 */ | ERR_RFLAG_COMMON) -# define ERR_R_SSL_LIB (ERR_LIB_SSL/* 20 */ | ERR_RFLAG_COMMON) # define ERR_R_BIO_LIB (ERR_LIB_BIO/* 32 */ | ERR_RFLAG_COMMON) # define ERR_R_PKCS7_LIB (ERR_LIB_PKCS7/* 33 */ | ERR_RFLAG_COMMON) # define ERR_R_X509V3_LIB (ERR_LIB_X509V3/* 34 */ | ERR_RFLAG_COMMON) -# define ERR_R_PKCS12_LIB (ERR_LIB_PKCS12/* 35 */ | ERR_RFLAG_COMMON) -# define ERR_R_RAND_LIB (ERR_LIB_RAND/* 36 */ | ERR_RFLAG_COMMON) -# define ERR_R_DSO_LIB (ERR_LIB_DSO/* 37 */ | ERR_RFLAG_COMMON) # define ERR_R_ENGINE_LIB (ERR_LIB_ENGINE/* 38 */ | ERR_RFLAG_COMMON) # define ERR_R_UI_LIB (ERR_LIB_UI/* 40 */ | ERR_RFLAG_COMMON) # define ERR_R_ECDSA_LIB (ERR_LIB_ECDSA/* 42 */ | ERR_RFLAG_COMMON) # define ERR_R_OSSL_STORE_LIB (ERR_LIB_OSSL_STORE/* 44 */ | ERR_RFLAG_COMMON) -# define ERR_R_CMS_LIB (ERR_LIB_CMS/* 46 */ | ERR_RFLAG_COMMON) -# define ERR_R_TS_LIB (ERR_LIB_TS/* 47 */ | ERR_RFLAG_COMMON) -# define ERR_R_CT_LIB (ERR_LIB_CT/* 50 */ | ERR_RFLAG_COMMON) -# define ERR_R_PROV_LIB (ERR_LIB_PROV/* 57 */ | ERR_RFLAG_COMMON) -# define ERR_R_ESS_LIB (ERR_LIB_ESS/* 54 */ | ERR_RFLAG_COMMON) -# define ERR_R_CMP_LIB (ERR_LIB_CMP/* 58 */ | ERR_RFLAG_COMMON) -# define ERR_R_OSSL_ENCODER_LIB (ERR_LIB_OSSL_ENCODER/* 59 */ | ERR_RFLAG_COMMON) # define ERR_R_OSSL_DECODER_LIB (ERR_LIB_OSSL_DECODER/* 60 */ | ERR_RFLAG_COMMON) /* Other common error codes, range 256..2^ERR_RFLAGS_OFFSET-1 */ @@ -372,7 +365,7 @@ typedef struct ERR_string_data_st { } ERR_STRING_DATA; DEFINE_LHASH_OF_INTERNAL(ERR_STRING_DATA); -#define lh_ERR_STRING_DATA_new(hfn, cmp) ((LHASH_OF(ERR_STRING_DATA) *)OPENSSL_LH_set_thunks(OPENSSL_LH_new(ossl_check_ERR_STRING_DATA_lh_hashfunc_type(hfn), ossl_check_ERR_STRING_DATA_lh_compfunc_type(cmp)), lh_ERR_STRING_DATA_hash_thunk, lh_ERR_STRING_DATA_comp_thunk, lh_ERR_STRING_DATA_doall_thunk, lh_ERR_STRING_DATA_doall_arg_thunk)) +#define lh_ERR_STRING_DATA_new(hfn, cmp) ((LHASH_OF(ERR_STRING_DATA) *)OPENSSL_LH_new(ossl_check_ERR_STRING_DATA_lh_hashfunc_type(hfn), ossl_check_ERR_STRING_DATA_lh_compfunc_type(cmp))) #define lh_ERR_STRING_DATA_free(lh) OPENSSL_LH_free(ossl_check_ERR_STRING_DATA_lh_type(lh)) #define lh_ERR_STRING_DATA_flush(lh) OPENSSL_LH_flush(ossl_check_ERR_STRING_DATA_lh_type(lh)) #define lh_ERR_STRING_DATA_insert(lh, ptr) ((ERR_STRING_DATA *)OPENSSL_LH_insert(ossl_check_ERR_STRING_DATA_lh_type(lh), ossl_check_ERR_STRING_DATA_lh_plain_type(ptr))) @@ -496,14 +489,6 @@ int ERR_get_next_error_library(void); int ERR_set_mark(void); int ERR_pop_to_mark(void); int ERR_clear_last_mark(void); -int ERR_count_to_mark(void); -int ERR_pop(void); - -ERR_STATE *OSSL_ERR_STATE_new(void); -void OSSL_ERR_STATE_save(ERR_STATE *es); -void OSSL_ERR_STATE_save_to_mark(ERR_STATE *es); -void OSSL_ERR_STATE_restore(const ERR_STATE *es); -void OSSL_ERR_STATE_free(ERR_STATE *es); #ifdef __cplusplus } diff --git a/openssl/include/openssl/err.h.in b/openssl/include/openssl/err.h.in index 9143704c0..db888da62 100644 --- a/openssl/include/openssl/err.h.in +++ b/openssl/include/openssl/err.h.in @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -122,6 +122,10 @@ struct err_state_st { # define ERR_LIB_OSSL_ENCODER 59 # define ERR_LIB_OSSL_DECODER 60 # define ERR_LIB_HTTP 61 +# define ERR_LIB_PAILLIER 62 +# define ERR_LIB_ZKP 63 +# define ERR_LIB_ZKP_BP 64 +# define ERR_LIB_ZKP_NIZK 65 # define ERR_LIB_USER 128 @@ -169,6 +173,7 @@ struct err_state_st { # define UIerr(f, r) ERR_raise_data(ERR_LIB_UI, (r), NULL) # define X509V3err(f, r) ERR_raise_data(ERR_LIB_X509V3, (r), NULL) # define X509err(f, r) ERR_raise_data(ERR_LIB_X509, (r), NULL) +# define PAILLIERerr(f, r) ERR_raise_data(ERR_LIB_PAILLIER, (r), NULL) # endif /*- @@ -325,27 +330,15 @@ static ossl_unused ossl_inline int ERR_COMMON_ERROR(unsigned long errcode) # define ERR_R_DSA_LIB (ERR_LIB_DSA/* 10 */ | ERR_RFLAG_COMMON) # define ERR_R_X509_LIB (ERR_LIB_X509/* 11 */ | ERR_RFLAG_COMMON) # define ERR_R_ASN1_LIB (ERR_LIB_ASN1/* 13 */ | ERR_RFLAG_COMMON) -# define ERR_R_CONF_LIB (ERR_LIB_CONF/* 14 */ | ERR_RFLAG_COMMON) # define ERR_R_CRYPTO_LIB (ERR_LIB_CRYPTO/* 15 */ | ERR_RFLAG_COMMON) # define ERR_R_EC_LIB (ERR_LIB_EC/* 16 */ | ERR_RFLAG_COMMON) -# define ERR_R_SSL_LIB (ERR_LIB_SSL/* 20 */ | ERR_RFLAG_COMMON) # define ERR_R_BIO_LIB (ERR_LIB_BIO/* 32 */ | ERR_RFLAG_COMMON) # define ERR_R_PKCS7_LIB (ERR_LIB_PKCS7/* 33 */ | ERR_RFLAG_COMMON) # define ERR_R_X509V3_LIB (ERR_LIB_X509V3/* 34 */ | ERR_RFLAG_COMMON) -# define ERR_R_PKCS12_LIB (ERR_LIB_PKCS12/* 35 */ | ERR_RFLAG_COMMON) -# define ERR_R_RAND_LIB (ERR_LIB_RAND/* 36 */ | ERR_RFLAG_COMMON) -# define ERR_R_DSO_LIB (ERR_LIB_DSO/* 37 */ | ERR_RFLAG_COMMON) # define ERR_R_ENGINE_LIB (ERR_LIB_ENGINE/* 38 */ | ERR_RFLAG_COMMON) # define ERR_R_UI_LIB (ERR_LIB_UI/* 40 */ | ERR_RFLAG_COMMON) # define ERR_R_ECDSA_LIB (ERR_LIB_ECDSA/* 42 */ | ERR_RFLAG_COMMON) # define ERR_R_OSSL_STORE_LIB (ERR_LIB_OSSL_STORE/* 44 */ | ERR_RFLAG_COMMON) -# define ERR_R_CMS_LIB (ERR_LIB_CMS/* 46 */ | ERR_RFLAG_COMMON) -# define ERR_R_TS_LIB (ERR_LIB_TS/* 47 */ | ERR_RFLAG_COMMON) -# define ERR_R_CT_LIB (ERR_LIB_CT/* 50 */ | ERR_RFLAG_COMMON) -# define ERR_R_PROV_LIB (ERR_LIB_PROV/* 57 */ | ERR_RFLAG_COMMON) -# define ERR_R_ESS_LIB (ERR_LIB_ESS/* 54 */ | ERR_RFLAG_COMMON) -# define ERR_R_CMP_LIB (ERR_LIB_CMP/* 58 */ | ERR_RFLAG_COMMON) -# define ERR_R_OSSL_ENCODER_LIB (ERR_LIB_OSSL_ENCODER/* 59 */ | ERR_RFLAG_COMMON) # define ERR_R_OSSL_DECODER_LIB (ERR_LIB_OSSL_DECODER/* 60 */ | ERR_RFLAG_COMMON) /* Other common error codes, range 256..2^ERR_RFLAGS_OFFSET-1 */ @@ -485,14 +478,6 @@ int ERR_get_next_error_library(void); int ERR_set_mark(void); int ERR_pop_to_mark(void); int ERR_clear_last_mark(void); -int ERR_count_to_mark(void); -int ERR_pop(void); - -ERR_STATE *OSSL_ERR_STATE_new(void); -void OSSL_ERR_STATE_save(ERR_STATE *es); -void OSSL_ERR_STATE_save_to_mark(ERR_STATE *es); -void OSSL_ERR_STATE_restore(const ERR_STATE *es); -void OSSL_ERR_STATE_free(ERR_STATE *es); #ifdef __cplusplus } diff --git a/openssl/include/openssl/evp.h b/openssl/include/openssl/evp.h index f70b9d744..efa47dfc6 100644 --- a/openssl/include/openssl/evp.h +++ b/openssl/include/openssl/evp.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -35,7 +35,6 @@ # define EVP_MAX_KEY_LENGTH 64 # define EVP_MAX_IV_LENGTH 16 # define EVP_MAX_BLOCK_LENGTH 32 -# define EVP_MAX_AEAD_TAG_LENGTH 16 # define PKCS5_SALT_LEN 8 /* Default PKCS#5 iteration count */ @@ -80,14 +79,13 @@ # define EVP_PKEY_ED25519 NID_ED25519 # define EVP_PKEY_X448 NID_X448 # define EVP_PKEY_ED448 NID_ED448 +# define EVP_PKEY_EIA3 NID_zuc_128_eia3 /* Special indicator that the object is uniquely provider side */ # define EVP_PKEY_KEYMGMT -1 /* Easy to use macros for EVP_PKEY related selections */ # define EVP_PKEY_KEY_PARAMETERS \ ( OSSL_KEYMGMT_SELECT_ALL_PARAMETERS ) -# define EVP_PKEY_PRIVATE_KEY \ - ( EVP_PKEY_KEY_PARAMETERS | OSSL_KEYMGMT_SELECT_PRIVATE_KEY ) # define EVP_PKEY_PUBLIC_KEY \ ( EVP_PKEY_KEY_PARAMETERS | OSSL_KEYMGMT_SELECT_PUBLIC_KEY ) # define EVP_PKEY_KEYPAIR \ @@ -229,8 +227,7 @@ int (*EVP_MD_meth_get_ctrl(const EVP_MD *md))(EVP_MD_CTX *ctx, int cmd, * if the following flag is set. */ # define EVP_MD_CTX_FLAG_FINALISE 0x0200 -/* NOTE: 0x0400 and 0x0800 are reserved for internal usage */ - +/* NOTE: 0x0400 is reserved for internal usage */ # ifndef OPENSSL_NO_DEPRECATED_3_0 OSSL_DEPRECATEDIN_3_0 EVP_CIPHER *EVP_CIPHER_meth_new(int cipher_type, int block_size, int key_len); @@ -310,7 +307,6 @@ OSSL_DEPRECATEDIN_3_0 int # define EVP_CIPH_WRAP_MODE 0x10002 # define EVP_CIPH_OCB_MODE 0x10003 # define EVP_CIPH_SIV_MODE 0x10004 -# define EVP_CIPH_GCM_SIV_MODE 0x10005 # define EVP_CIPH_MODE 0xF0007 /* Set if variable length cipher */ # define EVP_CIPH_VARIABLE_LENGTH 0x8 @@ -368,8 +364,6 @@ OSSL_DEPRECATEDIN_3_0 int # define EVP_CTRL_INIT 0x0 # define EVP_CTRL_SET_KEY_LENGTH 0x1 -# define EVP_CTRL_GET_RC2_KEY_BITS 0x2 -# define EVP_CTRL_SET_RC2_KEY_BITS 0x3 # define EVP_CTRL_GET_RC5_ROUNDS 0x4 # define EVP_CTRL_SET_RC5_ROUNDS 0x5 # define EVP_CTRL_RAND_KEY 0x6 @@ -527,6 +521,11 @@ typedef int (EVP_PBE_KEYGEN_EX) (EVP_CIPHER_CTX *ctx, const char *pass, EVP_PKEY_POLY1305,(polykey)) # endif +# ifndef OPENSSL_NO_ZUC +# define EVP_PKEY_assign_EIA3(pkey,eia3key) EVP_PKEY_assign((pkey),EVP_PKEY_EIA3,\ + (char *)(eia3key)) +# endif + /* Add some extra combinations */ # define EVP_get_digestbynid(a) EVP_get_digestbyname(OBJ_nid2sn(a)) # define EVP_get_digestbyobj(a) EVP_get_digestbynid(OBJ_obj2nid(a)) @@ -636,10 +635,11 @@ int EVP_CIPHER_CTX_get_original_iv(EVP_CIPHER_CTX *ctx, void *buf, size_t len); OSSL_DEPRECATEDIN_3_0 unsigned char *EVP_CIPHER_CTX_buf_noconst(EVP_CIPHER_CTX *ctx); # endif +int EVP_CIPHER_CTX_get_buf_len(const EVP_CIPHER_CTX *ctx); +int EVP_CIPHER_CTX_get_final_used(const EVP_CIPHER_CTX *ctx); int EVP_CIPHER_CTX_get_num(const EVP_CIPHER_CTX *ctx); # define EVP_CIPHER_CTX_num EVP_CIPHER_CTX_get_num int EVP_CIPHER_CTX_set_num(EVP_CIPHER_CTX *ctx, int num); -EVP_CIPHER_CTX *EVP_CIPHER_CTX_dup(const EVP_CIPHER_CTX *in); int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in); void *EVP_CIPHER_CTX_get_app_data(const EVP_CIPHER_CTX *ctx); void EVP_CIPHER_CTX_set_app_data(EVP_CIPHER_CTX *ctx, void *data); @@ -677,7 +677,7 @@ void BIO_set_md(BIO *, const EVP_MD *md); # define BIO_get_cipher_status(b) BIO_ctrl(b,BIO_C_GET_CIPHER_STATUS,0,NULL) # define BIO_get_cipher_ctx(b,c_pp) BIO_ctrl(b,BIO_C_GET_CIPHER_CTX,0,(c_pp)) -__owur int EVP_Cipher(EVP_CIPHER_CTX *c, +/*__owur*/ int EVP_Cipher(EVP_CIPHER_CTX *c, unsigned char *out, const unsigned char *in, unsigned int inl); @@ -705,7 +705,6 @@ void EVP_MD_CTX_free(EVP_MD_CTX *ctx); # define EVP_MD_CTX_create() EVP_MD_CTX_new() # define EVP_MD_CTX_init(ctx) EVP_MD_CTX_reset((ctx)) # define EVP_MD_CTX_destroy(ctx) EVP_MD_CTX_free((ctx)) -__owur EVP_MD_CTX *EVP_MD_CTX_dup(const EVP_MD_CTX *in); __owur int EVP_MD_CTX_copy_ex(EVP_MD_CTX *out, const EVP_MD_CTX *in); void EVP_MD_CTX_set_flags(EVP_MD_CTX *ctx, int flags); void EVP_MD_CTX_clear_flags(EVP_MD_CTX *ctx, int flags); @@ -729,10 +728,8 @@ __owur int EVP_MD_CTX_copy(EVP_MD_CTX *out, const EVP_MD_CTX *in); __owur int EVP_DigestInit(EVP_MD_CTX *ctx, const EVP_MD *type); __owur int EVP_DigestFinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s); -__owur int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *out, - size_t outlen); -__owur int EVP_DigestSqueeze(EVP_MD_CTX *ctx, unsigned char *out, - size_t outlen); +__owur int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, + size_t len); __owur EVP_MD *EVP_MD_fetch(OSSL_LIB_CTX *ctx, const char *algorithm, const char *properties); @@ -757,7 +754,7 @@ int EVP_CIPHER_CTX_test_flags(const EVP_CIPHER_CTX *ctx, int flags); __owur int EVP_EncryptInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, const unsigned char *key, const unsigned char *iv); -__owur int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, +/*__owur*/ int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv); @@ -765,16 +762,16 @@ __owur int EVP_EncryptInit_ex2(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, const unsigned char *key, const unsigned char *iv, const OSSL_PARAM params[]); -__owur int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, +/*__owur*/ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, const unsigned char *in, int inl); -__owur int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, +/*__owur*/ int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl); -__owur int EVP_EncryptFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, +/*__owur*/ int EVP_EncryptFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl); __owur int EVP_DecryptInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, const unsigned char *key, const unsigned char *iv); -__owur int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, +/*__owur*/ int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv); @@ -782,17 +779,17 @@ __owur int EVP_DecryptInit_ex2(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, const unsigned char *key, const unsigned char *iv, const OSSL_PARAM params[]); -__owur int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, +/*__owur*/ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, const unsigned char *in, int inl); __owur int EVP_DecryptFinal(EVP_CIPHER_CTX *ctx, unsigned char *outm, int *outl); -__owur int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *outm, +/*__owur*/ int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *outm, int *outl); __owur int EVP_CipherInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, const unsigned char *key, const unsigned char *iv, int enc); -__owur int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, +/*__owur*/ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv, int enc); @@ -826,18 +823,18 @@ __owur int EVP_DigestVerify(EVP_MD_CTX *ctx, const unsigned char *sigret, size_t siglen, const unsigned char *tbs, size_t tbslen); -__owur int EVP_DigestSignInit_ex(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, +int EVP_DigestSignInit_ex(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const char *mdname, OSSL_LIB_CTX *libctx, const char *props, EVP_PKEY *pkey, const OSSL_PARAM params[]); -__owur int EVP_DigestSignInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, +/*__owur*/ int EVP_DigestSignInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const EVP_MD *type, ENGINE *e, EVP_PKEY *pkey); -__owur int EVP_DigestSignUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize); +int EVP_DigestSignUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize); __owur int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret, size_t *siglen); -__owur int EVP_DigestVerifyInit_ex(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, +int EVP_DigestVerifyInit_ex(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const char *mdname, OSSL_LIB_CTX *libctx, const char *props, EVP_PKEY *pkey, const OSSL_PARAM params[]); @@ -903,20 +900,10 @@ __owur int BIO_set_cipher(BIO *b, const EVP_CIPHER *c, const unsigned char *k, const unsigned char *i, int enc); const EVP_MD *EVP_md_null(void); -# ifndef OPENSSL_NO_MD2 -const EVP_MD *EVP_md2(void); -# endif -# ifndef OPENSSL_NO_MD4 -const EVP_MD *EVP_md4(void); -# endif # ifndef OPENSSL_NO_MD5 const EVP_MD *EVP_md5(void); const EVP_MD *EVP_md5_sha1(void); # endif -# ifndef OPENSSL_NO_BLAKE2 -const EVP_MD *EVP_blake2b512(void); -const EVP_MD *EVP_blake2s256(void); -# endif const EVP_MD *EVP_sha1(void); const EVP_MD *EVP_sha224(void); const EVP_MD *EVP_sha256(void); @@ -931,15 +918,6 @@ const EVP_MD *EVP_sha3_512(void); const EVP_MD *EVP_shake128(void); const EVP_MD *EVP_shake256(void); -# ifndef OPENSSL_NO_MDC2 -const EVP_MD *EVP_mdc2(void); -# endif -# ifndef OPENSSL_NO_RMD160 -const EVP_MD *EVP_ripemd160(void); -# endif -# ifndef OPENSSL_NO_WHIRLPOOL -const EVP_MD *EVP_whirlpool(void); -# endif # ifndef OPENSSL_NO_SM3 const EVP_MD *EVP_sm3(void); # endif @@ -981,36 +959,6 @@ const EVP_CIPHER *EVP_rc4_40(void); const EVP_CIPHER *EVP_rc4_hmac_md5(void); # endif # endif -# ifndef OPENSSL_NO_IDEA -const EVP_CIPHER *EVP_idea_ecb(void); -const EVP_CIPHER *EVP_idea_cfb64(void); -# define EVP_idea_cfb EVP_idea_cfb64 -const EVP_CIPHER *EVP_idea_ofb(void); -const EVP_CIPHER *EVP_idea_cbc(void); -# endif -# ifndef OPENSSL_NO_RC2 -const EVP_CIPHER *EVP_rc2_ecb(void); -const EVP_CIPHER *EVP_rc2_cbc(void); -const EVP_CIPHER *EVP_rc2_40_cbc(void); -const EVP_CIPHER *EVP_rc2_64_cbc(void); -const EVP_CIPHER *EVP_rc2_cfb64(void); -# define EVP_rc2_cfb EVP_rc2_cfb64 -const EVP_CIPHER *EVP_rc2_ofb(void); -# endif -# ifndef OPENSSL_NO_BF -const EVP_CIPHER *EVP_bf_ecb(void); -const EVP_CIPHER *EVP_bf_cbc(void); -const EVP_CIPHER *EVP_bf_cfb64(void); -# define EVP_bf_cfb EVP_bf_cfb64 -const EVP_CIPHER *EVP_bf_ofb(void); -# endif -# ifndef OPENSSL_NO_CAST -const EVP_CIPHER *EVP_cast5_ecb(void); -const EVP_CIPHER *EVP_cast5_cbc(void); -const EVP_CIPHER *EVP_cast5_cfb64(void); -# define EVP_cast5_cfb EVP_cast5_cfb64 -const EVP_CIPHER *EVP_cast5_ofb(void); -# endif # ifndef OPENSSL_NO_RC5 const EVP_CIPHER *EVP_rc5_32_12_16_cbc(void); const EVP_CIPHER *EVP_rc5_32_12_16_ecb(void); @@ -1069,64 +1017,6 @@ const EVP_CIPHER *EVP_aes_128_cbc_hmac_sha1(void); const EVP_CIPHER *EVP_aes_256_cbc_hmac_sha1(void); const EVP_CIPHER *EVP_aes_128_cbc_hmac_sha256(void); const EVP_CIPHER *EVP_aes_256_cbc_hmac_sha256(void); -# ifndef OPENSSL_NO_ARIA -const EVP_CIPHER *EVP_aria_128_ecb(void); -const EVP_CIPHER *EVP_aria_128_cbc(void); -const EVP_CIPHER *EVP_aria_128_cfb1(void); -const EVP_CIPHER *EVP_aria_128_cfb8(void); -const EVP_CIPHER *EVP_aria_128_cfb128(void); -# define EVP_aria_128_cfb EVP_aria_128_cfb128 -const EVP_CIPHER *EVP_aria_128_ctr(void); -const EVP_CIPHER *EVP_aria_128_ofb(void); -const EVP_CIPHER *EVP_aria_128_gcm(void); -const EVP_CIPHER *EVP_aria_128_ccm(void); -const EVP_CIPHER *EVP_aria_192_ecb(void); -const EVP_CIPHER *EVP_aria_192_cbc(void); -const EVP_CIPHER *EVP_aria_192_cfb1(void); -const EVP_CIPHER *EVP_aria_192_cfb8(void); -const EVP_CIPHER *EVP_aria_192_cfb128(void); -# define EVP_aria_192_cfb EVP_aria_192_cfb128 -const EVP_CIPHER *EVP_aria_192_ctr(void); -const EVP_CIPHER *EVP_aria_192_ofb(void); -const EVP_CIPHER *EVP_aria_192_gcm(void); -const EVP_CIPHER *EVP_aria_192_ccm(void); -const EVP_CIPHER *EVP_aria_256_ecb(void); -const EVP_CIPHER *EVP_aria_256_cbc(void); -const EVP_CIPHER *EVP_aria_256_cfb1(void); -const EVP_CIPHER *EVP_aria_256_cfb8(void); -const EVP_CIPHER *EVP_aria_256_cfb128(void); -# define EVP_aria_256_cfb EVP_aria_256_cfb128 -const EVP_CIPHER *EVP_aria_256_ctr(void); -const EVP_CIPHER *EVP_aria_256_ofb(void); -const EVP_CIPHER *EVP_aria_256_gcm(void); -const EVP_CIPHER *EVP_aria_256_ccm(void); -# endif -# ifndef OPENSSL_NO_CAMELLIA -const EVP_CIPHER *EVP_camellia_128_ecb(void); -const EVP_CIPHER *EVP_camellia_128_cbc(void); -const EVP_CIPHER *EVP_camellia_128_cfb1(void); -const EVP_CIPHER *EVP_camellia_128_cfb8(void); -const EVP_CIPHER *EVP_camellia_128_cfb128(void); -# define EVP_camellia_128_cfb EVP_camellia_128_cfb128 -const EVP_CIPHER *EVP_camellia_128_ofb(void); -const EVP_CIPHER *EVP_camellia_128_ctr(void); -const EVP_CIPHER *EVP_camellia_192_ecb(void); -const EVP_CIPHER *EVP_camellia_192_cbc(void); -const EVP_CIPHER *EVP_camellia_192_cfb1(void); -const EVP_CIPHER *EVP_camellia_192_cfb8(void); -const EVP_CIPHER *EVP_camellia_192_cfb128(void); -# define EVP_camellia_192_cfb EVP_camellia_192_cfb128 -const EVP_CIPHER *EVP_camellia_192_ofb(void); -const EVP_CIPHER *EVP_camellia_192_ctr(void); -const EVP_CIPHER *EVP_camellia_256_ecb(void); -const EVP_CIPHER *EVP_camellia_256_cbc(void); -const EVP_CIPHER *EVP_camellia_256_cfb1(void); -const EVP_CIPHER *EVP_camellia_256_cfb8(void); -const EVP_CIPHER *EVP_camellia_256_cfb128(void); -# define EVP_camellia_256_cfb EVP_camellia_256_cfb128 -const EVP_CIPHER *EVP_camellia_256_ofb(void); -const EVP_CIPHER *EVP_camellia_256_ctr(void); -# endif # ifndef OPENSSL_NO_CHACHA const EVP_CIPHER *EVP_chacha20(void); # ifndef OPENSSL_NO_POLY1305 @@ -1134,14 +1024,6 @@ const EVP_CIPHER *EVP_chacha20_poly1305(void); # endif # endif -# ifndef OPENSSL_NO_SEED -const EVP_CIPHER *EVP_seed_ecb(void); -const EVP_CIPHER *EVP_seed_cbc(void); -const EVP_CIPHER *EVP_seed_cfb128(void); -# define EVP_seed_cfb EVP_seed_cfb128 -const EVP_CIPHER *EVP_seed_ofb(void); -# endif - # ifndef OPENSSL_NO_SM4 const EVP_CIPHER *EVP_sm4_ecb(void); const EVP_CIPHER *EVP_sm4_cbc(void); @@ -1149,6 +1031,12 @@ const EVP_CIPHER *EVP_sm4_cfb128(void); # define EVP_sm4_cfb EVP_sm4_cfb128 const EVP_CIPHER *EVP_sm4_ofb(void); const EVP_CIPHER *EVP_sm4_ctr(void); +const EVP_CIPHER *EVP_sm4_gcm(void); +const EVP_CIPHER *EVP_sm4_ccm(void); +# endif + +# ifndef OPENSSL_NO_ZUC +const EVP_CIPHER *EVP_eea3(void); # endif # ifndef OPENSSL_NO_DEPRECATED_1_1_0 @@ -1257,7 +1145,6 @@ const OSSL_PROVIDER *EVP_RAND_get0_provider(const EVP_RAND *rand); int EVP_RAND_get_params(EVP_RAND *rand, OSSL_PARAM params[]); EVP_RAND_CTX *EVP_RAND_CTX_new(EVP_RAND *rand, EVP_RAND_CTX *parent); -int EVP_RAND_CTX_up_ref(EVP_RAND_CTX *ctx); void EVP_RAND_CTX_free(EVP_RAND_CTX *ctx); EVP_RAND *EVP_RAND_CTX_get0_rand(EVP_RAND_CTX *ctx); int EVP_RAND_CTX_get_params(EVP_RAND_CTX *ctx, OSSL_PARAM params[]); @@ -1327,6 +1214,7 @@ int EVP_PKEY_can_sign(const EVP_PKEY *pkey); int EVP_PKEY_set_type(EVP_PKEY *pkey, int type); int EVP_PKEY_set_type_str(EVP_PKEY *pkey, const char *str, int len); int EVP_PKEY_set_type_by_keymgmt(EVP_PKEY *pkey, EVP_KEYMGMT *keymgmt); +int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type); # ifndef OPENSSL_NO_DEPRECATED_3_0 # ifndef OPENSSL_NO_ENGINE OSSL_DEPRECATEDIN_3_0 @@ -1385,6 +1273,10 @@ struct ec_key_st *EVP_PKEY_get1_EC_KEY(EVP_PKEY *pkey); # endif # endif /* OPENSSL_NO_DEPRECATED_3_0 */ +# ifndef OPENSSL_NO_ZUC +const unsigned char *EVP_PKEY_get0_eia3(const EVP_PKEY *pkey, size_t *len); +# endif + EVP_PKEY *EVP_PKEY_new(void); int EVP_PKEY_up_ref(EVP_PKEY *pkey); EVP_PKEY *EVP_PKEY_dup(EVP_PKEY *pkey); @@ -1725,8 +1617,6 @@ int EVP_PKEY_CTX_set_mac_key(EVP_PKEY_CTX *ctx, const unsigned char *key, # define EVP_PKEY_CTRL_PEER_KEY 2 # define EVP_PKEY_CTRL_SET_MAC_KEY 6 # define EVP_PKEY_CTRL_DIGESTINIT 7 -/* Used by GOST key encryption in TLS */ -# define EVP_PKEY_CTRL_SET_IV 8 # ifndef OPENSSL_NO_DEPRECATED_3_0 # define EVP_PKEY_CTRL_PKCS7_ENCRYPT 3 # define EVP_PKEY_CTRL_PKCS7_DECRYPT 4 @@ -1749,6 +1639,11 @@ int EVP_PKEY_CTX_set_mac_key(EVP_PKEY_CTX *ctx, const unsigned char *key, * Method handles all operations: don't assume any digest related defaults. */ # define EVP_PKEY_FLAG_SIGCTX_CUSTOM 4 + +/* Used by Chromium/QUIC */ +# define X25519_PRIVATE_KEY_LEN 32 +# define X25519_PUBLIC_VALUE_LEN 32 + # ifndef OPENSSL_NO_DEPRECATED_3_0 OSSL_DEPRECATEDIN_3_0 const EVP_PKEY_METHOD *EVP_PKEY_meth_find(int type); OSSL_DEPRECATEDIN_3_0 EVP_PKEY_METHOD *EVP_PKEY_meth_new(int id, int flags); @@ -1932,17 +1827,14 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer); int EVP_PKEY_derive(EVP_PKEY_CTX *ctx, unsigned char *key, size_t *keylen); int EVP_PKEY_encapsulate_init(EVP_PKEY_CTX *ctx, const OSSL_PARAM params[]); -int EVP_PKEY_auth_encapsulate_init(EVP_PKEY_CTX *ctx, EVP_PKEY *authpriv, - const OSSL_PARAM params[]); int EVP_PKEY_encapsulate(EVP_PKEY_CTX *ctx, unsigned char *wrappedkey, size_t *wrappedkeylen, unsigned char *genkey, size_t *genkeylen); int EVP_PKEY_decapsulate_init(EVP_PKEY_CTX *ctx, const OSSL_PARAM params[]); -int EVP_PKEY_auth_decapsulate_init(EVP_PKEY_CTX *ctx, EVP_PKEY *authpub, - const OSSL_PARAM params[]); int EVP_PKEY_decapsulate(EVP_PKEY_CTX *ctx, unsigned char *unwrapped, size_t *unwrappedlen, const unsigned char *wrapped, size_t wrappedlen); + typedef int EVP_PKEY_gen_cb(EVP_PKEY_CTX *ctx); int EVP_PKEY_fromdata_init(EVP_PKEY_CTX *ctx); @@ -2177,6 +2069,11 @@ OSSL_LIB_CTX *EVP_PKEY_CTX_get0_libctx(EVP_PKEY_CTX *ctx); const char *EVP_PKEY_CTX_get0_propq(const EVP_PKEY_CTX *ctx); const OSSL_PROVIDER *EVP_PKEY_CTX_get0_provider(const EVP_PKEY_CTX *ctx); +/* for compattibility usage */ +# ifndef OPENSSL_NO_SM2 +# define EVP_PKEY_is_sm2(pkey) EVP_PKEY_is_a(pkey, "SM2") +# endif + # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/evperr.h b/openssl/include/openssl/evperr.h index 11f3faa45..95b26ccab 100644 --- a/openssl/include/openssl/evperr.h +++ b/openssl/include/openssl/evperr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,13 +22,11 @@ * EVP reason codes. */ # define EVP_R_AES_KEY_SETUP_FAILED 143 -# define EVP_R_ARIA_KEY_SETUP_FAILED 176 # define EVP_R_BAD_ALGORITHM_NAME 200 # define EVP_R_BAD_DECRYPT 100 # define EVP_R_BAD_KEY_LENGTH 195 # define EVP_R_BUFFER_TOO_SMALL 155 # define EVP_R_CACHE_CONSTANTS_FAILED 225 -# define EVP_R_CAMELLIA_KEY_SETUP_FAILED 157 # define EVP_R_CANNOT_GET_PARAMETERS 197 # define EVP_R_CANNOT_SET_PARAMETERS 198 # define EVP_R_CIPHER_NOT_GCM_MODE 184 @@ -50,6 +48,7 @@ # define EVP_R_EXPECTING_A_DSA_KEY 129 # define EVP_R_EXPECTING_A_ECX_KEY 219 # define EVP_R_EXPECTING_A_EC_KEY 142 +# define EVP_R_EXPECTING_A_EIA3_KEY 166 # define EVP_R_EXPECTING_A_POLY1305_KEY 164 # define EVP_R_EXPECTING_A_SIPHASH_KEY 175 # define EVP_R_FINAL_ERROR 188 @@ -110,14 +109,11 @@ # define EVP_R_UNABLE_TO_GET_RANDOM_STRENGTH 216 # define EVP_R_UNABLE_TO_LOCK_CONTEXT 211 # define EVP_R_UNABLE_TO_SET_CALLBACKS 217 -# define EVP_R_UNKNOWN_BITS 166 # define EVP_R_UNKNOWN_CIPHER 160 # define EVP_R_UNKNOWN_DIGEST 161 # define EVP_R_UNKNOWN_KEY_TYPE 207 -# define EVP_R_UNKNOWN_MAX_SIZE 167 # define EVP_R_UNKNOWN_OPTION 169 # define EVP_R_UNKNOWN_PBE_ALGORITHM 121 -# define EVP_R_UNKNOWN_SECURITY_BITS 168 # define EVP_R_UNSUPPORTED_ALGORITHM 156 # define EVP_R_UNSUPPORTED_CIPHER 107 # define EVP_R_UNSUPPORTED_KEYLENGTH 123 diff --git a/openssl/include/openssl/fips_names.h b/openssl/include/openssl/fips_names.h index 5c77f6d69..0fdf5440c 100644 --- a/openssl/include/openssl/fips_names.h +++ b/openssl/include/openssl/fips_names.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -49,27 +49,10 @@ extern "C" { /* * A boolean that determines if the runtime FIPS security checks are performed. - * This is enabled by default. * Type: OSSL_PARAM_UTF8_STRING */ # define OSSL_PROV_FIPS_PARAM_SECURITY_CHECKS "security-checks" -/* - * A boolean that determines if the runtime FIPS check for TLS1_PRF EMS is performed. - * This is disabled by default. - * Type: OSSL_PARAM_UTF8_STRING - */ -# define OSSL_PROV_FIPS_PARAM_TLS1_PRF_EMS_CHECK "tls1-prf-ems-check" - -/* - * A boolean that determines if truncated digests can be used with Hash and HMAC - * DRBGs. FIPS 140-3 IG D.R disallows such use for efficiency rather than - * security reasons. - * This is disabled by default. - * Type: OSSL_PARAM_UTF8_STRING - */ -# define OSSL_PROV_FIPS_PARAM_DRBG_TRUNC_DIGEST "drbg-no-trunc-md" - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/hpke.h b/openssl/include/openssl/hpke.h deleted file mode 100644 index 482acd22c..000000000 --- a/openssl/include/openssl/hpke.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* APIs and data structures for HPKE (RFC9180) */ -#ifndef OSSL_HPKE_H -# define OSSL_HPKE_H -# pragma once - -# include - -/* HPKE modes */ -# define OSSL_HPKE_MODE_BASE 0 /* Base mode */ -# define OSSL_HPKE_MODE_PSK 1 /* Pre-shared key mode */ -# define OSSL_HPKE_MODE_AUTH 2 /* Authenticated mode */ -# define OSSL_HPKE_MODE_PSKAUTH 3 /* PSK+authenticated mode */ - -/* - * Max for ikm, psk, pskid, info and exporter contexts. - * RFC9180, section 7.2.1 RECOMMENDS 64 octets but we have test vectors from - * Appendix A.6.1 with a 66 octet IKM so we'll allow that. - */ -# define OSSL_HPKE_MAX_PARMLEN 66 -# define OSSL_HPKE_MIN_PSKLEN 32 -# define OSSL_HPKE_MAX_INFOLEN 1024 - -/* - * The (16bit) HPKE algorithm ID IANA codepoints - * If/when new IANA codepoints are added there are tables in - * crypto/hpke/hpke_util.c that must also be updated. - */ -# define OSSL_HPKE_KEM_ID_RESERVED 0x0000 /* not used */ -# define OSSL_HPKE_KEM_ID_P256 0x0010 /* NIST P-256 */ -# define OSSL_HPKE_KEM_ID_P384 0x0011 /* NIST P-384 */ -# define OSSL_HPKE_KEM_ID_P521 0x0012 /* NIST P-521 */ -# define OSSL_HPKE_KEM_ID_X25519 0x0020 /* Curve25519 */ -# define OSSL_HPKE_KEM_ID_X448 0x0021 /* Curve448 */ - -# define OSSL_HPKE_KDF_ID_RESERVED 0x0000 /* not used */ -# define OSSL_HPKE_KDF_ID_HKDF_SHA256 0x0001 /* HKDF-SHA256 */ -# define OSSL_HPKE_KDF_ID_HKDF_SHA384 0x0002 /* HKDF-SHA384 */ -# define OSSL_HPKE_KDF_ID_HKDF_SHA512 0x0003 /* HKDF-SHA512 */ - -# define OSSL_HPKE_AEAD_ID_RESERVED 0x0000 /* not used */ -# define OSSL_HPKE_AEAD_ID_AES_GCM_128 0x0001 /* AES-GCM-128 */ -# define OSSL_HPKE_AEAD_ID_AES_GCM_256 0x0002 /* AES-GCM-256 */ -# define OSSL_HPKE_AEAD_ID_CHACHA_POLY1305 0x0003 /* Chacha20-Poly1305 */ -# define OSSL_HPKE_AEAD_ID_EXPORTONLY 0xFFFF /* export-only fake ID */ - -/* strings for suite components */ -# define OSSL_HPKE_KEMSTR_P256 "P-256" /* KEM id 0x10 */ -# define OSSL_HPKE_KEMSTR_P384 "P-384" /* KEM id 0x11 */ -# define OSSL_HPKE_KEMSTR_P521 "P-521" /* KEM id 0x12 */ -# define OSSL_HPKE_KEMSTR_X25519 "X25519" /* KEM id 0x20 */ -# define OSSL_HPKE_KEMSTR_X448 "X448" /* KEM id 0x21 */ -# define OSSL_HPKE_KDFSTR_256 "hkdf-sha256" /* KDF id 1 */ -# define OSSL_HPKE_KDFSTR_384 "hkdf-sha384" /* KDF id 2 */ -# define OSSL_HPKE_KDFSTR_512 "hkdf-sha512" /* KDF id 3 */ -# define OSSL_HPKE_AEADSTR_AES128GCM "aes-128-gcm" /* AEAD id 1 */ -# define OSSL_HPKE_AEADSTR_AES256GCM "aes-256-gcm" /* AEAD id 2 */ -# define OSSL_HPKE_AEADSTR_CP "chacha20-poly1305" /* AEAD id 3 */ -# define OSSL_HPKE_AEADSTR_EXP "exporter" /* AEAD id 0xff */ - -/* - * Roles for use in creating an OSSL_HPKE_CTX, most - * important use of this is to control nonce reuse. - */ -# define OSSL_HPKE_ROLE_SENDER 0 -# define OSSL_HPKE_ROLE_RECEIVER 1 - -# ifdef __cplusplus -extern "C" { -# endif - -typedef struct { - uint16_t kem_id; /* Key Encapsulation Method id */ - uint16_t kdf_id; /* Key Derivation Function id */ - uint16_t aead_id; /* AEAD alg id */ -} OSSL_HPKE_SUITE; - -/** - * Suite constants, use this like: - * OSSL_HPKE_SUITE myvar = OSSL_HPKE_SUITE_DEFAULT; - */ -# ifndef OPENSSL_NO_ECX -# define OSSL_HPKE_SUITE_DEFAULT \ - {\ - OSSL_HPKE_KEM_ID_X25519, \ - OSSL_HPKE_KDF_ID_HKDF_SHA256, \ - OSSL_HPKE_AEAD_ID_AES_GCM_128 \ - } -# else -# define OSSL_HPKE_SUITE_DEFAULT \ - {\ - OSSL_HPKE_KEM_ID_P256, \ - OSSL_HPKE_KDF_ID_HKDF_SHA256, \ - OSSL_HPKE_AEAD_ID_AES_GCM_128 \ - } -#endif - -typedef struct ossl_hpke_ctx_st OSSL_HPKE_CTX; - -OSSL_HPKE_CTX *OSSL_HPKE_CTX_new(int mode, OSSL_HPKE_SUITE suite, int role, - OSSL_LIB_CTX *libctx, const char *propq); -void OSSL_HPKE_CTX_free(OSSL_HPKE_CTX *ctx); - -int OSSL_HPKE_encap(OSSL_HPKE_CTX *ctx, - unsigned char *enc, size_t *enclen, - const unsigned char *pub, size_t publen, - const unsigned char *info, size_t infolen); -int OSSL_HPKE_seal(OSSL_HPKE_CTX *ctx, - unsigned char *ct, size_t *ctlen, - const unsigned char *aad, size_t aadlen, - const unsigned char *pt, size_t ptlen); - -int OSSL_HPKE_keygen(OSSL_HPKE_SUITE suite, - unsigned char *pub, size_t *publen, EVP_PKEY **priv, - const unsigned char *ikm, size_t ikmlen, - OSSL_LIB_CTX *libctx, const char *propq); -int OSSL_HPKE_decap(OSSL_HPKE_CTX *ctx, - const unsigned char *enc, size_t enclen, - EVP_PKEY *recippriv, - const unsigned char *info, size_t infolen); -int OSSL_HPKE_open(OSSL_HPKE_CTX *ctx, - unsigned char *pt, size_t *ptlen, - const unsigned char *aad, size_t aadlen, - const unsigned char *ct, size_t ctlen); - -int OSSL_HPKE_export(OSSL_HPKE_CTX *ctx, - unsigned char *secret, - size_t secretlen, - const unsigned char *label, - size_t labellen); - -int OSSL_HPKE_CTX_set1_authpriv(OSSL_HPKE_CTX *ctx, EVP_PKEY *priv); -int OSSL_HPKE_CTX_set1_authpub(OSSL_HPKE_CTX *ctx, - const unsigned char *pub, - size_t publen); -int OSSL_HPKE_CTX_set1_psk(OSSL_HPKE_CTX *ctx, - const char *pskid, - const unsigned char *psk, size_t psklen); - -int OSSL_HPKE_CTX_set1_ikme(OSSL_HPKE_CTX *ctx, - const unsigned char *ikme, size_t ikmelen); - -int OSSL_HPKE_CTX_set_seq(OSSL_HPKE_CTX *ctx, uint64_t seq); -int OSSL_HPKE_CTX_get_seq(OSSL_HPKE_CTX *ctx, uint64_t *seq); - -int OSSL_HPKE_suite_check(OSSL_HPKE_SUITE suite); -int OSSL_HPKE_get_grease_value(const OSSL_HPKE_SUITE *suite_in, - OSSL_HPKE_SUITE *suite, - unsigned char *enc, size_t *enclen, - unsigned char *ct, size_t ctlen, - OSSL_LIB_CTX *libctx, const char *propq); -int OSSL_HPKE_str2suite(const char *str, OSSL_HPKE_SUITE *suite); -size_t OSSL_HPKE_get_ciphertext_size(OSSL_HPKE_SUITE suite, size_t clearlen); -size_t OSSL_HPKE_get_public_encap_size(OSSL_HPKE_SUITE suite); -size_t OSSL_HPKE_get_recommended_ikmelen(OSSL_HPKE_SUITE suite); - -# ifdef __cplusplus -} -# endif - -#endif diff --git a/openssl/include/openssl/http.h b/openssl/include/openssl/http.h index 8f4e9da30..f7ab21426 100644 --- a/openssl/include/openssl/http.h +++ b/openssl/include/openssl/http.h @@ -1,5 +1,5 @@ /* - * Copyright 2000-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Siemens AG 2018-2020 * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -33,12 +33,8 @@ extern "C" { # define OPENSSL_HTTP_PROXY "HTTP_PROXY" # define OPENSSL_HTTPS_PROXY "HTTPS_PROXY" -# ifndef OPENSSL_NO_HTTP - #define OSSL_HTTP_DEFAULT_MAX_LINE_LEN (4 * 1024) #define OSSL_HTTP_DEFAULT_MAX_RESP_LEN (100 * 1024) -#define OSSL_HTTP_DEFAULT_MAX_RESP_HDR_LINES 256 - /* Low-level HTTP API */ OSSL_HTTP_REQ_CTX *OSSL_HTTP_REQ_CTX_new(BIO *wbio, BIO *rbio, int buf_size); @@ -107,10 +103,6 @@ int OSSL_HTTP_parse_url(const char *url, int *pssl, char **puser, char **phost, const char *OSSL_HTTP_adapt_proxy(const char *proxy, const char *no_proxy, const char *server, int use_ssl); -void OSSL_HTTP_REQ_CTX_set_max_response_hdr_lines(OSSL_HTTP_REQ_CTX *rctx, - size_t count); - -# endif /* !defined(OPENSSL_NO_HTTP) */ # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/httperr.h b/openssl/include/openssl/httperr.h index ae7f00cac..ee0895920 100644 --- a/openssl/include/openssl/httperr.h +++ b/openssl/include/openssl/httperr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -44,7 +44,6 @@ # define HTTP_R_REDIRECTION_NOT_ENABLED 116 # define HTTP_R_RESPONSE_LINE_TOO_LONG 113 # define HTTP_R_RESPONSE_PARSE_ERROR 104 -# define HTTP_R_RESPONSE_TOO_MANY_HDRLINES 130 # define HTTP_R_RETRY_TIMEOUT 129 # define HTTP_R_SERVER_CANCELED_CONNECTION 127 # define HTTP_R_SOCK_NOT_SUPPORTED 122 diff --git a/openssl/include/openssl/idea.h b/openssl/include/openssl/idea.h deleted file mode 100644 index 1f9bb3b3c..000000000 --- a/openssl/include/openssl/idea.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_IDEA_H -# define OPENSSL_IDEA_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_IDEA_H -# endif - -# include - -# ifndef OPENSSL_NO_IDEA -# ifdef __cplusplus -extern "C" { -# endif - -# define IDEA_BLOCK 8 -# define IDEA_KEY_LENGTH 16 - -# ifndef OPENSSL_NO_DEPRECATED_3_0 - -typedef unsigned int IDEA_INT; - -# define IDEA_ENCRYPT 1 -# define IDEA_DECRYPT 0 - -typedef struct idea_key_st { - IDEA_INT data[9][6]; -} IDEA_KEY_SCHEDULE; -#endif -#ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 const char *IDEA_options(void); -OSSL_DEPRECATEDIN_3_0 void IDEA_ecb_encrypt(const unsigned char *in, - unsigned char *out, - IDEA_KEY_SCHEDULE *ks); -OSSL_DEPRECATEDIN_3_0 void IDEA_set_encrypt_key(const unsigned char *key, - IDEA_KEY_SCHEDULE *ks); -OSSL_DEPRECATEDIN_3_0 void IDEA_set_decrypt_key(IDEA_KEY_SCHEDULE *ek, - IDEA_KEY_SCHEDULE *dk); -OSSL_DEPRECATEDIN_3_0 void IDEA_cbc_encrypt(const unsigned char *in, - unsigned char *out, long length, - IDEA_KEY_SCHEDULE *ks, - unsigned char *iv, int enc); -OSSL_DEPRECATEDIN_3_0 void IDEA_cfb64_encrypt(const unsigned char *in, - unsigned char *out, long length, - IDEA_KEY_SCHEDULE *ks, - unsigned char *iv, int *num, - int enc); -OSSL_DEPRECATEDIN_3_0 void IDEA_ofb64_encrypt(const unsigned char *in, - unsigned char *out, long length, - IDEA_KEY_SCHEDULE *ks, - unsigned char *iv, int *num); -OSSL_DEPRECATEDIN_3_0 void IDEA_encrypt(unsigned long *in, - IDEA_KEY_SCHEDULE *ks); -#endif - -# ifndef OPENSSL_NO_DEPRECATED_1_1_0 -# define idea_options IDEA_options -# define idea_ecb_encrypt IDEA_ecb_encrypt -# define idea_set_encrypt_key IDEA_set_encrypt_key -# define idea_set_decrypt_key IDEA_set_decrypt_key -# define idea_cbc_encrypt IDEA_cbc_encrypt -# define idea_cfb64_encrypt IDEA_cfb64_encrypt -# define idea_ofb64_encrypt IDEA_ofb64_encrypt -# define idea_encrypt IDEA_encrypt -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/lhash.h b/openssl/include/openssl/lhash.h index 62c55b20f..39dd6254a 100644 --- a/openssl/include/openssl/lhash.h +++ b/openssl/include/openssl/lhash.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,9 +24,6 @@ # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -34,13 +31,9 @@ extern "C" { typedef struct lhash_node_st OPENSSL_LH_NODE; typedef int (*OPENSSL_LH_COMPFUNC) (const void *, const void *); -typedef int (*OPENSSL_LH_COMPFUNCTHUNK) (const void *, const void *, OPENSSL_LH_COMPFUNC cfn); typedef unsigned long (*OPENSSL_LH_HASHFUNC) (const void *); -typedef unsigned long (*OPENSSL_LH_HASHFUNCTHUNK) (const void *, OPENSSL_LH_HASHFUNC hfn); typedef void (*OPENSSL_LH_DOALL_FUNC) (void *); -typedef void (*OPENSSL_LH_DOALL_FUNC_THUNK) (void *, OPENSSL_LH_DOALL_FUNC doall); typedef void (*OPENSSL_LH_DOALL_FUNCARG) (void *, void *); -typedef void (*OPENSSL_LH_DOALL_FUNCARG_THUNK) (void *, void *, OPENSSL_LH_DOALL_FUNCARG doall); typedef struct lhash_st OPENSSL_LHASH; /* @@ -86,40 +79,26 @@ typedef struct lhash_st OPENSSL_LHASH; int OPENSSL_LH_error(OPENSSL_LHASH *lh); OPENSSL_LHASH *OPENSSL_LH_new(OPENSSL_LH_HASHFUNC h, OPENSSL_LH_COMPFUNC c); -OPENSSL_LHASH *OPENSSL_LH_set_thunks(OPENSSL_LHASH *lh, - OPENSSL_LH_HASHFUNCTHUNK hw, - OPENSSL_LH_COMPFUNCTHUNK cw, - OPENSSL_LH_DOALL_FUNC_THUNK daw, - OPENSSL_LH_DOALL_FUNCARG_THUNK daaw); void OPENSSL_LH_free(OPENSSL_LHASH *lh); void OPENSSL_LH_flush(OPENSSL_LHASH *lh); void *OPENSSL_LH_insert(OPENSSL_LHASH *lh, void *data); void *OPENSSL_LH_delete(OPENSSL_LHASH *lh, const void *data); void *OPENSSL_LH_retrieve(OPENSSL_LHASH *lh, const void *data); void OPENSSL_LH_doall(OPENSSL_LHASH *lh, OPENSSL_LH_DOALL_FUNC func); -void OPENSSL_LH_doall_arg(OPENSSL_LHASH *lh, - OPENSSL_LH_DOALL_FUNCARG func, void *arg); -void OPENSSL_LH_doall_arg_thunk(OPENSSL_LHASH *lh, - OPENSSL_LH_DOALL_FUNCARG_THUNK daaw, - OPENSSL_LH_DOALL_FUNCARG fn, void *arg); - +void OPENSSL_LH_doall_arg(OPENSSL_LHASH *lh, OPENSSL_LH_DOALL_FUNCARG func, void *arg); unsigned long OPENSSL_LH_strhash(const char *c); unsigned long OPENSSL_LH_num_items(const OPENSSL_LHASH *lh); unsigned long OPENSSL_LH_get_down_load(const OPENSSL_LHASH *lh); void OPENSSL_LH_set_down_load(OPENSSL_LHASH *lh, unsigned long down_load); # ifndef OPENSSL_NO_STDIO -# ifndef OPENSSL_NO_DEPRECATED_3_1 -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_stats(const OPENSSL_LHASH *lh, FILE *fp); -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_stats(const OPENSSL_LHASH *lh, FILE *fp); -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats(const OPENSSL_LHASH *lh, FILE *fp); -# endif -# endif -# ifndef OPENSSL_NO_DEPRECATED_3_1 -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_stats_bio(const OPENSSL_LHASH *lh, BIO *out); -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_stats_bio(const OPENSSL_LHASH *lh, BIO *out); -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH *lh, BIO *out); +void OPENSSL_LH_stats(const OPENSSL_LHASH *lh, FILE *fp); +void OPENSSL_LH_node_stats(const OPENSSL_LHASH *lh, FILE *fp); +void OPENSSL_LH_node_usage_stats(const OPENSSL_LHASH *lh, FILE *fp); # endif +void OPENSSL_LH_stats_bio(const OPENSSL_LHASH *lh, BIO *out); +void OPENSSL_LH_node_stats_bio(const OPENSSL_LHASH *lh, BIO *out); +void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH *lh, BIO *out); # ifndef OPENSSL_NO_DEPRECATED_1_1_0 # define _LHASH OPENSSL_LHASH @@ -150,190 +129,110 @@ OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH * /* Helper macro for internal use */ # define DEFINE_LHASH_OF_INTERNAL(type) \ - LHASH_OF(type) { \ - union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; \ - }; \ + LHASH_OF(type) { union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; }; \ typedef int (*lh_##type##_compfunc)(const type *a, const type *b); \ typedef unsigned long (*lh_##type##_hashfunc)(const type *a); \ typedef void (*lh_##type##_doallfunc)(type *a); \ - static ossl_inline unsigned long lh_##type##_hash_thunk(const void *data, OPENSSL_LH_HASHFUNC hfn) \ - { \ - unsigned long (*hfn_conv)(const type *) = (unsigned long (*)(const type *))hfn; \ - return hfn_conv((const type *)data); \ - } \ - static ossl_inline int lh_##type##_comp_thunk(const void *da, const void *db, OPENSSL_LH_COMPFUNC cfn) \ - { \ - int (*cfn_conv)(const type *, const type *) = (int (*)(const type *, const type *))cfn; \ - return cfn_conv((const type *)da, (const type *)db); \ - } \ - static ossl_inline void lh_##type##_doall_thunk(void *node, OPENSSL_LH_DOALL_FUNC doall) \ - { \ - void (*doall_conv)(type *) = (void (*)(type *))doall; \ - doall_conv((type *)node); \ - } \ - static ossl_inline void lh_##type##_doall_arg_thunk(void *node, void *arg, OPENSSL_LH_DOALL_FUNCARG doall) \ - { \ - void (*doall_conv)(type *, void *) = (void (*)(type *, void *))doall; \ - doall_conv((type *)node, arg); \ - } \ - static ossl_unused ossl_inline type *\ - ossl_check_##type##_lh_plain_type(type *ptr) \ + static ossl_unused ossl_inline type *ossl_check_##type##_lh_plain_type(type *ptr) \ { \ return ptr; \ } \ - static ossl_unused ossl_inline const type * \ - ossl_check_const_##type##_lh_plain_type(const type *ptr) \ + static ossl_unused ossl_inline const type *ossl_check_const_##type##_lh_plain_type(const type *ptr) \ { \ return ptr; \ } \ - static ossl_unused ossl_inline const OPENSSL_LHASH * \ - ossl_check_const_##type##_lh_type(const LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline const OPENSSL_LHASH *ossl_check_const_##type##_lh_type(const LHASH_OF(type) *lh) \ { \ return (const OPENSSL_LHASH *)lh; \ } \ - static ossl_unused ossl_inline OPENSSL_LHASH * \ - ossl_check_##type##_lh_type(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline OPENSSL_LHASH *ossl_check_##type##_lh_type(LHASH_OF(type) *lh) \ { \ return (OPENSSL_LHASH *)lh; \ } \ - static ossl_unused ossl_inline OPENSSL_LH_COMPFUNC \ - ossl_check_##type##_lh_compfunc_type(lh_##type##_compfunc cmp) \ + static ossl_unused ossl_inline OPENSSL_LH_COMPFUNC ossl_check_##type##_lh_compfunc_type(lh_##type##_compfunc cmp) \ { \ return (OPENSSL_LH_COMPFUNC)cmp; \ } \ - static ossl_unused ossl_inline OPENSSL_LH_HASHFUNC \ - ossl_check_##type##_lh_hashfunc_type(lh_##type##_hashfunc hfn) \ + static ossl_unused ossl_inline OPENSSL_LH_HASHFUNC ossl_check_##type##_lh_hashfunc_type(lh_##type##_hashfunc hfn) \ { \ return (OPENSSL_LH_HASHFUNC)hfn; \ } \ - static ossl_unused ossl_inline OPENSSL_LH_DOALL_FUNC \ - ossl_check_##type##_lh_doallfunc_type(lh_##type##_doallfunc dfn) \ + static ossl_unused ossl_inline OPENSSL_LH_DOALL_FUNC ossl_check_##type##_lh_doallfunc_type(lh_##type##_doallfunc dfn) \ { \ return (OPENSSL_LH_DOALL_FUNC)dfn; \ } \ LHASH_OF(type) -# ifndef OPENSSL_NO_DEPRECATED_3_1 -# define DEFINE_LHASH_OF_DEPRECATED(type) \ - static ossl_unused ossl_inline void \ - lh_##type##_node_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ - { \ - OPENSSL_LH_node_stats_bio((const OPENSSL_LHASH *)lh, out); \ - } \ - static ossl_unused ossl_inline void \ - lh_##type##_node_usage_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ - { \ - OPENSSL_LH_node_usage_stats_bio((const OPENSSL_LHASH *)lh, out); \ - } \ - static ossl_unused ossl_inline void \ - lh_##type##_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ - { \ - OPENSSL_LH_stats_bio((const OPENSSL_LHASH *)lh, out); \ - } -# else -# define DEFINE_LHASH_OF_DEPRECATED(type) -# endif - -# define DEFINE_LHASH_OF_EX(type) \ - LHASH_OF(type) { \ - union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; \ - }; \ - static unsigned long \ - lh_##type##_hfn_thunk(const void *data, OPENSSL_LH_HASHFUNC hfn) \ - { \ - unsigned long (*hfn_conv)(const type *) = (unsigned long (*)(const type *))hfn; \ - return hfn_conv((const type *)data); \ - } \ - static int lh_##type##_cfn_thunk(const void *da, const void *db, OPENSSL_LH_COMPFUNC cfn) \ +# define DEFINE_LHASH_OF(type) \ + LHASH_OF(type) { union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; }; \ + static ossl_unused ossl_inline LHASH_OF(type) *lh_##type##_new(unsigned long (*hfn)(const type *), \ + int (*cfn)(const type *, const type *)) \ { \ - int (*cfn_conv)(const type *, const type *) = (int (*)(const type *, const type *))cfn; \ - return cfn_conv((const type *)da, (const type *)db); \ + return (LHASH_OF(type) *) \ + OPENSSL_LH_new((OPENSSL_LH_HASHFUNC)hfn, (OPENSSL_LH_COMPFUNC)cfn); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_free(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline void lh_##type##_free(LHASH_OF(type) *lh) \ { \ OPENSSL_LH_free((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_flush(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline void lh_##type##_flush(LHASH_OF(type) *lh) \ { \ OPENSSL_LH_flush((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline type * \ - lh_##type##_insert(LHASH_OF(type) *lh, type *d) \ + static ossl_unused ossl_inline type *lh_##type##_insert(LHASH_OF(type) *lh, type *d) \ { \ return (type *)OPENSSL_LH_insert((OPENSSL_LHASH *)lh, d); \ } \ - static ossl_unused ossl_inline type * \ - lh_##type##_delete(LHASH_OF(type) *lh, const type *d) \ + static ossl_unused ossl_inline type *lh_##type##_delete(LHASH_OF(type) *lh, const type *d) \ { \ return (type *)OPENSSL_LH_delete((OPENSSL_LHASH *)lh, d); \ } \ - static ossl_unused ossl_inline type * \ - lh_##type##_retrieve(LHASH_OF(type) *lh, const type *d) \ + static ossl_unused ossl_inline type *lh_##type##_retrieve(LHASH_OF(type) *lh, const type *d) \ { \ return (type *)OPENSSL_LH_retrieve((OPENSSL_LHASH *)lh, d); \ } \ - static ossl_unused ossl_inline int \ - lh_##type##_error(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline int lh_##type##_error(LHASH_OF(type) *lh) \ { \ return OPENSSL_LH_error((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline unsigned long \ - lh_##type##_num_items(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline unsigned long lh_##type##_num_items(LHASH_OF(type) *lh) \ { \ return OPENSSL_LH_num_items((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline unsigned long \ - lh_##type##_get_down_load(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline void lh_##type##_node_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ { \ - return OPENSSL_LH_get_down_load((OPENSSL_LHASH *)lh); \ + OPENSSL_LH_node_stats_bio((const OPENSSL_LHASH *)lh, out); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_set_down_load(LHASH_OF(type) *lh, unsigned long dl) \ + static ossl_unused ossl_inline void lh_##type##_node_usage_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ { \ - OPENSSL_LH_set_down_load((OPENSSL_LHASH *)lh, dl); \ + OPENSSL_LH_node_usage_stats_bio((const OPENSSL_LHASH *)lh, out); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_doall_thunk(void *node, OPENSSL_LH_DOALL_FUNC doall) \ + static ossl_unused ossl_inline void lh_##type##_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ { \ - void (*doall_conv)(type *) = (void (*)(type *))doall; \ - doall_conv((type *)node); \ + OPENSSL_LH_stats_bio((const OPENSSL_LHASH *)lh, out); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_doall_arg_thunk(void *node, void *arg, OPENSSL_LH_DOALL_FUNCARG doall) \ + static ossl_unused ossl_inline unsigned long lh_##type##_get_down_load(LHASH_OF(type) *lh) \ { \ - void (*doall_conv)(type *, void *) = (void (*)(type *, void *))doall; \ - doall_conv((type *)node, arg); \ + return OPENSSL_LH_get_down_load((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_doall(LHASH_OF(type) *lh, void (*doall)(type *)) \ + static ossl_unused ossl_inline void lh_##type##_set_down_load(LHASH_OF(type) *lh, unsigned long dl) \ { \ - OPENSSL_LH_doall((OPENSSL_LHASH *)lh, (OPENSSL_LH_DOALL_FUNC)doall); \ + OPENSSL_LH_set_down_load((OPENSSL_LHASH *)lh, dl); \ } \ - static ossl_unused ossl_inline LHASH_OF(type) * \ - lh_##type##_new(unsigned long (*hfn)(const type *), \ - int (*cfn)(const type *, const type *)) \ + static ossl_unused ossl_inline void lh_##type##_doall(LHASH_OF(type) *lh, \ + void (*doall)(type *)) \ { \ - return (LHASH_OF(type) *)OPENSSL_LH_set_thunks(OPENSSL_LH_new((OPENSSL_LH_HASHFUNC)hfn, (OPENSSL_LH_COMPFUNC)cfn), \ - lh_##type##_hfn_thunk, lh_##type##_cfn_thunk, \ - lh_##type##_doall_thunk, \ - lh_##type##_doall_arg_thunk); \ + OPENSSL_LH_doall((OPENSSL_LHASH *)lh, (OPENSSL_LH_DOALL_FUNC)doall); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_doall_arg(LHASH_OF(type) *lh, \ - void (*doallarg)(type *, void *), void *arg) \ + static ossl_unused ossl_inline void lh_##type##_doall_arg(LHASH_OF(type) *lh, \ + void (*doallarg)(type *, void *), \ + void *arg) \ { \ OPENSSL_LH_doall_arg((OPENSSL_LHASH *)lh, \ (OPENSSL_LH_DOALL_FUNCARG)doallarg, arg); \ } \ LHASH_OF(type) -# define DEFINE_LHASH_OF(type) \ - DEFINE_LHASH_OF_EX(type); \ - DEFINE_LHASH_OF_DEPRECATED(type) \ - LHASH_OF(type) - #define IMPLEMENT_LHASH_DOALL_ARG_CONST(type, argtype) \ int_implement_lhash_doall(type, argtype, const type) @@ -341,26 +240,17 @@ OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH * int_implement_lhash_doall(type, argtype, type) #define int_implement_lhash_doall(type, argtype, cbargtype) \ - static ossl_unused ossl_inline void \ - lh_##type##_doall_##argtype##_thunk(void *node, void *arg, OPENSSL_LH_DOALL_FUNCARG fn) \ - { \ - void (*fn_conv)(cbargtype *, argtype *) = (void (*)(cbargtype *, argtype *))fn; \ - fn_conv((cbargtype *)node, (argtype *)arg); \ - } \ static ossl_unused ossl_inline void \ lh_##type##_doall_##argtype(LHASH_OF(type) *lh, \ void (*fn)(cbargtype *, argtype *), \ argtype *arg) \ { \ - OPENSSL_LH_doall_arg_thunk((OPENSSL_LHASH *)lh, \ - lh_##type##_doall_##argtype##_thunk, \ - (OPENSSL_LH_DOALL_FUNCARG)fn, \ - (void *)arg); \ + OPENSSL_LH_doall_arg((OPENSSL_LHASH *)lh, (OPENSSL_LH_DOALL_FUNCARG)fn, (void *)arg); \ } \ LHASH_OF(type) DEFINE_LHASH_OF_INTERNAL(OPENSSL_STRING); -#define lh_OPENSSL_STRING_new(hfn, cmp) ((LHASH_OF(OPENSSL_STRING) *)OPENSSL_LH_set_thunks(OPENSSL_LH_new(ossl_check_OPENSSL_STRING_lh_hashfunc_type(hfn), ossl_check_OPENSSL_STRING_lh_compfunc_type(cmp)), lh_OPENSSL_STRING_hash_thunk, lh_OPENSSL_STRING_comp_thunk, lh_OPENSSL_STRING_doall_thunk, lh_OPENSSL_STRING_doall_arg_thunk)) +#define lh_OPENSSL_STRING_new(hfn, cmp) ((LHASH_OF(OPENSSL_STRING) *)OPENSSL_LH_new(ossl_check_OPENSSL_STRING_lh_hashfunc_type(hfn), ossl_check_OPENSSL_STRING_lh_compfunc_type(cmp))) #define lh_OPENSSL_STRING_free(lh) OPENSSL_LH_free(ossl_check_OPENSSL_STRING_lh_type(lh)) #define lh_OPENSSL_STRING_flush(lh) OPENSSL_LH_flush(ossl_check_OPENSSL_STRING_lh_type(lh)) #define lh_OPENSSL_STRING_insert(lh, ptr) ((OPENSSL_STRING *)OPENSSL_LH_insert(ossl_check_OPENSSL_STRING_lh_type(lh), ossl_check_OPENSSL_STRING_lh_plain_type(ptr))) @@ -375,7 +265,7 @@ DEFINE_LHASH_OF_INTERNAL(OPENSSL_STRING); #define lh_OPENSSL_STRING_set_down_load(lh, dl) OPENSSL_LH_set_down_load(ossl_check_OPENSSL_STRING_lh_type(lh), dl) #define lh_OPENSSL_STRING_doall(lh, dfn) OPENSSL_LH_doall(ossl_check_OPENSSL_STRING_lh_type(lh), ossl_check_OPENSSL_STRING_lh_doallfunc_type(dfn)) DEFINE_LHASH_OF_INTERNAL(OPENSSL_CSTRING); -#define lh_OPENSSL_CSTRING_new(hfn, cmp) ((LHASH_OF(OPENSSL_CSTRING) *)OPENSSL_LH_set_thunks(OPENSSL_LH_new(ossl_check_OPENSSL_CSTRING_lh_hashfunc_type(hfn), ossl_check_OPENSSL_CSTRING_lh_compfunc_type(cmp)), lh_OPENSSL_CSTRING_hash_thunk, lh_OPENSSL_CSTRING_comp_thunk, lh_OPENSSL_CSTRING_doall_thunk, lh_OPENSSL_CSTRING_doall_arg_thunk)) +#define lh_OPENSSL_CSTRING_new(hfn, cmp) ((LHASH_OF(OPENSSL_CSTRING) *)OPENSSL_LH_new(ossl_check_OPENSSL_CSTRING_lh_hashfunc_type(hfn), ossl_check_OPENSSL_CSTRING_lh_compfunc_type(cmp))) #define lh_OPENSSL_CSTRING_free(lh) OPENSSL_LH_free(ossl_check_OPENSSL_CSTRING_lh_type(lh)) #define lh_OPENSSL_CSTRING_flush(lh) OPENSSL_LH_flush(ossl_check_OPENSSL_CSTRING_lh_type(lh)) #define lh_OPENSSL_CSTRING_insert(lh, ptr) ((OPENSSL_CSTRING *)OPENSSL_LH_insert(ossl_check_OPENSSL_CSTRING_lh_type(lh), ossl_check_OPENSSL_CSTRING_lh_plain_type(ptr))) diff --git a/openssl/include/openssl/lhash.h.in b/openssl/include/openssl/lhash.h.in index 63a9c53bf..febefa3c4 100644 --- a/openssl/include/openssl/lhash.h.in +++ b/openssl/include/openssl/lhash.h.in @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,9 +26,6 @@ use OpenSSL::stackhash qw(generate_lhash_macros); # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -36,13 +33,9 @@ extern "C" { typedef struct lhash_node_st OPENSSL_LH_NODE; typedef int (*OPENSSL_LH_COMPFUNC) (const void *, const void *); -typedef int (*OPENSSL_LH_COMPFUNCTHUNK) (const void *, const void *, OPENSSL_LH_COMPFUNC cfn); typedef unsigned long (*OPENSSL_LH_HASHFUNC) (const void *); -typedef unsigned long (*OPENSSL_LH_HASHFUNCTHUNK) (const void *, OPENSSL_LH_HASHFUNC hfn); typedef void (*OPENSSL_LH_DOALL_FUNC) (void *); -typedef void (*OPENSSL_LH_DOALL_FUNC_THUNK) (void *, OPENSSL_LH_DOALL_FUNC doall); typedef void (*OPENSSL_LH_DOALL_FUNCARG) (void *, void *); -typedef void (*OPENSSL_LH_DOALL_FUNCARG_THUNK) (void *, void *, OPENSSL_LH_DOALL_FUNCARG doall); typedef struct lhash_st OPENSSL_LHASH; /* @@ -88,40 +81,26 @@ typedef struct lhash_st OPENSSL_LHASH; int OPENSSL_LH_error(OPENSSL_LHASH *lh); OPENSSL_LHASH *OPENSSL_LH_new(OPENSSL_LH_HASHFUNC h, OPENSSL_LH_COMPFUNC c); -OPENSSL_LHASH *OPENSSL_LH_set_thunks(OPENSSL_LHASH *lh, - OPENSSL_LH_HASHFUNCTHUNK hw, - OPENSSL_LH_COMPFUNCTHUNK cw, - OPENSSL_LH_DOALL_FUNC_THUNK daw, - OPENSSL_LH_DOALL_FUNCARG_THUNK daaw); void OPENSSL_LH_free(OPENSSL_LHASH *lh); void OPENSSL_LH_flush(OPENSSL_LHASH *lh); void *OPENSSL_LH_insert(OPENSSL_LHASH *lh, void *data); void *OPENSSL_LH_delete(OPENSSL_LHASH *lh, const void *data); void *OPENSSL_LH_retrieve(OPENSSL_LHASH *lh, const void *data); void OPENSSL_LH_doall(OPENSSL_LHASH *lh, OPENSSL_LH_DOALL_FUNC func); -void OPENSSL_LH_doall_arg(OPENSSL_LHASH *lh, - OPENSSL_LH_DOALL_FUNCARG func, void *arg); -void OPENSSL_LH_doall_arg_thunk(OPENSSL_LHASH *lh, - OPENSSL_LH_DOALL_FUNCARG_THUNK daaw, - OPENSSL_LH_DOALL_FUNCARG fn, void *arg); - +void OPENSSL_LH_doall_arg(OPENSSL_LHASH *lh, OPENSSL_LH_DOALL_FUNCARG func, void *arg); unsigned long OPENSSL_LH_strhash(const char *c); unsigned long OPENSSL_LH_num_items(const OPENSSL_LHASH *lh); unsigned long OPENSSL_LH_get_down_load(const OPENSSL_LHASH *lh); void OPENSSL_LH_set_down_load(OPENSSL_LHASH *lh, unsigned long down_load); # ifndef OPENSSL_NO_STDIO -# ifndef OPENSSL_NO_DEPRECATED_3_1 -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_stats(const OPENSSL_LHASH *lh, FILE *fp); -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_stats(const OPENSSL_LHASH *lh, FILE *fp); -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats(const OPENSSL_LHASH *lh, FILE *fp); -# endif -# endif -# ifndef OPENSSL_NO_DEPRECATED_3_1 -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_stats_bio(const OPENSSL_LHASH *lh, BIO *out); -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_stats_bio(const OPENSSL_LHASH *lh, BIO *out); -OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH *lh, BIO *out); +void OPENSSL_LH_stats(const OPENSSL_LHASH *lh, FILE *fp); +void OPENSSL_LH_node_stats(const OPENSSL_LHASH *lh, FILE *fp); +void OPENSSL_LH_node_usage_stats(const OPENSSL_LHASH *lh, FILE *fp); # endif +void OPENSSL_LH_stats_bio(const OPENSSL_LHASH *lh, BIO *out); +void OPENSSL_LH_node_stats_bio(const OPENSSL_LHASH *lh, BIO *out); +void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH *lh, BIO *out); # ifndef OPENSSL_NO_DEPRECATED_1_1_0 # define _LHASH OPENSSL_LHASH @@ -152,190 +131,110 @@ OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH * /* Helper macro for internal use */ # define DEFINE_LHASH_OF_INTERNAL(type) \ - LHASH_OF(type) { \ - union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; \ - }; \ + LHASH_OF(type) { union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; }; \ typedef int (*lh_##type##_compfunc)(const type *a, const type *b); \ typedef unsigned long (*lh_##type##_hashfunc)(const type *a); \ typedef void (*lh_##type##_doallfunc)(type *a); \ - static ossl_inline unsigned long lh_##type##_hash_thunk(const void *data, OPENSSL_LH_HASHFUNC hfn) \ - { \ - unsigned long (*hfn_conv)(const type *) = (unsigned long (*)(const type *))hfn; \ - return hfn_conv((const type *)data); \ - } \ - static ossl_inline int lh_##type##_comp_thunk(const void *da, const void *db, OPENSSL_LH_COMPFUNC cfn) \ - { \ - int (*cfn_conv)(const type *, const type *) = (int (*)(const type *, const type *))cfn; \ - return cfn_conv((const type *)da, (const type *)db); \ - } \ - static ossl_inline void lh_##type##_doall_thunk(void *node, OPENSSL_LH_DOALL_FUNC doall) \ - { \ - void (*doall_conv)(type *) = (void (*)(type *))doall; \ - doall_conv((type *)node); \ - } \ - static ossl_inline void lh_##type##_doall_arg_thunk(void *node, void *arg, OPENSSL_LH_DOALL_FUNCARG doall) \ - { \ - void (*doall_conv)(type *, void *) = (void (*)(type *, void *))doall; \ - doall_conv((type *)node, arg); \ - } \ - static ossl_unused ossl_inline type *\ - ossl_check_##type##_lh_plain_type(type *ptr) \ + static ossl_unused ossl_inline type *ossl_check_##type##_lh_plain_type(type *ptr) \ { \ return ptr; \ } \ - static ossl_unused ossl_inline const type * \ - ossl_check_const_##type##_lh_plain_type(const type *ptr) \ + static ossl_unused ossl_inline const type *ossl_check_const_##type##_lh_plain_type(const type *ptr) \ { \ return ptr; \ } \ - static ossl_unused ossl_inline const OPENSSL_LHASH * \ - ossl_check_const_##type##_lh_type(const LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline const OPENSSL_LHASH *ossl_check_const_##type##_lh_type(const LHASH_OF(type) *lh) \ { \ return (const OPENSSL_LHASH *)lh; \ } \ - static ossl_unused ossl_inline OPENSSL_LHASH * \ - ossl_check_##type##_lh_type(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline OPENSSL_LHASH *ossl_check_##type##_lh_type(LHASH_OF(type) *lh) \ { \ return (OPENSSL_LHASH *)lh; \ } \ - static ossl_unused ossl_inline OPENSSL_LH_COMPFUNC \ - ossl_check_##type##_lh_compfunc_type(lh_##type##_compfunc cmp) \ + static ossl_unused ossl_inline OPENSSL_LH_COMPFUNC ossl_check_##type##_lh_compfunc_type(lh_##type##_compfunc cmp) \ { \ return (OPENSSL_LH_COMPFUNC)cmp; \ } \ - static ossl_unused ossl_inline OPENSSL_LH_HASHFUNC \ - ossl_check_##type##_lh_hashfunc_type(lh_##type##_hashfunc hfn) \ + static ossl_unused ossl_inline OPENSSL_LH_HASHFUNC ossl_check_##type##_lh_hashfunc_type(lh_##type##_hashfunc hfn) \ { \ return (OPENSSL_LH_HASHFUNC)hfn; \ } \ - static ossl_unused ossl_inline OPENSSL_LH_DOALL_FUNC \ - ossl_check_##type##_lh_doallfunc_type(lh_##type##_doallfunc dfn) \ + static ossl_unused ossl_inline OPENSSL_LH_DOALL_FUNC ossl_check_##type##_lh_doallfunc_type(lh_##type##_doallfunc dfn) \ { \ return (OPENSSL_LH_DOALL_FUNC)dfn; \ } \ LHASH_OF(type) -# ifndef OPENSSL_NO_DEPRECATED_3_1 -# define DEFINE_LHASH_OF_DEPRECATED(type) \ - static ossl_unused ossl_inline void \ - lh_##type##_node_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ - { \ - OPENSSL_LH_node_stats_bio((const OPENSSL_LHASH *)lh, out); \ - } \ - static ossl_unused ossl_inline void \ - lh_##type##_node_usage_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ - { \ - OPENSSL_LH_node_usage_stats_bio((const OPENSSL_LHASH *)lh, out); \ - } \ - static ossl_unused ossl_inline void \ - lh_##type##_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ - { \ - OPENSSL_LH_stats_bio((const OPENSSL_LHASH *)lh, out); \ - } -# else -# define DEFINE_LHASH_OF_DEPRECATED(type) -# endif - -# define DEFINE_LHASH_OF_EX(type) \ - LHASH_OF(type) { \ - union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; \ - }; \ - static unsigned long \ - lh_##type##_hfn_thunk(const void *data, OPENSSL_LH_HASHFUNC hfn) \ - { \ - unsigned long (*hfn_conv)(const type *) = (unsigned long (*)(const type *))hfn; \ - return hfn_conv((const type *)data); \ - } \ - static int lh_##type##_cfn_thunk(const void *da, const void *db, OPENSSL_LH_COMPFUNC cfn) \ +# define DEFINE_LHASH_OF(type) \ + LHASH_OF(type) { union lh_##type##_dummy { void* d1; unsigned long d2; int d3; } dummy; }; \ + static ossl_unused ossl_inline LHASH_OF(type) *lh_##type##_new(unsigned long (*hfn)(const type *), \ + int (*cfn)(const type *, const type *)) \ { \ - int (*cfn_conv)(const type *, const type *) = (int (*)(const type *, const type *))cfn; \ - return cfn_conv((const type *)da, (const type *)db); \ + return (LHASH_OF(type) *) \ + OPENSSL_LH_new((OPENSSL_LH_HASHFUNC)hfn, (OPENSSL_LH_COMPFUNC)cfn); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_free(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline void lh_##type##_free(LHASH_OF(type) *lh) \ { \ OPENSSL_LH_free((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_flush(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline void lh_##type##_flush(LHASH_OF(type) *lh) \ { \ OPENSSL_LH_flush((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline type * \ - lh_##type##_insert(LHASH_OF(type) *lh, type *d) \ + static ossl_unused ossl_inline type *lh_##type##_insert(LHASH_OF(type) *lh, type *d) \ { \ return (type *)OPENSSL_LH_insert((OPENSSL_LHASH *)lh, d); \ } \ - static ossl_unused ossl_inline type * \ - lh_##type##_delete(LHASH_OF(type) *lh, const type *d) \ + static ossl_unused ossl_inline type *lh_##type##_delete(LHASH_OF(type) *lh, const type *d) \ { \ return (type *)OPENSSL_LH_delete((OPENSSL_LHASH *)lh, d); \ } \ - static ossl_unused ossl_inline type * \ - lh_##type##_retrieve(LHASH_OF(type) *lh, const type *d) \ + static ossl_unused ossl_inline type *lh_##type##_retrieve(LHASH_OF(type) *lh, const type *d) \ { \ return (type *)OPENSSL_LH_retrieve((OPENSSL_LHASH *)lh, d); \ } \ - static ossl_unused ossl_inline int \ - lh_##type##_error(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline int lh_##type##_error(LHASH_OF(type) *lh) \ { \ return OPENSSL_LH_error((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline unsigned long \ - lh_##type##_num_items(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline unsigned long lh_##type##_num_items(LHASH_OF(type) *lh) \ { \ return OPENSSL_LH_num_items((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline unsigned long \ - lh_##type##_get_down_load(LHASH_OF(type) *lh) \ + static ossl_unused ossl_inline void lh_##type##_node_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ { \ - return OPENSSL_LH_get_down_load((OPENSSL_LHASH *)lh); \ + OPENSSL_LH_node_stats_bio((const OPENSSL_LHASH *)lh, out); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_set_down_load(LHASH_OF(type) *lh, unsigned long dl) \ + static ossl_unused ossl_inline void lh_##type##_node_usage_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ { \ - OPENSSL_LH_set_down_load((OPENSSL_LHASH *)lh, dl); \ + OPENSSL_LH_node_usage_stats_bio((const OPENSSL_LHASH *)lh, out); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_doall_thunk(void *node, OPENSSL_LH_DOALL_FUNC doall) \ + static ossl_unused ossl_inline void lh_##type##_stats_bio(const LHASH_OF(type) *lh, BIO *out) \ { \ - void (*doall_conv)(type *) = (void (*)(type *))doall; \ - doall_conv((type *)node); \ + OPENSSL_LH_stats_bio((const OPENSSL_LHASH *)lh, out); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_doall_arg_thunk(void *node, void *arg, OPENSSL_LH_DOALL_FUNCARG doall) \ + static ossl_unused ossl_inline unsigned long lh_##type##_get_down_load(LHASH_OF(type) *lh) \ { \ - void (*doall_conv)(type *, void *) = (void (*)(type *, void *))doall; \ - doall_conv((type *)node, arg); \ + return OPENSSL_LH_get_down_load((OPENSSL_LHASH *)lh); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_doall(LHASH_OF(type) *lh, void (*doall)(type *)) \ + static ossl_unused ossl_inline void lh_##type##_set_down_load(LHASH_OF(type) *lh, unsigned long dl) \ { \ - OPENSSL_LH_doall((OPENSSL_LHASH *)lh, (OPENSSL_LH_DOALL_FUNC)doall); \ + OPENSSL_LH_set_down_load((OPENSSL_LHASH *)lh, dl); \ } \ - static ossl_unused ossl_inline LHASH_OF(type) * \ - lh_##type##_new(unsigned long (*hfn)(const type *), \ - int (*cfn)(const type *, const type *)) \ + static ossl_unused ossl_inline void lh_##type##_doall(LHASH_OF(type) *lh, \ + void (*doall)(type *)) \ { \ - return (LHASH_OF(type) *)OPENSSL_LH_set_thunks(OPENSSL_LH_new((OPENSSL_LH_HASHFUNC)hfn, (OPENSSL_LH_COMPFUNC)cfn), \ - lh_##type##_hfn_thunk, lh_##type##_cfn_thunk, \ - lh_##type##_doall_thunk, \ - lh_##type##_doall_arg_thunk); \ + OPENSSL_LH_doall((OPENSSL_LHASH *)lh, (OPENSSL_LH_DOALL_FUNC)doall); \ } \ - static ossl_unused ossl_inline void \ - lh_##type##_doall_arg(LHASH_OF(type) *lh, \ - void (*doallarg)(type *, void *), void *arg) \ + static ossl_unused ossl_inline void lh_##type##_doall_arg(LHASH_OF(type) *lh, \ + void (*doallarg)(type *, void *), \ + void *arg) \ { \ OPENSSL_LH_doall_arg((OPENSSL_LHASH *)lh, \ (OPENSSL_LH_DOALL_FUNCARG)doallarg, arg); \ } \ LHASH_OF(type) -# define DEFINE_LHASH_OF(type) \ - DEFINE_LHASH_OF_EX(type); \ - DEFINE_LHASH_OF_DEPRECATED(type) \ - LHASH_OF(type) - #define IMPLEMENT_LHASH_DOALL_ARG_CONST(type, argtype) \ int_implement_lhash_doall(type, argtype, const type) @@ -343,21 +242,12 @@ OSSL_DEPRECATEDIN_3_1 void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH * int_implement_lhash_doall(type, argtype, type) #define int_implement_lhash_doall(type, argtype, cbargtype) \ - static ossl_unused ossl_inline void \ - lh_##type##_doall_##argtype##_thunk(void *node, void *arg, OPENSSL_LH_DOALL_FUNCARG fn) \ - { \ - void (*fn_conv)(cbargtype *, argtype *) = (void (*)(cbargtype *, argtype *))fn; \ - fn_conv((cbargtype *)node, (argtype *)arg); \ - } \ static ossl_unused ossl_inline void \ lh_##type##_doall_##argtype(LHASH_OF(type) *lh, \ void (*fn)(cbargtype *, argtype *), \ argtype *arg) \ { \ - OPENSSL_LH_doall_arg_thunk((OPENSSL_LHASH *)lh, \ - lh_##type##_doall_##argtype##_thunk, \ - (OPENSSL_LH_DOALL_FUNCARG)fn, \ - (void *)arg); \ + OPENSSL_LH_doall_arg((OPENSSL_LHASH *)lh, (OPENSSL_LH_DOALL_FUNCARG)fn, (void *)arg); \ } \ LHASH_OF(type) diff --git a/openssl/include/openssl/macros.h b/openssl/include/openssl/macros.h index e9ef93874..a6bc3f1fe 100644 --- a/openssl/include/openssl/macros.h +++ b/openssl/include/openssl/macros.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -158,7 +158,7 @@ /* * Define macros for deprecation and simulated removal purposes. * - * The macros OSSL_DEPRECATEDIN_{major}_{minor} are always defined for + * The macros OSSL_DEPRECATED_{major}_{minor} are always defined for * all OpenSSL versions we care for. They can be used as attributes * in function declarations where appropriate. * @@ -169,7 +169,6 @@ * 'no-deprecated'. */ -# undef OPENSSL_NO_DEPRECATED_3_1 # undef OPENSSL_NO_DEPRECATED_3_0 # undef OPENSSL_NO_DEPRECATED_1_1_1 # undef OPENSSL_NO_DEPRECATED_1_1_0 @@ -178,17 +177,6 @@ # undef OPENSSL_NO_DEPRECATED_1_0_0 # undef OPENSSL_NO_DEPRECATED_0_9_8 -# if OPENSSL_API_LEVEL >= 30100 -# ifndef OPENSSL_NO_DEPRECATED -# define OSSL_DEPRECATEDIN_3_1 OSSL_DEPRECATED(3.1) -# define OSSL_DEPRECATEDIN_3_1_FOR(msg) OSSL_DEPRECATED_FOR(3.1, msg) -# else -# define OPENSSL_NO_DEPRECATED_3_1 -# endif -# else -# define OSSL_DEPRECATEDIN_3_1 -# define OSSL_DEPRECATEDIN_3_1_FOR(msg) -# endif # if OPENSSL_API_LEVEL >= 30000 # ifndef OPENSSL_NO_DEPRECATED # define OSSL_DEPRECATEDIN_3_0 OSSL_DEPRECATED(3.0) @@ -313,14 +301,4 @@ # endif # endif -# ifndef OSSL_CRYPTO_ALLOC -# if defined(__GNUC__) -# define OSSL_CRYPTO_ALLOC __attribute__((__malloc__)) -# elif defined(_MSC_VER) -# define OSSL_CRYPTO_ALLOC __declspec(restrict) -# else -# define OSSL_CRYPTO_ALLOC -# endif -# endif - #endif /* OPENSSL_MACROS_H */ diff --git a/openssl/include/openssl/md2.h b/openssl/include/openssl/md2.h deleted file mode 100644 index 5d4cb77e7..000000000 --- a/openssl/include/openssl/md2.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_MD2_H -# define OPENSSL_MD2_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_MD2_H -# endif - -# include - -# ifndef OPENSSL_NO_MD2 -# include -# ifdef __cplusplus -extern "C" { -# endif - -# define MD2_DIGEST_LENGTH 16 - -# if !defined(OPENSSL_NO_DEPRECATED_3_0) - -typedef unsigned char MD2_INT; - -# define MD2_BLOCK 16 - -typedef struct MD2state_st { - unsigned int num; - unsigned char data[MD2_BLOCK]; - MD2_INT cksm[MD2_BLOCK]; - MD2_INT state[MD2_BLOCK]; -} MD2_CTX; -# endif -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 const char *MD2_options(void); -OSSL_DEPRECATEDIN_3_0 int MD2_Init(MD2_CTX *c); -OSSL_DEPRECATEDIN_3_0 int MD2_Update(MD2_CTX *c, const unsigned char *data, - size_t len); -OSSL_DEPRECATEDIN_3_0 int MD2_Final(unsigned char *md, MD2_CTX *c); -OSSL_DEPRECATEDIN_3_0 unsigned char *MD2(const unsigned char *d, size_t n, - unsigned char *md); -# endif - -# ifdef __cplusplus -} -# endif -# endif -#endif diff --git a/openssl/include/openssl/md4.h b/openssl/include/openssl/md4.h deleted file mode 100644 index 6c150a6cb..000000000 --- a/openssl/include/openssl/md4.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_MD4_H -# define OPENSSL_MD4_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_MD4_H -# endif - -# include - -# ifndef OPENSSL_NO_MD4 -# include -# include -# ifdef __cplusplus -extern "C" { -# endif - -# define MD4_DIGEST_LENGTH 16 - -# if !defined(OPENSSL_NO_DEPRECATED_3_0) - -/*- - * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - * ! MD4_LONG has to be at least 32 bits wide. ! - * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - */ -# define MD4_LONG unsigned int - -# define MD4_CBLOCK 64 -# define MD4_LBLOCK (MD4_CBLOCK/4) - -typedef struct MD4state_st { - MD4_LONG A, B, C, D; - MD4_LONG Nl, Nh; - MD4_LONG data[MD4_LBLOCK]; - unsigned int num; -} MD4_CTX; -# endif -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 int MD4_Init(MD4_CTX *c); -OSSL_DEPRECATEDIN_3_0 int MD4_Update(MD4_CTX *c, const void *data, size_t len); -OSSL_DEPRECATEDIN_3_0 int MD4_Final(unsigned char *md, MD4_CTX *c); -OSSL_DEPRECATEDIN_3_0 unsigned char *MD4(const unsigned char *d, size_t n, - unsigned char *md); -OSSL_DEPRECATEDIN_3_0 void MD4_Transform(MD4_CTX *c, const unsigned char *b); -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/mdc2.h b/openssl/include/openssl/mdc2.h deleted file mode 100644 index 5a7ee289d..000000000 --- a/openssl/include/openssl/mdc2.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_MDC2_H -# define OPENSSL_MDC2_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_MDC2_H -# endif - -# include - -# ifndef OPENSSL_NO_MDC2 -# include -# include -# ifdef __cplusplus -extern "C" { -# endif - -# define MDC2_DIGEST_LENGTH 16 - -# if !defined(OPENSSL_NO_DEPRECATED_3_0) - -# define MDC2_BLOCK 8 - -typedef struct mdc2_ctx_st { - unsigned int num; - unsigned char data[MDC2_BLOCK]; - DES_cblock h, hh; - unsigned int pad_type; /* either 1 or 2, default 1 */ -} MDC2_CTX; -# endif -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 int MDC2_Init(MDC2_CTX *c); -OSSL_DEPRECATEDIN_3_0 int MDC2_Update(MDC2_CTX *c, const unsigned char *data, - size_t len); -OSSL_DEPRECATEDIN_3_0 int MDC2_Final(unsigned char *md, MDC2_CTX *c); -OSSL_DEPRECATEDIN_3_0 unsigned char *MDC2(const unsigned char *d, size_t n, - unsigned char *md); -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/nizk.h b/openssl/include/openssl/nizk.h new file mode 100644 index 000000000..350ea402b --- /dev/null +++ b/openssl/include/openssl/nizk.h @@ -0,0 +1,124 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_NIZK_H +# define HEADER_NIZK_H + +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# ifndef OPENSSL_NO_NIZK +# ifdef __cplusplus +extern "C" { +# endif + +STACK_OF(EC_POINT); + +typedef struct nizk_pub_param_st NIZK_PUB_PARAM; +typedef struct nizk_witness_st NIZK_WITNESS; +typedef struct nizk_plaintext_knowledge_ctx_st NIZK_PLAINTEXT_KNOWLEDGE_CTX; +typedef struct nizk_plaintext_knowledge_proof_st NIZK_PLAINTEXT_KNOWLEDGE_PROOF; +typedef struct nizk_plaintext_equality_ctx_st NIZK_PLAINTEXT_EQUALITY_CTX; +typedef struct nizk_plaintext_equality_proof_st NIZK_PLAINTEXT_EQUALITY_PROOF; +typedef struct nizk_dlog_knowledge_ctx_st NIZK_DLOG_KNOWLEDGE_CTX; +typedef struct nizk_dlog_knowledge_proof_st NIZK_DLOG_KNOWLEDGE_PROOF; +typedef struct nizk_dlog_equality_ctx_st NIZK_DLOG_EQUALITY_CTX; +typedef struct nizk_dlog_equality_proof_st NIZK_DLOG_EQUALITY_PROOF; + +NIZK_PUB_PARAM *NIZK_PUB_PARAM_new(const EC_GROUP *group, const EC_POINT *G, + const EC_POINT *H); +void NIZK_PUB_PARAM_free(NIZK_PUB_PARAM *pp); +int NIZK_PUB_PARAM_up_ref(NIZK_PUB_PARAM *pp); +int NIZK_PUB_PARAM_down_ref(NIZK_PUB_PARAM *pp); +NIZK_WITNESS *NIZK_WITNESS_new(const NIZK_PUB_PARAM *pp, const BIGNUM *r, + const BIGNUM *v); +void NIZK_WITNESS_free(NIZK_WITNESS *witness); +int NIZK_WITNESS_up_ref(NIZK_WITNESS *witness); +int NIZK_WITNESS_down_ref(NIZK_WITNESS *witness); + +NIZK_PLAINTEXT_KNOWLEDGE_CTX *NIZK_PLAINTEXT_KNOWLEDGE_CTX_new(ZKP_TRANSCRIPT *transcript, + NIZK_PUB_PARAM *pp, + NIZK_WITNESS *witness, + EC_POINT *pk, + EC_ELGAMAL_CIPHERTEXT *ct); +void NIZK_PLAINTEXT_KNOWLEDGE_CTX_free(NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx); +NIZK_PLAINTEXT_KNOWLEDGE_PROOF *NIZK_PLAINTEXT_KNOWLEDGE_PROOF_new(NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx); +void NIZK_PLAINTEXT_KNOWLEDGE_PROOF_free(NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof); +NIZK_PLAINTEXT_KNOWLEDGE_PROOF *NIZK_PLAINTEXT_KNOWLEDGE_PROOF_prove(NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx); +int NIZK_PLAINTEXT_KNOWLEDGE_PROOF_verify(NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx, + NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof); + +NIZK_PLAINTEXT_EQUALITY_CTX *NIZK_PLAINTEXT_EQUALITY_CTX_new(ZKP_TRANSCRIPT *transcript, + NIZK_PUB_PARAM *pp, + NIZK_WITNESS *witness, + STACK_OF(EC_POINT) *pk, + EC_ELGAMAL_MR_CIPHERTEXT *ct); +void NIZK_PLAINTEXT_EQUALITY_CTX_free(NIZK_PLAINTEXT_EQUALITY_CTX *ctx); +NIZK_PLAINTEXT_EQUALITY_PROOF *NIZK_PLAINTEXT_EQUALITY_PROOF_new(NIZK_PLAINTEXT_EQUALITY_CTX *ctx); +void NIZK_PLAINTEXT_EQUALITY_PROOF_free(NIZK_PLAINTEXT_EQUALITY_PROOF *proof); +NIZK_PLAINTEXT_EQUALITY_PROOF *NIZK_PLAINTEXT_EQUALITY_PROOF_prove(NIZK_PLAINTEXT_EQUALITY_CTX *ctx); +int NIZK_PLAINTEXT_EQUALITY_PROOF_verify(NIZK_PLAINTEXT_EQUALITY_CTX *ctx, + NIZK_PLAINTEXT_EQUALITY_PROOF *proof); + +NIZK_DLOG_KNOWLEDGE_CTX *NIZK_DLOG_KNOWLEDGE_CTX_new(ZKP_TRANSCRIPT *transcript, + NIZK_PUB_PARAM *pp, + NIZK_WITNESS *witness); +void NIZK_DLOG_KNOWLEDGE_CTX_free(NIZK_DLOG_KNOWLEDGE_CTX *ctx); +NIZK_DLOG_KNOWLEDGE_PROOF *NIZK_DLOG_KNOWLEDGE_PROOF_new(NIZK_DLOG_KNOWLEDGE_CTX *ctx); +void NIZK_DLOG_KNOWLEDGE_PROOF_free(NIZK_DLOG_KNOWLEDGE_PROOF *proof); +NIZK_DLOG_KNOWLEDGE_PROOF *NIZK_DLOG_KNOWLEDGE_PROOF_prove(NIZK_DLOG_KNOWLEDGE_CTX *ctx); +int NIZK_DLOG_KNOWLEDGE_PROOF_verify(NIZK_DLOG_KNOWLEDGE_CTX *ctx, + NIZK_DLOG_KNOWLEDGE_PROOF *proof); + +NIZK_DLOG_EQUALITY_CTX *NIZK_DLOG_EQUALITY_CTX_new(ZKP_TRANSCRIPT *transcript, + NIZK_PUB_PARAM *pp, + NIZK_WITNESS *witness, + const EC_POINT *G, + const EC_POINT *H); +void NIZK_DLOG_EQUALITY_CTX_free(NIZK_DLOG_EQUALITY_CTX *ctx); +NIZK_DLOG_EQUALITY_PROOF *NIZK_DLOG_EQUALITY_PROOF_new(NIZK_DLOG_EQUALITY_CTX *ctx); +void NIZK_DLOG_EQUALITY_PROOF_free(NIZK_DLOG_EQUALITY_PROOF *proof); +NIZK_DLOG_EQUALITY_PROOF *NIZK_DLOG_EQUALITY_PROOF_prove(NIZK_DLOG_EQUALITY_CTX *ctx); +int NIZK_DLOG_EQUALITY_PROOF_verify(NIZK_DLOG_EQUALITY_CTX *ctx, NIZK_DLOG_EQUALITY_PROOF *proof); + +size_t NIZK_PUB_PARAM_encode(const NIZK_PUB_PARAM *pp, unsigned char *out, size_t size); +NIZK_PUB_PARAM *NIZK_PUB_PARAM_decode(const unsigned char *in, size_t size); +size_t NIZK_WITNESS_encode(const NIZK_WITNESS *witness, unsigned char *out, + size_t size, int flag); +NIZK_WITNESS *NIZK_WITNESS_decode(const unsigned char *in, size_t size, int flag); +size_t NIZK_PLAINTEXT_KNOWLEDGE_PROOF_encode(const NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof, + unsigned char *out, size_t size); +NIZK_PLAINTEXT_KNOWLEDGE_PROOF *NIZK_PLAINTEXT_KNOWLEDGE_PROOF_decode(const unsigned char *in, + size_t size); +size_t NIZK_PLAINTEXT_EQUALITY_PROOF_encode(const NIZK_PLAINTEXT_EQUALITY_PROOF *proof, + unsigned char *out, size_t size); +NIZK_PLAINTEXT_EQUALITY_PROOF *NIZK_PLAINTEXT_EQUALITY_PROOF_decode(const unsigned char *in, + size_t size); +size_t NIZK_DLOG_KNOWLEDGE_PROOF_encode(const NIZK_DLOG_KNOWLEDGE_PROOF *proof, + unsigned char *out, size_t size); +NIZK_DLOG_KNOWLEDGE_PROOF *NIZK_DLOG_KNOWLEDGE_PROOF_decode(const unsigned char *in, + size_t size); +size_t NIZK_DLOG_EQUALITY_PROOF_encode(const NIZK_DLOG_EQUALITY_PROOF *proof, + unsigned char *out, size_t size); +NIZK_DLOG_EQUALITY_PROOF *NIZK_DLOG_EQUALITY_PROOF_decode(const unsigned char *in, + size_t size); + +# ifdef __cplusplus +} +# endif +# endif + +#endif diff --git a/openssl/include/openssl/ntls.h b/openssl/include/openssl/ntls.h new file mode 100644 index 000000000..3dffd5000 --- /dev/null +++ b/openssl/include/openssl/ntls.h @@ -0,0 +1,92 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef OPENSSL_NTLS_H +# define OPENSSL_NTLS_H +# pragma once + +# include +# include + +# ifndef OPENSSL_NO_NTLS +# ifdef __cplusplus +extern "C" { +# endif + +/* NTLS version */ +# define NTLS1_1_VERSION_MAJOR 0x01 +# define NTLS1_1_VERSION_MINOR 0x01 +# define NTLS_VERSION NTLS1_1_VERSION +# define NTLS_VERSION_MAJOR NTLS1_1_VERSION_MAJOR +# define NTLS_VERSOIN_MINOR NTLS1_1_VERSION_MINOR +/* + * This tag is used to replace SSLv3 when use NTLS. + * SSLv3 is not used default, so it always be the min protocal version in test, + * but when add NTLS, the NTLS becomes the min version, and NTLS is commonly use, + * then will cause some problems, so add this tag + */ +# define MIN_VERSION_WITH_NTLS 0x0100 + +/* Compatible with GM/T 0024-2014 cipher suites name */ +# define NTLS_TXT_SM2DHE_WITH_SM4_SM3 "ECDHE-SM2-WITH-SM4-SM3" +# define NTLS_TXT_SM2_WITH_SM4_SM3 "ECC-SM2-WITH-SM4-SM3" + +/* GB/T 38636-2020 TLCP, cipher suites */ +# define NTLS_TXT_ECDHE_SM2_SM4_CBC_SM3 "ECDHE-SM2-SM4-CBC-SM3" +# define NTLS_TXT_ECDHE_SM2_SM4_GCM_SM3 "ECDHE-SM2-SM4-GCM-SM3" +# define NTLS_TXT_ECC_SM2_SM4_CBC_SM3 "ECC-SM2-SM4-CBC-SM3" +# define NTLS_TXT_ECC_SM2_SM4_GCM_SM3 "ECC-SM2-SM4-GCM-SM3" +# define NTLS_TXT_IBSDH_SM9_SM4_CBC_SM3 "IBSDH-SM9-SM4-CBC-SM3" +# define NTLS_TXT_IBSDH_SM9_SM4_GCM_SM3 "IBSDH-SM9-SM4-GCM-SM3" +# define NTLS_TXT_IBC_SM9_SM4_CBC_SM3 "IBC-SM9-SM4-CBC-SM3" +# define NTLS_TXT_IBC_SM9_SM4_GCM_SM3 "IBC-SM9-SM4-GCM-SM3" +# define NTLS_TXT_RSA_SM4_CBC_SM3 "RSA-SM4-CBC-SM3" +# define NTLS_TXT_RSA_SM4_GCM_SM3 "RSA-SM4-GCM-SM3" +# define NTLS_TXT_RSA_SM4_CBC_SHA256 "RSA-SM4-CBC-SHA256" +# define NTLS_TXT_RSA_SM4_GCM_SHA256 "RSA-SM4-GCM-SHA256" + +# define NTLS_GB_ECDHE_SM2_SM4_CBC_SM3 "ECDHE_SM4_CBC_SM3" +# define NTLS_GB_ECDHE_SM2_SM4_GCM_SM3 "ECDHE_SM4_GCM_SM3" +# define NTLS_GB_ECC_SM2_SM4_CBC_SM3 "ECC_SM4_CBC_SM3" +# define NTLS_GB_ECC_SM2_SM4_GCM_SM3 "ECC_SM4_GCM_SM3" +# define NTLS_GB_IBSDH_SM9_SM4_CBC_SM3 "IBSDH_SM4_CBC_SM3" +# define NTLS_GB_IBSDH_SM9_SM4_GCM_SM3 "IBSDH_SM4_GCM_SM3" +# define NTLS_GB_IBC_SM9_SM4_CBC_SM3 "IBC_SM4_CBC_SM3" +# define NTLS_GB_IBC_SM9_SM4_GCM_SM3 "IBC_SM4_GCM_SM3" +# define NTLS_GB_RSA_SM4_CBC_SM3 "RSA_SM4_CBC_SM3" +# define NTLS_GB_RSA_SM4_GCM_SM3 "RSA_SM4_GCM_SM3" +# define NTLS_GB_RSA_SM4_CBC_SHA256 "RSA_SM4_CBC_SHA256" +# define NTLS_GB_RSA_SM4_GCM_SHA256 "RSA_SM4_GCM_SHA256" + +# define NTLS_CK_ECDHE_SM2_SM4_CBC_SM3 0x0300E011 +# define NTLS_CK_ECDHE_SM2_SM4_GCM_SM3 0x0300E051 +# define NTLS_CK_ECC_SM2_SM4_CBC_SM3 0x0300E013 +# define NTLS_CK_ECC_SM2_SM4_GCM_SM3 0x0300E053 +# define NTLS_CK_IBSDH_SM9_SM4_CBC_SM3 0x0300E015 +# define NTLS_CK_IBSDH_SM9_SM4_GCM_SM3 0x0300E055 +# define NTLS_CK_IBC_SM9_SM4_CBC_SM3 0x0300E017 +# define NTLS_CK_IBC_SM9_SM4_GCM_SM3 0x0300E057 +# define NTLS_CK_RSA_SM4_CBC_SM3 0x0300E019 +# define NTLS_CK_RSA_SM4_GCM_SM3 0x0300E059 +# define NTLS_CK_RSA_SM4_CBC_SHA256 0x0300E01C +# define NTLS_CK_RSA_SM4_GCM_SHA256 0x0300E05a + + +# define NTLS_AD_UNSUPPORTED_SITE2SITE 200 +# define NTLS_AD_NO_AREA 201 +# define NTLS_AD_UNSUPPORTED_AREATYPE 202 +# define NTLS_AD_BAD_IBCPARAM 203 +# define NTLS_AD_UNSUPPORTED_IBCPARAM 204 +# define NTLS_AD_IDENTITY_NEED 205 + +# ifdef __cplusplus +} +# endif +# endif +#endif diff --git a/openssl/include/openssl/obj_mac.h b/openssl/include/openssl/obj_mac.h index 1b7d9240a..1b89df7c4 100644 --- a/openssl/include/openssl/obj_mac.h +++ b/openssl/include/openssl/obj_mac.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by crypto/objects/objects.pl * - * Copyright 2000-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2022 The OpenSSL Project Authors. All Rights Reserved. * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at @@ -494,27 +494,6 @@ #define NID_wap_wsg_idm_ecid_wtls12 745 #define OBJ_wap_wsg_idm_ecid_wtls12 OBJ_wap_wsg_idm_ecid,12L -#define SN_cast5_cbc "CAST5-CBC" -#define LN_cast5_cbc "cast5-cbc" -#define NID_cast5_cbc 108 -#define OBJ_cast5_cbc OBJ_ISO_US,113533L,7L,66L,10L - -#define SN_cast5_ecb "CAST5-ECB" -#define LN_cast5_ecb "cast5-ecb" -#define NID_cast5_ecb 109 - -#define SN_cast5_cfb64 "CAST5-CFB" -#define LN_cast5_cfb64 "cast5-cfb" -#define NID_cast5_cfb64 110 - -#define SN_cast5_ofb64 "CAST5-OFB" -#define LN_cast5_ofb64 "cast5-ofb" -#define NID_cast5_ofb64 111 - -#define LN_pbeWithMD5AndCast5_CBC "pbeWithMD5AndCast5CBC" -#define NID_pbeWithMD5AndCast5_CBC 112 -#define OBJ_pbeWithMD5AndCast5_CBC OBJ_ISO_US,113533L,7L,66L,12L - #define SN_id_PasswordBasedMAC "id-PasswordBasedMAC" #define LN_id_PasswordBasedMAC "password based MAC" #define NID_id_PasswordBasedMAC 782 @@ -543,16 +522,6 @@ #define NID_rsaEncryption 6 #define OBJ_rsaEncryption OBJ_pkcs1,1L -#define SN_md2WithRSAEncryption "RSA-MD2" -#define LN_md2WithRSAEncryption "md2WithRSAEncryption" -#define NID_md2WithRSAEncryption 7 -#define OBJ_md2WithRSAEncryption OBJ_pkcs1,2L - -#define SN_md4WithRSAEncryption "RSA-MD4" -#define LN_md4WithRSAEncryption "md4WithRSAEncryption" -#define NID_md4WithRSAEncryption 396 -#define OBJ_md4WithRSAEncryption OBJ_pkcs1,3L - #define SN_md5WithRSAEncryption "RSA-MD5" #define LN_md5WithRSAEncryption "md5WithRSAEncryption" #define NID_md5WithRSAEncryption 8 @@ -625,36 +594,16 @@ #define NID_pkcs5 187 #define OBJ_pkcs5 OBJ_pkcs,5L -#define SN_pbeWithMD2AndDES_CBC "PBE-MD2-DES" -#define LN_pbeWithMD2AndDES_CBC "pbeWithMD2AndDES-CBC" -#define NID_pbeWithMD2AndDES_CBC 9 -#define OBJ_pbeWithMD2AndDES_CBC OBJ_pkcs5,1L - #define SN_pbeWithMD5AndDES_CBC "PBE-MD5-DES" #define LN_pbeWithMD5AndDES_CBC "pbeWithMD5AndDES-CBC" #define NID_pbeWithMD5AndDES_CBC 10 #define OBJ_pbeWithMD5AndDES_CBC OBJ_pkcs5,3L -#define SN_pbeWithMD2AndRC2_CBC "PBE-MD2-RC2-64" -#define LN_pbeWithMD2AndRC2_CBC "pbeWithMD2AndRC2-CBC" -#define NID_pbeWithMD2AndRC2_CBC 168 -#define OBJ_pbeWithMD2AndRC2_CBC OBJ_pkcs5,4L - -#define SN_pbeWithMD5AndRC2_CBC "PBE-MD5-RC2-64" -#define LN_pbeWithMD5AndRC2_CBC "pbeWithMD5AndRC2-CBC" -#define NID_pbeWithMD5AndRC2_CBC 169 -#define OBJ_pbeWithMD5AndRC2_CBC OBJ_pkcs5,6L - #define SN_pbeWithSHA1AndDES_CBC "PBE-SHA1-DES" #define LN_pbeWithSHA1AndDES_CBC "pbeWithSHA1AndDES-CBC" #define NID_pbeWithSHA1AndDES_CBC 170 #define OBJ_pbeWithSHA1AndDES_CBC OBJ_pkcs5,10L -#define SN_pbeWithSHA1AndRC2_CBC "PBE-SHA1-RC2-64" -#define LN_pbeWithSHA1AndRC2_CBC "pbeWithSHA1AndRC2-CBC" -#define NID_pbeWithSHA1AndRC2_CBC 68 -#define OBJ_pbeWithSHA1AndRC2_CBC OBJ_pkcs5,11L - #define LN_id_pbkdf2 "PBKDF2" #define NID_id_pbkdf2 69 #define OBJ_id_pbkdf2 OBJ_pkcs5,12L @@ -886,18 +835,6 @@ #define NID_id_ct_signedChecklist 1247 #define OBJ_id_ct_signedChecklist OBJ_id_smime_ct,48L -#define SN_id_ct_ASPA "id-ct-ASPA" -#define NID_id_ct_ASPA 1250 -#define OBJ_id_ct_ASPA OBJ_id_smime_ct,49L - -#define SN_id_ct_signedTAL "id-ct-signedTAL" -#define NID_id_ct_signedTAL 1284 -#define OBJ_id_ct_signedTAL OBJ_id_smime_ct,50L - -#define SN_id_ct_rpkiSignedPrefixList "id-ct-rpkiSignedPrefixList" -#define NID_id_ct_rpkiSignedPrefixList 1320 -#define OBJ_id_ct_rpkiSignedPrefixList OBJ_id_smime_ct,51L - #define SN_id_smime_aa_receiptRequest "id-smime-aa-receiptRequest" #define NID_id_smime_aa_receiptRequest 212 #define OBJ_id_smime_aa_receiptRequest OBJ_id_smime_aa,1L @@ -1014,38 +951,18 @@ #define NID_id_smime_aa_dvcs_dvc 240 #define OBJ_id_smime_aa_dvcs_dvc OBJ_id_smime_aa,29L -#define SN_id_aa_ets_attrCertificateRefs "id-aa-ets-attrCertificateRefs" -#define NID_id_aa_ets_attrCertificateRefs 1261 -#define OBJ_id_aa_ets_attrCertificateRefs OBJ_id_smime_aa,44L - -#define SN_id_aa_ets_attrRevocationRefs "id-aa-ets-attrRevocationRefs" -#define NID_id_aa_ets_attrRevocationRefs 1262 -#define OBJ_id_aa_ets_attrRevocationRefs OBJ_id_smime_aa,45L - #define SN_id_smime_aa_signingCertificateV2 "id-smime-aa-signingCertificateV2" #define NID_id_smime_aa_signingCertificateV2 1086 #define OBJ_id_smime_aa_signingCertificateV2 OBJ_id_smime_aa,47L -#define SN_id_aa_ets_archiveTimestampV2 "id-aa-ets-archiveTimestampV2" -#define NID_id_aa_ets_archiveTimestampV2 1280 -#define OBJ_id_aa_ets_archiveTimestampV2 OBJ_id_smime_aa,48L - #define SN_id_smime_alg_ESDHwith3DES "id-smime-alg-ESDHwith3DES" #define NID_id_smime_alg_ESDHwith3DES 241 #define OBJ_id_smime_alg_ESDHwith3DES OBJ_id_smime_alg,1L -#define SN_id_smime_alg_ESDHwithRC2 "id-smime-alg-ESDHwithRC2" -#define NID_id_smime_alg_ESDHwithRC2 242 -#define OBJ_id_smime_alg_ESDHwithRC2 OBJ_id_smime_alg,2L - #define SN_id_smime_alg_3DESwrap "id-smime-alg-3DESwrap" #define NID_id_smime_alg_3DESwrap 243 #define OBJ_id_smime_alg_3DESwrap OBJ_id_smime_alg,3L -#define SN_id_smime_alg_RC2wrap "id-smime-alg-RC2wrap" -#define NID_id_smime_alg_RC2wrap 244 -#define OBJ_id_smime_alg_RC2wrap OBJ_id_smime_alg,4L - #define SN_id_smime_alg_ESDH "id-smime-alg-ESDH" #define NID_id_smime_alg_ESDH 245 #define OBJ_id_smime_alg_ESDH OBJ_id_smime_alg,5L @@ -1054,10 +971,6 @@ #define NID_id_smime_alg_CMS3DESwrap 246 #define OBJ_id_smime_alg_CMS3DESwrap OBJ_id_smime_alg,6L -#define SN_id_smime_alg_CMSRC2wrap "id-smime-alg-CMSRC2wrap" -#define NID_id_smime_alg_CMSRC2wrap 247 -#define OBJ_id_smime_alg_CMSRC2wrap OBJ_id_smime_alg,7L - #define SN_id_alg_PWRI_KEK "id-alg-PWRI-KEK" #define NID_id_alg_PWRI_KEK 893 #define OBJ_id_alg_PWRI_KEK OBJ_id_smime_alg,9L @@ -1106,17 +1019,15 @@ #define NID_localKeyID 157 #define OBJ_localKeyID OBJ_pkcs9,21L -#define OBJ_ms_corp 1L,3L,6L,1L,4L,1L,311L - #define SN_ms_csp_name "CSPName" #define LN_ms_csp_name "Microsoft CSP Name" #define NID_ms_csp_name 417 -#define OBJ_ms_csp_name OBJ_ms_corp,17L,1L +#define OBJ_ms_csp_name 1L,3L,6L,1L,4L,1L,311L,17L,1L #define SN_LocalKeySet "LocalKeySet" #define LN_LocalKeySet "Microsoft Local Key set" #define NID_LocalKeySet 856 -#define OBJ_LocalKeySet OBJ_ms_corp,17L,2L +#define OBJ_LocalKeySet 1L,3L,6L,1L,4L,1L,311L,17L,2L #define OBJ_certTypes OBJ_pkcs9,22L @@ -1134,10 +1045,6 @@ #define NID_x509Crl 160 #define OBJ_x509Crl OBJ_crlTypes,1L -#define SN_id_aa_CMSAlgorithmProtection "id-aa-CMSAlgorithmProtection" -#define NID_id_aa_CMSAlgorithmProtection 1263 -#define OBJ_id_aa_CMSAlgorithmProtection OBJ_pkcs9,52L - #define OBJ_pkcs12 OBJ_pkcs,12L #define OBJ_pkcs12_pbeids OBJ_pkcs12,1L @@ -1162,16 +1069,6 @@ #define NID_pbe_WithSHA1And2_Key_TripleDES_CBC 147 #define OBJ_pbe_WithSHA1And2_Key_TripleDES_CBC OBJ_pkcs12_pbeids,4L -#define SN_pbe_WithSHA1And128BitRC2_CBC "PBE-SHA1-RC2-128" -#define LN_pbe_WithSHA1And128BitRC2_CBC "pbeWithSHA1And128BitRC2-CBC" -#define NID_pbe_WithSHA1And128BitRC2_CBC 148 -#define OBJ_pbe_WithSHA1And128BitRC2_CBC OBJ_pkcs12_pbeids,5L - -#define SN_pbe_WithSHA1And40BitRC2_CBC "PBE-SHA1-RC2-40" -#define LN_pbe_WithSHA1And40BitRC2_CBC "pbeWithSHA1And40BitRC2-CBC" -#define NID_pbe_WithSHA1And40BitRC2_CBC 149 -#define OBJ_pbe_WithSHA1And40BitRC2_CBC OBJ_pkcs12_pbeids,6L - #define OBJ_pkcs12_Version1 OBJ_pkcs12,10L #define OBJ_pkcs12_BagIds OBJ_pkcs12_Version1,1L @@ -1200,16 +1097,6 @@ #define NID_safeContentsBag 155 #define OBJ_safeContentsBag OBJ_pkcs12_BagIds,6L -#define SN_md2 "MD2" -#define LN_md2 "md2" -#define NID_md2 3 -#define OBJ_md2 OBJ_rsadsi,2L,2L - -#define SN_md4 "MD4" -#define LN_md4 "md4" -#define NID_md4 257 -#define OBJ_md4 OBJ_rsadsi,2L,4L - #define SN_md5 "MD5" #define LN_md5 "md5" #define NID_md5 4 @@ -1248,9 +1135,24 @@ #define OBJ_SM2_with_SM3 OBJ_sm_scheme,501L #define LN_hmacWithSM3 "hmacWithSM3" -#define NID_hmacWithSM3 1281 +#define NID_hmacWithSM3 1257 #define OBJ_hmacWithSM3 OBJ_sm3,3L,1L +#define SN_zuc "ZUC" +#define LN_zuc "zuc" +#define NID_zuc 1248 +#define OBJ_zuc OBJ_sm_scheme,201L + +#define SN_zuc_128_eea3 "ZUC-128-EEA3" +#define LN_zuc_128_eea3 "zuc-128-eea3" +#define NID_zuc_128_eea3 1249 +#define OBJ_zuc_128_eea3 OBJ_sm_scheme,801L + +#define SN_zuc_128_eia3 "ZUC-128-EIA3" +#define LN_zuc_128_eia3 "zuc-128-eia3" +#define NID_zuc_128_eia3 1255 +#define OBJ_zuc_128_eia3 OBJ_sm_scheme,802L + #define LN_hmacWithSHA224 "hmacWithSHA224" #define NID_hmacWithSHA224 798 #define OBJ_hmacWithSHA224 OBJ_rsadsi,2L,8L @@ -1275,31 +1177,6 @@ #define NID_hmacWithSHA512_256 1194 #define OBJ_hmacWithSHA512_256 OBJ_rsadsi,2L,13L -#define SN_rc2_cbc "RC2-CBC" -#define LN_rc2_cbc "rc2-cbc" -#define NID_rc2_cbc 37 -#define OBJ_rc2_cbc OBJ_rsadsi,3L,2L - -#define SN_rc2_ecb "RC2-ECB" -#define LN_rc2_ecb "rc2-ecb" -#define NID_rc2_ecb 38 - -#define SN_rc2_cfb64 "RC2-CFB" -#define LN_rc2_cfb64 "rc2-cfb" -#define NID_rc2_cfb64 39 - -#define SN_rc2_ofb64 "RC2-OFB" -#define LN_rc2_ofb64 "rc2-ofb" -#define NID_rc2_ofb64 40 - -#define SN_rc2_40_cbc "RC2-40-CBC" -#define LN_rc2_40_cbc "rc2-40-cbc" -#define NID_rc2_40_cbc 98 - -#define SN_rc2_64_cbc "RC2-64-CBC" -#define LN_rc2_64_cbc "rc2-64-cbc" -#define NID_rc2_64_cbc 166 - #define SN_rc4 "RC4" #define LN_rc4 "rc4" #define NID_rc4 5 @@ -1334,96 +1211,42 @@ #define SN_ms_ext_req "msExtReq" #define LN_ms_ext_req "Microsoft Extension Request" #define NID_ms_ext_req 171 -#define OBJ_ms_ext_req OBJ_ms_corp,2L,1L,14L +#define OBJ_ms_ext_req 1L,3L,6L,1L,4L,1L,311L,2L,1L,14L #define SN_ms_code_ind "msCodeInd" #define LN_ms_code_ind "Microsoft Individual Code Signing" #define NID_ms_code_ind 134 -#define OBJ_ms_code_ind OBJ_ms_corp,2L,1L,21L +#define OBJ_ms_code_ind 1L,3L,6L,1L,4L,1L,311L,2L,1L,21L #define SN_ms_code_com "msCodeCom" #define LN_ms_code_com "Microsoft Commercial Code Signing" #define NID_ms_code_com 135 -#define OBJ_ms_code_com OBJ_ms_corp,2L,1L,22L +#define OBJ_ms_code_com 1L,3L,6L,1L,4L,1L,311L,2L,1L,22L #define SN_ms_ctl_sign "msCTLSign" #define LN_ms_ctl_sign "Microsoft Trust List Signing" #define NID_ms_ctl_sign 136 -#define OBJ_ms_ctl_sign OBJ_ms_corp,10L,3L,1L +#define OBJ_ms_ctl_sign 1L,3L,6L,1L,4L,1L,311L,10L,3L,1L #define SN_ms_sgc "msSGC" #define LN_ms_sgc "Microsoft Server Gated Crypto" #define NID_ms_sgc 137 -#define OBJ_ms_sgc OBJ_ms_corp,10L,3L,3L +#define OBJ_ms_sgc 1L,3L,6L,1L,4L,1L,311L,10L,3L,3L #define SN_ms_efs "msEFS" #define LN_ms_efs "Microsoft Encrypted File System" #define NID_ms_efs 138 -#define OBJ_ms_efs OBJ_ms_corp,10L,3L,4L +#define OBJ_ms_efs 1L,3L,6L,1L,4L,1L,311L,10L,3L,4L #define SN_ms_smartcard_login "msSmartcardLogin" #define LN_ms_smartcard_login "Microsoft Smartcard Login" #define NID_ms_smartcard_login 648 -#define OBJ_ms_smartcard_login OBJ_ms_corp,20L,2L,2L +#define OBJ_ms_smartcard_login 1L,3L,6L,1L,4L,1L,311L,20L,2L,2L #define SN_ms_upn "msUPN" #define LN_ms_upn "Microsoft User Principal Name" #define NID_ms_upn 649 -#define OBJ_ms_upn OBJ_ms_corp,20L,2L,3L - -#define SN_ms_ntds_sec_ext "ms-ntds-sec-ext" -#define LN_ms_ntds_sec_ext "Microsoft NTDS CA Extension" -#define NID_ms_ntds_sec_ext 1292 -#define OBJ_ms_ntds_sec_ext OBJ_ms_corp,25L,2L - -#define SN_ms_ntds_obj_sid "ms-ntds-obj-sid" -#define LN_ms_ntds_obj_sid "Microsoft NTDS AD objectSid" -#define NID_ms_ntds_obj_sid 1291 -#define OBJ_ms_ntds_obj_sid OBJ_ms_corp,25L,2L,1L - -#define SN_ms_cert_templ "ms-cert-templ" -#define LN_ms_cert_templ "Microsoft certificate template" -#define NID_ms_cert_templ 1293 -#define OBJ_ms_cert_templ OBJ_ms_corp,21L,7L - -#define SN_ms_app_policies "ms-app-policies" -#define LN_ms_app_policies "Microsoft Application Policies Extension" -#define NID_ms_app_policies 1294 -#define OBJ_ms_app_policies OBJ_ms_corp,21L,10L - -#define SN_idea_cbc "IDEA-CBC" -#define LN_idea_cbc "idea-cbc" -#define NID_idea_cbc 34 -#define OBJ_idea_cbc 1L,3L,6L,1L,4L,1L,188L,7L,1L,1L,2L - -#define SN_idea_ecb "IDEA-ECB" -#define LN_idea_ecb "idea-ecb" -#define NID_idea_ecb 36 - -#define SN_idea_cfb64 "IDEA-CFB" -#define LN_idea_cfb64 "idea-cfb" -#define NID_idea_cfb64 35 - -#define SN_idea_ofb64 "IDEA-OFB" -#define LN_idea_ofb64 "idea-ofb" -#define NID_idea_ofb64 46 - -#define SN_bf_cbc "BF-CBC" -#define LN_bf_cbc "bf-cbc" -#define NID_bf_cbc 91 -#define OBJ_bf_cbc 1L,3L,6L,1L,4L,1L,3029L,1L,2L - -#define SN_bf_ecb "BF-ECB" -#define LN_bf_ecb "bf-ecb" -#define NID_bf_ecb 92 - -#define SN_bf_cfb64 "BF-CFB" -#define LN_bf_cfb64 "bf-cfb" -#define NID_bf_cfb64 93 - -#define SN_bf_ofb64 "BF-OFB" -#define LN_bf_ofb64 "bf-ofb" -#define NID_bf_ofb64 94 +#define OBJ_ms_upn 1L,3L,6L,1L,4L,1L,311L,20L,2L,3L #define SN_id_pkix "PKIX" #define NID_id_pkix 127 @@ -1557,18 +1380,6 @@ #define NID_id_mod_cmp2000 284 #define OBJ_id_mod_cmp2000 OBJ_id_pkix_mod,16L -#define SN_id_mod_cmp2000_02 "id-mod-cmp2000-02" -#define NID_id_mod_cmp2000_02 1251 -#define OBJ_id_mod_cmp2000_02 OBJ_id_pkix_mod,50L - -#define SN_id_mod_cmp2021_88 "id-mod-cmp2021-88" -#define NID_id_mod_cmp2021_88 1252 -#define OBJ_id_mod_cmp2021_88 OBJ_id_pkix_mod,99L - -#define SN_id_mod_cmp2021_02 "id-mod-cmp2021-02" -#define NID_id_mod_cmp2021_02 1253 -#define OBJ_id_mod_cmp2021_02 OBJ_id_pkix_mod,100L - #define SN_info_access "authorityInfoAccess" #define LN_info_access "Authority Information Access" #define NID_info_access 177 @@ -1849,22 +1660,6 @@ #define NID_id_it_certReqTemplate 1225 #define OBJ_id_it_certReqTemplate OBJ_id_it,19L -#define SN_id_it_rootCaCert "id-it-rootCaCert" -#define NID_id_it_rootCaCert 1254 -#define OBJ_id_it_rootCaCert OBJ_id_it,20L - -#define SN_id_it_certProfile "id-it-certProfile" -#define NID_id_it_certProfile 1255 -#define OBJ_id_it_certProfile OBJ_id_it,21L - -#define SN_id_it_crlStatusList "id-it-crlStatusList" -#define NID_id_it_crlStatusList 1256 -#define OBJ_id_it_crlStatusList OBJ_id_it,22L - -#define SN_id_it_crls "id-it-crls" -#define NID_id_it_crls 1257 -#define OBJ_id_it_crls OBJ_id_it,23L - #define SN_id_regCtrl "id-regCtrl" #define NID_id_regCtrl 313 #define OBJ_id_regCtrl OBJ_id_pkip,1L @@ -1897,18 +1692,6 @@ #define NID_id_regCtrl_protocolEncrKey 320 #define OBJ_id_regCtrl_protocolEncrKey OBJ_id_regCtrl,6L -#define SN_id_regCtrl_altCertTemplate "id-regCtrl-altCertTemplate" -#define NID_id_regCtrl_altCertTemplate 1258 -#define OBJ_id_regCtrl_altCertTemplate OBJ_id_regCtrl,7L - -#define SN_id_regCtrl_algId "id-regCtrl-algId" -#define NID_id_regCtrl_algId 1259 -#define OBJ_id_regCtrl_algId OBJ_id_regCtrl,11L - -#define SN_id_regCtrl_rsaKeyLen "id-regCtrl-rsaKeyLen" -#define NID_id_regCtrl_rsaKeyLen 1260 -#define OBJ_id_regCtrl_rsaKeyLen OBJ_id_regCtrl,12L - #define SN_id_regInfo_utf8Pairs "id-regInfo-utf8Pairs" #define NID_id_regInfo_utf8Pairs 321 #define OBJ_id_regInfo_utf8Pairs OBJ_id_regInfo,1L @@ -2322,36 +2105,6 @@ #define NID_sha1WithRSA 115 #define OBJ_sha1WithRSA OBJ_algorithm,29L -#define SN_ripemd160 "RIPEMD160" -#define LN_ripemd160 "ripemd160" -#define NID_ripemd160 117 -#define OBJ_ripemd160 1L,3L,36L,3L,2L,1L - -#define SN_ripemd160WithRSA "RSA-RIPEMD160" -#define LN_ripemd160WithRSA "ripemd160WithRSA" -#define NID_ripemd160WithRSA 119 -#define OBJ_ripemd160WithRSA 1L,3L,36L,3L,3L,1L,2L - -#define SN_blake2bmac "BLAKE2BMAC" -#define LN_blake2bmac "blake2bmac" -#define NID_blake2bmac 1201 -#define OBJ_blake2bmac 1L,3L,6L,1L,4L,1L,1722L,12L,2L,1L - -#define SN_blake2smac "BLAKE2SMAC" -#define LN_blake2smac "blake2smac" -#define NID_blake2smac 1202 -#define OBJ_blake2smac 1L,3L,6L,1L,4L,1L,1722L,12L,2L,2L - -#define SN_blake2b512 "BLAKE2b512" -#define LN_blake2b512 "blake2b512" -#define NID_blake2b512 1056 -#define OBJ_blake2b512 OBJ_blake2bmac,16L - -#define SN_blake2s256 "BLAKE2s256" -#define LN_blake2s256 "blake2s256" -#define NID_blake2s256 1057 -#define OBJ_blake2s256 OBJ_blake2smac,8L - #define SN_sxnet "SXNetID" #define LN_sxnet "Strong Extranet ID" #define NID_sxnet 143 @@ -2624,16 +2377,6 @@ #define NID_rsa 19 #define OBJ_rsa OBJ_X500algorithms,1L,1L -#define SN_mdc2WithRSA "RSA-MDC2" -#define LN_mdc2WithRSA "mdc2WithRSA" -#define NID_mdc2WithRSA 96 -#define OBJ_mdc2WithRSA OBJ_X500algorithms,3L,100L - -#define SN_mdc2 "MDC2" -#define LN_mdc2 "mdc2" -#define NID_mdc2 95 -#define OBJ_mdc2 OBJ_X500algorithms,3L,101L - #define SN_id_ce "id-ce" #define NID_id_ce 81 #define OBJ_id_ce OBJ_X500,29L @@ -2743,56 +2486,16 @@ #define NID_ext_key_usage 126 #define OBJ_ext_key_usage OBJ_id_ce,37L -#define SN_authority_attribute_identifier "authorityAttributeIdentifier" -#define LN_authority_attribute_identifier "X509v3 Authority Attribute Identifier" -#define NID_authority_attribute_identifier 1295 -#define OBJ_authority_attribute_identifier OBJ_id_ce,38L - -#define SN_role_spec_cert_identifier "roleSpecCertIdentifier" -#define LN_role_spec_cert_identifier "X509v3 Role Specification Certificate Identifier" -#define NID_role_spec_cert_identifier 1296 -#define OBJ_role_spec_cert_identifier OBJ_id_ce,39L - -#define SN_basic_att_constraints "basicAttConstraints" -#define LN_basic_att_constraints "X509v3 Basic Attribute Certificate Constraints" -#define NID_basic_att_constraints 1297 -#define OBJ_basic_att_constraints OBJ_id_ce,41L - -#define SN_delegated_name_constraints "delegatedNameConstraints" -#define LN_delegated_name_constraints "X509v3 Delegated Name Constraints" -#define NID_delegated_name_constraints 1298 -#define OBJ_delegated_name_constraints OBJ_id_ce,42L - -#define SN_time_specification "timeSpecification" -#define LN_time_specification "X509v3 Time Specification" -#define NID_time_specification 1299 -#define OBJ_time_specification OBJ_id_ce,43L +#define SN_delegation_usage "delegationUsage" +#define LN_delegation_usage "X509v3 Delegation Usage" +#define NID_delegation_usage 1256 +#define OBJ_delegation_usage 1L,3L,6L,1L,4L,1L,44363L,44L #define SN_freshest_crl "freshestCRL" #define LN_freshest_crl "X509v3 Freshest CRL" #define NID_freshest_crl 857 #define OBJ_freshest_crl OBJ_id_ce,46L -#define SN_attribute_descriptor "attributeDescriptor" -#define LN_attribute_descriptor "X509v3 Attribute Descriptor" -#define NID_attribute_descriptor 1300 -#define OBJ_attribute_descriptor OBJ_id_ce,48L - -#define SN_user_notice "userNotice" -#define LN_user_notice "X509v3 User Notice" -#define NID_user_notice 1301 -#define OBJ_user_notice OBJ_id_ce,49L - -#define SN_soa_identifier "sOAIdentifier" -#define LN_soa_identifier "X509v3 Source of Authority Identifier" -#define NID_soa_identifier 1302 -#define OBJ_soa_identifier OBJ_id_ce,50L - -#define SN_acceptable_cert_policies "acceptableCertPolicies" -#define LN_acceptable_cert_policies "X509v3 Acceptable Certification Policies" -#define NID_acceptable_cert_policies 1303 -#define OBJ_acceptable_cert_policies OBJ_id_ce,52L - #define SN_inhibit_any_policy "inhibitAnyPolicy" #define LN_inhibit_any_policy "X509v3 Inhibit Any Policy" #define NID_inhibit_any_policy 748 @@ -2808,86 +2511,6 @@ #define NID_no_rev_avail 403 #define OBJ_no_rev_avail OBJ_id_ce,56L -#define SN_acceptable_privilege_policies "acceptablePrivPolicies" -#define LN_acceptable_privilege_policies "X509v3 Acceptable Privilege Policies" -#define NID_acceptable_privilege_policies 1304 -#define OBJ_acceptable_privilege_policies OBJ_id_ce,57L - -#define SN_indirect_issuer "indirectIssuer" -#define LN_indirect_issuer "X509v3 Indirect Issuer" -#define NID_indirect_issuer 1305 -#define OBJ_indirect_issuer OBJ_id_ce,61L - -#define SN_no_assertion "noAssertion" -#define LN_no_assertion "X509v3 No Assertion" -#define NID_no_assertion 1306 -#define OBJ_no_assertion OBJ_id_ce,62L - -#define SN_id_aa_issuing_distribution_point "aAissuingDistributionPoint" -#define LN_id_aa_issuing_distribution_point "X509v3 Attribute Authority Issuing Distribution Point" -#define NID_id_aa_issuing_distribution_point 1307 -#define OBJ_id_aa_issuing_distribution_point OBJ_id_ce,63L - -#define SN_issued_on_behalf_of "issuedOnBehalfOf" -#define LN_issued_on_behalf_of "X509v3 Issued On Behalf Of" -#define NID_issued_on_behalf_of 1308 -#define OBJ_issued_on_behalf_of OBJ_id_ce,64L - -#define SN_single_use "singleUse" -#define LN_single_use "X509v3 Single Use" -#define NID_single_use 1309 -#define OBJ_single_use OBJ_id_ce,65L - -#define SN_group_ac "groupAC" -#define LN_group_ac "X509v3 Group Attribute Certificate" -#define NID_group_ac 1310 -#define OBJ_group_ac OBJ_id_ce,66L - -#define SN_allowed_attribute_assignments "allowedAttributeAssignments" -#define LN_allowed_attribute_assignments "X509v3 Allowed Attribute Assignments" -#define NID_allowed_attribute_assignments 1311 -#define OBJ_allowed_attribute_assignments OBJ_id_ce,67L - -#define SN_attribute_mappings "attributeMappings" -#define LN_attribute_mappings "X509v3 Attribute Mappings" -#define NID_attribute_mappings 1312 -#define OBJ_attribute_mappings OBJ_id_ce,68L - -#define SN_holder_name_constraints "holderNameConstraints" -#define LN_holder_name_constraints "X509v3 Holder Name Constraints" -#define NID_holder_name_constraints 1313 -#define OBJ_holder_name_constraints OBJ_id_ce,69L - -#define SN_authorization_validation "authorizationValidation" -#define LN_authorization_validation "X509v3 Authorization Validation" -#define NID_authorization_validation 1314 -#define OBJ_authorization_validation OBJ_id_ce,70L - -#define SN_prot_restrict "protRestrict" -#define LN_prot_restrict "X509v3 Protocol Restriction" -#define NID_prot_restrict 1315 -#define OBJ_prot_restrict OBJ_id_ce,71L - -#define SN_subject_alt_public_key_info "subjectAltPublicKeyInfo" -#define LN_subject_alt_public_key_info "X509v3 Subject Alternative Public Key Info" -#define NID_subject_alt_public_key_info 1316 -#define OBJ_subject_alt_public_key_info OBJ_id_ce,72L - -#define SN_alt_signature_algorithm "altSignatureAlgorithm" -#define LN_alt_signature_algorithm "X509v3 Alternative Signature Algorithm" -#define NID_alt_signature_algorithm 1317 -#define OBJ_alt_signature_algorithm OBJ_id_ce,73L - -#define SN_alt_signature_value "altSignatureValue" -#define LN_alt_signature_value "X509v3 Alternative Signature Value" -#define NID_alt_signature_value 1318 -#define OBJ_alt_signature_value OBJ_id_ce,74L - -#define SN_associated_information "associatedInformation" -#define LN_associated_information "X509v3 Associated Information" -#define NID_associated_information 1319 -#define OBJ_associated_information OBJ_id_ce,75L - #define SN_anyExtendedKeyUsage "anyExtendedKeyUsage" #define LN_anyExtendedKeyUsage "Any Extended Key Usage" #define NID_anyExtendedKeyUsage 910 @@ -3439,70 +3062,6 @@ #define NID_hold_instruction_reject 433 #define OBJ_hold_instruction_reject OBJ_holdInstruction,3L -#define SN_itu_t_identified_organization "itu-t-identified-organization" -#define NID_itu_t_identified_organization 1264 -#define OBJ_itu_t_identified_organization OBJ_itu_t,4L - -#define SN_etsi "etsi" -#define NID_etsi 1265 -#define OBJ_etsi OBJ_itu_t_identified_organization,0L - -#define SN_electronic_signature_standard "electronic-signature-standard" -#define NID_electronic_signature_standard 1266 -#define OBJ_electronic_signature_standard OBJ_etsi,1733L - -#define SN_ess_attributes "ess-attributes" -#define NID_ess_attributes 1267 -#define OBJ_ess_attributes OBJ_electronic_signature_standard,2L - -#define SN_id_aa_ets_mimeType "id-aa-ets-mimeType" -#define NID_id_aa_ets_mimeType 1268 -#define OBJ_id_aa_ets_mimeType OBJ_ess_attributes,1L - -#define SN_id_aa_ets_longTermValidation "id-aa-ets-longTermValidation" -#define NID_id_aa_ets_longTermValidation 1269 -#define OBJ_id_aa_ets_longTermValidation OBJ_ess_attributes,2L - -#define SN_id_aa_ets_SignaturePolicyDocument "id-aa-ets-SignaturePolicyDocument" -#define NID_id_aa_ets_SignaturePolicyDocument 1270 -#define OBJ_id_aa_ets_SignaturePolicyDocument OBJ_ess_attributes,3L - -#define SN_id_aa_ets_archiveTimestampV3 "id-aa-ets-archiveTimestampV3" -#define NID_id_aa_ets_archiveTimestampV3 1271 -#define OBJ_id_aa_ets_archiveTimestampV3 OBJ_ess_attributes,4L - -#define SN_id_aa_ATSHashIndex "id-aa-ATSHashIndex" -#define NID_id_aa_ATSHashIndex 1272 -#define OBJ_id_aa_ATSHashIndex OBJ_ess_attributes,5L - -#define SN_cades "cades" -#define NID_cades 1273 -#define OBJ_cades OBJ_etsi,19122L - -#define SN_cades_attributes "cades-attributes" -#define NID_cades_attributes 1274 -#define OBJ_cades_attributes OBJ_cades,1L - -#define SN_id_aa_ets_signerAttrV2 "id-aa-ets-signerAttrV2" -#define NID_id_aa_ets_signerAttrV2 1275 -#define OBJ_id_aa_ets_signerAttrV2 OBJ_cades_attributes,1L - -#define SN_id_aa_ets_sigPolicyStore "id-aa-ets-sigPolicyStore" -#define NID_id_aa_ets_sigPolicyStore 1276 -#define OBJ_id_aa_ets_sigPolicyStore OBJ_cades_attributes,3L - -#define SN_id_aa_ATSHashIndex_v2 "id-aa-ATSHashIndex-v2" -#define NID_id_aa_ATSHashIndex_v2 1277 -#define OBJ_id_aa_ATSHashIndex_v2 OBJ_cades_attributes,4L - -#define SN_id_aa_ATSHashIndex_v3 "id-aa-ATSHashIndex-v3" -#define NID_id_aa_ATSHashIndex_v3 1278 -#define OBJ_id_aa_ATSHashIndex_v3 OBJ_cades_attributes,5L - -#define SN_signedAssertion "signedAssertion" -#define NID_signedAssertion 1279 -#define OBJ_signedAssertion OBJ_cades_attributes,6L - #define SN_data "data" #define NID_data 434 #define OBJ_data OBJ_itu_t,9L @@ -4351,855 +3910,6 @@ #define LN_ipsec4 "ipsec4" #define NID_ipsec4 750 -#define SN_whirlpool "whirlpool" -#define NID_whirlpool 804 -#define OBJ_whirlpool OBJ_iso,0L,10118L,3L,0L,55L - -#define SN_cryptopro "cryptopro" -#define NID_cryptopro 805 -#define OBJ_cryptopro OBJ_member_body,643L,2L,2L - -#define SN_cryptocom "cryptocom" -#define NID_cryptocom 806 -#define OBJ_cryptocom OBJ_member_body,643L,2L,9L - -#define SN_id_tc26 "id-tc26" -#define NID_id_tc26 974 -#define OBJ_id_tc26 OBJ_member_body,643L,7L,1L - -#define SN_id_GostR3411_94_with_GostR3410_2001 "id-GostR3411-94-with-GostR3410-2001" -#define LN_id_GostR3411_94_with_GostR3410_2001 "GOST R 34.11-94 with GOST R 34.10-2001" -#define NID_id_GostR3411_94_with_GostR3410_2001 807 -#define OBJ_id_GostR3411_94_with_GostR3410_2001 OBJ_cryptopro,3L - -#define SN_id_GostR3411_94_with_GostR3410_94 "id-GostR3411-94-with-GostR3410-94" -#define LN_id_GostR3411_94_with_GostR3410_94 "GOST R 34.11-94 with GOST R 34.10-94" -#define NID_id_GostR3411_94_with_GostR3410_94 808 -#define OBJ_id_GostR3411_94_with_GostR3410_94 OBJ_cryptopro,4L - -#define SN_id_GostR3411_94 "md_gost94" -#define LN_id_GostR3411_94 "GOST R 34.11-94" -#define NID_id_GostR3411_94 809 -#define OBJ_id_GostR3411_94 OBJ_cryptopro,9L - -#define SN_id_HMACGostR3411_94 "id-HMACGostR3411-94" -#define LN_id_HMACGostR3411_94 "HMAC GOST 34.11-94" -#define NID_id_HMACGostR3411_94 810 -#define OBJ_id_HMACGostR3411_94 OBJ_cryptopro,10L - -#define SN_id_GostR3410_2001 "gost2001" -#define LN_id_GostR3410_2001 "GOST R 34.10-2001" -#define NID_id_GostR3410_2001 811 -#define OBJ_id_GostR3410_2001 OBJ_cryptopro,19L - -#define SN_id_GostR3410_94 "gost94" -#define LN_id_GostR3410_94 "GOST R 34.10-94" -#define NID_id_GostR3410_94 812 -#define OBJ_id_GostR3410_94 OBJ_cryptopro,20L - -#define SN_id_Gost28147_89 "gost89" -#define LN_id_Gost28147_89 "GOST 28147-89" -#define NID_id_Gost28147_89 813 -#define OBJ_id_Gost28147_89 OBJ_cryptopro,21L - -#define SN_gost89_cnt "gost89-cnt" -#define NID_gost89_cnt 814 - -#define SN_gost89_cnt_12 "gost89-cnt-12" -#define NID_gost89_cnt_12 975 - -#define SN_gost89_cbc "gost89-cbc" -#define NID_gost89_cbc 1009 - -#define SN_gost89_ecb "gost89-ecb" -#define NID_gost89_ecb 1010 - -#define SN_gost89_ctr "gost89-ctr" -#define NID_gost89_ctr 1011 - -#define SN_id_Gost28147_89_MAC "gost-mac" -#define LN_id_Gost28147_89_MAC "GOST 28147-89 MAC" -#define NID_id_Gost28147_89_MAC 815 -#define OBJ_id_Gost28147_89_MAC OBJ_cryptopro,22L - -#define SN_gost_mac_12 "gost-mac-12" -#define NID_gost_mac_12 976 - -#define SN_id_GostR3411_94_prf "prf-gostr3411-94" -#define LN_id_GostR3411_94_prf "GOST R 34.11-94 PRF" -#define NID_id_GostR3411_94_prf 816 -#define OBJ_id_GostR3411_94_prf OBJ_cryptopro,23L - -#define SN_id_GostR3410_2001DH "id-GostR3410-2001DH" -#define LN_id_GostR3410_2001DH "GOST R 34.10-2001 DH" -#define NID_id_GostR3410_2001DH 817 -#define OBJ_id_GostR3410_2001DH OBJ_cryptopro,98L - -#define SN_id_GostR3410_94DH "id-GostR3410-94DH" -#define LN_id_GostR3410_94DH "GOST R 34.10-94 DH" -#define NID_id_GostR3410_94DH 818 -#define OBJ_id_GostR3410_94DH OBJ_cryptopro,99L - -#define SN_id_Gost28147_89_CryptoPro_KeyMeshing "id-Gost28147-89-CryptoPro-KeyMeshing" -#define NID_id_Gost28147_89_CryptoPro_KeyMeshing 819 -#define OBJ_id_Gost28147_89_CryptoPro_KeyMeshing OBJ_cryptopro,14L,1L - -#define SN_id_Gost28147_89_None_KeyMeshing "id-Gost28147-89-None-KeyMeshing" -#define NID_id_Gost28147_89_None_KeyMeshing 820 -#define OBJ_id_Gost28147_89_None_KeyMeshing OBJ_cryptopro,14L,0L - -#define SN_id_GostR3411_94_TestParamSet "id-GostR3411-94-TestParamSet" -#define NID_id_GostR3411_94_TestParamSet 821 -#define OBJ_id_GostR3411_94_TestParamSet OBJ_cryptopro,30L,0L - -#define SN_id_GostR3411_94_CryptoProParamSet "id-GostR3411-94-CryptoProParamSet" -#define NID_id_GostR3411_94_CryptoProParamSet 822 -#define OBJ_id_GostR3411_94_CryptoProParamSet OBJ_cryptopro,30L,1L - -#define SN_id_Gost28147_89_TestParamSet "id-Gost28147-89-TestParamSet" -#define NID_id_Gost28147_89_TestParamSet 823 -#define OBJ_id_Gost28147_89_TestParamSet OBJ_cryptopro,31L,0L - -#define SN_id_Gost28147_89_CryptoPro_A_ParamSet "id-Gost28147-89-CryptoPro-A-ParamSet" -#define NID_id_Gost28147_89_CryptoPro_A_ParamSet 824 -#define OBJ_id_Gost28147_89_CryptoPro_A_ParamSet OBJ_cryptopro,31L,1L - -#define SN_id_Gost28147_89_CryptoPro_B_ParamSet "id-Gost28147-89-CryptoPro-B-ParamSet" -#define NID_id_Gost28147_89_CryptoPro_B_ParamSet 825 -#define OBJ_id_Gost28147_89_CryptoPro_B_ParamSet OBJ_cryptopro,31L,2L - -#define SN_id_Gost28147_89_CryptoPro_C_ParamSet "id-Gost28147-89-CryptoPro-C-ParamSet" -#define NID_id_Gost28147_89_CryptoPro_C_ParamSet 826 -#define OBJ_id_Gost28147_89_CryptoPro_C_ParamSet OBJ_cryptopro,31L,3L - -#define SN_id_Gost28147_89_CryptoPro_D_ParamSet "id-Gost28147-89-CryptoPro-D-ParamSet" -#define NID_id_Gost28147_89_CryptoPro_D_ParamSet 827 -#define OBJ_id_Gost28147_89_CryptoPro_D_ParamSet OBJ_cryptopro,31L,4L - -#define SN_id_Gost28147_89_CryptoPro_Oscar_1_1_ParamSet "id-Gost28147-89-CryptoPro-Oscar-1-1-ParamSet" -#define NID_id_Gost28147_89_CryptoPro_Oscar_1_1_ParamSet 828 -#define OBJ_id_Gost28147_89_CryptoPro_Oscar_1_1_ParamSet OBJ_cryptopro,31L,5L - -#define SN_id_Gost28147_89_CryptoPro_Oscar_1_0_ParamSet "id-Gost28147-89-CryptoPro-Oscar-1-0-ParamSet" -#define NID_id_Gost28147_89_CryptoPro_Oscar_1_0_ParamSet 829 -#define OBJ_id_Gost28147_89_CryptoPro_Oscar_1_0_ParamSet OBJ_cryptopro,31L,6L - -#define SN_id_Gost28147_89_CryptoPro_RIC_1_ParamSet "id-Gost28147-89-CryptoPro-RIC-1-ParamSet" -#define NID_id_Gost28147_89_CryptoPro_RIC_1_ParamSet 830 -#define OBJ_id_Gost28147_89_CryptoPro_RIC_1_ParamSet OBJ_cryptopro,31L,7L - -#define SN_id_GostR3410_94_TestParamSet "id-GostR3410-94-TestParamSet" -#define NID_id_GostR3410_94_TestParamSet 831 -#define OBJ_id_GostR3410_94_TestParamSet OBJ_cryptopro,32L,0L - -#define SN_id_GostR3410_94_CryptoPro_A_ParamSet "id-GostR3410-94-CryptoPro-A-ParamSet" -#define NID_id_GostR3410_94_CryptoPro_A_ParamSet 832 -#define OBJ_id_GostR3410_94_CryptoPro_A_ParamSet OBJ_cryptopro,32L,2L - -#define SN_id_GostR3410_94_CryptoPro_B_ParamSet "id-GostR3410-94-CryptoPro-B-ParamSet" -#define NID_id_GostR3410_94_CryptoPro_B_ParamSet 833 -#define OBJ_id_GostR3410_94_CryptoPro_B_ParamSet OBJ_cryptopro,32L,3L - -#define SN_id_GostR3410_94_CryptoPro_C_ParamSet "id-GostR3410-94-CryptoPro-C-ParamSet" -#define NID_id_GostR3410_94_CryptoPro_C_ParamSet 834 -#define OBJ_id_GostR3410_94_CryptoPro_C_ParamSet OBJ_cryptopro,32L,4L - -#define SN_id_GostR3410_94_CryptoPro_D_ParamSet "id-GostR3410-94-CryptoPro-D-ParamSet" -#define NID_id_GostR3410_94_CryptoPro_D_ParamSet 835 -#define OBJ_id_GostR3410_94_CryptoPro_D_ParamSet OBJ_cryptopro,32L,5L - -#define SN_id_GostR3410_94_CryptoPro_XchA_ParamSet "id-GostR3410-94-CryptoPro-XchA-ParamSet" -#define NID_id_GostR3410_94_CryptoPro_XchA_ParamSet 836 -#define OBJ_id_GostR3410_94_CryptoPro_XchA_ParamSet OBJ_cryptopro,33L,1L - -#define SN_id_GostR3410_94_CryptoPro_XchB_ParamSet "id-GostR3410-94-CryptoPro-XchB-ParamSet" -#define NID_id_GostR3410_94_CryptoPro_XchB_ParamSet 837 -#define OBJ_id_GostR3410_94_CryptoPro_XchB_ParamSet OBJ_cryptopro,33L,2L - -#define SN_id_GostR3410_94_CryptoPro_XchC_ParamSet "id-GostR3410-94-CryptoPro-XchC-ParamSet" -#define NID_id_GostR3410_94_CryptoPro_XchC_ParamSet 838 -#define OBJ_id_GostR3410_94_CryptoPro_XchC_ParamSet OBJ_cryptopro,33L,3L - -#define SN_id_GostR3410_2001_TestParamSet "id-GostR3410-2001-TestParamSet" -#define NID_id_GostR3410_2001_TestParamSet 839 -#define OBJ_id_GostR3410_2001_TestParamSet OBJ_cryptopro,35L,0L - -#define SN_id_GostR3410_2001_CryptoPro_A_ParamSet "id-GostR3410-2001-CryptoPro-A-ParamSet" -#define NID_id_GostR3410_2001_CryptoPro_A_ParamSet 840 -#define OBJ_id_GostR3410_2001_CryptoPro_A_ParamSet OBJ_cryptopro,35L,1L - -#define SN_id_GostR3410_2001_CryptoPro_B_ParamSet "id-GostR3410-2001-CryptoPro-B-ParamSet" -#define NID_id_GostR3410_2001_CryptoPro_B_ParamSet 841 -#define OBJ_id_GostR3410_2001_CryptoPro_B_ParamSet OBJ_cryptopro,35L,2L - -#define SN_id_GostR3410_2001_CryptoPro_C_ParamSet "id-GostR3410-2001-CryptoPro-C-ParamSet" -#define NID_id_GostR3410_2001_CryptoPro_C_ParamSet 842 -#define OBJ_id_GostR3410_2001_CryptoPro_C_ParamSet OBJ_cryptopro,35L,3L - -#define SN_id_GostR3410_2001_CryptoPro_XchA_ParamSet "id-GostR3410-2001-CryptoPro-XchA-ParamSet" -#define NID_id_GostR3410_2001_CryptoPro_XchA_ParamSet 843 -#define OBJ_id_GostR3410_2001_CryptoPro_XchA_ParamSet OBJ_cryptopro,36L,0L - -#define SN_id_GostR3410_2001_CryptoPro_XchB_ParamSet "id-GostR3410-2001-CryptoPro-XchB-ParamSet" -#define NID_id_GostR3410_2001_CryptoPro_XchB_ParamSet 844 -#define OBJ_id_GostR3410_2001_CryptoPro_XchB_ParamSet OBJ_cryptopro,36L,1L - -#define SN_id_GostR3410_94_a "id-GostR3410-94-a" -#define NID_id_GostR3410_94_a 845 -#define OBJ_id_GostR3410_94_a OBJ_id_GostR3410_94,1L - -#define SN_id_GostR3410_94_aBis "id-GostR3410-94-aBis" -#define NID_id_GostR3410_94_aBis 846 -#define OBJ_id_GostR3410_94_aBis OBJ_id_GostR3410_94,2L - -#define SN_id_GostR3410_94_b "id-GostR3410-94-b" -#define NID_id_GostR3410_94_b 847 -#define OBJ_id_GostR3410_94_b OBJ_id_GostR3410_94,3L - -#define SN_id_GostR3410_94_bBis "id-GostR3410-94-bBis" -#define NID_id_GostR3410_94_bBis 848 -#define OBJ_id_GostR3410_94_bBis OBJ_id_GostR3410_94,4L - -#define SN_id_Gost28147_89_cc "id-Gost28147-89-cc" -#define LN_id_Gost28147_89_cc "GOST 28147-89 Cryptocom ParamSet" -#define NID_id_Gost28147_89_cc 849 -#define OBJ_id_Gost28147_89_cc OBJ_cryptocom,1L,6L,1L - -#define SN_id_GostR3410_94_cc "gost94cc" -#define LN_id_GostR3410_94_cc "GOST 34.10-94 Cryptocom" -#define NID_id_GostR3410_94_cc 850 -#define OBJ_id_GostR3410_94_cc OBJ_cryptocom,1L,5L,3L - -#define SN_id_GostR3410_2001_cc "gost2001cc" -#define LN_id_GostR3410_2001_cc "GOST 34.10-2001 Cryptocom" -#define NID_id_GostR3410_2001_cc 851 -#define OBJ_id_GostR3410_2001_cc OBJ_cryptocom,1L,5L,4L - -#define SN_id_GostR3411_94_with_GostR3410_94_cc "id-GostR3411-94-with-GostR3410-94-cc" -#define LN_id_GostR3411_94_with_GostR3410_94_cc "GOST R 34.11-94 with GOST R 34.10-94 Cryptocom" -#define NID_id_GostR3411_94_with_GostR3410_94_cc 852 -#define OBJ_id_GostR3411_94_with_GostR3410_94_cc OBJ_cryptocom,1L,3L,3L - -#define SN_id_GostR3411_94_with_GostR3410_2001_cc "id-GostR3411-94-with-GostR3410-2001-cc" -#define LN_id_GostR3411_94_with_GostR3410_2001_cc "GOST R 34.11-94 with GOST R 34.10-2001 Cryptocom" -#define NID_id_GostR3411_94_with_GostR3410_2001_cc 853 -#define OBJ_id_GostR3411_94_with_GostR3410_2001_cc OBJ_cryptocom,1L,3L,4L - -#define SN_id_GostR3410_2001_ParamSet_cc "id-GostR3410-2001-ParamSet-cc" -#define LN_id_GostR3410_2001_ParamSet_cc "GOST R 3410-2001 Parameter Set Cryptocom" -#define NID_id_GostR3410_2001_ParamSet_cc 854 -#define OBJ_id_GostR3410_2001_ParamSet_cc OBJ_cryptocom,1L,8L,1L - -#define SN_id_tc26_algorithms "id-tc26-algorithms" -#define NID_id_tc26_algorithms 977 -#define OBJ_id_tc26_algorithms OBJ_id_tc26,1L - -#define SN_id_tc26_sign "id-tc26-sign" -#define NID_id_tc26_sign 978 -#define OBJ_id_tc26_sign OBJ_id_tc26_algorithms,1L - -#define SN_id_GostR3410_2012_256 "gost2012_256" -#define LN_id_GostR3410_2012_256 "GOST R 34.10-2012 with 256 bit modulus" -#define NID_id_GostR3410_2012_256 979 -#define OBJ_id_GostR3410_2012_256 OBJ_id_tc26_sign,1L - -#define SN_id_GostR3410_2012_512 "gost2012_512" -#define LN_id_GostR3410_2012_512 "GOST R 34.10-2012 with 512 bit modulus" -#define NID_id_GostR3410_2012_512 980 -#define OBJ_id_GostR3410_2012_512 OBJ_id_tc26_sign,2L - -#define SN_id_tc26_digest "id-tc26-digest" -#define NID_id_tc26_digest 981 -#define OBJ_id_tc26_digest OBJ_id_tc26_algorithms,2L - -#define SN_id_GostR3411_2012_256 "md_gost12_256" -#define LN_id_GostR3411_2012_256 "GOST R 34.11-2012 with 256 bit hash" -#define NID_id_GostR3411_2012_256 982 -#define OBJ_id_GostR3411_2012_256 OBJ_id_tc26_digest,2L - -#define SN_id_GostR3411_2012_512 "md_gost12_512" -#define LN_id_GostR3411_2012_512 "GOST R 34.11-2012 with 512 bit hash" -#define NID_id_GostR3411_2012_512 983 -#define OBJ_id_GostR3411_2012_512 OBJ_id_tc26_digest,3L - -#define SN_id_tc26_signwithdigest "id-tc26-signwithdigest" -#define NID_id_tc26_signwithdigest 984 -#define OBJ_id_tc26_signwithdigest OBJ_id_tc26_algorithms,3L - -#define SN_id_tc26_signwithdigest_gost3410_2012_256 "id-tc26-signwithdigest-gost3410-2012-256" -#define LN_id_tc26_signwithdigest_gost3410_2012_256 "GOST R 34.10-2012 with GOST R 34.11-2012 (256 bit)" -#define NID_id_tc26_signwithdigest_gost3410_2012_256 985 -#define OBJ_id_tc26_signwithdigest_gost3410_2012_256 OBJ_id_tc26_signwithdigest,2L - -#define SN_id_tc26_signwithdigest_gost3410_2012_512 "id-tc26-signwithdigest-gost3410-2012-512" -#define LN_id_tc26_signwithdigest_gost3410_2012_512 "GOST R 34.10-2012 with GOST R 34.11-2012 (512 bit)" -#define NID_id_tc26_signwithdigest_gost3410_2012_512 986 -#define OBJ_id_tc26_signwithdigest_gost3410_2012_512 OBJ_id_tc26_signwithdigest,3L - -#define SN_id_tc26_mac "id-tc26-mac" -#define NID_id_tc26_mac 987 -#define OBJ_id_tc26_mac OBJ_id_tc26_algorithms,4L - -#define SN_id_tc26_hmac_gost_3411_2012_256 "id-tc26-hmac-gost-3411-2012-256" -#define LN_id_tc26_hmac_gost_3411_2012_256 "HMAC GOST 34.11-2012 256 bit" -#define NID_id_tc26_hmac_gost_3411_2012_256 988 -#define OBJ_id_tc26_hmac_gost_3411_2012_256 OBJ_id_tc26_mac,1L - -#define SN_id_tc26_hmac_gost_3411_2012_512 "id-tc26-hmac-gost-3411-2012-512" -#define LN_id_tc26_hmac_gost_3411_2012_512 "HMAC GOST 34.11-2012 512 bit" -#define NID_id_tc26_hmac_gost_3411_2012_512 989 -#define OBJ_id_tc26_hmac_gost_3411_2012_512 OBJ_id_tc26_mac,2L - -#define SN_id_tc26_cipher "id-tc26-cipher" -#define NID_id_tc26_cipher 990 -#define OBJ_id_tc26_cipher OBJ_id_tc26_algorithms,5L - -#define SN_id_tc26_cipher_gostr3412_2015_magma "id-tc26-cipher-gostr3412-2015-magma" -#define NID_id_tc26_cipher_gostr3412_2015_magma 1173 -#define OBJ_id_tc26_cipher_gostr3412_2015_magma OBJ_id_tc26_cipher,1L - -#define SN_magma_ctr_acpkm "magma-ctr-acpkm" -#define NID_magma_ctr_acpkm 1174 -#define OBJ_magma_ctr_acpkm OBJ_id_tc26_cipher_gostr3412_2015_magma,1L - -#define SN_magma_ctr_acpkm_omac "magma-ctr-acpkm-omac" -#define NID_magma_ctr_acpkm_omac 1175 -#define OBJ_magma_ctr_acpkm_omac OBJ_id_tc26_cipher_gostr3412_2015_magma,2L - -#define SN_id_tc26_cipher_gostr3412_2015_kuznyechik "id-tc26-cipher-gostr3412-2015-kuznyechik" -#define NID_id_tc26_cipher_gostr3412_2015_kuznyechik 1176 -#define OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik OBJ_id_tc26_cipher,2L - -#define SN_kuznyechik_ctr_acpkm "kuznyechik-ctr-acpkm" -#define NID_kuznyechik_ctr_acpkm 1177 -#define OBJ_kuznyechik_ctr_acpkm OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik,1L - -#define SN_kuznyechik_ctr_acpkm_omac "kuznyechik-ctr-acpkm-omac" -#define NID_kuznyechik_ctr_acpkm_omac 1178 -#define OBJ_kuznyechik_ctr_acpkm_omac OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik,2L - -#define SN_id_tc26_agreement "id-tc26-agreement" -#define NID_id_tc26_agreement 991 -#define OBJ_id_tc26_agreement OBJ_id_tc26_algorithms,6L - -#define SN_id_tc26_agreement_gost_3410_2012_256 "id-tc26-agreement-gost-3410-2012-256" -#define NID_id_tc26_agreement_gost_3410_2012_256 992 -#define OBJ_id_tc26_agreement_gost_3410_2012_256 OBJ_id_tc26_agreement,1L - -#define SN_id_tc26_agreement_gost_3410_2012_512 "id-tc26-agreement-gost-3410-2012-512" -#define NID_id_tc26_agreement_gost_3410_2012_512 993 -#define OBJ_id_tc26_agreement_gost_3410_2012_512 OBJ_id_tc26_agreement,2L - -#define SN_id_tc26_wrap "id-tc26-wrap" -#define NID_id_tc26_wrap 1179 -#define OBJ_id_tc26_wrap OBJ_id_tc26_algorithms,7L - -#define SN_id_tc26_wrap_gostr3412_2015_magma "id-tc26-wrap-gostr3412-2015-magma" -#define NID_id_tc26_wrap_gostr3412_2015_magma 1180 -#define OBJ_id_tc26_wrap_gostr3412_2015_magma OBJ_id_tc26_wrap,1L - -#define SN_magma_kexp15 "magma-kexp15" -#define NID_magma_kexp15 1181 -#define OBJ_magma_kexp15 OBJ_id_tc26_wrap_gostr3412_2015_magma,1L - -#define SN_id_tc26_wrap_gostr3412_2015_kuznyechik "id-tc26-wrap-gostr3412-2015-kuznyechik" -#define NID_id_tc26_wrap_gostr3412_2015_kuznyechik 1182 -#define OBJ_id_tc26_wrap_gostr3412_2015_kuznyechik OBJ_id_tc26_wrap,2L - -#define SN_kuznyechik_kexp15 "kuznyechik-kexp15" -#define NID_kuznyechik_kexp15 1183 -#define OBJ_kuznyechik_kexp15 OBJ_id_tc26_wrap_gostr3412_2015_kuznyechik,1L - -#define SN_id_tc26_constants "id-tc26-constants" -#define NID_id_tc26_constants 994 -#define OBJ_id_tc26_constants OBJ_id_tc26,2L - -#define SN_id_tc26_sign_constants "id-tc26-sign-constants" -#define NID_id_tc26_sign_constants 995 -#define OBJ_id_tc26_sign_constants OBJ_id_tc26_constants,1L - -#define SN_id_tc26_gost_3410_2012_256_constants "id-tc26-gost-3410-2012-256-constants" -#define NID_id_tc26_gost_3410_2012_256_constants 1147 -#define OBJ_id_tc26_gost_3410_2012_256_constants OBJ_id_tc26_sign_constants,1L - -#define SN_id_tc26_gost_3410_2012_256_paramSetA "id-tc26-gost-3410-2012-256-paramSetA" -#define LN_id_tc26_gost_3410_2012_256_paramSetA "GOST R 34.10-2012 (256 bit) ParamSet A" -#define NID_id_tc26_gost_3410_2012_256_paramSetA 1148 -#define OBJ_id_tc26_gost_3410_2012_256_paramSetA OBJ_id_tc26_gost_3410_2012_256_constants,1L - -#define SN_id_tc26_gost_3410_2012_256_paramSetB "id-tc26-gost-3410-2012-256-paramSetB" -#define LN_id_tc26_gost_3410_2012_256_paramSetB "GOST R 34.10-2012 (256 bit) ParamSet B" -#define NID_id_tc26_gost_3410_2012_256_paramSetB 1184 -#define OBJ_id_tc26_gost_3410_2012_256_paramSetB OBJ_id_tc26_gost_3410_2012_256_constants,2L - -#define SN_id_tc26_gost_3410_2012_256_paramSetC "id-tc26-gost-3410-2012-256-paramSetC" -#define LN_id_tc26_gost_3410_2012_256_paramSetC "GOST R 34.10-2012 (256 bit) ParamSet C" -#define NID_id_tc26_gost_3410_2012_256_paramSetC 1185 -#define OBJ_id_tc26_gost_3410_2012_256_paramSetC OBJ_id_tc26_gost_3410_2012_256_constants,3L - -#define SN_id_tc26_gost_3410_2012_256_paramSetD "id-tc26-gost-3410-2012-256-paramSetD" -#define LN_id_tc26_gost_3410_2012_256_paramSetD "GOST R 34.10-2012 (256 bit) ParamSet D" -#define NID_id_tc26_gost_3410_2012_256_paramSetD 1186 -#define OBJ_id_tc26_gost_3410_2012_256_paramSetD OBJ_id_tc26_gost_3410_2012_256_constants,4L - -#define SN_id_tc26_gost_3410_2012_512_constants "id-tc26-gost-3410-2012-512-constants" -#define NID_id_tc26_gost_3410_2012_512_constants 996 -#define OBJ_id_tc26_gost_3410_2012_512_constants OBJ_id_tc26_sign_constants,2L - -#define SN_id_tc26_gost_3410_2012_512_paramSetTest "id-tc26-gost-3410-2012-512-paramSetTest" -#define LN_id_tc26_gost_3410_2012_512_paramSetTest "GOST R 34.10-2012 (512 bit) testing parameter set" -#define NID_id_tc26_gost_3410_2012_512_paramSetTest 997 -#define OBJ_id_tc26_gost_3410_2012_512_paramSetTest OBJ_id_tc26_gost_3410_2012_512_constants,0L - -#define SN_id_tc26_gost_3410_2012_512_paramSetA "id-tc26-gost-3410-2012-512-paramSetA" -#define LN_id_tc26_gost_3410_2012_512_paramSetA "GOST R 34.10-2012 (512 bit) ParamSet A" -#define NID_id_tc26_gost_3410_2012_512_paramSetA 998 -#define OBJ_id_tc26_gost_3410_2012_512_paramSetA OBJ_id_tc26_gost_3410_2012_512_constants,1L - -#define SN_id_tc26_gost_3410_2012_512_paramSetB "id-tc26-gost-3410-2012-512-paramSetB" -#define LN_id_tc26_gost_3410_2012_512_paramSetB "GOST R 34.10-2012 (512 bit) ParamSet B" -#define NID_id_tc26_gost_3410_2012_512_paramSetB 999 -#define OBJ_id_tc26_gost_3410_2012_512_paramSetB OBJ_id_tc26_gost_3410_2012_512_constants,2L - -#define SN_id_tc26_gost_3410_2012_512_paramSetC "id-tc26-gost-3410-2012-512-paramSetC" -#define LN_id_tc26_gost_3410_2012_512_paramSetC "GOST R 34.10-2012 (512 bit) ParamSet C" -#define NID_id_tc26_gost_3410_2012_512_paramSetC 1149 -#define OBJ_id_tc26_gost_3410_2012_512_paramSetC OBJ_id_tc26_gost_3410_2012_512_constants,3L - -#define SN_id_tc26_digest_constants "id-tc26-digest-constants" -#define NID_id_tc26_digest_constants 1000 -#define OBJ_id_tc26_digest_constants OBJ_id_tc26_constants,2L - -#define SN_id_tc26_cipher_constants "id-tc26-cipher-constants" -#define NID_id_tc26_cipher_constants 1001 -#define OBJ_id_tc26_cipher_constants OBJ_id_tc26_constants,5L - -#define SN_id_tc26_gost_28147_constants "id-tc26-gost-28147-constants" -#define NID_id_tc26_gost_28147_constants 1002 -#define OBJ_id_tc26_gost_28147_constants OBJ_id_tc26_cipher_constants,1L - -#define SN_id_tc26_gost_28147_param_Z "id-tc26-gost-28147-param-Z" -#define LN_id_tc26_gost_28147_param_Z "GOST 28147-89 TC26 parameter set" -#define NID_id_tc26_gost_28147_param_Z 1003 -#define OBJ_id_tc26_gost_28147_param_Z OBJ_id_tc26_gost_28147_constants,1L - -#define SN_INN "INN" -#define LN_INN "INN" -#define NID_INN 1004 -#define OBJ_INN OBJ_member_body,643L,3L,131L,1L,1L - -#define SN_OGRN "OGRN" -#define LN_OGRN "OGRN" -#define NID_OGRN 1005 -#define OBJ_OGRN OBJ_member_body,643L,100L,1L - -#define SN_SNILS "SNILS" -#define LN_SNILS "SNILS" -#define NID_SNILS 1006 -#define OBJ_SNILS OBJ_member_body,643L,100L,3L - -#define SN_OGRNIP "OGRNIP" -#define LN_OGRNIP "OGRNIP" -#define NID_OGRNIP 1226 -#define OBJ_OGRNIP OBJ_member_body,643L,100L,5L - -#define SN_subjectSignTool "subjectSignTool" -#define LN_subjectSignTool "Signing Tool of Subject" -#define NID_subjectSignTool 1007 -#define OBJ_subjectSignTool OBJ_member_body,643L,100L,111L - -#define SN_issuerSignTool "issuerSignTool" -#define LN_issuerSignTool "Signing Tool of Issuer" -#define NID_issuerSignTool 1008 -#define OBJ_issuerSignTool OBJ_member_body,643L,100L,112L - -#define SN_classSignTool "classSignTool" -#define LN_classSignTool "Class of Signing Tool" -#define NID_classSignTool 1227 -#define OBJ_classSignTool OBJ_member_body,643L,100L,113L - -#define SN_classSignToolKC1 "classSignToolKC1" -#define LN_classSignToolKC1 "Class of Signing Tool KC1" -#define NID_classSignToolKC1 1228 -#define OBJ_classSignToolKC1 OBJ_member_body,643L,100L,113L,1L - -#define SN_classSignToolKC2 "classSignToolKC2" -#define LN_classSignToolKC2 "Class of Signing Tool KC2" -#define NID_classSignToolKC2 1229 -#define OBJ_classSignToolKC2 OBJ_member_body,643L,100L,113L,2L - -#define SN_classSignToolKC3 "classSignToolKC3" -#define LN_classSignToolKC3 "Class of Signing Tool KC3" -#define NID_classSignToolKC3 1230 -#define OBJ_classSignToolKC3 OBJ_member_body,643L,100L,113L,3L - -#define SN_classSignToolKB1 "classSignToolKB1" -#define LN_classSignToolKB1 "Class of Signing Tool KB1" -#define NID_classSignToolKB1 1231 -#define OBJ_classSignToolKB1 OBJ_member_body,643L,100L,113L,4L - -#define SN_classSignToolKB2 "classSignToolKB2" -#define LN_classSignToolKB2 "Class of Signing Tool KB2" -#define NID_classSignToolKB2 1232 -#define OBJ_classSignToolKB2 OBJ_member_body,643L,100L,113L,5L - -#define SN_classSignToolKA1 "classSignToolKA1" -#define LN_classSignToolKA1 "Class of Signing Tool KA1" -#define NID_classSignToolKA1 1233 -#define OBJ_classSignToolKA1 OBJ_member_body,643L,100L,113L,6L - -#define SN_kuznyechik_ecb "kuznyechik-ecb" -#define NID_kuznyechik_ecb 1012 - -#define SN_kuznyechik_ctr "kuznyechik-ctr" -#define NID_kuznyechik_ctr 1013 - -#define SN_kuznyechik_ofb "kuznyechik-ofb" -#define NID_kuznyechik_ofb 1014 - -#define SN_kuznyechik_cbc "kuznyechik-cbc" -#define NID_kuznyechik_cbc 1015 - -#define SN_kuznyechik_cfb "kuznyechik-cfb" -#define NID_kuznyechik_cfb 1016 - -#define SN_kuznyechik_mac "kuznyechik-mac" -#define NID_kuznyechik_mac 1017 - -#define SN_magma_ecb "magma-ecb" -#define NID_magma_ecb 1187 - -#define SN_magma_ctr "magma-ctr" -#define NID_magma_ctr 1188 - -#define SN_magma_ofb "magma-ofb" -#define NID_magma_ofb 1189 - -#define SN_magma_cbc "magma-cbc" -#define NID_magma_cbc 1190 - -#define SN_magma_cfb "magma-cfb" -#define NID_magma_cfb 1191 - -#define SN_magma_mac "magma-mac" -#define NID_magma_mac 1192 - -#define SN_camellia_128_cbc "CAMELLIA-128-CBC" -#define LN_camellia_128_cbc "camellia-128-cbc" -#define NID_camellia_128_cbc 751 -#define OBJ_camellia_128_cbc 1L,2L,392L,200011L,61L,1L,1L,1L,2L - -#define SN_camellia_192_cbc "CAMELLIA-192-CBC" -#define LN_camellia_192_cbc "camellia-192-cbc" -#define NID_camellia_192_cbc 752 -#define OBJ_camellia_192_cbc 1L,2L,392L,200011L,61L,1L,1L,1L,3L - -#define SN_camellia_256_cbc "CAMELLIA-256-CBC" -#define LN_camellia_256_cbc "camellia-256-cbc" -#define NID_camellia_256_cbc 753 -#define OBJ_camellia_256_cbc 1L,2L,392L,200011L,61L,1L,1L,1L,4L - -#define SN_id_camellia128_wrap "id-camellia128-wrap" -#define NID_id_camellia128_wrap 907 -#define OBJ_id_camellia128_wrap 1L,2L,392L,200011L,61L,1L,1L,3L,2L - -#define SN_id_camellia192_wrap "id-camellia192-wrap" -#define NID_id_camellia192_wrap 908 -#define OBJ_id_camellia192_wrap 1L,2L,392L,200011L,61L,1L,1L,3L,3L - -#define SN_id_camellia256_wrap "id-camellia256-wrap" -#define NID_id_camellia256_wrap 909 -#define OBJ_id_camellia256_wrap 1L,2L,392L,200011L,61L,1L,1L,3L,4L - -#define OBJ_ntt_ds 0L,3L,4401L,5L - -#define OBJ_camellia OBJ_ntt_ds,3L,1L,9L - -#define SN_camellia_128_ecb "CAMELLIA-128-ECB" -#define LN_camellia_128_ecb "camellia-128-ecb" -#define NID_camellia_128_ecb 754 -#define OBJ_camellia_128_ecb OBJ_camellia,1L - -#define SN_camellia_128_ofb128 "CAMELLIA-128-OFB" -#define LN_camellia_128_ofb128 "camellia-128-ofb" -#define NID_camellia_128_ofb128 766 -#define OBJ_camellia_128_ofb128 OBJ_camellia,3L - -#define SN_camellia_128_cfb128 "CAMELLIA-128-CFB" -#define LN_camellia_128_cfb128 "camellia-128-cfb" -#define NID_camellia_128_cfb128 757 -#define OBJ_camellia_128_cfb128 OBJ_camellia,4L - -#define SN_camellia_128_gcm "CAMELLIA-128-GCM" -#define LN_camellia_128_gcm "camellia-128-gcm" -#define NID_camellia_128_gcm 961 -#define OBJ_camellia_128_gcm OBJ_camellia,6L - -#define SN_camellia_128_ccm "CAMELLIA-128-CCM" -#define LN_camellia_128_ccm "camellia-128-ccm" -#define NID_camellia_128_ccm 962 -#define OBJ_camellia_128_ccm OBJ_camellia,7L - -#define SN_camellia_128_ctr "CAMELLIA-128-CTR" -#define LN_camellia_128_ctr "camellia-128-ctr" -#define NID_camellia_128_ctr 963 -#define OBJ_camellia_128_ctr OBJ_camellia,9L - -#define SN_camellia_128_cmac "CAMELLIA-128-CMAC" -#define LN_camellia_128_cmac "camellia-128-cmac" -#define NID_camellia_128_cmac 964 -#define OBJ_camellia_128_cmac OBJ_camellia,10L - -#define SN_camellia_192_ecb "CAMELLIA-192-ECB" -#define LN_camellia_192_ecb "camellia-192-ecb" -#define NID_camellia_192_ecb 755 -#define OBJ_camellia_192_ecb OBJ_camellia,21L - -#define SN_camellia_192_ofb128 "CAMELLIA-192-OFB" -#define LN_camellia_192_ofb128 "camellia-192-ofb" -#define NID_camellia_192_ofb128 767 -#define OBJ_camellia_192_ofb128 OBJ_camellia,23L - -#define SN_camellia_192_cfb128 "CAMELLIA-192-CFB" -#define LN_camellia_192_cfb128 "camellia-192-cfb" -#define NID_camellia_192_cfb128 758 -#define OBJ_camellia_192_cfb128 OBJ_camellia,24L - -#define SN_camellia_192_gcm "CAMELLIA-192-GCM" -#define LN_camellia_192_gcm "camellia-192-gcm" -#define NID_camellia_192_gcm 965 -#define OBJ_camellia_192_gcm OBJ_camellia,26L - -#define SN_camellia_192_ccm "CAMELLIA-192-CCM" -#define LN_camellia_192_ccm "camellia-192-ccm" -#define NID_camellia_192_ccm 966 -#define OBJ_camellia_192_ccm OBJ_camellia,27L - -#define SN_camellia_192_ctr "CAMELLIA-192-CTR" -#define LN_camellia_192_ctr "camellia-192-ctr" -#define NID_camellia_192_ctr 967 -#define OBJ_camellia_192_ctr OBJ_camellia,29L - -#define SN_camellia_192_cmac "CAMELLIA-192-CMAC" -#define LN_camellia_192_cmac "camellia-192-cmac" -#define NID_camellia_192_cmac 968 -#define OBJ_camellia_192_cmac OBJ_camellia,30L - -#define SN_camellia_256_ecb "CAMELLIA-256-ECB" -#define LN_camellia_256_ecb "camellia-256-ecb" -#define NID_camellia_256_ecb 756 -#define OBJ_camellia_256_ecb OBJ_camellia,41L - -#define SN_camellia_256_ofb128 "CAMELLIA-256-OFB" -#define LN_camellia_256_ofb128 "camellia-256-ofb" -#define NID_camellia_256_ofb128 768 -#define OBJ_camellia_256_ofb128 OBJ_camellia,43L - -#define SN_camellia_256_cfb128 "CAMELLIA-256-CFB" -#define LN_camellia_256_cfb128 "camellia-256-cfb" -#define NID_camellia_256_cfb128 759 -#define OBJ_camellia_256_cfb128 OBJ_camellia,44L - -#define SN_camellia_256_gcm "CAMELLIA-256-GCM" -#define LN_camellia_256_gcm "camellia-256-gcm" -#define NID_camellia_256_gcm 969 -#define OBJ_camellia_256_gcm OBJ_camellia,46L - -#define SN_camellia_256_ccm "CAMELLIA-256-CCM" -#define LN_camellia_256_ccm "camellia-256-ccm" -#define NID_camellia_256_ccm 970 -#define OBJ_camellia_256_ccm OBJ_camellia,47L - -#define SN_camellia_256_ctr "CAMELLIA-256-CTR" -#define LN_camellia_256_ctr "camellia-256-ctr" -#define NID_camellia_256_ctr 971 -#define OBJ_camellia_256_ctr OBJ_camellia,49L - -#define SN_camellia_256_cmac "CAMELLIA-256-CMAC" -#define LN_camellia_256_cmac "camellia-256-cmac" -#define NID_camellia_256_cmac 972 -#define OBJ_camellia_256_cmac OBJ_camellia,50L - -#define SN_camellia_128_cfb1 "CAMELLIA-128-CFB1" -#define LN_camellia_128_cfb1 "camellia-128-cfb1" -#define NID_camellia_128_cfb1 760 - -#define SN_camellia_192_cfb1 "CAMELLIA-192-CFB1" -#define LN_camellia_192_cfb1 "camellia-192-cfb1" -#define NID_camellia_192_cfb1 761 - -#define SN_camellia_256_cfb1 "CAMELLIA-256-CFB1" -#define LN_camellia_256_cfb1 "camellia-256-cfb1" -#define NID_camellia_256_cfb1 762 - -#define SN_camellia_128_cfb8 "CAMELLIA-128-CFB8" -#define LN_camellia_128_cfb8 "camellia-128-cfb8" -#define NID_camellia_128_cfb8 763 - -#define SN_camellia_192_cfb8 "CAMELLIA-192-CFB8" -#define LN_camellia_192_cfb8 "camellia-192-cfb8" -#define NID_camellia_192_cfb8 764 - -#define SN_camellia_256_cfb8 "CAMELLIA-256-CFB8" -#define LN_camellia_256_cfb8 "camellia-256-cfb8" -#define NID_camellia_256_cfb8 765 - -#define OBJ_aria 1L,2L,410L,200046L,1L,1L - -#define SN_aria_128_ecb "ARIA-128-ECB" -#define LN_aria_128_ecb "aria-128-ecb" -#define NID_aria_128_ecb 1065 -#define OBJ_aria_128_ecb OBJ_aria,1L - -#define SN_aria_128_cbc "ARIA-128-CBC" -#define LN_aria_128_cbc "aria-128-cbc" -#define NID_aria_128_cbc 1066 -#define OBJ_aria_128_cbc OBJ_aria,2L - -#define SN_aria_128_cfb128 "ARIA-128-CFB" -#define LN_aria_128_cfb128 "aria-128-cfb" -#define NID_aria_128_cfb128 1067 -#define OBJ_aria_128_cfb128 OBJ_aria,3L - -#define SN_aria_128_ofb128 "ARIA-128-OFB" -#define LN_aria_128_ofb128 "aria-128-ofb" -#define NID_aria_128_ofb128 1068 -#define OBJ_aria_128_ofb128 OBJ_aria,4L - -#define SN_aria_128_ctr "ARIA-128-CTR" -#define LN_aria_128_ctr "aria-128-ctr" -#define NID_aria_128_ctr 1069 -#define OBJ_aria_128_ctr OBJ_aria,5L - -#define SN_aria_192_ecb "ARIA-192-ECB" -#define LN_aria_192_ecb "aria-192-ecb" -#define NID_aria_192_ecb 1070 -#define OBJ_aria_192_ecb OBJ_aria,6L - -#define SN_aria_192_cbc "ARIA-192-CBC" -#define LN_aria_192_cbc "aria-192-cbc" -#define NID_aria_192_cbc 1071 -#define OBJ_aria_192_cbc OBJ_aria,7L - -#define SN_aria_192_cfb128 "ARIA-192-CFB" -#define LN_aria_192_cfb128 "aria-192-cfb" -#define NID_aria_192_cfb128 1072 -#define OBJ_aria_192_cfb128 OBJ_aria,8L - -#define SN_aria_192_ofb128 "ARIA-192-OFB" -#define LN_aria_192_ofb128 "aria-192-ofb" -#define NID_aria_192_ofb128 1073 -#define OBJ_aria_192_ofb128 OBJ_aria,9L - -#define SN_aria_192_ctr "ARIA-192-CTR" -#define LN_aria_192_ctr "aria-192-ctr" -#define NID_aria_192_ctr 1074 -#define OBJ_aria_192_ctr OBJ_aria,10L - -#define SN_aria_256_ecb "ARIA-256-ECB" -#define LN_aria_256_ecb "aria-256-ecb" -#define NID_aria_256_ecb 1075 -#define OBJ_aria_256_ecb OBJ_aria,11L - -#define SN_aria_256_cbc "ARIA-256-CBC" -#define LN_aria_256_cbc "aria-256-cbc" -#define NID_aria_256_cbc 1076 -#define OBJ_aria_256_cbc OBJ_aria,12L - -#define SN_aria_256_cfb128 "ARIA-256-CFB" -#define LN_aria_256_cfb128 "aria-256-cfb" -#define NID_aria_256_cfb128 1077 -#define OBJ_aria_256_cfb128 OBJ_aria,13L - -#define SN_aria_256_ofb128 "ARIA-256-OFB" -#define LN_aria_256_ofb128 "aria-256-ofb" -#define NID_aria_256_ofb128 1078 -#define OBJ_aria_256_ofb128 OBJ_aria,14L - -#define SN_aria_256_ctr "ARIA-256-CTR" -#define LN_aria_256_ctr "aria-256-ctr" -#define NID_aria_256_ctr 1079 -#define OBJ_aria_256_ctr OBJ_aria,15L - -#define SN_aria_128_cfb1 "ARIA-128-CFB1" -#define LN_aria_128_cfb1 "aria-128-cfb1" -#define NID_aria_128_cfb1 1080 - -#define SN_aria_192_cfb1 "ARIA-192-CFB1" -#define LN_aria_192_cfb1 "aria-192-cfb1" -#define NID_aria_192_cfb1 1081 - -#define SN_aria_256_cfb1 "ARIA-256-CFB1" -#define LN_aria_256_cfb1 "aria-256-cfb1" -#define NID_aria_256_cfb1 1082 - -#define SN_aria_128_cfb8 "ARIA-128-CFB8" -#define LN_aria_128_cfb8 "aria-128-cfb8" -#define NID_aria_128_cfb8 1083 - -#define SN_aria_192_cfb8 "ARIA-192-CFB8" -#define LN_aria_192_cfb8 "aria-192-cfb8" -#define NID_aria_192_cfb8 1084 - -#define SN_aria_256_cfb8 "ARIA-256-CFB8" -#define LN_aria_256_cfb8 "aria-256-cfb8" -#define NID_aria_256_cfb8 1085 - -#define SN_aria_128_ccm "ARIA-128-CCM" -#define LN_aria_128_ccm "aria-128-ccm" -#define NID_aria_128_ccm 1120 -#define OBJ_aria_128_ccm OBJ_aria,37L - -#define SN_aria_192_ccm "ARIA-192-CCM" -#define LN_aria_192_ccm "aria-192-ccm" -#define NID_aria_192_ccm 1121 -#define OBJ_aria_192_ccm OBJ_aria,38L - -#define SN_aria_256_ccm "ARIA-256-CCM" -#define LN_aria_256_ccm "aria-256-ccm" -#define NID_aria_256_ccm 1122 -#define OBJ_aria_256_ccm OBJ_aria,39L - -#define SN_aria_128_gcm "ARIA-128-GCM" -#define LN_aria_128_gcm "aria-128-gcm" -#define NID_aria_128_gcm 1123 -#define OBJ_aria_128_gcm OBJ_aria,34L - -#define SN_aria_192_gcm "ARIA-192-GCM" -#define LN_aria_192_gcm "aria-192-gcm" -#define NID_aria_192_gcm 1124 -#define OBJ_aria_192_gcm OBJ_aria,35L - -#define SN_aria_256_gcm "ARIA-256-GCM" -#define LN_aria_256_gcm "aria-256-gcm" -#define NID_aria_256_gcm 1125 -#define OBJ_aria_256_gcm OBJ_aria,36L - -#define SN_kisa "KISA" -#define LN_kisa "kisa" -#define NID_kisa 773 -#define OBJ_kisa OBJ_member_body,410L,200004L - -#define SN_seed_ecb "SEED-ECB" -#define LN_seed_ecb "seed-ecb" -#define NID_seed_ecb 776 -#define OBJ_seed_ecb OBJ_kisa,1L,3L - -#define SN_seed_cbc "SEED-CBC" -#define LN_seed_cbc "seed-cbc" -#define NID_seed_cbc 777 -#define OBJ_seed_cbc OBJ_kisa,1L,4L - -#define SN_seed_cfb128 "SEED-CFB" -#define LN_seed_cfb128 "seed-cfb" -#define NID_seed_cfb128 779 -#define OBJ_seed_cfb128 OBJ_kisa,1L,5L - -#define SN_seed_ofb128 "SEED-OFB" -#define LN_seed_ofb128 "seed-ofb" -#define NID_seed_ofb128 778 -#define OBJ_seed_ofb128 OBJ_kisa,1L,6L - #define SN_sm4_ecb "SM4-ECB" #define LN_sm4_ecb "sm4-ecb" #define NID_sm4_ecb 1133 @@ -5237,19 +3947,14 @@ #define SN_sm4_gcm "SM4-GCM" #define LN_sm4_gcm "sm4-gcm" -#define NID_sm4_gcm 1248 +#define NID_sm4_gcm 1250 #define OBJ_sm4_gcm OBJ_sm_scheme,104L,8L #define SN_sm4_ccm "SM4-CCM" #define LN_sm4_ccm "sm4-ccm" -#define NID_sm4_ccm 1249 +#define NID_sm4_ccm 1251 #define OBJ_sm4_ccm OBJ_sm_scheme,104L,9L -#define SN_sm4_xts "SM4-XTS" -#define LN_sm4_xts "sm4-xts" -#define NID_sm4_xts 1290 -#define OBJ_sm4_xts OBJ_sm_scheme,104L,10L - #define SN_hmac "HMAC" #define LN_hmac "hmac" #define NID_hmac 855 @@ -5327,9 +4032,6 @@ #define NID_brainpoolP256r1 927 #define OBJ_brainpoolP256r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,7L -#define SN_brainpoolP256r1tls13 "brainpoolP256r1tls13" -#define NID_brainpoolP256r1tls13 1285 - #define SN_brainpoolP256t1 "brainpoolP256t1" #define NID_brainpoolP256t1 928 #define OBJ_brainpoolP256t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,8L @@ -5346,9 +4048,6 @@ #define NID_brainpoolP384r1 931 #define OBJ_brainpoolP384r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,11L -#define SN_brainpoolP384r1tls13 "brainpoolP384r1tls13" -#define NID_brainpoolP384r1tls13 1286 - #define SN_brainpoolP384t1 "brainpoolP384t1" #define NID_brainpoolP384t1 932 #define OBJ_brainpoolP384t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,12L @@ -5357,9 +4056,6 @@ #define NID_brainpoolP512r1 933 #define OBJ_brainpoolP512r1 1L,3L,36L,3L,3L,2L,8L,1L,1L,13L -#define SN_brainpoolP512r1tls13 "brainpoolP512r1tls13" -#define NID_brainpoolP512r1tls13 1287 - #define SN_brainpoolP512t1 "brainpoolP512t1" #define NID_brainpoolP512t1 934 #define OBJ_brainpoolP512t1 1L,3L,36L,3L,3L,2L,8L,1L,1L,14L @@ -5437,17 +4133,17 @@ #define SN_jurisdictionLocalityName "jurisdictionL" #define LN_jurisdictionLocalityName "jurisdictionLocalityName" #define NID_jurisdictionLocalityName 955 -#define OBJ_jurisdictionLocalityName OBJ_ms_corp,60L,2L,1L,1L +#define OBJ_jurisdictionLocalityName 1L,3L,6L,1L,4L,1L,311L,60L,2L,1L,1L #define SN_jurisdictionStateOrProvinceName "jurisdictionST" #define LN_jurisdictionStateOrProvinceName "jurisdictionStateOrProvinceName" #define NID_jurisdictionStateOrProvinceName 956 -#define OBJ_jurisdictionStateOrProvinceName OBJ_ms_corp,60L,2L,1L,2L +#define OBJ_jurisdictionStateOrProvinceName 1L,3L,6L,1L,4L,1L,311L,60L,2L,1L,2L #define SN_jurisdictionCountryName "jurisdictionC" #define LN_jurisdictionCountryName "jurisdictionCountryName" #define NID_jurisdictionCountryName 957 -#define OBJ_jurisdictionCountryName OBJ_ms_corp,60L,2L,1L,3L +#define OBJ_jurisdictionCountryName 1L,3L,6L,1L,4L,1L,311L,60L,2L,1L,3L #define SN_id_scrypt "id-scrypt" #define LN_id_scrypt "scrypt" @@ -5540,18 +4236,18 @@ #define LN_kx_srp "kx-srp" #define NID_kx_srp 1044 -#define SN_kx_gost "KxGOST" -#define LN_kx_gost "kx-gost" -#define NID_kx_gost 1045 - -#define SN_kx_gost18 "KxGOST18" -#define LN_kx_gost18 "kx-gost18" -#define NID_kx_gost18 1218 - #define SN_kx_any "KxANY" #define LN_kx_any "kx-any" #define NID_kx_any 1063 +#define SN_kx_sm2 "KxSM2" +#define LN_kx_sm2 "kx-sm2" +#define NID_kx_sm2 1252 + +#define SN_kx_sm2dhe "KxSM2DHE" +#define LN_kx_sm2dhe "kx-sm2dhe" +#define NID_kx_sm2dhe 1253 + #define SN_auth_rsa "AuthRSA" #define LN_auth_rsa "auth-rsa" #define NID_auth_rsa 1046 @@ -5568,14 +4264,6 @@ #define LN_auth_dss "auth-dss" #define NID_auth_dss 1049 -#define SN_auth_gost01 "AuthGOST01" -#define LN_auth_gost01 "auth-gost01" -#define NID_auth_gost01 1050 - -#define SN_auth_gost12 "AuthGOST12" -#define LN_auth_gost12 "auth-gost12" -#define NID_auth_gost12 1051 - #define SN_auth_srp "AuthSRP" #define LN_auth_srp "auth-srp" #define NID_auth_srp 1052 @@ -5588,6 +4276,10 @@ #define LN_auth_any "auth-any" #define NID_auth_any 1064 +#define SN_auth_sm2 "AuthSM2" +#define LN_auth_sm2 "auth-sm2" +#define NID_auth_sm2 1254 + #define SN_poly1305 "Poly1305" #define LN_poly1305 "poly1305" #define NID_poly1305 1061 @@ -5637,36 +4329,6 @@ #define NID_ua_pki 1151 #define OBJ_ua_pki OBJ_ISO_UA,2L,1L,1L,1L -#define SN_dstu28147 "dstu28147" -#define LN_dstu28147 "DSTU Gost 28147-2009" -#define NID_dstu28147 1152 -#define OBJ_dstu28147 OBJ_ua_pki,1L,1L,1L - -#define SN_dstu28147_ofb "dstu28147-ofb" -#define LN_dstu28147_ofb "DSTU Gost 28147-2009 OFB mode" -#define NID_dstu28147_ofb 1153 -#define OBJ_dstu28147_ofb OBJ_dstu28147,2L - -#define SN_dstu28147_cfb "dstu28147-cfb" -#define LN_dstu28147_cfb "DSTU Gost 28147-2009 CFB mode" -#define NID_dstu28147_cfb 1154 -#define OBJ_dstu28147_cfb OBJ_dstu28147,3L - -#define SN_dstu28147_wrap "dstu28147-wrap" -#define LN_dstu28147_wrap "DSTU Gost 28147-2009 key wrap" -#define NID_dstu28147_wrap 1155 -#define OBJ_dstu28147_wrap OBJ_dstu28147,5L - -#define SN_hmacWithDstu34311 "hmacWithDstu34311" -#define LN_hmacWithDstu34311 "HMAC DSTU Gost 34311-95" -#define NID_hmacWithDstu34311 1156 -#define OBJ_hmacWithDstu34311 OBJ_ua_pki,1L,1L,2L - -#define SN_dstu34311 "dstu34311" -#define LN_dstu34311 "DSTU Gost 34311-95" -#define NID_dstu34311 1157 -#define OBJ_dstu34311 OBJ_ua_pki,1L,2L,1L - #define SN_dstu4145le "dstu4145le" #define LN_dstu4145le "DSTU 4145-2002 little endian" #define NID_dstu4145le 1158 @@ -5739,68 +4401,4 @@ #define LN_aes_256_siv "aes-256-siv" #define NID_aes_256_siv 1200 -#define SN_oracle "oracle-organization" -#define LN_oracle "Oracle organization" -#define NID_oracle 1282 -#define OBJ_oracle OBJ_joint_iso_itu_t,16L,840L,1L,113894L - -#define SN_oracle_jdk_trustedkeyusage "oracle-jdk-trustedkeyusage" -#define LN_oracle_jdk_trustedkeyusage "Trusted key usage (Oracle)" -#define NID_oracle_jdk_trustedkeyusage 1283 -#define OBJ_oracle_jdk_trustedkeyusage OBJ_oracle,746875L,1L,1L - -#define SN_brotli "brotli" -#define LN_brotli "Brotli compression" -#define NID_brotli 1288 - -#define SN_zstd "zstd" -#define LN_zstd "Zstandard compression" -#define NID_zstd 1289 - #endif /* OPENSSL_OBJ_MAC_H */ - -#ifndef OPENSSL_NO_DEPRECATED_3_0 - -#define SN_id_tc26_cipher_gostr3412_2015_magma_ctracpkm SN_magma_ctr_acpkm -#define NID_id_tc26_cipher_gostr3412_2015_magma_ctracpkm NID_magma_ctr_acpkm -#define OBJ_id_tc26_cipher_gostr3412_2015_magma_ctracpkm OBJ_magma_ctr_acpkm - -#define SN_id_tc26_cipher_gostr3412_2015_magma_ctracpkm_omac SN_magma_ctr_acpkm_omac -#define NID_id_tc26_cipher_gostr3412_2015_magma_ctracpkm_omac NID_magma_ctr_acpkm_omac -#define OBJ_id_tc26_cipher_gostr3412_2015_magma_ctracpkm_omac OBJ_magma_ctr_acpkm_omac - -#define SN_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm SN_kuznyechik_ctr_acpkm -#define NID_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm NID_kuznyechik_ctr_acpkm -#define OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm OBJ_kuznyechik_ctr_acpkm - -#define SN_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm_omac SN_kuznyechik_ctr_acpkm_omac -#define NID_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm_omac NID_kuznyechik_ctr_acpkm_omac -#define OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm_omac OBJ_kuznyechik_ctr_acpkm_omac - -#define SN_id_tc26_wrap_gostr3412_2015_magma_kexp15 SN_magma_kexp15 -#define NID_id_tc26_wrap_gostr3412_2015_magma_kexp15 NID_magma_kexp15 -#define OBJ_id_tc26_wrap_gostr3412_2015_magma_kexp15 OBJ_magma_kexp15 - -#define SN_id_tc26_wrap_gostr3412_2015_kuznyechik_kexp15 SN_kuznyechik_kexp15 -#define NID_id_tc26_wrap_gostr3412_2015_kuznyechik_kexp15 NID_kuznyechik_kexp15 -#define OBJ_id_tc26_wrap_gostr3412_2015_kuznyechik_kexp15 OBJ_kuznyechik_kexp15 - -#define SN_grasshopper_ecb SN_kuznyechik_ecb -#define NID_grasshopper_ecb NID_kuznyechik_ecb - -#define SN_grasshopper_ctr SN_kuznyechik_ctr -#define NID_grasshopper_ctr NID_kuznyechik_ctr - -#define SN_grasshopper_ofb SN_kuznyechik_ofb -#define NID_grasshopper_ofb NID_kuznyechik_ofb - -#define SN_grasshopper_cbc SN_kuznyechik_cbc -#define NID_grasshopper_cbc NID_kuznyechik_cbc - -#define SN_grasshopper_cfb SN_kuznyechik_cfb -#define NID_grasshopper_cfb NID_kuznyechik_cfb - -#define SN_grasshopper_mac SN_kuznyechik_mac -#define NID_grasshopper_mac NID_kuznyechik_mac - -#endif /* OPENSSL_NO_DEPRECATED_3_0 */ diff --git a/openssl/include/openssl/opensslconf.h b/openssl/include/openssl/opensslconf.h index 1e83371f1..e8fbb09f9 100644 --- a/openssl/include/openssl/opensslconf.h +++ b/openssl/include/openssl/opensslconf.h @@ -13,5 +13,6 @@ # include # include +# include #endif /* OPENSSL_OPENSSLCONF_H */ diff --git a/openssl/include/openssl/opensslv.h b/openssl/include/openssl/opensslv.h index 274a65ff0..117ce46c9 100644 --- a/openssl/include/openssl/opensslv.h +++ b/openssl/include/openssl/opensslv.h @@ -28,8 +28,12 @@ extern "C" { * These macros express version number MAJOR.MINOR.PATCH exactly */ # define OPENSSL_VERSION_MAJOR 3 -# define OPENSSL_VERSION_MINOR 3 -# define OPENSSL_VERSION_PATCH 1 +# define OPENSSL_VERSION_MINOR 0 +# define OPENSSL_VERSION_PATCH 3 + +# define TONGSUO_VERSION_MAJOR 8 +# define TONGSUO_VERSION_MINOR 4 +# define TONGSUO_VERSION_PATCH 0 /* * Additional version information @@ -44,6 +48,8 @@ extern "C" { /* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+vendor.1" */ # define OPENSSL_VERSION_BUILD_METADATA "" +# define TONGSUO_VERSION_PRE_RELEASE "" + /* * Note: The OpenSSL Project will never define OPENSSL_VERSION_BUILD_METADATA * to be anything but the empty string. Its use is entirely reserved for @@ -74,21 +80,21 @@ extern "C" { * longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and * OPENSSL_VERSION_BUILD_METADATA_STR appended. */ -# define OPENSSL_VERSION_STR "3.3.1" -# define OPENSSL_FULL_VERSION_STR "3.3.1" +# define OPENSSL_VERSION_STR "3.0.3" +# define OPENSSL_FULL_VERSION_STR "3.0.3" /* * SECTION 3: ADDITIONAL METADATA * * These strings are defined separately to allow them to be parsable. */ -# define OPENSSL_RELEASE_DATE "4 Jun 2024" +# define OPENSSL_RELEASE_DATE "3 May 2022" /* * SECTION 4: BACKWARD COMPATIBILITY */ -# define OPENSSL_VERSION_TEXT "OpenSSL 3.3.1 4 Jun 2024" +# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.3 3 May 2022" /* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */ # ifdef OPENSSL_VERSION_PRE_RELEASE @@ -102,6 +108,22 @@ extern "C" { |(OPENSSL_VERSION_PATCH<<4) \ |_OPENSSL_VERSION_PRE_RELEASE ) +/* Tongsuo/BabaSSL stuffs */ +# ifdef TONGSUO_VERSION_PRE_RELEASE +# define _TONGSUO_VERSION_PRE_RELEASE 0x0L +# else +# define _TONGSUO_VERSION_PRE_RELEASE 0xfL +# endif +# define TONGSUO_VERSION_NUMBER \ + ( (TONGSUO_VERSION_MAJOR<<28) \ + |(TONGSUO_VERSION_MINOR<<20) \ + |(TONGSUO_VERSION_PATCH<<4) \ + |_TONGSUO_VERSION_PRE_RELEASE ) + +# define TONGSUO_VERSION_TEXT "Tongsuo 8.4.0" +# define BABASSL_VERSION_NUMBER TONGSUO_VERSION_NUMBER +# define BABASSL_VERSION_TEXT TONGSUO_VERSION_TEXT + # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/opensslv.h.in b/openssl/include/openssl/opensslv.h.in index 3f47a2ac0..8bff0e3ee 100644 --- a/openssl/include/openssl/opensslv.h.in +++ b/openssl/include/openssl/opensslv.h.in @@ -30,6 +30,10 @@ extern "C" { # define OPENSSL_VERSION_MINOR {- $config{minor} -} # define OPENSSL_VERSION_PATCH {- $config{patch} -} +# define TONGSUO_VERSION_MAJOR {- $config{tongsuo_major} -} +# define TONGSUO_VERSION_MINOR {- $config{tongsuo_minor} -} +# define TONGSUO_VERSION_PATCH {- $config{tongsuo_patch} -} + /* * Additional version information * @@ -43,6 +47,8 @@ extern "C" { /* Could be: #define OPENSSL_VERSION_BUILD_METADATA "+vendor.1" */ # define OPENSSL_VERSION_BUILD_METADATA "{- $config{build_metadata} -}" +# define TONGSUO_VERSION_PRE_RELEASE "{- $config{tongsuo_prerelease} -}" + /* * Note: The OpenSSL Project will never define OPENSSL_VERSION_BUILD_METADATA * to be anything but the empty string. Its use is entirely reserved for @@ -101,6 +107,22 @@ extern "C" { |(OPENSSL_VERSION_PATCH<<4) \ |_OPENSSL_VERSION_PRE_RELEASE ) +/* Tongsuo/BabaSSL stuffs */ +# ifdef TONGSUO_VERSION_PRE_RELEASE +# define _TONGSUO_VERSION_PRE_RELEASE 0x0L +# else +# define _TONGSUO_VERSION_PRE_RELEASE 0xfL +# endif +# define TONGSUO_VERSION_NUMBER \ + ( (TONGSUO_VERSION_MAJOR<<28) \ + |(TONGSUO_VERSION_MINOR<<20) \ + |(TONGSUO_VERSION_PATCH<<4) \ + |_TONGSUO_VERSION_PRE_RELEASE ) + +# define TONGSUO_VERSION_TEXT "Tongsuo {- "$config{tongsuo_full_version}" -}" +# define BABASSL_VERSION_NUMBER TONGSUO_VERSION_NUMBER +# define BABASSL_VERSION_TEXT TONGSUO_VERSION_TEXT + # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/paillier.h b/openssl/include/openssl/paillier.h new file mode 100644 index 000000000..8895c3b29 --- /dev/null +++ b/openssl/include/openssl/paillier.h @@ -0,0 +1,237 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_PAILLIER_H +# define HEADER_PAILLIER_H + +# include +# include +# include +# include +# include + +# ifndef OPENSSL_NO_PAILLIER +# ifdef __cplusplus +extern "C" { +# endif + +# ifndef OPENSSL_PAILLIER_MAX_MODULUS_BITS +# define OPENSSL_PAILLIER_MAX_MODULUS_BITS 16384 +# endif + +# define PEM_STRING_PAILLIER_PRIVATE_KEY "PAILLIER PRIVATE KEY" +# define PEM_STRING_PAILLIER_PUBLIC_KEY "PAILLIER PUBLIC KEY" + +# define PAILLIER_ASN1_VERSION_DEFAULT 0 +# define PAILLIER_ASN1_VERSION_MULTI 1 + +# define PAILLIER_FLAG_G_OPTIMIZE 0x01 + +# define PAILLIER_KEY_TYPE_PUBLIC 0 +# define PAILLIER_KEY_TYPE_PRIVATE 1 + +# define PAILLIER_MAX_THRESHOLD ((((uint64_t)1) << 63) - 1) + +typedef struct paillier_key_st PAILLIER_KEY; +typedef struct paillier_ctx_st PAILLIER_CTX; +typedef struct paillier_ciphertext_st PAILLIER_CIPHERTEXT; + +DECLARE_PEM_rw(PAILLIER_PrivateKey, PAILLIER_KEY) +DECLARE_PEM_rw(PAILLIER_PublicKey, PAILLIER_KEY) +DECLARE_ASN1_ENCODE_FUNCTIONS_only(PAILLIER_KEY, PAILLIER_PrivateKey) +DECLARE_ASN1_ENCODE_FUNCTIONS_only(PAILLIER_KEY, PAILLIER_PublicKey) + +/** + * Creates a new PAILLIER_KEY object. + * \return PAILLIER_KEY object or NULL if an error occurred. + */ +PAILLIER_KEY *PAILLIER_KEY_new(void); + +/** Frees a PAILLIER_KEY object. + * \param key PAILLIER_KEY object to be freed. + */ +void PAILLIER_KEY_free(PAILLIER_KEY *key); + +/** Copies a PAILLIER_KEY object. + * \param dst destination PAILLIER_KEY object + * \param src src PAILLIER_KEY object + * \return dst or NULL if an error occurred. + */ +PAILLIER_KEY *PAILLIER_KEY_copy(PAILLIER_KEY *dest, PAILLIER_KEY *src); + +/** Creates a new PAILLIER_KEY object and copies the content from src to it. + * \param src the source PAILLIER_KEY object + * \return newly created PAILLIER_KEY object or NULL if an error occurred. + */ +PAILLIER_KEY *PAILLIER_KEY_dup(PAILLIER_KEY *key); + +/** Increases the internal reference count of a PAILLIER_KEY object. + * \param key PAILLIER_KEY object + * \return 1 on success and 0 if an error occurred. + */ +int PAILLIER_KEY_up_ref(PAILLIER_KEY *key); + +/** Creates a new paillier private (and optional a new public) key. + * \param key PAILLIER_KEY object + * \param bits use BN_generate_prime_ex() to generate a pseudo-random prime number + * of bit length + * \return 1 on success and 0 if an error occurred. + */ +int PAILLIER_KEY_generate_key(PAILLIER_KEY *key, int bits); + +/** Returns the type of the PAILLIER_KEY. + * \param key PAILLIER_KEY object + * \return PAILLIER_KEY_TYPE_PRIVATE or PAILLIER_KEY_TYPE_PUBLIC. + */ +int PAILLIER_KEY_type(PAILLIER_KEY *key); + +/** Encrypts an Integer with additadive homomorphic Paillier + * \param ctx PAILLIER_CTX object. + * \param r PAILLIER_CIPHERTEXT object that stores the result of + * the encryption + * \param m The plaintext integer to be encrypted + * \return 1 on success and 0 otherwise + */ +int PAILLIER_encrypt(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *out, int32_t m); + +/** Decrypts the ciphertext + * \param ctx PAILLIER_CTX object + * \param r The resulting plaintext integer + * \param c PAILLIER_CIPHERTEXT object to be decrypted + * \return 1 on success and 0 otherwise + */ +int PAILLIER_decrypt(PAILLIER_CTX *ctx, int32_t *out, PAILLIER_CIPHERTEXT *c); + +/** Adds two paillier ciphertext and stores it in r: + * E(r) = E(c1 + c2) = E(c1) * E(c2) + * \param ctx PAILLIER_CTX object + * \param r The PAILLIER_CIPHERTEXT object that stores the addition + * result + * \param c1 PAILLIER_CIPHERTEXT object + * \param c2 PAILLIER_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int PAILLIER_add(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + PAILLIER_CIPHERTEXT *c1, PAILLIER_CIPHERTEXT *c2); + +/** Add a paillier ciphertext to a plaintext, and stores it in r: + * E(r) = E(c1 + m) = E(c1) * g^m + * \param ctx PAILLIER_CTX object + * \param r The PAILLIER_CIPHERTEXT object that stores the addition + * result + * \param c1 PAILLIER_CIPHERTEXT object + * \param m The plaintext integer to be added + * \return 1 on success and 0 otherwise + */ +int PAILLIER_add_plain(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + PAILLIER_CIPHERTEXT *c1, int32_t m); + +/** Substracts two paillier ciphertext and stores it in r: + * E(r) = E(c1 - c2) = E(c1) * E(-c2) = E(c1) / E(c2) + * \param ctx PAILLIER_CTX object + * \param r The PAILLIER_CIPHERTEXT object that stores the + * subtraction result + * \param c1 PAILLIER_CIPHERTEXT object + * \param c2 PAILLIER_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int PAILLIER_sub(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + PAILLIER_CIPHERTEXT *c1, PAILLIER_CIPHERTEXT *c2); + +/** Ciphertext multiplication, computes E(r) = E(c * m) = E(c) ^ m + * \param ctx PAILLIER_CTX object + * \param r The PAILLIER_CIPHERTEXT object that stores the + * multiplication result + * \param c1 PAILLIER_CIPHERTEXT object + * \param m The plaintext integer to be multiplied + * \return 1 on success and 0 otherwise + */ +int PAILLIER_mul(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + PAILLIER_CIPHERTEXT *c, int32_t m); + +/** Creates a new PAILLIER object + * \param key PAILLIER_KEY to use + * \param threshold The threshold should be greater than the maximum integer + * that will be encrypted. + * \return newly created PAILLIER_CTX object or NULL in case of an error + */ +PAILLIER_CTX *PAILLIER_CTX_new(PAILLIER_KEY *key, int64_t threshold); + +/** Frees a PAILLIER_CTX object + * \param ctx PAILLIER_CTX object to be freed + */ +void PAILLIER_CTX_free(PAILLIER_CTX *ctx); + +/** Copies a PAILLIER_KEY object. + * \param dst destination PAILLIER_KEY object + * \param src src PAILLIER_KEY object + * \return dst or NULL if an error occurred. + */ +PAILLIER_CTX *PAILLIER_CTX_copy(PAILLIER_CTX *dest, PAILLIER_CTX *src); + +/** Creates a new PAILLIER_KEY object and copies the content from src to it. + * \param src the source PAILLIER_KEY object + * \return newly created PAILLIER_KEY object or NULL if an error occurred. + */ +PAILLIER_CTX *PAILLIER_CTX_dup(PAILLIER_CTX *src); + +#ifndef OPENSSL_NO_ENGINE +/** set ENGINE pointer to the PAILLIER object + * \param ctx PAILLIER_CTX object. + * \param engine ENGINE object to use + * \return 1 on success and 0 otherwise + */ +int PAILLIER_CTX_set_engine(PAILLIER_CTX *ctx, ENGINE *engine); +#endif + +/** Creates a new PAILLIER_CIPHERTEXT object for paillier oparations + * \param ctx PAILLIER_CTX object + * \return newly created PAILLIER_CIPHERTEXT object or NULL in case of an error + */ +PAILLIER_CIPHERTEXT *PAILLIER_CIPHERTEXT_new(PAILLIER_CTX *ctx); + +/** Frees a PAILLIER_CIPHERTEXT object + * \param ciphertext PAILLIER_CIPHERTEXT object to be freed + */ +void PAILLIER_CIPHERTEXT_free(PAILLIER_CIPHERTEXT *ciphertext); + +/** Encodes PAILLIER_CIPHERTEXT to binary + * \param ctx PAILLIER_CTX object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param ciphertext PAILLIER_CIPHERTEXT object + * \param compressed Whether to compress the encoding (either 0 or 1) + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t PAILLIER_CIPHERTEXT_encode(PAILLIER_CTX *ctx, unsigned char *out, + size_t size, + const PAILLIER_CIPHERTEXT *ciphertext, + int flag); + +/** Decodes binary to PAILLIER_CIPHERTEXT + * \param ctx PAILLIER_CTX object + * \param r the resulting ciphertext + * \param in Memory buffer with the encoded PAILLIER_CIPHERTEXT + * object + * \param size The memory size of the in pointer object + * \return 1 on success and 0 otherwise + */ +int PAILLIER_CIPHERTEXT_decode(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + unsigned char *in, size_t size); + +int PAILLIER_KEY_print_fp(FILE *fp, const PAILLIER_KEY *key, int indent); +int PAILLIER_KEY_print(BIO *bp, const PAILLIER_KEY *key, int indent); + +# ifdef __cplusplus +} +# endif +# endif + +#endif diff --git a/openssl/include/openssl/pem.h b/openssl/include/openssl/pem.h index 0446c7701..ab1e97144 100644 --- a/openssl/include/openssl/pem.h +++ b/openssl/include/openssl/pem.h @@ -22,9 +22,6 @@ # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -57,6 +54,7 @@ extern "C" { # define PEM_STRING_ECPRIVATEKEY "EC PRIVATE KEY" # define PEM_STRING_PARAMETERS "PARAMETERS" # define PEM_STRING_CMS "CMS" +# define PEM_STRING_SM2PRIVATEKEY "SM2 PRIVATE KEY" # define PEM_STRING_SM2PARAMETERS "SM2 PARAMETERS" # define PEM_TYPE_ENCRYPTED 10 diff --git a/openssl/include/openssl/pkcs12.h b/openssl/include/openssl/pkcs12.h index def2a8443..6b4ab69ac 100644 --- a/openssl/include/openssl/pkcs12.h +++ b/openssl/include/openssl/pkcs12.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/pkcs12.h.in * - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,9 +25,6 @@ # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -44,7 +41,6 @@ extern "C" { # define PKCS12_MAC_KEY_LENGTH 20 -/* The macro is expected to be used only internally. Kept for backwards compatibility. */ # define PKCS12_SALT_LEN 8 /* It's not clear if these are actually needed... */ @@ -134,9 +130,7 @@ int PKCS12_SAFEBAG_get_bag_nid(const PKCS12_SAFEBAG *bag); const ASN1_TYPE *PKCS12_SAFEBAG_get0_bag_obj(const PKCS12_SAFEBAG *bag); const ASN1_OBJECT *PKCS12_SAFEBAG_get0_bag_type(const PKCS12_SAFEBAG *bag); -X509 *PKCS12_SAFEBAG_get1_cert_ex(const PKCS12_SAFEBAG *bag, OSSL_LIB_CTX *libctx, const char *propq); X509 *PKCS12_SAFEBAG_get1_cert(const PKCS12_SAFEBAG *bag); -X509_CRL *PKCS12_SAFEBAG_get1_crl_ex(const PKCS12_SAFEBAG *bag, OSSL_LIB_CTX *libctx, const char *propq); X509_CRL *PKCS12_SAFEBAG_get1_crl(const PKCS12_SAFEBAG *bag); const STACK_OF(PKCS12_SAFEBAG) * PKCS12_SAFEBAG_get0_safes(const PKCS12_SAFEBAG *bag); @@ -224,7 +218,6 @@ ASN1_TYPE *PKCS12_get_attr_gen(const STACK_OF(X509_ATTRIBUTE) *attrs, char *PKCS12_get_friendlyname(PKCS12_SAFEBAG *bag); const STACK_OF(X509_ATTRIBUTE) * PKCS12_SAFEBAG_get0_attrs(const PKCS12_SAFEBAG *bag); -void PKCS12_SAFEBAG_set0_attrs(PKCS12_SAFEBAG *bag, STACK_OF(X509_ATTRIBUTE) *attrs); unsigned char *PKCS12_pbe_crypt(const X509_ALGOR *algor, const char *pass, int passlen, const unsigned char *in, int inlen, @@ -312,7 +305,6 @@ DECLARE_ASN1_ITEM(PKCS12_AUTHSAFES) void PKCS12_PBE_add(void); int PKCS12_parse(PKCS12 *p12, const char *pass, EVP_PKEY **pkey, X509 **cert, STACK_OF(X509) **ca); -typedef int PKCS12_create_cb(PKCS12_SAFEBAG *bag, void *cbarg); PKCS12 *PKCS12_create(const char *pass, const char *name, EVP_PKEY *pkey, X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, int iter, int mac_iter, int keytype); @@ -320,11 +312,6 @@ PKCS12 *PKCS12_create_ex(const char *pass, const char *name, EVP_PKEY *pkey, X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, int iter, int mac_iter, int keytype, OSSL_LIB_CTX *ctx, const char *propq); -PKCS12 *PKCS12_create_ex2(const char *pass, const char *name, EVP_PKEY *pkey, - X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, - int iter, int mac_iter, int keytype, - OSSL_LIB_CTX *ctx, const char *propq, - PKCS12_create_cb *cb, void *cbarg); PKCS12_SAFEBAG *PKCS12_add_cert(STACK_OF(PKCS12_SAFEBAG) **pbags, X509 *cert); PKCS12_SAFEBAG *PKCS12_add_key(STACK_OF(PKCS12_SAFEBAG) **pbags, diff --git a/openssl/include/openssl/pkcs12.h.in b/openssl/include/openssl/pkcs12.h.in index 35759d4de..c98eebfb3 100644 --- a/openssl/include/openssl/pkcs12.h.in +++ b/openssl/include/openssl/pkcs12.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,9 +26,6 @@ use OpenSSL::stackhash qw(generate_stack_macros); # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -45,7 +42,6 @@ extern "C" { # define PKCS12_MAC_KEY_LENGTH 20 -/* The macro is expected to be used only internally. Kept for backwards compatibility. */ # define PKCS12_SALT_LEN 8 /* It's not clear if these are actually needed... */ @@ -111,9 +107,7 @@ int PKCS12_SAFEBAG_get_bag_nid(const PKCS12_SAFEBAG *bag); const ASN1_TYPE *PKCS12_SAFEBAG_get0_bag_obj(const PKCS12_SAFEBAG *bag); const ASN1_OBJECT *PKCS12_SAFEBAG_get0_bag_type(const PKCS12_SAFEBAG *bag); -X509 *PKCS12_SAFEBAG_get1_cert_ex(const PKCS12_SAFEBAG *bag, OSSL_LIB_CTX *libctx, const char *propq); X509 *PKCS12_SAFEBAG_get1_cert(const PKCS12_SAFEBAG *bag); -X509_CRL *PKCS12_SAFEBAG_get1_crl_ex(const PKCS12_SAFEBAG *bag, OSSL_LIB_CTX *libctx, const char *propq); X509_CRL *PKCS12_SAFEBAG_get1_crl(const PKCS12_SAFEBAG *bag); const STACK_OF(PKCS12_SAFEBAG) * PKCS12_SAFEBAG_get0_safes(const PKCS12_SAFEBAG *bag); @@ -201,7 +195,6 @@ ASN1_TYPE *PKCS12_get_attr_gen(const STACK_OF(X509_ATTRIBUTE) *attrs, char *PKCS12_get_friendlyname(PKCS12_SAFEBAG *bag); const STACK_OF(X509_ATTRIBUTE) * PKCS12_SAFEBAG_get0_attrs(const PKCS12_SAFEBAG *bag); -void PKCS12_SAFEBAG_set0_attrs(PKCS12_SAFEBAG *bag, STACK_OF(X509_ATTRIBUTE) *attrs); unsigned char *PKCS12_pbe_crypt(const X509_ALGOR *algor, const char *pass, int passlen, const unsigned char *in, int inlen, @@ -289,7 +282,6 @@ DECLARE_ASN1_ITEM(PKCS12_AUTHSAFES) void PKCS12_PBE_add(void); int PKCS12_parse(PKCS12 *p12, const char *pass, EVP_PKEY **pkey, X509 **cert, STACK_OF(X509) **ca); -typedef int PKCS12_create_cb(PKCS12_SAFEBAG *bag, void *cbarg); PKCS12 *PKCS12_create(const char *pass, const char *name, EVP_PKEY *pkey, X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, int iter, int mac_iter, int keytype); @@ -297,11 +289,6 @@ PKCS12 *PKCS12_create_ex(const char *pass, const char *name, EVP_PKEY *pkey, X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, int iter, int mac_iter, int keytype, OSSL_LIB_CTX *ctx, const char *propq); -PKCS12 *PKCS12_create_ex2(const char *pass, const char *name, EVP_PKEY *pkey, - X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, - int iter, int mac_iter, int keytype, - OSSL_LIB_CTX *ctx, const char *propq, - PKCS12_create_cb *cb, void *cbarg); PKCS12_SAFEBAG *PKCS12_add_cert(STACK_OF(PKCS12_SAFEBAG) **pbags, X509 *cert); PKCS12_SAFEBAG *PKCS12_add_key(STACK_OF(PKCS12_SAFEBAG) **pbags, diff --git a/openssl/include/openssl/pkcs12err.h b/openssl/include/openssl/pkcs12err.h index abce37362..933c83299 100644 --- a/openssl/include/openssl/pkcs12err.h +++ b/openssl/include/openssl/pkcs12err.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,7 +21,6 @@ /* * PKCS12 reason codes. */ -# define PKCS12_R_CALLBACK_FAILED 115 # define PKCS12_R_CANT_PACK_STRUCTURE 100 # define PKCS12_R_CONTENT_TYPE_NOT_DATA 121 # define PKCS12_R_DECODE_ERROR 101 diff --git a/openssl/include/openssl/pkcs7.h b/openssl/include/openssl/pkcs7.h index 9b8c24867..59638e1b6 100644 --- a/openssl/include/openssl/pkcs7.h +++ b/openssl/include/openssl/pkcs7.h @@ -28,9 +28,6 @@ # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -59,8 +56,8 @@ typedef struct pkcs7_signer_info_st { PKCS7_ISSUER_AND_SERIAL *issuer_and_serial; X509_ALGOR *digest_alg; STACK_OF(X509_ATTRIBUTE) *auth_attr; /* [ 0 ] */ - X509_ALGOR *digest_enc_alg; /* confusing name, actually used for signing */ - ASN1_OCTET_STRING *enc_digest; /* confusing name, actually signature */ + X509_ALGOR *digest_enc_alg; + ASN1_OCTET_STRING *enc_digest; STACK_OF(X509_ATTRIBUTE) *unauth_attr; /* [ 1 ] */ /* The private key to sign with */ EVP_PKEY *pkey; @@ -134,8 +131,8 @@ SKM_DEFINE_STACK_OF_INTERNAL(PKCS7_RECIP_INFO, PKCS7_RECIP_INFO, PKCS7_RECIP_INF typedef struct pkcs7_signed_st { ASN1_INTEGER *version; /* version 1 */ STACK_OF(X509_ALGOR) *md_algs; /* md used */ - STACK_OF(X509) *cert; /* [ 0 ] */ /* name should be 'certificates' */ - STACK_OF(X509_CRL) *crl; /* [ 1 ] */ /* name should be 'crls' */ + STACK_OF(X509) *cert; /* [ 0 ] */ + STACK_OF(X509_CRL) *crl; /* [ 1 ] */ STACK_OF(PKCS7_SIGNER_INFO) *signer_info; struct pkcs7_st *contents; } PKCS7_SIGNED; @@ -161,8 +158,8 @@ typedef struct pkcs7_enveloped_st { typedef struct pkcs7_signedandenveloped_st { ASN1_INTEGER *version; /* version 1 */ STACK_OF(X509_ALGOR) *md_algs; /* md used */ - STACK_OF(X509) *cert; /* [ 0 ] */ /* name should be 'certificates' */ - STACK_OF(X509_CRL) *crl; /* [ 1 ] */ /* name should be 'crls' */ + STACK_OF(X509) *cert; /* [ 0 ] */ + STACK_OF(X509_CRL) *crl; /* [ 1 ] */ STACK_OF(PKCS7_SIGNER_INFO) *signer_info; PKCS7_ENC_CONTENT *enc_data; STACK_OF(PKCS7_RECIP_INFO) *recipientinfo; @@ -203,7 +200,7 @@ typedef struct pkcs7_st { /* NID_pkcs7_data */ ASN1_OCTET_STRING *data; /* NID_pkcs7_signed */ - PKCS7_SIGNED *sign; /* field name 'signed' would clash with C keyword */ + PKCS7_SIGNED *sign; /* NID_pkcs7_enveloped */ PKCS7_ENVELOPE *enveloped; /* NID_pkcs7_signedAndEnveloped */ @@ -344,13 +341,13 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey, const EVP_MD *dgst); int PKCS7_SIGNER_INFO_sign(PKCS7_SIGNER_INFO *si); int PKCS7_add_signer(PKCS7 *p7, PKCS7_SIGNER_INFO *p7i); -int PKCS7_add_certificate(PKCS7 *p7, X509 *cert); -int PKCS7_add_crl(PKCS7 *p7, X509_CRL *crl); +int PKCS7_add_certificate(PKCS7 *p7, X509 *x509); +int PKCS7_add_crl(PKCS7 *p7, X509_CRL *x509); int PKCS7_content_new(PKCS7 *p7, int nid); int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si); int PKCS7_signatureVerify(BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si, - X509 *signer); + X509 *x509); BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio); int PKCS7_dataFinal(PKCS7 *p7, BIO *bio); diff --git a/openssl/include/openssl/pkcs7.h.in b/openssl/include/openssl/pkcs7.h.in index 57e45dc2f..f5c55a3fb 100644 --- a/openssl/include/openssl/pkcs7.h.in +++ b/openssl/include/openssl/pkcs7.h.in @@ -29,9 +29,6 @@ use OpenSSL::stackhash qw(generate_stack_macros); # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -60,8 +57,8 @@ typedef struct pkcs7_signer_info_st { PKCS7_ISSUER_AND_SERIAL *issuer_and_serial; X509_ALGOR *digest_alg; STACK_OF(X509_ATTRIBUTE) *auth_attr; /* [ 0 ] */ - X509_ALGOR *digest_enc_alg; /* confusing name, actually used for signing */ - ASN1_OCTET_STRING *enc_digest; /* confusing name, actually signature */ + X509_ALGOR *digest_enc_alg; + ASN1_OCTET_STRING *enc_digest; STACK_OF(X509_ATTRIBUTE) *unauth_attr; /* [ 1 ] */ /* The private key to sign with */ EVP_PKEY *pkey; @@ -87,8 +84,8 @@ typedef struct pkcs7_recip_info_st { typedef struct pkcs7_signed_st { ASN1_INTEGER *version; /* version 1 */ STACK_OF(X509_ALGOR) *md_algs; /* md used */ - STACK_OF(X509) *cert; /* [ 0 ] */ /* name should be 'certificates' */ - STACK_OF(X509_CRL) *crl; /* [ 1 ] */ /* name should be 'crls' */ + STACK_OF(X509) *cert; /* [ 0 ] */ + STACK_OF(X509_CRL) *crl; /* [ 1 ] */ STACK_OF(PKCS7_SIGNER_INFO) *signer_info; struct pkcs7_st *contents; } PKCS7_SIGNED; @@ -114,8 +111,8 @@ typedef struct pkcs7_enveloped_st { typedef struct pkcs7_signedandenveloped_st { ASN1_INTEGER *version; /* version 1 */ STACK_OF(X509_ALGOR) *md_algs; /* md used */ - STACK_OF(X509) *cert; /* [ 0 ] */ /* name should be 'certificates' */ - STACK_OF(X509_CRL) *crl; /* [ 1 ] */ /* name should be 'crls' */ + STACK_OF(X509) *cert; /* [ 0 ] */ + STACK_OF(X509_CRL) *crl; /* [ 1 ] */ STACK_OF(PKCS7_SIGNER_INFO) *signer_info; PKCS7_ENC_CONTENT *enc_data; STACK_OF(PKCS7_RECIP_INFO) *recipientinfo; @@ -156,7 +153,7 @@ typedef struct pkcs7_st { /* NID_pkcs7_data */ ASN1_OCTET_STRING *data; /* NID_pkcs7_signed */ - PKCS7_SIGNED *sign; /* field name 'signed' would clash with C keyword */ + PKCS7_SIGNED *sign; /* NID_pkcs7_enveloped */ PKCS7_ENVELOPE *enveloped; /* NID_pkcs7_signedAndEnveloped */ @@ -273,13 +270,13 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey, const EVP_MD *dgst); int PKCS7_SIGNER_INFO_sign(PKCS7_SIGNER_INFO *si); int PKCS7_add_signer(PKCS7 *p7, PKCS7_SIGNER_INFO *p7i); -int PKCS7_add_certificate(PKCS7 *p7, X509 *cert); -int PKCS7_add_crl(PKCS7 *p7, X509_CRL *crl); +int PKCS7_add_certificate(PKCS7 *p7, X509 *x509); +int PKCS7_add_crl(PKCS7 *p7, X509_CRL *x509); int PKCS7_content_new(PKCS7 *p7, int nid); int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si); int PKCS7_signatureVerify(BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si, - X509 *signer); + X509 *x509); BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio); int PKCS7_dataFinal(PKCS7 *p7, BIO *bio); diff --git a/openssl/include/openssl/prov_ssl.h b/openssl/include/openssl/prov_ssl.h index 76d01e1eb..7464f9c6c 100644 --- a/openssl/include/openssl/prov_ssl.h +++ b/openssl/include/openssl/prov_ssl.h @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,7 +19,7 @@ extern "C" { # define SSL_MAX_MASTER_KEY_LENGTH 48 -/* SSL/TLS uses a 2 byte unsigned version number */ +# define NTLS1_1_VERSION 0x0101 # define SSL3_VERSION 0x0300 # define TLS1_VERSION 0x0301 # define TLS1_1_VERSION 0x0302 @@ -29,9 +29,6 @@ extern "C" { # define DTLS1_2_VERSION 0xFEFD # define DTLS1_BAD_VER 0x0100 -/* QUIC uses a 4 byte unsigned version number */ -# define OSSL_QUIC1_VERSION 0x0000001 - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/proverr.h b/openssl/include/openssl/proverr.h index d9ef56815..ad67a8f89 100644 --- a/openssl/include/openssl/proverr.h +++ b/openssl/include/openssl/proverr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -32,7 +32,6 @@ # define PROV_R_CIPHER_OPERATION_FAILED 102 # define PROV_R_DERIVATION_FUNCTION_INIT_FAILED 205 # define PROV_R_DIGEST_NOT_ALLOWED 174 -# define PROV_R_EMS_NOT_ENABLED 233 # define PROV_R_ENTROPY_SOURCE_STRENGTH_TOO_WEAK 186 # define PROV_R_ERROR_INSTANTIATING_DRBG 188 # define PROV_R_ERROR_RETRIEVING_ENTROPY 189 @@ -52,7 +51,6 @@ # define PROV_R_INDICATOR_INTEGRITY_FAILURE 210 # define PROV_R_INSUFFICIENT_DRBG_STRENGTH 181 # define PROV_R_INVALID_AAD 108 -# define PROV_R_INVALID_AEAD 231 # define PROV_R_INVALID_CONFIG_DATA 211 # define PROV_R_INVALID_CONSTANT_LENGTH 157 # define PROV_R_INVALID_CURVE 176 @@ -64,11 +62,9 @@ # define PROV_R_INVALID_INPUT_LENGTH 230 # define PROV_R_INVALID_ITERATION_COUNT 123 # define PROV_R_INVALID_IV_LENGTH 109 -# define PROV_R_INVALID_KDF 232 # define PROV_R_INVALID_KEY 158 # define PROV_R_INVALID_KEY_LENGTH 105 # define PROV_R_INVALID_MAC 151 -# define PROV_R_INVALID_MEMORY_SIZE 235 # define PROV_R_INVALID_MGF1_MD 167 # define PROV_R_INVALID_MODE 125 # define PROV_R_INVALID_OUTPUT_LENGTH 217 @@ -80,7 +76,6 @@ # define PROV_R_INVALID_STATE 212 # define PROV_R_INVALID_TAG 110 # define PROV_R_INVALID_TAG_LENGTH 118 -# define PROV_R_INVALID_THREAD_POOL_SIZE 234 # define PROV_R_INVALID_UKM_LENGTH 200 # define PROV_R_INVALID_X931_DIGEST 170 # define PROV_R_IN_ERROR_STATE 192 diff --git a/openssl/include/openssl/provider.h b/openssl/include/openssl/provider.h index 24ec0827b..dc86ff587 100644 --- a/openssl/include/openssl/provider.h +++ b/openssl/include/openssl/provider.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,19 +17,13 @@ extern "C" { # endif -/* Set and Get a library context search path */ +/* Set the default provider search path */ int OSSL_PROVIDER_set_default_search_path(OSSL_LIB_CTX *, const char *path); -const char *OSSL_PROVIDER_get0_default_search_path(OSSL_LIB_CTX *libctx); /* Load and unload a provider */ OSSL_PROVIDER *OSSL_PROVIDER_load(OSSL_LIB_CTX *, const char *name); -OSSL_PROVIDER *OSSL_PROVIDER_load_ex(OSSL_LIB_CTX *, const char *name, - OSSL_PARAM *params); OSSL_PROVIDER *OSSL_PROVIDER_try_load(OSSL_LIB_CTX *, const char *name, int retain_fallbacks); -OSSL_PROVIDER *OSSL_PROVIDER_try_load_ex(OSSL_LIB_CTX *, const char *name, - OSSL_PARAM *params, - int retain_fallbacks); int OSSL_PROVIDER_unload(OSSL_PROVIDER *prov); int OSSL_PROVIDER_available(OSSL_LIB_CTX *, const char *name); int OSSL_PROVIDER_do_all(OSSL_LIB_CTX *ctx, diff --git a/openssl/include/openssl/quic.h b/openssl/include/openssl/quic.h deleted file mode 100644 index 3dc2f5e74..000000000 --- a/openssl/include/openssl/quic.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_QUIC_H -# define OPENSSL_QUIC_H -# pragma once - -# include -# include - -# ifndef OPENSSL_NO_QUIC - -# ifdef __cplusplus -extern "C" { -# endif - -/* - * Method used for non-thread-assisted QUIC client operation. - */ -__owur const SSL_METHOD *OSSL_QUIC_client_method(void); - -/* - * Method used for thread-assisted QUIC client operation. - */ -__owur const SSL_METHOD *OSSL_QUIC_client_thread_method(void); - -/* - * QUIC transport error codes (RFC 9000 s. 20.1) - */ -# define OSSL_QUIC_ERR_NO_ERROR 0x00 -# define OSSL_QUIC_ERR_INTERNAL_ERROR 0x01 -# define OSSL_QUIC_ERR_CONNECTION_REFUSED 0x02 -# define OSSL_QUIC_ERR_FLOW_CONTROL_ERROR 0x03 -# define OSSL_QUIC_ERR_STREAM_LIMIT_ERROR 0x04 -# define OSSL_QUIC_ERR_STREAM_STATE_ERROR 0x05 -# define OSSL_QUIC_ERR_FINAL_SIZE_ERROR 0x06 -# define OSSL_QUIC_ERR_FRAME_ENCODING_ERROR 0x07 -# define OSSL_QUIC_ERR_TRANSPORT_PARAMETER_ERROR 0x08 -# define OSSL_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR 0x09 -# define OSSL_QUIC_ERR_PROTOCOL_VIOLATION 0x0A -# define OSSL_QUIC_ERR_INVALID_TOKEN 0x0B -# define OSSL_QUIC_ERR_APPLICATION_ERROR 0x0C -# define OSSL_QUIC_ERR_CRYPTO_BUFFER_EXCEEDED 0x0D -# define OSSL_QUIC_ERR_KEY_UPDATE_ERROR 0x0E -# define OSSL_QUIC_ERR_AEAD_LIMIT_REACHED 0x0F -# define OSSL_QUIC_ERR_NO_VIABLE_PATH 0x10 - -/* Inclusive range for handshake-specific errors. */ -# define OSSL_QUIC_ERR_CRYPTO_ERR_BEGIN 0x0100 -# define OSSL_QUIC_ERR_CRYPTO_ERR_END 0x01FF - -# define OSSL_QUIC_ERR_CRYPTO_ERR(X) \ - (OSSL_QUIC_ERR_CRYPTO_ERR_BEGIN + (X)) - -/* Local errors. */ -# define OSSL_QUIC_LOCAL_ERR_IDLE_TIMEOUT \ - ((uint64_t)0xFFFFFFFFFFFFFFFFULL) - -# ifdef __cplusplus -} -# endif - -# endif /* OPENSSL_NO_QUIC */ -#endif diff --git a/openssl/include/openssl/rand.h b/openssl/include/openssl/rand.h index 1fa1129e3..ad3054fd5 100644 --- a/openssl/include/openssl/rand.h +++ b/openssl/include/openssl/rand.h @@ -82,8 +82,6 @@ OSSL_DEPRECATEDIN_1_1_0 int RAND_pseudo_bytes(unsigned char *buf, int num); EVP_RAND_CTX *RAND_get0_primary(OSSL_LIB_CTX *ctx); EVP_RAND_CTX *RAND_get0_public(OSSL_LIB_CTX *ctx); EVP_RAND_CTX *RAND_get0_private(OSSL_LIB_CTX *ctx); -int RAND_set0_public(OSSL_LIB_CTX *ctx, EVP_RAND_CTX *rand); -int RAND_set0_private(OSSL_LIB_CTX *ctx, EVP_RAND_CTX *rand); int RAND_set_DRBG_type(OSSL_LIB_CTX *ctx, const char *drbg, const char *propq, const char *cipher, const char *digest); diff --git a/openssl/include/openssl/randerr.h b/openssl/include/openssl/randerr.h index 04880374a..b5e08e436 100644 --- a/openssl/include/openssl/randerr.h +++ b/openssl/include/openssl/randerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -41,7 +41,6 @@ # define RAND_R_GENERATE_ERROR 112 # define RAND_R_INSUFFICIENT_DRBG_STRENGTH 139 # define RAND_R_INTERNAL_ERROR 113 -# define RAND_R_INVALID_PROPERTY_QUERY 137 # define RAND_R_IN_ERROR_STATE 114 # define RAND_R_NOT_A_REGULAR_FILE 122 # define RAND_R_NOT_INSTANTIATED 115 diff --git a/openssl/include/openssl/rc2.h b/openssl/include/openssl/rc2.h deleted file mode 100644 index ff633fd80..000000000 --- a/openssl/include/openssl/rc2.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_RC2_H -# define OPENSSL_RC2_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_RC2_H -# endif - -# include - -# ifndef OPENSSL_NO_RC2 -# ifdef __cplusplus -extern "C" { -# endif - -# define RC2_BLOCK 8 -# define RC2_KEY_LENGTH 16 - -# ifndef OPENSSL_NO_DEPRECATED_3_0 -typedef unsigned int RC2_INT; - -# define RC2_ENCRYPT 1 -# define RC2_DECRYPT 0 - -typedef struct rc2_key_st { - RC2_INT data[64]; -} RC2_KEY; -# endif -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 void RC2_set_key(RC2_KEY *key, int len, - const unsigned char *data, int bits); -OSSL_DEPRECATEDIN_3_0 void RC2_ecb_encrypt(const unsigned char *in, - unsigned char *out, RC2_KEY *key, - int enc); -OSSL_DEPRECATEDIN_3_0 void RC2_encrypt(unsigned long *data, RC2_KEY *key); -OSSL_DEPRECATEDIN_3_0 void RC2_decrypt(unsigned long *data, RC2_KEY *key); -OSSL_DEPRECATEDIN_3_0 void RC2_cbc_encrypt(const unsigned char *in, - unsigned char *out, long length, - RC2_KEY *ks, unsigned char *iv, - int enc); -OSSL_DEPRECATEDIN_3_0 void RC2_cfb64_encrypt(const unsigned char *in, - unsigned char *out, long length, - RC2_KEY *schedule, - unsigned char *ivec, - int *num, int enc); -OSSL_DEPRECATEDIN_3_0 void RC2_ofb64_encrypt(const unsigned char *in, - unsigned char *out, long length, - RC2_KEY *schedule, - unsigned char *ivec, - int *num); -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/ripemd.h b/openssl/include/openssl/ripemd.h deleted file mode 100644 index 900ee317b..000000000 --- a/openssl/include/openssl/ripemd.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_RIPEMD_H -# define OPENSSL_RIPEMD_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_RIPEMD_H -# endif - -# include - -# ifndef OPENSSL_NO_RMD160 -# include -# include - -# define RIPEMD160_DIGEST_LENGTH 20 - -# ifdef __cplusplus -extern "C" { -# endif -# if !defined(OPENSSL_NO_DEPRECATED_3_0) - -# define RIPEMD160_LONG unsigned int - -# define RIPEMD160_CBLOCK 64 -# define RIPEMD160_LBLOCK (RIPEMD160_CBLOCK/4) - -typedef struct RIPEMD160state_st { - RIPEMD160_LONG A, B, C, D, E; - RIPEMD160_LONG Nl, Nh; - RIPEMD160_LONG data[RIPEMD160_LBLOCK]; - unsigned int num; -} RIPEMD160_CTX; -# endif -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 int RIPEMD160_Init(RIPEMD160_CTX *c); -OSSL_DEPRECATEDIN_3_0 int RIPEMD160_Update(RIPEMD160_CTX *c, const void *data, - size_t len); -OSSL_DEPRECATEDIN_3_0 int RIPEMD160_Final(unsigned char *md, RIPEMD160_CTX *c); -OSSL_DEPRECATEDIN_3_0 unsigned char *RIPEMD160(const unsigned char *d, size_t n, - unsigned char *md); -OSSL_DEPRECATEDIN_3_0 void RIPEMD160_Transform(RIPEMD160_CTX *c, - const unsigned char *b); -# endif - -# ifdef __cplusplus -} -# endif -# endif -#endif diff --git a/openssl/include/openssl/rsa.h b/openssl/include/openssl/rsa.h index 167427d3c..a55c9727c 100644 --- a/openssl/include/openssl/rsa.h +++ b/openssl/include/openssl/rsa.h @@ -27,9 +27,6 @@ # endif # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif # ifdef __cplusplus extern "C" { @@ -140,9 +137,6 @@ int EVP_PKEY_CTX_set_rsa_keygen_pubexp(EVP_PKEY_CTX *ctx, BIGNUM *pubexp); # define RSA_PSS_SALTLEN_AUTO -2 /* Set salt length to maximum possible */ # define RSA_PSS_SALTLEN_MAX -3 -/* Auto-detect on verify, set salt length to min(maximum possible, digest - * length) on sign */ -# define RSA_PSS_SALTLEN_AUTO_DIGEST_MAX -4 /* Old compatible max salt length for sign only */ # define RSA_PSS_SALTLEN_MAX_SIGN -2 @@ -189,8 +183,6 @@ int EVP_PKEY_CTX_get0_rsa_oaep_label(EVP_PKEY_CTX *ctx, unsigned char **label); # define EVP_PKEY_CTRL_RSA_KEYGEN_PRIMES (EVP_PKEY_ALG_CTRL + 13) -# define EVP_PKEY_CTRL_RSA_IMPLICIT_REJECTION (EVP_PKEY_ALG_CTRL + 14) - # define RSA_PKCS1_PADDING 1 # define RSA_NO_PADDING 3 # define RSA_PKCS1_OAEP_PADDING 4 @@ -200,9 +192,6 @@ int EVP_PKEY_CTX_get0_rsa_oaep_label(EVP_PKEY_CTX *ctx, unsigned char **label); # define RSA_PKCS1_PSS_PADDING 6 # define RSA_PKCS1_WITH_TLS_PADDING 7 -/* internal RSA_ only */ -# define RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING 8 - # define RSA_PKCS1_PADDING_SIZE 11 # define RSA_set_app_data(s,arg) RSA_set_ex_data(s,0,arg) diff --git a/openssl/include/openssl/seed.h b/openssl/include/openssl/seed.h deleted file mode 100644 index edb218ae6..000000000 --- a/openssl/include/openssl/seed.h +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Copyright (c) 2007 KISA(Korea Information Security Agency). All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Neither the name of author nor the names of its contributors may - * be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef OPENSSL_SEED_H -# define OPENSSL_SEED_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_SEED_H -# endif - -# include - -# ifndef OPENSSL_NO_SEED -# include -# include -# include - -# ifdef __cplusplus -extern "C" { -# endif - -# define SEED_BLOCK_SIZE 16 -# define SEED_KEY_LENGTH 16 - -# ifndef OPENSSL_NO_DEPRECATED_3_0 -/* look whether we need 'long' to get 32 bits */ -# ifdef AES_LONG -# ifndef SEED_LONG -# define SEED_LONG 1 -# endif -# endif - - -typedef struct seed_key_st { -# ifdef SEED_LONG - unsigned long data[32]; -# else - unsigned int data[32]; -# endif -} SEED_KEY_SCHEDULE; -# endif /* OPENSSL_NO_DEPRECATED_3_0 */ -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 -void SEED_set_key(const unsigned char rawkey[SEED_KEY_LENGTH], - SEED_KEY_SCHEDULE *ks); -OSSL_DEPRECATEDIN_3_0 -void SEED_encrypt(const unsigned char s[SEED_BLOCK_SIZE], - unsigned char d[SEED_BLOCK_SIZE], - const SEED_KEY_SCHEDULE *ks); -OSSL_DEPRECATEDIN_3_0 -void SEED_decrypt(const unsigned char s[SEED_BLOCK_SIZE], - unsigned char d[SEED_BLOCK_SIZE], - const SEED_KEY_SCHEDULE *ks); -OSSL_DEPRECATEDIN_3_0 -void SEED_ecb_encrypt(const unsigned char *in, - unsigned char *out, - const SEED_KEY_SCHEDULE *ks, int enc); -OSSL_DEPRECATEDIN_3_0 -void SEED_cbc_encrypt(const unsigned char *in, unsigned char *out, size_t len, - const SEED_KEY_SCHEDULE *ks, - unsigned char ivec[SEED_BLOCK_SIZE], - int enc); -OSSL_DEPRECATEDIN_3_0 -void SEED_cfb128_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const SEED_KEY_SCHEDULE *ks, - unsigned char ivec[SEED_BLOCK_SIZE], - int *num, int enc); -OSSL_DEPRECATEDIN_3_0 -void SEED_ofb128_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const SEED_KEY_SCHEDULE *ks, - unsigned char ivec[SEED_BLOCK_SIZE], - int *num); -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/self_test.h b/openssl/include/openssl/self_test.h index 17822049a..ee4949e5a 100644 --- a/openssl/include/openssl/self_test.h +++ b/openssl/include/openssl/self_test.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -30,8 +30,6 @@ extern "C" { # define OSSL_SELF_TEST_TYPE_INSTALL_INTEGRITY "Install_Integrity" # define OSSL_SELF_TEST_TYPE_CRNG "Continuous_RNG_Test" # define OSSL_SELF_TEST_TYPE_PCT "Conditional_PCT" -# define OSSL_SELF_TEST_TYPE_PCT_KAT "Conditional_KAT" -# define OSSL_SELF_TEST_TYPE_KAT_INTEGRITY "KAT_Integrity" # define OSSL_SELF_TEST_TYPE_KAT_CIPHER "KAT_Cipher" # define OSSL_SELF_TEST_TYPE_KAT_ASYM_CIPHER "KAT_AsymmetricCipher" # define OSSL_SELF_TEST_TYPE_KAT_DIGEST "KAT_Digest" @@ -46,7 +44,6 @@ extern "C" { # define OSSL_SELF_TEST_DESC_INTEGRITY_HMAC "HMAC" # define OSSL_SELF_TEST_DESC_PCT_RSA_PKCS1 "RSA" # define OSSL_SELF_TEST_DESC_PCT_ECDSA "ECDSA" -# define OSSL_SELF_TEST_DESC_PCT_EDDSA "EDDSA" # define OSSL_SELF_TEST_DESC_PCT_DSA "DSA" # define OSSL_SELF_TEST_DESC_CIPHER_AES_GCM "AES_GCM" # define OSSL_SELF_TEST_DESC_CIPHER_AES_ECB "AES_ECB_Decrypt" @@ -72,7 +69,6 @@ extern "C" { # define OSSL_SELF_TEST_DESC_KDF_SSHKDF "SSHKDF" # define OSSL_SELF_TEST_DESC_KDF_TLS12_PRF "TLS12_PRF" # define OSSL_SELF_TEST_DESC_KDF_KBKDF "KBKDF" -# define OSSL_SELF_TEST_DESC_KDF_KBKDF_KMAC "KBKDF_KMAC" # define OSSL_SELF_TEST_DESC_KDF_TLS13_EXTRACT "TLS13_KDF_EXTRACT" # define OSSL_SELF_TEST_DESC_KDF_TLS13_EXPAND "TLS13_KDF_EXPAND" # define OSSL_SELF_TEST_DESC_RNG "RNG" diff --git a/openssl/include/openssl/sha.h b/openssl/include/openssl/sha.h index 163a7d588..6e65a0408 100644 --- a/openssl/include/openssl/sha.h +++ b/openssl/include/openssl/sha.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -81,7 +81,6 @@ OSSL_DEPRECATEDIN_3_0 void SHA256_Transform(SHA256_CTX *c, unsigned char *SHA224(const unsigned char *d, size_t n, unsigned char *md); unsigned char *SHA256(const unsigned char *d, size_t n, unsigned char *md); -# define SHA256_192_DIGEST_LENGTH 24 # define SHA224_DIGEST_LENGTH 28 # define SHA256_DIGEST_LENGTH 32 # define SHA384_DIGEST_LENGTH 48 diff --git a/openssl/include/openssl/sm3.h b/openssl/include/openssl/sm3.h new file mode 100644 index 000000000..e3c8fc3ff --- /dev/null +++ b/openssl/include/openssl/sm3.h @@ -0,0 +1,55 @@ +/* + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017 Ribose Inc. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* This header can move into provider when legacy support is removed */ +#ifndef OPENSSL_SM3_H +# define OPENSSL_SM3_H +# pragma once + +# include +# ifndef OPENSSL_NO_DEPRECATED_3_0 +# define HEADER_SM3_H +# endif + +# include + +# ifndef OPENSSL_NO_SM3 +# include +# include +# ifdef __cplusplus +extern "C" { +# endif + +# ifndef OPENSSL_NO_DEPRECATED_3_0 +# define SM3_DIGEST_LENGTH 32 +# define SM3_WORD unsigned int + +# define SM3_CBLOCK 64 +# define SM3_LBLOCK (SM3_CBLOCK/4) + +typedef struct SM3state_st { + SM3_WORD A, B, C, D, E, F, G, H; + SM3_WORD Nl, Nh; + SM3_WORD data[SM3_LBLOCK]; + unsigned int num; +} SM3_CTX; + +OSSL_DEPRECATEDIN_3_0 int SM3_Init(SM3_CTX *c); +OSSL_DEPRECATEDIN_3_0 int SM3_Update(SM3_CTX *c, const void *data, size_t len); +OSSL_DEPRECATEDIN_3_0 int SM3_Final(unsigned char *md, SM3_CTX *c); +OSSL_DEPRECATEDIN_3_0 void SM3_Transform(SM3_CTX *c, const unsigned char *data); +# endif + +# ifdef __cplusplus +} +# endif +# endif + +#endif /* OPENSSL_SM3_H */ diff --git a/openssl/include/openssl/srtp.h b/openssl/include/openssl/srtp.h index 2c2c33444..d64606e5d 100644 --- a/openssl/include/openssl/srtp.h +++ b/openssl/include/openssl/srtp.h @@ -28,28 +28,16 @@ extern "C" { #endif -# define SRTP_AES128_CM_SHA1_80 0x0001 -# define SRTP_AES128_CM_SHA1_32 0x0002 -# define SRTP_AES128_F8_SHA1_80 0x0003 -# define SRTP_AES128_F8_SHA1_32 0x0004 -# define SRTP_NULL_SHA1_80 0x0005 -# define SRTP_NULL_SHA1_32 0x0006 +# define SRTP_AES128_CM_SHA1_80 0x0001 +# define SRTP_AES128_CM_SHA1_32 0x0002 +# define SRTP_AES128_F8_SHA1_80 0x0003 +# define SRTP_AES128_F8_SHA1_32 0x0004 +# define SRTP_NULL_SHA1_80 0x0005 +# define SRTP_NULL_SHA1_32 0x0006 /* AEAD SRTP protection profiles from RFC 7714 */ -# define SRTP_AEAD_AES_128_GCM 0x0007 -# define SRTP_AEAD_AES_256_GCM 0x0008 - -/* DOUBLE AEAD SRTP protection profiles from RFC 8723 */ -# define SRTP_DOUBLE_AEAD_AES_128_GCM_AEAD_AES_128_GCM 0x0009 -# define SRTP_DOUBLE_AEAD_AES_256_GCM_AEAD_AES_256_GCM 0x000A - -/* ARIA SRTP protection profiles from RFC 8269 */ -# define SRTP_ARIA_128_CTR_HMAC_SHA1_80 0x000B -# define SRTP_ARIA_128_CTR_HMAC_SHA1_32 0x000C -# define SRTP_ARIA_256_CTR_HMAC_SHA1_80 0x000D -# define SRTP_ARIA_256_CTR_HMAC_SHA1_32 0x000E -# define SRTP_AEAD_ARIA_128_GCM 0x000F -# define SRTP_AEAD_ARIA_256_GCM 0x0010 +# define SRTP_AEAD_AES_128_GCM 0x0007 +# define SRTP_AEAD_AES_256_GCM 0x0008 # ifndef OPENSSL_NO_SRTP diff --git a/openssl/include/openssl/ssl.h b/openssl/include/openssl/ssl.h index ef0c515f7..237136740 100644 --- a/openssl/include/openssl/ssl.h +++ b/openssl/include/openssl/ssl.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/ssl.h.in * - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -24,7 +24,6 @@ # endif # include -# include # include # include # include @@ -43,9 +42,7 @@ # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif +# include #ifdef __cplusplus extern "C" { @@ -96,9 +93,9 @@ extern "C" { # define SSL_TXT_kRSAPSK "kRSAPSK" # define SSL_TXT_kECDHEPSK "kECDHEPSK" # define SSL_TXT_kDHEPSK "kDHEPSK" -# define SSL_TXT_kGOST "kGOST" -# define SSL_TXT_kGOST18 "kGOST18" # define SSL_TXT_kSRP "kSRP" +# define SSL_TXT_kSM2 "kSM2" +# define SSL_TXT_kSM2DHE "kSM2DHE" # define SSL_TXT_aRSA "aRSA" # define SSL_TXT_aDSS "aDSS" @@ -106,11 +103,8 @@ extern "C" { # define SSL_TXT_aECDH "aECDH"/* this cipher class has been removed */ # define SSL_TXT_aECDSA "aECDSA" # define SSL_TXT_aPSK "aPSK" -# define SSL_TXT_aGOST94 "aGOST94" -# define SSL_TXT_aGOST01 "aGOST01" -# define SSL_TXT_aGOST12 "aGOST12" -# define SSL_TXT_aGOST "aGOST" # define SSL_TXT_aSRP "aSRP" +# define SSL_TXT_aSM2 "aSM2" # define SSL_TXT_DSS "DSS" # define SSL_TXT_DH "DH" @@ -125,45 +119,34 @@ extern "C" { # define SSL_TXT_ECDSA "ECDSA" # define SSL_TXT_PSK "PSK" # define SSL_TXT_SRP "SRP" +# define SSL_TXT_SM2 "SM2" # define SSL_TXT_DES "DES" # define SSL_TXT_3DES "3DES" # define SSL_TXT_RC4 "RC4" -# define SSL_TXT_RC2 "RC2" -# define SSL_TXT_IDEA "IDEA" -# define SSL_TXT_SEED "SEED" # define SSL_TXT_AES128 "AES128" # define SSL_TXT_AES256 "AES256" # define SSL_TXT_AES "AES" # define SSL_TXT_AES_GCM "AESGCM" # define SSL_TXT_AES_CCM "AESCCM" # define SSL_TXT_AES_CCM_8 "AESCCM8" -# define SSL_TXT_CAMELLIA128 "CAMELLIA128" -# define SSL_TXT_CAMELLIA256 "CAMELLIA256" -# define SSL_TXT_CAMELLIA "CAMELLIA" # define SSL_TXT_CHACHA20 "CHACHA20" -# define SSL_TXT_GOST "GOST89" -# define SSL_TXT_ARIA "ARIA" -# define SSL_TXT_ARIA_GCM "ARIAGCM" -# define SSL_TXT_ARIA128 "ARIA128" -# define SSL_TXT_ARIA256 "ARIA256" -# define SSL_TXT_GOST2012_GOST8912_GOST8912 "GOST2012-GOST8912-GOST8912" # define SSL_TXT_CBC "CBC" - +# define SSL_TXT_SM4 "SM4" # define SSL_TXT_MD5 "MD5" # define SSL_TXT_SHA1 "SHA1" # define SSL_TXT_SHA "SHA"/* same as "SHA1" */ -# define SSL_TXT_GOST94 "GOST94" -# define SSL_TXT_GOST89MAC "GOST89MAC" -# define SSL_TXT_GOST12 "GOST12" -# define SSL_TXT_GOST89MAC12 "GOST89MAC12" # define SSL_TXT_SHA256 "SHA256" # define SSL_TXT_SHA384 "SHA384" +# define SSL_TXT_SM3 "SM3" # define SSL_TXT_SSLV3 "SSLv3" # define SSL_TXT_TLSV1 "TLSv1" # define SSL_TXT_TLSV1_1 "TLSv1.1" # define SSL_TXT_TLSV1_2 "TLSv1.2" +# ifndef OPENSSL_NO_NTLS +# define SSL_TXT_NTLSV1_1 "NTLSv1.1" +# endif # define SSL_TXT_ALL "ALL" @@ -208,6 +191,11 @@ extern "C" { * throwing out anonymous and unencrypted ciphersuites! (The latter are not * actually enabled by ALL, but "ALL:RSA" would enable some of them.) */ +# ifdef SYSTEM_CIPHERS_FILE +# define SSL_SYSTEM_DEFAULT_CIPHER_LIST "PROFILE=SYSTEM" +# else +# define SSL_SYSTEM_DEFAULT_CIPHER_LIST OSSL_default_cipher_list() +# endif /* Used in SSL_set_shutdown()/SSL_get_shutdown(); */ # define SSL_SENT_SHUTDOWN 1 @@ -221,6 +209,12 @@ extern "C" { extern "C" { #endif +# ifndef OPENSSL_NO_NTLS +# define SSL_NORMAL_CERT 0 +# define SSL_SIGN_CERT 1 +# define SSL_ENC_CERT 2 +# endif + # define SSL_FILETYPE_ASN1 X509_FILETYPE_ASN1 # define SSL_FILETYPE_PEM X509_FILETYPE_PEM @@ -282,31 +276,28 @@ typedef int (*tls_session_secret_cb_fn)(SSL *s, void *secret, int *secret_len, /* Extension context codes */ /* This extension is only allowed in TLS */ -#define SSL_EXT_TLS_ONLY 0x00001 +#define SSL_EXT_TLS_ONLY 0x0001 /* This extension is only allowed in DTLS */ -#define SSL_EXT_DTLS_ONLY 0x00002 +#define SSL_EXT_DTLS_ONLY 0x0002 /* Some extensions may be allowed in DTLS but we don't implement them for it */ -#define SSL_EXT_TLS_IMPLEMENTATION_ONLY 0x00004 +#define SSL_EXT_TLS_IMPLEMENTATION_ONLY 0x0004 /* Most extensions are not defined for SSLv3 but EXT_TYPE_renegotiate is */ -#define SSL_EXT_SSL3_ALLOWED 0x00008 +#define SSL_EXT_SSL3_ALLOWED 0x0008 /* Extension is only defined for TLS1.2 and below */ -#define SSL_EXT_TLS1_2_AND_BELOW_ONLY 0x00010 +#define SSL_EXT_TLS1_2_AND_BELOW_ONLY 0x0010 /* Extension is only defined for TLS1.3 and above */ -#define SSL_EXT_TLS1_3_ONLY 0x00020 +#define SSL_EXT_TLS1_3_ONLY 0x0020 /* Ignore this extension during parsing if we are resuming */ -#define SSL_EXT_IGNORE_ON_RESUMPTION 0x00040 -#define SSL_EXT_CLIENT_HELLO 0x00080 +#define SSL_EXT_IGNORE_ON_RESUMPTION 0x0040 +#define SSL_EXT_CLIENT_HELLO 0x0080 /* Really means TLS1.2 or below */ -#define SSL_EXT_TLS1_2_SERVER_HELLO 0x00100 -#define SSL_EXT_TLS1_3_SERVER_HELLO 0x00200 -#define SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS 0x00400 -#define SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST 0x00800 -#define SSL_EXT_TLS1_3_CERTIFICATE 0x01000 -#define SSL_EXT_TLS1_3_NEW_SESSION_TICKET 0x02000 -#define SSL_EXT_TLS1_3_CERTIFICATE_REQUEST 0x04000 -#define SSL_EXT_TLS1_3_CERTIFICATE_COMPRESSION 0x08000 -/* When sending a raw public key in a certificate message */ -#define SSL_EXT_TLS1_3_RAW_PUBLIC_KEY 0x10000 +#define SSL_EXT_TLS1_2_SERVER_HELLO 0x0100 +#define SSL_EXT_TLS1_3_SERVER_HELLO 0x0200 +#define SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS 0x0400 +#define SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST 0x0800 +#define SSL_EXT_TLS1_3_CERTIFICATE 0x1000 +#define SSL_EXT_TLS1_3_NEW_SESSION_TICKET 0x2000 +#define SSL_EXT_TLS1_3_CERTIFICATE_REQUEST 0x4000 /* Typedefs for handling custom extensions */ @@ -428,28 +419,13 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); # define SSL_OP_NO_TLSv1_1 SSL_OP_BIT(28) # define SSL_OP_NO_TLSv1_3 SSL_OP_BIT(29) # define SSL_OP_NO_DTLSv1 SSL_OP_BIT(26) +# ifndef OPENSSL_NO_NTLS +/* Just use a reserved value, don't conflict with OP TLS */ +# define SSL_OP_NO_NTLS SSL_OP_BIT(5) +# endif # define SSL_OP_NO_DTLSv1_2 SSL_OP_BIT(27) /* Disallow all renegotiation */ # define SSL_OP_NO_RENEGOTIATION SSL_OP_BIT(30) - /* - * Make server add server-hello extension from early version of - * cryptopro draft, when GOST ciphersuite is negotiated. Required for - * interoperability with CryptoPro CSP 3.x - */ -# define SSL_OP_CRYPTOPRO_TLSEXT_BUG SSL_OP_BIT(31) -/* - * Disable RFC8879 certificate compression - * SSL_OP_NO_TX_CERTIFICATE_COMPRESSION: don't send compressed certificates, - * and ignore the extension when received. - * SSL_OP_NO_RX_CERTIFICATE_COMPRESSION: don't send the extension, and - * subsequently indicating that receiving is not supported - */ -# define SSL_OP_NO_TX_CERTIFICATE_COMPRESSION SSL_OP_BIT(32) -# define SSL_OP_NO_RX_CERTIFICATE_COMPRESSION SSL_OP_BIT(33) - /* Enable KTLS TX zerocopy on Linux */ -# define SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE SSL_OP_BIT(34) - -#define SSL_OP_PREFER_NO_DHE_KEX SSL_OP_BIT(35) /* * Option "collections." @@ -462,7 +438,7 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); /* Various bug workarounds that should be rather harmless. */ # define SSL_OP_ALL \ - ( SSL_OP_CRYPTOPRO_TLSEXT_BUG | SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS \ + ( SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS \ | SSL_OP_TLSEXT_PADDING | SSL_OP_SAFARI_ECDHE_ECDSA_BUG ) /* @@ -594,8 +570,6 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); # define CERT_PKEY_CERT_TYPE 0x400 /* Cert chain suitable to Suite B */ # define CERT_PKEY_SUITEB 0x800 -/* Cert pkey valid for raw public key use */ -# define CERT_PKEY_RPK 0x1000 # define SSL_CONF_FLAG_CMDLINE 0x1 # define SSL_CONF_FLAG_FILE 0x2 @@ -934,6 +908,9 @@ __owur int SSL_extension_supported(unsigned int ext_type); # define SSL_ASYNC_NO_JOBS 6 # define SSL_CLIENT_HELLO_CB 7 # define SSL_RETRY_VERIFY 8 +# ifndef OPENSSL_NO_SESSION_LOOKUP +# define SSL_SESS_LOOKUP 9 +# endif /* These will only be used when doing non-blocking IO */ # define SSL_want_nothing(s) (SSL_want(s) == SSL_NOTHING) @@ -944,11 +921,9 @@ __owur int SSL_extension_supported(unsigned int ext_type); # define SSL_want_async(s) (SSL_want(s) == SSL_ASYNC_PAUSED) # define SSL_want_async_job(s) (SSL_want(s) == SSL_ASYNC_NO_JOBS) # define SSL_want_client_hello_cb(s) (SSL_want(s) == SSL_CLIENT_HELLO_CB) - -# define SSL_MAC_FLAG_READ_MAC_STREAM 1 -# define SSL_MAC_FLAG_WRITE_MAC_STREAM 2 -# define SSL_MAC_FLAG_READ_MAC_TLSTREE 4 -# define SSL_MAC_FLAG_WRITE_MAC_TLSTREE 8 +# ifndef OPENSSL_NO_SESSION_LOOKUP +# define SSL_want_sess_lookup(s) (SSL_want(s) == SSL_SESS_LOOKUP) +# endif /* * A callback for logging out TLS key material. This callback should log out @@ -987,7 +962,6 @@ uint32_t SSL_get_recv_max_early_data(const SSL *s); # include /* This is mostly sslv3 with a few tweaks */ # include /* Datagram TLS */ # include /* Support for the use_srtp extension */ -# include #ifdef __cplusplus extern "C" { @@ -1089,7 +1063,6 @@ typedef enum { DTLS_ST_CR_HELLO_VERIFY_REQUEST, TLS_ST_CR_SRVR_HELLO, TLS_ST_CR_CERT, - TLS_ST_CR_COMP_CERT, TLS_ST_CR_CERT_STATUS, TLS_ST_CR_KEY_EXCH, TLS_ST_CR_CERT_REQ, @@ -1099,7 +1072,6 @@ typedef enum { TLS_ST_CR_FINISHED, TLS_ST_CW_CLNT_HELLO, TLS_ST_CW_CERT, - TLS_ST_CW_COMP_CERT, TLS_ST_CW_KEY_EXCH, TLS_ST_CW_CERT_VRFY, TLS_ST_CW_CHANGE, @@ -1110,12 +1082,10 @@ typedef enum { DTLS_ST_SW_HELLO_VERIFY_REQUEST, TLS_ST_SW_SRVR_HELLO, TLS_ST_SW_CERT, - TLS_ST_SW_COMP_CERT, TLS_ST_SW_KEY_EXCH, TLS_ST_SW_CERT_REQ, TLS_ST_SW_SRVR_DONE, TLS_ST_SR_CERT, - TLS_ST_SR_COMP_CERT, TLS_ST_SR_KEY_EXCH, TLS_ST_SR_CERT_VRFY, TLS_ST_SR_NEXT_PROTO, @@ -1201,6 +1171,10 @@ size_t SSL_get_peer_finished(const SSL *s, void *buf, size_t count); # define SSL_VERIFY_CLIENT_ONCE 0x04 # define SSL_VERIFY_POST_HANDSHAKE 0x08 +# ifndef OPENSSL_NO_VERIFY_SNI +# define SSL_VERIFY_FAIL_IF_SNI_NOT_MATCH_CERT 0x10 +# endif + # ifndef OPENSSL_NO_DEPRECATED_1_1_0 # define OpenSSL_add_ssl_algorithms() SSL_library_init() # define SSLeay_add_ssl_algorithms() SSL_library_init() @@ -1276,6 +1250,15 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) /* fatal */ # define SSL_AD_INAPPROPRIATE_FALLBACK TLS1_AD_INAPPROPRIATE_FALLBACK # define SSL_AD_NO_APPLICATION_PROTOCOL TLS1_AD_NO_APPLICATION_PROTOCOL +# ifndef OPENSSL_NO_NTLS +# define SSL_AD_UNSUPPORTED_SITE2SITE NTLS_AD_UNSUPPORTED_SITE2SITE +# define SSL_AD_NO_AREA NTLS_AD_NO_AREA +# define SSL_AD_UNSUPPORTED_AREATYPE NTLS_AD_UNSUPPORTED_AREATYPE +# define SSL_AD_BAD_IBCPARAM NTLS_AD_BAD_IBCPARAM +# define SSL_AD_UNSUPPORTED_IBCPARAM NTLS_AD_UNSUPPORTED_IBCPARAM +# define SSL_AD_IDENTITY_NEED NTLS_AD_IDENTITY_NEED +# endif + # define SSL_ERROR_NONE 0 # define SSL_ERROR_SSL 1 # define SSL_ERROR_WANT_READ 2 @@ -1290,6 +1273,10 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) # define SSL_ERROR_WANT_ASYNC_JOB 10 # define SSL_ERROR_WANT_CLIENT_HELLO_CB 11 # define SSL_ERROR_WANT_RETRY_VERIFY 12 +# ifndef OPENSSL_NO_SESSION_LOOKUP +# define SSL_ERROR_WANT_SESSION_LOOKUP 13 +# define SSL_ERROR_PENDING_SESSION 13 /* BoringSSL compatibility */ +# endif # ifndef OPENSSL_NO_DEPRECATED_3_0 # define SSL_CTRL_SET_TMP_DH 3 @@ -1407,7 +1394,6 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) # define SSL_CTRL_GET_SIGNATURE_NID 132 # define SSL_CTRL_GET_TMP_KEY 133 # define SSL_CTRL_GET_NEGOTIATED_GROUP 134 -# define SSL_CTRL_GET_IANA_GROUPS 135 # define SSL_CTRL_SET_RETRY_VERIFY 136 # define SSL_CTRL_GET_VERIFY_CERT_STORE 137 # define SSL_CTRL_GET_CHAIN_CERT_STORE 138 @@ -1513,8 +1499,6 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) # define SSL_get1_groups(s, glist) \ SSL_ctrl(s,SSL_CTRL_GET_GROUPS,0,(int*)(glist)) -# define SSL_get0_iana_groups(s, plst) \ - SSL_ctrl(s,SSL_CTRL_GET_IANA_GROUPS,0,(uint16_t **)(plst)) # define SSL_CTX_set1_groups(ctx, glist, glistlen) \ SSL_CTX_ctrl(ctx,SSL_CTRL_SET_GROUPS,glistlen,(int *)(glist)) # define SSL_CTX_set1_groups_list(ctx, s) \ @@ -1579,7 +1563,6 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) # define SSL_get_max_proto_version(s) \ SSL_ctrl(s, SSL_CTRL_GET_MAX_PROTO_VERSION, 0, NULL) -const char *SSL_get0_group_name(SSL *s); const char *SSL_group_to_name(SSL *s, int id); /* Backwards compatibility, original 1.1.0 names */ @@ -1636,6 +1619,7 @@ __owur SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, const SSL_METHOD *meth); int SSL_CTX_up_ref(SSL_CTX *ctx); void SSL_CTX_free(SSL_CTX *); +SSL_CTX *SSL_CTX_dup(SSL_CTX *ctx); __owur long SSL_CTX_set_timeout(SSL_CTX *ctx, long t); __owur long SSL_CTX_get_timeout(const SSL_CTX *ctx); __owur X509_STORE *SSL_CTX_get_cert_store(const SSL_CTX *); @@ -1660,6 +1644,16 @@ __owur int SSL_CIPHER_get_auth_nid(const SSL_CIPHER *c); __owur const EVP_MD *SSL_CIPHER_get_handshake_digest(const SSL_CIPHER *c); __owur int SSL_CIPHER_is_aead(const SSL_CIPHER *c); +# define SSL_CIPHER_get_mkey BABASSL_CIPHER_get_mkey +# define SSL_CIPHER_get_auth BABASSL_CIPHER_get_auth +# define SSL_CIPHER_get_enc BABASSL_CIPHER_get_enc +# define SSL_CIPHER_get_mac BABASSL_CIPHER_get_mac + +__owur unsigned long BABASSL_CIPHER_get_mkey(const SSL_CIPHER *c); +__owur unsigned long BABASSL_CIPHER_get_auth(const SSL_CIPHER *c); +__owur unsigned long BABASSL_CIPHER_get_enc(const SSL_CIPHER *c); +__owur unsigned long BABASSL_CIPHER_get_mac(const SSL_CIPHER *c); + __owur int SSL_get_fd(const SSL *s); __owur int SSL_get_rfd(const SSL *s); __owur int SSL_get_wfd(const SSL *s); @@ -1678,6 +1672,10 @@ void SSL_set0_wbio(SSL *s, BIO *wbio); void SSL_set_bio(SSL *s, BIO *rbio, BIO *wbio); __owur BIO *SSL_get_rbio(const SSL *s); __owur BIO *SSL_get_wbio(const SSL *s); + +# define SSL_get0_wbio BABASSL_get0_wbio +__owur BIO *BABASSL_get0_wbio(const SSL *s); + __owur int SSL_set_cipher_list(SSL *s, const char *str); __owur int SSL_CTX_set_ciphersuites(SSL_CTX *ctx, const char *str); __owur int SSL_set_ciphersuites(SSL *s, const char *str); @@ -1688,6 +1686,15 @@ __owur SSL_verify_cb SSL_get_verify_callback(const SSL *s); void SSL_set_verify(SSL *s, int mode, SSL_verify_cb callback); void SSL_set_verify_depth(SSL *s, int depth); void SSL_set_cert_cb(SSL *s, int (*cb) (SSL *ssl, void *arg), void *arg); + +typedef int (*SSL_cert_cb_fn) (SSL *s, void *arg); + +# define SSL_get_cert_cb BABASSL_get_cert_cb +# define SSL_get_cert_cb_arg BABASSL_get_cert_cb_arg + +SSL_cert_cb_fn BABASSL_get_cert_cb(SSL *s); +void *BABASSL_get_cert_cb_arg(SSL *s); + # ifndef OPENSSL_NO_DEPRECATED_3_0 OSSL_DEPRECATEDIN_3_0 __owur int SSL_use_RSAPrivateKey(SSL *ssl, RSA *rsa); OSSL_DEPRECATEDIN_3_0 @@ -1763,9 +1770,6 @@ __owur long SSL_SESSION_set_timeout(SSL_SESSION *s, long t); __owur int SSL_SESSION_get_protocol_version(const SSL_SESSION *s); __owur int SSL_SESSION_set_protocol_version(SSL_SESSION *s, int version); -__owur time_t SSL_SESSION_get_time_ex(const SSL_SESSION *s); -__owur time_t SSL_SESSION_set_time_ex(SSL_SESSION *s, time_t t); - __owur const char *SSL_SESSION_get0_hostname(const SSL_SESSION *s); __owur int SSL_SESSION_set1_hostname(SSL_SESSION *s, const char *hostname); void SSL_SESSION_get0_alpn_selected(const SSL_SESSION *s, @@ -1806,6 +1810,9 @@ int SSL_SESSION_print(BIO *fp, const SSL_SESSION *ses); int SSL_SESSION_print_keylog(BIO *bp, const SSL_SESSION *x); int SSL_SESSION_up_ref(SSL_SESSION *ses); void SSL_SESSION_free(SSL_SESSION *ses); +# ifndef OPENSSL_NO_SESSION_LOOKUP +SSL_SESSION *SSL_magic_pending_session_ptr(void); +# endif __owur int i2d_SSL_SESSION(const SSL_SESSION *in, unsigned char **pp); __owur int SSL_set_session(SSL *to, SSL_SESSION *session); int SSL_CTX_add_session(SSL_CTX *ctx, SSL_SESSION *session); @@ -1817,9 +1824,6 @@ __owur int SSL_has_matching_session_id(const SSL *s, unsigned int id_len); SSL_SESSION *d2i_SSL_SESSION(SSL_SESSION **a, const unsigned char **pp, long length); -SSL_SESSION *d2i_SSL_SESSION_ex(SSL_SESSION **a, const unsigned char **pp, - long length, OSSL_LIB_CTX *libctx, - const char *propq); # ifdef OPENSSL_X509_H __owur X509 *SSL_get0_peer_certificate(const SSL *s); @@ -1842,6 +1846,12 @@ void SSL_CTX_set_cert_verify_callback(SSL_CTX *ctx, void *arg); void SSL_CTX_set_cert_cb(SSL_CTX *c, int (*cb) (SSL *ssl, void *arg), void *arg); +# define SSL_CTX_get_cert_cb BABASSL_CTX_get_cert_cb +# define SSL_CTX_get_cert_cb_arg BABASSL_CTX_get_cert_cb_arg + +SSL_cert_cb_fn BABASSL_CTX_get_cert_cb(SSL_CTX *c); +void *BABASSL_CTX_get_cert_cb_arg(SSL_CTX *c); + # ifndef OPENSSL_NO_DEPRECATED_3_0 OSSL_DEPRECATEDIN_3_0 __owur int SSL_CTX_use_RSAPrivateKey(SSL_CTX *ctx, RSA *rsa); @@ -1850,9 +1860,73 @@ __owur int SSL_CTX_use_RSAPrivateKey_ASN1(SSL_CTX *ctx, const unsigned char *d, long len); # endif __owur int SSL_CTX_use_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey); +# ifndef OPENSSL_NO_NTLS +__owur int SSL_CTX_use_enc_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey); +__owur int SSL_CTX_use_sign_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey); +__owur int SSL_CTX_use_enc_PrivateKey_file(SSL_CTX *ctx, const char *file, + int type); +__owur int SSL_CTX_use_sign_PrivateKey_file(SSL_CTX *ctx, const char *file, + int type); +void SSL_CTX_enable_ntls(SSL_CTX *ctx); +void SSL_CTX_disable_ntls(SSL_CTX *ctx); +void SSL_CTX_enable_force_ntls(SSL_CTX *ctx); +void SSL_CTX_disable_force_ntls(SSL_CTX *ctx); +void SSL_enable_ntls(SSL *s); +void SSL_disable_ntls(SSL *s); +void SSL_enable_force_ntls(SSL *s); +void SSL_disable_force_ntls(SSL *s); +# endif + +# ifndef OPENSSL_NO_SM2 +void SSL_CTX_enable_sm_tls13_strict(SSL_CTX *ctx); +void SSL_CTX_disable_sm_tls13_strict(SSL_CTX *ctx); +void SSL_enable_sm_tls13_strict(SSL *s); +void SSL_disable_sm_tls13_strict(SSL *s); +# endif + +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +# define DC_REQ_HAS_BEEN_SEND_TO_PEER 0x01 +# define DC_HAS_BEEN_USED_FOR_VERIFY_PEER 0x02 +# define DC_HAS_BEEN_USED_FOR_SIGN 0x04 + +void SSL_CTX_enable_verify_peer_by_dc(SSL_CTX *ctx); +void SSL_CTX_disable_verify_peer_by_dc(SSL_CTX *ctx); +void SSL_enable_verify_peer_by_dc(SSL *s); +void SSL_disable_verify_peer_by_dc(SSL *s); +void SSL_CTX_enable_sign_by_dc(SSL_CTX *ctx); +void SSL_CTX_disable_sign_by_dc(SSL_CTX *ctx); +void SSL_enable_sign_by_dc(SSL *s); +void SSL_disable_sign_by_dc(SSL *s); +int SSL_get_delegated_credential_tag(SSL *s); +int SSL_verify_delegated_credential_signature(X509 *parent_cert, + DELEGATED_CREDENTIAL *dc, + int is_server); +int SSL_use_dc(SSL *ssl, DELEGATED_CREDENTIAL *dc); +int SSL_use_dc_file(SSL *ssl, const char *file, int type); +int SSL_use_dc_PrivateKey(SSL *ssl, EVP_PKEY *pkey); +int SSL_use_dc_PrivateKey_file(SSL *ssl, const char *file, int type); +int SSL_CTX_use_dc(SSL_CTX *ctx, DELEGATED_CREDENTIAL *dc); +int SSL_CTX_use_dc_file(SSL_CTX *ctx, const char *file, int type); +int SSL_CTX_use_dc_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey); +int SSL_CTX_use_dc_PrivateKey_file(SSL_CTX *ctx, const char *file, int type); +int DC_print(BIO *bp, DELEGATED_CREDENTIAL *dc); +int DC_sign(DELEGATED_CREDENTIAL *dc, EVP_PKEY *dc_pkey, + unsigned int valid_time, int expect_verify_hash, + X509 *ee_cert, EVP_PKEY *ee_pkey, const EVP_MD *md, + int is_server); +# endif + __owur int SSL_CTX_use_PrivateKey_ASN1(int pk, SSL_CTX *ctx, const unsigned char *d, long len); __owur int SSL_CTX_use_certificate(SSL_CTX *ctx, X509 *x); +# ifndef OPENSSL_NO_NTLS +__owur int SSL_CTX_use_enc_certificate(SSL_CTX *ctx, X509 *x); +__owur int SSL_CTX_use_sign_certificate(SSL_CTX *ctx, X509 *x); +__owur int SSL_CTX_use_enc_certificate_file(SSL_CTX *ctx, const char *file, + int type); +__owur int SSL_CTX_use_sign_certificate_file(SSL_CTX *ctx, const char *file, + int type); +# endif __owur int SSL_CTX_use_certificate_ASN1(SSL_CTX *ctx, int len, const unsigned char *d); __owur int SSL_CTX_use_cert_and_key(SSL_CTX *ctx, X509 *x509, EVP_PKEY *privatekey, @@ -1876,9 +1950,14 @@ __owur int SSL_CTX_set_session_id_context(SSL_CTX *ctx, SSL *SSL_new(SSL_CTX *ctx); int SSL_up_ref(SSL *s); + +# define SSL_SESSION_get_ref BABASSL_SESSION_get_ref +int BABASSL_SESSION_get_ref(SSL_SESSION *sess); + int SSL_is_dtls(const SSL *s); -int SSL_is_tls(const SSL *s); -int SSL_is_quic(const SSL *s); +# ifndef OPENSSL_NO_NTLS +int SSL_is_ntls(const SSL *s); +# endif __owur int SSL_set_session_id_context(SSL *ssl, const unsigned char *sid_ctx, unsigned int sid_ctx_len); @@ -1971,8 +2050,6 @@ size_t SSL_client_hello_get0_ciphers(SSL *s, const unsigned char **out); size_t SSL_client_hello_get0_compression_methods(SSL *s, const unsigned char **out); int SSL_client_hello_get1_extensions_present(SSL *s, int **out, size_t *outlen); -int SSL_client_hello_get_extension_order(SSL *s, uint16_t *exts, - size_t *num_exts); int SSL_client_hello_get0_ext(SSL *s, unsigned int type, const unsigned char **out, size_t *outlen); @@ -2019,12 +2096,6 @@ long SSL_callback_ctrl(SSL *, int, void (*)(void)); long SSL_CTX_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg); long SSL_CTX_callback_ctrl(SSL_CTX *, int, void (*)(void)); -# define SSL_WRITE_FLAG_CONCLUDE (1U << 0) - -__owur int SSL_write_ex2(SSL *s, const void *buf, size_t num, - uint64_t flags, - size_t *written); - # define SSL_EARLY_DATA_NOT_SENT 0 # define SSL_EARLY_DATA_REJECTED 1 # define SSL_EARLY_DATA_ACCEPTED 2 @@ -2033,7 +2104,6 @@ __owur int SSL_get_early_data_status(const SSL *s); __owur int SSL_get_error(const SSL *s, int ret_code); __owur const char *SSL_get_version(const SSL *s); -__owur int SSL_get_handshake_rtt(const SSL *s, uint64_t *rtt); /* This sets the 'default' SSL version that SSL_new() will create */ # ifndef OPENSSL_NO_DEPRECATED_3_0 @@ -2098,6 +2168,15 @@ OSSL_DEPRECATEDIN_1_1_0 __owur const SSL_METHOD *DTLSv1_2_server_method(void); OSSL_DEPRECATEDIN_1_1_0 __owur const SSL_METHOD *DTLSv1_2_client_method(void); # endif # endif +/* + * Have to write like this, because of make update can recognize + * this form only + */ +# ifndef OPENSSL_NO_NTLS +__owur const SSL_METHOD *NTLS_method(void); /* NTLSv1.1 */ +__owur const SSL_METHOD *NTLS_server_method(void); +__owur const SSL_METHOD *NTLS_client_method(void); +# endif __owur const SSL_METHOD *DTLS_method(void); /* DTLS 1.0 and 1.2 */ __owur const SSL_METHOD *DTLS_server_method(void); /* DTLS 1.0 and 1.2 */ @@ -2191,6 +2270,8 @@ __owur SSL_SESSION *SSL_get_session(const SSL *ssl); __owur SSL_SESSION *SSL_get1_session(SSL *ssl); /* obtain a reference count */ __owur SSL_CTX *SSL_get_SSL_CTX(const SSL *ssl); SSL_CTX *SSL_set_SSL_CTX(SSL *ssl, SSL_CTX *ctx); +# define SSL_set_SESSION_CTX BABASSL_set_SESSION_CTX +SSL_CTX *BABASSL_set_SESSION_CTX(SSL *ssl, SSL_CTX *ctx); void SSL_set_info_callback(SSL *ssl, void (*cb) (const SSL *ssl, int type, int val)); void (*SSL_get_info_callback(const SSL *ssl)) (const SSL *ssl, int type, @@ -2342,223 +2423,6 @@ size_t SSL_get_num_tickets(const SSL *s); int SSL_CTX_set_num_tickets(SSL_CTX *ctx, size_t num_tickets); size_t SSL_CTX_get_num_tickets(const SSL_CTX *ctx); -/* QUIC support */ -int SSL_handle_events(SSL *s); -__owur int SSL_get_event_timeout(SSL *s, struct timeval *tv, int *is_infinite); -__owur int SSL_get_rpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *desc); -__owur int SSL_get_wpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *desc); -__owur int SSL_net_read_desired(SSL *s); -__owur int SSL_net_write_desired(SSL *s); -__owur int SSL_set_blocking_mode(SSL *s, int blocking); -__owur int SSL_get_blocking_mode(SSL *s); -__owur int SSL_set1_initial_peer_addr(SSL *s, const BIO_ADDR *peer_addr); -__owur SSL *SSL_get0_connection(SSL *s); -__owur int SSL_is_connection(SSL *s); - -#define SSL_STREAM_TYPE_NONE 0 -#define SSL_STREAM_TYPE_READ (1U << 0) -#define SSL_STREAM_TYPE_WRITE (1U << 1) -#define SSL_STREAM_TYPE_BIDI (SSL_STREAM_TYPE_READ | SSL_STREAM_TYPE_WRITE) -__owur int SSL_get_stream_type(SSL *s); - -__owur uint64_t SSL_get_stream_id(SSL *s); -__owur int SSL_is_stream_local(SSL *s); - -#define SSL_DEFAULT_STREAM_MODE_NONE 0 -#define SSL_DEFAULT_STREAM_MODE_AUTO_BIDI 1 -#define SSL_DEFAULT_STREAM_MODE_AUTO_UNI 2 -__owur int SSL_set_default_stream_mode(SSL *s, uint32_t mode); - -#define SSL_STREAM_FLAG_UNI (1U << 0) -#define SSL_STREAM_FLAG_NO_BLOCK (1U << 1) -#define SSL_STREAM_FLAG_ADVANCE (1U << 2) -__owur SSL *SSL_new_stream(SSL *s, uint64_t flags); - -#define SSL_INCOMING_STREAM_POLICY_AUTO 0 -#define SSL_INCOMING_STREAM_POLICY_ACCEPT 1 -#define SSL_INCOMING_STREAM_POLICY_REJECT 2 -__owur int SSL_set_incoming_stream_policy(SSL *s, int policy, uint64_t aec); - -#define SSL_ACCEPT_STREAM_NO_BLOCK (1U << 0) -__owur SSL *SSL_accept_stream(SSL *s, uint64_t flags); -__owur size_t SSL_get_accept_stream_queue_len(SSL *s); - -# ifndef OPENSSL_NO_QUIC -__owur int SSL_inject_net_dgram(SSL *s, const unsigned char *buf, - size_t buf_len, - const BIO_ADDR *peer, - const BIO_ADDR *local); -# endif - -typedef struct ssl_shutdown_ex_args_st { - uint64_t quic_error_code; - const char *quic_reason; -} SSL_SHUTDOWN_EX_ARGS; - -#define SSL_SHUTDOWN_FLAG_RAPID (1U << 0) -#define SSL_SHUTDOWN_FLAG_NO_STREAM_FLUSH (1U << 1) -#define SSL_SHUTDOWN_FLAG_NO_BLOCK (1U << 2) -#define SSL_SHUTDOWN_FLAG_WAIT_PEER (1U << 3) - -__owur int SSL_shutdown_ex(SSL *ssl, uint64_t flags, - const SSL_SHUTDOWN_EX_ARGS *args, - size_t args_len); - -__owur int SSL_stream_conclude(SSL *ssl, uint64_t flags); - -typedef struct ssl_stream_reset_args_st { - uint64_t quic_error_code; -} SSL_STREAM_RESET_ARGS; - -__owur int SSL_stream_reset(SSL *ssl, - const SSL_STREAM_RESET_ARGS *args, - size_t args_len); - -#define SSL_STREAM_STATE_NONE 0 -#define SSL_STREAM_STATE_OK 1 -#define SSL_STREAM_STATE_WRONG_DIR 2 -#define SSL_STREAM_STATE_FINISHED 3 -#define SSL_STREAM_STATE_RESET_LOCAL 4 -#define SSL_STREAM_STATE_RESET_REMOTE 5 -#define SSL_STREAM_STATE_CONN_CLOSED 6 -__owur int SSL_get_stream_read_state(SSL *ssl); -__owur int SSL_get_stream_write_state(SSL *ssl); - -__owur int SSL_get_stream_read_error_code(SSL *ssl, uint64_t *app_error_code); -__owur int SSL_get_stream_write_error_code(SSL *ssl, uint64_t *app_error_code); - -#define SSL_CONN_CLOSE_FLAG_LOCAL (1U << 0) -#define SSL_CONN_CLOSE_FLAG_TRANSPORT (1U << 1) - -typedef struct ssl_conn_close_info_st { - uint64_t error_code, frame_type; - const char *reason; - size_t reason_len; - uint32_t flags; -} SSL_CONN_CLOSE_INFO; - -__owur int SSL_get_conn_close_info(SSL *ssl, - SSL_CONN_CLOSE_INFO *info, - size_t info_len); - -# define SSL_VALUE_CLASS_GENERIC 0 -# define SSL_VALUE_CLASS_FEATURE_REQUEST 1 -# define SSL_VALUE_CLASS_FEATURE_PEER_REQUEST 2 -# define SSL_VALUE_CLASS_FEATURE_NEGOTIATED 3 - -# define SSL_VALUE_NONE 0 -# define SSL_VALUE_QUIC_STREAM_BIDI_LOCAL_AVAIL 1 -# define SSL_VALUE_QUIC_STREAM_BIDI_REMOTE_AVAIL 2 -# define SSL_VALUE_QUIC_STREAM_UNI_LOCAL_AVAIL 3 -# define SSL_VALUE_QUIC_STREAM_UNI_REMOTE_AVAIL 4 -# define SSL_VALUE_QUIC_IDLE_TIMEOUT 5 -# define SSL_VALUE_EVENT_HANDLING_MODE 6 -# define SSL_VALUE_STREAM_WRITE_BUF_SIZE 7 -# define SSL_VALUE_STREAM_WRITE_BUF_USED 8 -# define SSL_VALUE_STREAM_WRITE_BUF_AVAIL 9 - -# define SSL_VALUE_EVENT_HANDLING_MODE_INHERIT 0 -# define SSL_VALUE_EVENT_HANDLING_MODE_IMPLICIT 1 -# define SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT 2 - -int SSL_get_value_uint(SSL *s, uint32_t class_, uint32_t id, uint64_t *v); -int SSL_set_value_uint(SSL *s, uint32_t class_, uint32_t id, uint64_t v); - -# define SSL_get_generic_value_uint(ssl, id, v) \ - SSL_get_value_uint((ssl), SSL_VALUE_CLASS_GENERIC, (id), (v)) -# define SSL_set_generic_value_uint(ssl, id, v) \ - SSL_set_value_uint((ssl), SSL_VALUE_CLASS_GENERIC, (id), (v)) -# define SSL_get_feature_request_uint(ssl, id, v) \ - SSL_get_value_uint((ssl), SSL_VALUE_CLASS_FEATURE_REQUEST, (id), (v)) -# define SSL_set_feature_request_uint(ssl, id, v) \ - SSL_set_value_uint((ssl), SSL_VALUE_CLASS_FEATURE_REQUEST, (id), (v)) -# define SSL_get_feature_peer_request_uint(ssl, id, v) \ - SSL_get_value_uint((ssl), SSL_VALUE_CLASS_FEATURE_PEER_REQUEST, (id), (v)) -# define SSL_get_feature_negotiated_uint(ssl, id, v) \ - SSL_get_value_uint((ssl), SSL_VALUE_CLASS_FEATURE_NEGOTIATED, (id), (v)) - -# define SSL_get_quic_stream_bidi_local_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_QUIC_STREAM_BIDI_LOCAL_AVAIL, \ - (value)) -# define SSL_get_quic_stream_bidi_remote_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_QUIC_STREAM_BIDI_REMOTE_AVAIL, \ - (value)) -# define SSL_get_quic_stream_uni_local_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_QUIC_STREAM_UNI_LOCAL_AVAIL, \ - (value)) -# define SSL_get_quic_stream_uni_remote_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_QUIC_STREAM_UNI_REMOTE_AVAIL, \ - (value)) - -# define SSL_get_event_handling_mode(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_EVENT_HANDLING_MODE, \ - (value)) -# define SSL_set_event_handling_mode(ssl, value) \ - SSL_set_generic_value_uint((ssl), SSL_VALUE_EVENT_HANDLING_MODE, \ - (value)) - -# define SSL_get_stream_write_buf_size(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_STREAM_WRITE_BUF_SIZE, \ - (value)) -# define SSL_get_stream_write_buf_used(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_STREAM_WRITE_BUF_USED, \ - (value)) -# define SSL_get_stream_write_buf_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_STREAM_WRITE_BUF_AVAIL, \ - (value)) - -# define SSL_POLL_EVENT_NONE 0 - -# define SSL_POLL_EVENT_F (1U << 0) /* F (Failure) */ -# define SSL_POLL_EVENT_EL (1U << 1) /* EL (Exception on Listener) */ -# define SSL_POLL_EVENT_EC (1U << 2) /* EC (Exception on Conn) */ -# define SSL_POLL_EVENT_ECD (1U << 3) /* ECD (Exception on Conn Drained) */ -# define SSL_POLL_EVENT_ER (1U << 4) /* ER (Exception on Read) */ -# define SSL_POLL_EVENT_EW (1U << 5) /* EW (Exception on Write) */ -# define SSL_POLL_EVENT_R (1U << 6) /* R (Readable) */ -# define SSL_POLL_EVENT_W (1U << 7) /* W (Writable) */ -# define SSL_POLL_EVENT_IC (1U << 8) /* IC (Incoming Connection) */ -# define SSL_POLL_EVENT_ISB (1U << 9) /* ISB (Incoming Stream: Bidi) */ -# define SSL_POLL_EVENT_ISU (1U << 10) /* ISU (Incoming Stream: Uni) */ -# define SSL_POLL_EVENT_OSB (1U << 11) /* OSB (Outgoing Stream: Bidi) */ -# define SSL_POLL_EVENT_OSU (1U << 12) /* OSU (Outgoing Stream: Uni) */ - -# define SSL_POLL_EVENT_RW (SSL_POLL_EVENT_R | SSL_POLL_EVENT_W) -# define SSL_POLL_EVENT_RE (SSL_POLL_EVENT_R | SSL_POLL_EVENT_ER) -# define SSL_POLL_EVENT_WE (SSL_POLL_EVENT_W | SSL_POLL_EVENT_EW) -# define SSL_POLL_EVENT_RWE (SSL_POLL_EVENT_RE | SSL_POLL_EVENT_WE) -# define SSL_POLL_EVENT_E (SSL_POLL_EVENT_EL | SSL_POLL_EVENT_EC \ - | SSL_POLL_EVENT_ER | SSL_POLL_EVENT_EW) -# define SSL_POLL_EVENT_IS (SSL_POLL_EVENT_ISB | SSL_POLL_EVENT_ISU) -# define SSL_POLL_EVENT_ISE (SSL_POLL_EVENT_IS | SSL_POLL_EVENT_EC) -# define SSL_POLL_EVENT_I (SSL_POLL_EVENT_IS | SSL_POLL_EVENT_IC) -# define SSL_POLL_EVENT_OS (SSL_POLL_EVENT_OSB | SSL_POLL_EVENT_OSU) -# define SSL_POLL_EVENT_OSE (SSL_POLL_EVENT_OS | SSL_POLL_EVENT_EC) - -typedef struct ssl_poll_item_st { - BIO_POLL_DESCRIPTOR desc; - uint64_t events, revents; -} SSL_POLL_ITEM; - -# define SSL_POLL_FLAG_NO_HANDLE_EVENTS (1U << 0) - -__owur int SSL_poll(SSL_POLL_ITEM *items, - size_t num_items, - size_t stride, - const struct timeval *timeout, - uint64_t flags, - size_t *result_count); - -static ossl_inline ossl_unused BIO_POLL_DESCRIPTOR -SSL_as_poll_descriptor(SSL *s) -{ - BIO_POLL_DESCRIPTOR d; - - d.type = BIO_POLL_DESCRIPTOR_TYPE_SSL; - d.value.ssl = s; - return d; -} - # ifndef OPENSSL_NO_DEPRECATED_1_1_0 # define SSL_cache_hit(s) SSL_session_reused(s) # endif @@ -2858,35 +2722,212 @@ void SSL_set_allow_early_data_cb(SSL *s, const char *OSSL_default_cipher_list(void); const char *OSSL_default_ciphersuites(void); -/* RFC8879 Certificate compression APIs */ - -int SSL_CTX_compress_certs(SSL_CTX *ctx, int alg); -int SSL_compress_certs(SSL *ssl, int alg); - -int SSL_CTX_set1_cert_comp_preference(SSL_CTX *ctx, int *algs, size_t len); -int SSL_set1_cert_comp_preference(SSL *ssl, int *algs, size_t len); - -int SSL_CTX_set1_compressed_cert(SSL_CTX *ctx, int algorithm, unsigned char *comp_data, - size_t comp_length, size_t orig_length); -int SSL_set1_compressed_cert(SSL *ssl, int algorithm, unsigned char *comp_data, - size_t comp_length, size_t orig_length); -size_t SSL_CTX_get1_compressed_cert(SSL_CTX *ctx, int alg, unsigned char **data, size_t *orig_len); -size_t SSL_get1_compressed_cert(SSL *ssl, int alg, unsigned char **data, size_t *orig_len); - -__owur int SSL_add_expected_rpk(SSL *s, EVP_PKEY *rpk); -__owur EVP_PKEY *SSL_get0_peer_rpk(const SSL *s); -__owur EVP_PKEY *SSL_SESSION_get0_peer_rpk(SSL_SESSION *s); -__owur int SSL_get_negotiated_client_cert_type(const SSL *s); -__owur int SSL_get_negotiated_server_cert_type(const SSL *s); - -__owur int SSL_set1_client_cert_type(SSL *s, const unsigned char *val, size_t len); -__owur int SSL_set1_server_cert_type(SSL *s, const unsigned char *val, size_t len); -__owur int SSL_CTX_set1_client_cert_type(SSL_CTX *ctx, const unsigned char *val, size_t len); -__owur int SSL_CTX_set1_server_cert_type(SSL_CTX *ctx, const unsigned char *val, size_t len); -__owur int SSL_get0_client_cert_type(const SSL *s, unsigned char **t, size_t *len); -__owur int SSL_get0_server_cert_type(const SSL *s, unsigned char **t, size_t *len); -__owur int SSL_CTX_get0_client_cert_type(const SSL_CTX *ctx, unsigned char **t, size_t *len); -__owur int SSL_CTX_get0_server_cert_type(const SSL_CTX *s, unsigned char **t, size_t *len); +# define SSL_client_hello_get1_extensions BABASSL_client_hello_get1_extensions +int BABASSL_client_hello_get1_extensions(SSL *s, int **out, size_t *outlen); + +# define SSL_get_use_certificate BABASSL_get_use_certificate +X509 *BABASSL_get_use_certificate(const SSL *s); +# define OPENSSL_GET_ALPN_PROPOSED 1 +# define SSL_get0_alpn_proposed BABASSL_get0_alpn_proposed +void BABASSL_get0_alpn_proposed(const SSL *ssl, const unsigned char **data, + unsigned *len); + +# define SSL_CTX_certs_clear BABASSL_CTX_certs_clear +void BABASSL_CTX_certs_clear(SSL_CTX *ctx); + +# define OPENSSL_CHECK_TLSEXT_STATUS 1 +# define SSL_check_tlsext_status BABASSL_check_tlsext_status +int BABASSL_check_tlsext_status(SSL *s); + +# define SSL_get_master_key BABASSL_get_master_key +int BABASSL_get_master_key(SSL *s, unsigned char **master_key, + int *master_key_len); + +# define SSL_debug BABASSL_debug +void BABASSL_debug(SSL *s, unsigned char *str, int len); + +# define SSL_get_desc_and_level SSL_get_alert_level +int SSL_get_alert_level(SSL *ssl, int *level, int *desc); + +# ifndef OPENSSL_NO_NTLS +# define SSL_get_sign_certificate_ntls BABASSL_get_sign_certificate_ntls +# define SSL_get_enc_certificate_ntls BABASSL_get_enc_certificate_ntls +X509 *BABASSL_get_sign_certificate_ntls(const SSL *s); +X509 *BABASSL_get_enc_certificate_ntls(const SSL *s); +__owur int SSL_use_sign_certificate(SSL *ssl, X509 *x); +__owur int SSL_use_sign_certificate_file(SSL *ssl, const char *file, int type); +__owur int SSL_use_enc_certificate(SSL *ssl, X509 *x); +__owur int SSL_use_enc_certificate_file(SSL *ssl, const char *file, int type); +__owur int SSL_use_enc_PrivateKey(SSL *ssl, EVP_PKEY *pkey); +__owur int SSL_use_enc_PrivateKey_file(SSL *ssl, const char *file, int type); +__owur int SSL_use_sign_PrivateKey(SSL *ssl, EVP_PKEY *pkey); +__owur int SSL_use_sign_PrivateKey_file(SSL *ssl, const char *file, int type); +# endif + +# ifndef OPENSSL_NO_SKIP_SCSV +# define SSL_set_no_scsv SSL_set_skip_scsv +void SSL_set_skip_scsv(SSL *s, int skip_scsv); +# endif + +# ifndef OPENSSL_NO_QUIC +/* + * QUIC integration - The QUIC interface matches BoringSSL + * + * ssl_encryption_level_t represents a specific QUIC encryption level used to + * transmit handshake messages. BoringSSL has this as an 'enum'. + */ +typedef enum ssl_encryption_level_t { + ssl_encryption_initial = 0, + ssl_encryption_early_data, + ssl_encryption_handshake, + ssl_encryption_application +} OSSL_ENCRYPTION_LEVEL; + +struct ssl_quic_method_st { + int (*set_read_secret)(SSL *ssl, enum ssl_encryption_level_t level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secret_len); + int (*set_write_secret)(SSL *ssl, enum ssl_encryption_level_t level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secret_len); + int (*add_handshake_data)(SSL *ssl, enum ssl_encryption_level_t level, + const uint8_t *data, size_t len); + int (*flush_flight)(SSL *ssl); + int (*send_alert)(SSL *ssl, enum ssl_encryption_level_t level, uint8_t alert); +}; + +__owur int SSL_CTX_set_quic_method(SSL_CTX *ctx, const SSL_QUIC_METHOD *quic_method); +__owur int SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method); +__owur int SSL_set_quic_transport_params(SSL *ssl, + const uint8_t *params, + size_t params_len); +void SSL_get_peer_quic_transport_params(const SSL *ssl, + const uint8_t **out_params, + size_t *out_params_len); +__owur size_t SSL_quic_max_handshake_flight_len(const SSL *ssl, OSSL_ENCRYPTION_LEVEL level); +__owur OSSL_ENCRYPTION_LEVEL SSL_quic_read_level(const SSL *ssl); +__owur OSSL_ENCRYPTION_LEVEL SSL_quic_write_level(const SSL *ssl); +__owur int SSL_provide_quic_data(SSL *ssl, OSSL_ENCRYPTION_LEVEL level, + const uint8_t *data, size_t len); +__owur int SSL_process_quic_post_handshake(SSL *ssl); + +__owur int SSL_is_quic(SSL *ssl); + +/* BoringSSL API */ +void SSL_set_quic_use_legacy_codepoint(SSL *ssl, int use_legacy); + +/* + * Set an explicit value that you want to use + * If 0 (default) the server will use the highest extenstion the client sent + * If 0 (default) the client will send both extensions + */ +void SSL_set_quic_transport_version(SSL *ssl, int version); +__owur int SSL_get_quic_transport_version(const SSL *ssl); +/* Returns the negotiated version, or -1 on error */ +__owur int SSL_get_peer_quic_transport_version(const SSL *ssl); + +int SSL_CIPHER_get_prf_nid(const SSL_CIPHER *c); + +void SSL_set_quic_early_data_enabled(SSL *ssl, int enabled); +__owur int SSL_set_quic_early_data_context(SSL *ssl, + const uint8_t *context, + size_t context_len); + +# endif + +# ifndef OPENSSL_NO_DYNAMIC_CIPHERS +# define OPENSSL_CIPHERS_CHANGEABLE 1 +# define OPENSSL_CIPHERS_DUP 1 + +# define SSL_set_ciphers SSL_set_cipher_list2 +# define SSL_set_ciphers_by_id SSL_set_cipher_list_by_id +# define SSL_dup_ciphers SSL_dup_cipher_list +# define SSL_dup_ciphers_by_id SSL_dup_cipher_list_by_id +# define SSL_CTX_set_ciphers SSL_CTX_set_cipher_list2 +# define SSL_CTX_get_ciphers_by_id SSL_CTX_get_cipher_list_by_id +# define SSL_CTX_set_ciphers_by_id SSL_CTX_set_cipher_list_by_id + +int SSL_set_cipher_list2(SSL *s, STACK_OF(SSL_CIPHER) *cipher_list); +int SSL_set_cipher_list_by_id(SSL *s, STACK_OF(SSL_CIPHER) *cipher_list_by_id); + +STACK_OF(SSL_CIPHER) *SSL_dup_cipher_list(SSL *s); +STACK_OF(SSL_CIPHER) *SSL_dup_cipher_list_by_id(SSL *s); + +int SSL_CTX_set_cipher_list2(SSL_CTX *ctx, STACK_OF(SSL_CIPHER) *cipher_list); + +STACK_OF(SSL_CIPHER) *SSL_CTX_get_cipher_list_by_id(const SSL_CTX *ctx); +int SSL_CTX_set_cipher_list_by_id(SSL_CTX *ctx, + STACK_OF(SSL_CIPHER) *cipher_list_by_id); +# endif + +# ifndef OPENSSL_NO_VERIFY_SNI +# define OPENSSL_VERIFY_SERVER_CERTIFICATE_HOST 1 +# define SSL_CTX_set_verify_server_certificate_host SSL_CTX_set_verify_cert_with_sni +# define SSL_CTX_get_verify_server_certificate_host SSL_CTX_get_verify_cert_with_sni +# define SSL_CTX_set_verify_sni SSL_CTX_set_verify_cert_with_sni +# define SSL_CTX_get_verify_sni SSL_CTX_get_verify_cert_with_sni + +void SSL_CTX_set_verify_cert_with_sni(SSL_CTX *ctx, int value); +int SSL_CTX_get_verify_cert_with_sni(SSL_CTX *ctx); +# endif + +# ifndef OPENSSL_NO_SESSION_REUSED_TYPE +# define SSL_SESSION_REUSED_TYPE_NOCACHE 0 +# define SSL_SESSION_REUSED_TYPE_CACHE 1 +# define SSL_SESSION_REUSED_TYPE_TICKET 2 + +# define SSL_get_session_reuse_type SSL_get_session_reused_type + +int SSL_get_session_reused_type(SSL *s); +# endif + +# ifndef OPENSSL_NO_STATUS +# define OPENSSL_STATUS_ENABLE 1 +/*SSL handshake status*/ +# define SSL_CLIENT_RPOTOCOL 0 +# define SSL_CLIENT_CIPHER 1 +# define SSL_CLIENT_V2_CIPHER 2 +# define SSL_SERVER_EXCHANGE_PUBKEY 3 +# define SSL_CLIENT_RSA_EXCHANGE 4 +# define SSL_CLIENT_SM2_EXCHANGE 5 +# define SSL_SERVER_DH_PUBKEY 6 +# define SSL_CLIENT_SESSION_ID 7 +# define SSL_CLIENT_ECC_CURVES 8 + +struct ssl_status_st { + void *arg; + void *parg; + int type; + int ssl_status_enable; +}; + +void SSL_set_status_callback(SSL *s, + int (*status_callback)(unsigned char *p, + unsigned int length, + SSL_status *param), + unsigned int ssl_status_enable, void *arg); +int (*SSL_get_status_callback(const SSL *s)) (unsigned char *p, + unsigned int length, + SSL_status *param); +# endif + +# ifndef OPENSSL_NO_CERT_COMPRESSION +typedef int (*SSL_cert_compress_cb_fn)(SSL *s, + const unsigned char *in, size_t inlen, + unsigned char *out, size_t *outlen); +typedef int (*SSL_cert_decompress_cb_fn)(SSL *s, + const unsigned char *in, size_t inlen, + unsigned char *out, size_t outlen); + +int SSL_get_cert_compression_compress_id(SSL *s); +int SSL_get_cert_compression_decompress_id(SSL *s); +int SSL_add_cert_compression_alg(SSL *s, int alg_id, + SSL_cert_compress_cb_fn compress, + SSL_cert_decompress_cb_fn decompress); +int SSL_CTX_add_cert_compression_alg(SSL_CTX *ctx, int alg_id, + SSL_cert_compress_cb_fn compress, + SSL_cert_decompress_cb_fn decompress); +# endif # ifdef __cplusplus } diff --git a/openssl/include/openssl/ssl.h.in b/openssl/include/openssl/ssl.h.in index 442d5cbc3..d800cf78f 100644 --- a/openssl/include/openssl/ssl.h.in +++ b/openssl/include/openssl/ssl.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -25,7 +25,6 @@ use OpenSSL::stackhash qw(generate_stack_macros generate_const_stack_macros); # endif # include -# include # include # include # include @@ -44,9 +43,7 @@ use OpenSSL::stackhash qw(generate_stack_macros generate_const_stack_macros); # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif +# include #ifdef __cplusplus extern "C" { @@ -97,9 +94,9 @@ extern "C" { # define SSL_TXT_kRSAPSK "kRSAPSK" # define SSL_TXT_kECDHEPSK "kECDHEPSK" # define SSL_TXT_kDHEPSK "kDHEPSK" -# define SSL_TXT_kGOST "kGOST" -# define SSL_TXT_kGOST18 "kGOST18" # define SSL_TXT_kSRP "kSRP" +# define SSL_TXT_kSM2 "kSM2" +# define SSL_TXT_kSM2DHE "kSM2DHE" # define SSL_TXT_aRSA "aRSA" # define SSL_TXT_aDSS "aDSS" @@ -107,11 +104,8 @@ extern "C" { # define SSL_TXT_aECDH "aECDH"/* this cipher class has been removed */ # define SSL_TXT_aECDSA "aECDSA" # define SSL_TXT_aPSK "aPSK" -# define SSL_TXT_aGOST94 "aGOST94" -# define SSL_TXT_aGOST01 "aGOST01" -# define SSL_TXT_aGOST12 "aGOST12" -# define SSL_TXT_aGOST "aGOST" # define SSL_TXT_aSRP "aSRP" +# define SSL_TXT_aSM2 "aSM2" # define SSL_TXT_DSS "DSS" # define SSL_TXT_DH "DH" @@ -126,45 +120,34 @@ extern "C" { # define SSL_TXT_ECDSA "ECDSA" # define SSL_TXT_PSK "PSK" # define SSL_TXT_SRP "SRP" +# define SSL_TXT_SM2 "SM2" # define SSL_TXT_DES "DES" # define SSL_TXT_3DES "3DES" # define SSL_TXT_RC4 "RC4" -# define SSL_TXT_RC2 "RC2" -# define SSL_TXT_IDEA "IDEA" -# define SSL_TXT_SEED "SEED" # define SSL_TXT_AES128 "AES128" # define SSL_TXT_AES256 "AES256" # define SSL_TXT_AES "AES" # define SSL_TXT_AES_GCM "AESGCM" # define SSL_TXT_AES_CCM "AESCCM" # define SSL_TXT_AES_CCM_8 "AESCCM8" -# define SSL_TXT_CAMELLIA128 "CAMELLIA128" -# define SSL_TXT_CAMELLIA256 "CAMELLIA256" -# define SSL_TXT_CAMELLIA "CAMELLIA" # define SSL_TXT_CHACHA20 "CHACHA20" -# define SSL_TXT_GOST "GOST89" -# define SSL_TXT_ARIA "ARIA" -# define SSL_TXT_ARIA_GCM "ARIAGCM" -# define SSL_TXT_ARIA128 "ARIA128" -# define SSL_TXT_ARIA256 "ARIA256" -# define SSL_TXT_GOST2012_GOST8912_GOST8912 "GOST2012-GOST8912-GOST8912" # define SSL_TXT_CBC "CBC" - +# define SSL_TXT_SM4 "SM4" # define SSL_TXT_MD5 "MD5" # define SSL_TXT_SHA1 "SHA1" # define SSL_TXT_SHA "SHA"/* same as "SHA1" */ -# define SSL_TXT_GOST94 "GOST94" -# define SSL_TXT_GOST89MAC "GOST89MAC" -# define SSL_TXT_GOST12 "GOST12" -# define SSL_TXT_GOST89MAC12 "GOST89MAC12" # define SSL_TXT_SHA256 "SHA256" # define SSL_TXT_SHA384 "SHA384" +# define SSL_TXT_SM3 "SM3" # define SSL_TXT_SSLV3 "SSLv3" # define SSL_TXT_TLSV1 "TLSv1" # define SSL_TXT_TLSV1_1 "TLSv1.1" # define SSL_TXT_TLSV1_2 "TLSv1.2" +# ifndef OPENSSL_NO_NTLS +# define SSL_TXT_NTLSV1_1 "NTLSv1.1" +# endif # define SSL_TXT_ALL "ALL" @@ -209,6 +192,11 @@ extern "C" { * throwing out anonymous and unencrypted ciphersuites! (The latter are not * actually enabled by ALL, but "ALL:RSA" would enable some of them.) */ +# ifdef SYSTEM_CIPHERS_FILE +# define SSL_SYSTEM_DEFAULT_CIPHER_LIST "PROFILE=SYSTEM" +# else +# define SSL_SYSTEM_DEFAULT_CIPHER_LIST OSSL_default_cipher_list() +# endif /* Used in SSL_set_shutdown()/SSL_get_shutdown(); */ # define SSL_SENT_SHUTDOWN 1 @@ -222,6 +210,12 @@ extern "C" { extern "C" { #endif +# ifndef OPENSSL_NO_NTLS +# define SSL_NORMAL_CERT 0 +# define SSL_SIGN_CERT 1 +# define SSL_ENC_CERT 2 +# endif + # define SSL_FILETYPE_ASN1 X509_FILETYPE_ASN1 # define SSL_FILETYPE_PEM X509_FILETYPE_PEM @@ -259,31 +253,28 @@ typedef int (*tls_session_secret_cb_fn)(SSL *s, void *secret, int *secret_len, /* Extension context codes */ /* This extension is only allowed in TLS */ -#define SSL_EXT_TLS_ONLY 0x00001 +#define SSL_EXT_TLS_ONLY 0x0001 /* This extension is only allowed in DTLS */ -#define SSL_EXT_DTLS_ONLY 0x00002 +#define SSL_EXT_DTLS_ONLY 0x0002 /* Some extensions may be allowed in DTLS but we don't implement them for it */ -#define SSL_EXT_TLS_IMPLEMENTATION_ONLY 0x00004 +#define SSL_EXT_TLS_IMPLEMENTATION_ONLY 0x0004 /* Most extensions are not defined for SSLv3 but EXT_TYPE_renegotiate is */ -#define SSL_EXT_SSL3_ALLOWED 0x00008 +#define SSL_EXT_SSL3_ALLOWED 0x0008 /* Extension is only defined for TLS1.2 and below */ -#define SSL_EXT_TLS1_2_AND_BELOW_ONLY 0x00010 +#define SSL_EXT_TLS1_2_AND_BELOW_ONLY 0x0010 /* Extension is only defined for TLS1.3 and above */ -#define SSL_EXT_TLS1_3_ONLY 0x00020 +#define SSL_EXT_TLS1_3_ONLY 0x0020 /* Ignore this extension during parsing if we are resuming */ -#define SSL_EXT_IGNORE_ON_RESUMPTION 0x00040 -#define SSL_EXT_CLIENT_HELLO 0x00080 +#define SSL_EXT_IGNORE_ON_RESUMPTION 0x0040 +#define SSL_EXT_CLIENT_HELLO 0x0080 /* Really means TLS1.2 or below */ -#define SSL_EXT_TLS1_2_SERVER_HELLO 0x00100 -#define SSL_EXT_TLS1_3_SERVER_HELLO 0x00200 -#define SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS 0x00400 -#define SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST 0x00800 -#define SSL_EXT_TLS1_3_CERTIFICATE 0x01000 -#define SSL_EXT_TLS1_3_NEW_SESSION_TICKET 0x02000 -#define SSL_EXT_TLS1_3_CERTIFICATE_REQUEST 0x04000 -#define SSL_EXT_TLS1_3_CERTIFICATE_COMPRESSION 0x08000 -/* When sending a raw public key in a certificate message */ -#define SSL_EXT_TLS1_3_RAW_PUBLIC_KEY 0x10000 +#define SSL_EXT_TLS1_2_SERVER_HELLO 0x0100 +#define SSL_EXT_TLS1_3_SERVER_HELLO 0x0200 +#define SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS 0x0400 +#define SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST 0x0800 +#define SSL_EXT_TLS1_3_CERTIFICATE 0x1000 +#define SSL_EXT_TLS1_3_NEW_SESSION_TICKET 0x2000 +#define SSL_EXT_TLS1_3_CERTIFICATE_REQUEST 0x4000 /* Typedefs for handling custom extensions */ @@ -405,28 +396,13 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); # define SSL_OP_NO_TLSv1_1 SSL_OP_BIT(28) # define SSL_OP_NO_TLSv1_3 SSL_OP_BIT(29) # define SSL_OP_NO_DTLSv1 SSL_OP_BIT(26) +# ifndef OPENSSL_NO_NTLS +/* Just use a reserved value, don't conflict with OP TLS */ +# define SSL_OP_NO_NTLS SSL_OP_BIT(5) +# endif # define SSL_OP_NO_DTLSv1_2 SSL_OP_BIT(27) /* Disallow all renegotiation */ # define SSL_OP_NO_RENEGOTIATION SSL_OP_BIT(30) - /* - * Make server add server-hello extension from early version of - * cryptopro draft, when GOST ciphersuite is negotiated. Required for - * interoperability with CryptoPro CSP 3.x - */ -# define SSL_OP_CRYPTOPRO_TLSEXT_BUG SSL_OP_BIT(31) -/* - * Disable RFC8879 certificate compression - * SSL_OP_NO_TX_CERTIFICATE_COMPRESSION: don't send compressed certificates, - * and ignore the extension when received. - * SSL_OP_NO_RX_CERTIFICATE_COMPRESSION: don't send the extension, and - * subsequently indicating that receiving is not supported - */ -# define SSL_OP_NO_TX_CERTIFICATE_COMPRESSION SSL_OP_BIT(32) -# define SSL_OP_NO_RX_CERTIFICATE_COMPRESSION SSL_OP_BIT(33) - /* Enable KTLS TX zerocopy on Linux */ -# define SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE SSL_OP_BIT(34) - -#define SSL_OP_PREFER_NO_DHE_KEX SSL_OP_BIT(35) /* * Option "collections." @@ -439,7 +415,7 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); /* Various bug workarounds that should be rather harmless. */ # define SSL_OP_ALL \ - ( SSL_OP_CRYPTOPRO_TLSEXT_BUG | SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS \ + ( SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS \ | SSL_OP_TLSEXT_PADDING | SSL_OP_SAFARI_ECDHE_ECDSA_BUG ) /* @@ -571,8 +547,6 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); # define CERT_PKEY_CERT_TYPE 0x400 /* Cert chain suitable to Suite B */ # define CERT_PKEY_SUITEB 0x800 -/* Cert pkey valid for raw public key use */ -# define CERT_PKEY_RPK 0x1000 # define SSL_CONF_FLAG_CMDLINE 0x1 # define SSL_CONF_FLAG_FILE 0x2 @@ -911,6 +885,9 @@ __owur int SSL_extension_supported(unsigned int ext_type); # define SSL_ASYNC_NO_JOBS 6 # define SSL_CLIENT_HELLO_CB 7 # define SSL_RETRY_VERIFY 8 +# ifndef OPENSSL_NO_SESSION_LOOKUP +# define SSL_SESS_LOOKUP 9 +# endif /* These will only be used when doing non-blocking IO */ # define SSL_want_nothing(s) (SSL_want(s) == SSL_NOTHING) @@ -921,11 +898,9 @@ __owur int SSL_extension_supported(unsigned int ext_type); # define SSL_want_async(s) (SSL_want(s) == SSL_ASYNC_PAUSED) # define SSL_want_async_job(s) (SSL_want(s) == SSL_ASYNC_NO_JOBS) # define SSL_want_client_hello_cb(s) (SSL_want(s) == SSL_CLIENT_HELLO_CB) - -# define SSL_MAC_FLAG_READ_MAC_STREAM 1 -# define SSL_MAC_FLAG_WRITE_MAC_STREAM 2 -# define SSL_MAC_FLAG_READ_MAC_TLSTREE 4 -# define SSL_MAC_FLAG_WRITE_MAC_TLSTREE 8 +# ifndef OPENSSL_NO_SESSION_LOOKUP +# define SSL_want_sess_lookup(s) (SSL_want(s) == SSL_SESS_LOOKUP) +# endif /* * A callback for logging out TLS key material. This callback should log out @@ -964,7 +939,6 @@ uint32_t SSL_get_recv_max_early_data(const SSL *s); # include /* This is mostly sslv3 with a few tweaks */ # include /* Datagram TLS */ # include /* Support for the use_srtp extension */ -# include #ifdef __cplusplus extern "C" { @@ -1017,7 +991,6 @@ typedef enum { DTLS_ST_CR_HELLO_VERIFY_REQUEST, TLS_ST_CR_SRVR_HELLO, TLS_ST_CR_CERT, - TLS_ST_CR_COMP_CERT, TLS_ST_CR_CERT_STATUS, TLS_ST_CR_KEY_EXCH, TLS_ST_CR_CERT_REQ, @@ -1027,7 +1000,6 @@ typedef enum { TLS_ST_CR_FINISHED, TLS_ST_CW_CLNT_HELLO, TLS_ST_CW_CERT, - TLS_ST_CW_COMP_CERT, TLS_ST_CW_KEY_EXCH, TLS_ST_CW_CERT_VRFY, TLS_ST_CW_CHANGE, @@ -1038,12 +1010,10 @@ typedef enum { DTLS_ST_SW_HELLO_VERIFY_REQUEST, TLS_ST_SW_SRVR_HELLO, TLS_ST_SW_CERT, - TLS_ST_SW_COMP_CERT, TLS_ST_SW_KEY_EXCH, TLS_ST_SW_CERT_REQ, TLS_ST_SW_SRVR_DONE, TLS_ST_SR_CERT, - TLS_ST_SR_COMP_CERT, TLS_ST_SR_KEY_EXCH, TLS_ST_SR_CERT_VRFY, TLS_ST_SR_NEXT_PROTO, @@ -1129,6 +1099,10 @@ size_t SSL_get_peer_finished(const SSL *s, void *buf, size_t count); # define SSL_VERIFY_CLIENT_ONCE 0x04 # define SSL_VERIFY_POST_HANDSHAKE 0x08 +# ifndef OPENSSL_NO_VERIFY_SNI +# define SSL_VERIFY_FAIL_IF_SNI_NOT_MATCH_CERT 0x10 +# endif + # ifndef OPENSSL_NO_DEPRECATED_1_1_0 # define OpenSSL_add_ssl_algorithms() SSL_library_init() # define SSLeay_add_ssl_algorithms() SSL_library_init() @@ -1204,6 +1178,15 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) /* fatal */ # define SSL_AD_INAPPROPRIATE_FALLBACK TLS1_AD_INAPPROPRIATE_FALLBACK # define SSL_AD_NO_APPLICATION_PROTOCOL TLS1_AD_NO_APPLICATION_PROTOCOL +# ifndef OPENSSL_NO_NTLS +# define SSL_AD_UNSUPPORTED_SITE2SITE NTLS_AD_UNSUPPORTED_SITE2SITE +# define SSL_AD_NO_AREA NTLS_AD_NO_AREA +# define SSL_AD_UNSUPPORTED_AREATYPE NTLS_AD_UNSUPPORTED_AREATYPE +# define SSL_AD_BAD_IBCPARAM NTLS_AD_BAD_IBCPARAM +# define SSL_AD_UNSUPPORTED_IBCPARAM NTLS_AD_UNSUPPORTED_IBCPARAM +# define SSL_AD_IDENTITY_NEED NTLS_AD_IDENTITY_NEED +# endif + # define SSL_ERROR_NONE 0 # define SSL_ERROR_SSL 1 # define SSL_ERROR_WANT_READ 2 @@ -1218,6 +1201,10 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) # define SSL_ERROR_WANT_ASYNC_JOB 10 # define SSL_ERROR_WANT_CLIENT_HELLO_CB 11 # define SSL_ERROR_WANT_RETRY_VERIFY 12 +# ifndef OPENSSL_NO_SESSION_LOOKUP +# define SSL_ERROR_WANT_SESSION_LOOKUP 13 +# define SSL_ERROR_PENDING_SESSION 13 /* BoringSSL compatibility */ +# endif # ifndef OPENSSL_NO_DEPRECATED_3_0 # define SSL_CTRL_SET_TMP_DH 3 @@ -1335,7 +1322,6 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) # define SSL_CTRL_GET_SIGNATURE_NID 132 # define SSL_CTRL_GET_TMP_KEY 133 # define SSL_CTRL_GET_NEGOTIATED_GROUP 134 -# define SSL_CTRL_GET_IANA_GROUPS 135 # define SSL_CTRL_SET_RETRY_VERIFY 136 # define SSL_CTRL_GET_VERIFY_CERT_STORE 137 # define SSL_CTRL_GET_CHAIN_CERT_STORE 138 @@ -1441,8 +1427,6 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) # define SSL_get1_groups(s, glist) \ SSL_ctrl(s,SSL_CTRL_GET_GROUPS,0,(int*)(glist)) -# define SSL_get0_iana_groups(s, plst) \ - SSL_ctrl(s,SSL_CTRL_GET_IANA_GROUPS,0,(uint16_t **)(plst)) # define SSL_CTX_set1_groups(ctx, glist, glistlen) \ SSL_CTX_ctrl(ctx,SSL_CTRL_SET_GROUPS,glistlen,(int *)(glist)) # define SSL_CTX_set1_groups_list(ctx, s) \ @@ -1507,7 +1491,6 @@ DECLARE_PEM_rw(SSL_SESSION, SSL_SESSION) # define SSL_get_max_proto_version(s) \ SSL_ctrl(s, SSL_CTRL_GET_MAX_PROTO_VERSION, 0, NULL) -const char *SSL_get0_group_name(SSL *s); const char *SSL_group_to_name(SSL *s, int id); /* Backwards compatibility, original 1.1.0 names */ @@ -1564,6 +1547,7 @@ __owur SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, const SSL_METHOD *meth); int SSL_CTX_up_ref(SSL_CTX *ctx); void SSL_CTX_free(SSL_CTX *); +SSL_CTX *SSL_CTX_dup(SSL_CTX *ctx); __owur long SSL_CTX_set_timeout(SSL_CTX *ctx, long t); __owur long SSL_CTX_get_timeout(const SSL_CTX *ctx); __owur X509_STORE *SSL_CTX_get_cert_store(const SSL_CTX *); @@ -1588,6 +1572,16 @@ __owur int SSL_CIPHER_get_auth_nid(const SSL_CIPHER *c); __owur const EVP_MD *SSL_CIPHER_get_handshake_digest(const SSL_CIPHER *c); __owur int SSL_CIPHER_is_aead(const SSL_CIPHER *c); +# define SSL_CIPHER_get_mkey BABASSL_CIPHER_get_mkey +# define SSL_CIPHER_get_auth BABASSL_CIPHER_get_auth +# define SSL_CIPHER_get_enc BABASSL_CIPHER_get_enc +# define SSL_CIPHER_get_mac BABASSL_CIPHER_get_mac + +__owur unsigned long BABASSL_CIPHER_get_mkey(const SSL_CIPHER *c); +__owur unsigned long BABASSL_CIPHER_get_auth(const SSL_CIPHER *c); +__owur unsigned long BABASSL_CIPHER_get_enc(const SSL_CIPHER *c); +__owur unsigned long BABASSL_CIPHER_get_mac(const SSL_CIPHER *c); + __owur int SSL_get_fd(const SSL *s); __owur int SSL_get_rfd(const SSL *s); __owur int SSL_get_wfd(const SSL *s); @@ -1606,6 +1600,10 @@ void SSL_set0_wbio(SSL *s, BIO *wbio); void SSL_set_bio(SSL *s, BIO *rbio, BIO *wbio); __owur BIO *SSL_get_rbio(const SSL *s); __owur BIO *SSL_get_wbio(const SSL *s); + +# define SSL_get0_wbio BABASSL_get0_wbio +__owur BIO *BABASSL_get0_wbio(const SSL *s); + __owur int SSL_set_cipher_list(SSL *s, const char *str); __owur int SSL_CTX_set_ciphersuites(SSL_CTX *ctx, const char *str); __owur int SSL_set_ciphersuites(SSL *s, const char *str); @@ -1616,6 +1614,15 @@ __owur SSL_verify_cb SSL_get_verify_callback(const SSL *s); void SSL_set_verify(SSL *s, int mode, SSL_verify_cb callback); void SSL_set_verify_depth(SSL *s, int depth); void SSL_set_cert_cb(SSL *s, int (*cb) (SSL *ssl, void *arg), void *arg); + +typedef int (*SSL_cert_cb_fn) (SSL *s, void *arg); + +# define SSL_get_cert_cb BABASSL_get_cert_cb +# define SSL_get_cert_cb_arg BABASSL_get_cert_cb_arg + +SSL_cert_cb_fn BABASSL_get_cert_cb(SSL *s); +void *BABASSL_get_cert_cb_arg(SSL *s); + # ifndef OPENSSL_NO_DEPRECATED_3_0 OSSL_DEPRECATEDIN_3_0 __owur int SSL_use_RSAPrivateKey(SSL *ssl, RSA *rsa); OSSL_DEPRECATEDIN_3_0 @@ -1691,9 +1698,6 @@ __owur long SSL_SESSION_set_timeout(SSL_SESSION *s, long t); __owur int SSL_SESSION_get_protocol_version(const SSL_SESSION *s); __owur int SSL_SESSION_set_protocol_version(SSL_SESSION *s, int version); -__owur time_t SSL_SESSION_get_time_ex(const SSL_SESSION *s); -__owur time_t SSL_SESSION_set_time_ex(SSL_SESSION *s, time_t t); - __owur const char *SSL_SESSION_get0_hostname(const SSL_SESSION *s); __owur int SSL_SESSION_set1_hostname(SSL_SESSION *s, const char *hostname); void SSL_SESSION_get0_alpn_selected(const SSL_SESSION *s, @@ -1734,6 +1738,9 @@ int SSL_SESSION_print(BIO *fp, const SSL_SESSION *ses); int SSL_SESSION_print_keylog(BIO *bp, const SSL_SESSION *x); int SSL_SESSION_up_ref(SSL_SESSION *ses); void SSL_SESSION_free(SSL_SESSION *ses); +# ifndef OPENSSL_NO_SESSION_LOOKUP +SSL_SESSION *SSL_magic_pending_session_ptr(void); +# endif __owur int i2d_SSL_SESSION(const SSL_SESSION *in, unsigned char **pp); __owur int SSL_set_session(SSL *to, SSL_SESSION *session); int SSL_CTX_add_session(SSL_CTX *ctx, SSL_SESSION *session); @@ -1745,9 +1752,6 @@ __owur int SSL_has_matching_session_id(const SSL *s, unsigned int id_len); SSL_SESSION *d2i_SSL_SESSION(SSL_SESSION **a, const unsigned char **pp, long length); -SSL_SESSION *d2i_SSL_SESSION_ex(SSL_SESSION **a, const unsigned char **pp, - long length, OSSL_LIB_CTX *libctx, - const char *propq); # ifdef OPENSSL_X509_H __owur X509 *SSL_get0_peer_certificate(const SSL *s); @@ -1770,6 +1774,12 @@ void SSL_CTX_set_cert_verify_callback(SSL_CTX *ctx, void *arg); void SSL_CTX_set_cert_cb(SSL_CTX *c, int (*cb) (SSL *ssl, void *arg), void *arg); +# define SSL_CTX_get_cert_cb BABASSL_CTX_get_cert_cb +# define SSL_CTX_get_cert_cb_arg BABASSL_CTX_get_cert_cb_arg + +SSL_cert_cb_fn BABASSL_CTX_get_cert_cb(SSL_CTX *c); +void *BABASSL_CTX_get_cert_cb_arg(SSL_CTX *c); + # ifndef OPENSSL_NO_DEPRECATED_3_0 OSSL_DEPRECATEDIN_3_0 __owur int SSL_CTX_use_RSAPrivateKey(SSL_CTX *ctx, RSA *rsa); @@ -1778,9 +1788,73 @@ __owur int SSL_CTX_use_RSAPrivateKey_ASN1(SSL_CTX *ctx, const unsigned char *d, long len); # endif __owur int SSL_CTX_use_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey); +# ifndef OPENSSL_NO_NTLS +__owur int SSL_CTX_use_enc_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey); +__owur int SSL_CTX_use_sign_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey); +__owur int SSL_CTX_use_enc_PrivateKey_file(SSL_CTX *ctx, const char *file, + int type); +__owur int SSL_CTX_use_sign_PrivateKey_file(SSL_CTX *ctx, const char *file, + int type); +void SSL_CTX_enable_ntls(SSL_CTX *ctx); +void SSL_CTX_disable_ntls(SSL_CTX *ctx); +void SSL_CTX_enable_force_ntls(SSL_CTX *ctx); +void SSL_CTX_disable_force_ntls(SSL_CTX *ctx); +void SSL_enable_ntls(SSL *s); +void SSL_disable_ntls(SSL *s); +void SSL_enable_force_ntls(SSL *s); +void SSL_disable_force_ntls(SSL *s); +# endif + +# ifndef OPENSSL_NO_SM2 +void SSL_CTX_enable_sm_tls13_strict(SSL_CTX *ctx); +void SSL_CTX_disable_sm_tls13_strict(SSL_CTX *ctx); +void SSL_enable_sm_tls13_strict(SSL *s); +void SSL_disable_sm_tls13_strict(SSL *s); +# endif + +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +# define DC_REQ_HAS_BEEN_SEND_TO_PEER 0x01 +# define DC_HAS_BEEN_USED_FOR_VERIFY_PEER 0x02 +# define DC_HAS_BEEN_USED_FOR_SIGN 0x04 + +void SSL_CTX_enable_verify_peer_by_dc(SSL_CTX *ctx); +void SSL_CTX_disable_verify_peer_by_dc(SSL_CTX *ctx); +void SSL_enable_verify_peer_by_dc(SSL *s); +void SSL_disable_verify_peer_by_dc(SSL *s); +void SSL_CTX_enable_sign_by_dc(SSL_CTX *ctx); +void SSL_CTX_disable_sign_by_dc(SSL_CTX *ctx); +void SSL_enable_sign_by_dc(SSL *s); +void SSL_disable_sign_by_dc(SSL *s); +int SSL_get_delegated_credential_tag(SSL *s); +int SSL_verify_delegated_credential_signature(X509 *parent_cert, + DELEGATED_CREDENTIAL *dc, + int is_server); +int SSL_use_dc(SSL *ssl, DELEGATED_CREDENTIAL *dc); +int SSL_use_dc_file(SSL *ssl, const char *file, int type); +int SSL_use_dc_PrivateKey(SSL *ssl, EVP_PKEY *pkey); +int SSL_use_dc_PrivateKey_file(SSL *ssl, const char *file, int type); +int SSL_CTX_use_dc(SSL_CTX *ctx, DELEGATED_CREDENTIAL *dc); +int SSL_CTX_use_dc_file(SSL_CTX *ctx, const char *file, int type); +int SSL_CTX_use_dc_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey); +int SSL_CTX_use_dc_PrivateKey_file(SSL_CTX *ctx, const char *file, int type); +int DC_print(BIO *bp, DELEGATED_CREDENTIAL *dc); +int DC_sign(DELEGATED_CREDENTIAL *dc, EVP_PKEY *dc_pkey, + unsigned int valid_time, int expect_verify_hash, + X509 *ee_cert, EVP_PKEY *ee_pkey, const EVP_MD *md, + int is_server); +# endif + __owur int SSL_CTX_use_PrivateKey_ASN1(int pk, SSL_CTX *ctx, const unsigned char *d, long len); __owur int SSL_CTX_use_certificate(SSL_CTX *ctx, X509 *x); +# ifndef OPENSSL_NO_NTLS +__owur int SSL_CTX_use_enc_certificate(SSL_CTX *ctx, X509 *x); +__owur int SSL_CTX_use_sign_certificate(SSL_CTX *ctx, X509 *x); +__owur int SSL_CTX_use_enc_certificate_file(SSL_CTX *ctx, const char *file, + int type); +__owur int SSL_CTX_use_sign_certificate_file(SSL_CTX *ctx, const char *file, + int type); +# endif __owur int SSL_CTX_use_certificate_ASN1(SSL_CTX *ctx, int len, const unsigned char *d); __owur int SSL_CTX_use_cert_and_key(SSL_CTX *ctx, X509 *x509, EVP_PKEY *privatekey, @@ -1804,9 +1878,14 @@ __owur int SSL_CTX_set_session_id_context(SSL_CTX *ctx, SSL *SSL_new(SSL_CTX *ctx); int SSL_up_ref(SSL *s); + +# define SSL_SESSION_get_ref BABASSL_SESSION_get_ref +int BABASSL_SESSION_get_ref(SSL_SESSION *sess); + int SSL_is_dtls(const SSL *s); -int SSL_is_tls(const SSL *s); -int SSL_is_quic(const SSL *s); +# ifndef OPENSSL_NO_NTLS +int SSL_is_ntls(const SSL *s); +# endif __owur int SSL_set_session_id_context(SSL *ssl, const unsigned char *sid_ctx, unsigned int sid_ctx_len); @@ -1899,8 +1978,6 @@ size_t SSL_client_hello_get0_ciphers(SSL *s, const unsigned char **out); size_t SSL_client_hello_get0_compression_methods(SSL *s, const unsigned char **out); int SSL_client_hello_get1_extensions_present(SSL *s, int **out, size_t *outlen); -int SSL_client_hello_get_extension_order(SSL *s, uint16_t *exts, - size_t *num_exts); int SSL_client_hello_get0_ext(SSL *s, unsigned int type, const unsigned char **out, size_t *outlen); @@ -1947,12 +2024,6 @@ long SSL_callback_ctrl(SSL *, int, void (*)(void)); long SSL_CTX_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg); long SSL_CTX_callback_ctrl(SSL_CTX *, int, void (*)(void)); -# define SSL_WRITE_FLAG_CONCLUDE (1U << 0) - -__owur int SSL_write_ex2(SSL *s, const void *buf, size_t num, - uint64_t flags, - size_t *written); - # define SSL_EARLY_DATA_NOT_SENT 0 # define SSL_EARLY_DATA_REJECTED 1 # define SSL_EARLY_DATA_ACCEPTED 2 @@ -1961,7 +2032,6 @@ __owur int SSL_get_early_data_status(const SSL *s); __owur int SSL_get_error(const SSL *s, int ret_code); __owur const char *SSL_get_version(const SSL *s); -__owur int SSL_get_handshake_rtt(const SSL *s, uint64_t *rtt); /* This sets the 'default' SSL version that SSL_new() will create */ # ifndef OPENSSL_NO_DEPRECATED_3_0 @@ -2026,6 +2096,15 @@ OSSL_DEPRECATEDIN_1_1_0 __owur const SSL_METHOD *DTLSv1_2_server_method(void); OSSL_DEPRECATEDIN_1_1_0 __owur const SSL_METHOD *DTLSv1_2_client_method(void); # endif # endif +/* + * Have to write like this, because of make update can recognize + * this form only + */ +# ifndef OPENSSL_NO_NTLS +__owur const SSL_METHOD *NTLS_method(void); /* NTLSv1.1 */ +__owur const SSL_METHOD *NTLS_server_method(void); +__owur const SSL_METHOD *NTLS_client_method(void); +# endif __owur const SSL_METHOD *DTLS_method(void); /* DTLS 1.0 and 1.2 */ __owur const SSL_METHOD *DTLS_server_method(void); /* DTLS 1.0 and 1.2 */ @@ -2119,6 +2198,8 @@ __owur SSL_SESSION *SSL_get_session(const SSL *ssl); __owur SSL_SESSION *SSL_get1_session(SSL *ssl); /* obtain a reference count */ __owur SSL_CTX *SSL_get_SSL_CTX(const SSL *ssl); SSL_CTX *SSL_set_SSL_CTX(SSL *ssl, SSL_CTX *ctx); +# define SSL_set_SESSION_CTX BABASSL_set_SESSION_CTX +SSL_CTX *BABASSL_set_SESSION_CTX(SSL *ssl, SSL_CTX *ctx); void SSL_set_info_callback(SSL *ssl, void (*cb) (const SSL *ssl, int type, int val)); void (*SSL_get_info_callback(const SSL *ssl)) (const SSL *ssl, int type, @@ -2270,223 +2351,6 @@ size_t SSL_get_num_tickets(const SSL *s); int SSL_CTX_set_num_tickets(SSL_CTX *ctx, size_t num_tickets); size_t SSL_CTX_get_num_tickets(const SSL_CTX *ctx); -/* QUIC support */ -int SSL_handle_events(SSL *s); -__owur int SSL_get_event_timeout(SSL *s, struct timeval *tv, int *is_infinite); -__owur int SSL_get_rpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *desc); -__owur int SSL_get_wpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *desc); -__owur int SSL_net_read_desired(SSL *s); -__owur int SSL_net_write_desired(SSL *s); -__owur int SSL_set_blocking_mode(SSL *s, int blocking); -__owur int SSL_get_blocking_mode(SSL *s); -__owur int SSL_set1_initial_peer_addr(SSL *s, const BIO_ADDR *peer_addr); -__owur SSL *SSL_get0_connection(SSL *s); -__owur int SSL_is_connection(SSL *s); - -#define SSL_STREAM_TYPE_NONE 0 -#define SSL_STREAM_TYPE_READ (1U << 0) -#define SSL_STREAM_TYPE_WRITE (1U << 1) -#define SSL_STREAM_TYPE_BIDI (SSL_STREAM_TYPE_READ | SSL_STREAM_TYPE_WRITE) -__owur int SSL_get_stream_type(SSL *s); - -__owur uint64_t SSL_get_stream_id(SSL *s); -__owur int SSL_is_stream_local(SSL *s); - -#define SSL_DEFAULT_STREAM_MODE_NONE 0 -#define SSL_DEFAULT_STREAM_MODE_AUTO_BIDI 1 -#define SSL_DEFAULT_STREAM_MODE_AUTO_UNI 2 -__owur int SSL_set_default_stream_mode(SSL *s, uint32_t mode); - -#define SSL_STREAM_FLAG_UNI (1U << 0) -#define SSL_STREAM_FLAG_NO_BLOCK (1U << 1) -#define SSL_STREAM_FLAG_ADVANCE (1U << 2) -__owur SSL *SSL_new_stream(SSL *s, uint64_t flags); - -#define SSL_INCOMING_STREAM_POLICY_AUTO 0 -#define SSL_INCOMING_STREAM_POLICY_ACCEPT 1 -#define SSL_INCOMING_STREAM_POLICY_REJECT 2 -__owur int SSL_set_incoming_stream_policy(SSL *s, int policy, uint64_t aec); - -#define SSL_ACCEPT_STREAM_NO_BLOCK (1U << 0) -__owur SSL *SSL_accept_stream(SSL *s, uint64_t flags); -__owur size_t SSL_get_accept_stream_queue_len(SSL *s); - -# ifndef OPENSSL_NO_QUIC -__owur int SSL_inject_net_dgram(SSL *s, const unsigned char *buf, - size_t buf_len, - const BIO_ADDR *peer, - const BIO_ADDR *local); -# endif - -typedef struct ssl_shutdown_ex_args_st { - uint64_t quic_error_code; - const char *quic_reason; -} SSL_SHUTDOWN_EX_ARGS; - -#define SSL_SHUTDOWN_FLAG_RAPID (1U << 0) -#define SSL_SHUTDOWN_FLAG_NO_STREAM_FLUSH (1U << 1) -#define SSL_SHUTDOWN_FLAG_NO_BLOCK (1U << 2) -#define SSL_SHUTDOWN_FLAG_WAIT_PEER (1U << 3) - -__owur int SSL_shutdown_ex(SSL *ssl, uint64_t flags, - const SSL_SHUTDOWN_EX_ARGS *args, - size_t args_len); - -__owur int SSL_stream_conclude(SSL *ssl, uint64_t flags); - -typedef struct ssl_stream_reset_args_st { - uint64_t quic_error_code; -} SSL_STREAM_RESET_ARGS; - -__owur int SSL_stream_reset(SSL *ssl, - const SSL_STREAM_RESET_ARGS *args, - size_t args_len); - -#define SSL_STREAM_STATE_NONE 0 -#define SSL_STREAM_STATE_OK 1 -#define SSL_STREAM_STATE_WRONG_DIR 2 -#define SSL_STREAM_STATE_FINISHED 3 -#define SSL_STREAM_STATE_RESET_LOCAL 4 -#define SSL_STREAM_STATE_RESET_REMOTE 5 -#define SSL_STREAM_STATE_CONN_CLOSED 6 -__owur int SSL_get_stream_read_state(SSL *ssl); -__owur int SSL_get_stream_write_state(SSL *ssl); - -__owur int SSL_get_stream_read_error_code(SSL *ssl, uint64_t *app_error_code); -__owur int SSL_get_stream_write_error_code(SSL *ssl, uint64_t *app_error_code); - -#define SSL_CONN_CLOSE_FLAG_LOCAL (1U << 0) -#define SSL_CONN_CLOSE_FLAG_TRANSPORT (1U << 1) - -typedef struct ssl_conn_close_info_st { - uint64_t error_code, frame_type; - const char *reason; - size_t reason_len; - uint32_t flags; -} SSL_CONN_CLOSE_INFO; - -__owur int SSL_get_conn_close_info(SSL *ssl, - SSL_CONN_CLOSE_INFO *info, - size_t info_len); - -# define SSL_VALUE_CLASS_GENERIC 0 -# define SSL_VALUE_CLASS_FEATURE_REQUEST 1 -# define SSL_VALUE_CLASS_FEATURE_PEER_REQUEST 2 -# define SSL_VALUE_CLASS_FEATURE_NEGOTIATED 3 - -# define SSL_VALUE_NONE 0 -# define SSL_VALUE_QUIC_STREAM_BIDI_LOCAL_AVAIL 1 -# define SSL_VALUE_QUIC_STREAM_BIDI_REMOTE_AVAIL 2 -# define SSL_VALUE_QUIC_STREAM_UNI_LOCAL_AVAIL 3 -# define SSL_VALUE_QUIC_STREAM_UNI_REMOTE_AVAIL 4 -# define SSL_VALUE_QUIC_IDLE_TIMEOUT 5 -# define SSL_VALUE_EVENT_HANDLING_MODE 6 -# define SSL_VALUE_STREAM_WRITE_BUF_SIZE 7 -# define SSL_VALUE_STREAM_WRITE_BUF_USED 8 -# define SSL_VALUE_STREAM_WRITE_BUF_AVAIL 9 - -# define SSL_VALUE_EVENT_HANDLING_MODE_INHERIT 0 -# define SSL_VALUE_EVENT_HANDLING_MODE_IMPLICIT 1 -# define SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT 2 - -int SSL_get_value_uint(SSL *s, uint32_t class_, uint32_t id, uint64_t *v); -int SSL_set_value_uint(SSL *s, uint32_t class_, uint32_t id, uint64_t v); - -# define SSL_get_generic_value_uint(ssl, id, v) \ - SSL_get_value_uint((ssl), SSL_VALUE_CLASS_GENERIC, (id), (v)) -# define SSL_set_generic_value_uint(ssl, id, v) \ - SSL_set_value_uint((ssl), SSL_VALUE_CLASS_GENERIC, (id), (v)) -# define SSL_get_feature_request_uint(ssl, id, v) \ - SSL_get_value_uint((ssl), SSL_VALUE_CLASS_FEATURE_REQUEST, (id), (v)) -# define SSL_set_feature_request_uint(ssl, id, v) \ - SSL_set_value_uint((ssl), SSL_VALUE_CLASS_FEATURE_REQUEST, (id), (v)) -# define SSL_get_feature_peer_request_uint(ssl, id, v) \ - SSL_get_value_uint((ssl), SSL_VALUE_CLASS_FEATURE_PEER_REQUEST, (id), (v)) -# define SSL_get_feature_negotiated_uint(ssl, id, v) \ - SSL_get_value_uint((ssl), SSL_VALUE_CLASS_FEATURE_NEGOTIATED, (id), (v)) - -# define SSL_get_quic_stream_bidi_local_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_QUIC_STREAM_BIDI_LOCAL_AVAIL, \ - (value)) -# define SSL_get_quic_stream_bidi_remote_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_QUIC_STREAM_BIDI_REMOTE_AVAIL, \ - (value)) -# define SSL_get_quic_stream_uni_local_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_QUIC_STREAM_UNI_LOCAL_AVAIL, \ - (value)) -# define SSL_get_quic_stream_uni_remote_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_QUIC_STREAM_UNI_REMOTE_AVAIL, \ - (value)) - -# define SSL_get_event_handling_mode(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_EVENT_HANDLING_MODE, \ - (value)) -# define SSL_set_event_handling_mode(ssl, value) \ - SSL_set_generic_value_uint((ssl), SSL_VALUE_EVENT_HANDLING_MODE, \ - (value)) - -# define SSL_get_stream_write_buf_size(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_STREAM_WRITE_BUF_SIZE, \ - (value)) -# define SSL_get_stream_write_buf_used(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_STREAM_WRITE_BUF_USED, \ - (value)) -# define SSL_get_stream_write_buf_avail(ssl, value) \ - SSL_get_generic_value_uint((ssl), SSL_VALUE_STREAM_WRITE_BUF_AVAIL, \ - (value)) - -# define SSL_POLL_EVENT_NONE 0 - -# define SSL_POLL_EVENT_F (1U << 0) /* F (Failure) */ -# define SSL_POLL_EVENT_EL (1U << 1) /* EL (Exception on Listener) */ -# define SSL_POLL_EVENT_EC (1U << 2) /* EC (Exception on Conn) */ -# define SSL_POLL_EVENT_ECD (1U << 3) /* ECD (Exception on Conn Drained) */ -# define SSL_POLL_EVENT_ER (1U << 4) /* ER (Exception on Read) */ -# define SSL_POLL_EVENT_EW (1U << 5) /* EW (Exception on Write) */ -# define SSL_POLL_EVENT_R (1U << 6) /* R (Readable) */ -# define SSL_POLL_EVENT_W (1U << 7) /* W (Writable) */ -# define SSL_POLL_EVENT_IC (1U << 8) /* IC (Incoming Connection) */ -# define SSL_POLL_EVENT_ISB (1U << 9) /* ISB (Incoming Stream: Bidi) */ -# define SSL_POLL_EVENT_ISU (1U << 10) /* ISU (Incoming Stream: Uni) */ -# define SSL_POLL_EVENT_OSB (1U << 11) /* OSB (Outgoing Stream: Bidi) */ -# define SSL_POLL_EVENT_OSU (1U << 12) /* OSU (Outgoing Stream: Uni) */ - -# define SSL_POLL_EVENT_RW (SSL_POLL_EVENT_R | SSL_POLL_EVENT_W) -# define SSL_POLL_EVENT_RE (SSL_POLL_EVENT_R | SSL_POLL_EVENT_ER) -# define SSL_POLL_EVENT_WE (SSL_POLL_EVENT_W | SSL_POLL_EVENT_EW) -# define SSL_POLL_EVENT_RWE (SSL_POLL_EVENT_RE | SSL_POLL_EVENT_WE) -# define SSL_POLL_EVENT_E (SSL_POLL_EVENT_EL | SSL_POLL_EVENT_EC \ - | SSL_POLL_EVENT_ER | SSL_POLL_EVENT_EW) -# define SSL_POLL_EVENT_IS (SSL_POLL_EVENT_ISB | SSL_POLL_EVENT_ISU) -# define SSL_POLL_EVENT_ISE (SSL_POLL_EVENT_IS | SSL_POLL_EVENT_EC) -# define SSL_POLL_EVENT_I (SSL_POLL_EVENT_IS | SSL_POLL_EVENT_IC) -# define SSL_POLL_EVENT_OS (SSL_POLL_EVENT_OSB | SSL_POLL_EVENT_OSU) -# define SSL_POLL_EVENT_OSE (SSL_POLL_EVENT_OS | SSL_POLL_EVENT_EC) - -typedef struct ssl_poll_item_st { - BIO_POLL_DESCRIPTOR desc; - uint64_t events, revents; -} SSL_POLL_ITEM; - -# define SSL_POLL_FLAG_NO_HANDLE_EVENTS (1U << 0) - -__owur int SSL_poll(SSL_POLL_ITEM *items, - size_t num_items, - size_t stride, - const struct timeval *timeout, - uint64_t flags, - size_t *result_count); - -static ossl_inline ossl_unused BIO_POLL_DESCRIPTOR -SSL_as_poll_descriptor(SSL *s) -{ - BIO_POLL_DESCRIPTOR d; - - d.type = BIO_POLL_DESCRIPTOR_TYPE_SSL; - d.value.ssl = s; - return d; -} - # ifndef OPENSSL_NO_DEPRECATED_1_1_0 # define SSL_cache_hit(s) SSL_session_reused(s) # endif @@ -2786,35 +2650,212 @@ void SSL_set_allow_early_data_cb(SSL *s, const char *OSSL_default_cipher_list(void); const char *OSSL_default_ciphersuites(void); -/* RFC8879 Certificate compression APIs */ - -int SSL_CTX_compress_certs(SSL_CTX *ctx, int alg); -int SSL_compress_certs(SSL *ssl, int alg); - -int SSL_CTX_set1_cert_comp_preference(SSL_CTX *ctx, int *algs, size_t len); -int SSL_set1_cert_comp_preference(SSL *ssl, int *algs, size_t len); - -int SSL_CTX_set1_compressed_cert(SSL_CTX *ctx, int algorithm, unsigned char *comp_data, - size_t comp_length, size_t orig_length); -int SSL_set1_compressed_cert(SSL *ssl, int algorithm, unsigned char *comp_data, - size_t comp_length, size_t orig_length); -size_t SSL_CTX_get1_compressed_cert(SSL_CTX *ctx, int alg, unsigned char **data, size_t *orig_len); -size_t SSL_get1_compressed_cert(SSL *ssl, int alg, unsigned char **data, size_t *orig_len); - -__owur int SSL_add_expected_rpk(SSL *s, EVP_PKEY *rpk); -__owur EVP_PKEY *SSL_get0_peer_rpk(const SSL *s); -__owur EVP_PKEY *SSL_SESSION_get0_peer_rpk(SSL_SESSION *s); -__owur int SSL_get_negotiated_client_cert_type(const SSL *s); -__owur int SSL_get_negotiated_server_cert_type(const SSL *s); - -__owur int SSL_set1_client_cert_type(SSL *s, const unsigned char *val, size_t len); -__owur int SSL_set1_server_cert_type(SSL *s, const unsigned char *val, size_t len); -__owur int SSL_CTX_set1_client_cert_type(SSL_CTX *ctx, const unsigned char *val, size_t len); -__owur int SSL_CTX_set1_server_cert_type(SSL_CTX *ctx, const unsigned char *val, size_t len); -__owur int SSL_get0_client_cert_type(const SSL *s, unsigned char **t, size_t *len); -__owur int SSL_get0_server_cert_type(const SSL *s, unsigned char **t, size_t *len); -__owur int SSL_CTX_get0_client_cert_type(const SSL_CTX *ctx, unsigned char **t, size_t *len); -__owur int SSL_CTX_get0_server_cert_type(const SSL_CTX *s, unsigned char **t, size_t *len); +# define SSL_client_hello_get1_extensions BABASSL_client_hello_get1_extensions +int BABASSL_client_hello_get1_extensions(SSL *s, int **out, size_t *outlen); + +# define SSL_get_use_certificate BABASSL_get_use_certificate +X509 *BABASSL_get_use_certificate(const SSL *s); +# define OPENSSL_GET_ALPN_PROPOSED 1 +# define SSL_get0_alpn_proposed BABASSL_get0_alpn_proposed +void BABASSL_get0_alpn_proposed(const SSL *ssl, const unsigned char **data, + unsigned *len); + +# define SSL_CTX_certs_clear BABASSL_CTX_certs_clear +void BABASSL_CTX_certs_clear(SSL_CTX *ctx); + +# define OPENSSL_CHECK_TLSEXT_STATUS 1 +# define SSL_check_tlsext_status BABASSL_check_tlsext_status +int BABASSL_check_tlsext_status(SSL *s); + +# define SSL_get_master_key BABASSL_get_master_key +int BABASSL_get_master_key(SSL *s, unsigned char **master_key, + int *master_key_len); + +# define SSL_debug BABASSL_debug +void BABASSL_debug(SSL *s, unsigned char *str, int len); + +# define SSL_get_desc_and_level SSL_get_alert_level +int SSL_get_alert_level(SSL *ssl, int *level, int *desc); + +# ifndef OPENSSL_NO_NTLS +# define SSL_get_sign_certificate_ntls BABASSL_get_sign_certificate_ntls +# define SSL_get_enc_certificate_ntls BABASSL_get_enc_certificate_ntls +X509 *BABASSL_get_sign_certificate_ntls(const SSL *s); +X509 *BABASSL_get_enc_certificate_ntls(const SSL *s); +__owur int SSL_use_sign_certificate(SSL *ssl, X509 *x); +__owur int SSL_use_sign_certificate_file(SSL *ssl, const char *file, int type); +__owur int SSL_use_enc_certificate(SSL *ssl, X509 *x); +__owur int SSL_use_enc_certificate_file(SSL *ssl, const char *file, int type); +__owur int SSL_use_enc_PrivateKey(SSL *ssl, EVP_PKEY *pkey); +__owur int SSL_use_enc_PrivateKey_file(SSL *ssl, const char *file, int type); +__owur int SSL_use_sign_PrivateKey(SSL *ssl, EVP_PKEY *pkey); +__owur int SSL_use_sign_PrivateKey_file(SSL *ssl, const char *file, int type); +# endif + +# ifndef OPENSSL_NO_SKIP_SCSV +# define SSL_set_no_scsv SSL_set_skip_scsv +void SSL_set_skip_scsv(SSL *s, int skip_scsv); +# endif + +# ifndef OPENSSL_NO_QUIC +/* + * QUIC integration - The QUIC interface matches BoringSSL + * + * ssl_encryption_level_t represents a specific QUIC encryption level used to + * transmit handshake messages. BoringSSL has this as an 'enum'. + */ +typedef enum ssl_encryption_level_t { + ssl_encryption_initial = 0, + ssl_encryption_early_data, + ssl_encryption_handshake, + ssl_encryption_application +} OSSL_ENCRYPTION_LEVEL; + +struct ssl_quic_method_st { + int (*set_read_secret)(SSL *ssl, enum ssl_encryption_level_t level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secret_len); + int (*set_write_secret)(SSL *ssl, enum ssl_encryption_level_t level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secret_len); + int (*add_handshake_data)(SSL *ssl, enum ssl_encryption_level_t level, + const uint8_t *data, size_t len); + int (*flush_flight)(SSL *ssl); + int (*send_alert)(SSL *ssl, enum ssl_encryption_level_t level, uint8_t alert); +}; + +__owur int SSL_CTX_set_quic_method(SSL_CTX *ctx, const SSL_QUIC_METHOD *quic_method); +__owur int SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method); +__owur int SSL_set_quic_transport_params(SSL *ssl, + const uint8_t *params, + size_t params_len); +void SSL_get_peer_quic_transport_params(const SSL *ssl, + const uint8_t **out_params, + size_t *out_params_len); +__owur size_t SSL_quic_max_handshake_flight_len(const SSL *ssl, OSSL_ENCRYPTION_LEVEL level); +__owur OSSL_ENCRYPTION_LEVEL SSL_quic_read_level(const SSL *ssl); +__owur OSSL_ENCRYPTION_LEVEL SSL_quic_write_level(const SSL *ssl); +__owur int SSL_provide_quic_data(SSL *ssl, OSSL_ENCRYPTION_LEVEL level, + const uint8_t *data, size_t len); +__owur int SSL_process_quic_post_handshake(SSL *ssl); + +__owur int SSL_is_quic(SSL *ssl); + +/* BoringSSL API */ +void SSL_set_quic_use_legacy_codepoint(SSL *ssl, int use_legacy); + +/* + * Set an explicit value that you want to use + * If 0 (default) the server will use the highest extenstion the client sent + * If 0 (default) the client will send both extensions + */ +void SSL_set_quic_transport_version(SSL *ssl, int version); +__owur int SSL_get_quic_transport_version(const SSL *ssl); +/* Returns the negotiated version, or -1 on error */ +__owur int SSL_get_peer_quic_transport_version(const SSL *ssl); + +int SSL_CIPHER_get_prf_nid(const SSL_CIPHER *c); + +void SSL_set_quic_early_data_enabled(SSL *ssl, int enabled); +__owur int SSL_set_quic_early_data_context(SSL *ssl, + const uint8_t *context, + size_t context_len); + +# endif + +# ifndef OPENSSL_NO_DYNAMIC_CIPHERS +# define OPENSSL_CIPHERS_CHANGEABLE 1 +# define OPENSSL_CIPHERS_DUP 1 + +# define SSL_set_ciphers SSL_set_cipher_list2 +# define SSL_set_ciphers_by_id SSL_set_cipher_list_by_id +# define SSL_dup_ciphers SSL_dup_cipher_list +# define SSL_dup_ciphers_by_id SSL_dup_cipher_list_by_id +# define SSL_CTX_set_ciphers SSL_CTX_set_cipher_list2 +# define SSL_CTX_get_ciphers_by_id SSL_CTX_get_cipher_list_by_id +# define SSL_CTX_set_ciphers_by_id SSL_CTX_set_cipher_list_by_id + +int SSL_set_cipher_list2(SSL *s, STACK_OF(SSL_CIPHER) *cipher_list); +int SSL_set_cipher_list_by_id(SSL *s, STACK_OF(SSL_CIPHER) *cipher_list_by_id); + +STACK_OF(SSL_CIPHER) *SSL_dup_cipher_list(SSL *s); +STACK_OF(SSL_CIPHER) *SSL_dup_cipher_list_by_id(SSL *s); + +int SSL_CTX_set_cipher_list2(SSL_CTX *ctx, STACK_OF(SSL_CIPHER) *cipher_list); + +STACK_OF(SSL_CIPHER) *SSL_CTX_get_cipher_list_by_id(const SSL_CTX *ctx); +int SSL_CTX_set_cipher_list_by_id(SSL_CTX *ctx, + STACK_OF(SSL_CIPHER) *cipher_list_by_id); +# endif + +# ifndef OPENSSL_NO_VERIFY_SNI +# define OPENSSL_VERIFY_SERVER_CERTIFICATE_HOST 1 +# define SSL_CTX_set_verify_server_certificate_host SSL_CTX_set_verify_cert_with_sni +# define SSL_CTX_get_verify_server_certificate_host SSL_CTX_get_verify_cert_with_sni +# define SSL_CTX_set_verify_sni SSL_CTX_set_verify_cert_with_sni +# define SSL_CTX_get_verify_sni SSL_CTX_get_verify_cert_with_sni + +void SSL_CTX_set_verify_cert_with_sni(SSL_CTX *ctx, int value); +int SSL_CTX_get_verify_cert_with_sni(SSL_CTX *ctx); +# endif + +# ifndef OPENSSL_NO_SESSION_REUSED_TYPE +# define SSL_SESSION_REUSED_TYPE_NOCACHE 0 +# define SSL_SESSION_REUSED_TYPE_CACHE 1 +# define SSL_SESSION_REUSED_TYPE_TICKET 2 + +# define SSL_get_session_reuse_type SSL_get_session_reused_type + +int SSL_get_session_reused_type(SSL *s); +# endif + +# ifndef OPENSSL_NO_STATUS +# define OPENSSL_STATUS_ENABLE 1 +/*SSL handshake status*/ +# define SSL_CLIENT_RPOTOCOL 0 +# define SSL_CLIENT_CIPHER 1 +# define SSL_CLIENT_V2_CIPHER 2 +# define SSL_SERVER_EXCHANGE_PUBKEY 3 +# define SSL_CLIENT_RSA_EXCHANGE 4 +# define SSL_CLIENT_SM2_EXCHANGE 5 +# define SSL_SERVER_DH_PUBKEY 6 +# define SSL_CLIENT_SESSION_ID 7 +# define SSL_CLIENT_ECC_CURVES 8 + +struct ssl_status_st { + void *arg; + void *parg; + int type; + int ssl_status_enable; +}; + +void SSL_set_status_callback(SSL *s, + int (*status_callback)(unsigned char *p, + unsigned int length, + SSL_status *param), + unsigned int ssl_status_enable, void *arg); +int (*SSL_get_status_callback(const SSL *s)) (unsigned char *p, + unsigned int length, + SSL_status *param); +# endif + +# ifndef OPENSSL_NO_CERT_COMPRESSION +typedef int (*SSL_cert_compress_cb_fn)(SSL *s, + const unsigned char *in, size_t inlen, + unsigned char *out, size_t *outlen); +typedef int (*SSL_cert_decompress_cb_fn)(SSL *s, + const unsigned char *in, size_t inlen, + unsigned char *out, size_t outlen); + +int SSL_get_cert_compression_compress_id(SSL *s); +int SSL_get_cert_compression_decompress_id(SSL *s); +int SSL_add_cert_compression_alg(SSL *s, int alg_id, + SSL_cert_compress_cb_fn compress, + SSL_cert_decompress_cb_fn decompress); +int SSL_CTX_add_cert_compression_alg(SSL_CTX *ctx, int alg_id, + SSL_cert_compress_cb_fn compress, + SSL_cert_decompress_cb_fn decompress); +# endif # ifdef __cplusplus } diff --git a/openssl/include/openssl/ssl3.h b/openssl/include/openssl/ssl3.h index 4f076c6c9..015948828 100644 --- a/openssl/include/openssl/ssl3.h +++ b/openssl/include/openssl/ssl3.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -43,8 +43,6 @@ extern "C" { # define SSL3_CK_RSA_RC4_40_MD5 0x03000003 # define SSL3_CK_RSA_RC4_128_MD5 0x03000004 # define SSL3_CK_RSA_RC4_128_SHA 0x03000005 -# define SSL3_CK_RSA_RC2_40_MD5 0x03000006 -# define SSL3_CK_RSA_IDEA_128_SHA 0x03000007 # define SSL3_CK_RSA_DES_40_CBC_SHA 0x03000008 # define SSL3_CK_RSA_DES_64_CBC_SHA 0x03000009 # define SSL3_CK_RSA_DES_192_CBC3_SHA 0x0300000A @@ -82,7 +80,6 @@ extern "C" { # define SSL3_RFC_DHE_DSS_DES_192_CBC3_SHA "TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA" # define SSL3_RFC_DHE_RSA_DES_192_CBC3_SHA "TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA" # define SSL3_RFC_ADH_DES_192_CBC_SHA "TLS_DH_anon_WITH_3DES_EDE_CBC_SHA" -# define SSL3_RFC_RSA_IDEA_128_SHA "TLS_RSA_WITH_IDEA_CBC_SHA" # define SSL3_RFC_RSA_RC4_128_MD5 "TLS_RSA_WITH_RC4_128_MD5" # define SSL3_RFC_RSA_RC4_128_SHA "TLS_RSA_WITH_RC4_128_SHA" # define SSL3_RFC_ADH_RC4_128_MD5 "TLS_DH_anon_WITH_RC4_128_MD5" @@ -92,8 +89,6 @@ extern "C" { # define SSL3_TXT_RSA_RC4_40_MD5 "EXP-RC4-MD5" # define SSL3_TXT_RSA_RC4_128_MD5 "RC4-MD5" # define SSL3_TXT_RSA_RC4_128_SHA "RC4-SHA" -# define SSL3_TXT_RSA_RC2_40_MD5 "EXP-RC2-CBC-MD5" -# define SSL3_TXT_RSA_IDEA_128_SHA "IDEA-CBC-SHA" # define SSL3_TXT_RSA_DES_40_CBC_SHA "EXP-DES-CBC-SHA" # define SSL3_TXT_RSA_DES_64_CBC_SHA "DES-CBC-SHA" # define SSL3_TXT_RSA_DES_192_CBC3_SHA "DES-CBC3-SHA" @@ -239,13 +234,6 @@ extern "C" { # define SSL3_RT_HEADER 0x100 # define SSL3_RT_INNER_CONTENT_TYPE 0x101 -/* Pseudo content types for QUIC */ -# define SSL3_RT_QUIC_DATAGRAM 0x200 -# define SSL3_RT_QUIC_PACKET 0x201 -# define SSL3_RT_QUIC_FRAME_FULL 0x202 -# define SSL3_RT_QUIC_FRAME_HEADER 0x203 -# define SSL3_RT_QUIC_FRAME_PADDING 0x204 - # define SSL3_AL_WARNING 1 # define SSL3_AL_FATAL 2 @@ -307,8 +295,6 @@ extern "C" { /* Set if extended master secret extension required on renegotiation */ # define TLS1_FLAGS_REQUIRED_EXTMS 0x1000 -/* 0x2000 is reserved for TLS1_FLAGS_QUIC (internal) */ - # define SSL3_MT_HELLO_REQUEST 0 # define SSL3_MT_CLIENT_HELLO 1 # define SSL3_MT_SERVER_HELLO 2 diff --git a/openssl/include/openssl/sslerr.h b/openssl/include/openssl/sslerr.h index ec35df64e..a99be63f8 100644 --- a/openssl/include/openssl/sslerr.h +++ b/openssl/include/openssl/sslerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,10 +25,10 @@ # define SSL_R_APP_DATA_IN_HANDSHAKE 100 # define SSL_R_ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT 272 # define SSL_R_AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE 158 -# define SSL_R_BAD_CERTIFICATE 348 +# define SSL_R_BAD_CERTIFICATE_SIGNATURE_TYPE 321 +# define SSL_R_BAD_CERTIFICATE_USAGE 322 # define SSL_R_BAD_CHANGE_CIPHER_SPEC 103 # define SSL_R_BAD_CIPHER 186 -# define SSL_R_BAD_COMPRESSION_ALGORITHM 326 # define SSL_R_BAD_DATA 390 # define SSL_R_BAD_DATA_RETURNED_BY_CALLBACK 106 # define SSL_R_BAD_DECOMPRESSION 107 @@ -84,7 +84,6 @@ # define SSL_R_COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE 307 # define SSL_R_COMPRESSION_LIBRARY_ERROR 142 # define SSL_R_CONNECTION_TYPE_NOT_SET 144 -# define SSL_R_CONN_USE_ONLY 356 # define SSL_R_CONTEXT_NOT_DANE_ENABLED 167 # define SSL_R_COOKIE_GEN_CALLBACK_FAILURE 400 # define SSL_R_COOKIE_MISMATCH 308 @@ -103,6 +102,7 @@ # define SSL_R_DANE_TLSA_NULL_DATA 203 # define SSL_R_DATA_BETWEEN_CCS_AND_FINISHED 145 # define SSL_R_DATA_LENGTH_TOO_LONG 146 +# define SSL_R_DC_VALID_TIME_TOO_LARGE 323 # define SSL_R_DECRYPTION_FAILED 147 # define SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC 281 # define SSL_R_DH_KEY_TOO_SMALL 394 @@ -112,8 +112,8 @@ # define SSL_R_DUPLICATE_COMPRESSION_ID 309 # define SSL_R_ECC_CERT_NOT_FOR_SIGNING 318 # define SSL_R_ECDH_REQUIRED_FOR_SUITEB_MODE 374 +# define SSL_R_EE_CERT_NOT_FOUND 326 # define SSL_R_EE_KEY_TOO_SMALL 399 -# define SSL_R_EMPTY_RAW_PUBLIC_KEY 349 # define SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST 354 # define SSL_R_ENCRYPTED_LENGTH_TOO_LONG 150 # define SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST 151 @@ -123,11 +123,10 @@ # define SSL_R_EXTENSION_NOT_RECEIVED 279 # define SSL_R_EXTRA_DATA_IN_MESSAGE 153 # define SSL_R_EXT_LENGTH_MISMATCH 163 -# define SSL_R_FAILED_TO_GET_PARAMETER 316 # define SSL_R_FAILED_TO_INIT_ASYNC 405 -# define SSL_R_FEATURE_NEGOTIATION_NOT_COMPLETE 417 -# define SSL_R_FEATURE_NOT_RENEGOTIABLE 413 +# define SSL_R_FAILED_TO_VERIFY_DC_SIGNATURE 327 # define SSL_R_FRAGMENTED_CLIENT_HELLO 401 +# define SSL_R_GET_SIG_AND_HASH_ERR 333 # define SSL_R_GOT_A_FIN_BEFORE_A_CCS 154 # define SSL_R_HTTPS_PROXY_REQUEST 155 # define SSL_R_HTTP_REQUEST 156 @@ -151,26 +150,24 @@ # define SSL_R_INVALID_KEY_UPDATE_TYPE 120 # define SSL_R_INVALID_MAX_EARLY_DATA 174 # define SSL_R_INVALID_NULL_CMD_NAME 385 -# define SSL_R_INVALID_RAW_PUBLIC_KEY 350 -# define SSL_R_INVALID_RECORD 317 # define SSL_R_INVALID_SEQUENCE_NUMBER 402 # define SSL_R_INVALID_SERVERINFO_DATA 388 # define SSL_R_INVALID_SESSION_ID 999 # define SSL_R_INVALID_SRP_USERNAME 357 # define SSL_R_INVALID_STATUS_RESPONSE 328 # define SSL_R_INVALID_TICKET_KEYS_LENGTH 325 -# define SSL_R_LEGACY_SIGALG_DISALLOWED_OR_UNSUPPORTED 333 # define SSL_R_LENGTH_MISMATCH 159 # define SSL_R_LENGTH_TOO_LONG 404 # define SSL_R_LENGTH_TOO_SHORT 160 # define SSL_R_LIBRARY_BUG 274 # define SSL_R_LIBRARY_HAS_NO_CIPHERS 161 -# define SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED 395 # define SSL_R_MISSING_DSA_SIGNING_CERT 165 # define SSL_R_MISSING_ECDSA_SIGNING_CERT 381 +# define SSL_R_MISSING_ENC_CERTIFICATE 346 # define SSL_R_MISSING_FATAL 256 # define SSL_R_MISSING_PARAMETERS 290 # define SSL_R_MISSING_PSK_KEX_MODES_EXTENSION 310 +# define SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION 313 # define SSL_R_MISSING_RSA_CERTIFICATE 168 # define SSL_R_MISSING_RSA_ENCRYPTING_CERT 169 # define SSL_R_MISSING_RSA_SIGNING_CERT 170 @@ -195,7 +192,6 @@ # define SSL_R_NO_CLIENT_CERT_METHOD 331 # define SSL_R_NO_COMPRESSION_SPECIFIED 187 # define SSL_R_NO_COOKIE_CALLBACK_SET 287 -# define SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER 330 # define SSL_R_NO_METHOD_SPECIFIED 188 # define SSL_R_NO_PEM_EXTENSIONS 389 # define SSL_R_NO_PRIVATE_KEY_ASSIGNED 190 @@ -206,11 +202,9 @@ # define SSL_R_NO_SHARED_GROUPS 410 # define SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS 376 # define SSL_R_NO_SRTP_PROFILES 359 -# define SSL_R_NO_STREAM 355 # define SSL_R_NO_SUITABLE_DIGEST_ALGORITHM 297 # define SSL_R_NO_SUITABLE_GROUPS 295 # define SSL_R_NO_SUITABLE_KEY_SHARE 101 -# define SSL_R_NO_SUITABLE_RECORD_LAYER 322 # define SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM 118 # define SSL_R_NO_VALID_SCTS 216 # define SSL_R_NO_VERIFY_COOKIE_CALLBACK 403 @@ -227,23 +221,16 @@ # define SSL_R_PEM_NAME_BAD_PREFIX 391 # define SSL_R_PEM_NAME_TOO_SHORT 392 # define SSL_R_PIPELINE_FAILURE 406 -# define SSL_R_POLL_REQUEST_NOT_SUPPORTED 418 # define SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR 278 # define SSL_R_PRIVATE_KEY_MISMATCH 288 # define SSL_R_PROTOCOL_IS_SHUTDOWN 207 # define SSL_R_PSK_IDENTITY_NOT_FOUND 223 # define SSL_R_PSK_NO_CLIENT_CB 224 # define SSL_R_PSK_NO_SERVER_CB 225 -# define SSL_R_QUIC_HANDSHAKE_LAYER_ERROR 393 -# define SSL_R_QUIC_NETWORK_ERROR 387 -# define SSL_R_QUIC_PROTOCOL_ERROR 382 # define SSL_R_READ_BIO_NOT_SET 211 # define SSL_R_READ_TIMEOUT_EXPIRED 312 -# define SSL_R_RECORDS_NOT_RELEASED 321 -# define SSL_R_RECORD_LAYER_FAILURE 313 # define SSL_R_RECORD_LENGTH_MISMATCH 213 # define SSL_R_RECORD_TOO_SMALL 298 -# define SSL_R_REMOTE_PEER_ADDRESS_NOT_SET 346 # define SSL_R_RENEGOTIATE_EXT_TOO_LONG 335 # define SSL_R_RENEGOTIATION_ENCODING_ERR 336 # define SSL_R_RENEGOTIATION_MISMATCH 337 @@ -253,7 +240,6 @@ # define SSL_R_REQUIRED_COMPRESSION_ALGORITHM_MISSING 342 # define SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING 345 # define SSL_R_SCT_VERIFICATION_FAILED 208 -# define SSL_R_SEQUENCE_CTR_WRAPPED 327 # define SSL_R_SERVERHELLO_TLSEXT 275 # define SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED 277 # define SSL_R_SHUTDOWN_WHILE_IN_INIT 407 @@ -292,12 +278,8 @@ # define SSL_R_SSL_SESSION_ID_HAS_BAD_LENGTH 303 # define SSL_R_SSL_SESSION_ID_TOO_LONG 408 # define SSL_R_SSL_SESSION_VERSION_MISMATCH 210 +# define SSL_R_STATUS_CALLBACK_ERROR 317 # define SSL_R_STILL_IN_INIT 121 -# define SSL_R_STREAM_COUNT_LIMITED 411 -# define SSL_R_STREAM_FINISHED 365 -# define SSL_R_STREAM_RECV_ONLY 366 -# define SSL_R_STREAM_RESET 375 -# define SSL_R_STREAM_SEND_ONLY 379 # define SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED 1116 # define SSL_R_TLSV13_ALERT_MISSING_EXTENSION 1109 # define SSL_R_TLSV1_ALERT_ACCESS_DENIED 1049 @@ -308,12 +290,10 @@ # define SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK 1086 # define SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY 1071 # define SSL_R_TLSV1_ALERT_INTERNAL_ERROR 1080 -# define SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL 1120 # define SSL_R_TLSV1_ALERT_NO_RENEGOTIATION 1100 # define SSL_R_TLSV1_ALERT_PROTOCOL_VERSION 1070 # define SSL_R_TLSV1_ALERT_RECORD_OVERFLOW 1022 # define SSL_R_TLSV1_ALERT_UNKNOWN_CA 1048 -# define SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY 1115 # define SSL_R_TLSV1_ALERT_USER_CANCELLED 1090 # define SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE 1114 # define SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE 1113 @@ -329,6 +309,7 @@ # define SSL_R_UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS 239 # define SSL_R_UNABLE_TO_LOAD_SSL3_MD5_ROUTINES 242 # define SSL_R_UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES 243 +# define SSL_R_UNABLE_TO_LOOKUP_CERT 348 # define SSL_R_UNEXPECTED_CCS_MESSAGE 262 # define SSL_R_UNEXPECTED_END_OF_EARLY_DATA 178 # define SSL_R_UNEXPECTED_EOF_WHILE_READING 294 @@ -343,7 +324,6 @@ # define SSL_R_UNKNOWN_COMMAND 139 # define SSL_R_UNKNOWN_DIGEST 368 # define SSL_R_UNKNOWN_KEY_EXCHANGE_TYPE 250 -# define SSL_R_UNKNOWN_MANDATORY_PARAMETER 323 # define SSL_R_UNKNOWN_PKEY_TYPE 251 # define SSL_R_UNKNOWN_PROTOCOL 252 # define SSL_R_UNKNOWN_SSL_VERSION 254 @@ -351,21 +331,18 @@ # define SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED 338 # define SSL_R_UNSOLICITED_EXTENSION 217 # define SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM 257 -# define SSL_R_UNSUPPORTED_CONFIG_VALUE 414 -# define SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS 415 -# define SSL_R_UNSUPPORTED_CONFIG_VALUE_OP 416 # define SSL_R_UNSUPPORTED_ELLIPTIC_CURVE 315 # define SSL_R_UNSUPPORTED_PROTOCOL 258 # define SSL_R_UNSUPPORTED_SSL_VERSION 259 # define SSL_R_UNSUPPORTED_STATUS_TYPE 329 -# define SSL_R_UNSUPPORTED_WRITE_FLAG 412 # define SSL_R_USE_SRTP_NOT_NEGOTIATED 369 # define SSL_R_VERSION_TOO_HIGH 166 # define SSL_R_VERSION_TOO_LOW 396 # define SSL_R_WRONG_CERTIFICATE_TYPE 383 # define SSL_R_WRONG_CIPHER_RETURNED 261 # define SSL_R_WRONG_CURVE 378 -# define SSL_R_WRONG_RPK_TYPE 351 +# define SSL_R_WRONG_ENCRYPTION_LEVEL_RECEIVED 316 +# define SSL_R_WRONG_PUBLIC_KEY_TYPE 349 # define SSL_R_WRONG_SIGNATURE_LENGTH 264 # define SSL_R_WRONG_SIGNATURE_SIZE 265 # define SSL_R_WRONG_SIGNATURE_TYPE 370 diff --git a/openssl/include/openssl/sslerr_legacy.h b/openssl/include/openssl/sslerr_legacy.h index 4c353671c..afb4c0b46 100644 --- a/openssl/include/openssl/sslerr_legacy.h +++ b/openssl/include/openssl/sslerr_legacy.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -143,6 +143,7 @@ OSSL_DEPRECATEDIN_3_0 int ERR_load_SSL_strings(void); # define SSL_F_SSL_ADD_SERVERHELLO_RENEGOTIATE_EXT 0 # define SSL_F_SSL_ADD_SERVERHELLO_TLSEXT 0 # define SSL_F_SSL_ADD_SERVERHELLO_USE_SRTP_EXT 0 +# define SSL_F_SSL_BAD_METHOD 0 # define SSL_F_SSL_BUILD_CERT_CHAIN 0 # define SSL_F_SSL_BYTES_TO_CIPHER_LIST 0 # define SSL_F_SSL_CACHE_CIPHERLIST 0 @@ -304,7 +305,6 @@ OSSL_DEPRECATEDIN_3_0 int ERR_load_SSL_strings(void); # define SSL_F_TLS_CONSTRUCT_CHANGE_CIPHER_SPEC 0 # define SSL_F_TLS_CONSTRUCT_CKE_DHE 0 # define SSL_F_TLS_CONSTRUCT_CKE_ECDHE 0 -# define SSL_F_TLS_CONSTRUCT_CKE_GOST 0 # define SSL_F_TLS_CONSTRUCT_CKE_PSK_PREAMBLE 0 # define SSL_F_TLS_CONSTRUCT_CKE_RSA 0 # define SSL_F_TLS_CONSTRUCT_CKE_SRP 0 @@ -354,7 +354,6 @@ OSSL_DEPRECATEDIN_3_0 int ERR_load_SSL_strings(void); # define SSL_F_TLS_CONSTRUCT_STOC_ALPN 0 # define SSL_F_TLS_CONSTRUCT_STOC_CERTIFICATE 0 # define SSL_F_TLS_CONSTRUCT_STOC_COOKIE 0 -# define SSL_F_TLS_CONSTRUCT_STOC_CRYPTOPRO_BUG 0 # define SSL_F_TLS_CONSTRUCT_STOC_DONE 0 # define SSL_F_TLS_CONSTRUCT_STOC_EARLY_DATA 0 # define SSL_F_TLS_CONSTRUCT_STOC_EARLY_DATA_INFO 0 @@ -428,7 +427,6 @@ OSSL_DEPRECATEDIN_3_0 int ERR_load_SSL_strings(void); # define SSL_F_TLS_PROCESS_CHANGE_CIPHER_SPEC 0 # define SSL_F_TLS_PROCESS_CKE_DHE 0 # define SSL_F_TLS_PROCESS_CKE_ECDHE 0 -# define SSL_F_TLS_PROCESS_CKE_GOST 0 # define SSL_F_TLS_PROCESS_CKE_PSK_PREAMBLE 0 # define SSL_F_TLS_PROCESS_CKE_RSA 0 # define SSL_F_TLS_PROCESS_CKE_SRP 0 diff --git a/openssl/include/openssl/store.h b/openssl/include/openssl/store.h index e6ea3cf87..3c1445e0e 100644 --- a/openssl/include/openssl/store.h +++ b/openssl/include/openssl/store.h @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -98,14 +98,6 @@ OSSL_DEPRECATEDIN_3_0 int OSSL_STORE_vctrl(OSSL_STORE_CTX *ctx, int cmd, */ OSSL_STORE_INFO *OSSL_STORE_load(OSSL_STORE_CTX *ctx); -/* - * Deletes the object in the store by URI. - * Returns 1 on success, 0 otherwise. - */ -int OSSL_STORE_delete(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, - const UI_METHOD *ui_method, void *ui_data, - const OSSL_PARAM params[]); - /* * Check if end of data (end of file) is reached * Returns 1 on end, 0 otherwise. @@ -353,7 +345,7 @@ int OSSL_STORE_LOADER_set_close(OSSL_STORE_LOADER *loader, OSSL_DEPRECATEDIN_3_0 const ENGINE *OSSL_STORE_LOADER_get0_engine(const OSSL_STORE_LOADER *loader); OSSL_DEPRECATEDIN_3_0 -const char *OSSL_STORE_LOADER_get0_scheme(const OSSL_STORE_LOADER *loader); +const char * OSSL_STORE_LOADER_get0_scheme(const OSSL_STORE_LOADER *loader); OSSL_DEPRECATEDIN_3_0 int OSSL_STORE_register_loader(OSSL_STORE_LOADER *loader); OSSL_DEPRECATEDIN_3_0 diff --git a/openssl/include/openssl/symbol_prefix.h b/openssl/include/openssl/symbol_prefix.h new file mode 100644 index 000000000..52585de3a --- /dev/null +++ b/openssl/include/openssl/symbol_prefix.h @@ -0,0 +1,11 @@ +#ifndef HEADER_SYMBOL_PREFIX_H +# define HEADER_SYMBOL_PREFIX_H + +# define SYMBOL_PREFIX "" + +/***************PARSED SYMBOLS***************/ + + +/***************CUSTOM SYMBOLS***************/ + +#endif /* HEADER_SYMBOL_PREFIX_H */ diff --git a/openssl/include/openssl/symhacks.h b/openssl/include/openssl/symhacks.h index 816f8f998..8af21c003 100644 --- a/openssl/include/openssl/symhacks.h +++ b/openssl/include/openssl/symhacks.h @@ -18,22 +18,4 @@ # include -/* Case insensitive linking causes problems.... */ -# if defined(OPENSSL_SYS_VMS) -# undef ERR_load_CRYPTO_strings -# define ERR_load_CRYPTO_strings ERR_load_CRYPTOlib_strings -# undef OCSP_crlID_new -# define OCSP_crlID_new OCSP_crlID2_new - -# undef d2i_ECPARAMETERS -# define d2i_ECPARAMETERS d2i_UC_ECPARAMETERS -# undef i2d_ECPARAMETERS -# define i2d_ECPARAMETERS i2d_UC_ECPARAMETERS -# undef d2i_ECPKPARAMETERS -# define d2i_ECPKPARAMETERS d2i_UC_ECPKPARAMETERS -# undef i2d_ECPKPARAMETERS -# define i2d_ECPKPARAMETERS i2d_UC_ECPKPARAMETERS - -# endif - #endif /* ! defined HEADER_VMS_IDHACKS_H */ diff --git a/openssl/include/openssl/thread.h b/openssl/include/openssl/thread.h deleted file mode 100644 index 3926ce54d..000000000 --- a/openssl/include/openssl/thread.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. - * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_THREAD_H -# define OPENSSL_THREAD_H - -# define OSSL_THREAD_SUPPORT_FLAG_THREAD_POOL (1U<<0) -# define OSSL_THREAD_SUPPORT_FLAG_DEFAULT_SPAWN (1U<<1) - -# include - -# ifdef __cplusplus -extern "C" { -# endif - -uint32_t OSSL_get_thread_support_flags(void); -int OSSL_set_max_threads(OSSL_LIB_CTX *ctx, uint64_t max_threads); -uint64_t OSSL_get_max_threads(OSSL_LIB_CTX *ctx); - -# ifdef __cplusplus -} -# endif - -#endif /* OPENSSL_THREAD_H */ diff --git a/openssl/include/openssl/tls1.h b/openssl/include/openssl/tls1.h index 7e3d1a725..fe485490d 100644 --- a/openssl/include/openssl/tls1.h +++ b/openssl/include/openssl/tls1.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -28,7 +28,7 @@ extern "C" { /* Default security level if not overridden at config time */ # ifndef OPENSSL_TLS_SECURITY_LEVEL -# define OPENSSL_TLS_SECURITY_LEVEL 2 +# define OPENSSL_TLS_SECURITY_LEVEL 1 # endif /* TLS*_VERSION constants are defined in prov_ssl.h */ @@ -122,14 +122,6 @@ extern "C" { */ # define TLSEXT_TYPE_signed_certificate_timestamp 18 -/* - * Extension type for Raw Public Keys - * https://tools.ietf.org/html/rfc7250 - * https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml - */ -# define TLSEXT_TYPE_client_cert_type 19 -# define TLSEXT_TYPE_server_cert_type 20 - /* * ExtensionType value for TLS padding extension. * http://tools.ietf.org/html/draft-agl-tls-padding @@ -145,6 +137,9 @@ extern "C" { /* ExtensionType value from RFC8879 */ # define TLSEXT_TYPE_compress_certificate 27 +/* ExtensionType value from draft-ietf-tls-subcerts-10 */ +# define TLSEXT_TYPE_delegated_credential 34 + /* ExtensionType value from RFC4507 */ # define TLSEXT_TYPE_session_ticket 35 @@ -158,11 +153,14 @@ extern "C" { # define TLSEXT_TYPE_post_handshake_auth 49 # define TLSEXT_TYPE_signature_algorithms_cert 50 # define TLSEXT_TYPE_key_share 51 -# define TLSEXT_TYPE_quic_transport_parameters 57 /* Temporary extension type */ # define TLSEXT_TYPE_renegotiate 0xff01 +/* ExtensionType value from draft-ietf-quic-tls-27 */ +# define TLSEXT_TYPE_quic_transport_parameters_draft 0xffa5 +# define TLSEXT_TYPE_quic_transport_parameters 0x0039 + # ifndef OPENSSL_NO_NEXTPROTONEG /* This is not an IANA defined extension number */ # define TLSEXT_TYPE_next_proto_neg 13172 @@ -185,9 +183,6 @@ extern "C" { # define TLSEXT_signature_rsa 1 # define TLSEXT_signature_dsa 2 # define TLSEXT_signature_ecdsa 3 -# define TLSEXT_signature_gostr34102001 237 -# define TLSEXT_signature_gostr34102012_256 238 -# define TLSEXT_signature_gostr34102012_512 239 /* Total number of different signature algorithms */ # define TLSEXT_signature_num 7 @@ -199,23 +194,11 @@ extern "C" { # define TLSEXT_hash_sha256 4 # define TLSEXT_hash_sha384 5 # define TLSEXT_hash_sha512 6 -# define TLSEXT_hash_gostr3411 237 -# define TLSEXT_hash_gostr34112012_256 238 -# define TLSEXT_hash_gostr34112012_512 239 /* Total number of different digest algorithms */ # define TLSEXT_hash_num 10 -/* Possible compression values from RFC8879 */ -/* Not defined in RFC8879, but used internally for no-compression */ -# define TLSEXT_comp_cert_none 0 -# define TLSEXT_comp_cert_zlib 1 -# define TLSEXT_comp_cert_brotli 2 -# define TLSEXT_comp_cert_zstd 3 -/* one more than the number of defined values - used as size of 0-terminated array */ -# define TLSEXT_comp_cert_limit 4 - /* Flag set for unrecognised algorithms */ # define TLSEXT_nid_unknown 0x1000000 @@ -224,6 +207,9 @@ extern "C" { # define TLSEXT_curve_P_256 23 # define TLSEXT_curve_P_384 24 +/* defined in RFC 8998 */ +# define TLSEXT_curve_SM2 41 + /* OpenSSL value to disable maximum fragment length extension */ # define TLSEXT_max_fragment_length_DISABLED 0 /* Allowed values for max fragment length extension */ @@ -231,15 +217,10 @@ extern "C" { # define TLSEXT_max_fragment_length_1024 2 # define TLSEXT_max_fragment_length_2048 3 # define TLSEXT_max_fragment_length_4096 4 - -/* - * TLS Certificate Type (for RFC7250) - * https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#tls-extensiontype-values-3 - */ -# define TLSEXT_cert_type_x509 0 -# define TLSEXT_cert_type_pgp 1 /* recognized, but not supported */ -# define TLSEXT_cert_type_rpk 2 -# define TLSEXT_cert_type_1609dot2 3 /* recognized, but not supported */ +/* TLS Certificate Compression Algorithm IDs from RFC8879 */ +# define TLSEXT_cert_compression_zlib 1 +# define TLSEXT_cert_compression_brotli 2 +# define TLSEXT_cert_compression_zstd 3 int SSL_CTX_set_tlsext_max_fragment_length(SSL_CTX *ctx, uint8_t mode); int SSL_set_tlsext_max_fragment_length(SSL *ssl, uint8_t mode); @@ -425,14 +406,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_CK_DH_RSA_WITH_AES_128_SHA256 0x0300003F # define TLS1_CK_DHE_DSS_WITH_AES_128_SHA256 0x03000040 -/* Camellia ciphersuites from RFC4132 */ -# define TLS1_CK_RSA_WITH_CAMELLIA_128_CBC_SHA 0x03000041 -# define TLS1_CK_DH_DSS_WITH_CAMELLIA_128_CBC_SHA 0x03000042 -# define TLS1_CK_DH_RSA_WITH_CAMELLIA_128_CBC_SHA 0x03000043 -# define TLS1_CK_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA 0x03000044 -# define TLS1_CK_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA 0x03000045 -# define TLS1_CK_ADH_WITH_CAMELLIA_128_CBC_SHA 0x03000046 - /* TLS v1.2 ciphersuites */ # define TLS1_CK_DHE_RSA_WITH_AES_128_SHA256 0x03000067 # define TLS1_CK_DH_DSS_WITH_AES_256_SHA256 0x03000068 @@ -442,22 +415,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_CK_ADH_WITH_AES_128_SHA256 0x0300006C # define TLS1_CK_ADH_WITH_AES_256_SHA256 0x0300006D -/* Camellia ciphersuites from RFC4132 */ -# define TLS1_CK_RSA_WITH_CAMELLIA_256_CBC_SHA 0x03000084 -# define TLS1_CK_DH_DSS_WITH_CAMELLIA_256_CBC_SHA 0x03000085 -# define TLS1_CK_DH_RSA_WITH_CAMELLIA_256_CBC_SHA 0x03000086 -# define TLS1_CK_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA 0x03000087 -# define TLS1_CK_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA 0x03000088 -# define TLS1_CK_ADH_WITH_CAMELLIA_256_CBC_SHA 0x03000089 - -/* SEED ciphersuites from RFC4162 */ -# define TLS1_CK_RSA_WITH_SEED_SHA 0x03000096 -# define TLS1_CK_DH_DSS_WITH_SEED_SHA 0x03000097 -# define TLS1_CK_DH_RSA_WITH_SEED_SHA 0x03000098 -# define TLS1_CK_DHE_DSS_WITH_SEED_SHA 0x03000099 -# define TLS1_CK_DHE_RSA_WITH_SEED_SHA 0x0300009A -# define TLS1_CK_ADH_WITH_SEED_SHA 0x0300009B - /* TLS v1.2 GCM ciphersuites from RFC5288 */ # define TLS1_CK_RSA_WITH_AES_128_GCM_SHA256 0x0300009C # define TLS1_CK_RSA_WITH_AES_256_GCM_SHA384 0x0300009D @@ -496,21 +453,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_CK_ECDHE_ECDSA_WITH_AES_128_CCM_8 0x0300C0AE # define TLS1_CK_ECDHE_ECDSA_WITH_AES_256_CCM_8 0x0300C0AF -/* TLS 1.2 Camellia SHA-256 ciphersuites from RFC5932 */ -# define TLS1_CK_RSA_WITH_CAMELLIA_128_CBC_SHA256 0x030000BA -# define TLS1_CK_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 0x030000BB -# define TLS1_CK_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 0x030000BC -# define TLS1_CK_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 0x030000BD -# define TLS1_CK_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 0x030000BE -# define TLS1_CK_ADH_WITH_CAMELLIA_128_CBC_SHA256 0x030000BF - -# define TLS1_CK_RSA_WITH_CAMELLIA_256_CBC_SHA256 0x030000C0 -# define TLS1_CK_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 0x030000C1 -# define TLS1_CK_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 0x030000C2 -# define TLS1_CK_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 0x030000C3 -# define TLS1_CK_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 0x030000C4 -# define TLS1_CK_ADH_WITH_CAMELLIA_256_CBC_SHA256 0x030000C5 - /* ECC ciphersuites from RFC4492 */ # define TLS1_CK_ECDH_ECDSA_WITH_NULL_SHA 0x0300C001 # define TLS1_CK_ECDH_ECDSA_WITH_RC4_128_SHA 0x0300C002 @@ -587,25 +529,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_CK_ECDHE_PSK_WITH_NULL_SHA256 0x0300C03A # define TLS1_CK_ECDHE_PSK_WITH_NULL_SHA384 0x0300C03B -/* Camellia-CBC ciphersuites from RFC6367 */ -# define TLS1_CK_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 0x0300C072 -# define TLS1_CK_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 0x0300C073 -# define TLS1_CK_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 0x0300C074 -# define TLS1_CK_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 0x0300C075 -# define TLS1_CK_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 0x0300C076 -# define TLS1_CK_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 0x0300C077 -# define TLS1_CK_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 0x0300C078 -# define TLS1_CK_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 0x0300C079 - -# define TLS1_CK_PSK_WITH_CAMELLIA_128_CBC_SHA256 0x0300C094 -# define TLS1_CK_PSK_WITH_CAMELLIA_256_CBC_SHA384 0x0300C095 -# define TLS1_CK_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 0x0300C096 -# define TLS1_CK_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 0x0300C097 -# define TLS1_CK_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 0x0300C098 -# define TLS1_CK_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 0x0300C099 -# define TLS1_CK_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 0x0300C09A -# define TLS1_CK_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 0x0300C09B - /* draft-ietf-tls-chacha20-poly1305-03 */ # define TLS1_CK_ECDHE_RSA_WITH_CHACHA20_POLY1305 0x0300CCA8 # define TLS1_CK_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 0x0300CCA9 @@ -621,34 +544,8 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_3_CK_CHACHA20_POLY1305_SHA256 0x03001303 # define TLS1_3_CK_AES_128_CCM_SHA256 0x03001304 # define TLS1_3_CK_AES_128_CCM_8_SHA256 0x03001305 - -/* Aria ciphersuites from RFC6209 */ -# define TLS1_CK_RSA_WITH_ARIA_128_GCM_SHA256 0x0300C050 -# define TLS1_CK_RSA_WITH_ARIA_256_GCM_SHA384 0x0300C051 -# define TLS1_CK_DHE_RSA_WITH_ARIA_128_GCM_SHA256 0x0300C052 -# define TLS1_CK_DHE_RSA_WITH_ARIA_256_GCM_SHA384 0x0300C053 -# define TLS1_CK_DH_RSA_WITH_ARIA_128_GCM_SHA256 0x0300C054 -# define TLS1_CK_DH_RSA_WITH_ARIA_256_GCM_SHA384 0x0300C055 -# define TLS1_CK_DHE_DSS_WITH_ARIA_128_GCM_SHA256 0x0300C056 -# define TLS1_CK_DHE_DSS_WITH_ARIA_256_GCM_SHA384 0x0300C057 -# define TLS1_CK_DH_DSS_WITH_ARIA_128_GCM_SHA256 0x0300C058 -# define TLS1_CK_DH_DSS_WITH_ARIA_256_GCM_SHA384 0x0300C059 -# define TLS1_CK_DH_anon_WITH_ARIA_128_GCM_SHA256 0x0300C05A -# define TLS1_CK_DH_anon_WITH_ARIA_256_GCM_SHA384 0x0300C05B -# define TLS1_CK_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 0x0300C05C -# define TLS1_CK_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 0x0300C05D -# define TLS1_CK_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 0x0300C05E -# define TLS1_CK_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 0x0300C05F -# define TLS1_CK_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 0x0300C060 -# define TLS1_CK_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 0x0300C061 -# define TLS1_CK_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 0x0300C062 -# define TLS1_CK_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 0x0300C063 -# define TLS1_CK_PSK_WITH_ARIA_128_GCM_SHA256 0x0300C06A -# define TLS1_CK_PSK_WITH_ARIA_256_GCM_SHA384 0x0300C06B -# define TLS1_CK_DHE_PSK_WITH_ARIA_128_GCM_SHA256 0x0300C06C -# define TLS1_CK_DHE_PSK_WITH_ARIA_256_GCM_SHA384 0x0300C06D -# define TLS1_CK_RSA_PSK_WITH_ARIA_128_GCM_SHA256 0x0300C06E -# define TLS1_CK_RSA_PSK_WITH_ARIA_256_GCM_SHA384 0x0300C06F +# define TLS1_3_CK_SM4_GCM_SM3 0x030000C6 +# define TLS1_3_CK_SM4_CCM_SM3 0x030000C7 /* a bundle of RFC standard cipher names, generated from ssl3_ciphers[] */ # define TLS1_RFC_RSA_WITH_AES_128_SHA "TLS_RSA_WITH_AES_128_CBC_SHA" @@ -701,6 +598,8 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_3_RFC_CHACHA20_POLY1305_SHA256 "TLS_CHACHA20_POLY1305_SHA256" # define TLS1_3_RFC_AES_128_CCM_SHA256 "TLS_AES_128_CCM_SHA256" # define TLS1_3_RFC_AES_128_CCM_8_SHA256 "TLS_AES_128_CCM_8_SHA256" +# define TLS1_3_RFC_SM4_CCM_SM3 "TLS_SM4_CCM_SM3" +# define TLS1_3_RFC_SM4_GCM_SM3 "TLS_SM4_GCM_SM3" # define TLS1_RFC_ECDHE_ECDSA_WITH_NULL_SHA "TLS_ECDHE_ECDSA_WITH_NULL_SHA" # define TLS1_RFC_ECDHE_ECDSA_WITH_DES_192_CBC3_SHA "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA" # define TLS1_RFC_ECDHE_ECDSA_WITH_AES_128_CBC_SHA "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA" @@ -775,38 +674,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_RFC_ECDHE_PSK_WITH_CHACHA20_POLY1305 "TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256" # define TLS1_RFC_DHE_PSK_WITH_CHACHA20_POLY1305 "TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256" # define TLS1_RFC_RSA_PSK_WITH_CHACHA20_POLY1305 "TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256" -# define TLS1_RFC_RSA_WITH_CAMELLIA_128_CBC_SHA256 "TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 "TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 "TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_ADH_WITH_CAMELLIA_128_CBC_SHA256 "TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_RSA_WITH_CAMELLIA_256_CBC_SHA256 "TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256" -# define TLS1_RFC_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 "TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256" -# define TLS1_RFC_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 "TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256" -# define TLS1_RFC_ADH_WITH_CAMELLIA_256_CBC_SHA256 "TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256" -# define TLS1_RFC_RSA_WITH_CAMELLIA_256_CBC_SHA "TLS_RSA_WITH_CAMELLIA_256_CBC_SHA" -# define TLS1_RFC_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA "TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA" -# define TLS1_RFC_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA "TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA" -# define TLS1_RFC_ADH_WITH_CAMELLIA_256_CBC_SHA "TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA" -# define TLS1_RFC_RSA_WITH_CAMELLIA_128_CBC_SHA "TLS_RSA_WITH_CAMELLIA_128_CBC_SHA" -# define TLS1_RFC_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA "TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA" -# define TLS1_RFC_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA "TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA" -# define TLS1_RFC_ADH_WITH_CAMELLIA_128_CBC_SHA "TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA" -# define TLS1_RFC_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 "TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 "TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384" -# define TLS1_RFC_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 "TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 "TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384" -# define TLS1_RFC_PSK_WITH_CAMELLIA_128_CBC_SHA256 "TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_PSK_WITH_CAMELLIA_256_CBC_SHA384 "TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384" -# define TLS1_RFC_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 "TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 "TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384" -# define TLS1_RFC_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 "TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 "TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384" -# define TLS1_RFC_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 "TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256" -# define TLS1_RFC_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 "TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384" -# define TLS1_RFC_RSA_WITH_SEED_SHA "TLS_RSA_WITH_SEED_CBC_SHA" -# define TLS1_RFC_DHE_DSS_WITH_SEED_SHA "TLS_DHE_DSS_WITH_SEED_CBC_SHA" -# define TLS1_RFC_DHE_RSA_WITH_SEED_SHA "TLS_DHE_RSA_WITH_SEED_CBC_SHA" -# define TLS1_RFC_ADH_WITH_SEED_SHA "TLS_DH_anon_WITH_SEED_CBC_SHA" # define TLS1_RFC_ECDHE_PSK_WITH_RC4_128_SHA "TLS_ECDHE_PSK_WITH_RC4_128_SHA" # define TLS1_RFC_ECDH_anon_WITH_RC4_128_SHA "TLS_ECDH_anon_WITH_RC4_128_SHA" # define TLS1_RFC_ECDHE_ECDSA_WITH_RC4_128_SHA "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA" @@ -814,32 +681,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_RFC_PSK_WITH_RC4_128_SHA "TLS_PSK_WITH_RC4_128_SHA" # define TLS1_RFC_RSA_PSK_WITH_RC4_128_SHA "TLS_RSA_PSK_WITH_RC4_128_SHA" # define TLS1_RFC_DHE_PSK_WITH_RC4_128_SHA "TLS_DHE_PSK_WITH_RC4_128_SHA" -# define TLS1_RFC_RSA_WITH_ARIA_128_GCM_SHA256 "TLS_RSA_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_RSA_WITH_ARIA_256_GCM_SHA384 "TLS_RSA_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_DHE_RSA_WITH_ARIA_128_GCM_SHA256 "TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_DHE_RSA_WITH_ARIA_256_GCM_SHA384 "TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_DH_RSA_WITH_ARIA_128_GCM_SHA256 "TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_DH_RSA_WITH_ARIA_256_GCM_SHA384 "TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_DHE_DSS_WITH_ARIA_128_GCM_SHA256 "TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_DHE_DSS_WITH_ARIA_256_GCM_SHA384 "TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_DH_DSS_WITH_ARIA_128_GCM_SHA256 "TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_DH_DSS_WITH_ARIA_256_GCM_SHA384 "TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_DH_anon_WITH_ARIA_128_GCM_SHA256 "TLS_DH_anon_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_DH_anon_WITH_ARIA_256_GCM_SHA384 "TLS_DH_anon_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 "TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 "TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 "TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 "TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 "TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 "TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 "TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 "TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_PSK_WITH_ARIA_128_GCM_SHA256 "TLS_PSK_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_PSK_WITH_ARIA_256_GCM_SHA384 "TLS_PSK_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_DHE_PSK_WITH_ARIA_128_GCM_SHA256 "TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_DHE_PSK_WITH_ARIA_256_GCM_SHA384 "TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384" -# define TLS1_RFC_RSA_PSK_WITH_ARIA_128_GCM_SHA256 "TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256" -# define TLS1_RFC_RSA_PSK_WITH_ARIA_256_GCM_SHA384 "TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384" /* @@ -949,53 +790,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_TXT_SRP_SHA_RSA_WITH_AES_256_CBC_SHA "SRP-RSA-AES-256-CBC-SHA" # define TLS1_TXT_SRP_SHA_DSS_WITH_AES_256_CBC_SHA "SRP-DSS-AES-256-CBC-SHA" -/* Camellia ciphersuites from RFC4132 */ -# define TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA "CAMELLIA128-SHA" -# define TLS1_TXT_DH_DSS_WITH_CAMELLIA_128_CBC_SHA "DH-DSS-CAMELLIA128-SHA" -# define TLS1_TXT_DH_RSA_WITH_CAMELLIA_128_CBC_SHA "DH-RSA-CAMELLIA128-SHA" -# define TLS1_TXT_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA "DHE-DSS-CAMELLIA128-SHA" -# define TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA "DHE-RSA-CAMELLIA128-SHA" -# define TLS1_TXT_ADH_WITH_CAMELLIA_128_CBC_SHA "ADH-CAMELLIA128-SHA" - -# define TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA "CAMELLIA256-SHA" -# define TLS1_TXT_DH_DSS_WITH_CAMELLIA_256_CBC_SHA "DH-DSS-CAMELLIA256-SHA" -# define TLS1_TXT_DH_RSA_WITH_CAMELLIA_256_CBC_SHA "DH-RSA-CAMELLIA256-SHA" -# define TLS1_TXT_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA "DHE-DSS-CAMELLIA256-SHA" -# define TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA "DHE-RSA-CAMELLIA256-SHA" -# define TLS1_TXT_ADH_WITH_CAMELLIA_256_CBC_SHA "ADH-CAMELLIA256-SHA" - -/* TLS 1.2 Camellia SHA-256 ciphersuites from RFC5932 */ -# define TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA256 "CAMELLIA128-SHA256" -# define TLS1_TXT_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 "DH-DSS-CAMELLIA128-SHA256" -# define TLS1_TXT_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 "DH-RSA-CAMELLIA128-SHA256" -# define TLS1_TXT_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 "DHE-DSS-CAMELLIA128-SHA256" -# define TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 "DHE-RSA-CAMELLIA128-SHA256" -# define TLS1_TXT_ADH_WITH_CAMELLIA_128_CBC_SHA256 "ADH-CAMELLIA128-SHA256" - -# define TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA256 "CAMELLIA256-SHA256" -# define TLS1_TXT_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 "DH-DSS-CAMELLIA256-SHA256" -# define TLS1_TXT_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 "DH-RSA-CAMELLIA256-SHA256" -# define TLS1_TXT_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 "DHE-DSS-CAMELLIA256-SHA256" -# define TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 "DHE-RSA-CAMELLIA256-SHA256" -# define TLS1_TXT_ADH_WITH_CAMELLIA_256_CBC_SHA256 "ADH-CAMELLIA256-SHA256" - -# define TLS1_TXT_PSK_WITH_CAMELLIA_128_CBC_SHA256 "PSK-CAMELLIA128-SHA256" -# define TLS1_TXT_PSK_WITH_CAMELLIA_256_CBC_SHA384 "PSK-CAMELLIA256-SHA384" -# define TLS1_TXT_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 "DHE-PSK-CAMELLIA128-SHA256" -# define TLS1_TXT_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 "DHE-PSK-CAMELLIA256-SHA384" -# define TLS1_TXT_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 "RSA-PSK-CAMELLIA128-SHA256" -# define TLS1_TXT_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 "RSA-PSK-CAMELLIA256-SHA384" -# define TLS1_TXT_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 "ECDHE-PSK-CAMELLIA128-SHA256" -# define TLS1_TXT_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 "ECDHE-PSK-CAMELLIA256-SHA384" - -/* SEED ciphersuites from RFC4162 */ -# define TLS1_TXT_RSA_WITH_SEED_SHA "SEED-SHA" -# define TLS1_TXT_DH_DSS_WITH_SEED_SHA "DH-DSS-SEED-SHA" -# define TLS1_TXT_DH_RSA_WITH_SEED_SHA "DH-RSA-SEED-SHA" -# define TLS1_TXT_DHE_DSS_WITH_SEED_SHA "DHE-DSS-SEED-SHA" -# define TLS1_TXT_DHE_RSA_WITH_SEED_SHA "DHE-RSA-SEED-SHA" -# define TLS1_TXT_ADH_WITH_SEED_SHA "ADH-SEED-SHA" - /* TLS v1.2 ciphersuites */ # define TLS1_TXT_RSA_WITH_NULL_SHA256 "NULL-SHA256" # define TLS1_TXT_RSA_WITH_AES_128_SHA256 "AES128-SHA256" @@ -1089,16 +883,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_TXT_ECDHE_PSK_WITH_NULL_SHA256 "ECDHE-PSK-NULL-SHA256" # define TLS1_TXT_ECDHE_PSK_WITH_NULL_SHA384 "ECDHE-PSK-NULL-SHA384" -/* Camellia-CBC ciphersuites from RFC6367 */ -# define TLS1_TXT_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 "ECDHE-ECDSA-CAMELLIA128-SHA256" -# define TLS1_TXT_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 "ECDHE-ECDSA-CAMELLIA256-SHA384" -# define TLS1_TXT_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 "ECDH-ECDSA-CAMELLIA128-SHA256" -# define TLS1_TXT_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 "ECDH-ECDSA-CAMELLIA256-SHA384" -# define TLS1_TXT_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 "ECDHE-RSA-CAMELLIA128-SHA256" -# define TLS1_TXT_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 "ECDHE-RSA-CAMELLIA256-SHA384" -# define TLS1_TXT_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 "ECDH-RSA-CAMELLIA128-SHA256" -# define TLS1_TXT_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 "ECDH-RSA-CAMELLIA256-SHA384" - /* draft-ietf-tls-chacha20-poly1305-03 */ # define TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 "ECDHE-RSA-CHACHA20-POLY1305" # define TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 "ECDHE-ECDSA-CHACHA20-POLY1305" @@ -1108,34 +892,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_TXT_DHE_PSK_WITH_CHACHA20_POLY1305 "DHE-PSK-CHACHA20-POLY1305" # define TLS1_TXT_RSA_PSK_WITH_CHACHA20_POLY1305 "RSA-PSK-CHACHA20-POLY1305" -/* Aria ciphersuites from RFC6209 */ -# define TLS1_TXT_RSA_WITH_ARIA_128_GCM_SHA256 "ARIA128-GCM-SHA256" -# define TLS1_TXT_RSA_WITH_ARIA_256_GCM_SHA384 "ARIA256-GCM-SHA384" -# define TLS1_TXT_DHE_RSA_WITH_ARIA_128_GCM_SHA256 "DHE-RSA-ARIA128-GCM-SHA256" -# define TLS1_TXT_DHE_RSA_WITH_ARIA_256_GCM_SHA384 "DHE-RSA-ARIA256-GCM-SHA384" -# define TLS1_TXT_DH_RSA_WITH_ARIA_128_GCM_SHA256 "DH-RSA-ARIA128-GCM-SHA256" -# define TLS1_TXT_DH_RSA_WITH_ARIA_256_GCM_SHA384 "DH-RSA-ARIA256-GCM-SHA384" -# define TLS1_TXT_DHE_DSS_WITH_ARIA_128_GCM_SHA256 "DHE-DSS-ARIA128-GCM-SHA256" -# define TLS1_TXT_DHE_DSS_WITH_ARIA_256_GCM_SHA384 "DHE-DSS-ARIA256-GCM-SHA384" -# define TLS1_TXT_DH_DSS_WITH_ARIA_128_GCM_SHA256 "DH-DSS-ARIA128-GCM-SHA256" -# define TLS1_TXT_DH_DSS_WITH_ARIA_256_GCM_SHA384 "DH-DSS-ARIA256-GCM-SHA384" -# define TLS1_TXT_DH_anon_WITH_ARIA_128_GCM_SHA256 "ADH-ARIA128-GCM-SHA256" -# define TLS1_TXT_DH_anon_WITH_ARIA_256_GCM_SHA384 "ADH-ARIA256-GCM-SHA384" -# define TLS1_TXT_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 "ECDHE-ECDSA-ARIA128-GCM-SHA256" -# define TLS1_TXT_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 "ECDHE-ECDSA-ARIA256-GCM-SHA384" -# define TLS1_TXT_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 "ECDH-ECDSA-ARIA128-GCM-SHA256" -# define TLS1_TXT_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 "ECDH-ECDSA-ARIA256-GCM-SHA384" -# define TLS1_TXT_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 "ECDHE-ARIA128-GCM-SHA256" -# define TLS1_TXT_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 "ECDHE-ARIA256-GCM-SHA384" -# define TLS1_TXT_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 "ECDH-ARIA128-GCM-SHA256" -# define TLS1_TXT_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 "ECDH-ARIA256-GCM-SHA384" -# define TLS1_TXT_PSK_WITH_ARIA_128_GCM_SHA256 "PSK-ARIA128-GCM-SHA256" -# define TLS1_TXT_PSK_WITH_ARIA_256_GCM_SHA384 "PSK-ARIA256-GCM-SHA384" -# define TLS1_TXT_DHE_PSK_WITH_ARIA_128_GCM_SHA256 "DHE-PSK-ARIA128-GCM-SHA256" -# define TLS1_TXT_DHE_PSK_WITH_ARIA_256_GCM_SHA384 "DHE-PSK-ARIA256-GCM-SHA384" -# define TLS1_TXT_RSA_PSK_WITH_ARIA_128_GCM_SHA256 "RSA-PSK-ARIA128-GCM-SHA256" -# define TLS1_TXT_RSA_PSK_WITH_ARIA_256_GCM_SHA384 "RSA-PSK-ARIA256-GCM-SHA384" - # define TLS_CT_RSA_SIGN 1 # define TLS_CT_DSS_SIGN 2 # define TLS_CT_RSA_FIXED_DH 3 @@ -1143,16 +899,6 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS_CT_ECDSA_SIGN 64 # define TLS_CT_RSA_FIXED_ECDH 65 # define TLS_CT_ECDSA_FIXED_ECDH 66 -# define TLS_CT_GOST01_SIGN 22 -# define TLS_CT_GOST12_IANA_SIGN 67 -# define TLS_CT_GOST12_IANA_512_SIGN 68 -# define TLS_CT_GOST12_LEGACY_SIGN 238 -# define TLS_CT_GOST12_LEGACY_512_SIGN 239 - -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define TLS_CT_GOST12_SIGN TLS_CT_GOST12_LEGACY_SIGN -# define TLS_CT_GOST12_512_SIGN TLS_CT_GOST12_LEGACY_512_SIGN -# endif /* * when correcting this number, correct also SSL3_CT_NUMBER in ssl3.h (see @@ -1168,35 +914,78 @@ int SSL_CTX_set_tlsext_ticket_key_evp_cb # define TLS1_FINISH_MAC_LENGTH 12 -# define TLS_MD_MAX_CONST_SIZE 22 - -/* ASCII: "client finished", in hex for EBCDIC compatibility */ -# define TLS_MD_CLIENT_FINISH_CONST "\x63\x6c\x69\x65\x6e\x74\x20\x66\x69\x6e\x69\x73\x68\x65\x64" -# define TLS_MD_CLIENT_FINISH_CONST_SIZE 15 -/* ASCII: "server finished", in hex for EBCDIC compatibility */ -# define TLS_MD_SERVER_FINISH_CONST "\x73\x65\x72\x76\x65\x72\x20\x66\x69\x6e\x69\x73\x68\x65\x64" -# define TLS_MD_SERVER_FINISH_CONST_SIZE 15 -/* ASCII: "server write key", in hex for EBCDIC compatibility */ -# define TLS_MD_SERVER_WRITE_KEY_CONST "\x73\x65\x72\x76\x65\x72\x20\x77\x72\x69\x74\x65\x20\x6b\x65\x79" -# define TLS_MD_SERVER_WRITE_KEY_CONST_SIZE 16 -/* ASCII: "key expansion", in hex for EBCDIC compatibility */ -# define TLS_MD_KEY_EXPANSION_CONST "\x6b\x65\x79\x20\x65\x78\x70\x61\x6e\x73\x69\x6f\x6e" -# define TLS_MD_KEY_EXPANSION_CONST_SIZE 13 -/* ASCII: "client write key", in hex for EBCDIC compatibility */ -# define TLS_MD_CLIENT_WRITE_KEY_CONST "\x63\x6c\x69\x65\x6e\x74\x20\x77\x72\x69\x74\x65\x20\x6b\x65\x79" -# define TLS_MD_CLIENT_WRITE_KEY_CONST_SIZE 16 -/* ASCII: "server write key", in hex for EBCDIC compatibility */ -# define TLS_MD_SERVER_WRITE_KEY_CONST "\x73\x65\x72\x76\x65\x72\x20\x77\x72\x69\x74\x65\x20\x6b\x65\x79" -# define TLS_MD_SERVER_WRITE_KEY_CONST_SIZE 16 -/* ASCII: "IV block", in hex for EBCDIC compatibility */ -# define TLS_MD_IV_BLOCK_CONST "\x49\x56\x20\x62\x6c\x6f\x63\x6b" -# define TLS_MD_IV_BLOCK_CONST_SIZE 8 -/* ASCII: "master secret", in hex for EBCDIC compatibility */ -# define TLS_MD_MASTER_SECRET_CONST "\x6d\x61\x73\x74\x65\x72\x20\x73\x65\x63\x72\x65\x74" -# define TLS_MD_MASTER_SECRET_CONST_SIZE 13 -/* ASCII: "extended master secret", in hex for EBCDIC compatibility */ -# define TLS_MD_EXTENDED_MASTER_SECRET_CONST "\x65\x78\x74\x65\x6e\x64\x65\x64\x20\x6d\x61\x73\x74\x65\x72\x20\x73\x65\x63\x72\x65\x74" -# define TLS_MD_EXTENDED_MASTER_SECRET_CONST_SIZE 22 +# define TLS_MD_MAX_CONST_SIZE 22 +# define TLS_MD_CLIENT_FINISH_CONST "client finished" +# define TLS_MD_CLIENT_FINISH_CONST_SIZE 15 +# define TLS_MD_SERVER_FINISH_CONST "server finished" +# define TLS_MD_SERVER_FINISH_CONST_SIZE 15 +# define TLS_MD_KEY_EXPANSION_CONST "key expansion" +# define TLS_MD_KEY_EXPANSION_CONST_SIZE 13 +# define TLS_MD_CLIENT_WRITE_KEY_CONST "client write key" +# define TLS_MD_CLIENT_WRITE_KEY_CONST_SIZE 16 +# define TLS_MD_SERVER_WRITE_KEY_CONST "server write key" +# define TLS_MD_SERVER_WRITE_KEY_CONST_SIZE 16 +# define TLS_MD_IV_BLOCK_CONST "IV block" +# define TLS_MD_IV_BLOCK_CONST_SIZE 8 +# define TLS_MD_MASTER_SECRET_CONST "master secret" +# define TLS_MD_MASTER_SECRET_CONST_SIZE 13 +# define TLS_MD_EXTENDED_MASTER_SECRET_CONST "extended master secret" +# define TLS_MD_EXTENDED_MASTER_SECRET_CONST_SIZE 22 + +# ifdef CHARSET_EBCDIC +# undef TLS_MD_CLIENT_FINISH_CONST +/* + * client finished + */ +# define TLS_MD_CLIENT_FINISH_CONST "\x63\x6c\x69\x65\x6e\x74\x20\x66\x69\x6e\x69\x73\x68\x65\x64" + +# undef TLS_MD_SERVER_FINISH_CONST +/* + * server finished + */ +# define TLS_MD_SERVER_FINISH_CONST "\x73\x65\x72\x76\x65\x72\x20\x66\x69\x6e\x69\x73\x68\x65\x64" + +# undef TLS_MD_SERVER_WRITE_KEY_CONST +/* + * server write key + */ +# define TLS_MD_SERVER_WRITE_KEY_CONST "\x73\x65\x72\x76\x65\x72\x20\x77\x72\x69\x74\x65\x20\x6b\x65\x79" + +# undef TLS_MD_KEY_EXPANSION_CONST +/* + * key expansion + */ +# define TLS_MD_KEY_EXPANSION_CONST "\x6b\x65\x79\x20\x65\x78\x70\x61\x6e\x73\x69\x6f\x6e" + +# undef TLS_MD_CLIENT_WRITE_KEY_CONST +/* + * client write key + */ +# define TLS_MD_CLIENT_WRITE_KEY_CONST "\x63\x6c\x69\x65\x6e\x74\x20\x77\x72\x69\x74\x65\x20\x6b\x65\x79" + +# undef TLS_MD_SERVER_WRITE_KEY_CONST +/* + * server write key + */ +# define TLS_MD_SERVER_WRITE_KEY_CONST "\x73\x65\x72\x76\x65\x72\x20\x77\x72\x69\x74\x65\x20\x6b\x65\x79" + +# undef TLS_MD_IV_BLOCK_CONST +/* + * IV block + */ +# define TLS_MD_IV_BLOCK_CONST "\x49\x56\x20\x62\x6c\x6f\x63\x6b" + +# undef TLS_MD_MASTER_SECRET_CONST +/* + * master secret + */ +# define TLS_MD_MASTER_SECRET_CONST "\x6d\x61\x73\x74\x65\x72\x20\x73\x65\x63\x72\x65\x74" +# undef TLS_MD_EXTENDED_MASTER_SECRET_CONST +/* + * extended master secret + */ +# define TLS_MD_EXTENDED_MASTER_SECRET_CONST "\x65\x78\x74\x65\x6e\x64\x65\x64\x20\x6d\x61\x73\x74\x65\x72\x20\x73\x65\x63\x72\x65\x74" +# endif /* TLS Session Ticket extension struct */ struct tls_session_ticket_ext_st { diff --git a/openssl/include/openssl/trace.h b/openssl/include/openssl/trace.h index 9a5b56ea5..282001336 100644 --- a/openssl/include/openssl/trace.h +++ b/openssl/include/openssl/trace.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -43,8 +43,10 @@ extern "C" { # define OSSL_TRACE_CATEGORY_TLS 3 # define OSSL_TRACE_CATEGORY_TLS_CIPHER 4 # define OSSL_TRACE_CATEGORY_CONF 5 -# define OSSL_TRACE_CATEGORY_ENGINE_TABLE 6 -# define OSSL_TRACE_CATEGORY_ENGINE_REF_COUNT 7 +# ifndef OPENSSL_NO_ENGINE +# define OSSL_TRACE_CATEGORY_ENGINE_TABLE 6 +# define OSSL_TRACE_CATEGORY_ENGINE_REF_COUNT 7 +# endif # define OSSL_TRACE_CATEGORY_PKCS5V2 8 # define OSSL_TRACE_CATEGORY_PKCS12_KEYGEN 9 # define OSSL_TRACE_CATEGORY_PKCS12_DECRYPT 10 @@ -55,10 +57,8 @@ extern "C" { # define OSSL_TRACE_CATEGORY_DECODER 15 # define OSSL_TRACE_CATEGORY_ENCODER 16 # define OSSL_TRACE_CATEGORY_REF_COUNT 17 -# define OSSL_TRACE_CATEGORY_HTTP 18 /* Count of available categories. */ -# define OSSL_TRACE_CATEGORY_NUM 19 -/* KEEP THIS LIST IN SYNC with trace_categories[] in crypto/trace.c */ +# define OSSL_TRACE_CATEGORY_NUM 18 /* Returns the trace category number for the given |name| */ int OSSL_trace_get_category_num(const char *name); @@ -305,14 +305,6 @@ void OSSL_trace_end(int category, BIO *channel); # define OSSL_TRACE9(category, format, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) \ OSSL_TRACEV(category, (trc_out, format, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9)) -#define OSSL_TRACE_STRING_MAX 80 -int OSSL_trace_string(BIO *out, int text, int full, - const unsigned char *data, size_t size); -#define OSSL_TRACE_STRING(category, text, full, data, len) \ - OSSL_TRACE_BEGIN(category) { \ - OSSL_trace_string(trc_out, text, full, data, len); \ - } OSSL_TRACE_END(category) - # ifdef __cplusplus } # endif diff --git a/openssl/include/openssl/ts.h b/openssl/include/openssl/ts.h index b09b646df..5136e4e97 100644 --- a/openssl/include/openssl/ts.h +++ b/openssl/include/openssl/ts.h @@ -30,15 +30,13 @@ # include # include # include -# include -# include -# ifndef OPENSSL_NO_STDIO -# include -# endif # ifdef __cplusplus extern "C" { # endif +# include +# include + typedef struct TS_msg_imprint_st TS_MSG_IMPRINT; typedef struct TS_req_st TS_REQ; typedef struct TS_accuracy_st TS_ACCURACY; diff --git a/openssl/include/openssl/types.h b/openssl/include/openssl/types.h index c28028681..4e3bce468 100644 --- a/openssl/include/openssl/types.h +++ b/openssl/include/openssl/types.h @@ -7,21 +7,9 @@ * https://www.openssl.org/source/license.html */ -/* - * Unfortunate workaround to avoid symbol conflict with wincrypt.h - * See https://github.com/openssl/openssl/issues/9981 - */ -#ifdef _WIN32 -# define WINCRYPT_USE_SYMBOL_PREFIX -# undef X509_NAME -# undef X509_EXTENSIONS -# undef PKCS7_SIGNER_INFO -# undef OCSP_REQUEST -# undef OCSP_RESPONSE -#endif - #ifndef OPENSSL_TYPES_H # define OPENSSL_TYPES_H +# pragma once # include @@ -82,6 +70,15 @@ typedef struct ASN1_ITEM_st ASN1_ITEM; typedef struct asn1_pctx_st ASN1_PCTX; typedef struct asn1_sctx_st ASN1_SCTX; +# ifdef _WIN32 +# undef X509_NAME +# undef X509_EXTENSIONS +# undef PKCS7_ISSUER_AND_SERIAL +# undef PKCS7_SIGNER_INFO +# undef OCSP_REQUEST +# undef OCSP_RESPONSE +# endif + # ifdef BIGNUM # undef BIGNUM # endif @@ -153,9 +150,15 @@ typedef struct ec_key_st EC_KEY; typedef struct ec_key_method_st EC_KEY_METHOD; # endif +typedef struct ec_point_method_st EC_POINT_METHOD; + typedef struct rand_meth_st RAND_METHOD; typedef struct rand_drbg_st RAND_DRBG; +# ifndef OPENSSL_NO_BN_METHOD +typedef struct bn_method_st BN_METHOD; +# endif + typedef struct ssl_dane_st SSL_DANE; typedef struct x509_st X509; typedef struct X509_algor_st X509_ALGOR; @@ -165,6 +168,9 @@ typedef struct x509_revoked_st X509_REVOKED; typedef struct X509_name_st X509_NAME; typedef struct X509_pubkey_st X509_PUBKEY; typedef struct x509_store_st X509_STORE; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +typedef struct delegated_credential_st DELEGATED_CREDENTIAL; +#endif typedef struct x509_store_ctx_st X509_STORE_CTX; typedef struct x509_object_st X509_OBJECT; @@ -187,6 +193,10 @@ typedef struct engine_st ENGINE; typedef struct ssl_st SSL; typedef struct ssl_ctx_st SSL_CTX; +# ifndef OPENSSL_NO_STATUS +typedef struct ssl_status_st SSL_status; +# endif + typedef struct comp_ctx_st COMP_CTX; typedef struct comp_method_st COMP_METHOD; @@ -215,6 +225,8 @@ typedef struct ct_policy_eval_ctx_st CT_POLICY_EVAL_CTX; typedef struct ossl_store_info_st OSSL_STORE_INFO; typedef struct ossl_store_search_st OSSL_STORE_SEARCH; +typedef struct ssl_quic_method_st SSL_QUIC_METHOD; + typedef struct ossl_lib_ctx_st OSSL_LIB_CTX; typedef struct ossl_dispatch_st OSSL_DISPATCH; diff --git a/openssl/include/openssl/whrlpool.h b/openssl/include/openssl/whrlpool.h deleted file mode 100644 index 05ba46324..000000000 --- a/openssl/include/openssl/whrlpool.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_WHRLPOOL_H -# define OPENSSL_WHRLPOOL_H -# pragma once - -# include -# ifndef OPENSSL_NO_DEPRECATED_3_0 -# define HEADER_WHRLPOOL_H -# endif - -# include - -# ifndef OPENSSL_NO_WHIRLPOOL -# include -# include -# ifdef __cplusplus -extern "C" { -# endif - -# define WHIRLPOOL_DIGEST_LENGTH (512/8) - -# if !defined(OPENSSL_NO_DEPRECATED_3_0) - -# define WHIRLPOOL_BBLOCK 512 -# define WHIRLPOOL_COUNTER (256/8) - -typedef struct { - union { - unsigned char c[WHIRLPOOL_DIGEST_LENGTH]; - /* double q is here to ensure 64-bit alignment */ - double q[WHIRLPOOL_DIGEST_LENGTH / sizeof(double)]; - } H; - unsigned char data[WHIRLPOOL_BBLOCK / 8]; - unsigned int bitoff; - size_t bitlen[WHIRLPOOL_COUNTER / sizeof(size_t)]; -} WHIRLPOOL_CTX; -# endif -# ifndef OPENSSL_NO_DEPRECATED_3_0 -OSSL_DEPRECATEDIN_3_0 int WHIRLPOOL_Init(WHIRLPOOL_CTX *c); -OSSL_DEPRECATEDIN_3_0 int WHIRLPOOL_Update(WHIRLPOOL_CTX *c, - const void *inp, size_t bytes); -OSSL_DEPRECATEDIN_3_0 void WHIRLPOOL_BitUpdate(WHIRLPOOL_CTX *c, - const void *inp, size_t bits); -OSSL_DEPRECATEDIN_3_0 int WHIRLPOOL_Final(unsigned char *md, WHIRLPOOL_CTX *c); -OSSL_DEPRECATEDIN_3_0 unsigned char *WHIRLPOOL(const void *inp, size_t bytes, - unsigned char *md); -# endif - -# ifdef __cplusplus -} -# endif -# endif - -#endif diff --git a/openssl/include/openssl/x509.h b/openssl/include/openssl/x509.h index bec46a737..3f9cf6ac5 100644 --- a/openssl/include/openssl/x509.h +++ b/openssl/include/openssl/x509.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/x509.h.in * - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -40,9 +40,6 @@ # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -512,11 +509,13 @@ void *X509_CRL_get_meth_data(X509_CRL *crl); const char *X509_verify_cert_error_string(long n); int X509_verify(X509 *a, EVP_PKEY *r); +int X509_verify_ctx(X509 *a, EVP_MD_CTX *ctx); int X509_self_signed(X509 *cert, int verify_signature); int X509_REQ_verify_ex(X509_REQ *a, EVP_PKEY *r, OSSL_LIB_CTX *libctx, const char *propq); int X509_REQ_verify(X509_REQ *a, EVP_PKEY *r); +int X509_REQ_verify_ctx(X509_REQ *a, EVP_MD_CTX *ctx); int X509_CRL_verify(X509_CRL *a, EVP_PKEY *r); int NETSCAPE_SPKI_verify(NETSCAPE_SPKI *a, EVP_PKEY *r); @@ -554,6 +553,55 @@ int X509_NAME_digest(const X509_NAME *data, const EVP_MD *type, X509 *X509_load_http(const char *url, BIO *bio, BIO *rbio, int timeout); X509_CRL *X509_CRL_load_http(const char *url, BIO *bio, BIO *rbio, int timeout); + +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +# define DC_FILETYPE_RAW 0 + +void DC_free(DELEGATED_CREDENTIAL *dc); +DELEGATED_CREDENTIAL *DC_new(void); +DELEGATED_CREDENTIAL *DC_new_ex(OSSL_LIB_CTX *libctx, const char *propq); +DELEGATED_CREDENTIAL *DC_new_from_raw_byte(const unsigned char *byte, + size_t len); +DELEGATED_CREDENTIAL *DC_new_from_raw_byte_ex(const unsigned char *byte, + size_t len, + OSSL_LIB_CTX *libctx, + const char *propq); +int DC_check_valid(X509 *parent_cert, DELEGATED_CREDENTIAL *dc); +int DC_check_time_valid(X509 *parent_cert, DELEGATED_CREDENTIAL *dc); +int DC_check_parent_cert_valid(X509 *parent_cert); +unsigned long DC_get_valid_time(DELEGATED_CREDENTIAL *dc); +unsigned int DC_get_expected_cert_verify_algorithm(DELEGATED_CREDENTIAL *dc); +size_t DC_get_dc_publickey_raw_len(DELEGATED_CREDENTIAL *dc); +unsigned char *DC_get0_dc_publickey_raw(DELEGATED_CREDENTIAL *dc); +unsigned int DC_get_signature_sign_algorithm(DELEGATED_CREDENTIAL *dc); +size_t DC_get_dc_signature_len(DELEGATED_CREDENTIAL *dc); +unsigned char *DC_get0_dc_signature(DELEGATED_CREDENTIAL *dc); +EVP_PKEY *DC_get0_publickey(DELEGATED_CREDENTIAL *dc); +unsigned char *DC_get0_raw_byte(DELEGATED_CREDENTIAL *dc); +size_t DC_get_raw_byte_len(DELEGATED_CREDENTIAL *dc); +int DC_set_valid_time(DELEGATED_CREDENTIAL *dc, unsigned long valid_time); +int DC_set_expected_cert_verify_algorithm(DELEGATED_CREDENTIAL *dc, + unsigned int alg); +int DC_set_dc_publickey_len(DELEGATED_CREDENTIAL *dc, size_t len); +int DC_set0_dc_publickey(DELEGATED_CREDENTIAL *dc, unsigned char *pub_key); +int DC_set_signature_sign_algorithm(DELEGATED_CREDENTIAL *dc, unsigned int alg); +int DC_set_dc_signature_len(DELEGATED_CREDENTIAL *dc, size_t len); +int DC_set0_dc_signature(DELEGATED_CREDENTIAL *dc, unsigned char *sig); +int DC_set0_raw_byte(DELEGATED_CREDENTIAL *dc, unsigned char *byte, size_t len); +int DC_set1_raw_byte(DELEGATED_CREDENTIAL *dc, const unsigned char *byte, + size_t len); +int DC_set0_publickey(DELEGATED_CREDENTIAL *dc, EVP_PKEY *pkey); + + +int DC_check_private_key(DELEGATED_CREDENTIAL *dc, EVP_PKEY *pkey); + +int DC_up_ref(DELEGATED_CREDENTIAL *dc); +DELEGATED_CREDENTIAL *DC_load_from_file(const char *file); +DELEGATED_CREDENTIAL *DC_load_from_file_ex(const char *file, + OSSL_LIB_CTX *libctx, + const char *propq); +# endif + # ifndef OPENSSL_NO_DEPRECATED_3_0 # include /* OSSL_HTTP_REQ_CTX_nbio_d2i */ # define X509_http_nbio(rctx, pcert) \ @@ -606,8 +654,6 @@ EVP_PKEY *d2i_PrivateKey_ex_fp(FILE *fp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, const char *propq); EVP_PKEY *d2i_PrivateKey_fp(FILE *fp, EVP_PKEY **a); int i2d_PUBKEY_fp(FILE *fp, const EVP_PKEY *pkey); -EVP_PKEY *d2i_PUBKEY_ex_fp(FILE *fp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, - const char *propq); EVP_PKEY *d2i_PUBKEY_fp(FILE *fp, EVP_PKEY **a); # endif @@ -656,8 +702,6 @@ EVP_PKEY *d2i_PrivateKey_ex_bio(BIO *bp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, const char *propq); EVP_PKEY *d2i_PrivateKey_bio(BIO *bp, EVP_PKEY **a); int i2d_PUBKEY_bio(BIO *bp, const EVP_PKEY *pkey); -EVP_PKEY *d2i_PUBKEY_ex_bio(BIO *bp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, - const char *propq); EVP_PKEY *d2i_PUBKEY_bio(BIO *bp, EVP_PKEY **a); DECLARE_ASN1_DUP_FUNCTION(X509) @@ -779,6 +823,14 @@ ASN1_OCTET_STRING *X509_get0_distinguishing_id(X509 *x); void X509_REQ_set0_distinguishing_id(X509_REQ *x, ASN1_OCTET_STRING *d_id); ASN1_OCTET_STRING *X509_REQ_get0_distinguishing_id(X509_REQ *x); +/* for compattibility usage */ +# ifndef OPENSSL_NO_SM2 +# define X509_set0_sm2_id X509_set0_distinguishing_id +# define X509_get0_sm2_id X509_get0_distinguishing_id +# define X509_REQ_set0_sm2_id X509_REQ_set0_distinguishing_id +# define X509_REQ_get0_sm2_id X509_REQ_get0_distinguishing_id +# endif + int X509_alias_set1(X509 *x, const unsigned char *name, int len); int X509_keyid_set1(X509 *x, const unsigned char *id, int len); unsigned char *X509_alias_get0(X509 *x, int *len); @@ -891,7 +943,7 @@ int X509_REQ_get_signature_nid(const X509_REQ *req); int i2d_re_X509_REQ_tbs(X509_REQ *req, unsigned char **pp); int X509_REQ_set_pubkey(X509_REQ *x, EVP_PKEY *pkey); EVP_PKEY *X509_REQ_get_pubkey(X509_REQ *req); -EVP_PKEY *X509_REQ_get0_pubkey(const X509_REQ *req); +EVP_PKEY *X509_REQ_get0_pubkey(X509_REQ *req); X509_PUBKEY *X509_REQ_get_X509_PUBKEY(X509_REQ *req); int X509_REQ_extension_nid(int nid); int *X509_REQ_get_extension_nids(void); @@ -957,14 +1009,13 @@ X509_REVOKED_get0_extensions(const X509_REVOKED *r); X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer, EVP_PKEY *skey, const EVP_MD *md, unsigned int flags); -int X509_REQ_check_private_key(const X509_REQ *req, EVP_PKEY *pkey); +int X509_REQ_check_private_key(X509_REQ *x509, EVP_PKEY *pkey); -int X509_check_private_key(const X509 *cert, const EVP_PKEY *pkey); +int X509_check_private_key(const X509 *x509, const EVP_PKEY *pkey); int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain, unsigned long flags); int X509_CRL_check_suiteb(X509_CRL *crl, EVP_PKEY *pk, unsigned long flags); -void OSSL_STACK_OF_X509_free(STACK_OF(X509) *certs); STACK_OF(X509) *X509_chain_up_ref(STACK_OF(X509) *chain); int X509_issuer_and_serial_cmp(const X509 *a, const X509 *b); @@ -1270,8 +1321,6 @@ int PKCS8_pkey_add1_attr_by_OBJ(PKCS8_PRIV_KEY_INFO *p8, const ASN1_OBJECT *obj, int type, const unsigned char *bytes, int len); -void X509_PUBKEY_set0_public_key(X509_PUBKEY *pub, - unsigned char *penc, int penclen); int X509_PUBKEY_set0_param(X509_PUBKEY *pub, ASN1_OBJECT *aobj, int ptype, void *pval, unsigned char *penc, int penclen); diff --git a/openssl/include/openssl/x509.h.in b/openssl/include/openssl/x509.h.in index 721039130..04f22f044 100644 --- a/openssl/include/openssl/x509.h.in +++ b/openssl/include/openssl/x509.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -41,9 +41,6 @@ use OpenSSL::stackhash qw(generate_stack_macros); # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -318,11 +315,13 @@ void *X509_CRL_get_meth_data(X509_CRL *crl); const char *X509_verify_cert_error_string(long n); int X509_verify(X509 *a, EVP_PKEY *r); +int X509_verify_ctx(X509 *a, EVP_MD_CTX *ctx); int X509_self_signed(X509 *cert, int verify_signature); int X509_REQ_verify_ex(X509_REQ *a, EVP_PKEY *r, OSSL_LIB_CTX *libctx, const char *propq); int X509_REQ_verify(X509_REQ *a, EVP_PKEY *r); +int X509_REQ_verify_ctx(X509_REQ *a, EVP_MD_CTX *ctx); int X509_CRL_verify(X509_CRL *a, EVP_PKEY *r); int NETSCAPE_SPKI_verify(NETSCAPE_SPKI *a, EVP_PKEY *r); @@ -360,6 +359,55 @@ int X509_NAME_digest(const X509_NAME *data, const EVP_MD *type, X509 *X509_load_http(const char *url, BIO *bio, BIO *rbio, int timeout); X509_CRL *X509_CRL_load_http(const char *url, BIO *bio, BIO *rbio, int timeout); + +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +# define DC_FILETYPE_RAW 0 + +void DC_free(DELEGATED_CREDENTIAL *dc); +DELEGATED_CREDENTIAL *DC_new(void); +DELEGATED_CREDENTIAL *DC_new_ex(OSSL_LIB_CTX *libctx, const char *propq); +DELEGATED_CREDENTIAL *DC_new_from_raw_byte(const unsigned char *byte, + size_t len); +DELEGATED_CREDENTIAL *DC_new_from_raw_byte_ex(const unsigned char *byte, + size_t len, + OSSL_LIB_CTX *libctx, + const char *propq); +int DC_check_valid(X509 *parent_cert, DELEGATED_CREDENTIAL *dc); +int DC_check_time_valid(X509 *parent_cert, DELEGATED_CREDENTIAL *dc); +int DC_check_parent_cert_valid(X509 *parent_cert); +unsigned long DC_get_valid_time(DELEGATED_CREDENTIAL *dc); +unsigned int DC_get_expected_cert_verify_algorithm(DELEGATED_CREDENTIAL *dc); +size_t DC_get_dc_publickey_raw_len(DELEGATED_CREDENTIAL *dc); +unsigned char *DC_get0_dc_publickey_raw(DELEGATED_CREDENTIAL *dc); +unsigned int DC_get_signature_sign_algorithm(DELEGATED_CREDENTIAL *dc); +size_t DC_get_dc_signature_len(DELEGATED_CREDENTIAL *dc); +unsigned char *DC_get0_dc_signature(DELEGATED_CREDENTIAL *dc); +EVP_PKEY *DC_get0_publickey(DELEGATED_CREDENTIAL *dc); +unsigned char *DC_get0_raw_byte(DELEGATED_CREDENTIAL *dc); +size_t DC_get_raw_byte_len(DELEGATED_CREDENTIAL *dc); +int DC_set_valid_time(DELEGATED_CREDENTIAL *dc, unsigned long valid_time); +int DC_set_expected_cert_verify_algorithm(DELEGATED_CREDENTIAL *dc, + unsigned int alg); +int DC_set_dc_publickey_len(DELEGATED_CREDENTIAL *dc, size_t len); +int DC_set0_dc_publickey(DELEGATED_CREDENTIAL *dc, unsigned char *pub_key); +int DC_set_signature_sign_algorithm(DELEGATED_CREDENTIAL *dc, unsigned int alg); +int DC_set_dc_signature_len(DELEGATED_CREDENTIAL *dc, size_t len); +int DC_set0_dc_signature(DELEGATED_CREDENTIAL *dc, unsigned char *sig); +int DC_set0_raw_byte(DELEGATED_CREDENTIAL *dc, unsigned char *byte, size_t len); +int DC_set1_raw_byte(DELEGATED_CREDENTIAL *dc, const unsigned char *byte, + size_t len); +int DC_set0_publickey(DELEGATED_CREDENTIAL *dc, EVP_PKEY *pkey); + + +int DC_check_private_key(DELEGATED_CREDENTIAL *dc, EVP_PKEY *pkey); + +int DC_up_ref(DELEGATED_CREDENTIAL *dc); +DELEGATED_CREDENTIAL *DC_load_from_file(const char *file); +DELEGATED_CREDENTIAL *DC_load_from_file_ex(const char *file, + OSSL_LIB_CTX *libctx, + const char *propq); +# endif + # ifndef OPENSSL_NO_DEPRECATED_3_0 # include /* OSSL_HTTP_REQ_CTX_nbio_d2i */ # define X509_http_nbio(rctx, pcert) \ @@ -412,8 +460,6 @@ EVP_PKEY *d2i_PrivateKey_ex_fp(FILE *fp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, const char *propq); EVP_PKEY *d2i_PrivateKey_fp(FILE *fp, EVP_PKEY **a); int i2d_PUBKEY_fp(FILE *fp, const EVP_PKEY *pkey); -EVP_PKEY *d2i_PUBKEY_ex_fp(FILE *fp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, - const char *propq); EVP_PKEY *d2i_PUBKEY_fp(FILE *fp, EVP_PKEY **a); # endif @@ -462,8 +508,6 @@ EVP_PKEY *d2i_PrivateKey_ex_bio(BIO *bp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, const char *propq); EVP_PKEY *d2i_PrivateKey_bio(BIO *bp, EVP_PKEY **a); int i2d_PUBKEY_bio(BIO *bp, const EVP_PKEY *pkey); -EVP_PKEY *d2i_PUBKEY_ex_bio(BIO *bp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, - const char *propq); EVP_PKEY *d2i_PUBKEY_bio(BIO *bp, EVP_PKEY **a); DECLARE_ASN1_DUP_FUNCTION(X509) @@ -585,6 +629,14 @@ ASN1_OCTET_STRING *X509_get0_distinguishing_id(X509 *x); void X509_REQ_set0_distinguishing_id(X509_REQ *x, ASN1_OCTET_STRING *d_id); ASN1_OCTET_STRING *X509_REQ_get0_distinguishing_id(X509_REQ *x); +/* for compattibility usage */ +# ifndef OPENSSL_NO_SM2 +# define X509_set0_sm2_id X509_set0_distinguishing_id +# define X509_get0_sm2_id X509_get0_distinguishing_id +# define X509_REQ_set0_sm2_id X509_REQ_set0_distinguishing_id +# define X509_REQ_get0_sm2_id X509_REQ_get0_distinguishing_id +# endif + int X509_alias_set1(X509 *x, const unsigned char *name, int len); int X509_keyid_set1(X509 *x, const unsigned char *id, int len); unsigned char *X509_alias_get0(X509 *x, int *len); @@ -697,7 +749,7 @@ int X509_REQ_get_signature_nid(const X509_REQ *req); int i2d_re_X509_REQ_tbs(X509_REQ *req, unsigned char **pp); int X509_REQ_set_pubkey(X509_REQ *x, EVP_PKEY *pkey); EVP_PKEY *X509_REQ_get_pubkey(X509_REQ *req); -EVP_PKEY *X509_REQ_get0_pubkey(const X509_REQ *req); +EVP_PKEY *X509_REQ_get0_pubkey(X509_REQ *req); X509_PUBKEY *X509_REQ_get_X509_PUBKEY(X509_REQ *req); int X509_REQ_extension_nid(int nid); int *X509_REQ_get_extension_nids(void); @@ -763,14 +815,13 @@ X509_REVOKED_get0_extensions(const X509_REVOKED *r); X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer, EVP_PKEY *skey, const EVP_MD *md, unsigned int flags); -int X509_REQ_check_private_key(const X509_REQ *req, EVP_PKEY *pkey); +int X509_REQ_check_private_key(X509_REQ *x509, EVP_PKEY *pkey); -int X509_check_private_key(const X509 *cert, const EVP_PKEY *pkey); +int X509_check_private_key(const X509 *x509, const EVP_PKEY *pkey); int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain, unsigned long flags); int X509_CRL_check_suiteb(X509_CRL *crl, EVP_PKEY *pk, unsigned long flags); -void OSSL_STACK_OF_X509_free(STACK_OF(X509) *certs); STACK_OF(X509) *X509_chain_up_ref(STACK_OF(X509) *chain); int X509_issuer_and_serial_cmp(const X509 *a, const X509 *b); @@ -1076,8 +1127,6 @@ int PKCS8_pkey_add1_attr_by_OBJ(PKCS8_PRIV_KEY_INFO *p8, const ASN1_OBJECT *obj, int type, const unsigned char *bytes, int len); -void X509_PUBKEY_set0_public_key(X509_PUBKEY *pub, - unsigned char *penc, int penclen); int X509_PUBKEY_set0_param(X509_PUBKEY *pub, ASN1_OBJECT *aobj, int ptype, void *pval, unsigned char *penc, int penclen); diff --git a/openssl/include/openssl/x509_vfy.h b/openssl/include/openssl/x509_vfy.h index cc50b520b..0c3dd1e57 100644 --- a/openssl/include/openssl/x509_vfy.h +++ b/openssl/include/openssl/x509_vfy.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/x509_vfy.h.in * - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -411,7 +411,6 @@ X509_LOOKUP_ctrl_ex((x), X509_L_ADD_STORE, (name), 0, NULL, \ # define X509_V_ERR_CA_CERT_MISSING_KEY_USAGE 92 # define X509_V_ERR_EXTENSIONS_REQUIRE_VERSION_3 93 # define X509_V_ERR_EC_KEY_EXPLICIT_PARAMS 94 -# define X509_V_ERR_RPK_UNTRUSTED 95 /* Certificate verify flags */ # ifndef OPENSSL_NO_DEPRECATED_1_1_0 @@ -492,72 +491,72 @@ int X509_OBJECT_set1_X509(X509_OBJECT *a, X509 *obj); X509_CRL *X509_OBJECT_get0_X509_CRL(const X509_OBJECT *a); int X509_OBJECT_set1_X509_CRL(X509_OBJECT *a, X509_CRL *obj); X509_STORE *X509_STORE_new(void); -void X509_STORE_free(X509_STORE *xs); -int X509_STORE_lock(X509_STORE *xs); -int X509_STORE_unlock(X509_STORE *xs); -int X509_STORE_up_ref(X509_STORE *xs); -STACK_OF(X509_OBJECT) *X509_STORE_get0_objects(const X509_STORE *xs); -STACK_OF(X509_OBJECT) *X509_STORE_get1_objects(X509_STORE *xs); -STACK_OF(X509) *X509_STORE_get1_all_certs(X509_STORE *xs); -STACK_OF(X509) *X509_STORE_CTX_get1_certs(X509_STORE_CTX *xs, +void X509_STORE_free(X509_STORE *v); +int X509_STORE_copy(X509_STORE *dest, const X509_STORE *src); +int X509_STORE_lock(X509_STORE *ctx); +int X509_STORE_unlock(X509_STORE *ctx); +int X509_STORE_up_ref(X509_STORE *v); +STACK_OF(X509_OBJECT) *X509_STORE_get0_objects(const X509_STORE *v); +STACK_OF(X509) *X509_STORE_get1_all_certs(X509_STORE *st); +STACK_OF(X509) *X509_STORE_CTX_get1_certs(X509_STORE_CTX *st, const X509_NAME *nm); STACK_OF(X509_CRL) *X509_STORE_CTX_get1_crls(const X509_STORE_CTX *st, const X509_NAME *nm); -int X509_STORE_set_flags(X509_STORE *xs, unsigned long flags); -int X509_STORE_set_purpose(X509_STORE *xs, int purpose); -int X509_STORE_set_trust(X509_STORE *xs, int trust); -int X509_STORE_set1_param(X509_STORE *xs, const X509_VERIFY_PARAM *pm); -X509_VERIFY_PARAM *X509_STORE_get0_param(const X509_STORE *xs); +int X509_STORE_set_flags(X509_STORE *ctx, unsigned long flags); +int X509_STORE_set_purpose(X509_STORE *ctx, int purpose); +int X509_STORE_set_trust(X509_STORE *ctx, int trust); +int X509_STORE_set1_param(X509_STORE *ctx, const X509_VERIFY_PARAM *pm); +X509_VERIFY_PARAM *X509_STORE_get0_param(const X509_STORE *ctx); -void X509_STORE_set_verify(X509_STORE *xs, X509_STORE_CTX_verify_fn verify); +void X509_STORE_set_verify(X509_STORE *ctx, X509_STORE_CTX_verify_fn verify); #define X509_STORE_set_verify_func(ctx, func) \ X509_STORE_set_verify((ctx),(func)) void X509_STORE_CTX_set_verify(X509_STORE_CTX *ctx, X509_STORE_CTX_verify_fn verify); -X509_STORE_CTX_verify_fn X509_STORE_get_verify(const X509_STORE *xs); -void X509_STORE_set_verify_cb(X509_STORE *xs, +X509_STORE_CTX_verify_fn X509_STORE_get_verify(const X509_STORE *ctx); +void X509_STORE_set_verify_cb(X509_STORE *ctx, X509_STORE_CTX_verify_cb verify_cb); # define X509_STORE_set_verify_cb_func(ctx,func) \ X509_STORE_set_verify_cb((ctx),(func)) -X509_STORE_CTX_verify_cb X509_STORE_get_verify_cb(const X509_STORE *xs); -void X509_STORE_set_get_issuer(X509_STORE *xs, +X509_STORE_CTX_verify_cb X509_STORE_get_verify_cb(const X509_STORE *ctx); +void X509_STORE_set_get_issuer(X509_STORE *ctx, X509_STORE_CTX_get_issuer_fn get_issuer); -X509_STORE_CTX_get_issuer_fn X509_STORE_get_get_issuer(const X509_STORE *xs); -void X509_STORE_set_check_issued(X509_STORE *xs, +X509_STORE_CTX_get_issuer_fn X509_STORE_get_get_issuer(const X509_STORE *ctx); +void X509_STORE_set_check_issued(X509_STORE *ctx, X509_STORE_CTX_check_issued_fn check_issued); -X509_STORE_CTX_check_issued_fn X509_STORE_get_check_issued(const X509_STORE *s); -void X509_STORE_set_check_revocation(X509_STORE *xs, +X509_STORE_CTX_check_issued_fn X509_STORE_get_check_issued(const X509_STORE *ctx); +void X509_STORE_set_check_revocation(X509_STORE *ctx, X509_STORE_CTX_check_revocation_fn check_revocation); X509_STORE_CTX_check_revocation_fn - X509_STORE_get_check_revocation(const X509_STORE *xs); -void X509_STORE_set_get_crl(X509_STORE *xs, + X509_STORE_get_check_revocation(const X509_STORE *ctx); +void X509_STORE_set_get_crl(X509_STORE *ctx, X509_STORE_CTX_get_crl_fn get_crl); -X509_STORE_CTX_get_crl_fn X509_STORE_get_get_crl(const X509_STORE *xs); -void X509_STORE_set_check_crl(X509_STORE *xs, +X509_STORE_CTX_get_crl_fn X509_STORE_get_get_crl(const X509_STORE *ctx); +void X509_STORE_set_check_crl(X509_STORE *ctx, X509_STORE_CTX_check_crl_fn check_crl); -X509_STORE_CTX_check_crl_fn X509_STORE_get_check_crl(const X509_STORE *xs); -void X509_STORE_set_cert_crl(X509_STORE *xs, +X509_STORE_CTX_check_crl_fn X509_STORE_get_check_crl(const X509_STORE *ctx); +void X509_STORE_set_cert_crl(X509_STORE *ctx, X509_STORE_CTX_cert_crl_fn cert_crl); -X509_STORE_CTX_cert_crl_fn X509_STORE_get_cert_crl(const X509_STORE *xs); -void X509_STORE_set_check_policy(X509_STORE *xs, +X509_STORE_CTX_cert_crl_fn X509_STORE_get_cert_crl(const X509_STORE *ctx); +void X509_STORE_set_check_policy(X509_STORE *ctx, X509_STORE_CTX_check_policy_fn check_policy); -X509_STORE_CTX_check_policy_fn X509_STORE_get_check_policy(const X509_STORE *s); -void X509_STORE_set_lookup_certs(X509_STORE *xs, +X509_STORE_CTX_check_policy_fn X509_STORE_get_check_policy(const X509_STORE *ctx); +void X509_STORE_set_lookup_certs(X509_STORE *ctx, X509_STORE_CTX_lookup_certs_fn lookup_certs); -X509_STORE_CTX_lookup_certs_fn X509_STORE_get_lookup_certs(const X509_STORE *s); -void X509_STORE_set_lookup_crls(X509_STORE *xs, +X509_STORE_CTX_lookup_certs_fn X509_STORE_get_lookup_certs(const X509_STORE *ctx); +void X509_STORE_set_lookup_crls(X509_STORE *ctx, X509_STORE_CTX_lookup_crls_fn lookup_crls); #define X509_STORE_set_lookup_crls_cb(ctx, func) \ X509_STORE_set_lookup_crls((ctx), (func)) -X509_STORE_CTX_lookup_crls_fn X509_STORE_get_lookup_crls(const X509_STORE *xs); -void X509_STORE_set_cleanup(X509_STORE *xs, +X509_STORE_CTX_lookup_crls_fn X509_STORE_get_lookup_crls(const X509_STORE *ctx); +void X509_STORE_set_cleanup(X509_STORE *ctx, X509_STORE_CTX_cleanup_fn cleanup); -X509_STORE_CTX_cleanup_fn X509_STORE_get_cleanup(const X509_STORE *xs); +X509_STORE_CTX_cleanup_fn X509_STORE_get_cleanup(const X509_STORE *ctx); #define X509_STORE_get_ex_new_index(l, p, newf, dupf, freef) \ CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_X509_STORE, l, p, newf, dupf, freef) -int X509_STORE_set_ex_data(X509_STORE *xs, int idx, void *data); -void *X509_STORE_get_ex_data(const X509_STORE *xs, int idx); +int X509_STORE_set_ex_data(X509_STORE *ctx, int idx, void *data); +void *X509_STORE_get_ex_data(const X509_STORE *ctx, int idx); X509_STORE_CTX *X509_STORE_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq); X509_STORE_CTX *X509_STORE_CTX_new(void); @@ -567,14 +566,11 @@ int X509_STORE_CTX_get1_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *x); void X509_STORE_CTX_free(X509_STORE_CTX *ctx); int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *trust_store, X509 *target, STACK_OF(X509) *untrusted); -int X509_STORE_CTX_init_rpk(X509_STORE_CTX *ctx, X509_STORE *trust_store, - EVP_PKEY* rpk); void X509_STORE_CTX_set0_trusted_stack(X509_STORE_CTX *ctx, STACK_OF(X509) *sk); void X509_STORE_CTX_cleanup(X509_STORE_CTX *ctx); X509_STORE *X509_STORE_CTX_get0_store(const X509_STORE_CTX *ctx); X509 *X509_STORE_CTX_get0_cert(const X509_STORE_CTX *ctx); -EVP_PKEY *X509_STORE_CTX_get0_rpk(const X509_STORE_CTX *ctx); STACK_OF(X509)* X509_STORE_CTX_get0_untrusted(const X509_STORE_CTX *ctx); void X509_STORE_CTX_set0_untrusted(X509_STORE_CTX *ctx, STACK_OF(X509) *sk); void X509_STORE_CTX_set_verify_cb(X509_STORE_CTX *ctx, @@ -584,8 +580,6 @@ X509_STORE_CTX_verify_fn X509_STORE_CTX_get_verify(const X509_STORE_CTX *ctx); X509_STORE_CTX_get_issuer_fn X509_STORE_CTX_get_get_issuer(const X509_STORE_CTX *ctx); X509_STORE_CTX_check_issued_fn X509_STORE_CTX_get_check_issued(const X509_STORE_CTX *ctx); X509_STORE_CTX_check_revocation_fn X509_STORE_CTX_get_check_revocation(const X509_STORE_CTX *ctx); -void X509_STORE_CTX_set_get_crl(X509_STORE_CTX *ctx, - X509_STORE_CTX_get_crl_fn get_crl); X509_STORE_CTX_get_crl_fn X509_STORE_CTX_get_get_crl(const X509_STORE_CTX *ctx); X509_STORE_CTX_check_crl_fn X509_STORE_CTX_get_check_crl(const X509_STORE_CTX *ctx); X509_STORE_CTX_cert_crl_fn X509_STORE_CTX_get_cert_crl(const X509_STORE_CTX *ctx); @@ -607,7 +601,7 @@ X509_STORE_CTX_cleanup_fn X509_STORE_CTX_get_cleanup(const X509_STORE_CTX *ctx); # define X509_STORE_get1_crl X509_STORE_CTX_get1_crls #endif -X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *xs, X509_LOOKUP_METHOD *m); +X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *v, X509_LOOKUP_METHOD *m); X509_LOOKUP_METHOD *X509_LOOKUP_hash_dir(void); X509_LOOKUP_METHOD *X509_LOOKUP_file(void); X509_LOOKUP_METHOD *X509_LOOKUP_store(void); @@ -692,8 +686,8 @@ X509_LOOKUP_get_by_alias_fn X509_LOOKUP_meth_get_get_by_alias( const X509_LOOKUP_METHOD *method); -int X509_STORE_add_cert(X509_STORE *xs, X509 *x); -int X509_STORE_add_crl(X509_STORE *xs, X509_CRL *x); +int X509_STORE_add_cert(X509_STORE *ctx, X509 *x); +int X509_STORE_add_crl(X509_STORE *ctx, X509_CRL *x); int X509_STORE_CTX_get_by_subject(const X509_STORE_CTX *vs, X509_LOOKUP_TYPE type, @@ -737,21 +731,23 @@ void *X509_LOOKUP_get_method_data(const X509_LOOKUP *ctx); X509_STORE *X509_LOOKUP_get_store(const X509_LOOKUP *ctx); int X509_LOOKUP_shutdown(X509_LOOKUP *ctx); -int X509_STORE_load_file(X509_STORE *xs, const char *file); -int X509_STORE_load_path(X509_STORE *xs, const char *path); -int X509_STORE_load_store(X509_STORE *xs, const char *store); -int X509_STORE_load_locations(X509_STORE *s, const char *file, const char *dir); -int X509_STORE_set_default_paths(X509_STORE *xs); +int X509_STORE_load_file(X509_STORE *ctx, const char *file); +int X509_STORE_load_path(X509_STORE *ctx, const char *path); +int X509_STORE_load_store(X509_STORE *ctx, const char *store); +int X509_STORE_load_locations(X509_STORE *ctx, + const char *file, + const char *dir); +int X509_STORE_set_default_paths(X509_STORE *ctx); -int X509_STORE_load_file_ex(X509_STORE *xs, const char *file, +int X509_STORE_load_file_ex(X509_STORE *ctx, const char *file, OSSL_LIB_CTX *libctx, const char *propq); -int X509_STORE_load_store_ex(X509_STORE *xs, const char *store, +int X509_STORE_load_store_ex(X509_STORE *ctx, const char *store, OSSL_LIB_CTX *libctx, const char *propq); -int X509_STORE_load_locations_ex(X509_STORE *xs, - const char *file, const char *dir, - OSSL_LIB_CTX *libctx, const char *propq); -int X509_STORE_set_default_paths_ex(X509_STORE *xs, - OSSL_LIB_CTX *libctx, const char *propq); +int X509_STORE_load_locations_ex(X509_STORE *ctx, const char *file, + const char *dir, OSSL_LIB_CTX *libctx, + const char *propq); +int X509_STORE_set_default_paths_ex(X509_STORE *ctx, OSSL_LIB_CTX *libctx, + const char *propq); #define X509_STORE_CTX_get_ex_new_index(l, p, newf, dupf, freef) \ CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_X509_STORE_CTX, l, p, newf, dupf, freef) @@ -769,9 +765,10 @@ X509_STORE_CTX *X509_STORE_CTX_get0_parent_ctx(const X509_STORE_CTX *ctx); STACK_OF(X509) *X509_STORE_CTX_get0_chain(const X509_STORE_CTX *ctx); STACK_OF(X509) *X509_STORE_CTX_get1_chain(const X509_STORE_CTX *ctx); void X509_STORE_CTX_set_cert(X509_STORE_CTX *ctx, X509 *target); -void X509_STORE_CTX_set0_rpk(X509_STORE_CTX *ctx, EVP_PKEY *target); void X509_STORE_CTX_set0_verified_chain(X509_STORE_CTX *c, STACK_OF(X509) *sk); void X509_STORE_CTX_set0_crls(X509_STORE_CTX *ctx, STACK_OF(X509_CRL) *sk); +void X509_STORE_CTX_set0_vfyopts(X509_STORE_CTX *ctx, + STACK_OF(OPENSSL_STRING) *vfyopts); int X509_STORE_CTX_set_purpose(X509_STORE_CTX *ctx, int purpose); int X509_STORE_CTX_set_trust(X509_STORE_CTX *ctx, int trust); int X509_STORE_CTX_purpose_inherit(X509_STORE_CTX *ctx, int def_purpose, @@ -779,8 +776,6 @@ int X509_STORE_CTX_purpose_inherit(X509_STORE_CTX *ctx, int def_purpose, void X509_STORE_CTX_set_flags(X509_STORE_CTX *ctx, unsigned long flags); void X509_STORE_CTX_set_time(X509_STORE_CTX *ctx, unsigned long flags, time_t t); -void X509_STORE_CTX_set_current_reasons(X509_STORE_CTX *ctx, - unsigned int current_reasons); X509_POLICY_TREE *X509_STORE_CTX_get0_policy_tree(const X509_STORE_CTX *ctx); int X509_STORE_CTX_get_explicit_policy(const X509_STORE_CTX *ctx); @@ -801,6 +796,7 @@ void X509_STORE_CTX_set0_dane(X509_STORE_CTX *ctx, SSL_DANE *dane); X509_VERIFY_PARAM *X509_VERIFY_PARAM_new(void); void X509_VERIFY_PARAM_free(X509_VERIFY_PARAM *param); +int X509_VERIFY_PARAM_copy(X509_VERIFY_PARAM *dest, const X509_VERIFY_PARAM *src); int X509_VERIFY_PARAM_inherit(X509_VERIFY_PARAM *to, const X509_VERIFY_PARAM *from); int X509_VERIFY_PARAM_set1(X509_VERIFY_PARAM *to, diff --git a/openssl/include/openssl/x509_vfy.h.in b/openssl/include/openssl/x509_vfy.h.in index a396193b8..a6bea7cf9 100644 --- a/openssl/include/openssl/x509_vfy.h.in +++ b/openssl/include/openssl/x509_vfy.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -314,7 +314,6 @@ X509_LOOKUP_ctrl_ex((x), X509_L_ADD_STORE, (name), 0, NULL, \ # define X509_V_ERR_CA_CERT_MISSING_KEY_USAGE 92 # define X509_V_ERR_EXTENSIONS_REQUIRE_VERSION_3 93 # define X509_V_ERR_EC_KEY_EXPLICIT_PARAMS 94 -# define X509_V_ERR_RPK_UNTRUSTED 95 /* Certificate verify flags */ # ifndef OPENSSL_NO_DEPRECATED_1_1_0 @@ -395,72 +394,72 @@ int X509_OBJECT_set1_X509(X509_OBJECT *a, X509 *obj); X509_CRL *X509_OBJECT_get0_X509_CRL(const X509_OBJECT *a); int X509_OBJECT_set1_X509_CRL(X509_OBJECT *a, X509_CRL *obj); X509_STORE *X509_STORE_new(void); -void X509_STORE_free(X509_STORE *xs); -int X509_STORE_lock(X509_STORE *xs); -int X509_STORE_unlock(X509_STORE *xs); -int X509_STORE_up_ref(X509_STORE *xs); -STACK_OF(X509_OBJECT) *X509_STORE_get0_objects(const X509_STORE *xs); -STACK_OF(X509_OBJECT) *X509_STORE_get1_objects(X509_STORE *xs); -STACK_OF(X509) *X509_STORE_get1_all_certs(X509_STORE *xs); -STACK_OF(X509) *X509_STORE_CTX_get1_certs(X509_STORE_CTX *xs, +void X509_STORE_free(X509_STORE *v); +int X509_STORE_copy(X509_STORE *dest, const X509_STORE *src); +int X509_STORE_lock(X509_STORE *ctx); +int X509_STORE_unlock(X509_STORE *ctx); +int X509_STORE_up_ref(X509_STORE *v); +STACK_OF(X509_OBJECT) *X509_STORE_get0_objects(const X509_STORE *v); +STACK_OF(X509) *X509_STORE_get1_all_certs(X509_STORE *st); +STACK_OF(X509) *X509_STORE_CTX_get1_certs(X509_STORE_CTX *st, const X509_NAME *nm); STACK_OF(X509_CRL) *X509_STORE_CTX_get1_crls(const X509_STORE_CTX *st, const X509_NAME *nm); -int X509_STORE_set_flags(X509_STORE *xs, unsigned long flags); -int X509_STORE_set_purpose(X509_STORE *xs, int purpose); -int X509_STORE_set_trust(X509_STORE *xs, int trust); -int X509_STORE_set1_param(X509_STORE *xs, const X509_VERIFY_PARAM *pm); -X509_VERIFY_PARAM *X509_STORE_get0_param(const X509_STORE *xs); +int X509_STORE_set_flags(X509_STORE *ctx, unsigned long flags); +int X509_STORE_set_purpose(X509_STORE *ctx, int purpose); +int X509_STORE_set_trust(X509_STORE *ctx, int trust); +int X509_STORE_set1_param(X509_STORE *ctx, const X509_VERIFY_PARAM *pm); +X509_VERIFY_PARAM *X509_STORE_get0_param(const X509_STORE *ctx); -void X509_STORE_set_verify(X509_STORE *xs, X509_STORE_CTX_verify_fn verify); +void X509_STORE_set_verify(X509_STORE *ctx, X509_STORE_CTX_verify_fn verify); #define X509_STORE_set_verify_func(ctx, func) \ X509_STORE_set_verify((ctx),(func)) void X509_STORE_CTX_set_verify(X509_STORE_CTX *ctx, X509_STORE_CTX_verify_fn verify); -X509_STORE_CTX_verify_fn X509_STORE_get_verify(const X509_STORE *xs); -void X509_STORE_set_verify_cb(X509_STORE *xs, +X509_STORE_CTX_verify_fn X509_STORE_get_verify(const X509_STORE *ctx); +void X509_STORE_set_verify_cb(X509_STORE *ctx, X509_STORE_CTX_verify_cb verify_cb); # define X509_STORE_set_verify_cb_func(ctx,func) \ X509_STORE_set_verify_cb((ctx),(func)) -X509_STORE_CTX_verify_cb X509_STORE_get_verify_cb(const X509_STORE *xs); -void X509_STORE_set_get_issuer(X509_STORE *xs, +X509_STORE_CTX_verify_cb X509_STORE_get_verify_cb(const X509_STORE *ctx); +void X509_STORE_set_get_issuer(X509_STORE *ctx, X509_STORE_CTX_get_issuer_fn get_issuer); -X509_STORE_CTX_get_issuer_fn X509_STORE_get_get_issuer(const X509_STORE *xs); -void X509_STORE_set_check_issued(X509_STORE *xs, +X509_STORE_CTX_get_issuer_fn X509_STORE_get_get_issuer(const X509_STORE *ctx); +void X509_STORE_set_check_issued(X509_STORE *ctx, X509_STORE_CTX_check_issued_fn check_issued); -X509_STORE_CTX_check_issued_fn X509_STORE_get_check_issued(const X509_STORE *s); -void X509_STORE_set_check_revocation(X509_STORE *xs, +X509_STORE_CTX_check_issued_fn X509_STORE_get_check_issued(const X509_STORE *ctx); +void X509_STORE_set_check_revocation(X509_STORE *ctx, X509_STORE_CTX_check_revocation_fn check_revocation); X509_STORE_CTX_check_revocation_fn - X509_STORE_get_check_revocation(const X509_STORE *xs); -void X509_STORE_set_get_crl(X509_STORE *xs, + X509_STORE_get_check_revocation(const X509_STORE *ctx); +void X509_STORE_set_get_crl(X509_STORE *ctx, X509_STORE_CTX_get_crl_fn get_crl); -X509_STORE_CTX_get_crl_fn X509_STORE_get_get_crl(const X509_STORE *xs); -void X509_STORE_set_check_crl(X509_STORE *xs, +X509_STORE_CTX_get_crl_fn X509_STORE_get_get_crl(const X509_STORE *ctx); +void X509_STORE_set_check_crl(X509_STORE *ctx, X509_STORE_CTX_check_crl_fn check_crl); -X509_STORE_CTX_check_crl_fn X509_STORE_get_check_crl(const X509_STORE *xs); -void X509_STORE_set_cert_crl(X509_STORE *xs, +X509_STORE_CTX_check_crl_fn X509_STORE_get_check_crl(const X509_STORE *ctx); +void X509_STORE_set_cert_crl(X509_STORE *ctx, X509_STORE_CTX_cert_crl_fn cert_crl); -X509_STORE_CTX_cert_crl_fn X509_STORE_get_cert_crl(const X509_STORE *xs); -void X509_STORE_set_check_policy(X509_STORE *xs, +X509_STORE_CTX_cert_crl_fn X509_STORE_get_cert_crl(const X509_STORE *ctx); +void X509_STORE_set_check_policy(X509_STORE *ctx, X509_STORE_CTX_check_policy_fn check_policy); -X509_STORE_CTX_check_policy_fn X509_STORE_get_check_policy(const X509_STORE *s); -void X509_STORE_set_lookup_certs(X509_STORE *xs, +X509_STORE_CTX_check_policy_fn X509_STORE_get_check_policy(const X509_STORE *ctx); +void X509_STORE_set_lookup_certs(X509_STORE *ctx, X509_STORE_CTX_lookup_certs_fn lookup_certs); -X509_STORE_CTX_lookup_certs_fn X509_STORE_get_lookup_certs(const X509_STORE *s); -void X509_STORE_set_lookup_crls(X509_STORE *xs, +X509_STORE_CTX_lookup_certs_fn X509_STORE_get_lookup_certs(const X509_STORE *ctx); +void X509_STORE_set_lookup_crls(X509_STORE *ctx, X509_STORE_CTX_lookup_crls_fn lookup_crls); #define X509_STORE_set_lookup_crls_cb(ctx, func) \ X509_STORE_set_lookup_crls((ctx), (func)) -X509_STORE_CTX_lookup_crls_fn X509_STORE_get_lookup_crls(const X509_STORE *xs); -void X509_STORE_set_cleanup(X509_STORE *xs, +X509_STORE_CTX_lookup_crls_fn X509_STORE_get_lookup_crls(const X509_STORE *ctx); +void X509_STORE_set_cleanup(X509_STORE *ctx, X509_STORE_CTX_cleanup_fn cleanup); -X509_STORE_CTX_cleanup_fn X509_STORE_get_cleanup(const X509_STORE *xs); +X509_STORE_CTX_cleanup_fn X509_STORE_get_cleanup(const X509_STORE *ctx); #define X509_STORE_get_ex_new_index(l, p, newf, dupf, freef) \ CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_X509_STORE, l, p, newf, dupf, freef) -int X509_STORE_set_ex_data(X509_STORE *xs, int idx, void *data); -void *X509_STORE_get_ex_data(const X509_STORE *xs, int idx); +int X509_STORE_set_ex_data(X509_STORE *ctx, int idx, void *data); +void *X509_STORE_get_ex_data(const X509_STORE *ctx, int idx); X509_STORE_CTX *X509_STORE_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq); X509_STORE_CTX *X509_STORE_CTX_new(void); @@ -470,14 +469,11 @@ int X509_STORE_CTX_get1_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *x); void X509_STORE_CTX_free(X509_STORE_CTX *ctx); int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *trust_store, X509 *target, STACK_OF(X509) *untrusted); -int X509_STORE_CTX_init_rpk(X509_STORE_CTX *ctx, X509_STORE *trust_store, - EVP_PKEY* rpk); void X509_STORE_CTX_set0_trusted_stack(X509_STORE_CTX *ctx, STACK_OF(X509) *sk); void X509_STORE_CTX_cleanup(X509_STORE_CTX *ctx); X509_STORE *X509_STORE_CTX_get0_store(const X509_STORE_CTX *ctx); X509 *X509_STORE_CTX_get0_cert(const X509_STORE_CTX *ctx); -EVP_PKEY *X509_STORE_CTX_get0_rpk(const X509_STORE_CTX *ctx); STACK_OF(X509)* X509_STORE_CTX_get0_untrusted(const X509_STORE_CTX *ctx); void X509_STORE_CTX_set0_untrusted(X509_STORE_CTX *ctx, STACK_OF(X509) *sk); void X509_STORE_CTX_set_verify_cb(X509_STORE_CTX *ctx, @@ -487,8 +483,6 @@ X509_STORE_CTX_verify_fn X509_STORE_CTX_get_verify(const X509_STORE_CTX *ctx); X509_STORE_CTX_get_issuer_fn X509_STORE_CTX_get_get_issuer(const X509_STORE_CTX *ctx); X509_STORE_CTX_check_issued_fn X509_STORE_CTX_get_check_issued(const X509_STORE_CTX *ctx); X509_STORE_CTX_check_revocation_fn X509_STORE_CTX_get_check_revocation(const X509_STORE_CTX *ctx); -void X509_STORE_CTX_set_get_crl(X509_STORE_CTX *ctx, - X509_STORE_CTX_get_crl_fn get_crl); X509_STORE_CTX_get_crl_fn X509_STORE_CTX_get_get_crl(const X509_STORE_CTX *ctx); X509_STORE_CTX_check_crl_fn X509_STORE_CTX_get_check_crl(const X509_STORE_CTX *ctx); X509_STORE_CTX_cert_crl_fn X509_STORE_CTX_get_cert_crl(const X509_STORE_CTX *ctx); @@ -510,7 +504,7 @@ X509_STORE_CTX_cleanup_fn X509_STORE_CTX_get_cleanup(const X509_STORE_CTX *ctx); # define X509_STORE_get1_crl X509_STORE_CTX_get1_crls #endif -X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *xs, X509_LOOKUP_METHOD *m); +X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *v, X509_LOOKUP_METHOD *m); X509_LOOKUP_METHOD *X509_LOOKUP_hash_dir(void); X509_LOOKUP_METHOD *X509_LOOKUP_file(void); X509_LOOKUP_METHOD *X509_LOOKUP_store(void); @@ -595,8 +589,8 @@ X509_LOOKUP_get_by_alias_fn X509_LOOKUP_meth_get_get_by_alias( const X509_LOOKUP_METHOD *method); -int X509_STORE_add_cert(X509_STORE *xs, X509 *x); -int X509_STORE_add_crl(X509_STORE *xs, X509_CRL *x); +int X509_STORE_add_cert(X509_STORE *ctx, X509 *x); +int X509_STORE_add_crl(X509_STORE *ctx, X509_CRL *x); int X509_STORE_CTX_get_by_subject(const X509_STORE_CTX *vs, X509_LOOKUP_TYPE type, @@ -640,21 +634,23 @@ void *X509_LOOKUP_get_method_data(const X509_LOOKUP *ctx); X509_STORE *X509_LOOKUP_get_store(const X509_LOOKUP *ctx); int X509_LOOKUP_shutdown(X509_LOOKUP *ctx); -int X509_STORE_load_file(X509_STORE *xs, const char *file); -int X509_STORE_load_path(X509_STORE *xs, const char *path); -int X509_STORE_load_store(X509_STORE *xs, const char *store); -int X509_STORE_load_locations(X509_STORE *s, const char *file, const char *dir); -int X509_STORE_set_default_paths(X509_STORE *xs); +int X509_STORE_load_file(X509_STORE *ctx, const char *file); +int X509_STORE_load_path(X509_STORE *ctx, const char *path); +int X509_STORE_load_store(X509_STORE *ctx, const char *store); +int X509_STORE_load_locations(X509_STORE *ctx, + const char *file, + const char *dir); +int X509_STORE_set_default_paths(X509_STORE *ctx); -int X509_STORE_load_file_ex(X509_STORE *xs, const char *file, +int X509_STORE_load_file_ex(X509_STORE *ctx, const char *file, OSSL_LIB_CTX *libctx, const char *propq); -int X509_STORE_load_store_ex(X509_STORE *xs, const char *store, +int X509_STORE_load_store_ex(X509_STORE *ctx, const char *store, OSSL_LIB_CTX *libctx, const char *propq); -int X509_STORE_load_locations_ex(X509_STORE *xs, - const char *file, const char *dir, - OSSL_LIB_CTX *libctx, const char *propq); -int X509_STORE_set_default_paths_ex(X509_STORE *xs, - OSSL_LIB_CTX *libctx, const char *propq); +int X509_STORE_load_locations_ex(X509_STORE *ctx, const char *file, + const char *dir, OSSL_LIB_CTX *libctx, + const char *propq); +int X509_STORE_set_default_paths_ex(X509_STORE *ctx, OSSL_LIB_CTX *libctx, + const char *propq); #define X509_STORE_CTX_get_ex_new_index(l, p, newf, dupf, freef) \ CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_X509_STORE_CTX, l, p, newf, dupf, freef) @@ -672,9 +668,10 @@ X509_STORE_CTX *X509_STORE_CTX_get0_parent_ctx(const X509_STORE_CTX *ctx); STACK_OF(X509) *X509_STORE_CTX_get0_chain(const X509_STORE_CTX *ctx); STACK_OF(X509) *X509_STORE_CTX_get1_chain(const X509_STORE_CTX *ctx); void X509_STORE_CTX_set_cert(X509_STORE_CTX *ctx, X509 *target); -void X509_STORE_CTX_set0_rpk(X509_STORE_CTX *ctx, EVP_PKEY *target); void X509_STORE_CTX_set0_verified_chain(X509_STORE_CTX *c, STACK_OF(X509) *sk); void X509_STORE_CTX_set0_crls(X509_STORE_CTX *ctx, STACK_OF(X509_CRL) *sk); +void X509_STORE_CTX_set0_vfyopts(X509_STORE_CTX *ctx, + STACK_OF(OPENSSL_STRING) *vfyopts); int X509_STORE_CTX_set_purpose(X509_STORE_CTX *ctx, int purpose); int X509_STORE_CTX_set_trust(X509_STORE_CTX *ctx, int trust); int X509_STORE_CTX_purpose_inherit(X509_STORE_CTX *ctx, int def_purpose, @@ -682,8 +679,6 @@ int X509_STORE_CTX_purpose_inherit(X509_STORE_CTX *ctx, int def_purpose, void X509_STORE_CTX_set_flags(X509_STORE_CTX *ctx, unsigned long flags); void X509_STORE_CTX_set_time(X509_STORE_CTX *ctx, unsigned long flags, time_t t); -void X509_STORE_CTX_set_current_reasons(X509_STORE_CTX *ctx, - unsigned int current_reasons); X509_POLICY_TREE *X509_STORE_CTX_get0_policy_tree(const X509_STORE_CTX *ctx); int X509_STORE_CTX_get_explicit_policy(const X509_STORE_CTX *ctx); @@ -704,6 +699,7 @@ void X509_STORE_CTX_set0_dane(X509_STORE_CTX *ctx, SSL_DANE *dane); X509_VERIFY_PARAM *X509_VERIFY_PARAM_new(void); void X509_VERIFY_PARAM_free(X509_VERIFY_PARAM *param); +int X509_VERIFY_PARAM_copy(X509_VERIFY_PARAM *dest, const X509_VERIFY_PARAM *src); int X509_VERIFY_PARAM_inherit(X509_VERIFY_PARAM *to, const X509_VERIFY_PARAM *from); int X509_VERIFY_PARAM_set1(X509_VERIFY_PARAM *to, diff --git a/openssl/include/openssl/x509err.h b/openssl/include/openssl/x509err.h index 71b557a3e..a56facd46 100644 --- a/openssl/include/openssl/x509err.h +++ b/openssl/include/openssl/x509err.h @@ -30,7 +30,6 @@ # define X509_R_CERT_ALREADY_IN_HASH_TABLE 101 # define X509_R_CRL_ALREADY_DELTA 127 # define X509_R_CRL_VERIFY_FAILURE 131 -# define X509_R_DUPLICATE_ATTRIBUTE 140 # define X509_R_ERROR_GETTING_MD_BY_NID 141 # define X509_R_ERROR_USING_SIGINF_SET 142 # define X509_R_IDP_MISMATCH 128 diff --git a/openssl/include/openssl/x509v3.h b/openssl/include/openssl/x509v3.h index 1fe395e0e..31addd1c8 100644 --- a/openssl/include/openssl/x509v3.h +++ b/openssl/include/openssl/x509v3.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by Makefile from ../../openssl/include/openssl/x509v3.h.in * - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,9 +25,6 @@ # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -396,13 +393,6 @@ typedef struct SXNET_st { STACK_OF(SXNETID) *ids; } SXNET; -typedef struct ISSUER_SIGN_TOOL_st { - ASN1_UTF8STRING *signTool; - ASN1_UTF8STRING *cATool; - ASN1_UTF8STRING *signToolCert; - ASN1_UTF8STRING *cAToolCert; -} ISSUER_SIGN_TOOL; - typedef struct NOTICEREF_st { ASN1_STRING *organization; STACK_OF(ASN1_INTEGER) *noticenos; @@ -627,13 +617,6 @@ struct ISSUING_DIST_POINT_st { 0,0,0,0, \ NULL} -#define EXT_UTF8STRING(nid) { nid, 0, ASN1_ITEM_ref(ASN1_UTF8STRING), \ - 0,0,0,0, \ - (X509V3_EXT_I2S)i2s_ASN1_UTF8STRING, \ - (X509V3_EXT_S2I)s2i_ASN1_UTF8STRING, \ - 0,0,0,0, \ - NULL} - # define EXT_END { -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} /* X509_PURPOSE stuff */ @@ -742,10 +725,9 @@ SKM_DEFINE_STACK_OF_INTERNAL(X509_PURPOSE, X509_PURPOSE, X509_PURPOSE) # define X509_PURPOSE_ANY 7 # define X509_PURPOSE_OCSP_HELPER 8 # define X509_PURPOSE_TIMESTAMP_SIGN 9 -# define X509_PURPOSE_CODE_SIGN 10 # define X509_PURPOSE_MIN 1 -# define X509_PURPOSE_MAX 10 +# define X509_PURPOSE_MAX 9 /* Flags for X509V3_EXT_print() */ @@ -775,8 +757,6 @@ DECLARE_ASN1_FUNCTIONS(BASIC_CONSTRAINTS) DECLARE_ASN1_FUNCTIONS(SXNET) DECLARE_ASN1_FUNCTIONS(SXNETID) -DECLARE_ASN1_FUNCTIONS(ISSUER_SIGN_TOOL) - int SXNET_add_id_asc(SXNET **psx, const char *zone, const char *user, int userlen); int SXNET_add_id_ulong(SXNET **psx, unsigned long lzone, const char *user, int userlen); @@ -804,9 +784,6 @@ STACK_OF(CONF_VALUE) *i2v_ASN1_BIT_STRING(X509V3_EXT_METHOD *method, char *i2s_ASN1_IA5STRING(X509V3_EXT_METHOD *method, ASN1_IA5STRING *ia5); ASN1_IA5STRING *s2i_ASN1_IA5STRING(X509V3_EXT_METHOD *method, X509V3_CTX *ctx, const char *str); -char *i2s_ASN1_UTF8STRING(X509V3_EXT_METHOD *method, ASN1_UTF8STRING *utf8); -ASN1_UTF8STRING *s2i_ASN1_UTF8STRING(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *str); STACK_OF(CONF_VALUE) *i2v_GENERAL_NAME(X509V3_EXT_METHOD *method, GENERAL_NAME *gen, diff --git a/openssl/include/openssl/x509v3.h.in b/openssl/include/openssl/x509v3.h.in index 569680378..9b9df6156 100644 --- a/openssl/include/openssl/x509v3.h.in +++ b/openssl/include/openssl/x509v3.h.in @@ -1,7 +1,7 @@ /* * {- join("\n * ", @autowarntext) -} * - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,9 +26,6 @@ use OpenSSL::stackhash qw(generate_stack_macros); # include # include # include -# ifndef OPENSSL_NO_STDIO -# include -# endif #ifdef __cplusplus extern "C" { @@ -252,13 +249,6 @@ typedef struct SXNET_st { STACK_OF(SXNETID) *ids; } SXNET; -typedef struct ISSUER_SIGN_TOOL_st { - ASN1_UTF8STRING *signTool; - ASN1_UTF8STRING *cATool; - ASN1_UTF8STRING *signToolCert; - ASN1_UTF8STRING *cAToolCert; -} ISSUER_SIGN_TOOL; - typedef struct NOTICEREF_st { ASN1_STRING *organization; STACK_OF(ASN1_INTEGER) *noticenos; @@ -387,13 +377,6 @@ struct ISSUING_DIST_POINT_st { 0,0,0,0, \ NULL} -#define EXT_UTF8STRING(nid) { nid, 0, ASN1_ITEM_ref(ASN1_UTF8STRING), \ - 0,0,0,0, \ - (X509V3_EXT_I2S)i2s_ASN1_UTF8STRING, \ - (X509V3_EXT_S2I)s2i_ASN1_UTF8STRING, \ - 0,0,0,0, \ - NULL} - # define EXT_END { -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} /* X509_PURPOSE stuff */ @@ -478,10 +461,9 @@ typedef struct x509_purpose_st { # define X509_PURPOSE_ANY 7 # define X509_PURPOSE_OCSP_HELPER 8 # define X509_PURPOSE_TIMESTAMP_SIGN 9 -# define X509_PURPOSE_CODE_SIGN 10 # define X509_PURPOSE_MIN 1 -# define X509_PURPOSE_MAX 10 +# define X509_PURPOSE_MAX 9 /* Flags for X509V3_EXT_print() */ @@ -511,8 +493,6 @@ DECLARE_ASN1_FUNCTIONS(BASIC_CONSTRAINTS) DECLARE_ASN1_FUNCTIONS(SXNET) DECLARE_ASN1_FUNCTIONS(SXNETID) -DECLARE_ASN1_FUNCTIONS(ISSUER_SIGN_TOOL) - int SXNET_add_id_asc(SXNET **psx, const char *zone, const char *user, int userlen); int SXNET_add_id_ulong(SXNET **psx, unsigned long lzone, const char *user, int userlen); @@ -540,9 +520,6 @@ STACK_OF(CONF_VALUE) *i2v_ASN1_BIT_STRING(X509V3_EXT_METHOD *method, char *i2s_ASN1_IA5STRING(X509V3_EXT_METHOD *method, ASN1_IA5STRING *ia5); ASN1_IA5STRING *s2i_ASN1_IA5STRING(X509V3_EXT_METHOD *method, X509V3_CTX *ctx, const char *str); -char *i2s_ASN1_UTF8STRING(X509V3_EXT_METHOD *method, ASN1_UTF8STRING *utf8); -ASN1_UTF8STRING *s2i_ASN1_UTF8STRING(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *str); STACK_OF(CONF_VALUE) *i2v_GENERAL_NAME(X509V3_EXT_METHOD *method, GENERAL_NAME *gen, diff --git a/openssl/include/openssl/x509v3err.h b/openssl/include/openssl/x509v3err.h index deede2795..1ae3a5620 100644 --- a/openssl/include/openssl/x509v3err.h +++ b/openssl/include/openssl/x509v3err.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,8 +23,6 @@ */ # define X509V3_R_BAD_IP_ADDRESS 118 # define X509V3_R_BAD_OBJECT 119 -# define X509V3_R_BAD_OPTION 170 -# define X509V3_R_BAD_VALUE 171 # define X509V3_R_BN_DEC2BN_ERROR 100 # define X509V3_R_BN_TO_ASN1_INTEGER_ERROR 101 # define X509V3_R_DIRNAME_ERROR 149 @@ -88,7 +86,6 @@ # define X509V3_R_UNKNOWN_EXTENSION 129 # define X509V3_R_UNKNOWN_EXTENSION_NAME 130 # define X509V3_R_UNKNOWN_OPTION 120 -# define X509V3_R_UNKNOWN_VALUE 172 # define X509V3_R_UNSUPPORTED_OPTION 117 # define X509V3_R_UNSUPPORTED_TYPE 167 # define X509V3_R_USER_TOO_LONG 132 diff --git a/openssl/include/openssl/zkp_gadget.h b/openssl/include/openssl/zkp_gadget.h new file mode 100644 index 000000000..2af9611b8 --- /dev/null +++ b/openssl/include/openssl/zkp_gadget.h @@ -0,0 +1,68 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_ZKP_GADGET_H +# define HEADER_ZKP_GADGET_H + +# include +# include +# include +# include +# include +# include +# include +# include + +# ifndef OPENSSL_NO_ZKP_GADGET +# ifdef __cplusplus +extern "C" { +# endif + +typedef struct zkp_range_pub_param_st ZKP_RANGE_PUB_PARAM; +typedef struct zkp_range_witness_st ZKP_RANGE_WITNESS; +typedef struct zkp_range_ctx_st ZKP_RANGE_CTX; +typedef struct zkp_range_proof_st ZKP_RANGE_PROOF; + +ZKP_RANGE_PUB_PARAM *ZKP_RANGE_PUB_PARAM_raw_new(BP_PUB_PARAM *bp_pp); +ZKP_RANGE_PUB_PARAM *ZKP_RANGE_PUB_PARAM_new(const EC_GROUP *group, int max_bits); +void ZKP_RANGE_PUB_PARAM_free(ZKP_RANGE_PUB_PARAM *pp); +int ZKP_RANGE_PUB_PARAM_up_ref(ZKP_RANGE_PUB_PARAM *pp); +int ZKP_RANGE_PUB_PARAM_down_ref(ZKP_RANGE_PUB_PARAM *pp); + +ZKP_RANGE_WITNESS *ZKP_RANGE_WITNESS_new(const ZKP_RANGE_PUB_PARAM *pp, + const BIGNUM *r, const BIGNUM *v); +void ZKP_RANGE_WITNESS_free(ZKP_RANGE_WITNESS *witness); +int ZKP_RANGE_WITNESS_up_ref(ZKP_RANGE_WITNESS *witness); +int ZKP_RANGE_WITNESS_down_ref(ZKP_RANGE_WITNESS *witness); + +ZKP_RANGE_CTX *ZKP_RANGE_CTX_raw_new(ZKP_TRANSCRIPT *transcript, + ZKP_RANGE_PUB_PARAM *pp, + ZKP_RANGE_WITNESS *witness, + const EC_POINT *pk, + EC_ELGAMAL_CTX *enc_ctx, + EC_ELGAMAL_CIPHERTEXT *enc_ct); +ZKP_RANGE_CTX *ZKP_RANGE_CTX_new(ZKP_TRANSCRIPT *transcript, + ZKP_RANGE_PUB_PARAM *pp, + ZKP_RANGE_WITNESS *witness, + EC_KEY *key); +void ZKP_RANGE_CTX_free(ZKP_RANGE_CTX *ctx); + +ZKP_RANGE_PROOF *ZKP_RANGE_PROOF_new(void); +void ZKP_RANGE_PROOF_free(ZKP_RANGE_PROOF *proof); +ZKP_RANGE_PROOF *ZKP_RANGE_PROOF_prove(ZKP_RANGE_CTX *ctx, int left_bound_bits, + int right_bound_bits); +int ZKP_RANGE_PROOF_verify(ZKP_RANGE_CTX *ctx, ZKP_RANGE_PROOF *proof, + int left_bound_bits, int right_bound_bits); + +# ifdef __cplusplus +} +# endif +# endif + +#endif diff --git a/openssl/include/openssl/zkp_transcript.h b/openssl/include/openssl/zkp_transcript.h new file mode 100644 index 000000000..3067f840e --- /dev/null +++ b/openssl/include/openssl/zkp_transcript.h @@ -0,0 +1,51 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_ZKP_TRANSCRIPT_H +# define HEADER_ZKP_TRANSCRIPT_H + +# include +# include +# include +# include +# include + +# ifndef OPENSSL_NO_ZKP_TRANSCRIPT +# ifdef __cplusplus +extern "C" { +# endif + +typedef struct zkp_transcript_method_st ZKP_TRANSCRIPT_METHOD; +typedef struct zkp_transcript_st ZKP_TRANSCRIPT; + +ZKP_TRANSCRIPT *ZKP_TRANSCRIPT_new(const ZKP_TRANSCRIPT_METHOD *method, + const char *label); +ZKP_TRANSCRIPT *ZKP_TRANSCRIPT_dup(const ZKP_TRANSCRIPT *src); +void ZKP_TRANSCRIPT_free(ZKP_TRANSCRIPT *transcript); +int ZKP_TRANSCRIPT_reset(ZKP_TRANSCRIPT *transcript); + +int ZKP_TRANSCRIPT_append_int64(ZKP_TRANSCRIPT *transcript, const char *label, + int64_t i64); +int ZKP_TRANSCRIPT_append_str(ZKP_TRANSCRIPT *transcript, const char *label, + const char *str, int len); +int ZKP_TRANSCRIPT_append_point(ZKP_TRANSCRIPT *transcript, const char *label, + const EC_POINT *point, const EC_GROUP *group); +int ZKP_TRANSCRIPT_append_bn(ZKP_TRANSCRIPT *transcript, const char *label, + const BIGNUM *bn); +int ZKP_TRANSCRIPT_challange(ZKP_TRANSCRIPT *transcript, const char *label, + BIGNUM *out); + +const ZKP_TRANSCRIPT_METHOD *ZKP_TRANSCRIPT_METHOD_sha256(void); + +# ifdef __cplusplus +} +# endif +# endif + +#endif diff --git a/openssl/include/openssl/zkpbperr.h b/openssl/include/openssl/zkpbperr.h new file mode 100644 index 000000000..05118c634 --- /dev/null +++ b/openssl/include/openssl/zkpbperr.h @@ -0,0 +1,40 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OPENSSL_ZKPBPERR_H +# define OPENSSL_ZKPBPERR_H +# pragma once + +# include +# include +# include + + + +/* + * ZKP_BP reason codes. + */ +# define ZKP_BP_R_EXCEEDS_GENS_CAPACITY 106 +# define ZKP_BP_R_EXCEEDS_MAX_AGG_NUM 100 +# define ZKP_BP_R_EXCEEDS_MAX_BITS 101 +# define ZKP_BP_R_EXCEEDS_PARTY_CAPACITY 104 +# define ZKP_BP_R_EXCEEDS_PP_CAPACITY 102 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_FORMAT_ERROR 109 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_NO_VAR 110 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_PROCESS_ERROR 111 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_NOT_FOUND 112 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_TOO_LONG 113 +# define ZKP_BP_R_RANGE_LEN_MUST_BE_POWER_OF_TWO 114 +# define ZKP_BP_R_TRANSCRIPT_INIT_FAILED 103 +# define ZKP_BP_R_VARIABLE_DUPLICATED 107 +# define ZKP_BP_R_VARIABLE_NAME_TOO_LONG 108 +# define ZKP_BP_R_WITNESS_INVALID 105 + +#endif diff --git a/openssl/include/openssl/zkperr.h b/openssl/include/openssl/zkperr.h new file mode 100644 index 000000000..15110ba26 --- /dev/null +++ b/openssl/include/openssl/zkperr.h @@ -0,0 +1,47 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OPENSSL_ZKPERR_H +# define OPENSSL_ZKPERR_H +# pragma once + +# include +# include +# include + + + +/* + * ZKP reason codes. + */ +# define ZKP_BP_R_EXCEEDS_GENS_CAPACITY 106 +# define ZKP_BP_R_EXCEEDS_MAX_AGG_NUM 100 +# define ZKP_BP_R_EXCEEDS_MAX_BITS 101 +# define ZKP_BP_R_EXCEEDS_PARTY_CAPACITY 104 +# define ZKP_BP_R_EXCEEDS_PP_CAPACITY 102 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_FORMAT_ERROR 109 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_NO_VAR 110 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_PROCESS_ERROR 111 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_NOT_FOUND 112 +# define ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_TOO_LONG 113 +# define ZKP_BP_R_TRANSCRIPT_INIT_FAILED 103 +# define ZKP_BP_R_VARIABLE_DUPLICATED 107 +# define ZKP_BP_R_VARIABLE_NAME_TOO_LONG 108 +# define ZKP_BP_R_WITNESS_INVALID 105 +# define ZKP_NIZK_R_TRANSCRIPT_INIT_FAILED 100 +# define ZKP_R_BULLETPROOFS_RANGE_PROVE_FAILED 101 +# define ZKP_R_BULLETPROOFS_RANGE_VERIFY_FAILED 102 +# define ZKP_R_NIZK_PLAINTEXT_KNOWLEDGE_PROVE_FAILED 103 +# define ZKP_R_NIZK_PLAINTEXT_KNOWLEDGE_VERIFY_FAILED 104 +# define ZKP_R_RANGE_PROVE_FAILED 105 +# define ZKP_R_RANGE_VERIFY_FAILED 106 +# define ZKP_R_TRANSCRIPT_INIT_FAILED 100 + +#endif diff --git a/openssl/include/openssl/zkpnizkerr.h b/openssl/include/openssl/zkpnizkerr.h new file mode 100644 index 000000000..264de3ffc --- /dev/null +++ b/openssl/include/openssl/zkpnizkerr.h @@ -0,0 +1,26 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OPENSSL_ZKPNIZKERR_H +# define OPENSSL_ZKPNIZKERR_H +# pragma once + +# include +# include +# include + + + +/* + * ZKP_NIZK reason codes. + */ +# define ZKP_NIZK_R_TRANSCRIPT_INIT_FAILED 100 + +#endif diff --git a/openssl/include/prov/der_digests.h b/openssl/include/prov/der_digests.h index b8f7356de..69def411a 100644 --- a/openssl/include/prov/der_digests.h +++ b/openssl/include/prov/der_digests.h @@ -30,14 +30,6 @@ extern const unsigned char ossl_der_oid_sigAlgs[DER_OID_SZ_sigAlgs]; #define DER_OID_SZ_id_sha1 7 extern const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1]; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -#define DER_OID_V_id_md2 DER_P_OBJECT, 8, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x02, 0x02 -#define DER_OID_SZ_id_md2 10 -extern const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2]; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/include/prov/der_rsa.h b/openssl/include/prov/der_rsa.h index ffabcf00e..4918c503e 100644 --- a/openssl/include/prov/der_rsa.h +++ b/openssl/include/prov/der_rsa.h @@ -50,13 +50,6 @@ extern const unsigned char ossl_der_oid_id_pSpecified[DER_OID_SZ_id_pSpecified]; #define DER_OID_SZ_id_RSASSA_PSS 11 extern const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS]; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -#define DER_OID_V_md2WithRSAEncryption DER_P_OBJECT, 9, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x01, 0x02 -#define DER_OID_SZ_md2WithRSAEncryption 11 -extern const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption]; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -148,31 +141,6 @@ extern const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_384[DER_O #define DER_OID_SZ_id_rsassa_pkcs1_v1_5_with_sha3_512 11 extern const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_id_rsassa_pkcs1_v1_5_with_sha3_512]; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -#define DER_OID_V_md4WithRSAEncryption DER_P_OBJECT, 9, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x01, 0x03 -#define DER_OID_SZ_md4WithRSAEncryption 11 -extern const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption]; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -#define DER_OID_V_ripemd160WithRSAEncryption DER_P_OBJECT, 6, 0x2B, 0x24, 0x03, 0x03, 0x01, 0x02 -#define DER_OID_SZ_ripemd160WithRSAEncryption 8 -extern const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption]; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -#define DER_OID_V_mdc2WithRSASignature DER_P_OBJECT, 5, 0x2B, 0x0E, 0x03, 0x02, 0x0E -#define DER_OID_SZ_mdc2WithRSASignature 7 -extern const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature]; - /* PSS parameters */ int ossl_DER_w_RSASSA_PSS_params(WPACKET *pkt, int tag, diff --git a/openssl/src/crypto/LPdir_nyi.c b/openssl/src/crypto/LPdir_nyi.c deleted file mode 100644 index a1540785f..000000000 --- a/openssl/src/crypto/LPdir_nyi.c +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2004-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is dual-licensed and is also available under the following - * terms: - * - * Copyright (c) 2004, Richard Levitte - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef LPDIR_H -# include "LPdir.h" -#endif - -struct LP_dir_context_st { - void *dummy; -}; -const char *LP_find_file(LP_DIR_CTX **ctx, const char *directory) -{ - errno = EINVAL; - return 0; -} - -int LP_find_file_end(LP_DIR_CTX **ctx) -{ - errno = EINVAL; - return 0; -} diff --git a/openssl/src/crypto/LPdir_unix.c b/openssl/src/crypto/LPdir_unix.c deleted file mode 100644 index b6dda7bce..000000000 --- a/openssl/src/crypto/LPdir_unix.c +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2004-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is dual-licensed and is also available under the following - * terms: - * - * Copyright (c) 2004, 2018, Richard Levitte - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#ifndef LPDIR_H -# include "LPdir.h" -#endif -#ifdef __VMS -# include -#endif - -/* - * The POSIX macro for the maximum number of characters in a file path is - * NAME_MAX. However, some operating systems use PATH_MAX instead. - * Therefore, it seems natural to first check for PATH_MAX and use that, and - * if it doesn't exist, use NAME_MAX. - */ -#if defined(PATH_MAX) -# define LP_ENTRY_SIZE PATH_MAX -#elif defined(NAME_MAX) -# define LP_ENTRY_SIZE NAME_MAX -#endif - -/* - * Of course, there's the possibility that neither PATH_MAX nor NAME_MAX - * exist. It's also possible that NAME_MAX exists but is define to a very - * small value (HP-UX offers 14), so we need to check if we got a result, and - * if it meets a minimum standard, and create or change it if not. - */ -#if !defined(LP_ENTRY_SIZE) || LP_ENTRY_SIZE<255 -# undef LP_ENTRY_SIZE -# define LP_ENTRY_SIZE 255 -#endif - -struct LP_dir_context_st { - DIR *dir; - char entry_name[LP_ENTRY_SIZE + 1]; -#ifdef __VMS - int expect_file_generations; - char previous_entry_name[LP_ENTRY_SIZE + 1]; -#endif -}; - -const char *LP_find_file(LP_DIR_CTX **ctx, const char *directory) -{ - struct dirent *direntry = NULL; - - if (ctx == NULL || directory == NULL) { - errno = EINVAL; - return 0; - } - - errno = 0; - if (*ctx == NULL) { - *ctx = malloc(sizeof(**ctx)); - if (*ctx == NULL) { - errno = ENOMEM; - return 0; - } - memset(*ctx, 0, sizeof(**ctx)); - -#ifdef __VMS - { - char c = directory[strlen(directory) - 1]; - - if (c == ']' || c == '>' || c == ':') - (*ctx)->expect_file_generations = 1; - } -#endif - - (*ctx)->dir = opendir(directory); - if ((*ctx)->dir == NULL) { - int save_errno = errno; /* Probably not needed, but I'm paranoid */ - free(*ctx); - *ctx = NULL; - errno = save_errno; - return 0; - } - } - -#ifdef __VMS - strncpy((*ctx)->previous_entry_name, (*ctx)->entry_name, - sizeof((*ctx)->previous_entry_name)); - - again: -#endif - - direntry = readdir((*ctx)->dir); - if (direntry == NULL) { - return 0; - } - - OPENSSL_strlcpy((*ctx)->entry_name, direntry->d_name, - sizeof((*ctx)->entry_name)); -#ifdef __VMS - if ((*ctx)->expect_file_generations) { - char *p = (*ctx)->entry_name + strlen((*ctx)->entry_name); - - while (p > (*ctx)->entry_name && isdigit((unsigned char)p[-1])) - p--; - if (p > (*ctx)->entry_name && p[-1] == ';') - p[-1] = '\0'; - if (OPENSSL_strcasecmp((*ctx)->entry_name, - (*ctx)->previous_entry_name) == 0) - goto again; - } -#endif - return (*ctx)->entry_name; -} - -int LP_find_file_end(LP_DIR_CTX **ctx) -{ - if (ctx != NULL && *ctx != NULL) { - int ret = closedir((*ctx)->dir); - - free(*ctx); - switch (ret) { - case 0: - return 1; - case -1: - return 0; - default: - break; - } - } - errno = EINVAL; - return 0; -} diff --git a/openssl/src/crypto/LPdir_vms.c b/openssl/src/crypto/LPdir_vms.c deleted file mode 100644 index 51043263a..000000000 --- a/openssl/src/crypto/LPdir_vms.c +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright 2004-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is dual-licensed and is also available under the following - * terms: - * - * Copyright (c) 2004, Richard Levitte - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef LPDIR_H -# include "LPdir.h" -#endif -#include "vms_rms.h" - -/* Some compiler options hide EVMSERR. */ -#ifndef EVMSERR -# define EVMSERR 65535 /* error for non-translatable VMS errors */ -#endif - -struct LP_dir_context_st { - unsigned long VMS_context; - char filespec[NAMX_MAXRSS + 1]; - char result[NAMX_MAXRSS + 1]; - struct dsc$descriptor_d filespec_dsc; - struct dsc$descriptor_d result_dsc; -}; - -const char *LP_find_file(LP_DIR_CTX **ctx, const char *directory) -{ - int status; - char *p, *r; - size_t l; - unsigned long flags = 0; - -/* Arrange 32-bit pointer to (copied) string storage, if needed. */ -#if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size save -# pragma pointer_size 32 - char *ctx_filespec_32p; -# pragma pointer_size restore - char ctx_filespec_32[NAMX_MAXRSS + 1]; -#endif /* __INITIAL_POINTER_SIZE == 64 */ - -#ifdef NAML$C_MAXRSS - flags |= LIB$M_FIL_LONG_NAMES; -#endif - - if (ctx == NULL || directory == NULL) { - errno = EINVAL; - return 0; - } - - errno = 0; - if (*ctx == NULL) { - size_t filespeclen = strlen(directory); - char *filespec = NULL; - - if (filespeclen == 0) { - errno = ENOENT; - return 0; - } - - /* MUST be a VMS directory specification! Let's estimate if it is. */ - if (directory[filespeclen - 1] != ']' - && directory[filespeclen - 1] != '>' - && directory[filespeclen - 1] != ':') { - errno = EINVAL; - return 0; - } - - filespeclen += 4; /* "*.*;" */ - - if (filespeclen > NAMX_MAXRSS) { - errno = ENAMETOOLONG; - return 0; - } - - *ctx = malloc(sizeof(**ctx)); - if (*ctx == NULL) { - errno = ENOMEM; - return 0; - } - memset(*ctx, 0, sizeof(**ctx)); - - strcpy((*ctx)->filespec, directory); - strcat((*ctx)->filespec, "*.*;"); - -/* Arrange 32-bit pointer to (copied) string storage, if needed. */ -#if __INITIAL_POINTER_SIZE == 64 -# define CTX_FILESPEC ctx_filespec_32p - /* Copy the file name to storage with a 32-bit pointer. */ - ctx_filespec_32p = ctx_filespec_32; - strcpy(ctx_filespec_32p, (*ctx)->filespec); -#else /* __INITIAL_POINTER_SIZE == 64 */ -# define CTX_FILESPEC (*ctx)->filespec -#endif /* __INITIAL_POINTER_SIZE == 64 [else] */ - - (*ctx)->filespec_dsc.dsc$w_length = filespeclen; - (*ctx)->filespec_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - (*ctx)->filespec_dsc.dsc$b_class = DSC$K_CLASS_S; - (*ctx)->filespec_dsc.dsc$a_pointer = CTX_FILESPEC; - } - - (*ctx)->result_dsc.dsc$w_length = 0; - (*ctx)->result_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - (*ctx)->result_dsc.dsc$b_class = DSC$K_CLASS_D; - (*ctx)->result_dsc.dsc$a_pointer = 0; - - status = lib$find_file(&(*ctx)->filespec_dsc, &(*ctx)->result_dsc, - &(*ctx)->VMS_context, 0, 0, 0, &flags); - - if (status == RMS$_NMF) { - errno = 0; - vaxc$errno = status; - return NULL; - } - - if (!$VMS_STATUS_SUCCESS(status)) { - errno = EVMSERR; - vaxc$errno = status; - return NULL; - } - - /* - * Quick, cheap and dirty way to discard any device and directory, since - * we only want file names - */ - l = (*ctx)->result_dsc.dsc$w_length; - p = (*ctx)->result_dsc.dsc$a_pointer; - r = p; - for (; *p; p++) { - if (*p == '^' && p[1] != '\0') { /* Take care of ODS-5 escapes */ - p++; - } else if (*p == ':' || *p == '>' || *p == ']') { - l -= p + 1 - r; - r = p + 1; - } else if (*p == ';') { - l = p - r; - break; - } - } - - strncpy((*ctx)->result, r, l); - (*ctx)->result[l] = '\0'; - str$free1_dx(&(*ctx)->result_dsc); - - return (*ctx)->result; -} - -int LP_find_file_end(LP_DIR_CTX **ctx) -{ - if (ctx != NULL && *ctx != NULL) { - int status = lib$find_file_end(&(*ctx)->VMS_context); - - free(*ctx); - - if (!$VMS_STATUS_SUCCESS(status)) { - errno = EVMSERR; - vaxc$errno = status; - return 0; - } - return 1; - } - errno = EINVAL; - return 0; -} diff --git a/openssl/src/crypto/LPdir_win.c b/openssl/src/crypto/LPdir_win.c deleted file mode 100644 index 83cbe1fc0..000000000 --- a/openssl/src/crypto/LPdir_win.c +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2004-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is dual-licensed and is also available under the following - * terms: - * - * Copyright (c) 2004, Richard Levitte - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include "internal/numbers.h" -#ifndef LPDIR_H -# include "LPdir.h" -#endif - -/* - * We're most likely overcautious here, but let's reserve for broken WinCE - * headers and explicitly opt for UNICODE call. Keep in mind that our WinCE - * builds are compiled with -DUNICODE [as well as -D_UNICODE]. - */ -#if defined(LP_SYS_WINCE) && !defined(FindFirstFile) -# define FindFirstFile FindFirstFileW -#endif -#if defined(LP_SYS_WINCE) && !defined(FindNextFile) -# define FindNextFile FindNextFileW -#endif - -#ifndef NAME_MAX -# define NAME_MAX 255 -#endif - -#ifdef CP_UTF8 -# define CP_DEFAULT CP_UTF8 -#else -# define CP_DEFAULT CP_ACP -#endif - -struct LP_dir_context_st { - WIN32_FIND_DATA ctx; - HANDLE handle; - char entry_name[NAME_MAX + 1]; -}; - -const char *LP_find_file(LP_DIR_CTX **ctx, const char *directory) -{ - if (ctx == NULL || directory == NULL) { - errno = EINVAL; - return 0; - } - - errno = 0; - if (*ctx == NULL) { - size_t dirlen = strlen(directory); - - if (dirlen == 0 || dirlen > INT_MAX - 3) { - errno = ENOENT; - return 0; - } - - *ctx = malloc(sizeof(**ctx)); - if (*ctx == NULL) { - errno = ENOMEM; - return 0; - } - memset(*ctx, 0, sizeof(**ctx)); - - if (sizeof(TCHAR) != sizeof(char)) { - TCHAR *wdir = NULL; - /* len_0 denotes string length *with* trailing 0 */ - size_t index = 0, len_0 = dirlen + 1; -#ifdef LP_MULTIBYTE_AVAILABLE - int sz = 0; - UINT cp; - - do { -# ifdef CP_UTF8 - if ((sz = MultiByteToWideChar((cp = CP_UTF8), 0, - directory, len_0, - NULL, 0)) > 0 || - GetLastError() != ERROR_NO_UNICODE_TRANSLATION) - break; -# endif - sz = MultiByteToWideChar((cp = CP_ACP), 0, - directory, len_0, - NULL, 0); - } while (0); - - if (sz > 0) { - /* - * allocate two additional characters in case we need to - * concatenate asterisk, |sz| covers trailing '\0'! - */ - wdir = _alloca((sz + 2) * sizeof(TCHAR)); - if (!MultiByteToWideChar(cp, 0, directory, len_0, - (WCHAR *)wdir, sz)) { - free(*ctx); - *ctx = NULL; - errno = EINVAL; - return 0; - } - } else -#endif - { - sz = len_0; - /* - * allocate two additional characters in case we need to - * concatenate asterisk, |sz| covers trailing '\0'! - */ - wdir = _alloca((sz + 2) * sizeof(TCHAR)); - for (index = 0; index < len_0; index++) - wdir[index] = (TCHAR)directory[index]; - } - - sz--; /* wdir[sz] is trailing '\0' now */ - if (wdir[sz - 1] != TEXT('*')) { - if (wdir[sz - 1] != TEXT('/') && wdir[sz - 1] != TEXT('\\')) - _tcscpy(wdir + sz, TEXT("/*")); - else - _tcscpy(wdir + sz, TEXT("*")); - } - - (*ctx)->handle = FindFirstFile(wdir, &(*ctx)->ctx); - } else { - if (directory[dirlen - 1] != '*') { - char *buf = _alloca(dirlen + 3); - - strcpy(buf, directory); - if (buf[dirlen - 1] != '/' && buf[dirlen - 1] != '\\') - strcpy(buf + dirlen, "/*"); - else - strcpy(buf + dirlen, "*"); - - directory = buf; - } - - (*ctx)->handle = FindFirstFile((TCHAR *)directory, &(*ctx)->ctx); - } - - if ((*ctx)->handle == INVALID_HANDLE_VALUE) { - free(*ctx); - *ctx = NULL; - errno = EINVAL; - return 0; - } - } else { - if (FindNextFile((*ctx)->handle, &(*ctx)->ctx) == FALSE) { - return 0; - } - } - if (sizeof(TCHAR) != sizeof(char)) { - TCHAR *wdir = (*ctx)->ctx.cFileName; - size_t index, len_0 = 0; - - while (wdir[len_0] && len_0 < (sizeof((*ctx)->entry_name) - 1)) - len_0++; - len_0++; - -#ifdef LP_MULTIBYTE_AVAILABLE - if (!WideCharToMultiByte(CP_DEFAULT, 0, (WCHAR *)wdir, len_0, - (*ctx)->entry_name, - sizeof((*ctx)->entry_name), NULL, 0)) -#endif - for (index = 0; index < len_0; index++) - (*ctx)->entry_name[index] = (char)wdir[index]; - } else - strncpy((*ctx)->entry_name, (const char *)(*ctx)->ctx.cFileName, - sizeof((*ctx)->entry_name) - 1); - - (*ctx)->entry_name[sizeof((*ctx)->entry_name) - 1] = '\0'; - - return (*ctx)->entry_name; -} - -int LP_find_file_end(LP_DIR_CTX **ctx) -{ - if (ctx != NULL && *ctx != NULL) { - FindClose((*ctx)->handle); - free(*ctx); - *ctx = NULL; - return 1; - } - errno = EINVAL; - return 0; -} diff --git a/openssl/src/crypto/LPdir_win32.c b/openssl/src/crypto/LPdir_win32.c deleted file mode 100644 index b29e096ff..000000000 --- a/openssl/src/crypto/LPdir_win32.c +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2004-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is dual-licensed and is also available under the following - * terms: - * - * Copyright (c) 2004, Richard Levitte - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#define LP_SYS_WIN32 -#define LP_MULTIBYTE_AVAILABLE -#include "LPdir_win.c" diff --git a/openssl/src/crypto/LPdir_wince.c b/openssl/src/crypto/LPdir_wince.c deleted file mode 100644 index ebf896282..000000000 --- a/openssl/src/crypto/LPdir_wince.c +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2004-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This file is dual-licensed and is also available under the following - * terms: - * - * Copyright (c) 2004, Richard Levitte - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#define LP_SYS_WINCE -/* - * We might want to define LP_MULTIBYTE_AVAILABLE here. It's currently under - * investigation what the exact conditions would be - */ -#include "LPdir_win.c" diff --git a/openssl/src/crypto/aes/aes_x86core.c b/openssl/src/crypto/aes/aes_x86core.c index 3e5b24ed1..da525b656 100644 --- a/openssl/src/crypto/aes/aes_x86core.c +++ b/openssl/src/crypto/aes/aes_x86core.c @@ -63,13 +63,12 @@ #if 1 static void prefetch256(const void *table) { - volatile unsigned long *t = (void *)table, ret; + volatile unsigned long *t=(void *)table,ret; unsigned long sum; int i; /* 32 is common least cache-line size */ - for (sum = 0, i = 0; i < 256/sizeof(t[0]); i += 32/sizeof(t[0])) - sum ^= t[i]; + for (sum=0,i=0;i<256/sizeof(t[0]);i+=32/sizeof(t[0])) sum ^= t[i]; ret = sum; } @@ -81,10 +80,13 @@ static void prefetch256(const void *table) #define GETU32(p) (*((u32*)(p))) #if (defined(_WIN32) || defined(_WIN64)) && !defined(__MINGW32__) +typedef unsigned __int64 u64; #define U64(C) C##UI64 #elif defined(__arch64__) +typedef unsigned long u64; #define U64(C) C##UL #else +typedef unsigned long long u64; #define U64(C) C##ULL #endif diff --git a/openssl/src/crypto/aes/gen/darwin_arm64/aesv8-armx.S b/openssl/src/crypto/aes/gen/darwin_arm64/aesv8-armx.S index 0807cc62f..ff4e581fb 100644 --- a/openssl/src/crypto/aes/gen/darwin_arm64/aesv8-armx.S +++ b/openssl/src/crypto/aes/gen/darwin_arm64/aesv8-armx.S @@ -14,8 +14,6 @@ Lrcon: .align 5 _aes_v8_set_encrypt_key: Lenc_key: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 mov x3,#-1 @@ -105,7 +103,7 @@ L192: Loop192: tbl v6.16b,{v4.16b},v2.16b ext v5.16b,v0.16b,v3.16b,#12 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ st1 {v4.4s},[x2],#16 sub x2,x2,#8 #else @@ -187,7 +185,7 @@ Lenc_key_abort: .align 5 _aes_v8_set_decrypt_key: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 bl Lenc_key @@ -221,14 +219,13 @@ Loop_imc: eor x0,x0,x0 // return value Ldec_key_abort: ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .globl _aes_v8_encrypt .align 5 _aes_v8_encrypt: - AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 ld1 {v2.16b},[x0] @@ -258,7 +255,6 @@ Loop_enc: .align 5 _aes_v8_decrypt: - AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 ld1 {v2.16b},[x0] @@ -288,7 +284,6 @@ Loop_dec: .align 5 _aes_v8_ecb_encrypt: - AARCH64_VALID_CALL_TARGET subs x2,x2,#16 // Original input data size bigger than 16, jump to big size processing. b.ne Lecb_big_size @@ -1035,8 +1030,6 @@ Lecb_Final_abort: .align 5 _aes_v8_cbc_encrypt: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 subs x2,x2,#16 @@ -1508,741 +1501,16 @@ Lcbc_abort: ldr x29,[sp],#16 ret -.globl _aes_v8_ctr32_encrypt_blocks_unroll12_eor3 - -.align 5 -_aes_v8_ctr32_encrypt_blocks_unroll12_eor3: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. - stp x29,x30,[sp,#-80]! - stp d8,d9,[sp, #16] - stp d10,d11,[sp, #32] - stp d12,d13,[sp, #48] - stp d14,d15,[sp, #64] - add x29,sp,#0 - - ldr w5,[x3,#240] - - ldr w8, [x4, #12] -#ifdef __AARCH64EB__ - ld1 {v24.16b},[x4] -#else - ld1 {v24.4s},[x4] -#endif - ld1 {v2.4s,v3.4s},[x3] // load key schedule... - sub w5,w5,#4 - cmp x2,#2 - add x7,x3,x5,lsl#4 // pointer to last round key - sub w5,w5,#2 - add x7, x7, #64 - ld1 {v1.4s},[x7] - add x7,x3,#32 - mov w6,w5 -#ifndef __AARCH64EB__ - rev w8, w8 -#endif - - orr v25.16b,v24.16b,v24.16b - add w10, w8, #1 - orr v26.16b,v24.16b,v24.16b - add w8, w8, #2 - orr v0.16b,v24.16b,v24.16b - rev w10, w10 - mov v25.s[3],w10 - b.ls Lctr32_tail_unroll - cmp x2,#6 - rev w12, w8 - sub x2,x2,#3 // bias - mov v26.s[3],w12 - b.lo Loop3x_ctr32_unroll - cmp x2,#9 - orr v27.16b,v24.16b,v24.16b - add w11, w8, #1 - orr v28.16b,v24.16b,v24.16b - add w13, w8, #2 - rev w11, w11 - orr v29.16b,v24.16b,v24.16b - add w8, w8, #3 - rev w13, w13 - mov v27.s[3],w11 - rev w14, w8 - mov v28.s[3],w13 - mov v29.s[3],w14 - sub x2,x2,#3 - b.lo Loop6x_ctr32_unroll - - // push regs to stack when 12 data chunks are interleaved - stp x19,x20,[sp,#-16]! - stp x21,x22,[sp,#-16]! - stp x23,x24,[sp,#-16]! - stp d8,d9,[sp,#-32]! - stp d10,d11,[sp,#-32]! - - add w15,w8,#1 - add w19,w8,#2 - add w20,w8,#3 - add w21,w8,#4 - add w22,w8,#5 - add w8,w8,#6 - orr v30.16b,v24.16b,v24.16b - rev w15,w15 - orr v31.16b,v24.16b,v24.16b - rev w19,w19 - orr v8.16b,v24.16b,v24.16b - rev w20,w20 - orr v9.16b,v24.16b,v24.16b - rev w21,w21 - orr v10.16b,v24.16b,v24.16b - rev w22,w22 - orr v11.16b,v24.16b,v24.16b - rev w23,w8 - - sub x2,x2,#6 // bias - mov v30.s[3],w15 - mov v31.s[3],w19 - mov v8.s[3],w20 - mov v9.s[3],w21 - mov v10.s[3],w22 - mov v11.s[3],w23 - b Loop12x_ctr32_unroll - -.align 4 -Loop12x_ctr32_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - aese v30.16b,v2.16b - aesmc v30.16b,v30.16b - aese v31.16b,v2.16b - aesmc v31.16b,v31.16b - aese v8.16b,v2.16b - aesmc v8.16b,v8.16b - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - aese v30.16b,v3.16b - aesmc v30.16b,v30.16b - aese v31.16b,v3.16b - aesmc v31.16b,v31.16b - aese v8.16b,v3.16b - aesmc v8.16b,v8.16b - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - aese v11.16b,v3.16b - aesmc v11.16b,v11.16b - ld1 {v3.4s},[x7],#16 - b.gt Loop12x_ctr32_unroll - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - aese v30.16b,v2.16b - aesmc v30.16b,v30.16b - aese v31.16b,v2.16b - aesmc v31.16b,v31.16b - aese v8.16b,v2.16b - aesmc v8.16b,v8.16b - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 - - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - aese v30.16b,v3.16b - aesmc v30.16b,v30.16b - aese v31.16b,v3.16b - aesmc v31.16b,v31.16b - aese v8.16b,v3.16b - aesmc v8.16b,v8.16b - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - aese v11.16b,v3.16b - aesmc v11.16b,v11.16b - ld1 {v3.4s},[x7],#16 - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - add w9,w8,#1 - add w10,w8,#2 - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - add w12,w8,#3 - add w11,w8,#4 - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - add w13,w8,#5 - add w14,w8,#6 - rev w9,w9 - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - add w15,w8,#7 - add w19,w8,#8 - rev w10,w10 - rev w12,w12 - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - add w20,w8,#9 - add w21,w8,#10 - rev w11,w11 - rev w13,w13 - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - add w22,w8,#11 - add w23,w8,#12 - rev w14,w14 - rev w15,w15 - aese v30.16b,v2.16b - aesmc v30.16b,v30.16b - rev w19,w19 - rev w20,w20 - aese v31.16b,v2.16b - aesmc v31.16b,v31.16b - rev w21,w21 - rev w22,w22 - aese v8.16b,v2.16b - aesmc v8.16b,v8.16b - rev w23,w23 - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 - - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64 - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - aese v30.16b,v3.16b - aesmc v30.16b,v30.16b - aese v31.16b,v3.16b - aesmc v31.16b,v31.16b - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x0],#64 - aese v8.16b,v3.16b - aesmc v8.16b,v8.16b - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - aese v11.16b,v3.16b - aesmc v11.16b,v11.16b - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x0],#64 - ld1 {v3.4s},[x7],#16 - - mov x7, x3 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - aese v30.16b,v2.16b - aesmc v30.16b,v30.16b - aese v31.16b,v2.16b - aesmc v31.16b,v31.16b - aese v8.16b,v2.16b - aesmc v8.16b,v8.16b - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 // re-pre-load rndkey[0] - - aese v24.16b,v3.16b -.long 0xce016084 //eor3 v4.16b,v4.16b,v1.16b,v24.16b - orr v24.16b,v0.16b,v0.16b - aese v25.16b,v3.16b -.long 0xce0164a5 //eor3 v5.16b,v5.16b,v1.16b,v25.16b - orr v25.16b,v0.16b,v0.16b - aese v26.16b,v3.16b -.long 0xce0168c6 //eor3 v6.16b,v6.16b,v1.16b,v26.16b - orr v26.16b,v0.16b,v0.16b - aese v27.16b,v3.16b -.long 0xce016ce7 //eor3 v7.16b,v7.16b,v1.16b,v27.16b - orr v27.16b,v0.16b,v0.16b - aese v28.16b,v3.16b -.long 0xce017210 //eor3 v16.16b,v16.16b,v1.16b,v28.16b - orr v28.16b,v0.16b,v0.16b - aese v29.16b,v3.16b -.long 0xce017631 //eor3 v17.16b,v17.16b,v1.16b,v29.16b - orr v29.16b,v0.16b,v0.16b - aese v30.16b,v3.16b -.long 0xce017a52 //eor3 v18.16b,v18.16b,v1.16b,v30.16b - orr v30.16b,v0.16b,v0.16b - aese v31.16b,v3.16b -.long 0xce017e73 //eor3 v19.16b,v19.16b,v1.16b,v31.16b - orr v31.16b,v0.16b,v0.16b - aese v8.16b,v3.16b -.long 0xce012294 //eor3 v20.16b,v20.16b,v1.16b,v8.16b - orr v8.16b,v0.16b,v0.16b - aese v9.16b,v3.16b -.long 0xce0126b5 //eor3 v21.16b,v21.16b,v1.16b,v9.16b - orr v9.16b,v0.16b,v0.16b - aese v10.16b,v3.16b -.long 0xce012ad6 //eor3 v22.16b,v22.16b,v1.16b,v10.16b - orr v10.16b,v0.16b,v0.16b - aese v11.16b,v3.16b -.long 0xce012ef7 //eor3 v23.16b,v23.16b,v1.16b,v11.16b - orr v11.16b,v0.16b,v0.16b - ld1 {v3.4s},[x7],#16 // re-pre-load rndkey[1] - - mov v24.s[3],w9 - mov v25.s[3],w10 - mov v26.s[3],w12 - mov v27.s[3],w11 - st1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 - mov v28.s[3],w13 - mov v29.s[3],w14 - mov v30.s[3],w15 - mov v31.s[3],w19 - st1 {v16.16b,v17.16b,v18.16b,v19.16b},[x1],#64 - mov v8.s[3],w20 - mov v9.s[3],w21 - mov v10.s[3],w22 - mov v11.s[3],w23 - st1 {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64 - - mov w6,w5 - - add w8,w8,#12 - subs x2,x2,#12 - b.hs Loop12x_ctr32_unroll - - // pop regs from stack when 12 data chunks are interleaved - ldp d10,d11,[sp],#32 - ldp d8,d9,[sp],#32 - ldp x23,x24,[sp],#16 - ldp x21,x22,[sp],#16 - ldp x19,x20,[sp],#16 - - add x2,x2,#12 - cbz x2,Lctr32_done_unroll - sub w8,w8,#12 - - cmp x2,#2 - b.ls Lctr32_tail_unroll - - cmp x2,#6 - sub x2,x2,#3 // bias - add w8,w8,#3 - b.lo Loop3x_ctr32_unroll - - sub x2,x2,#3 - add w8,w8,#3 - b.lo Loop6x_ctr32_unroll - -.align 4 -Loop6x_ctr32_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - ld1 {v3.4s},[x7],#16 - b.gt Loop6x_ctr32_unroll - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - ld1 {v2.4s},[x7],#16 - - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - ld1 {v3.4s},[x7],#16 - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - add w9,w8,#1 - add w10,w8,#2 - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - add w12,w8,#3 - add w11,w8,#4 - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - add w13,w8,#5 - add w14,w8,#6 - rev w9,w9 - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - rev w10,w10 - rev w12,w12 - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - rev w11,w11 - rev w13,w13 - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - rev w14,w14 - ld1 {v2.4s},[x7],#16 - - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64 - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - ld1 {v16.16b,v17.16b},[x0],#32 - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - ld1 {v3.4s},[x7],#16 - - mov x7, x3 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - ld1 {v2.4s},[x7],#16 // re-pre-load rndkey[0] - - aese v24.16b,v3.16b -.long 0xce016084 //eor3 v4.16b,v4.16b,v1.16b,v24.16b - aese v25.16b,v3.16b -.long 0xce0164a5 //eor3 v5.16b,v5.16b,v1.16b,v25.16b - aese v26.16b,v3.16b -.long 0xce0168c6 //eor3 v6.16b,v6.16b,v1.16b,v26.16b - aese v27.16b,v3.16b -.long 0xce016ce7 //eor3 v7.16b,v7.16b,v1.16b,v27.16b - aese v28.16b,v3.16b -.long 0xce017210 //eor3 v16.16b,v16.16b,v1.16b,v28.16b - aese v29.16b,v3.16b -.long 0xce017631 //eor3 v17.16b,v17.16b,v1.16b,v29.16b - ld1 {v3.4s},[x7],#16 // re-pre-load rndkey[1] - - orr v24.16b,v0.16b,v0.16b - orr v25.16b,v0.16b,v0.16b - orr v26.16b,v0.16b,v0.16b - orr v27.16b,v0.16b,v0.16b - orr v28.16b,v0.16b,v0.16b - orr v29.16b,v0.16b,v0.16b - - mov v24.s[3],w9 - mov v25.s[3],w10 - st1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 - mov v26.s[3],w12 - mov v27.s[3],w11 - st1 {v16.16b,v17.16b},[x1],#32 - mov v28.s[3],w13 - mov v29.s[3],w14 - - cbz x2,Lctr32_done_unroll - mov w6,w5 - - cmp x2,#2 - b.ls Lctr32_tail_unroll - - sub x2,x2,#3 // bias - add w8,w8,#3 - b Loop3x_ctr32_unroll - -.align 4 -Loop3x_ctr32_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - ld1 {v3.4s},[x7],#16 - b.gt Loop3x_ctr32_unroll - - aese v24.16b,v2.16b - aesmc v9.16b,v24.16b - aese v25.16b,v2.16b - aesmc v10.16b,v25.16b - ld1 {v4.16b,v5.16b,v6.16b},[x0],#48 - orr v24.16b,v0.16b,v0.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - ld1 {v2.4s},[x7],#16 - orr v25.16b,v0.16b,v0.16b - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - aese v26.16b,v3.16b - aesmc v11.16b,v26.16b - ld1 {v3.4s},[x7],#16 - orr v26.16b,v0.16b,v0.16b - add w9,w8,#1 - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - add w10,w8,#2 - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 - add w8,w8,#3 - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - - rev w9,w9 - aese v11.16b,v3.16b - aesmc v11.16b,v11.16b - ld1 {v3.4s},[x7],#16 - mov v24.s[3], w9 - mov x7,x3 - rev w10,w10 - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - mov v25.s[3], w10 - rev w12,w8 - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - mov v26.s[3], w12 - - aese v9.16b,v3.16b - aese v10.16b,v3.16b - aese v11.16b,v3.16b - -.long 0xce012484 //eor3 v4.16b,v4.16b,v1.16b,v9.16b - ld1 {v2.4s},[x7],#16 // re-pre-load rndkey[0] -.long 0xce0128a5 //eor3 v5.16b,v5.16b,v1.16b,v10.16b - mov w6,w5 -.long 0xce012cc6 //eor3 v6.16b,v6.16b,v1.16b,v11.16b - ld1 {v3.4s},[x7],#16 // re-pre-load rndkey[1] - st1 {v4.16b,v5.16b,v6.16b},[x1],#48 - - cbz x2,Lctr32_done_unroll - -Lctr32_tail_unroll: - cmp x2,#1 - b.eq Lctr32_tail_1_unroll - -Lctr32_tail_2_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - ld1 {v3.4s},[x7],#16 - b.gt Lctr32_tail_2_unroll - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - ld1 {v2.4s},[x7],#16 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - ld1 {v3.4s},[x7],#16 - ld1 {v4.16b,v5.16b},[x0],#32 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - ld1 {v2.4s},[x7],#16 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - ld1 {v3.4s},[x7],#16 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v24.16b,v3.16b - aese v25.16b,v3.16b - -.long 0xce016084 //eor3 v4.16b,v4.16b,v1.16b,v24.16b -.long 0xce0164a5 //eor3 v5.16b,v5.16b,v1.16b,v25.16b - st1 {v4.16b,v5.16b},[x1],#32 - b Lctr32_done_unroll - -Lctr32_tail_1_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - ld1 {v3.4s},[x7],#16 - b.gt Lctr32_tail_1_unroll - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - ld1 {v2.4s},[x7],#16 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - ld1 {v3.4s},[x7],#16 - ld1 {v4.16b},[x0] - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - ld1 {v2.4s},[x7],#16 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - ld1 {v3.4s},[x7],#16 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v24.16b,v3.16b - -.long 0xce016084 //eor3 v4.16b,v4.16b,v1.16b,v24.16b - st1 {v4.16b},[x1],#16 - -Lctr32_done_unroll: - ldp d8,d9,[sp, #16] - ldp d10,d11,[sp, #32] - ldp d12,d13,[sp, #48] - ldp d15,d16,[sp, #64] - ldr x29,[sp],#80 - ret - .globl _aes_v8_ctr32_encrypt_blocks .align 5 _aes_v8_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 ldr w5,[x3,#240] ldr w8, [x4, #12] -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ ld1 {v0.16b},[x4] #else ld1 {v0.4s},[x4] @@ -2259,7 +1527,7 @@ _aes_v8_ctr32_encrypt_blocks: add x7,x3,#32 mov w6,w5 csel x12,xzr,x12,lo -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev w8, w8 #endif orr v1.16b,v0.16b,v0.16b @@ -2586,16 +1854,15 @@ Lctr32_done: .align 5 _aes_v8_xts_encrypt: - AARCH64_VALID_CALL_TARGET cmp x2,#16 // Original input data size bigger than 16, jump to big size processing. b.ne Lxts_enc_big_size // Encrypt the iv with key2, as the first XEX iv. ldr w6,[x4,#240] - ld1 {v0.4s},[x4],#16 + ld1 {v0.16b},[x4],#16 ld1 {v6.16b},[x5] sub w6,w6,#2 - ld1 {v1.4s},[x4],#16 + ld1 {v1.16b},[x4],#16 Loop_enc_iv_enc: aese v6.16b,v0.16b @@ -3195,9 +2462,9 @@ Lxts_enc_load_done: // Encrypt the composite block to get the last second encrypted text block ldr w6,[x3,#240] // load key schedule... - ld1 {v0.4s},[x3],#16 + ld1 {v0.16b},[x3],#16 sub w6,w6,#2 - ld1 {v1.4s},[x3],#16 // load key schedule... + ld1 {v1.16b},[x3],#16 // load key schedule... Loop_final_enc: aese v26.16b,v0.16b aesmc v26.16b,v26.16b @@ -3228,16 +2495,15 @@ Lxts_enc_final_abort: .align 5 _aes_v8_xts_decrypt: - AARCH64_VALID_CALL_TARGET cmp x2,#16 // Original input data size bigger than 16, jump to big size processing. b.ne Lxts_dec_big_size // Encrypt the iv with key2, as the first XEX iv. ldr w6,[x4,#240] - ld1 {v0.4s},[x4],#16 + ld1 {v0.16b},[x4],#16 ld1 {v6.16b},[x5] sub w6,w6,#2 - ld1 {v1.4s},[x4],#16 + ld1 {v1.16b},[x4],#16 Loop_dec_small_iv_enc: aese v6.16b,v0.16b @@ -3315,10 +2581,10 @@ Lxts_dec_big_size: // Encrypt the iv with key2, as the first XEX iv ldr w6,[x4,#240] - ld1 {v0.4s},[x4],#16 + ld1 {v0.16b},[x4],#16 ld1 {v6.16b},[x5] sub w6,w6,#2 - ld1 {v1.4s},[x4],#16 + ld1 {v1.16b},[x4],#16 Loop_dec_iv_enc: aese v6.16b,v0.16b @@ -3658,7 +2924,7 @@ Lxts_dec_tail4x: st1 {v30.16b,v31.16b},[x1],#32 b.eq Lxts_dec_abort - ld1 {v0.16b},[x0],#16 + ld1 {v0.4s},[x0],#16 b Lxts_done .align 4 Lxts_outer_dec_tail: @@ -3836,9 +3102,9 @@ Lxts_done: // Processing the last two blocks with cipher stealing. mov x7,x3 cbnz x2,Lxts_dec_1st_done - ld1 {v0.16b},[x0],#16 + ld1 {v0.4s},[x0],#16 - // Decrypt the last second block to get the last plain text block + // Decrypt the last secod block to get the last plain text block Lxts_dec_1st_done: eor v26.16b,v0.16b,v8.16b ldr w6,[x3,#240] @@ -3881,9 +3147,9 @@ Lxts_dec_load_done: // Decrypt the composite block to get the last second plain text block ldr w6,[x7,#240] - ld1 {v0.4s},[x7],#16 + ld1 {v0.16b},[x7],#16 sub w6,w6,#2 - ld1 {v1.4s},[x7],#16 + ld1 {v1.16b},[x7],#16 Loop_final_dec: aesd v26.16b,v0.16b aesimc v26.16b,v26.16b diff --git a/openssl/src/crypto/aes/gen/darwin_arm64/bsaes-armv8.S b/openssl/src/crypto/aes/gen/darwin_arm64/bsaes-armv8.S deleted file mode 100644 index 6b972f5fc..000000000 --- a/openssl/src/crypto/aes/gen/darwin_arm64/bsaes-armv8.S +++ /dev/null @@ -1,2347 +0,0 @@ -// Copyright 2021-2024 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the OpenSSL license (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// ==================================================================== -// Written by Ben Avison for the OpenSSL -// project. Rights for redistribution and usage in source and binary -// forms are granted according to the OpenSSL license. -// ==================================================================== -// -// This implementation is a translation of bsaes-armv7 for AArch64. -// No attempt has been made to carry across the build switches for -// kernel targets, since the Linux kernel crypto support has moved on -// from when it was based on OpenSSL. - -// A lot of hand-scheduling has been performed. Consequently, this code -// doesn't factor out neatly into macros in the same way that the -// AArch32 version did, and there is little to be gained by wrapping it -// up in Perl, and it is presented as pure assembly. - - -#include "crypto/arm_arch.h" - -.text - - - - - - -.align 4 -// On entry: -// x9 -> key (previously expanded using _bsaes_key_convert) -// x10 = number of rounds -// v0-v7 input data -// On exit: -// x9-x11 corrupted -// other general-purpose registers preserved -// v0-v7 output data -// v11-v15 preserved -// other SIMD registers corrupted -_bsaes_decrypt8: - ldr q8, [x9], #16 - adr x11, LM0ISR - movi v9.16b, #0x55 - ldr q10, [x11], #16 - movi v16.16b, #0x33 - movi v17.16b, #0x0f - sub x10, x10, #1 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v8.16b - eor v2.16b, v2.16b, v8.16b - eor v4.16b, v4.16b, v8.16b - eor v3.16b, v3.16b, v8.16b - eor v5.16b, v5.16b, v8.16b - tbl v0.16b, {v0.16b}, v10.16b - tbl v1.16b, {v1.16b}, v10.16b - tbl v2.16b, {v2.16b}, v10.16b - tbl v4.16b, {v4.16b}, v10.16b - eor v6.16b, v6.16b, v8.16b - eor v7.16b, v7.16b, v8.16b - tbl v3.16b, {v3.16b}, v10.16b - tbl v5.16b, {v5.16b}, v10.16b - tbl v6.16b, {v6.16b}, v10.16b - ushr v8.2d, v0.2d, #1 - tbl v7.16b, {v7.16b}, v10.16b - ushr v10.2d, v4.2d, #1 - ushr v18.2d, v2.2d, #1 - eor v8.16b, v8.16b, v1.16b - ushr v19.2d, v6.2d, #1 - eor v10.16b, v10.16b, v5.16b - eor v18.16b, v18.16b, v3.16b - and v8.16b, v8.16b, v9.16b - eor v19.16b, v19.16b, v7.16b - and v10.16b, v10.16b, v9.16b - and v18.16b, v18.16b, v9.16b - eor v1.16b, v1.16b, v8.16b - shl v8.2d, v8.2d, #1 - and v9.16b, v19.16b, v9.16b - eor v5.16b, v5.16b, v10.16b - shl v10.2d, v10.2d, #1 - eor v3.16b, v3.16b, v18.16b - shl v18.2d, v18.2d, #1 - eor v0.16b, v0.16b, v8.16b - shl v8.2d, v9.2d, #1 - eor v7.16b, v7.16b, v9.16b - eor v4.16b, v4.16b, v10.16b - eor v2.16b, v2.16b, v18.16b - ushr v9.2d, v1.2d, #2 - eor v6.16b, v6.16b, v8.16b - ushr v8.2d, v0.2d, #2 - ushr v10.2d, v5.2d, #2 - ushr v18.2d, v4.2d, #2 - eor v9.16b, v9.16b, v3.16b - eor v8.16b, v8.16b, v2.16b - eor v10.16b, v10.16b, v7.16b - eor v18.16b, v18.16b, v6.16b - and v9.16b, v9.16b, v16.16b - and v8.16b, v8.16b, v16.16b - and v10.16b, v10.16b, v16.16b - and v16.16b, v18.16b, v16.16b - eor v3.16b, v3.16b, v9.16b - shl v9.2d, v9.2d, #2 - eor v2.16b, v2.16b, v8.16b - shl v8.2d, v8.2d, #2 - eor v7.16b, v7.16b, v10.16b - shl v10.2d, v10.2d, #2 - eor v6.16b, v6.16b, v16.16b - shl v16.2d, v16.2d, #2 - eor v1.16b, v1.16b, v9.16b - eor v0.16b, v0.16b, v8.16b - eor v5.16b, v5.16b, v10.16b - eor v4.16b, v4.16b, v16.16b - ushr v8.2d, v3.2d, #4 - ushr v9.2d, v2.2d, #4 - ushr v10.2d, v1.2d, #4 - ushr v16.2d, v0.2d, #4 - eor v8.16b, v8.16b, v7.16b - eor v9.16b, v9.16b, v6.16b - eor v10.16b, v10.16b, v5.16b - eor v16.16b, v16.16b, v4.16b - and v8.16b, v8.16b, v17.16b - and v9.16b, v9.16b, v17.16b - and v10.16b, v10.16b, v17.16b - and v16.16b, v16.16b, v17.16b - eor v7.16b, v7.16b, v8.16b - shl v8.2d, v8.2d, #4 - eor v6.16b, v6.16b, v9.16b - shl v9.2d, v9.2d, #4 - eor v5.16b, v5.16b, v10.16b - shl v10.2d, v10.2d, #4 - eor v4.16b, v4.16b, v16.16b - shl v16.2d, v16.2d, #4 - eor v3.16b, v3.16b, v8.16b - eor v2.16b, v2.16b, v9.16b - eor v1.16b, v1.16b, v10.16b - eor v0.16b, v0.16b, v16.16b - b Ldec_sbox -.align 4 -Ldec_loop: - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #64 - ldp q8, q9, [x9], #32 - eor v0.16b, v16.16b, v0.16b - ldr q10, [x9], #16 - eor v1.16b, v17.16b, v1.16b - ldr q16, [x9], #16 - eor v2.16b, v18.16b, v2.16b - eor v3.16b, v19.16b, v3.16b - eor v4.16b, v8.16b, v4.16b - eor v5.16b, v9.16b, v5.16b - eor v6.16b, v10.16b, v6.16b - eor v7.16b, v16.16b, v7.16b - tbl v0.16b, {v0.16b}, v28.16b - tbl v1.16b, {v1.16b}, v28.16b - tbl v2.16b, {v2.16b}, v28.16b - tbl v3.16b, {v3.16b}, v28.16b - tbl v4.16b, {v4.16b}, v28.16b - tbl v5.16b, {v5.16b}, v28.16b - tbl v6.16b, {v6.16b}, v28.16b - tbl v7.16b, {v7.16b}, v28.16b -Ldec_sbox: - eor v1.16b, v1.16b, v4.16b - eor v3.16b, v3.16b, v4.16b - subs x10, x10, #1 - eor v4.16b, v4.16b, v7.16b - eor v2.16b, v2.16b, v7.16b - eor v1.16b, v1.16b, v6.16b - eor v6.16b, v6.16b, v4.16b - eor v2.16b, v2.16b, v5.16b - eor v0.16b, v0.16b, v1.16b - eor v7.16b, v7.16b, v6.16b - eor v8.16b, v6.16b, v2.16b - and v9.16b, v4.16b, v6.16b - eor v10.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v16.16b, v7.16b, v4.16b - eor v17.16b, v4.16b, v0.16b - and v18.16b, v0.16b, v2.16b - eor v19.16b, v7.16b, v4.16b - eor v1.16b, v1.16b, v3.16b - eor v20.16b, v3.16b, v0.16b - eor v21.16b, v5.16b, v2.16b - eor v22.16b, v3.16b, v7.16b - and v8.16b, v17.16b, v8.16b - orr v17.16b, v3.16b, v5.16b - eor v23.16b, v1.16b, v6.16b - eor v24.16b, v20.16b, v16.16b - eor v25.16b, v1.16b, v5.16b - orr v26.16b, v20.16b, v21.16b - and v20.16b, v20.16b, v21.16b - and v27.16b, v7.16b, v1.16b - eor v21.16b, v21.16b, v23.16b - orr v28.16b, v16.16b, v23.16b - orr v29.16b, v22.16b, v25.16b - eor v26.16b, v26.16b, v8.16b - and v16.16b, v16.16b, v23.16b - and v22.16b, v22.16b, v25.16b - and v21.16b, v24.16b, v21.16b - eor v8.16b, v28.16b, v8.16b - eor v23.16b, v5.16b, v2.16b - eor v24.16b, v1.16b, v6.16b - eor v16.16b, v16.16b, v22.16b - eor v22.16b, v3.16b, v0.16b - eor v25.16b, v29.16b, v21.16b - eor v21.16b, v26.16b, v21.16b - eor v8.16b, v8.16b, v20.16b - eor v26.16b, v23.16b, v24.16b - eor v16.16b, v16.16b, v20.16b - eor v28.16b, v22.16b, v19.16b - eor v20.16b, v25.16b, v20.16b - eor v9.16b, v21.16b, v9.16b - eor v8.16b, v8.16b, v18.16b - eor v18.16b, v5.16b, v1.16b - eor v21.16b, v16.16b, v17.16b - eor v16.16b, v16.16b, v17.16b - eor v17.16b, v20.16b, v27.16b - eor v20.16b, v3.16b, v7.16b - eor v25.16b, v9.16b, v8.16b - eor v27.16b, v0.16b, v4.16b - and v29.16b, v9.16b, v17.16b - eor v30.16b, v8.16b, v29.16b - eor v31.16b, v21.16b, v29.16b - eor v29.16b, v21.16b, v29.16b - bsl v30.16b, v17.16b, v21.16b - bsl v31.16b, v9.16b, v8.16b - bsl v16.16b, v30.16b, v29.16b - bsl v21.16b, v29.16b, v30.16b - eor v8.16b, v31.16b, v30.16b - and v1.16b, v1.16b, v31.16b - and v9.16b, v16.16b, v31.16b - and v6.16b, v6.16b, v30.16b - eor v16.16b, v17.16b, v21.16b - and v4.16b, v4.16b, v30.16b - eor v17.16b, v8.16b, v30.16b - and v21.16b, v24.16b, v8.16b - eor v9.16b, v9.16b, v25.16b - and v19.16b, v19.16b, v8.16b - eor v24.16b, v30.16b, v16.16b - eor v25.16b, v30.16b, v16.16b - and v7.16b, v7.16b, v17.16b - and v10.16b, v10.16b, v16.16b - eor v29.16b, v9.16b, v16.16b - eor v30.16b, v31.16b, v9.16b - and v0.16b, v24.16b, v0.16b - and v9.16b, v18.16b, v9.16b - and v2.16b, v25.16b, v2.16b - eor v10.16b, v10.16b, v6.16b - eor v18.16b, v29.16b, v16.16b - and v5.16b, v30.16b, v5.16b - eor v24.16b, v8.16b, v29.16b - and v25.16b, v26.16b, v29.16b - and v26.16b, v28.16b, v29.16b - eor v8.16b, v8.16b, v29.16b - eor v17.16b, v17.16b, v18.16b - eor v5.16b, v1.16b, v5.16b - and v23.16b, v24.16b, v23.16b - eor v21.16b, v21.16b, v25.16b - eor v19.16b, v19.16b, v26.16b - eor v0.16b, v4.16b, v0.16b - and v3.16b, v17.16b, v3.16b - eor v1.16b, v9.16b, v1.16b - eor v9.16b, v25.16b, v23.16b - eor v5.16b, v5.16b, v21.16b - eor v2.16b, v6.16b, v2.16b - and v6.16b, v8.16b, v22.16b - eor v3.16b, v7.16b, v3.16b - and v8.16b, v20.16b, v18.16b - eor v10.16b, v10.16b, v9.16b - eor v0.16b, v0.16b, v19.16b - eor v9.16b, v1.16b, v9.16b - eor v1.16b, v2.16b, v21.16b - eor v3.16b, v3.16b, v19.16b - and v16.16b, v27.16b, v16.16b - eor v17.16b, v26.16b, v6.16b - eor v6.16b, v8.16b, v7.16b - eor v7.16b, v1.16b, v9.16b - eor v1.16b, v5.16b, v3.16b - eor v2.16b, v10.16b, v3.16b - eor v4.16b, v16.16b, v4.16b - eor v8.16b, v6.16b, v17.16b - eor v5.16b, v9.16b, v3.16b - eor v9.16b, v0.16b, v1.16b - eor v6.16b, v7.16b, v1.16b - eor v0.16b, v4.16b, v17.16b - eor v4.16b, v8.16b, v7.16b - eor v7.16b, v9.16b, v2.16b - eor v8.16b, v3.16b, v0.16b - eor v7.16b, v7.16b, v5.16b - eor v3.16b, v4.16b, v7.16b - eor v4.16b, v7.16b, v0.16b - eor v7.16b, v8.16b, v3.16b - bcc Ldec_done - ext v8.16b, v0.16b, v0.16b, #8 - ext v9.16b, v1.16b, v1.16b, #8 - ldr q28, [x11] // load from LISR in common case (x10 > 0) - ext v10.16b, v6.16b, v6.16b, #8 - ext v16.16b, v3.16b, v3.16b, #8 - ext v17.16b, v5.16b, v5.16b, #8 - ext v18.16b, v4.16b, v4.16b, #8 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v6.16b - eor v16.16b, v16.16b, v3.16b - eor v17.16b, v17.16b, v5.16b - ext v19.16b, v2.16b, v2.16b, #8 - ext v20.16b, v7.16b, v7.16b, #8 - eor v18.16b, v18.16b, v4.16b - eor v6.16b, v6.16b, v8.16b - eor v8.16b, v2.16b, v10.16b - eor v4.16b, v4.16b, v9.16b - eor v2.16b, v19.16b, v2.16b - eor v9.16b, v20.16b, v7.16b - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v16.16b - eor v6.16b, v6.16b, v17.16b - eor v8.16b, v8.16b, v16.16b - eor v7.16b, v7.16b, v18.16b - eor v4.16b, v4.16b, v16.16b - eor v2.16b, v3.16b, v2.16b - eor v1.16b, v1.16b, v17.16b - eor v3.16b, v5.16b, v9.16b - eor v5.16b, v8.16b, v17.16b - eor v7.16b, v7.16b, v17.16b - ext v8.16b, v0.16b, v0.16b, #12 - ext v9.16b, v6.16b, v6.16b, #12 - ext v10.16b, v4.16b, v4.16b, #12 - ext v16.16b, v1.16b, v1.16b, #12 - ext v17.16b, v5.16b, v5.16b, #12 - ext v18.16b, v7.16b, v7.16b, #12 - eor v0.16b, v0.16b, v8.16b - eor v6.16b, v6.16b, v9.16b - eor v4.16b, v4.16b, v10.16b - ext v19.16b, v2.16b, v2.16b, #12 - ext v20.16b, v3.16b, v3.16b, #12 - eor v1.16b, v1.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v7.16b, v7.16b, v18.16b - eor v2.16b, v2.16b, v19.16b - eor v16.16b, v16.16b, v0.16b - eor v3.16b, v3.16b, v20.16b - eor v17.16b, v17.16b, v4.16b - eor v10.16b, v10.16b, v6.16b - ext v0.16b, v0.16b, v0.16b, #8 - eor v9.16b, v9.16b, v1.16b - ext v1.16b, v1.16b, v1.16b, #8 - eor v8.16b, v8.16b, v3.16b - eor v16.16b, v16.16b, v3.16b - eor v18.16b, v18.16b, v5.16b - eor v19.16b, v19.16b, v7.16b - ext v21.16b, v5.16b, v5.16b, #8 - ext v5.16b, v7.16b, v7.16b, #8 - eor v7.16b, v20.16b, v2.16b - ext v4.16b, v4.16b, v4.16b, #8 - ext v20.16b, v3.16b, v3.16b, #8 - eor v17.16b, v17.16b, v3.16b - ext v2.16b, v2.16b, v2.16b, #8 - eor v3.16b, v10.16b, v3.16b - ext v10.16b, v6.16b, v6.16b, #8 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v16.16b - eor v5.16b, v5.16b, v18.16b - eor v3.16b, v3.16b, v4.16b - eor v7.16b, v20.16b, v7.16b - eor v6.16b, v2.16b, v19.16b - eor v4.16b, v21.16b, v17.16b - eor v2.16b, v10.16b, v9.16b - bne Ldec_loop - ldr q28, [x11, #16]! // load from LISRM0 on last round (x10 == 0) - b Ldec_loop -.align 4 -Ldec_done: - ushr v8.2d, v0.2d, #1 - movi v9.16b, #0x55 - ldr q10, [x9] - ushr v16.2d, v2.2d, #1 - movi v17.16b, #0x33 - ushr v18.2d, v6.2d, #1 - movi v19.16b, #0x0f - eor v8.16b, v8.16b, v1.16b - ushr v20.2d, v3.2d, #1 - eor v16.16b, v16.16b, v7.16b - eor v18.16b, v18.16b, v4.16b - and v8.16b, v8.16b, v9.16b - eor v20.16b, v20.16b, v5.16b - and v16.16b, v16.16b, v9.16b - and v18.16b, v18.16b, v9.16b - shl v21.2d, v8.2d, #1 - eor v1.16b, v1.16b, v8.16b - and v8.16b, v20.16b, v9.16b - eor v7.16b, v7.16b, v16.16b - shl v9.2d, v16.2d, #1 - eor v4.16b, v4.16b, v18.16b - shl v16.2d, v18.2d, #1 - eor v0.16b, v0.16b, v21.16b - shl v18.2d, v8.2d, #1 - eor v5.16b, v5.16b, v8.16b - eor v2.16b, v2.16b, v9.16b - eor v6.16b, v6.16b, v16.16b - ushr v8.2d, v1.2d, #2 - eor v3.16b, v3.16b, v18.16b - ushr v9.2d, v0.2d, #2 - ushr v16.2d, v7.2d, #2 - ushr v18.2d, v2.2d, #2 - eor v8.16b, v8.16b, v4.16b - eor v9.16b, v9.16b, v6.16b - eor v16.16b, v16.16b, v5.16b - eor v18.16b, v18.16b, v3.16b - and v8.16b, v8.16b, v17.16b - and v9.16b, v9.16b, v17.16b - and v16.16b, v16.16b, v17.16b - and v17.16b, v18.16b, v17.16b - eor v4.16b, v4.16b, v8.16b - shl v8.2d, v8.2d, #2 - eor v6.16b, v6.16b, v9.16b - shl v9.2d, v9.2d, #2 - eor v5.16b, v5.16b, v16.16b - shl v16.2d, v16.2d, #2 - eor v3.16b, v3.16b, v17.16b - shl v17.2d, v17.2d, #2 - eor v1.16b, v1.16b, v8.16b - eor v0.16b, v0.16b, v9.16b - eor v7.16b, v7.16b, v16.16b - eor v2.16b, v2.16b, v17.16b - ushr v8.2d, v4.2d, #4 - ushr v9.2d, v6.2d, #4 - ushr v16.2d, v1.2d, #4 - ushr v17.2d, v0.2d, #4 - eor v8.16b, v8.16b, v5.16b - eor v9.16b, v9.16b, v3.16b - eor v16.16b, v16.16b, v7.16b - eor v17.16b, v17.16b, v2.16b - and v8.16b, v8.16b, v19.16b - and v9.16b, v9.16b, v19.16b - and v16.16b, v16.16b, v19.16b - and v17.16b, v17.16b, v19.16b - eor v5.16b, v5.16b, v8.16b - shl v8.2d, v8.2d, #4 - eor v3.16b, v3.16b, v9.16b - shl v9.2d, v9.2d, #4 - eor v7.16b, v7.16b, v16.16b - shl v16.2d, v16.2d, #4 - eor v2.16b, v2.16b, v17.16b - shl v17.2d, v17.2d, #4 - eor v4.16b, v4.16b, v8.16b - eor v6.16b, v6.16b, v9.16b - eor v7.16b, v7.16b, v10.16b - eor v1.16b, v1.16b, v16.16b - eor v2.16b, v2.16b, v10.16b - eor v0.16b, v0.16b, v17.16b - eor v4.16b, v4.16b, v10.16b - eor v6.16b, v6.16b, v10.16b - eor v3.16b, v3.16b, v10.16b - eor v5.16b, v5.16b, v10.16b - eor v1.16b, v1.16b, v10.16b - eor v0.16b, v0.16b, v10.16b - ret - - - -.align 6 -_bsaes_const: -// InvShiftRows constants -// Used in _bsaes_decrypt8, which assumes contiguity -// .LM0ISR used with round 0 key -// .LISR used with middle round keys -// .LISRM0 used with final round key -LM0ISR: -.quad 0x0a0e0206070b0f03, 0x0004080c0d010509 -LISR: -.quad 0x0504070602010003, 0x0f0e0d0c080b0a09 -LISRM0: -.quad 0x01040b0e0205080f, 0x0306090c00070a0d - -// ShiftRows constants -// Used in _bsaes_encrypt8, which assumes contiguity -// .LM0SR used with round 0 key -// .LSR used with middle round keys -// .LSRM0 used with final round key -LM0SR: -.quad 0x0a0e02060f03070b, 0x0004080c05090d01 -LSR: -.quad 0x0504070600030201, 0x0f0e0d0c0a09080b -LSRM0: -.quad 0x0304090e00050a0f, 0x01060b0c0207080d - -LM0_bigendian: -.quad 0x02060a0e03070b0f, 0x0004080c0105090d -LM0_littleendian: -.quad 0x0105090d0004080c, 0x03070b0f02060a0e - -// Used in ossl_bsaes_ctr32_encrypt_blocks, prior to dropping into -// _bsaes_encrypt8_alt, for round 0 key in place of .LM0SR -LREVM0SR: -.quad 0x090d01050c000408, 0x03070b0f060a0e02 - -.align 6 - - - -.align 4 -// On entry: -// x9 -> key (previously expanded using _bsaes_key_convert) -// x10 = number of rounds -// v0-v7 input data -// On exit: -// x9-x11 corrupted -// other general-purpose registers preserved -// v0-v7 output data -// v11-v15 preserved -// other SIMD registers corrupted -_bsaes_encrypt8: - ldr q8, [x9], #16 - adr x11, LM0SR - ldr q9, [x11], #16 -_bsaes_encrypt8_alt: - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v8.16b - sub x10, x10, #1 - eor v2.16b, v2.16b, v8.16b - eor v4.16b, v4.16b, v8.16b - eor v3.16b, v3.16b, v8.16b - eor v5.16b, v5.16b, v8.16b - tbl v0.16b, {v0.16b}, v9.16b - tbl v1.16b, {v1.16b}, v9.16b - tbl v2.16b, {v2.16b}, v9.16b - tbl v4.16b, {v4.16b}, v9.16b - eor v6.16b, v6.16b, v8.16b - eor v7.16b, v7.16b, v8.16b - tbl v3.16b, {v3.16b}, v9.16b - tbl v5.16b, {v5.16b}, v9.16b - tbl v6.16b, {v6.16b}, v9.16b - ushr v8.2d, v0.2d, #1 - movi v10.16b, #0x55 - tbl v7.16b, {v7.16b}, v9.16b - ushr v9.2d, v4.2d, #1 - movi v16.16b, #0x33 - ushr v17.2d, v2.2d, #1 - eor v8.16b, v8.16b, v1.16b - movi v18.16b, #0x0f - ushr v19.2d, v6.2d, #1 - eor v9.16b, v9.16b, v5.16b - eor v17.16b, v17.16b, v3.16b - and v8.16b, v8.16b, v10.16b - eor v19.16b, v19.16b, v7.16b - and v9.16b, v9.16b, v10.16b - and v17.16b, v17.16b, v10.16b - eor v1.16b, v1.16b, v8.16b - shl v8.2d, v8.2d, #1 - and v10.16b, v19.16b, v10.16b - eor v5.16b, v5.16b, v9.16b - shl v9.2d, v9.2d, #1 - eor v3.16b, v3.16b, v17.16b - shl v17.2d, v17.2d, #1 - eor v0.16b, v0.16b, v8.16b - shl v8.2d, v10.2d, #1 - eor v7.16b, v7.16b, v10.16b - eor v4.16b, v4.16b, v9.16b - eor v2.16b, v2.16b, v17.16b - ushr v9.2d, v1.2d, #2 - eor v6.16b, v6.16b, v8.16b - ushr v8.2d, v0.2d, #2 - ushr v10.2d, v5.2d, #2 - ushr v17.2d, v4.2d, #2 - eor v9.16b, v9.16b, v3.16b - eor v8.16b, v8.16b, v2.16b - eor v10.16b, v10.16b, v7.16b - eor v17.16b, v17.16b, v6.16b - and v9.16b, v9.16b, v16.16b - and v8.16b, v8.16b, v16.16b - and v10.16b, v10.16b, v16.16b - and v16.16b, v17.16b, v16.16b - eor v3.16b, v3.16b, v9.16b - shl v9.2d, v9.2d, #2 - eor v2.16b, v2.16b, v8.16b - shl v8.2d, v8.2d, #2 - eor v7.16b, v7.16b, v10.16b - shl v10.2d, v10.2d, #2 - eor v6.16b, v6.16b, v16.16b - shl v16.2d, v16.2d, #2 - eor v1.16b, v1.16b, v9.16b - eor v0.16b, v0.16b, v8.16b - eor v5.16b, v5.16b, v10.16b - eor v4.16b, v4.16b, v16.16b - ushr v8.2d, v3.2d, #4 - ushr v9.2d, v2.2d, #4 - ushr v10.2d, v1.2d, #4 - ushr v16.2d, v0.2d, #4 - eor v8.16b, v8.16b, v7.16b - eor v9.16b, v9.16b, v6.16b - eor v10.16b, v10.16b, v5.16b - eor v16.16b, v16.16b, v4.16b - and v8.16b, v8.16b, v18.16b - and v9.16b, v9.16b, v18.16b - and v10.16b, v10.16b, v18.16b - and v16.16b, v16.16b, v18.16b - eor v7.16b, v7.16b, v8.16b - shl v8.2d, v8.2d, #4 - eor v6.16b, v6.16b, v9.16b - shl v9.2d, v9.2d, #4 - eor v5.16b, v5.16b, v10.16b - shl v10.2d, v10.2d, #4 - eor v4.16b, v4.16b, v16.16b - shl v16.2d, v16.2d, #4 - eor v3.16b, v3.16b, v8.16b - eor v2.16b, v2.16b, v9.16b - eor v1.16b, v1.16b, v10.16b - eor v0.16b, v0.16b, v16.16b - b Lenc_sbox -.align 4 -Lenc_loop: - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #64 - ldp q8, q9, [x9], #32 - eor v0.16b, v16.16b, v0.16b - ldr q10, [x9], #16 - eor v1.16b, v17.16b, v1.16b - ldr q16, [x9], #16 - eor v2.16b, v18.16b, v2.16b - eor v3.16b, v19.16b, v3.16b - eor v4.16b, v8.16b, v4.16b - eor v5.16b, v9.16b, v5.16b - eor v6.16b, v10.16b, v6.16b - eor v7.16b, v16.16b, v7.16b - tbl v0.16b, {v0.16b}, v28.16b - tbl v1.16b, {v1.16b}, v28.16b - tbl v2.16b, {v2.16b}, v28.16b - tbl v3.16b, {v3.16b}, v28.16b - tbl v4.16b, {v4.16b}, v28.16b - tbl v5.16b, {v5.16b}, v28.16b - tbl v6.16b, {v6.16b}, v28.16b - tbl v7.16b, {v7.16b}, v28.16b -Lenc_sbox: - eor v5.16b, v5.16b, v6.16b - eor v3.16b, v3.16b, v0.16b - subs x10, x10, #1 - eor v2.16b, v2.16b, v1.16b - eor v5.16b, v5.16b, v0.16b - eor v8.16b, v3.16b, v7.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v5.16b - eor v8.16b, v8.16b, v4.16b - eor v3.16b, v6.16b, v3.16b - eor v4.16b, v4.16b, v5.16b - eor v6.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v7.16b - eor v1.16b, v8.16b, v1.16b - eor v8.16b, v7.16b, v4.16b - eor v9.16b, v3.16b, v0.16b - eor v10.16b, v7.16b, v6.16b - eor v16.16b, v5.16b, v3.16b - eor v17.16b, v6.16b, v2.16b - eor v18.16b, v5.16b, v1.16b - eor v19.16b, v2.16b, v4.16b - eor v20.16b, v1.16b, v0.16b - orr v21.16b, v8.16b, v9.16b - orr v22.16b, v10.16b, v16.16b - eor v23.16b, v8.16b, v17.16b - eor v24.16b, v9.16b, v18.16b - and v19.16b, v19.16b, v20.16b - orr v20.16b, v17.16b, v18.16b - and v8.16b, v8.16b, v9.16b - and v9.16b, v17.16b, v18.16b - and v17.16b, v23.16b, v24.16b - and v10.16b, v10.16b, v16.16b - eor v16.16b, v21.16b, v19.16b - eor v18.16b, v20.16b, v19.16b - and v19.16b, v2.16b, v1.16b - and v20.16b, v6.16b, v5.16b - eor v21.16b, v22.16b, v17.16b - eor v9.16b, v9.16b, v10.16b - eor v10.16b, v16.16b, v17.16b - eor v16.16b, v18.16b, v8.16b - and v17.16b, v4.16b, v0.16b - orr v18.16b, v7.16b, v3.16b - eor v21.16b, v21.16b, v8.16b - eor v8.16b, v9.16b, v8.16b - eor v9.16b, v10.16b, v19.16b - eor v10.16b, v3.16b, v0.16b - eor v16.16b, v16.16b, v17.16b - eor v17.16b, v5.16b, v1.16b - eor v19.16b, v21.16b, v20.16b - eor v20.16b, v8.16b, v18.16b - eor v8.16b, v8.16b, v18.16b - eor v18.16b, v7.16b, v4.16b - eor v21.16b, v9.16b, v16.16b - eor v22.16b, v6.16b, v2.16b - and v23.16b, v9.16b, v19.16b - eor v24.16b, v10.16b, v17.16b - eor v25.16b, v0.16b, v1.16b - eor v26.16b, v7.16b, v6.16b - eor v27.16b, v18.16b, v22.16b - eor v28.16b, v3.16b, v5.16b - eor v29.16b, v16.16b, v23.16b - eor v30.16b, v20.16b, v23.16b - eor v23.16b, v20.16b, v23.16b - eor v31.16b, v4.16b, v2.16b - bsl v29.16b, v19.16b, v20.16b - bsl v30.16b, v9.16b, v16.16b - bsl v8.16b, v29.16b, v23.16b - bsl v20.16b, v23.16b, v29.16b - eor v9.16b, v30.16b, v29.16b - and v5.16b, v5.16b, v30.16b - and v8.16b, v8.16b, v30.16b - and v1.16b, v1.16b, v29.16b - eor v16.16b, v19.16b, v20.16b - and v2.16b, v2.16b, v29.16b - eor v19.16b, v9.16b, v29.16b - and v17.16b, v17.16b, v9.16b - eor v8.16b, v8.16b, v21.16b - and v20.16b, v22.16b, v9.16b - eor v21.16b, v29.16b, v16.16b - eor v22.16b, v29.16b, v16.16b - and v23.16b, v25.16b, v16.16b - and v6.16b, v6.16b, v19.16b - eor v25.16b, v8.16b, v16.16b - eor v29.16b, v30.16b, v8.16b - and v4.16b, v21.16b, v4.16b - and v8.16b, v28.16b, v8.16b - and v0.16b, v22.16b, v0.16b - eor v21.16b, v23.16b, v1.16b - eor v22.16b, v9.16b, v25.16b - eor v9.16b, v9.16b, v25.16b - eor v23.16b, v25.16b, v16.16b - and v3.16b, v29.16b, v3.16b - and v24.16b, v24.16b, v25.16b - and v25.16b, v27.16b, v25.16b - and v10.16b, v22.16b, v10.16b - and v9.16b, v9.16b, v18.16b - eor v18.16b, v19.16b, v23.16b - and v19.16b, v26.16b, v23.16b - eor v3.16b, v5.16b, v3.16b - eor v17.16b, v17.16b, v24.16b - eor v10.16b, v24.16b, v10.16b - and v16.16b, v31.16b, v16.16b - eor v20.16b, v20.16b, v25.16b - eor v9.16b, v25.16b, v9.16b - eor v4.16b, v2.16b, v4.16b - and v7.16b, v18.16b, v7.16b - eor v18.16b, v19.16b, v6.16b - eor v5.16b, v8.16b, v5.16b - eor v0.16b, v1.16b, v0.16b - eor v1.16b, v21.16b, v10.16b - eor v8.16b, v3.16b, v17.16b - eor v2.16b, v16.16b, v2.16b - eor v3.16b, v6.16b, v7.16b - eor v6.16b, v18.16b, v9.16b - eor v4.16b, v4.16b, v20.16b - eor v10.16b, v5.16b, v10.16b - eor v0.16b, v0.16b, v17.16b - eor v9.16b, v2.16b, v9.16b - eor v3.16b, v3.16b, v20.16b - eor v7.16b, v6.16b, v1.16b - eor v5.16b, v8.16b, v4.16b - eor v6.16b, v10.16b, v1.16b - eor v2.16b, v4.16b, v0.16b - eor v4.16b, v3.16b, v10.16b - eor v9.16b, v9.16b, v7.16b - eor v3.16b, v0.16b, v5.16b - eor v0.16b, v1.16b, v4.16b - eor v1.16b, v4.16b, v8.16b - eor v4.16b, v9.16b, v5.16b - eor v6.16b, v6.16b, v3.16b - bcc Lenc_done - ext v8.16b, v0.16b, v0.16b, #12 - ext v9.16b, v4.16b, v4.16b, #12 - ldr q28, [x11] - ext v10.16b, v6.16b, v6.16b, #12 - ext v16.16b, v1.16b, v1.16b, #12 - ext v17.16b, v3.16b, v3.16b, #12 - ext v18.16b, v7.16b, v7.16b, #12 - eor v0.16b, v0.16b, v8.16b - eor v4.16b, v4.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - ext v19.16b, v2.16b, v2.16b, #12 - ext v20.16b, v5.16b, v5.16b, #12 - eor v1.16b, v1.16b, v16.16b - eor v3.16b, v3.16b, v17.16b - eor v7.16b, v7.16b, v18.16b - eor v2.16b, v2.16b, v19.16b - eor v16.16b, v16.16b, v0.16b - eor v5.16b, v5.16b, v20.16b - eor v17.16b, v17.16b, v6.16b - eor v10.16b, v10.16b, v4.16b - ext v0.16b, v0.16b, v0.16b, #8 - eor v9.16b, v9.16b, v1.16b - ext v1.16b, v1.16b, v1.16b, #8 - eor v8.16b, v8.16b, v5.16b - eor v16.16b, v16.16b, v5.16b - eor v18.16b, v18.16b, v3.16b - eor v19.16b, v19.16b, v7.16b - ext v3.16b, v3.16b, v3.16b, #8 - ext v7.16b, v7.16b, v7.16b, #8 - eor v20.16b, v20.16b, v2.16b - ext v6.16b, v6.16b, v6.16b, #8 - ext v21.16b, v5.16b, v5.16b, #8 - eor v17.16b, v17.16b, v5.16b - ext v2.16b, v2.16b, v2.16b, #8 - eor v10.16b, v10.16b, v5.16b - ext v22.16b, v4.16b, v4.16b, #8 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v16.16b - eor v5.16b, v7.16b, v18.16b - eor v4.16b, v3.16b, v17.16b - eor v3.16b, v6.16b, v10.16b - eor v7.16b, v21.16b, v20.16b - eor v6.16b, v2.16b, v19.16b - eor v2.16b, v22.16b, v9.16b - bne Lenc_loop - ldr q28, [x11, #16]! // load from LSRM0 on last round (x10 == 0) - b Lenc_loop -.align 4 -Lenc_done: - ushr v8.2d, v0.2d, #1 - movi v9.16b, #0x55 - ldr q10, [x9] - ushr v16.2d, v3.2d, #1 - movi v17.16b, #0x33 - ushr v18.2d, v4.2d, #1 - movi v19.16b, #0x0f - eor v8.16b, v8.16b, v1.16b - ushr v20.2d, v2.2d, #1 - eor v16.16b, v16.16b, v7.16b - eor v18.16b, v18.16b, v6.16b - and v8.16b, v8.16b, v9.16b - eor v20.16b, v20.16b, v5.16b - and v16.16b, v16.16b, v9.16b - and v18.16b, v18.16b, v9.16b - shl v21.2d, v8.2d, #1 - eor v1.16b, v1.16b, v8.16b - and v8.16b, v20.16b, v9.16b - eor v7.16b, v7.16b, v16.16b - shl v9.2d, v16.2d, #1 - eor v6.16b, v6.16b, v18.16b - shl v16.2d, v18.2d, #1 - eor v0.16b, v0.16b, v21.16b - shl v18.2d, v8.2d, #1 - eor v5.16b, v5.16b, v8.16b - eor v3.16b, v3.16b, v9.16b - eor v4.16b, v4.16b, v16.16b - ushr v8.2d, v1.2d, #2 - eor v2.16b, v2.16b, v18.16b - ushr v9.2d, v0.2d, #2 - ushr v16.2d, v7.2d, #2 - ushr v18.2d, v3.2d, #2 - eor v8.16b, v8.16b, v6.16b - eor v9.16b, v9.16b, v4.16b - eor v16.16b, v16.16b, v5.16b - eor v18.16b, v18.16b, v2.16b - and v8.16b, v8.16b, v17.16b - and v9.16b, v9.16b, v17.16b - and v16.16b, v16.16b, v17.16b - and v17.16b, v18.16b, v17.16b - eor v6.16b, v6.16b, v8.16b - shl v8.2d, v8.2d, #2 - eor v4.16b, v4.16b, v9.16b - shl v9.2d, v9.2d, #2 - eor v5.16b, v5.16b, v16.16b - shl v16.2d, v16.2d, #2 - eor v2.16b, v2.16b, v17.16b - shl v17.2d, v17.2d, #2 - eor v1.16b, v1.16b, v8.16b - eor v0.16b, v0.16b, v9.16b - eor v7.16b, v7.16b, v16.16b - eor v3.16b, v3.16b, v17.16b - ushr v8.2d, v6.2d, #4 - ushr v9.2d, v4.2d, #4 - ushr v16.2d, v1.2d, #4 - ushr v17.2d, v0.2d, #4 - eor v8.16b, v8.16b, v5.16b - eor v9.16b, v9.16b, v2.16b - eor v16.16b, v16.16b, v7.16b - eor v17.16b, v17.16b, v3.16b - and v8.16b, v8.16b, v19.16b - and v9.16b, v9.16b, v19.16b - and v16.16b, v16.16b, v19.16b - and v17.16b, v17.16b, v19.16b - eor v5.16b, v5.16b, v8.16b - shl v8.2d, v8.2d, #4 - eor v2.16b, v2.16b, v9.16b - shl v9.2d, v9.2d, #4 - eor v7.16b, v7.16b, v16.16b - shl v16.2d, v16.2d, #4 - eor v3.16b, v3.16b, v17.16b - shl v17.2d, v17.2d, #4 - eor v6.16b, v6.16b, v8.16b - eor v4.16b, v4.16b, v9.16b - eor v7.16b, v7.16b, v10.16b - eor v1.16b, v1.16b, v16.16b - eor v3.16b, v3.16b, v10.16b - eor v0.16b, v0.16b, v17.16b - eor v6.16b, v6.16b, v10.16b - eor v4.16b, v4.16b, v10.16b - eor v2.16b, v2.16b, v10.16b - eor v5.16b, v5.16b, v10.16b - eor v1.16b, v1.16b, v10.16b - eor v0.16b, v0.16b, v10.16b - ret - - - -.align 4 -// On entry: -// x9 -> input key (big-endian) -// x10 = number of rounds -// x17 -> output key (native endianness) -// On exit: -// x9, x10 corrupted -// x11 -> .LM0_bigendian -// x17 -> last quadword of output key -// other general-purpose registers preserved -// v2-v6 preserved -// v7.16b[] = 0x63 -// v8-v14 preserved -// v15 = last round key (converted to native endianness) -// other SIMD registers corrupted -_bsaes_key_convert: -#ifdef __AARCH64EL__ - adr x11, LM0_littleendian -#else - adr x11, LM0_bigendian -#endif - ldr q0, [x9], #16 // load round 0 key - ldr q1, [x11] // LM0 - ldr q15, [x9], #16 // load round 1 key - - movi v7.16b, #0x63 // compose L63 - movi v16.16b, #0x01 // bit masks - movi v17.16b, #0x02 - movi v18.16b, #0x04 - movi v19.16b, #0x08 - movi v20.16b, #0x10 - movi v21.16b, #0x20 - movi v22.16b, #0x40 - movi v23.16b, #0x80 - -#ifdef __AARCH64EL__ - rev32 v0.16b, v0.16b -#endif - sub x10, x10, #1 - str q0, [x17], #16 // save round 0 key - -.align 4 -Lkey_loop: - tbl v0.16b, {v15.16b}, v1.16b - ldr q15, [x9], #16 // load next round key - - eor v0.16b, v0.16b, v7.16b - cmtst v24.16b, v0.16b, v16.16b - cmtst v25.16b, v0.16b, v17.16b - cmtst v26.16b, v0.16b, v18.16b - cmtst v27.16b, v0.16b, v19.16b - cmtst v28.16b, v0.16b, v20.16b - cmtst v29.16b, v0.16b, v21.16b - cmtst v30.16b, v0.16b, v22.16b - cmtst v31.16b, v0.16b, v23.16b - sub x10, x10, #1 - st1 {v24.16b,v25.16b,v26.16b,v27.16b}, [x17], #64 // write bit-sliced round key - st1 {v28.16b,v29.16b,v30.16b,v31.16b}, [x17], #64 - cbnz x10, Lkey_loop - - // don't save last round key -#ifdef __AARCH64EL__ - rev32 v15.16b, v15.16b - adr x11, LM0_bigendian -#endif - ret - - -.globl _ossl_bsaes_cbc_encrypt - -.align 4 -// On entry: -// x0 -> input ciphertext -// x1 -> output plaintext -// x2 = size of ciphertext and plaintext in bytes (assumed a multiple of 16) -// x3 -> key -// x4 -> 128-bit initialisation vector (or preceding 128-bit block of ciphertext if continuing after an earlier call) -// w5 must be == 0 -// On exit: -// Output plaintext filled in -// Initialisation vector overwritten with last quadword of ciphertext -// No output registers, usual AAPCS64 register preservation -_ossl_bsaes_cbc_encrypt: - AARCH64_VALID_CALL_TARGET - cmp x2, #128 - bhs Lcbc_do_bsaes - b _AES_cbc_encrypt -Lcbc_do_bsaes: - - // it is up to the caller to make sure we are called with enc == 0 - - stp x29, x30, [sp, #-48]! - stp d8, d9, [sp, #16] - stp d10, d15, [sp, #32] - lsr x2, x2, #4 // len in 16 byte blocks - - ldr w15, [x3, #240] // get # of rounds - mov x14, sp - - // allocate the key schedule on the stack - add x17, sp, #96 - sub x17, x17, x15, lsl #7 // 128 bytes per inner round key, less 96 bytes - - // populate the key schedule - mov x9, x3 // pass key - mov x10, x15 // pass # of rounds - mov sp, x17 // sp is sp - bl _bsaes_key_convert - ldr q6, [sp] - str q15, [x17] // save last round key - eor v6.16b, v6.16b, v7.16b // fix up round 0 key (by XORing with 0x63) - str q6, [sp] - - ldr q15, [x4] // load IV - b Lcbc_dec_loop - -.align 4 -Lcbc_dec_loop: - subs x2, x2, #0x8 - bmi Lcbc_dec_loop_finish - - ldr q0, [x0], #16 // load input - mov x9, sp // pass the key - ldr q1, [x0], #16 - mov x10, x15 - ldr q2, [x0], #16 - ldr q3, [x0], #16 - ldr q4, [x0], #16 - ldr q5, [x0], #16 - ldr q6, [x0], #16 - ldr q7, [x0], #-7*16 - - bl _bsaes_decrypt8 - - ldr q16, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - eor v1.16b, v1.16b, v16.16b - str q0, [x1], #16 // write output - ldr q0, [x0], #16 - str q1, [x1], #16 - ldr q1, [x0], #16 - eor v1.16b, v4.16b, v1.16b - ldr q4, [x0], #16 - eor v2.16b, v2.16b, v4.16b - eor v0.16b, v6.16b, v0.16b - ldr q4, [x0], #16 - str q0, [x1], #16 - str q1, [x1], #16 - eor v0.16b, v7.16b, v4.16b - ldr q1, [x0], #16 - str q2, [x1], #16 - ldr q2, [x0], #16 - ldr q15, [x0], #16 - str q0, [x1], #16 - eor v0.16b, v5.16b, v2.16b - eor v1.16b, v3.16b, v1.16b - str q1, [x1], #16 - str q0, [x1], #16 - - b Lcbc_dec_loop - -Lcbc_dec_loop_finish: - adds x2, x2, #8 - beq Lcbc_dec_done - - ldr q0, [x0], #16 // load input - cmp x2, #2 - blo Lcbc_dec_one - ldr q1, [x0], #16 - mov x9, sp // pass the key - mov x10, x15 - beq Lcbc_dec_two - ldr q2, [x0], #16 - cmp x2, #4 - blo Lcbc_dec_three - ldr q3, [x0], #16 - beq Lcbc_dec_four - ldr q4, [x0], #16 - cmp x2, #6 - blo Lcbc_dec_five - ldr q5, [x0], #16 - beq Lcbc_dec_six - ldr q6, [x0], #-6*16 - - bl _bsaes_decrypt8 - - ldr q5, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q8, [x0], #16 - ldr q9, [x0], #16 - ldr q10, [x0], #16 - str q0, [x1], #16 // write output - ldr q0, [x0], #16 - eor v1.16b, v1.16b, v5.16b - ldr q5, [x0], #16 - eor v6.16b, v6.16b, v8.16b - ldr q15, [x0] - eor v4.16b, v4.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - str q1, [x1], #16 - eor v0.16b, v7.16b, v0.16b - str q6, [x1], #16 - eor v1.16b, v3.16b, v5.16b - str q4, [x1], #16 - str q2, [x1], #16 - str q0, [x1], #16 - str q1, [x1] - b Lcbc_dec_done -.align 4 -Lcbc_dec_six: - sub x0, x0, #0x60 - bl _bsaes_decrypt8 - ldr q3, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q5, [x0], #16 - ldr q8, [x0], #16 - ldr q9, [x0], #16 - str q0, [x1], #16 // write output - ldr q0, [x0], #16 - eor v1.16b, v1.16b, v3.16b - ldr q15, [x0] - eor v3.16b, v6.16b, v5.16b - eor v4.16b, v4.16b, v8.16b - eor v2.16b, v2.16b, v9.16b - str q1, [x1], #16 - eor v0.16b, v7.16b, v0.16b - str q3, [x1], #16 - str q4, [x1], #16 - str q2, [x1], #16 - str q0, [x1] - b Lcbc_dec_done -.align 4 -Lcbc_dec_five: - sub x0, x0, #0x50 - bl _bsaes_decrypt8 - ldr q3, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q5, [x0], #16 - ldr q7, [x0], #16 - ldr q8, [x0], #16 - str q0, [x1], #16 // write output - ldr q15, [x0] - eor v0.16b, v1.16b, v3.16b - eor v1.16b, v6.16b, v5.16b - eor v3.16b, v4.16b, v7.16b - str q0, [x1], #16 - eor v0.16b, v2.16b, v8.16b - str q1, [x1], #16 - str q3, [x1], #16 - str q0, [x1] - b Lcbc_dec_done -.align 4 -Lcbc_dec_four: - sub x0, x0, #0x40 - bl _bsaes_decrypt8 - ldr q2, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q3, [x0], #16 - ldr q5, [x0], #16 - str q0, [x1], #16 // write output - ldr q15, [x0] - eor v0.16b, v1.16b, v2.16b - eor v1.16b, v6.16b, v3.16b - eor v2.16b, v4.16b, v5.16b - str q0, [x1], #16 - str q1, [x1], #16 - str q2, [x1] - b Lcbc_dec_done -.align 4 -Lcbc_dec_three: - sub x0, x0, #0x30 - bl _bsaes_decrypt8 - ldr q2, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q3, [x0], #16 - ldr q15, [x0] - str q0, [x1], #16 // write output - eor v0.16b, v1.16b, v2.16b - eor v1.16b, v6.16b, v3.16b - str q0, [x1], #16 - str q1, [x1] - b Lcbc_dec_done -.align 4 -Lcbc_dec_two: - sub x0, x0, #0x20 - bl _bsaes_decrypt8 - ldr q2, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q15, [x0] - str q0, [x1], #16 // write output - eor v0.16b, v1.16b, v2.16b - str q0, [x1] - b Lcbc_dec_done -.align 4 -Lcbc_dec_one: - sub x0, x0, #0x10 - stp x1, x4, [sp, #-32]! - str x14, [sp, #16] - mov v8.16b, v15.16b - mov v15.16b, v0.16b - mov x2, x3 - bl _AES_decrypt - ldr x14, [sp, #16] - ldp x1, x4, [sp], #32 - ldr q0, [x1] // load result - eor v0.16b, v0.16b, v8.16b // ^= IV - str q0, [x1] // write output - -.align 4 -Lcbc_dec_done: - movi v0.16b, #0 - movi v1.16b, #0 -Lcbc_dec_bzero: // wipe key schedule [if any] - stp q0, q1, [sp], #32 - cmp sp, x14 - bne Lcbc_dec_bzero - str q15, [x4] // return IV - ldp d8, d9, [sp, #16] - ldp d10, d15, [sp, #32] - ldp x29, x30, [sp], #48 - ret - - -.globl _ossl_bsaes_ctr32_encrypt_blocks - -.align 4 -// On entry: -// x0 -> input text (whole 16-byte blocks) -// x1 -> output text (whole 16-byte blocks) -// x2 = number of 16-byte blocks to encrypt/decrypt (> 0) -// x3 -> key -// x4 -> initial value of 128-bit counter (stored big-endian) which increments, modulo 2^32, for each block -// On exit: -// Output text filled in -// No output registers, usual AAPCS64 register preservation -_ossl_bsaes_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET - cmp x2, #8 // use plain AES for - blo Lctr_enc_short // small sizes - - stp x29, x30, [sp, #-80]! - stp d8, d9, [sp, #16] - stp d10, d11, [sp, #32] - stp d12, d13, [sp, #48] - stp d14, d15, [sp, #64] - - ldr w15, [x3, #240] // get # of rounds - mov x14, sp - - // allocate the key schedule on the stack - add x17, sp, #96 - sub x17, x17, x15, lsl #7 // 128 bytes per inner round key, less 96 bytes - - // populate the key schedule - mov x9, x3 // pass key - mov x10, x15 // pass # of rounds - mov sp, x17 // sp is sp - bl _bsaes_key_convert - eor v7.16b, v7.16b, v15.16b // fix up last round key - str q7, [x17] // save last round key - - ldr q0, [x4] // load counter - add x13, x11, #LREVM0SR-LM0_bigendian - ldr q4, [sp] // load round0 key - - movi v8.4s, #1 // compose 1<<96 - movi v9.16b, #0 - rev32 v15.16b, v0.16b - rev32 v0.16b, v0.16b - ext v11.16b, v9.16b, v8.16b, #4 - rev32 v4.16b, v4.16b - add v12.4s, v11.4s, v11.4s // compose 2<<96 - str q4, [sp] // save adjusted round0 key - add v13.4s, v11.4s, v12.4s // compose 3<<96 - add v14.4s, v12.4s, v12.4s // compose 4<<96 - b Lctr_enc_loop - -.align 4 -Lctr_enc_loop: - // Intermix prologue from _bsaes_encrypt8 to use the opportunity - // to flip byte order in 32-bit counter - - add v1.4s, v15.4s, v11.4s // +1 - add x9, sp, #0x10 // pass next round key - add v2.4s, v15.4s, v12.4s // +2 - ldr q9, [x13] // LREVM0SR - ldr q8, [sp] // load round0 key - add v3.4s, v15.4s, v13.4s // +3 - mov x10, x15 // pass rounds - sub x11, x13, #LREVM0SR-LSR // pass constants - add v6.4s, v2.4s, v14.4s - add v4.4s, v15.4s, v14.4s // +4 - add v7.4s, v3.4s, v14.4s - add v15.4s, v4.4s, v14.4s // next counter - add v5.4s, v1.4s, v14.4s - - bl _bsaes_encrypt8_alt - - subs x2, x2, #8 - blo Lctr_enc_loop_done - - ldr q16, [x0], #16 - ldr q17, [x0], #16 - eor v1.16b, v1.16b, v17.16b - ldr q17, [x0], #16 - eor v0.16b, v0.16b, v16.16b - eor v4.16b, v4.16b, v17.16b - str q0, [x1], #16 - ldr q16, [x0], #16 - str q1, [x1], #16 - mov v0.16b, v15.16b - str q4, [x1], #16 - ldr q1, [x0], #16 - eor v4.16b, v6.16b, v16.16b - eor v1.16b, v3.16b, v1.16b - ldr q3, [x0], #16 - eor v3.16b, v7.16b, v3.16b - ldr q6, [x0], #16 - eor v2.16b, v2.16b, v6.16b - ldr q6, [x0], #16 - eor v5.16b, v5.16b, v6.16b - str q4, [x1], #16 - str q1, [x1], #16 - str q3, [x1], #16 - str q2, [x1], #16 - str q5, [x1], #16 - - bne Lctr_enc_loop - b Lctr_enc_done - -.align 4 -Lctr_enc_loop_done: - add x2, x2, #8 - ldr q16, [x0], #16 // load input - eor v0.16b, v0.16b, v16.16b - str q0, [x1], #16 // write output - cmp x2, #2 - blo Lctr_enc_done - ldr q17, [x0], #16 - eor v1.16b, v1.16b, v17.16b - str q1, [x1], #16 - beq Lctr_enc_done - ldr q18, [x0], #16 - eor v4.16b, v4.16b, v18.16b - str q4, [x1], #16 - cmp x2, #4 - blo Lctr_enc_done - ldr q19, [x0], #16 - eor v6.16b, v6.16b, v19.16b - str q6, [x1], #16 - beq Lctr_enc_done - ldr q20, [x0], #16 - eor v3.16b, v3.16b, v20.16b - str q3, [x1], #16 - cmp x2, #6 - blo Lctr_enc_done - ldr q21, [x0], #16 - eor v7.16b, v7.16b, v21.16b - str q7, [x1], #16 - beq Lctr_enc_done - ldr q22, [x0] - eor v2.16b, v2.16b, v22.16b - str q2, [x1], #16 - -Lctr_enc_done: - movi v0.16b, #0 - movi v1.16b, #0 -Lctr_enc_bzero: // wipe key schedule [if any] - stp q0, q1, [sp], #32 - cmp sp, x14 - bne Lctr_enc_bzero - - ldp d8, d9, [sp, #16] - ldp d10, d11, [sp, #32] - ldp d12, d13, [sp, #48] - ldp d14, d15, [sp, #64] - ldp x29, x30, [sp], #80 - ret - -Lctr_enc_short: - stp x29, x30, [sp, #-96]! - stp x19, x20, [sp, #16] - stp x21, x22, [sp, #32] - str x23, [sp, #48] - - mov x19, x0 // copy arguments - mov x20, x1 - mov x21, x2 - mov x22, x3 - ldr w23, [x4, #12] // load counter LSW - ldr q1, [x4] // load whole counter value -#ifdef __AARCH64EL__ - rev w23, w23 -#endif - str q1, [sp, #80] // copy counter value - -Lctr_enc_short_loop: - add x0, sp, #80 // input counter value - add x1, sp, #64 // output on the stack - mov x2, x22 // key - - bl _AES_encrypt - - ldr q0, [x19], #16 // load input - ldr q1, [sp, #64] // load encrypted counter - add x23, x23, #1 -#ifdef __AARCH64EL__ - rev w0, w23 - str w0, [sp, #80+12] // next counter value -#else - str w23, [sp, #80+12] // next counter value -#endif - eor v0.16b, v0.16b, v1.16b - str q0, [x20], #16 // store output - subs x21, x21, #1 - bne Lctr_enc_short_loop - - movi v0.16b, #0 - movi v1.16b, #0 - stp q0, q1, [sp, #64] - - ldr x23, [sp, #48] - ldp x21, x22, [sp, #32] - ldp x19, x20, [sp, #16] - ldp x29, x30, [sp], #96 - ret - - -.globl _ossl_bsaes_xts_encrypt - -.align 4 -// On entry: -// x0 -> input plaintext -// x1 -> output ciphertext -// x2 -> length of text in bytes (must be at least 16) -// x3 -> key1 (used to encrypt the XORed plaintext blocks) -// x4 -> key2 (used to encrypt the initial vector to yield the initial tweak) -// x5 -> 16-byte initial vector (typically, sector number) -// On exit: -// Output ciphertext filled in -// No output registers, usual AAPCS64 register preservation -_ossl_bsaes_xts_encrypt: - AARCH64_VALID_CALL_TARGET - // Stack layout: - // sp -> - // nrounds*128-96 bytes: key schedule - // x19 -> - // 16 bytes: frame record - // 4*16 bytes: tweak storage across _bsaes_encrypt8 - // 6*8 bytes: storage for 5 callee-saved general-purpose registers - // 8*8 bytes: storage for 8 callee-saved SIMD registers - stp x29, x30, [sp, #-192]! - stp x19, x20, [sp, #80] - stp x21, x22, [sp, #96] - str x23, [sp, #112] - stp d8, d9, [sp, #128] - stp d10, d11, [sp, #144] - stp d12, d13, [sp, #160] - stp d14, d15, [sp, #176] - - mov x19, sp - mov x20, x0 - mov x21, x1 - mov x22, x2 - mov x23, x3 - - // generate initial tweak - sub sp, sp, #16 - mov x0, x5 // iv[] - mov x1, sp - mov x2, x4 // key2 - bl _AES_encrypt - ldr q11, [sp], #16 - - ldr w1, [x23, #240] // get # of rounds - // allocate the key schedule on the stack - add x17, sp, #96 - sub x17, x17, x1, lsl #7 // 128 bytes per inner round key, less 96 bytes - - // populate the key schedule - mov x9, x23 // pass key - mov x10, x1 // pass # of rounds - mov sp, x17 - bl _bsaes_key_convert - eor v15.16b, v15.16b, v7.16b // fix up last round key - str q15, [x17] // save last round key - - subs x22, x22, #0x80 - blo Lxts_enc_short - b Lxts_enc_loop - -.align 4 -Lxts_enc_loop: - ldr q8, Lxts_magic - mov x10, x1 // pass rounds - add x2, x19, #16 - ldr q0, [x20], #16 - sshr v1.2d, v11.2d, #63 - mov x9, sp // pass key schedule - ldr q6, Lxts_magic+16 - add v2.2d, v11.2d, v11.2d - cmtst v3.2d, v11.2d, v6.2d - and v1.16b, v1.16b, v8.16b - ext v1.16b, v1.16b, v1.16b, #8 - and v3.16b, v3.16b, v8.16b - ldr q4, [x20], #16 - eor v12.16b, v2.16b, v1.16b - eor v1.16b, v4.16b, v12.16b - eor v0.16b, v0.16b, v11.16b - cmtst v2.2d, v12.2d, v6.2d - add v4.2d, v12.2d, v12.2d - add x0, x19, #16 - ext v3.16b, v3.16b, v3.16b, #8 - and v2.16b, v2.16b, v8.16b - eor v13.16b, v4.16b, v3.16b - ldr q3, [x20], #16 - ext v4.16b, v2.16b, v2.16b, #8 - eor v2.16b, v3.16b, v13.16b - ldr q3, [x20], #16 - add v5.2d, v13.2d, v13.2d - cmtst v7.2d, v13.2d, v6.2d - and v7.16b, v7.16b, v8.16b - ldr q9, [x20], #16 - ext v7.16b, v7.16b, v7.16b, #8 - ldr q10, [x20], #16 - eor v14.16b, v5.16b, v4.16b - ldr q16, [x20], #16 - add v4.2d, v14.2d, v14.2d - eor v3.16b, v3.16b, v14.16b - eor v15.16b, v4.16b, v7.16b - add v5.2d, v15.2d, v15.2d - ldr q7, [x20], #16 - cmtst v4.2d, v14.2d, v6.2d - and v17.16b, v4.16b, v8.16b - cmtst v18.2d, v15.2d, v6.2d - eor v4.16b, v9.16b, v15.16b - ext v9.16b, v17.16b, v17.16b, #8 - eor v9.16b, v5.16b, v9.16b - add v17.2d, v9.2d, v9.2d - and v18.16b, v18.16b, v8.16b - eor v5.16b, v10.16b, v9.16b - str q9, [x2], #16 - ext v10.16b, v18.16b, v18.16b, #8 - cmtst v9.2d, v9.2d, v6.2d - and v9.16b, v9.16b, v8.16b - eor v10.16b, v17.16b, v10.16b - cmtst v17.2d, v10.2d, v6.2d - eor v6.16b, v16.16b, v10.16b - str q10, [x2], #16 - ext v9.16b, v9.16b, v9.16b, #8 - add v10.2d, v10.2d, v10.2d - eor v9.16b, v10.16b, v9.16b - str q9, [x2], #16 - eor v7.16b, v7.16b, v9.16b - add v9.2d, v9.2d, v9.2d - and v8.16b, v17.16b, v8.16b - ext v8.16b, v8.16b, v8.16b, #8 - eor v8.16b, v9.16b, v8.16b - str q8, [x2] // next round tweak - - bl _bsaes_encrypt8 - - ldr q8, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q9, [x0], #16 - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - ldr q10, [x0], #16 - eor v3.16b, v3.16b, v15.16b - subs x22, x22, #0x80 - str q0, [x21], #16 - ldr q11, [x0] // next round tweak - str q1, [x21], #16 - eor v0.16b, v7.16b, v8.16b - eor v1.16b, v2.16b, v9.16b - str q4, [x21], #16 - eor v2.16b, v5.16b, v10.16b - str q6, [x21], #16 - str q3, [x21], #16 - str q0, [x21], #16 - str q1, [x21], #16 - str q2, [x21], #16 - bpl Lxts_enc_loop - -Lxts_enc_short: - adds x22, x22, #0x70 - bmi Lxts_enc_done - - ldr q8, Lxts_magic - sshr v1.2d, v11.2d, #63 - add v2.2d, v11.2d, v11.2d - ldr q9, Lxts_magic+16 - subs x22, x22, #0x10 - ldr q0, [x20], #16 - and v1.16b, v1.16b, v8.16b - cmtst v3.2d, v11.2d, v9.2d - ext v1.16b, v1.16b, v1.16b, #8 - and v3.16b, v3.16b, v8.16b - eor v12.16b, v2.16b, v1.16b - ext v1.16b, v3.16b, v3.16b, #8 - add v2.2d, v12.2d, v12.2d - cmtst v3.2d, v12.2d, v9.2d - eor v13.16b, v2.16b, v1.16b - and v22.16b, v3.16b, v8.16b - bmi Lxts_enc_1 - - ext v2.16b, v22.16b, v22.16b, #8 - add v3.2d, v13.2d, v13.2d - ldr q1, [x20], #16 - cmtst v4.2d, v13.2d, v9.2d - subs x22, x22, #0x10 - eor v14.16b, v3.16b, v2.16b - and v23.16b, v4.16b, v8.16b - bmi Lxts_enc_2 - - ext v3.16b, v23.16b, v23.16b, #8 - add v4.2d, v14.2d, v14.2d - ldr q2, [x20], #16 - cmtst v5.2d, v14.2d, v9.2d - eor v0.16b, v0.16b, v11.16b - subs x22, x22, #0x10 - eor v15.16b, v4.16b, v3.16b - and v24.16b, v5.16b, v8.16b - bmi Lxts_enc_3 - - ext v4.16b, v24.16b, v24.16b, #8 - add v5.2d, v15.2d, v15.2d - ldr q3, [x20], #16 - cmtst v6.2d, v15.2d, v9.2d - eor v1.16b, v1.16b, v12.16b - subs x22, x22, #0x10 - eor v16.16b, v5.16b, v4.16b - and v25.16b, v6.16b, v8.16b - bmi Lxts_enc_4 - - ext v5.16b, v25.16b, v25.16b, #8 - add v6.2d, v16.2d, v16.2d - add x0, x19, #16 - cmtst v7.2d, v16.2d, v9.2d - ldr q4, [x20], #16 - eor v2.16b, v2.16b, v13.16b - str q16, [x0], #16 - subs x22, x22, #0x10 - eor v17.16b, v6.16b, v5.16b - and v26.16b, v7.16b, v8.16b - bmi Lxts_enc_5 - - ext v7.16b, v26.16b, v26.16b, #8 - add v18.2d, v17.2d, v17.2d - ldr q5, [x20], #16 - eor v3.16b, v3.16b, v14.16b - str q17, [x0], #16 - subs x22, x22, #0x10 - eor v18.16b, v18.16b, v7.16b - bmi Lxts_enc_6 - - ldr q6, [x20], #16 - eor v4.16b, v4.16b, v15.16b - eor v5.16b, v5.16b, v16.16b - str q18, [x0] // next round tweak - mov x9, sp // pass key schedule - mov x10, x1 - add x0, x19, #16 - sub x22, x22, #0x10 - eor v6.16b, v6.16b, v17.16b - - bl _bsaes_encrypt8 - - ldr q16, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q17, [x0], #16 - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - ldr q11, [x0] // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - eor v0.16b, v7.16b, v16.16b - eor v1.16b, v2.16b, v17.16b - str q4, [x21], #16 - str q6, [x21], #16 - str q3, [x21], #16 - str q0, [x21], #16 - str q1, [x21], #16 - b Lxts_enc_done - -.align 4 -Lxts_enc_6: - eor v4.16b, v4.16b, v15.16b - eor v5.16b, v5.16b, v16.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - ldr q16, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - ldr q11, [x0] // next round tweak - eor v3.16b, v3.16b, v15.16b - str q0, [x21], #16 - str q1, [x21], #16 - eor v0.16b, v7.16b, v16.16b - str q4, [x21], #16 - str q6, [x21], #16 - str q3, [x21], #16 - str q0, [x21], #16 - b Lxts_enc_done - -.align 4 -Lxts_enc_5: - eor v3.16b, v3.16b, v14.16b - eor v4.16b, v4.16b, v15.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q11, [x0] // next round tweak - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - str q0, [x21], #16 - str q1, [x21], #16 - str q4, [x21], #16 - str q6, [x21], #16 - str q3, [x21], #16 - b Lxts_enc_done - -.align 4 -Lxts_enc_4: - eor v2.16b, v2.16b, v13.16b - eor v3.16b, v3.16b, v14.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - mov v11.16b, v15.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - str q4, [x21], #16 - str q6, [x21], #16 - b Lxts_enc_done - -.align 4 -Lxts_enc_3: - eor v1.16b, v1.16b, v12.16b - eor v2.16b, v2.16b, v13.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v4.16b, v4.16b, v13.16b - mov v11.16b, v14.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - str q4, [x21], #16 - b Lxts_enc_done - -.align 4 -Lxts_enc_2: - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - mov v11.16b, v13.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - b Lxts_enc_done - -.align 4 -Lxts_enc_1: - eor v0.16b, v0.16b, v11.16b - sub x0, sp, #16 - sub x1, sp, #16 - mov x2, x23 - mov v13.d[0], v11.d[1] // just in case _AES_encrypt corrupts top half of callee-saved SIMD registers - mov v14.d[0], v12.d[1] - str q0, [sp, #-16]! - - bl _AES_encrypt - - ldr q0, [sp], #16 - trn1 v13.2d, v11.2d, v13.2d - trn1 v11.2d, v12.2d, v14.2d // next round tweak - eor v0.16b, v0.16b, v13.16b - str q0, [x21], #16 - -Lxts_enc_done: - adds x22, x22, #0x10 - beq Lxts_enc_ret - - sub x6, x21, #0x10 - // Penultimate plaintext block produces final ciphertext part-block - // plus remaining part of final plaintext block. Move ciphertext part - // to final position and reuse penultimate ciphertext block buffer to - // construct final plaintext block -Lxts_enc_steal: - ldrb w0, [x20], #1 - ldrb w1, [x21, #-0x10] - strb w0, [x21, #-0x10] - strb w1, [x21], #1 - - subs x22, x22, #1 - bhi Lxts_enc_steal - - // Finally encrypt the penultimate ciphertext block using the - // last tweak - ldr q0, [x6] - eor v0.16b, v0.16b, v11.16b - str q0, [sp, #-16]! - mov x0, sp - mov x1, sp - mov x2, x23 - mov x21, x6 - mov v13.d[0], v11.d[1] // just in case _AES_encrypt corrupts top half of callee-saved SIMD registers - - bl _AES_encrypt - - trn1 v11.2d, v11.2d, v13.2d - ldr q0, [sp], #16 - eor v0.16b, v0.16b, v11.16b - str q0, [x21] - -Lxts_enc_ret: - - movi v0.16b, #0 - movi v1.16b, #0 -Lxts_enc_bzero: // wipe key schedule - stp q0, q1, [sp], #32 - cmp sp, x19 - bne Lxts_enc_bzero - - ldp x19, x20, [sp, #80] - ldp x21, x22, [sp, #96] - ldr x23, [sp, #112] - ldp d8, d9, [sp, #128] - ldp d10, d11, [sp, #144] - ldp d12, d13, [sp, #160] - ldp d14, d15, [sp, #176] - ldp x29, x30, [sp], #192 - ret - - -// The assembler doesn't seem capable of de-duplicating these when expressed -// using `ldr qd,=` syntax, so assign a symbolic address -.align 5 -Lxts_magic: -.quad 1, 0x87, 0x4000000000000000, 0x4000000000000000 - -.globl _ossl_bsaes_xts_decrypt - -.align 4 -// On entry: -// x0 -> input ciphertext -// x1 -> output plaintext -// x2 -> length of text in bytes (must be at least 16) -// x3 -> key1 (used to decrypt the XORed ciphertext blocks) -// x4 -> key2 (used to encrypt the initial vector to yield the initial tweak) -// x5 -> 16-byte initial vector (typically, sector number) -// On exit: -// Output plaintext filled in -// No output registers, usual AAPCS64 register preservation -_ossl_bsaes_xts_decrypt: - AARCH64_VALID_CALL_TARGET - // Stack layout: - // sp -> - // nrounds*128-96 bytes: key schedule - // x19 -> - // 16 bytes: frame record - // 4*16 bytes: tweak storage across _bsaes_decrypt8 - // 6*8 bytes: storage for 5 callee-saved general-purpose registers - // 8*8 bytes: storage for 8 callee-saved SIMD registers - stp x29, x30, [sp, #-192]! - stp x19, x20, [sp, #80] - stp x21, x22, [sp, #96] - str x23, [sp, #112] - stp d8, d9, [sp, #128] - stp d10, d11, [sp, #144] - stp d12, d13, [sp, #160] - stp d14, d15, [sp, #176] - - mov x19, sp - mov x20, x0 - mov x21, x1 - mov x22, x2 - mov x23, x3 - - // generate initial tweak - sub sp, sp, #16 - mov x0, x5 // iv[] - mov x1, sp - mov x2, x4 // key2 - bl _AES_encrypt - ldr q11, [sp], #16 - - ldr w1, [x23, #240] // get # of rounds - // allocate the key schedule on the stack - add x17, sp, #96 - sub x17, x17, x1, lsl #7 // 128 bytes per inner round key, less 96 bytes - - // populate the key schedule - mov x9, x23 // pass key - mov x10, x1 // pass # of rounds - mov sp, x17 - bl _bsaes_key_convert - ldr q6, [sp] - str q15, [x17] // save last round key - eor v6.16b, v6.16b, v7.16b // fix up round 0 key (by XORing with 0x63) - str q6, [sp] - - sub x30, x22, #0x10 - tst x22, #0xf // if not multiple of 16 - csel x22, x30, x22, ne // subtract another 16 bytes - subs x22, x22, #0x80 - - blo Lxts_dec_short - b Lxts_dec_loop - -.align 4 -Lxts_dec_loop: - ldr q8, Lxts_magic - mov x10, x1 // pass rounds - add x2, x19, #16 - ldr q0, [x20], #16 - sshr v1.2d, v11.2d, #63 - mov x9, sp // pass key schedule - ldr q6, Lxts_magic+16 - add v2.2d, v11.2d, v11.2d - cmtst v3.2d, v11.2d, v6.2d - and v1.16b, v1.16b, v8.16b - ext v1.16b, v1.16b, v1.16b, #8 - and v3.16b, v3.16b, v8.16b - ldr q4, [x20], #16 - eor v12.16b, v2.16b, v1.16b - eor v1.16b, v4.16b, v12.16b - eor v0.16b, v0.16b, v11.16b - cmtst v2.2d, v12.2d, v6.2d - add v4.2d, v12.2d, v12.2d - add x0, x19, #16 - ext v3.16b, v3.16b, v3.16b, #8 - and v2.16b, v2.16b, v8.16b - eor v13.16b, v4.16b, v3.16b - ldr q3, [x20], #16 - ext v4.16b, v2.16b, v2.16b, #8 - eor v2.16b, v3.16b, v13.16b - ldr q3, [x20], #16 - add v5.2d, v13.2d, v13.2d - cmtst v7.2d, v13.2d, v6.2d - and v7.16b, v7.16b, v8.16b - ldr q9, [x20], #16 - ext v7.16b, v7.16b, v7.16b, #8 - ldr q10, [x20], #16 - eor v14.16b, v5.16b, v4.16b - ldr q16, [x20], #16 - add v4.2d, v14.2d, v14.2d - eor v3.16b, v3.16b, v14.16b - eor v15.16b, v4.16b, v7.16b - add v5.2d, v15.2d, v15.2d - ldr q7, [x20], #16 - cmtst v4.2d, v14.2d, v6.2d - and v17.16b, v4.16b, v8.16b - cmtst v18.2d, v15.2d, v6.2d - eor v4.16b, v9.16b, v15.16b - ext v9.16b, v17.16b, v17.16b, #8 - eor v9.16b, v5.16b, v9.16b - add v17.2d, v9.2d, v9.2d - and v18.16b, v18.16b, v8.16b - eor v5.16b, v10.16b, v9.16b - str q9, [x2], #16 - ext v10.16b, v18.16b, v18.16b, #8 - cmtst v9.2d, v9.2d, v6.2d - and v9.16b, v9.16b, v8.16b - eor v10.16b, v17.16b, v10.16b - cmtst v17.2d, v10.2d, v6.2d - eor v6.16b, v16.16b, v10.16b - str q10, [x2], #16 - ext v9.16b, v9.16b, v9.16b, #8 - add v10.2d, v10.2d, v10.2d - eor v9.16b, v10.16b, v9.16b - str q9, [x2], #16 - eor v7.16b, v7.16b, v9.16b - add v9.2d, v9.2d, v9.2d - and v8.16b, v17.16b, v8.16b - ext v8.16b, v8.16b, v8.16b, #8 - eor v8.16b, v9.16b, v8.16b - str q8, [x2] // next round tweak - - bl _bsaes_decrypt8 - - eor v6.16b, v6.16b, v13.16b - eor v0.16b, v0.16b, v11.16b - ldr q8, [x0], #16 - eor v7.16b, v7.16b, v8.16b - str q0, [x21], #16 - eor v0.16b, v1.16b, v12.16b - ldr q1, [x0], #16 - eor v1.16b, v3.16b, v1.16b - subs x22, x22, #0x80 - eor v2.16b, v2.16b, v15.16b - eor v3.16b, v4.16b, v14.16b - ldr q4, [x0], #16 - str q0, [x21], #16 - ldr q11, [x0] // next round tweak - eor v0.16b, v5.16b, v4.16b - str q6, [x21], #16 - str q3, [x21], #16 - str q2, [x21], #16 - str q7, [x21], #16 - str q1, [x21], #16 - str q0, [x21], #16 - bpl Lxts_dec_loop - -Lxts_dec_short: - adds x22, x22, #0x70 - bmi Lxts_dec_done - - ldr q8, Lxts_magic - sshr v1.2d, v11.2d, #63 - add v2.2d, v11.2d, v11.2d - ldr q9, Lxts_magic+16 - subs x22, x22, #0x10 - ldr q0, [x20], #16 - and v1.16b, v1.16b, v8.16b - cmtst v3.2d, v11.2d, v9.2d - ext v1.16b, v1.16b, v1.16b, #8 - and v3.16b, v3.16b, v8.16b - eor v12.16b, v2.16b, v1.16b - ext v1.16b, v3.16b, v3.16b, #8 - add v2.2d, v12.2d, v12.2d - cmtst v3.2d, v12.2d, v9.2d - eor v13.16b, v2.16b, v1.16b - and v22.16b, v3.16b, v8.16b - bmi Lxts_dec_1 - - ext v2.16b, v22.16b, v22.16b, #8 - add v3.2d, v13.2d, v13.2d - ldr q1, [x20], #16 - cmtst v4.2d, v13.2d, v9.2d - subs x22, x22, #0x10 - eor v14.16b, v3.16b, v2.16b - and v23.16b, v4.16b, v8.16b - bmi Lxts_dec_2 - - ext v3.16b, v23.16b, v23.16b, #8 - add v4.2d, v14.2d, v14.2d - ldr q2, [x20], #16 - cmtst v5.2d, v14.2d, v9.2d - eor v0.16b, v0.16b, v11.16b - subs x22, x22, #0x10 - eor v15.16b, v4.16b, v3.16b - and v24.16b, v5.16b, v8.16b - bmi Lxts_dec_3 - - ext v4.16b, v24.16b, v24.16b, #8 - add v5.2d, v15.2d, v15.2d - ldr q3, [x20], #16 - cmtst v6.2d, v15.2d, v9.2d - eor v1.16b, v1.16b, v12.16b - subs x22, x22, #0x10 - eor v16.16b, v5.16b, v4.16b - and v25.16b, v6.16b, v8.16b - bmi Lxts_dec_4 - - ext v5.16b, v25.16b, v25.16b, #8 - add v6.2d, v16.2d, v16.2d - add x0, x19, #16 - cmtst v7.2d, v16.2d, v9.2d - ldr q4, [x20], #16 - eor v2.16b, v2.16b, v13.16b - str q16, [x0], #16 - subs x22, x22, #0x10 - eor v17.16b, v6.16b, v5.16b - and v26.16b, v7.16b, v8.16b - bmi Lxts_dec_5 - - ext v7.16b, v26.16b, v26.16b, #8 - add v18.2d, v17.2d, v17.2d - ldr q5, [x20], #16 - eor v3.16b, v3.16b, v14.16b - str q17, [x0], #16 - subs x22, x22, #0x10 - eor v18.16b, v18.16b, v7.16b - bmi Lxts_dec_6 - - ldr q6, [x20], #16 - eor v4.16b, v4.16b, v15.16b - eor v5.16b, v5.16b, v16.16b - str q18, [x0] // next round tweak - mov x9, sp // pass key schedule - mov x10, x1 - add x0, x19, #16 - sub x22, x22, #0x10 - eor v6.16b, v6.16b, v17.16b - - bl _bsaes_decrypt8 - - ldr q16, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q17, [x0], #16 - eor v6.16b, v6.16b, v13.16b - eor v4.16b, v4.16b, v14.16b - eor v2.16b, v2.16b, v15.16b - ldr q11, [x0] // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - eor v0.16b, v7.16b, v16.16b - eor v1.16b, v3.16b, v17.16b - str q6, [x21], #16 - str q4, [x21], #16 - str q2, [x21], #16 - str q0, [x21], #16 - str q1, [x21], #16 - b Lxts_dec_done - -.align 4 -Lxts_dec_6: - eor v4.16b, v4.16b, v15.16b - eor v5.16b, v5.16b, v16.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - ldr q16, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v6.16b, v6.16b, v13.16b - eor v4.16b, v4.16b, v14.16b - ldr q11, [x0] // next round tweak - eor v2.16b, v2.16b, v15.16b - str q0, [x21], #16 - str q1, [x21], #16 - eor v0.16b, v7.16b, v16.16b - str q6, [x21], #16 - str q4, [x21], #16 - str q2, [x21], #16 - str q0, [x21], #16 - b Lxts_dec_done - -.align 4 -Lxts_dec_5: - eor v3.16b, v3.16b, v14.16b - eor v4.16b, v4.16b, v15.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q11, [x0] // next round tweak - eor v6.16b, v6.16b, v13.16b - eor v4.16b, v4.16b, v14.16b - eor v2.16b, v2.16b, v15.16b - str q0, [x21], #16 - str q1, [x21], #16 - str q6, [x21], #16 - str q4, [x21], #16 - str q2, [x21], #16 - b Lxts_dec_done - -.align 4 -Lxts_dec_4: - eor v2.16b, v2.16b, v13.16b - eor v3.16b, v3.16b, v14.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v6.16b, v6.16b, v13.16b - eor v4.16b, v4.16b, v14.16b - mov v11.16b, v15.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - str q6, [x21], #16 - str q4, [x21], #16 - b Lxts_dec_done - -.align 4 -Lxts_dec_3: - eor v1.16b, v1.16b, v12.16b - eor v2.16b, v2.16b, v13.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v6.16b, v6.16b, v13.16b - mov v11.16b, v14.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - str q6, [x21], #16 - b Lxts_dec_done - -.align 4 -Lxts_dec_2: - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - mov v11.16b, v13.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - b Lxts_dec_done - -.align 4 -Lxts_dec_1: - eor v0.16b, v0.16b, v11.16b - sub x0, sp, #16 - sub x1, sp, #16 - mov x2, x23 - mov v13.d[0], v11.d[1] // just in case _AES_decrypt corrupts top half of callee-saved SIMD registers - mov v14.d[0], v12.d[1] - str q0, [sp, #-16]! - - bl _AES_decrypt - - ldr q0, [sp], #16 - trn1 v13.2d, v11.2d, v13.2d - trn1 v11.2d, v12.2d, v14.2d // next round tweak - eor v0.16b, v0.16b, v13.16b - str q0, [x21], #16 - -Lxts_dec_done: - adds x22, x22, #0x10 - beq Lxts_dec_ret - - // calculate one round of extra tweak for the stolen ciphertext - ldr q8, Lxts_magic - sshr v6.2d, v11.2d, #63 - and v6.16b, v6.16b, v8.16b - add v12.2d, v11.2d, v11.2d - ext v6.16b, v6.16b, v6.16b, #8 - eor v12.16b, v12.16b, v6.16b - - // perform the final decryption with the last tweak value - ldr q0, [x20], #16 - eor v0.16b, v0.16b, v12.16b - str q0, [sp, #-16]! - mov x0, sp - mov x1, sp - mov x2, x23 - mov v13.d[0], v11.d[1] // just in case _AES_decrypt corrupts top half of callee-saved SIMD registers - mov v14.d[0], v12.d[1] - - bl _AES_decrypt - - trn1 v12.2d, v12.2d, v14.2d - trn1 v11.2d, v11.2d, v13.2d - ldr q0, [sp], #16 - eor v0.16b, v0.16b, v12.16b - str q0, [x21] - - mov x6, x21 - // Penultimate ciphertext block produces final plaintext part-block - // plus remaining part of final ciphertext block. Move plaintext part - // to final position and reuse penultimate plaintext block buffer to - // construct final ciphertext block -Lxts_dec_steal: - ldrb w1, [x21] - ldrb w0, [x20], #1 - strb w1, [x21, #0x10] - strb w0, [x21], #1 - - subs x22, x22, #1 - bhi Lxts_dec_steal - - // Finally decrypt the penultimate plaintext block using the - // penultimate tweak - ldr q0, [x6] - eor v0.16b, v0.16b, v11.16b - str q0, [sp, #-16]! - mov x0, sp - mov x1, sp - mov x2, x23 - mov x21, x6 - - bl _AES_decrypt - - trn1 v11.2d, v11.2d, v13.2d - ldr q0, [sp], #16 - eor v0.16b, v0.16b, v11.16b - str q0, [x21] - -Lxts_dec_ret: - - movi v0.16b, #0 - movi v1.16b, #0 -Lxts_dec_bzero: // wipe key schedule - stp q0, q1, [sp], #32 - cmp sp, x19 - bne Lxts_dec_bzero - - ldp x19, x20, [sp, #80] - ldp x21, x22, [sp, #96] - ldr x23, [sp, #112] - ldp d8, d9, [sp, #128] - ldp d10, d11, [sp, #144] - ldp d12, d13, [sp, #160] - ldp d14, d15, [sp, #176] - ldp x29, x30, [sp], #192 - ret - diff --git a/openssl/src/crypto/aes/gen/darwin_arm64/vpaes-armv8.S b/openssl/src/crypto/aes/gen/darwin_arm64/vpaes-armv8.S index d8a174588..0823766c7 100644 --- a/openssl/src/crypto/aes/gen/darwin_arm64/vpaes-armv8.S +++ b/openssl/src/crypto/aes/gen/darwin_arm64/vpaes-armv8.S @@ -1,5 +1,3 @@ -#include "arm_arch.h" - .text @@ -197,7 +195,7 @@ Lenc_entry: .align 4 _vpaes_encrypt: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -207,7 +205,7 @@ _vpaes_encrypt: st1 {v0.16b}, [x1] ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -430,7 +428,7 @@ Ldec_entry: .align 4 _vpaes_decrypt: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -440,7 +438,7 @@ _vpaes_decrypt: st1 {v0.16b}, [x1] ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -604,7 +602,7 @@ _vpaes_key_preheat: .align 4 _vpaes_schedule_core: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29, x30, [sp,#-16]! add x29,sp,#0 @@ -769,7 +767,7 @@ Lschedule_mangle_last_dec: eor v6.16b, v6.16b, v6.16b // vpxor %xmm6, %xmm6, %xmm6 eor v7.16b, v7.16b, v7.16b // vpxor %xmm7, %xmm7, %xmm7 ldp x29, x30, [sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -982,7 +980,7 @@ Lschedule_mangle_both: .align 4 _vpaes_set_encrypt_key: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -998,7 +996,7 @@ _vpaes_set_encrypt_key: ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -1006,7 +1004,7 @@ _vpaes_set_encrypt_key: .align 4 _vpaes_set_decrypt_key: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1026,18 +1024,18 @@ _vpaes_set_decrypt_key: ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .globl _vpaes_cbc_encrypt .align 4 _vpaes_cbc_encrypt: - AARCH64_SIGN_LINK_REGISTER cbz x2, Lcbc_abort cmp w5, #0 // check direction b.eq vpaes_cbc_decrypt +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1060,16 +1058,15 @@ Lcbc_enc_loop: st1 {v0.16b}, [x4] // write ivec ldp x29,x30,[sp],#16 +.long 0xd50323bf // autiasp Lcbc_abort: - AARCH64_VALIDATE_LINK_REGISTER ret .align 4 vpaes_cbc_decrypt: - // Not adding AARCH64_SIGN_LINK_REGISTER here because vpaes_cbc_decrypt is jumped to - // only from vpaes_cbc_encrypt which has already signed the return address. +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1111,14 +1108,14 @@ Lcbc_dec_done: ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .globl _vpaes_ecb_encrypt .align 4 _vpaes_ecb_encrypt: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1152,7 +1149,7 @@ Lecb_enc_done: ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -1160,7 +1157,7 @@ Lecb_enc_done: .align 4 _vpaes_ecb_decrypt: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1194,6 +1191,6 @@ Lecb_dec_done: ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret diff --git a/openssl/src/crypto/aes/gen/linux_arm/bsaes-armv7.S b/openssl/src/crypto/aes/gen/linux_arm/bsaes-armv7.S index 545d7f76c..2bdbe2f1d 100644 --- a/openssl/src/crypto/aes/gen/linux_arm/bsaes-armv7.S +++ b/openssl/src/crypto/aes/gen/linux_arm/bsaes-armv7.S @@ -1,4 +1,4 @@ -@ Copyright 2012-2023 The OpenSSL Project Authors. All Rights Reserved. +@ Copyright 2012-2021 The OpenSSL Project Authors. All Rights Reserved. @ @ Licensed under the Apache License 2.0 (the "License"). You may not use @ this file except in compliance with the License. You can obtain a copy @@ -13,7 +13,7 @@ @ details see http://www.openssl.org/~appro/cryptogams/. @ @ Specific modes and adaptation for Linux kernel by Ard Biesheuvel -@ of Linaro. +@ of Linaro. Permission to use under GPL terms is granted. @ ==================================================================== @ Bit-sliced AES for ARM NEON @@ -1131,7 +1131,7 @@ ossl_bsaes_cbc_encrypt: vstmia r4, {q7} .align 2 -0: + #endif vld1.8 {q15}, [r8] @ load IV @@ -1391,9 +1391,9 @@ ossl_bsaes_ctr32_encrypt_blocks: vstmia r12, {q7} @ save last round key .align 2 -0: add r12, r3, #248 + add r12, r3, #248 vld1.8 {q0}, [r8] @ load counter - add r8, r6, #.LREVM0SR-.LM0 @ borrow r8 + adrl r8, .LREVM0SR @ borrow r8 vldmia r12, {q4} @ load round0 key sub sp, #0x10 @ place for adjusted round0 key #endif @@ -1626,7 +1626,7 @@ ossl_bsaes_xts_encrypt: vstmia r12, {q7} .align 2 -0: sub sp, #0x90 @ place for tweak[9] + sub sp, #0x90 @ place for tweak[9] #endif vld1.8 {q8}, [r0] @ initial tweak @@ -2112,7 +2112,7 @@ ossl_bsaes_xts_decrypt: vstmia r4, {q7} .align 2 -0: sub sp, #0x90 @ place for tweak[9] + sub sp, #0x90 @ place for tweak[9] #endif vld1.8 {q8}, [r0] @ initial tweak adr r2, .Lxts_magic diff --git a/openssl/src/crypto/aes/gen/linux_arm64/aesv8-armx.S b/openssl/src/crypto/aes/gen/linux_arm64/aesv8-armx.S index df7b0369c..dcc07386a 100644 --- a/openssl/src/crypto/aes/gen/linux_arm64/aesv8-armx.S +++ b/openssl/src/crypto/aes/gen/linux_arm64/aesv8-armx.S @@ -14,8 +14,6 @@ .align 5 aes_v8_set_encrypt_key: .Lenc_key: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 mov x3,#-1 @@ -105,7 +103,7 @@ aes_v8_set_encrypt_key: .Loop192: tbl v6.16b,{v4.16b},v2.16b ext v5.16b,v0.16b,v3.16b,#12 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ st1 {v4.4s},[x2],#16 sub x2,x2,#8 #else @@ -187,7 +185,7 @@ aes_v8_set_encrypt_key: .type aes_v8_set_decrypt_key,%function .align 5 aes_v8_set_decrypt_key: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 bl .Lenc_key @@ -221,14 +219,13 @@ aes_v8_set_decrypt_key: eor x0,x0,x0 // return value .Ldec_key_abort: ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size aes_v8_set_decrypt_key,.-aes_v8_set_decrypt_key .globl aes_v8_encrypt .type aes_v8_encrypt,%function .align 5 aes_v8_encrypt: - AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 ld1 {v2.16b},[x0] @@ -258,7 +255,6 @@ aes_v8_encrypt: .type aes_v8_decrypt,%function .align 5 aes_v8_decrypt: - AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 ld1 {v2.16b},[x0] @@ -288,7 +284,6 @@ aes_v8_decrypt: .type aes_v8_ecb_encrypt,%function .align 5 aes_v8_ecb_encrypt: - AARCH64_VALID_CALL_TARGET subs x2,x2,#16 // Original input data size bigger than 16, jump to big size processing. b.ne .Lecb_big_size @@ -1035,8 +1030,6 @@ aes_v8_ecb_encrypt: .type aes_v8_cbc_encrypt,%function .align 5 aes_v8_cbc_encrypt: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 subs x2,x2,#16 @@ -1508,741 +1501,16 @@ aes_v8_cbc_encrypt: ldr x29,[sp],#16 ret .size aes_v8_cbc_encrypt,.-aes_v8_cbc_encrypt -.globl aes_v8_ctr32_encrypt_blocks_unroll12_eor3 -.type aes_v8_ctr32_encrypt_blocks_unroll12_eor3,%function -.align 5 -aes_v8_ctr32_encrypt_blocks_unroll12_eor3: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. - stp x29,x30,[sp,#-80]! - stp d8,d9,[sp, #16] - stp d10,d11,[sp, #32] - stp d12,d13,[sp, #48] - stp d14,d15,[sp, #64] - add x29,sp,#0 - - ldr w5,[x3,#240] - - ldr w8, [x4, #12] -#ifdef __AARCH64EB__ - ld1 {v24.16b},[x4] -#else - ld1 {v24.4s},[x4] -#endif - ld1 {v2.4s,v3.4s},[x3] // load key schedule... - sub w5,w5,#4 - cmp x2,#2 - add x7,x3,x5,lsl#4 // pointer to last round key - sub w5,w5,#2 - add x7, x7, #64 - ld1 {v1.4s},[x7] - add x7,x3,#32 - mov w6,w5 -#ifndef __AARCH64EB__ - rev w8, w8 -#endif - - orr v25.16b,v24.16b,v24.16b - add w10, w8, #1 - orr v26.16b,v24.16b,v24.16b - add w8, w8, #2 - orr v0.16b,v24.16b,v24.16b - rev w10, w10 - mov v25.s[3],w10 - b.ls .Lctr32_tail_unroll - cmp x2,#6 - rev w12, w8 - sub x2,x2,#3 // bias - mov v26.s[3],w12 - b.lo .Loop3x_ctr32_unroll - cmp x2,#9 - orr v27.16b,v24.16b,v24.16b - add w11, w8, #1 - orr v28.16b,v24.16b,v24.16b - add w13, w8, #2 - rev w11, w11 - orr v29.16b,v24.16b,v24.16b - add w8, w8, #3 - rev w13, w13 - mov v27.s[3],w11 - rev w14, w8 - mov v28.s[3],w13 - mov v29.s[3],w14 - sub x2,x2,#3 - b.lo .Loop6x_ctr32_unroll - - // push regs to stack when 12 data chunks are interleaved - stp x19,x20,[sp,#-16]! - stp x21,x22,[sp,#-16]! - stp x23,x24,[sp,#-16]! - stp d8,d9,[sp,#-32]! - stp d10,d11,[sp,#-32]! - - add w15,w8,#1 - add w19,w8,#2 - add w20,w8,#3 - add w21,w8,#4 - add w22,w8,#5 - add w8,w8,#6 - orr v30.16b,v24.16b,v24.16b - rev w15,w15 - orr v31.16b,v24.16b,v24.16b - rev w19,w19 - orr v8.16b,v24.16b,v24.16b - rev w20,w20 - orr v9.16b,v24.16b,v24.16b - rev w21,w21 - orr v10.16b,v24.16b,v24.16b - rev w22,w22 - orr v11.16b,v24.16b,v24.16b - rev w23,w8 - - sub x2,x2,#6 // bias - mov v30.s[3],w15 - mov v31.s[3],w19 - mov v8.s[3],w20 - mov v9.s[3],w21 - mov v10.s[3],w22 - mov v11.s[3],w23 - b .Loop12x_ctr32_unroll - -.align 4 -.Loop12x_ctr32_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - aese v30.16b,v2.16b - aesmc v30.16b,v30.16b - aese v31.16b,v2.16b - aesmc v31.16b,v31.16b - aese v8.16b,v2.16b - aesmc v8.16b,v8.16b - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - aese v30.16b,v3.16b - aesmc v30.16b,v30.16b - aese v31.16b,v3.16b - aesmc v31.16b,v31.16b - aese v8.16b,v3.16b - aesmc v8.16b,v8.16b - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - aese v11.16b,v3.16b - aesmc v11.16b,v11.16b - ld1 {v3.4s},[x7],#16 - b.gt .Loop12x_ctr32_unroll - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - aese v30.16b,v2.16b - aesmc v30.16b,v30.16b - aese v31.16b,v2.16b - aesmc v31.16b,v31.16b - aese v8.16b,v2.16b - aesmc v8.16b,v8.16b - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 - - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - aese v30.16b,v3.16b - aesmc v30.16b,v30.16b - aese v31.16b,v3.16b - aesmc v31.16b,v31.16b - aese v8.16b,v3.16b - aesmc v8.16b,v8.16b - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - aese v11.16b,v3.16b - aesmc v11.16b,v11.16b - ld1 {v3.4s},[x7],#16 - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - add w9,w8,#1 - add w10,w8,#2 - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - add w12,w8,#3 - add w11,w8,#4 - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - add w13,w8,#5 - add w14,w8,#6 - rev w9,w9 - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - add w15,w8,#7 - add w19,w8,#8 - rev w10,w10 - rev w12,w12 - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - add w20,w8,#9 - add w21,w8,#10 - rev w11,w11 - rev w13,w13 - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - add w22,w8,#11 - add w23,w8,#12 - rev w14,w14 - rev w15,w15 - aese v30.16b,v2.16b - aesmc v30.16b,v30.16b - rev w19,w19 - rev w20,w20 - aese v31.16b,v2.16b - aesmc v31.16b,v31.16b - rev w21,w21 - rev w22,w22 - aese v8.16b,v2.16b - aesmc v8.16b,v8.16b - rev w23,w23 - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 - - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64 - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - aese v30.16b,v3.16b - aesmc v30.16b,v30.16b - aese v31.16b,v3.16b - aesmc v31.16b,v31.16b - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x0],#64 - aese v8.16b,v3.16b - aesmc v8.16b,v8.16b - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - aese v11.16b,v3.16b - aesmc v11.16b,v11.16b - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x0],#64 - ld1 {v3.4s},[x7],#16 - - mov x7, x3 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - aese v30.16b,v2.16b - aesmc v30.16b,v30.16b - aese v31.16b,v2.16b - aesmc v31.16b,v31.16b - aese v8.16b,v2.16b - aesmc v8.16b,v8.16b - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 // re-pre-load rndkey[0] - - aese v24.16b,v3.16b -.inst 0xce016084 //eor3 v4.16b,v4.16b,v1.16b,v24.16b - orr v24.16b,v0.16b,v0.16b - aese v25.16b,v3.16b -.inst 0xce0164a5 //eor3 v5.16b,v5.16b,v1.16b,v25.16b - orr v25.16b,v0.16b,v0.16b - aese v26.16b,v3.16b -.inst 0xce0168c6 //eor3 v6.16b,v6.16b,v1.16b,v26.16b - orr v26.16b,v0.16b,v0.16b - aese v27.16b,v3.16b -.inst 0xce016ce7 //eor3 v7.16b,v7.16b,v1.16b,v27.16b - orr v27.16b,v0.16b,v0.16b - aese v28.16b,v3.16b -.inst 0xce017210 //eor3 v16.16b,v16.16b,v1.16b,v28.16b - orr v28.16b,v0.16b,v0.16b - aese v29.16b,v3.16b -.inst 0xce017631 //eor3 v17.16b,v17.16b,v1.16b,v29.16b - orr v29.16b,v0.16b,v0.16b - aese v30.16b,v3.16b -.inst 0xce017a52 //eor3 v18.16b,v18.16b,v1.16b,v30.16b - orr v30.16b,v0.16b,v0.16b - aese v31.16b,v3.16b -.inst 0xce017e73 //eor3 v19.16b,v19.16b,v1.16b,v31.16b - orr v31.16b,v0.16b,v0.16b - aese v8.16b,v3.16b -.inst 0xce012294 //eor3 v20.16b,v20.16b,v1.16b,v8.16b - orr v8.16b,v0.16b,v0.16b - aese v9.16b,v3.16b -.inst 0xce0126b5 //eor3 v21.16b,v21.16b,v1.16b,v9.16b - orr v9.16b,v0.16b,v0.16b - aese v10.16b,v3.16b -.inst 0xce012ad6 //eor3 v22.16b,v22.16b,v1.16b,v10.16b - orr v10.16b,v0.16b,v0.16b - aese v11.16b,v3.16b -.inst 0xce012ef7 //eor3 v23.16b,v23.16b,v1.16b,v11.16b - orr v11.16b,v0.16b,v0.16b - ld1 {v3.4s},[x7],#16 // re-pre-load rndkey[1] - - mov v24.s[3],w9 - mov v25.s[3],w10 - mov v26.s[3],w12 - mov v27.s[3],w11 - st1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 - mov v28.s[3],w13 - mov v29.s[3],w14 - mov v30.s[3],w15 - mov v31.s[3],w19 - st1 {v16.16b,v17.16b,v18.16b,v19.16b},[x1],#64 - mov v8.s[3],w20 - mov v9.s[3],w21 - mov v10.s[3],w22 - mov v11.s[3],w23 - st1 {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64 - - mov w6,w5 - - add w8,w8,#12 - subs x2,x2,#12 - b.hs .Loop12x_ctr32_unroll - - // pop regs from stack when 12 data chunks are interleaved - ldp d10,d11,[sp],#32 - ldp d8,d9,[sp],#32 - ldp x23,x24,[sp],#16 - ldp x21,x22,[sp],#16 - ldp x19,x20,[sp],#16 - - add x2,x2,#12 - cbz x2,.Lctr32_done_unroll - sub w8,w8,#12 - - cmp x2,#2 - b.ls .Lctr32_tail_unroll - - cmp x2,#6 - sub x2,x2,#3 // bias - add w8,w8,#3 - b.lo .Loop3x_ctr32_unroll - - sub x2,x2,#3 - add w8,w8,#3 - b.lo .Loop6x_ctr32_unroll - -.align 4 -.Loop6x_ctr32_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - ld1 {v3.4s},[x7],#16 - b.gt .Loop6x_ctr32_unroll - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - ld1 {v2.4s},[x7],#16 - - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - ld1 {v3.4s},[x7],#16 - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - add w9,w8,#1 - add w10,w8,#2 - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - add w12,w8,#3 - add w11,w8,#4 - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - add w13,w8,#5 - add w14,w8,#6 - rev w9,w9 - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - rev w10,w10 - rev w12,w12 - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - rev w11,w11 - rev w13,w13 - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - rev w14,w14 - ld1 {v2.4s},[x7],#16 - - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64 - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - aese v27.16b,v3.16b - aesmc v27.16b,v27.16b - ld1 {v16.16b,v17.16b},[x0],#32 - aese v28.16b,v3.16b - aesmc v28.16b,v28.16b - aese v29.16b,v3.16b - aesmc v29.16b,v29.16b - ld1 {v3.4s},[x7],#16 - - mov x7, x3 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - aese v27.16b,v2.16b - aesmc v27.16b,v27.16b - aese v28.16b,v2.16b - aesmc v28.16b,v28.16b - aese v29.16b,v2.16b - aesmc v29.16b,v29.16b - ld1 {v2.4s},[x7],#16 // re-pre-load rndkey[0] - - aese v24.16b,v3.16b -.inst 0xce016084 //eor3 v4.16b,v4.16b,v1.16b,v24.16b - aese v25.16b,v3.16b -.inst 0xce0164a5 //eor3 v5.16b,v5.16b,v1.16b,v25.16b - aese v26.16b,v3.16b -.inst 0xce0168c6 //eor3 v6.16b,v6.16b,v1.16b,v26.16b - aese v27.16b,v3.16b -.inst 0xce016ce7 //eor3 v7.16b,v7.16b,v1.16b,v27.16b - aese v28.16b,v3.16b -.inst 0xce017210 //eor3 v16.16b,v16.16b,v1.16b,v28.16b - aese v29.16b,v3.16b -.inst 0xce017631 //eor3 v17.16b,v17.16b,v1.16b,v29.16b - ld1 {v3.4s},[x7],#16 // re-pre-load rndkey[1] - - orr v24.16b,v0.16b,v0.16b - orr v25.16b,v0.16b,v0.16b - orr v26.16b,v0.16b,v0.16b - orr v27.16b,v0.16b,v0.16b - orr v28.16b,v0.16b,v0.16b - orr v29.16b,v0.16b,v0.16b - - mov v24.s[3],w9 - mov v25.s[3],w10 - st1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 - mov v26.s[3],w12 - mov v27.s[3],w11 - st1 {v16.16b,v17.16b},[x1],#32 - mov v28.s[3],w13 - mov v29.s[3],w14 - - cbz x2,.Lctr32_done_unroll - mov w6,w5 - - cmp x2,#2 - b.ls .Lctr32_tail_unroll - - sub x2,x2,#3 // bias - add w8,w8,#3 - b .Loop3x_ctr32_unroll - -.align 4 -.Loop3x_ctr32_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - aese v26.16b,v3.16b - aesmc v26.16b,v26.16b - ld1 {v3.4s},[x7],#16 - b.gt .Loop3x_ctr32_unroll - - aese v24.16b,v2.16b - aesmc v9.16b,v24.16b - aese v25.16b,v2.16b - aesmc v10.16b,v25.16b - ld1 {v4.16b,v5.16b,v6.16b},[x0],#48 - orr v24.16b,v0.16b,v0.16b - aese v26.16b,v2.16b - aesmc v26.16b,v26.16b - ld1 {v2.4s},[x7],#16 - orr v25.16b,v0.16b,v0.16b - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - aese v26.16b,v3.16b - aesmc v11.16b,v26.16b - ld1 {v3.4s},[x7],#16 - orr v26.16b,v0.16b,v0.16b - add w9,w8,#1 - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - add w10,w8,#2 - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - ld1 {v2.4s},[x7],#16 - add w8,w8,#3 - aese v9.16b,v3.16b - aesmc v9.16b,v9.16b - aese v10.16b,v3.16b - aesmc v10.16b,v10.16b - - rev w9,w9 - aese v11.16b,v3.16b - aesmc v11.16b,v11.16b - ld1 {v3.4s},[x7],#16 - mov v24.s[3], w9 - mov x7,x3 - rev w10,w10 - aese v9.16b,v2.16b - aesmc v9.16b,v9.16b - - aese v10.16b,v2.16b - aesmc v10.16b,v10.16b - mov v25.s[3], w10 - rev w12,w8 - aese v11.16b,v2.16b - aesmc v11.16b,v11.16b - mov v26.s[3], w12 - - aese v9.16b,v3.16b - aese v10.16b,v3.16b - aese v11.16b,v3.16b - -.inst 0xce012484 //eor3 v4.16b,v4.16b,v1.16b,v9.16b - ld1 {v2.4s},[x7],#16 // re-pre-load rndkey[0] -.inst 0xce0128a5 //eor3 v5.16b,v5.16b,v1.16b,v10.16b - mov w6,w5 -.inst 0xce012cc6 //eor3 v6.16b,v6.16b,v1.16b,v11.16b - ld1 {v3.4s},[x7],#16 // re-pre-load rndkey[1] - st1 {v4.16b,v5.16b,v6.16b},[x1],#48 - - cbz x2,.Lctr32_done_unroll - -.Lctr32_tail_unroll: - cmp x2,#1 - b.eq .Lctr32_tail_1_unroll - -.Lctr32_tail_2_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - ld1 {v3.4s},[x7],#16 - b.gt .Lctr32_tail_2_unroll - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - ld1 {v2.4s},[x7],#16 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - ld1 {v3.4s},[x7],#16 - ld1 {v4.16b,v5.16b},[x0],#32 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - ld1 {v2.4s},[x7],#16 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - aese v25.16b,v3.16b - aesmc v25.16b,v25.16b - ld1 {v3.4s},[x7],#16 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v25.16b,v2.16b - aesmc v25.16b,v25.16b - aese v24.16b,v3.16b - aese v25.16b,v3.16b - -.inst 0xce016084 //eor3 v4.16b,v4.16b,v1.16b,v24.16b -.inst 0xce0164a5 //eor3 v5.16b,v5.16b,v1.16b,v25.16b - st1 {v4.16b,v5.16b},[x1],#32 - b .Lctr32_done_unroll - -.Lctr32_tail_1_unroll: - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - ld1 {v2.4s},[x7],#16 - subs w6,w6,#2 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - ld1 {v3.4s},[x7],#16 - b.gt .Lctr32_tail_1_unroll - - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - ld1 {v2.4s},[x7],#16 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - ld1 {v3.4s},[x7],#16 - ld1 {v4.16b},[x0] - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - ld1 {v2.4s},[x7],#16 - aese v24.16b,v3.16b - aesmc v24.16b,v24.16b - ld1 {v3.4s},[x7],#16 - aese v24.16b,v2.16b - aesmc v24.16b,v24.16b - aese v24.16b,v3.16b - -.inst 0xce016084 //eor3 v4.16b,v4.16b,v1.16b,v24.16b - st1 {v4.16b},[x1],#16 - -.Lctr32_done_unroll: - ldp d8,d9,[sp, #16] - ldp d10,d11,[sp, #32] - ldp d12,d13,[sp, #48] - ldp d15,d16,[sp, #64] - ldr x29,[sp],#80 - ret -.size aes_v8_ctr32_encrypt_blocks_unroll12_eor3,.-aes_v8_ctr32_encrypt_blocks_unroll12_eor3 .globl aes_v8_ctr32_encrypt_blocks .type aes_v8_ctr32_encrypt_blocks,%function .align 5 aes_v8_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 ldr w5,[x3,#240] ldr w8, [x4, #12] -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ ld1 {v0.16b},[x4] #else ld1 {v0.4s},[x4] @@ -2259,7 +1527,7 @@ aes_v8_ctr32_encrypt_blocks: add x7,x3,#32 mov w6,w5 csel x12,xzr,x12,lo -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev w8, w8 #endif orr v1.16b,v0.16b,v0.16b @@ -2586,16 +1854,15 @@ aes_v8_ctr32_encrypt_blocks: .type aes_v8_xts_encrypt,%function .align 5 aes_v8_xts_encrypt: - AARCH64_VALID_CALL_TARGET cmp x2,#16 // Original input data size bigger than 16, jump to big size processing. b.ne .Lxts_enc_big_size // Encrypt the iv with key2, as the first XEX iv. ldr w6,[x4,#240] - ld1 {v0.4s},[x4],#16 + ld1 {v0.16b},[x4],#16 ld1 {v6.16b},[x5] sub w6,w6,#2 - ld1 {v1.4s},[x4],#16 + ld1 {v1.16b},[x4],#16 .Loop_enc_iv_enc: aese v6.16b,v0.16b @@ -3195,9 +2462,9 @@ aes_v8_xts_encrypt: // Encrypt the composite block to get the last second encrypted text block ldr w6,[x3,#240] // load key schedule... - ld1 {v0.4s},[x3],#16 + ld1 {v0.16b},[x3],#16 sub w6,w6,#2 - ld1 {v1.4s},[x3],#16 // load key schedule... + ld1 {v1.16b},[x3],#16 // load key schedule... .Loop_final_enc: aese v26.16b,v0.16b aesmc v26.16b,v26.16b @@ -3228,16 +2495,15 @@ aes_v8_xts_encrypt: .type aes_v8_xts_decrypt,%function .align 5 aes_v8_xts_decrypt: - AARCH64_VALID_CALL_TARGET cmp x2,#16 // Original input data size bigger than 16, jump to big size processing. b.ne .Lxts_dec_big_size // Encrypt the iv with key2, as the first XEX iv. ldr w6,[x4,#240] - ld1 {v0.4s},[x4],#16 + ld1 {v0.16b},[x4],#16 ld1 {v6.16b},[x5] sub w6,w6,#2 - ld1 {v1.4s},[x4],#16 + ld1 {v1.16b},[x4],#16 .Loop_dec_small_iv_enc: aese v6.16b,v0.16b @@ -3315,10 +2581,10 @@ aes_v8_xts_decrypt: // Encrypt the iv with key2, as the first XEX iv ldr w6,[x4,#240] - ld1 {v0.4s},[x4],#16 + ld1 {v0.16b},[x4],#16 ld1 {v6.16b},[x5] sub w6,w6,#2 - ld1 {v1.4s},[x4],#16 + ld1 {v1.16b},[x4],#16 .Loop_dec_iv_enc: aese v6.16b,v0.16b @@ -3658,7 +2924,7 @@ aes_v8_xts_decrypt: st1 {v30.16b,v31.16b},[x1],#32 b.eq .Lxts_dec_abort - ld1 {v0.16b},[x0],#16 + ld1 {v0.4s},[x0],#16 b .Lxts_done .align 4 .Lxts_outer_dec_tail: @@ -3836,9 +3102,9 @@ aes_v8_xts_decrypt: // Processing the last two blocks with cipher stealing. mov x7,x3 cbnz x2,.Lxts_dec_1st_done - ld1 {v0.16b},[x0],#16 + ld1 {v0.4s},[x0],#16 - // Decrypt the last second block to get the last plain text block + // Decrypt the last secod block to get the last plain text block .Lxts_dec_1st_done: eor v26.16b,v0.16b,v8.16b ldr w6,[x3,#240] @@ -3881,9 +3147,9 @@ aes_v8_xts_decrypt: // Decrypt the composite block to get the last second plain text block ldr w6,[x7,#240] - ld1 {v0.4s},[x7],#16 + ld1 {v0.16b},[x7],#16 sub w6,w6,#2 - ld1 {v1.4s},[x7],#16 + ld1 {v1.16b},[x7],#16 .Loop_final_dec: aesd v26.16b,v0.16b aesimc v26.16b,v26.16b diff --git a/openssl/src/crypto/aes/gen/linux_arm64/bsaes-armv8.S b/openssl/src/crypto/aes/gen/linux_arm64/bsaes-armv8.S deleted file mode 100644 index c550525fd..000000000 --- a/openssl/src/crypto/aes/gen/linux_arm64/bsaes-armv8.S +++ /dev/null @@ -1,2347 +0,0 @@ -// Copyright 2021-2024 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the OpenSSL license (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// ==================================================================== -// Written by Ben Avison for the OpenSSL -// project. Rights for redistribution and usage in source and binary -// forms are granted according to the OpenSSL license. -// ==================================================================== -// -// This implementation is a translation of bsaes-armv7 for AArch64. -// No attempt has been made to carry across the build switches for -// kernel targets, since the Linux kernel crypto support has moved on -// from when it was based on OpenSSL. - -// A lot of hand-scheduling has been performed. Consequently, this code -// doesn't factor out neatly into macros in the same way that the -// AArch32 version did, and there is little to be gained by wrapping it -// up in Perl, and it is presented as pure assembly. - - -#include "crypto/arm_arch.h" - -.text - - - - - -.type _bsaes_decrypt8,%function -.align 4 -// On entry: -// x9 -> key (previously expanded using _bsaes_key_convert) -// x10 = number of rounds -// v0-v7 input data -// On exit: -// x9-x11 corrupted -// other general-purpose registers preserved -// v0-v7 output data -// v11-v15 preserved -// other SIMD registers corrupted -_bsaes_decrypt8: - ldr q8, [x9], #16 - adr x11, .LM0ISR - movi v9.16b, #0x55 - ldr q10, [x11], #16 - movi v16.16b, #0x33 - movi v17.16b, #0x0f - sub x10, x10, #1 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v8.16b - eor v2.16b, v2.16b, v8.16b - eor v4.16b, v4.16b, v8.16b - eor v3.16b, v3.16b, v8.16b - eor v5.16b, v5.16b, v8.16b - tbl v0.16b, {v0.16b}, v10.16b - tbl v1.16b, {v1.16b}, v10.16b - tbl v2.16b, {v2.16b}, v10.16b - tbl v4.16b, {v4.16b}, v10.16b - eor v6.16b, v6.16b, v8.16b - eor v7.16b, v7.16b, v8.16b - tbl v3.16b, {v3.16b}, v10.16b - tbl v5.16b, {v5.16b}, v10.16b - tbl v6.16b, {v6.16b}, v10.16b - ushr v8.2d, v0.2d, #1 - tbl v7.16b, {v7.16b}, v10.16b - ushr v10.2d, v4.2d, #1 - ushr v18.2d, v2.2d, #1 - eor v8.16b, v8.16b, v1.16b - ushr v19.2d, v6.2d, #1 - eor v10.16b, v10.16b, v5.16b - eor v18.16b, v18.16b, v3.16b - and v8.16b, v8.16b, v9.16b - eor v19.16b, v19.16b, v7.16b - and v10.16b, v10.16b, v9.16b - and v18.16b, v18.16b, v9.16b - eor v1.16b, v1.16b, v8.16b - shl v8.2d, v8.2d, #1 - and v9.16b, v19.16b, v9.16b - eor v5.16b, v5.16b, v10.16b - shl v10.2d, v10.2d, #1 - eor v3.16b, v3.16b, v18.16b - shl v18.2d, v18.2d, #1 - eor v0.16b, v0.16b, v8.16b - shl v8.2d, v9.2d, #1 - eor v7.16b, v7.16b, v9.16b - eor v4.16b, v4.16b, v10.16b - eor v2.16b, v2.16b, v18.16b - ushr v9.2d, v1.2d, #2 - eor v6.16b, v6.16b, v8.16b - ushr v8.2d, v0.2d, #2 - ushr v10.2d, v5.2d, #2 - ushr v18.2d, v4.2d, #2 - eor v9.16b, v9.16b, v3.16b - eor v8.16b, v8.16b, v2.16b - eor v10.16b, v10.16b, v7.16b - eor v18.16b, v18.16b, v6.16b - and v9.16b, v9.16b, v16.16b - and v8.16b, v8.16b, v16.16b - and v10.16b, v10.16b, v16.16b - and v16.16b, v18.16b, v16.16b - eor v3.16b, v3.16b, v9.16b - shl v9.2d, v9.2d, #2 - eor v2.16b, v2.16b, v8.16b - shl v8.2d, v8.2d, #2 - eor v7.16b, v7.16b, v10.16b - shl v10.2d, v10.2d, #2 - eor v6.16b, v6.16b, v16.16b - shl v16.2d, v16.2d, #2 - eor v1.16b, v1.16b, v9.16b - eor v0.16b, v0.16b, v8.16b - eor v5.16b, v5.16b, v10.16b - eor v4.16b, v4.16b, v16.16b - ushr v8.2d, v3.2d, #4 - ushr v9.2d, v2.2d, #4 - ushr v10.2d, v1.2d, #4 - ushr v16.2d, v0.2d, #4 - eor v8.16b, v8.16b, v7.16b - eor v9.16b, v9.16b, v6.16b - eor v10.16b, v10.16b, v5.16b - eor v16.16b, v16.16b, v4.16b - and v8.16b, v8.16b, v17.16b - and v9.16b, v9.16b, v17.16b - and v10.16b, v10.16b, v17.16b - and v16.16b, v16.16b, v17.16b - eor v7.16b, v7.16b, v8.16b - shl v8.2d, v8.2d, #4 - eor v6.16b, v6.16b, v9.16b - shl v9.2d, v9.2d, #4 - eor v5.16b, v5.16b, v10.16b - shl v10.2d, v10.2d, #4 - eor v4.16b, v4.16b, v16.16b - shl v16.2d, v16.2d, #4 - eor v3.16b, v3.16b, v8.16b - eor v2.16b, v2.16b, v9.16b - eor v1.16b, v1.16b, v10.16b - eor v0.16b, v0.16b, v16.16b - b .Ldec_sbox -.align 4 -.Ldec_loop: - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #64 - ldp q8, q9, [x9], #32 - eor v0.16b, v16.16b, v0.16b - ldr q10, [x9], #16 - eor v1.16b, v17.16b, v1.16b - ldr q16, [x9], #16 - eor v2.16b, v18.16b, v2.16b - eor v3.16b, v19.16b, v3.16b - eor v4.16b, v8.16b, v4.16b - eor v5.16b, v9.16b, v5.16b - eor v6.16b, v10.16b, v6.16b - eor v7.16b, v16.16b, v7.16b - tbl v0.16b, {v0.16b}, v28.16b - tbl v1.16b, {v1.16b}, v28.16b - tbl v2.16b, {v2.16b}, v28.16b - tbl v3.16b, {v3.16b}, v28.16b - tbl v4.16b, {v4.16b}, v28.16b - tbl v5.16b, {v5.16b}, v28.16b - tbl v6.16b, {v6.16b}, v28.16b - tbl v7.16b, {v7.16b}, v28.16b -.Ldec_sbox: - eor v1.16b, v1.16b, v4.16b - eor v3.16b, v3.16b, v4.16b - subs x10, x10, #1 - eor v4.16b, v4.16b, v7.16b - eor v2.16b, v2.16b, v7.16b - eor v1.16b, v1.16b, v6.16b - eor v6.16b, v6.16b, v4.16b - eor v2.16b, v2.16b, v5.16b - eor v0.16b, v0.16b, v1.16b - eor v7.16b, v7.16b, v6.16b - eor v8.16b, v6.16b, v2.16b - and v9.16b, v4.16b, v6.16b - eor v10.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v16.16b, v7.16b, v4.16b - eor v17.16b, v4.16b, v0.16b - and v18.16b, v0.16b, v2.16b - eor v19.16b, v7.16b, v4.16b - eor v1.16b, v1.16b, v3.16b - eor v20.16b, v3.16b, v0.16b - eor v21.16b, v5.16b, v2.16b - eor v22.16b, v3.16b, v7.16b - and v8.16b, v17.16b, v8.16b - orr v17.16b, v3.16b, v5.16b - eor v23.16b, v1.16b, v6.16b - eor v24.16b, v20.16b, v16.16b - eor v25.16b, v1.16b, v5.16b - orr v26.16b, v20.16b, v21.16b - and v20.16b, v20.16b, v21.16b - and v27.16b, v7.16b, v1.16b - eor v21.16b, v21.16b, v23.16b - orr v28.16b, v16.16b, v23.16b - orr v29.16b, v22.16b, v25.16b - eor v26.16b, v26.16b, v8.16b - and v16.16b, v16.16b, v23.16b - and v22.16b, v22.16b, v25.16b - and v21.16b, v24.16b, v21.16b - eor v8.16b, v28.16b, v8.16b - eor v23.16b, v5.16b, v2.16b - eor v24.16b, v1.16b, v6.16b - eor v16.16b, v16.16b, v22.16b - eor v22.16b, v3.16b, v0.16b - eor v25.16b, v29.16b, v21.16b - eor v21.16b, v26.16b, v21.16b - eor v8.16b, v8.16b, v20.16b - eor v26.16b, v23.16b, v24.16b - eor v16.16b, v16.16b, v20.16b - eor v28.16b, v22.16b, v19.16b - eor v20.16b, v25.16b, v20.16b - eor v9.16b, v21.16b, v9.16b - eor v8.16b, v8.16b, v18.16b - eor v18.16b, v5.16b, v1.16b - eor v21.16b, v16.16b, v17.16b - eor v16.16b, v16.16b, v17.16b - eor v17.16b, v20.16b, v27.16b - eor v20.16b, v3.16b, v7.16b - eor v25.16b, v9.16b, v8.16b - eor v27.16b, v0.16b, v4.16b - and v29.16b, v9.16b, v17.16b - eor v30.16b, v8.16b, v29.16b - eor v31.16b, v21.16b, v29.16b - eor v29.16b, v21.16b, v29.16b - bsl v30.16b, v17.16b, v21.16b - bsl v31.16b, v9.16b, v8.16b - bsl v16.16b, v30.16b, v29.16b - bsl v21.16b, v29.16b, v30.16b - eor v8.16b, v31.16b, v30.16b - and v1.16b, v1.16b, v31.16b - and v9.16b, v16.16b, v31.16b - and v6.16b, v6.16b, v30.16b - eor v16.16b, v17.16b, v21.16b - and v4.16b, v4.16b, v30.16b - eor v17.16b, v8.16b, v30.16b - and v21.16b, v24.16b, v8.16b - eor v9.16b, v9.16b, v25.16b - and v19.16b, v19.16b, v8.16b - eor v24.16b, v30.16b, v16.16b - eor v25.16b, v30.16b, v16.16b - and v7.16b, v7.16b, v17.16b - and v10.16b, v10.16b, v16.16b - eor v29.16b, v9.16b, v16.16b - eor v30.16b, v31.16b, v9.16b - and v0.16b, v24.16b, v0.16b - and v9.16b, v18.16b, v9.16b - and v2.16b, v25.16b, v2.16b - eor v10.16b, v10.16b, v6.16b - eor v18.16b, v29.16b, v16.16b - and v5.16b, v30.16b, v5.16b - eor v24.16b, v8.16b, v29.16b - and v25.16b, v26.16b, v29.16b - and v26.16b, v28.16b, v29.16b - eor v8.16b, v8.16b, v29.16b - eor v17.16b, v17.16b, v18.16b - eor v5.16b, v1.16b, v5.16b - and v23.16b, v24.16b, v23.16b - eor v21.16b, v21.16b, v25.16b - eor v19.16b, v19.16b, v26.16b - eor v0.16b, v4.16b, v0.16b - and v3.16b, v17.16b, v3.16b - eor v1.16b, v9.16b, v1.16b - eor v9.16b, v25.16b, v23.16b - eor v5.16b, v5.16b, v21.16b - eor v2.16b, v6.16b, v2.16b - and v6.16b, v8.16b, v22.16b - eor v3.16b, v7.16b, v3.16b - and v8.16b, v20.16b, v18.16b - eor v10.16b, v10.16b, v9.16b - eor v0.16b, v0.16b, v19.16b - eor v9.16b, v1.16b, v9.16b - eor v1.16b, v2.16b, v21.16b - eor v3.16b, v3.16b, v19.16b - and v16.16b, v27.16b, v16.16b - eor v17.16b, v26.16b, v6.16b - eor v6.16b, v8.16b, v7.16b - eor v7.16b, v1.16b, v9.16b - eor v1.16b, v5.16b, v3.16b - eor v2.16b, v10.16b, v3.16b - eor v4.16b, v16.16b, v4.16b - eor v8.16b, v6.16b, v17.16b - eor v5.16b, v9.16b, v3.16b - eor v9.16b, v0.16b, v1.16b - eor v6.16b, v7.16b, v1.16b - eor v0.16b, v4.16b, v17.16b - eor v4.16b, v8.16b, v7.16b - eor v7.16b, v9.16b, v2.16b - eor v8.16b, v3.16b, v0.16b - eor v7.16b, v7.16b, v5.16b - eor v3.16b, v4.16b, v7.16b - eor v4.16b, v7.16b, v0.16b - eor v7.16b, v8.16b, v3.16b - bcc .Ldec_done - ext v8.16b, v0.16b, v0.16b, #8 - ext v9.16b, v1.16b, v1.16b, #8 - ldr q28, [x11] // load from .LISR in common case (x10 > 0) - ext v10.16b, v6.16b, v6.16b, #8 - ext v16.16b, v3.16b, v3.16b, #8 - ext v17.16b, v5.16b, v5.16b, #8 - ext v18.16b, v4.16b, v4.16b, #8 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v6.16b - eor v16.16b, v16.16b, v3.16b - eor v17.16b, v17.16b, v5.16b - ext v19.16b, v2.16b, v2.16b, #8 - ext v20.16b, v7.16b, v7.16b, #8 - eor v18.16b, v18.16b, v4.16b - eor v6.16b, v6.16b, v8.16b - eor v8.16b, v2.16b, v10.16b - eor v4.16b, v4.16b, v9.16b - eor v2.16b, v19.16b, v2.16b - eor v9.16b, v20.16b, v7.16b - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v16.16b - eor v6.16b, v6.16b, v17.16b - eor v8.16b, v8.16b, v16.16b - eor v7.16b, v7.16b, v18.16b - eor v4.16b, v4.16b, v16.16b - eor v2.16b, v3.16b, v2.16b - eor v1.16b, v1.16b, v17.16b - eor v3.16b, v5.16b, v9.16b - eor v5.16b, v8.16b, v17.16b - eor v7.16b, v7.16b, v17.16b - ext v8.16b, v0.16b, v0.16b, #12 - ext v9.16b, v6.16b, v6.16b, #12 - ext v10.16b, v4.16b, v4.16b, #12 - ext v16.16b, v1.16b, v1.16b, #12 - ext v17.16b, v5.16b, v5.16b, #12 - ext v18.16b, v7.16b, v7.16b, #12 - eor v0.16b, v0.16b, v8.16b - eor v6.16b, v6.16b, v9.16b - eor v4.16b, v4.16b, v10.16b - ext v19.16b, v2.16b, v2.16b, #12 - ext v20.16b, v3.16b, v3.16b, #12 - eor v1.16b, v1.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v7.16b, v7.16b, v18.16b - eor v2.16b, v2.16b, v19.16b - eor v16.16b, v16.16b, v0.16b - eor v3.16b, v3.16b, v20.16b - eor v17.16b, v17.16b, v4.16b - eor v10.16b, v10.16b, v6.16b - ext v0.16b, v0.16b, v0.16b, #8 - eor v9.16b, v9.16b, v1.16b - ext v1.16b, v1.16b, v1.16b, #8 - eor v8.16b, v8.16b, v3.16b - eor v16.16b, v16.16b, v3.16b - eor v18.16b, v18.16b, v5.16b - eor v19.16b, v19.16b, v7.16b - ext v21.16b, v5.16b, v5.16b, #8 - ext v5.16b, v7.16b, v7.16b, #8 - eor v7.16b, v20.16b, v2.16b - ext v4.16b, v4.16b, v4.16b, #8 - ext v20.16b, v3.16b, v3.16b, #8 - eor v17.16b, v17.16b, v3.16b - ext v2.16b, v2.16b, v2.16b, #8 - eor v3.16b, v10.16b, v3.16b - ext v10.16b, v6.16b, v6.16b, #8 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v16.16b - eor v5.16b, v5.16b, v18.16b - eor v3.16b, v3.16b, v4.16b - eor v7.16b, v20.16b, v7.16b - eor v6.16b, v2.16b, v19.16b - eor v4.16b, v21.16b, v17.16b - eor v2.16b, v10.16b, v9.16b - bne .Ldec_loop - ldr q28, [x11, #16]! // load from .LISRM0 on last round (x10 == 0) - b .Ldec_loop -.align 4 -.Ldec_done: - ushr v8.2d, v0.2d, #1 - movi v9.16b, #0x55 - ldr q10, [x9] - ushr v16.2d, v2.2d, #1 - movi v17.16b, #0x33 - ushr v18.2d, v6.2d, #1 - movi v19.16b, #0x0f - eor v8.16b, v8.16b, v1.16b - ushr v20.2d, v3.2d, #1 - eor v16.16b, v16.16b, v7.16b - eor v18.16b, v18.16b, v4.16b - and v8.16b, v8.16b, v9.16b - eor v20.16b, v20.16b, v5.16b - and v16.16b, v16.16b, v9.16b - and v18.16b, v18.16b, v9.16b - shl v21.2d, v8.2d, #1 - eor v1.16b, v1.16b, v8.16b - and v8.16b, v20.16b, v9.16b - eor v7.16b, v7.16b, v16.16b - shl v9.2d, v16.2d, #1 - eor v4.16b, v4.16b, v18.16b - shl v16.2d, v18.2d, #1 - eor v0.16b, v0.16b, v21.16b - shl v18.2d, v8.2d, #1 - eor v5.16b, v5.16b, v8.16b - eor v2.16b, v2.16b, v9.16b - eor v6.16b, v6.16b, v16.16b - ushr v8.2d, v1.2d, #2 - eor v3.16b, v3.16b, v18.16b - ushr v9.2d, v0.2d, #2 - ushr v16.2d, v7.2d, #2 - ushr v18.2d, v2.2d, #2 - eor v8.16b, v8.16b, v4.16b - eor v9.16b, v9.16b, v6.16b - eor v16.16b, v16.16b, v5.16b - eor v18.16b, v18.16b, v3.16b - and v8.16b, v8.16b, v17.16b - and v9.16b, v9.16b, v17.16b - and v16.16b, v16.16b, v17.16b - and v17.16b, v18.16b, v17.16b - eor v4.16b, v4.16b, v8.16b - shl v8.2d, v8.2d, #2 - eor v6.16b, v6.16b, v9.16b - shl v9.2d, v9.2d, #2 - eor v5.16b, v5.16b, v16.16b - shl v16.2d, v16.2d, #2 - eor v3.16b, v3.16b, v17.16b - shl v17.2d, v17.2d, #2 - eor v1.16b, v1.16b, v8.16b - eor v0.16b, v0.16b, v9.16b - eor v7.16b, v7.16b, v16.16b - eor v2.16b, v2.16b, v17.16b - ushr v8.2d, v4.2d, #4 - ushr v9.2d, v6.2d, #4 - ushr v16.2d, v1.2d, #4 - ushr v17.2d, v0.2d, #4 - eor v8.16b, v8.16b, v5.16b - eor v9.16b, v9.16b, v3.16b - eor v16.16b, v16.16b, v7.16b - eor v17.16b, v17.16b, v2.16b - and v8.16b, v8.16b, v19.16b - and v9.16b, v9.16b, v19.16b - and v16.16b, v16.16b, v19.16b - and v17.16b, v17.16b, v19.16b - eor v5.16b, v5.16b, v8.16b - shl v8.2d, v8.2d, #4 - eor v3.16b, v3.16b, v9.16b - shl v9.2d, v9.2d, #4 - eor v7.16b, v7.16b, v16.16b - shl v16.2d, v16.2d, #4 - eor v2.16b, v2.16b, v17.16b - shl v17.2d, v17.2d, #4 - eor v4.16b, v4.16b, v8.16b - eor v6.16b, v6.16b, v9.16b - eor v7.16b, v7.16b, v10.16b - eor v1.16b, v1.16b, v16.16b - eor v2.16b, v2.16b, v10.16b - eor v0.16b, v0.16b, v17.16b - eor v4.16b, v4.16b, v10.16b - eor v6.16b, v6.16b, v10.16b - eor v3.16b, v3.16b, v10.16b - eor v5.16b, v5.16b, v10.16b - eor v1.16b, v1.16b, v10.16b - eor v0.16b, v0.16b, v10.16b - ret -.size _bsaes_decrypt8,.-_bsaes_decrypt8 - -.type _bsaes_const,%object -.align 6 -_bsaes_const: -// InvShiftRows constants -// Used in _bsaes_decrypt8, which assumes contiguity -// .LM0ISR used with round 0 key -// .LISR used with middle round keys -// .LISRM0 used with final round key -.LM0ISR: -.quad 0x0a0e0206070b0f03, 0x0004080c0d010509 -.LISR: -.quad 0x0504070602010003, 0x0f0e0d0c080b0a09 -.LISRM0: -.quad 0x01040b0e0205080f, 0x0306090c00070a0d - -// ShiftRows constants -// Used in _bsaes_encrypt8, which assumes contiguity -// .LM0SR used with round 0 key -// .LSR used with middle round keys -// .LSRM0 used with final round key -.LM0SR: -.quad 0x0a0e02060f03070b, 0x0004080c05090d01 -.LSR: -.quad 0x0504070600030201, 0x0f0e0d0c0a09080b -.LSRM0: -.quad 0x0304090e00050a0f, 0x01060b0c0207080d - -.LM0_bigendian: -.quad 0x02060a0e03070b0f, 0x0004080c0105090d -.LM0_littleendian: -.quad 0x0105090d0004080c, 0x03070b0f02060a0e - -// Used in ossl_bsaes_ctr32_encrypt_blocks, prior to dropping into -// _bsaes_encrypt8_alt, for round 0 key in place of .LM0SR -.LREVM0SR: -.quad 0x090d01050c000408, 0x03070b0f060a0e02 - -.align 6 -.size _bsaes_const,.-_bsaes_const - -.type _bsaes_encrypt8,%function -.align 4 -// On entry: -// x9 -> key (previously expanded using _bsaes_key_convert) -// x10 = number of rounds -// v0-v7 input data -// On exit: -// x9-x11 corrupted -// other general-purpose registers preserved -// v0-v7 output data -// v11-v15 preserved -// other SIMD registers corrupted -_bsaes_encrypt8: - ldr q8, [x9], #16 - adr x11, .LM0SR - ldr q9, [x11], #16 -_bsaes_encrypt8_alt: - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v8.16b - sub x10, x10, #1 - eor v2.16b, v2.16b, v8.16b - eor v4.16b, v4.16b, v8.16b - eor v3.16b, v3.16b, v8.16b - eor v5.16b, v5.16b, v8.16b - tbl v0.16b, {v0.16b}, v9.16b - tbl v1.16b, {v1.16b}, v9.16b - tbl v2.16b, {v2.16b}, v9.16b - tbl v4.16b, {v4.16b}, v9.16b - eor v6.16b, v6.16b, v8.16b - eor v7.16b, v7.16b, v8.16b - tbl v3.16b, {v3.16b}, v9.16b - tbl v5.16b, {v5.16b}, v9.16b - tbl v6.16b, {v6.16b}, v9.16b - ushr v8.2d, v0.2d, #1 - movi v10.16b, #0x55 - tbl v7.16b, {v7.16b}, v9.16b - ushr v9.2d, v4.2d, #1 - movi v16.16b, #0x33 - ushr v17.2d, v2.2d, #1 - eor v8.16b, v8.16b, v1.16b - movi v18.16b, #0x0f - ushr v19.2d, v6.2d, #1 - eor v9.16b, v9.16b, v5.16b - eor v17.16b, v17.16b, v3.16b - and v8.16b, v8.16b, v10.16b - eor v19.16b, v19.16b, v7.16b - and v9.16b, v9.16b, v10.16b - and v17.16b, v17.16b, v10.16b - eor v1.16b, v1.16b, v8.16b - shl v8.2d, v8.2d, #1 - and v10.16b, v19.16b, v10.16b - eor v5.16b, v5.16b, v9.16b - shl v9.2d, v9.2d, #1 - eor v3.16b, v3.16b, v17.16b - shl v17.2d, v17.2d, #1 - eor v0.16b, v0.16b, v8.16b - shl v8.2d, v10.2d, #1 - eor v7.16b, v7.16b, v10.16b - eor v4.16b, v4.16b, v9.16b - eor v2.16b, v2.16b, v17.16b - ushr v9.2d, v1.2d, #2 - eor v6.16b, v6.16b, v8.16b - ushr v8.2d, v0.2d, #2 - ushr v10.2d, v5.2d, #2 - ushr v17.2d, v4.2d, #2 - eor v9.16b, v9.16b, v3.16b - eor v8.16b, v8.16b, v2.16b - eor v10.16b, v10.16b, v7.16b - eor v17.16b, v17.16b, v6.16b - and v9.16b, v9.16b, v16.16b - and v8.16b, v8.16b, v16.16b - and v10.16b, v10.16b, v16.16b - and v16.16b, v17.16b, v16.16b - eor v3.16b, v3.16b, v9.16b - shl v9.2d, v9.2d, #2 - eor v2.16b, v2.16b, v8.16b - shl v8.2d, v8.2d, #2 - eor v7.16b, v7.16b, v10.16b - shl v10.2d, v10.2d, #2 - eor v6.16b, v6.16b, v16.16b - shl v16.2d, v16.2d, #2 - eor v1.16b, v1.16b, v9.16b - eor v0.16b, v0.16b, v8.16b - eor v5.16b, v5.16b, v10.16b - eor v4.16b, v4.16b, v16.16b - ushr v8.2d, v3.2d, #4 - ushr v9.2d, v2.2d, #4 - ushr v10.2d, v1.2d, #4 - ushr v16.2d, v0.2d, #4 - eor v8.16b, v8.16b, v7.16b - eor v9.16b, v9.16b, v6.16b - eor v10.16b, v10.16b, v5.16b - eor v16.16b, v16.16b, v4.16b - and v8.16b, v8.16b, v18.16b - and v9.16b, v9.16b, v18.16b - and v10.16b, v10.16b, v18.16b - and v16.16b, v16.16b, v18.16b - eor v7.16b, v7.16b, v8.16b - shl v8.2d, v8.2d, #4 - eor v6.16b, v6.16b, v9.16b - shl v9.2d, v9.2d, #4 - eor v5.16b, v5.16b, v10.16b - shl v10.2d, v10.2d, #4 - eor v4.16b, v4.16b, v16.16b - shl v16.2d, v16.2d, #4 - eor v3.16b, v3.16b, v8.16b - eor v2.16b, v2.16b, v9.16b - eor v1.16b, v1.16b, v10.16b - eor v0.16b, v0.16b, v16.16b - b .Lenc_sbox -.align 4 -.Lenc_loop: - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #64 - ldp q8, q9, [x9], #32 - eor v0.16b, v16.16b, v0.16b - ldr q10, [x9], #16 - eor v1.16b, v17.16b, v1.16b - ldr q16, [x9], #16 - eor v2.16b, v18.16b, v2.16b - eor v3.16b, v19.16b, v3.16b - eor v4.16b, v8.16b, v4.16b - eor v5.16b, v9.16b, v5.16b - eor v6.16b, v10.16b, v6.16b - eor v7.16b, v16.16b, v7.16b - tbl v0.16b, {v0.16b}, v28.16b - tbl v1.16b, {v1.16b}, v28.16b - tbl v2.16b, {v2.16b}, v28.16b - tbl v3.16b, {v3.16b}, v28.16b - tbl v4.16b, {v4.16b}, v28.16b - tbl v5.16b, {v5.16b}, v28.16b - tbl v6.16b, {v6.16b}, v28.16b - tbl v7.16b, {v7.16b}, v28.16b -.Lenc_sbox: - eor v5.16b, v5.16b, v6.16b - eor v3.16b, v3.16b, v0.16b - subs x10, x10, #1 - eor v2.16b, v2.16b, v1.16b - eor v5.16b, v5.16b, v0.16b - eor v8.16b, v3.16b, v7.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v5.16b - eor v8.16b, v8.16b, v4.16b - eor v3.16b, v6.16b, v3.16b - eor v4.16b, v4.16b, v5.16b - eor v6.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v7.16b - eor v1.16b, v8.16b, v1.16b - eor v8.16b, v7.16b, v4.16b - eor v9.16b, v3.16b, v0.16b - eor v10.16b, v7.16b, v6.16b - eor v16.16b, v5.16b, v3.16b - eor v17.16b, v6.16b, v2.16b - eor v18.16b, v5.16b, v1.16b - eor v19.16b, v2.16b, v4.16b - eor v20.16b, v1.16b, v0.16b - orr v21.16b, v8.16b, v9.16b - orr v22.16b, v10.16b, v16.16b - eor v23.16b, v8.16b, v17.16b - eor v24.16b, v9.16b, v18.16b - and v19.16b, v19.16b, v20.16b - orr v20.16b, v17.16b, v18.16b - and v8.16b, v8.16b, v9.16b - and v9.16b, v17.16b, v18.16b - and v17.16b, v23.16b, v24.16b - and v10.16b, v10.16b, v16.16b - eor v16.16b, v21.16b, v19.16b - eor v18.16b, v20.16b, v19.16b - and v19.16b, v2.16b, v1.16b - and v20.16b, v6.16b, v5.16b - eor v21.16b, v22.16b, v17.16b - eor v9.16b, v9.16b, v10.16b - eor v10.16b, v16.16b, v17.16b - eor v16.16b, v18.16b, v8.16b - and v17.16b, v4.16b, v0.16b - orr v18.16b, v7.16b, v3.16b - eor v21.16b, v21.16b, v8.16b - eor v8.16b, v9.16b, v8.16b - eor v9.16b, v10.16b, v19.16b - eor v10.16b, v3.16b, v0.16b - eor v16.16b, v16.16b, v17.16b - eor v17.16b, v5.16b, v1.16b - eor v19.16b, v21.16b, v20.16b - eor v20.16b, v8.16b, v18.16b - eor v8.16b, v8.16b, v18.16b - eor v18.16b, v7.16b, v4.16b - eor v21.16b, v9.16b, v16.16b - eor v22.16b, v6.16b, v2.16b - and v23.16b, v9.16b, v19.16b - eor v24.16b, v10.16b, v17.16b - eor v25.16b, v0.16b, v1.16b - eor v26.16b, v7.16b, v6.16b - eor v27.16b, v18.16b, v22.16b - eor v28.16b, v3.16b, v5.16b - eor v29.16b, v16.16b, v23.16b - eor v30.16b, v20.16b, v23.16b - eor v23.16b, v20.16b, v23.16b - eor v31.16b, v4.16b, v2.16b - bsl v29.16b, v19.16b, v20.16b - bsl v30.16b, v9.16b, v16.16b - bsl v8.16b, v29.16b, v23.16b - bsl v20.16b, v23.16b, v29.16b - eor v9.16b, v30.16b, v29.16b - and v5.16b, v5.16b, v30.16b - and v8.16b, v8.16b, v30.16b - and v1.16b, v1.16b, v29.16b - eor v16.16b, v19.16b, v20.16b - and v2.16b, v2.16b, v29.16b - eor v19.16b, v9.16b, v29.16b - and v17.16b, v17.16b, v9.16b - eor v8.16b, v8.16b, v21.16b - and v20.16b, v22.16b, v9.16b - eor v21.16b, v29.16b, v16.16b - eor v22.16b, v29.16b, v16.16b - and v23.16b, v25.16b, v16.16b - and v6.16b, v6.16b, v19.16b - eor v25.16b, v8.16b, v16.16b - eor v29.16b, v30.16b, v8.16b - and v4.16b, v21.16b, v4.16b - and v8.16b, v28.16b, v8.16b - and v0.16b, v22.16b, v0.16b - eor v21.16b, v23.16b, v1.16b - eor v22.16b, v9.16b, v25.16b - eor v9.16b, v9.16b, v25.16b - eor v23.16b, v25.16b, v16.16b - and v3.16b, v29.16b, v3.16b - and v24.16b, v24.16b, v25.16b - and v25.16b, v27.16b, v25.16b - and v10.16b, v22.16b, v10.16b - and v9.16b, v9.16b, v18.16b - eor v18.16b, v19.16b, v23.16b - and v19.16b, v26.16b, v23.16b - eor v3.16b, v5.16b, v3.16b - eor v17.16b, v17.16b, v24.16b - eor v10.16b, v24.16b, v10.16b - and v16.16b, v31.16b, v16.16b - eor v20.16b, v20.16b, v25.16b - eor v9.16b, v25.16b, v9.16b - eor v4.16b, v2.16b, v4.16b - and v7.16b, v18.16b, v7.16b - eor v18.16b, v19.16b, v6.16b - eor v5.16b, v8.16b, v5.16b - eor v0.16b, v1.16b, v0.16b - eor v1.16b, v21.16b, v10.16b - eor v8.16b, v3.16b, v17.16b - eor v2.16b, v16.16b, v2.16b - eor v3.16b, v6.16b, v7.16b - eor v6.16b, v18.16b, v9.16b - eor v4.16b, v4.16b, v20.16b - eor v10.16b, v5.16b, v10.16b - eor v0.16b, v0.16b, v17.16b - eor v9.16b, v2.16b, v9.16b - eor v3.16b, v3.16b, v20.16b - eor v7.16b, v6.16b, v1.16b - eor v5.16b, v8.16b, v4.16b - eor v6.16b, v10.16b, v1.16b - eor v2.16b, v4.16b, v0.16b - eor v4.16b, v3.16b, v10.16b - eor v9.16b, v9.16b, v7.16b - eor v3.16b, v0.16b, v5.16b - eor v0.16b, v1.16b, v4.16b - eor v1.16b, v4.16b, v8.16b - eor v4.16b, v9.16b, v5.16b - eor v6.16b, v6.16b, v3.16b - bcc .Lenc_done - ext v8.16b, v0.16b, v0.16b, #12 - ext v9.16b, v4.16b, v4.16b, #12 - ldr q28, [x11] - ext v10.16b, v6.16b, v6.16b, #12 - ext v16.16b, v1.16b, v1.16b, #12 - ext v17.16b, v3.16b, v3.16b, #12 - ext v18.16b, v7.16b, v7.16b, #12 - eor v0.16b, v0.16b, v8.16b - eor v4.16b, v4.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - ext v19.16b, v2.16b, v2.16b, #12 - ext v20.16b, v5.16b, v5.16b, #12 - eor v1.16b, v1.16b, v16.16b - eor v3.16b, v3.16b, v17.16b - eor v7.16b, v7.16b, v18.16b - eor v2.16b, v2.16b, v19.16b - eor v16.16b, v16.16b, v0.16b - eor v5.16b, v5.16b, v20.16b - eor v17.16b, v17.16b, v6.16b - eor v10.16b, v10.16b, v4.16b - ext v0.16b, v0.16b, v0.16b, #8 - eor v9.16b, v9.16b, v1.16b - ext v1.16b, v1.16b, v1.16b, #8 - eor v8.16b, v8.16b, v5.16b - eor v16.16b, v16.16b, v5.16b - eor v18.16b, v18.16b, v3.16b - eor v19.16b, v19.16b, v7.16b - ext v3.16b, v3.16b, v3.16b, #8 - ext v7.16b, v7.16b, v7.16b, #8 - eor v20.16b, v20.16b, v2.16b - ext v6.16b, v6.16b, v6.16b, #8 - ext v21.16b, v5.16b, v5.16b, #8 - eor v17.16b, v17.16b, v5.16b - ext v2.16b, v2.16b, v2.16b, #8 - eor v10.16b, v10.16b, v5.16b - ext v22.16b, v4.16b, v4.16b, #8 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v16.16b - eor v5.16b, v7.16b, v18.16b - eor v4.16b, v3.16b, v17.16b - eor v3.16b, v6.16b, v10.16b - eor v7.16b, v21.16b, v20.16b - eor v6.16b, v2.16b, v19.16b - eor v2.16b, v22.16b, v9.16b - bne .Lenc_loop - ldr q28, [x11, #16]! // load from .LSRM0 on last round (x10 == 0) - b .Lenc_loop -.align 4 -.Lenc_done: - ushr v8.2d, v0.2d, #1 - movi v9.16b, #0x55 - ldr q10, [x9] - ushr v16.2d, v3.2d, #1 - movi v17.16b, #0x33 - ushr v18.2d, v4.2d, #1 - movi v19.16b, #0x0f - eor v8.16b, v8.16b, v1.16b - ushr v20.2d, v2.2d, #1 - eor v16.16b, v16.16b, v7.16b - eor v18.16b, v18.16b, v6.16b - and v8.16b, v8.16b, v9.16b - eor v20.16b, v20.16b, v5.16b - and v16.16b, v16.16b, v9.16b - and v18.16b, v18.16b, v9.16b - shl v21.2d, v8.2d, #1 - eor v1.16b, v1.16b, v8.16b - and v8.16b, v20.16b, v9.16b - eor v7.16b, v7.16b, v16.16b - shl v9.2d, v16.2d, #1 - eor v6.16b, v6.16b, v18.16b - shl v16.2d, v18.2d, #1 - eor v0.16b, v0.16b, v21.16b - shl v18.2d, v8.2d, #1 - eor v5.16b, v5.16b, v8.16b - eor v3.16b, v3.16b, v9.16b - eor v4.16b, v4.16b, v16.16b - ushr v8.2d, v1.2d, #2 - eor v2.16b, v2.16b, v18.16b - ushr v9.2d, v0.2d, #2 - ushr v16.2d, v7.2d, #2 - ushr v18.2d, v3.2d, #2 - eor v8.16b, v8.16b, v6.16b - eor v9.16b, v9.16b, v4.16b - eor v16.16b, v16.16b, v5.16b - eor v18.16b, v18.16b, v2.16b - and v8.16b, v8.16b, v17.16b - and v9.16b, v9.16b, v17.16b - and v16.16b, v16.16b, v17.16b - and v17.16b, v18.16b, v17.16b - eor v6.16b, v6.16b, v8.16b - shl v8.2d, v8.2d, #2 - eor v4.16b, v4.16b, v9.16b - shl v9.2d, v9.2d, #2 - eor v5.16b, v5.16b, v16.16b - shl v16.2d, v16.2d, #2 - eor v2.16b, v2.16b, v17.16b - shl v17.2d, v17.2d, #2 - eor v1.16b, v1.16b, v8.16b - eor v0.16b, v0.16b, v9.16b - eor v7.16b, v7.16b, v16.16b - eor v3.16b, v3.16b, v17.16b - ushr v8.2d, v6.2d, #4 - ushr v9.2d, v4.2d, #4 - ushr v16.2d, v1.2d, #4 - ushr v17.2d, v0.2d, #4 - eor v8.16b, v8.16b, v5.16b - eor v9.16b, v9.16b, v2.16b - eor v16.16b, v16.16b, v7.16b - eor v17.16b, v17.16b, v3.16b - and v8.16b, v8.16b, v19.16b - and v9.16b, v9.16b, v19.16b - and v16.16b, v16.16b, v19.16b - and v17.16b, v17.16b, v19.16b - eor v5.16b, v5.16b, v8.16b - shl v8.2d, v8.2d, #4 - eor v2.16b, v2.16b, v9.16b - shl v9.2d, v9.2d, #4 - eor v7.16b, v7.16b, v16.16b - shl v16.2d, v16.2d, #4 - eor v3.16b, v3.16b, v17.16b - shl v17.2d, v17.2d, #4 - eor v6.16b, v6.16b, v8.16b - eor v4.16b, v4.16b, v9.16b - eor v7.16b, v7.16b, v10.16b - eor v1.16b, v1.16b, v16.16b - eor v3.16b, v3.16b, v10.16b - eor v0.16b, v0.16b, v17.16b - eor v6.16b, v6.16b, v10.16b - eor v4.16b, v4.16b, v10.16b - eor v2.16b, v2.16b, v10.16b - eor v5.16b, v5.16b, v10.16b - eor v1.16b, v1.16b, v10.16b - eor v0.16b, v0.16b, v10.16b - ret -.size _bsaes_encrypt8,.-_bsaes_encrypt8 - -.type _bsaes_key_convert,%function -.align 4 -// On entry: -// x9 -> input key (big-endian) -// x10 = number of rounds -// x17 -> output key (native endianness) -// On exit: -// x9, x10 corrupted -// x11 -> .LM0_bigendian -// x17 -> last quadword of output key -// other general-purpose registers preserved -// v2-v6 preserved -// v7.16b[] = 0x63 -// v8-v14 preserved -// v15 = last round key (converted to native endianness) -// other SIMD registers corrupted -_bsaes_key_convert: -#ifdef __AARCH64EL__ - adr x11, .LM0_littleendian -#else - adr x11, .LM0_bigendian -#endif - ldr q0, [x9], #16 // load round 0 key - ldr q1, [x11] // .LM0 - ldr q15, [x9], #16 // load round 1 key - - movi v7.16b, #0x63 // compose .L63 - movi v16.16b, #0x01 // bit masks - movi v17.16b, #0x02 - movi v18.16b, #0x04 - movi v19.16b, #0x08 - movi v20.16b, #0x10 - movi v21.16b, #0x20 - movi v22.16b, #0x40 - movi v23.16b, #0x80 - -#ifdef __AARCH64EL__ - rev32 v0.16b, v0.16b -#endif - sub x10, x10, #1 - str q0, [x17], #16 // save round 0 key - -.align 4 -.Lkey_loop: - tbl v0.16b, {v15.16b}, v1.16b - ldr q15, [x9], #16 // load next round key - - eor v0.16b, v0.16b, v7.16b - cmtst v24.16b, v0.16b, v16.16b - cmtst v25.16b, v0.16b, v17.16b - cmtst v26.16b, v0.16b, v18.16b - cmtst v27.16b, v0.16b, v19.16b - cmtst v28.16b, v0.16b, v20.16b - cmtst v29.16b, v0.16b, v21.16b - cmtst v30.16b, v0.16b, v22.16b - cmtst v31.16b, v0.16b, v23.16b - sub x10, x10, #1 - st1 {v24.16b,v25.16b,v26.16b,v27.16b}, [x17], #64 // write bit-sliced round key - st1 {v28.16b,v29.16b,v30.16b,v31.16b}, [x17], #64 - cbnz x10, .Lkey_loop - - // don't save last round key -#ifdef __AARCH64EL__ - rev32 v15.16b, v15.16b - adr x11, .LM0_bigendian -#endif - ret -.size _bsaes_key_convert,.-_bsaes_key_convert - -.globl ossl_bsaes_cbc_encrypt -.type ossl_bsaes_cbc_encrypt,%function -.align 4 -// On entry: -// x0 -> input ciphertext -// x1 -> output plaintext -// x2 = size of ciphertext and plaintext in bytes (assumed a multiple of 16) -// x3 -> key -// x4 -> 128-bit initialisation vector (or preceding 128-bit block of ciphertext if continuing after an earlier call) -// w5 must be == 0 -// On exit: -// Output plaintext filled in -// Initialisation vector overwritten with last quadword of ciphertext -// No output registers, usual AAPCS64 register preservation -ossl_bsaes_cbc_encrypt: - AARCH64_VALID_CALL_TARGET - cmp x2, #128 - bhs .Lcbc_do_bsaes - b AES_cbc_encrypt -.Lcbc_do_bsaes: - - // it is up to the caller to make sure we are called with enc == 0 - - stp x29, x30, [sp, #-48]! - stp d8, d9, [sp, #16] - stp d10, d15, [sp, #32] - lsr x2, x2, #4 // len in 16 byte blocks - - ldr w15, [x3, #240] // get # of rounds - mov x14, sp - - // allocate the key schedule on the stack - add x17, sp, #96 - sub x17, x17, x15, lsl #7 // 128 bytes per inner round key, less 96 bytes - - // populate the key schedule - mov x9, x3 // pass key - mov x10, x15 // pass # of rounds - mov sp, x17 // sp is sp - bl _bsaes_key_convert - ldr q6, [sp] - str q15, [x17] // save last round key - eor v6.16b, v6.16b, v7.16b // fix up round 0 key (by XORing with 0x63) - str q6, [sp] - - ldr q15, [x4] // load IV - b .Lcbc_dec_loop - -.align 4 -.Lcbc_dec_loop: - subs x2, x2, #0x8 - bmi .Lcbc_dec_loop_finish - - ldr q0, [x0], #16 // load input - mov x9, sp // pass the key - ldr q1, [x0], #16 - mov x10, x15 - ldr q2, [x0], #16 - ldr q3, [x0], #16 - ldr q4, [x0], #16 - ldr q5, [x0], #16 - ldr q6, [x0], #16 - ldr q7, [x0], #-7*16 - - bl _bsaes_decrypt8 - - ldr q16, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - eor v1.16b, v1.16b, v16.16b - str q0, [x1], #16 // write output - ldr q0, [x0], #16 - str q1, [x1], #16 - ldr q1, [x0], #16 - eor v1.16b, v4.16b, v1.16b - ldr q4, [x0], #16 - eor v2.16b, v2.16b, v4.16b - eor v0.16b, v6.16b, v0.16b - ldr q4, [x0], #16 - str q0, [x1], #16 - str q1, [x1], #16 - eor v0.16b, v7.16b, v4.16b - ldr q1, [x0], #16 - str q2, [x1], #16 - ldr q2, [x0], #16 - ldr q15, [x0], #16 - str q0, [x1], #16 - eor v0.16b, v5.16b, v2.16b - eor v1.16b, v3.16b, v1.16b - str q1, [x1], #16 - str q0, [x1], #16 - - b .Lcbc_dec_loop - -.Lcbc_dec_loop_finish: - adds x2, x2, #8 - beq .Lcbc_dec_done - - ldr q0, [x0], #16 // load input - cmp x2, #2 - blo .Lcbc_dec_one - ldr q1, [x0], #16 - mov x9, sp // pass the key - mov x10, x15 - beq .Lcbc_dec_two - ldr q2, [x0], #16 - cmp x2, #4 - blo .Lcbc_dec_three - ldr q3, [x0], #16 - beq .Lcbc_dec_four - ldr q4, [x0], #16 - cmp x2, #6 - blo .Lcbc_dec_five - ldr q5, [x0], #16 - beq .Lcbc_dec_six - ldr q6, [x0], #-6*16 - - bl _bsaes_decrypt8 - - ldr q5, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q8, [x0], #16 - ldr q9, [x0], #16 - ldr q10, [x0], #16 - str q0, [x1], #16 // write output - ldr q0, [x0], #16 - eor v1.16b, v1.16b, v5.16b - ldr q5, [x0], #16 - eor v6.16b, v6.16b, v8.16b - ldr q15, [x0] - eor v4.16b, v4.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - str q1, [x1], #16 - eor v0.16b, v7.16b, v0.16b - str q6, [x1], #16 - eor v1.16b, v3.16b, v5.16b - str q4, [x1], #16 - str q2, [x1], #16 - str q0, [x1], #16 - str q1, [x1] - b .Lcbc_dec_done -.align 4 -.Lcbc_dec_six: - sub x0, x0, #0x60 - bl _bsaes_decrypt8 - ldr q3, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q5, [x0], #16 - ldr q8, [x0], #16 - ldr q9, [x0], #16 - str q0, [x1], #16 // write output - ldr q0, [x0], #16 - eor v1.16b, v1.16b, v3.16b - ldr q15, [x0] - eor v3.16b, v6.16b, v5.16b - eor v4.16b, v4.16b, v8.16b - eor v2.16b, v2.16b, v9.16b - str q1, [x1], #16 - eor v0.16b, v7.16b, v0.16b - str q3, [x1], #16 - str q4, [x1], #16 - str q2, [x1], #16 - str q0, [x1] - b .Lcbc_dec_done -.align 4 -.Lcbc_dec_five: - sub x0, x0, #0x50 - bl _bsaes_decrypt8 - ldr q3, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q5, [x0], #16 - ldr q7, [x0], #16 - ldr q8, [x0], #16 - str q0, [x1], #16 // write output - ldr q15, [x0] - eor v0.16b, v1.16b, v3.16b - eor v1.16b, v6.16b, v5.16b - eor v3.16b, v4.16b, v7.16b - str q0, [x1], #16 - eor v0.16b, v2.16b, v8.16b - str q1, [x1], #16 - str q3, [x1], #16 - str q0, [x1] - b .Lcbc_dec_done -.align 4 -.Lcbc_dec_four: - sub x0, x0, #0x40 - bl _bsaes_decrypt8 - ldr q2, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q3, [x0], #16 - ldr q5, [x0], #16 - str q0, [x1], #16 // write output - ldr q15, [x0] - eor v0.16b, v1.16b, v2.16b - eor v1.16b, v6.16b, v3.16b - eor v2.16b, v4.16b, v5.16b - str q0, [x1], #16 - str q1, [x1], #16 - str q2, [x1] - b .Lcbc_dec_done -.align 4 -.Lcbc_dec_three: - sub x0, x0, #0x30 - bl _bsaes_decrypt8 - ldr q2, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q3, [x0], #16 - ldr q15, [x0] - str q0, [x1], #16 // write output - eor v0.16b, v1.16b, v2.16b - eor v1.16b, v6.16b, v3.16b - str q0, [x1], #16 - str q1, [x1] - b .Lcbc_dec_done -.align 4 -.Lcbc_dec_two: - sub x0, x0, #0x20 - bl _bsaes_decrypt8 - ldr q2, [x0], #16 // reload input - eor v0.16b, v0.16b, v15.16b // ^= IV - ldr q15, [x0] - str q0, [x1], #16 // write output - eor v0.16b, v1.16b, v2.16b - str q0, [x1] - b .Lcbc_dec_done -.align 4 -.Lcbc_dec_one: - sub x0, x0, #0x10 - stp x1, x4, [sp, #-32]! - str x14, [sp, #16] - mov v8.16b, v15.16b - mov v15.16b, v0.16b - mov x2, x3 - bl AES_decrypt - ldr x14, [sp, #16] - ldp x1, x4, [sp], #32 - ldr q0, [x1] // load result - eor v0.16b, v0.16b, v8.16b // ^= IV - str q0, [x1] // write output - -.align 4 -.Lcbc_dec_done: - movi v0.16b, #0 - movi v1.16b, #0 -.Lcbc_dec_bzero: // wipe key schedule [if any] - stp q0, q1, [sp], #32 - cmp sp, x14 - bne .Lcbc_dec_bzero - str q15, [x4] // return IV - ldp d8, d9, [sp, #16] - ldp d10, d15, [sp, #32] - ldp x29, x30, [sp], #48 - ret -.size ossl_bsaes_cbc_encrypt,.-ossl_bsaes_cbc_encrypt - -.globl ossl_bsaes_ctr32_encrypt_blocks -.type ossl_bsaes_ctr32_encrypt_blocks,%function -.align 4 -// On entry: -// x0 -> input text (whole 16-byte blocks) -// x1 -> output text (whole 16-byte blocks) -// x2 = number of 16-byte blocks to encrypt/decrypt (> 0) -// x3 -> key -// x4 -> initial value of 128-bit counter (stored big-endian) which increments, modulo 2^32, for each block -// On exit: -// Output text filled in -// No output registers, usual AAPCS64 register preservation -ossl_bsaes_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET - cmp x2, #8 // use plain AES for - blo .Lctr_enc_short // small sizes - - stp x29, x30, [sp, #-80]! - stp d8, d9, [sp, #16] - stp d10, d11, [sp, #32] - stp d12, d13, [sp, #48] - stp d14, d15, [sp, #64] - - ldr w15, [x3, #240] // get # of rounds - mov x14, sp - - // allocate the key schedule on the stack - add x17, sp, #96 - sub x17, x17, x15, lsl #7 // 128 bytes per inner round key, less 96 bytes - - // populate the key schedule - mov x9, x3 // pass key - mov x10, x15 // pass # of rounds - mov sp, x17 // sp is sp - bl _bsaes_key_convert - eor v7.16b, v7.16b, v15.16b // fix up last round key - str q7, [x17] // save last round key - - ldr q0, [x4] // load counter - add x13, x11, #.LREVM0SR-.LM0_bigendian - ldr q4, [sp] // load round0 key - - movi v8.4s, #1 // compose 1<<96 - movi v9.16b, #0 - rev32 v15.16b, v0.16b - rev32 v0.16b, v0.16b - ext v11.16b, v9.16b, v8.16b, #4 - rev32 v4.16b, v4.16b - add v12.4s, v11.4s, v11.4s // compose 2<<96 - str q4, [sp] // save adjusted round0 key - add v13.4s, v11.4s, v12.4s // compose 3<<96 - add v14.4s, v12.4s, v12.4s // compose 4<<96 - b .Lctr_enc_loop - -.align 4 -.Lctr_enc_loop: - // Intermix prologue from _bsaes_encrypt8 to use the opportunity - // to flip byte order in 32-bit counter - - add v1.4s, v15.4s, v11.4s // +1 - add x9, sp, #0x10 // pass next round key - add v2.4s, v15.4s, v12.4s // +2 - ldr q9, [x13] // .LREVM0SR - ldr q8, [sp] // load round0 key - add v3.4s, v15.4s, v13.4s // +3 - mov x10, x15 // pass rounds - sub x11, x13, #.LREVM0SR-.LSR // pass constants - add v6.4s, v2.4s, v14.4s - add v4.4s, v15.4s, v14.4s // +4 - add v7.4s, v3.4s, v14.4s - add v15.4s, v4.4s, v14.4s // next counter - add v5.4s, v1.4s, v14.4s - - bl _bsaes_encrypt8_alt - - subs x2, x2, #8 - blo .Lctr_enc_loop_done - - ldr q16, [x0], #16 - ldr q17, [x0], #16 - eor v1.16b, v1.16b, v17.16b - ldr q17, [x0], #16 - eor v0.16b, v0.16b, v16.16b - eor v4.16b, v4.16b, v17.16b - str q0, [x1], #16 - ldr q16, [x0], #16 - str q1, [x1], #16 - mov v0.16b, v15.16b - str q4, [x1], #16 - ldr q1, [x0], #16 - eor v4.16b, v6.16b, v16.16b - eor v1.16b, v3.16b, v1.16b - ldr q3, [x0], #16 - eor v3.16b, v7.16b, v3.16b - ldr q6, [x0], #16 - eor v2.16b, v2.16b, v6.16b - ldr q6, [x0], #16 - eor v5.16b, v5.16b, v6.16b - str q4, [x1], #16 - str q1, [x1], #16 - str q3, [x1], #16 - str q2, [x1], #16 - str q5, [x1], #16 - - bne .Lctr_enc_loop - b .Lctr_enc_done - -.align 4 -.Lctr_enc_loop_done: - add x2, x2, #8 - ldr q16, [x0], #16 // load input - eor v0.16b, v0.16b, v16.16b - str q0, [x1], #16 // write output - cmp x2, #2 - blo .Lctr_enc_done - ldr q17, [x0], #16 - eor v1.16b, v1.16b, v17.16b - str q1, [x1], #16 - beq .Lctr_enc_done - ldr q18, [x0], #16 - eor v4.16b, v4.16b, v18.16b - str q4, [x1], #16 - cmp x2, #4 - blo .Lctr_enc_done - ldr q19, [x0], #16 - eor v6.16b, v6.16b, v19.16b - str q6, [x1], #16 - beq .Lctr_enc_done - ldr q20, [x0], #16 - eor v3.16b, v3.16b, v20.16b - str q3, [x1], #16 - cmp x2, #6 - blo .Lctr_enc_done - ldr q21, [x0], #16 - eor v7.16b, v7.16b, v21.16b - str q7, [x1], #16 - beq .Lctr_enc_done - ldr q22, [x0] - eor v2.16b, v2.16b, v22.16b - str q2, [x1], #16 - -.Lctr_enc_done: - movi v0.16b, #0 - movi v1.16b, #0 -.Lctr_enc_bzero: // wipe key schedule [if any] - stp q0, q1, [sp], #32 - cmp sp, x14 - bne .Lctr_enc_bzero - - ldp d8, d9, [sp, #16] - ldp d10, d11, [sp, #32] - ldp d12, d13, [sp, #48] - ldp d14, d15, [sp, #64] - ldp x29, x30, [sp], #80 - ret - -.Lctr_enc_short: - stp x29, x30, [sp, #-96]! - stp x19, x20, [sp, #16] - stp x21, x22, [sp, #32] - str x23, [sp, #48] - - mov x19, x0 // copy arguments - mov x20, x1 - mov x21, x2 - mov x22, x3 - ldr w23, [x4, #12] // load counter .LSW - ldr q1, [x4] // load whole counter value -#ifdef __AARCH64EL__ - rev w23, w23 -#endif - str q1, [sp, #80] // copy counter value - -.Lctr_enc_short_loop: - add x0, sp, #80 // input counter value - add x1, sp, #64 // output on the stack - mov x2, x22 // key - - bl AES_encrypt - - ldr q0, [x19], #16 // load input - ldr q1, [sp, #64] // load encrypted counter - add x23, x23, #1 -#ifdef __AARCH64EL__ - rev w0, w23 - str w0, [sp, #80+12] // next counter value -#else - str w23, [sp, #80+12] // next counter value -#endif - eor v0.16b, v0.16b, v1.16b - str q0, [x20], #16 // store output - subs x21, x21, #1 - bne .Lctr_enc_short_loop - - movi v0.16b, #0 - movi v1.16b, #0 - stp q0, q1, [sp, #64] - - ldr x23, [sp, #48] - ldp x21, x22, [sp, #32] - ldp x19, x20, [sp, #16] - ldp x29, x30, [sp], #96 - ret -.size ossl_bsaes_ctr32_encrypt_blocks,.-ossl_bsaes_ctr32_encrypt_blocks - -.globl ossl_bsaes_xts_encrypt -.type ossl_bsaes_xts_encrypt,%function -.align 4 -// On entry: -// x0 -> input plaintext -// x1 -> output ciphertext -// x2 -> length of text in bytes (must be at least 16) -// x3 -> key1 (used to encrypt the XORed plaintext blocks) -// x4 -> key2 (used to encrypt the initial vector to yield the initial tweak) -// x5 -> 16-byte initial vector (typically, sector number) -// On exit: -// Output ciphertext filled in -// No output registers, usual AAPCS64 register preservation -ossl_bsaes_xts_encrypt: - AARCH64_VALID_CALL_TARGET - // Stack layout: - // sp -> - // nrounds*128-96 bytes: key schedule - // x19 -> - // 16 bytes: frame record - // 4*16 bytes: tweak storage across _bsaes_encrypt8 - // 6*8 bytes: storage for 5 callee-saved general-purpose registers - // 8*8 bytes: storage for 8 callee-saved SIMD registers - stp x29, x30, [sp, #-192]! - stp x19, x20, [sp, #80] - stp x21, x22, [sp, #96] - str x23, [sp, #112] - stp d8, d9, [sp, #128] - stp d10, d11, [sp, #144] - stp d12, d13, [sp, #160] - stp d14, d15, [sp, #176] - - mov x19, sp - mov x20, x0 - mov x21, x1 - mov x22, x2 - mov x23, x3 - - // generate initial tweak - sub sp, sp, #16 - mov x0, x5 // iv[] - mov x1, sp - mov x2, x4 // key2 - bl AES_encrypt - ldr q11, [sp], #16 - - ldr w1, [x23, #240] // get # of rounds - // allocate the key schedule on the stack - add x17, sp, #96 - sub x17, x17, x1, lsl #7 // 128 bytes per inner round key, less 96 bytes - - // populate the key schedule - mov x9, x23 // pass key - mov x10, x1 // pass # of rounds - mov sp, x17 - bl _bsaes_key_convert - eor v15.16b, v15.16b, v7.16b // fix up last round key - str q15, [x17] // save last round key - - subs x22, x22, #0x80 - blo .Lxts_enc_short - b .Lxts_enc_loop - -.align 4 -.Lxts_enc_loop: - ldr q8, .Lxts_magic - mov x10, x1 // pass rounds - add x2, x19, #16 - ldr q0, [x20], #16 - sshr v1.2d, v11.2d, #63 - mov x9, sp // pass key schedule - ldr q6, .Lxts_magic+16 - add v2.2d, v11.2d, v11.2d - cmtst v3.2d, v11.2d, v6.2d - and v1.16b, v1.16b, v8.16b - ext v1.16b, v1.16b, v1.16b, #8 - and v3.16b, v3.16b, v8.16b - ldr q4, [x20], #16 - eor v12.16b, v2.16b, v1.16b - eor v1.16b, v4.16b, v12.16b - eor v0.16b, v0.16b, v11.16b - cmtst v2.2d, v12.2d, v6.2d - add v4.2d, v12.2d, v12.2d - add x0, x19, #16 - ext v3.16b, v3.16b, v3.16b, #8 - and v2.16b, v2.16b, v8.16b - eor v13.16b, v4.16b, v3.16b - ldr q3, [x20], #16 - ext v4.16b, v2.16b, v2.16b, #8 - eor v2.16b, v3.16b, v13.16b - ldr q3, [x20], #16 - add v5.2d, v13.2d, v13.2d - cmtst v7.2d, v13.2d, v6.2d - and v7.16b, v7.16b, v8.16b - ldr q9, [x20], #16 - ext v7.16b, v7.16b, v7.16b, #8 - ldr q10, [x20], #16 - eor v14.16b, v5.16b, v4.16b - ldr q16, [x20], #16 - add v4.2d, v14.2d, v14.2d - eor v3.16b, v3.16b, v14.16b - eor v15.16b, v4.16b, v7.16b - add v5.2d, v15.2d, v15.2d - ldr q7, [x20], #16 - cmtst v4.2d, v14.2d, v6.2d - and v17.16b, v4.16b, v8.16b - cmtst v18.2d, v15.2d, v6.2d - eor v4.16b, v9.16b, v15.16b - ext v9.16b, v17.16b, v17.16b, #8 - eor v9.16b, v5.16b, v9.16b - add v17.2d, v9.2d, v9.2d - and v18.16b, v18.16b, v8.16b - eor v5.16b, v10.16b, v9.16b - str q9, [x2], #16 - ext v10.16b, v18.16b, v18.16b, #8 - cmtst v9.2d, v9.2d, v6.2d - and v9.16b, v9.16b, v8.16b - eor v10.16b, v17.16b, v10.16b - cmtst v17.2d, v10.2d, v6.2d - eor v6.16b, v16.16b, v10.16b - str q10, [x2], #16 - ext v9.16b, v9.16b, v9.16b, #8 - add v10.2d, v10.2d, v10.2d - eor v9.16b, v10.16b, v9.16b - str q9, [x2], #16 - eor v7.16b, v7.16b, v9.16b - add v9.2d, v9.2d, v9.2d - and v8.16b, v17.16b, v8.16b - ext v8.16b, v8.16b, v8.16b, #8 - eor v8.16b, v9.16b, v8.16b - str q8, [x2] // next round tweak - - bl _bsaes_encrypt8 - - ldr q8, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q9, [x0], #16 - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - ldr q10, [x0], #16 - eor v3.16b, v3.16b, v15.16b - subs x22, x22, #0x80 - str q0, [x21], #16 - ldr q11, [x0] // next round tweak - str q1, [x21], #16 - eor v0.16b, v7.16b, v8.16b - eor v1.16b, v2.16b, v9.16b - str q4, [x21], #16 - eor v2.16b, v5.16b, v10.16b - str q6, [x21], #16 - str q3, [x21], #16 - str q0, [x21], #16 - str q1, [x21], #16 - str q2, [x21], #16 - bpl .Lxts_enc_loop - -.Lxts_enc_short: - adds x22, x22, #0x70 - bmi .Lxts_enc_done - - ldr q8, .Lxts_magic - sshr v1.2d, v11.2d, #63 - add v2.2d, v11.2d, v11.2d - ldr q9, .Lxts_magic+16 - subs x22, x22, #0x10 - ldr q0, [x20], #16 - and v1.16b, v1.16b, v8.16b - cmtst v3.2d, v11.2d, v9.2d - ext v1.16b, v1.16b, v1.16b, #8 - and v3.16b, v3.16b, v8.16b - eor v12.16b, v2.16b, v1.16b - ext v1.16b, v3.16b, v3.16b, #8 - add v2.2d, v12.2d, v12.2d - cmtst v3.2d, v12.2d, v9.2d - eor v13.16b, v2.16b, v1.16b - and v22.16b, v3.16b, v8.16b - bmi .Lxts_enc_1 - - ext v2.16b, v22.16b, v22.16b, #8 - add v3.2d, v13.2d, v13.2d - ldr q1, [x20], #16 - cmtst v4.2d, v13.2d, v9.2d - subs x22, x22, #0x10 - eor v14.16b, v3.16b, v2.16b - and v23.16b, v4.16b, v8.16b - bmi .Lxts_enc_2 - - ext v3.16b, v23.16b, v23.16b, #8 - add v4.2d, v14.2d, v14.2d - ldr q2, [x20], #16 - cmtst v5.2d, v14.2d, v9.2d - eor v0.16b, v0.16b, v11.16b - subs x22, x22, #0x10 - eor v15.16b, v4.16b, v3.16b - and v24.16b, v5.16b, v8.16b - bmi .Lxts_enc_3 - - ext v4.16b, v24.16b, v24.16b, #8 - add v5.2d, v15.2d, v15.2d - ldr q3, [x20], #16 - cmtst v6.2d, v15.2d, v9.2d - eor v1.16b, v1.16b, v12.16b - subs x22, x22, #0x10 - eor v16.16b, v5.16b, v4.16b - and v25.16b, v6.16b, v8.16b - bmi .Lxts_enc_4 - - ext v5.16b, v25.16b, v25.16b, #8 - add v6.2d, v16.2d, v16.2d - add x0, x19, #16 - cmtst v7.2d, v16.2d, v9.2d - ldr q4, [x20], #16 - eor v2.16b, v2.16b, v13.16b - str q16, [x0], #16 - subs x22, x22, #0x10 - eor v17.16b, v6.16b, v5.16b - and v26.16b, v7.16b, v8.16b - bmi .Lxts_enc_5 - - ext v7.16b, v26.16b, v26.16b, #8 - add v18.2d, v17.2d, v17.2d - ldr q5, [x20], #16 - eor v3.16b, v3.16b, v14.16b - str q17, [x0], #16 - subs x22, x22, #0x10 - eor v18.16b, v18.16b, v7.16b - bmi .Lxts_enc_6 - - ldr q6, [x20], #16 - eor v4.16b, v4.16b, v15.16b - eor v5.16b, v5.16b, v16.16b - str q18, [x0] // next round tweak - mov x9, sp // pass key schedule - mov x10, x1 - add x0, x19, #16 - sub x22, x22, #0x10 - eor v6.16b, v6.16b, v17.16b - - bl _bsaes_encrypt8 - - ldr q16, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q17, [x0], #16 - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - ldr q11, [x0] // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - eor v0.16b, v7.16b, v16.16b - eor v1.16b, v2.16b, v17.16b - str q4, [x21], #16 - str q6, [x21], #16 - str q3, [x21], #16 - str q0, [x21], #16 - str q1, [x21], #16 - b .Lxts_enc_done - -.align 4 -.Lxts_enc_6: - eor v4.16b, v4.16b, v15.16b - eor v5.16b, v5.16b, v16.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - ldr q16, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - ldr q11, [x0] // next round tweak - eor v3.16b, v3.16b, v15.16b - str q0, [x21], #16 - str q1, [x21], #16 - eor v0.16b, v7.16b, v16.16b - str q4, [x21], #16 - str q6, [x21], #16 - str q3, [x21], #16 - str q0, [x21], #16 - b .Lxts_enc_done - -.align 4 -.Lxts_enc_5: - eor v3.16b, v3.16b, v14.16b - eor v4.16b, v4.16b, v15.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q11, [x0] // next round tweak - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - str q0, [x21], #16 - str q1, [x21], #16 - str q4, [x21], #16 - str q6, [x21], #16 - str q3, [x21], #16 - b .Lxts_enc_done - -.align 4 -.Lxts_enc_4: - eor v2.16b, v2.16b, v13.16b - eor v3.16b, v3.16b, v14.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v4.16b, v4.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - mov v11.16b, v15.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - str q4, [x21], #16 - str q6, [x21], #16 - b .Lxts_enc_done - -.align 4 -.Lxts_enc_3: - eor v1.16b, v1.16b, v12.16b - eor v2.16b, v2.16b, v13.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v4.16b, v4.16b, v13.16b - mov v11.16b, v14.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - str q4, [x21], #16 - b .Lxts_enc_done - -.align 4 -.Lxts_enc_2: - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_encrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - mov v11.16b, v13.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - b .Lxts_enc_done - -.align 4 -.Lxts_enc_1: - eor v0.16b, v0.16b, v11.16b - sub x0, sp, #16 - sub x1, sp, #16 - mov x2, x23 - mov v13.d[0], v11.d[1] // just in case AES_encrypt corrupts top half of callee-saved SIMD registers - mov v14.d[0], v12.d[1] - str q0, [sp, #-16]! - - bl AES_encrypt - - ldr q0, [sp], #16 - trn1 v13.2d, v11.2d, v13.2d - trn1 v11.2d, v12.2d, v14.2d // next round tweak - eor v0.16b, v0.16b, v13.16b - str q0, [x21], #16 - -.Lxts_enc_done: - adds x22, x22, #0x10 - beq .Lxts_enc_ret - - sub x6, x21, #0x10 - // Penultimate plaintext block produces final ciphertext part-block - // plus remaining part of final plaintext block. Move ciphertext part - // to final position and reuse penultimate ciphertext block buffer to - // construct final plaintext block -.Lxts_enc_steal: - ldrb w0, [x20], #1 - ldrb w1, [x21, #-0x10] - strb w0, [x21, #-0x10] - strb w1, [x21], #1 - - subs x22, x22, #1 - bhi .Lxts_enc_steal - - // Finally encrypt the penultimate ciphertext block using the - // last tweak - ldr q0, [x6] - eor v0.16b, v0.16b, v11.16b - str q0, [sp, #-16]! - mov x0, sp - mov x1, sp - mov x2, x23 - mov x21, x6 - mov v13.d[0], v11.d[1] // just in case AES_encrypt corrupts top half of callee-saved SIMD registers - - bl AES_encrypt - - trn1 v11.2d, v11.2d, v13.2d - ldr q0, [sp], #16 - eor v0.16b, v0.16b, v11.16b - str q0, [x21] - -.Lxts_enc_ret: - - movi v0.16b, #0 - movi v1.16b, #0 -.Lxts_enc_bzero: // wipe key schedule - stp q0, q1, [sp], #32 - cmp sp, x19 - bne .Lxts_enc_bzero - - ldp x19, x20, [sp, #80] - ldp x21, x22, [sp, #96] - ldr x23, [sp, #112] - ldp d8, d9, [sp, #128] - ldp d10, d11, [sp, #144] - ldp d12, d13, [sp, #160] - ldp d14, d15, [sp, #176] - ldp x29, x30, [sp], #192 - ret -.size ossl_bsaes_xts_encrypt,.-ossl_bsaes_xts_encrypt - -// The assembler doesn't seem capable of de-duplicating these when expressed -// using `ldr qd,=` syntax, so assign a symbolic address -.align 5 -.Lxts_magic: -.quad 1, 0x87, 0x4000000000000000, 0x4000000000000000 - -.globl ossl_bsaes_xts_decrypt -.type ossl_bsaes_xts_decrypt,%function -.align 4 -// On entry: -// x0 -> input ciphertext -// x1 -> output plaintext -// x2 -> length of text in bytes (must be at least 16) -// x3 -> key1 (used to decrypt the XORed ciphertext blocks) -// x4 -> key2 (used to encrypt the initial vector to yield the initial tweak) -// x5 -> 16-byte initial vector (typically, sector number) -// On exit: -// Output plaintext filled in -// No output registers, usual AAPCS64 register preservation -ossl_bsaes_xts_decrypt: - AARCH64_VALID_CALL_TARGET - // Stack layout: - // sp -> - // nrounds*128-96 bytes: key schedule - // x19 -> - // 16 bytes: frame record - // 4*16 bytes: tweak storage across _bsaes_decrypt8 - // 6*8 bytes: storage for 5 callee-saved general-purpose registers - // 8*8 bytes: storage for 8 callee-saved SIMD registers - stp x29, x30, [sp, #-192]! - stp x19, x20, [sp, #80] - stp x21, x22, [sp, #96] - str x23, [sp, #112] - stp d8, d9, [sp, #128] - stp d10, d11, [sp, #144] - stp d12, d13, [sp, #160] - stp d14, d15, [sp, #176] - - mov x19, sp - mov x20, x0 - mov x21, x1 - mov x22, x2 - mov x23, x3 - - // generate initial tweak - sub sp, sp, #16 - mov x0, x5 // iv[] - mov x1, sp - mov x2, x4 // key2 - bl AES_encrypt - ldr q11, [sp], #16 - - ldr w1, [x23, #240] // get # of rounds - // allocate the key schedule on the stack - add x17, sp, #96 - sub x17, x17, x1, lsl #7 // 128 bytes per inner round key, less 96 bytes - - // populate the key schedule - mov x9, x23 // pass key - mov x10, x1 // pass # of rounds - mov sp, x17 - bl _bsaes_key_convert - ldr q6, [sp] - str q15, [x17] // save last round key - eor v6.16b, v6.16b, v7.16b // fix up round 0 key (by XORing with 0x63) - str q6, [sp] - - sub x30, x22, #0x10 - tst x22, #0xf // if not multiple of 16 - csel x22, x30, x22, ne // subtract another 16 bytes - subs x22, x22, #0x80 - - blo .Lxts_dec_short - b .Lxts_dec_loop - -.align 4 -.Lxts_dec_loop: - ldr q8, .Lxts_magic - mov x10, x1 // pass rounds - add x2, x19, #16 - ldr q0, [x20], #16 - sshr v1.2d, v11.2d, #63 - mov x9, sp // pass key schedule - ldr q6, .Lxts_magic+16 - add v2.2d, v11.2d, v11.2d - cmtst v3.2d, v11.2d, v6.2d - and v1.16b, v1.16b, v8.16b - ext v1.16b, v1.16b, v1.16b, #8 - and v3.16b, v3.16b, v8.16b - ldr q4, [x20], #16 - eor v12.16b, v2.16b, v1.16b - eor v1.16b, v4.16b, v12.16b - eor v0.16b, v0.16b, v11.16b - cmtst v2.2d, v12.2d, v6.2d - add v4.2d, v12.2d, v12.2d - add x0, x19, #16 - ext v3.16b, v3.16b, v3.16b, #8 - and v2.16b, v2.16b, v8.16b - eor v13.16b, v4.16b, v3.16b - ldr q3, [x20], #16 - ext v4.16b, v2.16b, v2.16b, #8 - eor v2.16b, v3.16b, v13.16b - ldr q3, [x20], #16 - add v5.2d, v13.2d, v13.2d - cmtst v7.2d, v13.2d, v6.2d - and v7.16b, v7.16b, v8.16b - ldr q9, [x20], #16 - ext v7.16b, v7.16b, v7.16b, #8 - ldr q10, [x20], #16 - eor v14.16b, v5.16b, v4.16b - ldr q16, [x20], #16 - add v4.2d, v14.2d, v14.2d - eor v3.16b, v3.16b, v14.16b - eor v15.16b, v4.16b, v7.16b - add v5.2d, v15.2d, v15.2d - ldr q7, [x20], #16 - cmtst v4.2d, v14.2d, v6.2d - and v17.16b, v4.16b, v8.16b - cmtst v18.2d, v15.2d, v6.2d - eor v4.16b, v9.16b, v15.16b - ext v9.16b, v17.16b, v17.16b, #8 - eor v9.16b, v5.16b, v9.16b - add v17.2d, v9.2d, v9.2d - and v18.16b, v18.16b, v8.16b - eor v5.16b, v10.16b, v9.16b - str q9, [x2], #16 - ext v10.16b, v18.16b, v18.16b, #8 - cmtst v9.2d, v9.2d, v6.2d - and v9.16b, v9.16b, v8.16b - eor v10.16b, v17.16b, v10.16b - cmtst v17.2d, v10.2d, v6.2d - eor v6.16b, v16.16b, v10.16b - str q10, [x2], #16 - ext v9.16b, v9.16b, v9.16b, #8 - add v10.2d, v10.2d, v10.2d - eor v9.16b, v10.16b, v9.16b - str q9, [x2], #16 - eor v7.16b, v7.16b, v9.16b - add v9.2d, v9.2d, v9.2d - and v8.16b, v17.16b, v8.16b - ext v8.16b, v8.16b, v8.16b, #8 - eor v8.16b, v9.16b, v8.16b - str q8, [x2] // next round tweak - - bl _bsaes_decrypt8 - - eor v6.16b, v6.16b, v13.16b - eor v0.16b, v0.16b, v11.16b - ldr q8, [x0], #16 - eor v7.16b, v7.16b, v8.16b - str q0, [x21], #16 - eor v0.16b, v1.16b, v12.16b - ldr q1, [x0], #16 - eor v1.16b, v3.16b, v1.16b - subs x22, x22, #0x80 - eor v2.16b, v2.16b, v15.16b - eor v3.16b, v4.16b, v14.16b - ldr q4, [x0], #16 - str q0, [x21], #16 - ldr q11, [x0] // next round tweak - eor v0.16b, v5.16b, v4.16b - str q6, [x21], #16 - str q3, [x21], #16 - str q2, [x21], #16 - str q7, [x21], #16 - str q1, [x21], #16 - str q0, [x21], #16 - bpl .Lxts_dec_loop - -.Lxts_dec_short: - adds x22, x22, #0x70 - bmi .Lxts_dec_done - - ldr q8, .Lxts_magic - sshr v1.2d, v11.2d, #63 - add v2.2d, v11.2d, v11.2d - ldr q9, .Lxts_magic+16 - subs x22, x22, #0x10 - ldr q0, [x20], #16 - and v1.16b, v1.16b, v8.16b - cmtst v3.2d, v11.2d, v9.2d - ext v1.16b, v1.16b, v1.16b, #8 - and v3.16b, v3.16b, v8.16b - eor v12.16b, v2.16b, v1.16b - ext v1.16b, v3.16b, v3.16b, #8 - add v2.2d, v12.2d, v12.2d - cmtst v3.2d, v12.2d, v9.2d - eor v13.16b, v2.16b, v1.16b - and v22.16b, v3.16b, v8.16b - bmi .Lxts_dec_1 - - ext v2.16b, v22.16b, v22.16b, #8 - add v3.2d, v13.2d, v13.2d - ldr q1, [x20], #16 - cmtst v4.2d, v13.2d, v9.2d - subs x22, x22, #0x10 - eor v14.16b, v3.16b, v2.16b - and v23.16b, v4.16b, v8.16b - bmi .Lxts_dec_2 - - ext v3.16b, v23.16b, v23.16b, #8 - add v4.2d, v14.2d, v14.2d - ldr q2, [x20], #16 - cmtst v5.2d, v14.2d, v9.2d - eor v0.16b, v0.16b, v11.16b - subs x22, x22, #0x10 - eor v15.16b, v4.16b, v3.16b - and v24.16b, v5.16b, v8.16b - bmi .Lxts_dec_3 - - ext v4.16b, v24.16b, v24.16b, #8 - add v5.2d, v15.2d, v15.2d - ldr q3, [x20], #16 - cmtst v6.2d, v15.2d, v9.2d - eor v1.16b, v1.16b, v12.16b - subs x22, x22, #0x10 - eor v16.16b, v5.16b, v4.16b - and v25.16b, v6.16b, v8.16b - bmi .Lxts_dec_4 - - ext v5.16b, v25.16b, v25.16b, #8 - add v6.2d, v16.2d, v16.2d - add x0, x19, #16 - cmtst v7.2d, v16.2d, v9.2d - ldr q4, [x20], #16 - eor v2.16b, v2.16b, v13.16b - str q16, [x0], #16 - subs x22, x22, #0x10 - eor v17.16b, v6.16b, v5.16b - and v26.16b, v7.16b, v8.16b - bmi .Lxts_dec_5 - - ext v7.16b, v26.16b, v26.16b, #8 - add v18.2d, v17.2d, v17.2d - ldr q5, [x20], #16 - eor v3.16b, v3.16b, v14.16b - str q17, [x0], #16 - subs x22, x22, #0x10 - eor v18.16b, v18.16b, v7.16b - bmi .Lxts_dec_6 - - ldr q6, [x20], #16 - eor v4.16b, v4.16b, v15.16b - eor v5.16b, v5.16b, v16.16b - str q18, [x0] // next round tweak - mov x9, sp // pass key schedule - mov x10, x1 - add x0, x19, #16 - sub x22, x22, #0x10 - eor v6.16b, v6.16b, v17.16b - - bl _bsaes_decrypt8 - - ldr q16, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q17, [x0], #16 - eor v6.16b, v6.16b, v13.16b - eor v4.16b, v4.16b, v14.16b - eor v2.16b, v2.16b, v15.16b - ldr q11, [x0] // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - eor v0.16b, v7.16b, v16.16b - eor v1.16b, v3.16b, v17.16b - str q6, [x21], #16 - str q4, [x21], #16 - str q2, [x21], #16 - str q0, [x21], #16 - str q1, [x21], #16 - b .Lxts_dec_done - -.align 4 -.Lxts_dec_6: - eor v4.16b, v4.16b, v15.16b - eor v5.16b, v5.16b, v16.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - ldr q16, [x0], #16 - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v6.16b, v6.16b, v13.16b - eor v4.16b, v4.16b, v14.16b - ldr q11, [x0] // next round tweak - eor v2.16b, v2.16b, v15.16b - str q0, [x21], #16 - str q1, [x21], #16 - eor v0.16b, v7.16b, v16.16b - str q6, [x21], #16 - str q4, [x21], #16 - str q2, [x21], #16 - str q0, [x21], #16 - b .Lxts_dec_done - -.align 4 -.Lxts_dec_5: - eor v3.16b, v3.16b, v14.16b - eor v4.16b, v4.16b, v15.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - ldr q11, [x0] // next round tweak - eor v6.16b, v6.16b, v13.16b - eor v4.16b, v4.16b, v14.16b - eor v2.16b, v2.16b, v15.16b - str q0, [x21], #16 - str q1, [x21], #16 - str q6, [x21], #16 - str q4, [x21], #16 - str q2, [x21], #16 - b .Lxts_dec_done - -.align 4 -.Lxts_dec_4: - eor v2.16b, v2.16b, v13.16b - eor v3.16b, v3.16b, v14.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v6.16b, v6.16b, v13.16b - eor v4.16b, v4.16b, v14.16b - mov v11.16b, v15.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - str q6, [x21], #16 - str q4, [x21], #16 - b .Lxts_dec_done - -.align 4 -.Lxts_dec_3: - eor v1.16b, v1.16b, v12.16b - eor v2.16b, v2.16b, v13.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - eor v6.16b, v6.16b, v13.16b - mov v11.16b, v14.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - str q6, [x21], #16 - b .Lxts_dec_done - -.align 4 -.Lxts_dec_2: - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - mov x9, sp // pass key schedule - mov x10, x1 // pass rounds - add x0, x19, #16 - - bl _bsaes_decrypt8 - - eor v0.16b, v0.16b, v11.16b - eor v1.16b, v1.16b, v12.16b - mov v11.16b, v13.16b // next round tweak - str q0, [x21], #16 - str q1, [x21], #16 - b .Lxts_dec_done - -.align 4 -.Lxts_dec_1: - eor v0.16b, v0.16b, v11.16b - sub x0, sp, #16 - sub x1, sp, #16 - mov x2, x23 - mov v13.d[0], v11.d[1] // just in case AES_decrypt corrupts top half of callee-saved SIMD registers - mov v14.d[0], v12.d[1] - str q0, [sp, #-16]! - - bl AES_decrypt - - ldr q0, [sp], #16 - trn1 v13.2d, v11.2d, v13.2d - trn1 v11.2d, v12.2d, v14.2d // next round tweak - eor v0.16b, v0.16b, v13.16b - str q0, [x21], #16 - -.Lxts_dec_done: - adds x22, x22, #0x10 - beq .Lxts_dec_ret - - // calculate one round of extra tweak for the stolen ciphertext - ldr q8, .Lxts_magic - sshr v6.2d, v11.2d, #63 - and v6.16b, v6.16b, v8.16b - add v12.2d, v11.2d, v11.2d - ext v6.16b, v6.16b, v6.16b, #8 - eor v12.16b, v12.16b, v6.16b - - // perform the final decryption with the last tweak value - ldr q0, [x20], #16 - eor v0.16b, v0.16b, v12.16b - str q0, [sp, #-16]! - mov x0, sp - mov x1, sp - mov x2, x23 - mov v13.d[0], v11.d[1] // just in case AES_decrypt corrupts top half of callee-saved SIMD registers - mov v14.d[0], v12.d[1] - - bl AES_decrypt - - trn1 v12.2d, v12.2d, v14.2d - trn1 v11.2d, v11.2d, v13.2d - ldr q0, [sp], #16 - eor v0.16b, v0.16b, v12.16b - str q0, [x21] - - mov x6, x21 - // Penultimate ciphertext block produces final plaintext part-block - // plus remaining part of final ciphertext block. Move plaintext part - // to final position and reuse penultimate plaintext block buffer to - // construct final ciphertext block -.Lxts_dec_steal: - ldrb w1, [x21] - ldrb w0, [x20], #1 - strb w1, [x21, #0x10] - strb w0, [x21], #1 - - subs x22, x22, #1 - bhi .Lxts_dec_steal - - // Finally decrypt the penultimate plaintext block using the - // penultimate tweak - ldr q0, [x6] - eor v0.16b, v0.16b, v11.16b - str q0, [sp, #-16]! - mov x0, sp - mov x1, sp - mov x2, x23 - mov x21, x6 - - bl AES_decrypt - - trn1 v11.2d, v11.2d, v13.2d - ldr q0, [sp], #16 - eor v0.16b, v0.16b, v11.16b - str q0, [x21] - -.Lxts_dec_ret: - - movi v0.16b, #0 - movi v1.16b, #0 -.Lxts_dec_bzero: // wipe key schedule - stp q0, q1, [sp], #32 - cmp sp, x19 - bne .Lxts_dec_bzero - - ldp x19, x20, [sp, #80] - ldp x21, x22, [sp, #96] - ldr x23, [sp, #112] - ldp d8, d9, [sp, #128] - ldp d10, d11, [sp, #144] - ldp d12, d13, [sp, #160] - ldp d14, d15, [sp, #176] - ldp x29, x30, [sp], #192 - ret -.size ossl_bsaes_xts_decrypt,.-ossl_bsaes_xts_decrypt diff --git a/openssl/src/crypto/aes/gen/linux_arm64/vpaes-armv8.S b/openssl/src/crypto/aes/gen/linux_arm64/vpaes-armv8.S index ff1747c69..4724cdfd2 100644 --- a/openssl/src/crypto/aes/gen/linux_arm64/vpaes-armv8.S +++ b/openssl/src/crypto/aes/gen/linux_arm64/vpaes-armv8.S @@ -1,5 +1,3 @@ -#include "arm_arch.h" - .text .type _vpaes_consts,%object @@ -197,7 +195,7 @@ _vpaes_encrypt_core: .type vpaes_encrypt,%function .align 4 vpaes_encrypt: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -207,7 +205,7 @@ vpaes_encrypt: st1 {v0.16b}, [x1] ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size vpaes_encrypt,.-vpaes_encrypt @@ -430,7 +428,7 @@ _vpaes_decrypt_core: .type vpaes_decrypt,%function .align 4 vpaes_decrypt: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -440,7 +438,7 @@ vpaes_decrypt: st1 {v0.16b}, [x1] ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size vpaes_decrypt,.-vpaes_decrypt @@ -604,7 +602,7 @@ _vpaes_key_preheat: .type _vpaes_schedule_core,%function .align 4 _vpaes_schedule_core: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29, x30, [sp,#-16]! add x29,sp,#0 @@ -769,7 +767,7 @@ _vpaes_schedule_core: eor v6.16b, v6.16b, v6.16b // vpxor %xmm6, %xmm6, %xmm6 eor v7.16b, v7.16b, v7.16b // vpxor %xmm7, %xmm7, %xmm7 ldp x29, x30, [sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size _vpaes_schedule_core,.-_vpaes_schedule_core @@ -982,7 +980,7 @@ _vpaes_schedule_mangle: .type vpaes_set_encrypt_key,%function .align 4 vpaes_set_encrypt_key: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -998,7 +996,7 @@ vpaes_set_encrypt_key: ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size vpaes_set_encrypt_key,.-vpaes_set_encrypt_key @@ -1006,7 +1004,7 @@ vpaes_set_encrypt_key: .type vpaes_set_decrypt_key,%function .align 4 vpaes_set_decrypt_key: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1026,18 +1024,18 @@ vpaes_set_decrypt_key: ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size vpaes_set_decrypt_key,.-vpaes_set_decrypt_key .globl vpaes_cbc_encrypt .type vpaes_cbc_encrypt,%function .align 4 vpaes_cbc_encrypt: - AARCH64_SIGN_LINK_REGISTER cbz x2, .Lcbc_abort cmp w5, #0 // check direction b.eq vpaes_cbc_decrypt +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1060,16 +1058,15 @@ vpaes_cbc_encrypt: st1 {v0.16b}, [x4] // write ivec ldp x29,x30,[sp],#16 +.inst 0xd50323bf // autiasp .Lcbc_abort: - AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_cbc_encrypt,.-vpaes_cbc_encrypt .type vpaes_cbc_decrypt,%function .align 4 vpaes_cbc_decrypt: - // Not adding AARCH64_SIGN_LINK_REGISTER here because vpaes_cbc_decrypt is jumped to - // only from vpaes_cbc_encrypt which has already signed the return address. +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1111,14 +1108,14 @@ vpaes_cbc_decrypt: ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size vpaes_cbc_decrypt,.-vpaes_cbc_decrypt .globl vpaes_ecb_encrypt .type vpaes_ecb_encrypt,%function .align 4 vpaes_ecb_encrypt: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1152,7 +1149,7 @@ vpaes_ecb_encrypt: ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size vpaes_ecb_encrypt,.-vpaes_ecb_encrypt @@ -1160,7 +1157,7 @@ vpaes_ecb_encrypt: .type vpaes_ecb_decrypt,%function .align 4 vpaes_ecb_decrypt: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1194,6 +1191,6 @@ vpaes_ecb_decrypt: ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size vpaes_ecb_decrypt,.-vpaes_ecb_decrypt diff --git a/openssl/src/crypto/aes/gen/linux_ia32/aes-586.S b/openssl/src/crypto/aes/gen/linux_ia32/aes-586.S index 9ee3fc9a5..3ca95e498 100644 --- a/openssl/src/crypto/aes/gen/linux_ia32/aes-586.S +++ b/openssl/src/crypto/aes/gen/linux_ia32/aes-586.S @@ -2,11 +2,7 @@ .type _x86_AES_encrypt_compact,@function .align 16 _x86_AES_encrypt_compact: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl %edi,20(%esp) xorl (%edi),%eax xorl 4(%edi),%ebx @@ -274,11 +270,7 @@ _x86_AES_encrypt_compact: .type _sse_AES_encrypt_compact,@function .align 16 _sse_AES_encrypt_compact: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pxor (%edi),%mm0 pxor 8(%edi),%mm4 movl 240(%edi),%esi @@ -436,11 +428,7 @@ _sse_AES_encrypt_compact: .type _x86_AES_encrypt,@function .align 16 _x86_AES_encrypt: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl %edi,20(%esp) xorl (%edi),%eax xorl 4(%edi),%ebx @@ -995,11 +983,7 @@ _x86_AES_encrypt: .align 16 AES_encrypt: .L_AES_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1063,11 +1047,7 @@ AES_encrypt: .type _x86_AES_decrypt_compact,@function .align 16 _x86_AES_decrypt_compact: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl %edi,20(%esp) xorl (%edi),%eax xorl 4(%edi),%ebx @@ -1423,11 +1403,7 @@ _x86_AES_decrypt_compact: .type _sse_AES_decrypt_compact,@function .align 16 _sse_AES_decrypt_compact: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pxor (%edi),%mm0 pxor 8(%edi),%mm4 movl 240(%edi),%esi @@ -1644,11 +1620,7 @@ _sse_AES_decrypt_compact: .type _x86_AES_decrypt,@function .align 16 _x86_AES_decrypt: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl %edi,20(%esp) xorl (%edi),%eax xorl 4(%edi),%ebx @@ -2207,11 +2179,7 @@ _x86_AES_decrypt: .align 16 AES_decrypt: .L_AES_decrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2277,11 +2245,7 @@ AES_decrypt: .align 16 AES_cbc_encrypt: .L_AES_cbc_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2813,11 +2777,7 @@ AES_cbc_encrypt: .type _x86_AES_set_encrypt_key,@function .align 16 _x86_AES_set_encrypt_key: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -3050,11 +3010,7 @@ _x86_AES_set_encrypt_key: .align 16 AES_set_encrypt_key: .L_AES_set_encrypt_key_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - call _x86_AES_set_encrypt_key ret .size AES_set_encrypt_key,.-.L_AES_set_encrypt_key_begin @@ -3063,11 +3019,7 @@ AES_set_encrypt_key: .align 16 AES_set_decrypt_key: .L_AES_set_decrypt_key_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - call _x86_AES_set_encrypt_key cmpl $0,%eax je .L054proceed diff --git a/openssl/src/crypto/aes/gen/linux_ia32/aesni-x86.S b/openssl/src/crypto/aes/gen/linux_ia32/aesni-x86.S index eb2d01b16..7174a9279 100644 --- a/openssl/src/crypto/aes/gen/linux_ia32/aesni-x86.S +++ b/openssl/src/crypto/aes/gen/linux_ia32/aesni-x86.S @@ -4,11 +4,7 @@ .align 16 aesni_encrypt: .L_aesni_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%eax movl 12(%esp),%edx movups (%eax),%xmm2 @@ -36,11 +32,7 @@ aesni_encrypt: .align 16 aesni_decrypt: .L_aesni_decrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%eax movl 12(%esp),%edx movups (%eax),%xmm2 @@ -66,11 +58,7 @@ aesni_decrypt: .type _aesni_encrypt2,@function .align 16 _aesni_encrypt2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movups (%edx),%xmm0 shll $4,%ecx movups 16(%edx),%xmm1 @@ -98,11 +86,7 @@ _aesni_encrypt2: .type _aesni_decrypt2,@function .align 16 _aesni_decrypt2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movups (%edx),%xmm0 shll $4,%ecx movups 16(%edx),%xmm1 @@ -130,11 +114,7 @@ _aesni_decrypt2: .type _aesni_encrypt3,@function .align 16 _aesni_encrypt3: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movups (%edx),%xmm0 shll $4,%ecx movups 16(%edx),%xmm1 @@ -167,11 +147,7 @@ _aesni_encrypt3: .type _aesni_decrypt3,@function .align 16 _aesni_decrypt3: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movups (%edx),%xmm0 shll $4,%ecx movups 16(%edx),%xmm1 @@ -204,11 +180,7 @@ _aesni_decrypt3: .type _aesni_encrypt4,@function .align 16 _aesni_encrypt4: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movups (%edx),%xmm0 movups 16(%edx),%xmm1 shll $4,%ecx @@ -247,11 +219,7 @@ _aesni_encrypt4: .type _aesni_decrypt4,@function .align 16 _aesni_decrypt4: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movups (%edx),%xmm0 movups 16(%edx),%xmm1 shll $4,%ecx @@ -290,11 +258,7 @@ _aesni_decrypt4: .type _aesni_encrypt6,@function .align 16 _aesni_encrypt6: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movups (%edx),%xmm0 shll $4,%ecx movups 16(%edx),%xmm1 @@ -349,11 +313,7 @@ _aesni_encrypt6: .type _aesni_decrypt6,@function .align 16 _aesni_decrypt6: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movups (%edx),%xmm0 shll $4,%ecx movups 16(%edx),%xmm1 @@ -410,11 +370,7 @@ _aesni_decrypt6: .align 16 aesni_ecb_encrypt: .L_aesni_ecb_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -650,11 +606,7 @@ aesni_ecb_encrypt: .align 16 aesni_ccm64_encrypt_blocks: .L_aesni_ccm64_encrypt_blocks_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -743,11 +695,7 @@ aesni_ccm64_encrypt_blocks: .align 16 aesni_ccm64_decrypt_blocks: .L_aesni_ccm64_decrypt_blocks_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -871,11 +819,7 @@ aesni_ccm64_decrypt_blocks: .align 16 aesni_ctr32_encrypt_blocks: .L_aesni_ctr32_encrypt_blocks_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1114,11 +1058,7 @@ aesni_ctr32_encrypt_blocks: .align 16 aesni_xts_encrypt: .L_aesni_xts_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1479,11 +1419,7 @@ aesni_xts_encrypt: .align 16 aesni_xts_decrypt: .L_aesni_xts_decrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1874,11 +1810,7 @@ aesni_xts_decrypt: .align 16 aesni_ocb_encrypt: .L_aesni_ocb_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2274,11 +2206,7 @@ aesni_ocb_encrypt: .align 16 aesni_ocb_decrypt: .L_aesni_ocb_decrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2674,11 +2602,7 @@ aesni_ocb_decrypt: .align 16 aesni_cbc_encrypt: .L_aesni_cbc_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2938,11 +2862,7 @@ aesni_cbc_encrypt: .type _aesni_set_encrypt_key,@function .align 16 _aesni_set_encrypt_key: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx testl %eax,%eax @@ -3278,11 +3198,7 @@ _aesni_set_encrypt_key: .align 16 aesni_set_encrypt_key: .L_aesni_set_encrypt_key_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%eax movl 8(%esp),%ecx movl 12(%esp),%edx @@ -3294,11 +3210,7 @@ aesni_set_encrypt_key: .align 16 aesni_set_decrypt_key: .L_aesni_set_decrypt_key_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%eax movl 8(%esp),%ecx movl 12(%esp),%edx diff --git a/openssl/src/crypto/aes/gen/linux_ia32/vpaes-x86.S b/openssl/src/crypto/aes/gen/linux_ia32/vpaes-x86.S index f3adb7054..cbee0c1d9 100644 --- a/openssl/src/crypto/aes/gen/linux_ia32/vpaes-x86.S +++ b/openssl/src/crypto/aes/gen/linux_ia32/vpaes-x86.S @@ -59,11 +59,7 @@ .type _vpaes_preheat,@function .align 16 _vpaes_preheat: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - addl (%esp),%ebp movdqa -48(%ebp),%xmm7 movdqa -16(%ebp),%xmm6 @@ -72,11 +68,7 @@ _vpaes_preheat: .type _vpaes_encrypt_core,@function .align 16 _vpaes_encrypt_core: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl $16,%ecx movl 240(%edx),%eax movdqa %xmm6,%xmm1 @@ -154,11 +146,7 @@ _vpaes_encrypt_core: .type _vpaes_decrypt_core,@function .align 16 _vpaes_decrypt_core: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - leal 608(%ebp),%ebx movl 240(%edx),%eax movdqa %xmm6,%xmm1 @@ -247,11 +235,7 @@ _vpaes_decrypt_core: .type _vpaes_schedule_core,@function .align 16 _vpaes_schedule_core: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - addl (%esp),%ebp movdqu (%esi),%xmm0 movdqa 320(%ebp),%xmm2 @@ -346,11 +330,7 @@ _vpaes_schedule_core: .type _vpaes_schedule_192_smear,@function .align 16 _vpaes_schedule_192_smear: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pshufd $128,%xmm6,%xmm1 pshufd $254,%xmm7,%xmm0 pxor %xmm1,%xmm6 @@ -363,11 +343,7 @@ _vpaes_schedule_192_smear: .type _vpaes_schedule_round,@function .align 16 _vpaes_schedule_round: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movdqa 8(%esp),%xmm2 pxor %xmm1,%xmm1 .byte 102,15,58,15,202,15 @@ -417,11 +393,7 @@ _vpaes_schedule_round: .type _vpaes_schedule_transform,@function .align 16 _vpaes_schedule_transform: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movdqa -16(%ebp),%xmm2 movdqa %xmm2,%xmm1 pandn %xmm0,%xmm1 @@ -437,11 +409,7 @@ _vpaes_schedule_transform: .type _vpaes_schedule_mangle,@function .align 16 _vpaes_schedule_mangle: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movdqa %xmm0,%xmm4 movdqa 128(%ebp),%xmm5 testl %edi,%edi @@ -503,11 +471,7 @@ _vpaes_schedule_mangle: .align 16 vpaes_set_encrypt_key: .L_vpaes_set_encrypt_key_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -541,11 +505,7 @@ vpaes_set_encrypt_key: .align 16 vpaes_set_decrypt_key: .L_vpaes_set_decrypt_key_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -584,11 +544,7 @@ vpaes_set_decrypt_key: .align 16 vpaes_encrypt: .L_vpaes_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -618,11 +574,7 @@ vpaes_encrypt: .align 16 vpaes_decrypt: .L_vpaes_decrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -652,11 +604,7 @@ vpaes_decrypt: .align 16 vpaes_cbc_encrypt: .L_vpaes_cbc_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/aes/gen/linux_loong64/vpaes-loongarch64.S b/openssl/src/crypto/aes/gen/linux_loong64/vpaes-loongarch64.S deleted file mode 100644 index 0b606c1b1..000000000 --- a/openssl/src/crypto/aes/gen/linux_loong64/vpaes-loongarch64.S +++ /dev/null @@ -1,915 +0,0 @@ - -## -## _aes_encrypt_core -## -## AES-encrypt %vr0. -## -## Inputs: -## %vr0 = input -## %vr9-%vr15 as in _vpaes_preheat -## (%a2) = scheduled keys -## -## Output in %vr0 -## Clobbers %vr1-%vr5, %r9, %r10, %r11, %t5 -## Preserves %vr6 - %vr8 so you get some local vectors -## -## -##.type _vpaes_encrypt_core -.align 4 -_vpaes_encrypt_core: -.cfi_startproc - move $r9,$r6 - li.d $r11,0x10 - ld.w $r17,$r6,240 - vori.b $vr1,$vr9,0 - la.local $r12,Lk_ipt - vld $vr2,$r12,0 # iptlo - vandn.v $vr1,$vr1,$vr0 - vld $vr5,$r9,0 # round0 key - vsrli.w $vr1,$vr1,4 - vand.v $vr0,$vr0,$vr9 - vshuf.b $vr2,$vr18,$vr2,$vr0 - vld $vr0,$r12,16 # ipthi - vshuf.b $vr0,$vr18,$vr0,$vr1 - vxor.v $vr2,$vr2,$vr5 - addi.d $r9,$r9,16 - vxor.v $vr0,$vr0,$vr2 - la.local $r10,Lk_mc_backward - b .Lenc_entry - -.align 4 -.Lenc_loop: - # middle of middle round - vori.b $vr4,$vr13,0 # 4 : sb1u - vori.b $vr0,$vr12,0 # 0 : sb1t - vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sb1u - vshuf.b $vr0,$vr18,$vr0,$vr3 # 0 = sb1t - vxor.v $vr4,$vr4,$vr5 # 4 = sb1u + k - vori.b $vr5,$vr15,0 # 4 : sb2u - vxor.v $vr0,$vr0,$vr4 # 0 = A - add.d $r12,$r11,$r10 # Lk_mc_forward[] - vld $vr1,$r12,-0x40 - vshuf.b $vr5,$vr18,$vr5,$vr2 # 4 = sb2u - vld $vr4,$r12,0 # Lk_mc_backward[] - vori.b $vr2,$vr14,0 # 2 : sb2t - vshuf.b $vr2,$vr18,$vr2,$vr3 # 2 = sb2t - vori.b $vr3,$vr0,0 # 3 = A - vxor.v $vr2,$vr5,$vr2 # 2 = 2A - vshuf.b $vr0,$vr18,$vr0,$vr1 # 0 = B - addi.d $r9,$r9,16 # next key - vxor.v $vr0,$vr0,$vr2 # 0 = 2A+B - vshuf.b $vr3,$vr18,$vr3,$vr4 # 3 = D - addi.d $r11,$r11,16 # next mc - vxor.v $vr3,$vr3,$vr0 # 3 = 2A+B+D - vshuf.b $vr0,$vr18,$vr0,$vr1 # 0 = 2B+C - andi $r11,$r11,0x30 # ... mod 4 - addi.d $r17,$r17,-1 # nr-- - vxor.v $vr0,$vr0,$vr3 # 0 = 2A+3B+C+D - -.Lenc_entry: - # top of round - vori.b $vr1,$vr9,0 # 1 : i - vori.b $vr5,$vr11,0 # 2 : a/k - vandn.v $vr1,$vr1,$vr0 # 1 = i<<4 - vsrli.w $vr1,$vr1,4 # 1 = i - vand.v $vr0,$vr0,$vr9 # 0 = k - vshuf.b $vr5,$vr18,$vr5,$vr0 # 2 = a/k - vori.b $vr3,$vr10,0 # 3 : 1/i - vxor.v $vr0,$vr0,$vr1 # 0 = j - vshuf.b $vr3,$vr18,$vr3,$vr1 # 3 = 1/i - vori.b $vr4,$vr10,0 # 4 : 1/j - vxor.v $vr3,$vr3,$vr5 # 3 = iak = 1/i + a/k - vshuf.b $vr4,$vr18,$vr4,$vr0 # 4 = 1/j - vori.b $vr2,$vr10,0 # 2 : 1/iak - vxor.v $vr4,$vr4,$vr5 # 4 = jak = 1/j + a/k - vshuf.b $vr2,$vr18,$vr2,$vr3 # 2 = 1/iak - vori.b $vr3,$vr10,0 # 3 : 1/jak - vxor.v $vr2,$vr2,$vr0 # 2 = io - vshuf.b $vr3,$vr18,$vr3,$vr4 # 3 = 1/jak - vld $vr5,$r9,0 - vxor.v $vr3,$vr3,$vr1 # 3 = jo - bnez $r17,.Lenc_loop - - # middle of last round - vld $vr4,$r10, -0x60 # 3 : sbou Lk_sbo - vld $vr0,$r10, -0x50 # 0 : sbot Lk_sbo+16 - vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbou - vxor.v $vr4,$vr4,$vr5 # 4 = sb1u + k - vshuf.b $vr0,$vr18,$vr0,$vr3 # 0 = sb1t - add.d $r12,$r11,$r10 # Lk_sr[] - vld $vr1,$r12,0x40 - vxor.v $vr0,$vr0,$vr4 # 0 = A - vshuf.b $vr0,$vr18,$vr0,$vr1 - jr $r1 -.cfi_endproc -.size _vpaes_encrypt_core,.-_vpaes_encrypt_core - -## -## Decryption core -## -## Same API as encryption core. -## -#.type _vpaes_decrypt_core,@abi-omnipotent -.align 4 -_vpaes_decrypt_core: -.cfi_startproc - move $r9,$r6 # load key - ld.w $r17,$r6,240 - vori.b $vr1,$vr9,0 - la.local $r12,Lk_dipt - vld $vr2,$r12,0 # iptlo - vandn.v $vr1,$vr1,$vr0 - move $r11,$r17 - vsrli.w $vr1,$vr1,4 - vld $vr5,$r9,0 # round0 key - slli.d $r11,$r11,4 - vand.v $vr0,$vr9,$vr0 - vshuf.b $vr2,$vr18,$vr2,$vr0 - vld $vr0,$r12,16 # ipthi - xori $r11,$r11,0x30 - la.local $r10,Lk_dsbd - vshuf.b $vr0,$vr18,$vr0,$vr1 - andi $r11,$r11,0x30 - vxor.v $vr2,$vr2,$vr5 - la.local $r12,Lk_mc_forward - vld $vr5,$r12,48 - vxor.v $vr0,$vr0,$vr2 - addi.d $r9,$r9,16 - add.d $r11,$r11,$r10 - b .Ldec_entry - -.align 4 -.Ldec_loop: -## -## Inverse mix columns -## - vld $vr4,$r10,-0x20 # 4 : sb9u - vld $vr1,$r10,-0x10 # 0 : sb9t - vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sb9u - vshuf.b $vr1,$vr18,$vr1,$vr3 # 0 = sb9t - vxor.v $vr0,$vr0,$vr4 - vld $vr4,$r10,0x0 # 4 : sbdu - vxor.v $vr0,$vr0,$vr1 # 0 = ch - vld $vr1,$r10,0x10 # 0 : sbdt - vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbdu - vshuf.b $vr0,$vr18,$vr0,$vr5 # MC ch - vshuf.b $vr1,$vr18,$vr1,$vr3 # 0 = sbdt - vxor.v $vr0,$vr0,$vr4 # 4 = ch - vld $vr4,$r10,0x20 # 4 : sbbu - vxor.v $vr0,$vr0,$vr1 # 0 = ch - vld $vr1,$r10,0x30 # 0 : sbbt - vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbbu - vshuf.b $vr0,$vr18,$vr0,$vr5 # MC ch - vshuf.b $vr1,$vr18,$vr1,$vr3 # 0 = sbbt - vxor.v $vr0,$vr0,$vr4 # 4 = ch - vld $vr4,$r10,0x40 # 4 : sbeu - vxor.v $vr0,$vr0,$vr1 # 0 = ch - vld $vr1,$r10,0x50 # 0 : sbet - vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbeu - vshuf.b $vr0,$vr18,$vr0,$vr5 # MC ch - vshuf.b $vr1,$vr18,$vr1,$vr3 # 0 = sbet - vxor.v $vr0,$vr0,$vr4 # 4 = ch - addi.d $r9,$r9, 16 # next round key - vbsrl.v $vr16,$vr5,0xc - vbsll.v $vr5,$vr5,0x4 - vor.v $vr5,$vr5,$vr16 - vxor.v $vr0,$vr0,$vr1 # 0 = ch - addi.d $r17,$r17,-1 # nr-- - -.Ldec_entry: - # top of round - vori.b $vr1,$vr9,0 # 1 : i - vandn.v $vr1,$vr1,$vr0 # 1 = i<<4 - vori.b $vr2,$vr11,0 # 2 : a/k - vsrli.w $vr1,$vr1,4 # 1 = i - vand.v $vr0,$vr0,$vr9 # 0 = k - vshuf.b $vr2,$vr18,$vr2,$vr0 # 2 = a/k - vori.b $vr3,$vr10,0 # 3 : 1/i - vxor.v $vr0,$vr0,$vr1 # 0 = j - vshuf.b $vr3,$vr18,$vr3,$vr1 # 3 = 1/i - vori.b $vr4,$vr10,0 # 4 : 1/j - vxor.v $vr3,$vr3,$vr2 # 3 = iak = 1/i + a/k - vshuf.b $vr4,$vr18,$vr4,$vr0 # 4 = 1/j - vxor.v $vr4,$vr4,$vr2 # 4 = jak = 1/j + a/k - vori.b $vr2,$vr10,0 # 2 : 1/iak - vshuf.b $vr2,$vr18,$vr2,$vr3 # 2 = 1/iak - vori.b $vr3,$vr10,0 # 3 : 1/jak - vxor.v $vr2,$vr2,$vr0 # 2 = io - vshuf.b $vr3,$vr18,$vr3,$vr4 # 3 = 1/jak - vld $vr0,$r9,0 - vxor.v $vr3,$vr3,$vr1 # 3 = jo - bnez $r17,.Ldec_loop - - # middle of last round - vld $vr4,$r10,0x60 # 3 : sbou - vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbou - vxor.v $vr4,$vr4,$vr0 # 4 = sb1u + k - vld $vr0,$r10,0x70 # 0 : sbot - vld $vr2,$r11,-0x160 # Lk_sr-.Lk_dsbd=-0x160 - vshuf.b $vr0,$vr18,$vr0,$vr3 # 0 = sb1t - vxor.v $vr0,$vr0,$vr4 # 0 = A - vshuf.b $vr0,$vr18,$vr0,$vr2 - jr $r1 -.cfi_endproc -.size _vpaes_decrypt_core,.-_vpaes_decrypt_core - -######################################################## -## ## -## AES key schedule ## -## ## -######################################################## -#.type _vpaes_schedule_core,@abi-omnipotent -.align 4 -_vpaes_schedule_core: -.cfi_startproc - # a0 = key - # a1 = size in bits - # a2 = buffer - # a3 = direction. 0=encrypt, 1=decrypt - - addi.d $r3,$r3,-48 - st.d $r1,$r3,40 - st.d $r22,$r3,32 - - bl _vpaes_preheat # load the tables - la.local $r12,Lk_rcon - vld $vr8,$r12,0 # load rcon - vld $vr0,$r4,0 # load key (unaligned) - - # input transform - vori.b $vr3,$vr0,0 - la.local $r11,Lk_ipt - bl _vpaes_schedule_transform - vori.b $vr7,$vr0,0 - - la.local $r10,Lk_sr - bnez $r7,.Lschedule_am_decrypting - - # encrypting, output zeroth round key after transform - vst $vr0,$r6,0 - b .Lschedule_go - -.Lschedule_am_decrypting: - # decrypting, output zeroth round key after shiftrows - add.d $r14,$r8,$r10 - vld $vr1,$r14,0 - vshuf.b $vr3,$vr18,$vr3,$vr1 - vst $vr3,$r6,0 - xori $r8,$r8,0x30 - -.Lschedule_go: - li.d $r18,192 - bltu $r18,$r5,.Lschedule_256 - beq $r18,$r5,.Lschedule_192 - # 128: fall though - -## -## .schedule_128 -## -## 128-bit specific part of key schedule. -## -## This schedule is really simple, because all its parts -## are accomplished by the subroutines. -## -.Lschedule_128: - li.w $r5,10 - -.Loop_schedule_128: - bl _vpaes_schedule_round - addi.w $r5,$r5,-1 - beqz $r5,.Lschedule_mangle_last - bl _vpaes_schedule_mangle - b .Loop_schedule_128 - -## -## .aes_schedule_192 -## -## 192-bit specific part of key schedule. -## -## The main body of this schedule is the same as the 128-bit -## schedule, but with more smearing. The long, high side is -## stored in %vr7 as before, and the short, low side is in -## the high bits of %vr6. -## -## This schedule is somewhat nastier, however, because each -## round produces 192 bits of key material, or 1.5 round keys. -## Therefore, on each cycle we do 2 rounds and produce 3 round -## keys. -## -.align 4 -.Lschedule_192: - vld $vr0,$r4,8 #load key part 2 - bl _vpaes_schedule_transform #input transform - vaddi.du $vr6,$vr0,0x0 #save short part - vxor.v $vr4,$vr4,$vr4 #clear 4 - vpackod.d $vr6,$vr6,$vr4 #clobber low side with zeros - li.w $r5,4 - -.Loop_schedule_192: - bl _vpaes_schedule_round - vbsrl.v $vr16,$vr6,0x8 - vbsll.v $vr0,$vr0,0x8 - vor.v $vr0,$vr0,$vr16 - - bl _vpaes_schedule_mangle # save key n - bl _vpaes_schedule_192_smear - bl _vpaes_schedule_mangle # save key n+1 - bl _vpaes_schedule_round - addi.w $r5,$r5,-1 - beqz $r5,.Lschedule_mangle_last - bl _vpaes_schedule_mangle # save key n+2 - bl _vpaes_schedule_192_smear - b .Loop_schedule_192 - -## -## .aes_schedule_256 -## -## 256-bit specific part of key schedule. -## -## The structure here is very similar to the 128-bit -## schedule, but with an additional "low side" in -## %vr6. The low side's rounds are the same as the -## high side's, except no rcon and no rotation. -## -.align 4 -.Lschedule_256: - vld $vr0,$r4,16 # load key part 2 (unaligned) - bl _vpaes_schedule_transform # input transform - addi.w $r5,$r0,7 - -.Loop_schedule_256: - bl _vpaes_schedule_mangle # output low result - vori.b $vr6,$vr0,0 # save cur_lo in vr6 - - # high round - bl _vpaes_schedule_round - addi.d $r5,$r5,-1 - beqz $r5,.Lschedule_mangle_last - bl _vpaes_schedule_mangle - - # low round. swap vr7 and vr6 - vshuf4i.w $vr0,$vr0,0xFF - vori.b $vr5,$vr7,0 - vori.b $vr7,$vr6,0 - bl _vpaes_schedule_low_round - vori.b $vr7,$vr5,0 - - b .Loop_schedule_256 - - -## -## .aes_schedule_mangle_last -## -## Mangler for last round of key schedule -## Mangles %vr0 -## when encrypting, outputs out(%vr0) ^ 63 -## when decrypting, outputs unskew(%vr0) -## -## Always called right before return... jumps to cleanup and exits -## -.align 4 -.Lschedule_mangle_last: - # schedule last round key from vr0 - la.local $r11,Lk_deskew # prepare to deskew - bnez $r7,.Lschedule_mangle_last_dec - - # encrypting - add.d $r12,$r8,$r10 - vld $vr1,$r12,0 - vshuf.b $vr0,$vr18,$vr0,$vr1 # output permute - la.local $r11,Lk_opt # prepare to output transform - addi.d $r6,$r6,32 - -.Lschedule_mangle_last_dec: - addi.d $r6,$r6,-16 - la.local $r12,Lk_s63 - vld $vr16,$r12,0 - vxor.v $vr0,$vr0,$vr16 - bl _vpaes_schedule_transform # output transform - vst $vr0,$r6,0 # save last key - - # cleanup - vxor.v $vr0,$vr0,$vr0 - vxor.v $vr1,$vr1,$vr1 - vxor.v $vr2,$vr2,$vr2 - vxor.v $vr3,$vr3,$vr3 - vxor.v $vr4,$vr4,$vr4 - vxor.v $vr5,$vr5,$vr5 - vxor.v $vr6,$vr6,$vr6 - vxor.v $vr7,$vr7,$vr7 - ld.d $r1,$r3,40 - ld.d $r22,$r3,32 - addi.d $r3,$r3,48 - jr $r1 -.cfi_endproc -.size _vpaes_schedule_core,.-_vpaes_schedule_core - -## -## .aes_schedule_192_smear -## -## Smear the short, low side in the 192-bit key schedule. -## -## Inputs: -## %vr7: high side, b a x y -## %vr6: low side, d c 0 0 -## %vr13: 0 -## -## Outputs: -## %vr6: b+c+d b+c 0 0 -## %vr0: b+c+d b+c b a -## -#.type _vpaes_schedule_192_smear,@abi-omnipotent -.align 4 -_vpaes_schedule_192_smear: -.cfi_startproc - vshuf4i.w $vr1,$vr6,0x80 # d c 0 0 -> c 0 0 0 - vshuf4i.w $vr0,$vr7,0xFE # b a _ _ -> b b b a - vxor.v $vr6,$vr6,$vr1 # -> c+d c 0 0 - vxor.v $vr1,$vr1,$vr1 - vxor.v $vr6,$vr6,$vr0 # -> b+c+d b+c b a - vori.b $vr0,$vr6,0 - vilvh.d $vr6,$vr6,$vr1 # clobber low side with zeros - jr $r1 -.cfi_endproc -.size _vpaes_schedule_192_smear,.-_vpaes_schedule_192_smear - -## -## .aes_schedule_round -## -## Runs one main round of the key schedule on %vr0, %vr7 -## -## Specifically, runs subbytes on the high dword of %vr0 -## then rotates it by one byte and xors into the low dword of -## %vr7. -## -## Adds rcon from low byte of %vr8, then rotates %vr8 for -## next rcon. -## -## Smears the dwords of %vr7 by xoring the low into the -## second low, result into third, result into highest. -## -## Returns results in %vr7 = %vr0. -## Clobbers %vr1-%vr4, %a7. -## -#.type _vpaes_schedule_round,@abi-omnipotent -.align 4 -_vpaes_schedule_round: -.cfi_startproc - # extract rcon from vr8 - vxor.v $vr1,$vr1,$vr1 - vbsrl.v $vr16,$vr8,0xf - vbsll.v $vr1,$vr1,0x1 - vor.v $vr1,$vr1,$vr16 - vbsrl.v $vr16,$vr8,0xf - vbsll.v $vr8,$vr8,0x1 - vor.v $vr8,$vr8,$vr16 - - vxor.v $vr7,$vr7,$vr1 - - # rotate - vshuf4i.w $vr0,$vr0,0xff #put $vr0 lowest 32 bit to each words - vbsrl.v $vr16,$vr0,0x1 - vbsll.v $vr0,$vr0,0xf - vor.v $vr0,$vr0,$vr16 - - # fall through... - - # low round: same as high round, but no rotation and no rcon. -_vpaes_schedule_low_round: - # smear vr7 - vaddi.du $vr1,$vr7,0x0 - vbsll.v $vr7,$vr7,0x4 - vxor.v $vr7,$vr7,$vr1 - vaddi.du $vr1,$vr7,0x0 - vbsll.v $vr7,$vr7,0x8 - vxor.v $vr7,$vr7,$vr1 - vxori.b $vr7,$vr7,0x5B - - # subbytes - vaddi.du $vr1,$vr9,0x0 - vandn.v $vr1,$vr1,$vr0 - vsrli.w $vr1,$vr1,0x4 # 1 = i - vand.v $vr0,$vr0,$vr9 # 0 = k - vaddi.du $vr2,$vr11,0x0 # 2 : a/k - vshuf.b $vr2,$vr18,$vr2,$vr0 # 2 = a/k - vxor.v $vr0,$vr0,$vr1 # 0 = j - vaddi.du $vr3,$vr10,0x0 # 3 : 1/i - vshuf.b $vr3,$vr18,$vr3,$vr1 # 3 = 1/i - vxor.v $vr3,$vr3,$vr2 # 3 = iak = 1/i + a/k - vaddi.du $vr4,$vr10,0x0 # 4 : 1/j - vshuf.b $vr4,$vr18,$vr4,$vr0 # 4 = 1/j - vxor.v $vr4,$vr4,$vr2 # 4 = jak = 1/j + a/k - vaddi.du $vr2,$vr10,0x0 # 2 : 1/iak - vshuf.b $vr2,$vr18,$vr2,$vr3 # 2 = 1/iak - vxor.v $vr2,$vr2,$vr0 # 2 = io - vaddi.du $vr3,$vr10,0x0 # 3 : 1/jak - vshuf.b $vr3,$vr18,$vr3,$vr4 # 3 = 1/jak - vxor.v $vr3,$vr3,$vr1 # 3 = jo - vaddi.du $vr4,$vr13,0x0 # 4 : sbou - vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbou - vaddi.du $vr0,$vr12,0x0 # 0 : sbot - vshuf.b $vr0,$vr18,$vr0,$vr3 # 0 = sb1t - vxor.v $vr0,$vr0,$vr4 # 0 = sbox output - - # add in smeared stuff - vxor.v $vr0,$vr0,$vr7 - vaddi.du $vr7,$vr0,0x0 - jr $r1 -.cfi_endproc -.size _vpaes_schedule_round,.-_vpaes_schedule_round - -## -## .aes_schedule_transform -## -## Linear-transform %vr0 according to tables at (%r11) -## -## Requires that %vr9 = 0x0F0F... as in preheat -## Output in %vr0 -## Clobbers %vr1, %vr2 -## -#.type _vpaes_schedule_transform,@abi-omnipotent -.align 4 -_vpaes_schedule_transform: -.cfi_startproc - vori.b $vr1,$vr9,0 - vandn.v $vr1,$vr1,$vr0 - vsrli.w $vr1,$vr1,4 - vand.v $vr0,$vr0,$vr9 - vld $vr2,$r11,0 # lo - vshuf.b $vr2,$vr18,$vr2,$vr0 - vld $vr0,$r11,16 # hi - vshuf.b $vr0,$vr18,$vr0,$vr1 - vxor.v $vr0,$vr0,$vr2 - jr $r1 -.cfi_endproc -.size _vpaes_schedule_transform,.-_vpaes_schedule_transform - -## -## .aes_schedule_mangle -## -## Mangle vr0 from (basis-transformed) standard version -## to our version. -## -## On encrypt, -## xor with 0x63 -## multiply by circulant 0,1,1,1 -## apply shiftrows transform -## -## On decrypt, -## xor with 0x63 -## multiply by "inverse mixcolumns" circulant E,B,D,9 -## deskew -## apply shiftrows transform -## -## -## Writes out to (%a2), and increments or decrements it -## Keeps track of round number mod 4 in %a4 -## Preserves vr0 -## Clobbers vr1-vr5 -## -#.type _vpaes_schedule_mangle,@abi-omnipotent -.align 4 -_vpaes_schedule_mangle: -.cfi_startproc - vori.b $vr4,$vr0,0 # save vr0 for later - la.local $r12,Lk_mc_forward - vld $vr5,$r12,0 - bnez $r7,.Lschedule_mangle_dec - - # encrypting - addi.d $r6,$r6,16 - la.local $r12,Lk_s63 - vld $vr16,$r12,0 - vxor.v $vr4,$vr4,$vr16 - vshuf.b $vr4,$vr18,$vr4,$vr5 - vori.b $vr3,$vr4,0 - vshuf.b $vr4,$vr18,$vr4,$vr5 - vxor.v $vr3,$vr3,$vr4 - vshuf.b $vr4,$vr18,$vr4,$vr5 - vxor.v $vr3,$vr3,$vr4 - - b .Lschedule_mangle_both -.align 4 -.Lschedule_mangle_dec: - # inverse mix columns - la.local $r11,Lk_dksd - vori.b $vr1,$vr9,0 - vandn.v $vr1,$vr1,$vr4 - vsrli.w $vr1,$vr1,4 # 1 = hi - vand.v $vr4,$vr4,$vr9 # 4 = lo - - vld $vr2,$r11,0 - vshuf.b $vr2,$vr18,$vr2,$vr4 - vld $vr3,$r11,0x10 - vshuf.b $vr3,$vr18,$vr3,$vr1 - vxor.v $vr3,$vr3,$vr2 - vshuf.b $vr3,$vr18,$vr3,$vr5 - - vld $vr2,$r11,0x20 - vshuf.b $vr2,$vr18,$vr2,$vr4 - vxor.v $vr2,$vr2,$vr3 - vld $vr3,$r11,0x30 - vshuf.b $vr3,$vr18,$vr3,$vr1 - vxor.v $vr3,$vr3,$vr2 - vshuf.b $vr3,$vr18,$vr3,$vr5 - - vld $vr2,$r11,0x40 - vshuf.b $vr2,$vr18,$vr2,$vr4 - vxor.v $vr2,$vr2,$vr3 - vld $vr3,$r11,0x50 - vshuf.b $vr3,$vr18,$vr3,$vr1 - vxor.v $vr3,$vr3,$vr2 - vshuf.b $vr3,$vr18,$vr3,$vr5 - - vld $vr2,$r11,0x60 - vshuf.b $vr2,$vr18,$vr2,$vr4 - vxor.v $vr2,$vr2,$vr3 - vld $vr3,$r11,0x70 - vshuf.b $vr3,$vr18,$vr3,$vr1 - vxor.v $vr3,$vr3,$vr2 - - addi.d $r6,$r6,-16 - -.Lschedule_mangle_both: - add.d $r14,$r8,$r10 - vld $vr1,$r14,0 - vshuf.b $vr3,$vr18,$vr3,$vr1 - addi.d $r8,$r8,-16 - andi $r8,$r8,0x30 - vst $vr3,$r6,0 - jirl $r0,$r1,0 -.cfi_endproc -.size _vpaes_schedule_mangle,.-_vpaes_schedule_mangle - -# -# Interface to OpenSSL -# -.globl vpaes_set_encrypt_key -#.type vpaes_set_encrypt_key,@function,3 -.align 4 -vpaes_set_encrypt_key: -.cfi_startproc - addi.d $r3,$r3,-48 - st.d $r1,$r3,40 - st.d $r22,$r3,32 - move $r17,$r5 - srli.w $r17,$r17,0x5 - addi.w $r17,$r17,0x5 - st.w $r17,$r6,240 # AES_KEY->rounds = nbits/32+5; - - move $r7,$r0 - li.d $r8,0x30 - bl _vpaes_schedule_core - xor $r4,$r4,$r4 - ld.d $r1,$r3,40 - ld.d $r22,$r3,32 - addi.d $r3,$r3,48 - jirl $r0,$r1,0 -.cfi_endproc -.size vpaes_set_encrypt_key,.-vpaes_set_encrypt_key - -.globl vpaes_set_decrypt_key -#.type vpaes_set_decrypt_key,@function,3 -.align 4 -vpaes_set_decrypt_key: -.cfi_startproc - -.Ldec_key_body: - addi.d $r3,$r3,-48 - st.d $r1,$r3,40 - st.d $r22,$r3,32 - - move $r17,$r5 - srli.w $r17,$r17,5 - addi.w $r17,$r17,5 - st.w $r17,$r6,240 # AES_KEY->rounds = nbits/32+5; - slli.w $r17,$r17,4 - add.d $r12,$r6,$r17 - addi.d $r6,$r12,16 - - li.d $r7,0x1 - move $r8,$r5 - srli.w $r8,$r8,1 - andi $r8,$r8,32 - xori $r8,$r8,32 # nbits==192?0:32 - bl _vpaes_schedule_core - -.Ldec_key_epilogue: - xor $r4,$r4,$r4 - ld.d $r1,$r3,40 - ld.d $r22,$r3,32 - addi.d $r3,$r3,48 - jirl $r0,$r1,0 -.cfi_endproc -.size vpaes_set_decrypt_key,.-vpaes_set_decrypt_key - -.globl vpaes_encrypt -#.type vpaes_encrypt,@function,3 -.align 4 -vpaes_encrypt: -.cfi_startproc -.Lenc_body: - addi.d $r3,$r3,-48 - st.d $r1,$r3,40 - st.d $r22,$r3,32 - vld $vr0,$r4,0x0 - bl _vpaes_preheat - bl _vpaes_encrypt_core - vst $vr0,$r5,0x0 -.Lenc_epilogue: - ld.d $r1,$r3,40 - ld.d $r22,$r3,32 - addi.d $r3,$r3,48 - jirl $r0,$r1,0 -.cfi_endproc -.size vpaes_encrypt,.-vpaes_encrypt - -.globl vpaes_decrypt -#.type vpaes_decrypt,@function,3 -.align 4 -vpaes_decrypt: -.cfi_startproc - addi.d $r3,$r3,-48 - st.d $r1,$r3,40 - st.d $r22,$r3,32 - vld $vr0,$r4,0x0 - bl _vpaes_preheat - bl _vpaes_decrypt_core - vst $vr0,$r5,0x0 - ld.d $r1,$r3,40 - ld.d $r22,$r3,32 - addi.d $r3,$r3,48 - jirl $r0,$r1,0 -.cfi_endproc -.size vpaes_decrypt,.-vpaes_decrypt -.globl vpaes_cbc_encrypt -#.type vpaes_cbc_encrypt,@function,6 -.align 4 -vpaes_cbc_encrypt: -.cfi_startproc - addi.d $r3,$r3,-48 - st.d $r1,$r3,40 - st.d $r22,$r3,32 - - ori $r12,$r6,0 - ori $r6,$r7,0 - ori $r7,$r12,0 - addi.d $r7,$r7,-16 - blt $r7,$r0,.Lcbc_abort - vld $vr6,$r8,0 # load IV - sub.d $r5,$r5,$r4 - bl _vpaes_preheat - beqz $r9,.Lcbc_dec_loop - b .Lcbc_enc_loop -.align 4 -.Lcbc_enc_loop: - vld $vr0,$r4,0 - vxor.v $vr0,$vr0,$vr6 - bl _vpaes_encrypt_core - vori.b $vr6,$vr0,0 - add.d $r12,$r5,$r4 - vst $vr0,$r12,0 - addi.d $r4,$r4,16 - addi.d $r7,$r7,-16 - bge $r7,$r0,.Lcbc_enc_loop - b .Lcbc_done -.align 4 -.Lcbc_dec_loop: - vld $vr0,$r4,0 - vori.b $vr7,$vr0,0 - bl _vpaes_decrypt_core - vxor.v $vr0,$vr0,$vr6 - vori.b $vr6,$vr7,0 - add.d $r12,$r5,$r4 - vst $vr0,$r12,0 - addi.d $r4,$r4,16 - addi.d $r7,$r7,-16 - bge $r7,$r0,.Lcbc_dec_loop -.Lcbc_done: - vst $vr6,$r8,0 # save IV -.Lcbc_abort: - ld.d $r1,$r3,40 - ld.d $r22,$r3,32 - addi.d $r3,$r3,48 - jirl $r0,$r1,0 -.cfi_endproc -.size vpaes_cbc_encrypt,.-vpaes_cbc_encrypt -## -## _aes_preheat -## -## Fills register %a6 -> .aes_consts (so you can -fPIC) -## and %vr9-%vr15 as specified below. -## -#.type _vpaes_preheat,@abi-omnipotent -.align 4 -_vpaes_preheat: -.cfi_startproc - la.local $r10,Lk_s0F - vld $vr10,$r10,-0x20 # Lk_inv - vld $vr11,$r10,-0x10 # Lk_inv+16 - vld $vr9,$r10,0 # Lk_s0F - vld $vr13,$r10,0x30 # Lk_sb1 - vld $vr12,$r10,0x40 # Lk_sb1+16 - vld $vr15,$r10,0x50 # Lk_sb2 - vld $vr14,$r10,0x60 # Lk_sb2+16 - vldi $vr18,0 # $vr18 in this program is equal to 0 - jirl $r0,$r1,0 -.cfi_endproc -.size _vpaes_preheat,.-_vpaes_preheat -.section .rodata -.align 6 -Lk_inv: # inv, inva - .quad 0x0E05060F0D080110, 0x040703090A0B0C02 - .quad 0x01040A060F0B0710, 0x030D0E0C02050809 - -Lk_s0F: # s0F - .quad 0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F0F0F0F0F - -Lk_ipt: # input transform (lo, hi) - .quad 0xC2B2E8985A2A7000, 0xCABAE09052227808 - .quad 0x4C01307D317C4D00, 0xCD80B1FCB0FDCC81 - -Lk_sb1: # sb1u, sb1t - .quad 0xB19BE18FCB503E00, 0xA5DF7A6E142AF544 - .quad 0x3618D415FAE22300, 0x3BF7CCC10D2ED9EF -Lk_sb2: # sb2u, sb2t - .quad 0xE27A93C60B712400, 0x5EB7E955BC982FCD - .quad 0x69EB88400AE12900, 0xC2A163C8AB82234A -Lk_sbo: # sbou, sbot - .quad 0xD0D26D176FBDC700, 0x15AABF7AC502A878 - .quad 0xCFE474A55FBB6A00, 0x8E1E90D1412B35FA - -Lk_mc_forward: # mc_forward - .quad 0x0407060500030201, 0x0C0F0E0D080B0A09 - .quad 0x080B0A0904070605, 0x000302010C0F0E0D - .quad 0x0C0F0E0D080B0A09, 0x0407060500030201 - .quad 0x000302010C0F0E0D, 0x080B0A0904070605 - -Lk_mc_backward:# mc_backward - .quad 0x0605040702010003, 0x0E0D0C0F0A09080B - .quad 0x020100030E0D0C0F, 0x0A09080B06050407 - .quad 0x0E0D0C0F0A09080B, 0x0605040702010003 - .quad 0x0A09080B06050407, 0x020100030E0D0C0F - -Lk_sr: # sr - .quad 0x0706050403020100, 0x0F0E0D0C0B0A0908 - .quad 0x030E09040F0A0500, 0x0B06010C07020D08 - .quad 0x0F060D040B020900, 0x070E050C030A0108 - .quad 0x0B0E0104070A0D00, 0x0306090C0F020508 - -Lk_rcon: # rcon - .quad 0x1F8391B9AF9DEEB6, 0x702A98084D7C7D81 - -Lk_s63: # s63: all equal to 0x63 transformed - .quad 0x5B5B5B5B5B5B5B5B, 0x5B5B5B5B5B5B5B5B - -Lk_opt: # output transform - .quad 0xFF9F4929D6B66000, 0xF7974121DEBE6808 - .quad 0x01EDBD5150BCEC00, 0xE10D5DB1B05C0CE0 - -Lk_deskew: # deskew tables: inverts the sbox's "skew" - .quad 0x07E4A34047A4E300, 0x1DFEB95A5DBEF91A - .quad 0x5F36B5DC83EA6900, 0x2841C2ABF49D1E77 - -## -## Decryption stuff -## Key schedule constants -## -Lk_dksd: # decryption key schedule: invskew x*D - .quad 0xFEB91A5DA3E44700, 0x0740E3A45A1DBEF9 - .quad 0x41C277F4B5368300, 0x5FDC69EAAB289D1E -Lk_dksb: # decryption key schedule: invskew x*B - .quad 0x9A4FCA1F8550D500, 0x03D653861CC94C99 - .quad 0x115BEDA7B6FC4A00, 0xD993256F7E3482C8 -Lk_dkse: # decryption key schedule: invskew x*E + 0x63 - .quad 0xD5031CCA1FC9D600, 0x53859A4C994F5086 - .quad 0xA23196054FDC7BE8, 0xCD5EF96A20B31487 -Lk_dks9: # decryption key schedule: invskew x*9 - .quad 0xB6116FC87ED9A700, 0x4AED933482255BFC - .quad 0x4576516227143300, 0x8BB89FACE9DAFDCE - -## -## Decryption stuff -## Round function constants -## -Lk_dipt: # decryption input transform - .quad 0x0F505B040B545F00, 0x154A411E114E451A - .quad 0x86E383E660056500, 0x12771772F491F194 - -Lk_dsb9: # decryption sbox output *9*u, *9*t - .quad 0x851C03539A86D600, 0xCAD51F504F994CC9 - .quad 0xC03B1789ECD74900, 0x725E2C9EB2FBA565 -Lk_dsbd: # decryption sbox output *D*u, *D*t - .quad 0x7D57CCDFE6B1A200, 0xF56E9B13882A4439 - .quad 0x3CE2FAF724C6CB00, 0x2931180D15DEEFD3 -Lk_dsbb: # decryption sbox output *B*u, *B*t - .quad 0xD022649296B44200, 0x602646F6B0F2D404 - .quad 0xC19498A6CD596700, 0xF3FF0C3E3255AA6B -Lk_dsbe: # decryption sbox output *E*u, *E*t - .quad 0x46F2929626D4D000, 0x2242600464B4F6B0 - .quad 0x0C55A6CDFFAAC100, 0x9467F36B98593E32 -Lk_dsbo: # decryption sbox final output - .quad 0x1387EA537EF94000, 0xC7AA6DB9D4943E2D - .quad 0x12D7560F93441D00, 0xCA4B8159D8C58E9C -.asciz "Vector Permutation AES for loongarch64/lsx, Mike Hamburg (Stanford University)" -.align 6 diff --git a/openssl/src/crypto/aes/gen/linux_ppc64/aesp8-ppc.s b/openssl/src/crypto/aes/gen/linux_ppc64/aesp8-ppc.s index 2577338d5..ae924ef9d 100644 --- a/openssl/src/crypto/aes/gen/linux_ppc64/aesp8-ppc.s +++ b/openssl/src/crypto/aes/gen/linux_ppc64/aesp8-ppc.s @@ -9,12 +9,11 @@ rcon: .byte 0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x1b .byte 0x0c,0x0f,0x0e,0x0d,0x0c,0x0f,0x0e,0x0d,0x0c,0x0f,0x0e,0x0d,0x0c,0x0f,0x0e,0x0d .byte 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 -.long 0x0f102132, 0x43546576, 0x8798a9ba, 0xcbdcedfe .Lconsts: mflr 0 bcl 20,31,$+4 mflr 6 - addi 6,6,-0x58 + addi 6,6,-0x48 mtlr 0 blr .long 0 @@ -2364,18 +2363,6 @@ _aesp8_xts_encrypt6x: li 31,0x70 or 0,0,0 - - xxlor 2, 32+10, 32+10 - vsldoi 10,11,10,1 - xxlor 1, 32+10, 32+10 - - - mr 31, 6 - bl .Lconsts - lxvw4x 0, 28, 6 - mr 6, 31 - li 31,0x70 - subi 9,9,3 lvx 23,0,6 @@ -2418,77 +2405,69 @@ _aesp8_xts_encrypt6x: vperm 31,22,31,7 lvx 25,3,7 - - - - - - - - vperm 0,2,4,5 subi 10,10,31 vxor 17,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vand 11,11,10 vxor 7,0,17 - xxlor 32+1, 0, 0 - vpermxor 8, 8, 11, 1 + vxor 8,8,11 .long 0x7C235699 vxor 18,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 1,1,1,6 vand 11,11,10 vxor 12,1,18 - xxlor 32+2, 0, 0 - vpermxor 8, 8, 11, 2 + vxor 8,8,11 .long 0x7C5A5699 andi. 31,5,15 vxor 19,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 2,2,2,6 vand 11,11,10 vxor 13,2,19 - xxlor 32+3, 0, 0 - vpermxor 8, 8, 11, 3 + vxor 8,8,11 .long 0x7C7B5699 sub 5,5,31 vxor 20,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 3,3,3,6 vand 11,11,10 vxor 14,3,20 - xxlor 32+4, 0, 0 - vpermxor 8, 8, 11, 4 + vxor 8,8,11 .long 0x7C9C5699 subi 5,5,0x60 vxor 21,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 4,4,4,6 vand 11,11,10 vxor 15,4,21 - xxlor 32+5, 0, 0 - vpermxor 8, 8, 11, 5 + vxor 8,8,11 .long 0x7CBD5699 addi 10,10,0x60 vxor 22,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 5,5,5,6 vand 11,11,10 vxor 16,5,22 - xxlor 32+0, 0, 0 - vpermxor 8, 8, 11, 0 + vxor 8,8,11 vxor 31,31,23 mtctr 9 @@ -2514,8 +2493,6 @@ _aesp8_xts_encrypt6x: lvx 25,3,7 bdnz .Loop_xts_enc6x - xxlor 32+10, 1, 1 - subic 5,5,96 vxor 0,17,31 .long 0x10E7C508 @@ -2525,6 +2502,7 @@ _aesp8_xts_encrypt6x: vaddubm 8,8,8 .long 0x11ADC508 .long 0x11CEC508 + vsldoi 11,11,11,15 .long 0x11EFC508 .long 0x1210C508 @@ -2532,8 +2510,7 @@ _aesp8_xts_encrypt6x: vand 11,11,10 .long 0x10E7CD08 .long 0x118CCD08 - xxlor 32+1, 0, 0 - vpermxor 8, 8, 11, 1 + vxor 8,8,11 .long 0x11ADCD08 .long 0x11CECD08 vxor 1,18,31 @@ -2544,13 +2521,13 @@ _aesp8_xts_encrypt6x: and 0,0,5 vaddubm 8,8,8 + vsldoi 11,11,11,15 .long 0x10E7D508 .long 0x118CD508 vand 11,11,10 .long 0x11ADD508 .long 0x11CED508 - xxlor 32+2, 0, 0 - vpermxor 8, 8, 11, 2 + vxor 8,8,11 .long 0x11EFD508 .long 0x1210D508 @@ -2564,6 +2541,7 @@ _aesp8_xts_encrypt6x: vaddubm 8,8,8 .long 0x10E7DD08 .long 0x118CDD08 + vsldoi 11,11,11,15 .long 0x11ADDD08 .long 0x11CEDD08 vand 11,11,10 @@ -2571,8 +2549,7 @@ _aesp8_xts_encrypt6x: .long 0x1210DD08 addi 7,1,64+15 - xxlor 32+3, 0, 0 - vpermxor 8, 8, 11, 3 + vxor 8,8,11 .long 0x10E7E508 .long 0x118CE508 vxor 3,20,31 @@ -2581,6 +2558,7 @@ _aesp8_xts_encrypt6x: .long 0x11ADE508 .long 0x11CEE508 vaddubm 8,8,8 + vsldoi 11,11,11,15 .long 0x11EFE508 .long 0x1210E508 lvx 24,0,7 @@ -2588,8 +2566,7 @@ _aesp8_xts_encrypt6x: .long 0x10E7ED08 .long 0x118CED08 - xxlor 32+4, 0, 0 - vpermxor 8, 8, 11, 4 + vxor 8,8,11 .long 0x11ADED08 .long 0x11CEED08 vxor 4,21,31 @@ -2599,14 +2576,14 @@ _aesp8_xts_encrypt6x: .long 0x1210ED08 lvx 25,3,7 vaddubm 8,8,8 + vsldoi 11,11,11,15 .long 0x10E7F508 .long 0x118CF508 vand 11,11,10 .long 0x11ADF508 .long 0x11CEF508 - xxlor 32+5, 0, 0 - vpermxor 8, 8, 11, 5 + vxor 8,8,11 .long 0x11EFF508 .long 0x1210F508 vxor 5,22,31 @@ -2616,6 +2593,7 @@ _aesp8_xts_encrypt6x: .long 0x10E70509 .long 0x7C005699 vaddubm 8,8,8 + vsldoi 11,11,11,15 .long 0x118C0D09 .long 0x7C235699 .long 0x11AD1509 @@ -2628,10 +2606,7 @@ _aesp8_xts_encrypt6x: .long 0x11EF2509 vperm 2,2,2,6 .long 0x7C9C5699 - xxlor 10, 32+0, 32+0 - xxlor 32+0, 0, 0 - vpermxor 8, 8, 11, 0 - xxlor 32+0, 10, 10 + vxor 8,8,11 .long 0x11702D09 vperm 3,3,3,6 @@ -2664,8 +2639,6 @@ _aesp8_xts_encrypt6x: mtctr 9 beq .Loop_xts_enc6x - xxlor 32+10, 2, 2 - addic. 5,5,0x60 beq .Lxts_enc6x_zero cmpwi 5,0x20 @@ -3042,18 +3015,6 @@ _aesp8_xts_decrypt6x: li 31,0x70 or 0,0,0 - - xxlor 2, 32+10, 32+10 - vsldoi 10,11,10,1 - xxlor 1, 32+10, 32+10 - - - mr 31, 6 - bl .Lconsts - lxvw4x 0, 28, 6 - mr 6, 31 - li 31,0x70 - subi 9,9,3 lvx 23,0,6 @@ -3101,64 +3062,64 @@ _aesp8_xts_decrypt6x: vxor 17,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vand 11,11,10 vxor 7,0,17 - xxlor 32+1, 0, 0 - vpermxor 8, 8, 11, 1 + vxor 8,8,11 .long 0x7C235699 vxor 18,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 1,1,1,6 vand 11,11,10 vxor 12,1,18 - xxlor 32+2, 0, 0 - vpermxor 8, 8, 11, 2 + vxor 8,8,11 .long 0x7C5A5699 andi. 31,5,15 vxor 19,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 2,2,2,6 vand 11,11,10 vxor 13,2,19 - xxlor 32+3, 0, 0 - vpermxor 8, 8, 11, 3 + vxor 8,8,11 .long 0x7C7B5699 sub 5,5,31 vxor 20,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 3,3,3,6 vand 11,11,10 vxor 14,3,20 - xxlor 32+4, 0, 0 - vpermxor 8, 8, 11, 4 + vxor 8,8,11 .long 0x7C9C5699 subi 5,5,0x60 vxor 21,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 4,4,4,6 vand 11,11,10 vxor 15,4,21 - xxlor 32+5, 0, 0 - vpermxor 8, 8, 11, 5 + vxor 8,8,11 .long 0x7CBD5699 addi 10,10,0x60 vxor 22,8,23 vsrab 11,8,9 vaddubm 8,8,8 + vsldoi 11,11,11,15 vperm 5,5,5,6 vand 11,11,10 vxor 16,5,22 - xxlor 32+0, 0, 0 - vpermxor 8, 8, 11, 0 + vxor 8,8,11 vxor 31,31,23 mtctr 9 @@ -3184,8 +3145,6 @@ _aesp8_xts_decrypt6x: lvx 25,3,7 bdnz .Loop_xts_dec6x - xxlor 32+10, 1, 1 - subic 5,5,96 vxor 0,17,31 .long 0x10E7C548 @@ -3195,6 +3154,7 @@ _aesp8_xts_decrypt6x: vaddubm 8,8,8 .long 0x11ADC548 .long 0x11CEC548 + vsldoi 11,11,11,15 .long 0x11EFC548 .long 0x1210C548 @@ -3202,8 +3162,7 @@ _aesp8_xts_decrypt6x: vand 11,11,10 .long 0x10E7CD48 .long 0x118CCD48 - xxlor 32+1, 0, 0 - vpermxor 8, 8, 11, 1 + vxor 8,8,11 .long 0x11ADCD48 .long 0x11CECD48 vxor 1,18,31 @@ -3214,13 +3173,13 @@ _aesp8_xts_decrypt6x: and 0,0,5 vaddubm 8,8,8 + vsldoi 11,11,11,15 .long 0x10E7D548 .long 0x118CD548 vand 11,11,10 .long 0x11ADD548 .long 0x11CED548 - xxlor 32+2, 0, 0 - vpermxor 8, 8, 11, 2 + vxor 8,8,11 .long 0x11EFD548 .long 0x1210D548 @@ -3234,6 +3193,7 @@ _aesp8_xts_decrypt6x: vaddubm 8,8,8 .long 0x10E7DD48 .long 0x118CDD48 + vsldoi 11,11,11,15 .long 0x11ADDD48 .long 0x11CEDD48 vand 11,11,10 @@ -3241,8 +3201,7 @@ _aesp8_xts_decrypt6x: .long 0x1210DD48 addi 7,1,64+15 - xxlor 32+3, 0, 0 - vpermxor 8, 8, 11, 3 + vxor 8,8,11 .long 0x10E7E548 .long 0x118CE548 vxor 3,20,31 @@ -3251,6 +3210,7 @@ _aesp8_xts_decrypt6x: .long 0x11ADE548 .long 0x11CEE548 vaddubm 8,8,8 + vsldoi 11,11,11,15 .long 0x11EFE548 .long 0x1210E548 lvx 24,0,7 @@ -3258,8 +3218,7 @@ _aesp8_xts_decrypt6x: .long 0x10E7ED48 .long 0x118CED48 - xxlor 32+4, 0, 0 - vpermxor 8, 8, 11, 4 + vxor 8,8,11 .long 0x11ADED48 .long 0x11CEED48 vxor 4,21,31 @@ -3269,14 +3228,14 @@ _aesp8_xts_decrypt6x: .long 0x1210ED48 lvx 25,3,7 vaddubm 8,8,8 + vsldoi 11,11,11,15 .long 0x10E7F548 .long 0x118CF548 vand 11,11,10 .long 0x11ADF548 .long 0x11CEF548 - xxlor 32+5, 0, 0 - vpermxor 8, 8, 11, 5 + vxor 8,8,11 .long 0x11EFF548 .long 0x1210F548 vxor 5,22,31 @@ -3286,6 +3245,7 @@ _aesp8_xts_decrypt6x: .long 0x10E70549 .long 0x7C005699 vaddubm 8,8,8 + vsldoi 11,11,11,15 .long 0x118C0D49 .long 0x7C235699 .long 0x11AD1549 @@ -3298,10 +3258,7 @@ _aesp8_xts_decrypt6x: .long 0x11EF2549 vperm 2,2,2,6 .long 0x7C9C5699 - xxlor 10, 32+0, 32+0 - xxlor 32+0, 0, 0 - vpermxor 8, 8, 11, 0 - xxlor 32+0, 10, 10 + vxor 8,8,11 .long 0x12102D49 vperm 3,3,3,6 .long 0x7CBD5699 @@ -3332,8 +3289,6 @@ _aesp8_xts_decrypt6x: mtctr 9 beq .Loop_xts_dec6x - xxlor 32+10, 2, 2 - addic. 5,5,0x60 beq .Lxts_dec6x_zero cmpwi 5,0x20 diff --git a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zkn.s b/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zkn.s deleted file mode 100644 index 80ae8736e..000000000 --- a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zkn.s +++ /dev/null @@ -1,704 +0,0 @@ -.text -.balign 16 -.globl rv64i_zkne_encrypt -.type rv64i_zkne_encrypt,@function -rv64i_zkne_encrypt: - addi sp,sp,-16 - sd x8,8(sp) - sd x9,0(sp) - - # Load input to block cipher - ld x6,0(x10) - ld x7,8(x10) - - # Load key - ld x13,0(x12) - ld x14,8(x12) - - # Load number of rounds - lwu x30,240(x12) - - # initial transformation - xor x6,x6,x13 - xor x7,x7,x14 - - # The main loop only executes the first N-1 rounds. - add x30,x30,-1 - - # Do Nr - 1 rounds (final round is special) -1: - .word 913507379 - .word 912491699 - - # Update key ptr to point to next key in schedule - add x12,x12,16 - - # Grab next key in schedule - ld x13,0(x12) - ld x14,8(x12) - xor x6,x8,x13 - xor x7,x9,x14 - - add x30,x30,-1 - bgtz x30,1b - - # final round - .word 846398515 - .word 845382835 - - # since not added 16 before - ld x13,16(x12) - ld x14,24(x12) - xor x6,x8,x13 - xor x7,x9,x14 - - sd x6,0(x11) - sd x7,8(x11) - - # Pop registers and return - ld x8,8(sp) - ld x9,0(sp) - addi sp,sp,16 - ret -.text -.balign 16 -.globl rv64i_zknd_decrypt -.type rv64i_zknd_decrypt,@function -rv64i_zknd_decrypt: - addi sp,sp,-16 - sd x8,8(sp) - sd x9,0(sp) - - # Load input to block cipher - ld x6,0(x10) - ld x7,8(x10) - - # Load number of rounds - lwu x30,240(x12) - - # Load the last key - slli x13,x30,4 - add x12,x12,x13 - ld x13,0(x12) - ld x14,8(x12) - - xor x6,x6,x13 - xor x7,x7,x14 - - # The main loop only executes the first N-1 rounds. - add x30,x30,-1 - - # Do Nr - 1 rounds (final round is special) -1: - .word 1047725107 - .word 1046709427 - - # Update key ptr to point to next key in schedule - add x12,x12,-16 - - # Grab next key in schedule - ld x13,0(x12) - ld x14,8(x12) - xor x6,x8,x13 - xor x7,x9,x14 - - add x30,x30,-1 - bgtz x30,1b - - # final round - .word 980616243 - .word 979600563 - - add x12,x12,-16 - ld x13,0(x12) - ld x14,8(x12) - xor x6,x8,x13 - xor x7,x9,x14 - - sd x6,0(x11) - sd x7,8(x11) - # Pop registers and return - ld x8,8(sp) - ld x9,0(sp) - addi sp,sp,16 - ret -.text -.balign 16 -.globl rv64i_zkne_set_encrypt_key -.type rv64i_zkne_set_encrypt_key,@function -rv64i_zkne_set_encrypt_key: - addi sp,sp,-16 - sd x8,0(sp) - bnez x10,1f # if (!userKey || !key) return -1; - bnez x12,1f - li a0,-1 - ret -1: - # Determine number of rounds from key size in bits - li x6,128 - bne x11,x6,1f - li x7,10 # key->rounds = 10 if bits == 128 - sw x7,240(x12) # store key->rounds - ld x6,0(x10) - ld x7,8(x10) - sd x6,0(x12) - sd x7,8(x12) - .word 822318099 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 823366675 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 824415251 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 825463827 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 826512403 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 827560979 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 828609555 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 829658131 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 830706707 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - .word 831755283 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - - j 4f -1: - li x6,192 - bne x11,x6,2f - li x7,12 # key->rounds = 12 if bits == 192 - sw x7,240(x12) # store key->rounds - ld x6,0(x10) - ld x7,8(x10) - ld x8,16(x10) - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - .word 822351507 - .word 2120647475 - .word 2121466803 - .word 2122548275 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - .word 823400083 - .word 2120647475 - .word 2121466803 - .word 2122548275 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - .word 824448659 - .word 2120647475 - .word 2121466803 - .word 2122548275 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - .word 825497235 - .word 2120647475 - .word 2121466803 - .word 2122548275 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - .word 826545811 - .word 2120647475 - .word 2121466803 - .word 2122548275 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - .word 827594387 - .word 2120647475 - .word 2121466803 - .word 2122548275 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - .word 828642963 - .word 2120647475 - .word 2121466803 - .word 2122548275 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - .word 829691539 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - - j 4f -2: - li x7,14 # key->rounds = 14 if bits == 256 - li x6,256 - beq x11,x6,3f - li a0,-2 # If bits != 128, 192, or 256, return -2 - j 5f -3: - sw x7,240(x12) # store key->rounds - ld x6,0(x10) - ld x7,8(x10) - ld x8,16(x10) - ld x13,24(x10) - sd x6,0(x12) - sd x7,8(x12) - sd x8,16(x12) - sd x13,24(x12) - .word 822515475 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - sd x6,0(x12) - sd x7,8(x12) - .word 832804627 - .word 2122777651 - .word 2127824563 - sd x8,16(x12) - sd x13,24(x12) - .word 823564051 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - sd x6,0(x12) - sd x7,8(x12) - .word 832804627 - .word 2122777651 - .word 2127824563 - sd x8,16(x12) - sd x13,24(x12) - .word 824612627 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - sd x6,0(x12) - sd x7,8(x12) - .word 832804627 - .word 2122777651 - .word 2127824563 - sd x8,16(x12) - sd x13,24(x12) - .word 825661203 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - sd x6,0(x12) - sd x7,8(x12) - .word 832804627 - .word 2122777651 - .word 2127824563 - sd x8,16(x12) - sd x13,24(x12) - .word 826709779 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - sd x6,0(x12) - sd x7,8(x12) - .word 832804627 - .word 2122777651 - .word 2127824563 - sd x8,16(x12) - sd x13,24(x12) - .word 827758355 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - sd x6,0(x12) - sd x7,8(x12) - .word 832804627 - .word 2122777651 - .word 2127824563 - sd x8,16(x12) - sd x13,24(x12) - .word 828806931 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - sd x6,0(x12) - sd x7,8(x12) - -4: # return 0 - li a0,0 -5: # return a0 - ld x8,0(sp) - addi sp,sp,16 - ret -.text -.balign 16 -.globl rv64i_zknd_set_decrypt_key -.type rv64i_zknd_set_decrypt_key,@function -rv64i_zknd_set_decrypt_key: - addi sp,sp,-16 - sd x8,0(sp) - bnez x10,1f # if (!userKey || !key) return -1; - bnez x12,1f - li a0,-1 - ret -1: - # Determine number of rounds from key size in bits - li x6,128 - bne x11,x6,1f - li x7,10 # key->rounds = 10 if bits == 128 - sw x7,240(x12) # store key->rounds - ld x6,0(x10) - ld x7,8(x10) - sd x6,0(x12) - sd x7,8(x12) - .word 822318099 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 823366675 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 824415251 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 825463827 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 826512403 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 827560979 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 828609555 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 829658131 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 830706707 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - .word 805508115 - sd x8,0(x12) - .word 805540883 - sd x8,8(x12) - .word 831755283 - .word 2120483635 - .word 2121466803 - add x12,x12,16 - sd x6,0(x12) - sd x7,8(x12) - - j 4f -1: - li x6,192 - bne x11,x6,2f - li x7,12 # key->rounds = 12 if bits == 192 - sw x7,240(x12) # store key->rounds - ld x6,0(x10) - ld x7,8(x10) - ld x8,16(x10) - sd x6,0(x12) - sd x7,8(x12) - .word 805574291 - sd x13,16(x12) - .word 822351507 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - .word 805508755 - sd x13,0(x12) - .word 805541523 - sd x13,8(x12) - # the reason is in ke192enc - .word 2122548275 - .word 805574291 - sd x13,16(x12) - .word 823400083 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - .word 805508755 - sd x13,0(x12) - .word 805541523 - sd x13,8(x12) - # the reason is in ke192enc - .word 2122548275 - .word 805574291 - sd x13,16(x12) - .word 824448659 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - .word 805508755 - sd x13,0(x12) - .word 805541523 - sd x13,8(x12) - # the reason is in ke192enc - .word 2122548275 - .word 805574291 - sd x13,16(x12) - .word 825497235 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - .word 805508755 - sd x13,0(x12) - .word 805541523 - sd x13,8(x12) - # the reason is in ke192enc - .word 2122548275 - .word 805574291 - sd x13,16(x12) - .word 826545811 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - .word 805508755 - sd x13,0(x12) - .word 805541523 - sd x13,8(x12) - # the reason is in ke192enc - .word 2122548275 - .word 805574291 - sd x13,16(x12) - .word 827594387 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - .word 805508755 - sd x13,0(x12) - .word 805541523 - sd x13,8(x12) - # the reason is in ke192enc - .word 2122548275 - .word 805574291 - sd x13,16(x12) - .word 828642963 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - .word 805508755 - sd x13,0(x12) - .word 805541523 - sd x13,8(x12) - # the reason is in ke192enc - .word 2122548275 - .word 805574291 - sd x13,16(x12) - .word 829691539 - .word 2120647475 - .word 2121466803 - add x12,x12,24 - sd x6,0(x12) - sd x7,8(x12) - - j 4f -2: - li x7,14 # key->rounds = 14 if bits == 256 - li x6,256 - beq x11,x6,3f - li a0,-2 # If bits != 128, 192, or 256, return -2 - j 5f -3: - sw x7,240(x12) # store key->rounds - ld x6,0(x10) - ld x7,8(x10) - ld x8,16(x10) - ld x13,24(x10) - sd x6,0(x12) - sd x7,8(x12) - .word 805574419 - sd x14,16(x12) - .word 805738259 - sd x14,24(x12) - .word 822515475 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - .word 832804627 - .word 2122777651 - .word 2127824563 - .word 805508883 - sd x14,0(x12) - .word 805541651 - sd x14,8(x12) - .word 805574419 - sd x14,16(x12) - .word 805738259 - sd x14,24(x12) - .word 823564051 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - .word 832804627 - .word 2122777651 - .word 2127824563 - .word 805508883 - sd x14,0(x12) - .word 805541651 - sd x14,8(x12) - .word 805574419 - sd x14,16(x12) - .word 805738259 - sd x14,24(x12) - .word 824612627 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - .word 832804627 - .word 2122777651 - .word 2127824563 - .word 805508883 - sd x14,0(x12) - .word 805541651 - sd x14,8(x12) - .word 805574419 - sd x14,16(x12) - .word 805738259 - sd x14,24(x12) - .word 825661203 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - .word 832804627 - .word 2122777651 - .word 2127824563 - .word 805508883 - sd x14,0(x12) - .word 805541651 - sd x14,8(x12) - .word 805574419 - sd x14,16(x12) - .word 805738259 - sd x14,24(x12) - .word 826709779 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - .word 832804627 - .word 2122777651 - .word 2127824563 - .word 805508883 - sd x14,0(x12) - .word 805541651 - sd x14,8(x12) - .word 805574419 - sd x14,16(x12) - .word 805738259 - sd x14,24(x12) - .word 827758355 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - .word 832804627 - .word 2122777651 - .word 2127824563 - .word 805508883 - sd x14,0(x12) - .word 805541651 - sd x14,8(x12) - .word 805574419 - sd x14,16(x12) - .word 805738259 - sd x14,24(x12) - .word 828806931 - .word 2120680243 - .word 2121466803 - add x12,x12,32 - sd x6,0(x12) - sd x7,8(x12) - # last two one dropped - -4: # return 0 - li a0,0 -5: # return a0 - ld x8,0(sp) - addi sp,sp,16 - ret diff --git a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvbb-zvkg-zvkned.s b/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvbb-zvkg-zvkned.s deleted file mode 100644 index f8a68e7c4..000000000 --- a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvbb-zvkg-zvkned.s +++ /dev/null @@ -1,943 +0,0 @@ -.text -.p2align 3 -.globl rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt -.type rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt,@function -rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt: - # Load number of rounds - lwu t0, 240(a4) - .word 3439489111 - .word 34074119 - .word 34041479 - .word 2815667831 - addi t0, t0, -1 - addi a4, a4, 16 -1: - .word 34041479 - .word 2815503991 - addi t0, t0, -1 - addi a4, a4, 16 - bnez t0, 1b - .word 34041479 - .word 2815536759 - - - # aes block size is 16 - andi a6, a2, 15 - mv t3, a2 - beqz a6, 1f - sub a2, a2, a6 - addi t3, a2, -16 -1: - # We make the `LENGTH` become e32 length here. - srli t4, a2, 2 - srli t3, t3, 2 - - # Load number of rounds - lwu t0, 240(a3) - li t1, 14 - li t2, 10 - beq t0, t1, aes_xts_enc_256 - beq t0, t2, aes_xts_enc_128 -.size rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt,.-rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt -.p2align 3 -aes_xts_enc_128: - # load input - .word 221182167 - .word 33909767 - - li t0, 5 - # We could simplify the initialization steps if we have `block<=1`. - blt t4, t0, 1f - - # Note: We use `vgmul` for GF(2^128) multiplication. The `vgmul` uses - # different order of coefficients. We should use`vbrev8` to reverse the - # data when we use `vgmul`. - .word 3439489111 - .word 1271144535 - .word 221179991 - .word 1577072727 - # v16: [r-IV0, r-IV0, ...] - .word 2785257591 - - # Prepare GF(2^128) multiplier [1, x, x^2, x^3, ...] in v8. - slli t0, t4, 2 - .word 218296407 - # v2: [`1`, `1`, `1`, `1`, ...] - .word 1577103703 - # v3: [`0`, `1`, `2`, `3`, ...] - .word 1376297431 - .word 227733591 - # v4: [`1`, 0, `1`, 0, `1`, 0, `1`, 0, ...] - .word 1243816535 - # v6: [`0`, 0, `1`, 0, `2`, 0, `3`, 0, ...] - .word 1244865367 - slli t0, t4, 1 - .word 219344983 - # v8: [1<<0=1, 0, 0, 0, 1<<1=x, 0, 0, 0, 1<<2=x^2, 0, 0, 0, ...] - .word 3594716247 - - # Compute [r-IV0*1, r-IV0*x, r-IV0*x^2, r-IV0*x^3, ...] in v16 - .word 221179991 - .word 1250174039 - .word 2726865015 - - # Compute [IV0*1, IV0*x, IV0*x^2, IV0*x^3, ...] in v28. - # Reverse the bits order back. - .word 1258565207 - - # Prepare the x^n multiplier in v20. The `n` is the aes-xts block number - # in a LMUL=4 register group. - # n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) - # = (VLEN/32) - # We could use vsetvli with `e32, m1` to compute the `n` number. - .word 218133207 - li t1, 1 - sll t0, t1, t0 - .word 3447812183 - .word 1577070679 - .word 3380670551 - .word 1577238615 - .word 3447812183 - .word 1241784407 - .word 221179991 - .word 1577073239 - .word 2785258103 - - j 2f -1: - .word 3439489111 - .word 1271146583 -2: - - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - - .word 221182167 - j 1f - -.Lenc_blocks_128: - .word 221182167 - # load plaintext into v24 - .word 33909767 - # update iv - .word 2739447927 - # reverse the iv's bits order back - .word 1258565207 -1: - .word 797838423 - slli t0, a7, 2 - sub t4, t4, a7 - add a0, a0, t0 - .word 2786307191 - .word 2787191927 - .word 2788240503 - .word 2789289079 - .word 2790337655 - .word 2791386231 - .word 2792434807 - .word 2793483383 - .word 2794531959 - .word 2795580535 - .word 2796661879 - - .word 797838423 - - # store ciphertext - .word 221147223 - .word 33942567 - add a1, a1, t0 - sub t3, t3, a7 - - bnez t4, .Lenc_blocks_128 - - bnez a6, 1f - ret -1: - # slidedown second to last block - addi a7, a7, -4 - .word 3441586263 - # ciphertext - .word 1065929815 - # multiplier - .word 1057540183 - - .word 3439489111 - .word 1577848023 - - # load last block into v24 - # note: We should load the last block before store the second to last block - # for in-place operation. - .word 134770775 - .word 33885191 - - # setup `x` multiplier with byte-reversed order - # 0b00000010 => 0b01000000 (0x40) - li t0, 0x40 - .word 3439489111 - .word 1577074263 - .word 3355504727 - .word 1577242199 - - # compute IV for last block - .word 3439489111 - .word 2747836535 - .word 1258565207 - - # store second to last block - .word 201879639 - .word 33918119 - - - # xts last block - .word 3439489111 - .word 797838423 - .word 2786307191 - .word 2787191927 - .word 2788240503 - .word 2789289079 - .word 2790337655 - .word 2791386231 - .word 2792434807 - .word 2793483383 - .word 2794531959 - .word 2795580535 - .word 2796661879 - - .word 797838423 - - # store last block ciphertext - addi a1, a1, -16 - .word 33942567 - - ret -.size aes_xts_enc_128,.-aes_xts_enc_128 -.p2align 3 -aes_xts_enc_256: - # load input - .word 221182167 - .word 33909767 - - li t0, 5 - # We could simplify the initialization steps if we have `block<=1`. - blt t4, t0, 1f - - # Note: We use `vgmul` for GF(2^128) multiplication. The `vgmul` uses - # different order of coefficients. We should use`vbrev8` to reverse the - # data when we use `vgmul`. - .word 3439489111 - .word 1271144535 - .word 221179991 - .word 1577072727 - # v16: [r-IV0, r-IV0, ...] - .word 2785257591 - - # Prepare GF(2^128) multiplier [1, x, x^2, x^3, ...] in v8. - slli t0, t4, 2 - .word 218296407 - # v2: [`1`, `1`, `1`, `1`, ...] - .word 1577103703 - # v3: [`0`, `1`, `2`, `3`, ...] - .word 1376297431 - .word 227733591 - # v4: [`1`, 0, `1`, 0, `1`, 0, `1`, 0, ...] - .word 1243816535 - # v6: [`0`, 0, `1`, 0, `2`, 0, `3`, 0, ...] - .word 1244865367 - slli t0, t4, 1 - .word 219344983 - # v8: [1<<0=1, 0, 0, 0, 1<<1=x, 0, 0, 0, 1<<2=x^2, 0, 0, 0, ...] - .word 3594716247 - - # Compute [r-IV0*1, r-IV0*x, r-IV0*x^2, r-IV0*x^3, ...] in v16 - .word 221179991 - .word 1250174039 - .word 2726865015 - - # Compute [IV0*1, IV0*x, IV0*x^2, IV0*x^3, ...] in v28. - # Reverse the bits order back. - .word 1258565207 - - # Prepare the x^n multiplier in v20. The `n` is the aes-xts block number - # in a LMUL=4 register group. - # n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) - # = (VLEN/32) - # We could use vsetvli with `e32, m1` to compute the `n` number. - .word 218133207 - li t1, 1 - sll t0, t1, t0 - .word 3447812183 - .word 1577070679 - .word 3380670551 - .word 1577238615 - .word 3447812183 - .word 1241784407 - .word 221179991 - .word 1577073239 - .word 2785258103 - - j 2f -1: - .word 3439489111 - .word 1271146583 -2: - - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - addi a3, a3, 16 - .word 34006791 - addi a3, a3, 16 - .word 34006919 - - - .word 221182167 - j 1f - -.Lenc_blocks_256: - .word 221182167 - # load plaintext into v24 - .word 33909767 - # update iv - .word 2739447927 - # reverse the iv's bits order back - .word 1258565207 -1: - .word 797838423 - slli t0, a7, 2 - sub t4, t4, a7 - add a0, a0, t0 - .word 2786307191 - .word 2787191927 - .word 2788240503 - .word 2789289079 - .word 2790337655 - .word 2791386231 - .word 2792434807 - .word 2793483383 - .word 2794531959 - .word 2795580535 - .word 2796629111 - .word 2797677687 - .word 2798726263 - .word 2799774839 - .word 2800856183 - - .word 797838423 - - # store ciphertext - .word 221147223 - .word 33942567 - add a1, a1, t0 - sub t3, t3, a7 - - bnez t4, .Lenc_blocks_256 - - bnez a6, 1f - ret -1: - # slidedown second to last block - addi a7, a7, -4 - .word 3441586263 - # ciphertext - .word 1065929815 - # multiplier - .word 1057540183 - - .word 3439489111 - .word 1577848023 - - # load last block into v24 - # note: We should load the last block before store the second to last block - # for in-place operation. - .word 134770775 - .word 33885191 - - # setup `x` multiplier with byte-reversed order - # 0b00000010 => 0b01000000 (0x40) - li t0, 0x40 - .word 3439489111 - .word 1577074263 - .word 3355504727 - .word 1577242199 - - # compute IV for last block - .word 3439489111 - .word 2747836535 - .word 1258565207 - - # store second to last block - .word 201879639 - .word 33918119 - - - # xts last block - .word 3439489111 - .word 797838423 - .word 2786307191 - .word 2787191927 - .word 2788240503 - .word 2789289079 - .word 2790337655 - .word 2791386231 - .word 2792434807 - .word 2793483383 - .word 2794531959 - .word 2795580535 - .word 2796629111 - .word 2797677687 - .word 2798726263 - .word 2799774839 - .word 2800856183 - - .word 797838423 - - # store last block ciphertext - addi a1, a1, -16 - .word 33942567 - - ret -.size aes_xts_enc_256,.-aes_xts_enc_256 -.p2align 3 -.globl rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt -.type rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt,@function -rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt: - # Load number of rounds - lwu t0, 240(a4) - .word 3439489111 - .word 34074119 - .word 34041479 - .word 2815667831 - addi t0, t0, -1 - addi a4, a4, 16 -1: - .word 34041479 - .word 2815503991 - addi t0, t0, -1 - addi a4, a4, 16 - bnez t0, 1b - .word 34041479 - .word 2815536759 - - - # aes block size is 16 - andi a6, a2, 15 - beqz a6, 1f - sub a2, a2, a6 - addi a2, a2, -16 -1: - # We make the `LENGTH` become e32 length here. - srli t4, a2, 2 - - # Load number of rounds - lwu t0, 240(a3) - li t1, 14 - li t2, 10 - beq t0, t1, aes_xts_dec_256 - beq t0, t2, aes_xts_dec_128 -.size rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt,.-rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt -.p2align 3 -aes_xts_dec_128: - # load input - .word 221182167 - .word 33909767 - - li t0, 5 - # We could simplify the initialization steps if we have `block<=1`. - blt t4, t0, 1f - - # Note: We use `vgmul` for GF(2^128) multiplication. The `vgmul` uses - # different order of coefficients. We should use`vbrev8` to reverse the - # data when we use `vgmul`. - .word 3439489111 - .word 1271144535 - .word 221179991 - .word 1577072727 - # v16: [r-IV0, r-IV0, ...] - .word 2785257591 - - # Prepare GF(2^128) multiplier [1, x, x^2, x^3, ...] in v8. - slli t0, t4, 2 - .word 218296407 - # v2: [`1`, `1`, `1`, `1`, ...] - .word 1577103703 - # v3: [`0`, `1`, `2`, `3`, ...] - .word 1376297431 - .word 227733591 - # v4: [`1`, 0, `1`, 0, `1`, 0, `1`, 0, ...] - .word 1243816535 - # v6: [`0`, 0, `1`, 0, `2`, 0, `3`, 0, ...] - .word 1244865367 - slli t0, t4, 1 - .word 219344983 - # v8: [1<<0=1, 0, 0, 0, 1<<1=x, 0, 0, 0, 1<<2=x^2, 0, 0, 0, ...] - .word 3594716247 - - # Compute [r-IV0*1, r-IV0*x, r-IV0*x^2, r-IV0*x^3, ...] in v16 - .word 221179991 - .word 1250174039 - .word 2726865015 - - # Compute [IV0*1, IV0*x, IV0*x^2, IV0*x^3, ...] in v28. - # Reverse the bits order back. - .word 1258565207 - - # Prepare the x^n multiplier in v20. The `n` is the aes-xts block number - # in a LMUL=4 register group. - # n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) - # = (VLEN/32) - # We could use vsetvli with `e32, m1` to compute the `n` number. - .word 218133207 - li t1, 1 - sll t0, t1, t0 - .word 3447812183 - .word 1577070679 - .word 3380670551 - .word 1577238615 - .word 3447812183 - .word 1241784407 - .word 221179991 - .word 1577073239 - .word 2785258103 - - j 2f -1: - .word 3439489111 - .word 1271146583 -2: - - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - - beqz t4, 2f - - .word 221182167 - j 1f - -.Ldec_blocks_128: - .word 221182167 - # load ciphertext into v24 - .word 33909767 - # update iv - .word 2739447927 - # reverse the iv's bits order back - .word 1258565207 -1: - .word 797838423 - slli t0, a7, 2 - sub t4, t4, a7 - add a0, a0, t0 - .word 2796792951 - .word 2795514999 - .word 2794466423 - .word 2793417847 - .word 2792369271 - .word 2791320695 - .word 2790272119 - .word 2789223543 - .word 2788174967 - .word 2787126391 - .word 2786110583 - - .word 797838423 - - # store plaintext - .word 33942567 - add a1, a1, t0 - - bnez t4, .Ldec_blocks_128 - -2: - bnez a6, 1f - ret -1: - # load second to last block's ciphertext - .word 3439489111 - .word 33909767 - addi a0, a0, 16 - - # setup `x` multiplier with byte-reversed order - # 0b00000010 => 0b01000000 (0x40) - li t0, 0x40 - .word 3439489111 - .word 1577073239 - .word 3355504727 - .word 1577241175 - - beqz a2, 1f - # slidedown third to last block - addi a7, a7, -4 - .word 3441586263 - # multiplier - .word 1057540183 - - # compute IV for last block - .word 3439489111 - .word 2739447927 - .word 1258565207 - - # compute IV for second to last block - .word 2739447927 - .word 1258565335 - j 2f -1: - # compute IV for second to last block - .word 3439489111 - .word 2739447927 - .word 1258565335 -2: - - - ## xts second to last block - .word 3439489111 - .word 797871191 - .word 2796792951 - .word 2795514999 - .word 2794466423 - .word 2793417847 - .word 2792369271 - .word 2791320695 - .word 2790272119 - .word 2789223543 - .word 2788174967 - .word 2787126391 - .word 2786110583 - - .word 797871191 - .word 1577848023 - - # load last block ciphertext - .word 134770775 - .word 33885191 - - # store second to last block plaintext - addi t0, a1, 16 - .word 33721511 - - ## xts last block - .word 3439489111 - .word 797838423 - .word 2796792951 - .word 2795514999 - .word 2794466423 - .word 2793417847 - .word 2792369271 - .word 2791320695 - .word 2790272119 - .word 2789223543 - .word 2788174967 - .word 2787126391 - .word 2786110583 - - .word 797838423 - - # store second to last block plaintext - .word 33942567 - - ret -.size aes_xts_dec_128,.-aes_xts_dec_128 -.p2align 3 -aes_xts_dec_256: - # load input - .word 221182167 - .word 33909767 - - li t0, 5 - # We could simplify the initialization steps if we have `block<=1`. - blt t4, t0, 1f - - # Note: We use `vgmul` for GF(2^128) multiplication. The `vgmul` uses - # different order of coefficients. We should use`vbrev8` to reverse the - # data when we use `vgmul`. - .word 3439489111 - .word 1271144535 - .word 221179991 - .word 1577072727 - # v16: [r-IV0, r-IV0, ...] - .word 2785257591 - - # Prepare GF(2^128) multiplier [1, x, x^2, x^3, ...] in v8. - slli t0, t4, 2 - .word 218296407 - # v2: [`1`, `1`, `1`, `1`, ...] - .word 1577103703 - # v3: [`0`, `1`, `2`, `3`, ...] - .word 1376297431 - .word 227733591 - # v4: [`1`, 0, `1`, 0, `1`, 0, `1`, 0, ...] - .word 1243816535 - # v6: [`0`, 0, `1`, 0, `2`, 0, `3`, 0, ...] - .word 1244865367 - slli t0, t4, 1 - .word 219344983 - # v8: [1<<0=1, 0, 0, 0, 1<<1=x, 0, 0, 0, 1<<2=x^2, 0, 0, 0, ...] - .word 3594716247 - - # Compute [r-IV0*1, r-IV0*x, r-IV0*x^2, r-IV0*x^3, ...] in v16 - .word 221179991 - .word 1250174039 - .word 2726865015 - - # Compute [IV0*1, IV0*x, IV0*x^2, IV0*x^3, ...] in v28. - # Reverse the bits order back. - .word 1258565207 - - # Prepare the x^n multiplier in v20. The `n` is the aes-xts block number - # in a LMUL=4 register group. - # n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) - # = (VLEN/32) - # We could use vsetvli with `e32, m1` to compute the `n` number. - .word 218133207 - li t1, 1 - sll t0, t1, t0 - .word 3447812183 - .word 1577070679 - .word 3380670551 - .word 1577238615 - .word 3447812183 - .word 1241784407 - .word 221179991 - .word 1577073239 - .word 2785258103 - - j 2f -1: - .word 3439489111 - .word 1271146583 -2: - - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - addi a3, a3, 16 - .word 34006791 - addi a3, a3, 16 - .word 34006919 - - - beqz t4, 2f - - .word 221182167 - j 1f - -.Ldec_blocks_256: - .word 221182167 - # load ciphertext into v24 - .word 33909767 - # update iv - .word 2739447927 - # reverse the iv's bits order back - .word 1258565207 -1: - .word 797838423 - slli t0, a7, 2 - sub t4, t4, a7 - add a0, a0, t0 - .word 2800987255 - .word 2799709303 - .word 2798660727 - .word 2797612151 - .word 2796563575 - .word 2795514999 - .word 2794466423 - .word 2793417847 - .word 2792369271 - .word 2791320695 - .word 2790272119 - .word 2789223543 - .word 2788174967 - .word 2787126391 - .word 2786110583 - - .word 797838423 - - # store plaintext - .word 33942567 - add a1, a1, t0 - - bnez t4, .Ldec_blocks_256 - -2: - bnez a6, 1f - ret -1: - # load second to last block's ciphertext - .word 3439489111 - .word 33909767 - addi a0, a0, 16 - - # setup `x` multiplier with byte-reversed order - # 0b00000010 => 0b01000000 (0x40) - li t0, 0x40 - .word 3439489111 - .word 1577073239 - .word 3355504727 - .word 1577241175 - - beqz a2, 1f - # slidedown third to last block - addi a7, a7, -4 - .word 3441586263 - # multiplier - .word 1057540183 - - # compute IV for last block - .word 3439489111 - .word 2739447927 - .word 1258565207 - - # compute IV for second to last block - .word 2739447927 - .word 1258565335 - j 2f -1: - # compute IV for second to last block - .word 3439489111 - .word 2739447927 - .word 1258565335 -2: - - - ## xts second to last block - .word 3439489111 - .word 797871191 - .word 2800987255 - .word 2799709303 - .word 2798660727 - .word 2797612151 - .word 2796563575 - .word 2795514999 - .word 2794466423 - .word 2793417847 - .word 2792369271 - .word 2791320695 - .word 2790272119 - .word 2789223543 - .word 2788174967 - .word 2787126391 - .word 2786110583 - - .word 797871191 - .word 1577848023 - - # load last block ciphertext - .word 134770775 - .word 33885191 - - # store second to last block plaintext - addi t0, a1, 16 - .word 33721511 - - ## xts last block - .word 3439489111 - .word 797838423 - .word 2800987255 - .word 2799709303 - .word 2798660727 - .word 2797612151 - .word 2796563575 - .word 2795514999 - .word 2794466423 - .word 2793417847 - .word 2792369271 - .word 2791320695 - .word 2790272119 - .word 2789223543 - .word 2788174967 - .word 2787126391 - .word 2786110583 - - .word 797838423 - - # store second to last block plaintext - .word 33942567 - - ret -.size aes_xts_dec_256,.-aes_xts_dec_256 diff --git a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvkb-zvkned.s b/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvkb-zvkned.s deleted file mode 100644 index 2d6a71e35..000000000 --- a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvkb-zvkned.s +++ /dev/null @@ -1,326 +0,0 @@ -.text -.p2align 3 -.globl rv64i_zvkb_zvkned_ctr32_encrypt_blocks -.type rv64i_zvkb_zvkned_ctr32_encrypt_blocks,@function -rv64i_zvkb_zvkned_ctr32_encrypt_blocks: - beqz a2, 1f - - # Load number of rounds - lwu t0, 240(a3) - li t1, 14 - li t2, 12 - li t3, 10 - - slli t5, a2, 2 - - beq t0, t1, ctr32_encrypt_blocks_256 - beq t0, t2, ctr32_encrypt_blocks_192 - beq t0, t3, ctr32_encrypt_blocks_128 - -1: - ret - -.size rv64i_zvkb_zvkned_ctr32_encrypt_blocks,.-rv64i_zvkb_zvkned_ctr32_encrypt_blocks -.p2align 3 -ctr32_encrypt_blocks_128: - # Load all 11 round keys to v1-v11 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - # Setup mask into v0 - # The mask pattern for 4*N-th elements - # mask v0: [000100010001....] - # Note: - # We could setup the mask just for the maximum element length instead of - # the VLMAX. - li t0, 0b10001000 - .word 201356247 - .word 1577238615 - # Load IV. - # v31:[IV0, IV1, IV2, big-endian count] - .word 3439489111 - .word 34041735 - # Convert the big-endian counter into little-endian. - .word 3305271383 - .word 1240772567 - # Splat the IV to v16 - .word 221212759 - .word 1577072727 - .word 2817763447 - # Prepare the ctr pattern into v20 - # v20: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...] - .word 1342712407 - # v16:[IV0, IV1, IV2, count+0, IV0, IV1, IV2, count+1, ...] - .word 86998743 - .word 17434711 - - - ##### AES body - j 2f -1: - .word 86998743 - # Increase ctr in v16. - .word 17811543 -2: - # Load plaintext into v20 - .word 33909255 - slli t0, t4, 2 - srli t6, t4, 2 - sub t5, t5, t4 - add a0, a0, t0 - # Prepare the AES ctr input into v24. - # The ctr data uses big-endian form. - .word 1577585751 - .word 1233431639 - - .word 2786307191 - .word 2787191927 - .word 2788240503 - .word 2789289079 - .word 2790337655 - .word 2791386231 - .word 2792434807 - .word 2793483383 - .word 2794531959 - .word 2795580535 - .word 2796661879 - - # ciphertext - .word 797576279 - - # Store the ciphertext. - .word 33942567 - add a1, a1, t0 - - bnez t5, 1b - - ret -.size ctr32_encrypt_blocks_128,.-ctr32_encrypt_blocks_128 -.p2align 3 -ctr32_encrypt_blocks_192: - # Load all 13 round keys to v1-v13 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - - # Setup mask into v0 - # The mask pattern for 4*N-th elements - # mask v0: [000100010001....] - # Note: - # We could setup the mask just for the maximum element length instead of - # the VLMAX. - li t0, 0b10001000 - .word 201356247 - .word 1577238615 - # Load IV. - # v31:[IV0, IV1, IV2, big-endian count] - .word 3439489111 - .word 34041735 - # Convert the big-endian counter into little-endian. - .word 3305271383 - .word 1240772567 - # Splat the IV to v16 - .word 221212759 - .word 1577072727 - .word 2817763447 - # Prepare the ctr pattern into v20 - # v20: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...] - .word 1342712407 - # v16:[IV0, IV1, IV2, count+0, IV0, IV1, IV2, count+1, ...] - .word 86998743 - .word 17434711 - - - ##### AES body - j 2f -1: - .word 86998743 - # Increase ctr in v16. - .word 17811543 -2: - # Load plaintext into v20 - .word 33909255 - slli t0, t4, 2 - srli t6, t4, 2 - sub t5, t5, t4 - add a0, a0, t0 - # Prepare the AES ctr input into v24. - # The ctr data uses big-endian form. - .word 1577585751 - .word 1233431639 - - .word 2786307191 - .word 2787191927 - .word 2788240503 - .word 2789289079 - .word 2790337655 - .word 2791386231 - .word 2792434807 - .word 2793483383 - .word 2794531959 - .word 2795580535 - .word 2796629111 - .word 2797677687 - .word 2798759031 - - # ciphertext - .word 797576279 - - # Store the ciphertext. - .word 33942567 - add a1, a1, t0 - - bnez t5, 1b - - ret -.size ctr32_encrypt_blocks_192,.-ctr32_encrypt_blocks_192 -.p2align 3 -ctr32_encrypt_blocks_256: - # Load all 15 round keys to v1-v15 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - addi a3, a3, 16 - .word 34006791 - addi a3, a3, 16 - .word 34006919 - - # Setup mask into v0 - # The mask pattern for 4*N-th elements - # mask v0: [000100010001....] - # Note: - # We could setup the mask just for the maximum element length instead of - # the VLMAX. - li t0, 0b10001000 - .word 201356247 - .word 1577238615 - # Load IV. - # v31:[IV0, IV1, IV2, big-endian count] - .word 3439489111 - .word 34041735 - # Convert the big-endian counter into little-endian. - .word 3305271383 - .word 1240772567 - # Splat the IV to v16 - .word 221212759 - .word 1577072727 - .word 2817763447 - # Prepare the ctr pattern into v20 - # v20: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...] - .word 1342712407 - # v16:[IV0, IV1, IV2, count+0, IV0, IV1, IV2, count+1, ...] - .word 86998743 - .word 17434711 - - - ##### AES body - j 2f -1: - .word 86998743 - # Increase ctr in v16. - .word 17811543 -2: - # Load plaintext into v20 - .word 33909255 - slli t0, t4, 2 - srli t6, t4, 2 - sub t5, t5, t4 - add a0, a0, t0 - # Prepare the AES ctr input into v24. - # The ctr data uses big-endian form. - .word 1577585751 - .word 1233431639 - - .word 2786307191 - .word 2787191927 - .word 2788240503 - .word 2789289079 - .word 2790337655 - .word 2791386231 - .word 2792434807 - .word 2793483383 - .word 2794531959 - .word 2795580535 - .word 2796629111 - .word 2797677687 - .word 2798726263 - .word 2799774839 - .word 2800856183 - - # ciphertext - .word 797576279 - - # Store the ciphertext. - .word 33942567 - add a1, a1, t0 - - bnez t5, 1b - - ret -.size ctr32_encrypt_blocks_256,.-ctr32_encrypt_blocks_256 diff --git a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvkned.s b/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvkned.s deleted file mode 100644 index 91d1f1394..000000000 --- a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64-zvkned.s +++ /dev/null @@ -1,1401 +0,0 @@ -.text -.p2align 3 -.globl rv64i_zvkned_cbc_encrypt -.type rv64i_zvkned_cbc_encrypt,@function -rv64i_zvkned_cbc_encrypt: - # check whether the length is a multiple of 16 and >= 16 - li t1, 16 - blt a2, t1, L_end - andi t1, a2, 15 - bnez t1, L_end - - # Load number of rounds - lwu t2, 240(a3) - - # Get proper routine for key size - li t0, 10 - beq t2, t0, L_cbc_enc_128 - - li t0, 12 - beq t2, t0, L_cbc_enc_192 - - li t0, 14 - beq t2, t0, L_cbc_enc_256 - - ret -.size rv64i_zvkned_cbc_encrypt,.-rv64i_zvkned_cbc_encrypt -.p2align 3 -L_cbc_enc_128: - # Load all 11 round keys to v1-v11 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - - # Load IV. - .word 34039815 - - .word 33909767 - .word 797445207 - j 2f - -1: - .word 33908871 - .word 797477975 - -2: - # AES body - .word 2786307191 # with round key w[ 0, 3] - .word 2787191927 # with round key w[ 4, 7] - .word 2788240503 # with round key w[ 8,11] - .word 2789289079 # with round key w[12,15] - .word 2790337655 # with round key w[16,19] - .word 2791386231 # with round key w[20,23] - .word 2792434807 # with round key w[24,27] - .word 2793483383 # with round key w[28,31] - .word 2794531959 # with round key w[32,35] - .word 2795580535 # with round key w[36,39] - .word 2796661879 # with round key w[40,43] - - - .word 33942567 - - addi a0, a0, 16 - addi a1, a1, 16 - addi a2, a2, -16 - - bnez a2, 1b - - .word 34040871 - - ret -.size L_cbc_enc_128,.-L_cbc_enc_128 -.p2align 3 -L_cbc_enc_192: - # Load all 13 round keys to v1-v13 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - - - # Load IV. - .word 34039815 - - .word 33909767 - .word 797445207 - j 2f - -1: - .word 33908871 - .word 797477975 - -2: - # AES body - .word 2786307191 # with round key w[ 0, 3] - .word 2787191927 # with round key w[ 4, 7] - .word 2788240503 # with round key w[ 8,11] - .word 2789289079 # with round key w[12,15] - .word 2790337655 # with round key w[16,19] - .word 2791386231 # with round key w[20,23] - .word 2792434807 # with round key w[24,27] - .word 2793483383 # with round key w[28,31] - .word 2794531959 # with round key w[32,35] - .word 2795580535 # with round key w[36,39] - .word 2796629111 # with round key w[40,43] - .word 2797677687 # with round key w[44,47] - .word 2798759031 # with round key w[48,51] - - - .word 33942567 - - addi a0, a0, 16 - addi a1, a1, 16 - addi a2, a2, -16 - - bnez a2, 1b - - .word 34040871 - - ret -.size L_cbc_enc_192,.-L_cbc_enc_192 -.p2align 3 -L_cbc_enc_256: - # Load all 15 round keys to v1-v15 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - addi a3, a3, 16 - .word 34006791 - addi a3, a3, 16 - .word 34006919 - - - # Load IV. - .word 34039815 - - .word 33909767 - .word 797445207 - j 2f - -1: - .word 33908871 - .word 797477975 - -2: - # AES body - .word 2786307191 # with round key w[ 0, 3] - .word 2787191927 # with round key w[ 4, 7] - .word 2788240503 # with round key w[ 8,11] - .word 2789289079 # with round key w[12,15] - .word 2790337655 # with round key w[16,19] - .word 2791386231 # with round key w[20,23] - .word 2792434807 # with round key w[24,27] - .word 2793483383 # with round key w[28,31] - .word 2794531959 # with round key w[32,35] - .word 2795580535 # with round key w[36,39] - .word 2796629111 # with round key w[40,43] - .word 2797677687 # with round key w[44,47] - .word 2798726263 # with round key w[48,51] - .word 2799774839 # with round key w[52,55] - .word 2800856183 # with round key w[56,59] - - - .word 33942567 - - addi a0, a0, 16 - addi a1, a1, 16 - addi a2, a2, -16 - - bnez a2, 1b - - .word 34040871 - - ret -.size L_cbc_enc_256,.-L_cbc_enc_256 -.p2align 3 -.globl rv64i_zvkned_cbc_decrypt -.type rv64i_zvkned_cbc_decrypt,@function -rv64i_zvkned_cbc_decrypt: - # check whether the length is a multiple of 16 and >= 16 - li t1, 16 - blt a2, t1, L_end - andi t1, a2, 15 - bnez t1, L_end - - # Load number of rounds - lwu t2, 240(a3) - - # Get proper routine for key size - li t0, 10 - beq t2, t0, L_cbc_dec_128 - - li t0, 12 - beq t2, t0, L_cbc_dec_192 - - li t0, 14 - beq t2, t0, L_cbc_dec_256 - - ret -.size rv64i_zvkned_cbc_decrypt,.-rv64i_zvkned_cbc_decrypt -.p2align 3 -L_cbc_dec_128: - # Load all 11 round keys to v1-v11 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - - # Load IV. - .word 34039815 - - .word 33909767 - .word 1577846999 - j 2f - -1: - .word 33909767 - .word 1577846999 - addi a1, a1, 16 - -2: - # AES body - .word 2796792951 # with round key w[40,43] - .word 2795514999 # with round key w[36,39] - .word 2794466423 # with round key w[32,35] - .word 2793417847 # with round key w[28,31] - .word 2792369271 # with round key w[24,27] - .word 2791320695 # with round key w[20,23] - .word 2790272119 # with round key w[16,19] - .word 2789223543 # with round key w[12,15] - .word 2788174967 # with round key w[ 8,11] - .word 2787126391 # with round key w[ 4, 7] - .word 2786110583 # with round key w[ 0, 3] - - - .word 797445207 - .word 33942567 - .word 1577617495 - - addi a2, a2, -16 - addi a0, a0, 16 - - bnez a2, 1b - - .word 34039847 - - ret -.size L_cbc_dec_128,.-L_cbc_dec_128 -.p2align 3 -L_cbc_dec_192: - # Load all 13 round keys to v1-v13 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - - - # Load IV. - .word 34039815 - - .word 33909767 - .word 1577846999 - j 2f - -1: - .word 33909767 - .word 1577846999 - addi a1, a1, 16 - -2: - # AES body - .word 2798890103 # with round key w[48,51] - .word 2797612151 # with round key w[44,47] - .word 2796563575 # with round key w[40,43] - .word 2795514999 # with round key w[36,39] - .word 2794466423 # with round key w[32,35] - .word 2793417847 # with round key w[28,31] - .word 2792369271 # with round key w[24,27] - .word 2791320695 # with round key w[20,23] - .word 2790272119 # with round key w[16,19] - .word 2789223543 # with round key w[12,15] - .word 2788174967 # with round key w[ 8,11] - .word 2787126391 # with round key w[ 4, 7] - .word 2786110583 # with round key w[ 0, 3] - - - .word 797445207 - .word 33942567 - .word 1577617495 - - addi a2, a2, -16 - addi a0, a0, 16 - - bnez a2, 1b - - .word 34039847 - - ret -.size L_cbc_dec_192,.-L_cbc_dec_192 -.p2align 3 -L_cbc_dec_256: - # Load all 15 round keys to v1-v15 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - addi a3, a3, 16 - .word 34006791 - addi a3, a3, 16 - .word 34006919 - - - # Load IV. - .word 34039815 - - .word 33909767 - .word 1577846999 - j 2f - -1: - .word 33909767 - .word 1577846999 - addi a1, a1, 16 - -2: - # AES body - .word 2800987255 # with round key w[56,59] - .word 2799709303 # with round key w[52,55] - .word 2798660727 # with round key w[48,51] - .word 2797612151 # with round key w[44,47] - .word 2796563575 # with round key w[40,43] - .word 2795514999 # with round key w[36,39] - .word 2794466423 # with round key w[32,35] - .word 2793417847 # with round key w[28,31] - .word 2792369271 # with round key w[24,27] - .word 2791320695 # with round key w[20,23] - .word 2790272119 # with round key w[16,19] - .word 2789223543 # with round key w[12,15] - .word 2788174967 # with round key w[ 8,11] - .word 2787126391 # with round key w[ 4, 7] - .word 2786110583 # with round key w[ 0, 3] - - - .word 797445207 - .word 33942567 - .word 1577617495 - - addi a2, a2, -16 - addi a0, a0, 16 - - bnez a2, 1b - - .word 34039847 - - ret -.size L_cbc_dec_256,.-L_cbc_dec_256 -.p2align 3 -.globl rv64i_zvkned_ecb_encrypt -.type rv64i_zvkned_ecb_encrypt,@function -rv64i_zvkned_ecb_encrypt: - # Make the LEN become e32 length. - srli t3, a2, 2 - - # Load number of rounds - lwu t2, 240(a3) - - # Get proper routine for key size - li t0, 10 - beq t2, t0, L_ecb_enc_128 - - li t0, 12 - beq t2, t0, L_ecb_enc_192 - - li t0, 14 - beq t2, t0, L_ecb_enc_256 - - ret -.size rv64i_zvkned_ecb_encrypt,.-rv64i_zvkned_ecb_encrypt -.p2align 3 -L_ecb_enc_128: - # Load all 11 round keys to v1-v11 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - -1: - .word 221149271 - slli t0, a6, 2 - sub t3, t3, a6 - - .word 33909767 - - # AES body - .word 2786307191 # with round key w[ 0, 3] - .word 2787191927 # with round key w[ 4, 7] - .word 2788240503 # with round key w[ 8,11] - .word 2789289079 # with round key w[12,15] - .word 2790337655 # with round key w[16,19] - .word 2791386231 # with round key w[20,23] - .word 2792434807 # with round key w[24,27] - .word 2793483383 # with round key w[28,31] - .word 2794531959 # with round key w[32,35] - .word 2795580535 # with round key w[36,39] - .word 2796661879 # with round key w[40,43] - - - .word 33942567 - - add a0, a0, t0 - add a1, a1, t0 - - bnez t3, 1b - - ret -.size L_ecb_enc_128,.-L_ecb_enc_128 -.p2align 3 -L_ecb_enc_192: - # Load all 13 round keys to v1-v13 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - - -1: - .word 221149271 - slli t0, a6, 2 - sub t3, t3, a6 - - .word 33909767 - - # AES body - .word 2786307191 # with round key w[ 0, 3] - .word 2787191927 # with round key w[ 4, 7] - .word 2788240503 # with round key w[ 8,11] - .word 2789289079 # with round key w[12,15] - .word 2790337655 # with round key w[16,19] - .word 2791386231 # with round key w[20,23] - .word 2792434807 # with round key w[24,27] - .word 2793483383 # with round key w[28,31] - .word 2794531959 # with round key w[32,35] - .word 2795580535 # with round key w[36,39] - .word 2796629111 # with round key w[40,43] - .word 2797677687 # with round key w[44,47] - .word 2798759031 # with round key w[48,51] - - - .word 33942567 - - add a0, a0, t0 - add a1, a1, t0 - - bnez t3, 1b - - ret -.size L_ecb_enc_192,.-L_ecb_enc_192 -.p2align 3 -L_ecb_enc_256: - # Load all 15 round keys to v1-v15 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - addi a3, a3, 16 - .word 34006791 - addi a3, a3, 16 - .word 34006919 - - -1: - .word 221149271 - slli t0, a6, 2 - sub t3, t3, a6 - - .word 33909767 - - # AES body - .word 2786307191 # with round key w[ 0, 3] - .word 2787191927 # with round key w[ 4, 7] - .word 2788240503 # with round key w[ 8,11] - .word 2789289079 # with round key w[12,15] - .word 2790337655 # with round key w[16,19] - .word 2791386231 # with round key w[20,23] - .word 2792434807 # with round key w[24,27] - .word 2793483383 # with round key w[28,31] - .word 2794531959 # with round key w[32,35] - .word 2795580535 # with round key w[36,39] - .word 2796629111 # with round key w[40,43] - .word 2797677687 # with round key w[44,47] - .word 2798726263 # with round key w[48,51] - .word 2799774839 # with round key w[52,55] - .word 2800856183 # with round key w[56,59] - - - .word 33942567 - - add a0, a0, t0 - add a1, a1, t0 - - bnez t3, 1b - - ret -.size L_ecb_enc_256,.-L_ecb_enc_256 -.p2align 3 -.globl rv64i_zvkned_ecb_decrypt -.type rv64i_zvkned_ecb_decrypt,@function -rv64i_zvkned_ecb_decrypt: - # Make the LEN become e32 length. - srli t3, a2, 2 - - # Load number of rounds - lwu t2, 240(a3) - - # Get proper routine for key size - li t0, 10 - beq t2, t0, L_ecb_dec_128 - - li t0, 12 - beq t2, t0, L_ecb_dec_192 - - li t0, 14 - beq t2, t0, L_ecb_dec_256 - - ret -.size rv64i_zvkned_ecb_decrypt,.-rv64i_zvkned_ecb_decrypt -.p2align 3 -L_ecb_dec_128: - # Load all 11 round keys to v1-v11 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - -1: - .word 221149271 - slli t0, a6, 2 - sub t3, t3, a6 - - .word 33909767 - - # AES body - .word 2796792951 # with round key w[40,43] - .word 2795514999 # with round key w[36,39] - .word 2794466423 # with round key w[32,35] - .word 2793417847 # with round key w[28,31] - .word 2792369271 # with round key w[24,27] - .word 2791320695 # with round key w[20,23] - .word 2790272119 # with round key w[16,19] - .word 2789223543 # with round key w[12,15] - .word 2788174967 # with round key w[ 8,11] - .word 2787126391 # with round key w[ 4, 7] - .word 2786110583 # with round key w[ 0, 3] - - - .word 33942567 - - add a0, a0, t0 - add a1, a1, t0 - - bnez t3, 1b - - ret -.size L_ecb_dec_128,.-L_ecb_dec_128 -.p2align 3 -L_ecb_dec_192: - # Load all 13 round keys to v1-v13 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - - -1: - .word 221149271 - slli t0, a6, 2 - sub t3, t3, a6 - - .word 33909767 - - # AES body - .word 2798890103 # with round key w[48,51] - .word 2797612151 # with round key w[44,47] - .word 2796563575 # with round key w[40,43] - .word 2795514999 # with round key w[36,39] - .word 2794466423 # with round key w[32,35] - .word 2793417847 # with round key w[28,31] - .word 2792369271 # with round key w[24,27] - .word 2791320695 # with round key w[20,23] - .word 2790272119 # with round key w[16,19] - .word 2789223543 # with round key w[12,15] - .word 2788174967 # with round key w[ 8,11] - .word 2787126391 # with round key w[ 4, 7] - .word 2786110583 # with round key w[ 0, 3] - - - .word 33942567 - - add a0, a0, t0 - add a1, a1, t0 - - bnez t3, 1b - - ret -.size L_ecb_dec_192,.-L_ecb_dec_192 -.p2align 3 -L_ecb_dec_256: - # Load all 15 round keys to v1-v15 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - addi a3, a3, 16 - .word 34006791 - addi a3, a3, 16 - .word 34006919 - - -1: - .word 221149271 - slli t0, a6, 2 - sub t3, t3, a6 - - .word 33909767 - - # AES body - .word 2800987255 # with round key w[56,59] - .word 2799709303 # with round key w[52,55] - .word 2798660727 # with round key w[48,51] - .word 2797612151 # with round key w[44,47] - .word 2796563575 # with round key w[40,43] - .word 2795514999 # with round key w[36,39] - .word 2794466423 # with round key w[32,35] - .word 2793417847 # with round key w[28,31] - .word 2792369271 # with round key w[24,27] - .word 2791320695 # with round key w[20,23] - .word 2790272119 # with round key w[16,19] - .word 2789223543 # with round key w[12,15] - .word 2788174967 # with round key w[ 8,11] - .word 2787126391 # with round key w[ 4, 7] - .word 2786110583 # with round key w[ 0, 3] - - - .word 33942567 - - add a0, a0, t0 - add a1, a1, t0 - - bnez t3, 1b - - ret -.size L_ecb_dec_256,.-L_ecb_dec_256 -.p2align 3 -.globl rv64i_zvkned_set_encrypt_key -.type rv64i_zvkned_set_encrypt_key,@function -rv64i_zvkned_set_encrypt_key: - beqz a0, L_fail_m1 - beqz a2, L_fail_m1 - - # Get proper routine for key size - li t1, 256 - beq a1, t1, L_set_key_256 - li t1, 128 - beq a1, t1, L_set_key_128 - - j L_fail_m2 - -.size rv64i_zvkned_set_encrypt_key,.-rv64i_zvkned_set_encrypt_key -.p2align 3 -.globl rv64i_zvkned_set_decrypt_key -.type rv64i_zvkned_set_decrypt_key,@function -rv64i_zvkned_set_decrypt_key: - beqz a0, L_fail_m1 - beqz a2, L_fail_m1 - - # Get proper routine for key size - li t1, 256 - beq a1, t1, L_set_key_256 - li t1, 128 - beq a1, t1, L_set_key_128 - - j L_fail_m2 - -.size rv64i_zvkned_set_decrypt_key,.-rv64i_zvkned_set_decrypt_key -.p2align 3 -L_set_key_128: - # Store the number of rounds - li t2, 10 - sw t2, 240(a2) - - .word 0xc1027057 - - # Load the key - .word 33907975 - - # Generate keys for round 2-11 into registers v11-v20. - .word 2325784055 # v11 <- rk2 (w[ 4, 7]) - .word 2326865527 # v12 <- rk3 (w[ 8,11]) - .word 2327946999 # v13 <- rk4 (w[12,15]) - .word 2329028471 # v14 <- rk5 (w[16,19]) - .word 2330109943 # v15 <- rk6 (w[20,23]) - .word 2331191415 # v16 <- rk7 (w[24,27]) - .word 2332272887 # v17 <- rk8 (w[28,31]) - .word 2333354359 # v18 <- rk9 (w[32,35]) - .word 2334435831 # v19 <- rk10 (w[36,39]) - .word 2335517303 # v20 <- rk11 (w[40,43]) - - # Store the round keys - .word 33973543 - addi a2, a2, 16 - .word 33973671 - addi a2, a2, 16 - .word 33973799 - addi a2, a2, 16 - .word 33973927 - addi a2, a2, 16 - .word 33974055 - addi a2, a2, 16 - .word 33974183 - addi a2, a2, 16 - .word 33974311 - addi a2, a2, 16 - .word 33974439 - addi a2, a2, 16 - .word 33974567 - addi a2, a2, 16 - .word 33974695 - addi a2, a2, 16 - .word 33974823 - - li a0, 1 - ret -.size L_set_key_128,.-L_set_key_128 -.p2align 3 -L_set_key_256: - # Store the number of rounds - li t2, 14 - sw t2, 240(a2) - - .word 0xc1027057 - - # Load the key - .word 33907975 - addi a0, a0, 16 - .word 33908103 - - .word 1577387607 - .word 2863736439 - .word 1577420503 - .word 2864817911 - .word 1577453399 - .word 2865899383 - .word 1577486295 - .word 2866980855 - .word 1577519191 - .word 2868062327 - .word 1577552087 - .word 2869143799 - .word 1577584983 - .word 2870225271 - .word 1577617879 - .word 2871306743 - .word 1577650775 - .word 2872388215 - .word 1577683671 - .word 2873469687 - .word 1577716567 - .word 2874551159 - .word 1577749463 - .word 2875632631 - .word 1577782359 - .word 2876714103 - - .word 33973543 - addi a2, a2, 16 - .word 33973671 - addi a2, a2, 16 - .word 33973799 - addi a2, a2, 16 - .word 33973927 - addi a2, a2, 16 - .word 33974055 - addi a2, a2, 16 - .word 33974183 - addi a2, a2, 16 - .word 33974311 - addi a2, a2, 16 - .word 33974439 - addi a2, a2, 16 - .word 33974567 - addi a2, a2, 16 - .word 33974695 - addi a2, a2, 16 - .word 33974823 - addi a2, a2, 16 - .word 33974951 - addi a2, a2, 16 - .word 33975079 - addi a2, a2, 16 - .word 33975207 - addi a2, a2, 16 - .word 33975335 - - li a0, 1 - ret -.size L_set_key_256,.-L_set_key_256 -.p2align 3 -.globl rv64i_zvkned_encrypt -.type rv64i_zvkned_encrypt,@function -rv64i_zvkned_encrypt: - # Load number of rounds - lwu t5, 240(a2) - - # Get proper routine for key size - li t6, 14 - beq t5, t6, L_enc_256 - li t6, 10 - beq t5, t6, L_enc_128 - li t6, 12 - beq t5, t6, L_enc_192 - - j L_fail_m2 -.size rv64i_zvkned_encrypt,.-rv64i_zvkned_encrypt -.p2align 3 -L_enc_128: - .word 3439489111 - - .word 33906823 - - .word 33973511 - .word 2795741431 # with round key w[ 0, 3] - addi a2, a2, 16 - .word 33973639 - .word 2796626167 # with round key w[ 4, 7] - addi a2, a2, 16 - .word 33973767 - .word 2797674743 # with round key w[ 8,11] - addi a2, a2, 16 - .word 33973895 - .word 2798723319 # with round key w[12,15] - addi a2, a2, 16 - .word 33974023 - .word 2799771895 # with round key w[16,19] - addi a2, a2, 16 - .word 33974151 - .word 2800820471 # with round key w[20,23] - addi a2, a2, 16 - .word 33974279 - .word 2801869047 # with round key w[24,27] - addi a2, a2, 16 - .word 33974407 - .word 2802917623 # with round key w[28,31] - addi a2, a2, 16 - .word 33974535 - .word 2803966199 # with round key w[32,35] - addi a2, a2, 16 - .word 33974663 - .word 2805014775 # with round key w[36,39] - addi a2, a2, 16 - .word 33974791 - .word 2806096119 # with round key w[40,43] - - .word 33939623 - - ret -.size L_enc_128,.-L_enc_128 -.p2align 3 -L_enc_192: - .word 3439489111 - - .word 33906823 - - .word 33973511 - .word 2795741431 # with round key w[ 0, 3] - addi a2, a2, 16 - .word 33973639 - .word 2796626167 - addi a2, a2, 16 - .word 33973767 - .word 2797674743 - addi a2, a2, 16 - .word 33973895 - .word 2798723319 - addi a2, a2, 16 - .word 33974023 - .word 2799771895 - addi a2, a2, 16 - .word 33974151 - .word 2800820471 - addi a2, a2, 16 - .word 33974279 - .word 2801869047 - addi a2, a2, 16 - .word 33974407 - .word 2802917623 - addi a2, a2, 16 - .word 33974535 - .word 2803966199 - addi a2, a2, 16 - .word 33974663 - .word 2805014775 - addi a2, a2, 16 - .word 33974791 - .word 2806063351 - addi a2, a2, 16 - .word 33974919 - .word 2807111927 - addi a2, a2, 16 - .word 33975047 - .word 2808193271 - - .word 33939623 - ret -.size L_enc_192,.-L_enc_192 -.p2align 3 -L_enc_256: - .word 3439489111 - - .word 33906823 - - .word 33973511 - .word 2795741431 # with round key w[ 0, 3] - addi a2, a2, 16 - .word 33973639 - .word 2796626167 - addi a2, a2, 16 - .word 33973767 - .word 2797674743 - addi a2, a2, 16 - .word 33973895 - .word 2798723319 - addi a2, a2, 16 - .word 33974023 - .word 2799771895 - addi a2, a2, 16 - .word 33974151 - .word 2800820471 - addi a2, a2, 16 - .word 33974279 - .word 2801869047 - addi a2, a2, 16 - .word 33974407 - .word 2802917623 - addi a2, a2, 16 - .word 33974535 - .word 2803966199 - addi a2, a2, 16 - .word 33974663 - .word 2805014775 - addi a2, a2, 16 - .word 33974791 - .word 2806063351 - addi a2, a2, 16 - .word 33974919 - .word 2807111927 - addi a2, a2, 16 - .word 33975047 - .word 2808160503 - addi a2, a2, 16 - .word 33975175 - .word 2809209079 - addi a2, a2, 16 - .word 33975303 - .word 2810290423 - - .word 33939623 - ret -.size L_enc_256,.-L_enc_256 -.p2align 3 -.globl rv64i_zvkned_decrypt -.type rv64i_zvkned_decrypt,@function -rv64i_zvkned_decrypt: - # Load number of rounds - lwu t5, 240(a2) - - # Get proper routine for key size - li t6, 14 - beq t5, t6, L_dec_256 - li t6, 10 - beq t5, t6, L_dec_128 - li t6, 12 - beq t5, t6, L_dec_192 - - j L_fail_m2 -.size rv64i_zvkned_decrypt,.-rv64i_zvkned_decrypt -.p2align 3 -L_dec_128: - .word 3439489111 - - .word 33906823 - - addi a2, a2, 160 - .word 33974791 - .word 2806227191 # with round key w[40,43] - addi a2, a2, -16 - .word 33974663 - .word 2804949239 # with round key w[36,39] - addi a2, a2, -16 - .word 33974535 - .word 2803900663 # with round key w[32,35] - addi a2, a2, -16 - .word 33974407 - .word 2802852087 # with round key w[28,31] - addi a2, a2, -16 - .word 33974279 - .word 2801803511 # with round key w[24,27] - addi a2, a2, -16 - .word 33974151 - .word 2800754935 # with round key w[20,23] - addi a2, a2, -16 - .word 33974023 - .word 2799706359 # with round key w[16,19] - addi a2, a2, -16 - .word 33973895 - .word 2798657783 # with round key w[12,15] - addi a2, a2, -16 - .word 33973767 - .word 2797609207 # with round key w[ 8,11] - addi a2, a2, -16 - .word 33973639 - .word 2796560631 # with round key w[ 4, 7] - addi a2, a2, -16 - .word 33973511 - .word 2795544823 # with round key w[ 0, 3] - - .word 33939623 - - ret -.size L_dec_128,.-L_dec_128 -.p2align 3 -L_dec_192: - .word 3439489111 - - .word 33906823 - - addi a2, a2, 192 - .word 33975047 - .word 2808324343 # with round key w[48,51] - addi a2, a2, -16 - .word 33974919 - .word 2807046391 # with round key w[44,47] - addi a2, a2, -16 - .word 33974791 - .word 2805997815 # with round key w[40,43] - addi a2, a2, -16 - .word 33974663 - .word 2804949239 # with round key w[36,39] - addi a2, a2, -16 - .word 33974535 - .word 2803900663 # with round key w[32,35] - addi a2, a2, -16 - .word 33974407 - .word 2802852087 # with round key w[28,31] - addi a2, a2, -16 - .word 33974279 - .word 2801803511 # with round key w[24,27] - addi a2, a2, -16 - .word 33974151 - .word 2800754935 # with round key w[20,23] - addi a2, a2, -16 - .word 33974023 - .word 2799706359 # with round key w[16,19] - addi a2, a2, -16 - .word 33973895 - .word 2798657783 # with round key w[12,15] - addi a2, a2, -16 - .word 33973767 - .word 2797609207 # with round key w[ 8,11] - addi a2, a2, -16 - .word 33973639 - .word 2796560631 # with round key w[ 4, 7] - addi a2, a2, -16 - .word 33973511 - .word 2795544823 # with round key w[ 0, 3] - - .word 33939623 - - ret -.size L_dec_192,.-L_dec_192 -.p2align 3 -L_dec_256: - .word 3439489111 - - .word 33906823 - - addi a2, a2, 224 - .word 33975303 - .word 2810421495 # with round key w[56,59] - addi a2, a2, -16 - .word 33975175 - .word 2809143543 # with round key w[52,55] - addi a2, a2, -16 - .word 33975047 - .word 2808094967 # with round key w[48,51] - addi a2, a2, -16 - .word 33974919 - .word 2807046391 # with round key w[44,47] - addi a2, a2, -16 - .word 33974791 - .word 2805997815 # with round key w[40,43] - addi a2, a2, -16 - .word 33974663 - .word 2804949239 # with round key w[36,39] - addi a2, a2, -16 - .word 33974535 - .word 2803900663 # with round key w[32,35] - addi a2, a2, -16 - .word 33974407 - .word 2802852087 # with round key w[28,31] - addi a2, a2, -16 - .word 33974279 - .word 2801803511 # with round key w[24,27] - addi a2, a2, -16 - .word 33974151 - .word 2800754935 # with round key w[20,23] - addi a2, a2, -16 - .word 33974023 - .word 2799706359 # with round key w[16,19] - addi a2, a2, -16 - .word 33973895 - .word 2798657783 # with round key w[12,15] - addi a2, a2, -16 - .word 33973767 - .word 2797609207 # with round key w[ 8,11] - addi a2, a2, -16 - .word 33973639 - .word 2796560631 # with round key w[ 4, 7] - addi a2, a2, -16 - .word 33973511 - .word 2795544823 # with round key w[ 0, 3] - - .word 33939623 - - ret -.size L_dec_256,.-L_dec_256 -L_fail_m1: - li a0, -1 - ret -.size L_fail_m1,.-L_fail_m1 - -L_fail_m2: - li a0, -2 - ret -.size L_fail_m2,.-L_fail_m2 - -L_end: - ret -.size L_end,.-L_end diff --git a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64.s b/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64.s deleted file mode 100644 index b8fb9e72e..000000000 --- a/openssl/src/crypto/aes/gen/linux_riscv64/aes-riscv64.s +++ /dev/null @@ -1,1870 +0,0 @@ -.text -.balign 16 -.globl AES_encrypt -.type AES_encrypt,@function -AES_encrypt: - addi sp,sp,-96 - sd x8,88(sp) - sd x9,80(sp) - sd x18,72(sp) - sd x19,64(sp) - sd x20,56(sp) - sd x21,48(sp) - sd x22,40(sp) - sd x23,32(sp) - sd x24,24(sp) - sd x25,16(sp) - sd x26,8(sp) - sd x27,0(sp) - - # Load input to block cipher - ld x6,0(x10) - ld x8,8(x10) - - - # Load key - ld x13,0(x12) - ld x15,8(x12) - - - # Load number of rounds - lwu x30,240(x12) - - # Load address of substitution table and wrap-around mask - la x31,AES_Te0 - li x5,~0xFFF - - # y = n xor k, stored in Q0-Q3 - - xor x6,x6,x13 - xor x8,x8,x15 - srli x7,x6,32 - srli x9,x8,32 - - # The main loop only executes the first N-1 rounds. - add x30,x30,-1 - - # Do Nr - 1 rounds (final round is special) - -1: - - slli x13,x6,0+2 - slli x14,x7,0+2 - slli x15,x8,0+2 - slli x16,x9,0+2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lwu x17,0(x29) - add x29,x31,x14 - lwu x18,0(x29) - add x29,x31,x15 - lwu x19,0(x29) - add x29,x31,x16 - lwu x20,0(x29) - - add x31,x31,1024 - - srli x13,x7,8-2 - srli x14,x8,8-2 - srli x15,x9,8-2 - srli x16,x6,8-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lwu x21,0(x29) - add x29,x31,x14 - lwu x22,0(x29) - add x29,x31,x15 - lwu x23,0(x29) - add x29,x31,x16 - lwu x24,0(x29) - - add x31,x31,1024 - - srli x13,x8,16-2 - srli x14,x9,16-2 - srli x15,x6,16-2 - srli x16,x7,16-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lwu x25,0(x29) - add x29,x31,x14 - lwu x26,0(x29) - add x29,x31,x15 - lwu x27,0(x29) - add x29,x31,x16 - lwu x28,0(x29) - - add x31,x31,1024 - - srli x13,x9,24-2 - srli x14,x6,24-2 - srli x15,x7,24-2 - srli x16,x8,24-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lwu x13,0(x29) - add x29,x31,x14 - lwu x14,0(x29) - add x29,x31,x15 - lwu x15,0(x29) - add x29,x31,x16 - lwu x16,0(x29) - - - # Combine table lookups - xor x17,x17,x21 - xor x18,x18,x22 - xor x19,x19,x23 - xor x20,x20,x24 - - xor x17,x17,x25 - xor x18,x18,x26 - xor x19,x19,x27 - xor x20,x20,x28 - - xor x13,x13,x17 - xor x14,x14,x18 - xor x15,x15,x19 - xor x16,x16,x20 - - # Update key ptr to point to next key in schedule - add x12,x12,16 - - # Grab next key in schedule - ld x17,0(x12) - ld x19,8(x12) - - # Round TBL back to 4k boundary - and x31,x31,x5 - - add x30,x30,-1 - - xor x6,x13,x17 - xor x8,x15,x19 - srli x18,x17,32 - xor x7,x14,x18 - srli x20,x19,32 - xor x9,x16,x20 - - bgtz x30,1b - -#================================FINAL ROUND==================================== - -# In the final round, all lookup table accesses would appear as follows: -# -# ... compute index I0 -# add I0,TBL,T0 -# lbu T0,1(I0) -# -# Instead of indexing with a 1 offset, we can add 1 to the TBL pointer, and use -# a 0 offset when indexing in the following code. This enables some instruction -# fusion opportunities. - - add x31,x31,1 - - ld x30,16(x12) - ld x12,24(x12) - - slli x13,x6,0+2 - slli x14,x7,0+2 - slli x15,x8,0+2 - slli x16,x9,0+2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lbu x17,0(x29) - add x29,x31,x14 - lbu x18,0(x29) - add x29,x31,x15 - lbu x19,0(x29) - add x29,x31,x16 - lbu x20,0(x29) - - - srli x13,x7,8-2 - srli x14,x8,8-2 - srli x15,x9,8-2 - srli x16,x6,8-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lbu x21,0(x29) - add x29,x31,x14 - lbu x22,0(x29) - add x29,x31,x15 - lbu x23,0(x29) - add x29,x31,x16 - lbu x24,0(x29) - - - srli x13,x8,16-2 - srli x14,x9,16-2 - srli x15,x6,16-2 - srli x16,x7,16-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lbu x25,0(x29) - add x29,x31,x14 - lbu x26,0(x29) - add x29,x31,x15 - lbu x27,0(x29) - add x29,x31,x16 - lbu x28,0(x29) - - - srli x13,x9,24-2 - srli x14,x6,24-2 - srli x15,x7,24-2 - srli x16,x8,24-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lbu x13,0(x29) - add x29,x31,x14 - lbu x14,0(x29) - add x29,x31,x15 - lbu x15,0(x29) - add x29,x31,x16 - lbu x16,0(x29) - - - # Combine table lookups into T0 and T2 - - slli x18,x18,32 - slli x20,x20,32 - slli x21,x21,8 - slli x22,x22,8+32 - slli x23,x23,8 - slli x24,x24,8+32 - slli x25,x25,16 - slli x26,x26,16+32 - slli x27,x27,16 - slli x28,x28,16+32 - - slli x13,x13,24 - slli x14,x14,24+32 - slli x15,x15,24 - slli x16,x16,24+32 - - xor x17,x17,x13 - xor x18,x18,x14 - xor x19,x19,x15 - xor x20,x20,x16 - - xor x21,x21,x25 - xor x22,x22,x26 - xor x23,x23,x27 - xor x24,x24,x28 - - xor x13,x17,x21 - xor x14,x18,x22 - xor x15,x19,x23 - xor x16,x20,x24 - - - xor x13,x13,x14 - # T0 = [T1 T13 T9 T5 T0 T12 T8 T4] - xor x13,x13,x30 # XOR in key - - xor x15,x15,x16 - # T2 = [T3 T15 T11 T7 T2 T14 T10 T6] - xor x15,x15,x12 # XOR in key - - sd x13,0(x11) - sd x15,8(x11) - - # Pop registers and return -2: - ld x8,88(sp) - ld x9,80(sp) - ld x18,72(sp) - ld x19,64(sp) - ld x20,56(sp) - ld x21,48(sp) - ld x22,40(sp) - ld x23,32(sp) - ld x24,24(sp) - ld x25,16(sp) - ld x26,8(sp) - ld x27,0(sp) - addi sp,sp,96 - ret -.text -.balign 16 -.globl AES_decrypt -.type AES_decrypt,@function -AES_decrypt: - addi sp,sp,-96 - sd x8,88(sp) - sd x9,80(sp) - sd x18,72(sp) - sd x19,64(sp) - sd x20,56(sp) - sd x21,48(sp) - sd x22,40(sp) - sd x23,32(sp) - sd x24,24(sp) - sd x25,16(sp) - sd x26,8(sp) - sd x27,0(sp) - - # Load input to block cipher - ld x6,0(x10) - ld x8,8(x10) - - # Load key - # Note that key is assumed in BE byte order - # (This routine was written against a key scheduling implementation that - # placed keys in BE byte order.) - ld x13,0(x12) - ld x15,8(x12) - - # Load number of rounds - lwu x30,240(x12) - - # Load address of substitution table and wrap-around mask - la x31,AES_Td0 - li x5,~0xFFF - - xor x6,x6,x13 - xor x8,x8,x15 - srli x7,x6,32 - srli x9,x8,32 - - # The main loop only executes the first N-1 rounds. - add x30,x30,-1 - - # Do Nr - 1 rounds (final round is special) -1: - - slli x13,x6,0+2 - slli x14,x7,0+2 - slli x15,x8,0+2 - slli x16,x9,0+2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lwu x17,0(x29) - add x29,x31,x14 - lwu x18,0(x29) - add x29,x31,x15 - lwu x19,0(x29) - add x29,x31,x16 - lwu x20,0(x29) - - add x31,x31,1024 - - srli x13,x9,8-2 - srli x14,x6,8-2 - srli x15,x7,8-2 - srli x16,x8,8-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lwu x21,0(x29) - add x29,x31,x14 - lwu x22,0(x29) - add x29,x31,x15 - lwu x23,0(x29) - add x29,x31,x16 - lwu x24,0(x29) - - add x31,x31,1024 - - srli x13,x8,16-2 - srli x14,x9,16-2 - srli x15,x6,16-2 - srli x16,x7,16-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lwu x25,0(x29) - add x29,x31,x14 - lwu x26,0(x29) - add x29,x31,x15 - lwu x27,0(x29) - add x29,x31,x16 - lwu x28,0(x29) - - add x31,x31,1024 - - srli x13,x7,24-2 - srli x14,x8,24-2 - srli x15,x9,24-2 - srli x16,x6,24-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table. - add x29,x31,x13 - lwu x13,0(x29) - add x29,x31,x14 - lwu x14,0(x29) - add x29,x31,x15 - lwu x15,0(x29) - add x29,x31,x16 - lwu x16,0(x29) - - xor x17,x17,x21 - xor x18,x18,x22 - xor x19,x19,x23 - xor x20,x20,x24 - - xor x17,x17,x25 - xor x18,x18,x26 - xor x19,x19,x27 - xor x20,x20,x28 - - xor x13,x13,x17 - xor x14,x14,x18 - xor x15,x15,x19 - xor x16,x16,x20 - - # Update key ptr to point to next key in schedule - add x12,x12,16 - - # Grab next key in schedule - ld x17,0(x12) - ld x19,8(x12) - - # Round TBL back to 4k boundary - and x31,x31,x5 - - add x30,x30,-1 - - xor x6,x13,x17 - xor x8,x15,x19 - srli x18,x17,32 - xor x7,x14,x18 - srli x20,x19,32 - xor x9,x16,x20 - - bgtz x30,1b - -#================================FINAL ROUND==================================== - - la x31,AES_Td4 - - # K0,K1 are aliases for loopcntr,KEYP - # As these registers will no longer be used after these loads, reuse them - # to store the final key in the schedule. - ld x30,16(x12) - ld x12,24(x12) - srli x14,x9,8 - srli x15,x8,16 - srli x16,x7,24 - - andi x13,x6,0xFF - andi x14,x14,0xFF - andi x15,x15,0xFF - andi x16,x16,0xFF - - add x29,x31,x13 - lbu x17,0(x29) - add x29,x31,x14 - lbu x18,0(x29) - add x29,x31,x15 - lbu x19,0(x29) - add x29,x31,x16 - lbu x20,0(x29) - - srli x14,x6,8 - srli x15,x9,16 - srli x16,x8,24 - - andi x13,x7,0xFF - andi x14,x14,0xFF - andi x15,x15,0xFF - andi x16,x16,0xFF - - add x29,x31,x13 - lbu x21,0(x29) - add x29,x31,x14 - lbu x22,0(x29) - add x29,x31,x15 - lbu x23,0(x29) - add x29,x31,x16 - lbu x24,0(x29) - - srli x14,x7,8 - srli x15,x6,16 - srli x16,x9,24 - - andi x13,x8,0xFF - andi x14,x14,0xFF - andi x15,x15,0xFF - andi x16,x16,0xFF - - add x29,x31,x13 - lbu x25,0(x29) - add x29,x31,x14 - lbu x26,0(x29) - add x29,x31,x15 - lbu x27,0(x29) - add x29,x31,x16 - lbu x28,0(x29) - - srli x14,x8,8 - srli x15,x7,16 - srli x16,x6,24 - - andi x13,x9,0xFF - andi x14,x14,0xFF - andi x15,x15,0xFF - andi x16,x16,0xFF - - add x29,x31,x13 - lbu x13,0(x29) - add x29,x31,x14 - lbu x14,0(x29) - add x29,x31,x15 - lbu x15,0(x29) - add x29,x31,x16 - lbu x16,0(x29) - - - # T0-T15 now contain the decrypted block, minus xoring with the final round - # key. We pack T0-T15 into the two 64-bit registers T0 and T4, then xor - # in the key and store. - - slli x18,x18,8 - slli x19,x19,16 - slli x20,x20,24 - slli x21,x21,32 - slli x22,x22,8+32 - slli x23,x23,16+32 - slli x24,x24,32+24 - slli x26,x26,8 - slli x27,x27,16 - slli x28,x28,24 - slli x13,x13,32 - slli x14,x14,8+32 - slli x15,x15,16+32 - slli x16,x16,24+32 - - xor x17,x17,x18 - xor x19,x19,x20 - xor x21,x21,x22 - xor x23,x23,x24 - - xor x25,x25,x26 - xor x27,x27,x28 - xor x13,x13,x14 - xor x15,x15,x16 - - xor x17,x17,x19 - xor x21,x21,x23 - xor x25,x25,x27 - xor x13,x13,x15 - - xor x17,x17,x21 - # T4 = [T11 T10 T9 T8 T7 T6 T5 T4] - xor x17,x17,x30 # xor in key - - xor x13,x13,x25 - # T0 = [T3 T2 T1 T0 T15 T14 T13 T12] - xor x13,x13,x12 # xor in key - - sd x17,0(x11) - sd x13,8(x11) - - # Pop registers and return - ld x8,88(sp) - ld x9,80(sp) - ld x18,72(sp) - ld x19,64(sp) - ld x20,56(sp) - ld x21,48(sp) - ld x22,40(sp) - ld x23,32(sp) - ld x24,24(sp) - ld x25,16(sp) - ld x26,8(sp) - ld x27,0(sp) - addi sp,sp,96 - ret -.text -.balign 16 -.globl AES_set_encrypt_key -.type AES_set_encrypt_key,@function -AES_set_encrypt_key: - addi sp,sp,-48 - sd x8,32(sp) - sd x9,24(sp) - sd x18,16(sp) - sd x19,8(sp) - sd x20,0(sp) - bnez x10,1f # if (!userKey || !key) return -1; - bnez x12,1f - li a0,-1 - ret -1: - la x9,AES_rcon - la x20,AES_Te0 - li x28,128 - li x29,192 - li x30,256 - - # Determine number of rounds from key size in bits - bne x11,x28,1f - li x13,10 # key->rounds = 10 if bits == 128 - j 3f -1: - bne x11,x29,2f - li x13,12 # key->rounds = 12 if bits == 192 - j 3f -2: - li x13,14 # key->rounds = 14 if bits == 256 - beq x11,x30,3f - li a0,-2 # If bits != 128, 192, or 256, return -2 - j 5f -3: - ld x6,0(x10) - ld x8,8(x10) - - sw x13,240(x12) - - li x19,0 # == i*4 - - srli x7,x6,32 - srli x13,x8,32 - - sd x6,0(x12) - sd x8,8(x12) - - # if bits == 128 - # jump into loop - beq x11,x28,1f - - ld x14,16(x10) - srli x15,x14,32 - sd x14,16(x12) - - # if bits == 192 - # jump into loop - beq x11,x29,2f - - ld x16,24(x10) - srli x17,x16,32 - sd x16,24(x12) - - # bits == 256 - j 3f -1: - addi x12,x12,16 -1: - - # Round TBL back to 4k boundary - srli x20,x20,12 - slli x20,x20,12 - - # Offset by 1 byte, since Te0[x] = S[x].[03, 01, 01, 02] - # So that, later on, a 0-offset lbu yields S[x].01 == S[x] - addi x20,x20,1 - srli x14,x13,8-2 - srli x15,x13,16-2 - srli x16,x13,24-2 - slli x17,x13,2-0 - - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - andi x17,x17,0x3FC - - # Index into tables Te0-Te3 (spread access across tables to help bring - # them into cache for later) - - add x18,x20,x14 - lbu x14,0(x18) - - add x20,x20,1025 # yes, 1025 - add x18,x20,x15 - lbu x15,0(x18) - - add x20,x20,1025 - add x18,x20,x16 - lbu x16,0(x18) - - add x20,x20,1022 - add x18,x20,x17 - lbu x17,0(x18) - - slli x15,x15,8 - slli x16,x16,16 - slli x17,x17,24 - - xor x14,x14,x15 - xor x16,x16,x17 - xor x14,x14,x16 - add x15,x9,x19 # rcon[i] (i increments by 4 so it can double as - # a word offset) - lwu x15,0(x15) - - addi x19,x19,4 - li x18,10*4 - - xor x6,x6,x14 - xor x6,x6,x15 - xor x7,x7,x6 - xor x8,x8,x7 - xor x13,x13,x8 - - sw x6,0(x12) - sw x7,4(x12) - sw x8,8(x12) - sw x13,12(x12) - - addi x12,x12,16 - - - bne x19,x18,1b - j 4f -2: - addi x12,x12,24 -2: - - # Round TBL back to 4k boundary - srli x20,x20,12 - slli x20,x20,12 - - # Offset by 1 byte, since Te0[x] = S[x].[03, 01, 01, 02] - # So that, later on, a 0-offset lbu yields S[x].01 == S[x] - addi x20,x20,1 - srli x16,x15,8-2 - srli x17,x15,16-2 - srli x28,x15,24-2 - slli x29,x15,2-0 - - andi x16,x16,0x3FC - andi x17,x17,0x3FC - andi x28,x28,0x3FC - andi x29,x29,0x3FC - - # Index into tables Te0-Te3 (spread access across tables to help bring - # them into cache for later) - - add x18,x20,x16 - lbu x16,0(x18) - - add x20,x20,1025 # yes, 1025 - add x18,x20,x17 - lbu x17,0(x18) - - add x20,x20,1025 - add x18,x20,x28 - lbu x28,0(x18) - - add x20,x20,1022 - add x18,x20,x29 - lbu x29,0(x18) - - slli x17,x17,8 - slli x28,x28,16 - slli x29,x29,24 - - xor x16,x16,x17 - xor x28,x28,x29 - xor x16,x16,x28 - add x17,x9,x19 # rcon[i] (i increments by 4 so it can double as - # a word offset) - lwu x17,0(x17) - - addi x19,x19,4 - li x18,8*4 - - xor x6,x6,x16 - xor x6,x6,x17 - xor x7,x7,x6 - xor x8,x8,x7 - xor x13,x13,x8 - - sw x6,0(x12) - sw x7,4(x12) - sw x8,8(x12) - sw x13,12(x12) - - beq x19,x18,4f - - xor x14,x14,x13 - xor x15,x15,x14 - sw x14,16(x12) - sw x15,20(x12) - - addi x12,x12,24 - j 2b -3: - addi x12,x12,32 -3: - - # Round TBL back to 4k boundary - srli x20,x20,12 - slli x20,x20,12 - - # Offset by 1 byte, since Te0[x] = S[x].[03, 01, 01, 02] - # So that, later on, a 0-offset lbu yields S[x].01 == S[x] - addi x20,x20,1 - srli x28,x17,8-2 - srli x29,x17,16-2 - srli x30,x17,24-2 - slli x31,x17,2-0 - - andi x28,x28,0x3FC - andi x29,x29,0x3FC - andi x30,x30,0x3FC - andi x31,x31,0x3FC - - # Index into tables Te0-Te3 (spread access across tables to help bring - # them into cache for later) - - add x18,x20,x28 - lbu x28,0(x18) - - add x20,x20,1025 # yes, 1025 - add x18,x20,x29 - lbu x29,0(x18) - - add x20,x20,1025 - add x18,x20,x30 - lbu x30,0(x18) - - add x20,x20,1022 - add x18,x20,x31 - lbu x31,0(x18) - - slli x29,x29,8 - slli x30,x30,16 - slli x31,x31,24 - - xor x28,x28,x29 - xor x30,x30,x31 - xor x28,x28,x30 - add x29,x9,x19 # rcon[i] (i increments by 4 so it can double as - # a word offset) - lwu x29,0(x29) - - addi x19,x19,4 - li x18,7*4 - - xor x6,x6,x28 - xor x6,x6,x29 - xor x7,x7,x6 - xor x8,x8,x7 - xor x13,x13,x8 - - sw x6,0(x12) - sw x7,4(x12) - sw x8,8(x12) - sw x13,12(x12) - - beq x19,x18,4f - - # Round TBL back to 4k boundary - srli x20,x20,12 - slli x20,x20,12 - - # Offset by 1 byte, since Te0[x] = S[x].[03, 01, 01, 02] - # So that, later on, a 0-offset lbu yields S[x].01 == S[x] - addi x20,x20,1 - slli x28,x13,2-0 - srli x29,x13,8-2 - srli x30,x13,16-2 - srli x31,x13,24-2 - - andi x28,x28,0x3FC - andi x29,x29,0x3FC - andi x30,x30,0x3FC - andi x31,x31,0x3FC - - # Index into tables Te0-Te3 (spread access across tables to help bring - # them into cache for later) - - add x18,x20,x28 - lbu x28,0(x18) - - add x20,x20,1025 # yes, 1025 - add x18,x20,x29 - lbu x29,0(x18) - - add x20,x20,1025 - add x18,x20,x30 - lbu x30,0(x18) - - add x20,x20,1022 - add x18,x20,x31 - lbu x31,0(x18) - - slli x29,x29,8 - slli x30,x30,16 - slli x31,x31,24 - - xor x28,x28,x29 - xor x30,x30,x31 - xor x28,x28,x30 - xor x14,x14,x28 - xor x15,x15,x14 - xor x16,x16,x15 - xor x17,x17,x16 - sw x14,16(x12) - sw x15,20(x12) - sw x16,24(x12) - sw x17,28(x12) - - addi x12,x12,32 - j 3b - -4: # return 0 - li a0,0 -5: # return a0 - ld x8,32(sp) - ld x9,24(sp) - ld x18,16(sp) - ld x19,8(sp) - ld x20,0(sp) - addi sp,sp,48 - ret -.text -.balign 16 -.globl AES_set_decrypt_key -.type AES_set_decrypt_key,@function -AES_set_decrypt_key: - # Call AES_set_encrypt_key first - addi sp,sp,-16 - sd x12,0(sp) # We need to hold onto this! - sd ra,8(sp) - la t0,AES_set_encrypt_key - jalr ra,t0 - ld x12,0(sp) - ld ra,8(sp) - addi sp,sp,16 - bgez a0,1f # If error, return error - ret -1: - addi sp,sp,-48 - sd x8,40(sp) - sd x9,32(sp) - sd x18,24(sp) - sd x19,16(sp) - sd x20,8(sp) - sd x21,0(sp) - - li x13,0 - lwu x17,240(x12) - slli x14,x17,4 - # Invert order of round keys -1: - add x19,x12,x13 - ld x6,0(x19) - ld x7,8(x19) - add x18,x12,x14 - ld x8,0(x18) - ld x9,8(x18) - addi x13,x13,16 - addi x14,x14,-16 - sd x6,0(x18) - sd x7,8(x18) - sd x8,0(x19) - sd x9,8(x19) - blt x13,x14,1b - - li x20,1 - -1: - addi x12,x12,16 - lwu x6,0(x12) - lwu x7,4(x12) - lwu x8,8(x12) - lwu x9,12(x12) - - la x21,AES_Te2 - - slli x13,x6,2 - srli x14,x6,8-2 - srli x15,x6,16-2 - srli x16,x6,24-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table Te2 - - add x19,x21,x13 - lwu x13,0(x19) - - add x19,x21,x14 - lwu x14,0(x19) - - add x19,x21,x15 - lwu x15,0(x19) - - add x19,x21,x16 - lwu x16,0(x19) - - andi x13,x13,0xFF - andi x14,x14,0xFF - andi x15,x15,0xFF - andi x16,x16,0xFF - - slli x13,x13,2 - slli x14,x14,2 - slli x15,x15,2 - slli x16,x16,2 - - la x21,AES_Td0 - - # Lookup in Td0-Td3 - - add x19,x21,x13 - lwu x13,0(x19) - - add x21,x21,1024 - add x19,x21,x14 - lwu x14,0(x19) - - add x21,x21,1024 - add x19,x21,x15 - lwu x15,0(x19) - - add x21,x21,1024 - add x19,x21,x16 - lwu x16,0(x19) - - xor x13,x13,x14 - xor x15,x15,x16 - xor x6,x13,x15 - - la x21,AES_Te2 - - slli x13,x7,2 - srli x14,x7,8-2 - srli x15,x7,16-2 - srli x16,x7,24-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table Te2 - - add x19,x21,x13 - lwu x13,0(x19) - - add x19,x21,x14 - lwu x14,0(x19) - - add x19,x21,x15 - lwu x15,0(x19) - - add x19,x21,x16 - lwu x16,0(x19) - - andi x13,x13,0xFF - andi x14,x14,0xFF - andi x15,x15,0xFF - andi x16,x16,0xFF - - slli x13,x13,2 - slli x14,x14,2 - slli x15,x15,2 - slli x16,x16,2 - - la x21,AES_Td0 - - # Lookup in Td0-Td3 - - add x19,x21,x13 - lwu x13,0(x19) - - add x21,x21,1024 - add x19,x21,x14 - lwu x14,0(x19) - - add x21,x21,1024 - add x19,x21,x15 - lwu x15,0(x19) - - add x21,x21,1024 - add x19,x21,x16 - lwu x16,0(x19) - - xor x13,x13,x14 - xor x15,x15,x16 - xor x7,x13,x15 - - la x21,AES_Te2 - - slli x13,x8,2 - srli x14,x8,8-2 - srli x15,x8,16-2 - srli x16,x8,24-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table Te2 - - add x19,x21,x13 - lwu x13,0(x19) - - add x19,x21,x14 - lwu x14,0(x19) - - add x19,x21,x15 - lwu x15,0(x19) - - add x19,x21,x16 - lwu x16,0(x19) - - andi x13,x13,0xFF - andi x14,x14,0xFF - andi x15,x15,0xFF - andi x16,x16,0xFF - - slli x13,x13,2 - slli x14,x14,2 - slli x15,x15,2 - slli x16,x16,2 - - la x21,AES_Td0 - - # Lookup in Td0-Td3 - - add x19,x21,x13 - lwu x13,0(x19) - - add x21,x21,1024 - add x19,x21,x14 - lwu x14,0(x19) - - add x21,x21,1024 - add x19,x21,x15 - lwu x15,0(x19) - - add x21,x21,1024 - add x19,x21,x16 - lwu x16,0(x19) - - xor x13,x13,x14 - xor x15,x15,x16 - xor x8,x13,x15 - - la x21,AES_Te2 - - slli x13,x9,2 - srli x14,x9,8-2 - srli x15,x9,16-2 - srli x16,x9,24-2 - - andi x13,x13,0x3FC - andi x14,x14,0x3FC - andi x15,x15,0x3FC - andi x16,x16,0x3FC - - # Index into table Te2 - - add x19,x21,x13 - lwu x13,0(x19) - - add x19,x21,x14 - lwu x14,0(x19) - - add x19,x21,x15 - lwu x15,0(x19) - - add x19,x21,x16 - lwu x16,0(x19) - - andi x13,x13,0xFF - andi x14,x14,0xFF - andi x15,x15,0xFF - andi x16,x16,0xFF - - slli x13,x13,2 - slli x14,x14,2 - slli x15,x15,2 - slli x16,x16,2 - - la x21,AES_Td0 - - # Lookup in Td0-Td3 - - add x19,x21,x13 - lwu x13,0(x19) - - add x21,x21,1024 - add x19,x21,x14 - lwu x14,0(x19) - - add x21,x21,1024 - add x19,x21,x15 - lwu x15,0(x19) - - add x21,x21,1024 - add x19,x21,x16 - lwu x16,0(x19) - - xor x13,x13,x14 - xor x15,x15,x16 - xor x9,x13,x15 - sw x6,0(x12) - sw x7,4(x12) - sw x8,8(x12) - sw x9,12(x12) - addi x20,x20,1 - blt x20,x17,1b - ld x8,40(sp) - ld x9,32(sp) - ld x18,24(sp) - ld x19,16(sp) - ld x20,8(sp) - ld x21,0(sp) - addi sp,sp,48 - li a0,0 - ret - -.section .rodata -.p2align 12 -.type AES_Te0,@object -AES_Te0: -.word 0xa56363c6U, 0x847c7cf8U, 0x997777eeU, 0x8d7b7bf6U -.word 0x0df2f2ffU, 0xbd6b6bd6U, 0xb16f6fdeU, 0x54c5c591U -.word 0x50303060U, 0x03010102U, 0xa96767ceU, 0x7d2b2b56U -.word 0x19fefee7U, 0x62d7d7b5U, 0xe6abab4dU, 0x9a7676ecU -.word 0x45caca8fU, 0x9d82821fU, 0x40c9c989U, 0x877d7dfaU -.word 0x15fafaefU, 0xeb5959b2U, 0xc947478eU, 0x0bf0f0fbU -.word 0xecadad41U, 0x67d4d4b3U, 0xfda2a25fU, 0xeaafaf45U -.word 0xbf9c9c23U, 0xf7a4a453U, 0x967272e4U, 0x5bc0c09bU -.word 0xc2b7b775U, 0x1cfdfde1U, 0xae93933dU, 0x6a26264cU -.word 0x5a36366cU, 0x413f3f7eU, 0x02f7f7f5U, 0x4fcccc83U -.word 0x5c343468U, 0xf4a5a551U, 0x34e5e5d1U, 0x08f1f1f9U -.word 0x937171e2U, 0x73d8d8abU, 0x53313162U, 0x3f15152aU -.word 0x0c040408U, 0x52c7c795U, 0x65232346U, 0x5ec3c39dU -.word 0x28181830U, 0xa1969637U, 0x0f05050aU, 0xb59a9a2fU -.word 0x0907070eU, 0x36121224U, 0x9b80801bU, 0x3de2e2dfU -.word 0x26ebebcdU, 0x6927274eU, 0xcdb2b27fU, 0x9f7575eaU -.word 0x1b090912U, 0x9e83831dU, 0x742c2c58U, 0x2e1a1a34U -.word 0x2d1b1b36U, 0xb26e6edcU, 0xee5a5ab4U, 0xfba0a05bU -.word 0xf65252a4U, 0x4d3b3b76U, 0x61d6d6b7U, 0xceb3b37dU -.word 0x7b292952U, 0x3ee3e3ddU, 0x712f2f5eU, 0x97848413U -.word 0xf55353a6U, 0x68d1d1b9U, 0x00000000U, 0x2cededc1U -.word 0x60202040U, 0x1ffcfce3U, 0xc8b1b179U, 0xed5b5bb6U -.word 0xbe6a6ad4U, 0x46cbcb8dU, 0xd9bebe67U, 0x4b393972U -.word 0xde4a4a94U, 0xd44c4c98U, 0xe85858b0U, 0x4acfcf85U -.word 0x6bd0d0bbU, 0x2aefefc5U, 0xe5aaaa4fU, 0x16fbfbedU -.word 0xc5434386U, 0xd74d4d9aU, 0x55333366U, 0x94858511U -.word 0xcf45458aU, 0x10f9f9e9U, 0x06020204U, 0x817f7ffeU -.word 0xf05050a0U, 0x443c3c78U, 0xba9f9f25U, 0xe3a8a84bU -.word 0xf35151a2U, 0xfea3a35dU, 0xc0404080U, 0x8a8f8f05U -.word 0xad92923fU, 0xbc9d9d21U, 0x48383870U, 0x04f5f5f1U -.word 0xdfbcbc63U, 0xc1b6b677U, 0x75dadaafU, 0x63212142U -.word 0x30101020U, 0x1affffe5U, 0x0ef3f3fdU, 0x6dd2d2bfU -.word 0x4ccdcd81U, 0x140c0c18U, 0x35131326U, 0x2fececc3U -.word 0xe15f5fbeU, 0xa2979735U, 0xcc444488U, 0x3917172eU -.word 0x57c4c493U, 0xf2a7a755U, 0x827e7efcU, 0x473d3d7aU -.word 0xac6464c8U, 0xe75d5dbaU, 0x2b191932U, 0x957373e6U -.word 0xa06060c0U, 0x98818119U, 0xd14f4f9eU, 0x7fdcdca3U -.word 0x66222244U, 0x7e2a2a54U, 0xab90903bU, 0x8388880bU -.word 0xca46468cU, 0x29eeeec7U, 0xd3b8b86bU, 0x3c141428U -.word 0x79dedea7U, 0xe25e5ebcU, 0x1d0b0b16U, 0x76dbdbadU -.word 0x3be0e0dbU, 0x56323264U, 0x4e3a3a74U, 0x1e0a0a14U -.word 0xdb494992U, 0x0a06060cU, 0x6c242448U, 0xe45c5cb8U -.word 0x5dc2c29fU, 0x6ed3d3bdU, 0xefacac43U, 0xa66262c4U -.word 0xa8919139U, 0xa4959531U, 0x37e4e4d3U, 0x8b7979f2U -.word 0x32e7e7d5U, 0x43c8c88bU, 0x5937376eU, 0xb76d6ddaU -.word 0x8c8d8d01U, 0x64d5d5b1U, 0xd24e4e9cU, 0xe0a9a949U -.word 0xb46c6cd8U, 0xfa5656acU, 0x07f4f4f3U, 0x25eaeacfU -.word 0xaf6565caU, 0x8e7a7af4U, 0xe9aeae47U, 0x18080810U -.word 0xd5baba6fU, 0x887878f0U, 0x6f25254aU, 0x722e2e5cU -.word 0x241c1c38U, 0xf1a6a657U, 0xc7b4b473U, 0x51c6c697U -.word 0x23e8e8cbU, 0x7cdddda1U, 0x9c7474e8U, 0x211f1f3eU -.word 0xdd4b4b96U, 0xdcbdbd61U, 0x868b8b0dU, 0x858a8a0fU -.word 0x907070e0U, 0x423e3e7cU, 0xc4b5b571U, 0xaa6666ccU -.word 0xd8484890U, 0x05030306U, 0x01f6f6f7U, 0x120e0e1cU -.word 0xa36161c2U, 0x5f35356aU, 0xf95757aeU, 0xd0b9b969U -.word 0x91868617U, 0x58c1c199U, 0x271d1d3aU, 0xb99e9e27U -.word 0x38e1e1d9U, 0x13f8f8ebU, 0xb398982bU, 0x33111122U -.word 0xbb6969d2U, 0x70d9d9a9U, 0x898e8e07U, 0xa7949433U -.word 0xb69b9b2dU, 0x221e1e3cU, 0x92878715U, 0x20e9e9c9U -.word 0x49cece87U, 0xff5555aaU, 0x78282850U, 0x7adfdfa5U -.word 0x8f8c8c03U, 0xf8a1a159U, 0x80898909U, 0x170d0d1aU -.word 0xdabfbf65U, 0x31e6e6d7U, 0xc6424284U, 0xb86868d0U -.word 0xc3414182U, 0xb0999929U, 0x772d2d5aU, 0x110f0f1eU -.word 0xcbb0b07bU, 0xfc5454a8U, 0xd6bbbb6dU, 0x3a16162cU - -.type AES_Te1,@object -AES_Te1: -.word 0x6363c6a5U, 0x7c7cf884U, 0x7777ee99U, 0x7b7bf68dU -.word 0xf2f2ff0dU, 0x6b6bd6bdU, 0x6f6fdeb1U, 0xc5c59154U -.word 0x30306050U, 0x01010203U, 0x6767cea9U, 0x2b2b567dU -.word 0xfefee719U, 0xd7d7b562U, 0xabab4de6U, 0x7676ec9aU -.word 0xcaca8f45U, 0x82821f9dU, 0xc9c98940U, 0x7d7dfa87U -.word 0xfafaef15U, 0x5959b2ebU, 0x47478ec9U, 0xf0f0fb0bU -.word 0xadad41ecU, 0xd4d4b367U, 0xa2a25ffdU, 0xafaf45eaU -.word 0x9c9c23bfU, 0xa4a453f7U, 0x7272e496U, 0xc0c09b5bU -.word 0xb7b775c2U, 0xfdfde11cU, 0x93933daeU, 0x26264c6aU -.word 0x36366c5aU, 0x3f3f7e41U, 0xf7f7f502U, 0xcccc834fU -.word 0x3434685cU, 0xa5a551f4U, 0xe5e5d134U, 0xf1f1f908U -.word 0x7171e293U, 0xd8d8ab73U, 0x31316253U, 0x15152a3fU -.word 0x0404080cU, 0xc7c79552U, 0x23234665U, 0xc3c39d5eU -.word 0x18183028U, 0x969637a1U, 0x05050a0fU, 0x9a9a2fb5U -.word 0x07070e09U, 0x12122436U, 0x80801b9bU, 0xe2e2df3dU -.word 0xebebcd26U, 0x27274e69U, 0xb2b27fcdU, 0x7575ea9fU -.word 0x0909121bU, 0x83831d9eU, 0x2c2c5874U, 0x1a1a342eU -.word 0x1b1b362dU, 0x6e6edcb2U, 0x5a5ab4eeU, 0xa0a05bfbU -.word 0x5252a4f6U, 0x3b3b764dU, 0xd6d6b761U, 0xb3b37dceU -.word 0x2929527bU, 0xe3e3dd3eU, 0x2f2f5e71U, 0x84841397U -.word 0x5353a6f5U, 0xd1d1b968U, 0x00000000U, 0xededc12cU -.word 0x20204060U, 0xfcfce31fU, 0xb1b179c8U, 0x5b5bb6edU -.word 0x6a6ad4beU, 0xcbcb8d46U, 0xbebe67d9U, 0x3939724bU -.word 0x4a4a94deU, 0x4c4c98d4U, 0x5858b0e8U, 0xcfcf854aU -.word 0xd0d0bb6bU, 0xefefc52aU, 0xaaaa4fe5U, 0xfbfbed16U -.word 0x434386c5U, 0x4d4d9ad7U, 0x33336655U, 0x85851194U -.word 0x45458acfU, 0xf9f9e910U, 0x02020406U, 0x7f7ffe81U -.word 0x5050a0f0U, 0x3c3c7844U, 0x9f9f25baU, 0xa8a84be3U -.word 0x5151a2f3U, 0xa3a35dfeU, 0x404080c0U, 0x8f8f058aU -.word 0x92923fadU, 0x9d9d21bcU, 0x38387048U, 0xf5f5f104U -.word 0xbcbc63dfU, 0xb6b677c1U, 0xdadaaf75U, 0x21214263U -.word 0x10102030U, 0xffffe51aU, 0xf3f3fd0eU, 0xd2d2bf6dU -.word 0xcdcd814cU, 0x0c0c1814U, 0x13132635U, 0xececc32fU -.word 0x5f5fbee1U, 0x979735a2U, 0x444488ccU, 0x17172e39U -.word 0xc4c49357U, 0xa7a755f2U, 0x7e7efc82U, 0x3d3d7a47U -.word 0x6464c8acU, 0x5d5dbae7U, 0x1919322bU, 0x7373e695U -.word 0x6060c0a0U, 0x81811998U, 0x4f4f9ed1U, 0xdcdca37fU -.word 0x22224466U, 0x2a2a547eU, 0x90903babU, 0x88880b83U -.word 0x46468ccaU, 0xeeeec729U, 0xb8b86bd3U, 0x1414283cU -.word 0xdedea779U, 0x5e5ebce2U, 0x0b0b161dU, 0xdbdbad76U -.word 0xe0e0db3bU, 0x32326456U, 0x3a3a744eU, 0x0a0a141eU -.word 0x494992dbU, 0x06060c0aU, 0x2424486cU, 0x5c5cb8e4U -.word 0xc2c29f5dU, 0xd3d3bd6eU, 0xacac43efU, 0x6262c4a6U -.word 0x919139a8U, 0x959531a4U, 0xe4e4d337U, 0x7979f28bU -.word 0xe7e7d532U, 0xc8c88b43U, 0x37376e59U, 0x6d6ddab7U -.word 0x8d8d018cU, 0xd5d5b164U, 0x4e4e9cd2U, 0xa9a949e0U -.word 0x6c6cd8b4U, 0x5656acfaU, 0xf4f4f307U, 0xeaeacf25U -.word 0x6565caafU, 0x7a7af48eU, 0xaeae47e9U, 0x08081018U -.word 0xbaba6fd5U, 0x7878f088U, 0x25254a6fU, 0x2e2e5c72U -.word 0x1c1c3824U, 0xa6a657f1U, 0xb4b473c7U, 0xc6c69751U -.word 0xe8e8cb23U, 0xdddda17cU, 0x7474e89cU, 0x1f1f3e21U -.word 0x4b4b96ddU, 0xbdbd61dcU, 0x8b8b0d86U, 0x8a8a0f85U -.word 0x7070e090U, 0x3e3e7c42U, 0xb5b571c4U, 0x6666ccaaU -.word 0x484890d8U, 0x03030605U, 0xf6f6f701U, 0x0e0e1c12U -.word 0x6161c2a3U, 0x35356a5fU, 0x5757aef9U, 0xb9b969d0U -.word 0x86861791U, 0xc1c19958U, 0x1d1d3a27U, 0x9e9e27b9U -.word 0xe1e1d938U, 0xf8f8eb13U, 0x98982bb3U, 0x11112233U -.word 0x6969d2bbU, 0xd9d9a970U, 0x8e8e0789U, 0x949433a7U -.word 0x9b9b2db6U, 0x1e1e3c22U, 0x87871592U, 0xe9e9c920U -.word 0xcece8749U, 0x5555aaffU, 0x28285078U, 0xdfdfa57aU -.word 0x8c8c038fU, 0xa1a159f8U, 0x89890980U, 0x0d0d1a17U -.word 0xbfbf65daU, 0xe6e6d731U, 0x424284c6U, 0x6868d0b8U -.word 0x414182c3U, 0x999929b0U, 0x2d2d5a77U, 0x0f0f1e11U -.word 0xb0b07bcbU, 0x5454a8fcU, 0xbbbb6dd6U, 0x16162c3aU - -.type AES_Te2,@object -AES_Te2: -.word 0x63c6a563U, 0x7cf8847cU, 0x77ee9977U, 0x7bf68d7bU -.word 0xf2ff0df2U, 0x6bd6bd6bU, 0x6fdeb16fU, 0xc59154c5U -.word 0x30605030U, 0x01020301U, 0x67cea967U, 0x2b567d2bU -.word 0xfee719feU, 0xd7b562d7U, 0xab4de6abU, 0x76ec9a76U -.word 0xca8f45caU, 0x821f9d82U, 0xc98940c9U, 0x7dfa877dU -.word 0xfaef15faU, 0x59b2eb59U, 0x478ec947U, 0xf0fb0bf0U -.word 0xad41ecadU, 0xd4b367d4U, 0xa25ffda2U, 0xaf45eaafU -.word 0x9c23bf9cU, 0xa453f7a4U, 0x72e49672U, 0xc09b5bc0U -.word 0xb775c2b7U, 0xfde11cfdU, 0x933dae93U, 0x264c6a26U -.word 0x366c5a36U, 0x3f7e413fU, 0xf7f502f7U, 0xcc834fccU -.word 0x34685c34U, 0xa551f4a5U, 0xe5d134e5U, 0xf1f908f1U -.word 0x71e29371U, 0xd8ab73d8U, 0x31625331U, 0x152a3f15U -.word 0x04080c04U, 0xc79552c7U, 0x23466523U, 0xc39d5ec3U -.word 0x18302818U, 0x9637a196U, 0x050a0f05U, 0x9a2fb59aU -.word 0x070e0907U, 0x12243612U, 0x801b9b80U, 0xe2df3de2U -.word 0xebcd26ebU, 0x274e6927U, 0xb27fcdb2U, 0x75ea9f75U -.word 0x09121b09U, 0x831d9e83U, 0x2c58742cU, 0x1a342e1aU -.word 0x1b362d1bU, 0x6edcb26eU, 0x5ab4ee5aU, 0xa05bfba0U -.word 0x52a4f652U, 0x3b764d3bU, 0xd6b761d6U, 0xb37dceb3U -.word 0x29527b29U, 0xe3dd3ee3U, 0x2f5e712fU, 0x84139784U -.word 0x53a6f553U, 0xd1b968d1U, 0x00000000U, 0xedc12cedU -.word 0x20406020U, 0xfce31ffcU, 0xb179c8b1U, 0x5bb6ed5bU -.word 0x6ad4be6aU, 0xcb8d46cbU, 0xbe67d9beU, 0x39724b39U -.word 0x4a94de4aU, 0x4c98d44cU, 0x58b0e858U, 0xcf854acfU -.word 0xd0bb6bd0U, 0xefc52aefU, 0xaa4fe5aaU, 0xfbed16fbU -.word 0x4386c543U, 0x4d9ad74dU, 0x33665533U, 0x85119485U -.word 0x458acf45U, 0xf9e910f9U, 0x02040602U, 0x7ffe817fU -.word 0x50a0f050U, 0x3c78443cU, 0x9f25ba9fU, 0xa84be3a8U -.word 0x51a2f351U, 0xa35dfea3U, 0x4080c040U, 0x8f058a8fU -.word 0x923fad92U, 0x9d21bc9dU, 0x38704838U, 0xf5f104f5U -.word 0xbc63dfbcU, 0xb677c1b6U, 0xdaaf75daU, 0x21426321U -.word 0x10203010U, 0xffe51affU, 0xf3fd0ef3U, 0xd2bf6dd2U -.word 0xcd814ccdU, 0x0c18140cU, 0x13263513U, 0xecc32fecU -.word 0x5fbee15fU, 0x9735a297U, 0x4488cc44U, 0x172e3917U -.word 0xc49357c4U, 0xa755f2a7U, 0x7efc827eU, 0x3d7a473dU -.word 0x64c8ac64U, 0x5dbae75dU, 0x19322b19U, 0x73e69573U -.word 0x60c0a060U, 0x81199881U, 0x4f9ed14fU, 0xdca37fdcU -.word 0x22446622U, 0x2a547e2aU, 0x903bab90U, 0x880b8388U -.word 0x468cca46U, 0xeec729eeU, 0xb86bd3b8U, 0x14283c14U -.word 0xdea779deU, 0x5ebce25eU, 0x0b161d0bU, 0xdbad76dbU -.word 0xe0db3be0U, 0x32645632U, 0x3a744e3aU, 0x0a141e0aU -.word 0x4992db49U, 0x060c0a06U, 0x24486c24U, 0x5cb8e45cU -.word 0xc29f5dc2U, 0xd3bd6ed3U, 0xac43efacU, 0x62c4a662U -.word 0x9139a891U, 0x9531a495U, 0xe4d337e4U, 0x79f28b79U -.word 0xe7d532e7U, 0xc88b43c8U, 0x376e5937U, 0x6ddab76dU -.word 0x8d018c8dU, 0xd5b164d5U, 0x4e9cd24eU, 0xa949e0a9U -.word 0x6cd8b46cU, 0x56acfa56U, 0xf4f307f4U, 0xeacf25eaU -.word 0x65caaf65U, 0x7af48e7aU, 0xae47e9aeU, 0x08101808U -.word 0xba6fd5baU, 0x78f08878U, 0x254a6f25U, 0x2e5c722eU -.word 0x1c38241cU, 0xa657f1a6U, 0xb473c7b4U, 0xc69751c6U -.word 0xe8cb23e8U, 0xdda17cddU, 0x74e89c74U, 0x1f3e211fU -.word 0x4b96dd4bU, 0xbd61dcbdU, 0x8b0d868bU, 0x8a0f858aU -.word 0x70e09070U, 0x3e7c423eU, 0xb571c4b5U, 0x66ccaa66U -.word 0x4890d848U, 0x03060503U, 0xf6f701f6U, 0x0e1c120eU -.word 0x61c2a361U, 0x356a5f35U, 0x57aef957U, 0xb969d0b9U -.word 0x86179186U, 0xc19958c1U, 0x1d3a271dU, 0x9e27b99eU -.word 0xe1d938e1U, 0xf8eb13f8U, 0x982bb398U, 0x11223311U -.word 0x69d2bb69U, 0xd9a970d9U, 0x8e07898eU, 0x9433a794U -.word 0x9b2db69bU, 0x1e3c221eU, 0x87159287U, 0xe9c920e9U -.word 0xce8749ceU, 0x55aaff55U, 0x28507828U, 0xdfa57adfU -.word 0x8c038f8cU, 0xa159f8a1U, 0x89098089U, 0x0d1a170dU -.word 0xbf65dabfU, 0xe6d731e6U, 0x4284c642U, 0x68d0b868U -.word 0x4182c341U, 0x9929b099U, 0x2d5a772dU, 0x0f1e110fU -.word 0xb07bcbb0U, 0x54a8fc54U, 0xbb6dd6bbU, 0x162c3a16U - -.type AES_Te3,@object -AES_Te3: -.word 0xc6a56363U, 0xf8847c7cU, 0xee997777U, 0xf68d7b7bU -.word 0xff0df2f2U, 0xd6bd6b6bU, 0xdeb16f6fU, 0x9154c5c5U -.word 0x60503030U, 0x02030101U, 0xcea96767U, 0x567d2b2bU -.word 0xe719fefeU, 0xb562d7d7U, 0x4de6ababU, 0xec9a7676U -.word 0x8f45cacaU, 0x1f9d8282U, 0x8940c9c9U, 0xfa877d7dU -.word 0xef15fafaU, 0xb2eb5959U, 0x8ec94747U, 0xfb0bf0f0U -.word 0x41ecadadU, 0xb367d4d4U, 0x5ffda2a2U, 0x45eaafafU -.word 0x23bf9c9cU, 0x53f7a4a4U, 0xe4967272U, 0x9b5bc0c0U -.word 0x75c2b7b7U, 0xe11cfdfdU, 0x3dae9393U, 0x4c6a2626U -.word 0x6c5a3636U, 0x7e413f3fU, 0xf502f7f7U, 0x834fccccU -.word 0x685c3434U, 0x51f4a5a5U, 0xd134e5e5U, 0xf908f1f1U -.word 0xe2937171U, 0xab73d8d8U, 0x62533131U, 0x2a3f1515U -.word 0x080c0404U, 0x9552c7c7U, 0x46652323U, 0x9d5ec3c3U -.word 0x30281818U, 0x37a19696U, 0x0a0f0505U, 0x2fb59a9aU -.word 0x0e090707U, 0x24361212U, 0x1b9b8080U, 0xdf3de2e2U -.word 0xcd26ebebU, 0x4e692727U, 0x7fcdb2b2U, 0xea9f7575U -.word 0x121b0909U, 0x1d9e8383U, 0x58742c2cU, 0x342e1a1aU -.word 0x362d1b1bU, 0xdcb26e6eU, 0xb4ee5a5aU, 0x5bfba0a0U -.word 0xa4f65252U, 0x764d3b3bU, 0xb761d6d6U, 0x7dceb3b3U -.word 0x527b2929U, 0xdd3ee3e3U, 0x5e712f2fU, 0x13978484U -.word 0xa6f55353U, 0xb968d1d1U, 0x00000000U, 0xc12cededU -.word 0x40602020U, 0xe31ffcfcU, 0x79c8b1b1U, 0xb6ed5b5bU -.word 0xd4be6a6aU, 0x8d46cbcbU, 0x67d9bebeU, 0x724b3939U -.word 0x94de4a4aU, 0x98d44c4cU, 0xb0e85858U, 0x854acfcfU -.word 0xbb6bd0d0U, 0xc52aefefU, 0x4fe5aaaaU, 0xed16fbfbU -.word 0x86c54343U, 0x9ad74d4dU, 0x66553333U, 0x11948585U -.word 0x8acf4545U, 0xe910f9f9U, 0x04060202U, 0xfe817f7fU -.word 0xa0f05050U, 0x78443c3cU, 0x25ba9f9fU, 0x4be3a8a8U -.word 0xa2f35151U, 0x5dfea3a3U, 0x80c04040U, 0x058a8f8fU -.word 0x3fad9292U, 0x21bc9d9dU, 0x70483838U, 0xf104f5f5U -.word 0x63dfbcbcU, 0x77c1b6b6U, 0xaf75dadaU, 0x42632121U -.word 0x20301010U, 0xe51affffU, 0xfd0ef3f3U, 0xbf6dd2d2U -.word 0x814ccdcdU, 0x18140c0cU, 0x26351313U, 0xc32fececU -.word 0xbee15f5fU, 0x35a29797U, 0x88cc4444U, 0x2e391717U -.word 0x9357c4c4U, 0x55f2a7a7U, 0xfc827e7eU, 0x7a473d3dU -.word 0xc8ac6464U, 0xbae75d5dU, 0x322b1919U, 0xe6957373U -.word 0xc0a06060U, 0x19988181U, 0x9ed14f4fU, 0xa37fdcdcU -.word 0x44662222U, 0x547e2a2aU, 0x3bab9090U, 0x0b838888U -.word 0x8cca4646U, 0xc729eeeeU, 0x6bd3b8b8U, 0x283c1414U -.word 0xa779dedeU, 0xbce25e5eU, 0x161d0b0bU, 0xad76dbdbU -.word 0xdb3be0e0U, 0x64563232U, 0x744e3a3aU, 0x141e0a0aU -.word 0x92db4949U, 0x0c0a0606U, 0x486c2424U, 0xb8e45c5cU -.word 0x9f5dc2c2U, 0xbd6ed3d3U, 0x43efacacU, 0xc4a66262U -.word 0x39a89191U, 0x31a49595U, 0xd337e4e4U, 0xf28b7979U -.word 0xd532e7e7U, 0x8b43c8c8U, 0x6e593737U, 0xdab76d6dU -.word 0x018c8d8dU, 0xb164d5d5U, 0x9cd24e4eU, 0x49e0a9a9U -.word 0xd8b46c6cU, 0xacfa5656U, 0xf307f4f4U, 0xcf25eaeaU -.word 0xcaaf6565U, 0xf48e7a7aU, 0x47e9aeaeU, 0x10180808U -.word 0x6fd5babaU, 0xf0887878U, 0x4a6f2525U, 0x5c722e2eU -.word 0x38241c1cU, 0x57f1a6a6U, 0x73c7b4b4U, 0x9751c6c6U -.word 0xcb23e8e8U, 0xa17cddddU, 0xe89c7474U, 0x3e211f1fU -.word 0x96dd4b4bU, 0x61dcbdbdU, 0x0d868b8bU, 0x0f858a8aU -.word 0xe0907070U, 0x7c423e3eU, 0x71c4b5b5U, 0xccaa6666U -.word 0x90d84848U, 0x06050303U, 0xf701f6f6U, 0x1c120e0eU -.word 0xc2a36161U, 0x6a5f3535U, 0xaef95757U, 0x69d0b9b9U -.word 0x17918686U, 0x9958c1c1U, 0x3a271d1dU, 0x27b99e9eU -.word 0xd938e1e1U, 0xeb13f8f8U, 0x2bb39898U, 0x22331111U -.word 0xd2bb6969U, 0xa970d9d9U, 0x07898e8eU, 0x33a79494U -.word 0x2db69b9bU, 0x3c221e1eU, 0x15928787U, 0xc920e9e9U -.word 0x8749ceceU, 0xaaff5555U, 0x50782828U, 0xa57adfdfU -.word 0x038f8c8cU, 0x59f8a1a1U, 0x09808989U, 0x1a170d0dU -.word 0x65dabfbfU, 0xd731e6e6U, 0x84c64242U, 0xd0b86868U -.word 0x82c34141U, 0x29b09999U, 0x5a772d2dU, 0x1e110f0fU -.word 0x7bcbb0b0U, 0xa8fc5454U, 0x6dd6bbbbU, 0x2c3a1616U - -.p2align 12 -.type AES_Td0,@object -AES_Td0: -.word 0x50a7f451U, 0x5365417eU, 0xc3a4171aU, 0x965e273aU -.word 0xcb6bab3bU, 0xf1459d1fU, 0xab58faacU, 0x9303e34bU -.word 0x55fa3020U, 0xf66d76adU, 0x9176cc88U, 0x254c02f5U -.word 0xfcd7e54fU, 0xd7cb2ac5U, 0x80443526U, 0x8fa362b5U -.word 0x495ab1deU, 0x671bba25U, 0x980eea45U, 0xe1c0fe5dU -.word 0x02752fc3U, 0x12f04c81U, 0xa397468dU, 0xc6f9d36bU -.word 0xe75f8f03U, 0x959c9215U, 0xeb7a6dbfU, 0xda595295U -.word 0x2d83bed4U, 0xd3217458U, 0x2969e049U, 0x44c8c98eU -.word 0x6a89c275U, 0x78798ef4U, 0x6b3e5899U, 0xdd71b927U -.word 0xb64fe1beU, 0x17ad88f0U, 0x66ac20c9U, 0xb43ace7dU -.word 0x184adf63U, 0x82311ae5U, 0x60335197U, 0x457f5362U -.word 0xe07764b1U, 0x84ae6bbbU, 0x1ca081feU, 0x942b08f9U -.word 0x58684870U, 0x19fd458fU, 0x876cde94U, 0xb7f87b52U -.word 0x23d373abU, 0xe2024b72U, 0x578f1fe3U, 0x2aab5566U -.word 0x0728ebb2U, 0x03c2b52fU, 0x9a7bc586U, 0xa50837d3U -.word 0xf2872830U, 0xb2a5bf23U, 0xba6a0302U, 0x5c8216edU -.word 0x2b1ccf8aU, 0x92b479a7U, 0xf0f207f3U, 0xa1e2694eU -.word 0xcdf4da65U, 0xd5be0506U, 0x1f6234d1U, 0x8afea6c4U -.word 0x9d532e34U, 0xa055f3a2U, 0x32e18a05U, 0x75ebf6a4U -.word 0x39ec830bU, 0xaaef6040U, 0x069f715eU, 0x51106ebdU -.word 0xf98a213eU, 0x3d06dd96U, 0xae053eddU, 0x46bde64dU -.word 0xb58d5491U, 0x055dc471U, 0x6fd40604U, 0xff155060U -.word 0x24fb9819U, 0x97e9bdd6U, 0xcc434089U, 0x779ed967U -.word 0xbd42e8b0U, 0x888b8907U, 0x385b19e7U, 0xdbeec879U -.word 0x470a7ca1U, 0xe90f427cU, 0xc91e84f8U, 0x00000000U -.word 0x83868009U, 0x48ed2b32U, 0xac70111eU, 0x4e725a6cU -.word 0xfbff0efdU, 0x5638850fU, 0x1ed5ae3dU, 0x27392d36U -.word 0x64d90f0aU, 0x21a65c68U, 0xd1545b9bU, 0x3a2e3624U -.word 0xb1670a0cU, 0x0fe75793U, 0xd296eeb4U, 0x9e919b1bU -.word 0x4fc5c080U, 0xa220dc61U, 0x694b775aU, 0x161a121cU -.word 0x0aba93e2U, 0xe52aa0c0U, 0x43e0223cU, 0x1d171b12U -.word 0x0b0d090eU, 0xadc78bf2U, 0xb9a8b62dU, 0xc8a91e14U -.word 0x8519f157U, 0x4c0775afU, 0xbbdd99eeU, 0xfd607fa3U -.word 0x9f2601f7U, 0xbcf5725cU, 0xc53b6644U, 0x347efb5bU -.word 0x7629438bU, 0xdcc623cbU, 0x68fcedb6U, 0x63f1e4b8U -.word 0xcadc31d7U, 0x10856342U, 0x40229713U, 0x2011c684U -.word 0x7d244a85U, 0xf83dbbd2U, 0x1132f9aeU, 0x6da129c7U -.word 0x4b2f9e1dU, 0xf330b2dcU, 0xec52860dU, 0xd0e3c177U -.word 0x6c16b32bU, 0x99b970a9U, 0xfa489411U, 0x2264e947U -.word 0xc48cfca8U, 0x1a3ff0a0U, 0xd82c7d56U, 0xef903322U -.word 0xc74e4987U, 0xc1d138d9U, 0xfea2ca8cU, 0x360bd498U -.word 0xcf81f5a6U, 0x28de7aa5U, 0x268eb7daU, 0xa4bfad3fU -.word 0xe49d3a2cU, 0x0d927850U, 0x9bcc5f6aU, 0x62467e54U -.word 0xc2138df6U, 0xe8b8d890U, 0x5ef7392eU, 0xf5afc382U -.word 0xbe805d9fU, 0x7c93d069U, 0xa92dd56fU, 0xb31225cfU -.word 0x3b99acc8U, 0xa77d1810U, 0x6e639ce8U, 0x7bbb3bdbU -.word 0x097826cdU, 0xf418596eU, 0x01b79aecU, 0xa89a4f83U -.word 0x656e95e6U, 0x7ee6ffaaU, 0x08cfbc21U, 0xe6e815efU -.word 0xd99be7baU, 0xce366f4aU, 0xd4099feaU, 0xd67cb029U -.word 0xafb2a431U, 0x31233f2aU, 0x3094a5c6U, 0xc066a235U -.word 0x37bc4e74U, 0xa6ca82fcU, 0xb0d090e0U, 0x15d8a733U -.word 0x4a9804f1U, 0xf7daec41U, 0x0e50cd7fU, 0x2ff69117U -.word 0x8dd64d76U, 0x4db0ef43U, 0x544daaccU, 0xdf0496e4U -.word 0xe3b5d19eU, 0x1b886a4cU, 0xb81f2cc1U, 0x7f516546U -.word 0x04ea5e9dU, 0x5d358c01U, 0x737487faU, 0x2e410bfbU -.word 0x5a1d67b3U, 0x52d2db92U, 0x335610e9U, 0x1347d66dU -.word 0x8c61d79aU, 0x7a0ca137U, 0x8e14f859U, 0x893c13ebU -.word 0xee27a9ceU, 0x35c961b7U, 0xede51ce1U, 0x3cb1477aU -.word 0x59dfd29cU, 0x3f73f255U, 0x79ce1418U, 0xbf37c773U -.word 0xeacdf753U, 0x5baafd5fU, 0x146f3ddfU, 0x86db4478U -.word 0x81f3afcaU, 0x3ec468b9U, 0x2c342438U, 0x5f40a3c2U -.word 0x72c31d16U, 0x0c25e2bcU, 0x8b493c28U, 0x41950dffU -.word 0x7101a839U, 0xdeb30c08U, 0x9ce4b4d8U, 0x90c15664U -.word 0x6184cb7bU, 0x70b632d5U, 0x745c6c48U, 0x4257b8d0U - -.type AES_Td1,@object -AES_Td1: -.word 0xa7f45150U, 0x65417e53U, 0xa4171ac3U, 0x5e273a96U -.word 0x6bab3bcbU, 0x459d1ff1U, 0x58faacabU, 0x03e34b93U -.word 0xfa302055U, 0x6d76adf6U, 0x76cc8891U, 0x4c02f525U -.word 0xd7e54ffcU, 0xcb2ac5d7U, 0x44352680U, 0xa362b58fU -.word 0x5ab1de49U, 0x1bba2567U, 0x0eea4598U, 0xc0fe5de1U -.word 0x752fc302U, 0xf04c8112U, 0x97468da3U, 0xf9d36bc6U -.word 0x5f8f03e7U, 0x9c921595U, 0x7a6dbfebU, 0x595295daU -.word 0x83bed42dU, 0x217458d3U, 0x69e04929U, 0xc8c98e44U -.word 0x89c2756aU, 0x798ef478U, 0x3e58996bU, 0x71b927ddU -.word 0x4fe1beb6U, 0xad88f017U, 0xac20c966U, 0x3ace7db4U -.word 0x4adf6318U, 0x311ae582U, 0x33519760U, 0x7f536245U -.word 0x7764b1e0U, 0xae6bbb84U, 0xa081fe1cU, 0x2b08f994U -.word 0x68487058U, 0xfd458f19U, 0x6cde9487U, 0xf87b52b7U -.word 0xd373ab23U, 0x024b72e2U, 0x8f1fe357U, 0xab55662aU -.word 0x28ebb207U, 0xc2b52f03U, 0x7bc5869aU, 0x0837d3a5U -.word 0x872830f2U, 0xa5bf23b2U, 0x6a0302baU, 0x8216ed5cU -.word 0x1ccf8a2bU, 0xb479a792U, 0xf207f3f0U, 0xe2694ea1U -.word 0xf4da65cdU, 0xbe0506d5U, 0x6234d11fU, 0xfea6c48aU -.word 0x532e349dU, 0x55f3a2a0U, 0xe18a0532U, 0xebf6a475U -.word 0xec830b39U, 0xef6040aaU, 0x9f715e06U, 0x106ebd51U -.word 0x8a213ef9U, 0x06dd963dU, 0x053eddaeU, 0xbde64d46U -.word 0x8d5491b5U, 0x5dc47105U, 0xd406046fU, 0x155060ffU -.word 0xfb981924U, 0xe9bdd697U, 0x434089ccU, 0x9ed96777U -.word 0x42e8b0bdU, 0x8b890788U, 0x5b19e738U, 0xeec879dbU -.word 0x0a7ca147U, 0x0f427ce9U, 0x1e84f8c9U, 0x00000000U -.word 0x86800983U, 0xed2b3248U, 0x70111eacU, 0x725a6c4eU -.word 0xff0efdfbU, 0x38850f56U, 0xd5ae3d1eU, 0x392d3627U -.word 0xd90f0a64U, 0xa65c6821U, 0x545b9bd1U, 0x2e36243aU -.word 0x670a0cb1U, 0xe757930fU, 0x96eeb4d2U, 0x919b1b9eU -.word 0xc5c0804fU, 0x20dc61a2U, 0x4b775a69U, 0x1a121c16U -.word 0xba93e20aU, 0x2aa0c0e5U, 0xe0223c43U, 0x171b121dU -.word 0x0d090e0bU, 0xc78bf2adU, 0xa8b62db9U, 0xa91e14c8U -.word 0x19f15785U, 0x0775af4cU, 0xdd99eebbU, 0x607fa3fdU -.word 0x2601f79fU, 0xf5725cbcU, 0x3b6644c5U, 0x7efb5b34U -.word 0x29438b76U, 0xc623cbdcU, 0xfcedb668U, 0xf1e4b863U -.word 0xdc31d7caU, 0x85634210U, 0x22971340U, 0x11c68420U -.word 0x244a857dU, 0x3dbbd2f8U, 0x32f9ae11U, 0xa129c76dU -.word 0x2f9e1d4bU, 0x30b2dcf3U, 0x52860decU, 0xe3c177d0U -.word 0x16b32b6cU, 0xb970a999U, 0x489411faU, 0x64e94722U -.word 0x8cfca8c4U, 0x3ff0a01aU, 0x2c7d56d8U, 0x903322efU -.word 0x4e4987c7U, 0xd138d9c1U, 0xa2ca8cfeU, 0x0bd49836U -.word 0x81f5a6cfU, 0xde7aa528U, 0x8eb7da26U, 0xbfad3fa4U -.word 0x9d3a2ce4U, 0x9278500dU, 0xcc5f6a9bU, 0x467e5462U -.word 0x138df6c2U, 0xb8d890e8U, 0xf7392e5eU, 0xafc382f5U -.word 0x805d9fbeU, 0x93d0697cU, 0x2dd56fa9U, 0x1225cfb3U -.word 0x99acc83bU, 0x7d1810a7U, 0x639ce86eU, 0xbb3bdb7bU -.word 0x7826cd09U, 0x18596ef4U, 0xb79aec01U, 0x9a4f83a8U -.word 0x6e95e665U, 0xe6ffaa7eU, 0xcfbc2108U, 0xe815efe6U -.word 0x9be7bad9U, 0x366f4aceU, 0x099fead4U, 0x7cb029d6U -.word 0xb2a431afU, 0x233f2a31U, 0x94a5c630U, 0x66a235c0U -.word 0xbc4e7437U, 0xca82fca6U, 0xd090e0b0U, 0xd8a73315U -.word 0x9804f14aU, 0xdaec41f7U, 0x50cd7f0eU, 0xf691172fU -.word 0xd64d768dU, 0xb0ef434dU, 0x4daacc54U, 0x0496e4dfU -.word 0xb5d19ee3U, 0x886a4c1bU, 0x1f2cc1b8U, 0x5165467fU -.word 0xea5e9d04U, 0x358c015dU, 0x7487fa73U, 0x410bfb2eU -.word 0x1d67b35aU, 0xd2db9252U, 0x5610e933U, 0x47d66d13U -.word 0x61d79a8cU, 0x0ca1377aU, 0x14f8598eU, 0x3c13eb89U -.word 0x27a9ceeeU, 0xc961b735U, 0xe51ce1edU, 0xb1477a3cU -.word 0xdfd29c59U, 0x73f2553fU, 0xce141879U, 0x37c773bfU -.word 0xcdf753eaU, 0xaafd5f5bU, 0x6f3ddf14U, 0xdb447886U -.word 0xf3afca81U, 0xc468b93eU, 0x3424382cU, 0x40a3c25fU -.word 0xc31d1672U, 0x25e2bc0cU, 0x493c288bU, 0x950dff41U -.word 0x01a83971U, 0xb30c08deU, 0xe4b4d89cU, 0xc1566490U -.word 0x84cb7b61U, 0xb632d570U, 0x5c6c4874U, 0x57b8d042U - -.type AES_Td2,@object -AES_Td2: -.word 0xf45150a7U, 0x417e5365U, 0x171ac3a4U, 0x273a965eU -.word 0xab3bcb6bU, 0x9d1ff145U, 0xfaacab58U, 0xe34b9303U -.word 0x302055faU, 0x76adf66dU, 0xcc889176U, 0x02f5254cU -.word 0xe54ffcd7U, 0x2ac5d7cbU, 0x35268044U, 0x62b58fa3U -.word 0xb1de495aU, 0xba25671bU, 0xea45980eU, 0xfe5de1c0U -.word 0x2fc30275U, 0x4c8112f0U, 0x468da397U, 0xd36bc6f9U -.word 0x8f03e75fU, 0x9215959cU, 0x6dbfeb7aU, 0x5295da59U -.word 0xbed42d83U, 0x7458d321U, 0xe0492969U, 0xc98e44c8U -.word 0xc2756a89U, 0x8ef47879U, 0x58996b3eU, 0xb927dd71U -.word 0xe1beb64fU, 0x88f017adU, 0x20c966acU, 0xce7db43aU -.word 0xdf63184aU, 0x1ae58231U, 0x51976033U, 0x5362457fU -.word 0x64b1e077U, 0x6bbb84aeU, 0x81fe1ca0U, 0x08f9942bU -.word 0x48705868U, 0x458f19fdU, 0xde94876cU, 0x7b52b7f8U -.word 0x73ab23d3U, 0x4b72e202U, 0x1fe3578fU, 0x55662aabU -.word 0xebb20728U, 0xb52f03c2U, 0xc5869a7bU, 0x37d3a508U -.word 0x2830f287U, 0xbf23b2a5U, 0x0302ba6aU, 0x16ed5c82U -.word 0xcf8a2b1cU, 0x79a792b4U, 0x07f3f0f2U, 0x694ea1e2U -.word 0xda65cdf4U, 0x0506d5beU, 0x34d11f62U, 0xa6c48afeU -.word 0x2e349d53U, 0xf3a2a055U, 0x8a0532e1U, 0xf6a475ebU -.word 0x830b39ecU, 0x6040aaefU, 0x715e069fU, 0x6ebd5110U -.word 0x213ef98aU, 0xdd963d06U, 0x3eddae05U, 0xe64d46bdU -.word 0x5491b58dU, 0xc471055dU, 0x06046fd4U, 0x5060ff15U -.word 0x981924fbU, 0xbdd697e9U, 0x4089cc43U, 0xd967779eU -.word 0xe8b0bd42U, 0x8907888bU, 0x19e7385bU, 0xc879dbeeU -.word 0x7ca1470aU, 0x427ce90fU, 0x84f8c91eU, 0x00000000U -.word 0x80098386U, 0x2b3248edU, 0x111eac70U, 0x5a6c4e72U -.word 0x0efdfbffU, 0x850f5638U, 0xae3d1ed5U, 0x2d362739U -.word 0x0f0a64d9U, 0x5c6821a6U, 0x5b9bd154U, 0x36243a2eU -.word 0x0a0cb167U, 0x57930fe7U, 0xeeb4d296U, 0x9b1b9e91U -.word 0xc0804fc5U, 0xdc61a220U, 0x775a694bU, 0x121c161aU -.word 0x93e20abaU, 0xa0c0e52aU, 0x223c43e0U, 0x1b121d17U -.word 0x090e0b0dU, 0x8bf2adc7U, 0xb62db9a8U, 0x1e14c8a9U -.word 0xf1578519U, 0x75af4c07U, 0x99eebbddU, 0x7fa3fd60U -.word 0x01f79f26U, 0x725cbcf5U, 0x6644c53bU, 0xfb5b347eU -.word 0x438b7629U, 0x23cbdcc6U, 0xedb668fcU, 0xe4b863f1U -.word 0x31d7cadcU, 0x63421085U, 0x97134022U, 0xc6842011U -.word 0x4a857d24U, 0xbbd2f83dU, 0xf9ae1132U, 0x29c76da1U -.word 0x9e1d4b2fU, 0xb2dcf330U, 0x860dec52U, 0xc177d0e3U -.word 0xb32b6c16U, 0x70a999b9U, 0x9411fa48U, 0xe9472264U -.word 0xfca8c48cU, 0xf0a01a3fU, 0x7d56d82cU, 0x3322ef90U -.word 0x4987c74eU, 0x38d9c1d1U, 0xca8cfea2U, 0xd498360bU -.word 0xf5a6cf81U, 0x7aa528deU, 0xb7da268eU, 0xad3fa4bfU -.word 0x3a2ce49dU, 0x78500d92U, 0x5f6a9bccU, 0x7e546246U -.word 0x8df6c213U, 0xd890e8b8U, 0x392e5ef7U, 0xc382f5afU -.word 0x5d9fbe80U, 0xd0697c93U, 0xd56fa92dU, 0x25cfb312U -.word 0xacc83b99U, 0x1810a77dU, 0x9ce86e63U, 0x3bdb7bbbU -.word 0x26cd0978U, 0x596ef418U, 0x9aec01b7U, 0x4f83a89aU -.word 0x95e6656eU, 0xffaa7ee6U, 0xbc2108cfU, 0x15efe6e8U -.word 0xe7bad99bU, 0x6f4ace36U, 0x9fead409U, 0xb029d67cU -.word 0xa431afb2U, 0x3f2a3123U, 0xa5c63094U, 0xa235c066U -.word 0x4e7437bcU, 0x82fca6caU, 0x90e0b0d0U, 0xa73315d8U -.word 0x04f14a98U, 0xec41f7daU, 0xcd7f0e50U, 0x91172ff6U -.word 0x4d768dd6U, 0xef434db0U, 0xaacc544dU, 0x96e4df04U -.word 0xd19ee3b5U, 0x6a4c1b88U, 0x2cc1b81fU, 0x65467f51U -.word 0x5e9d04eaU, 0x8c015d35U, 0x87fa7374U, 0x0bfb2e41U -.word 0x67b35a1dU, 0xdb9252d2U, 0x10e93356U, 0xd66d1347U -.word 0xd79a8c61U, 0xa1377a0cU, 0xf8598e14U, 0x13eb893cU -.word 0xa9ceee27U, 0x61b735c9U, 0x1ce1ede5U, 0x477a3cb1U -.word 0xd29c59dfU, 0xf2553f73U, 0x141879ceU, 0xc773bf37U -.word 0xf753eacdU, 0xfd5f5baaU, 0x3ddf146fU, 0x447886dbU -.word 0xafca81f3U, 0x68b93ec4U, 0x24382c34U, 0xa3c25f40U -.word 0x1d1672c3U, 0xe2bc0c25U, 0x3c288b49U, 0x0dff4195U -.word 0xa8397101U, 0x0c08deb3U, 0xb4d89ce4U, 0x566490c1U -.word 0xcb7b6184U, 0x32d570b6U, 0x6c48745cU, 0xb8d04257U - -.type AES_Td3,@object -AES_Td3: -.word 0x5150a7f4U, 0x7e536541U, 0x1ac3a417U, 0x3a965e27U -.word 0x3bcb6babU, 0x1ff1459dU, 0xacab58faU, 0x4b9303e3U -.word 0x2055fa30U, 0xadf66d76U, 0x889176ccU, 0xf5254c02U -.word 0x4ffcd7e5U, 0xc5d7cb2aU, 0x26804435U, 0xb58fa362U -.word 0xde495ab1U, 0x25671bbaU, 0x45980eeaU, 0x5de1c0feU -.word 0xc302752fU, 0x8112f04cU, 0x8da39746U, 0x6bc6f9d3U -.word 0x03e75f8fU, 0x15959c92U, 0xbfeb7a6dU, 0x95da5952U -.word 0xd42d83beU, 0x58d32174U, 0x492969e0U, 0x8e44c8c9U -.word 0x756a89c2U, 0xf478798eU, 0x996b3e58U, 0x27dd71b9U -.word 0xbeb64fe1U, 0xf017ad88U, 0xc966ac20U, 0x7db43aceU -.word 0x63184adfU, 0xe582311aU, 0x97603351U, 0x62457f53U -.word 0xb1e07764U, 0xbb84ae6bU, 0xfe1ca081U, 0xf9942b08U -.word 0x70586848U, 0x8f19fd45U, 0x94876cdeU, 0x52b7f87bU -.word 0xab23d373U, 0x72e2024bU, 0xe3578f1fU, 0x662aab55U -.word 0xb20728ebU, 0x2f03c2b5U, 0x869a7bc5U, 0xd3a50837U -.word 0x30f28728U, 0x23b2a5bfU, 0x02ba6a03U, 0xed5c8216U -.word 0x8a2b1ccfU, 0xa792b479U, 0xf3f0f207U, 0x4ea1e269U -.word 0x65cdf4daU, 0x06d5be05U, 0xd11f6234U, 0xc48afea6U -.word 0x349d532eU, 0xa2a055f3U, 0x0532e18aU, 0xa475ebf6U -.word 0x0b39ec83U, 0x40aaef60U, 0x5e069f71U, 0xbd51106eU -.word 0x3ef98a21U, 0x963d06ddU, 0xddae053eU, 0x4d46bde6U -.word 0x91b58d54U, 0x71055dc4U, 0x046fd406U, 0x60ff1550U -.word 0x1924fb98U, 0xd697e9bdU, 0x89cc4340U, 0x67779ed9U -.word 0xb0bd42e8U, 0x07888b89U, 0xe7385b19U, 0x79dbeec8U -.word 0xa1470a7cU, 0x7ce90f42U, 0xf8c91e84U, 0x00000000U -.word 0x09838680U, 0x3248ed2bU, 0x1eac7011U, 0x6c4e725aU -.word 0xfdfbff0eU, 0x0f563885U, 0x3d1ed5aeU, 0x3627392dU -.word 0x0a64d90fU, 0x6821a65cU, 0x9bd1545bU, 0x243a2e36U -.word 0x0cb1670aU, 0x930fe757U, 0xb4d296eeU, 0x1b9e919bU -.word 0x804fc5c0U, 0x61a220dcU, 0x5a694b77U, 0x1c161a12U -.word 0xe20aba93U, 0xc0e52aa0U, 0x3c43e022U, 0x121d171bU -.word 0x0e0b0d09U, 0xf2adc78bU, 0x2db9a8b6U, 0x14c8a91eU -.word 0x578519f1U, 0xaf4c0775U, 0xeebbdd99U, 0xa3fd607fU -.word 0xf79f2601U, 0x5cbcf572U, 0x44c53b66U, 0x5b347efbU -.word 0x8b762943U, 0xcbdcc623U, 0xb668fcedU, 0xb863f1e4U -.word 0xd7cadc31U, 0x42108563U, 0x13402297U, 0x842011c6U -.word 0x857d244aU, 0xd2f83dbbU, 0xae1132f9U, 0xc76da129U -.word 0x1d4b2f9eU, 0xdcf330b2U, 0x0dec5286U, 0x77d0e3c1U -.word 0x2b6c16b3U, 0xa999b970U, 0x11fa4894U, 0x472264e9U -.word 0xa8c48cfcU, 0xa01a3ff0U, 0x56d82c7dU, 0x22ef9033U -.word 0x87c74e49U, 0xd9c1d138U, 0x8cfea2caU, 0x98360bd4U -.word 0xa6cf81f5U, 0xa528de7aU, 0xda268eb7U, 0x3fa4bfadU -.word 0x2ce49d3aU, 0x500d9278U, 0x6a9bcc5fU, 0x5462467eU -.word 0xf6c2138dU, 0x90e8b8d8U, 0x2e5ef739U, 0x82f5afc3U -.word 0x9fbe805dU, 0x697c93d0U, 0x6fa92dd5U, 0xcfb31225U -.word 0xc83b99acU, 0x10a77d18U, 0xe86e639cU, 0xdb7bbb3bU -.word 0xcd097826U, 0x6ef41859U, 0xec01b79aU, 0x83a89a4fU -.word 0xe6656e95U, 0xaa7ee6ffU, 0x2108cfbcU, 0xefe6e815U -.word 0xbad99be7U, 0x4ace366fU, 0xead4099fU, 0x29d67cb0U -.word 0x31afb2a4U, 0x2a31233fU, 0xc63094a5U, 0x35c066a2U -.word 0x7437bc4eU, 0xfca6ca82U, 0xe0b0d090U, 0x3315d8a7U -.word 0xf14a9804U, 0x41f7daecU, 0x7f0e50cdU, 0x172ff691U -.word 0x768dd64dU, 0x434db0efU, 0xcc544daaU, 0xe4df0496U -.word 0x9ee3b5d1U, 0x4c1b886aU, 0xc1b81f2cU, 0x467f5165U -.word 0x9d04ea5eU, 0x015d358cU, 0xfa737487U, 0xfb2e410bU -.word 0xb35a1d67U, 0x9252d2dbU, 0xe9335610U, 0x6d1347d6U -.word 0x9a8c61d7U, 0x377a0ca1U, 0x598e14f8U, 0xeb893c13U -.word 0xceee27a9U, 0xb735c961U, 0xe1ede51cU, 0x7a3cb147U -.word 0x9c59dfd2U, 0x553f73f2U, 0x1879ce14U, 0x73bf37c7U -.word 0x53eacdf7U, 0x5f5baafdU, 0xdf146f3dU, 0x7886db44U -.word 0xca81f3afU, 0xb93ec468U, 0x382c3424U, 0xc25f40a3U -.word 0x1672c31dU, 0xbc0c25e2U, 0x288b493cU, 0xff41950dU -.word 0x397101a8U, 0x08deb30cU, 0xd89ce4b4U, 0x6490c156U -.word 0x7b6184cbU, 0xd570b632U, 0x48745c6cU, 0xd04257b8U - -.type AES_Td4,@object -AES_Td4: -.byte 0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U -.byte 0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU -.byte 0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U -.byte 0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU -.byte 0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU -.byte 0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU -.byte 0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U -.byte 0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U -.byte 0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U -.byte 0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U -.byte 0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU -.byte 0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U -.byte 0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU -.byte 0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U -.byte 0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U -.byte 0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU -.byte 0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU -.byte 0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U -.byte 0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U -.byte 0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU -.byte 0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U -.byte 0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU -.byte 0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U -.byte 0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U -.byte 0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U -.byte 0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU -.byte 0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU -.byte 0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU -.byte 0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U -.byte 0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U -.byte 0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U -.byte 0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU - -.type AES_rcon,@object -AES_rcon: -.word 0x00000001U, 0x00000002U, 0x00000004U, 0x00000008U -.word 0x00000010U, 0x00000020U, 0x00000040U, 0x00000080U -.word 0x0000001BU, 0x00000036U diff --git a/openssl/src/crypto/aes/gen/windows_ia32/aes-586.asm b/openssl/src/crypto/aes/gen/windows_ia32/aes-586.asm index 699cc2e3b..7a17e847c 100644 --- a/openssl/src/crypto/aes/gen/windows_ia32/aes-586.asm +++ b/openssl/src/crypto/aes/gen/windows_ia32/aes-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/aes/gen/windows_ia32/aesni-x86.asm b/openssl/src/crypto/aes/gen/windows_ia32/aesni-x86.asm index 512b6bf2c..14abad181 100644 --- a/openssl/src/crypto/aes/gen/windows_ia32/aesni-x86.asm +++ b/openssl/src/crypto/aes/gen/windows_ia32/aesni-x86.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/aes/gen/windows_ia32/vpaes-x86.asm b/openssl/src/crypto/aes/gen/windows_ia32/vpaes-x86.asm index 43c9efd34..dff9a4c61 100644 --- a/openssl/src/crypto/aes/gen/windows_ia32/vpaes-x86.asm +++ b/openssl/src/crypto/aes/gen/windows_ia32/vpaes-x86.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/aria/aria.c b/openssl/src/crypto/aria/aria.c deleted file mode 100644 index 84ddd00cd..000000000 --- a/openssl/src/crypto/aria/aria.c +++ /dev/null @@ -1,1212 +0,0 @@ -/* - * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Copyright (C) 2017 National Security Research Institute. All Rights Reserved. - * - * Information for ARIA - * http://210.104.33.10/ARIA/index-e.html (English) - * http://seed.kisa.or.kr/ (Korean) - * - * Public domain version is distributed above. - */ - -#include -#include "crypto/aria.h" - -#include -#include - -#ifndef OPENSSL_SMALL_FOOTPRINT - -/* Begin macro */ - -/* rotation */ -#define rotl32(v, r) (((uint32_t)(v) << (r)) | ((uint32_t)(v) >> (32 - r))) -#define rotr32(v, r) (((uint32_t)(v) >> (r)) | ((uint32_t)(v) << (32 - r))) - -#define bswap32(v) \ - (((v) << 24) ^ ((v) >> 24) ^ \ - (((v) & 0x0000ff00) << 8) ^ (((v) & 0x00ff0000) >> 8)) - -#define GET_U8_BE(X, Y) ((uint8_t)((X) >> ((3 - Y) * 8))) -#define GET_U32_BE(X, Y) ( \ - ((uint32_t)((const uint8_t *)(X))[Y * 4 ] << 24) ^ \ - ((uint32_t)((const uint8_t *)(X))[Y * 4 + 1] << 16) ^ \ - ((uint32_t)((const uint8_t *)(X))[Y * 4 + 2] << 8) ^ \ - ((uint32_t)((const uint8_t *)(X))[Y * 4 + 3] ) ) - -#define PUT_U32_BE(DEST, IDX, VAL) \ - do { \ - ((uint8_t *)(DEST))[IDX * 4 ] = GET_U8_BE(VAL, 0); \ - ((uint8_t *)(DEST))[IDX * 4 + 1] = GET_U8_BE(VAL, 1); \ - ((uint8_t *)(DEST))[IDX * 4 + 2] = GET_U8_BE(VAL, 2); \ - ((uint8_t *)(DEST))[IDX * 4 + 3] = GET_U8_BE(VAL, 3); \ - } while(0) - -#define MAKE_U32(V0, V1, V2, V3) ( \ - ((uint32_t)((uint8_t)(V0)) << 24) | \ - ((uint32_t)((uint8_t)(V1)) << 16) | \ - ((uint32_t)((uint8_t)(V2)) << 8) | \ - ((uint32_t)((uint8_t)(V3)) ) ) - -/* End Macro*/ - -/* Key Constant - * 128bit : 0, 1, 2 - * 192bit : 1, 2, 3(0) - * 256bit : 2, 3(0), 4(1) - */ -static const uint32_t Key_RC[5][4] = { - { 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0 }, - { 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 }, - { 0xdb92371d, 0x2126e970, 0x03249775, 0x04e8c90e }, - { 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0 }, - { 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 } -}; - -/* 32bit expanded s-box */ -static const uint32_t S1[256] = { - 0x00636363, 0x007c7c7c, 0x00777777, 0x007b7b7b, - 0x00f2f2f2, 0x006b6b6b, 0x006f6f6f, 0x00c5c5c5, - 0x00303030, 0x00010101, 0x00676767, 0x002b2b2b, - 0x00fefefe, 0x00d7d7d7, 0x00ababab, 0x00767676, - 0x00cacaca, 0x00828282, 0x00c9c9c9, 0x007d7d7d, - 0x00fafafa, 0x00595959, 0x00474747, 0x00f0f0f0, - 0x00adadad, 0x00d4d4d4, 0x00a2a2a2, 0x00afafaf, - 0x009c9c9c, 0x00a4a4a4, 0x00727272, 0x00c0c0c0, - 0x00b7b7b7, 0x00fdfdfd, 0x00939393, 0x00262626, - 0x00363636, 0x003f3f3f, 0x00f7f7f7, 0x00cccccc, - 0x00343434, 0x00a5a5a5, 0x00e5e5e5, 0x00f1f1f1, - 0x00717171, 0x00d8d8d8, 0x00313131, 0x00151515, - 0x00040404, 0x00c7c7c7, 0x00232323, 0x00c3c3c3, - 0x00181818, 0x00969696, 0x00050505, 0x009a9a9a, - 0x00070707, 0x00121212, 0x00808080, 0x00e2e2e2, - 0x00ebebeb, 0x00272727, 0x00b2b2b2, 0x00757575, - 0x00090909, 0x00838383, 0x002c2c2c, 0x001a1a1a, - 0x001b1b1b, 0x006e6e6e, 0x005a5a5a, 0x00a0a0a0, - 0x00525252, 0x003b3b3b, 0x00d6d6d6, 0x00b3b3b3, - 0x00292929, 0x00e3e3e3, 0x002f2f2f, 0x00848484, - 0x00535353, 0x00d1d1d1, 0x00000000, 0x00ededed, - 0x00202020, 0x00fcfcfc, 0x00b1b1b1, 0x005b5b5b, - 0x006a6a6a, 0x00cbcbcb, 0x00bebebe, 0x00393939, - 0x004a4a4a, 0x004c4c4c, 0x00585858, 0x00cfcfcf, - 0x00d0d0d0, 0x00efefef, 0x00aaaaaa, 0x00fbfbfb, - 0x00434343, 0x004d4d4d, 0x00333333, 0x00858585, - 0x00454545, 0x00f9f9f9, 0x00020202, 0x007f7f7f, - 0x00505050, 0x003c3c3c, 0x009f9f9f, 0x00a8a8a8, - 0x00515151, 0x00a3a3a3, 0x00404040, 0x008f8f8f, - 0x00929292, 0x009d9d9d, 0x00383838, 0x00f5f5f5, - 0x00bcbcbc, 0x00b6b6b6, 0x00dadada, 0x00212121, - 0x00101010, 0x00ffffff, 0x00f3f3f3, 0x00d2d2d2, - 0x00cdcdcd, 0x000c0c0c, 0x00131313, 0x00ececec, - 0x005f5f5f, 0x00979797, 0x00444444, 0x00171717, - 0x00c4c4c4, 0x00a7a7a7, 0x007e7e7e, 0x003d3d3d, - 0x00646464, 0x005d5d5d, 0x00191919, 0x00737373, - 0x00606060, 0x00818181, 0x004f4f4f, 0x00dcdcdc, - 0x00222222, 0x002a2a2a, 0x00909090, 0x00888888, - 0x00464646, 0x00eeeeee, 0x00b8b8b8, 0x00141414, - 0x00dedede, 0x005e5e5e, 0x000b0b0b, 0x00dbdbdb, - 0x00e0e0e0, 0x00323232, 0x003a3a3a, 0x000a0a0a, - 0x00494949, 0x00060606, 0x00242424, 0x005c5c5c, - 0x00c2c2c2, 0x00d3d3d3, 0x00acacac, 0x00626262, - 0x00919191, 0x00959595, 0x00e4e4e4, 0x00797979, - 0x00e7e7e7, 0x00c8c8c8, 0x00373737, 0x006d6d6d, - 0x008d8d8d, 0x00d5d5d5, 0x004e4e4e, 0x00a9a9a9, - 0x006c6c6c, 0x00565656, 0x00f4f4f4, 0x00eaeaea, - 0x00656565, 0x007a7a7a, 0x00aeaeae, 0x00080808, - 0x00bababa, 0x00787878, 0x00252525, 0x002e2e2e, - 0x001c1c1c, 0x00a6a6a6, 0x00b4b4b4, 0x00c6c6c6, - 0x00e8e8e8, 0x00dddddd, 0x00747474, 0x001f1f1f, - 0x004b4b4b, 0x00bdbdbd, 0x008b8b8b, 0x008a8a8a, - 0x00707070, 0x003e3e3e, 0x00b5b5b5, 0x00666666, - 0x00484848, 0x00030303, 0x00f6f6f6, 0x000e0e0e, - 0x00616161, 0x00353535, 0x00575757, 0x00b9b9b9, - 0x00868686, 0x00c1c1c1, 0x001d1d1d, 0x009e9e9e, - 0x00e1e1e1, 0x00f8f8f8, 0x00989898, 0x00111111, - 0x00696969, 0x00d9d9d9, 0x008e8e8e, 0x00949494, - 0x009b9b9b, 0x001e1e1e, 0x00878787, 0x00e9e9e9, - 0x00cecece, 0x00555555, 0x00282828, 0x00dfdfdf, - 0x008c8c8c, 0x00a1a1a1, 0x00898989, 0x000d0d0d, - 0x00bfbfbf, 0x00e6e6e6, 0x00424242, 0x00686868, - 0x00414141, 0x00999999, 0x002d2d2d, 0x000f0f0f, - 0x00b0b0b0, 0x00545454, 0x00bbbbbb, 0x00161616 -}; - -static const uint32_t S2[256] = { - 0xe200e2e2, 0x4e004e4e, 0x54005454, 0xfc00fcfc, - 0x94009494, 0xc200c2c2, 0x4a004a4a, 0xcc00cccc, - 0x62006262, 0x0d000d0d, 0x6a006a6a, 0x46004646, - 0x3c003c3c, 0x4d004d4d, 0x8b008b8b, 0xd100d1d1, - 0x5e005e5e, 0xfa00fafa, 0x64006464, 0xcb00cbcb, - 0xb400b4b4, 0x97009797, 0xbe00bebe, 0x2b002b2b, - 0xbc00bcbc, 0x77007777, 0x2e002e2e, 0x03000303, - 0xd300d3d3, 0x19001919, 0x59005959, 0xc100c1c1, - 0x1d001d1d, 0x06000606, 0x41004141, 0x6b006b6b, - 0x55005555, 0xf000f0f0, 0x99009999, 0x69006969, - 0xea00eaea, 0x9c009c9c, 0x18001818, 0xae00aeae, - 0x63006363, 0xdf00dfdf, 0xe700e7e7, 0xbb00bbbb, - 0x00000000, 0x73007373, 0x66006666, 0xfb00fbfb, - 0x96009696, 0x4c004c4c, 0x85008585, 0xe400e4e4, - 0x3a003a3a, 0x09000909, 0x45004545, 0xaa00aaaa, - 0x0f000f0f, 0xee00eeee, 0x10001010, 0xeb00ebeb, - 0x2d002d2d, 0x7f007f7f, 0xf400f4f4, 0x29002929, - 0xac00acac, 0xcf00cfcf, 0xad00adad, 0x91009191, - 0x8d008d8d, 0x78007878, 0xc800c8c8, 0x95009595, - 0xf900f9f9, 0x2f002f2f, 0xce00cece, 0xcd00cdcd, - 0x08000808, 0x7a007a7a, 0x88008888, 0x38003838, - 0x5c005c5c, 0x83008383, 0x2a002a2a, 0x28002828, - 0x47004747, 0xdb00dbdb, 0xb800b8b8, 0xc700c7c7, - 0x93009393, 0xa400a4a4, 0x12001212, 0x53005353, - 0xff00ffff, 0x87008787, 0x0e000e0e, 0x31003131, - 0x36003636, 0x21002121, 0x58005858, 0x48004848, - 0x01000101, 0x8e008e8e, 0x37003737, 0x74007474, - 0x32003232, 0xca00caca, 0xe900e9e9, 0xb100b1b1, - 0xb700b7b7, 0xab00abab, 0x0c000c0c, 0xd700d7d7, - 0xc400c4c4, 0x56005656, 0x42004242, 0x26002626, - 0x07000707, 0x98009898, 0x60006060, 0xd900d9d9, - 0xb600b6b6, 0xb900b9b9, 0x11001111, 0x40004040, - 0xec00ecec, 0x20002020, 0x8c008c8c, 0xbd00bdbd, - 0xa000a0a0, 0xc900c9c9, 0x84008484, 0x04000404, - 0x49004949, 0x23002323, 0xf100f1f1, 0x4f004f4f, - 0x50005050, 0x1f001f1f, 0x13001313, 0xdc00dcdc, - 0xd800d8d8, 0xc000c0c0, 0x9e009e9e, 0x57005757, - 0xe300e3e3, 0xc300c3c3, 0x7b007b7b, 0x65006565, - 0x3b003b3b, 0x02000202, 0x8f008f8f, 0x3e003e3e, - 0xe800e8e8, 0x25002525, 0x92009292, 0xe500e5e5, - 0x15001515, 0xdd00dddd, 0xfd00fdfd, 0x17001717, - 0xa900a9a9, 0xbf00bfbf, 0xd400d4d4, 0x9a009a9a, - 0x7e007e7e, 0xc500c5c5, 0x39003939, 0x67006767, - 0xfe00fefe, 0x76007676, 0x9d009d9d, 0x43004343, - 0xa700a7a7, 0xe100e1e1, 0xd000d0d0, 0xf500f5f5, - 0x68006868, 0xf200f2f2, 0x1b001b1b, 0x34003434, - 0x70007070, 0x05000505, 0xa300a3a3, 0x8a008a8a, - 0xd500d5d5, 0x79007979, 0x86008686, 0xa800a8a8, - 0x30003030, 0xc600c6c6, 0x51005151, 0x4b004b4b, - 0x1e001e1e, 0xa600a6a6, 0x27002727, 0xf600f6f6, - 0x35003535, 0xd200d2d2, 0x6e006e6e, 0x24002424, - 0x16001616, 0x82008282, 0x5f005f5f, 0xda00dada, - 0xe600e6e6, 0x75007575, 0xa200a2a2, 0xef00efef, - 0x2c002c2c, 0xb200b2b2, 0x1c001c1c, 0x9f009f9f, - 0x5d005d5d, 0x6f006f6f, 0x80008080, 0x0a000a0a, - 0x72007272, 0x44004444, 0x9b009b9b, 0x6c006c6c, - 0x90009090, 0x0b000b0b, 0x5b005b5b, 0x33003333, - 0x7d007d7d, 0x5a005a5a, 0x52005252, 0xf300f3f3, - 0x61006161, 0xa100a1a1, 0xf700f7f7, 0xb000b0b0, - 0xd600d6d6, 0x3f003f3f, 0x7c007c7c, 0x6d006d6d, - 0xed00eded, 0x14001414, 0xe000e0e0, 0xa500a5a5, - 0x3d003d3d, 0x22002222, 0xb300b3b3, 0xf800f8f8, - 0x89008989, 0xde00dede, 0x71007171, 0x1a001a1a, - 0xaf00afaf, 0xba00baba, 0xb500b5b5, 0x81008181 -}; - -static const uint32_t X1[256] = { - 0x52520052, 0x09090009, 0x6a6a006a, 0xd5d500d5, - 0x30300030, 0x36360036, 0xa5a500a5, 0x38380038, - 0xbfbf00bf, 0x40400040, 0xa3a300a3, 0x9e9e009e, - 0x81810081, 0xf3f300f3, 0xd7d700d7, 0xfbfb00fb, - 0x7c7c007c, 0xe3e300e3, 0x39390039, 0x82820082, - 0x9b9b009b, 0x2f2f002f, 0xffff00ff, 0x87870087, - 0x34340034, 0x8e8e008e, 0x43430043, 0x44440044, - 0xc4c400c4, 0xdede00de, 0xe9e900e9, 0xcbcb00cb, - 0x54540054, 0x7b7b007b, 0x94940094, 0x32320032, - 0xa6a600a6, 0xc2c200c2, 0x23230023, 0x3d3d003d, - 0xeeee00ee, 0x4c4c004c, 0x95950095, 0x0b0b000b, - 0x42420042, 0xfafa00fa, 0xc3c300c3, 0x4e4e004e, - 0x08080008, 0x2e2e002e, 0xa1a100a1, 0x66660066, - 0x28280028, 0xd9d900d9, 0x24240024, 0xb2b200b2, - 0x76760076, 0x5b5b005b, 0xa2a200a2, 0x49490049, - 0x6d6d006d, 0x8b8b008b, 0xd1d100d1, 0x25250025, - 0x72720072, 0xf8f800f8, 0xf6f600f6, 0x64640064, - 0x86860086, 0x68680068, 0x98980098, 0x16160016, - 0xd4d400d4, 0xa4a400a4, 0x5c5c005c, 0xcccc00cc, - 0x5d5d005d, 0x65650065, 0xb6b600b6, 0x92920092, - 0x6c6c006c, 0x70700070, 0x48480048, 0x50500050, - 0xfdfd00fd, 0xeded00ed, 0xb9b900b9, 0xdada00da, - 0x5e5e005e, 0x15150015, 0x46460046, 0x57570057, - 0xa7a700a7, 0x8d8d008d, 0x9d9d009d, 0x84840084, - 0x90900090, 0xd8d800d8, 0xabab00ab, 0x00000000, - 0x8c8c008c, 0xbcbc00bc, 0xd3d300d3, 0x0a0a000a, - 0xf7f700f7, 0xe4e400e4, 0x58580058, 0x05050005, - 0xb8b800b8, 0xb3b300b3, 0x45450045, 0x06060006, - 0xd0d000d0, 0x2c2c002c, 0x1e1e001e, 0x8f8f008f, - 0xcaca00ca, 0x3f3f003f, 0x0f0f000f, 0x02020002, - 0xc1c100c1, 0xafaf00af, 0xbdbd00bd, 0x03030003, - 0x01010001, 0x13130013, 0x8a8a008a, 0x6b6b006b, - 0x3a3a003a, 0x91910091, 0x11110011, 0x41410041, - 0x4f4f004f, 0x67670067, 0xdcdc00dc, 0xeaea00ea, - 0x97970097, 0xf2f200f2, 0xcfcf00cf, 0xcece00ce, - 0xf0f000f0, 0xb4b400b4, 0xe6e600e6, 0x73730073, - 0x96960096, 0xacac00ac, 0x74740074, 0x22220022, - 0xe7e700e7, 0xadad00ad, 0x35350035, 0x85850085, - 0xe2e200e2, 0xf9f900f9, 0x37370037, 0xe8e800e8, - 0x1c1c001c, 0x75750075, 0xdfdf00df, 0x6e6e006e, - 0x47470047, 0xf1f100f1, 0x1a1a001a, 0x71710071, - 0x1d1d001d, 0x29290029, 0xc5c500c5, 0x89890089, - 0x6f6f006f, 0xb7b700b7, 0x62620062, 0x0e0e000e, - 0xaaaa00aa, 0x18180018, 0xbebe00be, 0x1b1b001b, - 0xfcfc00fc, 0x56560056, 0x3e3e003e, 0x4b4b004b, - 0xc6c600c6, 0xd2d200d2, 0x79790079, 0x20200020, - 0x9a9a009a, 0xdbdb00db, 0xc0c000c0, 0xfefe00fe, - 0x78780078, 0xcdcd00cd, 0x5a5a005a, 0xf4f400f4, - 0x1f1f001f, 0xdddd00dd, 0xa8a800a8, 0x33330033, - 0x88880088, 0x07070007, 0xc7c700c7, 0x31310031, - 0xb1b100b1, 0x12120012, 0x10100010, 0x59590059, - 0x27270027, 0x80800080, 0xecec00ec, 0x5f5f005f, - 0x60600060, 0x51510051, 0x7f7f007f, 0xa9a900a9, - 0x19190019, 0xb5b500b5, 0x4a4a004a, 0x0d0d000d, - 0x2d2d002d, 0xe5e500e5, 0x7a7a007a, 0x9f9f009f, - 0x93930093, 0xc9c900c9, 0x9c9c009c, 0xefef00ef, - 0xa0a000a0, 0xe0e000e0, 0x3b3b003b, 0x4d4d004d, - 0xaeae00ae, 0x2a2a002a, 0xf5f500f5, 0xb0b000b0, - 0xc8c800c8, 0xebeb00eb, 0xbbbb00bb, 0x3c3c003c, - 0x83830083, 0x53530053, 0x99990099, 0x61610061, - 0x17170017, 0x2b2b002b, 0x04040004, 0x7e7e007e, - 0xbaba00ba, 0x77770077, 0xd6d600d6, 0x26260026, - 0xe1e100e1, 0x69690069, 0x14140014, 0x63630063, - 0x55550055, 0x21210021, 0x0c0c000c, 0x7d7d007d -}; - -static const uint32_t X2[256] = { - 0x30303000, 0x68686800, 0x99999900, 0x1b1b1b00, - 0x87878700, 0xb9b9b900, 0x21212100, 0x78787800, - 0x50505000, 0x39393900, 0xdbdbdb00, 0xe1e1e100, - 0x72727200, 0x09090900, 0x62626200, 0x3c3c3c00, - 0x3e3e3e00, 0x7e7e7e00, 0x5e5e5e00, 0x8e8e8e00, - 0xf1f1f100, 0xa0a0a000, 0xcccccc00, 0xa3a3a300, - 0x2a2a2a00, 0x1d1d1d00, 0xfbfbfb00, 0xb6b6b600, - 0xd6d6d600, 0x20202000, 0xc4c4c400, 0x8d8d8d00, - 0x81818100, 0x65656500, 0xf5f5f500, 0x89898900, - 0xcbcbcb00, 0x9d9d9d00, 0x77777700, 0xc6c6c600, - 0x57575700, 0x43434300, 0x56565600, 0x17171700, - 0xd4d4d400, 0x40404000, 0x1a1a1a00, 0x4d4d4d00, - 0xc0c0c000, 0x63636300, 0x6c6c6c00, 0xe3e3e300, - 0xb7b7b700, 0xc8c8c800, 0x64646400, 0x6a6a6a00, - 0x53535300, 0xaaaaaa00, 0x38383800, 0x98989800, - 0x0c0c0c00, 0xf4f4f400, 0x9b9b9b00, 0xededed00, - 0x7f7f7f00, 0x22222200, 0x76767600, 0xafafaf00, - 0xdddddd00, 0x3a3a3a00, 0x0b0b0b00, 0x58585800, - 0x67676700, 0x88888800, 0x06060600, 0xc3c3c300, - 0x35353500, 0x0d0d0d00, 0x01010100, 0x8b8b8b00, - 0x8c8c8c00, 0xc2c2c200, 0xe6e6e600, 0x5f5f5f00, - 0x02020200, 0x24242400, 0x75757500, 0x93939300, - 0x66666600, 0x1e1e1e00, 0xe5e5e500, 0xe2e2e200, - 0x54545400, 0xd8d8d800, 0x10101000, 0xcecece00, - 0x7a7a7a00, 0xe8e8e800, 0x08080800, 0x2c2c2c00, - 0x12121200, 0x97979700, 0x32323200, 0xababab00, - 0xb4b4b400, 0x27272700, 0x0a0a0a00, 0x23232300, - 0xdfdfdf00, 0xefefef00, 0xcacaca00, 0xd9d9d900, - 0xb8b8b800, 0xfafafa00, 0xdcdcdc00, 0x31313100, - 0x6b6b6b00, 0xd1d1d100, 0xadadad00, 0x19191900, - 0x49494900, 0xbdbdbd00, 0x51515100, 0x96969600, - 0xeeeeee00, 0xe4e4e400, 0xa8a8a800, 0x41414100, - 0xdadada00, 0xffffff00, 0xcdcdcd00, 0x55555500, - 0x86868600, 0x36363600, 0xbebebe00, 0x61616100, - 0x52525200, 0xf8f8f800, 0xbbbbbb00, 0x0e0e0e00, - 0x82828200, 0x48484800, 0x69696900, 0x9a9a9a00, - 0xe0e0e000, 0x47474700, 0x9e9e9e00, 0x5c5c5c00, - 0x04040400, 0x4b4b4b00, 0x34343400, 0x15151500, - 0x79797900, 0x26262600, 0xa7a7a700, 0xdedede00, - 0x29292900, 0xaeaeae00, 0x92929200, 0xd7d7d700, - 0x84848400, 0xe9e9e900, 0xd2d2d200, 0xbababa00, - 0x5d5d5d00, 0xf3f3f300, 0xc5c5c500, 0xb0b0b000, - 0xbfbfbf00, 0xa4a4a400, 0x3b3b3b00, 0x71717100, - 0x44444400, 0x46464600, 0x2b2b2b00, 0xfcfcfc00, - 0xebebeb00, 0x6f6f6f00, 0xd5d5d500, 0xf6f6f600, - 0x14141400, 0xfefefe00, 0x7c7c7c00, 0x70707000, - 0x5a5a5a00, 0x7d7d7d00, 0xfdfdfd00, 0x2f2f2f00, - 0x18181800, 0x83838300, 0x16161600, 0xa5a5a500, - 0x91919100, 0x1f1f1f00, 0x05050500, 0x95959500, - 0x74747400, 0xa9a9a900, 0xc1c1c100, 0x5b5b5b00, - 0x4a4a4a00, 0x85858500, 0x6d6d6d00, 0x13131300, - 0x07070700, 0x4f4f4f00, 0x4e4e4e00, 0x45454500, - 0xb2b2b200, 0x0f0f0f00, 0xc9c9c900, 0x1c1c1c00, - 0xa6a6a600, 0xbcbcbc00, 0xececec00, 0x73737300, - 0x90909000, 0x7b7b7b00, 0xcfcfcf00, 0x59595900, - 0x8f8f8f00, 0xa1a1a100, 0xf9f9f900, 0x2d2d2d00, - 0xf2f2f200, 0xb1b1b100, 0x00000000, 0x94949400, - 0x37373700, 0x9f9f9f00, 0xd0d0d000, 0x2e2e2e00, - 0x9c9c9c00, 0x6e6e6e00, 0x28282800, 0x3f3f3f00, - 0x80808000, 0xf0f0f000, 0x3d3d3d00, 0xd3d3d300, - 0x25252500, 0x8a8a8a00, 0xb5b5b500, 0xe7e7e700, - 0x42424200, 0xb3b3b300, 0xc7c7c700, 0xeaeaea00, - 0xf7f7f700, 0x4c4c4c00, 0x11111100, 0x33333300, - 0x03030300, 0xa2a2a200, 0xacacac00, 0x60606000 -}; - -/* Key XOR Layer */ -#define ARIA_ADD_ROUND_KEY(RK, T0, T1, T2, T3) \ - do { \ - (T0) ^= (RK)->u[0]; \ - (T1) ^= (RK)->u[1]; \ - (T2) ^= (RK)->u[2]; \ - (T3) ^= (RK)->u[3]; \ - } while(0) - -/* S-Box Layer 1 + M */ -#define ARIA_SBOX_LAYER1_WITH_PRE_DIFF(T0, T1, T2, T3) \ - do { \ - (T0) = \ - S1[GET_U8_BE(T0, 0)] ^ \ - S2[GET_U8_BE(T0, 1)] ^ \ - X1[GET_U8_BE(T0, 2)] ^ \ - X2[GET_U8_BE(T0, 3)]; \ - (T1) = \ - S1[GET_U8_BE(T1, 0)] ^ \ - S2[GET_U8_BE(T1, 1)] ^ \ - X1[GET_U8_BE(T1, 2)] ^ \ - X2[GET_U8_BE(T1, 3)]; \ - (T2) = \ - S1[GET_U8_BE(T2, 0)] ^ \ - S2[GET_U8_BE(T2, 1)] ^ \ - X1[GET_U8_BE(T2, 2)] ^ \ - X2[GET_U8_BE(T2, 3)]; \ - (T3) = \ - S1[GET_U8_BE(T3, 0)] ^ \ - S2[GET_U8_BE(T3, 1)] ^ \ - X1[GET_U8_BE(T3, 2)] ^ \ - X2[GET_U8_BE(T3, 3)]; \ - } while(0) - -/* S-Box Layer 2 + M */ -#define ARIA_SBOX_LAYER2_WITH_PRE_DIFF(T0, T1, T2, T3) \ - do { \ - (T0) = \ - X1[GET_U8_BE(T0, 0)] ^ \ - X2[GET_U8_BE(T0, 1)] ^ \ - S1[GET_U8_BE(T0, 2)] ^ \ - S2[GET_U8_BE(T0, 3)]; \ - (T1) = \ - X1[GET_U8_BE(T1, 0)] ^ \ - X2[GET_U8_BE(T1, 1)] ^ \ - S1[GET_U8_BE(T1, 2)] ^ \ - S2[GET_U8_BE(T1, 3)]; \ - (T2) = \ - X1[GET_U8_BE(T2, 0)] ^ \ - X2[GET_U8_BE(T2, 1)] ^ \ - S1[GET_U8_BE(T2, 2)] ^ \ - S2[GET_U8_BE(T2, 3)]; \ - (T3) = \ - X1[GET_U8_BE(T3, 0)] ^ \ - X2[GET_U8_BE(T3, 1)] ^ \ - S1[GET_U8_BE(T3, 2)] ^ \ - S2[GET_U8_BE(T3, 3)]; \ - } while(0) - -/* Word-level diffusion */ -#define ARIA_DIFF_WORD(T0,T1,T2,T3) \ - do { \ - (T1) ^= (T2); \ - (T2) ^= (T3); \ - (T0) ^= (T1); \ - \ - (T3) ^= (T1); \ - (T2) ^= (T0); \ - (T1) ^= (T2); \ - } while(0) - -/* Byte-level diffusion */ -#define ARIA_DIFF_BYTE(T0, T1, T2, T3) \ - do { \ - (T1) = (((T1) << 8) & 0xff00ff00) ^ (((T1) >> 8) & 0x00ff00ff); \ - (T2) = rotr32(T2, 16); \ - (T3) = bswap32(T3); \ - } while(0) - -/* Odd round Substitution & Diffusion */ -#define ARIA_SUBST_DIFF_ODD(T0, T1, T2, T3) \ - do { \ - ARIA_SBOX_LAYER1_WITH_PRE_DIFF(T0, T1, T2, T3); \ - ARIA_DIFF_WORD(T0, T1, T2, T3); \ - ARIA_DIFF_BYTE(T0, T1, T2, T3); \ - ARIA_DIFF_WORD(T0, T1, T2, T3); \ - } while(0) - -/* Even round Substitution & Diffusion */ -#define ARIA_SUBST_DIFF_EVEN(T0, T1, T2, T3) \ - do { \ - ARIA_SBOX_LAYER2_WITH_PRE_DIFF(T0, T1, T2, T3); \ - ARIA_DIFF_WORD(T0, T1, T2, T3); \ - ARIA_DIFF_BYTE(T2, T3, T0, T1); \ - ARIA_DIFF_WORD(T0, T1, T2, T3); \ - } while(0) - -/* Q, R Macro expanded ARIA GSRK */ -#define _ARIA_GSRK(RK, X, Y, Q, R) \ - do { \ - (RK)->u[0] = \ - ((X)[0]) ^ \ - (((Y)[((Q) ) % 4]) >> (R)) ^ \ - (((Y)[((Q) + 3) % 4]) << (32 - (R))); \ - (RK)->u[1] = \ - ((X)[1]) ^ \ - (((Y)[((Q) + 1) % 4]) >> (R)) ^ \ - (((Y)[((Q) ) % 4]) << (32 - (R))); \ - (RK)->u[2] = \ - ((X)[2]) ^ \ - (((Y)[((Q) + 2) % 4]) >> (R)) ^ \ - (((Y)[((Q) + 1) % 4]) << (32 - (R))); \ - (RK)->u[3] = \ - ((X)[3]) ^ \ - (((Y)[((Q) + 3) % 4]) >> (R)) ^ \ - (((Y)[((Q) + 2) % 4]) << (32 - (R))); \ - } while(0) - -#define ARIA_GSRK(RK, X, Y, N) _ARIA_GSRK(RK, X, Y, 4 - ((N) / 32), (N) % 32) - -#define ARIA_DEC_DIFF_BYTE(X, Y, TMP, TMP2) \ - do { \ - (TMP) = (X); \ - (TMP2) = rotr32((TMP), 8); \ - (Y) = (TMP2) ^ rotr32((TMP) ^ (TMP2), 16); \ - } while(0) - -void ossl_aria_encrypt(const unsigned char *in, unsigned char *out, - const ARIA_KEY *key) -{ - register uint32_t reg0, reg1, reg2, reg3; - int Nr; - const ARIA_u128 *rk; - - if (in == NULL || out == NULL || key == NULL) { - return; - } - - rk = key->rd_key; - Nr = key->rounds; - - if (Nr != 12 && Nr != 14 && Nr != 16) { - return; - } - - reg0 = GET_U32_BE(in, 0); - reg1 = GET_U32_BE(in, 1); - reg2 = GET_U32_BE(in, 2); - reg3 = GET_U32_BE(in, 3); - - ARIA_ADD_ROUND_KEY(rk, reg0, reg1, reg2, reg3); - rk++; - - ARIA_SUBST_DIFF_ODD(reg0, reg1, reg2, reg3); - ARIA_ADD_ROUND_KEY(rk, reg0, reg1, reg2, reg3); - rk++; - - while ((Nr -= 2) > 0) { - ARIA_SUBST_DIFF_EVEN(reg0, reg1, reg2, reg3); - ARIA_ADD_ROUND_KEY(rk, reg0, reg1, reg2, reg3); - rk++; - - ARIA_SUBST_DIFF_ODD(reg0, reg1, reg2, reg3); - ARIA_ADD_ROUND_KEY(rk, reg0, reg1, reg2, reg3); - rk++; - } - - reg0 = rk->u[0] ^ MAKE_U32( - (uint8_t)(X1[GET_U8_BE(reg0, 0)] ), - (uint8_t)(X2[GET_U8_BE(reg0, 1)] >> 8), - (uint8_t)(S1[GET_U8_BE(reg0, 2)] ), - (uint8_t)(S2[GET_U8_BE(reg0, 3)] )); - reg1 = rk->u[1] ^ MAKE_U32( - (uint8_t)(X1[GET_U8_BE(reg1, 0)] ), - (uint8_t)(X2[GET_U8_BE(reg1, 1)] >> 8), - (uint8_t)(S1[GET_U8_BE(reg1, 2)] ), - (uint8_t)(S2[GET_U8_BE(reg1, 3)] )); - reg2 = rk->u[2] ^ MAKE_U32( - (uint8_t)(X1[GET_U8_BE(reg2, 0)] ), - (uint8_t)(X2[GET_U8_BE(reg2, 1)] >> 8), - (uint8_t)(S1[GET_U8_BE(reg2, 2)] ), - (uint8_t)(S2[GET_U8_BE(reg2, 3)] )); - reg3 = rk->u[3] ^ MAKE_U32( - (uint8_t)(X1[GET_U8_BE(reg3, 0)] ), - (uint8_t)(X2[GET_U8_BE(reg3, 1)] >> 8), - (uint8_t)(S1[GET_U8_BE(reg3, 2)] ), - (uint8_t)(S2[GET_U8_BE(reg3, 3)] )); - - PUT_U32_BE(out, 0, reg0); - PUT_U32_BE(out, 1, reg1); - PUT_U32_BE(out, 2, reg2); - PUT_U32_BE(out, 3, reg3); -} - -int ossl_aria_set_encrypt_key(const unsigned char *userKey, const int bits, - ARIA_KEY *key) -{ - register uint32_t reg0, reg1, reg2, reg3; - uint32_t w0[4], w1[4], w2[4], w3[4]; - const uint32_t *ck; - - ARIA_u128 *rk; - int Nr = (bits + 256) / 32; - - if (userKey == NULL || key == NULL) { - return -1; - } - if (bits != 128 && bits != 192 && bits != 256) { - return -2; - } - - rk = key->rd_key; - key->rounds = Nr; - ck = &Key_RC[(bits - 128) / 64][0]; - - w0[0] = GET_U32_BE(userKey, 0); - w0[1] = GET_U32_BE(userKey, 1); - w0[2] = GET_U32_BE(userKey, 2); - w0[3] = GET_U32_BE(userKey, 3); - - reg0 = w0[0] ^ ck[0]; - reg1 = w0[1] ^ ck[1]; - reg2 = w0[2] ^ ck[2]; - reg3 = w0[3] ^ ck[3]; - - ARIA_SUBST_DIFF_ODD(reg0, reg1, reg2, reg3); - - if (bits > 128) { - w1[0] = GET_U32_BE(userKey, 4); - w1[1] = GET_U32_BE(userKey, 5); - if (bits > 192) { - w1[2] = GET_U32_BE(userKey, 6); - w1[3] = GET_U32_BE(userKey, 7); - } - else { - w1[2] = w1[3] = 0; - } - } - else { - w1[0] = w1[1] = w1[2] = w1[3] = 0; - } - - w1[0] ^= reg0; - w1[1] ^= reg1; - w1[2] ^= reg2; - w1[3] ^= reg3; - - reg0 = w1[0]; - reg1 = w1[1]; - reg2 = w1[2]; - reg3 = w1[3]; - - reg0 ^= ck[4]; - reg1 ^= ck[5]; - reg2 ^= ck[6]; - reg3 ^= ck[7]; - - ARIA_SUBST_DIFF_EVEN(reg0, reg1, reg2, reg3); - - reg0 ^= w0[0]; - reg1 ^= w0[1]; - reg2 ^= w0[2]; - reg3 ^= w0[3]; - - w2[0] = reg0; - w2[1] = reg1; - w2[2] = reg2; - w2[3] = reg3; - - reg0 ^= ck[8]; - reg1 ^= ck[9]; - reg2 ^= ck[10]; - reg3 ^= ck[11]; - - ARIA_SUBST_DIFF_ODD(reg0, reg1, reg2, reg3); - - w3[0] = reg0 ^ w1[0]; - w3[1] = reg1 ^ w1[1]; - w3[2] = reg2 ^ w1[2]; - w3[3] = reg3 ^ w1[3]; - - ARIA_GSRK(rk, w0, w1, 19); - rk++; - ARIA_GSRK(rk, w1, w2, 19); - rk++; - ARIA_GSRK(rk, w2, w3, 19); - rk++; - ARIA_GSRK(rk, w3, w0, 19); - - rk++; - ARIA_GSRK(rk, w0, w1, 31); - rk++; - ARIA_GSRK(rk, w1, w2, 31); - rk++; - ARIA_GSRK(rk, w2, w3, 31); - rk++; - ARIA_GSRK(rk, w3, w0, 31); - - rk++; - ARIA_GSRK(rk, w0, w1, 67); - rk++; - ARIA_GSRK(rk, w1, w2, 67); - rk++; - ARIA_GSRK(rk, w2, w3, 67); - rk++; - ARIA_GSRK(rk, w3, w0, 67); - - rk++; - ARIA_GSRK(rk, w0, w1, 97); - if (bits > 128) { - rk++; - ARIA_GSRK(rk, w1, w2, 97); - rk++; - ARIA_GSRK(rk, w2, w3, 97); - } - if (bits > 192) { - rk++; - ARIA_GSRK(rk, w3, w0, 97); - - rk++; - ARIA_GSRK(rk, w0, w1, 109); - } - - return 0; -} - -int ossl_aria_set_decrypt_key(const unsigned char *userKey, const int bits, - ARIA_KEY *key) -{ - ARIA_u128 *rk_head; - ARIA_u128 *rk_tail; - register uint32_t w1, w2; - register uint32_t reg0, reg1, reg2, reg3; - uint32_t s0, s1, s2, s3; - - const int r = ossl_aria_set_encrypt_key(userKey, bits, key); - - if (r != 0) { - return r; - } - - rk_head = key->rd_key; - rk_tail = rk_head + key->rounds; - - reg0 = rk_head->u[0]; - reg1 = rk_head->u[1]; - reg2 = rk_head->u[2]; - reg3 = rk_head->u[3]; - - memcpy(rk_head, rk_tail, ARIA_BLOCK_SIZE); - - rk_tail->u[0] = reg0; - rk_tail->u[1] = reg1; - rk_tail->u[2] = reg2; - rk_tail->u[3] = reg3; - - rk_head++; - rk_tail--; - - for (; rk_head < rk_tail; rk_head++, rk_tail--) { - ARIA_DEC_DIFF_BYTE(rk_head->u[0], reg0, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_head->u[1], reg1, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_head->u[2], reg2, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_head->u[3], reg3, w1, w2); - - ARIA_DIFF_WORD(reg0, reg1, reg2, reg3); - ARIA_DIFF_BYTE(reg0, reg1, reg2, reg3); - ARIA_DIFF_WORD(reg0, reg1, reg2, reg3); - - s0 = reg0; - s1 = reg1; - s2 = reg2; - s3 = reg3; - - ARIA_DEC_DIFF_BYTE(rk_tail->u[0], reg0, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_tail->u[1], reg1, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_tail->u[2], reg2, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_tail->u[3], reg3, w1, w2); - - ARIA_DIFF_WORD(reg0, reg1, reg2, reg3); - ARIA_DIFF_BYTE(reg0, reg1, reg2, reg3); - ARIA_DIFF_WORD(reg0, reg1, reg2, reg3); - - rk_head->u[0] = reg0; - rk_head->u[1] = reg1; - rk_head->u[2] = reg2; - rk_head->u[3] = reg3; - - rk_tail->u[0] = s0; - rk_tail->u[1] = s1; - rk_tail->u[2] = s2; - rk_tail->u[3] = s3; - } - ARIA_DEC_DIFF_BYTE(rk_head->u[0], reg0, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_head->u[1], reg1, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_head->u[2], reg2, w1, w2); - ARIA_DEC_DIFF_BYTE(rk_head->u[3], reg3, w1, w2); - - ARIA_DIFF_WORD(reg0, reg1, reg2, reg3); - ARIA_DIFF_BYTE(reg0, reg1, reg2, reg3); - ARIA_DIFF_WORD(reg0, reg1, reg2, reg3); - - rk_tail->u[0] = reg0; - rk_tail->u[1] = reg1; - rk_tail->u[2] = reg2; - rk_tail->u[3] = reg3; - - return 0; -} - -#else - -static const unsigned char sb1[256] = { - 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, - 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, - 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, - 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, - 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, - 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, - 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, - 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, - 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, - 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, - 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, - 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, - 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, - 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, - 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, - 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, - 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, - 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, - 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, - 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, - 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, - 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, - 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, - 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, - 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, - 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, - 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, - 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, - 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, - 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, - 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, - 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 -}; - -static const unsigned char sb2[256] = { - 0xe2, 0x4e, 0x54, 0xfc, 0x94, 0xc2, 0x4a, 0xcc, - 0x62, 0x0d, 0x6a, 0x46, 0x3c, 0x4d, 0x8b, 0xd1, - 0x5e, 0xfa, 0x64, 0xcb, 0xb4, 0x97, 0xbe, 0x2b, - 0xbc, 0x77, 0x2e, 0x03, 0xd3, 0x19, 0x59, 0xc1, - 0x1d, 0x06, 0x41, 0x6b, 0x55, 0xf0, 0x99, 0x69, - 0xea, 0x9c, 0x18, 0xae, 0x63, 0xdf, 0xe7, 0xbb, - 0x00, 0x73, 0x66, 0xfb, 0x96, 0x4c, 0x85, 0xe4, - 0x3a, 0x09, 0x45, 0xaa, 0x0f, 0xee, 0x10, 0xeb, - 0x2d, 0x7f, 0xf4, 0x29, 0xac, 0xcf, 0xad, 0x91, - 0x8d, 0x78, 0xc8, 0x95, 0xf9, 0x2f, 0xce, 0xcd, - 0x08, 0x7a, 0x88, 0x38, 0x5c, 0x83, 0x2a, 0x28, - 0x47, 0xdb, 0xb8, 0xc7, 0x93, 0xa4, 0x12, 0x53, - 0xff, 0x87, 0x0e, 0x31, 0x36, 0x21, 0x58, 0x48, - 0x01, 0x8e, 0x37, 0x74, 0x32, 0xca, 0xe9, 0xb1, - 0xb7, 0xab, 0x0c, 0xd7, 0xc4, 0x56, 0x42, 0x26, - 0x07, 0x98, 0x60, 0xd9, 0xb6, 0xb9, 0x11, 0x40, - 0xec, 0x20, 0x8c, 0xbd, 0xa0, 0xc9, 0x84, 0x04, - 0x49, 0x23, 0xf1, 0x4f, 0x50, 0x1f, 0x13, 0xdc, - 0xd8, 0xc0, 0x9e, 0x57, 0xe3, 0xc3, 0x7b, 0x65, - 0x3b, 0x02, 0x8f, 0x3e, 0xe8, 0x25, 0x92, 0xe5, - 0x15, 0xdd, 0xfd, 0x17, 0xa9, 0xbf, 0xd4, 0x9a, - 0x7e, 0xc5, 0x39, 0x67, 0xfe, 0x76, 0x9d, 0x43, - 0xa7, 0xe1, 0xd0, 0xf5, 0x68, 0xf2, 0x1b, 0x34, - 0x70, 0x05, 0xa3, 0x8a, 0xd5, 0x79, 0x86, 0xa8, - 0x30, 0xc6, 0x51, 0x4b, 0x1e, 0xa6, 0x27, 0xf6, - 0x35, 0xd2, 0x6e, 0x24, 0x16, 0x82, 0x5f, 0xda, - 0xe6, 0x75, 0xa2, 0xef, 0x2c, 0xb2, 0x1c, 0x9f, - 0x5d, 0x6f, 0x80, 0x0a, 0x72, 0x44, 0x9b, 0x6c, - 0x90, 0x0b, 0x5b, 0x33, 0x7d, 0x5a, 0x52, 0xf3, - 0x61, 0xa1, 0xf7, 0xb0, 0xd6, 0x3f, 0x7c, 0x6d, - 0xed, 0x14, 0xe0, 0xa5, 0x3d, 0x22, 0xb3, 0xf8, - 0x89, 0xde, 0x71, 0x1a, 0xaf, 0xba, 0xb5, 0x81 -}; - -static const unsigned char sb3[256] = { - 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, - 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, - 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, - 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, - 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, - 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, - 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, - 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, - 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, - 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, - 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, - 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, - 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, - 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, - 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, - 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, - 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, - 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, - 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, - 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, - 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, - 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, - 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, - 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, - 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, - 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, - 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, - 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, - 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, - 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, - 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, - 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d -}; - -static const unsigned char sb4[256] = { - 0x30, 0x68, 0x99, 0x1b, 0x87, 0xb9, 0x21, 0x78, - 0x50, 0x39, 0xdb, 0xe1, 0x72, 0x09, 0x62, 0x3c, - 0x3e, 0x7e, 0x5e, 0x8e, 0xf1, 0xa0, 0xcc, 0xa3, - 0x2a, 0x1d, 0xfb, 0xb6, 0xd6, 0x20, 0xc4, 0x8d, - 0x81, 0x65, 0xf5, 0x89, 0xcb, 0x9d, 0x77, 0xc6, - 0x57, 0x43, 0x56, 0x17, 0xd4, 0x40, 0x1a, 0x4d, - 0xc0, 0x63, 0x6c, 0xe3, 0xb7, 0xc8, 0x64, 0x6a, - 0x53, 0xaa, 0x38, 0x98, 0x0c, 0xf4, 0x9b, 0xed, - 0x7f, 0x22, 0x76, 0xaf, 0xdd, 0x3a, 0x0b, 0x58, - 0x67, 0x88, 0x06, 0xc3, 0x35, 0x0d, 0x01, 0x8b, - 0x8c, 0xc2, 0xe6, 0x5f, 0x02, 0x24, 0x75, 0x93, - 0x66, 0x1e, 0xe5, 0xe2, 0x54, 0xd8, 0x10, 0xce, - 0x7a, 0xe8, 0x08, 0x2c, 0x12, 0x97, 0x32, 0xab, - 0xb4, 0x27, 0x0a, 0x23, 0xdf, 0xef, 0xca, 0xd9, - 0xb8, 0xfa, 0xdc, 0x31, 0x6b, 0xd1, 0xad, 0x19, - 0x49, 0xbd, 0x51, 0x96, 0xee, 0xe4, 0xa8, 0x41, - 0xda, 0xff, 0xcd, 0x55, 0x86, 0x36, 0xbe, 0x61, - 0x52, 0xf8, 0xbb, 0x0e, 0x82, 0x48, 0x69, 0x9a, - 0xe0, 0x47, 0x9e, 0x5c, 0x04, 0x4b, 0x34, 0x15, - 0x79, 0x26, 0xa7, 0xde, 0x29, 0xae, 0x92, 0xd7, - 0x84, 0xe9, 0xd2, 0xba, 0x5d, 0xf3, 0xc5, 0xb0, - 0xbf, 0xa4, 0x3b, 0x71, 0x44, 0x46, 0x2b, 0xfc, - 0xeb, 0x6f, 0xd5, 0xf6, 0x14, 0xfe, 0x7c, 0x70, - 0x5a, 0x7d, 0xfd, 0x2f, 0x18, 0x83, 0x16, 0xa5, - 0x91, 0x1f, 0x05, 0x95, 0x74, 0xa9, 0xc1, 0x5b, - 0x4a, 0x85, 0x6d, 0x13, 0x07, 0x4f, 0x4e, 0x45, - 0xb2, 0x0f, 0xc9, 0x1c, 0xa6, 0xbc, 0xec, 0x73, - 0x90, 0x7b, 0xcf, 0x59, 0x8f, 0xa1, 0xf9, 0x2d, - 0xf2, 0xb1, 0x00, 0x94, 0x37, 0x9f, 0xd0, 0x2e, - 0x9c, 0x6e, 0x28, 0x3f, 0x80, 0xf0, 0x3d, 0xd3, - 0x25, 0x8a, 0xb5, 0xe7, 0x42, 0xb3, 0xc7, 0xea, - 0xf7, 0x4c, 0x11, 0x33, 0x03, 0xa2, 0xac, 0x60 -}; - -static const ARIA_u128 c1 = {{ - 0x51, 0x7c, 0xc1, 0xb7, 0x27, 0x22, 0x0a, 0x94, - 0xfe, 0x13, 0xab, 0xe8, 0xfa, 0x9a, 0x6e, 0xe0 -}}; - -static const ARIA_u128 c2 = {{ - 0x6d, 0xb1, 0x4a, 0xcc, 0x9e, 0x21, 0xc8, 0x20, - 0xff, 0x28, 0xb1, 0xd5, 0xef, 0x5d, 0xe2, 0xb0 -}}; - -static const ARIA_u128 c3 = {{ - 0xdb, 0x92, 0x37, 0x1d, 0x21, 0x26, 0xe9, 0x70, - 0x03, 0x24, 0x97, 0x75, 0x04, 0xe8, 0xc9, 0x0e -}}; - -/* - * Exclusive or two 128 bit values into the result. - * It is safe for the result to be the same as the either input. - */ -static void xor128(ARIA_c128 o, const ARIA_c128 x, const ARIA_u128 *y) -{ - int i; - - for (i = 0; i < ARIA_BLOCK_SIZE; i++) - o[i] = x[i] ^ y->c[i]; -} - -/* - * Generalised circular rotate right and exclusive or function. - * It is safe for the output to overlap either input. - */ -static ossl_inline void rotnr(unsigned int n, ARIA_u128 *o, - const ARIA_u128 *xor, const ARIA_u128 *z) -{ - const unsigned int bytes = n / 8, bits = n % 8; - unsigned int i; - ARIA_u128 t; - - for (i = 0; i < ARIA_BLOCK_SIZE; i++) - t.c[(i + bytes) % ARIA_BLOCK_SIZE] = z->c[i]; - for (i = 0; i < ARIA_BLOCK_SIZE; i++) - o->c[i] = ((t.c[i] >> bits) | - (t.c[i ? i - 1 : ARIA_BLOCK_SIZE - 1] << (8 - bits))) ^ - xor->c[i]; -} - -/* - * Circular rotate 19 bits right and xor. - * It is safe for the output to overlap either input. - */ -static void rot19r(ARIA_u128 *o, const ARIA_u128 *xor, const ARIA_u128 *z) -{ - rotnr(19, o, xor, z); -} - -/* - * Circular rotate 31 bits right and xor. - * It is safe for the output to overlap either input. - */ -static void rot31r(ARIA_u128 *o, const ARIA_u128 *xor, const ARIA_u128 *z) -{ - rotnr(31, o, xor, z); -} - -/* - * Circular rotate 61 bits left and xor. - * It is safe for the output to overlap either input. - */ -static void rot61l(ARIA_u128 *o, const ARIA_u128 *xor, const ARIA_u128 *z) -{ - rotnr(8 * ARIA_BLOCK_SIZE - 61, o, xor, z); -} - -/* - * Circular rotate 31 bits left and xor. - * It is safe for the output to overlap either input. - */ -static void rot31l(ARIA_u128 *o, const ARIA_u128 *xor, const ARIA_u128 *z) -{ - rotnr(8 * ARIA_BLOCK_SIZE - 31, o, xor, z); -} - -/* - * Circular rotate 19 bits left and xor. - * It is safe for the output to overlap either input. - */ -static void rot19l(ARIA_u128 *o, const ARIA_u128 *xor, const ARIA_u128 *z) -{ - rotnr(8 * ARIA_BLOCK_SIZE - 19, o, xor, z); -} - -/* - * First substitution and xor layer, used for odd steps. - * It is safe for the input and output to be the same. - */ -static void sl1(ARIA_u128 *o, const ARIA_u128 *x, const ARIA_u128 *y) -{ - unsigned int i; - for (i = 0; i < ARIA_BLOCK_SIZE; i += 4) { - o->c[i ] = sb1[x->c[i ] ^ y->c[i ]]; - o->c[i + 1] = sb2[x->c[i + 1] ^ y->c[i + 1]]; - o->c[i + 2] = sb3[x->c[i + 2] ^ y->c[i + 2]]; - o->c[i + 3] = sb4[x->c[i + 3] ^ y->c[i + 3]]; - } -} - -/* - * Second substitution and xor layer, used for even steps. - * It is safe for the input and output to be the same. - */ -static void sl2(ARIA_c128 o, const ARIA_u128 *x, const ARIA_u128 *y) -{ - unsigned int i; - for (i = 0; i < ARIA_BLOCK_SIZE; i += 4) { - o[i ] = sb3[x->c[i ] ^ y->c[i ]]; - o[i + 1] = sb4[x->c[i + 1] ^ y->c[i + 1]]; - o[i + 2] = sb1[x->c[i + 2] ^ y->c[i + 2]]; - o[i + 3] = sb2[x->c[i + 3] ^ y->c[i + 3]]; - } -} - -/* - * Diffusion layer step - * It is NOT safe for the input and output to overlap. - */ -static void a(ARIA_u128 *y, const ARIA_u128 *x) -{ - y->c[ 0] = x->c[ 3] ^ x->c[ 4] ^ x->c[ 6] ^ x->c[ 8] ^ - x->c[ 9] ^ x->c[13] ^ x->c[14]; - y->c[ 1] = x->c[ 2] ^ x->c[ 5] ^ x->c[ 7] ^ x->c[ 8] ^ - x->c[ 9] ^ x->c[12] ^ x->c[15]; - y->c[ 2] = x->c[ 1] ^ x->c[ 4] ^ x->c[ 6] ^ x->c[10] ^ - x->c[11] ^ x->c[12] ^ x->c[15]; - y->c[ 3] = x->c[ 0] ^ x->c[ 5] ^ x->c[ 7] ^ x->c[10] ^ - x->c[11] ^ x->c[13] ^ x->c[14]; - y->c[ 4] = x->c[ 0] ^ x->c[ 2] ^ x->c[ 5] ^ x->c[ 8] ^ - x->c[11] ^ x->c[14] ^ x->c[15]; - y->c[ 5] = x->c[ 1] ^ x->c[ 3] ^ x->c[ 4] ^ x->c[ 9] ^ - x->c[10] ^ x->c[14] ^ x->c[15]; - y->c[ 6] = x->c[ 0] ^ x->c[ 2] ^ x->c[ 7] ^ x->c[ 9] ^ - x->c[10] ^ x->c[12] ^ x->c[13]; - y->c[ 7] = x->c[ 1] ^ x->c[ 3] ^ x->c[ 6] ^ x->c[ 8] ^ - x->c[11] ^ x->c[12] ^ x->c[13]; - y->c[ 8] = x->c[ 0] ^ x->c[ 1] ^ x->c[ 4] ^ x->c[ 7] ^ - x->c[10] ^ x->c[13] ^ x->c[15]; - y->c[ 9] = x->c[ 0] ^ x->c[ 1] ^ x->c[ 5] ^ x->c[ 6] ^ - x->c[11] ^ x->c[12] ^ x->c[14]; - y->c[10] = x->c[ 2] ^ x->c[ 3] ^ x->c[ 5] ^ x->c[ 6] ^ - x->c[ 8] ^ x->c[13] ^ x->c[15]; - y->c[11] = x->c[ 2] ^ x->c[ 3] ^ x->c[ 4] ^ x->c[ 7] ^ - x->c[ 9] ^ x->c[12] ^ x->c[14]; - y->c[12] = x->c[ 1] ^ x->c[ 2] ^ x->c[ 6] ^ x->c[ 7] ^ - x->c[ 9] ^ x->c[11] ^ x->c[12]; - y->c[13] = x->c[ 0] ^ x->c[ 3] ^ x->c[ 6] ^ x->c[ 7] ^ - x->c[ 8] ^ x->c[10] ^ x->c[13]; - y->c[14] = x->c[ 0] ^ x->c[ 3] ^ x->c[ 4] ^ x->c[ 5] ^ - x->c[ 9] ^ x->c[11] ^ x->c[14]; - y->c[15] = x->c[ 1] ^ x->c[ 2] ^ x->c[ 4] ^ x->c[ 5] ^ - x->c[ 8] ^ x->c[10] ^ x->c[15]; -} - -/* - * Odd round function - * Apply the first substitution layer and then a diffusion step. - * It is safe for the input and output to overlap. - */ -static ossl_inline void FO(ARIA_u128 *o, const ARIA_u128 *d, - const ARIA_u128 *rk) -{ - ARIA_u128 y; - - sl1(&y, d, rk); - a(o, &y); -} - -/* - * Even round function - * Apply the second substitution layer and then a diffusion step. - * It is safe for the input and output to overlap. - */ -static ossl_inline void FE(ARIA_u128 *o, const ARIA_u128 *d, - const ARIA_u128 *rk) -{ - ARIA_u128 y; - - sl2(y.c, d, rk); - a(o, &y); -} - -/* - * Encrypt or decrypt a single block - * in and out can overlap - */ -static void do_encrypt(unsigned char *o, const unsigned char *pin, - unsigned int rounds, const ARIA_u128 *keys) -{ - ARIA_u128 p; - unsigned int i; - - memcpy(&p, pin, sizeof(p)); - for (i = 0; i < rounds - 2; i += 2) { - FO(&p, &p, &keys[i]); - FE(&p, &p, &keys[i + 1]); - } - FO(&p, &p, &keys[rounds - 2]); - sl2(o, &p, &keys[rounds - 1]); - xor128(o, o, &keys[rounds]); -} - -/* - * Encrypt a single block - * in and out can overlap - */ -void ossl_aria_encrypt(const unsigned char *in, unsigned char *out, - const ARIA_KEY *key) -{ - assert(in != NULL && out != NULL && key != NULL); - do_encrypt(out, in, key->rounds, key->rd_key); -} - - -/* - * Expand the cipher key into the encryption key schedule. - * We short circuit execution of the last two - * or four rotations based on the key size. - */ -int ossl_aria_set_encrypt_key(const unsigned char *userKey, const int bits, - ARIA_KEY *key) -{ - const ARIA_u128 *ck1, *ck2, *ck3; - ARIA_u128 kr, w0, w1, w2, w3; - - if (!userKey || !key) - return -1; - memcpy(w0.c, userKey, sizeof(w0)); - switch (bits) { - default: - return -2; - case 128: - key->rounds = 12; - ck1 = &c1; - ck2 = &c2; - ck3 = &c3; - memset(kr.c, 0, sizeof(kr)); - break; - - case 192: - key->rounds = 14; - ck1 = &c2; - ck2 = &c3; - ck3 = &c1; - memcpy(kr.c, userKey + ARIA_BLOCK_SIZE, sizeof(kr) / 2); - memset(kr.c + ARIA_BLOCK_SIZE / 2, 0, sizeof(kr) / 2); - break; - - case 256: - key->rounds = 16; - ck1 = &c3; - ck2 = &c1; - ck3 = &c2; - memcpy(kr.c, userKey + ARIA_BLOCK_SIZE, sizeof(kr)); - break; - } - - FO(&w3, &w0, ck1); xor128(w1.c, w3.c, &kr); - FE(&w3, &w1, ck2); xor128(w2.c, w3.c, &w0); - FO(&kr, &w2, ck3); xor128(w3.c, kr.c, &w1); - - rot19r(&key->rd_key[ 0], &w0, &w1); - rot19r(&key->rd_key[ 1], &w1, &w2); - rot19r(&key->rd_key[ 2], &w2, &w3); - rot19r(&key->rd_key[ 3], &w3, &w0); - - rot31r(&key->rd_key[ 4], &w0, &w1); - rot31r(&key->rd_key[ 5], &w1, &w2); - rot31r(&key->rd_key[ 6], &w2, &w3); - rot31r(&key->rd_key[ 7], &w3, &w0); - - rot61l(&key->rd_key[ 8], &w0, &w1); - rot61l(&key->rd_key[ 9], &w1, &w2); - rot61l(&key->rd_key[10], &w2, &w3); - rot61l(&key->rd_key[11], &w3, &w0); - - rot31l(&key->rd_key[12], &w0, &w1); - if (key->rounds > 12) { - rot31l(&key->rd_key[13], &w1, &w2); - rot31l(&key->rd_key[14], &w2, &w3); - - if (key->rounds > 14) { - rot31l(&key->rd_key[15], &w3, &w0); - rot19l(&key->rd_key[16], &w0, &w1); - } - } - return 0; -} - -/* - * Expand the cipher key into the decryption key schedule. - */ -int ossl_aria_set_decrypt_key(const unsigned char *userKey, const int bits, - ARIA_KEY *key) -{ - ARIA_KEY ek; - const int r = ossl_aria_set_encrypt_key(userKey, bits, &ek); - unsigned int i, rounds = ek.rounds; - - if (r == 0) { - key->rounds = rounds; - memcpy(&key->rd_key[0], &ek.rd_key[rounds], sizeof(key->rd_key[0])); - for (i = 1; i < rounds; i++) - a(&key->rd_key[i], &ek.rd_key[rounds - i]); - memcpy(&key->rd_key[rounds], &ek.rd_key[0], sizeof(key->rd_key[rounds])); - } - return r; -} - -#endif diff --git a/openssl/src/crypto/arm_arch.h b/openssl/src/crypto/arm_arch.h index 7ac978fec..357b92b9a 100644 --- a/openssl/src/crypto/arm_arch.h +++ b/openssl/src/crypto/arm_arch.h @@ -1,5 +1,5 @@ /* - * Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,6 +21,11 @@ # elif defined(__GNUC__) # if defined(__aarch64__) # define __ARM_ARCH__ 8 +# if __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define __ARMEB__ +# else +# define __ARMEL__ +# endif /* * Why doesn't gcc define __ARM_ARCH__? Instead it defines * bunch of below macros. See all_architectures[] table in @@ -49,8 +54,6 @@ # else # error "unsupported ARM architecture" # endif -# elif defined(__ARM_ARCH) -# define __ARM_ARCH__ __ARM_ARCH # endif # endif @@ -83,12 +86,6 @@ extern unsigned int OPENSSL_armv8_rsa_neonized; # define ARMV8_RNG (1<<8) # define ARMV8_SM3 (1<<9) # define ARMV8_SM4 (1<<10) -# define ARMV8_SHA3 (1<<11) -# define ARMV8_UNROLL8_EOR3 (1<<12) -# define ARMV8_SVE (1<<13) -# define ARMV8_SVE2 (1<<14) -# define ARMV8_HAVE_SHA3_AND_WORTH_USING (1<<15) -# define ARMV8_UNROLL12_EOR3 (1<<16) /* * MIDR_EL1 system register @@ -101,44 +98,22 @@ extern unsigned int OPENSSL_armv8_rsa_neonized; */ # define ARM_CPU_IMP_ARM 0x41 -# define HISI_CPU_IMP 0x48 -# define ARM_CPU_IMP_APPLE 0x61 -# define ARM_CPU_IMP_MICROSOFT 0x6D # define ARM_CPU_PART_CORTEX_A72 0xD08 # define ARM_CPU_PART_N1 0xD0C -# define ARM_CPU_PART_V1 0xD40 -# define ARM_CPU_PART_N2 0xD49 -# define HISI_CPU_PART_KP920 0xD01 -# define ARM_CPU_PART_V2 0xD4F - -# define APPLE_CPU_PART_M1_ICESTORM 0x022 -# define APPLE_CPU_PART_M1_FIRESTORM 0x023 -# define APPLE_CPU_PART_M1_ICESTORM_PRO 0x024 -# define APPLE_CPU_PART_M1_FIRESTORM_PRO 0x025 -# define APPLE_CPU_PART_M1_ICESTORM_MAX 0x028 -# define APPLE_CPU_PART_M1_FIRESTORM_MAX 0x029 -# define APPLE_CPU_PART_M2_BLIZZARD 0x032 -# define APPLE_CPU_PART_M2_AVALANCHE 0x033 -# define APPLE_CPU_PART_M2_BLIZZARD_PRO 0x034 -# define APPLE_CPU_PART_M2_AVALANCHE_PRO 0x035 -# define APPLE_CPU_PART_M2_BLIZZARD_MAX 0x038 -# define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 - -# define MICROSOFT_CPU_PART_COBALT_100 0xD49 # define MIDR_PARTNUM_SHIFT 4 -# define MIDR_PARTNUM_MASK (0xfffU << MIDR_PARTNUM_SHIFT) +# define MIDR_PARTNUM_MASK (0xfff << MIDR_PARTNUM_SHIFT) # define MIDR_PARTNUM(midr) \ (((midr) & MIDR_PARTNUM_MASK) >> MIDR_PARTNUM_SHIFT) # define MIDR_IMPLEMENTER_SHIFT 24 -# define MIDR_IMPLEMENTER_MASK (0xffU << MIDR_IMPLEMENTER_SHIFT) +# define MIDR_IMPLEMENTER_MASK (0xff << MIDR_IMPLEMENTER_SHIFT) # define MIDR_IMPLEMENTER(midr) \ (((midr) & MIDR_IMPLEMENTER_MASK) >> MIDR_IMPLEMENTER_SHIFT) # define MIDR_ARCHITECTURE_SHIFT 16 -# define MIDR_ARCHITECTURE_MASK (0xfU << MIDR_ARCHITECTURE_SHIFT) +# define MIDR_ARCHITECTURE_MASK (0xf << MIDR_ARCHITECTURE_SHIFT) # define MIDR_ARCHITECTURE(midr) \ (((midr) & MIDR_ARCHITECTURE_MASK) >> MIDR_ARCHITECTURE_SHIFT) @@ -149,70 +124,9 @@ extern unsigned int OPENSSL_armv8_rsa_neonized; # define MIDR_CPU_MODEL(imp, partnum) \ (((imp) << MIDR_IMPLEMENTER_SHIFT) | \ - (0xfU << MIDR_ARCHITECTURE_SHIFT) | \ + (0xf << MIDR_ARCHITECTURE_SHIFT) | \ ((partnum) << MIDR_PARTNUM_SHIFT)) # define MIDR_IS_CPU_MODEL(midr, imp, partnum) \ (((midr) & MIDR_CPU_MODEL_MASK) == MIDR_CPU_MODEL(imp, partnum)) - -#if defined(__ASSEMBLER__) - - /* - * Support macros for - * - Armv8.3-A Pointer Authentication and - * - Armv8.5-A Branch Target Identification - * features which require emitting a .note.gnu.property section with the - * appropriate architecture-dependent feature bits set. - * Read more: "ELF for the Arm® 64-bit Architecture" - */ - -# if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 -# define GNU_PROPERTY_AARCH64_BTI (1 << 0) /* Has Branch Target Identification */ -# define AARCH64_VALID_CALL_TARGET hint #34 /* BTI 'c' */ -# else -# define GNU_PROPERTY_AARCH64_BTI 0 /* No Branch Target Identification */ -# define AARCH64_VALID_CALL_TARGET -# endif - -# if defined(__ARM_FEATURE_PAC_DEFAULT) && \ - (__ARM_FEATURE_PAC_DEFAULT & 1) == 1 /* Signed with A-key */ -# define GNU_PROPERTY_AARCH64_POINTER_AUTH \ - (1 << 1) /* Has Pointer Authentication */ -# define AARCH64_SIGN_LINK_REGISTER hint #25 /* PACIASP */ -# define AARCH64_VALIDATE_LINK_REGISTER hint #29 /* AUTIASP */ -# elif defined(__ARM_FEATURE_PAC_DEFAULT) && \ - (__ARM_FEATURE_PAC_DEFAULT & 2) == 2 /* Signed with B-key */ -# define GNU_PROPERTY_AARCH64_POINTER_AUTH \ - (1 << 1) /* Has Pointer Authentication */ -# define AARCH64_SIGN_LINK_REGISTER hint #27 /* PACIBSP */ -# define AARCH64_VALIDATE_LINK_REGISTER hint #31 /* AUTIBSP */ -# else -# define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 /* No Pointer Authentication */ -# if GNU_PROPERTY_AARCH64_BTI != 0 -# define AARCH64_SIGN_LINK_REGISTER AARCH64_VALID_CALL_TARGET -# else -# define AARCH64_SIGN_LINK_REGISTER -# endif -# define AARCH64_VALIDATE_LINK_REGISTER -# endif - -# if GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_BTI != 0 - .pushsection .note.gnu.property, "a"; - .balign 8; - .long 4; - .long 0x10; - .long 0x5; - .asciz "GNU"; - .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ - .long 4; - .long (GNU_PROPERTY_AARCH64_POINTER_AUTH | GNU_PROPERTY_AARCH64_BTI); - .long 0; - .popsection; -# endif - -# endif /* defined __ASSEMBLER__ */ - -# define IS_CPU_SUPPORT_UNROLL8_EOR3() \ - (OPENSSL_armcap_P & ARMV8_UNROLL8_EOR3) - #endif diff --git a/openssl/src/crypto/armcap.c b/openssl/src/crypto/armcap.c index 781503eda..aaca45bbc 100644 --- a/openssl/src/crypto/armcap.c +++ b/openssl/src/crypto/armcap.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,57 +10,62 @@ #include #include #include +#include +#include #include #ifdef __APPLE__ #include -#else -#include -#include #endif #include "internal/cryptlib.h" -#ifdef _WIN32 -#include -#else -#include -#endif + #include "arm_arch.h" unsigned int OPENSSL_armcap_P = 0; unsigned int OPENSSL_arm_midr = 0; unsigned int OPENSSL_armv8_rsa_neonized = 0; -#ifdef _WIN32 +#if __ARM_MAX_ARCH__<7 void OPENSSL_cpuid_setup(void) { - OPENSSL_armcap_P |= ARMV7_NEON; - OPENSSL_armv8_rsa_neonized = 1; - if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE)) { - // These are all covered by one call in Windows - OPENSSL_armcap_P |= ARMV8_AES; - OPENSSL_armcap_P |= ARMV8_PMULL; - OPENSSL_armcap_P |= ARMV8_SHA1; - OPENSSL_armcap_P |= ARMV8_SHA256; - } } uint32_t OPENSSL_rdtsc(void) { return 0; } -#elif __ARM_MAX_ARCH__ < 7 -void OPENSSL_cpuid_setup(void) +#else +static sigset_t all_masked; + +static sigjmp_buf ill_jmp; +static void ill_handler(int sig) { + siglongjmp(ill_jmp, sig); } +/* + * Following subroutines could have been inlined, but it's not all + * ARM compilers support inline assembler... + */ +void _armv7_neon_probe(void); +void _armv8_aes_probe(void); +void _armv8_sha1_probe(void); +void _armv8_sha256_probe(void); +void _armv8_pmull_probe(void); +# ifdef __aarch64__ +void _armv8_sm3_probe(void); +void _armv8_sm4_probe(void); +void _armv8_sha512_probe(void); +unsigned int _armv8_cpuid_probe(void); +# endif +uint32_t _armv7_tick(void); + uint32_t OPENSSL_rdtsc(void) { - return 0; + if (OPENSSL_armcap_P & ARMV7_TICK) + return _armv7_tick(); + else + return 0; } -#else /* !_WIN32 && __ARM_MAX_ARCH__ >= 7 */ - - /* 3 ways of handling things here: __APPLE__, getauxval() or SIGILL detect */ - - /* First determine if getauxval() is available (OSSL_IMPLEMENT_GETAUXVAL) */ # if defined(__GNUC__) && __GNUC__>=2 void OPENSSL_cpuid_setup(void) __attribute__ ((constructor)); @@ -100,10 +105,10 @@ static unsigned long getauxval(unsigned long key) * Android: according to https://developer.android.com/ndk/guides/cpu-features, * getauxval is supported starting with API level 18 */ -# if defined(__ANDROID__) && defined(__ANDROID_API__) && __ANDROID_API__ >= 18 -# include -# define OSSL_IMPLEMENT_GETAUXVAL -# endif +# if defined(__ANDROID__) && defined(__ANDROID_API__) && __ANDROID_API__ >= 18 +# include +# define OSSL_IMPLEMENT_GETAUXVAL +# endif /* * ARM puts the feature bits for Crypto Extensions in AT_HWCAP2, whereas @@ -116,144 +121,34 @@ static unsigned long getauxval(unsigned long key) # define AT_HWCAP2 26 # endif # if defined(__arm__) || defined (__arm) -# define OSSL_HWCAP AT_HWCAP -# define OSSL_HWCAP_NEON (1 << 12) - -# define OSSL_HWCAP_CE AT_HWCAP2 -# define OSSL_HWCAP_CE_AES (1 << 0) -# define OSSL_HWCAP_CE_PMULL (1 << 1) -# define OSSL_HWCAP_CE_SHA1 (1 << 2) -# define OSSL_HWCAP_CE_SHA256 (1 << 3) +# define HWCAP AT_HWCAP +# define HWCAP_NEON (1 << 12) + +# define HWCAP_CE AT_HWCAP2 +# define HWCAP_CE_AES (1 << 0) +# define HWCAP_CE_PMULL (1 << 1) +# define HWCAP_CE_SHA1 (1 << 2) +# define HWCAP_CE_SHA256 (1 << 3) # elif defined(__aarch64__) -# define OSSL_HWCAP AT_HWCAP -# define OSSL_HWCAP_NEON (1 << 1) - -# define OSSL_HWCAP_CE AT_HWCAP -# define OSSL_HWCAP_CE_AES (1 << 3) -# define OSSL_HWCAP_CE_PMULL (1 << 4) -# define OSSL_HWCAP_CE_SHA1 (1 << 5) -# define OSSL_HWCAP_CE_SHA256 (1 << 6) -# define OSSL_HWCAP_CPUID (1 << 11) -# define OSSL_HWCAP_SHA3 (1 << 17) -# define OSSL_HWCAP_CE_SM3 (1 << 18) -# define OSSL_HWCAP_CE_SM4 (1 << 19) -# define OSSL_HWCAP_CE_SHA512 (1 << 21) -# define OSSL_HWCAP_SVE (1 << 22) - /* AT_HWCAP2 */ -# define OSSL_HWCAP2 26 -# define OSSL_HWCAP2_SVE2 (1 << 1) -# define OSSL_HWCAP2_RNG (1 << 16) -# endif - -uint32_t _armv7_tick(void); - -uint32_t OPENSSL_rdtsc(void) -{ - if (OPENSSL_armcap_P & ARMV7_TICK) - return _armv7_tick(); - else - return 0; -} - -# ifdef __aarch64__ -size_t OPENSSL_rndr_asm(unsigned char *buf, size_t len); -size_t OPENSSL_rndrrs_asm(unsigned char *buf, size_t len); - -size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len); -size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len); - -static size_t OPENSSL_rndr_wrapper(size_t (*func)(unsigned char *, size_t), unsigned char *buf, size_t len) -{ - size_t buffer_size = 0; - int i; - - for (i = 0; i < 8; i++) { - buffer_size = func(buf, len); - if (buffer_size == len) - break; - usleep(5000); /* 5000 microseconds (5 milliseconds) */ - } - return buffer_size; -} - -size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len) -{ - return OPENSSL_rndr_wrapper(OPENSSL_rndr_asm, buf, len); -} - -size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len) -{ - return OPENSSL_rndr_wrapper(OPENSSL_rndrrs_asm, buf, len); -} -# endif - -# if !defined(__APPLE__) && !defined(OSSL_IMPLEMENT_GETAUXVAL) -static sigset_t all_masked; - -static sigjmp_buf ill_jmp; -static void ill_handler(int sig) -{ - siglongjmp(ill_jmp, sig); -} - -/* - * Following subroutines could have been inlined, but not all - * ARM compilers support inline assembler, and we'd then have to - * worry about the compiler optimising out the detection code... - */ -void _armv7_neon_probe(void); -void _armv8_aes_probe(void); -void _armv8_sha1_probe(void); -void _armv8_sha256_probe(void); -void _armv8_pmull_probe(void); -# ifdef __aarch64__ -void _armv8_sm3_probe(void); -void _armv8_sm4_probe(void); -void _armv8_sha512_probe(void); -void _armv8_eor3_probe(void); -void _armv8_sve_probe(void); -void _armv8_sve2_probe(void); -void _armv8_rng_probe(void); -# endif -# endif /* !__APPLE__ && !OSSL_IMPLEMENT_GETAUXVAL */ - -/* We only call _armv8_cpuid_probe() if (OPENSSL_armcap_P & ARMV8_CPUID) != 0 */ -unsigned int _armv8_cpuid_probe(void); - -# if defined(__APPLE__) -/* - * Checks the specified integer sysctl, returning `value` if it's 1, otherwise returning 0. - */ -static unsigned int sysctl_query(const char *name, unsigned int value) -{ - unsigned int sys_value = 0; - size_t len = sizeof(sys_value); - - return (sysctlbyname(name, &sys_value, &len, NULL, 0) == 0 && sys_value == 1) ? value : 0; -} -# elif !defined(OSSL_IMPLEMENT_GETAUXVAL) -/* - * Calls a provided probe function, which may SIGILL. If it doesn't, return `value`, otherwise return 0. - */ -static unsigned int arm_probe_for(void (*probe)(void), volatile unsigned int value) -{ - if (sigsetjmp(ill_jmp, 1) == 0) { - probe(); - return value; - } else { - /* The probe function gave us SIGILL */ - return 0; - } -} +# define HWCAP AT_HWCAP +# define HWCAP_NEON (1 << 1) + +# define HWCAP_CE HWCAP +# define HWCAP_CE_AES (1 << 3) +# define HWCAP_CE_PMULL (1 << 4) +# define HWCAP_CE_SHA1 (1 << 5) +# define HWCAP_CE_SHA256 (1 << 6) +# define HWCAP_CPUID (1 << 11) +# define HWCAP_CE_SM3 (1 << 18) +# define HWCAP_CE_SM4 (1 << 19) +# define HWCAP_CE_SHA512 (1 << 21) # endif void OPENSSL_cpuid_setup(void) { const char *e; -# if !defined(__APPLE__) && !defined(OSSL_IMPLEMENT_GETAUXVAL) struct sigaction ill_oact, ill_act; sigset_t oset; -# endif static int trigger = 0; if (trigger) @@ -268,7 +163,7 @@ void OPENSSL_cpuid_setup(void) } # if defined(__APPLE__) -# if !defined(__aarch64__) +# if !defined(__aarch64__) /* * Capability probing by catching SIGILL appears to be problematic * on iOS. But since Apple universe is "monocultural", it's actually @@ -278,84 +173,56 @@ void OPENSSL_cpuid_setup(void) OPENSSL_armcap_P = ARMV7_NEON; return; } -# else + /* + * One could do same even for __aarch64__ iOS builds. It's not done + * exclusively for reasons of keeping code unified across platforms. + * Unified code works because it never triggers SIGILL on Apple + * devices... + */ +# else { - /* - * From - * https://github.com/llvm/llvm-project/blob/412237dcd07e5a2afbb1767858262a5f037149a3/llvm/lib/Target/AArch64/AArch64.td#L719 - * all of these have been available on 64-bit Apple Silicon from the - * beginning (the A7). - */ - OPENSSL_armcap_P |= ARMV7_NEON | ARMV8_PMULL | ARMV8_AES | ARMV8_SHA1 | ARMV8_SHA256; - - /* More recent extensions are indicated by sysctls */ - OPENSSL_armcap_P |= sysctl_query("hw.optional.armv8_2_sha512", ARMV8_SHA512); - OPENSSL_armcap_P |= sysctl_query("hw.optional.armv8_2_sha3", ARMV8_SHA3); - - if (OPENSSL_armcap_P & ARMV8_SHA3) { - char uarch[64]; - - size_t len = sizeof(uarch); - if ((sysctlbyname("machdep.cpu.brand_string", uarch, &len, NULL, 0) == 0) && - ((strncmp(uarch, "Apple M1", 8) == 0) || - (strncmp(uarch, "Apple M2", 8) == 0) || - (strncmp(uarch, "Apple M3", 8) == 0))) { - OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3; - OPENSSL_armcap_P |= ARMV8_HAVE_SHA3_AND_WORTH_USING; - } - } - } -# endif /* __aarch64__ */ + unsigned int sha512; + size_t len = sizeof(sha512); -# elif defined(OSSL_IMPLEMENT_GETAUXVAL) + if (sysctlbyname("hw.optional.armv8_2_sha512", &sha512, &len, NULL, 0) == 0 && sha512 == 1) + OPENSSL_armcap_P |= ARMV8_SHA512; + } +# endif +# endif - if (getauxval(OSSL_HWCAP) & OSSL_HWCAP_NEON) { - unsigned long hwcap = getauxval(OSSL_HWCAP_CE); +# ifdef OSSL_IMPLEMENT_GETAUXVAL + if (getauxval(HWCAP) & HWCAP_NEON) { + unsigned long hwcap = getauxval(HWCAP_CE); OPENSSL_armcap_P |= ARMV7_NEON; - if (hwcap & OSSL_HWCAP_CE_AES) + if (hwcap & HWCAP_CE_AES) OPENSSL_armcap_P |= ARMV8_AES; - if (hwcap & OSSL_HWCAP_CE_PMULL) + if (hwcap & HWCAP_CE_PMULL) OPENSSL_armcap_P |= ARMV8_PMULL; - if (hwcap & OSSL_HWCAP_CE_SHA1) + if (hwcap & HWCAP_CE_SHA1) OPENSSL_armcap_P |= ARMV8_SHA1; - if (hwcap & OSSL_HWCAP_CE_SHA256) + if (hwcap & HWCAP_CE_SHA256) OPENSSL_armcap_P |= ARMV8_SHA256; # ifdef __aarch64__ - if (hwcap & OSSL_HWCAP_CE_SM4) + if (hwcap & HWCAP_CE_SM4) OPENSSL_armcap_P |= ARMV8_SM4; - if (hwcap & OSSL_HWCAP_CE_SHA512) + if (hwcap & HWCAP_CE_SHA512) OPENSSL_armcap_P |= ARMV8_SHA512; - if (hwcap & OSSL_HWCAP_CPUID) + if (hwcap & HWCAP_CPUID) OPENSSL_armcap_P |= ARMV8_CPUID; - if (hwcap & OSSL_HWCAP_CE_SM3) + if (hwcap & HWCAP_CE_SM3) OPENSSL_armcap_P |= ARMV8_SM3; - if (hwcap & OSSL_HWCAP_SHA3) - OPENSSL_armcap_P |= ARMV8_SHA3; # endif } -# ifdef __aarch64__ - if (getauxval(OSSL_HWCAP) & OSSL_HWCAP_SVE) - OPENSSL_armcap_P |= ARMV8_SVE; - - if (getauxval(OSSL_HWCAP2) & OSSL_HWCAP2_SVE2) - OPENSSL_armcap_P |= ARMV8_SVE2; - - if (getauxval(OSSL_HWCAP2) & OSSL_HWCAP2_RNG) - OPENSSL_armcap_P |= ARMV8_RNG; -# endif - -# else /* !__APPLE__ && !OSSL_IMPLEMENT_GETAUXVAL */ - - /* If all else fails, do brute force SIGILL-based feature detection */ +# endif sigfillset(&all_masked); sigdelset(&all_masked, SIGILL); @@ -371,42 +238,54 @@ void OPENSSL_cpuid_setup(void) sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset); sigaction(SIGILL, &ill_act, &ill_oact); - OPENSSL_armcap_P |= arm_probe_for(_armv7_neon_probe, ARMV7_NEON); - - if (OPENSSL_armcap_P & ARMV7_NEON) { - - OPENSSL_armcap_P |= arm_probe_for(_armv8_pmull_probe, ARMV8_PMULL | ARMV8_AES); - if (!(OPENSSL_armcap_P & ARMV8_AES)) { - OPENSSL_armcap_P |= arm_probe_for(_armv8_aes_probe, ARMV8_AES); + /* If we used getauxval, we already have all the values */ +# ifndef OSSL_IMPLEMENT_GETAUXVAL + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv7_neon_probe(); + OPENSSL_armcap_P |= ARMV7_NEON; + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_pmull_probe(); + OPENSSL_armcap_P |= ARMV8_PMULL | ARMV8_AES; + } else if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_aes_probe(); + OPENSSL_armcap_P |= ARMV8_AES; + } + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_sha1_probe(); + OPENSSL_armcap_P |= ARMV8_SHA1; + } + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_sha256_probe(); + OPENSSL_armcap_P |= ARMV8_SHA256; + } +# if defined(__aarch64__) && !defined(__APPLE__) + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_sm4_probe(); + OPENSSL_armcap_P |= ARMV8_SM4; } - OPENSSL_armcap_P |= arm_probe_for(_armv8_sha1_probe, ARMV8_SHA1); - OPENSSL_armcap_P |= arm_probe_for(_armv8_sha256_probe, ARMV8_SHA256); + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_sha512_probe(); + OPENSSL_armcap_P |= ARMV8_SHA512; + } -# if defined(__aarch64__) - OPENSSL_armcap_P |= arm_probe_for(_armv8_sm3_probe, ARMV8_SM3); - OPENSSL_armcap_P |= arm_probe_for(_armv8_sm4_probe, ARMV8_SM4); - OPENSSL_armcap_P |= arm_probe_for(_armv8_sha512_probe, ARMV8_SHA512); - OPENSSL_armcap_P |= arm_probe_for(_armv8_eor3_probe, ARMV8_SHA3); + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_sm3_probe(); + OPENSSL_armcap_P |= ARMV8_SM3; + } # endif } -# ifdef __aarch64__ - OPENSSL_armcap_P |= arm_probe_for(_armv8_sve_probe, ARMV8_SVE); - OPENSSL_armcap_P |= arm_probe_for(_armv8_sve2_probe, ARMV8_SVE2); - OPENSSL_armcap_P |= arm_probe_for(_armv8_rng_probe, ARMV8_RNG); -# endif +# endif - /* - * Probing for ARMV7_TICK is known to produce unreliable results, - * so we only use the feature when the user explicitly enables it - * with OPENSSL_armcap. - */ + /* Things that getauxval didn't tell us */ + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv7_tick(); + OPENSSL_armcap_P |= ARMV7_TICK; + } sigaction(SIGILL, &ill_oact, NULL); sigprocmask(SIG_SETMASK, &oset, NULL); -# endif /* __APPLE__, OSSL_IMPLEMENT_GETAUXVAL */ - # ifdef __aarch64__ if (OPENSSL_armcap_P & ARMV8_CPUID) OPENSSL_arm_midr = _armv8_cpuid_probe(); @@ -416,30 +295,6 @@ void OPENSSL_cpuid_setup(void) (OPENSSL_armcap_P & ARMV7_NEON)) { OPENSSL_armv8_rsa_neonized = 1; } - if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V1) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N2) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_MICROSOFT, MICROSOFT_CPU_PART_COBALT_100) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V2)) && - (OPENSSL_armcap_P & ARMV8_SHA3)) - OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3; - if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V1) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V2)) && - (OPENSSL_armcap_P & ARMV8_SHA3)) - OPENSSL_armcap_P |= ARMV8_UNROLL12_EOR3; - if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_PRO) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_PRO) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_MAX) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_MAX) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_BLIZZARD) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_PRO) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_BLIZZARD_PRO) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_MAX) || - MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_BLIZZARD_MAX)) && - (OPENSSL_armcap_P & ARMV8_SHA3)) - OPENSSL_armcap_P |= ARMV8_HAVE_SHA3_AND_WORTH_USING; # endif } -#endif /* _WIN32, __ARM_MAX_ARCH__ >= 7 */ +#endif diff --git a/openssl/src/crypto/asn1/a_bitstr.c b/openssl/src/crypto/asn1/a_bitstr.c index d39407063..7c2564935 100644 --- a/openssl/src/crypto/asn1/a_bitstr.c +++ b/openssl/src/crypto/asn1/a_bitstr.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -82,7 +82,7 @@ ASN1_BIT_STRING *ossl_c2i_ASN1_BIT_STRING(ASN1_BIT_STRING **a, ASN1_BIT_STRING *ret = NULL; const unsigned char *p; unsigned char *s; - int i = 0; + int i; if (len < 1) { i = ASN1_R_STRING_TOO_SHORT; @@ -110,11 +110,13 @@ ASN1_BIT_STRING *ossl_c2i_ASN1_BIT_STRING(ASN1_BIT_STRING **a, * We do this to preserve the settings. If we modify the settings, via * the _set_bit function, we will recalculate on output */ - ossl_asn1_string_set_bits_left(ret, i); + ret->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); /* clear */ + ret->flags |= (ASN1_STRING_FLAG_BITS_LEFT | i); /* set */ if (len-- > 1) { /* using one because of the bits left byte */ s = OPENSSL_malloc((int)len); if (s == NULL) { + i = ERR_R_MALLOC_FAILURE; goto err; } memcpy(s, p, (int)len); @@ -123,15 +125,16 @@ ASN1_BIT_STRING *ossl_c2i_ASN1_BIT_STRING(ASN1_BIT_STRING **a, } else s = NULL; - ASN1_STRING_set0(ret, s, (int)len); + ret->length = (int)len; + OPENSSL_free(ret->data); + ret->data = s; ret->type = V_ASN1_BIT_STRING; if (a != NULL) (*a) = ret; *pp = p; return ret; err: - if (i != 0) - ERR_raise(ERR_LIB_ASN1, i); + ERR_raise(ERR_LIB_ASN1, i); if ((a == NULL) || (*a != ret)) ASN1_BIT_STRING_free(ret); return NULL; @@ -145,9 +148,6 @@ int ASN1_BIT_STRING_set_bit(ASN1_BIT_STRING *a, int n, int value) int w, v, iv; unsigned char *c; - if (n < 0) - return 0; - w = n / 8; v = 1 << (7 - (n & 0x07)); iv = ~v; @@ -163,8 +163,10 @@ int ASN1_BIT_STRING_set_bit(ASN1_BIT_STRING *a, int n, int value) if (!value) return 1; /* Don't need to set */ c = OPENSSL_clear_realloc(a->data, a->length, w + 1); - if (c == NULL) + if (c == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } if (w + 1 - a->length > 0) memset(c + a->length, 0, w + 1 - a->length); a->data = c; @@ -180,9 +182,6 @@ int ASN1_BIT_STRING_get_bit(const ASN1_BIT_STRING *a, int n) { int w, v; - if (n < 0) - return 0; - w = n / 8; v = 1 << (7 - (n & 0x07)); if ((a == NULL) || (a->length < (w + 1)) || (a->data == NULL)) diff --git a/openssl/src/crypto/asn1/a_d2i_fp.c b/openssl/src/crypto/asn1/a_d2i_fp.c index 4af2276a8..e8602053f 100644 --- a/openssl/src/crypto/asn1/a_d2i_fp.c +++ b/openssl/src/crypto/asn1/a_d2i_fp.c @@ -123,7 +123,7 @@ int asn1_d2i_read_bio(BIO *in, BUF_MEM **pb) b = BUF_MEM_new(); if (b == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } @@ -134,7 +134,7 @@ int asn1_d2i_read_bio(BIO *in, BUF_MEM **pb) want -= diff; if (len + want < len || !BUF_MEM_grow_clean(b, len + want)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } i = BIO_read(in, &(b->data[len]), want); @@ -206,7 +206,7 @@ int asn1_d2i_read_bio(BIO *in, BUF_MEM **pb) size_t chunk = want > chunk_max ? chunk_max : want; if (!BUF_MEM_grow_clean(b, len + chunk)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } want -= chunk; diff --git a/openssl/src/crypto/asn1/a_digest.c b/openssl/src/crypto/asn1/a_digest.c index 67e8a96ba..72cc88077 100644 --- a/openssl/src/crypto/asn1/a_digest.c +++ b/openssl/src/crypto/asn1/a_digest.c @@ -36,8 +36,10 @@ int ASN1_digest(i2d_of_void *i2d, const EVP_MD *type, char *data, ERR_raise(ERR_LIB_ASN1, ERR_R_INTERNAL_ERROR); return 0; } - if ((str = OPENSSL_malloc(inl)) == NULL) + if ((str = OPENSSL_malloc(inl)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } p = str; i2d(data, &p); diff --git a/openssl/src/crypto/asn1/a_dup.c b/openssl/src/crypto/asn1/a_dup.c index 23d1d6380..93e8b2aa8 100644 --- a/openssl/src/crypto/asn1/a_dup.c +++ b/openssl/src/crypto/asn1/a_dup.c @@ -28,8 +28,10 @@ void *ASN1_dup(i2d_of_void *i2d, d2i_of_void *d2i, const void *x) return NULL; b = OPENSSL_malloc(i + 10); - if (b == NULL) + if (b == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; + } p = b; i = i2d(x, &p); p2 = b; @@ -76,7 +78,7 @@ void *ASN1_item_dup(const ASN1_ITEM *it, const void *x) i = ASN1_item_i2d(x, &b, it); if (b == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; } p = b; diff --git a/openssl/src/crypto/asn1/a_i2d_fp.c b/openssl/src/crypto/asn1/a_i2d_fp.c index e30f1f2a1..4cc477366 100644 --- a/openssl/src/crypto/asn1/a_i2d_fp.c +++ b/openssl/src/crypto/asn1/a_i2d_fp.c @@ -42,8 +42,10 @@ int ASN1_i2d_bio(i2d_of_void *i2d, BIO *out, const void *x) return 0; b = OPENSSL_malloc(n); - if (b == NULL) + if (b == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } p = (unsigned char *)b; i2d(x, &p); @@ -89,7 +91,7 @@ int ASN1_item_i2d_bio(const ASN1_ITEM *it, BIO *out, const void *x) n = ASN1_item_i2d(x, &b, it); if (b == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } diff --git a/openssl/src/crypto/asn1/a_int.c b/openssl/src/crypto/asn1/a_int.c index dc962290d..19e41ec73 100644 --- a/openssl/src/crypto/asn1/a_int.c +++ b/openssl/src/crypto/asn1/a_int.c @@ -303,10 +303,8 @@ ASN1_INTEGER *ossl_c2i_ASN1_INTEGER(ASN1_INTEGER **a, const unsigned char **pp, } else ret = *a; - if (ASN1_STRING_set(ret, NULL, r) == 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + if (ASN1_STRING_set(ret, NULL, r) == 0) goto err; - } c2i_ibuf(ret->data, &neg, *pp, len); @@ -320,6 +318,7 @@ ASN1_INTEGER *ossl_c2i_ASN1_INTEGER(ASN1_INTEGER **a, const unsigned char **pp, (*a) = ret; return ret; err: + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); if (a == NULL || *a != ret) ASN1_INTEGER_free(ret); return NULL; @@ -401,7 +400,7 @@ ASN1_INTEGER *d2i_ASN1_UINTEGER(ASN1_INTEGER **a, const unsigned char **pp, unsigned char *s; long len = 0; int inf, tag, xclass; - int i = 0; + int i; if ((a == NULL) || ((*a) == NULL)) { if ((ret = ASN1_INTEGER_new()) == NULL) @@ -431,8 +430,10 @@ ASN1_INTEGER *d2i_ASN1_UINTEGER(ASN1_INTEGER **a, const unsigned char **pp, * a missing NULL parameter. */ s = OPENSSL_malloc((int)len + 1); - if (s == NULL) + if (s == NULL) { + i = ERR_R_MALLOC_FAILURE; goto err; + } ret->type = V_ASN1_INTEGER; if (len) { if ((*p == 0) && (len != 1)) { @@ -443,14 +444,15 @@ ASN1_INTEGER *d2i_ASN1_UINTEGER(ASN1_INTEGER **a, const unsigned char **pp, p += len; } - ASN1_STRING_set0(ret, s, (int)len); + OPENSSL_free(ret->data); + ret->data = s; + ret->length = (int)len; if (a != NULL) (*a) = ret; *pp = p; return ret; err: - if (i != 0) - ERR_raise(ERR_LIB_ASN1, i); + ERR_raise(ERR_LIB_ASN1, i); if ((a == NULL) || (*a != ret)) ASN1_INTEGER_free(ret); return NULL; @@ -483,7 +485,7 @@ static ASN1_STRING *bn_to_asn1_string(const BIGNUM *bn, ASN1_STRING *ai, len = 1; if (ASN1_STRING_set(ret, NULL, len) == 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } diff --git a/openssl/src/crypto/asn1/a_mbstr.c b/openssl/src/crypto/asn1/a_mbstr.c index c8170e162..22dea873e 100644 --- a/openssl/src/crypto/asn1/a_mbstr.c +++ b/openssl/src/crypto/asn1/a_mbstr.c @@ -139,13 +139,15 @@ int ASN1_mbstring_ncopy(ASN1_STRING **out, const unsigned char *in, int len, if (*out) { free_out = 0; dest = *out; - ASN1_STRING_set0(dest, NULL, 0); + OPENSSL_free(dest->data); + dest->data = NULL; + dest->length = 0; dest->type = str_type; } else { free_out = 1; dest = ASN1_STRING_type_new(str_type); if (dest == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } *out = dest; @@ -153,11 +155,7 @@ int ASN1_mbstring_ncopy(ASN1_STRING **out, const unsigned char *in, int len, /* If both the same type just copy across */ if (inform == outform) { if (!ASN1_STRING_set(dest, in, len)) { - if (free_out) { - ASN1_STRING_free(dest); - *out = NULL; - } - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } return str_type; @@ -187,10 +185,9 @@ int ASN1_mbstring_ncopy(ASN1_STRING **out, const unsigned char *in, int len, break; } if ((p = OPENSSL_malloc(outlen + 1)) == NULL) { - if (free_out) { + if (free_out) ASN1_STRING_free(dest); - *out = NULL; - } + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } dest->length = outlen; diff --git a/openssl/src/crypto/asn1/a_object.c b/openssl/src/crypto/asn1/a_object.c index 73c69eacd..c96c36e73 100644 --- a/openssl/src/crypto/asn1/a_object.c +++ b/openssl/src/crypto/asn1/a_object.c @@ -31,8 +31,10 @@ int i2d_ASN1_OBJECT(const ASN1_OBJECT *a, unsigned char **pp) return objsize; if (*pp == NULL) { - if ((p = allocated = OPENSSL_malloc(objsize)) == NULL) + if ((p = allocated = OPENSSL_malloc(objsize)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } } else { p = *pp; } @@ -133,8 +135,10 @@ int a2d_ASN1_OBJECT(unsigned char *out, int olen, const char *buf, int num) OPENSSL_free(tmp); tmpsize = blsize + 32; tmp = OPENSSL_malloc(tmpsize); - if (tmp == NULL) + if (tmp == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; + } } while (blsize--) { BN_ULONG t = BN_div_word(bl, 0x80L); @@ -192,8 +196,10 @@ int i2a_ASN1_OBJECT(BIO *bp, const ASN1_OBJECT *a) ERR_raise(ERR_LIB_ASN1, ASN1_R_LENGTH_TOO_LONG); return -1; } - if ((p = OPENSSL_malloc(i + 1)) == NULL) + if ((p = OPENSSL_malloc(i + 1)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; + } i2t_ASN1_OBJECT(p, i + 1, a); } if (i <= 0) { @@ -302,8 +308,10 @@ ASN1_OBJECT *ossl_c2i_ASN1_OBJECT(ASN1_OBJECT **a, const unsigned char **pp, ret->length = 0; OPENSSL_free(data); data = OPENSSL_malloc(length); - if (data == NULL) + if (data == NULL) { + i = ERR_R_MALLOC_FAILURE; goto err; + } ret->flags |= ASN1_OBJECT_FLAG_DYNAMIC_DATA; } memcpy(data, p, length); @@ -337,8 +345,10 @@ ASN1_OBJECT *ASN1_OBJECT_new(void) ASN1_OBJECT *ret; ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; + } ret->flags = ASN1_OBJECT_FLAG_DYNAMIC; return ret; } diff --git a/openssl/src/crypto/asn1/a_sign.c b/openssl/src/crypto/asn1/a_sign.c index 8507fc366..302045cfc 100644 --- a/openssl/src/crypto/asn1/a_sign.c +++ b/openssl/src/crypto/asn1/a_sign.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -35,7 +35,7 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2, X509_ALGOR *a; if (ctx == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } for (i = 0; i < 2; i++) { @@ -82,6 +82,7 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2, buf_out = OPENSSL_malloc(outll); if (buf_in == NULL || buf_out == NULL) { outl = 0; + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } p = buf_in; @@ -95,13 +96,16 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2, ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); goto err; } - ASN1_STRING_set0(signature, buf_out, outl); + OPENSSL_free(signature->data); + signature->data = buf_out; buf_out = NULL; + signature->length = outl; /* * In the interests of compatibility, I'll make sure that the bit string * has a 'not-used bits' value of 0 */ - ossl_asn1_string_set_bits_left(signature, 0); + signature->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + signature->flags |= ASN1_STRING_FLAG_BITS_LEFT; err: EVP_MD_CTX_free(ctx); OPENSSL_clear_free((char *)buf_in, inll); @@ -129,7 +133,7 @@ int ASN1_item_sign_ex(const ASN1_ITEM *it, X509_ALGOR *algor1, EVP_MD_CTX *ctx = evp_md_ctx_new_ex(pkey, id, libctx, propq); if (ctx == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } /* We can use the non _ex variant here since the pkey is already setup */ @@ -243,14 +247,16 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it, X509_ALGOR *algor1, goto err; } - paramtype = pkey->ameth->pkey_flags & ASN1_PKEY_SIGPARAM_NULL ? - V_ASN1_NULL : V_ASN1_UNDEF; - if (algor1 != NULL - && !X509_ALGOR_set0(algor1, OBJ_nid2obj(signid), paramtype, NULL)) - goto err; - if (algor2 != NULL - && !X509_ALGOR_set0(algor2, OBJ_nid2obj(signid), paramtype, NULL)) - goto err; + if (pkey->ameth->pkey_flags & ASN1_PKEY_SIGPARAM_NULL) + paramtype = V_ASN1_NULL; + else + paramtype = V_ASN1_UNDEF; + + if (algor1) + X509_ALGOR_set0(algor1, OBJ_nid2obj(signid), paramtype, NULL); + if (algor2) + X509_ALGOR_set0(algor2, OBJ_nid2obj(signid), paramtype, NULL); + } buf_len = ASN1_item_i2d(data, &buf_in, it); @@ -269,6 +275,7 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it, X509_ALGOR *algor1, buf_out = OPENSSL_malloc(outll); if (buf_in == NULL || buf_out == NULL) { outl = 0; + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } @@ -277,13 +284,16 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it, X509_ALGOR *algor1, ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); goto err; } - ASN1_STRING_set0(signature, buf_out, outl); + OPENSSL_free(signature->data); + signature->data = buf_out; buf_out = NULL; + signature->length = outl; /* * In the interests of compatibility, I'll make sure that the bit string * has a 'not-used bits' value of 0 */ - ossl_asn1_string_set_bits_left(signature, 0); + signature->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + signature->flags |= ASN1_STRING_FLAG_BITS_LEFT; err: OPENSSL_clear_free((char *)buf_in, inl); OPENSSL_clear_free((char *)buf_out, outll); diff --git a/openssl/src/crypto/asn1/a_strex.c b/openssl/src/crypto/asn1/a_strex.c index 29ea60596..b31761aae 100644 --- a/openssl/src/crypto/asn1/a_strex.c +++ b/openssl/src/crypto/asn1/a_strex.c @@ -282,8 +282,10 @@ static int do_dump(unsigned long lflags, char_io *io_ch, void *arg, der_len = i2d_ASN1_TYPE(&t, NULL); if (der_len <= 0) return -1; - if ((der_buf = OPENSSL_malloc(der_len)) == NULL) + if ((der_buf = OPENSSL_malloc(der_len)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; + } p = der_buf; i2d_ASN1_TYPE(&t, &p); outlen = do_hex_dump(io_ch, arg, der_buf, der_len); diff --git a/openssl/src/crypto/asn1/a_strnid.c b/openssl/src/crypto/asn1/a_strnid.c index 99ac2aed1..9e54db929 100644 --- a/openssl/src/crypto/asn1/a_strnid.c +++ b/openssl/src/crypto/asn1/a_strnid.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -50,10 +50,10 @@ int ASN1_STRING_set_default_mask_asc(const char *p) unsigned long mask; char *end; - if (CHECK_AND_SKIP_PREFIX(p, "MASK:")) { - if (*p == '\0') + if (strncmp(p, "MASK:", 5) == 0) { + if (p[5] == '\0') return 0; - mask = strtoul(p, &end, 0); + mask = strtoul(p + 5, &end, 0); if (*end) return 0; } else if (strcmp(p, "nombstr") == 0) @@ -129,15 +129,11 @@ ASN1_STRING_TABLE *ASN1_STRING_TABLE_get(int nid) int idx; ASN1_STRING_TABLE fnd; -#ifndef OPENSSL_NO_AUTOLOAD_CONFIG /* "stable" can be impacted by config, so load the config file first */ OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); -#endif fnd.nid = nid; - if (stable != NULL) { - /* Ideally, this would be done under lock */ - sk_ASN1_STRING_TABLE_sort(stable); + if (stable) { idx = sk_ASN1_STRING_TABLE_find(stable, &fnd); if (idx >= 0) return sk_ASN1_STRING_TABLE_value(stable, idx); @@ -163,8 +159,10 @@ static ASN1_STRING_TABLE *stable_get(int nid) tmp = ASN1_STRING_TABLE_get(nid); if (tmp != NULL && tmp->flags & STABLE_FLAGS_MALLOC) return tmp; - if ((rv = OPENSSL_zalloc(sizeof(*rv))) == NULL) + if ((rv = OPENSSL_zalloc(sizeof(*rv))) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; + } if (!sk_ASN1_STRING_TABLE_push(stable, rv)) { OPENSSL_free(rv); return NULL; @@ -192,7 +190,7 @@ int ASN1_STRING_TABLE_add(int nid, tmp = stable_get(nid); if (tmp == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } if (minsize >= 0) diff --git a/openssl/src/crypto/asn1/a_time.c b/openssl/src/crypto/asn1/a_time.c index 96ee63d31..9b3074e47 100644 --- a/openssl/src/crypto/asn1/a_time.c +++ b/openssl/src/crypto/asn1/a_time.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -79,7 +79,7 @@ int ossl_asn1_time_to_tm(struct tm *tm, const ASN1_TIME *d) static const int max[9] = { 99, 99, 12, 31, 23, 59, 59, 12, 59 }; static const int mdays[12] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; char *a; - int n, i, i2, l, o, min_l, strict = 0, end = 6, btz = 5, md; + int n, i, i2, l, o, min_l = 11, strict = 0, end = 6, btz = 5, md; struct tm tmp; #if defined(CHARSET_EBCDIC) const char upper_z = 0x5A, num_zero = 0x30, period = 0x2E, minus = 0x2D, plus = 0x2B; @@ -92,19 +92,21 @@ int ossl_asn1_time_to_tm(struct tm *tm, const ASN1_TIME *d) * * 1. "seconds" is a 'MUST' * 2. "Zulu" timezone is a 'MUST' - * 3. "+|-" is not allowed to indicate a timezone + * 3. "+|-" is not allowed to indicate a time zone */ if (d->type == V_ASN1_UTCTIME) { - min_l = 13; if (d->flags & ASN1_STRING_FLAG_X509_TIME) { + min_l = 13; strict = 1; } } else if (d->type == V_ASN1_GENERALIZEDTIME) { end = 7; btz = 6; - min_l = 15; if (d->flags & ASN1_STRING_FLAG_X509_TIME) { + min_l = 15; strict = 1; + } else { + min_l = 13; } } else { return 0; @@ -293,22 +295,16 @@ ASN1_TIME *ossl_asn1_time_from_tm(ASN1_TIME *s, struct tm *ts, int type) tmps->type = type; p = (char*)tmps->data; - if (ts->tm_mon > INT_MAX - 1) - goto err; - - if (type == V_ASN1_GENERALIZEDTIME) { - if (ts->tm_year > INT_MAX - 1900) - goto err; + if (type == V_ASN1_GENERALIZEDTIME) tmps->length = BIO_snprintf(p, len, "%04d%02d%02d%02d%02d%02dZ", ts->tm_year + 1900, ts->tm_mon + 1, ts->tm_mday, ts->tm_hour, ts->tm_min, ts->tm_sec); - } else { + else tmps->length = BIO_snprintf(p, len, "%02d%02d%02d%02d%02d%02dZ", ts->tm_year % 100, ts->tm_mon + 1, ts->tm_mday, ts->tm_hour, ts->tm_min, ts->tm_sec); - } #ifdef CHARSET_EBCDIC ebcdic2ascii(tmps->data, tmps->data, tmps->length); @@ -424,8 +420,10 @@ int ASN1_TIME_set_string_X509(ASN1_TIME *s, const char *str) * new t.data would be freed after ASN1_STRING_copy is done. */ t.data = OPENSSL_zalloc(t.length + 1); - if (t.data == NULL) + if (t.data == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto out; + } memcpy(t.data, str + 2, t.length); t.type = V_ASN1_UTCTIME; } @@ -573,7 +571,7 @@ int ASN1_TIME_normalize(ASN1_TIME *t) { struct tm tm; - if (t == NULL || !ASN1_TIME_to_tm(t, &tm)) + if (!ASN1_TIME_to_tm(t, &tm)) return 0; return ossl_asn1_time_from_tm(t, &tm, V_ASN1_UNDEF) != NULL; @@ -591,78 +589,3 @@ int ASN1_TIME_compare(const ASN1_TIME *a, const ASN1_TIME *b) return -1; return 0; } - -/* - * tweak for Windows - */ -#ifdef WIN32 -# define timezone _timezone -#endif - -#if defined(__FreeBSD__) || defined(__wasi__) -# define USE_TIMEGM -#endif - -time_t ossl_asn1_string_to_time_t(const char *asn1_string) -{ - ASN1_TIME *timestamp_asn1 = NULL; - struct tm *timestamp_tm = NULL; -#if defined(__DJGPP__) - char *tz = NULL; -#elif !defined(USE_TIMEGM) - time_t timestamp_local; -#endif - time_t timestamp_utc; - - timestamp_asn1 = ASN1_TIME_new(); - if (!ASN1_TIME_set_string(timestamp_asn1, asn1_string)) - { - ASN1_TIME_free(timestamp_asn1); - return -1; - } - - timestamp_tm = OPENSSL_malloc(sizeof(*timestamp_tm)); - if (timestamp_tm == NULL) { - ASN1_TIME_free(timestamp_asn1); - return -1; - } - if (!(ASN1_TIME_to_tm(timestamp_asn1, timestamp_tm))) { - OPENSSL_free(timestamp_tm); - ASN1_TIME_free(timestamp_asn1); - return -1; - } - ASN1_TIME_free(timestamp_asn1); - -#if defined(__DJGPP__) - /* - * This is NOT thread-safe. Do not use this method for platforms other - * than djgpp. - */ - tz = getenv("TZ"); - if (tz != NULL) { - tz = OPENSSL_strdup(tz); - if (tz == NULL) { - OPENSSL_free(timestamp_tm); - return -1; - } - } - setenv("TZ", "UTC", 1); - - timestamp_utc = mktime(timestamp_tm); - - if (tz != NULL) { - setenv("TZ", tz, 1); - OPENSSL_free(tz); - } else { - unsetenv("TZ"); - } -#elif defined(USE_TIMEGM) - timestamp_utc = timegm(timestamp_tm); -#else - timestamp_local = mktime(timestamp_tm); - timestamp_utc = timestamp_local - timezone; -#endif - OPENSSL_free(timestamp_tm); - - return timestamp_utc; -} diff --git a/openssl/src/crypto/asn1/a_verify.c b/openssl/src/crypto/asn1/a_verify.c index 94d29e7c2..ed5641790 100644 --- a/openssl/src/crypto/asn1/a_verify.c +++ b/openssl/src/crypto/asn1/a_verify.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,7 +33,7 @@ int ASN1_verify(i2d_of_void *i2d, X509_ALGOR *a, ASN1_BIT_STRING *signature, int ret = -1, i, inl; if (ctx == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } i = OBJ_obj2nid(a->algorithm); @@ -54,8 +54,10 @@ int ASN1_verify(i2d_of_void *i2d, X509_ALGOR *a, ASN1_BIT_STRING *signature, goto err; } buf_in = OPENSSL_malloc((unsigned int)inl); - if (buf_in == NULL) + if (buf_in == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; + } p = buf_in; i2d(data, &p); @@ -96,14 +98,73 @@ int ASN1_item_verify_ex(const ASN1_ITEM *it, const X509_ALGOR *alg, const ASN1_OCTET_STRING *id, EVP_PKEY *pkey, OSSL_LIB_CTX *libctx, const char *propq) { - EVP_MD_CTX *ctx; int rv = -1; + int mdnid, pknid; + const EVP_MD *md = NULL; + EVP_MD_CTX *ctx = evp_md_ctx_new_ex(pkey, id, libctx, propq); + + if (ctx == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + return rv; + } - if ((ctx = evp_md_ctx_new_ex(pkey, id, libctx, propq)) != NULL) { - rv = ASN1_item_verify_ctx(it, alg, signature, data, ctx); - EVP_PKEY_CTX_free(EVP_MD_CTX_get_pkey_ctx(ctx)); - EVP_MD_CTX_free(ctx); + /* Convert signature OID into digest and public key OIDs */ + if (!OBJ_find_sigid_algs(OBJ_obj2nid(alg->algorithm), &mdnid, &pknid)) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_UNKNOWN_SIGNATURE_ALGORITHM); + goto err; } + + if (mdnid != NID_undef) { + md = EVP_get_digestbynid(mdnid); + if (md == NULL) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM); + goto err; + } + } + +#ifndef OPENSSL_NO_SM2 + if (pknid == NID_sm2) { + EVP_PKEY_CTX *pctx = NULL, *opctx = EVP_MD_CTX_get_pkey_ctx(ctx); + + if (pknid == NID_sm2 && !EVP_PKEY_set_alias_type(pkey, EVP_PKEY_SM2)) { + ERR_raise(ERR_LIB_ASN1, ERR_R_INTERNAL_ERROR); + goto err; + } + + pctx = EVP_PKEY_CTX_new_from_pkey(opctx->libctx, pkey, + opctx->propquery); + if (pctx == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + goto err; + } + + if (EVP_PKEY_CTX_set1_id(pctx, + opctx->cached_parameters.dist_id, + opctx->cached_parameters.dist_id_len) != 1) { + ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + goto err; + } + + EVP_PKEY_CTX_free(opctx); + EVP_MD_CTX_set_pkey_ctx(ctx, pctx); + } +#endif + + /* + * Note that some algorithms (notably Ed25519 and Ed448) may allow + * a NULL digest value. + */ + if (!EVP_DigestVerifyInit(ctx, NULL, md, NULL, pkey)) { + ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + rv = 0; + goto err; + } + + rv = ASN1_item_verify_ctx(it, alg, signature, data, ctx); + + err: + EVP_PKEY_CTX_free(EVP_MD_CTX_get_pkey_ctx(ctx)); + EVP_MD_CTX_free(ctx); return rv; } @@ -152,8 +213,6 @@ int ASN1_item_verify_ctx(const ASN1_ITEM *it, const X509_ALGOR *alg, if (ret <= 1) goto err; } else { - const EVP_MD *type = NULL; - /* * We don't yet have the ability for providers to be able to handle * X509_ALGOR style parameters. Fortunately the only one that needs this @@ -176,26 +235,6 @@ int ASN1_item_verify_ctx(const ASN1_ITEM *it, const X509_ALGOR *alg, ERR_raise(ERR_LIB_ASN1, ASN1_R_WRONG_PUBLIC_KEY_TYPE); goto err; } - - if (mdnid != NID_undef) { - type = EVP_get_digestbynid(mdnid); - if (type == NULL) { - ERR_raise_data(ERR_LIB_ASN1, - ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM, - "nid=0x%x", mdnid); - goto err; - } - } - - /* - * Note that some algorithms (notably Ed25519 and Ed448) may allow - * a NULL digest value. - */ - if (!EVP_DigestVerifyInit(ctx, NULL, type, NULL, pkey)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); - ret = 0; - goto err; - } } } @@ -205,7 +244,7 @@ int ASN1_item_verify_ctx(const ASN1_ITEM *it, const X509_ALGOR *alg, goto err; } if (buf_in == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } inll = inl; diff --git a/openssl/src/crypto/asn1/ameth_lib.c b/openssl/src/crypto/asn1/ameth_lib.c index 6ba13dd7f..8b15da3be 100644 --- a/openssl/src/crypto/asn1/ameth_lib.c +++ b/openssl/src/crypto/asn1/ameth_lib.c @@ -222,8 +222,10 @@ EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_new(int id, int flags, { EVP_PKEY_ASN1_METHOD *ameth = OPENSSL_zalloc(sizeof(*ameth)); - if (ameth == NULL) + if (ameth == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; + } ameth->pkey_id = id; ameth->pkey_base_id = id; @@ -245,6 +247,7 @@ EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_new(int id, int flags, err: EVP_PKEY_asn1_free(ameth); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/asn1/asn1_err.c b/openssl/src/crypto/asn1/asn1_err.c index f52584244..a7b32e3a6 100644 --- a/openssl/src/crypto/asn1/asn1_err.c +++ b/openssl/src/crypto/asn1/asn1_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -55,8 +55,6 @@ static const ERR_STRING_DATA ASN1_str_reasons[] = { {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_FIELD_MISSING), "field missing"}, {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_FIRST_NUM_TOO_LARGE), "first num too large"}, - {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_GENERALIZEDTIME_IS_TOO_SHORT), - "generalizedtime is too short"}, {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_HEADER_TOO_LONG), "header too long"}, {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_ILLEGAL_BITSTRING_FORMAT), "illegal bitstring format"}, @@ -194,8 +192,6 @@ static const ERR_STRING_DATA ASN1_str_reasons[] = { {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE), "unsupported public key type"}, {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_UNSUPPORTED_TYPE), "unsupported type"}, - {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_UTCTIME_IS_TOO_SHORT), - "utctime is too short"}, {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_WRONG_INTEGER_TYPE), "wrong integer type"}, {ERR_PACK(ERR_LIB_ASN1, 0, ASN1_R_WRONG_PUBLIC_KEY_TYPE), diff --git a/openssl/src/crypto/asn1/asn1_gen.c b/openssl/src/crypto/asn1/asn1_gen.c index 6f73449cf..64620a4f2 100644 --- a/openssl/src/crypto/asn1/asn1_gen.c +++ b/openssl/src/crypto/asn1/asn1_gen.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,10 +7,9 @@ * https://www.openssl.org/source/license.html */ +#include "internal/cryptlib.h" #include #include -#include "internal/cryptlib.h" -#include "crypto/asn1.h" #define ASN1_GEN_FLAG 0x10000 #define ASN1_GEN_FLAG_IMP (ASN1_GEN_FLAG|1) @@ -325,13 +324,13 @@ static int asn1_cb(const char *elem, int len, void *bitstr) ERR_raise(ERR_LIB_ASN1, ASN1_R_UNKNOWN_FORMAT); return -1; } - if (HAS_PREFIX(vstart, "ASCII")) + if (strncmp(vstart, "ASCII", 5) == 0) arg->format = ASN1_GEN_FORMAT_ASCII; - else if (HAS_PREFIX(vstart, "UTF8")) + else if (strncmp(vstart, "UTF8", 4) == 0) arg->format = ASN1_GEN_FORMAT_UTF8; - else if (HAS_PREFIX(vstart, "HEX")) + else if (strncmp(vstart, "HEX", 3) == 0) arg->format = ASN1_GEN_FORMAT_HEX; - else if (HAS_PREFIX(vstart, "BITLIST")) + else if (strncmp(vstart, "BITLIST", 7) == 0) arg->format = ASN1_GEN_FORMAT_BITLIST; else { ERR_raise(ERR_LIB_ASN1, ASN1_R_UNKNOWN_FORMAT); @@ -582,7 +581,7 @@ static ASN1_TYPE *asn1_str2type(const char *str, int format, int utype) int no_unused = 1; if ((atmp = ASN1_TYPE_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; } @@ -643,11 +642,11 @@ static ASN1_TYPE *asn1_str2type(const char *str, int format, int utype) goto bad_form; } if ((atmp->value.asn1_string = ASN1_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto bad_str; } if (!ASN1_STRING_set(atmp->value.asn1_string, str, -1)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto bad_str; } atmp->value.asn1_string->type = utype; @@ -678,7 +677,7 @@ static ASN1_TYPE *asn1_str2type(const char *str, int format, int utype) if (ASN1_mbstring_copy(&atmp->value.asn1_string, (unsigned char *)str, -1, format, ASN1_tag2bit(utype)) <= 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto bad_str; } @@ -687,7 +686,7 @@ static ASN1_TYPE *asn1_str2type(const char *str, int format, int utype) case V_ASN1_BIT_STRING: case V_ASN1_OCTET_STRING: if ((atmp->value.asn1_string = ASN1_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto bad_form; } @@ -699,12 +698,9 @@ static ASN1_TYPE *asn1_str2type(const char *str, int format, int utype) atmp->value.asn1_string->data = rdata; atmp->value.asn1_string->length = rdlen; atmp->value.asn1_string->type = utype; - } else if (format == ASN1_GEN_FORMAT_ASCII) { - if (!ASN1_STRING_set(atmp->value.asn1_string, str, -1)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto bad_str; - } - } else if ((format == ASN1_GEN_FORMAT_BITLIST) + } else if (format == ASN1_GEN_FORMAT_ASCII) + ASN1_STRING_set(atmp->value.asn1_string, str, -1); + else if ((format == ASN1_GEN_FORMAT_BITLIST) && (utype == V_ASN1_BIT_STRING)) { if (!CONF_parse_list (str, ',', 1, bitstr_cb, atmp->value.bit_string)) { @@ -718,8 +714,11 @@ static ASN1_TYPE *asn1_str2type(const char *str, int format, int utype) goto bad_form; } - if ((utype == V_ASN1_BIT_STRING) && no_unused) - ossl_asn1_string_set_bits_left(atmp->value.asn1_string, 0); + if ((utype == V_ASN1_BIT_STRING) && no_unused) { + atmp->value.asn1_string->flags + &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + atmp->value.asn1_string->flags |= ASN1_STRING_FLAG_BITS_LEFT; + } break; @@ -754,7 +753,7 @@ static int bitstr_cb(const char *elem, int len, void *bitstr) return 0; } if (!ASN1_BIT_STRING_set_bit(bitstr, bitnum, 1)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } return 1; @@ -766,7 +765,7 @@ static int mask_cb(const char *elem, int len, void *arg) int tag; if (elem == NULL) return 0; - if (len == 3 && HAS_PREFIX(elem, "DIR")) { + if ((len == 3) && (strncmp(elem, "DIR", 3) == 0)) { *pmask |= B_ASN1_DIRECTORYSTRING; return 1; } diff --git a/openssl/src/crypto/asn1/asn1_item_list.h b/openssl/src/crypto/asn1/asn1_item_list.h index 72299a7b6..1c29b5744 100644 --- a/openssl/src/crypto/asn1/asn1_item_list.h +++ b/openssl/src/crypto/asn1/asn1_item_list.h @@ -52,7 +52,6 @@ static ASN1_ITEM_EXP *asn1_item_list[] = { ASN1_ITEM_ref(CERTIFICATEPOLICIES), #ifndef OPENSSL_NO_CMS ASN1_ITEM_ref(CMS_ContentInfo), - ASN1_ITEM_ref(CMS_EnvelopedData), ASN1_ITEM_ref(CMS_ReceiptRequest), #endif ASN1_ITEM_ref(CRL_DIST_POINTS), @@ -148,7 +147,6 @@ static ASN1_ITEM_EXP *asn1_item_list[] = { #endif ASN1_ITEM_ref(SXNETID), ASN1_ITEM_ref(SXNET), - ASN1_ITEM_ref(ISSUER_SIGN_TOOL), ASN1_ITEM_ref(USERNOTICE), ASN1_ITEM_ref(X509_ALGORS), ASN1_ITEM_ref(X509_ALGOR), diff --git a/openssl/src/crypto/asn1/asn1_lib.c b/openssl/src/crypto/asn1/asn1_lib.c index e3a8480ee..5359cbc11 100644 --- a/openssl/src/crypto/asn1/asn1_lib.c +++ b/openssl/src/crypto/asn1/asn1_lib.c @@ -248,12 +248,6 @@ int ASN1_object_size(int constructed, int length, int tag) return ret + length; } -void ossl_asn1_string_set_bits_left(ASN1_STRING *str, unsigned int num) -{ - str->flags &= ~0x07; - str->flags |= ASN1_STRING_FLAG_BITS_LEFT | (num & 0x07); -} - int ASN1_STRING_copy(ASN1_STRING *dst, const ASN1_STRING *str) { if (str == NULL) @@ -314,6 +308,7 @@ int ASN1_STRING_set(ASN1_STRING *str, const void *_data, int len_in) str->data = OPENSSL_realloc(c, len + 1); #endif if (str->data == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); str->data = c; return 0; } @@ -353,8 +348,10 @@ ASN1_STRING *ASN1_STRING_type_new(int type) ASN1_STRING *ret; ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; + } ret->type = type; return ret; } diff --git a/openssl/src/crypto/asn1/asn1_local.h b/openssl/src/crypto/asn1/asn1_local.h index 10e9fcb7d..f73bd8fc6 100644 --- a/openssl/src/crypto/asn1/asn1_local.h +++ b/openssl/src/crypto/asn1/asn1_local.h @@ -9,8 +9,6 @@ /* Internal ASN1 structures and functions: not for application use */ -#include "crypto/asn1.h" - typedef const ASN1_VALUE const_ASN1_VALUE; SKM_DEFINE_STACK_OF(const_ASN1_VALUE, const ASN1_VALUE, ASN1_VALUE) diff --git a/openssl/src/crypto/asn1/asn1_parse.c b/openssl/src/crypto/asn1/asn1_parse.c index 6a4618d25..04d7ef66c 100644 --- a/openssl/src/crypto/asn1/asn1_parse.c +++ b/openssl/src/crypto/asn1/asn1_parse.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -50,7 +50,7 @@ static int asn1_print_info(BIO *bp, long offset, int depth, int hl, long len, pop_f_prefix = 1; } saved_indent = BIO_get_indent(bp); - if (BIO_set_prefix(bp, str) <= 0 || BIO_set_indent(bp, indent) <= 0) + if (BIO_set_prefix(bp, str) <= 0 || BIO_set_indent(bp, indent) < 0) goto err; } diff --git a/openssl/src/crypto/asn1/asn_mime.c b/openssl/src/crypto/asn1/asn_mime.c index 3a7386f16..b4cff5da3 100644 --- a/openssl/src/crypto/asn1/asn_mime.c +++ b/openssl/src/crypto/asn1/asn_mime.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -69,20 +69,15 @@ static void mime_hdr_free(MIME_HEADER *hdr); int i2d_ASN1_bio_stream(BIO *out, ASN1_VALUE *val, BIO *in, int flags, const ASN1_ITEM *it) { - int rv = 1; - /* If streaming create stream BIO and copy all content through it */ if (flags & SMIME_STREAM) { BIO *bio, *tbio; bio = BIO_new_NDEF(out, val, it); if (!bio) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } - if (!SMIME_crlf_copy(in, bio, flags)) { - rv = 0; - } - + SMIME_crlf_copy(in, bio, flags); (void)BIO_flush(bio); /* Free up successive BIOs until we hit the old output BIO */ do { @@ -97,7 +92,7 @@ int i2d_ASN1_bio_stream(BIO *out, ASN1_VALUE *val, BIO *in, int flags, */ else ASN1_item_i2d_bio(it, out, val); - return rv; + return 1; } /* Base 64 read and write of ASN1 structure */ @@ -109,7 +104,7 @@ static int B64_write_ASN1(BIO *out, ASN1_VALUE *val, BIO *in, int flags, int r; b64 = BIO_new(BIO_f_base64()); if (b64 == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } /* @@ -142,7 +137,7 @@ static ASN1_VALUE *b64_read_asn1(BIO *bio, const ASN1_ITEM *it, ASN1_VALUE **x, ASN1_VALUE *val; if ((b64 = BIO_new(BIO_f_base64())) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } bio = BIO_push(b64, bio); @@ -202,18 +197,6 @@ static int asn1_write_micalg(BIO *out, STACK_OF(X509_ALGOR) *mdalgs) BIO_puts(out, "sha-512"); break; - case NID_id_GostR3411_94: - BIO_puts(out, "gostr3411-94"); - goto err; - - case NID_id_GostR3411_2012_256: - BIO_puts(out, "gostr3411-2012-256"); - goto err; - - case NID_id_GostR3411_2012_512: - BIO_puts(out, "gostr3411-2012-512"); - goto err; - default: if (have_unknown) { write_comma = 0; @@ -351,7 +334,8 @@ static int asn1_output_data(BIO *out, BIO *data, ASN1_VALUE *val, int flags, * set up to finalise when it is written through. */ if (!(flags & SMIME_DETACHED) || (flags & PKCS7_REUSE_DIGEST)) { - return SMIME_crlf_copy(data, out, flags); + SMIME_crlf_copy(data, out, flags); + return 1; } if (!aux || !aux->asn1_cb) { @@ -369,8 +353,7 @@ static int asn1_output_data(BIO *out, BIO *data, ASN1_VALUE *val, int flags, return 0; /* Copy data across, passing through filter BIOs for processing */ - if (!SMIME_crlf_copy(data, sarg.ndef_bio, flags)) - rv = 0; + SMIME_crlf_copy(data, sarg.ndef_bio, flags); /* Finalize structure */ if (aux->asn1_cb(ASN1_OP_DETACHED_POST, &val, it, &sarg) <= 0) @@ -515,22 +498,13 @@ int SMIME_crlf_copy(BIO *in, BIO *out, int flags) char eol; int len; char linebuf[MAX_SMLEN]; - int ret; - - if (in == NULL || out == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - /* * Buffer output so we don't write one line at a time. This is useful * when streaming as we don't end up with one OCTET STRING per line. */ bf = BIO_new(BIO_f_buffer()); - if (bf == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BIO_LIB); + if (bf == NULL) return 0; - } out = BIO_push(bf, out); if (flags & SMIME_BINARY) { while ((len = BIO_read(in, linebuf, MAX_SMLEN)) > 0) @@ -559,12 +533,9 @@ int SMIME_crlf_copy(BIO *in, BIO *out, int flags) } } } - ret = BIO_flush(out); + (void)BIO_flush(out); BIO_pop(out); BIO_free(bf); - if (ret <= 0) - return 0; - return 1; } @@ -685,7 +656,7 @@ static STACK_OF(MIME_HEADER) *mime_parse_hdr(BIO *bio) char linebuf[MAX_SMLEN]; MIME_HEADER *mhdr = NULL, *new_hdr = NULL; STACK_OF(MIME_HEADER) *headers; - int i, len, state, save_state = 0; + int len, state, save_state = 0; headers = sk_MIME_HEADER_new(mime_hdr_cmp); if (headers == NULL) @@ -791,12 +762,6 @@ static STACK_OF(MIME_HEADER) *mime_parse_hdr(BIO *bio) break; /* Blank line means end of headers */ } - /* Sort the headers and their params for faster searching */ - sk_MIME_HEADER_sort(headers); - for (i = 0; i < sk_MIME_HEADER_num(headers); i++) - if ((mhdr = sk_MIME_HEADER_value(headers, i)) != NULL - && mhdr->params != NULL) - sk_MIME_PARAM_sort(mhdr->params); return headers; err: @@ -995,8 +960,13 @@ static int mime_bound_check(char *line, int linelen, const char *bound, int blen if (blen + 2 > linelen) return 0; /* Check for part boundary */ - if ((CHECK_AND_SKIP_PREFIX(line, "--")) && strncmp(line, bound, blen) == 0) - return HAS_PREFIX(line + blen, "--") ? 2 : 1; + if ((strncmp(line, "--", 2) == 0) + && strncmp(line + 2, bound, blen) == 0) { + if (strncmp(line + blen + 2, "--", 2) == 0) + return 2; + else + return 1; + } return 0; } diff --git a/openssl/src/crypto/asn1/asn_moid.c b/openssl/src/crypto/asn1/asn_moid.c index 1e183f4f1..526219c1a 100644 --- a/openssl/src/crypto/asn1/asn_moid.c +++ b/openssl/src/crypto/asn1/asn_moid.c @@ -67,10 +67,6 @@ static int do_create(const char *value, const char *name) if (p == NULL) { ln = name; ostr = value; - } else if (p == value) { - /* we started with a leading comma */ - ln = name; - ostr = p + 1; } else { ln = value; ostr = p + 1; @@ -87,8 +83,10 @@ static int do_create(const char *value, const char *name) p--; } p++; - if ((lntmp = OPENSSL_malloc((p - ln) + 1)) == NULL) + if ((lntmp = OPENSSL_malloc((p - ln) + 1)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(lntmp, ln, p - ln); lntmp[p - ln] = '\0'; ln = lntmp; diff --git a/openssl/src/crypto/asn1/asn_mstbl.c b/openssl/src/crypto/asn1/asn_mstbl.c index b93095765..3543cd225 100644 --- a/openssl/src/crypto/asn1/asn_mstbl.c +++ b/openssl/src/crypto/asn1/asn_mstbl.c @@ -1,5 +1,5 @@ /* - * Copyright 2012-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2012-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -72,8 +72,6 @@ static int do_tcreate(const char *value, const char *name) goto err; for (i = 0; i < sk_CONF_VALUE_num(lst); i++) { cnf = sk_CONF_VALUE_value(lst, i); - if (cnf->value == NULL) - goto err; if (strcmp(cnf->name, "min") == 0) { tbl_min = strtoul(cnf->value, &eptr, 0); if (*eptr) @@ -100,9 +98,7 @@ static int do_tcreate(const char *value, const char *name) if (rv == 0) { if (cnf) ERR_raise_data(ERR_LIB_ASN1, ASN1_R_INVALID_STRING_TABLE_VALUE, - "field=%s, value=%s", cnf->name, - cnf->value != NULL ? cnf->value - : value); + "field=%s, value=%s", cnf->name, cnf->value); else ERR_raise_data(ERR_LIB_ASN1, ASN1_R_INVALID_STRING_TABLE_VALUE, "name=%s, value=%s", name, value); @@ -110,7 +106,7 @@ static int do_tcreate(const char *value, const char *name) rv = ASN1_STRING_TABLE_add(nid, tbl_min, tbl_max, tbl_mask, tbl_flags); if (!rv) - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); } sk_CONF_VALUE_pop_free(lst, X509V3_conf_free); return rv; diff --git a/openssl/src/crypto/asn1/asn_pack.c b/openssl/src/crypto/asn1/asn_pack.c index 54f4ae3a6..292e6d817 100644 --- a/openssl/src/crypto/asn1/asn_pack.c +++ b/openssl/src/crypto/asn1/asn_pack.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,23 +17,24 @@ ASN1_STRING *ASN1_item_pack(void *obj, const ASN1_ITEM *it, ASN1_STRING **oct) { ASN1_STRING *octmp; - if (oct == NULL || *oct == NULL) { + if (oct == NULL || *oct == NULL) { if ((octmp = ASN1_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; } } else { octmp = *oct; } - ASN1_STRING_set0(octmp, NULL, 0); + OPENSSL_free(octmp->data); + octmp->data = NULL; - if ((octmp->length = ASN1_item_i2d(obj, &octmp->data, it)) <= 0) { + if ((octmp->length = ASN1_item_i2d(obj, &octmp->data, it)) == 0) { ERR_raise(ERR_LIB_ASN1, ASN1_R_ENCODE_ERROR); goto err; } if (octmp->data == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } @@ -59,16 +60,3 @@ void *ASN1_item_unpack(const ASN1_STRING *oct, const ASN1_ITEM *it) ERR_raise(ERR_LIB_ASN1, ASN1_R_DECODE_ERROR); return ret; } - -void *ASN1_item_unpack_ex(const ASN1_STRING *oct, const ASN1_ITEM *it, - OSSL_LIB_CTX *libctx, const char *propq) -{ - const unsigned char *p; - void *ret; - - p = oct->data; - if ((ret = ASN1_item_d2i_ex(NULL, &p, oct->length, it,\ - libctx, propq)) == NULL) - ERR_raise(ERR_LIB_ASN1, ASN1_R_DECODE_ERROR); - return ret; -} diff --git a/openssl/src/crypto/asn1/bio_asn1.c b/openssl/src/crypto/asn1/bio_asn1.c index f14994250..f792c0880 100644 --- a/openssl/src/crypto/asn1/bio_asn1.c +++ b/openssl/src/crypto/asn1/bio_asn1.c @@ -100,8 +100,10 @@ static int asn1_bio_new(BIO *b) { BIO_ASN1_BUF_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } if (!asn1_bio_init(ctx, DEFAULT_ASN1_BUF_SIZE)) { OPENSSL_free(ctx); return 0; @@ -114,12 +116,10 @@ static int asn1_bio_new(BIO *b) static int asn1_bio_init(BIO_ASN1_BUF_CTX *ctx, int size) { - if (size <= 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_PASSED_INVALID_ARGUMENT); + if (size <= 0 || (ctx->buf = OPENSSL_malloc(size)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } - if ((ctx->buf = OPENSSL_malloc(size)) == NULL) - return 0; ctx->bufsize = size; ctx->asn1_class = V_ASN1_UNIVERSAL; ctx->asn1_tag = V_ASN1_OCTET_STRING; @@ -172,7 +172,7 @@ static int asn1_bio_write(BIO *b, const char *in, int inl) case ASN1_STATE_START: if (!asn1_bio_setup_ex(b, ctx, ctx->prefix, ASN1_STATE_PRE_COPY, ASN1_STATE_HEADER)) - return -1; + return 0; break; /* Copy any pre data first */ @@ -189,7 +189,7 @@ static int asn1_bio_write(BIO *b, const char *in, int inl) case ASN1_STATE_HEADER: ctx->buflen = ASN1_object_size(0, inl, ctx->asn1_tag) - inl; if (!ossl_assert(ctx->buflen <= ctx->bufsize)) - return -1; + return 0; p = ctx->buf; ASN1_put_object(&p, 0, inl, ctx->asn1_tag, ctx->asn1_class); ctx->copylen = inl; diff --git a/openssl/src/crypto/asn1/bio_ndef.c b/openssl/src/crypto/asn1/bio_ndef.c index 279609e60..b9df3a7a4 100644 --- a/openssl/src/crypto/asn1/bio_ndef.c +++ b/openssl/src/crypto/asn1/bio_ndef.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -140,8 +140,10 @@ static int ndef_prefix(BIO *b, unsigned char **pbuf, int *plen, void *parg) derlen = ASN1_item_ndef_i2d(ndef_aux->val, NULL, ndef_aux->it); if (derlen < 0) return 0; - if ((p = OPENSSL_malloc(derlen)) == NULL) + if ((p = OPENSSL_malloc(derlen)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } ndef_aux->derbuf = p; *pbuf = p; @@ -213,8 +215,10 @@ static int ndef_suffix(BIO *b, unsigned char **pbuf, int *plen, void *parg) derlen = ASN1_item_ndef_i2d(ndef_aux->val, NULL, ndef_aux->it); if (derlen < 0) return 0; - if ((p = OPENSSL_malloc(derlen)) == NULL) + if ((p = OPENSSL_malloc(derlen)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } ndef_aux->derbuf = p; *pbuf = p; diff --git a/openssl/src/crypto/asn1/charmap.h b/openssl/src/crypto/asn1/charmap.h index ac1eb076c..95928ca66 100644 --- a/openssl/src/crypto/asn1/charmap.h +++ b/openssl/src/crypto/asn1/charmap.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by crypto/asn1/charmap.pl * - * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/asn1/d2i_pr.c b/openssl/src/crypto/asn1/d2i_pr.c index 44e685c49..720b7fd6c 100644 --- a/openssl/src/crypto/asn1/d2i_pr.c +++ b/openssl/src/crypto/asn1/d2i_pr.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,7 +22,6 @@ #include "crypto/asn1.h" #include "crypto/evp.h" #include "internal/asn1.h" -#include "internal/sizes.h" static EVP_PKEY * d2i_PrivateKey_decoder(int keytype, EVP_PKEY **a, const unsigned char **pp, @@ -33,12 +32,8 @@ d2i_PrivateKey_decoder(int keytype, EVP_PKEY **a, const unsigned char **pp, EVP_PKEY *pkey = NULL, *bak_a = NULL; EVP_PKEY **ppkey = &pkey; const char *key_name = NULL; - char keytypebuf[OSSL_MAX_NAME_SIZE]; - int ret; - const unsigned char *p = *pp; - const char *structure; - PKCS8_PRIV_KEY_INFO *p8info; - const ASN1_OBJECT *algoid; + const char *input_structures[] = { "type-specific", "PrivateKeyInfo", NULL }; + int i, ret; if (keytype != EVP_PKEY_NONE) { key_name = evp_pkey_type2name(keytype); @@ -46,42 +41,34 @@ d2i_PrivateKey_decoder(int keytype, EVP_PKEY **a, const unsigned char **pp, return NULL; } - /* This is just a probe. It might fail, so we ignore errors */ - ERR_set_mark(); - p8info = d2i_PKCS8_PRIV_KEY_INFO(NULL, pp, len); - ERR_pop_to_mark(); - if (p8info != NULL) { - if (key_name == NULL - && PKCS8_pkey_get0(&algoid, NULL, NULL, NULL, p8info) - && OBJ_obj2txt(keytypebuf, sizeof(keytypebuf), algoid, 0)) - key_name = keytypebuf; - structure = "PrivateKeyInfo"; - PKCS8_PRIV_KEY_INFO_free(p8info); - } else { - structure = "type-specific"; - } - *pp = p; + for (i = 0; i < (int)OSSL_NELEM(input_structures); ++i) { + const unsigned char *p = *pp; - if (a != NULL && (bak_a = *a) != NULL) - ppkey = a; - dctx = OSSL_DECODER_CTX_new_for_pkey(ppkey, "DER", structure, key_name, - EVP_PKEY_KEYPAIR, libctx, propq); - if (a != NULL) - *a = bak_a; - if (dctx == NULL) - goto err; - - ret = OSSL_DECODER_from_data(dctx, pp, &len); - OSSL_DECODER_CTX_free(dctx); - if (ret - && *ppkey != NULL - && evp_keymgmt_util_has(*ppkey, OSSL_KEYMGMT_SELECT_PRIVATE_KEY)) { + if (a != NULL && (bak_a = *a) != NULL) + ppkey = a; + dctx = OSSL_DECODER_CTX_new_for_pkey(ppkey, "DER", + input_structures[i], key_name, + EVP_PKEY_KEYPAIR, libctx, propq); if (a != NULL) - *a = *ppkey; - return *ppkey; + *a = bak_a; + if (dctx == NULL) + continue; + + ret = OSSL_DECODER_from_data(dctx, pp, &len); + OSSL_DECODER_CTX_free(dctx); + if (ret) { + if (*ppkey != NULL + && evp_keymgmt_util_has(*ppkey, OSSL_KEYMGMT_SELECT_PRIVATE_KEY)) { + if (a != NULL) + *a = *ppkey; + return *ppkey; + } + *pp = p; + goto err; + } } - - err: + /* Fall through to error if all decodes failed */ +err: if (ppkey != a) EVP_PKEY_free(*ppkey); return NULL; diff --git a/openssl/src/crypto/asn1/f_int.c b/openssl/src/crypto/asn1/f_int.c index 20192b577..d41e0069a 100644 --- a/openssl/src/crypto/asn1/f_int.c +++ b/openssl/src/crypto/asn1/f_int.c @@ -108,6 +108,7 @@ int a2i_ASN1_INTEGER(BIO *bp, ASN1_INTEGER *bs, char *buf, int size) if (num + i > slen) { sp = OPENSSL_clear_realloc(s, slen, num + i * 2); if (sp == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); OPENSSL_free(s); return 0; } diff --git a/openssl/src/crypto/asn1/f_string.c b/openssl/src/crypto/asn1/f_string.c index 1da442a45..4b65110d9 100644 --- a/openssl/src/crypto/asn1/f_string.c +++ b/openssl/src/crypto/asn1/f_string.c @@ -99,6 +99,7 @@ int a2i_ASN1_STRING(BIO *bp, ASN1_STRING *bs, char *buf, int size) if (num + i > slen) { sp = OPENSSL_realloc(s, (unsigned int)num + i * 2); if (sp == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); OPENSSL_free(s); return 0; } diff --git a/openssl/src/crypto/asn1/i2d_evp.c b/openssl/src/crypto/asn1/i2d_evp.c index 106ea1527..b9cef0d94 100644 --- a/openssl/src/crypto/asn1/i2d_evp.c +++ b/openssl/src/crypto/asn1/i2d_evp.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -34,6 +34,7 @@ static int i2d_provided(const EVP_PKEY *a, int selection, const struct type_and_structure_st *output_info, unsigned char **pp) { + OSSL_ENCODER_CTX *ctx = NULL; int ret; for (ret = -1; @@ -48,7 +49,6 @@ static int i2d_provided(const EVP_PKEY *a, int selection, */ size_t len = INT_MAX; int pp_was_NULL = (pp == NULL || *pp == NULL); - OSSL_ENCODER_CTX *ctx; ctx = OSSL_ENCODER_CTX_new_for_pkey(a, selection, output_info->output_type, @@ -63,6 +63,7 @@ static int i2d_provided(const EVP_PKEY *a, int selection, ret = INT_MAX - (int)len; } OSSL_ENCODER_CTX_free(ctx); + ctx = NULL; } if (ret == -1) @@ -138,6 +139,9 @@ int i2d_PublicKey(const EVP_PKEY *a, unsigned char **pp) return i2d_DSAPublicKey(EVP_PKEY_get0_DSA(a), pp); #endif #ifndef OPENSSL_NO_EC +# ifndef OPENSSL_NO_SM2 + case EVP_PKEY_SM2: +# endif case EVP_PKEY_EC: return i2o_ECPublicKey(EVP_PKEY_get0_EC_KEY(a), pp); #endif diff --git a/openssl/src/crypto/asn1/local.h b/openssl/src/crypto/asn1/local.h new file mode 100644 index 000000000..9bae8106b --- /dev/null +++ b/openssl/src/crypto/asn1/local.h @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +DECLARE_ASN1_FUNCTIONS(NETSCAPE_ENCRYPTED_PKEY) +DECLARE_ASN1_ENCODE_FUNCTIONS_name(NETSCAPE_ENCRYPTED_PKEY, NETSCAPE_ENCRYPTED_PKEY) + +DECLARE_ASN1_FUNCTIONS(NETSCAPE_PKEY) +DECLARE_ASN1_ENCODE_FUNCTIONS_name(NETSCAPE_PKEY, NETSCAPE_PKEY) diff --git a/openssl/src/crypto/asn1/p5_pbe.c b/openssl/src/crypto/asn1/p5_pbe.c index a90c200d4..9bc8aaa7a 100644 --- a/openssl/src/crypto/asn1/p5_pbe.c +++ b/openssl/src/crypto/asn1/p5_pbe.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,6 @@ #include #include #include -#include "crypto/evp.h" /* PKCS#5 password based encryption structure */ @@ -35,24 +34,25 @@ int PKCS5_pbe_set0_algor_ex(X509_ALGOR *algor, int alg, int iter, pbe = PBEPARAM_new(); if (pbe == NULL) { - /* ERR_R_ASN1_LIB, because PBEPARAM_new() is defined in crypto/asn1 */ - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } if (iter <= 0) iter = PKCS5_DEFAULT_ITER; if (!ASN1_INTEGER_set(pbe->iter, iter)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } if (!saltlen) - saltlen = PKCS5_DEFAULT_PBE1_SALT_LEN; + saltlen = PKCS5_SALT_LEN; if (saltlen < 0) goto err; sstr = OPENSSL_malloc(saltlen); - if (sstr == NULL) + if (sstr == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; + } if (salt) memcpy(sstr, salt, saltlen); else if (RAND_bytes_ex(ctx, sstr, saltlen, 0) <= 0) @@ -62,7 +62,7 @@ int PKCS5_pbe_set0_algor_ex(X509_ALGOR *algor, int alg, int iter, sstr = NULL; if (!ASN1_item_pack(pbe, ASN1_ITEM_rptr(PBEPARAM), &pbe_str)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } @@ -94,7 +94,7 @@ X509_ALGOR *PKCS5_pbe_set_ex(int alg, int iter, X509_ALGOR *ret; ret = X509_ALGOR_new(); if (ret == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/asn1/p5_pbev2.c b/openssl/src/crypto/asn1/p5_pbev2.c index 8575d05bf..169c4b39f 100644 --- a/openssl/src/crypto/asn1/p5_pbev2.c +++ b/openssl/src/crypto/asn1/p5_pbev2.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,8 +9,6 @@ #include #include "internal/cryptlib.h" -#include "crypto/asn1.h" -#include "crypto/evp.h" #include #include #include @@ -58,18 +56,14 @@ X509_ALGOR *PKCS5_pbe2_set_iv_ex(const EVP_CIPHER *cipher, int iter, goto err; } - if ((pbe2 = PBE2PARAM_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if ((pbe2 = PBE2PARAM_new()) == NULL) + goto merr; /* Setup the AlgorithmIdentifier for the encryption scheme */ scheme = pbe2->encryption; scheme->algorithm = OBJ_nid2obj(alg_nid); - if ((scheme->parameter = ASN1_TYPE_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if ((scheme->parameter = ASN1_TYPE_new()) == NULL) + goto merr; /* Create random IV */ ivlen = EVP_CIPHER_get_iv_length(cipher); @@ -81,10 +75,8 @@ X509_ALGOR *PKCS5_pbe2_set_iv_ex(const EVP_CIPHER *cipher, int iter, } ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); - goto err; - } + if (ctx == NULL) + goto merr; /* Dummy cipherinit to just setup the IV, and PRF */ if (!EVP_CipherInit_ex(ctx, cipher, NULL, NULL, iv, 0)) @@ -106,12 +98,7 @@ X509_ALGOR *PKCS5_pbe2_set_iv_ex(const EVP_CIPHER *cipher, int iter, EVP_CIPHER_CTX_free(ctx); ctx = NULL; - /* If its RC2 then we'd better setup the key length */ - - if (alg_nid == NID_rc2_cbc) - keylen = EVP_CIPHER_get_key_length(cipher); - else - keylen = -1; + keylen = -1; /* Setup keyfunc */ @@ -120,33 +107,30 @@ X509_ALGOR *PKCS5_pbe2_set_iv_ex(const EVP_CIPHER *cipher, int iter, pbe2->keyfunc = PKCS5_pbkdf2_set_ex(iter, salt, saltlen, prf_nid, keylen, libctx); - if (pbe2->keyfunc == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (pbe2->keyfunc == NULL) + goto merr; /* Now set up top level AlgorithmIdentifier */ - if ((ret = X509_ALGOR_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_X509_LIB); - goto err; - } + if ((ret = X509_ALGOR_new()) == NULL) + goto merr; ret->algorithm = OBJ_nid2obj(NID_pbes2); /* Encode PBE2PARAM into parameter */ if (!ASN1_TYPE_pack_sequence(ASN1_ITEM_rptr(PBE2PARAM), pbe2, - &ret->parameter)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + &ret->parameter)) + goto merr; PBE2PARAM_free(pbe2); pbe2 = NULL; return ret; + merr: + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + err: EVP_CIPHER_CTX_free(ctx); PBE2PARAM_free(pbe2); @@ -180,89 +164,70 @@ X509_ALGOR *PKCS5_pbkdf2_set_ex(int iter, unsigned char *salt, int saltlen, PBKDF2PARAM *kdf = NULL; ASN1_OCTET_STRING *osalt = NULL; - if ((kdf = PBKDF2PARAM_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } - if ((osalt = ASN1_OCTET_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if ((kdf = PBKDF2PARAM_new()) == NULL) + goto merr; + if ((osalt = ASN1_OCTET_STRING_new()) == NULL) + goto merr; kdf->salt->value.octet_string = osalt; kdf->salt->type = V_ASN1_OCTET_STRING; - if (saltlen < 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } + if (saltlen < 0) + goto merr; if (saltlen == 0) - saltlen = PKCS5_DEFAULT_PBE2_SALT_LEN; + saltlen = PKCS5_SALT_LEN; if ((osalt->data = OPENSSL_malloc(saltlen)) == NULL) - goto err; - + goto merr; osalt->length = saltlen; - if (salt) { + if (salt) memcpy(osalt->data, salt, saltlen); - } else if (RAND_bytes_ex(libctx, osalt->data, saltlen, 0) <= 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_RAND_LIB); - goto err; - } + else if (RAND_bytes_ex(libctx, osalt->data, saltlen, 0) <= 0) + goto merr; if (iter <= 0) iter = PKCS5_DEFAULT_ITER; - if (!ASN1_INTEGER_set(kdf->iter, iter)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (!ASN1_INTEGER_set(kdf->iter, iter)) + goto merr; /* If have a key len set it up */ if (keylen > 0) { - if ((kdf->keylength = ASN1_INTEGER_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } - if (!ASN1_INTEGER_set(kdf->keylength, keylen)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if ((kdf->keylength = ASN1_INTEGER_new()) == NULL) + goto merr; + if (!ASN1_INTEGER_set(kdf->keylength, keylen)) + goto merr; } /* prf can stay NULL if we are using hmacWithSHA1 */ if (prf_nid > 0 && prf_nid != NID_hmacWithSHA1) { - kdf->prf = ossl_X509_ALGOR_from_nid(prf_nid, V_ASN1_NULL, NULL); - if (kdf->prf == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_X509_LIB); - goto err; - } + kdf->prf = X509_ALGOR_new(); + if (kdf->prf == NULL) + goto merr; + X509_ALGOR_set0(kdf->prf, OBJ_nid2obj(prf_nid), V_ASN1_NULL, NULL); } /* Finally setup the keyfunc structure */ keyfunc = X509_ALGOR_new(); - if (keyfunc == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_X509_LIB); - goto err; - } + if (keyfunc == NULL) + goto merr; keyfunc->algorithm = OBJ_nid2obj(NID_id_pbkdf2); /* Encode PBKDF2PARAM into parameter of pbe2 */ if (!ASN1_TYPE_pack_sequence(ASN1_ITEM_rptr(PBKDF2PARAM), kdf, - &keyfunc->parameter)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + &keyfunc->parameter)) + goto merr; PBKDF2PARAM_free(kdf); return keyfunc; - err: + merr: + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); PBKDF2PARAM_free(kdf); X509_ALGOR_free(keyfunc); return NULL; diff --git a/openssl/src/crypto/asn1/p5_scrypt.c b/openssl/src/crypto/asn1/p5_scrypt.c index 4f3dcecd4..b76cf8058 100644 --- a/openssl/src/crypto/asn1/p5_scrypt.c +++ b/openssl/src/crypto/asn1/p5_scrypt.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -67,20 +67,16 @@ X509_ALGOR *PKCS5_pbe2_set_scrypt(const EVP_CIPHER *cipher, } pbe2 = PBE2PARAM_new(); - if (pbe2 == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (pbe2 == NULL) + goto merr; /* Setup the AlgorithmIdentifier for the encryption scheme */ scheme = pbe2->encryption; scheme->algorithm = OBJ_nid2obj(alg_nid); scheme->parameter = ASN1_TYPE_new(); - if (scheme->parameter == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (scheme->parameter == NULL) + goto merr; /* Create random IV */ if (EVP_CIPHER_get_iv_length(cipher)) { @@ -91,10 +87,8 @@ X509_ALGOR *PKCS5_pbe2_set_scrypt(const EVP_CIPHER *cipher, } ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); - goto err; - } + if (ctx == NULL) + goto merr; /* Dummy cipherinit to just setup the IV */ if (EVP_CipherInit_ex(ctx, cipher, NULL, NULL, iv, 0) == 0) @@ -106,45 +100,37 @@ X509_ALGOR *PKCS5_pbe2_set_scrypt(const EVP_CIPHER *cipher, EVP_CIPHER_CTX_free(ctx); ctx = NULL; - /* If its RC2 then we'd better setup the key length */ - - if (alg_nid == NID_rc2_cbc) - keylen = EVP_CIPHER_get_key_length(cipher); - /* Setup keyfunc */ X509_ALGOR_free(pbe2->keyfunc); pbe2->keyfunc = pkcs5_scrypt_set(salt, saltlen, keylen, N, r, p); - if (pbe2->keyfunc == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (pbe2->keyfunc == NULL) + goto merr; /* Now set up top level AlgorithmIdentifier */ ret = X509_ALGOR_new(); - if (ret == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (ret == NULL) + goto merr; ret->algorithm = OBJ_nid2obj(NID_pbes2); /* Encode PBE2PARAM into parameter */ if (ASN1_TYPE_pack_sequence(ASN1_ITEM_rptr(PBE2PARAM), pbe2, - &ret->parameter) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + &ret->parameter) == NULL) + goto merr; PBE2PARAM_free(pbe2); pbe2 = NULL; return ret; + merr: + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + err: PBE2PARAM_free(pbe2); X509_ALGOR_free(ret); @@ -160,73 +146,57 @@ static X509_ALGOR *pkcs5_scrypt_set(const unsigned char *salt, size_t saltlen, X509_ALGOR *keyfunc = NULL; SCRYPT_PARAMS *sparam = SCRYPT_PARAMS_new(); - if (sparam == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (sparam == NULL) + goto merr; if (!saltlen) - saltlen = PKCS5_DEFAULT_PBE2_SALT_LEN; + saltlen = PKCS5_SALT_LEN; /* This will either copy salt or grow the buffer */ - if (ASN1_STRING_set(sparam->salt, salt, saltlen) == 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (ASN1_STRING_set(sparam->salt, salt, saltlen) == 0) + goto merr; if (salt == NULL && RAND_bytes(sparam->salt->data, saltlen) <= 0) goto err; - if (ASN1_INTEGER_set_uint64(sparam->costParameter, N) == 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (ASN1_INTEGER_set_uint64(sparam->costParameter, N) == 0) + goto merr; - if (ASN1_INTEGER_set_uint64(sparam->blockSize, r) == 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (ASN1_INTEGER_set_uint64(sparam->blockSize, r) == 0) + goto merr; - if (ASN1_INTEGER_set_uint64(sparam->parallelizationParameter, p) == 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (ASN1_INTEGER_set_uint64(sparam->parallelizationParameter, p) == 0) + goto merr; /* If have a key len set it up */ if (keylen > 0) { sparam->keyLength = ASN1_INTEGER_new(); - if (sparam->keyLength == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } - if (ASN1_INTEGER_set_int64(sparam->keyLength, keylen) == 0) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (sparam->keyLength == NULL) + goto merr; + if (ASN1_INTEGER_set_int64(sparam->keyLength, keylen) == 0) + goto merr; } /* Finally setup the keyfunc structure */ keyfunc = X509_ALGOR_new(); - if (keyfunc == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + if (keyfunc == NULL) + goto merr; keyfunc->algorithm = OBJ_nid2obj(NID_id_scrypt); /* Encode SCRYPT_PARAMS into parameter of pbe2 */ if (ASN1_TYPE_pack_sequence(ASN1_ITEM_rptr(SCRYPT_PARAMS), sparam, - &keyfunc->parameter) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - goto err; - } + &keyfunc->parameter) == NULL) + goto merr; SCRYPT_PARAMS_free(sparam); return keyfunc; + merr: + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); err: SCRYPT_PARAMS_free(sparam); X509_ALGOR_free(keyfunc); diff --git a/openssl/src/crypto/asn1/standard_methods.h b/openssl/src/crypto/asn1/standard_methods.h index 6b73d9a77..0b0c7ef68 100644 --- a/openssl/src/crypto/asn1/standard_methods.h +++ b/openssl/src/crypto/asn1/standard_methods.h @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -32,9 +32,11 @@ static const EVP_PKEY_ASN1_METHOD *standard_methods[] = { #ifndef OPENSSL_NO_DH &ossl_dhx_asn1_meth, #endif -#ifndef OPENSSL_NO_ECX +#ifndef OPENSSL_NO_EC &ossl_ecx25519_asn1_meth, &ossl_ecx448_asn1_meth, +#endif +#ifndef OPENSSL_NO_EC &ossl_ed25519_asn1_meth, &ossl_ed448_asn1_meth, #endif diff --git a/openssl/src/crypto/asn1/tasn_dec.c b/openssl/src/crypto/asn1/tasn_dec.c index c4f9d6151..11198087a 100644 --- a/openssl/src/crypto/asn1/tasn_dec.c +++ b/openssl/src/crypto/asn1/tasn_dec.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -629,7 +629,7 @@ static int asn1_template_noexp_d2i(ASN1_VALUE **val, } if (*val == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } @@ -658,7 +658,7 @@ static int asn1_template_noexp_d2i(ASN1_VALUE **val, } len -= p - q; if (!sk_ASN1_VALUE_push((STACK_OF(ASN1_VALUE) *)*val, skfield)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); ASN1_item_free(skfield, ASN1_ITEM_ptr(tt->item)); goto err; } @@ -802,7 +802,7 @@ static int asn1_d2i_ex_primitive(ASN1_VALUE **pval, len = buf.length; /* Append a final null to string */ if (!BUF_MEM_grow_clean(&buf, len + 1)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } buf.data[len] = 0; @@ -921,19 +921,11 @@ static int asn1_ex_c2i(ASN1_VALUE **pval, const unsigned char *cont, int len, ERR_raise(ERR_LIB_ASN1, ASN1_R_UNIVERSALSTRING_IS_WRONG_LENGTH); goto err; } - if (utype == V_ASN1_GENERALIZEDTIME && (len < 15)) { - ERR_raise(ERR_LIB_ASN1, ASN1_R_GENERALIZEDTIME_IS_TOO_SHORT); - goto err; - } - if (utype == V_ASN1_UTCTIME && (len < 13)) { - ERR_raise(ERR_LIB_ASN1, ASN1_R_UTCTIME_IS_TOO_SHORT); - goto err; - } /* All based on ASN1_STRING and handled the same */ if (*pval == NULL) { stmp = ASN1_STRING_type_new(utype); if (stmp == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } *pval = (ASN1_VALUE *)stmp; @@ -943,11 +935,13 @@ static int asn1_ex_c2i(ASN1_VALUE **pval, const unsigned char *cont, int len, } /* If we've already allocated a buffer use it */ if (*free_cont) { - ASN1_STRING_set0(stmp, (unsigned char *)cont /* UGLY CAST! */, len); + OPENSSL_free(stmp->data); + stmp->data = (unsigned char *)cont; /* UGLY CAST! RL */ + stmp->length = len; *free_cont = 0; } else { if (!ASN1_STRING_set(stmp, cont, len)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); ASN1_STRING_free(stmp); *pval = NULL; goto err; @@ -1106,7 +1100,7 @@ static int collect_data(BUF_MEM *buf, const unsigned char **p, long plen) if (buf) { len = buf->length; if (!BUF_MEM_grow_clean(buf, len + plen)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } memcpy(buf->data + len, *p, plen); diff --git a/openssl/src/crypto/asn1/tasn_enc.c b/openssl/src/crypto/asn1/tasn_enc.c index dab5f9f27..3ea18b028 100644 --- a/openssl/src/crypto/asn1/tasn_enc.c +++ b/openssl/src/crypto/asn1/tasn_enc.c @@ -62,8 +62,10 @@ static int asn1_item_flags_i2d(const ASN1_VALUE *val, unsigned char **out, len = ASN1_item_ex_i2d(&val, NULL, it, -1, flags); if (len <= 0) return len; - if ((buf = OPENSSL_malloc(len)) == NULL) + if ((buf = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; + } p = buf; ASN1_item_ex_i2d(&val, &p, it, -1, flags); *out = buf; @@ -413,11 +415,15 @@ static int asn1_set_seq_out(STACK_OF(const_ASN1_VALUE) *sk, else { derlst = OPENSSL_malloc(sk_const_ASN1_VALUE_num(sk) * sizeof(*derlst)); - if (derlst == NULL) + if (derlst == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } tmpdat = OPENSSL_malloc(skcontlen); - if (tmpdat == NULL) + if (tmpdat == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; + } } } /* If not sorting just output each item */ diff --git a/openssl/src/crypto/asn1/tasn_new.c b/openssl/src/crypto/asn1/tasn_new.c index 00a5397a5..4b624bbdd 100644 --- a/openssl/src/crypto/asn1/tasn_new.c +++ b/openssl/src/crypto/asn1/tasn_new.c @@ -78,10 +78,10 @@ int asn1_item_embed_new(ASN1_VALUE **pval, const ASN1_ITEM *it, int embed, if (ef != NULL) { if (ef->asn1_ex_new_ex != NULL) { if (!ef->asn1_ex_new_ex(pval, it, libctx, propq)) - goto asn1err; + goto memerr; } else if (ef->asn1_ex_new != NULL) { if (!ef->asn1_ex_new(pval, it)) - goto asn1err; + goto memerr; } } break; @@ -89,14 +89,14 @@ int asn1_item_embed_new(ASN1_VALUE **pval, const ASN1_ITEM *it, int embed, case ASN1_ITYPE_PRIMITIVE: if (it->templates) { if (!asn1_template_new(pval, it->templates, libctx, propq)) - goto asn1err; + goto memerr; } else if (!asn1_primitive_new(pval, it, embed)) - goto asn1err; + goto memerr; break; case ASN1_ITYPE_MSTRING: if (!asn1_primitive_new(pval, it, embed)) - goto asn1err; + goto memerr; break; case ASN1_ITYPE_CHOICE: @@ -113,7 +113,7 @@ int asn1_item_embed_new(ASN1_VALUE **pval, const ASN1_ITEM *it, int embed, } else { *pval = OPENSSL_zalloc(it->size); if (*pval == NULL) - return 0; + goto memerr; } ossl_asn1_set_choice_selector(pval, -1, it); if (asn1_cb && !asn1_cb(ASN1_OP_NEW_POST, pval, it, NULL)) @@ -135,7 +135,7 @@ int asn1_item_embed_new(ASN1_VALUE **pval, const ASN1_ITEM *it, int embed, } else { *pval = OPENSSL_zalloc(it->size); if (*pval == NULL) - return 0; + goto memerr; } /* 0 : init. lock */ if (ossl_asn1_do_lock(pval, 0, it) < 0) { @@ -143,13 +143,13 @@ int asn1_item_embed_new(ASN1_VALUE **pval, const ASN1_ITEM *it, int embed, OPENSSL_free(*pval); *pval = NULL; } - goto asn1err; + goto memerr; } ossl_asn1_enc_init(pval, it); for (i = 0, tt = it->templates; i < it->tcount; tt++, i++) { pseqval = ossl_asn1_get_field_ptr(pval, tt); if (!asn1_template_new(pseqval, tt, libctx, propq)) - goto asn1err2; + goto memerr2; } if (asn1_cb && !asn1_cb(ASN1_OP_NEW_POST, pval, it, NULL)) goto auxerr2; @@ -157,10 +157,10 @@ int asn1_item_embed_new(ASN1_VALUE **pval, const ASN1_ITEM *it, int embed, } return 1; - asn1err2: + memerr2: ossl_asn1_item_embed_free(pval, it, embed); - asn1err: - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); + memerr: + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; auxerr2: @@ -230,7 +230,7 @@ static int asn1_template_new(ASN1_VALUE **pval, const ASN1_TEMPLATE *tt, STACK_OF(ASN1_VALUE) *skval; skval = sk_ASN1_VALUE_new_null(); if (!skval) { - ERR_raise(ERR_LIB_ASN1, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); ret = 0; goto done; } @@ -298,8 +298,10 @@ static int asn1_primitive_new(ASN1_VALUE **pval, const ASN1_ITEM *it, return 1; case V_ASN1_ANY: - if ((typ = OPENSSL_malloc(sizeof(*typ))) == NULL) + if ((typ = OPENSSL_malloc(sizeof(*typ))) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } typ->value.ptr = NULL; typ->type = -1; *pval = (ASN1_VALUE *)typ; diff --git a/openssl/src/crypto/asn1/tasn_prn.c b/openssl/src/crypto/asn1/tasn_prn.c index 73eadc5fd..7d8618e26 100644 --- a/openssl/src/crypto/asn1/tasn_prn.c +++ b/openssl/src/crypto/asn1/tasn_prn.c @@ -37,8 +37,10 @@ ASN1_PCTX *ASN1_PCTX_new(void) ASN1_PCTX *ret; ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; + } return ret; } diff --git a/openssl/src/crypto/asn1/tasn_scn.c b/openssl/src/crypto/asn1/tasn_scn.c index 7ada313b9..bde697ee9 100644 --- a/openssl/src/crypto/asn1/tasn_scn.c +++ b/openssl/src/crypto/asn1/tasn_scn.c @@ -26,8 +26,10 @@ ASN1_SCTX *ASN1_SCTX_new(int (*scan_cb) (ASN1_SCTX *ctx)) { ASN1_SCTX *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; + } ret->scan_cb = scan_cb; return ret; } diff --git a/openssl/src/crypto/asn1/tasn_utl.c b/openssl/src/crypto/asn1/tasn_utl.c index 67a9ccde6..e5f25d88d 100644 --- a/openssl/src/crypto/asn1/tasn_utl.c +++ b/openssl/src/crypto/asn1/tasn_utl.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -59,7 +59,7 @@ int ossl_asn1_set_choice_selector(ASN1_VALUE **pval, int value, /* * Do atomic reference counting. The value 'op' decides what to do. * If it is +1 then the count is incremented. - * If |op| is 0, count is initialised and set to 1. + * If |op| is 0, lock is initialised and count is set to 1. * If |op| is -1, count is decremented and the return value is the current * reference count or 0 if no reference count is active. * It returns -1 on initialisation error. @@ -68,8 +68,8 @@ int ossl_asn1_set_choice_selector(ASN1_VALUE **pval, int value, int ossl_asn1_do_lock(ASN1_VALUE **pval, int op, const ASN1_ITEM *it) { const ASN1_AUX *aux; + CRYPTO_REF_COUNT *lck; CRYPTO_RWLOCK **lock; - CRYPTO_REF_COUNT *refcnt; int ret = -1; if ((it->itype != ASN1_ITYPE_SEQUENCE) @@ -78,34 +78,30 @@ int ossl_asn1_do_lock(ASN1_VALUE **pval, int op, const ASN1_ITEM *it) aux = it->funcs; if (aux == NULL || (aux->flags & ASN1_AFLG_REFCOUNT) == 0) return 0; + lck = offset2ptr(*pval, aux->ref_offset); lock = offset2ptr(*pval, aux->ref_lock); - refcnt = offset2ptr(*pval, aux->ref_offset); switch (op) { case 0: - if (!CRYPTO_NEW_REF(refcnt, 1)) - return -1; + *lck = ret = 1; *lock = CRYPTO_THREAD_lock_new(); if (*lock == NULL) { - CRYPTO_FREE_REF(refcnt); - ERR_raise(ERR_LIB_ASN1, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } - ret = 1; break; case 1: - if (!CRYPTO_UP_REF(refcnt, &ret)) + if (!CRYPTO_UP_REF(lck, &ret, *lock)) return -1; break; case -1: - if (!CRYPTO_DOWN_REF(refcnt, &ret)) + if (!CRYPTO_DOWN_REF(lck, &ret, *lock)) return -1; /* failed */ REF_PRINT_EX(it->sname, ret, (void *)it); REF_ASSERT_ISNT(ret < 0); if (ret == 0) { CRYPTO_THREAD_lock_free(*lock); *lock = NULL; - CRYPTO_FREE_REF(refcnt); } break; } @@ -172,8 +168,10 @@ int ossl_asn1_enc_save(ASN1_VALUE **pval, const unsigned char *in, int inlen, OPENSSL_free(enc->enc); if (inlen <= 0) return 0; - if ((enc->enc = OPENSSL_malloc(inlen)) == NULL) + if ((enc->enc = OPENSSL_malloc(inlen)) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(enc->enc, in, inlen); enc->len = inlen; enc->modified = 0; diff --git a/openssl/src/crypto/asn1/tbl_standard.h b/openssl/src/crypto/asn1/tbl_standard.h index 3e8fe81ee..122ea5ee6 100644 --- a/openssl/src/crypto/asn1/tbl_standard.h +++ b/openssl/src/crypto/asn1/tbl_standard.h @@ -51,9 +51,6 @@ static const ASN1_STRING_TABLE tbl_standard[] = { {NID_rfc822Mailbox, 1, ub_rfc822_mailbox, B_ASN1_IA5STRING, STABLE_NO_MASK}, {NID_jurisdictionCountryName, 2, 2, B_ASN1_PRINTABLESTRING, STABLE_NO_MASK}, - {NID_INN, 1, 12, B_ASN1_NUMERICSTRING, STABLE_NO_MASK}, - {NID_OGRN, 1, 13, B_ASN1_NUMERICSTRING, STABLE_NO_MASK}, - {NID_SNILS, 1, 11, B_ASN1_NUMERICSTRING, STABLE_NO_MASK}, {NID_countryCode3c, 3, 3, B_ASN1_PRINTABLESTRING, STABLE_NO_MASK}, {NID_countryCode3n, 3, 3, B_ASN1_NUMERICSTRING, STABLE_NO_MASK}, {NID_dnsName, 0, -1, B_ASN1_UTF8STRING, STABLE_NO_MASK}, diff --git a/openssl/src/crypto/asn1/x_algor.c b/openssl/src/crypto/asn1/x_algor.c index db9dd06e4..c0a5f7680 100644 --- a/openssl/src/crypto/asn1/x_algor.c +++ b/openssl/src/crypto/asn1/x_algor.c @@ -1,5 +1,5 @@ /* - * Copyright 1998-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1998-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,14 +33,17 @@ int X509_ALGOR_set0(X509_ALGOR *alg, ASN1_OBJECT *aobj, int ptype, void *pval) if (alg == NULL) return 0; - if (ptype != V_ASN1_UNDEF && alg->parameter == NULL - && (alg->parameter = ASN1_TYPE_new()) == NULL) - return 0; + if (ptype != V_ASN1_UNDEF) { + if (alg->parameter == NULL) + alg->parameter = ASN1_TYPE_new(); + if (alg->parameter == NULL) + return 0; + } ASN1_OBJECT_free(alg->algorithm); alg->algorithm = aobj; - if (ptype == V_ASN1_EOC) + if (ptype == 0) return 1; if (ptype == V_ASN1_UNDEF) { ASN1_TYPE_free(alg->parameter); @@ -50,25 +53,6 @@ int X509_ALGOR_set0(X509_ALGOR *alg, ASN1_OBJECT *aobj, int ptype, void *pval) return 1; } -X509_ALGOR *ossl_X509_ALGOR_from_nid(int nid, int ptype, void *pval) -{ - ASN1_OBJECT *algo = OBJ_nid2obj(nid); - X509_ALGOR *alg = NULL; - - if (algo == NULL) - return NULL; - if ((alg = X509_ALGOR_new()) == NULL) - goto err; - if (X509_ALGOR_set0(alg, algo, ptype, pval)) - return alg; - alg->algorithm = NULL; /* precaution to prevent double free */ - - err: - X509_ALGOR_free(alg); - /* ASN1_OBJECT_free(algo) is not needed due to OBJ_nid2obj() */ - return NULL; -} - void X509_ALGOR_get0(const ASN1_OBJECT **paobj, int *pptype, const void **ppval, const X509_ALGOR *algor) { @@ -86,12 +70,18 @@ void X509_ALGOR_get0(const ASN1_OBJECT **paobj, int *pptype, } /* Set up an X509_ALGOR DigestAlgorithmIdentifier from an EVP_MD */ + void X509_ALGOR_set_md(X509_ALGOR *alg, const EVP_MD *md) { - int type = md->flags & EVP_MD_FLAG_DIGALGID_ABSENT ? V_ASN1_UNDEF - : V_ASN1_NULL; + int param_type; + + if (md->flags & EVP_MD_FLAG_DIGALGID_ABSENT) + param_type = V_ASN1_UNDEF; + else + param_type = V_ASN1_NULL; + + X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_MD_get_type(md)), param_type, NULL); - (void)X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_MD_get_type(md)), type, NULL); } int X509_ALGOR_cmp(const X509_ALGOR *a, const X509_ALGOR *b) @@ -141,15 +131,13 @@ int X509_ALGOR_copy(X509_ALGOR *dest, const X509_ALGOR *src) /* allocate and set algorithm ID from EVP_MD, default SHA1 */ int ossl_x509_algor_new_from_md(X509_ALGOR **palg, const EVP_MD *md) { - X509_ALGOR *alg; - /* Default is SHA1 so no need to create it - still success */ if (md == NULL || EVP_MD_is_a(md, "SHA1")) return 1; - if ((alg = X509_ALGOR_new()) == NULL) + *palg = X509_ALGOR_new(); + if (*palg == NULL) return 0; - X509_ALGOR_set_md(alg, md); - *palg = alg; + X509_ALGOR_set_md(*palg, md); return 1; } @@ -188,12 +176,15 @@ int ossl_x509_algor_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md) goto err; if (ASN1_item_pack(algtmp, ASN1_ITEM_rptr(X509_ALGOR), &stmp) == NULL) goto err; - *palg = ossl_X509_ALGOR_from_nid(NID_mgf1, V_ASN1_SEQUENCE, stmp); + *palg = X509_ALGOR_new(); if (*palg == NULL) goto err; + X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp); stmp = NULL; err: ASN1_STRING_free(stmp); X509_ALGOR_free(algtmp); - return *palg != NULL; + if (*palg != NULL) + return 1; + return 0; } diff --git a/openssl/src/crypto/asn1/x_delegated_credential.c b/openssl/src/crypto/asn1/x_delegated_credential.c new file mode 100644 index 000000000..1dd0cfd20 --- /dev/null +++ b/openssl/src/crypto/asn1/x_delegated_credential.c @@ -0,0 +1,256 @@ +#include +#include "internal/cryptlib.h" +#include +#include +#include +#include +#include "crypto/x509.h" +#include "../../ssl/ssl_local.h" +#include "internal/refcount.h" + +#define DC_MAX_LEN 65535 + +DELEGATED_CREDENTIAL *DC_new(void) +{ + return DC_new_ex(NULL, NULL); +} + +DELEGATED_CREDENTIAL *DC_new_ex(OSSL_LIB_CTX *libctx, const char *propq) +{ + DELEGATED_CREDENTIAL *dc; + + dc = OPENSSL_zalloc(sizeof(DELEGATED_CREDENTIAL)); + if (dc == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + return NULL; + } + + dc->references = 1; + dc->lock = CRYPTO_THREAD_lock_new(); + if (dc->lock == NULL) + goto err; + + dc->libctx = libctx; + if (propq != NULL) { + dc->propq = OPENSSL_strdup(propq); + if (dc->propq == NULL) + goto err; + } + + return dc; +err: + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + + DC_free(dc); + return NULL; +} + +void DC_free(DELEGATED_CREDENTIAL *dc) +{ + int i; + + if (dc == NULL) + return; + + CRYPTO_DOWN_REF(&dc->references, &i, dc->lock); + REF_PRINT_COUNT("DC", dc); + if (i > 0) + return; + REF_ASSERT_ISNT(i < 0); + CRYPTO_THREAD_lock_free(dc->lock); + + OPENSSL_free(dc->dc_publickey_raw); + OPENSSL_free(dc->dc_signature); + EVP_PKEY_free(dc->pkey); + OPENSSL_free(dc->raw_byte); + OPENSSL_free(dc->propq); + + OPENSSL_free(dc); +} + +int DC_up_ref(DELEGATED_CREDENTIAL *dc) +{ + int i; + + if (CRYPTO_UP_REF(&dc->references, &i, dc->lock) <= 0) + return 0; + + REF_PRINT_COUNT("DC", dc); + REF_ASSERT_ISNT(i < 2); + return ((i > 1) ? 1 : 0); +} + +DELEGATED_CREDENTIAL *DC_new_from_raw_byte(const unsigned char *byte, + size_t len) +{ + return DC_new_from_raw_byte_ex(byte, len, NULL, NULL); +} + +DELEGATED_CREDENTIAL *DC_new_from_raw_byte_ex(const unsigned char *byte, + size_t len, + OSSL_LIB_CTX *libctx, + const char *propq) +{ + unsigned long valid_time; + unsigned int expected_cert_verify_algorithm; + unsigned long dc_publickey_raw_len; + unsigned char *dc_publickey_raw = NULL; + unsigned int signature_sign_algorithm; + unsigned int dc_signature_len; + unsigned char *dc_signature = NULL; + PACKET pkt; + DELEGATED_CREDENTIAL *dc = NULL; + EVP_PKEY *pkey = NULL; + + dc = DC_new_ex(libctx, propq); + if (dc == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if(!DC_set1_raw_byte(dc, byte, len)) + goto err; + + if (!PACKET_buf_init(&pkt, dc->raw_byte, dc->raw_byte_len)) + goto err; + + if (PACKET_remaining(&pkt) <= 0) + goto err; + + if (!PACKET_get_net_4(&pkt, &valid_time) + || !PACKET_get_net_2(&pkt, &expected_cert_verify_algorithm) + || !PACKET_get_net_3(&pkt, &dc_publickey_raw_len)) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_ILLEGAL_FORMAT); + goto err; + } + dc->valid_time = valid_time; + dc->expected_cert_verify_algorithm = expected_cert_verify_algorithm; + dc->dc_publickey_raw_len = dc_publickey_raw_len; + + if (dc_publickey_raw_len > pkt.remaining) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_INVALID_VALUE); + goto err; + } + dc_publickey_raw = OPENSSL_malloc(dc_publickey_raw_len); + if (dc_publickey_raw == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + goto err; + } + + dc->dc_publickey_raw = dc_publickey_raw; + + if (!PACKET_copy_bytes(&pkt, dc_publickey_raw, dc_publickey_raw_len)) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_INVALID_VALUE); + goto err; + } + + pkey = d2i_PUBKEY_ex(NULL, (const unsigned char **)&dc_publickey_raw, + dc_publickey_raw_len, libctx, propq); + if (pkey == NULL) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_INVALID_VALUE); + goto err; + } + + /* DC public key MUST NOT use the rsaEncryption OID */ + if (EVP_PKEY_is_a(pkey, "RSA")) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_WRONG_PUBLIC_KEY_TYPE); + goto err; + } + + dc->pkey = pkey; + + if (!PACKET_get_net_2(&pkt, &signature_sign_algorithm) + || !PACKET_get_net_2(&pkt, &dc_signature_len)) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_ILLEGAL_FORMAT); + goto err; + } + dc->signature_sign_algorithm = signature_sign_algorithm; + + if (dc_signature_len > pkt.remaining) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_INVALID_VALUE); + goto err; + } + dc->dc_signature_len = dc_signature_len; + dc_signature = OPENSSL_malloc(dc_signature_len); + if (dc_signature == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + goto err; + } + dc->dc_signature = dc_signature; + + if (!PACKET_copy_bytes(&pkt, dc_signature, dc_signature_len)) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_INVALID_VALUE); + goto err; + } + + return dc; +err: + DC_free(dc); + return NULL; +} + +DELEGATED_CREDENTIAL *DC_load_from_file(const char *file) +{ + return DC_load_from_file_ex(file, NULL, NULL); +} + +DELEGATED_CREDENTIAL *DC_load_from_file_ex(const char *file, + OSSL_LIB_CTX *libctx, + const char *propq) +{ + DELEGATED_CREDENTIAL *dc = NULL; + BIO *bio_dc = NULL; + char *dc_hex_buf = NULL; + unsigned char *dc_buf = NULL; + size_t dc_hex_len, len; + size_t dc_buf_len; + + dc_hex_buf = OPENSSL_malloc(DC_MAX_LEN); + if (dc_hex_buf == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + goto err; + } + + bio_dc = BIO_new_file(file, "r"); + if (bio_dc == NULL) { + goto err; + } + + dc_hex_len = BIO_read(bio_dc, dc_hex_buf, DC_MAX_LEN - 1); + if (dc_hex_len <= 0) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_UNKNOWN_FORMAT); + goto err; + } + + if (dc_hex_buf[dc_hex_len - 1] == '\n') + dc_hex_buf[dc_hex_len - 1] = '\0'; + else + dc_hex_buf[dc_hex_len] = '\0'; + + /* + * parse from hex byte, just for tmp, because there is no + * standard dc format define + */ + len = dc_hex_len / 2; + + dc_buf = OPENSSL_malloc(len); + if (dc_buf == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!OPENSSL_hexstr2buf_ex(dc_buf, len, &dc_buf_len, dc_hex_buf, '\0')) { + ERR_raise(ERR_LIB_ASN1, ASN1_R_UNKNOWN_FORMAT); + goto err; + } + + dc = DC_new_from_raw_byte_ex(dc_buf, dc_buf_len, libctx, propq); + if (dc == NULL) + goto err; + +err: + OPENSSL_free(dc_buf); + OPENSSL_free(dc_hex_buf); + BIO_free(bio_dc); + return dc; +} diff --git a/openssl/src/crypto/asn1/x_info.c b/openssl/src/crypto/asn1/x_info.c index 8a4d2dba0..f8bc47898 100644 --- a/openssl/src/crypto/asn1/x_info.c +++ b/openssl/src/crypto/asn1/x_info.c @@ -18,8 +18,10 @@ X509_INFO *X509_INFO_new(void) X509_INFO *ret; ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return NULL; + } return ret; } diff --git a/openssl/src/crypto/asn1/x_int64.c b/openssl/src/crypto/asn1/x_int64.c index b7251b8ad..eb78c7e36 100644 --- a/openssl/src/crypto/asn1/x_int64.c +++ b/openssl/src/crypto/asn1/x_int64.c @@ -28,8 +28,10 @@ static int uint64_new(ASN1_VALUE **pval, const ASN1_ITEM *it) { - if ((*pval = (ASN1_VALUE *)OPENSSL_zalloc(sizeof(uint64_t))) == NULL) + if ((*pval = (ASN1_VALUE *)OPENSSL_zalloc(sizeof(uint64_t))) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } return 1; } @@ -121,8 +123,10 @@ static int uint64_print(BIO *out, const ASN1_VALUE **pval, const ASN1_ITEM *it, static int uint32_new(ASN1_VALUE **pval, const ASN1_ITEM *it) { - if ((*pval = (ASN1_VALUE *)OPENSSL_zalloc(sizeof(uint32_t))) == NULL) + if ((*pval = (ASN1_VALUE *)OPENSSL_zalloc(sizeof(uint32_t))) == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } return 1; } @@ -220,8 +224,8 @@ static int uint32_print(BIO *out, const ASN1_VALUE **pval, const ASN1_ITEM *it, int indent, const ASN1_PCTX *pctx) { if ((it->size & INTxx_FLAG_SIGNED) == INTxx_FLAG_SIGNED) - return BIO_printf(out, "%d\n", (int)**(int32_t **)pval); - return BIO_printf(out, "%u\n", (unsigned int)**(uint32_t **)pval); + return BIO_printf(out, "%d\n", **(int32_t **)pval); + return BIO_printf(out, "%u\n", **(uint32_t **)pval); } diff --git a/openssl/src/crypto/asn1/x_pkey.c b/openssl/src/crypto/asn1/x_pkey.c index 34b7286d7..b63c7c648 100644 --- a/openssl/src/crypto/asn1/x_pkey.c +++ b/openssl/src/crypto/asn1/x_pkey.c @@ -19,17 +19,18 @@ X509_PKEY *X509_PKEY_new(void) ret = OPENSSL_zalloc(sizeof(*ret)); if (ret == NULL) - return NULL; + goto err; ret->enc_algor = X509_ALGOR_new(); ret->enc_pkey = ASN1_OCTET_STRING_new(); - if (ret->enc_algor == NULL || ret->enc_pkey == NULL) { - X509_PKEY_free(ret); - ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB); - return NULL; - } + if (ret->enc_algor == NULL || ret->enc_pkey == NULL) + goto err; return ret; +err: + X509_PKEY_free(ret); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); + return NULL; } void X509_PKEY_free(X509_PKEY *x) diff --git a/openssl/src/crypto/async/arch/async_null.c b/openssl/src/crypto/async/arch/async_null.c index 306c43df4..675c1d35b 100644 --- a/openssl/src/crypto/async/arch/async_null.c +++ b/openssl/src/crypto/async/arch/async_null.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -16,21 +16,6 @@ int ASYNC_is_capable(void) return 0; } -int ASYNC_set_mem_functions(ASYNC_stack_alloc_fn alloc_fn, - ASYNC_stack_free_fn free_fn) -{ - return 0; -} - -void ASYNC_get_mem_functions(ASYNC_stack_alloc_fn *alloc_fn, - ASYNC_stack_free_fn *free_fn) -{ - if (alloc_fn != NULL) - *alloc_fn = NULL; - if (free_fn != NULL) - *free_fn = NULL; -} - void async_local_cleanup(void) { } diff --git a/openssl/src/crypto/async/arch/async_null.h b/openssl/src/crypto/async/arch/async_null.h index 536c9829c..ef7abbdb9 100644 --- a/openssl/src/crypto/async/arch/async_null.h +++ b/openssl/src/crypto/async/arch/async_null.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,12 +21,21 @@ typedef struct async_fibre_st { int dummy; } async_fibre; - +# ifdef async_fibre_swapcontext +# undef async_fibre_swapcontext +# endif # define async_fibre_swapcontext(o,n,r) 0 +# ifdef async_fibre_makecontext +# undef async_fibre_makecontext +# endif # define async_fibre_makecontext(c) 0 +# ifdef async_fibre_free +# undef async_fibre_free +# endif # define async_fibre_free(f) +# ifdef async_fibre_init_dispatcher +# undef async_fibre_init_dispatcher +# endif # define async_fibre_init_dispatcher(f) -# define async_local_init() 1 -# define async_local_deinit() #endif diff --git a/openssl/src/crypto/async/arch/async_posix.c b/openssl/src/crypto/async/arch/async_posix.c index f2b507c7d..e107e09a3 100644 --- a/openssl/src/crypto/async/arch/async_posix.c +++ b/openssl/src/crypto/async/arch/async_posix.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,31 +14,9 @@ # include # include -# include -# include #define STACKSIZE 32768 -static CRYPTO_RWLOCK *async_mem_lock; - -static void *async_stack_alloc(size_t *num); -static void async_stack_free(void *addr); - -int async_local_init(void) -{ - async_mem_lock = CRYPTO_THREAD_lock_new(); - return async_mem_lock != NULL; -} - -void async_local_deinit(void) -{ - CRYPTO_THREAD_lock_free(async_mem_lock); -} - -static int allow_customize = 1; -static ASYNC_stack_alloc_fn stack_alloc_impl = async_stack_alloc; -static ASYNC_stack_free_fn stack_free_impl = async_stack_free; - int ASYNC_is_capable(void) { ucontext_t ctx; @@ -50,45 +28,6 @@ int ASYNC_is_capable(void) return getcontext(&ctx) == 0; } -int ASYNC_set_mem_functions(ASYNC_stack_alloc_fn alloc_fn, - ASYNC_stack_free_fn free_fn) -{ - OPENSSL_init_crypto(OPENSSL_INIT_ASYNC, NULL); - - if (!CRYPTO_THREAD_write_lock(async_mem_lock)) - return 0; - if (!allow_customize) { - CRYPTO_THREAD_unlock(async_mem_lock); - return 0; - } - CRYPTO_THREAD_unlock(async_mem_lock); - - if (alloc_fn != NULL) - stack_alloc_impl = alloc_fn; - if (free_fn != NULL) - stack_free_impl = free_fn; - return 1; -} - -void ASYNC_get_mem_functions(ASYNC_stack_alloc_fn *alloc_fn, - ASYNC_stack_free_fn *free_fn) -{ - if (alloc_fn != NULL) - *alloc_fn = stack_alloc_impl; - if (free_fn != NULL) - *free_fn = stack_free_impl; -} - -static void *async_stack_alloc(size_t *num) -{ - return OPENSSL_malloc(*num); -} - -static void async_stack_free(void *addr) -{ - OPENSSL_free(addr); -} - void async_local_cleanup(void) { } @@ -99,22 +38,9 @@ int async_fibre_makecontext(async_fibre *fibre) fibre->env_init = 0; #endif if (getcontext(&fibre->fibre) == 0) { - size_t num = STACKSIZE; - - /* - * Disallow customisation after the first - * stack is allocated. - */ - if (allow_customize) { - if (!CRYPTO_THREAD_write_lock(async_mem_lock)) - return 0; - allow_customize = 0; - CRYPTO_THREAD_unlock(async_mem_lock); - } - - fibre->fibre.uc_stack.ss_sp = stack_alloc_impl(&num); + fibre->fibre.uc_stack.ss_sp = OPENSSL_malloc(STACKSIZE); if (fibre->fibre.uc_stack.ss_sp != NULL) { - fibre->fibre.uc_stack.ss_size = num; + fibre->fibre.uc_stack.ss_size = STACKSIZE; fibre->fibre.uc_link = NULL; makecontext(&fibre->fibre, async_start_func, 0); return 1; @@ -127,7 +53,7 @@ int async_fibre_makecontext(async_fibre *fibre) void async_fibre_free(async_fibre *fibre) { - stack_free_impl(fibre->fibre.uc_stack.ss_sp); + OPENSSL_free(fibre->fibre.uc_stack.ss_sp); fibre->fibre.uc_stack.ss_sp = NULL; } diff --git a/openssl/src/crypto/async/arch/async_posix.h b/openssl/src/crypto/async/arch/async_posix.h index 603965310..86cb31b71 100644 --- a/openssl/src/crypto/async/arch/async_posix.h +++ b/openssl/src/crypto/async/arch/async_posix.h @@ -61,9 +61,6 @@ typedef struct async_fibre_st { # endif } async_fibre; -int async_local_init(void); -void async_local_deinit(void); - static ossl_inline int async_fibre_swapcontext(async_fibre *o, async_fibre *n, int r) { # ifdef USE_SWAPCONTEXT @@ -82,6 +79,9 @@ static ossl_inline int async_fibre_swapcontext(async_fibre *o, async_fibre *n, i return 1; } +# ifdef async_fibre_init_dispatcher +# undef async_fibre_init_dispatcher +# endif # define async_fibre_init_dispatcher(d) int async_fibre_makecontext(async_fibre *fibre); diff --git a/openssl/src/crypto/async/arch/async_win.c b/openssl/src/crypto/async/arch/async_win.c index cbb8ef24e..cbb130090 100644 --- a/openssl/src/crypto/async/arch/async_win.c +++ b/openssl/src/crypto/async/arch/async_win.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,49 +20,15 @@ int ASYNC_is_capable(void) return 1; } -int ASYNC_set_mem_functions(ASYNC_stack_alloc_fn alloc_fn, - ASYNC_stack_free_fn free_fn) -{ - return 0; -} - -void ASYNC_get_mem_functions(ASYNC_stack_alloc_fn *alloc_fn, - ASYNC_stack_free_fn *free_fn) -{ - if (alloc_fn != NULL) - *alloc_fn = NULL; - if (free_fn != NULL) - *free_fn = NULL; -} - void async_local_cleanup(void) { - async_ctx *ctx = async_get_ctx(); - if (ctx != NULL) { - async_fibre *fibre = &ctx->dispatcher; - if (fibre != NULL && fibre->fibre != NULL && fibre->converted) { - ConvertFiberToThread(); - fibre->fibre = NULL; - } - } + if (GetCurrentFiber()) + ConvertFiberToThread(); } -int async_fibre_init_dispatcher(async_fibre *fibre) +int async_fibre_init_dispatcher(async_ctx *ctx) { -# if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x600 - fibre->fibre = ConvertThreadToFiberEx(NULL, FIBER_FLAG_FLOAT_SWITCH); -# else - fibre->fibre = ConvertThreadToFiber(NULL); -# endif - if (fibre->fibre == NULL) { - fibre->converted = 0; - fibre->fibre = GetCurrentFiber(); - if (fibre->fibre == NULL) - return 0; - } else { - fibre->converted = 1; - } - + ConvertThreadToFiber(NULL); return 1; } diff --git a/openssl/src/crypto/async/arch/async_win.h b/openssl/src/crypto/async/arch/async_win.h index 6a61a9a38..9a0d7d9f2 100644 --- a/openssl/src/crypto/async/arch/async_win.h +++ b/openssl/src/crypto/async/arch/async_win.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,7 +25,7 @@ typedef struct async_fibre_st { } async_fibre; # define async_fibre_swapcontext(o,n,r) \ - (SwitchToFiber((n)->fibre), 1) + ((o)->fibre = GetCurrentFiber(), SwitchToFiber((n)->fibre), 1) # if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x600 # define async_fibre_makecontext(c) \ @@ -37,10 +37,8 @@ typedef struct async_fibre_st { # endif # define async_fibre_free(f) (DeleteFiber((f)->fibre)) -# define async_local_init() 1 -# define async_local_deinit() -int async_fibre_init_dispatcher(async_fibre *fibre); +int async_fibre_init_dispatcher(async_ctx *ctx); VOID CALLBACK async_start_func_win(PVOID unused); #endif diff --git a/openssl/src/crypto/async/async.c b/openssl/src/crypto/async/async.c index 46c87d6a5..c9de72cf0 100644 --- a/openssl/src/crypto/async/async.c +++ b/openssl/src/crypto/async/async.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -40,10 +40,12 @@ static async_ctx *async_ctx_new(void) return NULL; nctx = OPENSSL_malloc(sizeof(*nctx)); - if (nctx == NULL) + if (nctx == NULL) { + ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE); goto err; + } - async_fibre_init_dispatcher(&nctx->dispatcher); + async_fibre_init_dispatcher(nctx); nctx->currjob = NULL; nctx->blocked = 0; if (!CRYPTO_THREAD_set_local(&ctxkey, nctx)) @@ -80,8 +82,10 @@ static ASYNC_JOB *async_job_new(void) ASYNC_JOB *job = NULL; job = OPENSSL_zalloc(sizeof(*job)); - if (job == NULL) + if (job == NULL) { + ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE); return NULL; + } job->status = ASYNC_JOB_RUNNING; @@ -159,8 +163,7 @@ void async_start_func(void) /* Stop the job */ job->status = ASYNC_JOB_STOPPING; - if (!async_fibre_swapcontext(&job->fibrectx, - &ctx->dispatcher, 1)) { + if (!async_fibre_swapcontext(&job->fibrectx, &job->back, 1)) { /* * Should not happen. Getting here will close the thread...can't do * much about it @@ -175,6 +178,7 @@ int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret, { async_ctx *ctx; OSSL_LIB_CTX *libctx; + ASYNC_JOB *new; if (!OPENSSL_init_crypto(OPENSSL_INIT_ASYNC, NULL)) return ASYNC_ERR; @@ -185,16 +189,14 @@ int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret, if (ctx == NULL) return ASYNC_ERR; - if (*job != NULL) - ctx->currjob = *job; - for (;;) { - if (ctx->currjob != NULL) { + if (*job) { + ctx->currjob = *job; if (ctx->currjob->status == ASYNC_JOB_STOPPING) { *ret = ctx->currjob->ret; ctx->currjob->waitctx = NULL; async_release_job(ctx->currjob); - ctx->currjob = NULL; + ctx->currjob = ctx->currjob->prevjob; *job = NULL; return ASYNC_FINISH; } @@ -202,13 +204,15 @@ int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret, if (ctx->currjob->status == ASYNC_JOB_PAUSING) { *job = ctx->currjob; ctx->currjob->status = ASYNC_JOB_PAUSED; - ctx->currjob = NULL; + ctx->currjob = ctx->currjob->prevjob; return ASYNC_PAUSE; } if (ctx->currjob->status == ASYNC_JOB_PAUSED) { if (*job == NULL) return ASYNC_ERR; + if (*job != ctx->currjob) + (*job)->prevjob = ctx->currjob; ctx->currjob = *job; /* @@ -221,8 +225,10 @@ int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret, ERR_raise(ERR_LIB_ASYNC, ERR_R_INTERNAL_ERROR); goto err; } + + ctx->currjob->status = ASYNC_JOB_RUNNING; /* Resume previous job */ - if (!async_fibre_swapcontext(&ctx->dispatcher, + if (!async_fibre_swapcontext(&ctx->currjob->back, &ctx->currjob->fibrectx, 1)) { ctx->currjob->libctx = OSSL_LIB_CTX_set0_default(libctx); ERR_raise(ERR_LIB_ASYNC, ASYNC_R_FAILED_TO_SWAP_CONTEXT); @@ -234,37 +240,34 @@ int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret, * been changed to. */ ctx->currjob->libctx = OSSL_LIB_CTX_set0_default(libctx); + ctx->currjob = (*job)->prevjob; continue; } - - /* Should not happen */ - ERR_raise(ERR_LIB_ASYNC, ERR_R_INTERNAL_ERROR); - async_release_job(ctx->currjob); - ctx->currjob = NULL; - *job = NULL; - return ASYNC_ERR; } /* Start a new job */ - if ((ctx->currjob = async_get_pool_job()) == NULL) + if ((new = async_get_pool_job()) == NULL) return ASYNC_NO_JOBS; if (args != NULL) { - ctx->currjob->funcargs = OPENSSL_malloc(size); - if (ctx->currjob->funcargs == NULL) { - async_release_job(ctx->currjob); - ctx->currjob = NULL; + new->funcargs = OPENSSL_malloc(size); + if (new->funcargs == NULL) { + ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE); + async_release_job(new); return ASYNC_ERR; } - memcpy(ctx->currjob->funcargs, args, size); + memcpy(new->funcargs, args, size); } else { - ctx->currjob->funcargs = NULL; + new->funcargs = NULL; } - ctx->currjob->func = func; - ctx->currjob->waitctx = wctx; + new->func = func; + new->waitctx = wctx; + new->prevjob = ctx->currjob; + ctx->currjob = new; + *job = new; libctx = ossl_lib_ctx_get_concrete(NULL); - if (!async_fibre_swapcontext(&ctx->dispatcher, + if (!async_fibre_swapcontext(&ctx->currjob->back, &ctx->currjob->fibrectx, 1)) { ERR_raise(ERR_LIB_ASYNC, ASYNC_R_FAILED_TO_SWAP_CONTEXT); goto err; @@ -301,8 +304,7 @@ int ASYNC_pause_job(void) job = ctx->currjob; job->status = ASYNC_JOB_PAUSING; - if (!async_fibre_swapcontext(&job->fibrectx, - &ctx->dispatcher, 1)) { + if (!async_fibre_swapcontext(&job->fibrectx, &job->back, 1)) { ERR_raise(ERR_LIB_ASYNC, ASYNC_R_FAILED_TO_SWAP_CONTEXT); return 0; } @@ -335,14 +337,13 @@ int async_init(void) return 0; } - return async_local_init(); + return 1; } void async_deinit(void) { CRYPTO_THREAD_cleanup_local(&ctxkey); CRYPTO_THREAD_cleanup_local(&poolkey); - async_local_deinit(); } int ASYNC_init_thread(size_t max_size, size_t init_size) @@ -362,12 +363,14 @@ int ASYNC_init_thread(size_t max_size, size_t init_size) return 0; pool = OPENSSL_zalloc(sizeof(*pool)); - if (pool == NULL) + if (pool == NULL) { + ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE); return 0; + } pool->jobs = sk_ASYNC_JOB_new_reserve(NULL, init_size); if (pool->jobs == NULL) { - ERR_raise(ERR_LIB_ASYNC, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE); OPENSSL_free(pool); return 0; } diff --git a/openssl/src/crypto/async/async_local.h b/openssl/src/crypto/async/async_local.h index c06f413cf..309a48941 100644 --- a/openssl/src/crypto/async/async_local.h +++ b/openssl/src/crypto/async/async_local.h @@ -31,13 +31,14 @@ typedef struct async_pool_st async_pool; #include "arch/async_null.h" struct async_ctx_st { - async_fibre dispatcher; ASYNC_JOB *currjob; unsigned int blocked; }; struct async_job_st { async_fibre fibrectx; + async_fibre back; + struct async_job_st *prevjob; int (*func) (void *); void *funcargs; int ret; diff --git a/openssl/src/crypto/async/async_wait.c b/openssl/src/crypto/async/async_wait.c index c5d000a03..df7d29302 100644 --- a/openssl/src/crypto/async/async_wait.c +++ b/openssl/src/crypto/async/async_wait.c @@ -47,8 +47,10 @@ int ASYNC_WAIT_CTX_set_wait_fd(ASYNC_WAIT_CTX *ctx, const void *key, { struct fd_lookup_st *fdlookup; - if ((fdlookup = OPENSSL_zalloc(sizeof(*fdlookup))) == NULL) + if ((fdlookup = OPENSSL_zalloc(sizeof(*fdlookup))) == NULL) { + ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE); return 0; + } fdlookup->key = key; fdlookup->fd = fd; diff --git a/openssl/src/crypto/bf/bf_cfb64.c b/openssl/src/crypto/bf/bf_cfb64.c deleted file mode 100644 index d7a926620..000000000 --- a/openssl/src/crypto/bf/bf_cfb64.c +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * BF low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "bf_local.h" - -/* - * The input and output encrypted as though 64bit cfb mode is being used. - * The extra state information to record how much of the 64bit block we have - * used is contained in *num; - */ - -void BF_cfb64_encrypt(const unsigned char *in, unsigned char *out, - long length, const BF_KEY *schedule, - unsigned char *ivec, int *num, int encrypt) -{ - register BF_LONG v0, v1, t; - register int n = *num; - register long l = length; - BF_LONG ti[2]; - unsigned char *iv, c, cc; - - iv = (unsigned char *)ivec; - if (encrypt) { - while (l--) { - if (n == 0) { - n2l(iv, v0); - ti[0] = v0; - n2l(iv, v1); - ti[1] = v1; - BF_encrypt((BF_LONG *)ti, schedule); - iv = (unsigned char *)ivec; - t = ti[0]; - l2n(t, iv); - t = ti[1]; - l2n(t, iv); - iv = (unsigned char *)ivec; - } - c = *(in++) ^ iv[n]; - *(out++) = c; - iv[n] = c; - n = (n + 1) & 0x07; - } - } else { - while (l--) { - if (n == 0) { - n2l(iv, v0); - ti[0] = v0; - n2l(iv, v1); - ti[1] = v1; - BF_encrypt((BF_LONG *)ti, schedule); - iv = (unsigned char *)ivec; - t = ti[0]; - l2n(t, iv); - t = ti[1]; - l2n(t, iv); - iv = (unsigned char *)ivec; - } - cc = *(in++); - c = iv[n]; - iv[n] = cc; - *(out++) = c ^ cc; - n = (n + 1) & 0x07; - } - } - v0 = v1 = ti[0] = ti[1] = t = c = cc = 0; - *num = n; -} diff --git a/openssl/src/crypto/bf/bf_ecb.c b/openssl/src/crypto/bf/bf_ecb.c deleted file mode 100644 index b02768261..000000000 --- a/openssl/src/crypto/bf/bf_ecb.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * BF low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "bf_local.h" -#include - -/* - * Blowfish as implemented from 'Blowfish: Springer-Verlag paper' (From - * LECTURE NOTES IN COMPUTER SCIENCE 809, FAST SOFTWARE ENCRYPTION, CAMBRIDGE - * SECURITY WORKSHOP, CAMBRIDGE, U.K., DECEMBER 9-11, 1993) - */ - -const char *BF_options(void) -{ - return "blowfish(ptr)"; -} - -void BF_ecb_encrypt(const unsigned char *in, unsigned char *out, - const BF_KEY *key, int encrypt) -{ - BF_LONG l, d[2]; - - n2l(in, l); - d[0] = l; - n2l(in, l); - d[1] = l; - if (encrypt) - BF_encrypt(d, key); - else - BF_decrypt(d, key); - l = d[0]; - l2n(l, out); - l = d[1]; - l2n(l, out); - l = d[0] = d[1] = 0; -} diff --git a/openssl/src/crypto/bf/bf_enc.c b/openssl/src/crypto/bf/bf_enc.c deleted file mode 100644 index 40ddaf4af..000000000 --- a/openssl/src/crypto/bf/bf_enc.c +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * BF low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "bf_local.h" - -/* - * Blowfish as implemented from 'Blowfish: Springer-Verlag paper' (From - * LECTURE NOTES IN COMPUTER SCIENCE 809, FAST SOFTWARE ENCRYPTION, CAMBRIDGE - * SECURITY WORKSHOP, CAMBRIDGE, U.K., DECEMBER 9-11, 1993) - */ - -#if (BF_ROUNDS != 16) && (BF_ROUNDS != 20) -# error If you set BF_ROUNDS to some value other than 16 or 20, you will have \ -to modify the code. -#endif - -void BF_encrypt(BF_LONG *data, const BF_KEY *key) -{ - register BF_LONG l, r; - register const BF_LONG *p, *s; - - p = key->P; - s = &(key->S[0]); - l = data[0]; - r = data[1]; - - l ^= p[0]; - BF_ENC(r, l, s, p[1]); - BF_ENC(l, r, s, p[2]); - BF_ENC(r, l, s, p[3]); - BF_ENC(l, r, s, p[4]); - BF_ENC(r, l, s, p[5]); - BF_ENC(l, r, s, p[6]); - BF_ENC(r, l, s, p[7]); - BF_ENC(l, r, s, p[8]); - BF_ENC(r, l, s, p[9]); - BF_ENC(l, r, s, p[10]); - BF_ENC(r, l, s, p[11]); - BF_ENC(l, r, s, p[12]); - BF_ENC(r, l, s, p[13]); - BF_ENC(l, r, s, p[14]); - BF_ENC(r, l, s, p[15]); - BF_ENC(l, r, s, p[16]); -# if BF_ROUNDS == 20 - BF_ENC(r, l, s, p[17]); - BF_ENC(l, r, s, p[18]); - BF_ENC(r, l, s, p[19]); - BF_ENC(l, r, s, p[20]); -# endif - r ^= p[BF_ROUNDS + 1]; - - data[1] = l & 0xffffffffU; - data[0] = r & 0xffffffffU; -} - -void BF_decrypt(BF_LONG *data, const BF_KEY *key) -{ - register BF_LONG l, r; - register const BF_LONG *p, *s; - - p = key->P; - s = &(key->S[0]); - l = data[0]; - r = data[1]; - - l ^= p[BF_ROUNDS + 1]; -# if BF_ROUNDS == 20 - BF_ENC(r, l, s, p[20]); - BF_ENC(l, r, s, p[19]); - BF_ENC(r, l, s, p[18]); - BF_ENC(l, r, s, p[17]); -# endif - BF_ENC(r, l, s, p[16]); - BF_ENC(l, r, s, p[15]); - BF_ENC(r, l, s, p[14]); - BF_ENC(l, r, s, p[13]); - BF_ENC(r, l, s, p[12]); - BF_ENC(l, r, s, p[11]); - BF_ENC(r, l, s, p[10]); - BF_ENC(l, r, s, p[9]); - BF_ENC(r, l, s, p[8]); - BF_ENC(l, r, s, p[7]); - BF_ENC(r, l, s, p[6]); - BF_ENC(l, r, s, p[5]); - BF_ENC(r, l, s, p[4]); - BF_ENC(l, r, s, p[3]); - BF_ENC(r, l, s, p[2]); - BF_ENC(l, r, s, p[1]); - r ^= p[0]; - - data[1] = l & 0xffffffffU; - data[0] = r & 0xffffffffU; -} - -void BF_cbc_encrypt(const unsigned char *in, unsigned char *out, long length, - const BF_KEY *schedule, unsigned char *ivec, int encrypt) -{ - register BF_LONG tin0, tin1; - register BF_LONG tout0, tout1, xor0, xor1; - register long l = length; - BF_LONG tin[2]; - - if (encrypt) { - n2l(ivec, tout0); - n2l(ivec, tout1); - ivec -= 8; - for (l -= 8; l >= 0; l -= 8) { - n2l(in, tin0); - n2l(in, tin1); - tin0 ^= tout0; - tin1 ^= tout1; - tin[0] = tin0; - tin[1] = tin1; - BF_encrypt(tin, schedule); - tout0 = tin[0]; - tout1 = tin[1]; - l2n(tout0, out); - l2n(tout1, out); - } - if (l != -8) { - n2ln(in, tin0, tin1, l + 8); - tin0 ^= tout0; - tin1 ^= tout1; - tin[0] = tin0; - tin[1] = tin1; - BF_encrypt(tin, schedule); - tout0 = tin[0]; - tout1 = tin[1]; - l2n(tout0, out); - l2n(tout1, out); - } - l2n(tout0, ivec); - l2n(tout1, ivec); - } else { - n2l(ivec, xor0); - n2l(ivec, xor1); - ivec -= 8; - for (l -= 8; l >= 0; l -= 8) { - n2l(in, tin0); - n2l(in, tin1); - tin[0] = tin0; - tin[1] = tin1; - BF_decrypt(tin, schedule); - tout0 = tin[0] ^ xor0; - tout1 = tin[1] ^ xor1; - l2n(tout0, out); - l2n(tout1, out); - xor0 = tin0; - xor1 = tin1; - } - if (l != -8) { - n2l(in, tin0); - n2l(in, tin1); - tin[0] = tin0; - tin[1] = tin1; - BF_decrypt(tin, schedule); - tout0 = tin[0] ^ xor0; - tout1 = tin[1] ^ xor1; - l2nn(tout0, tout1, out, l + 8); - xor0 = tin0; - xor1 = tin1; - } - l2n(xor0, ivec); - l2n(xor1, ivec); - } - tin0 = tin1 = tout0 = tout1 = xor0 = xor1 = 0; - tin[0] = tin[1] = 0; -} diff --git a/openssl/src/crypto/bf/bf_local.h b/openssl/src/crypto/bf/bf_local.h deleted file mode 100644 index 53c6963e6..000000000 --- a/openssl/src/crypto/bf/bf_local.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_CRYPTO_BF_LOCAL_H -# define OSSL_CRYPTO_BF_LOCAL_H -# include - -/* NOTE - c is not incremented as per n2l */ -# define n2ln(c,l1,l2,n) { \ - c+=n; \ - l1=l2=0; \ - switch (n) { \ - case 8: l2 =((unsigned long)(*(--(c)))) ; \ - /* fall through */ \ - case 7: l2|=((unsigned long)(*(--(c))))<< 8; \ - /* fall through */ \ - case 6: l2|=((unsigned long)(*(--(c))))<<16; \ - /* fall through */ \ - case 5: l2|=((unsigned long)(*(--(c))))<<24; \ - /* fall through */ \ - case 4: l1 =((unsigned long)(*(--(c)))) ; \ - /* fall through */ \ - case 3: l1|=((unsigned long)(*(--(c))))<< 8; \ - /* fall through */ \ - case 2: l1|=((unsigned long)(*(--(c))))<<16; \ - /* fall through */ \ - case 1: l1|=((unsigned long)(*(--(c))))<<24; \ - } \ - } - -/* NOTE - c is not incremented as per l2n */ -# define l2nn(l1,l2,c,n) { \ - c+=n; \ - switch (n) { \ - case 8: *(--(c))=(unsigned char)(((l2) )&0xff); \ - /* fall through */ \ - case 7: *(--(c))=(unsigned char)(((l2)>> 8)&0xff); \ - /* fall through */ \ - case 6: *(--(c))=(unsigned char)(((l2)>>16)&0xff); \ - /* fall through */ \ - case 5: *(--(c))=(unsigned char)(((l2)>>24)&0xff); \ - /* fall through */ \ - case 4: *(--(c))=(unsigned char)(((l1) )&0xff); \ - /* fall through */ \ - case 3: *(--(c))=(unsigned char)(((l1)>> 8)&0xff); \ - /* fall through */ \ - case 2: *(--(c))=(unsigned char)(((l1)>>16)&0xff); \ - /* fall through */ \ - case 1: *(--(c))=(unsigned char)(((l1)>>24)&0xff); \ - } \ - } - -# undef n2l -# define n2l(c,l) (l =((unsigned long)(*((c)++)))<<24L, \ - l|=((unsigned long)(*((c)++)))<<16L, \ - l|=((unsigned long)(*((c)++)))<< 8L, \ - l|=((unsigned long)(*((c)++)))) - -# undef l2n -# define l2n(l,c) (*((c)++)=(unsigned char)(((l)>>24L)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16L)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8L)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff)) - -/* - * This is actually a big endian algorithm, the most significant byte is used - * to lookup array 0 - */ - -# define BF_ENC(LL,R,S,P) ( \ - LL^=P, \ - LL^=((( S[ ((R>>24)&0xff)] + \ - S[0x0100+((R>>16)&0xff)])^ \ - S[0x0200+((R>> 8)&0xff)])+ \ - S[0x0300+((R )&0xff)])&0xffffffffU \ - ) - -#endif diff --git a/openssl/src/crypto/bf/bf_ofb64.c b/openssl/src/crypto/bf/bf_ofb64.c deleted file mode 100644 index 086c3f07f..000000000 --- a/openssl/src/crypto/bf/bf_ofb64.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * BF low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "bf_local.h" - -/* - * The input and output encrypted as though 64bit ofb mode is being used. - * The extra state information to record how much of the 64bit block we have - * used is contained in *num; - */ -void BF_ofb64_encrypt(const unsigned char *in, unsigned char *out, - long length, const BF_KEY *schedule, - unsigned char *ivec, int *num) -{ - register BF_LONG v0, v1, t; - register int n = *num; - register long l = length; - unsigned char d[8]; - register char *dp; - BF_LONG ti[2]; - unsigned char *iv; - int save = 0; - - iv = (unsigned char *)ivec; - n2l(iv, v0); - n2l(iv, v1); - ti[0] = v0; - ti[1] = v1; - dp = (char *)d; - l2n(v0, dp); - l2n(v1, dp); - while (l--) { - if (n == 0) { - BF_encrypt((BF_LONG *)ti, schedule); - dp = (char *)d; - t = ti[0]; - l2n(t, dp); - t = ti[1]; - l2n(t, dp); - save++; - } - *(out++) = *(in++) ^ d[n]; - n = (n + 1) & 0x07; - } - if (save) { - v0 = ti[0]; - v1 = ti[1]; - iv = (unsigned char *)ivec; - l2n(v0, iv); - l2n(v1, iv); - } - t = v0 = v1 = ti[0] = ti[1] = 0; - *num = n; -} diff --git a/openssl/src/crypto/bf/bf_pi.h b/openssl/src/crypto/bf/bf_pi.h deleted file mode 100644 index 8b9896e7f..000000000 --- a/openssl/src/crypto/bf/bf_pi.h +++ /dev/null @@ -1,530 +0,0 @@ -/* - * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -static const BF_KEY bf_init = { - { - 0x243f6a88L, 0x85a308d3L, 0x13198a2eL, 0x03707344L, - 0xa4093822L, 0x299f31d0L, 0x082efa98L, 0xec4e6c89L, - 0x452821e6L, 0x38d01377L, 0xbe5466cfL, 0x34e90c6cL, - 0xc0ac29b7L, 0xc97c50ddL, 0x3f84d5b5L, 0xb5470917L, - 0x9216d5d9L, 0x8979fb1b}, { - 0xd1310ba6L, 0x98dfb5acL, 0x2ffd72dbL, - 0xd01adfb7L, - 0xb8e1afedL, 0x6a267e96L, 0xba7c9045L, - 0xf12c7f99L, - 0x24a19947L, 0xb3916cf7L, 0x0801f2e2L, - 0x858efc16L, - 0x636920d8L, 0x71574e69L, 0xa458fea3L, - 0xf4933d7eL, - 0x0d95748fL, 0x728eb658L, 0x718bcd58L, - 0x82154aeeL, - 0x7b54a41dL, 0xc25a59b5L, 0x9c30d539L, - 0x2af26013L, - 0xc5d1b023L, 0x286085f0L, 0xca417918L, - 0xb8db38efL, - 0x8e79dcb0L, 0x603a180eL, 0x6c9e0e8bL, - 0xb01e8a3eL, - 0xd71577c1L, 0xbd314b27L, 0x78af2fdaL, - 0x55605c60L, - 0xe65525f3L, 0xaa55ab94L, 0x57489862L, - 0x63e81440L, - 0x55ca396aL, 0x2aab10b6L, 0xb4cc5c34L, - 0x1141e8ceL, - 0xa15486afL, 0x7c72e993L, 0xb3ee1411L, - 0x636fbc2aL, - 0x2ba9c55dL, 0x741831f6L, 0xce5c3e16L, - 0x9b87931eL, - 0xafd6ba33L, 0x6c24cf5cL, 0x7a325381L, - 0x28958677L, - 0x3b8f4898L, 0x6b4bb9afL, 0xc4bfe81bL, - 0x66282193L, - 0x61d809ccL, 0xfb21a991L, 0x487cac60L, - 0x5dec8032L, - 0xef845d5dL, 0xe98575b1L, 0xdc262302L, - 0xeb651b88L, - 0x23893e81L, 0xd396acc5L, 0x0f6d6ff3L, - 0x83f44239L, - 0x2e0b4482L, 0xa4842004L, 0x69c8f04aL, - 0x9e1f9b5eL, - 0x21c66842L, 0xf6e96c9aL, 0x670c9c61L, - 0xabd388f0L, - 0x6a51a0d2L, 0xd8542f68L, 0x960fa728L, - 0xab5133a3L, - 0x6eef0b6cL, 0x137a3be4L, 0xba3bf050L, - 0x7efb2a98L, - 0xa1f1651dL, 0x39af0176L, 0x66ca593eL, - 0x82430e88L, - 0x8cee8619L, 0x456f9fb4L, 0x7d84a5c3L, - 0x3b8b5ebeL, - 0xe06f75d8L, 0x85c12073L, 0x401a449fL, - 0x56c16aa6L, - 0x4ed3aa62L, 0x363f7706L, 0x1bfedf72L, - 0x429b023dL, - 0x37d0d724L, 0xd00a1248L, 0xdb0fead3L, - 0x49f1c09bL, - 0x075372c9L, 0x80991b7bL, 0x25d479d8L, - 0xf6e8def7L, - 0xe3fe501aL, 0xb6794c3bL, 0x976ce0bdL, - 0x04c006baL, - 0xc1a94fb6L, 0x409f60c4L, 0x5e5c9ec2L, - 0x196a2463L, - 0x68fb6fafL, 0x3e6c53b5L, 0x1339b2ebL, - 0x3b52ec6fL, - 0x6dfc511fL, 0x9b30952cL, 0xcc814544L, - 0xaf5ebd09L, - 0xbee3d004L, 0xde334afdL, 0x660f2807L, - 0x192e4bb3L, - 0xc0cba857L, 0x45c8740fL, 0xd20b5f39L, - 0xb9d3fbdbL, - 0x5579c0bdL, 0x1a60320aL, 0xd6a100c6L, - 0x402c7279L, - 0x679f25feL, 0xfb1fa3ccL, 0x8ea5e9f8L, - 0xdb3222f8L, - 0x3c7516dfL, 0xfd616b15L, 0x2f501ec8L, - 0xad0552abL, - 0x323db5faL, 0xfd238760L, 0x53317b48L, - 0x3e00df82L, - 0x9e5c57bbL, 0xca6f8ca0L, 0x1a87562eL, - 0xdf1769dbL, - 0xd542a8f6L, 0x287effc3L, 0xac6732c6L, - 0x8c4f5573L, - 0x695b27b0L, 0xbbca58c8L, 0xe1ffa35dL, - 0xb8f011a0L, - 0x10fa3d98L, 0xfd2183b8L, 0x4afcb56cL, - 0x2dd1d35bL, - 0x9a53e479L, 0xb6f84565L, 0xd28e49bcL, - 0x4bfb9790L, - 0xe1ddf2daL, 0xa4cb7e33L, 0x62fb1341L, - 0xcee4c6e8L, - 0xef20cadaL, 0x36774c01L, 0xd07e9efeL, - 0x2bf11fb4L, - 0x95dbda4dL, 0xae909198L, 0xeaad8e71L, - 0x6b93d5a0L, - 0xd08ed1d0L, 0xafc725e0L, 0x8e3c5b2fL, - 0x8e7594b7L, - 0x8ff6e2fbL, 0xf2122b64L, 0x8888b812L, - 0x900df01cL, - 0x4fad5ea0L, 0x688fc31cL, 0xd1cff191L, - 0xb3a8c1adL, - 0x2f2f2218L, 0xbe0e1777L, 0xea752dfeL, - 0x8b021fa1L, - 0xe5a0cc0fL, 0xb56f74e8L, 0x18acf3d6L, - 0xce89e299L, - 0xb4a84fe0L, 0xfd13e0b7L, 0x7cc43b81L, - 0xd2ada8d9L, - 0x165fa266L, 0x80957705L, 0x93cc7314L, - 0x211a1477L, - 0xe6ad2065L, 0x77b5fa86L, 0xc75442f5L, - 0xfb9d35cfL, - 0xebcdaf0cL, 0x7b3e89a0L, 0xd6411bd3L, - 0xae1e7e49L, - 0x00250e2dL, 0x2071b35eL, 0x226800bbL, - 0x57b8e0afL, - 0x2464369bL, 0xf009b91eL, 0x5563911dL, - 0x59dfa6aaL, - 0x78c14389L, 0xd95a537fL, 0x207d5ba2L, - 0x02e5b9c5L, - 0x83260376L, 0x6295cfa9L, 0x11c81968L, - 0x4e734a41L, - 0xb3472dcaL, 0x7b14a94aL, 0x1b510052L, - 0x9a532915L, - 0xd60f573fL, 0xbc9bc6e4L, 0x2b60a476L, - 0x81e67400L, - 0x08ba6fb5L, 0x571be91fL, 0xf296ec6bL, - 0x2a0dd915L, - 0xb6636521L, 0xe7b9f9b6L, 0xff34052eL, - 0xc5855664L, - 0x53b02d5dL, 0xa99f8fa1L, 0x08ba4799L, - 0x6e85076aL, - 0x4b7a70e9L, 0xb5b32944L, 0xdb75092eL, - 0xc4192623L, - 0xad6ea6b0L, 0x49a7df7dL, 0x9cee60b8L, - 0x8fedb266L, - 0xecaa8c71L, 0x699a17ffL, 0x5664526cL, - 0xc2b19ee1L, - 0x193602a5L, 0x75094c29L, 0xa0591340L, - 0xe4183a3eL, - 0x3f54989aL, 0x5b429d65L, 0x6b8fe4d6L, - 0x99f73fd6L, - 0xa1d29c07L, 0xefe830f5L, 0x4d2d38e6L, - 0xf0255dc1L, - 0x4cdd2086L, 0x8470eb26L, 0x6382e9c6L, - 0x021ecc5eL, - 0x09686b3fL, 0x3ebaefc9L, 0x3c971814L, - 0x6b6a70a1L, - 0x687f3584L, 0x52a0e286L, 0xb79c5305L, - 0xaa500737L, - 0x3e07841cL, 0x7fdeae5cL, 0x8e7d44ecL, - 0x5716f2b8L, - 0xb03ada37L, 0xf0500c0dL, 0xf01c1f04L, - 0x0200b3ffL, - 0xae0cf51aL, 0x3cb574b2L, 0x25837a58L, - 0xdc0921bdL, - 0xd19113f9L, 0x7ca92ff6L, 0x94324773L, - 0x22f54701L, - 0x3ae5e581L, 0x37c2dadcL, 0xc8b57634L, - 0x9af3dda7L, - 0xa9446146L, 0x0fd0030eL, 0xecc8c73eL, - 0xa4751e41L, - 0xe238cd99L, 0x3bea0e2fL, 0x3280bba1L, - 0x183eb331L, - 0x4e548b38L, 0x4f6db908L, 0x6f420d03L, - 0xf60a04bfL, - 0x2cb81290L, 0x24977c79L, 0x5679b072L, - 0xbcaf89afL, - 0xde9a771fL, 0xd9930810L, 0xb38bae12L, - 0xdccf3f2eL, - 0x5512721fL, 0x2e6b7124L, 0x501adde6L, - 0x9f84cd87L, - 0x7a584718L, 0x7408da17L, 0xbc9f9abcL, - 0xe94b7d8cL, - 0xec7aec3aL, 0xdb851dfaL, 0x63094366L, - 0xc464c3d2L, - 0xef1c1847L, 0x3215d908L, 0xdd433b37L, - 0x24c2ba16L, - 0x12a14d43L, 0x2a65c451L, 0x50940002L, - 0x133ae4ddL, - 0x71dff89eL, 0x10314e55L, 0x81ac77d6L, - 0x5f11199bL, - 0x043556f1L, 0xd7a3c76bL, 0x3c11183bL, - 0x5924a509L, - 0xf28fe6edL, 0x97f1fbfaL, 0x9ebabf2cL, - 0x1e153c6eL, - 0x86e34570L, 0xeae96fb1L, 0x860e5e0aL, - 0x5a3e2ab3L, - 0x771fe71cL, 0x4e3d06faL, 0x2965dcb9L, - 0x99e71d0fL, - 0x803e89d6L, 0x5266c825L, 0x2e4cc978L, - 0x9c10b36aL, - 0xc6150ebaL, 0x94e2ea78L, 0xa5fc3c53L, - 0x1e0a2df4L, - 0xf2f74ea7L, 0x361d2b3dL, 0x1939260fL, - 0x19c27960L, - 0x5223a708L, 0xf71312b6L, 0xebadfe6eL, - 0xeac31f66L, - 0xe3bc4595L, 0xa67bc883L, 0xb17f37d1L, - 0x018cff28L, - 0xc332ddefL, 0xbe6c5aa5L, 0x65582185L, - 0x68ab9802L, - 0xeecea50fL, 0xdb2f953bL, 0x2aef7dadL, - 0x5b6e2f84L, - 0x1521b628L, 0x29076170L, 0xecdd4775L, - 0x619f1510L, - 0x13cca830L, 0xeb61bd96L, 0x0334fe1eL, - 0xaa0363cfL, - 0xb5735c90L, 0x4c70a239L, 0xd59e9e0bL, - 0xcbaade14L, - 0xeecc86bcL, 0x60622ca7L, 0x9cab5cabL, - 0xb2f3846eL, - 0x648b1eafL, 0x19bdf0caL, 0xa02369b9L, - 0x655abb50L, - 0x40685a32L, 0x3c2ab4b3L, 0x319ee9d5L, - 0xc021b8f7L, - 0x9b540b19L, 0x875fa099L, 0x95f7997eL, - 0x623d7da8L, - 0xf837889aL, 0x97e32d77L, 0x11ed935fL, - 0x16681281L, - 0x0e358829L, 0xc7e61fd6L, 0x96dedfa1L, - 0x7858ba99L, - 0x57f584a5L, 0x1b227263L, 0x9b83c3ffL, - 0x1ac24696L, - 0xcdb30aebL, 0x532e3054L, 0x8fd948e4L, - 0x6dbc3128L, - 0x58ebf2efL, 0x34c6ffeaL, 0xfe28ed61L, - 0xee7c3c73L, - 0x5d4a14d9L, 0xe864b7e3L, 0x42105d14L, - 0x203e13e0L, - 0x45eee2b6L, 0xa3aaabeaL, 0xdb6c4f15L, - 0xfacb4fd0L, - 0xc742f442L, 0xef6abbb5L, 0x654f3b1dL, - 0x41cd2105L, - 0xd81e799eL, 0x86854dc7L, 0xe44b476aL, - 0x3d816250L, - 0xcf62a1f2L, 0x5b8d2646L, 0xfc8883a0L, - 0xc1c7b6a3L, - 0x7f1524c3L, 0x69cb7492L, 0x47848a0bL, - 0x5692b285L, - 0x095bbf00L, 0xad19489dL, 0x1462b174L, - 0x23820e00L, - 0x58428d2aL, 0x0c55f5eaL, 0x1dadf43eL, - 0x233f7061L, - 0x3372f092L, 0x8d937e41L, 0xd65fecf1L, - 0x6c223bdbL, - 0x7cde3759L, 0xcbee7460L, 0x4085f2a7L, - 0xce77326eL, - 0xa6078084L, 0x19f8509eL, 0xe8efd855L, - 0x61d99735L, - 0xa969a7aaL, 0xc50c06c2L, 0x5a04abfcL, - 0x800bcadcL, - 0x9e447a2eL, 0xc3453484L, 0xfdd56705L, - 0x0e1e9ec9L, - 0xdb73dbd3L, 0x105588cdL, 0x675fda79L, - 0xe3674340L, - 0xc5c43465L, 0x713e38d8L, 0x3d28f89eL, - 0xf16dff20L, - 0x153e21e7L, 0x8fb03d4aL, 0xe6e39f2bL, - 0xdb83adf7L, - 0xe93d5a68L, 0x948140f7L, 0xf64c261cL, - 0x94692934L, - 0x411520f7L, 0x7602d4f7L, 0xbcf46b2eL, - 0xd4a20068L, - 0xd4082471L, 0x3320f46aL, 0x43b7d4b7L, - 0x500061afL, - 0x1e39f62eL, 0x97244546L, 0x14214f74L, - 0xbf8b8840L, - 0x4d95fc1dL, 0x96b591afL, 0x70f4ddd3L, - 0x66a02f45L, - 0xbfbc09ecL, 0x03bd9785L, 0x7fac6dd0L, - 0x31cb8504L, - 0x96eb27b3L, 0x55fd3941L, 0xda2547e6L, - 0xabca0a9aL, - 0x28507825L, 0x530429f4L, 0x0a2c86daL, - 0xe9b66dfbL, - 0x68dc1462L, 0xd7486900L, 0x680ec0a4L, - 0x27a18deeL, - 0x4f3ffea2L, 0xe887ad8cL, 0xb58ce006L, - 0x7af4d6b6L, - 0xaace1e7cL, 0xd3375fecL, 0xce78a399L, - 0x406b2a42L, - 0x20fe9e35L, 0xd9f385b9L, 0xee39d7abL, - 0x3b124e8bL, - 0x1dc9faf7L, 0x4b6d1856L, 0x26a36631L, - 0xeae397b2L, - 0x3a6efa74L, 0xdd5b4332L, 0x6841e7f7L, - 0xca7820fbL, - 0xfb0af54eL, 0xd8feb397L, 0x454056acL, - 0xba489527L, - 0x55533a3aL, 0x20838d87L, 0xfe6ba9b7L, - 0xd096954bL, - 0x55a867bcL, 0xa1159a58L, 0xcca92963L, - 0x99e1db33L, - 0xa62a4a56L, 0x3f3125f9L, 0x5ef47e1cL, - 0x9029317cL, - 0xfdf8e802L, 0x04272f70L, 0x80bb155cL, - 0x05282ce3L, - 0x95c11548L, 0xe4c66d22L, 0x48c1133fL, - 0xc70f86dcL, - 0x07f9c9eeL, 0x41041f0fL, 0x404779a4L, - 0x5d886e17L, - 0x325f51ebL, 0xd59bc0d1L, 0xf2bcc18fL, - 0x41113564L, - 0x257b7834L, 0x602a9c60L, 0xdff8e8a3L, - 0x1f636c1bL, - 0x0e12b4c2L, 0x02e1329eL, 0xaf664fd1L, - 0xcad18115L, - 0x6b2395e0L, 0x333e92e1L, 0x3b240b62L, - 0xeebeb922L, - 0x85b2a20eL, 0xe6ba0d99L, 0xde720c8cL, - 0x2da2f728L, - 0xd0127845L, 0x95b794fdL, 0x647d0862L, - 0xe7ccf5f0L, - 0x5449a36fL, 0x877d48faL, 0xc39dfd27L, - 0xf33e8d1eL, - 0x0a476341L, 0x992eff74L, 0x3a6f6eabL, - 0xf4f8fd37L, - 0xa812dc60L, 0xa1ebddf8L, 0x991be14cL, - 0xdb6e6b0dL, - 0xc67b5510L, 0x6d672c37L, 0x2765d43bL, - 0xdcd0e804L, - 0xf1290dc7L, 0xcc00ffa3L, 0xb5390f92L, - 0x690fed0bL, - 0x667b9ffbL, 0xcedb7d9cL, 0xa091cf0bL, - 0xd9155ea3L, - 0xbb132f88L, 0x515bad24L, 0x7b9479bfL, - 0x763bd6ebL, - 0x37392eb3L, 0xcc115979L, 0x8026e297L, - 0xf42e312dL, - 0x6842ada7L, 0xc66a2b3bL, 0x12754cccL, - 0x782ef11cL, - 0x6a124237L, 0xb79251e7L, 0x06a1bbe6L, - 0x4bfb6350L, - 0x1a6b1018L, 0x11caedfaL, 0x3d25bdd8L, - 0xe2e1c3c9L, - 0x44421659L, 0x0a121386L, 0xd90cec6eL, - 0xd5abea2aL, - 0x64af674eL, 0xda86a85fL, 0xbebfe988L, - 0x64e4c3feL, - 0x9dbc8057L, 0xf0f7c086L, 0x60787bf8L, - 0x6003604dL, - 0xd1fd8346L, 0xf6381fb0L, 0x7745ae04L, - 0xd736fcccL, - 0x83426b33L, 0xf01eab71L, 0xb0804187L, - 0x3c005e5fL, - 0x77a057beL, 0xbde8ae24L, 0x55464299L, - 0xbf582e61L, - 0x4e58f48fL, 0xf2ddfda2L, 0xf474ef38L, - 0x8789bdc2L, - 0x5366f9c3L, 0xc8b38e74L, 0xb475f255L, - 0x46fcd9b9L, - 0x7aeb2661L, 0x8b1ddf84L, 0x846a0e79L, - 0x915f95e2L, - 0x466e598eL, 0x20b45770L, 0x8cd55591L, - 0xc902de4cL, - 0xb90bace1L, 0xbb8205d0L, 0x11a86248L, - 0x7574a99eL, - 0xb77f19b6L, 0xe0a9dc09L, 0x662d09a1L, - 0xc4324633L, - 0xe85a1f02L, 0x09f0be8cL, 0x4a99a025L, - 0x1d6efe10L, - 0x1ab93d1dL, 0x0ba5a4dfL, 0xa186f20fL, - 0x2868f169L, - 0xdcb7da83L, 0x573906feL, 0xa1e2ce9bL, - 0x4fcd7f52L, - 0x50115e01L, 0xa70683faL, 0xa002b5c4L, - 0x0de6d027L, - 0x9af88c27L, 0x773f8641L, 0xc3604c06L, - 0x61a806b5L, - 0xf0177a28L, 0xc0f586e0L, 0x006058aaL, - 0x30dc7d62L, - 0x11e69ed7L, 0x2338ea63L, 0x53c2dd94L, - 0xc2c21634L, - 0xbbcbee56L, 0x90bcb6deL, 0xebfc7da1L, - 0xce591d76L, - 0x6f05e409L, 0x4b7c0188L, 0x39720a3dL, - 0x7c927c24L, - 0x86e3725fL, 0x724d9db9L, 0x1ac15bb4L, - 0xd39eb8fcL, - 0xed545578L, 0x08fca5b5L, 0xd83d7cd3L, - 0x4dad0fc4L, - 0x1e50ef5eL, 0xb161e6f8L, 0xa28514d9L, - 0x6c51133cL, - 0x6fd5c7e7L, 0x56e14ec4L, 0x362abfceL, - 0xddc6c837L, - 0xd79a3234L, 0x92638212L, 0x670efa8eL, - 0x406000e0L, - 0x3a39ce37L, 0xd3faf5cfL, 0xabc27737L, - 0x5ac52d1bL, - 0x5cb0679eL, 0x4fa33742L, 0xd3822740L, - 0x99bc9bbeL, - 0xd5118e9dL, 0xbf0f7315L, 0xd62d1c7eL, - 0xc700c47bL, - 0xb78c1b6bL, 0x21a19045L, 0xb26eb1beL, - 0x6a366eb4L, - 0x5748ab2fL, 0xbc946e79L, 0xc6a376d2L, - 0x6549c2c8L, - 0x530ff8eeL, 0x468dde7dL, 0xd5730a1dL, - 0x4cd04dc6L, - 0x2939bbdbL, 0xa9ba4650L, 0xac9526e8L, - 0xbe5ee304L, - 0xa1fad5f0L, 0x6a2d519aL, 0x63ef8ce2L, - 0x9a86ee22L, - 0xc089c2b8L, 0x43242ef6L, 0xa51e03aaL, - 0x9cf2d0a4L, - 0x83c061baL, 0x9be96a4dL, 0x8fe51550L, - 0xba645bd6L, - 0x2826a2f9L, 0xa73a3ae1L, 0x4ba99586L, - 0xef5562e9L, - 0xc72fefd3L, 0xf752f7daL, 0x3f046f69L, - 0x77fa0a59L, - 0x80e4a915L, 0x87b08601L, 0x9b09e6adL, - 0x3b3ee593L, - 0xe990fd5aL, 0x9e34d797L, 0x2cf0b7d9L, - 0x022b8b51L, - 0x96d5ac3aL, 0x017da67dL, 0xd1cf3ed6L, - 0x7c7d2d28L, - 0x1f9f25cfL, 0xadf2b89bL, 0x5ad6b472L, - 0x5a88f54cL, - 0xe029ac71L, 0xe019a5e6L, 0x47b0acfdL, - 0xed93fa9bL, - 0xe8d3c48dL, 0x283b57ccL, 0xf8d56629L, - 0x79132e28L, - 0x785f0191L, 0xed756055L, 0xf7960e44L, - 0xe3d35e8cL, - 0x15056dd4L, 0x88f46dbaL, 0x03a16125L, - 0x0564f0bdL, - 0xc3eb9e15L, 0x3c9057a2L, 0x97271aecL, - 0xa93a072aL, - 0x1b3f6d9bL, 0x1e6321f5L, 0xf59c66fbL, - 0x26dcf319L, - 0x7533d928L, 0xb155fdf5L, 0x03563482L, - 0x8aba3cbbL, - 0x28517711L, 0xc20ad9f8L, 0xabcc5167L, - 0xccad925fL, - 0x4de81751L, 0x3830dc8eL, 0x379d5862L, - 0x9320f991L, - 0xea7a90c2L, 0xfb3e7bceL, 0x5121ce64L, - 0x774fbe32L, - 0xa8b6e37eL, 0xc3293d46L, 0x48de5369L, - 0x6413e680L, - 0xa2ae0810L, 0xdd6db224L, 0x69852dfdL, - 0x09072166L, - 0xb39a460aL, 0x6445c0ddL, 0x586cdecfL, - 0x1c20c8aeL, - 0x5bbef7ddL, 0x1b588d40L, 0xccd2017fL, - 0x6bb4e3bbL, - 0xdda26a7eL, 0x3a59ff45L, 0x3e350a44L, - 0xbcb4cdd5L, - 0x72eacea8L, 0xfa6484bbL, 0x8d6612aeL, - 0xbf3c6f47L, - 0xd29be463L, 0x542f5d9eL, 0xaec2771bL, - 0xf64e6370L, - 0x740e0d8dL, 0xe75b1357L, 0xf8721671L, - 0xaf537d5dL, - 0x4040cb08L, 0x4eb4e2ccL, 0x34d2466aL, - 0x0115af84L, - 0xe1b00428L, 0x95983a1dL, 0x06b89fb4L, - 0xce6ea048L, - 0x6f3f3b82L, 0x3520ab82L, 0x011a1d4bL, - 0x277227f8L, - 0x611560b1L, 0xe7933fdcL, 0xbb3a792bL, - 0x344525bdL, - 0xa08839e1L, 0x51ce794bL, 0x2f32c9b7L, - 0xa01fbac9L, - 0xe01cc87eL, 0xbcc7d1f6L, 0xcf0111c3L, - 0xa1e8aac7L, - 0x1a908749L, 0xd44fbd9aL, 0xd0dadecbL, - 0xd50ada38L, - 0x0339c32aL, 0xc6913667L, 0x8df9317cL, - 0xe0b12b4fL, - 0xf79e59b7L, 0x43f5bb3aL, 0xf2d519ffL, - 0x27d9459cL, - 0xbf97222cL, 0x15e6fc2aL, 0x0f91fc71L, - 0x9b941525L, - 0xfae59361L, 0xceb69cebL, 0xc2a86459L, - 0x12baa8d1L, - 0xb6c1075eL, 0xe3056a0cL, 0x10d25065L, - 0xcb03a442L, - 0xe0ec6e0eL, 0x1698db3bL, 0x4c98a0beL, - 0x3278e964L, - 0x9f1f9532L, 0xe0d392dfL, 0xd3a0342bL, - 0x8971f21eL, - 0x1b0a7441L, 0x4ba3348cL, 0xc5be7120L, - 0xc37632d8L, - 0xdf359f8dL, 0x9b992f2eL, 0xe60b6f47L, - 0x0fe3f11dL, - 0xe54cda54L, 0x1edad891L, 0xce6279cfL, - 0xcd3e7e6fL, - 0x1618b166L, 0xfd2c1d05L, 0x848fd2c5L, - 0xf6fb2299L, - 0xf523f357L, 0xa6327623L, 0x93a83531L, - 0x56cccd02L, - 0xacf08162L, 0x5a75ebb5L, 0x6e163697L, - 0x88d273ccL, - 0xde966292L, 0x81b949d0L, 0x4c50901bL, - 0x71c65614L, - 0xe6c6c7bdL, 0x327a140aL, 0x45e1d006L, - 0xc3f27b9aL, - 0xc9aa53fdL, 0x62a80f00L, 0xbb25bfe2L, - 0x35bdd2f6L, - 0x71126905L, 0xb2040222L, 0xb6cbcf7cL, - 0xcd769c2bL, - 0x53113ec0L, 0x1640e3d3L, 0x38abbd60L, - 0x2547adf0L, - 0xba38209cL, 0xf746ce76L, 0x77afa1c5L, - 0x20756060L, - 0x85cbfe4eL, 0x8ae88dd8L, 0x7aaaf9b0L, - 0x4cf9aa7eL, - 0x1948c25cL, 0x02fb8a8cL, 0x01c36ae4L, - 0xd6ebe1f9L, - 0x90d4f869L, 0xa65cdea0L, 0x3f09252dL, - 0xc208e69fL, - 0xb74e6132L, 0xce77e25bL, 0x578fdfe3L, - 0x3ac372e6L, - } -}; diff --git a/openssl/src/crypto/bf/bf_skey.c b/openssl/src/crypto/bf/bf_skey.c deleted file mode 100644 index 9728be297..000000000 --- a/openssl/src/crypto/bf/bf_skey.c +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * BF low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include "bf_local.h" -#include "bf_pi.h" - -void BF_set_key(BF_KEY *key, int len, const unsigned char *data) -{ - int i; - BF_LONG *p, ri, in[2]; - const unsigned char *d, *end; - - memcpy(key, &bf_init, sizeof(BF_KEY)); - p = key->P; - - if (len > ((BF_ROUNDS + 2) * 4)) - len = (BF_ROUNDS + 2) * 4; - - d = data; - end = &(data[len]); - for (i = 0; i < (BF_ROUNDS + 2); i++) { - ri = *(d++); - if (d >= end) - d = data; - - ri <<= 8; - ri |= *(d++); - if (d >= end) - d = data; - - ri <<= 8; - ri |= *(d++); - if (d >= end) - d = data; - - ri <<= 8; - ri |= *(d++); - if (d >= end) - d = data; - - p[i] ^= ri; - } - - in[0] = 0L; - in[1] = 0L; - for (i = 0; i < (BF_ROUNDS + 2); i += 2) { - BF_encrypt(in, key); - p[i] = in[0]; - p[i + 1] = in[1]; - } - - p = key->S; - for (i = 0; i < 4 * 256; i += 2) { - BF_encrypt(in, key); - p[i] = in[0]; - p[i + 1] = in[1]; - } -} diff --git a/openssl/src/crypto/bf/gen/linux_ia32/bf-586.S b/openssl/src/crypto/bf/gen/linux_ia32/bf-586.S deleted file mode 100644 index 801b28d56..000000000 --- a/openssl/src/crypto/bf/gen/linux_ia32/bf-586.S +++ /dev/null @@ -1,962 +0,0 @@ -.text -.globl BF_encrypt -.type BF_encrypt,@function -.align 16 -BF_encrypt: -.L_BF_encrypt_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - - pushl %ebp - pushl %ebx - movl 12(%esp),%ebx - movl 16(%esp),%ebp - pushl %esi - pushl %edi - - movl (%ebx),%edi - movl 4(%ebx),%esi - xorl %eax,%eax - movl (%ebp),%ebx - xorl %ecx,%ecx - xorl %ebx,%edi - - - movl 4(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 8(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 12(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 16(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 20(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 24(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 28(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 32(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 36(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 40(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 44(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 48(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 52(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 56(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 60(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 64(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - - movl 20(%esp),%eax - xorl %ebx,%edi - movl 68(%ebp),%edx - xorl %edx,%esi - movl %edi,4(%eax) - movl %esi,(%eax) - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.size BF_encrypt,.-.L_BF_encrypt_begin -.globl BF_decrypt -.type BF_decrypt,@function -.align 16 -BF_decrypt: -.L_BF_decrypt_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - - pushl %ebp - pushl %ebx - movl 12(%esp),%ebx - movl 16(%esp),%ebp - pushl %esi - pushl %edi - - movl (%ebx),%edi - movl 4(%ebx),%esi - xorl %eax,%eax - movl 68(%ebp),%ebx - xorl %ecx,%ecx - xorl %ebx,%edi - - - movl 64(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 60(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 56(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 52(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 48(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 44(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 40(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 36(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 32(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 28(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 24(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 20(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 16(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 12(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%edi - - - movl 8(%ebp),%edx - movl %edi,%ebx - xorl %edx,%esi - shrl $16,%ebx - movl %edi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - xorl %eax,%eax - xorl %ebx,%esi - - - movl 4(%ebp),%edx - movl %esi,%ebx - xorl %edx,%edi - shrl $16,%ebx - movl %esi,%edx - movb %bh,%al - andl $255,%ebx - movb %dh,%cl - andl $255,%edx - movl 72(%ebp,%eax,4),%eax - movl 1096(%ebp,%ebx,4),%ebx - addl %eax,%ebx - movl 2120(%ebp,%ecx,4),%eax - xorl %eax,%ebx - movl 3144(%ebp,%edx,4),%edx - addl %edx,%ebx - - movl 20(%esp),%eax - xorl %ebx,%edi - movl (%ebp),%edx - xorl %edx,%esi - movl %edi,4(%eax) - movl %esi,(%eax) - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.size BF_decrypt,.-.L_BF_decrypt_begin -.globl BF_cbc_encrypt -.type BF_cbc_encrypt,@function -.align 16 -BF_cbc_encrypt: -.L_BF_cbc_encrypt_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - - pushl %ebp - pushl %ebx - pushl %esi - pushl %edi - movl 28(%esp),%ebp - - movl 36(%esp),%ebx - movl (%ebx),%esi - movl 4(%ebx),%edi - pushl %edi - pushl %esi - pushl %edi - pushl %esi - movl %esp,%ebx - movl 36(%esp),%esi - movl 40(%esp),%edi - - movl 56(%esp),%ecx - - movl 48(%esp),%eax - pushl %eax - pushl %ebx - cmpl $0,%ecx - jz .L000decrypt - andl $4294967288,%ebp - movl 8(%esp),%eax - movl 12(%esp),%ebx - jz .L001encrypt_finish -.L002encrypt_loop: - movl (%esi),%ecx - movl 4(%esi),%edx - xorl %ecx,%eax - xorl %edx,%ebx - bswap %eax - bswap %ebx - movl %eax,8(%esp) - movl %ebx,12(%esp) - call .L_BF_encrypt_begin - movl 8(%esp),%eax - movl 12(%esp),%ebx - bswap %eax - bswap %ebx - movl %eax,(%edi) - movl %ebx,4(%edi) - addl $8,%esi - addl $8,%edi - subl $8,%ebp - jnz .L002encrypt_loop -.L001encrypt_finish: - movl 52(%esp),%ebp - andl $7,%ebp - jz .L003finish - call .L004PIC_point -.L004PIC_point: - popl %edx - leal .L005cbc_enc_jmp_table-.L004PIC_point(%edx),%ecx - movl (%ecx,%ebp,4),%ebp - addl %edx,%ebp - xorl %ecx,%ecx - xorl %edx,%edx - jmp *%ebp -.L006ej7: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movb 6(%esi),%dh - shll $8,%edx -.L007ej6: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movb 5(%esi),%dh -.L008ej5: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movb 4(%esi),%dl -.L009ej4: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movl (%esi),%ecx - jmp .L010ejend -.L011ej3: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movb 2(%esi),%ch - shll $8,%ecx -.L012ej2: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movb 1(%esi),%ch -.L013ej1: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movb (%esi),%cl -.L010ejend: - xorl %ecx,%eax - xorl %edx,%ebx - bswap %eax - bswap %ebx - movl %eax,8(%esp) - movl %ebx,12(%esp) - call .L_BF_encrypt_begin - movl 8(%esp),%eax - movl 12(%esp),%ebx - bswap %eax - bswap %ebx - movl %eax,(%edi) - movl %ebx,4(%edi) - jmp .L003finish -.L000decrypt: - andl $4294967288,%ebp - movl 16(%esp),%eax - movl 20(%esp),%ebx - jz .L014decrypt_finish -.L015decrypt_loop: - movl (%esi),%eax - movl 4(%esi),%ebx - bswap %eax - bswap %ebx - movl %eax,8(%esp) - movl %ebx,12(%esp) - call .L_BF_decrypt_begin - movl 8(%esp),%eax - movl 12(%esp),%ebx - bswap %eax - bswap %ebx - movl 16(%esp),%ecx - movl 20(%esp),%edx - xorl %eax,%ecx - xorl %ebx,%edx - movl (%esi),%eax - movl 4(%esi),%ebx - movl %ecx,(%edi) - movl %edx,4(%edi) - movl %eax,16(%esp) - movl %ebx,20(%esp) - addl $8,%esi - addl $8,%edi - subl $8,%ebp - jnz .L015decrypt_loop -.L014decrypt_finish: - movl 52(%esp),%ebp - andl $7,%ebp - jz .L003finish - movl (%esi),%eax - movl 4(%esi),%ebx - bswap %eax - bswap %ebx - movl %eax,8(%esp) - movl %ebx,12(%esp) - call .L_BF_decrypt_begin - movl 8(%esp),%eax - movl 12(%esp),%ebx - bswap %eax - bswap %ebx - movl 16(%esp),%ecx - movl 20(%esp),%edx - xorl %eax,%ecx - xorl %ebx,%edx - movl (%esi),%eax - movl 4(%esi),%ebx -.L016dj7: - rorl $16,%edx - movb %dl,6(%edi) - shrl $16,%edx -.L017dj6: - movb %dh,5(%edi) -.L018dj5: - movb %dl,4(%edi) -.L019dj4: - movl %ecx,(%edi) - jmp .L020djend -.L021dj3: - rorl $16,%ecx - movb %cl,2(%edi) - shll $16,%ecx -.L022dj2: - movb %ch,1(%esi) -.L023dj1: - movb %cl,(%esi) -.L020djend: - jmp .L003finish -.L003finish: - movl 60(%esp),%ecx - addl $24,%esp - movl %eax,(%ecx) - movl %ebx,4(%ecx) - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.align 64 -.L005cbc_enc_jmp_table: -.long 0 -.long .L013ej1-.L004PIC_point -.long .L012ej2-.L004PIC_point -.long .L011ej3-.L004PIC_point -.long .L009ej4-.L004PIC_point -.long .L008ej5-.L004PIC_point -.long .L007ej6-.L004PIC_point -.long .L006ej7-.L004PIC_point -.align 64 -.size BF_cbc_encrypt,.-.L_BF_cbc_encrypt_begin - - .section ".note.gnu.property", "a" - .p2align 2 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - .asciz "GNU" -1: - .p2align 2 - .long 0xc0000002 - .long 3f - 2f -2: - .long 3 -3: - .p2align 2 -4: diff --git a/openssl/src/crypto/bf/gen/windows_ia32/bf-586.asm b/openssl/src/crypto/bf/gen/windows_ia32/bf-586.asm deleted file mode 100644 index 55e52e3f5..000000000 --- a/openssl/src/crypto/bf/gen/windows_ia32/bf-586.asm +++ /dev/null @@ -1,932 +0,0 @@ - -%ifidn __OUTPUT_FORMAT__,obj -section code use32 class=code align=64 -%elifidn __OUTPUT_FORMAT__,win32 -$@feat.00 equ 1 -section .text code align=64 -%else -section .text code -%endif -global _BF_encrypt -align 16 -_BF_encrypt: -L$_BF_encrypt_begin: - ; - push ebp - push ebx - mov ebx,DWORD [12+esp] - mov ebp,DWORD [16+esp] - push esi - push edi - ; Load the 2 words - mov edi,DWORD [ebx] - mov esi,DWORD [4+ebx] - xor eax,eax - mov ebx,DWORD [ebp] - xor ecx,ecx - xor edi,ebx - ; - ; Round 0 - mov edx,DWORD [4+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 1 - mov edx,DWORD [8+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 2 - mov edx,DWORD [12+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 3 - mov edx,DWORD [16+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 4 - mov edx,DWORD [20+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 5 - mov edx,DWORD [24+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 6 - mov edx,DWORD [28+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 7 - mov edx,DWORD [32+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 8 - mov edx,DWORD [36+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 9 - mov edx,DWORD [40+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 10 - mov edx,DWORD [44+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 11 - mov edx,DWORD [48+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 12 - mov edx,DWORD [52+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 13 - mov edx,DWORD [56+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 14 - mov edx,DWORD [60+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 15 - mov edx,DWORD [64+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - ; Load parameter 0 (16) enc=1 - mov eax,DWORD [20+esp] - xor edi,ebx - mov edx,DWORD [68+ebp] - xor esi,edx - mov DWORD [4+eax],edi - mov DWORD [eax],esi - pop edi - pop esi - pop ebx - pop ebp - ret -global _BF_decrypt -align 16 -_BF_decrypt: -L$_BF_decrypt_begin: - ; - push ebp - push ebx - mov ebx,DWORD [12+esp] - mov ebp,DWORD [16+esp] - push esi - push edi - ; Load the 2 words - mov edi,DWORD [ebx] - mov esi,DWORD [4+ebx] - xor eax,eax - mov ebx,DWORD [68+ebp] - xor ecx,ecx - xor edi,ebx - ; - ; Round 16 - mov edx,DWORD [64+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 15 - mov edx,DWORD [60+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 14 - mov edx,DWORD [56+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 13 - mov edx,DWORD [52+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 12 - mov edx,DWORD [48+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 11 - mov edx,DWORD [44+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 10 - mov edx,DWORD [40+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 9 - mov edx,DWORD [36+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 8 - mov edx,DWORD [32+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 7 - mov edx,DWORD [28+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 6 - mov edx,DWORD [24+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 5 - mov edx,DWORD [20+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 4 - mov edx,DWORD [16+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 3 - mov edx,DWORD [12+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor edi,ebx - ; - ; Round 2 - mov edx,DWORD [8+ebp] - mov ebx,edi - xor esi,edx - shr ebx,16 - mov edx,edi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - xor eax,eax - xor esi,ebx - ; - ; Round 1 - mov edx,DWORD [4+ebp] - mov ebx,esi - xor edi,edx - shr ebx,16 - mov edx,esi - mov al,bh - and ebx,255 - mov cl,dh - and edx,255 - mov eax,DWORD [72+eax*4+ebp] - mov ebx,DWORD [1096+ebx*4+ebp] - add ebx,eax - mov eax,DWORD [2120+ecx*4+ebp] - xor ebx,eax - mov edx,DWORD [3144+edx*4+ebp] - add ebx,edx - ; Load parameter 0 (1) enc=0 - mov eax,DWORD [20+esp] - xor edi,ebx - mov edx,DWORD [ebp] - xor esi,edx - mov DWORD [4+eax],edi - mov DWORD [eax],esi - pop edi - pop esi - pop ebx - pop ebp - ret -global _BF_cbc_encrypt -align 16 -_BF_cbc_encrypt: -L$_BF_cbc_encrypt_begin: - ; - push ebp - push ebx - push esi - push edi - mov ebp,DWORD [28+esp] - ; getting iv ptr from parameter 4 - mov ebx,DWORD [36+esp] - mov esi,DWORD [ebx] - mov edi,DWORD [4+ebx] - push edi - push esi - push edi - push esi - mov ebx,esp - mov esi,DWORD [36+esp] - mov edi,DWORD [40+esp] - ; getting encrypt flag from parameter 5 - mov ecx,DWORD [56+esp] - ; get and push parameter 3 - mov eax,DWORD [48+esp] - push eax - push ebx - cmp ecx,0 - jz NEAR L$000decrypt - and ebp,4294967288 - mov eax,DWORD [8+esp] - mov ebx,DWORD [12+esp] - jz NEAR L$001encrypt_finish -L$002encrypt_loop: - mov ecx,DWORD [esi] - mov edx,DWORD [4+esi] - xor eax,ecx - xor ebx,edx - bswap eax - bswap ebx - mov DWORD [8+esp],eax - mov DWORD [12+esp],ebx - call L$_BF_encrypt_begin - mov eax,DWORD [8+esp] - mov ebx,DWORD [12+esp] - bswap eax - bswap ebx - mov DWORD [edi],eax - mov DWORD [4+edi],ebx - add esi,8 - add edi,8 - sub ebp,8 - jnz NEAR L$002encrypt_loop -L$001encrypt_finish: - mov ebp,DWORD [52+esp] - and ebp,7 - jz NEAR L$003finish - call L$004PIC_point -L$004PIC_point: - pop edx - lea ecx,[(L$005cbc_enc_jmp_table-L$004PIC_point)+edx] - mov ebp,DWORD [ebp*4+ecx] - add ebp,edx - xor ecx,ecx - xor edx,edx - jmp ebp -L$006ej7: - - - - - - mov dh,BYTE [6+esi] - shl edx,8 -L$007ej6: - - - - - - mov dh,BYTE [5+esi] -L$008ej5: - - - - - - mov dl,BYTE [4+esi] -L$009ej4: - - - - - - mov ecx,DWORD [esi] - jmp NEAR L$010ejend -L$011ej3: - - - - - - mov ch,BYTE [2+esi] - shl ecx,8 -L$012ej2: - - - - - - mov ch,BYTE [1+esi] -L$013ej1: - - - - - - mov cl,BYTE [esi] -L$010ejend: - xor eax,ecx - xor ebx,edx - bswap eax - bswap ebx - mov DWORD [8+esp],eax - mov DWORD [12+esp],ebx - call L$_BF_encrypt_begin - mov eax,DWORD [8+esp] - mov ebx,DWORD [12+esp] - bswap eax - bswap ebx - mov DWORD [edi],eax - mov DWORD [4+edi],ebx - jmp NEAR L$003finish -L$000decrypt: - and ebp,4294967288 - mov eax,DWORD [16+esp] - mov ebx,DWORD [20+esp] - jz NEAR L$014decrypt_finish -L$015decrypt_loop: - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - bswap eax - bswap ebx - mov DWORD [8+esp],eax - mov DWORD [12+esp],ebx - call L$_BF_decrypt_begin - mov eax,DWORD [8+esp] - mov ebx,DWORD [12+esp] - bswap eax - bswap ebx - mov ecx,DWORD [16+esp] - mov edx,DWORD [20+esp] - xor ecx,eax - xor edx,ebx - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov DWORD [edi],ecx - mov DWORD [4+edi],edx - mov DWORD [16+esp],eax - mov DWORD [20+esp],ebx - add esi,8 - add edi,8 - sub ebp,8 - jnz NEAR L$015decrypt_loop -L$014decrypt_finish: - mov ebp,DWORD [52+esp] - and ebp,7 - jz NEAR L$003finish - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - bswap eax - bswap ebx - mov DWORD [8+esp],eax - mov DWORD [12+esp],ebx - call L$_BF_decrypt_begin - mov eax,DWORD [8+esp] - mov ebx,DWORD [12+esp] - bswap eax - bswap ebx - mov ecx,DWORD [16+esp] - mov edx,DWORD [20+esp] - xor ecx,eax - xor edx,ebx - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] -L$016dj7: - ror edx,16 - mov BYTE [6+edi],dl - shr edx,16 -L$017dj6: - mov BYTE [5+edi],dh -L$018dj5: - mov BYTE [4+edi],dl -L$019dj4: - mov DWORD [edi],ecx - jmp NEAR L$020djend -L$021dj3: - ror ecx,16 - mov BYTE [2+edi],cl - shl ecx,16 -L$022dj2: - mov BYTE [1+esi],ch -L$023dj1: - mov BYTE [esi],cl -L$020djend: - jmp NEAR L$003finish -L$003finish: - mov ecx,DWORD [60+esp] - add esp,24 - mov DWORD [ecx],eax - mov DWORD [4+ecx],ebx - pop edi - pop esi - pop ebx - pop ebp - ret -align 64 -L$005cbc_enc_jmp_table: -dd 0 -dd L$013ej1-L$004PIC_point -dd L$012ej2-L$004PIC_point -dd L$011ej3-L$004PIC_point -dd L$009ej4-L$004PIC_point -dd L$008ej5-L$004PIC_point -dd L$007ej6-L$004PIC_point -dd L$006ej7-L$004PIC_point -align 64 diff --git a/openssl/src/crypto/bio/bf_buff.c b/openssl/src/crypto/bio/bf_buff.c index 737910cc7..cfed63bd7 100644 --- a/openssl/src/crypto/bio/bf_buff.c +++ b/openssl/src/crypto/bio/bf_buff.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -291,7 +291,7 @@ static long buffer_ctrl(BIO *b, int cmd, long num, void *ptr) return 0; p1 = OPENSSL_malloc((size_t)num); if (p1 == NULL) - return 0; + goto malloc_error; OPENSSL_free(ctx->ibuf); ctx->ibuf = p1; } @@ -322,14 +322,14 @@ static long buffer_ctrl(BIO *b, int cmd, long num, void *ptr) return 0; p1 = OPENSSL_malloc((size_t)num); if (p1 == NULL) - return 0; + goto malloc_error; } if ((obs > DEFAULT_BUFFER_SIZE) && (obs != ctx->obuf_size)) { p2 = OPENSSL_malloc((size_t)num); if (p2 == NULL) { if (p1 != ctx->ibuf) OPENSSL_free(p1); - return 0; + goto malloc_error; } } if (ctx->ibuf != p1) { @@ -360,7 +360,6 @@ static long buffer_ctrl(BIO *b, int cmd, long num, void *ptr) return 0; if (ctx->obuf_len <= 0) { ret = BIO_ctrl(b->next_bio, cmd, num, ptr); - BIO_copy_next_retry(b); break; } @@ -381,12 +380,11 @@ static long buffer_ctrl(BIO *b, int cmd, long num, void *ptr) } } ret = BIO_ctrl(b->next_bio, cmd, num, ptr); - BIO_copy_next_retry(b); break; case BIO_CTRL_DUP: dbio = (BIO *)ptr; - if (BIO_set_read_buffer_size(dbio, ctx->ibuf_size) <= 0 || - BIO_set_write_buffer_size(dbio, ctx->obuf_size) <= 0) + if (!BIO_set_read_buffer_size(dbio, ctx->ibuf_size) || + !BIO_set_write_buffer_size(dbio, ctx->obuf_size)) ret = 0; break; case BIO_CTRL_PEEK: @@ -407,6 +405,9 @@ static long buffer_ctrl(BIO *b, int cmd, long num, void *ptr) break; } return ret; + malloc_error: + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); + return 0; } static long buffer_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp) diff --git a/openssl/src/crypto/bio/bf_lbuf.c b/openssl/src/crypto/bio/bf_lbuf.c index eed3dc463..73f121698 100644 --- a/openssl/src/crypto/bio/bf_lbuf.c +++ b/openssl/src/crypto/bio/bf_lbuf.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -57,10 +57,13 @@ static int linebuffer_new(BIO *bi) { BIO_LINEBUFFER_CTX *ctx; - if ((ctx = OPENSSL_malloc(sizeof(*ctx))) == NULL) + if ((ctx = OPENSSL_malloc(sizeof(*ctx))) == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } ctx->obuf = OPENSSL_malloc(DEFAULT_LINEBUFFER_SIZE); if (ctx->obuf == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); OPENSSL_free(ctx); return 0; } @@ -234,7 +237,7 @@ static long linebuffer_ctrl(BIO *b, int cmd, long num, void *ptr) if ((obs > DEFAULT_LINEBUFFER_SIZE) && (obs != ctx->obuf_size)) { p = OPENSSL_malloc((size_t)obs); if (p == NULL) - return 0; + goto malloc_error; } if (ctx->obuf != p) { if (ctx->obuf_len > obs) { @@ -259,7 +262,6 @@ static long linebuffer_ctrl(BIO *b, int cmd, long num, void *ptr) return 0; if (ctx->obuf_len <= 0) { ret = BIO_ctrl(b->next_bio, cmd, num, ptr); - BIO_copy_next_retry(b); break; } @@ -279,11 +281,10 @@ static long linebuffer_ctrl(BIO *b, int cmd, long num, void *ptr) } } ret = BIO_ctrl(b->next_bio, cmd, num, ptr); - BIO_copy_next_retry(b); break; case BIO_CTRL_DUP: dbio = (BIO *)ptr; - if (BIO_set_write_buffer_size(dbio, ctx->obuf_size) <= 0) + if (!BIO_set_write_buffer_size(dbio, ctx->obuf_size)) ret = 0; break; default: @@ -293,6 +294,9 @@ static long linebuffer_ctrl(BIO *b, int cmd, long num, void *ptr) break; } return ret; + malloc_error: + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); + return 0; } static long linebuffer_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp) diff --git a/openssl/src/crypto/bio/bf_nbio.c b/openssl/src/crypto/bio/bf_nbio.c index 01138729b..f9ea1730b 100644 --- a/openssl/src/crypto/bio/bf_nbio.c +++ b/openssl/src/crypto/bio/bf_nbio.c @@ -55,8 +55,10 @@ static int nbiof_new(BIO *bi) { NBIO_TEST *nt; - if ((nt = OPENSSL_zalloc(sizeof(*nt))) == NULL) + if ((nt = OPENSSL_zalloc(sizeof(*nt))) == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } nt->lrn = -1; nt->lwn = -1; bi->ptr = (char *)nt; diff --git a/openssl/src/crypto/bio/bio_addr.c b/openssl/src/crypto/bio/bio_addr.c index 0a64d0749..dd4f1e54e 100644 --- a/openssl/src/crypto/bio/bio_addr.c +++ b/openssl/src/crypto/bio/bio_addr.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -53,8 +53,10 @@ BIO_ADDR *BIO_ADDR_new(void) { BIO_ADDR *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return NULL; + } ret->sa.sa_family = AF_UNSPEC; return ret; @@ -65,33 +67,6 @@ void BIO_ADDR_free(BIO_ADDR *ap) OPENSSL_free(ap); } -int BIO_ADDR_copy(BIO_ADDR *dst, const BIO_ADDR *src) -{ - if (dst == NULL || src == NULL) - return 0; - - if (src->sa.sa_family == AF_UNSPEC) { - BIO_ADDR_clear(dst); - return 1; - } - - return BIO_ADDR_make(dst, &src->sa); -} - -BIO_ADDR *BIO_ADDR_dup(const BIO_ADDR *ap) -{ - BIO_ADDR *ret = NULL; - - if (ap != NULL) { - ret = BIO_ADDR_new(); - if (ret != NULL && !BIO_ADDR_copy(ret, ap)) { - BIO_ADDR_free(ret); - ret = NULL; - } - } - return ret; -} - void BIO_ADDR_clear(BIO_ADDR *ap) { memset(ap, 0, sizeof(*ap)); @@ -108,13 +83,13 @@ int BIO_ADDR_make(BIO_ADDR *ap, const struct sockaddr *sa) memcpy(&(ap->s_in), sa, sizeof(struct sockaddr_in)); return 1; } -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 if (sa->sa_family == AF_INET6) { memcpy(&(ap->s_in6), sa, sizeof(struct sockaddr_in6)); return 1; } #endif -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX if (sa->sa_family == AF_UNIX) { memcpy(&(ap->s_un), sa, sizeof(struct sockaddr_un)); return 1; @@ -128,7 +103,7 @@ int BIO_ADDR_rawmake(BIO_ADDR *ap, int family, const void *where, size_t wherelen, unsigned short port) { -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX if (family == AF_UNIX) { if (wherelen + 1 > sizeof(ap->s_un.sun_path)) return 0; @@ -147,7 +122,7 @@ int BIO_ADDR_rawmake(BIO_ADDR *ap, int family, ap->s_in.sin_addr = *(struct in_addr *)where; return 1; } -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 if (family == AF_INET6) { if (wherelen != sizeof(struct in6_addr)) return 0; @@ -176,13 +151,13 @@ int BIO_ADDR_rawaddress(const BIO_ADDR *ap, void *p, size_t *l) len = sizeof(ap->s_in.sin_addr); addrptr = &ap->s_in.sin_addr; } -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 else if (ap->sa.sa_family == AF_INET6) { len = sizeof(ap->s_in6.sin6_addr); addrptr = &ap->s_in6.sin6_addr; } #endif -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX else if (ap->sa.sa_family == AF_UNIX) { len = strlen(ap->s_un.sun_path); addrptr = &ap->s_un.sun_path; @@ -205,7 +180,7 @@ unsigned short BIO_ADDR_rawport(const BIO_ADDR *ap) { if (ap->sa.sa_family == AF_INET) return ap->s_in.sin_port; -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 if (ap->sa.sa_family == AF_INET6) return ap->s_in6.sin6_port; #endif @@ -218,7 +193,7 @@ unsigned short BIO_ADDR_rawport(const BIO_ADDR *ap) * @numeric: 0 if actual names should be returned, 1 if the numeric * representation should be returned. * @hostname: a pointer to a pointer to a memory area to store the - * hostname or numeric representation. Unused if NULL. + * host name or numeric representation. Unused if NULL. * @service: a pointer to a pointer to a memory area to store the * service name or numeric representation. Unused if NULL. * @@ -256,7 +231,7 @@ static int addr_strings(const BIO_ADDR *ap, int numeric, return 0; } - /* VMS getnameinfo() has a bug, it doesn't fill in serv, which + /* Obsolete: VMS getnameinfo() has a bug, it doesn't fill in serv, which * leaves it with whatever garbage that happens to be there. * However, we initialise serv with the empty string (serv[0] * is therefore NUL), so it gets real easy to detect when things @@ -292,6 +267,7 @@ static int addr_strings(const BIO_ADDR *ap, int numeric, OPENSSL_free(*service); *service = NULL; } + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; } @@ -320,7 +296,7 @@ char *BIO_ADDR_service_string(const BIO_ADDR *ap, int numeric) char *BIO_ADDR_path_string(const BIO_ADDR *ap) { -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX if (ap->sa.sa_family == AF_UNIX) return OPENSSL_strdup(ap->s_un.sun_path); #endif @@ -358,11 +334,11 @@ socklen_t BIO_ADDR_sockaddr_size(const BIO_ADDR *ap) { if (ap->sa.sa_family == AF_INET) return sizeof(ap->s_in); -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 if (ap->sa.sa_family == AF_INET6) return sizeof(ap->s_in6); #endif -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX if (ap->sa.sa_family == AF_UNIX) return sizeof(ap->s_un); #endif @@ -402,7 +378,7 @@ int BIO_ADDRINFO_protocol(const BIO_ADDRINFO *bai) if (bai->bai_protocol != 0) return bai->bai_protocol; -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX if (bai->bai_family == AF_UNIX) return 0; #endif @@ -454,7 +430,7 @@ void BIO_ADDRINFO_free(BIO_ADDRINFO *bai) return; #ifdef AI_PASSIVE -# ifndef OPENSSL_NO_UNIX_SOCK +# ifdef AF_UNIX # define _cond bai->bai_family != AF_UNIX # else # define _cond 1 @@ -562,7 +538,7 @@ int BIO_parse_hostserv(const char *hostserv, char **host, char **service, } else { *host = OPENSSL_strndup(h, hl); if (*host == NULL) - return 0; + goto memerr; } } if (p != NULL && service != NULL) { @@ -572,7 +548,7 @@ int BIO_parse_hostserv(const char *hostserv, char **host, char **service, } else { *service = OPENSSL_strndup(p, pl); if (*service == NULL) - return 0; + goto memerr; } } @@ -583,6 +559,9 @@ int BIO_parse_hostserv(const char *hostserv, char **host, char **service, spec_err: ERR_raise(ERR_LIB_BIO, BIO_R_MALFORMED_HOST_OR_SERVICE); return 0; + memerr: + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); + return 0; } /* addrinfo_wrap is used to build our own addrinfo "chain". @@ -599,8 +578,10 @@ static int addrinfo_wrap(int family, int socktype, unsigned short port, BIO_ADDRINFO **bai) { - if ((*bai = OPENSSL_zalloc(sizeof(**bai))) == NULL) + if ((*bai = OPENSSL_zalloc(sizeof(**bai))) == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } (*bai)->bai_family = family; (*bai)->bai_socktype = socktype; @@ -608,7 +589,7 @@ static int addrinfo_wrap(int family, int socktype, (*bai)->bai_protocol = IPPROTO_TCP; if (socktype == SOCK_DGRAM) (*bai)->bai_protocol = IPPROTO_UDP; -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX if (family == AF_UNIX) (*bai)->bai_protocol = 0; #endif @@ -673,12 +654,12 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, { int ret = 0; /* Assume failure */ - switch (family) { + switch(family) { case AF_INET: -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 case AF_INET6: #endif -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX case AF_UNIX: #endif #ifdef AF_UNSPEC @@ -690,12 +671,12 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, return 0; } -#ifndef OPENSSL_NO_UNIX_SOCK +#ifdef AF_UNIX if (family == AF_UNIX) { if (addrinfo_wrap(family, socktype, host, strlen(host), 0, res)) return 1; else - ERR_raise(ERR_LIB_BIO, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; } #endif @@ -739,8 +720,7 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, # endif # ifdef EAI_MEMORY case EAI_MEMORY: - ERR_raise_data(ERR_LIB_BIO, ERR_R_SYS_LIB, - gai_strerror(old_ret ? old_ret : gai_ret)); + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); break; # endif case 0: @@ -762,15 +742,6 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, } else { #endif const struct hostent *he; -/* - * Because struct hostent is defined for 32-bit pointers only with - * VMS C, we need to make sure that '&he_fallback_address' and - * '&he_fallback_addresses' are 32-bit pointers - */ -#if defined(OPENSSL_SYS_VMS) && defined(__DECC) -# pragma pointer_size save -# pragma pointer_size 32 -#endif /* Windows doesn't seem to have in_addr_t */ #if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_MSDOS) static uint32_t he_fallback_address; @@ -784,9 +755,6 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, static const struct hostent he_fallback = { NULL, NULL, AF_INET, sizeof(he_fallback_address), (char **)&he_fallback_addresses }; -#if defined(OPENSSL_SYS_VMS) && defined(__DECC) -# pragma pointer_size restore -#endif struct servent *se; /* Apparently, on WIN64, s_proto and s_port have traded places... */ @@ -797,8 +765,7 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, #endif if (!RUN_ONCE(&bio_lookup_init, do_bio_lookup_init)) { - /* Should this be raised inside do_bio_lookup_init()? */ - ERR_raise(ERR_LIB_BIO, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); ret = 0; goto err; } @@ -810,7 +777,7 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, he_fallback_address = INADDR_ANY; if (host == NULL) { he = &he_fallback; - switch (lookup_type) { + switch(lookup_type) { case BIO_LOOKUP_CLIENT: he_fallback_address = INADDR_LOOPBACK; break; @@ -864,19 +831,7 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, } else { char *endp = NULL; long portnum = strtol(service, &endp, 10); - -/* - * Because struct servent is defined for 32-bit pointers only with - * VMS C, we need to make sure that 'proto' is a 32-bit pointer. - */ -#if defined(OPENSSL_SYS_VMS) && defined(__DECC) -# pragma pointer_size save -# pragma pointer_size 32 -#endif char *proto = NULL; -#if defined(OPENSSL_SYS_VMS) && defined(__DECC) -# pragma pointer_size restore -#endif switch (socktype) { case SOCK_STREAM: @@ -909,41 +864,29 @@ int BIO_lookup_ex(const char *host, const char *service, int lookup_type, *res = NULL; { -/* - * Because hostent::h_addr_list is an array of 32-bit pointers with VMS C, - * we must make sure our iterator designates the same element type, hence - * the pointer size dance. - */ -#if defined(OPENSSL_SYS_VMS) && defined(__DECC) -# pragma pointer_size save -# pragma pointer_size 32 -#endif char **addrlistp; -#if defined(OPENSSL_SYS_VMS) && defined(__DECC) -# pragma pointer_size restore -#endif size_t addresses; BIO_ADDRINFO *tmp_bai = NULL; /* The easiest way to create a linked list from an array is to start from the back */ - for (addrlistp = he->h_addr_list; *addrlistp != NULL; - addrlistp++) + for(addrlistp = he->h_addr_list; *addrlistp != NULL; + addrlistp++) ; - for (addresses = addrlistp - he->h_addr_list; - addrlistp--, addresses-- > 0; ) { + for(addresses = addrlistp - he->h_addr_list; + addrlistp--, addresses-- > 0; ) { if (!addrinfo_wrap(he->h_addrtype, socktype, *addrlistp, he->h_length, se->s_port, &tmp_bai)) - goto addrinfo_wrap_err; + goto addrinfo_malloc_err; tmp_bai->bai_next = *res; *res = tmp_bai; continue; - addrinfo_wrap_err: + addrinfo_malloc_err: BIO_ADDRINFO_free(*res); *res = NULL; - ERR_raise(ERR_LIB_BIO, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); ret = 0; goto err; } diff --git a/openssl/src/crypto/bio/bio_cb.c b/openssl/src/crypto/bio/bio_cb.c index 8e4f79ea3..522a05369 100644 --- a/openssl/src/crypto/bio/bio_cb.c +++ b/openssl/src/crypto/bio/bio_cb.c @@ -24,8 +24,6 @@ long BIO_debug_callback_ex(BIO *bio, int cmd, const char *argp, size_t len, char *p; int left; size_t l = 0; - BIO_MMSG_CB_ARGS *args; - long ret_ = ret; if (processed != NULL) l = *processed; @@ -71,16 +69,6 @@ long BIO_debug_callback_ex(BIO *bio, int cmd, const char *argp, size_t len, BIO_snprintf(p, left, "ctrl(%d) - %s\n", argi, bio->method->name); break; - case BIO_CB_RECVMMSG: - args = (BIO_MMSG_CB_ARGS *)argp; - BIO_snprintf(p, left, "recvmmsg(%zu) - %s", - args->num_msg, bio->method->name); - break; - case BIO_CB_SENDMMSG: - args = (BIO_MMSG_CB_ARGS *)argp; - BIO_snprintf(p, left, "sendmmsg(%zu) - %s", - args->num_msg, bio->method->name); - break; case BIO_CB_RETURN | BIO_CB_READ: BIO_snprintf(p, left, "read return %d processed: %zu\n", ret, l); break; @@ -96,14 +84,6 @@ long BIO_debug_callback_ex(BIO *bio, int cmd, const char *argp, size_t len, case BIO_CB_RETURN | BIO_CB_CTRL: BIO_snprintf(p, left, "ctrl return %d\n", ret); break; - case BIO_CB_RETURN | BIO_CB_RECVMMSG: - BIO_snprintf(p, left, "recvmmsg processed: %zu\n", len); - ret_ = (long)len; - break; - case BIO_CB_RETURN | BIO_CB_SENDMMSG: - BIO_snprintf(p, left, "sendmmsg processed: %zu\n", len); - ret_ = (long)len; - break; default: BIO_snprintf(p, left, "bio callback - unknown type (%d)\n", cmd); break; @@ -116,7 +96,7 @@ long BIO_debug_callback_ex(BIO *bio, int cmd, const char *argp, size_t len, else fputs(buf, stderr); #endif - return ret_; + return ret; } #ifndef OPENSSL_NO_DEPRECATED_3_0 diff --git a/openssl/src/crypto/bio/bio_dump.c b/openssl/src/crypto/bio/bio_dump.c index 40c18410e..c453da626 100644 --- a/openssl/src/crypto/bio/bio_dump.c +++ b/openssl/src/crypto/bio/bio_dump.c @@ -141,10 +141,9 @@ int BIO_hex_string(BIO *out, int indent, int width, const void *data, BIO_printf(out, "%02X:", d[i]); - if (++j >= width) { - j = 0; + j = (j + 1) % width; + if (!j) BIO_printf(out, "\n"); - } } if (i && !j) diff --git a/openssl/src/crypto/bio/bio_err.c b/openssl/src/crypto/bio/bio_err.c index 6fe295ee5..7a36c6114 100644 --- a/openssl/src/crypto/bio/bio_err.c +++ b/openssl/src/crypto/bio/bio_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -46,10 +46,6 @@ static const ERR_STRING_DATA BIO_str_reasons[] = { "no hostname or service specified"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_NO_PORT_DEFINED), "no port defined"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_NO_SUCH_FILE), "no such file"}, - {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_PORT_MISMATCH), "port mismatch"}, - {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_TFO_DISABLED), "tfo disabled"}, - {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_TFO_NO_KERNEL_SUPPORT), - "tfo no kernel support"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_TRANSFER_ERROR), "transfer error"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_TRANSFER_TIMEOUT), "transfer timeout"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNABLE_TO_BIND_SOCKET), @@ -63,7 +59,6 @@ static const ERR_STRING_DATA BIO_str_reasons[] = { {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNABLE_TO_NODELAY), "unable to nodelay"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNABLE_TO_REUSEADDR), "unable to reuseaddr"}, - {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNABLE_TO_TFO), "unable to tfo"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNAVAILABLE_IP_FAMILY), "unavailable ip family"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_UNINITIALIZED), "uninitialized"}, @@ -76,14 +71,6 @@ static const ERR_STRING_DATA BIO_str_reasons[] = { {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_WRITE_TO_READ_ONLY_BIO), "write to read only BIO"}, {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_WSASTARTUP), "WSAStartup"}, - {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_LOCAL_ADDR_NOT_AVAILABLE), - "local address not available"}, - {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_PEER_ADDR_NOT_AVAILABLE), - "peer address not available"}, - {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_NON_FATAL), - "non-fatal or transient error"}, - {ERR_PACK(ERR_LIB_BIO, 0, BIO_R_PORT_MISMATCH), - "port mismatch"}, {0, NULL} }; @@ -97,18 +84,3 @@ int ossl_err_load_BIO_strings(void) #endif return 1; } - -#ifndef OPENSSL_NO_SOCK - -int BIO_err_is_non_fatal(unsigned int errcode) -{ - if (ERR_SYSTEM_ERROR(errcode)) - return BIO_sock_non_fatal_error(ERR_GET_REASON(errcode)); - else if (ERR_GET_LIB(errcode) == ERR_LIB_BIO - && ERR_GET_REASON(errcode) == BIO_R_NON_FATAL) - return 1; - else - return 0; -} - -#endif diff --git a/openssl/src/crypto/bio/bio_lib.c b/openssl/src/crypto/bio/bio_lib.c index 272189a9a..b5454f14b 100644 --- a/openssl/src/crypto/bio/bio_lib.c +++ b/openssl/src/crypto/bio/bio_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,6 @@ #include #include #include -#include "internal/numbers.h" #include "bio_local.h" /* @@ -82,22 +81,30 @@ BIO *BIO_new_ex(OSSL_LIB_CTX *libctx, const BIO_METHOD *method) { BIO *bio = OPENSSL_zalloc(sizeof(*bio)); - if (bio == NULL) + if (bio == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return NULL; + } bio->libctx = libctx; bio->method = method; bio->shutdown = 1; + bio->references = 1; - if (!CRYPTO_NEW_REF(&bio->references, 1)) + if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_BIO, bio, &bio->ex_data)) goto err; - if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_BIO, bio, &bio->ex_data)) + bio->lock = CRYPTO_THREAD_lock_new(); + if (bio->lock == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); + CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, bio, &bio->ex_data); goto err; + } if (method->create != NULL && !method->create(bio)) { ERR_raise(ERR_LIB_BIO, ERR_R_INIT_FAIL); CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, bio, &bio->ex_data); + CRYPTO_THREAD_lock_free(bio->lock); goto err; } if (method->create == NULL) @@ -106,7 +113,6 @@ BIO *BIO_new_ex(OSSL_LIB_CTX *libctx, const BIO_METHOD *method) return bio; err: - CRYPTO_FREE_REF(&bio->references); OPENSSL_free(bio); return NULL; } @@ -123,7 +129,7 @@ int BIO_free(BIO *a) if (a == NULL) return 0; - if (CRYPTO_DOWN_REF(&a->references, &ret) <= 0) + if (CRYPTO_DOWN_REF(&a->references, &ret, a->lock) <= 0) return 0; REF_PRINT_COUNT("BIO", a); @@ -142,7 +148,7 @@ int BIO_free(BIO *a) CRYPTO_free_ex_data(CRYPTO_EX_INDEX_BIO, a, &a->ex_data); - CRYPTO_FREE_REF(&a->references); + CRYPTO_THREAD_lock_free(a->lock); OPENSSL_free(a); @@ -188,7 +194,7 @@ int BIO_up_ref(BIO *a) { int i; - if (CRYPTO_UP_REF(&a->references, &i) <= 0) + if (CRYPTO_UP_REF(&a->references, &i, a->lock) <= 0) return 0; REF_PRINT_COUNT("BIO", a); @@ -391,110 +397,6 @@ int BIO_write_ex(BIO *b, const void *data, size_t dlen, size_t *written) || (b != NULL && dlen == 0); /* order is important for *written */ } -int BIO_sendmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, uint64_t flags, - size_t *msgs_processed) -{ - size_t ret; - BIO_MMSG_CB_ARGS args; - - if (b == NULL) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - if (b->method == NULL || b->method->bsendmmsg == NULL) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, BIO_R_UNSUPPORTED_METHOD); - return 0; - } - - if (HAS_CALLBACK(b)) { - args.msg = msg; - args.stride = stride; - args.num_msg = num_msg; - args.flags = flags; - args.msgs_processed = msgs_processed; - - ret = (size_t)bio_call_callback(b, BIO_CB_SENDMMSG, (void *)&args, - 0, 0, 0, 1, NULL); - if (ret <= 0) - return 0; - } - - if (!b->init) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, BIO_R_UNINITIALIZED); - return 0; - } - - ret = b->method->bsendmmsg(b, msg, stride, num_msg, flags, msgs_processed); - - if (HAS_CALLBACK(b)) - ret = (size_t)bio_call_callback(b, BIO_CB_SENDMMSG | BIO_CB_RETURN, - (void *)&args, ret, 0, 0, ret, NULL); - - return ret; -} - -int BIO_recvmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, uint64_t flags, - size_t *msgs_processed) -{ - size_t ret; - BIO_MMSG_CB_ARGS args; - - if (b == NULL) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - if (b->method == NULL || b->method->brecvmmsg == NULL) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, BIO_R_UNSUPPORTED_METHOD); - return 0; - } - - if (HAS_CALLBACK(b)) { - args.msg = msg; - args.stride = stride; - args.num_msg = num_msg; - args.flags = flags; - args.msgs_processed = msgs_processed; - - ret = bio_call_callback(b, BIO_CB_RECVMMSG, (void *)&args, - 0, 0, 0, 1, NULL); - if (ret <= 0) - return 0; - } - - if (!b->init) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, BIO_R_UNINITIALIZED); - return 0; - } - - ret = b->method->brecvmmsg(b, msg, stride, num_msg, flags, msgs_processed); - - if (HAS_CALLBACK(b)) - ret = (size_t)bio_call_callback(b, BIO_CB_RECVMMSG | BIO_CB_RETURN, - (void *)&args, ret, 0, 0, ret, NULL); - - return ret; -} - -int BIO_get_rpoll_descriptor(BIO *b, BIO_POLL_DESCRIPTOR *desc) -{ - return BIO_ctrl(b, BIO_CTRL_GET_RPOLL_DESCRIPTOR, 0, desc); -} - -int BIO_get_wpoll_descriptor(BIO *b, BIO_POLL_DESCRIPTOR *desc) -{ - return BIO_ctrl(b, BIO_CTRL_GET_WPOLL_DESCRIPTOR, 0, desc); -} - int BIO_puts(BIO *b, const char *buf) { int ret; @@ -718,28 +620,12 @@ long BIO_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp) */ size_t BIO_ctrl_pending(BIO *bio) { - long ret = BIO_ctrl(bio, BIO_CTRL_PENDING, 0, NULL); - - if (ret < 0) - ret = 0; -#if LONG_MAX > SIZE_MAX - if (ret > SIZE_MAX) - ret = SIZE_MAX; -#endif - return (size_t)ret; + return BIO_ctrl(bio, BIO_CTRL_PENDING, 0, NULL); } size_t BIO_ctrl_wpending(BIO *bio) { - long ret = BIO_ctrl(bio, BIO_CTRL_WPENDING, 0, NULL); - - if (ret < 0) - ret = 0; -#if LONG_MAX > SIZE_MAX - if (ret > SIZE_MAX) - ret = SIZE_MAX; -#endif - return (size_t)ret; + return BIO_ctrl(bio, BIO_CTRL_WPENDING, 0, NULL); } /* put the 'bio' on the end of b's list of operators */ @@ -817,7 +703,7 @@ BIO *BIO_find_type(BIO *bio, int type) ERR_raise(ERR_LIB_BIO, ERR_R_PASSED_NULL_PARAMETER); return NULL; } - mask = type & BIO_TYPE_MASK; + mask = type & 0xff; do { if (bio->method != NULL) { mt = bio->method->type; @@ -853,7 +739,7 @@ void BIO_free_all(BIO *bio) while (bio != NULL) { b = bio; - CRYPTO_GET_REF(&b->references, &ref); + ref = b->references; bio = bio->next_bio; BIO_free(b); /* Since ref count > 1, don't free anyone else. */ @@ -881,7 +767,7 @@ BIO *BIO_dup_chain(BIO *in) /* This will let SSL_s_sock() work with stdin/stdout */ new_bio->num = bio->num; - if (BIO_dup_state(bio, (char *)new_bio) <= 0) { + if (!BIO_dup_state(bio, (char *)new_bio)) { BIO_free(new_bio); goto err; } @@ -950,10 +836,11 @@ void bio_cleanup(void) CRYPTO_THREAD_lock_free(bio_lookup_lock); bio_lookup_lock = NULL; #endif - CRYPTO_FREE_REF(&bio_type_count); + CRYPTO_THREAD_lock_free(bio_type_lock); + bio_type_lock = NULL; } -/* Internal variant of the below BIO_wait() not calling ERR_raise(...) */ +/* Internal variant of the below BIO_wait() not calling BIOerr() */ static int bio_wait(BIO *bio, time_t max_time, unsigned int nap_milliseconds) { #ifndef OPENSSL_NO_SOCK @@ -965,12 +852,8 @@ static int bio_wait(BIO *bio, time_t max_time, unsigned int nap_milliseconds) return 1; #ifndef OPENSSL_NO_SOCK - if (BIO_get_fd(bio, &fd) > 0) { - int ret = BIO_socket_wait(fd, BIO_should_read(bio), max_time); - - if (ret != -1) - return ret; - } + if (BIO_get_fd(bio, &fd) > 0 && fd < FD_SETSIZE) + return BIO_socket_wait(fd, BIO_should_read(bio), max_time); #endif /* fall back to polling since no sockets are available */ @@ -986,7 +869,7 @@ static int bio_wait(BIO *bio, time_t max_time, unsigned int nap_milliseconds) if ((unsigned long)sec_diff * 1000 < nap_milliseconds) nap_milliseconds = (unsigned int)sec_diff * 1000; } - OSSL_sleep(nap_milliseconds); + ossl_sleep(nap_milliseconds); return 1; } @@ -995,7 +878,7 @@ static int bio_wait(BIO *bio, time_t max_time, unsigned int nap_milliseconds) * Succeed immediately if max_time == 0. * If sockets are not available support polling: succeed after waiting at most * the number of nap_milliseconds in order to avoid a tight busy loop. - * Call ERR_raise(ERR_LIB_BIO, ...) on timeout or error. + * Call BIOerr(...) on timeout or error. * Returns -1 on error, 0 on timeout, and 1 on success. */ int BIO_wait(BIO *bio, time_t max_time, unsigned int nap_milliseconds) diff --git a/openssl/src/crypto/bio/bio_local.h b/openssl/src/crypto/bio/bio_local.h index 05954f85b..e702423bc 100644 --- a/openssl/src/crypto/bio/bio_local.h +++ b/openssl/src/crypto/bio/bio_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2005-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2005-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,7 +9,6 @@ #include "internal/e_os.h" #include "internal/sockets.h" -#include "internal/bio_addr.h" /* BEGIN BIO_ADDRINFO/BIO_ADDR stuff. */ @@ -34,18 +33,14 @@ # error openssl/bio.h included before bio_local.h # endif -# ifdef AI_PASSIVE - /* - * There's a bug in VMS C header file netdb.h, where struct addrinfo - * always is the P32 variant, but the functions that handle that structure, - * such as getaddrinfo() and freeaddrinfo() adapt to the initial pointer - * size. The easiest workaround is to force struct addrinfo to be the - * 64-bit variant when compiling in P64 mode. + * Undefine AF_UNIX on systems that define it but don't support it. */ -# if defined(OPENSSL_SYS_VMS) && __INITIAL_POINTER_SIZE == 64 -# define addrinfo __addrinfo64 -# endif +# if defined(OPENSSL_SYS_WINDOWS) +# undef AF_UNIX +# endif + +# ifdef AI_PASSIVE # define bio_addrinfo_st addrinfo # define bai_family ai_family @@ -64,6 +59,17 @@ struct bio_addrinfo_st { struct bio_addrinfo_st *bai_next; }; # endif + +union bio_addr_st { + struct sockaddr sa; +# ifdef AF_INET6 + struct sockaddr_in6 s_in6; +# endif + struct sockaddr_in s_in; +# ifdef AF_UNIX + struct sockaddr_un s_un; +# endif +}; #endif /* END BIO_ADDRINFO/BIO_ADDR stuff. */ @@ -116,12 +122,10 @@ struct bio_st { uint64_t num_read; uint64_t num_write; CRYPTO_EX_DATA ex_data; + CRYPTO_RWLOCK *lock; }; #ifndef OPENSSL_NO_SOCK -# ifdef OPENSSL_SYS_VMS -typedef unsigned int socklen_t; -# endif extern CRYPTO_RWLOCK *bio_lookup_lock; @@ -131,15 +135,9 @@ struct sockaddr *BIO_ADDR_sockaddr_noconst(BIO_ADDR *ap); socklen_t BIO_ADDR_sockaddr_size(const BIO_ADDR *ap); socklen_t BIO_ADDRINFO_sockaddr_size(const BIO_ADDRINFO *bai); const struct sockaddr *BIO_ADDRINFO_sockaddr(const BIO_ADDRINFO *bai); - -# if defined(OPENSSL_SYS_WINDOWS) && defined(WSAID_WSARECVMSG) -# define BIO_HAVE_WSAMSG -extern LPFN_WSARECVMSG bio_WSARecvMsg; -extern LPFN_WSASENDMSG bio_WSASendMsg; -# endif #endif -extern CRYPTO_REF_COUNT bio_type_count; +extern CRYPTO_RWLOCK *bio_type_lock; void bio_sock_cleanup_int(void); diff --git a/openssl/src/crypto/bio/bio_meth.c b/openssl/src/crypto/bio/bio_meth.c index f6fdcb935..469715ba0 100644 --- a/openssl/src/crypto/bio/bio_meth.c +++ b/openssl/src/crypto/bio/bio_meth.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,26 +10,25 @@ #include "bio_local.h" #include "internal/thread_once.h" -CRYPTO_REF_COUNT bio_type_count; +CRYPTO_RWLOCK *bio_type_lock = NULL; static CRYPTO_ONCE bio_type_init = CRYPTO_ONCE_STATIC_INIT; DEFINE_RUN_ONCE_STATIC(do_bio_type_init) { - return CRYPTO_NEW_REF(&bio_type_count, BIO_TYPE_START); + bio_type_lock = CRYPTO_THREAD_lock_new(); + return bio_type_lock != NULL; } int BIO_get_new_index(void) { + static CRYPTO_REF_COUNT bio_count = BIO_TYPE_START; int newval; if (!RUN_ONCE(&bio_type_init, do_bio_type_init)) { - /* Perhaps the error should be raised in do_bio_type_init()? */ - ERR_raise(ERR_LIB_BIO, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return -1; } - if (!CRYPTO_UP_REF(&bio_type_count, &newval)) - return -1; - if (newval > BIO_TYPE_MASK) + if (!CRYPTO_UP_REF(&bio_count, &newval, bio_type_lock)) return -1; return newval; } @@ -41,6 +40,7 @@ BIO_METHOD *BIO_meth_new(int type, const char *name) if (biom == NULL || (biom->name = OPENSSL_strdup(name)) == NULL) { OPENSSL_free(biom); + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return NULL; } biom->type = type; @@ -218,25 +218,3 @@ int BIO_meth_set_callback_ctrl(BIO_METHOD *biom, biom->callback_ctrl = callback_ctrl; return 1; } - -int BIO_meth_set_sendmmsg(BIO_METHOD *biom, - int (*bsendmmsg) (BIO *, BIO_MSG *, size_t, size_t, uint64_t, size_t *)) -{ - biom->bsendmmsg = bsendmmsg; - return 1; -} - -int (*BIO_meth_get_sendmmsg(const BIO_METHOD *biom))(BIO *, BIO_MSG *, size_t, size_t, uint64_t, size_t *) { - return biom->bsendmmsg; -} - -int BIO_meth_set_recvmmsg(BIO_METHOD *biom, - int (*brecvmmsg) (BIO *, BIO_MSG *, size_t, size_t, uint64_t, size_t *)) -{ - biom->brecvmmsg = brecvmmsg; - return 1; -} - -int (*BIO_meth_get_recvmmsg(const BIO_METHOD *biom))(BIO *, BIO_MSG *, size_t, size_t, uint64_t, size_t *) { - return biom->brecvmmsg; -} diff --git a/openssl/src/crypto/bio/bio_print.c b/openssl/src/crypto/bio/bio_print.c index 5f2543030..4c9c3af7c 100644 --- a/openssl/src/crypto/bio/bio_print.c +++ b/openssl/src/crypto/bio/bio_print.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -62,7 +62,7 @@ static int _dopr(char **sbuffer, char **buffer, #define DP_F_NUM (1 << 3) /* print leading zeroes */ #define DP_F_ZERO (1 << 4) -/* print HEX in UPPERcase */ +/* print HEX in UPPPERcase */ #define DP_F_UP (1 << 5) /* treat value as unsigned */ #define DP_F_UNSIGNED (1 << 6) @@ -276,7 +276,7 @@ _dopr(char **sbuffer, break; case 'E': flags |= DP_F_UP; - /* fall through */ + /* fall thru */ case 'e': if (cflags == DP_C_LDOUBLE) fvalue = va_arg(args, LDOUBLE); @@ -288,7 +288,7 @@ _dopr(char **sbuffer, break; case 'G': flags |= DP_F_UP; - /* fall through */ + /* fall thru */ case 'g': if (cflags == DP_C_LDOUBLE) fvalue = va_arg(args, LDOUBLE); @@ -707,6 +707,8 @@ fmtfp(char **sbuffer, fracpart = (fracpart / 10); } + if (fplace == sizeof(fconvert)) + fplace--; fconvert[fplace] = 0; /* convert exponent part */ @@ -845,8 +847,10 @@ doapr_outch(char **sbuffer, *maxlen += BUFFER_INC; if (*buffer == NULL) { - if ((*buffer = OPENSSL_malloc(*maxlen)) == NULL) + if ((*buffer = OPENSSL_malloc(*maxlen)) == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } if (*currlen > 0) { if (!ossl_assert(*sbuffer != NULL)) return 0; @@ -857,8 +861,10 @@ doapr_outch(char **sbuffer, char *tmpbuf; tmpbuf = OPENSSL_realloc(*buffer, *maxlen); - if (tmpbuf == NULL) + if (tmpbuf == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } *buffer = tmpbuf; } } diff --git a/openssl/src/crypto/bio/bio_sock.c b/openssl/src/crypto/bio/bio_sock.c index ea28fd282..eb173282c 100644 --- a/openssl/src/crypto/bio/bio_sock.c +++ b/openssl/src/crypto/bio/bio_sock.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,17 +26,14 @@ static int wsa_init_done = 0; # if defined __TANDEM # include # include /* select */ +# if defined(OPENSSL_TANDEM_FLOSS) +# include +# endif # elif defined _WIN32 # include /* for type fd_set */ # else # include -# if defined __VMS -# include -# elif defined _HPUX_SOURCE -# include -# else -# include -# endif +# include # endif # ifndef OPENSSL_NO_DEPRECATED_1_1_0 @@ -127,11 +124,6 @@ struct hostent *BIO_gethostbyname(const char *name) } # endif -# ifdef BIO_HAVE_WSAMSG -LPFN_WSARECVMSG bio_WSARecvMsg; -LPFN_WSASENDMSG bio_WSASendMsg; -# endif - int BIO_sock_init(void) { # ifdef OPENSSL_SYS_WINDOWS @@ -152,39 +144,6 @@ int BIO_sock_init(void) ERR_raise(ERR_LIB_BIO, BIO_R_WSASTARTUP); return -1; } - - /* - * On Windows, some socket functions are not exposed as a prototype. - * Instead, their function pointers must be loaded via this elaborate - * process... - */ -# ifdef BIO_HAVE_WSAMSG - { - GUID id_WSARecvMsg = WSAID_WSARECVMSG; - GUID id_WSASendMsg = WSAID_WSASENDMSG; - DWORD len_out = 0; - SOCKET s; - - s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - if (s != INVALID_SOCKET) { - if (WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, - &id_WSARecvMsg, sizeof(id_WSARecvMsg), - &bio_WSARecvMsg, sizeof(bio_WSARecvMsg), - &len_out, NULL, NULL) != 0 - || len_out != sizeof(bio_WSARecvMsg)) - bio_WSARecvMsg = NULL; - - if (WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, - &id_WSASendMsg, sizeof(id_WSASendMsg), - &bio_WSASendMsg, sizeof(bio_WSASendMsg), - &len_out, NULL, NULL) != 0 - || len_out != sizeof(bio_WSASendMsg)) - bio_WSASendMsg = NULL; - - closesocket(s); - } - } -# endif } # endif /* OPENSSL_SYS_WINDOWS */ # ifdef WATT32 @@ -211,35 +170,7 @@ int BIO_socket_ioctl(int fd, long type, void *arg) { int i; -# ifdef __DJGPP__ - i = ioctlsocket(fd, type, (char *)arg); -# else -# if defined(OPENSSL_SYS_VMS) - /*- - * 2011-02-18 SMS. - * VMS ioctl() can't tolerate a 64-bit "void *arg", but we - * observe that all the consumers pass in an "unsigned long *", - * so we arrange a local copy with a short pointer, and use - * that, instead. - */ -# if __INITIAL_POINTER_SIZE == 64 -# define ARG arg_32p -# pragma pointer_size save -# pragma pointer_size 32 - unsigned long arg_32; - unsigned long *arg_32p; -# pragma pointer_size restore - arg_32p = &arg_32; - arg_32 = *((unsigned long *)arg); -# else /* __INITIAL_POINTER_SIZE == 64 */ -# define ARG arg -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ -# else /* defined(OPENSSL_SYS_VMS) */ -# define ARG arg -# endif /* defined(OPENSSL_SYS_VMS) [else] */ - - i = ioctlsocket(fd, type, ARG); -# endif /* __DJGPP__ */ + i = ioctlsocket(fd, type, arg); if (i < 0) ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), "calling ioctlsocket()"); @@ -302,14 +233,13 @@ int BIO_accept(int sock, char **ip_port) if (ip_port != NULL) { char *host = BIO_ADDR_hostname_string(&res, 1); char *port = BIO_ADDR_service_string(&res, 1); - if (host != NULL && port != NULL) { + if (host != NULL && port != NULL) *ip_port = OPENSSL_zalloc(strlen(host) + strlen(port) + 2); - } else { + else *ip_port = NULL; - ERR_raise(ERR_LIB_BIO, ERR_R_BIO_LIB); - } if (*ip_port == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); BIO_closesocket(ret); ret = (int)INVALID_SOCKET; } else { @@ -432,17 +362,13 @@ int BIO_socket_wait(int fd, int for_read, time_t max_time) struct timeval tv; time_t now; -#ifdef _WIN32 - if ((SOCKET)fd == INVALID_SOCKET) -#else if (fd < 0 || fd >= FD_SETSIZE) -#endif return -1; if (max_time == 0) return 1; now = time(NULL); - if (max_time < now) + if (max_time <= now) return 0; FD_ZERO(&confds); diff --git a/openssl/src/crypto/bio/bio_sock2.c b/openssl/src/crypto/bio/bio_sock2.c index 252a9ab07..b6c95913c 100644 --- a/openssl/src/crypto/bio/bio_sock2.c +++ b/openssl/src/crypto/bio/bio_sock2.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,7 +13,6 @@ #include "bio_local.h" #include "internal/ktls.h" -#include "internal/bio_tfo.h" #include @@ -53,6 +52,17 @@ int BIO_socket(int domain, int socktype, int protocol, int options) ERR_raise(ERR_LIB_BIO, BIO_R_UNABLE_TO_CREATE_SOCKET); return INVALID_SOCKET; } +# ifndef OPENSSL_NO_KTLS + { + /* + * The new socket is created successfully regardless of ktls_enable. + * ktls_enable doesn't change any functionality of the socket, except + * changing the setsockopt to enable the processing of ktls_start. + * Thus, it is not a problem to call it for non-TLS sockets. + */ + ktls_enable(sock); + } +# endif return sock; } @@ -69,7 +79,6 @@ int BIO_socket(int domain, int socktype, int protocol, int options) * - BIO_SOCK_KEEPALIVE: enable regularly sending keep-alive messages. * - BIO_SOCK_NONBLOCK: Make the socket non-blocking. * - BIO_SOCK_NODELAY: don't delay small messages. - * - BIO_SOCK_TFO: use TCP Fast Open * * options holds BIO socket options that can be used * You should call this for every address returned by BIO_lookup @@ -109,68 +118,6 @@ int BIO_connect(int sock, const BIO_ADDR *addr, int options) return 0; } } - if (options & BIO_SOCK_TFO) { -# if defined(OSSL_TFO_CLIENT_FLAG) -# if defined(OSSL_TFO_SYSCTL_CLIENT) - int enabled = 0; - size_t enabledlen = sizeof(enabled); - - /* Later FreeBSD */ - if (sysctlbyname(OSSL_TFO_SYSCTL_CLIENT, &enabled, &enabledlen, NULL, 0) < 0) { - ERR_raise(ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT); - return 0; - } - /* Need to check for client flag */ - if (!(enabled & OSSL_TFO_CLIENT_FLAG)) { - ERR_raise(ERR_LIB_BIO, BIO_R_TFO_DISABLED); - return 0; - } -# elif defined(OSSL_TFO_SYSCTL) - int enabled = 0; - size_t enabledlen = sizeof(enabled); - - /* macOS */ - if (sysctlbyname(OSSL_TFO_SYSCTL, &enabled, &enabledlen, NULL, 0) < 0) { - ERR_raise(ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT); - return 0; - } - /* Need to check for client flag */ - if (!(enabled & OSSL_TFO_CLIENT_FLAG)) { - ERR_raise(ERR_LIB_BIO, BIO_R_TFO_DISABLED); - return 0; - } -# endif -# endif -# if defined(OSSL_TFO_CONNECTX) - sa_endpoints_t sae; - - memset(&sae, 0, sizeof(sae)); - sae.sae_dstaddr = BIO_ADDR_sockaddr(addr); - sae.sae_dstaddrlen = BIO_ADDR_sockaddr_size(addr); - if (connectx(sock, &sae, SAE_ASSOCID_ANY, - CONNECT_DATA_IDEMPOTENT | CONNECT_RESUME_ON_READ_WRITE, - NULL, 0, NULL, NULL) == -1) { - if (!BIO_sock_should_retry(-1)) { - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling connectx()"); - ERR_raise(ERR_LIB_BIO, BIO_R_CONNECT_ERROR); - } - return 0; - } -# endif -# if defined(OSSL_TFO_CLIENT_SOCKOPT) - if (setsockopt(sock, IPPROTO_TCP, OSSL_TFO_CLIENT_SOCKOPT, - (const void *)&on, sizeof(on)) != 0) { - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); - ERR_raise(ERR_LIB_BIO, BIO_R_UNABLE_TO_TFO); - return 0; - } -# endif -# if defined(OSSL_TFO_DO_NOT_CONNECT) - return 1; -# endif - } if (connect(sock, BIO_ADDR_sockaddr(addr), BIO_ADDR_sockaddr_size(addr)) == -1) { @@ -181,15 +128,6 @@ int BIO_connect(int sock, const BIO_ADDR *addr, int options) } return 0; } -# ifndef OPENSSL_NO_KTLS - /* - * The new socket is created successfully regardless of ktls_enable. - * ktls_enable doesn't change any functionality of the socket, except - * changing the setsockopt to enable the processing of ktls_start. - * Thus, it is not a problem to call it for non-TLS sockets. - */ - ktls_enable(sock); -# endif return 1; } @@ -263,7 +201,6 @@ int BIO_bind(int sock, const BIO_ADDR *addr, int options) * for a recently closed port. * - BIO_SOCK_V6_ONLY: When creating an IPv6 socket, make it listen only * for IPv6 addresses and not IPv4 addresses mapped to IPv6. - * - BIO_SOCK_TFO: accept TCP fast open (set TCP_FASTOPEN) * * It's recommended that you set up both an IPv6 and IPv4 listen socket, and * then check both for new clients that connect to it. You want to set up @@ -327,7 +264,7 @@ int BIO_listen(int sock, const BIO_ADDR *addr, int options) } } - /* On OpenBSD it is always IPv6 only with IPv6 sockets thus read-only */ + /* On OpenBSD it is always ipv6 only with ipv6 sockets thus read-only */ # if defined(IPV6_V6ONLY) && !defined(__OpenBSD__) if (BIO_ADDR_family(addr) == AF_INET6) { /* @@ -355,54 +292,6 @@ int BIO_listen(int sock, const BIO_ADDR *addr, int options) return 0; } -# if defined(OSSL_TFO_SERVER_SOCKOPT) - /* - * Must do it explicitly after listen() for macOS, still - * works fine on other OS's - */ - if ((options & BIO_SOCK_TFO) && socktype != SOCK_DGRAM) { - int q = OSSL_TFO_SERVER_SOCKOPT_VALUE; -# if defined(OSSL_TFO_CLIENT_FLAG) -# if defined(OSSL_TFO_SYSCTL_SERVER) - int enabled = 0; - size_t enabledlen = sizeof(enabled); - - /* Later FreeBSD */ - if (sysctlbyname(OSSL_TFO_SYSCTL_SERVER, &enabled, &enabledlen, NULL, 0) < 0) { - ERR_raise(ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT); - return 0; - } - /* Need to check for server flag */ - if (!(enabled & OSSL_TFO_SERVER_FLAG)) { - ERR_raise(ERR_LIB_BIO, BIO_R_TFO_DISABLED); - return 0; - } -# elif defined(OSSL_TFO_SYSCTL) - int enabled = 0; - size_t enabledlen = sizeof(enabled); - - /* Early FreeBSD, macOS */ - if (sysctlbyname(OSSL_TFO_SYSCTL, &enabled, &enabledlen, NULL, 0) < 0) { - ERR_raise(ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT); - return 0; - } - /* Need to check for server flag */ - if (!(enabled & OSSL_TFO_SERVER_FLAG)) { - ERR_raise(ERR_LIB_BIO, BIO_R_TFO_DISABLED); - return 0; - } -# endif -# endif - if (setsockopt(sock, IPPROTO_TCP, OSSL_TFO_SERVER_SOCKOPT, - (void *)&q, sizeof(q)) < 0) { - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); - ERR_raise(ERR_LIB_BIO, BIO_R_UNABLE_TO_TFO); - return 0; - } - } -# endif - return 1; } diff --git a/openssl/src/crypto/bio/bss_acpt.c b/openssl/src/crypto/bio/bss_acpt.c index 4ccdca18e..1cda96733 100644 --- a/openssl/src/crypto/bio/bss_acpt.c +++ b/openssl/src/crypto/bio/bss_acpt.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -92,8 +92,10 @@ static BIO_ACCEPT *BIO_ACCEPT_new(void) { BIO_ACCEPT *ret; - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return NULL; + } ret->accept_family = BIO_FAMILY_IPANY; ret->accept_sock = (int)INVALID_SOCKET; return ret; @@ -186,7 +188,7 @@ static int acpt_state(BIO *b, BIO_ACCEPT *c) * at least the "else" part will always be * compiled. */ -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 family = AF_INET6; } else { #endif @@ -450,14 +452,10 @@ static long acpt_ctrl(BIO *b, int cmd, long num, void *ptr) data->bio_chain = (BIO *)ptr; } else if (num == 4) { data->accept_family = *(int *)ptr; - } else if (num == 5) { - data->bind_mode |= BIO_SOCK_TFO; } } else { if (num == 2) { data->bind_mode &= ~BIO_SOCK_NONBLOCK; - } else if (num == 5) { - data->bind_mode &= ~BIO_SOCK_TFO; } } break; @@ -499,7 +497,7 @@ static long acpt_ctrl(BIO *b, int cmd, long num, void *ptr) *pp = data->cache_peer_serv; } else if (num == 4) { switch (BIO_ADDRINFO_family(data->addr_iter)) { -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 case AF_INET6: ret = BIO_FAMILY_IPV6; break; @@ -568,7 +566,7 @@ BIO *BIO_new_accept(const char *str) ret = BIO_new(BIO_s_accept()); if (ret == NULL) return NULL; - if (BIO_set_accept_name(ret, str) > 0) + if (BIO_set_accept_name(ret, str)) return ret; BIO_free(ret); return NULL; diff --git a/openssl/src/crypto/bio/bss_bio.c b/openssl/src/crypto/bio/bss_bio.c index 3af3b27ea..5039a621f 100644 --- a/openssl/src/crypto/bio/bss_bio.c +++ b/openssl/src/crypto/bio/bss_bio.c @@ -273,7 +273,7 @@ static int bio_write(BIO *bio, const char *buf, int num_) BIO_clear_retry_flags(bio); - if (!bio->init || buf == NULL || num_ <= 0) + if (!bio->init || buf == NULL || num == 0) return 0; b = bio->ptr; @@ -620,16 +620,20 @@ static int bio_make_pair(BIO *bio1, BIO *bio2) if (b1->buf == NULL) { b1->buf = OPENSSL_malloc(b1->size); - if (b1->buf == NULL) + if (b1->buf == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } b1->len = 0; b1->offset = 0; } if (b2->buf == NULL) { b2->buf = OPENSSL_malloc(b2->size); - if (b2->buf == NULL) + if (b2->buf == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } b2->len = 0; b2->offset = 0; } diff --git a/openssl/src/crypto/bio/bss_conn.c b/openssl/src/crypto/bio/bss_conn.c index 9d00f1829..03d148014 100644 --- a/openssl/src/crypto/bio/bss_conn.c +++ b/openssl/src/crypto/bio/bss_conn.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,7 +11,6 @@ #include #include "bio_local.h" -#include "internal/bio_tfo.h" #include "internal/ktls.h" #ifndef OPENSSL_NO_SOCK @@ -19,14 +18,12 @@ typedef struct bio_connect_st { int state; int connect_family; - int connect_sock_type; char *param_hostname; char *param_service; int connect_mode; # ifndef OPENSSL_NO_KTLS unsigned char record_type; # endif - int tfo_first; BIO_ADDRINFO *addr_first; const BIO_ADDRINFO *addr_iter; @@ -40,25 +37,15 @@ typedef struct bio_connect_st { * ssl info_callback */ BIO_info_cb *info_callback; - /* - * Used when connect_sock_type is SOCK_DGRAM. Owned by us; we forward - * read/write(mmsg) calls to this if present. - */ - BIO *dgram_bio; } BIO_CONNECT; static int conn_write(BIO *h, const char *buf, int num); static int conn_read(BIO *h, char *buf, int size); static int conn_puts(BIO *h, const char *str); -static int conn_gets(BIO *h, char *buf, int size); static long conn_ctrl(BIO *h, int cmd, long arg1, void *arg2); static int conn_new(BIO *h); static int conn_free(BIO *data); static long conn_callback_ctrl(BIO *h, int cmd, BIO_info_cb *); -static int conn_sendmmsg(BIO *h, BIO_MSG *m, size_t s, size_t n, - uint64_t f, size_t *mp); -static int conn_recvmmsg(BIO *h, BIO_MSG *m, size_t s, size_t n, - uint64_t f, size_t *mp); static int conn_state(BIO *b, BIO_CONNECT *c); static void conn_close_socket(BIO *data); @@ -81,36 +68,16 @@ static const BIO_METHOD methods_connectp = { bread_conv, conn_read, conn_puts, - conn_gets, + NULL, /* conn_gets, */ conn_ctrl, conn_new, conn_free, conn_callback_ctrl, - conn_sendmmsg, - conn_recvmmsg, }; -static int conn_create_dgram_bio(BIO *b, BIO_CONNECT *c) -{ - if (c->connect_sock_type != SOCK_DGRAM) - return 1; - -#ifndef OPENSSL_NO_DGRAM - c->dgram_bio = BIO_new_dgram(b->num, 0); - if (c->dgram_bio == NULL) - goto err; - - return 1; - -err: -#endif - c->state = BIO_CONN_S_CONNECT_ERROR; - return 0; -} - static int conn_state(BIO *b, BIO_CONNECT *c) { - int ret = -1, i, opts; + int ret = -1, i; BIO_info_cb *cb = NULL; if (c->info_callback != NULL) @@ -138,7 +105,7 @@ static int conn_state(BIO *b, BIO_CONNECT *c) * at least the "else" part will always be * compiled. */ -#if OPENSSL_USE_IPV6 +#ifdef AF_INET6 family = AF_INET6; } else { #endif @@ -158,8 +125,7 @@ static int conn_state(BIO *b, BIO_CONNECT *c) } if (BIO_lookup(c->param_hostname, c->param_service, BIO_LOOKUP_CLIENT, - family, c->connect_sock_type, - &c->addr_first) == 0) + family, SOCK_STREAM, &c->addr_first) == 0) goto exit_loop; } if (c->addr_first == NULL) { @@ -188,12 +154,8 @@ static int conn_state(BIO *b, BIO_CONNECT *c) case BIO_CONN_S_CONNECT: BIO_clear_retry_flags(b); ERR_set_mark(); - - opts = c->connect_mode; - if (BIO_ADDRINFO_socktype(c->addr_iter) == SOCK_STREAM) - opts |= BIO_SOCK_KEEPALIVE; - - ret = BIO_connect(b->num, BIO_ADDRINFO_address(c->addr_iter), opts); + ret = BIO_connect(b->num, BIO_ADDRINFO_address(c->addr_iter), + BIO_SOCK_KEEPALIVE | c->connect_mode); b->retry_reason = 0; if (ret == 0) { if (BIO_sock_should_retry(ret)) { @@ -221,16 +183,11 @@ static int conn_state(BIO *b, BIO_CONNECT *c) goto exit_loop; } else { ERR_clear_last_mark(); - if (!conn_create_dgram_bio(b, c)) - break; c->state = BIO_CONN_S_OK; } break; case BIO_CONN_S_BLOCKED_CONNECT: - /* wait for socket being writable, before querying BIO_sock_error */ - if (BIO_socket_wait(b->num, 0, time(NULL)) == 0) - break; i = BIO_sock_error(b->num); if (i != 0) { BIO_clear_retry_flags(b); @@ -248,20 +205,8 @@ static int conn_state(BIO *b, BIO_CONNECT *c) ERR_raise(ERR_LIB_BIO, BIO_R_NBIO_CONNECT_ERROR); ret = 0; goto exit_loop; - } else { - if (!conn_create_dgram_bio(b, c)) - break; + } else c->state = BIO_CONN_S_OK; -# ifndef OPENSSL_NO_KTLS - /* - * The new socket is created successfully regardless of ktls_enable. - * ktls_enable doesn't change any functionality of the socket, except - * changing the setsockopt to enable the processing of ktls_start. - * Thus, it is not a problem to call it for non-TLS sockets. - */ - ktls_enable(b->num); -# endif - } break; case BIO_CONN_S_CONNECT_ERROR: @@ -291,19 +236,20 @@ static int conn_state(BIO *b, BIO_CONNECT *c) return ret; } -static BIO_CONNECT *BIO_CONNECT_new(void) +BIO_CONNECT *BIO_CONNECT_new(void) { BIO_CONNECT *ret; - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return NULL; + } ret->state = BIO_CONN_S_BEFORE; ret->connect_family = BIO_FAMILY_IPANY; - ret->connect_sock_type = SOCK_STREAM; return ret; } -static void BIO_CONNECT_free(BIO_CONNECT *a) +void BIO_CONNECT_free(BIO_CONNECT *a) { if (a == NULL) return; @@ -351,8 +297,6 @@ static int conn_free(BIO *a) return 0; data = (BIO_CONNECT *)a->ptr; - BIO_free(data->dgram_bio); - if (a->shutdown) { conn_close_socket(a); BIO_CONNECT_free(data); @@ -375,13 +319,6 @@ static int conn_read(BIO *b, char *out, int outl) return ret; } - if (data->dgram_bio != NULL) { - BIO_clear_retry_flags(b); - ret = BIO_read(data->dgram_bio, out, outl); - BIO_set_flags(b, BIO_get_retry_flags(data->dgram_bio)); - return ret; - } - if (out != NULL) { clear_socket_error(); # ifndef OPENSSL_NO_KTLS @@ -413,13 +350,6 @@ static int conn_write(BIO *b, const char *in, int inl) return ret; } - if (data->dgram_bio != NULL) { - BIO_clear_retry_flags(b); - ret = BIO_write(data->dgram_bio, in, inl); - BIO_set_flags(b, BIO_get_retry_flags(data->dgram_bio)); - return ret; - } - clear_socket_error(); # ifndef OPENSSL_NO_KTLS if (BIO_should_ktls_ctrl_msg_flag(b)) { @@ -429,15 +359,6 @@ static int conn_write(BIO *b, const char *in, int inl) BIO_clear_ktls_ctrl_msg_flag(b); } } else -# endif -# if defined(OSSL_TFO_SENDTO) - if (data->tfo_first) { - int peerlen = BIO_ADDRINFO_sockaddr_size(data->addr_iter); - - ret = sendto(b->num, in, inl, OSSL_TFO_SENDTO, - BIO_ADDRINFO_sockaddr(data->addr_iter), peerlen); - data->tfo_first = 0; - } else # endif ret = writesocket(b->num, in, inl); BIO_clear_retry_flags(b); @@ -455,7 +376,6 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr) const char **pptr = NULL; long ret = 1; BIO_CONNECT *data; - const BIO_ADDR *dg_addr; # ifndef OPENSSL_NO_KTLS ktls_crypto_info_t *crypto_info; # endif @@ -489,7 +409,7 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr) *pptr = (const char *)BIO_ADDRINFO_address(data->addr_iter); } else if (num == 3) { switch (BIO_ADDRINFO_family(data->addr_iter)) { -# if OPENSSL_USE_IPV6 +# ifdef AF_INET6 case AF_INET6: ret = BIO_FAMILY_IPV6; break; @@ -504,8 +424,6 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr) ret = -1; break; } - } else if (num == 4) { - ret = data->connect_mode; } else { ret = 0; } @@ -560,90 +478,14 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr) } } break; - case BIO_C_SET_SOCK_TYPE: - if ((num != SOCK_STREAM && num != SOCK_DGRAM) - || data->state >= BIO_CONN_S_GET_ADDR) { - ret = 0; - break; - } - - data->connect_sock_type = (int)num; - ret = 1; - break; - case BIO_C_GET_SOCK_TYPE: - ret = data->connect_sock_type; - break; - case BIO_C_GET_DGRAM_BIO: - if (data->dgram_bio != NULL) { - *(BIO **)ptr = data->dgram_bio; - ret = 1; - } else { - ret = 0; - } - break; - case BIO_CTRL_DGRAM_GET_PEER: - case BIO_CTRL_DGRAM_DETECT_PEER_ADDR: - if (data->state != BIO_CONN_S_OK) - conn_state(b, data); /* best effort */ - - if (data->state >= BIO_CONN_S_CREATE_SOCKET - && data->addr_iter != NULL - && (dg_addr = BIO_ADDRINFO_address(data->addr_iter)) != NULL) { - - ret = BIO_ADDR_sockaddr_size(dg_addr); - if (num == 0 || num > ret) - num = ret; - - memcpy(ptr, dg_addr, num); - ret = num; - } else { - ret = 0; - } - - break; - case BIO_CTRL_GET_RPOLL_DESCRIPTOR: - case BIO_CTRL_GET_WPOLL_DESCRIPTOR: - { - BIO_POLL_DESCRIPTOR *pd = ptr; - - if (data->state != BIO_CONN_S_OK) - conn_state(b, data); /* best effort */ - - if (data->state >= BIO_CONN_S_CREATE_SOCKET) { - pd->type = BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD; - pd->value.fd = b->num; - } else { - ret = 0; - } - } - break; case BIO_C_SET_NBIO: if (num != 0) data->connect_mode |= BIO_SOCK_NONBLOCK; else data->connect_mode &= ~BIO_SOCK_NONBLOCK; - - if (data->dgram_bio != NULL) - ret = BIO_set_nbio(data->dgram_bio, num); - - break; -#if defined(TCP_FASTOPEN) && !defined(OPENSSL_NO_TFO) - case BIO_C_SET_TFO: - if (num != 0) { - data->connect_mode |= BIO_SOCK_TFO; - data->tfo_first = 1; - } else { - data->connect_mode &= ~BIO_SOCK_TFO; - data->tfo_first = 0; - } break; -#endif case BIO_C_SET_CONNECT_MODE: data->connect_mode = (int)num; - if (num & BIO_SOCK_TFO) - data->tfo_first = 1; - else - data->tfo_first = 0; break; case BIO_C_GET_FD: if (b->init) { @@ -716,11 +558,6 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr) BIO_clear_ktls_ctrl_msg_flag(b); ret = 0; break; - case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE: - ret = ktls_enable_tx_zerocopy_sendfile(b->num); - if (ret) - BIO_set_ktls_zerocopy_sendfile_flag(b); - break; # endif default: ret = 0; @@ -758,123 +595,6 @@ static int conn_puts(BIO *bp, const char *str) return ret; } -int conn_gets(BIO *bio, char *buf, int size) -{ - BIO_CONNECT *data; - char *ptr = buf; - int ret = 0; - - if (buf == NULL) { - ERR_raise(ERR_LIB_BIO, ERR_R_PASSED_NULL_PARAMETER); - return -1; - } - if (size <= 0) { - ERR_raise(ERR_LIB_BIO, BIO_R_INVALID_ARGUMENT); - return -1; - } - *buf = '\0'; - - if (bio == NULL || bio->ptr == NULL) { - ERR_raise(ERR_LIB_BIO, ERR_R_PASSED_NULL_PARAMETER); - return -1; - } - data = (BIO_CONNECT *)bio->ptr; - if (data->state != BIO_CONN_S_OK) { - ret = conn_state(bio, data); - if (ret <= 0) - return ret; - } - - if (data->dgram_bio != NULL) { - ERR_raise(ERR_LIB_BIO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return -1; - } - - clear_socket_error(); - while (size-- > 1) { -# ifndef OPENSSL_NO_KTLS - if (BIO_get_ktls_recv(bio)) - ret = ktls_read_record(bio->num, ptr, 1); - else -# endif - ret = readsocket(bio->num, ptr, 1); - BIO_clear_retry_flags(bio); - if (ret <= 0) { - if (BIO_sock_should_retry(ret)) - BIO_set_retry_read(bio); - else if (ret == 0) - bio->flags |= BIO_FLAGS_IN_EOF; - break; - } - if (*ptr++ == '\n') - break; - } - *ptr = '\0'; - return ret > 0 || (bio->flags & BIO_FLAGS_IN_EOF) != 0 ? ptr - buf : ret; -} - -static int conn_sendmmsg(BIO *bio, BIO_MSG *msg, size_t stride, size_t num_msgs, - uint64_t flags, size_t *msgs_processed) -{ - int ret; - BIO_CONNECT *data; - - if (bio == NULL) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - data = (BIO_CONNECT *)bio->ptr; - if (data->state != BIO_CONN_S_OK) { - ret = conn_state(bio, data); - if (ret <= 0) { - *msgs_processed = 0; - return 0; - } - } - - if (data->dgram_bio == NULL) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - - return BIO_sendmmsg(data->dgram_bio, msg, stride, num_msgs, - flags, msgs_processed); -} - -static int conn_recvmmsg(BIO *bio, BIO_MSG *msg, size_t stride, size_t num_msgs, - uint64_t flags, size_t *msgs_processed) -{ - int ret; - BIO_CONNECT *data; - - if (bio == NULL) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - data = (BIO_CONNECT *)bio->ptr; - if (data->state != BIO_CONN_S_OK) { - ret = conn_state(bio, data); - if (ret <= 0) { - *msgs_processed = 0; - return 0; - } - } - - if (data->dgram_bio == NULL) { - *msgs_processed = 0; - ERR_raise(ERR_LIB_BIO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - - return BIO_recvmmsg(data->dgram_bio, msg, stride, num_msgs, - flags, msgs_processed); -} - BIO *BIO_new_connect(const char *str) { BIO *ret; diff --git a/openssl/src/crypto/bio/bss_core.c b/openssl/src/crypto/bio/bss_core.c index b9a8eff34..7a84b2046 100644 --- a/openssl/src/crypto/bio/bss_core.c +++ b/openssl/src/crypto/bio/bss_core.c @@ -10,7 +10,6 @@ #include #include "bio_local.h" #include "internal/cryptlib.h" -#include "crypto/context.h" typedef struct { OSSL_FUNC_BIO_read_ex_fn *c_bio_read_ex; @@ -22,19 +21,26 @@ typedef struct { OSSL_FUNC_BIO_free_fn *c_bio_free; } BIO_CORE_GLOBALS; -void ossl_bio_core_globals_free(void *vbcg) +static void bio_core_globals_free(void *vbcg) { OPENSSL_free(vbcg); } -void *ossl_bio_core_globals_new(OSSL_LIB_CTX *ctx) +static void *bio_core_globals_new(OSSL_LIB_CTX *ctx) { return OPENSSL_zalloc(sizeof(BIO_CORE_GLOBALS)); } +static const OSSL_LIB_CTX_METHOD bio_core_globals_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + bio_core_globals_new, + bio_core_globals_free, +}; + static ossl_inline BIO_CORE_GLOBALS *get_globals(OSSL_LIB_CTX *libctx) { - return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_BIO_CORE_INDEX); + return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_BIO_CORE_INDEX, + &bio_core_globals_method); } static int bio_core_read_ex(BIO *bio, char *data, size_t data_len, diff --git a/openssl/src/crypto/bio/bss_dgram.c b/openssl/src/crypto/bio/bss_dgram.c index f6d688b35..8e7daa199 100644 --- a/openssl/src/crypto/bio/bss_dgram.c +++ b/openssl/src/crypto/bio/bss_dgram.c @@ -1,5 +1,5 @@ /* - * Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,7 +14,6 @@ #include #include -#include "internal/time.h" #include "bio_local.h" #ifndef OPENSSL_NO_DGRAM @@ -43,102 +42,6 @@ ((a)->s6_addr32[2] == htonl(0x0000ffff))) # endif -/* Determine what method to use for BIO_sendmmsg and BIO_recvmmsg. */ -# define M_METHOD_NONE 0 -# define M_METHOD_RECVMMSG 1 -# define M_METHOD_RECVMSG 2 -# define M_METHOD_RECVFROM 3 -# define M_METHOD_WSARECVMSG 4 - -# if defined(__GLIBC__) && defined(__GLIBC_PREREQ) -# if !(__GLIBC_PREREQ(2, 14)) -# undef NO_RECVMMSG - /* - * Some old glibc versions may have recvmmsg and MSG_WAITFORONE flag, but - * not sendmmsg. We need both so force this to be disabled on these old - * versions - */ -# define NO_RECVMMSG -# endif -# endif -# if defined(__GNU__) - /* GNU/Hurd does not have IP_PKTINFO yet */ - #undef NO_RECVMSG - #define NO_RECVMSG -# endif -# if defined(__ANDROID_API__) && __ANDROID_API__ < 21 -# undef NO_RECVMMSG -# define NO_RECVMMSG -# endif -# if !defined(M_METHOD) -# if defined(OPENSSL_SYS_WINDOWS) && defined(BIO_HAVE_WSAMSG) && !defined(NO_WSARECVMSG) -# define M_METHOD M_METHOD_WSARECVMSG -# elif !defined(OPENSSL_SYS_WINDOWS) && defined(MSG_WAITFORONE) && !defined(NO_RECVMMSG) -# define M_METHOD M_METHOD_RECVMMSG -# elif !defined(OPENSSL_SYS_WINDOWS) && defined(CMSG_LEN) && !defined(NO_RECVMSG) -# define M_METHOD M_METHOD_RECVMSG -# elif !defined(NO_RECVFROM) -# define M_METHOD M_METHOD_RECVFROM -# else -# define M_METHOD M_METHOD_NONE -# endif -# endif - -# if defined(OPENSSL_SYS_WINDOWS) -# define BIO_CMSG_SPACE(x) WSA_CMSG_SPACE(x) -# define BIO_CMSG_FIRSTHDR(x) WSA_CMSG_FIRSTHDR(x) -# define BIO_CMSG_NXTHDR(x, y) WSA_CMSG_NXTHDR(x, y) -# define BIO_CMSG_DATA(x) WSA_CMSG_DATA(x) -# define BIO_CMSG_LEN(x) WSA_CMSG_LEN(x) -# define MSGHDR_TYPE WSAMSG -# define CMSGHDR_TYPE WSACMSGHDR -# else -# define MSGHDR_TYPE struct msghdr -# define CMSGHDR_TYPE struct cmsghdr -# define BIO_CMSG_SPACE(x) CMSG_SPACE(x) -# define BIO_CMSG_FIRSTHDR(x) CMSG_FIRSTHDR(x) -# define BIO_CMSG_NXTHDR(x, y) CMSG_NXTHDR(x, y) -# define BIO_CMSG_DATA(x) CMSG_DATA(x) -# define BIO_CMSG_LEN(x) CMSG_LEN(x) -# endif - -# if M_METHOD == M_METHOD_RECVMMSG \ - || M_METHOD == M_METHOD_RECVMSG \ - || M_METHOD == M_METHOD_WSARECVMSG -# if defined(__APPLE__) - /* - * CMSG_SPACE is not a constant expression on OSX even though POSIX - * says it's supposed to be. This should be adequate. - */ -# define BIO_CMSG_ALLOC_LEN 64 -# else -# if defined(IPV6_PKTINFO) -# define BIO_CMSG_ALLOC_LEN_1 BIO_CMSG_SPACE(sizeof(struct in6_pktinfo)) -# else -# define BIO_CMSG_ALLOC_LEN_1 0 -# endif -# if defined(IP_PKTINFO) -# define BIO_CMSG_ALLOC_LEN_2 BIO_CMSG_SPACE(sizeof(struct in_pktinfo)) -# else -# define BIO_CMSG_ALLOC_LEN_2 0 -# endif -# if defined(IP_RECVDSTADDR) -# define BIO_CMSG_ALLOC_LEN_3 BIO_CMSG_SPACE(sizeof(struct in_addr)) -# else -# define BIO_CMSG_ALLOC_LEN_3 0 -# endif -# define BIO_MAX(X,Y) ((X) > (Y) ? (X) : (Y)) -# define BIO_CMSG_ALLOC_LEN \ - BIO_MAX(BIO_CMSG_ALLOC_LEN_1, \ - BIO_MAX(BIO_CMSG_ALLOC_LEN_2, BIO_CMSG_ALLOC_LEN_3)) -# endif -# if (defined(IP_PKTINFO) || defined(IP_RECVDSTADDR)) && defined(IPV6_RECVPKTINFO) -# define SUPPORT_LOCAL_ADDR -# endif -# endif - -# define BIO_MSG_N(array, stride, n) (*(BIO_MSG *)((char *)(array) + (n)*(stride))) - static int dgram_write(BIO *h, const char *buf, int num); static int dgram_read(BIO *h, char *buf, int size); static int dgram_puts(BIO *h, const char *str); @@ -146,12 +49,6 @@ static long dgram_ctrl(BIO *h, int cmd, long arg1, void *arg2); static int dgram_new(BIO *h); static int dgram_free(BIO *data); static int dgram_clear(BIO *bio); -static int dgram_sendmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, - uint64_t flags, size_t *num_processed); -static int dgram_recvmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, - uint64_t flags, size_t *num_processed); # ifndef OPENSSL_NO_SCTP static int dgram_sctp_write(BIO *h, const char *buf, int num); @@ -170,6 +67,8 @@ static void dgram_sctp_handle_auth_free_key_event(BIO *b, union sctp_notificatio static int BIO_dgram_should_retry(int s); +static void get_current_time(struct timeval *t); + static const BIO_METHOD methods_dgramp = { BIO_TYPE_DGRAM, "datagram socket", @@ -183,8 +82,6 @@ static const BIO_METHOD methods_dgramp = { dgram_new, dgram_free, NULL, /* dgram_callback_ctrl */ - dgram_sendmmsg, - dgram_recvmmsg, }; # ifndef OPENSSL_NO_SCTP @@ -201,21 +98,17 @@ static const BIO_METHOD methods_dgramp_sctp = { dgram_sctp_new, dgram_sctp_free, NULL, /* dgram_callback_ctrl */ - NULL, /* sendmmsg */ - NULL, /* recvmmsg */ }; # endif typedef struct bio_dgram_data_st { BIO_ADDR peer; - BIO_ADDR local_addr; unsigned int connected; unsigned int _errno; unsigned int mtu; - OSSL_TIME next_timeout; - OSSL_TIME socket_timeout; + struct timeval next_timeout; + struct timeval socket_timeout; unsigned int peekmode; - char local_addr_enabled; } bio_dgram_data; # ifndef OPENSSL_NO_SCTP @@ -225,13 +118,11 @@ typedef struct bio_dgram_sctp_save_message_st { int length; } bio_dgram_sctp_save_message; -/* - * Note: bio_dgram_data must be first here - * as we use dgram_ctrl for underlying dgram operations - * which will cast this struct to a bio_dgram_data - */ typedef struct bio_dgram_sctp_data_st { - bio_dgram_data dgram; + BIO_ADDR peer; + unsigned int connected; + unsigned int _errno; + unsigned int mtu; struct bio_dgram_sctp_sndinfo sndinfo; struct bio_dgram_sctp_rcvinfo rcvinfo; struct bio_dgram_sctp_prinfo prinfo; @@ -304,102 +195,102 @@ static void dgram_adjust_rcv_timeout(BIO *b) { # if defined(SO_RCVTIMEO) bio_dgram_data *data = (bio_dgram_data *)b->ptr; - OSSL_TIME timeleft; + union { + size_t s; + int i; + } sz = { + 0 + }; /* Is a timer active? */ - if (!ossl_time_is_zero(data->next_timeout)) { + if (data->next_timeout.tv_sec > 0 || data->next_timeout.tv_usec > 0) { + struct timeval timenow, timeleft; + /* Read current socket timeout */ # ifdef OPENSSL_SYS_WINDOWS int timeout; - int sz = sizeof(timeout); + sz.i = sizeof(timeout); if (getsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, - (void *)&timeout, &sz) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling getsockopt()"); - else - data->socket_timeout = ossl_ms2time(timeout); + (void *)&timeout, &sz.i) < 0) { + perror("getsockopt"); + } else { + data->socket_timeout.tv_sec = timeout / 1000; + data->socket_timeout.tv_usec = (timeout % 1000) * 1000; + } # else - struct timeval tv; - socklen_t sz = sizeof(tv); - - if (getsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, &tv, &sz) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling getsockopt()"); - else - data->socket_timeout = ossl_time_from_timeval(tv); + sz.i = sizeof(data->socket_timeout); + if (getsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, + &(data->socket_timeout), (void *)&sz) < 0) { + perror("getsockopt"); + } else if (sizeof(sz.s) != sizeof(sz.i) && sz.i == 0) + OPENSSL_assert(sz.s <= sizeof(data->socket_timeout)); # endif + /* Get current time */ + get_current_time(&timenow); + /* Calculate time left until timer expires */ - timeleft = ossl_time_subtract(data->next_timeout, ossl_time_now()); - if (ossl_time_compare(timeleft, ossl_ticks2time(OSSL_TIME_US)) < 0) - timeleft = ossl_ticks2time(OSSL_TIME_US); + memcpy(&timeleft, &(data->next_timeout), sizeof(struct timeval)); + if (timeleft.tv_usec < timenow.tv_usec) { + timeleft.tv_usec = 1000000 - timenow.tv_usec + timeleft.tv_usec; + timeleft.tv_sec--; + } else { + timeleft.tv_usec -= timenow.tv_usec; + } + if (timeleft.tv_sec < timenow.tv_sec) { + timeleft.tv_sec = 0; + timeleft.tv_usec = 1; + } else { + timeleft.tv_sec -= timenow.tv_sec; + } /* * Adjust socket timeout if next handshake message timer will expire * earlier. */ - if (ossl_time_is_zero(data->socket_timeout) - || ossl_time_compare(data->socket_timeout, timeleft) >= 0) { + if ((data->socket_timeout.tv_sec == 0 + && data->socket_timeout.tv_usec == 0) + || (data->socket_timeout.tv_sec > timeleft.tv_sec) + || (data->socket_timeout.tv_sec == timeleft.tv_sec + && data->socket_timeout.tv_usec >= timeleft.tv_usec)) { # ifdef OPENSSL_SYS_WINDOWS - timeout = (int)ossl_time2ms(timeleft); + timeout = timeleft.tv_sec * 1000 + timeleft.tv_usec / 1000; if (setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, - (void *)&timeout, sizeof(timeout)) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + (void *)&timeout, sizeof(timeout)) < 0) { + perror("setsockopt"); + } # else - tv = ossl_time_to_timeval(timeleft); - if (setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, &tv, - sizeof(tv)) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + if (setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, &timeleft, + sizeof(struct timeval)) < 0) { + perror("setsockopt"); + } # endif } } # endif } -static void dgram_update_local_addr(BIO *b) -{ - bio_dgram_data *data = (bio_dgram_data *)b->ptr; - socklen_t addr_len = sizeof(data->local_addr); - - if (getsockname(b->num, &data->local_addr.sa, &addr_len) < 0) - /* - * This should not be possible, but zero-initialize and return - * anyway. - */ - BIO_ADDR_clear(&data->local_addr); -} - -# if M_METHOD == M_METHOD_RECVMMSG || M_METHOD == M_METHOD_RECVMSG || M_METHOD == M_METHOD_WSARECVMSG -static int dgram_get_sock_family(BIO *b) -{ - bio_dgram_data *data = (bio_dgram_data *)b->ptr; - return data->local_addr.sa.sa_family; -} -# endif - static void dgram_reset_rcv_timeout(BIO *b) { # if defined(SO_RCVTIMEO) bio_dgram_data *data = (bio_dgram_data *)b->ptr; /* Is a timer active? */ - if (!ossl_time_is_zero(data->next_timeout)) { + if (data->next_timeout.tv_sec > 0 || data->next_timeout.tv_usec > 0) { # ifdef OPENSSL_SYS_WINDOWS - int timeout = (int)ossl_time2ms(data->socket_timeout); - + int timeout = data->socket_timeout.tv_sec * 1000 + + data->socket_timeout.tv_usec / 1000; if (setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, - (void *)&timeout, sizeof(timeout)) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + (void *)&timeout, sizeof(timeout)) < 0) { + perror("setsockopt"); + } # else - struct timeval tv = ossl_time_to_timeval(data->socket_timeout); - - if (setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + if (setsockopt + (b->num, SOL_SOCKET, SO_RCVTIMEO, &(data->socket_timeout), + sizeof(struct timeval)) < 0) { + perror("setsockopt"); + } # endif } # endif @@ -416,7 +307,7 @@ static int dgram_read(BIO *b, char *out, int outl) if (out != NULL) { clear_socket_error(); - BIO_ADDR_clear(&peer); + memset(&peer, 0, sizeof(peer)); dgram_adjust_rcv_timeout(b); if (data->peekmode) flags = MSG_PEEK; @@ -503,56 +394,12 @@ static long dgram_get_mtu_overhead(bio_dgram_data *data) return ret; } -/* Enables appropriate destination address reception option on the socket. */ -# if defined(SUPPORT_LOCAL_ADDR) -static int enable_local_addr(BIO *b, int enable) { - int af = dgram_get_sock_family(b); - - if (af == AF_INET) { -# if defined(IP_PKTINFO) - /* IP_PKTINFO is preferred */ - if (setsockopt(b->num, IPPROTO_IP, IP_PKTINFO, - (void *)&enable, sizeof(enable)) < 0) - return 0; - - return 1; - -# elif defined(IP_RECVDSTADDR) - /* Fall back to IP_RECVDSTADDR */ - - if (setsockopt(b->num, IPPROTO_IP, IP_RECVDSTADDR, - &enable, sizeof(enable)) < 0) - return 0; - - return 1; -# endif - } - -# if OPENSSL_USE_IPV6 - if (af == AF_INET6) { -# if defined(IPV6_RECVPKTINFO) - if (setsockopt(b->num, IPPROTO_IPV6, IPV6_RECVPKTINFO, - &enable, sizeof(enable)) < 0) - return 0; - - return 1; -# endif - } -# endif - - return 0; -} -# endif - static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) { long ret = 1; int *ip; bio_dgram_data *data = NULL; -# ifndef __DJGPP__ - /* There are currently no cases where this is used on djgpp/watt32. */ int sockopt_val = 0; -# endif int d_errno; # if defined(OPENSSL_SYS_LINUX) && (defined(IP_MTU_DISCOVER) || defined(IP_MTU)) socklen_t sockopt_len; /* assume that system supporting IP_MTU is @@ -560,8 +407,6 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) socklen_t addr_len; BIO_ADDR addr; # endif - struct sockaddr_storage ss; - socklen_t ss_len = sizeof(ss); data = (bio_dgram_data *)b->ptr; @@ -578,17 +423,6 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) b->num = *((int *)ptr); b->shutdown = (int)num; b->init = 1; - dgram_update_local_addr(b); - if (getpeername(b->num, (struct sockaddr *)&ss, &ss_len) == 0) { - BIO_ADDR_make(&data->peer, BIO_ADDR_sockaddr((BIO_ADDR *)&ss)); - data->connected = 1; - } -# if defined(SUPPORT_LOCAL_ADDR) - if (data->local_addr_enabled) { - if (enable_local_addr(b, 1) < 1) - data->local_addr_enabled = 0; - } -# endif break; case BIO_C_GET_FD: if (b->init) { @@ -620,7 +454,7 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) case BIO_CTRL_DGRAM_MTU_DISCOVER: # if defined(OPENSSL_SYS_LINUX) && defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) addr_len = (socklen_t) sizeof(addr); - BIO_ADDR_clear(&addr); + memset(&addr, 0, sizeof(addr)); if (getsockname(b->num, &addr.sa, &addr_len) < 0) { ret = 0; break; @@ -630,16 +464,14 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) sockopt_val = IP_PMTUDISC_DO; if ((ret = setsockopt(b->num, IPPROTO_IP, IP_MTU_DISCOVER, &sockopt_val, sizeof(sockopt_val))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + perror("setsockopt"); break; # if OPENSSL_USE_IPV6 && defined(IPV6_MTU_DISCOVER) && defined(IPV6_PMTUDISC_DO) case AF_INET6: sockopt_val = IPV6_PMTUDISC_DO; if ((ret = setsockopt(b->num, IPPROTO_IPV6, IPV6_MTU_DISCOVER, &sockopt_val, sizeof(sockopt_val))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + perror("setsockopt"); break; # endif default: @@ -653,7 +485,7 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) case BIO_CTRL_DGRAM_QUERY_MTU: # if defined(OPENSSL_SYS_LINUX) && defined(IP_MTU) addr_len = (socklen_t) sizeof(addr); - BIO_ADDR_clear(&addr); + memset(&addr, 0, sizeof(addr)); if (getsockname(b->num, &addr.sa, &addr_len) < 0) { ret = 0; break; @@ -736,7 +568,7 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) BIO_ADDR_make(&data->peer, BIO_ADDR_sockaddr((BIO_ADDR *)ptr)); } else { data->connected = 0; - BIO_ADDR_clear(&data->peer); + memset(&data->peer, 0, sizeof(data->peer)); } break; case BIO_CTRL_DGRAM_GET_PEER: @@ -750,34 +582,8 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) case BIO_CTRL_DGRAM_SET_PEER: BIO_ADDR_make(&data->peer, BIO_ADDR_sockaddr((BIO_ADDR *)ptr)); break; - case BIO_CTRL_DGRAM_DETECT_PEER_ADDR: - { - BIO_ADDR xaddr, *p = &data->peer; - socklen_t xaddr_len = sizeof(xaddr.sa); - - if (BIO_ADDR_family(p) == AF_UNSPEC) { - if (getpeername(b->num, (void *)&xaddr.sa, &xaddr_len) == 0 - && BIO_ADDR_family(&xaddr) != AF_UNSPEC) { - p = &xaddr; - } else { - ret = 0; - break; - } - } - - ret = BIO_ADDR_sockaddr_size(p); - if (num == 0 || num > ret) - num = ret; - - memcpy(ptr, p, (ret = num)); - } - break; - case BIO_C_SET_NBIO: - if (!BIO_socket_nbio(b->num, num != 0)) - ret = 0; - break; case BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT: - data->next_timeout = ossl_time_from_timeval(*(struct timeval *)ptr); + memcpy(&(data->next_timeout), ptr, sizeof(struct timeval)); break; # if defined(SO_RCVTIMEO) case BIO_CTRL_DGRAM_SET_RECV_TIMEOUT: @@ -785,46 +591,53 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) { struct timeval *tv = (struct timeval *)ptr; int timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; - - if ((ret = setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, - (void *)&timeout, sizeof(timeout))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + if (setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, + (void *)&timeout, sizeof(timeout)) < 0) { + perror("setsockopt"); + ret = -1; + } } # else - if ((ret = setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, ptr, - sizeof(struct timeval))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + if (setsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, ptr, + sizeof(struct timeval)) < 0) { + perror("setsockopt"); + ret = -1; + } # endif break; case BIO_CTRL_DGRAM_GET_RECV_TIMEOUT: { + union { + size_t s; + int i; + } sz = { + 0 + }; # ifdef OPENSSL_SYS_WINDOWS - int sz = 0; int timeout; struct timeval *tv = (struct timeval *)ptr; - sz = sizeof(timeout); - if ((ret = getsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, - (void *)&timeout, &sz)) < 0) { - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling getsockopt()"); + sz.i = sizeof(timeout); + if (getsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, + (void *)&timeout, &sz.i) < 0) { + perror("getsockopt"); + ret = -1; } else { tv->tv_sec = timeout / 1000; tv->tv_usec = (timeout % 1000) * 1000; ret = sizeof(*tv); } # else - socklen_t sz = sizeof(struct timeval); - if ((ret = getsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, - ptr, &sz)) < 0) { - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling getsockopt()"); - } else { - OPENSSL_assert((size_t)sz <= sizeof(struct timeval)); - ret = (int)sz; - } + sz.i = sizeof(struct timeval); + if (getsockopt(b->num, SOL_SOCKET, SO_RCVTIMEO, + ptr, (void *)&sz) < 0) { + perror("getsockopt"); + ret = -1; + } else if (sizeof(sz.s) != sizeof(sz.i) && sz.i == 0) { + OPENSSL_assert(sz.s <= sizeof(struct timeval)); + ret = (int)sz.s; + } else + ret = sz.i; # endif } break; @@ -835,47 +648,53 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) { struct timeval *tv = (struct timeval *)ptr; int timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; - - if ((ret = setsockopt(b->num, SOL_SOCKET, SO_SNDTIMEO, - (void *)&timeout, sizeof(timeout))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + if (setsockopt(b->num, SOL_SOCKET, SO_SNDTIMEO, + (void *)&timeout, sizeof(timeout)) < 0) { + perror("setsockopt"); + ret = -1; + } } # else - if ((ret = setsockopt(b->num, SOL_SOCKET, SO_SNDTIMEO, ptr, - sizeof(struct timeval))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + if (setsockopt(b->num, SOL_SOCKET, SO_SNDTIMEO, ptr, + sizeof(struct timeval)) < 0) { + perror("setsockopt"); + ret = -1; + } # endif break; case BIO_CTRL_DGRAM_GET_SEND_TIMEOUT: { + union { + size_t s; + int i; + } sz = { + 0 + }; # ifdef OPENSSL_SYS_WINDOWS - int sz = 0; int timeout; struct timeval *tv = (struct timeval *)ptr; - sz = sizeof(timeout); - if ((ret = getsockopt(b->num, SOL_SOCKET, SO_SNDTIMEO, - (void *)&timeout, &sz)) < 0) { - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling getsockopt()"); + sz.i = sizeof(timeout); + if (getsockopt(b->num, SOL_SOCKET, SO_SNDTIMEO, + (void *)&timeout, &sz.i) < 0) { + perror("getsockopt"); + ret = -1; } else { tv->tv_sec = timeout / 1000; tv->tv_usec = (timeout % 1000) * 1000; ret = sizeof(*tv); } # else - socklen_t sz = sizeof(struct timeval); - - if ((ret = getsockopt(b->num, SOL_SOCKET, SO_SNDTIMEO, - ptr, &sz)) < 0) { - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling getsockopt()"); - } else { - OPENSSL_assert((size_t)sz <= sizeof(struct timeval)); - ret = (int)sz; - } + sz.i = sizeof(struct timeval); + if (getsockopt(b->num, SOL_SOCKET, SO_SNDTIMEO, + ptr, (void *)&sz) < 0) { + perror("getsockopt"); + ret = -1; + } else if (sizeof(sz.s) != sizeof(sz.i) && sz.i == 0) { + OPENSSL_assert(sz.s <= sizeof(struct timeval)); + ret = (int)sz.s; + } else + ret = sz.i; # endif } break; @@ -904,27 +723,30 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) break; # endif case BIO_CTRL_DGRAM_SET_DONT_FRAG: + sockopt_val = num ? 1 : 0; + switch (data->peer.sa.sa_family) { case AF_INET: # if defined(IP_DONTFRAG) - sockopt_val = num ? 1 : 0; if ((ret = setsockopt(b->num, IPPROTO_IP, IP_DONTFRAG, - &sockopt_val, sizeof(sockopt_val))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + &sockopt_val, sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } # elif defined(OPENSSL_SYS_LINUX) && defined(IP_MTU_DISCOVER) && defined (IP_PMTUDISC_PROBE) - sockopt_val = num ? IP_PMTUDISC_PROBE : IP_PMTUDISC_DONT; - if ((ret = setsockopt(b->num, IPPROTO_IP, IP_MTU_DISCOVER, - &sockopt_val, sizeof(sockopt_val))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + if ((sockopt_val = num ? IP_PMTUDISC_PROBE : IP_PMTUDISC_DONT), + (ret = setsockopt(b->num, IPPROTO_IP, IP_MTU_DISCOVER, + &sockopt_val, sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } # elif defined(OPENSSL_SYS_WINDOWS) && defined(IP_DONTFRAGMENT) - sockopt_val = num ? 1 : 0; if ((ret = setsockopt(b->num, IPPROTO_IP, IP_DONTFRAGMENT, (const char *)&sockopt_val, - sizeof(sockopt_val))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } # else ret = -1; # endif @@ -932,19 +754,19 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) # if OPENSSL_USE_IPV6 case AF_INET6: # if defined(IPV6_DONTFRAG) - sockopt_val = num ? 1 : 0; if ((ret = setsockopt(b->num, IPPROTO_IPV6, IPV6_DONTFRAG, (const void *)&sockopt_val, - sizeof(sockopt_val))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); - + sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } # elif defined(OPENSSL_SYS_LINUX) && defined(IPV6_MTUDISCOVER) - sockopt_val = num ? IP_PMTUDISC_PROBE : IP_PMTUDISC_DONT; - if ((ret = setsockopt(b->num, IPPROTO_IPV6, IPV6_MTU_DISCOVER, - &sockopt_val, sizeof(sockopt_val))) < 0) - ERR_raise_data(ERR_LIB_SYS, get_last_socket_error(), - "calling setsockopt()"); + if ((sockopt_val = num ? IP_PMTUDISC_PROBE : IP_PMTUDISC_DONT), + (ret = setsockopt(b->num, IPPROTO_IPV6, IPV6_MTU_DISCOVER, + &sockopt_val, sizeof(sockopt_val))) < 0) { + perror("setsockopt"); + ret = -1; + } # else ret = -1; # endif @@ -970,59 +792,10 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr) case BIO_CTRL_DGRAM_SET_PEEK_MODE: data->peekmode = (unsigned int)num; break; - - case BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP: -# if defined(SUPPORT_LOCAL_ADDR) - ret = 1; -# else - ret = 0; -# endif - break; - - case BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE: -# if defined(SUPPORT_LOCAL_ADDR) - num = num > 0; - if (num != data->local_addr_enabled) { - if (enable_local_addr(b, num) < 1) { - ret = 0; - break; - } - - data->local_addr_enabled = (char)num; - } -# else - ret = 0; -# endif - break; - - case BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE: - *(int *)ptr = data->local_addr_enabled; - break; - - case BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS: - ret = (long)(BIO_DGRAM_CAP_HANDLES_DST_ADDR - | BIO_DGRAM_CAP_HANDLES_SRC_ADDR - | BIO_DGRAM_CAP_PROVIDES_DST_ADDR - | BIO_DGRAM_CAP_PROVIDES_SRC_ADDR); - break; - - case BIO_CTRL_GET_RPOLL_DESCRIPTOR: - case BIO_CTRL_GET_WPOLL_DESCRIPTOR: - { - BIO_POLL_DESCRIPTOR *pd = ptr; - - pd->type = BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD; - pd->value.fd = b->num; - } - break; - default: ret = 0; break; } - /* Normalize if error */ - if (ret < 0) - ret = -1; return ret; } @@ -1035,726 +808,6 @@ static int dgram_puts(BIO *bp, const char *str) return ret; } -# if M_METHOD == M_METHOD_WSARECVMSG -static void translate_msg_win(BIO *b, WSAMSG *mh, WSABUF *iov, - unsigned char *control, BIO_MSG *msg) -{ - iov->len = msg->data_len; - iov->buf = msg->data; - - /* Windows requires namelen to be set exactly */ - mh->name = msg->peer != NULL ? &msg->peer->sa : NULL; - if (msg->peer != NULL && dgram_get_sock_family(b) == AF_INET) - mh->namelen = sizeof(struct sockaddr_in); -# if OPENSSL_USE_IPV6 - else if (msg->peer != NULL && dgram_get_sock_family(b) == AF_INET6) - mh->namelen = sizeof(struct sockaddr_in6); -# endif - else - mh->namelen = 0; - - /* - * When local address reception (IP_PKTINFO, etc.) is enabled, on Windows - * this causes WSARecvMsg to fail if the control buffer is too small to hold - * the structure, or if no control buffer is passed. So we need to give it - * the control buffer even if we aren't actually going to examine the - * result. - */ - mh->lpBuffers = iov; - mh->dwBufferCount = 1; - mh->Control.len = BIO_CMSG_ALLOC_LEN; - mh->Control.buf = control; - mh->dwFlags = 0; -} -# endif - -# if M_METHOD == M_METHOD_RECVMMSG || M_METHOD == M_METHOD_RECVMSG -/* Translates a BIO_MSG to a msghdr and iovec. */ -static void translate_msg(BIO *b, struct msghdr *mh, struct iovec *iov, - unsigned char *control, BIO_MSG *msg) -{ - bio_dgram_data *data; - - iov->iov_base = msg->data; - iov->iov_len = msg->data_len; - - data = (bio_dgram_data *)b->ptr; - if (data->connected == 0) { - /* macOS requires msg_namelen be 0 if msg_name is NULL */ - mh->msg_name = msg->peer != NULL ? &msg->peer->sa : NULL; - if (msg->peer != NULL && dgram_get_sock_family(b) == AF_INET) - mh->msg_namelen = sizeof(struct sockaddr_in); -# if OPENSSL_USE_IPV6 - else if (msg->peer != NULL && dgram_get_sock_family(b) == AF_INET6) - mh->msg_namelen = sizeof(struct sockaddr_in6); -# endif - else - mh->msg_namelen = 0; - } else { - mh->msg_name = NULL; - mh->msg_namelen = 0; - } - - mh->msg_iov = iov; - mh->msg_iovlen = 1; - mh->msg_control = msg->local != NULL ? control : NULL; - mh->msg_controllen = msg->local != NULL ? BIO_CMSG_ALLOC_LEN : 0; - mh->msg_flags = 0; -} -# endif - -# if M_METHOD == M_METHOD_RECVMMSG || M_METHOD == M_METHOD_RECVMSG || M_METHOD == M_METHOD_WSARECVMSG -/* Extracts destination address from the control buffer. */ -static int extract_local(BIO *b, MSGHDR_TYPE *mh, BIO_ADDR *local) { -# if defined(IP_PKTINFO) || defined(IP_RECVDSTADDR) || defined(IPV6_PKTINFO) - CMSGHDR_TYPE *cmsg; - int af = dgram_get_sock_family(b); - - for (cmsg = BIO_CMSG_FIRSTHDR(mh); cmsg != NULL; - cmsg = BIO_CMSG_NXTHDR(mh, cmsg)) { - if (af == AF_INET) { - if (cmsg->cmsg_level != IPPROTO_IP) - continue; - -# if defined(IP_PKTINFO) - if (cmsg->cmsg_type != IP_PKTINFO) - continue; - - local->s_in.sin_addr = - ((struct in_pktinfo *)BIO_CMSG_DATA(cmsg))->ipi_addr; - -# elif defined(IP_RECVDSTADDR) - if (cmsg->cmsg_type != IP_RECVDSTADDR) - continue; - - local->s_in.sin_addr = *(struct in_addr *)BIO_CMSG_DATA(cmsg); -# endif - -# if defined(IP_PKTINFO) || defined(IP_RECVDSTADDR) - { - bio_dgram_data *data = b->ptr; - - local->s_in.sin_family = AF_INET; - local->s_in.sin_port = data->local_addr.s_in.sin_port; - } - return 1; -# endif - } -# if OPENSSL_USE_IPV6 - else if (af == AF_INET6) { - if (cmsg->cmsg_level != IPPROTO_IPV6) - continue; - -# if defined(IPV6_RECVPKTINFO) - if (cmsg->cmsg_type != IPV6_PKTINFO) - continue; - - { - bio_dgram_data *data = b->ptr; - - local->s_in6.sin6_addr = - ((struct in6_pktinfo *)BIO_CMSG_DATA(cmsg))->ipi6_addr; - local->s_in6.sin6_family = AF_INET6; - local->s_in6.sin6_port = data->local_addr.s_in6.sin6_port; - local->s_in6.sin6_scope_id = - data->local_addr.s_in6.sin6_scope_id; - local->s_in6.sin6_flowinfo = 0; - } - return 1; -# endif - } -# endif - } -# endif - - return 0; -} - -static int pack_local(BIO *b, MSGHDR_TYPE *mh, const BIO_ADDR *local) { - int af = dgram_get_sock_family(b); -# if defined(IP_PKTINFO) || defined(IP_RECVDSTADDR) || defined(IPV6_PKTINFO) - CMSGHDR_TYPE *cmsg; - bio_dgram_data *data = b->ptr; -# endif - - if (af == AF_INET) { -# if defined(IP_PKTINFO) - struct in_pktinfo *info; - -# if defined(OPENSSL_SYS_WINDOWS) - cmsg = (CMSGHDR_TYPE *)mh->Control.buf; -# else - cmsg = (CMSGHDR_TYPE *)mh->msg_control; -# endif - - cmsg->cmsg_len = BIO_CMSG_LEN(sizeof(struct in_pktinfo)); - cmsg->cmsg_level = IPPROTO_IP; - cmsg->cmsg_type = IP_PKTINFO; - - info = (struct in_pktinfo *)BIO_CMSG_DATA(cmsg); -# if !defined(OPENSSL_SYS_WINDOWS) && !defined(OPENSSL_SYS_CYGWIN) && !defined(__FreeBSD__) - info->ipi_spec_dst = local->s_in.sin_addr; -# endif - info->ipi_addr.s_addr = 0; - info->ipi_ifindex = 0; - - /* - * We cannot override source port using this API, therefore - * ensure the application specified a source port of 0 - * or the one we are bound to. (Better to error than silently - * ignore this.) - */ - if (local->s_in.sin_port != 0 - && data->local_addr.s_in.sin_port != local->s_in.sin_port) { - ERR_raise(ERR_LIB_BIO, BIO_R_PORT_MISMATCH); - return 0; - } - -# if defined(OPENSSL_SYS_WINDOWS) - mh->Control.len = BIO_CMSG_SPACE(sizeof(struct in_pktinfo)); -# else - mh->msg_controllen = BIO_CMSG_SPACE(sizeof(struct in_pktinfo)); -# endif - return 1; - -# elif defined(IP_SENDSRCADDR) - struct in_addr *info; - - /* - * At least FreeBSD is very pedantic about using IP_SENDSRCADDR when we - * are not bound to 0.0.0.0 or ::, even if the address matches what we - * bound to. Support this by not packing the structure if the address - * matches our understanding of our local address. IP_SENDSRCADDR is a - * BSD thing, so we don't need an explicit test for BSD here. - */ - if (local->s_in.sin_addr.s_addr == data->local_addr.s_in.sin_addr.s_addr) { - mh->msg_control = NULL; - mh->msg_controllen = 0; - return 1; - } - - cmsg = (struct cmsghdr *)mh->msg_control; - cmsg->cmsg_len = BIO_CMSG_LEN(sizeof(struct in_addr)); - cmsg->cmsg_level = IPPROTO_IP; - cmsg->cmsg_type = IP_SENDSRCADDR; - - info = (struct in_addr *)BIO_CMSG_DATA(cmsg); - *info = local->s_in.sin_addr; - - /* See comment above. */ - if (local->s_in.sin_port != 0 - && data->local_addr.s_in.sin_port != local->s_in.sin_port) { - ERR_raise(ERR_LIB_BIO, BIO_R_PORT_MISMATCH); - return 0; - } - - mh->msg_controllen = BIO_CMSG_SPACE(sizeof(struct in_addr)); - return 1; -# endif - } -# if OPENSSL_USE_IPV6 - else if (af == AF_INET6) { -# if defined(IPV6_PKTINFO) - struct in6_pktinfo *info; - -# if defined(OPENSSL_SYS_WINDOWS) - cmsg = (CMSGHDR_TYPE *)mh->Control.buf; -# else - cmsg = (CMSGHDR_TYPE *)mh->msg_control; -# endif - cmsg->cmsg_len = BIO_CMSG_LEN(sizeof(struct in6_pktinfo)); - cmsg->cmsg_level = IPPROTO_IPV6; - cmsg->cmsg_type = IPV6_PKTINFO; - - info = (struct in6_pktinfo *)BIO_CMSG_DATA(cmsg); - info->ipi6_addr = local->s_in6.sin6_addr; - info->ipi6_ifindex = 0; - - /* - * See comment above, but also applies to the other fields - * in sockaddr_in6. - */ - if (local->s_in6.sin6_port != 0 - && data->local_addr.s_in6.sin6_port != local->s_in6.sin6_port) { - ERR_raise(ERR_LIB_BIO, BIO_R_PORT_MISMATCH); - return 0; - } - - if (local->s_in6.sin6_scope_id != 0 - && data->local_addr.s_in6.sin6_scope_id != local->s_in6.sin6_scope_id) { - ERR_raise(ERR_LIB_BIO, BIO_R_PORT_MISMATCH); - return 0; - } - -# if defined(OPENSSL_SYS_WINDOWS) - mh->Control.len = BIO_CMSG_SPACE(sizeof(struct in6_pktinfo)); -# else - mh->msg_controllen = BIO_CMSG_SPACE(sizeof(struct in6_pktinfo)); -# endif - return 1; -# endif - } -# endif - - return 0; -} -# endif - -/* - * Converts flags passed to BIO_sendmmsg or BIO_recvmmsg to syscall flags. You - * should mask out any system flags returned by this function you cannot support - * in a particular circumstance. Currently no flags are defined. - */ -# if M_METHOD != M_METHOD_NONE -static int translate_flags(uint64_t flags) { - return 0; -} -# endif - -static int dgram_sendmmsg(BIO *b, BIO_MSG *msg, size_t stride, - size_t num_msg, uint64_t flags, size_t *num_processed) -{ -# if M_METHOD != M_METHOD_NONE && M_METHOD != M_METHOD_RECVMSG - int ret; -# endif -# if M_METHOD == M_METHOD_RECVMMSG -# define BIO_MAX_MSGS_PER_CALL 64 - int sysflags; - bio_dgram_data *data = (bio_dgram_data *)b->ptr; - size_t i; - struct mmsghdr mh[BIO_MAX_MSGS_PER_CALL]; - struct iovec iov[BIO_MAX_MSGS_PER_CALL]; - unsigned char control[BIO_MAX_MSGS_PER_CALL][BIO_CMSG_ALLOC_LEN]; - int have_local_enabled = data->local_addr_enabled; -# elif M_METHOD == M_METHOD_RECVMSG - int sysflags; - bio_dgram_data *data = (bio_dgram_data *)b->ptr; - ossl_ssize_t l; - struct msghdr mh; - struct iovec iov; - unsigned char control[BIO_CMSG_ALLOC_LEN]; - int have_local_enabled = data->local_addr_enabled; -# elif M_METHOD == M_METHOD_WSARECVMSG - bio_dgram_data *data = (bio_dgram_data *)b->ptr; - int have_local_enabled = data->local_addr_enabled; - WSAMSG wmsg; - WSABUF wbuf; - DWORD num_bytes_sent = 0; - unsigned char control[BIO_CMSG_ALLOC_LEN]; -# endif -# if M_METHOD == M_METHOD_RECVFROM || M_METHOD == M_METHOD_WSARECVMSG - int sysflags; -# endif - - if (num_msg == 0) { - *num_processed = 0; - return 1; - } - - if (num_msg > OSSL_SSIZE_MAX) - num_msg = OSSL_SSIZE_MAX; - -# if M_METHOD != M_METHOD_NONE - sysflags = translate_flags(flags); -# endif - -# if M_METHOD == M_METHOD_RECVMMSG - /* - * In the sendmmsg/recvmmsg case, we need to allocate our translated struct - * msghdr and struct iovec on the stack to support multithreaded use. Thus - * we place a fixed limit on the number of messages per call, in the - * expectation that we will be called again if there were more messages to - * be sent. - */ - if (num_msg > BIO_MAX_MSGS_PER_CALL) - num_msg = BIO_MAX_MSGS_PER_CALL; - - for (i = 0; i < num_msg; ++i) { - translate_msg(b, &mh[i].msg_hdr, &iov[i], - control[i], &BIO_MSG_N(msg, stride, i)); - - /* If local address was requested, it must have been enabled */ - if (BIO_MSG_N(msg, stride, i).local != NULL) { - if (!have_local_enabled) { - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - - if (pack_local(b, &mh[i].msg_hdr, - BIO_MSG_N(msg, stride, i).local) < 1) { - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - } - } - - /* Do the batch */ - ret = sendmmsg(b->num, mh, num_msg, sysflags); - if (ret < 0) { - ERR_raise(ERR_LIB_SYS, get_last_socket_error()); - *num_processed = 0; - return 0; - } - - for (i = 0; i < (size_t)ret; ++i) { - BIO_MSG_N(msg, stride, i).data_len = mh[i].msg_len; - BIO_MSG_N(msg, stride, i).flags = 0; - } - - *num_processed = (size_t)ret; - return 1; - -# elif M_METHOD == M_METHOD_RECVMSG - /* - * If sendmsg is available, use it. - */ - translate_msg(b, &mh, &iov, control, msg); - - if (msg->local != NULL) { - if (!have_local_enabled) { - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - - if (pack_local(b, &mh, msg->local) < 1) { - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - } - - l = sendmsg(b->num, &mh, sysflags); - if (l < 0) { - ERR_raise(ERR_LIB_SYS, get_last_socket_error()); - *num_processed = 0; - return 0; - } - - msg->data_len = (size_t)l; - msg->flags = 0; - *num_processed = 1; - return 1; - -# elif M_METHOD == M_METHOD_WSARECVMSG || M_METHOD == M_METHOD_RECVFROM -# if M_METHOD == M_METHOD_WSARECVMSG - if (bio_WSASendMsg != NULL) { - /* WSASendMsg-based implementation for Windows. */ - translate_msg_win(b, &wmsg, &wbuf, control, msg); - - if (msg[0].local != NULL) { - if (!have_local_enabled) { - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - - if (pack_local(b, &wmsg, msg[0].local) < 1) { - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - } - - ret = WSASendMsg((SOCKET)b->num, &wmsg, 0, &num_bytes_sent, NULL, NULL); - if (ret < 0) { - ERR_raise(ERR_LIB_SYS, get_last_socket_error()); - *num_processed = 0; - return 0; - } - - msg[0].data_len = num_bytes_sent; - msg[0].flags = 0; - *num_processed = 1; - return 1; - } -# endif - - /* - * Fallback to sendto and send a single message. - */ - if (msg[0].local != NULL) { - /* - * We cannot set the local address if using sendto - * so fail in this case - */ - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - - ret = sendto(b->num, msg[0].data, -# if defined(OPENSSL_SYS_WINDOWS) - (int)msg[0].data_len, -# else - msg[0].data_len, -# endif - sysflags, - msg[0].peer != NULL ? BIO_ADDR_sockaddr(msg[0].peer) : NULL, - msg[0].peer != NULL ? BIO_ADDR_sockaddr_size(msg[0].peer) : 0); - if (ret <= 0) { - ERR_raise(ERR_LIB_SYS, get_last_socket_error()); - *num_processed = 0; - return 0; - } - - msg[0].data_len = ret; - msg[0].flags = 0; - *num_processed = 1; - return 1; - -# else - ERR_raise(ERR_LIB_BIO, BIO_R_UNSUPPORTED_METHOD); - *num_processed = 0; - return 0; -# endif -} - -static int dgram_recvmmsg(BIO *b, BIO_MSG *msg, - size_t stride, size_t num_msg, - uint64_t flags, size_t *num_processed) -{ -# if M_METHOD != M_METHOD_NONE && M_METHOD != M_METHOD_RECVMSG - int ret; -# endif -# if M_METHOD == M_METHOD_RECVMMSG - int sysflags; - bio_dgram_data *data = (bio_dgram_data *)b->ptr; - size_t i; - struct mmsghdr mh[BIO_MAX_MSGS_PER_CALL]; - struct iovec iov[BIO_MAX_MSGS_PER_CALL]; - unsigned char control[BIO_MAX_MSGS_PER_CALL][BIO_CMSG_ALLOC_LEN]; - int have_local_enabled = data->local_addr_enabled; -# elif M_METHOD == M_METHOD_RECVMSG - int sysflags; - bio_dgram_data *data = (bio_dgram_data *)b->ptr; - ossl_ssize_t l; - struct msghdr mh; - struct iovec iov; - unsigned char control[BIO_CMSG_ALLOC_LEN]; - int have_local_enabled = data->local_addr_enabled; -# elif M_METHOD == M_METHOD_WSARECVMSG - bio_dgram_data *data = (bio_dgram_data *)b->ptr; - int have_local_enabled = data->local_addr_enabled; - WSAMSG wmsg; - WSABUF wbuf; - DWORD num_bytes_received = 0; - unsigned char control[BIO_CMSG_ALLOC_LEN]; -# endif -# if M_METHOD == M_METHOD_RECVFROM || M_METHOD == M_METHOD_WSARECVMSG - int sysflags; - socklen_t slen; -# endif - - if (num_msg == 0) { - *num_processed = 0; - return 1; - } - - if (num_msg > OSSL_SSIZE_MAX) - num_msg = OSSL_SSIZE_MAX; - -# if M_METHOD != M_METHOD_NONE - sysflags = translate_flags(flags); -# endif - -# if M_METHOD == M_METHOD_RECVMMSG - /* - * In the sendmmsg/recvmmsg case, we need to allocate our translated struct - * msghdr and struct iovec on the stack to support multithreaded use. Thus - * we place a fixed limit on the number of messages per call, in the - * expectation that we will be called again if there were more messages to - * be sent. - */ - if (num_msg > BIO_MAX_MSGS_PER_CALL) - num_msg = BIO_MAX_MSGS_PER_CALL; - - for (i = 0; i < num_msg; ++i) { - translate_msg(b, &mh[i].msg_hdr, &iov[i], - control[i], &BIO_MSG_N(msg, stride, i)); - - /* If local address was requested, it must have been enabled */ - if (BIO_MSG_N(msg, stride, i).local != NULL && !have_local_enabled) { - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - } - - /* Do the batch */ - ret = recvmmsg(b->num, mh, num_msg, sysflags, NULL); - if (ret < 0) { - ERR_raise(ERR_LIB_SYS, get_last_socket_error()); - *num_processed = 0; - return 0; - } - - for (i = 0; i < (size_t)ret; ++i) { - BIO_MSG_N(msg, stride, i).data_len = mh[i].msg_len; - BIO_MSG_N(msg, stride, i).flags = 0; - /* - * *(msg->peer) will have been filled in by recvmmsg; - * for msg->local we parse the control data returned - */ - if (BIO_MSG_N(msg, stride, i).local != NULL) - if (extract_local(b, &mh[i].msg_hdr, - BIO_MSG_N(msg, stride, i).local) < 1) - /* - * It appears BSDs do not support local addresses for - * loopback sockets. In this case, just clear the local - * address, as for OS X and Windows in some circumstances - * (see below). - */ - BIO_ADDR_clear(msg->local); - } - - *num_processed = (size_t)ret; - return 1; - -# elif M_METHOD == M_METHOD_RECVMSG - /* - * If recvmsg is available, use it. - */ - translate_msg(b, &mh, &iov, control, msg); - - /* If local address was requested, it must have been enabled */ - if (msg->local != NULL && !have_local_enabled) { - /* - * If we have done at least one message, we must return the - * count; if we haven't done any, we can give an error code - */ - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - - l = recvmsg(b->num, &mh, sysflags); - if (l < 0) { - ERR_raise(ERR_LIB_SYS, get_last_socket_error()); - *num_processed = 0; - return 0; - } - - msg->data_len = (size_t)l; - msg->flags = 0; - - if (msg->local != NULL) - if (extract_local(b, &mh, msg->local) < 1) - /* - * OS X exhibits odd behaviour where it appears that if a packet is - * sent before the receiving interface enables IP_PKTINFO, it will - * sometimes not have any control data returned even if the - * receiving interface enables IP_PKTINFO before calling recvmsg(). - * This appears to occur non-deterministically. Presumably, OS X - * handles IP_PKTINFO at the time the packet is enqueued into a - * socket's receive queue, rather than at the time recvmsg() is - * called, unlike most other operating systems. Thus (if this - * hypothesis is correct) there is a race between where IP_PKTINFO - * is enabled by the process and when the kernel's network stack - * queues the incoming message. - * - * We cannot return the local address if we do not have it, but this - * is not a caller error either, so just return a zero address - * structure. This is similar to how we handle Windows loopback - * interfaces (see below). We enable this workaround for all - * platforms, not just Apple, as this kind of quirk in OS networking - * stacks seems to be common enough that failing hard if a local - * address is not provided appears to be too brittle. - */ - BIO_ADDR_clear(msg->local); - - *num_processed = 1; - return 1; - -# elif M_METHOD == M_METHOD_RECVFROM || M_METHOD == M_METHOD_WSARECVMSG -# if M_METHOD == M_METHOD_WSARECVMSG - if (bio_WSARecvMsg != NULL) { - /* WSARecvMsg-based implementation for Windows. */ - translate_msg_win(b, &wmsg, &wbuf, control, msg); - - /* If local address was requested, it must have been enabled */ - if (msg[0].local != NULL && !have_local_enabled) { - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - - ret = WSARecvMsg((SOCKET)b->num, &wmsg, &num_bytes_received, NULL, NULL); - if (ret < 0) { - ERR_raise(ERR_LIB_SYS, get_last_socket_error()); - *num_processed = 0; - return 0; - } - - msg[0].data_len = num_bytes_received; - msg[0].flags = 0; - if (msg[0].local != NULL) - if (extract_local(b, &wmsg, msg[0].local) < 1) - /* - * On Windows, loopback is not a "proper" interface and it works - * differently; packets are essentially short-circuited and - * don't go through all of the normal processing. A consequence - * of this is that packets sent from the local machine to the - * local machine _will not have IP_PKTINFO_ even if the - * IP_PKTINFO socket option is enabled. WSARecvMsg just sets - * Control.len to 0 on returning. - * - * This applies regardless of whether the loopback address, - * 127.0.0.1 is used, or a local interface address (e.g. - * 192.168.1.1); in both cases IP_PKTINFO will not be present. - * - * We report this condition by setting the local BIO_ADDR's - * family to 0. - */ - BIO_ADDR_clear(msg[0].local); - - *num_processed = 1; - return 1; - } -# endif - - /* - * Fallback to recvfrom and receive a single message. - */ - if (msg[0].local != NULL) { - /* - * We cannot determine the local address if using recvfrom - * so fail in this case - */ - ERR_raise(ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE); - *num_processed = 0; - return 0; - } - - slen = sizeof(*msg[0].peer); - ret = recvfrom(b->num, msg[0].data, -# if defined(OPENSSL_SYS_WINDOWS) - (int)msg[0].data_len, -# else - msg[0].data_len, -# endif - sysflags, - msg[0].peer != NULL ? &msg[0].peer->sa : NULL, - msg[0].peer != NULL ? &slen : NULL); - if (ret <= 0) { - ERR_raise(ERR_LIB_SYS, get_last_socket_error()); - return 0; - } - - msg[0].data_len = ret; - msg[0].flags = 0; - *num_processed = 1; - return 1; - -# else - ERR_raise(ERR_LIB_BIO, BIO_R_UNSUPPORTED_METHOD); - *num_processed = 0; - return 0; -# endif -} - # ifndef OPENSSL_NO_SCTP const BIO_METHOD *BIO_s_datagram_sctp(void) { @@ -1903,8 +956,10 @@ static int dgram_sctp_new(BIO *bi) bi->init = 0; bi->num = 0; - if ((data = OPENSSL_zalloc(sizeof(*data))) == NULL) + if ((data = OPENSSL_zalloc(sizeof(*data))) == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } # ifdef SCTP_PR_SCTP_NONE data->prinfo.pr_policy = SCTP_PR_SCTP_NONE; # endif @@ -2126,7 +1181,7 @@ static int dgram_sctp_read(BIO *b, char *out, int outl) if (ret < 0) { if (BIO_dgram_should_retry(ret)) { BIO_set_retry_read(b); - data->dgram._errno = get_last_socket_error(); + data->_errno = get_last_socket_error(); } } @@ -2139,8 +1194,10 @@ static int dgram_sctp_read(BIO *b, char *out, int outl) optlen = (socklen_t) (sizeof(sctp_assoc_t) + 256 * sizeof(uint8_t)); authchunks = OPENSSL_malloc(optlen); - if (authchunks == NULL) + if (authchunks == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return -1; + } memset(authchunks, 0, optlen); ii = getsockopt(b->num, IPPROTO_SCTP, SCTP_PEER_AUTH_CHUNKS, authchunks, &optlen); @@ -2278,7 +1335,7 @@ static int dgram_sctp_write(BIO *b, const char *in, int inl) if (ret <= 0) { if (BIO_dgram_should_retry(ret)) { BIO_set_retry_write(b); - data->dgram._errno = get_last_socket_error(); + data->_errno = get_last_socket_error(); } } return ret; @@ -2300,16 +1357,16 @@ static long dgram_sctp_ctrl(BIO *b, int cmd, long num, void *ptr) * Set to maximum (2^14) and ignore user input to enable transport * protocol fragmentation. Returns always 2^14. */ - data->dgram.mtu = 16384; - ret = data->dgram.mtu; + data->mtu = 16384; + ret = data->mtu; break; case BIO_CTRL_DGRAM_SET_MTU: /* * Set to maximum (2^14) and ignore input to enable transport * protocol fragmentation. Returns always 2^14. */ - data->dgram.mtu = 16384; - ret = data->dgram.mtu; + data->mtu = 16384; + ret = data->mtu; break; case BIO_CTRL_DGRAM_SET_CONNECTED: case BIO_CTRL_DGRAM_CONNECT: @@ -2857,4 +1914,27 @@ int BIO_dgram_non_fatal_error(int err) return 0; } +static void get_current_time(struct timeval *t) +{ +# if defined(_WIN32) + SYSTEMTIME st; + union { + unsigned __int64 ul; + FILETIME ft; + } now; + + GetSystemTime(&st); + SystemTimeToFileTime(&st, &now.ft); +# ifdef __MINGW32__ + now.ul -= 116444736000000000ULL; +# else + now.ul -= 116444736000000000UI64; /* re-bias to 1/1/1970 */ +# endif + t->tv_sec = (long)(now.ul / 10000000); + t->tv_usec = ((int)(now.ul % 10000000)) / 10; +# else + gettimeofday(t, NULL); +# endif +} + #endif diff --git a/openssl/src/crypto/bio/bss_dgram_pair.c b/openssl/src/crypto/bio/bss_dgram_pair.c deleted file mode 100644 index 08dd802d8..000000000 --- a/openssl/src/crypto/bio/bss_dgram_pair.c +++ /dev/null @@ -1,1328 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "bio_local.h" -#include "internal/cryptlib.h" -#include "internal/safe_math.h" - -#if !defined(OPENSSL_NO_DGRAM) && !defined(OPENSSL_NO_SOCK) - -OSSL_SAFE_MATH_UNSIGNED(size_t, size_t) - -/* =========================================================================== - * Byte-wise ring buffer which supports pushing and popping blocks of multiple - * bytes at a time. - */ -struct ring_buf { - unsigned char *start; /* start of buffer */ - size_t len; /* size of buffer allocation in bytes */ - size_t count; /* number of bytes currently pushed */ - /* - * These index into start. Where idx[0] == idx[1], the buffer is full - * (if count is nonzero) and empty otherwise. - */ - size_t idx[2]; /* 0: head, 1: tail */ -}; - -static int ring_buf_init(struct ring_buf *r, size_t nbytes) -{ - r->start = OPENSSL_malloc(nbytes); - if (r->start == NULL) - return 0; - - r->len = nbytes; - r->idx[0] = r->idx[1] = r->count = 0; - return 1; -} - -static void ring_buf_destroy(struct ring_buf *r) -{ - OPENSSL_free(r->start); - r->start = NULL; - r->len = 0; - r->count = 0; -} - -/* - * Get a pointer to the next place to write data to be pushed to the ring buffer - * (idx=0), or the next data to be popped from the ring buffer (idx=1). The - * pointer is written to *buf and the maximum number of bytes which can be - * read/written are written to *len. After writing data to the buffer, call - * ring_buf_push/pop() with the number of bytes actually read/written, which - * must not exceed the returned length. - */ -static void ring_buf_head_tail(struct ring_buf *r, int idx, uint8_t **buf, size_t *len) -{ - size_t max_len = r->len - r->idx[idx]; - - if (idx == 0 && max_len > r->len - r->count) - max_len = r->len - r->count; - if (idx == 1 && max_len > r->count) - max_len = r->count; - - *buf = (uint8_t *)r->start + r->idx[idx]; - *len = max_len; -} - -#define ring_buf_head(r, buf, len) ring_buf_head_tail((r), 0, (buf), (len)) -#define ring_buf_tail(r, buf, len) ring_buf_head_tail((r), 1, (buf), (len)) - -/* - * Commit bytes to the ring buffer previously filled after a call to - * ring_buf_head(). - */ -static void ring_buf_push_pop(struct ring_buf *r, int idx, size_t num_bytes) -{ - size_t new_idx; - - /* A single push/pop op cannot wrap around, though it can reach the end. - * If the caller adheres to the convention of using the length returned - * by ring_buf_head/tail(), this cannot happen. - */ - if (!ossl_assert(num_bytes <= r->len - r->idx[idx])) - return; - - /* - * Must not overfill the buffer, or pop more than is in the buffer either. - */ - if (!ossl_assert(idx != 0 ? num_bytes <= r->count - : num_bytes + r->count <= r->len)) - return; - - /* Update the index. */ - new_idx = r->idx[idx] + num_bytes; - if (new_idx == r->len) - new_idx = 0; - - r->idx[idx] = new_idx; - if (idx != 0) - r->count -= num_bytes; - else - r->count += num_bytes; -} - -#define ring_buf_push(r, num_bytes) ring_buf_push_pop((r), 0, (num_bytes)) -#define ring_buf_pop(r, num_bytes) ring_buf_push_pop((r), 1, (num_bytes)) - -static void ring_buf_clear(struct ring_buf *r) -{ - r->idx[0] = r->idx[1] = r->count = 0; -} - -static int ring_buf_resize(struct ring_buf *r, size_t nbytes) -{ - unsigned char *new_start; - - if (r->start == NULL) - return ring_buf_init(r, nbytes); - - if (nbytes == r->len) - return 1; - - if (r->count > 0 && nbytes < r->len) - /* fail shrinking the ring buffer when there is any data in it */ - return 0; - - new_start = OPENSSL_realloc(r->start, nbytes); - if (new_start == NULL) - return 0; - - /* Moving tail if it is after (or equal to) head */ - if (r->count > 0) { - if (r->idx[0] <= r->idx[1]) { - size_t offset = nbytes - r->len; - - memmove(new_start + r->idx[1] + offset, new_start + r->idx[1], - r->len - r->idx[1]); - r->idx[1] += offset; - } - } else { - /* just reset the head/tail because it might be pointing outside */ - r->idx[0] = r->idx[1] = 0; - } - - r->start = new_start; - r->len = nbytes; - - return 1; -} - -/* =========================================================================== - * BIO_s_dgram_pair is documented in BIO_s_dgram_pair(3). - * - * INTERNAL DATA STRUCTURE - * - * This is managed internally by using a bytewise ring buffer which supports - * pushing and popping spans of multiple bytes at once. The ring buffer stores - * internal packets which look like this: - * - * struct dgram_hdr hdr; - * uint8_t data[]; - * - * The header contains the length of the data and metadata such as - * source/destination addresses. - * - * The datagram pair BIO is designed to support both traditional - * BIO_read/BIO_write (likely to be used by applications) as well as - * BIO_recvmmsg/BIO_sendmmsg. - */ -struct bio_dgram_pair_st; -static int dgram_pair_write(BIO *bio, const char *buf, int sz_); -static int dgram_pair_read(BIO *bio, char *buf, int sz_); -static int dgram_mem_read(BIO *bio, char *buf, int sz_); -static long dgram_pair_ctrl(BIO *bio, int cmd, long num, void *ptr); -static long dgram_mem_ctrl(BIO *bio, int cmd, long num, void *ptr); -static int dgram_pair_init(BIO *bio); -static int dgram_mem_init(BIO *bio); -static int dgram_pair_free(BIO *bio); -static int dgram_pair_sendmmsg(BIO *b, BIO_MSG *msg, size_t stride, - size_t num_msg, uint64_t flags, - size_t *num_processed); -static int dgram_pair_recvmmsg(BIO *b, BIO_MSG *msg, size_t stride, - size_t num_msg, uint64_t flags, - size_t *num_processed); - -static int dgram_pair_ctrl_destroy_bio_pair(BIO *bio1); -static size_t dgram_pair_read_inner(struct bio_dgram_pair_st *b, uint8_t *buf, - size_t sz); - -#define BIO_MSG_N(array, n) (*(BIO_MSG *)((char *)(array) + (n)*stride)) - -static const BIO_METHOD dgram_pair_method = { - BIO_TYPE_DGRAM_PAIR, - "BIO dgram pair", - bwrite_conv, - dgram_pair_write, - bread_conv, - dgram_pair_read, - NULL, /* dgram_pair_puts */ - NULL, /* dgram_pair_gets */ - dgram_pair_ctrl, - dgram_pair_init, - dgram_pair_free, - NULL, /* dgram_pair_callback_ctrl */ - dgram_pair_sendmmsg, - dgram_pair_recvmmsg, -}; - -static const BIO_METHOD dgram_mem_method = { - BIO_TYPE_DGRAM_MEM, - "BIO dgram mem", - bwrite_conv, - dgram_pair_write, - bread_conv, - dgram_mem_read, - NULL, /* dgram_pair_puts */ - NULL, /* dgram_pair_gets */ - dgram_mem_ctrl, - dgram_mem_init, - dgram_pair_free, - NULL, /* dgram_pair_callback_ctrl */ - dgram_pair_sendmmsg, - dgram_pair_recvmmsg, -}; - -const BIO_METHOD *BIO_s_dgram_pair(void) -{ - return &dgram_pair_method; -} - -const BIO_METHOD *BIO_s_dgram_mem(void) -{ - return &dgram_mem_method; -} - -struct dgram_hdr { - size_t len; /* payload length in bytes, not including this struct */ - BIO_ADDR src_addr, dst_addr; /* family == 0: not present */ -}; - -struct bio_dgram_pair_st { - /* The other half of the BIO pair. NULL for dgram_mem. */ - BIO *peer; - /* Writes are directed to our own ringbuf and reads to our peer. */ - struct ring_buf rbuf; - /* Requested size of rbuf buffer in bytes once we initialize. */ - size_t req_buf_len; - /* Largest possible datagram size */ - size_t mtu; - /* Capability flags. */ - uint32_t cap; - /* - * This lock protects updates to our rbuf. Since writes are directed to our - * own rbuf, this means we use this lock for writes and our peer's lock for - * reads. - */ - CRYPTO_RWLOCK *lock; - unsigned int no_trunc : 1; /* Reads fail if they would truncate */ - unsigned int local_addr_enable : 1; /* Can use BIO_MSG->local? */ - unsigned int role : 1; /* Determines lock order */ - unsigned int grows_on_write : 1; /* Set for BIO_s_dgram_mem only */ -}; - -#define MIN_BUF_LEN (1024) - -#define is_dgram_pair(b) (b->peer != NULL) - -static int dgram_pair_init(BIO *bio) -{ - struct bio_dgram_pair_st *b = OPENSSL_zalloc(sizeof(*b)); - - if (b == NULL) - return 0; - - b->mtu = 1472; /* conservative default MTU */ - /* default buffer size */ - b->req_buf_len = 9 * (sizeof(struct dgram_hdr) + b->mtu); - - b->lock = CRYPTO_THREAD_lock_new(); - if (b->lock == NULL) { - OPENSSL_free(b); - return 0; - } - - bio->ptr = b; - return 1; -} - -static int dgram_mem_init(BIO *bio) -{ - struct bio_dgram_pair_st *b; - - if (!dgram_pair_init(bio)) - return 0; - - b = bio->ptr; - - if (ring_buf_init(&b->rbuf, b->req_buf_len) == 0) { - ERR_raise(ERR_LIB_BIO, ERR_R_BIO_LIB); - return 0; - } - - b->grows_on_write = 1; - - bio->init = 1; - return 1; -} - -static int dgram_pair_free(BIO *bio) -{ - struct bio_dgram_pair_st *b; - - if (bio == NULL) - return 0; - - b = bio->ptr; - if (!ossl_assert(b != NULL)) - return 0; - - /* We are being freed. Disconnect any peer and destroy buffers. */ - dgram_pair_ctrl_destroy_bio_pair(bio); - - CRYPTO_THREAD_lock_free(b->lock); - OPENSSL_free(b); - return 1; -} - -/* BIO_make_bio_pair (BIO_C_MAKE_BIO_PAIR) */ -static int dgram_pair_ctrl_make_bio_pair(BIO *bio1, BIO *bio2) -{ - struct bio_dgram_pair_st *b1, *b2; - - /* peer must be non-NULL. */ - if (bio1 == NULL || bio2 == NULL) { - ERR_raise(ERR_LIB_BIO, BIO_R_INVALID_ARGUMENT); - return 0; - } - - /* Ensure the BIO we have been passed is actually a dgram pair BIO. */ - if (bio1->method != &dgram_pair_method || bio2->method != &dgram_pair_method) { - ERR_raise_data(ERR_LIB_BIO, BIO_R_INVALID_ARGUMENT, - "both BIOs must be BIO_dgram_pair"); - return 0; - } - - b1 = bio1->ptr; - b2 = bio2->ptr; - - if (!ossl_assert(b1 != NULL && b2 != NULL)) { - ERR_raise(ERR_LIB_BIO, BIO_R_UNINITIALIZED); - return 0; - } - - /* - * This ctrl cannot be used to associate a BIO pair half which is already - * associated. - */ - if (b1->peer != NULL || b2->peer != NULL) { - ERR_raise_data(ERR_LIB_BIO, BIO_R_IN_USE, - "cannot associate a BIO_dgram_pair which is already in use"); - return 0; - } - - if (!ossl_assert(b1->req_buf_len >= MIN_BUF_LEN - && b2->req_buf_len >= MIN_BUF_LEN)) { - ERR_raise(ERR_LIB_BIO, BIO_R_UNINITIALIZED); - return 0; - } - - if (b1->rbuf.len != b1->req_buf_len) - if (ring_buf_init(&b1->rbuf, b1->req_buf_len) == 0) { - ERR_raise(ERR_LIB_BIO, ERR_R_BIO_LIB); - return 0; - } - - if (b2->rbuf.len != b2->req_buf_len) - if (ring_buf_init(&b2->rbuf, b2->req_buf_len) == 0) { - ERR_raise(ERR_LIB_BIO, ERR_R_BIO_LIB); - ring_buf_destroy(&b1->rbuf); - return 0; - } - - b1->peer = bio2; - b2->peer = bio1; - b1->role = 0; - b2->role = 1; - bio1->init = 1; - bio2->init = 1; - return 1; -} - -/* BIO_destroy_bio_pair (BIO_C_DESTROY_BIO_PAIR) */ -static int dgram_pair_ctrl_destroy_bio_pair(BIO *bio1) -{ - BIO *bio2; - struct bio_dgram_pair_st *b1 = bio1->ptr, *b2; - - ring_buf_destroy(&b1->rbuf); - bio1->init = 0; - - /* Early return if we don't have a peer. */ - if (b1->peer == NULL) - return 1; - - bio2 = b1->peer; - b2 = bio2->ptr; - - /* Invariant. */ - if (!ossl_assert(b2->peer == bio1)) - return 0; - - /* Free buffers. */ - ring_buf_destroy(&b2->rbuf); - - bio2->init = 0; - b1->peer = NULL; - b2->peer = NULL; - return 1; -} - -/* BIO_eof (BIO_CTRL_EOF) */ -static int dgram_pair_ctrl_eof(BIO *bio) -{ - struct bio_dgram_pair_st *b = bio->ptr, *peerb; - - if (!ossl_assert(b != NULL)) - return -1; - - /* If we aren't initialized, we can never read anything */ - if (!bio->init) - return 1; - if (!is_dgram_pair(b)) - return 0; - - - peerb = b->peer->ptr; - if (!ossl_assert(peerb != NULL)) - return -1; - - /* - * Since we are emulating datagram semantics, never indicate EOF so long as - * we have a peer. - */ - return 0; -} - -/* BIO_set_write_buf_size (BIO_C_SET_WRITE_BUF_SIZE) */ -static int dgram_pair_ctrl_set_write_buf_size(BIO *bio, size_t len) -{ - struct bio_dgram_pair_st *b = bio->ptr; - - /* Changing buffer sizes is not permitted while a peer is connected. */ - if (b->peer != NULL) { - ERR_raise(ERR_LIB_BIO, BIO_R_IN_USE); - return 0; - } - - /* Enforce minimum size. */ - if (len < MIN_BUF_LEN) - len = MIN_BUF_LEN; - - if (b->rbuf.start != NULL) { - if (!ring_buf_resize(&b->rbuf, len)) - return 0; - } - - b->req_buf_len = len; - b->grows_on_write = 0; - return 1; -} - -/* BIO_reset (BIO_CTRL_RESET) */ -static int dgram_pair_ctrl_reset(BIO *bio) -{ - struct bio_dgram_pair_st *b = bio->ptr; - - ring_buf_clear(&b->rbuf); - return 1; -} - -/* BIO_pending (BIO_CTRL_PENDING) (Threadsafe) */ -static size_t dgram_pair_ctrl_pending(BIO *bio) -{ - size_t saved_idx, saved_count; - struct bio_dgram_pair_st *b = bio->ptr, *readb; - struct dgram_hdr hdr; - size_t l; - - /* Safe to check; init may not change during this call */ - if (!bio->init) - return 0; - if (is_dgram_pair(b)) - readb = b->peer->ptr; - else - readb = b; - - if (CRYPTO_THREAD_write_lock(readb->lock) == 0) - return 0; - - saved_idx = readb->rbuf.idx[1]; - saved_count = readb->rbuf.count; - - l = dgram_pair_read_inner(readb, (uint8_t *)&hdr, sizeof(hdr)); - - readb->rbuf.idx[1] = saved_idx; - readb->rbuf.count = saved_count; - - CRYPTO_THREAD_unlock(readb->lock); - - if (!ossl_assert(l == 0 || l == sizeof(hdr))) - return 0; - - return l > 0 ? hdr.len : 0; -} - -/* BIO_get_write_guarantee (BIO_C_GET_WRITE_GUARANTEE) (Threadsafe) */ -static size_t dgram_pair_ctrl_get_write_guarantee(BIO *bio) -{ - size_t l; - struct bio_dgram_pair_st *b = bio->ptr; - - if (CRYPTO_THREAD_read_lock(b->lock) == 0) - return 0; - - l = b->rbuf.len - b->rbuf.count; - if (l >= sizeof(struct dgram_hdr)) - l -= sizeof(struct dgram_hdr); - - /* - * If the amount of buffer space would not be enough to accommodate the - * worst-case size of a datagram, report no space available. - */ - if (l < b->mtu) - l = 0; - - CRYPTO_THREAD_unlock(b->lock); - return l; -} - -/* BIO_dgram_get_local_addr_cap (BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP) */ -static int dgram_pair_ctrl_get_local_addr_cap(BIO *bio) -{ - struct bio_dgram_pair_st *b = bio->ptr, *readb; - - if (!bio->init) - return 0; - - if (is_dgram_pair(b)) - readb = b->peer->ptr; - else - readb = b; - - return (~readb->cap & (BIO_DGRAM_CAP_HANDLES_SRC_ADDR - | BIO_DGRAM_CAP_PROVIDES_DST_ADDR)) == 0; -} - -/* BIO_dgram_get_effective_caps (BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS) */ -static int dgram_pair_ctrl_get_effective_caps(BIO *bio) -{ - struct bio_dgram_pair_st *b = bio->ptr, *peerb; - - if (b->peer == NULL) - return 0; - - peerb = b->peer->ptr; - - return peerb->cap; -} - -/* BIO_dgram_get_caps (BIO_CTRL_DGRAM_GET_CAPS) */ -static uint32_t dgram_pair_ctrl_get_caps(BIO *bio) -{ - struct bio_dgram_pair_st *b = bio->ptr; - - return b->cap; -} - -/* BIO_dgram_set_caps (BIO_CTRL_DGRAM_SET_CAPS) */ -static int dgram_pair_ctrl_set_caps(BIO *bio, uint32_t caps) -{ - struct bio_dgram_pair_st *b = bio->ptr; - - b->cap = caps; - return 1; -} - -/* BIO_dgram_get_local_addr_enable (BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE) */ -static int dgram_pair_ctrl_get_local_addr_enable(BIO *bio) -{ - struct bio_dgram_pair_st *b = bio->ptr; - - return b->local_addr_enable; -} - -/* BIO_dgram_set_local_addr_enable (BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE) */ -static int dgram_pair_ctrl_set_local_addr_enable(BIO *bio, int enable) -{ - struct bio_dgram_pair_st *b = bio->ptr; - - if (dgram_pair_ctrl_get_local_addr_cap(bio) == 0) - return 0; - - b->local_addr_enable = (enable != 0 ? 1 : 0); - return 1; -} - -/* BIO_dgram_get_mtu (BIO_CTRL_DGRAM_GET_MTU) */ -static int dgram_pair_ctrl_get_mtu(BIO *bio) -{ - struct bio_dgram_pair_st *b = bio->ptr; - - return b->mtu; -} - -/* BIO_dgram_set_mtu (BIO_CTRL_DGRAM_SET_MTU) */ -static int dgram_pair_ctrl_set_mtu(BIO *bio, size_t mtu) -{ - struct bio_dgram_pair_st *b = bio->ptr, *peerb; - - b->mtu = mtu; - - if (b->peer != NULL) { - peerb = b->peer->ptr; - peerb->mtu = mtu; - } - - return 1; -} - -/* Partially threadsafe (some commands) */ -static long dgram_mem_ctrl(BIO *bio, int cmd, long num, void *ptr) -{ - long ret = 1; - struct bio_dgram_pair_st *b = bio->ptr; - - if (!ossl_assert(b != NULL)) - return 0; - - switch (cmd) { - /* - * BIO_set_write_buf_size: Set the size of the ring buffer used for storing - * datagrams. No more writes can be performed once the buffer is filled up, - * until reads are performed. This cannot be used after a peer is connected. - */ - case BIO_C_SET_WRITE_BUF_SIZE: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_set_write_buf_size(bio, (size_t)num); - break; - - /* - * BIO_get_write_buf_size: Get ring buffer size. - */ - case BIO_C_GET_WRITE_BUF_SIZE: /* Non-threadsafe */ - ret = (long)b->req_buf_len; - break; - - /* - * BIO_reset: Clear all data which was written to this side of the pair. - */ - case BIO_CTRL_RESET: /* Non-threadsafe */ - dgram_pair_ctrl_reset(bio); - break; - - /* - * BIO_get_write_guarantee: Any BIO_write providing a buffer less than or - * equal to this value is guaranteed to succeed. - */ - case BIO_C_GET_WRITE_GUARANTEE: /* Threadsafe */ - ret = (long)dgram_pair_ctrl_get_write_guarantee(bio); - break; - - /* BIO_pending: Bytes available to read. */ - case BIO_CTRL_PENDING: /* Threadsafe */ - ret = (long)dgram_pair_ctrl_pending(bio); - break; - - /* BIO_flush: No-op. */ - case BIO_CTRL_FLUSH: /* Threadsafe */ - break; - - /* BIO_dgram_get_no_trunc */ - case BIO_CTRL_DGRAM_GET_NO_TRUNC: /* Non-threadsafe */ - ret = (long)b->no_trunc; - break; - - /* BIO_dgram_set_no_trunc */ - case BIO_CTRL_DGRAM_SET_NO_TRUNC: /* Non-threadsafe */ - b->no_trunc = (num > 0); - break; - - /* BIO_dgram_get_local_addr_enable */ - case BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE: /* Non-threadsafe */ - *(int *)ptr = (int)dgram_pair_ctrl_get_local_addr_enable(bio); - break; - - /* BIO_dgram_set_local_addr_enable */ - case BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_set_local_addr_enable(bio, num); - break; - - /* BIO_dgram_get_local_addr_cap: Can local addresses be supported? */ - case BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_get_local_addr_cap(bio); - break; - - /* BIO_dgram_get_effective_caps */ - case BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS: /* Non-threadsafe */ - /* BIO_dgram_get_caps */ - case BIO_CTRL_DGRAM_GET_CAPS: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_get_caps(bio); - break; - - /* BIO_dgram_set_caps */ - case BIO_CTRL_DGRAM_SET_CAPS: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_set_caps(bio, (uint32_t)num); - break; - - /* BIO_dgram_get_mtu */ - case BIO_CTRL_DGRAM_GET_MTU: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_get_mtu(bio); - break; - - /* BIO_dgram_set_mtu */ - case BIO_CTRL_DGRAM_SET_MTU: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_set_mtu(bio, (uint32_t)num); - break; - - /* - * BIO_eof: Returns whether this half of the BIO pair is empty of data to - * read. - */ - case BIO_CTRL_EOF: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_eof(bio); - break; - - default: - ret = 0; - break; - } - - return ret; -} - -static long dgram_pair_ctrl(BIO *bio, int cmd, long num, void *ptr) -{ - long ret = 1; - - switch (cmd) { - /* - * BIO_make_bio_pair: this is usually used by BIO_new_dgram_pair, though it - * may be used manually after manually creating each half of a BIO pair - * using BIO_new. This only needs to be called on one of the BIOs. - */ - case BIO_C_MAKE_BIO_PAIR: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_make_bio_pair(bio, (BIO *)ptr); - break; - - /* - * BIO_destroy_bio_pair: Manually disconnect two halves of a BIO pair so - * that they are no longer peers. - */ - case BIO_C_DESTROY_BIO_PAIR: /* Non-threadsafe */ - dgram_pair_ctrl_destroy_bio_pair(bio); - break; - - /* BIO_dgram_get_effective_caps */ - case BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS: /* Non-threadsafe */ - ret = (long)dgram_pair_ctrl_get_effective_caps(bio); - break; - - default: - ret = dgram_mem_ctrl(bio, cmd, num, ptr); - break; - } - - return ret; -} - -int BIO_new_bio_dgram_pair(BIO **pbio1, size_t writebuf1, - BIO **pbio2, size_t writebuf2) -{ - int ret = 0; - long r; - BIO *bio1 = NULL, *bio2 = NULL; - - bio1 = BIO_new(BIO_s_dgram_pair()); - if (bio1 == NULL) - goto err; - - bio2 = BIO_new(BIO_s_dgram_pair()); - if (bio2 == NULL) - goto err; - - if (writebuf1 > 0) { - r = BIO_set_write_buf_size(bio1, writebuf1); - if (r == 0) - goto err; - } - - if (writebuf2 > 0) { - r = BIO_set_write_buf_size(bio2, writebuf2); - if (r == 0) - goto err; - } - - r = BIO_make_bio_pair(bio1, bio2); - if (r == 0) - goto err; - - ret = 1; -err: - if (ret == 0) { - BIO_free(bio1); - bio1 = NULL; - BIO_free(bio2); - bio2 = NULL; - } - - *pbio1 = bio1; - *pbio2 = bio2; - return ret; -} - -/* Must hold peer write lock */ -static size_t dgram_pair_read_inner(struct bio_dgram_pair_st *b, uint8_t *buf, size_t sz) -{ - size_t total_read = 0; - - /* - * We repeat pops from the ring buffer for as long as we have more - * application *buffer to fill until we fail. We may not be able to pop - * enough data to fill the buffer in one operation if the ring buffer wraps - * around, but there may still be more data available. - */ - while (sz > 0) { - uint8_t *src_buf = NULL; - size_t src_len = 0; - - /* - * There are two BIO instances, each with a ringbuf. We read from the - * peer ringbuf and write to our own ringbuf. - */ - ring_buf_tail(&b->rbuf, &src_buf, &src_len); - if (src_len == 0) - break; - - if (src_len > sz) - src_len = sz; - - if (buf != NULL) - memcpy(buf, src_buf, src_len); - - ring_buf_pop(&b->rbuf, src_len); - - if (buf != NULL) - buf += src_len; - total_read += src_len; - sz -= src_len; - } - - return total_read; -} - -/* - * Must hold peer write lock. Returns number of bytes processed or negated BIO - * response code. - */ -static ossl_ssize_t dgram_pair_read_actual(BIO *bio, char *buf, size_t sz, - BIO_ADDR *local, BIO_ADDR *peer, - int is_multi) -{ - size_t l, trunc = 0, saved_idx, saved_count; - struct bio_dgram_pair_st *b = bio->ptr, *readb; - struct dgram_hdr hdr; - - if (!is_multi) - BIO_clear_retry_flags(bio); - - if (!bio->init) - return -BIO_R_UNINITIALIZED; - - if (!ossl_assert(b != NULL)) - return -BIO_R_TRANSFER_ERROR; - - if (is_dgram_pair(b)) - readb = b->peer->ptr; - else - readb = b; - if (!ossl_assert(readb != NULL && readb->rbuf.start != NULL)) - return -BIO_R_TRANSFER_ERROR; - - if (sz > 0 && buf == NULL) - return -BIO_R_INVALID_ARGUMENT; - - /* If the caller wants to know the local address, it must be enabled */ - if (local != NULL && b->local_addr_enable == 0) - return -BIO_R_LOCAL_ADDR_NOT_AVAILABLE; - - /* Read the header. */ - saved_idx = readb->rbuf.idx[1]; - saved_count = readb->rbuf.count; - l = dgram_pair_read_inner(readb, (uint8_t *)&hdr, sizeof(hdr)); - if (l == 0) { - /* Buffer was empty. */ - if (!is_multi) - BIO_set_retry_read(bio); - return -BIO_R_NON_FATAL; - } - - if (!ossl_assert(l == sizeof(hdr))) - /* - * This should not be possible as headers (and their following payloads) - * should always be written atomically. - */ - return -BIO_R_BROKEN_PIPE; - - if (sz > hdr.len) { - sz = hdr.len; - } else if (sz < hdr.len) { - /* Truncation is occurring. */ - trunc = hdr.len - sz; - if (b->no_trunc) { - /* Restore original state. */ - readb->rbuf.idx[1] = saved_idx; - readb->rbuf.count = saved_count; - return -BIO_R_NON_FATAL; - } - } - - l = dgram_pair_read_inner(readb, (uint8_t *)buf, sz); - if (!ossl_assert(l == sz)) - /* We were somehow not able to read the entire datagram. */ - return -BIO_R_TRANSFER_ERROR; - - /* - * If the datagram was truncated due to an inadequate buffer, discard the - * remainder. - */ - if (trunc > 0 && !ossl_assert(dgram_pair_read_inner(readb, NULL, trunc) == trunc)) - /* We were somehow not able to read/skip the entire datagram. */ - return -BIO_R_TRANSFER_ERROR; - - if (local != NULL) - *local = hdr.dst_addr; - if (peer != NULL) - *peer = hdr.src_addr; - - return (ossl_ssize_t)l; -} - -/* Threadsafe */ -static int dgram_pair_lock_both_write(struct bio_dgram_pair_st *a, - struct bio_dgram_pair_st *b) -{ - struct bio_dgram_pair_st *x, *y; - - x = (a->role == 1) ? a : b; - y = (a->role == 1) ? b : a; - - if (!ossl_assert(a->role != b->role)) - return 0; - - if (!ossl_assert(a != b && x != y)) - return 0; - - if (CRYPTO_THREAD_write_lock(x->lock) == 0) - return 0; - - if (CRYPTO_THREAD_write_lock(y->lock) == 0) { - CRYPTO_THREAD_unlock(x->lock); - return 0; - } - - return 1; -} - -static void dgram_pair_unlock_both(struct bio_dgram_pair_st *a, - struct bio_dgram_pair_st *b) -{ - CRYPTO_THREAD_unlock(a->lock); - CRYPTO_THREAD_unlock(b->lock); -} - -/* Threadsafe */ -static int dgram_pair_read(BIO *bio, char *buf, int sz_) -{ - int ret; - ossl_ssize_t l; - struct bio_dgram_pair_st *b = bio->ptr, *peerb; - - if (sz_ < 0) { - ERR_raise(ERR_LIB_BIO, BIO_R_INVALID_ARGUMENT); - return -1; - } - - if (b->peer == NULL) { - ERR_raise(ERR_LIB_BIO, BIO_R_BROKEN_PIPE); - return -1; - } - - peerb = b->peer->ptr; - - /* - * For BIO_read we have to acquire both locks because we touch the retry - * flags on the local bio. (This is avoided in the recvmmsg case as it does - * not touch the retry flags.) - */ - if (dgram_pair_lock_both_write(peerb, b) == 0) { - ERR_raise(ERR_LIB_BIO, ERR_R_UNABLE_TO_GET_WRITE_LOCK); - return -1; - } - - l = dgram_pair_read_actual(bio, buf, (size_t)sz_, NULL, NULL, 0); - if (l < 0) { - if (l != -BIO_R_NON_FATAL) - ERR_raise(ERR_LIB_BIO, -l); - ret = -1; - } else { - ret = (int)l; - } - - dgram_pair_unlock_both(peerb, b); - return ret; -} - -/* Threadsafe */ -static int dgram_pair_recvmmsg(BIO *bio, BIO_MSG *msg, - size_t stride, size_t num_msg, - uint64_t flags, - size_t *num_processed) -{ - int ret; - ossl_ssize_t l; - BIO_MSG *m; - size_t i; - struct bio_dgram_pair_st *b = bio->ptr, *readb; - - if (num_msg == 0) { - *num_processed = 0; - return 1; - } - - if (!bio->init) { - ERR_raise(ERR_LIB_BIO, BIO_R_BROKEN_PIPE); - *num_processed = 0; - return 0; - } - - if (is_dgram_pair(b)) - readb = b->peer->ptr; - else - readb = b; - - if (CRYPTO_THREAD_write_lock(readb->lock) == 0) { - ERR_raise(ERR_LIB_BIO, ERR_R_UNABLE_TO_GET_WRITE_LOCK); - *num_processed = 0; - return 0; - } - - for (i = 0; i < num_msg; ++i) { - m = &BIO_MSG_N(msg, i); - l = dgram_pair_read_actual(bio, m->data, m->data_len, - m->local, m->peer, 1); - if (l < 0) { - *num_processed = i; - if (i > 0) { - ret = 1; - } else { - ERR_raise(ERR_LIB_BIO, -l); - ret = 0; - } - goto out; - } - - m->data_len = l; - m->flags = 0; - } - - *num_processed = i; - ret = 1; -out: - CRYPTO_THREAD_unlock(readb->lock); - return ret; -} - -/* Threadsafe */ -static int dgram_mem_read(BIO *bio, char *buf, int sz_) -{ - int ret; - ossl_ssize_t l; - struct bio_dgram_pair_st *b = bio->ptr; - - if (sz_ < 0) { - ERR_raise(ERR_LIB_BIO, BIO_R_INVALID_ARGUMENT); - return -1; - } - - if (CRYPTO_THREAD_write_lock(b->lock) == 0) { - ERR_raise(ERR_LIB_BIO, ERR_R_UNABLE_TO_GET_WRITE_LOCK); - return -1; - } - - l = dgram_pair_read_actual(bio, buf, (size_t)sz_, NULL, NULL, 0); - if (l < 0) { - if (l != -BIO_R_NON_FATAL) - ERR_raise(ERR_LIB_BIO, -l); - ret = -1; - } else { - ret = (int)l; - } - - CRYPTO_THREAD_unlock(b->lock); - return ret; -} - -/* - * Calculate the array growth based on the target size. - * - * The growth factor is a rational number and is defined by a numerator - * and a denominator. According to Andrew Koenig in his paper "Why Are - * Vectors Efficient?" from JOOP 11(5) 1998, this factor should be less - * than the golden ratio (1.618...). - * - * We use an expansion factor of 8 / 5 = 1.6 - */ -static const size_t max_rbuf_size = SIZE_MAX / 2; /* unlimited in practice */ -static ossl_inline size_t compute_rbuf_growth(size_t target, size_t current) -{ - int err = 0; - - while (current < target) { - if (current >= max_rbuf_size) - return 0; - - current = safe_muldiv_size_t(current, 8, 5, &err); - if (err) - return 0; - if (current >= max_rbuf_size) - current = max_rbuf_size; - } - return current; -} - -/* Must hold local write lock */ -static size_t dgram_pair_write_inner(struct bio_dgram_pair_st *b, - const uint8_t *buf, size_t sz) -{ - size_t total_written = 0; - - /* - * We repeat pushes to the ring buffer for as long as we have data until we - * fail. We may not be able to push in one operation if the ring buffer - * wraps around, but there may still be more room for data. - */ - while (sz > 0) { - size_t dst_len; - uint8_t *dst_buf; - - /* - * There are two BIO instances, each with a ringbuf. We write to our own - * ringbuf and read from the peer ringbuf. - */ - ring_buf_head(&b->rbuf, &dst_buf, &dst_len); - if (dst_len == 0) { - size_t new_len; - - if (!b->grows_on_write) /* resize only if size not set explicitly */ - break; - /* increase the size */ - new_len = compute_rbuf_growth(b->req_buf_len + sz, b->req_buf_len); - if (new_len == 0 || !ring_buf_resize(&b->rbuf, new_len)) - break; - b->req_buf_len = new_len; - } - - if (dst_len > sz) - dst_len = sz; - - memcpy(dst_buf, buf, dst_len); - ring_buf_push(&b->rbuf, dst_len); - - buf += dst_len; - sz -= dst_len; - total_written += dst_len; - } - - return total_written; -} - -/* - * Must hold local write lock. Returns number of bytes processed or negated BIO - * response code. - */ -static ossl_ssize_t dgram_pair_write_actual(BIO *bio, const char *buf, size_t sz, - const BIO_ADDR *local, const BIO_ADDR *peer, - int is_multi) -{ - static const BIO_ADDR zero_addr; - size_t saved_idx, saved_count; - struct bio_dgram_pair_st *b = bio->ptr, *readb; - struct dgram_hdr hdr = {0}; - - if (!is_multi) - BIO_clear_retry_flags(bio); - - if (!bio->init) - return -BIO_R_UNINITIALIZED; - - if (!ossl_assert(b != NULL && b->rbuf.start != NULL)) - return -BIO_R_TRANSFER_ERROR; - - if (sz > 0 && buf == NULL) - return -BIO_R_INVALID_ARGUMENT; - - if (local != NULL && b->local_addr_enable == 0) - return -BIO_R_LOCAL_ADDR_NOT_AVAILABLE; - - if (is_dgram_pair(b)) - readb = b->peer->ptr; - else - readb = b; - if (peer != NULL && (readb->cap & BIO_DGRAM_CAP_HANDLES_DST_ADDR) == 0) - return -BIO_R_PEER_ADDR_NOT_AVAILABLE; - - hdr.len = sz; - hdr.dst_addr = (peer != NULL ? *peer : zero_addr); - hdr.src_addr = (local != NULL ? *local : zero_addr); - - saved_idx = b->rbuf.idx[0]; - saved_count = b->rbuf.count; - if (dgram_pair_write_inner(b, (const uint8_t *)&hdr, sizeof(hdr)) != sizeof(hdr) - || dgram_pair_write_inner(b, (const uint8_t *)buf, sz) != sz) { - /* - * We were not able to push the header and the entirety of the payload - * onto the ring buffer, so abort and roll back the ring buffer state. - */ - b->rbuf.idx[0] = saved_idx; - b->rbuf.count = saved_count; - if (!is_multi) - BIO_set_retry_write(bio); - return -BIO_R_NON_FATAL; - } - - return sz; -} - -/* Threadsafe */ -static int dgram_pair_write(BIO *bio, const char *buf, int sz_) -{ - int ret; - ossl_ssize_t l; - struct bio_dgram_pair_st *b = bio->ptr; - - if (sz_ < 0) { - ERR_raise(ERR_LIB_BIO, BIO_R_INVALID_ARGUMENT); - return -1; - } - - if (CRYPTO_THREAD_write_lock(b->lock) == 0) { - ERR_raise(ERR_LIB_BIO, ERR_R_UNABLE_TO_GET_WRITE_LOCK); - return -1; - } - - l = dgram_pair_write_actual(bio, buf, (size_t)sz_, NULL, NULL, 0); - if (l < 0) { - ERR_raise(ERR_LIB_BIO, -l); - ret = -1; - } else { - ret = (int)l; - } - - CRYPTO_THREAD_unlock(b->lock); - return ret; -} - -/* Threadsafe */ -static int dgram_pair_sendmmsg(BIO *bio, BIO_MSG *msg, - size_t stride, size_t num_msg, - uint64_t flags, size_t *num_processed) -{ - ossl_ssize_t ret, l; - BIO_MSG *m; - size_t i; - struct bio_dgram_pair_st *b = bio->ptr; - - if (num_msg == 0) { - *num_processed = 0; - return 1; - } - - if (CRYPTO_THREAD_write_lock(b->lock) == 0) { - ERR_raise(ERR_LIB_BIO, ERR_R_UNABLE_TO_GET_WRITE_LOCK); - *num_processed = 0; - return 0; - } - - for (i = 0; i < num_msg; ++i) { - m = &BIO_MSG_N(msg, i); - l = dgram_pair_write_actual(bio, m->data, m->data_len, - m->local, m->peer, 1); - if (l < 0) { - *num_processed = i; - if (i > 0) { - ret = 1; - } else { - ERR_raise(ERR_LIB_BIO, -l); - ret = 0; - } - goto out; - } - - m->flags = 0; - } - - *num_processed = i; - ret = 1; -out: - CRYPTO_THREAD_unlock(b->lock); - return ret; -} - -#endif diff --git a/openssl/src/crypto/bio/bss_fd.c b/openssl/src/crypto/bio/bss_fd.c index 41514589a..f756225ed 100644 --- a/openssl/src/crypto/bio/bss_fd.c +++ b/openssl/src/crypto/bio/bss_fd.c @@ -149,7 +149,7 @@ static long fd_ctrl(BIO *b, int cmd, long num, void *ptr) switch (cmd) { case BIO_CTRL_RESET: num = 0; - /* fall through */ + /* fall thru */ case BIO_C_FILE_SEEK: ret = (long)UP_lseek(b->num, num, 0); break; diff --git a/openssl/src/crypto/bio/bss_file.c b/openssl/src/crypto/bio/bss_file.c index 05d87cfe9..44669aba6 100644 --- a/openssl/src/crypto/bio/bss_file.c +++ b/openssl/src/crypto/bio/bss_file.c @@ -7,11 +7,11 @@ * https://www.openssl.org/source/license.html */ -#if defined(__linux) || defined(__sun) || defined(__hpux) +#if defined(__linux) || defined(__sun) /* * Following definition aliases fopen to fopen64 on above mentioned * platforms. This makes it possible to open and sequentially access files - * larger than 2GB from 32-bit application. It does not allow one to traverse + * larger than 2GB from 32-bit application. It does not allow to traverse * them beyond 2GB with fseek/ftell, but on the other hand *no* 32-bit * platform permits that, not with fseek/ftell. Not to mention that breaking * 2GB limit for seeking would require surgery to *our* API. But sequential @@ -165,12 +165,6 @@ static int file_write(BIO *b, const char *in, int inl) ret = fwrite(in, (int)inl, 1, (FILE *)b->ptr); if (ret) ret = inl; - /* ret=fwrite(in,1,(int)inl,(FILE *)b->ptr); */ - /* - * according to Tim Hudson , the commented out - * version above can cause 'inl' write calls under some stupid stdio - * implementations (VMS) - */ } return ret; } diff --git a/openssl/src/crypto/bio/bss_log.c b/openssl/src/crypto/bio/bss_log.c index c22e603b0..66d8f8868 100644 --- a/openssl/src/crypto/bio/bss_log.c +++ b/openssl/src/crypto/bio/bss_log.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,7 @@ * * BIO_s_log is useful for system daemons (or services under NT). It is * one-way BIO, it sends all stuff to syslogd (on system that commonly use - * that), or event log (on NT), or OPCOM (on OpenVMS). + * that), or event log (on NT). * */ @@ -24,25 +24,6 @@ #if defined(OPENSSL_SYS_WINCE) #elif defined(OPENSSL_SYS_WIN32) -#elif defined(__wasi__) -# define NO_SYSLOG -#elif defined(OPENSSL_SYS_VMS) -# include -# include -# include -# include -/* Some compiler options may mask the declaration of "_malloc32". */ -# if __INITIAL_POINTER_SIZE && defined _ANSI_C_SOURCE -# if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size save -# pragma pointer_size 32 -void *_malloc32(__size_t); -# pragma pointer_size restore -# endif /* __INITIAL_POINTER_SIZE == 64 */ -# endif /* __INITIAL_POINTER_SIZE && defined - * _ANSI_C_SOURCE */ -#elif defined(__DJGPP__) && defined(OPENSSL_NO_SOCK) -# define NO_SYSLOG #elif (!defined(MSDOS) || defined(WATT32)) && !defined(OPENSSL_SYS_VXWORKS) && !defined(NO_SYSLOG) # include #endif @@ -63,18 +44,6 @@ void *_malloc32(__size_t); # define LOG_DEBUG 7 # define LOG_DAEMON (3<<3) -# elif defined(OPENSSL_SYS_VMS) -/* On VMS, we don't really care about these, but we need them to compile */ -# define LOG_EMERG 0 -# define LOG_ALERT 1 -# define LOG_CRIT 2 -# define LOG_ERR 3 -# define LOG_WARNING 4 -# define LOG_NOTICE 5 -# define LOG_INFO 6 -# define LOG_DEBUG 7 - -# define LOG_DAEMON OPC$M_NM_NTWORK # endif static int slg_write(BIO *h, const char *buf, int num); @@ -199,8 +168,10 @@ static int slg_write(BIO *b, const char *in, int inl) if (inl < 0) return 0; - if ((buf = OPENSSL_malloc(inl + 1)) == NULL) + if ((buf = OPENSSL_malloc(inl + 1)) == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(buf, in, inl); buf[inl] = '\0'; @@ -295,97 +266,6 @@ static void xcloselog(BIO *bp) bp->ptr = NULL; } -# elif defined(OPENSSL_SYS_VMS) - -static int VMS_OPC_target = LOG_DAEMON; - -static void xopenlog(BIO *bp, char *name, int level) -{ - VMS_OPC_target = level; -} - -static void xsyslog(BIO *bp, int priority, const char *string) -{ - struct dsc$descriptor_s opc_dsc; - -/* Arrange 32-bit pointer to opcdef buffer and malloc(), if needed. */ -# if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size save -# pragma pointer_size 32 -# define OPCDEF_TYPE __char_ptr32 -# define OPCDEF_MALLOC _malloc32 -# else /* __INITIAL_POINTER_SIZE == 64 */ -# define OPCDEF_TYPE char * -# define OPCDEF_MALLOC OPENSSL_malloc -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ - - struct opcdef *opcdef_p; - -# if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size restore -# endif /* __INITIAL_POINTER_SIZE == 64 */ - - char buf[10240]; - unsigned int len; - struct dsc$descriptor_s buf_dsc; - $DESCRIPTOR(fao_cmd, "!AZ: !AZ"); - char *priority_tag; - - switch (priority) { - case LOG_EMERG: - priority_tag = "Emergency"; - break; - case LOG_ALERT: - priority_tag = "Alert"; - break; - case LOG_CRIT: - priority_tag = "Critical"; - break; - case LOG_ERR: - priority_tag = "Error"; - break; - case LOG_WARNING: - priority_tag = "Warning"; - break; - case LOG_NOTICE: - priority_tag = "Notice"; - break; - case LOG_INFO: - priority_tag = "Info"; - break; - case LOG_DEBUG: - priority_tag = "DEBUG"; - break; - } - - buf_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - buf_dsc.dsc$b_class = DSC$K_CLASS_S; - buf_dsc.dsc$a_pointer = buf; - buf_dsc.dsc$w_length = sizeof(buf) - 1; - - lib$sys_fao(&fao_cmd, &len, &buf_dsc, priority_tag, string); - - /* We know there's an 8-byte header. That's documented. */ - opcdef_p = OPCDEF_MALLOC(8 + len); - opcdef_p->opc$b_ms_type = OPC$_RQ_RQST; - memcpy(opcdef_p->opc$z_ms_target_classes, &VMS_OPC_target, 3); - opcdef_p->opc$l_ms_rqstid = 0; - memcpy(&opcdef_p->opc$l_ms_text, buf, len); - - opc_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - opc_dsc.dsc$b_class = DSC$K_CLASS_S; - opc_dsc.dsc$a_pointer = (OPCDEF_TYPE) opcdef_p; - opc_dsc.dsc$w_length = len + 8; - - sys$sndopr(opc_dsc, 0); - - OPENSSL_free(opcdef_p); -} - -static void xcloselog(BIO *bp) -{ -} - # else /* Unix/Watt32 */ static void xopenlog(BIO *bp, char *name, int level) diff --git a/openssl/src/crypto/bio/bss_mem.c b/openssl/src/crypto/bio/bss_mem.c index 6deacba42..9153c1f1c 100644 --- a/openssl/src/crypto/bio/bss_mem.c +++ b/openssl/src/crypto/bio/bss_mem.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/bio/bss_sock.c b/openssl/src/crypto/bio/bss_sock.c index 82f7be85a..f5d881023 100644 --- a/openssl/src/crypto/bio/bss_sock.c +++ b/openssl/src/crypto/bio/bss_sock.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,7 +10,6 @@ #include #include #include "bio_local.h" -#include "internal/bio_tfo.h" #include "internal/cryptlib.h" #include "internal/ktls.h" @@ -28,14 +27,6 @@ # define sock_puts SockPuts # endif -struct bss_sock_st { - BIO_ADDR tfo_peer; - int tfo_first; -#ifndef OPENSSL_NO_KTLS - unsigned char ktls_record_type; -#endif -}; - static int sock_write(BIO *h, const char *buf, int num); static int sock_read(BIO *h, char *buf, int size); static int sock_puts(BIO *h, const char *str); @@ -90,10 +81,8 @@ static int sock_new(BIO *bi) { bi->init = 0; bi->num = 0; + bi->ptr = NULL; bi->flags = 0; - bi->ptr = OPENSSL_zalloc(sizeof(struct bss_sock_st)); - if (bi->ptr == NULL) - return 0; return 1; } @@ -108,8 +97,6 @@ static int sock_free(BIO *a) a->init = 0; a->flags = 0; } - OPENSSL_free(a->ptr); - a->ptr = NULL; return 1; } @@ -139,30 +126,17 @@ static int sock_read(BIO *b, char *out, int outl) static int sock_write(BIO *b, const char *in, int inl) { int ret = 0; -# if !defined(OPENSSL_NO_KTLS) || defined(OSSL_TFO_SENDTO) - struct bss_sock_st *data = (struct bss_sock_st *)b->ptr; -# endif clear_socket_error(); # ifndef OPENSSL_NO_KTLS if (BIO_should_ktls_ctrl_msg_flag(b)) { - unsigned char record_type = data->ktls_record_type; + unsigned char record_type = (intptr_t)b->ptr; ret = ktls_send_ctrl_message(b->num, record_type, in, inl); if (ret >= 0) { ret = inl; BIO_clear_ktls_ctrl_msg_flag(b); } } else -# endif -# if defined(OSSL_TFO_SENDTO) - if (data->tfo_first) { - struct bss_sock_st *data = (struct bss_sock_st *)b->ptr; - socklen_t peerlen = BIO_ADDR_sockaddr_size(&data->tfo_peer); - - ret = sendto(b->num, in, inl, OSSL_TFO_SENDTO, - BIO_ADDR_sockaddr(&data->tfo_peer), peerlen); - data->tfo_first = 0; - } else # endif ret = writesocket(b->num, in, inl); BIO_clear_retry_flags(b); @@ -177,24 +151,16 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr) { long ret = 1; int *ip; - struct bss_sock_st *data = (struct bss_sock_st *)b->ptr; # ifndef OPENSSL_NO_KTLS ktls_crypto_info_t *crypto_info; # endif switch (cmd) { case BIO_C_SET_FD: - /* minimal sock_free() */ - if (b->shutdown) { - if (b->init) - BIO_closesocket(b->num); - b->flags = 0; - } + sock_free(b); b->num = *((int *)ptr); b->shutdown = (int)num; b->init = 1; - data->tfo_first = 0; - memset(&data->tfo_peer, 0, sizeof(data->tfo_peer)); break; case BIO_C_GET_FD: if (b->init) { @@ -215,20 +181,6 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr) case BIO_CTRL_FLUSH: ret = 1; break; - case BIO_CTRL_GET_RPOLL_DESCRIPTOR: - case BIO_CTRL_GET_WPOLL_DESCRIPTOR: - { - BIO_POLL_DESCRIPTOR *pd = ptr; - - if (!b->init) { - ret = 0; - break; - } - - pd->type = BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD; - pd->value.fd = b->num; - } - break; # ifndef OPENSSL_NO_KTLS case BIO_CTRL_SET_KTLS: crypto_info = (ktls_crypto_info_t *)ptr; @@ -242,41 +194,17 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr) return BIO_should_ktls_flag(b, 0) != 0; case BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG: BIO_set_ktls_ctrl_msg_flag(b); - data->ktls_record_type = (unsigned char)num; + b->ptr = (void *)num; ret = 0; break; case BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG: BIO_clear_ktls_ctrl_msg_flag(b); ret = 0; break; - case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE: - ret = ktls_enable_tx_zerocopy_sendfile(b->num); - if (ret) - BIO_set_ktls_zerocopy_sendfile_flag(b); - break; # endif case BIO_CTRL_EOF: ret = (b->flags & BIO_FLAGS_IN_EOF) != 0; break; - case BIO_C_GET_CONNECT: - if (ptr != NULL && num == 2) { - const char **pptr = (const char **)ptr; - - *pptr = (const char *)&data->tfo_peer; - } else { - ret = 0; - } - break; - case BIO_C_SET_CONNECT: - if (ptr != NULL && num == 2) { - ret = BIO_ADDR_make(&data->tfo_peer, - BIO_ADDR_sockaddr((const BIO_ADDR *)ptr)); - if (ret) - data->tfo_first = 1; - } else { - ret = 0; - } - break; default: ret = 0; break; diff --git a/openssl/src/crypto/bio/ossl_core_bio.c b/openssl/src/crypto/bio/ossl_core_bio.c index 8d21115b6..328302ea3 100644 --- a/openssl/src/crypto/bio/ossl_core_bio.c +++ b/openssl/src/crypto/bio/ossl_core_bio.c @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,6 +17,7 @@ */ struct ossl_core_bio_st { CRYPTO_REF_COUNT ref_cnt; + CRYPTO_RWLOCK *ref_lock; BIO *bio; }; @@ -24,10 +25,11 @@ static OSSL_CORE_BIO *core_bio_new(void) { OSSL_CORE_BIO *cb = OPENSSL_malloc(sizeof(*cb)); - if (cb == NULL || !CRYPTO_NEW_REF(&cb->ref_cnt, 1)) { + if (cb == NULL || (cb->ref_lock = CRYPTO_THREAD_lock_new()) == NULL) { OPENSSL_free(cb); return NULL; } + cb->ref_cnt = 1; return cb; } @@ -35,7 +37,7 @@ int ossl_core_bio_up_ref(OSSL_CORE_BIO *cb) { int ref = 0; - return CRYPTO_UP_REF(&cb->ref_cnt, &ref); + return CRYPTO_UP_REF(&cb->ref_cnt, &ref, cb->ref_lock); } int ossl_core_bio_free(OSSL_CORE_BIO *cb) @@ -43,10 +45,10 @@ int ossl_core_bio_free(OSSL_CORE_BIO *cb) int ref = 0, res = 1; if (cb != NULL) { - CRYPTO_DOWN_REF(&cb->ref_cnt, &ref); + CRYPTO_DOWN_REF(&cb->ref_cnt, &ref, cb->ref_lock); if (ref <= 0) { res = BIO_free(cb->bio); - CRYPTO_FREE_REF(&cb->ref_cnt); + CRYPTO_THREAD_lock_free(cb->ref_lock); OPENSSL_free(cb); } } diff --git a/openssl/src/crypto/bn/bn_asm.c b/openssl/src/crypto/bn/bn_asm.c index c39907a7d..c2c8684d3 100644 --- a/openssl/src/crypto/bn/bn_asm.c +++ b/openssl/src/crypto/bn/bn_asm.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -381,33 +381,25 @@ BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, #ifndef OPENSSL_SMALL_FOOTPRINT while (n & ~3) { t1 = a[0]; - t2 = (t1 - c) & BN_MASK2; - c = (t2 > t1); - t1 = b[0]; - t1 = (t2 - t1) & BN_MASK2; - r[0] = t1; - c += (t1 > t2); + t2 = b[0]; + r[0] = (t1 - t2 - c) & BN_MASK2; + if (t1 != t2) + c = (t1 < t2); t1 = a[1]; - t2 = (t1 - c) & BN_MASK2; - c = (t2 > t1); - t1 = b[1]; - t1 = (t2 - t1) & BN_MASK2; - r[1] = t1; - c += (t1 > t2); + t2 = b[1]; + r[1] = (t1 - t2 - c) & BN_MASK2; + if (t1 != t2) + c = (t1 < t2); t1 = a[2]; - t2 = (t1 - c) & BN_MASK2; - c = (t2 > t1); - t1 = b[2]; - t1 = (t2 - t1) & BN_MASK2; - r[2] = t1; - c += (t1 > t2); + t2 = b[2]; + r[2] = (t1 - t2 - c) & BN_MASK2; + if (t1 != t2) + c = (t1 < t2); t1 = a[3]; - t2 = (t1 - c) & BN_MASK2; - c = (t2 > t1); - t1 = b[3]; - t1 = (t2 - t1) & BN_MASK2; - r[3] = t1; - c += (t1 > t2); + t2 = b[3]; + r[3] = (t1 - t2 - c) & BN_MASK2; + if (t1 != t2) + c = (t1 < t2); a += 4; b += 4; r += 4; @@ -416,12 +408,10 @@ BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, #endif while (n) { t1 = a[0]; - t2 = (t1 - c) & BN_MASK2; - c = (t2 > t1); - t1 = b[0]; - t1 = (t2 - t1) & BN_MASK2; - r[0] = t1; - c += (t1 > t2); + t2 = b[0]; + r[0] = (t1 - t2 - c) & BN_MASK2; + if (t1 != t2) + c = (t1 < t2); a++; b++; r++; @@ -432,6 +422,14 @@ BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, #if defined(BN_MUL_COMBA) && !defined(OPENSSL_SMALL_FOOTPRINT) +# undef bn_mul_comba8 +# undef bn_mul_comba4 +# undef bn_sqr_comba8 +# undef bn_sqr_comba4 + +# undef HEADER_SYMBOL_PREFIX_H +# include + /* mul_add_c(a,b,c0,c1,c2) -- c+=a*b for three word number c=(c2,c1,c0) */ /* mul_add_c2(a,b,c0,c1,c2) -- c+=2*a*b for three word number c=(c2,c1,c0) */ /* sqr_add_c(a,i,c0,c1,c2) -- c+=a[i]^2 for three word number c=(c2,c1,c0) */ @@ -451,7 +449,7 @@ BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, t += c0; /* no carry */ \ c0 = (BN_ULONG)Lw(t); \ hi = (BN_ULONG)Hw(t); \ - c1 = (c1+hi)&BN_MASK2; c2 += (c1 + void bn_sqr_comba4(BN_ULONG *r, const BN_ULONG *a) { BN_ULONG t[8]; diff --git a/openssl/src/crypto/bn/bn_blind.c b/openssl/src/crypto/bn/bn_blind.c index 6c6de1a30..6061ebb4c 100644 --- a/openssl/src/crypto/bn/bn_blind.c +++ b/openssl/src/crypto/bn/bn_blind.c @@ -1,5 +1,5 @@ /* - * Copyright 1998-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1998-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,32 +13,20 @@ #define BN_BLINDING_COUNTER 32 -struct bn_blinding_st { - BIGNUM *A; - BIGNUM *Ai; - BIGNUM *e; - BIGNUM *mod; /* just a reference */ - CRYPTO_THREAD_ID tid; - int counter; - unsigned long flags; - BN_MONT_CTX *m_ctx; - int (*bn_mod_exp) (BIGNUM *r, const BIGNUM *a, const BIGNUM *p, - const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx); - CRYPTO_RWLOCK *lock; -}; - BN_BLINDING *BN_BLINDING_new(const BIGNUM *A, const BIGNUM *Ai, BIGNUM *mod) { BN_BLINDING *ret = NULL; bn_check_top(mod); - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; + } ret->lock = CRYPTO_THREAD_lock_new(); if (ret->lock == NULL) { - ERR_raise(ERR_LIB_BN, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -189,8 +177,7 @@ int BN_BLINDING_invert_ex(BIGNUM *n, const BIGNUM *r, BN_BLINDING *b, n->top = (int)(rtop & ~mask) | (ntop & mask); n->flags |= (BN_FLG_FIXED_TOP & ~mask); } - ret = bn_mul_mont_fixed_top(n, n, r, b->m_ctx, ctx); - bn_correct_top_consttime(n); + ret = BN_mod_mul_montgomery(n, n, r, b->m_ctx, ctx); } else { ret = BN_mod_mul(n, n, r, b->mod, ctx); } diff --git a/openssl/src/crypto/bn/bn_const.c b/openssl/src/crypto/bn/bn_const.c index 190a36391..a36e0ac79 100644 --- a/openssl/src/crypto/bn/bn_const.c +++ b/openssl/src/crypto/bn/bn_const.c @@ -19,7 +19,7 @@ * The prime is: 2^768 - 2 ^704 - 1 + 2^64 * { [2^638 pi] + 149686 } * * RFC2409 specifies a generator of 2. - * RFC2412 specifies a generator of 22. + * RFC2412 specifies a generator of of 22. */ BIGNUM *BN_get_rfc2409_prime_768(BIGNUM *bn) diff --git a/openssl/src/crypto/bn/bn_conv.c b/openssl/src/crypto/bn/bn_conv.c index 849440e71..75054f5d6 100644 --- a/openssl/src/crypto/bn/bn_conv.c +++ b/openssl/src/crypto/bn/bn_conv.c @@ -23,8 +23,10 @@ char *BN_bn2hex(const BIGNUM *a) if (BN_is_zero(a)) return OPENSSL_strdup("0"); buf = OPENSSL_malloc(a->top * BN_BYTES * 2 + 2); - if (buf == NULL) + if (buf == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); goto err; + } p = buf; if (a->neg) *p++ = '-'; @@ -68,8 +70,10 @@ char *BN_bn2dec(const BIGNUM *a) bn_data_num = num / BN_DEC_NUM + 1; bn_data = OPENSSL_malloc(bn_data_num * sizeof(BN_ULONG)); buf = OPENSSL_malloc(tbytes); - if (buf == NULL || bn_data == NULL) + if (buf == NULL || bn_data == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); goto err; + } if ((t = BN_dup(a)) == NULL) goto err; diff --git a/openssl/src/crypto/bn/bn_ctx.c b/openssl/src/crypto/bn/bn_ctx.c index aa70ca7a3..0934262fb 100644 --- a/openssl/src/crypto/bn/bn_ctx.c +++ b/openssl/src/crypto/bn/bn_ctx.c @@ -10,73 +10,20 @@ #include #include "internal/cryptlib.h" #include "bn_local.h" +#ifndef OPENSSL_NO_BN_METHOD +# include +#endif -/* How many bignums are in each "pool item"; */ -#define BN_CTX_POOL_SIZE 16 -/* The stack frame info is resizing, set a first-time expansion size; */ -#define BN_CTX_START_FRAMES 32 - -/***********/ -/* BN_POOL */ -/***********/ - -/* A bundle of bignums that can be linked with other bundles */ -typedef struct bignum_pool_item { - /* The bignum values */ - BIGNUM vals[BN_CTX_POOL_SIZE]; - /* Linked-list admin */ - struct bignum_pool_item *prev, *next; -} BN_POOL_ITEM; -/* A linked-list of bignums grouped in bundles */ -typedef struct bignum_pool { - /* Linked-list admin */ - BN_POOL_ITEM *head, *current, *tail; - /* Stack depth and allocation size */ - unsigned used, size; -} BN_POOL; static void BN_POOL_init(BN_POOL *); static void BN_POOL_finish(BN_POOL *); static BIGNUM *BN_POOL_get(BN_POOL *, int); static void BN_POOL_release(BN_POOL *, unsigned int); -/************/ -/* BN_STACK */ -/************/ - -/* A wrapper to manage the "stack frames" */ -typedef struct bignum_ctx_stack { - /* Array of indexes into the bignum stack */ - unsigned int *indexes; - /* Number of stack frames, and the size of the allocated array */ - unsigned int depth, size; -} BN_STACK; static void BN_STACK_init(BN_STACK *); static void BN_STACK_finish(BN_STACK *); static int BN_STACK_push(BN_STACK *, unsigned int); static unsigned int BN_STACK_pop(BN_STACK *); -/**********/ -/* BN_CTX */ -/**********/ - -/* The opaque BN_CTX type */ -struct bignum_ctx { - /* The bignum bundles */ - BN_POOL pool; - /* The "stack frames", if you will */ - BN_STACK stack; - /* The number of bignums currently assigned */ - unsigned int used; - /* Depth of stack overflow */ - int err_stack; - /* Block "gets" until an "end" (compatibility behaviour) */ - int too_many; - /* Flags. */ - int flags; - /* The library context */ - OSSL_LIB_CTX *libctx; -}; - #ifndef FIPS_MODULE /* Debugging functionality */ static void ctxdbg(BIO *channel, const char *text, BN_CTX *ctx) @@ -119,8 +66,10 @@ BN_CTX *BN_CTX_new_ex(OSSL_LIB_CTX *ctx) { BN_CTX *ret; - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; + } /* Initialise the structure */ BN_POOL_init(&ret->pool); BN_STACK_init(&ret->stack); @@ -173,9 +122,48 @@ void BN_CTX_free(BN_CTX *ctx) #endif BN_STACK_finish(&ctx->stack); BN_POOL_finish(&ctx->pool); +#if !defined(OPENSSL_NO_BN_METHOD) && !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE) + ENGINE_finish(ctx->engine); +#endif OPENSSL_free(ctx); } +#ifndef OPENSSL_NO_BN_METHOD +# if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE) +int BN_CTX_set_engine(BN_CTX *ctx, ENGINE *engine) +{ + const BN_METHOD *bn_meth; + + if (!ENGINE_init(engine)) { + ERR_raise(ERR_LIB_BN, ERR_R_ENGINE_LIB); + return 0; + } + + bn_meth = ENGINE_get_bn_meth(engine); + if (bn_meth == NULL) { + ERR_raise(ERR_LIB_BN, BN_R_BN_METHOD_NOT_FOUND); + return 0; + } + + ctx->engine = engine; + ctx->bn_meth = bn_meth; + + return 1; +} + +const ENGINE *BN_CTX_get0_engine(BN_CTX *ctx) +{ + return ctx->engine; +} +# endif + +int BN_CTX_set_method(BN_CTX *ctx, const BN_METHOD *method) +{ + ctx->bn_meth = method; + return 1; +} +#endif + void BN_CTX_start(BN_CTX *ctx) { CTXDBG("ENTER BN_CTX_start()", ctx); @@ -266,8 +254,10 @@ static int BN_STACK_push(BN_STACK *st, unsigned int idx) st->size ? (st->size * 3 / 2) : BN_CTX_START_FRAMES; unsigned int *newitems; - if ((newitems = OPENSSL_malloc(sizeof(*newitems) * newsize)) == NULL) + if ((newitems = OPENSSL_malloc(sizeof(*newitems) * newsize)) == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return 0; + } if (st->depth) memcpy(newitems, st->indexes, sizeof(*newitems) * st->depth); OPENSSL_free(st->indexes); @@ -318,8 +308,10 @@ static BIGNUM *BN_POOL_get(BN_POOL *p, int flag) if (p->used == p->size) { BN_POOL_ITEM *item; - if ((item = OPENSSL_malloc(sizeof(*item))) == NULL) + if ((item = OPENSSL_malloc(sizeof(*item))) == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; + } for (loop = 0, bn = item->vals; loop++ < BN_CTX_POOL_SIZE; bn++) { bn_init(bn); if ((flag & BN_FLG_SECURE) != 0) diff --git a/openssl/src/crypto/bn/bn_div.c b/openssl/src/crypto/bn/bn_div.c index ff66baa48..7b5fb94ad 100644 --- a/openssl/src/crypto/bn/bn_div.c +++ b/openssl/src/crypto/bn/bn_div.c @@ -211,6 +211,11 @@ int BN_div(BIGNUM *dv, BIGNUM *rm, const BIGNUM *num, const BIGNUM *divisor, { int ret; +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->div) + return ctx->bn_meth->div(dv, rm, num, divisor, ctx); +#endif + if (BN_is_zero(divisor)) { ERR_raise(ERR_LIB_BN, BN_R_DIV_BY_ZERO); return 0; diff --git a/openssl/src/crypto/bn/bn_err.c b/openssl/src/crypto/bn/bn_err.c index 953be9ed4..4bbc2e829 100644 --- a/openssl/src/crypto/bn/bn_err.c +++ b/openssl/src/crypto/bn/bn_err.c @@ -19,6 +19,7 @@ static const ERR_STRING_DATA BN_str_reasons[] = { {ERR_PACK(ERR_LIB_BN, 0, BN_R_BAD_RECIPROCAL), "bad reciprocal"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_BIGNUM_TOO_LONG), "bignum too long"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_BITS_TOO_SMALL), "bits too small"}, + {ERR_PACK(ERR_LIB_BN, 0, BN_R_BN_METHOD_NOT_FOUND), "bn method not found"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_CALLED_WITH_EVEN_MODULUS), "called with even modulus"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_DIV_BY_ZERO), "div by zero"}, @@ -32,7 +33,6 @@ static const ERR_STRING_DATA BN_str_reasons[] = { {ERR_PACK(ERR_LIB_BN, 0, BN_R_NOT_A_SQUARE), "not a square"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_NOT_INITIALIZED), "not initialized"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_NO_INVERSE), "no inverse"}, - {ERR_PACK(ERR_LIB_BN, 0, BN_R_NO_PRIME_CANDIDATE), "no prime candidate"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_NO_SOLUTION), "no solution"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_NO_SUITABLE_DIGEST), "no suitable digest"}, {ERR_PACK(ERR_LIB_BN, 0, BN_R_PRIVATE_KEY_TOO_LARGE), diff --git a/openssl/src/crypto/bn/bn_exp.c b/openssl/src/crypto/bn/bn_exp.c index b876edbfa..e3decbc83 100644 --- a/openssl/src/crypto/bn/bn_exp.c +++ b/openssl/src/crypto/bn/bn_exp.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -27,24 +27,9 @@ #include "rsaz_exp.h" -#undef SPARC_T4_MONT -#if defined(OPENSSL_BN_ASM_MONT) && (defined(__sparc__) || defined(__sparc)) -# include "crypto/sparc_arch.h" -# define SPARC_T4_MONT -#endif - /* maximum precomputation table size for *variable* sliding windows */ #define TABLE_SIZE 32 -/* - * Beyond this limit the constant time code is disabled due to - * the possible overflow in the computation of powerbufLen in - * BN_mod_exp_mont_consttime. - * When this limit is exceeded, the computation will be done using - * non-constant time code, but it will take very long. - */ -#define BN_CONSTTIME_SIZE_LIMIT (INT_MAX / BN_BYTES / 256) - /* this one works - simple but works */ int BN_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) { @@ -99,6 +84,11 @@ int BN_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, { int ret; +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->mod_exp) + return ctx->bn_meth->mod_exp(r, a, p, m, ctx); +#endif + bn_check_top(a); bn_check_top(p); bn_check_top(m); @@ -169,7 +159,7 @@ int BN_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, int BN_mod_exp_recp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, BN_CTX *ctx) { - int i, j, bits, ret = 0, wstart, wend, window; + int i, j, bits, ret = 0, wstart, wend, window, wvalue; int start = 1; BIGNUM *aa; /* Table of variables obtained from 'ctx' */ @@ -239,23 +229,14 @@ int BN_mod_exp_recp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, start = 1; /* This is used to avoid multiplication etc * when there is only the value '1' in the * buffer. */ + wvalue = 0; /* The 'value' of the window */ wstart = bits - 1; /* The top bit of the window */ wend = 0; /* The bottom bit of the window */ - if (r == p) { - BIGNUM *p_dup = BN_CTX_get(ctx); - - if (p_dup == NULL || BN_copy(p_dup, p) == NULL) - goto err; - p = p_dup; - } - if (!BN_one(r)) goto err; for (;;) { - int wvalue; /* The 'value' of the window */ - if (BN_is_bit_set(p, wstart) == 0) { if (!start) if (!BN_mod_mul_reciprocal(r, r, r, &recp, ctx)) @@ -297,6 +278,7 @@ int BN_mod_exp_recp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, /* move the 'window' down further */ wstart -= wend + 1; + wvalue = 0; start = 0; if (wstart < 0) break; @@ -312,7 +294,7 @@ int BN_mod_exp_recp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *in_mont) { - int i, j, bits, ret = 0, wstart, wend, window; + int i, j, bits, ret = 0, wstart, wend, window, wvalue; int start = 1; BIGNUM *d, *r; const BIGNUM *aa; @@ -320,6 +302,12 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, BIGNUM *val[TABLE_SIZE]; BN_MONT_CTX *mont = NULL; + if (BN_get_flags(p, BN_FLG_CONSTTIME) != 0 + || BN_get_flags(a, BN_FLG_CONSTTIME) != 0 + || BN_get_flags(m, BN_FLG_CONSTTIME) != 0) { + return BN_mod_exp_mont_consttime(rr, a, p, m, ctx, in_mont); + } + bn_check_top(a); bn_check_top(p); bn_check_top(m); @@ -328,14 +316,6 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, ERR_raise(ERR_LIB_BN, BN_R_CALLED_WITH_EVEN_MODULUS); return 0; } - - if (m->top <= BN_CONSTTIME_SIZE_LIMIT - && (BN_get_flags(p, BN_FLG_CONSTTIME) != 0 - || BN_get_flags(a, BN_FLG_CONSTTIME) != 0 - || BN_get_flags(m, BN_FLG_CONSTTIME) != 0)) { - return BN_mod_exp_mont_consttime(rr, a, p, m, ctx, in_mont); - } - bits = BN_num_bits(p); if (bits == 0) { /* x**0 mod 1, or x**0 mod -1 is still zero. */ @@ -392,6 +372,7 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, start = 1; /* This is used to avoid multiplication etc * when there is only the value '1' in the * buffer. */ + wvalue = 0; /* The 'value' of the window */ wstart = bits - 1; /* The top bit of the window */ wend = 0; /* The bottom bit of the window */ @@ -411,8 +392,6 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, if (!bn_to_mont_fixed_top(r, BN_value_one(), mont, ctx)) goto err; for (;;) { - int wvalue; /* The 'value' of the window */ - if (BN_is_bit_set(p, wstart) == 0) { if (!start) { if (!bn_mul_mont_fixed_top(r, r, r, mont, ctx)) @@ -455,6 +434,7 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, /* move the 'window' down further */ wstart -= wend + 1; + wvalue = 0; start = 0; if (wstart < 0) break; @@ -464,17 +444,6 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, * removes padding [if any] and makes return value suitable for public * API consumer. */ -#if defined(SPARC_T4_MONT) - if (OPENSSL_sparcv9cap_P[0] & (SPARCV9_VIS3 | SPARCV9_PREFER_FPU)) { - j = mont->N.top; /* borrow j */ - val[0]->d[0] = 1; /* borrow val[0] */ - for (i = 1; i < j; i++) - val[0]->d[i] = 0; - val[0]->top = j; - if (!BN_mod_mul_montgomery(rr, r, val[0], mont, ctx)) - goto err; - } else -#endif if (!BN_from_montgomery(rr, r, mont, ctx)) goto err; ret = 1; @@ -619,9 +588,6 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, int powerbufLen = 0; unsigned char *powerbuf = NULL; BIGNUM tmp, am; -#if defined(SPARC_T4_MONT) - unsigned int t4 = 0; -#endif bn_check_top(a); bn_check_top(p); @@ -634,11 +600,6 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, top = m->top; - if (top > BN_CONSTTIME_SIZE_LIMIT) { - /* Prevent overflowing the powerbufLen computation below */ - return BN_mod_exp_mont(rr, a, p, m, ctx, in_mont); - } - /* * Use all bits stored in |p|, rather than |BN_num_bits|, so we do not leak * whether the top bits are zero. @@ -710,15 +671,8 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, /* Get the window size to use with size of p. */ window = BN_window_bits_for_ctime_exponent_size(bits); -#if defined(SPARC_T4_MONT) - if (window >= 5 && (top & 15) == 0 && top <= 64 && - (OPENSSL_sparcv9cap_P[1] & (CFR_MONTMUL | CFR_MONTSQR)) == - (CFR_MONTMUL | CFR_MONTSQR) && (t4 = OPENSSL_sparcv9cap_P[0])) - window = 5; - else -#endif #if defined(OPENSSL_BN_ASM_MONT5) - if (window >= 5 && top <= BN_SOFT_LIMIT) { + if (window >= 5) { window = 5; /* ~5% improvement for RSA2048 sign, and even * for RSA4096 */ /* reserve space for mont->N.d[] copy */ @@ -779,166 +733,17 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, if (!bn_to_mont_fixed_top(&am, a, mont, ctx)) goto err; - if (top > BN_SOFT_LIMIT) - goto fallback; - -#if defined(SPARC_T4_MONT) - if (t4) { - typedef int (*bn_pwr5_mont_f) (BN_ULONG *tp, const BN_ULONG *np, - const BN_ULONG *n0, const void *table, - int power, int bits); - int bn_pwr5_mont_t4_8(BN_ULONG *tp, const BN_ULONG *np, - const BN_ULONG *n0, const void *table, - int power, int bits); - int bn_pwr5_mont_t4_16(BN_ULONG *tp, const BN_ULONG *np, - const BN_ULONG *n0, const void *table, - int power, int bits); - int bn_pwr5_mont_t4_24(BN_ULONG *tp, const BN_ULONG *np, - const BN_ULONG *n0, const void *table, - int power, int bits); - int bn_pwr5_mont_t4_32(BN_ULONG *tp, const BN_ULONG *np, - const BN_ULONG *n0, const void *table, - int power, int bits); - static const bn_pwr5_mont_f pwr5_funcs[4] = { - bn_pwr5_mont_t4_8, bn_pwr5_mont_t4_16, - bn_pwr5_mont_t4_24, bn_pwr5_mont_t4_32 - }; - bn_pwr5_mont_f pwr5_worker = pwr5_funcs[top / 16 - 1]; - - typedef int (*bn_mul_mont_f) (BN_ULONG *rp, const BN_ULONG *ap, - const void *bp, const BN_ULONG *np, - const BN_ULONG *n0); - int bn_mul_mont_t4_8(BN_ULONG *rp, const BN_ULONG *ap, const void *bp, - const BN_ULONG *np, const BN_ULONG *n0); - int bn_mul_mont_t4_16(BN_ULONG *rp, const BN_ULONG *ap, - const void *bp, const BN_ULONG *np, - const BN_ULONG *n0); - int bn_mul_mont_t4_24(BN_ULONG *rp, const BN_ULONG *ap, - const void *bp, const BN_ULONG *np, - const BN_ULONG *n0); - int bn_mul_mont_t4_32(BN_ULONG *rp, const BN_ULONG *ap, - const void *bp, const BN_ULONG *np, - const BN_ULONG *n0); - static const bn_mul_mont_f mul_funcs[4] = { - bn_mul_mont_t4_8, bn_mul_mont_t4_16, - bn_mul_mont_t4_24, bn_mul_mont_t4_32 - }; - bn_mul_mont_f mul_worker = mul_funcs[top / 16 - 1]; - - void bn_mul_mont_vis3(BN_ULONG *rp, const BN_ULONG *ap, - const void *bp, const BN_ULONG *np, - const BN_ULONG *n0, int num); - void bn_mul_mont_t4(BN_ULONG *rp, const BN_ULONG *ap, - const void *bp, const BN_ULONG *np, - const BN_ULONG *n0, int num); - void bn_mul_mont_gather5_t4(BN_ULONG *rp, const BN_ULONG *ap, - const void *table, const BN_ULONG *np, - const BN_ULONG *n0, int num, int power); - void bn_flip_n_scatter5_t4(const BN_ULONG *inp, size_t num, - void *table, size_t power); - void bn_gather5_t4(BN_ULONG *out, size_t num, - void *table, size_t power); - void bn_flip_t4(BN_ULONG *dst, BN_ULONG *src, size_t num); - - BN_ULONG *np = mont->N.d, *n0 = mont->n0; - int stride = 5 * (6 - (top / 16 - 1)); /* multiple of 5, but less - * than 32 */ - - /* - * BN_to_montgomery can contaminate words above .top [in - * BN_DEBUG build... - */ - for (i = am.top; i < top; i++) - am.d[i] = 0; - for (i = tmp.top; i < top; i++) - tmp.d[i] = 0; - - bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, 0); - bn_flip_n_scatter5_t4(am.d, top, powerbuf, 1); - if (!(*mul_worker) (tmp.d, am.d, am.d, np, n0) && - !(*mul_worker) (tmp.d, am.d, am.d, np, n0)) - bn_mul_mont_vis3(tmp.d, am.d, am.d, np, n0, top); - bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, 2); - - for (i = 3; i < 32; i++) { - /* Calculate a^i = a^(i-1) * a */ - if (!(*mul_worker) (tmp.d, tmp.d, am.d, np, n0) && - !(*mul_worker) (tmp.d, tmp.d, am.d, np, n0)) - bn_mul_mont_vis3(tmp.d, tmp.d, am.d, np, n0, top); - bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, i); - } - - /* switch to 64-bit domain */ - np = alloca(top * sizeof(BN_ULONG)); - top /= 2; - bn_flip_t4(np, mont->N.d, top); - +#if defined(OPENSSL_BN_ASM_MONT5) + if (window == 5 && top > 1) { /* - * The exponent may not have a whole number of fixed-size windows. - * To simplify the main loop, the initial window has between 1 and - * full-window-size bits such that what remains is always a whole - * number of windows + * This optimization uses ideas from http://eprint.iacr.org/2011/239, + * specifically optimization of cache-timing attack countermeasures + * and pre-computation optimization. */ - window0 = (bits - 1) % 5 + 1; - wmask = (1 << window0) - 1; - bits -= window0; - wvalue = bn_get_bits(p, bits) & wmask; - bn_gather5_t4(tmp.d, top, powerbuf, wvalue); /* - * Scan the exponent one window at a time starting from the most - * significant bits. - */ - while (bits > 0) { - if (bits < stride) - stride = bits; - bits -= stride; - wvalue = bn_get_bits(p, bits); - - if ((*pwr5_worker) (tmp.d, np, n0, powerbuf, wvalue, stride)) - continue; - /* retry once and fall back */ - if ((*pwr5_worker) (tmp.d, np, n0, powerbuf, wvalue, stride)) - continue; - - bits += stride - 5; - wvalue >>= stride - 5; - wvalue &= 31; - bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top); - bn_mul_mont_gather5_t4(tmp.d, tmp.d, powerbuf, np, n0, top, - wvalue); - } - - bn_flip_t4(tmp.d, tmp.d, top); - top *= 2; - /* back to 32-bit domain */ - tmp.top = top; - bn_correct_top(&tmp); - OPENSSL_cleanse(np, top * sizeof(BN_ULONG)); - } else -#endif -#if defined(OPENSSL_BN_ASM_MONT5) - if (window == 5 && top > 1) { - /* - * This optimization uses ideas from https://eprint.iacr.org/2011/239, - * specifically optimization of cache-timing attack countermeasures, - * pre-computation optimization, and Almost Montgomery Multiplication. - * - * The paper discusses a 4-bit window to optimize 512-bit modular - * exponentiation, used in RSA-1024 with CRT, but RSA-1024 is no longer - * important. - * - * |bn_mul_mont_gather5| and |bn_power5| implement the "almost" - * reduction variant, so the values here may not be fully reduced. - * They are bounded by R (i.e. they fit in |top| words), not |m|. - * Additionally, we pass these "almost" reduced inputs into - * |bn_mul_mont|, which implements the normal reduction variant. - * Given those inputs, |bn_mul_mont| may not give reduced - * output, but it will still produce "almost" reduced output. + * Dedicated window==4 case improves 512-bit RSA sign by ~15%, but as + * 512-bit RSA is hardly relevant, we omit it to spare size... */ void bn_mul_mont_gather5(BN_ULONG *rp, const BN_ULONG *ap, const void *table, const BN_ULONG *np, @@ -950,6 +755,9 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, const void *table, const BN_ULONG *np, const BN_ULONG *n0, int num, int power); int bn_get_bits5(const BN_ULONG *ap, int off); + int bn_from_montgomery(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *not_used, const BN_ULONG *np, + const BN_ULONG *n0, int num); BN_ULONG *n0 = mont->n0, *np; @@ -1038,22 +846,17 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, } } + ret = bn_from_montgomery(tmp.d, tmp.d, NULL, np, n0, top); tmp.top = top; - /* - * The result is now in |tmp| in Montgomery form, but it may not be - * fully reduced. This is within bounds for |BN_from_montgomery| - * (tmp < R <= m*R) so it will, when converting from Montgomery form, - * produce a fully reduced result. - * - * This differs from Figure 2 of the paper, which uses AMM(h, 1) to - * convert from Montgomery form with unreduced output, followed by an - * extra reduction step. In the paper's terminology, we replace - * steps 9 and 10 with MM(h, 1). - */ + bn_correct_top(&tmp); + if (ret) { + if (!BN_copy(rr, &tmp)) + ret = 0; + goto err; /* non-zero ret means it's not error */ + } } else #endif { - fallback: if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, 0, window)) goto err; if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&am, top, powerbuf, 1, window)) @@ -1137,15 +940,6 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, * removes padding [if any] and makes return value suitable for public * API consumer. */ -#if defined(SPARC_T4_MONT) - if (OPENSSL_sparcv9cap_P[0] & (SPARCV9_VIS3 | SPARCV9_PREFER_FPU)) { - am.d[0] = 1; /* borrow am */ - for (i = 1; i < top; i++) - am.d[i] = 0; - if (!BN_mod_mul_montgomery(rr, &tmp, &am, mont, ctx)) - goto err; - } else -#endif if (!BN_from_montgomery(rr, &tmp, mont, ctx)) goto err; ret = 1; @@ -1311,7 +1105,7 @@ int BN_mod_exp_mont_word(BIGNUM *rr, BN_ULONG a, const BIGNUM *p, int BN_mod_exp_simple(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, BN_CTX *ctx) { - int i, j, bits, ret = 0, wstart, wend, window; + int i, j, bits, ret = 0, wstart, wend, window, wvalue; int start = 1; BIGNUM *d; /* Table of variables obtained from 'ctx' */ @@ -1325,11 +1119,6 @@ int BN_mod_exp_simple(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, return 0; } - if (r == m) { - ERR_raise(ERR_LIB_BN, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - bits = BN_num_bits(p); if (bits == 0) { /* x**0 mod 1, or x**0 mod -1 is still zero. */ @@ -1371,23 +1160,14 @@ int BN_mod_exp_simple(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, start = 1; /* This is used to avoid multiplication etc * when there is only the value '1' in the * buffer. */ + wvalue = 0; /* The 'value' of the window */ wstart = bits - 1; /* The top bit of the window */ wend = 0; /* The bottom bit of the window */ - if (r == p) { - BIGNUM *p_dup = BN_CTX_get(ctx); - - if (p_dup == NULL || BN_copy(p_dup, p) == NULL) - goto err; - p = p_dup; - } - if (!BN_one(r)) goto err; for (;;) { - int wvalue; /* The 'value' of the window */ - if (BN_is_bit_set(p, wstart) == 0) { if (!start) if (!BN_mod_mul(r, r, r, m, ctx)) @@ -1429,6 +1209,7 @@ int BN_mod_exp_simple(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, /* move the 'window' down further */ wstart -= wend + 1; + wvalue = 0; start = 0; if (wstart < 0) break; @@ -1460,20 +1241,12 @@ int BN_mod_exp_mont_consttime_x2(BIGNUM *rr1, const BIGNUM *a1, const BIGNUM *p1 BN_MONT_CTX *mont2 = NULL; if (ossl_rsaz_avx512ifma_eligible() && - (((a1->top == 16) && (p1->top == 16) && (BN_num_bits(m1) == 1024) && - (a2->top == 16) && (p2->top == 16) && (BN_num_bits(m2) == 1024)) || - ((a1->top == 24) && (p1->top == 24) && (BN_num_bits(m1) == 1536) && - (a2->top == 24) && (p2->top == 24) && (BN_num_bits(m2) == 1536)) || - ((a1->top == 32) && (p1->top == 32) && (BN_num_bits(m1) == 2048) && - (a2->top == 32) && (p2->top == 32) && (BN_num_bits(m2) == 2048)))) { - - int topn = a1->top; - /* Modulus bits of |m1| and |m2| are equal */ - int mod_bits = BN_num_bits(m1); - - if (bn_wexpand(rr1, topn) == NULL) + ((a1->top == 16) && (p1->top == 16) && (BN_num_bits(m1) == 1024) && + (a2->top == 16) && (p2->top == 16) && (BN_num_bits(m2) == 1024))) { + + if (bn_wexpand(rr1, 16) == NULL) goto err; - if (bn_wexpand(rr2, topn) == NULL) + if (bn_wexpand(rr2, 16) == NULL) goto err; /* Ensure that montgomery contexts are initialized */ @@ -1498,14 +1271,14 @@ int BN_mod_exp_mont_consttime_x2(BIGNUM *rr1, const BIGNUM *a1, const BIGNUM *p1 mont1->RR.d, mont1->n0[0], rr2->d, a2->d, p2->d, m2->d, mont2->RR.d, mont2->n0[0], - mod_bits); + 1024 /* factor bit size */); - rr1->top = topn; + rr1->top = 16; rr1->neg = 0; bn_correct_top(rr1); bn_check_top(rr1); - rr2->top = topn; + rr2->top = 16; rr2->neg = 0; bn_correct_top(rr2); bn_check_top(rr2); diff --git a/openssl/src/crypto/bn/bn_gcd.c b/openssl/src/crypto/bn/bn_gcd.c index 2cd8ee35e..142691874 100644 --- a/openssl/src/crypto/bn/bn_gcd.c +++ b/openssl/src/crypto/bn/bn_gcd.c @@ -47,8 +47,7 @@ BIGNUM *bn_mod_inverse_no_branch(BIGNUM *in, if (R == NULL) goto err; - if (!BN_one(X)) - goto err; + BN_one(X); BN_zero(Y); if (BN_copy(B, a) == NULL) goto err; @@ -236,8 +235,7 @@ BIGNUM *int_bn_mod_inverse(BIGNUM *in, if (R == NULL) goto err; - if (!BN_one(X)) - goto err; + BN_one(X); BN_zero(Y); if (BN_copy(B, a) == NULL) goto err; @@ -519,10 +517,15 @@ BIGNUM *BN_mod_inverse(BIGNUM *in, BIGNUM *rv; int noinv = 0; +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->mod_inverse) + return ctx->bn_meth->mod_inverse(in, a, n, ctx); +#endif + if (ctx == NULL) { ctx = new_ctx = BN_CTX_new_ex(NULL); if (ctx == NULL) { - ERR_raise(ERR_LIB_BN, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; } } @@ -534,37 +537,6 @@ BIGNUM *BN_mod_inverse(BIGNUM *in, return rv; } -/* - * The numbers a and b are coprime if the only positive integer that is a - * divisor of both of them is 1. - * i.e. gcd(a,b) = 1. - * - * Coprimes have the property: b has a multiplicative inverse modulo a - * i.e there is some value x such that bx = 1 (mod a). - * - * Testing the modulo inverse is currently much faster than the constant - * time version of BN_gcd(). - */ -int BN_are_coprime(BIGNUM *a, const BIGNUM *b, BN_CTX *ctx) -{ - int ret = 0; - BIGNUM *tmp; - - BN_CTX_start(ctx); - tmp = BN_CTX_get(ctx); - if (tmp == NULL) - goto end; - - ERR_set_mark(); - BN_set_flags(a, BN_FLG_CONSTTIME); - ret = (BN_mod_inverse(tmp, a, b, ctx) != NULL); - /* Clear any errors (an error is returned if there is no inverse) */ - ERR_pop_to_mark(); -end: - BN_CTX_end(ctx); - return ret; -} - /*- * This function is based on the constant-time GCD work by Bernstein and Yang: * https://eprint.iacr.org/2019/266 @@ -642,9 +614,9 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx) for (i = 0; i < m; i++) { /* conditionally flip signs if delta is positive and g is odd */ - cond = ((unsigned int)-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1 + cond = (-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1 /* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */ - & (~((unsigned int)(g->top - 1) >> (sizeof(g->top) * 8 - 1))); + & (~((g->top - 1) >> (sizeof(g->top) * 8 - 1))); delta = (-cond & -delta) | ((cond - 1) & delta); r->neg ^= cond; /* swap */ @@ -656,7 +628,7 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx) goto err; BN_consttime_swap(g->d[0] & 1 /* g is odd */ /* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */ - & (~((unsigned int)(g->top - 1) >> (sizeof(g->top) * 8 - 1))), + & (~((g->top - 1) >> (sizeof(g->top) * 8 - 1))), g, temp, top); if (!BN_rshift1(g, g)) goto err; diff --git a/openssl/src/crypto/bn/bn_gf2m.c b/openssl/src/crypto/bn/bn_gf2m.c index 444c5ca7a..304c2ea08 100644 --- a/openssl/src/crypto/bn/bn_gf2m.c +++ b/openssl/src/crypto/bn/bn_gf2m.c @@ -474,8 +474,10 @@ int BN_GF2m_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, bn_check_top(p); arr = OPENSSL_malloc(sizeof(*arr) * max); - if (arr == NULL) + if (arr == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return 0; + } ret = BN_GF2m_poly2arr(p, arr, max); if (!ret || ret > max) { ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH); @@ -534,8 +536,10 @@ int BN_GF2m_mod_sqr(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) bn_check_top(p); arr = OPENSSL_malloc(sizeof(*arr) * max); - if (arr == NULL) + if (arr == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return 0; + } ret = BN_GF2m_poly2arr(p, arr, max); if (!ret || ret > max) { ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH); @@ -730,20 +734,14 @@ int BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) { BIGNUM *b = NULL; int ret = 0; - int numbits; BN_CTX_start(ctx); if ((b = BN_CTX_get(ctx)) == NULL) goto err; - /* Fail on a non-sensical input p value */ - numbits = BN_num_bits(p); - if (numbits <= 1) - goto err; - /* generate blinding value */ do { - if (!BN_priv_rand_ex(b, numbits - 1, + if (!BN_priv_rand_ex(b, BN_num_bits(p) - 1, BN_RAND_TOP_ANY, BN_RAND_BOTTOM_ANY, 0, ctx)) goto err; } while (BN_is_zero(b)); @@ -917,8 +915,10 @@ int BN_GF2m_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, bn_check_top(p); arr = OPENSSL_malloc(sizeof(*arr) * max); - if (arr == NULL) + if (arr == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return 0; + } ret = BN_GF2m_poly2arr(p, arr, max); if (!ret || ret > max) { ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH); @@ -979,8 +979,10 @@ int BN_GF2m_mod_sqrt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) bn_check_top(p); arr = OPENSSL_malloc(sizeof(*arr) * max); - if (arr == NULL) + if (arr == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return 0; + } ret = BN_GF2m_poly2arr(p, arr, max); if (!ret || ret > max) { ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH); @@ -1113,8 +1115,10 @@ int BN_GF2m_mod_solve_quad(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, bn_check_top(p); arr = OPENSSL_malloc(sizeof(*arr) * max); - if (arr == NULL) + if (arr == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); goto err; + } ret = BN_GF2m_poly2arr(p, arr, max); if (!ret || ret > max) { ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH); diff --git a/openssl/src/crypto/bn/bn_intern.c b/openssl/src/crypto/bn/bn_intern.c index 505a9dfcc..c0f7f5fea 100644 --- a/openssl/src/crypto/bn/bn_intern.c +++ b/openssl/src/crypto/bn/bn_intern.c @@ -29,8 +29,10 @@ signed char *bn_compute_wNAF(const BIGNUM *scalar, int w, size_t *ret_len) if (BN_is_zero(scalar)) { r = OPENSSL_malloc(1); - if (r == NULL) + if (r == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); goto err; + } r[0] = 0; *ret_len = 1; return r; @@ -60,8 +62,10 @@ signed char *bn_compute_wNAF(const BIGNUM *scalar, int w, size_t *ret_len) * (*ret_len will be set to the actual length, i.e. at most * BN_num_bits(scalar) + 1) */ - if (r == NULL) + if (r == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); goto err; + } window_val = scalar->d[0] & mask; j = 0; while ((window_val != 0) || (j + w + 1 < len)) { /* if j+w+1 >= len, @@ -184,7 +188,7 @@ void bn_set_static_words(BIGNUM *a, const BN_ULONG *words, int size) int bn_set_words(BIGNUM *a, const BN_ULONG *words, int num_words) { if (bn_wexpand(a, num_words) == NULL) { - ERR_raise(ERR_LIB_BN, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return 0; } diff --git a/openssl/src/crypto/bn/bn_lib.c b/openssl/src/crypto/bn/bn_lib.c index 18c9d54f6..7ad684256 100644 --- a/openssl/src/crypto/bn/bn_lib.c +++ b/openssl/src/crypto/bn/bn_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -244,8 +244,10 @@ BIGNUM *BN_new(void) { BIGNUM *ret; - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; + } ret->flags = BN_FLG_MALLOCED; bn_check_top(ret); return ret; @@ -277,8 +279,10 @@ static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words) a = OPENSSL_secure_zalloc(words * sizeof(*a)); else a = OPENSSL_zalloc(words * sizeof(*a)); - if (a == NULL) + if (a == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; + } assert(b->top <= words); if (b->top > 0) @@ -426,102 +430,42 @@ int BN_set_word(BIGNUM *a, BN_ULONG w) return 1; } -typedef enum {BIG, LITTLE} endianness_t; -typedef enum {SIGNED, UNSIGNED} signedness_t; - -static BIGNUM *bin2bn(const unsigned char *s, int len, BIGNUM *ret, - endianness_t endianness, signedness_t signedness) +BIGNUM *BN_bin2bn(const unsigned char *s, int len, BIGNUM *ret) { - int inc; - const unsigned char *s2; - int inc2; - int neg = 0, xor = 0, carry = 0; - unsigned int i; + unsigned int i, m; unsigned int n; + BN_ULONG l; BIGNUM *bn = NULL; - /* Negative length is not acceptable */ - if (len < 0) - return NULL; - if (ret == NULL) ret = bn = BN_new(); if (ret == NULL) return NULL; bn_check_top(ret); - - /* - * If the input has no bits, the number is considered zero. - * This makes calls with s==NULL and len==0 safe. - */ - if (len == 0) { - BN_clear(ret); - return ret; - } - - /* - * The loop that does the work iterates from least to most - * significant BIGNUM chunk, so we adapt parameters to transfer - * input bytes accordingly. - */ - if (endianness == LITTLE) { - s2 = s + len - 1; - inc2 = -1; - inc = 1; - } else { - s2 = s; - inc2 = 1; - inc = -1; - s += len - 1; - } - - /* Take note of the signedness of the input bytes*/ - if (signedness == SIGNED) { - neg = !!(*s2 & 0x80); - xor = neg ? 0xff : 0x00; - carry = neg; - } - - /* - * Skip leading sign extensions (the value of |xor|). - * This is the only spot where |s2| and |inc2| are used. - */ - for ( ; len > 0 && *s2 == xor; s2 += inc2, len--) + /* Skip leading zero's. */ + for ( ; len > 0 && *s == 0; s++, len--) continue; - - /* - * If there was a set of 0xff, we backtrack one byte unless the next - * one has a sign bit, as the last 0xff is then part of the actual - * number, rather then a mere sign extension. - */ - if (xor == 0xff) { - if (len == 0 || !(*s2 & 0x80)) - len++; - } - /* If it was all zeros, we're done */ - if (len == 0) { + n = len; + if (n == 0) { ret->top = 0; return ret; } - n = ((len - 1) / BN_BYTES) + 1; /* Number of resulting bignum chunks */ - if (bn_wexpand(ret, (int)n) == NULL) { + i = ((n - 1) / BN_BYTES) + 1; + m = ((n - 1) % (BN_BYTES)); + if (bn_wexpand(ret, (int)i) == NULL) { BN_free(bn); return NULL; } - ret->top = n; - ret->neg = neg; - for (i = 0; n-- > 0; i++) { - BN_ULONG l = 0; /* Accumulator */ - unsigned int m = 0; /* Offset in a bignum chunk, in bits */ - - for (; len > 0 && m < BN_BYTES * 8; len--, s += inc, m += 8) { - BN_ULONG byte_xored = *s ^ xor; - BN_ULONG byte = (byte_xored + carry) & 0xff; - - carry = byte_xored > byte; /* Implicit 1 or 0 */ - l |= (byte << m); + ret->top = i; + ret->neg = 0; + l = 0; + while (n--) { + l = (l << 8L) | *(s++); + if (m-- == 0) { + ret->d[--i] = l; + l = 0; + m = BN_BYTES - 1; } - ret->d[i] = l; } /* * need to call this due to clear byte at top if avoiding having the top @@ -531,58 +475,30 @@ static BIGNUM *bin2bn(const unsigned char *s, int len, BIGNUM *ret, return ret; } -BIGNUM *BN_bin2bn(const unsigned char *s, int len, BIGNUM *ret) -{ - return bin2bn(s, len, ret, BIG, UNSIGNED); -} - -BIGNUM *BN_signed_bin2bn(const unsigned char *s, int len, BIGNUM *ret) -{ - return bin2bn(s, len, ret, BIG, SIGNED); -} +typedef enum {big, little} endianess_t; -static int bn2binpad(const BIGNUM *a, unsigned char *to, int tolen, - endianness_t endianness, signedness_t signedness) +/* ignore negative */ +static +int bn2binpad(const BIGNUM *a, unsigned char *to, int tolen, endianess_t endianess) { - int inc; - int n, n8; - int xor = 0, carry = 0, ext = 0; + int n; size_t i, lasti, j, atop, mask; BN_ULONG l; /* - * In case |a| is fixed-top, BN_num_bits can return bogus length, + * In case |a| is fixed-top, BN_num_bytes can return bogus length, * but it's assumed that fixed-top inputs ought to be "nominated" * even for padded output, so it works out... */ - n8 = BN_num_bits(a); - n = (n8 + 7) / 8; /* This is what BN_num_bytes() does */ - - /* Take note of the signedness of the bignum */ - if (signedness == SIGNED) { - xor = a->neg ? 0xff : 0x00; - carry = a->neg; - - /* - * if |n * 8 == n|, then the MSbit is set, otherwise unset. - * We must compensate with one extra byte if that doesn't - * correspond to the signedness of the bignum with regards - * to 2's complement. - */ - ext = (n * 8 == n8) - ? !a->neg /* MSbit set on nonnegative bignum */ - : a->neg; /* MSbit unset on negative bignum */ - } - + n = BN_num_bytes(a); if (tolen == -1) { - tolen = n + ext; - } else if (tolen < n + ext) { /* uncommon/unlike case */ + tolen = n; + } else if (tolen < n) { /* uncommon/unlike case */ BIGNUM temp = *a; bn_correct_top(&temp); - n8 = BN_num_bits(&temp); - n = (n8 + 7) / 8; /* This is what BN_num_bytes() does */ - if (tolen < n + ext) + n = BN_num_bytes(&temp); + if (tolen < n) return -1; } @@ -594,30 +510,19 @@ static int bn2binpad(const BIGNUM *a, unsigned char *to, int tolen, return tolen; } - /* - * The loop that does the work iterates from least significant - * to most significant BIGNUM limb, so we adapt parameters to - * transfer output bytes accordingly. - */ - if (endianness == LITTLE) { - inc = 1; - } else { - inc = -1; - to += tolen - 1; /* Move to the last byte, not beyond */ - } - lasti = atop - 1; atop = a->top * BN_BYTES; + if (endianess == big) + to += tolen; /* start from the end of the buffer */ for (i = 0, j = 0; j < (size_t)tolen; j++) { - unsigned char byte, byte_xored; - + unsigned char val; l = a->d[i / BN_BYTES]; mask = 0 - ((j - atop) >> (8 * sizeof(i) - 1)); - byte = (unsigned char)(l >> (8 * (i % BN_BYTES)) & mask); - byte_xored = byte ^ xor; - *to = (unsigned char)(byte_xored + carry); - carry = byte_xored > *to; /* Implicit 1 or 0 */ - to += inc; + val = (unsigned char)(l >> (8 * (i % BN_BYTES)) & mask); + if (endianess == big) + *--to = val; + else + *to++ = val; i += (i - lasti) >> (8 * sizeof(i) - 1); /* stay on last limb */ } @@ -628,43 +533,66 @@ int BN_bn2binpad(const BIGNUM *a, unsigned char *to, int tolen) { if (tolen < 0) return -1; - return bn2binpad(a, to, tolen, BIG, UNSIGNED); -} - -int BN_signed_bn2bin(const BIGNUM *a, unsigned char *to, int tolen) -{ - if (tolen < 0) - return -1; - return bn2binpad(a, to, tolen, BIG, SIGNED); + return bn2binpad(a, to, tolen, big); } int BN_bn2bin(const BIGNUM *a, unsigned char *to) { - return bn2binpad(a, to, -1, BIG, UNSIGNED); + return bn2binpad(a, to, -1, big); } BIGNUM *BN_lebin2bn(const unsigned char *s, int len, BIGNUM *ret) { - return bin2bn(s, len, ret, LITTLE, UNSIGNED); -} + unsigned int i, m; + unsigned int n; + BN_ULONG l; + BIGNUM *bn = NULL; -BIGNUM *BN_signed_lebin2bn(const unsigned char *s, int len, BIGNUM *ret) -{ - return bin2bn(s, len, ret, LITTLE, SIGNED); + if (ret == NULL) + ret = bn = BN_new(); + if (ret == NULL) + return NULL; + bn_check_top(ret); + s += len; + /* Skip trailing zeroes. */ + for ( ; len > 0 && s[-1] == 0; s--, len--) + continue; + n = len; + if (n == 0) { + ret->top = 0; + return ret; + } + i = ((n - 1) / BN_BYTES) + 1; + m = ((n - 1) % (BN_BYTES)); + if (bn_wexpand(ret, (int)i) == NULL) { + BN_free(bn); + return NULL; + } + ret->top = i; + ret->neg = 0; + l = 0; + while (n--) { + s--; + l = (l << 8L) | *s; + if (m-- == 0) { + ret->d[--i] = l; + l = 0; + m = BN_BYTES - 1; + } + } + /* + * need to call this due to clear byte at top if avoiding having the top + * bit set (-ve number) + */ + bn_correct_top(ret); + return ret; } int BN_bn2lebinpad(const BIGNUM *a, unsigned char *to, int tolen) { if (tolen < 0) return -1; - return bn2binpad(a, to, tolen, LITTLE, UNSIGNED); -} - -int BN_signed_bn2lebin(const BIGNUM *a, unsigned char *to, int tolen) -{ - if (tolen < 0) - return -1; - return bn2binpad(a, to, tolen, LITTLE, SIGNED); + return bn2binpad(a, to, tolen, little); } BIGNUM *BN_native2bn(const unsigned char *s, int len, BIGNUM *ret) @@ -676,15 +604,6 @@ BIGNUM *BN_native2bn(const unsigned char *s, int len, BIGNUM *ret) return BN_bin2bn(s, len, ret); } -BIGNUM *BN_signed_native2bn(const unsigned char *s, int len, BIGNUM *ret) -{ - DECLARE_IS_ENDIAN; - - if (IS_LITTLE_ENDIAN) - return BN_signed_lebin2bn(s, len, ret); - return BN_signed_bin2bn(s, len, ret); -} - int BN_bn2nativepad(const BIGNUM *a, unsigned char *to, int tolen) { DECLARE_IS_ENDIAN; @@ -694,43 +613,19 @@ int BN_bn2nativepad(const BIGNUM *a, unsigned char *to, int tolen) return BN_bn2binpad(a, to, tolen); } -int BN_signed_bn2native(const BIGNUM *a, unsigned char *to, int tolen) -{ - DECLARE_IS_ENDIAN; - - if (IS_LITTLE_ENDIAN) - return BN_signed_bn2lebin(a, to, tolen); - return BN_signed_bn2bin(a, to, tolen); -} - int BN_ucmp(const BIGNUM *a, const BIGNUM *b) { int i; BN_ULONG t1, t2, *ap, *bp; - ap = a->d; - bp = b->d; - - if (BN_get_flags(a, BN_FLG_CONSTTIME) - && a->top == b->top) { - int res = 0; - - for (i = 0; i < b->top; i++) { - res = constant_time_select_int(constant_time_lt_bn(ap[i], bp[i]), - -1, res); - res = constant_time_select_int(constant_time_lt_bn(bp[i], ap[i]), - 1, res); - } - return res; - } - bn_check_top(a); bn_check_top(b); i = a->top - b->top; if (i != 0) return i; - + ap = a->d; + bp = b->d; for (i = a->top - 1; i >= 0; i--) { t1 = ap[i]; t2 = bp[i]; @@ -842,10 +737,11 @@ int BN_is_bit_set(const BIGNUM *a, int n) return (int)(((a->d[i]) >> j) & ((BN_ULONG)1)); } -int ossl_bn_mask_bits_fixed_top(BIGNUM *a, int n) +int BN_mask_bits(BIGNUM *a, int n) { int b, w; + bn_check_top(a); if (n < 0) return 0; @@ -859,21 +755,10 @@ int ossl_bn_mask_bits_fixed_top(BIGNUM *a, int n) a->top = w + 1; a->d[w] &= ~(BN_MASK2 << b); } - a->flags |= BN_FLG_FIXED_TOP; + bn_correct_top(a); return 1; } -int BN_mask_bits(BIGNUM *a, int n) -{ - int ret; - - bn_check_top(a); - ret = ossl_bn_mask_bits_fixed_top(a, n); - if (ret) - bn_correct_top(a); - return ret; -} - void BN_set_negative(BIGNUM *a, int b) { if (b && !BN_is_zero(a)) @@ -944,6 +829,9 @@ void BN_consttime_swap(BN_ULONG condition, BIGNUM *a, BIGNUM *b, int nwords) BN_ULONG t; int i; + if (a == b) + return; + bn_wcheck_size(a, nwords); bn_wcheck_size(b, nwords); @@ -1047,22 +935,6 @@ int BN_is_word(const BIGNUM *a, const BN_ULONG w) return BN_abs_is_word(a, w) && (!w || !a->neg); } -int ossl_bn_is_word_fixed_top(const BIGNUM *a, const BN_ULONG w) -{ - int res, i; - const BN_ULONG *ap = a->d; - - if (a->neg || a->top == 0) - return 0; - - res = constant_time_select_int(constant_time_eq_bn(ap[0], w), 1, 0); - - for (i = 1; i < a->top; i++) - res = constant_time_select_int(constant_time_is_zero_bn(ap[i]), - res, 0); - return res; -} - int BN_is_odd(const BIGNUM *a) { return (a->top > 0) && (a->d[0] & 1); @@ -1094,8 +966,10 @@ BN_GENCB *BN_GENCB_new(void) { BN_GENCB *ret; - if ((ret = OPENSSL_malloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_malloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; + } return ret; } @@ -1147,28 +1021,6 @@ BIGNUM *bn_wexpand(BIGNUM *a, int words) return (words <= a->dmax) ? a : bn_expand2(a, words); } -void bn_correct_top_consttime(BIGNUM *a) -{ - int j, atop; - BN_ULONG limb; - unsigned int mask; - - for (j = 0, atop = 0; j < a->dmax; j++) { - limb = a->d[j]; - limb |= 0 - limb; - limb >>= BN_BITS2 - 1; - limb = 0 - limb; - mask = (unsigned int)limb; - mask &= constant_time_msb(j - a->top); - atop = constant_time_select_int(mask, j + 1, atop); - } - - mask = constant_time_eq_int(atop, 0); - a->top = atop; - a->neg = constant_time_select_int(mask, 0, a->neg); - a->flags &= ~BN_FLG_FIXED_TOP; -} - void bn_correct_top(BIGNUM *a) { BN_ULONG *ftl; diff --git a/openssl/src/crypto/bn/bn_local.h b/openssl/src/crypto/bn/bn_local.h index b5be37ba9..70deff09c 100644 --- a/openssl/src/crypto/bn/bn_local.h +++ b/openssl/src/crypto/bn/bn_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -42,26 +42,6 @@ # include # endif -/* - * This should limit the stack usage due to alloca to about 4K. - * BN_SOFT_LIMIT is a soft limit equivalent to 2*OPENSSL_RSA_MAX_MODULUS_BITS. - * Beyond that size bn_mul_mont is no longer used, and the constant time - * assembler code is disabled, due to the blatant alloca and bn_mul_mont usage. - * Note that bn_mul_mont does an alloca that is hidden away in assembly. - * It is not recommended to do computations with numbers exceeding this limit, - * since the result will be highly version dependent: - * While the current OpenSSL version will use non-optimized, but safe code, - * previous versions will use optimized code, that may crash due to unexpected - * stack overflow, and future versions may very well turn this into a hard - * limit. - * Note however, that it is possible to override the size limit using - * "./config -DBN_SOFT_LIMIT=" if necessary, and the O/S specific - * stack limit is known and taken into consideration. - */ -# ifndef BN_SOFT_LIMIT -# define BN_SOFT_LIMIT (4096 / BN_BYTES) -# endif - # ifndef OPENSSL_SMALL_FOOTPRINT # define BN_MUL_COMBA # define BN_SQR_COMBA @@ -243,11 +223,8 @@ BN_ULONG bn_sub_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, int num); struct bignum_st { - BN_ULONG *d; /* - * Pointer to an array of 'BN_BITS2' bit - * chunks. These chunks are organised in - * a least significant chunk first order. - */ + BN_ULONG *d; /* Pointer to an array of 'BN_BITS2' bit + * chunks. */ int top; /* Index of last used d +1. */ /* The next are internal book keeping for bn_expand. */ int dmax; /* Size of the d array. */ @@ -293,6 +270,20 @@ struct bn_gencb_st { } cb; }; +struct bn_blinding_st { + BIGNUM *A; + BIGNUM *Ai; + BIGNUM *e; + BIGNUM *mod; /* just a reference */ + CRYPTO_THREAD_ID tid; + int counter; + unsigned long flags; + BN_MONT_CTX *m_ctx; + int (*bn_mod_exp) (BIGNUM *r, const BIGNUM *a, const BIGNUM *p, + const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx); + CRYPTO_RWLOCK *lock; +}; + /*- * BN_window_bits_for_exponent_size -- macro for sliding window mod_exp functions * @@ -367,6 +358,10 @@ struct bn_gencb_st { # define BN_MUL_LOW_RECURSIVE_SIZE_NORMAL (32)/* 32 */ # define BN_MONT_CTX_SET_SIZE_WORD (64)/* 32 */ +# if !defined(PTR_SIZE_INT) +# define PTR_SIZE_INT size_t +# endif + # if !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) && !defined(PEDANTIC) /* * BN_UMULT_HIGH section. @@ -507,10 +502,10 @@ unsigned __int64 _umul128(unsigned __int64 a, unsigned __int64 b, ret = (r); \ BN_UMULT_LOHI(low,high,w,tmp); \ ret += (c); \ - (c) = (ret<(c)); \ + (c) = (ret<(c))?1:0; \ (c) += high; \ ret += low; \ - (c) += (ret>(BN_BITS4-1); \ m =(m&BN_MASK2l)<<(BN_BITS4+1); \ - l=(l+m)&BN_MASK2; h += (l < m); \ + l=(l+m)&BN_MASK2; if (l < m) h++; \ (lo)=l; \ (ho)=h; \ } @@ -615,9 +610,9 @@ unsigned __int64 _umul128(unsigned __int64 a, unsigned __int64 b, mul64(l,h,(bl),(bh)); \ \ /* non-multiply part */ \ - l=(l+(c))&BN_MASK2; h += (l < (c)); \ + l=(l+(c))&BN_MASK2; if (l < (c)) h++; \ (c)=(r); \ - l=(l+(c))&BN_MASK2; h += (l < (c)); \ + l=(l+(c))&BN_MASK2; if (l < (c)) h++; \ (c)=h&BN_MASK2; \ (r)=l; \ } @@ -631,12 +626,88 @@ unsigned __int64 _umul128(unsigned __int64 a, unsigned __int64 b, mul64(l,h,(bl),(bh)); \ \ /* non-multiply part */ \ - l+=(c); h += ((l&BN_MASK2) < (c)); \ + l+=(c); if ((l&BN_MASK2) < (c)) h++; \ (c)=h&BN_MASK2; \ (r)=l&BN_MASK2; \ } # endif /* !BN_LLONG */ +/* How many bignums are in each "pool item"; */ +# define BN_CTX_POOL_SIZE 16 +/* The stack frame info is resizing, set a first-time expansion size; */ +# define BN_CTX_START_FRAMES 32 + +/***********/ +/* BN_POOL */ +/***********/ + +/* A bundle of bignums that can be linked with other bundles */ +typedef struct bignum_pool_item { + /* The bignum values */ + BIGNUM vals[BN_CTX_POOL_SIZE]; + /* Linked-list admin */ + struct bignum_pool_item *prev, *next; +} BN_POOL_ITEM; +/* A linked-list of bignums grouped in bundles */ +typedef struct bignum_pool { + /* Linked-list admin */ + BN_POOL_ITEM *head, *current, *tail; + /* Stack depth and allocation size */ + unsigned used, size; +} BN_POOL; + +/************/ +/* BN_STACK */ +/************/ + +/* A wrapper to manage the "stack frames" */ +typedef struct bignum_ctx_stack { + /* Array of indexes into the bignum stack */ + unsigned int *indexes; + /* Number of stack frames, and the size of the allocated array */ + unsigned int depth, size; +} BN_STACK; + +/**********/ +/* BN_CTX */ +/**********/ + +/* The opaque BN_CTX type */ +struct bignum_ctx { + /* The bignum bundles */ + BN_POOL pool; + /* The "stack frames", if you will */ + BN_STACK stack; + /* The number of bignums currently assigned */ + unsigned int used; + /* Depth of stack overflow */ + int err_stack; + /* Block "gets" until an "end" (compatibility behaviour) */ + int too_many; + /* Flags. */ + int flags; + /* The library context */ + OSSL_LIB_CTX *libctx; +# ifndef OPENSSL_NO_BN_METHOD + ENGINE *engine; + const BN_METHOD *bn_meth; +# endif +}; + +# ifndef OPENSSL_NO_BN_METHOD +struct bn_method_st { + char *name; + int (*mod_add)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx); + int (*mod_sub)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx); + int (*mod_mul)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx); + int (*mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, BN_CTX *ctx); + int (*mod_sqr)(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx); + BIGNUM *(*mod_sqrt)(BIGNUM *r, const BIGNUM *a, const BIGNUM *n, BN_CTX *ctx); + BIGNUM *(*mod_inverse)(BIGNUM *r, const BIGNUM *a, const BIGNUM *n, BN_CTX *ctx); + int (*div)(BIGNUM *dv, BIGNUM *rem, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx); +}; +# endif + void BN_RECP_CTX_init(BN_RECP_CTX *recp); void BN_MONT_CTX_init(BN_MONT_CTX *ctx); @@ -661,7 +732,7 @@ BN_ULONG bn_sub_part_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, int cl, int dl); int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, int num); -void bn_correct_top_consttime(BIGNUM *a); + BIGNUM *int_bn_mod_inverse(BIGNUM *in, const BIGNUM *a, const BIGNUM *n, BN_CTX *ctx, int *noinv); diff --git a/openssl/src/crypto/bn/bn_meth.c b/openssl/src/crypto/bn/bn_meth.c new file mode 100644 index 000000000..51692c01b --- /dev/null +++ b/openssl/src/crypto/bn/bn_meth.c @@ -0,0 +1,179 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "bn_local.h" + +/** Creates a new BN_METHOD object for the specified name + * \param name the method name + * \return newly created BN_METHOD object or NULL if an error occurred + */ +BN_METHOD *BN_METHOD_new(const char *name) +{ + BN_METHOD *ret = OPENSSL_zalloc(sizeof(*ret)); + + if (ret == NULL) + return NULL; + + ret->name = OPENSSL_strdup(name); + if (ret->name == NULL) { + OPENSSL_free(ret); + return NULL; + } + + return ret; +} + +/** Frees a BN_METHOD object + * \param meth BN_METHOD object to be freed + */ +void BN_METHOD_free(BN_METHOD *meth) +{ + OPENSSL_free(meth->name); + OPENSSL_free(meth); +} + +/** Copies BN_METHOD object + * \param dst destination BN_METHOD object + * \param src source BN_METHOD object + * \return 1 on success and 0 if an error occurred + */ +int BN_METHOD_copy(BN_METHOD *dst, const BN_METHOD *src) +{ + char *name = NULL; + if (dst == src) + return 1; + + name = OPENSSL_strdup(src->name); + if (name == NULL) + return 0; + + if (dst->name) + OPENSSL_free(dst->name); + + memcpy(dst, src, sizeof(*dst)); + dst->name = name; + + return 1; +} + +/** Returns the name of a BN_METHOD object + * \param meth BN_METHOD object + */ +char *BN_METHOD_name(BN_METHOD *meth) +{ + return meth->name; +} + +int (*BN_METHOD_get_add(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx) +{ + return meth->mod_add; +} + +void BN_METHOD_set_add(BN_METHOD *meth, + int (*mod_add)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, + const BIGNUM *m, BN_CTX *ctx)) +{ + meth->mod_add = mod_add; +} + +int (*BN_METHOD_get_sub(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx) +{ + return meth->mod_sub; +} + +void BN_METHOD_set_sub(BN_METHOD *meth, + int (*mod_sub)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, + const BIGNUM *m, BN_CTX *ctx)) +{ + meth->mod_sub = mod_sub; +} + +int (*BN_METHOD_get_mul(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx) +{ + return meth->mod_mul; +} + +void BN_METHOD_set_mul(BN_METHOD *meth, + int (*mod_mul)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, + const BIGNUM *m, BN_CTX *ctx)) +{ + meth->mod_mul = mod_mul; +} + +int (*BN_METHOD_get_exp(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx) +{ + return meth->mod_exp; +} + +void BN_METHOD_set_exp(BN_METHOD *meth, + int (*mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, + const BIGNUM *m, BN_CTX *ctx)) +{ + meth->mod_exp = mod_exp; +} + +int (*BN_METHOD_get_sqr(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx) +{ + return meth->mod_sqr; +} + +void BN_METHOD_set_sqr(BN_METHOD *meth, + int (*mod_sqr)(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, + BN_CTX *ctx)) +{ + meth->mod_sqr = mod_sqr; +} + +int (*BN_METHOD_get_div(BN_METHOD *meth)) + (BIGNUM *dv, BIGNUM *rem, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx) +{ + return meth->div; +} + +void BN_METHOD_set_div(BN_METHOD *meth, + int (*div)(BIGNUM *dv, BIGNUM *rem, const BIGNUM *m, + const BIGNUM *d, BN_CTX *ctx)) +{ + meth->div = div; +} + +BIGNUM *(*BN_METHOD_get_sqrt(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *n, BN_CTX *ctx) +{ + return meth->mod_sqrt; +} + +void BN_METHOD_set_sqrt(BN_METHOD *meth, + BIGNUM *(*mod_sqrt)(BIGNUM *r, const BIGNUM *a, + const BIGNUM *n, BN_CTX *ctx)) +{ + meth->mod_sqrt = mod_sqrt; +} + +BIGNUM *(*BN_METHOD_get_inverse(BN_METHOD *meth)) + (BIGNUM *r, const BIGNUM *a, const BIGNUM *n, BN_CTX *ctx) +{ + return meth->mod_inverse; +} + +void BN_METHOD_set_inverse(BN_METHOD *meth, + BIGNUM *(*mod_inverse)(BIGNUM *r, const BIGNUM *a, + const BIGNUM *n, BN_CTX *ctx)) +{ + meth->mod_inverse = mod_inverse; +} diff --git a/openssl/src/crypto/bn/bn_mod.c b/openssl/src/crypto/bn/bn_mod.c index d7c2f4bd5..052c29ba8 100644 --- a/openssl/src/crypto/bn/bn_mod.c +++ b/openssl/src/crypto/bn/bn_mod.c @@ -17,11 +17,6 @@ int BN_nnmod(BIGNUM *r, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx) * always holds) */ - if (r == d) { - ERR_raise(ERR_LIB_BN, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (!(BN_mod(r, m, d, ctx))) return 0; if (!r->neg) @@ -33,6 +28,11 @@ int BN_nnmod(BIGNUM *r, const BIGNUM *m, const BIGNUM *d, BN_CTX *ctx) int BN_mod_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx) { +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->mod_add) + return ctx->bn_meth->mod_add(r, a, b, m, ctx); +#endif + if (!BN_add(r, a, b)) return 0; return BN_nnmod(r, r, m, ctx); @@ -63,8 +63,10 @@ int bn_mod_add_fixed_top(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, if (mtop > sizeof(storage) / sizeof(storage[0])) { tp = OPENSSL_malloc(mtop * sizeof(BN_ULONG)); - if (tp == NULL) + if (tp == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return 0; + } } ap = a->d != NULL ? a->d : tp; @@ -113,6 +115,11 @@ int BN_mod_add_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, int BN_mod_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BN_CTX *ctx) { +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->mod_sub) + return ctx->bn_meth->mod_sub(r, a, b, m, ctx); +#endif + if (!BN_sub(r, a, b)) return 0; return BN_nnmod(r, r, m, ctx); @@ -189,11 +196,6 @@ int bn_mod_sub_fixed_top(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, int BN_mod_sub_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m) { - if (r == m) { - ERR_raise(ERR_LIB_BN, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (!BN_sub(r, a, b)) return 0; if (r->neg) @@ -208,6 +210,11 @@ int BN_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, BIGNUM *t; int ret = 0; +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->mod_mul) + return ctx->bn_meth->mod_mul(r, a, b, m, ctx); +#endif + bn_check_top(a); bn_check_top(b); bn_check_top(m); @@ -233,6 +240,11 @@ int BN_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m, int BN_mod_sqr(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx) { +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->mod_sqr) + return ctx->bn_meth->mod_sqr(r, a, m, ctx); +#endif + if (!BN_sqr(r, a, ctx)) return 0; /* r->neg == 0, thus we don't need BN_nnmod */ diff --git a/openssl/src/crypto/bn/bn_mont.c b/openssl/src/crypto/bn/bn_mont.c index 8b4c7900a..735989d98 100644 --- a/openssl/src/crypto/bn/bn_mont.c +++ b/openssl/src/crypto/bn/bn_mont.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -42,7 +42,7 @@ int bn_mul_mont_fixed_top(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, int num = mont->N.top; #if defined(OPENSSL_BN_ASM_MONT) && defined(MONT_WORD) - if (num > 1 && num <= BN_SOFT_LIMIT && a->top == num && b->top == num) { + if (num > 1 && a->top == num && b->top == num) { if (bn_wexpand(r, num) == NULL) return 0; if (bn_mul_mont(r->d, a->d, b->d, mont->N.d, mont->n0, num)) { @@ -229,8 +229,10 @@ BN_MONT_CTX *BN_MONT_CTX_new(void) { BN_MONT_CTX *ret; - if ((ret = OPENSSL_malloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_malloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; + } BN_MONT_CTX_init(ret); ret->flags = BN_FLG_MALLOCED; diff --git a/openssl/src/crypto/bn/bn_mul.c b/openssl/src/crypto/bn/bn_mul.c index c3864b8c6..2c5ecc2e7 100644 --- a/openssl/src/crypto/bn/bn_mul.c +++ b/openssl/src/crypto/bn/bn_mul.c @@ -115,12 +115,12 @@ BN_ULONG bn_sub_part_words(BN_ULONG *r, r[1] = a[1]; if (--dl <= 0) break; - /* fall through */ + /* fall thru */ case 2: r[2] = a[2]; if (--dl <= 0) break; - /* fall through */ + /* fall thru */ case 3: r[3] = a[3]; if (--dl <= 0) @@ -496,6 +496,10 @@ void bn_mul_low_recursive(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n2, int BN_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx) { +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->mod_mul) + return ctx->bn_meth->mod_mul(r, a, b, NULL, ctx); +#endif int ret = bn_mul_fixed_top(r, a, b, ctx); bn_correct_top(r); diff --git a/openssl/src/crypto/bn/bn_nist.c b/openssl/src/crypto/bn/bn_nist.c index bc864346f..5b6d4e7e4 100644 --- a/openssl/src/crypto/bn/bn_nist.c +++ b/openssl/src/crypto/bn/bn_nist.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -251,7 +251,7 @@ const BIGNUM *BN_get0_nist_prime_521(void) /* * To avoid more recent compilers (specifically clang-14) from treating this - * code as a violation of the strict aliasing conditions and omitting it, this + * code as a violation of the strict aliasing conditions and omiting it, this * cannot be declared as a function. Moreover, the dst parameter cannot be * cached in a local since this no longer references the union and again falls * foul of the strict aliasing criteria. Refer to #18225 for the initial @@ -319,28 +319,6 @@ static void nist_cp_bn(BN_ULONG *dst, const BN_ULONG *src, int top) # endif #endif /* BN_BITS2 != 64 */ -#ifdef NIST_INT64 -/* Helpers to load/store a 32-bit word (uint32_t) from/into a memory - * location and avoid potential aliasing issue. */ -static ossl_inline uint32_t load_u32(const void *ptr) -{ - uint32_t tmp; - - memcpy(&tmp, ptr, sizeof(tmp)); - return tmp; -} - -static ossl_inline void store_lo32(void *ptr, NIST_INT64 val) -{ - /* A cast is needed for big-endian system: on a 32-bit BE system - * NIST_INT64 may be defined as well if the compiler supports 64-bit - * long long. */ - uint32_t tmp = (uint32_t)val; - - memcpy(ptr, &tmp, sizeof(tmp)); -} -#endif /* NIST_INT64 */ - #define nist_set_192(to, from, a1, a2, a3) \ { \ bn_cp_64(to, 0, from, (a3) - 3) \ @@ -360,6 +338,7 @@ int BN_nist_mod_192(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, sizeof(unsigned int)]; } buf; BN_ULONG c_d[BN_NIST_192_TOP], *res; + PTR_SIZE_INT mask; static const BIGNUM ossl_bignum_nist_p_192_sqr = { (BN_ULONG *)_nist_p_192_sqr, OSSL_NELEM(_nist_p_192_sqr), @@ -396,42 +375,42 @@ int BN_nist_mod_192(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, unsigned int *rp = (unsigned int *)r_d; const unsigned int *bp = (const unsigned int *)buf.ui; - acc = load_u32(&rp[0]); + acc = rp[0]; acc += bp[3 * 2 - 6]; acc += bp[5 * 2 - 6]; - store_lo32(&rp[0], acc); + rp[0] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[1]); + acc += rp[1]; acc += bp[3 * 2 - 5]; acc += bp[5 * 2 - 5]; - store_lo32(&rp[1], acc); + rp[1] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[2]); + acc += rp[2]; acc += bp[3 * 2 - 6]; acc += bp[4 * 2 - 6]; acc += bp[5 * 2 - 6]; - store_lo32(&rp[2], acc); + rp[2] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[3]); + acc += rp[3]; acc += bp[3 * 2 - 5]; acc += bp[4 * 2 - 5]; acc += bp[5 * 2 - 5]; - store_lo32(&rp[3], acc); + rp[3] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[4]); + acc += rp[4]; acc += bp[4 * 2 - 6]; acc += bp[5 * 2 - 6]; - store_lo32(&rp[4], acc); + rp[4] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[5]); + acc += rp[5]; acc += bp[4 * 2 - 5]; acc += bp[5 * 2 - 5]; - store_lo32(&rp[5], acc); + rp[5] = (unsigned int)acc; carry = (int)(acc >> 32); } @@ -460,9 +439,13 @@ int BN_nist_mod_192(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, * 'tmp=result-modulus; if (!carry || !borrow) result=tmp;' * this is what happens below, but without explicit if:-) a. */ - res = (bn_sub_words(c_d, r_d, _nist_p_192[0], BN_NIST_192_TOP) && carry) - ? r_d - : c_d; + mask = + 0 - (PTR_SIZE_INT) bn_sub_words(c_d, r_d, _nist_p_192[0], + BN_NIST_192_TOP); + mask &= 0 - (PTR_SIZE_INT) carry; + res = c_d; + res = (BN_ULONG *) + (((PTR_SIZE_INT) res & ~mask) | ((PTR_SIZE_INT) r_d & mask)); nist_cp_bn(r_d, res, BN_NIST_192_TOP); r->top = BN_NIST_192_TOP; bn_correct_top(r); @@ -496,7 +479,11 @@ int BN_nist_mod_224(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, sizeof(unsigned int)]; } buf; BN_ULONG c_d[BN_NIST_224_TOP], *res; - bn_addsub_f adjust; + PTR_SIZE_INT mask; + union { + bn_addsub_f f; + PTR_SIZE_INT p; + } u; static const BIGNUM ossl_bignum_nist_p_224_sqr = { (BN_ULONG *)_nist_p_224_sqr, OSSL_NELEM(_nist_p_224_sqr), @@ -610,7 +597,7 @@ int BN_nist_mod_224(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, # endif } #endif - adjust = bn_sub_words; + u.f = bn_sub_words; if (carry > 0) { carry = (int)bn_sub_words(r_d, r_d, _nist_p_224[carry - 1], @@ -629,14 +616,19 @@ int BN_nist_mod_224(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, carry = (int)bn_add_words(r_d, r_d, _nist_p_224[-carry - 1], BN_NIST_224_TOP); - adjust = carry ? bn_sub_words : bn_add_words; + mask = 0 - (PTR_SIZE_INT) carry; + u.p = ((PTR_SIZE_INT) bn_sub_words & mask) | + ((PTR_SIZE_INT) bn_add_words & ~mask); } else carry = 1; /* otherwise it's effectively same as in BN_nist_mod_192... */ - res = ((*adjust) (c_d, r_d, _nist_p_224[0], BN_NIST_224_TOP) && carry) - ? r_d - : c_d; + mask = + 0 - (PTR_SIZE_INT) (*u.f) (c_d, r_d, _nist_p_224[0], BN_NIST_224_TOP); + mask &= 0 - (PTR_SIZE_INT) carry; + res = c_d; + res = (BN_ULONG *)(((PTR_SIZE_INT) res & ~mask) | + ((PTR_SIZE_INT) r_d & mask)); nist_cp_bn(r_d, res, BN_NIST_224_TOP); r->top = BN_NIST_224_TOP; bn_correct_top(r); @@ -668,7 +660,11 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, sizeof(unsigned int)]; } buf; BN_ULONG c_d[BN_NIST_256_TOP], *res; - bn_addsub_f adjust; + PTR_SIZE_INT mask; + union { + bn_addsub_f f; + PTR_SIZE_INT p; + } u; static const BIGNUM ossl_bignum_nist_p_256_sqr = { (BN_ULONG *)_nist_p_256_sqr, OSSL_NELEM(_nist_p_256_sqr), @@ -705,36 +701,36 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, unsigned int *rp = (unsigned int *)r_d; const unsigned int *bp = (const unsigned int *)buf.ui; - acc = load_u32(&rp[0]); + acc = rp[0]; acc += bp[8 - 8]; acc += bp[9 - 8]; acc -= bp[11 - 8]; acc -= bp[12 - 8]; acc -= bp[13 - 8]; acc -= bp[14 - 8]; - store_lo32(&rp[0], acc); + rp[0] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[1]); + acc += rp[1]; acc += bp[9 - 8]; acc += bp[10 - 8]; acc -= bp[12 - 8]; acc -= bp[13 - 8]; acc -= bp[14 - 8]; acc -= bp[15 - 8]; - store_lo32(&rp[1], acc); + rp[1] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[2]); + acc += rp[2]; acc += bp[10 - 8]; acc += bp[11 - 8]; acc -= bp[13 - 8]; acc -= bp[14 - 8]; acc -= bp[15 - 8]; - store_lo32(&rp[2], acc); + rp[2] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[3]); + acc += rp[3]; acc += bp[11 - 8]; acc += bp[11 - 8]; acc += bp[12 - 8]; @@ -743,10 +739,10 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc -= bp[15 - 8]; acc -= bp[8 - 8]; acc -= bp[9 - 8]; - store_lo32(&rp[3], acc); + rp[3] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[4]); + acc += rp[4]; acc += bp[12 - 8]; acc += bp[12 - 8]; acc += bp[13 - 8]; @@ -754,10 +750,10 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc += bp[14 - 8]; acc -= bp[9 - 8]; acc -= bp[10 - 8]; - store_lo32(&rp[4], acc); + rp[4] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[5]); + acc += rp[5]; acc += bp[13 - 8]; acc += bp[13 - 8]; acc += bp[14 - 8]; @@ -765,10 +761,10 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc += bp[15 - 8]; acc -= bp[10 - 8]; acc -= bp[11 - 8]; - store_lo32(&rp[5], acc); + rp[5] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[6]); + acc += rp[6]; acc += bp[14 - 8]; acc += bp[14 - 8]; acc += bp[15 - 8]; @@ -777,10 +773,10 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc += bp[13 - 8]; acc -= bp[8 - 8]; acc -= bp[9 - 8]; - store_lo32(&rp[6], acc); + rp[6] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[7]); + acc += rp[7]; acc += bp[15 - 8]; acc += bp[15 - 8]; acc += bp[15 - 8]; @@ -789,7 +785,7 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc -= bp[11 - 8]; acc -= bp[12 - 8]; acc -= bp[13 - 8]; - store_lo32(&rp[7], acc); + rp[7] = (unsigned int)acc; carry = (int)(acc >> 32); } @@ -854,7 +850,7 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, } #endif /* see BN_nist_mod_224 for explanation */ - adjust = bn_sub_words; + u.f = bn_sub_words; if (carry > 0) carry = (int)bn_sub_words(r_d, r_d, _nist_p_256[carry - 1], @@ -863,13 +859,18 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, carry = (int)bn_add_words(r_d, r_d, _nist_p_256[-carry - 1], BN_NIST_256_TOP); - adjust = carry ? bn_sub_words : bn_add_words; + mask = 0 - (PTR_SIZE_INT) carry; + u.p = ((PTR_SIZE_INT) bn_sub_words & mask) | + ((PTR_SIZE_INT) bn_add_words & ~mask); } else carry = 1; - res = ((*adjust) (c_d, r_d, _nist_p_256[0], BN_NIST_256_TOP) && carry) - ? r_d - : c_d; + mask = + 0 - (PTR_SIZE_INT) (*u.f) (c_d, r_d, _nist_p_256[0], BN_NIST_256_TOP); + mask &= 0 - (PTR_SIZE_INT) carry; + res = c_d; + res = (BN_ULONG *)(((PTR_SIZE_INT) res & ~mask) | + ((PTR_SIZE_INT) r_d & mask)); nist_cp_bn(r_d, res, BN_NIST_256_TOP); r->top = BN_NIST_256_TOP; bn_correct_top(r); @@ -905,7 +906,11 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, sizeof(unsigned int)]; } buf; BN_ULONG c_d[BN_NIST_384_TOP], *res; - bn_addsub_f adjust; + PTR_SIZE_INT mask; + union { + bn_addsub_f f; + PTR_SIZE_INT p; + } u; static const BIGNUM ossl_bignum_nist_p_384_sqr = { (BN_ULONG *)_nist_p_384_sqr, OSSL_NELEM(_nist_p_384_sqr), @@ -942,32 +947,32 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, unsigned int *rp = (unsigned int *)r_d; const unsigned int *bp = (const unsigned int *)buf.ui; - acc = load_u32(&rp[0]); + acc = rp[0]; acc += bp[12 - 12]; acc += bp[21 - 12]; acc += bp[20 - 12]; acc -= bp[23 - 12]; - store_lo32(&rp[0], acc); + rp[0] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[1]); + acc += rp[1]; acc += bp[13 - 12]; acc += bp[22 - 12]; acc += bp[23 - 12]; acc -= bp[12 - 12]; acc -= bp[20 - 12]; - store_lo32(&rp[1], acc); + rp[1] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[2]); + acc += rp[2]; acc += bp[14 - 12]; acc += bp[23 - 12]; acc -= bp[13 - 12]; acc -= bp[21 - 12]; - store_lo32(&rp[2], acc); + rp[2] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[3]); + acc += rp[3]; acc += bp[15 - 12]; acc += bp[12 - 12]; acc += bp[20 - 12]; @@ -975,10 +980,10 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc -= bp[14 - 12]; acc -= bp[22 - 12]; acc -= bp[23 - 12]; - store_lo32(&rp[3], acc); + rp[3] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[4]); + acc += rp[4]; acc += bp[21 - 12]; acc += bp[21 - 12]; acc += bp[16 - 12]; @@ -989,10 +994,10 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc -= bp[15 - 12]; acc -= bp[23 - 12]; acc -= bp[23 - 12]; - store_lo32(&rp[4], acc); + rp[4] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[5]); + acc += rp[5]; acc += bp[22 - 12]; acc += bp[22 - 12]; acc += bp[17 - 12]; @@ -1001,10 +1006,10 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc += bp[21 - 12]; acc += bp[23 - 12]; acc -= bp[16 - 12]; - store_lo32(&rp[5], acc); + rp[5] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[6]); + acc += rp[6]; acc += bp[23 - 12]; acc += bp[23 - 12]; acc += bp[18 - 12]; @@ -1012,48 +1017,48 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, acc += bp[14 - 12]; acc += bp[22 - 12]; acc -= bp[17 - 12]; - store_lo32(&rp[6], acc); + rp[6] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[7]); + acc += rp[7]; acc += bp[19 - 12]; acc += bp[16 - 12]; acc += bp[15 - 12]; acc += bp[23 - 12]; acc -= bp[18 - 12]; - store_lo32(&rp[7], acc); + rp[7] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[8]); + acc += rp[8]; acc += bp[20 - 12]; acc += bp[17 - 12]; acc += bp[16 - 12]; acc -= bp[19 - 12]; - store_lo32(&rp[8], acc); + rp[8] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[9]); + acc += rp[9]; acc += bp[21 - 12]; acc += bp[18 - 12]; acc += bp[17 - 12]; acc -= bp[20 - 12]; - store_lo32(&rp[9], acc); + rp[9] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[10]); + acc += rp[10]; acc += bp[22 - 12]; acc += bp[19 - 12]; acc += bp[18 - 12]; acc -= bp[21 - 12]; - store_lo32(&rp[10], acc); + rp[10] = (unsigned int)acc; acc >>= 32; - acc += load_u32(&rp[11]); + acc += rp[11]; acc += bp[23 - 12]; acc += bp[20 - 12]; acc += bp[19 - 12]; acc -= bp[22 - 12]; - store_lo32(&rp[11], acc); + rp[11] = (unsigned int)acc; carry = (int)(acc >> 32); } @@ -1126,7 +1131,7 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, } #endif /* see BN_nist_mod_224 for explanation */ - adjust = bn_sub_words; + u.f = bn_sub_words; if (carry > 0) carry = (int)bn_sub_words(r_d, r_d, _nist_p_384[carry - 1], @@ -1135,13 +1140,18 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, carry = (int)bn_add_words(r_d, r_d, _nist_p_384[-carry - 1], BN_NIST_384_TOP); - adjust = carry ? bn_sub_words : bn_add_words; + mask = 0 - (PTR_SIZE_INT) carry; + u.p = ((PTR_SIZE_INT) bn_sub_words & mask) | + ((PTR_SIZE_INT) bn_add_words & ~mask); } else carry = 1; - res = ((*adjust) (c_d, r_d, _nist_p_384[0], BN_NIST_384_TOP) && carry) - ? r_d - : c_d; + mask = + 0 - (PTR_SIZE_INT) (*u.f) (c_d, r_d, _nist_p_384[0], BN_NIST_384_TOP); + mask &= 0 - (PTR_SIZE_INT) carry; + res = c_d; + res = (BN_ULONG *)(((PTR_SIZE_INT) res & ~mask) | + ((PTR_SIZE_INT) r_d & mask)); nist_cp_bn(r_d, res, BN_NIST_384_TOP); r->top = BN_NIST_384_TOP; bn_correct_top(r); @@ -1158,6 +1168,7 @@ int BN_nist_mod_521(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, { int top = a->top, i; BN_ULONG *r_d, *a_d = a->d, t_d[BN_NIST_521_TOP], val, tmp, *res; + PTR_SIZE_INT mask; static const BIGNUM ossl_bignum_nist_p_521_sqr = { (BN_ULONG *)_nist_p_521_sqr, OSSL_NELEM(_nist_p_521_sqr), @@ -1210,10 +1221,12 @@ int BN_nist_mod_521(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, r_d[i] &= BN_NIST_521_TOP_MASK; bn_add_words(r_d, r_d, t_d, BN_NIST_521_TOP); - res = bn_sub_words(t_d, r_d, _nist_p_521, - BN_NIST_521_TOP) - ? r_d - : t_d; + mask = + 0 - (PTR_SIZE_INT) bn_sub_words(t_d, r_d, _nist_p_521, + BN_NIST_521_TOP); + res = t_d; + res = (BN_ULONG *)(((PTR_SIZE_INT) res & ~mask) | + ((PTR_SIZE_INT) r_d & mask)); nist_cp_bn(r_d, res, BN_NIST_521_TOP); r->top = BN_NIST_521_TOP; bn_correct_top(r); diff --git a/openssl/src/crypto/bn/bn_prime.c b/openssl/src/crypto/bn/bn_prime.c index 96eb1b3c3..9e2f6861a 100644 --- a/openssl/src/crypto/bn/bn_prime.c +++ b/openssl/src/crypto/bn/bn_prime.c @@ -145,8 +145,10 @@ int BN_generate_prime_ex2(BIGNUM *ret, int bits, int safe, } mods = OPENSSL_zalloc(sizeof(*mods) * NUMPRIMES); - if (mods == NULL) + if (mods == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return 0; + } BN_CTX_start(ctx); t = BN_CTX_get(ctx); @@ -250,17 +252,6 @@ int ossl_bn_check_prime(const BIGNUM *w, int checks, BN_CTX *ctx, return bn_is_prime_int(w, checks, ctx, do_trial_division, cb); } -/* - * Use this only for key generation. - * It always uses trial division. The number of checks - * (MR rounds) passed in is used without being clamped to a minimum value. - */ -int ossl_bn_check_generated_prime(const BIGNUM *w, int checks, BN_CTX *ctx, - BN_GENCB *cb) -{ - return bn_is_prime_int(w, checks, ctx, 1, cb); -} - int BN_check_prime(const BIGNUM *p, BN_CTX *ctx, BN_GENCB *cb) { return ossl_bn_check_prime(p, 0, ctx, 1, cb); @@ -317,10 +308,9 @@ static int bn_is_prime_int(const BIGNUM *w, int checks, BN_CTX *ctx, goto err; #endif - if (!ossl_bn_miller_rabin_is_prime(w, checks, ctx, cb, 0, &status)) { - ret = -1; + ret = ossl_bn_miller_rabin_is_prime(w, checks, ctx, cb, 0, &status); + if (!ret) goto err; - } ret = (status == BN_PRIMETEST_PROBABLY_PRIME); err: #ifndef FIPS_MODULE diff --git a/openssl/src/crypto/bn/bn_prime.h b/openssl/src/crypto/bn/bn_prime.h index 8a859ac02..d92f6dfa6 100644 --- a/openssl/src/crypto/bn/bn_prime.h +++ b/openssl/src/crypto/bn/bn_prime.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by crypto/bn/bn_prime.pl * - * Copyright 1998-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1998-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/bn/bn_rand.c b/openssl/src/crypto/bn/bn_rand.c index da537a07a..1b4959699 100644 --- a/openssl/src/crypto/bn/bn_rand.c +++ b/openssl/src/crypto/bn/bn_rand.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -41,8 +41,10 @@ static int bnrand(BNRAND_FLAG flag, BIGNUM *rnd, int bits, int top, int bottom, mask = 0xff << (bit + 1); buf = OPENSSL_malloc(bytes); - if (buf == NULL) + if (buf == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); goto err; + } /* make a random number and set the top and bottom bits */ b = flag == NORMAL ? RAND_bytes_ex(libctx, buf, bytes, strength) @@ -134,11 +136,6 @@ static int bnrand_range(BNRAND_FLAG flag, BIGNUM *r, const BIGNUM *range, int n; int count = 100; - if (r == NULL) { - ERR_raise(ERR_LIB_BN, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (range->neg || BN_is_zero(range)) { ERR_raise(ERR_LIB_BN, BN_R_INVALID_RANGE); return 0; @@ -184,8 +181,8 @@ static int bnrand_range(BNRAND_FLAG flag, BIGNUM *r, const BIGNUM *range, } else { do { /* range = 11..._2 or range = 101..._2 */ - if (!bnrand(flag, r, n, BN_RAND_TOP_ANY, BN_RAND_BOTTOM_ANY, - strength, ctx)) + if (!bnrand(flag, r, n, BN_RAND_TOP_ANY, BN_RAND_BOTTOM_ANY, 0, + ctx)) return 0; if (!--count) { @@ -238,63 +235,17 @@ int BN_pseudo_rand_range(BIGNUM *r, const BIGNUM *range) # endif #endif -int ossl_bn_priv_rand_range_fixed_top(BIGNUM *r, const BIGNUM *range, - unsigned int strength, BN_CTX *ctx) -{ - int n; - int count = 100; - - if (r == NULL) { - ERR_raise(ERR_LIB_BN, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - if (range->neg || BN_is_zero(range)) { - ERR_raise(ERR_LIB_BN, BN_R_INVALID_RANGE); - return 0; - } - - n = BN_num_bits(range); /* n > 0 */ - - /* BN_is_bit_set(range, n - 1) always holds */ - - if (n == 1) { - BN_zero(r); - } else { - BN_set_flags(r, BN_FLG_CONSTTIME); - do { - if (!bnrand(PRIVATE, r, n + 1, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ANY, - strength, ctx)) - return 0; - - if (!--count) { - ERR_raise(ERR_LIB_BN, BN_R_TOO_MANY_ITERATIONS); - return 0; - } - ossl_bn_mask_bits_fixed_top(r, n); - } - while (BN_ucmp(r, range) >= 0); -#ifdef BN_DEBUG - /* With BN_DEBUG on a fixed top number cannot be returned */ - bn_correct_top(r); -#endif - } - - return 1; -} - /* - * ossl_bn_gen_dsa_nonce_fixed_top generates a random number 0 <= out < range. - * Unlike BN_rand_range, it also includes the contents of |priv| and |message| - * in the generation so that an RNG failure isn't fatal as long as |priv| + * BN_generate_dsa_nonce generates a random number 0 <= out < range. Unlike + * BN_rand_range, it also includes the contents of |priv| and |message| in + * the generation so that an RNG failure isn't fatal as long as |priv| * remains secret. This is intended for use in DSA and ECDSA where an RNG * weakness leads directly to private key exposure unless this function is * used. */ -int ossl_bn_gen_dsa_nonce_fixed_top(BIGNUM *out, const BIGNUM *range, - const BIGNUM *priv, - const unsigned char *message, - size_t message_len, BN_CTX *ctx) +int BN_generate_dsa_nonce(BIGNUM *out, const BIGNUM *range, + const BIGNUM *priv, const unsigned char *message, + size_t message_len, BN_CTX *ctx) { EVP_MD_CTX *mdctx = EVP_MD_CTX_new(); /* @@ -304,24 +255,20 @@ int ossl_bn_gen_dsa_nonce_fixed_top(BIGNUM *out, const BIGNUM *range, unsigned char random_bytes[64]; unsigned char digest[SHA512_DIGEST_LENGTH]; unsigned done, todo; - /* We generate |range|+1 bytes of random output. */ - const unsigned num_k_bytes = BN_num_bytes(range) + 1; + /* We generate |range|+8 bytes of random output. */ + const unsigned num_k_bytes = BN_num_bytes(range) + 8; unsigned char private_bytes[96]; unsigned char *k_bytes = NULL; - const int max_n = 64; /* Pr(failure to generate) < 2^max_n */ - int n; int ret = 0; EVP_MD *md = NULL; OSSL_LIB_CTX *libctx = ossl_bn_get_libctx(ctx); if (mdctx == NULL) - goto end; + goto err; k_bytes = OPENSSL_malloc(num_k_bytes); if (k_bytes == NULL) - goto end; - /* Ensure top byte is set to avoid non-constant time in bin2bn */ - k_bytes[0] = 0xff; + goto err; /* We copy |priv| into a local buffer to avoid exposing its length. */ if (BN_bn2binpad(priv, private_bytes, sizeof(private_bytes)) < 0) { @@ -331,82 +278,44 @@ int ossl_bn_gen_dsa_nonce_fixed_top(BIGNUM *out, const BIGNUM *range, * length of the private key. */ ERR_raise(ERR_LIB_BN, BN_R_PRIVATE_KEY_TOO_LARGE); - goto end; + goto err; } md = EVP_MD_fetch(libctx, "SHA512", NULL); if (md == NULL) { ERR_raise(ERR_LIB_BN, BN_R_NO_SUITABLE_DIGEST); - goto end; + goto err; } - for (n = 0; n < max_n; n++) { - unsigned char i = 0; - - for (done = 1; done < num_k_bytes;) { - if (RAND_priv_bytes_ex(libctx, random_bytes, sizeof(random_bytes), - 0) <= 0) - goto end; - - if (!EVP_DigestInit_ex(mdctx, md, NULL) - || !EVP_DigestUpdate(mdctx, &i, sizeof(i)) - || !EVP_DigestUpdate(mdctx, private_bytes, - sizeof(private_bytes)) - || !EVP_DigestUpdate(mdctx, message, message_len) - || !EVP_DigestUpdate(mdctx, random_bytes, - sizeof(random_bytes)) - || !EVP_DigestFinal_ex(mdctx, digest, NULL)) - goto end; - - todo = num_k_bytes - done; - if (todo > SHA512_DIGEST_LENGTH) - todo = SHA512_DIGEST_LENGTH; - memcpy(k_bytes + done, digest, todo); - done += todo; - ++i; - } - - if (!BN_bin2bn(k_bytes, num_k_bytes, out)) - goto end; - - /* Clear out the top bits and rejection filter into range */ - BN_set_flags(out, BN_FLG_CONSTTIME); - ossl_bn_mask_bits_fixed_top(out, BN_num_bits(range)); - - if (BN_ucmp(out, range) < 0) { - ret = 1; -#ifdef BN_DEBUG - /* With BN_DEBUG on a fixed top number cannot be returned */ - bn_correct_top(out); -#endif - goto end; - } + for (done = 0; done < num_k_bytes;) { + if (RAND_priv_bytes_ex(libctx, random_bytes, sizeof(random_bytes), 0) <= 0) + goto err; + + if (!EVP_DigestInit_ex(mdctx, md, NULL) + || !EVP_DigestUpdate(mdctx, &done, sizeof(done)) + || !EVP_DigestUpdate(mdctx, private_bytes, + sizeof(private_bytes)) + || !EVP_DigestUpdate(mdctx, message, message_len) + || !EVP_DigestUpdate(mdctx, random_bytes, sizeof(random_bytes)) + || !EVP_DigestFinal_ex(mdctx, digest, NULL)) + goto err; + + todo = num_k_bytes - done; + if (todo > SHA512_DIGEST_LENGTH) + todo = SHA512_DIGEST_LENGTH; + memcpy(k_bytes + done, digest, todo); + done += todo; } - /* Failed to generate anything */ - ERR_raise(ERR_LIB_BN, ERR_R_INTERNAL_ERROR); - end: + if (!BN_bin2bn(k_bytes, num_k_bytes, out)) + goto err; + if (BN_mod(out, out, range, ctx) != 1) + goto err; + ret = 1; + + err: EVP_MD_CTX_free(mdctx); EVP_MD_free(md); - OPENSSL_clear_free(k_bytes, num_k_bytes); - OPENSSL_cleanse(digest, sizeof(digest)); - OPENSSL_cleanse(random_bytes, sizeof(random_bytes)); + OPENSSL_free(k_bytes); OPENSSL_cleanse(private_bytes, sizeof(private_bytes)); return ret; } - -int BN_generate_dsa_nonce(BIGNUM *out, const BIGNUM *range, - const BIGNUM *priv, const unsigned char *message, - size_t message_len, BN_CTX *ctx) -{ - int ret; - - ret = ossl_bn_gen_dsa_nonce_fixed_top(out, range, priv, message, - message_len, ctx); - /* - * This call makes the BN_generate_dsa_nonce non-const-time, thus we - * do not use it internally. But fixed_top BNs currently cannot be returned - * from public API calls. - */ - bn_correct_top(out); - return ret; -} diff --git a/openssl/src/crypto/bn/bn_recp.c b/openssl/src/crypto/bn/bn_recp.c index 83fd175c4..96a6b19ab 100644 --- a/openssl/src/crypto/bn/bn_recp.c +++ b/openssl/src/crypto/bn/bn_recp.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,8 +21,10 @@ BN_RECP_CTX *BN_RECP_CTX_new(void) { BN_RECP_CTX *ret; - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE); return NULL; + } bn_init(&(ret->N)); bn_init(&(ret->Nr)); @@ -42,7 +44,7 @@ void BN_RECP_CTX_free(BN_RECP_CTX *recp) int BN_RECP_CTX_set(BN_RECP_CTX *recp, const BIGNUM *d, BN_CTX *ctx) { - if (BN_is_zero(d) || !BN_copy(&(recp->N), d)) + if (!BN_copy(&(recp->N), d)) return 0; BN_zero(&(recp->Nr)); recp->num_bits = BN_num_bits(d); diff --git a/openssl/src/crypto/bn/bn_rsa_fips186_4.c b/openssl/src/crypto/bn/bn_rsa_fips186_4.c index c967ca962..6ac42a5b4 100644 --- a/openssl/src/crypto/bn/bn_rsa_fips186_4.c +++ b/openssl/src/crypto/bn/bn_rsa_fips186_4.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2018-2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -48,34 +48,6 @@ const BIGNUM ossl_bn_inv_sqrt_2 = { BN_FLG_STATIC_DATA }; -/* - * Refer to FIPS 186-5 Table B.1 for minimum rounds of Miller Rabin - * required for generation of RSA aux primes (p1, p2, q1 and q2). - */ -static int bn_rsa_fips186_5_aux_prime_MR_rounds(int nbits) -{ - if (nbits >= 4096) - return 44; - if (nbits >= 3072) - return 41; - if (nbits >= 2048) - return 38; - return 0; /* Error */ -} - -/* - * Refer to FIPS 186-5 Table B.1 for minimum rounds of Miller Rabin - * required for generation of RSA primes (p and q) - */ -static int bn_rsa_fips186_5_prime_MR_rounds(int nbits) -{ - if (nbits >= 3072) - return 4; - if (nbits >= 2048) - return 5; - return 0; /* Error */ -} - /* * FIPS 186-5 Table A.1. "Min length of auxiliary primes p1, p2, q1, q2". * (FIPS 186-5 has an entry for >= 4096 bits). @@ -125,13 +97,11 @@ static int bn_rsa_fips186_5_aux_prime_max_sum_size_for_prob_primes(int nbits) * Xp1 The passed in starting point to find a probably prime. * p1 The returned probable prime (first odd integer >= Xp1) * ctx A BN_CTX object. - * rounds The number of Miller Rabin rounds * cb An optional BIGNUM callback. * Returns: 1 on success otherwise it returns 0. */ static int bn_rsa_fips186_4_find_aux_prob_prime(const BIGNUM *Xp1, BIGNUM *p1, BN_CTX *ctx, - int rounds, BN_GENCB *cb) { int ret = 0; @@ -143,11 +113,11 @@ static int bn_rsa_fips186_4_find_aux_prob_prime(const BIGNUM *Xp1, BN_set_flags(p1, BN_FLG_CONSTTIME); /* Find the first odd number >= Xp1 that is probably prime */ - for (;;) { + for(;;) { i++; BN_GENCB_call(cb, 0, i); /* MR test with trial division */ - tmp = ossl_bn_check_generated_prime(p1, rounds, ctx, cb); + tmp = BN_check_prime(p1, ctx, cb); if (tmp > 0) break; if (tmp < 0) @@ -190,7 +160,7 @@ int ossl_bn_rsa_fips186_4_gen_prob_primes(BIGNUM *p, BIGNUM *Xpout, { int ret = 0; BIGNUM *p1i = NULL, *p2i = NULL, *Xp1i = NULL, *Xp2i = NULL; - int bitlen, rounds; + int bitlen; if (p == NULL || Xpout == NULL) return 0; @@ -207,7 +177,6 @@ int ossl_bn_rsa_fips186_4_gen_prob_primes(BIGNUM *p, BIGNUM *Xpout, bitlen = bn_rsa_fips186_5_aux_prime_min_size(nlen); if (bitlen == 0) goto err; - rounds = bn_rsa_fips186_5_aux_prime_MR_rounds(nlen); /* (Steps 4.1/5.1): Randomly generate Xp1 if it is not passed in */ if (Xp1 == NULL) { @@ -225,8 +194,8 @@ int ossl_bn_rsa_fips186_4_gen_prob_primes(BIGNUM *p, BIGNUM *Xpout, } /* (Steps 4.2/5.2) - find first auxiliary probable primes */ - if (!bn_rsa_fips186_4_find_aux_prob_prime(Xp1i, p1i, ctx, rounds, cb) - || !bn_rsa_fips186_4_find_aux_prob_prime(Xp2i, p2i, ctx, rounds, cb)) + if (!bn_rsa_fips186_4_find_aux_prob_prime(Xp1i, p1i, ctx, cb) + || !bn_rsa_fips186_4_find_aux_prob_prime(Xp2i, p2i, ctx, cb)) goto err; /* (Table B.1) auxiliary prime Max length check */ if ((BN_num_bits(p1i) + BN_num_bits(p2i)) >= @@ -274,11 +243,11 @@ int ossl_bn_rsa_fips186_4_gen_prob_primes(BIGNUM *p, BIGNUM *Xpout, */ int ossl_bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin, const BIGNUM *r1, const BIGNUM *r2, - int nlen, const BIGNUM *e, - BN_CTX *ctx, BN_GENCB *cb) + int nlen, const BIGNUM *e, BN_CTX *ctx, + BN_GENCB *cb) { int ret = 0; - int i, imax, rounds; + int i, imax; int bits = nlen >> 1; BIGNUM *tmp, *R, *r1r2x2, *y1, *r1x2; BIGNUM *base, *range; @@ -317,20 +286,14 @@ int ossl_bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin, goto err; } - /* - * (Step 1) GCD(2r1, r2) = 1. - * Note: This algorithm was doing a gcd(2r1, r2)=1 test before doing an - * mod_inverse(2r1, r2) which are effectively the same operation. - * (The algorithm assumed that the gcd test would be faster). Since the - * mod_inverse is currently faster than calling the constant time - * BN_gcd(), the call to BN_gcd() has been omitted. The inverse result - * is used further down. - */ if (!(BN_lshift1(r1x2, r1) - && (BN_mod_inverse(tmp, r1x2, r2, ctx) != NULL) + /* (Step 1) GCD(2r1, r2) = 1 */ + && BN_gcd(tmp, r1x2, r2, ctx) + && BN_is_one(tmp) /* (Step 2) R = ((r2^-1 mod 2r1) * r2) - ((2r1^-1 mod r2)*2r1) */ - && (BN_mod_inverse(R, r2, r1x2, ctx) != NULL) + && BN_mod_inverse(R, r2, r1x2, ctx) && BN_mul(R, R, r2, ctx) /* R = (r2^-1 mod 2r1) * r2 */ + && BN_mod_inverse(tmp, r1x2, r2, ctx) && BN_mul(tmp, tmp, r1x2, ctx) /* tmp = (2r1^-1 mod r2)*2r1 */ && BN_sub(R, R, tmp) /* Calculate 2r1r2 */ @@ -340,16 +303,7 @@ int ossl_bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin, if (BN_is_negative(R) && !BN_add(R, R, r1r2x2)) goto err; - /* - * In FIPS 186-4 imax was set to 5 * nlen/2. - * Analysis by Allen Roginsky - * (See https://csrc.nist.gov/CSRC/media/Publications/fips/186/4/final/documents/comments-received-fips186-4-december-2015.pdf - * page 68) indicates this has a 1 in 2 million chance of failure. - * The number has been updated to 20 * nlen/2 as used in - * FIPS186-5 Appendix B.9 Step 9. - */ - rounds = bn_rsa_fips186_5_prime_MR_rounds(nlen); - imax = 20 * bits; /* max = 20/2 * nbits */ + imax = 5 * bits; /* max = 5/2 * nbits */ for (;;) { if (Xin == NULL) { /* @@ -357,7 +311,7 @@ int ossl_bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin, * sqrt(2) * 2^(nlen/2-1) <= Random X <= (2^(nlen/2)) - 1. */ if (!BN_priv_rand_range_ex(X, range, 0, ctx) || !BN_add(X, X, base)) - goto err; + goto end; } /* (Step 4) Y = X + ((R - X) mod 2r1r2) */ if (!BN_mod_sub(Y, R, X, r1r2x2, ctx) || !BN_add(Y, Y, X)) @@ -376,11 +330,11 @@ int ossl_bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin, /* (Step 7) If GCD(Y-1) == 1 & Y is probably prime then return Y */ if (BN_copy(y1, Y) == NULL - || !BN_sub_word(y1, 1)) + || !BN_sub_word(y1, 1) + || !BN_gcd(tmp, y1, e, ctx)) goto err; - - if (BN_are_coprime(y1, e, ctx)) { - int rv = ossl_bn_check_generated_prime(Y, rounds, ctx, cb); + if (BN_is_one(tmp)) { + int rv = BN_check_prime(Y, ctx, cb); if (rv > 0) goto end; @@ -388,11 +342,7 @@ int ossl_bn_rsa_fips186_4_derive_prime(BIGNUM *Y, BIGNUM *X, const BIGNUM *Xin, goto err; } /* (Step 8-10) */ - if (++i >= imax) { - ERR_raise(ERR_LIB_BN, BN_R_NO_PRIME_CANDIDATE); - goto err; - } - if (!BN_add(Y, Y, r1r2x2)) + if (++i >= imax || !BN_add(Y, Y, r1r2x2)) goto err; } } diff --git a/openssl/src/crypto/bn/bn_s390x.c b/openssl/src/crypto/bn/bn_s390x.c deleted file mode 100644 index 5449143f4..000000000 --- a/openssl/src/crypto/bn/bn_s390x.c +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "crypto/bn.h" -#include "crypto/s390x_arch.h" - -#ifdef S390X_MOD_EXP - -# include -# include -# include -# include -# include -# include -# include - -static int s390x_mod_exp_hw(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, - const BIGNUM *m) -{ - struct ica_rsa_modexpo me; - unsigned char *buffer; - size_t size; - int res = 0; - - if (OPENSSL_s390xcex == -1) - return 0; - size = BN_num_bytes(m); - buffer = OPENSSL_zalloc(4 * size); - if (buffer == NULL) - return 0; - me.inputdata = buffer; - me.inputdatalength = size; - me.outputdata = buffer + size; - me.outputdatalength = size; - me.b_key = buffer + 2 * size; - me.n_modulus = buffer + 3 * size; - if (BN_bn2binpad(a, me.inputdata, size) == -1 - || BN_bn2binpad(p, me.b_key, size) == -1 - || BN_bn2binpad(m, me.n_modulus, size) == -1) - goto dealloc; - if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) != -1) { - if (BN_bin2bn(me.outputdata, size, r) != NULL) - res = 1; - } else if (errno == EBADF) { - /*- - * In this cases, someone (e.g. a sandbox) closed the fd. - * Make sure to not further use this hardware acceleration. - */ - OPENSSL_s390xcex = -1; - } - dealloc: - OPENSSL_clear_free(buffer, 4 * size); - return res; -} - -int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, - const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx) -{ - if (s390x_mod_exp_hw(r, a, p, m) == 1) - return 1; - return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx); -} - -int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q, - const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp) -{ - struct ica_rsa_modexpo_crt crt; - unsigned char *buffer, *part; - size_t size, plen, qlen; - int res = 0; - - if (OPENSSL_s390xcex == -1) - return 0; - /*- - * Hardware-accelerated CRT can only deal with p>q. Fall back to - * software in the (hopefully rare) other cases. - */ - if (BN_ucmp(p, q) != 1) - return 0; - plen = BN_num_bytes(p); - qlen = BN_num_bytes(q); - size = (plen > qlen ? plen : qlen); - buffer = OPENSSL_zalloc(9 * size + 24); - if (buffer == NULL) - return 0; - part = buffer; - crt.inputdata = part; - crt.inputdatalength = 2 * size; - part += 2 * size; - crt.outputdata = part; - crt.outputdatalength = 2 * size; - part += 2 * size; - crt.bp_key = part; - part += size + 8; - crt.bq_key = part; - part += size; - crt.np_prime = part; - part += size + 8; - crt.nq_prime = part; - part += size; - crt.u_mult_inv = part; - if (BN_bn2binpad(i, crt.inputdata, crt.inputdatalength) == -1 - || BN_bn2binpad(p, crt.np_prime, size + 8) == -1 - || BN_bn2binpad(q, crt.nq_prime, size) == -1 - || BN_bn2binpad(dmp, crt.bp_key, size + 8) == -1 - || BN_bn2binpad(dmq, crt.bq_key, size) == -1 - || BN_bn2binpad(iqmp, crt.u_mult_inv, size + 8) == -1) - goto dealloc; - if (ioctl(OPENSSL_s390xcex, ICARSACRT, &crt) != -1) { - if (BN_bin2bn(crt.outputdata, crt.outputdatalength, r) != NULL) - res = 1; - } else if (errno == EBADF) { - /*- - * In this cases, someone (e.g. a sandbox) closed the fd. - * Make sure to not further use this hardware acceleration. - */ - OPENSSL_s390xcex = -1; - } - dealloc: - OPENSSL_clear_free(buffer, 9 * size + 24); - return res; -} - -#else -int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, - const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx) -{ - return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx); -} - -int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q, - const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp) -{ - return 0; -} - -#endif diff --git a/openssl/src/crypto/bn/bn_shift.c b/openssl/src/crypto/bn/bn_shift.c index d67331f1f..8fcb04324 100644 --- a/openssl/src/crypto/bn/bn_shift.c +++ b/openssl/src/crypto/bn/bn_shift.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -156,9 +156,6 @@ int BN_rshift(BIGNUM *r, const BIGNUM *a, int n) return 0; } - bn_check_top(r); - bn_check_top(a); - ret = bn_rshift_fixed_top(r, a, n); bn_correct_top(r); @@ -180,6 +177,9 @@ int bn_rshift_fixed_top(BIGNUM *r, const BIGNUM *a, int n) BN_ULONG *t, *f; BN_ULONG l, m, mask; + bn_check_top(r); + bn_check_top(a); + assert(n >= 0); nw = n / BN_BITS2; diff --git a/openssl/src/crypto/bn/bn_sm2.c b/openssl/src/crypto/bn/bn_sm2.c new file mode 100644 index 000000000..8d6d1517f --- /dev/null +++ b/openssl/src/crypto/bn/bn_sm2.c @@ -0,0 +1,475 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "bn_local.h" +#include "internal/cryptlib.h" + +#define BN_SM2_256_TOP (256+BN_BITS2-1)/BN_BITS2 + + +/* Pre-computed tables are "carry-less" values of modulus*(i+1), + * all values are in little-endian format. + */ +#if BN_BITS2 == 64 +/* + * The intermediate value of sm2 modular reduction needs to subtract at most + * 13p, so we need to precompute p, 2p, ... , 13p for modular reduction. + */ +static const BN_ULONG _sm2_p_256[][BN_SM2_256_TOP] = { + {0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFF00000000ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFEFFFFFFFFull}, + {0xFFFFFFFFFFFFFFFEull, 0xFFFFFFFE00000001ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFDFFFFFFFFull}, + {0xFFFFFFFFFFFFFFFDull, 0xFFFFFFFD00000002ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFCFFFFFFFFull}, + {0xFFFFFFFFFFFFFFFCull, 0xFFFFFFFC00000003ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFBFFFFFFFFull}, + {0xFFFFFFFFFFFFFFFBull, 0xFFFFFFFB00000004ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFAFFFFFFFFull}, + {0xFFFFFFFFFFFFFFFAull, 0xFFFFFFFA00000005ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFF9FFFFFFFFull}, + {0xFFFFFFFFFFFFFFF9ull, 0xFFFFFFF900000006ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFF8FFFFFFFFull}, + {0xFFFFFFFFFFFFFFF8ull, 0xFFFFFFF800000007ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFF7FFFFFFFFull}, + {0xFFFFFFFFFFFFFFF7ull, 0xFFFFFFF700000008ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFF6FFFFFFFFull}, + {0xFFFFFFFFFFFFFFF6ull, 0xFFFFFFF600000009ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFF5FFFFFFFFull}, + {0xFFFFFFFFFFFFFFF5ull, 0xFFFFFFF50000000Aull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFF4FFFFFFFFull}, + {0xFFFFFFFFFFFFFFF4ull, 0xFFFFFFF40000000Bull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFF3FFFFFFFFull}, + {0xFFFFFFFFFFFFFFF3ull, 0xFFFFFFF30000000Cull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFF2FFFFFFFFull} +}; + +/* pre-compute the value of p^2 check if the input satisfies input < p^2. */ +static const BN_ULONG _sm2_p_256_sqr[] = { + 0x0000000000000001ULL, 0x00000001FFFFFFFEULL, + 0xFFFFFFFE00000001ULL, 0x0000000200000000ULL, + 0xFFFFFFFDFFFFFFFEULL, 0xFFFFFFFE00000003ULL, + 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFE00000000ULL +}; + +#elif BN_BITS2 == 32 +static const BN_ULONG _sm2_p_256[][BN_SM2_256_TOP] = { + {0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFE}, + {0xFFFFFFFE, 0xFFFFFFFF, 0x00000001, 0xFFFFFFFE, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFD}, + {0xFFFFFFFD, 0xFFFFFFFF, 0x00000002, 0xFFFFFFFD, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFC}, + {0xFFFFFFFC, 0xFFFFFFFF, 0x00000003, 0xFFFFFFFC, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFB}, + {0xFFFFFFFB, 0xFFFFFFFF, 0x00000004, 0xFFFFFFFB, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFA}, + {0xFFFFFFFA, 0xFFFFFFFF, 0x00000005, 0xFFFFFFFA, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF9}, + {0xFFFFFFF9, 0xFFFFFFFF, 0x00000006, 0xFFFFFFF9, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF8}, + {0xFFFFFFF8, 0xFFFFFFFF, 0x00000007, 0xFFFFFFF8, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF7}, + {0xFFFFFFF7, 0xFFFFFFFF, 0x00000008, 0xFFFFFFF7, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF6}, + {0xFFFFFFF6, 0xFFFFFFFF, 0x00000009, 0xFFFFFFF6, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF5}, + {0xFFFFFFF5, 0xFFFFFFFF, 0x0000000A, 0xFFFFFFF5, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF4}, + {0xFFFFFFF4, 0xFFFFFFFF, 0x0000000B, 0xFFFFFFF4, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF3}, + {0xFFFFFFF3, 0xFFFFFFFF, 0x0000000C, 0xFFFFFFF3, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFF2}, +}; + +static const BN_ULONG _sm2_p_256_sqr[] = { + 0x00000001, 0x00000000, 0xFFFFFFFE, 0x00000001, + 0x00000001, 0xFFFFFFFE, 0x00000000, 0x00000002, + 0xFFFFFFFE, 0xFFFFFFFD, 0x00000003, 0xFFFFFFFE, + 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFE +}; +#else +# error "unsupported BN_BITS2" +#endif + +static const BIGNUM ossl_bignum_sm2_p_256 = { + (BN_ULONG *)_sm2_p_256[0], + BN_SM2_256_TOP, + BN_SM2_256_TOP, + 0, + BN_FLG_STATIC_DATA +}; + +const BIGNUM *BN_get0_sm2_prime_256(void) +{ + return &ossl_bignum_sm2_p_256; +} + +/* + * To avoid more recent compilers (specifically clang-14) from treating this + * code as a violation of the strict aliasing conditions and omitting it, this + * cannot be declared as a function. Moreover, the dst parameter cannot be + * cached in a local since this no longer references the union and again falls + * foul of the strict aliasing criteria. Refer to #18225 for the initial + * diagnostics and llvm/llvm-project#55255 for the later discussions with the + * LLVM developers. The problem boils down to if an array in the union is + * converted to a pointer or if it is used directly. + * + * This function was inlined regardless, so there is no space cost to be + * paid for making it a macro. + */ +#define sm2_cp_bn_0(dst, src_in, top, max) \ +{ \ + int ii; \ + const BN_ULONG *src = src_in; \ + \ + for (ii = 0; ii < top; ii++) \ + (dst)[ii] = src[ii]; \ + for (; ii < max; ii++) \ + (dst)[ii] = 0; \ +} + +static void sm2_cp_bn(BN_ULONG *dst, const BN_ULONG *src, int top) +{ + int i; + + for (i = 0; i < top; i++) + dst[i] = src[i]; +} + +#if BN_BITS2 == 64 +# define bn_cp_64(to, n, from, m) (to)[n] = (m>=0)?((from)[m]):0; +# define bn_64_set_0(to, n) (to)[n] = (BN_ULONG)0; +/* + * two following macros are implemented under assumption that they + * are called in a sequence with *ascending* n, i.e. as they are... + */ +# define bn_cp_32_naked(to, n, from, m) (((n)&1)?(to[(n)/2]|=((m)&1)?(from[(m)/2]&BN_MASK2h):(from[(m)/2]<<32))\ + :(to[(n)/2] =((m)&1)?(from[(m)/2]>>32):(from[(m)/2]&BN_MASK2l))) +# define bn_32_set_0(to, n) (((n)&1)?(to[(n)/2]&=BN_MASK2l):(to[(n)/2]=0)); +# define bn_cp_32(to,n,from,m) ((m)>=0)?bn_cp_32_naked(to,n,from,m):bn_32_set_0(to,n) +# if defined(L_ENDIAN) +# if defined(__arch64__) +# define SM2_INT64 long +# else +# define SM2_INT64 long long +# endif +# endif +#else +# define bn_cp_64(to, n, from, m) \ + { \ + bn_cp_32(to, (n)*2, from, (m)*2); \ + bn_cp_32(to, (n)*2+1, from, (m)*2+1); \ + } +# define bn_64_set_0(to, n) \ + { \ + bn_32_set_0(to, (n)*2); \ + bn_32_set_0(to, (n)*2+1); \ + } +# define bn_cp_32(to, n, from, m) (to)[n] = (m>=0)?((from)[m]):0; +# define bn_32_set_0(to, n) (to)[n] = (BN_ULONG)0; +# if defined(_WIN32) && !defined(__GNUC__) +# define SM2_INT64 __int64 +# elif defined(BN_LLONG) +# define SM2_INT64 long long +# endif +#endif /* BN_BITS2 != 64 */ + +typedef BN_ULONG (*bn_addsub_f) (BN_ULONG *, const BN_ULONG *, + const BN_ULONG *, int); + +#define sm2_set_256(to, from, a1, a2, a3, a4, a5, a6, a7, a8) \ + { \ + bn_cp_32(to, 0, from, (a8) - 8) \ + bn_cp_32(to, 1, from, (a7) - 8) \ + bn_cp_32(to, 2, from, (a6) - 8) \ + bn_cp_32(to, 3, from, (a5) - 8) \ + bn_cp_32(to, 4, from, (a4) - 8) \ + bn_cp_32(to, 5, from, (a3) - 8) \ + bn_cp_32(to, 6, from, (a2) - 8) \ + bn_cp_32(to, 7, from, (a1) - 8) \ + } + +/* + * A fast modular reduction algorithm based on generalized Mersenne prime + * for SM2 P256 parameter specialization. You can get more information from + * https://ieeexplore.ieee.org/document/7011249/ . + */ +int BN_sm2_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, + BN_CTX *ctx) +{ + int i, top = a->top; + int carry = 0; + register BN_ULONG *a_d = a->d, *r_d; + union { + BN_ULONG bn[BN_SM2_256_TOP]; + unsigned int ui[BN_SM2_256_TOP * sizeof(BN_ULONG) / + sizeof(unsigned int)]; + } buf; + BN_ULONG c_d[BN_SM2_256_TOP], *res; + PTR_SIZE_INT mask; + union { + bn_addsub_f f; + PTR_SIZE_INT p; + } u; + static const BIGNUM ossl_bignum_sm2_p_256_sqr = { + (BN_ULONG *)_sm2_p_256_sqr, + OSSL_NELEM(_sm2_p_256_sqr), + OSSL_NELEM(_sm2_p_256_sqr), + 0, BN_FLG_STATIC_DATA + }; + + field = &ossl_bignum_sm2_p_256; /* just to make sure */ + + if (BN_is_negative(a) || BN_ucmp(a, &ossl_bignum_sm2_p_256_sqr) >= 0) + return BN_nnmod(r, a, field, ctx); + + i = BN_ucmp(field, a); + if (i == 0) { + BN_zero(r); + return 1; + } else if (i > 0) + return (r == a) ? 1 : (BN_copy(r, a) != NULL); + + if (r != a) { + if (!bn_wexpand(r, BN_SM2_256_TOP)) + return 0; + r_d = r->d; + sm2_cp_bn(r_d, a_d, BN_SM2_256_TOP); + } else + r_d = a_d; + + sm2_cp_bn_0(buf.bn, a_d + BN_SM2_256_TOP, top - BN_SM2_256_TOP, + BN_SM2_256_TOP); + +#if defined(SM2_INT64) + { + SM2_INT64 acc; /* accumulator */ + unsigned int *rp = (unsigned int *)r_d; + const unsigned int *bp = (const unsigned int *)buf.ui; + + acc = rp[0]; + acc += bp[8 - 8]; + acc += bp[9 - 8]; + acc += bp[10 - 8]; + acc += bp[11 - 8]; + acc += bp[12 - 8]; + acc += bp[13 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + acc += bp[13 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + rp[0] = (unsigned int)acc; + acc >>= 32; + + acc += rp[1]; + acc += bp[9 - 8]; + acc += bp[10 - 8]; + acc += bp[11 - 8]; + acc += bp[12 - 8]; + acc += bp[13 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + rp[1] = (unsigned int)acc; + acc >>= 32; + + acc += rp[2]; + acc -= bp[8 - 8]; + acc -= bp[9 - 8]; + acc -= bp[13 - 8]; + acc -= bp[14 - 8]; + rp[2] = (unsigned int)acc; + acc >>= 32; + + acc += rp[3]; + acc += bp[8 - 8]; + acc += bp[11 - 8]; + acc += bp[12 - 8]; + acc += bp[13 - 8]; + acc += bp[13 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + rp[3] = (unsigned int)acc; + acc >>= 32; + + acc += rp[4]; + acc += bp[9 - 8]; + acc += bp[12 - 8]; + acc += bp[13 - 8]; + acc += bp[14 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + rp[4] = (unsigned int)acc; + acc >>= 32; + + acc += rp[5]; + acc += bp[10 - 8]; + acc += bp[13 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + acc += bp[15 - 8]; + rp[5] = (unsigned int)acc; + acc >>= 32; + + acc += rp[6]; + acc += bp[11 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + rp[6] = (unsigned int)acc; + acc >>= 32; + + acc += rp[7]; + acc += bp[8 - 8]; + acc += bp[9 - 8]; + acc += bp[10 - 8]; + acc += bp[11 - 8]; + acc += bp[12 - 8]; + acc += bp[13 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + acc += bp[12 - 8]; + acc += bp[13 - 8]; + acc += bp[14 - 8]; + acc += bp[15 - 8]; + acc += bp[15 - 8]; + rp[7] = (unsigned int)acc; + + carry = (int)(acc >> 32); + } +#else + { + BN_ULONG t_d[BN_SM2_256_TOP]; + + /* + * s3 = (c14, 0, c15, c14, c13, 0, c14, c13) + */ + sm2_set_256(t_d, buf.bn, 14, 0, 15, 14, 13, 0, 14, 13); + /* + * s4 = (c13, 0, 0, 0, 0, 0, c15, c14) + */ + sm2_set_256(c_d, buf.bn, 13, 0, 0, 0, 0, 0, 15, 14); + carry = (int)bn_add_words(t_d, t_d, c_d, BN_SM2_256_TOP); + /* + * s5 = (c12, 0, 0, 0, 0, 0, 0, c15) + */ + sm2_set_256(c_d, buf.bn, 12, 0, 0, 0, 0, 0, 0, 15); + carry += (int)bn_add_words(t_d, t_d, c_d, BN_SM2_256_TOP); + /* + * s10 = (c15, 0, 0, 0, 0, 0, 0, 0) + */ + sm2_set_256(c_d, buf.bn, 15, 0, 0, 0, 0, 0, 0, 0); + carry += (int)bn_add_words(t_d, t_d, c_d, BN_SM2_256_TOP); + + /* left shift */ + { + register BN_ULONG *ap, t, c; + ap = t_d; + c = 0; + for (i = BN_SM2_256_TOP; i != 0; --i) { + t = *ap; + *(ap++) = ((t << 1) | c) & BN_MASK2; + c = (t & BN_TBIT) ? 1 : 0; + } + carry <<= 1; + carry |= c; + } + + /* r_d += 2 * (s3 + s4 + s5 + s10) */ + carry += (int)bn_add_words(r_d, r_d, t_d, BN_SM2_256_TOP); + + /* + * r_d += s2 + s6 + s7 + s8 + s9 + * s2 = (c15, c14, c13, c12, c11, 0, c9, c8) + */ + sm2_set_256(t_d, buf.bn, 15, 14, 13, 12, 11, 0, 9, 8); + carry += (int)bn_add_words(r_d, r_d, t_d, BN_SM2_256_TOP); + /* + * s6 = (c11, c11, c10, c15, c14, 0, c13, c12) + */ + sm2_set_256(t_d, buf.bn, 11, 11, 10, 15, 14, 0, 13, 12); + carry += (int)bn_add_words(r_d, r_d, t_d, BN_SM2_256_TOP); + /* + * s7 = (c10, c15, c14, c13, c12, 0, c11, c10) + */ + sm2_set_256(t_d, buf.bn, 10, 15, 14, 13, 12, 0, 11, 10); + carry += (int)bn_add_words(r_d, r_d, t_d, BN_SM2_256_TOP); + /* + * s8 = (c9, 0, 0, c9, c8, 0, c10, c9) + */ + sm2_set_256(t_d, buf.bn, 9, 0, 0, 9, 8, 0, 10, 9); + carry += (int)bn_add_words(r_d, r_d, t_d, BN_SM2_256_TOP); + /* + * s9 = (c8, 0, 0, 0, c15, 0, c12, c11) + */ + sm2_set_256(t_d, buf.bn, 8, 0, 0, 0, 15, 0, 12, 11); + carry += (int)bn_add_words(r_d, r_d, t_d, BN_SM2_256_TOP); + + /* + * r_d = r_d - s11 - s12 - s13 - s14 + * s11 = (0, 0, 0, 0, 0, c14, 0, 0) + */ + sm2_set_256(t_d, buf.bn, 0, 0, 0, 0, 0, 14, 0, 0); + carry -= (int)bn_sub_words(r_d, r_d, t_d, BN_SM2_256_TOP); + /* + * s12 = (0, 0, 0, 0, 0, c13, 0, 0) + */ + sm2_set_256(t_d, buf.bn, 0, 0, 0, 0, 0, 13, 0, 0); + carry -= (int)bn_sub_words(r_d, r_d, t_d, BN_SM2_256_TOP); + /* + * s13 = (0, 0, 0, 0, 0, c9, 0, 0) + */ + sm2_set_256(t_d, buf.bn, 0, 0, 0, 0, 0, 9, 0, 0); + carry -= (int)bn_sub_words(r_d, r_d, t_d, BN_SM2_256_TOP); + /* + * s14 = (0, 0, 0, 0, 0, c8, 0, 0) + */ + sm2_set_256(t_d, buf.bn, 0, 0, 0, 0, 0, 8, 0, 0); + carry -= (int)bn_sub_words(r_d, r_d, t_d, BN_SM2_256_TOP); + } +#endif + /* see BN_nist_mod_224 for explanation */ + u.f = bn_sub_words; + if (carry > 0) + carry = + (int)bn_sub_words(r_d, r_d, _sm2_p_256[carry - 1], + BN_SM2_256_TOP); + else if (carry < 0) { + carry = + (int)bn_add_words(r_d, r_d, _sm2_p_256[-carry - 1], + BN_SM2_256_TOP); + mask = 0 - (PTR_SIZE_INT) carry; + u.p = ((PTR_SIZE_INT) bn_sub_words & mask) | + ((PTR_SIZE_INT) bn_add_words & ~mask); + } else + carry = 1; + + mask = + 0 - (PTR_SIZE_INT) (*u.f) (c_d, r_d, _sm2_p_256[0], BN_SM2_256_TOP); + mask &= 0 - (PTR_SIZE_INT) carry; + res = c_d; + res = (BN_ULONG *)(((PTR_SIZE_INT) res & ~mask) | + ((PTR_SIZE_INT) r_d & mask)); + sm2_cp_bn(r_d, res, BN_SM2_256_TOP); + r->top = BN_SM2_256_TOP; + bn_correct_top(r); + + return 1; +} diff --git a/openssl/src/crypto/bn/bn_sparc.c b/openssl/src/crypto/bn/bn_sparc.c deleted file mode 100644 index a810c3b1f..000000000 --- a/openssl/src/crypto/bn/bn_sparc.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "internal/cryptlib.h" -#include "crypto/sparc_arch.h" -#include "bn_local.h" /* for definition of bn_mul_mont */ - -int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, - const BN_ULONG *np, const BN_ULONG *n0, int num) -{ - int bn_mul_mont_vis3(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, - const BN_ULONG *np, const BN_ULONG *n0, int num); - int bn_mul_mont_fpu(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, - const BN_ULONG *np, const BN_ULONG *n0, int num); - int bn_mul_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, - const BN_ULONG *np, const BN_ULONG *n0, int num); - - if (!(num & 1) && num >= 6) { - if ((num & 15) == 0 && num <= 64 && - (OPENSSL_sparcv9cap_P[1] & (CFR_MONTMUL | CFR_MONTSQR)) == - (CFR_MONTMUL | CFR_MONTSQR)) { - typedef int (*bn_mul_mont_f) (BN_ULONG *rp, const BN_ULONG *ap, - const BN_ULONG *bp, - const BN_ULONG *np, - const BN_ULONG *n0); - int bn_mul_mont_t4_8(BN_ULONG *rp, const BN_ULONG *ap, - const BN_ULONG *bp, const BN_ULONG *np, - const BN_ULONG *n0); - int bn_mul_mont_t4_16(BN_ULONG *rp, const BN_ULONG *ap, - const BN_ULONG *bp, const BN_ULONG *np, - const BN_ULONG *n0); - int bn_mul_mont_t4_24(BN_ULONG *rp, const BN_ULONG *ap, - const BN_ULONG *bp, const BN_ULONG *np, - const BN_ULONG *n0); - int bn_mul_mont_t4_32(BN_ULONG *rp, const BN_ULONG *ap, - const BN_ULONG *bp, const BN_ULONG *np, - const BN_ULONG *n0); - static const bn_mul_mont_f funcs[4] = { - bn_mul_mont_t4_8, bn_mul_mont_t4_16, - bn_mul_mont_t4_24, bn_mul_mont_t4_32 - }; - bn_mul_mont_f worker = funcs[num / 16 - 1]; - - if ((*worker) (rp, ap, bp, np, n0)) - return 1; - /* retry once and fall back */ - if ((*worker) (rp, ap, bp, np, n0)) - return 1; - return bn_mul_mont_vis3(rp, ap, bp, np, n0, num); - } - if ((OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3)) - return bn_mul_mont_vis3(rp, ap, bp, np, n0, num); - else if (num >= 8 && - /* - * bn_mul_mont_fpu doesn't use FMADD, we just use the - * flag to detect when FPU path is preferable in cases - * when current heuristics is unreliable. [it works - * out because FMADD-capable processors where FPU - * code path is undesirable are also VIS3-capable and - * VIS3 code path takes precedence.] - */ - ( (OPENSSL_sparcv9cap_P[0] & SPARCV9_FMADD) || - (OPENSSL_sparcv9cap_P[0] & - (SPARCV9_PREFER_FPU | SPARCV9_VIS1)) == - (SPARCV9_PREFER_FPU | SPARCV9_VIS1) )) - return bn_mul_mont_fpu(rp, ap, bp, np, n0, num); - } - return bn_mul_mont_int(rp, ap, bp, np, n0, num); -} diff --git a/openssl/src/crypto/bn/bn_sqrt.c b/openssl/src/crypto/bn/bn_sqrt.c index 5c77e7213..e8c69c2a9 100644 --- a/openssl/src/crypto/bn/bn_sqrt.c +++ b/openssl/src/crypto/bn/bn_sqrt.c @@ -25,6 +25,11 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) int e, i, j; int used_ctx = 0; +#ifndef OPENSSL_NO_BN_METHOD + if (ctx && ctx->bn_meth && ctx->bn_meth->mod_sqrt) + return ctx->bn_meth->mod_sqrt(in, a, p, ctx); +#endif + if (!BN_is_odd(p) || BN_abs_is_word(p, 1)) { if (BN_abs_is_word(p, 2)) { if (ret == NULL) diff --git a/openssl/src/crypto/bn/gen/darwin_arm64/armv8-mont.S b/openssl/src/crypto/bn/gen/darwin_arm64/armv8-mont.S index 26e81e610..6fca712c4 100644 --- a/openssl/src/crypto/bn/gen/darwin_arm64/armv8-mont.S +++ b/openssl/src/crypto/bn/gen/darwin_arm64/armv8-mont.S @@ -1,5 +1,5 @@ -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .private_extern _OPENSSL_armv8_rsa_neonized #endif @@ -9,7 +9,6 @@ .align 5 _bn_mul_mont: - AARCH64_SIGN_LINK_REGISTER Lbn_mul_mont: tst x5,#3 b.ne Lmul_mont @@ -220,14 +219,11 @@ Lcond_copy: mov x0,#1 ldp x23,x24,[x29,#48] ldr x29,[sp],#64 - AARCH64_VALIDATE_LINK_REGISTER ret .align 5 bn_mul8x_mont_neon: - // Not adding AARCH64_SIGN_LINK_REGISTER here because bn_mul8x_mont_neon is jumped to - // only from bn_mul_mont which has already signed the return address. stp x29,x30,[sp,#-80]! mov x16,sp stp d8,d9,[sp,#16] @@ -920,7 +916,6 @@ LCopy_2: ldp d10,d11,[sp,#32] ldp d8,d9,[sp,#16] ldr x29,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER ret // bx lr @@ -930,8 +925,7 @@ __bn_sqr8x_mont: cmp x1,x2 b.ne __bn_mul4x_mont Lsqr8x_mont: - // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to - // only from bn_mul_mont which has already signed the return address. +.long 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1682,15 +1676,13 @@ Lsqr8x_done: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 - // x30 is loaded earlier - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .align 5 __bn_mul4x_mont: - // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to - // only from bn_mul_mont (or __bn_sqr8x_mont from bn_mul_mont) which has already signed the return address. +.long 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2124,8 +2116,7 @@ Lmul4x_done: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 - // x30 loaded earlier - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .byte 77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 diff --git a/openssl/src/crypto/bn/gen/darwin_x64/rsaz-2k-avx512.s b/openssl/src/crypto/bn/gen/darwin_x64/rsaz-2k-avx512.s deleted file mode 100644 index 5b62a2115..000000000 --- a/openssl/src/crypto/bn/gen/darwin_x64/rsaz-2k-avx512.s +++ /dev/null @@ -1,883 +0,0 @@ - -.globl _ossl_rsaz_avx512ifma_eligible - -.p2align 5 -_ossl_rsaz_avx512ifma_eligible: - movl _OPENSSL_ia32cap_P+8(%rip),%ecx - xorl %eax,%eax - andl $2149777408,%ecx - cmpl $2149777408,%ecx - cmovel %ecx,%eax - .byte 0xf3,0xc3 - -.text - -.globl _ossl_rsaz_amm52x20_x1_ifma256 - -.p2align 5 -_ossl_rsaz_amm52x20_x1_ifma256: - -.byte 243,15,30,250 - pushq %rbx - - pushq %rbp - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$ossl_rsaz_amm52x20_x1_ifma256_body: - - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - - xorl %r9d,%r9d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - - movl $5,%ebx - -.p2align 5 -L$loop5: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - movq 8(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - movq 16(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - movq 24(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - leaq 32(%r11),%r11 - decl %ebx - jne L$loop5 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm16,%ymm1 - vpsrlq $52,%ymm17,%ymm2 - vpsrlq $52,%ymm18,%ymm25 - vpsrlq $52,%ymm19,%ymm26 - - - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm2,%ymm25,%ymm25 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm16,%ymm16 - vpandq L$mask52x4(%rip),%ymm17,%ymm17 - vpandq L$mask52x4(%rip),%ymm18,%ymm18 - vpandq L$mask52x4(%rip),%ymm19,%ymm19 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm16,%ymm16 - vpaddq %ymm2,%ymm17,%ymm17 - vpaddq %ymm25,%ymm18,%ymm18 - vpaddq %ymm26,%ymm19,%ymm19 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm16,%k2 - vpcmpuq $6,L$mask52x4(%rip),%ymm17,%k3 - vpcmpuq $6,L$mask52x4(%rip),%ymm18,%k4 - vpcmpuq $6,L$mask52x4(%rip),%ymm19,%k5 - kmovb %k1,%r14d - kmovb %k2,%r13d - kmovb %k3,%r12d - kmovb %k4,%r11d - kmovb %k5,%r10d - - - vpcmpuq $0,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm16,%k2 - vpcmpuq $0,L$mask52x4(%rip),%ymm17,%k3 - vpcmpuq $0,L$mask52x4(%rip),%ymm18,%k4 - vpcmpuq $0,L$mask52x4(%rip),%ymm19,%k5 - kmovb %k1,%r9d - kmovb %k2,%r8d - kmovb %k3,%ebx - kmovb %k4,%ecx - kmovb %k5,%edx - - - - shlb $4,%r13b - orb %r13b,%r14b - shlb $4,%r11b - orb %r11b,%r12b - - addb %r14b,%r14b - adcb %r12b,%r12b - adcb %r10b,%r10b - - shlb $4,%r8b - orb %r8b,%r9b - shlb $4,%cl - orb %cl,%bl - - addb %r9b,%r14b - adcb %bl,%r12b - adcb %dl,%r10b - - xorb %r9b,%r14b - xorb %bl,%r12b - xorb %dl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r12d,%k3 - shrb $4,%r12b - kmovb %r12d,%k4 - kmovb %r10d,%k5 - - - vpsubq L$mask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq L$mask52x4(%rip),%ymm16,%ymm16{%k2} - vpsubq L$mask52x4(%rip),%ymm17,%ymm17{%k3} - vpsubq L$mask52x4(%rip),%ymm18,%ymm18{%k4} - vpsubq L$mask52x4(%rip),%ymm19,%ymm19{%k5} - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm16,%ymm16 - vpandq L$mask52x4(%rip),%ymm17,%ymm17 - vpandq L$mask52x4(%rip),%ymm18,%ymm18 - vpandq L$mask52x4(%rip),%ymm19,%ymm19 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm16,32(%rdi) - vmovdqu64 %ymm17,64(%rdi) - vmovdqu64 %ymm18,96(%rdi) - vmovdqu64 %ymm19,128(%rdi) - - vzeroupper - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%r12 - - movq 32(%rsp),%rbp - - movq 40(%rsp),%rbx - - leaq 48(%rsp),%rsp - -L$ossl_rsaz_amm52x20_x1_ifma256_epilogue: - .byte 0xf3,0xc3 - - -.data -.p2align 5 -L$mask52x4: -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.text - -.globl _ossl_rsaz_amm52x20_x2_ifma256 - -.p2align 5 -_ossl_rsaz_amm52x20_x2_ifma256: - -.byte 243,15,30,250 - pushq %rbx - - pushq %rbp - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$ossl_rsaz_amm52x20_x2_ifma256_body: - - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm20 - vmovdqa64 %ymm0,%ymm21 - vmovdqa64 %ymm0,%ymm22 - vmovdqa64 %ymm0,%ymm23 - - xorl %r9d,%r9d - xorl %r15d,%r15d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - movl $20,%ebx - -.p2align 5 -L$loop20: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq (%r8),%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - movq 160(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 160(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - movq %r12,%r10 - adcq $0,%r10 - - movq 8(%r8),%r13 - imulq %r15,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 160(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - adcq %r12,%r10 - - shrq $52,%r15 - salq $12,%r10 - orq %r10,%r15 - - vpmadd52luq 160(%rsi),%ymm1,%ymm4 - vpmadd52luq 192(%rsi),%ymm1,%ymm20 - vpmadd52luq 224(%rsi),%ymm1,%ymm21 - vpmadd52luq 256(%rsi),%ymm1,%ymm22 - vpmadd52luq 288(%rsi),%ymm1,%ymm23 - - vpmadd52luq 160(%rcx),%ymm2,%ymm4 - vpmadd52luq 192(%rcx),%ymm2,%ymm20 - vpmadd52luq 224(%rcx),%ymm2,%ymm21 - vpmadd52luq 256(%rcx),%ymm2,%ymm22 - vpmadd52luq 288(%rcx),%ymm2,%ymm23 - - - valignq $1,%ymm4,%ymm20,%ymm4 - valignq $1,%ymm20,%ymm21,%ymm20 - valignq $1,%ymm21,%ymm22,%ymm21 - valignq $1,%ymm22,%ymm23,%ymm22 - valignq $1,%ymm23,%ymm0,%ymm23 - - vmovq %xmm4,%r13 - addq %r13,%r15 - - vpmadd52huq 160(%rsi),%ymm1,%ymm4 - vpmadd52huq 192(%rsi),%ymm1,%ymm20 - vpmadd52huq 224(%rsi),%ymm1,%ymm21 - vpmadd52huq 256(%rsi),%ymm1,%ymm22 - vpmadd52huq 288(%rsi),%ymm1,%ymm23 - - vpmadd52huq 160(%rcx),%ymm2,%ymm4 - vpmadd52huq 192(%rcx),%ymm2,%ymm20 - vpmadd52huq 224(%rcx),%ymm2,%ymm21 - vpmadd52huq 256(%rcx),%ymm2,%ymm22 - vpmadd52huq 288(%rcx),%ymm2,%ymm23 - leaq 8(%r11),%r11 - decl %ebx - jne L$loop20 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm16,%ymm1 - vpsrlq $52,%ymm17,%ymm2 - vpsrlq $52,%ymm18,%ymm25 - vpsrlq $52,%ymm19,%ymm26 - - - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm2,%ymm25,%ymm25 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm16,%ymm16 - vpandq L$mask52x4(%rip),%ymm17,%ymm17 - vpandq L$mask52x4(%rip),%ymm18,%ymm18 - vpandq L$mask52x4(%rip),%ymm19,%ymm19 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm16,%ymm16 - vpaddq %ymm2,%ymm17,%ymm17 - vpaddq %ymm25,%ymm18,%ymm18 - vpaddq %ymm26,%ymm19,%ymm19 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm16,%k2 - vpcmpuq $6,L$mask52x4(%rip),%ymm17,%k3 - vpcmpuq $6,L$mask52x4(%rip),%ymm18,%k4 - vpcmpuq $6,L$mask52x4(%rip),%ymm19,%k5 - kmovb %k1,%r14d - kmovb %k2,%r13d - kmovb %k3,%r12d - kmovb %k4,%r11d - kmovb %k5,%r10d - - - vpcmpuq $0,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm16,%k2 - vpcmpuq $0,L$mask52x4(%rip),%ymm17,%k3 - vpcmpuq $0,L$mask52x4(%rip),%ymm18,%k4 - vpcmpuq $0,L$mask52x4(%rip),%ymm19,%k5 - kmovb %k1,%r9d - kmovb %k2,%r8d - kmovb %k3,%ebx - kmovb %k4,%ecx - kmovb %k5,%edx - - - - shlb $4,%r13b - orb %r13b,%r14b - shlb $4,%r11b - orb %r11b,%r12b - - addb %r14b,%r14b - adcb %r12b,%r12b - adcb %r10b,%r10b - - shlb $4,%r8b - orb %r8b,%r9b - shlb $4,%cl - orb %cl,%bl - - addb %r9b,%r14b - adcb %bl,%r12b - adcb %dl,%r10b - - xorb %r9b,%r14b - xorb %bl,%r12b - xorb %dl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r12d,%k3 - shrb $4,%r12b - kmovb %r12d,%k4 - kmovb %r10d,%k5 - - - vpsubq L$mask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq L$mask52x4(%rip),%ymm16,%ymm16{%k2} - vpsubq L$mask52x4(%rip),%ymm17,%ymm17{%k3} - vpsubq L$mask52x4(%rip),%ymm18,%ymm18{%k4} - vpsubq L$mask52x4(%rip),%ymm19,%ymm19{%k5} - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm16,%ymm16 - vpandq L$mask52x4(%rip),%ymm17,%ymm17 - vpandq L$mask52x4(%rip),%ymm18,%ymm18 - vpandq L$mask52x4(%rip),%ymm19,%ymm19 - - vpbroadcastq %r15,%ymm0 - vpblendd $3,%ymm0,%ymm4,%ymm4 - - - - vpsrlq $52,%ymm4,%ymm0 - vpsrlq $52,%ymm20,%ymm1 - vpsrlq $52,%ymm21,%ymm2 - vpsrlq $52,%ymm22,%ymm25 - vpsrlq $52,%ymm23,%ymm26 - - - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm2,%ymm25,%ymm25 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm20,%ymm20 - vpandq L$mask52x4(%rip),%ymm21,%ymm21 - vpandq L$mask52x4(%rip),%ymm22,%ymm22 - vpandq L$mask52x4(%rip),%ymm23,%ymm23 - - - vpaddq %ymm0,%ymm4,%ymm4 - vpaddq %ymm1,%ymm20,%ymm20 - vpaddq %ymm2,%ymm21,%ymm21 - vpaddq %ymm25,%ymm22,%ymm22 - vpaddq %ymm26,%ymm23,%ymm23 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm4,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm20,%k2 - vpcmpuq $6,L$mask52x4(%rip),%ymm21,%k3 - vpcmpuq $6,L$mask52x4(%rip),%ymm22,%k4 - vpcmpuq $6,L$mask52x4(%rip),%ymm23,%k5 - kmovb %k1,%r14d - kmovb %k2,%r13d - kmovb %k3,%r12d - kmovb %k4,%r11d - kmovb %k5,%r10d - - - vpcmpuq $0,L$mask52x4(%rip),%ymm4,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm20,%k2 - vpcmpuq $0,L$mask52x4(%rip),%ymm21,%k3 - vpcmpuq $0,L$mask52x4(%rip),%ymm22,%k4 - vpcmpuq $0,L$mask52x4(%rip),%ymm23,%k5 - kmovb %k1,%r9d - kmovb %k2,%r8d - kmovb %k3,%ebx - kmovb %k4,%ecx - kmovb %k5,%edx - - - - shlb $4,%r13b - orb %r13b,%r14b - shlb $4,%r11b - orb %r11b,%r12b - - addb %r14b,%r14b - adcb %r12b,%r12b - adcb %r10b,%r10b - - shlb $4,%r8b - orb %r8b,%r9b - shlb $4,%cl - orb %cl,%bl - - addb %r9b,%r14b - adcb %bl,%r12b - adcb %dl,%r10b - - xorb %r9b,%r14b - xorb %bl,%r12b - xorb %dl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r12d,%k3 - shrb $4,%r12b - kmovb %r12d,%k4 - kmovb %r10d,%k5 - - - vpsubq L$mask52x4(%rip),%ymm4,%ymm4{%k1} - vpsubq L$mask52x4(%rip),%ymm20,%ymm20{%k2} - vpsubq L$mask52x4(%rip),%ymm21,%ymm21{%k3} - vpsubq L$mask52x4(%rip),%ymm22,%ymm22{%k4} - vpsubq L$mask52x4(%rip),%ymm23,%ymm23{%k5} - - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm20,%ymm20 - vpandq L$mask52x4(%rip),%ymm21,%ymm21 - vpandq L$mask52x4(%rip),%ymm22,%ymm22 - vpandq L$mask52x4(%rip),%ymm23,%ymm23 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm16,32(%rdi) - vmovdqu64 %ymm17,64(%rdi) - vmovdqu64 %ymm18,96(%rdi) - vmovdqu64 %ymm19,128(%rdi) - - vmovdqu64 %ymm4,160(%rdi) - vmovdqu64 %ymm20,192(%rdi) - vmovdqu64 %ymm21,224(%rdi) - vmovdqu64 %ymm22,256(%rdi) - vmovdqu64 %ymm23,288(%rdi) - - vzeroupper - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%r12 - - movq 32(%rsp),%rbp - - movq 40(%rsp),%rbx - - leaq 48(%rsp),%rsp - -L$ossl_rsaz_amm52x20_x2_ifma256_epilogue: - .byte 0xf3,0xc3 - - -.text - -.p2align 5 -.globl _ossl_extract_multiplier_2x20_win5 - -_ossl_extract_multiplier_2x20_win5: - -.byte 243,15,30,250 - vmovdqa64 L$ones(%rip),%ymm24 - vpbroadcastq %rdx,%ymm22 - vpbroadcastq %rcx,%ymm23 - leaq 10240(%rsi),%rax - - - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %ymm0,%ymm21 - vmovdqa64 %ymm0,%ymm1 - vmovdqa64 %ymm0,%ymm2 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - -.p2align 5 -L$loop: - vpcmpq $0,%ymm21,%ymm22,%k1 - vpcmpq $0,%ymm21,%ymm23,%k2 - vmovdqu64 0(%rsi),%ymm20 - vpblendmq %ymm20,%ymm0,%ymm0{%k1} - vmovdqu64 32(%rsi),%ymm20 - vpblendmq %ymm20,%ymm1,%ymm1{%k1} - vmovdqu64 64(%rsi),%ymm20 - vpblendmq %ymm20,%ymm2,%ymm2{%k1} - vmovdqu64 96(%rsi),%ymm20 - vpblendmq %ymm20,%ymm3,%ymm3{%k1} - vmovdqu64 128(%rsi),%ymm20 - vpblendmq %ymm20,%ymm4,%ymm4{%k1} - vmovdqu64 160(%rsi),%ymm20 - vpblendmq %ymm20,%ymm5,%ymm5{%k2} - vmovdqu64 192(%rsi),%ymm20 - vpblendmq %ymm20,%ymm16,%ymm16{%k2} - vmovdqu64 224(%rsi),%ymm20 - vpblendmq %ymm20,%ymm17,%ymm17{%k2} - vmovdqu64 256(%rsi),%ymm20 - vpblendmq %ymm20,%ymm18,%ymm18{%k2} - vmovdqu64 288(%rsi),%ymm20 - vpblendmq %ymm20,%ymm19,%ymm19{%k2} - vpaddq %ymm24,%ymm21,%ymm21 - addq $320,%rsi - cmpq %rsi,%rax - jne L$loop - vmovdqu64 %ymm0,0(%rdi) - vmovdqu64 %ymm1,32(%rdi) - vmovdqu64 %ymm2,64(%rdi) - vmovdqu64 %ymm3,96(%rdi) - vmovdqu64 %ymm4,128(%rdi) - vmovdqu64 %ymm5,160(%rdi) - vmovdqu64 %ymm16,192(%rdi) - vmovdqu64 %ymm17,224(%rdi) - vmovdqu64 %ymm18,256(%rdi) - vmovdqu64 %ymm19,288(%rdi) - .byte 0xf3,0xc3 - - -.data -.p2align 5 -L$ones: -.quad 1,1,1,1 -L$zeros: -.quad 0,0,0,0 diff --git a/openssl/src/crypto/bn/gen/darwin_x64/rsaz-3k-avx512.s b/openssl/src/crypto/bn/gen/darwin_x64/rsaz-3k-avx512.s deleted file mode 100644 index befc45f44..000000000 --- a/openssl/src/crypto/bn/gen/darwin_x64/rsaz-3k-avx512.s +++ /dev/null @@ -1,1298 +0,0 @@ -.text - -.globl _ossl_rsaz_amm52x30_x1_ifma256 - -.p2align 5 -_ossl_rsaz_amm52x30_x1_ifma256: - -.byte 243,15,30,250 - pushq %rbx - - pushq %rbp - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm6 - vmovdqa64 %ymm0,%ymm7 - vmovdqa64 %ymm0,%ymm8 - vmovdqa64 %ymm0,%ymm9 - vmovdqa64 %ymm0,%ymm10 - - xorl %r9d,%r9d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - - movl $7,%ebx - -.p2align 5 -L$loop7: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 8(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 16(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 24(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - leaq 32(%r11),%r11 - decl %ebx - jne L$loop7 - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 8(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm4,%ymm1 - vpsrlq $52,%ymm5,%ymm2 - vpsrlq $52,%ymm6,%ymm19 - vpsrlq $52,%ymm7,%ymm20 - vpsrlq $52,%ymm8,%ymm21 - vpsrlq $52,%ymm9,%ymm22 - vpsrlq $52,%ymm10,%ymm23 - - - valignq $3,%ymm22,%ymm23,%ymm23 - valignq $3,%ymm21,%ymm22,%ymm22 - valignq $3,%ymm20,%ymm21,%ymm21 - valignq $3,%ymm19,%ymm20,%ymm20 - valignq $3,%ymm2,%ymm19,%ymm19 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm5,%ymm5 - vpandq L$mask52x4(%rip),%ymm6,%ymm6 - vpandq L$mask52x4(%rip),%ymm7,%ymm7 - vpandq L$mask52x4(%rip),%ymm8,%ymm8 - vpandq L$mask52x4(%rip),%ymm9,%ymm9 - vpandq L$mask52x4(%rip),%ymm10,%ymm10 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm4,%ymm4 - vpaddq %ymm2,%ymm5,%ymm5 - vpaddq %ymm19,%ymm6,%ymm6 - vpaddq %ymm20,%ymm7,%ymm7 - vpaddq %ymm21,%ymm8,%ymm8 - vpaddq %ymm22,%ymm9,%ymm9 - vpaddq %ymm23,%ymm10,%ymm10 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,L$mask52x4(%rip),%ymm5,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,L$mask52x4(%rip),%ymm7,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm8,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,L$mask52x4(%rip),%ymm9,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm10,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - - - vpcmpuq $0,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,L$mask52x4(%rip),%ymm5,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,L$mask52x4(%rip),%ymm7,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm8,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,L$mask52x4(%rip),%ymm9,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm10,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq L$mask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq L$mask52x4(%rip),%ymm4,%ymm4{%k2} - vpsubq L$mask52x4(%rip),%ymm5,%ymm5{%k3} - vpsubq L$mask52x4(%rip),%ymm6,%ymm6{%k4} - vpsubq L$mask52x4(%rip),%ymm7,%ymm7{%k5} - vpsubq L$mask52x4(%rip),%ymm8,%ymm8{%k6} - vpsubq L$mask52x4(%rip),%ymm9,%ymm9{%k7} - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm5,%ymm5 - vpandq L$mask52x4(%rip),%ymm6,%ymm6 - vpandq L$mask52x4(%rip),%ymm7,%ymm7 - vpandq L$mask52x4(%rip),%ymm8,%ymm8 - vpandq L$mask52x4(%rip),%ymm9,%ymm9 - - shrb $4,%r11b - kmovb %r11d,%k1 - - vpsubq L$mask52x4(%rip),%ymm10,%ymm10{%k1} - - vpandq L$mask52x4(%rip),%ymm10,%ymm10 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm4,32(%rdi) - vmovdqu64 %ymm5,64(%rdi) - vmovdqu64 %ymm6,96(%rdi) - vmovdqu64 %ymm7,128(%rdi) - vmovdqu64 %ymm8,160(%rdi) - vmovdqu64 %ymm9,192(%rdi) - vmovdqu64 %ymm10,224(%rdi) - - vzeroupper - leaq (%rsp),%rax - - movq 0(%rax),%r15 - - movq 8(%rax),%r14 - - movq 16(%rax),%r13 - - movq 24(%rax),%r12 - - movq 32(%rax),%rbp - - movq 40(%rax),%rbx - - leaq 48(%rax),%rsp - -L$ossl_rsaz_amm52x30_x1_ifma256_epilogue: - .byte 0xf3,0xc3 - - -.data -.p2align 5 -L$mask52x4: -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.text - -.globl _ossl_rsaz_amm52x30_x2_ifma256 - -.p2align 5 -_ossl_rsaz_amm52x30_x2_ifma256: - -.byte 243,15,30,250 - pushq %rbx - - pushq %rbp - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm6 - vmovdqa64 %ymm0,%ymm7 - vmovdqa64 %ymm0,%ymm8 - vmovdqa64 %ymm0,%ymm9 - vmovdqa64 %ymm0,%ymm10 - - vmovdqa64 %ymm0,%ymm11 - vmovdqa64 %ymm0,%ymm12 - vmovdqa64 %ymm0,%ymm13 - vmovdqa64 %ymm0,%ymm14 - vmovdqa64 %ymm0,%ymm15 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - - - xorl %r9d,%r9d - xorl %r15d,%r15d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - movl $30,%ebx - -.p2align 5 -L$loop30: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq (%r8),%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 256(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 256(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - movq %r12,%r10 - adcq $0,%r10 - - movq 8(%r8),%r13 - imulq %r15,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 256(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - adcq %r12,%r10 - - shrq $52,%r15 - salq $12,%r10 - orq %r10,%r15 - - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - vpmadd52luq 320(%rsi),%ymm1,%ymm13 - vpmadd52luq 352(%rsi),%ymm1,%ymm14 - vpmadd52luq 384(%rsi),%ymm1,%ymm15 - vpmadd52luq 416(%rsi),%ymm1,%ymm16 - vpmadd52luq 448(%rsi),%ymm1,%ymm17 - vpmadd52luq 480(%rsi),%ymm1,%ymm18 - - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - vpmadd52luq 320(%rcx),%ymm2,%ymm13 - vpmadd52luq 352(%rcx),%ymm2,%ymm14 - vpmadd52luq 384(%rcx),%ymm2,%ymm15 - vpmadd52luq 416(%rcx),%ymm2,%ymm16 - vpmadd52luq 448(%rcx),%ymm2,%ymm17 - vpmadd52luq 480(%rcx),%ymm2,%ymm18 - - - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm13,%ymm12 - valignq $1,%ymm13,%ymm14,%ymm13 - valignq $1,%ymm14,%ymm15,%ymm14 - valignq $1,%ymm15,%ymm16,%ymm15 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm0,%ymm18 - - vmovq %xmm11,%r13 - addq %r13,%r15 - - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - vpmadd52huq 320(%rsi),%ymm1,%ymm13 - vpmadd52huq 352(%rsi),%ymm1,%ymm14 - vpmadd52huq 384(%rsi),%ymm1,%ymm15 - vpmadd52huq 416(%rsi),%ymm1,%ymm16 - vpmadd52huq 448(%rsi),%ymm1,%ymm17 - vpmadd52huq 480(%rsi),%ymm1,%ymm18 - - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - vpmadd52huq 320(%rcx),%ymm2,%ymm13 - vpmadd52huq 352(%rcx),%ymm2,%ymm14 - vpmadd52huq 384(%rcx),%ymm2,%ymm15 - vpmadd52huq 416(%rcx),%ymm2,%ymm16 - vpmadd52huq 448(%rcx),%ymm2,%ymm17 - vpmadd52huq 480(%rcx),%ymm2,%ymm18 - leaq 8(%r11),%r11 - decl %ebx - jne L$loop30 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm4,%ymm1 - vpsrlq $52,%ymm5,%ymm2 - vpsrlq $52,%ymm6,%ymm19 - vpsrlq $52,%ymm7,%ymm20 - vpsrlq $52,%ymm8,%ymm21 - vpsrlq $52,%ymm9,%ymm22 - vpsrlq $52,%ymm10,%ymm23 - - - valignq $3,%ymm22,%ymm23,%ymm23 - valignq $3,%ymm21,%ymm22,%ymm22 - valignq $3,%ymm20,%ymm21,%ymm21 - valignq $3,%ymm19,%ymm20,%ymm20 - valignq $3,%ymm2,%ymm19,%ymm19 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm5,%ymm5 - vpandq L$mask52x4(%rip),%ymm6,%ymm6 - vpandq L$mask52x4(%rip),%ymm7,%ymm7 - vpandq L$mask52x4(%rip),%ymm8,%ymm8 - vpandq L$mask52x4(%rip),%ymm9,%ymm9 - vpandq L$mask52x4(%rip),%ymm10,%ymm10 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm4,%ymm4 - vpaddq %ymm2,%ymm5,%ymm5 - vpaddq %ymm19,%ymm6,%ymm6 - vpaddq %ymm20,%ymm7,%ymm7 - vpaddq %ymm21,%ymm8,%ymm8 - vpaddq %ymm22,%ymm9,%ymm9 - vpaddq %ymm23,%ymm10,%ymm10 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,L$mask52x4(%rip),%ymm5,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,L$mask52x4(%rip),%ymm7,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm8,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,L$mask52x4(%rip),%ymm9,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm10,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - - - vpcmpuq $0,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,L$mask52x4(%rip),%ymm5,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,L$mask52x4(%rip),%ymm7,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm8,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,L$mask52x4(%rip),%ymm9,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm10,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq L$mask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq L$mask52x4(%rip),%ymm4,%ymm4{%k2} - vpsubq L$mask52x4(%rip),%ymm5,%ymm5{%k3} - vpsubq L$mask52x4(%rip),%ymm6,%ymm6{%k4} - vpsubq L$mask52x4(%rip),%ymm7,%ymm7{%k5} - vpsubq L$mask52x4(%rip),%ymm8,%ymm8{%k6} - vpsubq L$mask52x4(%rip),%ymm9,%ymm9{%k7} - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm5,%ymm5 - vpandq L$mask52x4(%rip),%ymm6,%ymm6 - vpandq L$mask52x4(%rip),%ymm7,%ymm7 - vpandq L$mask52x4(%rip),%ymm8,%ymm8 - vpandq L$mask52x4(%rip),%ymm9,%ymm9 - - shrb $4,%r11b - kmovb %r11d,%k1 - - vpsubq L$mask52x4(%rip),%ymm10,%ymm10{%k1} - - vpandq L$mask52x4(%rip),%ymm10,%ymm10 - - vpbroadcastq %r15,%ymm0 - vpblendd $3,%ymm0,%ymm11,%ymm11 - - - - vpsrlq $52,%ymm11,%ymm0 - vpsrlq $52,%ymm12,%ymm1 - vpsrlq $52,%ymm13,%ymm2 - vpsrlq $52,%ymm14,%ymm19 - vpsrlq $52,%ymm15,%ymm20 - vpsrlq $52,%ymm16,%ymm21 - vpsrlq $52,%ymm17,%ymm22 - vpsrlq $52,%ymm18,%ymm23 - - - valignq $3,%ymm22,%ymm23,%ymm23 - valignq $3,%ymm21,%ymm22,%ymm22 - valignq $3,%ymm20,%ymm21,%ymm21 - valignq $3,%ymm19,%ymm20,%ymm20 - valignq $3,%ymm2,%ymm19,%ymm19 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm11,%ymm11 - vpandq L$mask52x4(%rip),%ymm12,%ymm12 - vpandq L$mask52x4(%rip),%ymm13,%ymm13 - vpandq L$mask52x4(%rip),%ymm14,%ymm14 - vpandq L$mask52x4(%rip),%ymm15,%ymm15 - vpandq L$mask52x4(%rip),%ymm16,%ymm16 - vpandq L$mask52x4(%rip),%ymm17,%ymm17 - vpandq L$mask52x4(%rip),%ymm18,%ymm18 - - - vpaddq %ymm0,%ymm11,%ymm11 - vpaddq %ymm1,%ymm12,%ymm12 - vpaddq %ymm2,%ymm13,%ymm13 - vpaddq %ymm19,%ymm14,%ymm14 - vpaddq %ymm20,%ymm15,%ymm15 - vpaddq %ymm21,%ymm16,%ymm16 - vpaddq %ymm22,%ymm17,%ymm17 - vpaddq %ymm23,%ymm18,%ymm18 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm11,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm12,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,L$mask52x4(%rip),%ymm13,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm14,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,L$mask52x4(%rip),%ymm15,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm16,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,L$mask52x4(%rip),%ymm17,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm18,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - - - vpcmpuq $0,L$mask52x4(%rip),%ymm11,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm12,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,L$mask52x4(%rip),%ymm13,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm14,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,L$mask52x4(%rip),%ymm15,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm16,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,L$mask52x4(%rip),%ymm17,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm18,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq L$mask52x4(%rip),%ymm11,%ymm11{%k1} - vpsubq L$mask52x4(%rip),%ymm12,%ymm12{%k2} - vpsubq L$mask52x4(%rip),%ymm13,%ymm13{%k3} - vpsubq L$mask52x4(%rip),%ymm14,%ymm14{%k4} - vpsubq L$mask52x4(%rip),%ymm15,%ymm15{%k5} - vpsubq L$mask52x4(%rip),%ymm16,%ymm16{%k6} - vpsubq L$mask52x4(%rip),%ymm17,%ymm17{%k7} - - vpandq L$mask52x4(%rip),%ymm11,%ymm11 - vpandq L$mask52x4(%rip),%ymm12,%ymm12 - vpandq L$mask52x4(%rip),%ymm13,%ymm13 - vpandq L$mask52x4(%rip),%ymm14,%ymm14 - vpandq L$mask52x4(%rip),%ymm15,%ymm15 - vpandq L$mask52x4(%rip),%ymm16,%ymm16 - vpandq L$mask52x4(%rip),%ymm17,%ymm17 - - shrb $4,%r11b - kmovb %r11d,%k1 - - vpsubq L$mask52x4(%rip),%ymm18,%ymm18{%k1} - - vpandq L$mask52x4(%rip),%ymm18,%ymm18 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm4,32(%rdi) - vmovdqu64 %ymm5,64(%rdi) - vmovdqu64 %ymm6,96(%rdi) - vmovdqu64 %ymm7,128(%rdi) - vmovdqu64 %ymm8,160(%rdi) - vmovdqu64 %ymm9,192(%rdi) - vmovdqu64 %ymm10,224(%rdi) - - vmovdqu64 %ymm11,256(%rdi) - vmovdqu64 %ymm12,288(%rdi) - vmovdqu64 %ymm13,320(%rdi) - vmovdqu64 %ymm14,352(%rdi) - vmovdqu64 %ymm15,384(%rdi) - vmovdqu64 %ymm16,416(%rdi) - vmovdqu64 %ymm17,448(%rdi) - vmovdqu64 %ymm18,480(%rdi) - - vzeroupper - leaq (%rsp),%rax - - movq 0(%rax),%r15 - - movq 8(%rax),%r14 - - movq 16(%rax),%r13 - - movq 24(%rax),%r12 - - movq 32(%rax),%rbp - - movq 40(%rax),%rbx - - leaq 48(%rax),%rsp - -L$ossl_rsaz_amm52x30_x2_ifma256_epilogue: - .byte 0xf3,0xc3 - - -.text - -.p2align 5 -.globl _ossl_extract_multiplier_2x30_win5 - -_ossl_extract_multiplier_2x30_win5: - -.byte 243,15,30,250 - vmovdqa64 L$ones(%rip),%ymm30 - vpbroadcastq %rdx,%ymm28 - vpbroadcastq %rcx,%ymm29 - leaq 16384(%rsi),%rax - - - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %ymm0,%ymm27 - vmovdqa64 %ymm0,%ymm1 - vmovdqa64 %ymm0,%ymm2 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - vmovdqa64 %ymm0,%ymm20 - vmovdqa64 %ymm0,%ymm21 - vmovdqa64 %ymm0,%ymm22 - vmovdqa64 %ymm0,%ymm23 - vmovdqa64 %ymm0,%ymm24 - vmovdqa64 %ymm0,%ymm25 - -.p2align 5 -L$loop: - vpcmpq $0,%ymm27,%ymm28,%k1 - vpcmpq $0,%ymm27,%ymm29,%k2 - vmovdqu64 0(%rsi),%ymm26 - vpblendmq %ymm26,%ymm0,%ymm0{%k1} - vmovdqu64 32(%rsi),%ymm26 - vpblendmq %ymm26,%ymm1,%ymm1{%k1} - vmovdqu64 64(%rsi),%ymm26 - vpblendmq %ymm26,%ymm2,%ymm2{%k1} - vmovdqu64 96(%rsi),%ymm26 - vpblendmq %ymm26,%ymm3,%ymm3{%k1} - vmovdqu64 128(%rsi),%ymm26 - vpblendmq %ymm26,%ymm4,%ymm4{%k1} - vmovdqu64 160(%rsi),%ymm26 - vpblendmq %ymm26,%ymm5,%ymm5{%k1} - vmovdqu64 192(%rsi),%ymm26 - vpblendmq %ymm26,%ymm16,%ymm16{%k1} - vmovdqu64 224(%rsi),%ymm26 - vpblendmq %ymm26,%ymm17,%ymm17{%k1} - vmovdqu64 256(%rsi),%ymm26 - vpblendmq %ymm26,%ymm18,%ymm18{%k2} - vmovdqu64 288(%rsi),%ymm26 - vpblendmq %ymm26,%ymm19,%ymm19{%k2} - vmovdqu64 320(%rsi),%ymm26 - vpblendmq %ymm26,%ymm20,%ymm20{%k2} - vmovdqu64 352(%rsi),%ymm26 - vpblendmq %ymm26,%ymm21,%ymm21{%k2} - vmovdqu64 384(%rsi),%ymm26 - vpblendmq %ymm26,%ymm22,%ymm22{%k2} - vmovdqu64 416(%rsi),%ymm26 - vpblendmq %ymm26,%ymm23,%ymm23{%k2} - vmovdqu64 448(%rsi),%ymm26 - vpblendmq %ymm26,%ymm24,%ymm24{%k2} - vmovdqu64 480(%rsi),%ymm26 - vpblendmq %ymm26,%ymm25,%ymm25{%k2} - vpaddq %ymm30,%ymm27,%ymm27 - addq $512,%rsi - cmpq %rsi,%rax - jne L$loop - vmovdqu64 %ymm0,0(%rdi) - vmovdqu64 %ymm1,32(%rdi) - vmovdqu64 %ymm2,64(%rdi) - vmovdqu64 %ymm3,96(%rdi) - vmovdqu64 %ymm4,128(%rdi) - vmovdqu64 %ymm5,160(%rdi) - vmovdqu64 %ymm16,192(%rdi) - vmovdqu64 %ymm17,224(%rdi) - vmovdqu64 %ymm18,256(%rdi) - vmovdqu64 %ymm19,288(%rdi) - vmovdqu64 %ymm20,320(%rdi) - vmovdqu64 %ymm21,352(%rdi) - vmovdqu64 %ymm22,384(%rdi) - vmovdqu64 %ymm23,416(%rdi) - vmovdqu64 %ymm24,448(%rdi) - vmovdqu64 %ymm25,480(%rdi) - - .byte 0xf3,0xc3 - - -.data -.p2align 5 -L$ones: -.quad 1,1,1,1 -L$zeros: -.quad 0,0,0,0 diff --git a/openssl/src/crypto/bn/gen/darwin_x64/rsaz-4k-avx512.s b/openssl/src/crypto/bn/gen/darwin_x64/rsaz-4k-avx512.s deleted file mode 100644 index 771425665..000000000 --- a/openssl/src/crypto/bn/gen/darwin_x64/rsaz-4k-avx512.s +++ /dev/null @@ -1,1341 +0,0 @@ -.text - -.globl _ossl_rsaz_amm52x40_x1_ifma256 - -.p2align 5 -_ossl_rsaz_amm52x40_x1_ifma256: - -.byte 243,15,30,250 - pushq %rbx - - pushq %rbp - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm6 - vmovdqa64 %ymm0,%ymm7 - vmovdqa64 %ymm0,%ymm8 - vmovdqa64 %ymm0,%ymm9 - vmovdqa64 %ymm0,%ymm10 - vmovdqa64 %ymm0,%ymm11 - vmovdqa64 %ymm0,%ymm12 - - xorl %r9d,%r9d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - - movl $10,%ebx - -.p2align 5 -L$loop10: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - movq 8(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - movq 16(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - movq 24(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - leaq 32(%r11),%r11 - decl %ebx - jne L$loop10 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm4,%ymm1 - vpsrlq $52,%ymm5,%ymm2 - vpsrlq $52,%ymm6,%ymm23 - vpsrlq $52,%ymm7,%ymm24 - vpsrlq $52,%ymm8,%ymm25 - vpsrlq $52,%ymm9,%ymm26 - vpsrlq $52,%ymm10,%ymm27 - vpsrlq $52,%ymm11,%ymm28 - vpsrlq $52,%ymm12,%ymm29 - - - valignq $3,%ymm28,%ymm29,%ymm29 - valignq $3,%ymm27,%ymm28,%ymm28 - valignq $3,%ymm26,%ymm27,%ymm27 - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm24,%ymm25,%ymm25 - valignq $3,%ymm23,%ymm24,%ymm24 - valignq $3,%ymm2,%ymm23,%ymm23 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm5,%ymm5 - vpandq L$mask52x4(%rip),%ymm6,%ymm6 - vpandq L$mask52x4(%rip),%ymm7,%ymm7 - vpandq L$mask52x4(%rip),%ymm8,%ymm8 - vpandq L$mask52x4(%rip),%ymm9,%ymm9 - vpandq L$mask52x4(%rip),%ymm10,%ymm10 - vpandq L$mask52x4(%rip),%ymm11,%ymm11 - vpandq L$mask52x4(%rip),%ymm12,%ymm12 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm4,%ymm4 - vpaddq %ymm2,%ymm5,%ymm5 - vpaddq %ymm23,%ymm6,%ymm6 - vpaddq %ymm24,%ymm7,%ymm7 - vpaddq %ymm25,%ymm8,%ymm8 - vpaddq %ymm26,%ymm9,%ymm9 - vpaddq %ymm27,%ymm10,%ymm10 - vpaddq %ymm28,%ymm11,%ymm11 - vpaddq %ymm29,%ymm12,%ymm12 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,L$mask52x4(%rip),%ymm5,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,L$mask52x4(%rip),%ymm7,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm8,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,L$mask52x4(%rip),%ymm9,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm10,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - vpcmpuq $6,L$mask52x4(%rip),%ymm11,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm12,%k2 - kmovb %k1,%r10d - kmovb %k2,%r9d - shlb $4,%r9b - orb %r9b,%r10b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - adcb %r10b,%r10b - - - vpcmpuq $0,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,L$mask52x4(%rip),%ymm5,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,L$mask52x4(%rip),%ymm7,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm8,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,L$mask52x4(%rip),%ymm9,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm10,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - vpcmpuq $0,L$mask52x4(%rip),%ymm11,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm12,%k2 - kmovb %k1,%ebx - kmovb %k2,%eax - shlb $4,%al - orb %al,%bl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - adcb %bl,%r10b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - xorb %bl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq L$mask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq L$mask52x4(%rip),%ymm4,%ymm4{%k2} - vpsubq L$mask52x4(%rip),%ymm5,%ymm5{%k3} - vpsubq L$mask52x4(%rip),%ymm6,%ymm6{%k4} - vpsubq L$mask52x4(%rip),%ymm7,%ymm7{%k5} - vpsubq L$mask52x4(%rip),%ymm8,%ymm8{%k6} - vpsubq L$mask52x4(%rip),%ymm9,%ymm9{%k7} - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm5,%ymm5 - vpandq L$mask52x4(%rip),%ymm6,%ymm6 - vpandq L$mask52x4(%rip),%ymm7,%ymm7 - vpandq L$mask52x4(%rip),%ymm8,%ymm8 - vpandq L$mask52x4(%rip),%ymm9,%ymm9 - - shrb $4,%r11b - kmovb %r11d,%k1 - kmovb %r10d,%k2 - shrb $4,%r10b - kmovb %r10d,%k3 - - vpsubq L$mask52x4(%rip),%ymm10,%ymm10{%k1} - vpsubq L$mask52x4(%rip),%ymm11,%ymm11{%k2} - vpsubq L$mask52x4(%rip),%ymm12,%ymm12{%k3} - - vpandq L$mask52x4(%rip),%ymm10,%ymm10 - vpandq L$mask52x4(%rip),%ymm11,%ymm11 - vpandq L$mask52x4(%rip),%ymm12,%ymm12 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm4,32(%rdi) - vmovdqu64 %ymm5,64(%rdi) - vmovdqu64 %ymm6,96(%rdi) - vmovdqu64 %ymm7,128(%rdi) - vmovdqu64 %ymm8,160(%rdi) - vmovdqu64 %ymm9,192(%rdi) - vmovdqu64 %ymm10,224(%rdi) - vmovdqu64 %ymm11,256(%rdi) - vmovdqu64 %ymm12,288(%rdi) - - vzeroupper - leaq (%rsp),%rax - - movq 0(%rax),%r15 - - movq 8(%rax),%r14 - - movq 16(%rax),%r13 - - movq 24(%rax),%r12 - - movq 32(%rax),%rbp - - movq 40(%rax),%rbx - - leaq 48(%rax),%rsp - -L$ossl_rsaz_amm52x40_x1_ifma256_epilogue: - - .byte 0xf3,0xc3 - - -.data -.p2align 5 -L$mask52x4: -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.text - -.globl _ossl_rsaz_amm52x40_x2_ifma256 - -.p2align 5 -_ossl_rsaz_amm52x40_x2_ifma256: - -.byte 243,15,30,250 - pushq %rbx - - pushq %rbp - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm6 - vmovdqa64 %ymm0,%ymm7 - vmovdqa64 %ymm0,%ymm8 - vmovdqa64 %ymm0,%ymm9 - vmovdqa64 %ymm0,%ymm10 - vmovdqa64 %ymm0,%ymm11 - vmovdqa64 %ymm0,%ymm12 - - vmovdqa64 %ymm0,%ymm13 - vmovdqa64 %ymm0,%ymm14 - vmovdqa64 %ymm0,%ymm15 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - vmovdqa64 %ymm0,%ymm20 - vmovdqa64 %ymm0,%ymm21 - vmovdqa64 %ymm0,%ymm22 - - - xorl %r9d,%r9d - xorl %r15d,%r15d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - movl $40,%ebx - -.p2align 5 -L$loop40: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq (%r8),%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - movq 320(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 320(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - movq %r12,%r10 - adcq $0,%r10 - - movq 8(%r8),%r13 - imulq %r15,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 320(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - adcq %r12,%r10 - - shrq $52,%r15 - salq $12,%r10 - orq %r10,%r15 - - vpmadd52luq 320(%rsi),%ymm1,%ymm13 - vpmadd52luq 352(%rsi),%ymm1,%ymm14 - vpmadd52luq 384(%rsi),%ymm1,%ymm15 - vpmadd52luq 416(%rsi),%ymm1,%ymm16 - vpmadd52luq 448(%rsi),%ymm1,%ymm17 - vpmadd52luq 480(%rsi),%ymm1,%ymm18 - vpmadd52luq 512(%rsi),%ymm1,%ymm19 - vpmadd52luq 544(%rsi),%ymm1,%ymm20 - vpmadd52luq 576(%rsi),%ymm1,%ymm21 - vpmadd52luq 608(%rsi),%ymm1,%ymm22 - - vpmadd52luq 320(%rcx),%ymm2,%ymm13 - vpmadd52luq 352(%rcx),%ymm2,%ymm14 - vpmadd52luq 384(%rcx),%ymm2,%ymm15 - vpmadd52luq 416(%rcx),%ymm2,%ymm16 - vpmadd52luq 448(%rcx),%ymm2,%ymm17 - vpmadd52luq 480(%rcx),%ymm2,%ymm18 - vpmadd52luq 512(%rcx),%ymm2,%ymm19 - vpmadd52luq 544(%rcx),%ymm2,%ymm20 - vpmadd52luq 576(%rcx),%ymm2,%ymm21 - vpmadd52luq 608(%rcx),%ymm2,%ymm22 - - - valignq $1,%ymm13,%ymm14,%ymm13 - valignq $1,%ymm14,%ymm15,%ymm14 - valignq $1,%ymm15,%ymm16,%ymm15 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm20,%ymm19 - valignq $1,%ymm20,%ymm21,%ymm20 - valignq $1,%ymm21,%ymm22,%ymm21 - valignq $1,%ymm22,%ymm0,%ymm22 - - vmovq %xmm13,%r13 - addq %r13,%r15 - - vpmadd52huq 320(%rsi),%ymm1,%ymm13 - vpmadd52huq 352(%rsi),%ymm1,%ymm14 - vpmadd52huq 384(%rsi),%ymm1,%ymm15 - vpmadd52huq 416(%rsi),%ymm1,%ymm16 - vpmadd52huq 448(%rsi),%ymm1,%ymm17 - vpmadd52huq 480(%rsi),%ymm1,%ymm18 - vpmadd52huq 512(%rsi),%ymm1,%ymm19 - vpmadd52huq 544(%rsi),%ymm1,%ymm20 - vpmadd52huq 576(%rsi),%ymm1,%ymm21 - vpmadd52huq 608(%rsi),%ymm1,%ymm22 - - vpmadd52huq 320(%rcx),%ymm2,%ymm13 - vpmadd52huq 352(%rcx),%ymm2,%ymm14 - vpmadd52huq 384(%rcx),%ymm2,%ymm15 - vpmadd52huq 416(%rcx),%ymm2,%ymm16 - vpmadd52huq 448(%rcx),%ymm2,%ymm17 - vpmadd52huq 480(%rcx),%ymm2,%ymm18 - vpmadd52huq 512(%rcx),%ymm2,%ymm19 - vpmadd52huq 544(%rcx),%ymm2,%ymm20 - vpmadd52huq 576(%rcx),%ymm2,%ymm21 - vpmadd52huq 608(%rcx),%ymm2,%ymm22 - leaq 8(%r11),%r11 - decl %ebx - jne L$loop40 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm4,%ymm1 - vpsrlq $52,%ymm5,%ymm2 - vpsrlq $52,%ymm6,%ymm23 - vpsrlq $52,%ymm7,%ymm24 - vpsrlq $52,%ymm8,%ymm25 - vpsrlq $52,%ymm9,%ymm26 - vpsrlq $52,%ymm10,%ymm27 - vpsrlq $52,%ymm11,%ymm28 - vpsrlq $52,%ymm12,%ymm29 - - - valignq $3,%ymm28,%ymm29,%ymm29 - valignq $3,%ymm27,%ymm28,%ymm28 - valignq $3,%ymm26,%ymm27,%ymm27 - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm24,%ymm25,%ymm25 - valignq $3,%ymm23,%ymm24,%ymm24 - valignq $3,%ymm2,%ymm23,%ymm23 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm5,%ymm5 - vpandq L$mask52x4(%rip),%ymm6,%ymm6 - vpandq L$mask52x4(%rip),%ymm7,%ymm7 - vpandq L$mask52x4(%rip),%ymm8,%ymm8 - vpandq L$mask52x4(%rip),%ymm9,%ymm9 - vpandq L$mask52x4(%rip),%ymm10,%ymm10 - vpandq L$mask52x4(%rip),%ymm11,%ymm11 - vpandq L$mask52x4(%rip),%ymm12,%ymm12 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm4,%ymm4 - vpaddq %ymm2,%ymm5,%ymm5 - vpaddq %ymm23,%ymm6,%ymm6 - vpaddq %ymm24,%ymm7,%ymm7 - vpaddq %ymm25,%ymm8,%ymm8 - vpaddq %ymm26,%ymm9,%ymm9 - vpaddq %ymm27,%ymm10,%ymm10 - vpaddq %ymm28,%ymm11,%ymm11 - vpaddq %ymm29,%ymm12,%ymm12 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,L$mask52x4(%rip),%ymm5,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,L$mask52x4(%rip),%ymm7,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm8,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,L$mask52x4(%rip),%ymm9,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm10,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - vpcmpuq $6,L$mask52x4(%rip),%ymm11,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm12,%k2 - kmovb %k1,%r10d - kmovb %k2,%r9d - shlb $4,%r9b - orb %r9b,%r10b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - adcb %r10b,%r10b - - - vpcmpuq $0,L$mask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,L$mask52x4(%rip),%ymm5,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,L$mask52x4(%rip),%ymm7,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm8,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,L$mask52x4(%rip),%ymm9,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm10,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - vpcmpuq $0,L$mask52x4(%rip),%ymm11,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm12,%k2 - kmovb %k1,%ebx - kmovb %k2,%eax - shlb $4,%al - orb %al,%bl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - adcb %bl,%r10b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - xorb %bl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq L$mask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq L$mask52x4(%rip),%ymm4,%ymm4{%k2} - vpsubq L$mask52x4(%rip),%ymm5,%ymm5{%k3} - vpsubq L$mask52x4(%rip),%ymm6,%ymm6{%k4} - vpsubq L$mask52x4(%rip),%ymm7,%ymm7{%k5} - vpsubq L$mask52x4(%rip),%ymm8,%ymm8{%k6} - vpsubq L$mask52x4(%rip),%ymm9,%ymm9{%k7} - - vpandq L$mask52x4(%rip),%ymm3,%ymm3 - vpandq L$mask52x4(%rip),%ymm4,%ymm4 - vpandq L$mask52x4(%rip),%ymm5,%ymm5 - vpandq L$mask52x4(%rip),%ymm6,%ymm6 - vpandq L$mask52x4(%rip),%ymm7,%ymm7 - vpandq L$mask52x4(%rip),%ymm8,%ymm8 - vpandq L$mask52x4(%rip),%ymm9,%ymm9 - - shrb $4,%r11b - kmovb %r11d,%k1 - kmovb %r10d,%k2 - shrb $4,%r10b - kmovb %r10d,%k3 - - vpsubq L$mask52x4(%rip),%ymm10,%ymm10{%k1} - vpsubq L$mask52x4(%rip),%ymm11,%ymm11{%k2} - vpsubq L$mask52x4(%rip),%ymm12,%ymm12{%k3} - - vpandq L$mask52x4(%rip),%ymm10,%ymm10 - vpandq L$mask52x4(%rip),%ymm11,%ymm11 - vpandq L$mask52x4(%rip),%ymm12,%ymm12 - - vpbroadcastq %r15,%ymm0 - vpblendd $3,%ymm0,%ymm13,%ymm13 - - - - vpsrlq $52,%ymm13,%ymm0 - vpsrlq $52,%ymm14,%ymm1 - vpsrlq $52,%ymm15,%ymm2 - vpsrlq $52,%ymm16,%ymm23 - vpsrlq $52,%ymm17,%ymm24 - vpsrlq $52,%ymm18,%ymm25 - vpsrlq $52,%ymm19,%ymm26 - vpsrlq $52,%ymm20,%ymm27 - vpsrlq $52,%ymm21,%ymm28 - vpsrlq $52,%ymm22,%ymm29 - - - valignq $3,%ymm28,%ymm29,%ymm29 - valignq $3,%ymm27,%ymm28,%ymm28 - valignq $3,%ymm26,%ymm27,%ymm27 - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm24,%ymm25,%ymm25 - valignq $3,%ymm23,%ymm24,%ymm24 - valignq $3,%ymm2,%ymm23,%ymm23 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,L$zeros(%rip),%ymm0,%ymm0 - - - vpandq L$mask52x4(%rip),%ymm13,%ymm13 - vpandq L$mask52x4(%rip),%ymm14,%ymm14 - vpandq L$mask52x4(%rip),%ymm15,%ymm15 - vpandq L$mask52x4(%rip),%ymm16,%ymm16 - vpandq L$mask52x4(%rip),%ymm17,%ymm17 - vpandq L$mask52x4(%rip),%ymm18,%ymm18 - vpandq L$mask52x4(%rip),%ymm19,%ymm19 - vpandq L$mask52x4(%rip),%ymm20,%ymm20 - vpandq L$mask52x4(%rip),%ymm21,%ymm21 - vpandq L$mask52x4(%rip),%ymm22,%ymm22 - - - vpaddq %ymm0,%ymm13,%ymm13 - vpaddq %ymm1,%ymm14,%ymm14 - vpaddq %ymm2,%ymm15,%ymm15 - vpaddq %ymm23,%ymm16,%ymm16 - vpaddq %ymm24,%ymm17,%ymm17 - vpaddq %ymm25,%ymm18,%ymm18 - vpaddq %ymm26,%ymm19,%ymm19 - vpaddq %ymm27,%ymm20,%ymm20 - vpaddq %ymm28,%ymm21,%ymm21 - vpaddq %ymm29,%ymm22,%ymm22 - - - - vpcmpuq $6,L$mask52x4(%rip),%ymm13,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm14,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,L$mask52x4(%rip),%ymm15,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm16,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,L$mask52x4(%rip),%ymm17,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm18,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,L$mask52x4(%rip),%ymm19,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm20,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - vpcmpuq $6,L$mask52x4(%rip),%ymm21,%k1 - vpcmpuq $6,L$mask52x4(%rip),%ymm22,%k2 - kmovb %k1,%r10d - kmovb %k2,%r9d - shlb $4,%r9b - orb %r9b,%r10b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - adcb %r10b,%r10b - - - vpcmpuq $0,L$mask52x4(%rip),%ymm13,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm14,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,L$mask52x4(%rip),%ymm15,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm16,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,L$mask52x4(%rip),%ymm17,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm18,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,L$mask52x4(%rip),%ymm19,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm20,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - vpcmpuq $0,L$mask52x4(%rip),%ymm21,%k1 - vpcmpuq $0,L$mask52x4(%rip),%ymm22,%k2 - kmovb %k1,%ebx - kmovb %k2,%eax - shlb $4,%al - orb %al,%bl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - adcb %bl,%r10b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - xorb %bl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq L$mask52x4(%rip),%ymm13,%ymm13{%k1} - vpsubq L$mask52x4(%rip),%ymm14,%ymm14{%k2} - vpsubq L$mask52x4(%rip),%ymm15,%ymm15{%k3} - vpsubq L$mask52x4(%rip),%ymm16,%ymm16{%k4} - vpsubq L$mask52x4(%rip),%ymm17,%ymm17{%k5} - vpsubq L$mask52x4(%rip),%ymm18,%ymm18{%k6} - vpsubq L$mask52x4(%rip),%ymm19,%ymm19{%k7} - - vpandq L$mask52x4(%rip),%ymm13,%ymm13 - vpandq L$mask52x4(%rip),%ymm14,%ymm14 - vpandq L$mask52x4(%rip),%ymm15,%ymm15 - vpandq L$mask52x4(%rip),%ymm16,%ymm16 - vpandq L$mask52x4(%rip),%ymm17,%ymm17 - vpandq L$mask52x4(%rip),%ymm18,%ymm18 - vpandq L$mask52x4(%rip),%ymm19,%ymm19 - - shrb $4,%r11b - kmovb %r11d,%k1 - kmovb %r10d,%k2 - shrb $4,%r10b - kmovb %r10d,%k3 - - vpsubq L$mask52x4(%rip),%ymm20,%ymm20{%k1} - vpsubq L$mask52x4(%rip),%ymm21,%ymm21{%k2} - vpsubq L$mask52x4(%rip),%ymm22,%ymm22{%k3} - - vpandq L$mask52x4(%rip),%ymm20,%ymm20 - vpandq L$mask52x4(%rip),%ymm21,%ymm21 - vpandq L$mask52x4(%rip),%ymm22,%ymm22 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm4,32(%rdi) - vmovdqu64 %ymm5,64(%rdi) - vmovdqu64 %ymm6,96(%rdi) - vmovdqu64 %ymm7,128(%rdi) - vmovdqu64 %ymm8,160(%rdi) - vmovdqu64 %ymm9,192(%rdi) - vmovdqu64 %ymm10,224(%rdi) - vmovdqu64 %ymm11,256(%rdi) - vmovdqu64 %ymm12,288(%rdi) - - vmovdqu64 %ymm13,320(%rdi) - vmovdqu64 %ymm14,352(%rdi) - vmovdqu64 %ymm15,384(%rdi) - vmovdqu64 %ymm16,416(%rdi) - vmovdqu64 %ymm17,448(%rdi) - vmovdqu64 %ymm18,480(%rdi) - vmovdqu64 %ymm19,512(%rdi) - vmovdqu64 %ymm20,544(%rdi) - vmovdqu64 %ymm21,576(%rdi) - vmovdqu64 %ymm22,608(%rdi) - - vzeroupper - leaq (%rsp),%rax - - movq 0(%rax),%r15 - - movq 8(%rax),%r14 - - movq 16(%rax),%r13 - - movq 24(%rax),%r12 - - movq 32(%rax),%rbp - - movq 40(%rax),%rbx - - leaq 48(%rax),%rsp - -L$ossl_rsaz_amm52x40_x2_ifma256_epilogue: - .byte 0xf3,0xc3 - - -.text - -.p2align 5 -.globl _ossl_extract_multiplier_2x40_win5 - -_ossl_extract_multiplier_2x40_win5: - -.byte 243,15,30,250 - vmovdqa64 L$ones(%rip),%ymm24 - vpbroadcastq %rdx,%ymm22 - vpbroadcastq %rcx,%ymm23 - leaq 20480(%rsi),%rax - - - movq %rsi,%r10 - - - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %ymm0,%ymm1 - vmovdqa64 %ymm0,%ymm2 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - vpxorq %ymm21,%ymm21,%ymm21 -.p2align 5 -L$loop_0: - vpcmpq $0,%ymm21,%ymm22,%k1 - vmovdqu64 0(%rsi),%ymm20 - vpblendmq %ymm20,%ymm0,%ymm0{%k1} - vmovdqu64 32(%rsi),%ymm20 - vpblendmq %ymm20,%ymm1,%ymm1{%k1} - vmovdqu64 64(%rsi),%ymm20 - vpblendmq %ymm20,%ymm2,%ymm2{%k1} - vmovdqu64 96(%rsi),%ymm20 - vpblendmq %ymm20,%ymm3,%ymm3{%k1} - vmovdqu64 128(%rsi),%ymm20 - vpblendmq %ymm20,%ymm4,%ymm4{%k1} - vmovdqu64 160(%rsi),%ymm20 - vpblendmq %ymm20,%ymm5,%ymm5{%k1} - vmovdqu64 192(%rsi),%ymm20 - vpblendmq %ymm20,%ymm16,%ymm16{%k1} - vmovdqu64 224(%rsi),%ymm20 - vpblendmq %ymm20,%ymm17,%ymm17{%k1} - vmovdqu64 256(%rsi),%ymm20 - vpblendmq %ymm20,%ymm18,%ymm18{%k1} - vmovdqu64 288(%rsi),%ymm20 - vpblendmq %ymm20,%ymm19,%ymm19{%k1} - vpaddq %ymm24,%ymm21,%ymm21 - addq $640,%rsi - cmpq %rsi,%rax - jne L$loop_0 - vmovdqu64 %ymm0,0(%rdi) - vmovdqu64 %ymm1,32(%rdi) - vmovdqu64 %ymm2,64(%rdi) - vmovdqu64 %ymm3,96(%rdi) - vmovdqu64 %ymm4,128(%rdi) - vmovdqu64 %ymm5,160(%rdi) - vmovdqu64 %ymm16,192(%rdi) - vmovdqu64 %ymm17,224(%rdi) - vmovdqu64 %ymm18,256(%rdi) - vmovdqu64 %ymm19,288(%rdi) - movq %r10,%rsi - vpxorq %ymm21,%ymm21,%ymm21 -.p2align 5 -L$loop_320: - vpcmpq $0,%ymm21,%ymm23,%k1 - vmovdqu64 320(%rsi),%ymm20 - vpblendmq %ymm20,%ymm0,%ymm0{%k1} - vmovdqu64 352(%rsi),%ymm20 - vpblendmq %ymm20,%ymm1,%ymm1{%k1} - vmovdqu64 384(%rsi),%ymm20 - vpblendmq %ymm20,%ymm2,%ymm2{%k1} - vmovdqu64 416(%rsi),%ymm20 - vpblendmq %ymm20,%ymm3,%ymm3{%k1} - vmovdqu64 448(%rsi),%ymm20 - vpblendmq %ymm20,%ymm4,%ymm4{%k1} - vmovdqu64 480(%rsi),%ymm20 - vpblendmq %ymm20,%ymm5,%ymm5{%k1} - vmovdqu64 512(%rsi),%ymm20 - vpblendmq %ymm20,%ymm16,%ymm16{%k1} - vmovdqu64 544(%rsi),%ymm20 - vpblendmq %ymm20,%ymm17,%ymm17{%k1} - vmovdqu64 576(%rsi),%ymm20 - vpblendmq %ymm20,%ymm18,%ymm18{%k1} - vmovdqu64 608(%rsi),%ymm20 - vpblendmq %ymm20,%ymm19,%ymm19{%k1} - vpaddq %ymm24,%ymm21,%ymm21 - addq $640,%rsi - cmpq %rsi,%rax - jne L$loop_320 - vmovdqu64 %ymm0,320(%rdi) - vmovdqu64 %ymm1,352(%rdi) - vmovdqu64 %ymm2,384(%rdi) - vmovdqu64 %ymm3,416(%rdi) - vmovdqu64 %ymm4,448(%rdi) - vmovdqu64 %ymm5,480(%rdi) - vmovdqu64 %ymm16,512(%rdi) - vmovdqu64 %ymm17,544(%rdi) - vmovdqu64 %ymm18,576(%rdi) - vmovdqu64 %ymm19,608(%rdi) - - .byte 0xf3,0xc3 - - -.data -.p2align 5 -L$ones: -.quad 1,1,1,1 -L$zeros: -.quad 0,0,0,0 diff --git a/openssl/src/crypto/bn/gen/darwin_x64/rsaz-avx512.s b/openssl/src/crypto/bn/gen/darwin_x64/rsaz-avx512.s new file mode 100644 index 000000000..fbaa2cd62 --- /dev/null +++ b/openssl/src/crypto/bn/gen/darwin_x64/rsaz-avx512.s @@ -0,0 +1,868 @@ + +.globl _ossl_rsaz_avx512ifma_eligible + +.p2align 5 +_ossl_rsaz_avx512ifma_eligible: + movl _OPENSSL_ia32cap_P+8(%rip),%ecx + xorl %eax,%eax + andl $2149777408,%ecx + cmpl $2149777408,%ecx + cmovel %ecx,%eax + .byte 0xf3,0xc3 + +.text + +.globl _ossl_rsaz_amm52x20_x1_256 + +.p2align 5 +_ossl_rsaz_amm52x20_x1_256: + +.byte 243,15,30,250 + pushq %rbx + + pushq %rbp + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + +L$rsaz_amm52x20_x1_256_body: + + + vpxord %ymm0,%ymm0,%ymm0 + vmovdqa64 %ymm0,%ymm1 + vmovdqa64 %ymm0,%ymm16 + vmovdqa64 %ymm0,%ymm17 + vmovdqa64 %ymm0,%ymm18 + vmovdqa64 %ymm0,%ymm19 + + xorl %r9d,%r9d + + movq %rdx,%r11 + movq $0xfffffffffffff,%rax + + + movl $5,%ebx + +.p2align 5 +L$loop5: + movq 0(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq %r8,%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + movq 8(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq %r8,%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + movq 16(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq %r8,%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + movq 24(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq %r8,%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + leaq 32(%r11),%r11 + decl %ebx + jne L$loop5 + + vmovdqa64 L$mask52x4(%rip),%ymm4 + + vpbroadcastq %r9,%ymm3 + vpblendd $3,%ymm3,%ymm1,%ymm1 + + + + vpsrlq $52,%ymm1,%ymm24 + vpsrlq $52,%ymm16,%ymm25 + vpsrlq $52,%ymm17,%ymm26 + vpsrlq $52,%ymm18,%ymm27 + vpsrlq $52,%ymm19,%ymm28 + + + valignq $3,%ymm27,%ymm28,%ymm28 + valignq $3,%ymm26,%ymm27,%ymm27 + valignq $3,%ymm25,%ymm26,%ymm26 + valignq $3,%ymm24,%ymm25,%ymm25 + valignq $3,%ymm0,%ymm24,%ymm24 + + + vpandq %ymm4,%ymm1,%ymm1 + vpandq %ymm4,%ymm16,%ymm16 + vpandq %ymm4,%ymm17,%ymm17 + vpandq %ymm4,%ymm18,%ymm18 + vpandq %ymm4,%ymm19,%ymm19 + + + vpaddq %ymm24,%ymm1,%ymm1 + vpaddq %ymm25,%ymm16,%ymm16 + vpaddq %ymm26,%ymm17,%ymm17 + vpaddq %ymm27,%ymm18,%ymm18 + vpaddq %ymm28,%ymm19,%ymm19 + + + + vpcmpuq $1,%ymm1,%ymm4,%k1 + vpcmpuq $1,%ymm16,%ymm4,%k2 + vpcmpuq $1,%ymm17,%ymm4,%k3 + vpcmpuq $1,%ymm18,%ymm4,%k4 + vpcmpuq $1,%ymm19,%ymm4,%k5 + kmovb %k1,%r14d + kmovb %k2,%r13d + kmovb %k3,%r12d + kmovb %k4,%r11d + kmovb %k5,%r10d + + + vpcmpuq $0,%ymm1,%ymm4,%k1 + vpcmpuq $0,%ymm16,%ymm4,%k2 + vpcmpuq $0,%ymm17,%ymm4,%k3 + vpcmpuq $0,%ymm18,%ymm4,%k4 + vpcmpuq $0,%ymm19,%ymm4,%k5 + kmovb %k1,%r9d + kmovb %k2,%r8d + kmovb %k3,%ebx + kmovb %k4,%ecx + kmovb %k5,%edx + + + + shlb $4,%r13b + orb %r13b,%r14b + shlb $4,%r11b + orb %r11b,%r12b + + addb %r14b,%r14b + adcb %r12b,%r12b + adcb %r10b,%r10b + + shlb $4,%r8b + orb %r8b,%r9b + shlb $4,%cl + orb %cl,%bl + + addb %r9b,%r14b + adcb %bl,%r12b + adcb %dl,%r10b + + xorb %r9b,%r14b + xorb %bl,%r12b + xorb %dl,%r10b + + kmovb %r14d,%k1 + shrb $4,%r14b + kmovb %r14d,%k2 + kmovb %r12d,%k3 + shrb $4,%r12b + kmovb %r12d,%k4 + kmovb %r10d,%k5 + + + vpsubq %ymm4,%ymm1,%ymm1{%k1} + vpsubq %ymm4,%ymm16,%ymm16{%k2} + vpsubq %ymm4,%ymm17,%ymm17{%k3} + vpsubq %ymm4,%ymm18,%ymm18{%k4} + vpsubq %ymm4,%ymm19,%ymm19{%k5} + + vpandq %ymm4,%ymm1,%ymm1 + vpandq %ymm4,%ymm16,%ymm16 + vpandq %ymm4,%ymm17,%ymm17 + vpandq %ymm4,%ymm18,%ymm18 + vpandq %ymm4,%ymm19,%ymm19 + + vmovdqu64 %ymm1,(%rdi) + vmovdqu64 %ymm16,32(%rdi) + vmovdqu64 %ymm17,64(%rdi) + vmovdqu64 %ymm18,96(%rdi) + vmovdqu64 %ymm19,128(%rdi) + + vzeroupper + movq 0(%rsp),%r15 + + movq 8(%rsp),%r14 + + movq 16(%rsp),%r13 + + movq 24(%rsp),%r12 + + movq 32(%rsp),%rbp + + movq 40(%rsp),%rbx + + leaq 48(%rsp),%rsp + +L$rsaz_amm52x20_x1_256_epilogue: + .byte 0xf3,0xc3 + + +.data +.p2align 5 +L$mask52x4: +.quad 0xfffffffffffff +.quad 0xfffffffffffff +.quad 0xfffffffffffff +.quad 0xfffffffffffff +.text + +.globl _ossl_rsaz_amm52x20_x2_256 + +.p2align 5 +_ossl_rsaz_amm52x20_x2_256: + +.byte 243,15,30,250 + pushq %rbx + + pushq %rbp + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + +L$rsaz_amm52x20_x2_256_body: + + + vpxord %ymm0,%ymm0,%ymm0 + vmovdqa64 %ymm0,%ymm1 + vmovdqa64 %ymm0,%ymm16 + vmovdqa64 %ymm0,%ymm17 + vmovdqa64 %ymm0,%ymm18 + vmovdqa64 %ymm0,%ymm19 + vmovdqa64 %ymm0,%ymm2 + vmovdqa64 %ymm0,%ymm20 + vmovdqa64 %ymm0,%ymm21 + vmovdqa64 %ymm0,%ymm22 + vmovdqa64 %ymm0,%ymm23 + + xorl %r9d,%r9d + xorl %r15d,%r15d + + movq %rdx,%r11 + movq $0xfffffffffffff,%rax + + movl $20,%ebx + +.p2align 5 +L$loop20: + movq 0(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq (%r8),%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + movq 160(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 160(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r15 + movq %r12,%r10 + adcq $0,%r10 + + movq 8(%r8),%r13 + imulq %r15,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 160(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r15 + adcq %r12,%r10 + + shrq $52,%r15 + salq $12,%r10 + orq %r10,%r15 + + vpmadd52luq 160(%rsi),%ymm3,%ymm2 + vpmadd52luq 192(%rsi),%ymm3,%ymm20 + vpmadd52luq 224(%rsi),%ymm3,%ymm21 + vpmadd52luq 256(%rsi),%ymm3,%ymm22 + vpmadd52luq 288(%rsi),%ymm3,%ymm23 + + vpmadd52luq 160(%rcx),%ymm4,%ymm2 + vpmadd52luq 192(%rcx),%ymm4,%ymm20 + vpmadd52luq 224(%rcx),%ymm4,%ymm21 + vpmadd52luq 256(%rcx),%ymm4,%ymm22 + vpmadd52luq 288(%rcx),%ymm4,%ymm23 + + + valignq $1,%ymm2,%ymm20,%ymm2 + valignq $1,%ymm20,%ymm21,%ymm20 + valignq $1,%ymm21,%ymm22,%ymm21 + valignq $1,%ymm22,%ymm23,%ymm22 + valignq $1,%ymm23,%ymm0,%ymm23 + + vmovq %xmm2,%r13 + addq %r13,%r15 + + vpmadd52huq 160(%rsi),%ymm3,%ymm2 + vpmadd52huq 192(%rsi),%ymm3,%ymm20 + vpmadd52huq 224(%rsi),%ymm3,%ymm21 + vpmadd52huq 256(%rsi),%ymm3,%ymm22 + vpmadd52huq 288(%rsi),%ymm3,%ymm23 + + vpmadd52huq 160(%rcx),%ymm4,%ymm2 + vpmadd52huq 192(%rcx),%ymm4,%ymm20 + vpmadd52huq 224(%rcx),%ymm4,%ymm21 + vpmadd52huq 256(%rcx),%ymm4,%ymm22 + vpmadd52huq 288(%rcx),%ymm4,%ymm23 + leaq 8(%r11),%r11 + decl %ebx + jne L$loop20 + + vmovdqa64 L$mask52x4(%rip),%ymm4 + + vpbroadcastq %r9,%ymm3 + vpblendd $3,%ymm3,%ymm1,%ymm1 + + + + vpsrlq $52,%ymm1,%ymm24 + vpsrlq $52,%ymm16,%ymm25 + vpsrlq $52,%ymm17,%ymm26 + vpsrlq $52,%ymm18,%ymm27 + vpsrlq $52,%ymm19,%ymm28 + + + valignq $3,%ymm27,%ymm28,%ymm28 + valignq $3,%ymm26,%ymm27,%ymm27 + valignq $3,%ymm25,%ymm26,%ymm26 + valignq $3,%ymm24,%ymm25,%ymm25 + valignq $3,%ymm0,%ymm24,%ymm24 + + + vpandq %ymm4,%ymm1,%ymm1 + vpandq %ymm4,%ymm16,%ymm16 + vpandq %ymm4,%ymm17,%ymm17 + vpandq %ymm4,%ymm18,%ymm18 + vpandq %ymm4,%ymm19,%ymm19 + + + vpaddq %ymm24,%ymm1,%ymm1 + vpaddq %ymm25,%ymm16,%ymm16 + vpaddq %ymm26,%ymm17,%ymm17 + vpaddq %ymm27,%ymm18,%ymm18 + vpaddq %ymm28,%ymm19,%ymm19 + + + + vpcmpuq $1,%ymm1,%ymm4,%k1 + vpcmpuq $1,%ymm16,%ymm4,%k2 + vpcmpuq $1,%ymm17,%ymm4,%k3 + vpcmpuq $1,%ymm18,%ymm4,%k4 + vpcmpuq $1,%ymm19,%ymm4,%k5 + kmovb %k1,%r14d + kmovb %k2,%r13d + kmovb %k3,%r12d + kmovb %k4,%r11d + kmovb %k5,%r10d + + + vpcmpuq $0,%ymm1,%ymm4,%k1 + vpcmpuq $0,%ymm16,%ymm4,%k2 + vpcmpuq $0,%ymm17,%ymm4,%k3 + vpcmpuq $0,%ymm18,%ymm4,%k4 + vpcmpuq $0,%ymm19,%ymm4,%k5 + kmovb %k1,%r9d + kmovb %k2,%r8d + kmovb %k3,%ebx + kmovb %k4,%ecx + kmovb %k5,%edx + + + + shlb $4,%r13b + orb %r13b,%r14b + shlb $4,%r11b + orb %r11b,%r12b + + addb %r14b,%r14b + adcb %r12b,%r12b + adcb %r10b,%r10b + + shlb $4,%r8b + orb %r8b,%r9b + shlb $4,%cl + orb %cl,%bl + + addb %r9b,%r14b + adcb %bl,%r12b + adcb %dl,%r10b + + xorb %r9b,%r14b + xorb %bl,%r12b + xorb %dl,%r10b + + kmovb %r14d,%k1 + shrb $4,%r14b + kmovb %r14d,%k2 + kmovb %r12d,%k3 + shrb $4,%r12b + kmovb %r12d,%k4 + kmovb %r10d,%k5 + + + vpsubq %ymm4,%ymm1,%ymm1{%k1} + vpsubq %ymm4,%ymm16,%ymm16{%k2} + vpsubq %ymm4,%ymm17,%ymm17{%k3} + vpsubq %ymm4,%ymm18,%ymm18{%k4} + vpsubq %ymm4,%ymm19,%ymm19{%k5} + + vpandq %ymm4,%ymm1,%ymm1 + vpandq %ymm4,%ymm16,%ymm16 + vpandq %ymm4,%ymm17,%ymm17 + vpandq %ymm4,%ymm18,%ymm18 + vpandq %ymm4,%ymm19,%ymm19 + + vpbroadcastq %r15,%ymm3 + vpblendd $3,%ymm3,%ymm2,%ymm2 + + + + vpsrlq $52,%ymm2,%ymm24 + vpsrlq $52,%ymm20,%ymm25 + vpsrlq $52,%ymm21,%ymm26 + vpsrlq $52,%ymm22,%ymm27 + vpsrlq $52,%ymm23,%ymm28 + + + valignq $3,%ymm27,%ymm28,%ymm28 + valignq $3,%ymm26,%ymm27,%ymm27 + valignq $3,%ymm25,%ymm26,%ymm26 + valignq $3,%ymm24,%ymm25,%ymm25 + valignq $3,%ymm0,%ymm24,%ymm24 + + + vpandq %ymm4,%ymm2,%ymm2 + vpandq %ymm4,%ymm20,%ymm20 + vpandq %ymm4,%ymm21,%ymm21 + vpandq %ymm4,%ymm22,%ymm22 + vpandq %ymm4,%ymm23,%ymm23 + + + vpaddq %ymm24,%ymm2,%ymm2 + vpaddq %ymm25,%ymm20,%ymm20 + vpaddq %ymm26,%ymm21,%ymm21 + vpaddq %ymm27,%ymm22,%ymm22 + vpaddq %ymm28,%ymm23,%ymm23 + + + + vpcmpuq $1,%ymm2,%ymm4,%k1 + vpcmpuq $1,%ymm20,%ymm4,%k2 + vpcmpuq $1,%ymm21,%ymm4,%k3 + vpcmpuq $1,%ymm22,%ymm4,%k4 + vpcmpuq $1,%ymm23,%ymm4,%k5 + kmovb %k1,%r14d + kmovb %k2,%r13d + kmovb %k3,%r12d + kmovb %k4,%r11d + kmovb %k5,%r10d + + + vpcmpuq $0,%ymm2,%ymm4,%k1 + vpcmpuq $0,%ymm20,%ymm4,%k2 + vpcmpuq $0,%ymm21,%ymm4,%k3 + vpcmpuq $0,%ymm22,%ymm4,%k4 + vpcmpuq $0,%ymm23,%ymm4,%k5 + kmovb %k1,%r9d + kmovb %k2,%r8d + kmovb %k3,%ebx + kmovb %k4,%ecx + kmovb %k5,%edx + + + + shlb $4,%r13b + orb %r13b,%r14b + shlb $4,%r11b + orb %r11b,%r12b + + addb %r14b,%r14b + adcb %r12b,%r12b + adcb %r10b,%r10b + + shlb $4,%r8b + orb %r8b,%r9b + shlb $4,%cl + orb %cl,%bl + + addb %r9b,%r14b + adcb %bl,%r12b + adcb %dl,%r10b + + xorb %r9b,%r14b + xorb %bl,%r12b + xorb %dl,%r10b + + kmovb %r14d,%k1 + shrb $4,%r14b + kmovb %r14d,%k2 + kmovb %r12d,%k3 + shrb $4,%r12b + kmovb %r12d,%k4 + kmovb %r10d,%k5 + + + vpsubq %ymm4,%ymm2,%ymm2{%k1} + vpsubq %ymm4,%ymm20,%ymm20{%k2} + vpsubq %ymm4,%ymm21,%ymm21{%k3} + vpsubq %ymm4,%ymm22,%ymm22{%k4} + vpsubq %ymm4,%ymm23,%ymm23{%k5} + + vpandq %ymm4,%ymm2,%ymm2 + vpandq %ymm4,%ymm20,%ymm20 + vpandq %ymm4,%ymm21,%ymm21 + vpandq %ymm4,%ymm22,%ymm22 + vpandq %ymm4,%ymm23,%ymm23 + + vmovdqu64 %ymm1,(%rdi) + vmovdqu64 %ymm16,32(%rdi) + vmovdqu64 %ymm17,64(%rdi) + vmovdqu64 %ymm18,96(%rdi) + vmovdqu64 %ymm19,128(%rdi) + + vmovdqu64 %ymm2,160(%rdi) + vmovdqu64 %ymm20,192(%rdi) + vmovdqu64 %ymm21,224(%rdi) + vmovdqu64 %ymm22,256(%rdi) + vmovdqu64 %ymm23,288(%rdi) + + vzeroupper + movq 0(%rsp),%r15 + + movq 8(%rsp),%r14 + + movq 16(%rsp),%r13 + + movq 24(%rsp),%r12 + + movq 32(%rsp),%rbp + + movq 40(%rsp),%rbx + + leaq 48(%rsp),%rsp + +L$rsaz_amm52x20_x2_256_epilogue: + .byte 0xf3,0xc3 + + +.text + +.p2align 5 +.globl _ossl_extract_multiplier_2x20_win5 + +_ossl_extract_multiplier_2x20_win5: + +.byte 243,15,30,250 + leaq (%rcx,%rcx,4),%rax + salq $5,%rax + addq %rax,%rsi + + vmovdqa64 L$ones(%rip),%ymm23 + vpbroadcastq %rdx,%ymm22 + leaq 10240(%rsi),%rax + + vpxor %xmm4,%xmm4,%xmm4 + vmovdqa64 %ymm4,%ymm3 + vmovdqa64 %ymm4,%ymm2 + vmovdqa64 %ymm4,%ymm1 + vmovdqa64 %ymm4,%ymm0 + vmovdqa64 %ymm4,%ymm21 + +.p2align 5 +L$loop: + vpcmpq $0,%ymm21,%ymm22,%k1 + addq $320,%rsi + vpaddq %ymm23,%ymm21,%ymm21 + vmovdqu64 -320(%rsi),%ymm16 + vmovdqu64 -288(%rsi),%ymm17 + vmovdqu64 -256(%rsi),%ymm18 + vmovdqu64 -224(%rsi),%ymm19 + vmovdqu64 -192(%rsi),%ymm20 + vpblendmq %ymm16,%ymm0,%ymm0{%k1} + vpblendmq %ymm17,%ymm1,%ymm1{%k1} + vpblendmq %ymm18,%ymm2,%ymm2{%k1} + vpblendmq %ymm19,%ymm3,%ymm3{%k1} + vpblendmq %ymm20,%ymm4,%ymm4{%k1} + cmpq %rsi,%rax + jne L$loop + + vmovdqu64 %ymm0,(%rdi) + vmovdqu64 %ymm1,32(%rdi) + vmovdqu64 %ymm2,64(%rdi) + vmovdqu64 %ymm3,96(%rdi) + vmovdqu64 %ymm4,128(%rdi) + + .byte 0xf3,0xc3 + + +.data +.p2align 5 +L$ones: +.quad 1,1,1,1 diff --git a/openssl/src/crypto/bn/gen/darwin_x64/x86_64-mont5.s b/openssl/src/crypto/bn/gen/darwin_x64/x86_64-mont5.s index 8520cd92f..6712682a6 100644 --- a/openssl/src/crypto/bn/gen/darwin_x64/x86_64-mont5.s +++ b/openssl/src/crypto/bn/gen/darwin_x64/x86_64-mont5.s @@ -2048,6 +2048,185 @@ L$sqr4x_sub_entry: .byte 0xf3,0xc3 +.globl _bn_from_montgomery + +.p2align 5 +_bn_from_montgomery: + + testl $7,%r9d + jz bn_from_mont8x + xorl %eax,%eax + .byte 0xf3,0xc3 + + + + +.p2align 5 +bn_from_mont8x: + +.byte 0x67 + movq %rsp,%rax + + pushq %rbx + + pushq %rbp + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + +L$from_prologue: + + shll $3,%r9d + leaq (%r9,%r9,2),%r10 + negq %r9 + movq (%r8),%r8 + + + + + + + + + leaq -320(%rsp,%r9,2),%r11 + movq %rsp,%rbp + subq %rdi,%r11 + andq $4095,%r11 + cmpq %r11,%r10 + jb L$from_sp_alt + subq %r11,%rbp + leaq -320(%rbp,%r9,2),%rbp + jmp L$from_sp_done + +.p2align 5 +L$from_sp_alt: + leaq 4096-320(,%r9,2),%r10 + leaq -320(%rbp,%r9,2),%rbp + subq %r10,%r11 + movq $0,%r10 + cmovcq %r10,%r11 + subq %r11,%rbp +L$from_sp_done: + andq $-64,%rbp + movq %rsp,%r11 + subq %rbp,%r11 + andq $-4096,%r11 + leaq (%r11,%rbp,1),%rsp + movq (%rsp),%r10 + cmpq %rbp,%rsp + ja L$from_page_walk + jmp L$from_page_walk_done + +L$from_page_walk: + leaq -4096(%rsp),%rsp + movq (%rsp),%r10 + cmpq %rbp,%rsp + ja L$from_page_walk +L$from_page_walk_done: + + movq %r9,%r10 + negq %r9 + + + + + + + + + + + movq %r8,32(%rsp) + movq %rax,40(%rsp) + +L$from_body: + movq %r9,%r11 + leaq 48(%rsp),%rax + pxor %xmm0,%xmm0 + jmp L$mul_by_1 + +.p2align 5 +L$mul_by_1: + movdqu (%rsi),%xmm1 + movdqu 16(%rsi),%xmm2 + movdqu 32(%rsi),%xmm3 + movdqa %xmm0,(%rax,%r9,1) + movdqu 48(%rsi),%xmm4 + movdqa %xmm0,16(%rax,%r9,1) +.byte 0x48,0x8d,0xb6,0x40,0x00,0x00,0x00 + movdqa %xmm1,(%rax) + movdqa %xmm0,32(%rax,%r9,1) + movdqa %xmm2,16(%rax) + movdqa %xmm0,48(%rax,%r9,1) + movdqa %xmm3,32(%rax) + movdqa %xmm4,48(%rax) + leaq 64(%rax),%rax + subq $64,%r11 + jnz L$mul_by_1 + +.byte 102,72,15,110,207 +.byte 102,72,15,110,209 +.byte 0x67 + movq %rcx,%rbp +.byte 102,73,15,110,218 + movl _OPENSSL_ia32cap_P+8(%rip),%r11d + andl $0x80108,%r11d + cmpl $0x80108,%r11d + jne L$from_mont_nox + + leaq (%rax,%r9,1),%rdi + call __bn_sqrx8x_reduction + call __bn_postx4x_internal + + pxor %xmm0,%xmm0 + leaq 48(%rsp),%rax + jmp L$from_mont_zero + +.p2align 5 +L$from_mont_nox: + call __bn_sqr8x_reduction + call __bn_post4x_internal + + pxor %xmm0,%xmm0 + leaq 48(%rsp),%rax + jmp L$from_mont_zero + +.p2align 5 +L$from_mont_zero: + movq 40(%rsp),%rsi + + movdqa %xmm0,0(%rax) + movdqa %xmm0,16(%rax) + movdqa %xmm0,32(%rax) + movdqa %xmm0,48(%rax) + leaq 64(%rax),%rax + subq $32,%r9 + jnz L$from_mont_zero + + movq $1,%rax + movq -48(%rsi),%r15 + + movq -40(%rsi),%r14 + + movq -32(%rsi),%r13 + + movq -24(%rsi),%r12 + + movq -16(%rsi),%rbp + + movq -8(%rsi),%rbx + + leaq (%rsi),%rsp + +L$from_epilogue: + .byte 0xf3,0xc3 + + .p2align 5 bn_mulx4x_mont_gather5: diff --git a/openssl/src/crypto/bn/gen/linux_arm/armv4-gf2m.S b/openssl/src/crypto/bn/gen/linux_arm/armv4-gf2m.S index a4b6355ea..72fbfa389 100644 --- a/openssl/src/crypto/bn/gen/linux_arm/armv4-gf2m.S +++ b/openssl/src/crypto/bn/gen/linux_arm/armv4-gf2m.S @@ -231,5 +231,5 @@ bn_GF2m_mul_2x2: .align 5 #if __ARM_MAX_ARCH__>=7 - +.comm OPENSSL_armcap_P,4,4 #endif diff --git a/openssl/src/crypto/bn/gen/linux_arm/armv4-mont.S b/openssl/src/crypto/bn/gen/linux_arm/armv4-mont.S index 4a745a6cc..99713f8af 100644 --- a/openssl/src/crypto/bn/gen/linux_arm/armv4-mont.S +++ b/openssl/src/crypto/bn/gen/linux_arm/armv4-mont.S @@ -956,5 +956,5 @@ bn_mul8x_mont_neon: .align 2 .align 2 #if __ARM_MAX_ARCH__>=7 - +.comm OPENSSL_armcap_P,4,4 #endif diff --git a/openssl/src/crypto/bn/gen/linux_arm64/armv8-mont.S b/openssl/src/crypto/bn/gen/linux_arm64/armv8-mont.S index a867dbb2d..98d06f934 100644 --- a/openssl/src/crypto/bn/gen/linux_arm64/armv8-mont.S +++ b/openssl/src/crypto/bn/gen/linux_arm64/armv8-mont.S @@ -1,5 +1,5 @@ -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .hidden OPENSSL_armv8_rsa_neonized #endif @@ -9,7 +9,6 @@ .type bn_mul_mont,%function .align 5 bn_mul_mont: - AARCH64_SIGN_LINK_REGISTER .Lbn_mul_mont: tst x5,#3 b.ne .Lmul_mont @@ -220,14 +219,11 @@ bn_mul_mont: mov x0,#1 ldp x23,x24,[x29,#48] ldr x29,[sp],#64 - AARCH64_VALIDATE_LINK_REGISTER ret .size bn_mul_mont,.-bn_mul_mont .type bn_mul8x_mont_neon,%function .align 5 bn_mul8x_mont_neon: - // Not adding AARCH64_SIGN_LINK_REGISTER here because bn_mul8x_mont_neon is jumped to - // only from bn_mul_mont which has already signed the return address. stp x29,x30,[sp,#-80]! mov x16,sp stp d8,d9,[sp,#16] @@ -920,7 +916,6 @@ bn_mul8x_mont_neon: ldp d10,d11,[sp,#32] ldp d8,d9,[sp,#16] ldr x29,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER ret // bx lr .size bn_mul8x_mont_neon,.-bn_mul8x_mont_neon @@ -930,8 +925,7 @@ __bn_sqr8x_mont: cmp x1,x2 b.ne __bn_mul4x_mont .Lsqr8x_mont: - // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to - // only from bn_mul_mont which has already signed the return address. +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1682,15 +1676,13 @@ __bn_sqr8x_mont: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 - // x30 is loaded earlier - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size __bn_sqr8x_mont,.-__bn_sqr8x_mont .type __bn_mul4x_mont,%function .align 5 __bn_mul4x_mont: - // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to - // only from bn_mul_mont (or __bn_sqr8x_mont from bn_mul_mont) which has already signed the return address. +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2124,8 +2116,7 @@ __bn_mul4x_mont: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 - // x30 loaded earlier - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size __bn_mul4x_mont,.-__bn_mul4x_mont .byte 77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 diff --git a/openssl/src/crypto/bn/gen/linux_ia32/bn-586.S b/openssl/src/crypto/bn/gen/linux_ia32/bn-586.S index a74e6470b..f6cd0ac64 100644 --- a/openssl/src/crypto/bn/gen/linux_ia32/bn-586.S +++ b/openssl/src/crypto/bn/gen/linux_ia32/bn-586.S @@ -4,11 +4,7 @@ .align 16 bn_mul_add_words: .L_bn_mul_add_words_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - call .L000PIC_me_up .L000PIC_me_up: popl %eax @@ -292,11 +288,7 @@ bn_mul_add_words: .align 16 bn_mul_words: .L_bn_mul_words_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - call .L010PIC_me_up .L010PIC_me_up: popl %eax @@ -479,11 +471,7 @@ bn_mul_words: .align 16 bn_sqr_words: .L_bn_sqr_words_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - call .L017PIC_me_up .L017PIC_me_up: popl %eax @@ -625,11 +613,7 @@ bn_sqr_words: .align 16 bn_div_words: .L_bn_div_words_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%edx movl 8(%esp),%eax movl 12(%esp),%ecx @@ -641,11 +625,7 @@ bn_div_words: .align 16 bn_add_words: .L_bn_add_words_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -828,11 +808,7 @@ bn_add_words: .align 16 bn_sub_words: .L_bn_sub_words_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1015,11 +991,7 @@ bn_sub_words: .align 16 bn_sub_part_words: .L_bn_sub_part_words_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/bn/gen/linux_ia32/co-586.S b/openssl/src/crypto/bn/gen/linux_ia32/co-586.S index bc8cd2888..fe2c77c0b 100644 --- a/openssl/src/crypto/bn/gen/linux_ia32/co-586.S +++ b/openssl/src/crypto/bn/gen/linux_ia32/co-586.S @@ -4,11 +4,7 @@ .align 16 bn_mul_comba8: .L_bn_mul_comba8_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %esi movl 12(%esp),%esi pushl %edi @@ -553,11 +549,7 @@ bn_mul_comba8: .align 16 bn_mul_comba4: .L_bn_mul_comba4_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %esi movl 12(%esp),%esi pushl %edi @@ -726,11 +718,7 @@ bn_mul_comba4: .align 16 bn_sqr_comba8: .L_bn_sqr_comba8_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %esi pushl %edi pushl %ebp @@ -1139,11 +1127,7 @@ bn_sqr_comba8: .align 16 bn_sqr_comba4: .L_bn_sqr_comba4_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %esi pushl %edi pushl %ebp diff --git a/openssl/src/crypto/bn/gen/linux_ia32/x86-gf2m.S b/openssl/src/crypto/bn/gen/linux_ia32/x86-gf2m.S index b784b7cbe..bdddc2fc0 100644 --- a/openssl/src/crypto/bn/gen/linux_ia32/x86-gf2m.S +++ b/openssl/src/crypto/bn/gen/linux_ia32/x86-gf2m.S @@ -2,11 +2,7 @@ .type _mul_1x1_mmx,@function .align 16 _mul_1x1_mmx: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - subl $36,%esp movl %eax,%ecx leal (%eax,%eax,1),%edx @@ -110,11 +106,7 @@ _mul_1x1_mmx: .type _mul_1x1_ialu,@function .align 16 _mul_1x1_ialu: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - subl $36,%esp movl %eax,%ecx leal (%eax,%eax,1),%edx @@ -249,11 +241,7 @@ _mul_1x1_ialu: .align 16 bn_GF2m_mul_2x2: .L_bn_GF2m_mul_2x2_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - call .L000PIC_me_up .L000PIC_me_up: popl %edx diff --git a/openssl/src/crypto/bn/gen/linux_ia32/x86-mont.S b/openssl/src/crypto/bn/gen/linux_ia32/x86-mont.S index 5d8a4c6cb..f1a37f6eb 100644 --- a/openssl/src/crypto/bn/gen/linux_ia32/x86-mont.S +++ b/openssl/src/crypto/bn/gen/linux_ia32/x86-mont.S @@ -4,11 +4,7 @@ .align 16 bn_mul_mont: .L_bn_mul_mont_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/bn/gen/linux_ppc64/ppc64-mont-fixed.s b/openssl/src/crypto/bn/gen/linux_ppc64/ppc64-mont-fixed.s index fa657ef77..b129bf623 100644 --- a/openssl/src/crypto/bn/gen/linux_ppc64/ppc64-mont-fixed.s +++ b/openssl/src/crypto/bn/gen/linux_ppc64/ppc64-mont-fixed.s @@ -19,7 +19,6 @@ bn_mul_mont_fixed_n6: std 26,-64(1) std 27,-72(1) - li 0,0 ld 7,0(7) ld 11,0(5) @@ -115,7 +114,7 @@ bn_mul_mont_fixed_n6: addze 10,10 addc 26,26,10 - addze 27,0 + addze 27,27 .align 4 .Lenter_6: mulld 11,20,7 @@ -245,7 +244,6 @@ bn_mul_mont_300_fixed_n6: std 26,-64(1) std 27,-72(1) - li 0,0 ld 7,0(7) ld 11,0(5) @@ -320,7 +318,7 @@ bn_mul_mont_300_fixed_n6: addze 10,10 addc 26,26,10 - addze 27,0 + addze 27,27 .align 4 .Lenter_300_6: mulld 11,20,7 diff --git a/openssl/src/crypto/bn/gen/linux_x64/rsaz-2k-avx512.s b/openssl/src/crypto/bn/gen/linux_x64/rsaz-2k-avx512.s deleted file mode 100644 index 33df34c36..000000000 --- a/openssl/src/crypto/bn/gen/linux_x64/rsaz-2k-avx512.s +++ /dev/null @@ -1,916 +0,0 @@ - -.globl ossl_rsaz_avx512ifma_eligible -.type ossl_rsaz_avx512ifma_eligible,@function -.align 32 -ossl_rsaz_avx512ifma_eligible: - movl OPENSSL_ia32cap_P+8(%rip),%ecx - xorl %eax,%eax - andl $2149777408,%ecx - cmpl $2149777408,%ecx - cmovel %ecx,%eax - .byte 0xf3,0xc3 -.size ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible -.text - -.globl ossl_rsaz_amm52x20_x1_ifma256 -.type ossl_rsaz_amm52x20_x1_ifma256,@function -.align 32 -ossl_rsaz_amm52x20_x1_ifma256: -.cfi_startproc -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lossl_rsaz_amm52x20_x1_ifma256_body: - - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - - xorl %r9d,%r9d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - - movl $5,%ebx - -.align 32 -.Lloop5: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - movq 8(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - movq 16(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - movq 24(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - leaq 32(%r11),%r11 - decl %ebx - jne .Lloop5 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm16,%ymm1 - vpsrlq $52,%ymm17,%ymm2 - vpsrlq $52,%ymm18,%ymm25 - vpsrlq $52,%ymm19,%ymm26 - - - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm2,%ymm25,%ymm25 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm16,%ymm16 - vpandq .Lmask52x4(%rip),%ymm17,%ymm17 - vpandq .Lmask52x4(%rip),%ymm18,%ymm18 - vpandq .Lmask52x4(%rip),%ymm19,%ymm19 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm16,%ymm16 - vpaddq %ymm2,%ymm17,%ymm17 - vpaddq %ymm25,%ymm18,%ymm18 - vpaddq %ymm26,%ymm19,%ymm19 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm16,%k2 - vpcmpuq $6,.Lmask52x4(%rip),%ymm17,%k3 - vpcmpuq $6,.Lmask52x4(%rip),%ymm18,%k4 - vpcmpuq $6,.Lmask52x4(%rip),%ymm19,%k5 - kmovb %k1,%r14d - kmovb %k2,%r13d - kmovb %k3,%r12d - kmovb %k4,%r11d - kmovb %k5,%r10d - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm16,%k2 - vpcmpuq $0,.Lmask52x4(%rip),%ymm17,%k3 - vpcmpuq $0,.Lmask52x4(%rip),%ymm18,%k4 - vpcmpuq $0,.Lmask52x4(%rip),%ymm19,%k5 - kmovb %k1,%r9d - kmovb %k2,%r8d - kmovb %k3,%ebx - kmovb %k4,%ecx - kmovb %k5,%edx - - - - shlb $4,%r13b - orb %r13b,%r14b - shlb $4,%r11b - orb %r11b,%r12b - - addb %r14b,%r14b - adcb %r12b,%r12b - adcb %r10b,%r10b - - shlb $4,%r8b - orb %r8b,%r9b - shlb $4,%cl - orb %cl,%bl - - addb %r9b,%r14b - adcb %bl,%r12b - adcb %dl,%r10b - - xorb %r9b,%r14b - xorb %bl,%r12b - xorb %dl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r12d,%k3 - shrb $4,%r12b - kmovb %r12d,%k4 - kmovb %r10d,%k5 - - - vpsubq .Lmask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq .Lmask52x4(%rip),%ymm16,%ymm16{%k2} - vpsubq .Lmask52x4(%rip),%ymm17,%ymm17{%k3} - vpsubq .Lmask52x4(%rip),%ymm18,%ymm18{%k4} - vpsubq .Lmask52x4(%rip),%ymm19,%ymm19{%k5} - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm16,%ymm16 - vpandq .Lmask52x4(%rip),%ymm17,%ymm17 - vpandq .Lmask52x4(%rip),%ymm18,%ymm18 - vpandq .Lmask52x4(%rip),%ymm19,%ymm19 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm16,32(%rdi) - vmovdqu64 %ymm17,64(%rdi) - vmovdqu64 %ymm18,96(%rdi) - vmovdqu64 %ymm19,128(%rdi) - - vzeroupper - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%r12 -.cfi_restore %r12 - movq 32(%rsp),%rbp -.cfi_restore %rbp - movq 40(%rsp),%rbx -.cfi_restore %rbx - leaq 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lossl_rsaz_amm52x20_x1_ifma256_epilogue: - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256 -.data -.align 32 -.Lmask52x4: -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.text - -.globl ossl_rsaz_amm52x20_x2_ifma256 -.type ossl_rsaz_amm52x20_x2_ifma256,@function -.align 32 -ossl_rsaz_amm52x20_x2_ifma256: -.cfi_startproc -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lossl_rsaz_amm52x20_x2_ifma256_body: - - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm20 - vmovdqa64 %ymm0,%ymm21 - vmovdqa64 %ymm0,%ymm22 - vmovdqa64 %ymm0,%ymm23 - - xorl %r9d,%r9d - xorl %r15d,%r15d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - movl $20,%ebx - -.align 32 -.Lloop20: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq (%r8),%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm16 - vpmadd52luq 64(%rsi),%ymm1,%ymm17 - vpmadd52luq 96(%rsi),%ymm1,%ymm18 - vpmadd52luq 128(%rsi),%ymm1,%ymm19 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm16 - vpmadd52luq 64(%rcx),%ymm2,%ymm17 - vpmadd52luq 96(%rcx),%ymm2,%ymm18 - vpmadd52luq 128(%rcx),%ymm2,%ymm19 - - - valignq $1,%ymm3,%ymm16,%ymm3 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm0,%ymm19 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm16 - vpmadd52huq 64(%rsi),%ymm1,%ymm17 - vpmadd52huq 96(%rsi),%ymm1,%ymm18 - vpmadd52huq 128(%rsi),%ymm1,%ymm19 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm16 - vpmadd52huq 64(%rcx),%ymm2,%ymm17 - vpmadd52huq 96(%rcx),%ymm2,%ymm18 - vpmadd52huq 128(%rcx),%ymm2,%ymm19 - movq 160(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 160(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - movq %r12,%r10 - adcq $0,%r10 - - movq 8(%r8),%r13 - imulq %r15,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 160(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - adcq %r12,%r10 - - shrq $52,%r15 - salq $12,%r10 - orq %r10,%r15 - - vpmadd52luq 160(%rsi),%ymm1,%ymm4 - vpmadd52luq 192(%rsi),%ymm1,%ymm20 - vpmadd52luq 224(%rsi),%ymm1,%ymm21 - vpmadd52luq 256(%rsi),%ymm1,%ymm22 - vpmadd52luq 288(%rsi),%ymm1,%ymm23 - - vpmadd52luq 160(%rcx),%ymm2,%ymm4 - vpmadd52luq 192(%rcx),%ymm2,%ymm20 - vpmadd52luq 224(%rcx),%ymm2,%ymm21 - vpmadd52luq 256(%rcx),%ymm2,%ymm22 - vpmadd52luq 288(%rcx),%ymm2,%ymm23 - - - valignq $1,%ymm4,%ymm20,%ymm4 - valignq $1,%ymm20,%ymm21,%ymm20 - valignq $1,%ymm21,%ymm22,%ymm21 - valignq $1,%ymm22,%ymm23,%ymm22 - valignq $1,%ymm23,%ymm0,%ymm23 - - vmovq %xmm4,%r13 - addq %r13,%r15 - - vpmadd52huq 160(%rsi),%ymm1,%ymm4 - vpmadd52huq 192(%rsi),%ymm1,%ymm20 - vpmadd52huq 224(%rsi),%ymm1,%ymm21 - vpmadd52huq 256(%rsi),%ymm1,%ymm22 - vpmadd52huq 288(%rsi),%ymm1,%ymm23 - - vpmadd52huq 160(%rcx),%ymm2,%ymm4 - vpmadd52huq 192(%rcx),%ymm2,%ymm20 - vpmadd52huq 224(%rcx),%ymm2,%ymm21 - vpmadd52huq 256(%rcx),%ymm2,%ymm22 - vpmadd52huq 288(%rcx),%ymm2,%ymm23 - leaq 8(%r11),%r11 - decl %ebx - jne .Lloop20 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm16,%ymm1 - vpsrlq $52,%ymm17,%ymm2 - vpsrlq $52,%ymm18,%ymm25 - vpsrlq $52,%ymm19,%ymm26 - - - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm2,%ymm25,%ymm25 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm16,%ymm16 - vpandq .Lmask52x4(%rip),%ymm17,%ymm17 - vpandq .Lmask52x4(%rip),%ymm18,%ymm18 - vpandq .Lmask52x4(%rip),%ymm19,%ymm19 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm16,%ymm16 - vpaddq %ymm2,%ymm17,%ymm17 - vpaddq %ymm25,%ymm18,%ymm18 - vpaddq %ymm26,%ymm19,%ymm19 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm16,%k2 - vpcmpuq $6,.Lmask52x4(%rip),%ymm17,%k3 - vpcmpuq $6,.Lmask52x4(%rip),%ymm18,%k4 - vpcmpuq $6,.Lmask52x4(%rip),%ymm19,%k5 - kmovb %k1,%r14d - kmovb %k2,%r13d - kmovb %k3,%r12d - kmovb %k4,%r11d - kmovb %k5,%r10d - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm16,%k2 - vpcmpuq $0,.Lmask52x4(%rip),%ymm17,%k3 - vpcmpuq $0,.Lmask52x4(%rip),%ymm18,%k4 - vpcmpuq $0,.Lmask52x4(%rip),%ymm19,%k5 - kmovb %k1,%r9d - kmovb %k2,%r8d - kmovb %k3,%ebx - kmovb %k4,%ecx - kmovb %k5,%edx - - - - shlb $4,%r13b - orb %r13b,%r14b - shlb $4,%r11b - orb %r11b,%r12b - - addb %r14b,%r14b - adcb %r12b,%r12b - adcb %r10b,%r10b - - shlb $4,%r8b - orb %r8b,%r9b - shlb $4,%cl - orb %cl,%bl - - addb %r9b,%r14b - adcb %bl,%r12b - adcb %dl,%r10b - - xorb %r9b,%r14b - xorb %bl,%r12b - xorb %dl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r12d,%k3 - shrb $4,%r12b - kmovb %r12d,%k4 - kmovb %r10d,%k5 - - - vpsubq .Lmask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq .Lmask52x4(%rip),%ymm16,%ymm16{%k2} - vpsubq .Lmask52x4(%rip),%ymm17,%ymm17{%k3} - vpsubq .Lmask52x4(%rip),%ymm18,%ymm18{%k4} - vpsubq .Lmask52x4(%rip),%ymm19,%ymm19{%k5} - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm16,%ymm16 - vpandq .Lmask52x4(%rip),%ymm17,%ymm17 - vpandq .Lmask52x4(%rip),%ymm18,%ymm18 - vpandq .Lmask52x4(%rip),%ymm19,%ymm19 - - vpbroadcastq %r15,%ymm0 - vpblendd $3,%ymm0,%ymm4,%ymm4 - - - - vpsrlq $52,%ymm4,%ymm0 - vpsrlq $52,%ymm20,%ymm1 - vpsrlq $52,%ymm21,%ymm2 - vpsrlq $52,%ymm22,%ymm25 - vpsrlq $52,%ymm23,%ymm26 - - - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm2,%ymm25,%ymm25 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm20,%ymm20 - vpandq .Lmask52x4(%rip),%ymm21,%ymm21 - vpandq .Lmask52x4(%rip),%ymm22,%ymm22 - vpandq .Lmask52x4(%rip),%ymm23,%ymm23 - - - vpaddq %ymm0,%ymm4,%ymm4 - vpaddq %ymm1,%ymm20,%ymm20 - vpaddq %ymm2,%ymm21,%ymm21 - vpaddq %ymm25,%ymm22,%ymm22 - vpaddq %ymm26,%ymm23,%ymm23 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm4,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm20,%k2 - vpcmpuq $6,.Lmask52x4(%rip),%ymm21,%k3 - vpcmpuq $6,.Lmask52x4(%rip),%ymm22,%k4 - vpcmpuq $6,.Lmask52x4(%rip),%ymm23,%k5 - kmovb %k1,%r14d - kmovb %k2,%r13d - kmovb %k3,%r12d - kmovb %k4,%r11d - kmovb %k5,%r10d - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm4,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm20,%k2 - vpcmpuq $0,.Lmask52x4(%rip),%ymm21,%k3 - vpcmpuq $0,.Lmask52x4(%rip),%ymm22,%k4 - vpcmpuq $0,.Lmask52x4(%rip),%ymm23,%k5 - kmovb %k1,%r9d - kmovb %k2,%r8d - kmovb %k3,%ebx - kmovb %k4,%ecx - kmovb %k5,%edx - - - - shlb $4,%r13b - orb %r13b,%r14b - shlb $4,%r11b - orb %r11b,%r12b - - addb %r14b,%r14b - adcb %r12b,%r12b - adcb %r10b,%r10b - - shlb $4,%r8b - orb %r8b,%r9b - shlb $4,%cl - orb %cl,%bl - - addb %r9b,%r14b - adcb %bl,%r12b - adcb %dl,%r10b - - xorb %r9b,%r14b - xorb %bl,%r12b - xorb %dl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r12d,%k3 - shrb $4,%r12b - kmovb %r12d,%k4 - kmovb %r10d,%k5 - - - vpsubq .Lmask52x4(%rip),%ymm4,%ymm4{%k1} - vpsubq .Lmask52x4(%rip),%ymm20,%ymm20{%k2} - vpsubq .Lmask52x4(%rip),%ymm21,%ymm21{%k3} - vpsubq .Lmask52x4(%rip),%ymm22,%ymm22{%k4} - vpsubq .Lmask52x4(%rip),%ymm23,%ymm23{%k5} - - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm20,%ymm20 - vpandq .Lmask52x4(%rip),%ymm21,%ymm21 - vpandq .Lmask52x4(%rip),%ymm22,%ymm22 - vpandq .Lmask52x4(%rip),%ymm23,%ymm23 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm16,32(%rdi) - vmovdqu64 %ymm17,64(%rdi) - vmovdqu64 %ymm18,96(%rdi) - vmovdqu64 %ymm19,128(%rdi) - - vmovdqu64 %ymm4,160(%rdi) - vmovdqu64 %ymm20,192(%rdi) - vmovdqu64 %ymm21,224(%rdi) - vmovdqu64 %ymm22,256(%rdi) - vmovdqu64 %ymm23,288(%rdi) - - vzeroupper - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%r12 -.cfi_restore %r12 - movq 32(%rsp),%rbp -.cfi_restore %rbp - movq 40(%rsp),%rbx -.cfi_restore %rbx - leaq 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lossl_rsaz_amm52x20_x2_ifma256_epilogue: - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_rsaz_amm52x20_x2_ifma256, .-ossl_rsaz_amm52x20_x2_ifma256 -.text - -.align 32 -.globl ossl_extract_multiplier_2x20_win5 -.type ossl_extract_multiplier_2x20_win5,@function -ossl_extract_multiplier_2x20_win5: -.cfi_startproc -.byte 243,15,30,250 - vmovdqa64 .Lones(%rip),%ymm24 - vpbroadcastq %rdx,%ymm22 - vpbroadcastq %rcx,%ymm23 - leaq 10240(%rsi),%rax - - - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %ymm0,%ymm21 - vmovdqa64 %ymm0,%ymm1 - vmovdqa64 %ymm0,%ymm2 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - -.align 32 -.Lloop: - vpcmpq $0,%ymm21,%ymm22,%k1 - vpcmpq $0,%ymm21,%ymm23,%k2 - vmovdqu64 0(%rsi),%ymm20 - vpblendmq %ymm20,%ymm0,%ymm0{%k1} - vmovdqu64 32(%rsi),%ymm20 - vpblendmq %ymm20,%ymm1,%ymm1{%k1} - vmovdqu64 64(%rsi),%ymm20 - vpblendmq %ymm20,%ymm2,%ymm2{%k1} - vmovdqu64 96(%rsi),%ymm20 - vpblendmq %ymm20,%ymm3,%ymm3{%k1} - vmovdqu64 128(%rsi),%ymm20 - vpblendmq %ymm20,%ymm4,%ymm4{%k1} - vmovdqu64 160(%rsi),%ymm20 - vpblendmq %ymm20,%ymm5,%ymm5{%k2} - vmovdqu64 192(%rsi),%ymm20 - vpblendmq %ymm20,%ymm16,%ymm16{%k2} - vmovdqu64 224(%rsi),%ymm20 - vpblendmq %ymm20,%ymm17,%ymm17{%k2} - vmovdqu64 256(%rsi),%ymm20 - vpblendmq %ymm20,%ymm18,%ymm18{%k2} - vmovdqu64 288(%rsi),%ymm20 - vpblendmq %ymm20,%ymm19,%ymm19{%k2} - vpaddq %ymm24,%ymm21,%ymm21 - addq $320,%rsi - cmpq %rsi,%rax - jne .Lloop - vmovdqu64 %ymm0,0(%rdi) - vmovdqu64 %ymm1,32(%rdi) - vmovdqu64 %ymm2,64(%rdi) - vmovdqu64 %ymm3,96(%rdi) - vmovdqu64 %ymm4,128(%rdi) - vmovdqu64 %ymm5,160(%rdi) - vmovdqu64 %ymm16,192(%rdi) - vmovdqu64 %ymm17,224(%rdi) - vmovdqu64 %ymm18,256(%rdi) - vmovdqu64 %ymm19,288(%rdi) - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_extract_multiplier_2x20_win5, .-ossl_extract_multiplier_2x20_win5 -.data -.align 32 -.Lones: -.quad 1,1,1,1 -.Lzeros: -.quad 0,0,0,0 - .section ".note.gnu.property", "a" - .p2align 3 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - # "GNU" encoded with .byte, since .asciz isn't supported - # on Solaris. - .byte 0x47 - .byte 0x4e - .byte 0x55 - .byte 0 -1: - .p2align 3 - .long 0xc0000002 - .long 3f - 2f -2: - .long 3 -3: - .p2align 3 -4: diff --git a/openssl/src/crypto/bn/gen/linux_x64/rsaz-3k-avx512.s b/openssl/src/crypto/bn/gen/linux_x64/rsaz-3k-avx512.s deleted file mode 100644 index 00a709cef..000000000 --- a/openssl/src/crypto/bn/gen/linux_x64/rsaz-3k-avx512.s +++ /dev/null @@ -1,1331 +0,0 @@ -.text - -.globl ossl_rsaz_amm52x30_x1_ifma256 -.type ossl_rsaz_amm52x30_x1_ifma256,@function -.align 32 -ossl_rsaz_amm52x30_x1_ifma256: -.cfi_startproc -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm6 - vmovdqa64 %ymm0,%ymm7 - vmovdqa64 %ymm0,%ymm8 - vmovdqa64 %ymm0,%ymm9 - vmovdqa64 %ymm0,%ymm10 - - xorl %r9d,%r9d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - - movl $7,%ebx - -.align 32 -.Lloop7: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 8(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 16(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 24(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - leaq 32(%r11),%r11 - decl %ebx - jne .Lloop7 - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 8(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm4,%ymm1 - vpsrlq $52,%ymm5,%ymm2 - vpsrlq $52,%ymm6,%ymm19 - vpsrlq $52,%ymm7,%ymm20 - vpsrlq $52,%ymm8,%ymm21 - vpsrlq $52,%ymm9,%ymm22 - vpsrlq $52,%ymm10,%ymm23 - - - valignq $3,%ymm22,%ymm23,%ymm23 - valignq $3,%ymm21,%ymm22,%ymm22 - valignq $3,%ymm20,%ymm21,%ymm21 - valignq $3,%ymm19,%ymm20,%ymm20 - valignq $3,%ymm2,%ymm19,%ymm19 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm5,%ymm5 - vpandq .Lmask52x4(%rip),%ymm6,%ymm6 - vpandq .Lmask52x4(%rip),%ymm7,%ymm7 - vpandq .Lmask52x4(%rip),%ymm8,%ymm8 - vpandq .Lmask52x4(%rip),%ymm9,%ymm9 - vpandq .Lmask52x4(%rip),%ymm10,%ymm10 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm4,%ymm4 - vpaddq %ymm2,%ymm5,%ymm5 - vpaddq %ymm19,%ymm6,%ymm6 - vpaddq %ymm20,%ymm7,%ymm7 - vpaddq %ymm21,%ymm8,%ymm8 - vpaddq %ymm22,%ymm9,%ymm9 - vpaddq %ymm23,%ymm10,%ymm10 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm5,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm7,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm8,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm9,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm10,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm5,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm7,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm8,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm9,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm10,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq .Lmask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq .Lmask52x4(%rip),%ymm4,%ymm4{%k2} - vpsubq .Lmask52x4(%rip),%ymm5,%ymm5{%k3} - vpsubq .Lmask52x4(%rip),%ymm6,%ymm6{%k4} - vpsubq .Lmask52x4(%rip),%ymm7,%ymm7{%k5} - vpsubq .Lmask52x4(%rip),%ymm8,%ymm8{%k6} - vpsubq .Lmask52x4(%rip),%ymm9,%ymm9{%k7} - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm5,%ymm5 - vpandq .Lmask52x4(%rip),%ymm6,%ymm6 - vpandq .Lmask52x4(%rip),%ymm7,%ymm7 - vpandq .Lmask52x4(%rip),%ymm8,%ymm8 - vpandq .Lmask52x4(%rip),%ymm9,%ymm9 - - shrb $4,%r11b - kmovb %r11d,%k1 - - vpsubq .Lmask52x4(%rip),%ymm10,%ymm10{%k1} - - vpandq .Lmask52x4(%rip),%ymm10,%ymm10 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm4,32(%rdi) - vmovdqu64 %ymm5,64(%rdi) - vmovdqu64 %ymm6,96(%rdi) - vmovdqu64 %ymm7,128(%rdi) - vmovdqu64 %ymm8,160(%rdi) - vmovdqu64 %ymm9,192(%rdi) - vmovdqu64 %ymm10,224(%rdi) - - vzeroupper - leaq (%rsp),%rax -.cfi_def_cfa_register %rax - movq 0(%rax),%r15 -.cfi_restore %r15 - movq 8(%rax),%r14 -.cfi_restore %r14 - movq 16(%rax),%r13 -.cfi_restore %r13 - movq 24(%rax),%r12 -.cfi_restore %r12 - movq 32(%rax),%rbp -.cfi_restore %rbp - movq 40(%rax),%rbx -.cfi_restore %rbx - leaq 48(%rax),%rsp -.cfi_def_cfa %rsp,8 -.Lossl_rsaz_amm52x30_x1_ifma256_epilogue: - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_rsaz_amm52x30_x1_ifma256, .-ossl_rsaz_amm52x30_x1_ifma256 -.data -.align 32 -.Lmask52x4: -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.text - -.globl ossl_rsaz_amm52x30_x2_ifma256 -.type ossl_rsaz_amm52x30_x2_ifma256,@function -.align 32 -ossl_rsaz_amm52x30_x2_ifma256: -.cfi_startproc -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm6 - vmovdqa64 %ymm0,%ymm7 - vmovdqa64 %ymm0,%ymm8 - vmovdqa64 %ymm0,%ymm9 - vmovdqa64 %ymm0,%ymm10 - - vmovdqa64 %ymm0,%ymm11 - vmovdqa64 %ymm0,%ymm12 - vmovdqa64 %ymm0,%ymm13 - vmovdqa64 %ymm0,%ymm14 - vmovdqa64 %ymm0,%ymm15 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - - - xorl %r9d,%r9d - xorl %r15d,%r15d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - movl $30,%ebx - -.align 32 -.Lloop30: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq (%r8),%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm0,%ymm10 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - movq 256(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 256(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - movq %r12,%r10 - adcq $0,%r10 - - movq 8(%r8),%r13 - imulq %r15,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 256(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - adcq %r12,%r10 - - shrq $52,%r15 - salq $12,%r10 - orq %r10,%r15 - - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - vpmadd52luq 320(%rsi),%ymm1,%ymm13 - vpmadd52luq 352(%rsi),%ymm1,%ymm14 - vpmadd52luq 384(%rsi),%ymm1,%ymm15 - vpmadd52luq 416(%rsi),%ymm1,%ymm16 - vpmadd52luq 448(%rsi),%ymm1,%ymm17 - vpmadd52luq 480(%rsi),%ymm1,%ymm18 - - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - vpmadd52luq 320(%rcx),%ymm2,%ymm13 - vpmadd52luq 352(%rcx),%ymm2,%ymm14 - vpmadd52luq 384(%rcx),%ymm2,%ymm15 - vpmadd52luq 416(%rcx),%ymm2,%ymm16 - vpmadd52luq 448(%rcx),%ymm2,%ymm17 - vpmadd52luq 480(%rcx),%ymm2,%ymm18 - - - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm13,%ymm12 - valignq $1,%ymm13,%ymm14,%ymm13 - valignq $1,%ymm14,%ymm15,%ymm14 - valignq $1,%ymm15,%ymm16,%ymm15 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm0,%ymm18 - - vmovq %xmm11,%r13 - addq %r13,%r15 - - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - vpmadd52huq 320(%rsi),%ymm1,%ymm13 - vpmadd52huq 352(%rsi),%ymm1,%ymm14 - vpmadd52huq 384(%rsi),%ymm1,%ymm15 - vpmadd52huq 416(%rsi),%ymm1,%ymm16 - vpmadd52huq 448(%rsi),%ymm1,%ymm17 - vpmadd52huq 480(%rsi),%ymm1,%ymm18 - - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - vpmadd52huq 320(%rcx),%ymm2,%ymm13 - vpmadd52huq 352(%rcx),%ymm2,%ymm14 - vpmadd52huq 384(%rcx),%ymm2,%ymm15 - vpmadd52huq 416(%rcx),%ymm2,%ymm16 - vpmadd52huq 448(%rcx),%ymm2,%ymm17 - vpmadd52huq 480(%rcx),%ymm2,%ymm18 - leaq 8(%r11),%r11 - decl %ebx - jne .Lloop30 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm4,%ymm1 - vpsrlq $52,%ymm5,%ymm2 - vpsrlq $52,%ymm6,%ymm19 - vpsrlq $52,%ymm7,%ymm20 - vpsrlq $52,%ymm8,%ymm21 - vpsrlq $52,%ymm9,%ymm22 - vpsrlq $52,%ymm10,%ymm23 - - - valignq $3,%ymm22,%ymm23,%ymm23 - valignq $3,%ymm21,%ymm22,%ymm22 - valignq $3,%ymm20,%ymm21,%ymm21 - valignq $3,%ymm19,%ymm20,%ymm20 - valignq $3,%ymm2,%ymm19,%ymm19 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm5,%ymm5 - vpandq .Lmask52x4(%rip),%ymm6,%ymm6 - vpandq .Lmask52x4(%rip),%ymm7,%ymm7 - vpandq .Lmask52x4(%rip),%ymm8,%ymm8 - vpandq .Lmask52x4(%rip),%ymm9,%ymm9 - vpandq .Lmask52x4(%rip),%ymm10,%ymm10 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm4,%ymm4 - vpaddq %ymm2,%ymm5,%ymm5 - vpaddq %ymm19,%ymm6,%ymm6 - vpaddq %ymm20,%ymm7,%ymm7 - vpaddq %ymm21,%ymm8,%ymm8 - vpaddq %ymm22,%ymm9,%ymm9 - vpaddq %ymm23,%ymm10,%ymm10 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm5,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm7,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm8,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm9,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm10,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm5,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm7,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm8,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm9,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm10,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq .Lmask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq .Lmask52x4(%rip),%ymm4,%ymm4{%k2} - vpsubq .Lmask52x4(%rip),%ymm5,%ymm5{%k3} - vpsubq .Lmask52x4(%rip),%ymm6,%ymm6{%k4} - vpsubq .Lmask52x4(%rip),%ymm7,%ymm7{%k5} - vpsubq .Lmask52x4(%rip),%ymm8,%ymm8{%k6} - vpsubq .Lmask52x4(%rip),%ymm9,%ymm9{%k7} - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm5,%ymm5 - vpandq .Lmask52x4(%rip),%ymm6,%ymm6 - vpandq .Lmask52x4(%rip),%ymm7,%ymm7 - vpandq .Lmask52x4(%rip),%ymm8,%ymm8 - vpandq .Lmask52x4(%rip),%ymm9,%ymm9 - - shrb $4,%r11b - kmovb %r11d,%k1 - - vpsubq .Lmask52x4(%rip),%ymm10,%ymm10{%k1} - - vpandq .Lmask52x4(%rip),%ymm10,%ymm10 - - vpbroadcastq %r15,%ymm0 - vpblendd $3,%ymm0,%ymm11,%ymm11 - - - - vpsrlq $52,%ymm11,%ymm0 - vpsrlq $52,%ymm12,%ymm1 - vpsrlq $52,%ymm13,%ymm2 - vpsrlq $52,%ymm14,%ymm19 - vpsrlq $52,%ymm15,%ymm20 - vpsrlq $52,%ymm16,%ymm21 - vpsrlq $52,%ymm17,%ymm22 - vpsrlq $52,%ymm18,%ymm23 - - - valignq $3,%ymm22,%ymm23,%ymm23 - valignq $3,%ymm21,%ymm22,%ymm22 - valignq $3,%ymm20,%ymm21,%ymm21 - valignq $3,%ymm19,%ymm20,%ymm20 - valignq $3,%ymm2,%ymm19,%ymm19 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm11,%ymm11 - vpandq .Lmask52x4(%rip),%ymm12,%ymm12 - vpandq .Lmask52x4(%rip),%ymm13,%ymm13 - vpandq .Lmask52x4(%rip),%ymm14,%ymm14 - vpandq .Lmask52x4(%rip),%ymm15,%ymm15 - vpandq .Lmask52x4(%rip),%ymm16,%ymm16 - vpandq .Lmask52x4(%rip),%ymm17,%ymm17 - vpandq .Lmask52x4(%rip),%ymm18,%ymm18 - - - vpaddq %ymm0,%ymm11,%ymm11 - vpaddq %ymm1,%ymm12,%ymm12 - vpaddq %ymm2,%ymm13,%ymm13 - vpaddq %ymm19,%ymm14,%ymm14 - vpaddq %ymm20,%ymm15,%ymm15 - vpaddq %ymm21,%ymm16,%ymm16 - vpaddq %ymm22,%ymm17,%ymm17 - vpaddq %ymm23,%ymm18,%ymm18 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm11,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm12,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm13,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm14,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm15,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm16,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm17,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm18,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm11,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm12,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm13,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm14,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm15,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm16,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm17,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm18,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq .Lmask52x4(%rip),%ymm11,%ymm11{%k1} - vpsubq .Lmask52x4(%rip),%ymm12,%ymm12{%k2} - vpsubq .Lmask52x4(%rip),%ymm13,%ymm13{%k3} - vpsubq .Lmask52x4(%rip),%ymm14,%ymm14{%k4} - vpsubq .Lmask52x4(%rip),%ymm15,%ymm15{%k5} - vpsubq .Lmask52x4(%rip),%ymm16,%ymm16{%k6} - vpsubq .Lmask52x4(%rip),%ymm17,%ymm17{%k7} - - vpandq .Lmask52x4(%rip),%ymm11,%ymm11 - vpandq .Lmask52x4(%rip),%ymm12,%ymm12 - vpandq .Lmask52x4(%rip),%ymm13,%ymm13 - vpandq .Lmask52x4(%rip),%ymm14,%ymm14 - vpandq .Lmask52x4(%rip),%ymm15,%ymm15 - vpandq .Lmask52x4(%rip),%ymm16,%ymm16 - vpandq .Lmask52x4(%rip),%ymm17,%ymm17 - - shrb $4,%r11b - kmovb %r11d,%k1 - - vpsubq .Lmask52x4(%rip),%ymm18,%ymm18{%k1} - - vpandq .Lmask52x4(%rip),%ymm18,%ymm18 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm4,32(%rdi) - vmovdqu64 %ymm5,64(%rdi) - vmovdqu64 %ymm6,96(%rdi) - vmovdqu64 %ymm7,128(%rdi) - vmovdqu64 %ymm8,160(%rdi) - vmovdqu64 %ymm9,192(%rdi) - vmovdqu64 %ymm10,224(%rdi) - - vmovdqu64 %ymm11,256(%rdi) - vmovdqu64 %ymm12,288(%rdi) - vmovdqu64 %ymm13,320(%rdi) - vmovdqu64 %ymm14,352(%rdi) - vmovdqu64 %ymm15,384(%rdi) - vmovdqu64 %ymm16,416(%rdi) - vmovdqu64 %ymm17,448(%rdi) - vmovdqu64 %ymm18,480(%rdi) - - vzeroupper - leaq (%rsp),%rax -.cfi_def_cfa_register %rax - movq 0(%rax),%r15 -.cfi_restore %r15 - movq 8(%rax),%r14 -.cfi_restore %r14 - movq 16(%rax),%r13 -.cfi_restore %r13 - movq 24(%rax),%r12 -.cfi_restore %r12 - movq 32(%rax),%rbp -.cfi_restore %rbp - movq 40(%rax),%rbx -.cfi_restore %rbx - leaq 48(%rax),%rsp -.cfi_def_cfa %rsp,8 -.Lossl_rsaz_amm52x30_x2_ifma256_epilogue: - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_rsaz_amm52x30_x2_ifma256, .-ossl_rsaz_amm52x30_x2_ifma256 -.text - -.align 32 -.globl ossl_extract_multiplier_2x30_win5 -.type ossl_extract_multiplier_2x30_win5,@function -ossl_extract_multiplier_2x30_win5: -.cfi_startproc -.byte 243,15,30,250 - vmovdqa64 .Lones(%rip),%ymm30 - vpbroadcastq %rdx,%ymm28 - vpbroadcastq %rcx,%ymm29 - leaq 16384(%rsi),%rax - - - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %ymm0,%ymm27 - vmovdqa64 %ymm0,%ymm1 - vmovdqa64 %ymm0,%ymm2 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - vmovdqa64 %ymm0,%ymm20 - vmovdqa64 %ymm0,%ymm21 - vmovdqa64 %ymm0,%ymm22 - vmovdqa64 %ymm0,%ymm23 - vmovdqa64 %ymm0,%ymm24 - vmovdqa64 %ymm0,%ymm25 - -.align 32 -.Lloop: - vpcmpq $0,%ymm27,%ymm28,%k1 - vpcmpq $0,%ymm27,%ymm29,%k2 - vmovdqu64 0(%rsi),%ymm26 - vpblendmq %ymm26,%ymm0,%ymm0{%k1} - vmovdqu64 32(%rsi),%ymm26 - vpblendmq %ymm26,%ymm1,%ymm1{%k1} - vmovdqu64 64(%rsi),%ymm26 - vpblendmq %ymm26,%ymm2,%ymm2{%k1} - vmovdqu64 96(%rsi),%ymm26 - vpblendmq %ymm26,%ymm3,%ymm3{%k1} - vmovdqu64 128(%rsi),%ymm26 - vpblendmq %ymm26,%ymm4,%ymm4{%k1} - vmovdqu64 160(%rsi),%ymm26 - vpblendmq %ymm26,%ymm5,%ymm5{%k1} - vmovdqu64 192(%rsi),%ymm26 - vpblendmq %ymm26,%ymm16,%ymm16{%k1} - vmovdqu64 224(%rsi),%ymm26 - vpblendmq %ymm26,%ymm17,%ymm17{%k1} - vmovdqu64 256(%rsi),%ymm26 - vpblendmq %ymm26,%ymm18,%ymm18{%k2} - vmovdqu64 288(%rsi),%ymm26 - vpblendmq %ymm26,%ymm19,%ymm19{%k2} - vmovdqu64 320(%rsi),%ymm26 - vpblendmq %ymm26,%ymm20,%ymm20{%k2} - vmovdqu64 352(%rsi),%ymm26 - vpblendmq %ymm26,%ymm21,%ymm21{%k2} - vmovdqu64 384(%rsi),%ymm26 - vpblendmq %ymm26,%ymm22,%ymm22{%k2} - vmovdqu64 416(%rsi),%ymm26 - vpblendmq %ymm26,%ymm23,%ymm23{%k2} - vmovdqu64 448(%rsi),%ymm26 - vpblendmq %ymm26,%ymm24,%ymm24{%k2} - vmovdqu64 480(%rsi),%ymm26 - vpblendmq %ymm26,%ymm25,%ymm25{%k2} - vpaddq %ymm30,%ymm27,%ymm27 - addq $512,%rsi - cmpq %rsi,%rax - jne .Lloop - vmovdqu64 %ymm0,0(%rdi) - vmovdqu64 %ymm1,32(%rdi) - vmovdqu64 %ymm2,64(%rdi) - vmovdqu64 %ymm3,96(%rdi) - vmovdqu64 %ymm4,128(%rdi) - vmovdqu64 %ymm5,160(%rdi) - vmovdqu64 %ymm16,192(%rdi) - vmovdqu64 %ymm17,224(%rdi) - vmovdqu64 %ymm18,256(%rdi) - vmovdqu64 %ymm19,288(%rdi) - vmovdqu64 %ymm20,320(%rdi) - vmovdqu64 %ymm21,352(%rdi) - vmovdqu64 %ymm22,384(%rdi) - vmovdqu64 %ymm23,416(%rdi) - vmovdqu64 %ymm24,448(%rdi) - vmovdqu64 %ymm25,480(%rdi) - - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_extract_multiplier_2x30_win5, .-ossl_extract_multiplier_2x30_win5 -.data -.align 32 -.Lones: -.quad 1,1,1,1 -.Lzeros: -.quad 0,0,0,0 - .section ".note.gnu.property", "a" - .p2align 3 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - # "GNU" encoded with .byte, since .asciz isn't supported - # on Solaris. - .byte 0x47 - .byte 0x4e - .byte 0x55 - .byte 0 -1: - .p2align 3 - .long 0xc0000002 - .long 3f - 2f -2: - .long 3 -3: - .p2align 3 -4: diff --git a/openssl/src/crypto/bn/gen/linux_x64/rsaz-4k-avx512.s b/openssl/src/crypto/bn/gen/linux_x64/rsaz-4k-avx512.s deleted file mode 100644 index 82b976907..000000000 --- a/openssl/src/crypto/bn/gen/linux_x64/rsaz-4k-avx512.s +++ /dev/null @@ -1,1374 +0,0 @@ -.text - -.globl ossl_rsaz_amm52x40_x1_ifma256 -.type ossl_rsaz_amm52x40_x1_ifma256,@function -.align 32 -ossl_rsaz_amm52x40_x1_ifma256: -.cfi_startproc -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm6 - vmovdqa64 %ymm0,%ymm7 - vmovdqa64 %ymm0,%ymm8 - vmovdqa64 %ymm0,%ymm9 - vmovdqa64 %ymm0,%ymm10 - vmovdqa64 %ymm0,%ymm11 - vmovdqa64 %ymm0,%ymm12 - - xorl %r9d,%r9d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - - movl $10,%ebx - -.align 32 -.Lloop10: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - movq 8(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - movq 16(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - movq 24(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq %r8,%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - leaq 32(%r11),%r11 - decl %ebx - jne .Lloop10 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm4,%ymm1 - vpsrlq $52,%ymm5,%ymm2 - vpsrlq $52,%ymm6,%ymm23 - vpsrlq $52,%ymm7,%ymm24 - vpsrlq $52,%ymm8,%ymm25 - vpsrlq $52,%ymm9,%ymm26 - vpsrlq $52,%ymm10,%ymm27 - vpsrlq $52,%ymm11,%ymm28 - vpsrlq $52,%ymm12,%ymm29 - - - valignq $3,%ymm28,%ymm29,%ymm29 - valignq $3,%ymm27,%ymm28,%ymm28 - valignq $3,%ymm26,%ymm27,%ymm27 - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm24,%ymm25,%ymm25 - valignq $3,%ymm23,%ymm24,%ymm24 - valignq $3,%ymm2,%ymm23,%ymm23 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm5,%ymm5 - vpandq .Lmask52x4(%rip),%ymm6,%ymm6 - vpandq .Lmask52x4(%rip),%ymm7,%ymm7 - vpandq .Lmask52x4(%rip),%ymm8,%ymm8 - vpandq .Lmask52x4(%rip),%ymm9,%ymm9 - vpandq .Lmask52x4(%rip),%ymm10,%ymm10 - vpandq .Lmask52x4(%rip),%ymm11,%ymm11 - vpandq .Lmask52x4(%rip),%ymm12,%ymm12 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm4,%ymm4 - vpaddq %ymm2,%ymm5,%ymm5 - vpaddq %ymm23,%ymm6,%ymm6 - vpaddq %ymm24,%ymm7,%ymm7 - vpaddq %ymm25,%ymm8,%ymm8 - vpaddq %ymm26,%ymm9,%ymm9 - vpaddq %ymm27,%ymm10,%ymm10 - vpaddq %ymm28,%ymm11,%ymm11 - vpaddq %ymm29,%ymm12,%ymm12 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm5,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm7,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm8,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm9,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm10,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm11,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm12,%k2 - kmovb %k1,%r10d - kmovb %k2,%r9d - shlb $4,%r9b - orb %r9b,%r10b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - adcb %r10b,%r10b - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm5,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm7,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm8,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm9,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm10,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm11,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm12,%k2 - kmovb %k1,%ebx - kmovb %k2,%eax - shlb $4,%al - orb %al,%bl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - adcb %bl,%r10b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - xorb %bl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq .Lmask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq .Lmask52x4(%rip),%ymm4,%ymm4{%k2} - vpsubq .Lmask52x4(%rip),%ymm5,%ymm5{%k3} - vpsubq .Lmask52x4(%rip),%ymm6,%ymm6{%k4} - vpsubq .Lmask52x4(%rip),%ymm7,%ymm7{%k5} - vpsubq .Lmask52x4(%rip),%ymm8,%ymm8{%k6} - vpsubq .Lmask52x4(%rip),%ymm9,%ymm9{%k7} - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm5,%ymm5 - vpandq .Lmask52x4(%rip),%ymm6,%ymm6 - vpandq .Lmask52x4(%rip),%ymm7,%ymm7 - vpandq .Lmask52x4(%rip),%ymm8,%ymm8 - vpandq .Lmask52x4(%rip),%ymm9,%ymm9 - - shrb $4,%r11b - kmovb %r11d,%k1 - kmovb %r10d,%k2 - shrb $4,%r10b - kmovb %r10d,%k3 - - vpsubq .Lmask52x4(%rip),%ymm10,%ymm10{%k1} - vpsubq .Lmask52x4(%rip),%ymm11,%ymm11{%k2} - vpsubq .Lmask52x4(%rip),%ymm12,%ymm12{%k3} - - vpandq .Lmask52x4(%rip),%ymm10,%ymm10 - vpandq .Lmask52x4(%rip),%ymm11,%ymm11 - vpandq .Lmask52x4(%rip),%ymm12,%ymm12 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm4,32(%rdi) - vmovdqu64 %ymm5,64(%rdi) - vmovdqu64 %ymm6,96(%rdi) - vmovdqu64 %ymm7,128(%rdi) - vmovdqu64 %ymm8,160(%rdi) - vmovdqu64 %ymm9,192(%rdi) - vmovdqu64 %ymm10,224(%rdi) - vmovdqu64 %ymm11,256(%rdi) - vmovdqu64 %ymm12,288(%rdi) - - vzeroupper - leaq (%rsp),%rax -.cfi_def_cfa_register %rax - movq 0(%rax),%r15 -.cfi_restore %r15 - movq 8(%rax),%r14 -.cfi_restore %r14 - movq 16(%rax),%r13 -.cfi_restore %r13 - movq 24(%rax),%r12 -.cfi_restore %r12 - movq 32(%rax),%rbp -.cfi_restore %rbp - movq 40(%rax),%rbx -.cfi_restore %rbx - leaq 48(%rax),%rsp -.cfi_def_cfa %rsp,8 -.Lossl_rsaz_amm52x40_x1_ifma256_epilogue: - - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_rsaz_amm52x40_x1_ifma256, .-ossl_rsaz_amm52x40_x1_ifma256 -.data -.align 32 -.Lmask52x4: -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.quad 0xfffffffffffff -.text - -.globl ossl_rsaz_amm52x40_x2_ifma256 -.type ossl_rsaz_amm52x40_x2_ifma256,@function -.align 32 -ossl_rsaz_amm52x40_x2_ifma256: -.cfi_startproc -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - - vpxord %ymm0,%ymm0,%ymm0 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm6 - vmovdqa64 %ymm0,%ymm7 - vmovdqa64 %ymm0,%ymm8 - vmovdqa64 %ymm0,%ymm9 - vmovdqa64 %ymm0,%ymm10 - vmovdqa64 %ymm0,%ymm11 - vmovdqa64 %ymm0,%ymm12 - - vmovdqa64 %ymm0,%ymm13 - vmovdqa64 %ymm0,%ymm14 - vmovdqa64 %ymm0,%ymm15 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - vmovdqa64 %ymm0,%ymm20 - vmovdqa64 %ymm0,%ymm21 - vmovdqa64 %ymm0,%ymm22 - - - xorl %r9d,%r9d - xorl %r15d,%r15d - - movq %rdx,%r11 - movq $0xfffffffffffff,%rax - - movl $40,%ebx - -.align 32 -.Lloop40: - movq 0(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 0(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - movq %r12,%r10 - adcq $0,%r10 - - movq (%r8),%r13 - imulq %r9,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 0(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r9 - adcq %r12,%r10 - - shrq $52,%r9 - salq $12,%r10 - orq %r10,%r9 - - vpmadd52luq 0(%rsi),%ymm1,%ymm3 - vpmadd52luq 32(%rsi),%ymm1,%ymm4 - vpmadd52luq 64(%rsi),%ymm1,%ymm5 - vpmadd52luq 96(%rsi),%ymm1,%ymm6 - vpmadd52luq 128(%rsi),%ymm1,%ymm7 - vpmadd52luq 160(%rsi),%ymm1,%ymm8 - vpmadd52luq 192(%rsi),%ymm1,%ymm9 - vpmadd52luq 224(%rsi),%ymm1,%ymm10 - vpmadd52luq 256(%rsi),%ymm1,%ymm11 - vpmadd52luq 288(%rsi),%ymm1,%ymm12 - - vpmadd52luq 0(%rcx),%ymm2,%ymm3 - vpmadd52luq 32(%rcx),%ymm2,%ymm4 - vpmadd52luq 64(%rcx),%ymm2,%ymm5 - vpmadd52luq 96(%rcx),%ymm2,%ymm6 - vpmadd52luq 128(%rcx),%ymm2,%ymm7 - vpmadd52luq 160(%rcx),%ymm2,%ymm8 - vpmadd52luq 192(%rcx),%ymm2,%ymm9 - vpmadd52luq 224(%rcx),%ymm2,%ymm10 - vpmadd52luq 256(%rcx),%ymm2,%ymm11 - vpmadd52luq 288(%rcx),%ymm2,%ymm12 - - - valignq $1,%ymm3,%ymm4,%ymm3 - valignq $1,%ymm4,%ymm5,%ymm4 - valignq $1,%ymm5,%ymm6,%ymm5 - valignq $1,%ymm6,%ymm7,%ymm6 - valignq $1,%ymm7,%ymm8,%ymm7 - valignq $1,%ymm8,%ymm9,%ymm8 - valignq $1,%ymm9,%ymm10,%ymm9 - valignq $1,%ymm10,%ymm11,%ymm10 - valignq $1,%ymm11,%ymm12,%ymm11 - valignq $1,%ymm12,%ymm0,%ymm12 - - vmovq %xmm3,%r13 - addq %r13,%r9 - - vpmadd52huq 0(%rsi),%ymm1,%ymm3 - vpmadd52huq 32(%rsi),%ymm1,%ymm4 - vpmadd52huq 64(%rsi),%ymm1,%ymm5 - vpmadd52huq 96(%rsi),%ymm1,%ymm6 - vpmadd52huq 128(%rsi),%ymm1,%ymm7 - vpmadd52huq 160(%rsi),%ymm1,%ymm8 - vpmadd52huq 192(%rsi),%ymm1,%ymm9 - vpmadd52huq 224(%rsi),%ymm1,%ymm10 - vpmadd52huq 256(%rsi),%ymm1,%ymm11 - vpmadd52huq 288(%rsi),%ymm1,%ymm12 - - vpmadd52huq 0(%rcx),%ymm2,%ymm3 - vpmadd52huq 32(%rcx),%ymm2,%ymm4 - vpmadd52huq 64(%rcx),%ymm2,%ymm5 - vpmadd52huq 96(%rcx),%ymm2,%ymm6 - vpmadd52huq 128(%rcx),%ymm2,%ymm7 - vpmadd52huq 160(%rcx),%ymm2,%ymm8 - vpmadd52huq 192(%rcx),%ymm2,%ymm9 - vpmadd52huq 224(%rcx),%ymm2,%ymm10 - vpmadd52huq 256(%rcx),%ymm2,%ymm11 - vpmadd52huq 288(%rcx),%ymm2,%ymm12 - movq 320(%r11),%r13 - - vpbroadcastq %r13,%ymm1 - movq 320(%rsi),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - movq %r12,%r10 - adcq $0,%r10 - - movq 8(%r8),%r13 - imulq %r15,%r13 - andq %rax,%r13 - - vpbroadcastq %r13,%ymm2 - movq 320(%rcx),%rdx - mulxq %r13,%r13,%r12 - addq %r13,%r15 - adcq %r12,%r10 - - shrq $52,%r15 - salq $12,%r10 - orq %r10,%r15 - - vpmadd52luq 320(%rsi),%ymm1,%ymm13 - vpmadd52luq 352(%rsi),%ymm1,%ymm14 - vpmadd52luq 384(%rsi),%ymm1,%ymm15 - vpmadd52luq 416(%rsi),%ymm1,%ymm16 - vpmadd52luq 448(%rsi),%ymm1,%ymm17 - vpmadd52luq 480(%rsi),%ymm1,%ymm18 - vpmadd52luq 512(%rsi),%ymm1,%ymm19 - vpmadd52luq 544(%rsi),%ymm1,%ymm20 - vpmadd52luq 576(%rsi),%ymm1,%ymm21 - vpmadd52luq 608(%rsi),%ymm1,%ymm22 - - vpmadd52luq 320(%rcx),%ymm2,%ymm13 - vpmadd52luq 352(%rcx),%ymm2,%ymm14 - vpmadd52luq 384(%rcx),%ymm2,%ymm15 - vpmadd52luq 416(%rcx),%ymm2,%ymm16 - vpmadd52luq 448(%rcx),%ymm2,%ymm17 - vpmadd52luq 480(%rcx),%ymm2,%ymm18 - vpmadd52luq 512(%rcx),%ymm2,%ymm19 - vpmadd52luq 544(%rcx),%ymm2,%ymm20 - vpmadd52luq 576(%rcx),%ymm2,%ymm21 - vpmadd52luq 608(%rcx),%ymm2,%ymm22 - - - valignq $1,%ymm13,%ymm14,%ymm13 - valignq $1,%ymm14,%ymm15,%ymm14 - valignq $1,%ymm15,%ymm16,%ymm15 - valignq $1,%ymm16,%ymm17,%ymm16 - valignq $1,%ymm17,%ymm18,%ymm17 - valignq $1,%ymm18,%ymm19,%ymm18 - valignq $1,%ymm19,%ymm20,%ymm19 - valignq $1,%ymm20,%ymm21,%ymm20 - valignq $1,%ymm21,%ymm22,%ymm21 - valignq $1,%ymm22,%ymm0,%ymm22 - - vmovq %xmm13,%r13 - addq %r13,%r15 - - vpmadd52huq 320(%rsi),%ymm1,%ymm13 - vpmadd52huq 352(%rsi),%ymm1,%ymm14 - vpmadd52huq 384(%rsi),%ymm1,%ymm15 - vpmadd52huq 416(%rsi),%ymm1,%ymm16 - vpmadd52huq 448(%rsi),%ymm1,%ymm17 - vpmadd52huq 480(%rsi),%ymm1,%ymm18 - vpmadd52huq 512(%rsi),%ymm1,%ymm19 - vpmadd52huq 544(%rsi),%ymm1,%ymm20 - vpmadd52huq 576(%rsi),%ymm1,%ymm21 - vpmadd52huq 608(%rsi),%ymm1,%ymm22 - - vpmadd52huq 320(%rcx),%ymm2,%ymm13 - vpmadd52huq 352(%rcx),%ymm2,%ymm14 - vpmadd52huq 384(%rcx),%ymm2,%ymm15 - vpmadd52huq 416(%rcx),%ymm2,%ymm16 - vpmadd52huq 448(%rcx),%ymm2,%ymm17 - vpmadd52huq 480(%rcx),%ymm2,%ymm18 - vpmadd52huq 512(%rcx),%ymm2,%ymm19 - vpmadd52huq 544(%rcx),%ymm2,%ymm20 - vpmadd52huq 576(%rcx),%ymm2,%ymm21 - vpmadd52huq 608(%rcx),%ymm2,%ymm22 - leaq 8(%r11),%r11 - decl %ebx - jne .Lloop40 - - vpbroadcastq %r9,%ymm0 - vpblendd $3,%ymm0,%ymm3,%ymm3 - - - - vpsrlq $52,%ymm3,%ymm0 - vpsrlq $52,%ymm4,%ymm1 - vpsrlq $52,%ymm5,%ymm2 - vpsrlq $52,%ymm6,%ymm23 - vpsrlq $52,%ymm7,%ymm24 - vpsrlq $52,%ymm8,%ymm25 - vpsrlq $52,%ymm9,%ymm26 - vpsrlq $52,%ymm10,%ymm27 - vpsrlq $52,%ymm11,%ymm28 - vpsrlq $52,%ymm12,%ymm29 - - - valignq $3,%ymm28,%ymm29,%ymm29 - valignq $3,%ymm27,%ymm28,%ymm28 - valignq $3,%ymm26,%ymm27,%ymm27 - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm24,%ymm25,%ymm25 - valignq $3,%ymm23,%ymm24,%ymm24 - valignq $3,%ymm2,%ymm23,%ymm23 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm5,%ymm5 - vpandq .Lmask52x4(%rip),%ymm6,%ymm6 - vpandq .Lmask52x4(%rip),%ymm7,%ymm7 - vpandq .Lmask52x4(%rip),%ymm8,%ymm8 - vpandq .Lmask52x4(%rip),%ymm9,%ymm9 - vpandq .Lmask52x4(%rip),%ymm10,%ymm10 - vpandq .Lmask52x4(%rip),%ymm11,%ymm11 - vpandq .Lmask52x4(%rip),%ymm12,%ymm12 - - - vpaddq %ymm0,%ymm3,%ymm3 - vpaddq %ymm1,%ymm4,%ymm4 - vpaddq %ymm2,%ymm5,%ymm5 - vpaddq %ymm23,%ymm6,%ymm6 - vpaddq %ymm24,%ymm7,%ymm7 - vpaddq %ymm25,%ymm8,%ymm8 - vpaddq %ymm26,%ymm9,%ymm9 - vpaddq %ymm27,%ymm10,%ymm10 - vpaddq %ymm28,%ymm11,%ymm11 - vpaddq %ymm29,%ymm12,%ymm12 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm5,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm7,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm8,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm9,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm10,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm11,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm12,%k2 - kmovb %k1,%r10d - kmovb %k2,%r9d - shlb $4,%r9b - orb %r9b,%r10b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - adcb %r10b,%r10b - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm3,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm4,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm5,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm6,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm7,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm8,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm9,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm10,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm11,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm12,%k2 - kmovb %k1,%ebx - kmovb %k2,%eax - shlb $4,%al - orb %al,%bl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - adcb %bl,%r10b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - xorb %bl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq .Lmask52x4(%rip),%ymm3,%ymm3{%k1} - vpsubq .Lmask52x4(%rip),%ymm4,%ymm4{%k2} - vpsubq .Lmask52x4(%rip),%ymm5,%ymm5{%k3} - vpsubq .Lmask52x4(%rip),%ymm6,%ymm6{%k4} - vpsubq .Lmask52x4(%rip),%ymm7,%ymm7{%k5} - vpsubq .Lmask52x4(%rip),%ymm8,%ymm8{%k6} - vpsubq .Lmask52x4(%rip),%ymm9,%ymm9{%k7} - - vpandq .Lmask52x4(%rip),%ymm3,%ymm3 - vpandq .Lmask52x4(%rip),%ymm4,%ymm4 - vpandq .Lmask52x4(%rip),%ymm5,%ymm5 - vpandq .Lmask52x4(%rip),%ymm6,%ymm6 - vpandq .Lmask52x4(%rip),%ymm7,%ymm7 - vpandq .Lmask52x4(%rip),%ymm8,%ymm8 - vpandq .Lmask52x4(%rip),%ymm9,%ymm9 - - shrb $4,%r11b - kmovb %r11d,%k1 - kmovb %r10d,%k2 - shrb $4,%r10b - kmovb %r10d,%k3 - - vpsubq .Lmask52x4(%rip),%ymm10,%ymm10{%k1} - vpsubq .Lmask52x4(%rip),%ymm11,%ymm11{%k2} - vpsubq .Lmask52x4(%rip),%ymm12,%ymm12{%k3} - - vpandq .Lmask52x4(%rip),%ymm10,%ymm10 - vpandq .Lmask52x4(%rip),%ymm11,%ymm11 - vpandq .Lmask52x4(%rip),%ymm12,%ymm12 - - vpbroadcastq %r15,%ymm0 - vpblendd $3,%ymm0,%ymm13,%ymm13 - - - - vpsrlq $52,%ymm13,%ymm0 - vpsrlq $52,%ymm14,%ymm1 - vpsrlq $52,%ymm15,%ymm2 - vpsrlq $52,%ymm16,%ymm23 - vpsrlq $52,%ymm17,%ymm24 - vpsrlq $52,%ymm18,%ymm25 - vpsrlq $52,%ymm19,%ymm26 - vpsrlq $52,%ymm20,%ymm27 - vpsrlq $52,%ymm21,%ymm28 - vpsrlq $52,%ymm22,%ymm29 - - - valignq $3,%ymm28,%ymm29,%ymm29 - valignq $3,%ymm27,%ymm28,%ymm28 - valignq $3,%ymm26,%ymm27,%ymm27 - valignq $3,%ymm25,%ymm26,%ymm26 - valignq $3,%ymm24,%ymm25,%ymm25 - valignq $3,%ymm23,%ymm24,%ymm24 - valignq $3,%ymm2,%ymm23,%ymm23 - valignq $3,%ymm1,%ymm2,%ymm2 - valignq $3,%ymm0,%ymm1,%ymm1 - valignq $3,.Lzeros(%rip),%ymm0,%ymm0 - - - vpandq .Lmask52x4(%rip),%ymm13,%ymm13 - vpandq .Lmask52x4(%rip),%ymm14,%ymm14 - vpandq .Lmask52x4(%rip),%ymm15,%ymm15 - vpandq .Lmask52x4(%rip),%ymm16,%ymm16 - vpandq .Lmask52x4(%rip),%ymm17,%ymm17 - vpandq .Lmask52x4(%rip),%ymm18,%ymm18 - vpandq .Lmask52x4(%rip),%ymm19,%ymm19 - vpandq .Lmask52x4(%rip),%ymm20,%ymm20 - vpandq .Lmask52x4(%rip),%ymm21,%ymm21 - vpandq .Lmask52x4(%rip),%ymm22,%ymm22 - - - vpaddq %ymm0,%ymm13,%ymm13 - vpaddq %ymm1,%ymm14,%ymm14 - vpaddq %ymm2,%ymm15,%ymm15 - vpaddq %ymm23,%ymm16,%ymm16 - vpaddq %ymm24,%ymm17,%ymm17 - vpaddq %ymm25,%ymm18,%ymm18 - vpaddq %ymm26,%ymm19,%ymm19 - vpaddq %ymm27,%ymm20,%ymm20 - vpaddq %ymm28,%ymm21,%ymm21 - vpaddq %ymm29,%ymm22,%ymm22 - - - - vpcmpuq $6,.Lmask52x4(%rip),%ymm13,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm14,%k2 - kmovb %k1,%r14d - kmovb %k2,%r13d - shlb $4,%r13b - orb %r13b,%r14b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm15,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm16,%k2 - kmovb %k1,%r13d - kmovb %k2,%r12d - shlb $4,%r12b - orb %r12b,%r13b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm17,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm18,%k2 - kmovb %k1,%r12d - kmovb %k2,%r11d - shlb $4,%r11b - orb %r11b,%r12b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm19,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm20,%k2 - kmovb %k1,%r11d - kmovb %k2,%r10d - shlb $4,%r10b - orb %r10b,%r11b - - vpcmpuq $6,.Lmask52x4(%rip),%ymm21,%k1 - vpcmpuq $6,.Lmask52x4(%rip),%ymm22,%k2 - kmovb %k1,%r10d - kmovb %k2,%r9d - shlb $4,%r9b - orb %r9b,%r10b - - addb %r14b,%r14b - adcb %r13b,%r13b - adcb %r12b,%r12b - adcb %r11b,%r11b - adcb %r10b,%r10b - - - vpcmpuq $0,.Lmask52x4(%rip),%ymm13,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm14,%k2 - kmovb %k1,%r9d - kmovb %k2,%r8d - shlb $4,%r8b - orb %r8b,%r9b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm15,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm16,%k2 - kmovb %k1,%r8d - kmovb %k2,%edx - shlb $4,%dl - orb %dl,%r8b - - vpcmpuq $0,.Lmask52x4(%rip),%ymm17,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm18,%k2 - kmovb %k1,%edx - kmovb %k2,%ecx - shlb $4,%cl - orb %cl,%dl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm19,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm20,%k2 - kmovb %k1,%ecx - kmovb %k2,%ebx - shlb $4,%bl - orb %bl,%cl - - vpcmpuq $0,.Lmask52x4(%rip),%ymm21,%k1 - vpcmpuq $0,.Lmask52x4(%rip),%ymm22,%k2 - kmovb %k1,%ebx - kmovb %k2,%eax - shlb $4,%al - orb %al,%bl - - addb %r9b,%r14b - adcb %r8b,%r13b - adcb %dl,%r12b - adcb %cl,%r11b - adcb %bl,%r10b - - xorb %r9b,%r14b - xorb %r8b,%r13b - xorb %dl,%r12b - xorb %cl,%r11b - xorb %bl,%r10b - - kmovb %r14d,%k1 - shrb $4,%r14b - kmovb %r14d,%k2 - kmovb %r13d,%k3 - shrb $4,%r13b - kmovb %r13d,%k4 - kmovb %r12d,%k5 - shrb $4,%r12b - kmovb %r12d,%k6 - kmovb %r11d,%k7 - - vpsubq .Lmask52x4(%rip),%ymm13,%ymm13{%k1} - vpsubq .Lmask52x4(%rip),%ymm14,%ymm14{%k2} - vpsubq .Lmask52x4(%rip),%ymm15,%ymm15{%k3} - vpsubq .Lmask52x4(%rip),%ymm16,%ymm16{%k4} - vpsubq .Lmask52x4(%rip),%ymm17,%ymm17{%k5} - vpsubq .Lmask52x4(%rip),%ymm18,%ymm18{%k6} - vpsubq .Lmask52x4(%rip),%ymm19,%ymm19{%k7} - - vpandq .Lmask52x4(%rip),%ymm13,%ymm13 - vpandq .Lmask52x4(%rip),%ymm14,%ymm14 - vpandq .Lmask52x4(%rip),%ymm15,%ymm15 - vpandq .Lmask52x4(%rip),%ymm16,%ymm16 - vpandq .Lmask52x4(%rip),%ymm17,%ymm17 - vpandq .Lmask52x4(%rip),%ymm18,%ymm18 - vpandq .Lmask52x4(%rip),%ymm19,%ymm19 - - shrb $4,%r11b - kmovb %r11d,%k1 - kmovb %r10d,%k2 - shrb $4,%r10b - kmovb %r10d,%k3 - - vpsubq .Lmask52x4(%rip),%ymm20,%ymm20{%k1} - vpsubq .Lmask52x4(%rip),%ymm21,%ymm21{%k2} - vpsubq .Lmask52x4(%rip),%ymm22,%ymm22{%k3} - - vpandq .Lmask52x4(%rip),%ymm20,%ymm20 - vpandq .Lmask52x4(%rip),%ymm21,%ymm21 - vpandq .Lmask52x4(%rip),%ymm22,%ymm22 - - vmovdqu64 %ymm3,0(%rdi) - vmovdqu64 %ymm4,32(%rdi) - vmovdqu64 %ymm5,64(%rdi) - vmovdqu64 %ymm6,96(%rdi) - vmovdqu64 %ymm7,128(%rdi) - vmovdqu64 %ymm8,160(%rdi) - vmovdqu64 %ymm9,192(%rdi) - vmovdqu64 %ymm10,224(%rdi) - vmovdqu64 %ymm11,256(%rdi) - vmovdqu64 %ymm12,288(%rdi) - - vmovdqu64 %ymm13,320(%rdi) - vmovdqu64 %ymm14,352(%rdi) - vmovdqu64 %ymm15,384(%rdi) - vmovdqu64 %ymm16,416(%rdi) - vmovdqu64 %ymm17,448(%rdi) - vmovdqu64 %ymm18,480(%rdi) - vmovdqu64 %ymm19,512(%rdi) - vmovdqu64 %ymm20,544(%rdi) - vmovdqu64 %ymm21,576(%rdi) - vmovdqu64 %ymm22,608(%rdi) - - vzeroupper - leaq (%rsp),%rax -.cfi_def_cfa_register %rax - movq 0(%rax),%r15 -.cfi_restore %r15 - movq 8(%rax),%r14 -.cfi_restore %r14 - movq 16(%rax),%r13 -.cfi_restore %r13 - movq 24(%rax),%r12 -.cfi_restore %r12 - movq 32(%rax),%rbp -.cfi_restore %rbp - movq 40(%rax),%rbx -.cfi_restore %rbx - leaq 48(%rax),%rsp -.cfi_def_cfa %rsp,8 -.Lossl_rsaz_amm52x40_x2_ifma256_epilogue: - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_rsaz_amm52x40_x2_ifma256, .-ossl_rsaz_amm52x40_x2_ifma256 -.text - -.align 32 -.globl ossl_extract_multiplier_2x40_win5 -.type ossl_extract_multiplier_2x40_win5,@function -ossl_extract_multiplier_2x40_win5: -.cfi_startproc -.byte 243,15,30,250 - vmovdqa64 .Lones(%rip),%ymm24 - vpbroadcastq %rdx,%ymm22 - vpbroadcastq %rcx,%ymm23 - leaq 20480(%rsi),%rax - - - movq %rsi,%r10 - - - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %ymm0,%ymm1 - vmovdqa64 %ymm0,%ymm2 - vmovdqa64 %ymm0,%ymm3 - vmovdqa64 %ymm0,%ymm4 - vmovdqa64 %ymm0,%ymm5 - vmovdqa64 %ymm0,%ymm16 - vmovdqa64 %ymm0,%ymm17 - vmovdqa64 %ymm0,%ymm18 - vmovdqa64 %ymm0,%ymm19 - vpxorq %ymm21,%ymm21,%ymm21 -.align 32 -.Lloop_0: - vpcmpq $0,%ymm21,%ymm22,%k1 - vmovdqu64 0(%rsi),%ymm20 - vpblendmq %ymm20,%ymm0,%ymm0{%k1} - vmovdqu64 32(%rsi),%ymm20 - vpblendmq %ymm20,%ymm1,%ymm1{%k1} - vmovdqu64 64(%rsi),%ymm20 - vpblendmq %ymm20,%ymm2,%ymm2{%k1} - vmovdqu64 96(%rsi),%ymm20 - vpblendmq %ymm20,%ymm3,%ymm3{%k1} - vmovdqu64 128(%rsi),%ymm20 - vpblendmq %ymm20,%ymm4,%ymm4{%k1} - vmovdqu64 160(%rsi),%ymm20 - vpblendmq %ymm20,%ymm5,%ymm5{%k1} - vmovdqu64 192(%rsi),%ymm20 - vpblendmq %ymm20,%ymm16,%ymm16{%k1} - vmovdqu64 224(%rsi),%ymm20 - vpblendmq %ymm20,%ymm17,%ymm17{%k1} - vmovdqu64 256(%rsi),%ymm20 - vpblendmq %ymm20,%ymm18,%ymm18{%k1} - vmovdqu64 288(%rsi),%ymm20 - vpblendmq %ymm20,%ymm19,%ymm19{%k1} - vpaddq %ymm24,%ymm21,%ymm21 - addq $640,%rsi - cmpq %rsi,%rax - jne .Lloop_0 - vmovdqu64 %ymm0,0(%rdi) - vmovdqu64 %ymm1,32(%rdi) - vmovdqu64 %ymm2,64(%rdi) - vmovdqu64 %ymm3,96(%rdi) - vmovdqu64 %ymm4,128(%rdi) - vmovdqu64 %ymm5,160(%rdi) - vmovdqu64 %ymm16,192(%rdi) - vmovdqu64 %ymm17,224(%rdi) - vmovdqu64 %ymm18,256(%rdi) - vmovdqu64 %ymm19,288(%rdi) - movq %r10,%rsi - vpxorq %ymm21,%ymm21,%ymm21 -.align 32 -.Lloop_320: - vpcmpq $0,%ymm21,%ymm23,%k1 - vmovdqu64 320(%rsi),%ymm20 - vpblendmq %ymm20,%ymm0,%ymm0{%k1} - vmovdqu64 352(%rsi),%ymm20 - vpblendmq %ymm20,%ymm1,%ymm1{%k1} - vmovdqu64 384(%rsi),%ymm20 - vpblendmq %ymm20,%ymm2,%ymm2{%k1} - vmovdqu64 416(%rsi),%ymm20 - vpblendmq %ymm20,%ymm3,%ymm3{%k1} - vmovdqu64 448(%rsi),%ymm20 - vpblendmq %ymm20,%ymm4,%ymm4{%k1} - vmovdqu64 480(%rsi),%ymm20 - vpblendmq %ymm20,%ymm5,%ymm5{%k1} - vmovdqu64 512(%rsi),%ymm20 - vpblendmq %ymm20,%ymm16,%ymm16{%k1} - vmovdqu64 544(%rsi),%ymm20 - vpblendmq %ymm20,%ymm17,%ymm17{%k1} - vmovdqu64 576(%rsi),%ymm20 - vpblendmq %ymm20,%ymm18,%ymm18{%k1} - vmovdqu64 608(%rsi),%ymm20 - vpblendmq %ymm20,%ymm19,%ymm19{%k1} - vpaddq %ymm24,%ymm21,%ymm21 - addq $640,%rsi - cmpq %rsi,%rax - jne .Lloop_320 - vmovdqu64 %ymm0,320(%rdi) - vmovdqu64 %ymm1,352(%rdi) - vmovdqu64 %ymm2,384(%rdi) - vmovdqu64 %ymm3,416(%rdi) - vmovdqu64 %ymm4,448(%rdi) - vmovdqu64 %ymm5,480(%rdi) - vmovdqu64 %ymm16,512(%rdi) - vmovdqu64 %ymm17,544(%rdi) - vmovdqu64 %ymm18,576(%rdi) - vmovdqu64 %ymm19,608(%rdi) - - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_extract_multiplier_2x40_win5, .-ossl_extract_multiplier_2x40_win5 -.data -.align 32 -.Lones: -.quad 1,1,1,1 -.Lzeros: -.quad 0,0,0,0 - .section ".note.gnu.property", "a" - .p2align 3 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - # "GNU" encoded with .byte, since .asciz isn't supported - # on Solaris. - .byte 0x47 - .byte 0x4e - .byte 0x55 - .byte 0 -1: - .p2align 3 - .long 0xc0000002 - .long 3f - 2f -2: - .long 3 -3: - .p2align 3 -4: diff --git a/openssl/src/crypto/bn/gen/linux_x64/rsaz-avx512.s b/openssl/src/crypto/bn/gen/linux_x64/rsaz-avx512.s new file mode 100644 index 000000000..9bbe8ebf7 --- /dev/null +++ b/openssl/src/crypto/bn/gen/linux_x64/rsaz-avx512.s @@ -0,0 +1,901 @@ + +.globl ossl_rsaz_avx512ifma_eligible +.type ossl_rsaz_avx512ifma_eligible,@function +.align 32 +ossl_rsaz_avx512ifma_eligible: + movl OPENSSL_ia32cap_P+8(%rip),%ecx + xorl %eax,%eax + andl $2149777408,%ecx + cmpl $2149777408,%ecx + cmovel %ecx,%eax + .byte 0xf3,0xc3 +.size ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible +.text + +.globl ossl_rsaz_amm52x20_x1_256 +.type ossl_rsaz_amm52x20_x1_256,@function +.align 32 +ossl_rsaz_amm52x20_x1_256: +.cfi_startproc +.byte 243,15,30,250 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-16 + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 +.Lrsaz_amm52x20_x1_256_body: + + + vpxord %ymm0,%ymm0,%ymm0 + vmovdqa64 %ymm0,%ymm1 + vmovdqa64 %ymm0,%ymm16 + vmovdqa64 %ymm0,%ymm17 + vmovdqa64 %ymm0,%ymm18 + vmovdqa64 %ymm0,%ymm19 + + xorl %r9d,%r9d + + movq %rdx,%r11 + movq $0xfffffffffffff,%rax + + + movl $5,%ebx + +.align 32 +.Lloop5: + movq 0(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq %r8,%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + movq 8(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq %r8,%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + movq 16(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq %r8,%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + movq 24(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq %r8,%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + leaq 32(%r11),%r11 + decl %ebx + jne .Lloop5 + + vmovdqa64 .Lmask52x4(%rip),%ymm4 + + vpbroadcastq %r9,%ymm3 + vpblendd $3,%ymm3,%ymm1,%ymm1 + + + + vpsrlq $52,%ymm1,%ymm24 + vpsrlq $52,%ymm16,%ymm25 + vpsrlq $52,%ymm17,%ymm26 + vpsrlq $52,%ymm18,%ymm27 + vpsrlq $52,%ymm19,%ymm28 + + + valignq $3,%ymm27,%ymm28,%ymm28 + valignq $3,%ymm26,%ymm27,%ymm27 + valignq $3,%ymm25,%ymm26,%ymm26 + valignq $3,%ymm24,%ymm25,%ymm25 + valignq $3,%ymm0,%ymm24,%ymm24 + + + vpandq %ymm4,%ymm1,%ymm1 + vpandq %ymm4,%ymm16,%ymm16 + vpandq %ymm4,%ymm17,%ymm17 + vpandq %ymm4,%ymm18,%ymm18 + vpandq %ymm4,%ymm19,%ymm19 + + + vpaddq %ymm24,%ymm1,%ymm1 + vpaddq %ymm25,%ymm16,%ymm16 + vpaddq %ymm26,%ymm17,%ymm17 + vpaddq %ymm27,%ymm18,%ymm18 + vpaddq %ymm28,%ymm19,%ymm19 + + + + vpcmpuq $1,%ymm1,%ymm4,%k1 + vpcmpuq $1,%ymm16,%ymm4,%k2 + vpcmpuq $1,%ymm17,%ymm4,%k3 + vpcmpuq $1,%ymm18,%ymm4,%k4 + vpcmpuq $1,%ymm19,%ymm4,%k5 + kmovb %k1,%r14d + kmovb %k2,%r13d + kmovb %k3,%r12d + kmovb %k4,%r11d + kmovb %k5,%r10d + + + vpcmpuq $0,%ymm1,%ymm4,%k1 + vpcmpuq $0,%ymm16,%ymm4,%k2 + vpcmpuq $0,%ymm17,%ymm4,%k3 + vpcmpuq $0,%ymm18,%ymm4,%k4 + vpcmpuq $0,%ymm19,%ymm4,%k5 + kmovb %k1,%r9d + kmovb %k2,%r8d + kmovb %k3,%ebx + kmovb %k4,%ecx + kmovb %k5,%edx + + + + shlb $4,%r13b + orb %r13b,%r14b + shlb $4,%r11b + orb %r11b,%r12b + + addb %r14b,%r14b + adcb %r12b,%r12b + adcb %r10b,%r10b + + shlb $4,%r8b + orb %r8b,%r9b + shlb $4,%cl + orb %cl,%bl + + addb %r9b,%r14b + adcb %bl,%r12b + adcb %dl,%r10b + + xorb %r9b,%r14b + xorb %bl,%r12b + xorb %dl,%r10b + + kmovb %r14d,%k1 + shrb $4,%r14b + kmovb %r14d,%k2 + kmovb %r12d,%k3 + shrb $4,%r12b + kmovb %r12d,%k4 + kmovb %r10d,%k5 + + + vpsubq %ymm4,%ymm1,%ymm1{%k1} + vpsubq %ymm4,%ymm16,%ymm16{%k2} + vpsubq %ymm4,%ymm17,%ymm17{%k3} + vpsubq %ymm4,%ymm18,%ymm18{%k4} + vpsubq %ymm4,%ymm19,%ymm19{%k5} + + vpandq %ymm4,%ymm1,%ymm1 + vpandq %ymm4,%ymm16,%ymm16 + vpandq %ymm4,%ymm17,%ymm17 + vpandq %ymm4,%ymm18,%ymm18 + vpandq %ymm4,%ymm19,%ymm19 + + vmovdqu64 %ymm1,(%rdi) + vmovdqu64 %ymm16,32(%rdi) + vmovdqu64 %ymm17,64(%rdi) + vmovdqu64 %ymm18,96(%rdi) + vmovdqu64 %ymm19,128(%rdi) + + vzeroupper + movq 0(%rsp),%r15 +.cfi_restore %r15 + movq 8(%rsp),%r14 +.cfi_restore %r14 + movq 16(%rsp),%r13 +.cfi_restore %r13 + movq 24(%rsp),%r12 +.cfi_restore %r12 + movq 32(%rsp),%rbp +.cfi_restore %rbp + movq 40(%rsp),%rbx +.cfi_restore %rbx + leaq 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lrsaz_amm52x20_x1_256_epilogue: + .byte 0xf3,0xc3 +.cfi_endproc +.size ossl_rsaz_amm52x20_x1_256, .-ossl_rsaz_amm52x20_x1_256 +.data +.align 32 +.Lmask52x4: +.quad 0xfffffffffffff +.quad 0xfffffffffffff +.quad 0xfffffffffffff +.quad 0xfffffffffffff +.text + +.globl ossl_rsaz_amm52x20_x2_256 +.type ossl_rsaz_amm52x20_x2_256,@function +.align 32 +ossl_rsaz_amm52x20_x2_256: +.cfi_startproc +.byte 243,15,30,250 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-16 + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 +.Lrsaz_amm52x20_x2_256_body: + + + vpxord %ymm0,%ymm0,%ymm0 + vmovdqa64 %ymm0,%ymm1 + vmovdqa64 %ymm0,%ymm16 + vmovdqa64 %ymm0,%ymm17 + vmovdqa64 %ymm0,%ymm18 + vmovdqa64 %ymm0,%ymm19 + vmovdqa64 %ymm0,%ymm2 + vmovdqa64 %ymm0,%ymm20 + vmovdqa64 %ymm0,%ymm21 + vmovdqa64 %ymm0,%ymm22 + vmovdqa64 %ymm0,%ymm23 + + xorl %r9d,%r9d + xorl %r15d,%r15d + + movq %rdx,%r11 + movq $0xfffffffffffff,%rax + + movl $20,%ebx + +.align 32 +.Lloop20: + movq 0(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 0(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + movq %r12,%r10 + adcq $0,%r10 + + movq (%r8),%r13 + imulq %r9,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 0(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r9 + adcq %r12,%r10 + + shrq $52,%r9 + salq $12,%r10 + orq %r10,%r9 + + vpmadd52luq 0(%rsi),%ymm3,%ymm1 + vpmadd52luq 32(%rsi),%ymm3,%ymm16 + vpmadd52luq 64(%rsi),%ymm3,%ymm17 + vpmadd52luq 96(%rsi),%ymm3,%ymm18 + vpmadd52luq 128(%rsi),%ymm3,%ymm19 + + vpmadd52luq 0(%rcx),%ymm4,%ymm1 + vpmadd52luq 32(%rcx),%ymm4,%ymm16 + vpmadd52luq 64(%rcx),%ymm4,%ymm17 + vpmadd52luq 96(%rcx),%ymm4,%ymm18 + vpmadd52luq 128(%rcx),%ymm4,%ymm19 + + + valignq $1,%ymm1,%ymm16,%ymm1 + valignq $1,%ymm16,%ymm17,%ymm16 + valignq $1,%ymm17,%ymm18,%ymm17 + valignq $1,%ymm18,%ymm19,%ymm18 + valignq $1,%ymm19,%ymm0,%ymm19 + + vmovq %xmm1,%r13 + addq %r13,%r9 + + vpmadd52huq 0(%rsi),%ymm3,%ymm1 + vpmadd52huq 32(%rsi),%ymm3,%ymm16 + vpmadd52huq 64(%rsi),%ymm3,%ymm17 + vpmadd52huq 96(%rsi),%ymm3,%ymm18 + vpmadd52huq 128(%rsi),%ymm3,%ymm19 + + vpmadd52huq 0(%rcx),%ymm4,%ymm1 + vpmadd52huq 32(%rcx),%ymm4,%ymm16 + vpmadd52huq 64(%rcx),%ymm4,%ymm17 + vpmadd52huq 96(%rcx),%ymm4,%ymm18 + vpmadd52huq 128(%rcx),%ymm4,%ymm19 + movq 160(%r11),%r13 + + vpbroadcastq %r13,%ymm3 + movq 160(%rsi),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r15 + movq %r12,%r10 + adcq $0,%r10 + + movq 8(%r8),%r13 + imulq %r15,%r13 + andq %rax,%r13 + + vpbroadcastq %r13,%ymm4 + movq 160(%rcx),%rdx + mulxq %r13,%r13,%r12 + addq %r13,%r15 + adcq %r12,%r10 + + shrq $52,%r15 + salq $12,%r10 + orq %r10,%r15 + + vpmadd52luq 160(%rsi),%ymm3,%ymm2 + vpmadd52luq 192(%rsi),%ymm3,%ymm20 + vpmadd52luq 224(%rsi),%ymm3,%ymm21 + vpmadd52luq 256(%rsi),%ymm3,%ymm22 + vpmadd52luq 288(%rsi),%ymm3,%ymm23 + + vpmadd52luq 160(%rcx),%ymm4,%ymm2 + vpmadd52luq 192(%rcx),%ymm4,%ymm20 + vpmadd52luq 224(%rcx),%ymm4,%ymm21 + vpmadd52luq 256(%rcx),%ymm4,%ymm22 + vpmadd52luq 288(%rcx),%ymm4,%ymm23 + + + valignq $1,%ymm2,%ymm20,%ymm2 + valignq $1,%ymm20,%ymm21,%ymm20 + valignq $1,%ymm21,%ymm22,%ymm21 + valignq $1,%ymm22,%ymm23,%ymm22 + valignq $1,%ymm23,%ymm0,%ymm23 + + vmovq %xmm2,%r13 + addq %r13,%r15 + + vpmadd52huq 160(%rsi),%ymm3,%ymm2 + vpmadd52huq 192(%rsi),%ymm3,%ymm20 + vpmadd52huq 224(%rsi),%ymm3,%ymm21 + vpmadd52huq 256(%rsi),%ymm3,%ymm22 + vpmadd52huq 288(%rsi),%ymm3,%ymm23 + + vpmadd52huq 160(%rcx),%ymm4,%ymm2 + vpmadd52huq 192(%rcx),%ymm4,%ymm20 + vpmadd52huq 224(%rcx),%ymm4,%ymm21 + vpmadd52huq 256(%rcx),%ymm4,%ymm22 + vpmadd52huq 288(%rcx),%ymm4,%ymm23 + leaq 8(%r11),%r11 + decl %ebx + jne .Lloop20 + + vmovdqa64 .Lmask52x4(%rip),%ymm4 + + vpbroadcastq %r9,%ymm3 + vpblendd $3,%ymm3,%ymm1,%ymm1 + + + + vpsrlq $52,%ymm1,%ymm24 + vpsrlq $52,%ymm16,%ymm25 + vpsrlq $52,%ymm17,%ymm26 + vpsrlq $52,%ymm18,%ymm27 + vpsrlq $52,%ymm19,%ymm28 + + + valignq $3,%ymm27,%ymm28,%ymm28 + valignq $3,%ymm26,%ymm27,%ymm27 + valignq $3,%ymm25,%ymm26,%ymm26 + valignq $3,%ymm24,%ymm25,%ymm25 + valignq $3,%ymm0,%ymm24,%ymm24 + + + vpandq %ymm4,%ymm1,%ymm1 + vpandq %ymm4,%ymm16,%ymm16 + vpandq %ymm4,%ymm17,%ymm17 + vpandq %ymm4,%ymm18,%ymm18 + vpandq %ymm4,%ymm19,%ymm19 + + + vpaddq %ymm24,%ymm1,%ymm1 + vpaddq %ymm25,%ymm16,%ymm16 + vpaddq %ymm26,%ymm17,%ymm17 + vpaddq %ymm27,%ymm18,%ymm18 + vpaddq %ymm28,%ymm19,%ymm19 + + + + vpcmpuq $1,%ymm1,%ymm4,%k1 + vpcmpuq $1,%ymm16,%ymm4,%k2 + vpcmpuq $1,%ymm17,%ymm4,%k3 + vpcmpuq $1,%ymm18,%ymm4,%k4 + vpcmpuq $1,%ymm19,%ymm4,%k5 + kmovb %k1,%r14d + kmovb %k2,%r13d + kmovb %k3,%r12d + kmovb %k4,%r11d + kmovb %k5,%r10d + + + vpcmpuq $0,%ymm1,%ymm4,%k1 + vpcmpuq $0,%ymm16,%ymm4,%k2 + vpcmpuq $0,%ymm17,%ymm4,%k3 + vpcmpuq $0,%ymm18,%ymm4,%k4 + vpcmpuq $0,%ymm19,%ymm4,%k5 + kmovb %k1,%r9d + kmovb %k2,%r8d + kmovb %k3,%ebx + kmovb %k4,%ecx + kmovb %k5,%edx + + + + shlb $4,%r13b + orb %r13b,%r14b + shlb $4,%r11b + orb %r11b,%r12b + + addb %r14b,%r14b + adcb %r12b,%r12b + adcb %r10b,%r10b + + shlb $4,%r8b + orb %r8b,%r9b + shlb $4,%cl + orb %cl,%bl + + addb %r9b,%r14b + adcb %bl,%r12b + adcb %dl,%r10b + + xorb %r9b,%r14b + xorb %bl,%r12b + xorb %dl,%r10b + + kmovb %r14d,%k1 + shrb $4,%r14b + kmovb %r14d,%k2 + kmovb %r12d,%k3 + shrb $4,%r12b + kmovb %r12d,%k4 + kmovb %r10d,%k5 + + + vpsubq %ymm4,%ymm1,%ymm1{%k1} + vpsubq %ymm4,%ymm16,%ymm16{%k2} + vpsubq %ymm4,%ymm17,%ymm17{%k3} + vpsubq %ymm4,%ymm18,%ymm18{%k4} + vpsubq %ymm4,%ymm19,%ymm19{%k5} + + vpandq %ymm4,%ymm1,%ymm1 + vpandq %ymm4,%ymm16,%ymm16 + vpandq %ymm4,%ymm17,%ymm17 + vpandq %ymm4,%ymm18,%ymm18 + vpandq %ymm4,%ymm19,%ymm19 + + vpbroadcastq %r15,%ymm3 + vpblendd $3,%ymm3,%ymm2,%ymm2 + + + + vpsrlq $52,%ymm2,%ymm24 + vpsrlq $52,%ymm20,%ymm25 + vpsrlq $52,%ymm21,%ymm26 + vpsrlq $52,%ymm22,%ymm27 + vpsrlq $52,%ymm23,%ymm28 + + + valignq $3,%ymm27,%ymm28,%ymm28 + valignq $3,%ymm26,%ymm27,%ymm27 + valignq $3,%ymm25,%ymm26,%ymm26 + valignq $3,%ymm24,%ymm25,%ymm25 + valignq $3,%ymm0,%ymm24,%ymm24 + + + vpandq %ymm4,%ymm2,%ymm2 + vpandq %ymm4,%ymm20,%ymm20 + vpandq %ymm4,%ymm21,%ymm21 + vpandq %ymm4,%ymm22,%ymm22 + vpandq %ymm4,%ymm23,%ymm23 + + + vpaddq %ymm24,%ymm2,%ymm2 + vpaddq %ymm25,%ymm20,%ymm20 + vpaddq %ymm26,%ymm21,%ymm21 + vpaddq %ymm27,%ymm22,%ymm22 + vpaddq %ymm28,%ymm23,%ymm23 + + + + vpcmpuq $1,%ymm2,%ymm4,%k1 + vpcmpuq $1,%ymm20,%ymm4,%k2 + vpcmpuq $1,%ymm21,%ymm4,%k3 + vpcmpuq $1,%ymm22,%ymm4,%k4 + vpcmpuq $1,%ymm23,%ymm4,%k5 + kmovb %k1,%r14d + kmovb %k2,%r13d + kmovb %k3,%r12d + kmovb %k4,%r11d + kmovb %k5,%r10d + + + vpcmpuq $0,%ymm2,%ymm4,%k1 + vpcmpuq $0,%ymm20,%ymm4,%k2 + vpcmpuq $0,%ymm21,%ymm4,%k3 + vpcmpuq $0,%ymm22,%ymm4,%k4 + vpcmpuq $0,%ymm23,%ymm4,%k5 + kmovb %k1,%r9d + kmovb %k2,%r8d + kmovb %k3,%ebx + kmovb %k4,%ecx + kmovb %k5,%edx + + + + shlb $4,%r13b + orb %r13b,%r14b + shlb $4,%r11b + orb %r11b,%r12b + + addb %r14b,%r14b + adcb %r12b,%r12b + adcb %r10b,%r10b + + shlb $4,%r8b + orb %r8b,%r9b + shlb $4,%cl + orb %cl,%bl + + addb %r9b,%r14b + adcb %bl,%r12b + adcb %dl,%r10b + + xorb %r9b,%r14b + xorb %bl,%r12b + xorb %dl,%r10b + + kmovb %r14d,%k1 + shrb $4,%r14b + kmovb %r14d,%k2 + kmovb %r12d,%k3 + shrb $4,%r12b + kmovb %r12d,%k4 + kmovb %r10d,%k5 + + + vpsubq %ymm4,%ymm2,%ymm2{%k1} + vpsubq %ymm4,%ymm20,%ymm20{%k2} + vpsubq %ymm4,%ymm21,%ymm21{%k3} + vpsubq %ymm4,%ymm22,%ymm22{%k4} + vpsubq %ymm4,%ymm23,%ymm23{%k5} + + vpandq %ymm4,%ymm2,%ymm2 + vpandq %ymm4,%ymm20,%ymm20 + vpandq %ymm4,%ymm21,%ymm21 + vpandq %ymm4,%ymm22,%ymm22 + vpandq %ymm4,%ymm23,%ymm23 + + vmovdqu64 %ymm1,(%rdi) + vmovdqu64 %ymm16,32(%rdi) + vmovdqu64 %ymm17,64(%rdi) + vmovdqu64 %ymm18,96(%rdi) + vmovdqu64 %ymm19,128(%rdi) + + vmovdqu64 %ymm2,160(%rdi) + vmovdqu64 %ymm20,192(%rdi) + vmovdqu64 %ymm21,224(%rdi) + vmovdqu64 %ymm22,256(%rdi) + vmovdqu64 %ymm23,288(%rdi) + + vzeroupper + movq 0(%rsp),%r15 +.cfi_restore %r15 + movq 8(%rsp),%r14 +.cfi_restore %r14 + movq 16(%rsp),%r13 +.cfi_restore %r13 + movq 24(%rsp),%r12 +.cfi_restore %r12 + movq 32(%rsp),%rbp +.cfi_restore %rbp + movq 40(%rsp),%rbx +.cfi_restore %rbx + leaq 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lrsaz_amm52x20_x2_256_epilogue: + .byte 0xf3,0xc3 +.cfi_endproc +.size ossl_rsaz_amm52x20_x2_256, .-ossl_rsaz_amm52x20_x2_256 +.text + +.align 32 +.globl ossl_extract_multiplier_2x20_win5 +.type ossl_extract_multiplier_2x20_win5,@function +ossl_extract_multiplier_2x20_win5: +.cfi_startproc +.byte 243,15,30,250 + leaq (%rcx,%rcx,4),%rax + salq $5,%rax + addq %rax,%rsi + + vmovdqa64 .Lones(%rip),%ymm23 + vpbroadcastq %rdx,%ymm22 + leaq 10240(%rsi),%rax + + vpxor %xmm4,%xmm4,%xmm4 + vmovdqa64 %ymm4,%ymm3 + vmovdqa64 %ymm4,%ymm2 + vmovdqa64 %ymm4,%ymm1 + vmovdqa64 %ymm4,%ymm0 + vmovdqa64 %ymm4,%ymm21 + +.align 32 +.Lloop: + vpcmpq $0,%ymm21,%ymm22,%k1 + addq $320,%rsi + vpaddq %ymm23,%ymm21,%ymm21 + vmovdqu64 -320(%rsi),%ymm16 + vmovdqu64 -288(%rsi),%ymm17 + vmovdqu64 -256(%rsi),%ymm18 + vmovdqu64 -224(%rsi),%ymm19 + vmovdqu64 -192(%rsi),%ymm20 + vpblendmq %ymm16,%ymm0,%ymm0{%k1} + vpblendmq %ymm17,%ymm1,%ymm1{%k1} + vpblendmq %ymm18,%ymm2,%ymm2{%k1} + vpblendmq %ymm19,%ymm3,%ymm3{%k1} + vpblendmq %ymm20,%ymm4,%ymm4{%k1} + cmpq %rsi,%rax + jne .Lloop + + vmovdqu64 %ymm0,(%rdi) + vmovdqu64 %ymm1,32(%rdi) + vmovdqu64 %ymm2,64(%rdi) + vmovdqu64 %ymm3,96(%rdi) + vmovdqu64 %ymm4,128(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size ossl_extract_multiplier_2x20_win5, .-ossl_extract_multiplier_2x20_win5 +.data +.align 32 +.Lones: +.quad 1,1,1,1 + .section ".note.gnu.property", "a" + .p2align 3 + .long 1f - 0f + .long 4f - 1f + .long 5 +0: + # "GNU" encoded with .byte, since .asciz isn't supported + # on Solaris. + .byte 0x47 + .byte 0x4e + .byte 0x55 + .byte 0 +1: + .p2align 3 + .long 0xc0000002 + .long 3f - 2f +2: + .long 3 +3: + .p2align 3 +4: diff --git a/openssl/src/crypto/bn/gen/linux_x64/x86_64-mont5.s b/openssl/src/crypto/bn/gen/linux_x64/x86_64-mont5.s index 0eb8b6c8b..71bf60c1a 100644 --- a/openssl/src/crypto/bn/gen/linux_x64/x86_64-mont5.s +++ b/openssl/src/crypto/bn/gen/linux_x64/x86_64-mont5.s @@ -2048,6 +2048,185 @@ __bn_post4x_internal: .byte 0xf3,0xc3 .cfi_endproc .size __bn_post4x_internal,.-__bn_post4x_internal +.globl bn_from_montgomery +.type bn_from_montgomery,@function +.align 32 +bn_from_montgomery: +.cfi_startproc + testl $7,%r9d + jz bn_from_mont8x + xorl %eax,%eax + .byte 0xf3,0xc3 +.cfi_endproc +.size bn_from_montgomery,.-bn_from_montgomery + +.type bn_from_mont8x,@function +.align 32 +bn_from_mont8x: +.cfi_startproc +.byte 0x67 + movq %rsp,%rax +.cfi_def_cfa_register %rax + pushq %rbx +.cfi_offset %rbx,-16 + pushq %rbp +.cfi_offset %rbp,-24 + pushq %r12 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_offset %r15,-56 +.Lfrom_prologue: + + shll $3,%r9d + leaq (%r9,%r9,2),%r10 + negq %r9 + movq (%r8),%r8 + + + + + + + + + leaq -320(%rsp,%r9,2),%r11 + movq %rsp,%rbp + subq %rdi,%r11 + andq $4095,%r11 + cmpq %r11,%r10 + jb .Lfrom_sp_alt + subq %r11,%rbp + leaq -320(%rbp,%r9,2),%rbp + jmp .Lfrom_sp_done + +.align 32 +.Lfrom_sp_alt: + leaq 4096-320(,%r9,2),%r10 + leaq -320(%rbp,%r9,2),%rbp + subq %r10,%r11 + movq $0,%r10 + cmovcq %r10,%r11 + subq %r11,%rbp +.Lfrom_sp_done: + andq $-64,%rbp + movq %rsp,%r11 + subq %rbp,%r11 + andq $-4096,%r11 + leaq (%r11,%rbp,1),%rsp + movq (%rsp),%r10 + cmpq %rbp,%rsp + ja .Lfrom_page_walk + jmp .Lfrom_page_walk_done + +.Lfrom_page_walk: + leaq -4096(%rsp),%rsp + movq (%rsp),%r10 + cmpq %rbp,%rsp + ja .Lfrom_page_walk +.Lfrom_page_walk_done: + + movq %r9,%r10 + negq %r9 + + + + + + + + + + + movq %r8,32(%rsp) + movq %rax,40(%rsp) +.cfi_escape 0x0f,0x05,0x77,0x28,0x06,0x23,0x08 +.Lfrom_body: + movq %r9,%r11 + leaq 48(%rsp),%rax + pxor %xmm0,%xmm0 + jmp .Lmul_by_1 + +.align 32 +.Lmul_by_1: + movdqu (%rsi),%xmm1 + movdqu 16(%rsi),%xmm2 + movdqu 32(%rsi),%xmm3 + movdqa %xmm0,(%rax,%r9,1) + movdqu 48(%rsi),%xmm4 + movdqa %xmm0,16(%rax,%r9,1) +.byte 0x48,0x8d,0xb6,0x40,0x00,0x00,0x00 + movdqa %xmm1,(%rax) + movdqa %xmm0,32(%rax,%r9,1) + movdqa %xmm2,16(%rax) + movdqa %xmm0,48(%rax,%r9,1) + movdqa %xmm3,32(%rax) + movdqa %xmm4,48(%rax) + leaq 64(%rax),%rax + subq $64,%r11 + jnz .Lmul_by_1 + +.byte 102,72,15,110,207 +.byte 102,72,15,110,209 +.byte 0x67 + movq %rcx,%rbp +.byte 102,73,15,110,218 + movl OPENSSL_ia32cap_P+8(%rip),%r11d + andl $0x80108,%r11d + cmpl $0x80108,%r11d + jne .Lfrom_mont_nox + + leaq (%rax,%r9,1),%rdi + call __bn_sqrx8x_reduction + call __bn_postx4x_internal + + pxor %xmm0,%xmm0 + leaq 48(%rsp),%rax + jmp .Lfrom_mont_zero + +.align 32 +.Lfrom_mont_nox: + call __bn_sqr8x_reduction + call __bn_post4x_internal + + pxor %xmm0,%xmm0 + leaq 48(%rsp),%rax + jmp .Lfrom_mont_zero + +.align 32 +.Lfrom_mont_zero: + movq 40(%rsp),%rsi +.cfi_def_cfa %rsi,8 + movdqa %xmm0,0(%rax) + movdqa %xmm0,16(%rax) + movdqa %xmm0,32(%rax) + movdqa %xmm0,48(%rax) + leaq 64(%rax),%rax + subq $32,%r9 + jnz .Lfrom_mont_zero + + movq $1,%rax + movq -48(%rsi),%r15 +.cfi_restore %r15 + movq -40(%rsi),%r14 +.cfi_restore %r14 + movq -32(%rsi),%r13 +.cfi_restore %r13 + movq -24(%rsi),%r12 +.cfi_restore %r12 + movq -16(%rsi),%rbp +.cfi_restore %rbp + movq -8(%rsi),%rbx +.cfi_restore %rbx + leaq (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Lfrom_epilogue: + .byte 0xf3,0xc3 +.cfi_endproc +.size bn_from_mont8x,.-bn_from_mont8x .type bn_mulx4x_mont_gather5,@function .align 32 bn_mulx4x_mont_gather5: diff --git a/openssl/src/crypto/bn/gen/windows_ia32/bn-586.asm b/openssl/src/crypto/bn/gen/windows_ia32/bn-586.asm index 236d3bb8d..82002b353 100644 --- a/openssl/src/crypto/bn/gen/windows_ia32/bn-586.asm +++ b/openssl/src/crypto/bn/gen/windows_ia32/bn-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/bn/gen/windows_ia32/co-586.asm b/openssl/src/crypto/bn/gen/windows_ia32/co-586.asm index b264f7f68..d57f0b5ff 100644 --- a/openssl/src/crypto/bn/gen/windows_ia32/co-586.asm +++ b/openssl/src/crypto/bn/gen/windows_ia32/co-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/bn/gen/windows_ia32/x86-gf2m.asm b/openssl/src/crypto/bn/gen/windows_ia32/x86-gf2m.asm index 169ddc962..709f4a9e5 100644 --- a/openssl/src/crypto/bn/gen/windows_ia32/x86-gf2m.asm +++ b/openssl/src/crypto/bn/gen/windows_ia32/x86-gf2m.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/bn/gen/windows_ia32/x86-mont.asm b/openssl/src/crypto/bn/gen/windows_ia32/x86-mont.asm index 6151bcc9b..090630c3a 100644 --- a/openssl/src/crypto/bn/gen/windows_ia32/x86-mont.asm +++ b/openssl/src/crypto/bn/gen/windows_ia32/x86-mont.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/bn/gen/windows_x64/rsaz-2k-avx512.asm b/openssl/src/crypto/bn/gen/windows_x64/rsaz-2k-avx512.asm deleted file mode 100644 index ae783c875..000000000 --- a/openssl/src/crypto/bn/gen/windows_x64/rsaz-2k-avx512.asm +++ /dev/null @@ -1,1026 +0,0 @@ -default rel -%define XMMWORD -%define YMMWORD -%define ZMMWORD -EXTERN OPENSSL_ia32cap_P -global ossl_rsaz_avx512ifma_eligible - -ALIGN 32 -ossl_rsaz_avx512ifma_eligible: - mov ecx,DWORD[((OPENSSL_ia32cap_P+8))] - xor eax,eax - and ecx,2149777408 - cmp ecx,2149777408 - cmove eax,ecx - DB 0F3h,0C3h ;repret - -section .text code align=64 - - -global ossl_rsaz_amm52x20_x1_ifma256 - -ALIGN 32 -ossl_rsaz_amm52x20_x1_ifma256: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ossl_rsaz_amm52x20_x1_ifma256: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - mov r8,QWORD[40+rsp] - - - -DB 243,15,30,250 - push rbx - - push rbp - - push r12 - - push r13 - - push r14 - - push r15 - -$L$ossl_rsaz_amm52x20_x1_ifma256_body: - - - vpxord ymm0,ymm0,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm16,ymm0 - vmovdqa64 ymm17,ymm0 - vmovdqa64 ymm18,ymm0 - vmovdqa64 ymm19,ymm0 - - xor r9d,r9d - - mov r11,rdx - mov rax,0xfffffffffffff - - - mov ebx,5 - -ALIGN 32 -$L$loop5: - mov r13,QWORD[r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm19,ymm2,YMMWORD[128+rcx] - - - valignq ymm3,ymm16,ymm3,1 - valignq ymm16,ymm17,ymm16,1 - valignq ymm17,ymm18,ymm17,1 - valignq ymm18,ymm19,ymm18,1 - valignq ymm19,ymm0,ymm19,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm19,ymm2,YMMWORD[128+rcx] - mov r13,QWORD[8+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm19,ymm2,YMMWORD[128+rcx] - - - valignq ymm3,ymm16,ymm3,1 - valignq ymm16,ymm17,ymm16,1 - valignq ymm17,ymm18,ymm17,1 - valignq ymm18,ymm19,ymm18,1 - valignq ymm19,ymm0,ymm19,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm19,ymm2,YMMWORD[128+rcx] - mov r13,QWORD[16+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm19,ymm2,YMMWORD[128+rcx] - - - valignq ymm3,ymm16,ymm3,1 - valignq ymm16,ymm17,ymm16,1 - valignq ymm17,ymm18,ymm17,1 - valignq ymm18,ymm19,ymm18,1 - valignq ymm19,ymm0,ymm19,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm19,ymm2,YMMWORD[128+rcx] - mov r13,QWORD[24+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm19,ymm2,YMMWORD[128+rcx] - - - valignq ymm3,ymm16,ymm3,1 - valignq ymm16,ymm17,ymm16,1 - valignq ymm17,ymm18,ymm17,1 - valignq ymm18,ymm19,ymm18,1 - valignq ymm19,ymm0,ymm19,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm19,ymm2,YMMWORD[128+rcx] - lea r11,[32+r11] - dec ebx - jne NEAR $L$loop5 - - vpbroadcastq ymm0,r9 - vpblendd ymm3,ymm3,ymm0,3 - - - - vpsrlq ymm0,ymm3,52 - vpsrlq ymm1,ymm16,52 - vpsrlq ymm2,ymm17,52 - vpsrlq ymm25,ymm18,52 - vpsrlq ymm26,ymm19,52 - - - valignq ymm26,ymm26,ymm25,3 - valignq ymm25,ymm25,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm16,ymm16,YMMWORD[$L$mask52x4] - vpandq ymm17,ymm17,YMMWORD[$L$mask52x4] - vpandq ymm18,ymm18,YMMWORD[$L$mask52x4] - vpandq ymm19,ymm19,YMMWORD[$L$mask52x4] - - - vpaddq ymm3,ymm3,ymm0 - vpaddq ymm16,ymm16,ymm1 - vpaddq ymm17,ymm17,ymm2 - vpaddq ymm18,ymm18,ymm25 - vpaddq ymm19,ymm19,ymm26 - - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm16,YMMWORD[$L$mask52x4],6 - vpcmpuq k3,ymm17,YMMWORD[$L$mask52x4],6 - vpcmpuq k4,ymm18,YMMWORD[$L$mask52x4],6 - vpcmpuq k5,ymm19,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - kmovb r12d,k3 - kmovb r11d,k4 - kmovb r10d,k5 - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm16,YMMWORD[$L$mask52x4],0 - vpcmpuq k3,ymm17,YMMWORD[$L$mask52x4],0 - vpcmpuq k4,ymm18,YMMWORD[$L$mask52x4],0 - vpcmpuq k5,ymm19,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - kmovb ebx,k3 - kmovb ecx,k4 - kmovb edx,k5 - - - - shl r13b,4 - or r14b,r13b - shl r11b,4 - or r12b,r11b - - add r14b,r14b - adc r12b,r12b - adc r10b,r10b - - shl r8b,4 - or r9b,r8b - shl cl,4 - or bl,cl - - add r14b,r9b - adc r12b,bl - adc r10b,dl - - xor r14b,r9b - xor r12b,bl - xor r10b,dl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r12d - shr r12b,4 - kmovb k4,r12d - kmovb k5,r10d - - - vpsubq ymm3{k1},ymm3,YMMWORD[$L$mask52x4] - vpsubq ymm16{k2},ymm16,YMMWORD[$L$mask52x4] - vpsubq ymm17{k3},ymm17,YMMWORD[$L$mask52x4] - vpsubq ymm18{k4},ymm18,YMMWORD[$L$mask52x4] - vpsubq ymm19{k5},ymm19,YMMWORD[$L$mask52x4] - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm16,ymm16,YMMWORD[$L$mask52x4] - vpandq ymm17,ymm17,YMMWORD[$L$mask52x4] - vpandq ymm18,ymm18,YMMWORD[$L$mask52x4] - vpandq ymm19,ymm19,YMMWORD[$L$mask52x4] - - vmovdqu64 YMMWORD[rdi],ymm3 - vmovdqu64 YMMWORD[32+rdi],ymm16 - vmovdqu64 YMMWORD[64+rdi],ymm17 - vmovdqu64 YMMWORD[96+rdi],ymm18 - vmovdqu64 YMMWORD[128+rdi],ymm19 - - vzeroupper - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov r12,QWORD[24+rsp] - - mov rbp,QWORD[32+rsp] - - mov rbx,QWORD[40+rsp] - - lea rsp,[48+rsp] - -$L$ossl_rsaz_amm52x20_x1_ifma256_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_ossl_rsaz_amm52x20_x1_ifma256: -section .data data align=8 - -ALIGN 32 -$L$mask52x4: - DQ 0xfffffffffffff - DQ 0xfffffffffffff - DQ 0xfffffffffffff - DQ 0xfffffffffffff -section .text code align=64 - - -global ossl_rsaz_amm52x20_x2_ifma256 - -ALIGN 32 -ossl_rsaz_amm52x20_x2_ifma256: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ossl_rsaz_amm52x20_x2_ifma256: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - mov r8,QWORD[40+rsp] - - - -DB 243,15,30,250 - push rbx - - push rbp - - push r12 - - push r13 - - push r14 - - push r15 - -$L$ossl_rsaz_amm52x20_x2_ifma256_body: - - - vpxord ymm0,ymm0,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm16,ymm0 - vmovdqa64 ymm17,ymm0 - vmovdqa64 ymm18,ymm0 - vmovdqa64 ymm19,ymm0 - vmovdqa64 ymm4,ymm0 - vmovdqa64 ymm20,ymm0 - vmovdqa64 ymm21,ymm0 - vmovdqa64 ymm22,ymm0 - vmovdqa64 ymm23,ymm0 - - xor r9d,r9d - xor r15d,r15d - - mov r11,rdx - mov rax,0xfffffffffffff - - mov ebx,20 - -ALIGN 32 -$L$loop20: - mov r13,QWORD[r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,QWORD[r8] - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm19,ymm2,YMMWORD[128+rcx] - - - valignq ymm3,ymm16,ymm3,1 - valignq ymm16,ymm17,ymm16,1 - valignq ymm17,ymm18,ymm17,1 - valignq ymm18,ymm19,ymm18,1 - valignq ymm19,ymm0,ymm19,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm16,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm17,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm18,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm19,ymm1,YMMWORD[128+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm16,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm17,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm18,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm19,ymm2,YMMWORD[128+rcx] - mov r13,QWORD[160+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[160+rsi] - mulx r12,r13,r13 - add r15,r13 - mov r10,r12 - adc r10,0 - - mov r13,QWORD[8+r8] - imul r13,r15 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[160+rcx] - mulx r12,r13,r13 - add r15,r13 - adc r10,r12 - - shr r15,52 - sal r10,12 - or r15,r10 - - vpmadd52luq ymm4,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm20,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm21,ymm1,YMMWORD[224+rsi] - vpmadd52luq ymm22,ymm1,YMMWORD[256+rsi] - vpmadd52luq ymm23,ymm1,YMMWORD[288+rsi] - - vpmadd52luq ymm4,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm20,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm21,ymm2,YMMWORD[224+rcx] - vpmadd52luq ymm22,ymm2,YMMWORD[256+rcx] - vpmadd52luq ymm23,ymm2,YMMWORD[288+rcx] - - - valignq ymm4,ymm20,ymm4,1 - valignq ymm20,ymm21,ymm20,1 - valignq ymm21,ymm22,ymm21,1 - valignq ymm22,ymm23,ymm22,1 - valignq ymm23,ymm0,ymm23,1 - - vmovq r13,xmm4 - add r15,r13 - - vpmadd52huq ymm4,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm20,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm21,ymm1,YMMWORD[224+rsi] - vpmadd52huq ymm22,ymm1,YMMWORD[256+rsi] - vpmadd52huq ymm23,ymm1,YMMWORD[288+rsi] - - vpmadd52huq ymm4,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm20,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm21,ymm2,YMMWORD[224+rcx] - vpmadd52huq ymm22,ymm2,YMMWORD[256+rcx] - vpmadd52huq ymm23,ymm2,YMMWORD[288+rcx] - lea r11,[8+r11] - dec ebx - jne NEAR $L$loop20 - - vpbroadcastq ymm0,r9 - vpblendd ymm3,ymm3,ymm0,3 - - - - vpsrlq ymm0,ymm3,52 - vpsrlq ymm1,ymm16,52 - vpsrlq ymm2,ymm17,52 - vpsrlq ymm25,ymm18,52 - vpsrlq ymm26,ymm19,52 - - - valignq ymm26,ymm26,ymm25,3 - valignq ymm25,ymm25,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm16,ymm16,YMMWORD[$L$mask52x4] - vpandq ymm17,ymm17,YMMWORD[$L$mask52x4] - vpandq ymm18,ymm18,YMMWORD[$L$mask52x4] - vpandq ymm19,ymm19,YMMWORD[$L$mask52x4] - - - vpaddq ymm3,ymm3,ymm0 - vpaddq ymm16,ymm16,ymm1 - vpaddq ymm17,ymm17,ymm2 - vpaddq ymm18,ymm18,ymm25 - vpaddq ymm19,ymm19,ymm26 - - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm16,YMMWORD[$L$mask52x4],6 - vpcmpuq k3,ymm17,YMMWORD[$L$mask52x4],6 - vpcmpuq k4,ymm18,YMMWORD[$L$mask52x4],6 - vpcmpuq k5,ymm19,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - kmovb r12d,k3 - kmovb r11d,k4 - kmovb r10d,k5 - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm16,YMMWORD[$L$mask52x4],0 - vpcmpuq k3,ymm17,YMMWORD[$L$mask52x4],0 - vpcmpuq k4,ymm18,YMMWORD[$L$mask52x4],0 - vpcmpuq k5,ymm19,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - kmovb ebx,k3 - kmovb ecx,k4 - kmovb edx,k5 - - - - shl r13b,4 - or r14b,r13b - shl r11b,4 - or r12b,r11b - - add r14b,r14b - adc r12b,r12b - adc r10b,r10b - - shl r8b,4 - or r9b,r8b - shl cl,4 - or bl,cl - - add r14b,r9b - adc r12b,bl - adc r10b,dl - - xor r14b,r9b - xor r12b,bl - xor r10b,dl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r12d - shr r12b,4 - kmovb k4,r12d - kmovb k5,r10d - - - vpsubq ymm3{k1},ymm3,YMMWORD[$L$mask52x4] - vpsubq ymm16{k2},ymm16,YMMWORD[$L$mask52x4] - vpsubq ymm17{k3},ymm17,YMMWORD[$L$mask52x4] - vpsubq ymm18{k4},ymm18,YMMWORD[$L$mask52x4] - vpsubq ymm19{k5},ymm19,YMMWORD[$L$mask52x4] - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm16,ymm16,YMMWORD[$L$mask52x4] - vpandq ymm17,ymm17,YMMWORD[$L$mask52x4] - vpandq ymm18,ymm18,YMMWORD[$L$mask52x4] - vpandq ymm19,ymm19,YMMWORD[$L$mask52x4] - - vpbroadcastq ymm0,r15 - vpblendd ymm4,ymm4,ymm0,3 - - - - vpsrlq ymm0,ymm4,52 - vpsrlq ymm1,ymm20,52 - vpsrlq ymm2,ymm21,52 - vpsrlq ymm25,ymm22,52 - vpsrlq ymm26,ymm23,52 - - - valignq ymm26,ymm26,ymm25,3 - valignq ymm25,ymm25,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm20,ymm20,YMMWORD[$L$mask52x4] - vpandq ymm21,ymm21,YMMWORD[$L$mask52x4] - vpandq ymm22,ymm22,YMMWORD[$L$mask52x4] - vpandq ymm23,ymm23,YMMWORD[$L$mask52x4] - - - vpaddq ymm4,ymm4,ymm0 - vpaddq ymm20,ymm20,ymm1 - vpaddq ymm21,ymm21,ymm2 - vpaddq ymm22,ymm22,ymm25 - vpaddq ymm23,ymm23,ymm26 - - - - vpcmpuq k1,ymm4,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm20,YMMWORD[$L$mask52x4],6 - vpcmpuq k3,ymm21,YMMWORD[$L$mask52x4],6 - vpcmpuq k4,ymm22,YMMWORD[$L$mask52x4],6 - vpcmpuq k5,ymm23,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - kmovb r12d,k3 - kmovb r11d,k4 - kmovb r10d,k5 - - - vpcmpuq k1,ymm4,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm20,YMMWORD[$L$mask52x4],0 - vpcmpuq k3,ymm21,YMMWORD[$L$mask52x4],0 - vpcmpuq k4,ymm22,YMMWORD[$L$mask52x4],0 - vpcmpuq k5,ymm23,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - kmovb ebx,k3 - kmovb ecx,k4 - kmovb edx,k5 - - - - shl r13b,4 - or r14b,r13b - shl r11b,4 - or r12b,r11b - - add r14b,r14b - adc r12b,r12b - adc r10b,r10b - - shl r8b,4 - or r9b,r8b - shl cl,4 - or bl,cl - - add r14b,r9b - adc r12b,bl - adc r10b,dl - - xor r14b,r9b - xor r12b,bl - xor r10b,dl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r12d - shr r12b,4 - kmovb k4,r12d - kmovb k5,r10d - - - vpsubq ymm4{k1},ymm4,YMMWORD[$L$mask52x4] - vpsubq ymm20{k2},ymm20,YMMWORD[$L$mask52x4] - vpsubq ymm21{k3},ymm21,YMMWORD[$L$mask52x4] - vpsubq ymm22{k4},ymm22,YMMWORD[$L$mask52x4] - vpsubq ymm23{k5},ymm23,YMMWORD[$L$mask52x4] - - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm20,ymm20,YMMWORD[$L$mask52x4] - vpandq ymm21,ymm21,YMMWORD[$L$mask52x4] - vpandq ymm22,ymm22,YMMWORD[$L$mask52x4] - vpandq ymm23,ymm23,YMMWORD[$L$mask52x4] - - vmovdqu64 YMMWORD[rdi],ymm3 - vmovdqu64 YMMWORD[32+rdi],ymm16 - vmovdqu64 YMMWORD[64+rdi],ymm17 - vmovdqu64 YMMWORD[96+rdi],ymm18 - vmovdqu64 YMMWORD[128+rdi],ymm19 - - vmovdqu64 YMMWORD[160+rdi],ymm4 - vmovdqu64 YMMWORD[192+rdi],ymm20 - vmovdqu64 YMMWORD[224+rdi],ymm21 - vmovdqu64 YMMWORD[256+rdi],ymm22 - vmovdqu64 YMMWORD[288+rdi],ymm23 - - vzeroupper - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov r12,QWORD[24+rsp] - - mov rbp,QWORD[32+rsp] - - mov rbx,QWORD[40+rsp] - - lea rsp,[48+rsp] - -$L$ossl_rsaz_amm52x20_x2_ifma256_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_ossl_rsaz_amm52x20_x2_ifma256: -section .text code align=64 - - -ALIGN 32 -global ossl_extract_multiplier_2x20_win5 - -ossl_extract_multiplier_2x20_win5: - -DB 243,15,30,250 - vmovdqa64 ymm24,YMMWORD[$L$ones] - vpbroadcastq ymm22,r8 - vpbroadcastq ymm23,r9 - lea rax,[10240+rdx] - - - vpxor xmm0,xmm0,xmm0 - vmovdqa64 ymm21,ymm0 - vmovdqa64 ymm1,ymm0 - vmovdqa64 ymm2,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm4,ymm0 - vmovdqa64 ymm5,ymm0 - vmovdqa64 ymm16,ymm0 - vmovdqa64 ymm17,ymm0 - vmovdqa64 ymm18,ymm0 - vmovdqa64 ymm19,ymm0 - -ALIGN 32 -$L$loop: - vpcmpq k1,ymm22,ymm21,0 - vpcmpq k2,ymm23,ymm21,0 - vmovdqu64 ymm20,YMMWORD[rdx] - vpblendmq ymm0{k1},ymm0,ymm20 - vmovdqu64 ymm20,YMMWORD[32+rdx] - vpblendmq ymm1{k1},ymm1,ymm20 - vmovdqu64 ymm20,YMMWORD[64+rdx] - vpblendmq ymm2{k1},ymm2,ymm20 - vmovdqu64 ymm20,YMMWORD[96+rdx] - vpblendmq ymm3{k1},ymm3,ymm20 - vmovdqu64 ymm20,YMMWORD[128+rdx] - vpblendmq ymm4{k1},ymm4,ymm20 - vmovdqu64 ymm20,YMMWORD[160+rdx] - vpblendmq ymm5{k2},ymm5,ymm20 - vmovdqu64 ymm20,YMMWORD[192+rdx] - vpblendmq ymm16{k2},ymm16,ymm20 - vmovdqu64 ymm20,YMMWORD[224+rdx] - vpblendmq ymm17{k2},ymm17,ymm20 - vmovdqu64 ymm20,YMMWORD[256+rdx] - vpblendmq ymm18{k2},ymm18,ymm20 - vmovdqu64 ymm20,YMMWORD[288+rdx] - vpblendmq ymm19{k2},ymm19,ymm20 - vpaddq ymm21,ymm21,ymm24 - add rdx,320 - cmp rax,rdx - jne NEAR $L$loop - vmovdqu64 YMMWORD[rcx],ymm0 - vmovdqu64 YMMWORD[32+rcx],ymm1 - vmovdqu64 YMMWORD[64+rcx],ymm2 - vmovdqu64 YMMWORD[96+rcx],ymm3 - vmovdqu64 YMMWORD[128+rcx],ymm4 - vmovdqu64 YMMWORD[160+rcx],ymm5 - vmovdqu64 YMMWORD[192+rcx],ymm16 - vmovdqu64 YMMWORD[224+rcx],ymm17 - vmovdqu64 YMMWORD[256+rcx],ymm18 - vmovdqu64 YMMWORD[288+rcx],ymm19 - DB 0F3h,0C3h ;repret - - -section .data data align=8 - -ALIGN 32 -$L$ones: - DQ 1,1,1,1 -$L$zeros: - DQ 0,0,0,0 -EXTERN __imp_RtlVirtualUnwind - -ALIGN 16 -rsaz_def_handler: - push rsi - push rdi - push rbx - push rbp - push r12 - push r13 - push r14 - push r15 - pushfq - sub rsp,64 - - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] - - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] - - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 - jb NEAR $L$common_seh_tail - - mov rax,QWORD[152+r8] - - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 - jae NEAR $L$common_seh_tail - - lea rax,[48+rax] - - mov rbx,QWORD[((-8))+rax] - mov rbp,QWORD[((-16))+rax] - mov r12,QWORD[((-24))+rax] - mov r13,QWORD[((-32))+rax] - mov r14,QWORD[((-40))+rax] - mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 - -$L$common_seh_tail: - mov rdi,QWORD[8+rax] - mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi - - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc - - mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx - call QWORD[__imp_RtlVirtualUnwind] - - mov eax,1 - add rsp,64 - popfq - pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx - pop rdi - pop rsi - DB 0F3h,0C3h ;repret - - -section .pdata rdata align=4 -ALIGN 4 - DD $L$SEH_begin_ossl_rsaz_amm52x20_x1_ifma256 wrt ..imagebase - DD $L$SEH_end_ossl_rsaz_amm52x20_x1_ifma256 wrt ..imagebase - DD $L$SEH_info_ossl_rsaz_amm52x20_x1_ifma256 wrt ..imagebase - - DD $L$SEH_begin_ossl_rsaz_amm52x20_x2_ifma256 wrt ..imagebase - DD $L$SEH_end_ossl_rsaz_amm52x20_x2_ifma256 wrt ..imagebase - DD $L$SEH_info_ossl_rsaz_amm52x20_x2_ifma256 wrt ..imagebase - -section .xdata rdata align=8 -ALIGN 8 -$L$SEH_info_ossl_rsaz_amm52x20_x1_ifma256: -DB 9,0,0,0 - DD rsaz_def_handler wrt ..imagebase - DD $L$ossl_rsaz_amm52x20_x1_ifma256_body wrt ..imagebase,$L$ossl_rsaz_amm52x20_x1_ifma256_epilogue wrt ..imagebase -$L$SEH_info_ossl_rsaz_amm52x20_x2_ifma256: -DB 9,0,0,0 - DD rsaz_def_handler wrt ..imagebase - DD $L$ossl_rsaz_amm52x20_x2_ifma256_body wrt ..imagebase,$L$ossl_rsaz_amm52x20_x2_ifma256_epilogue wrt ..imagebase diff --git a/openssl/src/crypto/bn/gen/windows_x64/rsaz-3k-avx512.asm b/openssl/src/crypto/bn/gen/windows_x64/rsaz-3k-avx512.asm deleted file mode 100644 index 63169e949..000000000 --- a/openssl/src/crypto/bn/gen/windows_x64/rsaz-3k-avx512.asm +++ /dev/null @@ -1,1492 +0,0 @@ -default rel -%define XMMWORD -%define YMMWORD -%define ZMMWORD -section .text code align=64 - - -global ossl_rsaz_amm52x30_x1_ifma256 - -ALIGN 32 -ossl_rsaz_amm52x30_x1_ifma256: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ossl_rsaz_amm52x30_x1_ifma256: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - mov r8,QWORD[40+rsp] - - - -DB 243,15,30,250 - push rbx - - push rbp - - push r12 - - push r13 - - push r14 - - push r15 - - lea rsp,[((-168))+rsp] - vmovdqa64 XMMWORD[rsp],xmm6 - vmovdqa64 XMMWORD[16+rsp],xmm7 - vmovdqa64 XMMWORD[32+rsp],xmm8 - vmovdqa64 XMMWORD[48+rsp],xmm9 - vmovdqa64 XMMWORD[64+rsp],xmm10 - vmovdqa64 XMMWORD[80+rsp],xmm11 - vmovdqa64 XMMWORD[96+rsp],xmm12 - vmovdqa64 XMMWORD[112+rsp],xmm13 - vmovdqa64 XMMWORD[128+rsp],xmm14 - vmovdqa64 XMMWORD[144+rsp],xmm15 -$L$ossl_rsaz_amm52x30_x1_ifma256_body: - - vpxord ymm0,ymm0,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm4,ymm0 - vmovdqa64 ymm5,ymm0 - vmovdqa64 ymm6,ymm0 - vmovdqa64 ymm7,ymm0 - vmovdqa64 ymm8,ymm0 - vmovdqa64 ymm9,ymm0 - vmovdqa64 ymm10,ymm0 - - xor r9d,r9d - - mov r11,rdx - mov rax,0xfffffffffffff - - - mov ebx,7 - -ALIGN 32 -$L$loop7: - mov r13,QWORD[r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm0,ymm10,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - mov r13,QWORD[8+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm0,ymm10,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - mov r13,QWORD[16+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm0,ymm10,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - mov r13,QWORD[24+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm0,ymm10,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - lea r11,[32+r11] - dec ebx - jne NEAR $L$loop7 - mov r13,QWORD[r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm0,ymm10,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - mov r13,QWORD[8+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm0,ymm10,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - - vpbroadcastq ymm0,r9 - vpblendd ymm3,ymm3,ymm0,3 - - - - vpsrlq ymm0,ymm3,52 - vpsrlq ymm1,ymm4,52 - vpsrlq ymm2,ymm5,52 - vpsrlq ymm19,ymm6,52 - vpsrlq ymm20,ymm7,52 - vpsrlq ymm21,ymm8,52 - vpsrlq ymm22,ymm9,52 - vpsrlq ymm23,ymm10,52 - - - valignq ymm23,ymm23,ymm22,3 - valignq ymm22,ymm22,ymm21,3 - valignq ymm21,ymm21,ymm20,3 - valignq ymm20,ymm20,ymm19,3 - valignq ymm19,ymm19,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm5,ymm5,YMMWORD[$L$mask52x4] - vpandq ymm6,ymm6,YMMWORD[$L$mask52x4] - vpandq ymm7,ymm7,YMMWORD[$L$mask52x4] - vpandq ymm8,ymm8,YMMWORD[$L$mask52x4] - vpandq ymm9,ymm9,YMMWORD[$L$mask52x4] - vpandq ymm10,ymm10,YMMWORD[$L$mask52x4] - - - vpaddq ymm3,ymm3,ymm0 - vpaddq ymm4,ymm4,ymm1 - vpaddq ymm5,ymm5,ymm2 - vpaddq ymm6,ymm6,ymm19 - vpaddq ymm7,ymm7,ymm20 - vpaddq ymm8,ymm8,ymm21 - vpaddq ymm9,ymm9,ymm22 - vpaddq ymm10,ymm10,ymm23 - - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm4,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - shl r13b,4 - or r14b,r13b - - vpcmpuq k1,ymm5,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm6,YMMWORD[$L$mask52x4],6 - kmovb r13d,k1 - kmovb r12d,k2 - shl r12b,4 - or r13b,r12b - - vpcmpuq k1,ymm7,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm8,YMMWORD[$L$mask52x4],6 - kmovb r12d,k1 - kmovb r11d,k2 - shl r11b,4 - or r12b,r11b - - vpcmpuq k1,ymm9,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm10,YMMWORD[$L$mask52x4],6 - kmovb r11d,k1 - kmovb r10d,k2 - shl r10b,4 - or r11b,r10b - - add r14b,r14b - adc r13b,r13b - adc r12b,r12b - adc r11b,r11b - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm4,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - shl r8b,4 - or r9b,r8b - - vpcmpuq k1,ymm5,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm6,YMMWORD[$L$mask52x4],0 - kmovb r8d,k1 - kmovb edx,k2 - shl dl,4 - or r8b,dl - - vpcmpuq k1,ymm7,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm8,YMMWORD[$L$mask52x4],0 - kmovb edx,k1 - kmovb ecx,k2 - shl cl,4 - or dl,cl - - vpcmpuq k1,ymm9,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm10,YMMWORD[$L$mask52x4],0 - kmovb ecx,k1 - kmovb ebx,k2 - shl bl,4 - or cl,bl - - add r14b,r9b - adc r13b,r8b - adc r12b,dl - adc r11b,cl - - xor r14b,r9b - xor r13b,r8b - xor r12b,dl - xor r11b,cl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r13d - shr r13b,4 - kmovb k4,r13d - kmovb k5,r12d - shr r12b,4 - kmovb k6,r12d - kmovb k7,r11d - - vpsubq ymm3{k1},ymm3,YMMWORD[$L$mask52x4] - vpsubq ymm4{k2},ymm4,YMMWORD[$L$mask52x4] - vpsubq ymm5{k3},ymm5,YMMWORD[$L$mask52x4] - vpsubq ymm6{k4},ymm6,YMMWORD[$L$mask52x4] - vpsubq ymm7{k5},ymm7,YMMWORD[$L$mask52x4] - vpsubq ymm8{k6},ymm8,YMMWORD[$L$mask52x4] - vpsubq ymm9{k7},ymm9,YMMWORD[$L$mask52x4] - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm5,ymm5,YMMWORD[$L$mask52x4] - vpandq ymm6,ymm6,YMMWORD[$L$mask52x4] - vpandq ymm7,ymm7,YMMWORD[$L$mask52x4] - vpandq ymm8,ymm8,YMMWORD[$L$mask52x4] - vpandq ymm9,ymm9,YMMWORD[$L$mask52x4] - - shr r11b,4 - kmovb k1,r11d - - vpsubq ymm10{k1},ymm10,YMMWORD[$L$mask52x4] - - vpandq ymm10,ymm10,YMMWORD[$L$mask52x4] - - vmovdqu64 YMMWORD[rdi],ymm3 - vmovdqu64 YMMWORD[32+rdi],ymm4 - vmovdqu64 YMMWORD[64+rdi],ymm5 - vmovdqu64 YMMWORD[96+rdi],ymm6 - vmovdqu64 YMMWORD[128+rdi],ymm7 - vmovdqu64 YMMWORD[160+rdi],ymm8 - vmovdqu64 YMMWORD[192+rdi],ymm9 - vmovdqu64 YMMWORD[224+rdi],ymm10 - - vzeroupper - lea rax,[rsp] - - vmovdqa64 xmm6,XMMWORD[rax] - vmovdqa64 xmm7,XMMWORD[16+rax] - vmovdqa64 xmm8,XMMWORD[32+rax] - vmovdqa64 xmm9,XMMWORD[48+rax] - vmovdqa64 xmm10,XMMWORD[64+rax] - vmovdqa64 xmm11,XMMWORD[80+rax] - vmovdqa64 xmm12,XMMWORD[96+rax] - vmovdqa64 xmm13,XMMWORD[112+rax] - vmovdqa64 xmm14,XMMWORD[128+rax] - vmovdqa64 xmm15,XMMWORD[144+rax] - lea rax,[168+rsp] - mov r15,QWORD[rax] - - mov r14,QWORD[8+rax] - - mov r13,QWORD[16+rax] - - mov r12,QWORD[24+rax] - - mov rbp,QWORD[32+rax] - - mov rbx,QWORD[40+rax] - - lea rsp,[48+rax] - -$L$ossl_rsaz_amm52x30_x1_ifma256_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_ossl_rsaz_amm52x30_x1_ifma256: -section .data data align=8 - -ALIGN 32 -$L$mask52x4: - DQ 0xfffffffffffff - DQ 0xfffffffffffff - DQ 0xfffffffffffff - DQ 0xfffffffffffff -section .text code align=64 - - -global ossl_rsaz_amm52x30_x2_ifma256 - -ALIGN 32 -ossl_rsaz_amm52x30_x2_ifma256: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ossl_rsaz_amm52x30_x2_ifma256: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - mov r8,QWORD[40+rsp] - - - -DB 243,15,30,250 - push rbx - - push rbp - - push r12 - - push r13 - - push r14 - - push r15 - - lea rsp,[((-168))+rsp] - vmovdqa64 XMMWORD[rsp],xmm6 - vmovdqa64 XMMWORD[16+rsp],xmm7 - vmovdqa64 XMMWORD[32+rsp],xmm8 - vmovdqa64 XMMWORD[48+rsp],xmm9 - vmovdqa64 XMMWORD[64+rsp],xmm10 - vmovdqa64 XMMWORD[80+rsp],xmm11 - vmovdqa64 XMMWORD[96+rsp],xmm12 - vmovdqa64 XMMWORD[112+rsp],xmm13 - vmovdqa64 XMMWORD[128+rsp],xmm14 - vmovdqa64 XMMWORD[144+rsp],xmm15 -$L$ossl_rsaz_amm52x30_x2_ifma256_body: - - vpxord ymm0,ymm0,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm4,ymm0 - vmovdqa64 ymm5,ymm0 - vmovdqa64 ymm6,ymm0 - vmovdqa64 ymm7,ymm0 - vmovdqa64 ymm8,ymm0 - vmovdqa64 ymm9,ymm0 - vmovdqa64 ymm10,ymm0 - - vmovdqa64 ymm11,ymm0 - vmovdqa64 ymm12,ymm0 - vmovdqa64 ymm13,ymm0 - vmovdqa64 ymm14,ymm0 - vmovdqa64 ymm15,ymm0 - vmovdqa64 ymm16,ymm0 - vmovdqa64 ymm17,ymm0 - vmovdqa64 ymm18,ymm0 - - - xor r9d,r9d - xor r15d,r15d - - mov r11,rdx - mov rax,0xfffffffffffff - - mov ebx,30 - -ALIGN 32 -$L$loop30: - mov r13,QWORD[r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,QWORD[r8] - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm0,ymm10,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - mov r13,QWORD[256+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[256+rsi] - mulx r12,r13,r13 - add r15,r13 - mov r10,r12 - adc r10,0 - - mov r13,QWORD[8+r8] - imul r13,r15 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[256+rcx] - mulx r12,r13,r13 - add r15,r13 - adc r10,r12 - - shr r15,52 - sal r10,12 - or r15,r10 - - vpmadd52luq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52luq ymm12,ymm1,YMMWORD[288+rsi] - vpmadd52luq ymm13,ymm1,YMMWORD[320+rsi] - vpmadd52luq ymm14,ymm1,YMMWORD[352+rsi] - vpmadd52luq ymm15,ymm1,YMMWORD[384+rsi] - vpmadd52luq ymm16,ymm1,YMMWORD[416+rsi] - vpmadd52luq ymm17,ymm1,YMMWORD[448+rsi] - vpmadd52luq ymm18,ymm1,YMMWORD[480+rsi] - - vpmadd52luq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52luq ymm12,ymm2,YMMWORD[288+rcx] - vpmadd52luq ymm13,ymm2,YMMWORD[320+rcx] - vpmadd52luq ymm14,ymm2,YMMWORD[352+rcx] - vpmadd52luq ymm15,ymm2,YMMWORD[384+rcx] - vpmadd52luq ymm16,ymm2,YMMWORD[416+rcx] - vpmadd52luq ymm17,ymm2,YMMWORD[448+rcx] - vpmadd52luq ymm18,ymm2,YMMWORD[480+rcx] - - - valignq ymm11,ymm12,ymm11,1 - valignq ymm12,ymm13,ymm12,1 - valignq ymm13,ymm14,ymm13,1 - valignq ymm14,ymm15,ymm14,1 - valignq ymm15,ymm16,ymm15,1 - valignq ymm16,ymm17,ymm16,1 - valignq ymm17,ymm18,ymm17,1 - valignq ymm18,ymm0,ymm18,1 - - vmovq r13,xmm11 - add r15,r13 - - vpmadd52huq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52huq ymm12,ymm1,YMMWORD[288+rsi] - vpmadd52huq ymm13,ymm1,YMMWORD[320+rsi] - vpmadd52huq ymm14,ymm1,YMMWORD[352+rsi] - vpmadd52huq ymm15,ymm1,YMMWORD[384+rsi] - vpmadd52huq ymm16,ymm1,YMMWORD[416+rsi] - vpmadd52huq ymm17,ymm1,YMMWORD[448+rsi] - vpmadd52huq ymm18,ymm1,YMMWORD[480+rsi] - - vpmadd52huq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52huq ymm12,ymm2,YMMWORD[288+rcx] - vpmadd52huq ymm13,ymm2,YMMWORD[320+rcx] - vpmadd52huq ymm14,ymm2,YMMWORD[352+rcx] - vpmadd52huq ymm15,ymm2,YMMWORD[384+rcx] - vpmadd52huq ymm16,ymm2,YMMWORD[416+rcx] - vpmadd52huq ymm17,ymm2,YMMWORD[448+rcx] - vpmadd52huq ymm18,ymm2,YMMWORD[480+rcx] - lea r11,[8+r11] - dec ebx - jne NEAR $L$loop30 - - vpbroadcastq ymm0,r9 - vpblendd ymm3,ymm3,ymm0,3 - - - - vpsrlq ymm0,ymm3,52 - vpsrlq ymm1,ymm4,52 - vpsrlq ymm2,ymm5,52 - vpsrlq ymm19,ymm6,52 - vpsrlq ymm20,ymm7,52 - vpsrlq ymm21,ymm8,52 - vpsrlq ymm22,ymm9,52 - vpsrlq ymm23,ymm10,52 - - - valignq ymm23,ymm23,ymm22,3 - valignq ymm22,ymm22,ymm21,3 - valignq ymm21,ymm21,ymm20,3 - valignq ymm20,ymm20,ymm19,3 - valignq ymm19,ymm19,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm5,ymm5,YMMWORD[$L$mask52x4] - vpandq ymm6,ymm6,YMMWORD[$L$mask52x4] - vpandq ymm7,ymm7,YMMWORD[$L$mask52x4] - vpandq ymm8,ymm8,YMMWORD[$L$mask52x4] - vpandq ymm9,ymm9,YMMWORD[$L$mask52x4] - vpandq ymm10,ymm10,YMMWORD[$L$mask52x4] - - - vpaddq ymm3,ymm3,ymm0 - vpaddq ymm4,ymm4,ymm1 - vpaddq ymm5,ymm5,ymm2 - vpaddq ymm6,ymm6,ymm19 - vpaddq ymm7,ymm7,ymm20 - vpaddq ymm8,ymm8,ymm21 - vpaddq ymm9,ymm9,ymm22 - vpaddq ymm10,ymm10,ymm23 - - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm4,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - shl r13b,4 - or r14b,r13b - - vpcmpuq k1,ymm5,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm6,YMMWORD[$L$mask52x4],6 - kmovb r13d,k1 - kmovb r12d,k2 - shl r12b,4 - or r13b,r12b - - vpcmpuq k1,ymm7,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm8,YMMWORD[$L$mask52x4],6 - kmovb r12d,k1 - kmovb r11d,k2 - shl r11b,4 - or r12b,r11b - - vpcmpuq k1,ymm9,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm10,YMMWORD[$L$mask52x4],6 - kmovb r11d,k1 - kmovb r10d,k2 - shl r10b,4 - or r11b,r10b - - add r14b,r14b - adc r13b,r13b - adc r12b,r12b - adc r11b,r11b - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm4,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - shl r8b,4 - or r9b,r8b - - vpcmpuq k1,ymm5,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm6,YMMWORD[$L$mask52x4],0 - kmovb r8d,k1 - kmovb edx,k2 - shl dl,4 - or r8b,dl - - vpcmpuq k1,ymm7,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm8,YMMWORD[$L$mask52x4],0 - kmovb edx,k1 - kmovb ecx,k2 - shl cl,4 - or dl,cl - - vpcmpuq k1,ymm9,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm10,YMMWORD[$L$mask52x4],0 - kmovb ecx,k1 - kmovb ebx,k2 - shl bl,4 - or cl,bl - - add r14b,r9b - adc r13b,r8b - adc r12b,dl - adc r11b,cl - - xor r14b,r9b - xor r13b,r8b - xor r12b,dl - xor r11b,cl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r13d - shr r13b,4 - kmovb k4,r13d - kmovb k5,r12d - shr r12b,4 - kmovb k6,r12d - kmovb k7,r11d - - vpsubq ymm3{k1},ymm3,YMMWORD[$L$mask52x4] - vpsubq ymm4{k2},ymm4,YMMWORD[$L$mask52x4] - vpsubq ymm5{k3},ymm5,YMMWORD[$L$mask52x4] - vpsubq ymm6{k4},ymm6,YMMWORD[$L$mask52x4] - vpsubq ymm7{k5},ymm7,YMMWORD[$L$mask52x4] - vpsubq ymm8{k6},ymm8,YMMWORD[$L$mask52x4] - vpsubq ymm9{k7},ymm9,YMMWORD[$L$mask52x4] - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm5,ymm5,YMMWORD[$L$mask52x4] - vpandq ymm6,ymm6,YMMWORD[$L$mask52x4] - vpandq ymm7,ymm7,YMMWORD[$L$mask52x4] - vpandq ymm8,ymm8,YMMWORD[$L$mask52x4] - vpandq ymm9,ymm9,YMMWORD[$L$mask52x4] - - shr r11b,4 - kmovb k1,r11d - - vpsubq ymm10{k1},ymm10,YMMWORD[$L$mask52x4] - - vpandq ymm10,ymm10,YMMWORD[$L$mask52x4] - - vpbroadcastq ymm0,r15 - vpblendd ymm11,ymm11,ymm0,3 - - - - vpsrlq ymm0,ymm11,52 - vpsrlq ymm1,ymm12,52 - vpsrlq ymm2,ymm13,52 - vpsrlq ymm19,ymm14,52 - vpsrlq ymm20,ymm15,52 - vpsrlq ymm21,ymm16,52 - vpsrlq ymm22,ymm17,52 - vpsrlq ymm23,ymm18,52 - - - valignq ymm23,ymm23,ymm22,3 - valignq ymm22,ymm22,ymm21,3 - valignq ymm21,ymm21,ymm20,3 - valignq ymm20,ymm20,ymm19,3 - valignq ymm19,ymm19,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm11,ymm11,YMMWORD[$L$mask52x4] - vpandq ymm12,ymm12,YMMWORD[$L$mask52x4] - vpandq ymm13,ymm13,YMMWORD[$L$mask52x4] - vpandq ymm14,ymm14,YMMWORD[$L$mask52x4] - vpandq ymm15,ymm15,YMMWORD[$L$mask52x4] - vpandq ymm16,ymm16,YMMWORD[$L$mask52x4] - vpandq ymm17,ymm17,YMMWORD[$L$mask52x4] - vpandq ymm18,ymm18,YMMWORD[$L$mask52x4] - - - vpaddq ymm11,ymm11,ymm0 - vpaddq ymm12,ymm12,ymm1 - vpaddq ymm13,ymm13,ymm2 - vpaddq ymm14,ymm14,ymm19 - vpaddq ymm15,ymm15,ymm20 - vpaddq ymm16,ymm16,ymm21 - vpaddq ymm17,ymm17,ymm22 - vpaddq ymm18,ymm18,ymm23 - - - - vpcmpuq k1,ymm11,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm12,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - shl r13b,4 - or r14b,r13b - - vpcmpuq k1,ymm13,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm14,YMMWORD[$L$mask52x4],6 - kmovb r13d,k1 - kmovb r12d,k2 - shl r12b,4 - or r13b,r12b - - vpcmpuq k1,ymm15,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm16,YMMWORD[$L$mask52x4],6 - kmovb r12d,k1 - kmovb r11d,k2 - shl r11b,4 - or r12b,r11b - - vpcmpuq k1,ymm17,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm18,YMMWORD[$L$mask52x4],6 - kmovb r11d,k1 - kmovb r10d,k2 - shl r10b,4 - or r11b,r10b - - add r14b,r14b - adc r13b,r13b - adc r12b,r12b - adc r11b,r11b - - - vpcmpuq k1,ymm11,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm12,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - shl r8b,4 - or r9b,r8b - - vpcmpuq k1,ymm13,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm14,YMMWORD[$L$mask52x4],0 - kmovb r8d,k1 - kmovb edx,k2 - shl dl,4 - or r8b,dl - - vpcmpuq k1,ymm15,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm16,YMMWORD[$L$mask52x4],0 - kmovb edx,k1 - kmovb ecx,k2 - shl cl,4 - or dl,cl - - vpcmpuq k1,ymm17,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm18,YMMWORD[$L$mask52x4],0 - kmovb ecx,k1 - kmovb ebx,k2 - shl bl,4 - or cl,bl - - add r14b,r9b - adc r13b,r8b - adc r12b,dl - adc r11b,cl - - xor r14b,r9b - xor r13b,r8b - xor r12b,dl - xor r11b,cl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r13d - shr r13b,4 - kmovb k4,r13d - kmovb k5,r12d - shr r12b,4 - kmovb k6,r12d - kmovb k7,r11d - - vpsubq ymm11{k1},ymm11,YMMWORD[$L$mask52x4] - vpsubq ymm12{k2},ymm12,YMMWORD[$L$mask52x4] - vpsubq ymm13{k3},ymm13,YMMWORD[$L$mask52x4] - vpsubq ymm14{k4},ymm14,YMMWORD[$L$mask52x4] - vpsubq ymm15{k5},ymm15,YMMWORD[$L$mask52x4] - vpsubq ymm16{k6},ymm16,YMMWORD[$L$mask52x4] - vpsubq ymm17{k7},ymm17,YMMWORD[$L$mask52x4] - - vpandq ymm11,ymm11,YMMWORD[$L$mask52x4] - vpandq ymm12,ymm12,YMMWORD[$L$mask52x4] - vpandq ymm13,ymm13,YMMWORD[$L$mask52x4] - vpandq ymm14,ymm14,YMMWORD[$L$mask52x4] - vpandq ymm15,ymm15,YMMWORD[$L$mask52x4] - vpandq ymm16,ymm16,YMMWORD[$L$mask52x4] - vpandq ymm17,ymm17,YMMWORD[$L$mask52x4] - - shr r11b,4 - kmovb k1,r11d - - vpsubq ymm18{k1},ymm18,YMMWORD[$L$mask52x4] - - vpandq ymm18,ymm18,YMMWORD[$L$mask52x4] - - vmovdqu64 YMMWORD[rdi],ymm3 - vmovdqu64 YMMWORD[32+rdi],ymm4 - vmovdqu64 YMMWORD[64+rdi],ymm5 - vmovdqu64 YMMWORD[96+rdi],ymm6 - vmovdqu64 YMMWORD[128+rdi],ymm7 - vmovdqu64 YMMWORD[160+rdi],ymm8 - vmovdqu64 YMMWORD[192+rdi],ymm9 - vmovdqu64 YMMWORD[224+rdi],ymm10 - - vmovdqu64 YMMWORD[256+rdi],ymm11 - vmovdqu64 YMMWORD[288+rdi],ymm12 - vmovdqu64 YMMWORD[320+rdi],ymm13 - vmovdqu64 YMMWORD[352+rdi],ymm14 - vmovdqu64 YMMWORD[384+rdi],ymm15 - vmovdqu64 YMMWORD[416+rdi],ymm16 - vmovdqu64 YMMWORD[448+rdi],ymm17 - vmovdqu64 YMMWORD[480+rdi],ymm18 - - vzeroupper - lea rax,[rsp] - - vmovdqa64 xmm6,XMMWORD[rax] - vmovdqa64 xmm7,XMMWORD[16+rax] - vmovdqa64 xmm8,XMMWORD[32+rax] - vmovdqa64 xmm9,XMMWORD[48+rax] - vmovdqa64 xmm10,XMMWORD[64+rax] - vmovdqa64 xmm11,XMMWORD[80+rax] - vmovdqa64 xmm12,XMMWORD[96+rax] - vmovdqa64 xmm13,XMMWORD[112+rax] - vmovdqa64 xmm14,XMMWORD[128+rax] - vmovdqa64 xmm15,XMMWORD[144+rax] - lea rax,[168+rsp] - mov r15,QWORD[rax] - - mov r14,QWORD[8+rax] - - mov r13,QWORD[16+rax] - - mov r12,QWORD[24+rax] - - mov rbp,QWORD[32+rax] - - mov rbx,QWORD[40+rax] - - lea rsp,[48+rax] - -$L$ossl_rsaz_amm52x30_x2_ifma256_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_ossl_rsaz_amm52x30_x2_ifma256: -section .text code align=64 - - -ALIGN 32 -global ossl_extract_multiplier_2x30_win5 - -ossl_extract_multiplier_2x30_win5: - -DB 243,15,30,250 - vmovdqa64 ymm30,YMMWORD[$L$ones] - vpbroadcastq ymm28,r8 - vpbroadcastq ymm29,r9 - lea rax,[16384+rdx] - - - vpxor xmm0,xmm0,xmm0 - vmovdqa64 ymm27,ymm0 - vmovdqa64 ymm1,ymm0 - vmovdqa64 ymm2,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm4,ymm0 - vmovdqa64 ymm5,ymm0 - vmovdqa64 ymm16,ymm0 - vmovdqa64 ymm17,ymm0 - vmovdqa64 ymm18,ymm0 - vmovdqa64 ymm19,ymm0 - vmovdqa64 ymm20,ymm0 - vmovdqa64 ymm21,ymm0 - vmovdqa64 ymm22,ymm0 - vmovdqa64 ymm23,ymm0 - vmovdqa64 ymm24,ymm0 - vmovdqa64 ymm25,ymm0 - -ALIGN 32 -$L$loop: - vpcmpq k1,ymm28,ymm27,0 - vpcmpq k2,ymm29,ymm27,0 - vmovdqu64 ymm26,YMMWORD[rdx] - vpblendmq ymm0{k1},ymm0,ymm26 - vmovdqu64 ymm26,YMMWORD[32+rdx] - vpblendmq ymm1{k1},ymm1,ymm26 - vmovdqu64 ymm26,YMMWORD[64+rdx] - vpblendmq ymm2{k1},ymm2,ymm26 - vmovdqu64 ymm26,YMMWORD[96+rdx] - vpblendmq ymm3{k1},ymm3,ymm26 - vmovdqu64 ymm26,YMMWORD[128+rdx] - vpblendmq ymm4{k1},ymm4,ymm26 - vmovdqu64 ymm26,YMMWORD[160+rdx] - vpblendmq ymm5{k1},ymm5,ymm26 - vmovdqu64 ymm26,YMMWORD[192+rdx] - vpblendmq ymm16{k1},ymm16,ymm26 - vmovdqu64 ymm26,YMMWORD[224+rdx] - vpblendmq ymm17{k1},ymm17,ymm26 - vmovdqu64 ymm26,YMMWORD[256+rdx] - vpblendmq ymm18{k2},ymm18,ymm26 - vmovdqu64 ymm26,YMMWORD[288+rdx] - vpblendmq ymm19{k2},ymm19,ymm26 - vmovdqu64 ymm26,YMMWORD[320+rdx] - vpblendmq ymm20{k2},ymm20,ymm26 - vmovdqu64 ymm26,YMMWORD[352+rdx] - vpblendmq ymm21{k2},ymm21,ymm26 - vmovdqu64 ymm26,YMMWORD[384+rdx] - vpblendmq ymm22{k2},ymm22,ymm26 - vmovdqu64 ymm26,YMMWORD[416+rdx] - vpblendmq ymm23{k2},ymm23,ymm26 - vmovdqu64 ymm26,YMMWORD[448+rdx] - vpblendmq ymm24{k2},ymm24,ymm26 - vmovdqu64 ymm26,YMMWORD[480+rdx] - vpblendmq ymm25{k2},ymm25,ymm26 - vpaddq ymm27,ymm27,ymm30 - add rdx,512 - cmp rax,rdx - jne NEAR $L$loop - vmovdqu64 YMMWORD[rcx],ymm0 - vmovdqu64 YMMWORD[32+rcx],ymm1 - vmovdqu64 YMMWORD[64+rcx],ymm2 - vmovdqu64 YMMWORD[96+rcx],ymm3 - vmovdqu64 YMMWORD[128+rcx],ymm4 - vmovdqu64 YMMWORD[160+rcx],ymm5 - vmovdqu64 YMMWORD[192+rcx],ymm16 - vmovdqu64 YMMWORD[224+rcx],ymm17 - vmovdqu64 YMMWORD[256+rcx],ymm18 - vmovdqu64 YMMWORD[288+rcx],ymm19 - vmovdqu64 YMMWORD[320+rcx],ymm20 - vmovdqu64 YMMWORD[352+rcx],ymm21 - vmovdqu64 YMMWORD[384+rcx],ymm22 - vmovdqu64 YMMWORD[416+rcx],ymm23 - vmovdqu64 YMMWORD[448+rcx],ymm24 - vmovdqu64 YMMWORD[480+rcx],ymm25 - - DB 0F3h,0C3h ;repret - - -section .data data align=8 - -ALIGN 32 -$L$ones: - DQ 1,1,1,1 -$L$zeros: - DQ 0,0,0,0 -EXTERN __imp_RtlVirtualUnwind - -ALIGN 16 -rsaz_avx_handler: - push rsi - push rdi - push rbx - push rbp - push r12 - push r13 - push r14 - push r15 - pushfq - sub rsp,64 - - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] - - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] - - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 - jb NEAR $L$common_seh_tail - - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 - jae NEAR $L$common_seh_tail - - mov rax,QWORD[152+r8] - - lea rsi,[rax] - lea rdi,[512+r8] - mov ecx,20 - DD 0xa548f3fc - - lea rax,[216+rax] - - mov rbx,QWORD[((-8))+rax] - mov rbp,QWORD[((-16))+rax] - mov r12,QWORD[((-24))+rax] - mov r13,QWORD[((-32))+rax] - mov r14,QWORD[((-40))+rax] - mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 - -$L$common_seh_tail: - mov rdi,QWORD[8+rax] - mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi - - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc - - mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx - call QWORD[__imp_RtlVirtualUnwind] - - mov eax,1 - add rsp,64 - popfq - pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx - pop rdi - pop rsi - DB 0F3h,0C3h ;repret - - -section .pdata rdata align=4 -ALIGN 4 - DD $L$SEH_begin_ossl_rsaz_amm52x30_x1_ifma256 wrt ..imagebase - DD $L$SEH_end_ossl_rsaz_amm52x30_x1_ifma256 wrt ..imagebase - DD $L$SEH_info_ossl_rsaz_amm52x30_x1_ifma256 wrt ..imagebase - - DD $L$SEH_begin_ossl_rsaz_amm52x30_x2_ifma256 wrt ..imagebase - DD $L$SEH_end_ossl_rsaz_amm52x30_x2_ifma256 wrt ..imagebase - DD $L$SEH_info_ossl_rsaz_amm52x30_x2_ifma256 wrt ..imagebase - -section .xdata rdata align=8 -ALIGN 8 -$L$SEH_info_ossl_rsaz_amm52x30_x1_ifma256: -DB 9,0,0,0 - DD rsaz_avx_handler wrt ..imagebase - DD $L$ossl_rsaz_amm52x30_x1_ifma256_body wrt ..imagebase,$L$ossl_rsaz_amm52x30_x1_ifma256_epilogue wrt ..imagebase -$L$SEH_info_ossl_rsaz_amm52x30_x2_ifma256: -DB 9,0,0,0 - DD rsaz_avx_handler wrt ..imagebase - DD $L$ossl_rsaz_amm52x30_x2_ifma256_body wrt ..imagebase,$L$ossl_rsaz_amm52x30_x2_ifma256_epilogue wrt ..imagebase diff --git a/openssl/src/crypto/bn/gen/windows_x64/rsaz-4k-avx512.asm b/openssl/src/crypto/bn/gen/windows_x64/rsaz-4k-avx512.asm deleted file mode 100644 index f1a13dcfa..000000000 --- a/openssl/src/crypto/bn/gen/windows_x64/rsaz-4k-avx512.asm +++ /dev/null @@ -1,1535 +0,0 @@ -default rel -%define XMMWORD -%define YMMWORD -%define ZMMWORD -section .text code align=64 - - -global ossl_rsaz_amm52x40_x1_ifma256 - -ALIGN 32 -ossl_rsaz_amm52x40_x1_ifma256: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ossl_rsaz_amm52x40_x1_ifma256: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - mov r8,QWORD[40+rsp] - - - -DB 243,15,30,250 - push rbx - - push rbp - - push r12 - - push r13 - - push r14 - - push r15 - - lea rsp,[((-168))+rsp] - vmovdqa64 XMMWORD[rsp],xmm6 - vmovdqa64 XMMWORD[16+rsp],xmm7 - vmovdqa64 XMMWORD[32+rsp],xmm8 - vmovdqa64 XMMWORD[48+rsp],xmm9 - vmovdqa64 XMMWORD[64+rsp],xmm10 - vmovdqa64 XMMWORD[80+rsp],xmm11 - vmovdqa64 XMMWORD[96+rsp],xmm12 - vmovdqa64 XMMWORD[112+rsp],xmm13 - vmovdqa64 XMMWORD[128+rsp],xmm14 - vmovdqa64 XMMWORD[144+rsp],xmm15 -$L$ossl_rsaz_amm52x40_x1_ifma256_body: - - vpxord ymm0,ymm0,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm4,ymm0 - vmovdqa64 ymm5,ymm0 - vmovdqa64 ymm6,ymm0 - vmovdqa64 ymm7,ymm0 - vmovdqa64 ymm8,ymm0 - vmovdqa64 ymm9,ymm0 - vmovdqa64 ymm10,ymm0 - vmovdqa64 ymm11,ymm0 - vmovdqa64 ymm12,ymm0 - - xor r9d,r9d - - mov r11,rdx - mov rax,0xfffffffffffff - - - mov ebx,10 - -ALIGN 32 -$L$loop10: - mov r13,QWORD[r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52luq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52luq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52luq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52luq ymm12,ymm2,YMMWORD[288+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm11,ymm10,1 - valignq ymm11,ymm12,ymm11,1 - valignq ymm12,ymm0,ymm12,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52huq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52huq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52huq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52huq ymm12,ymm2,YMMWORD[288+rcx] - mov r13,QWORD[8+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52luq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52luq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52luq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52luq ymm12,ymm2,YMMWORD[288+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm11,ymm10,1 - valignq ymm11,ymm12,ymm11,1 - valignq ymm12,ymm0,ymm12,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52huq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52huq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52huq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52huq ymm12,ymm2,YMMWORD[288+rcx] - mov r13,QWORD[16+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52luq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52luq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52luq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52luq ymm12,ymm2,YMMWORD[288+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm11,ymm10,1 - valignq ymm11,ymm12,ymm11,1 - valignq ymm12,ymm0,ymm12,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52huq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52huq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52huq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52huq ymm12,ymm2,YMMWORD[288+rcx] - mov r13,QWORD[24+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,r8 - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52luq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52luq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52luq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52luq ymm12,ymm2,YMMWORD[288+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm11,ymm10,1 - valignq ymm11,ymm12,ymm11,1 - valignq ymm12,ymm0,ymm12,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52huq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52huq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52huq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52huq ymm12,ymm2,YMMWORD[288+rcx] - lea r11,[32+r11] - dec ebx - jne NEAR $L$loop10 - - vpbroadcastq ymm0,r9 - vpblendd ymm3,ymm3,ymm0,3 - - - - vpsrlq ymm0,ymm3,52 - vpsrlq ymm1,ymm4,52 - vpsrlq ymm2,ymm5,52 - vpsrlq ymm23,ymm6,52 - vpsrlq ymm24,ymm7,52 - vpsrlq ymm25,ymm8,52 - vpsrlq ymm26,ymm9,52 - vpsrlq ymm27,ymm10,52 - vpsrlq ymm28,ymm11,52 - vpsrlq ymm29,ymm12,52 - - - valignq ymm29,ymm29,ymm28,3 - valignq ymm28,ymm28,ymm27,3 - valignq ymm27,ymm27,ymm26,3 - valignq ymm26,ymm26,ymm25,3 - valignq ymm25,ymm25,ymm24,3 - valignq ymm24,ymm24,ymm23,3 - valignq ymm23,ymm23,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm5,ymm5,YMMWORD[$L$mask52x4] - vpandq ymm6,ymm6,YMMWORD[$L$mask52x4] - vpandq ymm7,ymm7,YMMWORD[$L$mask52x4] - vpandq ymm8,ymm8,YMMWORD[$L$mask52x4] - vpandq ymm9,ymm9,YMMWORD[$L$mask52x4] - vpandq ymm10,ymm10,YMMWORD[$L$mask52x4] - vpandq ymm11,ymm11,YMMWORD[$L$mask52x4] - vpandq ymm12,ymm12,YMMWORD[$L$mask52x4] - - - vpaddq ymm3,ymm3,ymm0 - vpaddq ymm4,ymm4,ymm1 - vpaddq ymm5,ymm5,ymm2 - vpaddq ymm6,ymm6,ymm23 - vpaddq ymm7,ymm7,ymm24 - vpaddq ymm8,ymm8,ymm25 - vpaddq ymm9,ymm9,ymm26 - vpaddq ymm10,ymm10,ymm27 - vpaddq ymm11,ymm11,ymm28 - vpaddq ymm12,ymm12,ymm29 - - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm4,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - shl r13b,4 - or r14b,r13b - - vpcmpuq k1,ymm5,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm6,YMMWORD[$L$mask52x4],6 - kmovb r13d,k1 - kmovb r12d,k2 - shl r12b,4 - or r13b,r12b - - vpcmpuq k1,ymm7,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm8,YMMWORD[$L$mask52x4],6 - kmovb r12d,k1 - kmovb r11d,k2 - shl r11b,4 - or r12b,r11b - - vpcmpuq k1,ymm9,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm10,YMMWORD[$L$mask52x4],6 - kmovb r11d,k1 - kmovb r10d,k2 - shl r10b,4 - or r11b,r10b - - vpcmpuq k1,ymm11,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm12,YMMWORD[$L$mask52x4],6 - kmovb r10d,k1 - kmovb r9d,k2 - shl r9b,4 - or r10b,r9b - - add r14b,r14b - adc r13b,r13b - adc r12b,r12b - adc r11b,r11b - adc r10b,r10b - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm4,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - shl r8b,4 - or r9b,r8b - - vpcmpuq k1,ymm5,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm6,YMMWORD[$L$mask52x4],0 - kmovb r8d,k1 - kmovb edx,k2 - shl dl,4 - or r8b,dl - - vpcmpuq k1,ymm7,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm8,YMMWORD[$L$mask52x4],0 - kmovb edx,k1 - kmovb ecx,k2 - shl cl,4 - or dl,cl - - vpcmpuq k1,ymm9,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm10,YMMWORD[$L$mask52x4],0 - kmovb ecx,k1 - kmovb ebx,k2 - shl bl,4 - or cl,bl - - vpcmpuq k1,ymm11,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm12,YMMWORD[$L$mask52x4],0 - kmovb ebx,k1 - kmovb eax,k2 - shl al,4 - or bl,al - - add r14b,r9b - adc r13b,r8b - adc r12b,dl - adc r11b,cl - adc r10b,bl - - xor r14b,r9b - xor r13b,r8b - xor r12b,dl - xor r11b,cl - xor r10b,bl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r13d - shr r13b,4 - kmovb k4,r13d - kmovb k5,r12d - shr r12b,4 - kmovb k6,r12d - kmovb k7,r11d - - vpsubq ymm3{k1},ymm3,YMMWORD[$L$mask52x4] - vpsubq ymm4{k2},ymm4,YMMWORD[$L$mask52x4] - vpsubq ymm5{k3},ymm5,YMMWORD[$L$mask52x4] - vpsubq ymm6{k4},ymm6,YMMWORD[$L$mask52x4] - vpsubq ymm7{k5},ymm7,YMMWORD[$L$mask52x4] - vpsubq ymm8{k6},ymm8,YMMWORD[$L$mask52x4] - vpsubq ymm9{k7},ymm9,YMMWORD[$L$mask52x4] - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm5,ymm5,YMMWORD[$L$mask52x4] - vpandq ymm6,ymm6,YMMWORD[$L$mask52x4] - vpandq ymm7,ymm7,YMMWORD[$L$mask52x4] - vpandq ymm8,ymm8,YMMWORD[$L$mask52x4] - vpandq ymm9,ymm9,YMMWORD[$L$mask52x4] - - shr r11b,4 - kmovb k1,r11d - kmovb k2,r10d - shr r10b,4 - kmovb k3,r10d - - vpsubq ymm10{k1},ymm10,YMMWORD[$L$mask52x4] - vpsubq ymm11{k2},ymm11,YMMWORD[$L$mask52x4] - vpsubq ymm12{k3},ymm12,YMMWORD[$L$mask52x4] - - vpandq ymm10,ymm10,YMMWORD[$L$mask52x4] - vpandq ymm11,ymm11,YMMWORD[$L$mask52x4] - vpandq ymm12,ymm12,YMMWORD[$L$mask52x4] - - vmovdqu64 YMMWORD[rdi],ymm3 - vmovdqu64 YMMWORD[32+rdi],ymm4 - vmovdqu64 YMMWORD[64+rdi],ymm5 - vmovdqu64 YMMWORD[96+rdi],ymm6 - vmovdqu64 YMMWORD[128+rdi],ymm7 - vmovdqu64 YMMWORD[160+rdi],ymm8 - vmovdqu64 YMMWORD[192+rdi],ymm9 - vmovdqu64 YMMWORD[224+rdi],ymm10 - vmovdqu64 YMMWORD[256+rdi],ymm11 - vmovdqu64 YMMWORD[288+rdi],ymm12 - - vzeroupper - lea rax,[rsp] - - vmovdqa64 xmm6,XMMWORD[rax] - vmovdqa64 xmm7,XMMWORD[16+rax] - vmovdqa64 xmm8,XMMWORD[32+rax] - vmovdqa64 xmm9,XMMWORD[48+rax] - vmovdqa64 xmm10,XMMWORD[64+rax] - vmovdqa64 xmm11,XMMWORD[80+rax] - vmovdqa64 xmm12,XMMWORD[96+rax] - vmovdqa64 xmm13,XMMWORD[112+rax] - vmovdqa64 xmm14,XMMWORD[128+rax] - vmovdqa64 xmm15,XMMWORD[144+rax] - lea rax,[168+rsp] - mov r15,QWORD[rax] - - mov r14,QWORD[8+rax] - - mov r13,QWORD[16+rax] - - mov r12,QWORD[24+rax] - - mov rbp,QWORD[32+rax] - - mov rbx,QWORD[40+rax] - - lea rsp,[48+rax] - -$L$ossl_rsaz_amm52x40_x1_ifma256_epilogue: - - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_ossl_rsaz_amm52x40_x1_ifma256: -section .data data align=8 - -ALIGN 32 -$L$mask52x4: - DQ 0xfffffffffffff - DQ 0xfffffffffffff - DQ 0xfffffffffffff - DQ 0xfffffffffffff -section .text code align=64 - - -global ossl_rsaz_amm52x40_x2_ifma256 - -ALIGN 32 -ossl_rsaz_amm52x40_x2_ifma256: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ossl_rsaz_amm52x40_x2_ifma256: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - mov r8,QWORD[40+rsp] - - - -DB 243,15,30,250 - push rbx - - push rbp - - push r12 - - push r13 - - push r14 - - push r15 - - lea rsp,[((-168))+rsp] - vmovdqa64 XMMWORD[rsp],xmm6 - vmovdqa64 XMMWORD[16+rsp],xmm7 - vmovdqa64 XMMWORD[32+rsp],xmm8 - vmovdqa64 XMMWORD[48+rsp],xmm9 - vmovdqa64 XMMWORD[64+rsp],xmm10 - vmovdqa64 XMMWORD[80+rsp],xmm11 - vmovdqa64 XMMWORD[96+rsp],xmm12 - vmovdqa64 XMMWORD[112+rsp],xmm13 - vmovdqa64 XMMWORD[128+rsp],xmm14 - vmovdqa64 XMMWORD[144+rsp],xmm15 -$L$ossl_rsaz_amm52x40_x2_ifma256_body: - - vpxord ymm0,ymm0,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm4,ymm0 - vmovdqa64 ymm5,ymm0 - vmovdqa64 ymm6,ymm0 - vmovdqa64 ymm7,ymm0 - vmovdqa64 ymm8,ymm0 - vmovdqa64 ymm9,ymm0 - vmovdqa64 ymm10,ymm0 - vmovdqa64 ymm11,ymm0 - vmovdqa64 ymm12,ymm0 - - vmovdqa64 ymm13,ymm0 - vmovdqa64 ymm14,ymm0 - vmovdqa64 ymm15,ymm0 - vmovdqa64 ymm16,ymm0 - vmovdqa64 ymm17,ymm0 - vmovdqa64 ymm18,ymm0 - vmovdqa64 ymm19,ymm0 - vmovdqa64 ymm20,ymm0 - vmovdqa64 ymm21,ymm0 - vmovdqa64 ymm22,ymm0 - - - xor r9d,r9d - xor r15d,r15d - - mov r11,rdx - mov rax,0xfffffffffffff - - mov ebx,40 - -ALIGN 32 -$L$loop40: - mov r13,QWORD[r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[rsi] - mulx r12,r13,r13 - add r9,r13 - mov r10,r12 - adc r10,0 - - mov r13,QWORD[r8] - imul r13,r9 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[rcx] - mulx r12,r13,r13 - add r9,r13 - adc r10,r12 - - shr r9,52 - sal r10,12 - or r9,r10 - - vpmadd52luq ymm3,ymm1,YMMWORD[rsi] - vpmadd52luq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52luq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52luq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52luq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52luq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52luq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52luq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52luq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52luq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52luq ymm3,ymm2,YMMWORD[rcx] - vpmadd52luq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52luq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52luq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52luq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52luq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52luq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52luq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52luq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52luq ymm12,ymm2,YMMWORD[288+rcx] - - - valignq ymm3,ymm4,ymm3,1 - valignq ymm4,ymm5,ymm4,1 - valignq ymm5,ymm6,ymm5,1 - valignq ymm6,ymm7,ymm6,1 - valignq ymm7,ymm8,ymm7,1 - valignq ymm8,ymm9,ymm8,1 - valignq ymm9,ymm10,ymm9,1 - valignq ymm10,ymm11,ymm10,1 - valignq ymm11,ymm12,ymm11,1 - valignq ymm12,ymm0,ymm12,1 - - vmovq r13,xmm3 - add r9,r13 - - vpmadd52huq ymm3,ymm1,YMMWORD[rsi] - vpmadd52huq ymm4,ymm1,YMMWORD[32+rsi] - vpmadd52huq ymm5,ymm1,YMMWORD[64+rsi] - vpmadd52huq ymm6,ymm1,YMMWORD[96+rsi] - vpmadd52huq ymm7,ymm1,YMMWORD[128+rsi] - vpmadd52huq ymm8,ymm1,YMMWORD[160+rsi] - vpmadd52huq ymm9,ymm1,YMMWORD[192+rsi] - vpmadd52huq ymm10,ymm1,YMMWORD[224+rsi] - vpmadd52huq ymm11,ymm1,YMMWORD[256+rsi] - vpmadd52huq ymm12,ymm1,YMMWORD[288+rsi] - - vpmadd52huq ymm3,ymm2,YMMWORD[rcx] - vpmadd52huq ymm4,ymm2,YMMWORD[32+rcx] - vpmadd52huq ymm5,ymm2,YMMWORD[64+rcx] - vpmadd52huq ymm6,ymm2,YMMWORD[96+rcx] - vpmadd52huq ymm7,ymm2,YMMWORD[128+rcx] - vpmadd52huq ymm8,ymm2,YMMWORD[160+rcx] - vpmadd52huq ymm9,ymm2,YMMWORD[192+rcx] - vpmadd52huq ymm10,ymm2,YMMWORD[224+rcx] - vpmadd52huq ymm11,ymm2,YMMWORD[256+rcx] - vpmadd52huq ymm12,ymm2,YMMWORD[288+rcx] - mov r13,QWORD[320+r11] - - vpbroadcastq ymm1,r13 - mov rdx,QWORD[320+rsi] - mulx r12,r13,r13 - add r15,r13 - mov r10,r12 - adc r10,0 - - mov r13,QWORD[8+r8] - imul r13,r15 - and r13,rax - - vpbroadcastq ymm2,r13 - mov rdx,QWORD[320+rcx] - mulx r12,r13,r13 - add r15,r13 - adc r10,r12 - - shr r15,52 - sal r10,12 - or r15,r10 - - vpmadd52luq ymm13,ymm1,YMMWORD[320+rsi] - vpmadd52luq ymm14,ymm1,YMMWORD[352+rsi] - vpmadd52luq ymm15,ymm1,YMMWORD[384+rsi] - vpmadd52luq ymm16,ymm1,YMMWORD[416+rsi] - vpmadd52luq ymm17,ymm1,YMMWORD[448+rsi] - vpmadd52luq ymm18,ymm1,YMMWORD[480+rsi] - vpmadd52luq ymm19,ymm1,YMMWORD[512+rsi] - vpmadd52luq ymm20,ymm1,YMMWORD[544+rsi] - vpmadd52luq ymm21,ymm1,YMMWORD[576+rsi] - vpmadd52luq ymm22,ymm1,YMMWORD[608+rsi] - - vpmadd52luq ymm13,ymm2,YMMWORD[320+rcx] - vpmadd52luq ymm14,ymm2,YMMWORD[352+rcx] - vpmadd52luq ymm15,ymm2,YMMWORD[384+rcx] - vpmadd52luq ymm16,ymm2,YMMWORD[416+rcx] - vpmadd52luq ymm17,ymm2,YMMWORD[448+rcx] - vpmadd52luq ymm18,ymm2,YMMWORD[480+rcx] - vpmadd52luq ymm19,ymm2,YMMWORD[512+rcx] - vpmadd52luq ymm20,ymm2,YMMWORD[544+rcx] - vpmadd52luq ymm21,ymm2,YMMWORD[576+rcx] - vpmadd52luq ymm22,ymm2,YMMWORD[608+rcx] - - - valignq ymm13,ymm14,ymm13,1 - valignq ymm14,ymm15,ymm14,1 - valignq ymm15,ymm16,ymm15,1 - valignq ymm16,ymm17,ymm16,1 - valignq ymm17,ymm18,ymm17,1 - valignq ymm18,ymm19,ymm18,1 - valignq ymm19,ymm20,ymm19,1 - valignq ymm20,ymm21,ymm20,1 - valignq ymm21,ymm22,ymm21,1 - valignq ymm22,ymm0,ymm22,1 - - vmovq r13,xmm13 - add r15,r13 - - vpmadd52huq ymm13,ymm1,YMMWORD[320+rsi] - vpmadd52huq ymm14,ymm1,YMMWORD[352+rsi] - vpmadd52huq ymm15,ymm1,YMMWORD[384+rsi] - vpmadd52huq ymm16,ymm1,YMMWORD[416+rsi] - vpmadd52huq ymm17,ymm1,YMMWORD[448+rsi] - vpmadd52huq ymm18,ymm1,YMMWORD[480+rsi] - vpmadd52huq ymm19,ymm1,YMMWORD[512+rsi] - vpmadd52huq ymm20,ymm1,YMMWORD[544+rsi] - vpmadd52huq ymm21,ymm1,YMMWORD[576+rsi] - vpmadd52huq ymm22,ymm1,YMMWORD[608+rsi] - - vpmadd52huq ymm13,ymm2,YMMWORD[320+rcx] - vpmadd52huq ymm14,ymm2,YMMWORD[352+rcx] - vpmadd52huq ymm15,ymm2,YMMWORD[384+rcx] - vpmadd52huq ymm16,ymm2,YMMWORD[416+rcx] - vpmadd52huq ymm17,ymm2,YMMWORD[448+rcx] - vpmadd52huq ymm18,ymm2,YMMWORD[480+rcx] - vpmadd52huq ymm19,ymm2,YMMWORD[512+rcx] - vpmadd52huq ymm20,ymm2,YMMWORD[544+rcx] - vpmadd52huq ymm21,ymm2,YMMWORD[576+rcx] - vpmadd52huq ymm22,ymm2,YMMWORD[608+rcx] - lea r11,[8+r11] - dec ebx - jne NEAR $L$loop40 - - vpbroadcastq ymm0,r9 - vpblendd ymm3,ymm3,ymm0,3 - - - - vpsrlq ymm0,ymm3,52 - vpsrlq ymm1,ymm4,52 - vpsrlq ymm2,ymm5,52 - vpsrlq ymm23,ymm6,52 - vpsrlq ymm24,ymm7,52 - vpsrlq ymm25,ymm8,52 - vpsrlq ymm26,ymm9,52 - vpsrlq ymm27,ymm10,52 - vpsrlq ymm28,ymm11,52 - vpsrlq ymm29,ymm12,52 - - - valignq ymm29,ymm29,ymm28,3 - valignq ymm28,ymm28,ymm27,3 - valignq ymm27,ymm27,ymm26,3 - valignq ymm26,ymm26,ymm25,3 - valignq ymm25,ymm25,ymm24,3 - valignq ymm24,ymm24,ymm23,3 - valignq ymm23,ymm23,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm5,ymm5,YMMWORD[$L$mask52x4] - vpandq ymm6,ymm6,YMMWORD[$L$mask52x4] - vpandq ymm7,ymm7,YMMWORD[$L$mask52x4] - vpandq ymm8,ymm8,YMMWORD[$L$mask52x4] - vpandq ymm9,ymm9,YMMWORD[$L$mask52x4] - vpandq ymm10,ymm10,YMMWORD[$L$mask52x4] - vpandq ymm11,ymm11,YMMWORD[$L$mask52x4] - vpandq ymm12,ymm12,YMMWORD[$L$mask52x4] - - - vpaddq ymm3,ymm3,ymm0 - vpaddq ymm4,ymm4,ymm1 - vpaddq ymm5,ymm5,ymm2 - vpaddq ymm6,ymm6,ymm23 - vpaddq ymm7,ymm7,ymm24 - vpaddq ymm8,ymm8,ymm25 - vpaddq ymm9,ymm9,ymm26 - vpaddq ymm10,ymm10,ymm27 - vpaddq ymm11,ymm11,ymm28 - vpaddq ymm12,ymm12,ymm29 - - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm4,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - shl r13b,4 - or r14b,r13b - - vpcmpuq k1,ymm5,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm6,YMMWORD[$L$mask52x4],6 - kmovb r13d,k1 - kmovb r12d,k2 - shl r12b,4 - or r13b,r12b - - vpcmpuq k1,ymm7,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm8,YMMWORD[$L$mask52x4],6 - kmovb r12d,k1 - kmovb r11d,k2 - shl r11b,4 - or r12b,r11b - - vpcmpuq k1,ymm9,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm10,YMMWORD[$L$mask52x4],6 - kmovb r11d,k1 - kmovb r10d,k2 - shl r10b,4 - or r11b,r10b - - vpcmpuq k1,ymm11,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm12,YMMWORD[$L$mask52x4],6 - kmovb r10d,k1 - kmovb r9d,k2 - shl r9b,4 - or r10b,r9b - - add r14b,r14b - adc r13b,r13b - adc r12b,r12b - adc r11b,r11b - adc r10b,r10b - - - vpcmpuq k1,ymm3,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm4,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - shl r8b,4 - or r9b,r8b - - vpcmpuq k1,ymm5,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm6,YMMWORD[$L$mask52x4],0 - kmovb r8d,k1 - kmovb edx,k2 - shl dl,4 - or r8b,dl - - vpcmpuq k1,ymm7,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm8,YMMWORD[$L$mask52x4],0 - kmovb edx,k1 - kmovb ecx,k2 - shl cl,4 - or dl,cl - - vpcmpuq k1,ymm9,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm10,YMMWORD[$L$mask52x4],0 - kmovb ecx,k1 - kmovb ebx,k2 - shl bl,4 - or cl,bl - - vpcmpuq k1,ymm11,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm12,YMMWORD[$L$mask52x4],0 - kmovb ebx,k1 - kmovb eax,k2 - shl al,4 - or bl,al - - add r14b,r9b - adc r13b,r8b - adc r12b,dl - adc r11b,cl - adc r10b,bl - - xor r14b,r9b - xor r13b,r8b - xor r12b,dl - xor r11b,cl - xor r10b,bl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r13d - shr r13b,4 - kmovb k4,r13d - kmovb k5,r12d - shr r12b,4 - kmovb k6,r12d - kmovb k7,r11d - - vpsubq ymm3{k1},ymm3,YMMWORD[$L$mask52x4] - vpsubq ymm4{k2},ymm4,YMMWORD[$L$mask52x4] - vpsubq ymm5{k3},ymm5,YMMWORD[$L$mask52x4] - vpsubq ymm6{k4},ymm6,YMMWORD[$L$mask52x4] - vpsubq ymm7{k5},ymm7,YMMWORD[$L$mask52x4] - vpsubq ymm8{k6},ymm8,YMMWORD[$L$mask52x4] - vpsubq ymm9{k7},ymm9,YMMWORD[$L$mask52x4] - - vpandq ymm3,ymm3,YMMWORD[$L$mask52x4] - vpandq ymm4,ymm4,YMMWORD[$L$mask52x4] - vpandq ymm5,ymm5,YMMWORD[$L$mask52x4] - vpandq ymm6,ymm6,YMMWORD[$L$mask52x4] - vpandq ymm7,ymm7,YMMWORD[$L$mask52x4] - vpandq ymm8,ymm8,YMMWORD[$L$mask52x4] - vpandq ymm9,ymm9,YMMWORD[$L$mask52x4] - - shr r11b,4 - kmovb k1,r11d - kmovb k2,r10d - shr r10b,4 - kmovb k3,r10d - - vpsubq ymm10{k1},ymm10,YMMWORD[$L$mask52x4] - vpsubq ymm11{k2},ymm11,YMMWORD[$L$mask52x4] - vpsubq ymm12{k3},ymm12,YMMWORD[$L$mask52x4] - - vpandq ymm10,ymm10,YMMWORD[$L$mask52x4] - vpandq ymm11,ymm11,YMMWORD[$L$mask52x4] - vpandq ymm12,ymm12,YMMWORD[$L$mask52x4] - - vpbroadcastq ymm0,r15 - vpblendd ymm13,ymm13,ymm0,3 - - - - vpsrlq ymm0,ymm13,52 - vpsrlq ymm1,ymm14,52 - vpsrlq ymm2,ymm15,52 - vpsrlq ymm23,ymm16,52 - vpsrlq ymm24,ymm17,52 - vpsrlq ymm25,ymm18,52 - vpsrlq ymm26,ymm19,52 - vpsrlq ymm27,ymm20,52 - vpsrlq ymm28,ymm21,52 - vpsrlq ymm29,ymm22,52 - - - valignq ymm29,ymm29,ymm28,3 - valignq ymm28,ymm28,ymm27,3 - valignq ymm27,ymm27,ymm26,3 - valignq ymm26,ymm26,ymm25,3 - valignq ymm25,ymm25,ymm24,3 - valignq ymm24,ymm24,ymm23,3 - valignq ymm23,ymm23,ymm2,3 - valignq ymm2,ymm2,ymm1,3 - valignq ymm1,ymm1,ymm0,3 - valignq ymm0,ymm0,YMMWORD[$L$zeros],3 - - - vpandq ymm13,ymm13,YMMWORD[$L$mask52x4] - vpandq ymm14,ymm14,YMMWORD[$L$mask52x4] - vpandq ymm15,ymm15,YMMWORD[$L$mask52x4] - vpandq ymm16,ymm16,YMMWORD[$L$mask52x4] - vpandq ymm17,ymm17,YMMWORD[$L$mask52x4] - vpandq ymm18,ymm18,YMMWORD[$L$mask52x4] - vpandq ymm19,ymm19,YMMWORD[$L$mask52x4] - vpandq ymm20,ymm20,YMMWORD[$L$mask52x4] - vpandq ymm21,ymm21,YMMWORD[$L$mask52x4] - vpandq ymm22,ymm22,YMMWORD[$L$mask52x4] - - - vpaddq ymm13,ymm13,ymm0 - vpaddq ymm14,ymm14,ymm1 - vpaddq ymm15,ymm15,ymm2 - vpaddq ymm16,ymm16,ymm23 - vpaddq ymm17,ymm17,ymm24 - vpaddq ymm18,ymm18,ymm25 - vpaddq ymm19,ymm19,ymm26 - vpaddq ymm20,ymm20,ymm27 - vpaddq ymm21,ymm21,ymm28 - vpaddq ymm22,ymm22,ymm29 - - - - vpcmpuq k1,ymm13,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm14,YMMWORD[$L$mask52x4],6 - kmovb r14d,k1 - kmovb r13d,k2 - shl r13b,4 - or r14b,r13b - - vpcmpuq k1,ymm15,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm16,YMMWORD[$L$mask52x4],6 - kmovb r13d,k1 - kmovb r12d,k2 - shl r12b,4 - or r13b,r12b - - vpcmpuq k1,ymm17,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm18,YMMWORD[$L$mask52x4],6 - kmovb r12d,k1 - kmovb r11d,k2 - shl r11b,4 - or r12b,r11b - - vpcmpuq k1,ymm19,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm20,YMMWORD[$L$mask52x4],6 - kmovb r11d,k1 - kmovb r10d,k2 - shl r10b,4 - or r11b,r10b - - vpcmpuq k1,ymm21,YMMWORD[$L$mask52x4],6 - vpcmpuq k2,ymm22,YMMWORD[$L$mask52x4],6 - kmovb r10d,k1 - kmovb r9d,k2 - shl r9b,4 - or r10b,r9b - - add r14b,r14b - adc r13b,r13b - adc r12b,r12b - adc r11b,r11b - adc r10b,r10b - - - vpcmpuq k1,ymm13,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm14,YMMWORD[$L$mask52x4],0 - kmovb r9d,k1 - kmovb r8d,k2 - shl r8b,4 - or r9b,r8b - - vpcmpuq k1,ymm15,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm16,YMMWORD[$L$mask52x4],0 - kmovb r8d,k1 - kmovb edx,k2 - shl dl,4 - or r8b,dl - - vpcmpuq k1,ymm17,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm18,YMMWORD[$L$mask52x4],0 - kmovb edx,k1 - kmovb ecx,k2 - shl cl,4 - or dl,cl - - vpcmpuq k1,ymm19,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm20,YMMWORD[$L$mask52x4],0 - kmovb ecx,k1 - kmovb ebx,k2 - shl bl,4 - or cl,bl - - vpcmpuq k1,ymm21,YMMWORD[$L$mask52x4],0 - vpcmpuq k2,ymm22,YMMWORD[$L$mask52x4],0 - kmovb ebx,k1 - kmovb eax,k2 - shl al,4 - or bl,al - - add r14b,r9b - adc r13b,r8b - adc r12b,dl - adc r11b,cl - adc r10b,bl - - xor r14b,r9b - xor r13b,r8b - xor r12b,dl - xor r11b,cl - xor r10b,bl - - kmovb k1,r14d - shr r14b,4 - kmovb k2,r14d - kmovb k3,r13d - shr r13b,4 - kmovb k4,r13d - kmovb k5,r12d - shr r12b,4 - kmovb k6,r12d - kmovb k7,r11d - - vpsubq ymm13{k1},ymm13,YMMWORD[$L$mask52x4] - vpsubq ymm14{k2},ymm14,YMMWORD[$L$mask52x4] - vpsubq ymm15{k3},ymm15,YMMWORD[$L$mask52x4] - vpsubq ymm16{k4},ymm16,YMMWORD[$L$mask52x4] - vpsubq ymm17{k5},ymm17,YMMWORD[$L$mask52x4] - vpsubq ymm18{k6},ymm18,YMMWORD[$L$mask52x4] - vpsubq ymm19{k7},ymm19,YMMWORD[$L$mask52x4] - - vpandq ymm13,ymm13,YMMWORD[$L$mask52x4] - vpandq ymm14,ymm14,YMMWORD[$L$mask52x4] - vpandq ymm15,ymm15,YMMWORD[$L$mask52x4] - vpandq ymm16,ymm16,YMMWORD[$L$mask52x4] - vpandq ymm17,ymm17,YMMWORD[$L$mask52x4] - vpandq ymm18,ymm18,YMMWORD[$L$mask52x4] - vpandq ymm19,ymm19,YMMWORD[$L$mask52x4] - - shr r11b,4 - kmovb k1,r11d - kmovb k2,r10d - shr r10b,4 - kmovb k3,r10d - - vpsubq ymm20{k1},ymm20,YMMWORD[$L$mask52x4] - vpsubq ymm21{k2},ymm21,YMMWORD[$L$mask52x4] - vpsubq ymm22{k3},ymm22,YMMWORD[$L$mask52x4] - - vpandq ymm20,ymm20,YMMWORD[$L$mask52x4] - vpandq ymm21,ymm21,YMMWORD[$L$mask52x4] - vpandq ymm22,ymm22,YMMWORD[$L$mask52x4] - - vmovdqu64 YMMWORD[rdi],ymm3 - vmovdqu64 YMMWORD[32+rdi],ymm4 - vmovdqu64 YMMWORD[64+rdi],ymm5 - vmovdqu64 YMMWORD[96+rdi],ymm6 - vmovdqu64 YMMWORD[128+rdi],ymm7 - vmovdqu64 YMMWORD[160+rdi],ymm8 - vmovdqu64 YMMWORD[192+rdi],ymm9 - vmovdqu64 YMMWORD[224+rdi],ymm10 - vmovdqu64 YMMWORD[256+rdi],ymm11 - vmovdqu64 YMMWORD[288+rdi],ymm12 - - vmovdqu64 YMMWORD[320+rdi],ymm13 - vmovdqu64 YMMWORD[352+rdi],ymm14 - vmovdqu64 YMMWORD[384+rdi],ymm15 - vmovdqu64 YMMWORD[416+rdi],ymm16 - vmovdqu64 YMMWORD[448+rdi],ymm17 - vmovdqu64 YMMWORD[480+rdi],ymm18 - vmovdqu64 YMMWORD[512+rdi],ymm19 - vmovdqu64 YMMWORD[544+rdi],ymm20 - vmovdqu64 YMMWORD[576+rdi],ymm21 - vmovdqu64 YMMWORD[608+rdi],ymm22 - - vzeroupper - lea rax,[rsp] - - vmovdqa64 xmm6,XMMWORD[rax] - vmovdqa64 xmm7,XMMWORD[16+rax] - vmovdqa64 xmm8,XMMWORD[32+rax] - vmovdqa64 xmm9,XMMWORD[48+rax] - vmovdqa64 xmm10,XMMWORD[64+rax] - vmovdqa64 xmm11,XMMWORD[80+rax] - vmovdqa64 xmm12,XMMWORD[96+rax] - vmovdqa64 xmm13,XMMWORD[112+rax] - vmovdqa64 xmm14,XMMWORD[128+rax] - vmovdqa64 xmm15,XMMWORD[144+rax] - lea rax,[168+rsp] - mov r15,QWORD[rax] - - mov r14,QWORD[8+rax] - - mov r13,QWORD[16+rax] - - mov r12,QWORD[24+rax] - - mov rbp,QWORD[32+rax] - - mov rbx,QWORD[40+rax] - - lea rsp,[48+rax] - -$L$ossl_rsaz_amm52x40_x2_ifma256_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_ossl_rsaz_amm52x40_x2_ifma256: -section .text code align=64 - - -ALIGN 32 -global ossl_extract_multiplier_2x40_win5 - -ossl_extract_multiplier_2x40_win5: - -DB 243,15,30,250 - vmovdqa64 ymm24,YMMWORD[$L$ones] - vpbroadcastq ymm22,r8 - vpbroadcastq ymm23,r9 - lea rax,[20480+rdx] - - - mov r10,rdx - - - vpxor xmm0,xmm0,xmm0 - vmovdqa64 ymm1,ymm0 - vmovdqa64 ymm2,ymm0 - vmovdqa64 ymm3,ymm0 - vmovdqa64 ymm4,ymm0 - vmovdqa64 ymm5,ymm0 - vmovdqa64 ymm16,ymm0 - vmovdqa64 ymm17,ymm0 - vmovdqa64 ymm18,ymm0 - vmovdqa64 ymm19,ymm0 - vpxorq ymm21,ymm21,ymm21 -ALIGN 32 -$L$loop_0: - vpcmpq k1,ymm22,ymm21,0 - vmovdqu64 ymm20,YMMWORD[rdx] - vpblendmq ymm0{k1},ymm0,ymm20 - vmovdqu64 ymm20,YMMWORD[32+rdx] - vpblendmq ymm1{k1},ymm1,ymm20 - vmovdqu64 ymm20,YMMWORD[64+rdx] - vpblendmq ymm2{k1},ymm2,ymm20 - vmovdqu64 ymm20,YMMWORD[96+rdx] - vpblendmq ymm3{k1},ymm3,ymm20 - vmovdqu64 ymm20,YMMWORD[128+rdx] - vpblendmq ymm4{k1},ymm4,ymm20 - vmovdqu64 ymm20,YMMWORD[160+rdx] - vpblendmq ymm5{k1},ymm5,ymm20 - vmovdqu64 ymm20,YMMWORD[192+rdx] - vpblendmq ymm16{k1},ymm16,ymm20 - vmovdqu64 ymm20,YMMWORD[224+rdx] - vpblendmq ymm17{k1},ymm17,ymm20 - vmovdqu64 ymm20,YMMWORD[256+rdx] - vpblendmq ymm18{k1},ymm18,ymm20 - vmovdqu64 ymm20,YMMWORD[288+rdx] - vpblendmq ymm19{k1},ymm19,ymm20 - vpaddq ymm21,ymm21,ymm24 - add rdx,640 - cmp rax,rdx - jne NEAR $L$loop_0 - vmovdqu64 YMMWORD[rcx],ymm0 - vmovdqu64 YMMWORD[32+rcx],ymm1 - vmovdqu64 YMMWORD[64+rcx],ymm2 - vmovdqu64 YMMWORD[96+rcx],ymm3 - vmovdqu64 YMMWORD[128+rcx],ymm4 - vmovdqu64 YMMWORD[160+rcx],ymm5 - vmovdqu64 YMMWORD[192+rcx],ymm16 - vmovdqu64 YMMWORD[224+rcx],ymm17 - vmovdqu64 YMMWORD[256+rcx],ymm18 - vmovdqu64 YMMWORD[288+rcx],ymm19 - mov rdx,r10 - vpxorq ymm21,ymm21,ymm21 -ALIGN 32 -$L$loop_320: - vpcmpq k1,ymm23,ymm21,0 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpblendmq ymm0{k1},ymm0,ymm20 - vmovdqu64 ymm20,YMMWORD[352+rdx] - vpblendmq ymm1{k1},ymm1,ymm20 - vmovdqu64 ymm20,YMMWORD[384+rdx] - vpblendmq ymm2{k1},ymm2,ymm20 - vmovdqu64 ymm20,YMMWORD[416+rdx] - vpblendmq ymm3{k1},ymm3,ymm20 - vmovdqu64 ymm20,YMMWORD[448+rdx] - vpblendmq ymm4{k1},ymm4,ymm20 - vmovdqu64 ymm20,YMMWORD[480+rdx] - vpblendmq ymm5{k1},ymm5,ymm20 - vmovdqu64 ymm20,YMMWORD[512+rdx] - vpblendmq ymm16{k1},ymm16,ymm20 - vmovdqu64 ymm20,YMMWORD[544+rdx] - vpblendmq ymm17{k1},ymm17,ymm20 - vmovdqu64 ymm20,YMMWORD[576+rdx] - vpblendmq ymm18{k1},ymm18,ymm20 - vmovdqu64 ymm20,YMMWORD[608+rdx] - vpblendmq ymm19{k1},ymm19,ymm20 - vpaddq ymm21,ymm21,ymm24 - add rdx,640 - cmp rax,rdx - jne NEAR $L$loop_320 - vmovdqu64 YMMWORD[320+rcx],ymm0 - vmovdqu64 YMMWORD[352+rcx],ymm1 - vmovdqu64 YMMWORD[384+rcx],ymm2 - vmovdqu64 YMMWORD[416+rcx],ymm3 - vmovdqu64 YMMWORD[448+rcx],ymm4 - vmovdqu64 YMMWORD[480+rcx],ymm5 - vmovdqu64 YMMWORD[512+rcx],ymm16 - vmovdqu64 YMMWORD[544+rcx],ymm17 - vmovdqu64 YMMWORD[576+rcx],ymm18 - vmovdqu64 YMMWORD[608+rcx],ymm19 - - DB 0F3h,0C3h ;repret - - -section .data data align=8 - -ALIGN 32 -$L$ones: - DQ 1,1,1,1 -$L$zeros: - DQ 0,0,0,0 -EXTERN __imp_RtlVirtualUnwind - -ALIGN 16 -rsaz_avx_handler: - push rsi - push rdi - push rbx - push rbp - push r12 - push r13 - push r14 - push r15 - pushfq - sub rsp,64 - - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] - - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] - - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 - jb NEAR $L$common_seh_tail - - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 - jae NEAR $L$common_seh_tail - - mov rax,QWORD[152+r8] - - lea rsi,[rax] - lea rdi,[512+r8] - mov ecx,20 - DD 0xa548f3fc - - lea rax,[216+rax] - - mov rbx,QWORD[((-8))+rax] - mov rbp,QWORD[((-16))+rax] - mov r12,QWORD[((-24))+rax] - mov r13,QWORD[((-32))+rax] - mov r14,QWORD[((-40))+rax] - mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 - -$L$common_seh_tail: - mov rdi,QWORD[8+rax] - mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi - - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc - - mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx - call QWORD[__imp_RtlVirtualUnwind] - - mov eax,1 - add rsp,64 - popfq - pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx - pop rdi - pop rsi - DB 0F3h,0C3h ;repret - - -section .pdata rdata align=4 -ALIGN 4 - DD $L$SEH_begin_ossl_rsaz_amm52x40_x1_ifma256 wrt ..imagebase - DD $L$SEH_end_ossl_rsaz_amm52x40_x1_ifma256 wrt ..imagebase - DD $L$SEH_info_ossl_rsaz_amm52x40_x1_ifma256 wrt ..imagebase - - DD $L$SEH_begin_ossl_rsaz_amm52x40_x2_ifma256 wrt ..imagebase - DD $L$SEH_end_ossl_rsaz_amm52x40_x2_ifma256 wrt ..imagebase - DD $L$SEH_info_ossl_rsaz_amm52x40_x2_ifma256 wrt ..imagebase - -section .xdata rdata align=8 -ALIGN 8 -$L$SEH_info_ossl_rsaz_amm52x40_x1_ifma256: -DB 9,0,0,0 - DD rsaz_avx_handler wrt ..imagebase - DD $L$ossl_rsaz_amm52x40_x1_ifma256_body wrt ..imagebase,$L$ossl_rsaz_amm52x40_x1_ifma256_epilogue wrt ..imagebase -$L$SEH_info_ossl_rsaz_amm52x40_x2_ifma256: -DB 9,0,0,0 - DD rsaz_avx_handler wrt ..imagebase - DD $L$ossl_rsaz_amm52x40_x2_ifma256_body wrt ..imagebase,$L$ossl_rsaz_amm52x40_x2_ifma256_epilogue wrt ..imagebase diff --git a/openssl/src/crypto/bn/gen/windows_x64/rsaz-avx512.asm b/openssl/src/crypto/bn/gen/windows_x64/rsaz-avx512.asm new file mode 100644 index 000000000..8bedb848d --- /dev/null +++ b/openssl/src/crypto/bn/gen/windows_x64/rsaz-avx512.asm @@ -0,0 +1,1031 @@ +default rel +%define XMMWORD +%define YMMWORD +%define ZMMWORD +EXTERN OPENSSL_ia32cap_P +global ossl_rsaz_avx512ifma_eligible + +ALIGN 32 +ossl_rsaz_avx512ifma_eligible: + mov ecx,DWORD[((OPENSSL_ia32cap_P+8))] + xor eax,eax + and ecx,2149777408 + cmp ecx,2149777408 + cmove eax,ecx + DB 0F3h,0C3h ;repret + +section .text code align=64 + + +global ossl_rsaz_amm52x20_x1_256 + +ALIGN 32 +ossl_rsaz_amm52x20_x1_256: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ossl_rsaz_amm52x20_x1_256: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD[40+rsp] + + + +DB 243,15,30,250 + push rbx + + push rbp + + push r12 + + push r13 + + push r14 + + push r15 + +$L$rsaz_amm52x20_x1_256_body: + + + vpxord ymm0,ymm0,ymm0 + vmovdqa64 ymm1,ymm0 + vmovdqa64 ymm16,ymm0 + vmovdqa64 ymm17,ymm0 + vmovdqa64 ymm18,ymm0 + vmovdqa64 ymm19,ymm0 + + xor r9d,r9d + + mov r11,rdx + mov rax,0xfffffffffffff + + + mov ebx,5 + +ALIGN 32 +$L$loop5: + mov r13,QWORD[r11] + + vpbroadcastq ymm3,r13 + mov rdx,QWORD[rsi] + mulx r12,r13,r13 + add r9,r13 + mov r10,r12 + adc r10,0 + + mov r13,r8 + imul r13,r9 + and r13,rax + + vpbroadcastq ymm4,r13 + mov rdx,QWORD[rcx] + mulx r12,r13,r13 + add r9,r13 + adc r10,r12 + + shr r9,52 + sal r10,12 + or r9,r10 + + vpmadd52luq ymm1,ymm3,YMMWORD[rsi] + vpmadd52luq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52luq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52luq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52luq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52luq ymm1,ymm4,YMMWORD[rcx] + vpmadd52luq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52luq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52luq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52luq ymm19,ymm4,YMMWORD[128+rcx] + + + valignq ymm1,ymm16,ymm1,1 + valignq ymm16,ymm17,ymm16,1 + valignq ymm17,ymm18,ymm17,1 + valignq ymm18,ymm19,ymm18,1 + valignq ymm19,ymm0,ymm19,1 + + vmovq r13,xmm1 + add r9,r13 + + vpmadd52huq ymm1,ymm3,YMMWORD[rsi] + vpmadd52huq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52huq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52huq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52huq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52huq ymm1,ymm4,YMMWORD[rcx] + vpmadd52huq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52huq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52huq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52huq ymm19,ymm4,YMMWORD[128+rcx] + mov r13,QWORD[8+r11] + + vpbroadcastq ymm3,r13 + mov rdx,QWORD[rsi] + mulx r12,r13,r13 + add r9,r13 + mov r10,r12 + adc r10,0 + + mov r13,r8 + imul r13,r9 + and r13,rax + + vpbroadcastq ymm4,r13 + mov rdx,QWORD[rcx] + mulx r12,r13,r13 + add r9,r13 + adc r10,r12 + + shr r9,52 + sal r10,12 + or r9,r10 + + vpmadd52luq ymm1,ymm3,YMMWORD[rsi] + vpmadd52luq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52luq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52luq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52luq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52luq ymm1,ymm4,YMMWORD[rcx] + vpmadd52luq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52luq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52luq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52luq ymm19,ymm4,YMMWORD[128+rcx] + + + valignq ymm1,ymm16,ymm1,1 + valignq ymm16,ymm17,ymm16,1 + valignq ymm17,ymm18,ymm17,1 + valignq ymm18,ymm19,ymm18,1 + valignq ymm19,ymm0,ymm19,1 + + vmovq r13,xmm1 + add r9,r13 + + vpmadd52huq ymm1,ymm3,YMMWORD[rsi] + vpmadd52huq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52huq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52huq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52huq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52huq ymm1,ymm4,YMMWORD[rcx] + vpmadd52huq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52huq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52huq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52huq ymm19,ymm4,YMMWORD[128+rcx] + mov r13,QWORD[16+r11] + + vpbroadcastq ymm3,r13 + mov rdx,QWORD[rsi] + mulx r12,r13,r13 + add r9,r13 + mov r10,r12 + adc r10,0 + + mov r13,r8 + imul r13,r9 + and r13,rax + + vpbroadcastq ymm4,r13 + mov rdx,QWORD[rcx] + mulx r12,r13,r13 + add r9,r13 + adc r10,r12 + + shr r9,52 + sal r10,12 + or r9,r10 + + vpmadd52luq ymm1,ymm3,YMMWORD[rsi] + vpmadd52luq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52luq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52luq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52luq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52luq ymm1,ymm4,YMMWORD[rcx] + vpmadd52luq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52luq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52luq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52luq ymm19,ymm4,YMMWORD[128+rcx] + + + valignq ymm1,ymm16,ymm1,1 + valignq ymm16,ymm17,ymm16,1 + valignq ymm17,ymm18,ymm17,1 + valignq ymm18,ymm19,ymm18,1 + valignq ymm19,ymm0,ymm19,1 + + vmovq r13,xmm1 + add r9,r13 + + vpmadd52huq ymm1,ymm3,YMMWORD[rsi] + vpmadd52huq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52huq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52huq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52huq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52huq ymm1,ymm4,YMMWORD[rcx] + vpmadd52huq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52huq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52huq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52huq ymm19,ymm4,YMMWORD[128+rcx] + mov r13,QWORD[24+r11] + + vpbroadcastq ymm3,r13 + mov rdx,QWORD[rsi] + mulx r12,r13,r13 + add r9,r13 + mov r10,r12 + adc r10,0 + + mov r13,r8 + imul r13,r9 + and r13,rax + + vpbroadcastq ymm4,r13 + mov rdx,QWORD[rcx] + mulx r12,r13,r13 + add r9,r13 + adc r10,r12 + + shr r9,52 + sal r10,12 + or r9,r10 + + vpmadd52luq ymm1,ymm3,YMMWORD[rsi] + vpmadd52luq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52luq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52luq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52luq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52luq ymm1,ymm4,YMMWORD[rcx] + vpmadd52luq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52luq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52luq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52luq ymm19,ymm4,YMMWORD[128+rcx] + + + valignq ymm1,ymm16,ymm1,1 + valignq ymm16,ymm17,ymm16,1 + valignq ymm17,ymm18,ymm17,1 + valignq ymm18,ymm19,ymm18,1 + valignq ymm19,ymm0,ymm19,1 + + vmovq r13,xmm1 + add r9,r13 + + vpmadd52huq ymm1,ymm3,YMMWORD[rsi] + vpmadd52huq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52huq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52huq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52huq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52huq ymm1,ymm4,YMMWORD[rcx] + vpmadd52huq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52huq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52huq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52huq ymm19,ymm4,YMMWORD[128+rcx] + lea r11,[32+r11] + dec ebx + jne NEAR $L$loop5 + + vmovdqa64 ymm4,YMMWORD[$L$mask52x4] + + vpbroadcastq ymm3,r9 + vpblendd ymm1,ymm1,ymm3,3 + + + + vpsrlq ymm24,ymm1,52 + vpsrlq ymm25,ymm16,52 + vpsrlq ymm26,ymm17,52 + vpsrlq ymm27,ymm18,52 + vpsrlq ymm28,ymm19,52 + + + valignq ymm28,ymm28,ymm27,3 + valignq ymm27,ymm27,ymm26,3 + valignq ymm26,ymm26,ymm25,3 + valignq ymm25,ymm25,ymm24,3 + valignq ymm24,ymm24,ymm0,3 + + + vpandq ymm1,ymm1,ymm4 + vpandq ymm16,ymm16,ymm4 + vpandq ymm17,ymm17,ymm4 + vpandq ymm18,ymm18,ymm4 + vpandq ymm19,ymm19,ymm4 + + + vpaddq ymm1,ymm1,ymm24 + vpaddq ymm16,ymm16,ymm25 + vpaddq ymm17,ymm17,ymm26 + vpaddq ymm18,ymm18,ymm27 + vpaddq ymm19,ymm19,ymm28 + + + + vpcmpuq k1,ymm4,ymm1,1 + vpcmpuq k2,ymm4,ymm16,1 + vpcmpuq k3,ymm4,ymm17,1 + vpcmpuq k4,ymm4,ymm18,1 + vpcmpuq k5,ymm4,ymm19,1 + kmovb r14d,k1 + kmovb r13d,k2 + kmovb r12d,k3 + kmovb r11d,k4 + kmovb r10d,k5 + + + vpcmpuq k1,ymm4,ymm1,0 + vpcmpuq k2,ymm4,ymm16,0 + vpcmpuq k3,ymm4,ymm17,0 + vpcmpuq k4,ymm4,ymm18,0 + vpcmpuq k5,ymm4,ymm19,0 + kmovb r9d,k1 + kmovb r8d,k2 + kmovb ebx,k3 + kmovb ecx,k4 + kmovb edx,k5 + + + + shl r13b,4 + or r14b,r13b + shl r11b,4 + or r12b,r11b + + add r14b,r14b + adc r12b,r12b + adc r10b,r10b + + shl r8b,4 + or r9b,r8b + shl cl,4 + or bl,cl + + add r14b,r9b + adc r12b,bl + adc r10b,dl + + xor r14b,r9b + xor r12b,bl + xor r10b,dl + + kmovb k1,r14d + shr r14b,4 + kmovb k2,r14d + kmovb k3,r12d + shr r12b,4 + kmovb k4,r12d + kmovb k5,r10d + + + vpsubq ymm1{k1},ymm1,ymm4 + vpsubq ymm16{k2},ymm16,ymm4 + vpsubq ymm17{k3},ymm17,ymm4 + vpsubq ymm18{k4},ymm18,ymm4 + vpsubq ymm19{k5},ymm19,ymm4 + + vpandq ymm1,ymm1,ymm4 + vpandq ymm16,ymm16,ymm4 + vpandq ymm17,ymm17,ymm4 + vpandq ymm18,ymm18,ymm4 + vpandq ymm19,ymm19,ymm4 + + vmovdqu64 YMMWORD[rdi],ymm1 + vmovdqu64 YMMWORD[32+rdi],ymm16 + vmovdqu64 YMMWORD[64+rdi],ymm17 + vmovdqu64 YMMWORD[96+rdi],ymm18 + vmovdqu64 YMMWORD[128+rdi],ymm19 + + vzeroupper + mov r15,QWORD[rsp] + + mov r14,QWORD[8+rsp] + + mov r13,QWORD[16+rsp] + + mov r12,QWORD[24+rsp] + + mov rbp,QWORD[32+rsp] + + mov rbx,QWORD[40+rsp] + + lea rsp,[48+rsp] + +$L$rsaz_amm52x20_x1_256_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_ossl_rsaz_amm52x20_x1_256: +section .data data align=8 + +ALIGN 32 +$L$mask52x4: + DQ 0xfffffffffffff + DQ 0xfffffffffffff + DQ 0xfffffffffffff + DQ 0xfffffffffffff +section .text code align=64 + + +global ossl_rsaz_amm52x20_x2_256 + +ALIGN 32 +ossl_rsaz_amm52x20_x2_256: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ossl_rsaz_amm52x20_x2_256: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD[40+rsp] + + + +DB 243,15,30,250 + push rbx + + push rbp + + push r12 + + push r13 + + push r14 + + push r15 + +$L$rsaz_amm52x20_x2_256_body: + + + vpxord ymm0,ymm0,ymm0 + vmovdqa64 ymm1,ymm0 + vmovdqa64 ymm16,ymm0 + vmovdqa64 ymm17,ymm0 + vmovdqa64 ymm18,ymm0 + vmovdqa64 ymm19,ymm0 + vmovdqa64 ymm2,ymm0 + vmovdqa64 ymm20,ymm0 + vmovdqa64 ymm21,ymm0 + vmovdqa64 ymm22,ymm0 + vmovdqa64 ymm23,ymm0 + + xor r9d,r9d + xor r15d,r15d + + mov r11,rdx + mov rax,0xfffffffffffff + + mov ebx,20 + +ALIGN 32 +$L$loop20: + mov r13,QWORD[r11] + + vpbroadcastq ymm3,r13 + mov rdx,QWORD[rsi] + mulx r12,r13,r13 + add r9,r13 + mov r10,r12 + adc r10,0 + + mov r13,QWORD[r8] + imul r13,r9 + and r13,rax + + vpbroadcastq ymm4,r13 + mov rdx,QWORD[rcx] + mulx r12,r13,r13 + add r9,r13 + adc r10,r12 + + shr r9,52 + sal r10,12 + or r9,r10 + + vpmadd52luq ymm1,ymm3,YMMWORD[rsi] + vpmadd52luq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52luq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52luq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52luq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52luq ymm1,ymm4,YMMWORD[rcx] + vpmadd52luq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52luq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52luq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52luq ymm19,ymm4,YMMWORD[128+rcx] + + + valignq ymm1,ymm16,ymm1,1 + valignq ymm16,ymm17,ymm16,1 + valignq ymm17,ymm18,ymm17,1 + valignq ymm18,ymm19,ymm18,1 + valignq ymm19,ymm0,ymm19,1 + + vmovq r13,xmm1 + add r9,r13 + + vpmadd52huq ymm1,ymm3,YMMWORD[rsi] + vpmadd52huq ymm16,ymm3,YMMWORD[32+rsi] + vpmadd52huq ymm17,ymm3,YMMWORD[64+rsi] + vpmadd52huq ymm18,ymm3,YMMWORD[96+rsi] + vpmadd52huq ymm19,ymm3,YMMWORD[128+rsi] + + vpmadd52huq ymm1,ymm4,YMMWORD[rcx] + vpmadd52huq ymm16,ymm4,YMMWORD[32+rcx] + vpmadd52huq ymm17,ymm4,YMMWORD[64+rcx] + vpmadd52huq ymm18,ymm4,YMMWORD[96+rcx] + vpmadd52huq ymm19,ymm4,YMMWORD[128+rcx] + mov r13,QWORD[160+r11] + + vpbroadcastq ymm3,r13 + mov rdx,QWORD[160+rsi] + mulx r12,r13,r13 + add r15,r13 + mov r10,r12 + adc r10,0 + + mov r13,QWORD[8+r8] + imul r13,r15 + and r13,rax + + vpbroadcastq ymm4,r13 + mov rdx,QWORD[160+rcx] + mulx r12,r13,r13 + add r15,r13 + adc r10,r12 + + shr r15,52 + sal r10,12 + or r15,r10 + + vpmadd52luq ymm2,ymm3,YMMWORD[160+rsi] + vpmadd52luq ymm20,ymm3,YMMWORD[192+rsi] + vpmadd52luq ymm21,ymm3,YMMWORD[224+rsi] + vpmadd52luq ymm22,ymm3,YMMWORD[256+rsi] + vpmadd52luq ymm23,ymm3,YMMWORD[288+rsi] + + vpmadd52luq ymm2,ymm4,YMMWORD[160+rcx] + vpmadd52luq ymm20,ymm4,YMMWORD[192+rcx] + vpmadd52luq ymm21,ymm4,YMMWORD[224+rcx] + vpmadd52luq ymm22,ymm4,YMMWORD[256+rcx] + vpmadd52luq ymm23,ymm4,YMMWORD[288+rcx] + + + valignq ymm2,ymm20,ymm2,1 + valignq ymm20,ymm21,ymm20,1 + valignq ymm21,ymm22,ymm21,1 + valignq ymm22,ymm23,ymm22,1 + valignq ymm23,ymm0,ymm23,1 + + vmovq r13,xmm2 + add r15,r13 + + vpmadd52huq ymm2,ymm3,YMMWORD[160+rsi] + vpmadd52huq ymm20,ymm3,YMMWORD[192+rsi] + vpmadd52huq ymm21,ymm3,YMMWORD[224+rsi] + vpmadd52huq ymm22,ymm3,YMMWORD[256+rsi] + vpmadd52huq ymm23,ymm3,YMMWORD[288+rsi] + + vpmadd52huq ymm2,ymm4,YMMWORD[160+rcx] + vpmadd52huq ymm20,ymm4,YMMWORD[192+rcx] + vpmadd52huq ymm21,ymm4,YMMWORD[224+rcx] + vpmadd52huq ymm22,ymm4,YMMWORD[256+rcx] + vpmadd52huq ymm23,ymm4,YMMWORD[288+rcx] + lea r11,[8+r11] + dec ebx + jne NEAR $L$loop20 + + vmovdqa64 ymm4,YMMWORD[$L$mask52x4] + + vpbroadcastq ymm3,r9 + vpblendd ymm1,ymm1,ymm3,3 + + + + vpsrlq ymm24,ymm1,52 + vpsrlq ymm25,ymm16,52 + vpsrlq ymm26,ymm17,52 + vpsrlq ymm27,ymm18,52 + vpsrlq ymm28,ymm19,52 + + + valignq ymm28,ymm28,ymm27,3 + valignq ymm27,ymm27,ymm26,3 + valignq ymm26,ymm26,ymm25,3 + valignq ymm25,ymm25,ymm24,3 + valignq ymm24,ymm24,ymm0,3 + + + vpandq ymm1,ymm1,ymm4 + vpandq ymm16,ymm16,ymm4 + vpandq ymm17,ymm17,ymm4 + vpandq ymm18,ymm18,ymm4 + vpandq ymm19,ymm19,ymm4 + + + vpaddq ymm1,ymm1,ymm24 + vpaddq ymm16,ymm16,ymm25 + vpaddq ymm17,ymm17,ymm26 + vpaddq ymm18,ymm18,ymm27 + vpaddq ymm19,ymm19,ymm28 + + + + vpcmpuq k1,ymm4,ymm1,1 + vpcmpuq k2,ymm4,ymm16,1 + vpcmpuq k3,ymm4,ymm17,1 + vpcmpuq k4,ymm4,ymm18,1 + vpcmpuq k5,ymm4,ymm19,1 + kmovb r14d,k1 + kmovb r13d,k2 + kmovb r12d,k3 + kmovb r11d,k4 + kmovb r10d,k5 + + + vpcmpuq k1,ymm4,ymm1,0 + vpcmpuq k2,ymm4,ymm16,0 + vpcmpuq k3,ymm4,ymm17,0 + vpcmpuq k4,ymm4,ymm18,0 + vpcmpuq k5,ymm4,ymm19,0 + kmovb r9d,k1 + kmovb r8d,k2 + kmovb ebx,k3 + kmovb ecx,k4 + kmovb edx,k5 + + + + shl r13b,4 + or r14b,r13b + shl r11b,4 + or r12b,r11b + + add r14b,r14b + adc r12b,r12b + adc r10b,r10b + + shl r8b,4 + or r9b,r8b + shl cl,4 + or bl,cl + + add r14b,r9b + adc r12b,bl + adc r10b,dl + + xor r14b,r9b + xor r12b,bl + xor r10b,dl + + kmovb k1,r14d + shr r14b,4 + kmovb k2,r14d + kmovb k3,r12d + shr r12b,4 + kmovb k4,r12d + kmovb k5,r10d + + + vpsubq ymm1{k1},ymm1,ymm4 + vpsubq ymm16{k2},ymm16,ymm4 + vpsubq ymm17{k3},ymm17,ymm4 + vpsubq ymm18{k4},ymm18,ymm4 + vpsubq ymm19{k5},ymm19,ymm4 + + vpandq ymm1,ymm1,ymm4 + vpandq ymm16,ymm16,ymm4 + vpandq ymm17,ymm17,ymm4 + vpandq ymm18,ymm18,ymm4 + vpandq ymm19,ymm19,ymm4 + + vpbroadcastq ymm3,r15 + vpblendd ymm2,ymm2,ymm3,3 + + + + vpsrlq ymm24,ymm2,52 + vpsrlq ymm25,ymm20,52 + vpsrlq ymm26,ymm21,52 + vpsrlq ymm27,ymm22,52 + vpsrlq ymm28,ymm23,52 + + + valignq ymm28,ymm28,ymm27,3 + valignq ymm27,ymm27,ymm26,3 + valignq ymm26,ymm26,ymm25,3 + valignq ymm25,ymm25,ymm24,3 + valignq ymm24,ymm24,ymm0,3 + + + vpandq ymm2,ymm2,ymm4 + vpandq ymm20,ymm20,ymm4 + vpandq ymm21,ymm21,ymm4 + vpandq ymm22,ymm22,ymm4 + vpandq ymm23,ymm23,ymm4 + + + vpaddq ymm2,ymm2,ymm24 + vpaddq ymm20,ymm20,ymm25 + vpaddq ymm21,ymm21,ymm26 + vpaddq ymm22,ymm22,ymm27 + vpaddq ymm23,ymm23,ymm28 + + + + vpcmpuq k1,ymm4,ymm2,1 + vpcmpuq k2,ymm4,ymm20,1 + vpcmpuq k3,ymm4,ymm21,1 + vpcmpuq k4,ymm4,ymm22,1 + vpcmpuq k5,ymm4,ymm23,1 + kmovb r14d,k1 + kmovb r13d,k2 + kmovb r12d,k3 + kmovb r11d,k4 + kmovb r10d,k5 + + + vpcmpuq k1,ymm4,ymm2,0 + vpcmpuq k2,ymm4,ymm20,0 + vpcmpuq k3,ymm4,ymm21,0 + vpcmpuq k4,ymm4,ymm22,0 + vpcmpuq k5,ymm4,ymm23,0 + kmovb r9d,k1 + kmovb r8d,k2 + kmovb ebx,k3 + kmovb ecx,k4 + kmovb edx,k5 + + + + shl r13b,4 + or r14b,r13b + shl r11b,4 + or r12b,r11b + + add r14b,r14b + adc r12b,r12b + adc r10b,r10b + + shl r8b,4 + or r9b,r8b + shl cl,4 + or bl,cl + + add r14b,r9b + adc r12b,bl + adc r10b,dl + + xor r14b,r9b + xor r12b,bl + xor r10b,dl + + kmovb k1,r14d + shr r14b,4 + kmovb k2,r14d + kmovb k3,r12d + shr r12b,4 + kmovb k4,r12d + kmovb k5,r10d + + + vpsubq ymm2{k1},ymm2,ymm4 + vpsubq ymm20{k2},ymm20,ymm4 + vpsubq ymm21{k3},ymm21,ymm4 + vpsubq ymm22{k4},ymm22,ymm4 + vpsubq ymm23{k5},ymm23,ymm4 + + vpandq ymm2,ymm2,ymm4 + vpandq ymm20,ymm20,ymm4 + vpandq ymm21,ymm21,ymm4 + vpandq ymm22,ymm22,ymm4 + vpandq ymm23,ymm23,ymm4 + + vmovdqu64 YMMWORD[rdi],ymm1 + vmovdqu64 YMMWORD[32+rdi],ymm16 + vmovdqu64 YMMWORD[64+rdi],ymm17 + vmovdqu64 YMMWORD[96+rdi],ymm18 + vmovdqu64 YMMWORD[128+rdi],ymm19 + + vmovdqu64 YMMWORD[160+rdi],ymm2 + vmovdqu64 YMMWORD[192+rdi],ymm20 + vmovdqu64 YMMWORD[224+rdi],ymm21 + vmovdqu64 YMMWORD[256+rdi],ymm22 + vmovdqu64 YMMWORD[288+rdi],ymm23 + + vzeroupper + mov r15,QWORD[rsp] + + mov r14,QWORD[8+rsp] + + mov r13,QWORD[16+rsp] + + mov r12,QWORD[24+rsp] + + mov rbp,QWORD[32+rsp] + + mov rbx,QWORD[40+rsp] + + lea rsp,[48+rsp] + +$L$rsaz_amm52x20_x2_256_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_ossl_rsaz_amm52x20_x2_256: +section .text code align=64 + + +ALIGN 32 +global ossl_extract_multiplier_2x20_win5 + +ossl_extract_multiplier_2x20_win5: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ossl_extract_multiplier_2x20_win5: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + +DB 243,15,30,250 + lea rax,[rcx*4+rcx] + sal rax,5 + add rsi,rax + + vmovdqa64 ymm23,YMMWORD[$L$ones] + vpbroadcastq ymm22,rdx + lea rax,[10240+rsi] + + vpxor xmm4,xmm4,xmm4 + vmovdqa64 ymm3,ymm4 + vmovdqa64 ymm2,ymm4 + vmovdqa64 ymm1,ymm4 + vmovdqa64 ymm0,ymm4 + vmovdqa64 ymm21,ymm4 + +ALIGN 32 +$L$loop: + vpcmpq k1,ymm22,ymm21,0 + add rsi,320 + vpaddq ymm21,ymm21,ymm23 + vmovdqu64 ymm16,YMMWORD[((-320))+rsi] + vmovdqu64 ymm17,YMMWORD[((-288))+rsi] + vmovdqu64 ymm18,YMMWORD[((-256))+rsi] + vmovdqu64 ymm19,YMMWORD[((-224))+rsi] + vmovdqu64 ymm20,YMMWORD[((-192))+rsi] + vpblendmq ymm0{k1},ymm0,ymm16 + vpblendmq ymm1{k1},ymm1,ymm17 + vpblendmq ymm2{k1},ymm2,ymm18 + vpblendmq ymm3{k1},ymm3,ymm19 + vpblendmq ymm4{k1},ymm4,ymm20 + cmp rax,rsi + jne NEAR $L$loop + + vmovdqu64 YMMWORD[rdi],ymm0 + vmovdqu64 YMMWORD[32+rdi],ymm1 + vmovdqu64 YMMWORD[64+rdi],ymm2 + vmovdqu64 YMMWORD[96+rdi],ymm3 + vmovdqu64 YMMWORD[128+rdi],ymm4 + + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_ossl_extract_multiplier_2x20_win5: +section .data data align=8 + +ALIGN 32 +$L$ones: + DQ 1,1,1,1 +EXTERN __imp_RtlVirtualUnwind + +ALIGN 16 +rsaz_def_handler: + push rsi + push rdi + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + pushfq + sub rsp,64 + + mov rax,QWORD[120+r8] + mov rbx,QWORD[248+r8] + + mov rsi,QWORD[8+r9] + mov r11,QWORD[56+r9] + + mov r10d,DWORD[r11] + lea r10,[r10*1+rsi] + cmp rbx,r10 + jb NEAR $L$common_seh_tail + + mov rax,QWORD[152+r8] + + mov r10d,DWORD[4+r11] + lea r10,[r10*1+rsi] + cmp rbx,r10 + jae NEAR $L$common_seh_tail + + lea rax,[48+rax] + + mov rbx,QWORD[((-8))+rax] + mov rbp,QWORD[((-16))+rax] + mov r12,QWORD[((-24))+rax] + mov r13,QWORD[((-32))+rax] + mov r14,QWORD[((-40))+rax] + mov r15,QWORD[((-48))+rax] + mov QWORD[144+r8],rbx + mov QWORD[160+r8],rbp + mov QWORD[216+r8],r12 + mov QWORD[224+r8],r13 + mov QWORD[232+r8],r14 + mov QWORD[240+r8],r15 + +$L$common_seh_tail: + mov rdi,QWORD[8+rax] + mov rsi,QWORD[16+rax] + mov QWORD[152+r8],rax + mov QWORD[168+r8],rsi + mov QWORD[176+r8],rdi + + mov rdi,QWORD[40+r9] + mov rsi,r8 + mov ecx,154 + DD 0xa548f3fc + + mov rsi,r9 + xor rcx,rcx + mov rdx,QWORD[8+rsi] + mov r8,QWORD[rsi] + mov r9,QWORD[16+rsi] + mov r10,QWORD[40+rsi] + lea r11,[56+rsi] + lea r12,[24+rsi] + mov QWORD[32+rsp],r10 + mov QWORD[40+rsp],r11 + mov QWORD[48+rsp],r12 + mov QWORD[56+rsp],rcx + call QWORD[__imp_RtlVirtualUnwind] + + mov eax,1 + add rsp,64 + popfq + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + pop rdi + pop rsi + DB 0F3h,0C3h ;repret + + +section .pdata rdata align=4 +ALIGN 4 + DD $L$SEH_begin_ossl_rsaz_amm52x20_x1_256 wrt ..imagebase + DD $L$SEH_end_ossl_rsaz_amm52x20_x1_256 wrt ..imagebase + DD $L$SEH_info_ossl_rsaz_amm52x20_x1_256 wrt ..imagebase + + DD $L$SEH_begin_ossl_rsaz_amm52x20_x2_256 wrt ..imagebase + DD $L$SEH_end_ossl_rsaz_amm52x20_x2_256 wrt ..imagebase + DD $L$SEH_info_ossl_rsaz_amm52x20_x2_256 wrt ..imagebase + + DD $L$SEH_begin_ossl_extract_multiplier_2x20_win5 wrt ..imagebase + DD $L$SEH_end_ossl_extract_multiplier_2x20_win5 wrt ..imagebase + DD $L$SEH_info_ossl_extract_multiplier_2x20_win5 wrt ..imagebase + +section .xdata rdata align=8 +ALIGN 8 +$L$SEH_info_ossl_rsaz_amm52x20_x1_256: +DB 9,0,0,0 + DD rsaz_def_handler wrt ..imagebase + DD $L$rsaz_amm52x20_x1_256_body wrt ..imagebase,$L$rsaz_amm52x20_x1_256_epilogue wrt ..imagebase +$L$SEH_info_ossl_rsaz_amm52x20_x2_256: +DB 9,0,0,0 + DD rsaz_def_handler wrt ..imagebase + DD $L$rsaz_amm52x20_x2_256_body wrt ..imagebase,$L$rsaz_amm52x20_x2_256_epilogue wrt ..imagebase +$L$SEH_info_ossl_extract_multiplier_2x20_win5: +DB 9,0,0,0 + DD rsaz_def_handler wrt ..imagebase + DD $L$SEH_begin_ossl_extract_multiplier_2x20_win5 wrt ..imagebase,$L$SEH_begin_ossl_extract_multiplier_2x20_win5 wrt ..imagebase diff --git a/openssl/src/crypto/bn/gen/windows_x64/x86_64-mont5.asm b/openssl/src/crypto/bn/gen/windows_x64/x86_64-mont5.asm index 118676455..075671490 100644 --- a/openssl/src/crypto/bn/gen/windows_x64/x86_64-mont5.asm +++ b/openssl/src/crypto/bn/gen/windows_x64/x86_64-mont5.asm @@ -2095,6 +2095,199 @@ $L$sqr4x_sub_entry: DB 0F3h,0C3h ;repret +global bn_from_montgomery + +ALIGN 32 +bn_from_montgomery: + + test DWORD[48+rsp],7 + jz NEAR bn_from_mont8x + xor eax,eax + DB 0F3h,0C3h ;repret + + + + +ALIGN 32 +bn_from_mont8x: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_bn_from_mont8x: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD[40+rsp] + mov r9,QWORD[48+rsp] + + + +DB 0x67 + mov rax,rsp + + push rbx + + push rbp + + push r12 + + push r13 + + push r14 + + push r15 + +$L$from_prologue: + + shl r9d,3 + lea r10,[r9*2+r9] + neg r9 + mov r8,QWORD[r8] + + + + + + + + + lea r11,[((-320))+r9*2+rsp] + mov rbp,rsp + sub r11,rdi + and r11,4095 + cmp r10,r11 + jb NEAR $L$from_sp_alt + sub rbp,r11 + lea rbp,[((-320))+r9*2+rbp] + jmp NEAR $L$from_sp_done + +ALIGN 32 +$L$from_sp_alt: + lea r10,[((4096-320))+r9*2] + lea rbp,[((-320))+r9*2+rbp] + sub r11,r10 + mov r10,0 + cmovc r11,r10 + sub rbp,r11 +$L$from_sp_done: + and rbp,-64 + mov r11,rsp + sub r11,rbp + and r11,-4096 + lea rsp,[rbp*1+r11] + mov r10,QWORD[rsp] + cmp rsp,rbp + ja NEAR $L$from_page_walk + jmp NEAR $L$from_page_walk_done + +$L$from_page_walk: + lea rsp,[((-4096))+rsp] + mov r10,QWORD[rsp] + cmp rsp,rbp + ja NEAR $L$from_page_walk +$L$from_page_walk_done: + + mov r10,r9 + neg r9 + + + + + + + + + + + mov QWORD[32+rsp],r8 + mov QWORD[40+rsp],rax + +$L$from_body: + mov r11,r9 + lea rax,[48+rsp] + pxor xmm0,xmm0 + jmp NEAR $L$mul_by_1 + +ALIGN 32 +$L$mul_by_1: + movdqu xmm1,XMMWORD[rsi] + movdqu xmm2,XMMWORD[16+rsi] + movdqu xmm3,XMMWORD[32+rsi] + movdqa XMMWORD[r9*1+rax],xmm0 + movdqu xmm4,XMMWORD[48+rsi] + movdqa XMMWORD[16+r9*1+rax],xmm0 +DB 0x48,0x8d,0xb6,0x40,0x00,0x00,0x00 + movdqa XMMWORD[rax],xmm1 + movdqa XMMWORD[32+r9*1+rax],xmm0 + movdqa XMMWORD[16+rax],xmm2 + movdqa XMMWORD[48+r9*1+rax],xmm0 + movdqa XMMWORD[32+rax],xmm3 + movdqa XMMWORD[48+rax],xmm4 + lea rax,[64+rax] + sub r11,64 + jnz NEAR $L$mul_by_1 + +DB 102,72,15,110,207 +DB 102,72,15,110,209 +DB 0x67 + mov rbp,rcx +DB 102,73,15,110,218 + mov r11d,DWORD[((OPENSSL_ia32cap_P+8))] + and r11d,0x80108 + cmp r11d,0x80108 + jne NEAR $L$from_mont_nox + + lea rdi,[r9*1+rax] + call __bn_sqrx8x_reduction + call __bn_postx4x_internal + + pxor xmm0,xmm0 + lea rax,[48+rsp] + jmp NEAR $L$from_mont_zero + +ALIGN 32 +$L$from_mont_nox: + call __bn_sqr8x_reduction + call __bn_post4x_internal + + pxor xmm0,xmm0 + lea rax,[48+rsp] + jmp NEAR $L$from_mont_zero + +ALIGN 32 +$L$from_mont_zero: + mov rsi,QWORD[40+rsp] + + movdqa XMMWORD[rax],xmm0 + movdqa XMMWORD[16+rax],xmm0 + movdqa XMMWORD[32+rax],xmm0 + movdqa XMMWORD[48+rax],xmm0 + lea rax,[64+rax] + sub r9,32 + jnz NEAR $L$from_mont_zero + + mov rax,1 + mov r15,QWORD[((-48))+rsi] + + mov r14,QWORD[((-40))+rsi] + + mov r13,QWORD[((-32))+rsi] + + mov r12,QWORD[((-24))+rsi] + + mov rbp,QWORD[((-16))+rsi] + + mov rbx,QWORD[((-8))+rsi] + + lea rsp,[rsi] + +$L$from_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + DB 0F3h,0C3h ;repret + +$L$SEH_end_bn_from_mont8x: ALIGN 32 bn_mulx4x_mont_gather5: @@ -3797,6 +3990,10 @@ ALIGN 4 DD $L$SEH_begin_bn_power5 wrt ..imagebase DD $L$SEH_end_bn_power5 wrt ..imagebase DD $L$SEH_info_bn_power5 wrt ..imagebase + + DD $L$SEH_begin_bn_from_mont8x wrt ..imagebase + DD $L$SEH_end_bn_from_mont8x wrt ..imagebase + DD $L$SEH_info_bn_from_mont8x wrt ..imagebase DD $L$SEH_begin_bn_mulx4x_mont_gather5 wrt ..imagebase DD $L$SEH_end_bn_mulx4x_mont_gather5 wrt ..imagebase DD $L$SEH_info_bn_mulx4x_mont_gather5 wrt ..imagebase @@ -3825,6 +4022,11 @@ DB 9,0,0,0 DD mul_handler wrt ..imagebase DD $L$power5_prologue wrt ..imagebase,$L$power5_body wrt ..imagebase,$L$power5_epilogue wrt ..imagebase ALIGN 8 +$L$SEH_info_bn_from_mont8x: +DB 9,0,0,0 + DD mul_handler wrt ..imagebase + DD $L$from_prologue wrt ..imagebase,$L$from_body wrt ..imagebase,$L$from_epilogue wrt ..imagebase +ALIGN 8 $L$SEH_info_bn_mulx4x_mont_gather5: DB 9,0,0,0 DD mul_handler wrt ..imagebase diff --git a/openssl/src/crypto/bn/local.h b/openssl/src/crypto/bn/local.h new file mode 100644 index 000000000..6800cc359 --- /dev/null +++ b/openssl/src/crypto/bn/local.h @@ -0,0 +1,69 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +void bn_mul_mont_gather5(BN_ULONG *rp, const BN_ULONG *ap, + const void *table, const BN_ULONG *np, + const BN_ULONG *n0, int num, int power); +void bn_scatter5(const BN_ULONG *inp, size_t num, + void *table, size_t power); +void bn_gather5(BN_ULONG *out, size_t num, void *table, size_t power); +void bn_power5(BN_ULONG *rp, const BN_ULONG *ap, + const void *table, const BN_ULONG *np, + const BN_ULONG *n0, int num, int power); +int bn_get_bits5(const BN_ULONG *ap, int off); +int bn_from_montgomery(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *not_used, const BN_ULONG *np, + const BN_ULONG *n0, int num); + +void bn_GF2m_mul_2x2(BN_ULONG *r, BN_ULONG a1, BN_ULONG a0, BN_ULONG b1, + BN_ULONG b0); + +/* + * See crypto/bn/asm/rsaz-avx2.pl for further details. + */ +void rsaz_1024_norm2red_avx2(void *red, const void *norm); +void rsaz_1024_mul_avx2(void *ret, const void *a, const void *b, + const void *n, BN_ULONG k); +void rsaz_1024_sqr_avx2(void *ret, const void *a, const void *n, BN_ULONG k, + int cnt); +void rsaz_1024_scatter5_avx2(void *tbl, const void *val, int i); +void rsaz_1024_gather5_avx2(void *val, const void *tbl, int i); +void rsaz_1024_red2norm_avx2(void *norm, const void *red); + +/* + * See crypto/bn/rsaz-x86_64.pl for further details. + */ +void rsaz_512_mul(void *ret, const void *a, const void *b, const void *n, + BN_ULONG k); +void rsaz_512_mul_scatter4(void *ret, const void *a, const void *n, + BN_ULONG k, const void *tbl, unsigned int power); +void rsaz_512_mul_gather4(void *ret, const void *a, const void *tbl, + const void *n, BN_ULONG k, unsigned int power); +void rsaz_512_mul_by_one(void *ret, const void *a, const void *n, BN_ULONG k); +void rsaz_512_sqr(void *ret, const void *a, const void *n, BN_ULONG k, + int cnt); +void rsaz_512_scatter4(void *tbl, const BN_ULONG *val, int power); +void rsaz_512_gather4(BN_ULONG *val, const void *tbl, int power); + +/* + * See crypto/bn/asm/rsaz-avx512.pl for further details. + */ +void ossl_rsaz_amm52x20_x1_256(BN_ULONG *res, const BN_ULONG *base, + const BN_ULONG *exp, const BN_ULONG *m, + BN_ULONG k0); +void ossl_rsaz_amm52x20_x2_256(BN_ULONG *out, const BN_ULONG *a, + const BN_ULONG *b, const BN_ULONG *m, + const BN_ULONG k0[2]); +void ossl_extract_multiplier_2x20_win5(BN_ULONG *red_Y, + const BN_ULONG *red_table, + int red_table_idx, int tbl_idx); diff --git a/openssl/src/crypto/bn/rsa_sup_mul.c b/openssl/src/crypto/bn/rsa_sup_mul.c new file mode 100644 index 000000000..0e0d02e19 --- /dev/null +++ b/openssl/src/crypto/bn/rsa_sup_mul.c @@ -0,0 +1,604 @@ +#include +#include +#include +#include +#include +#include +#include +#include "internal/endian.h" +#include "internal/numbers.h" +#include "internal/constant_time.h" +#include "bn_local.h" + +# if BN_BYTES == 8 +typedef uint64_t limb_t; +# if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16 +typedef uint128_t limb2_t; +# define HAVE_LIMB2_T +# endif +# define LIMB_BIT_SIZE 64 +# define LIMB_BYTE_SIZE 8 +# elif BN_BYTES == 4 +typedef uint32_t limb_t; +typedef uint64_t limb2_t; +# define LIMB_BIT_SIZE 32 +# define LIMB_BYTE_SIZE 4 +# define HAVE_LIMB2_T +# else +# error "Not supported" +# endif + +/* + * For multiplication we're using schoolbook multiplication, + * so if we have two numbers, each with 6 "digits" (words) + * the multiplication is calculated as follows: + * A B C D E F + * x I J K L M N + * -------------- + * N*F + * N*E + * N*D + * N*C + * N*B + * N*A + * M*F + * M*E + * M*D + * M*C + * M*B + * M*A + * L*F + * L*E + * L*D + * L*C + * L*B + * L*A + * K*F + * K*E + * K*D + * K*C + * K*B + * K*A + * J*F + * J*E + * J*D + * J*C + * J*B + * J*A + * I*F + * I*E + * I*D + * I*C + * I*B + * + I*A + * ========================== + * N*B N*D N*F + * + N*A N*C N*E + * + M*B M*D M*F + * + M*A M*C M*E + * + L*B L*D L*F + * + L*A L*C L*E + * + K*B K*D K*F + * + K*A K*C K*E + * + J*B J*D J*F + * + J*A J*C J*E + * + I*B I*D I*F + * + I*A I*C I*E + * + * 1+1 1+3 1+5 + * 1+0 1+2 1+4 + * 0+1 0+3 0+5 + * 0+0 0+2 0+4 + * + * 0 1 2 3 4 5 6 + * which requires n^2 multiplications and 2n full length additions + * as we can keep every other result of limb multiplication in two separate + * limbs + */ + +#if defined HAVE_LIMB2_T +static ossl_inline void _mul_limb(limb_t *hi, limb_t *lo, limb_t a, limb_t b) +{ + limb2_t t; + /* + * this is idiomatic code to tell compiler to use the native mul + * those three lines will actually compile to single instruction + */ + + t = (limb2_t)a * b; + *hi = t >> LIMB_BIT_SIZE; + *lo = (limb_t)t; +} +#elif (BN_BYTES == 8) && (defined _MSC_VER) +/* https://learn.microsoft.com/en-us/cpp/intrinsics/umul128?view=msvc-170 */ +#pragma intrinsic(_umul128) +static ossl_inline void _mul_limb(limb_t *hi, limb_t *lo, limb_t a, limb_t b) +{ + *lo = _umul128(a, b, hi); +} +#else +/* + * if the compiler doesn't have either a 128bit data type nor a "return + * high 64 bits of multiplication" + */ +static ossl_inline void _mul_limb(limb_t *hi, limb_t *lo, limb_t a, limb_t b) +{ + limb_t a_low = (limb_t)(uint32_t)a; + limb_t a_hi = a >> 32; + limb_t b_low = (limb_t)(uint32_t)b; + limb_t b_hi = b >> 32; + + limb_t p0 = a_low * b_low; + limb_t p1 = a_low * b_hi; + limb_t p2 = a_hi * b_low; + limb_t p3 = a_hi * b_hi; + + uint32_t cy = (uint32_t)(((p0 >> 32) + (uint32_t)p1 + (uint32_t)p2) >> 32); + + *lo = p0 + (p1 << 32) + (p2 << 32); + *hi = p3 + (p1 >> 32) + (p2 >> 32) + cy; +} +#endif + +/* add two limbs with carry in, return carry out */ +static ossl_inline limb_t _add_limb(limb_t *ret, limb_t a, limb_t b, limb_t carry) +{ + limb_t carry1, carry2, t; + /* + * `c = a + b; if (c < a)` is idiomatic code that makes compilers + * use add with carry on assembly level + */ + + *ret = a + carry; + if (*ret < a) + carry1 = 1; + else + carry1 = 0; + + t = *ret; + *ret = t + b; + if (*ret < t) + carry2 = 1; + else + carry2 = 0; + + return carry1 + carry2; +} + +/* + * add two numbers of the same size, return overflow + * + * add a to b, place result in ret; all arrays need to be n limbs long + * return overflow from addition (0 or 1) + */ +static ossl_inline limb_t add(limb_t *ret, limb_t *a, limb_t *b, size_t n) +{ + limb_t c = 0; + ossl_ssize_t i; + + for(i = n - 1; i > -1; i--) + c = _add_limb(&ret[i], a[i], b[i], c); + + return c; +} + +/* + * return number of limbs necessary for temporary values + * when multiplying numbers n limbs large + */ +static ossl_inline size_t mul_limb_numb(size_t n) +{ + return 2 * n * 2; +} + +/* + * multiply two numbers of the same size + * + * multiply a by b, place result in ret; a and b need to be n limbs long + * ret needs to be 2*n limbs long, tmp needs to be mul_limb_numb(n) limbs + * long + */ +static void limb_mul(limb_t *ret, limb_t *a, limb_t *b, size_t n, limb_t *tmp) +{ + limb_t *r_odd, *r_even; + size_t i, j, k; + + r_odd = tmp; + r_even = &tmp[2 * n]; + + memset(ret, 0, 2 * n * sizeof(limb_t)); + + for (i = 0; i < n; i++) { + for (k = 0; k < i + n + 1; k++) { + r_even[k] = 0; + r_odd[k] = 0; + } + for (j = 0; j < n; j++) { + /* + * place results from even and odd limbs in separate arrays so that + * we don't have to calculate overflow every time we get individual + * limb multiplication result + */ + if (j % 2 == 0) + _mul_limb(&r_even[i + j], &r_even[i + j + 1], a[i], b[j]); + else + _mul_limb(&r_odd[i + j], &r_odd[i + j + 1], a[i], b[j]); + } + /* + * skip the least significant limbs when adding multiples of + * more significant limbs (they're zero anyway) + */ + add(ret, ret, r_even, n + i + 1); + add(ret, ret, r_odd, n + i + 1); + } +} + +/* modifies the value in place by performing a right shift by one bit */ +static ossl_inline void rshift1(limb_t *val, size_t n) +{ + limb_t shift_in = 0, shift_out = 0; + size_t i; + + for (i = 0; i < n; i++) { + shift_out = val[i] & 1; + val[i] = shift_in << (LIMB_BIT_SIZE - 1) | (val[i] >> 1); + shift_in = shift_out; + } +} + +/* extend the LSB of flag to all bits of limb */ +static ossl_inline limb_t mk_mask(limb_t flag) +{ + flag |= flag << 1; + flag |= flag << 2; + flag |= flag << 4; + flag |= flag << 8; + flag |= flag << 16; +#if (LIMB_BYTE_SIZE == 8) + flag |= flag << 32; +#endif + return flag; +} + +/* + * copy from either a or b to ret based on flag + * when flag == 0, then copies from b + * when flag == 1, then copies from a + */ +static ossl_inline void cselect(limb_t flag, limb_t *ret, limb_t *a, limb_t *b, size_t n) +{ + /* + * would be more efficient with non volatile mask, but then gcc + * generates code with jumps + */ + volatile limb_t mask; + size_t i; + + mask = mk_mask(flag); + for (i = 0; i < n; i++) { +#if (LIMB_BYTE_SIZE == 8) + ret[i] = constant_time_select_64(mask, a[i], b[i]); +#else + ret[i] = constant_time_select_32(mask, a[i], b[i]); +#endif + } +} + +static limb_t _sub_limb(limb_t *ret, limb_t a, limb_t b, limb_t borrow) +{ + limb_t borrow1, borrow2, t; + /* + * while it doesn't look constant-time, this is idiomatic code + * to tell compilers to use the carry bit from subtraction + */ + + *ret = a - borrow; + if (*ret > a) + borrow1 = 1; + else + borrow1 = 0; + + t = *ret; + *ret = t - b; + if (*ret > t) + borrow2 = 1; + else + borrow2 = 0; + + return borrow1 + borrow2; +} + +/* + * place the result of a - b into ret, return the borrow bit. + * All arrays need to be n limbs long + */ +static limb_t sub(limb_t *ret, limb_t *a, limb_t *b, size_t n) +{ + limb_t borrow = 0; + ossl_ssize_t i; + + for (i = n - 1; i > -1; i--) + borrow = _sub_limb(&ret[i], a[i], b[i], borrow); + + return borrow; +} + +/* return the number of limbs necessary to allocate for the mod() tmp operand */ +static ossl_inline size_t mod_limb_numb(size_t anum, size_t modnum) +{ + return (anum + modnum) * 3; +} + +/* + * calculate a % mod, place the result in ret + * size of a is defined by anum, size of ret and mod is modnum, + * size of tmp is returned by mod_limb_numb() + */ +static void mod(limb_t *ret, limb_t *a, size_t anum, limb_t *mod, + size_t modnum, limb_t *tmp) +{ + limb_t *atmp, *modtmp, *rettmp; + limb_t res; + size_t i; + + memset(tmp, 0, mod_limb_numb(anum, modnum) * LIMB_BYTE_SIZE); + + atmp = tmp; + modtmp = &tmp[anum + modnum]; + rettmp = &tmp[(anum + modnum) * 2]; + + for (i = modnum; i 0; i--, rp--) { + v = _mul_add_limb(rp, mod, modnum, rp[modnum-1] * ni0, tmp2); + v = v + carry + rp[-1]; + carry |= (v != rp[-1]); + carry &= (v <= rp[-1]); + rp[-1] = v; + } + + /* perform the final reduction by mod... */ + carry -= sub(ret, rp, mod, modnum); + + /* ...conditionally */ + cselect(carry, ret, rp, ret, modnum); +} + +/* allocated buffer should be freed afterwards */ +static void BN_to_limb(const BIGNUM *bn, limb_t *buf, size_t limbs) +{ + int i; + int real_limbs = (BN_num_bytes(bn) + LIMB_BYTE_SIZE - 1) / LIMB_BYTE_SIZE; + limb_t *ptr = buf + (limbs - real_limbs); + + for (i = 0; i < real_limbs; i++) + ptr[i] = bn->d[real_limbs - i - 1]; +} + +#if LIMB_BYTE_SIZE == 8 +static ossl_inline uint64_t be64(uint64_t host) +{ + uint64_t big = 0; + DECLARE_IS_ENDIAN; + + if (!IS_LITTLE_ENDIAN) + return host; + + big |= (host & 0xff00000000000000) >> 56; + big |= (host & 0x00ff000000000000) >> 40; + big |= (host & 0x0000ff0000000000) >> 24; + big |= (host & 0x000000ff00000000) >> 8; + big |= (host & 0x00000000ff000000) << 8; + big |= (host & 0x0000000000ff0000) << 24; + big |= (host & 0x000000000000ff00) << 40; + big |= (host & 0x00000000000000ff) << 56; + return big; +} + +#else +/* Not all platforms have htobe32(). */ +static ossl_inline uint32_t be32(uint32_t host) +{ + uint32_t big = 0; + DECLARE_IS_ENDIAN; + + if (!IS_LITTLE_ENDIAN) + return host; + + big |= (host & 0xff000000) >> 24; + big |= (host & 0x00ff0000) >> 8; + big |= (host & 0x0000ff00) << 8; + big |= (host & 0x000000ff) << 24; + return big; +} +#endif + +/* + * We assume that intermediate, possible_arg2, blinding, and ctx are used + * similar to BN_BLINDING_invert_ex() arguments. + * to_mod is RSA modulus. + * buf and num is the serialization buffer and its length. + * + * Here we use classic/Montgomery multiplication and modulo. After the calculation finished + * we serialize the new structure instead of BIGNUMs taking endianness into account. + */ +int ossl_bn_rsa_do_unblind(const BIGNUM *intermediate, + const BN_BLINDING *blinding, + const BIGNUM *possible_arg2, + const BIGNUM *to_mod, BN_CTX *ctx, + unsigned char *buf, int num) +{ + limb_t *l_im = NULL, *l_mul = NULL, *l_mod = NULL; + limb_t *l_ret = NULL, *l_tmp = NULL, l_buf; + size_t l_im_count = 0, l_mul_count = 0, l_size = 0, l_mod_count = 0; + size_t l_tmp_count = 0; + int ret = 0; + size_t i; + unsigned char *tmp; + const BIGNUM *arg1 = intermediate; + const BIGNUM *arg2 = (possible_arg2 == NULL) ? blinding->Ai : possible_arg2; + + l_im_count = (BN_num_bytes(arg1) + LIMB_BYTE_SIZE - 1) / LIMB_BYTE_SIZE; + l_mul_count = (BN_num_bytes(arg2) + LIMB_BYTE_SIZE - 1) / LIMB_BYTE_SIZE; + l_mod_count = (BN_num_bytes(to_mod) + LIMB_BYTE_SIZE - 1) / LIMB_BYTE_SIZE; + + l_size = l_im_count > l_mul_count ? l_im_count : l_mul_count; + l_im = OPENSSL_zalloc(l_size * LIMB_BYTE_SIZE); + l_mul = OPENSSL_zalloc(l_size * LIMB_BYTE_SIZE); + l_mod = OPENSSL_zalloc(l_mod_count * LIMB_BYTE_SIZE); + + if ((l_im == NULL) || (l_mul == NULL) || (l_mod == NULL)) + goto err; + + BN_to_limb(arg1, l_im, l_size); + BN_to_limb(arg2, l_mul, l_size); + BN_to_limb(to_mod, l_mod, l_mod_count); + + l_ret = OPENSSL_malloc(2 * l_size * LIMB_BYTE_SIZE); + + if (blinding->m_ctx != NULL) { + l_tmp_count = mul_limb_numb(l_size) > mod_montgomery_limb_numb(l_mod_count) ? + mul_limb_numb(l_size) : mod_montgomery_limb_numb(l_mod_count); + l_tmp = OPENSSL_malloc(l_tmp_count * LIMB_BYTE_SIZE); + } else { + l_tmp_count = mul_limb_numb(l_size) > mod_limb_numb(2 * l_size, l_mod_count) ? + mul_limb_numb(l_size) : mod_limb_numb(2 * l_size, l_mod_count); + l_tmp = OPENSSL_malloc(l_tmp_count * LIMB_BYTE_SIZE); + } + + if ((l_ret == NULL) || (l_tmp == NULL)) + goto err; + + if (blinding->m_ctx != NULL) { + limb_mul(l_ret, l_im, l_mul, l_size, l_tmp); + mod_montgomery(l_ret, l_ret, 2 * l_size, l_mod, l_mod_count, + blinding->m_ctx->n0[0], l_tmp); + } else { + limb_mul(l_ret, l_im, l_mul, l_size, l_tmp); + mod(l_ret, l_ret, 2 * l_size, l_mod, l_mod_count, l_tmp); + } + + /* modulus size in bytes can be equal to num but after limbs conversion it becomes bigger */ + if (num < BN_num_bytes(to_mod)) { + ERR_raise(ERR_LIB_BN, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + memset(buf, 0, num); + tmp = buf + num - BN_num_bytes(to_mod); + for (i = 0; i < l_mod_count; i++) { +#if LIMB_BYTE_SIZE == 8 + l_buf = be64(l_ret[i]); +#else + l_buf = be32(l_ret[i]); +#endif + if (i == 0) { + int delta = LIMB_BYTE_SIZE - ((l_mod_count * LIMB_BYTE_SIZE) - num); + + memcpy(tmp, ((char *)&l_buf) + LIMB_BYTE_SIZE - delta, delta); + tmp += delta; + } else { + memcpy(tmp, &l_buf, LIMB_BYTE_SIZE); + tmp += LIMB_BYTE_SIZE; + } + } + ret = num; + + err: + OPENSSL_free(l_im); + OPENSSL_free(l_mul); + OPENSSL_free(l_mod); + OPENSSL_free(l_tmp); + OPENSSL_free(l_ret); + + return ret; +} diff --git a/openssl/src/crypto/bn/rsaz_exp.c b/openssl/src/crypto/bn/rsaz_exp.c index 80b583f35..2dbcb88ac 100644 --- a/openssl/src/crypto/bn/rsaz_exp.c +++ b/openssl/src/crypto/bn/rsaz_exp.c @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2013-2016 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2012, Intel Corporation. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -13,7 +13,6 @@ */ #include -#include "internal/common.h" #include "rsaz_exp.h" #ifndef RSAZ_ENABLED @@ -32,8 +31,16 @@ void rsaz_1024_scatter5_avx2(void *tbl, const void *val, int i); void rsaz_1024_gather5_avx2(void *val, const void *tbl, int i); void rsaz_1024_red2norm_avx2(void *norm, const void *red); -#if defined(__SUNPRO_C) +#if defined(__GNUC__) +# define ALIGN64 __attribute__((aligned(64))) +#elif defined(_MSC_VER) +# define ALIGN64 __declspec(align(64)) +#elif defined(__SUNPRO_C) +# define ALIGN64 # pragma align 64(one,two80) +#else +/* not fatal, might hurt performance a little */ +# define ALIGN64 #endif ALIGN64 static const BN_ULONG one[40] = { @@ -59,7 +66,6 @@ void RSAZ_1024_mod_exp_avx2(BN_ULONG result_norm[16], unsigned char *R2 = table_s; /* borrow */ int index; int wvalue; - BN_ULONG tmp[16]; if ((((size_t)p_str & 4095) + 320) >> 12) { result = p_str; @@ -231,10 +237,7 @@ void RSAZ_1024_mod_exp_avx2(BN_ULONG result_norm[16], rsaz_1024_red2norm_avx2(result_norm, result); - bn_reduce_once_in_place(result_norm, /*carry=*/0, m_norm, tmp, 16); - OPENSSL_cleanse(storage, sizeof(storage)); - OPENSSL_cleanse(tmp, sizeof(tmp)); } /* @@ -263,7 +266,6 @@ void RSAZ_512_mod_exp(BN_ULONG result[8], unsigned char *p_str = (unsigned char *)exponent; int index; unsigned int wvalue; - BN_ULONG tmp[8]; /* table[0] = 1_inv */ temp[0] = 0 - m[0]; @@ -307,10 +309,7 @@ void RSAZ_512_mod_exp(BN_ULONG result[8], /* from Montgomery */ rsaz_512_mul_by_one(result, temp, m, k0); - bn_reduce_once_in_place(result, /*carry=*/0, m, tmp, 8); - OPENSSL_cleanse(storage, sizeof(storage)); - OPENSSL_cleanse(tmp, sizeof(tmp)); } #endif diff --git a/openssl/src/crypto/bn/rsaz_exp.h b/openssl/src/crypto/bn/rsaz_exp.h index 45dc9cc19..b4fd3cbbb 100644 --- a/openssl/src/crypto/bn/rsaz_exp.h +++ b/openssl/src/crypto/bn/rsaz_exp.h @@ -22,8 +22,6 @@ # define RSAZ_ENABLED # include -# include "internal/constant_time.h" -# include "bn_local.h" void RSAZ_1024_mod_exp_avx2(BN_ULONG result[16], const BN_ULONG base_norm[16], @@ -54,27 +52,6 @@ int ossl_rsaz_mod_exp_avx512_x2(BN_ULONG *res1, BN_ULONG k0_2, int factor_size); -static ossl_inline void bn_select_words(BN_ULONG *r, BN_ULONG mask, - const BN_ULONG *a, - const BN_ULONG *b, size_t num) -{ - size_t i; - - for (i = 0; i < num; i++) { - r[i] = constant_time_select_64(mask, a[i], b[i]); - } -} - -static ossl_inline BN_ULONG bn_reduce_once_in_place(BN_ULONG *r, - BN_ULONG carry, - const BN_ULONG *m, - BN_ULONG *tmp, size_t num) -{ - carry -= bn_sub_words(tmp, r, m, num); - bn_select_words(r, carry, r /* tmp < 0 */, tmp /* tmp >= 0 */, num); - return carry; -} - # endif #endif diff --git a/openssl/src/crypto/bn/rsaz_exp_x2.c b/openssl/src/crypto/bn/rsaz_exp_x2.c index 70705486a..0e0aff1f8 100644 --- a/openssl/src/crypto/bn/rsaz_exp_x2.c +++ b/openssl/src/crypto/bn/rsaz_exp_x2.c @@ -1,6 +1,6 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. - * Copyright (c) 2020-2021, Intel Corporation. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright (c) 2020, Intel Corporation. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,8 +8,7 @@ * https://www.openssl.org/source/license.html * * - * Originally written by Sergey Kirillov and Andrey Matyukov. - * Special thanks to Ilya Albrekht for his valuable hints. + * Originally written by Ilya Albrekht, Sergey Kirillov and Andrey Matyukov * Intel Corporation * */ @@ -24,6 +23,14 @@ NON_EMPTY_TRANSLATION_UNIT # include # include +# if defined(__GNUC__) +# define ALIGN64 __attribute__((aligned(64))) +# elif defined(_MSC_VER) +# define ALIGN64 __declspec(align(64)) +# else +# define ALIGN64 +# endif + # define ALIGN_OF(ptr, boundary) \ ((unsigned char *)(ptr) + (boundary - (((size_t)(ptr)) & (boundary - 1)))) @@ -35,12 +42,8 @@ NON_EMPTY_TRANSLATION_UNIT # define BITS2WORD8_SIZE(x) (((x) + 7) >> 3) # define BITS2WORD64_SIZE(x) (((x) + 63) >> 6) -/* Number of registers required to hold |digits_num| amount of qword digits */ -# define NUMBER_OF_REGISTERS(digits_num, register_size) \ - (((digits_num) * 64 + (register_size) - 1) / (register_size)) - -static ossl_inline uint64_t get_digit(const uint8_t *in, int in_len); -static ossl_inline void put_digit(uint8_t *out, int out_len, uint64_t digit); +static ossl_inline uint64_t get_digit52(const uint8_t *in, int in_len); +static ossl_inline void put_digit52(uint8_t *out, int out_len, uint64_t digit); static void to_words52(BN_ULONG *out, int out_len, const BN_ULONG *in, int in_bitsize); static void from_words52(BN_ULONG *bn_out, int out_bitsize, const BN_ULONG *in); @@ -52,52 +55,37 @@ static ossl_inline int number_of_digits(int bitsize, int digit_size) return (bitsize + digit_size - 1) / digit_size; } +typedef void (*AMM52)(BN_ULONG *res, const BN_ULONG *base, + const BN_ULONG *exp, const BN_ULONG *m, BN_ULONG k0); +typedef void (*EXP52_x2)(BN_ULONG *res, const BN_ULONG *base, + const BN_ULONG *exp[2], const BN_ULONG *m, + const BN_ULONG *rr, const BN_ULONG k0[2]); + /* * For details of the methods declared below please refer to * crypto/bn/asm/rsaz-avx512.pl * - * Naming conventions: + * Naming notes: * amm = Almost Montgomery Multiplication * ams = Almost Montgomery Squaring - * 52xZZ - data represented as array of ZZ digits in 52-bit radix + * 52x20 - data represented as array of 20 digits in 52-bit radix * _x1_/_x2_ - 1 or 2 independent inputs/outputs - * _ifma256 - uses 256-bit wide IFMA ISA (AVX512_IFMA256) + * _256 suffix - uses 256-bit (AVX512VL) registers */ -void ossl_rsaz_amm52x20_x1_ifma256(BN_ULONG *res, const BN_ULONG *a, - const BN_ULONG *b, const BN_ULONG *m, - BN_ULONG k0); -void ossl_rsaz_amm52x20_x2_ifma256(BN_ULONG *out, const BN_ULONG *a, - const BN_ULONG *b, const BN_ULONG *m, - const BN_ULONG k0[2]); +/*AMM = Almost Montgomery Multiplication. */ +void ossl_rsaz_amm52x20_x1_256(BN_ULONG *res, const BN_ULONG *base, + const BN_ULONG *exp, const BN_ULONG *m, + BN_ULONG k0); +static void RSAZ_exp52x20_x2_256(BN_ULONG *res, const BN_ULONG *base, + const BN_ULONG *exp[2], const BN_ULONG *m, + const BN_ULONG *rr, const BN_ULONG k0[2]); +void ossl_rsaz_amm52x20_x2_256(BN_ULONG *out, const BN_ULONG *a, + const BN_ULONG *b, const BN_ULONG *m, + const BN_ULONG k0[2]); void ossl_extract_multiplier_2x20_win5(BN_ULONG *red_Y, const BN_ULONG *red_table, - int red_table_idx1, int red_table_idx2); - -void ossl_rsaz_amm52x30_x1_ifma256(BN_ULONG *res, const BN_ULONG *a, - const BN_ULONG *b, const BN_ULONG *m, - BN_ULONG k0); -void ossl_rsaz_amm52x30_x2_ifma256(BN_ULONG *out, const BN_ULONG *a, - const BN_ULONG *b, const BN_ULONG *m, - const BN_ULONG k0[2]); -void ossl_extract_multiplier_2x30_win5(BN_ULONG *red_Y, - const BN_ULONG *red_table, - int red_table_idx1, int red_table_idx2); - -void ossl_rsaz_amm52x40_x1_ifma256(BN_ULONG *res, const BN_ULONG *a, - const BN_ULONG *b, const BN_ULONG *m, - BN_ULONG k0); -void ossl_rsaz_amm52x40_x2_ifma256(BN_ULONG *out, const BN_ULONG *a, - const BN_ULONG *b, const BN_ULONG *m, - const BN_ULONG k0[2]); -void ossl_extract_multiplier_2x40_win5(BN_ULONG *red_Y, - const BN_ULONG *red_table, - int red_table_idx1, int red_table_idx2); - -static int RSAZ_mod_exp_x2_ifma256(BN_ULONG *res, const BN_ULONG *base, - const BN_ULONG *exp[2], const BN_ULONG *m, - const BN_ULONG *rr, const BN_ULONG k0[2], - int modulus_bitsize); + int red_table_idx, int tbl_idx); /* * Dual Montgomery modular exponentiation using prime moduli of the @@ -110,10 +98,7 @@ static int RSAZ_mod_exp_x2_ifma256(BN_ULONG *res, const BN_ULONG *base, * * Each moduli shall be |factor_size| bit size. * - * Supported cases: - * - 2x1024 - * - 2x1536 - * - 2x2048 + * NOTE: currently only 2x1024 case is supported. * * [out] res|i| - result of modular exponentiation: array of qword values * in regular (2^64) radix. Size of array shall be enough @@ -142,8 +127,6 @@ int ossl_rsaz_mod_exp_avx512_x2(BN_ULONG *res1, BN_ULONG k0_2, int factor_size) { - typedef void (*AMM)(BN_ULONG *res, const BN_ULONG *a, - const BN_ULONG *b, const BN_ULONG *m, BN_ULONG k0); int ret = 0; /* @@ -152,60 +135,52 @@ int ossl_rsaz_mod_exp_avx512_x2(BN_ULONG *res1, */ int exp_digits = number_of_digits(factor_size + 2, DIGIT_SIZE); int coeff_pow = 4 * (DIGIT_SIZE * exp_digits - factor_size); - - /* Number of YMM registers required to store exponent's digits */ - int ymm_regs_num = NUMBER_OF_REGISTERS(exp_digits, 256 /* ymm bit size */); - /* Capacity of the register set (in qwords) to store exponent */ - int regs_capacity = ymm_regs_num * 4; - BN_ULONG *base1_red, *m1_red, *rr1_red; BN_ULONG *base2_red, *m2_red, *rr2_red; BN_ULONG *coeff_red; BN_ULONG *storage = NULL; BN_ULONG *storage_aligned = NULL; - int storage_len_bytes = 7 * regs_capacity * sizeof(BN_ULONG) - + 64 /* alignment */; + BN_ULONG storage_len_bytes = 7 * exp_digits * sizeof(BN_ULONG); + + /* AMM = Almost Montgomery Multiplication */ + AMM52 amm = NULL; + /* Dual (2-exps in parallel) exponentiation */ + EXP52_x2 exp_x2 = NULL; const BN_ULONG *exp[2] = {0}; BN_ULONG k0[2] = {0}; - /* AMM = Almost Montgomery Multiplication */ - AMM amm = NULL; + /* Only 1024-bit factor size is supported now */ switch (factor_size) { case 1024: - amm = ossl_rsaz_amm52x20_x1_ifma256; - break; - case 1536: - amm = ossl_rsaz_amm52x30_x1_ifma256; - break; - case 2048: - amm = ossl_rsaz_amm52x40_x1_ifma256; + amm = ossl_rsaz_amm52x20_x1_256; + exp_x2 = RSAZ_exp52x20_x2_256; break; default: goto err; } - storage = (BN_ULONG *)OPENSSL_malloc(storage_len_bytes); + storage = (BN_ULONG *)OPENSSL_malloc(storage_len_bytes + 64); if (storage == NULL) goto err; storage_aligned = (BN_ULONG *)ALIGN_OF(storage, 64); /* Memory layout for red(undant) representations */ base1_red = storage_aligned; - base2_red = storage_aligned + 1 * regs_capacity; - m1_red = storage_aligned + 2 * regs_capacity; - m2_red = storage_aligned + 3 * regs_capacity; - rr1_red = storage_aligned + 4 * regs_capacity; - rr2_red = storage_aligned + 5 * regs_capacity; - coeff_red = storage_aligned + 6 * regs_capacity; + base2_red = storage_aligned + 1 * exp_digits; + m1_red = storage_aligned + 2 * exp_digits; + m2_red = storage_aligned + 3 * exp_digits; + rr1_red = storage_aligned + 4 * exp_digits; + rr2_red = storage_aligned + 5 * exp_digits; + coeff_red = storage_aligned + 6 * exp_digits; /* Convert base_i, m_i, rr_i, from regular to 52-bit radix */ - to_words52(base1_red, regs_capacity, base1, factor_size); - to_words52(base2_red, regs_capacity, base2, factor_size); - to_words52(m1_red, regs_capacity, m1, factor_size); - to_words52(m2_red, regs_capacity, m2, factor_size); - to_words52(rr1_red, regs_capacity, rr1, factor_size); - to_words52(rr2_red, regs_capacity, rr2, factor_size); + to_words52(base1_red, exp_digits, base1, factor_size); + to_words52(base2_red, exp_digits, base2, factor_size); + to_words52(m1_red, exp_digits, m1, factor_size); + to_words52(m2_red, exp_digits, m2, factor_size); + to_words52(rr1_red, exp_digits, rr1, factor_size); + to_words52(rr2_red, exp_digits, rr2, factor_size); /* * Compute target domain Montgomery converters RR' for each modulus @@ -218,10 +193,10 @@ int ossl_rsaz_mod_exp_avx512_x2(BN_ULONG *res1, * where * k = 4 * (52 * digits52 - modlen) * R = 2^(64 * ceil(modlen/64)) mod m - * RR = R^2 mod m + * RR = R^2 mod M * R' = 2^(52 * ceil(modlen/52)) mod m * - * EX/ modlen = 1024: k = 64, RR = 2^2048 mod m, RR' = 2^2080 mod m + * modlen = 1024: k = 64, RR = 2^2048 mod m, RR' = 2^2080 mod m */ memset(coeff_red, 0, exp_digits * sizeof(BN_ULONG)); /* (1) in reduced domain representation */ @@ -239,22 +214,13 @@ int ossl_rsaz_mod_exp_avx512_x2(BN_ULONG *res1, k0[0] = k0_1; k0[1] = k0_2; - /* Dual (2-exps in parallel) exponentiation */ - ret = RSAZ_mod_exp_x2_ifma256(rr1_red, base1_red, exp, m1_red, rr1_red, - k0, factor_size); - if (!ret) - goto err; + exp_x2(rr1_red, base1_red, exp, m1_red, rr1_red, k0); /* Convert rr_i back to regular radix */ from_words52(res1, factor_size, rr1_red); from_words52(res2, factor_size, rr2_red); - /* bn_reduce_once_in_place expects number of BN_ULONG, not bit size */ - factor_size /= sizeof(BN_ULONG) * 8; - - bn_reduce_once_in_place(res1, /*carry=*/0, m1, storage, factor_size); - bn_reduce_once_in_place(res2, /*carry=*/0, m2, storage, factor_size); - + ret = 1; err: if (storage != NULL) { OPENSSL_cleanse(storage, storage_len_bytes); @@ -264,149 +230,91 @@ int ossl_rsaz_mod_exp_avx512_x2(BN_ULONG *res1, } /* - * Dual {1024,1536,2048}-bit w-ary modular exponentiation using prime moduli of - * the same bit size using Almost Montgomery Multiplication, optimized with - * AVX512_IFMA256 ISA. + * Dual 1024-bit w-ary modular exponentiation using prime moduli of the same + * bit size using Almost Montgomery Multiplication, optimized with AVX512_IFMA + * ISA. * * The parameter w (window size) = 5. * - * [out] res - result of modular exponentiation: 2x{20,30,40} qword + * [out] res - result of modular exponentiation: 2x20 qword * values in 2^52 radix. - * [in] base - base (2x{20,30,40} qword values in 2^52 radix) - * [in] exp - array of 2 pointers to {16,24,32} qword values in 2^64 radix. + * [in] base - base (2x20 qword values in 2^52 radix) + * [in] exp - array of 2 pointers to 16 qword values in 2^64 radix. * Exponent is not converted to redundant representation. - * [in] m - moduli (2x{20,30,40} qword values in 2^52 radix) - * [in] rr - Montgomery parameter for 2 moduli: - * RR(1024) = 2^2080 mod m. - * RR(1536) = 2^3120 mod m. - * RR(2048) = 2^4160 mod m. - * (2x{20,30,40} qword values in 2^52 radix) + * [in] m - moduli (2x20 qword values in 2^52 radix) + * [in] rr - Montgomery parameter for 2 moduli: RR = 2^2080 mod m. + * (2x20 qword values in 2^52 radix) * [in] k0 - Montgomery parameter for 2 moduli: k0 = -1/m mod 2^64 * * \return (void). */ -int RSAZ_mod_exp_x2_ifma256(BN_ULONG *out, - const BN_ULONG *base, - const BN_ULONG *exp[2], - const BN_ULONG *m, - const BN_ULONG *rr, - const BN_ULONG k0[2], - int modulus_bitsize) +static void RSAZ_exp52x20_x2_256(BN_ULONG *out, /* [2][20] */ + const BN_ULONG *base, /* [2][20] */ + const BN_ULONG *exp[2], /* 2x16 */ + const BN_ULONG *m, /* [2][20] */ + const BN_ULONG *rr, /* [2][20] */ + const BN_ULONG k0[2]) { - typedef void (*DAMM)(BN_ULONG *res, const BN_ULONG *a, - const BN_ULONG *b, const BN_ULONG *m, - const BN_ULONG k0[2]); - typedef void (*DEXTRACT)(BN_ULONG *res, const BN_ULONG *red_table, - int red_table_idx, int tbl_idx); - - int ret = 0; - int idx; - - /* Exponent window size */ - int exp_win_size = 5; - int exp_win_mask = (1U << exp_win_size) - 1; - - /* - * Number of digits (64-bit words) in redundant representation to handle - * modulus bits - */ - int red_digits = 0; - int exp_digits = 0; - - BN_ULONG *storage = NULL; - BN_ULONG *storage_aligned = NULL; - int storage_len_bytes = 0; - - /* Red(undant) result Y and multiplier X */ - BN_ULONG *red_Y = NULL; /* [2][red_digits] */ - BN_ULONG *red_X = NULL; /* [2][red_digits] */ - /* Pre-computed table of base powers */ - BN_ULONG *red_table = NULL; /* [1U << exp_win_size][2][red_digits] */ - /* Expanded exponent */ - BN_ULONG *expz = NULL; /* [2][exp_digits + 1] */ - - /* Dual AMM */ - DAMM damm = NULL; - /* Extractor from red_table */ - DEXTRACT extract = NULL; - +# define BITSIZE_MODULUS (1024) +# define EXP_WIN_SIZE (5) +# define EXP_WIN_MASK ((1U << EXP_WIN_SIZE) - 1) +/* + * Number of digits (64-bit words) in redundant representation to handle + * modulus bits + */ +# define RED_DIGITS (20) +# define EXP_DIGITS (16) +# define DAMM ossl_rsaz_amm52x20_x2_256 /* * Squaring is done using multiplication now. That can be a subject of * optimization in future. */ -# define DAMS(r,a,m,k0) damm((r),(a),(a),(m),(k0)) +# define DAMS(r,a,m,k0) \ + ossl_rsaz_amm52x20_x2_256((r),(a),(a),(m),(k0)) - switch (modulus_bitsize) { - case 1024: - red_digits = 20; - exp_digits = 16; - damm = ossl_rsaz_amm52x20_x2_ifma256; - extract = ossl_extract_multiplier_2x20_win5; - break; - case 1536: - /* Extended with 2 digits padding to avoid mask ops in high YMM register */ - red_digits = 30 + 2; - exp_digits = 24; - damm = ossl_rsaz_amm52x30_x2_ifma256; - extract = ossl_extract_multiplier_2x30_win5; - break; - case 2048: - red_digits = 40; - exp_digits = 32; - damm = ossl_rsaz_amm52x40_x2_ifma256; - extract = ossl_extract_multiplier_2x40_win5; - break; - default: - goto err; - } + /* Allocate stack for red(undant) result Y and multiplier X */ + ALIGN64 BN_ULONG red_Y[2][RED_DIGITS]; + ALIGN64 BN_ULONG red_X[2][RED_DIGITS]; - storage_len_bytes = (2 * red_digits /* red_Y */ - + 2 * red_digits /* red_X */ - + 2 * red_digits * (1U << exp_win_size) /* red_table */ - + 2 * (exp_digits + 1)) /* expz */ - * sizeof(BN_ULONG) - + 64; /* alignment */ + /* Allocate expanded exponent */ + ALIGN64 BN_ULONG expz[2][EXP_DIGITS + 1]; - storage = (BN_ULONG *)OPENSSL_zalloc(storage_len_bytes); - if (storage == NULL) - goto err; - storage_aligned = (BN_ULONG *)ALIGN_OF(storage, 64); + /* Pre-computed table of base powers */ + ALIGN64 BN_ULONG red_table[1U << EXP_WIN_SIZE][2][RED_DIGITS]; + + int idx; - red_Y = storage_aligned; - red_X = red_Y + 2 * red_digits; - red_table = red_X + 2 * red_digits; - expz = red_table + 2 * red_digits * (1U << exp_win_size); + memset(red_Y, 0, sizeof(red_Y)); + memset(red_table, 0, sizeof(red_table)); + memset(red_X, 0, sizeof(red_X)); /* * Compute table of powers base^i, i = 0, ..., (2^EXP_WIN_SIZE) - 1 * table[0] = mont(x^0) = mont(1) * table[1] = mont(x^1) = mont(x) */ - red_X[0 * red_digits] = 1; - red_X[1 * red_digits] = 1; - damm(&red_table[0 * 2 * red_digits], (const BN_ULONG*)red_X, rr, m, k0); - damm(&red_table[1 * 2 * red_digits], base, rr, m, k0); - - for (idx = 1; idx < (int)((1U << exp_win_size) / 2); idx++) { - DAMS(&red_table[(2 * idx + 0) * 2 * red_digits], - &red_table[(1 * idx) * 2 * red_digits], m, k0); - damm(&red_table[(2 * idx + 1) * 2 * red_digits], - &red_table[(2 * idx) * 2 * red_digits], - &red_table[1 * 2 * red_digits], m, k0); + red_X[0][0] = 1; + red_X[1][0] = 1; + DAMM(red_table[0][0], (const BN_ULONG*)red_X, rr, m, k0); + DAMM(red_table[1][0], base, rr, m, k0); + + for (idx = 1; idx < (int)((1U << EXP_WIN_SIZE) / 2); idx++) { + DAMS(red_table[2 * idx + 0][0], red_table[1 * idx][0], m, k0); + DAMM(red_table[2 * idx + 1][0], red_table[2 * idx][0], red_table[1][0], m, k0); } /* Copy and expand exponents */ - memcpy(&expz[0 * (exp_digits + 1)], exp[0], exp_digits * sizeof(BN_ULONG)); - expz[1 * (exp_digits + 1) - 1] = 0; - memcpy(&expz[1 * (exp_digits + 1)], exp[1], exp_digits * sizeof(BN_ULONG)); - expz[2 * (exp_digits + 1) - 1] = 0; + memcpy(expz[0], exp[0], EXP_DIGITS * sizeof(BN_ULONG)); + expz[0][EXP_DIGITS] = 0; + memcpy(expz[1], exp[1], EXP_DIGITS * sizeof(BN_ULONG)); + expz[1][EXP_DIGITS] = 0; /* Exponentiation */ { - const int rem = modulus_bitsize % exp_win_size; - const BN_ULONG table_idx_mask = exp_win_mask; + const int rem = BITSIZE_MODULUS % EXP_WIN_SIZE; + BN_ULONG table_idx_mask = EXP_WIN_MASK; - int exp_bit_no = modulus_bitsize - rem; + int exp_bit_no = BITSIZE_MODULUS - rem; int exp_chunk_no = exp_bit_no / 64; int exp_chunk_shift = exp_bit_no % 64; @@ -423,9 +331,8 @@ int RSAZ_mod_exp_x2_ifma256(BN_ULONG *out, OPENSSL_assert(rem != 0); /* Process 1-st exp window - just init result */ - red_table_idx_0 = expz[exp_chunk_no + 0 * (exp_digits + 1)]; - red_table_idx_1 = expz[exp_chunk_no + 1 * (exp_digits + 1)]; - + red_table_idx_0 = expz[0][exp_chunk_no]; + red_table_idx_1 = expz[1][exp_chunk_no]; /* * The function operates with fixed moduli sizes divisible by 64, * thus table index here is always in supported range [0, EXP_WIN_SIZE). @@ -433,10 +340,13 @@ int RSAZ_mod_exp_x2_ifma256(BN_ULONG *out, red_table_idx_0 >>= exp_chunk_shift; red_table_idx_1 >>= exp_chunk_shift; - extract(&red_Y[0 * red_digits], (const BN_ULONG*)red_table, (int)red_table_idx_0, (int)red_table_idx_1); + ossl_extract_multiplier_2x20_win5(red_Y[0], (const BN_ULONG*)red_table, + (int)red_table_idx_0, 0); + ossl_extract_multiplier_2x20_win5(red_Y[1], (const BN_ULONG*)red_table, + (int)red_table_idx_1, 1); /* Process other exp windows */ - for (exp_bit_no -= exp_win_size; exp_bit_no >= 0; exp_bit_no -= exp_win_size) { + for (exp_bit_no -= EXP_WIN_SIZE; exp_bit_no >= 0; exp_bit_no -= EXP_WIN_SIZE) { /* Extract pre-computed multiplier from the table */ { BN_ULONG T; @@ -444,37 +354,43 @@ int RSAZ_mod_exp_x2_ifma256(BN_ULONG *out, exp_chunk_no = exp_bit_no / 64; exp_chunk_shift = exp_bit_no % 64; { - red_table_idx_0 = expz[exp_chunk_no + 0 * (exp_digits + 1)]; - T = expz[exp_chunk_no + 1 + 0 * (exp_digits + 1)]; + red_table_idx_0 = expz[0][exp_chunk_no]; + T = expz[0][exp_chunk_no + 1]; red_table_idx_0 >>= exp_chunk_shift; /* * Get additional bits from then next quadword * when 64-bit boundaries are crossed. */ - if (exp_chunk_shift > 64 - exp_win_size) { + if (exp_chunk_shift > 64 - EXP_WIN_SIZE) { T <<= (64 - exp_chunk_shift); red_table_idx_0 ^= T; } red_table_idx_0 &= table_idx_mask; + + ossl_extract_multiplier_2x20_win5(red_X[0], + (const BN_ULONG*)red_table, + (int)red_table_idx_0, 0); } { - red_table_idx_1 = expz[exp_chunk_no + 1 * (exp_digits + 1)]; - T = expz[exp_chunk_no + 1 + 1 * (exp_digits + 1)]; + red_table_idx_1 = expz[1][exp_chunk_no]; + T = expz[1][exp_chunk_no + 1]; red_table_idx_1 >>= exp_chunk_shift; /* * Get additional bits from then next quadword * when 64-bit boundaries are crossed. */ - if (exp_chunk_shift > 64 - exp_win_size) { + if (exp_chunk_shift > 64 - EXP_WIN_SIZE) { T <<= (64 - exp_chunk_shift); red_table_idx_1 ^= T; } red_table_idx_1 &= table_idx_mask; - } - extract(&red_X[0 * red_digits], (const BN_ULONG*)red_table, (int)red_table_idx_0, (int)red_table_idx_1); + ossl_extract_multiplier_2x20_win5(red_X[1], + (const BN_ULONG*)red_table, + (int)red_table_idx_1, 1); + } } /* Series of squaring */ @@ -484,46 +400,43 @@ int RSAZ_mod_exp_x2_ifma256(BN_ULONG *out, DAMS((BN_ULONG*)red_Y, (const BN_ULONG*)red_Y, m, k0); DAMS((BN_ULONG*)red_Y, (const BN_ULONG*)red_Y, m, k0); - damm((BN_ULONG*)red_Y, (const BN_ULONG*)red_Y, (const BN_ULONG*)red_X, m, k0); + DAMM((BN_ULONG*)red_Y, (const BN_ULONG*)red_Y, (const BN_ULONG*)red_X, m, k0); } } /* * * NB: After the last AMM of exponentiation in Montgomery domain, the result - * may be (modulus_bitsize + 1), but the conversion out of Montgomery domain - * performs an AMM(x,1) which guarantees that the final result is less than - * |m|, so no conditional subtraction is needed here. See [1] for details. - * - * [1] Gueron, S. Efficient software implementations of modular exponentiation. - * DOI: 10.1007/s13389-012-0031-5 + * may be 1025-bit, but the conversion out of Montgomery domain performs an + * AMM(x,1) which guarantees that the final result is less than |m|, so no + * conditional subtraction is needed here. See "Efficient Software + * Implementations of Modular Exponentiation" (by Shay Gueron) paper for details. */ /* Convert result back in regular 2^52 domain */ - memset(red_X, 0, 2 * red_digits * sizeof(BN_ULONG)); - red_X[0 * red_digits] = 1; - red_X[1 * red_digits] = 1; - damm(out, (const BN_ULONG*)red_Y, (const BN_ULONG*)red_X, m, k0); - - ret = 1; - -err: - if (storage != NULL) { - /* Clear whole storage */ - OPENSSL_cleanse(storage, storage_len_bytes); - OPENSSL_free(storage); - } - -#undef DAMS - return ret; + memset(red_X, 0, sizeof(red_X)); + red_X[0][0] = 1; + red_X[1][0] = 1; + DAMM(out, (const BN_ULONG*)red_Y, (const BN_ULONG*)red_X, m, k0); + + /* Clear exponents */ + OPENSSL_cleanse(expz, sizeof(expz)); + OPENSSL_cleanse(red_Y, sizeof(red_Y)); + +# undef DAMS +# undef DAMM +# undef EXP_DIGITS +# undef RED_DIGITS +# undef EXP_WIN_MASK +# undef EXP_WIN_SIZE +# undef BITSIZE_MODULUS } -static ossl_inline uint64_t get_digit(const uint8_t *in, int in_len) +static ossl_inline uint64_t get_digit52(const uint8_t *in, int in_len) { uint64_t digit = 0; assert(in != NULL); - assert(in_len <= 8); for (; in_len > 0; in_len--) { digit <<= 8; @@ -549,29 +462,25 @@ static void to_words52(BN_ULONG *out, int out_len, in_str = (uint8_t *)in; for (; in_bitsize >= (2 * DIGIT_SIZE); in_bitsize -= (2 * DIGIT_SIZE), out += 2) { - uint64_t digit; - - memcpy(&digit, in_str, sizeof(digit)); - out[0] = digit & DIGIT_MASK; + out[0] = (*(uint64_t *)in_str) & DIGIT_MASK; in_str += 6; - memcpy(&digit, in_str, sizeof(digit)); - out[1] = (digit >> 4) & DIGIT_MASK; + out[1] = ((*(uint64_t *)in_str) >> 4) & DIGIT_MASK; in_str += 7; out_len -= 2; } if (in_bitsize > DIGIT_SIZE) { - uint64_t digit = get_digit(in_str, 7); + uint64_t digit = get_digit52(in_str, 7); out[0] = digit & DIGIT_MASK; in_str += 6; in_bitsize -= DIGIT_SIZE; - digit = get_digit(in_str, BITS2WORD8_SIZE(in_bitsize)); + digit = get_digit52(in_str, BITS2WORD8_SIZE(in_bitsize)); out[1] = digit >> 4; out += 2; out_len -= 2; } else if (in_bitsize > 0) { - out[0] = get_digit(in_str, BITS2WORD8_SIZE(in_bitsize)); + out[0] = get_digit52(in_str, BITS2WORD8_SIZE(in_bitsize)); out++; out_len--; } @@ -583,13 +492,12 @@ static void to_words52(BN_ULONG *out, int out_len, } } -static ossl_inline void put_digit(uint8_t *out, int out_len, uint64_t digit) +static ossl_inline void put_digit52(uint8_t *pStr, int strLen, uint64_t digit) { - assert(out != NULL); - assert(out_len <= 8); + assert(pStr != NULL); - for (; out_len > 0; out_len--) { - *out++ = (uint8_t)(digit & 0xFF); + for (; strLen > 0; strLen--) { + *pStr++ = (uint8_t)(digit & 0xFF); digit >>= 8; } } @@ -612,26 +520,21 @@ static void from_words52(BN_ULONG *out, int out_bitsize, const BN_ULONG *in) { uint8_t *out_str = (uint8_t *)out; - for (; out_bitsize >= (2 * DIGIT_SIZE); - out_bitsize -= (2 * DIGIT_SIZE), in += 2) { - uint64_t digit; - - digit = in[0]; - memcpy(out_str, &digit, sizeof(digit)); + for (; out_bitsize >= (2 * DIGIT_SIZE); out_bitsize -= (2 * DIGIT_SIZE), in += 2) { + (*(uint64_t *)out_str) = in[0]; out_str += 6; - digit = digit >> 48 | in[1] << 4; - memcpy(out_str, &digit, sizeof(digit)); + (*(uint64_t *)out_str) ^= in[1] << 4; out_str += 7; } if (out_bitsize > DIGIT_SIZE) { - put_digit(out_str, 7, in[0]); + put_digit52(out_str, 7, in[0]); out_str += 6; out_bitsize -= DIGIT_SIZE; - put_digit(out_str, BITS2WORD8_SIZE(out_bitsize), + put_digit52(out_str, BITS2WORD8_SIZE(out_bitsize), (in[1] << 4 | in[0] >> 48)); } else if (out_bitsize) { - put_digit(out_str, BITS2WORD8_SIZE(out_bitsize), in[0]); + put_digit52(out_str, BITS2WORD8_SIZE(out_bitsize), in[0]); } } } diff --git a/openssl/src/crypto/buffer/buffer.c b/openssl/src/crypto/buffer/buffer.c index 339434298..db1ea38b1 100644 --- a/openssl/src/crypto/buffer/buffer.c +++ b/openssl/src/crypto/buffer/buffer.c @@ -33,8 +33,10 @@ BUF_MEM *BUF_MEM_new(void) BUF_MEM *ret; ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_BUF, ERR_R_MALLOC_FAILURE); return NULL; + } return ret; } @@ -85,7 +87,7 @@ size_t BUF_MEM_grow(BUF_MEM *str, size_t len) } /* This limit is sufficient to ensure (len+3)/3*4 < 2**31 */ if (len > LIMIT_BEFORE_EXPANSION) { - ERR_raise(ERR_LIB_BUF, ERR_R_PASSED_INVALID_ARGUMENT); + ERR_raise(ERR_LIB_BUF, ERR_R_MALLOC_FAILURE); return 0; } n = (len + 3) / 3 * 4; @@ -94,6 +96,7 @@ size_t BUF_MEM_grow(BUF_MEM *str, size_t len) else ret = OPENSSL_realloc(str->data, n); if (ret == NULL) { + ERR_raise(ERR_LIB_BUF, ERR_R_MALLOC_FAILURE); len = 0; } else { str->data = ret; @@ -122,7 +125,7 @@ size_t BUF_MEM_grow_clean(BUF_MEM *str, size_t len) } /* This limit is sufficient to ensure (len+3)/3*4 < 2**31 */ if (len > LIMIT_BEFORE_EXPANSION) { - ERR_raise(ERR_LIB_BUF, ERR_R_PASSED_INVALID_ARGUMENT); + ERR_raise(ERR_LIB_BUF, ERR_R_MALLOC_FAILURE); return 0; } n = (len + 3) / 3 * 4; @@ -131,6 +134,7 @@ size_t BUF_MEM_grow_clean(BUF_MEM *str, size_t len) else ret = OPENSSL_clear_realloc(str->data, str->max, n); if (ret == NULL) { + ERR_raise(ERR_LIB_BUF, ERR_R_MALLOC_FAILURE); len = 0; } else { str->data = ret; diff --git a/openssl/src/crypto/camellia/camellia.c b/openssl/src/crypto/camellia/camellia.c deleted file mode 100644 index a4de9f891..000000000 --- a/openssl/src/crypto/camellia/camellia.c +++ /dev/null @@ -1,507 +0,0 @@ -/* - * Copyright 2006-2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* ==================================================================== - * Copyright 2006 NTT (Nippon Telegraph and Telephone Corporation) . - * ALL RIGHTS RESERVED. - * - * Intellectual Property information for Camellia: - * http://info.isl.ntt.co.jp/crypt/eng/info/chiteki.html - * - * News Release for Announcement of Camellia open source: - * http://www.ntt.co.jp/news/news06e/0604/060413a.html - * - * The Camellia Code included herein is developed by - * NTT (Nippon Telegraph and Telephone Corporation), and is contributed - * to the OpenSSL project. - */ - -/* - * Algorithm Specification - * http://info.isl.ntt.co.jp/crypt/eng/camellia/specifications.html - */ - -/* - * This release balances code size and performance. In particular key - * schedule setup is fully unrolled, because doing so *significantly* - * reduces amount of instructions per setup round and code increase is - * justifiable. In block functions on the other hand only inner loops - * are unrolled, as full unroll gives only nominal performance boost, - * while code size grows 4 or 7 times. Also, unlike previous versions - * this one "encourages" compiler to keep intermediate variables in - * registers, which should give better "all round" results, in other - * words reasonable performance even with not so modern compilers. - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "cmll_local.h" -#include -#include - -#define RightRotate(x, s) ( ((x) >> (s)) + ((x) << (32 - s)) ) -#define LeftRotate(x, s) ( ((x) << (s)) + ((x) >> (32 - s)) ) - -#define GETU32(p) (((u32)(p)[0] << 24) ^ ((u32)(p)[1] << 16) ^ ((u32)(p)[2] << 8) ^ ((u32)(p)[3])) -#define PUTU32(p,v) ((p)[0] = (u8)((v) >> 24), (p)[1] = (u8)((v) >> 16), (p)[2] = (u8)((v) >> 8), (p)[3] = (u8)(v)) - -/* S-box data */ -#define SBOX1_1110 Camellia_SBOX[0] -#define SBOX4_4404 Camellia_SBOX[1] -#define SBOX2_0222 Camellia_SBOX[2] -#define SBOX3_3033 Camellia_SBOX[3] -static const u32 Camellia_SBOX[][256] = { - {0x70707000, 0x82828200, 0x2c2c2c00, 0xececec00, 0xb3b3b300, 0x27272700, - 0xc0c0c000, 0xe5e5e500, 0xe4e4e400, 0x85858500, 0x57575700, 0x35353500, - 0xeaeaea00, 0x0c0c0c00, 0xaeaeae00, 0x41414100, 0x23232300, 0xefefef00, - 0x6b6b6b00, 0x93939300, 0x45454500, 0x19191900, 0xa5a5a500, 0x21212100, - 0xededed00, 0x0e0e0e00, 0x4f4f4f00, 0x4e4e4e00, 0x1d1d1d00, 0x65656500, - 0x92929200, 0xbdbdbd00, 0x86868600, 0xb8b8b800, 0xafafaf00, 0x8f8f8f00, - 0x7c7c7c00, 0xebebeb00, 0x1f1f1f00, 0xcecece00, 0x3e3e3e00, 0x30303000, - 0xdcdcdc00, 0x5f5f5f00, 0x5e5e5e00, 0xc5c5c500, 0x0b0b0b00, 0x1a1a1a00, - 0xa6a6a600, 0xe1e1e100, 0x39393900, 0xcacaca00, 0xd5d5d500, 0x47474700, - 0x5d5d5d00, 0x3d3d3d00, 0xd9d9d900, 0x01010100, 0x5a5a5a00, 0xd6d6d600, - 0x51515100, 0x56565600, 0x6c6c6c00, 0x4d4d4d00, 0x8b8b8b00, 0x0d0d0d00, - 0x9a9a9a00, 0x66666600, 0xfbfbfb00, 0xcccccc00, 0xb0b0b000, 0x2d2d2d00, - 0x74747400, 0x12121200, 0x2b2b2b00, 0x20202000, 0xf0f0f000, 0xb1b1b100, - 0x84848400, 0x99999900, 0xdfdfdf00, 0x4c4c4c00, 0xcbcbcb00, 0xc2c2c200, - 0x34343400, 0x7e7e7e00, 0x76767600, 0x05050500, 0x6d6d6d00, 0xb7b7b700, - 0xa9a9a900, 0x31313100, 0xd1d1d100, 0x17171700, 0x04040400, 0xd7d7d700, - 0x14141400, 0x58585800, 0x3a3a3a00, 0x61616100, 0xdedede00, 0x1b1b1b00, - 0x11111100, 0x1c1c1c00, 0x32323200, 0x0f0f0f00, 0x9c9c9c00, 0x16161600, - 0x53535300, 0x18181800, 0xf2f2f200, 0x22222200, 0xfefefe00, 0x44444400, - 0xcfcfcf00, 0xb2b2b200, 0xc3c3c300, 0xb5b5b500, 0x7a7a7a00, 0x91919100, - 0x24242400, 0x08080800, 0xe8e8e800, 0xa8a8a800, 0x60606000, 0xfcfcfc00, - 0x69696900, 0x50505000, 0xaaaaaa00, 0xd0d0d000, 0xa0a0a000, 0x7d7d7d00, - 0xa1a1a100, 0x89898900, 0x62626200, 0x97979700, 0x54545400, 0x5b5b5b00, - 0x1e1e1e00, 0x95959500, 0xe0e0e000, 0xffffff00, 0x64646400, 0xd2d2d200, - 0x10101000, 0xc4c4c400, 0x00000000, 0x48484800, 0xa3a3a300, 0xf7f7f700, - 0x75757500, 0xdbdbdb00, 0x8a8a8a00, 0x03030300, 0xe6e6e600, 0xdadada00, - 0x09090900, 0x3f3f3f00, 0xdddddd00, 0x94949400, 0x87878700, 0x5c5c5c00, - 0x83838300, 0x02020200, 0xcdcdcd00, 0x4a4a4a00, 0x90909000, 0x33333300, - 0x73737300, 0x67676700, 0xf6f6f600, 0xf3f3f300, 0x9d9d9d00, 0x7f7f7f00, - 0xbfbfbf00, 0xe2e2e200, 0x52525200, 0x9b9b9b00, 0xd8d8d800, 0x26262600, - 0xc8c8c800, 0x37373700, 0xc6c6c600, 0x3b3b3b00, 0x81818100, 0x96969600, - 0x6f6f6f00, 0x4b4b4b00, 0x13131300, 0xbebebe00, 0x63636300, 0x2e2e2e00, - 0xe9e9e900, 0x79797900, 0xa7a7a700, 0x8c8c8c00, 0x9f9f9f00, 0x6e6e6e00, - 0xbcbcbc00, 0x8e8e8e00, 0x29292900, 0xf5f5f500, 0xf9f9f900, 0xb6b6b600, - 0x2f2f2f00, 0xfdfdfd00, 0xb4b4b400, 0x59595900, 0x78787800, 0x98989800, - 0x06060600, 0x6a6a6a00, 0xe7e7e700, 0x46464600, 0x71717100, 0xbababa00, - 0xd4d4d400, 0x25252500, 0xababab00, 0x42424200, 0x88888800, 0xa2a2a200, - 0x8d8d8d00, 0xfafafa00, 0x72727200, 0x07070700, 0xb9b9b900, 0x55555500, - 0xf8f8f800, 0xeeeeee00, 0xacacac00, 0x0a0a0a00, 0x36363600, 0x49494900, - 0x2a2a2a00, 0x68686800, 0x3c3c3c00, 0x38383800, 0xf1f1f100, 0xa4a4a400, - 0x40404000, 0x28282800, 0xd3d3d300, 0x7b7b7b00, 0xbbbbbb00, 0xc9c9c900, - 0x43434300, 0xc1c1c100, 0x15151500, 0xe3e3e300, 0xadadad00, 0xf4f4f400, - 0x77777700, 0xc7c7c700, 0x80808000, 0x9e9e9e00}, - {0x70700070, 0x2c2c002c, 0xb3b300b3, 0xc0c000c0, 0xe4e400e4, 0x57570057, - 0xeaea00ea, 0xaeae00ae, 0x23230023, 0x6b6b006b, 0x45450045, 0xa5a500a5, - 0xeded00ed, 0x4f4f004f, 0x1d1d001d, 0x92920092, 0x86860086, 0xafaf00af, - 0x7c7c007c, 0x1f1f001f, 0x3e3e003e, 0xdcdc00dc, 0x5e5e005e, 0x0b0b000b, - 0xa6a600a6, 0x39390039, 0xd5d500d5, 0x5d5d005d, 0xd9d900d9, 0x5a5a005a, - 0x51510051, 0x6c6c006c, 0x8b8b008b, 0x9a9a009a, 0xfbfb00fb, 0xb0b000b0, - 0x74740074, 0x2b2b002b, 0xf0f000f0, 0x84840084, 0xdfdf00df, 0xcbcb00cb, - 0x34340034, 0x76760076, 0x6d6d006d, 0xa9a900a9, 0xd1d100d1, 0x04040004, - 0x14140014, 0x3a3a003a, 0xdede00de, 0x11110011, 0x32320032, 0x9c9c009c, - 0x53530053, 0xf2f200f2, 0xfefe00fe, 0xcfcf00cf, 0xc3c300c3, 0x7a7a007a, - 0x24240024, 0xe8e800e8, 0x60600060, 0x69690069, 0xaaaa00aa, 0xa0a000a0, - 0xa1a100a1, 0x62620062, 0x54540054, 0x1e1e001e, 0xe0e000e0, 0x64640064, - 0x10100010, 0x00000000, 0xa3a300a3, 0x75750075, 0x8a8a008a, 0xe6e600e6, - 0x09090009, 0xdddd00dd, 0x87870087, 0x83830083, 0xcdcd00cd, 0x90900090, - 0x73730073, 0xf6f600f6, 0x9d9d009d, 0xbfbf00bf, 0x52520052, 0xd8d800d8, - 0xc8c800c8, 0xc6c600c6, 0x81810081, 0x6f6f006f, 0x13130013, 0x63630063, - 0xe9e900e9, 0xa7a700a7, 0x9f9f009f, 0xbcbc00bc, 0x29290029, 0xf9f900f9, - 0x2f2f002f, 0xb4b400b4, 0x78780078, 0x06060006, 0xe7e700e7, 0x71710071, - 0xd4d400d4, 0xabab00ab, 0x88880088, 0x8d8d008d, 0x72720072, 0xb9b900b9, - 0xf8f800f8, 0xacac00ac, 0x36360036, 0x2a2a002a, 0x3c3c003c, 0xf1f100f1, - 0x40400040, 0xd3d300d3, 0xbbbb00bb, 0x43430043, 0x15150015, 0xadad00ad, - 0x77770077, 0x80800080, 0x82820082, 0xecec00ec, 0x27270027, 0xe5e500e5, - 0x85850085, 0x35350035, 0x0c0c000c, 0x41410041, 0xefef00ef, 0x93930093, - 0x19190019, 0x21210021, 0x0e0e000e, 0x4e4e004e, 0x65650065, 0xbdbd00bd, - 0xb8b800b8, 0x8f8f008f, 0xebeb00eb, 0xcece00ce, 0x30300030, 0x5f5f005f, - 0xc5c500c5, 0x1a1a001a, 0xe1e100e1, 0xcaca00ca, 0x47470047, 0x3d3d003d, - 0x01010001, 0xd6d600d6, 0x56560056, 0x4d4d004d, 0x0d0d000d, 0x66660066, - 0xcccc00cc, 0x2d2d002d, 0x12120012, 0x20200020, 0xb1b100b1, 0x99990099, - 0x4c4c004c, 0xc2c200c2, 0x7e7e007e, 0x05050005, 0xb7b700b7, 0x31310031, - 0x17170017, 0xd7d700d7, 0x58580058, 0x61610061, 0x1b1b001b, 0x1c1c001c, - 0x0f0f000f, 0x16160016, 0x18180018, 0x22220022, 0x44440044, 0xb2b200b2, - 0xb5b500b5, 0x91910091, 0x08080008, 0xa8a800a8, 0xfcfc00fc, 0x50500050, - 0xd0d000d0, 0x7d7d007d, 0x89890089, 0x97970097, 0x5b5b005b, 0x95950095, - 0xffff00ff, 0xd2d200d2, 0xc4c400c4, 0x48480048, 0xf7f700f7, 0xdbdb00db, - 0x03030003, 0xdada00da, 0x3f3f003f, 0x94940094, 0x5c5c005c, 0x02020002, - 0x4a4a004a, 0x33330033, 0x67670067, 0xf3f300f3, 0x7f7f007f, 0xe2e200e2, - 0x9b9b009b, 0x26260026, 0x37370037, 0x3b3b003b, 0x96960096, 0x4b4b004b, - 0xbebe00be, 0x2e2e002e, 0x79790079, 0x8c8c008c, 0x6e6e006e, 0x8e8e008e, - 0xf5f500f5, 0xb6b600b6, 0xfdfd00fd, 0x59590059, 0x98980098, 0x6a6a006a, - 0x46460046, 0xbaba00ba, 0x25250025, 0x42420042, 0xa2a200a2, 0xfafa00fa, - 0x07070007, 0x55550055, 0xeeee00ee, 0x0a0a000a, 0x49490049, 0x68680068, - 0x38380038, 0xa4a400a4, 0x28280028, 0x7b7b007b, 0xc9c900c9, 0xc1c100c1, - 0xe3e300e3, 0xf4f400f4, 0xc7c700c7, 0x9e9e009e}, - {0x00e0e0e0, 0x00050505, 0x00585858, 0x00d9d9d9, 0x00676767, 0x004e4e4e, - 0x00818181, 0x00cbcbcb, 0x00c9c9c9, 0x000b0b0b, 0x00aeaeae, 0x006a6a6a, - 0x00d5d5d5, 0x00181818, 0x005d5d5d, 0x00828282, 0x00464646, 0x00dfdfdf, - 0x00d6d6d6, 0x00272727, 0x008a8a8a, 0x00323232, 0x004b4b4b, 0x00424242, - 0x00dbdbdb, 0x001c1c1c, 0x009e9e9e, 0x009c9c9c, 0x003a3a3a, 0x00cacaca, - 0x00252525, 0x007b7b7b, 0x000d0d0d, 0x00717171, 0x005f5f5f, 0x001f1f1f, - 0x00f8f8f8, 0x00d7d7d7, 0x003e3e3e, 0x009d9d9d, 0x007c7c7c, 0x00606060, - 0x00b9b9b9, 0x00bebebe, 0x00bcbcbc, 0x008b8b8b, 0x00161616, 0x00343434, - 0x004d4d4d, 0x00c3c3c3, 0x00727272, 0x00959595, 0x00ababab, 0x008e8e8e, - 0x00bababa, 0x007a7a7a, 0x00b3b3b3, 0x00020202, 0x00b4b4b4, 0x00adadad, - 0x00a2a2a2, 0x00acacac, 0x00d8d8d8, 0x009a9a9a, 0x00171717, 0x001a1a1a, - 0x00353535, 0x00cccccc, 0x00f7f7f7, 0x00999999, 0x00616161, 0x005a5a5a, - 0x00e8e8e8, 0x00242424, 0x00565656, 0x00404040, 0x00e1e1e1, 0x00636363, - 0x00090909, 0x00333333, 0x00bfbfbf, 0x00989898, 0x00979797, 0x00858585, - 0x00686868, 0x00fcfcfc, 0x00ececec, 0x000a0a0a, 0x00dadada, 0x006f6f6f, - 0x00535353, 0x00626262, 0x00a3a3a3, 0x002e2e2e, 0x00080808, 0x00afafaf, - 0x00282828, 0x00b0b0b0, 0x00747474, 0x00c2c2c2, 0x00bdbdbd, 0x00363636, - 0x00222222, 0x00383838, 0x00646464, 0x001e1e1e, 0x00393939, 0x002c2c2c, - 0x00a6a6a6, 0x00303030, 0x00e5e5e5, 0x00444444, 0x00fdfdfd, 0x00888888, - 0x009f9f9f, 0x00656565, 0x00878787, 0x006b6b6b, 0x00f4f4f4, 0x00232323, - 0x00484848, 0x00101010, 0x00d1d1d1, 0x00515151, 0x00c0c0c0, 0x00f9f9f9, - 0x00d2d2d2, 0x00a0a0a0, 0x00555555, 0x00a1a1a1, 0x00414141, 0x00fafafa, - 0x00434343, 0x00131313, 0x00c4c4c4, 0x002f2f2f, 0x00a8a8a8, 0x00b6b6b6, - 0x003c3c3c, 0x002b2b2b, 0x00c1c1c1, 0x00ffffff, 0x00c8c8c8, 0x00a5a5a5, - 0x00202020, 0x00898989, 0x00000000, 0x00909090, 0x00474747, 0x00efefef, - 0x00eaeaea, 0x00b7b7b7, 0x00151515, 0x00060606, 0x00cdcdcd, 0x00b5b5b5, - 0x00121212, 0x007e7e7e, 0x00bbbbbb, 0x00292929, 0x000f0f0f, 0x00b8b8b8, - 0x00070707, 0x00040404, 0x009b9b9b, 0x00949494, 0x00212121, 0x00666666, - 0x00e6e6e6, 0x00cecece, 0x00ededed, 0x00e7e7e7, 0x003b3b3b, 0x00fefefe, - 0x007f7f7f, 0x00c5c5c5, 0x00a4a4a4, 0x00373737, 0x00b1b1b1, 0x004c4c4c, - 0x00919191, 0x006e6e6e, 0x008d8d8d, 0x00767676, 0x00030303, 0x002d2d2d, - 0x00dedede, 0x00969696, 0x00262626, 0x007d7d7d, 0x00c6c6c6, 0x005c5c5c, - 0x00d3d3d3, 0x00f2f2f2, 0x004f4f4f, 0x00191919, 0x003f3f3f, 0x00dcdcdc, - 0x00797979, 0x001d1d1d, 0x00525252, 0x00ebebeb, 0x00f3f3f3, 0x006d6d6d, - 0x005e5e5e, 0x00fbfbfb, 0x00696969, 0x00b2b2b2, 0x00f0f0f0, 0x00313131, - 0x000c0c0c, 0x00d4d4d4, 0x00cfcfcf, 0x008c8c8c, 0x00e2e2e2, 0x00757575, - 0x00a9a9a9, 0x004a4a4a, 0x00575757, 0x00848484, 0x00111111, 0x00454545, - 0x001b1b1b, 0x00f5f5f5, 0x00e4e4e4, 0x000e0e0e, 0x00737373, 0x00aaaaaa, - 0x00f1f1f1, 0x00dddddd, 0x00595959, 0x00141414, 0x006c6c6c, 0x00929292, - 0x00545454, 0x00d0d0d0, 0x00787878, 0x00707070, 0x00e3e3e3, 0x00494949, - 0x00808080, 0x00505050, 0x00a7a7a7, 0x00f6f6f6, 0x00777777, 0x00939393, - 0x00868686, 0x00838383, 0x002a2a2a, 0x00c7c7c7, 0x005b5b5b, 0x00e9e9e9, - 0x00eeeeee, 0x008f8f8f, 0x00010101, 0x003d3d3d}, - {0x38003838, 0x41004141, 0x16001616, 0x76007676, 0xd900d9d9, 0x93009393, - 0x60006060, 0xf200f2f2, 0x72007272, 0xc200c2c2, 0xab00abab, 0x9a009a9a, - 0x75007575, 0x06000606, 0x57005757, 0xa000a0a0, 0x91009191, 0xf700f7f7, - 0xb500b5b5, 0xc900c9c9, 0xa200a2a2, 0x8c008c8c, 0xd200d2d2, 0x90009090, - 0xf600f6f6, 0x07000707, 0xa700a7a7, 0x27002727, 0x8e008e8e, 0xb200b2b2, - 0x49004949, 0xde00dede, 0x43004343, 0x5c005c5c, 0xd700d7d7, 0xc700c7c7, - 0x3e003e3e, 0xf500f5f5, 0x8f008f8f, 0x67006767, 0x1f001f1f, 0x18001818, - 0x6e006e6e, 0xaf00afaf, 0x2f002f2f, 0xe200e2e2, 0x85008585, 0x0d000d0d, - 0x53005353, 0xf000f0f0, 0x9c009c9c, 0x65006565, 0xea00eaea, 0xa300a3a3, - 0xae00aeae, 0x9e009e9e, 0xec00ecec, 0x80008080, 0x2d002d2d, 0x6b006b6b, - 0xa800a8a8, 0x2b002b2b, 0x36003636, 0xa600a6a6, 0xc500c5c5, 0x86008686, - 0x4d004d4d, 0x33003333, 0xfd00fdfd, 0x66006666, 0x58005858, 0x96009696, - 0x3a003a3a, 0x09000909, 0x95009595, 0x10001010, 0x78007878, 0xd800d8d8, - 0x42004242, 0xcc00cccc, 0xef00efef, 0x26002626, 0xe500e5e5, 0x61006161, - 0x1a001a1a, 0x3f003f3f, 0x3b003b3b, 0x82008282, 0xb600b6b6, 0xdb00dbdb, - 0xd400d4d4, 0x98009898, 0xe800e8e8, 0x8b008b8b, 0x02000202, 0xeb00ebeb, - 0x0a000a0a, 0x2c002c2c, 0x1d001d1d, 0xb000b0b0, 0x6f006f6f, 0x8d008d8d, - 0x88008888, 0x0e000e0e, 0x19001919, 0x87008787, 0x4e004e4e, 0x0b000b0b, - 0xa900a9a9, 0x0c000c0c, 0x79007979, 0x11001111, 0x7f007f7f, 0x22002222, - 0xe700e7e7, 0x59005959, 0xe100e1e1, 0xda00dada, 0x3d003d3d, 0xc800c8c8, - 0x12001212, 0x04000404, 0x74007474, 0x54005454, 0x30003030, 0x7e007e7e, - 0xb400b4b4, 0x28002828, 0x55005555, 0x68006868, 0x50005050, 0xbe00bebe, - 0xd000d0d0, 0xc400c4c4, 0x31003131, 0xcb00cbcb, 0x2a002a2a, 0xad00adad, - 0x0f000f0f, 0xca00caca, 0x70007070, 0xff00ffff, 0x32003232, 0x69006969, - 0x08000808, 0x62006262, 0x00000000, 0x24002424, 0xd100d1d1, 0xfb00fbfb, - 0xba00baba, 0xed00eded, 0x45004545, 0x81008181, 0x73007373, 0x6d006d6d, - 0x84008484, 0x9f009f9f, 0xee00eeee, 0x4a004a4a, 0xc300c3c3, 0x2e002e2e, - 0xc100c1c1, 0x01000101, 0xe600e6e6, 0x25002525, 0x48004848, 0x99009999, - 0xb900b9b9, 0xb300b3b3, 0x7b007b7b, 0xf900f9f9, 0xce00cece, 0xbf00bfbf, - 0xdf00dfdf, 0x71007171, 0x29002929, 0xcd00cdcd, 0x6c006c6c, 0x13001313, - 0x64006464, 0x9b009b9b, 0x63006363, 0x9d009d9d, 0xc000c0c0, 0x4b004b4b, - 0xb700b7b7, 0xa500a5a5, 0x89008989, 0x5f005f5f, 0xb100b1b1, 0x17001717, - 0xf400f4f4, 0xbc00bcbc, 0xd300d3d3, 0x46004646, 0xcf00cfcf, 0x37003737, - 0x5e005e5e, 0x47004747, 0x94009494, 0xfa00fafa, 0xfc00fcfc, 0x5b005b5b, - 0x97009797, 0xfe00fefe, 0x5a005a5a, 0xac00acac, 0x3c003c3c, 0x4c004c4c, - 0x03000303, 0x35003535, 0xf300f3f3, 0x23002323, 0xb800b8b8, 0x5d005d5d, - 0x6a006a6a, 0x92009292, 0xd500d5d5, 0x21002121, 0x44004444, 0x51005151, - 0xc600c6c6, 0x7d007d7d, 0x39003939, 0x83008383, 0xdc00dcdc, 0xaa00aaaa, - 0x7c007c7c, 0x77007777, 0x56005656, 0x05000505, 0x1b001b1b, 0xa400a4a4, - 0x15001515, 0x34003434, 0x1e001e1e, 0x1c001c1c, 0xf800f8f8, 0x52005252, - 0x20002020, 0x14001414, 0xe900e9e9, 0xbd00bdbd, 0xdd00dddd, 0xe400e4e4, - 0xa100a1a1, 0xe000e0e0, 0x8a008a8a, 0xf100f1f1, 0xd600d6d6, 0x7a007a7a, - 0xbb00bbbb, 0xe300e3e3, 0x40004040, 0x4f004f4f} -}; - -/* Key generation constants */ -static const u32 SIGMA[] = { - 0xa09e667f, 0x3bcc908b, 0xb67ae858, 0x4caa73b2, 0xc6ef372f, 0xe94f82be, - 0x54ff53a5, 0xf1d36f1c, 0x10e527fa, 0xde682d1d, 0xb05688c2, 0xb3e6c1fd -}; - -/* The phi algorithm given in C.2.7 of the Camellia spec document. */ -/* - * This version does not attempt to minimize amount of temporary - * variables, but instead explicitly exposes algorithm's parallelism. - * It is therefore most appropriate for platforms with not less than - * ~16 registers. For platforms with less registers [well, x86 to be - * specific] assembler version should be/is provided anyway... - */ -#define Camellia_Feistel(_s0,_s1,_s2,_s3,_key) do {\ - register u32 _t0,_t1,_t2,_t3;\ -\ - _t0 = _s0 ^ (_key)[0];\ - _t3 = SBOX4_4404[_t0&0xff];\ - _t1 = _s1 ^ (_key)[1];\ - _t3 ^= SBOX3_3033[(_t0 >> 8)&0xff];\ - _t2 = SBOX1_1110[_t1&0xff];\ - _t3 ^= SBOX2_0222[(_t0 >> 16)&0xff];\ - _t2 ^= SBOX4_4404[(_t1 >> 8)&0xff];\ - _t3 ^= SBOX1_1110[(_t0 >> 24)];\ - _t2 ^= _t3;\ - _t3 = RightRotate(_t3,8);\ - _t2 ^= SBOX3_3033[(_t1 >> 16)&0xff];\ - _s3 ^= _t3;\ - _t2 ^= SBOX2_0222[(_t1 >> 24)];\ - _s2 ^= _t2; \ - _s3 ^= _t2;\ -} while(0) - -/* - * Note that n has to be less than 32. Rotations for larger amount - * of bits are achieved by "rotating" order of s-elements and - * adjusting n accordingly, e.g. RotLeft128(s1,s2,s3,s0,n-32). - */ -#define RotLeft128(_s0,_s1,_s2,_s3,_n) do {\ - u32 _t0=_s0>>(32-_n);\ - _s0 = (_s0<<_n) | (_s1>>(32-_n));\ - _s1 = (_s1<<_n) | (_s2>>(32-_n));\ - _s2 = (_s2<<_n) | (_s3>>(32-_n));\ - _s3 = (_s3<<_n) | _t0;\ -} while (0) - -int Camellia_Ekeygen(int keyBitLength, const u8 *rawKey, KEY_TABLE_TYPE k) -{ - register u32 s0, s1, s2, s3; - - k[0] = s0 = GETU32(rawKey); - k[1] = s1 = GETU32(rawKey + 4); - k[2] = s2 = GETU32(rawKey + 8); - k[3] = s3 = GETU32(rawKey + 12); - - if (keyBitLength != 128) { - k[8] = s0 = GETU32(rawKey + 16); - k[9] = s1 = GETU32(rawKey + 20); - if (keyBitLength == 192) { - k[10] = s2 = ~s0; - k[11] = s3 = ~s1; - } else { - k[10] = s2 = GETU32(rawKey + 24); - k[11] = s3 = GETU32(rawKey + 28); - } - s0 ^= k[0], s1 ^= k[1], s2 ^= k[2], s3 ^= k[3]; - } - - /* Use the Feistel routine to scramble the key material */ - Camellia_Feistel(s0, s1, s2, s3, SIGMA + 0); - Camellia_Feistel(s2, s3, s0, s1, SIGMA + 2); - - s0 ^= k[0], s1 ^= k[1], s2 ^= k[2], s3 ^= k[3]; - Camellia_Feistel(s0, s1, s2, s3, SIGMA + 4); - Camellia_Feistel(s2, s3, s0, s1, SIGMA + 6); - - /* Fill the keyTable. Requires many block rotations. */ - if (keyBitLength == 128) { - k[4] = s0, k[5] = s1, k[6] = s2, k[7] = s3; - RotLeft128(s0, s1, s2, s3, 15); /* KA <<< 15 */ - k[12] = s0, k[13] = s1, k[14] = s2, k[15] = s3; - RotLeft128(s0, s1, s2, s3, 15); /* KA <<< 30 */ - k[16] = s0, k[17] = s1, k[18] = s2, k[19] = s3; - RotLeft128(s0, s1, s2, s3, 15); /* KA <<< 45 */ - k[24] = s0, k[25] = s1; - RotLeft128(s0, s1, s2, s3, 15); /* KA <<< 60 */ - k[28] = s0, k[29] = s1, k[30] = s2, k[31] = s3; - RotLeft128(s1, s2, s3, s0, 2); /* KA <<< 94 */ - k[40] = s1, k[41] = s2, k[42] = s3, k[43] = s0; - RotLeft128(s1, s2, s3, s0, 17); /* KA <<<111 */ - k[48] = s1, k[49] = s2, k[50] = s3, k[51] = s0; - - s0 = k[0], s1 = k[1], s2 = k[2], s3 = k[3]; - RotLeft128(s0, s1, s2, s3, 15); /* KL <<< 15 */ - k[8] = s0, k[9] = s1, k[10] = s2, k[11] = s3; - RotLeft128(s0, s1, s2, s3, 30); /* KL <<< 45 */ - k[20] = s0, k[21] = s1, k[22] = s2, k[23] = s3; - RotLeft128(s0, s1, s2, s3, 15); /* KL <<< 60 */ - k[26] = s2, k[27] = s3; - RotLeft128(s0, s1, s2, s3, 17); /* KL <<< 77 */ - k[32] = s0, k[33] = s1, k[34] = s2, k[35] = s3; - RotLeft128(s0, s1, s2, s3, 17); /* KL <<< 94 */ - k[36] = s0, k[37] = s1, k[38] = s2, k[39] = s3; - RotLeft128(s0, s1, s2, s3, 17); /* KL <<<111 */ - k[44] = s0, k[45] = s1, k[46] = s2, k[47] = s3; - - return 3; /* grand rounds */ - } else { - k[12] = s0, k[13] = s1, k[14] = s2, k[15] = s3; - s0 ^= k[8], s1 ^= k[9], s2 ^= k[10], s3 ^= k[11]; - Camellia_Feistel(s0, s1, s2, s3, (SIGMA + 8)); - Camellia_Feistel(s2, s3, s0, s1, (SIGMA + 10)); - - k[4] = s0, k[5] = s1, k[6] = s2, k[7] = s3; - RotLeft128(s0, s1, s2, s3, 30); /* KB <<< 30 */ - k[20] = s0, k[21] = s1, k[22] = s2, k[23] = s3; - RotLeft128(s0, s1, s2, s3, 30); /* KB <<< 60 */ - k[40] = s0, k[41] = s1, k[42] = s2, k[43] = s3; - RotLeft128(s1, s2, s3, s0, 19); /* KB <<<111 */ - k[64] = s1, k[65] = s2, k[66] = s3, k[67] = s0; - - s0 = k[8], s1 = k[9], s2 = k[10], s3 = k[11]; - RotLeft128(s0, s1, s2, s3, 15); /* KR <<< 15 */ - k[8] = s0, k[9] = s1, k[10] = s2, k[11] = s3; - RotLeft128(s0, s1, s2, s3, 15); /* KR <<< 30 */ - k[16] = s0, k[17] = s1, k[18] = s2, k[19] = s3; - RotLeft128(s0, s1, s2, s3, 30); /* KR <<< 60 */ - k[36] = s0, k[37] = s1, k[38] = s2, k[39] = s3; - RotLeft128(s1, s2, s3, s0, 2); /* KR <<< 94 */ - k[52] = s1, k[53] = s2, k[54] = s3, k[55] = s0; - - s0 = k[12], s1 = k[13], s2 = k[14], s3 = k[15]; - RotLeft128(s0, s1, s2, s3, 15); /* KA <<< 15 */ - k[12] = s0, k[13] = s1, k[14] = s2, k[15] = s3; - RotLeft128(s0, s1, s2, s3, 30); /* KA <<< 45 */ - k[28] = s0, k[29] = s1, k[30] = s2, k[31] = s3; - /* KA <<< 77 */ - k[48] = s1, k[49] = s2, k[50] = s3, k[51] = s0; - RotLeft128(s1, s2, s3, s0, 17); /* KA <<< 94 */ - k[56] = s1, k[57] = s2, k[58] = s3, k[59] = s0; - - s0 = k[0], s1 = k[1], s2 = k[2], s3 = k[3]; - RotLeft128(s1, s2, s3, s0, 13); /* KL <<< 45 */ - k[24] = s1, k[25] = s2, k[26] = s3, k[27] = s0; - RotLeft128(s1, s2, s3, s0, 15); /* KL <<< 60 */ - k[32] = s1, k[33] = s2, k[34] = s3, k[35] = s0; - RotLeft128(s1, s2, s3, s0, 17); /* KL <<< 77 */ - k[44] = s1, k[45] = s2, k[46] = s3, k[47] = s0; - RotLeft128(s2, s3, s0, s1, 2); /* KL <<<111 */ - k[60] = s2, k[61] = s3, k[62] = s0, k[63] = s1; - - return 4; /* grand rounds */ - } - /* - * It is possible to perform certain precalculations, which - * would spare few cycles in block procedure. It's not done, - * because it upsets the performance balance between key - * setup and block procedures, negatively affecting overall - * throughput in applications operating on short messages - * and volatile keys. - */ -} - -void Camellia_EncryptBlock_Rounds(int grandRounds, const u8 plaintext[], - const KEY_TABLE_TYPE keyTable, - u8 ciphertext[]) -{ - register u32 s0, s1, s2, s3; - const u32 *k = keyTable, *kend = keyTable + grandRounds * 16; - - s0 = GETU32(plaintext) ^ k[0]; - s1 = GETU32(plaintext + 4) ^ k[1]; - s2 = GETU32(plaintext + 8) ^ k[2]; - s3 = GETU32(plaintext + 12) ^ k[3]; - k += 4; - - while (1) { - /* Camellia makes 6 Feistel rounds */ - Camellia_Feistel(s0, s1, s2, s3, k + 0); - Camellia_Feistel(s2, s3, s0, s1, k + 2); - Camellia_Feistel(s0, s1, s2, s3, k + 4); - Camellia_Feistel(s2, s3, s0, s1, k + 6); - Camellia_Feistel(s0, s1, s2, s3, k + 8); - Camellia_Feistel(s2, s3, s0, s1, k + 10); - k += 12; - - if (k == kend) - break; - - /* - * This is the same function as the diffusion function D of the - * accompanying documentation. See section 3.2 for properties of the - * FLlayer function. - */ - s1 ^= LeftRotate(s0 & k[0], 1); - s2 ^= s3 | k[3]; - s0 ^= s1 | k[1]; - s3 ^= LeftRotate(s2 & k[2], 1); - k += 4; - } - - s2 ^= k[0], s3 ^= k[1], s0 ^= k[2], s1 ^= k[3]; - - PUTU32(ciphertext, s2); - PUTU32(ciphertext + 4, s3); - PUTU32(ciphertext + 8, s0); - PUTU32(ciphertext + 12, s1); -} - -void Camellia_EncryptBlock(int keyBitLength, const u8 plaintext[], - const KEY_TABLE_TYPE keyTable, u8 ciphertext[]) -{ - Camellia_EncryptBlock_Rounds(keyBitLength == 128 ? 3 : 4, - plaintext, keyTable, ciphertext); -} - -void Camellia_DecryptBlock_Rounds(int grandRounds, const u8 ciphertext[], - const KEY_TABLE_TYPE keyTable, - u8 plaintext[]) -{ - u32 s0, s1, s2, s3; - const u32 *k = keyTable + grandRounds * 16, *kend = keyTable + 4; - - s0 = GETU32(ciphertext) ^ k[0]; - s1 = GETU32(ciphertext + 4) ^ k[1]; - s2 = GETU32(ciphertext + 8) ^ k[2]; - s3 = GETU32(ciphertext + 12) ^ k[3]; - - while (1) { - /* Camellia makes 6 Feistel rounds */ - k -= 12; - Camellia_Feistel(s0, s1, s2, s3, k + 10); - Camellia_Feistel(s2, s3, s0, s1, k + 8); - Camellia_Feistel(s0, s1, s2, s3, k + 6); - Camellia_Feistel(s2, s3, s0, s1, k + 4); - Camellia_Feistel(s0, s1, s2, s3, k + 2); - Camellia_Feistel(s2, s3, s0, s1, k + 0); - - if (k == kend) - break; - - /* - * This is the same function as the diffusion function D of the - * accompanying documentation. See section 3.2 for properties of the - * FLlayer function. - */ - k -= 4; - s1 ^= LeftRotate(s0 & k[2], 1); - s2 ^= s3 | k[1]; - s0 ^= s1 | k[3]; - s3 ^= LeftRotate(s2 & k[0], 1); - } - - k -= 4; - s2 ^= k[0], s3 ^= k[1], s0 ^= k[2], s1 ^= k[3]; - - PUTU32(plaintext, s2); - PUTU32(plaintext + 4, s3); - PUTU32(plaintext + 8, s0); - PUTU32(plaintext + 12, s1); -} - -void Camellia_DecryptBlock(int keyBitLength, const u8 ciphertext[], - const KEY_TABLE_TYPE keyTable, u8 plaintext[]) -{ - Camellia_DecryptBlock_Rounds(keyBitLength == 128 ? 3 : 4, - ciphertext, keyTable, plaintext); -} diff --git a/openssl/src/crypto/camellia/cmll_cbc.c b/openssl/src/crypto/camellia/cmll_cbc.c deleted file mode 100644 index 140681a9b..000000000 --- a/openssl/src/crypto/camellia/cmll_cbc.c +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include - -void Camellia_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const CAMELLIA_KEY *key, - unsigned char *ivec, const int enc) -{ - - if (enc) - CRYPTO_cbc128_encrypt(in, out, len, key, ivec, - (block128_f) Camellia_encrypt); - else - CRYPTO_cbc128_decrypt(in, out, len, key, ivec, - (block128_f) Camellia_decrypt); -} diff --git a/openssl/src/crypto/camellia/cmll_cfb.c b/openssl/src/crypto/camellia/cmll_cfb.c deleted file mode 100644 index 8a92572d9..000000000 --- a/openssl/src/crypto/camellia/cmll_cfb.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include - -/* - * The input and output encrypted as though 128bit cfb mode is being used. - * The extra state information to record how much of the 128bit block we have - * used is contained in *num; - */ - -void Camellia_cfb128_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const CAMELLIA_KEY *key, - unsigned char *ivec, int *num, const int enc) -{ - - CRYPTO_cfb128_encrypt(in, out, length, key, ivec, num, enc, - (block128_f) Camellia_encrypt); -} - -/* N.B. This expects the input to be packed, MS bit first */ -void Camellia_cfb1_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const CAMELLIA_KEY *key, - unsigned char *ivec, int *num, const int enc) -{ - CRYPTO_cfb128_1_encrypt(in, out, length, key, ivec, num, enc, - (block128_f) Camellia_encrypt); -} - -void Camellia_cfb8_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const CAMELLIA_KEY *key, - unsigned char *ivec, int *num, const int enc) -{ - CRYPTO_cfb128_8_encrypt(in, out, length, key, ivec, num, enc, - (block128_f) Camellia_encrypt); -} diff --git a/openssl/src/crypto/camellia/cmll_ctr.c b/openssl/src/crypto/camellia/cmll_ctr.c deleted file mode 100644 index 26d875e34..000000000 --- a/openssl/src/crypto/camellia/cmll_ctr.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include - -void Camellia_ctr128_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const CAMELLIA_KEY *key, - unsigned char ivec[CAMELLIA_BLOCK_SIZE], - unsigned char ecount_buf[CAMELLIA_BLOCK_SIZE], - unsigned int *num) -{ - - CRYPTO_ctr128_encrypt(in, out, length, key, ivec, ecount_buf, num, - (block128_f) Camellia_encrypt); -} diff --git a/openssl/src/crypto/camellia/cmll_ecb.c b/openssl/src/crypto/camellia/cmll_ecb.c deleted file mode 100644 index 86ffbd51e..000000000 --- a/openssl/src/crypto/camellia/cmll_ecb.c +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "cmll_local.h" - -void Camellia_ecb_encrypt(const unsigned char *in, unsigned char *out, - const CAMELLIA_KEY *key, const int enc) -{ - if (CAMELLIA_ENCRYPT == enc) - Camellia_encrypt(in, out, key); - else - Camellia_decrypt(in, out, key); -} diff --git a/openssl/src/crypto/camellia/cmll_local.h b/openssl/src/crypto/camellia/cmll_local.h deleted file mode 100644 index c1d940d3d..000000000 --- a/openssl/src/crypto/camellia/cmll_local.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2006-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* ==================================================================== - * Copyright 2006 NTT (Nippon Telegraph and Telephone Corporation) . - * ALL RIGHTS RESERVED. - * - * Intellectual Property information for Camellia: - * http://info.isl.ntt.co.jp/crypt/eng/info/chiteki.html - * - * News Release for Announcement of Camellia open source: - * http://www.ntt.co.jp/news/news06e/0604/060413a.html - * - * The Camellia Code included herein is developed by - * NTT (Nippon Telegraph and Telephone Corporation), and is contributed - * to the OpenSSL project. - */ - -#ifndef OSSL_CRYPTO_CAMELLIA_CMLL_LOCAL_H -# define OSSL_CRYPTO_CAMELLIA_CMLL_LOCAL_H - -typedef unsigned int u32; -typedef unsigned char u8; - -int Camellia_Ekeygen(int keyBitLength, const u8 *rawKey, - KEY_TABLE_TYPE keyTable); -void Camellia_EncryptBlock_Rounds(int grandRounds, const u8 plaintext[], - const KEY_TABLE_TYPE keyTable, - u8 ciphertext[]); -void Camellia_DecryptBlock_Rounds(int grandRounds, const u8 ciphertext[], - const KEY_TABLE_TYPE keyTable, - u8 plaintext[]); -void Camellia_EncryptBlock(int keyBitLength, const u8 plaintext[], - const KEY_TABLE_TYPE keyTable, u8 ciphertext[]); -void Camellia_DecryptBlock(int keyBitLength, const u8 ciphertext[], - const KEY_TABLE_TYPE keyTable, u8 plaintext[]); -#endif /* #ifndef OSSL_CRYPTO_CAMELLIA_CMLL_LOCAL_H */ diff --git a/openssl/src/crypto/camellia/cmll_misc.c b/openssl/src/crypto/camellia/cmll_misc.c deleted file mode 100644 index f98dff7e3..000000000 --- a/openssl/src/crypto/camellia/cmll_misc.c +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include "cmll_local.h" - -int Camellia_set_key(const unsigned char *userKey, const int bits, - CAMELLIA_KEY *key) -{ - if (!userKey || !key) - return -1; - if (bits != 128 && bits != 192 && bits != 256) - return -2; - key->grand_rounds = Camellia_Ekeygen(bits, userKey, key->u.rd_key); - return 0; -} - -void Camellia_encrypt(const unsigned char *in, unsigned char *out, - const CAMELLIA_KEY *key) -{ - Camellia_EncryptBlock_Rounds(key->grand_rounds, in, key->u.rd_key, out); -} - -void Camellia_decrypt(const unsigned char *in, unsigned char *out, - const CAMELLIA_KEY *key) -{ - Camellia_DecryptBlock_Rounds(key->grand_rounds, in, key->u.rd_key, out); -} diff --git a/openssl/src/crypto/camellia/cmll_ofb.c b/openssl/src/crypto/camellia/cmll_ofb.c deleted file mode 100644 index 4eeb0b5b7..000000000 --- a/openssl/src/crypto/camellia/cmll_ofb.c +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include - -/* - * The input and output encrypted as though 128bit ofb mode is being used. - * The extra state information to record how much of the 128bit block we have - * used is contained in *num; - */ -void Camellia_ofb128_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const CAMELLIA_KEY *key, - unsigned char *ivec, int *num) -{ - CRYPTO_ofb128_encrypt(in, out, length, key, ivec, num, - (block128_f) Camellia_encrypt); -} diff --git a/openssl/src/crypto/camellia/gen/darwin_x64/cmll-x86_64.s b/openssl/src/crypto/camellia/gen/darwin_x64/cmll-x86_64.s deleted file mode 100644 index b255fcfd1..000000000 --- a/openssl/src/crypto/camellia/gen/darwin_x64/cmll-x86_64.s +++ /dev/null @@ -1,1904 +0,0 @@ -.text - - -.globl _Camellia_EncryptBlock - -.p2align 4 -_Camellia_EncryptBlock: - - movl $128,%eax - subl %edi,%eax - movl $3,%edi - adcl $0,%edi - jmp L$enc_rounds - - - -.globl _Camellia_EncryptBlock_Rounds - -.p2align 4 -L$enc_rounds: -_Camellia_EncryptBlock_Rounds: - - pushq %rbx - - pushq %rbp - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$enc_prologue: - - - movq %rcx,%r13 - movq %rdx,%r14 - - shll $6,%edi - leaq L$Camellia_SBOX(%rip),%rbp - leaq (%r14,%rdi,1),%r15 - - movl 0(%rsi),%r8d - movl 4(%rsi),%r9d - movl 8(%rsi),%r10d - bswapl %r8d - movl 12(%rsi),%r11d - bswapl %r9d - bswapl %r10d - bswapl %r11d - - call _x86_64_Camellia_encrypt - - bswapl %r8d - bswapl %r9d - bswapl %r10d - movl %r8d,0(%r13) - bswapl %r11d - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%rbp - - movq 32(%rsp),%rbx - - leaq 40(%rsp),%rsp - -L$enc_epilogue: - .byte 0xf3,0xc3 - - - - -.p2align 4 -_x86_64_Camellia_encrypt: - - xorl 0(%r14),%r9d - xorl 4(%r14),%r8d - xorl 8(%r14),%r11d - xorl 12(%r14),%r10d -.p2align 4 -L$eloop: - movl 16(%r14),%ebx - movl 20(%r14),%eax - - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 24(%r14),%ebx - movl 28(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 32(%r14),%ebx - movl 36(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 40(%r14),%ebx - movl 44(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 48(%r14),%ebx - movl 52(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 56(%r14),%ebx - movl 60(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 64(%r14),%ebx - movl 68(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - leaq 64(%r14),%r14 - cmpq %r15,%r14 - movl 8(%r14),%edx - movl 12(%r14),%ecx - je L$edone - - andl %r8d,%eax - orl %r11d,%edx - roll $1,%eax - xorl %edx,%r10d - xorl %eax,%r9d - andl %r10d,%ecx - orl %r9d,%ebx - roll $1,%ecx - xorl %ebx,%r8d - xorl %ecx,%r11d - jmp L$eloop - -.p2align 4 -L$edone: - xorl %r10d,%eax - xorl %r11d,%ebx - xorl %r8d,%ecx - xorl %r9d,%edx - - movl %eax,%r8d - movl %ebx,%r9d - movl %ecx,%r10d - movl %edx,%r11d - -.byte 0xf3,0xc3 - - - - -.globl _Camellia_DecryptBlock - -.p2align 4 -_Camellia_DecryptBlock: - - movl $128,%eax - subl %edi,%eax - movl $3,%edi - adcl $0,%edi - jmp L$dec_rounds - - - -.globl _Camellia_DecryptBlock_Rounds - -.p2align 4 -L$dec_rounds: -_Camellia_DecryptBlock_Rounds: - - pushq %rbx - - pushq %rbp - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$dec_prologue: - - - movq %rcx,%r13 - movq %rdx,%r15 - - shll $6,%edi - leaq L$Camellia_SBOX(%rip),%rbp - leaq (%r15,%rdi,1),%r14 - - movl 0(%rsi),%r8d - movl 4(%rsi),%r9d - movl 8(%rsi),%r10d - bswapl %r8d - movl 12(%rsi),%r11d - bswapl %r9d - bswapl %r10d - bswapl %r11d - - call _x86_64_Camellia_decrypt - - bswapl %r8d - bswapl %r9d - bswapl %r10d - movl %r8d,0(%r13) - bswapl %r11d - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%rbp - - movq 32(%rsp),%rbx - - leaq 40(%rsp),%rsp - -L$dec_epilogue: - .byte 0xf3,0xc3 - - - - -.p2align 4 -_x86_64_Camellia_decrypt: - - xorl 0(%r14),%r9d - xorl 4(%r14),%r8d - xorl 8(%r14),%r11d - xorl 12(%r14),%r10d -.p2align 4 -L$dloop: - movl -8(%r14),%ebx - movl -4(%r14),%eax - - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -16(%r14),%ebx - movl -12(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -24(%r14),%ebx - movl -20(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -32(%r14),%ebx - movl -28(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -40(%r14),%ebx - movl -36(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -48(%r14),%ebx - movl -44(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -56(%r14),%ebx - movl -52(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - leaq -64(%r14),%r14 - cmpq %r15,%r14 - movl 0(%r14),%edx - movl 4(%r14),%ecx - je L$ddone - - andl %r8d,%eax - orl %r11d,%edx - roll $1,%eax - xorl %edx,%r10d - xorl %eax,%r9d - andl %r10d,%ecx - orl %r9d,%ebx - roll $1,%ecx - xorl %ebx,%r8d - xorl %ecx,%r11d - - jmp L$dloop - -.p2align 4 -L$ddone: - xorl %r10d,%ecx - xorl %r11d,%edx - xorl %r8d,%eax - xorl %r9d,%ebx - - movl %ecx,%r8d - movl %edx,%r9d - movl %eax,%r10d - movl %ebx,%r11d - -.byte 0xf3,0xc3 - - -.globl _Camellia_Ekeygen - -.p2align 4 -_Camellia_Ekeygen: - - pushq %rbx - - pushq %rbp - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$key_prologue: - - movl %edi,%r15d - movq %rdx,%r13 - - movl 0(%rsi),%r8d - movl 4(%rsi),%r9d - movl 8(%rsi),%r10d - movl 12(%rsi),%r11d - - bswapl %r8d - bswapl %r9d - bswapl %r10d - bswapl %r11d - movl %r9d,0(%r13) - movl %r8d,4(%r13) - movl %r11d,8(%r13) - movl %r10d,12(%r13) - cmpq $128,%r15 - je L$1st128 - - movl 16(%rsi),%r8d - movl 20(%rsi),%r9d - cmpq $192,%r15 - je L$1st192 - movl 24(%rsi),%r10d - movl 28(%rsi),%r11d - jmp L$1st256 -L$1st192: - movl %r8d,%r10d - movl %r9d,%r11d - notl %r10d - notl %r11d -L$1st256: - bswapl %r8d - bswapl %r9d - bswapl %r10d - bswapl %r11d - movl %r9d,32(%r13) - movl %r8d,36(%r13) - movl %r11d,40(%r13) - movl %r10d,44(%r13) - xorl 0(%r13),%r9d - xorl 4(%r13),%r8d - xorl 8(%r13),%r11d - xorl 12(%r13),%r10d - -L$1st128: - leaq L$Camellia_SIGMA(%rip),%r14 - leaq L$Camellia_SBOX(%rip),%rbp - - movl 0(%r14),%ebx - movl 4(%r14),%eax - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 8(%r14),%ebx - movl 12(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 16(%r14),%ebx - movl 20(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl 0(%r13),%r9d - xorl 4(%r13),%r8d - xorl 8(%r13),%r11d - xorl 12(%r13),%r10d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 24(%r14),%ebx - movl 28(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 32(%r14),%ebx - movl 36(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - cmpq $128,%r15 - jne L$2nd256 - - leaq 128(%r13),%r13 - shlq $32,%r8 - shlq $32,%r10 - orq %r9,%r8 - orq %r11,%r10 - movq -128(%r13),%rax - movq -120(%r13),%rbx - movq %r8,-112(%r13) - movq %r10,-104(%r13) - movq %rax,%r11 - shlq $15,%rax - movq %rbx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rax - shlq $15,%rbx - orq %r11,%rbx - movq %rax,-96(%r13) - movq %rbx,-88(%r13) - movq %r8,%r11 - shlq $15,%r8 - movq %r10,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r8 - shlq $15,%r10 - orq %r11,%r10 - movq %r8,-80(%r13) - movq %r10,-72(%r13) - movq %r8,%r11 - shlq $15,%r8 - movq %r10,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r8 - shlq $15,%r10 - orq %r11,%r10 - movq %r8,-64(%r13) - movq %r10,-56(%r13) - movq %rax,%r11 - shlq $30,%rax - movq %rbx,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%rax - shlq $30,%rbx - orq %r11,%rbx - movq %rax,-48(%r13) - movq %rbx,-40(%r13) - movq %r8,%r11 - shlq $15,%r8 - movq %r10,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r8 - shlq $15,%r10 - orq %r11,%r10 - movq %r8,-32(%r13) - movq %rax,%r11 - shlq $15,%rax - movq %rbx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rax - shlq $15,%rbx - orq %r11,%rbx - movq %rbx,-24(%r13) - movq %r8,%r11 - shlq $15,%r8 - movq %r10,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r8 - shlq $15,%r10 - orq %r11,%r10 - movq %r8,-16(%r13) - movq %r10,-8(%r13) - movq %rax,%r11 - shlq $17,%rax - movq %rbx,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%rax - shlq $17,%rbx - orq %r11,%rbx - movq %rax,0(%r13) - movq %rbx,8(%r13) - movq %rax,%r11 - shlq $17,%rax - movq %rbx,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%rax - shlq $17,%rbx - orq %r11,%rbx - movq %rax,16(%r13) - movq %rbx,24(%r13) - movq %r8,%r11 - shlq $34,%r8 - movq %r10,%r9 - shrq $30,%r9 - shrq $30,%r11 - orq %r9,%r8 - shlq $34,%r10 - orq %r11,%r10 - movq %r8,32(%r13) - movq %r10,40(%r13) - movq %rax,%r11 - shlq $17,%rax - movq %rbx,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%rax - shlq $17,%rbx - orq %r11,%rbx - movq %rax,48(%r13) - movq %rbx,56(%r13) - movq %r8,%r11 - shlq $17,%r8 - movq %r10,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%r8 - shlq $17,%r10 - orq %r11,%r10 - movq %r8,64(%r13) - movq %r10,72(%r13) - movl $3,%eax - jmp L$done -.p2align 4 -L$2nd256: - movl %r9d,48(%r13) - movl %r8d,52(%r13) - movl %r11d,56(%r13) - movl %r10d,60(%r13) - xorl 32(%r13),%r9d - xorl 36(%r13),%r8d - xorl 40(%r13),%r11d - xorl 44(%r13),%r10d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 40(%r14),%ebx - movl 44(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 48(%r14),%ebx - movl 52(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - movq 0(%r13),%rax - movq 8(%r13),%rbx - movq 32(%r13),%rcx - movq 40(%r13),%rdx - movq 48(%r13),%r14 - movq 56(%r13),%r15 - leaq 128(%r13),%r13 - shlq $32,%r8 - shlq $32,%r10 - orq %r9,%r8 - orq %r11,%r10 - movq %r8,-112(%r13) - movq %r10,-104(%r13) - movq %rcx,%r11 - shlq $15,%rcx - movq %rdx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rcx - shlq $15,%rdx - orq %r11,%rdx - movq %rcx,-96(%r13) - movq %rdx,-88(%r13) - movq %r14,%r11 - shlq $15,%r14 - movq %r15,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r14 - shlq $15,%r15 - orq %r11,%r15 - movq %r14,-80(%r13) - movq %r15,-72(%r13) - movq %rcx,%r11 - shlq $15,%rcx - movq %rdx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rcx - shlq $15,%rdx - orq %r11,%rdx - movq %rcx,-64(%r13) - movq %rdx,-56(%r13) - movq %r8,%r11 - shlq $30,%r8 - movq %r10,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%r8 - shlq $30,%r10 - orq %r11,%r10 - movq %r8,-48(%r13) - movq %r10,-40(%r13) - movq %rax,%r11 - shlq $45,%rax - movq %rbx,%r9 - shrq $19,%r9 - shrq $19,%r11 - orq %r9,%rax - shlq $45,%rbx - orq %r11,%rbx - movq %rax,-32(%r13) - movq %rbx,-24(%r13) - movq %r14,%r11 - shlq $30,%r14 - movq %r15,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%r14 - shlq $30,%r15 - orq %r11,%r15 - movq %r14,-16(%r13) - movq %r15,-8(%r13) - movq %rax,%r11 - shlq $15,%rax - movq %rbx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rax - shlq $15,%rbx - orq %r11,%rbx - movq %rax,0(%r13) - movq %rbx,8(%r13) - movq %rcx,%r11 - shlq $30,%rcx - movq %rdx,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%rcx - shlq $30,%rdx - orq %r11,%rdx - movq %rcx,16(%r13) - movq %rdx,24(%r13) - movq %r8,%r11 - shlq $30,%r8 - movq %r10,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%r8 - shlq $30,%r10 - orq %r11,%r10 - movq %r8,32(%r13) - movq %r10,40(%r13) - movq %rax,%r11 - shlq $17,%rax - movq %rbx,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%rax - shlq $17,%rbx - orq %r11,%rbx - movq %rax,48(%r13) - movq %rbx,56(%r13) - movq %r14,%r11 - shlq $32,%r14 - movq %r15,%r9 - shrq $32,%r9 - shrq $32,%r11 - orq %r9,%r14 - shlq $32,%r15 - orq %r11,%r15 - movq %r14,64(%r13) - movq %r15,72(%r13) - movq %rcx,%r11 - shlq $34,%rcx - movq %rdx,%r9 - shrq $30,%r9 - shrq $30,%r11 - orq %r9,%rcx - shlq $34,%rdx - orq %r11,%rdx - movq %rcx,80(%r13) - movq %rdx,88(%r13) - movq %r14,%r11 - shlq $17,%r14 - movq %r15,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%r14 - shlq $17,%r15 - orq %r11,%r15 - movq %r14,96(%r13) - movq %r15,104(%r13) - movq %rax,%r11 - shlq $34,%rax - movq %rbx,%r9 - shrq $30,%r9 - shrq $30,%r11 - orq %r9,%rax - shlq $34,%rbx - orq %r11,%rbx - movq %rax,112(%r13) - movq %rbx,120(%r13) - movq %r8,%r11 - shlq $51,%r8 - movq %r10,%r9 - shrq $13,%r9 - shrq $13,%r11 - orq %r9,%r8 - shlq $51,%r10 - orq %r11,%r10 - movq %r8,128(%r13) - movq %r10,136(%r13) - movl $4,%eax -L$done: - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%rbp - - movq 32(%rsp),%rbx - - leaq 40(%rsp),%rsp - -L$key_epilogue: - .byte 0xf3,0xc3 - - -.p2align 6 -L$Camellia_SIGMA: -.long 0x3bcc908b, 0xa09e667f, 0x4caa73b2, 0xb67ae858 -.long 0xe94f82be, 0xc6ef372f, 0xf1d36f1c, 0x54ff53a5 -.long 0xde682d1d, 0x10e527fa, 0xb3e6c1fd, 0xb05688c2 -.long 0, 0, 0, 0 -L$Camellia_SBOX: -.long 0x70707000,0x70700070 -.long 0x82828200,0x2c2c002c -.long 0x2c2c2c00,0xb3b300b3 -.long 0xececec00,0xc0c000c0 -.long 0xb3b3b300,0xe4e400e4 -.long 0x27272700,0x57570057 -.long 0xc0c0c000,0xeaea00ea -.long 0xe5e5e500,0xaeae00ae -.long 0xe4e4e400,0x23230023 -.long 0x85858500,0x6b6b006b -.long 0x57575700,0x45450045 -.long 0x35353500,0xa5a500a5 -.long 0xeaeaea00,0xeded00ed -.long 0x0c0c0c00,0x4f4f004f -.long 0xaeaeae00,0x1d1d001d -.long 0x41414100,0x92920092 -.long 0x23232300,0x86860086 -.long 0xefefef00,0xafaf00af -.long 0x6b6b6b00,0x7c7c007c -.long 0x93939300,0x1f1f001f -.long 0x45454500,0x3e3e003e -.long 0x19191900,0xdcdc00dc -.long 0xa5a5a500,0x5e5e005e -.long 0x21212100,0x0b0b000b -.long 0xededed00,0xa6a600a6 -.long 0x0e0e0e00,0x39390039 -.long 0x4f4f4f00,0xd5d500d5 -.long 0x4e4e4e00,0x5d5d005d -.long 0x1d1d1d00,0xd9d900d9 -.long 0x65656500,0x5a5a005a -.long 0x92929200,0x51510051 -.long 0xbdbdbd00,0x6c6c006c -.long 0x86868600,0x8b8b008b -.long 0xb8b8b800,0x9a9a009a -.long 0xafafaf00,0xfbfb00fb -.long 0x8f8f8f00,0xb0b000b0 -.long 0x7c7c7c00,0x74740074 -.long 0xebebeb00,0x2b2b002b -.long 0x1f1f1f00,0xf0f000f0 -.long 0xcecece00,0x84840084 -.long 0x3e3e3e00,0xdfdf00df -.long 0x30303000,0xcbcb00cb -.long 0xdcdcdc00,0x34340034 -.long 0x5f5f5f00,0x76760076 -.long 0x5e5e5e00,0x6d6d006d -.long 0xc5c5c500,0xa9a900a9 -.long 0x0b0b0b00,0xd1d100d1 -.long 0x1a1a1a00,0x04040004 -.long 0xa6a6a600,0x14140014 -.long 0xe1e1e100,0x3a3a003a -.long 0x39393900,0xdede00de -.long 0xcacaca00,0x11110011 -.long 0xd5d5d500,0x32320032 -.long 0x47474700,0x9c9c009c -.long 0x5d5d5d00,0x53530053 -.long 0x3d3d3d00,0xf2f200f2 -.long 0xd9d9d900,0xfefe00fe -.long 0x01010100,0xcfcf00cf -.long 0x5a5a5a00,0xc3c300c3 -.long 0xd6d6d600,0x7a7a007a -.long 0x51515100,0x24240024 -.long 0x56565600,0xe8e800e8 -.long 0x6c6c6c00,0x60600060 -.long 0x4d4d4d00,0x69690069 -.long 0x8b8b8b00,0xaaaa00aa -.long 0x0d0d0d00,0xa0a000a0 -.long 0x9a9a9a00,0xa1a100a1 -.long 0x66666600,0x62620062 -.long 0xfbfbfb00,0x54540054 -.long 0xcccccc00,0x1e1e001e -.long 0xb0b0b000,0xe0e000e0 -.long 0x2d2d2d00,0x64640064 -.long 0x74747400,0x10100010 -.long 0x12121200,0x00000000 -.long 0x2b2b2b00,0xa3a300a3 -.long 0x20202000,0x75750075 -.long 0xf0f0f000,0x8a8a008a -.long 0xb1b1b100,0xe6e600e6 -.long 0x84848400,0x09090009 -.long 0x99999900,0xdddd00dd -.long 0xdfdfdf00,0x87870087 -.long 0x4c4c4c00,0x83830083 -.long 0xcbcbcb00,0xcdcd00cd -.long 0xc2c2c200,0x90900090 -.long 0x34343400,0x73730073 -.long 0x7e7e7e00,0xf6f600f6 -.long 0x76767600,0x9d9d009d -.long 0x05050500,0xbfbf00bf -.long 0x6d6d6d00,0x52520052 -.long 0xb7b7b700,0xd8d800d8 -.long 0xa9a9a900,0xc8c800c8 -.long 0x31313100,0xc6c600c6 -.long 0xd1d1d100,0x81810081 -.long 0x17171700,0x6f6f006f -.long 0x04040400,0x13130013 -.long 0xd7d7d700,0x63630063 -.long 0x14141400,0xe9e900e9 -.long 0x58585800,0xa7a700a7 -.long 0x3a3a3a00,0x9f9f009f -.long 0x61616100,0xbcbc00bc -.long 0xdedede00,0x29290029 -.long 0x1b1b1b00,0xf9f900f9 -.long 0x11111100,0x2f2f002f -.long 0x1c1c1c00,0xb4b400b4 -.long 0x32323200,0x78780078 -.long 0x0f0f0f00,0x06060006 -.long 0x9c9c9c00,0xe7e700e7 -.long 0x16161600,0x71710071 -.long 0x53535300,0xd4d400d4 -.long 0x18181800,0xabab00ab -.long 0xf2f2f200,0x88880088 -.long 0x22222200,0x8d8d008d -.long 0xfefefe00,0x72720072 -.long 0x44444400,0xb9b900b9 -.long 0xcfcfcf00,0xf8f800f8 -.long 0xb2b2b200,0xacac00ac -.long 0xc3c3c300,0x36360036 -.long 0xb5b5b500,0x2a2a002a -.long 0x7a7a7a00,0x3c3c003c -.long 0x91919100,0xf1f100f1 -.long 0x24242400,0x40400040 -.long 0x08080800,0xd3d300d3 -.long 0xe8e8e800,0xbbbb00bb -.long 0xa8a8a800,0x43430043 -.long 0x60606000,0x15150015 -.long 0xfcfcfc00,0xadad00ad -.long 0x69696900,0x77770077 -.long 0x50505000,0x80800080 -.long 0xaaaaaa00,0x82820082 -.long 0xd0d0d000,0xecec00ec -.long 0xa0a0a000,0x27270027 -.long 0x7d7d7d00,0xe5e500e5 -.long 0xa1a1a100,0x85850085 -.long 0x89898900,0x35350035 -.long 0x62626200,0x0c0c000c -.long 0x97979700,0x41410041 -.long 0x54545400,0xefef00ef -.long 0x5b5b5b00,0x93930093 -.long 0x1e1e1e00,0x19190019 -.long 0x95959500,0x21210021 -.long 0xe0e0e000,0x0e0e000e -.long 0xffffff00,0x4e4e004e -.long 0x64646400,0x65650065 -.long 0xd2d2d200,0xbdbd00bd -.long 0x10101000,0xb8b800b8 -.long 0xc4c4c400,0x8f8f008f -.long 0x00000000,0xebeb00eb -.long 0x48484800,0xcece00ce -.long 0xa3a3a300,0x30300030 -.long 0xf7f7f700,0x5f5f005f -.long 0x75757500,0xc5c500c5 -.long 0xdbdbdb00,0x1a1a001a -.long 0x8a8a8a00,0xe1e100e1 -.long 0x03030300,0xcaca00ca -.long 0xe6e6e600,0x47470047 -.long 0xdadada00,0x3d3d003d -.long 0x09090900,0x01010001 -.long 0x3f3f3f00,0xd6d600d6 -.long 0xdddddd00,0x56560056 -.long 0x94949400,0x4d4d004d -.long 0x87878700,0x0d0d000d -.long 0x5c5c5c00,0x66660066 -.long 0x83838300,0xcccc00cc -.long 0x02020200,0x2d2d002d -.long 0xcdcdcd00,0x12120012 -.long 0x4a4a4a00,0x20200020 -.long 0x90909000,0xb1b100b1 -.long 0x33333300,0x99990099 -.long 0x73737300,0x4c4c004c -.long 0x67676700,0xc2c200c2 -.long 0xf6f6f600,0x7e7e007e -.long 0xf3f3f300,0x05050005 -.long 0x9d9d9d00,0xb7b700b7 -.long 0x7f7f7f00,0x31310031 -.long 0xbfbfbf00,0x17170017 -.long 0xe2e2e200,0xd7d700d7 -.long 0x52525200,0x58580058 -.long 0x9b9b9b00,0x61610061 -.long 0xd8d8d800,0x1b1b001b -.long 0x26262600,0x1c1c001c -.long 0xc8c8c800,0x0f0f000f -.long 0x37373700,0x16160016 -.long 0xc6c6c600,0x18180018 -.long 0x3b3b3b00,0x22220022 -.long 0x81818100,0x44440044 -.long 0x96969600,0xb2b200b2 -.long 0x6f6f6f00,0xb5b500b5 -.long 0x4b4b4b00,0x91910091 -.long 0x13131300,0x08080008 -.long 0xbebebe00,0xa8a800a8 -.long 0x63636300,0xfcfc00fc -.long 0x2e2e2e00,0x50500050 -.long 0xe9e9e900,0xd0d000d0 -.long 0x79797900,0x7d7d007d -.long 0xa7a7a700,0x89890089 -.long 0x8c8c8c00,0x97970097 -.long 0x9f9f9f00,0x5b5b005b -.long 0x6e6e6e00,0x95950095 -.long 0xbcbcbc00,0xffff00ff -.long 0x8e8e8e00,0xd2d200d2 -.long 0x29292900,0xc4c400c4 -.long 0xf5f5f500,0x48480048 -.long 0xf9f9f900,0xf7f700f7 -.long 0xb6b6b600,0xdbdb00db -.long 0x2f2f2f00,0x03030003 -.long 0xfdfdfd00,0xdada00da -.long 0xb4b4b400,0x3f3f003f -.long 0x59595900,0x94940094 -.long 0x78787800,0x5c5c005c -.long 0x98989800,0x02020002 -.long 0x06060600,0x4a4a004a -.long 0x6a6a6a00,0x33330033 -.long 0xe7e7e700,0x67670067 -.long 0x46464600,0xf3f300f3 -.long 0x71717100,0x7f7f007f -.long 0xbababa00,0xe2e200e2 -.long 0xd4d4d400,0x9b9b009b -.long 0x25252500,0x26260026 -.long 0xababab00,0x37370037 -.long 0x42424200,0x3b3b003b -.long 0x88888800,0x96960096 -.long 0xa2a2a200,0x4b4b004b -.long 0x8d8d8d00,0xbebe00be -.long 0xfafafa00,0x2e2e002e -.long 0x72727200,0x79790079 -.long 0x07070700,0x8c8c008c -.long 0xb9b9b900,0x6e6e006e -.long 0x55555500,0x8e8e008e -.long 0xf8f8f800,0xf5f500f5 -.long 0xeeeeee00,0xb6b600b6 -.long 0xacacac00,0xfdfd00fd -.long 0x0a0a0a00,0x59590059 -.long 0x36363600,0x98980098 -.long 0x49494900,0x6a6a006a -.long 0x2a2a2a00,0x46460046 -.long 0x68686800,0xbaba00ba -.long 0x3c3c3c00,0x25250025 -.long 0x38383800,0x42420042 -.long 0xf1f1f100,0xa2a200a2 -.long 0xa4a4a400,0xfafa00fa -.long 0x40404000,0x07070007 -.long 0x28282800,0x55550055 -.long 0xd3d3d300,0xeeee00ee -.long 0x7b7b7b00,0x0a0a000a -.long 0xbbbbbb00,0x49490049 -.long 0xc9c9c900,0x68680068 -.long 0x43434300,0x38380038 -.long 0xc1c1c100,0xa4a400a4 -.long 0x15151500,0x28280028 -.long 0xe3e3e300,0x7b7b007b -.long 0xadadad00,0xc9c900c9 -.long 0xf4f4f400,0xc1c100c1 -.long 0x77777700,0xe3e300e3 -.long 0xc7c7c700,0xf4f400f4 -.long 0x80808000,0xc7c700c7 -.long 0x9e9e9e00,0x9e9e009e -.long 0x00e0e0e0,0x38003838 -.long 0x00050505,0x41004141 -.long 0x00585858,0x16001616 -.long 0x00d9d9d9,0x76007676 -.long 0x00676767,0xd900d9d9 -.long 0x004e4e4e,0x93009393 -.long 0x00818181,0x60006060 -.long 0x00cbcbcb,0xf200f2f2 -.long 0x00c9c9c9,0x72007272 -.long 0x000b0b0b,0xc200c2c2 -.long 0x00aeaeae,0xab00abab -.long 0x006a6a6a,0x9a009a9a -.long 0x00d5d5d5,0x75007575 -.long 0x00181818,0x06000606 -.long 0x005d5d5d,0x57005757 -.long 0x00828282,0xa000a0a0 -.long 0x00464646,0x91009191 -.long 0x00dfdfdf,0xf700f7f7 -.long 0x00d6d6d6,0xb500b5b5 -.long 0x00272727,0xc900c9c9 -.long 0x008a8a8a,0xa200a2a2 -.long 0x00323232,0x8c008c8c -.long 0x004b4b4b,0xd200d2d2 -.long 0x00424242,0x90009090 -.long 0x00dbdbdb,0xf600f6f6 -.long 0x001c1c1c,0x07000707 -.long 0x009e9e9e,0xa700a7a7 -.long 0x009c9c9c,0x27002727 -.long 0x003a3a3a,0x8e008e8e -.long 0x00cacaca,0xb200b2b2 -.long 0x00252525,0x49004949 -.long 0x007b7b7b,0xde00dede -.long 0x000d0d0d,0x43004343 -.long 0x00717171,0x5c005c5c -.long 0x005f5f5f,0xd700d7d7 -.long 0x001f1f1f,0xc700c7c7 -.long 0x00f8f8f8,0x3e003e3e -.long 0x00d7d7d7,0xf500f5f5 -.long 0x003e3e3e,0x8f008f8f -.long 0x009d9d9d,0x67006767 -.long 0x007c7c7c,0x1f001f1f -.long 0x00606060,0x18001818 -.long 0x00b9b9b9,0x6e006e6e -.long 0x00bebebe,0xaf00afaf -.long 0x00bcbcbc,0x2f002f2f -.long 0x008b8b8b,0xe200e2e2 -.long 0x00161616,0x85008585 -.long 0x00343434,0x0d000d0d -.long 0x004d4d4d,0x53005353 -.long 0x00c3c3c3,0xf000f0f0 -.long 0x00727272,0x9c009c9c -.long 0x00959595,0x65006565 -.long 0x00ababab,0xea00eaea -.long 0x008e8e8e,0xa300a3a3 -.long 0x00bababa,0xae00aeae -.long 0x007a7a7a,0x9e009e9e -.long 0x00b3b3b3,0xec00ecec -.long 0x00020202,0x80008080 -.long 0x00b4b4b4,0x2d002d2d -.long 0x00adadad,0x6b006b6b -.long 0x00a2a2a2,0xa800a8a8 -.long 0x00acacac,0x2b002b2b -.long 0x00d8d8d8,0x36003636 -.long 0x009a9a9a,0xa600a6a6 -.long 0x00171717,0xc500c5c5 -.long 0x001a1a1a,0x86008686 -.long 0x00353535,0x4d004d4d -.long 0x00cccccc,0x33003333 -.long 0x00f7f7f7,0xfd00fdfd -.long 0x00999999,0x66006666 -.long 0x00616161,0x58005858 -.long 0x005a5a5a,0x96009696 -.long 0x00e8e8e8,0x3a003a3a -.long 0x00242424,0x09000909 -.long 0x00565656,0x95009595 -.long 0x00404040,0x10001010 -.long 0x00e1e1e1,0x78007878 -.long 0x00636363,0xd800d8d8 -.long 0x00090909,0x42004242 -.long 0x00333333,0xcc00cccc -.long 0x00bfbfbf,0xef00efef -.long 0x00989898,0x26002626 -.long 0x00979797,0xe500e5e5 -.long 0x00858585,0x61006161 -.long 0x00686868,0x1a001a1a -.long 0x00fcfcfc,0x3f003f3f -.long 0x00ececec,0x3b003b3b -.long 0x000a0a0a,0x82008282 -.long 0x00dadada,0xb600b6b6 -.long 0x006f6f6f,0xdb00dbdb -.long 0x00535353,0xd400d4d4 -.long 0x00626262,0x98009898 -.long 0x00a3a3a3,0xe800e8e8 -.long 0x002e2e2e,0x8b008b8b -.long 0x00080808,0x02000202 -.long 0x00afafaf,0xeb00ebeb -.long 0x00282828,0x0a000a0a -.long 0x00b0b0b0,0x2c002c2c -.long 0x00747474,0x1d001d1d -.long 0x00c2c2c2,0xb000b0b0 -.long 0x00bdbdbd,0x6f006f6f -.long 0x00363636,0x8d008d8d -.long 0x00222222,0x88008888 -.long 0x00383838,0x0e000e0e -.long 0x00646464,0x19001919 -.long 0x001e1e1e,0x87008787 -.long 0x00393939,0x4e004e4e -.long 0x002c2c2c,0x0b000b0b -.long 0x00a6a6a6,0xa900a9a9 -.long 0x00303030,0x0c000c0c -.long 0x00e5e5e5,0x79007979 -.long 0x00444444,0x11001111 -.long 0x00fdfdfd,0x7f007f7f -.long 0x00888888,0x22002222 -.long 0x009f9f9f,0xe700e7e7 -.long 0x00656565,0x59005959 -.long 0x00878787,0xe100e1e1 -.long 0x006b6b6b,0xda00dada -.long 0x00f4f4f4,0x3d003d3d -.long 0x00232323,0xc800c8c8 -.long 0x00484848,0x12001212 -.long 0x00101010,0x04000404 -.long 0x00d1d1d1,0x74007474 -.long 0x00515151,0x54005454 -.long 0x00c0c0c0,0x30003030 -.long 0x00f9f9f9,0x7e007e7e -.long 0x00d2d2d2,0xb400b4b4 -.long 0x00a0a0a0,0x28002828 -.long 0x00555555,0x55005555 -.long 0x00a1a1a1,0x68006868 -.long 0x00414141,0x50005050 -.long 0x00fafafa,0xbe00bebe -.long 0x00434343,0xd000d0d0 -.long 0x00131313,0xc400c4c4 -.long 0x00c4c4c4,0x31003131 -.long 0x002f2f2f,0xcb00cbcb -.long 0x00a8a8a8,0x2a002a2a -.long 0x00b6b6b6,0xad00adad -.long 0x003c3c3c,0x0f000f0f -.long 0x002b2b2b,0xca00caca -.long 0x00c1c1c1,0x70007070 -.long 0x00ffffff,0xff00ffff -.long 0x00c8c8c8,0x32003232 -.long 0x00a5a5a5,0x69006969 -.long 0x00202020,0x08000808 -.long 0x00898989,0x62006262 -.long 0x00000000,0x00000000 -.long 0x00909090,0x24002424 -.long 0x00474747,0xd100d1d1 -.long 0x00efefef,0xfb00fbfb -.long 0x00eaeaea,0xba00baba -.long 0x00b7b7b7,0xed00eded -.long 0x00151515,0x45004545 -.long 0x00060606,0x81008181 -.long 0x00cdcdcd,0x73007373 -.long 0x00b5b5b5,0x6d006d6d -.long 0x00121212,0x84008484 -.long 0x007e7e7e,0x9f009f9f -.long 0x00bbbbbb,0xee00eeee -.long 0x00292929,0x4a004a4a -.long 0x000f0f0f,0xc300c3c3 -.long 0x00b8b8b8,0x2e002e2e -.long 0x00070707,0xc100c1c1 -.long 0x00040404,0x01000101 -.long 0x009b9b9b,0xe600e6e6 -.long 0x00949494,0x25002525 -.long 0x00212121,0x48004848 -.long 0x00666666,0x99009999 -.long 0x00e6e6e6,0xb900b9b9 -.long 0x00cecece,0xb300b3b3 -.long 0x00ededed,0x7b007b7b -.long 0x00e7e7e7,0xf900f9f9 -.long 0x003b3b3b,0xce00cece -.long 0x00fefefe,0xbf00bfbf -.long 0x007f7f7f,0xdf00dfdf -.long 0x00c5c5c5,0x71007171 -.long 0x00a4a4a4,0x29002929 -.long 0x00373737,0xcd00cdcd -.long 0x00b1b1b1,0x6c006c6c -.long 0x004c4c4c,0x13001313 -.long 0x00919191,0x64006464 -.long 0x006e6e6e,0x9b009b9b -.long 0x008d8d8d,0x63006363 -.long 0x00767676,0x9d009d9d -.long 0x00030303,0xc000c0c0 -.long 0x002d2d2d,0x4b004b4b -.long 0x00dedede,0xb700b7b7 -.long 0x00969696,0xa500a5a5 -.long 0x00262626,0x89008989 -.long 0x007d7d7d,0x5f005f5f -.long 0x00c6c6c6,0xb100b1b1 -.long 0x005c5c5c,0x17001717 -.long 0x00d3d3d3,0xf400f4f4 -.long 0x00f2f2f2,0xbc00bcbc -.long 0x004f4f4f,0xd300d3d3 -.long 0x00191919,0x46004646 -.long 0x003f3f3f,0xcf00cfcf -.long 0x00dcdcdc,0x37003737 -.long 0x00797979,0x5e005e5e -.long 0x001d1d1d,0x47004747 -.long 0x00525252,0x94009494 -.long 0x00ebebeb,0xfa00fafa -.long 0x00f3f3f3,0xfc00fcfc -.long 0x006d6d6d,0x5b005b5b -.long 0x005e5e5e,0x97009797 -.long 0x00fbfbfb,0xfe00fefe -.long 0x00696969,0x5a005a5a -.long 0x00b2b2b2,0xac00acac -.long 0x00f0f0f0,0x3c003c3c -.long 0x00313131,0x4c004c4c -.long 0x000c0c0c,0x03000303 -.long 0x00d4d4d4,0x35003535 -.long 0x00cfcfcf,0xf300f3f3 -.long 0x008c8c8c,0x23002323 -.long 0x00e2e2e2,0xb800b8b8 -.long 0x00757575,0x5d005d5d -.long 0x00a9a9a9,0x6a006a6a -.long 0x004a4a4a,0x92009292 -.long 0x00575757,0xd500d5d5 -.long 0x00848484,0x21002121 -.long 0x00111111,0x44004444 -.long 0x00454545,0x51005151 -.long 0x001b1b1b,0xc600c6c6 -.long 0x00f5f5f5,0x7d007d7d -.long 0x00e4e4e4,0x39003939 -.long 0x000e0e0e,0x83008383 -.long 0x00737373,0xdc00dcdc -.long 0x00aaaaaa,0xaa00aaaa -.long 0x00f1f1f1,0x7c007c7c -.long 0x00dddddd,0x77007777 -.long 0x00595959,0x56005656 -.long 0x00141414,0x05000505 -.long 0x006c6c6c,0x1b001b1b -.long 0x00929292,0xa400a4a4 -.long 0x00545454,0x15001515 -.long 0x00d0d0d0,0x34003434 -.long 0x00787878,0x1e001e1e -.long 0x00707070,0x1c001c1c -.long 0x00e3e3e3,0xf800f8f8 -.long 0x00494949,0x52005252 -.long 0x00808080,0x20002020 -.long 0x00505050,0x14001414 -.long 0x00a7a7a7,0xe900e9e9 -.long 0x00f6f6f6,0xbd00bdbd -.long 0x00777777,0xdd00dddd -.long 0x00939393,0xe400e4e4 -.long 0x00868686,0xa100a1a1 -.long 0x00838383,0xe000e0e0 -.long 0x002a2a2a,0x8a008a8a -.long 0x00c7c7c7,0xf100f1f1 -.long 0x005b5b5b,0xd600d6d6 -.long 0x00e9e9e9,0x7a007a7a -.long 0x00eeeeee,0xbb00bbbb -.long 0x008f8f8f,0xe300e3e3 -.long 0x00010101,0x40004040 -.long 0x003d3d3d,0x4f004f4f -.globl _Camellia_cbc_encrypt - -.p2align 4 -_Camellia_cbc_encrypt: - -.byte 243,15,30,250 - cmpq $0,%rdx - je L$cbc_abort - pushq %rbx - - pushq %rbp - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$cbc_prologue: - - movq %rsp,%rbp - - subq $64,%rsp - andq $-64,%rsp - - - - leaq -64-63(%rcx),%r10 - subq %rsp,%r10 - negq %r10 - andq $0x3C0,%r10 - subq %r10,%rsp - - - movq %rdi,%r12 - movq %rsi,%r13 - movq %r8,%rbx - movq %rcx,%r14 - movl 272(%rcx),%r15d - - movq %r8,40(%rsp) - movq %rbp,48(%rsp) - - -L$cbc_body: - leaq L$Camellia_SBOX(%rip),%rbp - - movl $32,%ecx -.p2align 2 -L$cbc_prefetch_sbox: - movq 0(%rbp),%rax - movq 32(%rbp),%rsi - movq 64(%rbp),%rdi - movq 96(%rbp),%r11 - leaq 128(%rbp),%rbp - loop L$cbc_prefetch_sbox - subq $4096,%rbp - shlq $6,%r15 - movq %rdx,%rcx - leaq (%r14,%r15,1),%r15 - - cmpl $0,%r9d - je L$CBC_DECRYPT - - andq $-16,%rdx - andq $15,%rcx - leaq (%r12,%rdx,1),%rdx - movq %r14,0(%rsp) - movq %rdx,8(%rsp) - movq %rcx,16(%rsp) - - cmpq %r12,%rdx - movl 0(%rbx),%r8d - movl 4(%rbx),%r9d - movl 8(%rbx),%r10d - movl 12(%rbx),%r11d - je L$cbc_enc_tail - jmp L$cbc_eloop - -.p2align 4 -L$cbc_eloop: - xorl 0(%r12),%r8d - xorl 4(%r12),%r9d - xorl 8(%r12),%r10d - bswapl %r8d - xorl 12(%r12),%r11d - bswapl %r9d - bswapl %r10d - bswapl %r11d - - call _x86_64_Camellia_encrypt - - movq 0(%rsp),%r14 - bswapl %r8d - movq 8(%rsp),%rdx - bswapl %r9d - movq 16(%rsp),%rcx - bswapl %r10d - movl %r8d,0(%r13) - bswapl %r11d - movl %r9d,4(%r13) - movl %r10d,8(%r13) - leaq 16(%r12),%r12 - movl %r11d,12(%r13) - cmpq %rdx,%r12 - leaq 16(%r13),%r13 - jne L$cbc_eloop - - cmpq $0,%rcx - jne L$cbc_enc_tail - - movq 40(%rsp),%r13 - movl %r8d,0(%r13) - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - jmp L$cbc_done - -.p2align 4 -L$cbc_enc_tail: - xorq %rax,%rax - movq %rax,0+24(%rsp) - movq %rax,8+24(%rsp) - movq %rax,16(%rsp) - -L$cbc_enc_pushf: - pushfq - cld - movq %r12,%rsi - leaq 8+24(%rsp),%rdi -.long 0x9066A4F3 - popfq -L$cbc_enc_popf: - - leaq 24(%rsp),%r12 - leaq 16+24(%rsp),%rax - movq %rax,8(%rsp) - jmp L$cbc_eloop - -.p2align 4 -L$CBC_DECRYPT: - xchgq %r14,%r15 - addq $15,%rdx - andq $15,%rcx - andq $-16,%rdx - movq %r14,0(%rsp) - leaq (%r12,%rdx,1),%rdx - movq %rdx,8(%rsp) - movq %rcx,16(%rsp) - - movq (%rbx),%rax - movq 8(%rbx),%rbx - jmp L$cbc_dloop -.p2align 4 -L$cbc_dloop: - movl 0(%r12),%r8d - movl 4(%r12),%r9d - movl 8(%r12),%r10d - bswapl %r8d - movl 12(%r12),%r11d - bswapl %r9d - movq %rax,0+24(%rsp) - bswapl %r10d - movq %rbx,8+24(%rsp) - bswapl %r11d - - call _x86_64_Camellia_decrypt - - movq 0(%rsp),%r14 - movq 8(%rsp),%rdx - movq 16(%rsp),%rcx - - bswapl %r8d - movq (%r12),%rax - bswapl %r9d - movq 8(%r12),%rbx - bswapl %r10d - xorl 0+24(%rsp),%r8d - bswapl %r11d - xorl 4+24(%rsp),%r9d - xorl 8+24(%rsp),%r10d - leaq 16(%r12),%r12 - xorl 12+24(%rsp),%r11d - cmpq %rdx,%r12 - je L$cbc_ddone - - movl %r8d,0(%r13) - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - - leaq 16(%r13),%r13 - jmp L$cbc_dloop - -.p2align 4 -L$cbc_ddone: - movq 40(%rsp),%rdx - cmpq $0,%rcx - jne L$cbc_dec_tail - - movl %r8d,0(%r13) - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - - movq %rax,(%rdx) - movq %rbx,8(%rdx) - jmp L$cbc_done -.p2align 4 -L$cbc_dec_tail: - movl %r8d,0+24(%rsp) - movl %r9d,4+24(%rsp) - movl %r10d,8+24(%rsp) - movl %r11d,12+24(%rsp) - -L$cbc_dec_pushf: - pushfq - cld - leaq 8+24(%rsp),%rsi - leaq (%r13),%rdi -.long 0x9066A4F3 - popfq -L$cbc_dec_popf: - - movq %rax,(%rdx) - movq %rbx,8(%rdx) - jmp L$cbc_done - -.p2align 4 -L$cbc_done: - movq 48(%rsp),%rcx - - movq 0(%rcx),%r15 - - movq 8(%rcx),%r14 - - movq 16(%rcx),%r13 - - movq 24(%rcx),%r12 - - movq 32(%rcx),%rbp - - movq 40(%rcx),%rbx - - leaq 48(%rcx),%rsp - -L$cbc_abort: - .byte 0xf3,0xc3 - - - -.byte 67,97,109,101,108,108,105,97,32,102,111,114,32,120,56,54,95,54,52,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 diff --git a/openssl/src/crypto/camellia/gen/linux_ia32/cmll-x86.S b/openssl/src/crypto/camellia/gen/linux_ia32/cmll-x86.S deleted file mode 100644 index f4b77edc3..000000000 --- a/openssl/src/crypto/camellia/gen/linux_ia32/cmll-x86.S +++ /dev/null @@ -1,2446 +0,0 @@ -.text -.globl Camellia_EncryptBlock_Rounds -.type Camellia_EncryptBlock_Rounds,@function -.align 16 -Camellia_EncryptBlock_Rounds: -.L_Camellia_EncryptBlock_Rounds_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - pushl %ebp - pushl %ebx - pushl %esi - pushl %edi - movl 20(%esp),%eax - movl 24(%esp),%esi - movl 28(%esp),%edi - movl %esp,%ebx - subl $28,%esp - andl $-64,%esp - leal -127(%edi),%ecx - subl %esp,%ecx - negl %ecx - andl $960,%ecx - subl %ecx,%esp - addl $4,%esp - shll $6,%eax - leal (%edi,%eax,1),%eax - movl %ebx,20(%esp) - movl %eax,16(%esp) - call .L000pic_point -.L000pic_point: - popl %ebp - leal .LCamellia_SBOX-.L000pic_point(%ebp),%ebp - movl (%esi),%eax - movl 4(%esi),%ebx - movl 8(%esi),%ecx - bswap %eax - movl 12(%esi),%edx - bswap %ebx - bswap %ecx - bswap %edx - call _x86_Camellia_encrypt - movl 20(%esp),%esp - bswap %eax - movl 32(%esp),%esi - bswap %ebx - bswap %ecx - bswap %edx - movl %eax,(%esi) - movl %ebx,4(%esi) - movl %ecx,8(%esi) - movl %edx,12(%esi) - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.size Camellia_EncryptBlock_Rounds,.-.L_Camellia_EncryptBlock_Rounds_begin -.globl Camellia_EncryptBlock -.type Camellia_EncryptBlock,@function -.align 16 -Camellia_EncryptBlock: -.L_Camellia_EncryptBlock_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movl $128,%eax - subl 4(%esp),%eax - movl $3,%eax - adcl $0,%eax - movl %eax,4(%esp) - jmp .L_Camellia_EncryptBlock_Rounds_begin -.size Camellia_EncryptBlock,.-.L_Camellia_EncryptBlock_begin -.globl Camellia_encrypt -.type Camellia_encrypt,@function -.align 16 -Camellia_encrypt: -.L_Camellia_encrypt_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - pushl %ebp - pushl %ebx - pushl %esi - pushl %edi - movl 20(%esp),%esi - movl 28(%esp),%edi - movl %esp,%ebx - subl $28,%esp - andl $-64,%esp - movl 272(%edi),%eax - leal -127(%edi),%ecx - subl %esp,%ecx - negl %ecx - andl $960,%ecx - subl %ecx,%esp - addl $4,%esp - shll $6,%eax - leal (%edi,%eax,1),%eax - movl %ebx,20(%esp) - movl %eax,16(%esp) - call .L001pic_point -.L001pic_point: - popl %ebp - leal .LCamellia_SBOX-.L001pic_point(%ebp),%ebp - movl (%esi),%eax - movl 4(%esi),%ebx - movl 8(%esi),%ecx - bswap %eax - movl 12(%esi),%edx - bswap %ebx - bswap %ecx - bswap %edx - call _x86_Camellia_encrypt - movl 20(%esp),%esp - bswap %eax - movl 24(%esp),%esi - bswap %ebx - bswap %ecx - bswap %edx - movl %eax,(%esi) - movl %ebx,4(%esi) - movl %ecx,8(%esi) - movl %edx,12(%esi) - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.size Camellia_encrypt,.-.L_Camellia_encrypt_begin -.type _x86_Camellia_encrypt,@function -.align 16 -_x86_Camellia_encrypt: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - xorl (%edi),%eax - xorl 4(%edi),%ebx - xorl 8(%edi),%ecx - xorl 12(%edi),%edx - movl 16(%edi),%esi - movl %eax,4(%esp) - movl %ebx,8(%esp) - movl %ecx,12(%esp) - movl %edx,16(%esp) -.align 16 -.L002loop: - xorl %esi,%eax - xorl 20(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 16(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 12(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl 24(%edi),%esi - xorl %ecx,%edx - movl %edx,16(%esp) - xorl %ebx,%ecx - movl %ecx,12(%esp) - xorl %esi,%ecx - xorl 28(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 8(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl 4(%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl 32(%edi),%esi - xorl %eax,%ebx - movl %ebx,8(%esp) - xorl %edx,%eax - movl %eax,4(%esp) - xorl %esi,%eax - xorl 36(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 16(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 12(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl 40(%edi),%esi - xorl %ecx,%edx - movl %edx,16(%esp) - xorl %ebx,%ecx - movl %ecx,12(%esp) - xorl %esi,%ecx - xorl 44(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 8(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl 4(%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl 48(%edi),%esi - xorl %eax,%ebx - movl %ebx,8(%esp) - xorl %edx,%eax - movl %eax,4(%esp) - xorl %esi,%eax - xorl 52(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 16(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 12(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl 56(%edi),%esi - xorl %ecx,%edx - movl %edx,16(%esp) - xorl %ebx,%ecx - movl %ecx,12(%esp) - xorl %esi,%ecx - xorl 60(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 8(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl 4(%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl 64(%edi),%esi - xorl %eax,%ebx - movl %ebx,8(%esp) - xorl %edx,%eax - movl %eax,4(%esp) - addl $64,%edi - cmpl 20(%esp),%edi - je .L003done - andl %eax,%esi - movl 16(%esp),%edx - roll $1,%esi - movl %edx,%ecx - xorl %esi,%ebx - orl 12(%edi),%ecx - movl %ebx,8(%esp) - xorl 12(%esp),%ecx - movl 4(%edi),%esi - movl %ecx,12(%esp) - orl %ebx,%esi - andl 8(%edi),%ecx - xorl %esi,%eax - roll $1,%ecx - movl %eax,4(%esp) - xorl %ecx,%edx - movl 16(%edi),%esi - movl %edx,16(%esp) - jmp .L002loop -.align 8 -.L003done: - movl %eax,%ecx - movl %ebx,%edx - movl 12(%esp),%eax - movl 16(%esp),%ebx - xorl %esi,%eax - xorl 4(%edi),%ebx - xorl 8(%edi),%ecx - xorl 12(%edi),%edx - ret -.size _x86_Camellia_encrypt,.-_x86_Camellia_encrypt -.globl Camellia_DecryptBlock_Rounds -.type Camellia_DecryptBlock_Rounds,@function -.align 16 -Camellia_DecryptBlock_Rounds: -.L_Camellia_DecryptBlock_Rounds_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - pushl %ebp - pushl %ebx - pushl %esi - pushl %edi - movl 20(%esp),%eax - movl 24(%esp),%esi - movl 28(%esp),%edi - movl %esp,%ebx - subl $28,%esp - andl $-64,%esp - leal -127(%edi),%ecx - subl %esp,%ecx - negl %ecx - andl $960,%ecx - subl %ecx,%esp - addl $4,%esp - shll $6,%eax - movl %edi,16(%esp) - leal (%edi,%eax,1),%edi - movl %ebx,20(%esp) - call .L004pic_point -.L004pic_point: - popl %ebp - leal .LCamellia_SBOX-.L004pic_point(%ebp),%ebp - movl (%esi),%eax - movl 4(%esi),%ebx - movl 8(%esi),%ecx - bswap %eax - movl 12(%esi),%edx - bswap %ebx - bswap %ecx - bswap %edx - call _x86_Camellia_decrypt - movl 20(%esp),%esp - bswap %eax - movl 32(%esp),%esi - bswap %ebx - bswap %ecx - bswap %edx - movl %eax,(%esi) - movl %ebx,4(%esi) - movl %ecx,8(%esi) - movl %edx,12(%esi) - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.size Camellia_DecryptBlock_Rounds,.-.L_Camellia_DecryptBlock_Rounds_begin -.globl Camellia_DecryptBlock -.type Camellia_DecryptBlock,@function -.align 16 -Camellia_DecryptBlock: -.L_Camellia_DecryptBlock_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movl $128,%eax - subl 4(%esp),%eax - movl $3,%eax - adcl $0,%eax - movl %eax,4(%esp) - jmp .L_Camellia_DecryptBlock_Rounds_begin -.size Camellia_DecryptBlock,.-.L_Camellia_DecryptBlock_begin -.globl Camellia_decrypt -.type Camellia_decrypt,@function -.align 16 -Camellia_decrypt: -.L_Camellia_decrypt_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - pushl %ebp - pushl %ebx - pushl %esi - pushl %edi - movl 20(%esp),%esi - movl 28(%esp),%edi - movl %esp,%ebx - subl $28,%esp - andl $-64,%esp - movl 272(%edi),%eax - leal -127(%edi),%ecx - subl %esp,%ecx - negl %ecx - andl $960,%ecx - subl %ecx,%esp - addl $4,%esp - shll $6,%eax - movl %edi,16(%esp) - leal (%edi,%eax,1),%edi - movl %ebx,20(%esp) - call .L005pic_point -.L005pic_point: - popl %ebp - leal .LCamellia_SBOX-.L005pic_point(%ebp),%ebp - movl (%esi),%eax - movl 4(%esi),%ebx - movl 8(%esi),%ecx - bswap %eax - movl 12(%esi),%edx - bswap %ebx - bswap %ecx - bswap %edx - call _x86_Camellia_decrypt - movl 20(%esp),%esp - bswap %eax - movl 24(%esp),%esi - bswap %ebx - bswap %ecx - bswap %edx - movl %eax,(%esi) - movl %ebx,4(%esi) - movl %ecx,8(%esi) - movl %edx,12(%esi) - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.size Camellia_decrypt,.-.L_Camellia_decrypt_begin -.type _x86_Camellia_decrypt,@function -.align 16 -_x86_Camellia_decrypt: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - xorl (%edi),%eax - xorl 4(%edi),%ebx - xorl 8(%edi),%ecx - xorl 12(%edi),%edx - movl -8(%edi),%esi - movl %eax,4(%esp) - movl %ebx,8(%esp) - movl %ecx,12(%esp) - movl %edx,16(%esp) -.align 16 -.L006loop: - xorl %esi,%eax - xorl -4(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 16(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 12(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl -16(%edi),%esi - xorl %ecx,%edx - movl %edx,16(%esp) - xorl %ebx,%ecx - movl %ecx,12(%esp) - xorl %esi,%ecx - xorl -12(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 8(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl 4(%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl -24(%edi),%esi - xorl %eax,%ebx - movl %ebx,8(%esp) - xorl %edx,%eax - movl %eax,4(%esp) - xorl %esi,%eax - xorl -20(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 16(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 12(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl -32(%edi),%esi - xorl %ecx,%edx - movl %edx,16(%esp) - xorl %ebx,%ecx - movl %ecx,12(%esp) - xorl %esi,%ecx - xorl -28(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 8(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl 4(%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl -40(%edi),%esi - xorl %eax,%ebx - movl %ebx,8(%esp) - xorl %edx,%eax - movl %eax,4(%esp) - xorl %esi,%eax - xorl -36(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 16(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 12(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl -48(%edi),%esi - xorl %ecx,%edx - movl %edx,16(%esp) - xorl %ebx,%ecx - movl %ecx,12(%esp) - xorl %esi,%ecx - xorl -44(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 8(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl 4(%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl -56(%edi),%esi - xorl %eax,%ebx - movl %ebx,8(%esp) - xorl %edx,%eax - movl %eax,4(%esp) - subl $64,%edi - cmpl 20(%esp),%edi - je .L007done - andl %eax,%esi - movl 16(%esp),%edx - roll $1,%esi - movl %edx,%ecx - xorl %esi,%ebx - orl 4(%edi),%ecx - movl %ebx,8(%esp) - xorl 12(%esp),%ecx - movl 12(%edi),%esi - movl %ecx,12(%esp) - orl %ebx,%esi - andl (%edi),%ecx - xorl %esi,%eax - roll $1,%ecx - movl %eax,4(%esp) - xorl %ecx,%edx - movl -8(%edi),%esi - movl %edx,16(%esp) - jmp .L006loop -.align 8 -.L007done: - movl %eax,%ecx - movl %ebx,%edx - movl 12(%esp),%eax - movl 16(%esp),%ebx - xorl %esi,%ecx - xorl 12(%edi),%edx - xorl (%edi),%eax - xorl 4(%edi),%ebx - ret -.size _x86_Camellia_decrypt,.-_x86_Camellia_decrypt -.globl Camellia_Ekeygen -.type Camellia_Ekeygen,@function -.align 16 -Camellia_Ekeygen: -.L_Camellia_Ekeygen_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - pushl %ebp - pushl %ebx - pushl %esi - pushl %edi - subl $16,%esp - movl 36(%esp),%ebp - movl 40(%esp),%esi - movl 44(%esp),%edi - movl (%esi),%eax - movl 4(%esi),%ebx - movl 8(%esi),%ecx - movl 12(%esi),%edx - bswap %eax - bswap %ebx - bswap %ecx - bswap %edx - movl %eax,(%edi) - movl %ebx,4(%edi) - movl %ecx,8(%edi) - movl %edx,12(%edi) - cmpl $128,%ebp - je .L0081st128 - movl 16(%esi),%eax - movl 20(%esi),%ebx - cmpl $192,%ebp - je .L0091st192 - movl 24(%esi),%ecx - movl 28(%esi),%edx - jmp .L0101st256 -.align 4 -.L0091st192: - movl %eax,%ecx - movl %ebx,%edx - notl %ecx - notl %edx -.align 4 -.L0101st256: - bswap %eax - bswap %ebx - bswap %ecx - bswap %edx - movl %eax,32(%edi) - movl %ebx,36(%edi) - movl %ecx,40(%edi) - movl %edx,44(%edi) - xorl (%edi),%eax - xorl 4(%edi),%ebx - xorl 8(%edi),%ecx - xorl 12(%edi),%edx -.align 4 -.L0081st128: - call .L011pic_point -.L011pic_point: - popl %ebp - leal .LCamellia_SBOX-.L011pic_point(%ebp),%ebp - leal .LCamellia_SIGMA-.LCamellia_SBOX(%ebp),%edi - movl (%edi),%esi - movl %eax,(%esp) - movl %ebx,4(%esp) - movl %ecx,8(%esp) - movl %edx,12(%esp) - xorl %esi,%eax - xorl 4(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 12(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 8(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl 8(%edi),%esi - xorl %ecx,%edx - movl %edx,12(%esp) - xorl %ebx,%ecx - movl %ecx,8(%esp) - xorl %esi,%ecx - xorl 12(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 4(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl (%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl 16(%edi),%esi - xorl %eax,%ebx - movl %ebx,4(%esp) - xorl %edx,%eax - movl %eax,(%esp) - movl 8(%esp),%ecx - movl 12(%esp),%edx - movl 44(%esp),%esi - xorl (%esi),%eax - xorl 4(%esi),%ebx - xorl 8(%esi),%ecx - xorl 12(%esi),%edx - movl 16(%edi),%esi - movl %eax,(%esp) - movl %ebx,4(%esp) - movl %ecx,8(%esp) - movl %edx,12(%esp) - xorl %esi,%eax - xorl 20(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 12(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 8(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl 24(%edi),%esi - xorl %ecx,%edx - movl %edx,12(%esp) - xorl %ebx,%ecx - movl %ecx,8(%esp) - xorl %esi,%ecx - xorl 28(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 4(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl (%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl 32(%edi),%esi - xorl %eax,%ebx - movl %ebx,4(%esp) - xorl %edx,%eax - movl %eax,(%esp) - movl 8(%esp),%ecx - movl 12(%esp),%edx - movl 36(%esp),%esi - cmpl $128,%esi - jne .L0122nd256 - movl 44(%esp),%edi - leal 128(%edi),%edi - movl %eax,-112(%edi) - movl %ebx,-108(%edi) - movl %ecx,-104(%edi) - movl %edx,-100(%edi) - movl %eax,%ebp - shll $15,%eax - movl %ebx,%esi - shrl $17,%esi - shll $15,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $15,%ecx - movl %eax,-80(%edi) - shrl $17,%esi - orl %esi,%ebx - shrl $17,%ebp - movl %edx,%esi - shrl $17,%esi - movl %ebx,-76(%edi) - shll $15,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %ecx,-72(%edi) - movl %edx,-68(%edi) - movl %eax,%ebp - shll $15,%eax - movl %ebx,%esi - shrl $17,%esi - shll $15,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $15,%ecx - movl %eax,-64(%edi) - shrl $17,%esi - orl %esi,%ebx - shrl $17,%ebp - movl %edx,%esi - shrl $17,%esi - movl %ebx,-60(%edi) - shll $15,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %ecx,-56(%edi) - movl %edx,-52(%edi) - movl %eax,%ebp - shll $15,%eax - movl %ebx,%esi - shrl $17,%esi - shll $15,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $15,%ecx - movl %eax,-32(%edi) - shrl $17,%esi - orl %esi,%ebx - shrl $17,%ebp - movl %edx,%esi - shrl $17,%esi - movl %ebx,-28(%edi) - shll $15,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %eax,%ebp - shll $15,%eax - movl %ebx,%esi - shrl $17,%esi - shll $15,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $15,%ecx - movl %eax,-16(%edi) - shrl $17,%esi - orl %esi,%ebx - shrl $17,%ebp - movl %edx,%esi - shrl $17,%esi - movl %ebx,-12(%edi) - shll $15,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %ecx,-8(%edi) - movl %edx,-4(%edi) - movl %ebx,%ebp - shll $2,%ebx - movl %ecx,%esi - shrl $30,%esi - shll $2,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $2,%edx - movl %ebx,32(%edi) - shrl $30,%esi - orl %esi,%ecx - shrl $30,%ebp - movl %eax,%esi - shrl $30,%esi - movl %ecx,36(%edi) - shll $2,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,40(%edi) - movl %eax,44(%edi) - movl %ebx,%ebp - shll $17,%ebx - movl %ecx,%esi - shrl $15,%esi - shll $17,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $17,%edx - movl %ebx,64(%edi) - shrl $15,%esi - orl %esi,%ecx - shrl $15,%ebp - movl %eax,%esi - shrl $15,%esi - movl %ecx,68(%edi) - shll $17,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,72(%edi) - movl %eax,76(%edi) - movl -128(%edi),%ebx - movl -124(%edi),%ecx - movl -120(%edi),%edx - movl -116(%edi),%eax - movl %ebx,%ebp - shll $15,%ebx - movl %ecx,%esi - shrl $17,%esi - shll $15,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $15,%edx - movl %ebx,-96(%edi) - shrl $17,%esi - orl %esi,%ecx - shrl $17,%ebp - movl %eax,%esi - shrl $17,%esi - movl %ecx,-92(%edi) - shll $15,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,-88(%edi) - movl %eax,-84(%edi) - movl %ebx,%ebp - shll $30,%ebx - movl %ecx,%esi - shrl $2,%esi - shll $30,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $30,%edx - movl %ebx,-48(%edi) - shrl $2,%esi - orl %esi,%ecx - shrl $2,%ebp - movl %eax,%esi - shrl $2,%esi - movl %ecx,-44(%edi) - shll $30,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,-40(%edi) - movl %eax,-36(%edi) - movl %ebx,%ebp - shll $15,%ebx - movl %ecx,%esi - shrl $17,%esi - shll $15,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $15,%edx - shrl $17,%esi - orl %esi,%ecx - shrl $17,%ebp - movl %eax,%esi - shrl $17,%esi - shll $15,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,-24(%edi) - movl %eax,-20(%edi) - movl %ebx,%ebp - shll $17,%ebx - movl %ecx,%esi - shrl $15,%esi - shll $17,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $17,%edx - movl %ebx,(%edi) - shrl $15,%esi - orl %esi,%ecx - shrl $15,%ebp - movl %eax,%esi - shrl $15,%esi - movl %ecx,4(%edi) - shll $17,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,8(%edi) - movl %eax,12(%edi) - movl %ebx,%ebp - shll $17,%ebx - movl %ecx,%esi - shrl $15,%esi - shll $17,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $17,%edx - movl %ebx,16(%edi) - shrl $15,%esi - orl %esi,%ecx - shrl $15,%ebp - movl %eax,%esi - shrl $15,%esi - movl %ecx,20(%edi) - shll $17,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,24(%edi) - movl %eax,28(%edi) - movl %ebx,%ebp - shll $17,%ebx - movl %ecx,%esi - shrl $15,%esi - shll $17,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $17,%edx - movl %ebx,48(%edi) - shrl $15,%esi - orl %esi,%ecx - shrl $15,%ebp - movl %eax,%esi - shrl $15,%esi - movl %ecx,52(%edi) - shll $17,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,56(%edi) - movl %eax,60(%edi) - movl $3,%eax - jmp .L013done -.align 16 -.L0122nd256: - movl 44(%esp),%esi - movl %eax,48(%esi) - movl %ebx,52(%esi) - movl %ecx,56(%esi) - movl %edx,60(%esi) - xorl 32(%esi),%eax - xorl 36(%esi),%ebx - xorl 40(%esi),%ecx - xorl 44(%esi),%edx - movl 32(%edi),%esi - movl %eax,(%esp) - movl %ebx,4(%esp) - movl %ecx,8(%esp) - movl %edx,12(%esp) - xorl %esi,%eax - xorl 36(%edi),%ebx - movzbl %ah,%esi - movl 2052(%ebp,%esi,8),%edx - movzbl %al,%esi - xorl 4(%ebp,%esi,8),%edx - shrl $16,%eax - movzbl %bl,%esi - movl (%ebp,%esi,8),%ecx - movzbl %ah,%esi - xorl (%ebp,%esi,8),%edx - movzbl %bh,%esi - xorl 4(%ebp,%esi,8),%ecx - shrl $16,%ebx - movzbl %al,%eax - xorl 2048(%ebp,%eax,8),%edx - movzbl %bh,%esi - movl 12(%esp),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl 2048(%ebp,%esi,8),%ecx - movzbl %bl,%esi - movl 8(%esp),%ebx - xorl %eax,%edx - xorl 2052(%ebp,%esi,8),%ecx - movl 40(%edi),%esi - xorl %ecx,%edx - movl %edx,12(%esp) - xorl %ebx,%ecx - movl %ecx,8(%esp) - xorl %esi,%ecx - xorl 44(%edi),%edx - movzbl %ch,%esi - movl 2052(%ebp,%esi,8),%ebx - movzbl %cl,%esi - xorl 4(%ebp,%esi,8),%ebx - shrl $16,%ecx - movzbl %dl,%esi - movl (%ebp,%esi,8),%eax - movzbl %ch,%esi - xorl (%ebp,%esi,8),%ebx - movzbl %dh,%esi - xorl 4(%ebp,%esi,8),%eax - shrl $16,%edx - movzbl %cl,%ecx - xorl 2048(%ebp,%ecx,8),%ebx - movzbl %dh,%esi - movl 4(%esp),%ecx - xorl %ebx,%eax - rorl $8,%ebx - xorl 2048(%ebp,%esi,8),%eax - movzbl %dl,%esi - movl (%esp),%edx - xorl %ecx,%ebx - xorl 2052(%ebp,%esi,8),%eax - movl 48(%edi),%esi - xorl %eax,%ebx - movl %ebx,4(%esp) - xorl %edx,%eax - movl %eax,(%esp) - movl 8(%esp),%ecx - movl 12(%esp),%edx - movl 44(%esp),%edi - leal 128(%edi),%edi - movl %eax,-112(%edi) - movl %ebx,-108(%edi) - movl %ecx,-104(%edi) - movl %edx,-100(%edi) - movl %eax,%ebp - shll $30,%eax - movl %ebx,%esi - shrl $2,%esi - shll $30,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $30,%ecx - movl %eax,-48(%edi) - shrl $2,%esi - orl %esi,%ebx - shrl $2,%ebp - movl %edx,%esi - shrl $2,%esi - movl %ebx,-44(%edi) - shll $30,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %ecx,-40(%edi) - movl %edx,-36(%edi) - movl %eax,%ebp - shll $30,%eax - movl %ebx,%esi - shrl $2,%esi - shll $30,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $30,%ecx - movl %eax,32(%edi) - shrl $2,%esi - orl %esi,%ebx - shrl $2,%ebp - movl %edx,%esi - shrl $2,%esi - movl %ebx,36(%edi) - shll $30,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %ecx,40(%edi) - movl %edx,44(%edi) - movl %ebx,%ebp - shll $19,%ebx - movl %ecx,%esi - shrl $13,%esi - shll $19,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $19,%edx - movl %ebx,128(%edi) - shrl $13,%esi - orl %esi,%ecx - shrl $13,%ebp - movl %eax,%esi - shrl $13,%esi - movl %ecx,132(%edi) - shll $19,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,136(%edi) - movl %eax,140(%edi) - movl -96(%edi),%ebx - movl -92(%edi),%ecx - movl -88(%edi),%edx - movl -84(%edi),%eax - movl %ebx,%ebp - shll $15,%ebx - movl %ecx,%esi - shrl $17,%esi - shll $15,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $15,%edx - movl %ebx,-96(%edi) - shrl $17,%esi - orl %esi,%ecx - shrl $17,%ebp - movl %eax,%esi - shrl $17,%esi - movl %ecx,-92(%edi) - shll $15,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,-88(%edi) - movl %eax,-84(%edi) - movl %ebx,%ebp - shll $15,%ebx - movl %ecx,%esi - shrl $17,%esi - shll $15,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $15,%edx - movl %ebx,-64(%edi) - shrl $17,%esi - orl %esi,%ecx - shrl $17,%ebp - movl %eax,%esi - shrl $17,%esi - movl %ecx,-60(%edi) - shll $15,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,-56(%edi) - movl %eax,-52(%edi) - movl %ebx,%ebp - shll $30,%ebx - movl %ecx,%esi - shrl $2,%esi - shll $30,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $30,%edx - movl %ebx,16(%edi) - shrl $2,%esi - orl %esi,%ecx - shrl $2,%ebp - movl %eax,%esi - shrl $2,%esi - movl %ecx,20(%edi) - shll $30,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,24(%edi) - movl %eax,28(%edi) - movl %ecx,%ebp - shll $2,%ecx - movl %edx,%esi - shrl $30,%esi - shll $2,%edx - orl %esi,%ecx - movl %eax,%esi - shll $2,%eax - movl %ecx,80(%edi) - shrl $30,%esi - orl %esi,%edx - shrl $30,%ebp - movl %ebx,%esi - shrl $30,%esi - movl %edx,84(%edi) - shll $2,%ebx - orl %esi,%eax - orl %ebp,%ebx - movl %eax,88(%edi) - movl %ebx,92(%edi) - movl -80(%edi),%ecx - movl -76(%edi),%edx - movl -72(%edi),%eax - movl -68(%edi),%ebx - movl %ecx,%ebp - shll $15,%ecx - movl %edx,%esi - shrl $17,%esi - shll $15,%edx - orl %esi,%ecx - movl %eax,%esi - shll $15,%eax - movl %ecx,-80(%edi) - shrl $17,%esi - orl %esi,%edx - shrl $17,%ebp - movl %ebx,%esi - shrl $17,%esi - movl %edx,-76(%edi) - shll $15,%ebx - orl %esi,%eax - orl %ebp,%ebx - movl %eax,-72(%edi) - movl %ebx,-68(%edi) - movl %ecx,%ebp - shll $30,%ecx - movl %edx,%esi - shrl $2,%esi - shll $30,%edx - orl %esi,%ecx - movl %eax,%esi - shll $30,%eax - movl %ecx,-16(%edi) - shrl $2,%esi - orl %esi,%edx - shrl $2,%ebp - movl %ebx,%esi - shrl $2,%esi - movl %edx,-12(%edi) - shll $30,%ebx - orl %esi,%eax - orl %ebp,%ebx - movl %eax,-8(%edi) - movl %ebx,-4(%edi) - movl %edx,64(%edi) - movl %eax,68(%edi) - movl %ebx,72(%edi) - movl %ecx,76(%edi) - movl %edx,%ebp - shll $17,%edx - movl %eax,%esi - shrl $15,%esi - shll $17,%eax - orl %esi,%edx - movl %ebx,%esi - shll $17,%ebx - movl %edx,96(%edi) - shrl $15,%esi - orl %esi,%eax - shrl $15,%ebp - movl %ecx,%esi - shrl $15,%esi - movl %eax,100(%edi) - shll $17,%ecx - orl %esi,%ebx - orl %ebp,%ecx - movl %ebx,104(%edi) - movl %ecx,108(%edi) - movl -128(%edi),%edx - movl -124(%edi),%eax - movl -120(%edi),%ebx - movl -116(%edi),%ecx - movl %eax,%ebp - shll $13,%eax - movl %ebx,%esi - shrl $19,%esi - shll $13,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $13,%ecx - movl %eax,-32(%edi) - shrl $19,%esi - orl %esi,%ebx - shrl $19,%ebp - movl %edx,%esi - shrl $19,%esi - movl %ebx,-28(%edi) - shll $13,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %ecx,-24(%edi) - movl %edx,-20(%edi) - movl %eax,%ebp - shll $15,%eax - movl %ebx,%esi - shrl $17,%esi - shll $15,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $15,%ecx - movl %eax,(%edi) - shrl $17,%esi - orl %esi,%ebx - shrl $17,%ebp - movl %edx,%esi - shrl $17,%esi - movl %ebx,4(%edi) - shll $15,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %ecx,8(%edi) - movl %edx,12(%edi) - movl %eax,%ebp - shll $17,%eax - movl %ebx,%esi - shrl $15,%esi - shll $17,%ebx - orl %esi,%eax - movl %ecx,%esi - shll $17,%ecx - movl %eax,48(%edi) - shrl $15,%esi - orl %esi,%ebx - shrl $15,%ebp - movl %edx,%esi - shrl $15,%esi - movl %ebx,52(%edi) - shll $17,%edx - orl %esi,%ecx - orl %ebp,%edx - movl %ecx,56(%edi) - movl %edx,60(%edi) - movl %ebx,%ebp - shll $2,%ebx - movl %ecx,%esi - shrl $30,%esi - shll $2,%ecx - orl %esi,%ebx - movl %edx,%esi - shll $2,%edx - movl %ebx,112(%edi) - shrl $30,%esi - orl %esi,%ecx - shrl $30,%ebp - movl %eax,%esi - shrl $30,%esi - movl %ecx,116(%edi) - shll $2,%eax - orl %esi,%edx - orl %ebp,%eax - movl %edx,120(%edi) - movl %eax,124(%edi) - movl $4,%eax -.L013done: - leal 144(%edi),%edx - addl $16,%esp - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.size Camellia_Ekeygen,.-.L_Camellia_Ekeygen_begin -.globl Camellia_set_key -.type Camellia_set_key,@function -.align 16 -Camellia_set_key: -.L_Camellia_set_key_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - pushl %ebx - movl 8(%esp),%ecx - movl 12(%esp),%ebx - movl 16(%esp),%edx - movl $-1,%eax - testl %ecx,%ecx - jz .L014done - testl %edx,%edx - jz .L014done - movl $-2,%eax - cmpl $256,%ebx - je .L015arg_ok - cmpl $192,%ebx - je .L015arg_ok - cmpl $128,%ebx - jne .L014done -.align 4 -.L015arg_ok: - pushl %edx - pushl %ecx - pushl %ebx - call .L_Camellia_Ekeygen_begin - addl $12,%esp - movl %eax,(%edx) - xorl %eax,%eax -.align 4 -.L014done: - popl %ebx - ret -.size Camellia_set_key,.-.L_Camellia_set_key_begin -.align 64 -.LCamellia_SIGMA: -.long 2694735487,1003262091,3061508184,1286239154,3337565999,3914302142,1426019237,4057165596,283453434,3731369245,2958461122,3018244605,0,0,0,0 -.align 64 -.LCamellia_SBOX: -.long 1886416896,1886388336 -.long 2189591040,741081132 -.long 741092352,3014852787 -.long 3974949888,3233808576 -.long 3014898432,3840147684 -.long 656877312,1465319511 -.long 3233857536,3941204202 -.long 3857048832,2930639022 -.long 3840205824,589496355 -.long 2240120064,1802174571 -.long 1465341696,1162149957 -.long 892679424,2779054245 -.long 3941263872,3991732461 -.long 202116096,1330577487 -.long 2930683392,488439837 -.long 1094795520,2459041938 -.long 589505280,2256928902 -.long 4025478912,2947481775 -.long 1802201856,2088501372 -.long 2475922176,522125343 -.long 1162167552,1044250686 -.long 421075200,3705405660 -.long 2779096320,1583218782 -.long 555819264,185270283 -.long 3991792896,2795896998 -.long 235802112,960036921 -.long 1330597632,3587506389 -.long 1313754624,1566376029 -.long 488447232,3654877401 -.long 1701143808,1515847770 -.long 2459079168,1364262993 -.long 3183328512,1819017324 -.long 2256963072,2341142667 -.long 3099113472,2593783962 -.long 2947526400,4227531003 -.long 2408550144,2964324528 -.long 2088532992,1953759348 -.long 3958106880,724238379 -.long 522133248,4042260720 -.long 3469659648,2223243396 -.long 1044266496,3755933919 -.long 808464384,3419078859 -.long 3705461760,875823156 -.long 1600085760,1987444854 -.long 1583242752,1835860077 -.long 3318072576,2846425257 -.long 185273088,3520135377 -.long 437918208,67371012 -.long 2795939328,336855060 -.long 3789676800,976879674 -.long 960051456,3739091166 -.long 3402287616,286326801 -.long 3587560704,842137650 -.long 1195853568,2627469468 -.long 1566399744,1397948499 -.long 1027423488,4075946226 -.long 3654932736,4278059262 -.long 16843008,3486449871 -.long 1515870720,3284336835 -.long 3604403712,2054815866 -.long 1364283648,606339108 -.long 1448498688,3907518696 -.long 1819044864,1616904288 -.long 1296911616,1768489065 -.long 2341178112,2863268010 -.long 218959104,2694840480 -.long 2593823232,2711683233 -.long 1717986816,1650589794 -.long 4227595008,1414791252 -.long 3435973632,505282590 -.long 2964369408,3772776672 -.long 757935360,1684275300 -.long 1953788928,269484048 -.long 303174144,0 -.long 724249344,2745368739 -.long 538976256,1970602101 -.long 4042321920,2324299914 -.long 2981212416,3873833190 -.long 2223277056,151584777 -.long 2576980224,3722248413 -.long 3755990784,2273771655 -.long 1280068608,2206400643 -.long 3419130624,3452764365 -.long 3267543552,2425356432 -.long 875836416,1936916595 -.long 2122219008,4143317238 -.long 1987474944,2644312221 -.long 84215040,3216965823 -.long 1835887872,1381105746 -.long 3082270464,3638034648 -.long 2846468352,3368550600 -.long 825307392,3334865094 -.long 3520188672,2172715137 -.long 387389184,1869545583 -.long 67372032,320012307 -.long 3621246720,1667432547 -.long 336860160,3924361449 -.long 1482184704,2812739751 -.long 976894464,2677997727 -.long 1633771776,3166437564 -.long 3739147776,690552873 -.long 454761216,4193845497 -.long 286331136,791609391 -.long 471604224,3031695540 -.long 842150400,2021130360 -.long 252645120,101056518 -.long 2627509248,3890675943 -.long 370546176,1903231089 -.long 1397969664,3570663636 -.long 404232192,2880110763 -.long 4076007936,2290614408 -.long 572662272,2374828173 -.long 4278124032,1920073842 -.long 1145324544,3115909305 -.long 3486502656,4177002744 -.long 2998055424,2896953516 -.long 3284386560,909508662 -.long 3048584448,707395626 -.long 2054846976,1010565180 -.long 2442236160,4059103473 -.long 606348288,1077936192 -.long 134744064,3553820883 -.long 3907577856,3149594811 -.long 2829625344,1128464451 -.long 1616928768,353697813 -.long 4244438016,2913796269 -.long 1768515840,2004287607 -.long 1347440640,2155872384 -.long 2863311360,2189557890 -.long 3503345664,3974889708 -.long 2694881280,656867367 -.long 2105376000,3856990437 -.long 2711724288,2240086149 -.long 2307492096,892665909 -.long 1650614784,202113036 -.long 2543294208,1094778945 -.long 1414812672,4025417967 -.long 1532713728,2475884691 -.long 505290240,421068825 -.long 2509608192,555810849 -.long 3772833792,235798542 -.long 4294967040,1313734734 -.long 1684300800,1701118053 -.long 3537031680,3183280317 -.long 269488128,3099066552 -.long 3301229568,2408513679 -.long 0,3958046955 -.long 1212696576,3469607118 -.long 2745410304,808452144 -.long 4160222976,1600061535 -.long 1970631936,3318022341 -.long 3688618752,437911578 -.long 2324335104,3789619425 -.long 50529024,3402236106 -.long 3873891840,1195835463 -.long 3671775744,1027407933 -.long 151587072,16842753 -.long 1061109504,3604349142 -.long 3722304768,1448476758 -.long 2492765184,1296891981 -.long 2273806080,218955789 -.long 1549556736,1717960806 -.long 2206434048,3435921612 -.long 33686016,757923885 -.long 3452816640,303169554 -.long 1246382592,538968096 -.long 2425393152,2981167281 -.long 858993408,2576941209 -.long 1936945920,1280049228 -.long 1734829824,3267494082 -.long 4143379968,2122186878 -.long 4092850944,84213765 -.long 2644352256,3082223799 -.long 2139062016,825294897 -.long 3217014528,387383319 -.long 3806519808,3621191895 -.long 1381126656,1482162264 -.long 2610666240,1633747041 -.long 3638089728,454754331 -.long 640034304,471597084 -.long 3368601600,252641295 -.long 926365440,370540566 -.long 3334915584,404226072 -.long 993737472,572653602 -.long 2172748032,1145307204 -.long 2526451200,2998010034 -.long 1869573888,3048538293 -.long 1263225600,2442199185 -.long 320017152,134742024 -.long 3200171520,2829582504 -.long 1667457792,4244373756 -.long 774778368,1347420240 -.long 3924420864,3503292624 -.long 2038003968,2105344125 -.long 2812782336,2307457161 -.long 2358021120,2543255703 -.long 2678038272,1532690523 -.long 1852730880,2509570197 -.long 3166485504,4294902015 -.long 2391707136,3536978130 -.long 690563328,3301179588 -.long 4126536960,1212678216 -.long 4193908992,4160159991 -.long 3065427456,3688562907 -.long 791621376,50528259 -.long 4261281024,3671720154 -.long 3031741440,1061093439 -.long 1499027712,2492727444 -.long 2021160960,1549533276 -.long 2560137216,33685506 -.long 101058048,1246363722 -.long 1785358848,858980403 -.long 3890734848,1734803559 -.long 1179010560,4092788979 -.long 1903259904,2139029631 -.long 3132799488,3806462178 -.long 3570717696,2610626715 -.long 623191296,640024614 -.long 2880154368,926351415 -.long 1111638528,993722427 -.long 2290649088,2526412950 -.long 2728567296,1263206475 -.long 2374864128,3200123070 -.long 4210752000,774766638 -.long 1920102912,2037973113 -.long 117901056,2357985420 -.long 3115956480,1852702830 -.long 1431655680,2391670926 -.long 4177065984,4126474485 -.long 4008635904,3065381046 -.long 2896997376,4261216509 -.long 168430080,1499005017 -.long 909522432,2560098456 -.long 1229539584,1785331818 -.long 707406336,1178992710 -.long 1751672832,3132752058 -.long 1010580480,623181861 -.long 943208448,1111621698 -.long 4059164928,2728525986 -.long 2762253312,4210688250 -.long 1077952512,117899271 -.long 673720320,1431634005 -.long 3553874688,4008575214 -.long 2071689984,168427530 -.long 3149642496,1229520969 -.long 3385444608,1751646312 -.long 1128481536,943194168 -.long 3250700544,2762211492 -.long 353703168,673710120 -.long 3823362816,2071658619 -.long 2913840384,3385393353 -.long 4109693952,3250651329 -.long 2004317952,3823304931 -.long 3351758592,4109631732 -.long 2155905024,3351707847 -.long 2661195264,2661154974 -.long 14737632,939538488 -.long 328965,1090535745 -.long 5789784,369104406 -.long 14277081,1979741814 -.long 6776679,3640711641 -.long 5131854,2466288531 -.long 8487297,1610637408 -.long 13355979,4060148466 -.long 13224393,1912631922 -.long 723723,3254829762 -.long 11447982,2868947883 -.long 6974058,2583730842 -.long 14013909,1962964341 -.long 1579032,100664838 -.long 6118749,1459640151 -.long 8553090,2684395680 -.long 4605510,2432733585 -.long 14671839,4144035831 -.long 14079702,3036722613 -.long 2565927,3372272073 -.long 9079434,2717950626 -.long 3289650,2348846220 -.long 4934475,3523269330 -.long 4342338,2415956112 -.long 14408667,4127258358 -.long 1842204,117442311 -.long 10395294,2801837991 -.long 10263708,654321447 -.long 3815994,2382401166 -.long 13290186,2986390194 -.long 2434341,1224755529 -.long 8092539,3724599006 -.long 855309,1124090691 -.long 7434609,1543527516 -.long 6250335,3607156695 -.long 2039583,3338717127 -.long 16316664,1040203326 -.long 14145495,4110480885 -.long 4079166,2399178639 -.long 10329501,1728079719 -.long 8158332,520101663 -.long 6316128,402659352 -.long 12171705,1845522030 -.long 12500670,2936057775 -.long 12369084,788541231 -.long 9145227,3791708898 -.long 1447446,2231403909 -.long 3421236,218107149 -.long 5066061,1392530259 -.long 12829635,4026593520 -.long 7500402,2617285788 -.long 9803157,1694524773 -.long 11250603,3925928682 -.long 9342606,2734728099 -.long 12237498,2919280302 -.long 8026746,2650840734 -.long 11776947,3959483628 -.long 131586,2147516544 -.long 11842740,754986285 -.long 11382189,1795189611 -.long 10658466,2818615464 -.long 11316396,721431339 -.long 14211288,905983542 -.long 10132122,2785060518 -.long 1513239,3305162181 -.long 1710618,2248181382 -.long 3487029,1291865421 -.long 13421772,855651123 -.long 16250871,4244700669 -.long 10066329,1711302246 -.long 6381921,1476417624 -.long 5921370,2516620950 -.long 15263976,973093434 -.long 2368548,150997257 -.long 5658198,2499843477 -.long 4210752,268439568 -.long 14803425,2013296760 -.long 6513507,3623934168 -.long 592137,1107313218 -.long 3355443,3422604492 -.long 12566463,4009816047 -.long 10000536,637543974 -.long 9934743,3842041317 -.long 8750469,1627414881 -.long 6842472,436214298 -.long 16579836,1056980799 -.long 15527148,989870907 -.long 657930,2181071490 -.long 14342874,3053500086 -.long 7303023,3674266587 -.long 5460819,3556824276 -.long 6447714,2550175896 -.long 10724259,3892373736 -.long 3026478,2332068747 -.long 526344,33554946 -.long 11513775,3942706155 -.long 2631720,167774730 -.long 11579568,738208812 -.long 7631988,486546717 -.long 12763842,2952835248 -.long 12434877,1862299503 -.long 3552822,2365623693 -.long 2236962,2281736328 -.long 3684408,234884622 -.long 6579300,419436825 -.long 1973790,2264958855 -.long 3750201,1308642894 -.long 2894892,184552203 -.long 10921638,2835392937 -.long 3158064,201329676 -.long 15066597,2030074233 -.long 4473924,285217041 -.long 16645629,2130739071 -.long 8947848,570434082 -.long 10461087,3875596263 -.long 6645093,1493195097 -.long 8882055,3774931425 -.long 7039851,3657489114 -.long 16053492,1023425853 -.long 2302755,3355494600 -.long 4737096,301994514 -.long 1052688,67109892 -.long 13750737,1946186868 -.long 5329233,1409307732 -.long 12632256,805318704 -.long 16382457,2113961598 -.long 13816530,3019945140 -.long 10526880,671098920 -.long 5592405,1426085205 -.long 10592673,1744857192 -.long 4276545,1342197840 -.long 16448250,3187719870 -.long 4408131,3489714384 -.long 1250067,3288384708 -.long 12895428,822096177 -.long 3092271,3405827019 -.long 11053224,704653866 -.long 11974326,2902502829 -.long 3947580,251662095 -.long 2829099,3389049546 -.long 12698049,1879076976 -.long 16777215,4278255615 -.long 13158600,838873650 -.long 10855845,1761634665 -.long 2105376,134219784 -.long 9013641,1644192354 -.long 0,0 -.long 9474192,603989028 -.long 4671303,3506491857 -.long 15724527,4211145723 -.long 15395562,3120609978 -.long 12040119,3976261101 -.long 1381653,1157645637 -.long 394758,2164294017 -.long 13487565,1929409395 -.long 11908533,1828744557 -.long 1184274,2214626436 -.long 8289918,2667618207 -.long 12303291,3993038574 -.long 2697513,1241533002 -.long 986895,3271607235 -.long 12105912,771763758 -.long 460551,3238052289 -.long 263172,16777473 -.long 10197915,3858818790 -.long 9737364,620766501 -.long 2171169,1207978056 -.long 6710886,2566953369 -.long 15132390,3103832505 -.long 13553358,3003167667 -.long 15592941,2063629179 -.long 15198183,4177590777 -.long 3881787,3456159438 -.long 16711422,3204497343 -.long 8355711,3741376479 -.long 12961221,1895854449 -.long 10790052,687876393 -.long 3618615,3439381965 -.long 11645361,1811967084 -.long 5000268,318771987 -.long 9539985,1677747300 -.long 7237230,2600508315 -.long 9276813,1660969827 -.long 7763574,2634063261 -.long 197379,3221274816 -.long 2960685,1258310475 -.long 14606046,3070277559 -.long 9868950,2768283045 -.long 2500134,2298513801 -.long 8224125,1593859935 -.long 13027014,2969612721 -.long 6052956,385881879 -.long 13882323,4093703412 -.long 15921906,3154164924 -.long 5197647,3540046803 -.long 1644825,1174423110 -.long 4144959,3472936911 -.long 14474460,922761015 -.long 7960953,1577082462 -.long 1907997,1191200583 -.long 5395026,2483066004 -.long 15461355,4194368250 -.long 15987699,4227923196 -.long 7171437,1526750043 -.long 6184542,2533398423 -.long 16514043,4261478142 -.long 6908265,1509972570 -.long 11711154,2885725356 -.long 15790320,1006648380 -.long 3223857,1275087948 -.long 789516,50332419 -.long 13948116,889206069 -.long 13619151,4076925939 -.long 9211020,587211555 -.long 14869218,3087055032 -.long 7697781,1560304989 -.long 11119017,1778412138 -.long 4868682,2449511058 -.long 5723991,3573601749 -.long 8684676,553656609 -.long 1118481,1140868164 -.long 4539717,1358975313 -.long 1776411,3321939654 -.long 16119285,2097184125 -.long 15000804,956315961 -.long 921102,2197848963 -.long 7566195,3691044060 -.long 11184810,2852170410 -.long 15856113,2080406652 -.long 14540253,1996519287 -.long 5855577,1442862678 -.long 1315860,83887365 -.long 7105644,452991771 -.long 9605778,2751505572 -.long 5526612,352326933 -.long 13684944,872428596 -.long 7895160,503324190 -.long 7368816,469769244 -.long 14935011,4160813304 -.long 4802889,1375752786 -.long 8421504,536879136 -.long 5263440,335549460 -.long 10987431,3909151209 -.long 16185078,3170942397 -.long 7829367,3707821533 -.long 9671571,3825263844 -.long 8816262,2701173153 -.long 8618883,3758153952 -.long 2763306,2315291274 -.long 13092807,4043370993 -.long 5987163,3590379222 -.long 15329769,2046851706 -.long 15658734,3137387451 -.long 9408399,3808486371 -.long 65793,1073758272 -.long 4013373,1325420367 -.globl Camellia_cbc_encrypt -.type Camellia_cbc_encrypt,@function -.align 16 -Camellia_cbc_encrypt: -.L_Camellia_cbc_encrypt_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - pushl %ebp - pushl %ebx - pushl %esi - pushl %edi - movl 28(%esp),%ecx - cmpl $0,%ecx - je .L016enc_out - pushfl - cld - movl 24(%esp),%eax - movl 28(%esp),%ebx - movl 36(%esp),%edx - movl 40(%esp),%ebp - leal -64(%esp),%esi - andl $-64,%esi - leal -127(%edx),%edi - subl %esi,%edi - negl %edi - andl $960,%edi - subl %edi,%esi - movl 44(%esp),%edi - xchgl %esi,%esp - addl $4,%esp - movl %esi,20(%esp) - movl %eax,24(%esp) - movl %ebx,28(%esp) - movl %ecx,32(%esp) - movl %edx,36(%esp) - movl %ebp,40(%esp) - call .L017pic_point -.L017pic_point: - popl %ebp - leal .LCamellia_SBOX-.L017pic_point(%ebp),%ebp - movl $32,%esi -.align 4 -.L018prefetch_sbox: - movl (%ebp),%eax - movl 32(%ebp),%ebx - movl 64(%ebp),%ecx - movl 96(%ebp),%edx - leal 128(%ebp),%ebp - decl %esi - jnz .L018prefetch_sbox - movl 36(%esp),%eax - subl $4096,%ebp - movl 24(%esp),%esi - movl 272(%eax),%edx - cmpl $0,%edi - je .L019DECRYPT - movl 32(%esp),%ecx - movl 40(%esp),%edi - shll $6,%edx - leal (%eax,%edx,1),%edx - movl %edx,16(%esp) - testl $4294967280,%ecx - jz .L020enc_tail - movl (%edi),%eax - movl 4(%edi),%ebx -.align 4 -.L021enc_loop: - movl 8(%edi),%ecx - movl 12(%edi),%edx - xorl (%esi),%eax - xorl 4(%esi),%ebx - xorl 8(%esi),%ecx - bswap %eax - xorl 12(%esi),%edx - bswap %ebx - movl 36(%esp),%edi - bswap %ecx - bswap %edx - call _x86_Camellia_encrypt - movl 24(%esp),%esi - movl 28(%esp),%edi - bswap %eax - bswap %ebx - bswap %ecx - movl %eax,(%edi) - bswap %edx - movl %ebx,4(%edi) - movl %ecx,8(%edi) - movl %edx,12(%edi) - movl 32(%esp),%ecx - leal 16(%esi),%esi - movl %esi,24(%esp) - leal 16(%edi),%edx - movl %edx,28(%esp) - subl $16,%ecx - testl $4294967280,%ecx - movl %ecx,32(%esp) - jnz .L021enc_loop - testl $15,%ecx - jnz .L020enc_tail - movl 40(%esp),%esi - movl 8(%edi),%ecx - movl 12(%edi),%edx - movl %eax,(%esi) - movl %ebx,4(%esi) - movl %ecx,8(%esi) - movl %edx,12(%esi) - movl 20(%esp),%esp - popfl -.L016enc_out: - popl %edi - popl %esi - popl %ebx - popl %ebp - ret - pushfl -.align 4 -.L020enc_tail: - movl %edi,%eax - movl 28(%esp),%edi - pushl %eax - movl $16,%ebx - subl %ecx,%ebx - cmpl %esi,%edi - je .L022enc_in_place -.align 4 -.long 2767451785 - jmp .L023enc_skip_in_place -.L022enc_in_place: - leal (%edi,%ecx,1),%edi -.L023enc_skip_in_place: - movl %ebx,%ecx - xorl %eax,%eax -.align 4 -.long 2868115081 - popl %edi - movl 28(%esp),%esi - movl (%edi),%eax - movl 4(%edi),%ebx - movl $16,32(%esp) - jmp .L021enc_loop -.align 16 -.L019DECRYPT: - shll $6,%edx - leal (%eax,%edx,1),%edx - movl %eax,16(%esp) - movl %edx,36(%esp) - cmpl 28(%esp),%esi - je .L024dec_in_place - movl 40(%esp),%edi - movl %edi,44(%esp) -.align 4 -.L025dec_loop: - movl (%esi),%eax - movl 4(%esi),%ebx - movl 8(%esi),%ecx - bswap %eax - movl 12(%esi),%edx - bswap %ebx - movl 36(%esp),%edi - bswap %ecx - bswap %edx - call _x86_Camellia_decrypt - movl 44(%esp),%edi - movl 32(%esp),%esi - bswap %eax - bswap %ebx - bswap %ecx - xorl (%edi),%eax - bswap %edx - xorl 4(%edi),%ebx - xorl 8(%edi),%ecx - xorl 12(%edi),%edx - subl $16,%esi - jc .L026dec_partial - movl %esi,32(%esp) - movl 24(%esp),%esi - movl 28(%esp),%edi - movl %eax,(%edi) - movl %ebx,4(%edi) - movl %ecx,8(%edi) - movl %edx,12(%edi) - movl %esi,44(%esp) - leal 16(%esi),%esi - movl %esi,24(%esp) - leal 16(%edi),%edi - movl %edi,28(%esp) - jnz .L025dec_loop - movl 44(%esp),%edi -.L027dec_end: - movl 40(%esp),%esi - movl (%edi),%eax - movl 4(%edi),%ebx - movl 8(%edi),%ecx - movl 12(%edi),%edx - movl %eax,(%esi) - movl %ebx,4(%esi) - movl %ecx,8(%esi) - movl %edx,12(%esi) - jmp .L028dec_out -.align 4 -.L026dec_partial: - leal 44(%esp),%edi - movl %eax,(%edi) - movl %ebx,4(%edi) - movl %ecx,8(%edi) - movl %edx,12(%edi) - leal 16(%esi),%ecx - movl %edi,%esi - movl 28(%esp),%edi -.long 2767451785 - movl 24(%esp),%edi - jmp .L027dec_end -.align 4 -.L024dec_in_place: -.L029dec_in_place_loop: - leal 44(%esp),%edi - movl (%esi),%eax - movl 4(%esi),%ebx - movl 8(%esi),%ecx - movl 12(%esi),%edx - movl %eax,(%edi) - movl %ebx,4(%edi) - movl %ecx,8(%edi) - bswap %eax - movl %edx,12(%edi) - bswap %ebx - movl 36(%esp),%edi - bswap %ecx - bswap %edx - call _x86_Camellia_decrypt - movl 40(%esp),%edi - movl 28(%esp),%esi - bswap %eax - bswap %ebx - bswap %ecx - xorl (%edi),%eax - bswap %edx - xorl 4(%edi),%ebx - xorl 8(%edi),%ecx - xorl 12(%edi),%edx - movl %eax,(%esi) - movl %ebx,4(%esi) - movl %ecx,8(%esi) - movl %edx,12(%esi) - leal 16(%esi),%esi - movl %esi,28(%esp) - leal 44(%esp),%esi - movl (%esi),%eax - movl 4(%esi),%ebx - movl 8(%esi),%ecx - movl 12(%esi),%edx - movl %eax,(%edi) - movl %ebx,4(%edi) - movl %ecx,8(%edi) - movl %edx,12(%edi) - movl 24(%esp),%esi - leal 16(%esi),%esi - movl %esi,24(%esp) - movl 32(%esp),%ecx - subl $16,%ecx - jc .L030dec_in_place_partial - movl %ecx,32(%esp) - jnz .L029dec_in_place_loop - jmp .L028dec_out -.align 4 -.L030dec_in_place_partial: - movl 28(%esp),%edi - leal 44(%esp),%esi - leal (%edi,%ecx,1),%edi - leal 16(%esi,%ecx,1),%esi - negl %ecx -.long 2767451785 -.align 4 -.L028dec_out: - movl 20(%esp),%esp - popfl - popl %edi - popl %esi - popl %ebx - popl %ebp - ret -.size Camellia_cbc_encrypt,.-.L_Camellia_cbc_encrypt_begin -.byte 67,97,109,101,108,108,105,97,32,102,111,114,32,120,56,54 -.byte 32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115 -.byte 115,108,46,111,114,103,62,0 - - .section ".note.gnu.property", "a" - .p2align 2 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - .asciz "GNU" -1: - .p2align 2 - .long 0xc0000002 - .long 3f - 2f -2: - .long 3 -3: - .p2align 2 -4: diff --git a/openssl/src/crypto/camellia/gen/linux_x64/cmll-x86_64.s b/openssl/src/crypto/camellia/gen/linux_x64/cmll-x86_64.s deleted file mode 100644 index 605c25d74..000000000 --- a/openssl/src/crypto/camellia/gen/linux_x64/cmll-x86_64.s +++ /dev/null @@ -1,1946 +0,0 @@ -.text - - -.globl Camellia_EncryptBlock -.type Camellia_EncryptBlock,@function -.align 16 -Camellia_EncryptBlock: -.cfi_startproc - movl $128,%eax - subl %edi,%eax - movl $3,%edi - adcl $0,%edi - jmp .Lenc_rounds -.cfi_endproc -.size Camellia_EncryptBlock,.-Camellia_EncryptBlock - -.globl Camellia_EncryptBlock_Rounds -.type Camellia_EncryptBlock_Rounds,@function -.align 16 -.Lenc_rounds: -Camellia_EncryptBlock_Rounds: -.cfi_startproc - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-32 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-40 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-48 -.Lenc_prologue: - - - movq %rcx,%r13 - movq %rdx,%r14 - - shll $6,%edi - leaq .LCamellia_SBOX(%rip),%rbp - leaq (%r14,%rdi,1),%r15 - - movl 0(%rsi),%r8d - movl 4(%rsi),%r9d - movl 8(%rsi),%r10d - bswapl %r8d - movl 12(%rsi),%r11d - bswapl %r9d - bswapl %r10d - bswapl %r11d - - call _x86_64_Camellia_encrypt - - bswapl %r8d - bswapl %r9d - bswapl %r10d - movl %r8d,0(%r13) - bswapl %r11d - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%rbp -.cfi_restore %rbp - movq 32(%rsp),%rbx -.cfi_restore %rbx - leaq 40(%rsp),%rsp -.cfi_adjust_cfa_offset -40 -.Lenc_epilogue: - .byte 0xf3,0xc3 -.cfi_endproc -.size Camellia_EncryptBlock_Rounds,.-Camellia_EncryptBlock_Rounds - -.type _x86_64_Camellia_encrypt,@function -.align 16 -_x86_64_Camellia_encrypt: -.cfi_startproc - xorl 0(%r14),%r9d - xorl 4(%r14),%r8d - xorl 8(%r14),%r11d - xorl 12(%r14),%r10d -.align 16 -.Leloop: - movl 16(%r14),%ebx - movl 20(%r14),%eax - - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 24(%r14),%ebx - movl 28(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 32(%r14),%ebx - movl 36(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 40(%r14),%ebx - movl 44(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 48(%r14),%ebx - movl 52(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 56(%r14),%ebx - movl 60(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 64(%r14),%ebx - movl 68(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - leaq 64(%r14),%r14 - cmpq %r15,%r14 - movl 8(%r14),%edx - movl 12(%r14),%ecx - je .Ledone - - andl %r8d,%eax - orl %r11d,%edx - roll $1,%eax - xorl %edx,%r10d - xorl %eax,%r9d - andl %r10d,%ecx - orl %r9d,%ebx - roll $1,%ecx - xorl %ebx,%r8d - xorl %ecx,%r11d - jmp .Leloop - -.align 16 -.Ledone: - xorl %r10d,%eax - xorl %r11d,%ebx - xorl %r8d,%ecx - xorl %r9d,%edx - - movl %eax,%r8d - movl %ebx,%r9d - movl %ecx,%r10d - movl %edx,%r11d - -.byte 0xf3,0xc3 -.cfi_endproc -.size _x86_64_Camellia_encrypt,.-_x86_64_Camellia_encrypt - - -.globl Camellia_DecryptBlock -.type Camellia_DecryptBlock,@function -.align 16 -Camellia_DecryptBlock: -.cfi_startproc - movl $128,%eax - subl %edi,%eax - movl $3,%edi - adcl $0,%edi - jmp .Ldec_rounds -.cfi_endproc -.size Camellia_DecryptBlock,.-Camellia_DecryptBlock - -.globl Camellia_DecryptBlock_Rounds -.type Camellia_DecryptBlock_Rounds,@function -.align 16 -.Ldec_rounds: -Camellia_DecryptBlock_Rounds: -.cfi_startproc - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-32 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-40 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-48 -.Ldec_prologue: - - - movq %rcx,%r13 - movq %rdx,%r15 - - shll $6,%edi - leaq .LCamellia_SBOX(%rip),%rbp - leaq (%r15,%rdi,1),%r14 - - movl 0(%rsi),%r8d - movl 4(%rsi),%r9d - movl 8(%rsi),%r10d - bswapl %r8d - movl 12(%rsi),%r11d - bswapl %r9d - bswapl %r10d - bswapl %r11d - - call _x86_64_Camellia_decrypt - - bswapl %r8d - bswapl %r9d - bswapl %r10d - movl %r8d,0(%r13) - bswapl %r11d - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%rbp -.cfi_restore %rbp - movq 32(%rsp),%rbx -.cfi_restore %rbx - leaq 40(%rsp),%rsp -.cfi_adjust_cfa_offset -40 -.Ldec_epilogue: - .byte 0xf3,0xc3 -.cfi_endproc -.size Camellia_DecryptBlock_Rounds,.-Camellia_DecryptBlock_Rounds - -.type _x86_64_Camellia_decrypt,@function -.align 16 -_x86_64_Camellia_decrypt: -.cfi_startproc - xorl 0(%r14),%r9d - xorl 4(%r14),%r8d - xorl 8(%r14),%r11d - xorl 12(%r14),%r10d -.align 16 -.Ldloop: - movl -8(%r14),%ebx - movl -4(%r14),%eax - - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -16(%r14),%ebx - movl -12(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -24(%r14),%ebx - movl -20(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -32(%r14),%ebx - movl -28(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -40(%r14),%ebx - movl -36(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -48(%r14),%ebx - movl -44(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl -56(%r14),%ebx - movl -52(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - leaq -64(%r14),%r14 - cmpq %r15,%r14 - movl 0(%r14),%edx - movl 4(%r14),%ecx - je .Lddone - - andl %r8d,%eax - orl %r11d,%edx - roll $1,%eax - xorl %edx,%r10d - xorl %eax,%r9d - andl %r10d,%ecx - orl %r9d,%ebx - roll $1,%ecx - xorl %ebx,%r8d - xorl %ecx,%r11d - - jmp .Ldloop - -.align 16 -.Lddone: - xorl %r10d,%ecx - xorl %r11d,%edx - xorl %r8d,%eax - xorl %r9d,%ebx - - movl %ecx,%r8d - movl %edx,%r9d - movl %eax,%r10d - movl %ebx,%r11d - -.byte 0xf3,0xc3 -.cfi_endproc -.size _x86_64_Camellia_decrypt,.-_x86_64_Camellia_decrypt -.globl Camellia_Ekeygen -.type Camellia_Ekeygen,@function -.align 16 -Camellia_Ekeygen: -.cfi_startproc - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-32 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-40 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-48 -.Lkey_prologue: - - movl %edi,%r15d - movq %rdx,%r13 - - movl 0(%rsi),%r8d - movl 4(%rsi),%r9d - movl 8(%rsi),%r10d - movl 12(%rsi),%r11d - - bswapl %r8d - bswapl %r9d - bswapl %r10d - bswapl %r11d - movl %r9d,0(%r13) - movl %r8d,4(%r13) - movl %r11d,8(%r13) - movl %r10d,12(%r13) - cmpq $128,%r15 - je .L1st128 - - movl 16(%rsi),%r8d - movl 20(%rsi),%r9d - cmpq $192,%r15 - je .L1st192 - movl 24(%rsi),%r10d - movl 28(%rsi),%r11d - jmp .L1st256 -.L1st192: - movl %r8d,%r10d - movl %r9d,%r11d - notl %r10d - notl %r11d -.L1st256: - bswapl %r8d - bswapl %r9d - bswapl %r10d - bswapl %r11d - movl %r9d,32(%r13) - movl %r8d,36(%r13) - movl %r11d,40(%r13) - movl %r10d,44(%r13) - xorl 0(%r13),%r9d - xorl 4(%r13),%r8d - xorl 8(%r13),%r11d - xorl 12(%r13),%r10d - -.L1st128: - leaq .LCamellia_SIGMA(%rip),%r14 - leaq .LCamellia_SBOX(%rip),%rbp - - movl 0(%r14),%ebx - movl 4(%r14),%eax - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 8(%r14),%ebx - movl 12(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 16(%r14),%ebx - movl 20(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - xorl 0(%r13),%r9d - xorl 4(%r13),%r8d - xorl 8(%r13),%r11d - xorl 12(%r13),%r10d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 24(%r14),%ebx - movl 28(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 32(%r14),%ebx - movl 36(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - cmpq $128,%r15 - jne .L2nd256 - - leaq 128(%r13),%r13 - shlq $32,%r8 - shlq $32,%r10 - orq %r9,%r8 - orq %r11,%r10 - movq -128(%r13),%rax - movq -120(%r13),%rbx - movq %r8,-112(%r13) - movq %r10,-104(%r13) - movq %rax,%r11 - shlq $15,%rax - movq %rbx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rax - shlq $15,%rbx - orq %r11,%rbx - movq %rax,-96(%r13) - movq %rbx,-88(%r13) - movq %r8,%r11 - shlq $15,%r8 - movq %r10,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r8 - shlq $15,%r10 - orq %r11,%r10 - movq %r8,-80(%r13) - movq %r10,-72(%r13) - movq %r8,%r11 - shlq $15,%r8 - movq %r10,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r8 - shlq $15,%r10 - orq %r11,%r10 - movq %r8,-64(%r13) - movq %r10,-56(%r13) - movq %rax,%r11 - shlq $30,%rax - movq %rbx,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%rax - shlq $30,%rbx - orq %r11,%rbx - movq %rax,-48(%r13) - movq %rbx,-40(%r13) - movq %r8,%r11 - shlq $15,%r8 - movq %r10,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r8 - shlq $15,%r10 - orq %r11,%r10 - movq %r8,-32(%r13) - movq %rax,%r11 - shlq $15,%rax - movq %rbx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rax - shlq $15,%rbx - orq %r11,%rbx - movq %rbx,-24(%r13) - movq %r8,%r11 - shlq $15,%r8 - movq %r10,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r8 - shlq $15,%r10 - orq %r11,%r10 - movq %r8,-16(%r13) - movq %r10,-8(%r13) - movq %rax,%r11 - shlq $17,%rax - movq %rbx,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%rax - shlq $17,%rbx - orq %r11,%rbx - movq %rax,0(%r13) - movq %rbx,8(%r13) - movq %rax,%r11 - shlq $17,%rax - movq %rbx,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%rax - shlq $17,%rbx - orq %r11,%rbx - movq %rax,16(%r13) - movq %rbx,24(%r13) - movq %r8,%r11 - shlq $34,%r8 - movq %r10,%r9 - shrq $30,%r9 - shrq $30,%r11 - orq %r9,%r8 - shlq $34,%r10 - orq %r11,%r10 - movq %r8,32(%r13) - movq %r10,40(%r13) - movq %rax,%r11 - shlq $17,%rax - movq %rbx,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%rax - shlq $17,%rbx - orq %r11,%rbx - movq %rax,48(%r13) - movq %rbx,56(%r13) - movq %r8,%r11 - shlq $17,%r8 - movq %r10,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%r8 - shlq $17,%r10 - orq %r11,%r10 - movq %r8,64(%r13) - movq %r10,72(%r13) - movl $3,%eax - jmp .Ldone -.align 16 -.L2nd256: - movl %r9d,48(%r13) - movl %r8d,52(%r13) - movl %r11d,56(%r13) - movl %r10d,60(%r13) - xorl 32(%r13),%r9d - xorl 36(%r13),%r8d - xorl 40(%r13),%r11d - xorl 44(%r13),%r10d - xorl %r8d,%eax - xorl %r9d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 40(%r14),%ebx - movl 44(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r10d - xorl %ecx,%r11d - xorl %edx,%r11d - xorl %r10d,%eax - xorl %r11d,%ebx - movzbl %ah,%esi - movzbl %bl,%edi - movl 2052(%rbp,%rsi,8),%edx - movl 0(%rbp,%rdi,8),%ecx - movzbl %al,%esi - shrl $16,%eax - movzbl %bh,%edi - xorl 4(%rbp,%rsi,8),%edx - shrl $16,%ebx - xorl 4(%rbp,%rdi,8),%ecx - movzbl %ah,%esi - movzbl %bl,%edi - xorl 0(%rbp,%rsi,8),%edx - xorl 2052(%rbp,%rdi,8),%ecx - movzbl %al,%esi - movzbl %bh,%edi - xorl 2048(%rbp,%rsi,8),%edx - xorl 2048(%rbp,%rdi,8),%ecx - movl 48(%r14),%ebx - movl 52(%r14),%eax - xorl %edx,%ecx - rorl $8,%edx - xorl %ecx,%r8d - xorl %ecx,%r9d - xorl %edx,%r9d - movq 0(%r13),%rax - movq 8(%r13),%rbx - movq 32(%r13),%rcx - movq 40(%r13),%rdx - movq 48(%r13),%r14 - movq 56(%r13),%r15 - leaq 128(%r13),%r13 - shlq $32,%r8 - shlq $32,%r10 - orq %r9,%r8 - orq %r11,%r10 - movq %r8,-112(%r13) - movq %r10,-104(%r13) - movq %rcx,%r11 - shlq $15,%rcx - movq %rdx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rcx - shlq $15,%rdx - orq %r11,%rdx - movq %rcx,-96(%r13) - movq %rdx,-88(%r13) - movq %r14,%r11 - shlq $15,%r14 - movq %r15,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%r14 - shlq $15,%r15 - orq %r11,%r15 - movq %r14,-80(%r13) - movq %r15,-72(%r13) - movq %rcx,%r11 - shlq $15,%rcx - movq %rdx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rcx - shlq $15,%rdx - orq %r11,%rdx - movq %rcx,-64(%r13) - movq %rdx,-56(%r13) - movq %r8,%r11 - shlq $30,%r8 - movq %r10,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%r8 - shlq $30,%r10 - orq %r11,%r10 - movq %r8,-48(%r13) - movq %r10,-40(%r13) - movq %rax,%r11 - shlq $45,%rax - movq %rbx,%r9 - shrq $19,%r9 - shrq $19,%r11 - orq %r9,%rax - shlq $45,%rbx - orq %r11,%rbx - movq %rax,-32(%r13) - movq %rbx,-24(%r13) - movq %r14,%r11 - shlq $30,%r14 - movq %r15,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%r14 - shlq $30,%r15 - orq %r11,%r15 - movq %r14,-16(%r13) - movq %r15,-8(%r13) - movq %rax,%r11 - shlq $15,%rax - movq %rbx,%r9 - shrq $49,%r9 - shrq $49,%r11 - orq %r9,%rax - shlq $15,%rbx - orq %r11,%rbx - movq %rax,0(%r13) - movq %rbx,8(%r13) - movq %rcx,%r11 - shlq $30,%rcx - movq %rdx,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%rcx - shlq $30,%rdx - orq %r11,%rdx - movq %rcx,16(%r13) - movq %rdx,24(%r13) - movq %r8,%r11 - shlq $30,%r8 - movq %r10,%r9 - shrq $34,%r9 - shrq $34,%r11 - orq %r9,%r8 - shlq $30,%r10 - orq %r11,%r10 - movq %r8,32(%r13) - movq %r10,40(%r13) - movq %rax,%r11 - shlq $17,%rax - movq %rbx,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%rax - shlq $17,%rbx - orq %r11,%rbx - movq %rax,48(%r13) - movq %rbx,56(%r13) - movq %r14,%r11 - shlq $32,%r14 - movq %r15,%r9 - shrq $32,%r9 - shrq $32,%r11 - orq %r9,%r14 - shlq $32,%r15 - orq %r11,%r15 - movq %r14,64(%r13) - movq %r15,72(%r13) - movq %rcx,%r11 - shlq $34,%rcx - movq %rdx,%r9 - shrq $30,%r9 - shrq $30,%r11 - orq %r9,%rcx - shlq $34,%rdx - orq %r11,%rdx - movq %rcx,80(%r13) - movq %rdx,88(%r13) - movq %r14,%r11 - shlq $17,%r14 - movq %r15,%r9 - shrq $47,%r9 - shrq $47,%r11 - orq %r9,%r14 - shlq $17,%r15 - orq %r11,%r15 - movq %r14,96(%r13) - movq %r15,104(%r13) - movq %rax,%r11 - shlq $34,%rax - movq %rbx,%r9 - shrq $30,%r9 - shrq $30,%r11 - orq %r9,%rax - shlq $34,%rbx - orq %r11,%rbx - movq %rax,112(%r13) - movq %rbx,120(%r13) - movq %r8,%r11 - shlq $51,%r8 - movq %r10,%r9 - shrq $13,%r9 - shrq $13,%r11 - orq %r9,%r8 - shlq $51,%r10 - orq %r11,%r10 - movq %r8,128(%r13) - movq %r10,136(%r13) - movl $4,%eax -.Ldone: - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%rbp -.cfi_restore %rbp - movq 32(%rsp),%rbx -.cfi_restore %rbx - leaq 40(%rsp),%rsp -.cfi_adjust_cfa_offset -40 -.Lkey_epilogue: - .byte 0xf3,0xc3 -.cfi_endproc -.size Camellia_Ekeygen,.-Camellia_Ekeygen -.align 64 -.LCamellia_SIGMA: -.long 0x3bcc908b, 0xa09e667f, 0x4caa73b2, 0xb67ae858 -.long 0xe94f82be, 0xc6ef372f, 0xf1d36f1c, 0x54ff53a5 -.long 0xde682d1d, 0x10e527fa, 0xb3e6c1fd, 0xb05688c2 -.long 0, 0, 0, 0 -.LCamellia_SBOX: -.long 0x70707000,0x70700070 -.long 0x82828200,0x2c2c002c -.long 0x2c2c2c00,0xb3b300b3 -.long 0xececec00,0xc0c000c0 -.long 0xb3b3b300,0xe4e400e4 -.long 0x27272700,0x57570057 -.long 0xc0c0c000,0xeaea00ea -.long 0xe5e5e500,0xaeae00ae -.long 0xe4e4e400,0x23230023 -.long 0x85858500,0x6b6b006b -.long 0x57575700,0x45450045 -.long 0x35353500,0xa5a500a5 -.long 0xeaeaea00,0xeded00ed -.long 0x0c0c0c00,0x4f4f004f -.long 0xaeaeae00,0x1d1d001d -.long 0x41414100,0x92920092 -.long 0x23232300,0x86860086 -.long 0xefefef00,0xafaf00af -.long 0x6b6b6b00,0x7c7c007c -.long 0x93939300,0x1f1f001f -.long 0x45454500,0x3e3e003e -.long 0x19191900,0xdcdc00dc -.long 0xa5a5a500,0x5e5e005e -.long 0x21212100,0x0b0b000b -.long 0xededed00,0xa6a600a6 -.long 0x0e0e0e00,0x39390039 -.long 0x4f4f4f00,0xd5d500d5 -.long 0x4e4e4e00,0x5d5d005d -.long 0x1d1d1d00,0xd9d900d9 -.long 0x65656500,0x5a5a005a -.long 0x92929200,0x51510051 -.long 0xbdbdbd00,0x6c6c006c -.long 0x86868600,0x8b8b008b -.long 0xb8b8b800,0x9a9a009a -.long 0xafafaf00,0xfbfb00fb -.long 0x8f8f8f00,0xb0b000b0 -.long 0x7c7c7c00,0x74740074 -.long 0xebebeb00,0x2b2b002b -.long 0x1f1f1f00,0xf0f000f0 -.long 0xcecece00,0x84840084 -.long 0x3e3e3e00,0xdfdf00df -.long 0x30303000,0xcbcb00cb -.long 0xdcdcdc00,0x34340034 -.long 0x5f5f5f00,0x76760076 -.long 0x5e5e5e00,0x6d6d006d -.long 0xc5c5c500,0xa9a900a9 -.long 0x0b0b0b00,0xd1d100d1 -.long 0x1a1a1a00,0x04040004 -.long 0xa6a6a600,0x14140014 -.long 0xe1e1e100,0x3a3a003a -.long 0x39393900,0xdede00de -.long 0xcacaca00,0x11110011 -.long 0xd5d5d500,0x32320032 -.long 0x47474700,0x9c9c009c -.long 0x5d5d5d00,0x53530053 -.long 0x3d3d3d00,0xf2f200f2 -.long 0xd9d9d900,0xfefe00fe -.long 0x01010100,0xcfcf00cf -.long 0x5a5a5a00,0xc3c300c3 -.long 0xd6d6d600,0x7a7a007a -.long 0x51515100,0x24240024 -.long 0x56565600,0xe8e800e8 -.long 0x6c6c6c00,0x60600060 -.long 0x4d4d4d00,0x69690069 -.long 0x8b8b8b00,0xaaaa00aa -.long 0x0d0d0d00,0xa0a000a0 -.long 0x9a9a9a00,0xa1a100a1 -.long 0x66666600,0x62620062 -.long 0xfbfbfb00,0x54540054 -.long 0xcccccc00,0x1e1e001e -.long 0xb0b0b000,0xe0e000e0 -.long 0x2d2d2d00,0x64640064 -.long 0x74747400,0x10100010 -.long 0x12121200,0x00000000 -.long 0x2b2b2b00,0xa3a300a3 -.long 0x20202000,0x75750075 -.long 0xf0f0f000,0x8a8a008a -.long 0xb1b1b100,0xe6e600e6 -.long 0x84848400,0x09090009 -.long 0x99999900,0xdddd00dd -.long 0xdfdfdf00,0x87870087 -.long 0x4c4c4c00,0x83830083 -.long 0xcbcbcb00,0xcdcd00cd -.long 0xc2c2c200,0x90900090 -.long 0x34343400,0x73730073 -.long 0x7e7e7e00,0xf6f600f6 -.long 0x76767600,0x9d9d009d -.long 0x05050500,0xbfbf00bf -.long 0x6d6d6d00,0x52520052 -.long 0xb7b7b700,0xd8d800d8 -.long 0xa9a9a900,0xc8c800c8 -.long 0x31313100,0xc6c600c6 -.long 0xd1d1d100,0x81810081 -.long 0x17171700,0x6f6f006f -.long 0x04040400,0x13130013 -.long 0xd7d7d700,0x63630063 -.long 0x14141400,0xe9e900e9 -.long 0x58585800,0xa7a700a7 -.long 0x3a3a3a00,0x9f9f009f -.long 0x61616100,0xbcbc00bc -.long 0xdedede00,0x29290029 -.long 0x1b1b1b00,0xf9f900f9 -.long 0x11111100,0x2f2f002f -.long 0x1c1c1c00,0xb4b400b4 -.long 0x32323200,0x78780078 -.long 0x0f0f0f00,0x06060006 -.long 0x9c9c9c00,0xe7e700e7 -.long 0x16161600,0x71710071 -.long 0x53535300,0xd4d400d4 -.long 0x18181800,0xabab00ab -.long 0xf2f2f200,0x88880088 -.long 0x22222200,0x8d8d008d -.long 0xfefefe00,0x72720072 -.long 0x44444400,0xb9b900b9 -.long 0xcfcfcf00,0xf8f800f8 -.long 0xb2b2b200,0xacac00ac -.long 0xc3c3c300,0x36360036 -.long 0xb5b5b500,0x2a2a002a -.long 0x7a7a7a00,0x3c3c003c -.long 0x91919100,0xf1f100f1 -.long 0x24242400,0x40400040 -.long 0x08080800,0xd3d300d3 -.long 0xe8e8e800,0xbbbb00bb -.long 0xa8a8a800,0x43430043 -.long 0x60606000,0x15150015 -.long 0xfcfcfc00,0xadad00ad -.long 0x69696900,0x77770077 -.long 0x50505000,0x80800080 -.long 0xaaaaaa00,0x82820082 -.long 0xd0d0d000,0xecec00ec -.long 0xa0a0a000,0x27270027 -.long 0x7d7d7d00,0xe5e500e5 -.long 0xa1a1a100,0x85850085 -.long 0x89898900,0x35350035 -.long 0x62626200,0x0c0c000c -.long 0x97979700,0x41410041 -.long 0x54545400,0xefef00ef -.long 0x5b5b5b00,0x93930093 -.long 0x1e1e1e00,0x19190019 -.long 0x95959500,0x21210021 -.long 0xe0e0e000,0x0e0e000e -.long 0xffffff00,0x4e4e004e -.long 0x64646400,0x65650065 -.long 0xd2d2d200,0xbdbd00bd -.long 0x10101000,0xb8b800b8 -.long 0xc4c4c400,0x8f8f008f -.long 0x00000000,0xebeb00eb -.long 0x48484800,0xcece00ce -.long 0xa3a3a300,0x30300030 -.long 0xf7f7f700,0x5f5f005f -.long 0x75757500,0xc5c500c5 -.long 0xdbdbdb00,0x1a1a001a -.long 0x8a8a8a00,0xe1e100e1 -.long 0x03030300,0xcaca00ca -.long 0xe6e6e600,0x47470047 -.long 0xdadada00,0x3d3d003d -.long 0x09090900,0x01010001 -.long 0x3f3f3f00,0xd6d600d6 -.long 0xdddddd00,0x56560056 -.long 0x94949400,0x4d4d004d -.long 0x87878700,0x0d0d000d -.long 0x5c5c5c00,0x66660066 -.long 0x83838300,0xcccc00cc -.long 0x02020200,0x2d2d002d -.long 0xcdcdcd00,0x12120012 -.long 0x4a4a4a00,0x20200020 -.long 0x90909000,0xb1b100b1 -.long 0x33333300,0x99990099 -.long 0x73737300,0x4c4c004c -.long 0x67676700,0xc2c200c2 -.long 0xf6f6f600,0x7e7e007e -.long 0xf3f3f300,0x05050005 -.long 0x9d9d9d00,0xb7b700b7 -.long 0x7f7f7f00,0x31310031 -.long 0xbfbfbf00,0x17170017 -.long 0xe2e2e200,0xd7d700d7 -.long 0x52525200,0x58580058 -.long 0x9b9b9b00,0x61610061 -.long 0xd8d8d800,0x1b1b001b -.long 0x26262600,0x1c1c001c -.long 0xc8c8c800,0x0f0f000f -.long 0x37373700,0x16160016 -.long 0xc6c6c600,0x18180018 -.long 0x3b3b3b00,0x22220022 -.long 0x81818100,0x44440044 -.long 0x96969600,0xb2b200b2 -.long 0x6f6f6f00,0xb5b500b5 -.long 0x4b4b4b00,0x91910091 -.long 0x13131300,0x08080008 -.long 0xbebebe00,0xa8a800a8 -.long 0x63636300,0xfcfc00fc -.long 0x2e2e2e00,0x50500050 -.long 0xe9e9e900,0xd0d000d0 -.long 0x79797900,0x7d7d007d -.long 0xa7a7a700,0x89890089 -.long 0x8c8c8c00,0x97970097 -.long 0x9f9f9f00,0x5b5b005b -.long 0x6e6e6e00,0x95950095 -.long 0xbcbcbc00,0xffff00ff -.long 0x8e8e8e00,0xd2d200d2 -.long 0x29292900,0xc4c400c4 -.long 0xf5f5f500,0x48480048 -.long 0xf9f9f900,0xf7f700f7 -.long 0xb6b6b600,0xdbdb00db -.long 0x2f2f2f00,0x03030003 -.long 0xfdfdfd00,0xdada00da -.long 0xb4b4b400,0x3f3f003f -.long 0x59595900,0x94940094 -.long 0x78787800,0x5c5c005c -.long 0x98989800,0x02020002 -.long 0x06060600,0x4a4a004a -.long 0x6a6a6a00,0x33330033 -.long 0xe7e7e700,0x67670067 -.long 0x46464600,0xf3f300f3 -.long 0x71717100,0x7f7f007f -.long 0xbababa00,0xe2e200e2 -.long 0xd4d4d400,0x9b9b009b -.long 0x25252500,0x26260026 -.long 0xababab00,0x37370037 -.long 0x42424200,0x3b3b003b -.long 0x88888800,0x96960096 -.long 0xa2a2a200,0x4b4b004b -.long 0x8d8d8d00,0xbebe00be -.long 0xfafafa00,0x2e2e002e -.long 0x72727200,0x79790079 -.long 0x07070700,0x8c8c008c -.long 0xb9b9b900,0x6e6e006e -.long 0x55555500,0x8e8e008e -.long 0xf8f8f800,0xf5f500f5 -.long 0xeeeeee00,0xb6b600b6 -.long 0xacacac00,0xfdfd00fd -.long 0x0a0a0a00,0x59590059 -.long 0x36363600,0x98980098 -.long 0x49494900,0x6a6a006a -.long 0x2a2a2a00,0x46460046 -.long 0x68686800,0xbaba00ba -.long 0x3c3c3c00,0x25250025 -.long 0x38383800,0x42420042 -.long 0xf1f1f100,0xa2a200a2 -.long 0xa4a4a400,0xfafa00fa -.long 0x40404000,0x07070007 -.long 0x28282800,0x55550055 -.long 0xd3d3d300,0xeeee00ee -.long 0x7b7b7b00,0x0a0a000a -.long 0xbbbbbb00,0x49490049 -.long 0xc9c9c900,0x68680068 -.long 0x43434300,0x38380038 -.long 0xc1c1c100,0xa4a400a4 -.long 0x15151500,0x28280028 -.long 0xe3e3e300,0x7b7b007b -.long 0xadadad00,0xc9c900c9 -.long 0xf4f4f400,0xc1c100c1 -.long 0x77777700,0xe3e300e3 -.long 0xc7c7c700,0xf4f400f4 -.long 0x80808000,0xc7c700c7 -.long 0x9e9e9e00,0x9e9e009e -.long 0x00e0e0e0,0x38003838 -.long 0x00050505,0x41004141 -.long 0x00585858,0x16001616 -.long 0x00d9d9d9,0x76007676 -.long 0x00676767,0xd900d9d9 -.long 0x004e4e4e,0x93009393 -.long 0x00818181,0x60006060 -.long 0x00cbcbcb,0xf200f2f2 -.long 0x00c9c9c9,0x72007272 -.long 0x000b0b0b,0xc200c2c2 -.long 0x00aeaeae,0xab00abab -.long 0x006a6a6a,0x9a009a9a -.long 0x00d5d5d5,0x75007575 -.long 0x00181818,0x06000606 -.long 0x005d5d5d,0x57005757 -.long 0x00828282,0xa000a0a0 -.long 0x00464646,0x91009191 -.long 0x00dfdfdf,0xf700f7f7 -.long 0x00d6d6d6,0xb500b5b5 -.long 0x00272727,0xc900c9c9 -.long 0x008a8a8a,0xa200a2a2 -.long 0x00323232,0x8c008c8c -.long 0x004b4b4b,0xd200d2d2 -.long 0x00424242,0x90009090 -.long 0x00dbdbdb,0xf600f6f6 -.long 0x001c1c1c,0x07000707 -.long 0x009e9e9e,0xa700a7a7 -.long 0x009c9c9c,0x27002727 -.long 0x003a3a3a,0x8e008e8e -.long 0x00cacaca,0xb200b2b2 -.long 0x00252525,0x49004949 -.long 0x007b7b7b,0xde00dede -.long 0x000d0d0d,0x43004343 -.long 0x00717171,0x5c005c5c -.long 0x005f5f5f,0xd700d7d7 -.long 0x001f1f1f,0xc700c7c7 -.long 0x00f8f8f8,0x3e003e3e -.long 0x00d7d7d7,0xf500f5f5 -.long 0x003e3e3e,0x8f008f8f -.long 0x009d9d9d,0x67006767 -.long 0x007c7c7c,0x1f001f1f -.long 0x00606060,0x18001818 -.long 0x00b9b9b9,0x6e006e6e -.long 0x00bebebe,0xaf00afaf -.long 0x00bcbcbc,0x2f002f2f -.long 0x008b8b8b,0xe200e2e2 -.long 0x00161616,0x85008585 -.long 0x00343434,0x0d000d0d -.long 0x004d4d4d,0x53005353 -.long 0x00c3c3c3,0xf000f0f0 -.long 0x00727272,0x9c009c9c -.long 0x00959595,0x65006565 -.long 0x00ababab,0xea00eaea -.long 0x008e8e8e,0xa300a3a3 -.long 0x00bababa,0xae00aeae -.long 0x007a7a7a,0x9e009e9e -.long 0x00b3b3b3,0xec00ecec -.long 0x00020202,0x80008080 -.long 0x00b4b4b4,0x2d002d2d -.long 0x00adadad,0x6b006b6b -.long 0x00a2a2a2,0xa800a8a8 -.long 0x00acacac,0x2b002b2b -.long 0x00d8d8d8,0x36003636 -.long 0x009a9a9a,0xa600a6a6 -.long 0x00171717,0xc500c5c5 -.long 0x001a1a1a,0x86008686 -.long 0x00353535,0x4d004d4d -.long 0x00cccccc,0x33003333 -.long 0x00f7f7f7,0xfd00fdfd -.long 0x00999999,0x66006666 -.long 0x00616161,0x58005858 -.long 0x005a5a5a,0x96009696 -.long 0x00e8e8e8,0x3a003a3a -.long 0x00242424,0x09000909 -.long 0x00565656,0x95009595 -.long 0x00404040,0x10001010 -.long 0x00e1e1e1,0x78007878 -.long 0x00636363,0xd800d8d8 -.long 0x00090909,0x42004242 -.long 0x00333333,0xcc00cccc -.long 0x00bfbfbf,0xef00efef -.long 0x00989898,0x26002626 -.long 0x00979797,0xe500e5e5 -.long 0x00858585,0x61006161 -.long 0x00686868,0x1a001a1a -.long 0x00fcfcfc,0x3f003f3f -.long 0x00ececec,0x3b003b3b -.long 0x000a0a0a,0x82008282 -.long 0x00dadada,0xb600b6b6 -.long 0x006f6f6f,0xdb00dbdb -.long 0x00535353,0xd400d4d4 -.long 0x00626262,0x98009898 -.long 0x00a3a3a3,0xe800e8e8 -.long 0x002e2e2e,0x8b008b8b -.long 0x00080808,0x02000202 -.long 0x00afafaf,0xeb00ebeb -.long 0x00282828,0x0a000a0a -.long 0x00b0b0b0,0x2c002c2c -.long 0x00747474,0x1d001d1d -.long 0x00c2c2c2,0xb000b0b0 -.long 0x00bdbdbd,0x6f006f6f -.long 0x00363636,0x8d008d8d -.long 0x00222222,0x88008888 -.long 0x00383838,0x0e000e0e -.long 0x00646464,0x19001919 -.long 0x001e1e1e,0x87008787 -.long 0x00393939,0x4e004e4e -.long 0x002c2c2c,0x0b000b0b -.long 0x00a6a6a6,0xa900a9a9 -.long 0x00303030,0x0c000c0c -.long 0x00e5e5e5,0x79007979 -.long 0x00444444,0x11001111 -.long 0x00fdfdfd,0x7f007f7f -.long 0x00888888,0x22002222 -.long 0x009f9f9f,0xe700e7e7 -.long 0x00656565,0x59005959 -.long 0x00878787,0xe100e1e1 -.long 0x006b6b6b,0xda00dada -.long 0x00f4f4f4,0x3d003d3d -.long 0x00232323,0xc800c8c8 -.long 0x00484848,0x12001212 -.long 0x00101010,0x04000404 -.long 0x00d1d1d1,0x74007474 -.long 0x00515151,0x54005454 -.long 0x00c0c0c0,0x30003030 -.long 0x00f9f9f9,0x7e007e7e -.long 0x00d2d2d2,0xb400b4b4 -.long 0x00a0a0a0,0x28002828 -.long 0x00555555,0x55005555 -.long 0x00a1a1a1,0x68006868 -.long 0x00414141,0x50005050 -.long 0x00fafafa,0xbe00bebe -.long 0x00434343,0xd000d0d0 -.long 0x00131313,0xc400c4c4 -.long 0x00c4c4c4,0x31003131 -.long 0x002f2f2f,0xcb00cbcb -.long 0x00a8a8a8,0x2a002a2a -.long 0x00b6b6b6,0xad00adad -.long 0x003c3c3c,0x0f000f0f -.long 0x002b2b2b,0xca00caca -.long 0x00c1c1c1,0x70007070 -.long 0x00ffffff,0xff00ffff -.long 0x00c8c8c8,0x32003232 -.long 0x00a5a5a5,0x69006969 -.long 0x00202020,0x08000808 -.long 0x00898989,0x62006262 -.long 0x00000000,0x00000000 -.long 0x00909090,0x24002424 -.long 0x00474747,0xd100d1d1 -.long 0x00efefef,0xfb00fbfb -.long 0x00eaeaea,0xba00baba -.long 0x00b7b7b7,0xed00eded -.long 0x00151515,0x45004545 -.long 0x00060606,0x81008181 -.long 0x00cdcdcd,0x73007373 -.long 0x00b5b5b5,0x6d006d6d -.long 0x00121212,0x84008484 -.long 0x007e7e7e,0x9f009f9f -.long 0x00bbbbbb,0xee00eeee -.long 0x00292929,0x4a004a4a -.long 0x000f0f0f,0xc300c3c3 -.long 0x00b8b8b8,0x2e002e2e -.long 0x00070707,0xc100c1c1 -.long 0x00040404,0x01000101 -.long 0x009b9b9b,0xe600e6e6 -.long 0x00949494,0x25002525 -.long 0x00212121,0x48004848 -.long 0x00666666,0x99009999 -.long 0x00e6e6e6,0xb900b9b9 -.long 0x00cecece,0xb300b3b3 -.long 0x00ededed,0x7b007b7b -.long 0x00e7e7e7,0xf900f9f9 -.long 0x003b3b3b,0xce00cece -.long 0x00fefefe,0xbf00bfbf -.long 0x007f7f7f,0xdf00dfdf -.long 0x00c5c5c5,0x71007171 -.long 0x00a4a4a4,0x29002929 -.long 0x00373737,0xcd00cdcd -.long 0x00b1b1b1,0x6c006c6c -.long 0x004c4c4c,0x13001313 -.long 0x00919191,0x64006464 -.long 0x006e6e6e,0x9b009b9b -.long 0x008d8d8d,0x63006363 -.long 0x00767676,0x9d009d9d -.long 0x00030303,0xc000c0c0 -.long 0x002d2d2d,0x4b004b4b -.long 0x00dedede,0xb700b7b7 -.long 0x00969696,0xa500a5a5 -.long 0x00262626,0x89008989 -.long 0x007d7d7d,0x5f005f5f -.long 0x00c6c6c6,0xb100b1b1 -.long 0x005c5c5c,0x17001717 -.long 0x00d3d3d3,0xf400f4f4 -.long 0x00f2f2f2,0xbc00bcbc -.long 0x004f4f4f,0xd300d3d3 -.long 0x00191919,0x46004646 -.long 0x003f3f3f,0xcf00cfcf -.long 0x00dcdcdc,0x37003737 -.long 0x00797979,0x5e005e5e -.long 0x001d1d1d,0x47004747 -.long 0x00525252,0x94009494 -.long 0x00ebebeb,0xfa00fafa -.long 0x00f3f3f3,0xfc00fcfc -.long 0x006d6d6d,0x5b005b5b -.long 0x005e5e5e,0x97009797 -.long 0x00fbfbfb,0xfe00fefe -.long 0x00696969,0x5a005a5a -.long 0x00b2b2b2,0xac00acac -.long 0x00f0f0f0,0x3c003c3c -.long 0x00313131,0x4c004c4c -.long 0x000c0c0c,0x03000303 -.long 0x00d4d4d4,0x35003535 -.long 0x00cfcfcf,0xf300f3f3 -.long 0x008c8c8c,0x23002323 -.long 0x00e2e2e2,0xb800b8b8 -.long 0x00757575,0x5d005d5d -.long 0x00a9a9a9,0x6a006a6a -.long 0x004a4a4a,0x92009292 -.long 0x00575757,0xd500d5d5 -.long 0x00848484,0x21002121 -.long 0x00111111,0x44004444 -.long 0x00454545,0x51005151 -.long 0x001b1b1b,0xc600c6c6 -.long 0x00f5f5f5,0x7d007d7d -.long 0x00e4e4e4,0x39003939 -.long 0x000e0e0e,0x83008383 -.long 0x00737373,0xdc00dcdc -.long 0x00aaaaaa,0xaa00aaaa -.long 0x00f1f1f1,0x7c007c7c -.long 0x00dddddd,0x77007777 -.long 0x00595959,0x56005656 -.long 0x00141414,0x05000505 -.long 0x006c6c6c,0x1b001b1b -.long 0x00929292,0xa400a4a4 -.long 0x00545454,0x15001515 -.long 0x00d0d0d0,0x34003434 -.long 0x00787878,0x1e001e1e -.long 0x00707070,0x1c001c1c -.long 0x00e3e3e3,0xf800f8f8 -.long 0x00494949,0x52005252 -.long 0x00808080,0x20002020 -.long 0x00505050,0x14001414 -.long 0x00a7a7a7,0xe900e9e9 -.long 0x00f6f6f6,0xbd00bdbd -.long 0x00777777,0xdd00dddd -.long 0x00939393,0xe400e4e4 -.long 0x00868686,0xa100a1a1 -.long 0x00838383,0xe000e0e0 -.long 0x002a2a2a,0x8a008a8a -.long 0x00c7c7c7,0xf100f1f1 -.long 0x005b5b5b,0xd600d6d6 -.long 0x00e9e9e9,0x7a007a7a -.long 0x00eeeeee,0xbb00bbbb -.long 0x008f8f8f,0xe300e3e3 -.long 0x00010101,0x40004040 -.long 0x003d3d3d,0x4f004f4f -.globl Camellia_cbc_encrypt -.type Camellia_cbc_encrypt,@function -.align 16 -Camellia_cbc_encrypt: -.cfi_startproc -.byte 243,15,30,250 - cmpq $0,%rdx - je .Lcbc_abort - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lcbc_prologue: - - movq %rsp,%rbp -.cfi_def_cfa_register %rbp - subq $64,%rsp - andq $-64,%rsp - - - - leaq -64-63(%rcx),%r10 - subq %rsp,%r10 - negq %r10 - andq $0x3C0,%r10 - subq %r10,%rsp - - - movq %rdi,%r12 - movq %rsi,%r13 - movq %r8,%rbx - movq %rcx,%r14 - movl 272(%rcx),%r15d - - movq %r8,40(%rsp) - movq %rbp,48(%rsp) -.cfi_escape 0x0f,0x05,0x77,0x30,0x06,0x23,0x38 - -.Lcbc_body: - leaq .LCamellia_SBOX(%rip),%rbp - - movl $32,%ecx -.align 4 -.Lcbc_prefetch_sbox: - movq 0(%rbp),%rax - movq 32(%rbp),%rsi - movq 64(%rbp),%rdi - movq 96(%rbp),%r11 - leaq 128(%rbp),%rbp - loop .Lcbc_prefetch_sbox - subq $4096,%rbp - shlq $6,%r15 - movq %rdx,%rcx - leaq (%r14,%r15,1),%r15 - - cmpl $0,%r9d - je .LCBC_DECRYPT - - andq $-16,%rdx - andq $15,%rcx - leaq (%r12,%rdx,1),%rdx - movq %r14,0(%rsp) - movq %rdx,8(%rsp) - movq %rcx,16(%rsp) - - cmpq %r12,%rdx - movl 0(%rbx),%r8d - movl 4(%rbx),%r9d - movl 8(%rbx),%r10d - movl 12(%rbx),%r11d - je .Lcbc_enc_tail - jmp .Lcbc_eloop - -.align 16 -.Lcbc_eloop: - xorl 0(%r12),%r8d - xorl 4(%r12),%r9d - xorl 8(%r12),%r10d - bswapl %r8d - xorl 12(%r12),%r11d - bswapl %r9d - bswapl %r10d - bswapl %r11d - - call _x86_64_Camellia_encrypt - - movq 0(%rsp),%r14 - bswapl %r8d - movq 8(%rsp),%rdx - bswapl %r9d - movq 16(%rsp),%rcx - bswapl %r10d - movl %r8d,0(%r13) - bswapl %r11d - movl %r9d,4(%r13) - movl %r10d,8(%r13) - leaq 16(%r12),%r12 - movl %r11d,12(%r13) - cmpq %rdx,%r12 - leaq 16(%r13),%r13 - jne .Lcbc_eloop - - cmpq $0,%rcx - jne .Lcbc_enc_tail - - movq 40(%rsp),%r13 - movl %r8d,0(%r13) - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - jmp .Lcbc_done - -.align 16 -.Lcbc_enc_tail: - xorq %rax,%rax - movq %rax,0+24(%rsp) - movq %rax,8+24(%rsp) - movq %rax,16(%rsp) - -.Lcbc_enc_pushf: - pushfq - cld - movq %r12,%rsi - leaq 8+24(%rsp),%rdi -.long 0x9066A4F3 - popfq -.Lcbc_enc_popf: - - leaq 24(%rsp),%r12 - leaq 16+24(%rsp),%rax - movq %rax,8(%rsp) - jmp .Lcbc_eloop - -.align 16 -.LCBC_DECRYPT: - xchgq %r14,%r15 - addq $15,%rdx - andq $15,%rcx - andq $-16,%rdx - movq %r14,0(%rsp) - leaq (%r12,%rdx,1),%rdx - movq %rdx,8(%rsp) - movq %rcx,16(%rsp) - - movq (%rbx),%rax - movq 8(%rbx),%rbx - jmp .Lcbc_dloop -.align 16 -.Lcbc_dloop: - movl 0(%r12),%r8d - movl 4(%r12),%r9d - movl 8(%r12),%r10d - bswapl %r8d - movl 12(%r12),%r11d - bswapl %r9d - movq %rax,0+24(%rsp) - bswapl %r10d - movq %rbx,8+24(%rsp) - bswapl %r11d - - call _x86_64_Camellia_decrypt - - movq 0(%rsp),%r14 - movq 8(%rsp),%rdx - movq 16(%rsp),%rcx - - bswapl %r8d - movq (%r12),%rax - bswapl %r9d - movq 8(%r12),%rbx - bswapl %r10d - xorl 0+24(%rsp),%r8d - bswapl %r11d - xorl 4+24(%rsp),%r9d - xorl 8+24(%rsp),%r10d - leaq 16(%r12),%r12 - xorl 12+24(%rsp),%r11d - cmpq %rdx,%r12 - je .Lcbc_ddone - - movl %r8d,0(%r13) - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - - leaq 16(%r13),%r13 - jmp .Lcbc_dloop - -.align 16 -.Lcbc_ddone: - movq 40(%rsp),%rdx - cmpq $0,%rcx - jne .Lcbc_dec_tail - - movl %r8d,0(%r13) - movl %r9d,4(%r13) - movl %r10d,8(%r13) - movl %r11d,12(%r13) - - movq %rax,(%rdx) - movq %rbx,8(%rdx) - jmp .Lcbc_done -.align 16 -.Lcbc_dec_tail: - movl %r8d,0+24(%rsp) - movl %r9d,4+24(%rsp) - movl %r10d,8+24(%rsp) - movl %r11d,12+24(%rsp) - -.Lcbc_dec_pushf: - pushfq - cld - leaq 8+24(%rsp),%rsi - leaq (%r13),%rdi -.long 0x9066A4F3 - popfq -.Lcbc_dec_popf: - - movq %rax,(%rdx) - movq %rbx,8(%rdx) - jmp .Lcbc_done - -.align 16 -.Lcbc_done: - movq 48(%rsp),%rcx -.cfi_def_cfa %rcx,56 - movq 0(%rcx),%r15 -.cfi_restore %r15 - movq 8(%rcx),%r14 -.cfi_restore %r14 - movq 16(%rcx),%r13 -.cfi_restore %r13 - movq 24(%rcx),%r12 -.cfi_restore %r12 - movq 32(%rcx),%rbp -.cfi_restore %rbp - movq 40(%rcx),%rbx -.cfi_restore %rbx - leaq 48(%rcx),%rsp -.cfi_def_cfa %rsp,8 -.Lcbc_abort: - .byte 0xf3,0xc3 -.cfi_endproc -.size Camellia_cbc_encrypt,.-Camellia_cbc_encrypt - -.byte 67,97,109,101,108,108,105,97,32,102,111,114,32,120,56,54,95,54,52,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 - .section ".note.gnu.property", "a" - .p2align 3 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - # "GNU" encoded with .byte, since .asciz isn't supported - # on Solaris. - .byte 0x47 - .byte 0x4e - .byte 0x55 - .byte 0 -1: - .p2align 3 - .long 0xc0000002 - .long 3f - 2f -2: - .long 3 -3: - .p2align 3 -4: diff --git a/openssl/src/crypto/camellia/gen/windows_ia32/cmll-x86.asm b/openssl/src/crypto/camellia/gen/windows_ia32/cmll-x86.asm deleted file mode 100644 index 7a03d4dfd..000000000 --- a/openssl/src/crypto/camellia/gen/windows_ia32/cmll-x86.asm +++ /dev/null @@ -1,2360 +0,0 @@ - -%ifidn __OUTPUT_FORMAT__,obj -section code use32 class=code align=64 -%elifidn __OUTPUT_FORMAT__,win32 -$@feat.00 equ 1 -section .text code align=64 -%else -section .text code -%endif -global _Camellia_EncryptBlock_Rounds -align 16 -_Camellia_EncryptBlock_Rounds: -L$_Camellia_EncryptBlock_Rounds_begin: - push ebp - push ebx - push esi - push edi - mov eax,DWORD [20+esp] - mov esi,DWORD [24+esp] - mov edi,DWORD [28+esp] - mov ebx,esp - sub esp,28 - and esp,-64 - lea ecx,[edi-127] - sub ecx,esp - neg ecx - and ecx,960 - sub esp,ecx - add esp,4 - shl eax,6 - lea eax,[eax*1+edi] - mov DWORD [20+esp],ebx - mov DWORD [16+esp],eax - call L$000pic_point -L$000pic_point: - pop ebp - lea ebp,[(L$Camellia_SBOX-L$000pic_point)+ebp] - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov ecx,DWORD [8+esi] - bswap eax - mov edx,DWORD [12+esi] - bswap ebx - bswap ecx - bswap edx - call __x86_Camellia_encrypt - mov esp,DWORD [20+esp] - bswap eax - mov esi,DWORD [32+esp] - bswap ebx - bswap ecx - bswap edx - mov DWORD [esi],eax - mov DWORD [4+esi],ebx - mov DWORD [8+esi],ecx - mov DWORD [12+esi],edx - pop edi - pop esi - pop ebx - pop ebp - ret -global _Camellia_EncryptBlock -align 16 -_Camellia_EncryptBlock: -L$_Camellia_EncryptBlock_begin: - mov eax,128 - sub eax,DWORD [4+esp] - mov eax,3 - adc eax,0 - mov DWORD [4+esp],eax - jmp NEAR L$_Camellia_EncryptBlock_Rounds_begin -global _Camellia_encrypt -align 16 -_Camellia_encrypt: -L$_Camellia_encrypt_begin: - push ebp - push ebx - push esi - push edi - mov esi,DWORD [20+esp] - mov edi,DWORD [28+esp] - mov ebx,esp - sub esp,28 - and esp,-64 - mov eax,DWORD [272+edi] - lea ecx,[edi-127] - sub ecx,esp - neg ecx - and ecx,960 - sub esp,ecx - add esp,4 - shl eax,6 - lea eax,[eax*1+edi] - mov DWORD [20+esp],ebx - mov DWORD [16+esp],eax - call L$001pic_point -L$001pic_point: - pop ebp - lea ebp,[(L$Camellia_SBOX-L$001pic_point)+ebp] - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov ecx,DWORD [8+esi] - bswap eax - mov edx,DWORD [12+esi] - bswap ebx - bswap ecx - bswap edx - call __x86_Camellia_encrypt - mov esp,DWORD [20+esp] - bswap eax - mov esi,DWORD [24+esp] - bswap ebx - bswap ecx - bswap edx - mov DWORD [esi],eax - mov DWORD [4+esi],ebx - mov DWORD [8+esi],ecx - mov DWORD [12+esi],edx - pop edi - pop esi - pop ebx - pop ebp - ret -align 16 -__x86_Camellia_encrypt: - xor eax,DWORD [edi] - xor ebx,DWORD [4+edi] - xor ecx,DWORD [8+edi] - xor edx,DWORD [12+edi] - mov esi,DWORD [16+edi] - mov DWORD [4+esp],eax - mov DWORD [8+esp],ebx - mov DWORD [12+esp],ecx - mov DWORD [16+esp],edx -align 16 -L$002loop: - xor eax,esi - xor ebx,DWORD [20+edi] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [16+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [12+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [24+edi] - xor edx,ecx - mov DWORD [16+esp],edx - xor ecx,ebx - mov DWORD [12+esp],ecx - xor ecx,esi - xor edx,DWORD [28+edi] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [8+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [4+esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [32+edi] - xor ebx,eax - mov DWORD [8+esp],ebx - xor eax,edx - mov DWORD [4+esp],eax - xor eax,esi - xor ebx,DWORD [36+edi] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [16+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [12+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [40+edi] - xor edx,ecx - mov DWORD [16+esp],edx - xor ecx,ebx - mov DWORD [12+esp],ecx - xor ecx,esi - xor edx,DWORD [44+edi] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [8+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [4+esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [48+edi] - xor ebx,eax - mov DWORD [8+esp],ebx - xor eax,edx - mov DWORD [4+esp],eax - xor eax,esi - xor ebx,DWORD [52+edi] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [16+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [12+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [56+edi] - xor edx,ecx - mov DWORD [16+esp],edx - xor ecx,ebx - mov DWORD [12+esp],ecx - xor ecx,esi - xor edx,DWORD [60+edi] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [8+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [4+esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [64+edi] - xor ebx,eax - mov DWORD [8+esp],ebx - xor eax,edx - mov DWORD [4+esp],eax - add edi,64 - cmp edi,DWORD [20+esp] - je NEAR L$003done - and esi,eax - mov edx,DWORD [16+esp] - rol esi,1 - mov ecx,edx - xor ebx,esi - or ecx,DWORD [12+edi] - mov DWORD [8+esp],ebx - xor ecx,DWORD [12+esp] - mov esi,DWORD [4+edi] - mov DWORD [12+esp],ecx - or esi,ebx - and ecx,DWORD [8+edi] - xor eax,esi - rol ecx,1 - mov DWORD [4+esp],eax - xor edx,ecx - mov esi,DWORD [16+edi] - mov DWORD [16+esp],edx - jmp NEAR L$002loop -align 8 -L$003done: - mov ecx,eax - mov edx,ebx - mov eax,DWORD [12+esp] - mov ebx,DWORD [16+esp] - xor eax,esi - xor ebx,DWORD [4+edi] - xor ecx,DWORD [8+edi] - xor edx,DWORD [12+edi] - ret -global _Camellia_DecryptBlock_Rounds -align 16 -_Camellia_DecryptBlock_Rounds: -L$_Camellia_DecryptBlock_Rounds_begin: - push ebp - push ebx - push esi - push edi - mov eax,DWORD [20+esp] - mov esi,DWORD [24+esp] - mov edi,DWORD [28+esp] - mov ebx,esp - sub esp,28 - and esp,-64 - lea ecx,[edi-127] - sub ecx,esp - neg ecx - and ecx,960 - sub esp,ecx - add esp,4 - shl eax,6 - mov DWORD [16+esp],edi - lea edi,[eax*1+edi] - mov DWORD [20+esp],ebx - call L$004pic_point -L$004pic_point: - pop ebp - lea ebp,[(L$Camellia_SBOX-L$004pic_point)+ebp] - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov ecx,DWORD [8+esi] - bswap eax - mov edx,DWORD [12+esi] - bswap ebx - bswap ecx - bswap edx - call __x86_Camellia_decrypt - mov esp,DWORD [20+esp] - bswap eax - mov esi,DWORD [32+esp] - bswap ebx - bswap ecx - bswap edx - mov DWORD [esi],eax - mov DWORD [4+esi],ebx - mov DWORD [8+esi],ecx - mov DWORD [12+esi],edx - pop edi - pop esi - pop ebx - pop ebp - ret -global _Camellia_DecryptBlock -align 16 -_Camellia_DecryptBlock: -L$_Camellia_DecryptBlock_begin: - mov eax,128 - sub eax,DWORD [4+esp] - mov eax,3 - adc eax,0 - mov DWORD [4+esp],eax - jmp NEAR L$_Camellia_DecryptBlock_Rounds_begin -global _Camellia_decrypt -align 16 -_Camellia_decrypt: -L$_Camellia_decrypt_begin: - push ebp - push ebx - push esi - push edi - mov esi,DWORD [20+esp] - mov edi,DWORD [28+esp] - mov ebx,esp - sub esp,28 - and esp,-64 - mov eax,DWORD [272+edi] - lea ecx,[edi-127] - sub ecx,esp - neg ecx - and ecx,960 - sub esp,ecx - add esp,4 - shl eax,6 - mov DWORD [16+esp],edi - lea edi,[eax*1+edi] - mov DWORD [20+esp],ebx - call L$005pic_point -L$005pic_point: - pop ebp - lea ebp,[(L$Camellia_SBOX-L$005pic_point)+ebp] - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov ecx,DWORD [8+esi] - bswap eax - mov edx,DWORD [12+esi] - bswap ebx - bswap ecx - bswap edx - call __x86_Camellia_decrypt - mov esp,DWORD [20+esp] - bswap eax - mov esi,DWORD [24+esp] - bswap ebx - bswap ecx - bswap edx - mov DWORD [esi],eax - mov DWORD [4+esi],ebx - mov DWORD [8+esi],ecx - mov DWORD [12+esi],edx - pop edi - pop esi - pop ebx - pop ebp - ret -align 16 -__x86_Camellia_decrypt: - xor eax,DWORD [edi] - xor ebx,DWORD [4+edi] - xor ecx,DWORD [8+edi] - xor edx,DWORD [12+edi] - mov esi,DWORD [edi-8] - mov DWORD [4+esp],eax - mov DWORD [8+esp],ebx - mov DWORD [12+esp],ecx - mov DWORD [16+esp],edx -align 16 -L$006loop: - xor eax,esi - xor ebx,DWORD [edi-4] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [16+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [12+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [edi-16] - xor edx,ecx - mov DWORD [16+esp],edx - xor ecx,ebx - mov DWORD [12+esp],ecx - xor ecx,esi - xor edx,DWORD [edi-12] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [8+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [4+esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [edi-24] - xor ebx,eax - mov DWORD [8+esp],ebx - xor eax,edx - mov DWORD [4+esp],eax - xor eax,esi - xor ebx,DWORD [edi-20] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [16+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [12+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [edi-32] - xor edx,ecx - mov DWORD [16+esp],edx - xor ecx,ebx - mov DWORD [12+esp],ecx - xor ecx,esi - xor edx,DWORD [edi-28] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [8+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [4+esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [edi-40] - xor ebx,eax - mov DWORD [8+esp],ebx - xor eax,edx - mov DWORD [4+esp],eax - xor eax,esi - xor ebx,DWORD [edi-36] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [16+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [12+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [edi-48] - xor edx,ecx - mov DWORD [16+esp],edx - xor ecx,ebx - mov DWORD [12+esp],ecx - xor ecx,esi - xor edx,DWORD [edi-44] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [8+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [4+esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [edi-56] - xor ebx,eax - mov DWORD [8+esp],ebx - xor eax,edx - mov DWORD [4+esp],eax - sub edi,64 - cmp edi,DWORD [20+esp] - je NEAR L$007done - and esi,eax - mov edx,DWORD [16+esp] - rol esi,1 - mov ecx,edx - xor ebx,esi - or ecx,DWORD [4+edi] - mov DWORD [8+esp],ebx - xor ecx,DWORD [12+esp] - mov esi,DWORD [12+edi] - mov DWORD [12+esp],ecx - or esi,ebx - and ecx,DWORD [edi] - xor eax,esi - rol ecx,1 - mov DWORD [4+esp],eax - xor edx,ecx - mov esi,DWORD [edi-8] - mov DWORD [16+esp],edx - jmp NEAR L$006loop -align 8 -L$007done: - mov ecx,eax - mov edx,ebx - mov eax,DWORD [12+esp] - mov ebx,DWORD [16+esp] - xor ecx,esi - xor edx,DWORD [12+edi] - xor eax,DWORD [edi] - xor ebx,DWORD [4+edi] - ret -global _Camellia_Ekeygen -align 16 -_Camellia_Ekeygen: -L$_Camellia_Ekeygen_begin: - push ebp - push ebx - push esi - push edi - sub esp,16 - mov ebp,DWORD [36+esp] - mov esi,DWORD [40+esp] - mov edi,DWORD [44+esp] - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov ecx,DWORD [8+esi] - mov edx,DWORD [12+esi] - bswap eax - bswap ebx - bswap ecx - bswap edx - mov DWORD [edi],eax - mov DWORD [4+edi],ebx - mov DWORD [8+edi],ecx - mov DWORD [12+edi],edx - cmp ebp,128 - je NEAR L$0081st128 - mov eax,DWORD [16+esi] - mov ebx,DWORD [20+esi] - cmp ebp,192 - je NEAR L$0091st192 - mov ecx,DWORD [24+esi] - mov edx,DWORD [28+esi] - jmp NEAR L$0101st256 -align 4 -L$0091st192: - mov ecx,eax - mov edx,ebx - not ecx - not edx -align 4 -L$0101st256: - bswap eax - bswap ebx - bswap ecx - bswap edx - mov DWORD [32+edi],eax - mov DWORD [36+edi],ebx - mov DWORD [40+edi],ecx - mov DWORD [44+edi],edx - xor eax,DWORD [edi] - xor ebx,DWORD [4+edi] - xor ecx,DWORD [8+edi] - xor edx,DWORD [12+edi] -align 4 -L$0081st128: - call L$011pic_point -L$011pic_point: - pop ebp - lea ebp,[(L$Camellia_SBOX-L$011pic_point)+ebp] - lea edi,[(L$Camellia_SIGMA-L$Camellia_SBOX)+ebp] - mov esi,DWORD [edi] - mov DWORD [esp],eax - mov DWORD [4+esp],ebx - mov DWORD [8+esp],ecx - mov DWORD [12+esp],edx - xor eax,esi - xor ebx,DWORD [4+edi] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [12+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [8+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [8+edi] - xor edx,ecx - mov DWORD [12+esp],edx - xor ecx,ebx - mov DWORD [8+esp],ecx - xor ecx,esi - xor edx,DWORD [12+edi] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [4+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [16+edi] - xor ebx,eax - mov DWORD [4+esp],ebx - xor eax,edx - mov DWORD [esp],eax - mov ecx,DWORD [8+esp] - mov edx,DWORD [12+esp] - mov esi,DWORD [44+esp] - xor eax,DWORD [esi] - xor ebx,DWORD [4+esi] - xor ecx,DWORD [8+esi] - xor edx,DWORD [12+esi] - mov esi,DWORD [16+edi] - mov DWORD [esp],eax - mov DWORD [4+esp],ebx - mov DWORD [8+esp],ecx - mov DWORD [12+esp],edx - xor eax,esi - xor ebx,DWORD [20+edi] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [12+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [8+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [24+edi] - xor edx,ecx - mov DWORD [12+esp],edx - xor ecx,ebx - mov DWORD [8+esp],ecx - xor ecx,esi - xor edx,DWORD [28+edi] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [4+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [32+edi] - xor ebx,eax - mov DWORD [4+esp],ebx - xor eax,edx - mov DWORD [esp],eax - mov ecx,DWORD [8+esp] - mov edx,DWORD [12+esp] - mov esi,DWORD [36+esp] - cmp esi,128 - jne NEAR L$0122nd256 - mov edi,DWORD [44+esp] - lea edi,[128+edi] - mov DWORD [edi-112],eax - mov DWORD [edi-108],ebx - mov DWORD [edi-104],ecx - mov DWORD [edi-100],edx - mov ebp,eax - shl eax,15 - mov esi,ebx - shr esi,17 - shl ebx,15 - or eax,esi - mov esi,ecx - shl ecx,15 - mov DWORD [edi-80],eax - shr esi,17 - or ebx,esi - shr ebp,17 - mov esi,edx - shr esi,17 - mov DWORD [edi-76],ebx - shl edx,15 - or ecx,esi - or edx,ebp - mov DWORD [edi-72],ecx - mov DWORD [edi-68],edx - mov ebp,eax - shl eax,15 - mov esi,ebx - shr esi,17 - shl ebx,15 - or eax,esi - mov esi,ecx - shl ecx,15 - mov DWORD [edi-64],eax - shr esi,17 - or ebx,esi - shr ebp,17 - mov esi,edx - shr esi,17 - mov DWORD [edi-60],ebx - shl edx,15 - or ecx,esi - or edx,ebp - mov DWORD [edi-56],ecx - mov DWORD [edi-52],edx - mov ebp,eax - shl eax,15 - mov esi,ebx - shr esi,17 - shl ebx,15 - or eax,esi - mov esi,ecx - shl ecx,15 - mov DWORD [edi-32],eax - shr esi,17 - or ebx,esi - shr ebp,17 - mov esi,edx - shr esi,17 - mov DWORD [edi-28],ebx - shl edx,15 - or ecx,esi - or edx,ebp - mov ebp,eax - shl eax,15 - mov esi,ebx - shr esi,17 - shl ebx,15 - or eax,esi - mov esi,ecx - shl ecx,15 - mov DWORD [edi-16],eax - shr esi,17 - or ebx,esi - shr ebp,17 - mov esi,edx - shr esi,17 - mov DWORD [edi-12],ebx - shl edx,15 - or ecx,esi - or edx,ebp - mov DWORD [edi-8],ecx - mov DWORD [edi-4],edx - mov ebp,ebx - shl ebx,2 - mov esi,ecx - shr esi,30 - shl ecx,2 - or ebx,esi - mov esi,edx - shl edx,2 - mov DWORD [32+edi],ebx - shr esi,30 - or ecx,esi - shr ebp,30 - mov esi,eax - shr esi,30 - mov DWORD [36+edi],ecx - shl eax,2 - or edx,esi - or eax,ebp - mov DWORD [40+edi],edx - mov DWORD [44+edi],eax - mov ebp,ebx - shl ebx,17 - mov esi,ecx - shr esi,15 - shl ecx,17 - or ebx,esi - mov esi,edx - shl edx,17 - mov DWORD [64+edi],ebx - shr esi,15 - or ecx,esi - shr ebp,15 - mov esi,eax - shr esi,15 - mov DWORD [68+edi],ecx - shl eax,17 - or edx,esi - or eax,ebp - mov DWORD [72+edi],edx - mov DWORD [76+edi],eax - mov ebx,DWORD [edi-128] - mov ecx,DWORD [edi-124] - mov edx,DWORD [edi-120] - mov eax,DWORD [edi-116] - mov ebp,ebx - shl ebx,15 - mov esi,ecx - shr esi,17 - shl ecx,15 - or ebx,esi - mov esi,edx - shl edx,15 - mov DWORD [edi-96],ebx - shr esi,17 - or ecx,esi - shr ebp,17 - mov esi,eax - shr esi,17 - mov DWORD [edi-92],ecx - shl eax,15 - or edx,esi - or eax,ebp - mov DWORD [edi-88],edx - mov DWORD [edi-84],eax - mov ebp,ebx - shl ebx,30 - mov esi,ecx - shr esi,2 - shl ecx,30 - or ebx,esi - mov esi,edx - shl edx,30 - mov DWORD [edi-48],ebx - shr esi,2 - or ecx,esi - shr ebp,2 - mov esi,eax - shr esi,2 - mov DWORD [edi-44],ecx - shl eax,30 - or edx,esi - or eax,ebp - mov DWORD [edi-40],edx - mov DWORD [edi-36],eax - mov ebp,ebx - shl ebx,15 - mov esi,ecx - shr esi,17 - shl ecx,15 - or ebx,esi - mov esi,edx - shl edx,15 - shr esi,17 - or ecx,esi - shr ebp,17 - mov esi,eax - shr esi,17 - shl eax,15 - or edx,esi - or eax,ebp - mov DWORD [edi-24],edx - mov DWORD [edi-20],eax - mov ebp,ebx - shl ebx,17 - mov esi,ecx - shr esi,15 - shl ecx,17 - or ebx,esi - mov esi,edx - shl edx,17 - mov DWORD [edi],ebx - shr esi,15 - or ecx,esi - shr ebp,15 - mov esi,eax - shr esi,15 - mov DWORD [4+edi],ecx - shl eax,17 - or edx,esi - or eax,ebp - mov DWORD [8+edi],edx - mov DWORD [12+edi],eax - mov ebp,ebx - shl ebx,17 - mov esi,ecx - shr esi,15 - shl ecx,17 - or ebx,esi - mov esi,edx - shl edx,17 - mov DWORD [16+edi],ebx - shr esi,15 - or ecx,esi - shr ebp,15 - mov esi,eax - shr esi,15 - mov DWORD [20+edi],ecx - shl eax,17 - or edx,esi - or eax,ebp - mov DWORD [24+edi],edx - mov DWORD [28+edi],eax - mov ebp,ebx - shl ebx,17 - mov esi,ecx - shr esi,15 - shl ecx,17 - or ebx,esi - mov esi,edx - shl edx,17 - mov DWORD [48+edi],ebx - shr esi,15 - or ecx,esi - shr ebp,15 - mov esi,eax - shr esi,15 - mov DWORD [52+edi],ecx - shl eax,17 - or edx,esi - or eax,ebp - mov DWORD [56+edi],edx - mov DWORD [60+edi],eax - mov eax,3 - jmp NEAR L$013done -align 16 -L$0122nd256: - mov esi,DWORD [44+esp] - mov DWORD [48+esi],eax - mov DWORD [52+esi],ebx - mov DWORD [56+esi],ecx - mov DWORD [60+esi],edx - xor eax,DWORD [32+esi] - xor ebx,DWORD [36+esi] - xor ecx,DWORD [40+esi] - xor edx,DWORD [44+esi] - mov esi,DWORD [32+edi] - mov DWORD [esp],eax - mov DWORD [4+esp],ebx - mov DWORD [8+esp],ecx - mov DWORD [12+esp],edx - xor eax,esi - xor ebx,DWORD [36+edi] - movzx esi,ah - mov edx,DWORD [2052+esi*8+ebp] - movzx esi,al - xor edx,DWORD [4+esi*8+ebp] - shr eax,16 - movzx esi,bl - mov ecx,DWORD [esi*8+ebp] - movzx esi,ah - xor edx,DWORD [esi*8+ebp] - movzx esi,bh - xor ecx,DWORD [4+esi*8+ebp] - shr ebx,16 - movzx eax,al - xor edx,DWORD [2048+eax*8+ebp] - movzx esi,bh - mov eax,DWORD [12+esp] - xor ecx,edx - ror edx,8 - xor ecx,DWORD [2048+esi*8+ebp] - movzx esi,bl - mov ebx,DWORD [8+esp] - xor edx,eax - xor ecx,DWORD [2052+esi*8+ebp] - mov esi,DWORD [40+edi] - xor edx,ecx - mov DWORD [12+esp],edx - xor ecx,ebx - mov DWORD [8+esp],ecx - xor ecx,esi - xor edx,DWORD [44+edi] - movzx esi,ch - mov ebx,DWORD [2052+esi*8+ebp] - movzx esi,cl - xor ebx,DWORD [4+esi*8+ebp] - shr ecx,16 - movzx esi,dl - mov eax,DWORD [esi*8+ebp] - movzx esi,ch - xor ebx,DWORD [esi*8+ebp] - movzx esi,dh - xor eax,DWORD [4+esi*8+ebp] - shr edx,16 - movzx ecx,cl - xor ebx,DWORD [2048+ecx*8+ebp] - movzx esi,dh - mov ecx,DWORD [4+esp] - xor eax,ebx - ror ebx,8 - xor eax,DWORD [2048+esi*8+ebp] - movzx esi,dl - mov edx,DWORD [esp] - xor ebx,ecx - xor eax,DWORD [2052+esi*8+ebp] - mov esi,DWORD [48+edi] - xor ebx,eax - mov DWORD [4+esp],ebx - xor eax,edx - mov DWORD [esp],eax - mov ecx,DWORD [8+esp] - mov edx,DWORD [12+esp] - mov edi,DWORD [44+esp] - lea edi,[128+edi] - mov DWORD [edi-112],eax - mov DWORD [edi-108],ebx - mov DWORD [edi-104],ecx - mov DWORD [edi-100],edx - mov ebp,eax - shl eax,30 - mov esi,ebx - shr esi,2 - shl ebx,30 - or eax,esi - mov esi,ecx - shl ecx,30 - mov DWORD [edi-48],eax - shr esi,2 - or ebx,esi - shr ebp,2 - mov esi,edx - shr esi,2 - mov DWORD [edi-44],ebx - shl edx,30 - or ecx,esi - or edx,ebp - mov DWORD [edi-40],ecx - mov DWORD [edi-36],edx - mov ebp,eax - shl eax,30 - mov esi,ebx - shr esi,2 - shl ebx,30 - or eax,esi - mov esi,ecx - shl ecx,30 - mov DWORD [32+edi],eax - shr esi,2 - or ebx,esi - shr ebp,2 - mov esi,edx - shr esi,2 - mov DWORD [36+edi],ebx - shl edx,30 - or ecx,esi - or edx,ebp - mov DWORD [40+edi],ecx - mov DWORD [44+edi],edx - mov ebp,ebx - shl ebx,19 - mov esi,ecx - shr esi,13 - shl ecx,19 - or ebx,esi - mov esi,edx - shl edx,19 - mov DWORD [128+edi],ebx - shr esi,13 - or ecx,esi - shr ebp,13 - mov esi,eax - shr esi,13 - mov DWORD [132+edi],ecx - shl eax,19 - or edx,esi - or eax,ebp - mov DWORD [136+edi],edx - mov DWORD [140+edi],eax - mov ebx,DWORD [edi-96] - mov ecx,DWORD [edi-92] - mov edx,DWORD [edi-88] - mov eax,DWORD [edi-84] - mov ebp,ebx - shl ebx,15 - mov esi,ecx - shr esi,17 - shl ecx,15 - or ebx,esi - mov esi,edx - shl edx,15 - mov DWORD [edi-96],ebx - shr esi,17 - or ecx,esi - shr ebp,17 - mov esi,eax - shr esi,17 - mov DWORD [edi-92],ecx - shl eax,15 - or edx,esi - or eax,ebp - mov DWORD [edi-88],edx - mov DWORD [edi-84],eax - mov ebp,ebx - shl ebx,15 - mov esi,ecx - shr esi,17 - shl ecx,15 - or ebx,esi - mov esi,edx - shl edx,15 - mov DWORD [edi-64],ebx - shr esi,17 - or ecx,esi - shr ebp,17 - mov esi,eax - shr esi,17 - mov DWORD [edi-60],ecx - shl eax,15 - or edx,esi - or eax,ebp - mov DWORD [edi-56],edx - mov DWORD [edi-52],eax - mov ebp,ebx - shl ebx,30 - mov esi,ecx - shr esi,2 - shl ecx,30 - or ebx,esi - mov esi,edx - shl edx,30 - mov DWORD [16+edi],ebx - shr esi,2 - or ecx,esi - shr ebp,2 - mov esi,eax - shr esi,2 - mov DWORD [20+edi],ecx - shl eax,30 - or edx,esi - or eax,ebp - mov DWORD [24+edi],edx - mov DWORD [28+edi],eax - mov ebp,ecx - shl ecx,2 - mov esi,edx - shr esi,30 - shl edx,2 - or ecx,esi - mov esi,eax - shl eax,2 - mov DWORD [80+edi],ecx - shr esi,30 - or edx,esi - shr ebp,30 - mov esi,ebx - shr esi,30 - mov DWORD [84+edi],edx - shl ebx,2 - or eax,esi - or ebx,ebp - mov DWORD [88+edi],eax - mov DWORD [92+edi],ebx - mov ecx,DWORD [edi-80] - mov edx,DWORD [edi-76] - mov eax,DWORD [edi-72] - mov ebx,DWORD [edi-68] - mov ebp,ecx - shl ecx,15 - mov esi,edx - shr esi,17 - shl edx,15 - or ecx,esi - mov esi,eax - shl eax,15 - mov DWORD [edi-80],ecx - shr esi,17 - or edx,esi - shr ebp,17 - mov esi,ebx - shr esi,17 - mov DWORD [edi-76],edx - shl ebx,15 - or eax,esi - or ebx,ebp - mov DWORD [edi-72],eax - mov DWORD [edi-68],ebx - mov ebp,ecx - shl ecx,30 - mov esi,edx - shr esi,2 - shl edx,30 - or ecx,esi - mov esi,eax - shl eax,30 - mov DWORD [edi-16],ecx - shr esi,2 - or edx,esi - shr ebp,2 - mov esi,ebx - shr esi,2 - mov DWORD [edi-12],edx - shl ebx,30 - or eax,esi - or ebx,ebp - mov DWORD [edi-8],eax - mov DWORD [edi-4],ebx - mov DWORD [64+edi],edx - mov DWORD [68+edi],eax - mov DWORD [72+edi],ebx - mov DWORD [76+edi],ecx - mov ebp,edx - shl edx,17 - mov esi,eax - shr esi,15 - shl eax,17 - or edx,esi - mov esi,ebx - shl ebx,17 - mov DWORD [96+edi],edx - shr esi,15 - or eax,esi - shr ebp,15 - mov esi,ecx - shr esi,15 - mov DWORD [100+edi],eax - shl ecx,17 - or ebx,esi - or ecx,ebp - mov DWORD [104+edi],ebx - mov DWORD [108+edi],ecx - mov edx,DWORD [edi-128] - mov eax,DWORD [edi-124] - mov ebx,DWORD [edi-120] - mov ecx,DWORD [edi-116] - mov ebp,eax - shl eax,13 - mov esi,ebx - shr esi,19 - shl ebx,13 - or eax,esi - mov esi,ecx - shl ecx,13 - mov DWORD [edi-32],eax - shr esi,19 - or ebx,esi - shr ebp,19 - mov esi,edx - shr esi,19 - mov DWORD [edi-28],ebx - shl edx,13 - or ecx,esi - or edx,ebp - mov DWORD [edi-24],ecx - mov DWORD [edi-20],edx - mov ebp,eax - shl eax,15 - mov esi,ebx - shr esi,17 - shl ebx,15 - or eax,esi - mov esi,ecx - shl ecx,15 - mov DWORD [edi],eax - shr esi,17 - or ebx,esi - shr ebp,17 - mov esi,edx - shr esi,17 - mov DWORD [4+edi],ebx - shl edx,15 - or ecx,esi - or edx,ebp - mov DWORD [8+edi],ecx - mov DWORD [12+edi],edx - mov ebp,eax - shl eax,17 - mov esi,ebx - shr esi,15 - shl ebx,17 - or eax,esi - mov esi,ecx - shl ecx,17 - mov DWORD [48+edi],eax - shr esi,15 - or ebx,esi - shr ebp,15 - mov esi,edx - shr esi,15 - mov DWORD [52+edi],ebx - shl edx,17 - or ecx,esi - or edx,ebp - mov DWORD [56+edi],ecx - mov DWORD [60+edi],edx - mov ebp,ebx - shl ebx,2 - mov esi,ecx - shr esi,30 - shl ecx,2 - or ebx,esi - mov esi,edx - shl edx,2 - mov DWORD [112+edi],ebx - shr esi,30 - or ecx,esi - shr ebp,30 - mov esi,eax - shr esi,30 - mov DWORD [116+edi],ecx - shl eax,2 - or edx,esi - or eax,ebp - mov DWORD [120+edi],edx - mov DWORD [124+edi],eax - mov eax,4 -L$013done: - lea edx,[144+edi] - add esp,16 - pop edi - pop esi - pop ebx - pop ebp - ret -global _Camellia_set_key -align 16 -_Camellia_set_key: -L$_Camellia_set_key_begin: - push ebx - mov ecx,DWORD [8+esp] - mov ebx,DWORD [12+esp] - mov edx,DWORD [16+esp] - mov eax,-1 - test ecx,ecx - jz NEAR L$014done - test edx,edx - jz NEAR L$014done - mov eax,-2 - cmp ebx,256 - je NEAR L$015arg_ok - cmp ebx,192 - je NEAR L$015arg_ok - cmp ebx,128 - jne NEAR L$014done -align 4 -L$015arg_ok: - push edx - push ecx - push ebx - call L$_Camellia_Ekeygen_begin - add esp,12 - mov DWORD [edx],eax - xor eax,eax -align 4 -L$014done: - pop ebx - ret -align 64 -L$Camellia_SIGMA: -dd 2694735487,1003262091,3061508184,1286239154,3337565999,3914302142,1426019237,4057165596,283453434,3731369245,2958461122,3018244605,0,0,0,0 -align 64 -L$Camellia_SBOX: -dd 1886416896,1886388336 -dd 2189591040,741081132 -dd 741092352,3014852787 -dd 3974949888,3233808576 -dd 3014898432,3840147684 -dd 656877312,1465319511 -dd 3233857536,3941204202 -dd 3857048832,2930639022 -dd 3840205824,589496355 -dd 2240120064,1802174571 -dd 1465341696,1162149957 -dd 892679424,2779054245 -dd 3941263872,3991732461 -dd 202116096,1330577487 -dd 2930683392,488439837 -dd 1094795520,2459041938 -dd 589505280,2256928902 -dd 4025478912,2947481775 -dd 1802201856,2088501372 -dd 2475922176,522125343 -dd 1162167552,1044250686 -dd 421075200,3705405660 -dd 2779096320,1583218782 -dd 555819264,185270283 -dd 3991792896,2795896998 -dd 235802112,960036921 -dd 1330597632,3587506389 -dd 1313754624,1566376029 -dd 488447232,3654877401 -dd 1701143808,1515847770 -dd 2459079168,1364262993 -dd 3183328512,1819017324 -dd 2256963072,2341142667 -dd 3099113472,2593783962 -dd 2947526400,4227531003 -dd 2408550144,2964324528 -dd 2088532992,1953759348 -dd 3958106880,724238379 -dd 522133248,4042260720 -dd 3469659648,2223243396 -dd 1044266496,3755933919 -dd 808464384,3419078859 -dd 3705461760,875823156 -dd 1600085760,1987444854 -dd 1583242752,1835860077 -dd 3318072576,2846425257 -dd 185273088,3520135377 -dd 437918208,67371012 -dd 2795939328,336855060 -dd 3789676800,976879674 -dd 960051456,3739091166 -dd 3402287616,286326801 -dd 3587560704,842137650 -dd 1195853568,2627469468 -dd 1566399744,1397948499 -dd 1027423488,4075946226 -dd 3654932736,4278059262 -dd 16843008,3486449871 -dd 1515870720,3284336835 -dd 3604403712,2054815866 -dd 1364283648,606339108 -dd 1448498688,3907518696 -dd 1819044864,1616904288 -dd 1296911616,1768489065 -dd 2341178112,2863268010 -dd 218959104,2694840480 -dd 2593823232,2711683233 -dd 1717986816,1650589794 -dd 4227595008,1414791252 -dd 3435973632,505282590 -dd 2964369408,3772776672 -dd 757935360,1684275300 -dd 1953788928,269484048 -dd 303174144,0 -dd 724249344,2745368739 -dd 538976256,1970602101 -dd 4042321920,2324299914 -dd 2981212416,3873833190 -dd 2223277056,151584777 -dd 2576980224,3722248413 -dd 3755990784,2273771655 -dd 1280068608,2206400643 -dd 3419130624,3452764365 -dd 3267543552,2425356432 -dd 875836416,1936916595 -dd 2122219008,4143317238 -dd 1987474944,2644312221 -dd 84215040,3216965823 -dd 1835887872,1381105746 -dd 3082270464,3638034648 -dd 2846468352,3368550600 -dd 825307392,3334865094 -dd 3520188672,2172715137 -dd 387389184,1869545583 -dd 67372032,320012307 -dd 3621246720,1667432547 -dd 336860160,3924361449 -dd 1482184704,2812739751 -dd 976894464,2677997727 -dd 1633771776,3166437564 -dd 3739147776,690552873 -dd 454761216,4193845497 -dd 286331136,791609391 -dd 471604224,3031695540 -dd 842150400,2021130360 -dd 252645120,101056518 -dd 2627509248,3890675943 -dd 370546176,1903231089 -dd 1397969664,3570663636 -dd 404232192,2880110763 -dd 4076007936,2290614408 -dd 572662272,2374828173 -dd 4278124032,1920073842 -dd 1145324544,3115909305 -dd 3486502656,4177002744 -dd 2998055424,2896953516 -dd 3284386560,909508662 -dd 3048584448,707395626 -dd 2054846976,1010565180 -dd 2442236160,4059103473 -dd 606348288,1077936192 -dd 134744064,3553820883 -dd 3907577856,3149594811 -dd 2829625344,1128464451 -dd 1616928768,353697813 -dd 4244438016,2913796269 -dd 1768515840,2004287607 -dd 1347440640,2155872384 -dd 2863311360,2189557890 -dd 3503345664,3974889708 -dd 2694881280,656867367 -dd 2105376000,3856990437 -dd 2711724288,2240086149 -dd 2307492096,892665909 -dd 1650614784,202113036 -dd 2543294208,1094778945 -dd 1414812672,4025417967 -dd 1532713728,2475884691 -dd 505290240,421068825 -dd 2509608192,555810849 -dd 3772833792,235798542 -dd 4294967040,1313734734 -dd 1684300800,1701118053 -dd 3537031680,3183280317 -dd 269488128,3099066552 -dd 3301229568,2408513679 -dd 0,3958046955 -dd 1212696576,3469607118 -dd 2745410304,808452144 -dd 4160222976,1600061535 -dd 1970631936,3318022341 -dd 3688618752,437911578 -dd 2324335104,3789619425 -dd 50529024,3402236106 -dd 3873891840,1195835463 -dd 3671775744,1027407933 -dd 151587072,16842753 -dd 1061109504,3604349142 -dd 3722304768,1448476758 -dd 2492765184,1296891981 -dd 2273806080,218955789 -dd 1549556736,1717960806 -dd 2206434048,3435921612 -dd 33686016,757923885 -dd 3452816640,303169554 -dd 1246382592,538968096 -dd 2425393152,2981167281 -dd 858993408,2576941209 -dd 1936945920,1280049228 -dd 1734829824,3267494082 -dd 4143379968,2122186878 -dd 4092850944,84213765 -dd 2644352256,3082223799 -dd 2139062016,825294897 -dd 3217014528,387383319 -dd 3806519808,3621191895 -dd 1381126656,1482162264 -dd 2610666240,1633747041 -dd 3638089728,454754331 -dd 640034304,471597084 -dd 3368601600,252641295 -dd 926365440,370540566 -dd 3334915584,404226072 -dd 993737472,572653602 -dd 2172748032,1145307204 -dd 2526451200,2998010034 -dd 1869573888,3048538293 -dd 1263225600,2442199185 -dd 320017152,134742024 -dd 3200171520,2829582504 -dd 1667457792,4244373756 -dd 774778368,1347420240 -dd 3924420864,3503292624 -dd 2038003968,2105344125 -dd 2812782336,2307457161 -dd 2358021120,2543255703 -dd 2678038272,1532690523 -dd 1852730880,2509570197 -dd 3166485504,4294902015 -dd 2391707136,3536978130 -dd 690563328,3301179588 -dd 4126536960,1212678216 -dd 4193908992,4160159991 -dd 3065427456,3688562907 -dd 791621376,50528259 -dd 4261281024,3671720154 -dd 3031741440,1061093439 -dd 1499027712,2492727444 -dd 2021160960,1549533276 -dd 2560137216,33685506 -dd 101058048,1246363722 -dd 1785358848,858980403 -dd 3890734848,1734803559 -dd 1179010560,4092788979 -dd 1903259904,2139029631 -dd 3132799488,3806462178 -dd 3570717696,2610626715 -dd 623191296,640024614 -dd 2880154368,926351415 -dd 1111638528,993722427 -dd 2290649088,2526412950 -dd 2728567296,1263206475 -dd 2374864128,3200123070 -dd 4210752000,774766638 -dd 1920102912,2037973113 -dd 117901056,2357985420 -dd 3115956480,1852702830 -dd 1431655680,2391670926 -dd 4177065984,4126474485 -dd 4008635904,3065381046 -dd 2896997376,4261216509 -dd 168430080,1499005017 -dd 909522432,2560098456 -dd 1229539584,1785331818 -dd 707406336,1178992710 -dd 1751672832,3132752058 -dd 1010580480,623181861 -dd 943208448,1111621698 -dd 4059164928,2728525986 -dd 2762253312,4210688250 -dd 1077952512,117899271 -dd 673720320,1431634005 -dd 3553874688,4008575214 -dd 2071689984,168427530 -dd 3149642496,1229520969 -dd 3385444608,1751646312 -dd 1128481536,943194168 -dd 3250700544,2762211492 -dd 353703168,673710120 -dd 3823362816,2071658619 -dd 2913840384,3385393353 -dd 4109693952,3250651329 -dd 2004317952,3823304931 -dd 3351758592,4109631732 -dd 2155905024,3351707847 -dd 2661195264,2661154974 -dd 14737632,939538488 -dd 328965,1090535745 -dd 5789784,369104406 -dd 14277081,1979741814 -dd 6776679,3640711641 -dd 5131854,2466288531 -dd 8487297,1610637408 -dd 13355979,4060148466 -dd 13224393,1912631922 -dd 723723,3254829762 -dd 11447982,2868947883 -dd 6974058,2583730842 -dd 14013909,1962964341 -dd 1579032,100664838 -dd 6118749,1459640151 -dd 8553090,2684395680 -dd 4605510,2432733585 -dd 14671839,4144035831 -dd 14079702,3036722613 -dd 2565927,3372272073 -dd 9079434,2717950626 -dd 3289650,2348846220 -dd 4934475,3523269330 -dd 4342338,2415956112 -dd 14408667,4127258358 -dd 1842204,117442311 -dd 10395294,2801837991 -dd 10263708,654321447 -dd 3815994,2382401166 -dd 13290186,2986390194 -dd 2434341,1224755529 -dd 8092539,3724599006 -dd 855309,1124090691 -dd 7434609,1543527516 -dd 6250335,3607156695 -dd 2039583,3338717127 -dd 16316664,1040203326 -dd 14145495,4110480885 -dd 4079166,2399178639 -dd 10329501,1728079719 -dd 8158332,520101663 -dd 6316128,402659352 -dd 12171705,1845522030 -dd 12500670,2936057775 -dd 12369084,788541231 -dd 9145227,3791708898 -dd 1447446,2231403909 -dd 3421236,218107149 -dd 5066061,1392530259 -dd 12829635,4026593520 -dd 7500402,2617285788 -dd 9803157,1694524773 -dd 11250603,3925928682 -dd 9342606,2734728099 -dd 12237498,2919280302 -dd 8026746,2650840734 -dd 11776947,3959483628 -dd 131586,2147516544 -dd 11842740,754986285 -dd 11382189,1795189611 -dd 10658466,2818615464 -dd 11316396,721431339 -dd 14211288,905983542 -dd 10132122,2785060518 -dd 1513239,3305162181 -dd 1710618,2248181382 -dd 3487029,1291865421 -dd 13421772,855651123 -dd 16250871,4244700669 -dd 10066329,1711302246 -dd 6381921,1476417624 -dd 5921370,2516620950 -dd 15263976,973093434 -dd 2368548,150997257 -dd 5658198,2499843477 -dd 4210752,268439568 -dd 14803425,2013296760 -dd 6513507,3623934168 -dd 592137,1107313218 -dd 3355443,3422604492 -dd 12566463,4009816047 -dd 10000536,637543974 -dd 9934743,3842041317 -dd 8750469,1627414881 -dd 6842472,436214298 -dd 16579836,1056980799 -dd 15527148,989870907 -dd 657930,2181071490 -dd 14342874,3053500086 -dd 7303023,3674266587 -dd 5460819,3556824276 -dd 6447714,2550175896 -dd 10724259,3892373736 -dd 3026478,2332068747 -dd 526344,33554946 -dd 11513775,3942706155 -dd 2631720,167774730 -dd 11579568,738208812 -dd 7631988,486546717 -dd 12763842,2952835248 -dd 12434877,1862299503 -dd 3552822,2365623693 -dd 2236962,2281736328 -dd 3684408,234884622 -dd 6579300,419436825 -dd 1973790,2264958855 -dd 3750201,1308642894 -dd 2894892,184552203 -dd 10921638,2835392937 -dd 3158064,201329676 -dd 15066597,2030074233 -dd 4473924,285217041 -dd 16645629,2130739071 -dd 8947848,570434082 -dd 10461087,3875596263 -dd 6645093,1493195097 -dd 8882055,3774931425 -dd 7039851,3657489114 -dd 16053492,1023425853 -dd 2302755,3355494600 -dd 4737096,301994514 -dd 1052688,67109892 -dd 13750737,1946186868 -dd 5329233,1409307732 -dd 12632256,805318704 -dd 16382457,2113961598 -dd 13816530,3019945140 -dd 10526880,671098920 -dd 5592405,1426085205 -dd 10592673,1744857192 -dd 4276545,1342197840 -dd 16448250,3187719870 -dd 4408131,3489714384 -dd 1250067,3288384708 -dd 12895428,822096177 -dd 3092271,3405827019 -dd 11053224,704653866 -dd 11974326,2902502829 -dd 3947580,251662095 -dd 2829099,3389049546 -dd 12698049,1879076976 -dd 16777215,4278255615 -dd 13158600,838873650 -dd 10855845,1761634665 -dd 2105376,134219784 -dd 9013641,1644192354 -dd 0,0 -dd 9474192,603989028 -dd 4671303,3506491857 -dd 15724527,4211145723 -dd 15395562,3120609978 -dd 12040119,3976261101 -dd 1381653,1157645637 -dd 394758,2164294017 -dd 13487565,1929409395 -dd 11908533,1828744557 -dd 1184274,2214626436 -dd 8289918,2667618207 -dd 12303291,3993038574 -dd 2697513,1241533002 -dd 986895,3271607235 -dd 12105912,771763758 -dd 460551,3238052289 -dd 263172,16777473 -dd 10197915,3858818790 -dd 9737364,620766501 -dd 2171169,1207978056 -dd 6710886,2566953369 -dd 15132390,3103832505 -dd 13553358,3003167667 -dd 15592941,2063629179 -dd 15198183,4177590777 -dd 3881787,3456159438 -dd 16711422,3204497343 -dd 8355711,3741376479 -dd 12961221,1895854449 -dd 10790052,687876393 -dd 3618615,3439381965 -dd 11645361,1811967084 -dd 5000268,318771987 -dd 9539985,1677747300 -dd 7237230,2600508315 -dd 9276813,1660969827 -dd 7763574,2634063261 -dd 197379,3221274816 -dd 2960685,1258310475 -dd 14606046,3070277559 -dd 9868950,2768283045 -dd 2500134,2298513801 -dd 8224125,1593859935 -dd 13027014,2969612721 -dd 6052956,385881879 -dd 13882323,4093703412 -dd 15921906,3154164924 -dd 5197647,3540046803 -dd 1644825,1174423110 -dd 4144959,3472936911 -dd 14474460,922761015 -dd 7960953,1577082462 -dd 1907997,1191200583 -dd 5395026,2483066004 -dd 15461355,4194368250 -dd 15987699,4227923196 -dd 7171437,1526750043 -dd 6184542,2533398423 -dd 16514043,4261478142 -dd 6908265,1509972570 -dd 11711154,2885725356 -dd 15790320,1006648380 -dd 3223857,1275087948 -dd 789516,50332419 -dd 13948116,889206069 -dd 13619151,4076925939 -dd 9211020,587211555 -dd 14869218,3087055032 -dd 7697781,1560304989 -dd 11119017,1778412138 -dd 4868682,2449511058 -dd 5723991,3573601749 -dd 8684676,553656609 -dd 1118481,1140868164 -dd 4539717,1358975313 -dd 1776411,3321939654 -dd 16119285,2097184125 -dd 15000804,956315961 -dd 921102,2197848963 -dd 7566195,3691044060 -dd 11184810,2852170410 -dd 15856113,2080406652 -dd 14540253,1996519287 -dd 5855577,1442862678 -dd 1315860,83887365 -dd 7105644,452991771 -dd 9605778,2751505572 -dd 5526612,352326933 -dd 13684944,872428596 -dd 7895160,503324190 -dd 7368816,469769244 -dd 14935011,4160813304 -dd 4802889,1375752786 -dd 8421504,536879136 -dd 5263440,335549460 -dd 10987431,3909151209 -dd 16185078,3170942397 -dd 7829367,3707821533 -dd 9671571,3825263844 -dd 8816262,2701173153 -dd 8618883,3758153952 -dd 2763306,2315291274 -dd 13092807,4043370993 -dd 5987163,3590379222 -dd 15329769,2046851706 -dd 15658734,3137387451 -dd 9408399,3808486371 -dd 65793,1073758272 -dd 4013373,1325420367 -global _Camellia_cbc_encrypt -align 16 -_Camellia_cbc_encrypt: -L$_Camellia_cbc_encrypt_begin: - push ebp - push ebx - push esi - push edi - mov ecx,DWORD [28+esp] - cmp ecx,0 - je NEAR L$016enc_out - pushfd - cld - mov eax,DWORD [24+esp] - mov ebx,DWORD [28+esp] - mov edx,DWORD [36+esp] - mov ebp,DWORD [40+esp] - lea esi,[esp-64] - and esi,-64 - lea edi,[edx-127] - sub edi,esi - neg edi - and edi,960 - sub esi,edi - mov edi,DWORD [44+esp] - xchg esp,esi - add esp,4 - mov DWORD [20+esp],esi - mov DWORD [24+esp],eax - mov DWORD [28+esp],ebx - mov DWORD [32+esp],ecx - mov DWORD [36+esp],edx - mov DWORD [40+esp],ebp - call L$017pic_point -L$017pic_point: - pop ebp - lea ebp,[(L$Camellia_SBOX-L$017pic_point)+ebp] - mov esi,32 -align 4 -L$018prefetch_sbox: - mov eax,DWORD [ebp] - mov ebx,DWORD [32+ebp] - mov ecx,DWORD [64+ebp] - mov edx,DWORD [96+ebp] - lea ebp,[128+ebp] - dec esi - jnz NEAR L$018prefetch_sbox - mov eax,DWORD [36+esp] - sub ebp,4096 - mov esi,DWORD [24+esp] - mov edx,DWORD [272+eax] - cmp edi,0 - je NEAR L$019DECRYPT - mov ecx,DWORD [32+esp] - mov edi,DWORD [40+esp] - shl edx,6 - lea edx,[edx*1+eax] - mov DWORD [16+esp],edx - test ecx,4294967280 - jz NEAR L$020enc_tail - mov eax,DWORD [edi] - mov ebx,DWORD [4+edi] -align 4 -L$021enc_loop: - mov ecx,DWORD [8+edi] - mov edx,DWORD [12+edi] - xor eax,DWORD [esi] - xor ebx,DWORD [4+esi] - xor ecx,DWORD [8+esi] - bswap eax - xor edx,DWORD [12+esi] - bswap ebx - mov edi,DWORD [36+esp] - bswap ecx - bswap edx - call __x86_Camellia_encrypt - mov esi,DWORD [24+esp] - mov edi,DWORD [28+esp] - bswap eax - bswap ebx - bswap ecx - mov DWORD [edi],eax - bswap edx - mov DWORD [4+edi],ebx - mov DWORD [8+edi],ecx - mov DWORD [12+edi],edx - mov ecx,DWORD [32+esp] - lea esi,[16+esi] - mov DWORD [24+esp],esi - lea edx,[16+edi] - mov DWORD [28+esp],edx - sub ecx,16 - test ecx,4294967280 - mov DWORD [32+esp],ecx - jnz NEAR L$021enc_loop - test ecx,15 - jnz NEAR L$020enc_tail - mov esi,DWORD [40+esp] - mov ecx,DWORD [8+edi] - mov edx,DWORD [12+edi] - mov DWORD [esi],eax - mov DWORD [4+esi],ebx - mov DWORD [8+esi],ecx - mov DWORD [12+esi],edx - mov esp,DWORD [20+esp] - popfd -L$016enc_out: - pop edi - pop esi - pop ebx - pop ebp - ret - pushfd -align 4 -L$020enc_tail: - mov eax,edi - mov edi,DWORD [28+esp] - push eax - mov ebx,16 - sub ebx,ecx - cmp edi,esi - je NEAR L$022enc_in_place -align 4 -dd 2767451785 - jmp NEAR L$023enc_skip_in_place -L$022enc_in_place: - lea edi,[ecx*1+edi] -L$023enc_skip_in_place: - mov ecx,ebx - xor eax,eax -align 4 -dd 2868115081 - pop edi - mov esi,DWORD [28+esp] - mov eax,DWORD [edi] - mov ebx,DWORD [4+edi] - mov DWORD [32+esp],16 - jmp NEAR L$021enc_loop -align 16 -L$019DECRYPT: - shl edx,6 - lea edx,[edx*1+eax] - mov DWORD [16+esp],eax - mov DWORD [36+esp],edx - cmp esi,DWORD [28+esp] - je NEAR L$024dec_in_place - mov edi,DWORD [40+esp] - mov DWORD [44+esp],edi -align 4 -L$025dec_loop: - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov ecx,DWORD [8+esi] - bswap eax - mov edx,DWORD [12+esi] - bswap ebx - mov edi,DWORD [36+esp] - bswap ecx - bswap edx - call __x86_Camellia_decrypt - mov edi,DWORD [44+esp] - mov esi,DWORD [32+esp] - bswap eax - bswap ebx - bswap ecx - xor eax,DWORD [edi] - bswap edx - xor ebx,DWORD [4+edi] - xor ecx,DWORD [8+edi] - xor edx,DWORD [12+edi] - sub esi,16 - jc NEAR L$026dec_partial - mov DWORD [32+esp],esi - mov esi,DWORD [24+esp] - mov edi,DWORD [28+esp] - mov DWORD [edi],eax - mov DWORD [4+edi],ebx - mov DWORD [8+edi],ecx - mov DWORD [12+edi],edx - mov DWORD [44+esp],esi - lea esi,[16+esi] - mov DWORD [24+esp],esi - lea edi,[16+edi] - mov DWORD [28+esp],edi - jnz NEAR L$025dec_loop - mov edi,DWORD [44+esp] -L$027dec_end: - mov esi,DWORD [40+esp] - mov eax,DWORD [edi] - mov ebx,DWORD [4+edi] - mov ecx,DWORD [8+edi] - mov edx,DWORD [12+edi] - mov DWORD [esi],eax - mov DWORD [4+esi],ebx - mov DWORD [8+esi],ecx - mov DWORD [12+esi],edx - jmp NEAR L$028dec_out -align 4 -L$026dec_partial: - lea edi,[44+esp] - mov DWORD [edi],eax - mov DWORD [4+edi],ebx - mov DWORD [8+edi],ecx - mov DWORD [12+edi],edx - lea ecx,[16+esi] - mov esi,edi - mov edi,DWORD [28+esp] -dd 2767451785 - mov edi,DWORD [24+esp] - jmp NEAR L$027dec_end -align 4 -L$024dec_in_place: -L$029dec_in_place_loop: - lea edi,[44+esp] - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov ecx,DWORD [8+esi] - mov edx,DWORD [12+esi] - mov DWORD [edi],eax - mov DWORD [4+edi],ebx - mov DWORD [8+edi],ecx - bswap eax - mov DWORD [12+edi],edx - bswap ebx - mov edi,DWORD [36+esp] - bswap ecx - bswap edx - call __x86_Camellia_decrypt - mov edi,DWORD [40+esp] - mov esi,DWORD [28+esp] - bswap eax - bswap ebx - bswap ecx - xor eax,DWORD [edi] - bswap edx - xor ebx,DWORD [4+edi] - xor ecx,DWORD [8+edi] - xor edx,DWORD [12+edi] - mov DWORD [esi],eax - mov DWORD [4+esi],ebx - mov DWORD [8+esi],ecx - mov DWORD [12+esi],edx - lea esi,[16+esi] - mov DWORD [28+esp],esi - lea esi,[44+esp] - mov eax,DWORD [esi] - mov ebx,DWORD [4+esi] - mov ecx,DWORD [8+esi] - mov edx,DWORD [12+esi] - mov DWORD [edi],eax - mov DWORD [4+edi],ebx - mov DWORD [8+edi],ecx - mov DWORD [12+edi],edx - mov esi,DWORD [24+esp] - lea esi,[16+esi] - mov DWORD [24+esp],esi - mov ecx,DWORD [32+esp] - sub ecx,16 - jc NEAR L$030dec_in_place_partial - mov DWORD [32+esp],ecx - jnz NEAR L$029dec_in_place_loop - jmp NEAR L$028dec_out -align 4 -L$030dec_in_place_partial: - mov edi,DWORD [28+esp] - lea esi,[44+esp] - lea edi,[ecx*1+edi] - lea esi,[16+ecx*1+esi] - neg ecx -dd 2767451785 -align 4 -L$028dec_out: - mov esp,DWORD [20+esp] - popfd - pop edi - pop esi - pop ebx - pop ebp - ret -db 67,97,109,101,108,108,105,97,32,102,111,114,32,120,56,54 -db 32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115 -db 115,108,46,111,114,103,62,0 diff --git a/openssl/src/crypto/camellia/gen/windows_x64/cmll-x86_64.asm b/openssl/src/crypto/camellia/gen/windows_x64/cmll-x86_64.asm deleted file mode 100644 index ac057de92..000000000 --- a/openssl/src/crypto/camellia/gen/windows_x64/cmll-x86_64.asm +++ /dev/null @@ -1,2160 +0,0 @@ -default rel -%define XMMWORD -%define YMMWORD -%define ZMMWORD -section .text code align=64 - - - -global Camellia_EncryptBlock - -ALIGN 16 -Camellia_EncryptBlock: - - mov eax,128 - sub eax,ecx - mov ecx,3 - adc ecx,0 - jmp NEAR $L$enc_rounds - - - -global Camellia_EncryptBlock_Rounds - -ALIGN 16 -$L$enc_rounds: -Camellia_EncryptBlock_Rounds: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_Camellia_EncryptBlock_Rounds: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - - - - push rbx - - push rbp - - push r13 - - push r14 - - push r15 - -$L$enc_prologue: - - - mov r13,rcx - mov r14,rdx - - shl edi,6 - lea rbp,[$L$Camellia_SBOX] - lea r15,[rdi*1+r14] - - mov r8d,DWORD[rsi] - mov r9d,DWORD[4+rsi] - mov r10d,DWORD[8+rsi] - bswap r8d - mov r11d,DWORD[12+rsi] - bswap r9d - bswap r10d - bswap r11d - - call _x86_64_Camellia_encrypt - - bswap r8d - bswap r9d - bswap r10d - mov DWORD[r13],r8d - bswap r11d - mov DWORD[4+r13],r9d - mov DWORD[8+r13],r10d - mov DWORD[12+r13],r11d - - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov rbp,QWORD[24+rsp] - - mov rbx,QWORD[32+rsp] - - lea rsp,[40+rsp] - -$L$enc_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_Camellia_EncryptBlock_Rounds: - - -ALIGN 16 -_x86_64_Camellia_encrypt: - - xor r9d,DWORD[r14] - xor r8d,DWORD[4+r14] - xor r11d,DWORD[8+r14] - xor r10d,DWORD[12+r14] -ALIGN 16 -$L$eloop: - mov ebx,DWORD[16+r14] - mov eax,DWORD[20+r14] - - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[24+r14] - mov eax,DWORD[28+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[32+r14] - mov eax,DWORD[36+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[40+r14] - mov eax,DWORD[44+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[48+r14] - mov eax,DWORD[52+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[56+r14] - mov eax,DWORD[60+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[64+r14] - mov eax,DWORD[68+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - lea r14,[64+r14] - cmp r14,r15 - mov edx,DWORD[8+r14] - mov ecx,DWORD[12+r14] - je NEAR $L$edone - - and eax,r8d - or edx,r11d - rol eax,1 - xor r10d,edx - xor r9d,eax - and ecx,r10d - or ebx,r9d - rol ecx,1 - xor r8d,ebx - xor r11d,ecx - jmp NEAR $L$eloop - -ALIGN 16 -$L$edone: - xor eax,r10d - xor ebx,r11d - xor ecx,r8d - xor edx,r9d - - mov r8d,eax - mov r9d,ebx - mov r10d,ecx - mov r11d,edx - -DB 0xf3,0xc3 - - - - -global Camellia_DecryptBlock - -ALIGN 16 -Camellia_DecryptBlock: - - mov eax,128 - sub eax,ecx - mov ecx,3 - adc ecx,0 - jmp NEAR $L$dec_rounds - - - -global Camellia_DecryptBlock_Rounds - -ALIGN 16 -$L$dec_rounds: -Camellia_DecryptBlock_Rounds: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_Camellia_DecryptBlock_Rounds: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - - - - push rbx - - push rbp - - push r13 - - push r14 - - push r15 - -$L$dec_prologue: - - - mov r13,rcx - mov r15,rdx - - shl edi,6 - lea rbp,[$L$Camellia_SBOX] - lea r14,[rdi*1+r15] - - mov r8d,DWORD[rsi] - mov r9d,DWORD[4+rsi] - mov r10d,DWORD[8+rsi] - bswap r8d - mov r11d,DWORD[12+rsi] - bswap r9d - bswap r10d - bswap r11d - - call _x86_64_Camellia_decrypt - - bswap r8d - bswap r9d - bswap r10d - mov DWORD[r13],r8d - bswap r11d - mov DWORD[4+r13],r9d - mov DWORD[8+r13],r10d - mov DWORD[12+r13],r11d - - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov rbp,QWORD[24+rsp] - - mov rbx,QWORD[32+rsp] - - lea rsp,[40+rsp] - -$L$dec_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_Camellia_DecryptBlock_Rounds: - - -ALIGN 16 -_x86_64_Camellia_decrypt: - - xor r9d,DWORD[r14] - xor r8d,DWORD[4+r14] - xor r11d,DWORD[8+r14] - xor r10d,DWORD[12+r14] -ALIGN 16 -$L$dloop: - mov ebx,DWORD[((-8))+r14] - mov eax,DWORD[((-4))+r14] - - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[((-16))+r14] - mov eax,DWORD[((-12))+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[((-24))+r14] - mov eax,DWORD[((-20))+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[((-32))+r14] - mov eax,DWORD[((-28))+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[((-40))+r14] - mov eax,DWORD[((-36))+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[((-48))+r14] - mov eax,DWORD[((-44))+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[((-56))+r14] - mov eax,DWORD[((-52))+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - lea r14,[((-64))+r14] - cmp r14,r15 - mov edx,DWORD[r14] - mov ecx,DWORD[4+r14] - je NEAR $L$ddone - - and eax,r8d - or edx,r11d - rol eax,1 - xor r10d,edx - xor r9d,eax - and ecx,r10d - or ebx,r9d - rol ecx,1 - xor r8d,ebx - xor r11d,ecx - - jmp NEAR $L$dloop - -ALIGN 16 -$L$ddone: - xor ecx,r10d - xor edx,r11d - xor eax,r8d - xor ebx,r9d - - mov r8d,ecx - mov r9d,edx - mov r10d,eax - mov r11d,ebx - -DB 0xf3,0xc3 - - -global Camellia_Ekeygen - -ALIGN 16 -Camellia_Ekeygen: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_Camellia_Ekeygen: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - - - - push rbx - - push rbp - - push r13 - - push r14 - - push r15 - -$L$key_prologue: - - mov r15d,edi - mov r13,rdx - - mov r8d,DWORD[rsi] - mov r9d,DWORD[4+rsi] - mov r10d,DWORD[8+rsi] - mov r11d,DWORD[12+rsi] - - bswap r8d - bswap r9d - bswap r10d - bswap r11d - mov DWORD[r13],r9d - mov DWORD[4+r13],r8d - mov DWORD[8+r13],r11d - mov DWORD[12+r13],r10d - cmp r15,128 - je NEAR $L$1st128 - - mov r8d,DWORD[16+rsi] - mov r9d,DWORD[20+rsi] - cmp r15,192 - je NEAR $L$1st192 - mov r10d,DWORD[24+rsi] - mov r11d,DWORD[28+rsi] - jmp NEAR $L$1st256 -$L$1st192: - mov r10d,r8d - mov r11d,r9d - not r10d - not r11d -$L$1st256: - bswap r8d - bswap r9d - bswap r10d - bswap r11d - mov DWORD[32+r13],r9d - mov DWORD[36+r13],r8d - mov DWORD[40+r13],r11d - mov DWORD[44+r13],r10d - xor r9d,DWORD[r13] - xor r8d,DWORD[4+r13] - xor r11d,DWORD[8+r13] - xor r10d,DWORD[12+r13] - -$L$1st128: - lea r14,[$L$Camellia_SIGMA] - lea rbp,[$L$Camellia_SBOX] - - mov ebx,DWORD[r14] - mov eax,DWORD[4+r14] - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[8+r14] - mov eax,DWORD[12+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[16+r14] - mov eax,DWORD[20+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - xor r9d,DWORD[r13] - xor r8d,DWORD[4+r13] - xor r11d,DWORD[8+r13] - xor r10d,DWORD[12+r13] - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[24+r14] - mov eax,DWORD[28+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[32+r14] - mov eax,DWORD[36+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - cmp r15,128 - jne NEAR $L$2nd256 - - lea r13,[128+r13] - shl r8,32 - shl r10,32 - or r8,r9 - or r10,r11 - mov rax,QWORD[((-128))+r13] - mov rbx,QWORD[((-120))+r13] - mov QWORD[((-112))+r13],r8 - mov QWORD[((-104))+r13],r10 - mov r11,rax - shl rax,15 - mov r9,rbx - shr r9,49 - shr r11,49 - or rax,r9 - shl rbx,15 - or rbx,r11 - mov QWORD[((-96))+r13],rax - mov QWORD[((-88))+r13],rbx - mov r11,r8 - shl r8,15 - mov r9,r10 - shr r9,49 - shr r11,49 - or r8,r9 - shl r10,15 - or r10,r11 - mov QWORD[((-80))+r13],r8 - mov QWORD[((-72))+r13],r10 - mov r11,r8 - shl r8,15 - mov r9,r10 - shr r9,49 - shr r11,49 - or r8,r9 - shl r10,15 - or r10,r11 - mov QWORD[((-64))+r13],r8 - mov QWORD[((-56))+r13],r10 - mov r11,rax - shl rax,30 - mov r9,rbx - shr r9,34 - shr r11,34 - or rax,r9 - shl rbx,30 - or rbx,r11 - mov QWORD[((-48))+r13],rax - mov QWORD[((-40))+r13],rbx - mov r11,r8 - shl r8,15 - mov r9,r10 - shr r9,49 - shr r11,49 - or r8,r9 - shl r10,15 - or r10,r11 - mov QWORD[((-32))+r13],r8 - mov r11,rax - shl rax,15 - mov r9,rbx - shr r9,49 - shr r11,49 - or rax,r9 - shl rbx,15 - or rbx,r11 - mov QWORD[((-24))+r13],rbx - mov r11,r8 - shl r8,15 - mov r9,r10 - shr r9,49 - shr r11,49 - or r8,r9 - shl r10,15 - or r10,r11 - mov QWORD[((-16))+r13],r8 - mov QWORD[((-8))+r13],r10 - mov r11,rax - shl rax,17 - mov r9,rbx - shr r9,47 - shr r11,47 - or rax,r9 - shl rbx,17 - or rbx,r11 - mov QWORD[r13],rax - mov QWORD[8+r13],rbx - mov r11,rax - shl rax,17 - mov r9,rbx - shr r9,47 - shr r11,47 - or rax,r9 - shl rbx,17 - or rbx,r11 - mov QWORD[16+r13],rax - mov QWORD[24+r13],rbx - mov r11,r8 - shl r8,34 - mov r9,r10 - shr r9,30 - shr r11,30 - or r8,r9 - shl r10,34 - or r10,r11 - mov QWORD[32+r13],r8 - mov QWORD[40+r13],r10 - mov r11,rax - shl rax,17 - mov r9,rbx - shr r9,47 - shr r11,47 - or rax,r9 - shl rbx,17 - or rbx,r11 - mov QWORD[48+r13],rax - mov QWORD[56+r13],rbx - mov r11,r8 - shl r8,17 - mov r9,r10 - shr r9,47 - shr r11,47 - or r8,r9 - shl r10,17 - or r10,r11 - mov QWORD[64+r13],r8 - mov QWORD[72+r13],r10 - mov eax,3 - jmp NEAR $L$done -ALIGN 16 -$L$2nd256: - mov DWORD[48+r13],r9d - mov DWORD[52+r13],r8d - mov DWORD[56+r13],r11d - mov DWORD[60+r13],r10d - xor r9d,DWORD[32+r13] - xor r8d,DWORD[36+r13] - xor r11d,DWORD[40+r13] - xor r10d,DWORD[44+r13] - xor eax,r8d - xor ebx,r9d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[40+r14] - mov eax,DWORD[44+r14] - xor ecx,edx - ror edx,8 - xor r10d,ecx - xor r11d,ecx - xor r11d,edx - xor eax,r10d - xor ebx,r11d - movzx esi,ah - movzx edi,bl - mov edx,DWORD[2052+rsi*8+rbp] - mov ecx,DWORD[rdi*8+rbp] - movzx esi,al - shr eax,16 - movzx edi,bh - xor edx,DWORD[4+rsi*8+rbp] - shr ebx,16 - xor ecx,DWORD[4+rdi*8+rbp] - movzx esi,ah - movzx edi,bl - xor edx,DWORD[rsi*8+rbp] - xor ecx,DWORD[2052+rdi*8+rbp] - movzx esi,al - movzx edi,bh - xor edx,DWORD[2048+rsi*8+rbp] - xor ecx,DWORD[2048+rdi*8+rbp] - mov ebx,DWORD[48+r14] - mov eax,DWORD[52+r14] - xor ecx,edx - ror edx,8 - xor r8d,ecx - xor r9d,ecx - xor r9d,edx - mov rax,QWORD[r13] - mov rbx,QWORD[8+r13] - mov rcx,QWORD[32+r13] - mov rdx,QWORD[40+r13] - mov r14,QWORD[48+r13] - mov r15,QWORD[56+r13] - lea r13,[128+r13] - shl r8,32 - shl r10,32 - or r8,r9 - or r10,r11 - mov QWORD[((-112))+r13],r8 - mov QWORD[((-104))+r13],r10 - mov r11,rcx - shl rcx,15 - mov r9,rdx - shr r9,49 - shr r11,49 - or rcx,r9 - shl rdx,15 - or rdx,r11 - mov QWORD[((-96))+r13],rcx - mov QWORD[((-88))+r13],rdx - mov r11,r14 - shl r14,15 - mov r9,r15 - shr r9,49 - shr r11,49 - or r14,r9 - shl r15,15 - or r15,r11 - mov QWORD[((-80))+r13],r14 - mov QWORD[((-72))+r13],r15 - mov r11,rcx - shl rcx,15 - mov r9,rdx - shr r9,49 - shr r11,49 - or rcx,r9 - shl rdx,15 - or rdx,r11 - mov QWORD[((-64))+r13],rcx - mov QWORD[((-56))+r13],rdx - mov r11,r8 - shl r8,30 - mov r9,r10 - shr r9,34 - shr r11,34 - or r8,r9 - shl r10,30 - or r10,r11 - mov QWORD[((-48))+r13],r8 - mov QWORD[((-40))+r13],r10 - mov r11,rax - shl rax,45 - mov r9,rbx - shr r9,19 - shr r11,19 - or rax,r9 - shl rbx,45 - or rbx,r11 - mov QWORD[((-32))+r13],rax - mov QWORD[((-24))+r13],rbx - mov r11,r14 - shl r14,30 - mov r9,r15 - shr r9,34 - shr r11,34 - or r14,r9 - shl r15,30 - or r15,r11 - mov QWORD[((-16))+r13],r14 - mov QWORD[((-8))+r13],r15 - mov r11,rax - shl rax,15 - mov r9,rbx - shr r9,49 - shr r11,49 - or rax,r9 - shl rbx,15 - or rbx,r11 - mov QWORD[r13],rax - mov QWORD[8+r13],rbx - mov r11,rcx - shl rcx,30 - mov r9,rdx - shr r9,34 - shr r11,34 - or rcx,r9 - shl rdx,30 - or rdx,r11 - mov QWORD[16+r13],rcx - mov QWORD[24+r13],rdx - mov r11,r8 - shl r8,30 - mov r9,r10 - shr r9,34 - shr r11,34 - or r8,r9 - shl r10,30 - or r10,r11 - mov QWORD[32+r13],r8 - mov QWORD[40+r13],r10 - mov r11,rax - shl rax,17 - mov r9,rbx - shr r9,47 - shr r11,47 - or rax,r9 - shl rbx,17 - or rbx,r11 - mov QWORD[48+r13],rax - mov QWORD[56+r13],rbx - mov r11,r14 - shl r14,32 - mov r9,r15 - shr r9,32 - shr r11,32 - or r14,r9 - shl r15,32 - or r15,r11 - mov QWORD[64+r13],r14 - mov QWORD[72+r13],r15 - mov r11,rcx - shl rcx,34 - mov r9,rdx - shr r9,30 - shr r11,30 - or rcx,r9 - shl rdx,34 - or rdx,r11 - mov QWORD[80+r13],rcx - mov QWORD[88+r13],rdx - mov r11,r14 - shl r14,17 - mov r9,r15 - shr r9,47 - shr r11,47 - or r14,r9 - shl r15,17 - or r15,r11 - mov QWORD[96+r13],r14 - mov QWORD[104+r13],r15 - mov r11,rax - shl rax,34 - mov r9,rbx - shr r9,30 - shr r11,30 - or rax,r9 - shl rbx,34 - or rbx,r11 - mov QWORD[112+r13],rax - mov QWORD[120+r13],rbx - mov r11,r8 - shl r8,51 - mov r9,r10 - shr r9,13 - shr r11,13 - or r8,r9 - shl r10,51 - or r10,r11 - mov QWORD[128+r13],r8 - mov QWORD[136+r13],r10 - mov eax,4 -$L$done: - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov rbp,QWORD[24+rsp] - - mov rbx,QWORD[32+rsp] - - lea rsp,[40+rsp] - -$L$key_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_Camellia_Ekeygen: -ALIGN 64 -$L$Camellia_SIGMA: - DD 0x3bcc908b,0xa09e667f,0x4caa73b2,0xb67ae858 - DD 0xe94f82be,0xc6ef372f,0xf1d36f1c,0x54ff53a5 - DD 0xde682d1d,0x10e527fa,0xb3e6c1fd,0xb05688c2 - DD 0,0,0,0 -$L$Camellia_SBOX: - DD 0x70707000,0x70700070 - DD 0x82828200,0x2c2c002c - DD 0x2c2c2c00,0xb3b300b3 - DD 0xececec00,0xc0c000c0 - DD 0xb3b3b300,0xe4e400e4 - DD 0x27272700,0x57570057 - DD 0xc0c0c000,0xeaea00ea - DD 0xe5e5e500,0xaeae00ae - DD 0xe4e4e400,0x23230023 - DD 0x85858500,0x6b6b006b - DD 0x57575700,0x45450045 - DD 0x35353500,0xa5a500a5 - DD 0xeaeaea00,0xeded00ed - DD 0x0c0c0c00,0x4f4f004f - DD 0xaeaeae00,0x1d1d001d - DD 0x41414100,0x92920092 - DD 0x23232300,0x86860086 - DD 0xefefef00,0xafaf00af - DD 0x6b6b6b00,0x7c7c007c - DD 0x93939300,0x1f1f001f - DD 0x45454500,0x3e3e003e - DD 0x19191900,0xdcdc00dc - DD 0xa5a5a500,0x5e5e005e - DD 0x21212100,0x0b0b000b - DD 0xededed00,0xa6a600a6 - DD 0x0e0e0e00,0x39390039 - DD 0x4f4f4f00,0xd5d500d5 - DD 0x4e4e4e00,0x5d5d005d - DD 0x1d1d1d00,0xd9d900d9 - DD 0x65656500,0x5a5a005a - DD 0x92929200,0x51510051 - DD 0xbdbdbd00,0x6c6c006c - DD 0x86868600,0x8b8b008b - DD 0xb8b8b800,0x9a9a009a - DD 0xafafaf00,0xfbfb00fb - DD 0x8f8f8f00,0xb0b000b0 - DD 0x7c7c7c00,0x74740074 - DD 0xebebeb00,0x2b2b002b - DD 0x1f1f1f00,0xf0f000f0 - DD 0xcecece00,0x84840084 - DD 0x3e3e3e00,0xdfdf00df - DD 0x30303000,0xcbcb00cb - DD 0xdcdcdc00,0x34340034 - DD 0x5f5f5f00,0x76760076 - DD 0x5e5e5e00,0x6d6d006d - DD 0xc5c5c500,0xa9a900a9 - DD 0x0b0b0b00,0xd1d100d1 - DD 0x1a1a1a00,0x04040004 - DD 0xa6a6a600,0x14140014 - DD 0xe1e1e100,0x3a3a003a - DD 0x39393900,0xdede00de - DD 0xcacaca00,0x11110011 - DD 0xd5d5d500,0x32320032 - DD 0x47474700,0x9c9c009c - DD 0x5d5d5d00,0x53530053 - DD 0x3d3d3d00,0xf2f200f2 - DD 0xd9d9d900,0xfefe00fe - DD 0x01010100,0xcfcf00cf - DD 0x5a5a5a00,0xc3c300c3 - DD 0xd6d6d600,0x7a7a007a - DD 0x51515100,0x24240024 - DD 0x56565600,0xe8e800e8 - DD 0x6c6c6c00,0x60600060 - DD 0x4d4d4d00,0x69690069 - DD 0x8b8b8b00,0xaaaa00aa - DD 0x0d0d0d00,0xa0a000a0 - DD 0x9a9a9a00,0xa1a100a1 - DD 0x66666600,0x62620062 - DD 0xfbfbfb00,0x54540054 - DD 0xcccccc00,0x1e1e001e - DD 0xb0b0b000,0xe0e000e0 - DD 0x2d2d2d00,0x64640064 - DD 0x74747400,0x10100010 - DD 0x12121200,0x00000000 - DD 0x2b2b2b00,0xa3a300a3 - DD 0x20202000,0x75750075 - DD 0xf0f0f000,0x8a8a008a - DD 0xb1b1b100,0xe6e600e6 - DD 0x84848400,0x09090009 - DD 0x99999900,0xdddd00dd - DD 0xdfdfdf00,0x87870087 - DD 0x4c4c4c00,0x83830083 - DD 0xcbcbcb00,0xcdcd00cd - DD 0xc2c2c200,0x90900090 - DD 0x34343400,0x73730073 - DD 0x7e7e7e00,0xf6f600f6 - DD 0x76767600,0x9d9d009d - DD 0x05050500,0xbfbf00bf - DD 0x6d6d6d00,0x52520052 - DD 0xb7b7b700,0xd8d800d8 - DD 0xa9a9a900,0xc8c800c8 - DD 0x31313100,0xc6c600c6 - DD 0xd1d1d100,0x81810081 - DD 0x17171700,0x6f6f006f - DD 0x04040400,0x13130013 - DD 0xd7d7d700,0x63630063 - DD 0x14141400,0xe9e900e9 - DD 0x58585800,0xa7a700a7 - DD 0x3a3a3a00,0x9f9f009f - DD 0x61616100,0xbcbc00bc - DD 0xdedede00,0x29290029 - DD 0x1b1b1b00,0xf9f900f9 - DD 0x11111100,0x2f2f002f - DD 0x1c1c1c00,0xb4b400b4 - DD 0x32323200,0x78780078 - DD 0x0f0f0f00,0x06060006 - DD 0x9c9c9c00,0xe7e700e7 - DD 0x16161600,0x71710071 - DD 0x53535300,0xd4d400d4 - DD 0x18181800,0xabab00ab - DD 0xf2f2f200,0x88880088 - DD 0x22222200,0x8d8d008d - DD 0xfefefe00,0x72720072 - DD 0x44444400,0xb9b900b9 - DD 0xcfcfcf00,0xf8f800f8 - DD 0xb2b2b200,0xacac00ac - DD 0xc3c3c300,0x36360036 - DD 0xb5b5b500,0x2a2a002a - DD 0x7a7a7a00,0x3c3c003c - DD 0x91919100,0xf1f100f1 - DD 0x24242400,0x40400040 - DD 0x08080800,0xd3d300d3 - DD 0xe8e8e800,0xbbbb00bb - DD 0xa8a8a800,0x43430043 - DD 0x60606000,0x15150015 - DD 0xfcfcfc00,0xadad00ad - DD 0x69696900,0x77770077 - DD 0x50505000,0x80800080 - DD 0xaaaaaa00,0x82820082 - DD 0xd0d0d000,0xecec00ec - DD 0xa0a0a000,0x27270027 - DD 0x7d7d7d00,0xe5e500e5 - DD 0xa1a1a100,0x85850085 - DD 0x89898900,0x35350035 - DD 0x62626200,0x0c0c000c - DD 0x97979700,0x41410041 - DD 0x54545400,0xefef00ef - DD 0x5b5b5b00,0x93930093 - DD 0x1e1e1e00,0x19190019 - DD 0x95959500,0x21210021 - DD 0xe0e0e000,0x0e0e000e - DD 0xffffff00,0x4e4e004e - DD 0x64646400,0x65650065 - DD 0xd2d2d200,0xbdbd00bd - DD 0x10101000,0xb8b800b8 - DD 0xc4c4c400,0x8f8f008f - DD 0x00000000,0xebeb00eb - DD 0x48484800,0xcece00ce - DD 0xa3a3a300,0x30300030 - DD 0xf7f7f700,0x5f5f005f - DD 0x75757500,0xc5c500c5 - DD 0xdbdbdb00,0x1a1a001a - DD 0x8a8a8a00,0xe1e100e1 - DD 0x03030300,0xcaca00ca - DD 0xe6e6e600,0x47470047 - DD 0xdadada00,0x3d3d003d - DD 0x09090900,0x01010001 - DD 0x3f3f3f00,0xd6d600d6 - DD 0xdddddd00,0x56560056 - DD 0x94949400,0x4d4d004d - DD 0x87878700,0x0d0d000d - DD 0x5c5c5c00,0x66660066 - DD 0x83838300,0xcccc00cc - DD 0x02020200,0x2d2d002d - DD 0xcdcdcd00,0x12120012 - DD 0x4a4a4a00,0x20200020 - DD 0x90909000,0xb1b100b1 - DD 0x33333300,0x99990099 - DD 0x73737300,0x4c4c004c - DD 0x67676700,0xc2c200c2 - DD 0xf6f6f600,0x7e7e007e - DD 0xf3f3f300,0x05050005 - DD 0x9d9d9d00,0xb7b700b7 - DD 0x7f7f7f00,0x31310031 - DD 0xbfbfbf00,0x17170017 - DD 0xe2e2e200,0xd7d700d7 - DD 0x52525200,0x58580058 - DD 0x9b9b9b00,0x61610061 - DD 0xd8d8d800,0x1b1b001b - DD 0x26262600,0x1c1c001c - DD 0xc8c8c800,0x0f0f000f - DD 0x37373700,0x16160016 - DD 0xc6c6c600,0x18180018 - DD 0x3b3b3b00,0x22220022 - DD 0x81818100,0x44440044 - DD 0x96969600,0xb2b200b2 - DD 0x6f6f6f00,0xb5b500b5 - DD 0x4b4b4b00,0x91910091 - DD 0x13131300,0x08080008 - DD 0xbebebe00,0xa8a800a8 - DD 0x63636300,0xfcfc00fc - DD 0x2e2e2e00,0x50500050 - DD 0xe9e9e900,0xd0d000d0 - DD 0x79797900,0x7d7d007d - DD 0xa7a7a700,0x89890089 - DD 0x8c8c8c00,0x97970097 - DD 0x9f9f9f00,0x5b5b005b - DD 0x6e6e6e00,0x95950095 - DD 0xbcbcbc00,0xffff00ff - DD 0x8e8e8e00,0xd2d200d2 - DD 0x29292900,0xc4c400c4 - DD 0xf5f5f500,0x48480048 - DD 0xf9f9f900,0xf7f700f7 - DD 0xb6b6b600,0xdbdb00db - DD 0x2f2f2f00,0x03030003 - DD 0xfdfdfd00,0xdada00da - DD 0xb4b4b400,0x3f3f003f - DD 0x59595900,0x94940094 - DD 0x78787800,0x5c5c005c - DD 0x98989800,0x02020002 - DD 0x06060600,0x4a4a004a - DD 0x6a6a6a00,0x33330033 - DD 0xe7e7e700,0x67670067 - DD 0x46464600,0xf3f300f3 - DD 0x71717100,0x7f7f007f - DD 0xbababa00,0xe2e200e2 - DD 0xd4d4d400,0x9b9b009b - DD 0x25252500,0x26260026 - DD 0xababab00,0x37370037 - DD 0x42424200,0x3b3b003b - DD 0x88888800,0x96960096 - DD 0xa2a2a200,0x4b4b004b - DD 0x8d8d8d00,0xbebe00be - DD 0xfafafa00,0x2e2e002e - DD 0x72727200,0x79790079 - DD 0x07070700,0x8c8c008c - DD 0xb9b9b900,0x6e6e006e - DD 0x55555500,0x8e8e008e - DD 0xf8f8f800,0xf5f500f5 - DD 0xeeeeee00,0xb6b600b6 - DD 0xacacac00,0xfdfd00fd - DD 0x0a0a0a00,0x59590059 - DD 0x36363600,0x98980098 - DD 0x49494900,0x6a6a006a - DD 0x2a2a2a00,0x46460046 - DD 0x68686800,0xbaba00ba - DD 0x3c3c3c00,0x25250025 - DD 0x38383800,0x42420042 - DD 0xf1f1f100,0xa2a200a2 - DD 0xa4a4a400,0xfafa00fa - DD 0x40404000,0x07070007 - DD 0x28282800,0x55550055 - DD 0xd3d3d300,0xeeee00ee - DD 0x7b7b7b00,0x0a0a000a - DD 0xbbbbbb00,0x49490049 - DD 0xc9c9c900,0x68680068 - DD 0x43434300,0x38380038 - DD 0xc1c1c100,0xa4a400a4 - DD 0x15151500,0x28280028 - DD 0xe3e3e300,0x7b7b007b - DD 0xadadad00,0xc9c900c9 - DD 0xf4f4f400,0xc1c100c1 - DD 0x77777700,0xe3e300e3 - DD 0xc7c7c700,0xf4f400f4 - DD 0x80808000,0xc7c700c7 - DD 0x9e9e9e00,0x9e9e009e - DD 0x00e0e0e0,0x38003838 - DD 0x00050505,0x41004141 - DD 0x00585858,0x16001616 - DD 0x00d9d9d9,0x76007676 - DD 0x00676767,0xd900d9d9 - DD 0x004e4e4e,0x93009393 - DD 0x00818181,0x60006060 - DD 0x00cbcbcb,0xf200f2f2 - DD 0x00c9c9c9,0x72007272 - DD 0x000b0b0b,0xc200c2c2 - DD 0x00aeaeae,0xab00abab - DD 0x006a6a6a,0x9a009a9a - DD 0x00d5d5d5,0x75007575 - DD 0x00181818,0x06000606 - DD 0x005d5d5d,0x57005757 - DD 0x00828282,0xa000a0a0 - DD 0x00464646,0x91009191 - DD 0x00dfdfdf,0xf700f7f7 - DD 0x00d6d6d6,0xb500b5b5 - DD 0x00272727,0xc900c9c9 - DD 0x008a8a8a,0xa200a2a2 - DD 0x00323232,0x8c008c8c - DD 0x004b4b4b,0xd200d2d2 - DD 0x00424242,0x90009090 - DD 0x00dbdbdb,0xf600f6f6 - DD 0x001c1c1c,0x07000707 - DD 0x009e9e9e,0xa700a7a7 - DD 0x009c9c9c,0x27002727 - DD 0x003a3a3a,0x8e008e8e - DD 0x00cacaca,0xb200b2b2 - DD 0x00252525,0x49004949 - DD 0x007b7b7b,0xde00dede - DD 0x000d0d0d,0x43004343 - DD 0x00717171,0x5c005c5c - DD 0x005f5f5f,0xd700d7d7 - DD 0x001f1f1f,0xc700c7c7 - DD 0x00f8f8f8,0x3e003e3e - DD 0x00d7d7d7,0xf500f5f5 - DD 0x003e3e3e,0x8f008f8f - DD 0x009d9d9d,0x67006767 - DD 0x007c7c7c,0x1f001f1f - DD 0x00606060,0x18001818 - DD 0x00b9b9b9,0x6e006e6e - DD 0x00bebebe,0xaf00afaf - DD 0x00bcbcbc,0x2f002f2f - DD 0x008b8b8b,0xe200e2e2 - DD 0x00161616,0x85008585 - DD 0x00343434,0x0d000d0d - DD 0x004d4d4d,0x53005353 - DD 0x00c3c3c3,0xf000f0f0 - DD 0x00727272,0x9c009c9c - DD 0x00959595,0x65006565 - DD 0x00ababab,0xea00eaea - DD 0x008e8e8e,0xa300a3a3 - DD 0x00bababa,0xae00aeae - DD 0x007a7a7a,0x9e009e9e - DD 0x00b3b3b3,0xec00ecec - DD 0x00020202,0x80008080 - DD 0x00b4b4b4,0x2d002d2d - DD 0x00adadad,0x6b006b6b - DD 0x00a2a2a2,0xa800a8a8 - DD 0x00acacac,0x2b002b2b - DD 0x00d8d8d8,0x36003636 - DD 0x009a9a9a,0xa600a6a6 - DD 0x00171717,0xc500c5c5 - DD 0x001a1a1a,0x86008686 - DD 0x00353535,0x4d004d4d - DD 0x00cccccc,0x33003333 - DD 0x00f7f7f7,0xfd00fdfd - DD 0x00999999,0x66006666 - DD 0x00616161,0x58005858 - DD 0x005a5a5a,0x96009696 - DD 0x00e8e8e8,0x3a003a3a - DD 0x00242424,0x09000909 - DD 0x00565656,0x95009595 - DD 0x00404040,0x10001010 - DD 0x00e1e1e1,0x78007878 - DD 0x00636363,0xd800d8d8 - DD 0x00090909,0x42004242 - DD 0x00333333,0xcc00cccc - DD 0x00bfbfbf,0xef00efef - DD 0x00989898,0x26002626 - DD 0x00979797,0xe500e5e5 - DD 0x00858585,0x61006161 - DD 0x00686868,0x1a001a1a - DD 0x00fcfcfc,0x3f003f3f - DD 0x00ececec,0x3b003b3b - DD 0x000a0a0a,0x82008282 - DD 0x00dadada,0xb600b6b6 - DD 0x006f6f6f,0xdb00dbdb - DD 0x00535353,0xd400d4d4 - DD 0x00626262,0x98009898 - DD 0x00a3a3a3,0xe800e8e8 - DD 0x002e2e2e,0x8b008b8b - DD 0x00080808,0x02000202 - DD 0x00afafaf,0xeb00ebeb - DD 0x00282828,0x0a000a0a - DD 0x00b0b0b0,0x2c002c2c - DD 0x00747474,0x1d001d1d - DD 0x00c2c2c2,0xb000b0b0 - DD 0x00bdbdbd,0x6f006f6f - DD 0x00363636,0x8d008d8d - DD 0x00222222,0x88008888 - DD 0x00383838,0x0e000e0e - DD 0x00646464,0x19001919 - DD 0x001e1e1e,0x87008787 - DD 0x00393939,0x4e004e4e - DD 0x002c2c2c,0x0b000b0b - DD 0x00a6a6a6,0xa900a9a9 - DD 0x00303030,0x0c000c0c - DD 0x00e5e5e5,0x79007979 - DD 0x00444444,0x11001111 - DD 0x00fdfdfd,0x7f007f7f - DD 0x00888888,0x22002222 - DD 0x009f9f9f,0xe700e7e7 - DD 0x00656565,0x59005959 - DD 0x00878787,0xe100e1e1 - DD 0x006b6b6b,0xda00dada - DD 0x00f4f4f4,0x3d003d3d - DD 0x00232323,0xc800c8c8 - DD 0x00484848,0x12001212 - DD 0x00101010,0x04000404 - DD 0x00d1d1d1,0x74007474 - DD 0x00515151,0x54005454 - DD 0x00c0c0c0,0x30003030 - DD 0x00f9f9f9,0x7e007e7e - DD 0x00d2d2d2,0xb400b4b4 - DD 0x00a0a0a0,0x28002828 - DD 0x00555555,0x55005555 - DD 0x00a1a1a1,0x68006868 - DD 0x00414141,0x50005050 - DD 0x00fafafa,0xbe00bebe - DD 0x00434343,0xd000d0d0 - DD 0x00131313,0xc400c4c4 - DD 0x00c4c4c4,0x31003131 - DD 0x002f2f2f,0xcb00cbcb - DD 0x00a8a8a8,0x2a002a2a - DD 0x00b6b6b6,0xad00adad - DD 0x003c3c3c,0x0f000f0f - DD 0x002b2b2b,0xca00caca - DD 0x00c1c1c1,0x70007070 - DD 0x00ffffff,0xff00ffff - DD 0x00c8c8c8,0x32003232 - DD 0x00a5a5a5,0x69006969 - DD 0x00202020,0x08000808 - DD 0x00898989,0x62006262 - DD 0x00000000,0x00000000 - DD 0x00909090,0x24002424 - DD 0x00474747,0xd100d1d1 - DD 0x00efefef,0xfb00fbfb - DD 0x00eaeaea,0xba00baba - DD 0x00b7b7b7,0xed00eded - DD 0x00151515,0x45004545 - DD 0x00060606,0x81008181 - DD 0x00cdcdcd,0x73007373 - DD 0x00b5b5b5,0x6d006d6d - DD 0x00121212,0x84008484 - DD 0x007e7e7e,0x9f009f9f - DD 0x00bbbbbb,0xee00eeee - DD 0x00292929,0x4a004a4a - DD 0x000f0f0f,0xc300c3c3 - DD 0x00b8b8b8,0x2e002e2e - DD 0x00070707,0xc100c1c1 - DD 0x00040404,0x01000101 - DD 0x009b9b9b,0xe600e6e6 - DD 0x00949494,0x25002525 - DD 0x00212121,0x48004848 - DD 0x00666666,0x99009999 - DD 0x00e6e6e6,0xb900b9b9 - DD 0x00cecece,0xb300b3b3 - DD 0x00ededed,0x7b007b7b - DD 0x00e7e7e7,0xf900f9f9 - DD 0x003b3b3b,0xce00cece - DD 0x00fefefe,0xbf00bfbf - DD 0x007f7f7f,0xdf00dfdf - DD 0x00c5c5c5,0x71007171 - DD 0x00a4a4a4,0x29002929 - DD 0x00373737,0xcd00cdcd - DD 0x00b1b1b1,0x6c006c6c - DD 0x004c4c4c,0x13001313 - DD 0x00919191,0x64006464 - DD 0x006e6e6e,0x9b009b9b - DD 0x008d8d8d,0x63006363 - DD 0x00767676,0x9d009d9d - DD 0x00030303,0xc000c0c0 - DD 0x002d2d2d,0x4b004b4b - DD 0x00dedede,0xb700b7b7 - DD 0x00969696,0xa500a5a5 - DD 0x00262626,0x89008989 - DD 0x007d7d7d,0x5f005f5f - DD 0x00c6c6c6,0xb100b1b1 - DD 0x005c5c5c,0x17001717 - DD 0x00d3d3d3,0xf400f4f4 - DD 0x00f2f2f2,0xbc00bcbc - DD 0x004f4f4f,0xd300d3d3 - DD 0x00191919,0x46004646 - DD 0x003f3f3f,0xcf00cfcf - DD 0x00dcdcdc,0x37003737 - DD 0x00797979,0x5e005e5e - DD 0x001d1d1d,0x47004747 - DD 0x00525252,0x94009494 - DD 0x00ebebeb,0xfa00fafa - DD 0x00f3f3f3,0xfc00fcfc - DD 0x006d6d6d,0x5b005b5b - DD 0x005e5e5e,0x97009797 - DD 0x00fbfbfb,0xfe00fefe - DD 0x00696969,0x5a005a5a - DD 0x00b2b2b2,0xac00acac - DD 0x00f0f0f0,0x3c003c3c - DD 0x00313131,0x4c004c4c - DD 0x000c0c0c,0x03000303 - DD 0x00d4d4d4,0x35003535 - DD 0x00cfcfcf,0xf300f3f3 - DD 0x008c8c8c,0x23002323 - DD 0x00e2e2e2,0xb800b8b8 - DD 0x00757575,0x5d005d5d - DD 0x00a9a9a9,0x6a006a6a - DD 0x004a4a4a,0x92009292 - DD 0x00575757,0xd500d5d5 - DD 0x00848484,0x21002121 - DD 0x00111111,0x44004444 - DD 0x00454545,0x51005151 - DD 0x001b1b1b,0xc600c6c6 - DD 0x00f5f5f5,0x7d007d7d - DD 0x00e4e4e4,0x39003939 - DD 0x000e0e0e,0x83008383 - DD 0x00737373,0xdc00dcdc - DD 0x00aaaaaa,0xaa00aaaa - DD 0x00f1f1f1,0x7c007c7c - DD 0x00dddddd,0x77007777 - DD 0x00595959,0x56005656 - DD 0x00141414,0x05000505 - DD 0x006c6c6c,0x1b001b1b - DD 0x00929292,0xa400a4a4 - DD 0x00545454,0x15001515 - DD 0x00d0d0d0,0x34003434 - DD 0x00787878,0x1e001e1e - DD 0x00707070,0x1c001c1c - DD 0x00e3e3e3,0xf800f8f8 - DD 0x00494949,0x52005252 - DD 0x00808080,0x20002020 - DD 0x00505050,0x14001414 - DD 0x00a7a7a7,0xe900e9e9 - DD 0x00f6f6f6,0xbd00bdbd - DD 0x00777777,0xdd00dddd - DD 0x00939393,0xe400e4e4 - DD 0x00868686,0xa100a1a1 - DD 0x00838383,0xe000e0e0 - DD 0x002a2a2a,0x8a008a8a - DD 0x00c7c7c7,0xf100f1f1 - DD 0x005b5b5b,0xd600d6d6 - DD 0x00e9e9e9,0x7a007a7a - DD 0x00eeeeee,0xbb00bbbb - DD 0x008f8f8f,0xe300e3e3 - DD 0x00010101,0x40004040 - DD 0x003d3d3d,0x4f004f4f -global Camellia_cbc_encrypt - -ALIGN 16 -Camellia_cbc_encrypt: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_Camellia_cbc_encrypt: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 - mov r8,QWORD[40+rsp] - mov r9,QWORD[48+rsp] - - - -DB 243,15,30,250 - cmp rdx,0 - je NEAR $L$cbc_abort - push rbx - - push rbp - - push r12 - - push r13 - - push r14 - - push r15 - -$L$cbc_prologue: - - mov rbp,rsp - - sub rsp,64 - and rsp,-64 - - - - lea r10,[((-64-63))+rcx] - sub r10,rsp - neg r10 - and r10,0x3C0 - sub rsp,r10 - - - mov r12,rdi - mov r13,rsi - mov rbx,r8 - mov r14,rcx - mov r15d,DWORD[272+rcx] - - mov QWORD[40+rsp],r8 - mov QWORD[48+rsp],rbp - - -$L$cbc_body: - lea rbp,[$L$Camellia_SBOX] - - mov ecx,32 -ALIGN 4 -$L$cbc_prefetch_sbox: - mov rax,QWORD[rbp] - mov rsi,QWORD[32+rbp] - mov rdi,QWORD[64+rbp] - mov r11,QWORD[96+rbp] - lea rbp,[128+rbp] - loop $L$cbc_prefetch_sbox - sub rbp,4096 - shl r15,6 - mov rcx,rdx - lea r15,[r15*1+r14] - - cmp r9d,0 - je NEAR $L$CBC_DECRYPT - - and rdx,-16 - and rcx,15 - lea rdx,[rdx*1+r12] - mov QWORD[rsp],r14 - mov QWORD[8+rsp],rdx - mov QWORD[16+rsp],rcx - - cmp rdx,r12 - mov r8d,DWORD[rbx] - mov r9d,DWORD[4+rbx] - mov r10d,DWORD[8+rbx] - mov r11d,DWORD[12+rbx] - je NEAR $L$cbc_enc_tail - jmp NEAR $L$cbc_eloop - -ALIGN 16 -$L$cbc_eloop: - xor r8d,DWORD[r12] - xor r9d,DWORD[4+r12] - xor r10d,DWORD[8+r12] - bswap r8d - xor r11d,DWORD[12+r12] - bswap r9d - bswap r10d - bswap r11d - - call _x86_64_Camellia_encrypt - - mov r14,QWORD[rsp] - bswap r8d - mov rdx,QWORD[8+rsp] - bswap r9d - mov rcx,QWORD[16+rsp] - bswap r10d - mov DWORD[r13],r8d - bswap r11d - mov DWORD[4+r13],r9d - mov DWORD[8+r13],r10d - lea r12,[16+r12] - mov DWORD[12+r13],r11d - cmp r12,rdx - lea r13,[16+r13] - jne NEAR $L$cbc_eloop - - cmp rcx,0 - jne NEAR $L$cbc_enc_tail - - mov r13,QWORD[40+rsp] - mov DWORD[r13],r8d - mov DWORD[4+r13],r9d - mov DWORD[8+r13],r10d - mov DWORD[12+r13],r11d - jmp NEAR $L$cbc_done - -ALIGN 16 -$L$cbc_enc_tail: - xor rax,rax - mov QWORD[((0+24))+rsp],rax - mov QWORD[((8+24))+rsp],rax - mov QWORD[16+rsp],rax - -$L$cbc_enc_pushf: - pushfq - cld - mov rsi,r12 - lea rdi,[((8+24))+rsp] - DD 0x9066A4F3 - popfq -$L$cbc_enc_popf: - - lea r12,[24+rsp] - lea rax,[((16+24))+rsp] - mov QWORD[8+rsp],rax - jmp NEAR $L$cbc_eloop - -ALIGN 16 -$L$CBC_DECRYPT: - xchg r15,r14 - add rdx,15 - and rcx,15 - and rdx,-16 - mov QWORD[rsp],r14 - lea rdx,[rdx*1+r12] - mov QWORD[8+rsp],rdx - mov QWORD[16+rsp],rcx - - mov rax,QWORD[rbx] - mov rbx,QWORD[8+rbx] - jmp NEAR $L$cbc_dloop -ALIGN 16 -$L$cbc_dloop: - mov r8d,DWORD[r12] - mov r9d,DWORD[4+r12] - mov r10d,DWORD[8+r12] - bswap r8d - mov r11d,DWORD[12+r12] - bswap r9d - mov QWORD[((0+24))+rsp],rax - bswap r10d - mov QWORD[((8+24))+rsp],rbx - bswap r11d - - call _x86_64_Camellia_decrypt - - mov r14,QWORD[rsp] - mov rdx,QWORD[8+rsp] - mov rcx,QWORD[16+rsp] - - bswap r8d - mov rax,QWORD[r12] - bswap r9d - mov rbx,QWORD[8+r12] - bswap r10d - xor r8d,DWORD[((0+24))+rsp] - bswap r11d - xor r9d,DWORD[((4+24))+rsp] - xor r10d,DWORD[((8+24))+rsp] - lea r12,[16+r12] - xor r11d,DWORD[((12+24))+rsp] - cmp r12,rdx - je NEAR $L$cbc_ddone - - mov DWORD[r13],r8d - mov DWORD[4+r13],r9d - mov DWORD[8+r13],r10d - mov DWORD[12+r13],r11d - - lea r13,[16+r13] - jmp NEAR $L$cbc_dloop - -ALIGN 16 -$L$cbc_ddone: - mov rdx,QWORD[40+rsp] - cmp rcx,0 - jne NEAR $L$cbc_dec_tail - - mov DWORD[r13],r8d - mov DWORD[4+r13],r9d - mov DWORD[8+r13],r10d - mov DWORD[12+r13],r11d - - mov QWORD[rdx],rax - mov QWORD[8+rdx],rbx - jmp NEAR $L$cbc_done -ALIGN 16 -$L$cbc_dec_tail: - mov DWORD[((0+24))+rsp],r8d - mov DWORD[((4+24))+rsp],r9d - mov DWORD[((8+24))+rsp],r10d - mov DWORD[((12+24))+rsp],r11d - -$L$cbc_dec_pushf: - pushfq - cld - lea rsi,[((8+24))+rsp] - lea rdi,[r13] - DD 0x9066A4F3 - popfq -$L$cbc_dec_popf: - - mov QWORD[rdx],rax - mov QWORD[8+rdx],rbx - jmp NEAR $L$cbc_done - -ALIGN 16 -$L$cbc_done: - mov rcx,QWORD[48+rsp] - - mov r15,QWORD[rcx] - - mov r14,QWORD[8+rcx] - - mov r13,QWORD[16+rcx] - - mov r12,QWORD[24+rcx] - - mov rbp,QWORD[32+rcx] - - mov rbx,QWORD[40+rcx] - - lea rsp,[48+rcx] - -$L$cbc_abort: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - DB 0F3h,0C3h ;repret - -$L$SEH_end_Camellia_cbc_encrypt: - -DB 67,97,109,101,108,108,105,97,32,102,111,114,32,120,56,54 -DB 95,54,52,32,98,121,32,60,97,112,112,114,111,64,111,112 -DB 101,110,115,115,108,46,111,114,103,62,0 -EXTERN __imp_RtlVirtualUnwind - -ALIGN 16 -common_se_handler: - push rsi - push rdi - push rbx - push rbp - push r12 - push r13 - push r14 - push r15 - pushfq - lea rsp,[((-64))+rsp] - - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] - - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] - - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 - jb NEAR $L$in_prologue - - mov rax,QWORD[152+r8] - - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 - jae NEAR $L$in_prologue - - lea rax,[40+rax] - mov rbx,QWORD[((-8))+rax] - mov rbp,QWORD[((-16))+rax] - mov r13,QWORD[((-24))+rax] - mov r14,QWORD[((-32))+rax] - mov r15,QWORD[((-40))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 - -$L$in_prologue: - mov rdi,QWORD[8+rax] - mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi - - jmp NEAR $L$common_seh_exit - - - -ALIGN 16 -cbc_se_handler: - push rsi - push rdi - push rbx - push rbp - push r12 - push r13 - push r14 - push r15 - pushfq - lea rsp,[((-64))+rsp] - - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] - - lea r10,[$L$cbc_prologue] - cmp rbx,r10 - jb NEAR $L$in_cbc_prologue - - lea r10,[$L$cbc_body] - cmp rbx,r10 - jb NEAR $L$in_cbc_frame_setup - - mov rax,QWORD[152+r8] - - lea r10,[$L$cbc_abort] - cmp rbx,r10 - jae NEAR $L$in_cbc_prologue - - - lea r10,[$L$cbc_enc_pushf] - cmp rbx,r10 - jbe NEAR $L$in_cbc_no_flag - lea rax,[8+rax] - lea r10,[$L$cbc_enc_popf] - cmp rbx,r10 - jb NEAR $L$in_cbc_no_flag - lea rax,[((-8))+rax] - lea r10,[$L$cbc_dec_pushf] - cmp rbx,r10 - jbe NEAR $L$in_cbc_no_flag - lea rax,[8+rax] - lea r10,[$L$cbc_dec_popf] - cmp rbx,r10 - jb NEAR $L$in_cbc_no_flag - lea rax,[((-8))+rax] - -$L$in_cbc_no_flag: - mov rax,QWORD[48+rax] - lea rax,[48+rax] - -$L$in_cbc_frame_setup: - mov rbx,QWORD[((-8))+rax] - mov rbp,QWORD[((-16))+rax] - mov r12,QWORD[((-24))+rax] - mov r13,QWORD[((-32))+rax] - mov r14,QWORD[((-40))+rax] - mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 - -$L$in_cbc_prologue: - mov rdi,QWORD[8+rax] - mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi - -ALIGN 4 -$L$common_seh_exit: - - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc - - mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx - call QWORD[__imp_RtlVirtualUnwind] - - mov eax,1 - lea rsp,[64+rsp] - popfq - pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx - pop rdi - pop rsi - DB 0F3h,0C3h ;repret - - -section .pdata rdata align=4 -ALIGN 4 - DD $L$SEH_begin_Camellia_EncryptBlock_Rounds wrt ..imagebase - DD $L$SEH_end_Camellia_EncryptBlock_Rounds wrt ..imagebase - DD $L$SEH_info_Camellia_EncryptBlock_Rounds wrt ..imagebase - - DD $L$SEH_begin_Camellia_DecryptBlock_Rounds wrt ..imagebase - DD $L$SEH_end_Camellia_DecryptBlock_Rounds wrt ..imagebase - DD $L$SEH_info_Camellia_DecryptBlock_Rounds wrt ..imagebase - - DD $L$SEH_begin_Camellia_Ekeygen wrt ..imagebase - DD $L$SEH_end_Camellia_Ekeygen wrt ..imagebase - DD $L$SEH_info_Camellia_Ekeygen wrt ..imagebase - - DD $L$SEH_begin_Camellia_cbc_encrypt wrt ..imagebase - DD $L$SEH_end_Camellia_cbc_encrypt wrt ..imagebase - DD $L$SEH_info_Camellia_cbc_encrypt wrt ..imagebase - -section .xdata rdata align=8 -ALIGN 8 -$L$SEH_info_Camellia_EncryptBlock_Rounds: -DB 9,0,0,0 - DD common_se_handler wrt ..imagebase - DD $L$enc_prologue wrt ..imagebase,$L$enc_epilogue wrt ..imagebase -$L$SEH_info_Camellia_DecryptBlock_Rounds: -DB 9,0,0,0 - DD common_se_handler wrt ..imagebase - DD $L$dec_prologue wrt ..imagebase,$L$dec_epilogue wrt ..imagebase -$L$SEH_info_Camellia_Ekeygen: -DB 9,0,0,0 - DD common_se_handler wrt ..imagebase - DD $L$key_prologue wrt ..imagebase,$L$key_epilogue wrt ..imagebase -$L$SEH_info_Camellia_cbc_encrypt: -DB 9,0,0,0 - DD cbc_se_handler wrt ..imagebase diff --git a/openssl/src/crypto/cast/c_cfb64.c b/openssl/src/crypto/cast/c_cfb64.c deleted file mode 100644 index 97081734e..000000000 --- a/openssl/src/crypto/cast/c_cfb64.c +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * CAST low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "cast_local.h" - -/* - * The input and output encrypted as though 64bit cfb mode is being used. - * The extra state information to record how much of the 64bit block we have - * used is contained in *num; - */ - -void CAST_cfb64_encrypt(const unsigned char *in, unsigned char *out, - long length, const CAST_KEY *schedule, - unsigned char *ivec, int *num, int enc) -{ - register CAST_LONG v0, v1, t; - register int n = *num; - register long l = length; - CAST_LONG ti[2]; - unsigned char *iv, c, cc; - - iv = ivec; - if (enc) { - while (l--) { - if (n == 0) { - n2l(iv, v0); - ti[0] = v0; - n2l(iv, v1); - ti[1] = v1; - CAST_encrypt((CAST_LONG *)ti, schedule); - iv = ivec; - t = ti[0]; - l2n(t, iv); - t = ti[1]; - l2n(t, iv); - iv = ivec; - } - c = *(in++) ^ iv[n]; - *(out++) = c; - iv[n] = c; - n = (n + 1) & 0x07; - } - } else { - while (l--) { - if (n == 0) { - n2l(iv, v0); - ti[0] = v0; - n2l(iv, v1); - ti[1] = v1; - CAST_encrypt((CAST_LONG *)ti, schedule); - iv = ivec; - t = ti[0]; - l2n(t, iv); - t = ti[1]; - l2n(t, iv); - iv = ivec; - } - cc = *(in++); - c = iv[n]; - iv[n] = cc; - *(out++) = c ^ cc; - n = (n + 1) & 0x07; - } - } - v0 = v1 = ti[0] = ti[1] = t = c = cc = 0; - *num = n; -} diff --git a/openssl/src/crypto/cast/c_ecb.c b/openssl/src/crypto/cast/c_ecb.c deleted file mode 100644 index a0ab660f8..000000000 --- a/openssl/src/crypto/cast/c_ecb.c +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * CAST low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "cast_local.h" -#include - -void CAST_ecb_encrypt(const unsigned char *in, unsigned char *out, - const CAST_KEY *ks, int enc) -{ - CAST_LONG l, d[2]; - - n2l(in, l); - d[0] = l; - n2l(in, l); - d[1] = l; - if (enc) - CAST_encrypt(d, ks); - else - CAST_decrypt(d, ks); - l = d[0]; - l2n(l, out); - l = d[1]; - l2n(l, out); - l = d[0] = d[1] = 0; -} diff --git a/openssl/src/crypto/cast/c_enc.c b/openssl/src/crypto/cast/c_enc.c deleted file mode 100644 index 4ed945a50..000000000 --- a/openssl/src/crypto/cast/c_enc.c +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * CAST low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "cast_local.h" - -void CAST_encrypt(CAST_LONG *data, const CAST_KEY *key) -{ - CAST_LONG l, r, t; - const CAST_LONG *k; - - k = &(key->data[0]); - l = data[0]; - r = data[1]; - - E_CAST(0, k, l, r, +, ^, -); - E_CAST(1, k, r, l, ^, -, +); - E_CAST(2, k, l, r, -, +, ^); - E_CAST(3, k, r, l, +, ^, -); - E_CAST(4, k, l, r, ^, -, +); - E_CAST(5, k, r, l, -, +, ^); - E_CAST(6, k, l, r, +, ^, -); - E_CAST(7, k, r, l, ^, -, +); - E_CAST(8, k, l, r, -, +, ^); - E_CAST(9, k, r, l, +, ^, -); - E_CAST(10, k, l, r, ^, -, +); - E_CAST(11, k, r, l, -, +, ^); - if (!key->short_key) { - E_CAST(12, k, l, r, +, ^, -); - E_CAST(13, k, r, l, ^, -, +); - E_CAST(14, k, l, r, -, +, ^); - E_CAST(15, k, r, l, +, ^, -); - } - - data[1] = l & 0xffffffffL; - data[0] = r & 0xffffffffL; -} - -void CAST_decrypt(CAST_LONG *data, const CAST_KEY *key) -{ - CAST_LONG l, r, t; - const CAST_LONG *k; - - k = &(key->data[0]); - l = data[0]; - r = data[1]; - - if (!key->short_key) { - E_CAST(15, k, l, r, +, ^, -); - E_CAST(14, k, r, l, -, +, ^); - E_CAST(13, k, l, r, ^, -, +); - E_CAST(12, k, r, l, +, ^, -); - } - E_CAST(11, k, l, r, -, +, ^); - E_CAST(10, k, r, l, ^, -, +); - E_CAST(9, k, l, r, +, ^, -); - E_CAST(8, k, r, l, -, +, ^); - E_CAST(7, k, l, r, ^, -, +); - E_CAST(6, k, r, l, +, ^, -); - E_CAST(5, k, l, r, -, +, ^); - E_CAST(4, k, r, l, ^, -, +); - E_CAST(3, k, l, r, +, ^, -); - E_CAST(2, k, r, l, -, +, ^); - E_CAST(1, k, l, r, ^, -, +); - E_CAST(0, k, r, l, +, ^, -); - - data[1] = l & 0xffffffffL; - data[0] = r & 0xffffffffL; -} - -void CAST_cbc_encrypt(const unsigned char *in, unsigned char *out, - long length, const CAST_KEY *ks, unsigned char *iv, - int enc) -{ - register CAST_LONG tin0, tin1; - register CAST_LONG tout0, tout1, xor0, xor1; - register long l = length; - CAST_LONG tin[2]; - - if (enc) { - n2l(iv, tout0); - n2l(iv, tout1); - iv -= 8; - for (l -= 8; l >= 0; l -= 8) { - n2l(in, tin0); - n2l(in, tin1); - tin0 ^= tout0; - tin1 ^= tout1; - tin[0] = tin0; - tin[1] = tin1; - CAST_encrypt(tin, ks); - tout0 = tin[0]; - tout1 = tin[1]; - l2n(tout0, out); - l2n(tout1, out); - } - if (l != -8) { - n2ln(in, tin0, tin1, l + 8); - tin0 ^= tout0; - tin1 ^= tout1; - tin[0] = tin0; - tin[1] = tin1; - CAST_encrypt(tin, ks); - tout0 = tin[0]; - tout1 = tin[1]; - l2n(tout0, out); - l2n(tout1, out); - } - l2n(tout0, iv); - l2n(tout1, iv); - } else { - n2l(iv, xor0); - n2l(iv, xor1); - iv -= 8; - for (l -= 8; l >= 0; l -= 8) { - n2l(in, tin0); - n2l(in, tin1); - tin[0] = tin0; - tin[1] = tin1; - CAST_decrypt(tin, ks); - tout0 = tin[0] ^ xor0; - tout1 = tin[1] ^ xor1; - l2n(tout0, out); - l2n(tout1, out); - xor0 = tin0; - xor1 = tin1; - } - if (l != -8) { - n2l(in, tin0); - n2l(in, tin1); - tin[0] = tin0; - tin[1] = tin1; - CAST_decrypt(tin, ks); - tout0 = tin[0] ^ xor0; - tout1 = tin[1] ^ xor1; - l2nn(tout0, tout1, out, l + 8); - xor0 = tin0; - xor1 = tin1; - } - l2n(xor0, iv); - l2n(xor1, iv); - } - tin0 = tin1 = tout0 = tout1 = xor0 = xor1 = 0; - tin[0] = tin[1] = 0; -} diff --git a/openssl/src/crypto/cast/c_ofb64.c b/openssl/src/crypto/cast/c_ofb64.c deleted file mode 100644 index d4aa10ff6..000000000 --- a/openssl/src/crypto/cast/c_ofb64.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * CAST low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "cast_local.h" - -/* - * The input and output encrypted as though 64bit ofb mode is being used. - * The extra state information to record how much of the 64bit block we have - * used is contained in *num; - */ -void CAST_ofb64_encrypt(const unsigned char *in, unsigned char *out, - long length, const CAST_KEY *schedule, - unsigned char *ivec, int *num) -{ - register CAST_LONG v0, v1, t; - register int n = *num; - register long l = length; - unsigned char d[8]; - register char *dp; - CAST_LONG ti[2]; - unsigned char *iv; - int save = 0; - - iv = ivec; - n2l(iv, v0); - n2l(iv, v1); - ti[0] = v0; - ti[1] = v1; - dp = (char *)d; - l2n(v0, dp); - l2n(v1, dp); - while (l--) { - if (n == 0) { - CAST_encrypt((CAST_LONG *)ti, schedule); - dp = (char *)d; - t = ti[0]; - l2n(t, dp); - t = ti[1]; - l2n(t, dp); - save++; - } - *(out++) = *(in++) ^ d[n]; - n = (n + 1) & 0x07; - } - if (save) { - v0 = ti[0]; - v1 = ti[1]; - iv = ivec; - l2n(v0, iv); - l2n(v1, iv); - } - t = v0 = v1 = ti[0] = ti[1] = 0; - *num = n; -} diff --git a/openssl/src/crypto/cast/c_skey.c b/openssl/src/crypto/cast/c_skey.c deleted file mode 100644 index 030e20ea3..000000000 --- a/openssl/src/crypto/cast/c_skey.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * CAST low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "cast_local.h" -#include "cast_s.h" - -#define CAST_exp(l,A,a,n) \ - A[n/4]=l; \ - a[n+3]=(l )&0xff; \ - a[n+2]=(l>> 8)&0xff; \ - a[n+1]=(l>>16)&0xff; \ - a[n+0]=(l>>24)&0xff; - -#define S4 CAST_S_table4 -#define S5 CAST_S_table5 -#define S6 CAST_S_table6 -#define S7 CAST_S_table7 - -void CAST_set_key(CAST_KEY *key, int len, const unsigned char *data) -{ - CAST_LONG x[16]; - CAST_LONG z[16]; - CAST_LONG k[32]; - CAST_LONG X[4], Z[4]; - CAST_LONG l, *K; - int i; - - for (i = 0; i < 16; i++) - x[i] = 0; - if (len > 16) - len = 16; - for (i = 0; i < len; i++) - x[i] = data[i]; - if (len <= 10) - key->short_key = 1; - else - key->short_key = 0; - - K = &k[0]; - X[0] = ((x[0] << 24) | (x[1] << 16) | (x[2] << 8) | x[3]) & 0xffffffffL; - X[1] = ((x[4] << 24) | (x[5] << 16) | (x[6] << 8) | x[7]) & 0xffffffffL; - X[2] = ((x[8] << 24) | (x[9] << 16) | (x[10] << 8) | x[11]) & 0xffffffffL; - X[3] = - ((x[12] << 24) | (x[13] << 16) | (x[14] << 8) | x[15]) & 0xffffffffL; - - for (;;) { - l = X[0] ^ S4[x[13]] ^ S5[x[15]] ^ S6[x[12]] ^ S7[x[14]] ^ S6[x[8]]; - CAST_exp(l, Z, z, 0); - l = X[2] ^ S4[z[0]] ^ S5[z[2]] ^ S6[z[1]] ^ S7[z[3]] ^ S7[x[10]]; - CAST_exp(l, Z, z, 4); - l = X[3] ^ S4[z[7]] ^ S5[z[6]] ^ S6[z[5]] ^ S7[z[4]] ^ S4[x[9]]; - CAST_exp(l, Z, z, 8); - l = X[1] ^ S4[z[10]] ^ S5[z[9]] ^ S6[z[11]] ^ S7[z[8]] ^ S5[x[11]]; - CAST_exp(l, Z, z, 12); - - K[0] = S4[z[8]] ^ S5[z[9]] ^ S6[z[7]] ^ S7[z[6]] ^ S4[z[2]]; - K[1] = S4[z[10]] ^ S5[z[11]] ^ S6[z[5]] ^ S7[z[4]] ^ S5[z[6]]; - K[2] = S4[z[12]] ^ S5[z[13]] ^ S6[z[3]] ^ S7[z[2]] ^ S6[z[9]]; - K[3] = S4[z[14]] ^ S5[z[15]] ^ S6[z[1]] ^ S7[z[0]] ^ S7[z[12]]; - - l = Z[2] ^ S4[z[5]] ^ S5[z[7]] ^ S6[z[4]] ^ S7[z[6]] ^ S6[z[0]]; - CAST_exp(l, X, x, 0); - l = Z[0] ^ S4[x[0]] ^ S5[x[2]] ^ S6[x[1]] ^ S7[x[3]] ^ S7[z[2]]; - CAST_exp(l, X, x, 4); - l = Z[1] ^ S4[x[7]] ^ S5[x[6]] ^ S6[x[5]] ^ S7[x[4]] ^ S4[z[1]]; - CAST_exp(l, X, x, 8); - l = Z[3] ^ S4[x[10]] ^ S5[x[9]] ^ S6[x[11]] ^ S7[x[8]] ^ S5[z[3]]; - CAST_exp(l, X, x, 12); - - K[4] = S4[x[3]] ^ S5[x[2]] ^ S6[x[12]] ^ S7[x[13]] ^ S4[x[8]]; - K[5] = S4[x[1]] ^ S5[x[0]] ^ S6[x[14]] ^ S7[x[15]] ^ S5[x[13]]; - K[6] = S4[x[7]] ^ S5[x[6]] ^ S6[x[8]] ^ S7[x[9]] ^ S6[x[3]]; - K[7] = S4[x[5]] ^ S5[x[4]] ^ S6[x[10]] ^ S7[x[11]] ^ S7[x[7]]; - - l = X[0] ^ S4[x[13]] ^ S5[x[15]] ^ S6[x[12]] ^ S7[x[14]] ^ S6[x[8]]; - CAST_exp(l, Z, z, 0); - l = X[2] ^ S4[z[0]] ^ S5[z[2]] ^ S6[z[1]] ^ S7[z[3]] ^ S7[x[10]]; - CAST_exp(l, Z, z, 4); - l = X[3] ^ S4[z[7]] ^ S5[z[6]] ^ S6[z[5]] ^ S7[z[4]] ^ S4[x[9]]; - CAST_exp(l, Z, z, 8); - l = X[1] ^ S4[z[10]] ^ S5[z[9]] ^ S6[z[11]] ^ S7[z[8]] ^ S5[x[11]]; - CAST_exp(l, Z, z, 12); - - K[8] = S4[z[3]] ^ S5[z[2]] ^ S6[z[12]] ^ S7[z[13]] ^ S4[z[9]]; - K[9] = S4[z[1]] ^ S5[z[0]] ^ S6[z[14]] ^ S7[z[15]] ^ S5[z[12]]; - K[10] = S4[z[7]] ^ S5[z[6]] ^ S6[z[8]] ^ S7[z[9]] ^ S6[z[2]]; - K[11] = S4[z[5]] ^ S5[z[4]] ^ S6[z[10]] ^ S7[z[11]] ^ S7[z[6]]; - - l = Z[2] ^ S4[z[5]] ^ S5[z[7]] ^ S6[z[4]] ^ S7[z[6]] ^ S6[z[0]]; - CAST_exp(l, X, x, 0); - l = Z[0] ^ S4[x[0]] ^ S5[x[2]] ^ S6[x[1]] ^ S7[x[3]] ^ S7[z[2]]; - CAST_exp(l, X, x, 4); - l = Z[1] ^ S4[x[7]] ^ S5[x[6]] ^ S6[x[5]] ^ S7[x[4]] ^ S4[z[1]]; - CAST_exp(l, X, x, 8); - l = Z[3] ^ S4[x[10]] ^ S5[x[9]] ^ S6[x[11]] ^ S7[x[8]] ^ S5[z[3]]; - CAST_exp(l, X, x, 12); - - K[12] = S4[x[8]] ^ S5[x[9]] ^ S6[x[7]] ^ S7[x[6]] ^ S4[x[3]]; - K[13] = S4[x[10]] ^ S5[x[11]] ^ S6[x[5]] ^ S7[x[4]] ^ S5[x[7]]; - K[14] = S4[x[12]] ^ S5[x[13]] ^ S6[x[3]] ^ S7[x[2]] ^ S6[x[8]]; - K[15] = S4[x[14]] ^ S5[x[15]] ^ S6[x[1]] ^ S7[x[0]] ^ S7[x[13]]; - if (K != k) - break; - K += 16; - } - - for (i = 0; i < 16; i++) { - key->data[i * 2] = k[i]; - key->data[i * 2 + 1] = ((k[i + 16]) + 16) & 0x1f; - } -} diff --git a/openssl/src/crypto/cast/cast_local.h b/openssl/src/crypto/cast/cast_local.h deleted file mode 100644 index e99fe0882..000000000 --- a/openssl/src/crypto/cast/cast_local.h +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifdef OPENSSL_SYS_WIN32 -# include -#endif - -/* NOTE - c is not incremented as per n2l */ -#define n2ln(c,l1,l2,n) { \ - c+=n; \ - l1=l2=0; \ - switch (n) { \ - case 8: l2 =((unsigned long)(*(--(c)))) ; \ - /* fall through */ \ - case 7: l2|=((unsigned long)(*(--(c))))<< 8; \ - /* fall through */ \ - case 6: l2|=((unsigned long)(*(--(c))))<<16; \ - /* fall through */ \ - case 5: l2|=((unsigned long)(*(--(c))))<<24; \ - /* fall through */ \ - case 4: l1 =((unsigned long)(*(--(c)))) ; \ - /* fall through */ \ - case 3: l1|=((unsigned long)(*(--(c))))<< 8; \ - /* fall through */ \ - case 2: l1|=((unsigned long)(*(--(c))))<<16; \ - /* fall through */ \ - case 1: l1|=((unsigned long)(*(--(c))))<<24; \ - } \ - } - -/* NOTE - c is not incremented as per l2n */ -#define l2nn(l1,l2,c,n) { \ - c+=n; \ - switch (n) { \ - case 8: *(--(c))=(unsigned char)(((l2) )&0xff); \ - /* fall through */ \ - case 7: *(--(c))=(unsigned char)(((l2)>> 8)&0xff); \ - /* fall through */ \ - case 6: *(--(c))=(unsigned char)(((l2)>>16)&0xff); \ - /* fall through */ \ - case 5: *(--(c))=(unsigned char)(((l2)>>24)&0xff); \ - /* fall through */ \ - case 4: *(--(c))=(unsigned char)(((l1) )&0xff); \ - /* fall through */ \ - case 3: *(--(c))=(unsigned char)(((l1)>> 8)&0xff); \ - /* fall through */ \ - case 2: *(--(c))=(unsigned char)(((l1)>>16)&0xff); \ - /* fall through */ \ - case 1: *(--(c))=(unsigned char)(((l1)>>24)&0xff); \ - } \ - } - -#undef n2l -#define n2l(c,l) (l =((unsigned long)(*((c)++)))<<24L, \ - l|=((unsigned long)(*((c)++)))<<16L, \ - l|=((unsigned long)(*((c)++)))<< 8L, \ - l|=((unsigned long)(*((c)++)))) - -#undef l2n -#define l2n(l,c) (*((c)++)=(unsigned char)(((l)>>24L)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16L)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8L)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff)) - -#if defined(OPENSSL_SYS_WIN32) && defined(_MSC_VER) -# define ROTL(a,n) (_lrotl(a,n)) -#else -# define ROTL(a,n) ((((a)<<(n))&0xffffffffL)|((a)>>((32-(n))&31))) -#endif - -#define C_M 0x3fc -#define C_0 22L -#define C_1 14L -#define C_2 6L -#define C_3 2L /* left shift */ - -/* The rotate has an extra 16 added to it to help the x86 asm */ -#if defined(CAST_PTR) -# define E_CAST(n,key,L,R,OP1,OP2,OP3) \ - { \ - int i; \ - t=(key[n*2] OP1 R)&0xffffffffL; \ - i=key[n*2+1]; \ - t=ROTL(t,i); \ - L^= (((((*(CAST_LONG *)((unsigned char *) \ - CAST_S_table0+((t>>C_2)&C_M)) OP2 \ - *(CAST_LONG *)((unsigned char *) \ - CAST_S_table1+((t<>C_0)&C_M)))&0xffffffffL) OP1 \ - *(CAST_LONG *)((unsigned char *) \ - CAST_S_table3+((t>>C_1)&C_M)))&0xffffffffL; \ - } -#elif defined(CAST_PTR2) -# define E_CAST(n,key,L,R,OP1,OP2,OP3) \ - { \ - int i; \ - CAST_LONG u,v,w; \ - w=(key[n*2] OP1 R)&0xffffffffL; \ - i=key[n*2+1]; \ - w=ROTL(w,i); \ - u=w>>C_2; \ - v=w<>C_0; \ - t=(t OP2 *(CAST_LONG *)((unsigned char *)CAST_S_table1+v))&0xffffffffL;\ - v=w>>C_1; \ - u&=C_M; \ - v&=C_M; \ - t=(t OP3 *(CAST_LONG *)((unsigned char *)CAST_S_table2+u)&0xffffffffL);\ - t=(t OP1 *(CAST_LONG *)((unsigned char *)CAST_S_table3+v)&0xffffffffL);\ - L^=(t&0xffffffff); \ - } -#else -# define E_CAST(n,key,L,R,OP1,OP2,OP3) \ - { \ - CAST_LONG a,b,c,d; \ - t=(key[n*2] OP1 R)&0xffffffff; \ - t=ROTL(t,(key[n*2+1])); \ - a=CAST_S_table0[(t>> 8)&0xff]; \ - b=CAST_S_table1[(t )&0xff]; \ - c=CAST_S_table2[(t>>24)&0xff]; \ - d=CAST_S_table3[(t>>16)&0xff]; \ - L^=(((((a OP2 b)&0xffffffffL) OP3 c)&0xffffffffL) OP1 d)&0xffffffffL; \ - } -#endif - -extern const CAST_LONG CAST_S_table0[256]; -extern const CAST_LONG CAST_S_table1[256]; -extern const CAST_LONG CAST_S_table2[256]; -extern const CAST_LONG CAST_S_table3[256]; -extern const CAST_LONG CAST_S_table4[256]; -extern const CAST_LONG CAST_S_table5[256]; -extern const CAST_LONG CAST_S_table6[256]; -extern const CAST_LONG CAST_S_table7[256]; diff --git a/openssl/src/crypto/cast/cast_s.h b/openssl/src/crypto/cast/cast_s.h deleted file mode 100644 index 57163d17e..000000000 --- a/openssl/src/crypto/cast/cast_s.h +++ /dev/null @@ -1,544 +0,0 @@ -/* - * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -const CAST_LONG CAST_S_table0[256] = { - 0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, - 0x1e213f2f, 0x9c004dd3, 0x6003e540, 0xcf9fc949, - 0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, - 0x6e63a0e0, 0x15c361d2, 0xc2e7661d, 0x22d4ff8e, - 0x28683b6f, 0xc07fd059, 0xff2379c8, 0x775f50e2, - 0x43c340d3, 0xdf2f8656, 0x887ca41a, 0xa2d2bd2d, - 0xa1c9e0d6, 0x346c4819, 0x61b76d87, 0x22540f2f, - 0x2abe32e1, 0xaa54166b, 0x22568e3a, 0xa2d341d0, - 0x66db40c8, 0xa784392f, 0x004dff2f, 0x2db9d2de, - 0x97943fac, 0x4a97c1d8, 0x527644b7, 0xb5f437a7, - 0xb82cbaef, 0xd751d159, 0x6ff7f0ed, 0x5a097a1f, - 0x827b68d0, 0x90ecf52e, 0x22b0c054, 0xbc8e5935, - 0x4b6d2f7f, 0x50bb64a2, 0xd2664910, 0xbee5812d, - 0xb7332290, 0xe93b159f, 0xb48ee411, 0x4bff345d, - 0xfd45c240, 0xad31973f, 0xc4f6d02e, 0x55fc8165, - 0xd5b1caad, 0xa1ac2dae, 0xa2d4b76d, 0xc19b0c50, - 0x882240f2, 0x0c6e4f38, 0xa4e4bfd7, 0x4f5ba272, - 0x564c1d2f, 0xc59c5319, 0xb949e354, 0xb04669fe, - 0xb1b6ab8a, 0xc71358dd, 0x6385c545, 0x110f935d, - 0x57538ad5, 0x6a390493, 0xe63d37e0, 0x2a54f6b3, - 0x3a787d5f, 0x6276a0b5, 0x19a6fcdf, 0x7a42206a, - 0x29f9d4d5, 0xf61b1891, 0xbb72275e, 0xaa508167, - 0x38901091, 0xc6b505eb, 0x84c7cb8c, 0x2ad75a0f, - 0x874a1427, 0xa2d1936b, 0x2ad286af, 0xaa56d291, - 0xd7894360, 0x425c750d, 0x93b39e26, 0x187184c9, - 0x6c00b32d, 0x73e2bb14, 0xa0bebc3c, 0x54623779, - 0x64459eab, 0x3f328b82, 0x7718cf82, 0x59a2cea6, - 0x04ee002e, 0x89fe78e6, 0x3fab0950, 0x325ff6c2, - 0x81383f05, 0x6963c5c8, 0x76cb5ad6, 0xd49974c9, - 0xca180dcf, 0x380782d5, 0xc7fa5cf6, 0x8ac31511, - 0x35e79e13, 0x47da91d0, 0xf40f9086, 0xa7e2419e, - 0x31366241, 0x051ef495, 0xaa573b04, 0x4a805d8d, - 0x548300d0, 0x00322a3c, 0xbf64cddf, 0xba57a68e, - 0x75c6372b, 0x50afd341, 0xa7c13275, 0x915a0bf5, - 0x6b54bfab, 0x2b0b1426, 0xab4cc9d7, 0x449ccd82, - 0xf7fbf265, 0xab85c5f3, 0x1b55db94, 0xaad4e324, - 0xcfa4bd3f, 0x2deaa3e2, 0x9e204d02, 0xc8bd25ac, - 0xeadf55b3, 0xd5bd9e98, 0xe31231b2, 0x2ad5ad6c, - 0x954329de, 0xadbe4528, 0xd8710f69, 0xaa51c90f, - 0xaa786bf6, 0x22513f1e, 0xaa51a79b, 0x2ad344cc, - 0x7b5a41f0, 0xd37cfbad, 0x1b069505, 0x41ece491, - 0xb4c332e6, 0x032268d4, 0xc9600acc, 0xce387e6d, - 0xbf6bb16c, 0x6a70fb78, 0x0d03d9c9, 0xd4df39de, - 0xe01063da, 0x4736f464, 0x5ad328d8, 0xb347cc96, - 0x75bb0fc3, 0x98511bfb, 0x4ffbcc35, 0xb58bcf6a, - 0xe11f0abc, 0xbfc5fe4a, 0xa70aec10, 0xac39570a, - 0x3f04442f, 0x6188b153, 0xe0397a2e, 0x5727cb79, - 0x9ceb418f, 0x1cacd68d, 0x2ad37c96, 0x0175cb9d, - 0xc69dff09, 0xc75b65f0, 0xd9db40d8, 0xec0e7779, - 0x4744ead4, 0xb11c3274, 0xdd24cb9e, 0x7e1c54bd, - 0xf01144f9, 0xd2240eb1, 0x9675b3fd, 0xa3ac3755, - 0xd47c27af, 0x51c85f4d, 0x56907596, 0xa5bb15e6, - 0x580304f0, 0xca042cf1, 0x011a37ea, 0x8dbfaadb, - 0x35ba3e4a, 0x3526ffa0, 0xc37b4d09, 0xbc306ed9, - 0x98a52666, 0x5648f725, 0xff5e569d, 0x0ced63d0, - 0x7c63b2cf, 0x700b45e1, 0xd5ea50f1, 0x85a92872, - 0xaf1fbda7, 0xd4234870, 0xa7870bf3, 0x2d3b4d79, - 0x42e04198, 0x0cd0ede7, 0x26470db8, 0xf881814c, - 0x474d6ad7, 0x7c0c5e5c, 0xd1231959, 0x381b7298, - 0xf5d2f4db, 0xab838653, 0x6e2f1e23, 0x83719c9e, - 0xbd91e046, 0x9a56456e, 0xdc39200c, 0x20c8c571, - 0x962bda1c, 0xe1e696ff, 0xb141ab08, 0x7cca89b9, - 0x1a69e783, 0x02cc4843, 0xa2f7c579, 0x429ef47d, - 0x427b169c, 0x5ac9f049, 0xdd8f0f00, 0x5c8165bf, -}; - -const CAST_LONG CAST_S_table1[256] = { - 0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, - 0xfe61cf7a, 0xeec5207a, 0x55889c94, 0x72fc0651, - 0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, - 0x99c430ef, 0x5f0c0794, 0x18dcdb7d, 0xa1d6eff3, - 0xa0b52f7b, 0x59e83605, 0xee15b094, 0xe9ffd909, - 0xdc440086, 0xef944459, 0xba83ccb3, 0xe0c3cdfb, - 0xd1da4181, 0x3b092ab1, 0xf997f1c1, 0xa5e6cf7b, - 0x01420ddb, 0xe4e7ef5b, 0x25a1ff41, 0xe180f806, - 0x1fc41080, 0x179bee7a, 0xd37ac6a9, 0xfe5830a4, - 0x98de8b7f, 0x77e83f4e, 0x79929269, 0x24fa9f7b, - 0xe113c85b, 0xacc40083, 0xd7503525, 0xf7ea615f, - 0x62143154, 0x0d554b63, 0x5d681121, 0xc866c359, - 0x3d63cf73, 0xcee234c0, 0xd4d87e87, 0x5c672b21, - 0x071f6181, 0x39f7627f, 0x361e3084, 0xe4eb573b, - 0x602f64a4, 0xd63acd9c, 0x1bbc4635, 0x9e81032d, - 0x2701f50c, 0x99847ab4, 0xa0e3df79, 0xba6cf38c, - 0x10843094, 0x2537a95e, 0xf46f6ffe, 0xa1ff3b1f, - 0x208cfb6a, 0x8f458c74, 0xd9e0a227, 0x4ec73a34, - 0xfc884f69, 0x3e4de8df, 0xef0e0088, 0x3559648d, - 0x8a45388c, 0x1d804366, 0x721d9bfd, 0xa58684bb, - 0xe8256333, 0x844e8212, 0x128d8098, 0xfed33fb4, - 0xce280ae1, 0x27e19ba5, 0xd5a6c252, 0xe49754bd, - 0xc5d655dd, 0xeb667064, 0x77840b4d, 0xa1b6a801, - 0x84db26a9, 0xe0b56714, 0x21f043b7, 0xe5d05860, - 0x54f03084, 0x066ff472, 0xa31aa153, 0xdadc4755, - 0xb5625dbf, 0x68561be6, 0x83ca6b94, 0x2d6ed23b, - 0xeccf01db, 0xa6d3d0ba, 0xb6803d5c, 0xaf77a709, - 0x33b4a34c, 0x397bc8d6, 0x5ee22b95, 0x5f0e5304, - 0x81ed6f61, 0x20e74364, 0xb45e1378, 0xde18639b, - 0x881ca122, 0xb96726d1, 0x8049a7e8, 0x22b7da7b, - 0x5e552d25, 0x5272d237, 0x79d2951c, 0xc60d894c, - 0x488cb402, 0x1ba4fe5b, 0xa4b09f6b, 0x1ca815cf, - 0xa20c3005, 0x8871df63, 0xb9de2fcb, 0x0cc6c9e9, - 0x0beeff53, 0xe3214517, 0xb4542835, 0x9f63293c, - 0xee41e729, 0x6e1d2d7c, 0x50045286, 0x1e6685f3, - 0xf33401c6, 0x30a22c95, 0x31a70850, 0x60930f13, - 0x73f98417, 0xa1269859, 0xec645c44, 0x52c877a9, - 0xcdff33a6, 0xa02b1741, 0x7cbad9a2, 0x2180036f, - 0x50d99c08, 0xcb3f4861, 0xc26bd765, 0x64a3f6ab, - 0x80342676, 0x25a75e7b, 0xe4e6d1fc, 0x20c710e6, - 0xcdf0b680, 0x17844d3b, 0x31eef84d, 0x7e0824e4, - 0x2ccb49eb, 0x846a3bae, 0x8ff77888, 0xee5d60f6, - 0x7af75673, 0x2fdd5cdb, 0xa11631c1, 0x30f66f43, - 0xb3faec54, 0x157fd7fa, 0xef8579cc, 0xd152de58, - 0xdb2ffd5e, 0x8f32ce19, 0x306af97a, 0x02f03ef8, - 0x99319ad5, 0xc242fa0f, 0xa7e3ebb0, 0xc68e4906, - 0xb8da230c, 0x80823028, 0xdcdef3c8, 0xd35fb171, - 0x088a1bc8, 0xbec0c560, 0x61a3c9e8, 0xbca8f54d, - 0xc72feffa, 0x22822e99, 0x82c570b4, 0xd8d94e89, - 0x8b1c34bc, 0x301e16e6, 0x273be979, 0xb0ffeaa6, - 0x61d9b8c6, 0x00b24869, 0xb7ffce3f, 0x08dc283b, - 0x43daf65a, 0xf7e19798, 0x7619b72f, 0x8f1c9ba4, - 0xdc8637a0, 0x16a7d3b1, 0x9fc393b7, 0xa7136eeb, - 0xc6bcc63e, 0x1a513742, 0xef6828bc, 0x520365d6, - 0x2d6a77ab, 0x3527ed4b, 0x821fd216, 0x095c6e2e, - 0xdb92f2fb, 0x5eea29cb, 0x145892f5, 0x91584f7f, - 0x5483697b, 0x2667a8cc, 0x85196048, 0x8c4bacea, - 0x833860d4, 0x0d23e0f9, 0x6c387e8a, 0x0ae6d249, - 0xb284600c, 0xd835731d, 0xdcb1c647, 0xac4c56ea, - 0x3ebd81b3, 0x230eabb0, 0x6438bc87, 0xf0b5b1fa, - 0x8f5ea2b3, 0xfc184642, 0x0a036b7a, 0x4fb089bd, - 0x649da589, 0xa345415e, 0x5c038323, 0x3e5d3bb9, - 0x43d79572, 0x7e6dd07c, 0x06dfdf1e, 0x6c6cc4ef, - 0x7160a539, 0x73bfbe70, 0x83877605, 0x4523ecf1, -}; - -const CAST_LONG CAST_S_table2[256] = { - 0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, - 0x47607fff, 0x369fe44b, 0x8c1fc644, 0xaececa90, - 0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, - 0x920e8806, 0xf0ad0548, 0xe13c8d83, 0x927010d5, - 0x11107d9f, 0x07647db9, 0xb2e3e4d4, 0x3d4f285e, - 0xb9afa820, 0xfade82e0, 0xa067268b, 0x8272792e, - 0x553fb2c0, 0x489ae22b, 0xd4ef9794, 0x125e3fbc, - 0x21fffcee, 0x825b1bfd, 0x9255c5ed, 0x1257a240, - 0x4e1a8302, 0xbae07fff, 0x528246e7, 0x8e57140e, - 0x3373f7bf, 0x8c9f8188, 0xa6fc4ee8, 0xc982b5a5, - 0xa8c01db7, 0x579fc264, 0x67094f31, 0xf2bd3f5f, - 0x40fff7c1, 0x1fb78dfc, 0x8e6bd2c1, 0x437be59b, - 0x99b03dbf, 0xb5dbc64b, 0x638dc0e6, 0x55819d99, - 0xa197c81c, 0x4a012d6e, 0xc5884a28, 0xccc36f71, - 0xb843c213, 0x6c0743f1, 0x8309893c, 0x0feddd5f, - 0x2f7fe850, 0xd7c07f7e, 0x02507fbf, 0x5afb9a04, - 0xa747d2d0, 0x1651192e, 0xaf70bf3e, 0x58c31380, - 0x5f98302e, 0x727cc3c4, 0x0a0fb402, 0x0f7fef82, - 0x8c96fdad, 0x5d2c2aae, 0x8ee99a49, 0x50da88b8, - 0x8427f4a0, 0x1eac5790, 0x796fb449, 0x8252dc15, - 0xefbd7d9b, 0xa672597d, 0xada840d8, 0x45f54504, - 0xfa5d7403, 0xe83ec305, 0x4f91751a, 0x925669c2, - 0x23efe941, 0xa903f12e, 0x60270df2, 0x0276e4b6, - 0x94fd6574, 0x927985b2, 0x8276dbcb, 0x02778176, - 0xf8af918d, 0x4e48f79e, 0x8f616ddf, 0xe29d840e, - 0x842f7d83, 0x340ce5c8, 0x96bbb682, 0x93b4b148, - 0xef303cab, 0x984faf28, 0x779faf9b, 0x92dc560d, - 0x224d1e20, 0x8437aa88, 0x7d29dc96, 0x2756d3dc, - 0x8b907cee, 0xb51fd240, 0xe7c07ce3, 0xe566b4a1, - 0xc3e9615e, 0x3cf8209d, 0x6094d1e3, 0xcd9ca341, - 0x5c76460e, 0x00ea983b, 0xd4d67881, 0xfd47572c, - 0xf76cedd9, 0xbda8229c, 0x127dadaa, 0x438a074e, - 0x1f97c090, 0x081bdb8a, 0x93a07ebe, 0xb938ca15, - 0x97b03cff, 0x3dc2c0f8, 0x8d1ab2ec, 0x64380e51, - 0x68cc7bfb, 0xd90f2788, 0x12490181, 0x5de5ffd4, - 0xdd7ef86a, 0x76a2e214, 0xb9a40368, 0x925d958f, - 0x4b39fffa, 0xba39aee9, 0xa4ffd30b, 0xfaf7933b, - 0x6d498623, 0x193cbcfa, 0x27627545, 0x825cf47a, - 0x61bd8ba0, 0xd11e42d1, 0xcead04f4, 0x127ea392, - 0x10428db7, 0x8272a972, 0x9270c4a8, 0x127de50b, - 0x285ba1c8, 0x3c62f44f, 0x35c0eaa5, 0xe805d231, - 0x428929fb, 0xb4fcdf82, 0x4fb66a53, 0x0e7dc15b, - 0x1f081fab, 0x108618ae, 0xfcfd086d, 0xf9ff2889, - 0x694bcc11, 0x236a5cae, 0x12deca4d, 0x2c3f8cc5, - 0xd2d02dfe, 0xf8ef5896, 0xe4cf52da, 0x95155b67, - 0x494a488c, 0xb9b6a80c, 0x5c8f82bc, 0x89d36b45, - 0x3a609437, 0xec00c9a9, 0x44715253, 0x0a874b49, - 0xd773bc40, 0x7c34671c, 0x02717ef6, 0x4feb5536, - 0xa2d02fff, 0xd2bf60c4, 0xd43f03c0, 0x50b4ef6d, - 0x07478cd1, 0x006e1888, 0xa2e53f55, 0xb9e6d4bc, - 0xa2048016, 0x97573833, 0xd7207d67, 0xde0f8f3d, - 0x72f87b33, 0xabcc4f33, 0x7688c55d, 0x7b00a6b0, - 0x947b0001, 0x570075d2, 0xf9bb88f8, 0x8942019e, - 0x4264a5ff, 0x856302e0, 0x72dbd92b, 0xee971b69, - 0x6ea22fde, 0x5f08ae2b, 0xaf7a616d, 0xe5c98767, - 0xcf1febd2, 0x61efc8c2, 0xf1ac2571, 0xcc8239c2, - 0x67214cb8, 0xb1e583d1, 0xb7dc3e62, 0x7f10bdce, - 0xf90a5c38, 0x0ff0443d, 0x606e6dc6, 0x60543a49, - 0x5727c148, 0x2be98a1d, 0x8ab41738, 0x20e1be24, - 0xaf96da0f, 0x68458425, 0x99833be5, 0x600d457d, - 0x282f9350, 0x8334b362, 0xd91d1120, 0x2b6d8da0, - 0x642b1e31, 0x9c305a00, 0x52bce688, 0x1b03588a, - 0xf7baefd5, 0x4142ed9c, 0xa4315c11, 0x83323ec5, - 0xdfef4636, 0xa133c501, 0xe9d3531c, 0xee353783, -}; - -const CAST_LONG CAST_S_table3[256] = { - 0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, - 0x4a4f7bdb, 0x64ad8c57, 0x85510443, 0xfa020ed1, - 0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, - 0xfd059d43, 0x6497b7b1, 0xf3641f63, 0x241e4adf, - 0x28147f5f, 0x4fa2b8cd, 0xc9430040, 0x0cc32220, - 0xfdd30b30, 0xc0a5374f, 0x1d2d00d9, 0x24147b15, - 0xee4d111a, 0x0fca5167, 0x71ff904c, 0x2d195ffe, - 0x1a05645f, 0x0c13fefe, 0x081b08ca, 0x05170121, - 0x80530100, 0xe83e5efe, 0xac9af4f8, 0x7fe72701, - 0xd2b8ee5f, 0x06df4261, 0xbb9e9b8a, 0x7293ea25, - 0xce84ffdf, 0xf5718801, 0x3dd64b04, 0xa26f263b, - 0x7ed48400, 0x547eebe6, 0x446d4ca0, 0x6cf3d6f5, - 0x2649abdf, 0xaea0c7f5, 0x36338cc1, 0x503f7e93, - 0xd3772061, 0x11b638e1, 0x72500e03, 0xf80eb2bb, - 0xabe0502e, 0xec8d77de, 0x57971e81, 0xe14f6746, - 0xc9335400, 0x6920318f, 0x081dbb99, 0xffc304a5, - 0x4d351805, 0x7f3d5ce3, 0xa6c866c6, 0x5d5bcca9, - 0xdaec6fea, 0x9f926f91, 0x9f46222f, 0x3991467d, - 0xa5bf6d8e, 0x1143c44f, 0x43958302, 0xd0214eeb, - 0x022083b8, 0x3fb6180c, 0x18f8931e, 0x281658e6, - 0x26486e3e, 0x8bd78a70, 0x7477e4c1, 0xb506e07c, - 0xf32d0a25, 0x79098b02, 0xe4eabb81, 0x28123b23, - 0x69dead38, 0x1574ca16, 0xdf871b62, 0x211c40b7, - 0xa51a9ef9, 0x0014377b, 0x041e8ac8, 0x09114003, - 0xbd59e4d2, 0xe3d156d5, 0x4fe876d5, 0x2f91a340, - 0x557be8de, 0x00eae4a7, 0x0ce5c2ec, 0x4db4bba6, - 0xe756bdff, 0xdd3369ac, 0xec17b035, 0x06572327, - 0x99afc8b0, 0x56c8c391, 0x6b65811c, 0x5e146119, - 0x6e85cb75, 0xbe07c002, 0xc2325577, 0x893ff4ec, - 0x5bbfc92d, 0xd0ec3b25, 0xb7801ab7, 0x8d6d3b24, - 0x20c763ef, 0xc366a5fc, 0x9c382880, 0x0ace3205, - 0xaac9548a, 0xeca1d7c7, 0x041afa32, 0x1d16625a, - 0x6701902c, 0x9b757a54, 0x31d477f7, 0x9126b031, - 0x36cc6fdb, 0xc70b8b46, 0xd9e66a48, 0x56e55a79, - 0x026a4ceb, 0x52437eff, 0x2f8f76b4, 0x0df980a5, - 0x8674cde3, 0xedda04eb, 0x17a9be04, 0x2c18f4df, - 0xb7747f9d, 0xab2af7b4, 0xefc34d20, 0x2e096b7c, - 0x1741a254, 0xe5b6a035, 0x213d42f6, 0x2c1c7c26, - 0x61c2f50f, 0x6552daf9, 0xd2c231f8, 0x25130f69, - 0xd8167fa2, 0x0418f2c8, 0x001a96a6, 0x0d1526ab, - 0x63315c21, 0x5e0a72ec, 0x49bafefd, 0x187908d9, - 0x8d0dbd86, 0x311170a7, 0x3e9b640c, 0xcc3e10d7, - 0xd5cad3b6, 0x0caec388, 0xf73001e1, 0x6c728aff, - 0x71eae2a1, 0x1f9af36e, 0xcfcbd12f, 0xc1de8417, - 0xac07be6b, 0xcb44a1d8, 0x8b9b0f56, 0x013988c3, - 0xb1c52fca, 0xb4be31cd, 0xd8782806, 0x12a3a4e2, - 0x6f7de532, 0x58fd7eb6, 0xd01ee900, 0x24adffc2, - 0xf4990fc5, 0x9711aac5, 0x001d7b95, 0x82e5e7d2, - 0x109873f6, 0x00613096, 0xc32d9521, 0xada121ff, - 0x29908415, 0x7fbb977f, 0xaf9eb3db, 0x29c9ed2a, - 0x5ce2a465, 0xa730f32c, 0xd0aa3fe8, 0x8a5cc091, - 0xd49e2ce7, 0x0ce454a9, 0xd60acd86, 0x015f1919, - 0x77079103, 0xdea03af6, 0x78a8565e, 0xdee356df, - 0x21f05cbe, 0x8b75e387, 0xb3c50651, 0xb8a5c3ef, - 0xd8eeb6d2, 0xe523be77, 0xc2154529, 0x2f69efdf, - 0xafe67afb, 0xf470c4b2, 0xf3e0eb5b, 0xd6cc9876, - 0x39e4460c, 0x1fda8538, 0x1987832f, 0xca007367, - 0xa99144f8, 0x296b299e, 0x492fc295, 0x9266beab, - 0xb5676e69, 0x9bd3ddda, 0xdf7e052f, 0xdb25701c, - 0x1b5e51ee, 0xf65324e6, 0x6afce36c, 0x0316cc04, - 0x8644213e, 0xb7dc59d0, 0x7965291f, 0xccd6fd43, - 0x41823979, 0x932bcdf6, 0xb657c34d, 0x4edfd282, - 0x7ae5290c, 0x3cb9536b, 0x851e20fe, 0x9833557e, - 0x13ecf0b0, 0xd3ffb372, 0x3f85c5c1, 0x0aef7ed2, -}; - -const CAST_LONG CAST_S_table4[256] = { - 0x7ec90c04, 0x2c6e74b9, 0x9b0e66df, 0xa6337911, - 0xb86a7fff, 0x1dd358f5, 0x44dd9d44, 0x1731167f, - 0x08fbf1fa, 0xe7f511cc, 0xd2051b00, 0x735aba00, - 0x2ab722d8, 0x386381cb, 0xacf6243a, 0x69befd7a, - 0xe6a2e77f, 0xf0c720cd, 0xc4494816, 0xccf5c180, - 0x38851640, 0x15b0a848, 0xe68b18cb, 0x4caadeff, - 0x5f480a01, 0x0412b2aa, 0x259814fc, 0x41d0efe2, - 0x4e40b48d, 0x248eb6fb, 0x8dba1cfe, 0x41a99b02, - 0x1a550a04, 0xba8f65cb, 0x7251f4e7, 0x95a51725, - 0xc106ecd7, 0x97a5980a, 0xc539b9aa, 0x4d79fe6a, - 0xf2f3f763, 0x68af8040, 0xed0c9e56, 0x11b4958b, - 0xe1eb5a88, 0x8709e6b0, 0xd7e07156, 0x4e29fea7, - 0x6366e52d, 0x02d1c000, 0xc4ac8e05, 0x9377f571, - 0x0c05372a, 0x578535f2, 0x2261be02, 0xd642a0c9, - 0xdf13a280, 0x74b55bd2, 0x682199c0, 0xd421e5ec, - 0x53fb3ce8, 0xc8adedb3, 0x28a87fc9, 0x3d959981, - 0x5c1ff900, 0xfe38d399, 0x0c4eff0b, 0x062407ea, - 0xaa2f4fb1, 0x4fb96976, 0x90c79505, 0xb0a8a774, - 0xef55a1ff, 0xe59ca2c2, 0xa6b62d27, 0xe66a4263, - 0xdf65001f, 0x0ec50966, 0xdfdd55bc, 0x29de0655, - 0x911e739a, 0x17af8975, 0x32c7911c, 0x89f89468, - 0x0d01e980, 0x524755f4, 0x03b63cc9, 0x0cc844b2, - 0xbcf3f0aa, 0x87ac36e9, 0xe53a7426, 0x01b3d82b, - 0x1a9e7449, 0x64ee2d7e, 0xcddbb1da, 0x01c94910, - 0xb868bf80, 0x0d26f3fd, 0x9342ede7, 0x04a5c284, - 0x636737b6, 0x50f5b616, 0xf24766e3, 0x8eca36c1, - 0x136e05db, 0xfef18391, 0xfb887a37, 0xd6e7f7d4, - 0xc7fb7dc9, 0x3063fcdf, 0xb6f589de, 0xec2941da, - 0x26e46695, 0xb7566419, 0xf654efc5, 0xd08d58b7, - 0x48925401, 0xc1bacb7f, 0xe5ff550f, 0xb6083049, - 0x5bb5d0e8, 0x87d72e5a, 0xab6a6ee1, 0x223a66ce, - 0xc62bf3cd, 0x9e0885f9, 0x68cb3e47, 0x086c010f, - 0xa21de820, 0xd18b69de, 0xf3f65777, 0xfa02c3f6, - 0x407edac3, 0xcbb3d550, 0x1793084d, 0xb0d70eba, - 0x0ab378d5, 0xd951fb0c, 0xded7da56, 0x4124bbe4, - 0x94ca0b56, 0x0f5755d1, 0xe0e1e56e, 0x6184b5be, - 0x580a249f, 0x94f74bc0, 0xe327888e, 0x9f7b5561, - 0xc3dc0280, 0x05687715, 0x646c6bd7, 0x44904db3, - 0x66b4f0a3, 0xc0f1648a, 0x697ed5af, 0x49e92ff6, - 0x309e374f, 0x2cb6356a, 0x85808573, 0x4991f840, - 0x76f0ae02, 0x083be84d, 0x28421c9a, 0x44489406, - 0x736e4cb8, 0xc1092910, 0x8bc95fc6, 0x7d869cf4, - 0x134f616f, 0x2e77118d, 0xb31b2be1, 0xaa90b472, - 0x3ca5d717, 0x7d161bba, 0x9cad9010, 0xaf462ba2, - 0x9fe459d2, 0x45d34559, 0xd9f2da13, 0xdbc65487, - 0xf3e4f94e, 0x176d486f, 0x097c13ea, 0x631da5c7, - 0x445f7382, 0x175683f4, 0xcdc66a97, 0x70be0288, - 0xb3cdcf72, 0x6e5dd2f3, 0x20936079, 0x459b80a5, - 0xbe60e2db, 0xa9c23101, 0xeba5315c, 0x224e42f2, - 0x1c5c1572, 0xf6721b2c, 0x1ad2fff3, 0x8c25404e, - 0x324ed72f, 0x4067b7fd, 0x0523138e, 0x5ca3bc78, - 0xdc0fd66e, 0x75922283, 0x784d6b17, 0x58ebb16e, - 0x44094f85, 0x3f481d87, 0xfcfeae7b, 0x77b5ff76, - 0x8c2302bf, 0xaaf47556, 0x5f46b02a, 0x2b092801, - 0x3d38f5f7, 0x0ca81f36, 0x52af4a8a, 0x66d5e7c0, - 0xdf3b0874, 0x95055110, 0x1b5ad7a8, 0xf61ed5ad, - 0x6cf6e479, 0x20758184, 0xd0cefa65, 0x88f7be58, - 0x4a046826, 0x0ff6f8f3, 0xa09c7f70, 0x5346aba0, - 0x5ce96c28, 0xe176eda3, 0x6bac307f, 0x376829d2, - 0x85360fa9, 0x17e3fe2a, 0x24b79767, 0xf5a96b20, - 0xd6cd2595, 0x68ff1ebf, 0x7555442c, 0xf19f06be, - 0xf9e0659a, 0xeeb9491d, 0x34010718, 0xbb30cab8, - 0xe822fe15, 0x88570983, 0x750e6249, 0xda627e55, - 0x5e76ffa8, 0xb1534546, 0x6d47de08, 0xefe9e7d4, -}; - -const CAST_LONG CAST_S_table5[256] = { - 0xf6fa8f9d, 0x2cac6ce1, 0x4ca34867, 0xe2337f7c, - 0x95db08e7, 0x016843b4, 0xeced5cbc, 0x325553ac, - 0xbf9f0960, 0xdfa1e2ed, 0x83f0579d, 0x63ed86b9, - 0x1ab6a6b8, 0xde5ebe39, 0xf38ff732, 0x8989b138, - 0x33f14961, 0xc01937bd, 0xf506c6da, 0xe4625e7e, - 0xa308ea99, 0x4e23e33c, 0x79cbd7cc, 0x48a14367, - 0xa3149619, 0xfec94bd5, 0xa114174a, 0xeaa01866, - 0xa084db2d, 0x09a8486f, 0xa888614a, 0x2900af98, - 0x01665991, 0xe1992863, 0xc8f30c60, 0x2e78ef3c, - 0xd0d51932, 0xcf0fec14, 0xf7ca07d2, 0xd0a82072, - 0xfd41197e, 0x9305a6b0, 0xe86be3da, 0x74bed3cd, - 0x372da53c, 0x4c7f4448, 0xdab5d440, 0x6dba0ec3, - 0x083919a7, 0x9fbaeed9, 0x49dbcfb0, 0x4e670c53, - 0x5c3d9c01, 0x64bdb941, 0x2c0e636a, 0xba7dd9cd, - 0xea6f7388, 0xe70bc762, 0x35f29adb, 0x5c4cdd8d, - 0xf0d48d8c, 0xb88153e2, 0x08a19866, 0x1ae2eac8, - 0x284caf89, 0xaa928223, 0x9334be53, 0x3b3a21bf, - 0x16434be3, 0x9aea3906, 0xefe8c36e, 0xf890cdd9, - 0x80226dae, 0xc340a4a3, 0xdf7e9c09, 0xa694a807, - 0x5b7c5ecc, 0x221db3a6, 0x9a69a02f, 0x68818a54, - 0xceb2296f, 0x53c0843a, 0xfe893655, 0x25bfe68a, - 0xb4628abc, 0xcf222ebf, 0x25ac6f48, 0xa9a99387, - 0x53bddb65, 0xe76ffbe7, 0xe967fd78, 0x0ba93563, - 0x8e342bc1, 0xe8a11be9, 0x4980740d, 0xc8087dfc, - 0x8de4bf99, 0xa11101a0, 0x7fd37975, 0xda5a26c0, - 0xe81f994f, 0x9528cd89, 0xfd339fed, 0xb87834bf, - 0x5f04456d, 0x22258698, 0xc9c4c83b, 0x2dc156be, - 0x4f628daa, 0x57f55ec5, 0xe2220abe, 0xd2916ebf, - 0x4ec75b95, 0x24f2c3c0, 0x42d15d99, 0xcd0d7fa0, - 0x7b6e27ff, 0xa8dc8af0, 0x7345c106, 0xf41e232f, - 0x35162386, 0xe6ea8926, 0x3333b094, 0x157ec6f2, - 0x372b74af, 0x692573e4, 0xe9a9d848, 0xf3160289, - 0x3a62ef1d, 0xa787e238, 0xf3a5f676, 0x74364853, - 0x20951063, 0x4576698d, 0xb6fad407, 0x592af950, - 0x36f73523, 0x4cfb6e87, 0x7da4cec0, 0x6c152daa, - 0xcb0396a8, 0xc50dfe5d, 0xfcd707ab, 0x0921c42f, - 0x89dff0bb, 0x5fe2be78, 0x448f4f33, 0x754613c9, - 0x2b05d08d, 0x48b9d585, 0xdc049441, 0xc8098f9b, - 0x7dede786, 0xc39a3373, 0x42410005, 0x6a091751, - 0x0ef3c8a6, 0x890072d6, 0x28207682, 0xa9a9f7be, - 0xbf32679d, 0xd45b5b75, 0xb353fd00, 0xcbb0e358, - 0x830f220a, 0x1f8fb214, 0xd372cf08, 0xcc3c4a13, - 0x8cf63166, 0x061c87be, 0x88c98f88, 0x6062e397, - 0x47cf8e7a, 0xb6c85283, 0x3cc2acfb, 0x3fc06976, - 0x4e8f0252, 0x64d8314d, 0xda3870e3, 0x1e665459, - 0xc10908f0, 0x513021a5, 0x6c5b68b7, 0x822f8aa0, - 0x3007cd3e, 0x74719eef, 0xdc872681, 0x073340d4, - 0x7e432fd9, 0x0c5ec241, 0x8809286c, 0xf592d891, - 0x08a930f6, 0x957ef305, 0xb7fbffbd, 0xc266e96f, - 0x6fe4ac98, 0xb173ecc0, 0xbc60b42a, 0x953498da, - 0xfba1ae12, 0x2d4bd736, 0x0f25faab, 0xa4f3fceb, - 0xe2969123, 0x257f0c3d, 0x9348af49, 0x361400bc, - 0xe8816f4a, 0x3814f200, 0xa3f94043, 0x9c7a54c2, - 0xbc704f57, 0xda41e7f9, 0xc25ad33a, 0x54f4a084, - 0xb17f5505, 0x59357cbe, 0xedbd15c8, 0x7f97c5ab, - 0xba5ac7b5, 0xb6f6deaf, 0x3a479c3a, 0x5302da25, - 0x653d7e6a, 0x54268d49, 0x51a477ea, 0x5017d55b, - 0xd7d25d88, 0x44136c76, 0x0404a8c8, 0xb8e5a121, - 0xb81a928a, 0x60ed5869, 0x97c55b96, 0xeaec991b, - 0x29935913, 0x01fdb7f1, 0x088e8dfa, 0x9ab6f6f5, - 0x3b4cbf9f, 0x4a5de3ab, 0xe6051d35, 0xa0e1d855, - 0xd36b4cf1, 0xf544edeb, 0xb0e93524, 0xbebb8fbd, - 0xa2d762cf, 0x49c92f54, 0x38b5f331, 0x7128a454, - 0x48392905, 0xa65b1db8, 0x851c97bd, 0xd675cf2f, -}; - -const CAST_LONG CAST_S_table6[256] = { - 0x85e04019, 0x332bf567, 0x662dbfff, 0xcfc65693, - 0x2a8d7f6f, 0xab9bc912, 0xde6008a1, 0x2028da1f, - 0x0227bce7, 0x4d642916, 0x18fac300, 0x50f18b82, - 0x2cb2cb11, 0xb232e75c, 0x4b3695f2, 0xb28707de, - 0xa05fbcf6, 0xcd4181e9, 0xe150210c, 0xe24ef1bd, - 0xb168c381, 0xfde4e789, 0x5c79b0d8, 0x1e8bfd43, - 0x4d495001, 0x38be4341, 0x913cee1d, 0x92a79c3f, - 0x089766be, 0xbaeeadf4, 0x1286becf, 0xb6eacb19, - 0x2660c200, 0x7565bde4, 0x64241f7a, 0x8248dca9, - 0xc3b3ad66, 0x28136086, 0x0bd8dfa8, 0x356d1cf2, - 0x107789be, 0xb3b2e9ce, 0x0502aa8f, 0x0bc0351e, - 0x166bf52a, 0xeb12ff82, 0xe3486911, 0xd34d7516, - 0x4e7b3aff, 0x5f43671b, 0x9cf6e037, 0x4981ac83, - 0x334266ce, 0x8c9341b7, 0xd0d854c0, 0xcb3a6c88, - 0x47bc2829, 0x4725ba37, 0xa66ad22b, 0x7ad61f1e, - 0x0c5cbafa, 0x4437f107, 0xb6e79962, 0x42d2d816, - 0x0a961288, 0xe1a5c06e, 0x13749e67, 0x72fc081a, - 0xb1d139f7, 0xf9583745, 0xcf19df58, 0xbec3f756, - 0xc06eba30, 0x07211b24, 0x45c28829, 0xc95e317f, - 0xbc8ec511, 0x38bc46e9, 0xc6e6fa14, 0xbae8584a, - 0xad4ebc46, 0x468f508b, 0x7829435f, 0xf124183b, - 0x821dba9f, 0xaff60ff4, 0xea2c4e6d, 0x16e39264, - 0x92544a8b, 0x009b4fc3, 0xaba68ced, 0x9ac96f78, - 0x06a5b79a, 0xb2856e6e, 0x1aec3ca9, 0xbe838688, - 0x0e0804e9, 0x55f1be56, 0xe7e5363b, 0xb3a1f25d, - 0xf7debb85, 0x61fe033c, 0x16746233, 0x3c034c28, - 0xda6d0c74, 0x79aac56c, 0x3ce4e1ad, 0x51f0c802, - 0x98f8f35a, 0x1626a49f, 0xeed82b29, 0x1d382fe3, - 0x0c4fb99a, 0xbb325778, 0x3ec6d97b, 0x6e77a6a9, - 0xcb658b5c, 0xd45230c7, 0x2bd1408b, 0x60c03eb7, - 0xb9068d78, 0xa33754f4, 0xf430c87d, 0xc8a71302, - 0xb96d8c32, 0xebd4e7be, 0xbe8b9d2d, 0x7979fb06, - 0xe7225308, 0x8b75cf77, 0x11ef8da4, 0xe083c858, - 0x8d6b786f, 0x5a6317a6, 0xfa5cf7a0, 0x5dda0033, - 0xf28ebfb0, 0xf5b9c310, 0xa0eac280, 0x08b9767a, - 0xa3d9d2b0, 0x79d34217, 0x021a718d, 0x9ac6336a, - 0x2711fd60, 0x438050e3, 0x069908a8, 0x3d7fedc4, - 0x826d2bef, 0x4eeb8476, 0x488dcf25, 0x36c9d566, - 0x28e74e41, 0xc2610aca, 0x3d49a9cf, 0xbae3b9df, - 0xb65f8de6, 0x92aeaf64, 0x3ac7d5e6, 0x9ea80509, - 0xf22b017d, 0xa4173f70, 0xdd1e16c3, 0x15e0d7f9, - 0x50b1b887, 0x2b9f4fd5, 0x625aba82, 0x6a017962, - 0x2ec01b9c, 0x15488aa9, 0xd716e740, 0x40055a2c, - 0x93d29a22, 0xe32dbf9a, 0x058745b9, 0x3453dc1e, - 0xd699296e, 0x496cff6f, 0x1c9f4986, 0xdfe2ed07, - 0xb87242d1, 0x19de7eae, 0x053e561a, 0x15ad6f8c, - 0x66626c1c, 0x7154c24c, 0xea082b2a, 0x93eb2939, - 0x17dcb0f0, 0x58d4f2ae, 0x9ea294fb, 0x52cf564c, - 0x9883fe66, 0x2ec40581, 0x763953c3, 0x01d6692e, - 0xd3a0c108, 0xa1e7160e, 0xe4f2dfa6, 0x693ed285, - 0x74904698, 0x4c2b0edd, 0x4f757656, 0x5d393378, - 0xa132234f, 0x3d321c5d, 0xc3f5e194, 0x4b269301, - 0xc79f022f, 0x3c997e7e, 0x5e4f9504, 0x3ffafbbd, - 0x76f7ad0e, 0x296693f4, 0x3d1fce6f, 0xc61e45be, - 0xd3b5ab34, 0xf72bf9b7, 0x1b0434c0, 0x4e72b567, - 0x5592a33d, 0xb5229301, 0xcfd2a87f, 0x60aeb767, - 0x1814386b, 0x30bcc33d, 0x38a0c07d, 0xfd1606f2, - 0xc363519b, 0x589dd390, 0x5479f8e6, 0x1cb8d647, - 0x97fd61a9, 0xea7759f4, 0x2d57539d, 0x569a58cf, - 0xe84e63ad, 0x462e1b78, 0x6580f87e, 0xf3817914, - 0x91da55f4, 0x40a230f3, 0xd1988f35, 0xb6e318d2, - 0x3ffa50bc, 0x3d40f021, 0xc3c0bdae, 0x4958c24c, - 0x518f36b2, 0x84b1d370, 0x0fedce83, 0x878ddada, - 0xf2a279c7, 0x94e01be8, 0x90716f4b, 0x954b8aa3, -}; - -const CAST_LONG CAST_S_table7[256] = { - 0xe216300d, 0xbbddfffc, 0xa7ebdabd, 0x35648095, - 0x7789f8b7, 0xe6c1121b, 0x0e241600, 0x052ce8b5, - 0x11a9cfb0, 0xe5952f11, 0xece7990a, 0x9386d174, - 0x2a42931c, 0x76e38111, 0xb12def3a, 0x37ddddfc, - 0xde9adeb1, 0x0a0cc32c, 0xbe197029, 0x84a00940, - 0xbb243a0f, 0xb4d137cf, 0xb44e79f0, 0x049eedfd, - 0x0b15a15d, 0x480d3168, 0x8bbbde5a, 0x669ded42, - 0xc7ece831, 0x3f8f95e7, 0x72df191b, 0x7580330d, - 0x94074251, 0x5c7dcdfa, 0xabbe6d63, 0xaa402164, - 0xb301d40a, 0x02e7d1ca, 0x53571dae, 0x7a3182a2, - 0x12a8ddec, 0xfdaa335d, 0x176f43e8, 0x71fb46d4, - 0x38129022, 0xce949ad4, 0xb84769ad, 0x965bd862, - 0x82f3d055, 0x66fb9767, 0x15b80b4e, 0x1d5b47a0, - 0x4cfde06f, 0xc28ec4b8, 0x57e8726e, 0x647a78fc, - 0x99865d44, 0x608bd593, 0x6c200e03, 0x39dc5ff6, - 0x5d0b00a3, 0xae63aff2, 0x7e8bd632, 0x70108c0c, - 0xbbd35049, 0x2998df04, 0x980cf42a, 0x9b6df491, - 0x9e7edd53, 0x06918548, 0x58cb7e07, 0x3b74ef2e, - 0x522fffb1, 0xd24708cc, 0x1c7e27cd, 0xa4eb215b, - 0x3cf1d2e2, 0x19b47a38, 0x424f7618, 0x35856039, - 0x9d17dee7, 0x27eb35e6, 0xc9aff67b, 0x36baf5b8, - 0x09c467cd, 0xc18910b1, 0xe11dbf7b, 0x06cd1af8, - 0x7170c608, 0x2d5e3354, 0xd4de495a, 0x64c6d006, - 0xbcc0c62c, 0x3dd00db3, 0x708f8f34, 0x77d51b42, - 0x264f620f, 0x24b8d2bf, 0x15c1b79e, 0x46a52564, - 0xf8d7e54e, 0x3e378160, 0x7895cda5, 0x859c15a5, - 0xe6459788, 0xc37bc75f, 0xdb07ba0c, 0x0676a3ab, - 0x7f229b1e, 0x31842e7b, 0x24259fd7, 0xf8bef472, - 0x835ffcb8, 0x6df4c1f2, 0x96f5b195, 0xfd0af0fc, - 0xb0fe134c, 0xe2506d3d, 0x4f9b12ea, 0xf215f225, - 0xa223736f, 0x9fb4c428, 0x25d04979, 0x34c713f8, - 0xc4618187, 0xea7a6e98, 0x7cd16efc, 0x1436876c, - 0xf1544107, 0xbedeee14, 0x56e9af27, 0xa04aa441, - 0x3cf7c899, 0x92ecbae6, 0xdd67016d, 0x151682eb, - 0xa842eedf, 0xfdba60b4, 0xf1907b75, 0x20e3030f, - 0x24d8c29e, 0xe139673b, 0xefa63fb8, 0x71873054, - 0xb6f2cf3b, 0x9f326442, 0xcb15a4cc, 0xb01a4504, - 0xf1e47d8d, 0x844a1be5, 0xbae7dfdc, 0x42cbda70, - 0xcd7dae0a, 0x57e85b7a, 0xd53f5af6, 0x20cf4d8c, - 0xcea4d428, 0x79d130a4, 0x3486ebfb, 0x33d3cddc, - 0x77853b53, 0x37effcb5, 0xc5068778, 0xe580b3e6, - 0x4e68b8f4, 0xc5c8b37e, 0x0d809ea2, 0x398feb7c, - 0x132a4f94, 0x43b7950e, 0x2fee7d1c, 0x223613bd, - 0xdd06caa2, 0x37df932b, 0xc4248289, 0xacf3ebc3, - 0x5715f6b7, 0xef3478dd, 0xf267616f, 0xc148cbe4, - 0x9052815e, 0x5e410fab, 0xb48a2465, 0x2eda7fa4, - 0xe87b40e4, 0xe98ea084, 0x5889e9e1, 0xefd390fc, - 0xdd07d35b, 0xdb485694, 0x38d7e5b2, 0x57720101, - 0x730edebc, 0x5b643113, 0x94917e4f, 0x503c2fba, - 0x646f1282, 0x7523d24a, 0xe0779695, 0xf9c17a8f, - 0x7a5b2121, 0xd187b896, 0x29263a4d, 0xba510cdf, - 0x81f47c9f, 0xad1163ed, 0xea7b5965, 0x1a00726e, - 0x11403092, 0x00da6d77, 0x4a0cdd61, 0xad1f4603, - 0x605bdfb0, 0x9eedc364, 0x22ebe6a8, 0xcee7d28a, - 0xa0e736a0, 0x5564a6b9, 0x10853209, 0xc7eb8f37, - 0x2de705ca, 0x8951570f, 0xdf09822b, 0xbd691a6c, - 0xaa12e4f2, 0x87451c0f, 0xe0f6a27a, 0x3ada4819, - 0x4cf1764f, 0x0d771c2b, 0x67cdb156, 0x350d8384, - 0x5938fa0f, 0x42399ef3, 0x36997b07, 0x0e84093d, - 0x4aa93e61, 0x8360d87b, 0x1fa98b0c, 0x1149382c, - 0xe97625a5, 0x0614d1b7, 0x0e25244b, 0x0c768347, - 0x589e8d82, 0x0d2059d1, 0xa466bb1e, 0xf8da0a82, - 0x04f19130, 0xba6e4ec0, 0x99265164, 0x1ee7230d, - 0x50b2ad80, 0xeaee6801, 0x8db2a283, 0xea8bf59e, -}; diff --git a/openssl/src/crypto/chacha/chacha_enc.c b/openssl/src/crypto/chacha/chacha_enc.c index f6fbc1198..c5d1d63d8 100644 --- a/openssl/src/crypto/chacha/chacha_enc.c +++ b/openssl/src/crypto/chacha/chacha_enc.c @@ -24,28 +24,6 @@ typedef union { # define ROTATE(v, n) (((v) << (n)) | ((v) >> (32 - (n)))) -# ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && \ - !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if defined(__riscv_zbb) || defined(__riscv_zbkb) -# if __riscv_xlen == 64 -# undef ROTATE -# define ROTATE(x, n) ({ u32 ret; \ - asm ("roriw %0, %1, %2" \ - : "=r"(ret) \ - : "r"(x), "i"(32 - (n))); ret;}) -# endif -# if __riscv_xlen == 32 -# undef ROTATE -# define ROTATE(x, n) ({ u32 ret; \ - asm ("rori %0, %1, %2" \ - : "=r"(ret) \ - : "r"(x), "i"(32 - (n))); ret;}) -# endif -# endif -# endif -# endif - # define U32TO8_LITTLE(p, v) do { \ (p)[0] = (u8)(v >> 0); \ (p)[1] = (u8)(v >> 8); \ @@ -90,13 +68,9 @@ static void chacha20_core(chacha_buf *output, const u32 input[16]) } } -#ifdef INCLUDE_C_CHACHA20 -void ChaCha20_ctr32_c(unsigned char *out, const unsigned char *inp, size_t len, - const unsigned int key[8], const unsigned int counter[4]) -#else -void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len, - const unsigned int key[8], const unsigned int counter[4]) -#endif +void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, + size_t len, const unsigned int key[8], + const unsigned int counter[4]) { u32 input[16]; chacha_buf buf; diff --git a/openssl/src/crypto/chacha/chacha_ppc.c b/openssl/src/crypto/chacha/chacha_ppc.c index 91ed85eaf..5319040cc 100644 --- a/openssl/src/crypto/chacha/chacha_ppc.c +++ b/openssl/src/crypto/chacha/chacha_ppc.c @@ -1,5 +1,5 @@ /* - * Copyright 2009-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2009-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,20 +23,13 @@ void ChaCha20_ctr32_vmx(unsigned char *out, const unsigned char *inp, void ChaCha20_ctr32_vsx(unsigned char *out, const unsigned char *inp, size_t len, const unsigned int key[8], const unsigned int counter[4]); -void ChaCha20_ctr32_vsx_p10(unsigned char *out, const unsigned char *inp, - size_t len, const unsigned int key[8], - const unsigned int counter[4]); void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len, const unsigned int key[8], const unsigned int counter[4]) { -#if !defined(OPENSSL_SYS_AIX) && !defined(OPENSSL_SYS_MACOSX) - OPENSSL_ppccap_P & PPC_BRD31 - ? ChaCha20_ctr32_vsx_p10(out, inp, len, key, counter) : -#endif - OPENSSL_ppccap_P & PPC_CRYPTO207 - ? ChaCha20_ctr32_vsx(out, inp, len, key, counter) - : OPENSSL_ppccap_P & PPC_ALTIVEC - ? ChaCha20_ctr32_vmx(out, inp, len, key, counter) - : ChaCha20_ctr32_int(out, inp, len, key, counter); + OPENSSL_ppccap_P & PPC_CRYPTO207 + ? ChaCha20_ctr32_vsx(out, inp, len, key, counter) + : OPENSSL_ppccap_P & PPC_ALTIVEC + ? ChaCha20_ctr32_vmx(out, inp, len, key, counter) + : ChaCha20_ctr32_int(out, inp, len, key, counter); } diff --git a/openssl/src/crypto/chacha/chacha_riscv.c b/openssl/src/crypto/chacha/chacha_riscv.c deleted file mode 100644 index 06e0400ba..000000000 --- a/openssl/src/crypto/chacha/chacha_riscv.c +++ /dev/null @@ -1,56 +0,0 @@ -/* - * This file is dual-licensed, meaning that you can use it under your - * choice of either of the following two licenses: - * - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - * - * or - * - * Copyright (c) 2023, Jerry Shih - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include "crypto/chacha.h" -#include "crypto/riscv_arch.h" - -void ChaCha20_ctr32_zbb_zvkb(unsigned char *out, const unsigned char *inp, - size_t len, const unsigned int key[8], - const unsigned int counter[4]); - -void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len, - const unsigned int key[8], const unsigned int counter[4]) -{ - if (len > CHACHA_BLK_SIZE && RISCV_HAS_ZVKB() && RISCV_HAS_ZBB() && - riscv_vlen() >= 128) { - ChaCha20_ctr32_zbb_zvkb(out, inp, len, key, counter); - } else { - ChaCha20_ctr32_c(out, inp, len, key, counter); - } -} diff --git a/openssl/src/crypto/chacha/gen/darwin_arm64/chacha-armv8-sve.S b/openssl/src/crypto/chacha/gen/darwin_arm64/chacha-armv8-sve.S deleted file mode 100644 index 3a13652f8..000000000 --- a/openssl/src/crypto/chacha/gen/darwin_arm64/chacha-armv8-sve.S +++ /dev/null @@ -1,3868 +0,0 @@ -// Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// -// ChaCha20 for ARMv8 via SVE -// -// $output is the last argument if it looks like a file (it has an extension) -// $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" - - - - -.private_extern _OPENSSL_armcap_P - -.text -.align 5 -Lchacha20_consts: -.quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral -Lrot8: -.word 0x02010003,0x04040404,0x02010003,0x04040404 -.globl _ChaCha20_ctr32_sve - -.align 5 -_ChaCha20_ctr32_sve: - AARCH64_VALID_CALL_TARGET -.long 0x04a0e3e5 //cntw x5, ALL, MUL #1 - cmp x2,x5,lsl #6 - b.lt Lreturn - mov x7,0 - adrp x6,_OPENSSL_armcap_P@PAGE - ldr w6,[x6,_OPENSSL_armcap_P@PAGEOFF] - tst w6,#ARMV8_SVE2 - b.eq 1f - mov x7,1 - b 2f -1: - cmp x5,4 - b.le Lreturn - adr x6,Lrot8 - ldp w9,w10,[x6] -.long 0x04aa4d3f //index z31.s,w9,w10 -2: - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,-192]! - stp d10,d11,[sp,16] - stp d12,d13,[sp,32] - stp d14,d15,[sp,48] - stp x16,x17,[sp,64] - stp x18,x19,[sp,80] - stp x20,x21,[sp,96] - stp x22,x23,[sp,112] - stp x24,x25,[sp,128] - stp x26,x27,[sp,144] - stp x28,x29,[sp,160] - str x30,[sp,176] - - adr x6,Lchacha20_consts - ldp x23,x24,[x6] - ldp x25,x26,[x3] - ldp x27,x28,[x3, 16] - ldp x29,x30,[x4] -.long 0x2599e3e0 //ptrues p0.s,ALL -#ifdef __AARCH64EB__ - ror x25,x25,#32 - ror x26,x26,#32 - ror x27,x27,#32 - ror x28,x28,#32 - ror x29,x29,#32 - ror x30,x30,#32 -#endif - cbz x7, 1f -.align 5 -100: - subs x7,x2,x5,lsl #6 - b.lt 110f - mov x2,x7 - b.eq 101f - cmp x2,64 - b.lt 101f - mixin=1 - lsr x8,x23,#32 -.long 0x05a03ae0 //dup z0.s,w23 -.long 0x05a03af9 //dup z25.s,w23 -.if mixin == 1 - mov w7,w23 -.endif -.long 0x05a03904 //dup z4.s,w8 -.long 0x05a0391a //dup z26.s,w8 - lsr x10,x24,#32 -.long 0x05a03b08 //dup z8.s,w24 -.long 0x05a03b1b //dup z27.s,w24 -.if mixin == 1 - mov w9,w24 -.endif -.long 0x05a0394c //dup z12.s,w10 -.long 0x05a0395c //dup z28.s,w10 - lsr x12,x25,#32 -.long 0x05a03b21 //dup z1.s,w25 -.long 0x05a03b3d //dup z29.s,w25 -.if mixin == 1 - mov w11,w25 -.endif -.long 0x05a03985 //dup z5.s,w12 -.long 0x05a0399e //dup z30.s,w12 - lsr x14,x26,#32 -.long 0x05a03b49 //dup z9.s,w26 -.long 0x05a03b55 //dup z21.s,w26 -.if mixin == 1 - mov w13,w26 -.endif -.long 0x05a039cd //dup z13.s,w14 -.long 0x05a039d6 //dup z22.s,w14 - lsr x16,x27,#32 -.long 0x05a03b62 //dup z2.s,w27 -.long 0x05a03b77 //dup z23.s,w27 -.if mixin == 1 - mov w15,w27 -.endif -.long 0x05a03a06 //dup z6.s,w16 -.long 0x05a03a18 //dup z24.s,w16 - lsr x18,x28,#32 -.long 0x05a03b8a //dup z10.s,w28 -.long 0x05a03b91 //dup z17.s,w28 -.if mixin == 1 - mov w17,w28 -.endif -.long 0x05a03a4e //dup z14.s,w18 -.long 0x05a03a52 //dup z18.s,w18 - lsr x22,x30,#32 -.long 0x05a03bcb //dup z11.s,w30 -.long 0x05a03bd4 //dup z20.s,w30 -.if mixin == 1 - mov w21,w30 -.endif -.long 0x05a03acf //dup z15.s,w22 -.long 0x05a03adf //dup z31.s,w22 -.if mixin == 1 - add w20,w29,#1 - mov w19,w29 -.long 0x04a14690 //index z16.s,w20,1 -.long 0x04a14683 //index z3.s,w20,1 -.else -.long 0x04a147b0 //index z16.s,w29,1 -.long 0x04a147a3 //index z3.s,w29,1 -.endif - lsr x20,x29,#32 -.long 0x05a03a87 //dup z7.s,w20 -.long 0x05a03a93 //dup z19.s,w20 - mov x6,#10 -10: -.align 5 -.long 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.long 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.long 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.long 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.if mixin == 1 - eor w19,w19,w7 -.endif -.long 0x04703403 //xar z3.s,z3.s,z0.s,16 -.if mixin == 1 - ror w19,w19,16 -.endif -.if mixin == 1 - eor w20,w20,w8 -.endif -.long 0x04703487 //xar z7.s,z7.s,z4.s,16 -.if mixin == 1 - ror w20,w20,16 -.endif -.if mixin == 1 - eor w21,w21,w9 -.endif -.long 0x0470350b //xar z11.s,z11.s,z8.s,16 -.if mixin == 1 - ror w21,w21,16 -.endif -.if mixin == 1 - eor w22,w22,w10 -.endif -.long 0x0470358f //xar z15.s,z15.s,z12.s,16 -.if mixin == 1 - ror w22,w22,16 -.endif -.long 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.long 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.long 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.long 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.if mixin == 1 - eor w11,w11,w15 -.endif -.long 0x046c3441 //xar z1.s,z1.s,z2.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.if mixin == 1 - eor w12,w12,w16 -.endif -.long 0x046c34c5 //xar z5.s,z5.s,z6.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.if mixin == 1 - eor w13,w13,w17 -.endif -.long 0x046c3549 //xar z9.s,z9.s,z10.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.if mixin == 1 - eor w14,w14,w18 -.endif -.long 0x046c35cd //xar z13.s,z13.s,z14.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.long 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.long 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.long 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.long 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.if mixin == 1 - eor w19,w19,w7 -.endif -.long 0x04683403 //xar z3.s,z3.s,z0.s,24 -.if mixin == 1 - ror w19,w19,24 -.endif -.if mixin == 1 - eor w20,w20,w8 -.endif -.long 0x04683487 //xar z7.s,z7.s,z4.s,24 -.if mixin == 1 - ror w20,w20,24 -.endif -.if mixin == 1 - eor w21,w21,w9 -.endif -.long 0x0468350b //xar z11.s,z11.s,z8.s,24 -.if mixin == 1 - ror w21,w21,24 -.endif -.if mixin == 1 - eor w22,w22,w10 -.endif -.long 0x0468358f //xar z15.s,z15.s,z12.s,24 -.if mixin == 1 - ror w22,w22,24 -.endif -.long 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.long 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.long 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.long 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.if mixin == 1 - eor w11,w11,w15 -.endif -.long 0x04673441 //xar z1.s,z1.s,z2.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.if mixin == 1 - eor w12,w12,w16 -.endif -.long 0x046734c5 //xar z5.s,z5.s,z6.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.if mixin == 1 - eor w13,w13,w17 -.endif -.long 0x04673549 //xar z9.s,z9.s,z10.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.if mixin == 1 - eor w14,w14,w18 -.endif -.long 0x046735cd //xar z13.s,z13.s,z14.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.long 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.long 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.long 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.long 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.if mixin == 1 - eor w22,w22,w7 -.endif -.long 0x0470340f //xar z15.s,z15.s,z0.s,16 -.if mixin == 1 - ror w22,w22,16 -.endif -.if mixin == 1 - eor w19,w19,w8 -.endif -.long 0x04703483 //xar z3.s,z3.s,z4.s,16 -.if mixin == 1 - ror w19,w19,16 -.endif -.if mixin == 1 - eor w20,w20,w9 -.endif -.long 0x04703507 //xar z7.s,z7.s,z8.s,16 -.if mixin == 1 - ror w20,w20,16 -.endif -.if mixin == 1 - eor w21,w21,w10 -.endif -.long 0x0470358b //xar z11.s,z11.s,z12.s,16 -.if mixin == 1 - ror w21,w21,16 -.endif -.long 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.long 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.long 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.long 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.if mixin == 1 - eor w12,w12,w17 -.endif -.long 0x046c3545 //xar z5.s,z5.s,z10.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.if mixin == 1 - eor w13,w13,w18 -.endif -.long 0x046c35c9 //xar z9.s,z9.s,z14.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.if mixin == 1 - eor w14,w14,w15 -.endif -.long 0x046c344d //xar z13.s,z13.s,z2.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.if mixin == 1 - eor w11,w11,w16 -.endif -.long 0x046c34c1 //xar z1.s,z1.s,z6.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.long 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.long 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.long 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.long 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.if mixin == 1 - eor w22,w22,w7 -.endif -.long 0x0468340f //xar z15.s,z15.s,z0.s,24 -.if mixin == 1 - ror w22,w22,24 -.endif -.if mixin == 1 - eor w19,w19,w8 -.endif -.long 0x04683483 //xar z3.s,z3.s,z4.s,24 -.if mixin == 1 - ror w19,w19,24 -.endif -.if mixin == 1 - eor w20,w20,w9 -.endif -.long 0x04683507 //xar z7.s,z7.s,z8.s,24 -.if mixin == 1 - ror w20,w20,24 -.endif -.if mixin == 1 - eor w21,w21,w10 -.endif -.long 0x0468358b //xar z11.s,z11.s,z12.s,24 -.if mixin == 1 - ror w21,w21,24 -.endif -.long 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.long 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.long 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.long 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.if mixin == 1 - eor w12,w12,w17 -.endif -.long 0x04673545 //xar z5.s,z5.s,z10.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.if mixin == 1 - eor w13,w13,w18 -.endif -.long 0x046735c9 //xar z9.s,z9.s,z14.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.if mixin == 1 - eor w14,w14,w15 -.endif -.long 0x0467344d //xar z13.s,z13.s,z2.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.if mixin == 1 - eor w11,w11,w16 -.endif -.long 0x046734c1 //xar z1.s,z1.s,z6.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif - sub x6,x6,1 - cbnz x6,10b -.if mixin == 1 - add w7,w7,w23 -.endif -.long 0x04b90000 //add z0.s,z0.s,z25.s -.if mixin == 1 - add x8,x8,x23,lsr #32 -.endif -.long 0x04ba0084 //add z4.s,z4.s,z26.s -.if mixin == 1 - add x7,x7,x8,lsl #32 // pack -.endif -.if mixin == 1 - add w9,w9,w24 -.endif -.long 0x04bb0108 //add z8.s,z8.s,z27.s -.if mixin == 1 - add x10,x10,x24,lsr #32 -.endif -.long 0x04bc018c //add z12.s,z12.s,z28.s -.if mixin == 1 - add x9,x9,x10,lsl #32 // pack -.endif -.if mixin == 1 - ldp x8,x10,[x1],#16 -.endif -.if mixin == 1 - add w11,w11,w25 -.endif -.long 0x04bd0021 //add z1.s,z1.s,z29.s -.if mixin == 1 - add x12,x12,x25,lsr #32 -.endif -.long 0x04be00a5 //add z5.s,z5.s,z30.s -.if mixin == 1 - add x11,x11,x12,lsl #32 // pack -.endif -.if mixin == 1 - add w13,w13,w26 -.endif -.long 0x04b50129 //add z9.s,z9.s,z21.s -.if mixin == 1 - add x14,x14,x26,lsr #32 -.endif -.long 0x04b601ad //add z13.s,z13.s,z22.s -.if mixin == 1 - add x13,x13,x14,lsl #32 // pack -.endif -.if mixin == 1 - ldp x12,x14,[x1],#16 -.endif -.if mixin == 1 - add w15,w15,w27 -.endif -.long 0x04b70042 //add z2.s,z2.s,z23.s -.if mixin == 1 - add x16,x16,x27,lsr #32 -.endif -.long 0x04b800c6 //add z6.s,z6.s,z24.s -.if mixin == 1 - add x15,x15,x16,lsl #32 // pack -.endif -.if mixin == 1 - add w17,w17,w28 -.endif -.long 0x04b1014a //add z10.s,z10.s,z17.s -.if mixin == 1 - add x18,x18,x28,lsr #32 -.endif -.long 0x04b201ce //add z14.s,z14.s,z18.s -.if mixin == 1 - add x17,x17,x18,lsl #32 // pack -.endif -.if mixin == 1 - ldp x16,x18,[x1],#16 -.endif -.if mixin == 1 - add w19,w19,w29 -.endif -.long 0x04b00063 //add z3.s,z3.s,z16.s -.if mixin == 1 - add x20,x20,x29,lsr #32 -.endif -.long 0x04b300e7 //add z7.s,z7.s,z19.s -.if mixin == 1 - add x19,x19,x20,lsl #32 // pack -.endif -.if mixin == 1 - add w21,w21,w30 -.endif -.long 0x04b4016b //add z11.s,z11.s,z20.s -.if mixin == 1 - add x22,x22,x30,lsr #32 -.endif -.long 0x04bf01ef //add z15.s,z15.s,z31.s -.if mixin == 1 - add x21,x21,x22,lsl #32 // pack -.endif -.if mixin == 1 - ldp x20,x22,[x1],#16 -.endif -#ifdef __AARCH64EB__ - rev x7,x7 - rev x9,x9 - rev x11,x11 - rev x13,x13 - rev x15,x15 - rev x17,x17 - rev x19,x19 - rev x21,x21 -#endif -.if mixin == 1 - add x29,x29,#1 -.endif - cmp x5,4 - b.ne 200f -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.if mixin == 1 - eor x11,x11,x12 -.endif -.long 0x05a46011 //zip1 z17.s,z0.s,z4.s -.long 0x05a46412 //zip2 z18.s,z0.s,z4.s -.long 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.long 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.long 0x05a56035 //zip1 z21.s,z1.s,z5.s -.long 0x05a56436 //zip2 z22.s,z1.s,z5.s -.long 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.long 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36624 //zip2 z4.d,z17.d,z19.d -.long 0x05f46248 //zip1 z8.d,z18.d,z20.d -.long 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.long 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.long 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x13,x13,x14 -.endif -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.long 0x05a66051 //zip1 z17.s,z2.s,z6.s -.long 0x05a66452 //zip2 z18.s,z2.s,z6.s -.long 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.long 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.long 0x05a76075 //zip1 z21.s,z3.s,z7.s -.long 0x05a76476 //zip2 z22.s,z3.s,z7.s -.long 0x05af6177 //zip1 z23.s,z11.s,z15.s -.long 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.long 0x05f36222 //zip1 z2.d,z17.d,z19.d -.long 0x05f36626 //zip2 z6.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.long 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.long 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.long 0x05f862cb //zip1 z11.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.long 0x04b13000 //eor z0.d,z0.d,z17.d -.long 0x04b23021 //eor z1.d,z1.d,z18.d -.long 0x04b33042 //eor z2.d,z2.d,z19.d -.long 0x04b43063 //eor z3.d,z3.d,z20.d -.long 0x04b53084 //eor z4.d,z4.d,z21.d -.long 0x04b630a5 //eor z5.d,z5.d,z22.d -.long 0x04b730c6 //eor z6.d,z6.d,z23.d -.long 0x04b830e7 //eor z7.d,z7.d,z24.d - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.long 0x04b13108 //eor z8.d,z8.d,z17.d -.long 0x04b23129 //eor z9.d,z9.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.long 0x04b3314a //eor z10.d,z10.d,z19.d -.long 0x04b4316b //eor z11.d,z11.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.long 0x04b5318c //eor z12.d,z12.d,z21.d -.long 0x04b631ad //eor z13.d,z13.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.long 0x04b731ce //eor z14.d,z14.d,z23.d -.long 0x04b831ef //eor z15.d,z15.d,z24.d - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x0],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - st1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - st1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - b 210f -200: -.long 0x05a16011 //zip1 z17.s,z0.s,z1.s -.long 0x05a16412 //zip2 z18.s,z0.s,z1.s -.long 0x05a36053 //zip1 z19.s,z2.s,z3.s -.long 0x05a36454 //zip2 z20.s,z2.s,z3.s - -.long 0x05a56095 //zip1 z21.s,z4.s,z5.s -.long 0x05a56496 //zip2 z22.s,z4.s,z5.s -.long 0x05a760d7 //zip1 z23.s,z6.s,z7.s -.long 0x05a764d8 //zip2 z24.s,z6.s,z7.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36621 //zip2 z1.d,z17.d,z19.d -.long 0x05f46242 //zip1 z2.d,z18.d,z20.d -.long 0x05f46643 //zip2 z3.d,z18.d,z20.d - -.long 0x05f762a4 //zip1 z4.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c6 //zip1 z6.d,z22.d,z24.d -.long 0x05f866c7 //zip2 z7.d,z22.d,z24.d -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.long 0x05a96111 //zip1 z17.s,z8.s,z9.s -.long 0x05a96512 //zip2 z18.s,z8.s,z9.s -.long 0x05ab6153 //zip1 z19.s,z10.s,z11.s -.long 0x05ab6554 //zip2 z20.s,z10.s,z11.s - -.long 0x05ad6195 //zip1 z21.s,z12.s,z13.s -.long 0x05ad6596 //zip2 z22.s,z12.s,z13.s -.long 0x05af61d7 //zip1 z23.s,z14.s,z15.s -.long 0x05af65d8 //zip2 z24.s,z14.s,z15.s - -.long 0x05f36228 //zip1 z8.d,z17.d,z19.d -.long 0x05f36629 //zip2 z9.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664b //zip2 z11.d,z18.d,z20.d - -.long 0x05f762ac //zip1 z12.d,z21.d,z23.d -.long 0x05f766ad //zip2 z13.d,z21.d,z23.d -.long 0x05f862ce //zip1 z14.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x11,x11,x12 -.endif -.if mixin == 1 - eor x13,x13,x14 -.endif -.long 0x05a46011 //zip1 z17.s,z0.s,z4.s -.long 0x05a46412 //zip2 z18.s,z0.s,z4.s -.long 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.long 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.long 0x05a56035 //zip1 z21.s,z1.s,z5.s -.long 0x05a56436 //zip2 z22.s,z1.s,z5.s -.long 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.long 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36624 //zip2 z4.d,z17.d,z19.d -.long 0x05f46248 //zip1 z8.d,z18.d,z20.d -.long 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.long 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.long 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.long 0x05a66051 //zip1 z17.s,z2.s,z6.s -.long 0x05a66452 //zip2 z18.s,z2.s,z6.s -.long 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.long 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.long 0x05a76075 //zip1 z21.s,z3.s,z7.s -.long 0x05a76476 //zip2 z22.s,z3.s,z7.s -.long 0x05af6177 //zip1 z23.s,z11.s,z15.s -.long 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.long 0x05f36222 //zip1 z2.d,z17.d,z19.d -.long 0x05f36626 //zip2 z6.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.long 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.long 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.long 0x05f862cb //zip1 z11.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif -.long 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.long 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.long 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.long 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.long 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.long 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.long 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.long 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.long 0x04215101 //addvl x1,x1,8 -.long 0x04b13000 //eor z0.d,z0.d,z17.d -.long 0x04b23084 //eor z4.d,z4.d,z18.d -.long 0x04b33108 //eor z8.d,z8.d,z19.d -.long 0x04b4318c //eor z12.d,z12.d,z20.d -.long 0x04b53021 //eor z1.d,z1.d,z21.d -.long 0x04b630a5 //eor z5.d,z5.d,z22.d -.long 0x04b73129 //eor z9.d,z9.d,z23.d -.long 0x04b831ad //eor z13.d,z13.d,z24.d -.long 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.long 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.long 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.long 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.long 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.long 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.long 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.long 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.long 0x04215101 //addvl x1,x1,8 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.long 0x04b13042 //eor z2.d,z2.d,z17.d -.long 0x04b230c6 //eor z6.d,z6.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.long 0x04b3314a //eor z10.d,z10.d,z19.d -.long 0x04b431ce //eor z14.d,z14.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.long 0x04b53063 //eor z3.d,z3.d,z21.d -.long 0x04b630e7 //eor z7.d,z7.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.long 0x04b7316b //eor z11.d,z11.d,z23.d -.long 0x04b831ef //eor z15.d,z15.d,z24.d -#ifdef __AARCH64EB__ -.long 0x05a48000 //revb z0.s,p0/m,z0.s -#endif -.long 0xe540e000 //st1w {z0.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48084 //revb z4.s,p0/m,z4.s -#endif -.long 0xe541e004 //st1w {z4.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48108 //revb z8.s,p0/m,z8.s -#endif -.long 0xe542e008 //st1w {z8.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4818c //revb z12.s,p0/m,z12.s -#endif -.long 0xe543e00c //st1w {z12.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48021 //revb z1.s,p0/m,z1.s -#endif -.long 0xe544e001 //st1w {z1.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480a5 //revb z5.s,p0/m,z5.s -#endif -.long 0xe545e005 //st1w {z5.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48129 //revb z9.s,p0/m,z9.s -#endif -.long 0xe546e009 //st1w {z9.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ad //revb z13.s,p0/m,z13.s -#endif -.long 0xe547e00d //st1w {z13.s},p0,[x0,#7,MUL VL] -.long 0x04205100 //addvl x0,x0,8 -#ifdef __AARCH64EB__ -.long 0x05a48042 //revb z2.s,p0/m,z2.s -#endif -.long 0xe540e002 //st1w {z2.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480c6 //revb z6.s,p0/m,z6.s -#endif -.long 0xe541e006 //st1w {z6.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4814a //revb z10.s,p0/m,z10.s -#endif -.long 0xe542e00a //st1w {z10.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ce //revb z14.s,p0/m,z14.s -#endif -.long 0xe543e00e //st1w {z14.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48063 //revb z3.s,p0/m,z3.s -#endif -.long 0xe544e003 //st1w {z3.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480e7 //revb z7.s,p0/m,z7.s -#endif -.long 0xe545e007 //st1w {z7.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4816b //revb z11.s,p0/m,z11.s -#endif -.long 0xe546e00b //st1w {z11.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ef //revb z15.s,p0/m,z15.s -#endif -.long 0xe547e00f //st1w {z15.s},p0,[x0,#7,MUL VL] -.long 0x04205100 //addvl x0,x0,8 -210: -.long 0x04b0e3fd //incw x29, ALL, MUL #1 - subs x2,x2,64 - b.gt 100b - b 110f -101: - mixin=0 - lsr x8,x23,#32 -.long 0x05a03ae0 //dup z0.s,w23 -.long 0x05a03af9 //dup z25.s,w23 -.if mixin == 1 - mov w7,w23 -.endif -.long 0x05a03904 //dup z4.s,w8 -.long 0x05a0391a //dup z26.s,w8 - lsr x10,x24,#32 -.long 0x05a03b08 //dup z8.s,w24 -.long 0x05a03b1b //dup z27.s,w24 -.if mixin == 1 - mov w9,w24 -.endif -.long 0x05a0394c //dup z12.s,w10 -.long 0x05a0395c //dup z28.s,w10 - lsr x12,x25,#32 -.long 0x05a03b21 //dup z1.s,w25 -.long 0x05a03b3d //dup z29.s,w25 -.if mixin == 1 - mov w11,w25 -.endif -.long 0x05a03985 //dup z5.s,w12 -.long 0x05a0399e //dup z30.s,w12 - lsr x14,x26,#32 -.long 0x05a03b49 //dup z9.s,w26 -.long 0x05a03b55 //dup z21.s,w26 -.if mixin == 1 - mov w13,w26 -.endif -.long 0x05a039cd //dup z13.s,w14 -.long 0x05a039d6 //dup z22.s,w14 - lsr x16,x27,#32 -.long 0x05a03b62 //dup z2.s,w27 -.long 0x05a03b77 //dup z23.s,w27 -.if mixin == 1 - mov w15,w27 -.endif -.long 0x05a03a06 //dup z6.s,w16 -.long 0x05a03a18 //dup z24.s,w16 - lsr x18,x28,#32 -.long 0x05a03b8a //dup z10.s,w28 -.long 0x05a03b91 //dup z17.s,w28 -.if mixin == 1 - mov w17,w28 -.endif -.long 0x05a03a4e //dup z14.s,w18 -.long 0x05a03a52 //dup z18.s,w18 - lsr x22,x30,#32 -.long 0x05a03bcb //dup z11.s,w30 -.long 0x05a03bd4 //dup z20.s,w30 -.if mixin == 1 - mov w21,w30 -.endif -.long 0x05a03acf //dup z15.s,w22 -.long 0x05a03adf //dup z31.s,w22 -.if mixin == 1 - add w20,w29,#1 - mov w19,w29 -.long 0x04a14690 //index z16.s,w20,1 -.long 0x04a14683 //index z3.s,w20,1 -.else -.long 0x04a147b0 //index z16.s,w29,1 -.long 0x04a147a3 //index z3.s,w29,1 -.endif - lsr x20,x29,#32 -.long 0x05a03a87 //dup z7.s,w20 -.long 0x05a03a93 //dup z19.s,w20 - mov x6,#10 -10: -.align 5 -.long 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.long 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.long 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.long 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.if mixin == 1 - eor w19,w19,w7 -.endif -.long 0x04703403 //xar z3.s,z3.s,z0.s,16 -.if mixin == 1 - ror w19,w19,16 -.endif -.if mixin == 1 - eor w20,w20,w8 -.endif -.long 0x04703487 //xar z7.s,z7.s,z4.s,16 -.if mixin == 1 - ror w20,w20,16 -.endif -.if mixin == 1 - eor w21,w21,w9 -.endif -.long 0x0470350b //xar z11.s,z11.s,z8.s,16 -.if mixin == 1 - ror w21,w21,16 -.endif -.if mixin == 1 - eor w22,w22,w10 -.endif -.long 0x0470358f //xar z15.s,z15.s,z12.s,16 -.if mixin == 1 - ror w22,w22,16 -.endif -.long 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.long 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.long 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.long 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.if mixin == 1 - eor w11,w11,w15 -.endif -.long 0x046c3441 //xar z1.s,z1.s,z2.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.if mixin == 1 - eor w12,w12,w16 -.endif -.long 0x046c34c5 //xar z5.s,z5.s,z6.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.if mixin == 1 - eor w13,w13,w17 -.endif -.long 0x046c3549 //xar z9.s,z9.s,z10.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.if mixin == 1 - eor w14,w14,w18 -.endif -.long 0x046c35cd //xar z13.s,z13.s,z14.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.long 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.long 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.long 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.long 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.if mixin == 1 - eor w19,w19,w7 -.endif -.long 0x04683403 //xar z3.s,z3.s,z0.s,24 -.if mixin == 1 - ror w19,w19,24 -.endif -.if mixin == 1 - eor w20,w20,w8 -.endif -.long 0x04683487 //xar z7.s,z7.s,z4.s,24 -.if mixin == 1 - ror w20,w20,24 -.endif -.if mixin == 1 - eor w21,w21,w9 -.endif -.long 0x0468350b //xar z11.s,z11.s,z8.s,24 -.if mixin == 1 - ror w21,w21,24 -.endif -.if mixin == 1 - eor w22,w22,w10 -.endif -.long 0x0468358f //xar z15.s,z15.s,z12.s,24 -.if mixin == 1 - ror w22,w22,24 -.endif -.long 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.long 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.long 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.long 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.if mixin == 1 - eor w11,w11,w15 -.endif -.long 0x04673441 //xar z1.s,z1.s,z2.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.if mixin == 1 - eor w12,w12,w16 -.endif -.long 0x046734c5 //xar z5.s,z5.s,z6.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.if mixin == 1 - eor w13,w13,w17 -.endif -.long 0x04673549 //xar z9.s,z9.s,z10.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.if mixin == 1 - eor w14,w14,w18 -.endif -.long 0x046735cd //xar z13.s,z13.s,z14.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.long 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.long 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.long 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.long 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.if mixin == 1 - eor w22,w22,w7 -.endif -.long 0x0470340f //xar z15.s,z15.s,z0.s,16 -.if mixin == 1 - ror w22,w22,16 -.endif -.if mixin == 1 - eor w19,w19,w8 -.endif -.long 0x04703483 //xar z3.s,z3.s,z4.s,16 -.if mixin == 1 - ror w19,w19,16 -.endif -.if mixin == 1 - eor w20,w20,w9 -.endif -.long 0x04703507 //xar z7.s,z7.s,z8.s,16 -.if mixin == 1 - ror w20,w20,16 -.endif -.if mixin == 1 - eor w21,w21,w10 -.endif -.long 0x0470358b //xar z11.s,z11.s,z12.s,16 -.if mixin == 1 - ror w21,w21,16 -.endif -.long 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.long 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.long 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.long 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.if mixin == 1 - eor w12,w12,w17 -.endif -.long 0x046c3545 //xar z5.s,z5.s,z10.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.if mixin == 1 - eor w13,w13,w18 -.endif -.long 0x046c35c9 //xar z9.s,z9.s,z14.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.if mixin == 1 - eor w14,w14,w15 -.endif -.long 0x046c344d //xar z13.s,z13.s,z2.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.if mixin == 1 - eor w11,w11,w16 -.endif -.long 0x046c34c1 //xar z1.s,z1.s,z6.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.long 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.long 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.long 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.long 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.if mixin == 1 - eor w22,w22,w7 -.endif -.long 0x0468340f //xar z15.s,z15.s,z0.s,24 -.if mixin == 1 - ror w22,w22,24 -.endif -.if mixin == 1 - eor w19,w19,w8 -.endif -.long 0x04683483 //xar z3.s,z3.s,z4.s,24 -.if mixin == 1 - ror w19,w19,24 -.endif -.if mixin == 1 - eor w20,w20,w9 -.endif -.long 0x04683507 //xar z7.s,z7.s,z8.s,24 -.if mixin == 1 - ror w20,w20,24 -.endif -.if mixin == 1 - eor w21,w21,w10 -.endif -.long 0x0468358b //xar z11.s,z11.s,z12.s,24 -.if mixin == 1 - ror w21,w21,24 -.endif -.long 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.long 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.long 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.long 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.if mixin == 1 - eor w12,w12,w17 -.endif -.long 0x04673545 //xar z5.s,z5.s,z10.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.if mixin == 1 - eor w13,w13,w18 -.endif -.long 0x046735c9 //xar z9.s,z9.s,z14.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.if mixin == 1 - eor w14,w14,w15 -.endif -.long 0x0467344d //xar z13.s,z13.s,z2.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.if mixin == 1 - eor w11,w11,w16 -.endif -.long 0x046734c1 //xar z1.s,z1.s,z6.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif - sub x6,x6,1 - cbnz x6,10b -.if mixin == 1 - add w7,w7,w23 -.endif -.long 0x04b90000 //add z0.s,z0.s,z25.s -.if mixin == 1 - add x8,x8,x23,lsr #32 -.endif -.long 0x04ba0084 //add z4.s,z4.s,z26.s -.if mixin == 1 - add x7,x7,x8,lsl #32 // pack -.endif -.if mixin == 1 - add w9,w9,w24 -.endif -.long 0x04bb0108 //add z8.s,z8.s,z27.s -.if mixin == 1 - add x10,x10,x24,lsr #32 -.endif -.long 0x04bc018c //add z12.s,z12.s,z28.s -.if mixin == 1 - add x9,x9,x10,lsl #32 // pack -.endif -.if mixin == 1 - ldp x8,x10,[x1],#16 -.endif -.if mixin == 1 - add w11,w11,w25 -.endif -.long 0x04bd0021 //add z1.s,z1.s,z29.s -.if mixin == 1 - add x12,x12,x25,lsr #32 -.endif -.long 0x04be00a5 //add z5.s,z5.s,z30.s -.if mixin == 1 - add x11,x11,x12,lsl #32 // pack -.endif -.if mixin == 1 - add w13,w13,w26 -.endif -.long 0x04b50129 //add z9.s,z9.s,z21.s -.if mixin == 1 - add x14,x14,x26,lsr #32 -.endif -.long 0x04b601ad //add z13.s,z13.s,z22.s -.if mixin == 1 - add x13,x13,x14,lsl #32 // pack -.endif -.if mixin == 1 - ldp x12,x14,[x1],#16 -.endif -.if mixin == 1 - add w15,w15,w27 -.endif -.long 0x04b70042 //add z2.s,z2.s,z23.s -.if mixin == 1 - add x16,x16,x27,lsr #32 -.endif -.long 0x04b800c6 //add z6.s,z6.s,z24.s -.if mixin == 1 - add x15,x15,x16,lsl #32 // pack -.endif -.if mixin == 1 - add w17,w17,w28 -.endif -.long 0x04b1014a //add z10.s,z10.s,z17.s -.if mixin == 1 - add x18,x18,x28,lsr #32 -.endif -.long 0x04b201ce //add z14.s,z14.s,z18.s -.if mixin == 1 - add x17,x17,x18,lsl #32 // pack -.endif -.if mixin == 1 - ldp x16,x18,[x1],#16 -.endif -.if mixin == 1 - add w19,w19,w29 -.endif -.long 0x04b00063 //add z3.s,z3.s,z16.s -.if mixin == 1 - add x20,x20,x29,lsr #32 -.endif -.long 0x04b300e7 //add z7.s,z7.s,z19.s -.if mixin == 1 - add x19,x19,x20,lsl #32 // pack -.endif -.if mixin == 1 - add w21,w21,w30 -.endif -.long 0x04b4016b //add z11.s,z11.s,z20.s -.if mixin == 1 - add x22,x22,x30,lsr #32 -.endif -.long 0x04bf01ef //add z15.s,z15.s,z31.s -.if mixin == 1 - add x21,x21,x22,lsl #32 // pack -.endif -.if mixin == 1 - ldp x20,x22,[x1],#16 -.endif -#ifdef __AARCH64EB__ - rev x7,x7 - rev x9,x9 - rev x11,x11 - rev x13,x13 - rev x15,x15 - rev x17,x17 - rev x19,x19 - rev x21,x21 -#endif -.if mixin == 1 - add x29,x29,#1 -.endif - cmp x5,4 - b.ne 200f -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.if mixin == 1 - eor x11,x11,x12 -.endif -.long 0x05a46011 //zip1 z17.s,z0.s,z4.s -.long 0x05a46412 //zip2 z18.s,z0.s,z4.s -.long 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.long 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.long 0x05a56035 //zip1 z21.s,z1.s,z5.s -.long 0x05a56436 //zip2 z22.s,z1.s,z5.s -.long 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.long 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36624 //zip2 z4.d,z17.d,z19.d -.long 0x05f46248 //zip1 z8.d,z18.d,z20.d -.long 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.long 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.long 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x13,x13,x14 -.endif -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.long 0x05a66051 //zip1 z17.s,z2.s,z6.s -.long 0x05a66452 //zip2 z18.s,z2.s,z6.s -.long 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.long 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.long 0x05a76075 //zip1 z21.s,z3.s,z7.s -.long 0x05a76476 //zip2 z22.s,z3.s,z7.s -.long 0x05af6177 //zip1 z23.s,z11.s,z15.s -.long 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.long 0x05f36222 //zip1 z2.d,z17.d,z19.d -.long 0x05f36626 //zip2 z6.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.long 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.long 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.long 0x05f862cb //zip1 z11.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.long 0x04b13000 //eor z0.d,z0.d,z17.d -.long 0x04b23021 //eor z1.d,z1.d,z18.d -.long 0x04b33042 //eor z2.d,z2.d,z19.d -.long 0x04b43063 //eor z3.d,z3.d,z20.d -.long 0x04b53084 //eor z4.d,z4.d,z21.d -.long 0x04b630a5 //eor z5.d,z5.d,z22.d -.long 0x04b730c6 //eor z6.d,z6.d,z23.d -.long 0x04b830e7 //eor z7.d,z7.d,z24.d - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.long 0x04b13108 //eor z8.d,z8.d,z17.d -.long 0x04b23129 //eor z9.d,z9.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.long 0x04b3314a //eor z10.d,z10.d,z19.d -.long 0x04b4316b //eor z11.d,z11.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.long 0x04b5318c //eor z12.d,z12.d,z21.d -.long 0x04b631ad //eor z13.d,z13.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.long 0x04b731ce //eor z14.d,z14.d,z23.d -.long 0x04b831ef //eor z15.d,z15.d,z24.d - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x0],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - st1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - st1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - b 210f -200: -.long 0x05a16011 //zip1 z17.s,z0.s,z1.s -.long 0x05a16412 //zip2 z18.s,z0.s,z1.s -.long 0x05a36053 //zip1 z19.s,z2.s,z3.s -.long 0x05a36454 //zip2 z20.s,z2.s,z3.s - -.long 0x05a56095 //zip1 z21.s,z4.s,z5.s -.long 0x05a56496 //zip2 z22.s,z4.s,z5.s -.long 0x05a760d7 //zip1 z23.s,z6.s,z7.s -.long 0x05a764d8 //zip2 z24.s,z6.s,z7.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36621 //zip2 z1.d,z17.d,z19.d -.long 0x05f46242 //zip1 z2.d,z18.d,z20.d -.long 0x05f46643 //zip2 z3.d,z18.d,z20.d - -.long 0x05f762a4 //zip1 z4.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c6 //zip1 z6.d,z22.d,z24.d -.long 0x05f866c7 //zip2 z7.d,z22.d,z24.d -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.long 0x05a96111 //zip1 z17.s,z8.s,z9.s -.long 0x05a96512 //zip2 z18.s,z8.s,z9.s -.long 0x05ab6153 //zip1 z19.s,z10.s,z11.s -.long 0x05ab6554 //zip2 z20.s,z10.s,z11.s - -.long 0x05ad6195 //zip1 z21.s,z12.s,z13.s -.long 0x05ad6596 //zip2 z22.s,z12.s,z13.s -.long 0x05af61d7 //zip1 z23.s,z14.s,z15.s -.long 0x05af65d8 //zip2 z24.s,z14.s,z15.s - -.long 0x05f36228 //zip1 z8.d,z17.d,z19.d -.long 0x05f36629 //zip2 z9.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664b //zip2 z11.d,z18.d,z20.d - -.long 0x05f762ac //zip1 z12.d,z21.d,z23.d -.long 0x05f766ad //zip2 z13.d,z21.d,z23.d -.long 0x05f862ce //zip1 z14.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x11,x11,x12 -.endif -.if mixin == 1 - eor x13,x13,x14 -.endif -.long 0x05a46011 //zip1 z17.s,z0.s,z4.s -.long 0x05a46412 //zip2 z18.s,z0.s,z4.s -.long 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.long 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.long 0x05a56035 //zip1 z21.s,z1.s,z5.s -.long 0x05a56436 //zip2 z22.s,z1.s,z5.s -.long 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.long 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36624 //zip2 z4.d,z17.d,z19.d -.long 0x05f46248 //zip1 z8.d,z18.d,z20.d -.long 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.long 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.long 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.long 0x05a66051 //zip1 z17.s,z2.s,z6.s -.long 0x05a66452 //zip2 z18.s,z2.s,z6.s -.long 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.long 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.long 0x05a76075 //zip1 z21.s,z3.s,z7.s -.long 0x05a76476 //zip2 z22.s,z3.s,z7.s -.long 0x05af6177 //zip1 z23.s,z11.s,z15.s -.long 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.long 0x05f36222 //zip1 z2.d,z17.d,z19.d -.long 0x05f36626 //zip2 z6.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.long 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.long 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.long 0x05f862cb //zip1 z11.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif -.long 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.long 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.long 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.long 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.long 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.long 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.long 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.long 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.long 0x04215101 //addvl x1,x1,8 -.long 0x04b13000 //eor z0.d,z0.d,z17.d -.long 0x04b23084 //eor z4.d,z4.d,z18.d -.long 0x04b33108 //eor z8.d,z8.d,z19.d -.long 0x04b4318c //eor z12.d,z12.d,z20.d -.long 0x04b53021 //eor z1.d,z1.d,z21.d -.long 0x04b630a5 //eor z5.d,z5.d,z22.d -.long 0x04b73129 //eor z9.d,z9.d,z23.d -.long 0x04b831ad //eor z13.d,z13.d,z24.d -.long 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.long 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.long 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.long 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.long 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.long 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.long 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.long 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.long 0x04215101 //addvl x1,x1,8 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.long 0x04b13042 //eor z2.d,z2.d,z17.d -.long 0x04b230c6 //eor z6.d,z6.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.long 0x04b3314a //eor z10.d,z10.d,z19.d -.long 0x04b431ce //eor z14.d,z14.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.long 0x04b53063 //eor z3.d,z3.d,z21.d -.long 0x04b630e7 //eor z7.d,z7.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.long 0x04b7316b //eor z11.d,z11.d,z23.d -.long 0x04b831ef //eor z15.d,z15.d,z24.d -#ifdef __AARCH64EB__ -.long 0x05a48000 //revb z0.s,p0/m,z0.s -#endif -.long 0xe540e000 //st1w {z0.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48084 //revb z4.s,p0/m,z4.s -#endif -.long 0xe541e004 //st1w {z4.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48108 //revb z8.s,p0/m,z8.s -#endif -.long 0xe542e008 //st1w {z8.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4818c //revb z12.s,p0/m,z12.s -#endif -.long 0xe543e00c //st1w {z12.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48021 //revb z1.s,p0/m,z1.s -#endif -.long 0xe544e001 //st1w {z1.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480a5 //revb z5.s,p0/m,z5.s -#endif -.long 0xe545e005 //st1w {z5.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48129 //revb z9.s,p0/m,z9.s -#endif -.long 0xe546e009 //st1w {z9.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ad //revb z13.s,p0/m,z13.s -#endif -.long 0xe547e00d //st1w {z13.s},p0,[x0,#7,MUL VL] -.long 0x04205100 //addvl x0,x0,8 -#ifdef __AARCH64EB__ -.long 0x05a48042 //revb z2.s,p0/m,z2.s -#endif -.long 0xe540e002 //st1w {z2.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480c6 //revb z6.s,p0/m,z6.s -#endif -.long 0xe541e006 //st1w {z6.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4814a //revb z10.s,p0/m,z10.s -#endif -.long 0xe542e00a //st1w {z10.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ce //revb z14.s,p0/m,z14.s -#endif -.long 0xe543e00e //st1w {z14.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48063 //revb z3.s,p0/m,z3.s -#endif -.long 0xe544e003 //st1w {z3.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480e7 //revb z7.s,p0/m,z7.s -#endif -.long 0xe545e007 //st1w {z7.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4816b //revb z11.s,p0/m,z11.s -#endif -.long 0xe546e00b //st1w {z11.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ef //revb z15.s,p0/m,z15.s -#endif -.long 0xe547e00f //st1w {z15.s},p0,[x0,#7,MUL VL] -.long 0x04205100 //addvl x0,x0,8 -210: -.long 0x04b0e3fd //incw x29, ALL, MUL #1 -110: - b 2f -1: -.align 5 -100: - subs x7,x2,x5,lsl #6 - b.lt 110f - mov x2,x7 - b.eq 101f - cmp x2,64 - b.lt 101f - mixin=1 - lsr x8,x23,#32 -.long 0x05a03ae0 //dup z0.s,w23 -.long 0x05a03af9 //dup z25.s,w23 -.if mixin == 1 - mov w7,w23 -.endif -.long 0x05a03904 //dup z4.s,w8 -.long 0x05a0391a //dup z26.s,w8 - lsr x10,x24,#32 -.long 0x05a03b08 //dup z8.s,w24 -.long 0x05a03b1b //dup z27.s,w24 -.if mixin == 1 - mov w9,w24 -.endif -.long 0x05a0394c //dup z12.s,w10 -.long 0x05a0395c //dup z28.s,w10 - lsr x12,x25,#32 -.long 0x05a03b21 //dup z1.s,w25 -.long 0x05a03b3d //dup z29.s,w25 -.if mixin == 1 - mov w11,w25 -.endif -.long 0x05a03985 //dup z5.s,w12 -.long 0x05a0399e //dup z30.s,w12 - lsr x14,x26,#32 -.long 0x05a03b49 //dup z9.s,w26 -.long 0x05a03b55 //dup z21.s,w26 -.if mixin == 1 - mov w13,w26 -.endif -.long 0x05a039cd //dup z13.s,w14 -.long 0x05a039d6 //dup z22.s,w14 - lsr x16,x27,#32 -.long 0x05a03b62 //dup z2.s,w27 -.long 0x05a03b77 //dup z23.s,w27 -.if mixin == 1 - mov w15,w27 -.endif -.long 0x05a03a06 //dup z6.s,w16 -.long 0x05a03a18 //dup z24.s,w16 - lsr x18,x28,#32 -.long 0x05a03b8a //dup z10.s,w28 -.if mixin == 1 - mov w17,w28 -.endif -.long 0x05a03a4e //dup z14.s,w18 - lsr x22,x30,#32 -.long 0x05a03bcb //dup z11.s,w30 -.if mixin == 1 - mov w21,w30 -.endif -.long 0x05a03acf //dup z15.s,w22 -.if mixin == 1 - add w20,w29,#1 - mov w19,w29 -.long 0x04a14690 //index z16.s,w20,1 -.long 0x04a14683 //index z3.s,w20,1 -.else -.long 0x04a147b0 //index z16.s,w29,1 -.long 0x04a147a3 //index z3.s,w29,1 -.endif - lsr x20,x29,#32 -.long 0x05a03a87 //dup z7.s,w20 - mov x6,#10 -10: -.align 5 -.long 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.long 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.long 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.long 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.long 0x04a03063 //eor z3.d,z3.d,z0.d -.if mixin == 1 - eor w19,w19,w7 -.endif -.long 0x04a430e7 //eor z7.d,z7.d,z4.d -.if mixin == 1 - eor w20,w20,w8 -.endif -.long 0x04a8316b //eor z11.d,z11.d,z8.d -.if mixin == 1 - eor w21,w21,w9 -.endif -.long 0x04ac31ef //eor z15.d,z15.d,z12.d -.if mixin == 1 - eor w22,w22,w10 -.endif -.long 0x05a58063 //revh z3.s,p0/m,z3.s -.if mixin == 1 - ror w19,w19,#16 -.endif -.long 0x05a580e7 //revh z7.s,p0/m,z7.s -.if mixin == 1 - ror w20,w20,#16 -.endif -.long 0x05a5816b //revh z11.s,p0/m,z11.s -.if mixin == 1 - ror w21,w21,#16 -.endif -.long 0x05a581ef //revh z15.s,p0/m,z15.s -.if mixin == 1 - ror w22,w22,#16 -.endif -.long 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.long 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.long 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.long 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.long 0x04a23021 //eor z1.d,z1.d,z2.d -.if mixin == 1 - eor w11,w11,w15 -.endif -.long 0x04a630a5 //eor z5.d,z5.d,z6.d -.if mixin == 1 - eor w12,w12,w16 -.endif -.long 0x04aa3129 //eor z9.d,z9.d,z10.d -.if mixin == 1 - eor w13,w13,w17 -.endif -.long 0x04ae31ad //eor z13.d,z13.d,z14.d -.if mixin == 1 - eor w14,w14,w18 -.endif -.long 0x046c9c31 //lsl z17.s,z1.s,12 -.long 0x046c9cb2 //lsl z18.s,z5.s,12 -.long 0x046c9d33 //lsl z19.s,z9.s,12 -.long 0x046c9db4 //lsl z20.s,z13.s,12 -.long 0x046c9421 //lsr z1.s,z1.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.long 0x046c94a5 //lsr z5.s,z5.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.long 0x046c9529 //lsr z9.s,z9.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.long 0x046c95ad //lsr z13.s,z13.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.long 0x04713021 //orr z1.d,z1.d,z17.d -.long 0x047230a5 //orr z5.d,z5.d,z18.d -.long 0x04733129 //orr z9.d,z9.d,z19.d -.long 0x047431ad //orr z13.d,z13.d,z20.d -.long 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.long 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.long 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.long 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.long 0x04a03063 //eor z3.d,z3.d,z0.d -.if mixin == 1 - eor w19,w19,w7 -.endif -.long 0x04a430e7 //eor z7.d,z7.d,z4.d -.if mixin == 1 - eor w20,w20,w8 -.endif -.long 0x04a8316b //eor z11.d,z11.d,z8.d -.if mixin == 1 - eor w21,w21,w9 -.endif -.long 0x04ac31ef //eor z15.d,z15.d,z12.d -.if mixin == 1 - eor w22,w22,w10 -.endif -.long 0x053f3063 //tbl z3.b,{z3.b},z31.b -.if mixin == 1 - ror w19,w19,#24 -.endif -.long 0x053f30e7 //tbl z7.b,{z7.b},z31.b -.if mixin == 1 - ror w20,w20,#24 -.endif -.long 0x053f316b //tbl z11.b,{z11.b},z31.b -.if mixin == 1 - ror w21,w21,#24 -.endif -.long 0x053f31ef //tbl z15.b,{z15.b},z31.b -.if mixin == 1 - ror w22,w22,#24 -.endif -.long 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.long 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.long 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.long 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.long 0x04a23021 //eor z1.d,z1.d,z2.d -.if mixin == 1 - eor w11,w11,w15 -.endif -.long 0x04a630a5 //eor z5.d,z5.d,z6.d -.if mixin == 1 - eor w12,w12,w16 -.endif -.long 0x04aa3129 //eor z9.d,z9.d,z10.d -.if mixin == 1 - eor w13,w13,w17 -.endif -.long 0x04ae31ad //eor z13.d,z13.d,z14.d -.if mixin == 1 - eor w14,w14,w18 -.endif -.long 0x04679c31 //lsl z17.s,z1.s,7 -.long 0x04679cb2 //lsl z18.s,z5.s,7 -.long 0x04679d33 //lsl z19.s,z9.s,7 -.long 0x04679db4 //lsl z20.s,z13.s,7 -.long 0x04679421 //lsr z1.s,z1.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.long 0x046794a5 //lsr z5.s,z5.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.long 0x04679529 //lsr z9.s,z9.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.long 0x046795ad //lsr z13.s,z13.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.long 0x04713021 //orr z1.d,z1.d,z17.d -.long 0x047230a5 //orr z5.d,z5.d,z18.d -.long 0x04733129 //orr z9.d,z9.d,z19.d -.long 0x047431ad //orr z13.d,z13.d,z20.d -.long 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.long 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.long 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.long 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.long 0x04a031ef //eor z15.d,z15.d,z0.d -.if mixin == 1 - eor w22,w22,w7 -.endif -.long 0x04a43063 //eor z3.d,z3.d,z4.d -.if mixin == 1 - eor w19,w19,w8 -.endif -.long 0x04a830e7 //eor z7.d,z7.d,z8.d -.if mixin == 1 - eor w20,w20,w9 -.endif -.long 0x04ac316b //eor z11.d,z11.d,z12.d -.if mixin == 1 - eor w21,w21,w10 -.endif -.long 0x05a581ef //revh z15.s,p0/m,z15.s -.if mixin == 1 - ror w22,w22,#16 -.endif -.long 0x05a58063 //revh z3.s,p0/m,z3.s -.if mixin == 1 - ror w19,w19,#16 -.endif -.long 0x05a580e7 //revh z7.s,p0/m,z7.s -.if mixin == 1 - ror w20,w20,#16 -.endif -.long 0x05a5816b //revh z11.s,p0/m,z11.s -.if mixin == 1 - ror w21,w21,#16 -.endif -.long 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.long 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.long 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.long 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.long 0x04aa30a5 //eor z5.d,z5.d,z10.d -.if mixin == 1 - eor w12,w12,w17 -.endif -.long 0x04ae3129 //eor z9.d,z9.d,z14.d -.if mixin == 1 - eor w13,w13,w18 -.endif -.long 0x04a231ad //eor z13.d,z13.d,z2.d -.if mixin == 1 - eor w14,w14,w15 -.endif -.long 0x04a63021 //eor z1.d,z1.d,z6.d -.if mixin == 1 - eor w11,w11,w16 -.endif -.long 0x046c9cb1 //lsl z17.s,z5.s,12 -.long 0x046c9d32 //lsl z18.s,z9.s,12 -.long 0x046c9db3 //lsl z19.s,z13.s,12 -.long 0x046c9c34 //lsl z20.s,z1.s,12 -.long 0x046c94a5 //lsr z5.s,z5.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.long 0x046c9529 //lsr z9.s,z9.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.long 0x046c95ad //lsr z13.s,z13.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.long 0x046c9421 //lsr z1.s,z1.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.long 0x047130a5 //orr z5.d,z5.d,z17.d -.long 0x04723129 //orr z9.d,z9.d,z18.d -.long 0x047331ad //orr z13.d,z13.d,z19.d -.long 0x04743021 //orr z1.d,z1.d,z20.d -.long 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.long 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.long 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.long 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.long 0x04a031ef //eor z15.d,z15.d,z0.d -.if mixin == 1 - eor w22,w22,w7 -.endif -.long 0x04a43063 //eor z3.d,z3.d,z4.d -.if mixin == 1 - eor w19,w19,w8 -.endif -.long 0x04a830e7 //eor z7.d,z7.d,z8.d -.if mixin == 1 - eor w20,w20,w9 -.endif -.long 0x04ac316b //eor z11.d,z11.d,z12.d -.if mixin == 1 - eor w21,w21,w10 -.endif -.long 0x053f31ef //tbl z15.b,{z15.b},z31.b -.if mixin == 1 - ror w22,w22,#24 -.endif -.long 0x053f3063 //tbl z3.b,{z3.b},z31.b -.if mixin == 1 - ror w19,w19,#24 -.endif -.long 0x053f30e7 //tbl z7.b,{z7.b},z31.b -.if mixin == 1 - ror w20,w20,#24 -.endif -.long 0x053f316b //tbl z11.b,{z11.b},z31.b -.if mixin == 1 - ror w21,w21,#24 -.endif -.long 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.long 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.long 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.long 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.long 0x04aa30a5 //eor z5.d,z5.d,z10.d -.if mixin == 1 - eor w12,w12,w17 -.endif -.long 0x04ae3129 //eor z9.d,z9.d,z14.d -.if mixin == 1 - eor w13,w13,w18 -.endif -.long 0x04a231ad //eor z13.d,z13.d,z2.d -.if mixin == 1 - eor w14,w14,w15 -.endif -.long 0x04a63021 //eor z1.d,z1.d,z6.d -.if mixin == 1 - eor w11,w11,w16 -.endif -.long 0x04679cb1 //lsl z17.s,z5.s,7 -.long 0x04679d32 //lsl z18.s,z9.s,7 -.long 0x04679db3 //lsl z19.s,z13.s,7 -.long 0x04679c34 //lsl z20.s,z1.s,7 -.long 0x046794a5 //lsr z5.s,z5.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.long 0x04679529 //lsr z9.s,z9.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.long 0x046795ad //lsr z13.s,z13.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.long 0x04679421 //lsr z1.s,z1.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.long 0x047130a5 //orr z5.d,z5.d,z17.d -.long 0x04723129 //orr z9.d,z9.d,z18.d -.long 0x047331ad //orr z13.d,z13.d,z19.d -.long 0x04743021 //orr z1.d,z1.d,z20.d - sub x6,x6,1 - cbnz x6,10b - lsr x6,x28,#32 -.long 0x05a03b91 //dup z17.s,w28 -.long 0x05a038d2 //dup z18.s,w6 - lsr x6,x29,#32 -.long 0x05a038d3 //dup z19.s,w6 - lsr x6,x30,#32 -.if mixin == 1 - add w7,w7,w23 -.endif -.long 0x04b90000 //add z0.s,z0.s,z25.s -.if mixin == 1 - add x8,x8,x23,lsr #32 -.endif -.long 0x04ba0084 //add z4.s,z4.s,z26.s -.if mixin == 1 - add x7,x7,x8,lsl #32 // pack -.endif -.if mixin == 1 - add w9,w9,w24 -.endif -.long 0x04bb0108 //add z8.s,z8.s,z27.s -.if mixin == 1 - add x10,x10,x24,lsr #32 -.endif -.long 0x04bc018c //add z12.s,z12.s,z28.s -.if mixin == 1 - add x9,x9,x10,lsl #32 // pack -.endif -.if mixin == 1 - ldp x8,x10,[x1],#16 -.endif -.if mixin == 1 - add w11,w11,w25 -.endif -.long 0x04bd0021 //add z1.s,z1.s,z29.s -.if mixin == 1 - add x12,x12,x25,lsr #32 -.endif -.long 0x04be00a5 //add z5.s,z5.s,z30.s -.if mixin == 1 - add x11,x11,x12,lsl #32 // pack -.endif -.if mixin == 1 - add w13,w13,w26 -.endif -.long 0x04b50129 //add z9.s,z9.s,z21.s -.if mixin == 1 - add x14,x14,x26,lsr #32 -.endif -.long 0x04b601ad //add z13.s,z13.s,z22.s -.if mixin == 1 - add x13,x13,x14,lsl #32 // pack -.endif -.if mixin == 1 - ldp x12,x14,[x1],#16 -.endif -.if mixin == 1 - add w15,w15,w27 -.endif -.long 0x04b70042 //add z2.s,z2.s,z23.s -.if mixin == 1 - add x16,x16,x27,lsr #32 -.endif -.long 0x04b800c6 //add z6.s,z6.s,z24.s -.if mixin == 1 - add x15,x15,x16,lsl #32 // pack -.endif -.if mixin == 1 - add w17,w17,w28 -.endif -.long 0x04b1014a //add z10.s,z10.s,z17.s -.if mixin == 1 - add x18,x18,x28,lsr #32 -.endif -.long 0x04b201ce //add z14.s,z14.s,z18.s -.if mixin == 1 - add x17,x17,x18,lsl #32 // pack -.endif -.if mixin == 1 - ldp x16,x18,[x1],#16 -.endif -.long 0x05a03bd4 //dup z20.s,w30 -.long 0x05a038d9 //dup z25.s,w6 // bak[15] not available for SVE -.if mixin == 1 - add w19,w19,w29 -.endif -.long 0x04b00063 //add z3.s,z3.s,z16.s -.if mixin == 1 - add x20,x20,x29,lsr #32 -.endif -.long 0x04b300e7 //add z7.s,z7.s,z19.s -.if mixin == 1 - add x19,x19,x20,lsl #32 // pack -.endif -.if mixin == 1 - add w21,w21,w30 -.endif -.long 0x04b4016b //add z11.s,z11.s,z20.s -.if mixin == 1 - add x22,x22,x30,lsr #32 -.endif -.long 0x04b901ef //add z15.s,z15.s,z25.s -.if mixin == 1 - add x21,x21,x22,lsl #32 // pack -.endif -.if mixin == 1 - ldp x20,x22,[x1],#16 -.endif -#ifdef __AARCH64EB__ - rev x7,x7 - rev x9,x9 - rev x11,x11 - rev x13,x13 - rev x15,x15 - rev x17,x17 - rev x19,x19 - rev x21,x21 -#endif -.if mixin == 1 - add x29,x29,#1 -.endif - cmp x5,4 - b.ne 200f -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.if mixin == 1 - eor x11,x11,x12 -.endif -.long 0x05a46011 //zip1 z17.s,z0.s,z4.s -.long 0x05a46412 //zip2 z18.s,z0.s,z4.s -.long 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.long 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.long 0x05a56035 //zip1 z21.s,z1.s,z5.s -.long 0x05a56436 //zip2 z22.s,z1.s,z5.s -.long 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.long 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36624 //zip2 z4.d,z17.d,z19.d -.long 0x05f46248 //zip1 z8.d,z18.d,z20.d -.long 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.long 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.long 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x13,x13,x14 -.endif -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.long 0x05a66051 //zip1 z17.s,z2.s,z6.s -.long 0x05a66452 //zip2 z18.s,z2.s,z6.s -.long 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.long 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.long 0x05a76075 //zip1 z21.s,z3.s,z7.s -.long 0x05a76476 //zip2 z22.s,z3.s,z7.s -.long 0x05af6177 //zip1 z23.s,z11.s,z15.s -.long 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.long 0x05f36222 //zip1 z2.d,z17.d,z19.d -.long 0x05f36626 //zip2 z6.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.long 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.long 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.long 0x05f862cb //zip1 z11.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.long 0x04b13000 //eor z0.d,z0.d,z17.d -.long 0x04b23021 //eor z1.d,z1.d,z18.d -.long 0x04b33042 //eor z2.d,z2.d,z19.d -.long 0x04b43063 //eor z3.d,z3.d,z20.d -.long 0x04b53084 //eor z4.d,z4.d,z21.d -.long 0x04b630a5 //eor z5.d,z5.d,z22.d -.long 0x04b730c6 //eor z6.d,z6.d,z23.d -.long 0x04b830e7 //eor z7.d,z7.d,z24.d - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.long 0x04b13108 //eor z8.d,z8.d,z17.d -.long 0x04b23129 //eor z9.d,z9.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.long 0x04b3314a //eor z10.d,z10.d,z19.d -.long 0x04b4316b //eor z11.d,z11.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.long 0x04b5318c //eor z12.d,z12.d,z21.d -.long 0x04b631ad //eor z13.d,z13.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.long 0x04b731ce //eor z14.d,z14.d,z23.d -.long 0x04b831ef //eor z15.d,z15.d,z24.d - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x0],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - st1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - st1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - b 210f -200: -.long 0x05a16011 //zip1 z17.s,z0.s,z1.s -.long 0x05a16412 //zip2 z18.s,z0.s,z1.s -.long 0x05a36053 //zip1 z19.s,z2.s,z3.s -.long 0x05a36454 //zip2 z20.s,z2.s,z3.s - -.long 0x05a56095 //zip1 z21.s,z4.s,z5.s -.long 0x05a56496 //zip2 z22.s,z4.s,z5.s -.long 0x05a760d7 //zip1 z23.s,z6.s,z7.s -.long 0x05a764d8 //zip2 z24.s,z6.s,z7.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36621 //zip2 z1.d,z17.d,z19.d -.long 0x05f46242 //zip1 z2.d,z18.d,z20.d -.long 0x05f46643 //zip2 z3.d,z18.d,z20.d - -.long 0x05f762a4 //zip1 z4.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c6 //zip1 z6.d,z22.d,z24.d -.long 0x05f866c7 //zip2 z7.d,z22.d,z24.d -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.long 0x05a96111 //zip1 z17.s,z8.s,z9.s -.long 0x05a96512 //zip2 z18.s,z8.s,z9.s -.long 0x05ab6153 //zip1 z19.s,z10.s,z11.s -.long 0x05ab6554 //zip2 z20.s,z10.s,z11.s - -.long 0x05ad6195 //zip1 z21.s,z12.s,z13.s -.long 0x05ad6596 //zip2 z22.s,z12.s,z13.s -.long 0x05af61d7 //zip1 z23.s,z14.s,z15.s -.long 0x05af65d8 //zip2 z24.s,z14.s,z15.s - -.long 0x05f36228 //zip1 z8.d,z17.d,z19.d -.long 0x05f36629 //zip2 z9.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664b //zip2 z11.d,z18.d,z20.d - -.long 0x05f762ac //zip1 z12.d,z21.d,z23.d -.long 0x05f766ad //zip2 z13.d,z21.d,z23.d -.long 0x05f862ce //zip1 z14.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x11,x11,x12 -.endif -.if mixin == 1 - eor x13,x13,x14 -.endif -.long 0x05a46011 //zip1 z17.s,z0.s,z4.s -.long 0x05a46412 //zip2 z18.s,z0.s,z4.s -.long 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.long 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.long 0x05a56035 //zip1 z21.s,z1.s,z5.s -.long 0x05a56436 //zip2 z22.s,z1.s,z5.s -.long 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.long 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36624 //zip2 z4.d,z17.d,z19.d -.long 0x05f46248 //zip1 z8.d,z18.d,z20.d -.long 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.long 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.long 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.long 0x05a66051 //zip1 z17.s,z2.s,z6.s -.long 0x05a66452 //zip2 z18.s,z2.s,z6.s -.long 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.long 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.long 0x05a76075 //zip1 z21.s,z3.s,z7.s -.long 0x05a76476 //zip2 z22.s,z3.s,z7.s -.long 0x05af6177 //zip1 z23.s,z11.s,z15.s -.long 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.long 0x05f36222 //zip1 z2.d,z17.d,z19.d -.long 0x05f36626 //zip2 z6.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.long 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.long 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.long 0x05f862cb //zip1 z11.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif -.long 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.long 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.long 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.long 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.long 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.long 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.long 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.long 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.long 0x04215101 //addvl x1,x1,8 -.long 0x04b13000 //eor z0.d,z0.d,z17.d -.long 0x04b23084 //eor z4.d,z4.d,z18.d -.long 0x04b33108 //eor z8.d,z8.d,z19.d -.long 0x04b4318c //eor z12.d,z12.d,z20.d -.long 0x04b53021 //eor z1.d,z1.d,z21.d -.long 0x04b630a5 //eor z5.d,z5.d,z22.d -.long 0x04b73129 //eor z9.d,z9.d,z23.d -.long 0x04b831ad //eor z13.d,z13.d,z24.d -.long 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.long 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.long 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.long 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.long 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.long 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.long 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.long 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.long 0x04215101 //addvl x1,x1,8 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.long 0x04b13042 //eor z2.d,z2.d,z17.d -.long 0x04b230c6 //eor z6.d,z6.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.long 0x04b3314a //eor z10.d,z10.d,z19.d -.long 0x04b431ce //eor z14.d,z14.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.long 0x04b53063 //eor z3.d,z3.d,z21.d -.long 0x04b630e7 //eor z7.d,z7.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.long 0x04b7316b //eor z11.d,z11.d,z23.d -.long 0x04b831ef //eor z15.d,z15.d,z24.d -#ifdef __AARCH64EB__ -.long 0x05a48000 //revb z0.s,p0/m,z0.s -#endif -.long 0xe540e000 //st1w {z0.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48084 //revb z4.s,p0/m,z4.s -#endif -.long 0xe541e004 //st1w {z4.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48108 //revb z8.s,p0/m,z8.s -#endif -.long 0xe542e008 //st1w {z8.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4818c //revb z12.s,p0/m,z12.s -#endif -.long 0xe543e00c //st1w {z12.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48021 //revb z1.s,p0/m,z1.s -#endif -.long 0xe544e001 //st1w {z1.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480a5 //revb z5.s,p0/m,z5.s -#endif -.long 0xe545e005 //st1w {z5.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48129 //revb z9.s,p0/m,z9.s -#endif -.long 0xe546e009 //st1w {z9.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ad //revb z13.s,p0/m,z13.s -#endif -.long 0xe547e00d //st1w {z13.s},p0,[x0,#7,MUL VL] -.long 0x04205100 //addvl x0,x0,8 -#ifdef __AARCH64EB__ -.long 0x05a48042 //revb z2.s,p0/m,z2.s -#endif -.long 0xe540e002 //st1w {z2.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480c6 //revb z6.s,p0/m,z6.s -#endif -.long 0xe541e006 //st1w {z6.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4814a //revb z10.s,p0/m,z10.s -#endif -.long 0xe542e00a //st1w {z10.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ce //revb z14.s,p0/m,z14.s -#endif -.long 0xe543e00e //st1w {z14.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48063 //revb z3.s,p0/m,z3.s -#endif -.long 0xe544e003 //st1w {z3.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480e7 //revb z7.s,p0/m,z7.s -#endif -.long 0xe545e007 //st1w {z7.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4816b //revb z11.s,p0/m,z11.s -#endif -.long 0xe546e00b //st1w {z11.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ef //revb z15.s,p0/m,z15.s -#endif -.long 0xe547e00f //st1w {z15.s},p0,[x0,#7,MUL VL] -.long 0x04205100 //addvl x0,x0,8 -210: -.long 0x04b0e3fd //incw x29, ALL, MUL #1 - subs x2,x2,64 - b.gt 100b - b 110f -101: - mixin=0 - lsr x8,x23,#32 -.long 0x05a03ae0 //dup z0.s,w23 -.long 0x05a03af9 //dup z25.s,w23 -.if mixin == 1 - mov w7,w23 -.endif -.long 0x05a03904 //dup z4.s,w8 -.long 0x05a0391a //dup z26.s,w8 - lsr x10,x24,#32 -.long 0x05a03b08 //dup z8.s,w24 -.long 0x05a03b1b //dup z27.s,w24 -.if mixin == 1 - mov w9,w24 -.endif -.long 0x05a0394c //dup z12.s,w10 -.long 0x05a0395c //dup z28.s,w10 - lsr x12,x25,#32 -.long 0x05a03b21 //dup z1.s,w25 -.long 0x05a03b3d //dup z29.s,w25 -.if mixin == 1 - mov w11,w25 -.endif -.long 0x05a03985 //dup z5.s,w12 -.long 0x05a0399e //dup z30.s,w12 - lsr x14,x26,#32 -.long 0x05a03b49 //dup z9.s,w26 -.long 0x05a03b55 //dup z21.s,w26 -.if mixin == 1 - mov w13,w26 -.endif -.long 0x05a039cd //dup z13.s,w14 -.long 0x05a039d6 //dup z22.s,w14 - lsr x16,x27,#32 -.long 0x05a03b62 //dup z2.s,w27 -.long 0x05a03b77 //dup z23.s,w27 -.if mixin == 1 - mov w15,w27 -.endif -.long 0x05a03a06 //dup z6.s,w16 -.long 0x05a03a18 //dup z24.s,w16 - lsr x18,x28,#32 -.long 0x05a03b8a //dup z10.s,w28 -.if mixin == 1 - mov w17,w28 -.endif -.long 0x05a03a4e //dup z14.s,w18 - lsr x22,x30,#32 -.long 0x05a03bcb //dup z11.s,w30 -.if mixin == 1 - mov w21,w30 -.endif -.long 0x05a03acf //dup z15.s,w22 -.if mixin == 1 - add w20,w29,#1 - mov w19,w29 -.long 0x04a14690 //index z16.s,w20,1 -.long 0x04a14683 //index z3.s,w20,1 -.else -.long 0x04a147b0 //index z16.s,w29,1 -.long 0x04a147a3 //index z3.s,w29,1 -.endif - lsr x20,x29,#32 -.long 0x05a03a87 //dup z7.s,w20 - mov x6,#10 -10: -.align 5 -.long 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.long 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.long 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.long 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.long 0x04a03063 //eor z3.d,z3.d,z0.d -.if mixin == 1 - eor w19,w19,w7 -.endif -.long 0x04a430e7 //eor z7.d,z7.d,z4.d -.if mixin == 1 - eor w20,w20,w8 -.endif -.long 0x04a8316b //eor z11.d,z11.d,z8.d -.if mixin == 1 - eor w21,w21,w9 -.endif -.long 0x04ac31ef //eor z15.d,z15.d,z12.d -.if mixin == 1 - eor w22,w22,w10 -.endif -.long 0x05a58063 //revh z3.s,p0/m,z3.s -.if mixin == 1 - ror w19,w19,#16 -.endif -.long 0x05a580e7 //revh z7.s,p0/m,z7.s -.if mixin == 1 - ror w20,w20,#16 -.endif -.long 0x05a5816b //revh z11.s,p0/m,z11.s -.if mixin == 1 - ror w21,w21,#16 -.endif -.long 0x05a581ef //revh z15.s,p0/m,z15.s -.if mixin == 1 - ror w22,w22,#16 -.endif -.long 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.long 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.long 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.long 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.long 0x04a23021 //eor z1.d,z1.d,z2.d -.if mixin == 1 - eor w11,w11,w15 -.endif -.long 0x04a630a5 //eor z5.d,z5.d,z6.d -.if mixin == 1 - eor w12,w12,w16 -.endif -.long 0x04aa3129 //eor z9.d,z9.d,z10.d -.if mixin == 1 - eor w13,w13,w17 -.endif -.long 0x04ae31ad //eor z13.d,z13.d,z14.d -.if mixin == 1 - eor w14,w14,w18 -.endif -.long 0x046c9c31 //lsl z17.s,z1.s,12 -.long 0x046c9cb2 //lsl z18.s,z5.s,12 -.long 0x046c9d33 //lsl z19.s,z9.s,12 -.long 0x046c9db4 //lsl z20.s,z13.s,12 -.long 0x046c9421 //lsr z1.s,z1.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.long 0x046c94a5 //lsr z5.s,z5.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.long 0x046c9529 //lsr z9.s,z9.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.long 0x046c95ad //lsr z13.s,z13.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.long 0x04713021 //orr z1.d,z1.d,z17.d -.long 0x047230a5 //orr z5.d,z5.d,z18.d -.long 0x04733129 //orr z9.d,z9.d,z19.d -.long 0x047431ad //orr z13.d,z13.d,z20.d -.long 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.long 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.long 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.long 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.long 0x04a03063 //eor z3.d,z3.d,z0.d -.if mixin == 1 - eor w19,w19,w7 -.endif -.long 0x04a430e7 //eor z7.d,z7.d,z4.d -.if mixin == 1 - eor w20,w20,w8 -.endif -.long 0x04a8316b //eor z11.d,z11.d,z8.d -.if mixin == 1 - eor w21,w21,w9 -.endif -.long 0x04ac31ef //eor z15.d,z15.d,z12.d -.if mixin == 1 - eor w22,w22,w10 -.endif -.long 0x053f3063 //tbl z3.b,{z3.b},z31.b -.if mixin == 1 - ror w19,w19,#24 -.endif -.long 0x053f30e7 //tbl z7.b,{z7.b},z31.b -.if mixin == 1 - ror w20,w20,#24 -.endif -.long 0x053f316b //tbl z11.b,{z11.b},z31.b -.if mixin == 1 - ror w21,w21,#24 -.endif -.long 0x053f31ef //tbl z15.b,{z15.b},z31.b -.if mixin == 1 - ror w22,w22,#24 -.endif -.long 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.long 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.long 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.long 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.long 0x04a23021 //eor z1.d,z1.d,z2.d -.if mixin == 1 - eor w11,w11,w15 -.endif -.long 0x04a630a5 //eor z5.d,z5.d,z6.d -.if mixin == 1 - eor w12,w12,w16 -.endif -.long 0x04aa3129 //eor z9.d,z9.d,z10.d -.if mixin == 1 - eor w13,w13,w17 -.endif -.long 0x04ae31ad //eor z13.d,z13.d,z14.d -.if mixin == 1 - eor w14,w14,w18 -.endif -.long 0x04679c31 //lsl z17.s,z1.s,7 -.long 0x04679cb2 //lsl z18.s,z5.s,7 -.long 0x04679d33 //lsl z19.s,z9.s,7 -.long 0x04679db4 //lsl z20.s,z13.s,7 -.long 0x04679421 //lsr z1.s,z1.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.long 0x046794a5 //lsr z5.s,z5.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.long 0x04679529 //lsr z9.s,z9.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.long 0x046795ad //lsr z13.s,z13.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.long 0x04713021 //orr z1.d,z1.d,z17.d -.long 0x047230a5 //orr z5.d,z5.d,z18.d -.long 0x04733129 //orr z9.d,z9.d,z19.d -.long 0x047431ad //orr z13.d,z13.d,z20.d -.long 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.long 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.long 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.long 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.long 0x04a031ef //eor z15.d,z15.d,z0.d -.if mixin == 1 - eor w22,w22,w7 -.endif -.long 0x04a43063 //eor z3.d,z3.d,z4.d -.if mixin == 1 - eor w19,w19,w8 -.endif -.long 0x04a830e7 //eor z7.d,z7.d,z8.d -.if mixin == 1 - eor w20,w20,w9 -.endif -.long 0x04ac316b //eor z11.d,z11.d,z12.d -.if mixin == 1 - eor w21,w21,w10 -.endif -.long 0x05a581ef //revh z15.s,p0/m,z15.s -.if mixin == 1 - ror w22,w22,#16 -.endif -.long 0x05a58063 //revh z3.s,p0/m,z3.s -.if mixin == 1 - ror w19,w19,#16 -.endif -.long 0x05a580e7 //revh z7.s,p0/m,z7.s -.if mixin == 1 - ror w20,w20,#16 -.endif -.long 0x05a5816b //revh z11.s,p0/m,z11.s -.if mixin == 1 - ror w21,w21,#16 -.endif -.long 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.long 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.long 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.long 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.long 0x04aa30a5 //eor z5.d,z5.d,z10.d -.if mixin == 1 - eor w12,w12,w17 -.endif -.long 0x04ae3129 //eor z9.d,z9.d,z14.d -.if mixin == 1 - eor w13,w13,w18 -.endif -.long 0x04a231ad //eor z13.d,z13.d,z2.d -.if mixin == 1 - eor w14,w14,w15 -.endif -.long 0x04a63021 //eor z1.d,z1.d,z6.d -.if mixin == 1 - eor w11,w11,w16 -.endif -.long 0x046c9cb1 //lsl z17.s,z5.s,12 -.long 0x046c9d32 //lsl z18.s,z9.s,12 -.long 0x046c9db3 //lsl z19.s,z13.s,12 -.long 0x046c9c34 //lsl z20.s,z1.s,12 -.long 0x046c94a5 //lsr z5.s,z5.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.long 0x046c9529 //lsr z9.s,z9.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.long 0x046c95ad //lsr z13.s,z13.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.long 0x046c9421 //lsr z1.s,z1.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.long 0x047130a5 //orr z5.d,z5.d,z17.d -.long 0x04723129 //orr z9.d,z9.d,z18.d -.long 0x047331ad //orr z13.d,z13.d,z19.d -.long 0x04743021 //orr z1.d,z1.d,z20.d -.long 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.long 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.long 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.long 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.long 0x04a031ef //eor z15.d,z15.d,z0.d -.if mixin == 1 - eor w22,w22,w7 -.endif -.long 0x04a43063 //eor z3.d,z3.d,z4.d -.if mixin == 1 - eor w19,w19,w8 -.endif -.long 0x04a830e7 //eor z7.d,z7.d,z8.d -.if mixin == 1 - eor w20,w20,w9 -.endif -.long 0x04ac316b //eor z11.d,z11.d,z12.d -.if mixin == 1 - eor w21,w21,w10 -.endif -.long 0x053f31ef //tbl z15.b,{z15.b},z31.b -.if mixin == 1 - ror w22,w22,#24 -.endif -.long 0x053f3063 //tbl z3.b,{z3.b},z31.b -.if mixin == 1 - ror w19,w19,#24 -.endif -.long 0x053f30e7 //tbl z7.b,{z7.b},z31.b -.if mixin == 1 - ror w20,w20,#24 -.endif -.long 0x053f316b //tbl z11.b,{z11.b},z31.b -.if mixin == 1 - ror w21,w21,#24 -.endif -.long 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.long 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.long 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.long 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.long 0x04aa30a5 //eor z5.d,z5.d,z10.d -.if mixin == 1 - eor w12,w12,w17 -.endif -.long 0x04ae3129 //eor z9.d,z9.d,z14.d -.if mixin == 1 - eor w13,w13,w18 -.endif -.long 0x04a231ad //eor z13.d,z13.d,z2.d -.if mixin == 1 - eor w14,w14,w15 -.endif -.long 0x04a63021 //eor z1.d,z1.d,z6.d -.if mixin == 1 - eor w11,w11,w16 -.endif -.long 0x04679cb1 //lsl z17.s,z5.s,7 -.long 0x04679d32 //lsl z18.s,z9.s,7 -.long 0x04679db3 //lsl z19.s,z13.s,7 -.long 0x04679c34 //lsl z20.s,z1.s,7 -.long 0x046794a5 //lsr z5.s,z5.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.long 0x04679529 //lsr z9.s,z9.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.long 0x046795ad //lsr z13.s,z13.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.long 0x04679421 //lsr z1.s,z1.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.long 0x047130a5 //orr z5.d,z5.d,z17.d -.long 0x04723129 //orr z9.d,z9.d,z18.d -.long 0x047331ad //orr z13.d,z13.d,z19.d -.long 0x04743021 //orr z1.d,z1.d,z20.d - sub x6,x6,1 - cbnz x6,10b - lsr x6,x28,#32 -.long 0x05a03b91 //dup z17.s,w28 -.long 0x05a038d2 //dup z18.s,w6 - lsr x6,x29,#32 -.long 0x05a038d3 //dup z19.s,w6 - lsr x6,x30,#32 -.if mixin == 1 - add w7,w7,w23 -.endif -.long 0x04b90000 //add z0.s,z0.s,z25.s -.if mixin == 1 - add x8,x8,x23,lsr #32 -.endif -.long 0x04ba0084 //add z4.s,z4.s,z26.s -.if mixin == 1 - add x7,x7,x8,lsl #32 // pack -.endif -.if mixin == 1 - add w9,w9,w24 -.endif -.long 0x04bb0108 //add z8.s,z8.s,z27.s -.if mixin == 1 - add x10,x10,x24,lsr #32 -.endif -.long 0x04bc018c //add z12.s,z12.s,z28.s -.if mixin == 1 - add x9,x9,x10,lsl #32 // pack -.endif -.if mixin == 1 - ldp x8,x10,[x1],#16 -.endif -.if mixin == 1 - add w11,w11,w25 -.endif -.long 0x04bd0021 //add z1.s,z1.s,z29.s -.if mixin == 1 - add x12,x12,x25,lsr #32 -.endif -.long 0x04be00a5 //add z5.s,z5.s,z30.s -.if mixin == 1 - add x11,x11,x12,lsl #32 // pack -.endif -.if mixin == 1 - add w13,w13,w26 -.endif -.long 0x04b50129 //add z9.s,z9.s,z21.s -.if mixin == 1 - add x14,x14,x26,lsr #32 -.endif -.long 0x04b601ad //add z13.s,z13.s,z22.s -.if mixin == 1 - add x13,x13,x14,lsl #32 // pack -.endif -.if mixin == 1 - ldp x12,x14,[x1],#16 -.endif -.if mixin == 1 - add w15,w15,w27 -.endif -.long 0x04b70042 //add z2.s,z2.s,z23.s -.if mixin == 1 - add x16,x16,x27,lsr #32 -.endif -.long 0x04b800c6 //add z6.s,z6.s,z24.s -.if mixin == 1 - add x15,x15,x16,lsl #32 // pack -.endif -.if mixin == 1 - add w17,w17,w28 -.endif -.long 0x04b1014a //add z10.s,z10.s,z17.s -.if mixin == 1 - add x18,x18,x28,lsr #32 -.endif -.long 0x04b201ce //add z14.s,z14.s,z18.s -.if mixin == 1 - add x17,x17,x18,lsl #32 // pack -.endif -.if mixin == 1 - ldp x16,x18,[x1],#16 -.endif -.long 0x05a03bd4 //dup z20.s,w30 -.long 0x05a038d9 //dup z25.s,w6 // bak[15] not available for SVE -.if mixin == 1 - add w19,w19,w29 -.endif -.long 0x04b00063 //add z3.s,z3.s,z16.s -.if mixin == 1 - add x20,x20,x29,lsr #32 -.endif -.long 0x04b300e7 //add z7.s,z7.s,z19.s -.if mixin == 1 - add x19,x19,x20,lsl #32 // pack -.endif -.if mixin == 1 - add w21,w21,w30 -.endif -.long 0x04b4016b //add z11.s,z11.s,z20.s -.if mixin == 1 - add x22,x22,x30,lsr #32 -.endif -.long 0x04b901ef //add z15.s,z15.s,z25.s -.if mixin == 1 - add x21,x21,x22,lsl #32 // pack -.endif -.if mixin == 1 - ldp x20,x22,[x1],#16 -.endif -#ifdef __AARCH64EB__ - rev x7,x7 - rev x9,x9 - rev x11,x11 - rev x13,x13 - rev x15,x15 - rev x17,x17 - rev x19,x19 - rev x21,x21 -#endif -.if mixin == 1 - add x29,x29,#1 -.endif - cmp x5,4 - b.ne 200f -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.if mixin == 1 - eor x11,x11,x12 -.endif -.long 0x05a46011 //zip1 z17.s,z0.s,z4.s -.long 0x05a46412 //zip2 z18.s,z0.s,z4.s -.long 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.long 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.long 0x05a56035 //zip1 z21.s,z1.s,z5.s -.long 0x05a56436 //zip2 z22.s,z1.s,z5.s -.long 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.long 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36624 //zip2 z4.d,z17.d,z19.d -.long 0x05f46248 //zip1 z8.d,z18.d,z20.d -.long 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.long 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.long 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x13,x13,x14 -.endif -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.long 0x05a66051 //zip1 z17.s,z2.s,z6.s -.long 0x05a66452 //zip2 z18.s,z2.s,z6.s -.long 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.long 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.long 0x05a76075 //zip1 z21.s,z3.s,z7.s -.long 0x05a76476 //zip2 z22.s,z3.s,z7.s -.long 0x05af6177 //zip1 z23.s,z11.s,z15.s -.long 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.long 0x05f36222 //zip1 z2.d,z17.d,z19.d -.long 0x05f36626 //zip2 z6.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.long 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.long 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.long 0x05f862cb //zip1 z11.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.long 0x04b13000 //eor z0.d,z0.d,z17.d -.long 0x04b23021 //eor z1.d,z1.d,z18.d -.long 0x04b33042 //eor z2.d,z2.d,z19.d -.long 0x04b43063 //eor z3.d,z3.d,z20.d -.long 0x04b53084 //eor z4.d,z4.d,z21.d -.long 0x04b630a5 //eor z5.d,z5.d,z22.d -.long 0x04b730c6 //eor z6.d,z6.d,z23.d -.long 0x04b830e7 //eor z7.d,z7.d,z24.d - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.long 0x04b13108 //eor z8.d,z8.d,z17.d -.long 0x04b23129 //eor z9.d,z9.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.long 0x04b3314a //eor z10.d,z10.d,z19.d -.long 0x04b4316b //eor z11.d,z11.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.long 0x04b5318c //eor z12.d,z12.d,z21.d -.long 0x04b631ad //eor z13.d,z13.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.long 0x04b731ce //eor z14.d,z14.d,z23.d -.long 0x04b831ef //eor z15.d,z15.d,z24.d - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x0],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - st1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - st1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - b 210f -200: -.long 0x05a16011 //zip1 z17.s,z0.s,z1.s -.long 0x05a16412 //zip2 z18.s,z0.s,z1.s -.long 0x05a36053 //zip1 z19.s,z2.s,z3.s -.long 0x05a36454 //zip2 z20.s,z2.s,z3.s - -.long 0x05a56095 //zip1 z21.s,z4.s,z5.s -.long 0x05a56496 //zip2 z22.s,z4.s,z5.s -.long 0x05a760d7 //zip1 z23.s,z6.s,z7.s -.long 0x05a764d8 //zip2 z24.s,z6.s,z7.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36621 //zip2 z1.d,z17.d,z19.d -.long 0x05f46242 //zip1 z2.d,z18.d,z20.d -.long 0x05f46643 //zip2 z3.d,z18.d,z20.d - -.long 0x05f762a4 //zip1 z4.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c6 //zip1 z6.d,z22.d,z24.d -.long 0x05f866c7 //zip2 z7.d,z22.d,z24.d -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.long 0x05a96111 //zip1 z17.s,z8.s,z9.s -.long 0x05a96512 //zip2 z18.s,z8.s,z9.s -.long 0x05ab6153 //zip1 z19.s,z10.s,z11.s -.long 0x05ab6554 //zip2 z20.s,z10.s,z11.s - -.long 0x05ad6195 //zip1 z21.s,z12.s,z13.s -.long 0x05ad6596 //zip2 z22.s,z12.s,z13.s -.long 0x05af61d7 //zip1 z23.s,z14.s,z15.s -.long 0x05af65d8 //zip2 z24.s,z14.s,z15.s - -.long 0x05f36228 //zip1 z8.d,z17.d,z19.d -.long 0x05f36629 //zip2 z9.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664b //zip2 z11.d,z18.d,z20.d - -.long 0x05f762ac //zip1 z12.d,z21.d,z23.d -.long 0x05f766ad //zip2 z13.d,z21.d,z23.d -.long 0x05f862ce //zip1 z14.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x11,x11,x12 -.endif -.if mixin == 1 - eor x13,x13,x14 -.endif -.long 0x05a46011 //zip1 z17.s,z0.s,z4.s -.long 0x05a46412 //zip2 z18.s,z0.s,z4.s -.long 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.long 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.long 0x05a56035 //zip1 z21.s,z1.s,z5.s -.long 0x05a56436 //zip2 z22.s,z1.s,z5.s -.long 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.long 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.long 0x05f36220 //zip1 z0.d,z17.d,z19.d -.long 0x05f36624 //zip2 z4.d,z17.d,z19.d -.long 0x05f46248 //zip1 z8.d,z18.d,z20.d -.long 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.long 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.long 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.long 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.long 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.long 0x05a66051 //zip1 z17.s,z2.s,z6.s -.long 0x05a66452 //zip2 z18.s,z2.s,z6.s -.long 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.long 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.long 0x05a76075 //zip1 z21.s,z3.s,z7.s -.long 0x05a76476 //zip2 z22.s,z3.s,z7.s -.long 0x05af6177 //zip1 z23.s,z11.s,z15.s -.long 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.long 0x05f36222 //zip1 z2.d,z17.d,z19.d -.long 0x05f36626 //zip2 z6.d,z17.d,z19.d -.long 0x05f4624a //zip1 z10.d,z18.d,z20.d -.long 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.long 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.long 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.long 0x05f862cb //zip1 z11.d,z22.d,z24.d -.long 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif -.long 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.long 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.long 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.long 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.long 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.long 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.long 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.long 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.long 0x04215101 //addvl x1,x1,8 -.long 0x04b13000 //eor z0.d,z0.d,z17.d -.long 0x04b23084 //eor z4.d,z4.d,z18.d -.long 0x04b33108 //eor z8.d,z8.d,z19.d -.long 0x04b4318c //eor z12.d,z12.d,z20.d -.long 0x04b53021 //eor z1.d,z1.d,z21.d -.long 0x04b630a5 //eor z5.d,z5.d,z22.d -.long 0x04b73129 //eor z9.d,z9.d,z23.d -.long 0x04b831ad //eor z13.d,z13.d,z24.d -.long 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.long 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.long 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.long 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.long 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.long 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.long 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.long 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.long 0x04215101 //addvl x1,x1,8 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.long 0x04b13042 //eor z2.d,z2.d,z17.d -.long 0x04b230c6 //eor z6.d,z6.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.long 0x04b3314a //eor z10.d,z10.d,z19.d -.long 0x04b431ce //eor z14.d,z14.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.long 0x04b53063 //eor z3.d,z3.d,z21.d -.long 0x04b630e7 //eor z7.d,z7.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.long 0x04b7316b //eor z11.d,z11.d,z23.d -.long 0x04b831ef //eor z15.d,z15.d,z24.d -#ifdef __AARCH64EB__ -.long 0x05a48000 //revb z0.s,p0/m,z0.s -#endif -.long 0xe540e000 //st1w {z0.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48084 //revb z4.s,p0/m,z4.s -#endif -.long 0xe541e004 //st1w {z4.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48108 //revb z8.s,p0/m,z8.s -#endif -.long 0xe542e008 //st1w {z8.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4818c //revb z12.s,p0/m,z12.s -#endif -.long 0xe543e00c //st1w {z12.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48021 //revb z1.s,p0/m,z1.s -#endif -.long 0xe544e001 //st1w {z1.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480a5 //revb z5.s,p0/m,z5.s -#endif -.long 0xe545e005 //st1w {z5.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48129 //revb z9.s,p0/m,z9.s -#endif -.long 0xe546e009 //st1w {z9.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ad //revb z13.s,p0/m,z13.s -#endif -.long 0xe547e00d //st1w {z13.s},p0,[x0,#7,MUL VL] -.long 0x04205100 //addvl x0,x0,8 -#ifdef __AARCH64EB__ -.long 0x05a48042 //revb z2.s,p0/m,z2.s -#endif -.long 0xe540e002 //st1w {z2.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480c6 //revb z6.s,p0/m,z6.s -#endif -.long 0xe541e006 //st1w {z6.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4814a //revb z10.s,p0/m,z10.s -#endif -.long 0xe542e00a //st1w {z10.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ce //revb z14.s,p0/m,z14.s -#endif -.long 0xe543e00e //st1w {z14.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a48063 //revb z3.s,p0/m,z3.s -#endif -.long 0xe544e003 //st1w {z3.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a480e7 //revb z7.s,p0/m,z7.s -#endif -.long 0xe545e007 //st1w {z7.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a4816b //revb z11.s,p0/m,z11.s -#endif -.long 0xe546e00b //st1w {z11.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.long 0x05a481ef //revb z15.s,p0/m,z15.s -#endif -.long 0xe547e00f //st1w {z15.s},p0,[x0,#7,MUL VL] -.long 0x04205100 //addvl x0,x0,8 -210: -.long 0x04b0e3fd //incw x29, ALL, MUL #1 -110: -2: - str w29,[x4] - ldp d10,d11,[sp,16] - ldp d12,d13,[sp,32] - ldp d14,d15,[sp,48] - ldp x16,x17,[sp,64] - ldp x18,x19,[sp,80] - ldp x20,x21,[sp,96] - ldp x22,x23,[sp,112] - ldp x24,x25,[sp,128] - ldp x26,x27,[sp,144] - ldp x28,x29,[sp,160] - ldr x30,[sp,176] - ldp d8,d9,[sp],192 - AARCH64_VALIDATE_LINK_REGISTER -Lreturn: - ret - diff --git a/openssl/src/crypto/chacha/gen/darwin_arm64/chacha-armv8.S b/openssl/src/crypto/chacha/gen/darwin_arm64/chacha-armv8.S index ad4250b6f..1374eb450 100644 --- a/openssl/src/crypto/chacha/gen/darwin_arm64/chacha-armv8.S +++ b/openssl/src/crypto/chacha/gen/darwin_arm64/chacha-armv8.S @@ -1,9 +1,7 @@ -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .private_extern _OPENSSL_armcap_P - - #endif .text @@ -18,22 +16,23 @@ Lrot24: .byte 67,104,97,67,104,97,50,48,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 .align 2 -.globl _ChaCha20_ctr32_dflt +.globl _ChaCha20_ctr32 .align 5 -_ChaCha20_ctr32_dflt: - AARCH64_SIGN_LINK_REGISTER +_ChaCha20_ctr32: + cbz x2,Labort cmp x2,#192 b.lo Lshort + #ifndef __KERNEL__ adrp x17,_OPENSSL_armcap_P@PAGE ldr w17,[x17,_OPENSSL_armcap_P@PAGEOFF] -Lcheck_neon: tst w17,#ARMV7_NEON b.ne LChaCha20_neon #endif Lshort: +.long 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -245,8 +244,8 @@ Loop: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 +.long 0xd50323bf // autiasp Labort: - AARCH64_VALIDATE_LINK_REGISTER ret .align 4 @@ -302,43 +301,8 @@ Loop_tail: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -.globl _ChaCha20_ctr32 - -.align 5 -_ChaCha20_ctr32: - AARCH64_SIGN_LINK_REGISTER - cbz x2,Labort - cmp x2,#192 - b.lo Lshort -#ifndef __KERNEL__ - adrp x17,_OPENSSL_armcap_P@PAGE - ldr w17,[x17,_OPENSSL_armcap_P@PAGEOFF] - tst w17,#ARMV8_SVE - b.eq Lcheck_neon - stp x29,x30,[sp,#-16]! - sub sp,sp,#16 - // SVE handling will inevitably increment the counter - // Neon/Scalar code that follows to process tail data needs to - // use new counter, unfortunately the input counter buffer - // pointed to by ctr is meant to be read-only per API contract - // we have to copy the buffer to stack to be writable by SVE - ldp x5,x6,[x4] - stp x5,x6,[sp] - mov x4,sp - bl _ChaCha20_ctr32_sve - cbz x2,1f - bl _ChaCha20_ctr32_dflt -1: - add sp,sp,#16 - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret -#endif - b Lshort #ifdef __KERNEL__ @@ -347,8 +311,8 @@ _ChaCha20_ctr32: .align 5 _ChaCha20_neon: - AARCH64_SIGN_LINK_REGISTER LChaCha20_neon: +.long 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -783,7 +747,7 @@ Loop_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .align 4 @@ -915,13 +879,13 @@ Ldone_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .align 5 ChaCha20_512_neon: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -2065,6 +2029,6 @@ Ldone_512_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret diff --git a/openssl/src/crypto/chacha/gen/linux_arm/chacha-armv4.S b/openssl/src/crypto/chacha/gen/linux_arm/chacha-armv4.S index 95805c659..0badd6fea 100644 --- a/openssl/src/crypto/chacha/gen/linux_arm/chacha-armv4.S +++ b/openssl/src/crypto/chacha/gen/linux_arm/chacha-armv4.S @@ -1473,5 +1473,5 @@ ChaCha20_neon: add sp,sp,#4*(16+3) ldmia sp!,{r4,r5,r6,r7,r8,r9,r10,r11,pc} .size ChaCha20_neon,.-ChaCha20_neon - +.comm OPENSSL_armcap_P,4,4 #endif diff --git a/openssl/src/crypto/chacha/gen/linux_arm64/chacha-armv8-sve.S b/openssl/src/crypto/chacha/gen/linux_arm64/chacha-armv8-sve.S deleted file mode 100644 index 29fd6b144..000000000 --- a/openssl/src/crypto/chacha/gen/linux_arm64/chacha-armv8-sve.S +++ /dev/null @@ -1,3868 +0,0 @@ -// Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// -// ChaCha20 for ARMv8 via SVE -// -// $output is the last argument if it looks like a file (it has an extension) -// $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" - -.arch armv8-a - - -.hidden OPENSSL_armcap_P - -.text -.align 5 -.Lchacha20_consts: -.quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral -.Lrot8: -.word 0x02010003,0x04040404,0x02010003,0x04040404 -.globl ChaCha20_ctr32_sve -.type ChaCha20_ctr32_sve,%function -.align 5 -ChaCha20_ctr32_sve: - AARCH64_VALID_CALL_TARGET -.inst 0x04a0e3e5 //cntw x5, ALL, MUL #1 - cmp x2,x5,lsl #6 - b.lt .Lreturn - mov x7,0 - adrp x6,OPENSSL_armcap_P - ldr w6,[x6,#:lo12:OPENSSL_armcap_P] - tst w6,#ARMV8_SVE2 - b.eq 1f - mov x7,1 - b 2f -1: - cmp x5,4 - b.le .Lreturn - adr x6,.Lrot8 - ldp w9,w10,[x6] -.inst 0x04aa4d3f //index z31.s,w9,w10 -2: - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,-192]! - stp d10,d11,[sp,16] - stp d12,d13,[sp,32] - stp d14,d15,[sp,48] - stp x16,x17,[sp,64] - stp x18,x19,[sp,80] - stp x20,x21,[sp,96] - stp x22,x23,[sp,112] - stp x24,x25,[sp,128] - stp x26,x27,[sp,144] - stp x28,x29,[sp,160] - str x30,[sp,176] - - adr x6,.Lchacha20_consts - ldp x23,x24,[x6] - ldp x25,x26,[x3] - ldp x27,x28,[x3, 16] - ldp x29,x30,[x4] -.inst 0x2599e3e0 //ptrues p0.s,ALL -#ifdef __AARCH64EB__ - ror x25,x25,#32 - ror x26,x26,#32 - ror x27,x27,#32 - ror x28,x28,#32 - ror x29,x29,#32 - ror x30,x30,#32 -#endif - cbz x7, 1f -.align 5 -100: - subs x7,x2,x5,lsl #6 - b.lt 110f - mov x2,x7 - b.eq 101f - cmp x2,64 - b.lt 101f - mixin=1 - lsr x8,x23,#32 -.inst 0x05a03ae0 //dup z0.s,w23 -.inst 0x05a03af9 //dup z25.s,w23 -.if mixin == 1 - mov w7,w23 -.endif -.inst 0x05a03904 //dup z4.s,w8 -.inst 0x05a0391a //dup z26.s,w8 - lsr x10,x24,#32 -.inst 0x05a03b08 //dup z8.s,w24 -.inst 0x05a03b1b //dup z27.s,w24 -.if mixin == 1 - mov w9,w24 -.endif -.inst 0x05a0394c //dup z12.s,w10 -.inst 0x05a0395c //dup z28.s,w10 - lsr x12,x25,#32 -.inst 0x05a03b21 //dup z1.s,w25 -.inst 0x05a03b3d //dup z29.s,w25 -.if mixin == 1 - mov w11,w25 -.endif -.inst 0x05a03985 //dup z5.s,w12 -.inst 0x05a0399e //dup z30.s,w12 - lsr x14,x26,#32 -.inst 0x05a03b49 //dup z9.s,w26 -.inst 0x05a03b55 //dup z21.s,w26 -.if mixin == 1 - mov w13,w26 -.endif -.inst 0x05a039cd //dup z13.s,w14 -.inst 0x05a039d6 //dup z22.s,w14 - lsr x16,x27,#32 -.inst 0x05a03b62 //dup z2.s,w27 -.inst 0x05a03b77 //dup z23.s,w27 -.if mixin == 1 - mov w15,w27 -.endif -.inst 0x05a03a06 //dup z6.s,w16 -.inst 0x05a03a18 //dup z24.s,w16 - lsr x18,x28,#32 -.inst 0x05a03b8a //dup z10.s,w28 -.inst 0x05a03b91 //dup z17.s,w28 -.if mixin == 1 - mov w17,w28 -.endif -.inst 0x05a03a4e //dup z14.s,w18 -.inst 0x05a03a52 //dup z18.s,w18 - lsr x22,x30,#32 -.inst 0x05a03bcb //dup z11.s,w30 -.inst 0x05a03bd4 //dup z20.s,w30 -.if mixin == 1 - mov w21,w30 -.endif -.inst 0x05a03acf //dup z15.s,w22 -.inst 0x05a03adf //dup z31.s,w22 -.if mixin == 1 - add w20,w29,#1 - mov w19,w29 -.inst 0x04a14690 //index z16.s,w20,1 -.inst 0x04a14683 //index z3.s,w20,1 -.else -.inst 0x04a147b0 //index z16.s,w29,1 -.inst 0x04a147a3 //index z3.s,w29,1 -.endif - lsr x20,x29,#32 -.inst 0x05a03a87 //dup z7.s,w20 -.inst 0x05a03a93 //dup z19.s,w20 - mov x6,#10 -10: -.align 5 -.inst 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.inst 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.inst 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.inst 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.if mixin == 1 - eor w19,w19,w7 -.endif -.inst 0x04703403 //xar z3.s,z3.s,z0.s,16 -.if mixin == 1 - ror w19,w19,16 -.endif -.if mixin == 1 - eor w20,w20,w8 -.endif -.inst 0x04703487 //xar z7.s,z7.s,z4.s,16 -.if mixin == 1 - ror w20,w20,16 -.endif -.if mixin == 1 - eor w21,w21,w9 -.endif -.inst 0x0470350b //xar z11.s,z11.s,z8.s,16 -.if mixin == 1 - ror w21,w21,16 -.endif -.if mixin == 1 - eor w22,w22,w10 -.endif -.inst 0x0470358f //xar z15.s,z15.s,z12.s,16 -.if mixin == 1 - ror w22,w22,16 -.endif -.inst 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.inst 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.inst 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.inst 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.if mixin == 1 - eor w11,w11,w15 -.endif -.inst 0x046c3441 //xar z1.s,z1.s,z2.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.if mixin == 1 - eor w12,w12,w16 -.endif -.inst 0x046c34c5 //xar z5.s,z5.s,z6.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.if mixin == 1 - eor w13,w13,w17 -.endif -.inst 0x046c3549 //xar z9.s,z9.s,z10.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.if mixin == 1 - eor w14,w14,w18 -.endif -.inst 0x046c35cd //xar z13.s,z13.s,z14.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.inst 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.inst 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.inst 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.inst 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.if mixin == 1 - eor w19,w19,w7 -.endif -.inst 0x04683403 //xar z3.s,z3.s,z0.s,24 -.if mixin == 1 - ror w19,w19,24 -.endif -.if mixin == 1 - eor w20,w20,w8 -.endif -.inst 0x04683487 //xar z7.s,z7.s,z4.s,24 -.if mixin == 1 - ror w20,w20,24 -.endif -.if mixin == 1 - eor w21,w21,w9 -.endif -.inst 0x0468350b //xar z11.s,z11.s,z8.s,24 -.if mixin == 1 - ror w21,w21,24 -.endif -.if mixin == 1 - eor w22,w22,w10 -.endif -.inst 0x0468358f //xar z15.s,z15.s,z12.s,24 -.if mixin == 1 - ror w22,w22,24 -.endif -.inst 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.inst 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.inst 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.inst 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.if mixin == 1 - eor w11,w11,w15 -.endif -.inst 0x04673441 //xar z1.s,z1.s,z2.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.if mixin == 1 - eor w12,w12,w16 -.endif -.inst 0x046734c5 //xar z5.s,z5.s,z6.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.if mixin == 1 - eor w13,w13,w17 -.endif -.inst 0x04673549 //xar z9.s,z9.s,z10.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.if mixin == 1 - eor w14,w14,w18 -.endif -.inst 0x046735cd //xar z13.s,z13.s,z14.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.inst 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.inst 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.inst 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.inst 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.if mixin == 1 - eor w22,w22,w7 -.endif -.inst 0x0470340f //xar z15.s,z15.s,z0.s,16 -.if mixin == 1 - ror w22,w22,16 -.endif -.if mixin == 1 - eor w19,w19,w8 -.endif -.inst 0x04703483 //xar z3.s,z3.s,z4.s,16 -.if mixin == 1 - ror w19,w19,16 -.endif -.if mixin == 1 - eor w20,w20,w9 -.endif -.inst 0x04703507 //xar z7.s,z7.s,z8.s,16 -.if mixin == 1 - ror w20,w20,16 -.endif -.if mixin == 1 - eor w21,w21,w10 -.endif -.inst 0x0470358b //xar z11.s,z11.s,z12.s,16 -.if mixin == 1 - ror w21,w21,16 -.endif -.inst 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.inst 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.inst 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.inst 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.if mixin == 1 - eor w12,w12,w17 -.endif -.inst 0x046c3545 //xar z5.s,z5.s,z10.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.if mixin == 1 - eor w13,w13,w18 -.endif -.inst 0x046c35c9 //xar z9.s,z9.s,z14.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.if mixin == 1 - eor w14,w14,w15 -.endif -.inst 0x046c344d //xar z13.s,z13.s,z2.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.if mixin == 1 - eor w11,w11,w16 -.endif -.inst 0x046c34c1 //xar z1.s,z1.s,z6.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.inst 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.inst 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.inst 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.inst 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.if mixin == 1 - eor w22,w22,w7 -.endif -.inst 0x0468340f //xar z15.s,z15.s,z0.s,24 -.if mixin == 1 - ror w22,w22,24 -.endif -.if mixin == 1 - eor w19,w19,w8 -.endif -.inst 0x04683483 //xar z3.s,z3.s,z4.s,24 -.if mixin == 1 - ror w19,w19,24 -.endif -.if mixin == 1 - eor w20,w20,w9 -.endif -.inst 0x04683507 //xar z7.s,z7.s,z8.s,24 -.if mixin == 1 - ror w20,w20,24 -.endif -.if mixin == 1 - eor w21,w21,w10 -.endif -.inst 0x0468358b //xar z11.s,z11.s,z12.s,24 -.if mixin == 1 - ror w21,w21,24 -.endif -.inst 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.inst 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.inst 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.inst 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.if mixin == 1 - eor w12,w12,w17 -.endif -.inst 0x04673545 //xar z5.s,z5.s,z10.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.if mixin == 1 - eor w13,w13,w18 -.endif -.inst 0x046735c9 //xar z9.s,z9.s,z14.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.if mixin == 1 - eor w14,w14,w15 -.endif -.inst 0x0467344d //xar z13.s,z13.s,z2.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.if mixin == 1 - eor w11,w11,w16 -.endif -.inst 0x046734c1 //xar z1.s,z1.s,z6.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif - sub x6,x6,1 - cbnz x6,10b -.if mixin == 1 - add w7,w7,w23 -.endif -.inst 0x04b90000 //add z0.s,z0.s,z25.s -.if mixin == 1 - add x8,x8,x23,lsr #32 -.endif -.inst 0x04ba0084 //add z4.s,z4.s,z26.s -.if mixin == 1 - add x7,x7,x8,lsl #32 // pack -.endif -.if mixin == 1 - add w9,w9,w24 -.endif -.inst 0x04bb0108 //add z8.s,z8.s,z27.s -.if mixin == 1 - add x10,x10,x24,lsr #32 -.endif -.inst 0x04bc018c //add z12.s,z12.s,z28.s -.if mixin == 1 - add x9,x9,x10,lsl #32 // pack -.endif -.if mixin == 1 - ldp x8,x10,[x1],#16 -.endif -.if mixin == 1 - add w11,w11,w25 -.endif -.inst 0x04bd0021 //add z1.s,z1.s,z29.s -.if mixin == 1 - add x12,x12,x25,lsr #32 -.endif -.inst 0x04be00a5 //add z5.s,z5.s,z30.s -.if mixin == 1 - add x11,x11,x12,lsl #32 // pack -.endif -.if mixin == 1 - add w13,w13,w26 -.endif -.inst 0x04b50129 //add z9.s,z9.s,z21.s -.if mixin == 1 - add x14,x14,x26,lsr #32 -.endif -.inst 0x04b601ad //add z13.s,z13.s,z22.s -.if mixin == 1 - add x13,x13,x14,lsl #32 // pack -.endif -.if mixin == 1 - ldp x12,x14,[x1],#16 -.endif -.if mixin == 1 - add w15,w15,w27 -.endif -.inst 0x04b70042 //add z2.s,z2.s,z23.s -.if mixin == 1 - add x16,x16,x27,lsr #32 -.endif -.inst 0x04b800c6 //add z6.s,z6.s,z24.s -.if mixin == 1 - add x15,x15,x16,lsl #32 // pack -.endif -.if mixin == 1 - add w17,w17,w28 -.endif -.inst 0x04b1014a //add z10.s,z10.s,z17.s -.if mixin == 1 - add x18,x18,x28,lsr #32 -.endif -.inst 0x04b201ce //add z14.s,z14.s,z18.s -.if mixin == 1 - add x17,x17,x18,lsl #32 // pack -.endif -.if mixin == 1 - ldp x16,x18,[x1],#16 -.endif -.if mixin == 1 - add w19,w19,w29 -.endif -.inst 0x04b00063 //add z3.s,z3.s,z16.s -.if mixin == 1 - add x20,x20,x29,lsr #32 -.endif -.inst 0x04b300e7 //add z7.s,z7.s,z19.s -.if mixin == 1 - add x19,x19,x20,lsl #32 // pack -.endif -.if mixin == 1 - add w21,w21,w30 -.endif -.inst 0x04b4016b //add z11.s,z11.s,z20.s -.if mixin == 1 - add x22,x22,x30,lsr #32 -.endif -.inst 0x04bf01ef //add z15.s,z15.s,z31.s -.if mixin == 1 - add x21,x21,x22,lsl #32 // pack -.endif -.if mixin == 1 - ldp x20,x22,[x1],#16 -.endif -#ifdef __AARCH64EB__ - rev x7,x7 - rev x9,x9 - rev x11,x11 - rev x13,x13 - rev x15,x15 - rev x17,x17 - rev x19,x19 - rev x21,x21 -#endif -.if mixin == 1 - add x29,x29,#1 -.endif - cmp x5,4 - b.ne 200f -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.if mixin == 1 - eor x11,x11,x12 -.endif -.inst 0x05a46011 //zip1 z17.s,z0.s,z4.s -.inst 0x05a46412 //zip2 z18.s,z0.s,z4.s -.inst 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.inst 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.inst 0x05a56035 //zip1 z21.s,z1.s,z5.s -.inst 0x05a56436 //zip2 z22.s,z1.s,z5.s -.inst 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.inst 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36624 //zip2 z4.d,z17.d,z19.d -.inst 0x05f46248 //zip1 z8.d,z18.d,z20.d -.inst 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.inst 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.inst 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x13,x13,x14 -.endif -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.inst 0x05a66051 //zip1 z17.s,z2.s,z6.s -.inst 0x05a66452 //zip2 z18.s,z2.s,z6.s -.inst 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.inst 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.inst 0x05a76075 //zip1 z21.s,z3.s,z7.s -.inst 0x05a76476 //zip2 z22.s,z3.s,z7.s -.inst 0x05af6177 //zip1 z23.s,z11.s,z15.s -.inst 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.inst 0x05f36222 //zip1 z2.d,z17.d,z19.d -.inst 0x05f36626 //zip2 z6.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.inst 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.inst 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.inst 0x05f862cb //zip1 z11.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.inst 0x04b13000 //eor z0.d,z0.d,z17.d -.inst 0x04b23021 //eor z1.d,z1.d,z18.d -.inst 0x04b33042 //eor z2.d,z2.d,z19.d -.inst 0x04b43063 //eor z3.d,z3.d,z20.d -.inst 0x04b53084 //eor z4.d,z4.d,z21.d -.inst 0x04b630a5 //eor z5.d,z5.d,z22.d -.inst 0x04b730c6 //eor z6.d,z6.d,z23.d -.inst 0x04b830e7 //eor z7.d,z7.d,z24.d - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.inst 0x04b13108 //eor z8.d,z8.d,z17.d -.inst 0x04b23129 //eor z9.d,z9.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.inst 0x04b3314a //eor z10.d,z10.d,z19.d -.inst 0x04b4316b //eor z11.d,z11.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.inst 0x04b5318c //eor z12.d,z12.d,z21.d -.inst 0x04b631ad //eor z13.d,z13.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.inst 0x04b731ce //eor z14.d,z14.d,z23.d -.inst 0x04b831ef //eor z15.d,z15.d,z24.d - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x0],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - st1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - st1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - b 210f -200: -.inst 0x05a16011 //zip1 z17.s,z0.s,z1.s -.inst 0x05a16412 //zip2 z18.s,z0.s,z1.s -.inst 0x05a36053 //zip1 z19.s,z2.s,z3.s -.inst 0x05a36454 //zip2 z20.s,z2.s,z3.s - -.inst 0x05a56095 //zip1 z21.s,z4.s,z5.s -.inst 0x05a56496 //zip2 z22.s,z4.s,z5.s -.inst 0x05a760d7 //zip1 z23.s,z6.s,z7.s -.inst 0x05a764d8 //zip2 z24.s,z6.s,z7.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36621 //zip2 z1.d,z17.d,z19.d -.inst 0x05f46242 //zip1 z2.d,z18.d,z20.d -.inst 0x05f46643 //zip2 z3.d,z18.d,z20.d - -.inst 0x05f762a4 //zip1 z4.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c6 //zip1 z6.d,z22.d,z24.d -.inst 0x05f866c7 //zip2 z7.d,z22.d,z24.d -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.inst 0x05a96111 //zip1 z17.s,z8.s,z9.s -.inst 0x05a96512 //zip2 z18.s,z8.s,z9.s -.inst 0x05ab6153 //zip1 z19.s,z10.s,z11.s -.inst 0x05ab6554 //zip2 z20.s,z10.s,z11.s - -.inst 0x05ad6195 //zip1 z21.s,z12.s,z13.s -.inst 0x05ad6596 //zip2 z22.s,z12.s,z13.s -.inst 0x05af61d7 //zip1 z23.s,z14.s,z15.s -.inst 0x05af65d8 //zip2 z24.s,z14.s,z15.s - -.inst 0x05f36228 //zip1 z8.d,z17.d,z19.d -.inst 0x05f36629 //zip2 z9.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664b //zip2 z11.d,z18.d,z20.d - -.inst 0x05f762ac //zip1 z12.d,z21.d,z23.d -.inst 0x05f766ad //zip2 z13.d,z21.d,z23.d -.inst 0x05f862ce //zip1 z14.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x11,x11,x12 -.endif -.if mixin == 1 - eor x13,x13,x14 -.endif -.inst 0x05a46011 //zip1 z17.s,z0.s,z4.s -.inst 0x05a46412 //zip2 z18.s,z0.s,z4.s -.inst 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.inst 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.inst 0x05a56035 //zip1 z21.s,z1.s,z5.s -.inst 0x05a56436 //zip2 z22.s,z1.s,z5.s -.inst 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.inst 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36624 //zip2 z4.d,z17.d,z19.d -.inst 0x05f46248 //zip1 z8.d,z18.d,z20.d -.inst 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.inst 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.inst 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.inst 0x05a66051 //zip1 z17.s,z2.s,z6.s -.inst 0x05a66452 //zip2 z18.s,z2.s,z6.s -.inst 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.inst 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.inst 0x05a76075 //zip1 z21.s,z3.s,z7.s -.inst 0x05a76476 //zip2 z22.s,z3.s,z7.s -.inst 0x05af6177 //zip1 z23.s,z11.s,z15.s -.inst 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.inst 0x05f36222 //zip1 z2.d,z17.d,z19.d -.inst 0x05f36626 //zip2 z6.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.inst 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.inst 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.inst 0x05f862cb //zip1 z11.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif -.inst 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.inst 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.inst 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.inst 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.inst 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.inst 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.inst 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.inst 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.inst 0x04215101 //addvl x1,x1,8 -.inst 0x04b13000 //eor z0.d,z0.d,z17.d -.inst 0x04b23084 //eor z4.d,z4.d,z18.d -.inst 0x04b33108 //eor z8.d,z8.d,z19.d -.inst 0x04b4318c //eor z12.d,z12.d,z20.d -.inst 0x04b53021 //eor z1.d,z1.d,z21.d -.inst 0x04b630a5 //eor z5.d,z5.d,z22.d -.inst 0x04b73129 //eor z9.d,z9.d,z23.d -.inst 0x04b831ad //eor z13.d,z13.d,z24.d -.inst 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.inst 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.inst 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.inst 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.inst 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.inst 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.inst 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.inst 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.inst 0x04215101 //addvl x1,x1,8 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.inst 0x04b13042 //eor z2.d,z2.d,z17.d -.inst 0x04b230c6 //eor z6.d,z6.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.inst 0x04b3314a //eor z10.d,z10.d,z19.d -.inst 0x04b431ce //eor z14.d,z14.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.inst 0x04b53063 //eor z3.d,z3.d,z21.d -.inst 0x04b630e7 //eor z7.d,z7.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.inst 0x04b7316b //eor z11.d,z11.d,z23.d -.inst 0x04b831ef //eor z15.d,z15.d,z24.d -#ifdef __AARCH64EB__ -.inst 0x05a48000 //revb z0.s,p0/m,z0.s -#endif -.inst 0xe540e000 //st1w {z0.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48084 //revb z4.s,p0/m,z4.s -#endif -.inst 0xe541e004 //st1w {z4.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48108 //revb z8.s,p0/m,z8.s -#endif -.inst 0xe542e008 //st1w {z8.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4818c //revb z12.s,p0/m,z12.s -#endif -.inst 0xe543e00c //st1w {z12.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48021 //revb z1.s,p0/m,z1.s -#endif -.inst 0xe544e001 //st1w {z1.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480a5 //revb z5.s,p0/m,z5.s -#endif -.inst 0xe545e005 //st1w {z5.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48129 //revb z9.s,p0/m,z9.s -#endif -.inst 0xe546e009 //st1w {z9.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ad //revb z13.s,p0/m,z13.s -#endif -.inst 0xe547e00d //st1w {z13.s},p0,[x0,#7,MUL VL] -.inst 0x04205100 //addvl x0,x0,8 -#ifdef __AARCH64EB__ -.inst 0x05a48042 //revb z2.s,p0/m,z2.s -#endif -.inst 0xe540e002 //st1w {z2.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480c6 //revb z6.s,p0/m,z6.s -#endif -.inst 0xe541e006 //st1w {z6.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4814a //revb z10.s,p0/m,z10.s -#endif -.inst 0xe542e00a //st1w {z10.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ce //revb z14.s,p0/m,z14.s -#endif -.inst 0xe543e00e //st1w {z14.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48063 //revb z3.s,p0/m,z3.s -#endif -.inst 0xe544e003 //st1w {z3.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480e7 //revb z7.s,p0/m,z7.s -#endif -.inst 0xe545e007 //st1w {z7.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4816b //revb z11.s,p0/m,z11.s -#endif -.inst 0xe546e00b //st1w {z11.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ef //revb z15.s,p0/m,z15.s -#endif -.inst 0xe547e00f //st1w {z15.s},p0,[x0,#7,MUL VL] -.inst 0x04205100 //addvl x0,x0,8 -210: -.inst 0x04b0e3fd //incw x29, ALL, MUL #1 - subs x2,x2,64 - b.gt 100b - b 110f -101: - mixin=0 - lsr x8,x23,#32 -.inst 0x05a03ae0 //dup z0.s,w23 -.inst 0x05a03af9 //dup z25.s,w23 -.if mixin == 1 - mov w7,w23 -.endif -.inst 0x05a03904 //dup z4.s,w8 -.inst 0x05a0391a //dup z26.s,w8 - lsr x10,x24,#32 -.inst 0x05a03b08 //dup z8.s,w24 -.inst 0x05a03b1b //dup z27.s,w24 -.if mixin == 1 - mov w9,w24 -.endif -.inst 0x05a0394c //dup z12.s,w10 -.inst 0x05a0395c //dup z28.s,w10 - lsr x12,x25,#32 -.inst 0x05a03b21 //dup z1.s,w25 -.inst 0x05a03b3d //dup z29.s,w25 -.if mixin == 1 - mov w11,w25 -.endif -.inst 0x05a03985 //dup z5.s,w12 -.inst 0x05a0399e //dup z30.s,w12 - lsr x14,x26,#32 -.inst 0x05a03b49 //dup z9.s,w26 -.inst 0x05a03b55 //dup z21.s,w26 -.if mixin == 1 - mov w13,w26 -.endif -.inst 0x05a039cd //dup z13.s,w14 -.inst 0x05a039d6 //dup z22.s,w14 - lsr x16,x27,#32 -.inst 0x05a03b62 //dup z2.s,w27 -.inst 0x05a03b77 //dup z23.s,w27 -.if mixin == 1 - mov w15,w27 -.endif -.inst 0x05a03a06 //dup z6.s,w16 -.inst 0x05a03a18 //dup z24.s,w16 - lsr x18,x28,#32 -.inst 0x05a03b8a //dup z10.s,w28 -.inst 0x05a03b91 //dup z17.s,w28 -.if mixin == 1 - mov w17,w28 -.endif -.inst 0x05a03a4e //dup z14.s,w18 -.inst 0x05a03a52 //dup z18.s,w18 - lsr x22,x30,#32 -.inst 0x05a03bcb //dup z11.s,w30 -.inst 0x05a03bd4 //dup z20.s,w30 -.if mixin == 1 - mov w21,w30 -.endif -.inst 0x05a03acf //dup z15.s,w22 -.inst 0x05a03adf //dup z31.s,w22 -.if mixin == 1 - add w20,w29,#1 - mov w19,w29 -.inst 0x04a14690 //index z16.s,w20,1 -.inst 0x04a14683 //index z3.s,w20,1 -.else -.inst 0x04a147b0 //index z16.s,w29,1 -.inst 0x04a147a3 //index z3.s,w29,1 -.endif - lsr x20,x29,#32 -.inst 0x05a03a87 //dup z7.s,w20 -.inst 0x05a03a93 //dup z19.s,w20 - mov x6,#10 -10: -.align 5 -.inst 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.inst 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.inst 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.inst 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.if mixin == 1 - eor w19,w19,w7 -.endif -.inst 0x04703403 //xar z3.s,z3.s,z0.s,16 -.if mixin == 1 - ror w19,w19,16 -.endif -.if mixin == 1 - eor w20,w20,w8 -.endif -.inst 0x04703487 //xar z7.s,z7.s,z4.s,16 -.if mixin == 1 - ror w20,w20,16 -.endif -.if mixin == 1 - eor w21,w21,w9 -.endif -.inst 0x0470350b //xar z11.s,z11.s,z8.s,16 -.if mixin == 1 - ror w21,w21,16 -.endif -.if mixin == 1 - eor w22,w22,w10 -.endif -.inst 0x0470358f //xar z15.s,z15.s,z12.s,16 -.if mixin == 1 - ror w22,w22,16 -.endif -.inst 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.inst 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.inst 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.inst 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.if mixin == 1 - eor w11,w11,w15 -.endif -.inst 0x046c3441 //xar z1.s,z1.s,z2.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.if mixin == 1 - eor w12,w12,w16 -.endif -.inst 0x046c34c5 //xar z5.s,z5.s,z6.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.if mixin == 1 - eor w13,w13,w17 -.endif -.inst 0x046c3549 //xar z9.s,z9.s,z10.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.if mixin == 1 - eor w14,w14,w18 -.endif -.inst 0x046c35cd //xar z13.s,z13.s,z14.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.inst 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.inst 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.inst 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.inst 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.if mixin == 1 - eor w19,w19,w7 -.endif -.inst 0x04683403 //xar z3.s,z3.s,z0.s,24 -.if mixin == 1 - ror w19,w19,24 -.endif -.if mixin == 1 - eor w20,w20,w8 -.endif -.inst 0x04683487 //xar z7.s,z7.s,z4.s,24 -.if mixin == 1 - ror w20,w20,24 -.endif -.if mixin == 1 - eor w21,w21,w9 -.endif -.inst 0x0468350b //xar z11.s,z11.s,z8.s,24 -.if mixin == 1 - ror w21,w21,24 -.endif -.if mixin == 1 - eor w22,w22,w10 -.endif -.inst 0x0468358f //xar z15.s,z15.s,z12.s,24 -.if mixin == 1 - ror w22,w22,24 -.endif -.inst 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.inst 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.inst 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.inst 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.if mixin == 1 - eor w11,w11,w15 -.endif -.inst 0x04673441 //xar z1.s,z1.s,z2.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.if mixin == 1 - eor w12,w12,w16 -.endif -.inst 0x046734c5 //xar z5.s,z5.s,z6.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.if mixin == 1 - eor w13,w13,w17 -.endif -.inst 0x04673549 //xar z9.s,z9.s,z10.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.if mixin == 1 - eor w14,w14,w18 -.endif -.inst 0x046735cd //xar z13.s,z13.s,z14.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.inst 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.inst 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.inst 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.inst 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.if mixin == 1 - eor w22,w22,w7 -.endif -.inst 0x0470340f //xar z15.s,z15.s,z0.s,16 -.if mixin == 1 - ror w22,w22,16 -.endif -.if mixin == 1 - eor w19,w19,w8 -.endif -.inst 0x04703483 //xar z3.s,z3.s,z4.s,16 -.if mixin == 1 - ror w19,w19,16 -.endif -.if mixin == 1 - eor w20,w20,w9 -.endif -.inst 0x04703507 //xar z7.s,z7.s,z8.s,16 -.if mixin == 1 - ror w20,w20,16 -.endif -.if mixin == 1 - eor w21,w21,w10 -.endif -.inst 0x0470358b //xar z11.s,z11.s,z12.s,16 -.if mixin == 1 - ror w21,w21,16 -.endif -.inst 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.inst 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.inst 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.inst 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.if mixin == 1 - eor w12,w12,w17 -.endif -.inst 0x046c3545 //xar z5.s,z5.s,z10.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.if mixin == 1 - eor w13,w13,w18 -.endif -.inst 0x046c35c9 //xar z9.s,z9.s,z14.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.if mixin == 1 - eor w14,w14,w15 -.endif -.inst 0x046c344d //xar z13.s,z13.s,z2.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.if mixin == 1 - eor w11,w11,w16 -.endif -.inst 0x046c34c1 //xar z1.s,z1.s,z6.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.inst 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.inst 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.inst 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.inst 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.if mixin == 1 - eor w22,w22,w7 -.endif -.inst 0x0468340f //xar z15.s,z15.s,z0.s,24 -.if mixin == 1 - ror w22,w22,24 -.endif -.if mixin == 1 - eor w19,w19,w8 -.endif -.inst 0x04683483 //xar z3.s,z3.s,z4.s,24 -.if mixin == 1 - ror w19,w19,24 -.endif -.if mixin == 1 - eor w20,w20,w9 -.endif -.inst 0x04683507 //xar z7.s,z7.s,z8.s,24 -.if mixin == 1 - ror w20,w20,24 -.endif -.if mixin == 1 - eor w21,w21,w10 -.endif -.inst 0x0468358b //xar z11.s,z11.s,z12.s,24 -.if mixin == 1 - ror w21,w21,24 -.endif -.inst 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.inst 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.inst 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.inst 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.if mixin == 1 - eor w12,w12,w17 -.endif -.inst 0x04673545 //xar z5.s,z5.s,z10.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.if mixin == 1 - eor w13,w13,w18 -.endif -.inst 0x046735c9 //xar z9.s,z9.s,z14.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.if mixin == 1 - eor w14,w14,w15 -.endif -.inst 0x0467344d //xar z13.s,z13.s,z2.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.if mixin == 1 - eor w11,w11,w16 -.endif -.inst 0x046734c1 //xar z1.s,z1.s,z6.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif - sub x6,x6,1 - cbnz x6,10b -.if mixin == 1 - add w7,w7,w23 -.endif -.inst 0x04b90000 //add z0.s,z0.s,z25.s -.if mixin == 1 - add x8,x8,x23,lsr #32 -.endif -.inst 0x04ba0084 //add z4.s,z4.s,z26.s -.if mixin == 1 - add x7,x7,x8,lsl #32 // pack -.endif -.if mixin == 1 - add w9,w9,w24 -.endif -.inst 0x04bb0108 //add z8.s,z8.s,z27.s -.if mixin == 1 - add x10,x10,x24,lsr #32 -.endif -.inst 0x04bc018c //add z12.s,z12.s,z28.s -.if mixin == 1 - add x9,x9,x10,lsl #32 // pack -.endif -.if mixin == 1 - ldp x8,x10,[x1],#16 -.endif -.if mixin == 1 - add w11,w11,w25 -.endif -.inst 0x04bd0021 //add z1.s,z1.s,z29.s -.if mixin == 1 - add x12,x12,x25,lsr #32 -.endif -.inst 0x04be00a5 //add z5.s,z5.s,z30.s -.if mixin == 1 - add x11,x11,x12,lsl #32 // pack -.endif -.if mixin == 1 - add w13,w13,w26 -.endif -.inst 0x04b50129 //add z9.s,z9.s,z21.s -.if mixin == 1 - add x14,x14,x26,lsr #32 -.endif -.inst 0x04b601ad //add z13.s,z13.s,z22.s -.if mixin == 1 - add x13,x13,x14,lsl #32 // pack -.endif -.if mixin == 1 - ldp x12,x14,[x1],#16 -.endif -.if mixin == 1 - add w15,w15,w27 -.endif -.inst 0x04b70042 //add z2.s,z2.s,z23.s -.if mixin == 1 - add x16,x16,x27,lsr #32 -.endif -.inst 0x04b800c6 //add z6.s,z6.s,z24.s -.if mixin == 1 - add x15,x15,x16,lsl #32 // pack -.endif -.if mixin == 1 - add w17,w17,w28 -.endif -.inst 0x04b1014a //add z10.s,z10.s,z17.s -.if mixin == 1 - add x18,x18,x28,lsr #32 -.endif -.inst 0x04b201ce //add z14.s,z14.s,z18.s -.if mixin == 1 - add x17,x17,x18,lsl #32 // pack -.endif -.if mixin == 1 - ldp x16,x18,[x1],#16 -.endif -.if mixin == 1 - add w19,w19,w29 -.endif -.inst 0x04b00063 //add z3.s,z3.s,z16.s -.if mixin == 1 - add x20,x20,x29,lsr #32 -.endif -.inst 0x04b300e7 //add z7.s,z7.s,z19.s -.if mixin == 1 - add x19,x19,x20,lsl #32 // pack -.endif -.if mixin == 1 - add w21,w21,w30 -.endif -.inst 0x04b4016b //add z11.s,z11.s,z20.s -.if mixin == 1 - add x22,x22,x30,lsr #32 -.endif -.inst 0x04bf01ef //add z15.s,z15.s,z31.s -.if mixin == 1 - add x21,x21,x22,lsl #32 // pack -.endif -.if mixin == 1 - ldp x20,x22,[x1],#16 -.endif -#ifdef __AARCH64EB__ - rev x7,x7 - rev x9,x9 - rev x11,x11 - rev x13,x13 - rev x15,x15 - rev x17,x17 - rev x19,x19 - rev x21,x21 -#endif -.if mixin == 1 - add x29,x29,#1 -.endif - cmp x5,4 - b.ne 200f -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.if mixin == 1 - eor x11,x11,x12 -.endif -.inst 0x05a46011 //zip1 z17.s,z0.s,z4.s -.inst 0x05a46412 //zip2 z18.s,z0.s,z4.s -.inst 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.inst 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.inst 0x05a56035 //zip1 z21.s,z1.s,z5.s -.inst 0x05a56436 //zip2 z22.s,z1.s,z5.s -.inst 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.inst 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36624 //zip2 z4.d,z17.d,z19.d -.inst 0x05f46248 //zip1 z8.d,z18.d,z20.d -.inst 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.inst 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.inst 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x13,x13,x14 -.endif -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.inst 0x05a66051 //zip1 z17.s,z2.s,z6.s -.inst 0x05a66452 //zip2 z18.s,z2.s,z6.s -.inst 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.inst 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.inst 0x05a76075 //zip1 z21.s,z3.s,z7.s -.inst 0x05a76476 //zip2 z22.s,z3.s,z7.s -.inst 0x05af6177 //zip1 z23.s,z11.s,z15.s -.inst 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.inst 0x05f36222 //zip1 z2.d,z17.d,z19.d -.inst 0x05f36626 //zip2 z6.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.inst 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.inst 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.inst 0x05f862cb //zip1 z11.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.inst 0x04b13000 //eor z0.d,z0.d,z17.d -.inst 0x04b23021 //eor z1.d,z1.d,z18.d -.inst 0x04b33042 //eor z2.d,z2.d,z19.d -.inst 0x04b43063 //eor z3.d,z3.d,z20.d -.inst 0x04b53084 //eor z4.d,z4.d,z21.d -.inst 0x04b630a5 //eor z5.d,z5.d,z22.d -.inst 0x04b730c6 //eor z6.d,z6.d,z23.d -.inst 0x04b830e7 //eor z7.d,z7.d,z24.d - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.inst 0x04b13108 //eor z8.d,z8.d,z17.d -.inst 0x04b23129 //eor z9.d,z9.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.inst 0x04b3314a //eor z10.d,z10.d,z19.d -.inst 0x04b4316b //eor z11.d,z11.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.inst 0x04b5318c //eor z12.d,z12.d,z21.d -.inst 0x04b631ad //eor z13.d,z13.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.inst 0x04b731ce //eor z14.d,z14.d,z23.d -.inst 0x04b831ef //eor z15.d,z15.d,z24.d - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x0],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - st1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - st1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - b 210f -200: -.inst 0x05a16011 //zip1 z17.s,z0.s,z1.s -.inst 0x05a16412 //zip2 z18.s,z0.s,z1.s -.inst 0x05a36053 //zip1 z19.s,z2.s,z3.s -.inst 0x05a36454 //zip2 z20.s,z2.s,z3.s - -.inst 0x05a56095 //zip1 z21.s,z4.s,z5.s -.inst 0x05a56496 //zip2 z22.s,z4.s,z5.s -.inst 0x05a760d7 //zip1 z23.s,z6.s,z7.s -.inst 0x05a764d8 //zip2 z24.s,z6.s,z7.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36621 //zip2 z1.d,z17.d,z19.d -.inst 0x05f46242 //zip1 z2.d,z18.d,z20.d -.inst 0x05f46643 //zip2 z3.d,z18.d,z20.d - -.inst 0x05f762a4 //zip1 z4.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c6 //zip1 z6.d,z22.d,z24.d -.inst 0x05f866c7 //zip2 z7.d,z22.d,z24.d -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.inst 0x05a96111 //zip1 z17.s,z8.s,z9.s -.inst 0x05a96512 //zip2 z18.s,z8.s,z9.s -.inst 0x05ab6153 //zip1 z19.s,z10.s,z11.s -.inst 0x05ab6554 //zip2 z20.s,z10.s,z11.s - -.inst 0x05ad6195 //zip1 z21.s,z12.s,z13.s -.inst 0x05ad6596 //zip2 z22.s,z12.s,z13.s -.inst 0x05af61d7 //zip1 z23.s,z14.s,z15.s -.inst 0x05af65d8 //zip2 z24.s,z14.s,z15.s - -.inst 0x05f36228 //zip1 z8.d,z17.d,z19.d -.inst 0x05f36629 //zip2 z9.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664b //zip2 z11.d,z18.d,z20.d - -.inst 0x05f762ac //zip1 z12.d,z21.d,z23.d -.inst 0x05f766ad //zip2 z13.d,z21.d,z23.d -.inst 0x05f862ce //zip1 z14.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x11,x11,x12 -.endif -.if mixin == 1 - eor x13,x13,x14 -.endif -.inst 0x05a46011 //zip1 z17.s,z0.s,z4.s -.inst 0x05a46412 //zip2 z18.s,z0.s,z4.s -.inst 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.inst 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.inst 0x05a56035 //zip1 z21.s,z1.s,z5.s -.inst 0x05a56436 //zip2 z22.s,z1.s,z5.s -.inst 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.inst 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36624 //zip2 z4.d,z17.d,z19.d -.inst 0x05f46248 //zip1 z8.d,z18.d,z20.d -.inst 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.inst 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.inst 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.inst 0x05a66051 //zip1 z17.s,z2.s,z6.s -.inst 0x05a66452 //zip2 z18.s,z2.s,z6.s -.inst 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.inst 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.inst 0x05a76075 //zip1 z21.s,z3.s,z7.s -.inst 0x05a76476 //zip2 z22.s,z3.s,z7.s -.inst 0x05af6177 //zip1 z23.s,z11.s,z15.s -.inst 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.inst 0x05f36222 //zip1 z2.d,z17.d,z19.d -.inst 0x05f36626 //zip2 z6.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.inst 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.inst 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.inst 0x05f862cb //zip1 z11.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif -.inst 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.inst 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.inst 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.inst 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.inst 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.inst 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.inst 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.inst 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.inst 0x04215101 //addvl x1,x1,8 -.inst 0x04b13000 //eor z0.d,z0.d,z17.d -.inst 0x04b23084 //eor z4.d,z4.d,z18.d -.inst 0x04b33108 //eor z8.d,z8.d,z19.d -.inst 0x04b4318c //eor z12.d,z12.d,z20.d -.inst 0x04b53021 //eor z1.d,z1.d,z21.d -.inst 0x04b630a5 //eor z5.d,z5.d,z22.d -.inst 0x04b73129 //eor z9.d,z9.d,z23.d -.inst 0x04b831ad //eor z13.d,z13.d,z24.d -.inst 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.inst 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.inst 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.inst 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.inst 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.inst 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.inst 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.inst 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.inst 0x04215101 //addvl x1,x1,8 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.inst 0x04b13042 //eor z2.d,z2.d,z17.d -.inst 0x04b230c6 //eor z6.d,z6.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.inst 0x04b3314a //eor z10.d,z10.d,z19.d -.inst 0x04b431ce //eor z14.d,z14.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.inst 0x04b53063 //eor z3.d,z3.d,z21.d -.inst 0x04b630e7 //eor z7.d,z7.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.inst 0x04b7316b //eor z11.d,z11.d,z23.d -.inst 0x04b831ef //eor z15.d,z15.d,z24.d -#ifdef __AARCH64EB__ -.inst 0x05a48000 //revb z0.s,p0/m,z0.s -#endif -.inst 0xe540e000 //st1w {z0.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48084 //revb z4.s,p0/m,z4.s -#endif -.inst 0xe541e004 //st1w {z4.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48108 //revb z8.s,p0/m,z8.s -#endif -.inst 0xe542e008 //st1w {z8.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4818c //revb z12.s,p0/m,z12.s -#endif -.inst 0xe543e00c //st1w {z12.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48021 //revb z1.s,p0/m,z1.s -#endif -.inst 0xe544e001 //st1w {z1.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480a5 //revb z5.s,p0/m,z5.s -#endif -.inst 0xe545e005 //st1w {z5.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48129 //revb z9.s,p0/m,z9.s -#endif -.inst 0xe546e009 //st1w {z9.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ad //revb z13.s,p0/m,z13.s -#endif -.inst 0xe547e00d //st1w {z13.s},p0,[x0,#7,MUL VL] -.inst 0x04205100 //addvl x0,x0,8 -#ifdef __AARCH64EB__ -.inst 0x05a48042 //revb z2.s,p0/m,z2.s -#endif -.inst 0xe540e002 //st1w {z2.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480c6 //revb z6.s,p0/m,z6.s -#endif -.inst 0xe541e006 //st1w {z6.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4814a //revb z10.s,p0/m,z10.s -#endif -.inst 0xe542e00a //st1w {z10.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ce //revb z14.s,p0/m,z14.s -#endif -.inst 0xe543e00e //st1w {z14.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48063 //revb z3.s,p0/m,z3.s -#endif -.inst 0xe544e003 //st1w {z3.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480e7 //revb z7.s,p0/m,z7.s -#endif -.inst 0xe545e007 //st1w {z7.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4816b //revb z11.s,p0/m,z11.s -#endif -.inst 0xe546e00b //st1w {z11.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ef //revb z15.s,p0/m,z15.s -#endif -.inst 0xe547e00f //st1w {z15.s},p0,[x0,#7,MUL VL] -.inst 0x04205100 //addvl x0,x0,8 -210: -.inst 0x04b0e3fd //incw x29, ALL, MUL #1 -110: - b 2f -1: -.align 5 -100: - subs x7,x2,x5,lsl #6 - b.lt 110f - mov x2,x7 - b.eq 101f - cmp x2,64 - b.lt 101f - mixin=1 - lsr x8,x23,#32 -.inst 0x05a03ae0 //dup z0.s,w23 -.inst 0x05a03af9 //dup z25.s,w23 -.if mixin == 1 - mov w7,w23 -.endif -.inst 0x05a03904 //dup z4.s,w8 -.inst 0x05a0391a //dup z26.s,w8 - lsr x10,x24,#32 -.inst 0x05a03b08 //dup z8.s,w24 -.inst 0x05a03b1b //dup z27.s,w24 -.if mixin == 1 - mov w9,w24 -.endif -.inst 0x05a0394c //dup z12.s,w10 -.inst 0x05a0395c //dup z28.s,w10 - lsr x12,x25,#32 -.inst 0x05a03b21 //dup z1.s,w25 -.inst 0x05a03b3d //dup z29.s,w25 -.if mixin == 1 - mov w11,w25 -.endif -.inst 0x05a03985 //dup z5.s,w12 -.inst 0x05a0399e //dup z30.s,w12 - lsr x14,x26,#32 -.inst 0x05a03b49 //dup z9.s,w26 -.inst 0x05a03b55 //dup z21.s,w26 -.if mixin == 1 - mov w13,w26 -.endif -.inst 0x05a039cd //dup z13.s,w14 -.inst 0x05a039d6 //dup z22.s,w14 - lsr x16,x27,#32 -.inst 0x05a03b62 //dup z2.s,w27 -.inst 0x05a03b77 //dup z23.s,w27 -.if mixin == 1 - mov w15,w27 -.endif -.inst 0x05a03a06 //dup z6.s,w16 -.inst 0x05a03a18 //dup z24.s,w16 - lsr x18,x28,#32 -.inst 0x05a03b8a //dup z10.s,w28 -.if mixin == 1 - mov w17,w28 -.endif -.inst 0x05a03a4e //dup z14.s,w18 - lsr x22,x30,#32 -.inst 0x05a03bcb //dup z11.s,w30 -.if mixin == 1 - mov w21,w30 -.endif -.inst 0x05a03acf //dup z15.s,w22 -.if mixin == 1 - add w20,w29,#1 - mov w19,w29 -.inst 0x04a14690 //index z16.s,w20,1 -.inst 0x04a14683 //index z3.s,w20,1 -.else -.inst 0x04a147b0 //index z16.s,w29,1 -.inst 0x04a147a3 //index z3.s,w29,1 -.endif - lsr x20,x29,#32 -.inst 0x05a03a87 //dup z7.s,w20 - mov x6,#10 -10: -.align 5 -.inst 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.inst 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.inst 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.inst 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.inst 0x04a03063 //eor z3.d,z3.d,z0.d -.if mixin == 1 - eor w19,w19,w7 -.endif -.inst 0x04a430e7 //eor z7.d,z7.d,z4.d -.if mixin == 1 - eor w20,w20,w8 -.endif -.inst 0x04a8316b //eor z11.d,z11.d,z8.d -.if mixin == 1 - eor w21,w21,w9 -.endif -.inst 0x04ac31ef //eor z15.d,z15.d,z12.d -.if mixin == 1 - eor w22,w22,w10 -.endif -.inst 0x05a58063 //revh z3.s,p0/m,z3.s -.if mixin == 1 - ror w19,w19,#16 -.endif -.inst 0x05a580e7 //revh z7.s,p0/m,z7.s -.if mixin == 1 - ror w20,w20,#16 -.endif -.inst 0x05a5816b //revh z11.s,p0/m,z11.s -.if mixin == 1 - ror w21,w21,#16 -.endif -.inst 0x05a581ef //revh z15.s,p0/m,z15.s -.if mixin == 1 - ror w22,w22,#16 -.endif -.inst 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.inst 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.inst 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.inst 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.inst 0x04a23021 //eor z1.d,z1.d,z2.d -.if mixin == 1 - eor w11,w11,w15 -.endif -.inst 0x04a630a5 //eor z5.d,z5.d,z6.d -.if mixin == 1 - eor w12,w12,w16 -.endif -.inst 0x04aa3129 //eor z9.d,z9.d,z10.d -.if mixin == 1 - eor w13,w13,w17 -.endif -.inst 0x04ae31ad //eor z13.d,z13.d,z14.d -.if mixin == 1 - eor w14,w14,w18 -.endif -.inst 0x046c9c31 //lsl z17.s,z1.s,12 -.inst 0x046c9cb2 //lsl z18.s,z5.s,12 -.inst 0x046c9d33 //lsl z19.s,z9.s,12 -.inst 0x046c9db4 //lsl z20.s,z13.s,12 -.inst 0x046c9421 //lsr z1.s,z1.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.inst 0x046c94a5 //lsr z5.s,z5.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.inst 0x046c9529 //lsr z9.s,z9.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.inst 0x046c95ad //lsr z13.s,z13.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.inst 0x04713021 //orr z1.d,z1.d,z17.d -.inst 0x047230a5 //orr z5.d,z5.d,z18.d -.inst 0x04733129 //orr z9.d,z9.d,z19.d -.inst 0x047431ad //orr z13.d,z13.d,z20.d -.inst 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.inst 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.inst 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.inst 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.inst 0x04a03063 //eor z3.d,z3.d,z0.d -.if mixin == 1 - eor w19,w19,w7 -.endif -.inst 0x04a430e7 //eor z7.d,z7.d,z4.d -.if mixin == 1 - eor w20,w20,w8 -.endif -.inst 0x04a8316b //eor z11.d,z11.d,z8.d -.if mixin == 1 - eor w21,w21,w9 -.endif -.inst 0x04ac31ef //eor z15.d,z15.d,z12.d -.if mixin == 1 - eor w22,w22,w10 -.endif -.inst 0x053f3063 //tbl z3.b,{z3.b},z31.b -.if mixin == 1 - ror w19,w19,#24 -.endif -.inst 0x053f30e7 //tbl z7.b,{z7.b},z31.b -.if mixin == 1 - ror w20,w20,#24 -.endif -.inst 0x053f316b //tbl z11.b,{z11.b},z31.b -.if mixin == 1 - ror w21,w21,#24 -.endif -.inst 0x053f31ef //tbl z15.b,{z15.b},z31.b -.if mixin == 1 - ror w22,w22,#24 -.endif -.inst 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.inst 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.inst 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.inst 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.inst 0x04a23021 //eor z1.d,z1.d,z2.d -.if mixin == 1 - eor w11,w11,w15 -.endif -.inst 0x04a630a5 //eor z5.d,z5.d,z6.d -.if mixin == 1 - eor w12,w12,w16 -.endif -.inst 0x04aa3129 //eor z9.d,z9.d,z10.d -.if mixin == 1 - eor w13,w13,w17 -.endif -.inst 0x04ae31ad //eor z13.d,z13.d,z14.d -.if mixin == 1 - eor w14,w14,w18 -.endif -.inst 0x04679c31 //lsl z17.s,z1.s,7 -.inst 0x04679cb2 //lsl z18.s,z5.s,7 -.inst 0x04679d33 //lsl z19.s,z9.s,7 -.inst 0x04679db4 //lsl z20.s,z13.s,7 -.inst 0x04679421 //lsr z1.s,z1.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.inst 0x046794a5 //lsr z5.s,z5.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.inst 0x04679529 //lsr z9.s,z9.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.inst 0x046795ad //lsr z13.s,z13.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.inst 0x04713021 //orr z1.d,z1.d,z17.d -.inst 0x047230a5 //orr z5.d,z5.d,z18.d -.inst 0x04733129 //orr z9.d,z9.d,z19.d -.inst 0x047431ad //orr z13.d,z13.d,z20.d -.inst 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.inst 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.inst 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.inst 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.inst 0x04a031ef //eor z15.d,z15.d,z0.d -.if mixin == 1 - eor w22,w22,w7 -.endif -.inst 0x04a43063 //eor z3.d,z3.d,z4.d -.if mixin == 1 - eor w19,w19,w8 -.endif -.inst 0x04a830e7 //eor z7.d,z7.d,z8.d -.if mixin == 1 - eor w20,w20,w9 -.endif -.inst 0x04ac316b //eor z11.d,z11.d,z12.d -.if mixin == 1 - eor w21,w21,w10 -.endif -.inst 0x05a581ef //revh z15.s,p0/m,z15.s -.if mixin == 1 - ror w22,w22,#16 -.endif -.inst 0x05a58063 //revh z3.s,p0/m,z3.s -.if mixin == 1 - ror w19,w19,#16 -.endif -.inst 0x05a580e7 //revh z7.s,p0/m,z7.s -.if mixin == 1 - ror w20,w20,#16 -.endif -.inst 0x05a5816b //revh z11.s,p0/m,z11.s -.if mixin == 1 - ror w21,w21,#16 -.endif -.inst 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.inst 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.inst 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.inst 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.inst 0x04aa30a5 //eor z5.d,z5.d,z10.d -.if mixin == 1 - eor w12,w12,w17 -.endif -.inst 0x04ae3129 //eor z9.d,z9.d,z14.d -.if mixin == 1 - eor w13,w13,w18 -.endif -.inst 0x04a231ad //eor z13.d,z13.d,z2.d -.if mixin == 1 - eor w14,w14,w15 -.endif -.inst 0x04a63021 //eor z1.d,z1.d,z6.d -.if mixin == 1 - eor w11,w11,w16 -.endif -.inst 0x046c9cb1 //lsl z17.s,z5.s,12 -.inst 0x046c9d32 //lsl z18.s,z9.s,12 -.inst 0x046c9db3 //lsl z19.s,z13.s,12 -.inst 0x046c9c34 //lsl z20.s,z1.s,12 -.inst 0x046c94a5 //lsr z5.s,z5.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.inst 0x046c9529 //lsr z9.s,z9.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.inst 0x046c95ad //lsr z13.s,z13.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.inst 0x046c9421 //lsr z1.s,z1.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.inst 0x047130a5 //orr z5.d,z5.d,z17.d -.inst 0x04723129 //orr z9.d,z9.d,z18.d -.inst 0x047331ad //orr z13.d,z13.d,z19.d -.inst 0x04743021 //orr z1.d,z1.d,z20.d -.inst 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.inst 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.inst 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.inst 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.inst 0x04a031ef //eor z15.d,z15.d,z0.d -.if mixin == 1 - eor w22,w22,w7 -.endif -.inst 0x04a43063 //eor z3.d,z3.d,z4.d -.if mixin == 1 - eor w19,w19,w8 -.endif -.inst 0x04a830e7 //eor z7.d,z7.d,z8.d -.if mixin == 1 - eor w20,w20,w9 -.endif -.inst 0x04ac316b //eor z11.d,z11.d,z12.d -.if mixin == 1 - eor w21,w21,w10 -.endif -.inst 0x053f31ef //tbl z15.b,{z15.b},z31.b -.if mixin == 1 - ror w22,w22,#24 -.endif -.inst 0x053f3063 //tbl z3.b,{z3.b},z31.b -.if mixin == 1 - ror w19,w19,#24 -.endif -.inst 0x053f30e7 //tbl z7.b,{z7.b},z31.b -.if mixin == 1 - ror w20,w20,#24 -.endif -.inst 0x053f316b //tbl z11.b,{z11.b},z31.b -.if mixin == 1 - ror w21,w21,#24 -.endif -.inst 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.inst 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.inst 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.inst 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.inst 0x04aa30a5 //eor z5.d,z5.d,z10.d -.if mixin == 1 - eor w12,w12,w17 -.endif -.inst 0x04ae3129 //eor z9.d,z9.d,z14.d -.if mixin == 1 - eor w13,w13,w18 -.endif -.inst 0x04a231ad //eor z13.d,z13.d,z2.d -.if mixin == 1 - eor w14,w14,w15 -.endif -.inst 0x04a63021 //eor z1.d,z1.d,z6.d -.if mixin == 1 - eor w11,w11,w16 -.endif -.inst 0x04679cb1 //lsl z17.s,z5.s,7 -.inst 0x04679d32 //lsl z18.s,z9.s,7 -.inst 0x04679db3 //lsl z19.s,z13.s,7 -.inst 0x04679c34 //lsl z20.s,z1.s,7 -.inst 0x046794a5 //lsr z5.s,z5.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.inst 0x04679529 //lsr z9.s,z9.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.inst 0x046795ad //lsr z13.s,z13.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.inst 0x04679421 //lsr z1.s,z1.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.inst 0x047130a5 //orr z5.d,z5.d,z17.d -.inst 0x04723129 //orr z9.d,z9.d,z18.d -.inst 0x047331ad //orr z13.d,z13.d,z19.d -.inst 0x04743021 //orr z1.d,z1.d,z20.d - sub x6,x6,1 - cbnz x6,10b - lsr x6,x28,#32 -.inst 0x05a03b91 //dup z17.s,w28 -.inst 0x05a038d2 //dup z18.s,w6 - lsr x6,x29,#32 -.inst 0x05a038d3 //dup z19.s,w6 - lsr x6,x30,#32 -.if mixin == 1 - add w7,w7,w23 -.endif -.inst 0x04b90000 //add z0.s,z0.s,z25.s -.if mixin == 1 - add x8,x8,x23,lsr #32 -.endif -.inst 0x04ba0084 //add z4.s,z4.s,z26.s -.if mixin == 1 - add x7,x7,x8,lsl #32 // pack -.endif -.if mixin == 1 - add w9,w9,w24 -.endif -.inst 0x04bb0108 //add z8.s,z8.s,z27.s -.if mixin == 1 - add x10,x10,x24,lsr #32 -.endif -.inst 0x04bc018c //add z12.s,z12.s,z28.s -.if mixin == 1 - add x9,x9,x10,lsl #32 // pack -.endif -.if mixin == 1 - ldp x8,x10,[x1],#16 -.endif -.if mixin == 1 - add w11,w11,w25 -.endif -.inst 0x04bd0021 //add z1.s,z1.s,z29.s -.if mixin == 1 - add x12,x12,x25,lsr #32 -.endif -.inst 0x04be00a5 //add z5.s,z5.s,z30.s -.if mixin == 1 - add x11,x11,x12,lsl #32 // pack -.endif -.if mixin == 1 - add w13,w13,w26 -.endif -.inst 0x04b50129 //add z9.s,z9.s,z21.s -.if mixin == 1 - add x14,x14,x26,lsr #32 -.endif -.inst 0x04b601ad //add z13.s,z13.s,z22.s -.if mixin == 1 - add x13,x13,x14,lsl #32 // pack -.endif -.if mixin == 1 - ldp x12,x14,[x1],#16 -.endif -.if mixin == 1 - add w15,w15,w27 -.endif -.inst 0x04b70042 //add z2.s,z2.s,z23.s -.if mixin == 1 - add x16,x16,x27,lsr #32 -.endif -.inst 0x04b800c6 //add z6.s,z6.s,z24.s -.if mixin == 1 - add x15,x15,x16,lsl #32 // pack -.endif -.if mixin == 1 - add w17,w17,w28 -.endif -.inst 0x04b1014a //add z10.s,z10.s,z17.s -.if mixin == 1 - add x18,x18,x28,lsr #32 -.endif -.inst 0x04b201ce //add z14.s,z14.s,z18.s -.if mixin == 1 - add x17,x17,x18,lsl #32 // pack -.endif -.if mixin == 1 - ldp x16,x18,[x1],#16 -.endif -.inst 0x05a03bd4 //dup z20.s,w30 -.inst 0x05a038d9 //dup z25.s,w6 // bak[15] not available for SVE -.if mixin == 1 - add w19,w19,w29 -.endif -.inst 0x04b00063 //add z3.s,z3.s,z16.s -.if mixin == 1 - add x20,x20,x29,lsr #32 -.endif -.inst 0x04b300e7 //add z7.s,z7.s,z19.s -.if mixin == 1 - add x19,x19,x20,lsl #32 // pack -.endif -.if mixin == 1 - add w21,w21,w30 -.endif -.inst 0x04b4016b //add z11.s,z11.s,z20.s -.if mixin == 1 - add x22,x22,x30,lsr #32 -.endif -.inst 0x04b901ef //add z15.s,z15.s,z25.s -.if mixin == 1 - add x21,x21,x22,lsl #32 // pack -.endif -.if mixin == 1 - ldp x20,x22,[x1],#16 -.endif -#ifdef __AARCH64EB__ - rev x7,x7 - rev x9,x9 - rev x11,x11 - rev x13,x13 - rev x15,x15 - rev x17,x17 - rev x19,x19 - rev x21,x21 -#endif -.if mixin == 1 - add x29,x29,#1 -.endif - cmp x5,4 - b.ne 200f -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.if mixin == 1 - eor x11,x11,x12 -.endif -.inst 0x05a46011 //zip1 z17.s,z0.s,z4.s -.inst 0x05a46412 //zip2 z18.s,z0.s,z4.s -.inst 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.inst 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.inst 0x05a56035 //zip1 z21.s,z1.s,z5.s -.inst 0x05a56436 //zip2 z22.s,z1.s,z5.s -.inst 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.inst 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36624 //zip2 z4.d,z17.d,z19.d -.inst 0x05f46248 //zip1 z8.d,z18.d,z20.d -.inst 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.inst 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.inst 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x13,x13,x14 -.endif -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.inst 0x05a66051 //zip1 z17.s,z2.s,z6.s -.inst 0x05a66452 //zip2 z18.s,z2.s,z6.s -.inst 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.inst 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.inst 0x05a76075 //zip1 z21.s,z3.s,z7.s -.inst 0x05a76476 //zip2 z22.s,z3.s,z7.s -.inst 0x05af6177 //zip1 z23.s,z11.s,z15.s -.inst 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.inst 0x05f36222 //zip1 z2.d,z17.d,z19.d -.inst 0x05f36626 //zip2 z6.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.inst 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.inst 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.inst 0x05f862cb //zip1 z11.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.inst 0x04b13000 //eor z0.d,z0.d,z17.d -.inst 0x04b23021 //eor z1.d,z1.d,z18.d -.inst 0x04b33042 //eor z2.d,z2.d,z19.d -.inst 0x04b43063 //eor z3.d,z3.d,z20.d -.inst 0x04b53084 //eor z4.d,z4.d,z21.d -.inst 0x04b630a5 //eor z5.d,z5.d,z22.d -.inst 0x04b730c6 //eor z6.d,z6.d,z23.d -.inst 0x04b830e7 //eor z7.d,z7.d,z24.d - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.inst 0x04b13108 //eor z8.d,z8.d,z17.d -.inst 0x04b23129 //eor z9.d,z9.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.inst 0x04b3314a //eor z10.d,z10.d,z19.d -.inst 0x04b4316b //eor z11.d,z11.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.inst 0x04b5318c //eor z12.d,z12.d,z21.d -.inst 0x04b631ad //eor z13.d,z13.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.inst 0x04b731ce //eor z14.d,z14.d,z23.d -.inst 0x04b831ef //eor z15.d,z15.d,z24.d - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x0],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - st1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - st1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - b 210f -200: -.inst 0x05a16011 //zip1 z17.s,z0.s,z1.s -.inst 0x05a16412 //zip2 z18.s,z0.s,z1.s -.inst 0x05a36053 //zip1 z19.s,z2.s,z3.s -.inst 0x05a36454 //zip2 z20.s,z2.s,z3.s - -.inst 0x05a56095 //zip1 z21.s,z4.s,z5.s -.inst 0x05a56496 //zip2 z22.s,z4.s,z5.s -.inst 0x05a760d7 //zip1 z23.s,z6.s,z7.s -.inst 0x05a764d8 //zip2 z24.s,z6.s,z7.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36621 //zip2 z1.d,z17.d,z19.d -.inst 0x05f46242 //zip1 z2.d,z18.d,z20.d -.inst 0x05f46643 //zip2 z3.d,z18.d,z20.d - -.inst 0x05f762a4 //zip1 z4.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c6 //zip1 z6.d,z22.d,z24.d -.inst 0x05f866c7 //zip2 z7.d,z22.d,z24.d -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.inst 0x05a96111 //zip1 z17.s,z8.s,z9.s -.inst 0x05a96512 //zip2 z18.s,z8.s,z9.s -.inst 0x05ab6153 //zip1 z19.s,z10.s,z11.s -.inst 0x05ab6554 //zip2 z20.s,z10.s,z11.s - -.inst 0x05ad6195 //zip1 z21.s,z12.s,z13.s -.inst 0x05ad6596 //zip2 z22.s,z12.s,z13.s -.inst 0x05af61d7 //zip1 z23.s,z14.s,z15.s -.inst 0x05af65d8 //zip2 z24.s,z14.s,z15.s - -.inst 0x05f36228 //zip1 z8.d,z17.d,z19.d -.inst 0x05f36629 //zip2 z9.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664b //zip2 z11.d,z18.d,z20.d - -.inst 0x05f762ac //zip1 z12.d,z21.d,z23.d -.inst 0x05f766ad //zip2 z13.d,z21.d,z23.d -.inst 0x05f862ce //zip1 z14.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x11,x11,x12 -.endif -.if mixin == 1 - eor x13,x13,x14 -.endif -.inst 0x05a46011 //zip1 z17.s,z0.s,z4.s -.inst 0x05a46412 //zip2 z18.s,z0.s,z4.s -.inst 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.inst 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.inst 0x05a56035 //zip1 z21.s,z1.s,z5.s -.inst 0x05a56436 //zip2 z22.s,z1.s,z5.s -.inst 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.inst 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36624 //zip2 z4.d,z17.d,z19.d -.inst 0x05f46248 //zip1 z8.d,z18.d,z20.d -.inst 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.inst 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.inst 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.inst 0x05a66051 //zip1 z17.s,z2.s,z6.s -.inst 0x05a66452 //zip2 z18.s,z2.s,z6.s -.inst 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.inst 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.inst 0x05a76075 //zip1 z21.s,z3.s,z7.s -.inst 0x05a76476 //zip2 z22.s,z3.s,z7.s -.inst 0x05af6177 //zip1 z23.s,z11.s,z15.s -.inst 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.inst 0x05f36222 //zip1 z2.d,z17.d,z19.d -.inst 0x05f36626 //zip2 z6.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.inst 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.inst 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.inst 0x05f862cb //zip1 z11.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif -.inst 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.inst 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.inst 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.inst 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.inst 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.inst 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.inst 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.inst 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.inst 0x04215101 //addvl x1,x1,8 -.inst 0x04b13000 //eor z0.d,z0.d,z17.d -.inst 0x04b23084 //eor z4.d,z4.d,z18.d -.inst 0x04b33108 //eor z8.d,z8.d,z19.d -.inst 0x04b4318c //eor z12.d,z12.d,z20.d -.inst 0x04b53021 //eor z1.d,z1.d,z21.d -.inst 0x04b630a5 //eor z5.d,z5.d,z22.d -.inst 0x04b73129 //eor z9.d,z9.d,z23.d -.inst 0x04b831ad //eor z13.d,z13.d,z24.d -.inst 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.inst 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.inst 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.inst 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.inst 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.inst 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.inst 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.inst 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.inst 0x04215101 //addvl x1,x1,8 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.inst 0x04b13042 //eor z2.d,z2.d,z17.d -.inst 0x04b230c6 //eor z6.d,z6.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.inst 0x04b3314a //eor z10.d,z10.d,z19.d -.inst 0x04b431ce //eor z14.d,z14.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.inst 0x04b53063 //eor z3.d,z3.d,z21.d -.inst 0x04b630e7 //eor z7.d,z7.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.inst 0x04b7316b //eor z11.d,z11.d,z23.d -.inst 0x04b831ef //eor z15.d,z15.d,z24.d -#ifdef __AARCH64EB__ -.inst 0x05a48000 //revb z0.s,p0/m,z0.s -#endif -.inst 0xe540e000 //st1w {z0.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48084 //revb z4.s,p0/m,z4.s -#endif -.inst 0xe541e004 //st1w {z4.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48108 //revb z8.s,p0/m,z8.s -#endif -.inst 0xe542e008 //st1w {z8.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4818c //revb z12.s,p0/m,z12.s -#endif -.inst 0xe543e00c //st1w {z12.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48021 //revb z1.s,p0/m,z1.s -#endif -.inst 0xe544e001 //st1w {z1.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480a5 //revb z5.s,p0/m,z5.s -#endif -.inst 0xe545e005 //st1w {z5.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48129 //revb z9.s,p0/m,z9.s -#endif -.inst 0xe546e009 //st1w {z9.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ad //revb z13.s,p0/m,z13.s -#endif -.inst 0xe547e00d //st1w {z13.s},p0,[x0,#7,MUL VL] -.inst 0x04205100 //addvl x0,x0,8 -#ifdef __AARCH64EB__ -.inst 0x05a48042 //revb z2.s,p0/m,z2.s -#endif -.inst 0xe540e002 //st1w {z2.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480c6 //revb z6.s,p0/m,z6.s -#endif -.inst 0xe541e006 //st1w {z6.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4814a //revb z10.s,p0/m,z10.s -#endif -.inst 0xe542e00a //st1w {z10.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ce //revb z14.s,p0/m,z14.s -#endif -.inst 0xe543e00e //st1w {z14.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48063 //revb z3.s,p0/m,z3.s -#endif -.inst 0xe544e003 //st1w {z3.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480e7 //revb z7.s,p0/m,z7.s -#endif -.inst 0xe545e007 //st1w {z7.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4816b //revb z11.s,p0/m,z11.s -#endif -.inst 0xe546e00b //st1w {z11.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ef //revb z15.s,p0/m,z15.s -#endif -.inst 0xe547e00f //st1w {z15.s},p0,[x0,#7,MUL VL] -.inst 0x04205100 //addvl x0,x0,8 -210: -.inst 0x04b0e3fd //incw x29, ALL, MUL #1 - subs x2,x2,64 - b.gt 100b - b 110f -101: - mixin=0 - lsr x8,x23,#32 -.inst 0x05a03ae0 //dup z0.s,w23 -.inst 0x05a03af9 //dup z25.s,w23 -.if mixin == 1 - mov w7,w23 -.endif -.inst 0x05a03904 //dup z4.s,w8 -.inst 0x05a0391a //dup z26.s,w8 - lsr x10,x24,#32 -.inst 0x05a03b08 //dup z8.s,w24 -.inst 0x05a03b1b //dup z27.s,w24 -.if mixin == 1 - mov w9,w24 -.endif -.inst 0x05a0394c //dup z12.s,w10 -.inst 0x05a0395c //dup z28.s,w10 - lsr x12,x25,#32 -.inst 0x05a03b21 //dup z1.s,w25 -.inst 0x05a03b3d //dup z29.s,w25 -.if mixin == 1 - mov w11,w25 -.endif -.inst 0x05a03985 //dup z5.s,w12 -.inst 0x05a0399e //dup z30.s,w12 - lsr x14,x26,#32 -.inst 0x05a03b49 //dup z9.s,w26 -.inst 0x05a03b55 //dup z21.s,w26 -.if mixin == 1 - mov w13,w26 -.endif -.inst 0x05a039cd //dup z13.s,w14 -.inst 0x05a039d6 //dup z22.s,w14 - lsr x16,x27,#32 -.inst 0x05a03b62 //dup z2.s,w27 -.inst 0x05a03b77 //dup z23.s,w27 -.if mixin == 1 - mov w15,w27 -.endif -.inst 0x05a03a06 //dup z6.s,w16 -.inst 0x05a03a18 //dup z24.s,w16 - lsr x18,x28,#32 -.inst 0x05a03b8a //dup z10.s,w28 -.if mixin == 1 - mov w17,w28 -.endif -.inst 0x05a03a4e //dup z14.s,w18 - lsr x22,x30,#32 -.inst 0x05a03bcb //dup z11.s,w30 -.if mixin == 1 - mov w21,w30 -.endif -.inst 0x05a03acf //dup z15.s,w22 -.if mixin == 1 - add w20,w29,#1 - mov w19,w29 -.inst 0x04a14690 //index z16.s,w20,1 -.inst 0x04a14683 //index z3.s,w20,1 -.else -.inst 0x04a147b0 //index z16.s,w29,1 -.inst 0x04a147a3 //index z3.s,w29,1 -.endif - lsr x20,x29,#32 -.inst 0x05a03a87 //dup z7.s,w20 - mov x6,#10 -10: -.align 5 -.inst 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.inst 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.inst 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.inst 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.inst 0x04a03063 //eor z3.d,z3.d,z0.d -.if mixin == 1 - eor w19,w19,w7 -.endif -.inst 0x04a430e7 //eor z7.d,z7.d,z4.d -.if mixin == 1 - eor w20,w20,w8 -.endif -.inst 0x04a8316b //eor z11.d,z11.d,z8.d -.if mixin == 1 - eor w21,w21,w9 -.endif -.inst 0x04ac31ef //eor z15.d,z15.d,z12.d -.if mixin == 1 - eor w22,w22,w10 -.endif -.inst 0x05a58063 //revh z3.s,p0/m,z3.s -.if mixin == 1 - ror w19,w19,#16 -.endif -.inst 0x05a580e7 //revh z7.s,p0/m,z7.s -.if mixin == 1 - ror w20,w20,#16 -.endif -.inst 0x05a5816b //revh z11.s,p0/m,z11.s -.if mixin == 1 - ror w21,w21,#16 -.endif -.inst 0x05a581ef //revh z15.s,p0/m,z15.s -.if mixin == 1 - ror w22,w22,#16 -.endif -.inst 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.inst 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.inst 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.inst 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.inst 0x04a23021 //eor z1.d,z1.d,z2.d -.if mixin == 1 - eor w11,w11,w15 -.endif -.inst 0x04a630a5 //eor z5.d,z5.d,z6.d -.if mixin == 1 - eor w12,w12,w16 -.endif -.inst 0x04aa3129 //eor z9.d,z9.d,z10.d -.if mixin == 1 - eor w13,w13,w17 -.endif -.inst 0x04ae31ad //eor z13.d,z13.d,z14.d -.if mixin == 1 - eor w14,w14,w18 -.endif -.inst 0x046c9c31 //lsl z17.s,z1.s,12 -.inst 0x046c9cb2 //lsl z18.s,z5.s,12 -.inst 0x046c9d33 //lsl z19.s,z9.s,12 -.inst 0x046c9db4 //lsl z20.s,z13.s,12 -.inst 0x046c9421 //lsr z1.s,z1.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.inst 0x046c94a5 //lsr z5.s,z5.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.inst 0x046c9529 //lsr z9.s,z9.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.inst 0x046c95ad //lsr z13.s,z13.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.inst 0x04713021 //orr z1.d,z1.d,z17.d -.inst 0x047230a5 //orr z5.d,z5.d,z18.d -.inst 0x04733129 //orr z9.d,z9.d,z19.d -.inst 0x047431ad //orr z13.d,z13.d,z20.d -.inst 0x04a10000 //add z0.s,z0.s,z1.s -.if mixin == 1 - add w7,w7,w11 -.endif -.inst 0x04a50084 //add z4.s,z4.s,z5.s -.if mixin == 1 - add w8,w8,w12 -.endif -.inst 0x04a90108 //add z8.s,z8.s,z9.s -.if mixin == 1 - add w9,w9,w13 -.endif -.inst 0x04ad018c //add z12.s,z12.s,z13.s -.if mixin == 1 - add w10,w10,w14 -.endif -.inst 0x04a03063 //eor z3.d,z3.d,z0.d -.if mixin == 1 - eor w19,w19,w7 -.endif -.inst 0x04a430e7 //eor z7.d,z7.d,z4.d -.if mixin == 1 - eor w20,w20,w8 -.endif -.inst 0x04a8316b //eor z11.d,z11.d,z8.d -.if mixin == 1 - eor w21,w21,w9 -.endif -.inst 0x04ac31ef //eor z15.d,z15.d,z12.d -.if mixin == 1 - eor w22,w22,w10 -.endif -.inst 0x053f3063 //tbl z3.b,{z3.b},z31.b -.if mixin == 1 - ror w19,w19,#24 -.endif -.inst 0x053f30e7 //tbl z7.b,{z7.b},z31.b -.if mixin == 1 - ror w20,w20,#24 -.endif -.inst 0x053f316b //tbl z11.b,{z11.b},z31.b -.if mixin == 1 - ror w21,w21,#24 -.endif -.inst 0x053f31ef //tbl z15.b,{z15.b},z31.b -.if mixin == 1 - ror w22,w22,#24 -.endif -.inst 0x04a30042 //add z2.s,z2.s,z3.s -.if mixin == 1 - add w15,w15,w19 -.endif -.inst 0x04a700c6 //add z6.s,z6.s,z7.s -.if mixin == 1 - add w16,w16,w20 -.endif -.inst 0x04ab014a //add z10.s,z10.s,z11.s -.if mixin == 1 - add w17,w17,w21 -.endif -.inst 0x04af01ce //add z14.s,z14.s,z15.s -.if mixin == 1 - add w18,w18,w22 -.endif -.inst 0x04a23021 //eor z1.d,z1.d,z2.d -.if mixin == 1 - eor w11,w11,w15 -.endif -.inst 0x04a630a5 //eor z5.d,z5.d,z6.d -.if mixin == 1 - eor w12,w12,w16 -.endif -.inst 0x04aa3129 //eor z9.d,z9.d,z10.d -.if mixin == 1 - eor w13,w13,w17 -.endif -.inst 0x04ae31ad //eor z13.d,z13.d,z14.d -.if mixin == 1 - eor w14,w14,w18 -.endif -.inst 0x04679c31 //lsl z17.s,z1.s,7 -.inst 0x04679cb2 //lsl z18.s,z5.s,7 -.inst 0x04679d33 //lsl z19.s,z9.s,7 -.inst 0x04679db4 //lsl z20.s,z13.s,7 -.inst 0x04679421 //lsr z1.s,z1.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.inst 0x046794a5 //lsr z5.s,z5.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.inst 0x04679529 //lsr z9.s,z9.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.inst 0x046795ad //lsr z13.s,z13.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.inst 0x04713021 //orr z1.d,z1.d,z17.d -.inst 0x047230a5 //orr z5.d,z5.d,z18.d -.inst 0x04733129 //orr z9.d,z9.d,z19.d -.inst 0x047431ad //orr z13.d,z13.d,z20.d -.inst 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.inst 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.inst 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.inst 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.inst 0x04a031ef //eor z15.d,z15.d,z0.d -.if mixin == 1 - eor w22,w22,w7 -.endif -.inst 0x04a43063 //eor z3.d,z3.d,z4.d -.if mixin == 1 - eor w19,w19,w8 -.endif -.inst 0x04a830e7 //eor z7.d,z7.d,z8.d -.if mixin == 1 - eor w20,w20,w9 -.endif -.inst 0x04ac316b //eor z11.d,z11.d,z12.d -.if mixin == 1 - eor w21,w21,w10 -.endif -.inst 0x05a581ef //revh z15.s,p0/m,z15.s -.if mixin == 1 - ror w22,w22,#16 -.endif -.inst 0x05a58063 //revh z3.s,p0/m,z3.s -.if mixin == 1 - ror w19,w19,#16 -.endif -.inst 0x05a580e7 //revh z7.s,p0/m,z7.s -.if mixin == 1 - ror w20,w20,#16 -.endif -.inst 0x05a5816b //revh z11.s,p0/m,z11.s -.if mixin == 1 - ror w21,w21,#16 -.endif -.inst 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.inst 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.inst 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.inst 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.inst 0x04aa30a5 //eor z5.d,z5.d,z10.d -.if mixin == 1 - eor w12,w12,w17 -.endif -.inst 0x04ae3129 //eor z9.d,z9.d,z14.d -.if mixin == 1 - eor w13,w13,w18 -.endif -.inst 0x04a231ad //eor z13.d,z13.d,z2.d -.if mixin == 1 - eor w14,w14,w15 -.endif -.inst 0x04a63021 //eor z1.d,z1.d,z6.d -.if mixin == 1 - eor w11,w11,w16 -.endif -.inst 0x046c9cb1 //lsl z17.s,z5.s,12 -.inst 0x046c9d32 //lsl z18.s,z9.s,12 -.inst 0x046c9db3 //lsl z19.s,z13.s,12 -.inst 0x046c9c34 //lsl z20.s,z1.s,12 -.inst 0x046c94a5 //lsr z5.s,z5.s,20 -.if mixin == 1 - ror w12,w12,20 -.endif -.inst 0x046c9529 //lsr z9.s,z9.s,20 -.if mixin == 1 - ror w13,w13,20 -.endif -.inst 0x046c95ad //lsr z13.s,z13.s,20 -.if mixin == 1 - ror w14,w14,20 -.endif -.inst 0x046c9421 //lsr z1.s,z1.s,20 -.if mixin == 1 - ror w11,w11,20 -.endif -.inst 0x047130a5 //orr z5.d,z5.d,z17.d -.inst 0x04723129 //orr z9.d,z9.d,z18.d -.inst 0x047331ad //orr z13.d,z13.d,z19.d -.inst 0x04743021 //orr z1.d,z1.d,z20.d -.inst 0x04a50000 //add z0.s,z0.s,z5.s -.if mixin == 1 - add w7,w7,w12 -.endif -.inst 0x04a90084 //add z4.s,z4.s,z9.s -.if mixin == 1 - add w8,w8,w13 -.endif -.inst 0x04ad0108 //add z8.s,z8.s,z13.s -.if mixin == 1 - add w9,w9,w14 -.endif -.inst 0x04a1018c //add z12.s,z12.s,z1.s -.if mixin == 1 - add w10,w10,w11 -.endif -.inst 0x04a031ef //eor z15.d,z15.d,z0.d -.if mixin == 1 - eor w22,w22,w7 -.endif -.inst 0x04a43063 //eor z3.d,z3.d,z4.d -.if mixin == 1 - eor w19,w19,w8 -.endif -.inst 0x04a830e7 //eor z7.d,z7.d,z8.d -.if mixin == 1 - eor w20,w20,w9 -.endif -.inst 0x04ac316b //eor z11.d,z11.d,z12.d -.if mixin == 1 - eor w21,w21,w10 -.endif -.inst 0x053f31ef //tbl z15.b,{z15.b},z31.b -.if mixin == 1 - ror w22,w22,#24 -.endif -.inst 0x053f3063 //tbl z3.b,{z3.b},z31.b -.if mixin == 1 - ror w19,w19,#24 -.endif -.inst 0x053f30e7 //tbl z7.b,{z7.b},z31.b -.if mixin == 1 - ror w20,w20,#24 -.endif -.inst 0x053f316b //tbl z11.b,{z11.b},z31.b -.if mixin == 1 - ror w21,w21,#24 -.endif -.inst 0x04af014a //add z10.s,z10.s,z15.s -.if mixin == 1 - add w17,w17,w22 -.endif -.inst 0x04a301ce //add z14.s,z14.s,z3.s -.if mixin == 1 - add w18,w18,w19 -.endif -.inst 0x04a70042 //add z2.s,z2.s,z7.s -.if mixin == 1 - add w15,w15,w20 -.endif -.inst 0x04ab00c6 //add z6.s,z6.s,z11.s -.if mixin == 1 - add w16,w16,w21 -.endif -.inst 0x04aa30a5 //eor z5.d,z5.d,z10.d -.if mixin == 1 - eor w12,w12,w17 -.endif -.inst 0x04ae3129 //eor z9.d,z9.d,z14.d -.if mixin == 1 - eor w13,w13,w18 -.endif -.inst 0x04a231ad //eor z13.d,z13.d,z2.d -.if mixin == 1 - eor w14,w14,w15 -.endif -.inst 0x04a63021 //eor z1.d,z1.d,z6.d -.if mixin == 1 - eor w11,w11,w16 -.endif -.inst 0x04679cb1 //lsl z17.s,z5.s,7 -.inst 0x04679d32 //lsl z18.s,z9.s,7 -.inst 0x04679db3 //lsl z19.s,z13.s,7 -.inst 0x04679c34 //lsl z20.s,z1.s,7 -.inst 0x046794a5 //lsr z5.s,z5.s,25 -.if mixin == 1 - ror w12,w12,25 -.endif -.inst 0x04679529 //lsr z9.s,z9.s,25 -.if mixin == 1 - ror w13,w13,25 -.endif -.inst 0x046795ad //lsr z13.s,z13.s,25 -.if mixin == 1 - ror w14,w14,25 -.endif -.inst 0x04679421 //lsr z1.s,z1.s,25 -.if mixin == 1 - ror w11,w11,25 -.endif -.inst 0x047130a5 //orr z5.d,z5.d,z17.d -.inst 0x04723129 //orr z9.d,z9.d,z18.d -.inst 0x047331ad //orr z13.d,z13.d,z19.d -.inst 0x04743021 //orr z1.d,z1.d,z20.d - sub x6,x6,1 - cbnz x6,10b - lsr x6,x28,#32 -.inst 0x05a03b91 //dup z17.s,w28 -.inst 0x05a038d2 //dup z18.s,w6 - lsr x6,x29,#32 -.inst 0x05a038d3 //dup z19.s,w6 - lsr x6,x30,#32 -.if mixin == 1 - add w7,w7,w23 -.endif -.inst 0x04b90000 //add z0.s,z0.s,z25.s -.if mixin == 1 - add x8,x8,x23,lsr #32 -.endif -.inst 0x04ba0084 //add z4.s,z4.s,z26.s -.if mixin == 1 - add x7,x7,x8,lsl #32 // pack -.endif -.if mixin == 1 - add w9,w9,w24 -.endif -.inst 0x04bb0108 //add z8.s,z8.s,z27.s -.if mixin == 1 - add x10,x10,x24,lsr #32 -.endif -.inst 0x04bc018c //add z12.s,z12.s,z28.s -.if mixin == 1 - add x9,x9,x10,lsl #32 // pack -.endif -.if mixin == 1 - ldp x8,x10,[x1],#16 -.endif -.if mixin == 1 - add w11,w11,w25 -.endif -.inst 0x04bd0021 //add z1.s,z1.s,z29.s -.if mixin == 1 - add x12,x12,x25,lsr #32 -.endif -.inst 0x04be00a5 //add z5.s,z5.s,z30.s -.if mixin == 1 - add x11,x11,x12,lsl #32 // pack -.endif -.if mixin == 1 - add w13,w13,w26 -.endif -.inst 0x04b50129 //add z9.s,z9.s,z21.s -.if mixin == 1 - add x14,x14,x26,lsr #32 -.endif -.inst 0x04b601ad //add z13.s,z13.s,z22.s -.if mixin == 1 - add x13,x13,x14,lsl #32 // pack -.endif -.if mixin == 1 - ldp x12,x14,[x1],#16 -.endif -.if mixin == 1 - add w15,w15,w27 -.endif -.inst 0x04b70042 //add z2.s,z2.s,z23.s -.if mixin == 1 - add x16,x16,x27,lsr #32 -.endif -.inst 0x04b800c6 //add z6.s,z6.s,z24.s -.if mixin == 1 - add x15,x15,x16,lsl #32 // pack -.endif -.if mixin == 1 - add w17,w17,w28 -.endif -.inst 0x04b1014a //add z10.s,z10.s,z17.s -.if mixin == 1 - add x18,x18,x28,lsr #32 -.endif -.inst 0x04b201ce //add z14.s,z14.s,z18.s -.if mixin == 1 - add x17,x17,x18,lsl #32 // pack -.endif -.if mixin == 1 - ldp x16,x18,[x1],#16 -.endif -.inst 0x05a03bd4 //dup z20.s,w30 -.inst 0x05a038d9 //dup z25.s,w6 // bak[15] not available for SVE -.if mixin == 1 - add w19,w19,w29 -.endif -.inst 0x04b00063 //add z3.s,z3.s,z16.s -.if mixin == 1 - add x20,x20,x29,lsr #32 -.endif -.inst 0x04b300e7 //add z7.s,z7.s,z19.s -.if mixin == 1 - add x19,x19,x20,lsl #32 // pack -.endif -.if mixin == 1 - add w21,w21,w30 -.endif -.inst 0x04b4016b //add z11.s,z11.s,z20.s -.if mixin == 1 - add x22,x22,x30,lsr #32 -.endif -.inst 0x04b901ef //add z15.s,z15.s,z25.s -.if mixin == 1 - add x21,x21,x22,lsl #32 // pack -.endif -.if mixin == 1 - ldp x20,x22,[x1],#16 -.endif -#ifdef __AARCH64EB__ - rev x7,x7 - rev x9,x9 - rev x11,x11 - rev x13,x13 - rev x15,x15 - rev x17,x17 - rev x19,x19 - rev x21,x21 -#endif -.if mixin == 1 - add x29,x29,#1 -.endif - cmp x5,4 - b.ne 200f -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.if mixin == 1 - eor x11,x11,x12 -.endif -.inst 0x05a46011 //zip1 z17.s,z0.s,z4.s -.inst 0x05a46412 //zip2 z18.s,z0.s,z4.s -.inst 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.inst 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.inst 0x05a56035 //zip1 z21.s,z1.s,z5.s -.inst 0x05a56436 //zip2 z22.s,z1.s,z5.s -.inst 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.inst 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36624 //zip2 z4.d,z17.d,z19.d -.inst 0x05f46248 //zip1 z8.d,z18.d,z20.d -.inst 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.inst 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.inst 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x13,x13,x14 -.endif -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.inst 0x05a66051 //zip1 z17.s,z2.s,z6.s -.inst 0x05a66452 //zip2 z18.s,z2.s,z6.s -.inst 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.inst 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.inst 0x05a76075 //zip1 z21.s,z3.s,z7.s -.inst 0x05a76476 //zip2 z22.s,z3.s,z7.s -.inst 0x05af6177 //zip1 z23.s,z11.s,z15.s -.inst 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.inst 0x05f36222 //zip1 z2.d,z17.d,z19.d -.inst 0x05f36626 //zip2 z6.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.inst 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.inst 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.inst 0x05f862cb //zip1 z11.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.inst 0x04b13000 //eor z0.d,z0.d,z17.d -.inst 0x04b23021 //eor z1.d,z1.d,z18.d -.inst 0x04b33042 //eor z2.d,z2.d,z19.d -.inst 0x04b43063 //eor z3.d,z3.d,z20.d -.inst 0x04b53084 //eor z4.d,z4.d,z21.d -.inst 0x04b630a5 //eor z5.d,z5.d,z22.d -.inst 0x04b730c6 //eor z6.d,z6.d,z23.d -.inst 0x04b830e7 //eor z7.d,z7.d,z24.d - ld1 {v17.4s,v18.4s,v19.4s,v20.4s},[x1],#64 - ld1 {v21.4s,v22.4s,v23.4s,v24.4s},[x1],#64 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.inst 0x04b13108 //eor z8.d,z8.d,z17.d -.inst 0x04b23129 //eor z9.d,z9.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.inst 0x04b3314a //eor z10.d,z10.d,z19.d -.inst 0x04b4316b //eor z11.d,z11.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.inst 0x04b5318c //eor z12.d,z12.d,z21.d -.inst 0x04b631ad //eor z13.d,z13.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.inst 0x04b731ce //eor z14.d,z14.d,z23.d -.inst 0x04b831ef //eor z15.d,z15.d,z24.d - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x0],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - st1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - st1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - b 210f -200: -.inst 0x05a16011 //zip1 z17.s,z0.s,z1.s -.inst 0x05a16412 //zip2 z18.s,z0.s,z1.s -.inst 0x05a36053 //zip1 z19.s,z2.s,z3.s -.inst 0x05a36454 //zip2 z20.s,z2.s,z3.s - -.inst 0x05a56095 //zip1 z21.s,z4.s,z5.s -.inst 0x05a56496 //zip2 z22.s,z4.s,z5.s -.inst 0x05a760d7 //zip1 z23.s,z6.s,z7.s -.inst 0x05a764d8 //zip2 z24.s,z6.s,z7.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36621 //zip2 z1.d,z17.d,z19.d -.inst 0x05f46242 //zip1 z2.d,z18.d,z20.d -.inst 0x05f46643 //zip2 z3.d,z18.d,z20.d - -.inst 0x05f762a4 //zip1 z4.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c6 //zip1 z6.d,z22.d,z24.d -.inst 0x05f866c7 //zip2 z7.d,z22.d,z24.d -.if mixin == 1 - eor x7,x7,x8 -.endif -.if mixin == 1 - eor x9,x9,x10 -.endif -.inst 0x05a96111 //zip1 z17.s,z8.s,z9.s -.inst 0x05a96512 //zip2 z18.s,z8.s,z9.s -.inst 0x05ab6153 //zip1 z19.s,z10.s,z11.s -.inst 0x05ab6554 //zip2 z20.s,z10.s,z11.s - -.inst 0x05ad6195 //zip1 z21.s,z12.s,z13.s -.inst 0x05ad6596 //zip2 z22.s,z12.s,z13.s -.inst 0x05af61d7 //zip1 z23.s,z14.s,z15.s -.inst 0x05af65d8 //zip2 z24.s,z14.s,z15.s - -.inst 0x05f36228 //zip1 z8.d,z17.d,z19.d -.inst 0x05f36629 //zip2 z9.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664b //zip2 z11.d,z18.d,z20.d - -.inst 0x05f762ac //zip1 z12.d,z21.d,z23.d -.inst 0x05f766ad //zip2 z13.d,z21.d,z23.d -.inst 0x05f862ce //zip1 z14.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x11,x11,x12 -.endif -.if mixin == 1 - eor x13,x13,x14 -.endif -.inst 0x05a46011 //zip1 z17.s,z0.s,z4.s -.inst 0x05a46412 //zip2 z18.s,z0.s,z4.s -.inst 0x05ac6113 //zip1 z19.s,z8.s,z12.s -.inst 0x05ac6514 //zip2 z20.s,z8.s,z12.s - -.inst 0x05a56035 //zip1 z21.s,z1.s,z5.s -.inst 0x05a56436 //zip2 z22.s,z1.s,z5.s -.inst 0x05ad6137 //zip1 z23.s,z9.s,z13.s -.inst 0x05ad6538 //zip2 z24.s,z9.s,z13.s - -.inst 0x05f36220 //zip1 z0.d,z17.d,z19.d -.inst 0x05f36624 //zip2 z4.d,z17.d,z19.d -.inst 0x05f46248 //zip1 z8.d,z18.d,z20.d -.inst 0x05f4664c //zip2 z12.d,z18.d,z20.d - -.inst 0x05f762a1 //zip1 z1.d,z21.d,z23.d -.inst 0x05f766a5 //zip2 z5.d,z21.d,z23.d -.inst 0x05f862c9 //zip1 z9.d,z22.d,z24.d -.inst 0x05f866cd //zip2 z13.d,z22.d,z24.d -.if mixin == 1 - eor x15,x15,x16 -.endif -.if mixin == 1 - eor x17,x17,x18 -.endif -.inst 0x05a66051 //zip1 z17.s,z2.s,z6.s -.inst 0x05a66452 //zip2 z18.s,z2.s,z6.s -.inst 0x05ae6153 //zip1 z19.s,z10.s,z14.s -.inst 0x05ae6554 //zip2 z20.s,z10.s,z14.s - -.inst 0x05a76075 //zip1 z21.s,z3.s,z7.s -.inst 0x05a76476 //zip2 z22.s,z3.s,z7.s -.inst 0x05af6177 //zip1 z23.s,z11.s,z15.s -.inst 0x05af6578 //zip2 z24.s,z11.s,z15.s - -.inst 0x05f36222 //zip1 z2.d,z17.d,z19.d -.inst 0x05f36626 //zip2 z6.d,z17.d,z19.d -.inst 0x05f4624a //zip1 z10.d,z18.d,z20.d -.inst 0x05f4664e //zip2 z14.d,z18.d,z20.d - -.inst 0x05f762a3 //zip1 z3.d,z21.d,z23.d -.inst 0x05f766a7 //zip2 z7.d,z21.d,z23.d -.inst 0x05f862cb //zip1 z11.d,z22.d,z24.d -.inst 0x05f866cf //zip2 z15.d,z22.d,z24.d -.if mixin == 1 - eor x19,x19,x20 -.endif -.if mixin == 1 - eor x21,x21,x22 -.endif -.inst 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.inst 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.inst 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.inst 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.inst 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.inst 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.inst 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.inst 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.inst 0x04215101 //addvl x1,x1,8 -.inst 0x04b13000 //eor z0.d,z0.d,z17.d -.inst 0x04b23084 //eor z4.d,z4.d,z18.d -.inst 0x04b33108 //eor z8.d,z8.d,z19.d -.inst 0x04b4318c //eor z12.d,z12.d,z20.d -.inst 0x04b53021 //eor z1.d,z1.d,z21.d -.inst 0x04b630a5 //eor z5.d,z5.d,z22.d -.inst 0x04b73129 //eor z9.d,z9.d,z23.d -.inst 0x04b831ad //eor z13.d,z13.d,z24.d -.inst 0xa540a031 //ld1w {z17.s},p0/z,[x1,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48231 //revb z17.s,p0/m,z17.s -#endif -.inst 0xa541a032 //ld1w {z18.s},p0/z,[x1,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48252 //revb z18.s,p0/m,z18.s -#endif -.inst 0xa542a033 //ld1w {z19.s},p0/z,[x1,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48273 //revb z19.s,p0/m,z19.s -#endif -.inst 0xa543a034 //ld1w {z20.s},p0/z,[x1,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48294 //revb z20.s,p0/m,z20.s -#endif -.inst 0xa544a035 //ld1w {z21.s},p0/z,[x1,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482b5 //revb z21.s,p0/m,z21.s -#endif -.inst 0xa545a036 //ld1w {z22.s},p0/z,[x1,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482d6 //revb z22.s,p0/m,z22.s -#endif -.inst 0xa546a037 //ld1w {z23.s},p0/z,[x1,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a482f7 //revb z23.s,p0/m,z23.s -#endif -.inst 0xa547a038 //ld1w {z24.s},p0/z,[x1,#7,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48318 //revb z24.s,p0/m,z24.s -#endif -.inst 0x04215101 //addvl x1,x1,8 -.if mixin == 1 - stp x7,x9,[x0],#16 -.endif -.inst 0x04b13042 //eor z2.d,z2.d,z17.d -.inst 0x04b230c6 //eor z6.d,z6.d,z18.d -.if mixin == 1 - stp x11,x13,[x0],#16 -.endif -.inst 0x04b3314a //eor z10.d,z10.d,z19.d -.inst 0x04b431ce //eor z14.d,z14.d,z20.d -.if mixin == 1 - stp x15,x17,[x0],#16 -.endif -.inst 0x04b53063 //eor z3.d,z3.d,z21.d -.inst 0x04b630e7 //eor z7.d,z7.d,z22.d -.if mixin == 1 - stp x19,x21,[x0],#16 -.endif -.inst 0x04b7316b //eor z11.d,z11.d,z23.d -.inst 0x04b831ef //eor z15.d,z15.d,z24.d -#ifdef __AARCH64EB__ -.inst 0x05a48000 //revb z0.s,p0/m,z0.s -#endif -.inst 0xe540e000 //st1w {z0.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48084 //revb z4.s,p0/m,z4.s -#endif -.inst 0xe541e004 //st1w {z4.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48108 //revb z8.s,p0/m,z8.s -#endif -.inst 0xe542e008 //st1w {z8.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4818c //revb z12.s,p0/m,z12.s -#endif -.inst 0xe543e00c //st1w {z12.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48021 //revb z1.s,p0/m,z1.s -#endif -.inst 0xe544e001 //st1w {z1.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480a5 //revb z5.s,p0/m,z5.s -#endif -.inst 0xe545e005 //st1w {z5.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48129 //revb z9.s,p0/m,z9.s -#endif -.inst 0xe546e009 //st1w {z9.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ad //revb z13.s,p0/m,z13.s -#endif -.inst 0xe547e00d //st1w {z13.s},p0,[x0,#7,MUL VL] -.inst 0x04205100 //addvl x0,x0,8 -#ifdef __AARCH64EB__ -.inst 0x05a48042 //revb z2.s,p0/m,z2.s -#endif -.inst 0xe540e002 //st1w {z2.s},p0,[x0,#0,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480c6 //revb z6.s,p0/m,z6.s -#endif -.inst 0xe541e006 //st1w {z6.s},p0,[x0,#1,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4814a //revb z10.s,p0/m,z10.s -#endif -.inst 0xe542e00a //st1w {z10.s},p0,[x0,#2,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ce //revb z14.s,p0/m,z14.s -#endif -.inst 0xe543e00e //st1w {z14.s},p0,[x0,#3,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a48063 //revb z3.s,p0/m,z3.s -#endif -.inst 0xe544e003 //st1w {z3.s},p0,[x0,#4,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a480e7 //revb z7.s,p0/m,z7.s -#endif -.inst 0xe545e007 //st1w {z7.s},p0,[x0,#5,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a4816b //revb z11.s,p0/m,z11.s -#endif -.inst 0xe546e00b //st1w {z11.s},p0,[x0,#6,MUL VL] -#ifdef __AARCH64EB__ -.inst 0x05a481ef //revb z15.s,p0/m,z15.s -#endif -.inst 0xe547e00f //st1w {z15.s},p0,[x0,#7,MUL VL] -.inst 0x04205100 //addvl x0,x0,8 -210: -.inst 0x04b0e3fd //incw x29, ALL, MUL #1 -110: -2: - str w29,[x4] - ldp d10,d11,[sp,16] - ldp d12,d13,[sp,32] - ldp d14,d15,[sp,48] - ldp x16,x17,[sp,64] - ldp x18,x19,[sp,80] - ldp x20,x21,[sp,96] - ldp x22,x23,[sp,112] - ldp x24,x25,[sp,128] - ldp x26,x27,[sp,144] - ldp x28,x29,[sp,160] - ldr x30,[sp,176] - ldp d8,d9,[sp],192 - AARCH64_VALIDATE_LINK_REGISTER -.Lreturn: - ret -.size ChaCha20_ctr32_sve,.-ChaCha20_ctr32_sve diff --git a/openssl/src/crypto/chacha/gen/linux_arm64/chacha-armv8.S b/openssl/src/crypto/chacha/gen/linux_arm64/chacha-armv8.S index 02991495b..66eb96ec5 100644 --- a/openssl/src/crypto/chacha/gen/linux_arm64/chacha-armv8.S +++ b/openssl/src/crypto/chacha/gen/linux_arm64/chacha-armv8.S @@ -1,9 +1,7 @@ -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .hidden OPENSSL_armcap_P - - #endif .text @@ -18,22 +16,23 @@ .byte 67,104,97,67,104,97,50,48,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 .align 2 -.globl ChaCha20_ctr32_dflt -.type ChaCha20_ctr32_dflt,%function +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function .align 5 -ChaCha20_ctr32_dflt: - AARCH64_SIGN_LINK_REGISTER +ChaCha20_ctr32: + cbz x2,.Labort cmp x2,#192 b.lo .Lshort + #ifndef __KERNEL__ adrp x17,OPENSSL_armcap_P ldr w17,[x17,#:lo12:OPENSSL_armcap_P] -.Lcheck_neon: tst w17,#ARMV7_NEON b.ne .LChaCha20_neon #endif .Lshort: +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -245,8 +244,8 @@ ChaCha20_ctr32_dflt: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 +.inst 0xd50323bf // autiasp .Labort: - AARCH64_VALIDATE_LINK_REGISTER ret .align 4 @@ -302,43 +301,8 @@ ChaCha20_ctr32_dflt: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ChaCha20_ctr32_dflt,.-ChaCha20_ctr32_dflt - -.globl ChaCha20_ctr32 -.type ChaCha20_ctr32,%function -.align 5 -ChaCha20_ctr32: - AARCH64_SIGN_LINK_REGISTER - cbz x2,.Labort - cmp x2,#192 - b.lo .Lshort -#ifndef __KERNEL__ - adrp x17,OPENSSL_armcap_P - ldr w17,[x17,#:lo12:OPENSSL_armcap_P] - tst w17,#ARMV8_SVE - b.eq .Lcheck_neon - stp x29,x30,[sp,#-16]! - sub sp,sp,#16 - // SVE handling will inevitably increment the counter - // Neon/Scalar code that follows to process tail data needs to - // use new counter, unfortunately the input counter buffer - // pointed to by ctr is meant to be read-only per API contract - // we have to copy the buffer to stack to be writable by SVE - ldp x5,x6,[x4] - stp x5,x6,[sp] - mov x4,sp - bl ChaCha20_ctr32_sve - cbz x2,1f - bl ChaCha20_ctr32_dflt -1: - add sp,sp,#16 - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret -#endif - b .Lshort .size ChaCha20_ctr32,.-ChaCha20_ctr32 #ifdef __KERNEL__ @@ -347,8 +311,8 @@ ChaCha20_ctr32: .type ChaCha20_neon,%function .align 5 ChaCha20_neon: - AARCH64_SIGN_LINK_REGISTER .LChaCha20_neon: +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -783,7 +747,7 @@ ChaCha20_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .align 4 @@ -915,13 +879,13 @@ ChaCha20_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ChaCha20_neon,.-ChaCha20_neon .type ChaCha20_512_neon,%function .align 5 ChaCha20_512_neon: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -2065,6 +2029,6 @@ ChaCha20_512_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ChaCha20_512_neon,.-ChaCha20_512_neon diff --git a/openssl/src/crypto/chacha/gen/linux_ia32/chacha-x86.S b/openssl/src/crypto/chacha/gen/linux_ia32/chacha-x86.S index ec98cb32f..a0d95910f 100644 --- a/openssl/src/crypto/chacha/gen/linux_ia32/chacha-x86.S +++ b/openssl/src/crypto/chacha/gen/linux_ia32/chacha-x86.S @@ -4,11 +4,7 @@ .align 16 ChaCha20_ctr32: .L_ChaCha20_ctr32_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -382,11 +378,7 @@ ChaCha20_ctr32: .align 16 ChaCha20_ssse3: .L_ChaCha20_ssse3_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -979,11 +971,7 @@ ChaCha20_ssse3: .align 16 ChaCha20_xop: .L_ChaCha20_xop_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/chacha/gen/linux_loong64/chacha-loongarch64.S b/openssl/src/crypto/chacha/gen/linux_loong64/chacha-loongarch64.S deleted file mode 100644 index 196fb6131..000000000 --- a/openssl/src/crypto/chacha/gen/linux_loong64/chacha-loongarch64.S +++ /dev/null @@ -1,1446 +0,0 @@ -#include "loongarch_arch.h" - -.text - -.extern OPENSSL_loongarch_hwcap_P - -.align 6 -.Lsigma: -.ascii "expand 32-byte k" -.Linc8x: -.long 0,1,2,3,4,5,6,7 -.Linc4x: -.long 0,1,2,3 - -.globl ChaCha20_ctr32 -.type ChaCha20_ctr32 function - -.align 6 -ChaCha20_ctr32: - # $r4 = arg #1 (out pointer) - # $r5 = arg #2 (inp pointer) - # $r6 = arg #3 (len) - # $r7 = arg #4 (key array) - # $r8 = arg #5 (counter array) - - beqz $r6,.Lno_data - ori $r15,$r0,64 - la.pcrel $r12,OPENSSL_loongarch_hwcap_P - ld.w $r12,$r12,0 - - bleu $r6,$r15,.LChaCha20_1x # goto 1x when len <= 64 - - andi $r12,$r12,LOONGARCH_HWCAP_LASX | LOONGARCH_HWCAP_LSX - beqz $r12,.LChaCha20_1x - - addi.d $r3,$r3,-64 - fst.d $f24,$r3,0 - fst.d $f25,$r3,8 - fst.d $f26,$r3,16 - fst.d $f27,$r3,24 - fst.d $f28,$r3,32 - fst.d $f29,$r3,40 - fst.d $f30,$r3,48 - fst.d $f31,$r3,56 - - andi $r13,$r12,LOONGARCH_HWCAP_LASX - bnez $r13,.LChaCha20_8x - - b .LChaCha20_4x - -.align 6 -.LChaCha20_1x: - addi.d $r3,$r3,-256 - st.d $r23,$r3,0 - st.d $r24,$r3,8 - st.d $r25,$r3,16 - st.d $r26,$r3,24 - st.d $r27,$r3,32 - st.d $r28,$r3,40 - st.d $r29,$r3,48 - st.d $r30,$r3,56 - st.d $r31,$r3,64 - - # Save the initial block counter in $r31 - ld.w $r31,$r8,0 - b .Loop_outer_1x - -.align 5 -.Loop_outer_1x: - # Load constants - la.local $r20,.Lsigma - ld.w $r12,$r20,0 # 'expa' - ld.w $r13,$r20,4 # 'nd 3' - ld.w $r14,$r20,8 # '2-by' - ld.w $r15,$r20,12 # 'te k' - - # Load key - ld.w $r16,$r7,4*0 - ld.w $r17,$r7,4*1 - ld.w $r18,$r7,4*2 - ld.w $r19,$r7,4*3 - ld.w $r23,$r7,4*4 - ld.w $r24,$r7,4*5 - ld.w $r25,$r7,4*6 - ld.w $r26,$r7,4*7 - - # Load block counter - move $r27,$r31 - - # Load nonce - ld.w $r28,$r8,4*1 - ld.w $r29,$r8,4*2 - ld.w $r30,$r8,4*3 - - # Update states in @x[*] for 20 rounds - ori $r20,$r0,10 - b .Loop_1x - -.align 5 -.Loop_1x: - add.w $r12,$r12,$r16 - xor $r27,$r27,$r12 - rotri.w $r27,$r27,16 # rotate left 16 bits - add.w $r13,$r13,$r17 - xor $r28,$r28,$r13 - rotri.w $r28,$r28,16 - - add.w $r23,$r23,$r27 - xor $r16,$r16,$r23 - rotri.w $r16,$r16,20 # rotate left 12 bits - add.w $r24,$r24,$r28 - xor $r17,$r17,$r24 - rotri.w $r17,$r17,20 - - add.w $r12,$r12,$r16 - xor $r27,$r27,$r12 - rotri.w $r27,$r27,24 # rotate left 8 bits - add.w $r13,$r13,$r17 - xor $r28,$r28,$r13 - rotri.w $r28,$r28,24 - - add.w $r23,$r23,$r27 - xor $r16,$r16,$r23 - rotri.w $r16,$r16,25 # rotate left 7 bits - add.w $r24,$r24,$r28 - xor $r17,$r17,$r24 - rotri.w $r17,$r17,25 - - add.w $r14,$r14,$r18 - xor $r29,$r29,$r14 - rotri.w $r29,$r29,16 - add.w $r15,$r15,$r19 - xor $r30,$r30,$r15 - rotri.w $r30,$r30,16 - - add.w $r25,$r25,$r29 - xor $r18,$r18,$r25 - rotri.w $r18,$r18,20 - add.w $r26,$r26,$r30 - xor $r19,$r19,$r26 - rotri.w $r19,$r19,20 - - add.w $r14,$r14,$r18 - xor $r29,$r29,$r14 - rotri.w $r29,$r29,24 - add.w $r15,$r15,$r19 - xor $r30,$r30,$r15 - rotri.w $r30,$r30,24 - - add.w $r25,$r25,$r29 - xor $r18,$r18,$r25 - rotri.w $r18,$r18,25 - add.w $r26,$r26,$r30 - xor $r19,$r19,$r26 - rotri.w $r19,$r19,25 - - add.w $r12,$r12,$r17 - xor $r30,$r30,$r12 - rotri.w $r30,$r30,16 # rotate left 16 bits - add.w $r13,$r13,$r18 - xor $r27,$r27,$r13 - rotri.w $r27,$r27,16 - - add.w $r25,$r25,$r30 - xor $r17,$r17,$r25 - rotri.w $r17,$r17,20 # rotate left 12 bits - add.w $r26,$r26,$r27 - xor $r18,$r18,$r26 - rotri.w $r18,$r18,20 - - add.w $r12,$r12,$r17 - xor $r30,$r30,$r12 - rotri.w $r30,$r30,24 # rotate left 8 bits - add.w $r13,$r13,$r18 - xor $r27,$r27,$r13 - rotri.w $r27,$r27,24 - - add.w $r25,$r25,$r30 - xor $r17,$r17,$r25 - rotri.w $r17,$r17,25 # rotate left 7 bits - add.w $r26,$r26,$r27 - xor $r18,$r18,$r26 - rotri.w $r18,$r18,25 - - add.w $r14,$r14,$r19 - xor $r28,$r28,$r14 - rotri.w $r28,$r28,16 - add.w $r15,$r15,$r16 - xor $r29,$r29,$r15 - rotri.w $r29,$r29,16 - - add.w $r23,$r23,$r28 - xor $r19,$r19,$r23 - rotri.w $r19,$r19,20 - add.w $r24,$r24,$r29 - xor $r16,$r16,$r24 - rotri.w $r16,$r16,20 - - add.w $r14,$r14,$r19 - xor $r28,$r28,$r14 - rotri.w $r28,$r28,24 - add.w $r15,$r15,$r16 - xor $r29,$r29,$r15 - rotri.w $r29,$r29,24 - - add.w $r23,$r23,$r28 - xor $r19,$r19,$r23 - rotri.w $r19,$r19,25 - add.w $r24,$r24,$r29 - xor $r16,$r16,$r24 - rotri.w $r16,$r16,25 - - addi.w $r20,$r20,-1 - bnez $r20,.Loop_1x - - # Get the final states by adding the initial states - la.local $r20,.Lsigma - ld.w $r11,$r20,4*0 - ld.w $r10,$r20,4*1 - ld.w $r9,$r20,4*2 - add.w $r12,$r12,$r11 - add.w $r13,$r13,$r10 - add.w $r14,$r14,$r9 - ld.w $r11,$r20,4*3 - add.w $r15,$r15,$r11 - - ld.w $r20,$r7,4*0 - ld.w $r11,$r7,4*1 - ld.w $r10,$r7,4*2 - ld.w $r9,$r7,4*3 - add.w $r16,$r16,$r20 - add.w $r17,$r17,$r11 - add.w $r18,$r18,$r10 - add.w $r19,$r19,$r9 - - ld.w $r20,$r7,4*4 - ld.w $r11,$r7,4*5 - ld.w $r10,$r7,4*6 - ld.w $r9,$r7,4*7 - add.w $r23,$r23,$r20 - add.w $r24,$r24,$r11 - add.w $r25,$r25,$r10 - add.w $r26,$r26,$r9 - - add.w $r27,$r27,$r31 - - ld.w $r20,$r8,4*1 - ld.w $r11,$r8,4*2 - ld.w $r10,$r8,4*3 - add.w $r28,$r28,$r20 - add.w $r29,$r29,$r11 - add.w $r30,$r30,$r10 - - ori $r20,$r0,64 - bltu $r6,$r20,.Ltail_1x - - # Get the encrypted message by xor states with plaintext - ld.w $r20,$r5,4*0 - ld.w $r11,$r5,4*1 - ld.w $r10,$r5,4*2 - ld.w $r9,$r5,4*3 - xor $r20,$r20,$r12 - xor $r11,$r11,$r13 - xor $r10,$r10,$r14 - xor $r9,$r9,$r15 - st.w $r20,$r4,4*0 - st.w $r11,$r4,4*1 - st.w $r10,$r4,4*2 - st.w $r9,$r4,4*3 - - ld.w $r20,$r5,4*4 - ld.w $r11,$r5,4*5 - ld.w $r10,$r5,4*6 - ld.w $r9,$r5,4*7 - xor $r20,$r20,$r16 - xor $r11,$r11,$r17 - xor $r10,$r10,$r18 - xor $r9,$r9,$r19 - st.w $r20,$r4,4*4 - st.w $r11,$r4,4*5 - st.w $r10,$r4,4*6 - st.w $r9,$r4,4*7 - - ld.w $r20,$r5,4*8 - ld.w $r11,$r5,4*9 - ld.w $r10,$r5,4*10 - ld.w $r9,$r5,4*11 - xor $r20,$r20,$r23 - xor $r11,$r11,$r24 - xor $r10,$r10,$r25 - xor $r9,$r9,$r26 - st.w $r20,$r4,4*8 - st.w $r11,$r4,4*9 - st.w $r10,$r4,4*10 - st.w $r9,$r4,4*11 - - ld.w $r20,$r5,4*12 - ld.w $r11,$r5,4*13 - ld.w $r10,$r5,4*14 - ld.w $r9,$r5,4*15 - xor $r20,$r20,$r27 - xor $r11,$r11,$r28 - xor $r10,$r10,$r29 - xor $r9,$r9,$r30 - st.w $r20,$r4,4*12 - st.w $r11,$r4,4*13 - st.w $r10,$r4,4*14 - st.w $r9,$r4,4*15 - - addi.d $r6,$r6,-64 - beqz $r6,.Ldone_1x - addi.d $r5,$r5,64 - addi.d $r4,$r4,64 - addi.w $r31,$r31,1 - b .Loop_outer_1x - -.align 4 -.Ltail_1x: - # Handle the tail for 1x (1 <= tail_len <= 63) - addi.d $r11,$r3,72 - st.w $r12,$r11,4*0 - st.w $r13,$r11,4*1 - st.w $r14,$r11,4*2 - st.w $r15,$r11,4*3 - st.w $r16,$r11,4*4 - st.w $r17,$r11,4*5 - st.w $r18,$r11,4*6 - st.w $r19,$r11,4*7 - st.w $r23,$r11,4*8 - st.w $r24,$r11,4*9 - st.w $r25,$r11,4*10 - st.w $r26,$r11,4*11 - st.w $r27,$r11,4*12 - st.w $r28,$r11,4*13 - st.w $r29,$r11,4*14 - st.w $r30,$r11,4*15 - - move $r20,$r0 - -.Loop_tail_1x: - # Xor input with states byte by byte - ldx.bu $r10,$r5,$r20 - ldx.bu $r9,$r11,$r20 - xor $r10,$r10,$r9 - stx.b $r10,$r4,$r20 - addi.w $r20,$r20,1 - addi.d $r6,$r6,-1 - bnez $r6,.Loop_tail_1x - b .Ldone_1x - -.Ldone_1x: - ld.d $r23,$r3,0 - ld.d $r24,$r3,8 - ld.d $r25,$r3,16 - ld.d $r26,$r3,24 - ld.d $r27,$r3,32 - ld.d $r28,$r3,40 - ld.d $r29,$r3,48 - ld.d $r30,$r3,56 - ld.d $r31,$r3,64 - addi.d $r3,$r3,256 - - b .Lend - -.align 6 -.LChaCha20_4x: - addi.d $r3,$r3,-128 - - # Save the initial block counter in $r16 - ld.w $r16,$r8,0 - b .Loop_outer_4x - -.align 5 -.Loop_outer_4x: - # Load constant - la.local $r20,.Lsigma - vldrepl.w $vr0,$r20,4*0 # 'expa' - vldrepl.w $vr1,$r20,4*1 # 'nd 3' - vldrepl.w $vr2,$r20,4*2 # '2-by' - vldrepl.w $vr3,$r20,4*3 # 'te k' - - # Load key - vldrepl.w $vr4,$r7,4*0 - vldrepl.w $vr5,$r7,4*1 - vldrepl.w $vr6,$r7,4*2 - vldrepl.w $vr7,$r7,4*3 - vldrepl.w $vr8,$r7,4*4 - vldrepl.w $vr9,$r7,4*5 - vldrepl.w $vr10,$r7,4*6 - vldrepl.w $vr11,$r7,4*7 - - # Load block counter - vreplgr2vr.w $vr12,$r16 - - # Load nonce - vldrepl.w $vr13,$r8,4*1 - vldrepl.w $vr14,$r8,4*2 - vldrepl.w $vr15,$r8,4*3 - - # Get the correct block counter for each block - la.local $r20,.Linc4x - vld $vr16,$r20,0 - vadd.w $vr12,$vr12,$vr16 - - # Copy the initial states from @x[*] to @y[*] - vori.b $vr16,$vr0,0 - vori.b $vr17,$vr1,0 - vori.b $vr18,$vr2,0 - vori.b $vr19,$vr3,0 - vori.b $vr20,$vr4,0 - vori.b $vr21,$vr5,0 - vori.b $vr22,$vr6,0 - vori.b $vr23,$vr7,0 - vori.b $vr24,$vr8,0 - vori.b $vr25,$vr9,0 - vori.b $vr26,$vr10,0 - vori.b $vr27,$vr11,0 - vori.b $vr28,$vr12,0 - vori.b $vr29,$vr13,0 - vori.b $vr30,$vr14,0 - vori.b $vr31,$vr15,0 - - # Update states in @x[*] for 20 rounds - ori $r20,$r0,10 - b .Loop_4x - -.align 5 -.Loop_4x: - vadd.w $vr0,$vr0,$vr4 - vxor.v $vr12,$vr12,$vr0 - vrotri.w $vr12,$vr12,16 # rotate left 16 bits - vadd.w $vr1,$vr1,$vr5 - vxor.v $vr13,$vr13,$vr1 - vrotri.w $vr13,$vr13,16 - - vadd.w $vr8,$vr8,$vr12 - vxor.v $vr4,$vr4,$vr8 - vrotri.w $vr4,$vr4,20 # rotate left 12 bits - vadd.w $vr9,$vr9,$vr13 - vxor.v $vr5,$vr5,$vr9 - vrotri.w $vr5,$vr5,20 - - vadd.w $vr0,$vr0,$vr4 - vxor.v $vr12,$vr12,$vr0 - vrotri.w $vr12,$vr12,24 # rotate left 8 bits - vadd.w $vr1,$vr1,$vr5 - vxor.v $vr13,$vr13,$vr1 - vrotri.w $vr13,$vr13,24 - - vadd.w $vr8,$vr8,$vr12 - vxor.v $vr4,$vr4,$vr8 - vrotri.w $vr4,$vr4,25 # rotate left 7 bits - vadd.w $vr9,$vr9,$vr13 - vxor.v $vr5,$vr5,$vr9 - vrotri.w $vr5,$vr5,25 - - vadd.w $vr2,$vr2,$vr6 - vxor.v $vr14,$vr14,$vr2 - vrotri.w $vr14,$vr14,16 - vadd.w $vr3,$vr3,$vr7 - vxor.v $vr15,$vr15,$vr3 - vrotri.w $vr15,$vr15,16 - - vadd.w $vr10,$vr10,$vr14 - vxor.v $vr6,$vr6,$vr10 - vrotri.w $vr6,$vr6,20 - vadd.w $vr11,$vr11,$vr15 - vxor.v $vr7,$vr7,$vr11 - vrotri.w $vr7,$vr7,20 - - vadd.w $vr2,$vr2,$vr6 - vxor.v $vr14,$vr14,$vr2 - vrotri.w $vr14,$vr14,24 - vadd.w $vr3,$vr3,$vr7 - vxor.v $vr15,$vr15,$vr3 - vrotri.w $vr15,$vr15,24 - - vadd.w $vr10,$vr10,$vr14 - vxor.v $vr6,$vr6,$vr10 - vrotri.w $vr6,$vr6,25 - vadd.w $vr11,$vr11,$vr15 - vxor.v $vr7,$vr7,$vr11 - vrotri.w $vr7,$vr7,25 - - vadd.w $vr0,$vr0,$vr5 - vxor.v $vr15,$vr15,$vr0 - vrotri.w $vr15,$vr15,16 # rotate left 16 bits - vadd.w $vr1,$vr1,$vr6 - vxor.v $vr12,$vr12,$vr1 - vrotri.w $vr12,$vr12,16 - - vadd.w $vr10,$vr10,$vr15 - vxor.v $vr5,$vr5,$vr10 - vrotri.w $vr5,$vr5,20 # rotate left 12 bits - vadd.w $vr11,$vr11,$vr12 - vxor.v $vr6,$vr6,$vr11 - vrotri.w $vr6,$vr6,20 - - vadd.w $vr0,$vr0,$vr5 - vxor.v $vr15,$vr15,$vr0 - vrotri.w $vr15,$vr15,24 # rotate left 8 bits - vadd.w $vr1,$vr1,$vr6 - vxor.v $vr12,$vr12,$vr1 - vrotri.w $vr12,$vr12,24 - - vadd.w $vr10,$vr10,$vr15 - vxor.v $vr5,$vr5,$vr10 - vrotri.w $vr5,$vr5,25 # rotate left 7 bits - vadd.w $vr11,$vr11,$vr12 - vxor.v $vr6,$vr6,$vr11 - vrotri.w $vr6,$vr6,25 - - vadd.w $vr2,$vr2,$vr7 - vxor.v $vr13,$vr13,$vr2 - vrotri.w $vr13,$vr13,16 - vadd.w $vr3,$vr3,$vr4 - vxor.v $vr14,$vr14,$vr3 - vrotri.w $vr14,$vr14,16 - - vadd.w $vr8,$vr8,$vr13 - vxor.v $vr7,$vr7,$vr8 - vrotri.w $vr7,$vr7,20 - vadd.w $vr9,$vr9,$vr14 - vxor.v $vr4,$vr4,$vr9 - vrotri.w $vr4,$vr4,20 - - vadd.w $vr2,$vr2,$vr7 - vxor.v $vr13,$vr13,$vr2 - vrotri.w $vr13,$vr13,24 - vadd.w $vr3,$vr3,$vr4 - vxor.v $vr14,$vr14,$vr3 - vrotri.w $vr14,$vr14,24 - - vadd.w $vr8,$vr8,$vr13 - vxor.v $vr7,$vr7,$vr8 - vrotri.w $vr7,$vr7,25 - vadd.w $vr9,$vr9,$vr14 - vxor.v $vr4,$vr4,$vr9 - vrotri.w $vr4,$vr4,25 - - addi.w $r20,$r20,-1 - bnez $r20,.Loop_4x - - # Get the final states by adding the initial states - vadd.w $vr0,$vr0,$vr16 - vadd.w $vr1,$vr1,$vr17 - vadd.w $vr2,$vr2,$vr18 - vadd.w $vr3,$vr3,$vr19 - vadd.w $vr4,$vr4,$vr20 - vadd.w $vr5,$vr5,$vr21 - vadd.w $vr6,$vr6,$vr22 - vadd.w $vr7,$vr7,$vr23 - vadd.w $vr8,$vr8,$vr24 - vadd.w $vr9,$vr9,$vr25 - vadd.w $vr10,$vr10,$vr26 - vadd.w $vr11,$vr11,$vr27 - vadd.w $vr12,$vr12,$vr28 - vadd.w $vr13,$vr13,$vr29 - vadd.w $vr14,$vr14,$vr30 - vadd.w $vr15,$vr15,$vr31 - - # Get the transpose of @x[*] and save them in @x[*] - vilvl.w $vr16,$vr1,$vr0 - vilvh.w $vr17,$vr1,$vr0 - vilvl.w $vr18,$vr3,$vr2 - vilvh.w $vr19,$vr3,$vr2 - vilvl.w $vr20,$vr5,$vr4 - vilvh.w $vr21,$vr5,$vr4 - vilvl.w $vr22,$vr7,$vr6 - vilvh.w $vr23,$vr7,$vr6 - vilvl.w $vr24,$vr9,$vr8 - vilvh.w $vr25,$vr9,$vr8 - vilvl.w $vr26,$vr11,$vr10 - vilvh.w $vr27,$vr11,$vr10 - vilvl.w $vr28,$vr13,$vr12 - vilvh.w $vr29,$vr13,$vr12 - vilvl.w $vr30,$vr15,$vr14 - vilvh.w $vr31,$vr15,$vr14 - - vilvl.d $vr0,$vr18,$vr16 - vilvh.d $vr1,$vr18,$vr16 - vilvl.d $vr2,$vr19,$vr17 - vilvh.d $vr3,$vr19,$vr17 - vilvl.d $vr4,$vr22,$vr20 - vilvh.d $vr5,$vr22,$vr20 - vilvl.d $vr6,$vr23,$vr21 - vilvh.d $vr7,$vr23,$vr21 - vilvl.d $vr8,$vr26,$vr24 - vilvh.d $vr9,$vr26,$vr24 - vilvl.d $vr10,$vr27,$vr25 - vilvh.d $vr11,$vr27,$vr25 - vilvl.d $vr12,$vr30,$vr28 - vilvh.d $vr13,$vr30,$vr28 - vilvl.d $vr14,$vr31,$vr29 - vilvh.d $vr15,$vr31,$vr29 - ori $r20,$r0,64*4 - bltu $r6,$r20,.Ltail_4x - - # Get the encrypted message by xor states with plaintext - vld $vr16,$r5,16*0 - vld $vr17,$r5,16*1 - vld $vr18,$r5,16*2 - vld $vr19,$r5,16*3 - vxor.v $vr16,$vr16,$vr0 - vxor.v $vr17,$vr17,$vr4 - vxor.v $vr18,$vr18,$vr8 - vxor.v $vr19,$vr19,$vr12 - vst $vr16,$r4,16*0 - vst $vr17,$r4,16*1 - vst $vr18,$r4,16*2 - vst $vr19,$r4,16*3 - - vld $vr16,$r5,16*4 - vld $vr17,$r5,16*5 - vld $vr18,$r5,16*6 - vld $vr19,$r5,16*7 - vxor.v $vr16,$vr16,$vr1 - vxor.v $vr17,$vr17,$vr5 - vxor.v $vr18,$vr18,$vr9 - vxor.v $vr19,$vr19,$vr13 - vst $vr16,$r4,16*4 - vst $vr17,$r4,16*5 - vst $vr18,$r4,16*6 - vst $vr19,$r4,16*7 - - vld $vr16,$r5,16*8 - vld $vr17,$r5,16*9 - vld $vr18,$r5,16*10 - vld $vr19,$r5,16*11 - vxor.v $vr16,$vr16,$vr2 - vxor.v $vr17,$vr17,$vr6 - vxor.v $vr18,$vr18,$vr10 - vxor.v $vr19,$vr19,$vr14 - vst $vr16,$r4,16*8 - vst $vr17,$r4,16*9 - vst $vr18,$r4,16*10 - vst $vr19,$r4,16*11 - - vld $vr16,$r5,16*12 - vld $vr17,$r5,16*13 - vld $vr18,$r5,16*14 - vld $vr19,$r5,16*15 - vxor.v $vr16,$vr16,$vr3 - vxor.v $vr17,$vr17,$vr7 - vxor.v $vr18,$vr18,$vr11 - vxor.v $vr19,$vr19,$vr15 - vst $vr16,$r4,16*12 - vst $vr17,$r4,16*13 - vst $vr18,$r4,16*14 - vst $vr19,$r4,16*15 - - addi.d $r6,$r6,-64*4 - beqz $r6,.Ldone_4x - addi.d $r5,$r5,64*4 - addi.d $r4,$r4,64*4 - addi.w $r16,$r16,4 - b .Loop_outer_4x - -.Ltail_4x: - # Handle the tail for 4x (1 <= tail_len <= 255) - ori $r20,$r0,192 - bgeu $r6,$r20,.L192_or_more4x - ori $r20,$r0,128 - bgeu $r6,$r20,.L128_or_more4x - ori $r20,$r0,64 - bgeu $r6,$r20,.L64_or_more4x - - vst $vr0,$r3,16*0 - vst $vr4,$r3,16*1 - vst $vr8,$r3,16*2 - vst $vr12,$r3,16*3 - move $r20,$r0 - b .Loop_tail_4x - -.align 5 -.L64_or_more4x: - vld $vr16,$r5,16*0 - vld $vr17,$r5,16*1 - vld $vr18,$r5,16*2 - vld $vr19,$r5,16*3 - vxor.v $vr16,$vr16,$vr0 - vxor.v $vr17,$vr17,$vr4 - vxor.v $vr18,$vr18,$vr8 - vxor.v $vr19,$vr19,$vr12 - vst $vr16,$r4,16*0 - vst $vr17,$r4,16*1 - vst $vr18,$r4,16*2 - vst $vr19,$r4,16*3 - - addi.d $r6,$r6,-64 - beqz $r6,.Ldone_4x - addi.d $r5,$r5,64 - addi.d $r4,$r4,64 - vst $vr1,$r3,16*0 - vst $vr5,$r3,16*1 - vst $vr9,$r3,16*2 - vst $vr13,$r3,16*3 - move $r20,$r0 - b .Loop_tail_4x - -.align 5 -.L128_or_more4x: - vld $vr16,$r5,16*0 - vld $vr17,$r5,16*1 - vld $vr18,$r5,16*2 - vld $vr19,$r5,16*3 - vxor.v $vr16,$vr16,$vr0 - vxor.v $vr17,$vr17,$vr4 - vxor.v $vr18,$vr18,$vr8 - vxor.v $vr19,$vr19,$vr12 - vst $vr16,$r4,16*0 - vst $vr17,$r4,16*1 - vst $vr18,$r4,16*2 - vst $vr19,$r4,16*3 - - vld $vr16,$r5,16*4 - vld $vr17,$r5,16*5 - vld $vr18,$r5,16*6 - vld $vr19,$r5,16*7 - vxor.v $vr16,$vr16,$vr1 - vxor.v $vr17,$vr17,$vr5 - vxor.v $vr18,$vr18,$vr9 - vxor.v $vr19,$vr19,$vr13 - vst $vr16,$r4,16*4 - vst $vr17,$r4,16*5 - vst $vr18,$r4,16*6 - vst $vr19,$r4,16*7 - - addi.d $r6,$r6,-128 - beqz $r6,.Ldone_4x - addi.d $r5,$r5,128 - addi.d $r4,$r4,128 - vst $vr2,$r3,16*0 - vst $vr6,$r3,16*1 - vst $vr10,$r3,16*2 - vst $vr14,$r3,16*3 - move $r20,$r0 - b .Loop_tail_4x - -.align 5 -.L192_or_more4x: - vld $vr16,$r5,16*0 - vld $vr17,$r5,16*1 - vld $vr18,$r5,16*2 - vld $vr19,$r5,16*3 - vxor.v $vr16,$vr16,$vr0 - vxor.v $vr17,$vr17,$vr4 - vxor.v $vr18,$vr18,$vr8 - vxor.v $vr19,$vr19,$vr12 - vst $vr16,$r4,16*0 - vst $vr17,$r4,16*1 - vst $vr18,$r4,16*2 - vst $vr19,$r4,16*3 - - vld $vr16,$r5,16*4 - vld $vr17,$r5,16*5 - vld $vr18,$r5,16*6 - vld $vr19,$r5,16*7 - vxor.v $vr16,$vr16,$vr1 - vxor.v $vr17,$vr17,$vr5 - vxor.v $vr18,$vr18,$vr9 - vxor.v $vr19,$vr19,$vr13 - vst $vr16,$r4,16*4 - vst $vr17,$r4,16*5 - vst $vr18,$r4,16*6 - vst $vr19,$r4,16*7 - - vld $vr16,$r5,16*8 - vld $vr17,$r5,16*9 - vld $vr18,$r5,16*10 - vld $vr19,$r5,16*11 - vxor.v $vr16,$vr16,$vr2 - vxor.v $vr17,$vr17,$vr6 - vxor.v $vr18,$vr18,$vr10 - vxor.v $vr19,$vr19,$vr14 - vst $vr16,$r4,16*8 - vst $vr17,$r4,16*9 - vst $vr18,$r4,16*10 - vst $vr19,$r4,16*11 - - addi.d $r6,$r6,-192 - beqz $r6,.Ldone_4x - addi.d $r5,$r5,192 - addi.d $r4,$r4,192 - vst $vr3,$r3,16*0 - vst $vr7,$r3,16*1 - vst $vr11,$r3,16*2 - vst $vr15,$r3,16*3 - move $r20,$r0 - b .Loop_tail_4x - -.Loop_tail_4x: - # Xor input with states byte by byte - ldx.bu $r17,$r5,$r20 - ldx.bu $r18,$r3,$r20 - xor $r17,$r17,$r18 - stx.b $r17,$r4,$r20 - addi.w $r20,$r20,1 - addi.d $r6,$r6,-1 - bnez $r6,.Loop_tail_4x - b .Ldone_4x - -.Ldone_4x: - addi.d $r3,$r3,128 - b .Lrestore_saved_fpr - -.align 6 -.LChaCha20_8x: - addi.d $r3,$r3,-128 - - # Save the initial block counter in $r16 - ld.w $r16,$r8,0 - b .Loop_outer_8x - -.align 5 -.Loop_outer_8x: - # Load constant - la.local $r20,.Lsigma - xvldrepl.w $xr0,$r20,4*0 # 'expa' - xvldrepl.w $xr1,$r20,4*1 # 'nd 3' - xvldrepl.w $xr2,$r20,4*2 # '2-by' - xvldrepl.w $xr3,$r20,4*3 # 'te k' - - # Load key - xvldrepl.w $xr4,$r7,4*0 - xvldrepl.w $xr5,$r7,4*1 - xvldrepl.w $xr6,$r7,4*2 - xvldrepl.w $xr7,$r7,4*3 - xvldrepl.w $xr8,$r7,4*4 - xvldrepl.w $xr9,$r7,4*5 - xvldrepl.w $xr10,$r7,4*6 - xvldrepl.w $xr11,$r7,4*7 - - # Load block counter - xvreplgr2vr.w $xr12,$r16 - - # Load nonce - xvldrepl.w $xr13,$r8,4*1 - xvldrepl.w $xr14,$r8,4*2 - xvldrepl.w $xr15,$r8,4*3 - - # Get the correct block counter for each block - la.local $r20,.Linc8x - xvld $xr16,$r20,0 - xvadd.w $xr12,$xr12,$xr16 - - # Copy the initial states from @x[*] to @y[*] - xvori.b $xr16,$xr0,0 - xvori.b $xr17,$xr1,0 - xvori.b $xr18,$xr2,0 - xvori.b $xr19,$xr3,0 - xvori.b $xr20,$xr4,0 - xvori.b $xr21,$xr5,0 - xvori.b $xr22,$xr6,0 - xvori.b $xr23,$xr7,0 - xvori.b $xr24,$xr8,0 - xvori.b $xr25,$xr9,0 - xvori.b $xr26,$xr10,0 - xvori.b $xr27,$xr11,0 - xvori.b $xr28,$xr12,0 - xvori.b $xr29,$xr13,0 - xvori.b $xr30,$xr14,0 - xvori.b $xr31,$xr15,0 - - # Update states in @x[*] for 20 rounds - ori $r20,$r0,10 - b .Loop_8x - -.align 5 -.Loop_8x: - xvadd.w $xr0,$xr0,$xr4 - xvxor.v $xr12,$xr12,$xr0 - xvrotri.w $xr12,$xr12,16 # rotate left 16 bits - xvadd.w $xr1,$xr1,$xr5 - xvxor.v $xr13,$xr13,$xr1 - xvrotri.w $xr13,$xr13,16 - - xvadd.w $xr8,$xr8,$xr12 - xvxor.v $xr4,$xr4,$xr8 - xvrotri.w $xr4,$xr4,20 # rotate left 12 bits - xvadd.w $xr9,$xr9,$xr13 - xvxor.v $xr5,$xr5,$xr9 - xvrotri.w $xr5,$xr5,20 - - xvadd.w $xr0,$xr0,$xr4 - xvxor.v $xr12,$xr12,$xr0 - xvrotri.w $xr12,$xr12,24 # rotate left 8 bits - xvadd.w $xr1,$xr1,$xr5 - xvxor.v $xr13,$xr13,$xr1 - xvrotri.w $xr13,$xr13,24 - - xvadd.w $xr8,$xr8,$xr12 - xvxor.v $xr4,$xr4,$xr8 - xvrotri.w $xr4,$xr4,25 # rotate left 7 bits - xvadd.w $xr9,$xr9,$xr13 - xvxor.v $xr5,$xr5,$xr9 - xvrotri.w $xr5,$xr5,25 - - xvadd.w $xr2,$xr2,$xr6 - xvxor.v $xr14,$xr14,$xr2 - xvrotri.w $xr14,$xr14,16 - xvadd.w $xr3,$xr3,$xr7 - xvxor.v $xr15,$xr15,$xr3 - xvrotri.w $xr15,$xr15,16 - - xvadd.w $xr10,$xr10,$xr14 - xvxor.v $xr6,$xr6,$xr10 - xvrotri.w $xr6,$xr6,20 - xvadd.w $xr11,$xr11,$xr15 - xvxor.v $xr7,$xr7,$xr11 - xvrotri.w $xr7,$xr7,20 - - xvadd.w $xr2,$xr2,$xr6 - xvxor.v $xr14,$xr14,$xr2 - xvrotri.w $xr14,$xr14,24 - xvadd.w $xr3,$xr3,$xr7 - xvxor.v $xr15,$xr15,$xr3 - xvrotri.w $xr15,$xr15,24 - - xvadd.w $xr10,$xr10,$xr14 - xvxor.v $xr6,$xr6,$xr10 - xvrotri.w $xr6,$xr6,25 - xvadd.w $xr11,$xr11,$xr15 - xvxor.v $xr7,$xr7,$xr11 - xvrotri.w $xr7,$xr7,25 - - xvadd.w $xr0,$xr0,$xr5 - xvxor.v $xr15,$xr15,$xr0 - xvrotri.w $xr15,$xr15,16 # rotate left 16 bits - xvadd.w $xr1,$xr1,$xr6 - xvxor.v $xr12,$xr12,$xr1 - xvrotri.w $xr12,$xr12,16 - - xvadd.w $xr10,$xr10,$xr15 - xvxor.v $xr5,$xr5,$xr10 - xvrotri.w $xr5,$xr5,20 # rotate left 12 bits - xvadd.w $xr11,$xr11,$xr12 - xvxor.v $xr6,$xr6,$xr11 - xvrotri.w $xr6,$xr6,20 - - xvadd.w $xr0,$xr0,$xr5 - xvxor.v $xr15,$xr15,$xr0 - xvrotri.w $xr15,$xr15,24 # rotate left 8 bits - xvadd.w $xr1,$xr1,$xr6 - xvxor.v $xr12,$xr12,$xr1 - xvrotri.w $xr12,$xr12,24 - - xvadd.w $xr10,$xr10,$xr15 - xvxor.v $xr5,$xr5,$xr10 - xvrotri.w $xr5,$xr5,25 # rotate left 7 bits - xvadd.w $xr11,$xr11,$xr12 - xvxor.v $xr6,$xr6,$xr11 - xvrotri.w $xr6,$xr6,25 - - xvadd.w $xr2,$xr2,$xr7 - xvxor.v $xr13,$xr13,$xr2 - xvrotri.w $xr13,$xr13,16 - xvadd.w $xr3,$xr3,$xr4 - xvxor.v $xr14,$xr14,$xr3 - xvrotri.w $xr14,$xr14,16 - - xvadd.w $xr8,$xr8,$xr13 - xvxor.v $xr7,$xr7,$xr8 - xvrotri.w $xr7,$xr7,20 - xvadd.w $xr9,$xr9,$xr14 - xvxor.v $xr4,$xr4,$xr9 - xvrotri.w $xr4,$xr4,20 - - xvadd.w $xr2,$xr2,$xr7 - xvxor.v $xr13,$xr13,$xr2 - xvrotri.w $xr13,$xr13,24 - xvadd.w $xr3,$xr3,$xr4 - xvxor.v $xr14,$xr14,$xr3 - xvrotri.w $xr14,$xr14,24 - - xvadd.w $xr8,$xr8,$xr13 - xvxor.v $xr7,$xr7,$xr8 - xvrotri.w $xr7,$xr7,25 - xvadd.w $xr9,$xr9,$xr14 - xvxor.v $xr4,$xr4,$xr9 - xvrotri.w $xr4,$xr4,25 - - addi.w $r20,$r20,-1 - bnez $r20,.Loop_8x - - # Get the final states by adding the initial states - xvadd.w $xr0,$xr0,$xr16 - xvadd.w $xr1,$xr1,$xr17 - xvadd.w $xr2,$xr2,$xr18 - xvadd.w $xr3,$xr3,$xr19 - xvadd.w $xr4,$xr4,$xr20 - xvadd.w $xr5,$xr5,$xr21 - xvadd.w $xr6,$xr6,$xr22 - xvadd.w $xr7,$xr7,$xr23 - xvadd.w $xr8,$xr8,$xr24 - xvadd.w $xr9,$xr9,$xr25 - xvadd.w $xr10,$xr10,$xr26 - xvadd.w $xr11,$xr11,$xr27 - xvadd.w $xr12,$xr12,$xr28 - xvadd.w $xr13,$xr13,$xr29 - xvadd.w $xr14,$xr14,$xr30 - xvadd.w $xr15,$xr15,$xr31 - - # Get the transpose of @x[*] and save them in @y[*] - xvilvl.w $xr16,$xr1,$xr0 - xvilvh.w $xr17,$xr1,$xr0 - xvilvl.w $xr18,$xr3,$xr2 - xvilvh.w $xr19,$xr3,$xr2 - xvilvl.w $xr20,$xr5,$xr4 - xvilvh.w $xr21,$xr5,$xr4 - xvilvl.w $xr22,$xr7,$xr6 - xvilvh.w $xr23,$xr7,$xr6 - xvilvl.w $xr24,$xr9,$xr8 - xvilvh.w $xr25,$xr9,$xr8 - xvilvl.w $xr26,$xr11,$xr10 - xvilvh.w $xr27,$xr11,$xr10 - xvilvl.w $xr28,$xr13,$xr12 - xvilvh.w $xr29,$xr13,$xr12 - xvilvl.w $xr30,$xr15,$xr14 - xvilvh.w $xr31,$xr15,$xr14 - - xvilvl.d $xr0,$xr18,$xr16 - xvilvh.d $xr1,$xr18,$xr16 - xvilvl.d $xr2,$xr19,$xr17 - xvilvh.d $xr3,$xr19,$xr17 - xvilvl.d $xr4,$xr22,$xr20 - xvilvh.d $xr5,$xr22,$xr20 - xvilvl.d $xr6,$xr23,$xr21 - xvilvh.d $xr7,$xr23,$xr21 - xvilvl.d $xr8,$xr26,$xr24 - xvilvh.d $xr9,$xr26,$xr24 - xvilvl.d $xr10,$xr27,$xr25 - xvilvh.d $xr11,$xr27,$xr25 - xvilvl.d $xr12,$xr30,$xr28 - xvilvh.d $xr13,$xr30,$xr28 - xvilvl.d $xr14,$xr31,$xr29 - xvilvh.d $xr15,$xr31,$xr29 - - xvori.b $xr16,$xr4,0 - xvpermi.q $xr16,$xr0,0x20 - xvori.b $xr17,$xr5,0 - xvpermi.q $xr17,$xr1,0x20 - xvori.b $xr18,$xr6,0 - xvpermi.q $xr18,$xr2,0x20 - xvori.b $xr19,$xr7,0 - xvpermi.q $xr19,$xr3,0x20 - xvori.b $xr20,$xr4,0 - xvpermi.q $xr20,$xr0,0x31 - xvori.b $xr21,$xr5,0 - xvpermi.q $xr21,$xr1,0x31 - xvori.b $xr22,$xr6,0 - xvpermi.q $xr22,$xr2,0x31 - xvori.b $xr23,$xr7,0 - xvpermi.q $xr23,$xr3,0x31 - xvori.b $xr24,$xr12,0 - xvpermi.q $xr24,$xr8,0x20 - xvori.b $xr25,$xr13,0 - xvpermi.q $xr25,$xr9,0x20 - xvori.b $xr26,$xr14,0 - xvpermi.q $xr26,$xr10,0x20 - xvori.b $xr27,$xr15,0 - xvpermi.q $xr27,$xr11,0x20 - xvori.b $xr28,$xr12,0 - xvpermi.q $xr28,$xr8,0x31 - xvori.b $xr29,$xr13,0 - xvpermi.q $xr29,$xr9,0x31 - xvori.b $xr30,$xr14,0 - xvpermi.q $xr30,$xr10,0x31 - xvori.b $xr31,$xr15,0 - xvpermi.q $xr31,$xr11,0x31 - - ori $r20,$r0,64*8 - bltu $r6,$r20,.Ltail_8x - - # Get the encrypted message by xor states with plaintext - xvld $xr0,$r5,32*0 - xvld $xr1,$r5,32*1 - xvld $xr2,$r5,32*2 - xvld $xr3,$r5,32*3 - xvxor.v $xr0,$xr0,$xr16 - xvxor.v $xr1,$xr1,$xr24 - xvxor.v $xr2,$xr2,$xr17 - xvxor.v $xr3,$xr3,$xr25 - xvst $xr0,$r4,32*0 - xvst $xr1,$r4,32*1 - xvst $xr2,$r4,32*2 - xvst $xr3,$r4,32*3 - - xvld $xr0,$r5,32*4 - xvld $xr1,$r5,32*5 - xvld $xr2,$r5,32*6 - xvld $xr3,$r5,32*7 - xvxor.v $xr0,$xr0,$xr18 - xvxor.v $xr1,$xr1,$xr26 - xvxor.v $xr2,$xr2,$xr19 - xvxor.v $xr3,$xr3,$xr27 - xvst $xr0,$r4,32*4 - xvst $xr1,$r4,32*5 - xvst $xr2,$r4,32*6 - xvst $xr3,$r4,32*7 - - xvld $xr0,$r5,32*8 - xvld $xr1,$r5,32*9 - xvld $xr2,$r5,32*10 - xvld $xr3,$r5,32*11 - xvxor.v $xr0,$xr0,$xr20 - xvxor.v $xr1,$xr1,$xr28 - xvxor.v $xr2,$xr2,$xr21 - xvxor.v $xr3,$xr3,$xr29 - xvst $xr0,$r4,32*8 - xvst $xr1,$r4,32*9 - xvst $xr2,$r4,32*10 - xvst $xr3,$r4,32*11 - - xvld $xr0,$r5,32*12 - xvld $xr1,$r5,32*13 - xvld $xr2,$r5,32*14 - xvld $xr3,$r5,32*15 - xvxor.v $xr0,$xr0,$xr22 - xvxor.v $xr1,$xr1,$xr30 - xvxor.v $xr2,$xr2,$xr23 - xvxor.v $xr3,$xr3,$xr31 - xvst $xr0,$r4,32*12 - xvst $xr1,$r4,32*13 - xvst $xr2,$r4,32*14 - xvst $xr3,$r4,32*15 - - addi.d $r6,$r6,-64*8 - beqz $r6,.Ldone_8x - addi.d $r5,$r5,64*8 - addi.d $r4,$r4,64*8 - addi.w $r16,$r16,8 - b .Loop_outer_8x - -.Ltail_8x: - # Handle the tail for 8x (1 <= tail_len <= 511) - ori $r20,$r0,448 - bgeu $r6,$r20,.L448_or_more8x - ori $r20,$r0,384 - bgeu $r6,$r20,.L384_or_more8x - ori $r20,$r0,320 - bgeu $r6,$r20,.L320_or_more8x - ori $r20,$r0,256 - bgeu $r6,$r20,.L256_or_more8x - ori $r20,$r0,192 - bgeu $r6,$r20,.L192_or_more8x - ori $r20,$r0,128 - bgeu $r6,$r20,.L128_or_more8x - ori $r20,$r0,64 - bgeu $r6,$r20,.L64_or_more8x - - xvst $xr16,$r3,32*0 - xvst $xr24,$r3,32*1 - move $r20,$r0 - b .Loop_tail_8x - -.align 5 -.L64_or_more8x: - xvld $xr0,$r5,32*0 - xvld $xr1,$r5,32*1 - xvxor.v $xr0,$xr0,$xr16 - xvxor.v $xr1,$xr1,$xr24 - xvst $xr0,$r4,32*0 - xvst $xr1,$r4,32*1 - - addi.d $r6,$r6,-64 - beqz $r6,.Ldone_8x - addi.d $r5,$r5,64 - addi.d $r4,$r4,64 - xvst $xr17,$r3,32*0 - xvst $xr25,$r3,32*1 - move $r20,$r0 - b .Loop_tail_8x - -.align 5 -.L128_or_more8x: - xvld $xr0,$r5,32*0 - xvld $xr1,$r5,32*1 - xvld $xr2,$r5,32*2 - xvld $xr3,$r5,32*3 - xvxor.v $xr0,$xr0,$xr16 - xvxor.v $xr1,$xr1,$xr24 - xvxor.v $xr2,$xr2,$xr17 - xvxor.v $xr3,$xr3,$xr25 - xvst $xr0,$r4,32*0 - xvst $xr1,$r4,32*1 - xvst $xr2,$r4,32*2 - xvst $xr3,$r4,32*3 - - addi.d $r6,$r6,-128 - beqz $r6,.Ldone_8x - addi.d $r5,$r5,128 - addi.d $r4,$r4,128 - xvst $xr18,$r3,32*0 - xvst $xr26,$r3,32*1 - move $r20,$r0 - b .Loop_tail_8x - -.align 5 -.L192_or_more8x: - xvld $xr0,$r5,32*0 - xvld $xr1,$r5,32*1 - xvld $xr2,$r5,32*2 - xvld $xr3,$r5,32*3 - xvxor.v $xr0,$xr0,$xr16 - xvxor.v $xr1,$xr1,$xr24 - xvxor.v $xr2,$xr2,$xr17 - xvxor.v $xr3,$xr3,$xr25 - xvst $xr0,$r4,32*0 - xvst $xr1,$r4,32*1 - xvst $xr2,$r4,32*2 - xvst $xr3,$r4,32*3 - - xvld $xr0,$r5,32*4 - xvld $xr1,$r5,32*5 - xvxor.v $xr0,$xr0,$xr18 - xvxor.v $xr1,$xr1,$xr26 - xvst $xr0,$r4,32*4 - xvst $xr1,$r4,32*5 - - addi.d $r6,$r6,-192 - beqz $r6,.Ldone_8x - addi.d $r5,$r5,192 - addi.d $r4,$r4,192 - xvst $xr19,$r3,32*0 - xvst $xr27,$r3,32*1 - move $r20,$r0 - b .Loop_tail_8x - -.align 5 -.L256_or_more8x: - xvld $xr0,$r5,32*0 - xvld $xr1,$r5,32*1 - xvld $xr2,$r5,32*2 - xvld $xr3,$r5,32*3 - xvxor.v $xr0,$xr0,$xr16 - xvxor.v $xr1,$xr1,$xr24 - xvxor.v $xr2,$xr2,$xr17 - xvxor.v $xr3,$xr3,$xr25 - xvst $xr0,$r4,32*0 - xvst $xr1,$r4,32*1 - xvst $xr2,$r4,32*2 - xvst $xr3,$r4,32*3 - - xvld $xr0,$r5,32*4 - xvld $xr1,$r5,32*5 - xvld $xr2,$r5,32*6 - xvld $xr3,$r5,32*7 - xvxor.v $xr0,$xr0,$xr18 - xvxor.v $xr1,$xr1,$xr26 - xvxor.v $xr2,$xr2,$xr19 - xvxor.v $xr3,$xr3,$xr27 - xvst $xr0,$r4,32*4 - xvst $xr1,$r4,32*5 - xvst $xr2,$r4,32*6 - xvst $xr3,$r4,32*7 - - addi.d $r6,$r6,-256 - beqz $r6,.Ldone_8x - addi.d $r5,$r5,256 - addi.d $r4,$r4,256 - xvst $xr20,$r3,32*0 - xvst $xr28,$r3,32*1 - move $r20,$r0 - b .Loop_tail_8x - -.align 5 -.L320_or_more8x: - xvld $xr0,$r5,32*0 - xvld $xr1,$r5,32*1 - xvld $xr2,$r5,32*2 - xvld $xr3,$r5,32*3 - xvxor.v $xr0,$xr0,$xr16 - xvxor.v $xr1,$xr1,$xr24 - xvxor.v $xr2,$xr2,$xr17 - xvxor.v $xr3,$xr3,$xr25 - xvst $xr0,$r4,32*0 - xvst $xr1,$r4,32*1 - xvst $xr2,$r4,32*2 - xvst $xr3,$r4,32*3 - - xvld $xr0,$r5,32*4 - xvld $xr1,$r5,32*5 - xvld $xr2,$r5,32*6 - xvld $xr3,$r5,32*7 - xvxor.v $xr0,$xr0,$xr18 - xvxor.v $xr1,$xr1,$xr26 - xvxor.v $xr2,$xr2,$xr19 - xvxor.v $xr3,$xr3,$xr27 - xvst $xr0,$r4,32*4 - xvst $xr1,$r4,32*5 - xvst $xr2,$r4,32*6 - xvst $xr3,$r4,32*7 - - xvld $xr0,$r5,32*8 - xvld $xr1,$r5,32*9 - xvxor.v $xr0,$xr0,$xr20 - xvxor.v $xr1,$xr1,$xr28 - xvst $xr0,$r4,32*8 - xvst $xr1,$r4,32*9 - - addi.d $r6,$r6,-320 - beqz $r6,.Ldone_8x - addi.d $r5,$r5,320 - addi.d $r4,$r4,320 - xvst $xr21,$r3,32*0 - xvst $xr29,$r3,32*1 - move $r20,$r0 - b .Loop_tail_8x - -.align 5 -.L384_or_more8x: - xvld $xr0,$r5,32*0 - xvld $xr1,$r5,32*1 - xvld $xr2,$r5,32*2 - xvld $xr3,$r5,32*3 - xvxor.v $xr0,$xr0,$xr16 - xvxor.v $xr1,$xr1,$xr24 - xvxor.v $xr2,$xr2,$xr17 - xvxor.v $xr3,$xr3,$xr25 - xvst $xr0,$r4,32*0 - xvst $xr1,$r4,32*1 - xvst $xr2,$r4,32*2 - xvst $xr3,$r4,32*3 - - xvld $xr0,$r5,32*4 - xvld $xr1,$r5,32*5 - xvld $xr2,$r5,32*6 - xvld $xr3,$r5,32*7 - xvxor.v $xr0,$xr0,$xr18 - xvxor.v $xr1,$xr1,$xr26 - xvxor.v $xr2,$xr2,$xr19 - xvxor.v $xr3,$xr3,$xr27 - xvst $xr0,$r4,32*4 - xvst $xr1,$r4,32*5 - xvst $xr2,$r4,32*6 - xvst $xr3,$r4,32*7 - - xvld $xr0,$r5,32*8 - xvld $xr1,$r5,32*9 - xvld $xr2,$r5,32*10 - xvld $xr3,$r5,32*11 - xvxor.v $xr0,$xr0,$xr20 - xvxor.v $xr1,$xr1,$xr28 - xvxor.v $xr2,$xr2,$xr21 - xvxor.v $xr3,$xr3,$xr29 - xvst $xr0,$r4,32*8 - xvst $xr1,$r4,32*9 - xvst $xr2,$r4,32*10 - xvst $xr3,$r4,32*11 - - addi.d $r6,$r6,-384 - beqz $r6,.Ldone_8x - addi.d $r5,$r5,384 - addi.d $r4,$r4,384 - xvst $xr22,$r3,32*0 - xvst $xr30,$r3,32*1 - move $r20,$r0 - b .Loop_tail_8x - -.align 5 -.L448_or_more8x: - xvld $xr0,$r5,32*0 - xvld $xr1,$r5,32*1 - xvld $xr2,$r5,32*2 - xvld $xr3,$r5,32*3 - xvxor.v $xr0,$xr0,$xr16 - xvxor.v $xr1,$xr1,$xr24 - xvxor.v $xr2,$xr2,$xr17 - xvxor.v $xr3,$xr3,$xr25 - xvst $xr0,$r4,32*0 - xvst $xr1,$r4,32*1 - xvst $xr2,$r4,32*2 - xvst $xr3,$r4,32*3 - - xvld $xr0,$r5,32*4 - xvld $xr1,$r5,32*5 - xvld $xr2,$r5,32*6 - xvld $xr3,$r5,32*7 - xvxor.v $xr0,$xr0,$xr18 - xvxor.v $xr1,$xr1,$xr26 - xvxor.v $xr2,$xr2,$xr19 - xvxor.v $xr3,$xr3,$xr27 - xvst $xr0,$r4,32*4 - xvst $xr1,$r4,32*5 - xvst $xr2,$r4,32*6 - xvst $xr3,$r4,32*7 - - xvld $xr0,$r5,32*8 - xvld $xr1,$r5,32*9 - xvld $xr2,$r5,32*10 - xvld $xr3,$r5,32*11 - xvxor.v $xr0,$xr0,$xr20 - xvxor.v $xr1,$xr1,$xr28 - xvxor.v $xr2,$xr2,$xr21 - xvxor.v $xr3,$xr3,$xr29 - xvst $xr0,$r4,32*8 - xvst $xr1,$r4,32*9 - xvst $xr2,$r4,32*10 - xvst $xr3,$r4,32*11 - - xvld $xr0,$r5,32*12 - xvld $xr1,$r5,32*13 - xvxor.v $xr0,$xr0,$xr22 - xvxor.v $xr1,$xr1,$xr30 - xvst $xr0,$r4,32*12 - xvst $xr1,$r4,32*13 - - addi.d $r6,$r6,-448 - beqz $r6,.Ldone_8x - addi.d $r5,$r5,448 - addi.d $r4,$r4,448 - xvst $xr23,$r3,32*0 - xvst $xr31,$r3,32*1 - move $r20,$r0 - b .Loop_tail_8x - -.Loop_tail_8x: - # Xor input with states byte by byte - ldx.bu $r17,$r5,$r20 - ldx.bu $r18,$r3,$r20 - xor $r17,$r17,$r18 - stx.b $r17,$r4,$r20 - addi.w $r20,$r20,1 - addi.d $r6,$r6,-1 - bnez $r6,.Loop_tail_8x - b .Ldone_8x - -.Ldone_8x: - addi.d $r3,$r3,128 - b .Lrestore_saved_fpr - -.Lrestore_saved_fpr: - fld.d $f24,$r3,0 - fld.d $f25,$r3,8 - fld.d $f26,$r3,16 - fld.d $f27,$r3,24 - fld.d $f28,$r3,32 - fld.d $f29,$r3,40 - fld.d $f30,$r3,48 - fld.d $f31,$r3,56 - addi.d $r3,$r3,64 -.Lno_data: -.Lend: - jr $r1 -.size ChaCha20_ctr32,.-ChaCha20_ctr32 diff --git a/openssl/src/crypto/chacha/gen/linux_ppc64/chachap10-ppc.s b/openssl/src/crypto/chacha/gen/linux_ppc64/chachap10-ppc.s deleted file mode 100644 index dc098ec4d..000000000 --- a/openssl/src/crypto/chacha/gen/linux_ppc64/chachap10-ppc.s +++ /dev/null @@ -1,1227 +0,0 @@ - -.globl ChaCha20_ctr32_vsx_p10 -.type ChaCha20_ctr32_vsx_p10,@function -.align 5 -ChaCha20_ctr32_vsx_p10: -.localentry ChaCha20_ctr32_vsx_p10,0 - - cmpldi 5,255 - ble .Not_greater_than_8x - b ChaCha20_ctr32_vsx_8x -.Not_greater_than_8x: - stdu 1,-224(1) - mflr 0 - li 10,127 - li 11,143 - li 12,-1 - stvx 26,10,1 - addi 10,10,32 - stvx 27,11,1 - addi 11,11,32 - stvx 28,10,1 - addi 10,10,32 - stvx 29,11,1 - addi 11,11,32 - stvx 30,10,1 - stvx 31,11,1 - stw 12,220(1) - li 12,-4096+63 - std 0, 240(1) - or 12,12,12 - - bl .Lconsts - .long 0x7E006619 - addi 12,12,0x70 - li 8,16 - li 9,32 - li 10,48 - li 11,64 - - .long 0x7E203619 - .long 0x7E483619 - .long 0x7E603E19 - - vxor 27,27,27 - .long 0x7F8B6619 - vspltw 26,19,0 - vsldoi 19,19,27,4 - vsldoi 19,27,19,12 - vadduwm 26,26,28 - - - - - - li 0,10 - mtctr 0 - b .Loop_outer_vsx - -.align 5 -.Loop_outer_vsx: - lvx 0,0,12 - lvx 1,8,12 - lvx 2,9,12 - lvx 3,10,12 - - vspltw 4,17,0 - vspltw 5,17,1 - vspltw 6,17,2 - vspltw 7,17,3 - - vspltw 8,18,0 - vspltw 9,18,1 - vspltw 10,18,2 - vspltw 11,18,3 - - vor 12,26,26 - vspltw 13,19,1 - vspltw 14,19,2 - vspltw 15,19,3 - - vspltisw 27,-16 - vspltisw 28,12 - vspltisw 29,8 - vspltisw 30,7 - -.Loop_vsx_4x: - vadduwm 0,0,4 - vadduwm 1,1,5 - vadduwm 2,2,6 - vadduwm 3,3,7 - vxor 12,12,0 - vxor 13,13,1 - vxor 14,14,2 - vxor 15,15,3 - vrlw 12,12,27 - vrlw 13,13,27 - vrlw 14,14,27 - vrlw 15,15,27 - vadduwm 8,8,12 - vadduwm 9,9,13 - vadduwm 10,10,14 - vadduwm 11,11,15 - vxor 4,4,8 - vxor 5,5,9 - vxor 6,6,10 - vxor 7,7,11 - vrlw 4,4,28 - vrlw 5,5,28 - vrlw 6,6,28 - vrlw 7,7,28 - vadduwm 0,0,4 - vadduwm 1,1,5 - vadduwm 2,2,6 - vadduwm 3,3,7 - vxor 12,12,0 - vxor 13,13,1 - vxor 14,14,2 - vxor 15,15,3 - vrlw 12,12,29 - vrlw 13,13,29 - vrlw 14,14,29 - vrlw 15,15,29 - vadduwm 8,8,12 - vadduwm 9,9,13 - vadduwm 10,10,14 - vadduwm 11,11,15 - vxor 4,4,8 - vxor 5,5,9 - vxor 6,6,10 - vxor 7,7,11 - vrlw 4,4,30 - vrlw 5,5,30 - vrlw 6,6,30 - vrlw 7,7,30 - vadduwm 0,0,5 - vadduwm 1,1,6 - vadduwm 2,2,7 - vadduwm 3,3,4 - vxor 15,15,0 - vxor 12,12,1 - vxor 13,13,2 - vxor 14,14,3 - vrlw 15,15,27 - vrlw 12,12,27 - vrlw 13,13,27 - vrlw 14,14,27 - vadduwm 10,10,15 - vadduwm 11,11,12 - vadduwm 8,8,13 - vadduwm 9,9,14 - vxor 5,5,10 - vxor 6,6,11 - vxor 7,7,8 - vxor 4,4,9 - vrlw 5,5,28 - vrlw 6,6,28 - vrlw 7,7,28 - vrlw 4,4,28 - vadduwm 0,0,5 - vadduwm 1,1,6 - vadduwm 2,2,7 - vadduwm 3,3,4 - vxor 15,15,0 - vxor 12,12,1 - vxor 13,13,2 - vxor 14,14,3 - vrlw 15,15,29 - vrlw 12,12,29 - vrlw 13,13,29 - vrlw 14,14,29 - vadduwm 10,10,15 - vadduwm 11,11,12 - vadduwm 8,8,13 - vadduwm 9,9,14 - vxor 5,5,10 - vxor 6,6,11 - vxor 7,7,8 - vxor 4,4,9 - vrlw 5,5,30 - vrlw 6,6,30 - vrlw 7,7,30 - vrlw 4,4,30 - - bdnz .Loop_vsx_4x - - vadduwm 12,12,26 - - .long 0x13600F8C - .long 0x13821F8C - .long 0x10000E8C - .long 0x10421E8C - .long 0x13A42F8C - .long 0x13C63F8C - .long 0xF0201057 - .long 0xF0601357 - .long 0xF01BE057 - .long 0xF05BE357 - - .long 0x10842E8C - .long 0x10C63E8C - .long 0x13684F8C - .long 0x138A5F8C - .long 0xF0A43057 - .long 0xF0E43357 - .long 0xF09DF057 - .long 0xF0DDF357 - - .long 0x11084E8C - .long 0x114A5E8C - .long 0x13AC6F8C - .long 0x13CE7F8C - .long 0xF1285057 - .long 0xF1685357 - .long 0xF11BE057 - .long 0xF15BE357 - - .long 0x118C6E8C - .long 0x11CE7E8C - vspltisw 27,4 - vadduwm 26,26,27 - .long 0xF1AC7057 - .long 0xF1EC7357 - .long 0xF19DF057 - .long 0xF1DDF357 - - vadduwm 0,0,16 - vadduwm 4,4,17 - vadduwm 8,8,18 - vadduwm 12,12,19 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx - - .long 0x7F602619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 27,27,0 - vxor 28,28,4 - vxor 29,29,8 - vxor 30,30,12 - - .long 0x7F601F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx - - vadduwm 0,1,16 - vadduwm 4,5,17 - vadduwm 8,9,18 - vadduwm 12,13,19 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx - - .long 0x7F602619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 27,27,0 - vxor 28,28,4 - vxor 29,29,8 - vxor 30,30,12 - - .long 0x7F601F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx - - vadduwm 0,2,16 - vadduwm 4,6,17 - vadduwm 8,10,18 - vadduwm 12,14,19 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx - - .long 0x7F602619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 27,27,0 - vxor 28,28,4 - vxor 29,29,8 - vxor 30,30,12 - - .long 0x7F601F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx - - vadduwm 0,3,16 - vadduwm 4,7,17 - vadduwm 8,11,18 - vadduwm 12,15,19 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx - - .long 0x7F602619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 27,27,0 - vxor 28,28,4 - vxor 29,29,8 - vxor 30,30,12 - - .long 0x7F601F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - mtctr 0 - bne .Loop_outer_vsx - -.Ldone_vsx: - lwz 12,220(1) - li 10,127 - li 11,143 - ld 0, 240(1) - or 12,12,12 - lvx 26,10,1 - addi 10,10,32 - lvx 27,11,1 - addi 11,11,32 - lvx 28,10,1 - addi 10,10,32 - lvx 29,11,1 - addi 11,11,32 - lvx 30,10,1 - lvx 31,11,1 - mtlr 0 - addi 1,1,224 - blr - -.align 4 -.Ltail_vsx: - addi 11,1,48 - mtctr 5 - .long 0x7C005F19 - .long 0x7C885F19 - .long 0x7D095F19 - .long 0x7D8A5F19 - subi 12,11,1 - subi 4,4,1 - subi 3,3,1 - -.Loop_tail_vsx: - lbzu 6,1(12) - lbzu 7,1(4) - xor 6,6,7 - stbu 6,1(3) - bdnz .Loop_tail_vsx - - .long 0x7E005F19 - .long 0x7E085F19 - .long 0x7E095F19 - .long 0x7E0A5F19 - - b .Ldone_vsx -.long 0 -.byte 0,12,0x04,1,0x80,0,5,0 -.long 0 -.size ChaCha20_ctr32_vsx_p10,.-ChaCha20_ctr32_vsx_p10 - -.globl ChaCha20_ctr32_vsx_8x -.type ChaCha20_ctr32_vsx_8x,@function -.align 5 -ChaCha20_ctr32_vsx_8x: -.localentry ChaCha20_ctr32_vsx_8x,0 - - stdu 1,-256(1) - mflr 0 - li 10,127 - li 11,143 - li 12,-1 - stvx 24,10,1 - addi 10,10,32 - stvx 25,11,1 - addi 11,11,32 - stvx 26,10,1 - addi 10,10,32 - stvx 27,11,1 - addi 11,11,32 - stvx 28,10,1 - addi 10,10,32 - stvx 29,11,1 - addi 11,11,32 - stvx 30,10,1 - stvx 31,11,1 - stw 12,252(1) - li 12,-4096+63 - std 0, 272(1) - or 12,12,12 - - bl .Lconsts - - .long 0x7F606619 - addi 12,12,0x70 - li 8,16 - li 9,32 - li 10,48 - li 11,64 - - vspltisw 16,-16 - vspltisw 20,12 - vspltisw 24,8 - vspltisw 28,7 - - lvx 0,0,12 - lvx 1,8,12 - lvx 2,9,12 - lvx 3,10,12 - - .long 0xF1308496 - .long 0xF154A496 - .long 0xF178C496 - .long 0xF19CE496 - .long 0xF2C00496 - .long 0xF2E10C96 - .long 0xF3021496 - .long 0xF3231C96 - - .long 0x7F003619 - .long 0x7F283619 - .long 0x7F403E19 - vspltisw 30,4 - - - vxor 29,29,29 - .long 0x7F8B6619 - vspltw 2,26,0 - vsldoi 26,26,29,4 - vsldoi 26,29,26,12 - vadduwm 28,2,28 - vadduwm 30,28,30 - vspltw 0,25,2 - - - - - - - .long 0xF01BDC96 - .long 0xF038C496 - .long 0xF059CC96 - .long 0xF07AD496 - .long 0xF09CE496 - .long 0xF0BEF496 - .long 0xF1000496 - - li 0,10 - mtctr 0 - b .Loop_outer_vsx_8x - -.align 5 -.Loop_outer_vsx_8x: - .long 0xF016B491 - .long 0xF037BC91 - .long 0xF058C491 - .long 0xF079CC91 - .long 0xF216B491 - .long 0xF237BC91 - .long 0xF258C491 - .long 0xF279CC91 - - vspltw 4,24,0 - vspltw 5,24,1 - vspltw 6,24,2 - vspltw 7,24,3 - vspltw 20,24,0 - vspltw 21,24,1 - vspltw 22,24,2 - vspltw 23,24,3 - - vspltw 8,25,0 - vspltw 9,25,1 - vspltw 10,25,2 - vspltw 11,25,3 - vspltw 24,25,0 - vspltw 27,25,3 - vspltw 25,25,1 - - .long 0xF1842491 - vspltw 13,26,1 - vspltw 14,26,2 - vspltw 15,26,3 - .long 0xF3852C91 - vspltw 29,26,1 - vspltw 30,26,2 - vspltw 31,26,3 - .long 0xF3484491 - -.Loop_vsx_8x: - .long 0xF1FBDC96 - .long 0xF3694C91 - vadduwm 0,0,4 - vadduwm 1,1,5 - vadduwm 2,2,6 - vadduwm 3,3,7 - vadduwm 16,16,20 - vadduwm 17,17,21 - vadduwm 18,18,22 - vadduwm 19,19,23 - vxor 12,12,0 - vxor 13,13,1 - vxor 14,14,2 - vxor 15,15,3 - vxor 28,28,16 - vxor 29,29,17 - vxor 30,30,18 - vxor 31,31,19 - vrlw 12,12,27 - vrlw 13,13,27 - vrlw 14,14,27 - vrlw 15,15,27 - vrlw 28,28,27 - vrlw 29,29,27 - vrlw 30,30,27 - vrlw 31,31,27 - .long 0xF1B39C96 - .long 0xF36F7C91 - .long 0xF26A5491 - vadduwm 8,8,12 - vadduwm 9,9,13 - vadduwm 10,10,14 - vadduwm 11,11,15 - vadduwm 24,24,28 - vadduwm 25,25,29 - vadduwm 26,26,30 - vadduwm 27,27,31 - vxor 4,4,8 - vxor 5,5,9 - vxor 6,6,10 - vxor 7,7,11 - vxor 20,20,24 - vxor 21,21,25 - vxor 22,22,26 - vxor 23,23,27 - vrlw 4,4,19 - vrlw 5,5,19 - vrlw 6,6,19 - vrlw 7,7,19 - vrlw 20,20,19 - vrlw 21,21,19 - vrlw 22,22,19 - vrlw 23,23,19 - .long 0xF26D6C91 - .long 0xF1FBDC96 - .long 0xF36B5C91 - vadduwm 0,0,4 - vadduwm 1,1,5 - vadduwm 2,2,6 - vadduwm 3,3,7 - vadduwm 16,16,20 - vadduwm 17,17,21 - vadduwm 18,18,22 - vadduwm 19,19,23 - vxor 12,12,0 - vxor 13,13,1 - vxor 14,14,2 - vxor 15,15,3 - vxor 28,28,16 - vxor 29,29,17 - vxor 30,30,18 - vxor 31,31,19 - vrlw 12,12,27 - vrlw 13,13,27 - vrlw 14,14,27 - vrlw 15,15,27 - vrlw 28,28,27 - vrlw 29,29,27 - vrlw 30,30,27 - vrlw 31,31,27 - .long 0xF36F7C91 - .long 0xF1B39C96 - .long 0xF26C6491 - vadduwm 8,8,12 - vadduwm 9,9,13 - vadduwm 10,10,14 - vadduwm 11,11,15 - vadduwm 24,24,28 - vadduwm 25,25,29 - vadduwm 26,26,30 - vadduwm 27,27,31 - vxor 4,4,8 - vxor 5,5,9 - vxor 6,6,10 - vxor 7,7,11 - vxor 20,20,24 - vxor 21,21,25 - vxor 22,22,26 - vxor 23,23,27 - vrlw 4,4,19 - vrlw 5,5,19 - vrlw 6,6,19 - vrlw 7,7,19 - vrlw 20,20,19 - vrlw 21,21,19 - vrlw 22,22,19 - vrlw 23,23,19 - .long 0xF26D6C91 - .long 0xF1F9CC96 - .long 0xF3294C91 - vadduwm 0,0,5 - vadduwm 1,1,6 - vadduwm 2,2,7 - vadduwm 3,3,4 - vadduwm 16,16,21 - vadduwm 17,17,22 - vadduwm 18,18,23 - vadduwm 19,19,20 - vxor 15,15,0 - vxor 12,12,1 - vxor 13,13,2 - vxor 14,14,3 - vxor 31,31,16 - vxor 28,28,17 - vxor 29,29,18 - vxor 30,30,19 - vrlw 15,15,25 - vrlw 12,12,25 - vrlw 13,13,25 - vrlw 14,14,25 - vrlw 31,31,25 - vrlw 28,28,25 - vrlw 29,29,25 - vrlw 30,30,25 - .long 0xF1B39C96 - .long 0xF32F7C91 - .long 0xF26A5491 - vadduwm 10,10,15 - vadduwm 11,11,12 - vadduwm 8,8,13 - vadduwm 9,9,14 - vadduwm 26,26,31 - vadduwm 27,27,28 - vadduwm 24,24,29 - vadduwm 25,25,30 - vxor 5,5,10 - vxor 6,6,11 - vxor 7,7,8 - vxor 4,4,9 - vxor 21,21,26 - vxor 22,22,27 - vxor 23,23,24 - vxor 20,20,25 - vrlw 5,5,19 - vrlw 6,6,19 - vrlw 7,7,19 - vrlw 4,4,19 - vrlw 21,21,19 - vrlw 22,22,19 - vrlw 23,23,19 - vrlw 20,20,19 - .long 0xF26D6C91 - .long 0xF1F9CC96 - .long 0xF32B5C91 - vadduwm 0,0,5 - vadduwm 1,1,6 - vadduwm 2,2,7 - vadduwm 3,3,4 - vadduwm 16,16,21 - vadduwm 17,17,22 - vadduwm 18,18,23 - vadduwm 19,19,20 - vxor 15,15,0 - vxor 12,12,1 - vxor 13,13,2 - vxor 14,14,3 - vxor 31,31,16 - vxor 28,28,17 - vxor 29,29,18 - vxor 30,30,19 - vrlw 15,15,25 - vrlw 12,12,25 - vrlw 13,13,25 - vrlw 14,14,25 - vrlw 31,31,25 - vrlw 28,28,25 - vrlw 29,29,25 - vrlw 30,30,25 - .long 0xF32F7C91 - .long 0xF1B39C96 - .long 0xF26C6491 - vadduwm 10,10,15 - vadduwm 11,11,12 - vadduwm 8,8,13 - vadduwm 9,9,14 - vadduwm 26,26,31 - vadduwm 27,27,28 - vadduwm 24,24,29 - vadduwm 25,25,30 - vxor 5,5,10 - vxor 6,6,11 - vxor 7,7,8 - vxor 4,4,9 - vxor 21,21,26 - vxor 22,22,27 - vxor 23,23,24 - vxor 20,20,25 - vrlw 5,5,19 - vrlw 6,6,19 - vrlw 7,7,19 - vrlw 4,4,19 - vrlw 21,21,19 - vrlw 22,22,19 - vrlw 23,23,19 - vrlw 20,20,19 - .long 0xF26D6C91 - - bdnz .Loop_vsx_8x - .long 0xF1BCE496 - .long 0xF1DDEC96 - .long 0xF1FEF496 - .long 0xF21FFC96 - - .long 0xF258C496 - .long 0xF279CC96 - .long 0xF29AD496 - .long 0xF2BBDC96 - - .long 0xF0D6B496 - .long 0xF0F7BC96 - - - .long 0xF3600491 - .long 0xF3010C91 - .long 0xF3221491 - .long 0xF3431C91 - .long 0xF2C42491 - - - .long 0x12E00F8C - .long 0x13821F8C - .long 0x10000E8C - .long 0x10421E8C - - .long 0x13A42F8C - .long 0x13C63F8C - .long 0x10842E8C - .long 0x10C63E8C - - vadduwm 12,12,22 - - .long 0xF0201057 - .long 0xF0601357 - .long 0xF017E057 - .long 0xF057E357 - .long 0xF0A43057 - .long 0xF0E43357 - .long 0xF09DF057 - .long 0xF0DDF357 - - .long 0x12E84F8C - .long 0x138A5F8C - .long 0x11084E8C - .long 0x114A5E8C - .long 0x13AC6F8C - .long 0x13CE7F8C - .long 0x118C6E8C - .long 0x11CE7E8C - - .long 0xF1285057 - .long 0xF1685357 - .long 0xF117E057 - .long 0xF157E357 - .long 0xF1AC7057 - .long 0xF1EC7357 - .long 0xF19DF057 - .long 0xF1DDF357 - - vspltisw 23,8 - vadduwm 22,22,23 - .long 0xF096B496 - - vadduwm 0,0,27 - vadduwm 4,4,24 - vadduwm 8,8,25 - vadduwm 12,12,26 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx_8x - - .long 0x7EE02619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 23,23,0 - vxor 28,28,4 - vxor 29,29,8 - vxor 30,30,12 - - .long 0x7EE01F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx_8x - - vadduwm 0,1,27 - vadduwm 4,5,24 - vadduwm 8,9,25 - vadduwm 12,13,26 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx_8x - - .long 0x7EE02619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 23,23,0 - vxor 28,28,4 - vxor 29,29,8 - vxor 30,30,12 - - .long 0x7EE01F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx_8x - - vadduwm 0,2,27 - vadduwm 4,6,24 - vadduwm 8,10,25 - vadduwm 12,14,26 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx_8x - - .long 0x7EE02619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 23,23,0 - vxor 28,28,4 - vxor 29,29,8 - vxor 30,30,12 - - .long 0x7EE01F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx_8x - - vadduwm 0,3,27 - vadduwm 4,7,24 - vadduwm 8,11,25 - vadduwm 12,15,26 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx_8x - - .long 0x7EE02619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 23,23,0 - vxor 28,28,4 - vxor 29,29,8 - vxor 30,30,12 - - .long 0x7EE01F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx_8x - - - - - .long 0xF0A52C91 - - .long 0xF1129491 - .long 0xF1339C91 - .long 0xF154A491 - .long 0xF175AC91 - - .long 0xF18D6C91 - .long 0xF1AE7491 - .long 0xF1CF7C91 - .long 0xF1F08491 - vadduwm 12,12,5 - - .long 0xF2C63491 - .long 0xF2E73C91 - - - .long 0x10908F8C - .long 0x13929F8C - .long 0x12108E8C - .long 0x12529E8C - .long 0x13B4AF8C - .long 0x13D6BF8C - .long 0x1294AE8C - .long 0x12D6BE8C - - .long 0xF2309057 - .long 0xF2709357 - .long 0xF204E057 - .long 0xF244E357 - .long 0xF2B4B057 - .long 0xF2F4B357 - .long 0xF29DF057 - .long 0xF2DDF357 - - .long 0x10884F8C - .long 0x138A5F8C - .long 0x11084E8C - .long 0x114A5E8C - .long 0x13AC6F8C - .long 0x13CE7F8C - .long 0x118C6E8C - .long 0x11CE7E8C - - .long 0xF1285057 - .long 0xF1685357 - .long 0xF104E057 - .long 0xF144E357 - .long 0xF1AC7057 - .long 0xF1EC7357 - .long 0xF19DF057 - .long 0xF1DDF357 - - vspltisw 4,8 - vadduwm 5,5,4 - .long 0xF0A52C96 - - vadduwm 0,16,27 - vadduwm 1,20,24 - vadduwm 2,8,25 - vadduwm 3,12,26 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx_8x_1 - - .long 0x7C802619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 4,4,0 - vxor 28,28,1 - vxor 29,29,2 - vxor 30,30,3 - - .long 0x7C801F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx_8x - - vadduwm 0,17,27 - vadduwm 1,21,24 - vadduwm 2,9,25 - vadduwm 3,13,26 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx_8x_1 - - .long 0x7C802619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 4,4,0 - vxor 28,28,1 - vxor 29,29,2 - vxor 30,30,3 - - .long 0x7C801F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx_8x - - vadduwm 0,18,27 - vadduwm 1,22,24 - vadduwm 2,10,25 - vadduwm 3,14,26 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx_8x_1 - - .long 0x7C802619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 4,4,0 - vxor 28,28,1 - vxor 29,29,2 - vxor 30,30,3 - - .long 0x7C801F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx_8x - - vadduwm 0,19,27 - vadduwm 1,23,24 - vadduwm 2,11,25 - vadduwm 3,15,26 - - - - - - - cmpldi 5,0x40 - blt .Ltail_vsx_8x_1 - - .long 0x7C802619 - .long 0x7F882619 - .long 0x7FA92619 - .long 0x7FCA2619 - - vxor 4,4,0 - vxor 28,28,1 - vxor 29,29,2 - vxor 30,30,3 - - .long 0x7C801F19 - .long 0x7F881F19 - addi 4,4,0x40 - .long 0x7FA91F19 - subi 5,5,0x40 - .long 0x7FCA1F19 - addi 3,3,0x40 - beq .Ldone_vsx_8x - - mtctr 0 - bne .Loop_outer_vsx_8x - -.Ldone_vsx_8x: - lwz 12,252(1) - li 10,127 - li 11,143 - ld 0, 272(1) - or 12,12,12 - lvx 24,10,1 - addi 10,10,32 - lvx 25,11,1 - addi 11,11,32 - lvx 26,10,1 - addi 10,10,32 - lvx 27,11,1 - addi 11,11,32 - lvx 28,10,1 - addi 10,10,32 - lvx 29,11,1 - addi 11,11,32 - lvx 30,10,1 - lvx 31,11,1 - mtlr 0 - addi 1,1,256 - blr - -.align 4 -.Ltail_vsx_8x: - addi 11,1,48 - mtctr 5 - .long 0x7C005F19 - .long 0x7C885F19 - .long 0x7D095F19 - .long 0x7D8A5F19 - subi 12,11,1 - subi 4,4,1 - subi 3,3,1 - bl .Loop_tail_vsx_8x -.Ltail_vsx_8x_1: - addi 11,1,48 - mtctr 5 - .long 0x7C005F19 - .long 0x7C285F19 - .long 0x7C495F19 - .long 0x7C6A5F19 - subi 12,11,1 - subi 4,4,1 - subi 3,3,1 - bl .Loop_tail_vsx_8x - -.Loop_tail_vsx_8x: - lbzu 6,1(12) - lbzu 7,1(4) - xor 6,6,7 - stbu 6,1(3) - bdnz .Loop_tail_vsx_8x - - .long 0x7F605F19 - .long 0x7F685F19 - .long 0x7F695F19 - .long 0x7F6A5F19 - - b .Ldone_vsx_8x -.long 0 -.byte 0,12,0x04,1,0x80,0,5,0 -.long 0 -.size ChaCha20_ctr32_vsx_8x,.-ChaCha20_ctr32_vsx_8x -.align 5 -.Lconsts: - mflr 0 - bcl 20,31,$+4 - mflr 12 - addi 12,12,56 - mtlr 0 - blr -.long 0 -.byte 0,12,0x14,0,0,0,0,0 -.space 28 -.Lsigma: -.long 0x61707865,0x3320646e,0x79622d32,0x6b206574 -.long 1,0,0,0 -.long 2,0,0,0 -.long 3,0,0,0 -.long 4,0,0,0 -.long 0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001 -.long 0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300 -.long 0x61707865,0x61707865,0x61707865,0x61707865 -.long 0x3320646e,0x3320646e,0x3320646e,0x3320646e -.long 0x79622d32,0x79622d32,0x79622d32,0x79622d32 -.long 0x6b206574,0x6b206574,0x6b206574,0x6b206574 -.long 0,1,2,3 -.long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c -.byte 67,104,97,67,104,97,50,48,32,102,111,114,32,80,111,119,101,114,80,67,47,65,108,116,105,86,101,99,44,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 -.align 2 -.align 2 diff --git a/openssl/src/crypto/chacha/gen/linux_riscv64/chacha-riscv64-zbb-zvkb.s b/openssl/src/crypto/chacha/gen/linux_riscv64/chacha-riscv64-zbb-zvkb.s deleted file mode 100644 index 2e38d893a..000000000 --- a/openssl/src/crypto/chacha/gen/linux_riscv64/chacha-riscv64-zbb-zvkb.s +++ /dev/null @@ -1,444 +0,0 @@ -.text -.p2align 3 -.globl ChaCha20_ctr32_zbb_zvkb -.type ChaCha20_ctr32_zbb_zvkb,@function -ChaCha20_ctr32_zbb_zvkb: - addi sp, sp, -96 - sd s0, 0(sp) - sd s1, 8(sp) - sd s2, 16(sp) - sd s3, 24(sp) - sd s4, 32(sp) - sd s5, 40(sp) - sd s6, 48(sp) - sd s7, 56(sp) - sd s8, 64(sp) - sd s9, 72(sp) - sd s10, 80(sp) - sd s11, 88(sp) - addi sp, sp, -64 - - lw t2, 0(a4) - -.Lblock_loop: - # We will use the scalar ALU for 1 chacha block. - srli t3, a2, 6 - .word 219050839 - slli t4, t1, 6 - bltu t4, a2, 1f - # Since there is no more chacha block existed, we need to split 1 block - # from vector ALU. - addi t4, t1, -1 - .word 219083607 -1: - - #### chacha block data - # init chacha const states - # "expa" little endian - li a5, 0x61707865 - .word 1577566295 - # "nd 3" little endian - li a6, 0x3320646e - .word 1577599191 - # "2-by" little endian - li a7, 0x79622d32 - .word 1577632087 - # "te k" little endian - li s0, 0x6b206574 - lw s1, 0(a3) - .word 1577337303 - - # init chacha key states - lw s2, 4(a3) - .word 1577370199 - lw s3, 8(a3) - .word 1577665239 - lw s4, 12(a3) - .word 1577698135 - lw s5, 16(a3) - .word 1577731031 - lw s6, 20(a3) - .word 1577763927 - lw s7, 24(a3) - .word 1577796823 - lw s8, 28(a3) - .word 1577829719 - .word 1577862615 - - # init chacha key states - lw s10, 4(a4) - .word 1376298583 - lw s11, 8(a4) - .word 46384727 - lw t0, 12(a4) - .word 1577928407 - add s9, t2, t1 - - # init chacha nonce states - .word 1577961303 - .word 1577240535 - - li t3, 64 - # load the top-half of input data - .word 3955615751 - - # 20 round groups - li t3, 10 -.Lround_loop: - addi t3, t3, -1 - # a += b; d ^= a; d <<<= 16; - .word 33685591 - add a5, a5, s1 - .word 34767063 - add a6, a6, s2 - .word 35848535 - add a7, a7, s3 - .word 36930007 - add s0, s0, s4 - .word 784336471 - xor s9, s9, a5 - .word 785417943 - xor s10, s10, a6 - .word 786499415 - xor s11, s11, a7 - .word 787580887 - xor t0, t0, s0 - .word 1388852823 - .word 1628232859 - .word 1389901527 - .word 1628265755 - .word 1390950231 - .word 1628298651 - .word 1391998935 - .word 1627574939 - # c += d; b ^= c; b <<<= 12; - .word 42337367 - add s5, s5, s9 - .word 43418839 - add s6, s6, s10 - .word 44500311 - add s7, s7, s11 - .word 45581783 - add s8, s8, t0 - .word 776208983 - xor s1, s1, s5 - .word 777290455 - xor s2, s2, s6 - .word 778371927 - xor s3, s3, s7 - .word 779453399 - xor s4, s4, s8 - .word 1380594263 - .word 1631900827 - .word 1381642967 - .word 1632196891 - .word 1382691671 - .word 1632229787 - .word 1383740375 - .word 1632262683 - # a += b; d ^= a; d <<<= 8; - .word 33685591 - add a5, a5, s1 - .word 34767063 - add a6, a6, s2 - .word 35848535 - add a7, a7, s3 - .word 36930007 - add s0, s0, s4 - .word 784336471 - xor s9, s9, a5 - .word 785417943 - xor s10, s10, a6 - .word 786499415 - xor s11, s11, a7 - .word 787580887 - xor t0, t0, s0 - .word 1389114967 - .word 1636621467 - .word 1390163671 - .word 1636654363 - .word 1391212375 - .word 1636687259 - .word 1392261079 - .word 1635963547 - # c += d; b ^= c; b <<<= 7; - .word 42337367 - add s5, s5, s9 - .word 43418839 - add s6, s6, s10 - .word 44500311 - add s7, s7, s11 - .word 45581783 - add s8, s8, t0 - .word 776208983 - xor s1, s1, s5 - .word 777290455 - xor s2, s2, s6 - .word 778371927 - xor s3, s3, s7 - .word 779453399 - xor s4, s4, s8 - .word 1380758103 - .word 1637143707 - .word 1381806807 - .word 1637439771 - .word 1382855511 - .word 1637472667 - .word 1383904215 - .word 1637505563 - - # a += b; d ^= a; d <<<= 16; - .word 36831703 - add s0, s0, s1 - .word 33718359 - add a5, a5, s2 - .word 34799831 - add a6, a6, s3 - .word 35881303 - add a7, a7, s4 - .word 786532183 - xor s11, s11, s0 - .word 787482583 - xor t0, t0, a5 - .word 784369239 - xor s9, s9, a6 - .word 785450711 - xor s10, s10, a7 - .word 1390950231 - .word 1628298651 - .word 1391998935 - .word 1627574939 - .word 1388852823 - .word 1628232859 - .word 1389901527 - .word 1628265755 - # c += d; b ^= c; b <<<= 12; - .word 43451607 - add s6, s6, s11 - .word 44533079 - add s7, s7, t0 - .word 45483479 - add s8, s8, s9 - .word 42370135 - add s5, s5, s10 - .word 776241751 - xor s1, s1, s6 - .word 777323223 - xor s2, s2, s7 - .word 778404695 - xor s3, s3, s8 - .word 779355095 - xor s4, s4, s5 - .word 1380594263 - .word 1631900827 - .word 1381642967 - .word 1632196891 - .word 1382691671 - .word 1632229787 - .word 1383740375 - .word 1632262683 - # a += b; d ^= a; d <<<= 8; - .word 36831703 - add s0, s0, s1 - .word 33718359 - add a5, a5, s2 - .word 34799831 - add a6, a6, s3 - .word 35881303 - add a7, a7, s4 - .word 786532183 - xor s11, s11, s0 - .word 787482583 - xor t0, t0, a5 - .word 784369239 - xor s9, s9, a6 - .word 785450711 - xor s10, s10, a7 - .word 1391212375 - .word 1636687259 - .word 1392261079 - .word 1635963547 - .word 1389114967 - .word 1636621467 - .word 1390163671 - .word 1636654363 - # c += d; b ^= c; b <<<= 7; - .word 43451607 - add s6, s6, s11 - .word 44533079 - add s7, s7, t0 - .word 45483479 - add s8, s8, s9 - .word 42370135 - add s5, s5, s10 - .word 776241751 - xor s1, s1, s6 - .word 777323223 - xor s2, s2, s7 - .word 778404695 - xor s3, s3, s8 - .word 779355095 - xor s4, s4, s5 - .word 1380758103 - .word 1637143707 - .word 1381806807 - .word 1637439771 - .word 1382855511 - .word 1637472667 - .word 1383904215 - .word 1637505563 - - bnez t3, .Lround_loop - - li t3, 64 - # load the bottom-half of input data - addi t4, a1, 32 - .word 3956206599 - - # add chacha top-half initial block states - # "expa" little endian - li t3, 0x61707865 - .word 34488407 - add a5, a5, t3 - # "nd 3" little endian - li t4, 0x3320646e - .word 35569879 - add a6, a6, t4 - lw t3, 0(a3) - # "2-by" little endian - li t5, 0x79622d32 - .word 36651351 - add a7, a7, t5 - lw t4, 4(a3) - # "te k" little endian - li t6, 0x6b206574 - .word 37732823 - add s0, s0, t6 - lw t5, 8(a3) - .word 38683223 - add s1, s1, t3 - lw t6, 12(a3) - .word 39764695 - add s2, s2, t4 - .word 40846167 - add s3, s3, t5 - .word 41927639 - add s4, s4, t6 - - # xor with the top-half input - .word 788531287 - sw a5, 0(sp) - sw a6, 4(sp) - .word 789612759 - sw a7, 8(sp) - sw s0, 12(sp) - .word 790694231 - sw s1, 16(sp) - sw s2, 20(sp) - .word 791775703 - sw s3, 24(sp) - sw s4, 28(sp) - .word 792857175 - lw t3, 16(a3) - .word 793938647 - lw t4, 20(a3) - .word 795020119 - lw t5, 24(a3) - .word 796101591 - - # save the top-half of output - li t6, 64 - .word 3958728743 - - # add chacha bottom-half initial block states - .word 42878039 - add s5, s5, t3 - lw t6, 28(a3) - .word 43959511 - add s6, s6, t4 - lw t3, 4(a4) - .word 45040983 - add s7, s7, t5 - lw t4, 8(a4) - .word 46122455 - add s8, s8, t6 - lw t5, 12(a4) - .word 1376297047 - add s9, s9, t2 - .word 46384727 - add s9, s9, t1 - .word 48121559 - add s10, s10, t3 - .word 49203031 - add s11, s11, t4 - .word 50284503 - add t0, t0, t5 - .word 46138967 - # xor with the bottom-half input - .word 797183063 - sw s5, 32(sp) - .word 798264535 - sw s6, 36(sp) - .word 799346007 - sw s7, 40(sp) - .word 800427479 - sw s8, 44(sp) - .word 802590423 - sw s9, 48(sp) - .word 801508951 - sw s10, 52(sp) - .word 803671895 - sw s11, 56(sp) - .word 804753367 - sw t0, 60(sp) - - # save the bottom-half of output - li t3, 64 - addi t4, a0, 32 - .word 3956206631 - - # the computed vector parts: `64 * VL` - slli t3, t1, 6 - - add a1, a1, t3 - add a0, a0, t3 - sub a2, a2, t3 - add t2, t2, t1 - - # process the scalar data block - addi t2, t2, 1 - li t3, 64 - .word 197549747 - sub a2, a2, t4 - mv t5, sp -.Lscalar_data_loop: - .word 205452119 - .word 33915911 - .word 34539527 - .word 780665943 - .word 33883175 - add a1, a1, t1 - add a0, a0, t1 - add t5, t5, t1 - sub t4, t4, t1 - bnez t4, .Lscalar_data_loop - - bnez a2, .Lblock_loop - - addi sp, sp, 64 - ld s0, 0(sp) - ld s1, 8(sp) - ld s2, 16(sp) - ld s3, 24(sp) - ld s4, 32(sp) - ld s5, 40(sp) - ld s6, 48(sp) - ld s7, 56(sp) - ld s8, 64(sp) - ld s9, 72(sp) - ld s10, 80(sp) - ld s11, 88(sp) - addi sp, sp, 96 - - ret -.size ChaCha20_ctr32_zbb_zvkb,.-ChaCha20_ctr32_zbb_zvkb diff --git a/openssl/src/crypto/chacha/gen/windows_ia32/chacha-x86.asm b/openssl/src/crypto/chacha/gen/windows_ia32/chacha-x86.asm index 13aba2160..16ede5f93 100644 --- a/openssl/src/crypto/chacha/gen/windows_ia32/chacha-x86.asm +++ b/openssl/src/crypto/chacha/gen/windows_ia32/chacha-x86.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/cmac/cmac.c b/openssl/src/crypto/cmac/cmac.c index 2012774f8..218eb9425 100644 --- a/openssl/src/crypto/cmac/cmac.c +++ b/openssl/src/crypto/cmac/cmac.c @@ -1,5 +1,5 @@ /* - * Copyright 2010-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2010-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,7 +20,6 @@ #include #include -#define LOCAL_BUF_SIZE 2048 struct CMAC_CTX_st { /* Cipher context to use */ EVP_CIPHER_CTX *cctx; @@ -54,8 +53,10 @@ CMAC_CTX *CMAC_CTX_new(void) { CMAC_CTX *ctx; - if ((ctx = OPENSSL_malloc(sizeof(*ctx))) == NULL) + if ((ctx = OPENSSL_malloc(sizeof(*ctx))) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->cctx = EVP_CIPHER_CTX_new(); if (ctx->cctx == NULL) { OPENSSL_free(ctx); @@ -95,7 +96,7 @@ int CMAC_CTX_copy(CMAC_CTX *out, const CMAC_CTX *in) if (in->nlast_block == -1) return 0; - if ((bl = EVP_CIPHER_CTX_get_block_size(in->cctx)) == 0) + if ((bl = EVP_CIPHER_CTX_get_block_size(in->cctx)) < 0) return 0; if (!EVP_CIPHER_CTX_copy(out->cctx, in->cctx)) return 0; @@ -111,7 +112,6 @@ int CMAC_Init(CMAC_CTX *ctx, const void *key, size_t keylen, const EVP_CIPHER *cipher, ENGINE *impl) { static const unsigned char zero_iv[EVP_MAX_BLOCK_LENGTH] = { 0 }; - int block_len; /* All zeros means restart */ if (!key && !cipher && !impl && keylen == 0) { @@ -120,10 +120,7 @@ int CMAC_Init(CMAC_CTX *ctx, const void *key, size_t keylen, return 0; if (!EVP_EncryptInit_ex(ctx->cctx, NULL, NULL, NULL, zero_iv)) return 0; - block_len = EVP_CIPHER_CTX_get_block_size(ctx->cctx); - if (block_len == 0) - return 0; - memset(ctx->tbl, 0, block_len); + memset(ctx->tbl, 0, EVP_CIPHER_CTX_get_block_size(ctx->cctx)); ctx->nlast_block = 0; return 1; } @@ -140,9 +137,9 @@ int CMAC_Init(CMAC_CTX *ctx, const void *key, size_t keylen, /* If anything fails then ensure we can't use this ctx */ ctx->nlast_block = -1; - if (EVP_CIPHER_CTX_get0_cipher(ctx->cctx) == NULL) + if (!EVP_CIPHER_CTX_get0_cipher(ctx->cctx)) return 0; - if (EVP_CIPHER_CTX_set_key_length(ctx->cctx, keylen) <= 0) + if (!EVP_CIPHER_CTX_set_key_length(ctx->cctx, keylen)) return 0; if (!EVP_EncryptInit_ex(ctx->cctx, NULL, NULL, key, zero_iv)) return 0; @@ -167,14 +164,12 @@ int CMAC_Update(CMAC_CTX *ctx, const void *in, size_t dlen) { const unsigned char *data = in; int bl; - size_t max_burst_blocks, cipher_blocks; - unsigned char buf[LOCAL_BUF_SIZE]; if (ctx->nlast_block == -1) return 0; if (dlen == 0) return 1; - if ((bl = EVP_CIPHER_CTX_get_block_size(ctx->cctx)) == 0) + if ((bl = EVP_CIPHER_CTX_get_block_size(ctx->cctx)) < 0) return 0; /* Copy into partial block if we need to */ if (ctx->nlast_block > 0) { @@ -195,35 +190,11 @@ int CMAC_Update(CMAC_CTX *ctx, const void *in, size_t dlen) return 0; } /* Encrypt all but one of the complete blocks left */ - - max_burst_blocks = LOCAL_BUF_SIZE / bl; - cipher_blocks = (dlen - 1) / bl; - if (max_burst_blocks == 0) { - /* - * When block length is greater than local buffer size, - * use ctx->tbl as cipher output. - */ - while (dlen > (size_t)bl) { - if (EVP_Cipher(ctx->cctx, ctx->tbl, data, bl) <= 0) - return 0; - dlen -= bl; - data += bl; - } - } else { - while (cipher_blocks > max_burst_blocks) { - if (EVP_Cipher(ctx->cctx, buf, data, max_burst_blocks * bl) <= 0) - return 0; - dlen -= max_burst_blocks * bl; - data += max_burst_blocks * bl; - cipher_blocks -= max_burst_blocks; - } - if (cipher_blocks > 0) { - if (EVP_Cipher(ctx->cctx, buf, data, cipher_blocks * bl) <= 0) - return 0; - dlen -= cipher_blocks * bl; - data += cipher_blocks * bl; - memcpy(ctx->tbl, &buf[(cipher_blocks - 1) * bl], bl); - } + while (dlen > (size_t)bl) { + if (EVP_Cipher(ctx->cctx, ctx->tbl, data, bl) <= 0) + return 0; + dlen -= bl; + data += bl; } /* Copy any data left to last block buffer */ memcpy(ctx->last_block, data, dlen); @@ -238,7 +209,7 @@ int CMAC_Final(CMAC_CTX *ctx, unsigned char *out, size_t *poutlen) if (ctx->nlast_block == -1) return 0; - if ((bl = EVP_CIPHER_CTX_get_block_size(ctx->cctx)) == 0) + if ((bl = EVP_CIPHER_CTX_get_block_size(ctx->cctx)) < 0) return 0; if (poutlen != NULL) *poutlen = (size_t)bl; diff --git a/openssl/src/crypto/cmp/cmp_asn.c b/openssl/src/crypto/cmp/cmp_asn.c index 3285cbf42..0ca107554 100644 --- a/openssl/src/crypto/cmp/cmp_asn.c +++ b/openssl/src/crypto/cmp/cmp_asn.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -28,6 +28,7 @@ ASN1_SEQUENCE(OSSL_CMP_REVANNCONTENT) = { } ASN1_SEQUENCE_END(OSSL_CMP_REVANNCONTENT) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_REVANNCONTENT) + ASN1_SEQUENCE(OSSL_CMP_CHALLENGE) = { ASN1_OPT(OSSL_CMP_CHALLENGE, owf, X509_ALGOR), ASN1_SIMPLE(OSSL_CMP_CHALLENGE, witness, ASN1_OCTET_STRING), @@ -35,16 +36,19 @@ ASN1_SEQUENCE(OSSL_CMP_CHALLENGE) = { } ASN1_SEQUENCE_END(OSSL_CMP_CHALLENGE) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_CHALLENGE) + ASN1_ITEM_TEMPLATE(OSSL_CMP_POPODECKEYCHALLCONTENT) = ASN1_EX_TEMPLATE_TYPE(ASN1_TFLG_SEQUENCE_OF, 0, OSSL_CMP_POPODECKEYCHALLCONTENT, OSSL_CMP_CHALLENGE) ASN1_ITEM_TEMPLATE_END(OSSL_CMP_POPODECKEYCHALLCONTENT) + ASN1_ITEM_TEMPLATE(OSSL_CMP_POPODECKEYRESPCONTENT) = ASN1_EX_TEMPLATE_TYPE(ASN1_TFLG_SEQUENCE_OF, 0, OSSL_CMP_POPODECKEYRESPCONTENT, ASN1_INTEGER) ASN1_ITEM_TEMPLATE_END(OSSL_CMP_POPODECKEYRESPCONTENT) + ASN1_SEQUENCE(OSSL_CMP_CAKEYUPDANNCONTENT) = { /* OSSL_CMP_CMPCERTIFICATE is effectively X509 so it is used directly */ ASN1_SIMPLE(OSSL_CMP_CAKEYUPDANNCONTENT, oldWithNew, X509), @@ -55,10 +59,15 @@ ASN1_SEQUENCE(OSSL_CMP_CAKEYUPDANNCONTENT) = { } ASN1_SEQUENCE_END(OSSL_CMP_CAKEYUPDANNCONTENT) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_CAKEYUPDANNCONTENT) + ASN1_SEQUENCE(OSSL_CMP_ERRORMSGCONTENT) = { ASN1_SIMPLE(OSSL_CMP_ERRORMSGCONTENT, pKIStatusInfo, OSSL_CMP_PKISI), ASN1_OPT(OSSL_CMP_ERRORMSGCONTENT, errorCode, ASN1_INTEGER), - /* OSSL_CMP_PKIFREETEXT is a ASN1_UTF8STRING sequence, so used directly */ + /* + * OSSL_CMP_PKIFREETEXT is effectively a sequence of ASN1_UTF8STRING + * so it is used directly + * + */ ASN1_SEQUENCE_OF_OPT(OSSL_CMP_ERRORMSGCONTENT, errorDetails, ASN1_UTF8STRING) } ASN1_SEQUENCE_END(OSSL_CMP_ERRORMSGCONTENT) @@ -110,19 +119,10 @@ ASN1_ADB(OSSL_CMP_ITAV) = { ADB_ENTRY(NID_id_it_suppLangTags, ASN1_SEQUENCE_OF_OPT(OSSL_CMP_ITAV, infoValue.suppLangTagsValue, ASN1_UTF8STRING)), - ADB_ENTRY(NID_id_it_caCerts, - ASN1_SEQUENCE_OF_OPT(OSSL_CMP_ITAV, infoValue.caCerts, X509)), - ADB_ENTRY(NID_id_it_rootCaCert, - ASN1_OPT(OSSL_CMP_ITAV, infoValue.rootCaCert, X509)), - ADB_ENTRY(NID_id_it_rootCaKeyUpdate, - ASN1_OPT(OSSL_CMP_ITAV, infoValue.rootCaKeyUpdate, - OSSL_CMP_ROOTCAKEYUPDATE)), - ADB_ENTRY(NID_id_it_certProfile, - ASN1_SEQUENCE_OF_OPT(OSSL_CMP_ITAV, infoValue.certProfile, - ASN1_UTF8STRING)), } ASN1_ADB_END(OSSL_CMP_ITAV, 0, infoType, 0, &infotypeandvalue_default_tt, NULL); + ASN1_SEQUENCE(OSSL_CMP_ITAV) = { ASN1_SIMPLE(OSSL_CMP_ITAV, infoType, ASN1_OBJECT), ASN1_ADB_OBJECT(OSSL_CMP_ITAV) @@ -130,14 +130,6 @@ ASN1_SEQUENCE(OSSL_CMP_ITAV) = { IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_ITAV) IMPLEMENT_ASN1_DUP_FUNCTION(OSSL_CMP_ITAV) -ASN1_SEQUENCE(OSSL_CMP_ROOTCAKEYUPDATE) = { - /* OSSL_CMP_CMPCERTIFICATE is effectively X509 so it is used directly */ - ASN1_SIMPLE(OSSL_CMP_ROOTCAKEYUPDATE, newWithNew, X509), - ASN1_EXP_OPT(OSSL_CMP_ROOTCAKEYUPDATE, newWithOld, X509, 0), - ASN1_EXP_OPT(OSSL_CMP_ROOTCAKEYUPDATE, oldWithNew, X509, 1) -} ASN1_SEQUENCE_END(OSSL_CMP_ROOTCAKEYUPDATE) -IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_ROOTCAKEYUPDATE) - OSSL_CMP_ITAV *OSSL_CMP_ITAV_create(ASN1_OBJECT *type, ASN1_TYPE *value) { OSSL_CMP_ITAV *itav; @@ -189,178 +181,35 @@ int OSSL_CMP_ITAV_push0_stack_item(STACK_OF(OSSL_CMP_ITAV) **itav_sk_p, return 1; err: - if (created) { + if (created != 0) { sk_OSSL_CMP_ITAV_free(*itav_sk_p); *itav_sk_p = NULL; } return 0; } -OSSL_CMP_ITAV -*OSSL_CMP_ITAV_new0_certProfile(STACK_OF(ASN1_UTF8STRING) *certProfile) -{ - OSSL_CMP_ITAV *itav; - - if ((itav = OSSL_CMP_ITAV_new()) == NULL) - return NULL; - itav->infoType = OBJ_nid2obj(NID_id_it_certProfile); - itav->infoValue.certProfile = certProfile; - return itav; -} - -int OSSL_CMP_ITAV_get0_certProfile(const OSSL_CMP_ITAV *itav, - STACK_OF(ASN1_UTF8STRING) **out) -{ - if (itav == NULL || out == NULL) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (OBJ_obj2nid(itav->infoType) != NID_id_it_certProfile) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - *out = itav->infoValue.certProfile; - return 1; -} - -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_caCerts(const STACK_OF(X509) *caCerts) -{ - OSSL_CMP_ITAV *itav = OSSL_CMP_ITAV_new(); - - if (itav == NULL) - return NULL; - if (sk_X509_num(caCerts) > 0 - && (itav->infoValue.caCerts = - sk_X509_deep_copy(caCerts, X509_dup, X509_free)) == NULL) { - OSSL_CMP_ITAV_free(itav); - return NULL; - } - itav->infoType = OBJ_nid2obj(NID_id_it_caCerts); - return itav; -} - -int OSSL_CMP_ITAV_get0_caCerts(const OSSL_CMP_ITAV *itav, STACK_OF(X509) **out) -{ - if (itav == NULL || out == NULL) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (OBJ_obj2nid(itav->infoType) != NID_id_it_caCerts) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - *out = sk_X509_num(itav->infoValue.caCerts) > 0 - ? itav->infoValue.caCerts : NULL; - return 1; -} - -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_rootCaCert(const X509 *rootCaCert) -{ - OSSL_CMP_ITAV *itav = OSSL_CMP_ITAV_new(); - - if (itav == NULL) - return NULL; - if (rootCaCert != NULL - && (itav->infoValue.rootCaCert = X509_dup(rootCaCert)) == NULL) { - OSSL_CMP_ITAV_free(itav); - return NULL; - } - itav->infoType = OBJ_nid2obj(NID_id_it_rootCaCert); - return itav; -} - -int OSSL_CMP_ITAV_get0_rootCaCert(const OSSL_CMP_ITAV *itav, X509 **out) -{ - if (itav == NULL || out == NULL) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (OBJ_obj2nid(itav->infoType) != NID_id_it_rootCaCert) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - *out = itav->infoValue.rootCaCert; - return 1; -} -OSSL_CMP_ITAV *OSSL_CMP_ITAV_new_rootCaKeyUpdate(const X509 *newWithNew, - const X509 *newWithOld, - const X509 *oldWithNew) -{ - OSSL_CMP_ITAV *itav; - OSSL_CMP_ROOTCAKEYUPDATE *upd = NULL; - - if (newWithNew != NULL) { - upd = OSSL_CMP_ROOTCAKEYUPDATE_new(); - if (upd == NULL) - return NULL; - - if ((upd->newWithNew = X509_dup(newWithNew)) == NULL) - goto err; - if (newWithOld != NULL - && (upd->newWithOld = X509_dup(newWithOld)) == NULL) - goto err; - if (oldWithNew != NULL - && (upd->oldWithNew = X509_dup(oldWithNew)) == NULL) - goto err; - } - - if ((itav = OSSL_CMP_ITAV_new()) == NULL) - goto err; - itav->infoType = OBJ_nid2obj(NID_id_it_rootCaKeyUpdate); - itav->infoValue.rootCaKeyUpdate = upd; - return itav; - - err: - OSSL_CMP_ROOTCAKEYUPDATE_free(upd); - return NULL; -} - -int OSSL_CMP_ITAV_get0_rootCaKeyUpdate(const OSSL_CMP_ITAV *itav, - X509 **newWithNew, - X509 **newWithOld, - X509 **oldWithNew) -{ - OSSL_CMP_ROOTCAKEYUPDATE *upd; - - if (itav == NULL || newWithNew == NULL) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (OBJ_obj2nid(itav->infoType) != NID_id_it_rootCaKeyUpdate) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - upd = itav->infoValue.rootCaKeyUpdate; - *newWithNew = upd != NULL ? upd->newWithNew : NULL; - if (newWithOld != NULL) - *newWithOld = upd != NULL ? upd->newWithOld : NULL; - if (oldWithNew != NULL) - *oldWithNew = upd != NULL ? upd->oldWithNew : NULL; - return 1; -} - -/* get ASN.1 encoded integer, return -2 on error; -1 is valid for certReqId */ +/* get ASN.1 encoded integer, return -1 on error */ int ossl_cmp_asn1_get_int(const ASN1_INTEGER *a) { int64_t res; if (!ASN1_INTEGER_get_int64(&res, a)) { ERR_raise(ERR_LIB_CMP, ASN1_R_INVALID_NUMBER); - return -2; + return -1; } if (res < INT_MIN) { ERR_raise(ERR_LIB_CMP, ASN1_R_TOO_SMALL); - return -2; + return -1; } if (res > INT_MAX) { ERR_raise(ERR_LIB_CMP, ASN1_R_TOO_LARGE); - return -2; + return -1; } return (int)res; } static int ossl_cmp_msg_cb(int operation, ASN1_VALUE **pval, - ossl_unused const ASN1_ITEM *it, void *exarg) + const ASN1_ITEM *it, void *exarg) { OSSL_CMP_MSG *msg = (OSSL_CMP_MSG *)*pval; @@ -406,6 +255,7 @@ ASN1_CHOICE(OSSL_CMP_CERTORENCCERT) = { } ASN1_CHOICE_END(OSSL_CMP_CERTORENCCERT) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_CERTORENCCERT) + ASN1_SEQUENCE(OSSL_CMP_CERTIFIEDKEYPAIR) = { ASN1_SIMPLE(OSSL_CMP_CERTIFIEDKEYPAIR, certOrEncCert, OSSL_CMP_CERTORENCCERT), @@ -416,17 +266,20 @@ ASN1_SEQUENCE(OSSL_CMP_CERTIFIEDKEYPAIR) = { } ASN1_SEQUENCE_END(OSSL_CMP_CERTIFIEDKEYPAIR) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_CERTIFIEDKEYPAIR) + ASN1_SEQUENCE(OSSL_CMP_REVDETAILS) = { ASN1_SIMPLE(OSSL_CMP_REVDETAILS, certDetails, OSSL_CRMF_CERTTEMPLATE), ASN1_OPT(OSSL_CMP_REVDETAILS, crlEntryDetails, X509_EXTENSIONS) } ASN1_SEQUENCE_END(OSSL_CMP_REVDETAILS) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_REVDETAILS) + ASN1_ITEM_TEMPLATE(OSSL_CMP_REVREQCONTENT) = ASN1_EX_TEMPLATE_TYPE(ASN1_TFLG_SEQUENCE_OF, 0, OSSL_CMP_REVREQCONTENT, OSSL_CMP_REVDETAILS) ASN1_ITEM_TEMPLATE_END(OSSL_CMP_REVREQCONTENT) + ASN1_SEQUENCE(OSSL_CMP_REVREPCONTENT) = { ASN1_SEQUENCE_OF(OSSL_CMP_REVREPCONTENT, status, OSSL_CMP_PKISI), ASN1_EXP_SEQUENCE_OF_OPT(OSSL_CMP_REVREPCONTENT, revCerts, OSSL_CRMF_CERTID, @@ -435,6 +288,7 @@ ASN1_SEQUENCE(OSSL_CMP_REVREPCONTENT) = { } ASN1_SEQUENCE_END(OSSL_CMP_REVREPCONTENT) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_REVREPCONTENT) + ASN1_SEQUENCE(OSSL_CMP_KEYRECREPCONTENT) = { ASN1_SIMPLE(OSSL_CMP_KEYRECREPCONTENT, status, OSSL_CMP_PKISI), ASN1_EXP_OPT(OSSL_CMP_KEYRECREPCONTENT, newSigCert, X509, 0), @@ -444,15 +298,21 @@ ASN1_SEQUENCE(OSSL_CMP_KEYRECREPCONTENT) = { } ASN1_SEQUENCE_END(OSSL_CMP_KEYRECREPCONTENT) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_KEYRECREPCONTENT) + ASN1_ITEM_TEMPLATE(OSSL_CMP_PKISTATUS) = ASN1_EX_TEMPLATE_TYPE(ASN1_TFLG_UNIVERSAL, 0, status, ASN1_INTEGER) ASN1_ITEM_TEMPLATE_END(OSSL_CMP_PKISTATUS) ASN1_SEQUENCE(OSSL_CMP_PKISI) = { ASN1_SIMPLE(OSSL_CMP_PKISI, status, OSSL_CMP_PKISTATUS), - /* OSSL_CMP_PKIFREETEXT is a ASN1_UTF8STRING sequence, so used directly */ + /* + * CMP_PKIFREETEXT is effectively a sequence of ASN1_UTF8STRING + * so it is used directly + */ ASN1_SEQUENCE_OF_OPT(OSSL_CMP_PKISI, statusString, ASN1_UTF8STRING), - /* OSSL_CMP_PKIFAILUREINFO is effectively ASN1_BIT_STRING, used directly */ + /* + * OSSL_CMP_PKIFAILUREINFO is effectively ASN1_BIT_STRING so used directly + */ ASN1_OPT(OSSL_CMP_PKISI, failInfo, ASN1_BIT_STRING) } ASN1_SEQUENCE_END(OSSL_CMP_PKISI) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_PKISI) @@ -461,8 +321,7 @@ IMPLEMENT_ASN1_DUP_FUNCTION(OSSL_CMP_PKISI) ASN1_SEQUENCE(OSSL_CMP_CERTSTATUS) = { ASN1_SIMPLE(OSSL_CMP_CERTSTATUS, certHash, ASN1_OCTET_STRING), ASN1_SIMPLE(OSSL_CMP_CERTSTATUS, certReqId, ASN1_INTEGER), - ASN1_OPT(OSSL_CMP_CERTSTATUS, statusInfo, OSSL_CMP_PKISI), - ASN1_EXP_OPT(OSSL_CMP_CERTSTATUS, hashAlg, X509_ALGOR, 0) + ASN1_OPT(OSSL_CMP_CERTSTATUS, statusInfo, OSSL_CMP_PKISI) } ASN1_SEQUENCE_END(OSSL_CMP_CERTSTATUS) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CMP_CERTSTATUS) @@ -569,7 +428,10 @@ ASN1_SEQUENCE(OSSL_CMP_PKIHEADER) = { ASN1_EXP_OPT(OSSL_CMP_PKIHEADER, transactionID, ASN1_OCTET_STRING, 4), ASN1_EXP_OPT(OSSL_CMP_PKIHEADER, senderNonce, ASN1_OCTET_STRING, 5), ASN1_EXP_OPT(OSSL_CMP_PKIHEADER, recipNonce, ASN1_OCTET_STRING, 6), - /* OSSL_CMP_PKIFREETEXT is a ASN1_UTF8STRING sequence, so used directly */ + /* + * OSSL_CMP_PKIFREETEXT is effectively a sequence of ASN1_UTF8STRING + * so it is used directly + */ ASN1_EXP_SEQUENCE_OF_OPT(OSSL_CMP_PKIHEADER, freeText, ASN1_UTF8STRING, 7), ASN1_EXP_SEQUENCE_OF_OPT(OSSL_CMP_PKIHEADER, generalInfo, OSSL_CMP_ITAV, 8) diff --git a/openssl/src/crypto/cmp/cmp_client.c b/openssl/src/crypto/cmp/cmp_client.c index d588bb358..e10002106 100644 --- a/openssl/src/crypto/cmp/cmp_client.c +++ b/openssl/src/crypto/cmp/cmp_client.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -11,6 +11,7 @@ #include "cmp_local.h" #include "internal/cryptlib.h" +#include "internal/e_os.h" /* ossl_sleep() */ /* explicit #includes not strictly needed since implied by the above: */ #include @@ -31,7 +32,7 @@ static int unprotected_exception(const OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *rep, int invalid_protection, - ossl_unused int expected_type) + int expected_type /* ignored here */) { int rcvd_type = OSSL_CMP_MSG_get_bodytype(rep /* may be NULL */); const char *msg_type = NULL; @@ -63,10 +64,10 @@ static int unprotected_exception(const OSSL_CMP_CTX *ctx, break; default: if (IS_CREP(rcvd_type)) { - int any_rid = OSSL_CMP_CERTREQID_NONE; OSSL_CMP_CERTREPMESSAGE *crepmsg = rep->body->value.ip; OSSL_CMP_CERTRESPONSE *crep = - ossl_cmp_certrepmessage_get0_certresponse(crepmsg, any_rid); + ossl_cmp_certrepmessage_get0_certresponse(crepmsg, + -1 /* any rid */); if (sk_OSSL_CMP_CERTRESPONSE_num(crepmsg->response) > 1) return -1; @@ -93,11 +94,16 @@ static int save_statusInfo(OSSL_CMP_CTX *ctx, OSSL_CMP_PKISI *si) if (!ossl_assert(ctx != NULL && si != NULL)) return 0; - ctx->status = ossl_cmp_pkisi_get_status(si); - if (ctx->status < OSSL_CMP_PKISTATUS_accepted) + if ((ctx->status = ossl_cmp_pkisi_get_status(si)) < 0) return 0; - ctx->failInfoCode = ossl_cmp_pkisi_get_pkifailureinfo(si); + ctx->failInfoCode = 0; + if (si->failInfo != NULL) { + for (i = 0; i <= OSSL_CMP_PKIFAILUREINFO_MAX; i++) { + if (ASN1_BIT_STRING_get_bit(si->failInfo, i)) + ctx->failInfoCode |= (1 << i); + } + } if (!ossl_cmp_ctx_set0_statusString(ctx, sk_ASN1_UTF8STRING_new_null()) || (ctx->statusString == NULL)) @@ -113,23 +119,6 @@ static int save_statusInfo(OSSL_CMP_CTX *ctx, OSSL_CMP_PKISI *si) return 1; } -static int is_crep_with_waiting(const OSSL_CMP_MSG *resp, int rid) -{ - OSSL_CMP_CERTREPMESSAGE *crepmsg; - OSSL_CMP_CERTRESPONSE *crep; - int bt = OSSL_CMP_MSG_get_bodytype(resp); - - if (!IS_CREP(bt)) - return 0; - - crepmsg = resp->body->value.ip; /* same for cp and kup */ - crep = ossl_cmp_certrepmessage_get0_certresponse(crepmsg, rid); - - return (crep != NULL - && ossl_cmp_pkisi_get_status(crep->status) - == OSSL_CMP_PKISTATUS_waiting); -} - /*- * Perform the generic aspects of sending a request and receiving a response. * Returns 1 on success and provides the received PKIMESSAGE in *rep. @@ -139,32 +128,29 @@ static int is_crep_with_waiting(const OSSL_CMP_MSG *resp, int rid) static int send_receive_check(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req, OSSL_CMP_MSG **rep, int expected_type) { - int begin_transaction = - expected_type != OSSL_CMP_PKIBODY_POLLREP - && expected_type != OSSL_CMP_PKIBODY_PKICONF; + int is_enrollment = IS_CREP(expected_type) + || expected_type == OSSL_CMP_PKIBODY_POLLREP + || expected_type == OSSL_CMP_PKIBODY_PKICONF; const char *req_type_str = ossl_cmp_bodytype_to_string(OSSL_CMP_MSG_get_bodytype(req)); const char *expected_type_str = ossl_cmp_bodytype_to_string(expected_type); - int bak_msg_timeout = ctx->msg_timeout; + int msg_timeout; int bt; time_t now = time(NULL); int time_left; OSSL_CMP_transfer_cb_t transfer_cb = ctx->transfer_cb; -#ifndef OPENSSL_NO_HTTP if (transfer_cb == NULL) transfer_cb = OSSL_CMP_MSG_http_perform; -#endif - *rep = NULL; - if (ctx->total_timeout != 0 /* not waiting indefinitely */) { - if (begin_transaction) - ctx->end_time = now + ctx->total_timeout; + *rep = NULL; + msg_timeout = ctx->msg_timeout; /* backup original value */ + if (is_enrollment && ctx->total_timeout > 0 /* timeout is not infinite */) { if (now >= ctx->end_time) { ERR_raise(ERR_LIB_CMP, CMP_R_TOTAL_TIMEOUT); return 0; } - if (!ossl_assert(ctx->end_time - now < INT_MAX)) { + if (!ossl_assert(ctx->end_time - time(NULL) < INT_MAX)) { /* actually cannot happen due to assignment in initial_certreq() */ ERR_raise(ERR_LIB_CMP, CMP_R_INVALID_ARGS); return 0; @@ -177,15 +163,14 @@ static int send_receive_check(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req, /* should print error queue since transfer_cb may call ERR_clear_error() */ OSSL_CMP_CTX_print_errors(ctx); - if (ctx->server != NULL) - ossl_cmp_log1(INFO, ctx, "sending %s", req_type_str); + ossl_cmp_log1(INFO, ctx, "sending %s", req_type_str); *rep = (*transfer_cb)(ctx, req); - ctx->msg_timeout = bak_msg_timeout; + ctx->msg_timeout = msg_timeout; /* restore original value */ if (*rep == NULL) { ERR_raise_data(ERR_LIB_CMP, - ctx->total_timeout != 0 && time(NULL) >= ctx->end_time ? + ctx->total_timeout > 0 && time(NULL) >= ctx->end_time ? CMP_R_TOTAL_TIMEOUT : CMP_R_TRANSFER_ERROR, "request sent: %s, expected response: %s", req_type_str, expected_type_str); @@ -198,8 +183,7 @@ static int send_receive_check(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req, * Still we use this preliminary value already for a progress report because * the following msg verification may also produce log entries and may fail. */ - ossl_cmp_log2(INFO, ctx, "received %s%s", ossl_cmp_bodytype_to_string(bt), - ossl_cmp_is_error_with_waiting(*rep) ? " (waiting)" : ""); + ossl_cmp_log1(INFO, ctx, "received %s", ossl_cmp_bodytype_to_string(bt)); /* copy received extraCerts to ctx->extraCertsIn so they can be retrieved */ if (bt != OSSL_CMP_PKIBODY_POLLREP && bt != OSSL_CMP_PKIBODY_PKICONF @@ -210,17 +194,9 @@ static int send_receive_check(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req, expected_type)) return 0; - /* - * rep can have the expected response type, which during polling is pollRep. - * When polling, also any other non-error response (the final response) - * is fine here. When not yet polling, delayed delivery may be initiated - * by the server returning an error message with 'waiting' status (or a - * response message of expected type ip/cp/kup with 'waiting' status). - */ if (bt == expected_type - || (expected_type == OSSL_CMP_PKIBODY_POLLREP - ? bt != OSSL_CMP_PKIBODY_ERROR - : ossl_cmp_is_error_with_waiting(*rep))) + /* as an answer to polling, there could be IP/CP/KUP: */ + || (IS_CREP(bt) && expected_type == OSSL_CMP_PKIBODY_POLLREP)) return 1; /* received message type is not one of the expected ones (e.g., error) */ @@ -262,19 +238,18 @@ static int send_receive_check(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req, /*- * When a 'waiting' PKIStatus has been received, this function is used to - * poll, which should yield a pollRep or the final response. + * poll, which should yield a pollRep or finally a CertRepMessage in ip/cp/kup. * On receiving a pollRep, which includes a checkAfter value, it return this * value if sleep == 0, else it sleeps as long as indicated and retries. * - * A transaction timeout is enabled if ctx->total_timeout is != 0. + * A transaction timeout is enabled if ctx->total_timeout is > 0. * In this case polling will continue until the timeout is reached and then * polling is done a last time even if this is before the "checkAfter" time. * * Returns -1 on receiving pollRep if sleep == 0, setting the checkAfter value. * Returns 1 on success and provides the received PKIMESSAGE in *rep. * In this case the caller is responsible for freeing *rep. - * Returns 0 on error (which includes the cases that timeout has been reached - * or a response with 'waiting' status has been received). + * Returns 0 on error (which includes the case that timeout has been reached). */ static int poll_for_response(OSSL_CMP_CTX *ctx, int sleep, int rid, OSSL_CMP_MSG **rep, int *checkAfter) @@ -339,8 +314,8 @@ static int poll_for_response(OSSL_CMP_CTX *ctx, int sleep, int rid, "received polling response%s; checkAfter = %ld seconds", str, check_after); - if (ctx->total_timeout != 0) { /* timeout is not infinite */ - const int exp = OSSL_CMP_EXPECTED_RESP_TIME; + if (ctx->total_timeout > 0) { /* timeout is not infinite */ + const int exp = 5; /* expected max time per msg round trip */ int64_t time_left = (int64_t)(ctx->end_time - exp - time(NULL)); if (time_left <= 0) { @@ -357,25 +332,15 @@ static int poll_for_response(OSSL_CMP_CTX *ctx, int sleep, int rid, OSSL_CMP_MSG_free(prep); prep = NULL; if (sleep) { - OSSL_sleep((unsigned long)(1000 * check_after)); + ossl_sleep((unsigned long)(1000 * check_after)); } else { if (checkAfter != NULL) *checkAfter = (int)check_after; return -1; /* exits the loop */ } - } else if (is_crep_with_waiting(prep, rid) - || ossl_cmp_is_error_with_waiting(prep)) { - /* received status must not be 'waiting' */ - (void)ossl_cmp_exchange_error(ctx, OSSL_CMP_PKISTATUS_rejection, - OSSL_CMP_CTX_FAILINFO_badRequest, - "polling already started", - 0 /* errorCode */, NULL); - ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKISTATUS); - goto err; } else { - ossl_cmp_info(ctx, "received final response after polling"); - if (!ossl_cmp_ctx_set1_first_senderNonce(ctx, NULL)) - return 0; + ossl_cmp_info(ctx, "received ip/cp/kup after polling"); + /* any other body type has been rejected by send_receive_check() */ break; } } @@ -387,81 +352,24 @@ static int poll_for_response(OSSL_CMP_CTX *ctx, int sleep, int rid, return 1; err: - (void)ossl_cmp_ctx_set1_first_senderNonce(ctx, NULL); OSSL_CMP_MSG_free(preq); OSSL_CMP_MSG_free(prep); return 0; } -static int save_senderNonce_if_waiting(OSSL_CMP_CTX *ctx, - const OSSL_CMP_MSG *rep, int rid) -{ - /* - * Lightweight CMP Profile section 4.4 states: the senderNonce of the - * preceding request message because this value will be needed for checking - * the recipNonce of the final response to be received after polling. - */ - if ((is_crep_with_waiting(rep, rid) - || ossl_cmp_is_error_with_waiting(rep)) - && !ossl_cmp_ctx_set1_first_senderNonce(ctx, ctx->senderNonce)) - return 0; - - return 1; -} - -/* - * Send request and get response possibly with polling initiated by error msg. - * Polling for ip/cp/kup/ with 'waiting' status is handled by cert_response(). - */ -static int send_receive_also_delayed(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req, - OSSL_CMP_MSG **rep, int expected_type) -{ - - if (!send_receive_check(ctx, req, rep, expected_type)) - return 0; - - if (ossl_cmp_is_error_with_waiting(*rep)) { - if (!save_senderNonce_if_waiting(ctx, *rep, OSSL_CMP_CERTREQID_NONE)) - return 0; - /* not modifying ctx->status during certConf and error exchanges */ - if (expected_type != OSSL_CMP_PKIBODY_PKICONF - && !save_statusInfo(ctx, (*rep)->body->value.error->pKIStatusInfo)) - return 0; - - OSSL_CMP_MSG_free(*rep); - *rep = NULL; - - if (poll_for_response(ctx, 1 /* can sleep */, OSSL_CMP_CERTREQID_NONE, - rep, NULL /* checkAfter */) <= 0) { - ERR_raise(ERR_LIB_CMP, CMP_R_POLLING_FAILED); - return 0; - } - } - if (OSSL_CMP_MSG_get_bodytype(*rep) != expected_type) { - ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); - return 0; - } - - return 1; -} -/* - * Send certConf for IR, CR or KUR sequences and check response, - * not modifying ctx->status during the certConf exchange - */ -int ossl_cmp_exchange_certConf(OSSL_CMP_CTX *ctx, int certReqId, - int fail_info, const char *txt) +/* Send certConf for IR, CR or KUR sequences and check response */ +int ossl_cmp_exchange_certConf(OSSL_CMP_CTX *ctx, int fail_info, + const char *txt) { OSSL_CMP_MSG *certConf; OSSL_CMP_MSG *PKIconf = NULL; int res = 0; /* OSSL_CMP_certConf_new() also checks if all necessary options are set */ - certConf = ossl_cmp_certConf_new(ctx, certReqId, fail_info, txt); - if (certConf == NULL) + if ((certConf = ossl_cmp_certConf_new(ctx, fail_info, txt)) == NULL) goto err; - res = send_receive_also_delayed(ctx, certConf, &PKIconf, - OSSL_CMP_PKIBODY_PKICONF); + res = send_receive_check(ctx, certConf, &PKIconf, OSSL_CMP_PKIBODY_PKICONF); err: OSSL_CMP_MSG_free(certConf); @@ -478,15 +386,13 @@ int ossl_cmp_exchange_error(OSSL_CMP_CTX *ctx, int status, int fail_info, OSSL_CMP_MSG *PKIconf = NULL; int res = 0; - /* not overwriting ctx->status on error exchange */ if ((si = OSSL_CMP_STATUSINFO_new(status, fail_info, txt)) == NULL) goto err; /* ossl_cmp_error_new() also checks if all necessary options are set */ if ((error = ossl_cmp_error_new(ctx, si, errorCode, details, 0)) == NULL) goto err; - res = send_receive_also_delayed(ctx, error, - &PKIconf, OSSL_CMP_PKIBODY_PKICONF); + res = send_receive_check(ctx, error, &PKIconf, OSSL_CMP_PKIBODY_PKICONF); err: OSSL_CMP_MSG_free(error); @@ -505,10 +411,12 @@ static X509 *get1_cert_status(OSSL_CMP_CTX *ctx, int bodytype, { char buf[OSSL_CMP_PKISI_BUFLEN]; X509 *crt = NULL; + EVP_PKEY *privkey; if (!ossl_assert(ctx != NULL && crep != NULL)) return NULL; + privkey = OSSL_CMP_CTX_get0_newPkey(ctx, 1); switch (ossl_cmp_pkisi_get_status(crep->status)) { case OSSL_CMP_PKISTATUS_waiting: ossl_cmp_err(ctx, @@ -546,7 +454,7 @@ static X509 *get1_cert_status(OSSL_CMP_CTX *ctx, int bodytype, ERR_raise(ERR_LIB_CMP, CMP_R_UNKNOWN_PKISTATUS); goto err; } - crt = ossl_cmp_certresponse_get1_cert(ctx, crep); + crt = ossl_cmp_certresponse_get1_cert(crep, ctx, privkey); if (crt == NULL) /* according to PKIStatus, we can expect a cert */ ERR_raise(ERR_LIB_CMP, CMP_R_CERTIFICATE_NOT_FOUND); @@ -580,52 +488,23 @@ int OSSL_CMP_certConf_cb(OSSL_CMP_CTX *ctx, X509 *cert, int fail_info, { X509_STORE *out_trusted = OSSL_CMP_CTX_get_certConf_cb_arg(ctx); STACK_OF(X509) *chain = NULL; - (void)text; /* make (artificial) use of var to prevent compiler warning */ if (fail_info != 0) /* accept any error flagged by CMP core library */ return fail_info; - if (out_trusted == NULL) { - ossl_cmp_debug(ctx, "trying to build chain for newly enrolled cert"); - chain = X509_build_chain(cert, ctx->untrusted, out_trusted, - 0, ctx->libctx, ctx->propq); - } else { - X509_STORE_CTX *csc = X509_STORE_CTX_new_ex(ctx->libctx, ctx->propq); - - ossl_cmp_debug(ctx, "validating newly enrolled cert"); - if (csc == NULL) - goto err; - if (!X509_STORE_CTX_init(csc, out_trusted, cert, ctx->untrusted)) - goto err; - /* disable any cert status/revocation checking etc. */ - X509_VERIFY_PARAM_clear_flags(X509_STORE_CTX_get0_param(csc), - ~(X509_V_FLAG_USE_CHECK_TIME - | X509_V_FLAG_NO_CHECK_TIME - | X509_V_FLAG_PARTIAL_CHAIN - | X509_V_FLAG_POLICY_CHECK)); - if (X509_verify_cert(csc) <= 0) - goto err; - - if (!ossl_x509_add_certs_new(&chain, X509_STORE_CTX_get0_chain(csc), - X509_ADD_FLAG_UP_REF | X509_ADD_FLAG_NO_DUP - | X509_ADD_FLAG_NO_SS)) { - sk_X509_free(chain); - chain = NULL; - } - err: - X509_STORE_CTX_free(csc); - } - + ossl_cmp_debug(ctx, "trying to build chain for newly enrolled cert"); + chain = X509_build_chain(cert, ctx->untrusted, out_trusted /* maybe NULL */, + 0, ctx->libctx, ctx->propq); if (sk_X509_num(chain) > 0) X509_free(sk_X509_shift(chain)); /* remove leaf (EE) cert */ if (out_trusted != NULL) { if (chain == NULL) { - ossl_cmp_err(ctx, "failed to validate newly enrolled cert"); + ossl_cmp_err(ctx, "failed building chain for newly enrolled cert"); fail_info = 1 << OSSL_CMP_PKIFAILUREINFO_incorrectData; } else { ossl_cmp_debug(ctx, - "success validating newly enrolled cert"); + "succeeded building proper chain for newly enrolled cert"); } } else if (chain == NULL) { ossl_cmp_warn(ctx, "could not build approximate chain for newly enrolled cert, resorting to received extraCerts"); @@ -635,14 +514,13 @@ int OSSL_CMP_certConf_cb(OSSL_CMP_CTX *ctx, X509 *cert, int fail_info, "success building approximate chain for newly enrolled cert"); } (void)ossl_cmp_ctx_set1_newChain(ctx, chain); - OSSL_STACK_OF_X509_free(chain); + sk_X509_pop_free(chain, X509_free); return fail_info; } /*- * Perform the generic handling of certificate responses for IR/CR/KUR/P10CR. - * |rid| must be OSSL_CMP_CERTREQID_NONE if not available, namely for p10cr * Returns -1 on receiving pollRep if sleep == 0, setting the checkAfter value. * Returns 1 on success and provides the received PKIMESSAGE in *resp. * Returns 0 on error (which includes the case that timeout has been reached). @@ -650,84 +528,54 @@ int OSSL_CMP_certConf_cb(OSSL_CMP_CTX *ctx, X509 *cert, int fail_info, */ static int cert_response(OSSL_CMP_CTX *ctx, int sleep, int rid, OSSL_CMP_MSG **resp, int *checkAfter, - ossl_unused int req_type, - ossl_unused int expected_type) + int req_type, int expected_type) { - EVP_PKEY *rkey = ossl_cmp_ctx_get0_newPubkey(ctx); + EVP_PKEY *rkey = OSSL_CMP_CTX_get0_newPkey(ctx /* may be NULL */, 0); int fail_info = 0; /* no failure */ const char *txt = NULL; - OSSL_CMP_CERTREPMESSAGE *crepmsg = NULL; - OSSL_CMP_CERTRESPONSE *crep = NULL; + OSSL_CMP_CERTREPMESSAGE *crepmsg; + OSSL_CMP_CERTRESPONSE *crep; OSSL_CMP_certConf_cb_t cb; X509 *cert; char *subj = NULL; int ret = 1; - int rcvd_type; - OSSL_CMP_PKISI *si; if (!ossl_assert(ctx != NULL)) return 0; retry: - rcvd_type = OSSL_CMP_MSG_get_bodytype(*resp); - if (IS_CREP(rcvd_type)) { - crepmsg = (*resp)->body->value.ip; /* same for cp and kup */ - if (sk_OSSL_CMP_CERTRESPONSE_num(crepmsg->response) > 1) { - ERR_raise(ERR_LIB_CMP, CMP_R_MULTIPLE_RESPONSES_NOT_SUPPORTED); - return 0; - } - crep = ossl_cmp_certrepmessage_get0_certresponse(crepmsg, rid); - if (crep == NULL) - return 0; - si = crep->status; - - if (rid == OSSL_CMP_CERTREQID_NONE) { - /* for OSSL_CMP_PKIBODY_P10CR learn CertReqId from response */ - rid = ossl_cmp_asn1_get_int(crep->certReqId); - if (rid < OSSL_CMP_CERTREQID_NONE) { - ERR_raise(ERR_LIB_CMP, CMP_R_BAD_REQUEST_ID); - return 0; - } - } - } else if (rcvd_type == OSSL_CMP_PKIBODY_ERROR) { - si = (*resp)->body->value.error->pKIStatusInfo; - } else { - ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); + crepmsg = (*resp)->body->value.ip; /* same for cp and kup */ + if (sk_OSSL_CMP_CERTRESPONSE_num(crepmsg->response) > 1) { + ERR_raise(ERR_LIB_CMP, CMP_R_MULTIPLE_RESPONSES_NOT_SUPPORTED); return 0; } - - if (!save_statusInfo(ctx, si)) + crep = ossl_cmp_certrepmessage_get0_certresponse(crepmsg, rid); + if (crep == NULL) + return 0; + if (!save_statusInfo(ctx, crep->status)) return 0; + if (rid == -1) { + /* for OSSL_CMP_PKIBODY_P10CR learn CertReqId from response */ + rid = ossl_cmp_asn1_get_int(crep->certReqId); + if (rid == -1) { + ERR_raise(ERR_LIB_CMP, CMP_R_BAD_REQUEST_ID); + return 0; + } + } - if (ossl_cmp_pkisi_get_status(si) == OSSL_CMP_PKISTATUS_waiting) { - /* - * Here we allow both and error message with waiting indication - * as well as a certificate response with waiting indication, where - * its flavor (ip, cp, or kup) may not strictly match ir/cr/p10cr/kur. - */ + if (ossl_cmp_pkisi_get_status(crep->status) == OSSL_CMP_PKISTATUS_waiting) { OSSL_CMP_MSG_free(*resp); *resp = NULL; if ((ret = poll_for_response(ctx, sleep, rid, resp, checkAfter)) != 0) { if (ret == -1) /* at this point implies sleep == 0 */ return ret; /* waiting */ - goto retry; /* got some response other than pollRep */ + goto retry; /* got ip/cp/kup, which may still indicate 'waiting' */ } else { ERR_raise(ERR_LIB_CMP, CMP_R_POLLING_FAILED); return 0; } } - /* at this point, we have received ip/cp/kup/error without waiting */ - if (rcvd_type == OSSL_CMP_PKIBODY_ERROR) { - ERR_raise(ERR_LIB_CMP, CMP_R_RECEIVED_ERROR); - return 0; - } - /* here we are strict on the flavor of ip/cp/kup: must match request */ - if (rcvd_type != expected_type) { - ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); - return 0; - } - cert = get1_cert_status(ctx, (*resp)->body->type, crep); if (cert == NULL) { ERR_add_error_data(1, "; cannot extract certificate from response"); @@ -740,7 +588,7 @@ static int cert_response(OSSL_CMP_CTX *ctx, int sleep, int rid, * if the CMP server returned certificates in the caPubs field, copy them * to the context so that they can be retrieved if necessary */ - if (crepmsg != NULL && crepmsg->caPubs != NULL + if (crepmsg->caPubs != NULL && !ossl_cmp_ctx_set1_caPubs(ctx, crepmsg->caPubs)) return 0; @@ -771,13 +619,9 @@ static int cert_response(OSSL_CMP_CTX *ctx, int sleep, int rid, if (fail_info != 0) /* immediately log error before any certConf exchange */ ossl_cmp_log1(ERROR, ctx, "rejecting newly enrolled cert with subject: %s", subj); - /* - * certConf exchange should better be moved to do_certreq_seq() such that - * also more low-level errors with CertReqMessages get reported to server - */ if (!ctx->disableConfirm && !ossl_cmp_hdr_has_implicitConfirm((*resp)->header)) { - if (!ossl_cmp_exchange_certConf(ctx, rid, fail_info, txt)) + if (!ossl_cmp_exchange_certConf(ctx, fail_info, txt)) ret = 0; } @@ -786,7 +630,6 @@ static int cert_response(OSSL_CMP_CTX *ctx, int sleep, int rid, ERR_raise_data(ERR_LIB_CMP, CMP_R_CERTIFICATE_NOT_ACCEPTED, "rejecting newly enrolled cert with subject: %s; %s", subj, txt); - ctx->status = OSSL_CMP_PKISTATUS_rejection; ret = 0; } OPENSSL_free(subj); @@ -800,15 +643,17 @@ static int initial_certreq(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *req; int res; - ctx->status = OSSL_CMP_PKISTATUS_request; + ctx->status = -1; if (!ossl_cmp_ctx_set0_newCert(ctx, NULL)) return 0; + if (ctx->total_timeout > 0) /* else ctx->end_time is not used */ + ctx->end_time = time(NULL) + ctx->total_timeout; + /* also checks if all necessary options are set */ if ((req = ossl_cmp_certreq_new(ctx, req_type, crm)) == NULL) return 0; - ctx->status = OSSL_CMP_PKISTATUS_trans; res = send_receive_check(ctx, req, p_rep, rep_type); OSSL_CMP_MSG_free(req); return res; @@ -819,7 +664,7 @@ int OSSL_CMP_try_certreq(OSSL_CMP_CTX *ctx, int req_type, { OSSL_CMP_MSG *rep = NULL; int is_p10 = req_type == OSSL_CMP_PKIBODY_P10CR; - int rid = is_p10 ? OSSL_CMP_CERTREQID_NONE : OSSL_CMP_CERTREQID; + int rid = is_p10 ? -1 : OSSL_CMP_CERTREQID; int rep_type = is_p10 ? OSSL_CMP_PKIBODY_CP : req_type + 1; int res = 0; @@ -831,9 +676,6 @@ int OSSL_CMP_try_certreq(OSSL_CMP_CTX *ctx, int req_type, if (ctx->status != OSSL_CMP_PKISTATUS_waiting) { /* not polling already */ if (!initial_certreq(ctx, req_type, crm, &rep, rep_type)) goto err; - - if (!save_senderNonce_if_waiting(ctx, rep, rid)) - return 0; } else { if (req_type < 0) return ossl_cmp_exchange_error(ctx, OSSL_CMP_PKISTATUS_rejection, @@ -861,9 +703,10 @@ int OSSL_CMP_try_certreq(OSSL_CMP_CTX *ctx, int req_type, X509 *OSSL_CMP_exec_certreq(OSSL_CMP_CTX *ctx, int req_type, const OSSL_CRMF_MSG *crm) { + OSSL_CMP_MSG *rep = NULL; int is_p10 = req_type == OSSL_CMP_PKIBODY_P10CR; - int rid = is_p10 ? OSSL_CMP_CERTREQID_NONE : OSSL_CMP_CERTREQID; + int rid = is_p10 ? -1 : OSSL_CMP_CERTREQID; int rep_type = is_p10 ? OSSL_CMP_PKIBODY_CP : req_type + 1; X509 *result = NULL; @@ -875,9 +718,6 @@ X509 *OSSL_CMP_exec_certreq(OSSL_CMP_CTX *ctx, int req_type, if (!initial_certreq(ctx, req_type, crm, &rep, rep_type)) goto err; - if (!save_senderNonce_if_waiting(ctx, rep, rid)) - return 0; - if (cert_response(ctx, 1 /* sleep */, rid, &rep, NULL, req_type, rep_type) <= 0) goto err; @@ -903,19 +743,17 @@ int OSSL_CMP_exec_RR_ses(OSSL_CMP_CTX *ctx) ERR_raise(ERR_LIB_CMP, CMP_R_INVALID_ARGS); return 0; } - ctx->status = OSSL_CMP_PKISTATUS_request; - if (ctx->oldCert == NULL && ctx->p10CSR == NULL - && (ctx->serialNumber == NULL || ctx->issuer == NULL)) { + if (ctx->oldCert == NULL && ctx->p10CSR == NULL) { ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_REFERENCE_CERT); return 0; } + ctx->status = -1; /* OSSL_CMP_rr_new() also checks if all necessary options are set */ if ((rr = ossl_cmp_rr_new(ctx)) == NULL) goto end; - ctx->status = OSSL_CMP_PKISTATUS_trans; - if (!send_receive_also_delayed(ctx, rr, &rp, OSSL_CMP_PKIBODY_RP)) + if (!send_receive_check(ctx, rr, &rp, OSSL_CMP_PKIBODY_RP)) goto end; rrep = rp->body->value.rp; @@ -972,8 +810,7 @@ int OSSL_CMP_exec_RR_ses(OSSL_CMP_CTX *ctx) OSSL_CRMF_CERTTEMPLATE *tmpl = sk_OSSL_CMP_REVDETAILS_value(rr->body->value.rr, rsid)->certDetails; const X509_NAME *issuer = OSSL_CRMF_CERTTEMPLATE_get0_issuer(tmpl); - const ASN1_INTEGER *serial = - OSSL_CRMF_CERTTEMPLATE_get0_serialNumber(tmpl); + const ASN1_INTEGER *serial = OSSL_CRMF_CERTTEMPLATE_get0_serialNumber(tmpl); if (sk_OSSL_CRMF_CERTID_num(rrep->revCerts) != num_RevDetails) { ERR_raise(ERR_LIB_CMP, CMP_R_WRONG_RP_COMPONENT_COUNT); @@ -1024,31 +861,27 @@ STACK_OF(OSSL_CMP_ITAV) *OSSL_CMP_exec_GENM_ses(OSSL_CMP_CTX *ctx) { OSSL_CMP_MSG *genm; OSSL_CMP_MSG *genp = NULL; - STACK_OF(OSSL_CMP_ITAV) *itavs = NULL; + STACK_OF(OSSL_CMP_ITAV) *rcvd_itavs = NULL; if (ctx == NULL) { ERR_raise(ERR_LIB_CMP, CMP_R_INVALID_ARGS); - return NULL; + return 0; } - ctx->status = OSSL_CMP_PKISTATUS_request; + ctx->status = -1; if ((genm = ossl_cmp_genm_new(ctx)) == NULL) goto err; - ctx->status = OSSL_CMP_PKISTATUS_trans; - if (!send_receive_also_delayed(ctx, genm, &genp, OSSL_CMP_PKIBODY_GENP)) + if (!send_receive_check(ctx, genm, &genp, OSSL_CMP_PKIBODY_GENP)) goto err; - ctx->status = OSSL_CMP_PKISTATUS_accepted; - itavs = genp->body->value.genp; - if (itavs == NULL) - itavs = sk_OSSL_CMP_ITAV_new_null(); /* received stack of itavs not to be freed with the genp */ + rcvd_itavs = genp->body->value.genp; genp->body->value.genp = NULL; err: OSSL_CMP_MSG_free(genm); OSSL_CMP_MSG_free(genp); - return itavs; /* NULL indicates error case */ + return rcvd_itavs; /* recv_itavs == NULL indicates an error */ } diff --git a/openssl/src/crypto/cmp/cmp_ctx.c b/openssl/src/crypto/cmp/cmp_ctx.c index 7b78ab160..f514ab27e 100644 --- a/openssl/src/crypto/cmp/cmp_ctx.c +++ b/openssl/src/crypto/cmp/cmp_ctx.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -20,35 +20,16 @@ #include #include -#define DEFINE_OSSL_CMP_CTX_get0(FIELD, TYPE) \ - DEFINE_OSSL_CMP_CTX_get0_NAME(FIELD, FIELD, TYPE) -#define DEFINE_OSSL_CMP_CTX_get0_NAME(NAME, FIELD, TYPE) \ -TYPE *OSSL_CMP_CTX_get0_##NAME(const OSSL_CMP_CTX *ctx) \ -{ \ - if (ctx == NULL) { \ - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); \ - return NULL; \ - } \ - return ctx->FIELD; \ -} - /* * Get current certificate store containing trusted root CA certs */ -DEFINE_OSSL_CMP_CTX_get0_NAME(trusted, trusted, X509_STORE) - -#define DEFINE_OSSL_set0(PREFIX, FIELD, TYPE) \ - DEFINE_OSSL_set0_NAME(PREFIX, FIELD, FIELD, TYPE) -#define DEFINE_OSSL_set0_NAME(PREFIX, NAME, FIELD, TYPE) \ -int PREFIX##_set0##_##NAME(OSSL_CMP_CTX *ctx, TYPE *val) \ -{ \ - if (ctx == NULL) { \ - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); \ - return 0; \ - } \ - TYPE##_free(ctx->FIELD); \ - ctx->FIELD = val; \ - return 1; \ +X509_STORE *OSSL_CMP_CTX_get0_trustedStore(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return ctx->trusted; } /* @@ -56,13 +37,26 @@ int PREFIX##_set0##_##NAME(OSSL_CMP_CTX *ctx, TYPE *val) \ * and a cert verification callback function used for CMP server authentication. * Any already existing store entry is freed. Given NULL, the entry is reset. */ -DEFINE_OSSL_set0_NAME(OSSL_CMP_CTX, trusted, trusted, X509_STORE) - -DEFINE_OSSL_CMP_CTX_get0(libctx, OSSL_LIB_CTX) -DEFINE_OSSL_CMP_CTX_get0(propq, const char) +int OSSL_CMP_CTX_set0_trustedStore(OSSL_CMP_CTX *ctx, X509_STORE *store) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + X509_STORE_free(ctx->trusted); + ctx->trusted = store; + return 1; +} /* Get current list of non-trusted intermediate certs */ -DEFINE_OSSL_CMP_CTX_get0(untrusted, STACK_OF(X509)) +STACK_OF(X509) *OSSL_CMP_CTX_get0_untrusted(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return ctx->untrusted; +} /* * Set untrusted certificates for path construction in authentication of @@ -79,11 +73,11 @@ int OSSL_CMP_CTX_set1_untrusted(OSSL_CMP_CTX *ctx, STACK_OF(X509) *certs) if (!ossl_x509_add_certs_new(&untrusted, certs, X509_ADD_FLAG_UP_REF | X509_ADD_FLAG_NO_DUP)) goto err; - OSSL_STACK_OF_X509_free(ctx->untrusted); + sk_X509_pop_free(ctx->untrusted, X509_free); ctx->untrusted = untrusted; return 1; err: - OSSL_STACK_OF_X509_free(untrusted); + sk_X509_pop_free(untrusted, X509_free); return 0; } @@ -114,21 +108,18 @@ OSSL_CMP_CTX *OSSL_CMP_CTX_new(OSSL_LIB_CTX *libctx, const char *propq) ctx->libctx = libctx; if (propq != NULL && (ctx->propq = OPENSSL_strdup(propq)) == NULL) - goto err; + goto oom; ctx->log_verbosity = OSSL_CMP_LOG_INFO; - ctx->status = OSSL_CMP_PKISTATUS_unspecified; + ctx->status = -1; ctx->failInfoCode = -1; ctx->keep_alive = 1; ctx->msg_timeout = -1; - ctx->tls_used = -1; /* default for backward compatibility */ - if ((ctx->untrusted = sk_X509_new_null()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); - goto err; - } + if ((ctx->untrusted = sk_X509_new_null()) == NULL) + goto oom; ctx->pbm_slen = 16; if (!cmp_ctx_set_md(ctx, &ctx->pbm_owf, NID_sha256)) @@ -144,18 +135,13 @@ OSSL_CMP_CTX *OSSL_CMP_CTX_new(OSSL_LIB_CTX *libctx, const char *propq) /* all other elements are initialized to 0 or NULL, respectively */ return ctx; + oom: + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); err: OSSL_CMP_CTX_free(ctx); return NULL; } -#define OSSL_CMP_ITAVs_free(itavs) \ - sk_OSSL_CMP_ITAV_pop_free(itavs, OSSL_CMP_ITAV_free); -#define X509_EXTENSIONS_free(exts) \ - sk_X509_EXTENSION_pop_free(exts, X509_EXTENSION_free) -#define OSSL_CMP_PKIFREETEXT_free(text) \ - sk_ASN1_UTF8STRING_pop_free(text, ASN1_UTF8STRING_free) - /* Prepare the OSSL_CMP_CTX for next use, partly re-initializing OSSL_CMP_CTX */ int OSSL_CMP_CTX_reinit(OSSL_CMP_CTX *ctx) { @@ -164,26 +150,20 @@ int OSSL_CMP_CTX_reinit(OSSL_CMP_CTX *ctx) return 0; } -#ifndef OPENSSL_NO_HTTP if (ctx->http_ctx != NULL) { (void)OSSL_HTTP_close(ctx->http_ctx, 1); ossl_cmp_debug(ctx, "disconnected from CMP server"); ctx->http_ctx = NULL; } -#endif - ctx->status = OSSL_CMP_PKISTATUS_unspecified; + ctx->status = -1; ctx->failInfoCode = -1; - OSSL_CMP_ITAVs_free(ctx->genm_ITAVs); - ctx->genm_ITAVs = NULL; - return ossl_cmp_ctx_set0_statusString(ctx, NULL) && ossl_cmp_ctx_set0_newCert(ctx, NULL) && ossl_cmp_ctx_set1_newChain(ctx, NULL) && ossl_cmp_ctx_set1_caPubs(ctx, NULL) && ossl_cmp_ctx_set1_extraCertsIn(ctx, NULL) - && ossl_cmp_ctx_set1_validatedSrvCert(ctx, NULL) - && ossl_cmp_ctx_set1_first_senderNonce(ctx, NULL) + && ossl_cmp_ctx_set0_validatedSrvCert(ctx, NULL) && OSSL_CMP_CTX_set1_transactionID(ctx, NULL) && OSSL_CMP_CTX_set1_senderNonce(ctx, NULL) && ossl_cmp_ctx_set1_recipNonce(ctx, NULL); @@ -195,12 +175,10 @@ void OSSL_CMP_CTX_free(OSSL_CMP_CTX *ctx) if (ctx == NULL) return; -#ifndef OPENSSL_NO_HTTP if (ctx->http_ctx != NULL) { (void)OSSL_HTTP_close(ctx->http_ctx, 1); ossl_cmp_debug(ctx, "disconnected from CMP server"); } -#endif OPENSSL_free(ctx->propq); OPENSSL_free(ctx->serverPath); OPENSSL_free(ctx->server); @@ -211,10 +189,10 @@ void OSSL_CMP_CTX_free(OSSL_CMP_CTX *ctx) X509_free(ctx->validatedSrvCert); X509_NAME_free(ctx->expected_sender); X509_STORE_free(ctx->trusted); - OSSL_STACK_OF_X509_free(ctx->untrusted); + sk_X509_pop_free(ctx->untrusted, X509_free); X509_free(ctx->cert); - OSSL_STACK_OF_X509_free(ctx->chain); + sk_X509_pop_free(ctx->chain, X509_free); EVP_PKEY_free(ctx->pkey); ASN1_OCTET_STRING_free(ctx->referenceValue); if (ctx->secretValue != NULL) @@ -227,83 +205,120 @@ void OSSL_CMP_CTX_free(OSSL_CMP_CTX *ctx) ASN1_OCTET_STRING_free(ctx->transactionID); ASN1_OCTET_STRING_free(ctx->senderNonce); ASN1_OCTET_STRING_free(ctx->recipNonce); - ASN1_OCTET_STRING_free(ctx->first_senderNonce); - OSSL_CMP_ITAVs_free(ctx->geninfo_ITAVs); - OSSL_STACK_OF_X509_free(ctx->extraCertsOut); + sk_OSSL_CMP_ITAV_pop_free(ctx->geninfo_ITAVs, OSSL_CMP_ITAV_free); + sk_X509_pop_free(ctx->extraCertsOut, X509_free); EVP_PKEY_free(ctx->newPkey); X509_NAME_free(ctx->issuer); - ASN1_INTEGER_free(ctx->serialNumber); X509_NAME_free(ctx->subjectName); sk_GENERAL_NAME_pop_free(ctx->subjectAltNames, GENERAL_NAME_free); - X509_EXTENSIONS_free(ctx->reqExtensions); + sk_X509_EXTENSION_pop_free(ctx->reqExtensions, X509_EXTENSION_free); sk_POLICYINFO_pop_free(ctx->policies, POLICYINFO_free); X509_free(ctx->oldCert); X509_REQ_free(ctx->p10CSR); - OSSL_CMP_ITAVs_free(ctx->genm_ITAVs); + sk_OSSL_CMP_ITAV_pop_free(ctx->genm_ITAVs, OSSL_CMP_ITAV_free); - OSSL_CMP_PKIFREETEXT_free(ctx->statusString); + sk_ASN1_UTF8STRING_pop_free(ctx->statusString, ASN1_UTF8STRING_free); X509_free(ctx->newCert); - OSSL_STACK_OF_X509_free(ctx->newChain); - OSSL_STACK_OF_X509_free(ctx->caPubs); - OSSL_STACK_OF_X509_free(ctx->extraCertsIn); + sk_X509_pop_free(ctx->newChain, X509_free); + sk_X509_pop_free(ctx->caPubs, X509_free); + sk_X509_pop_free(ctx->extraCertsIn, X509_free); OPENSSL_free(ctx); } -#define DEFINE_OSSL_set(PREFIX, FIELD, TYPE) \ -int PREFIX##_set_##FIELD(OSSL_CMP_CTX *ctx, TYPE val) \ -{ \ - if (ctx == NULL) { \ - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); \ - return 0; \ - } \ - ctx->FIELD = val; \ - return 1; \ -} - -DEFINE_OSSL_set(ossl_cmp_ctx, status, int) - -#define DEFINE_OSSL_get(PREFIX, FIELD, TYPE, ERR_RET) \ -TYPE PREFIX##_get_##FIELD(const OSSL_CMP_CTX *ctx) \ -{ \ - if (ctx == NULL) { \ - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); \ - return ERR_RET; \ - } \ - return ctx->FIELD; \ +int ossl_cmp_ctx_set_status(OSSL_CMP_CTX *ctx, int status) +{ + if (!ossl_assert(ctx != NULL)) + return 0; + ctx->status = status; + return 1; } /* * Returns the PKIStatus from the last CertRepMessage * or Revocation Response or error message, -1 on error */ -DEFINE_OSSL_get(OSSL_CMP_CTX, status, int, -1) +int OSSL_CMP_CTX_get_status(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return -1; + } + return ctx->status; +} /* * Returns the statusString from the last CertRepMessage * or Revocation Response or error message, NULL on error */ -DEFINE_OSSL_CMP_CTX_get0(statusString, OSSL_CMP_PKIFREETEXT) +OSSL_CMP_PKIFREETEXT *OSSL_CMP_CTX_get0_statusString(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return ctx->statusString; +} -DEFINE_OSSL_set0(ossl_cmp_ctx, statusString, OSSL_CMP_PKIFREETEXT) +int ossl_cmp_ctx_set0_statusString(OSSL_CMP_CTX *ctx, + OSSL_CMP_PKIFREETEXT *text) +{ + if (!ossl_assert(ctx != NULL)) + return 0; + sk_ASN1_UTF8STRING_pop_free(ctx->statusString, ASN1_UTF8STRING_free); + ctx->statusString = text; + return 1; +} + +int ossl_cmp_ctx_set0_validatedSrvCert(OSSL_CMP_CTX *ctx, X509 *cert) +{ + if (!ossl_assert(ctx != NULL)) + return 0; + X509_free(ctx->validatedSrvCert); + ctx->validatedSrvCert = cert; + return 1; +} /* Set callback function for checking if the cert is ok or should be rejected */ -DEFINE_OSSL_set(OSSL_CMP_CTX, certConf_cb, OSSL_CMP_certConf_cb_t) +int OSSL_CMP_CTX_set_certConf_cb(OSSL_CMP_CTX *ctx, OSSL_CMP_certConf_cb_t cb) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + ctx->certConf_cb = cb; + return 1; +} /* * Set argument, respectively a pointer to a structure containing arguments, * optionally to be used by the certConf callback. */ -DEFINE_OSSL_set(OSSL_CMP_CTX, certConf_cb_arg, void *) +int OSSL_CMP_CTX_set_certConf_cb_arg(OSSL_CMP_CTX *ctx, void *arg) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + ctx->certConf_cb_arg = arg; + return 1; +} /* * Get argument, respectively the pointer to a structure containing arguments, * optionally to be used by certConf callback. * Returns callback argument set previously (NULL if not set or on error) */ -DEFINE_OSSL_get(OSSL_CMP_CTX, certConf_cb_arg, void *, NULL) +void *OSSL_CMP_CTX_get_certConf_cb_arg(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return ctx->certConf_cb_arg; +} #ifndef OPENSSL_NO_TRACE static size_t ossl_cmp_log_trace_cb(const char *buf, size_t cnt, @@ -427,16 +442,15 @@ int OSSL_CMP_CTX_set1_referenceValue(OSSL_CMP_CTX *ctx, ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); return 0; } - return - ossl_cmp_asn1_octet_string_set1_bytes(&ctx->referenceValue, ref, len); + return ossl_cmp_asn1_octet_string_set1_bytes(&ctx->referenceValue, ref, + len); } /* Set or clear the password to be used for protecting messages with PBMAC */ -int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, - const unsigned char *sec, int len) +int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, const unsigned char *sec, + const int len) { ASN1_OCTET_STRING *secretValue = NULL; - if (ctx == NULL) { ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); return 0; @@ -451,51 +465,74 @@ int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, return 1; } -#define DEFINE_OSSL_CMP_CTX_get1_certs(FIELD) \ -STACK_OF(X509) *OSSL_CMP_CTX_get1_##FIELD(const OSSL_CMP_CTX *ctx) \ -{ \ - if (ctx == NULL) { \ - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); \ - return NULL; \ - } \ - return X509_chain_up_ref(ctx->FIELD); \ -} - /* Returns the cert chain computed by OSSL_CMP_certConf_cb(), NULL on error */ -DEFINE_OSSL_CMP_CTX_get1_certs(newChain) - -#define DEFINE_OSSL_set1_certs(PREFIX, FIELD) \ -int PREFIX##_set1_##FIELD(OSSL_CMP_CTX *ctx, STACK_OF(X509) *certs) \ -{ \ - if (ctx == NULL) { \ - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); \ - return 0; \ - } \ - OSSL_STACK_OF_X509_free(ctx->FIELD); \ - ctx->FIELD = NULL; \ - return certs == NULL || (ctx->FIELD = X509_chain_up_ref(certs)) != NULL; \ +STACK_OF(X509) *OSSL_CMP_CTX_get1_newChain(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return X509_chain_up_ref(ctx->newChain); } /* * Copies any given stack of inbound X509 certificates to newChain * of the OSSL_CMP_CTX structure so that they may be retrieved later. */ -DEFINE_OSSL_set1_certs(ossl_cmp_ctx, newChain) +int ossl_cmp_ctx_set1_newChain(OSSL_CMP_CTX *ctx, STACK_OF(X509) *newChain) +{ + if (!ossl_assert(ctx != NULL)) + return 0; + + sk_X509_pop_free(ctx->newChain, X509_free); + ctx->newChain = NULL; + return newChain == NULL || + (ctx->newChain = X509_chain_up_ref(newChain)) != NULL; +} /* Returns the stack of extraCerts received in CertRepMessage, NULL on error */ -DEFINE_OSSL_CMP_CTX_get1_certs(extraCertsIn) +STACK_OF(X509) *OSSL_CMP_CTX_get1_extraCertsIn(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return X509_chain_up_ref(ctx->extraCertsIn); +} /* * Copies any given stack of inbound X509 certificates to extraCertsIn * of the OSSL_CMP_CTX structure so that they may be retrieved later. */ -DEFINE_OSSL_set1_certs(ossl_cmp_ctx, extraCertsIn) +int ossl_cmp_ctx_set1_extraCertsIn(OSSL_CMP_CTX *ctx, + STACK_OF(X509) *extraCertsIn) +{ + if (!ossl_assert(ctx != NULL)) + return 0; + + sk_X509_pop_free(ctx->extraCertsIn, X509_free); + ctx->extraCertsIn = NULL; + return extraCertsIn == NULL + || (ctx->extraCertsIn = X509_chain_up_ref(extraCertsIn)) != NULL; +} /* * Copies any given stack as the new stack of X509 * certificates to send out in the extraCerts field. */ -DEFINE_OSSL_set1_certs(OSSL_CMP_CTX, extraCertsOut) +int OSSL_CMP_CTX_set1_extraCertsOut(OSSL_CMP_CTX *ctx, + STACK_OF(X509) *extraCertsOut) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + + sk_X509_pop_free(ctx->extraCertsOut, X509_free); + ctx->extraCertsOut = NULL; + return extraCertsOut == NULL + || (ctx->extraCertsOut = X509_chain_up_ref(extraCertsOut)) != NULL; +} /* * Add the given policy info object @@ -525,19 +562,6 @@ int OSSL_CMP_CTX_push0_geninfo_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav) return OSSL_CMP_ITAV_push0_stack_item(&ctx->geninfo_ITAVs, itav); } -int OSSL_CMP_CTX_reset_geninfo_ITAVs(OSSL_CMP_CTX *ctx) -{ - if (ctx == NULL) { - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); - return 0; - } - OSSL_CMP_ITAVs_free(ctx->geninfo_ITAVs); - ctx->geninfo_ITAVs = NULL; - return 1; -} - -DEFINE_OSSL_CMP_CTX_get0(geninfo_ITAVs, STACK_OF(OSSL_CMP_ITAV)) - /* Add an itav for the body of outgoing general messages */ int OSSL_CMP_CTX_push0_genm_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav) { @@ -553,13 +577,28 @@ int OSSL_CMP_CTX_push0_genm_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav) * were received in the caPubs field of the last CertRepMessage. * Returns NULL on error */ -DEFINE_OSSL_CMP_CTX_get1_certs(caPubs) +STACK_OF(X509) *OSSL_CMP_CTX_get1_caPubs(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return X509_chain_up_ref(ctx->caPubs); +} /* * Copies any given stack of certificates to the given * OSSL_CMP_CTX structure so that they may be retrieved later. */ -DEFINE_OSSL_set1_certs(ossl_cmp_ctx, caPubs) +int ossl_cmp_ctx_set1_caPubs(OSSL_CMP_CTX *ctx, STACK_OF(X509) *caPubs) +{ + if (!ossl_assert(ctx != NULL)) + return 0; + + sk_X509_pop_free(ctx->caPubs, X509_free); + ctx->caPubs = NULL; + return caPubs == NULL || (ctx->caPubs = X509_chain_up_ref(caPubs)) != NULL; +} #define char_dup OPENSSL_strdup #define char_free OPENSSL_free @@ -582,9 +621,8 @@ int OSSL_CMP_CTX_set1_##FIELD(OSSL_CMP_CTX *ctx, const TYPE *val) \ #define X509_invalid(cert) (!ossl_x509v3_cache_extensions(cert)) #define EVP_PKEY_invalid(key) 0 - -#define DEFINE_OSSL_set1_up_ref(PREFIX, FIELD, TYPE) \ -int PREFIX##_set1_##FIELD(OSSL_CMP_CTX *ctx, TYPE *val) \ +#define DEFINE_OSSL_CMP_CTX_set1_up_ref(FIELD, TYPE) \ +int OSSL_CMP_CTX_set1_##FIELD(OSSL_CMP_CTX *ctx, TYPE *val) \ { \ if (ctx == NULL) { \ ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); \ @@ -603,26 +641,22 @@ int PREFIX##_set1_##FIELD(OSSL_CMP_CTX *ctx, TYPE *val) \ return 1; \ } -DEFINE_OSSL_set1_up_ref(ossl_cmp_ctx, validatedSrvCert, X509) - /* * Pins the server certificate to be directly trusted (even if it is expired) * for verifying response messages. * Cert pointer is not consumed. It may be NULL to clear the entry. */ -DEFINE_OSSL_set1_up_ref(OSSL_CMP_CTX, srvCert, X509) +DEFINE_OSSL_CMP_CTX_set1_up_ref(srvCert, X509) -/* Set the X509 name of the recipient to be placed in the PKIHeader */ +/* Set the X509 name of the recipient. Set in the PKIHeader */ DEFINE_OSSL_CMP_CTX_set1(recipient, X509_NAME) /* Store the X509 name of the expected sender in the PKIHeader of responses */ DEFINE_OSSL_CMP_CTX_set1(expected_sender, X509_NAME) -/* Set the X509 name of the issuer to be placed in the certTemplate */ +/* Set the X509 name of the issuer. Set in the PKIHeader */ DEFINE_OSSL_CMP_CTX_set1(issuer, X509_NAME) -/* Set the ASN1_INTEGER serial to be placed in the certTemplate for rr */ -DEFINE_OSSL_CMP_CTX_set1(serialNumber, ASN1_INTEGER) /* * Set the subject name that will be placed in the certificate * request. This will be the subject name on the received certificate. @@ -642,7 +676,7 @@ int OSSL_CMP_CTX_set0_reqExtensions(OSSL_CMP_CTX *ctx, X509_EXTENSIONS *exts) ERR_raise(ERR_LIB_CMP, CMP_R_MULTIPLE_SAN_SOURCES); return 0; } - X509_EXTENSIONS_free(ctx->reqExtensions); + sk_X509_EXTENSION_pop_free(ctx->reqExtensions, X509_EXTENSION_free); ctx->reqExtensions = exts; return 1; } @@ -695,7 +729,7 @@ int OSSL_CMP_CTX_push1_subjectAltName(OSSL_CMP_CTX *ctx, * Set our own client certificate, used for example in KUR and when * doing the IR with existing certificate. */ -DEFINE_OSSL_set1_up_ref(OSSL_CMP_CTX, cert, X509) +DEFINE_OSSL_CMP_CTX_set1_up_ref(cert, X509) int OSSL_CMP_CTX_build_cert_chain(OSSL_CMP_CTX *ctx, X509_STORE *own_trusted, STACK_OF(X509) *candidates) @@ -729,7 +763,7 @@ int OSSL_CMP_CTX_build_cert_chain(OSSL_CMP_CTX *ctx, X509_STORE *own_trusted, * Also used as reference cert (defaulting to cert) for deriving subject DN * and SANs. Its issuer is used as default recipient in the CMP message header. */ -DEFINE_OSSL_set1_up_ref(OSSL_CMP_CTX, oldCert, X509) +DEFINE_OSSL_CMP_CTX_set1_up_ref(oldCert, X509) /* Set the PKCS#10 CSR to be sent in P10CR */ DEFINE_OSSL_CMP_CTX_set1(p10CSR, X509_REQ) @@ -738,19 +772,31 @@ DEFINE_OSSL_CMP_CTX_set1(p10CSR, X509_REQ) * Set the (newly received in IP/KUP/CP) certificate in the context. * This only permits for one cert to be enrolled at a time. */ -DEFINE_OSSL_set0(ossl_cmp_ctx, newCert, X509) +int ossl_cmp_ctx_set0_newCert(OSSL_CMP_CTX *ctx, X509 *cert) +{ + if (!ossl_assert(ctx != NULL)) + return 0; -/* Get successfully validated server cert, if any, of current transaction */ -DEFINE_OSSL_CMP_CTX_get0(validatedSrvCert, X509) + X509_free(ctx->newCert); + ctx->newCert = cert; + return 1; +} /* * Get the (newly received in IP/KUP/CP) client certificate from the context * This only permits for one client cert to be received... */ -DEFINE_OSSL_CMP_CTX_get0(newCert, X509) +X509 *OSSL_CMP_CTX_get0_newCert(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return ctx->newCert; +} /* Set the client's current private key */ -DEFINE_OSSL_set1_up_ref(OSSL_CMP_CTX, pkey, EVP_PKEY) +DEFINE_OSSL_CMP_CTX_set1_up_ref(pkey, EVP_PKEY) /* Set new key pair. Used e.g. when doing Key Update */ int OSSL_CMP_CTX_set0_newPkey(OSSL_CMP_CTX *ctx, int priv, EVP_PKEY *pkey) @@ -767,7 +813,6 @@ int OSSL_CMP_CTX_set0_newPkey(OSSL_CMP_CTX *ctx, int priv, EVP_PKEY *pkey) } /* Get the private/public key to use for cert enrollment, or NULL on error */ -/* In case |priv| == 0, better use ossl_cmp_ctx_get0_newPubkey() below */ EVP_PKEY *OSSL_CMP_CTX_get0_newPkey(const OSSL_CMP_CTX *ctx, int priv) { if (ctx == NULL) { @@ -782,92 +827,151 @@ EVP_PKEY *OSSL_CMP_CTX_get0_newPkey(const OSSL_CMP_CTX *ctx, int priv) return ctx->pkey; /* may be NULL */ } -EVP_PKEY *ossl_cmp_ctx_get0_newPubkey(const OSSL_CMP_CTX *ctx) +/* Set the given transactionID to the context */ +int OSSL_CMP_CTX_set1_transactionID(OSSL_CMP_CTX *ctx, + const ASN1_OCTET_STRING *id) { - if (!ossl_assert(ctx != NULL)) - return NULL; - if (ctx->newPkey != NULL) - return ctx->newPkey; - if (ctx->p10CSR != NULL) - return X509_REQ_get0_pubkey(ctx->p10CSR); - if (ctx->oldCert != NULL) - return X509_get0_pubkey(ctx->oldCert); - if (ctx->cert != NULL) - return X509_get0_pubkey(ctx->cert); - return ctx->pkey; -} - -#define DEFINE_set1_ASN1_OCTET_STRING(PREFIX, FIELD) \ -int PREFIX##_set1_##FIELD(OSSL_CMP_CTX *ctx, const ASN1_OCTET_STRING *id) \ -{ \ - if (ctx == NULL) { \ - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); \ - return 0; \ - } \ - return ossl_cmp_asn1_octet_string_set1(&ctx->FIELD, id); \ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + return ossl_cmp_asn1_octet_string_set1(&ctx->transactionID, id); } -/* Set the given transactionID to the context */ -DEFINE_set1_ASN1_OCTET_STRING(OSSL_CMP_CTX, transactionID) - /* Set the nonce to be used for the recipNonce in the message created next */ -DEFINE_set1_ASN1_OCTET_STRING(ossl_cmp_ctx, recipNonce) +int ossl_cmp_ctx_set1_recipNonce(OSSL_CMP_CTX *ctx, + const ASN1_OCTET_STRING *nonce) +{ + if (!ossl_assert(ctx != NULL)) + return 0; + return ossl_cmp_asn1_octet_string_set1(&ctx->recipNonce, nonce); +} /* Stores the given nonce as the last senderNonce sent out */ -DEFINE_set1_ASN1_OCTET_STRING(OSSL_CMP_CTX, senderNonce) - -/* store the first req sender nonce for verifying delayed delivery */ -DEFINE_set1_ASN1_OCTET_STRING(ossl_cmp_ctx, first_senderNonce) +int OSSL_CMP_CTX_set1_senderNonce(OSSL_CMP_CTX *ctx, + const ASN1_OCTET_STRING *nonce) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + return ossl_cmp_asn1_octet_string_set1(&ctx->senderNonce, nonce); +} /* Set the proxy server to use for HTTP(S) connections */ DEFINE_OSSL_CMP_CTX_set1(proxy, char) -/* Set the (HTTP) hostname of the CMP server */ +/* Set the (HTTP) host name of the CMP server */ DEFINE_OSSL_CMP_CTX_set1(server, char) /* Set the server exclusion list of the HTTP proxy server */ DEFINE_OSSL_CMP_CTX_set1(no_proxy, char) -#ifndef OPENSSL_NO_HTTP /* Set the http connect/disconnect callback function to be used for HTTP(S) */ -DEFINE_OSSL_set(OSSL_CMP_CTX, http_cb, OSSL_HTTP_bio_cb_t) +int OSSL_CMP_CTX_set_http_cb(OSSL_CMP_CTX *ctx, OSSL_HTTP_bio_cb_t cb) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + ctx->http_cb = cb; + return 1; +} /* Set argument optionally to be used by the http connect/disconnect callback */ -DEFINE_OSSL_set(OSSL_CMP_CTX, http_cb_arg, void *) +int OSSL_CMP_CTX_set_http_cb_arg(OSSL_CMP_CTX *ctx, void *arg) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + ctx->http_cb_arg = arg; + return 1; +} /* * Get argument optionally to be used by the http connect/disconnect callback * Returns callback argument set previously (NULL if not set or on error) */ -DEFINE_OSSL_get(OSSL_CMP_CTX, http_cb_arg, void *, NULL) -#endif +void *OSSL_CMP_CTX_get_http_cb_arg(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return ctx->http_cb_arg; +} /* Set callback function for sending CMP request and receiving response */ -DEFINE_OSSL_set(OSSL_CMP_CTX, transfer_cb, OSSL_CMP_transfer_cb_t) +int OSSL_CMP_CTX_set_transfer_cb(OSSL_CMP_CTX *ctx, OSSL_CMP_transfer_cb_t cb) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + ctx->transfer_cb = cb; + return 1; +} /* Set argument optionally to be used by the transfer callback */ -DEFINE_OSSL_set(OSSL_CMP_CTX, transfer_cb_arg, void *) +int OSSL_CMP_CTX_set_transfer_cb_arg(OSSL_CMP_CTX *ctx, void *arg) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + ctx->transfer_cb_arg = arg; + return 1; +} /* * Get argument optionally to be used by the transfer callback. * Returns callback argument set previously (NULL if not set or on error) */ -DEFINE_OSSL_get(OSSL_CMP_CTX, transfer_cb_arg, void *, NULL) +void *OSSL_CMP_CTX_get_transfer_cb_arg(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return NULL; + } + return ctx->transfer_cb_arg; +} /** Set the HTTP server port to be used */ -DEFINE_OSSL_set(OSSL_CMP_CTX, serverPort, int) +int OSSL_CMP_CTX_set_serverPort(OSSL_CMP_CTX *ctx, int port) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return 0; + } + ctx->serverPort = port; + return 1; +} /* Set the HTTP path to be used on the server (e.g "pkix/") */ DEFINE_OSSL_CMP_CTX_set1(serverPath, char) /* Set the failInfo error code as bit encoding in OSSL_CMP_CTX */ -DEFINE_OSSL_set(ossl_cmp_ctx, failInfoCode, int) +int ossl_cmp_ctx_set_failInfoCode(OSSL_CMP_CTX *ctx, int fail_info) +{ + if (!ossl_assert(ctx != NULL)) + return 0; + ctx->failInfoCode = fail_info; + return 1; +} /* * Get the failInfo error code in OSSL_CMP_CTX as bit encoding. * Returns bit string as integer on success, -1 on error */ -DEFINE_OSSL_get(OSSL_CMP_CTX, failInfoCode, int, -1) +int OSSL_CMP_CTX_get_failInfoCode(const OSSL_CMP_CTX *ctx) +{ + if (ctx == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); + return -1; + } + return ctx->failInfoCode; +} /* Set a Boolean or integer option of the context to the "val" arg */ int OSSL_CMP_CTX_set_option(OSSL_CMP_CTX *ctx, int opt, int val) @@ -915,9 +1019,6 @@ int OSSL_CMP_CTX_set_option(OSSL_CMP_CTX *ctx, int opt, int val) case OSSL_CMP_OPT_UNPROTECTED_ERRORS: ctx->unprotectedErrors = val; break; - case OSSL_CMP_OPT_NO_CACHE_EXTRACERTS: - ctx->noCacheExtraCerts = val; - break; case OSSL_CMP_OPT_VALIDITY_DAYS: ctx->days = val; break; @@ -960,9 +1061,6 @@ int OSSL_CMP_CTX_set_option(OSSL_CMP_CTX *ctx, int opt, int val) case OSSL_CMP_OPT_TOTAL_TIMEOUT: ctx->total_timeout = val; break; - case OSSL_CMP_OPT_USE_TLS: - ctx->tls_used = val; - break; case OSSL_CMP_OPT_PERMIT_TA_IN_EXTRACERTS_FOR_IR: ctx->permitTAInExtraCertsForIR = val; break; @@ -1003,8 +1101,6 @@ int OSSL_CMP_CTX_get_option(const OSSL_CMP_CTX *ctx, int opt) return ctx->unprotectedSend; case OSSL_CMP_OPT_UNPROTECTED_ERRORS: return ctx->unprotectedErrors; - case OSSL_CMP_OPT_NO_CACHE_EXTRACERTS: - return ctx->noCacheExtraCerts; case OSSL_CMP_OPT_VALIDITY_DAYS: return ctx->days; case OSSL_CMP_OPT_SUBJECTALTNAME_NODEFAULT: @@ -1029,8 +1125,6 @@ int OSSL_CMP_CTX_get_option(const OSSL_CMP_CTX *ctx, int opt) return ctx->msg_timeout; case OSSL_CMP_OPT_TOTAL_TIMEOUT: return ctx->total_timeout; - case OSSL_CMP_OPT_USE_TLS: - return ctx->tls_used; case OSSL_CMP_OPT_PERMIT_TA_IN_EXTRACERTS_FOR_IR: return ctx->permitTAInExtraCertsForIR; case OSSL_CMP_OPT_REVOCATION_REASON: diff --git a/openssl/src/crypto/cmp/cmp_err.c b/openssl/src/crypto/cmp/cmp_err.c index 56ac3691d..67d9964db 100644 --- a/openssl/src/crypto/cmp/cmp_err.c +++ b/openssl/src/crypto/cmp/cmp_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -76,7 +76,6 @@ static const ERR_STRING_DATA CMP_str_reasons[] = { "error validating protection"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_ERROR_VALIDATING_SIGNATURE), "error validating signature"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_EXPECTED_POLLREQ), "expected pollreq"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_FAILED_BUILDING_OWN_CHAIN), "failed building own chain"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_FAILED_EXTRACTING_PUBKEY), @@ -85,12 +84,8 @@ static const ERR_STRING_DATA CMP_str_reasons[] = { "failure obtaining random"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_FAIL_INFO_OUT_OF_RANGE), "fail info out of range"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_GETTING_GENP), "getting genp"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_INVALID_ARGS), "invalid args"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_INVALID_GENP), "invalid genp"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_INVALID_OPTION), "invalid option"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_INVALID_ROOTCAKEYUPDATE), - "invalid rootcakeyupdate"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_CERTID), "missing certid"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_KEY_INPUT_FOR_CREATING_PROTECTION), "missing key input for creating protection"}, @@ -100,17 +95,11 @@ static const ERR_STRING_DATA CMP_str_reasons[] = { {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_PBM_SECRET), "missing pbm secret"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_PRIVATE_KEY), "missing private key"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_PRIVATE_KEY_FOR_POPO), - "missing private key for popo"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_PROTECTION), "missing protection"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_PUBLIC_KEY), "missing public key"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_REFERENCE_CERT), "missing reference cert"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_SECRET), "missing secret"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_SENDER_IDENTIFICATION), "missing sender identification"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_TRUST_ANCHOR), - "missing trust anchor"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MISSING_TRUST_STORE), "missing trust store"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_MULTIPLE_REQUESTS_NOT_SUPPORTED), @@ -144,15 +133,10 @@ static const ERR_STRING_DATA CMP_str_reasons[] = { {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_TRANSACTIONID_UNMATCHED), "transactionid unmatched"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_TRANSFER_ERROR), "transfer error"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNCLEAN_CTX), "unclean ctx"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNEXPECTED_CERTPROFILE), - "unexpected certprofile"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNEXPECTED_PKIBODY), "unexpected pkibody"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNEXPECTED_PKISTATUS), "unexpected pkistatus"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNEXPECTED_POLLREQ), "unexpected pollreq"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNEXPECTED_PVNO), "unexpected pvno"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNEXPECTED_SENDER), "unexpected sender"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNKNOWN_ALGORITHM_ID), "unknown algorithm id"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNKNOWN_CERT_TYPE), "unknown cert type"}, @@ -161,8 +145,6 @@ static const ERR_STRING_DATA CMP_str_reasons[] = { "unsupported algorithm"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNSUPPORTED_KEY_TYPE), "unsupported key type"}, - {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNSUPPORTED_PKIBODY), - "unsupported pkibody"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC), "unsupported protection alg dhbasedmac"}, {ERR_PACK(ERR_LIB_CMP, 0, CMP_R_VALUE_TOO_LARGE), "value too large"}, diff --git a/openssl/src/crypto/cmp/cmp_genm.c b/openssl/src/crypto/cmp/cmp_genm.c deleted file mode 100644 index 5986036f5..000000000 --- a/openssl/src/crypto/cmp/cmp_genm.c +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * Copyright Siemens AG 2022 - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "cmp_local.h" -#include - -static const X509_VERIFY_PARAM *get0_trustedStore_vpm(const OSSL_CMP_CTX *ctx) -{ - const X509_STORE *ts = OSSL_CMP_CTX_get0_trustedStore(ctx); - - return ts == NULL ? NULL : X509_STORE_get0_param(ts); -} - -static void cert_msg(const char *func, const char *file, int lineno, - OSSL_CMP_severity level, OSSL_CMP_CTX *ctx, - const char *source, X509 *cert, const char *msg) -{ - char *subj = X509_NAME_oneline(X509_get_subject_name(cert), NULL, 0); - - ossl_cmp_print_log(level, ctx, func, file, lineno, - level == OSSL_CMP_LOG_WARNING ? "WARN" : "ERR", - "certificate from '%s' with subject '%s' %s", - source, subj, msg); - OPENSSL_free(subj); -} - -/* use |type_CA| -1 (no CA type check) or 0 (must be EE) or 1 (must be CA) */ -static int ossl_X509_check(OSSL_CMP_CTX *ctx, const char *source, X509 *cert, - int type_CA, const X509_VERIFY_PARAM *vpm) -{ - uint32_t ex_flags = X509_get_extension_flags(cert); - int res = X509_cmp_timeframe(vpm, X509_get0_notBefore(cert), - X509_get0_notAfter(cert)); - int ret = res == 0; - OSSL_CMP_severity level = - vpm == NULL ? OSSL_CMP_LOG_WARNING : OSSL_CMP_LOG_ERR; - - if (!ret) - cert_msg(OPENSSL_FUNC, OPENSSL_FILE, OPENSSL_LINE, level, ctx, - source, cert, res > 0 ? "has expired" : "not yet valid"); - if (type_CA >= 0 && (ex_flags & EXFLAG_V1) == 0) { - int is_CA = (ex_flags & EXFLAG_CA) != 0; - - if ((type_CA != 0) != is_CA) { - cert_msg(OPENSSL_FUNC, OPENSSL_FILE, OPENSSL_LINE, level, ctx, - source, cert, - is_CA ? "is not an EE cert" : "is not a CA cert"); - ret = 0; - } - } - return ret; -} - -static int ossl_X509_check_all(OSSL_CMP_CTX *ctx, const char *source, - STACK_OF(X509) *certs, - int type_CA, const X509_VERIFY_PARAM *vpm) -{ - int i; - int ret = 1; - - for (i = 0; i < sk_X509_num(certs /* may be NULL */); i++) - ret = ossl_X509_check(ctx, source, - sk_X509_value(certs, i), type_CA, vpm) - && ret; /* Having 'ret' after the '&&', all certs are checked. */ - return ret; -} - -static OSSL_CMP_ITAV *get_genm_itav(OSSL_CMP_CTX *ctx, - OSSL_CMP_ITAV *req, /* gets consumed */ - int expected, const char *desc) -{ - STACK_OF(OSSL_CMP_ITAV) *itavs = NULL; - int i, n; - - if (ctx == NULL) { - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); - goto err; - } - if (OSSL_CMP_CTX_get_status(ctx) != OSSL_CMP_PKISTATUS_unspecified) { - ERR_raise_data(ERR_LIB_CMP, CMP_R_UNCLEAN_CTX, - "client context in unsuitable state; should call CMPclient_reinit() before"); - goto err; - } - - if (!OSSL_CMP_CTX_push0_genm_ITAV(ctx, req)) - goto err; - req = NULL; - itavs = OSSL_CMP_exec_GENM_ses(ctx); - if (itavs == NULL) { - if (OSSL_CMP_CTX_get_status(ctx) != OSSL_CMP_PKISTATUS_request) - ERR_raise_data(ERR_LIB_CMP, CMP_R_GETTING_GENP, - "with infoType %s", desc); - return NULL; - } - - if ((n = sk_OSSL_CMP_ITAV_num(itavs)) <= 0) { - ERR_raise_data(ERR_LIB_CMP, CMP_R_INVALID_GENP, - "response on genm requesting infoType %s does not include suitable value", desc); - sk_OSSL_CMP_ITAV_free(itavs); - return NULL; - } - - if (n > 1) - ossl_cmp_log2(WARN, ctx, - "response on genm contains %d ITAVs; will use the first ITAV with infoType id-it-%s", - n, desc); - for (i = 0; i < n; i++) { - OSSL_CMP_ITAV *itav = sk_OSSL_CMP_ITAV_shift(itavs); - ASN1_OBJECT *obj = OSSL_CMP_ITAV_get0_type(itav); - char name[128] = "genp contains InfoType '"; - size_t offset = strlen(name); - - if (OBJ_obj2nid(obj) == expected) { - for (i++; i < n; i++) - OSSL_CMP_ITAV_free(sk_OSSL_CMP_ITAV_shift(itavs)); - sk_OSSL_CMP_ITAV_free(itavs); - return itav; - } - - if (OBJ_obj2txt(name + offset, sizeof(name) - offset, obj, 0) < 0) - strcat(name, ""); - ossl_cmp_log2(WARN, ctx, "%s' while expecting 'id-it-%s'", name, desc); - OSSL_CMP_ITAV_free(itav); - } - ERR_raise_data(ERR_LIB_CMP, CMP_R_INVALID_GENP, - "could not find any ITAV for %s", desc); - - err: - sk_OSSL_CMP_ITAV_free(itavs); - OSSL_CMP_ITAV_free(req); - return NULL; -} - -int OSSL_CMP_get1_caCerts(OSSL_CMP_CTX *ctx, STACK_OF(X509) **out) -{ - OSSL_CMP_ITAV *req, *itav; - STACK_OF(X509) *certs = NULL; - int ret = 0; - - if (out == NULL) { - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); - return 0; - } - *out = NULL; - - if ((req = OSSL_CMP_ITAV_new_caCerts(NULL)) == NULL) - return 0; - if ((itav = get_genm_itav(ctx, req, NID_id_it_caCerts, "caCerts")) == NULL) - return 0; - if (!OSSL_CMP_ITAV_get0_caCerts(itav, &certs)) - goto end; - ret = 1; - if (certs == NULL) /* no CA certificate available */ - goto end; - - if (!ossl_X509_check_all(ctx, "genp", certs, 1 /* CA */, - get0_trustedStore_vpm(ctx))) { - ret = 0; - goto end; - } - *out = sk_X509_new_reserve(NULL, sk_X509_num(certs)); - if (!X509_add_certs(*out, certs, - X509_ADD_FLAG_UP_REF | X509_ADD_FLAG_NO_DUP)) { - sk_X509_pop_free(*out, X509_free); - *out = NULL; - ret = 0; - } - - end: - OSSL_CMP_ITAV_free(itav); - return ret; -} - -static int selfsigned_verify_cb(int ok, X509_STORE_CTX *store_ctx) -{ - if (ok == 0 - && X509_STORE_CTX_get_error_depth(store_ctx) == 0 - && X509_STORE_CTX_get_error(store_ctx) - == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT) { - /* in this case, custom chain building */ - int i; - STACK_OF(X509) *trust; - STACK_OF(X509) *chain = X509_STORE_CTX_get0_chain(store_ctx); - STACK_OF(X509) *untrusted = X509_STORE_CTX_get0_untrusted(store_ctx); - X509_STORE_CTX_check_issued_fn check_issued = - X509_STORE_CTX_get_check_issued(store_ctx); - X509 *cert = sk_X509_value(chain, 0); /* target cert */ - X509 *issuer; - - for (i = 0; i < sk_X509_num(untrusted); i++) { - cert = sk_X509_value(untrusted, i); - if (!X509_add_cert(chain, cert, X509_ADD_FLAG_UP_REF)) - return 0; - } - - trust = X509_STORE_get1_all_certs(X509_STORE_CTX_get0_store(store_ctx)); - for (i = 0; i < sk_X509_num(trust); i++) { - issuer = sk_X509_value(trust, i); - if ((*check_issued)(store_ctx, cert, issuer)) { - if (X509_add_cert(chain, cert, X509_ADD_FLAG_UP_REF)) - ok = 1; - break; - } - } - sk_X509_pop_free(trust, X509_free); - return ok; - } else { - X509_STORE *ts = X509_STORE_CTX_get0_store(store_ctx); - X509_STORE_CTX_verify_cb verify_cb; - - if (ts == NULL || (verify_cb = X509_STORE_get_verify_cb(ts)) == NULL) - return ok; - return (*verify_cb)(ok, store_ctx); - } -} - -/* vanilla X509_verify_cert() does not support self-signed certs as target */ -static int verify_ss_cert(OSSL_LIB_CTX *libctx, const char *propq, - X509_STORE *ts, STACK_OF(X509) *untrusted, - X509 *target) -{ - X509_STORE_CTX *csc = NULL; - int ok = 0; - - if (ts == NULL || target == NULL) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - if ((csc = X509_STORE_CTX_new_ex(libctx, propq)) == NULL - || !X509_STORE_CTX_init(csc, ts, target, untrusted)) - goto err; - X509_STORE_CTX_set_verify_cb(csc, selfsigned_verify_cb); - ok = X509_verify_cert(csc) > 0; - - err: - X509_STORE_CTX_free(csc); - return ok; -} - -static int -verify_ss_cert_trans(OSSL_CMP_CTX *ctx, X509 *trusted /* may be NULL */, - X509 *trans /* the only untrusted cert, may be NULL */, - X509 *target, const char *desc) -{ - X509_STORE *ts = OSSL_CMP_CTX_get0_trusted(ctx); - STACK_OF(X509) *untrusted = NULL; - int res = 0; - - if (trusted != NULL) { - X509_VERIFY_PARAM *vpm = X509_STORE_get0_param(ts); - - if ((ts = X509_STORE_new()) == NULL) - return 0; - if (!X509_STORE_set1_param(ts, vpm) - || !X509_STORE_add_cert(ts, trusted)) - goto err; - } - - if (trans != NULL - && !ossl_x509_add_cert_new(&untrusted, trans, X509_ADD_FLAG_UP_REF)) - goto err; - - res = verify_ss_cert(OSSL_CMP_CTX_get0_libctx(ctx), - OSSL_CMP_CTX_get0_propq(ctx), - ts, untrusted, target); - if (!res) - ERR_raise_data(ERR_LIB_CMP, CMP_R_INVALID_ROOTCAKEYUPDATE, - "failed to validate %s certificate received in genp %s", - desc, trusted == NULL ? "using trust store" - : "with given certificate as trust anchor"); - - err: - sk_X509_pop_free(untrusted, X509_free); - if (trusted != NULL) - X509_STORE_free(ts); - return res; -} - -int OSSL_CMP_get1_rootCaKeyUpdate(OSSL_CMP_CTX *ctx, - const X509 *oldWithOld, X509 **newWithNew, - X509 **newWithOld, X509 **oldWithNew) -{ - X509 *oldWithOld_copy = NULL, *my_newWithOld, *my_oldWithNew; - OSSL_CMP_ITAV *req, *itav; - int res = 0; - - if (newWithNew == NULL) { - ERR_raise(ERR_LIB_CMP, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - *newWithNew = NULL; - - if ((req = OSSL_CMP_ITAV_new_rootCaCert(oldWithOld)) == NULL) - return 0; - itav = get_genm_itav(ctx, req, NID_id_it_rootCaKeyUpdate, "rootCaKeyUpdate"); - if (itav == NULL) - return 0; - - if (!OSSL_CMP_ITAV_get0_rootCaKeyUpdate(itav, newWithNew, - &my_newWithOld, &my_oldWithNew)) - goto end; - /* no root CA cert update available */ - if (*newWithNew == NULL) { - res = 1; - goto end; - } - if ((oldWithOld_copy = X509_dup(oldWithOld)) == NULL && oldWithOld != NULL) - goto end; - if (!verify_ss_cert_trans(ctx, oldWithOld_copy, my_newWithOld, - *newWithNew, "newWithNew")) { - ERR_raise(ERR_LIB_CMP, CMP_R_INVALID_ROOTCAKEYUPDATE); - goto end; - } - if (oldWithOld != NULL && my_oldWithNew != NULL - && !verify_ss_cert_trans(ctx, *newWithNew, my_oldWithNew, - oldWithOld_copy, "oldWithOld")) { - ERR_raise(ERR_LIB_CMP, CMP_R_INVALID_ROOTCAKEYUPDATE); - goto end; - } - - if (!X509_up_ref(*newWithNew)) - goto end; - if (newWithOld != NULL && - (*newWithOld = my_newWithOld) != NULL && !X509_up_ref(*newWithOld)) - goto free; - if (oldWithNew == NULL || - (*oldWithNew = my_oldWithNew) == NULL || X509_up_ref(*oldWithNew)) { - res = 1; - goto end; - } - if (newWithOld != NULL) - X509_free(*newWithOld); - free: - X509_free(*newWithNew); - - end: - OSSL_CMP_ITAV_free(itav); - X509_free(oldWithOld_copy); - return res; -} diff --git a/openssl/src/crypto/cmp/cmp_hdr.c b/openssl/src/crypto/cmp/cmp_hdr.c index 4358b3887..8c553af61 100644 --- a/openssl/src/crypto/cmp/cmp_hdr.c +++ b/openssl/src/crypto/cmp/cmp_hdr.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -72,16 +72,6 @@ ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_recipNonce(const OSSL_CMP_PKIHEADER *hdr) return hdr->recipNonce; } -STACK_OF(OSSL_CMP_ITAV) - *OSSL_CMP_HDR_get0_geninfo_ITAVs(const OSSL_CMP_PKIHEADER *hdr) -{ - if (hdr == NULL) { - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); - return NULL; - } - return hdr->generalInfo; -} - /* a NULL-DN as an empty sequence of RDNs */ int ossl_cmp_general_name_is_NULL_DN(GENERAL_NAME *name) { @@ -286,7 +276,8 @@ int ossl_cmp_hdr_set_transactionID(OSSL_CMP_CTX *ctx, OSSL_CMP_PKIHEADER *hdr) if (!set_random(&ctx->transactionID, ctx, OSSL_CMP_TRANSACTIONID_LENGTH)) return 0; - tid = i2s_ASN1_OCTET_STRING(NULL, ctx->transactionID); + tid = OPENSSL_buf2hexstr(ctx->transactionID->data, + ctx->transactionID->length); if (tid != NULL) ossl_cmp_log1(DEBUG, ctx, "Starting new transaction with ID=%s", tid); @@ -311,12 +302,11 @@ int ossl_cmp_hdr_init(OSSL_CMP_CTX *ctx, OSSL_CMP_PKIHEADER *hdr) return 0; /* - * If no protection cert nor oldCert nor CSR nor subject is given, + * If neither protection cert nor oldCert nor subject are given, * sender name is not known to the client and thus set to NULL-DN */ sender = ctx->cert != NULL ? X509_get_subject_name(ctx->cert) : ctx->oldCert != NULL ? X509_get_subject_name(ctx->oldCert) : - ctx->p10CSR != NULL ? X509_REQ_get_subject_name(ctx->p10CSR) : ctx->subjectName; if (!ossl_cmp_hdr_set1_sender(hdr, sender)) return 0; diff --git a/openssl/src/crypto/cmp/cmp_http.c b/openssl/src/crypto/cmp/cmp_http.c index d08c362a7..6ac4212db 100644 --- a/openssl/src/crypto/cmp/cmp_http.c +++ b/openssl/src/crypto/cmp/cmp_http.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -14,6 +14,7 @@ #include #include +#include "internal/sockets.h" #include #include "cmp_local.h" @@ -24,15 +25,13 @@ #include #include #include +#include #include static int keep_alive(int keep_alive, int body_type) { if (keep_alive != 0 - /* - * Ask for persistent connection only if may need more round trips. - * Do so even with disableConfirm because polling might be needed. - */ + /* Ask for persistent connection only if may need more round trips */ && body_type != OSSL_CMP_PKIBODY_IR && body_type != OSSL_CMP_PKIBODY_CR && body_type != OSSL_CMP_PKIBODY_P10CR @@ -44,6 +43,7 @@ static int keep_alive(int keep_alive, int body_type) /* * Send the PKIMessage req and on success return the response, else NULL. + * Any previous error queue entries will likely be removed by ERR_clear_error(). */ OSSL_CMP_MSG *OSSL_CMP_MSG_http_perform(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *req) @@ -68,8 +68,7 @@ OSSL_CMP_MSG *OSSL_CMP_MSG_http_perform(OSSL_CMP_CTX *ctx, if (ctx->serverPort != 0) BIO_snprintf(server_port, sizeof(server_port), "%d", ctx->serverPort); - tls_used = ctx->tls_used >= 0 ? ctx->tls_used != 0 - : OSSL_CMP_CTX_get_http_cb_arg(ctx) != NULL; /* backward compat */ + tls_used = OSSL_CMP_CTX_get_http_cb_arg(ctx) != NULL; if (ctx->http_ctx == NULL) ossl_cmp_log3(DEBUG, ctx, "connecting to CMP server %s:%s%s", ctx->server, server_port, tls_used ? " using TLS" : ""); diff --git a/openssl/src/crypto/cmp/cmp_local.h b/openssl/src/crypto/cmp/cmp_local.h index 89f05d753..3da021043 100644 --- a/openssl/src/crypto/cmp/cmp_local.h +++ b/openssl/src/crypto/cmp/cmp_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2007-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -25,7 +25,7 @@ # include # include "crypto/x509.h" -# define IS_NULL_DN(name) (X509_NAME_get_entry(name, 0) == NULL) +#define IS_NULL_DN(name) (X509_NAME_get_entry(name, 0) == NULL) /* * this structure is used to store the context for CMP sessions @@ -49,13 +49,10 @@ struct ossl_cmp_ctx_st { int keep_alive; /* persistent connection: 0=no, 1=prefer, 2=require */ int msg_timeout; /* max seconds to wait for each CMP message round trip */ int total_timeout; /* max number of seconds an enrollment may take, incl. */ - int tls_used; /* whether to use TLS for client-side HTTP connections */ /* attempts polling for a response if a 'waiting' PKIStatus is received */ time_t end_time; /* session start time + totaltimeout */ -# ifndef OPENSSL_NO_HTTP OSSL_HTTP_bio_cb_t http_cb; void *http_cb_arg; /* allows to store optional argument to cb */ -# endif /* server authentication */ /* @@ -64,7 +61,6 @@ struct ossl_cmp_ctx_st { * certificate responses (ip/cp/kup), revocation responses (rp), and PKIConf */ int unprotectedErrors; - int noCacheExtraCerts; X509 *srvCert; /* certificate used to identify the server */ X509 *validatedSrvCert; /* caches any already validated server cert */ X509_NAME *expected_sender; /* expected sender in header of response */ @@ -82,7 +78,7 @@ struct ossl_cmp_ctx_st { X509 *cert; /* protection cert used to identify and sign for MSG_SIG_ALG */ STACK_OF(X509) *chain; /* (cached) chain of protection cert including it */ EVP_PKEY *pkey; /* the key pair corresponding to cert */ - ASN1_OCTET_STRING *referenceValue; /* optional username for MSG_MAC_ALG */ + ASN1_OCTET_STRING *referenceValue; /* optional user name for MSG_MAC_ALG */ ASN1_OCTET_STRING *secretValue; /* password/shared secret for MSG_MAC_ALG */ /* PBMParameters for MSG_MAC_ALG */ size_t pbm_slen; /* salt length, currently fixed to 16 */ @@ -96,7 +92,6 @@ struct ossl_cmp_ctx_st { ASN1_OCTET_STRING *transactionID; /* the current transaction ID */ ASN1_OCTET_STRING *senderNonce; /* last nonce sent */ ASN1_OCTET_STRING *recipNonce; /* last nonce received */ - ASN1_OCTET_STRING *first_senderNonce; /* sender nonce when starting to poll */ ASN1_UTF8STRING *freeText; /* optional string to include each msg */ STACK_OF(OSSL_CMP_ITAV) *geninfo_ITAVs; int implicitConfirm; /* set implicitConfirm in IR/KUR/CR messages */ @@ -106,8 +101,7 @@ struct ossl_cmp_ctx_st { /* certificate template */ EVP_PKEY *newPkey; /* explicit new private/public key for cert enrollment */ int newPkey_priv; /* flag indicating if newPkey contains private key */ - X509_NAME *issuer; /* issuer name to used in cert template, also in rr */ - ASN1_INTEGER *serialNumber; /* certificate serial number to use in rr */ + X509_NAME *issuer; /* issuer name to used in cert template */ int days; /* Number of days new certificates are asked to be valid for */ X509_NAME *subjectName; /* subject name to be used in cert template */ STACK_OF(GENERAL_NAME) *subjectAltNames; /* to add to the cert template */ @@ -124,7 +118,7 @@ struct ossl_cmp_ctx_st { int revocationReason; /* revocation reason code to be included in RR */ STACK_OF(OSSL_CMP_ITAV) *genm_ITAVs; /* content of general message */ - /* result returned in responses, so far supporting only one certResponse */ + /* result returned in responses */ int status; /* PKIStatus of last received IP/CP/KUP/RP/error or -1 */ OSSL_CMP_PKIFREETEXT *statusString; /* of last IP/CP/KUP/RP/error */ int failInfoCode; /* failInfoCode of last received IP/CP/KUP/error, or -1 */ @@ -208,9 +202,6 @@ typedef struct ossl_cmp_cakeyupdanncontent_st { } OSSL_CMP_CAKEYUPDANNCONTENT; DECLARE_ASN1_FUNCTIONS(OSSL_CMP_CAKEYUPDANNCONTENT) -typedef struct ossl_cmp_rootcakeyupdate_st OSSL_CMP_ROOTCAKEYUPDATE; -DECLARE_ASN1_FUNCTIONS(OSSL_CMP_ROOTCAKEYUPDATE) - /*- * declared already here as it will be used in OSSL_CMP_MSG (nested) and * infoType and infoValue @@ -256,14 +247,6 @@ struct ossl_cmp_itav_st { OSSL_CMP_MSGS *origPKIMessage; /* NID_id_it_suppLangTags - Supported Language Tags */ STACK_OF(ASN1_UTF8STRING) *suppLangTagsValue; - /* NID_id_it_certProfile - Certificate Profile */ - STACK_OF(ASN1_UTF8STRING) *certProfile; - /* NID_id_it_caCerts - CA Certificates */ - STACK_OF(X509) *caCerts; - /* NID_id_it_rootCaCert - Root CA Certificate */ - X509 *rootCaCert; - /* NID_id_it_rootCaKeyUpdate - Root CA Certificate Update */ - OSSL_CMP_ROOTCAKEYUPDATE *rootCaKeyUpdate; /* this is to be used for so far undeclared objects */ ASN1_TYPE *other; } infoValue; @@ -386,15 +369,13 @@ DECLARE_ASN1_FUNCTIONS(OSSL_CMP_ERRORMSGCONTENT) * -- as is used to create and verify the certificate signature * certReqId INTEGER, * -- to match this confirmation with the corresponding req/rep - * statusInfo PKIStatusInfo OPTIONAL, - * hashAlg [0] AlgorithmIdentifier OPTIONAL + * statusInfo PKIStatusInfo OPTIONAL * } */ struct ossl_cmp_certstatus_st { ASN1_OCTET_STRING *certHash; ASN1_INTEGER *certReqId; OSSL_CMP_PKISI *statusInfo; - X509_ALGOR *hashAlg; /* 0 */ } /* OSSL_CMP_CERTSTATUS */; DECLARE_ASN1_FUNCTIONS(OSSL_CMP_CERTSTATUS) typedef STACK_OF(OSSL_CMP_CERTSTATUS) OSSL_CMP_CERTCONFIRMCONTENT; @@ -465,7 +446,7 @@ DECLARE_ASN1_FUNCTIONS(OSSL_CMP_POLLREPCONTENT) /*- * PKIHeader ::= SEQUENCE { - * pvno INTEGER { cmp1999(1), cmp2000(2), cmp2021(3) }, + * pvno INTEGER { cmp1999(1), cmp2000(2) }, * sender GeneralName, * -- identifies the sender * recipient GeneralName, @@ -727,7 +708,6 @@ DECLARE_ASN1_FUNCTIONS(OSSL_CMP_PROTECTEDPART) * } -- or HMAC [RFC2104, RFC2202]) */ /*- - * Not supported: * id-DHBasedMac OBJECT IDENTIFIER ::= {1 2 840 113533 7 66 30} * DHBMParameter ::= SEQUENCE { * owf AlgorithmIdentifier, @@ -750,21 +730,6 @@ DECLARE_ASN1_FUNCTIONS(OSSL_CMP_PROTECTEDPART) * } */ -/* - * RootCaKeyUpdateContent ::= SEQUENCE { - * newWithNew CMPCertificate, - * newWithOld [0] CMPCertificate OPTIONAL, - * oldWithNew [1] CMPCertificate OPTIONAL - * } - */ - -struct ossl_cmp_rootcakeyupdate_st { - X509 *newWithNew; - X509 *newWithOld; - X509 *oldWithNew; -} /* OSSL_CMP_ROOTCAKEYUPDATE */; -DECLARE_ASN1_FUNCTIONS(OSSL_CMP_ROOTCAKEYUPDATE) - /* from cmp_asn.c */ int ossl_cmp_asn1_get_int(const ASN1_INTEGER *a); @@ -812,7 +777,7 @@ int ossl_cmp_print_log(OSSL_CMP_severity level, const OSSL_CMP_CTX *ctx, # define ossl_cmp_info(ctx, msg) ossl_cmp_log(INFO, ctx, msg) # define ossl_cmp_debug(ctx, msg) ossl_cmp_log(DEBUG, ctx, msg) # define ossl_cmp_trace(ctx, msg) ossl_cmp_log(TRACE, ctx, msg) -int ossl_cmp_ctx_set1_validatedSrvCert(OSSL_CMP_CTX *ctx, X509 *cert); +int ossl_cmp_ctx_set0_validatedSrvCert(OSSL_CMP_CTX *ctx, X509 *cert); int ossl_cmp_ctx_set_status(OSSL_CMP_CTX *ctx, int status); int ossl_cmp_ctx_set0_statusString(OSSL_CMP_CTX *ctx, OSSL_CMP_PKIFREETEXT *text); @@ -824,9 +789,6 @@ int ossl_cmp_ctx_set1_extraCertsIn(OSSL_CMP_CTX *ctx, STACK_OF(X509) *extraCertsIn); int ossl_cmp_ctx_set1_recipNonce(OSSL_CMP_CTX *ctx, const ASN1_OCTET_STRING *nonce); -EVP_PKEY *ossl_cmp_ctx_get0_newPubkey(const OSSL_CMP_CTX *ctx); -int ossl_cmp_ctx_set1_first_senderNonce(OSSL_CMP_CTX *ctx, - const ASN1_OCTET_STRING *nonce); /* from cmp_status.c */ int ossl_cmp_pkisi_get_status(const OSSL_CMP_PKISI *si); @@ -890,9 +852,7 @@ int ossl_cmp_hdr_init(OSSL_CMP_CTX *ctx, OSSL_CMP_PKIHEADER *hdr); # define OSSL_CMP_PKIBODY_POLLREP 26 # define OSSL_CMP_PKIBODY_TYPE_MAX OSSL_CMP_PKIBODY_POLLREP /* certReqId for the first - and so far only - certificate request */ -# define OSSL_CMP_CERTREQID 0 -# define OSSL_CMP_CERTREQID_NONE -1 -# define OSSL_CMP_CERTREQID_INVALID -2 +# define OSSL_CMP_CERTREQID 0 /* sequence id for the first - and so far only - revocation request */ # define OSSL_CMP_REVREQSID 0 int ossl_cmp_msg_set0_libctx(OSSL_CMP_MSG *msg, OSSL_LIB_CTX *libctx, @@ -925,8 +885,8 @@ OSSL_CMP_MSG *ossl_cmp_error_new(OSSL_CMP_CTX *ctx, const OSSL_CMP_PKISI *si, int unprotected); int ossl_cmp_certstatus_set0_certHash(OSSL_CMP_CERTSTATUS *certStatus, ASN1_OCTET_STRING *hash); -OSSL_CMP_MSG *ossl_cmp_certConf_new(OSSL_CMP_CTX *ctx, int certReqId, - int fail_info, const char *text); +OSSL_CMP_MSG *ossl_cmp_certConf_new(OSSL_CMP_CTX *ctx, int fail_info, + const char *text); OSSL_CMP_MSG *ossl_cmp_pollReq_new(OSSL_CMP_CTX *ctx, int crid); OSSL_CMP_MSG *ossl_cmp_pollRep_new(OSSL_CMP_CTX *ctx, int crid, int64_t poll_after); @@ -940,10 +900,9 @@ ossl_cmp_pollrepcontent_get0_pollrep(const OSSL_CMP_POLLREPCONTENT *prc, OSSL_CMP_CERTRESPONSE * ossl_cmp_certrepmessage_get0_certresponse(const OSSL_CMP_CERTREPMESSAGE *crm, int rid); -X509 *ossl_cmp_certresponse_get1_cert(const OSSL_CMP_CTX *ctx, - const OSSL_CMP_CERTRESPONSE *crep); +X509 *ossl_cmp_certresponse_get1_cert(const OSSL_CMP_CERTRESPONSE *crep, + const OSSL_CMP_CTX *ctx, EVP_PKEY *pkey); OSSL_CMP_MSG *ossl_cmp_msg_load(const char *file); -int ossl_cmp_is_error_with_waiting(const OSSL_CMP_MSG *msg); /* from cmp_protect.c */ int ossl_cmp_msg_add_extraCerts(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); @@ -963,10 +922,8 @@ int ossl_cmp_verify_popo(const OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg, int accept_RAVerified); /* from cmp_client.c */ -/* expected max time per msg round trip, used for last try during polling: */ -# define OSSL_CMP_EXPECTED_RESP_TIME 2 -int ossl_cmp_exchange_certConf(OSSL_CMP_CTX *ctx, int certReqId, - int fail_info, const char *txt); +int ossl_cmp_exchange_certConf(OSSL_CMP_CTX *ctx, int fail_info, + const char *txt); int ossl_cmp_exchange_error(OSSL_CMP_CTX *ctx, int status, int fail_info, const char *txt, int errorCode, const char *detail); diff --git a/openssl/src/crypto/cmp/cmp_msg.c b/openssl/src/crypto/cmp/cmp_msg.c index 4ba7b8108..2da95248a 100644 --- a/openssl/src/crypto/cmp/cmp_msg.c +++ b/openssl/src/crypto/cmp/cmp_msg.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -59,6 +59,7 @@ int ossl_cmp_msg_set0_libctx(OSSL_CMP_MSG *msg, OSSL_LIB_CTX *libctx, return 1; } + OSSL_CMP_PKIHEADER *OSSL_CMP_MSG_get0_header(const OSSL_CMP_MSG *msg) { if (msg == NULL) { @@ -100,34 +101,6 @@ int OSSL_CMP_MSG_get_bodytype(const OSSL_CMP_MSG *msg) return msg->body->type; } -X509_PUBKEY *OSSL_CMP_MSG_get0_certreq_publickey(const OSSL_CMP_MSG *msg) -{ - const OSSL_CRMF_MSGS *reqs; - const OSSL_CRMF_MSG *crm; - const OSSL_CRMF_CERTTEMPLATE *tmpl; - X509_PUBKEY *pubkey; - - switch (OSSL_CMP_MSG_get_bodytype(msg)) { - case OSSL_CMP_PKIBODY_IR: - case OSSL_CMP_PKIBODY_CR: - case OSSL_CMP_PKIBODY_KUR: - reqs = msg->body->value.ir; /* value.ir is same for cr and kur */ - if ((crm = sk_OSSL_CRMF_MSG_value(reqs, 0)) == NULL) { - ERR_raise(ERR_LIB_CMP, CMP_R_CERTREQMSG_NOT_FOUND); - return NULL; - } - if ((tmpl = OSSL_CRMF_MSG_get0_tmpl(crm)) == NULL - || (pubkey = OSSL_CRMF_CERTTEMPLATE_get0_publicKey(tmpl)) == NULL) { - ERR_raise(ERR_LIB_CMP, CRMF_R_POPO_MISSING_PUBLIC_KEY); - return NULL; - } - return pubkey; - default: - ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); - return NULL; - } -} - /* Add an extension to the referenced extension stack, which may be NULL */ static int add1_extension(X509_EXTENSIONS **pexts, int nid, int crit, void *ex) { @@ -280,16 +253,16 @@ OSSL_CMP_MSG *ossl_cmp_msg_create(OSSL_CMP_CTX *ctx, int bodytype) (sk_GENERAL_NAME_num((ctx)->subjectAltNames) > 0 \ || OSSL_CMP_CTX_reqExtensions_have_SAN(ctx) == 1) -static const X509_NAME *determine_subj(OSSL_CMP_CTX *ctx, int for_KUR, - const X509_NAME *ref_subj) +static const X509_NAME *determine_subj(OSSL_CMP_CTX *ctx, + const X509_NAME *ref_subj, + int for_KUR) { if (ctx->subjectName != NULL) return IS_NULL_DN(ctx->subjectName) ? NULL : ctx->subjectName; - if (ctx->p10CSR != NULL) /* first default is from any given CSR */ - return X509_REQ_get_subject_name(ctx->p10CSR); - if (for_KUR || !HAS_SAN(ctx)) + + if (ref_subj != NULL && (ctx->p10CSR != NULL || for_KUR || !HAS_SAN(ctx))) /* - * For KUR, copy subject from any reference cert as fallback. + * For KUR, copy subject from the reference. * For IR or CR, do the same only if there is no subjectAltName. */ return ref_subj; @@ -301,11 +274,12 @@ OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid) OSSL_CRMF_MSG *crm = NULL; X509 *refcert = ctx->oldCert != NULL ? ctx->oldCert : ctx->cert; /* refcert defaults to current client cert */ - EVP_PKEY *rkey = ossl_cmp_ctx_get0_newPubkey(ctx); + EVP_PKEY *rkey = OSSL_CMP_CTX_get0_newPkey(ctx, 0); STACK_OF(GENERAL_NAME) *default_sans = NULL; const X509_NAME *ref_subj = + ctx->p10CSR != NULL ? X509_REQ_get_subject_name(ctx->p10CSR) : refcert != NULL ? X509_get_subject_name(refcert) : NULL; - const X509_NAME *subject = determine_subj(ctx, for_KUR, ref_subj); + const X509_NAME *subject = determine_subj(ctx, ref_subj, for_KUR); const X509_NAME *issuer = ctx->issuer != NULL || refcert == NULL ? (IS_NULL_DN(ctx->issuer) ? NULL : ctx->issuer) : X509_get_issuer_name(refcert); @@ -313,9 +287,15 @@ OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid) /* RFC5280: subjectAltName MUST be critical if subject is null */ X509_EXTENSIONS *exts = NULL; + if (rkey == NULL && ctx->p10CSR != NULL) + rkey = X509_REQ_get0_pubkey(ctx->p10CSR); + if (rkey == NULL && refcert != NULL) + rkey = X509_get0_pubkey(refcert); + if (rkey == NULL) + rkey = ctx->pkey; /* default is independent of ctx->oldCert */ if (rkey == NULL) { #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_PUBLIC_KEY); + ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); return NULL; #endif } @@ -353,9 +333,9 @@ OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid) && (exts = X509_REQ_get_extensions(ctx->p10CSR)) == NULL) goto err; if (!ctx->SubjectAltName_nodefault && !HAS_SAN(ctx) && refcert != NULL - && (default_sans = X509V3_get_d2i(X509_get0_extensions(refcert), - NID_subject_alt_name, NULL, NULL)) - != NULL + && (default_sans = X509V3_get_d2i(X509_get0_extensions(refcert), + NID_subject_alt_name, NULL, NULL)) + != NULL && !add1_extension(&exts, NID_subject_alt_name, crit, default_sans)) goto err; if (ctx->reqExtensions != NULL /* augment/override existing ones */ @@ -432,9 +412,15 @@ OSSL_CMP_MSG *ossl_cmp_certreq_new(OSSL_CMP_CTX *ctx, int type, if (type != OSSL_CMP_PKIBODY_P10CR) { EVP_PKEY *privkey = OSSL_CMP_CTX_get0_newPkey(ctx, 1); - /* privkey is ctx->newPkey (if private, else NULL) or ctx->pkey */ - if (ctx->popoMethod >= OSSL_CRMF_POPO_SIGNATURE && privkey == NULL) { - ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_PRIVATE_KEY_FOR_POPO); + /* + * privkey is NULL in case ctx->newPkey does not include a private key. + * We then may try to use ctx->pkey as fallback/default, but only + * if ctx-> newPkey does not include a (non-matching) public key: + */ + if (privkey == NULL && OSSL_CMP_CTX_get0_newPkey(ctx, 0) == NULL) + privkey = ctx->pkey; /* default is independent of ctx->oldCert */ + if (ctx->popoMethod == OSSL_CRMF_POPO_SIGNATURE && privkey == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_PRIVATE_KEY); goto err; } if (crm == NULL) { @@ -478,7 +464,7 @@ OSSL_CMP_MSG *ossl_cmp_certrep_new(OSSL_CMP_CTX *ctx, int bodytype, OSSL_CMP_MSG *msg = NULL; OSSL_CMP_CERTREPMESSAGE *repMsg = NULL; OSSL_CMP_CERTRESPONSE *resp = NULL; - int status = OSSL_CMP_PKISTATUS_unspecified; + int status = -1; if (!ossl_assert(ctx != NULL && si != NULL)) return NULL; @@ -546,37 +532,27 @@ OSSL_CMP_MSG *ossl_cmp_certrep_new(OSSL_CMP_CTX *ctx, int bodytype, OSSL_CMP_MSG *ossl_cmp_rr_new(OSSL_CMP_CTX *ctx) { OSSL_CMP_MSG *msg = NULL; - const X509_NAME *issuer = NULL; - const X509_NAME *subject = NULL; - const ASN1_INTEGER *serialNumber = NULL; - EVP_PKEY *pubkey = NULL; OSSL_CMP_REVDETAILS *rd; int ret; - if (!ossl_assert(ctx != NULL - && (ctx->oldCert != NULL || ctx->p10CSR != NULL - || (ctx->serialNumber != NULL && ctx->issuer != NULL)))) + if (!ossl_assert(ctx != NULL && (ctx->oldCert != NULL + || ctx->p10CSR != NULL))) return NULL; if ((rd = OSSL_CMP_REVDETAILS_new()) == NULL) goto err; - if (ctx->serialNumber != NULL && ctx->issuer != NULL) { - issuer = ctx->issuer; - serialNumber = ctx->serialNumber; - } else if (ctx->oldCert != NULL) { - issuer = X509_get_issuer_name(ctx->oldCert); - serialNumber = X509_get0_serialNumber(ctx->oldCert); - } else if (ctx->p10CSR != NULL) { - pubkey = X509_REQ_get0_pubkey(ctx->p10CSR); - subject = X509_REQ_get_subject_name(ctx->p10CSR); - } else { - goto err; - } - /* Fill the template from the contents of the certificate to be revoked */ - ret = OSSL_CRMF_CERTTEMPLATE_fill(rd->certDetails, pubkey, subject, - issuer, serialNumber); + ret = ctx->oldCert != NULL + ? OSSL_CRMF_CERTTEMPLATE_fill(rd->certDetails, + NULL /* pubkey would be redundant */, + NULL /* subject would be redundant */, + X509_get_issuer_name(ctx->oldCert), + X509_get0_serialNumber(ctx->oldCert)) + : OSSL_CRMF_CERTTEMPLATE_fill(rd->certDetails, + X509_REQ_get0_pubkey(ctx->p10CSR), + X509_REQ_get_subject_name(ctx->p10CSR), + NULL, NULL); if (!ret) goto err; @@ -710,7 +686,7 @@ int ossl_cmp_msg_gen_push1_ITAVs(OSSL_CMP_MSG *msg, } /* - * Creates a new General Message/Response with a copy of the given itav stack + * Creates a new General Message/Response with an empty itav stack * returns a pointer to the PKIMessage on success, NULL on error */ static OSSL_CMP_MSG *gen_new(OSSL_CMP_CTX *ctx, @@ -725,7 +701,8 @@ static OSSL_CMP_MSG *gen_new(OSSL_CMP_CTX *ctx, if ((msg = ossl_cmp_msg_create(ctx, body_type)) == NULL) return NULL; - if (itavs != NULL && !ossl_cmp_msg_gen_push1_ITAVs(msg, itavs)) + if (ctx->genm_ITAVs != NULL + && !ossl_cmp_msg_gen_push1_ITAVs(msg, itavs)) goto err; if (!ossl_cmp_msg_protect(ctx, msg)) @@ -819,19 +796,15 @@ int ossl_cmp_certstatus_set0_certHash(OSSL_CMP_CERTSTATUS *certStatus, return 1; } -OSSL_CMP_MSG *ossl_cmp_certConf_new(OSSL_CMP_CTX *ctx, int certReqId, - int fail_info, const char *text) +OSSL_CMP_MSG *ossl_cmp_certConf_new(OSSL_CMP_CTX *ctx, int fail_info, + const char *text) { OSSL_CMP_MSG *msg = NULL; OSSL_CMP_CERTSTATUS *certStatus = NULL; - EVP_MD *md; - int is_fallback; ASN1_OCTET_STRING *certHash = NULL; OSSL_CMP_PKISI *sinfo; - if (!ossl_assert(ctx != NULL && ctx->newCert != NULL - && (certReqId == OSSL_CMP_CERTREQID - || certReqId == OSSL_CMP_CERTREQID_NONE))) + if (!ossl_assert(ctx != NULL && ctx->newCert != NULL)) return NULL; if ((unsigned)fail_info > OSSL_CMP_PKIFAILUREINFO_MAX_BIT_PATTERN) { @@ -845,31 +818,18 @@ OSSL_CMP_MSG *ossl_cmp_certConf_new(OSSL_CMP_CTX *ctx, int certReqId, if ((certStatus = OSSL_CMP_CERTSTATUS_new()) == NULL) goto err; /* consume certStatus into msg right away so it gets deallocated with msg */ - if (sk_OSSL_CMP_CERTSTATUS_push(msg->body->value.certConf, certStatus) < 1) { - OSSL_CMP_CERTSTATUS_free(certStatus); + if (!sk_OSSL_CMP_CERTSTATUS_push(msg->body->value.certConf, certStatus)) goto err; - } - /* set the ID of the certReq */ - if (!ASN1_INTEGER_set(certStatus->certReqId, certReqId)) + if (!ASN1_INTEGER_set(certStatus->certReqId, OSSL_CMP_CERTREQID)) goto err; - - certStatus->hashAlg = NULL; /* * The hash of the certificate, using the same hash algorithm * as is used to create and verify the certificate signature. - * If not available, a fallback hash algorithm is used. + * If not available, a default hash algorithm is used. */ - if ((certHash = X509_digest_sig(ctx->newCert, &md, &is_fallback)) == NULL) + if ((certHash = X509_digest_sig(ctx->newCert, NULL, NULL)) == NULL) goto err; - if (is_fallback) { - if (!ossl_cmp_hdr_set_pvno(msg->header, OSSL_CMP_PVNO_3)) - goto err; - if ((certStatus->hashAlg = X509_ALGOR_new()) == NULL) - goto err; - X509_ALGOR_set_md(certStatus->hashAlg, md); - } - EVP_MD_free(md); if (!ossl_cmp_certstatus_set0_certHash(certStatus, certHash)) goto err; @@ -1007,11 +967,12 @@ static int suitable_rid(const ASN1_INTEGER *certReqId, int rid) { int trid; - if (rid == OSSL_CMP_CERTREQID_NONE) + if (rid == -1) return 1; trid = ossl_cmp_asn1_get_int(certReqId); - if (trid <= OSSL_CMP_CERTREQID_INVALID) { + + if (trid == -1) { ERR_raise(ERR_LIB_CMP, CMP_R_BAD_REQUEST_ID); return 0; } @@ -1072,15 +1033,14 @@ ossl_cmp_certrepmessage_get0_certresponse(const OSSL_CMP_CERTREPMESSAGE *crm, /*- * Retrieve the newly enrolled certificate from the given certResponse crep. - * Uses libctx and propq from ctx, in case of indirect POPO also private key. + * In case of indirect POPO uses the libctx and propq from ctx and private key. * Returns a pointer to a copy of the found certificate, or NULL if not found. */ -X509 *ossl_cmp_certresponse_get1_cert(const OSSL_CMP_CTX *ctx, - const OSSL_CMP_CERTRESPONSE *crep) +X509 *ossl_cmp_certresponse_get1_cert(const OSSL_CMP_CERTRESPONSE *crep, + const OSSL_CMP_CTX *ctx, EVP_PKEY *pkey) { OSSL_CMP_CERTORENCCERT *coec; X509 *crt = NULL; - EVP_PKEY *pkey; if (!ossl_assert(crep != NULL && ctx != NULL)) return NULL; @@ -1093,8 +1053,6 @@ X509 *ossl_cmp_certresponse_get1_cert(const OSSL_CMP_CTX *ctx, break; case OSSL_CMP_CERTORENCCERT_ENCRYPTEDCERT: /* cert encrypted for indirect PoP; RFC 4210, 5.2.8.2 */ - pkey = OSSL_CMP_CTX_get0_newPkey(ctx, 1); - /* pkey is ctx->newPkey (if private, else NULL) or ctx->pkey */ if (pkey == NULL) { ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_PRIVATE_KEY); return NULL; @@ -1128,20 +1086,6 @@ int OSSL_CMP_MSG_update_transactionID(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) || ossl_cmp_msg_protect(ctx, msg); } -int OSSL_CMP_MSG_update_recipNonce(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) -{ - if (ctx == NULL || msg == NULL || msg->header == NULL) { - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); - return 0; - } - if (ctx->recipNonce == NULL) /* nothing to do for 1st msg in transaction */ - return 1; - if (!ossl_cmp_asn1_octet_string_set1(&msg->header->recipNonce, - ctx->recipNonce)) - return 0; - return msg->header->protectionAlg == NULL || ossl_cmp_msg_protect(ctx, msg); -} - OSSL_CMP_MSG *OSSL_CMP_MSG_read(const char *file, OSSL_LIB_CTX *libctx, const char *propq) { @@ -1154,8 +1098,8 @@ OSSL_CMP_MSG *OSSL_CMP_MSG_read(const char *file, OSSL_LIB_CTX *libctx, } msg = OSSL_CMP_MSG_new(libctx, propq); - if (msg == NULL) { - ERR_raise(ERR_LIB_CMP, ERR_R_CMP_LIB); + if (msg == NULL){ + ERR_raise(ERR_LIB_CMP, ERR_R_MALLOC_FAILURE); return NULL; } @@ -1226,13 +1170,3 @@ int i2d_OSSL_CMP_MSG_bio(BIO *bio, const OSSL_CMP_MSG *msg) { return ASN1_i2d_bio_of(OSSL_CMP_MSG, i2d_OSSL_CMP_MSG, bio, msg); } - -int ossl_cmp_is_error_with_waiting(const OSSL_CMP_MSG *msg) -{ - if (!ossl_assert(msg != NULL)) - return 0; - - return (OSSL_CMP_MSG_get_bodytype(msg) == OSSL_CMP_PKIBODY_ERROR - && ossl_cmp_pkisi_get_status(msg->body->value.error->pKIStatusInfo) - == OSSL_CMP_PKISTATUS_waiting); -} diff --git a/openssl/src/crypto/cmp/cmp_protect.c b/openssl/src/crypto/cmp/cmp_protect.c index f59fee44e..a7ca580cc 100644 --- a/openssl/src/crypto/cmp/cmp_protect.c +++ b/openssl/src/crypto/cmp/cmp_protect.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -10,7 +10,6 @@ */ #include "cmp_local.h" -#include "crypto/asn1.h" /* explicit #includes not strictly needed since implied by the above: */ #include @@ -22,11 +21,9 @@ /* * This function is also used by the internal verify_PBMAC() in cmp_vfy.c. * - * Calculate protection for |msg| according to |msg->header->protectionAlg| + * Calculate protection for given PKImessage according to + * the algorithm and parameters in the message header's protectionAlg * using the credentials, library context, and property criteria in the ctx. - * Unless |msg->header->protectionAlg| is PasswordBasedMAC, - * its value is completed according to |ctx->pkey| and |ctx->digest|, - * where the latter irrelevant in the case of Edwards curves. * * returns ASN1_BIT_STRING representing the protection on success, else NULL */ @@ -93,9 +90,10 @@ ASN1_BIT_STRING *ossl_cmp_calc_protection(const OSSL_CMP_CTX *ctx, goto end; if ((prot = ASN1_BIT_STRING_new()) == NULL) - goto end; - /* OpenSSL by default encodes all bit strings as ASN.1 NamedBitList */ - ossl_asn1_string_set_bits_left(prot, 0); + return NULL; + /* OpenSSL defaults all bit strings to be encoded as ASN.1 NamedBitList */ + prot->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + prot->flags |= ASN1_STRING_FLAG_BITS_LEFT; if (!ASN1_BIT_STRING_set(prot, protection, sig_len)) { ASN1_BIT_STRING_free(prot); prot = NULL; @@ -106,22 +104,23 @@ ASN1_BIT_STRING *ossl_cmp_calc_protection(const OSSL_CMP_CTX *ctx, OPENSSL_free(prot_part_der); return prot; } else { - const EVP_MD *md = ctx->digest; - char name[80] = ""; + int md_nid; + const EVP_MD *md = NULL; if (ctx->pkey == NULL) { ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_KEY_INPUT_FOR_CREATING_PROTECTION); return NULL; } - if (EVP_PKEY_get_default_digest_name(ctx->pkey, name, sizeof(name)) > 0 - && strcmp(name, "UNDEF") == 0) /* at least for Ed25519, Ed448 */ - md = NULL; + if (!OBJ_find_sigid_algs(OBJ_obj2nid(algorOID), &md_nid, NULL) + || (md = EVP_get_digestbynid(md_nid)) == NULL) { + ERR_raise(ERR_LIB_CMP, CMP_R_UNKNOWN_ALGORITHM_ID); + return NULL; + } if ((prot = ASN1_BIT_STRING_new()) == NULL) return NULL; - if (ASN1_item_sign_ex(ASN1_ITEM_rptr(OSSL_CMP_PROTECTEDPART), - msg->header->protectionAlg, /* sets X509_ALGOR */ + if (ASN1_item_sign_ex(ASN1_ITEM_rptr(OSSL_CMP_PROTECTEDPART), NULL, NULL, prot, &prot_part, NULL, ctx->pkey, md, ctx->libctx, ctx->propq)) return prot; @@ -130,7 +129,6 @@ ASN1_BIT_STRING *ossl_cmp_calc_protection(const OSSL_CMP_CTX *ctx, } } -/* ctx is not const just because ctx->chain may get adapted */ int ossl_cmp_msg_add_extraCerts(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) { if (!ossl_assert(ctx != NULL && msg != NULL)) @@ -186,16 +184,15 @@ int ossl_cmp_msg_add_extraCerts(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) * Create an X509_ALGOR structure for PasswordBasedMAC protection based on * the pbm settings in the context */ -static X509_ALGOR *pbmac_algor(const OSSL_CMP_CTX *ctx) +static int set_pbmac_algor(const OSSL_CMP_CTX *ctx, X509_ALGOR **alg) { OSSL_CRMF_PBMPARAMETER *pbm = NULL; unsigned char *pbm_der = NULL; int pbm_der_len; ASN1_STRING *pbm_str = NULL; - X509_ALGOR *alg = NULL; if (!ossl_assert(ctx != NULL)) - return NULL; + return 0; pbm = OSSL_CRMF_pbmp_new(ctx->libctx, ctx->pbm_slen, EVP_MD_get_type(ctx->pbm_owf), ctx->pbm_itercnt, @@ -203,18 +200,47 @@ static X509_ALGOR *pbmac_algor(const OSSL_CMP_CTX *ctx) pbm_str = ASN1_STRING_new(); if (pbm == NULL || pbm_str == NULL) goto err; + if ((pbm_der_len = i2d_OSSL_CRMF_PBMPARAMETER(pbm, &pbm_der)) < 0) goto err; + if (!ASN1_STRING_set(pbm_str, pbm_der, pbm_der_len)) goto err; - alg = ossl_X509_ALGOR_from_nid(NID_id_PasswordBasedMAC, - V_ASN1_SEQUENCE, pbm_str); + if (*alg == NULL && (*alg = X509_ALGOR_new()) == NULL) + goto err; + OPENSSL_free(pbm_der); + + X509_ALGOR_set0(*alg, OBJ_nid2obj(NID_id_PasswordBasedMAC), + V_ASN1_SEQUENCE, pbm_str); + OSSL_CRMF_PBMPARAMETER_free(pbm); + return 1; + err: - if (alg == NULL) - ASN1_STRING_free(pbm_str); + ASN1_STRING_free(pbm_str); OPENSSL_free(pbm_der); OSSL_CRMF_PBMPARAMETER_free(pbm); - return alg; + return 0; +} + +static int set_sig_algor(const OSSL_CMP_CTX *ctx, X509_ALGOR **alg) +{ + int nid = 0; + ASN1_OBJECT *algo = NULL; + + if (!OBJ_find_sigid_by_algs(&nid, EVP_MD_get_type(ctx->digest), + EVP_PKEY_get_id(ctx->pkey))) { + ERR_raise(ERR_LIB_CMP, CMP_R_UNSUPPORTED_KEY_TYPE); + return 0; + } + if ((algo = OBJ_nid2obj(nid)) == NULL) + return 0; + if (*alg == NULL && (*alg = X509_ALGOR_new()) == NULL) + return 0; + + if (X509_ALGOR_set0(*alg, algo, V_ASN1_UNDEF, NULL)) + return 1; + ASN1_OBJECT_free(algo); + return 0; } static int set_senderKID(const OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg, @@ -225,7 +251,6 @@ static int set_senderKID(const OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg, return id == NULL || ossl_cmp_hdr_set1_senderKID(msg->header, id); } -/* ctx is not const just because ctx->chain may get adapted */ int ossl_cmp_msg_protect(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) { if (!ossl_assert(ctx != NULL && msg != NULL)) @@ -233,7 +258,6 @@ int ossl_cmp_msg_protect(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) /* * For the case of re-protection remove pre-existing protection. - * Does not remove any pre-existing extraCerts. */ X509_ALGOR_free(msg->header->protectionAlg); msg->header->protectionAlg = NULL; @@ -245,7 +269,7 @@ int ossl_cmp_msg_protect(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) goto err; } else if (ctx->secretValue != NULL) { /* use PasswordBasedMac according to 5.1.3.1 if secretValue is given */ - if ((msg->header->protectionAlg = pbmac_algor(ctx)) == NULL) + if (!set_pbmac_algor(ctx, &msg->header->protectionAlg)) goto err; if (!set_senderKID(ctx, msg, NULL)) goto err; @@ -264,7 +288,7 @@ int ossl_cmp_msg_protect(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) goto err; } - if ((msg->header->protectionAlg = X509_ALGOR_new()) == NULL) + if (!set_sig_algor(ctx, &msg->header->protectionAlg)) goto err; /* set senderKID to keyIdentifier of the cert according to 5.1.1 */ if (!set_senderKID(ctx, msg, X509_get0_subject_key_id(ctx->cert))) @@ -280,7 +304,6 @@ int ossl_cmp_msg_protect(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg) goto err; } if (!ctx->unprotectedSend - /* protect according to msg->header->protectionAlg partly set above */ && ((msg->protection = ossl_cmp_calc_protection(ctx, msg)) == NULL)) goto err; diff --git a/openssl/src/crypto/cmp/cmp_server.c b/openssl/src/crypto/cmp/cmp_server.c index 53c41bc96..7ce4662ae 100644 --- a/openssl/src/crypto/cmp/cmp_server.c +++ b/openssl/src/crypto/cmp/cmp_server.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -22,10 +22,8 @@ /* the context for the generic CMP server */ struct ossl_cmp_srv_ctx_st { - OSSL_CMP_CTX *ctx; /* CMP client context reused for transactionID etc. */ - void *custom_ctx; /* application-specific server context */ - int certReqId; /* of ir/cr/kur, OSSL_CMP_CERTREQID_NONE for p10cr */ - int polling; /* current transaction is in polling mode */ + OSSL_CMP_CTX *ctx; /* Client CMP context, partly reused for srv */ + void *custom_ctx; /* pointer to specific server context */ OSSL_CMP_SRV_cert_request_cb_t process_cert_request; OSSL_CMP_SRV_rr_cb_t process_rr; @@ -33,8 +31,6 @@ struct ossl_cmp_srv_ctx_st OSSL_CMP_SRV_error_cb_t process_error; OSSL_CMP_SRV_certConf_cb_t process_certConf; OSSL_CMP_SRV_pollReq_cb_t process_pollReq; - OSSL_CMP_SRV_delayed_delivery_cb_t delayed_delivery; - OSSL_CMP_SRV_clean_transaction_cb_t clean_transaction; int sendUnprotectedErrors; /* Send error and rejection msgs unprotected */ int acceptUnprotected; /* Accept requests with no/invalid prot. */ @@ -61,8 +57,6 @@ OSSL_CMP_SRV_CTX *OSSL_CMP_SRV_CTX_new(OSSL_LIB_CTX *libctx, const char *propq) if ((ctx->ctx = OSSL_CMP_CTX_new(libctx, propq)) == NULL) goto err; - ctx->certReqId = OSSL_CMP_CERTREQID_INVALID; - ctx->polling = 0; /* all other elements are initialized to 0 or NULL, respectively */ return ctx; @@ -93,19 +87,6 @@ int OSSL_CMP_SRV_CTX_init(OSSL_CMP_SRV_CTX *srv_ctx, void *custom_ctx, return 1; } -int OSSL_CMP_SRV_CTX_init_trans(OSSL_CMP_SRV_CTX *srv_ctx, - OSSL_CMP_SRV_delayed_delivery_cb_t delay, - OSSL_CMP_SRV_clean_transaction_cb_t clean) -{ - if (srv_ctx == NULL) { - ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); - return 0; - } - srv_ctx->delayed_delivery = delay; - srv_ctx->clean_transaction = clean; - return 1; -} - OSSL_CMP_CTX *OSSL_CMP_SRV_CTX_get0_cmp_ctx(const OSSL_CMP_SRV_CTX *srv_ctx) { if (srv_ctx == NULL) { @@ -166,46 +147,6 @@ int OSSL_CMP_SRV_CTX_set_grant_implicit_confirm(OSSL_CMP_SRV_CTX *srv_ctx, return 1; } -/* return error msg with waiting status if polling is initiated, else NULL */ -static OSSL_CMP_MSG *delayed_delivery(OSSL_CMP_SRV_CTX *srv_ctx, - const OSSL_CMP_MSG *req) -{ - int ret; - unsigned long err; - int status = OSSL_CMP_PKISTATUS_waiting, - fail_info = 0, errorCode = 0; - const char *txt = NULL, *details = NULL; - OSSL_CMP_PKISI *si; - OSSL_CMP_MSG *msg; - - if (!ossl_assert(srv_ctx != NULL && srv_ctx->ctx != NULL && req != NULL - && srv_ctx->delayed_delivery != NULL)) - return NULL; - - ret = srv_ctx->delayed_delivery(srv_ctx, req); - if (ret == 0) - return NULL; - if (ret == 1) { - srv_ctx->polling = 1; - } else { - status = OSSL_CMP_PKISTATUS_rejection; - fail_info = 1 << OSSL_CMP_PKIFAILUREINFO_systemFailure; - txt = "server application error"; - err = ERR_peek_error(); - errorCode = ERR_GET_REASON(err); - details = ERR_reason_error_string(err); - } - - si = OSSL_CMP_STATUSINFO_new(status, fail_info, txt); - if (si == NULL) - return NULL; - - msg = ossl_cmp_error_new(srv_ctx->ctx, si, errorCode, details, - srv_ctx->sendUnprotectedErrors); - OSSL_CMP_PKISI_free(si); - return msg; -} - /* * Processes an ir/cr/p10cr/kur and returns a certification response. * Only handles the first certification request contained in req @@ -243,7 +184,7 @@ static OSSL_CMP_MSG *process_cert_request(OSSL_CMP_SRV_CTX *srv_ctx, } if (OSSL_CMP_MSG_get_bodytype(req) == OSSL_CMP_PKIBODY_P10CR) { - certReqId = OSSL_CMP_CERTREQID_NONE; /* p10cr does not include an Id */ + certReqId = OSSL_CMP_CERTREQID; p10cr = req->body->value.p10cr; } else { OSSL_CRMF_MSGS *reqs = req->body->value.ir; /* same for cr and kur */ @@ -252,17 +193,13 @@ static OSSL_CMP_MSG *process_cert_request(OSSL_CMP_SRV_CTX *srv_ctx, ERR_raise(ERR_LIB_CMP, CMP_R_MULTIPLE_REQUESTS_NOT_SUPPORTED); return NULL; } - if ((crm = sk_OSSL_CRMF_MSG_value(reqs, 0)) == NULL) { + + if ((crm = sk_OSSL_CRMF_MSG_value(reqs, OSSL_CMP_CERTREQID)) == NULL) { ERR_raise(ERR_LIB_CMP, CMP_R_CERTREQMSG_NOT_FOUND); return NULL; } certReqId = OSSL_CRMF_MSG_get_certReqId(crm); - if (certReqId != OSSL_CMP_CERTREQID) { /* so far, only possible value */ - ERR_raise(ERR_LIB_CMP, CMP_R_BAD_REQUEST_ID); - return NULL; - } } - srv_ctx->certReqId = certReqId; if (!ossl_cmp_verify_popo(srv_ctx->ctx, req, srv_ctx->acceptRAVerified)) { /* Proof of possession could not be verified */ @@ -278,8 +215,6 @@ static OSSL_CMP_MSG *process_cert_request(OSSL_CMP_SRV_CTX *srv_ctx, &certOut, &chainOut, &caPubs); if (si == NULL) goto err; - if (ossl_cmp_pkisi_get_status(si) == OSSL_CMP_PKISTATUS_waiting) - srv_ctx->polling = 1; /* set OSSL_CMP_OPT_IMPLICIT_CONFIRM if and only if transaction ends */ if (!OSSL_CMP_CTX_set_option(srv_ctx->ctx, OSSL_CMP_OPT_IMPLICIT_CONFIRM, @@ -293,15 +228,14 @@ static OSSL_CMP_MSG *process_cert_request(OSSL_CMP_SRV_CTX *srv_ctx, msg = ossl_cmp_certrep_new(srv_ctx->ctx, bodytype, certReqId, si, certOut, NULL /* enc */, chainOut, caPubs, srv_ctx->sendUnprotectedErrors); - /* When supporting OSSL_CRMF_POPO_KEYENC, "enc" will need to be set */ if (msg == NULL) ERR_raise(ERR_LIB_CMP, CMP_R_ERROR_CREATING_CERTREP); err: OSSL_CMP_PKISI_free(si); X509_free(certOut); - OSSL_STACK_OF_X509_free(chainOut); - OSSL_STACK_OF_X509_free(caPubs); + sk_X509_pop_free(chainOut, X509_free); + sk_X509_pop_free(caPubs, X509_free); return msg; } @@ -323,8 +257,9 @@ static OSSL_CMP_MSG *process_rr(OSSL_CMP_SRV_CTX *srv_ctx, ERR_raise(ERR_LIB_CMP, CMP_R_MULTIPLE_REQUESTS_NOT_SUPPORTED); return NULL; } - details = sk_OSSL_CMP_REVDETAILS_value(req->body->value.rr, 0); - if (details == NULL) { + + if ((details = sk_OSSL_CMP_REVDETAILS_value(req->body->value.rr, + OSSL_CMP_REVREQSID)) == NULL) { ERR_raise(ERR_LIB_CMP, CMP_R_ERROR_PROCESSING_MESSAGE); return NULL; } @@ -403,7 +338,7 @@ static OSSL_CMP_MSG *process_certConf(OSSL_CMP_SRV_CTX *srv_ctx, num = sk_OSSL_CMP_CERTSTATUS_num(ccc); if (OSSL_CMP_CTX_get_option(ctx, OSSL_CMP_OPT_IMPLICIT_CONFIRM) == 1 - || ctx->status != OSSL_CMP_PKISTATUS_trans) { + || ctx->status != -2 /* transaction not open */) { ERR_raise(ERR_LIB_CMP, CMP_R_ERROR_UNEXPECTED_CERTCONF); return NULL; } @@ -413,7 +348,7 @@ static OSSL_CMP_MSG *process_certConf(OSSL_CMP_SRV_CTX *srv_ctx, } else { if (num > 1) ossl_cmp_warn(ctx, "All CertStatus but the first will be ignored"); - status = sk_OSSL_CMP_CERTSTATUS_value(ccc, 0); + status = sk_OSSL_CMP_CERTSTATUS_value(ccc, OSSL_CMP_CERTREQID); } if (status != NULL) { @@ -421,15 +356,11 @@ static OSSL_CMP_MSG *process_certConf(OSSL_CMP_SRV_CTX *srv_ctx, ASN1_OCTET_STRING *certHash = status->certHash; OSSL_CMP_PKISI *si = status->statusInfo; - if (certReqId != srv_ctx->certReqId) { - ERR_raise(ERR_LIB_CMP, CMP_R_BAD_REQUEST_ID); - return NULL; - } if (!srv_ctx->process_certConf(srv_ctx, req, certReqId, certHash, si)) return NULL; /* reason code may be: CMP_R_CERTHASH_UNMATCHED */ - if (si != NULL - && ossl_cmp_pkisi_get_status(si) != OSSL_CMP_PKISTATUS_accepted) { + if (si != NULL && ossl_cmp_pkisi_get_status(si) + != OSSL_CMP_PKISTATUS_accepted) { int pki_status = ossl_cmp_pkisi_get_status(si); const char *str = ossl_cmp_PKIStatus_to_string(pki_status); @@ -444,80 +375,19 @@ static OSSL_CMP_MSG *process_certConf(OSSL_CMP_SRV_CTX *srv_ctx, return msg; } -/* pollReq is handled separately, to avoid recursive call */ -static OSSL_CMP_MSG *process_non_polling_request(OSSL_CMP_SRV_CTX *srv_ctx, - const OSSL_CMP_MSG *req) -{ - OSSL_CMP_MSG *rsp = NULL; - - if (!ossl_assert(srv_ctx != NULL && srv_ctx->ctx != NULL && req != NULL - && req->body != NULL)) - return NULL; - - switch (OSSL_CMP_MSG_get_bodytype(req)) { - case OSSL_CMP_PKIBODY_IR: - case OSSL_CMP_PKIBODY_CR: - case OSSL_CMP_PKIBODY_P10CR: - case OSSL_CMP_PKIBODY_KUR: - if (srv_ctx->process_cert_request == NULL) - ERR_raise(ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY); - else - rsp = process_cert_request(srv_ctx, req); - break; - case OSSL_CMP_PKIBODY_RR: - if (srv_ctx->process_rr == NULL) - ERR_raise(ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY); - else - rsp = process_rr(srv_ctx, req); - break; - case OSSL_CMP_PKIBODY_GENM: - if (srv_ctx->process_genm == NULL) - ERR_raise(ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY); - else - rsp = process_genm(srv_ctx, req); - break; - case OSSL_CMP_PKIBODY_ERROR: - if (srv_ctx->process_error == NULL) - ERR_raise(ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY); - else - rsp = process_error(srv_ctx, req); - break; - case OSSL_CMP_PKIBODY_CERTCONF: - if (srv_ctx->process_certConf == NULL) - ERR_raise(ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY); - else - rsp = process_certConf(srv_ctx, req); - break; - - case OSSL_CMP_PKIBODY_POLLREQ: - ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); - break; - default: - ERR_raise(ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY); - break; - } - - return rsp; -} - static OSSL_CMP_MSG *process_pollReq(OSSL_CMP_SRV_CTX *srv_ctx, const OSSL_CMP_MSG *req) { OSSL_CMP_POLLREQCONTENT *prc; OSSL_CMP_POLLREQ *pr; int certReqId; - OSSL_CMP_MSG *orig_req; + OSSL_CMP_MSG *certReq; int64_t check_after = 0; OSSL_CMP_MSG *msg = NULL; if (!ossl_assert(srv_ctx != NULL && srv_ctx->ctx != NULL && req != NULL)) return NULL; - if (!srv_ctx->polling) { - ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); - return NULL; - } - prc = req->body->value.pollReq; if (sk_OSSL_CMP_POLLREQ_num(prc) != 1) { ERR_raise(ERR_LIB_CMP, CMP_R_MULTIPLE_REQUESTS_NOT_SUPPORTED); @@ -527,13 +397,12 @@ static OSSL_CMP_MSG *process_pollReq(OSSL_CMP_SRV_CTX *srv_ctx, pr = sk_OSSL_CMP_POLLREQ_value(prc, 0); certReqId = ossl_cmp_asn1_get_int(pr->certReqId); if (!srv_ctx->process_pollReq(srv_ctx, req, certReqId, - &orig_req, &check_after)) + &certReq, &check_after)) return NULL; - if (orig_req != NULL) { - srv_ctx->polling = 0; - msg = process_non_polling_request(srv_ctx, orig_req); - OSSL_CMP_MSG_free(orig_req); + if (certReq != NULL) { + msg = process_cert_request(srv_ctx, certReq); + OSSL_CMP_MSG_free(certReq); } else { if ((msg = ossl_cmp_pollRep_new(srv_ctx->ctx, certReqId, check_after)) == NULL) @@ -577,7 +446,7 @@ OSSL_CMP_MSG *OSSL_CMP_SRV_process_request(OSSL_CMP_SRV_CTX *srv_ctx, ASN1_OCTET_STRING *backup_secret; OSSL_CMP_PKIHEADER *hdr; int req_type, rsp_type; - int req_verified = 0; + int res; OSSL_CMP_MSG *rsp = NULL; if (srv_ctx == NULL || srv_ctx->ctx == NULL @@ -603,12 +472,6 @@ OSSL_CMP_MSG *OSSL_CMP_SRV_process_request(OSSL_CMP_SRV_CTX *srv_ctx, if (!OSSL_CMP_CTX_set1_recipient(ctx, hdr->sender->d.directoryName)) goto err; - if (srv_ctx->polling && req_type != OSSL_CMP_PKIBODY_POLLREQ - && req_type != OSSL_CMP_PKIBODY_ERROR) { - ERR_raise(ERR_LIB_CMP, CMP_R_EXPECTED_POLLREQ); - goto err; - } - switch (req_type) { case OSSL_CMP_PKIBODY_IR: case OSSL_CMP_PKIBODY_CR: @@ -618,8 +481,10 @@ OSSL_CMP_MSG *OSSL_CMP_SRV_process_request(OSSL_CMP_SRV_CTX *srv_ctx, case OSSL_CMP_PKIBODY_GENM: case OSSL_CMP_PKIBODY_ERROR: if (ctx->transactionID != NULL) { - char *tid = i2s_ASN1_OCTET_STRING(NULL, ctx->transactionID); + char *tid; + tid = OPENSSL_buf2hexstr(ctx->transactionID->data, + ctx->transactionID->length); if (tid != NULL) ossl_cmp_log1(WARN, ctx, "Assuming that last transaction with ID=%s got aborted", @@ -630,13 +495,6 @@ OSSL_CMP_MSG *OSSL_CMP_SRV_process_request(OSSL_CMP_SRV_CTX *srv_ctx, if (!OSSL_CMP_CTX_set1_transactionID(ctx, NULL) || !OSSL_CMP_CTX_set1_senderNonce(ctx, NULL)) goto err; - - if (srv_ctx->clean_transaction != NULL - && !srv_ctx->clean_transaction(srv_ctx, NULL)) { - ERR_raise(ERR_LIB_CMP, CMP_R_ERROR_PROCESSING_MESSAGE); - goto err; - } - break; default: /* transactionID should be already initialized */ @@ -648,25 +506,57 @@ OSSL_CMP_MSG *OSSL_CMP_SRV_process_request(OSSL_CMP_SRV_CTX *srv_ctx, } } - req_verified = ossl_cmp_msg_check_update(ctx, req, unprotected_exception, - srv_ctx->acceptUnprotected); + res = ossl_cmp_msg_check_update(ctx, req, unprotected_exception, + srv_ctx->acceptUnprotected); if (ctx->secretValue != NULL && ctx->pkey != NULL && ossl_cmp_hdr_get_protection_nid(hdr) != NID_id_PasswordBasedMAC) ctx->secretValue = NULL; /* use MSG_SIG_ALG when protecting rsp */ - if (!req_verified) + if (!res) goto err; - if (req_type == OSSL_CMP_PKIBODY_POLLREQ) { + switch (req_type) { + case OSSL_CMP_PKIBODY_IR: + case OSSL_CMP_PKIBODY_CR: + case OSSL_CMP_PKIBODY_P10CR: + case OSSL_CMP_PKIBODY_KUR: + if (srv_ctx->process_cert_request == NULL) + ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); + else + rsp = process_cert_request(srv_ctx, req); + break; + case OSSL_CMP_PKIBODY_RR: + if (srv_ctx->process_rr == NULL) + ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); + else + rsp = process_rr(srv_ctx, req); + break; + case OSSL_CMP_PKIBODY_GENM: + if (srv_ctx->process_genm == NULL) + ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); + else + rsp = process_genm(srv_ctx, req); + break; + case OSSL_CMP_PKIBODY_ERROR: + if (srv_ctx->process_error == NULL) + ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); + else + rsp = process_error(srv_ctx, req); + break; + case OSSL_CMP_PKIBODY_CERTCONF: + if (srv_ctx->process_certConf == NULL) + ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); + else + rsp = process_certConf(srv_ctx, req); + break; + case OSSL_CMP_PKIBODY_POLLREQ: if (srv_ctx->process_pollReq == NULL) - ERR_raise(ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY); + ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); else rsp = process_pollReq(srv_ctx, req); - } else { - if (srv_ctx->delayed_delivery != NULL - && (rsp = delayed_delivery(srv_ctx, req)) != NULL) { - goto err; - } - rsp = process_non_polling_request(srv_ctx, req); + break; + default: + ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY); + break; } err: @@ -676,18 +566,11 @@ OSSL_CMP_MSG *OSSL_CMP_SRV_process_request(OSSL_CMP_SRV_CTX *srv_ctx, int flags = 0; unsigned long err = ERR_peek_error_data(&data, &flags); int fail_info = 1 << OSSL_CMP_PKIFAILUREINFO_badRequest; - /* fail_info is not very specific */ OSSL_CMP_PKISI *si = NULL; - if (!req_verified) { - /* - * Above ossl_cmp_msg_check_update() was not successfully executed, - * which normally would set ctx->transactionID and ctx->recipNonce. - * So anyway try to provide the right transactionID and recipNonce, - * while ignoring any (extra) error in next two function calls. - */ - if (ctx->transactionID == NULL) - (void)OSSL_CMP_CTX_set1_transactionID(ctx, hdr->transactionID); + if (ctx->transactionID == NULL) { + /* ignore any (extra) error in next two function calls: */ + (void)OSSL_CMP_CTX_set1_transactionID(ctx, hdr->transactionID); (void)ossl_cmp_ctx_set1_recipNonce(ctx, hdr->senderNonce); } @@ -712,8 +595,8 @@ OSSL_CMP_MSG *OSSL_CMP_SRV_process_request(OSSL_CMP_SRV_CTX *srv_ctx, else ossl_cmp_log(ERR, ctx, "cannot send proper CMP response"); - /* determine whether to keep the transaction open or not */ - ctx->status = OSSL_CMP_PKISTATUS_trans; + /* possibly close the transaction */ + ctx->status = -2; /* this indicates transaction is open */ switch (rsp_type) { case OSSL_CMP_PKIBODY_IP: case OSSL_CMP_PKIBODY_CP: @@ -722,22 +605,13 @@ OSSL_CMP_MSG *OSSL_CMP_SRV_process_request(OSSL_CMP_SRV_CTX *srv_ctx, break; /* fall through */ - case OSSL_CMP_PKIBODY_ERROR: - if (rsp != NULL && ossl_cmp_is_error_with_waiting(rsp)) - break; - /* fall through */ - case OSSL_CMP_PKIBODY_RP: case OSSL_CMP_PKIBODY_PKICONF: case OSSL_CMP_PKIBODY_GENP: - /* Other terminating response message types are not supported */ - srv_ctx->certReqId = OSSL_CMP_CERTREQID_INVALID; - /* Prepare for next transaction, ignoring any errors here: */ - if (srv_ctx->clean_transaction != NULL) - (void)srv_ctx->clean_transaction(srv_ctx, ctx->transactionID); + case OSSL_CMP_PKIBODY_ERROR: (void)OSSL_CMP_CTX_set1_transactionID(ctx, NULL); (void)OSSL_CMP_CTX_set1_senderNonce(ctx, NULL); - ctx->status = OSSL_CMP_PKISTATUS_unspecified; /* transaction closed */ + ctx->status = -1; /* transaction closed */ default: /* not closing transaction in other cases */ break; diff --git a/openssl/src/crypto/cmp/cmp_status.c b/openssl/src/crypto/cmp/cmp_status.c index b9086d84f..46be6b689 100644 --- a/openssl/src/crypto/cmp/cmp_status.c +++ b/openssl/src/crypto/cmp/cmp_status.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -30,12 +30,9 @@ int ossl_cmp_pkisi_get_status(const OSSL_CMP_PKISI *si) { - int res ; - if (!ossl_assert(si != NULL && si->status != NULL)) return -1; - res = ossl_cmp_asn1_get_int(si->status); - return res == -2 ? -1 : res; + return ossl_cmp_asn1_get_int(si->status); } const char *ossl_cmp_PKIStatus_to_string(int status) @@ -76,10 +73,9 @@ int ossl_cmp_pkisi_get_pkifailureinfo(const OSSL_CMP_PKISI *si) if (!ossl_assert(si != NULL)) return -1; - if (si->failInfo != NULL) - for (i = 0; i <= OSSL_CMP_PKIFAILUREINFO_MAX; i++) - if (ASN1_BIT_STRING_get_bit(si->failInfo, i)) - res |= 1 << i; + for (i = 0; i <= OSSL_CMP_PKIFAILUREINFO_MAX; i++) + if (ASN1_BIT_STRING_get_bit(si->failInfo, i)) + res |= 1 << i; return res; } @@ -184,20 +180,17 @@ char *snprint_PKIStatusInfo_parts(int status, int fail_info, || (status_string = ossl_cmp_PKIStatus_to_string(status)) == NULL) return NULL; -#define ADVANCE_BUFFER \ - if (printed_chars < 0 || (size_t)printed_chars >= bufsize) \ - return NULL; \ - write_ptr += printed_chars; \ - bufsize -= printed_chars; +#define ADVANCE_BUFFER \ + if (printed_chars < 0 || (size_t)printed_chars >= bufsize) \ + return NULL; \ + write_ptr += printed_chars; \ + bufsize -= printed_chars; printed_chars = BIO_snprintf(write_ptr, bufsize, "%s", status_string); ADVANCE_BUFFER; - /* - * failInfo is optional and may be empty; - * if present, print failInfo before statusString because it is more concise - */ - if (fail_info != -1 && fail_info != 0) { + /* failInfo is optional and may be empty */ + if (fail_info != 0) { printed_chars = BIO_snprintf(write_ptr, bufsize, "; PKIFailureInfo: "); ADVANCE_BUFFER; for (failure = 0; failure <= OSSL_CMP_PKIFAILUREINFO_MAX; failure++) { diff --git a/openssl/src/crypto/cmp/cmp_util.c b/openssl/src/crypto/cmp/cmp_util.c index f3c21c5d9..ed611d64d 100644 --- a/openssl/src/crypto/cmp/cmp_util.c +++ b/openssl/src/crypto/cmp/cmp_util.c @@ -53,7 +53,8 @@ static OSSL_CMP_severity parse_level(const char *level) if (end_level == NULL) return -1; - if (HAS_PREFIX(level, OSSL_CMP_LOG_PREFIX)) + if (strncmp(level, OSSL_CMP_LOG_PREFIX, + strlen(OSSL_CMP_LOG_PREFIX)) == 0) level += strlen(OSSL_CMP_LOG_PREFIX); len = end_level - level; if (len > max_level_len) @@ -100,8 +101,8 @@ const char *ossl_cmp_log_parse_metadata(const char *buf, *file = OPENSSL_strndup(p_file, p_line - 1 - p_file); /* no real problem if OPENSSL_strndup() returns NULL */ *line = (int)line_number; - msg = strchr(p_level, ':'); - if (msg != NULL && *++msg == ' ') + msg = strchr(p_level, ':') + 1; + if (*msg == ' ') msg++; } } @@ -189,7 +190,7 @@ void OSSL_CMP_print_errors_cb(OSSL_CMP_log_cb_t log_fn) BIO_free(bio); } #else - /* ERR_raise(..., CMP_R_NO_STDIO) would make no sense here */ + /* ERR_raise(ERR_LIB_CMP, CMP_R_NO_STDIO) makes no sense during error printing */ #endif } else { if (log_fn(component, file, line, OSSL_CMP_LOG_ERR, msg) <= 0) @@ -243,7 +244,6 @@ int ossl_cmp_asn1_octet_string_set1(ASN1_OCTET_STRING **tgt, const ASN1_OCTET_STRING *src) { ASN1_OCTET_STRING *new; - if (tgt == NULL) { ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT); return 0; diff --git a/openssl/src/crypto/cmp/cmp_vfy.c b/openssl/src/crypto/cmp/cmp_vfy.c index ec99ab7fe..b9d6fc2bd 100644 --- a/openssl/src/crypto/cmp/cmp_vfy.c +++ b/openssl/src/crypto/cmp/cmp_vfy.c @@ -1,5 +1,5 @@ /* - * Copyright 2007-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2020 * Copyright Siemens AG 2015-2020 * @@ -34,8 +34,7 @@ static int verify_signature(const OSSL_CMP_CTX *cmp_ctx, return 0; bio = BIO_new(BIO_s_mem()); /* may be NULL */ - if (bio == NULL) - return 0; + /* verify that keyUsage, if present, contains digitalSignature */ if (!cmp_ctx->ignore_keyusage && (X509_get_key_usage(cert) & X509v3_KU_DIGITAL_SIGNATURE) == 0) { @@ -139,24 +138,6 @@ int OSSL_CMP_validate_cert_path(const OSSL_CMP_CTX *ctx, return valid; } -static int verify_cb_cert(X509_STORE *ts, X509 *cert, int err) -{ - X509_STORE_CTX_verify_cb verify_cb; - X509_STORE_CTX *csc; - int ok = 0; - - if (ts == NULL || (verify_cb = X509_STORE_get_verify_cb(ts)) == NULL) - return ok; - if ((csc = X509_STORE_CTX_new()) != NULL - && X509_STORE_CTX_init(csc, ts, cert, NULL)) { - X509_STORE_CTX_set_error(csc, err); - X509_STORE_CTX_set_current_cert(csc, cert); - ok = (*verify_cb)(0, csc); - } - X509_STORE_CTX_free(csc); - return ok; -} - /* Return 0 if expect_name != NULL and there is no matching actual_name */ static int check_name(const OSSL_CMP_CTX *ctx, int log_success, const char *actual_desc, const X509_NAME *actual_name, @@ -175,8 +156,8 @@ static int check_name(const OSSL_CMP_CTX *ctx, int log_success, str = X509_NAME_oneline(actual_name, NULL, 0); if (X509_NAME_cmp(actual_name, expect_name) == 0) { if (log_success && str != NULL) - ossl_cmp_log3(INFO, ctx, " %s matches %s: %s", - actual_desc, expect_desc, str); + ossl_cmp_log2(INFO, ctx, " subject matches %s: %s", expect_desc, + str); OPENSSL_free(str); return 1; } @@ -205,7 +186,7 @@ static int check_kid(const OSSL_CMP_CTX *ctx, ossl_cmp_warn(ctx, "missing Subject Key Identifier in certificate"); return 0; } - str = i2s_ASN1_OCTET_STRING(NULL, ckid); + str = OPENSSL_buf2hexstr(ckid->data, ckid->length); if (ASN1_OCTET_STRING_cmp(ckid, skid) == 0) { if (str != NULL) ossl_cmp_log1(INFO, ctx, " subjectKID matches senderKID: %s", str); @@ -216,7 +197,7 @@ static int check_kid(const OSSL_CMP_CTX *ctx, if (str != NULL) ossl_cmp_log1(INFO, ctx, " cert Subject Key Identifier = %s", str); OPENSSL_free(str); - if ((str = i2s_ASN1_OCTET_STRING(NULL, skid)) != NULL) + if ((str = OPENSSL_buf2hexstr(skid->data, skid->length)) != NULL) ossl_cmp_log1(INFO, ctx, " does not match senderKID = %s", str); OPENSSL_free(str); return 0; @@ -274,14 +255,9 @@ static int cert_acceptable(const OSSL_CMP_CTX *ctx, time_cmp = X509_cmp_timeframe(vpm, X509_get0_notBefore(cert), X509_get0_notAfter(cert)); if (time_cmp != 0) { - int err = time_cmp > 0 ? X509_V_ERR_CERT_HAS_EXPIRED - : X509_V_ERR_CERT_NOT_YET_VALID; - ossl_cmp_warn(ctx, time_cmp > 0 ? "cert has expired" : "cert is not yet valid"); - if (ctx->log_cb != NULL /* logging not temporarily disabled */ - && verify_cb_cert(ts, cert, err) <= 0) - return 0; + return 0; } if (!check_name(ctx, 1, @@ -347,11 +323,11 @@ static int check_cert_path_3gpp(const OSSL_CMP_CTX *ctx, * verify that the newly enrolled certificate (which assumed rid == * OSSL_CMP_CERTREQID) can also be validated with the same trusted store */ + EVP_PKEY *pkey = OSSL_CMP_CTX_get0_newPkey(ctx, 1); OSSL_CMP_CERTRESPONSE *crep = ossl_cmp_certrepmessage_get0_certresponse(msg->body->value.ip, OSSL_CMP_CERTREQID); - X509 *newcrt = ossl_cmp_certresponse_get1_cert(ctx, crep); - + X509 *newcrt = ossl_cmp_certresponse_get1_cert(crep, ctx, pkey); /* * maybe better use get_cert_status() from cmp_client.c, which catches * errors @@ -377,7 +353,7 @@ static int check_msg_given_cert(const OSSL_CMP_CTX *ctx, X509 *cert, /*- * Try all certs in given list for verifying msg, normally or in 3GPP mode. * If already_checked1 == NULL then certs are assumed to be the msg->extraCerts. - * On success cache the found cert using ossl_cmp_ctx_set1_validatedSrvCert(). + * On success cache the found cert using ossl_cmp_ctx_set0_validatedSrvCert(). */ static int check_msg_with_certs(OSSL_CMP_CTX *ctx, const STACK_OF(X509) *certs, const char *desc, @@ -406,7 +382,13 @@ static int check_msg_with_certs(OSSL_CMP_CTX *ctx, const STACK_OF(X509) *certs, if (mode_3gpp ? check_cert_path_3gpp(ctx, msg, cert) : check_cert_path(ctx, ctx->trusted, cert)) { /* store successful sender cert for further msgs in transaction */ - return ossl_cmp_ctx_set1_validatedSrvCert(ctx, cert); + if (!X509_up_ref(cert)) + return 0; + if (!ossl_cmp_ctx_set0_validatedSrvCert(ctx, cert)) { + X509_free(cert); + return 0; + } + return 1; } } if (in_extraCerts && n_acceptable_certs == 0) @@ -417,21 +399,21 @@ static int check_msg_with_certs(OSSL_CMP_CTX *ctx, const STACK_OF(X509) *certs, /*- * Verify msg trying first ctx->untrusted, which should include extraCerts * at its front, then trying the trusted certs in truststore (if any) of ctx. - * On success cache the found cert using ossl_cmp_ctx_set1_validatedSrvCert(). + * On success cache the found cert using ossl_cmp_ctx_set0_validatedSrvCert(). */ static int check_msg_all_certs(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg, int mode_3gpp) { int ret = 0; - if (ctx->permitTAInExtraCertsForIR - && OSSL_CMP_MSG_get_bodytype(msg) == OSSL_CMP_PKIBODY_IP) - ossl_cmp_info(ctx, mode_3gpp ? - "normal mode failed; trying now 3GPP mode trusting extraCerts" - : "trying first normal mode using trust store"); - else if (mode_3gpp) + if (mode_3gpp + && ((!ctx->permitTAInExtraCertsForIR + || OSSL_CMP_MSG_get_bodytype(msg) != OSSL_CMP_PKIBODY_IP))) return 0; + ossl_cmp_info(ctx, + mode_3gpp ? "normal mode failed; trying now 3GPP mode trusting extraCerts" + : "trying first normal mode using trust store"); if (check_msg_with_certs(ctx, msg->extraCerts, "extraCerts", NULL, NULL, msg, mode_3gpp)) return 1; @@ -444,20 +426,25 @@ static int check_msg_all_certs(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg, : "no trusted store"); } else { STACK_OF(X509) *trusted = X509_STORE_get1_all_certs(ctx->trusted); - ret = check_msg_with_certs(ctx, trusted, mode_3gpp ? "self-issued extraCerts" : "certs in trusted store", msg->extraCerts, ctx->untrusted, msg, mode_3gpp); - OSSL_STACK_OF_X509_free(trusted); + sk_X509_pop_free(trusted, X509_free); } return ret; } +static int no_log_cb(const char *func, const char *file, int line, + OSSL_CMP_severity level, const char *msg) +{ + return 1; +} + /*- * Verify message signature with any acceptable and valid candidate cert. - * On success cache the found cert using ossl_cmp_ctx_set1_validatedSrvCert(). + * On success cache the found cert using ossl_cmp_ctx_set0_validatedSrvCert(). */ static int check_msg_find_cert(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) { @@ -472,7 +459,6 @@ static int check_msg_find_cert(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) if (sender == NULL || msg->body == NULL) return 0; /* other NULL cases already have been checked */ if (sender->type != GEN_DIRNAME) { - /* So far, only X509_NAME is supported */ ERR_raise(ERR_LIB_CMP, CMP_R_SENDER_GENERALNAME_TYPE_NOT_SUPPORTED); return 0; } @@ -482,7 +468,7 @@ static int check_msg_find_cert(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) /* enable clearing irrelevant errors in attempts to validate sender certs */ (void)ERR_set_mark(); - ctx->log_cb = NULL; /* temporarily disable logging */ + ctx->log_cb = no_log_cb; /* temporarily disable logging */ /* * try first cached scrt, used successfully earlier in same transaction, @@ -495,7 +481,7 @@ static int check_msg_find_cert(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) return 1; } /* cached sender cert has shown to be no more successfully usable */ - (void)ossl_cmp_ctx_set1_validatedSrvCert(ctx, NULL); + (void)ossl_cmp_ctx_set0_validatedSrvCert(ctx, NULL); /* re-do the above check (just) for adding diagnostic information */ ossl_cmp_info(ctx, "trying to verify msg signature with previously validated cert"); @@ -514,7 +500,8 @@ static int check_msg_find_cert(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) (void)ERR_clear_last_mark(); sname = X509_NAME_oneline(sender->d.directoryName, NULL, 0); - skid_str = skid == NULL ? NULL : i2s_ASN1_OCTET_STRING(NULL, skid); + skid_str = skid == NULL ? NULL + : OPENSSL_buf2hexstr(skid->data, skid->length); if (ctx->log_cb != NULL) { ossl_cmp_info(ctx, "trying to verify msg signature with a valid cert that.."); if (sname != NULL) @@ -550,7 +537,7 @@ static int check_msg_find_cert(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) * the sender certificate can have been pinned by providing it in ctx->srvCert, * else it is searched in msg->extraCerts, ctx->untrusted, in ctx->trusted * (in this order) and is path is validated against ctx->trusted. - * On success cache the found cert using ossl_cmp_ctx_set1_validatedSrvCert(). + * On success cache the found cert using ossl_cmp_ctx_set0_validatedSrvCert(). * * If ctx->permitTAInExtraCertsForIR is true and when validating a CMP IP msg, * the trust anchor for validating the IP msg may be taken from msg->extraCerts @@ -581,9 +568,8 @@ int OSSL_CMP_validate_msg(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) /* 5.1.3.1. Shared Secret Information */ case NID_id_PasswordBasedMAC: if (ctx->secretValue == NULL) { - ossl_cmp_info(ctx, "no secret available for verifying PBM-based CMP message protection"); - ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_SECRET); - return 0; + ossl_cmp_warn(ctx, "no secret available for verifying PBM-based CMP message protection"); + return 1; } if (verify_PBMAC(ctx, msg)) { /* @@ -612,7 +598,7 @@ int OSSL_CMP_validate_msg(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) break; } ossl_cmp_debug(ctx, - "successfully validated PBM-based CMP message protection"); + "sucessfully validated PBM-based CMP message protection"); return 1; } ossl_cmp_warn(ctx, "verifying PBM-based CMP message protection failed"); @@ -633,21 +619,18 @@ int OSSL_CMP_validate_msg(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) scrt = ctx->srvCert; if (scrt == NULL) { if (ctx->trusted == NULL) { - ossl_cmp_info(ctx, "no trust store nor pinned server cert available for verifying signature-based CMP message protection"); - ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_TRUST_ANCHOR); - return 0; - } - if (check_msg_find_cert(ctx, msg)) { - ossl_cmp_debug(ctx, - "successfully validated signature-based CMP message protection using trust store"); + ossl_cmp_warn(ctx, "no trust store nor pinned server cert available for verifying signature-based CMP message protection"); return 1; } + if (check_msg_find_cert(ctx, msg)) + return 1; } else { /* use pinned sender cert */ /* use ctx->srvCert for signature check even if not acceptable */ if (verify_signature(ctx, msg, scrt)) { ossl_cmp_debug(ctx, - "successfully validated signature-based CMP message protection using pinned server cert"); - return ossl_cmp_ctx_set1_validatedSrvCert(ctx, scrt); + "sucessfully validated signature-based CMP message protection"); + + return 1; } ossl_cmp_warn(ctx, "CMP message signature verification failed"); ERR_raise(ERR_LIB_CMP, CMP_R_SRVCERT_DOES_NOT_VALIDATE_MSG); @@ -657,35 +640,13 @@ int OSSL_CMP_validate_msg(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg) return 0; } -static int check_transactionID_or_nonce(ASN1_OCTET_STRING *expected, - ASN1_OCTET_STRING *actual, int reason) -{ - if (expected != NULL - && (actual == NULL || ASN1_OCTET_STRING_cmp(expected, actual) != 0)) { -#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - char *expected_str, *actual_str; - - expected_str = i2s_ASN1_OCTET_STRING(NULL, expected); - actual_str = actual == NULL ? NULL: i2s_ASN1_OCTET_STRING(NULL, actual); - ERR_raise_data(ERR_LIB_CMP, reason, - "expected = %s, actual = %s", - expected_str == NULL ? "?" : expected_str, - actual == NULL ? "(none)" : - actual_str == NULL ? "?" : actual_str); - OPENSSL_free(expected_str); - OPENSSL_free(actual_str); - return 0; -#endif - } - return 1; -} /*- * Check received message (i.e., response by server or request from client) * Any msg->extraCerts are prepended to ctx->untrusted. * * Ensures that: - * its sender is of appropriate type (currently only X509_NAME) and + * its sender is of appropriate type (curently only X509_NAME) and * matches any expected sender or srvCert subject given in the ctx * it has a valid body type * its protection is valid (or invalid/absent, but only if a callback function @@ -705,94 +666,68 @@ int ossl_cmp_msg_check_update(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg, { OSSL_CMP_PKIHEADER *hdr; const X509_NAME *expected_sender; - int num_untrusted, num_added, res; if (!ossl_assert(ctx != NULL && msg != NULL && msg->header != NULL)) return 0; hdr = OSSL_CMP_MSG_get0_header(msg); - /* If expected_sender is given, validate sender name of received msg */ + /* validate sender name of received msg */ + if (hdr->sender->type != GEN_DIRNAME) { + ERR_raise(ERR_LIB_CMP, CMP_R_SENDER_GENERALNAME_TYPE_NOT_SUPPORTED); + return 0; + } + /* + * Compare actual sender name of response with expected sender name. + * Mitigates risk to accept misused PBM secret + * or misused certificate of an unauthorized entity of a trusted hierarchy. + */ expected_sender = ctx->expected_sender; if (expected_sender == NULL && ctx->srvCert != NULL) expected_sender = X509_get_subject_name(ctx->srvCert); - if (expected_sender != NULL) { - const X509_NAME *actual_sender; - char *str; - - if (hdr->sender->type != GEN_DIRNAME) { - ERR_raise(ERR_LIB_CMP, CMP_R_SENDER_GENERALNAME_TYPE_NOT_SUPPORTED); - return 0; - } - actual_sender = hdr->sender->d.directoryName; - /* - * Compare actual sender name of response with expected sender name. - * Mitigates risk of accepting misused PBM secret or - * misused certificate of an unauthorized entity of a trusted hierarchy. - */ - if (!check_name(ctx, 0, "sender DN field", actual_sender, - "expected sender", expected_sender)) { - str = X509_NAME_oneline(actual_sender, NULL, 0); - ERR_raise_data(ERR_LIB_CMP, CMP_R_UNEXPECTED_SENDER, - str != NULL ? str : ""); - OPENSSL_free(str); - return 0; - } - } + if (!check_name(ctx, 0, "sender DN field", hdr->sender->d.directoryName, + "expected sender", expected_sender)) + return 0; /* Note: if recipient was NULL-DN it could be learned here if needed */ - num_added = sk_X509_num(msg->extraCerts); - if (num_added > 10) - ossl_cmp_log1(WARN, ctx, "received CMP message contains %d extraCerts", - num_added); + if (sk_X509_num(msg->extraCerts) > 10) + ossl_cmp_warn(ctx, + "received CMP message contains more than 10 extraCerts"); /* * Store any provided extraCerts in ctx for use in OSSL_CMP_validate_msg() * and for future use, such that they are available to ctx->certConf_cb and * the peer does not need to send them again in the same transaction. * Note that it does not help validating the message before storing the * extraCerts because they do not belong to the protected msg part anyway. - * The extraCerts are prepended. Allows simple removal if they shall not be - * cached. Also they get used first, which is likely good for efficiency. + * For efficiency, the extraCerts are prepended so they get used first. */ - num_untrusted = ctx->untrusted == NULL ? 0 : sk_X509_num(ctx->untrusted); - res = ossl_x509_add_certs_new(&ctx->untrusted, msg->extraCerts, - /* this allows self-signed certs */ - X509_ADD_FLAG_UP_REF | X509_ADD_FLAG_NO_DUP - | X509_ADD_FLAG_PREPEND); - num_added = (ctx->untrusted == NULL ? 0 : sk_X509_num(ctx->untrusted)) - - num_untrusted; - if (!res) { - while (num_added-- > 0) - X509_free(sk_X509_shift(ctx->untrusted)); + if (!X509_add_certs(ctx->untrusted, msg->extraCerts, + /* this allows self-signed certs */ + X509_ADD_FLAG_UP_REF | X509_ADD_FLAG_NO_DUP + | X509_ADD_FLAG_PREPEND)) return 0; - } - if (hdr->protectionAlg != NULL) - res = OSSL_CMP_validate_msg(ctx, msg) - /* explicitly permitted exceptions for invalid protection: */ - || (cb != NULL && (*cb)(ctx, msg, 1, cb_arg) > 0); - else - /* explicitly permitted exceptions for missing protection: */ - res = cb != NULL && (*cb)(ctx, msg, 0, cb_arg) > 0; -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - res = 1; /* support more aggressive fuzzing by letting invalid msg pass */ -#endif - - /* remove extraCerts again if not caching */ - if (ctx->noCacheExtraCerts) - while (num_added-- > 0) - X509_free(sk_X509_shift(ctx->untrusted)); - - if (!res) { - if (hdr->protectionAlg != NULL) + /* validate message protection */ + if (hdr->protectionAlg != NULL) { + /* detect explicitly permitted exceptions for invalid protection */ + if (!OSSL_CMP_validate_msg(ctx, msg) + && (cb == NULL || (*cb)(ctx, msg, 1, cb_arg) <= 0)) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION ERR_raise(ERR_LIB_CMP, CMP_R_ERROR_VALIDATING_PROTECTION); - else + return 0; +#endif + } + } else { + /* detect explicitly permitted exceptions for missing protection */ + if (cb == NULL || (*cb)(ctx, msg, 0, cb_arg) <= 0) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION ERR_raise(ERR_LIB_CMP, CMP_R_MISSING_PROTECTION); - return 0; + return 0; +#endif + } } /* check CMP version number in header */ - if (ossl_cmp_hdr_get_pvno(hdr) != OSSL_CMP_PVNO_2 - && ossl_cmp_hdr_get_pvno(hdr) != OSSL_CMP_PVNO_3) { + if (ossl_cmp_hdr_get_pvno(hdr) != OSSL_CMP_PVNO) { #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION ERR_raise(ERR_LIB_CMP, CMP_R_UNEXPECTED_PVNO); return 0; @@ -807,35 +742,26 @@ int ossl_cmp_msg_check_update(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg, } /* compare received transactionID with the expected one in previous msg */ - if (!check_transactionID_or_nonce(ctx->transactionID, hdr->transactionID, - CMP_R_TRANSACTIONID_UNMATCHED)) + if (ctx->transactionID != NULL + && (hdr->transactionID == NULL + || ASN1_OCTET_STRING_cmp(ctx->transactionID, + hdr->transactionID) != 0)) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + ERR_raise(ERR_LIB_CMP, CMP_R_TRANSACTIONID_UNMATCHED); return 0; - - /* - * enable clearing irrelevant errors - * in attempts to validate recipient nonce in case of delayed delivery. - */ - (void)ERR_set_mark(); - /* compare received nonce with the one we sent */ - if (!check_transactionID_or_nonce(ctx->senderNonce, hdr->recipNonce, - CMP_R_RECIPNONCE_UNMATCHED)) { - /* check if we are polling and received final response */ - if (ctx->first_senderNonce == NULL - || OSSL_CMP_MSG_get_bodytype(msg) == OSSL_CMP_PKIBODY_POLLREP - /* compare received nonce with our sender nonce at poll start */ - || !check_transactionID_or_nonce(ctx->first_senderNonce, - hdr->recipNonce, - CMP_R_RECIPNONCE_UNMATCHED)) { - (void)ERR_clear_last_mark(); - return 0; - } +#endif } - (void)ERR_pop_to_mark(); - /* if not yet present, learn transactionID */ - if (ctx->transactionID == NULL - && !OSSL_CMP_CTX_set1_transactionID(ctx, hdr->transactionID)) + /* compare received nonce with the one we sent */ + if (ctx->senderNonce != NULL + && (msg->header->recipNonce == NULL + || ASN1_OCTET_STRING_cmp(ctx->senderNonce, + hdr->recipNonce) != 0)) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + ERR_raise(ERR_LIB_CMP, CMP_R_RECIPNONCE_UNMATCHED); return 0; +#endif + } /* * RFC 4210 section 5.1.1 states: the recipNonce is copied from @@ -845,6 +771,23 @@ int ossl_cmp_msg_check_update(OSSL_CMP_CTX *ctx, const OSSL_CMP_MSG *msg, if (!ossl_cmp_ctx_set1_recipNonce(ctx, hdr->senderNonce)) return 0; + /* if not yet present, learn transactionID */ + if (ctx->transactionID == NULL + && !OSSL_CMP_CTX_set1_transactionID(ctx, hdr->transactionID)) + return -1; + + /* + * Store any provided extraCerts in ctx for future use, + * such that they are available to ctx->certConf_cb and + * the peer does not need to send them again in the same transaction. + * For efficiency, the extraCerts are prepended so they get used first. + */ + if (!X509_add_certs(ctx->untrusted, msg->extraCerts, + /* this allows self-signed certs */ + X509_ADD_FLAG_UP_REF | X509_ADD_FLAG_NO_DUP + | X509_ADD_FLAG_PREPEND)) + return -1; + if (ossl_cmp_hdr_get_protection_nid(hdr) == NID_id_PasswordBasedMAC) { /* * RFC 4210, 5.3.2: 'Note that if the PKI Message Protection is diff --git a/openssl/src/crypto/cms/cms_asn1.c b/openssl/src/crypto/cms/cms_asn1.c index ecf5a4479..72cd14317 100644 --- a/openssl/src/crypto/cms/cms_asn1.c +++ b/openssl/src/crypto/cms/cms_asn1.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -51,7 +51,6 @@ static int cms_si_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, EVP_PKEY_free(si->pkey); X509_free(si->signer); EVP_MD_CTX_free(si->mctx); - EVP_PKEY_CTX_free(si->pctx); } return 1; } @@ -84,28 +83,17 @@ ASN1_NDEF_SEQUENCE(CMS_SignedData) = { ASN1_IMP_SET_OF_OPT(CMS_SignedData, crls, CMS_RevocationInfoChoice, 1), ASN1_SET_OF(CMS_SignedData, signerInfos, CMS_SignerInfo) } ASN1_NDEF_SEQUENCE_END(CMS_SignedData) -IMPLEMENT_ASN1_ALLOC_FUNCTIONS(CMS_SignedData) ASN1_SEQUENCE(CMS_OriginatorInfo) = { ASN1_IMP_SET_OF_OPT(CMS_OriginatorInfo, certificates, CMS_CertificateChoices, 0), ASN1_IMP_SET_OF_OPT(CMS_OriginatorInfo, crls, CMS_RevocationInfoChoice, 1) } static_ASN1_SEQUENCE_END(CMS_OriginatorInfo) -static int cms_ec_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, - void *exarg) -{ - CMS_EncryptedContentInfo *ec = (CMS_EncryptedContentInfo *)*pval; - - if (operation == ASN1_OP_FREE_POST) - OPENSSL_clear_free(ec->key, ec->keylen); - return 1; -} - -ASN1_NDEF_SEQUENCE_cb(CMS_EncryptedContentInfo, cms_ec_cb) = { +ASN1_NDEF_SEQUENCE(CMS_EncryptedContentInfo) = { ASN1_SIMPLE(CMS_EncryptedContentInfo, contentType, ASN1_OBJECT), ASN1_SIMPLE(CMS_EncryptedContentInfo, contentEncryptionAlgorithm, X509_ALGOR), ASN1_IMP_OPT(CMS_EncryptedContentInfo, encryptedContent, ASN1_OCTET_STRING_NDEF, 0) -} ASN1_NDEF_SEQUENCE_END_cb(CMS_EncryptedContentInfo, CMS_EncryptedContentInfo) +} static_ASN1_NDEF_SEQUENCE_END(CMS_EncryptedContentInfo) ASN1_SEQUENCE(CMS_KeyTransRecipientInfo) = { ASN1_EMBED(CMS_KeyTransRecipientInfo, version, INT32), @@ -316,7 +304,7 @@ static int cms_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, case ASN1_OP_STREAM_PRE: if (CMS_stream(&sarg->boundary, cms) <= 0) return 0; - /* fall through */ + /* fall thru */ case ASN1_OP_DETACHED_PRE: sarg->ndef_bio = CMS_dataInit(cms, sarg->out); if (!sarg->ndef_bio) @@ -329,10 +317,6 @@ static int cms_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, return 0; break; - case ASN1_OP_FREE_POST: - OPENSSL_free(cms->ctx.propq); - break; - } return 1; } diff --git a/openssl/src/crypto/cms/cms_att.c b/openssl/src/crypto/cms/cms_att.c index 64acda726..5b99516b2 100644 --- a/openssl/src/crypto/cms/cms_att.c +++ b/openssl/src/crypto/cms/cms_att.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,9 +12,8 @@ #include #include #include -#include "internal/nelem.h" -#include "crypto/x509.h" #include "cms_local.h" +#include "internal/nelem.h" /*- * Attribute flags. @@ -95,7 +94,7 @@ X509_ATTRIBUTE *CMS_signed_delete_attr(CMS_SignerInfo *si, int loc) int CMS_signed_add1_attr(CMS_SignerInfo *si, X509_ATTRIBUTE *attr) { - if (ossl_x509at_add1_attr(&si->signedAttrs, attr)) + if (X509at_add1_attr(&si->signedAttrs, attr)) return 1; return 0; } @@ -104,7 +103,7 @@ int CMS_signed_add1_attr_by_OBJ(CMS_SignerInfo *si, const ASN1_OBJECT *obj, int type, const void *bytes, int len) { - if (ossl_x509at_add1_attr_by_OBJ(&si->signedAttrs, obj, type, bytes, len)) + if (X509at_add1_attr_by_OBJ(&si->signedAttrs, obj, type, bytes, len)) return 1; return 0; } @@ -112,7 +111,7 @@ int CMS_signed_add1_attr_by_OBJ(CMS_SignerInfo *si, int CMS_signed_add1_attr_by_NID(CMS_SignerInfo *si, int nid, int type, const void *bytes, int len) { - if (ossl_x509at_add1_attr_by_NID(&si->signedAttrs, nid, type, bytes, len)) + if (X509at_add1_attr_by_NID(&si->signedAttrs, nid, type, bytes, len)) return 1; return 0; } @@ -121,8 +120,7 @@ int CMS_signed_add1_attr_by_txt(CMS_SignerInfo *si, const char *attrname, int type, const void *bytes, int len) { - if (ossl_x509at_add1_attr_by_txt(&si->signedAttrs, attrname, type, bytes, - len)) + if (X509at_add1_attr_by_txt(&si->signedAttrs, attrname, type, bytes, len)) return 1; return 0; } @@ -163,7 +161,7 @@ X509_ATTRIBUTE *CMS_unsigned_delete_attr(CMS_SignerInfo *si, int loc) int CMS_unsigned_add1_attr(CMS_SignerInfo *si, X509_ATTRIBUTE *attr) { - if (ossl_x509at_add1_attr(&si->unsignedAttrs, attr)) + if (X509at_add1_attr(&si->unsignedAttrs, attr)) return 1; return 0; } @@ -172,7 +170,7 @@ int CMS_unsigned_add1_attr_by_OBJ(CMS_SignerInfo *si, const ASN1_OBJECT *obj, int type, const void *bytes, int len) { - if (ossl_x509at_add1_attr_by_OBJ(&si->unsignedAttrs, obj, type, bytes, len)) + if (X509at_add1_attr_by_OBJ(&si->unsignedAttrs, obj, type, bytes, len)) return 1; return 0; } @@ -181,7 +179,7 @@ int CMS_unsigned_add1_attr_by_NID(CMS_SignerInfo *si, int nid, int type, const void *bytes, int len) { - if (ossl_x509at_add1_attr_by_NID(&si->unsignedAttrs, nid, type, bytes, len)) + if (X509at_add1_attr_by_NID(&si->unsignedAttrs, nid, type, bytes, len)) return 1; return 0; } @@ -190,8 +188,8 @@ int CMS_unsigned_add1_attr_by_txt(CMS_SignerInfo *si, const char *attrname, int type, const void *bytes, int len) { - if (ossl_x509at_add1_attr_by_txt(&si->unsignedAttrs, attrname, - type, bytes, len)) + if (X509at_add1_attr_by_txt(&si->unsignedAttrs, attrname, + type, bytes, len)) return 1; return 0; } diff --git a/openssl/src/crypto/cms/cms_cd.c b/openssl/src/crypto/cms/cms_cd.c index 300026848..6de6d55e5 100644 --- a/openssl/src/crypto/cms/cms_cd.c +++ b/openssl/src/crypto/cms/cms_cd.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,7 +17,7 @@ #include #include "cms_local.h" -#ifndef OPENSSL_NO_ZLIB +#ifdef ZLIB /* CMS CompressedData Utilities */ @@ -50,9 +50,8 @@ CMS_ContentInfo *ossl_cms_CompressedData_create(int comp_nid, cd->version = 0; - (void)X509_ALGOR_set0(cd->compressionAlgorithm, - OBJ_nid2obj(NID_zlib_compression), - V_ASN1_UNDEF, NULL); /* cannot fail */ + X509_ALGOR_set0(cd->compressionAlgorithm, + OBJ_nid2obj(NID_zlib_compression), V_ASN1_UNDEF, NULL); cd->encapContentInfo->eContentType = OBJ_nid2obj(NID_pkcs7_data); diff --git a/openssl/src/crypto/cms/cms_dd.c b/openssl/src/crypto/cms/cms_dd.c index 40b20249a..6a7c049ef 100644 --- a/openssl/src/crypto/cms/cms_dd.c +++ b/openssl/src/crypto/cms/cms_dd.c @@ -66,7 +66,7 @@ int ossl_cms_DigestedData_do_final(const CMS_ContentInfo *cms, BIO *chain, CMS_DigestedData *dd; if (mctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } diff --git a/openssl/src/crypto/cms/cms_dh.c b/openssl/src/crypto/cms/cms_dh.c index c6e8c076d..f14546c70 100644 --- a/openssl/src/crypto/cms/cms_dh.c +++ b/openssl/src/crypto/cms/cms_dh.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,7 +13,6 @@ #include #include #include "internal/sizes.h" -#include "crypto/asn1.h" #include "crypto/evp.h" #include "cms_local.h" @@ -235,16 +234,17 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri) if (penclen <= 0) goto err; ASN1_STRING_set0(pubkey, penc, penclen); - ossl_asn1_string_set_bits_left(pubkey, 0); + pubkey->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + pubkey->flags |= ASN1_STRING_FLAG_BITS_LEFT; penc = NULL; - (void)X509_ALGOR_set0(talg, OBJ_nid2obj(NID_dhpublicnumber), - V_ASN1_UNDEF, NULL); /* cannot fail */ + X509_ALGOR_set0(talg, OBJ_nid2obj(NID_dhpublicnumber), + V_ASN1_UNDEF, NULL); } /* See if custom parameters set */ kdf_type = EVP_PKEY_CTX_get_dh_kdf_type(pctx); - if (kdf_type <= 0 || EVP_PKEY_CTX_get_dh_kdf_md(pctx, &kdf_md) <= 0) + if (kdf_type <= 0 || !EVP_PKEY_CTX_get_dh_kdf_md(pctx, &kdf_md)) goto err; if (kdf_type == EVP_PKEY_DH_KDF_NONE) { @@ -309,17 +309,17 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri) */ penc = NULL; penclen = i2d_X509_ALGOR(wrap_alg, &penc); - if (penclen <= 0) + if (penc == NULL || penclen == 0) goto err; wrap_str = ASN1_STRING_new(); if (wrap_str == NULL) goto err; ASN1_STRING_set0(wrap_str, penc, penclen); penc = NULL; - rv = X509_ALGOR_set0(talg, OBJ_nid2obj(NID_id_smime_alg_ESDH), - V_ASN1_SEQUENCE, wrap_str); - if (!rv) - ASN1_STRING_free(wrap_str); + X509_ALGOR_set0(talg, OBJ_nid2obj(NID_id_smime_alg_ESDH), + V_ASN1_SEQUENCE, wrap_str); + + rv = 1; err: OPENSSL_free(penc); diff --git a/openssl/src/crypto/cms/cms_ec.c b/openssl/src/crypto/cms/cms_ec.c index a4427d7ee..b07af92ba 100644 --- a/openssl/src/crypto/cms/cms_ec.c +++ b/openssl/src/crypto/cms/cms_ec.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,12 +8,10 @@ */ #include -#include #include #include #include #include "internal/sizes.h" -#include "crypto/asn1.h" #include "crypto/evp.h" #include "cms_local.h" @@ -50,7 +48,7 @@ static EVP_PKEY *pkey_type2param(int ptype, const void *pval, if (pctx == NULL || EVP_PKEY_paramgen_init(pctx) <= 0) goto err; if (OBJ_obj2txt(groupname, sizeof(groupname), poid, 0) <= 0 - || EVP_PKEY_CTX_set_group_name(pctx, groupname) <= 0) { + || !EVP_PKEY_CTX_set_group_name(pctx, groupname)) { ERR_raise(ERR_LIB_CMS, CMS_R_DECODE_ERROR); goto err; } @@ -259,7 +257,7 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri) ASN1_STRING *wrap_str; ASN1_OCTET_STRING *ukm; unsigned char *penc = NULL; - int penclen; + size_t penclen; int rv = 0; int ecdh_nid, kdf_type, kdf_nid, wrap_nid; const EVP_MD *kdf_md; @@ -276,24 +274,22 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri) /* Is everything uninitialised? */ if (aoid == OBJ_nid2obj(NID_undef)) { /* Set the key */ - size_t enckeylen; - enckeylen = EVP_PKEY_get1_encoded_public_key(pkey, &penc); - if (enckeylen > INT_MAX || enckeylen == 0) - goto err; - ASN1_STRING_set0(pubkey, penc, (int)enckeylen); - ossl_asn1_string_set_bits_left(pubkey, 0); + penclen = EVP_PKEY_get1_encoded_public_key(pkey, &penc); + ASN1_STRING_set0(pubkey, penc, penclen); + pubkey->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + pubkey->flags |= ASN1_STRING_FLAG_BITS_LEFT; penc = NULL; - (void)X509_ALGOR_set0(talg, OBJ_nid2obj(NID_X9_62_id_ecPublicKey), - V_ASN1_UNDEF, NULL); /* cannot fail */ + X509_ALGOR_set0(talg, OBJ_nid2obj(NID_X9_62_id_ecPublicKey), + V_ASN1_UNDEF, NULL); } /* See if custom parameters set */ kdf_type = EVP_PKEY_CTX_get_ecdh_kdf_type(pctx); if (kdf_type <= 0) goto err; - if (EVP_PKEY_CTX_get_ecdh_kdf_md(pctx, &kdf_md) <= 0) + if (!EVP_PKEY_CTX_get_ecdh_kdf_md(pctx, &kdf_md)) goto err; ecdh_nid = EVP_PKEY_CTX_get_ecdh_cofactor_mode(pctx); if (ecdh_nid < 0) @@ -350,7 +346,7 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri) penclen = CMS_SharedInfo_encode(&penc, wrap_alg, ukm, keylen); - if (penclen <= 0) + if (penclen == 0) goto err; if (EVP_PKEY_CTX_set0_ecdh_kdf_ukm(pctx, penc, penclen) <= 0) @@ -362,16 +358,16 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri) * of another AlgorithmIdentifier. */ penclen = i2d_X509_ALGOR(wrap_alg, &penc); - if (penclen <= 0) + if (penc == NULL || penclen == 0) goto err; wrap_str = ASN1_STRING_new(); if (wrap_str == NULL) goto err; ASN1_STRING_set0(wrap_str, penc, penclen); penc = NULL; - rv = X509_ALGOR_set0(talg, OBJ_nid2obj(kdf_nid), V_ASN1_SEQUENCE, wrap_str); - if (!rv) - ASN1_STRING_free(wrap_str); + X509_ALGOR_set0(talg, OBJ_nid2obj(kdf_nid), V_ASN1_SEQUENCE, wrap_str); + + rv = 1; err: OPENSSL_free(penc); @@ -392,3 +388,26 @@ int ossl_cms_ecdh_envelope(CMS_RecipientInfo *ri, int decrypt) ERR_raise(ERR_LIB_CMS, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); return 0; } + +/* ECDSA and DSA implementation is the same */ +int ossl_cms_ecdsa_dsa_sign(CMS_SignerInfo *si, int verify) +{ + assert(verify == 0 || verify == 1); + + if (verify == 0) { + int snid, hnid; + X509_ALGOR *alg1, *alg2; + EVP_PKEY *pkey = si->pkey; + + CMS_SignerInfo_get0_algs(si, NULL, NULL, &alg1, &alg2); + if (alg1 == NULL || alg1->algorithm == NULL) + return -1; + hnid = OBJ_obj2nid(alg1->algorithm); + if (hnid == NID_undef) + return -1; + if (!OBJ_find_sigid_by_algs(&snid, hnid, EVP_PKEY_get_id(pkey))) + return -1; + X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0); + } + return 1; +} diff --git a/openssl/src/crypto/cms/cms_enc.c b/openssl/src/crypto/cms/cms_enc.c index ea8f07e1a..a896148dd 100644 --- a/openssl/src/crypto/cms/cms_enc.c +++ b/openssl/src/crypto/cms/cms_enc.c @@ -15,7 +15,6 @@ #include #include #include "crypto/evp.h" -#include "crypto/asn1.h" #include "cms_local.h" /* CMS EncryptedData Utilities */ @@ -45,7 +44,7 @@ BIO *ossl_cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec, b = BIO_new(BIO_f_cipher()); if (b == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return NULL; } @@ -82,17 +81,8 @@ BIO *ossl_cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec, if (enc) { calg->algorithm = OBJ_nid2obj(EVP_CIPHER_CTX_get_type(ctx)); - if (calg->algorithm == NULL || calg->algorithm->nid == NID_undef) { - ERR_raise(ERR_LIB_CMS, CMS_R_UNSUPPORTED_CONTENT_ENCRYPTION_ALGORITHM); - goto err; - } /* Generate a random IV if we need one */ ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); - if (ivlen < 0) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); - goto err; - } - if (ivlen > 0) { if (RAND_bytes_ex(libctx, iv, ivlen, 0) <= 0) goto err; @@ -121,8 +111,10 @@ BIO *ossl_cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec, /* Generate random session key */ if (!enc || !ec->key) { tkey = OPENSSL_malloc(tkeylen); - if (tkey == NULL) + if (tkey == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; + } if (EVP_CIPHER_CTX_rand_key(ctx, tkey) <= 0) goto err; } @@ -166,7 +158,7 @@ BIO *ossl_cms_EncryptedContent_init_bio(CMS_EncryptedContentInfo *ec, if (enc) { calg->parameter = ASN1_TYPE_new(); if (calg->parameter == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } if ((EVP_CIPHER_get_flags(cipher) & EVP_CIPH_FLAG_AEAD_CIPHER)) { @@ -209,8 +201,10 @@ int ossl_cms_EncryptedContent_init(CMS_EncryptedContentInfo *ec, { ec->cipher = cipher; if (key) { - if ((ec->key = OPENSSL_malloc(keylen)) == NULL) + if ((ec->key = OPENSSL_malloc(keylen)) == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(ec->key, key, keylen); } ec->keylen = keylen; @@ -231,7 +225,7 @@ int CMS_EncryptedData_set1_key(CMS_ContentInfo *cms, const EVP_CIPHER *ciph, if (ciph) { cms->d.encryptedData = M_ASN1_new_of(CMS_EncryptedData); if (!cms->d.encryptedData) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; } cms->contentType = OBJ_nid2obj(NID_pkcs7_encrypted); diff --git a/openssl/src/crypto/cms/cms_env.c b/openssl/src/crypto/cms/cms_env.c index 2d87738ee..51a1d7df8 100644 --- a/openssl/src/crypto/cms/cms_env.c +++ b/openssl/src/crypto/cms/cms_env.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,7 +26,7 @@ static void cms_env_set_version(CMS_EnvelopedData *env); #define CMS_ENVELOPED_STANDARD 1 #define CMS_ENVELOPED_AUTH 2 -static int cms_get_enveloped_type_simple(const CMS_ContentInfo *cms) +static int cms_get_enveloped_type(const CMS_ContentInfo *cms) { int nid = OBJ_obj2nid(cms->contentType); @@ -38,19 +38,11 @@ static int cms_get_enveloped_type_simple(const CMS_ContentInfo *cms) return CMS_ENVELOPED_AUTH; default: + ERR_raise(ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_ENVELOPED_DATA); return 0; } } -static int cms_get_enveloped_type(const CMS_ContentInfo *cms) -{ - int ret = cms_get_enveloped_type_simple(cms); - - if (ret == 0) - ERR_raise(ERR_LIB_CMS, CMS_R_CONTENT_TYPE_NOT_ENVELOPED_DATA); - return ret; -} - CMS_EnvelopedData *ossl_cms_get0_enveloped(CMS_ContentInfo *cms) { if (OBJ_obj2nid(cms->contentType) != NID_pkcs7_enveloped) { @@ -74,7 +66,7 @@ static CMS_EnvelopedData *cms_enveloped_data_init(CMS_ContentInfo *cms) if (cms->d.other == NULL) { cms->d.envelopedData = M_ASN1_new_of(CMS_EnvelopedData); if (cms->d.envelopedData == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return NULL; } cms->d.envelopedData->version = 0; @@ -93,7 +85,7 @@ cms_auth_enveloped_data_init(CMS_ContentInfo *cms) if (cms->d.other == NULL) { cms->d.authEnvelopedData = M_ASN1_new_of(CMS_AuthEnvelopedData); if (cms->d.authEnvelopedData == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return NULL; } /* Defined in RFC 5083 - Section 2.1. "AuthEnvelopedData Type" */ @@ -146,16 +138,14 @@ int ossl_cms_env_asn1_ctrl(CMS_RecipientInfo *ri, int cmd) return 1; } -CMS_EncryptedContentInfo *ossl_cms_get0_env_enc_content(const CMS_ContentInfo *cms) +CMS_EncryptedContentInfo* ossl_cms_get0_env_enc_content(const CMS_ContentInfo *cms) { switch (cms_get_enveloped_type(cms)) { case CMS_ENVELOPED_STANDARD: - return cms->d.envelopedData == NULL ? NULL - : cms->d.envelopedData->encryptedContentInfo; + return cms->d.envelopedData->encryptedContentInfo; case CMS_ENVELOPED_AUTH: - return cms->d.authEnvelopedData == NULL ? NULL - : cms->d.authEnvelopedData->authEncryptedContentInfo; + return cms->d.authEnvelopedData->authEncryptedContentInfo; default: return NULL; @@ -232,18 +222,18 @@ CMS_ContentInfo *CMS_EnvelopedData_create_ex(const EVP_CIPHER *cipher, cms = CMS_ContentInfo_new_ex(libctx, propq); if (cms == NULL) - goto err; + goto merr; env = cms_enveloped_data_init(cms); if (env == NULL) - goto err; + goto merr; if (!ossl_cms_EncryptedContent_init(env->encryptedContentInfo, cipher, NULL, 0, ossl_cms_get0_cmsctx(cms))) - goto err; + goto merr; return cms; - err: + merr: CMS_ContentInfo_free(cms); - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return NULL; } @@ -252,46 +242,6 @@ CMS_ContentInfo *CMS_EnvelopedData_create(const EVP_CIPHER *cipher) return CMS_EnvelopedData_create_ex(cipher, NULL, NULL); } -BIO *CMS_EnvelopedData_decrypt(CMS_EnvelopedData *env, BIO *detached_data, - EVP_PKEY *pkey, X509 *cert, - ASN1_OCTET_STRING *secret, unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq) -{ - CMS_ContentInfo *ci; - BIO *bio = NULL; - int res = 0; - - if (env == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - - if ((ci = CMS_ContentInfo_new_ex(libctx, propq)) == NULL - || (bio = BIO_new(BIO_s_mem())) == NULL) - goto end; - ci->contentType = OBJ_nid2obj(NID_pkcs7_enveloped); - ci->d.envelopedData = env; - if (secret != NULL - && CMS_decrypt_set1_password(ci, (unsigned char *) - ASN1_STRING_get0_data(secret), - ASN1_STRING_length(secret)) != 1) - goto end; - res = CMS_decrypt(ci, secret == NULL ? pkey : NULL, - secret == NULL ? cert : NULL, detached_data, bio, flags); - - end: - if (ci != NULL) { - ci->d.envelopedData = NULL; /* do not indirectly free |env| */ - ci->contentType = NULL; - } - CMS_ContentInfo_free(ci); - if (!res) { - BIO_free(bio); - bio = NULL; - } - return bio; -} - CMS_ContentInfo * CMS_AuthEnvelopedData_create_ex(const EVP_CIPHER *cipher, OSSL_LIB_CTX *libctx, const char *propq) @@ -312,7 +262,7 @@ CMS_AuthEnvelopedData_create_ex(const EVP_CIPHER *cipher, OSSL_LIB_CTX *libctx, return cms; merr: CMS_ContentInfo_free(cms); - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return NULL; } @@ -395,10 +345,8 @@ CMS_RecipientInfo *CMS_add1_recipient(CMS_ContentInfo *cms, X509 *recip, /* Initialize recipient info */ ri = M_ASN1_new_of(CMS_RecipientInfo); - if (ri == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (ri == NULL) + goto merr; pk = X509_get0_pubkey(recip); if (pk == NULL) { @@ -425,13 +373,13 @@ CMS_RecipientInfo *CMS_add1_recipient(CMS_ContentInfo *cms, X509 *recip, } - if (!sk_CMS_RecipientInfo_push(ris, ri)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); - goto err; - } + if (!sk_CMS_RecipientInfo_push(ris, ri)) + goto merr; return ri; + merr: + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); err: M_ASN1_free_of(ri, CMS_RecipientInfo); return NULL; @@ -542,8 +490,11 @@ static int cms_RecipientInfo_ktri_encrypt(const CMS_ContentInfo *cms, goto err; ek = OPENSSL_malloc(eklen); - if (ek == NULL) + + if (ek == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; + } if (EVP_PKEY_encrypt(pctx, ek, &eklen, ec->key, ec->keylen) <= 0) goto err; @@ -620,17 +571,25 @@ static int cms_RecipientInfo_ktri_decrypt(CMS_ContentInfo *cms, if (!ossl_cms_env_asn1_ctrl(ri, 1)) goto err; - if (EVP_PKEY_is_a(pkey, "RSA")) - /* upper layer CMS code incorrectly assumes that a successful RSA - * decryption means that the key matches ciphertext (which never - * was the case, implicit rejection or not), so to make it work - * disable implicit rejection for RSA keys */ - EVP_PKEY_CTX_ctrl_str(ktri->pctx, "rsa_pkcs1_implicit_rejection", "0"); + if (EVP_PKEY_decrypt(ktri->pctx, NULL, &eklen, + ktri->encryptedKey->data, + ktri->encryptedKey->length) <= 0) + goto err; - if (evp_pkey_decrypt_alloc(ktri->pctx, &ek, &eklen, fixlen, - ktri->encryptedKey->data, - ktri->encryptedKey->length) <= 0) + ek = OPENSSL_malloc(eklen); + if (ek == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (EVP_PKEY_decrypt(ktri->pctx, ek, &eklen, + ktri->encryptedKey->data, + ktri->encryptedKey->length) <= 0 + || eklen == 0 + || (fixlen != 0 && eklen != fixlen)) { + ERR_raise(ERR_LIB_CMS, CMS_R_CMS_LIB); goto err; + } ret = 1; @@ -736,32 +695,24 @@ CMS_RecipientInfo *CMS_add0_recipient_key(CMS_ContentInfo *cms, int nid, /* Initialize recipient info */ ri = M_ASN1_new_of(CMS_RecipientInfo); - if (!ri) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (!ri) + goto merr; ri->d.kekri = M_ASN1_new_of(CMS_KEKRecipientInfo); - if (!ri->d.kekri) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (!ri->d.kekri) + goto merr; ri->type = CMS_RECIPINFO_KEK; kekri = ri->d.kekri; if (otherTypeId) { kekri->kekid->other = M_ASN1_new_of(CMS_OtherKeyAttribute); - if (kekri->kekid->other == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (kekri->kekid->other == NULL) + goto merr; } - if (!sk_CMS_RecipientInfo_push(ris, ri)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); - goto err; - } + if (!sk_CMS_RecipientInfo_push(ris, ri)) + goto merr; /* After this point no calls can fail */ @@ -779,11 +730,13 @@ CMS_RecipientInfo *CMS_add0_recipient_key(CMS_ContentInfo *cms, int nid, kekri->kekid->other->keyAttr = otherType; } - (void)X509_ALGOR_set0(kekri->keyEncryptionAlgorithm, OBJ_nid2obj(nid), - V_ASN1_UNDEF, NULL); /* cannot fail */ + X509_ALGOR_set0(kekri->keyEncryptionAlgorithm, + OBJ_nid2obj(nid), V_ASN1_UNDEF, NULL); return ri; + merr: + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); err: M_ASN1_free_of(ri, CMS_RecipientInfo); return NULL; @@ -842,7 +795,7 @@ static EVP_CIPHER *cms_get_key_wrap_cipher(size_t keylen, const CMS_CTX *ctx) { const char *alg = NULL; - switch (keylen) { + switch(keylen) { case 16: alg = "AES-128-WRAP"; break; @@ -894,12 +847,14 @@ static int cms_RecipientInfo_kekri_encrypt(const CMS_ContentInfo *cms, /* 8 byte prefix for AES wrap ciphers */ wkey = OPENSSL_malloc(ec->keylen + 8); - if (wkey == NULL) + if (wkey == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; + } ctx = EVP_CIPHER_CTX_new(); if (ctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } @@ -975,12 +930,14 @@ static int cms_RecipientInfo_kekri_decrypt(CMS_ContentInfo *cms, } ukey = OPENSSL_malloc(kekri->encryptedKey->length - 8); - if (ukey == NULL) + if (ukey == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; + } ctx = EVP_CIPHER_CTX_new(); if (ctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } @@ -1278,7 +1235,7 @@ int ossl_cms_EnvelopedData_final(CMS_ContentInfo *cms, BIO *chain) env->unprotectedAttrs = sk_X509_ATTRIBUTE_new_null(); if (env->unprotectedAttrs == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; } diff --git a/openssl/src/crypto/cms/cms_err.c b/openssl/src/crypto/cms/cms_err.c index 40aeb7088..1fba9d850 100644 --- a/openssl/src/crypto/cms/cms_err.c +++ b/openssl/src/crypto/cms/cms_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -109,8 +109,6 @@ static const ERR_STRING_DATA CMS_str_reasons[] = { {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_NO_PUBLIC_KEY), "no public key"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_NO_RECEIPT_REQUEST), "no receipt request"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_NO_SIGNERS), "no signers"}, - {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_OPERATION_UNSUPPORTED), - "operation unsupported"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_PEER_KEY_ERROR), "peer key error"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE), "private key does not match certificate"}, @@ -140,8 +138,6 @@ static const ERR_STRING_DATA CMS_str_reasons[] = { {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNKNOWN_ID), "unknown id"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNSUPPORTED_COMPRESSION_ALGORITHM), "unsupported compression algorithm"}, - {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNSUPPORTED_CONTENT_ENCRYPTION_ALGORITHM), - "unsupported content encryption algorithm"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNSUPPORTED_CONTENT_TYPE), "unsupported content type"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNSUPPORTED_ENCRYPTION_TYPE), @@ -156,8 +152,6 @@ static const ERR_STRING_DATA CMS_str_reasons[] = { "unsupported recipientinfo type"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNSUPPORTED_RECIPIENT_TYPE), "unsupported recipient type"}, - {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNSUPPORTED_SIGNATURE_ALGORITHM), - "unsupported signature algorithm"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNSUPPORTED_TYPE), "unsupported type"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNWRAP_ERROR), "unwrap error"}, {ERR_PACK(ERR_LIB_CMS, 0, CMS_R_UNWRAP_FAILURE), "unwrap failure"}, diff --git a/openssl/src/crypto/cms/cms_ess.c b/openssl/src/crypto/cms/cms_ess.c index 0885a6821..6c43dd102 100644 --- a/openssl/src/crypto/cms/cms_ess.c +++ b/openssl/src/crypto/cms/cms_ess.c @@ -121,17 +121,13 @@ CMS_ReceiptRequest *CMS_ReceiptRequest_create0_ex( CMS_ReceiptRequest *rr; rr = CMS_ReceiptRequest_new(); - if (rr == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + if (rr == NULL) + goto merr; if (id) ASN1_STRING_set0(rr->signedContentIdentifier, id, idlen); else { - if (!ASN1_STRING_set(rr->signedContentIdentifier, NULL, 32)) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (!ASN1_STRING_set(rr->signedContentIdentifier, NULL, 32)) + goto merr; if (RAND_bytes_ex(libctx, rr->signedContentIdentifier->data, 32, 0) <= 0) goto err; @@ -150,6 +146,9 @@ CMS_ReceiptRequest *CMS_ReceiptRequest_create0_ex( return rr; + merr: + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); + err: CMS_ReceiptRequest_free(rr); return NULL; @@ -170,20 +169,19 @@ int CMS_add1_ReceiptRequest(CMS_SignerInfo *si, CMS_ReceiptRequest *rr) int rrderlen, r = 0; rrderlen = i2d_CMS_ReceiptRequest(rr, &rrder); - if (rrderlen < 0) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + if (rrderlen < 0) + goto merr; if (!CMS_signed_add1_attr_by_NID(si, NID_id_smime_aa_receiptRequest, - V_ASN1_SEQUENCE, rrder, rrderlen)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + V_ASN1_SEQUENCE, rrder, rrderlen)) + goto merr; r = 1; - err: + merr: + if (!r) + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); + OPENSSL_free(rrder); return r; @@ -243,7 +241,7 @@ int ossl_cms_msgSigDigest_add1(CMS_SignerInfo *dest, CMS_SignerInfo *src) } if (!CMS_signed_add1_attr_by_NID(dest, NID_id_smime_aa_msgSigDigest, V_ASN1_OCTET_STRING, dig, diglen)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; } return 1; diff --git a/openssl/src/crypto/cms/cms_io.c b/openssl/src/crypto/cms/cms_io.c index f5d70e84c..dab70af73 100644 --- a/openssl/src/crypto/cms/cms_io.c +++ b/openssl/src/crypto/cms/cms_io.c @@ -18,7 +18,6 @@ int CMS_stream(unsigned char ***boundary, CMS_ContentInfo *cms) { ASN1_OCTET_STRING **pos; - pos = CMS_get0_content(cms); if (pos == NULL) return 0; @@ -30,7 +29,7 @@ int CMS_stream(unsigned char ***boundary, CMS_ContentInfo *cms) *boundary = &(*pos)->data; return 1; } - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; } diff --git a/openssl/src/crypto/cms/cms_lib.c b/openssl/src/crypto/cms/cms_lib.c index 4ef614162..4ad930291 100644 --- a/openssl/src/crypto/cms/cms_lib.c +++ b/openssl/src/crypto/cms/cms_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,6 +14,7 @@ #include #include #include +#include #include "internal/sizes.h" #include "crypto/x509.h" #include "cms_local.h" @@ -21,7 +22,6 @@ static STACK_OF(CMS_CertificateChoices) **cms_get0_certificate_choices(CMS_ContentInfo *cms); -IMPLEMENT_ASN1_ALLOC_FUNCTIONS(CMS_ContentInfo) IMPLEMENT_ASN1_PRINT_FUNCTION(CMS_ContentInfo) CMS_ContentInfo *d2i_CMS_ContentInfo(CMS_ContentInfo **a, @@ -34,11 +34,8 @@ CMS_ContentInfo *d2i_CMS_ContentInfo(CMS_ContentInfo **a, (CMS_ContentInfo_it()), ossl_cms_ctx_get0_libctx(ctx), ossl_cms_ctx_get0_propq(ctx)); - if (ci != NULL) { - ERR_set_mark(); + if (ci != NULL) ossl_cms_resolve_libctx(ci); - ERR_pop_to_mark(); - } return ci; } @@ -61,12 +58,26 @@ CMS_ContentInfo *CMS_ContentInfo_new_ex(OSSL_LIB_CTX *libctx, const char *propq) if (ci->ctx.propq == NULL) { CMS_ContentInfo_free(ci); ci = NULL; + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); } } } return ci; } +CMS_ContentInfo *CMS_ContentInfo_new(void) +{ + return CMS_ContentInfo_new_ex(NULL, NULL); +} + +void CMS_ContentInfo_free(CMS_ContentInfo *cms) +{ + if (cms != NULL) { + OPENSSL_free(cms->ctx.propq); + ASN1_item_free((ASN1_VALUE *)cms, ASN1_ITEM_rptr(CMS_ContentInfo)); + } +} + const CMS_CTX *ossl_cms_get0_cmsctx(const CMS_ContentInfo *cms) { return cms != NULL ? &cms->ctx : NULL; @@ -162,7 +173,7 @@ BIO *CMS_dataInit(CMS_ContentInfo *cms, BIO *icont) case NID_pkcs7_digest: cmsbio = ossl_cms_DigestedData_init_bio(cms); break; -#ifndef OPENSSL_NO_ZLIB +#ifdef ZLIB case NID_id_smime_ct_compressedData: cmsbio = ossl_cms_CompressedData_init_bio(cms); break; @@ -196,13 +207,6 @@ BIO *CMS_dataInit(CMS_ContentInfo *cms, BIO *icont) /* unfortunately cannot constify SMIME_write_ASN1() due to this function */ int CMS_dataFinal(CMS_ContentInfo *cms, BIO *cmsbio) -{ - return ossl_cms_DataFinal(cms, cmsbio, NULL, 0); -} - -int ossl_cms_DataFinal(CMS_ContentInfo *cms, BIO *cmsbio, - const unsigned char *precomp_md, - unsigned int precomp_mdlen) { ASN1_OCTET_STRING **pos = CMS_get0_content(cms); @@ -241,7 +245,7 @@ int ossl_cms_DataFinal(CMS_ContentInfo *cms, BIO *cmsbio, return ossl_cms_AuthEnvelopedData_final(cms, cmsbio); case NID_pkcs7_signed: - return ossl_cms_SignedData_final(cms, cmsbio, precomp_md, precomp_mdlen); + return ossl_cms_SignedData_final(cms, cmsbio); case NID_pkcs7_digest: return ossl_cms_DigestedData_do_final(cms, cmsbio, 0); @@ -391,7 +395,7 @@ int CMS_set_detached(CMS_ContentInfo *cms, int detached) (*pos)->flags |= ASN1_STRING_FLAG_CONT; return 1; } - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; } @@ -425,7 +429,7 @@ BIO *ossl_cms_DigestAlgorithm_init_bio(X509_ALGOR *digestAlgorithm, (void)ERR_pop_to_mark(); mdbio = BIO_new(BIO_f_md()); - if (mdbio == NULL || BIO_set_md(mdbio, digest) <= 0) { + if (mdbio == NULL || !BIO_set_md(mdbio, digest)) { ERR_raise(ERR_LIB_CMS, CMS_R_MD_BIO_INIT_ERROR); goto err; } @@ -525,9 +529,9 @@ int CMS_add0_cert(CMS_ContentInfo *cms, X509 *cert) for (i = 0; i < sk_CMS_CertificateChoices_num(*pcerts); i++) { cch = sk_CMS_CertificateChoices_value(*pcerts, i); if (cch->type == CMS_CERTCHOICE_CERT) { - if (X509_cmp(cch->d.certificate, cert) == 0) { - X509_free(cert); - return 1; /* cert already present */ + if (!X509_cmp(cch->d.certificate, cert)) { + ERR_raise(ERR_LIB_CMS, CMS_R_CERTIFICATE_ALREADY_PRESENT); + return 0; } } } @@ -541,12 +545,11 @@ int CMS_add0_cert(CMS_ContentInfo *cms, X509 *cert) int CMS_add1_cert(CMS_ContentInfo *cms, X509 *cert) { - if (!X509_up_ref(cert)) - return 0; - if (CMS_add0_cert(cms, cert)) - return 1; - X509_free(cert); - return 0; + int r; + r = CMS_add0_cert(cms, cert); + if (r > 0) + X509_up_ref(cert); + return r; } static STACK_OF(CMS_RevocationInfoChoice) @@ -598,9 +601,9 @@ CMS_RevocationInfoChoice *CMS_add0_RevocationInfoChoice(CMS_ContentInfo *cms) int CMS_add0_crl(CMS_ContentInfo *cms, X509_CRL *crl) { - CMS_RevocationInfoChoice *rch = CMS_add0_RevocationInfoChoice(cms); - - if (rch == NULL) + CMS_RevocationInfoChoice *rch; + rch = CMS_add0_RevocationInfoChoice(cms); + if (!rch) return 0; rch->type = CMS_REVCHOICE_CRL; rch->d.crl = crl; @@ -609,12 +612,11 @@ int CMS_add0_crl(CMS_ContentInfo *cms, X509_CRL *crl) int CMS_add1_crl(CMS_ContentInfo *cms, X509_CRL *crl) { - if (!X509_CRL_up_ref(crl)) - return 0; - if (CMS_add0_crl(cms, crl)) - return 1; - X509_CRL_free(crl); - return 0; + int r; + r = CMS_add0_crl(cms, crl); + if (r > 0) + X509_CRL_up_ref(crl); + return r; } STACK_OF(X509) *CMS_get1_certs(CMS_ContentInfo *cms) @@ -632,7 +634,7 @@ STACK_OF(X509) *CMS_get1_certs(CMS_ContentInfo *cms) if (cch->type == 0) { if (!ossl_x509_add_cert_new(&certs, cch->d.certificate, X509_ADD_FLAG_UP_REF)) { - OSSL_STACK_OF_X509_free(certs); + sk_X509_pop_free(certs, X509_free); return NULL; } } @@ -654,15 +656,16 @@ STACK_OF(X509_CRL) *CMS_get1_crls(CMS_ContentInfo *cms) for (i = 0; i < sk_CMS_RevocationInfoChoice_num(*pcrls); i++) { rch = sk_CMS_RevocationInfoChoice_value(*pcrls, i); if (rch->type == 0) { - if (crls == NULL) { - if ((crls = sk_X509_CRL_new_null()) == NULL) + if (!crls) { + crls = sk_X509_CRL_new_null(); + if (!crls) return NULL; } - if (!sk_X509_CRL_push(crls, rch->d.crl) - || !X509_CRL_up_ref(rch->d.crl)) { + if (!sk_X509_CRL_push(crls, rch->d.crl)) { sk_X509_CRL_pop_free(crls, X509_CRL_free); return NULL; } + X509_CRL_up_ref(rch->d.crl); } } return crls; @@ -690,23 +693,18 @@ int ossl_cms_set1_ias(CMS_IssuerAndSerialNumber **pias, X509 *cert) { CMS_IssuerAndSerialNumber *ias; ias = M_ASN1_new_of(CMS_IssuerAndSerialNumber); - if (!ias) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + if (!ias) goto err; - } - if (!X509_NAME_set(&ias->issuer, X509_get_issuer_name(cert))) { - ERR_raise(ERR_LIB_CMS, ERR_R_X509_LIB); + if (!X509_NAME_set(&ias->issuer, X509_get_issuer_name(cert))) goto err; - } - if (!ASN1_STRING_copy(ias->serialNumber, X509_get0_serialNumber(cert))) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + if (!ASN1_STRING_copy(ias->serialNumber, X509_get0_serialNumber(cert))) goto err; - } M_ASN1_free_of(*pias, CMS_IssuerAndSerialNumber); *pias = ias; return 1; err: M_ASN1_free_of(ias, CMS_IssuerAndSerialNumber); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; } @@ -721,7 +719,7 @@ int ossl_cms_set1_keyid(ASN1_OCTET_STRING **pkeyid, X509 *cert) } keyid = ASN1_STRING_dup(cert_keyid); if (!keyid) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; } ASN1_OCTET_STRING_free(*pkeyid); diff --git a/openssl/src/crypto/cms/cms_local.h b/openssl/src/crypto/cms/cms_local.h index fd5c7c9a6..15b4a29ce 100644 --- a/openssl/src/crypto/cms/cms_local.h +++ b/openssl/src/crypto/cms/cms_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,9 +21,11 @@ typedef struct CMS_IssuerAndSerialNumber_st CMS_IssuerAndSerialNumber; typedef struct CMS_EncapsulatedContentInfo_st CMS_EncapsulatedContentInfo; typedef struct CMS_SignerIdentifier_st CMS_SignerIdentifier; +typedef struct CMS_SignedData_st CMS_SignedData; typedef struct CMS_OtherRevocationInfoFormat_st CMS_OtherRevocationInfoFormat; typedef struct CMS_OriginatorInfo_st CMS_OriginatorInfo; typedef struct CMS_EncryptedContentInfo_st CMS_EncryptedContentInfo; +typedef struct CMS_EnvelopedData_st CMS_EnvelopedData; typedef struct CMS_DigestedData_st CMS_DigestedData; typedef struct CMS_EncryptedData_st CMS_EncryptedData; typedef struct CMS_AuthenticatedData_st CMS_AuthenticatedData; @@ -366,7 +368,6 @@ struct CMS_Receipt_st { DECLARE_ASN1_FUNCTIONS(CMS_ContentInfo) DECLARE_ASN1_ITEM(CMS_SignerInfo) -DECLARE_ASN1_ITEM(CMS_EncryptedContentInfo) DECLARE_ASN1_ITEM(CMS_IssuerAndSerialNumber) DECLARE_ASN1_ITEM(CMS_Attributes_Sign) DECLARE_ASN1_ITEM(CMS_Attributes_Verify) @@ -394,9 +395,6 @@ const char *ossl_cms_ctx_get0_propq(const CMS_CTX *ctx); void ossl_cms_resolve_libctx(CMS_ContentInfo *ci); CMS_ContentInfo *ossl_cms_Data_create(OSSL_LIB_CTX *ctx, const char *propq); -int ossl_cms_DataFinal(CMS_ContentInfo *cms, BIO *cmsbio, - const unsigned char *precomp_md, - unsigned int precomp_mdlen); CMS_ContentInfo *ossl_cms_DigestedData_create(const EVP_MD *md, OSSL_LIB_CTX *libctx, @@ -406,9 +404,7 @@ int ossl_cms_DigestedData_do_final(const CMS_ContentInfo *cms, BIO *chain, int verify); BIO *ossl_cms_SignedData_init_bio(CMS_ContentInfo *cms); -int ossl_cms_SignedData_final(CMS_ContentInfo *cms, BIO *chain, - const unsigned char *precomp_md, - unsigned int precomp_mdlen); +int ossl_cms_SignedData_final(CMS_ContentInfo *cms, BIO *chain); int ossl_cms_set1_SignerIdentifier(CMS_SignerIdentifier *sid, X509 *cert, int type, const CMS_CTX *ctx); int ossl_cms_SignerIdentifier_get0_signer_id(CMS_SignerIdentifier *sid, @@ -483,6 +479,7 @@ int ossl_cms_check_signing_certs(const CMS_SignerInfo *si, int ossl_cms_dh_envelope(CMS_RecipientInfo *ri, int decrypt); int ossl_cms_ecdh_envelope(CMS_RecipientInfo *ri, int decrypt); int ossl_cms_rsa_envelope(CMS_RecipientInfo *ri, int decrypt); +int ossl_cms_ecdsa_dsa_sign(CMS_SignerInfo *si, int verify); int ossl_cms_rsa_sign(CMS_SignerInfo *si, int verify); DECLARE_ASN1_ITEM(CMS_CertificateChoices) diff --git a/openssl/src/crypto/cms/cms_pwri.c b/openssl/src/crypto/cms/cms_pwri.c index 67efa8766..380240561 100644 --- a/openssl/src/crypto/cms/cms_pwri.c +++ b/openssl/src/crypto/cms/cms_pwri.c @@ -1,5 +1,5 @@ /* - * Copyright 2009-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2009-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -82,12 +82,11 @@ CMS_RecipientInfo *CMS_add0_recipient_password(CMS_ContentInfo *cms, /* Setup algorithm identifier for cipher */ encalg = X509_ALGOR_new(); if (encalg == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; + goto merr; } ctx = EVP_CIPHER_CTX_new(); if (ctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } @@ -97,10 +96,6 @@ CMS_RecipientInfo *CMS_add0_recipient_password(CMS_ContentInfo *cms, } ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); - if (ivlen < 0) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); - goto err; - } if (ivlen > 0) { if (RAND_bytes_ex(ossl_cms_ctx_get0_libctx(cms_ctx), iv, ivlen, 0) <= 0) @@ -111,7 +106,7 @@ CMS_RecipientInfo *CMS_add0_recipient_password(CMS_ContentInfo *cms, } encalg->parameter = ASN1_TYPE_new(); if (!encalg->parameter) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } if (EVP_CIPHER_param_to_asn1(ctx, encalg->parameter) <= 0) { @@ -127,16 +122,12 @@ CMS_RecipientInfo *CMS_add0_recipient_password(CMS_ContentInfo *cms, /* Initialize recipient info */ ri = M_ASN1_new_of(CMS_RecipientInfo); - if (ri == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (ri == NULL) + goto merr; ri->d.pwri = M_ASN1_new_of(CMS_PasswordRecipientInfo); - if (ri->d.pwri == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (ri->d.pwri == NULL) + goto merr; ri->type = CMS_RECIPINFO_PASS; pwri = ri->d.pwri; @@ -144,23 +135,17 @@ CMS_RecipientInfo *CMS_add0_recipient_password(CMS_ContentInfo *cms, /* Since this is overwritten, free up empty structure already there */ X509_ALGOR_free(pwri->keyEncryptionAlgorithm); pwri->keyEncryptionAlgorithm = X509_ALGOR_new(); - if (pwri->keyEncryptionAlgorithm == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (pwri->keyEncryptionAlgorithm == NULL) + goto merr; pwri->keyEncryptionAlgorithm->algorithm = OBJ_nid2obj(wrap_nid); pwri->keyEncryptionAlgorithm->parameter = ASN1_TYPE_new(); - if (pwri->keyEncryptionAlgorithm->parameter == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (pwri->keyEncryptionAlgorithm->parameter == NULL) + goto merr; if (!ASN1_item_pack(encalg, ASN1_ITEM_rptr(X509_ALGOR), &pwri->keyEncryptionAlgorithm->parameter-> - value.sequence)) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + value.sequence)) + goto merr; pwri->keyEncryptionAlgorithm->parameter->type = V_ASN1_SEQUENCE; X509_ALGOR_free(encalg); @@ -176,13 +161,13 @@ CMS_RecipientInfo *CMS_add0_recipient_password(CMS_ContentInfo *cms, CMS_RecipientInfo_set0_password(ri, pass, passlen); pwri->version = 0; - if (!sk_CMS_RecipientInfo_push(ris, ri)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); - goto err; - } + if (!sk_CMS_RecipientInfo_push(ris, ri)) + goto merr; return ri; + merr: + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); err: EVP_CIPHER_CTX_free(ctx); if (ri) @@ -204,10 +189,6 @@ static int kek_unwrap_key(unsigned char *out, size_t *outlen, size_t blocklen = EVP_CIPHER_CTX_get_block_size(ctx); unsigned char *tmp; int outl, rv = 0; - - if (blocklen == 0) - return 0; - if (inlen < 2 * blocklen) { /* too small */ return 0; @@ -216,8 +197,10 @@ static int kek_unwrap_key(unsigned char *out, size_t *outlen, /* Invalid size */ return 0; } - if ((tmp = OPENSSL_malloc(inlen)) == NULL) + if ((tmp = OPENSSL_malloc(inlen)) == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; + } /* setup IV by decrypting last two blocks */ if (!EVP_DecryptUpdate(ctx, tmp + inlen - 2 * blocklen, &outl, in + inlen - 2 * blocklen, blocklen * 2) @@ -261,10 +244,6 @@ static int kek_wrap_key(unsigned char *out, size_t *outlen, size_t blocklen = EVP_CIPHER_CTX_get_block_size(ctx); size_t olen; int dummy; - - if (blocklen == 0) - return 0; - /* * First decide length of output buffer: need header and round up to * multiple of block length. @@ -352,7 +331,7 @@ int ossl_cms_RecipientInfo_pwri_crypt(const CMS_ContentInfo *cms, kekctx = EVP_CIPHER_CTX_new(); if (kekctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } /* Fixup cipher based on AlgorithmIdentifier to set IV etc */ @@ -393,8 +372,11 @@ int ossl_cms_RecipientInfo_pwri_crypt(const CMS_ContentInfo *cms, pwri->encryptedKey->length = keylen; } else { key = OPENSSL_malloc(pwri->encryptedKey->length); - if (key == NULL) + + if (key == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; + } if (!kek_unwrap_key(key, &keylen, pwri->encryptedKey->data, pwri->encryptedKey->length, kekctx)) { diff --git a/openssl/src/crypto/cms/cms_rsa.c b/openssl/src/crypto/cms/cms_rsa.c index 31436d4d6..20ed81691 100644 --- a/openssl/src/crypto/cms/cms_rsa.c +++ b/openssl/src/crypto/cms/cms_rsa.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,10 +10,8 @@ #include #include #include -#include #include "crypto/asn1.h" #include "crypto/rsa.h" -#include "crypto/evp.h" #include "cms_local.h" static RSA_OAEP_PARAMS *rsa_oaep_decode(const X509_ALGOR *alg) @@ -99,10 +97,8 @@ static int rsa_cms_decrypt(CMS_RecipientInfo *ri) if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0) goto err; if (label != NULL - && EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0) { - OPENSSL_free(label); + && EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0) goto err; - } /* Carry on */ rv = 1; @@ -116,7 +112,6 @@ static int rsa_cms_encrypt(CMS_RecipientInfo *ri) const EVP_MD *md, *mgf1md; RSA_OAEP_PARAMS *oaep = NULL; ASN1_STRING *os = NULL; - ASN1_OCTET_STRING *los = NULL; X509_ALGOR *alg; EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri); int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen; @@ -128,10 +123,10 @@ static int rsa_cms_encrypt(CMS_RecipientInfo *ri) if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0) return 0; } - if (pad_mode == RSA_PKCS1_PADDING) - return X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), - V_ASN1_NULL, NULL); - + if (pad_mode == RSA_PKCS1_PADDING) { + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0); + return 1; + } /* Not supported */ if (pad_mode != RSA_PKCS1_OAEP_PADDING) return 0; @@ -150,31 +145,30 @@ static int rsa_cms_encrypt(CMS_RecipientInfo *ri) if (!ossl_x509_algor_md_to_mgf1(&oaep->maskGenFunc, mgf1md)) goto err; if (labellen > 0) { - los = ASN1_OCTET_STRING_new(); + ASN1_OCTET_STRING *los; - if (los == NULL) + oaep->pSourceFunc = X509_ALGOR_new(); + if (oaep->pSourceFunc == NULL) goto err; - if (!ASN1_OCTET_STRING_set(los, label, labellen)) + los = ASN1_OCTET_STRING_new(); + if (los == NULL) goto err; - - oaep->pSourceFunc = ossl_X509_ALGOR_from_nid(NID_pSpecified, - V_ASN1_OCTET_STRING, los); - if (oaep->pSourceFunc == NULL) + if (!ASN1_OCTET_STRING_set(los, label, labellen)) { + ASN1_OCTET_STRING_free(los); goto err; - - los = NULL; + } + X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified), + V_ASN1_OCTET_STRING, los); } - /* create string with oaep parameter encoding. */ + /* create string with pss parameter encoding. */ if (!ASN1_item_pack(oaep, ASN1_ITEM_rptr(RSA_OAEP_PARAMS), &os)) - goto err; - if (!X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os)) - goto err; + goto err; + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os); os = NULL; rv = 1; err: RSA_OAEP_PARAMS_free(oaep); ASN1_STRING_free(os); - ASN1_OCTET_STRING_free(los); return rv; } @@ -197,44 +191,24 @@ static int rsa_cms_sign(CMS_SignerInfo *si) int pad_mode = RSA_PKCS1_PADDING; X509_ALGOR *alg; EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si); - unsigned char aid[128]; - const unsigned char *pp = aid; - size_t aid_len = 0; - OSSL_PARAM params[2]; + ASN1_STRING *os = NULL; CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg); if (pkctx != NULL) { if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0) return 0; } - if (pad_mode == RSA_PKCS1_PADDING) - return X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), - V_ASN1_NULL, NULL); - + if (pad_mode == RSA_PKCS1_PADDING) { + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0); + return 1; + } /* We don't support it */ if (pad_mode != RSA_PKCS1_PSS_PADDING) return 0; - - if (evp_pkey_ctx_is_legacy(pkctx)) { - /* No provider -> we cannot query it for algorithm ID. */ - ASN1_STRING *os = NULL; - - os = ossl_rsa_ctx_to_pss_string(pkctx); - if (os == NULL) - return 0; - return X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_PKEY_RSA_PSS), V_ASN1_SEQUENCE, os); - } - - params[0] = OSSL_PARAM_construct_octet_string( - OSSL_SIGNATURE_PARAM_ALGORITHM_ID, aid, sizeof(aid)); - params[1] = OSSL_PARAM_construct_end(); - - if (EVP_PKEY_CTX_get_params(pkctx, params) <= 0) - return 0; - if ((aid_len = params[0].return_size) == 0) - return 0; - if (d2i_X509_ALGOR(&alg, &pp, aid_len) == NULL) + os = ossl_rsa_ctx_to_pss_string(pkctx); + if (os == NULL) return 0; + X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_PKEY_RSA_PSS), V_ASN1_SEQUENCE, os); return 1; } diff --git a/openssl/src/crypto/cms/cms_sd.c b/openssl/src/crypto/cms/cms_sd.c index 8ad94a9ed..f8fe3bee8 100644 --- a/openssl/src/crypto/cms/cms_sd.c +++ b/openssl/src/crypto/cms/cms_sd.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -38,7 +38,7 @@ static CMS_SignedData *cms_signed_data_init(CMS_ContentInfo *cms) if (cms->d.other == NULL) { cms->d.signedData = M_ASN1_new_of(CMS_SignedData); if (!cms->d.signedData) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return NULL; } cms->d.signedData->version = 1; @@ -53,6 +53,7 @@ static CMS_SignedData *cms_signed_data_init(CMS_ContentInfo *cms) } /* Just initialise SignedData e.g. for certs only structure */ + int CMS_SignedData_init(CMS_ContentInfo *cms) { if (cms_signed_data_init(cms)) @@ -61,7 +62,9 @@ int CMS_SignedData_init(CMS_ContentInfo *cms) return 0; } + /* Check structures and fixup version numbers (if necessary) */ + static void cms_sd_set_version(CMS_SignedData *sd) { int i; @@ -102,9 +105,8 @@ static void cms_sd_set_version(CMS_SignedData *sd) si->version = 3; if (sd->version < 3) sd->version = 3; - } else if (si->version < 1) { + } else if (si->version < 1) si->version = 1; - } } if (sd->version < 1) @@ -134,6 +136,7 @@ static int cms_set_si_contentType_attr(CMS_ContentInfo *cms, CMS_SignerInfo *si) } /* Copy an existing messageDigest value */ + static int cms_copy_messageDigest(CMS_ContentInfo *cms, CMS_SignerInfo *si) { STACK_OF(CMS_SignerInfo) *sinfos; @@ -209,9 +212,8 @@ int ossl_cms_SignerIdentifier_get0_signer_id(CMS_SignerIdentifier *sid, } else if (sid->type == CMS_SIGNERINFO_KEYIDENTIFIER) { if (keyid) *keyid = sid->d.subjectKeyIdentifier; - } else { + } else return 0; - } return 1; } @@ -225,50 +227,19 @@ int ossl_cms_SignerIdentifier_cert_cmp(CMS_SignerIdentifier *sid, X509 *cert) return -1; } -/* Method to map any, incl. provider-implemented PKEY types to OIDs */ -/* (EC)DSA and all provider-delivered signatures implementation is the same */ -static int cms_generic_sign(CMS_SignerInfo *si, int verify) -{ - if (!ossl_assert(verify == 0 || verify == 1)) - return -1; - - if (!verify) { - EVP_PKEY *pkey = si->pkey; - int snid, hnid, pknid = EVP_PKEY_get_id(pkey); - X509_ALGOR *alg1, *alg2; - - CMS_SignerInfo_get0_algs(si, NULL, NULL, &alg1, &alg2); - if (alg1 == NULL || alg1->algorithm == NULL) - return -1; - hnid = OBJ_obj2nid(alg1->algorithm); - if (hnid == NID_undef) - return -1; - if (pknid <= 0) { /* check whether a provider registered a NID */ - const char *typename = EVP_PKEY_get0_type_name(pkey); - - if (typename != NULL) - pknid = OBJ_txt2nid(typename); - } - if (!OBJ_find_sigid_by_algs(&snid, hnid, pknid)) - return -1; - return X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, NULL); - } - return 1; -} - static int cms_sd_asn1_ctrl(CMS_SignerInfo *si, int cmd) { EVP_PKEY *pkey = si->pkey; int i; if (EVP_PKEY_is_a(pkey, "DSA") || EVP_PKEY_is_a(pkey, "EC")) - return cms_generic_sign(si, cmd) > 0; + return ossl_cms_ecdsa_dsa_sign(si, cmd); else if (EVP_PKEY_is_a(pkey, "RSA") || EVP_PKEY_is_a(pkey, "RSA-PSS")) - return ossl_cms_rsa_sign(si, cmd) > 0; + return ossl_cms_rsa_sign(si, cmd); - /* Now give engines, providers, etc a chance to handle this */ + /* Something else? We'll give engines etc a chance to handle this */ if (pkey->ameth == NULL || pkey->ameth->pkey_ctrl == NULL) - return cms_generic_sign(si, cmd) > 0; + return 1; i = pkey->ameth->pkey_ctrl(pkey, ASN1_PKEY_CTRL_CMS_SIGN, cmd, si); if (i == -2) { ERR_raise(ERR_LIB_CMS, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); @@ -349,10 +320,8 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, if (!sd) goto err; si = M_ASN1_new_of(CMS_SignerInfo); - if (!si) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + if (!si) + goto merr; /* Call for side-effect of computing hash and caching extensions */ X509_check_purpose(signer, -1, -1); @@ -366,7 +335,7 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, si->pctx = NULL; if (si->mctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } @@ -385,20 +354,25 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, if (md == NULL) { int def_nid; - - if (EVP_PKEY_get_default_digest_nid(pk, &def_nid) <= 0) { - ERR_raise_data(ERR_LIB_CMS, CMS_R_NO_DEFAULT_DIGEST, - "pkey nid=%d", EVP_PKEY_get_id(pk)); + if (EVP_PKEY_get_default_digest_nid(pk, &def_nid) <= 0) goto err; - } md = EVP_get_digestbynid(def_nid); if (md == NULL) { - ERR_raise_data(ERR_LIB_CMS, CMS_R_NO_DEFAULT_DIGEST, - "default md nid=%d", def_nid); + ERR_raise(ERR_LIB_CMS, CMS_R_NO_DEFAULT_DIGEST); goto err; } } + if (!md) { + ERR_raise(ERR_LIB_CMS, CMS_R_NO_DIGEST_SET); + goto err; + } + + if (md == NULL) { + ERR_raise(ERR_LIB_CMS, CMS_R_NO_DIGEST_SET); + goto err; + } + X509_ALGOR_set_md(si->digestAlgorithm, md); /* See if digest is present in digestAlgorithms */ @@ -414,23 +388,18 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, } if (i == sk_X509_ALGOR_num(sd->digestAlgorithms)) { - if ((alg = X509_ALGOR_new()) == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB); - goto err; - } + alg = X509_ALGOR_new(); + if (alg == NULL) + goto merr; X509_ALGOR_set_md(alg, md); if (!sk_X509_ALGOR_push(sd->digestAlgorithms, alg)) { X509_ALGOR_free(alg); - ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); - goto err; + goto merr; } } - if (!(flags & CMS_KEY_PARAM) && !cms_sd_asn1_ctrl(si, 0)) { - ERR_raise_data(ERR_LIB_CMS, CMS_R_UNSUPPORTED_SIGNATURE_ALGORITHM, - "pkey nid=%d", EVP_PKEY_get_id(pk)); + if (!(flags & CMS_KEY_PARAM) && !cms_sd_asn1_ctrl(si, 0)) goto err; - } if (!(flags & CMS_NOATTR)) { /* * Initialize signed attributes structure so other attributes @@ -438,23 +407,18 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, */ if (!si->signedAttrs) { si->signedAttrs = sk_X509_ATTRIBUTE_new_null(); - if (!si->signedAttrs) { - ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); - goto err; - } + if (!si->signedAttrs) + goto merr; } if (!(flags & CMS_NOSMIMECAP)) { STACK_OF(X509_ALGOR) *smcap = NULL; - i = CMS_add_standard_smimecap(&smcap); if (i) i = CMS_add_smimecap(si, smcap); sk_X509_ALGOR_pop_free(smcap, X509_ALGOR_free); - if (!i) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + if (!i) + goto merr; } if (flags & CMS_CADES) { ESS_SIGNING_CERT *sc = NULL; @@ -490,10 +454,8 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, if (!(flags & CMS_NOCERTS)) { /* NB ignore -1 return for duplicate cert */ - if (!CMS_add1_cert(cms, signer)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + if (!CMS_add1_cert(cms, signer)) + goto merr; } if (flags & CMS_KEY_PARAM) { @@ -512,23 +474,19 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms, ossl_cms_ctx_get0_libctx(ctx), ossl_cms_ctx_get0_propq(ctx), pk, NULL) <= 0) { - si->pctx = NULL; goto err; } - else { - EVP_MD_CTX_set_flags(si->mctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX); - } } - if (sd->signerInfos == NULL) + if (!sd->signerInfos) sd->signerInfos = sk_CMS_SignerInfo_new_null(); - if (sd->signerInfos == NULL || !sk_CMS_SignerInfo_push(sd->signerInfos, si)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB); - goto err; - } + if (!sd->signerInfos || !sk_CMS_SignerInfo_push(sd->signerInfos, si)) + goto merr; return si; + merr: + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); err: M_ASN1_free_of(si, CMS_SignerInfo); return NULL; @@ -563,22 +521,21 @@ static int cms_add1_signingTime(CMS_SignerInfo *si, ASN1_TIME *t) else tt = X509_gmtime_adj(NULL, 0); - if (tt == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_X509_LIB); - goto err; - } + if (tt == NULL) + goto merr; if (CMS_signed_add1_attr_by_NID(si, NID_pkcs9_signingTime, - tt->type, tt, -1) <= 0) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + tt->type, tt, -1) <= 0) + goto merr; r = 1; - err: + merr: if (t == NULL) ASN1_TIME_free(tt); + if (!r) + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); + return r; } @@ -678,7 +635,7 @@ int CMS_set1_signers_certs(CMS_ContentInfo *cms, STACK_OF(X509) *scerts, for (j = 0; j < sk_CMS_CertificateChoices_num(certs); j++) { cch = sk_CMS_CertificateChoices_value(certs, j); - if (cch->type != CMS_CERTCHOICE_CERT) + if (cch->type != 0) continue; x = cch->d.certificate; if (CMS_SignerInfo_cert_cmp(si, x) == 0) { @@ -711,9 +668,7 @@ ASN1_OCTET_STRING *CMS_SignerInfo_get0_signature(CMS_SignerInfo *si) } static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, - CMS_SignerInfo *si, BIO *chain, - const unsigned char *md, - unsigned int mdlen) + CMS_SignerInfo *si, BIO *chain) { EVP_MD_CTX *mctx = EVP_MD_CTX_new(); int r = 0; @@ -721,7 +676,7 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, const CMS_CTX *ctx = ossl_cms_get0_cmsctx(cms); if (mctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); return 0; } @@ -739,14 +694,13 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, /* * If any signed attributes calculate and add messageDigest attribute */ + if (CMS_signed_get_attr_count(si) >= 0) { - unsigned char computed_md[EVP_MAX_MD_SIZE]; + unsigned char md[EVP_MAX_MD_SIZE]; + unsigned int mdlen; - if (md == NULL) { - if (!EVP_DigestFinal_ex(mctx, computed_md, &mdlen)) - goto err; - md = computed_md; - } + if (!EVP_DigestFinal_ex(mctx, md, &mdlen)) + goto err; if (!CMS_signed_add1_attr_by_NID(si, NID_pkcs9_messageDigest, V_ASN1_OCTET_STRING, md, mdlen)) goto err; @@ -759,18 +713,18 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, } else if (si->pctx) { unsigned char *sig; size_t siglen; - unsigned char computed_md[EVP_MAX_MD_SIZE]; + unsigned char md[EVP_MAX_MD_SIZE]; + unsigned int mdlen; pctx = si->pctx; - si->pctx = NULL; - if (md == NULL) { - if (!EVP_DigestFinal_ex(mctx, computed_md, &mdlen)) - goto err; - md = computed_md; - } + if (!EVP_DigestFinal_ex(mctx, md, &mdlen)) + goto err; siglen = EVP_PKEY_get_size(si->pkey); - if (siglen == 0 || (sig = OPENSSL_malloc(siglen)) == NULL) + sig = OPENSSL_malloc(siglen); + if (sig == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; + } if (EVP_PKEY_sign(pctx, sig, &siglen, md, mdlen) <= 0) { OPENSSL_free(sig); goto err; @@ -780,13 +734,11 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, unsigned char *sig; unsigned int siglen; - if (md != NULL) { - ERR_raise(ERR_LIB_CMS, CMS_R_OPERATION_UNSUPPORTED); + sig = OPENSSL_malloc(EVP_PKEY_get_size(si->pkey)); + if (sig == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } - siglen = EVP_PKEY_get_size(si->pkey); - if (siglen == 0 || (sig = OPENSSL_malloc(siglen)) == NULL) - goto err; if (!EVP_SignFinal_ex(mctx, sig, &siglen, si->pkey, ossl_cms_ctx_get0_libctx(ctx), ossl_cms_ctx_get0_propq(ctx))) { @@ -806,9 +758,7 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms, } -int ossl_cms_SignedData_final(CMS_ContentInfo *cms, BIO *chain, - const unsigned char *precomp_md, - unsigned int precomp_mdlen) +int ossl_cms_SignedData_final(CMS_ContentInfo *cms, BIO *chain) { STACK_OF(CMS_SignerInfo) *sinfos; CMS_SignerInfo *si; @@ -817,8 +767,7 @@ int ossl_cms_SignedData_final(CMS_ContentInfo *cms, BIO *chain, sinfos = CMS_get0_SignerInfos(cms); for (i = 0; i < sk_CMS_SignerInfo_num(sinfos); i++) { si = sk_CMS_SignerInfo_value(sinfos, i); - if (!cms_SignerInfo_content_sign(cms, si, chain, - precomp_md, precomp_mdlen)) + if (!cms_SignerInfo_content_sign(cms, si, chain)) return 0; } cms->d.signedData->encapContentInfo->partial = 0; @@ -836,7 +785,7 @@ int CMS_SignerInfo_sign(CMS_SignerInfo *si) char md_name[OSSL_MAX_NAME_SIZE]; if (OBJ_obj2txt(md_name, sizeof(md_name), - si->digestAlgorithm->algorithm, 0) <= 0) + si->digestAlgorithm->algorithm, 0) <= 0) return 0; if (CMS_signed_get_attr_by_NID(si, NID_pkcs9_signingTime, -1) < 0) { @@ -847,16 +796,15 @@ int CMS_SignerInfo_sign(CMS_SignerInfo *si) if (!ossl_cms_si_check_attributes(si)) goto err; - if (si->pctx) { + if (si->pctx) pctx = si->pctx; - } else { + else { EVP_MD_CTX_reset(mctx); if (EVP_DigestSignInit_ex(mctx, &pctx, md_name, ossl_cms_ctx_get0_libctx(ctx), ossl_cms_ctx_get0_propq(ctx), si->pkey, NULL) <= 0) goto err; - EVP_MD_CTX_set_flags(mctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX); si->pctx = pctx; } @@ -924,20 +872,13 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si) (void)ERR_pop_to_mark(); if (si->mctx == NULL && (si->mctx = EVP_MD_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } mctx = si->mctx; - if (si->pctx != NULL) { - EVP_PKEY_CTX_free(si->pctx); - si->pctx = NULL; - } if (EVP_DigestVerifyInit_ex(mctx, &si->pctx, EVP_MD_get0_name(md), libctx, - propq, si->pkey, NULL) <= 0) { - si->pctx = NULL; + propq, si->pkey, NULL) <= 0) goto err; - } - EVP_MD_CTX_set_flags(mctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX); if (!cms_sd_asn1_ctrl(si, 1)) goto err; @@ -963,6 +904,7 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si) } /* Create a chain of digest BIOs from a CMS ContentInfo */ + BIO *ossl_cms_SignedData_init_bio(CMS_ContentInfo *cms) { int i; @@ -1004,7 +946,7 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) unsigned int mlen; if (mctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } /* If we have any signed attributes look for messageDigest value */ @@ -1027,6 +969,7 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) } /* If messageDigest found compare it */ + if (os != NULL) { if (mlen != (unsigned int)os->length) { ERR_raise(ERR_LIB_CMS, CMS_R_MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH); @@ -1036,9 +979,8 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) if (memcmp(mval, os->data, mlen)) { ERR_raise(ERR_LIB_CMS, CMS_R_VERIFICATION_FAILURE); r = 0; - } else { + } else r = 1; - } } else { const EVP_MD *md = EVP_MD_CTX_get0_md(mctx); const CMS_CTX *ctx = si->cms_ctx; @@ -1053,11 +995,8 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) if (EVP_PKEY_CTX_set_signature_md(pkctx, md) <= 0) goto err; si->pctx = pkctx; - if (!cms_sd_asn1_ctrl(si, 1)) { - si->pctx = NULL; + if (!cms_sd_asn1_ctrl(si, 1)) goto err; - } - si->pctx = NULL; r = EVP_PKEY_verify(pkctx, si->signature->data, si->signature->length, mval, mlen); if (r <= 0) { @@ -1073,47 +1012,6 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain) } -BIO *CMS_SignedData_verify(CMS_SignedData *sd, BIO *detached_data, - STACK_OF(X509) *scerts, X509_STORE *store, - STACK_OF(X509) *extra, STACK_OF(X509_CRL) *crls, - unsigned int flags, - OSSL_LIB_CTX *libctx, const char *propq) -{ - CMS_ContentInfo *ci; - BIO *bio = NULL; - int i, res = 0; - - if (sd == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - - if ((ci = CMS_ContentInfo_new_ex(libctx, propq)) == NULL) - return NULL; - if ((bio = BIO_new(BIO_s_mem())) == NULL) - goto end; - ci->contentType = OBJ_nid2obj(NID_pkcs7_signed); - ci->d.signedData = sd; - - for (i = 0; i < sk_X509_num(extra); i++) - if (!CMS_add1_cert(ci, sk_X509_value(extra, i))) - goto end; - for (i = 0; i < sk_X509_CRL_num(crls); i++) - if (!CMS_add1_crl(ci, sk_X509_CRL_value(crls, i))) - goto end; - res = CMS_verify(ci, scerts, store, detached_data, bio, flags); - - end: - if (ci != NULL) - ci->d.signedData = NULL; /* do not indirectly free |sd| */ - CMS_ContentInfo_free(ci); - if (!res) { - BIO_free(bio); - bio = NULL; - } - return bio; -} - int CMS_add_smimecap(CMS_SignerInfo *si, STACK_OF(X509_ALGOR) *algs) { unsigned char *smder = NULL; @@ -1141,13 +1039,14 @@ int CMS_add_simple_smimecap(STACK_OF(X509_ALGOR) **algs, return 0; } } - alg = ossl_X509_ALGOR_from_nid(algnid, key != NULL ? V_ASN1_INTEGER : - V_ASN1_UNDEF, key); + alg = X509_ALGOR_new(); if (alg == NULL) { ASN1_INTEGER_free(key); return 0; } + X509_ALGOR_set0(alg, OBJ_nid2obj(algnid), + key ? V_ASN1_INTEGER : V_ASN1_UNDEF, key); if (*algs == NULL) *algs = sk_X509_ALGOR_new_null(); if (*algs == NULL || !sk_X509_ALGOR_push(*algs, alg)) { @@ -1158,6 +1057,7 @@ int CMS_add_simple_smimecap(STACK_OF(X509_ALGOR) **algs, } /* Check to see if a cipher exists and if so add S/MIME capabilities */ + static int cms_add_cipher_smcap(STACK_OF(X509_ALGOR) **sk, int nid, int arg) { if (EVP_get_cipherbynid(nid)) @@ -1165,27 +1065,13 @@ static int cms_add_cipher_smcap(STACK_OF(X509_ALGOR) **sk, int nid, int arg) return 1; } -static int cms_add_digest_smcap(STACK_OF(X509_ALGOR) **sk, int nid, int arg) -{ - if (EVP_get_digestbynid(nid)) - return CMS_add_simple_smimecap(sk, nid, arg); - return 1; -} - int CMS_add_standard_smimecap(STACK_OF(X509_ALGOR) **smcap) { if (!cms_add_cipher_smcap(smcap, NID_aes_256_cbc, -1) - || !cms_add_digest_smcap(smcap, NID_id_GostR3411_2012_256, -1) - || !cms_add_digest_smcap(smcap, NID_id_GostR3411_2012_512, -1) - || !cms_add_digest_smcap(smcap, NID_id_GostR3411_94, -1) - || !cms_add_cipher_smcap(smcap, NID_id_Gost28147_89, -1) || !cms_add_cipher_smcap(smcap, NID_aes_192_cbc, -1) || !cms_add_cipher_smcap(smcap, NID_aes_128_cbc, -1) || !cms_add_cipher_smcap(smcap, NID_des_ede3_cbc, -1) - || !cms_add_cipher_smcap(smcap, NID_rc2_cbc, 128) - || !cms_add_cipher_smcap(smcap, NID_rc2_cbc, 64) - || !cms_add_cipher_smcap(smcap, NID_des_cbc, -1) - || !cms_add_cipher_smcap(smcap, NID_rc2_cbc, 40)) + || !cms_add_cipher_smcap(smcap, NID_des_cbc, -1)) return 0; return 1; } diff --git a/openssl/src/crypto/cms/cms_smime.c b/openssl/src/crypto/cms/cms_smime.c index 3a8b13d6e..4e80a13b4 100644 --- a/openssl/src/crypto/cms/cms_smime.c +++ b/openssl/src/crypto/cms/cms_smime.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2008-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -39,7 +39,7 @@ static int cms_copy_content(BIO *out, BIO *in, unsigned int flags) tmpout = cms_get_text_bio(out, flags); if (tmpout == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } @@ -48,7 +48,7 @@ static int cms_copy_content(BIO *out, BIO *in, unsigned int flags) i = BIO_read(in, buf, sizeof(buf)); if (i <= 0) { if (BIO_method_type(in) == BIO_TYPE_CIPHER) { - if (BIO_get_cipher_status(in) <= 0) + if (!BIO_get_cipher_status(in)) goto err; } if (i < 0) @@ -236,7 +236,7 @@ CMS_ContentInfo *CMS_EncryptedData_encrypt_ex(BIO *in, const EVP_CIPHER *cipher, if (cms == NULL) return NULL; if (!CMS_EncryptedData_set1_key(cms, cipher, key, keylen)) - goto err; + return NULL; if (!(flags & CMS_DETACHED)) CMS_set_detached(cms, 0); @@ -245,7 +245,6 @@ CMS_ContentInfo *CMS_EncryptedData_encrypt_ex(BIO *in, const EVP_CIPHER *cipher, || CMS_final(cms, in, NULL, flags)) return cms; - err: CMS_ContentInfo_free(cms); return NULL; } @@ -260,7 +259,7 @@ CMS_ContentInfo *CMS_EncryptedData_encrypt(BIO *in, const EVP_CIPHER *cipher, static int cms_signerinfo_verify_cert(CMS_SignerInfo *si, X509_STORE *store, - STACK_OF(X509) *untrusted, + STACK_OF(X509) *certs, STACK_OF(X509_CRL) *crls, STACK_OF(X509) **chain, const CMS_CTX *cms_ctx) @@ -272,11 +271,11 @@ static int cms_signerinfo_verify_cert(CMS_SignerInfo *si, ctx = X509_STORE_CTX_new_ex(ossl_cms_ctx_get0_libctx(cms_ctx), ossl_cms_ctx_get0_propq(cms_ctx)); if (ctx == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } CMS_SignerInfo_get0_algs(si, NULL, &signer, NULL, NULL); - if (!X509_STORE_CTX_init(ctx, store, signer, untrusted)) { + if (!X509_STORE_CTX_init(ctx, store, signer, certs)) { ERR_raise(ERR_LIB_CMS, CMS_R_STORE_INIT_ERROR); goto err; } @@ -302,7 +301,6 @@ static int cms_signerinfo_verify_cert(CMS_SignerInfo *si, } -/* This strongly overlaps with PKCS7_verify() */ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, X509_STORE *store, BIO *dcont, BIO *out, unsigned int flags) { @@ -338,7 +336,7 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, for (i = 0; i < sk_CMS_SignerInfo_num(sinfos); i++) { si = sk_CMS_SignerInfo_value(sinfos, i); CMS_SignerInfo_get0_algs(si, NULL, &signer, NULL, NULL); - if (signer != NULL) + if (signer) scount++; } @@ -357,8 +355,10 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, if (cadesVerify) { /* Certificate trust chain is required to check CAdES signature */ si_chains = OPENSSL_zalloc(scount * sizeof(si_chains[0])); - if (si_chains == NULL) + if (si_chains == NULL) { + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; + } } cms_certs = CMS_get1_certs(cms); if (!(flags & CMS_NOCRL)) @@ -405,7 +405,7 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, len = BIO_get_mem_data(dcont, &ptr); tmpin = (len == 0) ? dcont : BIO_new_mem_buf(ptr, len); if (tmpin == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err2; } } else { @@ -422,7 +422,7 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, */ tmpout = cms_get_text_bio(out, flags); if (tmpout == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); goto err; } cmsbio = CMS_dataInit(cms, tmpout); @@ -432,8 +432,7 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, * Don't use SMIME_TEXT for verify: it adds headers and we want to * remove them. */ - if (!SMIME_crlf_copy(dcont, cmsbio, flags & ~SMIME_TEXT)) - goto err; + SMIME_crlf_copy(dcont, cmsbio, flags & ~SMIME_TEXT); if (flags & CMS_TEXT) { if (!SMIME_text(tmpout, out)) { @@ -479,10 +478,10 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs, err2: if (si_chains != NULL) { for (i = 0; i < scount; ++i) - OSSL_STACK_OF_X509_free(si_chains[i]); + sk_X509_pop_free(si_chains[i], X509_free); OPENSSL_free(si_chains); } - OSSL_STACK_OF_X509_free(cms_certs); + sk_X509_pop_free(cms_certs, X509_free); sk_X509_CRL_pop_free(crls, X509_CRL_free); return ret; @@ -510,16 +509,12 @@ CMS_ContentInfo *CMS_sign_ex(X509 *signcert, EVP_PKEY *pkey, int i; cms = CMS_ContentInfo_new_ex(libctx, propq); - if (cms == NULL || !CMS_SignedData_init(cms)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + if (cms == NULL || !CMS_SignedData_init(cms)) + goto merr; if (flags & CMS_ASCIICRLF && !CMS_set1_eContentType(cms, - OBJ_nid2obj(NID_id_ct_asciiTextWithCRLF))) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + OBJ_nid2obj(NID_id_ct_asciiTextWithCRLF))) goto err; - } if (pkey != NULL && !CMS_add1_signer(cms, signcert, pkey, NULL, flags)) { ERR_raise(ERR_LIB_CMS, CMS_R_ADD_SIGNER_ERROR); @@ -529,10 +524,8 @@ CMS_ContentInfo *CMS_sign_ex(X509 *signcert, EVP_PKEY *pkey, for (i = 0; i < sk_X509_num(certs); i++) { X509 *x = sk_X509_value(certs, i); - if (!CMS_add1_cert(cms, x)) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + if (!CMS_add1_cert(cms, x)) + goto merr; } if (!(flags & CMS_DETACHED)) @@ -544,6 +537,9 @@ CMS_ContentInfo *CMS_sign_ex(X509 *signcert, EVP_PKEY *pkey, else goto err; + merr: + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); + err: CMS_ContentInfo_free(cms); return NULL; @@ -561,7 +557,7 @@ CMS_ContentInfo *CMS_sign_receipt(CMS_SignerInfo *si, { CMS_SignerInfo *rct_si; CMS_ContentInfo *cms = NULL; - ASN1_OCTET_STRING **pos, *os = NULL; + ASN1_OCTET_STRING **pos, *os; BIO *rct_cont = NULL; int r = 0; const CMS_CTX *ctx = si->cms_ctx; @@ -623,7 +619,6 @@ CMS_ContentInfo *CMS_sign_receipt(CMS_SignerInfo *si, if (r) return cms; CMS_ContentInfo_free(cms); - ASN1_OCTET_STRING_free(os); return NULL; } @@ -640,10 +635,8 @@ CMS_ContentInfo *CMS_encrypt_ex(STACK_OF(X509) *certs, BIO *data, cms = (EVP_CIPHER_get_flags(cipher) & EVP_CIPH_FLAG_AEAD_CIPHER) ? CMS_AuthEnvelopedData_create_ex(cipher, libctx, propq) : CMS_EnvelopedData_create_ex(cipher, libctx, propq); - if (cms == NULL) { - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); - goto err; - } + if (cms == NULL) + goto merr; for (i = 0; i < sk_X509_num(certs); i++) { recip = sk_X509_value(certs, i); if (!CMS_add1_recipient_cert(cms, recip, flags)) { @@ -659,8 +652,10 @@ CMS_ContentInfo *CMS_encrypt_ex(STACK_OF(X509) *certs, BIO *data, || CMS_final(cms, data, NULL, flags)) return cms; else - ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB); + goto err; + merr: + ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE); err: CMS_ContentInfo_free(cms); return NULL; @@ -705,21 +700,14 @@ int CMS_decrypt_set1_pkey(CMS_ContentInfo *cms, EVP_PKEY *pk, X509 *cert) int CMS_decrypt_set1_pkey_and_peer(CMS_ContentInfo *cms, EVP_PKEY *pk, X509 *cert, X509 *peer) { - STACK_OF(CMS_RecipientInfo) *ris = CMS_get0_RecipientInfos(cms); + STACK_OF(CMS_RecipientInfo) *ris; CMS_RecipientInfo *ri; int i, r, cms_pkey_ri_type; int debug = 0, match_ri = 0; - CMS_EncryptedContentInfo *ec = ossl_cms_get0_env_enc_content(cms); - - /* Prevent mem leak on earlier CMS_decrypt_set1_{pkey_and_peer,password} */ - if (ec != NULL) { - OPENSSL_clear_free(ec->key, ec->keylen); - ec->key = NULL; - ec->keylen = 0; - } - if (ris != NULL && ec != NULL) - debug = ec->debug; + ris = CMS_get0_RecipientInfos(cms); + if (ris != NULL) + debug = ossl_cms_get0_env_enc_content(cms)->debug; cms_pkey_ri_type = ossl_cms_pkey_get_ri_type(pk); if (cms_pkey_ri_type == CMS_RECIPINFO_NONE) { @@ -742,8 +730,11 @@ int CMS_decrypt_set1_pkey_and_peer(CMS_ContentInfo *cms, EVP_PKEY *pk, if (r < 0) return 0; } - /* If we have a cert, try matching RecipientInfo, else try them all */ - else if (cert == NULL || !CMS_RecipientInfo_ktri_cert_cmp(ri, cert)) { + /* + * If we have a cert try matching RecipientInfo otherwise try them + * all. + */ + else if (cert == NULL|| !CMS_RecipientInfo_ktri_cert_cmp(ri, cert)) { EVP_PKEY_up_ref(pk); CMS_RecipientInfo_set0_pkey(ri, pk); r = CMS_RecipientInfo_decrypt(cms, ri); @@ -780,8 +771,7 @@ int CMS_decrypt_set1_pkey_and_peer(CMS_ContentInfo *cms, EVP_PKEY *pk, return 1; } - if (!match_ri) - ERR_raise(ERR_LIB_CMS, CMS_R_NO_MATCHING_RECIPIENT); + ERR_raise(ERR_LIB_CMS, CMS_R_NO_MATCHING_RECIPIENT); return 0; } @@ -792,7 +782,7 @@ int CMS_decrypt_set1_key(CMS_ContentInfo *cms, { STACK_OF(CMS_RecipientInfo) *ris; CMS_RecipientInfo *ri; - int i, r, match_ri = 0; + int i, r; ris = CMS_get0_RecipientInfos(cms); for (i = 0; i < sk_CMS_RecipientInfo_num(ris); i++) { @@ -800,10 +790,11 @@ int CMS_decrypt_set1_key(CMS_ContentInfo *cms, if (CMS_RecipientInfo_type(ri) != CMS_RECIPINFO_KEK) continue; - /* If we have an id, try matching RecipientInfo, else try them all */ - if (id == NULL - || (CMS_RecipientInfo_kekri_id_cmp(ri, id, idlen) == 0)) { - match_ri = 1; + /* + * If we have an id try matching RecipientInfo otherwise try them + * all. + */ + if (id == NULL || (CMS_RecipientInfo_kekri_id_cmp(ri, id, idlen) == 0)) { CMS_RecipientInfo_set0_key(ri, key, keylen); r = CMS_RecipientInfo_decrypt(cms, ri); CMS_RecipientInfo_set0_key(ri, NULL, 0); @@ -817,8 +808,7 @@ int CMS_decrypt_set1_key(CMS_ContentInfo *cms, } } - if (!match_ri) - ERR_raise(ERR_LIB_CMS, CMS_R_NO_MATCHING_RECIPIENT); + ERR_raise(ERR_LIB_CMS, CMS_R_NO_MATCHING_RECIPIENT); return 0; } @@ -826,25 +816,15 @@ int CMS_decrypt_set1_key(CMS_ContentInfo *cms, int CMS_decrypt_set1_password(CMS_ContentInfo *cms, unsigned char *pass, ossl_ssize_t passlen) { - STACK_OF(CMS_RecipientInfo) *ris = CMS_get0_RecipientInfos(cms); + STACK_OF(CMS_RecipientInfo) *ris; CMS_RecipientInfo *ri; - int i, r, match_ri = 0; - CMS_EncryptedContentInfo *ec = ossl_cms_get0_env_enc_content(cms); - - /* Prevent mem leak on earlier CMS_decrypt_set1_{pkey_and_peer,password} */ - if (ec != NULL) { - OPENSSL_clear_free(ec->key, ec->keylen); - ec->key = NULL; - ec->keylen = 0; - } + int i, r; + ris = CMS_get0_RecipientInfos(cms); for (i = 0; i < sk_CMS_RecipientInfo_num(ris); i++) { ri = sk_CMS_RecipientInfo_value(ris, i); if (CMS_RecipientInfo_type(ri) != CMS_RECIPINFO_PASS) continue; - - /* Must try each PasswordRecipientInfo */ - match_ri = 1; CMS_RecipientInfo_set0_password(ri, pass, passlen); r = CMS_RecipientInfo_decrypt(cms, ri); CMS_RecipientInfo_set0_password(ri, NULL, 0); @@ -852,8 +832,7 @@ int CMS_decrypt_set1_password(CMS_ContentInfo *cms, return 1; } - if (!match_ri) - ERR_raise(ERR_LIB_CMS, CMS_R_NO_MATCHING_RECIPIENT); + ERR_raise(ERR_LIB_CMS, CMS_R_NO_MATCHING_RECIPIENT); return 0; } @@ -863,7 +842,7 @@ int CMS_decrypt(CMS_ContentInfo *cms, EVP_PKEY *pk, X509 *cert, { int r; BIO *cont; - CMS_EncryptedContentInfo *ec; + int nid = OBJ_obj2nid(CMS_get0_type(cms)); if (nid != NID_pkcs7_enveloped @@ -873,9 +852,14 @@ int CMS_decrypt(CMS_ContentInfo *cms, EVP_PKEY *pk, X509 *cert, } if (dcont == NULL && !check_content(cms)) return 0; - ec = ossl_cms_get0_env_enc_content(cms); - ec->debug = (flags & CMS_DEBUG_DECRYPT) != 0; - ec->havenocert = cert == NULL; + if (flags & CMS_DEBUG_DECRYPT) + ossl_cms_get0_env_enc_content(cms)->debug = 1; + else + ossl_cms_get0_env_enc_content(cms)->debug = 0; + if (cert == NULL) + ossl_cms_get0_env_enc_content(cms)->havenocert = 1; + else + ossl_cms_get0_env_enc_content(cms)->havenocert = 0; if (pk == NULL && cert == NULL && dcont == NULL && out == NULL) return 1; if (pk != NULL && !CMS_decrypt_set1_pkey(cms, pk, cert)) @@ -898,9 +882,7 @@ int CMS_final(CMS_ContentInfo *cms, BIO *data, BIO *dcont, unsigned int flags) return 0; } - if (!SMIME_crlf_copy(data, cmsbio, flags)) { - goto err; - } + ret = SMIME_crlf_copy(data, cmsbio, flags); (void)BIO_flush(cmsbio); @@ -908,9 +890,6 @@ int CMS_final(CMS_ContentInfo *cms, BIO *data, BIO *dcont, unsigned int flags) ERR_raise(ERR_LIB_CMS, CMS_R_CMS_DATAFINAL_ERROR); goto err; } - - ret = 1; - err: do_free_upto(cmsbio, dcont); @@ -918,32 +897,7 @@ int CMS_final(CMS_ContentInfo *cms, BIO *data, BIO *dcont, unsigned int flags) } -int CMS_final_digest(CMS_ContentInfo *cms, - const unsigned char *md, unsigned int mdlen, - BIO *dcont, unsigned int flags) -{ - BIO *cmsbio; - int ret = 0; - - if ((cmsbio = CMS_dataInit(cms, dcont)) == NULL) { - ERR_raise(ERR_LIB_CMS, CMS_R_CMS_LIB); - return 0; - } - - (void)BIO_flush(cmsbio); - - if (!ossl_cms_DataFinal(cms, cmsbio, md, mdlen)) { - ERR_raise(ERR_LIB_CMS, CMS_R_CMS_DATAFINAL_ERROR); - goto err; - } - ret = 1; - -err: - do_free_upto(cmsbio, dcont); - return ret; -} - -#ifndef OPENSSL_NO_ZLIB +#ifdef ZLIB int CMS_uncompress(CMS_ContentInfo *cms, BIO *dcont, BIO *out, unsigned int flags) diff --git a/openssl/src/crypto/comp/c_brotli.c b/openssl/src/crypto/comp/c_brotli.c deleted file mode 100644 index 07e1e7647..000000000 --- a/openssl/src/crypto/comp/c_brotli.c +++ /dev/null @@ -1,800 +0,0 @@ -/* - * Copyright 1998-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - * - * Uses brotli compression library from https://github.com/google/brotli - */ - -#include -#include -#include -#include -#include "internal/comp.h" -#include -#include "crypto/cryptlib.h" -#include "internal/bio.h" -#include "internal/thread_once.h" -#include "comp_local.h" - -COMP_METHOD *COMP_brotli(void); - -#ifdef OPENSSL_NO_BROTLI -# undef BROTLI_SHARED -#else - -# include -# include - -/* memory allocations functions for brotli initialisation */ -static void *brotli_alloc(void *opaque, size_t size) -{ - return OPENSSL_zalloc(size); -} - -static void brotli_free(void *opaque, void *address) -{ - OPENSSL_free(address); -} - -/* - * When OpenSSL is built on Windows, we do not want to require that - * the BROTLI.DLL be available in order for the OpenSSL DLLs to - * work. Therefore, all BROTLI routines are loaded at run time - * and we do not link to a .LIB file when BROTLI_SHARED is set. - */ -# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32) -# include -# endif - -# ifdef BROTLI_SHARED -# include "internal/dso.h" - -/* Function pointers */ -typedef BrotliEncoderState *(*encode_init_ft)(brotli_alloc_func, brotli_free_func, void *); -typedef BROTLI_BOOL (*encode_stream_ft)(BrotliEncoderState *, BrotliEncoderOperation, size_t *, const uint8_t **, size_t *, uint8_t **, size_t *); -typedef BROTLI_BOOL (*encode_has_more_ft)(BrotliEncoderState *); -typedef void (*encode_end_ft)(BrotliEncoderState *); -typedef BROTLI_BOOL (*encode_oneshot_ft)(int, int, BrotliEncoderMode, size_t, const uint8_t in[], size_t *, uint8_t out[]); - -typedef BrotliDecoderState *(*decode_init_ft)(brotli_alloc_func, brotli_free_func, void *); -typedef BROTLI_BOOL (*decode_stream_ft)(BrotliDecoderState *, size_t *, const uint8_t **, size_t *, uint8_t **, size_t *); -typedef BROTLI_BOOL (*decode_has_more_ft)(BrotliDecoderState *); -typedef void (*decode_end_ft)(BrotliDecoderState *); -typedef BrotliDecoderErrorCode (*decode_error_ft)(BrotliDecoderState *); -typedef const char *(*decode_error_string_ft)(BrotliDecoderErrorCode); -typedef BROTLI_BOOL (*decode_is_finished_ft)(BrotliDecoderState *); -typedef BrotliDecoderResult (*decode_oneshot_ft)(size_t, const uint8_t in[], size_t *, uint8_t out[]); - -static encode_init_ft p_encode_init = NULL; -static encode_stream_ft p_encode_stream = NULL; -static encode_has_more_ft p_encode_has_more = NULL; -static encode_end_ft p_encode_end = NULL; -static encode_oneshot_ft p_encode_oneshot = NULL; - -static decode_init_ft p_decode_init = NULL; -static decode_stream_ft p_decode_stream = NULL; -static decode_has_more_ft p_decode_has_more = NULL; -static decode_end_ft p_decode_end = NULL; -static decode_error_ft p_decode_error = NULL; -static decode_error_string_ft p_decode_error_string = NULL; -static decode_is_finished_ft p_decode_is_finished = NULL; -static decode_oneshot_ft p_decode_oneshot = NULL; - -static DSO *brotli_encode_dso = NULL; -static DSO *brotli_decode_dso = NULL; - -# define BrotliEncoderCreateInstance p_encode_init -# define BrotliEncoderCompressStream p_encode_stream -# define BrotliEncoderHasMoreOutput p_encode_has_more -# define BrotliEncoderDestroyInstance p_encode_end -# define BrotliEncoderCompress p_encode_oneshot - -# define BrotliDecoderCreateInstance p_decode_init -# define BrotliDecoderDecompressStream p_decode_stream -# define BrotliDecoderHasMoreOutput p_decode_has_more -# define BrotliDecoderDestroyInstance p_decode_end -# define BrotliDecoderGetErrorCode p_decode_error -# define BrotliDecoderErrorString p_decode_error_string -# define BrotliDecoderIsFinished p_decode_is_finished -# define BrotliDecoderDecompress p_decode_oneshot - -# endif /* ifdef BROTLI_SHARED */ - - -struct brotli_state { - BrotliEncoderState *encoder; - BrotliDecoderState *decoder; -}; - -static int brotli_stateful_init(COMP_CTX *ctx) -{ - struct brotli_state *state = OPENSSL_zalloc(sizeof(*state)); - - if (state == NULL) - return 0; - - state->encoder = BrotliEncoderCreateInstance(brotli_alloc, brotli_free, NULL); - if (state->encoder == NULL) - goto err; - - state->decoder = BrotliDecoderCreateInstance(brotli_alloc, brotli_free, NULL); - if (state->decoder == NULL) - goto err; - - ctx->data = state; - return 1; - err: - BrotliDecoderDestroyInstance(state->decoder); - BrotliEncoderDestroyInstance(state->encoder); - OPENSSL_free(state); - return 0; -} - -static void brotli_stateful_finish(COMP_CTX *ctx) -{ - struct brotli_state *state = ctx->data; - - if (state != NULL) { - BrotliDecoderDestroyInstance(state->decoder); - BrotliEncoderDestroyInstance(state->encoder); - OPENSSL_free(state); - ctx->data = NULL; - } -} - -static ossl_ssize_t brotli_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - BROTLI_BOOL done; - struct brotli_state *state = ctx->data; - size_t in_avail = ilen; - size_t out_avail = olen; - - if (state == NULL || olen > OSSL_SSIZE_MAX) - return -1; - - if (ilen == 0) - return 0; - - /* - * The finish API does not provide a final output buffer, - * so each compress operation has to be flushed, if all - * the input data can't be accepted, or there is more output, - * this has to be considered an error, since there is no more - * output buffer space - */ - done = BrotliEncoderCompressStream(state->encoder, BROTLI_OPERATION_FLUSH, - &in_avail, (const uint8_t**)&in, - &out_avail, &out, NULL); - if (done == BROTLI_FALSE - || in_avail != 0 - || BrotliEncoderHasMoreOutput(state->encoder)) - return -1; - - if (out_avail > olen) - return -1; - return (ossl_ssize_t)(olen - out_avail); -} - -static ossl_ssize_t brotli_stateful_expand_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - BrotliDecoderResult result; - struct brotli_state *state = ctx->data; - size_t in_avail = ilen; - size_t out_avail = olen; - - if (state == NULL || olen > OSSL_SSIZE_MAX) - return -1; - - if (ilen == 0) - return 0; - - result = BrotliDecoderDecompressStream(state->decoder, &in_avail, - (const uint8_t**)&in, &out_avail, - &out, NULL); - if (result == BROTLI_DECODER_RESULT_ERROR - || in_avail != 0 - || BrotliDecoderHasMoreOutput(state->decoder)) - return -1; - - if (out_avail > olen) - return -1; - return (ossl_ssize_t)(olen - out_avail); -} - -static COMP_METHOD brotli_stateful_method = { - NID_brotli, - LN_brotli, - brotli_stateful_init, - brotli_stateful_finish, - brotli_stateful_compress_block, - brotli_stateful_expand_block -}; - -static int brotli_oneshot_init(COMP_CTX *ctx) -{ - return 1; -} - -static void brotli_oneshot_finish(COMP_CTX *ctx) -{ -} - -static ossl_ssize_t brotli_oneshot_compress_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - size_t out_size = olen; - ossl_ssize_t ret; - - if (ilen == 0) - return 0; - - if (BrotliEncoderCompress(BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, - BROTLI_DEFAULT_MODE, ilen, in, - &out_size, out) == BROTLI_FALSE) - return -1; - - if (out_size > OSSL_SSIZE_MAX) - return -1; - ret = (ossl_ssize_t)out_size; - if (ret < 0) - return -1; - return ret; -} - -static ossl_ssize_t brotli_oneshot_expand_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - size_t out_size = olen; - ossl_ssize_t ret; - - if (ilen == 0) - return 0; - - if (BrotliDecoderDecompress(ilen, in, &out_size, out) != BROTLI_DECODER_RESULT_SUCCESS) - return -1; - - if (out_size > OSSL_SSIZE_MAX) - return -1; - ret = (ossl_ssize_t)out_size; - if (ret < 0) - return -1; - return ret; -} - -static COMP_METHOD brotli_oneshot_method = { - NID_brotli, - LN_brotli, - brotli_oneshot_init, - brotli_oneshot_finish, - brotli_oneshot_compress_block, - brotli_oneshot_expand_block -}; - -static CRYPTO_ONCE brotli_once = CRYPTO_ONCE_STATIC_INIT; -DEFINE_RUN_ONCE_STATIC(ossl_comp_brotli_init) -{ -# ifdef BROTLI_SHARED -# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32) -# define LIBBROTLIENC "BROTLIENC" -# define LIBBROTLIDEC "BROTLIDEC" -# else -# define LIBBROTLIENC "brotlienc" -# define LIBBROTLIDEC "brotlidec" -# endif - - brotli_encode_dso = DSO_load(NULL, LIBBROTLIENC, NULL, 0); - if (brotli_encode_dso != NULL) { - p_encode_init = (encode_init_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCreateInstance"); - p_encode_stream = (encode_stream_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCompressStream"); - p_encode_has_more = (encode_has_more_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderHasMoreOutput"); - p_encode_end = (encode_end_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderDestroyInstance"); - p_encode_oneshot = (encode_oneshot_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCompress"); - } - - brotli_decode_dso = DSO_load(NULL, LIBBROTLIDEC, NULL, 0); - if (brotli_decode_dso != NULL) { - p_decode_init = (decode_init_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderCreateInstance"); - p_decode_stream = (decode_stream_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDecompressStream"); - p_decode_has_more = (decode_has_more_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderHasMoreOutput"); - p_decode_end = (decode_end_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDestroyInstance"); - p_decode_error = (decode_error_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderGetErrorCode"); - p_decode_error_string = (decode_error_string_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderErrorString"); - p_decode_is_finished = (decode_is_finished_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderIsFinished"); - p_decode_oneshot = (decode_oneshot_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDecompress"); - } - - if (p_encode_init == NULL || p_encode_stream == NULL || p_encode_has_more == NULL - || p_encode_end == NULL || p_encode_oneshot == NULL || p_decode_init == NULL - || p_decode_stream == NULL || p_decode_has_more == NULL || p_decode_end == NULL - || p_decode_error == NULL || p_decode_error_string == NULL || p_decode_is_finished == NULL - || p_decode_oneshot == NULL) { - ossl_comp_brotli_cleanup(); - return 0; - } -# endif - return 1; -} -#endif /* ifndef BROTLI / else */ - -COMP_METHOD *COMP_brotli(void) -{ - COMP_METHOD *meth = NULL; - -#ifndef OPENSSL_NO_BROTLI - if (RUN_ONCE(&brotli_once, ossl_comp_brotli_init)) - meth = &brotli_stateful_method; -#endif - return meth; -} - -COMP_METHOD *COMP_brotli_oneshot(void) -{ - COMP_METHOD *meth = NULL; - -#ifndef OPENSSL_NO_BROTLI - if (RUN_ONCE(&brotli_once, ossl_comp_brotli_init)) - meth = &brotli_oneshot_method; -#endif - return meth; -} - -/* Also called from OPENSSL_cleanup() */ -void ossl_comp_brotli_cleanup(void) -{ -#ifdef BROTLI_SHARED - DSO_free(brotli_encode_dso); - brotli_encode_dso = NULL; - DSO_free(brotli_decode_dso); - brotli_decode_dso = NULL; - p_encode_init = NULL; - p_encode_stream = NULL; - p_encode_has_more = NULL; - p_encode_end = NULL; - p_encode_oneshot = NULL; - p_decode_init = NULL; - p_decode_stream = NULL; - p_decode_has_more = NULL; - p_decode_end = NULL; - p_decode_error = NULL; - p_decode_error_string = NULL; - p_decode_is_finished = NULL; - p_decode_oneshot = NULL; -#endif -} - -#ifndef OPENSSL_NO_BROTLI - -/* Brotli-based compression/decompression filter BIO */ - -typedef struct { - struct { /* input structure */ - size_t avail_in; - unsigned char *next_in; - size_t avail_out; - unsigned char *next_out; - unsigned char *buf; - size_t bufsize; - BrotliDecoderState *state; - } decode; - struct { /* output structure */ - size_t avail_in; - unsigned char *next_in; - size_t avail_out; - unsigned char *next_out; - unsigned char *buf; - size_t bufsize; - BrotliEncoderState *state; - int mode; /* Encoder mode to use */ - int done; - unsigned char *ptr; - size_t count; - } encode; -} BIO_BROTLI_CTX; - -# define BROTLI_DEFAULT_BUFSIZE 1024 - -static int bio_brotli_new(BIO *bi); -static int bio_brotli_free(BIO *bi); -static int bio_brotli_read(BIO *b, char *out, int outl); -static int bio_brotli_write(BIO *b, const char *in, int inl); -static long bio_brotli_ctrl(BIO *b, int cmd, long num, void *ptr); -static long bio_brotli_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp); - -static const BIO_METHOD bio_meth_brotli = { - BIO_TYPE_COMP, - "brotli", - /* TODO: Convert to new style write function */ - bwrite_conv, - bio_brotli_write, - /* TODO: Convert to new style read function */ - bread_conv, - bio_brotli_read, - NULL, /* bio_brotli_puts, */ - NULL, /* bio_brotli_gets, */ - bio_brotli_ctrl, - bio_brotli_new, - bio_brotli_free, - bio_brotli_callback_ctrl -}; -#endif - -const BIO_METHOD *BIO_f_brotli(void) -{ -#ifndef OPENSSL_NO_BROTLI - if (RUN_ONCE(&brotli_once, ossl_comp_brotli_init)) - return &bio_meth_brotli; -#endif - return NULL; -} - -#ifndef OPENSSL_NO_BROTLI - -static int bio_brotli_new(BIO *bi) -{ - BIO_BROTLI_CTX *ctx; - -# ifdef BROTLI_SHARED - if (!RUN_ONCE(&brotli_once, ossl_comp_brotli_init)) { - ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_NOT_SUPPORTED); - return 0; - } -# endif - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) { - ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); - return 0; - } - ctx->decode.bufsize = BROTLI_DEFAULT_BUFSIZE; - ctx->decode.state = BrotliDecoderCreateInstance(brotli_alloc, brotli_free, NULL); - if (ctx->decode.state == NULL) - goto err; - ctx->encode.bufsize = BROTLI_DEFAULT_BUFSIZE; - ctx->encode.state = BrotliEncoderCreateInstance(brotli_alloc, brotli_free, NULL); - if (ctx->encode.state == NULL) - goto err; - ctx->encode.mode = BROTLI_DEFAULT_MODE; - BIO_set_init(bi, 1); - BIO_set_data(bi, ctx); - - return 1; - - err: - ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); - BrotliDecoderDestroyInstance(ctx->decode.state); - BrotliEncoderDestroyInstance(ctx->encode.state); - OPENSSL_free(ctx); - return 0; -} - -static int bio_brotli_free(BIO *bi) -{ - BIO_BROTLI_CTX *ctx; - - if (bi == NULL) - return 0; - - ctx = BIO_get_data(bi); - if (ctx != NULL) { - BrotliDecoderDestroyInstance(ctx->decode.state); - OPENSSL_free(ctx->decode.buf); - BrotliEncoderDestroyInstance(ctx->encode.state); - OPENSSL_free(ctx->encode.buf); - OPENSSL_free(ctx); - } - BIO_set_data(bi, NULL); - BIO_set_init(bi, 0); - - return 1; -} - -static int bio_brotli_read(BIO *b, char *out, int outl) -{ - BIO_BROTLI_CTX *ctx; - BrotliDecoderResult bret; - int ret; - BIO *next = BIO_next(b); - - if (out == NULL || outl <= 0) { - ERR_raise(ERR_LIB_COMP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } -#if INT_MAX > SIZE_MAX - if ((unsigned int)outl > SIZE_MAX) { - ERR_raise(ERR_LIB_COMP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } -#endif - - ctx = BIO_get_data(b); - BIO_clear_retry_flags(b); - if (ctx->decode.buf == NULL) { - ctx->decode.buf = OPENSSL_malloc(ctx->decode.bufsize); - if (ctx->decode.buf == NULL) { - ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); - return 0; - } - ctx->decode.next_in = ctx->decode.buf; - ctx->decode.avail_in = 0; - } - - /* Copy output data directly to supplied buffer */ - ctx->decode.next_out = (unsigned char *)out; - ctx->decode.avail_out = (size_t)outl; - for (;;) { - /* Decompress while data available */ - while (ctx->decode.avail_in > 0 || BrotliDecoderHasMoreOutput(ctx->decode.state)) { - bret = BrotliDecoderDecompressStream(ctx->decode.state, &ctx->decode.avail_in, (const uint8_t**)&ctx->decode.next_in, - &ctx->decode.avail_out, &ctx->decode.next_out, NULL); - if (bret == BROTLI_DECODER_RESULT_ERROR) { - ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_DECODE_ERROR); - ERR_add_error_data(1, BrotliDecoderErrorString(BrotliDecoderGetErrorCode(ctx->decode.state))); - return 0; - } - /* If EOF or we've read everything then return */ - if (BrotliDecoderIsFinished(ctx->decode.state) || ctx->decode.avail_out == 0) - return (int)(outl - ctx->decode.avail_out); - } - - /* If EOF */ - if (BrotliDecoderIsFinished(ctx->decode.state)) - return 0; - - /* - * No data in input buffer try to read some in, if an error then - * return the total data read. - */ - ret = BIO_read(next, ctx->decode.buf, ctx->decode.bufsize); - if (ret <= 0) { - /* Total data read */ - int tot = outl - ctx->decode.avail_out; - - BIO_copy_next_retry(b); - if (ret < 0) - return (tot > 0) ? tot : ret; - return tot; - } - ctx->decode.avail_in = ret; - ctx->decode.next_in = ctx->decode.buf; - } -} - -static int bio_brotli_write(BIO *b, const char *in, int inl) -{ - BIO_BROTLI_CTX *ctx; - BROTLI_BOOL brret; - int ret; - BIO *next = BIO_next(b); - - if (in == NULL || inl <= 0) { - ERR_raise(ERR_LIB_COMP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } -#if INT_MAX > SIZE_MAX - if ((unsigned int)inl > SIZE_MAX) { - ERR_raise(ERR_LIB_COMP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } -#endif - - ctx = BIO_get_data(b); - if (ctx->encode.done) - return 0; - - BIO_clear_retry_flags(b); - if (ctx->encode.buf == NULL) { - ctx->encode.buf = OPENSSL_malloc(ctx->encode.bufsize); - if (ctx->encode.buf == NULL) { - ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); - return 0; - } - ctx->encode.ptr = ctx->encode.buf; - ctx->encode.count = 0; - ctx->encode.next_out = ctx->encode.buf; - ctx->encode.avail_out = ctx->encode.bufsize; - } - /* Obtain input data directly from supplied buffer */ - ctx->encode.next_in = (unsigned char *)in; - ctx->encode.avail_in = (size_t)inl; - for (;;) { - /* If data in output buffer write it first */ - while (ctx->encode.count > 0) { - ret = BIO_write(next, ctx->encode.ptr, ctx->encode.count); - if (ret <= 0) { - /* Total data written */ - int tot = inl - ctx->encode.avail_in; - - BIO_copy_next_retry(b); - if (ret < 0) - return (tot > 0) ? tot : ret; - return tot; - } - ctx->encode.ptr += ret; - ctx->encode.count -= ret; - } - - /* Have we consumed all supplied data? */ - if (ctx->encode.avail_in == 0 && !BrotliEncoderHasMoreOutput(ctx->encode.state)) - return inl; - - /* Compress some more */ - - /* Reset buffer */ - ctx->encode.ptr = ctx->encode.buf; - ctx->encode.next_out = ctx->encode.buf; - ctx->encode.avail_out = ctx->encode.bufsize; - /* Compress some more */ - brret = BrotliEncoderCompressStream(ctx->encode.state, BROTLI_OPERATION_FLUSH, &ctx->encode.avail_in, (const uint8_t**)&ctx->encode.next_in, - &ctx->encode.avail_out, &ctx->encode.next_out, NULL); - if (brret != BROTLI_TRUE) { - ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_ENCODE_ERROR); - ERR_add_error_data(1, "brotli encoder error"); - return 0; - } - ctx->encode.count = ctx->encode.bufsize - ctx->encode.avail_out; - } -} - -static int bio_brotli_flush(BIO *b) -{ - BIO_BROTLI_CTX *ctx; - BROTLI_BOOL brret; - int ret; - BIO *next = BIO_next(b); - - ctx = BIO_get_data(b); - - /* If no data written or already flush show success */ - if (ctx->encode.buf == NULL || (ctx->encode.done && ctx->encode.count == 0)) - return 1; - - BIO_clear_retry_flags(b); - /* No more input data */ - ctx->encode.next_in = NULL; - ctx->encode.avail_in = 0; - for (;;) { - /* If data in output buffer write it first */ - while (ctx->encode.count > 0) { - ret = BIO_write(next, ctx->encode.ptr, ctx->encode.count); - if (ret <= 0) { - BIO_copy_next_retry(b); - return ret; - } - ctx->encode.ptr += ret; - ctx->encode.count -= ret; - } - if (ctx->encode.done) - return 1; - - /* Compress some more */ - - /* Reset buffer */ - ctx->encode.ptr = ctx->encode.buf; - ctx->encode.next_out = ctx->encode.buf; - ctx->encode.avail_out = ctx->encode.bufsize; - /* Compress some more */ - brret = BrotliEncoderCompressStream(ctx->encode.state, BROTLI_OPERATION_FINISH, &ctx->encode.avail_in, - (const uint8_t**)&ctx->encode.next_in, &ctx->encode.avail_out, &ctx->encode.next_out, NULL); - if (brret != BROTLI_TRUE) { - ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_DECODE_ERROR); - ERR_add_error_data(1, "brotli encoder error"); - return 0; - } - if (!BrotliEncoderHasMoreOutput(ctx->encode.state) && ctx->encode.avail_in == 0) - ctx->encode.done = 1; - ctx->encode.count = ctx->encode.bufsize - ctx->encode.avail_out; - } -} - -static long bio_brotli_ctrl(BIO *b, int cmd, long num, void *ptr) -{ - BIO_BROTLI_CTX *ctx; - unsigned char *tmp; - int ret = 0, *ip; - size_t ibs, obs; - BIO *next = BIO_next(b); - - if (next == NULL) - return 0; - ctx = BIO_get_data(b); - switch (cmd) { - - case BIO_CTRL_RESET: - ctx->encode.count = 0; - ctx->encode.done = 0; - ret = 1; - break; - - case BIO_CTRL_FLUSH: - ret = bio_brotli_flush(b); - if (ret > 0) { - ret = BIO_flush(next); - BIO_copy_next_retry(b); - } - break; - - case BIO_C_SET_BUFF_SIZE: - ibs = ctx->decode.bufsize; - obs = ctx->encode.bufsize; - if (ptr != NULL) { - ip = ptr; - if (*ip == 0) - ibs = (size_t)num; - else - obs = (size_t)num; - } else { - ibs = (size_t)num; - obs = ibs; - } - - if (ibs > 0 && ibs != ctx->decode.bufsize) { - /* Do not free/alloc, only reallocate */ - if (ctx->decode.buf != NULL) { - tmp = OPENSSL_realloc(ctx->decode.buf, ibs); - if (tmp == NULL) - return 0; - ctx->decode.buf = tmp; - } - ctx->decode.bufsize = ibs; - } - - if (obs > 0 && obs != ctx->encode.bufsize) { - /* Do not free/alloc, only reallocate */ - if (ctx->encode.buf != NULL) { - tmp = OPENSSL_realloc(ctx->encode.buf, obs); - if (tmp == NULL) - return 0; - ctx->encode.buf = tmp; - } - ctx->encode.bufsize = obs; - } - ret = 1; - break; - - case BIO_C_DO_STATE_MACHINE: - BIO_clear_retry_flags(b); - ret = BIO_ctrl(next, cmd, num, ptr); - BIO_copy_next_retry(b); - break; - - case BIO_CTRL_WPENDING: - if (BrotliEncoderHasMoreOutput(ctx->encode.state)) - ret = 1; - else - ret = BIO_ctrl(next, cmd, num, ptr); - break; - - case BIO_CTRL_PENDING: - if (!BrotliDecoderIsFinished(ctx->decode.state)) - ret = 1; - else - ret = BIO_ctrl(next, cmd, num, ptr); - break; - - default: - ret = BIO_ctrl(next, cmd, num, ptr); - break; - - } - - return ret; -} - -static long bio_brotli_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp) -{ - BIO *next = BIO_next(b); - if (next == NULL) - return 0; - return BIO_callback_ctrl(next, cmd, fp); -} - -#endif diff --git a/openssl/src/crypto/comp/c_zlib.c b/openssl/src/crypto/comp/c_zlib.c index 0fbab8f01..599662d22 100644 --- a/openssl/src/crypto/comp/c_zlib.c +++ b/openssl/src/crypto/comp/c_zlib.c @@ -1,5 +1,5 @@ /* - * Copyright 1998-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1998-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,7 +20,16 @@ COMP_METHOD *COMP_zlib(void); -#ifdef OPENSSL_NO_ZLIB +static COMP_METHOD zlib_method_nozlib = { + NID_undef, + "(undef)", + NULL, + NULL, + NULL, + NULL, +}; + +#ifndef ZLIB # undef ZLIB_SHARED #else @@ -28,12 +37,12 @@ COMP_METHOD *COMP_zlib(void); static int zlib_stateful_init(COMP_CTX *ctx); static void zlib_stateful_finish(COMP_CTX *ctx); -static ossl_ssize_t zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen); -static ossl_ssize_t zlib_stateful_expand_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen); +static int zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, + unsigned int olen, unsigned char *in, + unsigned int ilen); +static int zlib_stateful_expand_block(COMP_CTX *ctx, unsigned char *out, + unsigned int olen, unsigned char *in, + unsigned int ilen); /* memory allocations functions for zlib initialisation */ static void *zlib_zalloc(void *opaque, unsigned int no, unsigned int size) @@ -74,10 +83,8 @@ static COMP_METHOD zlib_stateful_method = { # include "internal/dso.h" /* Function pointers */ -typedef int (*compress_ft) (Bytef *dest, uLongf *destLen, +typedef int (*compress_ft) (Bytef *dest, uLongf * destLen, const Bytef *source, uLong sourceLen); -typedef int (*uncompress_ft) (Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen); typedef int (*inflateEnd_ft) (z_streamp strm); typedef int (*inflate_ft) (z_streamp strm, int flush); typedef int (*inflateInit__ft) (z_streamp strm, @@ -88,7 +95,6 @@ typedef int (*deflateInit__ft) (z_streamp strm, int level, const char *version, int stream_size); typedef const char *(*zError__ft) (int err); static compress_ft p_compress = NULL; -static uncompress_ft p_uncompress = NULL; static inflateEnd_ft p_inflateEnd = NULL; static inflate_ft p_inflate = NULL; static inflateInit__ft p_inflateInit_ = NULL; @@ -100,7 +106,6 @@ static zError__ft p_zError = NULL; static DSO *zlib_dso = NULL; # define compress p_compress -# define uncompress p_uncompress # define inflateEnd p_inflateEnd # define inflate p_inflate # define inflateInit_ p_inflateInit_ @@ -157,9 +162,9 @@ static void zlib_stateful_finish(COMP_CTX *ctx) OPENSSL_free(state); } -static ossl_ssize_t zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) +static int zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, + unsigned int olen, unsigned char *in, + unsigned int ilen) { int err = Z_OK; struct zlib_state *state = ctx->data; @@ -175,14 +180,12 @@ static ossl_ssize_t zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *o err = deflate(&state->ostream, Z_SYNC_FLUSH); if (err != Z_OK) return -1; - if (state->ostream.avail_out > olen) - return -1; - return (ossl_ssize_t)(olen - state->ostream.avail_out); + return olen - state->ostream.avail_out; } -static ossl_ssize_t zlib_stateful_expand_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) +static int zlib_stateful_expand_block(COMP_CTX *ctx, unsigned char *out, + unsigned int olen, unsigned char *in, + unsigned int ilen) { int err = Z_OK; struct zlib_state *state = ctx->data; @@ -198,75 +201,9 @@ static ossl_ssize_t zlib_stateful_expand_block(COMP_CTX *ctx, unsigned char *out err = inflate(&state->istream, Z_SYNC_FLUSH); if (err != Z_OK) return -1; - if (state->istream.avail_out > olen) - return -1; - return (ossl_ssize_t)(olen - state->istream.avail_out); -} - -/* ONESHOT COMPRESSION/DECOMPRESSION */ - -static int zlib_oneshot_init(COMP_CTX *ctx) -{ - return 1; -} - -static void zlib_oneshot_finish(COMP_CTX *ctx) -{ -} - -static ossl_ssize_t zlib_oneshot_compress_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - uLongf out_size; - - if (ilen == 0) - return 0; - - /* zlib's uLongf defined as unsigned long FAR */ - if (olen > ULONG_MAX) - return -1; - out_size = (uLongf)olen; - - if (compress(out, &out_size, in, ilen) != Z_OK) - return -1; - - if (out_size > OSSL_SSIZE_MAX) - return -1; - return (ossl_ssize_t)out_size; + return olen - state->istream.avail_out; } -static ossl_ssize_t zlib_oneshot_expand_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - uLongf out_size; - - if (ilen == 0) - return 0; - - /* zlib's uLongf defined as unsigned long FAR */ - if (olen > ULONG_MAX) - return -1; - out_size = (uLongf)olen; - - if (uncompress(out, &out_size, in, ilen) != Z_OK) - return -1; - - if (out_size > OSSL_SSIZE_MAX) - return -1; - return (ossl_ssize_t)out_size; -} - -static COMP_METHOD zlib_oneshot_method = { - NID_zlib_compression, - LN_zlib_compression, - zlib_oneshot_init, - zlib_oneshot_finish, - zlib_oneshot_compress_block, - zlib_oneshot_expand_block -}; - static CRYPTO_ONCE zlib_once = CRYPTO_ONCE_STATIC_INIT; DEFINE_RUN_ONCE_STATIC(ossl_comp_zlib_init) { @@ -275,8 +212,6 @@ DEFINE_RUN_ONCE_STATIC(ossl_comp_zlib_init) # ifndef LIBZ # if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32) # define LIBZ "ZLIB1" -# elif defined(OPENSSL_SYS_VMS) -# define LIBZ "LIBZ" # else # define LIBZ "z" # endif @@ -285,7 +220,6 @@ DEFINE_RUN_ONCE_STATIC(ossl_comp_zlib_init) zlib_dso = DSO_load(NULL, LIBZ, NULL, 0); if (zlib_dso != NULL) { p_compress = (compress_ft) DSO_bind_func(zlib_dso, "compress"); - p_uncompress = (compress_ft) DSO_bind_func(zlib_dso, "uncompress"); p_inflateEnd = (inflateEnd_ft) DSO_bind_func(zlib_dso, "inflateEnd"); p_inflate = (inflate_ft) DSO_bind_func(zlib_dso, "inflate"); p_inflateInit_ = (inflateInit__ft) DSO_bind_func(zlib_dso, "inflateInit_"); @@ -294,7 +228,7 @@ DEFINE_RUN_ONCE_STATIC(ossl_comp_zlib_init) p_deflateInit_ = (deflateInit__ft) DSO_bind_func(zlib_dso, "deflateInit_"); p_zError = (zError__ft) DSO_bind_func(zlib_dso, "zError"); - if (p_compress == NULL || p_uncompress == NULL || p_inflateEnd == NULL + if (p_compress == NULL || p_inflateEnd == NULL || p_inflate == NULL || p_inflateInit_ == NULL || p_deflateEnd == NULL || p_deflate == NULL || p_deflateInit_ == NULL || p_zError == NULL) { @@ -309,9 +243,9 @@ DEFINE_RUN_ONCE_STATIC(ossl_comp_zlib_init) COMP_METHOD *COMP_zlib(void) { - COMP_METHOD *meth = NULL; + COMP_METHOD *meth = &zlib_method_nozlib; -#ifndef OPENSSL_NO_ZLIB +#ifdef ZLIB if (RUN_ONCE(&zlib_once, ossl_comp_zlib_init)) meth = &zlib_stateful_method; #endif @@ -319,18 +253,6 @@ COMP_METHOD *COMP_zlib(void) return meth; } -COMP_METHOD *COMP_zlib_oneshot(void) -{ - COMP_METHOD *meth = NULL; - -#ifndef OPENSSL_NO_ZLIB - if (RUN_ONCE(&zlib_once, ossl_comp_zlib_init)) - meth = &zlib_oneshot_method; -#endif - - return meth; -} - /* Also called from OPENSSL_cleanup() */ void ossl_comp_zlib_cleanup(void) { @@ -340,7 +262,7 @@ void ossl_comp_zlib_cleanup(void) #endif } -#ifndef OPENSSL_NO_ZLIB +#ifdef ZLIB /* Zlib based compression/decompression filter BIO */ @@ -380,18 +302,12 @@ static const BIO_METHOD bio_meth_zlib = { bio_zlib_free, bio_zlib_callback_ctrl }; -#endif const BIO_METHOD *BIO_f_zlib(void) { -#ifndef OPENSSL_NO_ZLIB - if (RUN_ONCE(&zlib_once, ossl_comp_zlib_init)) - return &bio_meth_zlib; -#endif - return NULL; + return &bio_meth_zlib; } -#ifndef OPENSSL_NO_ZLIB static int bio_zlib_new(BIO *bi) { BIO_ZLIB_CTX *ctx; @@ -403,8 +319,10 @@ static int bio_zlib_new(BIO *bi) } # endif ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); return 0; + } ctx->ibufsize = ZLIB_DEFAULT_BUFSIZE; ctx->obufsize = ZLIB_DEFAULT_BUFSIZE; ctx->zin.zalloc = Z_NULL; @@ -456,8 +374,10 @@ static int bio_zlib_read(BIO *b, char *out, int outl) BIO_clear_retry_flags(b); if (!ctx->ibuf) { ctx->ibuf = OPENSSL_malloc(ctx->ibufsize); - if (ctx->ibuf == NULL) + if (ctx->ibuf == NULL) { + ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); return 0; + } if ((ret = inflateInit(zin)) != Z_OK) { ERR_raise_data(ERR_LIB_COMP, COMP_R_ZLIB_INFLATE_ERROR, "zlib error: %s", zError(ret)); @@ -519,8 +439,10 @@ static int bio_zlib_write(BIO *b, const char *in, int inl) if (!ctx->obuf) { ctx->obuf = OPENSSL_malloc(ctx->obufsize); /* Need error here */ - if (ctx->obuf == NULL) + if (ctx->obuf == NULL) { + ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); return 0; + } ctx->optr = ctx->obuf; ctx->ocount = 0; if ((ret = deflateInit(zout, ctx->comp_level)) != Z_OK) { @@ -640,10 +562,8 @@ static long bio_zlib_ctrl(BIO *b, int cmd, long num, void *ptr) case BIO_CTRL_FLUSH: ret = bio_zlib_flush(b); - if (ret > 0) { + if (ret > 0) ret = BIO_flush(next); - BIO_copy_next_retry(b); - } break; case BIO_C_SET_BUFF_SIZE: diff --git a/openssl/src/crypto/comp/c_zstd.c b/openssl/src/crypto/comp/c_zstd.c deleted file mode 100644 index b4667649f..000000000 --- a/openssl/src/crypto/comp/c_zstd.c +++ /dev/null @@ -1,845 +0,0 @@ -/* - * Copyright 1998-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - * - * Uses zstd compression library from https://github.com/facebook/zstd - * Requires version 1.4.x (latest as of this writing is 1.4.5) - * Using custom free functions require static linking, so that is disabled when - * using the shared library. - */ - -#include -#include -#include -#include -#include "internal/comp.h" -#include -#include "crypto/cryptlib.h" -#include "internal/bio.h" -#include "internal/thread_once.h" -#include "comp_local.h" - -COMP_METHOD *COMP_zstd(void); - -#ifdef OPENSSL_NO_ZSTD -# undef ZSTD_SHARED -#else - -# ifndef ZSTD_SHARED -# define ZSTD_STATIC_LINKING_ONLY -# endif -# include - -/* Note: There is also a linux zstd.h file in the kernel source */ -# ifndef ZSTD_H_235446 -# error Wrong (i.e. linux) zstd.h included. -# endif - -# if ZSTD_VERSION_MAJOR != 1 && ZSTD_VERSION_MINOR < 4 -# error Expecting version 1.4 or greater of ZSTD -# endif - -# ifndef ZSTD_SHARED -/* memory allocations functions for zstd initialisation */ -static void *zstd_alloc(void *opaque, size_t size) -{ - return OPENSSL_zalloc(size); -} - -static void zstd_free(void *opaque, void *address) -{ - OPENSSL_free(address); -} - -static ZSTD_customMem zstd_mem_funcs = { - zstd_alloc, - zstd_free, - NULL -}; -# endif - -/* - * When OpenSSL is built on Windows, we do not want to require that - * the LIBZSTD.DLL be available in order for the OpenSSL DLLs to - * work. Therefore, all ZSTD routines are loaded at run time - * and we do not link to a .LIB file when ZSTD_SHARED is set. - */ -# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32) -# include -# endif - -# ifdef ZSTD_SHARED -# include "internal/dso.h" - -/* Function pointers */ -typedef ZSTD_CStream* (*createCStream_ft)(void); -typedef size_t (*initCStream_ft)(ZSTD_CStream*, int); -typedef size_t (*freeCStream_ft)(ZSTD_CStream*); -typedef size_t (*compressStream2_ft)(ZSTD_CCtx*, ZSTD_outBuffer*, ZSTD_inBuffer*, ZSTD_EndDirective); -typedef size_t (*flushStream_ft)(ZSTD_CStream*, ZSTD_outBuffer*); -typedef size_t (*endStream_ft)(ZSTD_CStream*, ZSTD_outBuffer*); -typedef size_t (*compress_ft)(void*, size_t, const void*, size_t, int); -typedef ZSTD_DStream* (*createDStream_ft)(void); -typedef size_t (*initDStream_ft)(ZSTD_DStream*); -typedef size_t (*freeDStream_ft)(ZSTD_DStream*); -typedef size_t (*decompressStream_ft)(ZSTD_DStream*, ZSTD_outBuffer*, ZSTD_inBuffer*); -typedef size_t (*decompress_ft)(void*, size_t, const void*, size_t); -typedef unsigned (*isError_ft)(size_t); -typedef const char* (*getErrorName_ft)(size_t); -typedef size_t (*DStreamInSize_ft)(void); -typedef size_t (*CStreamInSize_ft)(void); - -static createCStream_ft p_createCStream = NULL; -static initCStream_ft p_initCStream = NULL; -static freeCStream_ft p_freeCStream = NULL; -static compressStream2_ft p_compressStream2 = NULL; -static flushStream_ft p_flushStream = NULL; -static endStream_ft p_endStream = NULL; -static compress_ft p_compress = NULL; -static createDStream_ft p_createDStream = NULL; -static initDStream_ft p_initDStream = NULL; -static freeDStream_ft p_freeDStream = NULL; -static decompressStream_ft p_decompressStream = NULL; -static decompress_ft p_decompress = NULL; -static isError_ft p_isError = NULL; -static getErrorName_ft p_getErrorName = NULL; -static DStreamInSize_ft p_DStreamInSize = NULL; -static CStreamInSize_ft p_CStreamInSize = NULL; - -static DSO *zstd_dso = NULL; - -# define ZSTD_createCStream p_createCStream -# define ZSTD_initCStream p_initCStream -# define ZSTD_freeCStream p_freeCStream -# define ZSTD_compressStream2 p_compressStream2 -# define ZSTD_flushStream p_flushStream -# define ZSTD_endStream p_endStream -# define ZSTD_compress p_compress -# define ZSTD_createDStream p_createDStream -# define ZSTD_initDStream p_initDStream -# define ZSTD_freeDStream p_freeDStream -# define ZSTD_decompressStream p_decompressStream -# define ZSTD_decompress p_decompress -# define ZSTD_isError p_isError -# define ZSTD_getErrorName p_getErrorName -# define ZSTD_DStreamInSize p_DStreamInSize -# define ZSTD_CStreamInSize p_CStreamInSize - -# endif /* ifdef ZSTD_SHARED */ - -struct zstd_state { - ZSTD_CStream *compressor; - ZSTD_DStream *decompressor; -}; - -static int zstd_stateful_init(COMP_CTX *ctx) -{ - struct zstd_state *state = OPENSSL_zalloc(sizeof(*state)); - - if (state == NULL) - return 0; - -# ifdef ZSTD_SHARED - state->compressor = ZSTD_createCStream(); -# else - state->compressor = ZSTD_createCStream_advanced(zstd_mem_funcs); -# endif - if (state->compressor == NULL) - goto err; - ZSTD_initCStream(state->compressor, ZSTD_CLEVEL_DEFAULT); - -# ifdef ZSTD_SHARED - state->decompressor = ZSTD_createDStream(); -# else - state->decompressor = ZSTD_createDStream_advanced(zstd_mem_funcs); -# endif - if (state->decompressor == NULL) - goto err; - ZSTD_initDStream(state->decompressor); - - ctx->data = state; - return 1; - err: - ZSTD_freeCStream(state->compressor); - ZSTD_freeDStream(state->decompressor); - OPENSSL_free(state); - return 0; -} - -static void zstd_stateful_finish(COMP_CTX *ctx) -{ - struct zstd_state *state = ctx->data; - - if (state != NULL) { - ZSTD_freeCStream(state->compressor); - ZSTD_freeDStream(state->decompressor); - OPENSSL_free(state); - ctx->data = NULL; - } -} - -static ossl_ssize_t zstd_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - ZSTD_inBuffer inbuf; - ZSTD_outBuffer outbuf; - size_t ret; - ossl_ssize_t fret; - struct zstd_state *state = ctx->data; - - inbuf.src = in; - inbuf.size = ilen; - inbuf.pos = 0; - outbuf.dst = out; - outbuf.size = olen; - outbuf.pos = 0; - - if (state == NULL) - return -1; - - /* If input length is zero, end the stream/frame ? */ - if (ilen == 0) { - ret = ZSTD_endStream(state->compressor, &outbuf); - if (ZSTD_isError(ret)) - return -1; - goto end; - } - - /* - * The finish API does not provide a final output buffer, - * so each compress operation has to be ended, if all - * the input data can't be accepted, or there is more output, - * this has to be considered an error, since there is no more - * output buffer space. - */ - do { - ret = ZSTD_compressStream2(state->compressor, &outbuf, &inbuf, ZSTD_e_continue); - if (ZSTD_isError(ret)) - return -1; - /* do I need to check for ret == 0 ? */ - } while (inbuf.pos < inbuf.size); - - /* Did not consume all the data */ - if (inbuf.pos < inbuf.size) - return -1; - - ret = ZSTD_flushStream(state->compressor, &outbuf); - if (ZSTD_isError(ret)) - return -1; - - end: - if (outbuf.pos > OSSL_SSIZE_MAX) - return -1; - fret = (ossl_ssize_t)outbuf.pos; - if (fret < 0) - return -1; - return fret; -} - -static ossl_ssize_t zstd_stateful_expand_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - ZSTD_inBuffer inbuf; - ZSTD_outBuffer outbuf; - size_t ret; - ossl_ssize_t fret; - struct zstd_state *state = ctx->data; - - inbuf.src = in; - inbuf.size = ilen; - inbuf.pos = 0; - outbuf.dst = out; - outbuf.size = olen; - outbuf.pos = 0; - - if (state == NULL) - return -1; - - if (ilen == 0) - return 0; - - do { - ret = ZSTD_decompressStream(state->decompressor, &outbuf, &inbuf); - if (ZSTD_isError(ret)) - return -1; - /* If we completed a frame, and there's more data, try again */ - } while (ret == 0 && inbuf.pos < inbuf.size); - - /* Did not consume all the data */ - if (inbuf.pos < inbuf.size) - return -1; - - if (outbuf.pos > OSSL_SSIZE_MAX) - return -1; - fret = (ossl_ssize_t)outbuf.pos; - if (fret < 0) - return -1; - return fret; -} - - -static COMP_METHOD zstd_stateful_method = { - NID_zstd, - LN_zstd, - zstd_stateful_init, - zstd_stateful_finish, - zstd_stateful_compress_block, - zstd_stateful_expand_block -}; - -static int zstd_oneshot_init(COMP_CTX *ctx) -{ - return 1; -} - -static void zstd_oneshot_finish(COMP_CTX *ctx) -{ -} - -static ossl_ssize_t zstd_oneshot_compress_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - size_t out_size; - ossl_ssize_t ret; - - if (ilen == 0) - return 0; - - /* Note: uses STDLIB memory allocators */ - out_size = ZSTD_compress(out, olen, in, ilen, ZSTD_CLEVEL_DEFAULT); - if (ZSTD_isError(out_size)) - return -1; - - if (out_size > OSSL_SSIZE_MAX) - return -1; - ret = (ossl_ssize_t)out_size; - if (ret < 0) - return -1; - return ret; -} - -static ossl_ssize_t zstd_oneshot_expand_block(COMP_CTX *ctx, unsigned char *out, - size_t olen, unsigned char *in, - size_t ilen) -{ - size_t out_size; - ossl_ssize_t ret; - - if (ilen == 0) - return 0; - - /* Note: uses STDLIB memory allocators */ - out_size = ZSTD_decompress(out, olen, in, ilen); - if (ZSTD_isError(out_size)) - return -1; - - if (out_size > OSSL_SSIZE_MAX) - return -1; - ret = (ossl_ssize_t)out_size; - if (ret < 0) - return -1; - return ret; -} - -static COMP_METHOD zstd_oneshot_method = { - NID_zstd, - LN_zstd, - zstd_oneshot_init, - zstd_oneshot_finish, - zstd_oneshot_compress_block, - zstd_oneshot_expand_block -}; - -static CRYPTO_ONCE zstd_once = CRYPTO_ONCE_STATIC_INIT; -DEFINE_RUN_ONCE_STATIC(ossl_comp_zstd_init) -{ -# ifdef ZSTD_SHARED -# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32) -# define LIBZSTD "LIBZSTD" -# else -# define LIBZSTD "zstd" -# endif - - zstd_dso = DSO_load(NULL, LIBZSTD, NULL, 0); - if (zstd_dso != NULL) { - p_createCStream = (createCStream_ft)DSO_bind_func(zstd_dso, "ZSTD_createCStream"); - p_initCStream = (initCStream_ft)DSO_bind_func(zstd_dso, "ZSTD_initCStream"); - p_freeCStream = (freeCStream_ft)DSO_bind_func(zstd_dso, "ZSTD_freeCStream"); - p_compressStream2 = (compressStream2_ft)DSO_bind_func(zstd_dso, "ZSTD_compressStream2"); - p_flushStream = (flushStream_ft)DSO_bind_func(zstd_dso, "ZSTD_flushStream"); - p_endStream = (endStream_ft)DSO_bind_func(zstd_dso, "ZSTD_endStream"); - p_compress = (compress_ft)DSO_bind_func(zstd_dso, "ZSTD_compress"); - p_createDStream = (createDStream_ft)DSO_bind_func(zstd_dso, "ZSTD_createDStream"); - p_initDStream = (initDStream_ft)DSO_bind_func(zstd_dso, "ZSTD_initDStream"); - p_freeDStream = (freeDStream_ft)DSO_bind_func(zstd_dso, "ZSTD_freeDStream"); - p_decompressStream = (decompressStream_ft)DSO_bind_func(zstd_dso, "ZSTD_decompressStream"); - p_decompress = (decompress_ft)DSO_bind_func(zstd_dso, "ZSTD_decompress"); - p_isError = (isError_ft)DSO_bind_func(zstd_dso, "ZSTD_isError"); - p_getErrorName = (getErrorName_ft)DSO_bind_func(zstd_dso, "ZSTD_getErrorName"); - p_DStreamInSize = (DStreamInSize_ft)DSO_bind_func(zstd_dso, "ZSTD_DStreamInSize"); - p_CStreamInSize = (CStreamInSize_ft)DSO_bind_func(zstd_dso, "ZSTD_CStreamInSize"); - } - - if (p_createCStream == NULL || p_initCStream == NULL || p_freeCStream == NULL - || p_compressStream2 == NULL || p_flushStream == NULL || p_endStream == NULL - || p_compress == NULL || p_createDStream == NULL || p_initDStream == NULL - || p_freeDStream == NULL || p_decompressStream == NULL || p_decompress == NULL - || p_isError == NULL || p_getErrorName == NULL || p_DStreamInSize == NULL - || p_CStreamInSize == NULL) { - ossl_comp_zstd_cleanup(); - return 0; - } -# endif - return 1; -} -#endif /* ifndef ZSTD / else */ - -COMP_METHOD *COMP_zstd(void) -{ - COMP_METHOD *meth = NULL; - -#ifndef OPENSSL_NO_ZSTD - if (RUN_ONCE(&zstd_once, ossl_comp_zstd_init)) - meth = &zstd_stateful_method; -#endif - return meth; -} - -COMP_METHOD *COMP_zstd_oneshot(void) -{ - COMP_METHOD *meth = NULL; - -#ifndef OPENSSL_NO_ZSTD - if (RUN_ONCE(&zstd_once, ossl_comp_zstd_init)) - meth = &zstd_oneshot_method; -#endif - return meth; -} - -/* Also called from OPENSSL_cleanup() */ -void ossl_comp_zstd_cleanup(void) -{ -#ifdef ZSTD_SHARED - DSO_free(zstd_dso); - zstd_dso = NULL; - p_createCStream = NULL; - p_initCStream = NULL; - p_freeCStream = NULL; - p_compressStream2 = NULL; - p_flushStream = NULL; - p_endStream = NULL; - p_compress = NULL; - p_createDStream = NULL; - p_initDStream = NULL; - p_freeDStream = NULL; - p_decompressStream = NULL; - p_decompress = NULL; - p_isError = NULL; - p_getErrorName = NULL; - p_DStreamInSize = NULL; - p_CStreamInSize = NULL; -#endif -} - -#ifndef OPENSSL_NO_ZSTD - -/* Zstd-based compression/decompression filter BIO */ - -typedef struct { - struct { /* input structure */ - ZSTD_DStream *state; - ZSTD_inBuffer inbuf; /* has const src */ - size_t bufsize; - void* buffer; - } decompress; - struct { /* output structure */ - ZSTD_CStream *state; - ZSTD_outBuffer outbuf; - size_t bufsize; - size_t write_pos; - } compress; -} BIO_ZSTD_CTX; - -# define ZSTD_DEFAULT_BUFSIZE 1024 - -static int bio_zstd_new(BIO *bi); -static int bio_zstd_free(BIO *bi); -static int bio_zstd_read(BIO *b, char *out, int outl); -static int bio_zstd_write(BIO *b, const char *in, int inl); -static long bio_zstd_ctrl(BIO *b, int cmd, long num, void *ptr); -static long bio_zstd_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp); - -static const BIO_METHOD bio_meth_zstd = { - BIO_TYPE_COMP, - "zstd", - /* TODO: Convert to new style write function */ - bwrite_conv, - bio_zstd_write, - /* TODO: Convert to new style read function */ - bread_conv, - bio_zstd_read, - NULL, /* bio_zstd_puts, */ - NULL, /* bio_zstd_gets, */ - bio_zstd_ctrl, - bio_zstd_new, - bio_zstd_free, - bio_zstd_callback_ctrl -}; -#endif - -const BIO_METHOD *BIO_f_zstd(void) -{ -#ifndef OPENSSL_NO_ZSTD - if (RUN_ONCE(&zstd_once, ossl_comp_zstd_init)) - return &bio_meth_zstd; -#endif - return NULL; -} - -#ifndef OPENSSL_NO_ZSTD -static int bio_zstd_new(BIO *bi) -{ - BIO_ZSTD_CTX *ctx; - -# ifdef ZSTD_SHARED - (void)COMP_zstd(); - if (zstd_dso == NULL) { - ERR_raise(ERR_LIB_COMP, COMP_R_ZSTD_NOT_SUPPORTED); - return 0; - } -# endif - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) { - ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); - return 0; - } - -# ifdef ZSTD_SHARED - ctx->decompress.state = ZSTD_createDStream(); -# else - ctx->decompress.state = ZSTD_createDStream_advanced(zstd_mem_funcs); -# endif - if (ctx->decompress.state == NULL) - goto err; - ZSTD_initDStream(ctx->decompress.state); - ctx->decompress.bufsize = ZSTD_DStreamInSize(); - -# ifdef ZSTD_SHARED - ctx->compress.state = ZSTD_createCStream(); -# else - ctx->compress.state = ZSTD_createCStream_advanced(zstd_mem_funcs); -# endif - if (ctx->compress.state == NULL) - goto err; - ZSTD_initCStream(ctx->compress.state, ZSTD_CLEVEL_DEFAULT); - ctx->compress.bufsize = ZSTD_CStreamInSize(); - - BIO_set_init(bi, 1); - BIO_set_data(bi, ctx); - - return 1; - err: - ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); - ZSTD_freeDStream(ctx->decompress.state); - ZSTD_freeCStream(ctx->compress.state); - OPENSSL_free(ctx); - return 0; -} - -static int bio_zstd_free(BIO *bi) -{ - BIO_ZSTD_CTX *ctx; - - if (bi == NULL) - return 0; - - ctx = BIO_get_data(bi); - if (ctx != NULL) { - ZSTD_freeDStream(ctx->decompress.state); - OPENSSL_free(ctx->decompress.buffer); - ZSTD_freeCStream(ctx->compress.state); - OPENSSL_free(ctx->compress.outbuf.dst); - OPENSSL_free(ctx); - } - BIO_set_data(bi, NULL); - BIO_set_init(bi, 0); - - return 1; -} - -static int bio_zstd_read(BIO *b, char *out, int outl) -{ - BIO_ZSTD_CTX *ctx; - size_t zret; - int ret; - ZSTD_outBuffer outBuf; - BIO *next = BIO_next(b); - - if (out == NULL || outl <= 0) - return 0; - - ctx = BIO_get_data(b); - BIO_clear_retry_flags(b); - if (ctx->decompress.buffer == NULL) { - ctx->decompress.buffer = OPENSSL_malloc(ctx->decompress.bufsize); - if (ctx->decompress.buffer == NULL) { - ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); - return 0; - } - ctx->decompress.inbuf.src = ctx->decompress.buffer; - ctx->decompress.inbuf.size = 0; - ctx->decompress.inbuf.pos = 0; - } - - /* Copy output data directly to supplied buffer */ - outBuf.dst = out; - outBuf.size = (size_t)outl; - outBuf.pos = 0; - for (;;) { - /* Decompress while data available */ - do { - zret = ZSTD_decompressStream(ctx->decompress.state, &outBuf, &ctx->decompress.inbuf); - if (ZSTD_isError(zret)) { - ERR_raise(ERR_LIB_COMP, COMP_R_ZSTD_DECOMPRESS_ERROR); - ERR_add_error_data(1, ZSTD_getErrorName(zret)); - return -1; - } - /* No more output space */ - if (outBuf.pos == outBuf.size) - return outBuf.pos; - } while (ctx->decompress.inbuf.pos < ctx->decompress.inbuf.size); - - /* - * No data in input buffer try to read some in, if an error then - * return the total data read. - */ - ret = BIO_read(next, ctx->decompress.buffer, ctx->decompress.bufsize); - if (ret <= 0) { - BIO_copy_next_retry(b); - if (ret < 0 && outBuf.pos == 0) - return ret; - return outBuf.pos; - } - ctx->decompress.inbuf.size = ret; - ctx->decompress.inbuf.pos = 0; - } -} - -static int bio_zstd_write(BIO *b, const char *in, int inl) -{ - BIO_ZSTD_CTX *ctx; - size_t zret; - ZSTD_inBuffer inBuf; - int ret; - int done = 0; - BIO *next = BIO_next(b); - - if (in == NULL || inl <= 0) - return 0; - - ctx = BIO_get_data(b); - - BIO_clear_retry_flags(b); - if (ctx->compress.outbuf.dst == NULL) { - ctx->compress.outbuf.dst = OPENSSL_malloc(ctx->compress.bufsize); - if (ctx->compress.outbuf.dst == NULL) { - ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); - return 0; - } - ctx->compress.outbuf.size = ctx->compress.bufsize; - ctx->compress.outbuf.pos = 0; - ctx->compress.write_pos = 0; - } - /* Obtain input data directly from supplied buffer */ - inBuf.src = in; - inBuf.size = inl; - inBuf.pos = 0; - for (;;) { - /* If data in output buffer write it first */ - while (ctx->compress.write_pos < ctx->compress.outbuf.pos) { - ret = BIO_write(next, (unsigned char*)ctx->compress.outbuf.dst + ctx->compress.write_pos, - ctx->compress.outbuf.pos - ctx->compress.write_pos); - if (ret <= 0) { - BIO_copy_next_retry(b); - if (ret < 0 && inBuf.pos == 0) - return ret; - return inBuf.pos; - } - ctx->compress.write_pos += ret; - } - - /* Have we consumed all supplied data? */ - if (done) - return inBuf.pos; - - /* Reset buffer */ - ctx->compress.outbuf.pos = 0; - ctx->compress.outbuf.size = ctx->compress.bufsize; - ctx->compress.write_pos = 0; - /* Compress some more */ - zret = ZSTD_compressStream2(ctx->compress.state, &ctx->compress.outbuf, &inBuf, ZSTD_e_end); - if (ZSTD_isError(zret)) { - ERR_raise(ERR_LIB_COMP, COMP_R_ZSTD_COMPRESS_ERROR); - ERR_add_error_data(1, ZSTD_getErrorName(zret)); - return 0; - } else if (zret == 0) { - done = 1; - } - } -} - -static int bio_zstd_flush(BIO *b) -{ - BIO_ZSTD_CTX *ctx; - size_t zret; - int ret; - BIO *next = BIO_next(b); - - ctx = BIO_get_data(b); - - /* If no data written or already flush show success */ - if (ctx->compress.outbuf.dst == NULL) - return 1; - - BIO_clear_retry_flags(b); - /* No more input data */ - ctx->compress.outbuf.pos = 0; - ctx->compress.outbuf.size = ctx->compress.bufsize; - ctx->compress.write_pos = 0; - for (;;) { - /* If data in output buffer write it first */ - while (ctx->compress.write_pos < ctx->compress.outbuf.pos) { - ret = BIO_write(next, (unsigned char*)ctx->compress.outbuf.dst + ctx->compress.write_pos, - ctx->compress.outbuf.pos - ctx->compress.write_pos); - if (ret <= 0) { - BIO_copy_next_retry(b); - return ret; - } - ctx->compress.write_pos += ret; - } - - /* Reset buffer */ - ctx->compress.outbuf.pos = 0; - ctx->compress.outbuf.size = ctx->compress.bufsize; - ctx->compress.write_pos = 0; - /* Compress some more */ - zret = ZSTD_flushStream(ctx->compress.state, &ctx->compress.outbuf); - if (ZSTD_isError(zret)) { - ERR_raise(ERR_LIB_COMP, COMP_R_ZSTD_DECODE_ERROR); - ERR_add_error_data(1, ZSTD_getErrorName(zret)); - return 0; - } - if (zret == 0) - return 1; - } -} - -static long bio_zstd_ctrl(BIO *b, int cmd, long num, void *ptr) -{ - BIO_ZSTD_CTX *ctx; - int ret = 0, *ip; - size_t ibs, obs; - unsigned char *tmp; - BIO *next = BIO_next(b); - - if (next == NULL) - return 0; - ctx = BIO_get_data(b); - switch (cmd) { - - case BIO_CTRL_RESET: - ctx->compress.write_pos = 0; - ctx->compress.bufsize = 0; - ret = 1; - break; - - case BIO_CTRL_FLUSH: - ret = bio_zstd_flush(b); - if (ret > 0) { - ret = BIO_flush(next); - BIO_copy_next_retry(b); - } - break; - - case BIO_C_SET_BUFF_SIZE: - ibs = ctx->decompress.bufsize; - obs = ctx->compress.bufsize; - if (ptr != NULL) { - ip = ptr; - if (*ip == 0) - ibs = (size_t)num; - else - obs = (size_t)num; - } else { - obs = ibs = (size_t)num; - } - - if (ibs > 0 && ibs != ctx->decompress.bufsize) { - if (ctx->decompress.buffer != NULL) { - tmp = OPENSSL_realloc(ctx->decompress.buffer, ibs); - if (tmp == NULL) - return 0; - if (ctx->decompress.inbuf.src == ctx->decompress.buffer) - ctx->decompress.inbuf.src = tmp; - ctx->decompress.buffer = tmp; - } - ctx->decompress.bufsize = ibs; - } - - if (obs > 0 && obs != ctx->compress.bufsize) { - if (ctx->compress.outbuf.dst != NULL) { - tmp = OPENSSL_realloc(ctx->compress.outbuf.dst, obs); - if (tmp == NULL) - return 0; - ctx->compress.outbuf.dst = tmp; - } - ctx->compress.bufsize = obs; - } - ret = 1; - break; - - case BIO_C_DO_STATE_MACHINE: - BIO_clear_retry_flags(b); - ret = BIO_ctrl(next, cmd, num, ptr); - BIO_copy_next_retry(b); - break; - - case BIO_CTRL_WPENDING: - if (ctx->compress.outbuf.pos < ctx->compress.outbuf.size) - ret = 1; - else - ret = BIO_ctrl(next, cmd, num, ptr); - break; - - case BIO_CTRL_PENDING: - if (ctx->decompress.inbuf.pos < ctx->decompress.inbuf.size) - ret = 1; - else - ret = BIO_ctrl(next, cmd, num, ptr); - break; - - default: - ret = BIO_ctrl(next, cmd, num, ptr); - break; - - } - - return ret; -} - -static long bio_zstd_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp) -{ - BIO *next = BIO_next(b); - if (next == NULL) - return 0; - return BIO_callback_ctrl(next, cmd, fp); -} - -#endif diff --git a/openssl/src/crypto/comp/comp_err.c b/openssl/src/crypto/comp/comp_err.c index 2345da693..70a6eea0f 100644 --- a/openssl/src/crypto/comp/comp_err.c +++ b/openssl/src/crypto/comp/comp_err.c @@ -17,25 +17,12 @@ # ifndef OPENSSL_NO_ERR static const ERR_STRING_DATA COMP_str_reasons[] = { - {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_BROTLI_DECODE_ERROR), - "brotli decode error"}, - {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_BROTLI_ENCODE_ERROR), - "brotli encode error"}, - {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_BROTLI_NOT_SUPPORTED), - "brotli not supported"}, {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZLIB_DEFLATE_ERROR), "zlib deflate error"}, {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZLIB_INFLATE_ERROR), "zlib inflate error"}, {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZLIB_NOT_SUPPORTED), "zlib not supported"}, - {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZSTD_COMPRESS_ERROR), - "zstd compress error"}, - {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZSTD_DECODE_ERROR), "zstd decode error"}, - {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZSTD_DECOMPRESS_ERROR), - "zstd decompress error"}, - {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZSTD_NOT_SUPPORTED), - "zstd not supported"}, {0, NULL} }; diff --git a/openssl/src/crypto/comp/comp_lib.c b/openssl/src/crypto/comp/comp_lib.c index 56ca17a7a..bf9069d87 100644 --- a/openssl/src/crypto/comp/comp_lib.c +++ b/openssl/src/crypto/comp/comp_lib.c @@ -19,11 +19,10 @@ COMP_CTX *COMP_CTX_new(COMP_METHOD *meth) { COMP_CTX *ret; - if (meth == NULL) - return NULL; - - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); return NULL; + } ret->meth = meth; if ((ret->meth->init != NULL) && !ret->meth->init(ret)) { OPENSSL_free(ret); @@ -39,15 +38,11 @@ const COMP_METHOD *COMP_CTX_get_method(const COMP_CTX *ctx) int COMP_get_type(const COMP_METHOD *meth) { - if (meth == NULL) - return NID_undef; return meth->type; } const char *COMP_get_name(const COMP_METHOD *meth) { - if (meth == NULL) - return NULL; return meth->name; } diff --git a/openssl/src/crypto/comp/comp_local.h b/openssl/src/crypto/comp/comp_local.h index d8be9271a..acf113e31 100644 --- a/openssl/src/crypto/comp/comp_local.h +++ b/openssl/src/crypto/comp/comp_local.h @@ -12,12 +12,12 @@ struct comp_method_st { const char *name; /* A text string to identify the library */ int (*init) (COMP_CTX *ctx); void (*finish) (COMP_CTX *ctx); - ossl_ssize_t (*compress) (COMP_CTX *ctx, - unsigned char *out, size_t olen, - unsigned char *in, size_t ilen); - ossl_ssize_t (*expand) (COMP_CTX *ctx, - unsigned char *out, size_t olen, - unsigned char *in, size_t ilen); + int (*compress) (COMP_CTX *ctx, + unsigned char *out, unsigned int olen, + unsigned char *in, unsigned int ilen); + int (*expand) (COMP_CTX *ctx, + unsigned char *out, unsigned int olen, + unsigned char *in, unsigned int ilen); }; struct comp_ctx_st { diff --git a/openssl/src/crypto/conf/conf_def.c b/openssl/src/crypto/conf/conf_def.c index e047746f6..4dced334a 100644 --- a/openssl/src/crypto/conf/conf_def.c +++ b/openssl/src/crypto/conf/conf_def.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -169,11 +169,7 @@ static int def_load(CONF *conf, const char *name, long *line) int ret; BIO *in = NULL; -#ifdef OPENSSL_SYS_VMS - in = BIO_new_file(name, "r"); -#else in = BIO_new_file(name, "rb"); -#endif if (in == NULL) { if (ERR_GET_REASON(ERR_peek_last_error()) == BIO_R_NO_SUCH_FILE) ERR_raise(ERR_LIB_CONF, CONF_R_NO_SUCH_FILE); @@ -193,10 +189,10 @@ static int def_load(CONF *conf, const char *name, long *line) static int parsebool(const char *pval, int *flag) { if (OPENSSL_strcasecmp(pval, "on") == 0 - || OPENSSL_strcasecmp(pval, "true") == 0) { + || OPENSSL_strcasecmp(pval, "true") == 0) { *flag = 1; } else if (OPENSSL_strcasecmp(pval, "off") == 0 - || OPENSSL_strcasecmp(pval, "false") == 0) { + || OPENSSL_strcasecmp(pval, "false") == 0) { *flag = 0; } else { ERR_raise(ERR_LIB_CONF, CONF_R_INVALID_PRAGMA); @@ -226,9 +222,6 @@ static int def_load_bio(CONF *conf, BIO *in, long *line) char *dirpath = NULL; OPENSSL_DIR_CTX *dirctx = NULL; #endif -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - int numincludes = 0; -#endif if ((buff = BUF_MEM_new()) == NULL) { ERR_raise(ERR_LIB_CONF, ERR_R_BUF_LIB); @@ -236,11 +229,13 @@ static int def_load_bio(CONF *conf, BIO *in, long *line) } section = OPENSSL_strdup("default"); - if (section == NULL) + if (section == NULL) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); goto err; + } if (_CONF_new_data(conf) == 0) { - ERR_raise(ERR_LIB_CONF, ERR_R_CONF_LIB); + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); goto err; } @@ -390,8 +385,8 @@ static int def_load_bio(CONF *conf, BIO *in, long *line) psection = section; } p = eat_ws(conf, end); - if (CHECK_AND_SKIP_PREFIX(pname, ".pragma") - && (p != pname || *p == '=')) { + if (strncmp(pname, ".pragma", 7) == 0 + && (p != pname + 7 || *p == '=')) { char *pval; if (*p == '=') { @@ -426,35 +421,23 @@ static int def_load_bio(CONF *conf, BIO *in, long *line) goto err; } else if (strcmp(p, "includedir") == 0) { OPENSSL_free(conf->includedir); - if ((conf->includedir = OPENSSL_strdup(pval)) == NULL) + if ((conf->includedir = OPENSSL_strdup(pval)) == NULL) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); goto err; + } } /* * We *ignore* any unknown pragma. */ continue; - } else if (CHECK_AND_SKIP_PREFIX(pname, ".include") - && (p != pname || *p == '=')) { + } else if (strncmp(pname, ".include", 8) == 0 + && (p != pname + 8 || *p == '=')) { char *include = NULL; BIO *next; const char *include_dir = ossl_safe_getenv("OPENSSL_CONF_INCLUDE"); char *include_path = NULL; -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - /* - * The include processing below can cause the "conf" fuzzer to - * timeout due to the fuzzer inserting large and complicated - * includes - with a large amount of time spent in - * OPENSSL_strlcat/OPENSSL_strcpy. This is not a security - * concern because config files should never come from untrusted - * sources. We just set an arbitrary limit on the allowed - * number of includes when fuzzing to prevent this timeout. - */ - if (numincludes++ > 10) - goto err; -#endif - if (include_dir == NULL) include_dir = conf->includedir; @@ -471,6 +454,7 @@ static int def_load_bio(CONF *conf, BIO *in, long *line) include_path = OPENSSL_malloc(newlen); if (include_path == NULL) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); OPENSSL_free(include); goto err; } @@ -507,13 +491,13 @@ static int def_load_bio(CONF *conf, BIO *in, long *line) /* push the currently processing BIO onto stack */ if (biosk == NULL) { if ((biosk = sk_BIO_new_null()) == NULL) { - ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); BIO_free(next); goto err; } } if (!sk_BIO_push(biosk, in)) { - ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); BIO_free(next); goto err; } @@ -531,12 +515,16 @@ static int def_load_bio(CONF *conf, BIO *in, long *line) start = eat_ws(conf, p); trim_ws(conf, start); - if ((v = OPENSSL_malloc(sizeof(*v))) == NULL) + if ((v = OPENSSL_malloc(sizeof(*v))) == NULL) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); goto err; + } v->name = OPENSSL_strdup(pname); v->value = NULL; - if (v->name == NULL) + if (v->name == NULL) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); goto err; + } if (!str_copy(conf, psection, &(v->value), start)) goto err; @@ -552,7 +540,7 @@ static int def_load_bio(CONF *conf, BIO *in, long *line) } else tv = sv; if (_CONF_add_string(conf, tv, v) == 0) { - ERR_raise(ERR_LIB_CONF, ERR_R_CONF_LIB); + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); goto err; } v = NULL; @@ -765,7 +753,7 @@ static int str_copy(CONF *conf, char *section, char **pto, char *from) goto err; } if (!BUF_MEM_grow_clean(buf, newsize)) { - ERR_raise(ERR_LIB_CONF, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); goto err; } while (*p) @@ -849,28 +837,18 @@ static BIO *get_next_file(const char *path, OPENSSL_DIR_CTX **dirctx) if ((namelen > 5 && OPENSSL_strcasecmp(filename + namelen - 5, ".conf") == 0) - || (namelen > 4 - && OPENSSL_strcasecmp(filename + namelen - 4, ".cnf") == 0)) { + || (namelen > 4 + && OPENSSL_strcasecmp(filename + namelen - 4, ".cnf") == 0)) { size_t newlen; char *newpath; BIO *bio; newlen = pathlen + namelen + 2; newpath = OPENSSL_zalloc(newlen); - if (newpath == NULL) + if (newpath == NULL) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); break; -#ifdef OPENSSL_SYS_VMS - /* - * If the given path isn't clear VMS syntax, - * we treat it as on Unix. - */ - if (path[pathlen - 1] == ']' - || path[pathlen - 1] == '>' - || path[pathlen - 1] == ':') { - /* Clear VMS directory syntax, just copy as is */ - OPENSSL_strlcpy(newpath, path, newlen); } -#endif if (newpath[0] == '\0') { OPENSSL_strlcpy(newpath, path, newlen); OPENSSL_strlcat(newpath, "/", newlen); @@ -892,7 +870,7 @@ static BIO *get_next_file(const char *path, OPENSSL_DIR_CTX **dirctx) static int is_keytype(const CONF *conf, char c, unsigned short type) { - const unsigned short *keytypes = (const unsigned short *) conf->meth_data; + const unsigned short * keytypes = (const unsigned short *) conf->meth_data; unsigned char key = (unsigned char)c; #ifdef CHARSET_EBCDIC diff --git a/openssl/src/crypto/conf/conf_def.h b/openssl/src/crypto/conf/conf_def.h index 1f66a58e0..e5321bd30 100644 --- a/openssl/src/crypto/conf/conf_def.h +++ b/openssl/src/crypto/conf/conf_def.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by crypto/conf/keysets.pl * - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at diff --git a/openssl/src/crypto/conf/conf_err.c b/openssl/src/crypto/conf/conf_err.c index 9f1309c50..68ee90b97 100644 --- a/openssl/src/crypto/conf/conf_err.c +++ b/openssl/src/crypto/conf/conf_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -41,8 +41,6 @@ static const ERR_STRING_DATA CONF_str_reasons[] = { "openssl conf references missing section"}, {ERR_PACK(ERR_LIB_CONF, 0, CONF_R_RECURSIVE_DIRECTORY_INCLUDE), "recursive directory include"}, - {ERR_PACK(ERR_LIB_CONF, 0, CONF_R_RECURSIVE_SECTION_REFERENCE), - "recursive section reference"}, {ERR_PACK(ERR_LIB_CONF, 0, CONF_R_RELATIVE_PATH), "relative path"}, {ERR_PACK(ERR_LIB_CONF, 0, CONF_R_SSL_COMMAND_SECTION_EMPTY), "ssl command section empty"}, diff --git a/openssl/src/crypto/conf/conf_lib.c b/openssl/src/crypto/conf/conf_lib.c index 601f49430..765ba362f 100644 --- a/openssl/src/crypto/conf/conf_lib.c +++ b/openssl/src/crypto/conf/conf_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -49,11 +49,7 @@ LHASH_OF(CONF_VALUE) *CONF_load(LHASH_OF(CONF_VALUE) *conf, const char *file, LHASH_OF(CONF_VALUE) *ltmp; BIO *in = NULL; -#ifdef OPENSSL_SYS_VMS - in = BIO_new_file(file, "r"); -#else in = BIO_new_file(file, "rb"); -#endif if (in == NULL) { ERR_raise(ERR_LIB_CONF, ERR_R_SYS_LIB); return NULL; @@ -188,7 +184,7 @@ CONF *NCONF_new_ex(OSSL_LIB_CTX *libctx, CONF_METHOD *meth) ret = meth->create(meth); if (ret == NULL) { - ERR_raise(ERR_LIB_CONF, ERR_R_CONF_LIB); + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); return NULL; } ret->libctx = libctx; @@ -421,12 +417,6 @@ OPENSSL_INIT_SETTINGS *OPENSSL_INIT_new(void) #ifndef OPENSSL_NO_STDIO -/* - * If CRYPTO_set_mem_functions is called after this, then - * memory allocation and deallocation in this function can - * become disjointed. Avoid this by always using standard - * strdup & free instead of OPENSSL_strdup & OPENSSL_free. - */ int OPENSSL_INIT_set_config_filename(OPENSSL_INIT_SETTINGS *settings, const char *filename) { @@ -450,12 +440,6 @@ void OPENSSL_INIT_set_config_file_flags(OPENSSL_INIT_SETTINGS *settings, settings->flags = flags; } -/* - * If CRYPTO_set_mem_functions is called after this, then - * memory allocation and deallocation in this function can - * become disjointed. Avoid this by always using standard - * strdup & free instead of OPENSSL_strdup & OPENSSL_free. - */ int OPENSSL_INIT_set_config_appname(OPENSSL_INIT_SETTINGS *settings, const char *appname) { diff --git a/openssl/src/crypto/conf/conf_mod.c b/openssl/src/crypto/conf/conf_mod.c index a19575af3..5d0b7407c 100644 --- a/openssl/src/crypto/conf/conf_mod.c +++ b/openssl/src/crypto/conf/conf_mod.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,7 +11,6 @@ #define OPENSSL_SUPPRESS_DEPRECATED #include "internal/cryptlib.h" -#include "internal/rcu.h" #include #include #include @@ -63,10 +62,8 @@ struct conf_imodule_st { void *usr_data; }; -static CRYPTO_ONCE init_module_list_lock = CRYPTO_ONCE_STATIC_INIT; -static CRYPTO_RCU_LOCK *module_list_lock = NULL; -static STACK_OF(CONF_MODULE) *supported_modules = NULL; /* protected by lock */ -static STACK_OF(CONF_IMODULE) *initialized_modules = NULL; /* protected by lock */ +static STACK_OF(CONF_MODULE) *supported_modules = NULL; +static STACK_OF(CONF_IMODULE) *initialized_modules = NULL; static CRYPTO_ONCE load_builtin_modules = CRYPTO_ONCE_STATIC_INIT; @@ -83,31 +80,6 @@ static int module_init(CONF_MODULE *pmod, const char *name, const char *value, static CONF_MODULE *module_load_dso(const CONF *cnf, const char *name, const char *value); -static int conf_modules_finish_int(void); - -static void module_lists_free(void) -{ - ossl_rcu_lock_free(module_list_lock); - module_list_lock = NULL; - - sk_CONF_MODULE_free(supported_modules); - supported_modules = NULL; - - sk_CONF_IMODULE_free(initialized_modules); - initialized_modules = NULL; -} - -DEFINE_RUN_ONCE_STATIC(do_init_module_list_lock) -{ - module_list_lock = ossl_rcu_lock_new(1); - if (module_list_lock == NULL) { - ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB); - return 0; - } - - return 1; -} - static int conf_diagnostics(const CONF *cnf) { return _CONF_get_number(cnf, NULL, "config_diagnostics") != 0; @@ -185,21 +157,15 @@ int CONF_modules_load_file_ex(OSSL_LIB_CTX *libctx, const char *filename, CONF *conf = NULL; int ret = 0, diagnostics = 0; - ERR_set_mark(); - if (filename == NULL) { file = CONF_get1_default_config_file(); if (file == NULL) goto err; - if (*file == '\0') { - /* Do not try to load an empty file name but do not error out */ - ret = 1; - goto err; - } } else { file = (char *)filename; } + ERR_set_mark(); conf = NCONF_new_ex(libctx, NULL); if (conf == NULL) goto err; @@ -328,53 +294,31 @@ static CONF_MODULE *module_add(DSO *dso, const char *name, conf_init_func *ifunc, conf_finish_func *ffunc) { CONF_MODULE *tmod = NULL; - STACK_OF(CONF_MODULE) *old_modules; - STACK_OF(CONF_MODULE) *new_modules; - - if (!RUN_ONCE(&init_module_list_lock, do_init_module_list_lock)) + if (supported_modules == NULL) + supported_modules = sk_CONF_MODULE_new_null(); + if (supported_modules == NULL) return NULL; - - ossl_rcu_write_lock(module_list_lock); - - old_modules = ossl_rcu_deref(&supported_modules); - - if (old_modules == NULL) - new_modules = sk_CONF_MODULE_new_null(); - else - new_modules = sk_CONF_MODULE_dup(old_modules); - - if (new_modules == NULL) - goto err; - - if ((tmod = OPENSSL_zalloc(sizeof(*tmod))) == NULL) - goto err; + if ((tmod = OPENSSL_zalloc(sizeof(*tmod))) == NULL) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); + return NULL; + } tmod->dso = dso; tmod->name = OPENSSL_strdup(name); tmod->init = ifunc; tmod->finish = ffunc; - if (tmod->name == NULL) - goto err; - - if (!sk_CONF_MODULE_push(new_modules, tmod)) - goto err; - - ossl_rcu_assign_ptr(&supported_modules, &new_modules); - ossl_rcu_write_unlock(module_list_lock); - ossl_synchronize_rcu(module_list_lock); - - sk_CONF_MODULE_free(old_modules); - return tmod; + if (tmod->name == NULL) { + OPENSSL_free(tmod); + return NULL; + } - err: - ossl_rcu_write_unlock(module_list_lock); - sk_CONF_MODULE_free(new_modules); - if (tmod != NULL) { + if (!sk_CONF_MODULE_push(supported_modules, tmod)) { OPENSSL_free(tmod->name); OPENSSL_free(tmod); + return NULL; } - sk_CONF_MODULE_free(new_modules); - return NULL; + + return tmod; } /* @@ -388,8 +332,6 @@ static CONF_MODULE *module_find(const char *name) CONF_MODULE *tmod; int i, nchar; char *p; - STACK_OF(CONF_MODULE) *mods; - p = strrchr(name, '.'); if (p) @@ -397,22 +339,14 @@ static CONF_MODULE *module_find(const char *name) else nchar = strlen(name); - if (!RUN_ONCE(&init_module_list_lock, do_init_module_list_lock)) - return NULL; - - ossl_rcu_read_lock(module_list_lock); - mods = ossl_rcu_deref(&supported_modules); - - for (i = 0; i < sk_CONF_MODULE_num(mods); i++) { - tmod = sk_CONF_MODULE_value(mods, i); - if (strncmp(tmod->name, name, nchar) == 0) { - ossl_rcu_read_unlock(module_list_lock); + for (i = 0; i < sk_CONF_MODULE_num(supported_modules); i++) { + tmod = sk_CONF_MODULE_value(supported_modules, i); + if (strncmp(tmod->name, name, nchar) == 0) return tmod; - } } - ossl_rcu_read_unlock(module_list_lock); return NULL; + } /* initialize a module */ @@ -422,8 +356,6 @@ static int module_init(CONF_MODULE *pmod, const char *name, const char *value, int ret = 1; int init_called = 0; CONF_IMODULE *imod = NULL; - STACK_OF(CONF_IMODULE) *old_modules; - STACK_OF(CONF_IMODULE) *new_modules; /* Otherwise add initialized module to list */ imod = OPENSSL_malloc(sizeof(*imod)); @@ -447,37 +379,21 @@ static int module_init(CONF_MODULE *pmod, const char *name, const char *value, goto err; } - if (!RUN_ONCE(&init_module_list_lock, do_init_module_list_lock)) - goto err; - - ossl_rcu_write_lock(module_list_lock); - - old_modules = ossl_rcu_deref(&initialized_modules); - - if (old_modules == NULL) - new_modules = sk_CONF_IMODULE_new_null(); - else - new_modules = sk_CONF_IMODULE_dup(old_modules); - - if (new_modules == NULL) { - ossl_rcu_write_unlock(module_list_lock); - ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB); - goto err; + if (initialized_modules == NULL) { + initialized_modules = sk_CONF_IMODULE_new_null(); + if (!initialized_modules) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); + goto err; + } } - if (!sk_CONF_IMODULE_push(new_modules, imod)) { - ossl_rcu_write_unlock(module_list_lock); - sk_CONF_IMODULE_free(new_modules); - ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB); + if (!sk_CONF_IMODULE_push(initialized_modules, imod)) { + ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE); goto err; } pmod->links++; - ossl_rcu_assign_ptr(&initialized_modules, &new_modules); - ossl_rcu_write_unlock(module_list_lock); - ossl_synchronize_rcu(module_list_lock); - sk_CONF_IMODULE_free(old_modules); return ret; err: @@ -507,46 +423,21 @@ void CONF_modules_unload(int all) { int i; CONF_MODULE *md; - STACK_OF(CONF_MODULE) *old_modules; - STACK_OF(CONF_MODULE) *new_modules; - STACK_OF(CONF_MODULE) *to_delete; - - if (!conf_modules_finish_int()) /* also inits module list lock */ - return; - - ossl_rcu_write_lock(module_list_lock); - - old_modules = ossl_rcu_deref(&supported_modules); - new_modules = sk_CONF_MODULE_dup(old_modules); - to_delete = sk_CONF_MODULE_new_null(); - - if (new_modules == NULL) { - ossl_rcu_write_unlock(module_list_lock); - return; - } - + CONF_modules_finish(); /* unload modules in reverse order */ - for (i = sk_CONF_MODULE_num(new_modules) - 1; i >= 0; i--) { - md = sk_CONF_MODULE_value(new_modules, i); + for (i = sk_CONF_MODULE_num(supported_modules) - 1; i >= 0; i--) { + md = sk_CONF_MODULE_value(supported_modules, i); /* If static or in use and 'all' not set ignore it */ if (((md->links > 0) || !md->dso) && !all) continue; /* Since we're working in reverse this is OK */ - (void)sk_CONF_MODULE_delete(new_modules, i); - sk_CONF_MODULE_push(to_delete, md); + (void)sk_CONF_MODULE_delete(supported_modules, i); + module_free(md); } - - if (sk_CONF_MODULE_num(new_modules) == 0) { - sk_CONF_MODULE_free(new_modules); - new_modules = NULL; + if (sk_CONF_MODULE_num(supported_modules) == 0) { + sk_CONF_MODULE_free(supported_modules); + supported_modules = NULL; } - - ossl_rcu_assign_ptr(&supported_modules, &new_modules); - ossl_rcu_write_unlock(module_list_lock); - ossl_synchronize_rcu(module_list_lock); - sk_CONF_MODULE_free(old_modules); - sk_CONF_MODULE_pop_free(to_delete, module_free); - } /* unload a single module */ @@ -559,37 +450,15 @@ static void module_free(CONF_MODULE *md) /* finish and free up all modules instances */ -static int conf_modules_finish_int(void) +void CONF_modules_finish(void) { CONF_IMODULE *imod; - STACK_OF(CONF_IMODULE) *old_modules; - STACK_OF(CONF_IMODULE) *new_modules = NULL; - - if (!RUN_ONCE(&init_module_list_lock, do_init_module_list_lock)) - return 0; - - /* If module_list_lock is NULL here it means we were already unloaded */ - if (module_list_lock == NULL) - return 0; - - ossl_rcu_write_lock(module_list_lock); - old_modules = ossl_rcu_deref(&initialized_modules); - ossl_rcu_assign_ptr(&initialized_modules, &new_modules); - ossl_rcu_write_unlock(module_list_lock); - ossl_synchronize_rcu(module_list_lock); - - while (sk_CONF_IMODULE_num(old_modules) > 0) { - imod = sk_CONF_IMODULE_pop(old_modules); + while (sk_CONF_IMODULE_num(initialized_modules) > 0) { + imod = sk_CONF_IMODULE_pop(initialized_modules); module_finish(imod); } - sk_CONF_IMODULE_free(old_modules); - - return 1; -} - -void CONF_modules_finish(void) -{ - conf_modules_finish_int(); + sk_CONF_IMODULE_free(initialized_modules); + initialized_modules = NULL; } /* finish a module instance */ @@ -619,8 +488,8 @@ int CONF_module_add(const char *name, conf_init_func *ifunc, void ossl_config_modules_free(void) { - CONF_modules_unload(1); /* calls CONF_modules_finish */ - module_lists_free(); + CONF_modules_finish(); + CONF_modules_unload(1); } /* Utility functions */ @@ -674,22 +543,19 @@ void CONF_module_set_usr_data(CONF_MODULE *pmod, void *usr_data) char *CONF_get1_default_config_file(void) { const char *t; - char *file, *sep = ""; + char *file; size_t size; if ((file = ossl_safe_getenv("OPENSSL_CONF")) != NULL) return OPENSSL_strdup(file); t = X509_get_default_cert_area(); -#ifndef OPENSSL_SYS_VMS - sep = "/"; -#endif - size = strlen(t) + strlen(sep) + strlen(OPENSSL_CONF) + 1; + size = strlen(t) + strlen("/") + strlen(OPENSSL_CONF) + 1; file = OPENSSL_malloc(size); if (file == NULL) return NULL; - BIO_snprintf(file, size, "%s%s%s", t, sep, OPENSSL_CONF); + BIO_snprintf(file, size, "%s/%s", t, OPENSSL_CONF); return file; } diff --git a/openssl/src/crypto/conf/conf_sap.c b/openssl/src/crypto/conf/conf_sap.c index 6b3defe0f..6742ecf87 100644 --- a/openssl/src/crypto/conf/conf_sap.c +++ b/openssl/src/crypto/conf/conf_sap.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -44,20 +44,16 @@ void OPENSSL_config(const char *appname) int ossl_config_int(const OPENSSL_INIT_SETTINGS *settings) { int ret = 0; -#if defined(OPENSSL_INIT_DEBUG) || !defined(OPENSSL_SYS_UEFI) const char *filename; const char *appname; unsigned long flags; -#endif if (openssl_configured) return 1; -#if defined(OPENSSL_INIT_DEBUG) || !defined(OPENSSL_SYS_UEFI) filename = settings ? settings->filename : NULL; appname = settings ? settings->appname : NULL; flags = settings ? settings->flags : DEFAULT_CONF_MFLAGS; -#endif #ifdef OPENSSL_INIT_DEBUG fprintf(stderr, "OPENSSL_INIT: ossl_config_int(%s, %s, %lu)\n", @@ -65,10 +61,7 @@ int ossl_config_int(const OPENSSL_INIT_SETTINGS *settings) #endif #ifndef OPENSSL_SYS_UEFI - ret = CONF_modules_load_file_ex(OSSL_LIB_CTX_get0_global_default(), - filename, appname, flags); -#else - ret = 1; + ret = CONF_modules_load_file(filename, appname, flags); #endif openssl_configured = 1; return ret; diff --git a/openssl/src/crypto/context.c b/openssl/src/crypto/context.c index 33d52a964..bdfc4d02a 100644 --- a/openssl/src/crypto/context.c +++ b/openssl/src/crypto/context.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,40 +14,33 @@ #include "internal/core.h" #include "internal/bio.h" #include "internal/provider.h" -#include "crypto/decoder.h" -#include "crypto/context.h" +#include "crypto/ctype.h" + +struct ossl_lib_ctx_onfree_list_st { + ossl_lib_ctx_onfree_fn *fn; + struct ossl_lib_ctx_onfree_list_st *next; +}; struct ossl_lib_ctx_st { - CRYPTO_RWLOCK *lock, *rand_crngt_lock; + CRYPTO_RWLOCK *lock; + CRYPTO_EX_DATA data; + + /* + * For most data in the OSSL_LIB_CTX we just use ex_data to store it. But + * that doesn't work for ex_data itself - so we store that directly. + */ OSSL_EX_DATA_GLOBAL global; - void *property_string_data; - void *evp_method_store; - void *provider_store; - void *namemap; - void *property_defns; - void *global_properties; - void *drbg; - void *drbg_nonce; -#ifndef FIPS_MODULE - void *provider_conf; - void *bio_core; - void *child_provider; - OSSL_METHOD_STORE *decoder_store; - void *decoder_cache; - OSSL_METHOD_STORE *encoder_store; - OSSL_METHOD_STORE *store_loader_store; - void *self_test_cb; -#endif -#if defined(OPENSSL_THREADS) - void *threads; -#endif - void *rand_crngt; -#ifdef FIPS_MODULE - void *thread_event_handler; - void *fips_prov; -#endif + /* Map internal static indexes to dynamically created indexes */ + int dyn_indexes[OSSL_LIB_CTX_MAX_INDEXES]; + /* Keep a separate lock for each index */ + CRYPTO_RWLOCK *index_locks[OSSL_LIB_CTX_MAX_INDEXES]; + + CRYPTO_RWLOCK *oncelock; + int run_once_done[OSSL_LIB_CTX_MAX_RUN_ONCE]; + int run_once_ret[OSSL_LIB_CTX_MAX_RUN_ONCE]; + struct ossl_lib_ctx_onfree_list_st *onfreelist; unsigned int ischild:1; }; @@ -75,285 +68,75 @@ int ossl_lib_ctx_is_child(OSSL_LIB_CTX *ctx) return ctx->ischild; } -static void context_deinit_objs(OSSL_LIB_CTX *ctx); - static int context_init(OSSL_LIB_CTX *ctx) { + size_t i; int exdata_done = 0; ctx->lock = CRYPTO_THREAD_lock_new(); if (ctx->lock == NULL) return 0; - ctx->rand_crngt_lock = CRYPTO_THREAD_lock_new(); - if (ctx->rand_crngt_lock == NULL) + ctx->oncelock = CRYPTO_THREAD_lock_new(); + if (ctx->oncelock == NULL) goto err; - /* Initialize ex_data. */ + for (i = 0; i < OSSL_LIB_CTX_MAX_INDEXES; i++) { + ctx->index_locks[i] = CRYPTO_THREAD_lock_new(); + ctx->dyn_indexes[i] = -1; + if (ctx->index_locks[i] == NULL) + goto err; + } + + /* OSSL_LIB_CTX is built on top of ex_data so we initialise that directly */ if (!ossl_do_ex_data_init(ctx)) goto err; exdata_done = 1; - /* P2. We want evp_method_store to be cleaned up before the provider store */ - ctx->evp_method_store = ossl_method_store_new(ctx); - if (ctx->evp_method_store == NULL) - goto err; - -#ifndef FIPS_MODULE - /* P2. Must be freed before the provider store is freed */ - ctx->provider_conf = ossl_prov_conf_ctx_new(ctx); - if (ctx->provider_conf == NULL) - goto err; -#endif - - /* P2. */ - ctx->drbg = ossl_rand_ctx_new(ctx); - if (ctx->drbg == NULL) - goto err; - -#ifndef FIPS_MODULE - /* - * P2. We want decoder_store/decoder_cache to be cleaned up before the - * provider store - */ - ctx->decoder_store = ossl_method_store_new(ctx); - if (ctx->decoder_store == NULL) - goto err; - ctx->decoder_cache = ossl_decoder_cache_new(ctx); - if (ctx->decoder_cache == NULL) - goto err; - - /* P2. We want encoder_store to be cleaned up before the provider store */ - ctx->encoder_store = ossl_method_store_new(ctx); - if (ctx->encoder_store == NULL) + if (!ossl_crypto_new_ex_data_ex(ctx, CRYPTO_EX_INDEX_OSSL_LIB_CTX, NULL, + &ctx->data)) goto err; - /* P2. We want loader_store to be cleaned up before the provider store */ - ctx->store_loader_store = ossl_method_store_new(ctx); - if (ctx->store_loader_store == NULL) - goto err; -#endif - - /* P1. Needs to be freed before the child provider data is freed */ - ctx->provider_store = ossl_provider_store_new(ctx); - if (ctx->provider_store == NULL) - goto err; - - /* Default priority. */ - ctx->property_string_data = ossl_property_string_data_new(ctx); - if (ctx->property_string_data == NULL) - goto err; - - ctx->namemap = ossl_stored_namemap_new(ctx); - if (ctx->namemap == NULL) - goto err; - - ctx->property_defns = ossl_property_defns_new(ctx); - if (ctx->property_defns == NULL) - goto err; - - ctx->global_properties = ossl_ctx_global_properties_new(ctx); - if (ctx->global_properties == NULL) - goto err; - -#ifndef FIPS_MODULE - ctx->bio_core = ossl_bio_core_globals_new(ctx); - if (ctx->bio_core == NULL) - goto err; -#endif - - ctx->drbg_nonce = ossl_prov_drbg_nonce_ctx_new(ctx); - if (ctx->drbg_nonce == NULL) - goto err; - -#ifndef FIPS_MODULE - ctx->self_test_cb = ossl_self_test_set_callback_new(ctx); - if (ctx->self_test_cb == NULL) - goto err; -#endif - -#ifdef FIPS_MODULE - ctx->thread_event_handler = ossl_thread_event_ctx_new(ctx); - if (ctx->thread_event_handler == NULL) - goto err; - - ctx->fips_prov = ossl_fips_prov_ossl_ctx_new(ctx); - if (ctx->fips_prov == NULL) - goto err; -#endif - -#ifndef OPENSSL_NO_THREAD_POOL - ctx->threads = ossl_threads_ctx_new(ctx); - if (ctx->threads == NULL) - goto err; -#endif - - /* Low priority. */ -#ifndef FIPS_MODULE - ctx->child_provider = ossl_child_prov_ctx_new(ctx); - if (ctx->child_provider == NULL) - goto err; -#endif - /* Everything depends on properties, so we also pre-initialise that */ if (!ossl_property_parse_init(ctx)) goto err; return 1; - err: - context_deinit_objs(ctx); - if (exdata_done) ossl_crypto_cleanup_all_ex_data_int(ctx); - - CRYPTO_THREAD_lock_free(ctx->rand_crngt_lock); + for (i = 0; i < OSSL_LIB_CTX_MAX_INDEXES; i++) + CRYPTO_THREAD_lock_free(ctx->index_locks[i]); + CRYPTO_THREAD_lock_free(ctx->oncelock); CRYPTO_THREAD_lock_free(ctx->lock); memset(ctx, '\0', sizeof(*ctx)); return 0; } -static void context_deinit_objs(OSSL_LIB_CTX *ctx) -{ - /* P2. We want evp_method_store to be cleaned up before the provider store */ - if (ctx->evp_method_store != NULL) { - ossl_method_store_free(ctx->evp_method_store); - ctx->evp_method_store = NULL; - } - - /* P2. */ - if (ctx->drbg != NULL) { - ossl_rand_ctx_free(ctx->drbg); - ctx->drbg = NULL; - } - -#ifndef FIPS_MODULE - /* P2. */ - if (ctx->provider_conf != NULL) { - ossl_prov_conf_ctx_free(ctx->provider_conf); - ctx->provider_conf = NULL; - } - - /* - * P2. We want decoder_store/decoder_cache to be cleaned up before the - * provider store - */ - if (ctx->decoder_store != NULL) { - ossl_method_store_free(ctx->decoder_store); - ctx->decoder_store = NULL; - } - if (ctx->decoder_cache != NULL) { - ossl_decoder_cache_free(ctx->decoder_cache); - ctx->decoder_cache = NULL; - } - - - /* P2. We want encoder_store to be cleaned up before the provider store */ - if (ctx->encoder_store != NULL) { - ossl_method_store_free(ctx->encoder_store); - ctx->encoder_store = NULL; - } - - /* P2. We want loader_store to be cleaned up before the provider store */ - if (ctx->store_loader_store != NULL) { - ossl_method_store_free(ctx->store_loader_store); - ctx->store_loader_store = NULL; - } -#endif - - /* P1. Needs to be freed before the child provider data is freed */ - if (ctx->provider_store != NULL) { - ossl_provider_store_free(ctx->provider_store); - ctx->provider_store = NULL; - } - - /* Default priority. */ - if (ctx->property_string_data != NULL) { - ossl_property_string_data_free(ctx->property_string_data); - ctx->property_string_data = NULL; - } - - if (ctx->namemap != NULL) { - ossl_stored_namemap_free(ctx->namemap); - ctx->namemap = NULL; - } - - if (ctx->property_defns != NULL) { - ossl_property_defns_free(ctx->property_defns); - ctx->property_defns = NULL; - } - - if (ctx->global_properties != NULL) { - ossl_ctx_global_properties_free(ctx->global_properties); - ctx->global_properties = NULL; - } - -#ifndef FIPS_MODULE - if (ctx->bio_core != NULL) { - ossl_bio_core_globals_free(ctx->bio_core); - ctx->bio_core = NULL; - } -#endif - - if (ctx->drbg_nonce != NULL) { - ossl_prov_drbg_nonce_ctx_free(ctx->drbg_nonce); - ctx->drbg_nonce = NULL; - } - -#ifndef FIPS_MODULE - if (ctx->self_test_cb != NULL) { - ossl_self_test_set_callback_free(ctx->self_test_cb); - ctx->self_test_cb = NULL; - } -#endif - - if (ctx->rand_crngt != NULL) { - ossl_rand_crng_ctx_free(ctx->rand_crngt); - ctx->rand_crngt = NULL; - } - -#ifdef FIPS_MODULE - if (ctx->thread_event_handler != NULL) { - ossl_thread_event_ctx_free(ctx->thread_event_handler); - ctx->thread_event_handler = NULL; - } - - if (ctx->fips_prov != NULL) { - ossl_fips_prov_ossl_ctx_free(ctx->fips_prov); - ctx->fips_prov = NULL; - } -#endif - -#ifndef OPENSSL_NO_THREAD_POOL - if (ctx->threads != NULL) { - ossl_threads_ctx_free(ctx->threads); - ctx->threads = NULL; - } -#endif - - /* Low priority. */ -#ifndef FIPS_MODULE - if (ctx->child_provider != NULL) { - ossl_child_prov_ctx_free(ctx->child_provider); - ctx->child_provider = NULL; - } -#endif -} - static int context_deinit(OSSL_LIB_CTX *ctx) { + struct ossl_lib_ctx_onfree_list_st *tmp, *onfree; + int i; + if (ctx == NULL) return 1; ossl_ctx_thread_stop(ctx); - context_deinit_objs(ctx); - + onfree = ctx->onfreelist; + while (onfree != NULL) { + onfree->fn(ctx); + tmp = onfree; + onfree = onfree->next; + OPENSSL_free(tmp); + } + CRYPTO_free_ex_data(CRYPTO_EX_INDEX_OSSL_LIB_CTX, NULL, &ctx->data); ossl_crypto_cleanup_all_ex_data_int(ctx); + for (i = 0; i < OSSL_LIB_CTX_MAX_INDEXES; i++) + CRYPTO_THREAD_lock_free(ctx->index_locks[i]); - CRYPTO_THREAD_lock_free(ctx->rand_crngt_lock); + CRYPTO_THREAD_lock_free(ctx->oncelock); CRYPTO_THREAD_lock_free(ctx->lock); - ctx->rand_crngt_lock = NULL; ctx->lock = NULL; return 1; } @@ -364,32 +147,17 @@ static OSSL_LIB_CTX default_context_int; static CRYPTO_ONCE default_context_init = CRYPTO_ONCE_STATIC_INIT; static CRYPTO_THREAD_LOCAL default_context_thread_local; -static int default_context_inited = 0; DEFINE_RUN_ONCE_STATIC(default_context_do_init) { - if (!CRYPTO_THREAD_init_local(&default_context_thread_local, NULL)) - goto err; - - if (!context_init(&default_context_int)) - goto deinit_thread; - - default_context_inited = 1; - return 1; - -deinit_thread: - CRYPTO_THREAD_cleanup_local(&default_context_thread_local); -err: - return 0; + return CRYPTO_THREAD_init_local(&default_context_thread_local, NULL) + && context_init(&default_context_int); } void ossl_lib_ctx_default_deinit(void) { - if (!default_context_inited) - return; context_deinit(&default_context_int); CRYPTO_THREAD_cleanup_local(&default_context_thread_local); - default_context_inited = 0; } static OSSL_LIB_CTX *get_thread_default_context(void) @@ -503,15 +271,6 @@ OSSL_LIB_CTX *OSSL_LIB_CTX_set0_default(OSSL_LIB_CTX *libctx) return NULL; } - -void ossl_release_default_drbg_ctx(void) -{ - /* early release of the DRBG in global default libctx */ - if (default_context_int.drbg != NULL) { - ossl_rand_ctx_free(default_context_int.drbg); - default_context_int.drbg = NULL; - } -} #endif OSSL_LIB_CTX *ossl_lib_ctx_get_concrete(OSSL_LIB_CTX *ctx) @@ -541,95 +300,127 @@ int ossl_lib_ctx_is_global_default(OSSL_LIB_CTX *ctx) return 0; } -void *ossl_lib_ctx_get_data(OSSL_LIB_CTX *ctx, int index) +static void ossl_lib_ctx_generic_new(void *parent_ign, void *ptr_ign, + CRYPTO_EX_DATA *ad, int index, + long argl_ign, void *argp) +{ + const OSSL_LIB_CTX_METHOD *meth = argp; + OSSL_LIB_CTX *ctx = ossl_crypto_ex_data_get_ossl_lib_ctx(ad); + void *ptr = meth->new_func(ctx); + + if (ptr != NULL) { + if (!CRYPTO_THREAD_write_lock(ctx->lock)) + /* + * Can't return something, so best to hope that something will + * fail later. :( + */ + return; + CRYPTO_set_ex_data(ad, index, ptr); + CRYPTO_THREAD_unlock(ctx->lock); + } +} +static void ossl_lib_ctx_generic_free(void *parent_ign, void *ptr, + CRYPTO_EX_DATA *ad, int index, + long argl_ign, void *argp) +{ + const OSSL_LIB_CTX_METHOD *meth = argp; + + meth->free_func(ptr); +} + +static int ossl_lib_ctx_init_index(OSSL_LIB_CTX *ctx, int static_index, + const OSSL_LIB_CTX_METHOD *meth) { - void *p; + int idx; ctx = ossl_lib_ctx_get_concrete(ctx); if (ctx == NULL) - return NULL; + return 0; - switch (index) { - case OSSL_LIB_CTX_PROPERTY_STRING_INDEX: - return ctx->property_string_data; - case OSSL_LIB_CTX_EVP_METHOD_STORE_INDEX: - return ctx->evp_method_store; - case OSSL_LIB_CTX_PROVIDER_STORE_INDEX: - return ctx->provider_store; - case OSSL_LIB_CTX_NAMEMAP_INDEX: - return ctx->namemap; - case OSSL_LIB_CTX_PROPERTY_DEFN_INDEX: - return ctx->property_defns; - case OSSL_LIB_CTX_GLOBAL_PROPERTIES: - return ctx->global_properties; - case OSSL_LIB_CTX_DRBG_INDEX: - return ctx->drbg; - case OSSL_LIB_CTX_DRBG_NONCE_INDEX: - return ctx->drbg_nonce; -#ifndef FIPS_MODULE - case OSSL_LIB_CTX_PROVIDER_CONF_INDEX: - return ctx->provider_conf; - case OSSL_LIB_CTX_BIO_CORE_INDEX: - return ctx->bio_core; - case OSSL_LIB_CTX_CHILD_PROVIDER_INDEX: - return ctx->child_provider; - case OSSL_LIB_CTX_DECODER_STORE_INDEX: - return ctx->decoder_store; - case OSSL_LIB_CTX_DECODER_CACHE_INDEX: - return ctx->decoder_cache; - case OSSL_LIB_CTX_ENCODER_STORE_INDEX: - return ctx->encoder_store; - case OSSL_LIB_CTX_STORE_LOADER_STORE_INDEX: - return ctx->store_loader_store; - case OSSL_LIB_CTX_SELF_TEST_CB_INDEX: - return ctx->self_test_cb; -#endif -#ifndef OPENSSL_NO_THREAD_POOL - case OSSL_LIB_CTX_THREAD_INDEX: - return ctx->threads; -#endif + idx = ossl_crypto_get_ex_new_index_ex(ctx, CRYPTO_EX_INDEX_OSSL_LIB_CTX, 0, + (void *)meth, + ossl_lib_ctx_generic_new, + NULL, ossl_lib_ctx_generic_free, + meth->priority); + if (idx < 0) + return 0; - case OSSL_LIB_CTX_RAND_CRNGT_INDEX: { - - /* - * rand_crngt must be lazily initialized because it calls into - * libctx, so must not be called from context_init, else a deadlock - * will occur. - * - * We use a separate lock because code called by the instantiation - * of rand_crngt is liable to try and take the libctx lock. - */ - if (CRYPTO_THREAD_read_lock(ctx->rand_crngt_lock) != 1) - return NULL; + ctx->dyn_indexes[static_index] = idx; + return 1; +} - if (ctx->rand_crngt == NULL) { - CRYPTO_THREAD_unlock(ctx->rand_crngt_lock); +void *ossl_lib_ctx_get_data(OSSL_LIB_CTX *ctx, int index, + const OSSL_LIB_CTX_METHOD *meth) +{ + void *data = NULL; + int dynidx; - if (CRYPTO_THREAD_write_lock(ctx->rand_crngt_lock) != 1) - return NULL; + ctx = ossl_lib_ctx_get_concrete(ctx); + if (ctx == NULL) + return NULL; - if (ctx->rand_crngt == NULL) - ctx->rand_crngt = ossl_rand_crng_ctx_new(ctx); - } + if (!CRYPTO_THREAD_read_lock(ctx->lock)) + return NULL; + dynidx = ctx->dyn_indexes[index]; + CRYPTO_THREAD_unlock(ctx->lock); - p = ctx->rand_crngt; + if (dynidx != -1) { + if (!CRYPTO_THREAD_read_lock(ctx->index_locks[index])) + return NULL; + if (!CRYPTO_THREAD_read_lock(ctx->lock)) { + CRYPTO_THREAD_unlock(ctx->index_locks[index]); + return NULL; + } + data = CRYPTO_get_ex_data(&ctx->data, dynidx); + CRYPTO_THREAD_unlock(ctx->lock); + CRYPTO_THREAD_unlock(ctx->index_locks[index]); + return data; + } - CRYPTO_THREAD_unlock(ctx->rand_crngt_lock); + if (!CRYPTO_THREAD_write_lock(ctx->index_locks[index])) + return NULL; + if (!CRYPTO_THREAD_write_lock(ctx->lock)) { + CRYPTO_THREAD_unlock(ctx->index_locks[index]); + return NULL; + } - return p; + dynidx = ctx->dyn_indexes[index]; + if (dynidx != -1) { + data = CRYPTO_get_ex_data(&ctx->data, dynidx); + CRYPTO_THREAD_unlock(ctx->lock); + CRYPTO_THREAD_unlock(ctx->index_locks[index]); + return data; } -#ifdef FIPS_MODULE - case OSSL_LIB_CTX_THREAD_EVENT_HANDLER_INDEX: - return ctx->thread_event_handler; + if (!ossl_lib_ctx_init_index(ctx, index, meth)) { + CRYPTO_THREAD_unlock(ctx->lock); + CRYPTO_THREAD_unlock(ctx->index_locks[index]); + return NULL; + } - case OSSL_LIB_CTX_FIPS_PROV_INDEX: - return ctx->fips_prov; -#endif + CRYPTO_THREAD_unlock(ctx->lock); - default: - return NULL; + /* + * The alloc call ensures there's a value there. We release the ctx->lock + * for this, because the allocation itself may recursively call + * ossl_lib_ctx_get_data for other indexes (never this one). The allocation + * will itself aquire the ctx->lock when it actually comes to store the + * allocated data (see ossl_lib_ctx_generic_new() above). We call + * ossl_crypto_alloc_ex_data_intern() here instead of CRYPTO_alloc_ex_data(). + * They do the same thing except that the latter calls CRYPTO_get_ex_data() + * as well - which we must not do without holding the ctx->lock. + */ + if (ossl_crypto_alloc_ex_data_intern(CRYPTO_EX_INDEX_OSSL_LIB_CTX, NULL, + &ctx->data, ctx->dyn_indexes[index])) { + if (!CRYPTO_THREAD_read_lock(ctx->lock)) + goto end; + data = CRYPTO_get_ex_data(&ctx->data, ctx->dyn_indexes[index]); + CRYPTO_THREAD_unlock(ctx->lock); } + +end: + CRYPTO_THREAD_unlock(ctx->index_locks[index]); + return data; } OSSL_EX_DATA_GLOBAL *ossl_lib_ctx_get_ex_data_global(OSSL_LIB_CTX *ctx) @@ -640,6 +431,56 @@ OSSL_EX_DATA_GLOBAL *ossl_lib_ctx_get_ex_data_global(OSSL_LIB_CTX *ctx) return &ctx->global; } +int ossl_lib_ctx_run_once(OSSL_LIB_CTX *ctx, unsigned int idx, + ossl_lib_ctx_run_once_fn run_once_fn) +{ + int done = 0, ret = 0; + + ctx = ossl_lib_ctx_get_concrete(ctx); + if (ctx == NULL) + return 0; + + if (!CRYPTO_THREAD_read_lock(ctx->oncelock)) + return 0; + done = ctx->run_once_done[idx]; + if (done) + ret = ctx->run_once_ret[idx]; + CRYPTO_THREAD_unlock(ctx->oncelock); + + if (done) + return ret; + + if (!CRYPTO_THREAD_write_lock(ctx->oncelock)) + return 0; + if (ctx->run_once_done[idx]) { + ret = ctx->run_once_ret[idx]; + CRYPTO_THREAD_unlock(ctx->oncelock); + return ret; + } + + ret = run_once_fn(ctx); + ctx->run_once_done[idx] = 1; + ctx->run_once_ret[idx] = ret; + CRYPTO_THREAD_unlock(ctx->oncelock); + + return ret; +} + +int ossl_lib_ctx_onfree(OSSL_LIB_CTX *ctx, ossl_lib_ctx_onfree_fn onfreefn) +{ + struct ossl_lib_ctx_onfree_list_st *newonfree + = OPENSSL_malloc(sizeof(*newonfree)); + + if (newonfree == NULL) + return 0; + + newonfree->fn = onfreefn; + newonfree->next = ctx->onfreelist; + ctx->onfreelist = newonfree; + + return 1; +} + const char *ossl_lib_ctx_get_descriptor(OSSL_LIB_CTX *libctx) { #ifdef FIPS_MODULE diff --git a/openssl/src/crypto/core_algorithm.c b/openssl/src/crypto/core_algorithm.c index 16055bad3..5ff33eff7 100644 --- a/openssl/src/crypto/core_algorithm.c +++ b/openssl/src/crypto/core_algorithm.c @@ -16,92 +16,18 @@ struct algorithm_data_st { OSSL_LIB_CTX *libctx; int operation_id; /* May be zero for finding them all */ - int (*pre)(OSSL_PROVIDER *, int operation_id, int no_store, void *data, - int *result); - int (*reserve_store)(int no_store, void *data); + int (*pre)(OSSL_PROVIDER *, int operation_id, void *data, int *result); void (*fn)(OSSL_PROVIDER *, const OSSL_ALGORITHM *, int no_store, void *data); - int (*unreserve_store)(void *data); int (*post)(OSSL_PROVIDER *, int operation_id, int no_store, void *data, int *result); void *data; }; -/* - * Process one OSSL_ALGORITHM array, for the operation |cur_operation|, - * by constructing methods for all its implementations and adding those - * to the appropriate method store. - * Which method store is appropriate is given by |no_store| ("permanent" - * if 0, temporary if 1) and other data in |data->data|. - * - * Returns: - * -1 to quit adding algorithm implementations immediately - * 0 if not successful, but adding should continue - * 1 if successful so far, and adding should continue - */ -static int algorithm_do_map(OSSL_PROVIDER *provider, const OSSL_ALGORITHM *map, - int cur_operation, int no_store, void *cbdata) -{ - struct algorithm_data_st *data = cbdata; - int ret = 0; - - if (!data->reserve_store(no_store, data->data)) - /* Error, bail out! */ - return -1; - - /* Do we fulfill pre-conditions? */ - if (data->pre == NULL) { - /* If there is no pre-condition function, assume "yes" */ - ret = 1; - } else if (!data->pre(provider, cur_operation, no_store, data->data, - &ret)) { - /* Error, bail out! */ - ret = -1; - goto end; - } - - /* - * If pre-condition not fulfilled don't add this set of implementations, - * but do continue with the next. This simply means that another thread - * got to it first. - */ - if (ret == 0) { - ret = 1; - goto end; - } - - if (map != NULL) { - const OSSL_ALGORITHM *thismap; - - for (thismap = map; thismap->algorithm_names != NULL; thismap++) - data->fn(provider, thismap, no_store, data->data); - } - - /* Do we fulfill post-conditions? */ - if (data->post == NULL) { - /* If there is no post-condition function, assume "yes" */ - ret = 1; - } else if (!data->post(provider, cur_operation, no_store, data->data, - &ret)) { - /* Error, bail out! */ - ret = -1; - } - - end: - data->unreserve_store(data->data); - - return ret; -} - -/* - * Given a provider, process one operation given by |data->operation_id|, or - * if that's zero, process all known operations. - * For each such operation, query the associated OSSL_ALGORITHM array from - * the provider, then process that array with |algorithm_do_map()|. - */ static int algorithm_do_this(OSSL_PROVIDER *provider, void *cbdata) { struct algorithm_data_st *data = cbdata; + int no_store = 0; /* Assume caching is ok */ int first_operation = 1; int last_operation = OSSL_OP__HIGHEST; int cur_operation; @@ -113,18 +39,43 @@ static int algorithm_do_this(OSSL_PROVIDER *provider, void *cbdata) for (cur_operation = first_operation; cur_operation <= last_operation; cur_operation++) { - int no_store = 0; /* Assume caching is ok */ const OSSL_ALGORITHM *map = NULL; - int ret = 0; + int ret; + + /* Do we fulfill pre-conditions? */ + if (data->pre == NULL) { + /* If there is no pre-condition function, assume "yes" */ + ret = 1; + } else { + if (!data->pre(provider, cur_operation, data->data, &ret)) + /* Error, bail out! */ + return 0; + } + + /* If pre-condition not fulfilled, go to the next operation */ + if (!ret) + continue; map = ossl_provider_query_operation(provider, cur_operation, &no_store); - ret = algorithm_do_map(provider, map, cur_operation, no_store, data); + if (map != NULL) { + const OSSL_ALGORITHM *thismap; + + for (thismap = map; thismap->algorithm_names != NULL; thismap++) + data->fn(provider, thismap, no_store, data->data); + } ossl_provider_unquery_operation(provider, cur_operation, map); - if (ret < 0) - /* Hard error, bail out immediately! */ - return 0; + /* Do we fulfill post-conditions? */ + if (data->post == NULL) { + /* If there is no post-condition function, assume "yes" */ + ret = 1; + } else { + if (!data->post(provider, cur_operation, no_store, data->data, + &ret)) + /* Error, bail out! */ + return 0; + } /* If post-condition not fulfilled, set general failure */ if (!ret) @@ -137,12 +88,10 @@ static int algorithm_do_this(OSSL_PROVIDER *provider, void *cbdata) void ossl_algorithm_do_all(OSSL_LIB_CTX *libctx, int operation_id, OSSL_PROVIDER *provider, int (*pre)(OSSL_PROVIDER *, int operation_id, - int no_store, void *data, int *result), - int (*reserve_store)(int no_store, void *data), + void *data, int *result), void (*fn)(OSSL_PROVIDER *provider, const OSSL_ALGORITHM *algo, int no_store, void *data), - int (*unreserve_store)(void *data), int (*post)(OSSL_PROVIDER *, int operation_id, int no_store, void *data, int *result), void *data) @@ -152,9 +101,7 @@ void ossl_algorithm_do_all(OSSL_LIB_CTX *libctx, int operation_id, cbdata.libctx = libctx; cbdata.operation_id = operation_id; cbdata.pre = pre; - cbdata.reserve_store = reserve_store; cbdata.fn = fn; - cbdata.unreserve_store = unreserve_store; cbdata.post = post; cbdata.data = data; @@ -193,5 +140,7 @@ char *ossl_algorithm_get1_first_name(const OSSL_ALGORITHM *algo) first_name_len = first_name_end - algo->algorithm_names; ret = OPENSSL_strndup(algo->algorithm_names, first_name_len); + if (ret == NULL) + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return ret; } diff --git a/openssl/src/crypto/core_fetch.c b/openssl/src/crypto/core_fetch.c index d311158d7..367f6ba8a 100644 --- a/openssl/src/crypto/core_fetch.c +++ b/openssl/src/crypto/core_fetch.c @@ -24,53 +24,16 @@ struct construct_data_st { void *mcm_data; }; -static int is_temporary_method_store(int no_store, void *cbdata) -{ - struct construct_data_st *data = cbdata; - - return no_store && !data->force_store; -} - -static int ossl_method_construct_reserve_store(int no_store, void *cbdata) -{ - struct construct_data_st *data = cbdata; - - if (is_temporary_method_store(no_store, data) && data->store == NULL) { - /* - * If we have been told not to store the method "permanently", we - * ask for a temporary store, and store the method there. - * The owner of |data->mcm| is completely responsible for managing - * that temporary store. - */ - if ((data->store = data->mcm->get_tmp_store(data->mcm_data)) == NULL) - return 0; - } - - return data->mcm->lock_store(data->store, data->mcm_data); -} - -static int ossl_method_construct_unreserve_store(void *cbdata) -{ - struct construct_data_st *data = cbdata; - - return data->mcm->unlock_store(data->store, data->mcm_data); -} - static int ossl_method_construct_precondition(OSSL_PROVIDER *provider, - int operation_id, int no_store, - void *cbdata, int *result) + int operation_id, void *cbdata, + int *result) { if (!ossl_assert(result != NULL)) { ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); return 0; } - /* Assume that no bits are set */ - *result = 0; - - /* No flag bits for temporary stores */ - if (!is_temporary_method_store(no_store, cbdata) - && !ossl_provider_test_operation_bit(provider, operation_id, result)) + if (!ossl_provider_test_operation_bit(provider, operation_id, result)) return 0; /* @@ -93,9 +56,7 @@ static int ossl_method_construct_postcondition(OSSL_PROVIDER *provider, } *result = 1; - - /* No flag bits for temporary stores */ - return is_temporary_method_store(no_store, cbdata) + return no_store != 0 || ossl_provider_set_operation_bit(provider, operation_id); } @@ -120,8 +81,24 @@ static void ossl_method_construct_this(OSSL_PROVIDER *provider, * It is *expected* that the put function increments the refcnt * of the passed method. */ - data->mcm->put(data->store, method, provider, algo->algorithm_names, - algo->property_definition, data->mcm_data); + + if (data->force_store || !no_store) { + /* If we haven't been told not to store, add to the global store */ + data->mcm->put(NULL, method, provider, algo->algorithm_names, + algo->property_definition, data->mcm_data); + } else { + /* + * If we have been told not to store the method "permanently", we + * ask for a temporary store, and store the method there. + * The owner of |data->mcm| is completely responsible for managing + * that temporary store. + */ + if ((data->store = data->mcm->get_tmp_store(data->mcm_data)) == NULL) + return; + + data->mcm->put(data->store, method, provider, algo->algorithm_names, + algo->property_definition, data->mcm_data); + } /* refcnt-- because we're dropping the reference */ data->mcm->destruct(method, data->mcm_data); @@ -132,40 +109,31 @@ void *ossl_method_construct(OSSL_LIB_CTX *libctx, int operation_id, OSSL_METHOD_CONSTRUCT_METHOD *mcm, void *mcm_data) { void *method = NULL; - OSSL_PROVIDER *provider = provider_rw != NULL ? *provider_rw : NULL; - struct construct_data_st cbdata; - /* - * We might be tempted to try to look into the method store without - * constructing to see if we can find our method there already. - * Unfortunately that does not work well if the query contains - * optional properties as newly loaded providers can match them better. - * We trust that ossl_method_construct_precondition() and - * ossl_method_construct_postcondition() make sure that the - * ossl_algorithm_do_all() does very little when methods from - * a provider have already been constructed. - */ - - cbdata.store = NULL; - cbdata.force_store = force_store; - cbdata.mcm = mcm; - cbdata.mcm_data = mcm_data; - ossl_algorithm_do_all(libctx, operation_id, provider, - ossl_method_construct_precondition, - ossl_method_construct_reserve_store, - ossl_method_construct_this, - ossl_method_construct_unreserve_store, - ossl_method_construct_postcondition, - &cbdata); - - /* If there is a temporary store, try there first */ - if (cbdata.store != NULL) - method = mcm->get(cbdata.store, (const OSSL_PROVIDER **)provider_rw, - mcm_data); - - /* If no method was found yet, try the global store */ - if (method == NULL) - method = mcm->get(NULL, (const OSSL_PROVIDER **)provider_rw, mcm_data); + if ((method = mcm->get(NULL, (const OSSL_PROVIDER **)provider_rw, + mcm_data)) == NULL) { + OSSL_PROVIDER *provider = provider_rw != NULL ? *provider_rw : NULL; + struct construct_data_st cbdata; + + cbdata.store = NULL; + cbdata.force_store = force_store; + cbdata.mcm = mcm; + cbdata.mcm_data = mcm_data; + ossl_algorithm_do_all(libctx, operation_id, provider, + ossl_method_construct_precondition, + ossl_method_construct_this, + ossl_method_construct_postcondition, + &cbdata); + + /* If there is a temporary store, try there first */ + if (cbdata.store != NULL) + method = mcm->get(cbdata.store, (const OSSL_PROVIDER **)provider_rw, + mcm_data); + + /* If no method was found yet, try the global store */ + if (method == NULL) + method = mcm->get(NULL, (const OSSL_PROVIDER **)provider_rw, mcm_data); + } return method; } diff --git a/openssl/src/crypto/core_namemap.c b/openssl/src/crypto/core_namemap.c index 1dcf390fc..7e11ab1c8 100644 --- a/openssl/src/crypto/core_namemap.c +++ b/openssl/src/crypto/core_namemap.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,6 @@ #include "crypto/lhash.h" /* ossl_lh_strcasehash */ #include "internal/tsan_assist.h" #include "internal/sizes.h" -#include "crypto/context.h" /*- * The namenum entry @@ -23,7 +22,7 @@ typedef struct { int number; } NAMENUM_ENTRY; -DEFINE_LHASH_OF_EX(NAMENUM_ENTRY); +DEFINE_LHASH_OF(NAMENUM_ENTRY); /*- * The namemap itself @@ -61,7 +60,7 @@ static void namenum_free(NAMENUM_ENTRY *n) /* OSSL_LIB_CTX_METHOD functions for a namemap stored in a library context */ -void *ossl_stored_namemap_new(OSSL_LIB_CTX *libctx) +static void *stored_namemap_new(OSSL_LIB_CTX *libctx) { OSSL_NAMEMAP *namemap = ossl_namemap_new(); @@ -71,7 +70,7 @@ void *ossl_stored_namemap_new(OSSL_LIB_CTX *libctx) return namemap; } -void ossl_stored_namemap_free(void *vnamemap) +static void stored_namemap_free(void *vnamemap) { OSSL_NAMEMAP *namemap = vnamemap; @@ -82,6 +81,12 @@ void ossl_stored_namemap_free(void *vnamemap) } } +static const OSSL_LIB_CTX_METHOD stored_namemap_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + stored_namemap_new, + stored_namemap_free, +}; + /*- * API functions * ============= @@ -137,9 +142,6 @@ int ossl_namemap_doall_names(const OSSL_NAMEMAP *namemap, int number, cbdata.number = number; cbdata.found = 0; - if (namemap == NULL) - return 0; - /* * We collect all the names first under a read lock. Subsequently we call * the user function, so that we're not holding the read lock when in user @@ -169,20 +171,22 @@ int ossl_namemap_doall_names(const OSSL_NAMEMAP *namemap, int number, return 1; } -/* This function is not thread safe, the namemap must be locked */ -static int namemap_name2num(const OSSL_NAMEMAP *namemap, - const char *name) +static int namemap_name2num_n(const OSSL_NAMEMAP *namemap, + const char *name, size_t name_len) { NAMENUM_ENTRY *namenum_entry, namenum_tmpl; - namenum_tmpl.name = (char *)name; + if ((namenum_tmpl.name = OPENSSL_strndup(name, name_len)) == NULL) + return 0; namenum_tmpl.number = 0; namenum_entry = lh_NAMENUM_ENTRY_retrieve(namemap->namenum, &namenum_tmpl); + OPENSSL_free(namenum_tmpl.name); return namenum_entry != NULL ? namenum_entry->number : 0; } -int ossl_namemap_name2num(const OSSL_NAMEMAP *namemap, const char *name) +int ossl_namemap_name2num_n(const OSSL_NAMEMAP *namemap, + const char *name, size_t name_len) { int number; @@ -196,24 +200,18 @@ int ossl_namemap_name2num(const OSSL_NAMEMAP *namemap, const char *name) if (!CRYPTO_THREAD_read_lock(namemap->lock)) return 0; - number = namemap_name2num(namemap, name); + number = namemap_name2num_n(namemap, name, name_len); CRYPTO_THREAD_unlock(namemap->lock); return number; } -int ossl_namemap_name2num_n(const OSSL_NAMEMAP *namemap, - const char *name, size_t name_len) +int ossl_namemap_name2num(const OSSL_NAMEMAP *namemap, const char *name) { - char *tmp; - int ret; - - if (name == NULL || (tmp = OPENSSL_strndup(name, name_len)) == NULL) + if (name == NULL) return 0; - ret = ossl_namemap_name2num(namemap, tmp); - OPENSSL_free(tmp); - return ret; + return ossl_namemap_name2num_n(namemap, name, strlen(name)); } struct num2name_data_st { @@ -243,21 +241,18 @@ const char *ossl_namemap_num2name(const OSSL_NAMEMAP *namemap, int number, return data.name; } -/* This function is not thread safe, the namemap must be locked */ -static int namemap_add_name(OSSL_NAMEMAP *namemap, int number, - const char *name) +static int namemap_add_name_n(OSSL_NAMEMAP *namemap, int number, + const char *name, size_t name_len) { NAMENUM_ENTRY *namenum = NULL; int tmp_number; /* If it already exists, we don't add it */ - if ((tmp_number = namemap_name2num(namemap, name)) != 0) + if ((tmp_number = namemap_name2num_n(namemap, name, name_len)) != 0) return tmp_number; - if ((namenum = OPENSSL_zalloc(sizeof(*namenum))) == NULL) - return 0; - - if ((namenum->name = OPENSSL_strdup(name)) == NULL) + if ((namenum = OPENSSL_zalloc(sizeof(*namenum))) == NULL + || (namenum->name = OPENSSL_strndup(name, name_len)) == NULL) goto err; /* The tsan_counter use here is safe since we're under lock */ @@ -274,8 +269,8 @@ static int namemap_add_name(OSSL_NAMEMAP *namemap, int number, return 0; } -int ossl_namemap_add_name(OSSL_NAMEMAP *namemap, int number, - const char *name) +int ossl_namemap_add_name_n(OSSL_NAMEMAP *namemap, int number, + const char *name, size_t name_len) { int tmp_number; @@ -284,20 +279,29 @@ int ossl_namemap_add_name(OSSL_NAMEMAP *namemap, int number, namemap = ossl_namemap_stored(NULL); #endif - if (name == NULL || *name == 0 || namemap == NULL) + if (name == NULL || name_len == 0 || namemap == NULL) return 0; if (!CRYPTO_THREAD_write_lock(namemap->lock)) return 0; - tmp_number = namemap_add_name(namemap, number, name); + tmp_number = namemap_add_name_n(namemap, number, name, name_len); CRYPTO_THREAD_unlock(namemap->lock); return tmp_number; } +int ossl_namemap_add_name(OSSL_NAMEMAP *namemap, int number, const char *name) +{ + if (name == NULL) + return 0; + + return ossl_namemap_add_name_n(namemap, number, name, strlen(name)); +} + int ossl_namemap_add_names(OSSL_NAMEMAP *namemap, int number, const char *names, const char separator) { - char *tmp, *p, *q, *endp; + const char *p, *q; + size_t l; /* Check that we have a namemap */ if (!ossl_assert(namemap != NULL)) { @@ -305,71 +309,62 @@ int ossl_namemap_add_names(OSSL_NAMEMAP *namemap, int number, return 0; } - if ((tmp = OPENSSL_strdup(names)) == NULL) - return 0; - - if (!CRYPTO_THREAD_write_lock(namemap->lock)) { - OPENSSL_free(tmp); + if (!CRYPTO_THREAD_write_lock(namemap->lock)) return 0; - } /* * Check that no name is an empty string, and that all names have at * most one numeric identity together. */ - for (p = tmp; *p != '\0'; p = q) { + for (p = names; *p != '\0'; p = (q == NULL ? p + l : q + 1)) { int this_number; - size_t l; - if ((q = strchr(p, separator)) == NULL) { + if ((q = strchr(p, separator)) == NULL) l = strlen(p); /* offset to \0 */ - q = p + l; - } else { + else l = q - p; /* offset to the next separator */ - *q++ = '\0'; - } - if (*p == '\0') { + this_number = namemap_name2num_n(namemap, p, l); + + if (*p == '\0' || *p == separator) { ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_BAD_ALGORITHM_NAME); - number = 0; - goto end; + goto err; } - - this_number = namemap_name2num(namemap, p); - if (number == 0) { number = this_number; } else if (this_number != 0 && this_number != number) { ERR_raise_data(ERR_LIB_CRYPTO, CRYPTO_R_CONFLICTING_NAMES, - "\"%s\" has an existing different identity %d (from \"%s\")", - p, this_number, names); - number = 0; - goto end; + "\"%.*s\" has an existing different identity %d (from \"%s\")", + l, p, this_number, names); + goto err; } } - endp = p; /* Now that we have checked, register all names */ - for (p = tmp; p < endp; p = q) { + for (p = names; *p != '\0'; p = (q == NULL ? p + l : q + 1)) { int this_number; - q = p + strlen(p) + 1; + if ((q = strchr(p, separator)) == NULL) + l = strlen(p); /* offset to \0 */ + else + l = q - p; /* offset to the next separator */ - this_number = namemap_add_name(namemap, number, p); + this_number = namemap_add_name_n(namemap, number, p, l); if (number == 0) { number = this_number; } else if (this_number != number) { ERR_raise_data(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR, "Got number %d when expecting %d", this_number, number); - number = 0; - goto end; + goto err; } } - end: CRYPTO_THREAD_unlock(namemap->lock); - OPENSSL_free(tmp); return number; + + err: + CRYPTO_THREAD_unlock(namemap->lock); + return 0; } /*- @@ -473,7 +468,8 @@ OSSL_NAMEMAP *ossl_namemap_stored(OSSL_LIB_CTX *libctx) int nms; #endif OSSL_NAMEMAP *namemap = - ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_NAMEMAP_INDEX); + ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_NAMEMAP_INDEX, + &stored_namemap_method); if (namemap == NULL) return NULL; diff --git a/openssl/src/crypto/cpt_err.c b/openssl/src/crypto/cpt_err.c index 02d631466..8574f31a8 100644 --- a/openssl/src/crypto/cpt_err.c +++ b/openssl/src/crypto/cpt_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -29,32 +29,14 @@ static const ERR_STRING_DATA CRYPTO_str_reasons[] = { "insufficient param size"}, {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_INSUFFICIENT_SECURE_DATA_SPACE), "insufficient secure data space"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_INTEGER_OVERFLOW), - "integer overflow"}, {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_INVALID_NEGATIVE_VALUE), "invalid negative value"}, {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_INVALID_NULL_ARGUMENT), "invalid null argument"}, {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_INVALID_OSSL_PARAM_TYPE), "invalid ossl param type"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_NO_PARAMS_TO_MERGE), - "no params to merge"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_NO_SPACE_FOR_TERMINATING_NULL), - "no space for terminating null"}, {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_ODD_NUMBER_OF_DIGITS), "odd number of digits"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_PARAM_CANNOT_BE_REPRESENTED_EXACTLY), - "param cannot be represented exactly"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_PARAM_NOT_INTEGER_TYPE), - "param not integer type"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_PARAM_OF_INCOMPATIBLE_TYPE), - "param of incompatible type"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_PARAM_UNSIGNED_INTEGER_NEGATIVE_VALUE_UNSUPPORTED), - "param unsigned integer negative value unsupported"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_PARAM_UNSUPPORTED_FLOATING_POINT_FORMAT), - "param unsupported floating point format"}, - {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_PARAM_VALUE_TOO_LARGE_FOR_DESTINATION), - "param value too large for destination"}, {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_PROVIDER_ALREADY_EXISTS), "provider already exists"}, {ERR_PACK(ERR_LIB_CRYPTO, 0, CRYPTO_R_PROVIDER_SECTION_ERROR), diff --git a/openssl/src/crypto/cpuid.c b/openssl/src/crypto/cpuid.c index 51cbe5ea0..21b2a59b4 100644 --- a/openssl/src/crypto/cpuid.c +++ b/openssl/src/crypto/cpuid.c @@ -1,5 +1,5 @@ /* - * Copyright 1998-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1998-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -34,7 +34,7 @@ static variant_char *ossl_getenv(const char *name) { /* * Since we pull only one environment variable, it's simpler to - * just ignore |name| and use equivalent wide-char L-literal. + * to just ignore |name| and use equivalent wide-char L-literal. * As well as to ignore excessively long values... */ static WCHAR value[48]; @@ -71,7 +71,7 @@ static uint64_t ossl_strtouint64(const variant_char *str) base = 16, str++; } - while ((digit = todigit(*str++)) < base) + while((digit = todigit(*str++)) < base) ret = ret * base + digit; return ret; @@ -80,7 +80,7 @@ static uint64_t ossl_strtouint64(const variant_char *str) static variant_char *ossl_strchr(const variant_char *str, char srch) { variant_char c; - while ((c = *str)) { + while((c = *str)) { if (c == srch) return (variant_char *)str; str++; @@ -173,7 +173,7 @@ void OPENSSL_cpuid_setup(void) */ /* - * The volatile is used to ensure that the compiler generates code that reads + * The volatile is used to to ensure that the compiler generates code that reads * all values from the array and doesn't try to optimize this away. The standard * doesn't actually require this behavior if the original data pointed to is * not volatile, but compilers do this in practice anyway. @@ -181,7 +181,7 @@ void OPENSSL_cpuid_setup(void) * There are also assembler versions of this function. */ # undef CRYPTO_memcmp -int CRYPTO_memcmp(const void *in_a, const void *in_b, size_t len) +int CRYPTO_memcmp(const void * in_a, const void * in_b, size_t len) { size_t i; const volatile unsigned char *a = in_a; diff --git a/openssl/src/crypto/crmf/crmf_asn.c b/openssl/src/crypto/crmf/crmf_asn.c index 85b421393..3354b8973 100644 --- a/openssl/src/crypto/crmf/crmf_asn.c +++ b/openssl/src/crypto/crmf/crmf_asn.c @@ -26,14 +26,14 @@ ASN1_SEQUENCE(OSSL_CRMF_PRIVATEKEYINFO) = { } ASN1_SEQUENCE_END(OSSL_CRMF_PRIVATEKEYINFO) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_PRIVATEKEYINFO) + ASN1_CHOICE(OSSL_CRMF_ENCKEYWITHID_IDENTIFIER) = { - ASN1_SIMPLE(OSSL_CRMF_ENCKEYWITHID_IDENTIFIER, - value.string, ASN1_UTF8STRING), - ASN1_SIMPLE(OSSL_CRMF_ENCKEYWITHID_IDENTIFIER, - value.generalName, GENERAL_NAME) + ASN1_SIMPLE(OSSL_CRMF_ENCKEYWITHID_IDENTIFIER, value.string, ASN1_UTF8STRING), + ASN1_SIMPLE(OSSL_CRMF_ENCKEYWITHID_IDENTIFIER, value.generalName, GENERAL_NAME) } ASN1_CHOICE_END(OSSL_CRMF_ENCKEYWITHID_IDENTIFIER) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_ENCKEYWITHID_IDENTIFIER) + ASN1_SEQUENCE(OSSL_CRMF_ENCKEYWITHID) = { ASN1_SIMPLE(OSSL_CRMF_ENCKEYWITHID, privateKey, OSSL_CRMF_PRIVATEKEYINFO), ASN1_OPT(OSSL_CRMF_ENCKEYWITHID, identifier, @@ -41,6 +41,7 @@ ASN1_SEQUENCE(OSSL_CRMF_ENCKEYWITHID) = { } ASN1_SEQUENCE_END(OSSL_CRMF_ENCKEYWITHID) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_ENCKEYWITHID) + ASN1_SEQUENCE(OSSL_CRMF_CERTID) = { ASN1_SIMPLE(OSSL_CRMF_CERTID, issuer, GENERAL_NAME), ASN1_SIMPLE(OSSL_CRMF_CERTID, serialNumber, ASN1_INTEGER) @@ -48,6 +49,7 @@ ASN1_SEQUENCE(OSSL_CRMF_CERTID) = { IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_CERTID) IMPLEMENT_ASN1_DUP_FUNCTION(OSSL_CRMF_CERTID) + ASN1_SEQUENCE(OSSL_CRMF_ENCRYPTEDVALUE) = { ASN1_IMP_OPT(OSSL_CRMF_ENCRYPTEDVALUE, intendedAlg, X509_ALGOR, 0), ASN1_IMP_OPT(OSSL_CRMF_ENCRYPTEDVALUE, symmAlg, X509_ALGOR, 1), @@ -64,6 +66,7 @@ ASN1_SEQUENCE(OSSL_CRMF_SINGLEPUBINFO) = { } ASN1_SEQUENCE_END(OSSL_CRMF_SINGLEPUBINFO) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_SINGLEPUBINFO) + ASN1_SEQUENCE(OSSL_CRMF_PKIPUBLICATIONINFO) = { ASN1_SIMPLE(OSSL_CRMF_PKIPUBLICATIONINFO, action, ASN1_INTEGER), ASN1_SEQUENCE_OF_OPT(OSSL_CRMF_PKIPUBLICATIONINFO, pubInfos, @@ -72,22 +75,24 @@ ASN1_SEQUENCE(OSSL_CRMF_PKIPUBLICATIONINFO) = { IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_PKIPUBLICATIONINFO) IMPLEMENT_ASN1_DUP_FUNCTION(OSSL_CRMF_PKIPUBLICATIONINFO) + ASN1_SEQUENCE(OSSL_CRMF_PKMACVALUE) = { ASN1_SIMPLE(OSSL_CRMF_PKMACVALUE, algId, X509_ALGOR), ASN1_SIMPLE(OSSL_CRMF_PKMACVALUE, value, ASN1_BIT_STRING) } ASN1_SEQUENCE_END(OSSL_CRMF_PKMACVALUE) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_PKMACVALUE) + ASN1_CHOICE(OSSL_CRMF_POPOPRIVKEY) = { ASN1_IMP(OSSL_CRMF_POPOPRIVKEY, value.thisMessage, ASN1_BIT_STRING, 0), ASN1_IMP(OSSL_CRMF_POPOPRIVKEY, value.subsequentMessage, ASN1_INTEGER, 1), ASN1_IMP(OSSL_CRMF_POPOPRIVKEY, value.dhMAC, ASN1_BIT_STRING, 2), ASN1_IMP(OSSL_CRMF_POPOPRIVKEY, value.agreeMAC, OSSL_CRMF_PKMACVALUE, 3), ASN1_IMP(OSSL_CRMF_POPOPRIVKEY, value.encryptedKey, ASN1_NULL, 4), - /* When supported, ASN1_NULL needs to be replaced by CMS_ENVELOPEDDATA */ } ASN1_CHOICE_END(OSSL_CRMF_POPOPRIVKEY) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_POPOPRIVKEY) + ASN1_SEQUENCE(OSSL_CRMF_PBMPARAMETER) = { ASN1_SIMPLE(OSSL_CRMF_PBMPARAMETER, salt, ASN1_OCTET_STRING), ASN1_SIMPLE(OSSL_CRMF_PBMPARAMETER, owf, X509_ALGOR), @@ -96,6 +101,7 @@ ASN1_SEQUENCE(OSSL_CRMF_PBMPARAMETER) = { } ASN1_SEQUENCE_END(OSSL_CRMF_PBMPARAMETER) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_PBMPARAMETER) + ASN1_CHOICE(OSSL_CRMF_POPOSIGNINGKEYINPUT_AUTHINFO) = { ASN1_EXP(OSSL_CRMF_POPOSIGNINGKEYINPUT_AUTHINFO, value.sender, GENERAL_NAME, 0), @@ -104,6 +110,7 @@ ASN1_CHOICE(OSSL_CRMF_POPOSIGNINGKEYINPUT_AUTHINFO) = { } ASN1_CHOICE_END(OSSL_CRMF_POPOSIGNINGKEYINPUT_AUTHINFO) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_POPOSIGNINGKEYINPUT_AUTHINFO) + ASN1_SEQUENCE(OSSL_CRMF_POPOSIGNINGKEYINPUT) = { ASN1_SIMPLE(OSSL_CRMF_POPOSIGNINGKEYINPUT, authInfo, OSSL_CRMF_POPOSIGNINGKEYINPUT_AUTHINFO), @@ -111,6 +118,7 @@ ASN1_SEQUENCE(OSSL_CRMF_POPOSIGNINGKEYINPUT) = { } ASN1_SEQUENCE_END(OSSL_CRMF_POPOSIGNINGKEYINPUT) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_POPOSIGNINGKEYINPUT) + ASN1_SEQUENCE(OSSL_CRMF_POPOSIGNINGKEY) = { ASN1_IMP_OPT(OSSL_CRMF_POPOSIGNINGKEY, poposkInput, OSSL_CRMF_POPOSIGNINGKEYINPUT, 0), @@ -119,6 +127,7 @@ ASN1_SEQUENCE(OSSL_CRMF_POPOSIGNINGKEY) = { } ASN1_SEQUENCE_END(OSSL_CRMF_POPOSIGNINGKEY) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_POPOSIGNINGKEY) + ASN1_CHOICE(OSSL_CRMF_POPO) = { ASN1_IMP(OSSL_CRMF_POPO, value.raVerified, ASN1_NULL, 0), ASN1_IMP(OSSL_CRMF_POPO, value.signature, OSSL_CRMF_POPOSIGNINGKEY, 1), @@ -127,6 +136,7 @@ ASN1_CHOICE(OSSL_CRMF_POPO) = { } ASN1_CHOICE_END(OSSL_CRMF_POPO) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_POPO) + ASN1_ADB_TEMPLATE(attributetypeandvalue_default) = ASN1_OPT(OSSL_CRMF_ATTRIBUTETYPEANDVALUE, value.other, ASN1_ANY); ASN1_ADB(OSSL_CRMF_ATTRIBUTETYPEANDVALUE) = { @@ -155,6 +165,7 @@ ASN1_ADB(OSSL_CRMF_ATTRIBUTETYPEANDVALUE) = { } ASN1_ADB_END(OSSL_CRMF_ATTRIBUTETYPEANDVALUE, 0, type, 0, &attributetypeandvalue_default_tt, NULL); + ASN1_SEQUENCE(OSSL_CRMF_ATTRIBUTETYPEANDVALUE) = { ASN1_SIMPLE(OSSL_CRMF_ATTRIBUTETYPEANDVALUE, type, ASN1_OBJECT), ASN1_ADB_OBJECT(OSSL_CRMF_ATTRIBUTETYPEANDVALUE) @@ -163,12 +174,14 @@ ASN1_SEQUENCE(OSSL_CRMF_ATTRIBUTETYPEANDVALUE) = { IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_ATTRIBUTETYPEANDVALUE) IMPLEMENT_ASN1_DUP_FUNCTION(OSSL_CRMF_ATTRIBUTETYPEANDVALUE) + ASN1_SEQUENCE(OSSL_CRMF_OPTIONALVALIDITY) = { ASN1_EXP_OPT(OSSL_CRMF_OPTIONALVALIDITY, notBefore, ASN1_TIME, 0), ASN1_EXP_OPT(OSSL_CRMF_OPTIONALVALIDITY, notAfter, ASN1_TIME, 1) } ASN1_SEQUENCE_END(OSSL_CRMF_OPTIONALVALIDITY) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_OPTIONALVALIDITY) + ASN1_SEQUENCE(OSSL_CRMF_CERTTEMPLATE) = { ASN1_IMP_OPT(OSSL_CRMF_CERTTEMPLATE, version, ASN1_INTEGER, 0), /* @@ -195,6 +208,7 @@ ASN1_SEQUENCE(OSSL_CRMF_CERTTEMPLATE) = { } ASN1_SEQUENCE_END(OSSL_CRMF_CERTTEMPLATE) IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_CERTTEMPLATE) + ASN1_SEQUENCE(OSSL_CRMF_CERTREQUEST) = { ASN1_SIMPLE(OSSL_CRMF_CERTREQUEST, certReqId, ASN1_INTEGER), ASN1_SIMPLE(OSSL_CRMF_CERTREQUEST, certTemplate, OSSL_CRMF_CERTTEMPLATE), @@ -204,6 +218,7 @@ ASN1_SEQUENCE(OSSL_CRMF_CERTREQUEST) = { IMPLEMENT_ASN1_FUNCTIONS(OSSL_CRMF_CERTREQUEST) IMPLEMENT_ASN1_DUP_FUNCTION(OSSL_CRMF_CERTREQUEST) + ASN1_SEQUENCE(OSSL_CRMF_MSG) = { ASN1_SIMPLE(OSSL_CRMF_MSG, certReq, OSSL_CRMF_CERTREQUEST), ASN1_OPT(OSSL_CRMF_MSG, popo, OSSL_CRMF_POPO), diff --git a/openssl/src/crypto/crmf/crmf_lib.c b/openssl/src/crypto/crmf/crmf_lib.c index cb077e41d..ea1e2dda0 100644 --- a/openssl/src/crypto/crmf/crmf_lib.c +++ b/openssl/src/crypto/crmf/crmf_lib.c @@ -1,5 +1,5 @@ /*- - * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2018 * Copyright Siemens AG 2015-2019 * @@ -29,9 +29,8 @@ #include #include "crmf_local.h" +#include "internal/constant_time.h" #include "internal/sizes.h" -#include "crypto/evp.h" -#include "crypto/x509.h" /* explicit #includes not strictly needed since implied by the above: */ #include @@ -81,6 +80,7 @@ int OSSL_CRMF_MSG_set1_##ctrlinf##_##atyp(OSSL_CRMF_MSG *msg, const valt *in) \ return 0; \ } + /*- * Pushes the given control attribute into the controls stack of a CertRequest * (section 6) @@ -243,6 +243,7 @@ IMPLEMENT_CRMF_CTRL_FUNC(utf8Pairs, ASN1_UTF8STRING, regInfo) /* id-regInfo-certReq to regInfo (section 7.2) */ IMPLEMENT_CRMF_CTRL_FUNC(certReq, OSSL_CRMF_CERTREQUEST, regInfo) + /* retrieves the certificate template of crm */ OSSL_CRMF_CERTTEMPLATE *OSSL_CRMF_MSG_get0_tmpl(const OSSL_CRMF_MSG *crm) { @@ -253,6 +254,7 @@ OSSL_CRMF_CERTTEMPLATE *OSSL_CRMF_MSG_get0_tmpl(const OSSL_CRMF_MSG *crm) return crm->certReq->certTemplate; } + int OSSL_CRMF_MSG_set0_validity(OSSL_CRMF_MSG *crm, ASN1_TIME *notBefore, ASN1_TIME *notAfter) { @@ -272,6 +274,7 @@ int OSSL_CRMF_MSG_set0_validity(OSSL_CRMF_MSG *crm, return 1; } + int OSSL_CRMF_MSG_set_certReqId(OSSL_CRMF_MSG *crm, int rid) { if (crm == NULL || crm->certReq == NULL || crm->certReq->certReqId == NULL) { @@ -311,6 +314,7 @@ int OSSL_CRMF_MSG_get_certReqId(const OSSL_CRMF_MSG *crm) return crmf_asn1_get_int(crm->certReq->certReqId); } + int OSSL_CRMF_MSG_set0_extensions(OSSL_CRMF_MSG *crm, X509_EXTENSIONS *exts) { @@ -331,6 +335,7 @@ int OSSL_CRMF_MSG_set0_extensions(OSSL_CRMF_MSG *crm, return 1; } + int OSSL_CRMF_MSG_push0_extension(OSSL_CRMF_MSG *crm, X509_EXTENSION *ext) { @@ -364,33 +369,22 @@ static int create_popo_signature(OSSL_CRMF_POPOSIGNINGKEY *ps, EVP_PKEY *pkey, const EVP_MD *digest, OSSL_LIB_CTX *libctx, const char *propq) { - char name[80] = ""; - EVP_PKEY *pub; - if (ps == NULL || cr == NULL || pkey == NULL) { ERR_raise(ERR_LIB_CRMF, CRMF_R_NULL_ARGUMENT); return 0; } - pub = X509_PUBKEY_get0(cr->certTemplate->publicKey); - if (!ossl_x509_check_private_key(pub, pkey)) - return 0; - if (ps->poposkInput != NULL) { /* We do not support cases 1+2 defined in RFC 4211, section 4.1 */ ERR_raise(ERR_LIB_CRMF, CRMF_R_POPOSKINPUT_NOT_SUPPORTED); return 0; } - if (EVP_PKEY_get_default_digest_name(pkey, name, sizeof(name)) > 0 - && strcmp(name, "UNDEF") == 0) /* at least for Ed25519, Ed448 */ - digest = NULL; - return ASN1_item_sign_ex(ASN1_ITEM_rptr(OSSL_CRMF_CERTREQUEST), - ps->algorithmIdentifier, /* sets this X509_ALGOR */ - NULL, ps->signature, /* sets the ASN1_BIT_STRING */ - cr, NULL, pkey, digest, libctx, propq); + ps->algorithmIdentifier, NULL, ps->signature, cr, + NULL, pkey, digest, libctx, propq); } + int OSSL_CRMF_MSG_create_popo(int meth, OSSL_CRMF_MSG *crm, EVP_PKEY *pkey, const EVP_MD *digest, OSSL_LIB_CTX *libctx, const char *propq) @@ -506,12 +500,6 @@ int OSSL_CRMF_MSGS_verify_popo(const OSSL_CRMF_MSGS *reqs, ERR_raise(ERR_LIB_CRMF, CRMF_R_POPO_INCONSISTENT_PUBLIC_KEY); return 0; } - - /* - * Should check at this point the contents of the authInfo sub-field - * as requested in FR #19807 according to RFC 4211 section 4.1. - */ - it = ASN1_ITEM_rptr(OSSL_CRMF_POPOSIGNINGKEYINPUT); asn = sig->poposkInput; } else { @@ -528,12 +516,6 @@ int OSSL_CRMF_MSGS_verify_popo(const OSSL_CRMF_MSGS *reqs, return 0; break; case OSSL_CRMF_POPO_KEYENC: - /* - * When OSSL_CMP_certrep_new() supports encrypted certs, - * should return 1 if the type of req->popo->value.keyEncipherment - * is OSSL_CRMF_POPOPRIVKEY_SUBSEQUENTMESSAGE and - * its value.subsequentMessage == OSSL_CRMF_SUBSEQUENTMESSAGE_ENCRCERT - */ case OSSL_CRMF_POPO_KEYAGREE: default: ERR_raise(ERR_LIB_CRMF, CRMF_R_UNSUPPORTED_POPO_METHOD); @@ -542,12 +524,7 @@ int OSSL_CRMF_MSGS_verify_popo(const OSSL_CRMF_MSGS *reqs, return 1; } -X509_PUBKEY -*OSSL_CRMF_CERTTEMPLATE_get0_publicKey(const OSSL_CRMF_CERTTEMPLATE *tmpl) -{ - return tmpl != NULL ? tmpl->publicKey : NULL; -} - +/* retrieves the serialNumber of the given cert template or NULL on error */ const ASN1_INTEGER *OSSL_CRMF_CERTTEMPLATE_get0_serialNumber(const OSSL_CRMF_CERTTEMPLATE *tmpl) { @@ -555,38 +532,40 @@ const ASN1_INTEGER } const X509_NAME -*OSSL_CRMF_CERTTEMPLATE_get0_subject(const OSSL_CRMF_CERTTEMPLATE *tmpl) + *OSSL_CRMF_CERTTEMPLATE_get0_subject(const OSSL_CRMF_CERTTEMPLATE *tmpl) { return tmpl != NULL ? tmpl->subject : NULL; } +/* retrieves the issuer name of the given cert template or NULL on error */ const X509_NAME -*OSSL_CRMF_CERTTEMPLATE_get0_issuer(const OSSL_CRMF_CERTTEMPLATE *tmpl) + *OSSL_CRMF_CERTTEMPLATE_get0_issuer(const OSSL_CRMF_CERTTEMPLATE *tmpl) { return tmpl != NULL ? tmpl->issuer : NULL; } X509_EXTENSIONS -*OSSL_CRMF_CERTTEMPLATE_get0_extensions(const OSSL_CRMF_CERTTEMPLATE *tmpl) + *OSSL_CRMF_CERTTEMPLATE_get0_extensions(const OSSL_CRMF_CERTTEMPLATE *tmpl) { return tmpl != NULL ? tmpl->extensions : NULL; } +/* retrieves the issuer name of the given CertId or NULL on error */ const X509_NAME *OSSL_CRMF_CERTID_get0_issuer(const OSSL_CRMF_CERTID *cid) { return cid != NULL && cid->issuer->type == GEN_DIRNAME ? cid->issuer->d.directoryName : NULL; } -const ASN1_INTEGER *OSSL_CRMF_CERTID_get0_serialNumber(const OSSL_CRMF_CERTID - *cid) +/* retrieves the serialNumber of the given CertId or NULL on error */ +const ASN1_INTEGER *OSSL_CRMF_CERTID_get0_serialNumber(const OSSL_CRMF_CERTID *cid) { return cid != NULL ? cid->serialNumber : NULL; } /*- - * Fill in the certificate template |tmpl|. - * Any other NULL argument will leave the respective field unchanged. + * fill in certificate template. + * Any value argument that is NULL will leave the respective field unchanged. */ int OSSL_CRMF_CERTTEMPLATE_fill(OSSL_CRMF_CERTTEMPLATE *tmpl, EVP_PKEY *pubkey, @@ -612,6 +591,7 @@ int OSSL_CRMF_CERTTEMPLATE_fill(OSSL_CRMF_CERTTEMPLATE *tmpl, return 1; } + /*- * Decrypts the certificate in the given encryptedValue using private key pkey. * This is needed for the indirect PoP method as in RFC 4210 section 5.2.8.2. @@ -662,12 +642,28 @@ X509 cikeysize = EVP_CIPHER_get_key_length(cipher); /* first the symmetric key needs to be decrypted */ pkctx = EVP_PKEY_CTX_new_from_pkey(libctx, pkey, propq); - if (pkctx == NULL || EVP_PKEY_decrypt_init(pkctx) <= 0 - || evp_pkey_decrypt_alloc(pkctx, &ek, &eksize, (size_t)cikeysize, - ecert->encSymmKey->data, - ecert->encSymmKey->length) <= 0) + if (pkctx != NULL && EVP_PKEY_decrypt_init(pkctx)) { + ASN1_BIT_STRING *encKey = ecert->encSymmKey; + size_t failure; + int retval; + + if (EVP_PKEY_decrypt(pkctx, NULL, &eksize, + encKey->data, encKey->length) <= 0 + || (ek = OPENSSL_malloc(eksize)) == NULL) + goto end; + retval = EVP_PKEY_decrypt(pkctx, ek, &eksize, + encKey->data, encKey->length); + ERR_clear_error(); /* error state may have sensitive information */ + failure = ~constant_time_is_zero_s(constant_time_msb(retval) + | constant_time_is_zero(retval)); + failure |= ~constant_time_eq_s(eksize, (size_t)cikeysize); + if (failure) { + ERR_raise(ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_SYMMETRIC_KEY); + goto end; + } + } else { goto end; - + } if ((iv = OPENSSL_malloc(EVP_CIPHER_get_iv_length(cipher))) == NULL) goto end; if (ASN1_TYPE_get_octetstring(ecert->symmAlg->parameter, iv, diff --git a/openssl/src/crypto/crmf/crmf_local.h b/openssl/src/crypto/crmf/crmf_local.h index e8937b423..3b8c3701b 100644 --- a/openssl/src/crypto/crmf/crmf_local.h +++ b/openssl/src/crypto/crmf/crmf_local.h @@ -126,6 +126,7 @@ struct ossl_crmf_singlepubinfo_st { DEFINE_STACK_OF(OSSL_CRMF_SINGLEPUBINFO) typedef STACK_OF(OSSL_CRMF_SINGLEPUBINFO) OSSL_CRMF_PUBINFOS; + /*- * PKIPublicationInfo ::= SEQUENCE { * action INTEGER { @@ -188,7 +189,6 @@ typedef struct ossl_crmf_popoprivkey_st { ASN1_BIT_STRING *dhMAC; /* 2 */ /* Deprecated */ OSSL_CRMF_PKMACVALUE *agreeMAC; /* 3 */ ASN1_NULL *encryptedKey; /* 4 */ - /* When supported, ASN1_NULL needs to be replaced by CMS_ENVELOPEDDATA */ } value; } OSSL_CRMF_POPOPRIVKEY; DECLARE_ASN1_FUNCTIONS(OSSL_CRMF_POPOPRIVKEY) @@ -330,7 +330,7 @@ struct ossl_crmf_certtemplate_st { struct ossl_crmf_certrequest_st { ASN1_INTEGER *certReqId; OSSL_CRMF_CERTTEMPLATE *certTemplate; - STACK_OF(OSSL_CRMF_ATTRIBUTETYPEANDVALUE /* Controls expanded */) *controls; + STACK_OF(OSSL_CRMF_ATTRIBUTETYPEANDVALUE) *controls; } /* OSSL_CRMF_CERTREQUEST */; DECLARE_ASN1_FUNCTIONS(OSSL_CRMF_CERTREQUEST) DECLARE_ASN1_DUP_FUNCTION(OSSL_CRMF_CERTREQUEST) diff --git a/openssl/src/crypto/crmf/crmf_pbm.c b/openssl/src/crypto/crmf/crmf_pbm.c index d4c7af38c..88a8480cf 100644 --- a/openssl/src/crypto/crmf/crmf_pbm.c +++ b/openssl/src/crypto/crmf/crmf_pbm.c @@ -11,6 +11,7 @@ * CRMF implementation by Martin Peylo, Miikka Viljanen, and David von Oheimb. */ + #include #include @@ -21,6 +22,7 @@ #include #include #include +#include #include #include @@ -123,7 +125,6 @@ OSSL_CRMF_PBMPARAMETER *OSSL_CRMF_pbmp_new(OSSL_LIB_CTX *libctx, size_t slen, * |outlen| if not NULL, will set variable to the length of the mac on success * returns 1 on success, 0 on error */ -/* could be combined with other MAC calculations in the library */ int OSSL_CRMF_pbm_new(OSSL_LIB_CTX *libctx, const char *propq, const OSSL_CRMF_PBMPARAMETER *pbmp, const unsigned char *msg, size_t msglen, @@ -200,11 +201,10 @@ int OSSL_CRMF_pbm_new(OSSL_LIB_CTX *libctx, const char *propq, if (!EVP_PBE_find(EVP_PBE_TYPE_PRF, mac_nid, NULL, &hmac_md_nid, NULL) || OBJ_obj2txt(hmac_mdname, sizeof(hmac_mdname), - OBJ_nid2obj(hmac_md_nid), 0) <= 0) { + OBJ_nid2obj(hmac_md_nid), 0) <= 0) { ERR_raise(ERR_LIB_CRMF, CRMF_R_UNSUPPORTED_ALGORITHM); goto err; } - /* could be generalized to allow non-HMAC: */ if (EVP_Q_mac(libctx, "HMAC", propq, hmac_mdname, NULL, basekey, bklen, msg, msglen, mac_res, EVP_MAX_MD_SIZE, outlen) == NULL) goto err; diff --git a/openssl/src/crypto/cryptlib.c b/openssl/src/crypto/cryptlib.c index 6c1947943..b722d2bb5 100644 --- a/openssl/src/crypto/cryptlib.c +++ b/openssl/src/crypto/cryptlib.c @@ -1,5 +1,5 @@ /* - * Copyright 1998-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1998-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -12,7 +12,7 @@ #include "crypto/cryptlib.h" #include -#if defined(_WIN32) && !defined(OPENSSL_SYS_UEFI) +#if defined(_WIN32) # include # include # ifdef __WATCOMC__ @@ -256,7 +256,7 @@ void OPENSSL_die(const char *message, const char *file, int line) { OPENSSL_showfatal("%s:%d: OpenSSL internal error: %s\n", file, line, message); -#if !defined(_WIN32) || defined(OPENSSL_SYS_UEFI) +#if !defined(_WIN32) abort(); #else /* diff --git a/openssl/src/crypto/ct/ct_b64.c b/openssl/src/crypto/ct/ct_b64.c index 253544206..d3f783962 100644 --- a/openssl/src/crypto/ct/ct_b64.c +++ b/openssl/src/crypto/ct/ct_b64.c @@ -34,8 +34,10 @@ static int ct_base64_decode(const char *in, unsigned char **out) outlen = (inlen / 4) * 3; outbuf = OPENSSL_malloc(outlen); - if (outbuf == NULL) + if (outbuf == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); goto err; + } outlen = EVP_DecodeBlock(outbuf, (unsigned char *)in, inlen); if (outlen < 0) { @@ -69,7 +71,7 @@ SCT *SCT_new_from_base64(unsigned char version, const char *logid_base64, int declen; if (sct == NULL) { - ERR_raise(ERR_LIB_CT, ERR_R_CT_LIB); + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/ct/ct_log.c b/openssl/src/crypto/ct/ct_log.c index 95084dc76..9b77d7a96 100644 --- a/openssl/src/crypto/ct/ct_log.c +++ b/openssl/src/crypto/ct/ct_log.c @@ -62,6 +62,9 @@ static CTLOG_STORE_LOAD_CTX *ctlog_store_load_ctx_new(void) { CTLOG_STORE_LOAD_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); + return ctx; } @@ -101,19 +104,23 @@ CTLOG_STORE *CTLOG_STORE_new_ex(OSSL_LIB_CTX *libctx, const char *propq) { CTLOG_STORE *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return NULL; + } ret->libctx = libctx; if (propq != NULL) { ret->propq = OPENSSL_strdup(propq); - if (ret->propq == NULL) + if (ret->propq == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); goto err; + } } ret->logs = sk_CTLOG_new_null(); if (ret->logs == NULL) { - ERR_raise(ERR_LIB_CT, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); goto err; } @@ -189,7 +196,7 @@ static int ctlog_store_load_log(const char *log_name, int log_name_len, tmp = OPENSSL_strndup(log_name, log_name_len); if (tmp == NULL) - return -1; + goto mem_err; ret = ctlog_new_from_conf(load_ctx->log_store, &ct_log, load_ctx->conf, tmp); OPENSSL_free(tmp); @@ -205,11 +212,14 @@ static int ctlog_store_load_log(const char *log_name, int log_name_len, } if (!sk_CTLOG_push(load_ctx->log_store->logs, ct_log)) { - CTLOG_free(ct_log); - ERR_raise(ERR_LIB_CT, ERR_R_CRYPTO_LIB); - return -1; + goto mem_err; } return 1; + +mem_err: + CTLOG_free(ct_log); + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); + return -1; } int CTLOG_STORE_load_file(CTLOG_STORE *store, const char *file) @@ -259,19 +269,25 @@ CTLOG *CTLOG_new_ex(EVP_PKEY *public_key, const char *name, OSSL_LIB_CTX *libctx { CTLOG *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return NULL; + } ret->libctx = libctx; if (propq != NULL) { - ret->propq = OPENSSL_strdup(propq); - if (ret->propq == NULL) + ret->name = OPENSSL_strdup(propq); + if (ret->propq == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); goto err; + } } ret->name = OPENSSL_strdup(name); - if (ret->name == NULL) + if (ret->name == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); goto err; + } if (ct_v1_log_id_from_pkey(ret, public_key) != 1) goto err; diff --git a/openssl/src/crypto/ct/ct_oct.c b/openssl/src/crypto/ct/ct_oct.c index 145b27710..72a433747 100644 --- a/openssl/src/crypto/ct/ct_oct.c +++ b/openssl/src/crypto/ct/ct_oct.c @@ -178,8 +178,10 @@ int i2o_SCT_signature(const SCT *sct, unsigned char **out) *out += len; } else { pstart = p = OPENSSL_malloc(len); - if (p == NULL) + if (p == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); goto err; + } *out = p; } @@ -223,8 +225,10 @@ int i2o_SCT(const SCT *sct, unsigned char **out) *out += len; } else { pstart = p = OPENSSL_malloc(len); - if (p == NULL) + if (p == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); goto err; + } *out = p; } @@ -326,8 +330,10 @@ int i2o_SCT_LIST(const STACK_OF(SCT) *a, unsigned char **pp) ERR_raise(ERR_LIB_CT, CT_R_SCT_LIST_INVALID); return -1; } - if ((*pp = OPENSSL_malloc(len)) == NULL) + if ((*pp = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return -1; + } is_pp_new = 1; } p = *pp + 2; diff --git a/openssl/src/crypto/ct/ct_policy.c b/openssl/src/crypto/ct/ct_policy.c index 725be7ce2..80a8baabe 100644 --- a/openssl/src/crypto/ct/ct_policy.c +++ b/openssl/src/crypto/ct/ct_policy.c @@ -13,7 +13,7 @@ #include #include -#include "internal/time.h" +#include #include "ct_local.h" @@ -29,23 +29,25 @@ CT_POLICY_EVAL_CTX *CT_POLICY_EVAL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq) { CT_POLICY_EVAL_CTX *ctx = OPENSSL_zalloc(sizeof(CT_POLICY_EVAL_CTX)); - OSSL_TIME now; - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->libctx = libctx; if (propq != NULL) { ctx->propq = OPENSSL_strdup(propq); if (ctx->propq == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); OPENSSL_free(ctx); return NULL; } } - now = ossl_time_add(ossl_time_now(), - ossl_seconds2time(SCT_CLOCK_DRIFT_TOLERANCE)); - ctx->epoch_time_in_ms = ossl_time2ms(now); + /* time(NULL) shouldn't ever fail, so don't bother checking for -1. */ + ctx->epoch_time_in_ms = (uint64_t)(time(NULL) + SCT_CLOCK_DRIFT_TOLERANCE) * + 1000; return ctx; } diff --git a/openssl/src/crypto/ct/ct_sct.c b/openssl/src/crypto/ct/ct_sct.c index ec87d0230..10a67ed6d 100644 --- a/openssl/src/crypto/ct/ct_sct.c +++ b/openssl/src/crypto/ct/ct_sct.c @@ -23,8 +23,10 @@ SCT *SCT_new(void) { SCT *sct = OPENSSL_zalloc(sizeof(*sct)); - if (sct == NULL) + if (sct == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return NULL; + } sct->entry_type = CT_LOG_ENTRY_TYPE_NOT_SET; sct->version = SCT_VERSION_NOT_SET; @@ -103,8 +105,10 @@ int SCT_set1_log_id(SCT *sct, const unsigned char *log_id, size_t log_id_len) if (log_id != NULL && log_id_len > 0) { sct->log_id = OPENSSL_memdup(log_id, log_id_len); - if (sct->log_id == NULL) + if (sct->log_id == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return 0; + } sct->log_id_len = log_id_len; } return 1; @@ -153,8 +157,10 @@ int SCT_set1_extensions(SCT *sct, const unsigned char *ext, size_t ext_len) if (ext != NULL && ext_len > 0) { sct->ext = OPENSSL_memdup(ext, ext_len); - if (sct->ext == NULL) + if (sct->ext == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return 0; + } sct->ext_len = ext_len; } return 1; @@ -177,8 +183,10 @@ int SCT_set1_signature(SCT *sct, const unsigned char *sig, size_t sig_len) if (sig != NULL && sig_len > 0) { sct->sig = OPENSSL_memdup(sig, sig_len); - if (sct->sig == NULL) + if (sct->sig == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return 0; + } sct->sig_len = sig_len; } return 1; diff --git a/openssl/src/crypto/ct/ct_sct_ctx.c b/openssl/src/crypto/ct/ct_sct_ctx.c index effd724a0..865368481 100644 --- a/openssl/src/crypto/ct/ct_sct_ctx.c +++ b/openssl/src/crypto/ct/ct_sct_ctx.c @@ -24,13 +24,16 @@ SCT_CTX *SCT_CTX_new(OSSL_LIB_CTX *libctx, const char *propq) { SCT_CTX *sctx = OPENSSL_zalloc(sizeof(*sctx)); - if (sctx == NULL) + if (sctx == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); return NULL; + } sctx->libctx = libctx; if (propq != NULL) { sctx->propq = OPENSSL_strdup(propq); if (sctx->propq == NULL) { + ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE); OPENSSL_free(sctx); return NULL; } diff --git a/openssl/src/crypto/ctype.c b/openssl/src/crypto/ctype.c index 48b3025ba..d8798218d 100644 --- a/openssl/src/crypto/ctype.c +++ b/openssl/src/crypto/ctype.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -223,9 +223,9 @@ static const unsigned short ctype_char_map[128] = { /* 7F del */ CTYPE_MASK_cntrl }; -#ifdef CHARSET_EBCDIC int ossl_toascii(int c) { +#ifdef CHARSET_EBCDIC if (c < -128 || c > 256 || c == EOF) return c; /* @@ -237,17 +237,23 @@ int ossl_toascii(int c) if (c < 0) c += 256; return os_toascii[c]; +#else + return c; +#endif } int ossl_fromascii(int c) { +#ifdef CHARSET_EBCDIC if (c < -128 || c > 256 || c == EOF) return c; if (c < 0) c += 256; return os_toebcdic[c]; -} +#else + return c; #endif +} int ossl_ctype_check(int c, unsigned int mask) { @@ -258,7 +264,7 @@ int ossl_ctype_check(int c, unsigned int mask) } /* - * Implement some of the simpler functions directly to avoid the overhead of + * Implement some of the simplier functions directly to avoid the overhead of * accessing memory via ctype_char_map[]. */ diff --git a/openssl/src/crypto/cversion.c b/openssl/src/crypto/cversion.c index 530b0e805..eb3f1dff6 100644 --- a/openssl/src/crypto/cversion.c +++ b/openssl/src/crypto/cversion.c @@ -41,6 +41,11 @@ const char *OPENSSL_version_build_metadata(void) return OPENSSL_VERSION_BUILD_METADATA; } +unsigned long Tongsuo_version_num(void) +{ + return TONGSUO_VERSION_NUMBER; +} + extern char ossl_cpu_info_str[]; const char *OpenSSL_version(int t) @@ -48,6 +53,8 @@ const char *OpenSSL_version(int t) switch (t) { case OPENSSL_VERSION: return OPENSSL_VERSION_TEXT; + case TONGSUO_VERSION: + return TONGSUO_VERSION_TEXT; case OPENSSL_VERSION_STRING: return OPENSSL_VERSION_STR; case OPENSSL_FULL_VERSION_STRING: @@ -81,6 +88,10 @@ const char *OpenSSL_version(int t) return ossl_cpu_info_str; else return "CPUINFO: N/A"; +#ifndef OPENSSL_NO_QUIC + case OPENSSL_INFO_QUIC: + return "QUIC"; +#endif } return "not available"; } diff --git a/openssl/src/crypto/der_writer.c b/openssl/src/crypto/der_writer.c index bd330785f..c6fd4c429 100644 --- a/openssl/src/crypto/der_writer.c +++ b/openssl/src/crypto/der_writer.c @@ -106,11 +106,11 @@ static int int_der_w_integer(WPACKET *pkt, int tag, && int_end_context(pkt, tag); } -static int int_put_bytes_uint32(WPACKET *pkt, const void *v, +static int int_put_bytes_ulong(WPACKET *pkt, const void *v, unsigned int *top_byte) { - const uint32_t *value = v; - uint32_t tmp = *value; + const unsigned long *value = v; + unsigned long tmp = *value; size_t n = 0; while (tmp != 0) { @@ -125,9 +125,9 @@ static int int_put_bytes_uint32(WPACKET *pkt, const void *v, } /* For integers, we only support unsigned values for now */ -int ossl_DER_w_uint32(WPACKET *pkt, int tag, uint32_t v) +int ossl_DER_w_ulong(WPACKET *pkt, int tag, unsigned long v) { - return int_der_w_integer(pkt, tag, int_put_bytes_uint32, &v); + return int_der_w_integer(pkt, tag, int_put_bytes_ulong, &v); } static int int_put_bytes_bn(WPACKET *pkt, const void *v, @@ -153,7 +153,7 @@ int ossl_DER_w_bn(WPACKET *pkt, int tag, const BIGNUM *v) if (v == NULL || BN_is_negative(v)) return 0; if (BN_is_zero(v)) - return ossl_DER_w_uint32(pkt, tag, 0); + return ossl_DER_w_ulong(pkt, tag, 0); return int_der_w_integer(pkt, tag, int_put_bytes_bn, v); } diff --git a/openssl/src/crypto/des/des_local.h b/openssl/src/crypto/des/des_local.h index d43f2c873..f888cb800 100644 --- a/openssl/src/crypto/des/des_local.h +++ b/openssl/src/crypto/des/des_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -37,19 +37,19 @@ l1=l2=0; \ switch (n) { \ case 8: l2 =((DES_LONG)(*(--(c))))<<24L; \ - /* fall through */ \ + /* fall thru */ \ case 7: l2|=((DES_LONG)(*(--(c))))<<16L; \ - /* fall through */ \ + /* fall thru */ \ case 6: l2|=((DES_LONG)(*(--(c))))<< 8L; \ - /* fall through */ \ + /* fall thru */ \ case 5: l2|=((DES_LONG)(*(--(c)))); \ - /* fall through */ \ + /* fall thru */ \ case 4: l1 =((DES_LONG)(*(--(c))))<<24L; \ - /* fall through */ \ + /* fall thru */ \ case 3: l1|=((DES_LONG)(*(--(c))))<<16L; \ - /* fall through */ \ + /* fall thru */ \ case 2: l1|=((DES_LONG)(*(--(c))))<< 8L; \ - /* fall through */ \ + /* fall thru */ \ case 1: l1|=((DES_LONG)(*(--(c)))); \ } \ } @@ -59,24 +59,39 @@ *((c)++)=(unsigned char)(((l)>>16L)&0xff), \ *((c)++)=(unsigned char)(((l)>>24L)&0xff)) +/* + * replacements for htonl and ntohl since I have no idea what to do when + * faced with machines with 8 byte longs. + */ + +# define n2l(c,l) (l =((DES_LONG)(*((c)++)))<<24L, \ + l|=((DES_LONG)(*((c)++)))<<16L, \ + l|=((DES_LONG)(*((c)++)))<< 8L, \ + l|=((DES_LONG)(*((c)++)))) + +# define l2n(l,c) (*((c)++)=(unsigned char)(((l)>>24L)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16L)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8L)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff)) + /* NOTE - c is not incremented as per l2c */ # define l2cn(l1,l2,c,n) { \ c+=n; \ switch (n) { \ case 8: *(--(c))=(unsigned char)(((l2)>>24L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 7: *(--(c))=(unsigned char)(((l2)>>16L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 6: *(--(c))=(unsigned char)(((l2)>> 8L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 5: *(--(c))=(unsigned char)(((l2) )&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 4: *(--(c))=(unsigned char)(((l1)>>24L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 3: *(--(c))=(unsigned char)(((l1)>>16L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 2: *(--(c))=(unsigned char)(((l1)>> 8L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 1: *(--(c))=(unsigned char)(((l1) )&0xff); \ } \ } @@ -94,19 +109,6 @@ : "cc"); \ ret; \ }) -# elif defined(__riscv_zbb) || defined(__riscv_zbkb) -# if __riscv_xlen == 64 -# define ROTATE(x, n) ({ register unsigned int ret; \ - asm ("roriw %0, %1, %2" \ - : "=r"(ret) \ - : "r"(x), "i"(n)); ret; }) -# endif -# if __riscv_xlen == 32 -# define ROTATE(x, n) ({ register unsigned int ret; \ - asm ("rori %0, %1, %2" \ - : "=r"(ret) \ - : "r"(x), "i"(n)); ret; }) -# endif # endif # endif # ifndef ROTATE diff --git a/openssl/src/crypto/des/fcrypt.c b/openssl/src/crypto/des/fcrypt.c index c3827a61c..088ddf2f3 100644 --- a/openssl/src/crypto/des/fcrypt.c +++ b/openssl/src/crypto/des/fcrypt.c @@ -15,6 +15,9 @@ /* NOCW */ #include +#include +#include "des_local.h" + #ifdef _OSD_POSIX # ifndef CHARSET_EBCDIC # define CHARSET_EBCDIC 1 @@ -24,9 +27,6 @@ # include #endif -#include -#include "des_local.h" - /* * Added more values to handle illegal salt values the way normal crypt() * implementations do. diff --git a/openssl/src/crypto/des/gen/linux_ia32/crypt586.S b/openssl/src/crypto/des/gen/linux_ia32/crypt586.S index 29ba0fa70..8d053be96 100644 --- a/openssl/src/crypto/des/gen/linux_ia32/crypt586.S +++ b/openssl/src/crypto/des/gen/linux_ia32/crypt586.S @@ -4,11 +4,7 @@ .align 16 fcrypt_body: .L_fcrypt_body_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/des/gen/linux_ia32/des-586.S b/openssl/src/crypto/des/gen/linux_ia32/des-586.S index b3839fa88..faaf030ab 100644 --- a/openssl/src/crypto/des/gen/linux_ia32/des-586.S +++ b/openssl/src/crypto/des/gen/linux_ia32/des-586.S @@ -3,11 +3,7 @@ .type _x86_DES_encrypt,@function .align 16 _x86_DES_encrypt: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ecx movl (%ecx),%eax @@ -479,11 +475,7 @@ _x86_DES_encrypt: .type _x86_DES_decrypt,@function .align 16 _x86_DES_decrypt: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ecx movl 120(%ecx),%eax @@ -957,11 +949,7 @@ _x86_DES_decrypt: .align 16 DES_encrypt1: .L_DES_encrypt1_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %esi pushl %edi @@ -1075,11 +1063,7 @@ DES_encrypt1: .align 16 DES_encrypt2: .L_DES_encrypt2_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %esi pushl %edi @@ -1123,11 +1107,7 @@ DES_encrypt2: .align 16 DES_encrypt3: .L_DES_encrypt3_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebx movl 8(%esp),%ebx pushl %ebp @@ -1249,11 +1229,7 @@ DES_encrypt3: .align 16 DES_decrypt3: .L_DES_decrypt3_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebx movl 8(%esp),%ebx pushl %ebp @@ -1375,11 +1351,7 @@ DES_decrypt3: .align 16 DES_ncbc_encrypt: .L_DES_ncbc_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx @@ -1441,56 +1413,28 @@ DES_ncbc_encrypt: xorl %edx,%edx jmp *%ebp .L012ej7: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 6(%esi),%dh shll $8,%edx .L013ej6: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 5(%esi),%dh .L014ej5: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 4(%esi),%dl .L015ej4: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl (%esi),%ecx jmp .L016ejend .L017ej3: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 2(%esi),%ch shll $8,%ecx .L018ej2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 1(%esi),%ch .L019ej1: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb (%esi),%cl .L016ejend: xorl %ecx,%eax @@ -1595,11 +1539,7 @@ DES_ncbc_encrypt: .align 16 DES_ede3_cbc_encrypt: .L_DES_ede3_cbc_encrypt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx @@ -1665,56 +1605,28 @@ DES_ede3_cbc_encrypt: xorl %edx,%edx jmp *%ebp .L036ej7: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 6(%esi),%dh shll $8,%edx .L037ej6: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 5(%esi),%dh .L038ej5: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 4(%esi),%dl .L039ej4: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl (%esi),%ecx jmp .L040ejend .L041ej3: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 2(%esi),%ch shll $8,%ecx .L042ej2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb 1(%esi),%ch .L043ej1: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movb (%esi),%cl .L040ejend: xorl %ecx,%eax diff --git a/openssl/src/crypto/des/gen/windows_ia32/crypt586.asm b/openssl/src/crypto/des/gen/windows_ia32/crypt586.asm index 6260ddb09..2af8af39d 100644 --- a/openssl/src/crypto/des/gen/windows_ia32/crypt586.asm +++ b/openssl/src/crypto/des/gen/windows_ia32/crypt586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/des/gen/windows_ia32/des-586.asm b/openssl/src/crypto/des/gen/windows_ia32/des-586.asm index 117a41d22..481e0627c 100644 --- a/openssl/src/crypto/des/gen/windows_ia32/des-586.asm +++ b/openssl/src/crypto/des/gen/windows_ia32/des-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 @@ -1401,56 +1400,28 @@ L$010PIC_point: xor edx,edx jmp ebp L$012ej7: - - - - - +db 243,15,30,251 mov dh,BYTE [6+esi] shl edx,8 L$013ej6: - - - - - +db 243,15,30,251 mov dh,BYTE [5+esi] L$014ej5: - - - - - +db 243,15,30,251 mov dl,BYTE [4+esi] L$015ej4: - - - - - +db 243,15,30,251 mov ecx,DWORD [esi] jmp NEAR L$016ejend L$017ej3: - - - - - +db 243,15,30,251 mov ch,BYTE [2+esi] shl ecx,8 L$018ej2: - - - - - +db 243,15,30,251 mov ch,BYTE [1+esi] L$019ej1: - - - - - +db 243,15,30,251 mov cl,BYTE [esi] L$016ejend: xor eax,ecx @@ -1618,56 +1589,28 @@ L$034PIC_point: xor edx,edx jmp ebp L$036ej7: - - - - - +db 243,15,30,251 mov dh,BYTE [6+esi] shl edx,8 L$037ej6: - - - - - +db 243,15,30,251 mov dh,BYTE [5+esi] L$038ej5: - - - - - +db 243,15,30,251 mov dl,BYTE [4+esi] L$039ej4: - - - - - +db 243,15,30,251 mov ecx,DWORD [esi] jmp NEAR L$040ejend L$041ej3: - - - - - +db 243,15,30,251 mov ch,BYTE [2+esi] shl ecx,8 L$042ej2: - - - - - +db 243,15,30,251 mov ch,BYTE [1+esi] L$043ej1: - - - - - +db 243,15,30,251 mov cl,BYTE [esi] L$040ejend: xor eax,ecx diff --git a/openssl/src/crypto/des/set_key.c b/openssl/src/crypto/des/set_key.c index 068fb9133..adcfb7f12 100644 --- a/openssl/src/crypto/des/set_key.c +++ b/openssl/src/crypto/des/set_key.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/deterministic_nonce.c b/openssl/src/crypto/deterministic_nonce.c deleted file mode 100644 index 3da9ba420..000000000 --- a/openssl/src/crypto/deterministic_nonce.c +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include -#include "internal/deterministic_nonce.h" -#include "crypto/bn.h" - -/* - * Convert a Bit String to an Integer (See RFC 6979 Section 2.3.2) - * - * Params: - * out The returned Integer as a BIGNUM - * qlen_bits The maximum size of the returned integer in bits. The returned - * Integer is shifted right if inlen is larger than qlen_bits.. - * in, inlen The input Bit String (in bytes). - * Returns: 1 if successful, or 0 otherwise. - */ -static int bits2int(BIGNUM *out, int qlen_bits, - const unsigned char *in, size_t inlen) -{ - int blen_bits = inlen * 8; - int shift; - - if (BN_bin2bn(in, (int)inlen, out) == NULL) - return 0; - - shift = blen_bits - qlen_bits; - if (shift > 0) - return BN_rshift(out, out, shift); - return 1; -} - -/* - * Convert as above a Bit String in const time to an Integer w fixed top - * - * Params: - * out The returned Integer as a BIGNUM - * qlen_bits The maximum size of the returned integer in bits. The returned - * Integer is shifted right if inlen is larger than qlen_bits.. - * in, inlen The input Bit String (in bytes). It has sizeof(BN_ULONG) bytes - * prefix with all bits set that needs to be cleared out after - * the conversion. - * Returns: 1 if successful, or 0 otherwise. - */ -static int bits2int_consttime(BIGNUM *out, int qlen_bits, - const unsigned char *in, size_t inlen) -{ - int blen_bits = (inlen - sizeof(BN_ULONG)) * 8; - int shift; - - if (BN_bin2bn(in, (int)inlen, out) == NULL) - return 0; - - BN_set_flags(out, BN_FLG_CONSTTIME); - ossl_bn_mask_bits_fixed_top(out, blen_bits); - - shift = blen_bits - qlen_bits; - if (shift > 0) - return bn_rshift_fixed_top(out, out, shift); - return 1; -} - -/* - * Convert an Integer to an Octet String (See RFC 6979 2.3.3). - * The value is zero padded if required. - * - * Params: - * out The returned Octet String - * num The input Integer - * rlen The required size of the returned Octet String in bytes - * Returns: 1 if successful, or 0 otherwise. - */ -static int int2octets(unsigned char *out, const BIGNUM *num, int rlen) -{ - return BN_bn2binpad(num, out, rlen) >= 0; -} - -/* - * Convert a Bit String to an Octet String (See RFC 6979 Section 2.3.4) - * - * Params: - * out The returned octet string. - * q The modulus - * qlen_bits The length of q in bits - * rlen The value of qlen_bits rounded up to the nearest 8 bits. - * in, inlen The input bit string (in bytes) - * Returns: 1 if successful, or 0 otherwise. - */ -static int bits2octets(unsigned char *out, const BIGNUM *q, int qlen_bits, - int rlen, const unsigned char *in, size_t inlen) -{ - int ret = 0; - BIGNUM *z = BN_new(); - - if (z == NULL - || !bits2int(z, qlen_bits, in, inlen)) - goto err; - - /* z2 = z1 mod q (Do a simple subtract, since z1 < 2^qlen_bits) */ - if (BN_cmp(z, q) >= 0 - && !BN_usub(z, z, q)) - goto err; - - ret = int2octets(out, z, rlen); -err: - BN_free(z); - return ret; -} - -/* - * Setup a KDF HMAC_DRBG object using fixed entropy and nonce data. - * - * Params: - * digestname The digest name for the HMAC - * entropy, entropylen A fixed input entropy buffer - * nonce, noncelen A fixed input nonce buffer - * libctx, propq Are used for fetching algorithms - * - * Returns: The created KDF HMAC_DRBG object if successful, or NULL otherwise. - */ -static EVP_KDF_CTX *kdf_setup(const char *digestname, - const unsigned char *entropy, size_t entropylen, - const unsigned char *nonce, size_t noncelen, - OSSL_LIB_CTX *libctx, const char *propq) -{ - EVP_KDF_CTX *ctx = NULL; - EVP_KDF *kdf = NULL; - OSSL_PARAM params[5], *p; - - kdf = EVP_KDF_fetch(libctx, "HMAC-DRBG-KDF", propq); - ctx = EVP_KDF_CTX_new(kdf); - EVP_KDF_free(kdf); - if (ctx == NULL) - goto err; - - p = params; - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST, - (char *)digestname, 0); - if (propq != NULL) - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_PROPERTIES, - (char *)propq, 0); - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_HMACDRBG_ENTROPY, - (void *)entropy, entropylen); - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_HMACDRBG_NONCE, - (void *)nonce, noncelen); - *p = OSSL_PARAM_construct_end(); - - if (EVP_KDF_CTX_set_params(ctx, params) <= 0) - goto err; - - return ctx; -err: - EVP_KDF_CTX_free(ctx); - return NULL; -} - -/* - * Generate a Deterministic nonce 'k' for DSA/ECDSA as defined in - * RFC 6979 Section 3.3. "Alternate Description of the Generation of k" - * - * Params: - * out Returns the generated deterministic nonce 'k' - * q A large prime number used for modulus operations for DSA and ECDSA. - * priv The private key in the range [1, q-1] - * hm, hmlen The digested message buffer in bytes - * digestname The digest name used for signing. It is used as the HMAC digest. - * libctx, propq Used for fetching algorithms - * - * Returns: 1 if successful, or 0 otherwise. - */ -int ossl_gen_deterministic_nonce_rfc6979(BIGNUM *out, const BIGNUM *q, - const BIGNUM *priv, - const unsigned char *hm, size_t hmlen, - const char *digestname, - OSSL_LIB_CTX *libctx, - const char *propq) -{ - EVP_KDF_CTX *kdfctx = NULL; - int ret = 0, rlen = 0, qlen_bits = 0; - unsigned char *entropyx = NULL, *nonceh = NULL, *rbits = NULL, *T = NULL; - size_t allocsz = 0; - const size_t prefsz = sizeof(BN_ULONG); - - if (out == NULL) - return 0; - - qlen_bits = BN_num_bits(q); - if (qlen_bits == 0) - return 0; - - /* Note rlen used here is in bytes since the input values are byte arrays */ - rlen = (qlen_bits + 7) / 8; - allocsz = prefsz + 3 * rlen; - - /* Use a single alloc for the buffers T, nonceh and entropyx */ - T = (unsigned char *)OPENSSL_zalloc(allocsz); - if (T == NULL) - return 0; - rbits = T + prefsz; - nonceh = rbits + rlen; - entropyx = nonceh + rlen; - - memset(T, 0xff, prefsz); - - if (!int2octets(entropyx, priv, rlen) - || !bits2octets(nonceh, q, qlen_bits, rlen, hm, hmlen)) - goto end; - - kdfctx = kdf_setup(digestname, entropyx, rlen, nonceh, rlen, libctx, propq); - if (kdfctx == NULL) - goto end; - - do { - if (!EVP_KDF_derive(kdfctx, rbits, rlen, NULL) - || !bits2int_consttime(out, qlen_bits, T, rlen + prefsz)) - goto end; - } while (ossl_bn_is_word_fixed_top(out, 0) - || ossl_bn_is_word_fixed_top(out, 1) - || BN_ucmp(out, q) >= 0); -#ifdef BN_DEBUG - /* With BN_DEBUG on a fixed top number cannot be returned */ - bn_correct_top(out); -#endif - ret = 1; - -end: - EVP_KDF_CTX_free(kdfctx); - OPENSSL_clear_free(T, allocsz); - return ret; -} diff --git a/openssl/src/crypto/dh/dh_ameth.c b/openssl/src/crypto/dh/dh_ameth.c index 80e161225..b2ff8c3eb 100644 --- a/openssl/src/crypto/dh/dh_ameth.c +++ b/openssl/src/crypto/dh/dh_ameth.c @@ -121,12 +121,12 @@ static int dh_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) str = ASN1_STRING_new(); if (str == NULL) { - ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); goto err; } str->length = i2d_dhp(pkey, dh, &str->data); if (str->length <= 0) { - ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); goto err; } ptype = V_ASN1_SEQUENCE; @@ -140,7 +140,7 @@ static int dh_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) ASN1_INTEGER_free(pub_key); if (penclen <= 0) { - ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); goto err; } @@ -184,13 +184,13 @@ static int dh_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) params = ASN1_STRING_new(); if (params == NULL) { - ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); goto err; } params->length = i2d_dhp(pkey, pkey->pkey.dh, ¶ms->data); if (params->length <= 0) { - ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); goto err; } params->type = V_ASN1_SEQUENCE; @@ -206,21 +206,18 @@ static int dh_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) dplen = i2d_ASN1_INTEGER(prkey, &dp); ASN1_STRING_clear_free(prkey); - - if (dplen <= 0) { - ERR_raise(ERR_LIB_DH, DH_R_BN_ERROR); - goto err; - } + prkey = NULL; if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(pkey->ameth->pkey_id), 0, - V_ASN1_SEQUENCE, params, dp, dplen)) { - OPENSSL_clear_free(dp, dplen); + V_ASN1_SEQUENCE, params, dp, dplen)) goto err; - } + return 1; err: + OPENSSL_free(dp); ASN1_STRING_free(params); + ASN1_STRING_clear_free(prkey); return 0; } @@ -396,21 +393,14 @@ int DHparams_print(BIO *bp, const DH *x) static int dh_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2) { - DH *dh; switch (op) { case ASN1_PKEY_CTRL_SET1_TLS_ENCPT: /* We should only be here if we have a legacy key */ if (!ossl_assert(evp_pkey_is_legacy(pkey))) return 0; - dh = (DH *) evp_pkey_get0_DH_int(pkey); - if (dh == NULL) - return 0; - return ossl_dh_buf2key(dh, arg2, arg1); + return ossl_dh_buf2key(evp_pkey_get0_DH_int(pkey), arg2, arg1); case ASN1_PKEY_CTRL_GET1_TLS_ENCPT: - dh = (DH *) EVP_PKEY_get0_DH(pkey); - if (dh == NULL) - return 0; - return ossl_dh_key2buf(dh, arg2, 0, 1); + return ossl_dh_key2buf(EVP_PKEY_get0_DH(pkey), arg2, 0, 1); default: return -2; } @@ -514,7 +504,7 @@ static int dh_pkey_import_from_type(const OSSL_PARAM params[], void *vpctx, DH *dh = ossl_dh_new_ex(pctx->libctx); if (dh == NULL) { - ERR_raise(ERR_LIB_DH, ERR_R_DH_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return 0; } DH_clear_flags(dh, DH_FLAG_TYPE_MASK); diff --git a/openssl/src/crypto/dh/dh_asn1.c b/openssl/src/crypto/dh/dh_asn1.c index 5fa91a8ec..9998bdc17 100644 --- a/openssl/src/crypto/dh/dh_asn1.c +++ b/openssl/src/crypto/dh/dh_asn1.c @@ -84,11 +84,11 @@ ASN1_SEQUENCE(DHxparams) = { ASN1_OPT(int_dhx942_dh, vparams, DHvparams), } static_ASN1_SEQUENCE_END_name(int_dhx942_dh, DHxparams) -int_dhx942_dh *d2i_int_dhx(int_dhx942_dh **a, - const unsigned char **pp, long length); -int i2d_int_dhx(const int_dhx942_dh *a, unsigned char **pp); +static int_dhx942_dh *d2i_int_dhx(int_dhx942_dh **a, + const unsigned char **pp, long length); +static int i2d_int_dhx(const int_dhx942_dh *a, unsigned char **pp); -IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(int_dhx942_dh, DHxparams, int_dhx) +IMPLEMENT_STATIC_ASN1_ENCODE_FUNCTIONS_fname(int_dhx942_dh, DHxparams, int_dhx) DH *d2i_DHxparams(DH **a, const unsigned char **pp, long length) { diff --git a/openssl/src/crypto/dh/dh_backend.c b/openssl/src/crypto/dh/dh_backend.c index 1aaa88dac..726843fd3 100644 --- a/openssl/src/crypto/dh/dh_backend.c +++ b/openssl/src/crypto/dh/dh_backend.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -31,7 +31,13 @@ static int dh_ffc_params_fromdata(DH *dh, const OSSL_PARAM params[]) { int ret; - FFC_PARAMS *ffc = ossl_dh_get0_params(dh); + FFC_PARAMS *ffc; + + if (dh == NULL) + return 0; + ffc = ossl_dh_get0_params(dh); + if (ffc == NULL) + return 0; ret = ossl_ffc_params_fromdata(ffc, params); if (ret) diff --git a/openssl/src/crypto/dh/dh_check.c b/openssl/src/crypto/dh/dh_check.c index ae23f6183..ab31d1285 100644 --- a/openssl/src/crypto/dh/dh_check.c +++ b/openssl/src/crypto/dh/dh_check.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -73,7 +73,7 @@ int DH_check_params(const DH *dh, int *ret) BN_CTX *ctx = NULL; *ret = 0; - ctx = BN_CTX_new_ex(dh->libctx); + ctx = BN_CTX_new(); if (ctx == NULL) goto err; BN_CTX_start(ctx); @@ -155,14 +155,13 @@ int DH_check(const DH *dh, int *ret) /* Don't do any checks at all with an excessively large modulus */ if (BN_num_bits(dh->params.p) > OPENSSL_DH_CHECK_MAX_MODULUS_BITS) { ERR_raise(ERR_LIB_DH, DH_R_MODULUS_TOO_LARGE); - *ret = DH_MODULUS_TOO_LARGE | DH_CHECK_P_NOT_PRIME; return 0; } if (!DH_check_params(dh, ret)) return 0; - ctx = BN_CTX_new_ex(dh->libctx); + ctx = BN_CTX_new(); if (ctx == NULL) goto err; BN_CTX_start(ctx); @@ -271,8 +270,7 @@ int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *ret) */ int ossl_dh_check_pub_key_partial(const DH *dh, const BIGNUM *pub_key, int *ret) { - return ossl_ffc_validate_public_key_partial(&dh->params, pub_key, ret) - && *ret == 0; + return ossl_ffc_validate_public_key_partial(&dh->params, pub_key, ret); } int ossl_dh_check_priv_key(const DH *dh, const BIGNUM *priv_key, int *ret) @@ -284,43 +282,22 @@ int ossl_dh_check_priv_key(const DH *dh, const BIGNUM *priv_key, int *ret) two_powN = BN_new(); if (two_powN == NULL) return 0; - - if (dh->params.q != NULL) { - upper = dh->params.q; -#ifndef FIPS_MODULE - } else if (dh->params.p != NULL) { - /* - * We do not have q so we just check the key is within some - * reasonable range, or the number of bits is equal to dh->length. - */ - int length = dh->length; - - if (length == 0) { - length = BN_num_bits(dh->params.p) - 1; - if (BN_num_bits(priv_key) <= length - && BN_num_bits(priv_key) > 1) - ok = 1; - } else if (BN_num_bits(priv_key) == length) { - ok = 1; - } - goto end; -#endif - } else { - goto end; - } + if (dh->params.q == NULL) + goto err; + upper = dh->params.q; /* Is it from an approved Safe prime group ?*/ if (DH_get_nid((DH *)dh) != NID_undef && dh->length != 0) { if (!BN_lshift(two_powN, BN_value_one(), dh->length)) - goto end; + goto err; if (BN_cmp(two_powN, dh->params.q) < 0) upper = two_powN; } if (!ossl_ffc_validate_private_key(upper, priv_key, ret)) - goto end; + goto err; ok = 1; -end: +err: BN_free(two_powN); return ok; } @@ -351,7 +328,7 @@ int ossl_dh_check_pairwise(const DH *dh) /* recalculate the public key = (g ^ priv) mod p */ if (!ossl_dh_generate_public_key(ctx, dh, dh->priv_key, pub_key)) goto err; - /* check it matches the existing public_key */ + /* check it matches the existing pubic_key */ ret = BN_cmp(pub_key, dh->pub_key) == 0; err: BN_free(pub_key); diff --git a/openssl/src/crypto/dh/dh_err.c b/openssl/src/crypto/dh/dh_err.c index 4d6d2acd9..f76ac0dd1 100644 --- a/openssl/src/crypto/dh/dh_err.c +++ b/openssl/src/crypto/dh/dh_err.c @@ -42,7 +42,6 @@ static const ERR_STRING_DATA DH_str_reasons[] = { "invalid parameter nid"}, {ERR_PACK(ERR_LIB_DH, 0, DH_R_INVALID_PUBKEY), "invalid public key"}, {ERR_PACK(ERR_LIB_DH, 0, DH_R_INVALID_SECRET), "invalid secret"}, - {ERR_PACK(ERR_LIB_DH, 0, DH_R_INVALID_SIZE), "invalid size"}, {ERR_PACK(ERR_LIB_DH, 0, DH_R_KDF_PARAMETER_ERROR), "kdf parameter error"}, {ERR_PACK(ERR_LIB_DH, 0, DH_R_KEYS_NOT_SET), "keys not set"}, {ERR_PACK(ERR_LIB_DH, 0, DH_R_MISSING_PUBKEY), "missing pubkey"}, diff --git a/openssl/src/crypto/dh/dh_gen.c b/openssl/src/crypto/dh/dh_gen.c index b73bfb7f3..66d1f94bc 100644 --- a/openssl/src/crypto/dh/dh_gen.c +++ b/openssl/src/crypto/dh/dh_gen.c @@ -28,7 +28,6 @@ #include #include #include "crypto/dh.h" -#include "crypto/security_bits.h" #include "dh_local.h" #ifndef FIPS_MODULE @@ -170,7 +169,7 @@ static int dh_builtin_genparams(DH *ret, int prime_len, int generator, return 0; } - ctx = BN_CTX_new_ex(ret->libctx); + ctx = BN_CTX_new(); if (ctx == NULL) goto err; BN_CTX_start(ctx); @@ -214,15 +213,12 @@ static int dh_builtin_genparams(DH *ret, int prime_len, int generator, g = generator; } - if (!BN_generate_prime_ex2(ret->params.p, prime_len, 1, t1, t2, cb, ctx)) + if (!BN_generate_prime_ex(ret->params.p, prime_len, 1, t1, t2, cb)) goto err; if (!BN_GENCB_call(cb, 3, 0)) goto err; if (!BN_set_word(ret->params.g, g)) goto err; - /* We are using safe prime p, set key length equivalent to RFC 7919 */ - ret->length = (2 * ossl_ifc_ffc_compute_security_bits(prime_len) - + 24) / 25 * 25; ret->dirty_cnt++; ok = 1; err: diff --git a/openssl/src/crypto/dh/dh_group_params.c b/openssl/src/crypto/dh/dh_group_params.c index 460bd8f00..3f843fe95 100644 --- a/openssl/src/crypto/dh/dh_group_params.c +++ b/openssl/src/crypto/dh/dh_group_params.c @@ -31,7 +31,7 @@ static DH *dh_param_init(OSSL_LIB_CTX *libctx, const DH_NAMED_GROUP *group) if (dh == NULL) return NULL; - ossl_ffc_named_group_set(&dh->params, group); + ossl_ffc_named_group_set_pqg(&dh->params, group); dh->params.nid = ossl_ffc_named_group_get_uid(group); dh->dirty_cnt++; return dh; @@ -72,9 +72,8 @@ void ossl_dh_cache_named_group(DH *dh) dh->params.g)) != NULL) { if (dh->params.q == NULL) dh->params.q = (BIGNUM *)ossl_ffc_named_group_get_q(group); - /* cache the nid and default key length */ + /* cache the nid */ dh->params.nid = ossl_ffc_named_group_get_uid(group); - dh->params.keylength = ossl_ffc_named_group_get_keylength(group); dh->dirty_cnt++; } } diff --git a/openssl/src/crypto/dh/dh_key.c b/openssl/src/crypto/dh/dh_key.c index 7132b9b68..982630a0b 100644 --- a/openssl/src/crypto/dh/dh_key.c +++ b/openssl/src/crypto/dh/dh_key.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -190,16 +190,13 @@ static int dh_bn_mod_exp(const DH *dh, BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx) { -#ifdef S390X_MOD_EXP - return s390x_mod_exp(r, a, p, m, ctx, m_ctx); -#else return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx); -#endif } static int dh_init(DH *dh) { dh->flags |= DH_FLAG_CACHE_MONT_P; + ossl_ffc_params_init(&dh->params); dh->dirty_cnt++; return 1; } @@ -433,15 +430,14 @@ size_t ossl_dh_key2buf(const DH *dh, unsigned char **pbuf_out, size_t size, if (!alloc) { if (size >= (size_t)p_size) pbuf = *pbuf_out; - if (pbuf == NULL) - ERR_raise(ERR_LIB_DH, DH_R_INVALID_SIZE); } else { pbuf = OPENSSL_malloc(p_size); } - /* Errors raised above */ - if (pbuf == NULL) + if (pbuf == NULL) { + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return 0; + } /* * As per Section 4.2.8.1 of RFC 8446 left pad public * key with zeros to the size of p diff --git a/openssl/src/crypto/dh/dh_lib.c b/openssl/src/crypto/dh/dh_lib.c index 9d5a6b0b6..29cda5d7b 100644 --- a/openssl/src/crypto/dh/dh_lib.c +++ b/openssl/src/crypto/dh/dh_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -75,18 +75,15 @@ static DH *dh_new_intern(ENGINE *engine, OSSL_LIB_CTX *libctx) { DH *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return NULL; + } + ret->references = 1; ret->lock = CRYPTO_THREAD_lock_new(); if (ret->lock == NULL) { - ERR_raise(ERR_LIB_DH, ERR_R_CRYPTO_LIB); - OPENSSL_free(ret); - return NULL; - } - - if (!CRYPTO_NEW_REF(&ret->references, 1)) { - CRYPTO_THREAD_lock_free(ret->lock); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -119,8 +116,6 @@ static DH *dh_new_intern(ENGINE *engine, OSSL_LIB_CTX *libctx) goto err; #endif /* FIPS_MODULE */ - ossl_ffc_params_init(&ret->params); - if ((ret->meth->init != NULL) && !ret->meth->init(ret)) { ERR_raise(ERR_LIB_DH, ERR_R_INIT_FAIL); goto err; @@ -140,7 +135,7 @@ void DH_free(DH *r) if (r == NULL) return; - CRYPTO_DOWN_REF(&r->references, &i); + CRYPTO_DOWN_REF(&r->references, &i, r->lock); REF_PRINT_COUNT("DH", r); if (i > 0) return; @@ -156,7 +151,6 @@ void DH_free(DH *r) #endif CRYPTO_THREAD_lock_free(r->lock); - CRYPTO_FREE_REF(&r->references); ossl_ffc_params_cleanup(&r->params); BN_clear_free(r->pub_key); @@ -168,7 +162,7 @@ int DH_up_ref(DH *r) { int i; - if (CRYPTO_UP_REF(&r->references, &i) <= 0) + if (CRYPTO_UP_REF(&r->references, &i, r->lock) <= 0) return 0; REF_PRINT_COUNT("DH", r); diff --git a/openssl/src/crypto/dh/dh_meth.c b/openssl/src/crypto/dh/dh_meth.c index f5652e078..5c15cd2b8 100644 --- a/openssl/src/crypto/dh/dh_meth.c +++ b/openssl/src/crypto/dh/dh_meth.c @@ -31,6 +31,7 @@ DH_METHOD *DH_meth_new(const char *name, int flags) OPENSSL_free(dhm); } + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return NULL; } @@ -56,6 +57,7 @@ DH_METHOD *DH_meth_dup(const DH_METHOD *dhm) OPENSSL_free(ret); } + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return NULL; } @@ -68,8 +70,10 @@ int DH_meth_set1_name(DH_METHOD *dhm, const char *name) { char *tmpname = OPENSSL_strdup(name); - if (tmpname == NULL) + if (tmpname == NULL) { + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return 0; + } OPENSSL_free(dhm->name); dhm->name = tmpname; diff --git a/openssl/src/crypto/dh/dh_pmeth.c b/openssl/src/crypto/dh/dh_pmeth.c index c11ada982..1ad50b850 100644 --- a/openssl/src/crypto/dh/dh_pmeth.c +++ b/openssl/src/crypto/dh/dh_pmeth.c @@ -55,8 +55,10 @@ static int pkey_dh_init(EVP_PKEY_CTX *ctx) { DH_PKEY_CTX *dctx; - if ((dctx = OPENSSL_zalloc(sizeof(*dctx))) == NULL) + if ((dctx = OPENSSL_zalloc(sizeof(*dctx))) == NULL) { + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return 0; + } dctx->prime_len = 2048; dctx->subprime_len = -1; dctx->generator = 2; @@ -430,8 +432,7 @@ static int pkey_dh_derive(EVP_PKEY_CTX *ctx, unsigned char *key, else if (dctx->kdf_type == EVP_PKEY_DH_KDF_X9_42) { unsigned char *Z = NULL; - int Zlen = 0; - + size_t Zlen = 0; if (!dctx->kdf_outlen || !dctx->kdf_oid) return 0; if (key == NULL) { @@ -443,8 +444,10 @@ static int pkey_dh_derive(EVP_PKEY_CTX *ctx, unsigned char *key, ret = 0; if ((Zlen = DH_size(dh)) <= 0) return 0; - if ((Z = OPENSSL_malloc(Zlen)) == NULL) + if ((Z = OPENSSL_malloc(Zlen)) == NULL) { + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return 0; + } if (DH_compute_key_padded(Z, dhpubbn, dh) <= 0) goto err; if (!DH_KDF_X9_42(key, *keylen, Z, Zlen, dctx->kdf_oid, diff --git a/openssl/src/crypto/dsa/dsa_ameth.c b/openssl/src/crypto/dsa/dsa_ameth.c index 15a5266ca..234fc44ed 100644 --- a/openssl/src/crypto/dsa/dsa_ameth.c +++ b/openssl/src/crypto/dsa/dsa_ameth.c @@ -54,7 +54,7 @@ static int dsa_pub_decode(EVP_PKEY *pkey, const X509_PUBKEY *pubkey) } else if ((ptype == V_ASN1_NULL) || (ptype == V_ASN1_UNDEF)) { if ((dsa = DSA_new()) == NULL) { - ERR_raise(ERR_LIB_DSA, ERR_R_DSA_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto err; } } else { @@ -101,12 +101,12 @@ static int dsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) && dsa->params.g != NULL) { str = ASN1_STRING_new(); if (str == NULL) { - ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto err; } str->length = i2d_DSAparams(dsa, &str->data); if (str->length <= 0) { - ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto err; } ptype = V_ASN1_SEQUENCE; @@ -116,7 +116,7 @@ static int dsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) pubint = BN_to_ASN1_INTEGER(dsa->pub_key, NULL); if (pubint == NULL) { - ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto err; } @@ -124,7 +124,7 @@ static int dsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) ASN1_INTEGER_free(pubint); if (penclen <= 0) { - ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto err; } @@ -175,13 +175,13 @@ static int dsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) params = ASN1_STRING_new(); if (params == NULL) { - ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto err; } params->length = i2d_DSAparams(pkey->pkey.dsa, ¶ms->data); if (params->length <= 0) { - ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto err; } params->type = V_ASN1_SEQUENCE; @@ -197,21 +197,18 @@ static int dsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) dplen = i2d_ASN1_INTEGER(prkey, &dp); ASN1_STRING_clear_free(prkey); - - if (dplen <= 0) { - ERR_raise(ERR_LIB_DSA, DSA_R_BN_ERROR); - goto err; - } + prkey = NULL; if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(NID_dsa), 0, - V_ASN1_SEQUENCE, params, dp, dplen)) { - OPENSSL_clear_free(dp, dplen); + V_ASN1_SEQUENCE, params, dp, dplen)) goto err; - } + return 1; err: + OPENSSL_free(dp); ASN1_STRING_free(params); + ASN1_STRING_clear_free(prkey); return 0; } @@ -483,7 +480,7 @@ static int dsa_pkey_import_from(const OSSL_PARAM params[], void *vpctx) DSA *dsa = ossl_dsa_new(pctx->libctx); if (dsa == NULL) { - ERR_raise(ERR_LIB_DSA, ERR_R_DSA_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); return 0; } diff --git a/openssl/src/crypto/dsa/dsa_backend.c b/openssl/src/crypto/dsa/dsa_backend.c index 8bd4b8ad7..f9a71bdc9 100644 --- a/openssl/src/crypto/dsa/dsa_backend.c +++ b/openssl/src/crypto/dsa/dsa_backend.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -158,11 +158,11 @@ DSA *ossl_dsa_key_from_pkcs8(const PKCS8_PRIV_KEY_INFO *p8inf, } /* Calculate public key */ if ((dsa_pubkey = BN_new()) == NULL) { - ERR_raise(ERR_LIB_DSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto dsaerr; } if ((ctx = BN_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_DSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); goto dsaerr; } @@ -173,10 +173,7 @@ DSA *ossl_dsa_key_from_pkcs8(const PKCS8_PRIV_KEY_INFO *p8inf, ERR_raise(ERR_LIB_DSA, DSA_R_BN_ERROR); goto dsaerr; } - if (!DSA_set0_key(dsa, dsa_pubkey, dsa_privkey)) { - ERR_raise(ERR_LIB_DSA, ERR_R_INTERNAL_ERROR); - goto dsaerr; - } + DSA_set0_key(dsa, dsa_pubkey, dsa_privkey); goto done; diff --git a/openssl/src/crypto/dsa/dsa_check.c b/openssl/src/crypto/dsa/dsa_check.c index e1375dfad..7ee914a47 100644 --- a/openssl/src/crypto/dsa/dsa_check.c +++ b/openssl/src/crypto/dsa/dsa_check.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,34 +19,8 @@ #include "dsa_local.h" #include "crypto/dsa.h" -static int dsa_precheck_params(const DSA *dsa, int *ret) -{ - if (dsa->params.p == NULL || dsa->params.q == NULL) { - ERR_raise(ERR_LIB_DSA, DSA_R_BAD_FFC_PARAMETERS); - *ret = FFC_CHECK_INVALID_PQ; - return 0; - } - - if (BN_num_bits(dsa->params.p) > OPENSSL_DSA_MAX_MODULUS_BITS) { - ERR_raise(ERR_LIB_DSA, DSA_R_MODULUS_TOO_LARGE); - *ret = FFC_CHECK_INVALID_PQ; - return 0; - } - - if (BN_num_bits(dsa->params.q) >= BN_num_bits(dsa->params.p)) { - ERR_raise(ERR_LIB_DSA, DSA_R_BAD_Q_VALUE); - *ret = FFC_CHECK_INVALID_PQ; - return 0; - } - - return 1; -} - int ossl_dsa_check_params(const DSA *dsa, int checktype, int *ret) { - if (!dsa_precheck_params(dsa, ret)) - return 0; - if (checktype == OSSL_KEYMGMT_VALIDATE_QUICK_CHECK) return ossl_ffc_params_simple_validate(dsa->libctx, &dsa->params, FFC_PARAM_TYPE_DSA, ret); @@ -65,11 +39,7 @@ int ossl_dsa_check_params(const DSA *dsa, int checktype, int *ret) */ int ossl_dsa_check_pub_key(const DSA *dsa, const BIGNUM *pub_key, int *ret) { - if (!dsa_precheck_params(dsa, ret)) - return 0; - - return ossl_ffc_validate_public_key(&dsa->params, pub_key, ret) - && *ret == 0; + return ossl_ffc_validate_public_key(&dsa->params, pub_key, ret); } /* @@ -79,21 +49,15 @@ int ossl_dsa_check_pub_key(const DSA *dsa, const BIGNUM *pub_key, int *ret) */ int ossl_dsa_check_pub_key_partial(const DSA *dsa, const BIGNUM *pub_key, int *ret) { - if (!dsa_precheck_params(dsa, ret)) - return 0; - - return ossl_ffc_validate_public_key_partial(&dsa->params, pub_key, ret) - && *ret == 0; + return ossl_ffc_validate_public_key_partial(&dsa->params, pub_key, ret); } int ossl_dsa_check_priv_key(const DSA *dsa, const BIGNUM *priv_key, int *ret) { *ret = 0; - if (!dsa_precheck_params(dsa, ret)) - return 0; - - return ossl_ffc_validate_private_key(dsa->params.q, priv_key, ret); + return (dsa->params.q != NULL + && ossl_ffc_validate_private_key(dsa->params.q, priv_key, ret)); } /* @@ -106,10 +70,8 @@ int ossl_dsa_check_pairwise(const DSA *dsa) BN_CTX *ctx = NULL; BIGNUM *pub_key = NULL; - if (!dsa_precheck_params(dsa, &ret)) - return 0; - - if (dsa->params.g == NULL + if (dsa->params.p == NULL + || dsa->params.g == NULL || dsa->priv_key == NULL || dsa->pub_key == NULL) return 0; @@ -124,7 +86,7 @@ int ossl_dsa_check_pairwise(const DSA *dsa) /* recalculate the public key = (g ^ priv) mod p */ if (!ossl_dsa_generate_public_key(ctx, dsa, dsa->priv_key, pub_key)) goto err; - /* check it matches the existing public_key */ + /* check it matches the existing pubic_key */ ret = BN_cmp(pub_key, dsa->pub_key) == 0; err: BN_free(pub_key); diff --git a/openssl/src/crypto/dsa/dsa_err.c b/openssl/src/crypto/dsa/dsa_err.c index a92ca6166..5685d5e83 100644 --- a/openssl/src/crypto/dsa/dsa_err.c +++ b/openssl/src/crypto/dsa/dsa_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -36,7 +36,6 @@ static const ERR_STRING_DATA DSA_str_reasons[] = { {ERR_PACK(ERR_LIB_DSA, 0, DSA_R_Q_NOT_PRIME), "q not prime"}, {ERR_PACK(ERR_LIB_DSA, 0, DSA_R_SEED_LEN_SMALL), "seed_len is less than the length of q"}, - {ERR_PACK(ERR_LIB_DSA, 0, DSA_R_TOO_MANY_RETRIES), "too many retries"}, {0, NULL} }; diff --git a/openssl/src/crypto/dsa/dsa_key.c b/openssl/src/crypto/dsa/dsa_key.c index 1c2bab171..1f951a9d3 100644 --- a/openssl/src/crypto/dsa/dsa_key.c +++ b/openssl/src/crypto/dsa/dsa_key.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -28,7 +28,8 @@ # define MIN_STRENGTH 80 #endif -static int dsa_keygen(DSA *dsa); +static int dsa_keygen(DSA *dsa, int pairwise_test); +static int dsa_keygen_pairwise_test(DSA *dsa, OSSL_CALLBACK *cb, void *cbarg); int DSA_generate_key(DSA *dsa) { @@ -36,7 +37,7 @@ int DSA_generate_key(DSA *dsa) if (dsa->meth->dsa_keygen != NULL) return dsa->meth->dsa_keygen(dsa); #endif - return dsa_keygen(dsa); + return dsa_keygen(dsa, 0); } int ossl_dsa_generate_public_key(BN_CTX *ctx, const DSA *dsa, @@ -58,93 +59,7 @@ int ossl_dsa_generate_public_key(BN_CTX *ctx, const DSA *dsa, return ret; } -#ifdef FIPS_MODULE -/* - * Refer: FIPS 140-3 IG 10.3.A Additional Comment 1 - * Perform a KAT by duplicating the public key generation. - * - * NOTE: This issue requires a background understanding, provided in a separate - * document; the current IG 10.3.A AC1 is insufficient regarding the PCT for - * the key agreement scenario. - * - * Currently IG 10.3.A requires PCT in the mode of use prior to use of the - * key pair, citing the PCT defined in the associated standard. For key - * agreement, the only PCT defined in SP 800-56A is that of Section 5.6.2.4: - * the comparison of the original public key to a newly calculated public key. - */ -static int dsa_keygen_knownanswer_test(DSA *dsa, BN_CTX *ctx, - OSSL_CALLBACK *cb, void *cbarg) -{ - int len, ret = 0; - OSSL_SELF_TEST *st = NULL; - unsigned char bytes[512] = {0}; - BIGNUM *pub_key2 = BN_new(); - - if (pub_key2 == NULL) - return 0; - - st = OSSL_SELF_TEST_new(cb, cbarg); - if (st == NULL) - goto err; - - OSSL_SELF_TEST_onbegin(st, OSSL_SELF_TEST_TYPE_PCT_KAT, - OSSL_SELF_TEST_DESC_PCT_DSA); - - if (!ossl_dsa_generate_public_key(ctx, dsa, dsa->priv_key, pub_key2)) - goto err; - - if (BN_num_bytes(pub_key2) > (int)sizeof(bytes)) - goto err; - len = BN_bn2bin(pub_key2, bytes); - OSSL_SELF_TEST_oncorrupt_byte(st, bytes); - if (BN_bin2bn(bytes, len, pub_key2) != NULL) - ret = !BN_cmp(dsa->pub_key, pub_key2); - -err: - OSSL_SELF_TEST_onend(st, ret); - OSSL_SELF_TEST_free(st); - BN_free(pub_key2); - return ret; -} - -/* - * FIPS 140-2 IG 9.9 AS09.33 - * Perform a sign/verify operation. - */ -static int dsa_keygen_pairwise_test(DSA *dsa, OSSL_CALLBACK *cb, void *cbarg) -{ - int ret = 0; - unsigned char dgst[16] = {0}; - unsigned int dgst_len = (unsigned int)sizeof(dgst); - DSA_SIG *sig = NULL; - OSSL_SELF_TEST *st = NULL; - - st = OSSL_SELF_TEST_new(cb, cbarg); - if (st == NULL) - goto err; - - OSSL_SELF_TEST_onbegin(st, OSSL_SELF_TEST_TYPE_PCT, - OSSL_SELF_TEST_DESC_PCT_DSA); - - sig = DSA_do_sign(dgst, (int)dgst_len, dsa); - if (sig == NULL) - goto err; - - OSSL_SELF_TEST_oncorrupt_byte(st, dgst); - - if (DSA_do_verify(dgst, dgst_len, sig, dsa) != 1) - goto err; - - ret = 1; -err: - OSSL_SELF_TEST_onend(st, ret); - OSSL_SELF_TEST_free(st); - DSA_SIG_free(sig); - return ret; -} -#endif /* FIPS_MODULE */ - -static int dsa_keygen(DSA *dsa) +static int dsa_keygen(DSA *dsa, int pairwise_test) { int ok = 0; BN_CTX *ctx = NULL; @@ -188,15 +103,17 @@ static int dsa_keygen(DSA *dsa) dsa->priv_key = priv_key; dsa->pub_key = pub_key; - ok = 1; #ifdef FIPS_MODULE - { + pairwise_test = 1; +#endif /* FIPS_MODULE */ + + ok = 1; + if (pairwise_test) { OSSL_CALLBACK *cb = NULL; void *cbarg = NULL; OSSL_SELF_TEST_get_callback(dsa->libctx, &cb, &cbarg); - ok = dsa_keygen_pairwise_test(dsa, cb, cbarg) - && dsa_keygen_knownanswer_test(dsa, ctx, cb, cbarg); + ok = dsa_keygen_pairwise_test(dsa, cb, cbarg); if (!ok) { ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT); BN_free(dsa->pub_key); @@ -207,7 +124,6 @@ static int dsa_keygen(DSA *dsa) return ok; } } -#endif dsa->dirty_cnt++; err: @@ -219,3 +135,39 @@ static int dsa_keygen(DSA *dsa) return ok; } + +/* + * FIPS 140-2 IG 9.9 AS09.33 + * Perform a sign/verify operation. + */ +static int dsa_keygen_pairwise_test(DSA *dsa, OSSL_CALLBACK *cb, void *cbarg) +{ + int ret = 0; + unsigned char dgst[16] = {0}; + unsigned int dgst_len = (unsigned int)sizeof(dgst); + DSA_SIG *sig = NULL; + OSSL_SELF_TEST *st = NULL; + + st = OSSL_SELF_TEST_new(cb, cbarg); + if (st == NULL) + goto err; + + OSSL_SELF_TEST_onbegin(st, OSSL_SELF_TEST_TYPE_PCT, + OSSL_SELF_TEST_DESC_PCT_DSA); + + sig = DSA_do_sign(dgst, (int)dgst_len, dsa); + if (sig == NULL) + goto err; + + OSSL_SELF_TEST_oncorrupt_byte(st, dgst); + + if (DSA_do_verify(dgst, dgst_len, sig, dsa) != 1) + goto err; + + ret = 1; +err: + OSSL_SELF_TEST_onend(st, ret); + OSSL_SELF_TEST_free(st); + DSA_SIG_free(sig); + return ret; +} diff --git a/openssl/src/crypto/dsa/dsa_lib.c b/openssl/src/crypto/dsa/dsa_lib.c index 7997c2ac2..ccc701659 100644 --- a/openssl/src/crypto/dsa/dsa_lib.c +++ b/openssl/src/crypto/dsa/dsa_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -134,18 +134,15 @@ static DSA *dsa_new_intern(ENGINE *engine, OSSL_LIB_CTX *libctx) { DSA *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); return NULL; + } + ret->references = 1; ret->lock = CRYPTO_THREAD_lock_new(); if (ret->lock == NULL) { - ERR_raise(ERR_LIB_DSA, ERR_R_CRYPTO_LIB); - OPENSSL_free(ret); - return NULL; - } - - if (!CRYPTO_NEW_REF(&ret->references, 1)) { - CRYPTO_THREAD_lock_free(ret->lock); + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -179,8 +176,6 @@ static DSA *dsa_new_intern(ENGINE *engine, OSSL_LIB_CTX *libctx) goto err; #endif - ossl_ffc_params_init(&ret->params); - if ((ret->meth->init != NULL) && !ret->meth->init(ret)) { ERR_raise(ERR_LIB_DSA, ERR_R_INIT_FAIL); goto err; @@ -217,7 +212,7 @@ void DSA_free(DSA *r) if (r == NULL) return; - CRYPTO_DOWN_REF(&r->references, &i); + CRYPTO_DOWN_REF(&r->references, &i, r->lock); REF_PRINT_COUNT("DSA", r); if (i > 0) return; @@ -234,7 +229,6 @@ void DSA_free(DSA *r) #endif CRYPTO_THREAD_lock_free(r->lock); - CRYPTO_FREE_REF(&r->references); ossl_ffc_params_cleanup(&r->params); BN_clear_free(r->pub_key); @@ -246,7 +240,7 @@ int DSA_up_ref(DSA *r) { int i; - if (CRYPTO_UP_REF(&r->references, &i) <= 0) + if (CRYPTO_UP_REF(&r->references, &i, r->lock) <= 0) return 0; REF_PRINT_COUNT("DSA", r); @@ -353,7 +347,13 @@ FFC_PARAMS *ossl_dsa_get0_params(DSA *dsa) int ossl_dsa_ffc_params_fromdata(DSA *dsa, const OSSL_PARAM params[]) { int ret; - FFC_PARAMS *ffc = ossl_dsa_get0_params(dsa); + FFC_PARAMS *ffc; + + if (dsa == NULL) + return 0; + ffc = ossl_dsa_get0_params(dsa); + if (ffc == NULL) + return 0; ret = ossl_ffc_params_fromdata(ffc, params); if (ret) diff --git a/openssl/src/crypto/dsa/dsa_local.h b/openssl/src/crypto/dsa/dsa_local.h index 38cb64a82..9e33fae13 100644 --- a/openssl/src/crypto/dsa/dsa_local.h +++ b/openssl/src/crypto/dsa/dsa_local.h @@ -14,7 +14,7 @@ struct dsa_st { /* * This first variable is used to pick up errors where a DSA is passed - * instead of an EVP_PKEY + * instead of of a EVP_PKEY */ int pad; int32_t version; @@ -69,6 +69,4 @@ struct dsa_method { int (*dsa_keygen) (DSA *dsa); }; -DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa, - unsigned int nonce_type, const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq); +DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa); diff --git a/openssl/src/crypto/dsa/dsa_meth.c b/openssl/src/crypto/dsa/dsa_meth.c index f2b759a9d..2f0a0bf46 100644 --- a/openssl/src/crypto/dsa/dsa_meth.c +++ b/openssl/src/crypto/dsa/dsa_meth.c @@ -32,6 +32,7 @@ DSA_METHOD *DSA_meth_new(const char *name, int flags) OPENSSL_free(dsam); } + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); return NULL; } @@ -57,6 +58,7 @@ DSA_METHOD *DSA_meth_dup(const DSA_METHOD *dsam) OPENSSL_free(ret); } + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); return NULL; } @@ -69,8 +71,10 @@ int DSA_meth_set1_name(DSA_METHOD *dsam, const char *name) { char *tmpname = OPENSSL_strdup(name); - if (tmpname == NULL) + if (tmpname == NULL) { + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); return 0; + } OPENSSL_free(dsam->name); dsam->name = tmpname; diff --git a/openssl/src/crypto/dsa/dsa_ossl.c b/openssl/src/crypto/dsa/dsa_ossl.c index 59b26d736..86d89f4c7 100644 --- a/openssl/src/crypto/dsa/dsa_ossl.c +++ b/openssl/src/crypto/dsa/dsa_ossl.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,18 +20,12 @@ #include #include "dsa_local.h" #include -#include "internal/deterministic_nonce.h" - -#define MIN_DSA_SIGN_QBITS 128 -#define MAX_DSA_SIGN_RETRIES 8 static DSA_SIG *dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa); static int dsa_sign_setup_no_digest(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp); static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, - BIGNUM **rp, const unsigned char *dgst, int dlen, - unsigned int nonce_type, const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq); + BIGNUM **rp, const unsigned char *dgst, int dlen); static int dsa_do_verify(const unsigned char *dgst, int dgst_len, DSA_SIG *sig, DSA *dsa); static int dsa_init(DSA *dsa); @@ -73,9 +67,7 @@ const DSA_METHOD *DSA_OpenSSL(void) return &openssl_dsa_meth; } -DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa, - unsigned int nonce_type, const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq) +DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa) { BIGNUM *kinv = NULL; BIGNUM *m, *blind, *blindm, *tmp; @@ -83,7 +75,6 @@ DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa, int reason = ERR_R_BN_LIB; DSA_SIG *ret = NULL; int rv = 0; - int retries = 0; if (dsa->params.p == NULL || dsa->params.q == NULL @@ -115,8 +106,7 @@ DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa, goto err; redo: - if (!dsa_sign_setup(dsa, ctx, &kinv, &ret->r, dgst, dlen, - nonce_type, digestname, libctx, propq)) + if (!dsa_sign_setup(dsa, ctx, &kinv, &ret->r, dgst, dlen)) goto err; if (dlen > BN_num_bytes(dsa->params.q)) @@ -139,10 +129,7 @@ DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa, * s := blind^-1 * k^-1 * (blind * m + blind * r * priv_key) mod q */ - /* - * Generate a blinding value - * The size of q is tested in dsa_sign_setup() so there should not be an infinite loop here. - */ + /* Generate a blinding value */ do { if (!BN_priv_rand_ex(blind, BN_num_bits(dsa->params.q) - 1, BN_RAND_TOP_ANY, BN_RAND_BOTTOM_ANY, 0, ctx)) @@ -177,19 +164,14 @@ DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa, goto err; /* - * Redo if r or s is zero as required by FIPS 186-4: Section 4.6 - * This is very unlikely. - * Limit the retries so there is no possibility of an infinite - * loop for bad domain parameter values. + * Redo if r or s is zero as required by FIPS 186-3: this is very + * unlikely. */ - if (BN_is_zero(ret->r) || BN_is_zero(ret->s)) { - if (retries++ > MAX_DSA_SIGN_RETRIES) { - reason = DSA_R_TOO_MANY_RETRIES; - goto err; - } + if (BN_is_zero(ret->r) || BN_is_zero(ret->s)) goto redo; - } + rv = 1; + err: if (rv == 0) { ERR_raise(ERR_LIB_DSA, reason); @@ -203,22 +185,18 @@ DSA_SIG *ossl_dsa_do_sign_int(const unsigned char *dgst, int dlen, DSA *dsa, static DSA_SIG *dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa) { - return ossl_dsa_do_sign_int(dgst, dlen, dsa, - 0, NULL, NULL, NULL); + return ossl_dsa_do_sign_int(dgst, dlen, dsa); } static int dsa_sign_setup_no_digest(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp) { - return dsa_sign_setup(dsa, ctx_in, kinvp, rp, NULL, 0, - 0, NULL, NULL, NULL); + return dsa_sign_setup(dsa, ctx_in, kinvp, rp, NULL, 0); } static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp, - const unsigned char *dgst, int dlen, - unsigned int nonce_type, const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq) + const unsigned char *dgst, int dlen) { BN_CTX *ctx = NULL; BIGNUM *k, *kinv = NULL, *r = *rp; @@ -234,10 +212,7 @@ static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, /* Reject obviously invalid parameters */ if (BN_is_zero(dsa->params.p) || BN_is_zero(dsa->params.q) - || BN_is_zero(dsa->params.g) - || BN_is_negative(dsa->params.p) - || BN_is_negative(dsa->params.q) - || BN_is_negative(dsa->params.g)) { + || BN_is_zero(dsa->params.g)) { ERR_raise(ERR_LIB_DSA, DSA_R_INVALID_PARAMETERS); return 0; } @@ -245,6 +220,7 @@ static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, ERR_raise(ERR_LIB_DSA, DSA_R_MISSING_PRIVATE_KEY); return 0; } + k = BN_new(); l = BN_new(); if (k == NULL || l == NULL) @@ -260,36 +236,23 @@ static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, /* Preallocate space */ q_bits = BN_num_bits(dsa->params.q); q_words = bn_get_top(dsa->params.q); - if (q_bits < MIN_DSA_SIGN_QBITS - || !bn_wexpand(k, q_words + 2) + if (!bn_wexpand(k, q_words + 2) || !bn_wexpand(l, q_words + 2)) goto err; /* Get random k */ do { if (dgst != NULL) { - if (nonce_type == 1) { -#ifndef FIPS_MODULE - if (!ossl_gen_deterministic_nonce_rfc6979(k, dsa->params.q, - dsa->priv_key, - dgst, dlen, - digestname, - libctx, propq)) -#endif - goto err; - } else { - /* - * We calculate k from SHA512(private_key + H(message) + random). - * This protects the private key from a weak PRNG. - */ - if (!ossl_bn_gen_dsa_nonce_fixed_top(k, dsa->params.q, - dsa->priv_key, dgst, - dlen, ctx)) - goto err; - } - } else if (!ossl_bn_priv_rand_range_fixed_top(k, dsa->params.q, 0, ctx)) + /* + * We calculate k from SHA512(private_key + H(message) + random). + * This protects the private key from a weak PRNG. + */ + if (!BN_generate_dsa_nonce(k, dsa->params.q, dsa->priv_key, dgst, + dlen, ctx)) + goto err; + } else if (!BN_priv_rand_range_ex(k, dsa->params.q, 0, ctx)) goto err; - } while (ossl_bn_is_word_fixed_top(k, 0)); + } while (BN_is_zero(k)); BN_set_flags(k, BN_FLG_CONSTTIME); BN_set_flags(l, BN_FLG_CONSTTIME); @@ -463,6 +426,7 @@ static int dsa_do_verify(const unsigned char *dgst, int dgst_len, static int dsa_init(DSA *dsa) { dsa->flags |= DSA_FLAG_CACHE_MONT_P; + ossl_ffc_params_init(&dsa->params); dsa->dirty_cnt++; return 1; } diff --git a/openssl/src/crypto/dsa/dsa_sign.c b/openssl/src/crypto/dsa/dsa_sign.c index aab7953c0..21b0cbd5f 100644 --- a/openssl/src/crypto/dsa/dsa_sign.c +++ b/openssl/src/crypto/dsa/dsa_sign.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -34,7 +34,8 @@ int DSA_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp) DSA_SIG *DSA_SIG_new(void) { DSA_SIG *sig = OPENSSL_zalloc(sizeof(*sig)); - + if (sig == NULL) + ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE); return sig; } @@ -151,23 +152,15 @@ int DSA_SIG_set0(DSA_SIG *sig, BIGNUM *r, BIGNUM *s) } int ossl_dsa_sign_int(int type, const unsigned char *dgst, int dlen, - unsigned char *sig, unsigned int *siglen, DSA *dsa, - unsigned int nonce_type, const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq) + unsigned char *sig, unsigned int *siglen, DSA *dsa) { DSA_SIG *s; - if (sig == NULL) { - *siglen = DSA_size(dsa); - return 1; - } - /* legacy case uses the method table */ if (dsa->libctx == NULL || dsa->meth != DSA_get_default_method()) s = DSA_do_sign(dgst, dlen, dsa); else - s = ossl_dsa_do_sign_int(dgst, dlen, dsa, - nonce_type, digestname, libctx, propq); + s = ossl_dsa_do_sign_int(dgst, dlen, dsa); if (s == NULL) { *siglen = 0; return 0; @@ -180,8 +173,7 @@ int ossl_dsa_sign_int(int type, const unsigned char *dgst, int dlen, int DSA_sign(int type, const unsigned char *dgst, int dlen, unsigned char *sig, unsigned int *siglen, DSA *dsa) { - return ossl_dsa_sign_int(type, dgst, dlen, sig, siglen, dsa, - 0, NULL, NULL, NULL); + return ossl_dsa_sign_int(type, dgst, dlen, sig, siglen, dsa); } /* data has already been hashed (probably with SHA or SHA-1). */ diff --git a/openssl/src/crypto/dso/dso_dl.c b/openssl/src/crypto/dso/dso_dl.c index 451523911..f4e6e5f45 100644 --- a/openssl/src/crypto/dso/dso_dl.c +++ b/openssl/src/crypto/dso/dso_dl.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -165,16 +165,20 @@ static char *dl_merger(DSO *dso, const char *filespec1, const char *filespec2) */ if (!filespec2 || filespec1[0] == '/') { merged = OPENSSL_strdup(filespec1); - if (merged == NULL) + if (merged == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } } /* * If the first file specification is missing, the second one rules. */ else if (!filespec1) { merged = OPENSSL_strdup(filespec2); - if (merged == NULL) + if (merged == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } } else /* * This part isn't as trivial as it looks. It assumes that the @@ -194,8 +198,10 @@ static char *dl_merger(DSO *dso, const char *filespec1, const char *filespec2) len--; } merged = OPENSSL_malloc(len + 2); - if (merged == NULL) + if (merged == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } strcpy(merged, filespec2); merged[spec2len] = '/'; strcpy(&merged[spec2len + 1], filespec1); @@ -217,7 +223,7 @@ static char *dl_name_converter(DSO *dso, const char *filename) len = strlen(filename); rsize = len + 1; - transform = (strchr(filename, '/') == NULL); + transform = (strstr(filename, "/") == NULL); if (transform) { /* We will convert this to "%s.s?" or "lib%s.s?" */ rsize += strlen(DSO_EXTENSION); /* The length of ".s?" */ diff --git a/openssl/src/crypto/dso/dso_dlfcn.c b/openssl/src/crypto/dso/dso_dlfcn.c index 76737fa7b..c292b41c4 100644 --- a/openssl/src/crypto/dso/dso_dlfcn.c +++ b/openssl/src/crypto/dso/dso_dlfcn.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -207,16 +207,20 @@ static char *dlfcn_merger(DSO *dso, const char *filespec1, */ if (!filespec2 || (filespec1 != NULL && filespec1[0] == '/')) { merged = OPENSSL_strdup(filespec1); - if (merged == NULL) + if (merged == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } } /* * If the first file specification is missing, the second one rules. */ else if (!filespec1) { merged = OPENSSL_strdup(filespec2); - if (merged == NULL) + if (merged == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } } else { /* * This part isn't as trivial as it looks. It assumes that the @@ -235,8 +239,10 @@ static char *dlfcn_merger(DSO *dso, const char *filespec1, len--; } merged = OPENSSL_malloc(len + 2); - if (merged == NULL) + if (merged == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } strcpy(merged, filespec2); merged[spec2len] = '/'; strcpy(&merged[spec2len + 1], filespec1); @@ -251,7 +257,7 @@ static char *dlfcn_name_converter(DSO *dso, const char *filename) len = strlen(filename); rsize = len + 1; - transform = (strchr(filename, '/') == NULL); + transform = (strstr(filename, "/") == NULL); if (transform) { /* We will convert this to "%s.so" or "lib%s.so" etc */ rsize += strlen(DSO_EXTENSION); /* The length of ".so" */ diff --git a/openssl/src/crypto/dso/dso_lib.c b/openssl/src/crypto/dso/dso_lib.c index 8f3387e9b..9d755986d 100644 --- a/openssl/src/crypto/dso/dso_lib.c +++ b/openssl/src/crypto/dso/dso_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,17 +15,22 @@ static DSO *DSO_new_method(DSO_METHOD *meth) DSO *ret; ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } ret->meth_data = sk_void_new_null(); if (ret->meth_data == NULL) { /* sk_new doesn't generate any errors so we do */ - ERR_raise(ERR_LIB_DSO, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } ret->meth = DSO_METHOD_openssl(); - if (!CRYPTO_NEW_REF(&ret->references, 1)) { + ret->references = 1; + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); sk_void_free(ret->meth_data); OPENSSL_free(ret); return NULL; @@ -51,7 +56,7 @@ int DSO_free(DSO *dso) if (dso == NULL) return 1; - if (CRYPTO_DOWN_REF(&dso->references, &i) <= 0) + if (CRYPTO_DOWN_REF(&dso->references, &i, dso->lock) <= 0) return 0; REF_PRINT_COUNT("DSO", dso); @@ -74,7 +79,7 @@ int DSO_free(DSO *dso) sk_void_free(dso->meth_data); OPENSSL_free(dso->filename); OPENSSL_free(dso->loaded_filename); - CRYPTO_FREE_REF(&dso->references); + CRYPTO_THREAD_lock_free(dso->lock); OPENSSL_free(dso); return 1; } @@ -93,7 +98,7 @@ int DSO_up_ref(DSO *dso) return 0; } - if (CRYPTO_UP_REF(&dso->references, &i) <= 0) + if (CRYPTO_UP_REF(&dso->references, &i, dso->lock) <= 0) return 0; REF_PRINT_COUNT("DSO", dso); @@ -109,7 +114,7 @@ DSO *DSO_load(DSO *dso, const char *filename, DSO_METHOD *meth, int flags) if (dso == NULL) { ret = DSO_new_method(meth); if (ret == NULL) { - ERR_raise(ERR_LIB_DSO, ERR_R_DSO_LIB); + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); goto err; } allocated = 1; @@ -236,8 +241,10 @@ int DSO_set_filename(DSO *dso, const char *filename) } /* We'll duplicate filename */ copied = OPENSSL_strdup(filename); - if (copied == NULL) + if (copied == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return 0; + } OPENSSL_free(dso->filename); dso->filename = copied; return 1; @@ -282,8 +289,10 @@ char *DSO_convert_filename(DSO *dso, const char *filename) } if (result == NULL) { result = OPENSSL_strdup(filename); - if (result == NULL) + if (result == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } } return result; } diff --git a/openssl/src/crypto/dso/dso_local.h b/openssl/src/crypto/dso/dso_local.h index d7af0b064..f7a555216 100644 --- a/openssl/src/crypto/dso/dso_local.h +++ b/openssl/src/crypto/dso/dso_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,9 +19,8 @@ struct dso_st { DSO_METHOD *meth; /* - * Standard dlopen uses a (void *). Win32 uses a HANDLE. VMS doesn't use - * anything but will need to cache the filename for use in the dso_bind - * handler. All in all, let each method control its own destiny. + * Standard dlopen uses a (void *). Win32 uses a HANDLE. All in all, let + * each method control its own destiny. * "Handles" and such go in a STACK. */ STACK_OF(void) *meth_data; @@ -61,6 +60,7 @@ struct dso_st { * loaded. */ char *loaded_filename; + CRYPTO_RWLOCK *lock; }; struct dso_meth_st { diff --git a/openssl/src/crypto/dso/dso_vms.c b/openssl/src/crypto/dso/dso_vms.c deleted file mode 100644 index 6c84bb198..000000000 --- a/openssl/src/crypto/dso/dso_vms.c +++ /dev/null @@ -1,501 +0,0 @@ -/* - * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "dso_local.h" - -#ifdef OPENSSL_SYS_VMS - -# pragma message disable DOLLARID -# include -# include -# include -# include -# include -# include -# include -# include "../vms_rms.h" - -/* Some compiler options may mask the declaration of "_malloc32". */ -# define DSO_MALLOC OPENSSL_malloc -# if __INITIAL_POINTER_SIZE && defined _ANSI_C_SOURCE -# if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size save -# pragma pointer_size 32 -void *_malloc32(__size_t); -static void *dso_malloc(__size_t num, const char *file, int line) -{ - void *ret = _malloc32(num); - if (ret == NULL && (file != NULL || line != 0)) { - ERR_new(); - ERR_set_debug(file, line, NULL); - ERR_set_error(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE, NULL); - } - return ret; -} -# undef DSO_MALLOC -# define DSO_MALLOC(num) dso_malloc((num), OPENSSL_FILE, OPENSSL_LINE) -# pragma pointer_size restore -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ -# endif /* __INITIAL_POINTER_SIZE && defined - * _ANSI_C_SOURCE */ - -# pragma message disable DOLLARID - -static int vms_load(DSO *dso); -static int vms_unload(DSO *dso); -static DSO_FUNC_TYPE vms_bind_func(DSO *dso, const char *symname); -static char *vms_name_converter(DSO *dso, const char *filename); -static char *vms_merger(DSO *dso, const char *filespec1, - const char *filespec2); - -static DSO_METHOD dso_meth_vms = { - "OpenSSL 'VMS' shared library method", - vms_load, - NULL, /* unload */ - vms_bind_func, - NULL, /* ctrl */ - vms_name_converter, - vms_merger, - NULL, /* init */ - NULL, /* finish */ - NULL, /* pathbyaddr */ - NULL /* globallookup */ -}; - -/* - * On VMS, the only "handle" is the file name. LIB$FIND_IMAGE_SYMBOL depends - * on the reference to the file name being the same for all calls regarding - * one shared image, so we'll just store it in an instance of the following - * structure and put a pointer to that instance in the meth_data stack. - */ -typedef struct dso_internal_st { - /* - * This should contain the name only, no directory, no extension, nothing - * but a name. - */ - struct dsc$descriptor_s filename_dsc; - char filename[NAMX_MAXRSS + 1]; - /* - * This contains whatever is not in filename, if needed. Normally not - * defined. - */ - struct dsc$descriptor_s imagename_dsc; - char imagename[NAMX_MAXRSS + 1]; -} DSO_VMS_INTERNAL; - -DSO_METHOD *DSO_METHOD_openssl(void) -{ - return &dso_meth_vms; -} - -static int vms_load(DSO *dso) -{ - void *ptr = NULL; - /* See applicable comments in dso_dl.c */ - char *filename = DSO_convert_filename(dso, NULL); - -/* Ensure 32-bit pointer for "p", and appropriate malloc() function. */ -# if __INITIAL_POINTER_SIZE && defined _ANSI_C_SOURCE -# if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size save -# pragma pointer_size 32 -# endif /* __INITIAL_POINTER_SIZE == 64 */ -# endif /* __INITIAL_POINTER_SIZE && defined - * _ANSI_C_SOURCE */ - - DSO_VMS_INTERNAL *p = NULL; - -# if __INITIAL_POINTER_SIZE && defined _ANSI_C_SOURCE -# if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size restore -# endif /* __INITIAL_POINTER_SIZE == 64 */ -# endif /* __INITIAL_POINTER_SIZE && defined - * _ANSI_C_SOURCE */ - - const char *sp1, *sp2; /* Search result */ - const char *ext = NULL; /* possible extension to add */ - - if (filename == NULL) { - ERR_raise(ERR_LIB_DSO, DSO_R_NO_FILENAME); - goto err; - } - - /*- - * A file specification may look like this: - * - * node::dev:[dir-spec]name.type;ver - * - * or (for compatibility with TOPS-20): - * - * node::dev:name.type;ver - * - * and the dir-spec uses '.' as separator. Also, a dir-spec - * may consist of several parts, with mixed use of [] and <>: - * - * [dir1.] - * - * We need to split the file specification into the name and - * the rest (both before and after the name itself). - */ - /* - * Start with trying to find the end of a dir-spec, and save the position - * of the byte after in sp1 - */ - sp1 = strrchr(filename, ']'); - sp2 = strrchr(filename, '>'); - if (sp1 == NULL) - sp1 = sp2; - if (sp2 != NULL && sp2 > sp1) - sp1 = sp2; - if (sp1 == NULL) - sp1 = strrchr(filename, ':'); - if (sp1 == NULL) - sp1 = filename; - else - sp1++; /* The byte after the found character */ - /* Now, let's see if there's a type, and save the position in sp2 */ - sp2 = strchr(sp1, '.'); - /* - * If there is a period and the next character is a semi-colon, - * we need to add an extension - */ - if (sp2 != NULL && sp2[1] == ';') - ext = ".EXE"; - /* - * If we found it, that's where we'll cut. Otherwise, look for a version - * number and save the position in sp2 - */ - if (sp2 == NULL) { - sp2 = strchr(sp1, ';'); - ext = ".EXE"; - } - /* - * If there was still nothing to find, set sp2 to point at the end of the - * string - */ - if (sp2 == NULL) - sp2 = sp1 + strlen(sp1); - - /* Check that we won't get buffer overflows */ - if (sp2 - sp1 > FILENAME_MAX - || (sp1 - filename) + strlen(sp2) > FILENAME_MAX) { - ERR_raise(ERR_LIB_DSO, DSO_R_FILENAME_TOO_BIG); - goto err; - } - - p = DSO_MALLOC(sizeof(*p)); - if (p == NULL) - goto err; - - strncpy(p->filename, sp1, sp2 - sp1); - p->filename[sp2 - sp1] = '\0'; - - strncpy(p->imagename, filename, sp1 - filename); - p->imagename[sp1 - filename] = '\0'; - if (ext) { - strcat(p->imagename, ext); - if (*sp2 == '.') - sp2++; - } - strcat(p->imagename, sp2); - - p->filename_dsc.dsc$w_length = strlen(p->filename); - p->filename_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - p->filename_dsc.dsc$b_class = DSC$K_CLASS_S; - p->filename_dsc.dsc$a_pointer = p->filename; - p->imagename_dsc.dsc$w_length = strlen(p->imagename); - p->imagename_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - p->imagename_dsc.dsc$b_class = DSC$K_CLASS_S; - p->imagename_dsc.dsc$a_pointer = p->imagename; - - if (!sk_void_push(dso->meth_data, (char *)p)) { - ERR_raise(ERR_LIB_DSO, DSO_R_STACK_ERROR); - goto err; - } - - /* Success (for now, we lie. We actually do not know...) */ - dso->loaded_filename = filename; - return 1; - err: - /* Cleanup! */ - OPENSSL_free(p); - OPENSSL_free(filename); - return 0; -} - -/* - * Note that this doesn't actually unload the shared image, as there is no - * such thing in VMS. Next time it get loaded again, a new copy will - * actually be loaded. - */ -static int vms_unload(DSO *dso) -{ - DSO_VMS_INTERNAL *p; - if (dso == NULL) { - ERR_raise(ERR_LIB_DSO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (sk_void_num(dso->meth_data) < 1) - return 1; - p = (DSO_VMS_INTERNAL *)sk_void_pop(dso->meth_data); - if (p == NULL) { - ERR_raise(ERR_LIB_DSO, DSO_R_NULL_HANDLE); - return 0; - } - /* Cleanup */ - OPENSSL_free(p); - return 1; -} - -/* - * We must do this in a separate function because of the way the exception - * handler works (it makes this function return - */ -static int do_find_symbol(DSO_VMS_INTERNAL *ptr, - struct dsc$descriptor_s *symname_dsc, void **sym, - unsigned long flags) -{ - /* - * Make sure that signals are caught and returned instead of aborting the - * program. The exception handler gets unestablished automatically on - * return from this function. - */ - lib$establish(lib$sig_to_ret); - - if (ptr->imagename_dsc.dsc$w_length) - return lib$find_image_symbol(&ptr->filename_dsc, - symname_dsc, sym, - &ptr->imagename_dsc, flags); - else - return lib$find_image_symbol(&ptr->filename_dsc, - symname_dsc, sym, 0, flags); -} - -# ifndef LIB$M_FIS_MIXEDCASE -# define LIB$M_FIS_MIXEDCASE (1 << 4); -# endif -void vms_bind_sym(DSO *dso, const char *symname, void **sym) -{ - DSO_VMS_INTERNAL *ptr; - int status = 0; - struct dsc$descriptor_s symname_dsc; - -/* Arrange 32-bit pointer to (copied) string storage, if needed. */ -# if __INITIAL_POINTER_SIZE == 64 -# define SYMNAME symname_32p -# pragma pointer_size save -# pragma pointer_size 32 - char *symname_32p; -# pragma pointer_size restore - char symname_32[NAMX_MAXRSS + 1]; -# else /* __INITIAL_POINTER_SIZE == 64 */ -# define SYMNAME ((char *) symname) -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ - - *sym = NULL; - - if ((dso == NULL) || (symname == NULL)) { - ERR_raise(ERR_LIB_DSO, ERR_R_PASSED_NULL_PARAMETER); - return; - } -# if __INITIAL_POINTER_SIZE == 64 - /* Copy the symbol name to storage with a 32-bit pointer. */ - symname_32p = symname_32; - strcpy(symname_32p, symname); -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ - - symname_dsc.dsc$w_length = strlen(SYMNAME); - symname_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - symname_dsc.dsc$b_class = DSC$K_CLASS_S; - symname_dsc.dsc$a_pointer = SYMNAME; - - if (sk_void_num(dso->meth_data) < 1) { - ERR_raise(ERR_LIB_DSO, DSO_R_STACK_ERROR); - return; - } - ptr = (DSO_VMS_INTERNAL *)sk_void_value(dso->meth_data, - sk_void_num(dso->meth_data) - 1); - if (ptr == NULL) { - ERR_raise(ERR_LIB_DSO, DSO_R_NULL_HANDLE); - return; - } - - status = do_find_symbol(ptr, &symname_dsc, sym, LIB$M_FIS_MIXEDCASE); - - if (!$VMS_STATUS_SUCCESS(status)) - status = do_find_symbol(ptr, &symname_dsc, sym, 0); - - if (!$VMS_STATUS_SUCCESS(status)) { - unsigned short length; - char errstring[257]; - struct dsc$descriptor_s errstring_dsc; - - errstring_dsc.dsc$w_length = sizeof(errstring); - errstring_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - errstring_dsc.dsc$b_class = DSC$K_CLASS_S; - errstring_dsc.dsc$a_pointer = errstring; - - *sym = NULL; - - status = sys$getmsg(status, &length, &errstring_dsc, 1, 0); - - if (!$VMS_STATUS_SUCCESS(status)) - lib$signal(status); /* This is really bad. Abort! */ - else { - errstring[length] = '\0'; - - if (ptr->imagename_dsc.dsc$w_length) - ERR_raise_data(ERR_LIB_DSO, DSO_R_SYM_FAILURE, - "Symbol %s in %s (%s): %s", - symname, ptr->filename, ptr->imagename, - errstring); - else - ERR_raise_data(ERR_LIB_DSO, DSO_R_SYM_FAILURE, - "Symbol %s in %s: %s", - symname, ptr->filename, errstring); - } - return; - } - return; -} - -static DSO_FUNC_TYPE vms_bind_func(DSO *dso, const char *symname) -{ - DSO_FUNC_TYPE sym = 0; - vms_bind_sym(dso, symname, (void **)&sym); - return sym; -} - -static char *vms_merger(DSO *dso, const char *filespec1, - const char *filespec2) -{ - int status; - int filespec1len, filespec2len; - struct FAB fab; - struct NAMX_STRUCT nam; - char esa[NAMX_MAXRSS + 1]; - char *merged; - -/* Arrange 32-bit pointer to (copied) string storage, if needed. */ -# if __INITIAL_POINTER_SIZE == 64 -# define FILESPEC1 filespec1_32p; -# define FILESPEC2 filespec2_32p; -# pragma pointer_size save -# pragma pointer_size 32 - char *filespec1_32p; - char *filespec2_32p; -# pragma pointer_size restore - char filespec1_32[NAMX_MAXRSS + 1]; - char filespec2_32[NAMX_MAXRSS + 1]; -# else /* __INITIAL_POINTER_SIZE == 64 */ -# define FILESPEC1 ((char *) filespec1) -# define FILESPEC2 ((char *) filespec2) -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ - - if (!filespec1) - filespec1 = ""; - if (!filespec2) - filespec2 = ""; - filespec1len = strlen(filespec1); - filespec2len = strlen(filespec2); - -# if __INITIAL_POINTER_SIZE == 64 - /* Copy the file names to storage with a 32-bit pointer. */ - filespec1_32p = filespec1_32; - filespec2_32p = filespec2_32; - strcpy(filespec1_32p, filespec1); - strcpy(filespec2_32p, filespec2); -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ - - fab = cc$rms_fab; - nam = CC_RMS_NAMX; - - FAB_OR_NAML(fab, nam).FAB_OR_NAML_FNA = FILESPEC1; - FAB_OR_NAML(fab, nam).FAB_OR_NAML_FNS = filespec1len; - FAB_OR_NAML(fab, nam).FAB_OR_NAML_DNA = FILESPEC2; - FAB_OR_NAML(fab, nam).FAB_OR_NAML_DNS = filespec2len; - NAMX_DNA_FNA_SET(fab) - - nam.NAMX_ESA = esa; - nam.NAMX_ESS = NAMX_MAXRSS; - nam.NAMX_NOP = NAM$M_SYNCHK | NAM$M_PWD; - SET_NAMX_NO_SHORT_UPCASE(nam); - - fab.FAB_NAMX = &nam; - - status = sys$parse(&fab, 0, 0); - - if (!$VMS_STATUS_SUCCESS(status)) { - unsigned short length; - char errstring[257]; - struct dsc$descriptor_s errstring_dsc; - - errstring_dsc.dsc$w_length = sizeof(errstring); - errstring_dsc.dsc$b_dtype = DSC$K_DTYPE_T; - errstring_dsc.dsc$b_class = DSC$K_CLASS_S; - errstring_dsc.dsc$a_pointer = errstring; - - status = sys$getmsg(status, &length, &errstring_dsc, 1, 0); - - if (!$VMS_STATUS_SUCCESS(status)) - lib$signal(status); /* This is really bad. Abort! */ - else { - errstring[length] = '\0'; - - ERR_raise_data(ERR_LIB_DSO, DSO_R_FAILURE, - "filespec \"%s\", default \"%s\": %s", - filespec1, filespec2, errstring); - } - return NULL; - } - - merged = OPENSSL_malloc(nam.NAMX_ESL + 1); - if (merged == NULL) - return NULL; - strncpy(merged, nam.NAMX_ESA, nam.NAMX_ESL); - merged[nam.NAMX_ESL] = '\0'; - return merged; -} - -static char *vms_name_converter(DSO *dso, const char *filename) -{ - char *translated; - int len, transform; - const char *p; - - len = strlen(filename); - - p = strchr(filename, ':'); - if (p != NULL) { - transform = 0; - } else { - p = filename; - transform = (strrchr(p, '>') == NULL && strrchr(p, ']') == NULL); - } - - if (transform) { - int rsize = len + sizeof(DSO_EXTENSION); - - if ((translated = OPENSSL_malloc(rsize)) != NULL) { - p = strrchr(filename, ';'); - if (p != NULL) - len = p - filename; - strncpy(translated, filename, len); - translated[len] = '\0'; - strcat(translated, DSO_EXTENSION); - if (p != NULL) - strcat(translated, p); - } - } else { - translated = OPENSSL_strdup(filename); - } - return translated; -} - -#endif /* OPENSSL_SYS_VMS */ diff --git a/openssl/src/crypto/dso/dso_win32.c b/openssl/src/crypto/dso/dso_win32.c index 43210e3d9..20fa3dce7 100644 --- a/openssl/src/crypto/dso/dso_win32.c +++ b/openssl/src/crypto/dso/dso_win32.c @@ -110,8 +110,10 @@ static int win32_load(DSO *dso) goto err; } p = OPENSSL_malloc(sizeof(*p)); - if (p == NULL) + if (p == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); goto err; + } *p = h; if (!sk_void_push(dso->meth_data, p)) { ERR_raise(ERR_LIB_DSO, DSO_R_STACK_ERROR); @@ -212,8 +214,10 @@ static struct file_st *win32_splitter(DSO *dso, const char *filename, } result = OPENSSL_zalloc(sizeof(*result)); - if (result == NULL) + if (result == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } position = IN_DEVICE; @@ -329,8 +333,10 @@ static char *win32_joiner(DSO *dso, const struct file_st *file_split) } result = OPENSSL_malloc(len + 1); - if (result == NULL) + if (result == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } if (file_split->node) { strcpy(&result[offset], "\\\\"); @@ -393,21 +399,25 @@ static char *win32_merger(DSO *dso, const char *filespec1, } if (!filespec2) { merged = OPENSSL_strdup(filespec1); - if (merged == NULL) + if (merged == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } } else if (!filespec1) { merged = OPENSSL_strdup(filespec2); - if (merged == NULL) + if (merged == NULL) { + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; + } } else { filespec1_split = win32_splitter(dso, filespec1, 0); if (!filespec1_split) { - ERR_raise(ERR_LIB_DSO, ERR_R_DSO_LIB); + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); return NULL; } filespec2_split = win32_splitter(dso, filespec2, 1); if (!filespec2_split) { - ERR_raise(ERR_LIB_DSO, ERR_R_DSO_LIB); + ERR_raise(ERR_LIB_DSO, ERR_R_MALLOC_FAILURE); OPENSSL_free(filespec1_split); return NULL; } diff --git a/openssl/src/crypto/ec/curve25519.c b/openssl/src/crypto/ec/curve25519.c index 6287eb93b..5271f4ed2 100644 --- a/openssl/src/crypto/ec/curve25519.c +++ b/openssl/src/crypto/ec/curve25519.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -1868,7 +1868,7 @@ static int ge_frombytes_vartime(ge_p3 *h, const uint8_t *s) { fe u; fe v; - fe w; + fe v3; fe vxx; fe check; @@ -1879,10 +1879,15 @@ static int ge_frombytes_vartime(ge_p3 *h, const uint8_t *s) fe_sub(u, u, h->Z); /* u = y^2-1 */ fe_add(v, v, h->Z); /* v = dy^2+1 */ - fe_mul(w, u, v); /* w = u*v */ + fe_sq(v3, v); + fe_mul(v3, v3, v); /* v3 = v^3 */ + fe_sq(h->X, v3); + fe_mul(h->X, h->X, v); + fe_mul(h->X, h->X, u); /* x = uv^7 */ - fe_pow22523(h->X, w); /* x = w^((q-5)/8) */ - fe_mul(h->X, h->X, u); /* x = u * w^((q-5)/8) */ + fe_pow22523(h->X, h->X); /* x = (uv^7)^((q-5)/8) */ + fe_mul(h->X, h->X, v3); + fe_mul(h->X, h->X, u); /* x = uv^3(uv^7)^((q-5)/8) */ fe_sq(vxx, h->X); fe_mul(vxx, vxx, v); @@ -5434,47 +5439,9 @@ static void sc_muladd(uint8_t *s, const uint8_t *a, const uint8_t *b, s[31] = (uint8_t) (s11 >> 17); } -static int hash_init_with_dom(EVP_MD_CTX *hash_ctx, - EVP_MD *sha512, - const uint8_t dom2flag, - const uint8_t phflag, - const uint8_t *context, - const size_t context_len) -{ - /* ASCII: "SigEd25519 no Ed25519 collisions", in hex for EBCDIC compatibility */ - const char dom_s[] = - "\x53\x69\x67\x45\x64\x32\x35\x35\x31\x39\x20\x6e" - "\x6f\x20\x45\x64\x32\x35\x35\x31\x39\x20\x63\x6f" - "\x6c\x6c\x69\x73\x69\x6f\x6e\x73"; - uint8_t dom[2]; - - if (!EVP_DigestInit_ex(hash_ctx, sha512, NULL)) - return 0; - - /* return early if dom2flag is not set */ - if (!dom2flag) - return 1; - - if (context_len > UINT8_MAX) - return 0; - - dom[0] = (uint8_t)(phflag >= 1 ? 1 : 0); - dom[1] = (uint8_t)context_len; - - if (!EVP_DigestUpdate(hash_ctx, dom_s, sizeof(dom_s)-1) - || !EVP_DigestUpdate(hash_ctx, dom, sizeof(dom)) - || !EVP_DigestUpdate(hash_ctx, context, context_len)) { - return 0; - } - - return 1; -} - int -ossl_ed25519_sign(uint8_t *out_sig, const uint8_t *tbs, size_t tbs_len, +ossl_ed25519_sign(uint8_t *out_sig, const uint8_t *message, size_t message_len, const uint8_t public_key[32], const uint8_t private_key[32], - const uint8_t dom2flag, const uint8_t phflag, const uint8_t csflag, - const uint8_t *context, size_t context_len, OSSL_LIB_CTX *libctx, const char *propq) { uint8_t az[SHA512_DIGEST_LENGTH]; @@ -5486,17 +5453,6 @@ ossl_ed25519_sign(uint8_t *out_sig, const uint8_t *tbs, size_t tbs_len, unsigned int sz; int res = 0; - if (context == NULL) - context_len = 0; - - /* if csflag is set, then a non-empty context-string is required */ - if (csflag && context_len == 0) - goto err; - - /* if dom2flag is not set, then an empty context-string is required */ - if (!dom2flag && context_len > 0) - goto err; - if (sha512 == NULL || hash_ctx == NULL) goto err; @@ -5509,9 +5465,9 @@ ossl_ed25519_sign(uint8_t *out_sig, const uint8_t *tbs, size_t tbs_len, az[31] &= 63; az[31] |= 64; - if (!hash_init_with_dom(hash_ctx, sha512, dom2flag, phflag, context, context_len) + if (!EVP_DigestInit_ex(hash_ctx, sha512, NULL) || !EVP_DigestUpdate(hash_ctx, az + 32, 32) - || !EVP_DigestUpdate(hash_ctx, tbs, tbs_len) + || !EVP_DigestUpdate(hash_ctx, message, message_len) || !EVP_DigestFinal_ex(hash_ctx, nonce, &sz)) goto err; @@ -5519,10 +5475,10 @@ ossl_ed25519_sign(uint8_t *out_sig, const uint8_t *tbs, size_t tbs_len, ge_scalarmult_base(&R, nonce); ge_p3_tobytes(out_sig, &R); - if (!hash_init_with_dom(hash_ctx, sha512, dom2flag, phflag, context, context_len) + if (!EVP_DigestInit_ex(hash_ctx, sha512, NULL) || !EVP_DigestUpdate(hash_ctx, out_sig, 32) || !EVP_DigestUpdate(hash_ctx, public_key, 32) - || !EVP_DigestUpdate(hash_ctx, tbs, tbs_len) + || !EVP_DigestUpdate(hash_ctx, message, message_len) || !EVP_DigestFinal_ex(hash_ctx, hram, &sz)) goto err; @@ -5541,10 +5497,8 @@ ossl_ed25519_sign(uint8_t *out_sig, const uint8_t *tbs, size_t tbs_len, static const char allzeroes[15]; int -ossl_ed25519_verify(const uint8_t *tbs, size_t tbs_len, +ossl_ed25519_verify(const uint8_t *message, size_t message_len, const uint8_t signature[64], const uint8_t public_key[32], - const uint8_t dom2flag, const uint8_t phflag, const uint8_t csflag, - const uint8_t *context, size_t context_len, OSSL_LIB_CTX *libctx, const char *propq) { int i; @@ -5563,17 +5517,6 @@ ossl_ed25519_verify(const uint8_t *tbs, size_t tbs_len, 0xDE, 0xF9, 0xDE, 0x14 }; - if (context == NULL) - context_len = 0; - - /* if csflag is set, then a non-empty context-string is required */ - if (csflag && context_len == 0) - return 0; - - /* if dom2flag is not set, then an empty context-string is required */ - if (!dom2flag && context_len > 0) - return 0; - r = signature; s = signature + 32; @@ -5618,10 +5561,10 @@ ossl_ed25519_verify(const uint8_t *tbs, size_t tbs_len, if (hash_ctx == NULL) goto err; - if (!hash_init_with_dom(hash_ctx, sha512, dom2flag, phflag, context, context_len) + if (!EVP_DigestInit_ex(hash_ctx, sha512, NULL) || !EVP_DigestUpdate(hash_ctx, r, 32) || !EVP_DigestUpdate(hash_ctx, public_key, 32) - || !EVP_DigestUpdate(hash_ctx, tbs, tbs_len) + || !EVP_DigestUpdate(hash_ctx, message, message_len) || !EVP_DigestFinal_ex(hash_ctx, h, &sz)) goto err; @@ -5632,14 +5575,6 @@ ossl_ed25519_verify(const uint8_t *tbs, size_t tbs_len, ge_tobytes(rcheck, &R); res = CRYPTO_memcmp(rcheck, r, sizeof(rcheck)) == 0; - - /* note that we have used the strict verification equation here. - * we checked that ENC( [h](-A) + [s]B ) == r - * B is the base point. - * - * the less strict verification equation uses the curve cofactor: - * [h*8](-A) + [s*8]B == [8]R - */ err: EVP_MD_free(sha512); EVP_MD_CTX_free(hash_ctx); diff --git a/openssl/src/crypto/ec/curve448/arch_32/f_impl32.c b/openssl/src/crypto/ec/curve448/arch_32/f_impl32.c index 140c73c64..14f7b786f 100644 --- a/openssl/src/crypto/ec/curve448/arch_32/f_impl32.c +++ b/openssl/src/crypto/ec/curve448/arch_32/f_impl32.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2014 Cryptography Research, Inc. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -21,7 +21,7 @@ NON_EMPTY_TRANSLATION_UNIT # include "../field.h" -void ossl_gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) +void gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) { const uint32_t *a = as->limb, *b = bs->limb; uint32_t *c = cs->limb; @@ -70,7 +70,7 @@ void ossl_gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) c[1] += ((uint32_t)(accum1)); } -void ossl_gf_mulw_unsigned(gf_s * RESTRICT cs, const gf as, uint32_t b) +void gf_mulw_unsigned(gf_s * RESTRICT cs, const gf as, uint32_t b) { const uint32_t *a = as->limb; uint32_t *c = cs->limb; @@ -98,8 +98,8 @@ void ossl_gf_mulw_unsigned(gf_s * RESTRICT cs, const gf as, uint32_t b) c[1] += (uint32_t)(accum8 >> 28); } -void ossl_gf_sqr(gf_s * RESTRICT cs, const gf as) +void gf_sqr(gf_s * RESTRICT cs, const gf as) { - ossl_gf_mul(cs, as, as); /* Performs better with a dedicated square */ + gf_mul(cs, as, as); /* Performs better with a dedicated square */ } #endif diff --git a/openssl/src/crypto/ec/curve448/arch_64/f_impl64.c b/openssl/src/crypto/ec/curve448/arch_64/f_impl64.c index 06cc33a96..10a9b065e 100644 --- a/openssl/src/crypto/ec/curve448/arch_64/f_impl64.c +++ b/openssl/src/crypto/ec/curve448/arch_64/f_impl64.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2014 Cryptography Research, Inc. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -21,7 +21,7 @@ NON_EMPTY_TRANSLATION_UNIT # include "../field.h" -void ossl_gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) +void gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) { const uint64_t *a = as->limb, *b = bs->limb; uint64_t *c = cs->limb; @@ -45,9 +45,9 @@ void ossl_gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) accum0 += widemul(a[j + 4], b[i - j + 4]); } for (; j < 4; j++) { - accum2 += widemul(a[j], b[i + 8 - j]); - accum1 += widemul(aa[j], bbb[i + 4 - j]); - accum0 += widemul(a[j + 4], bb[i + 4 - j]); + accum2 += widemul(a[j], b[i - j + 8]); + accum1 += widemul(aa[j], bbb[i - j + 4]); + accum0 += widemul(a[j + 4], bb[i - j + 4]); } accum1 -= accum2; @@ -73,7 +73,7 @@ void ossl_gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) c[1] += ((uint64_t)(accum1)); } -void ossl_gf_mulw_unsigned(gf_s * RESTRICT cs, const gf as, uint32_t b) +void gf_mulw_unsigned(gf_s * RESTRICT cs, const gf as, uint32_t b) { const uint64_t *a = as->limb; uint64_t *c = cs->limb; @@ -99,7 +99,7 @@ void ossl_gf_mulw_unsigned(gf_s * RESTRICT cs, const gf as, uint32_t b) c[1] += accum4 >> 56; } -void ossl_gf_sqr(gf_s * RESTRICT cs, const gf as) +void gf_sqr(gf_s * RESTRICT cs, const gf as) { const uint64_t *a = as->limb; uint64_t *c = cs->limb; diff --git a/openssl/src/crypto/ec/curve448/curve448.c b/openssl/src/crypto/ec/curve448/curve448.c index 2422d068a..6928d9693 100644 --- a/openssl/src/crypto/ec/curve448/curve448.c +++ b/openssl/src/crypto/ec/curve448/curve448.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2015-2016 Cryptography Research, Inc. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -44,13 +44,13 @@ static void gf_invert(gf y, const gf x, int assert_nonzero) mask_t ret; gf t1, t2; - ossl_gf_sqr(t1, x); /* o^2 */ + gf_sqr(t1, x); /* o^2 */ ret = gf_isr(t2, t1); /* +-1/sqrt(o^2) = +-1/o */ (void)ret; if (assert_nonzero) assert(ret); - ossl_gf_sqr(t1, t2); - ossl_gf_mul(t2, t1, x); /* not direct to y in case of alias. */ + gf_sqr(t1, t2); + gf_mul(t2, t1, x); /* not direct to y in case of alias. */ gf_copy(y, t2); } @@ -63,23 +63,23 @@ static void point_double_internal(curve448_point_t p, const curve448_point_t q, { gf a, b, c, d; - ossl_gf_sqr(c, q->x); - ossl_gf_sqr(a, q->y); + gf_sqr(c, q->x); + gf_sqr(a, q->y); gf_add_nr(d, c, a); /* 2+e */ gf_add_nr(p->t, q->y, q->x); /* 2+e */ - ossl_gf_sqr(b, p->t); + gf_sqr(b, p->t); gf_subx_nr(b, b, d, 3); /* 4+e */ gf_sub_nr(p->t, a, c); /* 3+e */ - ossl_gf_sqr(p->x, q->z); + gf_sqr(p->x, q->z); gf_add_nr(p->z, p->x, p->x); /* 2+e */ gf_subx_nr(a, p->z, p->t, 4); /* 6+e */ if (GF_HEADROOM == 5) gf_weak_reduce(a); /* or 1+e */ - ossl_gf_mul(p->x, a, b); - ossl_gf_mul(p->z, p->t, a); - ossl_gf_mul(p->y, p->t, d); + gf_mul(p->x, a, b); + gf_mul(p->z, p->t, a); + gf_mul(p->y, p->t, d); if (!before_double) - ossl_gf_mul(p->t, b, d); + gf_mul(p->t, b, d); } void ossl_curve448_point_double(curve448_point_t p, const curve448_point_t q) @@ -108,17 +108,17 @@ static void pniels_to_pt(curve448_point_t e, const pniels_t d) gf_add(eu, d->n->b, d->n->a); gf_sub(e->y, d->n->b, d->n->a); - ossl_gf_mul(e->t, e->y, eu); - ossl_gf_mul(e->x, d->z, e->y); - ossl_gf_mul(e->y, d->z, eu); - ossl_gf_sqr(e->z, d->z); + gf_mul(e->t, e->y, eu); + gf_mul(e->x, d->z, e->y); + gf_mul(e->y, d->z, eu); + gf_sqr(e->z, d->z); } static void niels_to_pt(curve448_point_t e, const niels_t n) { gf_add(e->y, n->b, n->a); gf_sub(e->x, n->b, n->a); - ossl_gf_mul(e->t, e->y, e->x); + gf_mul(e->t, e->y, e->x); gf_copy(e->z, ONE); } @@ -128,19 +128,19 @@ static void add_niels_to_pt(curve448_point_t d, const niels_t e, gf a, b, c; gf_sub_nr(b, d->y, d->x); /* 3+e */ - ossl_gf_mul(a, e->a, b); + gf_mul(a, e->a, b); gf_add_nr(b, d->x, d->y); /* 2+e */ - ossl_gf_mul(d->y, e->b, b); - ossl_gf_mul(d->x, e->c, d->t); + gf_mul(d->y, e->b, b); + gf_mul(d->x, e->c, d->t); gf_add_nr(c, a, d->y); /* 2+e */ gf_sub_nr(b, d->y, a); /* 3+e */ gf_sub_nr(d->y, d->z, d->x); /* 3+e */ gf_add_nr(a, d->x, d->z); /* 2+e */ - ossl_gf_mul(d->z, a, d->y); - ossl_gf_mul(d->x, d->y, b); - ossl_gf_mul(d->y, a, c); + gf_mul(d->z, a, d->y); + gf_mul(d->x, d->y, b); + gf_mul(d->y, a, c); if (!before_double) - ossl_gf_mul(d->t, b, c); + gf_mul(d->t, b, c); } static void sub_niels_from_pt(curve448_point_t d, const niels_t e, @@ -149,19 +149,19 @@ static void sub_niels_from_pt(curve448_point_t d, const niels_t e, gf a, b, c; gf_sub_nr(b, d->y, d->x); /* 3+e */ - ossl_gf_mul(a, e->b, b); + gf_mul(a, e->b, b); gf_add_nr(b, d->x, d->y); /* 2+e */ - ossl_gf_mul(d->y, e->a, b); - ossl_gf_mul(d->x, e->c, d->t); + gf_mul(d->y, e->a, b); + gf_mul(d->x, e->c, d->t); gf_add_nr(c, a, d->y); /* 2+e */ gf_sub_nr(b, d->y, a); /* 3+e */ gf_add_nr(d->y, d->z, d->x); /* 2+e */ gf_sub_nr(a, d->z, d->x); /* 3+e */ - ossl_gf_mul(d->z, a, d->y); - ossl_gf_mul(d->x, d->y, b); - ossl_gf_mul(d->y, a, c); + gf_mul(d->z, a, d->y); + gf_mul(d->x, d->y, b); + gf_mul(d->y, a, c); if (!before_double) - ossl_gf_mul(d->t, b, c); + gf_mul(d->t, b, c); } static void add_pniels_to_pt(curve448_point_t p, const pniels_t pn, @@ -169,7 +169,7 @@ static void add_pniels_to_pt(curve448_point_t p, const pniels_t pn, { gf L0; - ossl_gf_mul(L0, p->z, pn->z); + gf_mul(L0, p->z, pn->z); gf_copy(p->z, L0); add_niels_to_pt(p, pn->n, before_double); } @@ -179,7 +179,7 @@ static void sub_pniels_from_pt(curve448_point_t p, const pniels_t pn, { gf L0; - ossl_gf_mul(L0, p->z, pn->z); + gf_mul(L0, p->z, pn->z); gf_copy(p->z, L0); sub_niels_from_pt(p, pn->n, before_double); } @@ -192,8 +192,8 @@ ossl_curve448_point_eq(const curve448_point_t p, gf a, b; /* equality mod 2-torsion compares x/y */ - ossl_gf_mul(a, p->y, q->x); - ossl_gf_mul(b, q->y, p->x); + gf_mul(a, p->y, q->x); + gf_mul(b, q->y, p->x); succ = gf_eq(a, b); return mask_to_bool(succ); @@ -205,15 +205,15 @@ ossl_curve448_point_valid(const curve448_point_t p) mask_t out; gf a, b, c; - ossl_gf_mul(a, p->x, p->y); - ossl_gf_mul(b, p->z, p->t); + gf_mul(a, p->x, p->y); + gf_mul(b, p->z, p->t); out = gf_eq(a, b); - ossl_gf_sqr(a, p->x); - ossl_gf_sqr(b, p->y); + gf_sqr(a, p->x); + gf_sqr(b, p->y); gf_sub(a, b, a); - ossl_gf_sqr(b, p->t); + gf_sqr(b, p->t); gf_mulw(c, b, TWISTED_D); - ossl_gf_sqr(b, p->z); + gf_sqr(b, p->z); gf_add(b, b, c); out &= gf_eq(a, b); out &= ~gf_eq(p->z, ZERO); @@ -221,7 +221,7 @@ ossl_curve448_point_valid(const curve448_point_t p) } static ossl_inline void constant_time_lookup_niels(niels_s * RESTRICT ni, - const niels_t *table, + const niels_t * table, int nelts, int idx) { constant_time_lookup(ni, table, sizeof(niels_s), nelts, idx); @@ -229,7 +229,7 @@ static ossl_inline void constant_time_lookup_niels(niels_s * RESTRICT ni, void ossl_curve448_precomputed_scalarmul(curve448_point_t out, - const curve448_precomputed_s *table, + const curve448_precomputed_s * table, const curve448_scalar_t scalar) { unsigned int i, j, k; @@ -290,26 +290,26 @@ ossl_curve448_point_mul_by_ratio_and_encode_like_eddsa( /* 4-isogeny: 2xy/(y^+x^2), (y^2-x^2)/(2z^2-y^2+x^2) */ gf u; - ossl_gf_sqr(x, q->x); - ossl_gf_sqr(t, q->y); + gf_sqr(x, q->x); + gf_sqr(t, q->y); gf_add(u, x, t); gf_add(z, q->y, q->x); - ossl_gf_sqr(y, z); + gf_sqr(y, z); gf_sub(y, y, u); gf_sub(z, t, x); - ossl_gf_sqr(x, q->z); + gf_sqr(x, q->z); gf_add(t, x, x); gf_sub(t, t, z); - ossl_gf_mul(x, t, y); - ossl_gf_mul(y, z, u); - ossl_gf_mul(z, u, t); + gf_mul(x, t, y); + gf_mul(y, z, u); + gf_mul(z, u, t); OPENSSL_cleanse(u, sizeof(u)); } /* Affinize */ gf_invert(z, z, 1); - ossl_gf_mul(t, x, z); - ossl_gf_mul(x, y, z); + gf_mul(t, x, z); + gf_mul(x, y, z); /* Encode */ enc[EDDSA_448_PRIVATE_BYTES - 1] = 0; @@ -340,15 +340,15 @@ ossl_curve448_point_decode_like_eddsa_and_mul_by_ratio( succ = gf_deserialize(p->y, enc2, 1, 0); succ &= word_is_zero(enc2[EDDSA_448_PRIVATE_BYTES - 1]); - ossl_gf_sqr(p->x, p->y); + gf_sqr(p->x, p->y); gf_sub(p->z, ONE, p->x); /* num = 1-y^2 */ gf_mulw(p->t, p->x, EDWARDS_D); /* dy^2 */ gf_sub(p->t, ONE, p->t); /* denom = 1-dy^2 or 1-d + dy^2 */ - ossl_gf_mul(p->x, p->z, p->t); + gf_mul(p->x, p->z, p->t); succ &= gf_isr(p->t, p->x); /* 1/sqrt(num * denom) */ - ossl_gf_mul(p->x, p->t, p->z); /* sqrt(num / denom) */ + gf_mul(p->x, p->t, p->z); /* sqrt(num / denom) */ gf_cond_neg(p->x, gf_lobit(p->x) ^ low); gf_copy(p->z, ONE); @@ -356,20 +356,20 @@ ossl_curve448_point_decode_like_eddsa_and_mul_by_ratio( gf a, b, c, d; /* 4-isogeny 2xy/(y^2-ax^2), (y^2+ax^2)/(2-y^2-ax^2) */ - ossl_gf_sqr(c, p->x); - ossl_gf_sqr(a, p->y); + gf_sqr(c, p->x); + gf_sqr(a, p->y); gf_add(d, c, a); gf_add(p->t, p->y, p->x); - ossl_gf_sqr(b, p->t); + gf_sqr(b, p->t); gf_sub(b, b, d); gf_sub(p->t, a, c); - ossl_gf_sqr(p->x, p->z); + gf_sqr(p->x, p->z); gf_add(p->z, p->x, p->x); gf_sub(a, p->z, d); - ossl_gf_mul(p->x, a, b); - ossl_gf_mul(p->z, p->t, a); - ossl_gf_mul(p->y, p->t, d); - ossl_gf_mul(p->t, b, d); + gf_mul(p->x, a, b); + gf_mul(p->z, p->t, a); + gf_mul(p->y, p->t, d); + gf_mul(p->t, b, d); OPENSSL_cleanse(a, sizeof(a)); OPENSSL_cleanse(b, sizeof(b)); OPENSSL_cleanse(c, sizeof(c)); @@ -424,30 +424,30 @@ ossl_x448_int(uint8_t out[X_PUBLIC_BYTES], gf_add_nr(t1, x2, z2); /* A = x2 + z2 */ /* 2+e */ gf_sub_nr(t2, x2, z2); /* B = x2 - z2 */ /* 3+e */ gf_sub_nr(z2, x3, z3); /* D = x3 - z3 */ /* 3+e */ - ossl_gf_mul(x2, t1, z2); /* DA */ + gf_mul(x2, t1, z2); /* DA */ gf_add_nr(z2, z3, x3); /* C = x3 + z3 */ /* 2+e */ - ossl_gf_mul(x3, t2, z2); /* CB */ + gf_mul(x3, t2, z2); /* CB */ gf_sub_nr(z3, x2, x3); /* DA-CB */ /* 3+e */ - ossl_gf_sqr(z2, z3); /* (DA-CB)^2 */ - ossl_gf_mul(z3, x1, z2); /* z3 = x1(DA-CB)^2 */ + gf_sqr(z2, z3); /* (DA-CB)^2 */ + gf_mul(z3, x1, z2); /* z3 = x1(DA-CB)^2 */ gf_add_nr(z2, x2, x3); /* (DA+CB) */ /* 2+e */ - ossl_gf_sqr(x3, z2); /* x3 = (DA+CB)^2 */ + gf_sqr(x3, z2); /* x3 = (DA+CB)^2 */ - ossl_gf_sqr(z2, t1); /* AA = A^2 */ - ossl_gf_sqr(t1, t2); /* BB = B^2 */ - ossl_gf_mul(x2, z2, t1); /* x2 = AA*BB */ + gf_sqr(z2, t1); /* AA = A^2 */ + gf_sqr(t1, t2); /* BB = B^2 */ + gf_mul(x2, z2, t1); /* x2 = AA*BB */ gf_sub_nr(t2, z2, t1); /* E = AA-BB */ /* 3+e */ gf_mulw(t1, t2, -EDWARDS_D); /* E*-d = a24*E */ gf_add_nr(t1, t1, z2); /* AA + a24*E */ /* 2+e */ - ossl_gf_mul(z2, t2, t1); /* z2 = E(AA+a24*E) */ + gf_mul(z2, t2, t1); /* z2 = E(AA+a24*E) */ } /* Finish */ gf_cond_swap(x2, x3, swap); gf_cond_swap(z2, z3, swap); gf_invert(z2, z2, 0); - ossl_gf_mul(x1, x2, z2); + gf_mul(x1, x2, z2); gf_serialize(out, x1, 1); nz = ~gf_eq(x1, ZERO); @@ -471,8 +471,8 @@ ossl_curve448_point_mul_by_ratio_and_encode_like_x448(uint8_t curve448_point_copy(q, p); gf_invert(q->t, q->x, 0); /* 1/x */ - ossl_gf_mul(q->z, q->t, q->y); /* y/x */ - ossl_gf_sqr(q->y, q->z); /* (y/x)^2 */ + gf_mul(q->z, q->t, q->y); /* y/x */ + gf_sqr(q->y, q->z); /* (y/x)^2 */ gf_serialize(out, q->y, 1); ossl_curve448_point_destroy(q); } @@ -586,15 +586,9 @@ static int recode_wnaf(struct smvt_control *control, int32_t delta = odd & mask; assert(position >= 0); + assert(pos < 32); /* can't fail since current & 0xFFFF != 0 */ if (odd & (1 << (table_bits + 1))) delta -= (1 << (table_bits + 1)); - /* - * Coverity gets confused by the value of pos, thinking it might be - * 32. This would require current & 0xFFFF to be zero which isn't - * possible. Suppress this false positive, since adding a check - * isn't desirable. - */ - /* coverity[overflow_before_widen] */ current -= delta * (1 << pos); control[position].power = pos + 16 * (w - 1); control[position].addend = delta; @@ -612,7 +606,7 @@ static int recode_wnaf(struct smvt_control *control, return n - 1; } -static void prepare_wnaf_table(pniels_t *output, +static void prepare_wnaf_table(pniels_t * output, const curve448_point_t working, unsigned int tbits) { diff --git a/openssl/src/crypto/ec/curve448/curve448_local.h b/openssl/src/crypto/ec/curve448/curve448_local.h index 5c569ea8b..3410f091a 100644 --- a/openssl/src/crypto/ec/curve448/curve448_local.h +++ b/openssl/src/crypto/ec/curve448/curve448_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,4 +10,15 @@ # define OSSL_CRYPTO_EC_CURVE448_LOCAL_H # include "curve448utils.h" +int +ossl_ed448ph_sign(OSSL_LIB_CTX *ctx, uint8_t *out_sig, const uint8_t hash[64], + const uint8_t public_key[57], const uint8_t private_key[57], + const uint8_t *context, size_t context_len, const char *propq); + +int +ossl_ed448ph_verify(OSSL_LIB_CTX *ctx, const uint8_t hash[64], + const uint8_t signature[114], const uint8_t public_key[57], + const uint8_t *context, size_t context_len, + const char *propq); + #endif /* OSSL_CRYPTO_EC_CURVE448_LOCAL_H */ diff --git a/openssl/src/crypto/ec/curve448/ed448.h b/openssl/src/crypto/ec/curve448/ed448.h index f4f01892d..00b2bae58 100644 --- a/openssl/src/crypto/ec/curve448/ed448.h +++ b/openssl/src/crypto/ec/curve448/ed448.h @@ -21,7 +21,7 @@ /* Number of bytes in an EdDSA private key. */ # define EDDSA_448_PRIVATE_BYTES EDDSA_448_PUBLIC_BYTES -/* Number of bytes in an EdDSA signature. */ +/* Number of bytes in an EdDSA private key. */ # define EDDSA_448_SIGNATURE_BYTES (EDDSA_448_PUBLIC_BYTES + \ EDDSA_448_PRIVATE_BYTES) diff --git a/openssl/src/crypto/ec/curve448/eddsa.c b/openssl/src/crypto/ec/curve448/eddsa.c index ff7f11dd3..6648692ff 100644 --- a/openssl/src/crypto/ec/curve448/eddsa.c +++ b/openssl/src/crypto/ec/curve448/eddsa.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2015-2016 Cryptography Research, Inc. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -61,8 +61,12 @@ static c448_error_t hash_init_with_dom(OSSL_LIB_CTX *ctx, EVP_MD_CTX *hashctx, size_t context_len, const char *propq) { - /* ASCII: "SigEd448", in hex for EBCDIC compatibility */ - const char dom_s[] = "\x53\x69\x67\x45\x64\x34\x34\x38"; +#ifdef CHARSET_EBCDIC + const char dom_s[] = {0x53, 0x69, 0x67, 0x45, + 0x64, 0x34, 0x34, 0x38, 0x00}; +#else + const char dom_s[] = "SigEd448"; +#endif uint8_t dom[2]; EVP_MD *shake256 = NULL; @@ -78,7 +82,7 @@ static c448_error_t hash_init_with_dom(OSSL_LIB_CTX *ctx, EVP_MD_CTX *hashctx, return C448_FAILURE; if (!EVP_DigestInit_ex(hashctx, shake256, NULL) - || !EVP_DigestUpdate(hashctx, dom_s, sizeof(dom_s)-1) + || !EVP_DigestUpdate(hashctx, dom_s, strlen(dom_s)) || !EVP_DigestUpdate(hashctx, dom, sizeof(dom)) || !EVP_DigestUpdate(hashctx, context, context_len)) { EVP_MD_free(shake256); @@ -369,29 +373,47 @@ ossl_c448_ed448_verify_prehash( } int -ossl_ed448_sign(OSSL_LIB_CTX *ctx, uint8_t *out_sig, - const uint8_t *message, size_t message_len, - const uint8_t public_key[57], const uint8_t private_key[57], - const uint8_t *context, size_t context_len, - const uint8_t phflag, const char *propq) +ossl_ed448_sign(OSSL_LIB_CTX *ctx, uint8_t *out_sig, const uint8_t *message, + size_t message_len, const uint8_t public_key[57], + const uint8_t private_key[57], const uint8_t *context, + size_t context_len, const char *propq) { return ossl_c448_ed448_sign(ctx, out_sig, private_key, public_key, message, - message_len, phflag, context, context_len, + message_len, 0, context, context_len, propq) == C448_SUCCESS; } int -ossl_ed448_verify(OSSL_LIB_CTX *ctx, - const uint8_t *message, size_t message_len, +ossl_ed448_verify(OSSL_LIB_CTX *ctx, const uint8_t *message, size_t message_len, const uint8_t signature[114], const uint8_t public_key[57], - const uint8_t *context, size_t context_len, - const uint8_t phflag, const char *propq) + const uint8_t *context, size_t context_len, const char *propq) { return ossl_c448_ed448_verify(ctx, signature, public_key, message, - message_len, phflag, context, (uint8_t)context_len, + message_len, 0, context, (uint8_t)context_len, propq) == C448_SUCCESS; } +int +ossl_ed448ph_sign(OSSL_LIB_CTX *ctx, uint8_t *out_sig, const uint8_t hash[64], + const uint8_t public_key[57], const uint8_t private_key[57], + const uint8_t *context, size_t context_len, const char *propq) +{ + return ossl_c448_ed448_sign_prehash(ctx, out_sig, private_key, public_key, + hash, context, context_len, + propq) == C448_SUCCESS; +} + +int +ossl_ed448ph_verify(OSSL_LIB_CTX *ctx, const uint8_t hash[64], + const uint8_t signature[114], const uint8_t public_key[57], + const uint8_t *context, size_t context_len, + const char *propq) +{ + return ossl_c448_ed448_verify_prehash(ctx, signature, public_key, hash, + context, (uint8_t)context_len, + propq) == C448_SUCCESS; +} + int ossl_ed448_public_from_private(OSSL_LIB_CTX *ctx, uint8_t out_public_key[57], const uint8_t private_key[57], const char *propq) diff --git a/openssl/src/crypto/ec/curve448/f_generic.c b/openssl/src/crypto/ec/curve448/f_generic.c index 9a4675a8b..4c571810d 100644 --- a/openssl/src/crypto/ec/curve448/f_generic.c +++ b/openssl/src/crypto/ec/curve448/f_generic.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2015-2016 Cryptography Research, Inc. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -18,7 +18,7 @@ static const gf MODULUS = { }; /* Serialize to wire format. */ -void gf_serialize(uint8_t serial[SER_BYTES], const gf x, int with_hibit) +void gf_serialize(uint8_t *serial, const gf x, int with_hibit) { unsigned int j = 0, fill = 0; dword_t buffer = 0; @@ -173,32 +173,32 @@ mask_t gf_isr(gf a, const gf x) { gf L0, L1, L2; - ossl_gf_sqr(L1, x); - ossl_gf_mul(L2, x, L1); - ossl_gf_sqr(L1, L2); - ossl_gf_mul(L2, x, L1); + gf_sqr(L1, x); + gf_mul(L2, x, L1); + gf_sqr(L1, L2); + gf_mul(L2, x, L1); gf_sqrn(L1, L2, 3); - ossl_gf_mul(L0, L2, L1); + gf_mul(L0, L2, L1); gf_sqrn(L1, L0, 3); - ossl_gf_mul(L0, L2, L1); + gf_mul(L0, L2, L1); gf_sqrn(L2, L0, 9); - ossl_gf_mul(L1, L0, L2); - ossl_gf_sqr(L0, L1); - ossl_gf_mul(L2, x, L0); + gf_mul(L1, L0, L2); + gf_sqr(L0, L1); + gf_mul(L2, x, L0); gf_sqrn(L0, L2, 18); - ossl_gf_mul(L2, L1, L0); + gf_mul(L2, L1, L0); gf_sqrn(L0, L2, 37); - ossl_gf_mul(L1, L2, L0); + gf_mul(L1, L2, L0); gf_sqrn(L0, L1, 37); - ossl_gf_mul(L1, L2, L0); + gf_mul(L1, L2, L0); gf_sqrn(L0, L1, 111); - ossl_gf_mul(L2, L1, L0); - ossl_gf_sqr(L0, L2); - ossl_gf_mul(L1, x, L0); + gf_mul(L2, L1, L0); + gf_sqr(L0, L2); + gf_mul(L1, x, L0); gf_sqrn(L0, L1, 223); - ossl_gf_mul(L1, L2, L0); - ossl_gf_sqr(L2, L1); - ossl_gf_mul(L0, L2, x); + gf_mul(L1, L2, L0); + gf_sqr(L2, L1); + gf_mul(L0, L2, x); gf_copy(a, L1); return gf_eq(L0, ONE); } diff --git a/openssl/src/crypto/ec/curve448/field.h b/openssl/src/crypto/ec/curve448/field.h index 80b1355b7..e1c633378 100644 --- a/openssl/src/crypto/ec/curve448/field.h +++ b/openssl/src/crypto/ec/curve448/field.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2014 Cryptography Research, Inc. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -54,15 +54,15 @@ static INLINE_UNUSED void gf_weak_reduce(gf inout); void gf_strong_reduce(gf inout); void gf_add(gf out, const gf a, const gf b); void gf_sub(gf out, const gf a, const gf b); -void ossl_gf_mul(gf_s * RESTRICT out, const gf a, const gf b); -void ossl_gf_mulw_unsigned(gf_s * RESTRICT out, const gf a, uint32_t b); -void ossl_gf_sqr(gf_s * RESTRICT out, const gf a); +void gf_mul(gf_s * RESTRICT out, const gf a, const gf b); +void gf_mulw_unsigned(gf_s * RESTRICT out, const gf a, uint32_t b); +void gf_sqr(gf_s * RESTRICT out, const gf a); mask_t gf_isr(gf a, const gf x); /** a^2 x = 1, QNR, or 0 if x=0. Return true if successful */ mask_t gf_eq(const gf x, const gf y); mask_t gf_lobit(const gf x); mask_t gf_hibit(const gf x); -void gf_serialize(uint8_t serial[SER_BYTES], const gf x, int with_highbit); +void gf_serialize(uint8_t *serial, const gf x, int with_highbit); mask_t gf_deserialize(gf x, const uint8_t serial[SER_BYTES], int with_hibit, uint8_t hi_nmask); @@ -85,16 +85,16 @@ static ossl_inline void gf_sqrn(gf_s * RESTRICT y, const gf x, int n) assert(n > 0); if (n & 1) { - ossl_gf_sqr(y, x); + gf_sqr(y, x); n--; } else { - ossl_gf_sqr(tmp, x); - ossl_gf_sqr(y, tmp); + gf_sqr(tmp, x); + gf_sqr(y, tmp); n -= 2; } for (; n; n -= 2) { - ossl_gf_sqr(tmp, y); - ossl_gf_sqr(y, tmp); + gf_sqr(tmp, y); + gf_sqr(y, tmp); } } @@ -122,9 +122,9 @@ static ossl_inline void gf_subx_nr(gf c, const gf a, const gf b, int amt) static ossl_inline void gf_mulw(gf c, const gf a, int32_t w) { if (w > 0) { - ossl_gf_mulw_unsigned(c, a, w); + gf_mulw_unsigned(c, a, w); } else { - ossl_gf_mulw_unsigned(c, a, -w); + gf_mulw_unsigned(c, a, -w); gf_sub(c, ZERO, c); } } diff --git a/openssl/src/crypto/ec/curve448/point_448.h b/openssl/src/crypto/ec/curve448/point_448.h index 96a54558d..e67ea6804 100644 --- a/openssl/src/crypto/ec/curve448/point_448.h +++ b/openssl/src/crypto/ec/curve448/point_448.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2015-2016 Cryptography Research, Inc. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -271,7 +271,7 @@ ossl_x448_derive_public_key(uint8_t out[X448_PUBLIC_BYTES], */ void ossl_curve448_precomputed_scalarmul(curve448_point_t scaled, - const curve448_precomputed_s *base, + const curve448_precomputed_s * base, const curve448_scalar_t scalar); /* diff --git a/openssl/src/crypto/ec/ec2_oct.c b/openssl/src/crypto/ec/ec2_oct.c index 4ed628756..10a493259 100644 --- a/openssl/src/crypto/ec/ec2_oct.c +++ b/openssl/src/crypto/ec/ec2_oct.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -272,7 +272,7 @@ int ossl_ec_GF2m_simple_oct2point(const EC_GROUP *group, EC_POINT *point, } /* - * The first octet is the point conversion octet PC, see X9.62, page 4 + * The first octet is the point converison octet PC, see X9.62, page 4 * and section 4.4.2. It must be: * 0x00 for the point at infinity * 0x02 or 0x03 for compressed form diff --git a/openssl/src/crypto/ec/ec2_smpl.c b/openssl/src/crypto/ec/ec2_smpl.c index 13e702684..3a59544c8 100644 --- a/openssl/src/crypto/ec/ec2_smpl.c +++ b/openssl/src/crypto/ec/ec2_smpl.c @@ -9,7 +9,7 @@ */ /* - * ECDSA low-level APIs are deprecated for public use, but still ok for + * ECDSA low level APIs are deprecated for public use, but still ok for * internal use. */ #include "internal/deprecated.h" @@ -188,7 +188,7 @@ int ossl_ec_GF2m_simple_group_check_discriminant(const EC_GROUP *group, if (ctx == NULL) { ctx = new_ctx = BN_CTX_new(); if (ctx == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -545,7 +545,7 @@ int ossl_ec_GF2m_simple_is_on_curve(const EC_GROUP *group, const EC_POINT *point * We have a curve defined by a Weierstrass equation * y^2 + x*y = x^3 + a*x^2 + b. * <=> x^3 + a*x^2 + x*y + b + y^2 = 0 - * <=> ((x + a) * x + y) * x + b + y^2 = 0 + * <=> ((x + a) * x + y ) * x + b + y^2 = 0 */ if (!BN_GF2m_add(lh, point->X, group->a)) goto err; @@ -826,7 +826,7 @@ int ec_GF2m_simple_ladder_post(const EC_GROUP *group, t1 = BN_CTX_get(ctx); t2 = BN_CTX_get(ctx); if (t2 == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -905,7 +905,7 @@ int ec_GF2m_simple_points_mul(const EC_GROUP *group, EC_POINT *r, */ if ((t = EC_POINT_new(group)) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } @@ -923,7 +923,7 @@ int ec_GF2m_simple_points_mul(const EC_GROUP *group, EC_POINT *r, /*- * Computes the multiplicative inverse of a in GF(2^m), storing the result in r. - * If a is zero (or equivalent), you'll get an EC_R_CANNOT_INVERT error. + * If a is zero (or equivalent), you'll get a EC_R_CANNOT_INVERT error. * SCA hardening is with blinding: BN_GF2m_mod_inv does that. */ static int ec_GF2m_simple_field_inv(const EC_GROUP *group, BIGNUM *r, diff --git a/openssl/src/crypto/ec/ec_ameth.c b/openssl/src/crypto/ec/ec_ameth.c index ad28ba682..9ca023bbe 100644 --- a/openssl/src/crypto/ec/ec_ameth.c +++ b/openssl/src/crypto/ec/ec_ameth.c @@ -42,6 +42,7 @@ static int eckey_param2type(int *pptype, void **ppval, const EC_KEY *ec_key) ASN1_OBJECT *asn1obj = OBJ_nid2obj(nid); if (asn1obj == NULL || OBJ_length(asn1obj) == 0) { + ASN1_OBJECT_free(asn1obj); ERR_raise(ERR_LIB_EC, EC_R_MISSING_OID); return 0; } @@ -91,7 +92,9 @@ static int eckey_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) ptype, pval, penc, penclen)) return 1; err: - if (ptype == V_ASN1_SEQUENCE) + if (ptype == V_ASN1_OBJECT) + ASN1_OBJECT_free(pval); + else ASN1_STRING_free(pval); OPENSSL_free(penc); return 0; @@ -162,7 +165,7 @@ static int eckey_priv_decode_ex(EVP_PKEY *pkey, const PKCS8_PRIV_KEY_INFO *p8, static int eckey_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) { EC_KEY ec_key = *(pkey->pkey.ec); - unsigned char *ep = NULL; + unsigned char *ep, *p; int eplen, ptype; void *pval; unsigned int old_flags; @@ -181,25 +184,30 @@ static int eckey_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) old_flags = EC_KEY_get_enc_flags(&ec_key); EC_KEY_set_enc_flags(&ec_key, old_flags | EC_PKEY_NO_PARAMETERS); - eplen = i2d_ECPrivateKey(&ec_key, &ep); - if (eplen <= 0) { + eplen = i2d_ECPrivateKey(&ec_key, NULL); + if (!eplen) { ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); - goto err; + return 0; + } + ep = OPENSSL_malloc(eplen); + if (ep == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return 0; + } + p = ep; + if (!i2d_ECPrivateKey(&ec_key, &p)) { + OPENSSL_free(ep); + ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + return 0; } if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(NID_X9_62_id_ecPublicKey), 0, ptype, pval, ep, eplen)) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); - OPENSSL_clear_free(ep, eplen); - goto err; + OPENSSL_free(ep); + return 0; } return 1; - - err: - if (ptype == V_ASN1_SEQUENCE) - ASN1_STRING_free(pval); - return 0; } static int int_ec_size(const EVP_PKEY *pkey) @@ -513,10 +521,8 @@ int ec_pkey_export_to(const EVP_PKEY *from, void *to_keydata, if (pub_point != NULL) { /* convert pub_point to a octet string according to the SECG standard */ - point_conversion_form_t format = EC_KEY_get_conv_form(eckey); - if ((pub_key_buflen = EC_POINT_point2buf(ecg, pub_point, - format, + POINT_CONVERSION_COMPRESSED, &pub_key_buf, bnctx)) == 0 || !OSSL_PARAM_BLD_push_octet_string(tmpl, OSSL_PKEY_PARAM_PUB_KEY, @@ -568,7 +574,7 @@ int ec_pkey_export_to(const EVP_PKEY *from, void *to_keydata, if (ecbits <= 0) goto err; - sz = (ecbits + 7) / 8; + sz = (ecbits + 7 ) / 8; if (!OSSL_PARAM_BLD_push_BN_pad(tmpl, OSSL_PKEY_PARAM_PRIV_KEY, priv_key, sz)) @@ -613,7 +619,7 @@ static int ec_pkey_import_from(const OSSL_PARAM params[], void *vpctx) EC_KEY *ec = EC_KEY_new_ex(pctx->libctx, pctx->propquery); if (ec == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return 0; } diff --git a/openssl/src/crypto/ec/ec_asn1.c b/openssl/src/crypto/ec/ec_asn1.c index b32697fb8..d60b06a74 100644 --- a/openssl/src/crypto/ec/ec_asn1.c +++ b/openssl/src/crypto/ec/ec_asn1.c @@ -19,7 +19,6 @@ #include #include #include "internal/nelem.h" -#include "crypto/asn1.h" #include "crypto/asn1_dsa.h" #ifndef FIPS_MODULE @@ -105,8 +104,8 @@ ASN1_SEQUENCE(X9_62_PENTANOMIAL) = { ASN1_EMBED(X9_62_PENTANOMIAL, k3, INT32) } static_ASN1_SEQUENCE_END(X9_62_PENTANOMIAL) -DECLARE_ASN1_ALLOC_FUNCTIONS(X9_62_PENTANOMIAL) -IMPLEMENT_ASN1_ALLOC_FUNCTIONS(X9_62_PENTANOMIAL) +DECLARE_STATIC_ASN1_ALLOC_FUNCTIONS(X9_62_PENTANOMIAL) +IMPLEMENT_STATIC_ASN1_ALLOC_FUNCTIONS(X9_62_PENTANOMIAL) ASN1_ADB_TEMPLATE(char_two_def) = ASN1_SIMPLE(X9_62_CHARACTERISTIC_TWO, p.other, ASN1_ANY); @@ -122,8 +121,8 @@ ASN1_SEQUENCE(X9_62_CHARACTERISTIC_TWO) = { ASN1_ADB_OBJECT(X9_62_CHARACTERISTIC_TWO) } static_ASN1_SEQUENCE_END(X9_62_CHARACTERISTIC_TWO) -DECLARE_ASN1_ALLOC_FUNCTIONS(X9_62_CHARACTERISTIC_TWO) -IMPLEMENT_ASN1_ALLOC_FUNCTIONS(X9_62_CHARACTERISTIC_TWO) +DECLARE_STATIC_ASN1_ALLOC_FUNCTIONS(X9_62_CHARACTERISTIC_TWO) +IMPLEMENT_STATIC_ASN1_ALLOC_FUNCTIONS(X9_62_CHARACTERISTIC_TWO) ASN1_ADB_TEMPLATE(fieldID_def) = ASN1_SIMPLE(X9_62_FIELDID, p.other, ASN1_ANY); @@ -206,7 +205,7 @@ static int ec_asn1_group2fieldid(const EC_GROUP *group, X9_62_FIELDID *field) if (nid == NID_X9_62_prime_field) { if ((tmp = BN_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } /* the parameters are specified by the prime number p */ @@ -235,7 +234,7 @@ static int ec_asn1_group2fieldid(const EC_GROUP *group, X9_62_FIELDID *field) char_two = field->p.char_two; if (char_two == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -261,7 +260,7 @@ static int ec_asn1_group2fieldid(const EC_GROUP *group, X9_62_FIELDID *field) char_two->p.tpBasis = ASN1_INTEGER_new(); if (char_two->p.tpBasis == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } if (!ASN1_INTEGER_set(char_two->p.tpBasis, (long)k)) { @@ -276,7 +275,7 @@ static int ec_asn1_group2fieldid(const EC_GROUP *group, X9_62_FIELDID *field) char_two->p.ppBasis = X9_62_PENTANOMIAL_new(); if (char_two->p.ppBasis == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -289,7 +288,7 @@ static int ec_asn1_group2fieldid(const EC_GROUP *group, X9_62_FIELDID *field) /* for ONB the parameters are (asn1) NULL */ char_two->p.onBasis = ASN1_NULL_new(); if (char_two->p.onBasis == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -303,6 +302,9 @@ static int ec_asn1_group2fieldid(const EC_GROUP *group, X9_62_FIELDID *field) ok = 1; err: + /* Just to avoid compiler warnings */ + X9_62_PENTANOMIAL_free(NULL); + X9_62_CHARACTERISTIC_TWO_free(NULL); BN_free(tmp); return ok; } @@ -318,7 +320,7 @@ static int ec_asn1_group2curve(const EC_GROUP *group, X9_62_CURVE *curve) return 0; if ((tmp_1 = BN_new()) == NULL || (tmp_2 = BN_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -335,8 +337,10 @@ static int ec_asn1_group2curve(const EC_GROUP *group, X9_62_CURVE *curve) */ len = ((size_t)EC_GROUP_get_degree(group) + 7) / 8; if ((a_buf = OPENSSL_malloc(len)) == NULL - || (b_buf = OPENSSL_malloc(len)) == NULL) + || (b_buf = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } if (BN_bn2binpad(tmp_1, a_buf, len) < 0 || BN_bn2binpad(tmp_2, b_buf, len) < 0) { ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); @@ -354,10 +358,11 @@ static int ec_asn1_group2curve(const EC_GROUP *group, X9_62_CURVE *curve) if (group->seed) { if (!curve->seed) if ((curve->seed = ASN1_BIT_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } - ossl_asn1_string_set_bits_left(curve->seed, 0); + curve->seed->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + curve->seed->flags |= ASN1_STRING_FLAG_BITS_LEFT; if (!ASN1_BIT_STRING_set(curve->seed, group->seed, (int)group->seed_len)) { ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); @@ -391,7 +396,7 @@ ECPARAMETERS *EC_GROUP_get_ecparameters(const EC_GROUP *group, if (params == NULL) { if ((ret = ECPARAMETERS_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } else @@ -427,7 +432,7 @@ ECPARAMETERS *EC_GROUP_get_ecparameters(const EC_GROUP *group, } if (ret->base == NULL && (ret->base = ASN1_OCTET_STRING_new()) == NULL) { OPENSSL_free(buffer); - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } ASN1_STRING_set0(ret->base, buffer, len); @@ -472,7 +477,7 @@ ECPKPARAMETERS *EC_GROUP_get_ecpkparameters(const EC_GROUP *group, if (ret == NULL) { if ((ret = ECPKPARAMETERS_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NULL; } } else { @@ -578,7 +583,7 @@ EC_GROUP *EC_GROUP_new_from_ecparameters(const ECPARAMETERS *params) } if ((p = BN_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -685,19 +690,11 @@ EC_GROUP *EC_GROUP_new_from_ecparameters(const ECPARAMETERS *params) /* extract seed (optional) */ if (params->curve->seed != NULL) { - /* - * This happens for instance with - * fuzz/corpora/asn1/65cf44e85614c62f10cf3b7a7184c26293a19e4a - * and causes the OPENSSL_malloc below to choke on the - * zero length allocation request. - */ - if (params->curve->seed->length == 0) { - ERR_raise(ERR_LIB_EC, EC_R_ASN1_ERROR); - goto err; - } OPENSSL_free(ret->seed); - if ((ret->seed = OPENSSL_malloc(params->curve->seed->length)) == NULL) + if ((ret->seed = OPENSSL_malloc(params->curve->seed->length)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } memcpy(ret->seed, params->curve->seed->data, params->curve->seed->length); ret->seed_len = params->curve->seed->length; @@ -726,7 +723,7 @@ EC_GROUP *EC_GROUP_new_from_ecparameters(const ECPARAMETERS *params) } /* extract the order */ - if (ASN1_INTEGER_to_BN(params->order, a) == NULL) { + if ((a = ASN1_INTEGER_to_BN(params->order, a)) == NULL) { ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); goto err; } @@ -743,7 +740,7 @@ EC_GROUP *EC_GROUP_new_from_ecparameters(const ECPARAMETERS *params) if (params->cofactor == NULL) { BN_free(b); b = NULL; - } else if (ASN1_INTEGER_to_BN(params->cofactor, b) == NULL) { + } else if ((b = ASN1_INTEGER_to_BN(params->cofactor, b)) == NULL) { ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); goto err; } @@ -941,7 +938,7 @@ EC_KEY *d2i_ECPrivateKey(EC_KEY **a, const unsigned char **in, long len) if (a == NULL || *a == NULL) { if ((ret = EC_KEY_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } else @@ -1029,7 +1026,7 @@ int i2d_ECPrivateKey(const EC_KEY *a, unsigned char **out) } if ((priv_key = EC_PRIVATEKEY_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -1057,7 +1054,7 @@ int i2d_ECPrivateKey(const EC_KEY *a, unsigned char **out) if (!(a->enc_flag & EC_PKEY_NO_PUBKEY)) { priv_key->publicKey = ASN1_BIT_STRING_new(); if (priv_key->publicKey == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -1068,7 +1065,8 @@ int i2d_ECPrivateKey(const EC_KEY *a, unsigned char **out) goto err; } - ossl_asn1_string_set_bits_left(priv_key->publicKey, 0); + priv_key->publicKey->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + priv_key->publicKey->flags |= ASN1_STRING_FLAG_BITS_LEFT; ASN1_STRING_set0(priv_key->publicKey, pub, publen); pub = NULL; } @@ -1105,7 +1103,7 @@ EC_KEY *d2i_ECParameters(EC_KEY **a, const unsigned char **in, long len) if (a == NULL || *a == NULL) { if ((ret = EC_KEY_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NULL; } } else @@ -1169,8 +1167,10 @@ int i2o_ECPublicKey(const EC_KEY *a, unsigned char **out) return buf_len; if (*out == NULL) { - if ((*out = OPENSSL_malloc(buf_len)) == NULL) + if ((*out = OPENSSL_malloc(buf_len)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; + } new_buffer = 1; } if (!EC_POINT_point2oct(a->group, a->pub_key, a->conv_form, @@ -1195,7 +1195,8 @@ DECLARE_ASN1_ENCODE_FUNCTIONS_name(ECDSA_SIG, ECDSA_SIG) ECDSA_SIG *ECDSA_SIG_new(void) { ECDSA_SIG *sig = OPENSSL_zalloc(sizeof(*sig)); - + if (sig == NULL) + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return sig; } diff --git a/openssl/src/crypto/ec/ec_backend.c b/openssl/src/crypto/ec/ec_backend.c index b9ebc9646..9159bb808 100644 --- a/openssl/src/crypto/ec/ec_backend.c +++ b/openssl/src/crypto/ec/ec_backend.c @@ -25,7 +25,6 @@ #include "crypto/ec.h" #include "ec_local.h" #include "internal/e_os.h" -#include "internal/nelem.h" #include "internal/param_build_set.h" /* Mapping between a flag and a name */ @@ -191,7 +190,7 @@ static int ec_group_explicit_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, BIGNUM *b = BN_CTX_get(bnctx); if (b == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -202,7 +201,7 @@ static int ec_group_explicit_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, if (!ossl_param_build_set_bn(tmpl, params, OSSL_PKEY_PARAM_EC_P, p) || !ossl_param_build_set_bn(tmpl, params, OSSL_PKEY_PARAM_EC_A, a) || !ossl_param_build_set_bn(tmpl, params, OSSL_PKEY_PARAM_EC_B, b)) { - ERR_raise(ERR_LIB_EC, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -217,7 +216,7 @@ static int ec_group_explicit_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, } if (!ossl_param_build_set_bn(tmpl, params, OSSL_PKEY_PARAM_EC_ORDER, order)) { - ERR_raise(ERR_LIB_EC, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -227,7 +226,7 @@ static int ec_group_explicit_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, if (!ossl_param_build_set_utf8_string(tmpl, params, OSSL_PKEY_PARAM_EC_FIELD_TYPE, field_type)) { - ERR_raise(ERR_LIB_EC, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -250,7 +249,7 @@ static int ec_group_explicit_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, if (!ossl_param_build_set_octet_string(tmpl, params, OSSL_PKEY_PARAM_EC_GENERATOR, *genbuf, genbuf_len)) { - ERR_raise(ERR_LIB_EC, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -262,7 +261,7 @@ static int ec_group_explicit_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, if (cofactor != NULL && !ossl_param_build_set_bn(tmpl, params, OSSL_PKEY_PARAM_EC_COFACTOR, cofactor)) { - ERR_raise(ERR_LIB_EC, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -277,7 +276,7 @@ static int ec_group_explicit_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, && !ossl_param_build_set_octet_string(tmpl, params, OSSL_PKEY_PARAM_EC_SEED, seed, seed_len)) { - ERR_raise(ERR_LIB_EC, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -296,7 +295,7 @@ int ossl_ec_group_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, point_conversion_form_t genform; if (group == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_PASSED_NULL_PARAMETER); + ERR_raise(ERR_LIB_EC,EC_R_PASSED_NULL_PARAMETER); return 0; } @@ -319,11 +318,6 @@ int ossl_ec_group_todata(const EC_GROUP *group, OSSL_PARAM_BLD *tmpl, return 0; } - if (!ossl_param_build_set_int(tmpl, params, - OSSL_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS, - group->decoded_from_explicit_params)) - return 0; - curve_nid = EC_GROUP_get_curve_name(group); /* @@ -524,7 +518,7 @@ static int ec_key_point_format_fromdata(EC_KEY *ec, const OSSL_PARAM params[]) p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT); if (p != NULL) { if (!ossl_ec_pt_format_param2id(p, &format)) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_FORM); + ECerr(0, EC_R_INVALID_FORM); return 0; } EC_KEY_set_conv_form(ec, format); @@ -774,7 +768,7 @@ EC_KEY *ossl_ec_key_param_from_x509_algor(const X509_ALGOR *palg, X509_ALGOR_get0(NULL, &ptype, &pval, palg); if ((eckey = EC_KEY_new_ex(libctx, propq)) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto ecerr; } diff --git a/openssl/src/crypto/ec/ec_check.c b/openssl/src/crypto/ec/ec_check.c index 9ed94b328..484124915 100644 --- a/openssl/src/crypto/ec/ec_check.c +++ b/openssl/src/crypto/ec/ec_check.c @@ -30,7 +30,7 @@ int EC_GROUP_check_named_curve(const EC_GROUP *group, int nist_only, if (ctx == NULL) { ctx = new_ctx = BN_CTX_new_ex(NULL); if (ctx == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NID_undef; } } @@ -69,7 +69,7 @@ int EC_GROUP_check(const EC_GROUP *group, BN_CTX *ctx) if (ctx == NULL) { ctx = new_ctx = BN_CTX_new(); if (ctx == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } diff --git a/openssl/src/crypto/ec/ec_curve.c b/openssl/src/crypto/ec/ec_curve.c index d703d16b3..d21ae1e30 100644 --- a/openssl/src/crypto/ec/ec_curve.c +++ b/openssl/src/crypto/ec/ec_curve.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -2838,8 +2838,6 @@ static const ec_list_element curve_list[] = { {NID_secp384r1, &_EC_NIST_PRIME_384.h, # if defined(S390X_EC_ASM) EC_GFp_s390x_nistp384_method, -# elif !defined(OPENSSL_NO_EC_NISTP_64_GCC_128) - ossl_ec_GFp_nistp384_method, # else 0, # endif @@ -2933,8 +2931,6 @@ static const ec_list_element curve_list[] = { {NID_secp384r1, &_EC_NIST_PRIME_384.h, # if defined(S390X_EC_ASM) EC_GFp_s390x_nistp384_method, -# elif !defined(OPENSSL_NO_EC_NISTP_64_GCC_128) - ossl_ec_GFp_nistp384_method, # else 0, # endif @@ -3111,9 +3107,9 @@ static const ec_list_element curve_list[] = { "RFC 5639 curve over a 512 bit prime field"}, {NID_brainpoolP512t1, &_EC_brainpoolP512t1.h, 0, "RFC 5639 curve over a 512 bit prime field"}, -#ifndef OPENSSL_NO_SM2 +# ifndef OPENSSL_NO_SM2 {NID_sm2, &_EC_sm2p256v1.h, -# ifdef ECP_SM2P256_ASM +# if !defined(OPENSSL_NO_EC_SM2P_64_GCC_128) EC_GFp_sm2p256_method, # else 0, @@ -3160,7 +3156,7 @@ static EC_GROUP *ec_group_new_from_data(OSSL_LIB_CTX *libctx, curve.meth != NULL ? curve.meth() : NULL); if ((ctx = BN_CTX_new_ex(libctx)) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } diff --git a/openssl/src/crypto/ec/ec_deprecated.c b/openssl/src/crypto/ec/ec_deprecated.c index 905b56063..765894c32 100644 --- a/openssl/src/crypto/ec/ec_deprecated.c +++ b/openssl/src/crypto/ec/ec_deprecated.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2020 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -47,8 +47,10 @@ EC_POINT *EC_POINT_bn2point(const EC_GROUP *group, if ((buf_len = BN_num_bytes(bn)) == 0) buf_len = 1; - if ((buf = OPENSSL_malloc(buf_len)) == NULL) + if ((buf = OPENSSL_malloc(buf_len)) == NULL) { + ECerr(EC_F_EC_POINT_BN2POINT, ERR_R_MALLOC_FAILURE); return NULL; + } if (BN_bn2binpad(bn, buf, buf_len) < 0) { OPENSSL_free(buf); diff --git a/openssl/src/crypto/ec/ec_elgamal.h b/openssl/src/crypto/ec/ec_elgamal.h new file mode 100644 index 000000000..af6e6f0c8 --- /dev/null +++ b/openssl/src/crypto/ec/ec_elgamal.h @@ -0,0 +1,88 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_EC_ELGAMAL_H +# define HEADER_EC_ELGAMAL_H + +# include + +# ifndef OPENSSL_NO_EC_ELGAMAL +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include +# include +# include +# include +# include + +struct ec_elgamal_ciphertext_st { + EC_POINT *C1; + EC_POINT *C2; +}; + +struct ec_elgamal_mr_ciphertext_st { + STACK_OF(EC_POINT) *sk_C1; + EC_POINT *C2; +}; + +typedef struct ec_elgamal_decrypt_table_entry_st { + int32_t value; + uint32_t key_len; + unsigned char *key; +} EC_ELGAMAL_dec_tbl_entry; + +DEFINE_LHASH_OF(EC_ELGAMAL_dec_tbl_entry); + +struct ec_elgamal_decrypt_table_st { + CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; + int32_t flag; + int32_t size; + uint32_t baby_step_bits; + uint32_t giant_step_bits; + EC_POINT *mG_inv; + LHASH_OF(EC_ELGAMAL_dec_tbl_entry) *positive_entries; + LHASH_OF(EC_ELGAMAL_dec_tbl_entry) *negative_entries; +}; + +struct ec_elgamal_ctx_st { + EC_KEY *key; + EC_ELGAMAL_DECRYPT_TABLE *decrypt_table; + int32_t flag; +# ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + EC_POINT *h; + BIGNUM *pk_inv; +# endif +}; + +struct ec_elgamal_mr_ctx_st { + EC_GROUP *group; + STACK_OF(EC_KEY) *sk_key; + EC_ELGAMAL_DECRYPT_TABLE *decrypt_table; + int32_t flag; +# ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + EC_POINT *h; + BIGNUM *pk_inv; +# endif +}; + +int EC_ELGAMAL_dlog_brute(EC_ELGAMAL_CTX *ctx, int32_t *r, EC_POINT *M); +int EC_ELGAMAL_dlog_bsgs(EC_ELGAMAL_CTX *ctx, int32_t *r, EC_POINT *M); + +# ifdef __cplusplus +} +# endif +# endif + +#endif diff --git a/openssl/src/crypto/ec/ec_elgamal_crypt.c b/openssl/src/crypto/ec/ec_elgamal_crypt.c new file mode 100644 index 000000000..697e47c63 --- /dev/null +++ b/openssl/src/crypto/ec/ec_elgamal_crypt.c @@ -0,0 +1,714 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include "ec_elgamal.h" +#include +#include + +DEFINE_STACK_OF(EC_KEY) +DEFINE_STACK_OF(EC_POINT) + +/** Creates a new EC_ELGAMAL_CTX object + * \param key EC_KEY to use + * \param h EC_POINT object pointer + * \param flag flag of ctx + * \return newly created EC_ELGAMAL_CTX object or NULL in case of an error + */ +EC_ELGAMAL_CTX *EC_ELGAMAL_CTX_new(EC_KEY *key, const EC_POINT *h, int32_t flag) +{ +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + size_t len; + unsigned char *buf = NULL; + BN_CTX *bn_ctx = NULL; +#endif + EC_ELGAMAL_CTX *ctx = NULL; + + if (key == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return NULL; + } + +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + if (flag == EC_ELGAMAL_FLAG_TWISTED) { + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (h != NULL) { + if (!(ctx->h = EC_POINT_dup(h, key->group))) + return 0; + } else { + ctx->h = EC_POINT_new(key->group); + if (ctx->h == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + len = EC_POINT_point2oct(key->group, EC_GROUP_get0_generator(key->group), + POINT_CONVERSION_COMPRESSED, NULL, 0, bn_ctx); + if (len <= 0) + goto err; + + buf = OPENSSL_zalloc(len); + if (buf == NULL) + goto err; + + if (!EC_POINT_point2oct(key->group, EC_GROUP_get0_generator(key->group), + POINT_CONVERSION_COMPRESSED, buf, len, bn_ctx)) + goto err; + + if (!EC_POINT_from_string(key->group, ctx->h, buf, len)) + goto err; + } + + if (key->priv_key) { + ctx->pk_inv = BN_new(); + if (ctx->pk_inv == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!BN_mod_inverse(ctx->pk_inv, key->priv_key, + EC_GROUP_get0_order(key->group), bn_ctx)) + goto err; + } + + OPENSSL_free(buf); + BN_CTX_free(bn_ctx); + } +#endif + + EC_KEY_up_ref(key); + ctx->key = key; + ctx->flag = flag; + + return ctx; +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL +err: + OPENSSL_free(buf); + BN_CTX_free(bn_ctx); + EC_ELGAMAL_CTX_free(ctx); + return NULL; +#endif +} + +EC_ELGAMAL_CTX *EC_ELGAMAL_CTX_dup(EC_ELGAMAL_CTX *ctx) +{ +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + return EC_ELGAMAL_CTX_new(ctx->key, ctx->h, ctx->flag); +#else + return EC_ELGAMAL_CTX_new(ctx->key, NULL, ctx->flag); +#endif +} + +/** Frees a EC_ELGAMAL_CTX object + * \param ctx EC_ELGAMAL_CTX object to be freed + */ +void EC_ELGAMAL_CTX_free(EC_ELGAMAL_CTX *ctx) +{ + if (ctx == NULL) + return; + + EC_KEY_free(ctx->key); + EC_ELGAMAL_DECRYPT_TABLE_free(ctx->decrypt_table); +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + EC_POINT_free(ctx->h); + BN_free(ctx->pk_inv); +#endif + OPENSSL_free(ctx); +} + +/** Creates a new EC_ELGAMAL_MR_CTX object + * \param key EC_KEY to use + * \param flag flag of ctx + * \return newly created EC_ELGAMAL_MR_CTX object or NULL in case of an error + */ +EC_ELGAMAL_MR_CTX *EC_ELGAMAL_MR_CTX_new(STACK_OF(EC_KEY) *keys, const EC_POINT *h, + int32_t flag) +{ +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + size_t len; + unsigned char *buf = NULL; + BN_CTX *bn_ctx = NULL; +#endif + int i; + EC_KEY *key; + EC_GROUP *group; + EC_ELGAMAL_MR_CTX *ctx = NULL; + + if (keys == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (sk_EC_KEY_num(keys) == 0) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + key = sk_EC_KEY_value(keys, 0); + group = key->group; + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!(ctx->group = EC_GROUP_dup(group))) + goto err; + + ctx->sk_key = sk_EC_KEY_dup(keys); + if (ctx->sk_key == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + for (i = 0; i < sk_EC_KEY_num(keys); i++) { + key = sk_EC_KEY_value(keys, i); + if (!ec_point_is_compat(key->pub_key, group)) + goto err; + + if (!EC_KEY_up_ref(key)) + goto err; + } + + ctx->flag = flag; + +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + if (flag == EC_ELGAMAL_FLAG_TWISTED) { + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (h != NULL) { + if (!(ctx->h = EC_POINT_dup(h, ctx->group))) + return 0; + } else { + ctx->h = EC_POINT_new(group); + if (ctx->h == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + POINT_CONVERSION_COMPRESSED, NULL, 0, bn_ctx); + if (len <= 0) + goto err; + + buf = OPENSSL_zalloc(len); + if (buf == NULL) + goto err; + + if (!EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + POINT_CONVERSION_COMPRESSED, buf, len, bn_ctx)) + goto err; + + if (!EC_POINT_from_string(group, ctx->h, buf, len)) + goto err; + } + + if (key->priv_key) { + ctx->pk_inv = BN_new(); + if (ctx->pk_inv == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!BN_mod_inverse(ctx->pk_inv, key->priv_key, + EC_GROUP_get0_order(group), bn_ctx)) + goto err; + } + + OPENSSL_free(buf); + BN_CTX_free(bn_ctx); + } +#endif + + return ctx; +err: +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + OPENSSL_free(buf); + BN_CTX_free(bn_ctx); +#endif + EC_ELGAMAL_MR_CTX_free(ctx); + return NULL; +} + +/** Frees a EC_ELGAMAL_MR_CTX object + * \param ctx EC_ELGAMAL_MR_CTX object to be freed + */ +void EC_ELGAMAL_MR_CTX_free(EC_ELGAMAL_MR_CTX *ctx) +{ + if (ctx == NULL) + return; + + EC_GROUP_free(ctx->group); + sk_EC_KEY_pop_free(ctx->sk_key, EC_KEY_free); + EC_ELGAMAL_DECRYPT_TABLE_free(ctx->decrypt_table); +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + EC_POINT_free(ctx->h); + BN_free(ctx->pk_inv); +#endif + OPENSSL_free(ctx); +} + +/** Encrypts an Integer with additadive homomorphic EC-ElGamal + * \param ctx EC_ELGAMAL_CTX object. + * \param r EC_ELGAMAL_CIPHERTEXT object that stores the result of + * the encryption + * \param plaintext The plaintext integer to be encrypted + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_encrypt(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, int32_t plaintext) +{ + int ret = 0; + BIGNUM *bn_plain = NULL; + + if (ctx == NULL || ctx->key == NULL || ctx->key->pub_key == NULL || r == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_plain = BN_new(); + if (bn_plain == NULL) + return ret; + + BN_set_word(bn_plain, (BN_ULONG)(plaintext > 0 ? plaintext : -(int64_t)plaintext)); + BN_set_negative(bn_plain, plaintext < 0 ? 1 : 0); + + ret = EC_ELGAMAL_bn_encrypt(ctx, r, bn_plain, NULL); + + BN_free(bn_plain); + return ret; +} + +int EC_ELGAMAL_bn_encrypt(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + const BIGNUM *plaintext, const BIGNUM *rand) +{ + int ret = 0; + BN_CTX *bn_ctx = NULL; + BIGNUM *random = NULL; + + if (ctx == NULL || ctx->key == NULL || ctx->key->pub_key == NULL + || r == NULL || plaintext == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return ret; + } + + BN_CTX_start(bn_ctx); + random = BN_CTX_get(bn_ctx); + if (random == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (r->C1 == NULL) { + r->C1 = EC_POINT_new(ctx->key->group); + if (r->C1 == NULL) + goto err; + } + + if (r->C2 == NULL) { + r->C2 = EC_POINT_new(ctx->key->group); + if (r->C2 == NULL) + goto err; + } + + if (rand == NULL) + BN_rand_range(random, EC_GROUP_get0_order(ctx->key->group)); + else + random = (BIGNUM *)rand; + +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + if (ctx->flag == EC_ELGAMAL_FLAG_TWISTED) { + if (!EC_POINT_mul(ctx->key->group, r->C1, NULL, ctx->key->pub_key, + random, bn_ctx)) + goto err; + + if (!EC_POINT_mul(ctx->key->group, r->C2, random, ctx->h, + plaintext, bn_ctx)) + goto err; + } else { +#endif + if (!EC_POINT_mul(ctx->key->group, r->C1, random, NULL, NULL, bn_ctx)) + goto err; + + if (!EC_POINT_mul(ctx->key->group, r->C2, plaintext, ctx->key->pub_key, + random, bn_ctx)) + goto err; +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + } +#endif + + ret = 1; + +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + + if (!ret) { + EC_POINT_free(r->C1); + EC_POINT_free(r->C2); + r->C1 = NULL; + r->C2 = NULL; + } + + return ret; +} + +/** Encryption with one plaintext for multiple recipients. + * \param ctx EC_ELGAMAL_CTX object. + * \param r EC_ELGAMAL_CIPHERTEXT_MR object that stores the result of + * the encryption + * \param plaintext The plaintext BIGNUM object to be encrypted + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_MR_encrypt(EC_ELGAMAL_MR_CTX *ctx, EC_ELGAMAL_MR_CIPHERTEXT *r, + const BIGNUM *plaintext, BIGNUM *rand) +{ + int ret = 0, i; + EC_KEY *key; + BN_CTX *bn_ctx = NULL; + BIGNUM *random = NULL; + EC_POINT *C1 = NULL; + + if (ctx == NULL || ctx->sk_key == NULL || r == NULL || plaintext == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return ret; + } + + BN_CTX_start(bn_ctx); + random = BN_CTX_get(bn_ctx); + if (random == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (r->sk_C1 && sk_EC_POINT_num(r->sk_C1) != 0) { + sk_EC_POINT_pop_free(r->sk_C1, EC_POINT_free); + r->sk_C1 = NULL; + } + + if (r->sk_C1 == NULL) { + r->sk_C1 = sk_EC_POINT_new_null(); + if (r->sk_C1 == NULL) + goto err; + } + + if (r->C2 == NULL) { + r->C2 = EC_POINT_new(ctx->group); + if (r->C2 == NULL) + goto err; + } + + if (rand == NULL) { + BN_rand_range(random, EC_GROUP_get0_order(ctx->group)); + rand = random; + } + +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + if (ctx->flag == EC_ELGAMAL_FLAG_TWISTED) { + for (i = 0; i < sk_EC_KEY_num(ctx->sk_key); i++) { + key = sk_EC_KEY_value(ctx->sk_key, i); + + C1 = EC_POINT_new(ctx->group); + if (C1 == NULL) + goto err; + + if (!EC_POINT_mul(ctx->group, C1, NULL, key->pub_key, rand, bn_ctx)) + goto err; + + if (sk_EC_POINT_push(r->sk_C1, C1) <= 0) + goto err; + + C1 = NULL; + } + + if (!EC_POINT_mul(ctx->group, r->C2, rand, ctx->h, plaintext, bn_ctx)) + goto err; + } else { +#endif + for (i = 0; i < sk_EC_KEY_num(ctx->sk_key); i++) { + key = sk_EC_KEY_value(ctx->sk_key, i); + + C1 = EC_POINT_new(ctx->group); + if (C1 == NULL) + goto err; + + if (!EC_POINT_mul(ctx->group, C1, plaintext, key->pub_key, rand, bn_ctx)) + goto err; + + if (sk_EC_POINT_push(r->sk_C1, C1) <= 0) + goto err; + + C1 = NULL; + } + + if (!EC_POINT_mul(ctx->group, r->C2, rand, NULL, NULL, bn_ctx)) + goto err; +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + } +#endif + + ret = 1; + +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + + if (!ret) { + EC_POINT_free(C1); + EC_POINT_free(r->C2); + sk_EC_POINT_pop_free(r->sk_C1, EC_POINT_free); + r->sk_C1 = NULL; + r->C2 = NULL; + } + + return ret; +} + +/** Decrypts the ciphertext + * \param ctx EC_ELGAMAL_CTX object + * \param r The resulting plaintext integer + * \param cihpertext EC_ELGAMAL_CIPHERTEXT object to be decrypted + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_decrypt(EC_ELGAMAL_CTX *ctx, int32_t *r, + const EC_ELGAMAL_CIPHERTEXT *ciphertext) +{ + int ret = 0; + int32_t plaintext = 0; + EC_POINT *M = NULL; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL || ctx->key == NULL || ctx->key->priv_key == NULL || r == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + M = EC_POINT_new(ctx->key->group); + if (M == NULL) + goto err; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + BN_CTX_start(bn_ctx); +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + if (ctx->flag == EC_ELGAMAL_FLAG_TWISTED) { + if (!EC_POINT_mul(ctx->key->group, M, NULL, ciphertext->C1, + ctx->pk_inv, bn_ctx)) + goto err; + } else { +#endif + if (!EC_POINT_mul(ctx->key->group, M, NULL, ciphertext->C1, + ctx->key->priv_key, bn_ctx)) + goto err; +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + } +#endif + + if (!EC_POINT_invert(ctx->key->group, M, bn_ctx)) + goto err; + + if (!EC_POINT_add(ctx->key->group, M, ciphertext->C2, M, bn_ctx)) + goto err; + + if (ctx->decrypt_table != NULL) { + if (!EC_ELGAMAL_dlog_bsgs(ctx, &plaintext, M)) + goto err; + } else { + if (!EC_ELGAMAL_dlog_brute(ctx, &plaintext, M)) + goto err; + } + + *r = plaintext; + + ret = 1; + +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + EC_POINT_free(M); + return ret; +} + +/** Adds two EC-Elgamal ciphertext and stores it in r (r = c1 + c2). + * \param ctx EC_ELGAMAL_CTX object + * \param r The EC_ELGAMAL_CIPHERTEXT object that stores the addition + * result + * \param c1 EC_ELGAMAL_CIPHERTEXT object + * \param c2 EC_ELGAMAL_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_add(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + const EC_ELGAMAL_CIPHERTEXT *c1, + const EC_ELGAMAL_CIPHERTEXT *c2) +{ + int ret = 0; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL || ctx->key == NULL || r == NULL || c1 == NULL || c2 == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!EC_POINT_add(ctx->key->group, r->C1, c1->C1, c2->C1, bn_ctx)) + goto err; + + if (!EC_POINT_add(ctx->key->group, r->C2, c1->C2, c2->C2, bn_ctx)) + goto err; + + ret = 1; + +err: + BN_CTX_free(bn_ctx); + return ret; +} + +/** Substracts two EC-Elgamal ciphertext and stores it in r (r = c1 - c2). + * \param ctx EC_ELGAMAL_CTX object + * \param r The EC_ELGAMAL_CIPHERTEXT object that stores the + * subtraction result + * \param c1 EC_ELGAMAL_CIPHERTEXT object + * \param c2 EC_ELGAMAL_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_sub(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + const EC_ELGAMAL_CIPHERTEXT *c1, + const EC_ELGAMAL_CIPHERTEXT *c2) +{ + int ret = 0; + BN_CTX *bn_ctx = NULL; + EC_POINT *C1_inv = NULL, *C2_inv = NULL; + + if (ctx == NULL || ctx->key == NULL || r == NULL || c1 == NULL || c2 == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if ((C1_inv = EC_POINT_dup(c2->C1, ctx->key->group)) == NULL) + goto err; + + if ((C2_inv = EC_POINT_dup(c2->C2, ctx->key->group)) == NULL) + goto err; + + if (!EC_POINT_invert(ctx->key->group, C1_inv, bn_ctx)) + goto err; + + if (!EC_POINT_invert(ctx->key->group, C2_inv, bn_ctx)) + goto err; + + if (!EC_POINT_add(ctx->key->group, r->C1, c1->C1, C1_inv, bn_ctx)) + goto err; + + if (!EC_POINT_add(ctx->key->group, r->C2, c1->C2, C2_inv, bn_ctx)) + goto err; + + ret = 1; + +err: + EC_POINT_free(C1_inv); + EC_POINT_free(C2_inv); + BN_CTX_free(bn_ctx); + return ret; +} + +/** Ciphertext multiplication, computes r = c * m + * \param ctx EC_ELGAMAL_CTX object + * \param r The EC_ELGAMAL_CIPHERTEXT object that stores the + * multiplication result + * \param c1 EC_ELGAMAL_CIPHERTEXT object + * \param c2 EC_ELGAMAL_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_mul(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + const EC_ELGAMAL_CIPHERTEXT *c, int32_t m) +{ + int ret = 0; + BIGNUM *bn_m; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL || ctx->key == NULL || r == NULL || c == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return ret; + } + + BN_CTX_start(bn_ctx); + + if (m == 0) { + ret = EC_ELGAMAL_encrypt(ctx, r, 0); + goto end; + } + + bn_m = BN_CTX_get(bn_ctx); + if (bn_m == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto end; + } + BN_set_word(bn_m, (BN_ULONG)(m > 0 ? m : -(int64_t)m)); + BN_set_negative(bn_m, m < 0 ? 1 : 0); + + if (!EC_POINT_mul(ctx->key->group, r->C1, NULL, c->C1, bn_m, bn_ctx)) + goto end; + + if (!EC_POINT_mul(ctx->key->group, r->C2, NULL, c->C2, bn_m, bn_ctx)) + goto end; + + ret = 1; + +end: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} diff --git a/openssl/src/crypto/ec/ec_elgamal_dlog.c b/openssl/src/crypto/ec/ec_elgamal_dlog.c new file mode 100644 index 000000000..bafaea276 --- /dev/null +++ b/openssl/src/crypto/ec/ec_elgamal_dlog.c @@ -0,0 +1,491 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include "ec_elgamal.h" +#include +#include + +#define EC_ELGAMAL_MSG_BITS 32 +#define EC_ELGAMAL_ECDLP_BABY_BITS 11 +#define EC_ELGAMAL_ECDLP_GIANT_BITS (EC_ELGAMAL_MSG_BITS-EC_ELGAMAL_ECDLP_BABY_BITS) + +static EC_ELGAMAL_dec_tbl_entry *EC_ELGAMAL_dec_tbl_entry_new(EC_ELGAMAL_CTX *ctx, + EC_POINT *point, + int32_t value); +static void EC_ELGAMAL_dec_tbl_entry_free(EC_ELGAMAL_dec_tbl_entry *entry); + +static unsigned long EC_ELGAMAL_dec_tbl_entry_hash(const EC_ELGAMAL_dec_tbl_entry *e) +{ + int i = e->key_len; + unsigned char *p = e->key; + + while (*p == 0 && i-- > 0) + p++; + + return ossl_lh_strcasehash((const char *)p); +} + +static int EC_ELGAMAL_dec_tbl_entry_cmp(const EC_ELGAMAL_dec_tbl_entry *a, + const EC_ELGAMAL_dec_tbl_entry *b) +{ + if (a->key_len != b->key_len) + return -1; + + return memcmp(a->key, b->key, a->key_len); +} + +/** Finds the value r with brute force s.t. M=rG + * \param ctx EC_ELGAMAL_CTX object + * \param r The resulting integer + * \param M EC_POINT object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_dlog_brute(EC_ELGAMAL_CTX *ctx, int32_t *r, EC_POINT *M) +{ + int ret = 0; + int64_t i = 1, max = 1LL << EC_ELGAMAL_MAX_BITS; + const EC_POINT *G; + EC_POINT *P = NULL; + BN_CTX *bn_ctx = NULL; + + if (EC_POINT_is_at_infinity(ctx->key->group, M)) { + ERR_raise(ERR_LIB_EC, EC_R_POINT_AT_INFINITY); + goto err; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + P = EC_POINT_new(ctx->key->group); + if (P == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + G = EC_GROUP_get0_generator(ctx->key->group); + EC_POINT_set_to_infinity(ctx->key->group, P); + + for (; i < max; i++) { + if (!EC_POINT_add(ctx->key->group, P, P, G, bn_ctx)) + goto err; + if (EC_POINT_cmp(ctx->key->group, P, M, bn_ctx) == 0) + break; + } + + if (i >= max) { + ERR_raise(ERR_LIB_EC, EC_R_ELGAMAL_DLOG_FAILED); + goto err; + } + + *r = (int32_t)i; + ret = 1; + +err: + EC_POINT_free(P); + BN_CTX_free(bn_ctx); + return ret; +} + +static +int EC_ELGAMAL_dec_tbl_entries_dlog(EC_ELGAMAL_CTX *ctx, int32_t *r, + LHASH_OF(EC_ELGAMAL_dec_tbl_entry) *entries, + EC_POINT *M, EC_POINT *Q, BN_CTX *bn_ctx) +{ + int ret = 0; + int64_t i, max; + EC_POINT *P = NULL; + EC_ELGAMAL_dec_tbl_entry *entry = NULL, *entry_res = NULL; + + max = 1L << ctx->decrypt_table->baby_step_bits; + + if ((P = EC_POINT_dup(M, ctx->key->group)) == NULL) + goto err; + + for (i = 0; i < max; i++) { + entry = EC_ELGAMAL_dec_tbl_entry_new(ctx, P, (int32_t)i); + if (entry == NULL) + goto err; + + entry_res = lh_EC_ELGAMAL_dec_tbl_entry_retrieve(entries, entry); + if (entry_res != NULL) { + ret = 1; + *r = (int32_t)(i * ctx->decrypt_table->size + entry_res->value); + break; + } + + if (!EC_POINT_add(ctx->key->group, P, P, Q, bn_ctx)) + goto err; + + EC_ELGAMAL_dec_tbl_entry_free(entry); + entry = NULL; + } + +err: + EC_ELGAMAL_dec_tbl_entry_free(entry); + EC_POINT_free(P); + return ret; +} + +/** Finds the value r with ecdlp bsgs hashtable. + * \param ctx EC_ELGAMAL_CTX object + * \param r The resulting integer + * \param M EC_POINT object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_dlog_bsgs(EC_ELGAMAL_CTX *ctx, int32_t *r, EC_POINT *M) +{ + int ret = 0; + int32_t result = 0; + EC_POINT *Q = NULL; + EC_ELGAMAL_DECRYPT_TABLE *table = ctx->decrypt_table; + BN_CTX *bn_ctx = NULL; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return ret; + } + + + if (table->flag == 0 || (table->flag & EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE)) + ret = EC_ELGAMAL_dec_tbl_entries_dlog(ctx, &result, table->positive_entries, + M, table->mG_inv, bn_ctx); + + if (ret) { + *r = result; + } else if (table->flag & (EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE | + EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE_FIRST | + EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE_ONLY)) { + if ((Q = EC_POINT_dup(table->mG_inv, ctx->key->group)) == NULL) + goto err; + + if (!EC_POINT_invert(ctx->key->group, Q, bn_ctx)) + goto err; + + ret = EC_ELGAMAL_dec_tbl_entries_dlog(ctx, &result, + table->negative_entries, M, Q, + bn_ctx); + if (!ret && (table->flag & EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE_FIRST)) + ret = EC_ELGAMAL_dec_tbl_entries_dlog(ctx, &result, table->positive_entries, + M, table->mG_inv, bn_ctx); + + if (ret) + *r = -result; + else + ERR_raise(ERR_LIB_EC, EC_R_ELGAMAL_DLOG_FAILED); + } + +err: + EC_POINT_free(Q); + BN_CTX_free(bn_ctx); + return ret; +} + +/** Creates a new EC_ELGAMAL_dec_tbl_entry object + * \param ctx EC_ELGAMAL_CTX object + * \param point EC_POINT object + * \return newly created EC_ELGAMAL_dec_tbl_entry object or NULL in case of an error + */ +static EC_ELGAMAL_dec_tbl_entry *EC_ELGAMAL_dec_tbl_entry_new(EC_ELGAMAL_CTX *ctx, + EC_POINT *point, + int32_t value) +{ + EC_ELGAMAL_dec_tbl_entry *entry = NULL; + size_t point_size = 0, len = 0; + unsigned char *point_key = NULL; + BN_CTX *bn_ctx = NULL; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + point_size = EC_POINT_point2oct(ctx->key->group, point, + POINT_CONVERSION_COMPRESSED, NULL, 0, + bn_ctx); + if (point_size <= 0) + goto err; + + entry = OPENSSL_zalloc(sizeof(*entry)); + if (entry == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + point_key = OPENSSL_zalloc(point_size + 1); + if (point_key == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + if ((len = EC_POINT_point2oct(ctx->key->group, point, + POINT_CONVERSION_COMPRESSED, point_key, + point_size, bn_ctx)) != point_size) + goto err; + + entry->key_len = (int)point_size; + entry->key = point_key; + entry->value = value; + + BN_CTX_free(bn_ctx); + + return entry; + +err: + OPENSSL_free(point_key); + OPENSSL_free(entry); + BN_CTX_free(bn_ctx); + return NULL; +} + +/** Frees a EC_ELGAMAL_dec_tbl_entry object + * \param entry EC_ELGAMAL_dec_tbl_entry object to be freed + */ +static void EC_ELGAMAL_dec_tbl_entry_free(EC_ELGAMAL_dec_tbl_entry *entry) +{ + if (entry == NULL) + return; + + OPENSSL_free(entry->key); + OPENSSL_free(entry); +} + +static int EC_ELGAMAL_dec_table_entries_init(EC_ELGAMAL_CTX *ctx, + LHASH_OF(EC_ELGAMAL_dec_tbl_entry) *entries, + int32_t size, const EC_POINT *G, + BN_CTX *bn_ctx) +{ + int ret = 0; + int32_t i; + EC_POINT *P; + EC_GROUP *group; + EC_ELGAMAL_dec_tbl_entry *entry = NULL, *entry_old = NULL; + + if (ctx == NULL || ctx->key == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + group = ctx->key->group; + + P = EC_POINT_new(group); + if (P == NULL) + return ret; + + EC_POINT_set_to_infinity(group, P); + + for (i = 0; i < size; i++) { + entry = EC_ELGAMAL_dec_tbl_entry_new(ctx, P, i); + if (entry == NULL) + goto err; + + entry_old = lh_EC_ELGAMAL_dec_tbl_entry_insert(entries, entry); + if (lh_EC_ELGAMAL_dec_tbl_entry_error(entries) && entry_old == NULL) + goto err; + + if (entry_old != NULL) + EC_ELGAMAL_dec_tbl_entry_free(entry_old); + + entry = NULL; + + if (!EC_POINT_add(group, P, P, G, bn_ctx)) + goto err; + } + + ret = 1; +err: + EC_ELGAMAL_dec_tbl_entry_free(entry); + EC_POINT_free(P); + return ret; +} + +/** Creates a new EC_ELGAMAL_DECRYPT_TABLE object + * \param ctx EC_ELGAMAL_CTX object + * \param decrypt_negative Whether negative numbers can be decrypted (1 or 0) + * \return newly created EC_ELGAMAL_DECRYPT_TABLE object or NULL in case of an error + */ +EC_ELGAMAL_DECRYPT_TABLE *EC_ELGAMAL_DECRYPT_TABLE_new(EC_ELGAMAL_CTX *ctx, + int32_t decrypt_negative) +{ + return EC_ELGAMAL_DECRYPT_TABLE_new_ex(ctx, EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE, + EC_ELGAMAL_ECDLP_BABY_BITS, + EC_ELGAMAL_ECDLP_GIANT_BITS); +} + +/** Creates a new EC_ELGAMAL_DECRYPT_TABLE object with some extra paramers + * \param ctx EC_ELGAMAL_CTX object + * \param flag the flag of decrypt table + * \param baby_step_bits baby step exponent/bits + * \param giant_step_bits giant step exponent/bits + * \return newly created EC_ELGAMAL_DECRYPT_TABLE object or NULL in case of an error + */ +EC_ELGAMAL_DECRYPT_TABLE *EC_ELGAMAL_DECRYPT_TABLE_new_ex(EC_ELGAMAL_CTX *ctx, + int32_t flag, + uint32_t baby_step_bits, + uint32_t giant_step_bits) +{ + int32_t size; + EC_ELGAMAL_DECRYPT_TABLE *table = NULL; + LHASH_OF(EC_ELGAMAL_dec_tbl_entry) *positive_entries = NULL, *negative_entries = NULL; + EC_GROUP *group; + EC_POINT *mG_inv = NULL, *Q = NULL; + const EC_POINT *G; + BIGNUM *bn_size = NULL; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL || ctx->key == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (giant_step_bits > 32 || baby_step_bits > 32 || + (giant_step_bits + baby_step_bits) > 32) + return NULL; + + size = 1L << giant_step_bits; + group = ctx->key->group; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + table = OPENSSL_zalloc(sizeof(*table)); + if (table == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + table->size = size; + + bn_size = BN_CTX_get(bn_ctx); + if (bn_size == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + BN_set_word(bn_size, (BN_ULONG)size); + BN_set_negative(bn_size, 0); + + G = EC_GROUP_get0_generator(group); + + mG_inv = EC_POINT_new(group); + if (mG_inv == NULL) + goto err; + +#ifndef OPENSSL_NO_TWISTED_EC_ELGAMAL + if (ctx->flag == EC_ELGAMAL_FLAG_TWISTED) { + if (!EC_POINT_mul(group, mG_inv, NULL, ctx->h, bn_size, bn_ctx)) + goto err; + G = ctx->h; + } else +#endif + if (!EC_POINT_mul(group, mG_inv, bn_size, NULL, NULL, bn_ctx)) + goto err; + + if (!EC_POINT_invert(group, mG_inv, bn_ctx)) + goto err; + + positive_entries = lh_EC_ELGAMAL_dec_tbl_entry_new(EC_ELGAMAL_dec_tbl_entry_hash, + EC_ELGAMAL_dec_tbl_entry_cmp); + if (positive_entries == NULL + || !EC_ELGAMAL_dec_table_entries_init(ctx, positive_entries, size, G, bn_ctx)) + goto err; + + if (flag & (EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE | + EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE_FIRST | + EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE_ONLY)) { + negative_entries = lh_EC_ELGAMAL_dec_tbl_entry_new(EC_ELGAMAL_dec_tbl_entry_hash, + EC_ELGAMAL_dec_tbl_entry_cmp); + if (negative_entries == NULL) + goto err; + + if ((Q = EC_POINT_dup(G, group)) == NULL) + goto err; + + if (!EC_POINT_invert(group, Q, bn_ctx)) + goto err; + + if (!EC_ELGAMAL_dec_table_entries_init(ctx, negative_entries, size, Q, bn_ctx)) + goto err; + + EC_POINT_free(Q); + table->flag |= EC_ELGAMAL_DECRYPT_TABLE_FLAG_NEGATIVE; + } + + table->mG_inv = mG_inv; + table->positive_entries = positive_entries; + table->negative_entries = negative_entries; + table->flag = flag; + table->baby_step_bits = baby_step_bits; + table->giant_step_bits = giant_step_bits; + + table->references = 1; + table->lock = CRYPTO_THREAD_lock_new(); + + BN_CTX_free(bn_ctx); + + return table; + +err: + lh_EC_ELGAMAL_dec_tbl_entry_doall(positive_entries, EC_ELGAMAL_dec_tbl_entry_free); + lh_EC_ELGAMAL_dec_tbl_entry_free(positive_entries); + + lh_EC_ELGAMAL_dec_tbl_entry_doall(negative_entries, EC_ELGAMAL_dec_tbl_entry_free); + lh_EC_ELGAMAL_dec_tbl_entry_free(negative_entries); + + EC_POINT_free(Q); + EC_POINT_free(mG_inv); + OPENSSL_free(table); + BN_CTX_free(bn_ctx); + return NULL; +} + +/** Frees a EC_ELGAMAL_DECRYPT_TABLE object + * \param table EC_ELGAMAL_DECRYPT_TABLE object to be freed + */ +void EC_ELGAMAL_DECRYPT_TABLE_free(EC_ELGAMAL_DECRYPT_TABLE *table) +{ + int i; + + if (table == NULL) + return; + + CRYPTO_DOWN_REF(&table->references, &i, table->lock); + + if (i > 0) + return; + + lh_EC_ELGAMAL_dec_tbl_entry_doall(table->positive_entries, EC_ELGAMAL_dec_tbl_entry_free); + lh_EC_ELGAMAL_dec_tbl_entry_free(table->positive_entries); + + lh_EC_ELGAMAL_dec_tbl_entry_doall(table->negative_entries, EC_ELGAMAL_dec_tbl_entry_free); + lh_EC_ELGAMAL_dec_tbl_entry_free(table->negative_entries); + + EC_POINT_free(table->mG_inv); + CRYPTO_THREAD_lock_free(table->lock); + OPENSSL_free(table); +} + +/** Sets a EC_ELGAMAL_DECRYPT_TABLE object for decryption. + * \param ctx EC_ELGAMAL_CTX object + * \param table EC_ELGAMAL_DECRYPT_TABLE object + */ +void EC_ELGAMAL_CTX_set_decrypt_table(EC_ELGAMAL_CTX *ctx, + EC_ELGAMAL_DECRYPT_TABLE *table) +{ + int i; + + ctx->decrypt_table = table; + CRYPTO_UP_REF(&table->references, &i, table->lock); +} diff --git a/openssl/src/crypto/ec/ec_elgamal_encode.c b/openssl/src/crypto/ec/ec_elgamal_encode.c new file mode 100644 index 000000000..b616521ff --- /dev/null +++ b/openssl/src/crypto/ec/ec_elgamal_encode.c @@ -0,0 +1,541 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include "ec_elgamal.h" +#include +#include + +#ifdef __bswap_constant_32 +# undef __bswap_constant_32 +#endif +#define __bswap_constant_32(x) \ + ((((uint32_t)(x) & 0xff000000u) >> 24) | \ + (((uint32_t)(x) & 0x00ff0000u) >> 8) | \ + (((uint32_t)(x) & 0x0000ff00u) << 8) | \ + (((uint32_t)(x) & 0x000000ffu) << 24)) + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define n2l(x) (x) +# define l2n(x) (x) +#else +# define n2l(x) __bswap_constant_32(x) +# define l2n(x) __bswap_constant_32(x) +#endif + +DEFINE_STACK_OF(EC_POINT) + +static int stack_of_point_encode(STACK_OF(EC_POINT) *sk, unsigned char *out, + point_conversion_form_t form, + const EC_GROUP *group, BN_CTX *bn_ctx) +{ + int i, n, *q; + size_t point_len; + unsigned char *p = out; + EC_POINT *P; + + if (sk == NULL || group == NULL) + return 0; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + n = sk_EC_POINT_num(sk); + if (out == NULL) + return 1 + sizeof(n) + n * point_len; + + *p++ = form == POINT_CONVERSION_COMPRESSED ? 0x1 : 0; + + q = (int *)p; + *q++ = l2n((int)n); + p = (unsigned char *)q; + + for (i = 0; i < n; i++) { + P = sk_EC_POINT_value(sk, i); + if (P == NULL) + goto end; + + if (EC_POINT_point2oct(group, P, form, p, point_len, bn_ctx) == 0) + goto end; + + p += point_len; + } + +end: + return p - out; +} + +static STACK_OF(EC_POINT) *stack_of_point_decode(const unsigned char *in, + int *len, + const EC_GROUP *group, + BN_CTX *bn_ctx) +{ + unsigned char *p = (unsigned char *)in; + int *q, n, i; + size_t point_len; + EC_POINT *P = NULL; + STACK_OF(EC_POINT) *ret = NULL; + point_conversion_form_t form; + + if (in == NULL || group == NULL) + return 0; + + form = *p == 0x1 ? POINT_CONVERSION_COMPRESSED : POINT_CONVERSION_UNCOMPRESSED; + p++; + q = (int *)p; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + + n = (int)n2l(*q); + q++; + p = (unsigned char *)q; + + if (n < 0) { + return NULL; + } + + if (!(ret = sk_EC_POINT_new_reserve(NULL, n))) + return NULL; + + for (i = 0; i < n; i++) { + if (!(P = EC_POINT_new(group))) + goto err; + + if (!EC_POINT_oct2point(group, P, p, point_len, bn_ctx)) + goto err; + + if (sk_EC_POINT_push(ret, P) <= 0) + goto err; + + p += point_len; + } + + if (len != NULL) + *len = p - in; + + return ret; +err: + EC_POINT_free(P); + sk_EC_POINT_pop_free(ret, EC_POINT_free); + return NULL; +} + +/** Creates a new EC_ELGAMAL_CIPHERTEXT object for EC-ELGAMAL oparations + * \param ctx EC_ELGAMAL_CTX object + * \return newly created EC_ELGAMAL_CIPHERTEXT object or NULL in case of an error + */ +EC_ELGAMAL_CIPHERTEXT *EC_ELGAMAL_CIPHERTEXT_new(EC_ELGAMAL_CTX *ctx) +{ + EC_POINT *C1 = NULL, *C2 = NULL; + EC_ELGAMAL_CIPHERTEXT *ciphertext; + + ciphertext = OPENSSL_zalloc(sizeof(*ciphertext)); + if (ciphertext == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return NULL; + } + + C1 = EC_POINT_new(ctx->key->group); + if (C1 == NULL) + goto err; + + C2 = EC_POINT_new(ctx->key->group); + if (C2 == NULL) + goto err; + + ciphertext->C1 = C1; + ciphertext->C2 = C2; + + return ciphertext; + +err: + EC_POINT_free(C1); + EC_POINT_free(C2); + OPENSSL_free(ciphertext); + return NULL; +} + +EC_ELGAMAL_CIPHERTEXT *EC_ELGAMAL_CIPHERTEXT_dup(const EC_ELGAMAL_CIPHERTEXT *ct, + const EC_GROUP *group) +{ + EC_ELGAMAL_CIPHERTEXT *ret; + + if (ct == NULL || group == NULL) + return NULL; + + ret = OPENSSL_zalloc(sizeof(*ret)); + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ret->C1 = EC_POINT_dup(ct->C1, group); + ret->C2 = EC_POINT_dup(ct->C2, group); + if (ret->C1 == NULL || ret->C2 == NULL) + goto err; + + return ret; + +err: + EC_ELGAMAL_CIPHERTEXT_free(ret); + return NULL; +} + +/** Frees a EC_ELGAMAL_CIPHERTEXT object + * \param ciphertext EC_ELGAMAL_CIPHERTEXT object to be freed + */ +void EC_ELGAMAL_CIPHERTEXT_free(EC_ELGAMAL_CIPHERTEXT *ciphertext) +{ + if (ciphertext == NULL) + return; + + EC_POINT_free(ciphertext->C1); + EC_POINT_free(ciphertext->C2); + + OPENSSL_free(ciphertext); +} + +/** Encodes EC_ELGAMAL_CIPHERTEXT to binary + * \param ctx EC_ELGAMAL_CTX object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param ciphertext EC_ELGAMAL_CIPHERTEXT object + * \param compressed Whether to compress the encoding (either 0 or 1) + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t EC_ELGAMAL_CIPHERTEXT_encode(EC_ELGAMAL_CTX *ctx, unsigned char *out, + size_t size, + const EC_ELGAMAL_CIPHERTEXT *ciphertext, + int compressed) +{ + size_t point_len, ret = 0, len, plen; + unsigned char *p = out; + point_conversion_form_t form = compressed ? POINT_CONVERSION_COMPRESSED : + POINT_CONVERSION_UNCOMPRESSED; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL || ctx->key == NULL || ciphertext == NULL || + ciphertext->C1 == NULL || ciphertext->C2 == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto end; + } + + point_len = EC_POINT_point2oct(ctx->key->group, + EC_GROUP_get0_generator(ctx->key->group), + form, NULL, 0, bn_ctx); + len = point_len * 2; + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + plen = EC_POINT_point2oct(ctx->key->group, ciphertext->C1, form, p, + point_len, bn_ctx); + if (plen == 0) + goto end; + + p += point_len; + + plen = EC_POINT_point2oct(ctx->key->group, ciphertext->C2, form, p, + point_len, bn_ctx); + if (plen == 0) + goto end; + + ret = len; + +end: + BN_CTX_free(bn_ctx); + return ret; +} + +/** Decodes binary to EC_ELGAMAL_CIPHERTEXT + * \param ctx EC_ELGAMAL_CTX object + * \param r the resulting ciphertext + * \param in Memory buffer with the encoded EC_ELGAMAL_CIPHERTEXT + * object + * \param size The memory size of the in pointer object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_CIPHERTEXT_decode(EC_ELGAMAL_CTX *ctx, EC_ELGAMAL_CIPHERTEXT *r, + unsigned char *in, size_t size) +{ + int ret = 0; + size_t point_len; + unsigned char *p = in, zero[128]; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL || ctx->key == NULL || r == NULL || r->C1 == NULL || + r->C2 == NULL || size % 2 != 0 || in == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + point_len = size / 2; + memset(zero, 0, sizeof(zero)); + + if (!EC_POINT_oct2point(ctx->key->group, r->C1, p, point_len, bn_ctx)) { + if (memcmp(p, zero, point_len) != 0 || + !EC_POINT_set_to_infinity(ctx->key->group, r->C1)) + goto err; + } + + p += point_len; + + if (!EC_POINT_oct2point(ctx->key->group, r->C2, p, point_len, bn_ctx)) { + if (memcmp(p, zero, point_len) != 0 || + !EC_POINT_set_to_infinity(ctx->key->group, r->C2)) + goto err; + } + + ret = 1; + +err: + BN_CTX_free(bn_ctx); + return ret; +} + +/** Creates a new EC_ELGAMAL_MR_CIPHERTEXT object for EC-ELGAMAL oparations + * \param ctx EC_ELGAMAL_MR_CTX object + * \return newly created EC_ELGAMAL_MR_CIPHERTEXT object or NULL in case of an error + */ +EC_ELGAMAL_MR_CIPHERTEXT *EC_ELGAMAL_MR_CIPHERTEXT_new(EC_ELGAMAL_MR_CTX *ctx) +{ + STACK_OF(EC_POINT) *sk_C1 = NULL; + EC_ELGAMAL_MR_CIPHERTEXT *ciphertext; + EC_POINT *C2 = NULL; + + ciphertext = OPENSSL_zalloc(sizeof(*ciphertext)); + if (ciphertext == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return NULL; + } + + sk_C1 = sk_EC_POINT_new_null(); + if (sk_C1 == NULL) + goto err; + + C2 = EC_POINT_new(ctx->group); + if (C2 == NULL) + goto err; + + ciphertext->sk_C1 = sk_C1; + ciphertext->C2 = C2; + + return ciphertext; + +err: + sk_EC_POINT_free(sk_C1); + EC_POINT_free(C2); + OPENSSL_free(ciphertext); + return NULL; +} + +EC_ELGAMAL_MR_CIPHERTEXT *EC_ELGAMAL_MR_CIPHERTEXT_dup(const EC_ELGAMAL_MR_CIPHERTEXT *ct, + const EC_GROUP *group) +{ + int i; + STACK_OF(EC_POINT) *sk_C1 = NULL; + EC_ELGAMAL_MR_CIPHERTEXT *ret; + EC_POINT *P = NULL, *C1, *C2 = NULL; + + if (ct == NULL || group == NULL) + return NULL; + + ret = OPENSSL_zalloc(sizeof(*ret)); + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return NULL; + } + + sk_C1 = sk_EC_POINT_new_reserve(NULL, sk_EC_POINT_num(ct->sk_C1)); + if (sk_C1 == NULL) + goto err; + + for (i = 0; i < sk_EC_POINT_num(ct->sk_C1); i++) { + C1 = sk_EC_POINT_value(ct->sk_C1, i); + if (!(P = EC_POINT_dup(C1, group))) + goto err; + + if (sk_EC_POINT_push(sk_C1, P) <= 0) + goto err; + } + + C2 = EC_POINT_dup(ct->C2, group); + if (C2 == NULL) + goto err; + + ret->sk_C1 = sk_C1; + ret->C2 = C2; + + return ret; + +err: + sk_EC_POINT_pop_free(sk_C1, EC_POINT_free); + EC_POINT_free(P); + EC_POINT_free(C2); + OPENSSL_free(ret); + return NULL; +} + +/** Frees a EC_ELGAMAL_MR_CIPHERTEXT object + * \param ciphertext EC_ELGAMAL_MR_CIPHERTEXT object to be freed + */ +void EC_ELGAMAL_MR_CIPHERTEXT_free(EC_ELGAMAL_MR_CIPHERTEXT *ciphertext) +{ + if (ciphertext == NULL) + return; + + sk_EC_POINT_pop_free(ciphertext->sk_C1, EC_POINT_free); + EC_POINT_free(ciphertext->C2); + + OPENSSL_free(ciphertext); +} + +/** Encodes EC_ELGAMAL_MR_CIPHERTEXT to binary + * \param ctx EC_ELGAMAL_MR_CTX object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param ciphertext EC_ELGAMAL_MR_CIPHERTEXT object + * \param compressed Whether to compress the encoding (either 0 or 1) + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t EC_ELGAMAL_MR_CIPHERTEXT_encode(EC_ELGAMAL_MR_CTX *ctx, unsigned char *out, + size_t size, + const EC_ELGAMAL_MR_CIPHERTEXT *ciphertext, + int compressed) +{ + size_t point_len, ret = 0, len, plen; + unsigned char *p = out; + point_conversion_form_t form = compressed ? POINT_CONVERSION_COMPRESSED : + POINT_CONVERSION_UNCOMPRESSED; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL || ctx->group == NULL || ciphertext == NULL || + ciphertext->sk_C1 == NULL || ciphertext->C2 == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto end; + } + + point_len = EC_POINT_point2oct(ctx->group, + EC_GROUP_get0_generator(ctx->group), + form, NULL, 0, bn_ctx); + + len = stack_of_point_encode(ciphertext->sk_C1, NULL, form, ctx->group, bn_ctx); + + len += point_len; + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + len = stack_of_point_encode(ciphertext->sk_C1, p, form, ctx->group, bn_ctx); + if (len <= 0) + goto end; + + p += len; + + plen = EC_POINT_point2oct(ctx->group, ciphertext->C2, form, p, + point_len, bn_ctx); + if (plen == 0) + goto end; + + ret = len; + +end: + BN_CTX_free(bn_ctx); + return ret; +} + +/** Decodes binary to EC_ELGAMAL_MR_CIPHERTEXT + * \param ctx EC_ELGAMAL_MR_CTX object + * \param r the resulting ciphertext + * \param in Memory buffer with the encoded EC_ELGAMAL_MR_CIPHERTEXT + * object + * \param size The memory size of the in pointer object + * \return 1 on success and 0 otherwise + */ +int EC_ELGAMAL_MR_CIPHERTEXT_decode(EC_ELGAMAL_MR_CTX *ctx, EC_ELGAMAL_MR_CIPHERTEXT *r, + unsigned char *in, size_t size) +{ + int ret = 0, len = 0; + size_t point_len; + unsigned char *p = in, zero[128]; + BN_CTX *bn_ctx = NULL; + STACK_OF(EC_POINT) *sk_C1 = NULL; + point_conversion_form_t form; + + if (ctx == NULL || ctx->group == NULL || r == NULL || r->C2 == NULL || in == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + memset(zero, 0, sizeof(zero)); + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + form = *p == 0x1 ? POINT_CONVERSION_COMPRESSED : POINT_CONVERSION_UNCOMPRESSED; + + point_len = EC_POINT_point2oct(ctx->group, EC_GROUP_get0_generator(ctx->group), + form, NULL, 0, bn_ctx); + + sk_C1 = stack_of_point_decode(p, &len, ctx->group, bn_ctx); + p += len; + + if (!EC_POINT_oct2point(ctx->group, r->C2, p, point_len, bn_ctx)) { + if (memcmp(p, zero, point_len) != 0 || + !EC_POINT_set_to_infinity(ctx->group, r->C2)) + goto err; + } + + if (r->sk_C1 != NULL) { + sk_EC_POINT_pop_free(r->sk_C1, EC_POINT_free); + } + + r->sk_C1 = sk_C1; + sk_C1 = NULL; + ret = 1; + +err: + sk_EC_POINT_pop_free(sk_C1, EC_POINT_free); + BN_CTX_free(bn_ctx); + return ret; +} diff --git a/openssl/src/crypto/ec/ec_err.c b/openssl/src/crypto/ec/ec_err.c index 480376686..2d378e928 100644 --- a/openssl/src/crypto/ec/ec_err.c +++ b/openssl/src/crypto/ec/ec_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -35,6 +35,9 @@ static const ERR_STRING_DATA EC_str_reasons[] = { "discriminant is zero"}, {ERR_PACK(ERR_LIB_EC, 0, EC_R_EC_GROUP_NEW_BY_NAME_FAILURE), "ec group new by name failure"}, + {ERR_PACK(ERR_LIB_EC, 0, EC_R_EC_POINT_METHOD_NOT_FOUND), + "ec point method not found"}, + {ERR_PACK(ERR_LIB_EC, 0, EC_R_ELGAMAL_DLOG_FAILED), "elgamal dlog failed"}, {ERR_PACK(ERR_LIB_EC, 0, EC_R_EXPLICIT_PARAMS_NOT_SUPPORTED), "explicit params not supported"}, {ERR_PACK(ERR_LIB_EC, 0, EC_R_FAILED_MAKING_PUBLIC_KEY), @@ -108,7 +111,6 @@ static const ERR_STRING_DATA EC_str_reasons[] = { "random number generation failed"}, {ERR_PACK(ERR_LIB_EC, 0, EC_R_SHARED_INFO_ERROR), "shared info error"}, {ERR_PACK(ERR_LIB_EC, 0, EC_R_SLOT_FULL), "slot full"}, - {ERR_PACK(ERR_LIB_EC, 0, EC_R_TOO_MANY_RETRIES), "too many retries"}, {ERR_PACK(ERR_LIB_EC, 0, EC_R_UNDEFINED_GENERATOR), "undefined generator"}, {ERR_PACK(ERR_LIB_EC, 0, EC_R_UNDEFINED_ORDER), "undefined order"}, {ERR_PACK(ERR_LIB_EC, 0, EC_R_UNKNOWN_COFACTOR), "unknown cofactor"}, diff --git a/openssl/src/crypto/ec/ec_key.c b/openssl/src/crypto/ec/ec_key.c index 9bc4e032c..eb14f4e40 100644 --- a/openssl/src/crypto/ec/ec_key.c +++ b/openssl/src/crypto/ec/ec_key.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -24,7 +24,6 @@ #endif #include #include "prov/providercommon.h" -#include "prov/ecx.h" #include "crypto/bn.h" static int ecdsa_keygen_pairwise_test(EC_KEY *eckey, OSSL_CALLBACK *cb, @@ -75,7 +74,7 @@ void EC_KEY_free(EC_KEY *r) if (r == NULL) return; - CRYPTO_DOWN_REF(&r->references, &i); + CRYPTO_DOWN_REF(&r->references, &i, r->lock); REF_PRINT_COUNT("EC_KEY", r); if (i > 0) return; @@ -94,7 +93,7 @@ void EC_KEY_free(EC_KEY *r) #ifndef FIPS_MODULE CRYPTO_free_ex_data(CRYPTO_EX_INDEX_EC_KEY, r, &r->ex_data); #endif - CRYPTO_FREE_REF(&r->references); + CRYPTO_THREAD_lock_free(r->lock); EC_GROUP_free(r->group); EC_POINT_free(r->pub_key); BN_clear_free(r->priv_key); @@ -194,7 +193,7 @@ int EC_KEY_up_ref(EC_KEY *r) { int i; - if (CRYPTO_UP_REF(&r->references, &i) <= 0) + if (CRYPTO_UP_REF(&r->references, &i, r->lock) <= 0) return 0; REF_PRINT_COUNT("EC_KEY", r); @@ -237,56 +236,6 @@ int ossl_ec_key_gen(EC_KEY *eckey) return ret; } -/* - * Refer: FIPS 140-3 IG 10.3.A Additional Comment 1 - * Perform a KAT by duplicating the public key generation. - * - * NOTE: This issue requires a background understanding, provided in a separate - * document; the current IG 10.3.A AC1 is insufficient regarding the PCT for - * the key agreement scenario. - * - * Currently IG 10.3.A requires PCT in the mode of use prior to use of the - * key pair, citing the PCT defined in the associated standard. For key - * agreement, the only PCT defined in SP 800-56A is that of Section 5.6.2.4: - * the comparison of the original public key to a newly calculated public key. - */ -static int ecdsa_keygen_knownanswer_test(EC_KEY *eckey, BN_CTX *ctx, - OSSL_CALLBACK *cb, void *cbarg) -{ - int len, ret = 0; - OSSL_SELF_TEST *st = NULL; - unsigned char bytes[512] = {0}; - EC_POINT *pub_key2 = EC_POINT_new(eckey->group); - - if (pub_key2 == NULL) - return 0; - - st = OSSL_SELF_TEST_new(cb, cbarg); - if (st == NULL) - return 0; - - OSSL_SELF_TEST_onbegin(st, OSSL_SELF_TEST_TYPE_PCT_KAT, - OSSL_SELF_TEST_DESC_PCT_ECDSA); - - /* pub_key = priv_key * G (where G is a point on the curve) */ - if (!EC_POINT_mul(eckey->group, pub_key2, eckey->priv_key, NULL, NULL, ctx)) - goto err; - - if (BN_num_bytes(pub_key2->X) > (int)sizeof(bytes)) - goto err; - len = BN_bn2bin(pub_key2->X, bytes); - if (OSSL_SELF_TEST_oncorrupt_byte(st, bytes) - && BN_bin2bn(bytes, len, pub_key2->X) == NULL) - goto err; - ret = !EC_POINT_cmp(eckey->group, eckey->pub_key, pub_key2, ctx); - -err: - OSSL_SELF_TEST_onend(st, ret); - OSSL_SELF_TEST_free(st); - EC_POINT_free(pub_key2); - return ret; -} - /* * ECC Key generation. * See SP800-56AR3 5.6.1.2.2 "Key Pair Generation by Testing Candidates" @@ -383,8 +332,7 @@ static int ec_generate_key(EC_KEY *eckey, int pairwise_test) void *cbarg = NULL; OSSL_SELF_TEST_get_callback(eckey->libctx, &cb, &cbarg); - ok = ecdsa_keygen_pairwise_test(eckey, cb, cbarg) - && ecdsa_keygen_knownanswer_test(eckey, ctx, cb, cbarg); + ok = ecdsa_keygen_pairwise_test(eckey, cb, cbarg); } err: /* Step (9): If there is an error return an invalid keypair. */ @@ -402,43 +350,6 @@ static int ec_generate_key(EC_KEY *eckey, int pairwise_test) return ok; } -#ifndef FIPS_MODULE -/* - * This is similar to ec_generate_key(), except it uses an ikm to - * derive the private key. - */ -int ossl_ec_generate_key_dhkem(EC_KEY *eckey, - const unsigned char *ikm, size_t ikmlen) -{ - int ok = 0; - - if (eckey->priv_key == NULL) { - eckey->priv_key = BN_secure_new(); - if (eckey->priv_key == NULL) - goto err; - } - if (ossl_ec_dhkem_derive_private(eckey, eckey->priv_key, ikm, ikmlen) <= 0) - goto err; - if (eckey->pub_key == NULL) { - eckey->pub_key = EC_POINT_new(eckey->group); - if (eckey->pub_key == NULL) - goto err; - } - if (!ossl_ec_key_simple_generate_public_key(eckey)) - goto err; - - ok = 1; -err: - if (!ok) { - BN_clear_free(eckey->priv_key); - eckey->priv_key = NULL; - if (eckey->pub_key != NULL) - EC_POINT_set_to_infinity(eckey->group, eckey->pub_key); - } - return ok; -} -#endif - int ossl_ec_key_simple_generate_key(EC_KEY *eckey) { return ec_generate_key(eckey, 0); @@ -809,16 +720,6 @@ int EC_KEY_set_private_key(EC_KEY *key, const BIGNUM *priv_key) && key->meth->set_private(key, priv_key) == 0) return 0; - /* - * Return `0` to comply with legacy behavior for this function, see - * https://github.com/openssl/openssl/issues/18744#issuecomment-1195175696 - */ - if (priv_key == NULL) { - BN_clear_free(key->priv_key); - key->priv_key = NULL; - return 0; /* intentional for legacy compatibility */ - } - /* * We should never leak the bit length of the secret scalar in the key, * so we always set the `BN_FLG_CONSTTIME` flag on the internal `BIGNUM` @@ -1043,10 +944,11 @@ int ossl_ec_key_simple_oct2priv(EC_KEY *eckey, const unsigned char *buf, if (eckey->priv_key == NULL) eckey->priv_key = BN_secure_new(); if (eckey->priv_key == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } - if (BN_bin2bn(buf, len, eckey->priv_key) == NULL) { + eckey->priv_key = BN_bin2bn(buf, len, eckey->priv_key); + if (eckey->priv_key == NULL) { ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); return 0; } @@ -1062,8 +964,10 @@ size_t EC_KEY_priv2buf(const EC_KEY *eckey, unsigned char **pbuf) len = EC_KEY_priv2oct(eckey, NULL, 0); if (len == 0) return 0; - if ((buf = OPENSSL_malloc(len)) == NULL) + if ((buf = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; + } len = EC_KEY_priv2oct(eckey, buf, len); if (len == 0) { OPENSSL_free(buf); diff --git a/openssl/src/crypto/ec/ec_kmeth.c b/openssl/src/crypto/ec/ec_kmeth.c index 054a3333a..8c011635c 100644 --- a/openssl/src/crypto/ec/ec_kmeth.c +++ b/openssl/src/crypto/ec/ec_kmeth.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -83,19 +83,25 @@ EC_KEY *ossl_ec_key_new_method_int(OSSL_LIB_CTX *libctx, const char *propq, { EC_KEY *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - - if (!CRYPTO_NEW_REF(&ret->references, 1)) { - OPENSSL_free(ret); + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NULL; } ret->libctx = libctx; if (propq != NULL) { ret->propq = OPENSSL_strdup(propq); - if (ret->propq == NULL) + if (ret->propq == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } + } + + ret->references = 1; + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; } ret->meth = EC_KEY_get_default_method(); @@ -123,7 +129,6 @@ EC_KEY *ossl_ec_key_new_method_int(OSSL_LIB_CTX *libctx, const char *propq, /* No ex_data inside the FIPS provider */ #ifndef FIPS_MODULE if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_EC_KEY, ret, &ret->ex_data)) { - ERR_raise(ERR_LIB_EC, ERR_R_CRYPTO_LIB); goto err; } #endif diff --git a/openssl/src/crypto/ec/ec_lib.c b/openssl/src/crypto/ec/ec_lib.c index c92b4dcb0..e92f582db 100644 --- a/openssl/src/crypto/ec/ec_lib.c +++ b/openssl/src/crypto/ec/ec_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -19,11 +19,13 @@ #include #include #include -#include +#include #include "crypto/ec.h" #include "internal/nelem.h" #include "ec_local.h" +#define HASH_TO_EC_POINT_TRY_COUNT 1000 + /* functions for EC_GROUP objects */ EC_GROUP *ossl_ec_group_new_ex(OSSL_LIB_CTX *libctx, const char *propq, @@ -41,14 +43,18 @@ EC_GROUP *ossl_ec_group_new_ex(OSSL_LIB_CTX *libctx, const char *propq, } ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NULL; + } ret->libctx = libctx; if (propq != NULL) { ret->propq = OPENSSL_strdup(propq); - if (ret->propq == NULL) + if (ret->propq == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } } ret->meth = meth; if ((ret->meth->flags & EC_FLAGS_CUSTOM_CURVE) == 0) { @@ -82,6 +88,24 @@ EC_GROUP *EC_GROUP_new(const EC_METHOD *meth) # endif #endif +#ifndef FIPS_MODULE +# ifndef OPENSSL_NO_ENGINE +EC_GROUP *EC_GROUP_new_ex(const EC_METHOD *meth, ENGINE *engine) +{ + EC_GROUP *ret = EC_GROUP_new(meth); + if (ret == NULL) + return NULL; + + if (!EC_GROUP_set_engine(ret, engine)) { + EC_GROUP_free(ret); + return NULL; + } + + return ret; +} +# endif +#endif + void EC_pre_comp_free(EC_GROUP *group) { switch (group->pre_comp_type) { @@ -90,6 +114,11 @@ void EC_pre_comp_free(EC_GROUP *group) case PCT_nistz256: #ifdef ECP_NISTZ256_ASM EC_nistz256_pre_comp_free(group->pre_comp.nistz256); +#endif + break; + case PCT_sm2p256: +#if !defined(OPENSSL_NO_SM2) && !defined(OPENSSL_NO_EC_SM2P_64_GCC_128) + EC_sm2p256_pre_comp_free(group->pre_comp.sm2p256); #endif break; #ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 @@ -99,16 +128,12 @@ void EC_pre_comp_free(EC_GROUP *group) case PCT_nistp256: EC_nistp256_pre_comp_free(group->pre_comp.nistp256); break; - case PCT_nistp384: - ossl_ec_nistp384_pre_comp_free(group->pre_comp.nistp384); - break; case PCT_nistp521: EC_nistp521_pre_comp_free(group->pre_comp.nistp521); break; #else case PCT_nistp224: case PCT_nistp256: - case PCT_nistp384: case PCT_nistp521: break; #endif @@ -127,6 +152,12 @@ void EC_GROUP_free(EC_GROUP *group) if (group->meth->group_finish != 0) group->meth->group_finish(group); +#ifndef FIPS_MODULE +# ifndef OPENSSL_NO_ENGINE + ENGINE_finish(group->engine); +# endif +#endif + EC_pre_comp_free(group); BN_MONT_CTX_free(group->mont_data); EC_POINT_free(group->generator); @@ -148,6 +179,12 @@ void EC_GROUP_clear_free(EC_GROUP *group) else if (group->meth->group_finish != 0) group->meth->group_finish(group); +# ifndef FIPS_MODULE +# ifndef OPENSSL_NO_ENGINE + ENGINE_finish(group->engine); +# endif +# endif + EC_pre_comp_free(group); BN_MONT_CTX_free(group->mont_data); EC_POINT_clear_free(group->generator); @@ -183,6 +220,11 @@ int EC_GROUP_copy(EC_GROUP *dest, const EC_GROUP *src) case PCT_nistz256: #ifdef ECP_NISTZ256_ASM dest->pre_comp.nistz256 = EC_nistz256_pre_comp_dup(src->pre_comp.nistz256); +#endif + break; + case PCT_sm2p256: +#if !defined(OPENSSL_NO_SM2) && !defined(OPENSSL_NO_EC_SM2P_64_GCC_128) + dest->pre_comp.sm2p256 = EC_sm2p256_pre_comp_dup(src->pre_comp.sm2p256); #endif break; #ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 @@ -192,16 +234,12 @@ int EC_GROUP_copy(EC_GROUP *dest, const EC_GROUP *src) case PCT_nistp256: dest->pre_comp.nistp256 = EC_nistp256_pre_comp_dup(src->pre_comp.nistp256); break; - case PCT_nistp384: - dest->pre_comp.nistp384 = ossl_ec_nistp384_pre_comp_dup(src->pre_comp.nistp384); - break; case PCT_nistp521: dest->pre_comp.nistp521 = EC_nistp521_pre_comp_dup(src->pre_comp.nistp521); break; #else case PCT_nistp224: case PCT_nistp256: - case PCT_nistp384: case PCT_nistp521: break; #endif @@ -251,8 +289,10 @@ int EC_GROUP_copy(EC_GROUP *dest, const EC_GROUP *src) if (src->seed) { OPENSSL_free(dest->seed); - if ((dest->seed = OPENSSL_malloc(src->seed_len)) == NULL) + if ((dest->seed = OPENSSL_malloc(src->seed_len)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; + } if (!memcpy(dest->seed, src->seed, src->seed_len)) return 0; dest->seed_len = src->seed_len; @@ -262,6 +302,20 @@ int EC_GROUP_copy(EC_GROUP *dest, const EC_GROUP *src) dest->seed_len = 0; } +#ifndef FIPS_MODULE +# ifndef OPENSSL_NO_ENGINE + if (src->engine) { + dest->engine = src->engine; + if (!ENGINE_init(dest->engine)) { + ERR_raise(ERR_LIB_EC, ERR_R_ENGINE_LIB); + return 0; + } + + dest->ecp_meth = src->ecp_meth; + } +# endif +#endif + return dest->meth->group_copy(dest, src); } @@ -535,8 +589,10 @@ size_t EC_GROUP_set_seed(EC_GROUP *group, const unsigned char *p, size_t len) if (!len || !p) return 1; - if ((group->seed = OPENSSL_malloc(len)) == NULL) + if ((group->seed = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(group->seed, p, len); group->seed_len = len; @@ -711,6 +767,128 @@ int EC_GROUP_cmp(const EC_GROUP *a, const EC_GROUP *b, BN_CTX *ctx) return r; } +/* functions for EC_POINTS objects */ + +EC_POINTS *EC_POINTS_new(const EC_GROUP *group, int count) +{ + int i; + EC_POINT *point; + EC_POINTS *ret; + + if (group == NULL) { + ECerr(EC_F_EC_POINTS_NEW, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + if (group->meth->point_init == NULL) { + ECerr(EC_F_EC_POINTS_NEW, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return NULL; + } + + ret = OPENSSL_zalloc(sizeof(*ret) + count * sizeof(EC_POINT *)); + if (ret == NULL) { + ECerr(EC_F_EC_POINTS_NEW, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ret->count = count; + ret->items = (EC_POINT **)(ret + 1); + + for (i = 0; i < count; i++) { + point = EC_POINT_new(group); + if (point == NULL) + EC_POINTS_free(ret); + + ret->items[i] = point; + } + + return ret; +} + +void EC_POINTS_free(EC_POINTS *points) +{ + int i; + + if (points == NULL) + return; + + for (i = 0; i < points->count; i++) + EC_POINT_free(points->items[i]); + + OPENSSL_free(points); +} + +void EC_POINTS_clear_free(EC_POINTS *points) +{ + int i; + + if (points == NULL) + return; + + for (i = 0; i < points->count; i++) + EC_POINT_clear_free(points->items[i]); + + OPENSSL_clear_free(points, sizeof(*points) + points->count * sizeof(EC_POINT *)); +} + +int EC_POINTS_copy(EC_POINTS *dest, const EC_POINTS *src) +{ + int i; + + if (dest == src) + return 1; + + if (dest->count != src->count) { + ECerr(EC_F_EC_POINTS_COPY, EC_R_INCOMPATIBLE_OBJECTS); + return 0; + } + + for (i = 0; i < src->count; i++) { + if (!EC_POINT_copy(dest->items[i], src->items[i])) + return 0; + } + + return 1; +} + +EC_POINTS *EC_POINTS_dup(const EC_POINTS *a, const EC_GROUP *group) +{ + EC_POINTS *t; + int r; + + if (a == NULL) + return NULL; + + t = EC_POINTS_new(group, a->count); + if (t == NULL) + return NULL; + r = EC_POINTS_copy(t, a); + if (!r) { + EC_POINTS_free(t); + return NULL; + } + return t; +} + +EC_POINT *EC_POINTS_get_item(EC_POINTS *p, int i) +{ + return p != NULL && i >= 0 && i < p->count ? p->items[i] : NULL; +} + +int EC_POINTS_set_item(EC_POINTS *p, int i, EC_POINT *point) +{ + if (p == NULL || i > p->count) + return 0; + + p->items[i] = point; + + return 1; +} + +int EC_POINTS_count(EC_POINTS *p) +{ + return p->count; +} + /* functions for EC_POINT objects */ EC_POINT *EC_POINT_new(const EC_GROUP *group) @@ -727,8 +905,10 @@ EC_POINT *EC_POINT_new(const EC_GROUP *group) } ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NULL; + } ret->meth = group->meth; ret->curve_name = group->curve_name; @@ -800,6 +980,60 @@ EC_POINT *EC_POINT_dup(const EC_POINT *a, const EC_GROUP *group) return t; } +#ifndef FIPS_MODULE +/* + * Functions for convert string to ec_point on the elliptic curve. + * This implementation belongs to the ad-hoc method, but it is also the + * recommended implementation in the mcl library, the google open source project + * and the cryptography conference paper. + * \param group underlying EC_GROUP object + * \param r EC_POINT object for the result + * \param str string pointer + * \param len length of the string + * \return 1 on success and 0 if an error occurred + */ +int EC_POINT_from_string(const EC_GROUP *group, EC_POINT *r, + const unsigned char *str, size_t len) +{ + int ret = 0, i = 0; + unsigned char hash_res[SHA256_DIGEST_LENGTH]; + unsigned char *p = (unsigned char *)str; + BN_CTX *bn_ctx = NULL; + BIGNUM *x; + + memset(hash_res, 0, sizeof(hash_res)); + + if ((bn_ctx = BN_CTX_new_ex(group->libctx)) == NULL) + goto end; + + BN_CTX_start(bn_ctx); + if ((x = BN_CTX_get(bn_ctx)) == NULL) + goto end; + + do { + if (!SHA256(p, len, hash_res)) + goto end; + + BN_bin2bn(hash_res, SHA256_DIGEST_LENGTH, x); + + p = &hash_res[0]; + len = sizeof(hash_res); + + if(EC_POINT_set_compressed_coordinates(group, r, x, 0, bn_ctx) == 1) { + ret = 1; + break; + } + + ERR_clear_error(); + } while (i++ < HASH_TO_EC_POINT_TRY_COUNT); + +end: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} +#endif + #ifndef OPENSSL_NO_DEPRECATED_3_0 const EC_METHOD *EC_POINT_method_of(const EC_POINT *point) { @@ -807,6 +1041,11 @@ const EC_METHOD *EC_POINT_method_of(const EC_POINT *point) } #endif +int EC_POINT_get_curve_name(const EC_POINT *point) +{ + return point->curve_name; +} + int EC_POINT_set_to_infinity(const EC_GROUP *group, EC_POINT *point) { if (group->meth->point_set_to_infinity == 0) { @@ -945,6 +1184,10 @@ int EC_POINT_add(const EC_GROUP *group, EC_POINT *r, const EC_POINT *a, ERR_raise(ERR_LIB_EC, EC_R_INCOMPATIBLE_OBJECTS); return 0; } +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL && group->ecp_meth->add != NULL) + return group->ecp_meth->add(group, r, a, b, ctx); +#endif return group->meth->add(group, r, a, b, ctx); } @@ -959,6 +1202,10 @@ int EC_POINT_dbl(const EC_GROUP *group, EC_POINT *r, const EC_POINT *a, ERR_raise(ERR_LIB_EC, EC_R_INCOMPATIBLE_OBJECTS); return 0; } +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL && group->ecp_meth->dbl != NULL) + return group->ecp_meth->dbl(group, r, a, ctx); +#endif return group->meth->dbl(group, r, a, ctx); } @@ -972,6 +1219,10 @@ int EC_POINT_invert(const EC_GROUP *group, EC_POINT *a, BN_CTX *ctx) ERR_raise(ERR_LIB_EC, EC_R_INCOMPATIBLE_OBJECTS); return 0; } +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL && group->ecp_meth->invert != NULL) + return group->ecp_meth->invert(group, a, ctx); +#endif return group->meth->invert(group, a, ctx); } @@ -1097,6 +1348,12 @@ int EC_POINTs_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, return 0; } +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL && group->ecp_meth->mul != NULL) + ret = group->ecp_meth->mul(group, r, scalar, num, points, scalars, ctx); + else +#endif + if (group->meth->mul != NULL) ret = group->meth->mul(group, r, scalar, num, points, scalars, ctx); else @@ -1138,6 +1395,13 @@ int EC_POINT_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *g_scalar, } num = (point != NULL && p_scalar != NULL) ? 1 : 0; + +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL && group->ecp_meth->mul != NULL) + ret = group->ecp_meth->mul(group, r, g_scalar, num, &point, &p_scalar, ctx); + else +#endif + if (group->meth->mul != NULL) ret = group->meth->mul(group, r, g_scalar, num, &point, &p_scalar, ctx); else @@ -1150,6 +1414,235 @@ int EC_POINT_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *g_scalar, return ret; } +/* + * Functions for point multiplication. + * r[i] = points[i] * scalars[i] + */ +int EC_POINTs_scalars_mul(const EC_GROUP *group, EC_POINTS **r, size_t num, + const EC_POINT *points[], const BIGNUM *scalars[], + BN_CTX *ctx) +{ + int ret = 0; + size_t i = 0; + BN_CTX *new_ctx = NULL; + EC_POINTS *result = NULL; + + if (r == NULL || points == NULL || scalars == NULL || num <= 0) + return 0; + + if (*r == NULL) { + result = EC_POINTS_new(group, (uint32_t)num); + if (result == NULL) + return 0; + } else { + result = *r; + } + + for (i = 0; i < num; i++) { + if (!ec_point_is_compat(points[i], group)) { + ERR_raise(ERR_LIB_EC, EC_R_INCOMPATIBLE_OBJECTS); + goto err; + } + } + + if (ctx == NULL && (ctx = new_ctx = BN_CTX_secure_new_ex(NULL)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_INTERNAL_ERROR); + goto err; + } + +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL && group->ecp_meth->scalars_mul != NULL) { + if (!group->ecp_meth->scalars_mul(group, result->items, num, points, + scalars, ctx)) + goto err; + } else +#endif + + for (i = 0; i < num; i++) { + if (!EC_POINT_mul(group, result->items[i], NULL, points[i], scalars[i], + ctx)) + goto err; + } + + *r = result; + result = NULL; + ret = 1; +err: + BN_CTX_free(new_ctx); + EC_POINTS_free(result); + return ret; +} + +/* + * Functions for point multiplication. + * r[i] = points[i] * scalar + */ +int EC_POINTs_scalar_mul(const EC_GROUP *group, EC_POINTS **r, size_t num, + const EC_POINT *points[], const BIGNUM *scalar, + BN_CTX *ctx) +{ + int ret = 0; + size_t i = 0; + BN_CTX *new_ctx = NULL; + EC_POINTS *result = NULL; + + if (r == NULL || points == NULL || scalar == NULL || num <= 0) + return 0; + + if (*r == NULL) { + result = EC_POINTS_new(group, (uint32_t)num); + if (result == NULL) + return 0; + } else { + result = *r; + } + + for (i = 0; i < num; i++) { + if (!ec_point_is_compat(points[i], group)) { + ERR_raise(ERR_LIB_EC, EC_R_INCOMPATIBLE_OBJECTS); + goto err; + } + } + + if (ctx == NULL && (ctx = new_ctx = BN_CTX_secure_new_ex(NULL)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_INTERNAL_ERROR); + goto err; + } + +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL && group->ecp_meth->scalar_mul != NULL) { + if (!group->ecp_meth->scalar_mul(group, result->items, num, points, + scalar, ctx)) + goto err; + } else +#endif + + for (i = 0; i < num; i++) { + if (!EC_POINT_mul(group, result->items[i], NULL, points[i], scalar, ctx)) + goto err; + } + + *r = result; + result = NULL; + ret = 1; +err: + BN_CTX_free(new_ctx); + EC_POINTS_free(result); + return ret; +} + +/* + * Functions for convert some strings to some points on the elliptic curve. + * r[i]->X = hash(strings[i]) + * r[i]->Y = F(hash(strings[i])), the Y coordinate can be calculated by taking + * the X coordinate into the equation + * r[i]->Z = 1 + */ +int EC_POINTs_from_strings(const EC_GROUP *group, EC_POINTS **r, + size_t num, const unsigned char *strings[], + BN_CTX *ctx) +{ + int ret = 0; + BN_CTX *new_ctx = NULL; + EC_POINTS *result = NULL; + + if (r == NULL || strings == NULL || num <= 0) + return 0; + + if (*r == NULL) { + result = EC_POINTS_new(group, (uint32_t)num); + if (result == NULL) + return 0; + } else { + result = *r; + } + + if (ctx == NULL && (ctx = new_ctx = BN_CTX_secure_new_ex(NULL)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_INTERNAL_ERROR); + goto err; + } + +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL && group->ecp_meth->strings_to_points != NULL) { + if (!group->ecp_meth->strings_to_points(group, result->items, num, + strings, ctx)) { + goto err; + } else { + *r = result; + ret = 1; + } + } +#endif + + result = NULL; + + /* + * TODO + */ +err: + BN_CTX_free(new_ctx); + EC_POINTS_free(result); + return ret; +} + +/* + * Functions for convert some strings to some points on the elliptic curve, then + * multiply with scalar. + * point[i]->X = hash(strings[i]) + * point[i]->Y = F(hash(strings[i])), the Y coordinate can be calculated by taking + * the X coordinate into the equation + * point[i]->Z = 1 + * r[i] = scalar * point[i] + */ +int EC_POINTs_from_strings_scalar_mul(const EC_GROUP *group, EC_POINTS **r, + size_t num, const unsigned char *strings[], + const BIGNUM *scalar, BN_CTX *ctx) +{ + int ret = 0; + BN_CTX *new_ctx = NULL; + EC_POINTS *result = NULL; + + if (r == NULL || strings == NULL || num <= 0) + return 0; + + if (*r == NULL) { + result = EC_POINTS_new(group, (uint32_t)num); + if (result == NULL) + return 0; + } else { + result = *r; + } + + if (ctx == NULL && (ctx = new_ctx = BN_CTX_secure_new_ex(NULL)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_INTERNAL_ERROR); + goto err; + } + +#ifndef OPENSSL_NO_ENGINE + if (group->ecp_meth != NULL + && group->ecp_meth->strings_to_points_scalar_mul != NULL) { + if (!group->ecp_meth->strings_to_points_scalar_mul(group, result->items, + num, strings, scalar, + ctx)) { + goto err; + } else { + *r = result; + ret = 1; + } + } +#endif + + result = NULL; + + /* + * TODO + */ +err: + BN_CTX_free(new_ctx); + EC_POINTS_free(result); + return ret; +} + #ifndef OPENSSL_NO_DEPRECATED_3_0 int EC_GROUP_precompute_mult(EC_GROUP *group, BN_CTX *ctx) { @@ -1341,6 +1834,36 @@ int EC_GROUP_get_basis_type(const EC_GROUP *group) return 0; } +#ifndef FIPS_MODULE +# ifndef OPENSSL_NO_ENGINE +int EC_GROUP_set_engine(EC_GROUP *group, ENGINE *engine) +{ + const EC_POINT_METHOD *ecp_meth; + + if (!ENGINE_init(engine)) { + ERR_raise(ERR_LIB_EC, ERR_R_ENGINE_LIB); + return 0; + } + + ecp_meth = ENGINE_get_ecp_meth(engine, group->curve_name); + if (ecp_meth == NULL) { + ERR_raise(ERR_LIB_EC, EC_R_EC_POINT_METHOD_NOT_FOUND); + return 0; + } + + group->ecp_meth = ecp_meth; + group->engine = engine; + + return 1; +} + +const ENGINE *EC_GROUP_get0_engine(const EC_GROUP *group) +{ + return group->engine; +} +# endif +#endif + #ifndef OPENSSL_NO_EC2M int EC_GROUP_get_trinomial_basis(const EC_GROUP *group, unsigned int *k) { @@ -1506,7 +2029,7 @@ int ossl_ec_group_set_params(EC_GROUP *group, const OSSL_PARAM params[]) p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT); if (p != NULL) { if (!ossl_ec_pt_format_param2id(p, &format)) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_FORM); + ECerr(0, EC_R_INVALID_FORM); return 0; } EC_GROUP_set_point_conversion_form(group, format); @@ -1515,7 +2038,7 @@ int ossl_ec_group_set_params(EC_GROUP *group, const OSSL_PARAM params[]) p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_EC_ENCODING); if (p != NULL) { if (!ossl_ec_encoding_param2id(p, &encoding_flag)) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_FORM); + ECerr(0, EC_R_INVALID_FORM); return 0; } EC_GROUP_set_asn1_flag(group, encoding_flag); @@ -1526,7 +2049,7 @@ int ossl_ec_group_set_params(EC_GROUP *group, const OSSL_PARAM params[]) /* The seed is allowed to be NULL */ if (p->data_type != OSSL_PARAM_OCTET_STRING || !EC_GROUP_set_seed(group, p->data, p->data_size)) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_SEED); + ECerr(0, EC_R_INVALID_SEED); return 0; } } @@ -1555,23 +2078,13 @@ EC_GROUP *EC_GROUP_new_from_params(const OSSL_PARAM params[], /* This is the simple named group case */ ptmp = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_GROUP_NAME); if (ptmp != NULL) { - int decoded = 0; - - if ((group = group_new_from_name(ptmp, libctx, propq)) == NULL) - return NULL; - if (!ossl_ec_group_set_params(group, params)) { - EC_GROUP_free(group); - return NULL; - } - - ptmp = OSSL_PARAM_locate_const(params, - OSSL_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS); - if (ptmp != NULL && !OSSL_PARAM_get_int(ptmp, &decoded)) { - ERR_raise(ERR_LIB_EC, EC_R_WRONG_CURVE_PARAMETERS); - EC_GROUP_free(group); - return NULL; + group = group_new_from_name(ptmp, libctx, propq); + if (group != NULL) { + if (!ossl_ec_group_set_params(group, params)) { + EC_GROUP_free(group); + group = NULL; + } } - group->decoded_from_explicit_params = decoded > 0; return group; } #ifdef FIPS_MODULE @@ -1581,7 +2094,7 @@ EC_GROUP *EC_GROUP_new_from_params(const OSSL_PARAM params[], /* If it gets here then we are trying explicit parameters */ bnctx = BN_CTX_new_ex(libctx); if (bnctx == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } BN_CTX_start(bnctx); @@ -1591,7 +2104,7 @@ EC_GROUP *EC_GROUP_new_from_params(const OSSL_PARAM params[], b = BN_CTX_get(bnctx); order = BN_CTX_get(bnctx); if (order == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -1742,8 +2255,6 @@ EC_GROUP *EC_GROUP_new_from_params(const OSSL_PARAM params[], EC_GROUP_free(group); group = named_group; } - /* We've imported the group from explicit parameters, set it so. */ - group->decoded_from_explicit_params = 1; ok = 1; err: if (!ok) { @@ -1757,38 +2268,3 @@ EC_GROUP *EC_GROUP_new_from_params(const OSSL_PARAM params[], return group; #endif /* FIPS_MODULE */ } - -OSSL_PARAM *EC_GROUP_to_params(const EC_GROUP *group, OSSL_LIB_CTX *libctx, - const char *propq, BN_CTX *bnctx) -{ - OSSL_PARAM_BLD *tmpl = NULL; - BN_CTX *new_bnctx = NULL; - unsigned char *gen_buf = NULL; - OSSL_PARAM *params = NULL; - - if (group == NULL) - goto err; - - tmpl = OSSL_PARAM_BLD_new(); - if (tmpl == NULL) - goto err; - - if (bnctx == NULL) - bnctx = new_bnctx = BN_CTX_new_ex(libctx); - if (bnctx == NULL) - goto err; - BN_CTX_start(bnctx); - - if (!ossl_ec_group_todata( - group, tmpl, NULL, libctx, propq, bnctx, &gen_buf)) - goto err; - - params = OSSL_PARAM_BLD_to_param(tmpl); - - err: - OSSL_PARAM_BLD_free(tmpl); - OPENSSL_free(gen_buf); - BN_CTX_end(bnctx); - BN_CTX_free(new_bnctx); - return params; -} diff --git a/openssl/src/crypto/ec/ec_local.h b/openssl/src/crypto/ec/ec_local.h index 2814d8739..a104815ae 100644 --- a/openssl/src/crypto/ec/ec_local.h +++ b/openssl/src/crypto/ec/ec_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -13,6 +13,7 @@ #include #include #include +#include #include "internal/refcount.h" #include "crypto/ec.h" @@ -198,14 +199,46 @@ struct ec_method_st { EC_POINT *p, BN_CTX *ctx); }; +struct ec_point_method_st { + int curve_id; + + /* used by EC_POINT_add */ + int (*add)(const EC_GROUP *, EC_POINT *r, const EC_POINT *a, + const EC_POINT *b, BN_CTX *); + /* used by EC_POINT_dbl */ + int (*dbl)(const EC_GROUP *, EC_POINT *r, const EC_POINT *a, BN_CTX *); + /* used by ECP_POINT_invert */ + int (*invert)(const EC_GROUP *, EC_POINT *, BN_CTX *); + /* used by ECP_POINT_mul */ + int (*mul)(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, + size_t num, const EC_POINT *points[], const BIGNUM *scalars[], + BN_CTX *); + /* used by ECP_POINTs_scalars_mul */ + int (*scalars_mul)(const EC_GROUP *group, EC_POINT *r[], size_t num, + const EC_POINT *points[], const BIGNUM *scalars[], + BN_CTX *ctx); + /* used by ECP_POINTs_scalar_mul */ + int (*scalar_mul)(const EC_GROUP *group, EC_POINT *r[], size_t num, + const EC_POINT *points[], const BIGNUM *scalar, + BN_CTX *ctx); + /* used by ECP_POINTs_string2_to_points */ + int (*strings_to_points)(const EC_GROUP *group, EC_POINT *r[], + size_t num, const unsigned char *strings[], + BN_CTX *ctx); + /* used by ECP_POINTs_string2_to_points_scalar_mul */ + int (*strings_to_points_scalar_mul)(const EC_GROUP *group, EC_POINT *r[], + size_t num, const unsigned char *strings[], + const BIGNUM *scalar, BN_CTX *ctx); +}; + /* * Types and functions to manipulate pre-computed values. */ typedef struct nistp224_pre_comp_st NISTP224_PRE_COMP; typedef struct nistp256_pre_comp_st NISTP256_PRE_COMP; -typedef struct nistp384_pre_comp_st NISTP384_PRE_COMP; typedef struct nistp521_pre_comp_st NISTP521_PRE_COMP; typedef struct nistz256_pre_comp_st NISTZ256_PRE_COMP; +typedef struct sm2p256_pre_comp_st SM2P256_PRE_COMP; typedef struct ec_pre_comp_st EC_PRE_COMP; struct ec_group_st { @@ -246,7 +279,7 @@ struct ec_group_st { * equation of the form y^2 + x*y = x^3 + a*x^2 + b. */ BIGNUM *a, *b; - /* enable optimized point arithmetic for special case */ + /* enable optimized point arithmetics for special case */ int a_is_minus3; /* method-specific (e.g., Montgomery structure) */ void *field_data1; @@ -265,20 +298,24 @@ struct ec_group_st { */ enum { PCT_none, - PCT_nistp224, PCT_nistp256, PCT_nistp384, PCT_nistp521, PCT_nistz256, + PCT_nistp224, PCT_nistp256, PCT_nistp521, PCT_nistz256, PCT_sm2p256, PCT_ec } pre_comp_type; union { NISTP224_PRE_COMP *nistp224; NISTP256_PRE_COMP *nistp256; - NISTP384_PRE_COMP *nistp384; NISTP521_PRE_COMP *nistp521; NISTZ256_PRE_COMP *nistz256; + SM2P256_PRE_COMP *sm2p256; EC_PRE_COMP *ec; } pre_comp; OSSL_LIB_CTX *libctx; char *propq; +#ifndef OPENSSL_NO_ENGINE + ENGINE *engine; + const EC_POINT_METHOD *ecp_meth; +#endif }; #define SETPRECOMP(g, type, pre) \ @@ -300,6 +337,7 @@ struct ec_key_st { #ifndef FIPS_MODULE CRYPTO_EX_DATA ex_data; #endif + CRYPTO_RWLOCK *lock; OSSL_LIB_CTX *libctx; char *propq; @@ -319,10 +357,15 @@ struct ec_point_st { BIGNUM *Y; BIGNUM *Z; /* Jacobian projective coordinates: * (X, Y, * Z) represents (X/Z^2, Y/Z^3) if Z != 0 */ - int Z_is_one; /* enable optimized point arithmetic for + int Z_is_one; /* enable optimized point arithmetics for * special case */ }; +struct ec_points_st { + int count; + EC_POINT **items; +}; + static ossl_inline int ec_point_is_compat(const EC_POINT *point, const EC_GROUP *group) { @@ -334,18 +377,18 @@ static ossl_inline int ec_point_is_compat(const EC_POINT *point, NISTP224_PRE_COMP *EC_nistp224_pre_comp_dup(NISTP224_PRE_COMP *); NISTP256_PRE_COMP *EC_nistp256_pre_comp_dup(NISTP256_PRE_COMP *); -NISTP384_PRE_COMP *ossl_ec_nistp384_pre_comp_dup(NISTP384_PRE_COMP *); NISTP521_PRE_COMP *EC_nistp521_pre_comp_dup(NISTP521_PRE_COMP *); NISTZ256_PRE_COMP *EC_nistz256_pre_comp_dup(NISTZ256_PRE_COMP *); NISTP256_PRE_COMP *EC_nistp256_pre_comp_dup(NISTP256_PRE_COMP *); +SM2P256_PRE_COMP *EC_sm2p256_pre_comp_dup(SM2P256_PRE_COMP *); EC_PRE_COMP *EC_ec_pre_comp_dup(EC_PRE_COMP *); void EC_pre_comp_free(EC_GROUP *group); void EC_nistp224_pre_comp_free(NISTP224_PRE_COMP *); void EC_nistp256_pre_comp_free(NISTP256_PRE_COMP *); -void ossl_ec_nistp384_pre_comp_free(NISTP384_PRE_COMP *); void EC_nistp521_pre_comp_free(NISTP521_PRE_COMP *); void EC_nistz256_pre_comp_free(NISTZ256_PRE_COMP *); +void EC_sm2p256_pre_comp_free(SM2P256_PRE_COMP *); void EC_ec_pre_comp_free(EC_PRE_COMP *); /* @@ -510,11 +553,12 @@ int ossl_ec_GF2m_simple_field_sqr(const EC_GROUP *, BIGNUM *r, const BIGNUM *a, int ossl_ec_GF2m_simple_field_div(const EC_GROUP *, BIGNUM *r, const BIGNUM *a, const BIGNUM *b, BN_CTX *); -#ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 +#if !defined(OPENSSL_NO_EC_NISTP_64_GCC_128) || !defined(OPENSSL_NO_EC_SM2P_64_GCC_128) # ifdef B_ENDIAN -# error "Can not enable ec_nistp_64_gcc_128 on big-endian systems" +# error "Can not enable ec_nistp_64_gcc_128 and ec_sm2p_64_gcc_128 on big-endian systems" # endif +# ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 /* method functions in ecp_nistp224.c */ int ossl_ec_GFp_nistp224_group_init(EC_GROUP *group); int ossl_ec_GFp_nistp224_group_set_curve(EC_GROUP *group, const BIGNUM *p, @@ -555,46 +599,48 @@ int ossl_ec_GFp_nistp256_points_mul(const EC_GROUP *group, EC_POINT *r, int ossl_ec_GFp_nistp256_precompute_mult(EC_GROUP *group, BN_CTX *ctx); int ossl_ec_GFp_nistp256_have_precompute_mult(const EC_GROUP *group); -/* method functions in ecp_nistp384.c */ -int ossl_ec_GFp_nistp384_group_init(EC_GROUP *group); -int ossl_ec_GFp_nistp384_group_set_curve(EC_GROUP *group, const BIGNUM *p, +/* method functions in ecp_nistp521.c */ +int ossl_ec_GFp_nistp521_group_init(EC_GROUP *group); +int ossl_ec_GFp_nistp521_group_set_curve(EC_GROUP *group, const BIGNUM *p, const BIGNUM *a, const BIGNUM *n, BN_CTX *); -int ossl_ec_GFp_nistp384_point_get_affine_coordinates(const EC_GROUP *group, +int ossl_ec_GFp_nistp521_point_get_affine_coordinates(const EC_GROUP *group, const EC_POINT *point, BIGNUM *x, BIGNUM *y, BN_CTX *ctx); -int ossl_ec_GFp_nistp384_mul(const EC_GROUP *group, EC_POINT *r, +int ossl_ec_GFp_nistp521_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, size_t num, const EC_POINT *points[], const BIGNUM *scalars[], BN_CTX *); -int ossl_ec_GFp_nistp384_points_mul(const EC_GROUP *group, EC_POINT *r, +int ossl_ec_GFp_nistp521_points_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, size_t num, const EC_POINT *points[], const BIGNUM *scalars[], BN_CTX *ctx); -int ossl_ec_GFp_nistp384_precompute_mult(EC_GROUP *group, BN_CTX *ctx); -int ossl_ec_GFp_nistp384_have_precompute_mult(const EC_GROUP *group); -const EC_METHOD *ossl_ec_GFp_nistp384_method(void); +int ossl_ec_GFp_nistp521_precompute_mult(EC_GROUP *group, BN_CTX *ctx); +int ossl_ec_GFp_nistp521_have_precompute_mult(const EC_GROUP *group); +# endif -/* method functions in ecp_nistp521.c */ -int ossl_ec_GFp_nistp521_group_init(EC_GROUP *group); -int ossl_ec_GFp_nistp521_group_set_curve(EC_GROUP *group, const BIGNUM *p, +# if !defined(OPENSSL_NO_EC_SM2P_64_GCC_128) && !defined(OPENSSL_NO_SM2) +/* method functions in ecp_sm2p256.c */ +int ossl_ec_GFp_sm2p256_group_init(EC_GROUP *group); +int ossl_ec_GFp_sm2p256_group_set_curve(EC_GROUP *group, const BIGNUM *p, const BIGNUM *a, const BIGNUM *n, BN_CTX *); -int ossl_ec_GFp_nistp521_point_get_affine_coordinates(const EC_GROUP *group, +int ossl_ec_GFp_sm2p256_point_get_affine_coordinates(const EC_GROUP *group, const EC_POINT *point, BIGNUM *x, BIGNUM *y, BN_CTX *ctx); -int ossl_ec_GFp_nistp521_mul(const EC_GROUP *group, EC_POINT *r, +int ossl_ec_GFp_sm2p256_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, size_t num, const EC_POINT *points[], const BIGNUM *scalars[], BN_CTX *); -int ossl_ec_GFp_nistp521_points_mul(const EC_GROUP *group, EC_POINT *r, +int ossl_ec_GFp_sm2p256_points_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, size_t num, const EC_POINT *points[], const BIGNUM *scalars[], BN_CTX *ctx); -int ossl_ec_GFp_nistp521_precompute_mult(EC_GROUP *group, BN_CTX *ctx); -int ossl_ec_GFp_nistp521_have_precompute_mult(const EC_GROUP *group); +int ossl_ec_GFp_sm2p256_precompute_mult(EC_GROUP *group, BN_CTX *ctx); +int ossl_ec_GFp_sm2p256_have_precompute_mult(const EC_GROUP *group); +# endif /* utility functions in ecp_nistputil.c */ void ossl_ec_GFp_nistp_points_make_affine_internal(size_t num, void *point_array, @@ -653,11 +699,6 @@ int ossl_ec_key_simple_generate_key(EC_KEY *eckey); int ossl_ec_key_simple_generate_public_key(EC_KEY *eckey); int ossl_ec_key_simple_check_key(const EC_KEY *eckey); -#ifdef ECP_SM2P256_ASM -/* Returns optimized methods for SM2 */ -const EC_METHOD *EC_GFp_sm2p256_method(void); -#endif - int ossl_ec_curve_nid_from_params(const EC_GROUP *group, BN_CTX *ctx); /* EC_METHOD definitions */ diff --git a/openssl/src/crypto/ec/ec_mult.c b/openssl/src/crypto/ec/ec_mult.c index 9eb007cdf..c6ec2964b 100644 --- a/openssl/src/crypto/ec/ec_mult.c +++ b/openssl/src/crypto/ec/ec_mult.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -45,6 +45,7 @@ struct ec_pre_comp_st { * objects followed by a NULL */ size_t num; /* numblocks * 2^(w-1) */ CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; }; static EC_PRE_COMP *ec_pre_comp_new(const EC_GROUP *group) @@ -55,14 +56,19 @@ static EC_PRE_COMP *ec_pre_comp_new(const EC_GROUP *group) return NULL; ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return ret; + } ret->group = group; ret->blocksize = 8; /* default */ ret->w = 4; /* default */ + ret->references = 1; - if (!CRYPTO_NEW_REF(&ret->references, 1)) { + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -73,7 +79,7 @@ EC_PRE_COMP *EC_ec_pre_comp_dup(EC_PRE_COMP *pre) { int i; if (pre != NULL) - CRYPTO_UP_REF(&pre->references, &i); + CRYPTO_UP_REF(&pre->references, &i, pre->lock); return pre; } @@ -84,7 +90,7 @@ void EC_ec_pre_comp_free(EC_PRE_COMP *pre) if (pre == NULL) return; - CRYPTO_DOWN_REF(&pre->references, &i); + CRYPTO_DOWN_REF(&pre->references, &i, pre->lock); REF_PRINT_COUNT("EC_ec", pre); if (i > 0) return; @@ -97,7 +103,7 @@ void EC_ec_pre_comp_free(EC_PRE_COMP *pre) EC_POINT_free(*pts); OPENSSL_free(pre->points); } - CRYPTO_FREE_REF(&pre->references); + CRYPTO_THREAD_lock_free(pre->lock); OPENSSL_free(pre); } @@ -165,7 +171,7 @@ int ossl_ec_scalar_mul_ladder(const EC_GROUP *group, EC_POINT *r, if (((p = EC_POINT_new(group)) == NULL) || ((s = EC_POINT_new(group)) == NULL)) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -189,7 +195,7 @@ int ossl_ec_scalar_mul_ladder(const EC_GROUP *group, EC_POINT *r, lambda = BN_CTX_get(ctx); k = BN_CTX_get(ctx); if (k == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -514,8 +520,10 @@ int ossl_ec_wNAF_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, if (wNAF != NULL) wNAF[0] = NULL; /* preliminary pivot */ - if (wsize == NULL || wNAF_len == NULL || wNAF == NULL || val_sub == NULL) + if (wsize == NULL || wNAF_len == NULL || wNAF == NULL || val_sub == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } /* * num_val will be the total number of temporarily precomputed points @@ -625,6 +633,7 @@ int ossl_ec_wNAF_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, wNAF[i + 1] = NULL; wNAF[i] = OPENSSL_malloc(wNAF_len[i]); if (wNAF[i] == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); OPENSSL_free(tmp_wNAF); goto err; } @@ -652,8 +661,10 @@ int ossl_ec_wNAF_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, * subarray of 'pre_comp->points' if we already have precomputation. */ val = OPENSSL_malloc((num_val + 1) * sizeof(val[0])); - if (val == NULL) + if (val == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } val[num_val] = NULL; /* pivot element */ /* allocate points for precomputation */ @@ -882,21 +893,23 @@ int ossl_ec_wNAF_precompute_mult(EC_GROUP *group, BN_CTX *ctx) * and store */ points = OPENSSL_malloc(sizeof(*points) * (num + 1)); - if (points == NULL) + if (points == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } var = points; var[num] = NULL; /* pivot */ for (i = 0; i < num; i++) { if ((var[i] = EC_POINT_new(group)) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } if ((tmp_point = EC_POINT_new(group)) == NULL || (base = EC_POINT_new(group)) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } diff --git a/openssl/src/crypto/ec/ec_oct.c b/openssl/src/crypto/ec/ec_oct.c index 0ad3394c8..790a0b290 100644 --- a/openssl/src/crypto/ec/ec_oct.c +++ b/openssl/src/crypto/ec/ec_oct.c @@ -140,8 +140,10 @@ size_t EC_POINT_point2buf(const EC_GROUP *group, const EC_POINT *point, len = EC_POINT_point2oct(group, point, form, NULL, 0, NULL); if (len == 0) return 0; - if ((buf = OPENSSL_malloc(len)) == NULL) + if ((buf = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; + } len = EC_POINT_point2oct(group, point, form, buf, len, ctx); if (len == 0) { OPENSSL_free(buf); diff --git a/openssl/src/crypto/ec/ec_pmeth.c b/openssl/src/crypto/ec/ec_pmeth.c index 716b1860b..19e2f0d0c 100644 --- a/openssl/src/crypto/ec/ec_pmeth.c +++ b/openssl/src/crypto/ec/ec_pmeth.c @@ -48,8 +48,10 @@ static int pkey_ec_init(EVP_PKEY_CTX *ctx) { EC_PKEY_CTX *dctx; - if ((dctx = OPENSSL_zalloc(sizeof(*dctx))) == NULL) + if ((dctx = OPENSSL_zalloc(sizeof(*dctx))) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; + } dctx->cofactor_mode = -1; dctx->kdf_type = EVP_PKEY_ECDH_KDF_NONE; @@ -227,8 +229,10 @@ static int pkey_ec_kdf_derive(EVP_PKEY_CTX *ctx, return 0; if (!pkey_ec_derive(ctx, NULL, &ktmplen)) return 0; - if ((ktmp = OPENSSL_malloc(ktmplen)) == NULL) + if ((ktmp = OPENSSL_malloc(ktmplen)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; + } if (!pkey_ec_derive(ctx, ktmp, &ktmplen)) goto err; /* Do KDF stuff */ diff --git a/openssl/src/crypto/ec/ecdh_ossl.c b/openssl/src/crypto/ec/ecdh_ossl.c index 41f7e3904..8016c6d7a 100644 --- a/openssl/src/crypto/ec/ecdh_ossl.c +++ b/openssl/src/crypto/ec/ecdh_ossl.c @@ -63,7 +63,7 @@ int ossl_ecdh_simple_compute_key(unsigned char **pout, size_t *poutlen, BN_CTX_start(ctx); x = BN_CTX_get(ctx); if (x == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -80,19 +80,16 @@ int ossl_ecdh_simple_compute_key(unsigned char **pout, size_t *poutlen, * * peer_public_key. */ if (EC_KEY_get_flags(ecdh) & EC_FLAG_COFACTOR_ECDH) { - if (!EC_GROUP_get_cofactor(group, x, NULL)) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); - goto err; - } - if (!BN_mul(x, x, priv_key, ctx)) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + if (!EC_GROUP_get_cofactor(group, x, NULL) || + !BN_mul(x, x, priv_key, ctx)) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } priv_key = x; } if ((tmp = EC_POINT_new(group)) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -121,8 +118,10 @@ int ossl_ecdh_simple_compute_key(unsigned char **pout, size_t *poutlen, ERR_raise(ERR_LIB_EC, ERR_R_INTERNAL_ERROR); goto err; } - if ((buf = OPENSSL_malloc(buflen)) == NULL) + if ((buf = OPENSSL_malloc(buflen)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } memset(buf, 0, buflen - len); if (len != (size_t)BN_bn2bin(x, buf + buflen - len)) { diff --git a/openssl/src/crypto/ec/ecdsa_ossl.c b/openssl/src/crypto/ec/ecdsa_ossl.c index 8b4d25d59..fe9b3cf59 100644 --- a/openssl/src/crypto/ec/ecdsa_ossl.c +++ b/openssl/src/crypto/ec/ecdsa_ossl.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,22 +19,6 @@ #include #include "crypto/bn.h" #include "ec_local.h" -#include "internal/deterministic_nonce.h" - -#define MIN_ECDSA_SIGN_ORDERBITS 64 -/* - * It is highly unlikely that a retry will happen, - * Multiple retries would indicate that something is wrong - * with the group parameters (which would normally only happen - * with a bad custom group). - */ -#define MAX_ECDSA_SIGN_RETRIES 8 - -static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, - BIGNUM **kinvp, BIGNUM **rp, - const unsigned char *dgst, int dlen, - unsigned int nonce_type, const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq); int ossl_ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp) @@ -77,59 +61,19 @@ int ossl_ecdsa_sign(int type, const unsigned char *dgst, int dlen, { ECDSA_SIG *s; - if (sig == NULL && (kinv == NULL || r == NULL)) { - *siglen = ECDSA_size(eckey); - return 1; - } - s = ECDSA_do_sign_ex(dgst, dlen, kinv, r, eckey); if (s == NULL) { *siglen = 0; return 0; } - *siglen = i2d_ECDSA_SIG(s, sig != NULL ? &sig : NULL); - ECDSA_SIG_free(s); - return 1; -} - -int ossl_ecdsa_deterministic_sign(const unsigned char *dgst, int dlen, - unsigned char *sig, unsigned int *siglen, - EC_KEY *eckey, unsigned int nonce_type, - const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq) -{ - ECDSA_SIG *s; - BIGNUM *kinv = NULL, *r = NULL; - int ret = 0; - - if (sig == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - *siglen = 0; - if (!ecdsa_sign_setup(eckey, NULL, &kinv, &r, dgst, dlen, - nonce_type, digestname, libctx, propq)) - return 0; - - s = ECDSA_do_sign_ex(dgst, dlen, kinv, r, eckey); - if (s == NULL) - goto end; - *siglen = i2d_ECDSA_SIG(s, &sig); ECDSA_SIG_free(s); - ret = 1; -end: - BN_clear_free(kinv); - BN_clear_free(r); - return ret; + return 1; } static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp, - const unsigned char *dgst, int dlen, - unsigned int nonce_type, const char *digestname, - OSSL_LIB_CTX *libctx, const char *propq) + const unsigned char *dgst, int dlen) { BN_CTX *ctx = NULL; BIGNUM *k = NULL, *r = NULL, *X = NULL; @@ -156,7 +100,7 @@ static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, if ((ctx = ctx_in) == NULL) { if ((ctx = BN_CTX_new_ex(eckey->libctx)) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } } @@ -165,7 +109,7 @@ static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, r = BN_new(); /* this value is later returned in *rp */ X = BN_new(); if (k == NULL || r == NULL || X == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } if ((tmp_point = EC_POINT_new(group)) == NULL) { @@ -176,39 +120,27 @@ static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, /* Preallocate space */ order_bits = BN_num_bits(order); - /* Check the number of bits here so that an infinite loop is not possible */ - if (order_bits < MIN_ECDSA_SIGN_ORDERBITS - || !BN_set_bit(k, order_bits) + if (!BN_set_bit(k, order_bits) || !BN_set_bit(r, order_bits) || !BN_set_bit(X, order_bits)) goto err; do { - /* get random or deterministic value of k */ + /* get random k */ do { - int res = 0; - if (dgst != NULL) { - if (nonce_type == 1) { -#ifndef FIPS_MODULE - res = ossl_gen_deterministic_nonce_rfc6979(k, order, - priv_key, - dgst, dlen, - digestname, - libctx, propq); -#endif - } else { - res = ossl_bn_gen_dsa_nonce_fixed_top(k, order, priv_key, - dgst, dlen, ctx); + if (!BN_generate_dsa_nonce(k, order, priv_key, + dgst, dlen, ctx)) { + ERR_raise(ERR_LIB_EC, EC_R_RANDOM_NUMBER_GENERATION_FAILED); + goto err; } } else { - res = ossl_bn_priv_rand_range_fixed_top(k, order, 0, ctx); - } - if (!res) { - ERR_raise(ERR_LIB_EC, EC_R_RANDOM_NUMBER_GENERATION_FAILED); - goto err; + if (!BN_priv_rand_range_ex(k, order, 0, ctx)) { + ERR_raise(ERR_LIB_EC, EC_R_RANDOM_NUMBER_GENERATION_FAILED); + goto err; + } } - } while (ossl_bn_is_word_fixed_top(k, 0)); + } while (BN_is_zero(k)); /* compute r the x-coordinate of generator * k */ if (!EC_POINT_mul(group, tmp_point, k, NULL, NULL, ctx)) { @@ -255,8 +187,7 @@ static int ecdsa_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, int ossl_ecdsa_simple_sign_setup(EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp) { - return ecdsa_sign_setup(eckey, ctx_in, kinvp, rp, NULL, 0, - 0, NULL, NULL, NULL); + return ecdsa_sign_setup(eckey, ctx_in, kinvp, rp, NULL, 0); } ECDSA_SIG *ossl_ecdsa_simple_sign_sig(const unsigned char *dgst, int dgst_len, @@ -264,7 +195,6 @@ ECDSA_SIG *ossl_ecdsa_simple_sign_sig(const unsigned char *dgst, int dgst_len, EC_KEY *eckey) { int ok = 0, i; - int retries = 0; BIGNUM *kinv = NULL, *s, *m = NULL; const BIGNUM *order, *ckinv; BN_CTX *ctx = NULL; @@ -291,20 +221,20 @@ ECDSA_SIG *ossl_ecdsa_simple_sign_sig(const unsigned char *dgst, int dgst_len, ret = ECDSA_SIG_new(); if (ret == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ECDSA_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NULL; } ret->r = BN_new(); ret->s = BN_new(); if (ret->r == NULL || ret->s == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } s = ret->s; if ((ctx = BN_CTX_new_ex(eckey->libctx)) == NULL || (m = BN_new()) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -326,8 +256,7 @@ ECDSA_SIG *ossl_ecdsa_simple_sign_sig(const unsigned char *dgst, int dgst_len, } do { if (in_kinv == NULL || in_r == NULL) { - if (!ecdsa_sign_setup(eckey, ctx, &kinv, &ret->r, dgst, dgst_len, - 0, NULL, NULL, NULL)) { + if (!ecdsa_sign_setup(eckey, ctx, &kinv, &ret->r, dgst, dgst_len)) { ERR_raise(ERR_LIB_EC, ERR_R_ECDSA_LIB); goto err; } @@ -335,7 +264,7 @@ ECDSA_SIG *ossl_ecdsa_simple_sign_sig(const unsigned char *dgst, int dgst_len, } else { ckinv = in_kinv; if (BN_copy(ret->r, in_r) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -375,11 +304,6 @@ ECDSA_SIG *ossl_ecdsa_simple_sign_sig(const unsigned char *dgst, int dgst_len, ERR_raise(ERR_LIB_EC, EC_R_NEED_NEW_SETUP_VALUES); goto err; } - /* Avoid infinite loops cause by invalid group parameters */ - if (retries++ > MAX_ECDSA_SIGN_RETRIES) { - ERR_raise(ERR_LIB_EC, EC_R_TOO_MANY_RETRIES); - goto err; - } } else { /* s != 0 => we have a valid signature */ break; @@ -454,7 +378,7 @@ int ossl_ecdsa_simple_verify_sig(const unsigned char *dgst, int dgst_len, ctx = BN_CTX_new_ex(eckey->libctx); if (ctx == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return -1; } BN_CTX_start(ctx); @@ -513,7 +437,7 @@ int ossl_ecdsa_simple_verify_sig(const unsigned char *dgst, int dgst_len, } if ((point = EC_POINT_new(group)) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } if (!EC_POINT_mul(group, point, u1, pub_key, u2, ctx)) { diff --git a/openssl/src/crypto/ec/eck_prn.c b/openssl/src/crypto/ec/eck_prn.c index 1bb58c6f3..96ced7c8b 100644 --- a/openssl/src/crypto/ec/eck_prn.c +++ b/openssl/src/crypto/ec/eck_prn.c @@ -89,7 +89,7 @@ int ECPKParameters_print(BIO *bp, const EC_GROUP *x, int off) ctx = BN_CTX_new(); if (ctx == NULL) { - reason = ERR_R_BN_LIB; + reason = ERR_R_MALLOC_FAILURE; goto err; } @@ -127,7 +127,7 @@ int ECPKParameters_print(BIO *bp, const EC_GROUP *x, int off) if ((p = BN_new()) == NULL || (a = BN_new()) == NULL || (b = BN_new()) == NULL) { - reason = ERR_R_BN_LIB; + reason = ERR_R_MALLOC_FAILURE; goto err; } diff --git a/openssl/src/crypto/ec/ecp_meth.c b/openssl/src/crypto/ec/ecp_meth.c new file mode 100644 index 000000000..e0d32ce70 --- /dev/null +++ b/openssl/src/crypto/ec/ecp_meth.c @@ -0,0 +1,191 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "ec_local.h" + +/** Creates a new EC_POINT_METHOD object for the specified curve_id + * \param curve_id the elliptic curve id + * \return newly created EC_POINT_METHOD object or NULL if an error occurred + */ +EC_POINT_METHOD *EC_POINT_METHOD_new(int curve_id) +{ + EC_POINT_METHOD *ret = OPENSSL_zalloc(sizeof(*ret)); + + if (ret == NULL) + return NULL; + + ret->curve_id = curve_id; + + return ret; +} + +/** Frees a EC_POINT_METHOD object + * \param meth EC_POINT_METHOD object to be freed + */ +void EC_POINT_METHOD_free(EC_POINT_METHOD *meth) +{ + OPENSSL_free(meth); +} + +/** Copies EC_POINT_METHOD object + * \param dst destination EC_POINT_METHOD object + * \param src source EC_POINT_METHOD object + * \return 1 on success and 0 if an error occurred + */ +int EC_POINT_METHOD_copy(EC_POINT_METHOD *dst, const EC_POINT_METHOD *src) +{ + if (dst == src) + return 1; + + if (dst->curve_id != src->curve_id) { + ERR_raise(ERR_LIB_EC, EC_R_INCOMPATIBLE_OBJECTS); + return 0; + } + + memcpy(dst, src, sizeof(*dst)); + + return 1; +} + +/** Returns the curve_id of a EC_POINT_METHOD object + * \param meth EC_POINT_METHOD object + * \return NID of the curve name OID or 0 if not set. + */ +int EC_POINT_METHOD_curve_id(EC_POINT_METHOD *meth) +{ + return meth->curve_id; +} + +int (*EC_POINT_METHOD_get_add(EC_POINT_METHOD *meth)) + (const EC_GROUP *, EC_POINT *r, const EC_POINT *a, const EC_POINT *b, + BN_CTX *) +{ + return meth->add; +} + +void EC_POINT_METHOD_set_add(EC_POINT_METHOD *meth, + int (*add)(const EC_GROUP *, EC_POINT *r, + const EC_POINT *a, const EC_POINT *b, + BN_CTX *)) +{ + meth->add = add; +} + +int (*EC_POINT_METHOD_get_dbl(EC_POINT_METHOD *meth)) + (const EC_GROUP *, EC_POINT *r, const EC_POINT *a, BN_CTX *) +{ + return meth->dbl; +} + +void EC_POINT_METHOD_set_dbl(EC_POINT_METHOD *meth, + int (*dbl)(const EC_GROUP *, EC_POINT *r, + const EC_POINT *a, BN_CTX *)) +{ + meth->dbl = dbl; +} + +int (*EC_POINT_METHOD_get_invert(EC_POINT_METHOD *meth)) + (const EC_GROUP *, EC_POINT *point, BN_CTX *) +{ + return meth->invert; +} + +void EC_POINT_METHOD_set_invert(EC_POINT_METHOD *meth, + int (*invert)(const EC_GROUP *, EC_POINT *point, + BN_CTX *)) +{ + meth->invert = invert; +} + +int (*EC_POINT_METHOD_get_mul(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar, size_t num, + const EC_POINT *points[], const BIGNUM *scalars[], BN_CTX *) +{ + return meth->mul; +} + +void EC_POINT_METHOD_set_mul(EC_POINT_METHOD *meth, + int (*mul)(const EC_GROUP *group, EC_POINT *r, + const BIGNUM *scalar, size_t num, + const EC_POINT *points[], + const BIGNUM *scalars[], BN_CTX *)) +{ + meth->mul = mul; +} + +int (*EC_POINT_METHOD_get_scalars_mul(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r[], size_t num, const EC_POINT *points[], + const BIGNUM *scalars[], BN_CTX *ctx) +{ + return meth->scalars_mul; +} + +void EC_POINT_METHOD_set_scalars_mul(EC_POINT_METHOD *meth, + int (*scalars_mul)(const EC_GROUP *group, + EC_POINT *r[], size_t num, + const EC_POINT *points[], + const BIGNUM *scalars[], + BN_CTX *ctx)) +{ + meth->scalars_mul = scalars_mul; +} + +int (*EC_POINT_METHOD_get_scalar_mul(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r[], size_t num, const EC_POINT *points[], + const BIGNUM *scalar, BN_CTX *ctx) +{ + return meth->scalar_mul; +} + +void EC_POINT_METHOD_set_scalar_mul(EC_POINT_METHOD *meth, + int (*scalar_mul)(const EC_GROUP *group, + EC_POINT *r[], size_t num, + const EC_POINT *points[], + const BIGNUM *scalar, + BN_CTX *ctx)) +{ + meth->scalar_mul = scalar_mul; +} + +int (*EC_POINT_METHOD_get_strings_to_points(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r[], size_t num, const unsigned char *strings[], + BN_CTX *ctx) +{ + return meth->strings_to_points; +} + +void EC_POINT_METHOD_set_strings_to_points(EC_POINT_METHOD *meth, + int (*func)(const EC_GROUP *group, + EC_POINT *r[], size_t num, + const unsigned char *strings[], + BN_CTX *ctx)) +{ + meth->strings_to_points = func; +} + +int (*EC_POINT_METHOD_get_strings_to_points_scalar_mul(EC_POINT_METHOD *meth)) + (const EC_GROUP *group, EC_POINT *r[], size_t num, const unsigned char *strings[], + const BIGNUM *scalar, BN_CTX *ctx) +{ + return meth->strings_to_points_scalar_mul; +} + +void EC_POINT_METHOD_set_strings_to_points_scalar_mul(EC_POINT_METHOD *meth, + int (*func)(const EC_GROUP *group, + EC_POINT *r[], size_t num, + const unsigned char *strings[], + const BIGNUM *scalar, + BN_CTX *ctx)) +{ + meth->strings_to_points_scalar_mul = func; +} diff --git a/openssl/src/crypto/ec/ecp_mont.c b/openssl/src/crypto/ec/ecp_mont.c index 12750b600..35b492453 100644 --- a/openssl/src/crypto/ec/ecp_mont.c +++ b/openssl/src/crypto/ec/ecp_mont.c @@ -9,7 +9,7 @@ */ /* - * ECDSA low-level APIs are deprecated for public use, but still ok for + * ECDSA low level APIs are deprecated for public use, but still ok for * internal use. */ #include "internal/deprecated.h" @@ -217,7 +217,7 @@ int ossl_ec_GFp_mont_field_sqr(const EC_GROUP *group, BIGNUM *r, const BIGNUM *a /*- * Computes the multiplicative inverse of a in GF(p), storing the result in r. - * If a is zero (or equivalent), you'll get an EC_R_CANNOT_INVERT error. + * If a is zero (or equivalent), you'll get a EC_R_CANNOT_INVERT error. * We have a Mont structure, so SCA hardening is FLT inversion. */ int ossl_ec_GFp_mont_field_inv(const EC_GROUP *group, BIGNUM *r, const BIGNUM *a, diff --git a/openssl/src/crypto/ec/ecp_nistp224.c b/openssl/src/crypto/ec/ecp_nistp224.c index debfdb3dc..5ab0dd7be 100644 --- a/openssl/src/crypto/ec/ecp_nistp224.c +++ b/openssl/src/crypto/ec/ecp_nistp224.c @@ -1,5 +1,5 @@ /* - * Copyright 2010-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2010-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -238,6 +238,7 @@ static const felem gmul[2][16][3] = { struct nistp224_pre_comp_st { felem g_pre_comp[2][16][3]; CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; }; const EC_METHOD *EC_GFp_nistp224_method(void) @@ -1237,11 +1238,16 @@ static NISTP224_PRE_COMP *nistp224_pre_comp_new(void) { NISTP224_PRE_COMP *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (!ret) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return ret; + } + ret->references = 1; - if (!CRYPTO_NEW_REF(&ret->references, 1)) { + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -1252,7 +1258,7 @@ NISTP224_PRE_COMP *EC_nistp224_pre_comp_dup(NISTP224_PRE_COMP *p) { int i; if (p != NULL) - CRYPTO_UP_REF(&p->references, &i); + CRYPTO_UP_REF(&p->references, &i, p->lock); return p; } @@ -1263,13 +1269,13 @@ void EC_nistp224_pre_comp_free(NISTP224_PRE_COMP *p) if (p == NULL) return; - CRYPTO_DOWN_REF(&p->references, &i); + CRYPTO_DOWN_REF(&p->references, &i, p->lock); REF_PRINT_COUNT("EC_nistp224", p); if (i > 0) return; REF_ASSERT_ISNT(i < 0); - CRYPTO_FREE_REF(&p->references); + CRYPTO_THREAD_lock_free(p->lock); OPENSSL_free(p); } @@ -1481,8 +1487,10 @@ int ossl_ec_GFp_nistp224_points_mul(const EC_GROUP *group, EC_POINT *r, tmp_felems = OPENSSL_malloc(sizeof(felem) * (num_points * 17 + 1)); if ((secrets == NULL) || (pre_comp == NULL) - || (mixed && (tmp_felems == NULL))) + || (mixed && (tmp_felems == NULL))) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } /* * we treat NULL scalars as 0, and NULL points as points at infinity, diff --git a/openssl/src/crypto/ec/ecp_nistp256.c b/openssl/src/crypto/ec/ecp_nistp256.c index d28306a6b..4a55f925c 100644 --- a/openssl/src/crypto/ec/ecp_nistp256.c +++ b/openssl/src/crypto/ec/ecp_nistp256.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -97,7 +97,7 @@ static const felem_bytearray nistp256_curve_params[5] = { * values, or four 64-bit values. The field element represented is: * v[0]*2^0 + v[1]*2^64 + v[2]*2^128 + v[3]*2^192 (mod p) * or: - * v[0]*2^0 + v[1]*2^64 + v[2]*2^128 + ... + v[7]*2^448 (mod p) + * v[0]*2^0 + v[1]*2^64 + v[2]*2^128 + ... + v[8]*2^512 (mod p) * * 128-bit values are called 'limbs'. Since the limbs are spaced only 64 bits * apart, but are 128-bits wide, the most significant bits of each limb overlap @@ -1773,6 +1773,7 @@ static void batch_mul(felem x_out, felem y_out, felem z_out, struct nistp256_pre_comp_st { smallfelem g_pre_comp[2][16][3]; CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; }; const EC_METHOD *EC_GFp_nistp256_method(void) @@ -1848,10 +1849,16 @@ static NISTP256_PRE_COMP *nistp256_pre_comp_new(void) { NISTP256_PRE_COMP *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return ret; + } + + ret->references = 1; - if (!CRYPTO_NEW_REF(&ret->references, 1)) { + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -1862,7 +1869,7 @@ NISTP256_PRE_COMP *EC_nistp256_pre_comp_dup(NISTP256_PRE_COMP *p) { int i; if (p != NULL) - CRYPTO_UP_REF(&p->references, &i); + CRYPTO_UP_REF(&p->references, &i, p->lock); return p; } @@ -1873,13 +1880,13 @@ void EC_nistp256_pre_comp_free(NISTP256_PRE_COMP *pre) if (pre == NULL) return; - CRYPTO_DOWN_REF(&pre->references, &i); + CRYPTO_DOWN_REF(&pre->references, &i, pre->lock); REF_PRINT_COUNT("EC_nistp256", pre); if (i > 0) return; REF_ASSERT_ISNT(i < 0); - CRYPTO_FREE_REF(&pre->references); + CRYPTO_THREAD_lock_free(pre->lock); OPENSSL_free(pre); } @@ -2092,8 +2099,10 @@ int ossl_ec_GFp_nistp256_points_mul(const EC_GROUP *group, EC_POINT *r, tmp_smallfelems = OPENSSL_malloc(sizeof(*tmp_smallfelems) * (num_points * 17 + 1)); if ((secrets == NULL) || (pre_comp == NULL) - || (mixed && (tmp_smallfelems == NULL))) + || (mixed && (tmp_smallfelems == NULL))) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } /* * we treat NULL scalars as 0, and NULL points as points at infinity, diff --git a/openssl/src/crypto/ec/ecp_nistp384.c b/openssl/src/crypto/ec/ecp_nistp384.c deleted file mode 100644 index ff68f9cc7..000000000 --- a/openssl/src/crypto/ec/ecp_nistp384.c +++ /dev/null @@ -1,1997 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Copyright 2023 IBM Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Designed for 56-bit limbs by Rohan McLure . - * The layout is based on that of ecp_nistp{224,521}.c, allowing even for asm - * acceleration of felem_{square,mul} as supported in these files. - */ - -#include - -#include -#include -#include "ec_local.h" - -#include "internal/numbers.h" - -#ifndef INT128_MAX -# error "Your compiler doesn't appear to support 128-bit integer types" -#endif - -typedef uint8_t u8; -typedef uint64_t u64; - -/* - * The underlying field. P384 operates over GF(2^384-2^128-2^96+2^32-1). We - * can serialize an element of this field into 48 bytes. We call this an - * felem_bytearray. - */ - -typedef u8 felem_bytearray[48]; - -/* - * These are the parameters of P384, taken from FIPS 186-3, section D.1.2.4. - * These values are big-endian. - */ -static const felem_bytearray nistp384_curve_params[5] = { - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, /* p */ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, /* a = -3 */ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFC}, - {0xB3, 0x31, 0x2F, 0xA7, 0xE2, 0x3E, 0xE7, 0xE4, 0x98, 0x8E, 0x05, 0x6B, /* b */ - 0xE3, 0xF8, 0x2D, 0x19, 0x18, 0x1D, 0x9C, 0x6E, 0xFE, 0x81, 0x41, 0x12, - 0x03, 0x14, 0x08, 0x8F, 0x50, 0x13, 0x87, 0x5A, 0xC6, 0x56, 0x39, 0x8D, - 0x8A, 0x2E, 0xD1, 0x9D, 0x2A, 0x85, 0xC8, 0xED, 0xD3, 0xEC, 0x2A, 0xEF}, - {0xAA, 0x87, 0xCA, 0x22, 0xBE, 0x8B, 0x05, 0x37, 0x8E, 0xB1, 0xC7, 0x1E, /* x */ - 0xF3, 0x20, 0xAD, 0x74, 0x6E, 0x1D, 0x3B, 0x62, 0x8B, 0xA7, 0x9B, 0x98, - 0x59, 0xF7, 0x41, 0xE0, 0x82, 0x54, 0x2A, 0x38, 0x55, 0x02, 0xF2, 0x5D, - 0xBF, 0x55, 0x29, 0x6C, 0x3A, 0x54, 0x5E, 0x38, 0x72, 0x76, 0x0A, 0xB7}, - {0x36, 0x17, 0xDE, 0x4A, 0x96, 0x26, 0x2C, 0x6F, 0x5D, 0x9E, 0x98, 0xBF, /* y */ - 0x92, 0x92, 0xDC, 0x29, 0xF8, 0xF4, 0x1D, 0xBD, 0x28, 0x9A, 0x14, 0x7C, - 0xE9, 0xDA, 0x31, 0x13, 0xB5, 0xF0, 0xB8, 0xC0, 0x0A, 0x60, 0xB1, 0xCE, - 0x1D, 0x7E, 0x81, 0x9D, 0x7A, 0x43, 0x1D, 0x7C, 0x90, 0xEA, 0x0E, 0x5F}, -}; - -/*- - * The representation of field elements. - * ------------------------------------ - * - * We represent field elements with seven values. These values are either 64 or - * 128 bits and the field element represented is: - * v[0]*2^0 + v[1]*2^56 + v[2]*2^112 + ... + v[6]*2^336 (mod p) - * Each of the seven values is called a 'limb'. Since the limbs are spaced only - * 56 bits apart, but are greater than 56 bits in length, the most significant - * bits of each limb overlap with the least significant bits of the next - * - * This representation is considered to be 'redundant' in the sense that - * intermediate values can each contain more than a 56-bit value in each limb. - * Reduction causes all but the final limb to be reduced to contain a value less - * than 2^56, with the final value represented allowed to be larger than 2^384, - * inasmuch as we can be sure that arithmetic overflow remains impossible. The - * reduced value must of course be congruent to the unreduced value. - * - * A field element with 64-bit limbs is an 'felem'. One with 128-bit limbs is a - * 'widefelem', featuring enough bits to store the result of a multiplication - * and even some further arithmetic without need for immediate reduction. - */ - -#define NLIMBS 7 - -typedef uint64_t limb; -typedef uint128_t widelimb; -typedef limb limb_aX __attribute((__aligned__(1))); -typedef limb felem[NLIMBS]; -typedef widelimb widefelem[2*NLIMBS-1]; - -static const limb bottom56bits = 0xffffffffffffff; - -/* Helper functions (de)serialising reduced field elements in little endian */ -static void bin48_to_felem(felem out, const u8 in[48]) -{ - memset(out, 0, 56); - out[0] = (*((limb *) & in[0])) & bottom56bits; - out[1] = (*((limb_aX *) & in[7])) & bottom56bits; - out[2] = (*((limb_aX *) & in[14])) & bottom56bits; - out[3] = (*((limb_aX *) & in[21])) & bottom56bits; - out[4] = (*((limb_aX *) & in[28])) & bottom56bits; - out[5] = (*((limb_aX *) & in[35])) & bottom56bits; - memmove(&out[6], &in[42], 6); -} - -static void felem_to_bin48(u8 out[48], const felem in) -{ - memset(out, 0, 48); - (*((limb *) & out[0])) |= (in[0] & bottom56bits); - (*((limb_aX *) & out[7])) |= (in[1] & bottom56bits); - (*((limb_aX *) & out[14])) |= (in[2] & bottom56bits); - (*((limb_aX *) & out[21])) |= (in[3] & bottom56bits); - (*((limb_aX *) & out[28])) |= (in[4] & bottom56bits); - (*((limb_aX *) & out[35])) |= (in[5] & bottom56bits); - memmove(&out[42], &in[6], 6); -} - -/* BN_to_felem converts an OpenSSL BIGNUM into an felem */ -static int BN_to_felem(felem out, const BIGNUM *bn) -{ - felem_bytearray b_out; - int num_bytes; - - if (BN_is_negative(bn)) { - ERR_raise(ERR_LIB_EC, EC_R_BIGNUM_OUT_OF_RANGE); - return 0; - } - num_bytes = BN_bn2lebinpad(bn, b_out, sizeof(b_out)); - if (num_bytes < 0) { - ERR_raise(ERR_LIB_EC, EC_R_BIGNUM_OUT_OF_RANGE); - return 0; - } - bin48_to_felem(out, b_out); - return 1; -} - -/* felem_to_BN converts an felem into an OpenSSL BIGNUM */ -static BIGNUM *felem_to_BN(BIGNUM *out, const felem in) -{ - felem_bytearray b_out; - - felem_to_bin48(b_out, in); - return BN_lebin2bn(b_out, sizeof(b_out), out); -} - -/*- - * Field operations - * ---------------- - */ - -static void felem_one(felem out) -{ - out[0] = 1; - memset(&out[1], 0, sizeof(limb) * (NLIMBS-1)); -} - -static void felem_assign(felem out, const felem in) -{ - memcpy(out, in, sizeof(felem)); -} - -/* felem_sum64 sets out = out + in. */ -static void felem_sum64(felem out, const felem in) -{ - unsigned int i; - - for (i = 0; i < NLIMBS; i++) - out[i] += in[i]; -} - -/* felem_scalar sets out = in * scalar */ -static void felem_scalar(felem out, const felem in, limb scalar) -{ - unsigned int i; - - for (i = 0; i < NLIMBS; i++) - out[i] = in[i] * scalar; -} - -/* felem_scalar64 sets out = out * scalar */ -static void felem_scalar64(felem out, limb scalar) -{ - unsigned int i; - - for (i = 0; i < NLIMBS; i++) - out[i] *= scalar; -} - -/* felem_scalar128 sets out = out * scalar */ -static void felem_scalar128(widefelem out, limb scalar) -{ - unsigned int i; - - for (i = 0; i < 2*NLIMBS-1; i++) - out[i] *= scalar; -} - -/*- - * felem_neg sets |out| to |-in| - * On entry: - * in[i] < 2^60 - 2^29 - * On exit: - * out[i] < 2^60 - */ -static void felem_neg(felem out, const felem in) -{ - /* - * In order to prevent underflow, we add a multiple of p before subtracting. - * Use telescopic sums to represent 2^12 * p redundantly with each limb - * of the form 2^60 + ... - */ - static const limb two60m52m4 = (((limb) 1) << 60) - - (((limb) 1) << 52) - - (((limb) 1) << 4); - static const limb two60p44m12 = (((limb) 1) << 60) - + (((limb) 1) << 44) - - (((limb) 1) << 12); - static const limb two60m28m4 = (((limb) 1) << 60) - - (((limb) 1) << 28) - - (((limb) 1) << 4); - static const limb two60m4 = (((limb) 1) << 60) - - (((limb) 1) << 4); - - out[0] = two60p44m12 - in[0]; - out[1] = two60m52m4 - in[1]; - out[2] = two60m28m4 - in[2]; - out[3] = two60m4 - in[3]; - out[4] = two60m4 - in[4]; - out[5] = two60m4 - in[5]; - out[6] = two60m4 - in[6]; -} - -/*- - * felem_diff64 subtracts |in| from |out| - * On entry: - * in[i] < 2^60 - 2^52 - 2^4 - * On exit: - * out[i] < out_orig[i] + 2^60 + 2^44 - */ -static void felem_diff64(felem out, const felem in) -{ - /* - * In order to prevent underflow, we add a multiple of p before subtracting. - * Use telescopic sums to represent 2^12 * p redundantly with each limb - * of the form 2^60 + ... - */ - - static const limb two60m52m4 = (((limb) 1) << 60) - - (((limb) 1) << 52) - - (((limb) 1) << 4); - static const limb two60p44m12 = (((limb) 1) << 60) - + (((limb) 1) << 44) - - (((limb) 1) << 12); - static const limb two60m28m4 = (((limb) 1) << 60) - - (((limb) 1) << 28) - - (((limb) 1) << 4); - static const limb two60m4 = (((limb) 1) << 60) - - (((limb) 1) << 4); - - out[0] += two60p44m12 - in[0]; - out[1] += two60m52m4 - in[1]; - out[2] += two60m28m4 - in[2]; - out[3] += two60m4 - in[3]; - out[4] += two60m4 - in[4]; - out[5] += two60m4 - in[5]; - out[6] += two60m4 - in[6]; -} - -/* - * in[i] < 2^63 - * out[i] < out_orig[i] + 2^64 + 2^48 - */ -static void felem_diff_128_64(widefelem out, const felem in) -{ - /* - * In order to prevent underflow, we add a multiple of p before subtracting. - * Use telescopic sums to represent 2^16 * p redundantly with each limb - * of the form 2^64 + ... - */ - - static const widelimb two64m56m8 = (((widelimb) 1) << 64) - - (((widelimb) 1) << 56) - - (((widelimb) 1) << 8); - static const widelimb two64m32m8 = (((widelimb) 1) << 64) - - (((widelimb) 1) << 32) - - (((widelimb) 1) << 8); - static const widelimb two64m8 = (((widelimb) 1) << 64) - - (((widelimb) 1) << 8); - static const widelimb two64p48m16 = (((widelimb) 1) << 64) - + (((widelimb) 1) << 48) - - (((widelimb) 1) << 16); - unsigned int i; - - out[0] += two64p48m16; - out[1] += two64m56m8; - out[2] += two64m32m8; - out[3] += two64m8; - out[4] += two64m8; - out[5] += two64m8; - out[6] += two64m8; - - for (i = 0; i < NLIMBS; i++) - out[i] -= in[i]; -} - -/* - * in[i] < 2^127 - 2^119 - 2^71 - * out[i] < out_orig[i] + 2^127 + 2^111 - */ -static void felem_diff128(widefelem out, const widefelem in) -{ - /* - * In order to prevent underflow, we add a multiple of p before subtracting. - * Use telescopic sums to represent 2^415 * p redundantly with each limb - * of the form 2^127 + ... - */ - - static const widelimb two127 = ((widelimb) 1) << 127; - static const widelimb two127m71 = (((widelimb) 1) << 127) - - (((widelimb) 1) << 71); - static const widelimb two127p111m79m71 = (((widelimb) 1) << 127) - + (((widelimb) 1) << 111) - - (((widelimb) 1) << 79) - - (((widelimb) 1) << 71); - static const widelimb two127m119m71 = (((widelimb) 1) << 127) - - (((widelimb) 1) << 119) - - (((widelimb) 1) << 71); - static const widelimb two127m95m71 = (((widelimb) 1) << 127) - - (((widelimb) 1) << 95) - - (((widelimb) 1) << 71); - unsigned int i; - - out[0] += two127; - out[1] += two127m71; - out[2] += two127m71; - out[3] += two127m71; - out[4] += two127m71; - out[5] += two127m71; - out[6] += two127p111m79m71; - out[7] += two127m119m71; - out[8] += two127m95m71; - out[9] += two127m71; - out[10] += two127m71; - out[11] += two127m71; - out[12] += two127m71; - - for (i = 0; i < 2*NLIMBS-1; i++) - out[i] -= in[i]; -} - -static void felem_square_ref(widefelem out, const felem in) -{ - felem inx2; - felem_scalar(inx2, in, 2); - - out[0] = ((uint128_t) in[0]) * in[0]; - - out[1] = ((uint128_t) in[0]) * inx2[1]; - - out[2] = ((uint128_t) in[0]) * inx2[2] - + ((uint128_t) in[1]) * in[1]; - - out[3] = ((uint128_t) in[0]) * inx2[3] - + ((uint128_t) in[1]) * inx2[2]; - - out[4] = ((uint128_t) in[0]) * inx2[4] - + ((uint128_t) in[1]) * inx2[3] - + ((uint128_t) in[2]) * in[2]; - - out[5] = ((uint128_t) in[0]) * inx2[5] - + ((uint128_t) in[1]) * inx2[4] - + ((uint128_t) in[2]) * inx2[3]; - - out[6] = ((uint128_t) in[0]) * inx2[6] - + ((uint128_t) in[1]) * inx2[5] - + ((uint128_t) in[2]) * inx2[4] - + ((uint128_t) in[3]) * in[3]; - - out[7] = ((uint128_t) in[1]) * inx2[6] - + ((uint128_t) in[2]) * inx2[5] - + ((uint128_t) in[3]) * inx2[4]; - - out[8] = ((uint128_t) in[2]) * inx2[6] - + ((uint128_t) in[3]) * inx2[5] - + ((uint128_t) in[4]) * in[4]; - - out[9] = ((uint128_t) in[3]) * inx2[6] - + ((uint128_t) in[4]) * inx2[5]; - - out[10] = ((uint128_t) in[4]) * inx2[6] - + ((uint128_t) in[5]) * in[5]; - - out[11] = ((uint128_t) in[5]) * inx2[6]; - - out[12] = ((uint128_t) in[6]) * in[6]; -} - -static void felem_mul_ref(widefelem out, const felem in1, const felem in2) -{ - out[0] = ((uint128_t) in1[0]) * in2[0]; - - out[1] = ((uint128_t) in1[0]) * in2[1] - + ((uint128_t) in1[1]) * in2[0]; - - out[2] = ((uint128_t) in1[0]) * in2[2] - + ((uint128_t) in1[1]) * in2[1] - + ((uint128_t) in1[2]) * in2[0]; - - out[3] = ((uint128_t) in1[0]) * in2[3] - + ((uint128_t) in1[1]) * in2[2] - + ((uint128_t) in1[2]) * in2[1] - + ((uint128_t) in1[3]) * in2[0]; - - out[4] = ((uint128_t) in1[0]) * in2[4] - + ((uint128_t) in1[1]) * in2[3] - + ((uint128_t) in1[2]) * in2[2] - + ((uint128_t) in1[3]) * in2[1] - + ((uint128_t) in1[4]) * in2[0]; - - out[5] = ((uint128_t) in1[0]) * in2[5] - + ((uint128_t) in1[1]) * in2[4] - + ((uint128_t) in1[2]) * in2[3] - + ((uint128_t) in1[3]) * in2[2] - + ((uint128_t) in1[4]) * in2[1] - + ((uint128_t) in1[5]) * in2[0]; - - out[6] = ((uint128_t) in1[0]) * in2[6] - + ((uint128_t) in1[1]) * in2[5] - + ((uint128_t) in1[2]) * in2[4] - + ((uint128_t) in1[3]) * in2[3] - + ((uint128_t) in1[4]) * in2[2] - + ((uint128_t) in1[5]) * in2[1] - + ((uint128_t) in1[6]) * in2[0]; - - out[7] = ((uint128_t) in1[1]) * in2[6] - + ((uint128_t) in1[2]) * in2[5] - + ((uint128_t) in1[3]) * in2[4] - + ((uint128_t) in1[4]) * in2[3] - + ((uint128_t) in1[5]) * in2[2] - + ((uint128_t) in1[6]) * in2[1]; - - out[8] = ((uint128_t) in1[2]) * in2[6] - + ((uint128_t) in1[3]) * in2[5] - + ((uint128_t) in1[4]) * in2[4] - + ((uint128_t) in1[5]) * in2[3] - + ((uint128_t) in1[6]) * in2[2]; - - out[9] = ((uint128_t) in1[3]) * in2[6] - + ((uint128_t) in1[4]) * in2[5] - + ((uint128_t) in1[5]) * in2[4] - + ((uint128_t) in1[6]) * in2[3]; - - out[10] = ((uint128_t) in1[4]) * in2[6] - + ((uint128_t) in1[5]) * in2[5] - + ((uint128_t) in1[6]) * in2[4]; - - out[11] = ((uint128_t) in1[5]) * in2[6] - + ((uint128_t) in1[6]) * in2[5]; - - out[12] = ((uint128_t) in1[6]) * in2[6]; -} - -/*- - * Reduce thirteen 128-bit coefficients to seven 64-bit coefficients. - * in[i] < 2^128 - 2^125 - * out[i] < 2^56 for i < 6, - * out[6] <= 2^48 - * - * The technique in use here stems from the format of the prime modulus: - * P384 = 2^384 - delta - * - * Thus we can reduce numbers of the form (X + 2^384 * Y) by substituting - * them with (X + delta Y), with delta = 2^128 + 2^96 + (-2^32 + 1). These - * coefficients are still quite large, and so we repeatedly apply this - * technique on high-order bits in order to guarantee the desired bounds on - * the size of our output. - * - * The three phases of elimination are as follows: - * [1]: Y = 2^120 (in[12] | in[11] | in[10] | in[9]) - * [2]: Y = 2^8 (acc[8] | acc[7]) - * [3]: Y = 2^48 (acc[6] >> 48) - * (Where a | b | c | d = (2^56)^3 a + (2^56)^2 b + (2^56) c + d) - */ -static void felem_reduce(felem out, const widefelem in) -{ - /* - * In order to prevent underflow, we add a multiple of p before subtracting. - * Use telescopic sums to represent 2^76 * p redundantly with each limb - * of the form 2^124 + ... - */ - static const widelimb two124m68 = (((widelimb) 1) << 124) - - (((widelimb) 1) << 68); - static const widelimb two124m116m68 = (((widelimb) 1) << 124) - - (((widelimb) 1) << 116) - - (((widelimb) 1) << 68); - static const widelimb two124p108m76 = (((widelimb) 1) << 124) - + (((widelimb) 1) << 108) - - (((widelimb) 1) << 76); - static const widelimb two124m92m68 = (((widelimb) 1) << 124) - - (((widelimb) 1) << 92) - - (((widelimb) 1) << 68); - widelimb temp, acc[9]; - unsigned int i; - - memcpy(acc, in, sizeof(widelimb) * 9); - - acc[0] += two124p108m76; - acc[1] += two124m116m68; - acc[2] += two124m92m68; - acc[3] += two124m68; - acc[4] += two124m68; - acc[5] += two124m68; - acc[6] += two124m68; - - /* [1]: Eliminate in[9], ..., in[12] */ - acc[8] += in[12] >> 32; - acc[7] += (in[12] & 0xffffffff) << 24; - acc[7] += in[12] >> 8; - acc[6] += (in[12] & 0xff) << 48; - acc[6] -= in[12] >> 16; - acc[5] -= (in[12] & 0xffff) << 40; - acc[6] += in[12] >> 48; - acc[5] += (in[12] & 0xffffffffffff) << 8; - - acc[7] += in[11] >> 32; - acc[6] += (in[11] & 0xffffffff) << 24; - acc[6] += in[11] >> 8; - acc[5] += (in[11] & 0xff) << 48; - acc[5] -= in[11] >> 16; - acc[4] -= (in[11] & 0xffff) << 40; - acc[5] += in[11] >> 48; - acc[4] += (in[11] & 0xffffffffffff) << 8; - - acc[6] += in[10] >> 32; - acc[5] += (in[10] & 0xffffffff) << 24; - acc[5] += in[10] >> 8; - acc[4] += (in[10] & 0xff) << 48; - acc[4] -= in[10] >> 16; - acc[3] -= (in[10] & 0xffff) << 40; - acc[4] += in[10] >> 48; - acc[3] += (in[10] & 0xffffffffffff) << 8; - - acc[5] += in[9] >> 32; - acc[4] += (in[9] & 0xffffffff) << 24; - acc[4] += in[9] >> 8; - acc[3] += (in[9] & 0xff) << 48; - acc[3] -= in[9] >> 16; - acc[2] -= (in[9] & 0xffff) << 40; - acc[3] += in[9] >> 48; - acc[2] += (in[9] & 0xffffffffffff) << 8; - - /* - * [2]: Eliminate acc[7], acc[8], that is the 7 and eighth limbs, as - * well as the contributions made from eliminating higher limbs. - * acc[7] < in[7] + 2^120 + 2^56 < in[7] + 2^121 - * acc[8] < in[8] + 2^96 - */ - acc[4] += acc[8] >> 32; - acc[3] += (acc[8] & 0xffffffff) << 24; - acc[3] += acc[8] >> 8; - acc[2] += (acc[8] & 0xff) << 48; - acc[2] -= acc[8] >> 16; - acc[1] -= (acc[8] & 0xffff) << 40; - acc[2] += acc[8] >> 48; - acc[1] += (acc[8] & 0xffffffffffff) << 8; - - acc[3] += acc[7] >> 32; - acc[2] += (acc[7] & 0xffffffff) << 24; - acc[2] += acc[7] >> 8; - acc[1] += (acc[7] & 0xff) << 48; - acc[1] -= acc[7] >> 16; - acc[0] -= (acc[7] & 0xffff) << 40; - acc[1] += acc[7] >> 48; - acc[0] += (acc[7] & 0xffffffffffff) << 8; - - /*- - * acc[k] < in[k] + 2^124 + 2^121 - * < in[k] + 2^125 - * < 2^128, for k <= 6 - */ - - /* - * Carry 4 -> 5 -> 6 - * This has the effect of ensuring that these more significant limbs - * will be small in value after eliminating high bits from acc[6]. - */ - acc[5] += acc[4] >> 56; - acc[4] &= 0x00ffffffffffffff; - - acc[6] += acc[5] >> 56; - acc[5] &= 0x00ffffffffffffff; - - /*- - * acc[6] < in[6] + 2^124 + 2^121 + 2^72 + 2^16 - * < in[6] + 2^125 - * < 2^128 - */ - - /* [3]: Eliminate high bits of acc[6] */ - temp = acc[6] >> 48; - acc[6] &= 0x0000ffffffffffff; - - /* temp < 2^80 */ - - acc[3] += temp >> 40; - acc[2] += (temp & 0xffffffffff) << 16; - acc[2] += temp >> 16; - acc[1] += (temp & 0xffff) << 40; - acc[1] -= temp >> 24; - acc[0] -= (temp & 0xffffff) << 32; - acc[0] += temp; - - /*- - * acc[k] < acc_old[k] + 2^64 + 2^56 - * < in[k] + 2^124 + 2^121 + 2^72 + 2^64 + 2^56 + 2^16 , k < 4 - */ - - /* Carry 0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 */ - acc[1] += acc[0] >> 56; /* acc[1] < acc_old[1] + 2^72 */ - acc[0] &= 0x00ffffffffffffff; - - acc[2] += acc[1] >> 56; /* acc[2] < acc_old[2] + 2^72 + 2^16 */ - acc[1] &= 0x00ffffffffffffff; - - acc[3] += acc[2] >> 56; /* acc[3] < acc_old[3] + 2^72 + 2^16 */ - acc[2] &= 0x00ffffffffffffff; - - /*- - * acc[k] < acc_old[k] + 2^72 + 2^16 - * < in[k] + 2^124 + 2^121 + 2^73 + 2^64 + 2^56 + 2^17 - * < in[k] + 2^125 - * < 2^128 , k < 4 - */ - - acc[4] += acc[3] >> 56; /*- - * acc[4] < acc_old[4] + 2^72 + 2^16 - * < 2^72 + 2^56 + 2^16 - */ - acc[3] &= 0x00ffffffffffffff; - - acc[5] += acc[4] >> 56; /*- - * acc[5] < acc_old[5] + 2^16 + 1 - * < 2^56 + 2^16 + 1 - */ - acc[4] &= 0x00ffffffffffffff; - - acc[6] += acc[5] >> 56; /* acc[6] < 2^48 + 1 <= 2^48 */ - acc[5] &= 0x00ffffffffffffff; - - for (i = 0; i < NLIMBS; i++) - out[i] = acc[i]; -} - -#if defined(ECP_NISTP384_ASM) -static void felem_square_wrapper(widefelem out, const felem in); -static void felem_mul_wrapper(widefelem out, const felem in1, const felem in2); - -static void (*felem_square_p)(widefelem out, const felem in) = - felem_square_wrapper; -static void (*felem_mul_p)(widefelem out, const felem in1, const felem in2) = - felem_mul_wrapper; - -void p384_felem_square(widefelem out, const felem in); -void p384_felem_mul(widefelem out, const felem in1, const felem in2); - -# if defined(_ARCH_PPC64) -# include "crypto/ppc_arch.h" -# endif - -static void felem_select(void) -{ -# if defined(_ARCH_PPC64) - if ((OPENSSL_ppccap_P & PPC_MADD300) && (OPENSSL_ppccap_P & PPC_ALTIVEC)) { - felem_square_p = p384_felem_square; - felem_mul_p = p384_felem_mul; - - return; - } -# endif - - /* Default */ - felem_square_p = felem_square_ref; - felem_mul_p = felem_mul_ref; -} - -static void felem_square_wrapper(widefelem out, const felem in) -{ - felem_select(); - felem_square_p(out, in); -} - -static void felem_mul_wrapper(widefelem out, const felem in1, const felem in2) -{ - felem_select(); - felem_mul_p(out, in1, in2); -} - -# define felem_square felem_square_p -# define felem_mul felem_mul_p -#else -# define felem_square felem_square_ref -# define felem_mul felem_mul_ref -#endif - -static ossl_inline void felem_square_reduce(felem out, const felem in) -{ - widefelem tmp; - - felem_square(tmp, in); - felem_reduce(out, tmp); -} - -static ossl_inline void felem_mul_reduce(felem out, const felem in1, const felem in2) -{ - widefelem tmp; - - felem_mul(tmp, in1, in2); - felem_reduce(out, tmp); -} - -/*- - * felem_inv calculates |out| = |in|^{-1} - * - * Based on Fermat's Little Theorem: - * a^p = a (mod p) - * a^{p-1} = 1 (mod p) - * a^{p-2} = a^{-1} (mod p) - */ -static void felem_inv(felem out, const felem in) -{ - felem ftmp, ftmp2, ftmp3, ftmp4, ftmp5, ftmp6; - unsigned int i = 0; - - felem_square_reduce(ftmp, in); /* 2^1 */ - felem_mul_reduce(ftmp, ftmp, in); /* 2^1 + 2^0 */ - felem_assign(ftmp2, ftmp); - - felem_square_reduce(ftmp, ftmp); /* 2^2 + 2^1 */ - felem_mul_reduce(ftmp, ftmp, in); /* 2^2 + 2^1 * 2^0 */ - felem_assign(ftmp3, ftmp); - - for (i = 0; i < 3; i++) - felem_square_reduce(ftmp, ftmp); /* 2^5 + 2^4 + 2^3 */ - felem_mul_reduce(ftmp, ftmp3, ftmp); /* 2^5 + 2^4 + 2^3 + 2^2 + 2^1 + 2^0 */ - felem_assign(ftmp4, ftmp); - - for (i = 0; i < 6; i++) - felem_square_reduce(ftmp, ftmp); /* 2^11 + ... + 2^6 */ - felem_mul_reduce(ftmp, ftmp4, ftmp); /* 2^11 + ... + 2^0 */ - - for (i = 0; i < 3; i++) - felem_square_reduce(ftmp, ftmp); /* 2^14 + ... + 2^3 */ - felem_mul_reduce(ftmp, ftmp3, ftmp); /* 2^14 + ... + 2^0 */ - felem_assign(ftmp5, ftmp); - - for (i = 0; i < 15; i++) - felem_square_reduce(ftmp, ftmp); /* 2^29 + ... + 2^15 */ - felem_mul_reduce(ftmp, ftmp5, ftmp); /* 2^29 + ... + 2^0 */ - felem_assign(ftmp6, ftmp); - - for (i = 0; i < 30; i++) - felem_square_reduce(ftmp, ftmp); /* 2^59 + ... + 2^30 */ - felem_mul_reduce(ftmp, ftmp6, ftmp); /* 2^59 + ... + 2^0 */ - felem_assign(ftmp4, ftmp); - - for (i = 0; i < 60; i++) - felem_square_reduce(ftmp, ftmp); /* 2^119 + ... + 2^60 */ - felem_mul_reduce(ftmp, ftmp4, ftmp); /* 2^119 + ... + 2^0 */ - felem_assign(ftmp4, ftmp); - - for (i = 0; i < 120; i++) - felem_square_reduce(ftmp, ftmp); /* 2^239 + ... + 2^120 */ - felem_mul_reduce(ftmp, ftmp4, ftmp); /* 2^239 + ... + 2^0 */ - - for (i = 0; i < 15; i++) - felem_square_reduce(ftmp, ftmp); /* 2^254 + ... + 2^15 */ - felem_mul_reduce(ftmp, ftmp5, ftmp); /* 2^254 + ... + 2^0 */ - - for (i = 0; i < 31; i++) - felem_square_reduce(ftmp, ftmp); /* 2^285 + ... + 2^31 */ - felem_mul_reduce(ftmp, ftmp6, ftmp); /* 2^285 + ... + 2^31 + 2^29 + ... + 2^0 */ - - for (i = 0; i < 2; i++) - felem_square_reduce(ftmp, ftmp); /* 2^287 + ... + 2^33 + 2^31 + ... + 2^2 */ - felem_mul_reduce(ftmp, ftmp2, ftmp); /* 2^287 + ... + 2^33 + 2^31 + ... + 2^0 */ - - for (i = 0; i < 94; i++) - felem_square_reduce(ftmp, ftmp); /* 2^381 + ... + 2^127 + 2^125 + ... + 2^94 */ - felem_mul_reduce(ftmp, ftmp6, ftmp); /* 2^381 + ... + 2^127 + 2^125 + ... + 2^94 + 2^29 + ... + 2^0 */ - - for (i = 0; i < 2; i++) - felem_square_reduce(ftmp, ftmp); /* 2^383 + ... + 2^129 + 2^127 + ... + 2^96 + 2^31 + ... + 2^2 */ - felem_mul_reduce(ftmp, in, ftmp); /* 2^383 + ... + 2^129 + 2^127 + ... + 2^96 + 2^31 + ... + 2^2 + 2^0 */ - - memcpy(out, ftmp, sizeof(felem)); -} - -/* - * Zero-check: returns a limb with all bits set if |in| == 0 (mod p) - * and 0 otherwise. We know that field elements are reduced to - * 0 < in < 2p, so we only need to check two cases: - * 0 and 2^384 - 2^128 - 2^96 + 2^32 - 1 - * in[k] < 2^56, k < 6 - * in[6] <= 2^48 - */ -static limb felem_is_zero(const felem in) -{ - limb zero, p384; - - zero = in[0] | in[1] | in[2] | in[3] | in[4] | in[5] | in[6]; - zero = ((int64_t) (zero) - 1) >> 63; - p384 = (in[0] ^ 0x000000ffffffff) | (in[1] ^ 0xffff0000000000) - | (in[2] ^ 0xfffffffffeffff) | (in[3] ^ 0xffffffffffffff) - | (in[4] ^ 0xffffffffffffff) | (in[5] ^ 0xffffffffffffff) - | (in[6] ^ 0xffffffffffff); - p384 = ((int64_t) (p384) - 1) >> 63; - - return (zero | p384); -} - -static int felem_is_zero_int(const void *in) -{ - return (int)(felem_is_zero(in) & ((limb) 1)); -} - -/*- - * felem_contract converts |in| to its unique, minimal representation. - * Assume we've removed all redundant bits. - * On entry: - * in[k] < 2^56, k < 6 - * in[6] <= 2^48 - */ -static void felem_contract(felem out, const felem in) -{ - static const int64_t two56 = ((limb) 1) << 56; - - /* - * We know for a fact that 0 <= |in| < 2*p, for p = 2^384 - 2^128 - 2^96 + 2^32 - 1 - * Perform two successive, idempotent subtractions to reduce if |in| >= p. - */ - - int64_t tmp[NLIMBS], cond[5], a; - unsigned int i; - - memcpy(tmp, in, sizeof(felem)); - - /* Case 1: a = 1 iff |in| >= 2^384 */ - a = (in[6] >> 48); - tmp[0] += a; - tmp[0] -= a << 32; - tmp[1] += a << 40; - tmp[2] += a << 16; - tmp[6] &= 0x0000ffffffffffff; - - /* - * eliminate negative coefficients: if tmp[0] is negative, tmp[1] must be - * non-zero, so we only need one step - */ - - a = tmp[0] >> 63; - tmp[0] += a & two56; - tmp[1] -= a & 1; - - /* Carry 1 -> 2 -> 3 -> 4 -> 5 -> 6 */ - tmp[2] += tmp[1] >> 56; - tmp[1] &= 0x00ffffffffffffff; - - tmp[3] += tmp[2] >> 56; - tmp[2] &= 0x00ffffffffffffff; - - tmp[4] += tmp[3] >> 56; - tmp[3] &= 0x00ffffffffffffff; - - tmp[5] += tmp[4] >> 56; - tmp[4] &= 0x00ffffffffffffff; - - tmp[6] += tmp[5] >> 56; /* tmp[6] < 2^48 */ - tmp[5] &= 0x00ffffffffffffff; - - /* - * Case 2: a = all ones if p <= |in| < 2^384, 0 otherwise - */ - - /* 0 iff (2^129..2^383) are all one */ - cond[0] = ((tmp[6] | 0xff000000000000) & tmp[5] & tmp[4] & tmp[3] & (tmp[2] | 0x0000000001ffff)) + 1; - /* 0 iff 2^128 bit is one */ - cond[1] = (tmp[2] | ~0x00000000010000) + 1; - /* 0 iff (2^96..2^127) bits are all one */ - cond[2] = ((tmp[2] | 0xffffffffff0000) & (tmp[1] | 0x0000ffffffffff)) + 1; - /* 0 iff (2^32..2^95) bits are all zero */ - cond[3] = (tmp[1] & ~0xffff0000000000) | (tmp[0] & ~((int64_t) 0x000000ffffffff)); - /* 0 iff (2^0..2^31) bits are all one */ - cond[4] = (tmp[0] | 0xffffff00000000) + 1; - - /* - * In effect, invert our conditions, so that 0 values become all 1's, - * any non-zero value in the low-order 56 bits becomes all 0's - */ - for (i = 0; i < 5; i++) - cond[i] = ((cond[i] & 0x00ffffffffffffff) - 1) >> 63; - - /* - * The condition for determining whether in is greater than our - * prime is given by the following condition. - */ - - /* First subtract 2^384 - 2^129 cheaply */ - a = cond[0] & (cond[1] | (cond[2] & (~cond[3] | cond[4]))); - tmp[6] &= ~a; - tmp[5] &= ~a; - tmp[4] &= ~a; - tmp[3] &= ~a; - tmp[2] &= ~a | 0x0000000001ffff; - - /* - * Subtract 2^128 - 2^96 by - * means of disjoint cases. - */ - - /* subtract 2^128 if that bit is present, and add 2^96 */ - a = cond[0] & cond[1]; - tmp[2] &= ~a | 0xfffffffffeffff; - tmp[1] += a & ((int64_t) 1 << 40); - - /* otherwise, clear bits 2^127 .. 2^96 */ - a = cond[0] & ~cond[1] & (cond[2] & (~cond[3] | cond[4])); - tmp[2] &= ~a | 0xffffffffff0000; - tmp[1] &= ~a | 0x0000ffffffffff; - - /* finally, subtract the last 2^32 - 1 */ - a = cond[0] & (cond[1] | (cond[2] & (~cond[3] | cond[4]))); - tmp[0] += a & (-((int64_t) 1 << 32) + 1); - - /* - * eliminate negative coefficients: if tmp[0] is negative, tmp[1] must be - * non-zero, so we only need one step - */ - a = tmp[0] >> 63; - tmp[0] += a & two56; - tmp[1] -= a & 1; - - /* Carry 1 -> 2 -> 3 -> 4 -> 5 -> 6 */ - tmp[2] += tmp[1] >> 56; - tmp[1] &= 0x00ffffffffffffff; - - tmp[3] += tmp[2] >> 56; - tmp[2] &= 0x00ffffffffffffff; - - tmp[4] += tmp[3] >> 56; - tmp[3] &= 0x00ffffffffffffff; - - tmp[5] += tmp[4] >> 56; - tmp[4] &= 0x00ffffffffffffff; - - tmp[6] += tmp[5] >> 56; - tmp[5] &= 0x00ffffffffffffff; - - memcpy(out, tmp, sizeof(felem)); -} - -/*- - * Group operations - * ---------------- - * - * Building on top of the field operations we have the operations on the - * elliptic curve group itself. Points on the curve are represented in Jacobian - * coordinates - */ - -/*- - * point_double calculates 2*(x_in, y_in, z_in) - * - * The method is taken from: - * http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b - * - * Outputs can equal corresponding inputs, i.e., x_out == x_in is allowed. - * while x_out == y_in is not (maybe this works, but it's not tested). - */ -static void -point_double(felem x_out, felem y_out, felem z_out, - const felem x_in, const felem y_in, const felem z_in) -{ - widefelem tmp, tmp2; - felem delta, gamma, beta, alpha, ftmp, ftmp2; - - felem_assign(ftmp, x_in); - felem_assign(ftmp2, x_in); - - /* delta = z^2 */ - felem_square_reduce(delta, z_in); /* delta[i] < 2^56 */ - - /* gamma = y^2 */ - felem_square_reduce(gamma, y_in); /* gamma[i] < 2^56 */ - - /* beta = x*gamma */ - felem_mul_reduce(beta, x_in, gamma); /* beta[i] < 2^56 */ - - /* alpha = 3*(x-delta)*(x+delta) */ - felem_diff64(ftmp, delta); /* ftmp[i] < 2^60 + 2^58 + 2^44 */ - felem_sum64(ftmp2, delta); /* ftmp2[i] < 2^59 */ - felem_scalar64(ftmp2, 3); /* ftmp2[i] < 2^61 */ - felem_mul_reduce(alpha, ftmp, ftmp2); /* alpha[i] < 2^56 */ - - /* x' = alpha^2 - 8*beta */ - felem_square(tmp, alpha); /* tmp[i] < 2^115 */ - felem_assign(ftmp, beta); /* ftmp[i] < 2^56 */ - felem_scalar64(ftmp, 8); /* ftmp[i] < 2^59 */ - felem_diff_128_64(tmp, ftmp); /* tmp[i] < 2^115 + 2^64 + 2^48 */ - felem_reduce(x_out, tmp); /* x_out[i] < 2^56 */ - - /* z' = (y + z)^2 - gamma - delta */ - felem_sum64(delta, gamma); /* delta[i] < 2^57 */ - felem_assign(ftmp, y_in); /* ftmp[i] < 2^56 */ - felem_sum64(ftmp, z_in); /* ftmp[i] < 2^56 */ - felem_square(tmp, ftmp); /* tmp[i] < 2^115 */ - felem_diff_128_64(tmp, delta); /* tmp[i] < 2^115 + 2^64 + 2^48 */ - felem_reduce(z_out, tmp); /* z_out[i] < 2^56 */ - - /* y' = alpha*(4*beta - x') - 8*gamma^2 */ - felem_scalar64(beta, 4); /* beta[i] < 2^58 */ - felem_diff64(beta, x_out); /* beta[i] < 2^60 + 2^58 + 2^44 */ - felem_mul(tmp, alpha, beta); /* tmp[i] < 2^119 */ - felem_square(tmp2, gamma); /* tmp2[i] < 2^115 */ - felem_scalar128(tmp2, 8); /* tmp2[i] < 2^118 */ - felem_diff128(tmp, tmp2); /* tmp[i] < 2^127 + 2^119 + 2^111 */ - felem_reduce(y_out, tmp); /* tmp[i] < 2^56 */ -} - -/* copy_conditional copies in to out iff mask is all ones. */ -static void copy_conditional(felem out, const felem in, limb mask) -{ - unsigned int i; - - for (i = 0; i < NLIMBS; i++) - out[i] ^= mask & (in[i] ^ out[i]); -} - -/*- - * point_add calculates (x1, y1, z1) + (x2, y2, z2) - * - * The method is taken from - * http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl, - * adapted for mixed addition (z2 = 1, or z2 = 0 for the point at infinity). - * - * This function includes a branch for checking whether the two input points - * are equal (while not equal to the point at infinity). See comment below - * on constant-time. - */ -static void point_add(felem x3, felem y3, felem z3, - const felem x1, const felem y1, const felem z1, - const int mixed, const felem x2, const felem y2, - const felem z2) -{ - felem ftmp, ftmp2, ftmp3, ftmp4, ftmp5, ftmp6, x_out, y_out, z_out; - widefelem tmp, tmp2; - limb x_equal, y_equal, z1_is_zero, z2_is_zero; - limb points_equal; - - z1_is_zero = felem_is_zero(z1); - z2_is_zero = felem_is_zero(z2); - - /* ftmp = z1z1 = z1**2 */ - felem_square_reduce(ftmp, z1); /* ftmp[i] < 2^56 */ - - if (!mixed) { - /* ftmp2 = z2z2 = z2**2 */ - felem_square_reduce(ftmp2, z2); /* ftmp2[i] < 2^56 */ - - /* u1 = ftmp3 = x1*z2z2 */ - felem_mul_reduce(ftmp3, x1, ftmp2); /* ftmp3[i] < 2^56 */ - - /* ftmp5 = z1 + z2 */ - felem_assign(ftmp5, z1); /* ftmp5[i] < 2^56 */ - felem_sum64(ftmp5, z2); /* ftmp5[i] < 2^57 */ - - /* ftmp5 = (z1 + z2)**2 - z1z1 - z2z2 = 2*z1z2 */ - felem_square(tmp, ftmp5); /* tmp[i] < 2^117 */ - felem_diff_128_64(tmp, ftmp); /* tmp[i] < 2^117 + 2^64 + 2^48 */ - felem_diff_128_64(tmp, ftmp2); /* tmp[i] < 2^117 + 2^65 + 2^49 */ - felem_reduce(ftmp5, tmp); /* ftmp5[i] < 2^56 */ - - /* ftmp2 = z2 * z2z2 */ - felem_mul_reduce(ftmp2, ftmp2, z2); /* ftmp2[i] < 2^56 */ - - /* s1 = ftmp6 = y1 * z2**3 */ - felem_mul_reduce(ftmp6, y1, ftmp2); /* ftmp6[i] < 2^56 */ - } else { - /* - * We'll assume z2 = 1 (special case z2 = 0 is handled later) - */ - - /* u1 = ftmp3 = x1*z2z2 */ - felem_assign(ftmp3, x1); /* ftmp3[i] < 2^56 */ - - /* ftmp5 = 2*z1z2 */ - felem_scalar(ftmp5, z1, 2); /* ftmp5[i] < 2^57 */ - - /* s1 = ftmp6 = y1 * z2**3 */ - felem_assign(ftmp6, y1); /* ftmp6[i] < 2^56 */ - } - /* ftmp3[i] < 2^56, ftmp5[i] < 2^57, ftmp6[i] < 2^56 */ - - /* u2 = x2*z1z1 */ - felem_mul(tmp, x2, ftmp); /* tmp[i] < 2^115 */ - - /* h = ftmp4 = u2 - u1 */ - felem_diff_128_64(tmp, ftmp3); /* tmp[i] < 2^115 + 2^64 + 2^48 */ - felem_reduce(ftmp4, tmp); /* ftmp[4] < 2^56 */ - - x_equal = felem_is_zero(ftmp4); - - /* z_out = ftmp5 * h */ - felem_mul_reduce(z_out, ftmp5, ftmp4); /* z_out[i] < 2^56 */ - - /* ftmp = z1 * z1z1 */ - felem_mul_reduce(ftmp, ftmp, z1); /* ftmp[i] < 2^56 */ - - /* s2 = tmp = y2 * z1**3 */ - felem_mul(tmp, y2, ftmp); /* tmp[i] < 2^115 */ - - /* r = ftmp5 = (s2 - s1)*2 */ - felem_diff_128_64(tmp, ftmp6); /* tmp[i] < 2^115 + 2^64 + 2^48 */ - felem_reduce(ftmp5, tmp); /* ftmp5[i] < 2^56 */ - y_equal = felem_is_zero(ftmp5); - felem_scalar64(ftmp5, 2); /* ftmp5[i] < 2^57 */ - - /* - * The formulae are incorrect if the points are equal, in affine coordinates - * (X_1, Y_1) == (X_2, Y_2), so we check for this and do doubling if this - * happens. - * - * We use bitwise operations to avoid potential side-channels introduced by - * the short-circuiting behaviour of boolean operators. - * - * The special case of either point being the point at infinity (z1 and/or - * z2 are zero), is handled separately later on in this function, so we - * avoid jumping to point_double here in those special cases. - * - * Notice the comment below on the implications of this branching for timing - * leaks and why it is considered practically irrelevant. - */ - points_equal = (x_equal & y_equal & (~z1_is_zero) & (~z2_is_zero)); - - if (points_equal) { - /* - * This is obviously not constant-time but it will almost-never happen - * for ECDH / ECDSA. - */ - point_double(x3, y3, z3, x1, y1, z1); - return; - } - - /* I = ftmp = (2h)**2 */ - felem_assign(ftmp, ftmp4); /* ftmp[i] < 2^56 */ - felem_scalar64(ftmp, 2); /* ftmp[i] < 2^57 */ - felem_square_reduce(ftmp, ftmp); /* ftmp[i] < 2^56 */ - - /* J = ftmp2 = h * I */ - felem_mul_reduce(ftmp2, ftmp4, ftmp); /* ftmp2[i] < 2^56 */ - - /* V = ftmp4 = U1 * I */ - felem_mul_reduce(ftmp4, ftmp3, ftmp); /* ftmp4[i] < 2^56 */ - - /* x_out = r**2 - J - 2V */ - felem_square(tmp, ftmp5); /* tmp[i] < 2^117 */ - felem_diff_128_64(tmp, ftmp2); /* tmp[i] < 2^117 + 2^64 + 2^48 */ - felem_assign(ftmp3, ftmp4); /* ftmp3[i] < 2^56 */ - felem_scalar64(ftmp4, 2); /* ftmp4[i] < 2^57 */ - felem_diff_128_64(tmp, ftmp4); /* tmp[i] < 2^117 + 2^65 + 2^49 */ - felem_reduce(x_out, tmp); /* x_out[i] < 2^56 */ - - /* y_out = r(V-x_out) - 2 * s1 * J */ - felem_diff64(ftmp3, x_out); /* ftmp3[i] < 2^60 + 2^56 + 2^44 */ - felem_mul(tmp, ftmp5, ftmp3); /* tmp[i] < 2^116 */ - felem_mul(tmp2, ftmp6, ftmp2); /* tmp2[i] < 2^115 */ - felem_scalar128(tmp2, 2); /* tmp2[i] < 2^116 */ - felem_diff128(tmp, tmp2); /* tmp[i] < 2^127 + 2^116 + 2^111 */ - felem_reduce(y_out, tmp); /* y_out[i] < 2^56 */ - - copy_conditional(x_out, x2, z1_is_zero); - copy_conditional(x_out, x1, z2_is_zero); - copy_conditional(y_out, y2, z1_is_zero); - copy_conditional(y_out, y1, z2_is_zero); - copy_conditional(z_out, z2, z1_is_zero); - copy_conditional(z_out, z1, z2_is_zero); - felem_assign(x3, x_out); - felem_assign(y3, y_out); - felem_assign(z3, z_out); -} - -/*- - * Base point pre computation - * -------------------------- - * - * Two different sorts of precomputed tables are used in the following code. - * Each contain various points on the curve, where each point is three field - * elements (x, y, z). - * - * For the base point table, z is usually 1 (0 for the point at infinity). - * This table has 16 elements: - * index | bits | point - * ------+---------+------------------------------ - * 0 | 0 0 0 0 | 0G - * 1 | 0 0 0 1 | 1G - * 2 | 0 0 1 0 | 2^95G - * 3 | 0 0 1 1 | (2^95 + 1)G - * 4 | 0 1 0 0 | 2^190G - * 5 | 0 1 0 1 | (2^190 + 1)G - * 6 | 0 1 1 0 | (2^190 + 2^95)G - * 7 | 0 1 1 1 | (2^190 + 2^95 + 1)G - * 8 | 1 0 0 0 | 2^285G - * 9 | 1 0 0 1 | (2^285 + 1)G - * 10 | 1 0 1 0 | (2^285 + 2^95)G - * 11 | 1 0 1 1 | (2^285 + 2^95 + 1)G - * 12 | 1 1 0 0 | (2^285 + 2^190)G - * 13 | 1 1 0 1 | (2^285 + 2^190 + 1)G - * 14 | 1 1 1 0 | (2^285 + 2^190 + 2^95)G - * 15 | 1 1 1 1 | (2^285 + 2^190 + 2^95 + 1)G - * - * The reason for this is so that we can clock bits into four different - * locations when doing simple scalar multiplies against the base point. - * - * Tables for other points have table[i] = iG for i in 0 .. 16. - */ - -/* gmul is the table of precomputed base points */ -static const felem gmul[16][3] = { -{{0, 0, 0, 0, 0, 0, 0}, - {0, 0, 0, 0, 0, 0, 0}, - {0, 0, 0, 0, 0, 0, 0}}, -{{0x00545e3872760ab7, 0x00f25dbf55296c3a, 0x00e082542a385502, 0x008ba79b9859f741, - 0x0020ad746e1d3b62, 0x0005378eb1c71ef3, 0x0000aa87ca22be8b}, - {0x00431d7c90ea0e5f, 0x00b1ce1d7e819d7a, 0x0013b5f0b8c00a60, 0x00289a147ce9da31, - 0x0092dc29f8f41dbd, 0x002c6f5d9e98bf92, 0x00003617de4a9626}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x00024711cc902a90, 0x00acb2e579ab4fe1, 0x00af818a4b4d57b1, 0x00a17c7bec49c3de, - 0x004280482d726a8b, 0x00128dd0f0a90f3b, 0x00004387c1c3fa3c}, - {0x002ce76543cf5c3a, 0x00de6cee5ef58f0a, 0x00403e42fa561ca6, 0x00bc54d6f9cb9731, - 0x007155f925fb4ff1, 0x004a9ce731b7b9bc, 0x00002609076bd7b2}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x00e74c9182f0251d, 0x0039bf54bb111974, 0x00b9d2f2eec511d2, 0x0036b1594eb3a6a4, - 0x00ac3bb82d9d564b, 0x00f9313f4615a100, 0x00006716a9a91b10}, - {0x0046698116e2f15c, 0x00f34347067d3d33, 0x008de4ccfdebd002, 0x00e838c6b8e8c97b, - 0x006faf0798def346, 0x007349794a57563c, 0x00002629e7e6ad84}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x0075300e34fd163b, 0x0092e9db4e8d0ad3, 0x00254be9f625f760, 0x00512c518c72ae68, - 0x009bfcf162bede5a, 0x00bf9341566ce311, 0x0000cd6175bd41cf}, - {0x007dfe52af4ac70f, 0x0002159d2d5c4880, 0x00b504d16f0af8d0, 0x0014585e11f5e64c, - 0x0089c6388e030967, 0x00ffb270cbfa5f71, 0x00009a15d92c3947}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x0033fc1278dc4fe5, 0x00d53088c2caa043, 0x0085558827e2db66, 0x00c192bef387b736, - 0x00df6405a2225f2c, 0x0075205aa90fd91a, 0x0000137e3f12349d}, - {0x00ce5b115efcb07e, 0x00abc3308410deeb, 0x005dc6fc1de39904, 0x00907c1c496f36b4, - 0x0008e6ad3926cbe1, 0x00110747b787928c, 0x0000021b9162eb7e}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x008180042cfa26e1, 0x007b826a96254967, 0x0082473694d6b194, 0x007bd6880a45b589, - 0x00c0a5097072d1a3, 0x0019186555e18b4e, 0x000020278190e5ca}, - {0x00b4bef17de61ac0, 0x009535e3c38ed348, 0x002d4aa8e468ceab, 0x00ef40b431036ad3, - 0x00defd52f4542857, 0x0086edbf98234266, 0x00002025b3a7814d}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x00b238aa97b886be, 0x00ef3192d6dd3a32, 0x0079f9e01fd62df8, 0x00742e890daba6c5, - 0x008e5289144408ce, 0x0073bbcc8e0171a5, 0x0000c4fd329d3b52}, - {0x00c6f64a15ee23e7, 0x00dcfb7b171cad8b, 0x00039f6cbd805867, 0x00de024e428d4562, - 0x00be6a594d7c64c5, 0x0078467b70dbcd64, 0x0000251f2ed7079b}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x000e5cc25fc4b872, 0x005ebf10d31ef4e1, 0x0061e0ebd11e8256, 0x0076e026096f5a27, - 0x0013e6fc44662e9a, 0x0042b00289d3597e, 0x000024f089170d88}, - {0x001604d7e0effbe6, 0x0048d77cba64ec2c, 0x008166b16da19e36, 0x006b0d1a0f28c088, - 0x000259fcd47754fd, 0x00cc643e4d725f9a, 0x00007b10f3c79c14}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x00430155e3b908af, 0x00b801e4fec25226, 0x00b0d4bcfe806d26, 0x009fc4014eb13d37, - 0x0066c94e44ec07e8, 0x00d16adc03874ba2, 0x000030c917a0d2a7}, - {0x00edac9e21eb891c, 0x00ef0fb768102eff, 0x00c088cef272a5f3, 0x00cbf782134e2964, - 0x0001044a7ba9a0e3, 0x00e363f5b194cf3c, 0x00009ce85249e372}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x001dd492dda5a7eb, 0x008fd577be539fd1, 0x002ff4b25a5fc3f1, 0x0074a8a1b64df72f, - 0x002ba3d8c204a76c, 0x009d5cff95c8235a, 0x0000e014b9406e0f}, - {0x008c2e4dbfc98aba, 0x00f30bb89f1a1436, 0x00b46f7aea3e259c, 0x009224454ac02f54, - 0x00906401f5645fa2, 0x003a1d1940eabc77, 0x00007c9351d680e6}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x005a35d872ef967c, 0x0049f1b7884e1987, 0x0059d46d7e31f552, 0x00ceb4869d2d0fb6, - 0x00e8e89eee56802a, 0x0049d806a774aaf2, 0x0000147e2af0ae24}, - {0x005fd1bd852c6e5e, 0x00b674b7b3de6885, 0x003b9ea5eb9b6c08, 0x005c9f03babf3ef7, - 0x00605337fecab3c7, 0x009a3f85b11bbcc8, 0x0000455470f330ec}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x002197ff4d55498d, 0x00383e8916c2d8af, 0x00eb203f34d1c6d2, 0x0080367cbd11b542, - 0x00769b3be864e4f5, 0x0081a8458521c7bb, 0x0000c531b34d3539}, - {0x00e2a3d775fa2e13, 0x00534fc379573844, 0x00ff237d2a8db54a, 0x00d301b2335a8882, - 0x000f75ea96103a80, 0x0018fecb3cdd96fa, 0x0000304bf61e94eb}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x00b2afc332a73dbd, 0x0029a0d5bb007bc5, 0x002d628eb210f577, 0x009f59a36dd05f50, - 0x006d339de4eca613, 0x00c75a71addc86bc, 0x000060384c5ea93c}, - {0x00aa9641c32a30b4, 0x00cc73ae8cce565d, 0x00ec911a4df07f61, 0x00aa4b762ea4b264, - 0x0096d395bb393629, 0x004efacfb7632fe0, 0x00006f252f46fa3f}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x00567eec597c7af6, 0x0059ba6795204413, 0x00816d4e6f01196f, 0x004ae6b3eb57951d, - 0x00420f5abdda2108, 0x003401d1f57ca9d9, 0x0000cf5837b0b67a}, - {0x00eaa64b8aeeabf9, 0x00246ddf16bcb4de, 0x000e7e3c3aecd751, 0x0008449f04fed72e, - 0x00307b67ccf09183, 0x0017108c3556b7b1, 0x0000229b2483b3bf}, - {1, 0, 0, 0, 0, 0, 0}}, -{{0x00e7c491a7bb78a1, 0x00eafddd1d3049ab, 0x00352c05e2bc7c98, 0x003d6880c165fa5c, - 0x00b6ac61cc11c97d, 0x00beeb54fcf90ce5, 0x0000dc1f0b455edc}, - {0x002db2e7aee34d60, 0x0073b5f415a2d8c0, 0x00dd84e4193e9a0c, 0x00d02d873467c572, - 0x0018baaeda60aee5, 0x0013fb11d697c61e, 0x000083aafcc3a973}, - {1, 0, 0, 0, 0, 0, 0}} -}; - -/* - * select_point selects the |idx|th point from a precomputation table and - * copies it to out. - * - * pre_comp below is of the size provided in |size|. - */ -static void select_point(const limb idx, unsigned int size, - const felem pre_comp[][3], felem out[3]) -{ - unsigned int i, j; - limb *outlimbs = &out[0][0]; - - memset(out, 0, sizeof(*out) * 3); - - for (i = 0; i < size; i++) { - const limb *inlimbs = &pre_comp[i][0][0]; - limb mask = i ^ idx; - - mask |= mask >> 4; - mask |= mask >> 2; - mask |= mask >> 1; - mask &= 1; - mask--; - for (j = 0; j < NLIMBS * 3; j++) - outlimbs[j] |= inlimbs[j] & mask; - } -} - -/* get_bit returns the |i|th bit in |in| */ -static char get_bit(const felem_bytearray in, int i) -{ - if (i < 0 || i >= 384) - return 0; - return (in[i >> 3] >> (i & 7)) & 1; -} - -/* - * Interleaved point multiplication using precomputed point multiples: The - * small point multiples 0*P, 1*P, ..., 16*P are in pre_comp[], the scalars - * in scalars[]. If g_scalar is non-NULL, we also add this multiple of the - * generator, using certain (large) precomputed multiples in g_pre_comp. - * Output point (X, Y, Z) is stored in x_out, y_out, z_out - */ -static void batch_mul(felem x_out, felem y_out, felem z_out, - const felem_bytearray scalars[], - const unsigned int num_points, const u8 *g_scalar, - const int mixed, const felem pre_comp[][17][3], - const felem g_pre_comp[16][3]) -{ - int i, skip; - unsigned int num, gen_mul = (g_scalar != NULL); - felem nq[3], tmp[4]; - limb bits; - u8 sign, digit; - - /* set nq to the point at infinity */ - memset(nq, 0, sizeof(nq)); - - /* - * Loop over all scalars msb-to-lsb, interleaving additions of multiples - * of the generator (last quarter of rounds) and additions of other - * points multiples (every 5th round). - */ - skip = 1; /* save two point operations in the first - * round */ - for (i = (num_points ? 380 : 98); i >= 0; --i) { - /* double */ - if (!skip) - point_double(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2]); - - /* add multiples of the generator */ - if (gen_mul && (i <= 98)) { - bits = get_bit(g_scalar, i + 285) << 3; - if (i < 95) { - bits |= get_bit(g_scalar, i + 190) << 2; - bits |= get_bit(g_scalar, i + 95) << 1; - bits |= get_bit(g_scalar, i); - } - /* select the point to add, in constant time */ - select_point(bits, 16, g_pre_comp, tmp); - if (!skip) { - /* The 1 argument below is for "mixed" */ - point_add(nq[0], nq[1], nq[2], - nq[0], nq[1], nq[2], 1, - tmp[0], tmp[1], tmp[2]); - } else { - memcpy(nq, tmp, 3 * sizeof(felem)); - skip = 0; - } - } - - /* do other additions every 5 doublings */ - if (num_points && (i % 5 == 0)) { - /* loop over all scalars */ - for (num = 0; num < num_points; ++num) { - bits = get_bit(scalars[num], i + 4) << 5; - bits |= get_bit(scalars[num], i + 3) << 4; - bits |= get_bit(scalars[num], i + 2) << 3; - bits |= get_bit(scalars[num], i + 1) << 2; - bits |= get_bit(scalars[num], i) << 1; - bits |= get_bit(scalars[num], i - 1); - ossl_ec_GFp_nistp_recode_scalar_bits(&sign, &digit, bits); - - /* - * select the point to add or subtract, in constant time - */ - select_point(digit, 17, pre_comp[num], tmp); - felem_neg(tmp[3], tmp[1]); /* (X, -Y, Z) is the negative - * point */ - copy_conditional(tmp[1], tmp[3], (-(limb) sign)); - - if (!skip) { - point_add(nq[0], nq[1], nq[2], - nq[0], nq[1], nq[2], mixed, - tmp[0], tmp[1], tmp[2]); - } else { - memcpy(nq, tmp, 3 * sizeof(felem)); - skip = 0; - } - } - } - } - felem_assign(x_out, nq[0]); - felem_assign(y_out, nq[1]); - felem_assign(z_out, nq[2]); -} - -/* Precomputation for the group generator. */ -struct nistp384_pre_comp_st { - felem g_pre_comp[16][3]; - CRYPTO_REF_COUNT references; -}; - -const EC_METHOD *ossl_ec_GFp_nistp384_method(void) -{ - static const EC_METHOD ret = { - EC_FLAGS_DEFAULT_OCT, - NID_X9_62_prime_field, - ossl_ec_GFp_nistp384_group_init, - ossl_ec_GFp_simple_group_finish, - ossl_ec_GFp_simple_group_clear_finish, - ossl_ec_GFp_nist_group_copy, - ossl_ec_GFp_nistp384_group_set_curve, - ossl_ec_GFp_simple_group_get_curve, - ossl_ec_GFp_simple_group_get_degree, - ossl_ec_group_simple_order_bits, - ossl_ec_GFp_simple_group_check_discriminant, - ossl_ec_GFp_simple_point_init, - ossl_ec_GFp_simple_point_finish, - ossl_ec_GFp_simple_point_clear_finish, - ossl_ec_GFp_simple_point_copy, - ossl_ec_GFp_simple_point_set_to_infinity, - ossl_ec_GFp_simple_point_set_affine_coordinates, - ossl_ec_GFp_nistp384_point_get_affine_coordinates, - 0, /* point_set_compressed_coordinates */ - 0, /* point2oct */ - 0, /* oct2point */ - ossl_ec_GFp_simple_add, - ossl_ec_GFp_simple_dbl, - ossl_ec_GFp_simple_invert, - ossl_ec_GFp_simple_is_at_infinity, - ossl_ec_GFp_simple_is_on_curve, - ossl_ec_GFp_simple_cmp, - ossl_ec_GFp_simple_make_affine, - ossl_ec_GFp_simple_points_make_affine, - ossl_ec_GFp_nistp384_points_mul, - ossl_ec_GFp_nistp384_precompute_mult, - ossl_ec_GFp_nistp384_have_precompute_mult, - ossl_ec_GFp_nist_field_mul, - ossl_ec_GFp_nist_field_sqr, - 0, /* field_div */ - ossl_ec_GFp_simple_field_inv, - 0, /* field_encode */ - 0, /* field_decode */ - 0, /* field_set_to_one */ - ossl_ec_key_simple_priv2oct, - ossl_ec_key_simple_oct2priv, - 0, /* set private */ - ossl_ec_key_simple_generate_key, - ossl_ec_key_simple_check_key, - ossl_ec_key_simple_generate_public_key, - 0, /* keycopy */ - 0, /* keyfinish */ - ossl_ecdh_simple_compute_key, - ossl_ecdsa_simple_sign_setup, - ossl_ecdsa_simple_sign_sig, - ossl_ecdsa_simple_verify_sig, - 0, /* field_inverse_mod_ord */ - 0, /* blind_coordinates */ - 0, /* ladder_pre */ - 0, /* ladder_step */ - 0 /* ladder_post */ - }; - - return &ret; -} - -/******************************************************************************/ -/* - * FUNCTIONS TO MANAGE PRECOMPUTATION - */ - -static NISTP384_PRE_COMP *nistp384_pre_comp_new(void) -{ - NISTP384_PRE_COMP *ret = OPENSSL_zalloc(sizeof(*ret)); - - if (ret == NULL) - return ret; - - if (!CRYPTO_NEW_REF(&ret->references, 1)) { - OPENSSL_free(ret); - return NULL; - } - return ret; -} - -NISTP384_PRE_COMP *ossl_ec_nistp384_pre_comp_dup(NISTP384_PRE_COMP *p) -{ - int i; - - if (p != NULL) - CRYPTO_UP_REF(&p->references, &i); - return p; -} - -void ossl_ec_nistp384_pre_comp_free(NISTP384_PRE_COMP *p) -{ - int i; - - if (p == NULL) - return; - - CRYPTO_DOWN_REF(&p->references, &i); - REF_PRINT_COUNT("ossl_ec_nistp384", p); - if (i > 0) - return; - REF_ASSERT_ISNT(i < 0); - - CRYPTO_FREE_REF(&p->references); - OPENSSL_free(p); -} - -/******************************************************************************/ -/* - * OPENSSL EC_METHOD FUNCTIONS - */ - -int ossl_ec_GFp_nistp384_group_init(EC_GROUP *group) -{ - int ret; - - ret = ossl_ec_GFp_simple_group_init(group); - group->a_is_minus3 = 1; - return ret; -} - -int ossl_ec_GFp_nistp384_group_set_curve(EC_GROUP *group, const BIGNUM *p, - const BIGNUM *a, const BIGNUM *b, - BN_CTX *ctx) -{ - int ret = 0; - BIGNUM *curve_p, *curve_a, *curve_b; -#ifndef FIPS_MODULE - BN_CTX *new_ctx = NULL; - - if (ctx == NULL) - ctx = new_ctx = BN_CTX_new(); -#endif - if (ctx == NULL) - return 0; - - BN_CTX_start(ctx); - curve_p = BN_CTX_get(ctx); - curve_a = BN_CTX_get(ctx); - curve_b = BN_CTX_get(ctx); - if (curve_b == NULL) - goto err; - BN_bin2bn(nistp384_curve_params[0], sizeof(felem_bytearray), curve_p); - BN_bin2bn(nistp384_curve_params[1], sizeof(felem_bytearray), curve_a); - BN_bin2bn(nistp384_curve_params[2], sizeof(felem_bytearray), curve_b); - if ((BN_cmp(curve_p, p)) || (BN_cmp(curve_a, a)) || (BN_cmp(curve_b, b))) { - ERR_raise(ERR_LIB_EC, EC_R_WRONG_CURVE_PARAMETERS); - goto err; - } - group->field_mod_func = BN_nist_mod_384; - ret = ossl_ec_GFp_simple_group_set_curve(group, p, a, b, ctx); - err: - BN_CTX_end(ctx); -#ifndef FIPS_MODULE - BN_CTX_free(new_ctx); -#endif - return ret; -} - -/* - * Takes the Jacobian coordinates (X, Y, Z) of a point and returns (X', Y') = - * (X/Z^2, Y/Z^3) - */ -int ossl_ec_GFp_nistp384_point_get_affine_coordinates(const EC_GROUP *group, - const EC_POINT *point, - BIGNUM *x, BIGNUM *y, - BN_CTX *ctx) -{ - felem z1, z2, x_in, y_in, x_out, y_out; - widefelem tmp; - - if (EC_POINT_is_at_infinity(group, point)) { - ERR_raise(ERR_LIB_EC, EC_R_POINT_AT_INFINITY); - return 0; - } - if ((!BN_to_felem(x_in, point->X)) || (!BN_to_felem(y_in, point->Y)) || - (!BN_to_felem(z1, point->Z))) - return 0; - felem_inv(z2, z1); - felem_square(tmp, z2); - felem_reduce(z1, tmp); - felem_mul(tmp, x_in, z1); - felem_reduce(x_in, tmp); - felem_contract(x_out, x_in); - if (x != NULL) { - if (!felem_to_BN(x, x_out)) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); - return 0; - } - } - felem_mul(tmp, z1, z2); - felem_reduce(z1, tmp); - felem_mul(tmp, y_in, z1); - felem_reduce(y_in, tmp); - felem_contract(y_out, y_in); - if (y != NULL) { - if (!felem_to_BN(y, y_out)) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); - return 0; - } - } - return 1; -} - -/* points below is of size |num|, and tmp_felems is of size |num+1/ */ -static void make_points_affine(size_t num, felem points[][3], - felem tmp_felems[]) -{ - /* - * Runs in constant time, unless an input is the point at infinity (which - * normally shouldn't happen). - */ - ossl_ec_GFp_nistp_points_make_affine_internal(num, - points, - sizeof(felem), - tmp_felems, - (void (*)(void *))felem_one, - felem_is_zero_int, - (void (*)(void *, const void *)) - felem_assign, - (void (*)(void *, const void *)) - felem_square_reduce, - (void (*)(void *, const void *, const void*)) - felem_mul_reduce, - (void (*)(void *, const void *)) - felem_inv, - (void (*)(void *, const void *)) - felem_contract); -} - -/* - * Computes scalar*generator + \sum scalars[i]*points[i], ignoring NULL - * values Result is stored in r (r can equal one of the inputs). - */ -int ossl_ec_GFp_nistp384_points_mul(const EC_GROUP *group, EC_POINT *r, - const BIGNUM *scalar, size_t num, - const EC_POINT *points[], - const BIGNUM *scalars[], BN_CTX *ctx) -{ - int ret = 0; - int j; - int mixed = 0; - BIGNUM *x, *y, *z, *tmp_scalar; - felem_bytearray g_secret; - felem_bytearray *secrets = NULL; - felem (*pre_comp)[17][3] = NULL; - felem *tmp_felems = NULL; - unsigned int i; - int num_bytes; - int have_pre_comp = 0; - size_t num_points = num; - felem x_in, y_in, z_in, x_out, y_out, z_out; - NISTP384_PRE_COMP *pre = NULL; - felem(*g_pre_comp)[3] = NULL; - EC_POINT *generator = NULL; - const EC_POINT *p = NULL; - const BIGNUM *p_scalar = NULL; - - BN_CTX_start(ctx); - x = BN_CTX_get(ctx); - y = BN_CTX_get(ctx); - z = BN_CTX_get(ctx); - tmp_scalar = BN_CTX_get(ctx); - if (tmp_scalar == NULL) - goto err; - - if (scalar != NULL) { - pre = group->pre_comp.nistp384; - if (pre) - /* we have precomputation, try to use it */ - g_pre_comp = &pre->g_pre_comp[0]; - else - /* try to use the standard precomputation */ - g_pre_comp = (felem(*)[3]) gmul; - generator = EC_POINT_new(group); - if (generator == NULL) - goto err; - /* get the generator from precomputation */ - if (!felem_to_BN(x, g_pre_comp[1][0]) || - !felem_to_BN(y, g_pre_comp[1][1]) || - !felem_to_BN(z, g_pre_comp[1][2])) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); - goto err; - } - if (!ossl_ec_GFp_simple_set_Jprojective_coordinates_GFp(group, - generator, - x, y, z, ctx)) - goto err; - if (0 == EC_POINT_cmp(group, generator, group->generator, ctx)) - /* precomputation matches generator */ - have_pre_comp = 1; - else - /* - * we don't have valid precomputation: treat the generator as a - * random point - */ - num_points++; - } - - if (num_points > 0) { - if (num_points >= 2) { - /* - * unless we precompute multiples for just one point, converting - * those into affine form is time well spent - */ - mixed = 1; - } - secrets = OPENSSL_zalloc(sizeof(*secrets) * num_points); - pre_comp = OPENSSL_zalloc(sizeof(*pre_comp) * num_points); - if (mixed) - tmp_felems = - OPENSSL_malloc(sizeof(*tmp_felems) * (num_points * 17 + 1)); - if ((secrets == NULL) || (pre_comp == NULL) - || (mixed && (tmp_felems == NULL))) - goto err; - - /* - * we treat NULL scalars as 0, and NULL points as points at infinity, - * i.e., they contribute nothing to the linear combination - */ - for (i = 0; i < num_points; ++i) { - if (i == num) { - /* - * we didn't have a valid precomputation, so we pick the - * generator - */ - p = EC_GROUP_get0_generator(group); - p_scalar = scalar; - } else { - /* the i^th point */ - p = points[i]; - p_scalar = scalars[i]; - } - if (p_scalar != NULL && p != NULL) { - /* reduce scalar to 0 <= scalar < 2^384 */ - if ((BN_num_bits(p_scalar) > 384) - || (BN_is_negative(p_scalar))) { - /* - * this is an unusual input, and we don't guarantee - * constant-timeness - */ - if (!BN_nnmod(tmp_scalar, p_scalar, group->order, ctx)) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); - goto err; - } - num_bytes = BN_bn2lebinpad(tmp_scalar, - secrets[i], sizeof(secrets[i])); - } else { - num_bytes = BN_bn2lebinpad(p_scalar, - secrets[i], sizeof(secrets[i])); - } - if (num_bytes < 0) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); - goto err; - } - /* precompute multiples */ - if ((!BN_to_felem(x_out, p->X)) || - (!BN_to_felem(y_out, p->Y)) || - (!BN_to_felem(z_out, p->Z))) - goto err; - memcpy(pre_comp[i][1][0], x_out, sizeof(felem)); - memcpy(pre_comp[i][1][1], y_out, sizeof(felem)); - memcpy(pre_comp[i][1][2], z_out, sizeof(felem)); - for (j = 2; j <= 16; ++j) { - if (j & 1) { - point_add(pre_comp[i][j][0], pre_comp[i][j][1], pre_comp[i][j][2], - pre_comp[i][1][0], pre_comp[i][1][1], pre_comp[i][1][2], 0, - pre_comp[i][j - 1][0], pre_comp[i][j - 1][1], pre_comp[i][j - 1][2]); - } else { - point_double(pre_comp[i][j][0], pre_comp[i][j][1], pre_comp[i][j][2], - pre_comp[i][j / 2][0], pre_comp[i][j / 2][1], pre_comp[i][j / 2][2]); - } - } - } - } - if (mixed) - make_points_affine(num_points * 17, pre_comp[0], tmp_felems); - } - - /* the scalar for the generator */ - if (scalar != NULL && have_pre_comp) { - memset(g_secret, 0, sizeof(g_secret)); - /* reduce scalar to 0 <= scalar < 2^384 */ - if ((BN_num_bits(scalar) > 384) || (BN_is_negative(scalar))) { - /* - * this is an unusual input, and we don't guarantee - * constant-timeness - */ - if (!BN_nnmod(tmp_scalar, scalar, group->order, ctx)) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); - goto err; - } - num_bytes = BN_bn2lebinpad(tmp_scalar, g_secret, sizeof(g_secret)); - } else { - num_bytes = BN_bn2lebinpad(scalar, g_secret, sizeof(g_secret)); - } - /* do the multiplication with generator precomputation */ - batch_mul(x_out, y_out, z_out, - (const felem_bytearray(*))secrets, num_points, - g_secret, - mixed, (const felem(*)[17][3])pre_comp, - (const felem(*)[3])g_pre_comp); - } else { - /* do the multiplication without generator precomputation */ - batch_mul(x_out, y_out, z_out, - (const felem_bytearray(*))secrets, num_points, - NULL, mixed, (const felem(*)[17][3])pre_comp, NULL); - } - /* reduce the output to its unique minimal representation */ - felem_contract(x_in, x_out); - felem_contract(y_in, y_out); - felem_contract(z_in, z_out); - if ((!felem_to_BN(x, x_in)) || (!felem_to_BN(y, y_in)) || - (!felem_to_BN(z, z_in))) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); - goto err; - } - ret = ossl_ec_GFp_simple_set_Jprojective_coordinates_GFp(group, r, x, y, z, - ctx); - - err: - BN_CTX_end(ctx); - EC_POINT_free(generator); - OPENSSL_free(secrets); - OPENSSL_free(pre_comp); - OPENSSL_free(tmp_felems); - return ret; -} - -int ossl_ec_GFp_nistp384_precompute_mult(EC_GROUP *group, BN_CTX *ctx) -{ - int ret = 0; - NISTP384_PRE_COMP *pre = NULL; - int i, j; - BIGNUM *x, *y; - EC_POINT *generator = NULL; - felem tmp_felems[16]; -#ifndef FIPS_MODULE - BN_CTX *new_ctx = NULL; -#endif - - /* throw away old precomputation */ - EC_pre_comp_free(group); - -#ifndef FIPS_MODULE - if (ctx == NULL) - ctx = new_ctx = BN_CTX_new(); -#endif - if (ctx == NULL) - return 0; - - BN_CTX_start(ctx); - x = BN_CTX_get(ctx); - y = BN_CTX_get(ctx); - if (y == NULL) - goto err; - /* get the generator */ - if (group->generator == NULL) - goto err; - generator = EC_POINT_new(group); - if (generator == NULL) - goto err; - BN_bin2bn(nistp384_curve_params[3], sizeof(felem_bytearray), x); - BN_bin2bn(nistp384_curve_params[4], sizeof(felem_bytearray), y); - if (!EC_POINT_set_affine_coordinates(group, generator, x, y, ctx)) - goto err; - if ((pre = nistp384_pre_comp_new()) == NULL) - goto err; - /* - * if the generator is the standard one, use built-in precomputation - */ - if (0 == EC_POINT_cmp(group, generator, group->generator, ctx)) { - memcpy(pre->g_pre_comp, gmul, sizeof(pre->g_pre_comp)); - goto done; - } - if ((!BN_to_felem(pre->g_pre_comp[1][0], group->generator->X)) || - (!BN_to_felem(pre->g_pre_comp[1][1], group->generator->Y)) || - (!BN_to_felem(pre->g_pre_comp[1][2], group->generator->Z))) - goto err; - /* compute 2^95*G, 2^190*G, 2^285*G */ - for (i = 1; i <= 4; i <<= 1) { - point_double(pre->g_pre_comp[2 * i][0], pre->g_pre_comp[2 * i][1], pre->g_pre_comp[2 * i][2], - pre->g_pre_comp[i][0], pre->g_pre_comp[i][1], pre->g_pre_comp[i][2]); - for (j = 0; j < 94; ++j) { - point_double(pre->g_pre_comp[2 * i][0], pre->g_pre_comp[2 * i][1], pre->g_pre_comp[2 * i][2], - pre->g_pre_comp[2 * i][0], pre->g_pre_comp[2 * i][1], pre->g_pre_comp[2 * i][2]); - } - } - /* g_pre_comp[0] is the point at infinity */ - memset(pre->g_pre_comp[0], 0, sizeof(pre->g_pre_comp[0])); - /* the remaining multiples */ - /* 2^95*G + 2^190*G */ - point_add(pre->g_pre_comp[6][0], pre->g_pre_comp[6][1], pre->g_pre_comp[6][2], - pre->g_pre_comp[4][0], pre->g_pre_comp[4][1], pre->g_pre_comp[4][2], 0, - pre->g_pre_comp[2][0], pre->g_pre_comp[2][1], pre->g_pre_comp[2][2]); - /* 2^95*G + 2^285*G */ - point_add(pre->g_pre_comp[10][0], pre->g_pre_comp[10][1], pre->g_pre_comp[10][2], - pre->g_pre_comp[8][0], pre->g_pre_comp[8][1], pre->g_pre_comp[8][2], 0, - pre->g_pre_comp[2][0], pre->g_pre_comp[2][1], pre->g_pre_comp[2][2]); - /* 2^190*G + 2^285*G */ - point_add(pre->g_pre_comp[12][0], pre->g_pre_comp[12][1], pre->g_pre_comp[12][2], - pre->g_pre_comp[8][0], pre->g_pre_comp[8][1], pre->g_pre_comp[8][2], 0, - pre->g_pre_comp[4][0], pre->g_pre_comp[4][1], pre->g_pre_comp[4][2]); - /* 2^95*G + 2^190*G + 2^285*G */ - point_add(pre->g_pre_comp[14][0], pre->g_pre_comp[14][1], pre->g_pre_comp[14][2], - pre->g_pre_comp[12][0], pre->g_pre_comp[12][1], pre->g_pre_comp[12][2], 0, - pre->g_pre_comp[2][0], pre->g_pre_comp[2][1], pre->g_pre_comp[2][2]); - for (i = 1; i < 8; ++i) { - /* odd multiples: add G */ - point_add(pre->g_pre_comp[2 * i + 1][0], pre->g_pre_comp[2 * i + 1][1], pre->g_pre_comp[2 * i + 1][2], - pre->g_pre_comp[2 * i][0], pre->g_pre_comp[2 * i][1], pre->g_pre_comp[2 * i][2], 0, - pre->g_pre_comp[1][0], pre->g_pre_comp[1][1], pre->g_pre_comp[1][2]); - } - make_points_affine(15, &(pre->g_pre_comp[1]), tmp_felems); - - done: - SETPRECOMP(group, nistp384, pre); - ret = 1; - pre = NULL; - err: - BN_CTX_end(ctx); - EC_POINT_free(generator); -#ifndef FIPS_MODULE - BN_CTX_free(new_ctx); -#endif - ossl_ec_nistp384_pre_comp_free(pre); - return ret; -} - -int ossl_ec_GFp_nistp384_have_precompute_mult(const EC_GROUP *group) -{ - return HAVEPRECOMP(group, nistp384); -} diff --git a/openssl/src/crypto/ec/ecp_nistp521.c b/openssl/src/crypto/ec/ecp_nistp521.c index db5a9dd5d..31a97d793 100644 --- a/openssl/src/crypto/ec/ecp_nistp521.c +++ b/openssl/src/crypto/ec/ecp_nistp521.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -676,8 +676,8 @@ static void felem_reduce(felem out, const largefelem in) } #if defined(ECP_NISTP521_ASM) -static void felem_square_wrapper(largefelem out, const felem in); -static void felem_mul_wrapper(largefelem out, const felem in1, const felem in2); +void felem_square_wrapper(largefelem out, const felem in); +void felem_mul_wrapper(largefelem out, const felem in1, const felem in2); static void (*felem_square_p)(largefelem out, const felem in) = felem_square_wrapper; @@ -691,7 +691,7 @@ void p521_felem_mul(largefelem out, const felem in1, const felem in2); # include "crypto/ppc_arch.h" # endif -static void felem_select(void) +void felem_select(void) { # if defined(_ARCH_PPC64) if ((OPENSSL_ppccap_P & PPC_MADD300) && (OPENSSL_ppccap_P & PPC_ALTIVEC)) { @@ -707,13 +707,13 @@ static void felem_select(void) felem_mul_p = felem_mul_ref; } -static void felem_square_wrapper(largefelem out, const felem in) +void felem_square_wrapper(largefelem out, const felem in) { felem_select(); felem_square_p(out, in); } -static void felem_mul_wrapper(largefelem out, const felem in1, const felem in2) +void felem_mul_wrapper(largefelem out, const felem in1, const felem in2) { felem_select(); felem_mul_p(out, in1, in2); @@ -782,6 +782,7 @@ static void felem_inv(felem out, const felem in) felem_reduce(ftmp3, tmp); /* 2^7 - 2^3 */ felem_square(tmp, ftmp3); felem_reduce(ftmp3, tmp); /* 2^8 - 2^4 */ + felem_assign(ftmp4, ftmp3); felem_mul(tmp, ftmp3, ftmp); felem_reduce(ftmp4, tmp); /* 2^8 - 2^1 */ felem_square(tmp, ftmp4); @@ -842,9 +843,9 @@ static void felem_inv(felem out, const felem in) felem_reduce(ftmp3, tmp); /* 2^521 - 2^9 */ } felem_mul(tmp, ftmp3, ftmp4); - felem_reduce(ftmp3, tmp); /* 2^521 - 2^2 */ + felem_reduce(ftmp3, tmp); /* 2^512 - 2^2 */ felem_mul(tmp, ftmp3, in); - felem_reduce(out, tmp); /* 2^521 - 3 */ + felem_reduce(out, tmp); /* 2^512 - 3 */ } /* This is 2^521-1, expressed as an felem */ @@ -1665,6 +1666,7 @@ static void batch_mul(felem x_out, felem y_out, felem z_out, struct nistp521_pre_comp_st { felem g_pre_comp[16][3]; CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; }; const EC_METHOD *EC_GFp_nistp521_method(void) @@ -1740,10 +1742,16 @@ static NISTP521_PRE_COMP *nistp521_pre_comp_new(void) { NISTP521_PRE_COMP *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return ret; + } + + ret->references = 1; - if (!CRYPTO_NEW_REF(&ret->references, 1)) { + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -1754,7 +1762,7 @@ NISTP521_PRE_COMP *EC_nistp521_pre_comp_dup(NISTP521_PRE_COMP *p) { int i; if (p != NULL) - CRYPTO_UP_REF(&p->references, &i); + CRYPTO_UP_REF(&p->references, &i, p->lock); return p; } @@ -1765,13 +1773,13 @@ void EC_nistp521_pre_comp_free(NISTP521_PRE_COMP *p) if (p == NULL) return; - CRYPTO_DOWN_REF(&p->references, &i); + CRYPTO_DOWN_REF(&p->references, &i, p->lock); REF_PRINT_COUNT("EC_nistp521", p); if (i > 0) return; REF_ASSERT_ISNT(i < 0); - CRYPTO_FREE_REF(&p->references); + CRYPTO_THREAD_lock_free(p->lock); OPENSSL_free(p); } @@ -1984,8 +1992,10 @@ int ossl_ec_GFp_nistp521_points_mul(const EC_GROUP *group, EC_POINT *r, tmp_felems = OPENSSL_malloc(sizeof(*tmp_felems) * (num_points * 17 + 1)); if ((secrets == NULL) || (pre_comp == NULL) - || (mixed && (tmp_felems == NULL))) + || (mixed && (tmp_felems == NULL))) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } /* * we treat NULL scalars as 0, and NULL points as points at infinity, diff --git a/openssl/src/crypto/ec/ecp_nistz256.c b/openssl/src/crypto/ec/ecp_nistz256.c index 5760639a2..d65f6984d 100644 --- a/openssl/src/crypto/ec/ecp_nistz256.c +++ b/openssl/src/crypto/ec/ecp_nistz256.c @@ -1,5 +1,5 @@ /* - * Copyright 2014-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2014, Intel Corporation. All Rights Reserved. * Copyright (c) 2015, CloudFlare, Inc. * @@ -37,6 +37,14 @@ # define TOBN(hi,lo) ((BN_ULONG)hi<<32|lo) #endif +#if defined(__GNUC__) +# define ALIGN32 __attribute((aligned(32))) +#elif defined(_MSC_VER) +# define ALIGN32 __declspec(align(32)) +#else +# define ALIGN32 +#endif + #define ALIGNPTR(p,N) ((unsigned char *)p+N-(size_t)p%N) #define P256_LIMBS (256/BN_BITS2) @@ -67,6 +75,7 @@ struct nistz256_pre_comp_st { PRECOMP256_ROW *precomp; void *precomp_storage; CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; }; /* Functions implemented in assembly */ @@ -627,8 +636,10 @@ __owur static int ecp_nistz256_windowed_mul(const EC_GROUP *group, OPENSSL_malloc((num * 16 + 5) * sizeof(P256_POINT) + 64)) == NULL || (p_str = OPENSSL_malloc(num * 33 * sizeof(unsigned char))) == NULL - || (scalars = OPENSSL_malloc(num * sizeof(BIGNUM *))) == NULL) + || (scalars = OPENSSL_malloc(num * sizeof(BIGNUM *))) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } table = (void *)ALIGNPTR(table_storage, 64); temp = (P256_POINT *)(table + num); @@ -857,8 +868,10 @@ __owur static int ecp_nistz256_mult_precompute(EC_GROUP *group, BN_CTX *ctx) w = 7; if ((precomp_storage = - OPENSSL_malloc(37 * 64 * sizeof(P256_POINT_AFFINE) + 64)) == NULL) + OPENSSL_malloc(37 * 64 * sizeof(P256_POINT_AFFINE) + 64)) == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } preComputedTable = (void *)ALIGNPTR(precomp_storage, 64); @@ -961,7 +974,7 @@ __owur static int ecp_nistz256_points_mul(const EC_GROUP *group, BIGNUM *tmp_scalar; if ((num + 1) == 0 || (num + 1) > OPENSSL_MALLOC_MAX_NELEMS(void *)) { - ERR_raise(ERR_LIB_EC, ERR_R_PASSED_INVALID_ARGUMENT); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } @@ -1110,12 +1123,16 @@ __owur static int ecp_nistz256_points_mul(const EC_GROUP *group, * handled like a normal point. */ new_scalars = OPENSSL_malloc((num + 1) * sizeof(BIGNUM *)); - if (new_scalars == NULL) + if (new_scalars == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } new_points = OPENSSL_malloc((num + 1) * sizeof(EC_POINT *)); - if (new_points == NULL) + if (new_points == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; + } memcpy(new_scalars, scalars, num * sizeof(BIGNUM *)); new_scalars[num] = scalar; @@ -1209,13 +1226,18 @@ static NISTZ256_PRE_COMP *ecp_nistz256_pre_comp_new(const EC_GROUP *group) ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return ret; + } ret->group = group; ret->w = 6; /* default */ + ret->references = 1; - if (!CRYPTO_NEW_REF(&ret->references, 1)) { + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -1226,7 +1248,7 @@ NISTZ256_PRE_COMP *EC_nistz256_pre_comp_dup(NISTZ256_PRE_COMP *p) { int i; if (p != NULL) - CRYPTO_UP_REF(&p->references, &i); + CRYPTO_UP_REF(&p->references, &i, p->lock); return p; } @@ -1237,14 +1259,14 @@ void EC_nistz256_pre_comp_free(NISTZ256_PRE_COMP *pre) if (pre == NULL) return; - CRYPTO_DOWN_REF(&pre->references, &i); + CRYPTO_DOWN_REF(&pre->references, &i, pre->lock); REF_PRINT_COUNT("EC_nistz256", pre); if (i > 0) return; REF_ASSERT_ISNT(i < 0); OPENSSL_free(pre->precomp_storage); - CRYPTO_FREE_REF(&pre->references); + CRYPTO_THREAD_lock_free(pre->lock); OPENSSL_free(pre); } @@ -1356,7 +1378,7 @@ static int ecp_nistz256_inv_mod_ord(const EC_GROUP *group, BIGNUM *r, /* * The bottom 128 bit of the exponent are processed with fixed 4-bit window */ - for (i = 0; i < 32; i++) { + for(i = 0; i < 32; i++) { /* expLo - the low 128 bits of the exponent we use (ord(p256) - 2), * split into nibbles */ static const unsigned char expLo[32] = { diff --git a/openssl/src/crypto/ec/ecp_s390x_nistp.c b/openssl/src/crypto/ec/ecp_s390x_nistp.c index f13f8bed2..0c10196ea 100644 --- a/openssl/src/crypto/ec/ecp_s390x_nistp.c +++ b/openssl/src/crypto/ec/ecp_s390x_nistp.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -152,14 +152,14 @@ static ECDSA_SIG *ecdsa_s390x_nistp_sign_sig(const unsigned char *dgst, k = BN_secure_new(); sig = ECDSA_SIG_new(); if (k == NULL || sig == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_ECDSA_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto ret; } sig->r = BN_new(); sig->s = BN_new(); if (sig->r == NULL || sig->s == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto ret; } @@ -178,7 +178,7 @@ static ECDSA_SIG *ecdsa_s390x_nistp_sign_sig(const unsigned char *dgst, goto ret; } /* - * Generate random k and copy to param block. RAND_priv_bytes_ex + * Generate random k and copy to param param block. RAND_priv_bytes_ex * is used instead of BN_priv_rand_range or BN_generate_dsa_nonce * because kdsa instruction constructs an in-range, invertible nonce * internally implementing counter-measures for RNG weakness. @@ -247,7 +247,7 @@ static int ecdsa_s390x_nistp_verify_sig(const unsigned char *dgst, int dgstlen, ctx = BN_CTX_new_ex(group->libctx); if (ctx == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return -1; } @@ -256,7 +256,7 @@ static int ecdsa_s390x_nistp_verify_sig(const unsigned char *dgst, int dgstlen, x = BN_CTX_get(ctx); y = BN_CTX_get(ctx); if (x == NULL || y == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto ret; } diff --git a/openssl/src/crypto/ec/ecp_sm2p256.c b/openssl/src/crypto/ec/ecp_sm2p256.c index 7668b6137..81c9c7cae 100644 --- a/openssl/src/crypto/ec/ecp_sm2p256.c +++ b/openssl/src/crypto/ec/ecp_sm2p256.c @@ -1,742 +1,1729 @@ /* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at * https://www.openssl.org/source/license.html + */ + +/* Copyright 2011 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ /* - * SM2 low level APIs are deprecated for public use, but still ok for + * ECDSA low level APIs are deprecated for public use, but still ok for * internal use. */ #include "internal/deprecated.h" +/* + * A 64-bit implementation of the SM2 P-256 elliptic curve point multiplication + * + * OpenSSL integration was taken from Emilia Kasper's work in ecp_nistp224.c. + * Otherwise based on Emilia's P224 work, which was inspired by my curve25519 + * work which got its smarts from Daniel J. Bernstein's work on the same. + * + * The optimization of SM2 was also inspired by GuanZhi‘s work on GMSSL. + */ + +#include + +#include #include #include -#include "crypto/bn.h" #include "ec_local.h" -#include "internal/common.h" -#include "internal/constant_time.h" -#define P256_LIMBS (256 / BN_BITS2) +#include "internal/numbers.h" -#if !defined(OPENSSL_NO_SM2_PRECOMP) -extern const BN_ULONG ecp_sm2p256_precomputed[8 * 32 * 256]; +#ifndef INT128_MAX +# error "Your compiler doesn't appear to support 128-bit integer types" #endif -typedef struct { - BN_ULONG X[P256_LIMBS]; - BN_ULONG Y[P256_LIMBS]; - BN_ULONG Z[P256_LIMBS]; -} P256_POINT; - -typedef struct { - BN_ULONG X[P256_LIMBS]; - BN_ULONG Y[P256_LIMBS]; -} P256_POINT_AFFINE; - -#if !defined(OPENSSL_NO_SM2_PRECOMP) -/* Coordinates of G, for which we have precomputed tables */ -ALIGN32 static const BN_ULONG def_xG[P256_LIMBS] = { - 0x715a4589334c74c7, 0x8fe30bbff2660be1, - 0x5f9904466a39c994, 0x32c4ae2c1f198119 -}; - -ALIGN32 static const BN_ULONG def_yG[P256_LIMBS] = { - 0x02df32e52139f0a0, 0xd0a9877cc62a4740, - 0x59bdcee36b692153, 0xbc3736a2f4f6779c, -}; -#endif +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; -/* p and order for SM2 according to GB/T 32918.5-2017 */ -ALIGN32 static const BN_ULONG def_p[P256_LIMBS] = { - 0xffffffffffffffff, 0xffffffff00000000, - 0xffffffffffffffff, 0xfffffffeffffffff -}; -ALIGN32 static const BN_ULONG def_ord[P256_LIMBS] = { - 0x53bbf40939d54123, 0x7203df6b21c6052b, - 0xffffffffffffffff, 0xfffffffeffffffff -}; +/* + * The underlying field. SM2 P-256 operates over GF(2^256-2^224-2^96+2^64-1). We + * can serialize an element of this field into 32 bytes. We call this an + * felem_bytearray. + */ -ALIGN32 static const BN_ULONG ONE[P256_LIMBS] = {1, 0, 0, 0}; +typedef u8 felem_bytearray[32]; -/* Functions implemented in assembly */ /* - * Most of below mentioned functions *preserve* the property of inputs - * being fully reduced, i.e. being in [0, modulus) range. Simply put if - * inputs are fully reduced, then output is too. + * These are the parameters of SM2 P-256, taken from GM/T 0003.5-2012, page 1. These + * values are big-endian. */ -/* Right shift: a >> 1 */ -void bn_rshift1(BN_ULONG *a); -/* Sub: r = a - b */ -void bn_sub(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -/* Modular div by 2: r = a / 2 mod p */ -void ecp_sm2p256_div_by_2(BN_ULONG *r, const BN_ULONG *a); -/* Modular div by 2: r = a / 2 mod n, where n = ord(p) */ -void ecp_sm2p256_div_by_2_mod_ord(BN_ULONG *r, const BN_ULONG *a); -/* Modular add: r = a + b mod p */ -void ecp_sm2p256_add(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -/* Modular sub: r = a - b mod p */ -void ecp_sm2p256_sub(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -/* Modular sub: r = a - b mod n, where n = ord(p) */ -void ecp_sm2p256_sub_mod_ord(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -/* Modular mul by 3: out = 3 * a mod p */ -void ecp_sm2p256_mul_by_3(BN_ULONG *r, const BN_ULONG *a); -/* Modular mul: r = a * b mod p */ -void ecp_sm2p256_mul(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -/* Modular sqr: r = a ^ 2 mod p */ -void ecp_sm2p256_sqr(BN_ULONG *r, const BN_ULONG *a); +static const felem_bytearray sm2p256_curve_params[5] = { + {0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0xff, /* p */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + {0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0xff, /* a = -3 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc}, + {0x28, 0xe9, 0xfa, 0x9e, 0x9d, 0x9f, 0x5e, 0x34, /* b */ + 0x4d, 0x5a, 0x9e, 0x4b, 0xcf, 0x65, 0x09, 0xa7, + 0xf3, 0x97, 0x89, 0xf5, 0x15, 0xab, 0x8f, 0x92, + 0xdd, 0xbc, 0xbd, 0x41, 0x4d, 0x94, 0x0e, 0x93}, + {0x32, 0xc4, 0xae, 0x2c, 0x1f, 0x19, 0x81, 0x19, /* x */ + 0x5f, 0x99, 0x04, 0x46, 0x6a, 0x39, 0xc9, 0x94, + 0x8f, 0xe3, 0x0b, 0xbf, 0xf2, 0x66, 0x0b, 0xe1, + 0x71, 0x5a, 0x45, 0x89, 0x33, 0x4c, 0x74, 0xc7}, + {0xbc, 0x37, 0x36, 0xa2, 0xf4, 0xf6, 0x77, 0x9c, /* y */ + 0x59, 0xbd, 0xce, 0xe3, 0x6b, 0x69, 0x21, 0x53, + 0xd0, 0xa9, 0x87, 0x7c, 0xc6, 0x2a, 0x47, 0x40, + 0x02, 0xdf, 0x32, 0xe5, 0x21, 0x39, 0xf0, 0xa0} +}; -static ossl_inline BN_ULONG is_zeros(const BN_ULONG *a) -{ - BN_ULONG res; +/*- + * The representation of field elements. + * ------------------------------------ + * + * We represent field elements with either four 128-bit values, eight 128-bit + * values, or four 64-bit values. The field element represented is: + * v[0]*2^0 + v[1]*2^64 + v[2]*2^128 + v[3]*2^192 (mod p) + * or: + * v[0]*2^0 + v[1]*2^64 + v[2]*2^128 + ... + v[7]*2^448 (mod p) + * + * 128-bit values are called 'limbs'. Since the limbs are spaced only 64 bits + * apart, but are 128-bits wide, the most significant bits of each limb overlap + * with the least significant bits of the next. + * + * A field element with four limbs is an 'felem'. One with eight limbs is a + * 'longfelem' + * + * A field element with four, 64-bit values is called a 'smallfelem'. Small + * values are used as intermediate values before multiplication. + */ - res = a[0] | a[1] | a[2] | a[3]; +#define NLIMBS 4 - return constant_time_is_zero_64(res); -} +typedef uint128_t limb; +typedef limb felem[NLIMBS]; +typedef limb longfelem[NLIMBS * 2]; +typedef u64 smallfelem[NLIMBS]; -static ossl_inline int is_equal(const BN_ULONG *a, const BN_ULONG *b) -{ - BN_ULONG res; +/* This is the value of the prime as four 64-bit words, little-endian. */ +static const u64 kPrime[4] = + { 0xfffffffffffffffful, 0xffffffff00000000ul, 0xfffffffffffffffful, 0xfffffffefffffffful }; +static const u64 bottom63bits = 0x7ffffffffffffffful; - res = a[0] ^ b[0]; - res |= a[1] ^ b[1]; - res |= a[2] ^ b[2]; - res |= a[3] ^ b[3]; +/* + * bin32_to_felem takes a little-endian byte array and converts it into felem + * form. This assumes that the CPU is little-endian. + */ +static void bin32_to_felem(felem out, const u8 in[32]) +{ + out[0] = *((u64 *)&in[0]); + out[1] = *((u64 *)&in[8]); + out[2] = *((u64 *)&in[16]); + out[3] = *((u64 *)&in[24]); +} - return constant_time_is_zero_64(res); +/* + * smallfelem_to_bin32 takes a smallfelem and serializes into a little + * endian, 32 byte array. This assumes that the CPU is little-endian. + */ +static void smallfelem_to_bin32(u8 out[32], const smallfelem in) +{ + *((u64 *)&out[0]) = in[0]; + *((u64 *)&out[8]) = in[1]; + *((u64 *)&out[16]) = in[2]; + *((u64 *)&out[24]) = in[3]; } -static ossl_inline int is_greater(const BN_ULONG *a, const BN_ULONG *b) +/* BN_to_felem converts an OpenSSL BIGNUM into an felem */ +static int BN_to_felem(felem out, const BIGNUM *bn) { - int i; + felem_bytearray b_out; + int num_bytes; - for (i = P256_LIMBS - 1; i >= 0; --i) { - if (a[i] > b[i]) - return 1; - if (a[i] < b[i]) - return -1; + if (BN_is_negative(bn)) { + ERR_raise(ERR_LIB_EC, EC_R_BIGNUM_OUT_OF_RANGE); + return 0; } - - return 0; -} - -#define is_one(a) is_equal(a, ONE) -#define is_even(a) !(a[0] & 1) -#define is_point_equal(a, b) \ - is_equal(a->X, b->X) && \ - is_equal(a->Y, b->Y) && \ - is_equal(a->Z, b->Z) - -/* Bignum and field elements conversion */ -#define ecp_sm2p256_bignum_field_elem(out, in) \ - bn_copy_words(out, in, P256_LIMBS) - -/* Binary algorithm for inversion in Fp */ -#define BN_MOD_INV(out, in, mod_div, mod_sub, mod) \ - do { \ - ALIGN32 BN_ULONG u[4]; \ - ALIGN32 BN_ULONG v[4]; \ - ALIGN32 BN_ULONG x1[4] = {1, 0, 0, 0}; \ - ALIGN32 BN_ULONG x2[4] = {0}; \ - \ - if (is_zeros(in)) \ - return; \ - memcpy(u, in, 32); \ - memcpy(v, mod, 32); \ - while (!is_one(u) && !is_one(v)) { \ - while (is_even(u)) { \ - bn_rshift1(u); \ - mod_div(x1, x1); \ - } \ - while (is_even(v)) { \ - bn_rshift1(v); \ - mod_div(x2, x2); \ - } \ - if (is_greater(u, v) == 1) { \ - bn_sub(u, u, v); \ - mod_sub(x1, x1, x2); \ - } else { \ - bn_sub(v, v, u); \ - mod_sub(x2, x2, x1); \ - } \ - } \ - if (is_one(u)) \ - memcpy(out, x1, 32); \ - else \ - memcpy(out, x2, 32); \ - } while (0) - -/* Modular inverse |out| = |in|^(-1) mod |p|. */ -static ossl_inline void ecp_sm2p256_mod_inverse(BN_ULONG* out, - const BN_ULONG* in) { - BN_MOD_INV(out, in, ecp_sm2p256_div_by_2, ecp_sm2p256_sub, def_p); -} - -/* Modular inverse mod order |out| = |in|^(-1) % |ord|. */ -static ossl_inline void ecp_sm2p256_mod_ord_inverse(BN_ULONG* out, - const BN_ULONG* in) { - BN_MOD_INV(out, in, ecp_sm2p256_div_by_2_mod_ord, ecp_sm2p256_sub_mod_ord, - def_ord); -} - -/* Point double: R <- P + P */ -static void ecp_sm2p256_point_double(P256_POINT *R, const P256_POINT *P) -{ - unsigned int i; - ALIGN32 BN_ULONG tmp0[P256_LIMBS]; - ALIGN32 BN_ULONG tmp1[P256_LIMBS]; - ALIGN32 BN_ULONG tmp2[P256_LIMBS]; - - /* zero-check P->Z */ - if (is_zeros(P->Z)) { - for (i = 0; i < P256_LIMBS; ++i) - R->Z[i] = 0; - - return; + num_bytes = BN_bn2lebinpad(bn, b_out, sizeof(b_out)); + if (num_bytes < 0) { + ERR_raise(ERR_LIB_EC, EC_R_BIGNUM_OUT_OF_RANGE); + return 0; } + bin32_to_felem(out, b_out); + return 1; +} - ecp_sm2p256_sqr(tmp0, P->Z); - ecp_sm2p256_sub(tmp1, P->X, tmp0); - ecp_sm2p256_add(tmp0, P->X, tmp0); - ecp_sm2p256_mul(tmp1, tmp1, tmp0); - ecp_sm2p256_mul_by_3(tmp1, tmp1); - ecp_sm2p256_add(R->Y, P->Y, P->Y); - ecp_sm2p256_mul(R->Z, R->Y, P->Z); - ecp_sm2p256_sqr(R->Y, R->Y); - ecp_sm2p256_mul(tmp2, R->Y, P->X); - ecp_sm2p256_sqr(R->Y, R->Y); - ecp_sm2p256_div_by_2(R->Y, R->Y); - ecp_sm2p256_sqr(R->X, tmp1); - ecp_sm2p256_add(tmp0, tmp2, tmp2); - ecp_sm2p256_sub(R->X, R->X, tmp0); - ecp_sm2p256_sub(tmp0, tmp2, R->X); - ecp_sm2p256_mul(tmp0, tmp0, tmp1); - ecp_sm2p256_sub(tmp1, tmp0, R->Y); - memcpy(R->Y, tmp1, 32); -} - -/* Point add affine: R <- P + Q */ -static void ecp_sm2p256_point_add_affine(P256_POINT *R, const P256_POINT *P, - const P256_POINT_AFFINE *Q) -{ - unsigned int i; - ALIGN32 BN_ULONG tmp0[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG tmp1[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG tmp2[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG tmp3[P256_LIMBS] = {0}; - - /* zero-check P->Z */ - if (is_zeros(P->Z)) { - for (i = 0; i < P256_LIMBS; ++i) { - R->X[i] = Q->X[i]; - R->Y[i] = Q->Y[i]; - R->Z[i] = 0; - } - R->Z[0] = 1; +/* felem_to_BN converts an felem into an OpenSSL BIGNUM */ +static BIGNUM *smallfelem_to_BN(BIGNUM *out, const smallfelem in) +{ + felem_bytearray b_out; + smallfelem_to_bin32(b_out, in); + return BN_lebin2bn(b_out, sizeof(b_out), out); +} - return; - } +/*- + * Field operations + * ---------------- + */ - ecp_sm2p256_sqr(tmp0, P->Z); - ecp_sm2p256_mul(tmp1, tmp0, P->Z); - ecp_sm2p256_mul(tmp0, tmp0, Q->X); - ecp_sm2p256_mul(tmp1, tmp1, Q->Y); - ecp_sm2p256_sub(tmp0, tmp0, P->X); - ecp_sm2p256_sub(tmp1, tmp1, P->Y); - - /* zero-check tmp0, tmp1 */ - if (is_zeros(tmp0)) { - if (is_zeros(tmp1)) { - P256_POINT K; - - for (i = 0; i < P256_LIMBS; ++i) { - K.X[i] = Q->X[i]; - K.Y[i] = Q->Y[i]; - K.Z[i] = 0; - } - K.Z[0] = 1; - ecp_sm2p256_point_double(R, &K); - } else { - for (i = 0; i < P256_LIMBS; ++i) - R->Z[i] = 0; - } +static void smallfelem_one(smallfelem out) +{ + out[0] = 1; + out[1] = 0; + out[2] = 0; + out[3] = 0; +} - return; - } +static void smallfelem_assign(smallfelem out, const smallfelem in) +{ + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; +} - ecp_sm2p256_mul(R->Z, P->Z, tmp0); - ecp_sm2p256_sqr(tmp2, tmp0); - ecp_sm2p256_mul(tmp3, tmp2, tmp0); - ecp_sm2p256_mul(tmp2, tmp2, P->X); - ecp_sm2p256_add(tmp0, tmp2, tmp2); - ecp_sm2p256_sqr(R->X, tmp1); - ecp_sm2p256_sub(R->X, R->X, tmp0); - ecp_sm2p256_sub(R->X, R->X, tmp3); - ecp_sm2p256_sub(tmp2, tmp2, R->X); - ecp_sm2p256_mul(tmp2, tmp2, tmp1); - ecp_sm2p256_mul(tmp3, tmp3, P->Y); - ecp_sm2p256_sub(R->Y, tmp2, tmp3); -} - -/* Point add: R <- P + Q */ -static void ecp_sm2p256_point_add(P256_POINT *R, const P256_POINT *P, - const P256_POINT *Q) -{ - unsigned int i; - ALIGN32 BN_ULONG tmp0[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG tmp1[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG tmp2[P256_LIMBS] = {0}; - - /* zero-check P | Q ->Z */ - if (is_zeros(P->Z)) { - for (i = 0; i < P256_LIMBS; ++i) { - R->X[i] = Q->X[i]; - R->Y[i] = Q->Y[i]; - R->Z[i] = Q->Z[i]; - } +static void felem_assign(felem out, const felem in) +{ + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; +} - return; - } else if (is_zeros(Q->Z)) { - for (i = 0; i < P256_LIMBS; ++i) { - R->X[i] = P->X[i]; - R->Y[i] = P->Y[i]; - R->Z[i] = P->Z[i]; - } +/* felem_sum sets out = out + in. */ +static void felem_sum(felem out, const felem in) +{ + out[0] += in[0]; + out[1] += in[1]; + out[2] += in[2]; + out[3] += in[3]; +} - return; - } else if (is_point_equal(P, Q)) { - ecp_sm2p256_point_double(R, Q); +/* felem_small_sum sets out = out + in. */ +static void felem_small_sum(felem out, const smallfelem in) +{ + out[0] += in[0]; + out[1] += in[1]; + out[2] += in[2]; + out[3] += in[3]; +} - return; - } +/* felem_scalar sets out = out * scalar */ +static void felem_scalar(felem out, const u64 scalar) +{ + out[0] *= scalar; + out[1] *= scalar; + out[2] *= scalar; + out[3] *= scalar; +} - ecp_sm2p256_sqr(tmp0, P->Z); - ecp_sm2p256_mul(tmp1, tmp0, P->Z); - ecp_sm2p256_mul(tmp0, tmp0, Q->X); - ecp_sm2p256_mul(tmp1, tmp1, Q->Y); - ecp_sm2p256_mul(R->Y, P->Y, Q->Z); - ecp_sm2p256_mul(R->Z, Q->Z, P->Z); - ecp_sm2p256_sqr(tmp2, Q->Z); - ecp_sm2p256_mul(R->Y, tmp2, R->Y); - ecp_sm2p256_mul(R->X, tmp2, P->X); - ecp_sm2p256_sub(tmp0, tmp0, R->X); - ecp_sm2p256_mul(R->Z, tmp0, R->Z); - ecp_sm2p256_sub(tmp1, tmp1, R->Y); - ecp_sm2p256_sqr(tmp2, tmp0); - ecp_sm2p256_mul(tmp0, tmp0, tmp2); - ecp_sm2p256_mul(tmp2, tmp2, R->X); - ecp_sm2p256_sqr(R->X, tmp1); - ecp_sm2p256_sub(R->X, R->X, tmp2); - ecp_sm2p256_sub(R->X, R->X, tmp2); - ecp_sm2p256_sub(R->X, R->X, tmp0); - ecp_sm2p256_sub(tmp2, tmp2, R->X); - ecp_sm2p256_mul(tmp2, tmp1, tmp2); - ecp_sm2p256_mul(tmp0, tmp0, R->Y); - ecp_sm2p256_sub(R->Y, tmp2, tmp0); -} - -#if !defined(OPENSSL_NO_SM2_PRECOMP) -/* Base point mul by scalar: k - scalar, G - base point */ -static void ecp_sm2p256_point_G_mul_by_scalar(P256_POINT *R, const BN_ULONG *k) -{ - unsigned int i, index, mask = 0xff; - P256_POINT_AFFINE Q; - - memset(R, 0, sizeof(P256_POINT)); - - if (is_zeros(k)) - return; +/* longfelem_scalar sets out = out * scalar */ +static void longfelem_scalar(longfelem out, const u64 scalar) +{ + out[0] *= scalar; + out[1] *= scalar; + out[2] *= scalar; + out[3] *= scalar; + out[4] *= scalar; + out[5] *= scalar; + out[6] *= scalar; + out[7] *= scalar; +} - index = k[0] & mask; - if (index) { - index = index * 8; - memcpy(R->X, ecp_sm2p256_precomputed + index, 32); - memcpy(R->Y, ecp_sm2p256_precomputed + index + P256_LIMBS, 32); - R->Z[0] = 1; - } +#define two105m73m41 (((limb)1) << 105) - (((limb)1) << 73) - (((limb)1) << 41) +#define two105m73 (((limb)1) << 105) - (((limb)1) << 73) +#define two105m41 (((limb)1) << 105) - (((limb)1) << 41) - for (i = 1; i < 32; ++i) { - index = (k[i / 8] >> (8 * (i % 8))) & mask; +/* zero105 is (2^41 * p) mod p = 0 mod p*/ +static const felem zero105 = + { two105m41, two105m73, two105m41, two105m73m41 }; - if (index) { - index = index + i * 256; - index = index * 8; - memcpy(Q.X, ecp_sm2p256_precomputed + index, 32); - memcpy(Q.Y, ecp_sm2p256_precomputed + index + P256_LIMBS, 32); - ecp_sm2p256_point_add_affine(R, R, &Q); - } - } +/*- + * smallfelem_neg sets |out| to |-small| + * On exit: + * out[i] < out[i] + 2^105 + */ +static void smallfelem_neg(felem out, const smallfelem small) +{ + /* In order to prevent underflow, we subtract from 0 mod p. */ + out[0] = zero105[0] - small[0]; + out[1] = zero105[1] - small[1]; + out[2] = zero105[2] - small[2]; + out[3] = zero105[3] - small[3]; } -#endif -/* - * Affine point mul by scalar: k - scalar, P - affine point +/*- + * felem_diff subtracts |in| from |out| + * On entry: + * in[i] < 2^104 + * On exit: + * out[i] < out[i] + 2^105 + */ +static void felem_diff(felem out, const felem in) +{ + /* + * In order to prevent underflow, we add 0 mod p before subtracting. + */ + out[0] += zero105[0]; + out[1] += zero105[1]; + out[2] += zero105[2]; + out[3] += zero105[3]; + + out[0] -= in[0]; + out[1] -= in[1]; + out[2] -= in[2]; + out[3] -= in[3]; +} + +#define two107m75m43 (((limb)1) << 107) - (((limb)1) << 75) - (((limb)1) << 43) +#define two107m75 (((limb)1) << 107) - (((limb)1) << 75) +#define two107m43 (((limb)1) << 107) - (((limb)1) << 43) + +/* zero107 is (2^43 * p) mod p = 0 mod p*/ +static const felem zero107 = + { two107m43, two107m75, two107m43, two107m75m43 }; + +/*- + * An alternative felem_diff for larger inputs |in| + * felem_diff_zero107 subtracts |in| from |out| + * On entry: + * in[i] < 2^106 + * On exit: + * out[i] < out[i] + 2^107 */ -static void ecp_sm2p256_point_P_mul_by_scalar(P256_POINT *R, const BN_ULONG *k, - P256_POINT_AFFINE P) +static void felem_diff_zero107(felem out, const felem in) { - int i, init = 0; - unsigned int index, mask = 0x0f; - ALIGN64 P256_POINT precomputed[16]; + /* + * In order to prevent underflow, we add 0 mod p before subtracting. + */ + out[0] += zero107[0]; + out[1] += zero107[1]; + out[2] += zero107[2]; + out[3] += zero107[3]; + + out[0] -= in[0]; + out[1] -= in[1]; + out[2] -= in[2]; + out[3] -= in[3]; +} - memset(R, 0, sizeof(P256_POINT)); +/*- + * longfelem_diff subtracts |in| from |out| + * On entry: + * in[i] < 7*2^67 + * On exit: + * out[i] < out[i] + 2^70 + 2^40 + */ +static void longfelem_diff(longfelem out, const longfelem in) +{ + /* construct a zero num equals to + * (2^262+2^230+2^198+2^166+2^134+2^103+2^71+2^39+192) * p + */ + static const limb two70m39m7m6 = + (((limb) 1) << 70) - (((limb) 1) << 39) - (((limb) 1) << 7) - (((limb) 1) << 6); + static const limb two70m40p38 = (((limb) 1) << 70) - (((limb) 1) << 40) + (((limb) 1) << 38); + static const limb two70m38m7 = (((limb) 1) << 70) - (((limb) 1) << 38) - (((limb) 1) << 7); + static const limb two70m40m7m6 = + (((limb) 1) << 70) - (((limb) 1) << 40) - (((limb) 1) << 7) - + (((limb) 1) << 6); + static const limb two70m6 = (((limb) 1) << 70) - (((limb) 1) << 6); + + /* add 0 mod p to avoid underflow */ + out[0] += two70m39m7m6; + out[1] += two70m40p38; + out[2] += two70m38m7; + out[3] += two70m40m7m6; + out[4] += two70m6; + out[5] += two70m6; + out[6] += two70m6; + out[7] += two70m6; + + /* in[i] < 7*2^67 < 2^70 - 2^40 - 2^38 + 2^6 */ + out[0] -= in[0]; + out[1] -= in[1]; + out[2] -= in[2]; + out[3] -= in[3]; + out[4] -= in[4]; + out[5] -= in[5]; + out[6] -= in[6]; + out[7] -= in[7]; +} - if (is_zeros(k)) - return; +#define two64m0 (((limb)1) << 64) - 1 +#define two64m32 (((limb)1) << 64) - (((limb)1) << 32) +#define two64m32m0 (((limb)1) << 64) - (((limb)1) << 32) - 1 - /* The first value of the precomputed table is P. */ - memcpy(precomputed[1].X, P.X, 32); - memcpy(precomputed[1].Y, P.Y, 32); - precomputed[1].Z[0] = 1; - precomputed[1].Z[1] = 0; - precomputed[1].Z[2] = 0; - precomputed[1].Z[3] = 0; +/* zerop is equal to p */ +static const felem zerop = { two64m0, two64m32, two64m0, two64m32m0 }; - /* The second value of the precomputed table is 2P. */ - ecp_sm2p256_point_double(&precomputed[2], &precomputed[1]); +/*- + * felem_shrink converts an felem into a smallfelem. The result isn't quite + * minimal as the value may be greater than p. + * + * On entry: + * in[i] < 2^109 + * On exit: + * out[i] < 2^64 + */ +static void felem_shrink(smallfelem out, const felem in) +{ + felem tmp; + u64 a, b, mask; + u64 high, low; + static const u64 kPrime3Test = 0x7ffffffefffffffful; /* 2^63 - 2^32 - 1 */ + + /* Carry 2->3 */ + tmp[3] = zerop[3] + in[3] + ((u64)(in[2] >> 64)); + /* tmp[3] < 2^110 */ + + tmp[2] = zerop[2] + (u64)in[2]; + tmp[0] = zerop[0] + in[0]; + tmp[1] = zerop[1] + in[1]; + /* tmp[0] < 2**110, tmp[1] < 2^111, tmp[2] < 2**65 */ + + /* + * We perform two partial reductions where we eliminate the high-word of + * tmp[3]. We don't update the other words till the end. + */ + a = tmp[3] >> 64; /* a < 2^46 */ + tmp[3] = (u64)tmp[3]; + tmp[3] += ((limb) a) << 32; + /* tmp[3] < 2^79 */ + + b = a; + a = tmp[3] >> 64; /* a < 2^15 */ + b += a; /* b < 2^46 + 2^15 < 2^47 */ + tmp[3] = (u64)tmp[3]; + tmp[3] += ((limb) a) << 32; + /* tmp[3] < 2^64 + 2^47 */ + + /* + * This adjusts the other two words to complete the two partial + * reductions. + */ + tmp[0] += b; + tmp[1] += (((limb) b) << 32); + tmp[1] -= b; + + /* + * In order to make space in tmp[3] for the carry from 2 -> 3, we + * conditionally subtract kPrime if tmp[3] is large enough. + */ + high = (u64)(tmp[3] >> 64); + /* As tmp[3] < 2^65, high is either 1 or 0 */ + high = 0 - high; + /*- + * high is: + * all ones if the high word of tmp[3] is 1 + * all zeros if the high word of tmp[3] if 0 + */ + low = (u64)tmp[3]; + mask = 0 - (low >> 63); + /*- + * mask is: + * all ones if the MSB of low is 1 + * all zeros if the MSB of low if 0 + */ + low &= bottom63bits; + low -= kPrime3Test; + /* if low was greater than kPrime3Test then the MSB is zero */ + low = ~low; + low = 0 - (low >> 63); + /*- + * low is: + * all ones if low was > kPrime3Test + * all zeros if low was <= kPrime3Test + */ + mask = (mask & low) | high; + tmp[0] -= mask & kPrime[0]; + tmp[1] -= mask & kPrime[1]; + tmp[2] -= mask & kPrime[2]; + tmp[3] -= mask & kPrime[3]; + /* tmp[3] < 2**64 - 2**32 + 1 */ + + tmp[1] += ((u64)(tmp[0] >> 64)); + tmp[0] = (u64)tmp[0]; + tmp[2] += ((u64)(tmp[1] >> 64)); + tmp[1] = (u64)tmp[1]; + tmp[3] += ((u64)(tmp[2] >> 64)); + tmp[2] = (u64)tmp[2]; + /* tmp[i] < 2^64 */ + + out[0] = tmp[0]; + out[1] = tmp[1]; + out[2] = tmp[2]; + out[3] = tmp[3]; +} - /* The subsequent elements are 3P, 4P, and so on. */ - for (i = 3; i < 16; ++i) - ecp_sm2p256_point_add_affine(&precomputed[i], &precomputed[i - 1], &P); +/* smallfelem_expand converts a smallfelem to an felem */ +static void smallfelem_expand(felem out, const smallfelem in) +{ + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; +} - for (i = 64 - 1; i >= 0; --i) { - index = (k[i / 16] >> (4 * (i % 16))) & mask; +/*- + * smallfelem_square sets |out| = |small|^2 + * On entry: + * small[i] < 2^64 + * On exit: + * out[i] < 7 * 2^64 < 2^67 + */ +static void smallfelem_square(longfelem out, const smallfelem small) +{ + limb a; + u64 high, low; + + a = ((uint128_t) small[0]) * small[0]; + low = a; + high = a >> 64; + out[0] = low; + out[1] = high; + + a = ((uint128_t) small[0]) * small[1]; + low = a; + high = a >> 64; + out[1] += low; + out[1] += low; + out[2] = high; + + a = ((uint128_t) small[0]) * small[2]; + low = a; + high = a >> 64; + out[2] += low; + out[2] *= 2; + out[3] = high; + + a = ((uint128_t) small[0]) * small[3]; + low = a; + high = a >> 64; + out[3] += low; + out[4] = high; + + a = ((uint128_t) small[1]) * small[2]; + low = a; + high = a >> 64; + out[3] += low; + out[3] *= 2; + out[4] += high; + + a = ((uint128_t) small[1]) * small[1]; + low = a; + high = a >> 64; + out[2] += low; + out[3] += high; + + a = ((uint128_t) small[1]) * small[3]; + low = a; + high = a >> 64; + out[4] += low; + out[4] *= 2; + out[5] = high; + + a = ((uint128_t) small[2]) * small[3]; + low = a; + high = a >> 64; + out[5] += low; + out[5] *= 2; + out[6] = high; + out[6] += high; + + a = ((uint128_t) small[2]) * small[2]; + low = a; + high = a >> 64; + out[4] += low; + out[5] += high; + + a = ((uint128_t) small[3]) * small[3]; + low = a; + high = a >> 64; + out[6] += low; + out[7] = high; +} - if (init == 0) { - if (index) { - memcpy(R, &precomputed[index], sizeof(P256_POINT)); - init = 1; - } - } else { - ecp_sm2p256_point_double(R, R); - ecp_sm2p256_point_double(R, R); - ecp_sm2p256_point_double(R, R); - ecp_sm2p256_point_double(R, R); - if (index) - ecp_sm2p256_point_add(R, R, &precomputed[index]); - } - } +/*- + * felem_square sets |out| = |in|^2 + * On entry: + * in[i] < 2^109 + * On exit: + * out[i] < 7 * 2^64 < 2^67 + */ +static void felem_square(longfelem out, const felem in) +{ + u64 small[4]; + felem_shrink(small, in); + smallfelem_square(out, small); } -/* Get affine point */ -static void ecp_sm2p256_point_get_affine(P256_POINT_AFFINE *R, - const P256_POINT *P) +/*- + * smallfelem_mul sets |out| = |small1| * |small2| + * On entry: + * small1[i] < 2^64 + * small2[i] < 2^64 + * On exit: + * out[i] < 7 * 2^64 < 2^67 + */ +static void smallfelem_mul(longfelem out, const smallfelem small1, + const smallfelem small2) { - ALIGN32 BN_ULONG z_inv3[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG z_inv2[P256_LIMBS] = {0}; + limb a; + u64 high, low; + + a = ((uint128_t) small1[0]) * small2[0]; + low = a; + high = a >> 64; + out[0] = low; + out[1] = high; + + a = ((uint128_t) small1[0]) * small2[1]; + low = a; + high = a >> 64; + out[1] += low; + out[2] = high; + + a = ((uint128_t) small1[1]) * small2[0]; + low = a; + high = a >> 64; + out[1] += low; + out[2] += high; + + a = ((uint128_t) small1[0]) * small2[2]; + low = a; + high = a >> 64; + out[2] += low; + out[3] = high; + + a = ((uint128_t) small1[1]) * small2[1]; + low = a; + high = a >> 64; + out[2] += low; + out[3] += high; + + a = ((uint128_t) small1[2]) * small2[0]; + low = a; + high = a >> 64; + out[2] += low; + out[3] += high; + + a = ((uint128_t) small1[0]) * small2[3]; + low = a; + high = a >> 64; + out[3] += low; + out[4] = high; + + a = ((uint128_t) small1[1]) * small2[2]; + low = a; + high = a >> 64; + out[3] += low; + out[4] += high; + + a = ((uint128_t) small1[2]) * small2[1]; + low = a; + high = a >> 64; + out[3] += low; + out[4] += high; + + a = ((uint128_t) small1[3]) * small2[0]; + low = a; + high = a >> 64; + out[3] += low; + out[4] += high; + + a = ((uint128_t) small1[1]) * small2[3]; + low = a; + high = a >> 64; + out[4] += low; + out[5] = high; + + a = ((uint128_t) small1[2]) * small2[2]; + low = a; + high = a >> 64; + out[4] += low; + out[5] += high; + + a = ((uint128_t) small1[3]) * small2[1]; + low = a; + high = a >> 64; + out[4] += low; + out[5] += high; + + a = ((uint128_t) small1[2]) * small2[3]; + low = a; + high = a >> 64; + out[5] += low; + out[6] = high; + + a = ((uint128_t) small1[3]) * small2[2]; + low = a; + high = a >> 64; + out[5] += low; + out[6] += high; + + a = ((uint128_t) small1[3]) * small2[3]; + low = a; + high = a >> 64; + out[6] += low; + out[7] = high; +} - if (is_one(P->Z)) { - memcpy(R->X, P->X, 32); - memcpy(R->Y, P->Y, 32); - return; - } +/*- + * felem_mul sets |out| = |in1| * |in2| + * On entry: + * in1[i] < 2^109 + * in2[i] < 2^109 + * On exit: + * out[i] < 7 * 2^64 < 2^67 + */ +static void felem_mul(longfelem out, const felem in1, const felem in2) +{ + smallfelem small1, small2; + felem_shrink(small1, in1); + felem_shrink(small2, in2); + smallfelem_mul(out, small1, small2); +} - ecp_sm2p256_mod_inverse(z_inv3, P->Z); - ecp_sm2p256_sqr(z_inv2, z_inv3); - ecp_sm2p256_mul(R->X, P->X, z_inv2); - ecp_sm2p256_mul(z_inv3, z_inv3, z_inv2); - ecp_sm2p256_mul(R->Y, P->Y, z_inv3); +/*- + * felem_small_mul sets |out| = |small1| * |in2| + * On entry: + * small1[i] < 2^64 + * in2[i] < 2^109 + * On exit: + * out[i] < 7 * 2^64 < 2^67 + */ +static void felem_small_mul(longfelem out, const smallfelem small1, + const felem in2) +{ + smallfelem small2; + felem_shrink(small2, in2); + smallfelem_mul(out, small1, small2); } -#if !defined(OPENSSL_NO_SM2_PRECOMP) -static int ecp_sm2p256_is_affine_G(const EC_POINT *generator) +/*- + * felem_reduce converts a longfelem into an felem. + * To be called directly after felem_square or felem_mul. + * On entry: + * in[0] < 2^64, in[1] < 3*2^64, in[2] < 5*2^64, in[3] < 7*2^64 + * in[4] < 7*2^64, in[5] < 5*2^64, in[6] < 3*2^64, in[7] < 2*64 + * On exit: + * out[i] < 2^101 + */ +static void felem_reduce(felem out, const longfelem in) { - return (bn_get_top(generator->X) == P256_LIMBS) - && (bn_get_top(generator->Y) == P256_LIMBS) - && is_equal(bn_get_words(generator->X), def_xG) - && is_equal(bn_get_words(generator->Y), def_yG) - && (generator->Z_is_one == 1); + uint128_t a, b, c, d; + a = in[6] + in[7]; + b = in[5] + in[7]; + c = in[4] + in[7]; + d = a + b; + + out[3] = in[3] + ((in[4] + in[5] + a * 2) << 32) + in[7]; + out[2] = in[2] + (b << 32) + a + in[7]; + out[1] = in[1] + ((c + in[6]) << 32) - c; + out[0] = in[0] + (d << 32) + d + in[4]; } -#endif /* - * Convert Jacobian coordinate point into affine coordinate (x,y) + * subtract_u64 sets *result = *result - v and *carry to one if the + * subtraction underflowed. */ -static int ecp_sm2p256_get_affine(const EC_GROUP *group, - const EC_POINT *point, - BIGNUM *x, BIGNUM *y, BN_CTX *ctx) +static void subtract_u64(u64 *result, u64 *carry, u64 v) { - ALIGN32 BN_ULONG z_inv2[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG z_inv3[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG x_aff[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG y_aff[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG point_x[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG point_y[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG point_z[P256_LIMBS] = {0}; + uint128_t r = *result; + r -= v; + *carry = (r >> 64) & 1; + *result = (u64)r; +} - if (EC_POINT_is_at_infinity(group, point)) { - ECerr(ERR_LIB_EC, EC_R_POINT_AT_INFINITY); - return 0; +/* + * felem_contract converts |in| to its unique, minimal representation. On + * entry: in[i] < 2^109 + */ +static void felem_contract(smallfelem out, const felem in) +{ + unsigned i; + u64 all_equal_so_far = 0, result = 0, carry; + + felem_shrink(out, in); + /* small is minimal except that the value might be > p */ + + all_equal_so_far--; + /* + * We are doing a constant time test if out >= kPrime. We need to compare + * each u64, from most-significant to least significant. For each one, if + * all words so far have been equal (m is all ones) then a non-equal + * result is the answer. Otherwise we continue. + */ + for (i = 3; i < 4; i--) { + u64 equal; + uint128_t a = ((uint128_t) kPrime[i]) - out[i]; + /* + * if out[i] > kPrime[i] then a will underflow and the high 64-bits + * will all be set. + */ + result |= all_equal_so_far & ((u64)(a >> 64)); + + /* + * if kPrime[i] == out[i] then |equal| will be all zeros and the + * decrement will make it all ones. + */ + equal = kPrime[i] ^ out[i]; + equal--; + equal &= equal << 32; + equal &= equal << 16; + equal &= equal << 8; + equal &= equal << 4; + equal &= equal << 2; + equal &= equal << 1; + equal = 0 - (equal >> 63); + + all_equal_so_far &= equal; } - if (ecp_sm2p256_bignum_field_elem(point_x, point->X) <= 0 - || ecp_sm2p256_bignum_field_elem(point_y, point->Y) <= 0 - || ecp_sm2p256_bignum_field_elem(point_z, point->Z) <= 0) { - ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE); - return 0; - } + /* + * if all_equal_so_far is still all ones then the two values are equal + * and so out >= kPrime is true. + */ + result |= all_equal_so_far; - ecp_sm2p256_mod_inverse(z_inv3, point_z); - ecp_sm2p256_sqr(z_inv2, z_inv3); + /* if out >= kPrime then we subtract kPrime. */ + subtract_u64(&out[0], &carry, result & kPrime[0]); + subtract_u64(&out[1], &carry, carry); + subtract_u64(&out[2], &carry, carry); + subtract_u64(&out[3], &carry, carry); - if (x != NULL) { - ecp_sm2p256_mul(x_aff, point_x, z_inv2); - if (!bn_set_words(x, x_aff, P256_LIMBS)) - return 0; - } + subtract_u64(&out[1], &carry, result & kPrime[1]); + subtract_u64(&out[2], &carry, carry); + subtract_u64(&out[3], &carry, carry); - if (y != NULL) { - ecp_sm2p256_mul(z_inv3, z_inv3, z_inv2); - ecp_sm2p256_mul(y_aff, point_y, z_inv3); - if (!bn_set_words(y, y_aff, P256_LIMBS)) - return 0; - } + subtract_u64(&out[2], &carry, result & kPrime[2]); + subtract_u64(&out[3], &carry, carry); - return 1; + subtract_u64(&out[3], &carry, result & kPrime[3]); } -/* r = sum(scalar[i]*point[i]) */ -static int ecp_sm2p256_windowed_mul(const EC_GROUP *group, - P256_POINT *r, - const BIGNUM **scalar, - const EC_POINT **point, - size_t num, BN_CTX *ctx) +static void smallfelem_square_contract(smallfelem out, const smallfelem in) { - unsigned int i; - int ret = 0; - const BIGNUM **scalars = NULL; - ALIGN32 BN_ULONG k[P256_LIMBS] = {0}; - P256_POINT kP; - ALIGN32 union { - P256_POINT p; - P256_POINT_AFFINE a; - } t, p; - - if (num > OPENSSL_MALLOC_MAX_NELEMS(P256_POINT) - || (scalars = OPENSSL_malloc(num * sizeof(BIGNUM *))) == NULL) { - ECerr(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); - goto err; - } + longfelem longtmp; + felem tmp; - memset(r, 0, sizeof(P256_POINT)); + smallfelem_square(longtmp, in); + felem_reduce(tmp, longtmp); + felem_contract(out, tmp); +} - for (i = 0; i < num; i++) { - if (EC_POINT_is_at_infinity(group, point[i])) - continue; +static void smallfelem_mul_contract(smallfelem out, const smallfelem in1, + const smallfelem in2) +{ + longfelem longtmp; + felem tmp; - if ((BN_num_bits(scalar[i]) > 256) || BN_is_negative(scalar[i])) { - BIGNUM *tmp; + smallfelem_mul(longtmp, in1, in2); + felem_reduce(tmp, longtmp); + felem_contract(out, tmp); +} - if ((tmp = BN_CTX_get(ctx)) == NULL) - goto err; - if (!BN_nnmod(tmp, scalar[i], group->order, ctx)) { - ECerr(ERR_LIB_EC, ERR_R_BN_LIB); - goto err; - } - scalars[i] = tmp; - } else { - scalars[i] = scalar[i]; - } +/*- + * felem_is_zero returns a limb with all bits set if |in| == 0 (mod p) and 0 + * otherwise. + * On entry: + * small[i] < 2^64 + */ +static limb smallfelem_is_zero(const smallfelem small) +{ + limb result; + u64 is_p; + + u64 is_zero = small[0] | small[1] | small[2] | small[3]; + is_zero--; + is_zero &= is_zero << 32; + is_zero &= is_zero << 16; + is_zero &= is_zero << 8; + is_zero &= is_zero << 4; + is_zero &= is_zero << 2; + is_zero &= is_zero << 1; + is_zero = 0 - (is_zero >> 63); + + is_p = (small[0] ^ kPrime[0]) | + (small[1] ^ kPrime[1]) | + (small[2] ^ kPrime[2]) | (small[3] ^ kPrime[3]); + is_p--; + is_p &= is_p << 32; + is_p &= is_p << 16; + is_p &= is_p << 8; + is_p &= is_p << 4; + is_p &= is_p << 2; + is_p &= is_p << 1; + is_p = 0 - (is_p >> 63); + + is_zero |= is_p; + + result = is_zero; + result |= ((limb) is_zero) << 64; + return result; +} - if (ecp_sm2p256_bignum_field_elem(k, scalars[i]) <= 0 - || ecp_sm2p256_bignum_field_elem(p.p.X, point[i]->X) <= 0 - || ecp_sm2p256_bignum_field_elem(p.p.Y, point[i]->Y) <= 0 - || ecp_sm2p256_bignum_field_elem(p.p.Z, point[i]->Z) <= 0) { - ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE); - goto err; - } +static int smallfelem_is_zero_int(const void *small) +{ + return (int)(smallfelem_is_zero(small) & ((limb) 1)); +} - ecp_sm2p256_point_get_affine(&t.a, &p.p); - ecp_sm2p256_point_P_mul_by_scalar(&kP, k, t.a); - ecp_sm2p256_point_add(r, r, &kP); +/*- + * felem_inv calculates |out| = |in|^{-1} + * + * Based on Fermat's Little Theorem: + * a^p = a (mod p) + * a^{p-1} = 1 (mod p) + * a^{p-2} = a^{-1} (mod p) + * + * Use Addition chain to obtain a faster algorithm with fewer operations (255S + 14M). + * You can get more information from https://jeit.ac.cn/cn/article/doi/10.11999/JEIT211049 . + */ +static void felem_inv(felem out, const felem in) +{ + felem t0, t1, t2, t3; + felem ftmp; + longfelem tmp; + unsigned i; + + /* Step 1: t0 = a^3 = (2^2 - 2^0) * a */ + felem_square(tmp, in); + felem_reduce(ftmp, tmp); + felem_mul(tmp, ftmp, in); + felem_reduce(t0, tmp); + /* Step 2: t1 = t0^2 * a = (2^3 - 2^0) * a */ + felem_square(tmp, t0); + felem_reduce(ftmp, tmp); + felem_mul(tmp, ftmp, in); + felem_reduce(t1, tmp); + /* Step 3: t2= t1^(2^3) * t1 = (2^6 - 2^0) * a */ + felem_assign(ftmp, t1); + for (i = 0; i < 3; i++) { + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + } + felem_mul(tmp, ftmp, t1); + felem_reduce(t2, tmp); + /* Step 4: t1 = t2^2 = (2^7 - 2^1) * a */ + felem_square(tmp, t2); + felem_reduce(t1, tmp); + /* Step 5: t3 = t1 * z = (2^7 - 2^0) * a */ + felem_mul(tmp, t1, in); + felem_reduce(t3, tmp); + /* Step 6: t1= t1^(2^5) * t2 = (2^12 - 2^0) * a */ + felem_assign(ftmp, t1); + for (i = 0; i < 5; i++) { + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + } + felem_mul(tmp, ftmp, t2); + felem_reduce(t1, tmp); + /* Step 7: t2= t1^(2^12) * t1 = (2^24 - 2^0) * a */ + felem_assign(ftmp, t1); + for (i = 0; i < 12; i++) { + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + } + felem_mul(tmp, ftmp, t1); + felem_reduce(t2, tmp); + /* Step 8: t1= t2^(2^7) * t3 = (2^31 - 2^0) * a */ + felem_assign(ftmp, t2); + for (i = 0; i < 7; i++) { + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + } + felem_mul(tmp, ftmp, t3); + felem_reduce(t1, tmp); + /* Step 9: t2 = t1^4 = (2^33 - 2^2) * a */ + felem_square(tmp, t1); + felem_reduce(ftmp, tmp); + felem_square(tmp, ftmp); + felem_reduce(t2, tmp); + /* Step 10: t3= t2^(2^29) = (2^62 - 2^31) * a */ + felem_assign(t3, t2); + for (i = 0; i < 29; i++) { + felem_square(tmp, t3); + felem_reduce(t3, tmp); + } + /* Step 11: t1 = t1 * t3 = (2^62 - 2^0) * a */ + felem_mul(tmp, t1, t3); + felem_reduce(t1, tmp); + /* Step 12: t3 = t3^4 = (2^64 - 2^33) * a */ + felem_square(tmp, t3); + felem_reduce(t3, tmp); + felem_square(tmp, t3); + felem_reduce(t3, tmp); + /* Step 13: t0 = t0 * t3 * t2 = (2^64 - 2^0) * a */ + felem_mul(tmp, t0, t3); + felem_reduce(t0, tmp); + felem_mul(tmp, t0, t2); + felem_reduce(t0, tmp); + /* Step 14: t2= ((t3^(2^32) * t0)^(2^64) * t0)^(2^94) = (2^254 - 2^222 - 2^94) * a */ + felem_assign(ftmp, t3); + for (i = 0; i < 32; i++) { + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + } + felem_mul(tmp, ftmp, t0); + felem_reduce(ftmp, tmp); + for (i = 0; i < 64; i++) { + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); } - - ret = 1; -err: - OPENSSL_free(scalars); - return ret; + felem_mul(tmp, ftmp, t0); + felem_reduce(ftmp, tmp); + for (i = 0; i < 94; i++) { + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + } + felem_assign(t2, ftmp); + /* Step 15: out = (t1 * t2)^4 * a = (2^256 - 2^224 - 2^96 + 2^64 -1) * a */ + felem_mul(tmp, t1, t2); + felem_reduce(ftmp, tmp); + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + felem_mul(tmp, ftmp, in); + felem_reduce(out, tmp); } -/* r = scalar*G + sum(scalars[i]*points[i]) */ -static int ecp_sm2p256_points_mul(const EC_GROUP *group, - EC_POINT *r, - const BIGNUM *scalar, - size_t num, - const EC_POINT *points[], - const BIGNUM *scalars[], BN_CTX *ctx) -{ - int ret = 0, p_is_infinity = 0; - const EC_POINT *generator = NULL; - ALIGN32 BN_ULONG k[P256_LIMBS] = {0}; - ALIGN32 union { - P256_POINT p; - P256_POINT_AFFINE a; - } t, p; - - if ((num + 1) == 0 || (num + 1) > OPENSSL_MALLOC_MAX_NELEMS(void *)) { - ECerr(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); - goto err; - } +static void smallfelem_inv_contract(smallfelem out, const smallfelem in) +{ + felem tmp; - BN_CTX_start(ctx); + smallfelem_expand(tmp, in); + felem_inv(tmp, tmp); + felem_contract(out, tmp); +} - if (scalar) { - generator = EC_GROUP_get0_generator(group); - if (generator == NULL) { - ECerr(ERR_LIB_EC, EC_R_UNDEFINED_GENERATOR); - goto err; - } +/*- + * Group operations + * ---------------- + * + * Building on top of the field operations we have the operations on the + * elliptic curve group itself. Points on the curve are represented in Jacobian + * coordinates + */ - if (!ecp_sm2p256_bignum_field_elem(k, scalar)) { - ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE); - goto err; - } -#if !defined(OPENSSL_NO_SM2_PRECOMP) - if (ecp_sm2p256_is_affine_G(generator)) { - ecp_sm2p256_point_G_mul_by_scalar(&p.p, k); - } else -#endif - { - /* if no precomputed table */ - const EC_POINT *new_generator[1]; - const BIGNUM *g_scalars[1]; +/*- + * point_double calculates 2*(x_in, y_in, z_in) + * + * The method is taken from: + * http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b + * + * Outputs can equal corresponding inputs, i.e., x_out == x_in is allowed. + * while x_out == y_in is not (maybe this works, but it's not tested). + */ +static void +point_double(felem x_out, felem y_out, felem z_out, + const felem x_in, const felem y_in, const felem z_in) +{ + longfelem tmp, tmp2; + felem delta, gamma, beta, alpha, ftmp, ftmp2; + smallfelem small1, small2; + + felem_assign(ftmp, x_in); + /* ftmp[i] < 2^106 */ + felem_assign(ftmp2, x_in); + /* ftmp2[i] < 2^106 */ + + /* delta = z^2 */ + felem_square(tmp, z_in); + felem_reduce(delta, tmp); + /* delta[i] < 2^101 */ + + /* gamma = y^2 */ + felem_square(tmp, y_in); + felem_reduce(gamma, tmp); + /* gamma[i] < 2^101 */ + felem_shrink(small1, gamma); + + /* beta = x*gamma */ + felem_small_mul(tmp, small1, x_in); + felem_reduce(beta, tmp); + /* beta[i] < 2^101 */ + + /* alpha = 3*(x-delta)*(x+delta) */ + felem_diff(ftmp, delta); + /* ftmp[i] < 2^105 + 2^106 < 2^107 */ + felem_sum(ftmp2, delta); + /* ftmp2[i] < 2^105 + 2^106 < 2^107 */ + felem_scalar(ftmp2, 3); + /* ftmp2[i] < 3 * 2^107 < 2^109 */ + felem_mul(tmp, ftmp, ftmp2); + felem_reduce(alpha, tmp); + /* alpha[i] < 2^101 */ + felem_shrink(small2, alpha); + + /* x' = alpha^2 - 8*beta */ + smallfelem_square(tmp, small2); + felem_reduce(x_out, tmp); + felem_assign(ftmp, beta); + felem_scalar(ftmp, 8); + /* ftmp[i] < 8 * 2^101 = 2^104 */ + felem_diff(x_out, ftmp); + /* x_out[i] < 2^105 + 2^101 < 2^106 */ + + /* z' = (y + z)^2 - gamma - delta */ + felem_sum(delta, gamma); + /* delta[i] < 2^101 + 2^101 = 2^102 */ + felem_assign(ftmp, y_in); + felem_sum(ftmp, z_in); + /* ftmp[i] < 2^106 + 2^106 = 2^107 */ + felem_square(tmp, ftmp); + felem_reduce(z_out, tmp); + felem_diff(z_out, delta); + /* z_out[i] < 2^105 + 2^101 < 2^106 */ + + /* y' = alpha*(4*beta - x') - 8*gamma^2 */ + felem_scalar(beta, 4); + /* beta[i] < 4 * 2^101 = 2^103 */ + felem_diff_zero107(beta, x_out); + /* beta[i] < 2^107 + 2^103 < 2^108 */ + felem_small_mul(tmp, small2, beta); + /* tmp[i] < 7 * 2^64 < 2^67 */ + smallfelem_square(tmp2, small1); + /* tmp2[i] < 7 * 2^64 */ + longfelem_scalar(tmp2, 8); + /* tmp2[i] < 8 * 7 * 2^64 = 7 * 2^67 */ + longfelem_diff(tmp, tmp2); + /* tmp[i] < 2^67 + 2^70 + 2^40 < 2^71 */ + felem_reduce(y_out, tmp); + /* y_out[i] < 2^106 */ +} - new_generator[0] = generator; - g_scalars[0] = scalar; +/* + * point_double_small is the same as point_double, except that it operates on + * smallfelems + */ +static void +point_double_small(smallfelem x_out, smallfelem y_out, smallfelem z_out, + const smallfelem x_in, const smallfelem y_in, + const smallfelem z_in) +{ + felem felem_x_out, felem_y_out, felem_z_out; + felem felem_x_in, felem_y_in, felem_z_in; + + smallfelem_expand(felem_x_in, x_in); + smallfelem_expand(felem_y_in, y_in); + smallfelem_expand(felem_z_in, z_in); + point_double(felem_x_out, felem_y_out, felem_z_out, + felem_x_in, felem_y_in, felem_z_in); + felem_shrink(x_out, felem_x_out); + felem_shrink(y_out, felem_y_out); + felem_shrink(z_out, felem_z_out); +} - if (!ecp_sm2p256_windowed_mul(group, &p.p, g_scalars, new_generator, - (new_generator[0] != NULL - && g_scalars[0] != NULL), ctx)) - goto err; - } - } else { - p_is_infinity = 1; +/* copy_conditional copies in to out iff mask is all ones. */ +static void copy_conditional(felem out, const felem in, limb mask) +{ + unsigned i; + for (i = 0; i < NLIMBS; ++i) { + const limb tmp = mask & (in[i] ^ out[i]); + out[i] ^= tmp; } - if (num) { - P256_POINT *out = &t.p; - - if (p_is_infinity) - out = &p.p; - - if (!ecp_sm2p256_windowed_mul(group, out, scalars, points, num, ctx)) - goto err; +} - if (!p_is_infinity) - ecp_sm2p256_point_add(&p.p, &p.p, out); +/* copy_small_conditional copies in to out iff mask is all ones. */ +static void copy_small_conditional(felem out, const smallfelem in, limb mask) +{ + unsigned i; + const u64 mask64 = mask; + for (i = 0; i < NLIMBS; ++i) { + out[i] = ((limb) (in[i] & mask64)) | (out[i] & ~mask); } - - /* Not constant-time, but we're only operating on the public output. */ - if (!bn_set_words(r->X, p.p.X, P256_LIMBS) - || !bn_set_words(r->Y, p.p.Y, P256_LIMBS) - || !bn_set_words(r->Z, p.p.Z, P256_LIMBS)) - goto err; - r->Z_is_one = is_equal(bn_get_words(r->Z), ONE) & 1; - - ret = 1; -err: - BN_CTX_end(ctx); - return ret; } -static int ecp_sm2p256_field_mul(const EC_GROUP *group, BIGNUM *r, - const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx) +/*- + * point_add calculates (x1, y1, z1) + (x2, y2, z2) + * + * The method is taken from: + * http://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl, + * adapted for mixed addition (z2 = 1, or z2 = 0 for the point at infinity). + * + * This function includes a branch for checking whether the two input points + * are equal, (while not equal to the point at infinity). This case never + * happens during single point multiplication, so there is no timing leak for + * ECDH or ECDSA signing. + */ +static void point_add(felem x3, felem y3, felem z3, + const felem x1, const felem y1, const felem z1, + const int mixed, const smallfelem x2, + const smallfelem y2, const smallfelem z2) { - ALIGN32 BN_ULONG a_fe[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG b_fe[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG r_fe[P256_LIMBS] = {0}; - - if (a == NULL || b == NULL || r == NULL) - return 0; - - if (!ecp_sm2p256_bignum_field_elem(a_fe, a) - || !ecp_sm2p256_bignum_field_elem(b_fe, b)) { - ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE); - return 0; + felem ftmp, ftmp2, ftmp3, ftmp4, ftmp5, ftmp6, x_out, y_out, z_out; + longfelem tmp, tmp2; + smallfelem small1, small2, small3, small4, small5; + limb x_equal, y_equal, z1_is_zero, z2_is_zero; + limb points_equal; + + felem_shrink(small3, z1); + + z1_is_zero = smallfelem_is_zero(small3); + z2_is_zero = smallfelem_is_zero(z2); + + /* ftmp = z1z1 = z1**2 */ + smallfelem_square(tmp, small3); + felem_reduce(ftmp, tmp); + /* ftmp[i] < 2^101 */ + felem_shrink(small1, ftmp); + + if (!mixed) { + /* ftmp2 = z2z2 = z2**2 */ + smallfelem_square(tmp, z2); + felem_reduce(ftmp2, tmp); + /* ftmp2[i] < 2^101 */ + felem_shrink(small2, ftmp2); + + felem_shrink(small5, x1); + + /* u1 = ftmp3 = x1*z2z2 */ + smallfelem_mul(tmp, small5, small2); + felem_reduce(ftmp3, tmp); + /* ftmp3[i] < 2^101 */ + + /* ftmp5 = z1 + z2 */ + felem_assign(ftmp5, z1); + felem_small_sum(ftmp5, z2); + /* ftmp5[i] < 2^107 */ + + /* ftmp5 = (z1 + z2)**2 - (z1z1 + z2z2) = 2z1z2 */ + felem_square(tmp, ftmp5); + felem_reduce(ftmp5, tmp); + /* ftmp2 = z2z2 + z1z1 */ + felem_sum(ftmp2, ftmp); + /* ftmp2[i] < 2^101 + 2^101 = 2^102 */ + felem_diff(ftmp5, ftmp2); + /* ftmp5[i] < 2^105 + 2^101 < 2^106 */ + + /* ftmp2 = z2 * z2z2 */ + smallfelem_mul(tmp, small2, z2); + felem_reduce(ftmp2, tmp); + + /* s1 = ftmp2 = y1 * z2**3 */ + felem_mul(tmp, y1, ftmp2); + felem_reduce(ftmp6, tmp); + /* ftmp6[i] < 2^101 */ + } else { + /* + * We'll assume z2 = 1 (special case z2 = 0 is handled later) + */ + + /* u1 = ftmp3 = x1*z2z2 */ + felem_assign(ftmp3, x1); + /* ftmp3[i] < 2^106 */ + + /* ftmp5 = 2z1z2 */ + felem_assign(ftmp5, z1); + felem_scalar(ftmp5, 2); + /* ftmp5[i] < 2*2^106 = 2^107 */ + + /* s1 = ftmp2 = y1 * z2**3 */ + felem_assign(ftmp6, y1); + /* ftmp6[i] < 2^106 */ } - ecp_sm2p256_mul(r_fe, a_fe, b_fe); - - if (!bn_set_words(r, r_fe, P256_LIMBS)) - return 0; + /* u2 = x2*z1z1 */ + smallfelem_mul(tmp, x2, small1); + felem_reduce(ftmp4, tmp); + + /* h = ftmp4 = u2 - u1 */ + felem_diff_zero107(ftmp4, ftmp3); + /* ftmp4[i] < 2^107 + 2^101 < 2^108 */ + felem_shrink(small4, ftmp4); + + x_equal = smallfelem_is_zero(small4); + + /* z_out = ftmp5 * h */ + felem_small_mul(tmp, small4, ftmp5); + felem_reduce(z_out, tmp); + /* z_out[i] < 2^101 */ + + /* ftmp = z1 * z1z1 */ + smallfelem_mul(tmp, small1, small3); + felem_reduce(ftmp, tmp); + + /* s2 = tmp = y2 * z1**3 */ + felem_small_mul(tmp, y2, ftmp); + felem_reduce(ftmp5, tmp); + + /* r = ftmp5 = (s2 - s1)*2 */ + felem_diff_zero107(ftmp5, ftmp6); + /* ftmp5[i] < 2^107 + 2^107 = 2^108 */ + felem_scalar(ftmp5, 2); + /* ftmp5[i] < 2^109 */ + felem_shrink(small1, ftmp5); + y_equal = smallfelem_is_zero(small1); + + /* + * The formulae are incorrect if the points are equal, in affine coordinates + * (X_1, Y_1) == (X_2, Y_2), so we check for this and do doubling if this + * happens. + * + * We use bitwise operations to avoid potential side-channels introduced by + * the short-circuiting behaviour of boolean operators. + * + * The special case of either point being the point at infinity (z1 and/or + * z2 are zero), is handled separately later on in this function, so we + * avoid jumping to point_double here in those special cases. + */ + points_equal = (x_equal & y_equal & (~z1_is_zero) & (~z2_is_zero)); + + if (points_equal) { + /* + * This is obviously not constant-time but, as mentioned before, this + * case never happens during single point multiplication, so there is no + * timing leak for ECDH or ECDSA signing. + */ + point_double(x3, y3, z3, x1, y1, z1); + return; + } - return 1; + /* I = ftmp = (2h)**2 */ + felem_assign(ftmp, ftmp4); + felem_scalar(ftmp, 2); + /* ftmp[i] < 2*2^108 = 2^109 */ + felem_square(tmp, ftmp); + felem_reduce(ftmp, tmp); + + /* J = ftmp2 = h * I */ + felem_mul(tmp, ftmp4, ftmp); + felem_reduce(ftmp2, tmp); + + /* V = ftmp4 = U1 * I */ + felem_mul(tmp, ftmp3, ftmp); + felem_reduce(ftmp4, tmp); + + /* x_out = r**2 - J - 2V */ + smallfelem_square(tmp, small1); + felem_reduce(x_out, tmp); + felem_assign(ftmp3, ftmp4); + felem_scalar(ftmp4, 2); + felem_sum(ftmp4, ftmp2); + /* ftmp4[i] < 2*2^101 + 2^101 < 2^103 */ + felem_diff(x_out, ftmp4); + /* x_out[i] < 2^105 + 2^101 */ + + /* y_out = r(V-x_out) - 2 * s1 * J */ + felem_diff_zero107(ftmp3, x_out); + /* ftmp3[i] < 2^107 + 2^101 < 2^108 */ + felem_small_mul(tmp, small1, ftmp3); + felem_mul(tmp2, ftmp6, ftmp2); + longfelem_scalar(tmp2, 2); + /* tmp2[i] < 2*2^67 = 2^68 */ + longfelem_diff(tmp, tmp2); + /* tmp[i] < 2^67 + 2^70 + 2^40 < 2^71 */ + felem_reduce(y_out, tmp); + /* y_out[i] < 2^106 */ + + copy_small_conditional(x_out, x2, z1_is_zero); + copy_conditional(x_out, x1, z2_is_zero); + copy_small_conditional(y_out, y2, z1_is_zero); + copy_conditional(y_out, y1, z2_is_zero); + copy_small_conditional(z_out, z2, z1_is_zero); + copy_conditional(z_out, z1, z2_is_zero); + felem_assign(x3, x_out); + felem_assign(y3, y_out); + felem_assign(z3, z_out); } -static int ecp_sm2p256_field_sqr(const EC_GROUP *group, BIGNUM *r, - const BIGNUM *a, BN_CTX *ctx) +/* + * point_add_small is the same as point_add, except that it operates on + * smallfelems + */ +static void point_add_small(smallfelem x3, smallfelem y3, smallfelem z3, + smallfelem x1, smallfelem y1, smallfelem z1, + smallfelem x2, smallfelem y2, smallfelem z2) { - ALIGN32 BN_ULONG a_fe[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG r_fe[P256_LIMBS] = {0}; + felem felem_x3, felem_y3, felem_z3; + felem felem_x1, felem_y1, felem_z1; + smallfelem_expand(felem_x1, x1); + smallfelem_expand(felem_y1, y1); + smallfelem_expand(felem_z1, z1); + point_add(felem_x3, felem_y3, felem_z3, felem_x1, felem_y1, felem_z1, 0, + x2, y2, z2); + felem_shrink(x3, felem_x3); + felem_shrink(y3, felem_y3); + felem_shrink(z3, felem_z3); +} - if (a == NULL || r == NULL) - return 0; +/*- + * Base point pre computation + * -------------------------- + * + * Two different sorts of precomputed tables are used in the following code. + * Each contain various points on the curve, where each point is three field + * elements (x, y, z). + * + * For the base point table, z is usually 1 (0 for the point at infinity). + * This table has 2 * 16 elements, starting with the following: + * index | bits | point + * ------+---------+------------------------------ + * 0 | 0 0 0 0 | 0G + * 1 | 0 0 0 1 | 1G + * 2 | 0 0 1 0 | 2^64G + * 3 | 0 0 1 1 | (2^64 + 1)G + * 4 | 0 1 0 0 | 2^128G + * 5 | 0 1 0 1 | (2^128 + 1)G + * 6 | 0 1 1 0 | (2^128 + 2^64)G + * 7 | 0 1 1 1 | (2^128 + 2^64 + 1)G + * 8 | 1 0 0 0 | 2^192G + * 9 | 1 0 0 1 | (2^192 + 1)G + * 10 | 1 0 1 0 | (2^192 + 2^64)G + * 11 | 1 0 1 1 | (2^192 + 2^64 + 1)G + * 12 | 1 1 0 0 | (2^192 + 2^128)G + * 13 | 1 1 0 1 | (2^192 + 2^128 + 1)G + * 14 | 1 1 1 0 | (2^192 + 2^128 + 2^64)G + * 15 | 1 1 1 1 | (2^192 + 2^128 + 2^64 + 1)G + * followed by a copy of this with each element multiplied by 2^32. + * + * The reason for this is so that we can clock bits into four different + * locations when doing simple scalar multiplies against the base point, + * and then another four locations using the second 16 elements. + * + * Tables for other points have table[i] = iG for i in 0 .. 16. */ + +/* gmul is the table of precomputed base points */ +static const smallfelem gmul[2][16][3] = { + {{{0, 0, 0, 0}, + {0, 0, 0, 0}, + {0, 0, 0, 0}}, + {{0x715a4589334c74c7, 0x8fe30bbff2660be1, 0x5f9904466a39c994, + 0x32c4ae2c1f198119}, + {0x2df32e52139f0a0, 0xd0a9877cc62a4740, 0x59bdcee36b692153, + 0xbc3736a2f4f6779c}, + {1, 0, 0, 0}}, + {{0xe18bd546b5824517, 0x673891d791caa486, 0xba220b99df9f9a14, + 0x95afbd1155c1da54}, + {0x8e4450eb334acdcb, 0xc3c7d1898a53f20d, 0x2eee750f4053017c, + 0xe8a6d82c517388c2}, + {1, 0, 0, 0}}, + {{0xf81c8da9b99fba55, 0x137f6c6149feef6e, 0xcb129aa494da9ad4, + 0x82a0f5407d123db6}, + {0xfdeca00772c4dbc9, 0xa961b58f0cf58373, 0xecacab94e973f9c3, + 0xf12fa4696a22ca3f}, + {1, 0, 0, 0}}, + {{0xeae3d9a9d13a42ed, 0x2b2308f6484e1b38, 0x3db7b24888c21f3a, + 0xb692e5b574d55da9}, + {0xd186469de295e5ab, 0xdb61ac1773438e6d, 0x5a924f85544926f9, + 0xa175051b0f3fb613}, + {1, 0, 0, 0}}, + {{0xa72d084f62c8d58b, 0xe3d6467deaf48fd7, 0x8fe75e5a128a56a7, + 0xc0023fe7ff2b68bd}, + {0x64f67782316815f9, 0xb52b6d9b19a69cd2, 0x5d1ed6fa89cbbade, + 0x796c910ee7f4ccdb}, + {1, 0, 0, 0}}, + {{0x1b2150c1c5f13015, 0xdaaba91b5d952c9b, 0xe8cc24c3f546142, + 0x75a34b243705f260}, + {0x77d195421cef1339, 0x636644aa0c3a0623, 0x4683df176eeb2444, + 0x642ce3bd3535e74d}, + {1, 0, 0, 0}}, + {{0x4a59ac2c6e7ecc08, 0xaf2b71164f191d63, 0x3622a87fb284554f, + 0xd9eb397b441e9cd0}, + {0xa66b8a4893b6a54d, 0x26fb89a40b4a663a, 0xafa87501eedfc9f4, + 0xf3f000bc66f98108}, + {1, 0, 0, 0}}, + {{0xad8bc68ce031d616, 0x16888d8ee4003187, 0x44c0757f3bb8b600, + 0x793fae7af0164245}, + {0x210cd042973f333b, 0x8666ff52dbd25f9, 0x65c5b129f5f7ad5d, + 0xe03d7a8d19b3219a}, + {1, 0, 0, 0}}, + {{0xd68bfbace0e00392, 0x261014f7d3445dc7, 0xd9f46b2714a071ee, + 0x1b200af30810b682}, + {0xd91d8b12ae69bcd, 0x74a08f17bf8cd981, 0xd822913cf0d2b82d, + 0x248b7af0b05bfad2}, + {1, 0, 0, 0}}, + {{0xba119a049e62f2e2, 0xf278e8a34df05ae5, 0xd269f3564eb5d180, + 0x8e74ad0f4f957cb1}, + {0x112ff4dabd76e2dd, 0x91373f20630fdb7f, 0xf43eab474992904c, + 0x55a5ccc7af3b6db4}, + {1, 0, 0, 0}}, + {{0x5ad104a8bdd23de9, 0xf5a9e515eb71c2c1, 0x390542a0ba95c174, + 0x4c55fb20426491bf}, + {0x91525735ef626289, 0xd2ed977f88f09635, 0xfd48731b7a8a8521, + 0x8f89a03b8fdebea}, + {1, 0, 0, 0}}, + {{0x7e8e61ea35eb8e2e, 0x1bb2700db98a762c, 0xd81ea23b7738c17c, + 0xf9def2a46dba26a3}, + {0x183a7912d05e329f, 0x34664a0896ccde0e, 0x56c22652614283bb, + 0x91692899d5ff0513}, + {1, 0, 0, 0}}, + {{0x449d48d8f3bdbe19, 0xab95de03cc8510cb, 0xaef159463f8bfb25, + 0xda72c379dae3ca8b}, + {0xcba9315ce82cc3ea, 0x4e524bac38a58020, 0x36ba2752538e348c, + 0xb170d0da75ed450f}, + {1, 0, 0, 0}}, + {{0x947af0f52b4f8da6, 0x7eda17d917827976, 0x5ba79a0c705853a0, + 0xa5d9873b3fb2ddc7}, + {0xc2a48162a5fd9ce9, 0x80ee8ae526f25f02, 0xf60c8ef6633be6a9, + 0xe2e23f0229a84a35}, + {1, 0, 0, 0}}, + {{0xbc4945bd86bb6afb, 0x237eb711eba46fee, 0x7c1db58b7b86eb33, + 0xd94eb728273b3ac7}, + {0xbe1717e59568d0a4, 0x4a6067cc45f70212, 0x19b32eb5afc2fb17, + 0xbe3c1e7ac3ac9d3c}, + {1, 0, 0, 0}}}, + {{{0, 0, 0, 0}, + {0, 0, 0, 0}, + {0, 0, 0, 0}}, + {{0x68a88405ae53c1e9, 0x51e46707fd558656, 0x71e834cf86896c10, + 0x3d251b54e10d581f}, + {0x1884d5b0eeb19032, 0xeeaf729853e526fe, 0x5931f6831a8d8c11, + 0x87891d33fb98b4d8}, + {1, 0, 0, 0}}, + {{0x9047673fcac14893, 0xf5df5d83bfb58659, 0xa6230c81642e71a, + 0xef14b33800777791}, + {0xcf1e99afa3386fca, 0x7ace937791313d53, 0x36fe159b6dcd01bb, + 0xc9bc50d02e2b960a}, + {1, 0, 0, 0}}, + {{0x716e5a7ee12e162d, 0xbbf9bb2c62dd5a00, 0xca235ccb4144dd05, + 0xbcb7de0f8f70520e}, + {0x981e8964947cb8eb, 0x53c7102ea04de08d, 0xe9076332afc6a10d, + 0x93d90f776b58c35d}, + {1, 0, 0, 0}}, + {{0x834dbff6678337ee, 0xc607e811fef0785a, 0xaaefc62be30a298b, + 0xeb5ca335326afad3}, + {0x9774fe1384af54a8, 0xca4b6ef5785388b4, 0x1346c82d66f6c642, + 0xedcc0c2aaa2d53ce}, + {1, 0, 0, 0}}, + {{0xb896b3f764b9e6f4, 0x47e4018c736fb3d0, 0xfc2fc86707413920, + 0x1a8526428e1aeae7}, + {0x1386802650e2ae60, 0x7474dedc995384d0, 0x2c4cc396dd43b011, + 0x63b0e9c7141de1b0}, + {1, 0, 0, 0}}, + {{0xeb5fb3b369d17771, 0x1fe07b18933ed257, 0xdfc4c81ce3673912, + 0x913614c66a91a647}, + {0x18aee853c0ba877f, 0x3109c2deceff091, 0x8532307e7e4ee08c, + 0xcef0791a6e6ce0bb}, + {1, 0, 0, 0}}, + {{0xf0e9f5d8057a4a0f, 0xbbf7f8b49f125aa9, 0x51e8fdd6283187c2, + 0xe0997d4759d36298}, + {0x67ec3c5c6f4221c3, 0x3ea275dbc860722f, 0x152d01e23859f5e2, + 0xfb57404312680f44}, + {1, 0, 0, 0}}, + {{0x21ac3df849be2a1f, 0x11006e9fc51d112f, 0x9151aa584775c857, + 0x5159d218ba04a8d9}, + {0x98b7d1a925fd1866, 0x8f4753cafc2ad9d8, 0x8eb91ec1569c05a9, + 0x4abbd1ae27e13f11}, + {1, 0, 0, 0}}, + {{0x616f6644b2c11f4c, 0x251cd7140e540758, 0xf927a40110f02017, + 0x92ff3cc3c1c941b6}, + {0x3249906213f565fe, 0x4633e3ddeb9dbd4e, 0xea9a9d1ec402e6c2, + 0xdc84ce34b14bb7cf}, + {1, 0, 0, 0}}, + {{0xa93e23e5436ff69a, 0x52dcb0a79b63efce, 0x34f6538a9e90cb41, + 0x9cac08f200234bc0}, + {0x6661825b5174a02d, 0x7d4d06de036be57, 0x589d74610ae6bd27, + 0xa296f5577fc91a93}, + {1, 0, 0, 0}}, + {{0x10acefa9d29721d0, 0x8b0f6b8bb5bcd340, 0x921d318c3d86785c, + 0xd6916f3bc16aa378}, + {0x2a0d646a7ad84a0e, 0x7b93256c2fe7e97a, 0x5765e27626479e41, + 0xae9da2272daaced3}, + {1, 0, 0, 0}}, + {{0x56fdc215f7f34ac5, 0xebcb4ff2da3877d3, 0x1eb96792aba6b832, + 0x807ce6bea24741aa}, + {0xff1c10109c721fb4, 0xd187d4bc796353a7, 0x7639ae749af2d303, + 0xaff6d783d56c9286}, + {1, 0, 0, 0}}, + {{0x6002d51b6290dd01, 0xcba3ab0099a836a5, 0x71776611e00d2528, + 0xfaf2cb8c87fce119}, + {0xd445228bdf6882ae, 0xcbbfade17cbce919, 0x837b6335a2eb2453, + 0x11ad7c4b8597f6b6}, + {1, 0, 0, 0}}, + {{0x48de8f368cf2e399, 0x7ae3d25630a74277, 0xdef1a9a6c505323f, + 0xe55f203b4b8d9672}, + {0xc58d8f0d9a1e6e97, 0xe160e6d4b2737a76, 0xd60bd087d47cbdd8, + 0x687d41364d5fef53}, + {1, 0, 0, 0}}, + {{0x83f21bbe056bbf9b, 0x4c2a9d120b4ba5ab, 0xff383d1845b64e4f, + 0x8f13cc8d06dd7867}, + {0xf3a292d8424f0995, 0xfd2546eae7cbe44b, 0x67d14dee6c1e75a3, + 0x53b49e6cc93fb5a8}, + {1, 0, 0, 0}}} +}; - if (!ecp_sm2p256_bignum_field_elem(a_fe, a)) { - ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE); - return 0; +/* + * select_point selects the |idx|th point from a precomputation table and + * copies it to out. + */ +static void select_point(const u64 idx, unsigned int size, + const smallfelem pre_comp[16][3], smallfelem out[3]) +{ + unsigned i, j; + u64 *outlimbs = &out[0][0]; + + memset(out, 0, sizeof(*out) * 3); + + for (i = 0; i < size; i++) { + const u64 *inlimbs = (u64 *)&pre_comp[i][0][0]; + u64 mask = i ^ idx; + mask |= mask >> 4; + mask |= mask >> 2; + mask |= mask >> 1; + mask &= 1; + mask--; + for (j = 0; j < NLIMBS * 3; j++) + outlimbs[j] |= inlimbs[j] & mask; } +} - ecp_sm2p256_sqr(r_fe, a_fe); - - if (!bn_set_words(r, r_fe, P256_LIMBS)) +/* get_bit returns the |i|th bit in |in| */ +static char get_bit(const felem_bytearray in, int i) +{ + if ((i < 0) || (i >= 256)) return 0; - - return 1; + return (in[i >> 3] >> (i & 7)) & 1; } -static int ecp_sm2p256_inv_mod_ord(const EC_GROUP *group, BIGNUM *r, - const BIGNUM *x, BN_CTX *ctx) +/* + * Interleaved point multiplication using precomputed point multiples: The + * small point multiples 0*P, 1*P, ..., 17*P are in pre_comp[], the scalars + * in scalars[]. If g_scalar is non-NULL, we also add this multiple of the + * generator, using certain (large) precomputed multiples in g_pre_comp. + * Output point (X, Y, Z) is stored in x_out, y_out, z_out + */ +static void batch_mul(felem x_out, felem y_out, felem z_out, + const felem_bytearray scalars[], + const unsigned num_points, const u8 *g_scalar, + const int mixed, const smallfelem pre_comp[][17][3], + const smallfelem g_pre_comp[2][16][3]) { - int ret = 0; - ALIGN32 BN_ULONG t[P256_LIMBS] = {0}; - ALIGN32 BN_ULONG out[P256_LIMBS] = {0}; - - if (bn_wexpand(r, P256_LIMBS) == NULL) { - ECerr(ERR_LIB_EC, ERR_R_BN_LIB); - goto err; - } - - if ((BN_num_bits(x) > 256) || BN_is_negative(x)) { - BIGNUM *tmp; + int i, skip; + unsigned num, gen_mul = (g_scalar != NULL); + felem nq[3], ftmp; + smallfelem tmp[3]; + u64 bits; + u8 sign, digit; + + /* set nq to the point at infinity */ + memset(nq, 0, sizeof(nq)); + + /* + * Loop over all scalars msb-to-lsb, interleaving additions of multiples + * of the generator (two in each of the last 32 rounds) and additions of + * other points multiples (every 5th round). + */ + skip = 1; /* save two point operations in the first + * round */ + for (i = (num_points ? 255 : 31); i >= 0; --i) { + /* double */ + if (!skip) + point_double(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2]); + + /* add multiples of the generator */ + if (gen_mul && (i <= 31)) { + /* first, look 32 bits upwards */ + bits = get_bit(g_scalar, i + 224) << 3; + bits |= get_bit(g_scalar, i + 160) << 2; + bits |= get_bit(g_scalar, i + 96) << 1; + bits |= get_bit(g_scalar, i + 32); + /* select the point to add, in constant time */ + select_point(bits, 16, g_pre_comp[1], tmp); + + if (!skip) { + /* Arg 1 below is for "mixed" */ + point_add(nq[0], nq[1], nq[2], + nq[0], nq[1], nq[2], 1, tmp[0], tmp[1], tmp[2]); + } else { + smallfelem_expand(nq[0], tmp[0]); + smallfelem_expand(nq[1], tmp[1]); + smallfelem_expand(nq[2], tmp[2]); + skip = 0; + } - if ((tmp = BN_CTX_get(ctx)) == NULL - || !BN_nnmod(tmp, x, group->order, ctx)) { - ECerr(ERR_LIB_EC, ERR_R_BN_LIB); - goto err; + /* second, look at the current position */ + bits = get_bit(g_scalar, i + 192) << 3; + bits |= get_bit(g_scalar, i + 128) << 2; + bits |= get_bit(g_scalar, i + 64) << 1; + bits |= get_bit(g_scalar, i); + /* select the point to add, in constant time */ + select_point(bits, 16, g_pre_comp[0], tmp); + /* Arg 1 below is for "mixed" */ + point_add(nq[0], nq[1], nq[2], + nq[0], nq[1], nq[2], 1, tmp[0], tmp[1], tmp[2]); } - x = tmp; - } - if (!ecp_sm2p256_bignum_field_elem(t, x)) { - ECerr(ERR_LIB_EC, EC_R_COORDINATES_OUT_OF_RANGE); - goto err; + /* do other additions every 5 doublings */ + if (num_points && (i % 5 == 0)) { + /* loop over all scalars */ + for (num = 0; num < num_points; ++num) { + bits = get_bit(scalars[num], i + 4) << 5; + bits |= get_bit(scalars[num], i + 3) << 4; + bits |= get_bit(scalars[num], i + 2) << 3; + bits |= get_bit(scalars[num], i + 1) << 2; + bits |= get_bit(scalars[num], i) << 1; + bits |= get_bit(scalars[num], i - 1); + ossl_ec_GFp_nistp_recode_scalar_bits(&sign, &digit, bits); + + /* + * select the point to add or subtract, in constant time + */ + select_point(digit, 17, pre_comp[num], tmp); + smallfelem_neg(ftmp, tmp[1]); /* (X, -Y, Z) is the negative + * point */ + copy_small_conditional(ftmp, tmp[1], (((limb) sign) - 1)); + felem_contract(tmp[1], ftmp); + + if (!skip) { + point_add(nq[0], nq[1], nq[2], + nq[0], nq[1], nq[2], + mixed, tmp[0], tmp[1], tmp[2]); + } else { + smallfelem_expand(nq[0], tmp[0]); + smallfelem_expand(nq[1], tmp[1]); + smallfelem_expand(nq[2], tmp[2]); + skip = 0; + } + } + } } - - ecp_sm2p256_mod_ord_inverse(out, t); - - if (!bn_set_words(r, out, P256_LIMBS)) - goto err; - - ret = 1; -err: - return ret; + felem_assign(x_out, nq[0]); + felem_assign(y_out, nq[1]); + felem_assign(z_out, nq[2]); } +/* Precomputation for the group generator. */ +struct sm2p256_pre_comp_st { + smallfelem g_pre_comp[2][16][3]; + CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; +}; + const EC_METHOD *EC_GFp_sm2p256_method(void) { static const EC_METHOD ret = { EC_FLAGS_DEFAULT_OCT, NID_X9_62_prime_field, - ossl_ec_GFp_simple_group_init, + ossl_ec_GFp_sm2p256_group_init, ossl_ec_GFp_simple_group_finish, ossl_ec_GFp_simple_group_clear_finish, - ossl_ec_GFp_simple_group_copy, - ossl_ec_GFp_simple_group_set_curve, + ossl_ec_GFp_nist_group_copy, + ossl_ec_GFp_sm2p256_group_set_curve, ossl_ec_GFp_simple_group_get_curve, ossl_ec_GFp_simple_group_get_degree, ossl_ec_group_simple_order_bits, @@ -747,8 +1734,10 @@ const EC_METHOD *EC_GFp_sm2p256_method(void) ossl_ec_GFp_simple_point_copy, ossl_ec_GFp_simple_point_set_to_infinity, ossl_ec_GFp_simple_point_set_affine_coordinates, - ecp_sm2p256_get_affine, - 0, 0, 0, + ossl_ec_GFp_sm2p256_point_get_affine_coordinates, + 0 /* point_set_compressed_coordinates */ , + 0 /* point2oct */ , + 0 /* oct2point */ , ossl_ec_GFp_simple_add, ossl_ec_GFp_simple_dbl, ossl_ec_GFp_simple_invert, @@ -757,16 +1746,16 @@ const EC_METHOD *EC_GFp_sm2p256_method(void) ossl_ec_GFp_simple_cmp, ossl_ec_GFp_simple_make_affine, ossl_ec_GFp_simple_points_make_affine, - ecp_sm2p256_points_mul, /* mul */ - 0 /* precompute_mult */, - 0 /* have_precompute_mult */, - ecp_sm2p256_field_mul, - ecp_sm2p256_field_sqr, - 0 /* field_div */, - 0 /* field_inv */, - 0 /* field_encode */, - 0 /* field_decode */, - 0 /* field_set_to_one */, + ossl_ec_GFp_sm2p256_points_mul, + ossl_ec_GFp_sm2p256_precompute_mult, + ossl_ec_GFp_sm2p256_have_precompute_mult, + ossl_ec_GFp_nist_field_mul, + ossl_ec_GFp_nist_field_sqr, + 0 /* field_div */ , + ossl_ec_GFp_simple_field_inv, + 0 /* field_encode */ , + 0 /* field_decode */ , + 0, /* field_set_to_one */ ossl_ec_key_simple_priv2oct, ossl_ec_key_simple_oct2priv, 0, /* set private */ @@ -779,7 +1768,7 @@ const EC_METHOD *EC_GFp_sm2p256_method(void) ossl_ecdsa_simple_sign_setup, ossl_ecdsa_simple_sign_sig, ossl_ecdsa_simple_verify_sig, - ecp_sm2p256_inv_mod_ord, + 0, /* field_inverse_mod_ord */ 0, /* blind_coordinates */ 0, /* ladder_pre */ 0, /* ladder_step */ @@ -788,3 +1777,540 @@ const EC_METHOD *EC_GFp_sm2p256_method(void) return &ret; } + +/******************************************************************************/ +/* + * FUNCTIONS TO MANAGE PRECOMPUTATION + */ + +static SM2P256_PRE_COMP *sm2p256_pre_comp_new(void) +{ + SM2P256_PRE_COMP *ret = OPENSSL_zalloc(sizeof(*ret)); + + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + return ret; + } + + ret->references = 1; + + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + OPENSSL_free(ret); + return NULL; + } + return ret; +} + +SM2P256_PRE_COMP *EC_sm2p256_pre_comp_dup(SM2P256_PRE_COMP *p) +{ + int i; + if (p != NULL) + CRYPTO_UP_REF(&p->references, &i, p->lock); + return p; +} + +void EC_sm2p256_pre_comp_free(SM2P256_PRE_COMP *pre) +{ + int i; + + if (pre == NULL) + return; + + CRYPTO_DOWN_REF(&pre->references, &i, pre->lock); + REF_PRINT_COUNT("EC_sm2p256", pre); + if (i > 0) + return; + REF_ASSERT_ISNT(i < 0); + + CRYPTO_THREAD_lock_free(pre->lock); + OPENSSL_free(pre); +} + +/******************************************************************************/ +/* + * OPENSSL EC_METHOD FUNCTIONS + */ + +int ossl_ec_GFp_sm2p256_group_init(EC_GROUP *group) +{ + int ret; + ret = ossl_ec_GFp_simple_group_init(group); + group->a_is_minus3 = 1; + return ret; +} + +int ossl_ec_GFp_sm2p256_group_set_curve(EC_GROUP *group, const BIGNUM *p, + const BIGNUM *a, const BIGNUM *b, + BN_CTX *ctx) +{ + int ret = 0; + BIGNUM *curve_p, *curve_a, *curve_b; +#ifndef FIPS_MODULE + BN_CTX *new_ctx = NULL; + + if (ctx == NULL) + ctx = new_ctx = BN_CTX_new(); +#endif + if (ctx == NULL) + return 0; + + BN_CTX_start(ctx); + curve_p = BN_CTX_get(ctx); + curve_a = BN_CTX_get(ctx); + curve_b = BN_CTX_get(ctx); + if (curve_b == NULL) + goto err; + BN_bin2bn(sm2p256_curve_params[0], sizeof(felem_bytearray), curve_p); + BN_bin2bn(sm2p256_curve_params[1], sizeof(felem_bytearray), curve_a); + BN_bin2bn(sm2p256_curve_params[2], sizeof(felem_bytearray), curve_b); + if ((BN_cmp(curve_p, p)) || (BN_cmp(curve_a, a)) || (BN_cmp(curve_b, b))) { + ERR_raise(ERR_LIB_EC, EC_R_WRONG_CURVE_PARAMETERS); + goto err; + } + group->field_mod_func = BN_sm2_mod_256; + ret = ossl_ec_GFp_simple_group_set_curve(group, p, a, b, ctx); + err: + BN_CTX_end(ctx); +#ifndef FIPS_MODULE + BN_CTX_free(new_ctx); +#endif + return ret; +} + +/* + * Takes the Jacobian coordinates (X, Y, Z) of a point and returns (X', Y') = + * (X/Z^2, Y/Z^3) + */ +int ossl_ec_GFp_sm2p256_point_get_affine_coordinates(const EC_GROUP *group, + const EC_POINT *point, + BIGNUM *x, BIGNUM *y, + BN_CTX *ctx) +{ + felem z1, z2, x_in, y_in; + smallfelem x_out, y_out; + longfelem tmp; + + if (EC_POINT_is_at_infinity(group, point)) { + ERR_raise(ERR_LIB_EC, EC_R_POINT_AT_INFINITY); + return 0; + } + if ((!BN_to_felem(x_in, point->X)) || (!BN_to_felem(y_in, point->Y)) || + (!BN_to_felem(z1, point->Z))) + return 0; + felem_inv(z2, z1); + felem_square(tmp, z2); + felem_reduce(z1, tmp); + felem_mul(tmp, x_in, z1); + felem_reduce(x_in, tmp); + felem_contract(x_out, x_in); + if (x != NULL) { + if (!smallfelem_to_BN(x, x_out)) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + return 0; + } + } + felem_mul(tmp, z1, z2); + felem_reduce(z1, tmp); + felem_mul(tmp, y_in, z1); + felem_reduce(y_in, tmp); + felem_contract(y_out, y_in); + if (y != NULL) { + if (!smallfelem_to_BN(y, y_out)) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + return 0; + } + } + return 1; +} + +/* points below is of size |num|, and tmp_smallfelems is of size |num+1| */ +static void make_points_affine(size_t num, smallfelem points[][3], + smallfelem tmp_smallfelems[]) +{ + /* + * Runs in constant time, unless an input is the point at infinity (which + * normally shouldn't happen). + */ + ossl_ec_GFp_nistp_points_make_affine_internal(num, + points, + sizeof(smallfelem), + tmp_smallfelems, + (void (*)(void *))smallfelem_one, + smallfelem_is_zero_int, + (void (*)(void *, const void *)) + smallfelem_assign, + (void (*)(void *, const void *)) + smallfelem_square_contract, + (void (*) + (void *, const void *, + const void *)) + smallfelem_mul_contract, + (void (*)(void *, const void *)) + smallfelem_inv_contract, + /* nothing to contract */ + (void (*)(void *, const void *)) + smallfelem_assign); +} + +/* + * Computes scalar*generator + \sum scalars[i]*points[i], ignoring NULL + * values Result is stored in r (r can equal one of the inputs). + */ +int ossl_ec_GFp_sm2p256_points_mul(const EC_GROUP *group, EC_POINT *r, + const BIGNUM *scalar, size_t num, + const EC_POINT *points[], + const BIGNUM *scalars[], BN_CTX *ctx) +{ + int ret = 0; + int j; + int mixed = 0; + BIGNUM *x, *y, *z, *tmp_scalar; + felem_bytearray g_secret; + felem_bytearray *secrets = NULL; + smallfelem (*pre_comp)[17][3] = NULL; + smallfelem *tmp_smallfelems = NULL; + unsigned i; + int num_bytes; + int have_pre_comp = 0; + size_t num_points = num; + smallfelem x_in, y_in, z_in; + felem x_out, y_out, z_out; + SM2P256_PRE_COMP *pre = NULL; + const smallfelem(*g_pre_comp)[16][3] = NULL; + EC_POINT *generator = NULL; + const EC_POINT *p = NULL; + const BIGNUM *p_scalar = NULL; + + BN_CTX_start(ctx); + x = BN_CTX_get(ctx); + y = BN_CTX_get(ctx); + z = BN_CTX_get(ctx); + tmp_scalar = BN_CTX_get(ctx); + if (tmp_scalar == NULL) + goto err; + + if (scalar != NULL) { + pre = group->pre_comp.sm2p256; + if (pre) + /* we have precomputation, try to use it */ + g_pre_comp = (const smallfelem(*)[16][3])pre->g_pre_comp; + else + /* try to use the standard precomputation */ + g_pre_comp = &gmul[0]; + generator = EC_POINT_new(group); + if (generator == NULL) + goto err; + /* get the generator from precomputation */ + if (!smallfelem_to_BN(x, g_pre_comp[0][1][0]) || + !smallfelem_to_BN(y, g_pre_comp[0][1][1]) || + !smallfelem_to_BN(z, g_pre_comp[0][1][2])) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + if (!ossl_ec_GFp_simple_set_Jprojective_coordinates_GFp(group, + generator, + x, y, z, ctx)) + goto err; + if (0 == EC_POINT_cmp(group, generator, group->generator, ctx)) + /* precomputation matches generator */ + have_pre_comp = 1; + else + /* + * we don't have valid precomputation: treat the generator as a + * random point + */ + num_points++; + } + if (num_points > 0) { + if (num_points >= 3) { + /* + * unless we precompute multiples for just one or two points, + * converting those into affine form is time well spent + */ + mixed = 1; + } + secrets = OPENSSL_malloc(sizeof(*secrets) * num_points); + pre_comp = OPENSSL_malloc(sizeof(*pre_comp) * num_points); + if (mixed) + tmp_smallfelems = + OPENSSL_malloc(sizeof(*tmp_smallfelems) * (num_points * 17 + 1)); + if ((secrets == NULL) || (pre_comp == NULL) + || (mixed && (tmp_smallfelems == NULL))) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + /* + * we treat NULL scalars as 0, and NULL points as points at infinity, + * i.e., they contribute nothing to the linear combination + */ + memset(secrets, 0, sizeof(*secrets) * num_points); + memset(pre_comp, 0, sizeof(*pre_comp) * num_points); + for (i = 0; i < num_points; ++i) { + if (i == num) { + /* + * we didn't have a valid precomputation, so we pick the + * generator + */ + p = EC_GROUP_get0_generator(group); + p_scalar = scalar; + } else { + /* the i^th point */ + p = points[i]; + p_scalar = scalars[i]; + } + if ((p_scalar != NULL) && (p != NULL)) { + /* reduce scalar to 0 <= scalar < 2^256 */ + if ((BN_num_bits(p_scalar) > 256) + || (BN_is_negative(p_scalar))) { + /* + * this is an unusual input, and we don't guarantee + * constant-timeness + */ + if (!BN_nnmod(tmp_scalar, p_scalar, group->order, ctx)) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + num_bytes = BN_bn2lebinpad(tmp_scalar, + secrets[i], sizeof(secrets[i])); + } else { + num_bytes = BN_bn2lebinpad(p_scalar, + secrets[i], sizeof(secrets[i])); + } + if (num_bytes < 0) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + /* precompute multiples */ + if ((!BN_to_felem(x_out, p->X)) || + (!BN_to_felem(y_out, p->Y)) || + (!BN_to_felem(z_out, p->Z))) + goto err; + felem_shrink(pre_comp[i][1][0], x_out); + felem_shrink(pre_comp[i][1][1], y_out); + felem_shrink(pre_comp[i][1][2], z_out); + for (j = 2; j <= 16; ++j) { + if (j & 1) { + point_add_small(pre_comp[i][j][0], pre_comp[i][j][1], + pre_comp[i][j][2], pre_comp[i][1][0], + pre_comp[i][1][1], pre_comp[i][1][2], + pre_comp[i][j - 1][0], + pre_comp[i][j - 1][1], + pre_comp[i][j - 1][2]); + } else { + point_double_small(pre_comp[i][j][0], + pre_comp[i][j][1], + pre_comp[i][j][2], + pre_comp[i][j / 2][0], + pre_comp[i][j / 2][1], + pre_comp[i][j / 2][2]); + } + } + } + } + if (mixed) + make_points_affine(num_points * 17, pre_comp[0], tmp_smallfelems); + } + + /* the scalar for the generator */ + if ((scalar != NULL) && (have_pre_comp)) { + memset(g_secret, 0, sizeof(g_secret)); + /* reduce scalar to 0 <= scalar < 2^256 */ + if ((BN_num_bits(scalar) > 256) || (BN_is_negative(scalar))) { + /* + * this is an unusual input, and we don't guarantee + * constant-timeness + */ + if (!BN_nnmod(tmp_scalar, scalar, group->order, ctx)) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + num_bytes = BN_bn2lebinpad(tmp_scalar, g_secret, sizeof(g_secret)); + } else { + num_bytes = BN_bn2lebinpad(scalar, g_secret, sizeof(g_secret)); + } + /* do the multiplication with generator precomputation */ + batch_mul(x_out, y_out, z_out, + (const felem_bytearray(*))secrets, num_points, + g_secret, + mixed, (const smallfelem(*)[17][3])pre_comp, g_pre_comp); + } else { + /* do the multiplication without generator precomputation */ + batch_mul(x_out, y_out, z_out, + (const felem_bytearray(*))secrets, num_points, + NULL, mixed, (const smallfelem(*)[17][3])pre_comp, NULL); + } + /* reduce the output to its unique minimal representation */ + felem_contract(x_in, x_out); + felem_contract(y_in, y_out); + felem_contract(z_in, z_out); + if ((!smallfelem_to_BN(x, x_in)) || (!smallfelem_to_BN(y, y_in)) || + (!smallfelem_to_BN(z, z_in))) { + ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + goto err; + } + ret = ossl_ec_GFp_simple_set_Jprojective_coordinates_GFp(group, r, x, y, z, + ctx); + + err: + BN_CTX_end(ctx); + EC_POINT_free(generator); + OPENSSL_free(secrets); + OPENSSL_free(pre_comp); + OPENSSL_free(tmp_smallfelems); + return ret; +} + +int ossl_ec_GFp_sm2p256_precompute_mult(EC_GROUP *group, BN_CTX *ctx) +{ + int ret = 0; + SM2P256_PRE_COMP *pre = NULL; + int i, j; + BIGNUM *x, *y; + EC_POINT *generator = NULL; + smallfelem tmp_smallfelems[32]; + felem x_tmp, y_tmp, z_tmp; +#ifndef FIPS_MODULE + BN_CTX *new_ctx = NULL; +#endif + + /* throw away old precomputation */ + EC_pre_comp_free(group); + +#ifndef FIPS_MODULE + if (ctx == NULL) + ctx = new_ctx = BN_CTX_new(); +#endif + if (ctx == NULL) + return 0; + + BN_CTX_start(ctx); + x = BN_CTX_get(ctx); + y = BN_CTX_get(ctx); + if (y == NULL) + goto err; + /* get the generator */ + if (group->generator == NULL) + goto err; + generator = EC_POINT_new(group); + if (generator == NULL) + goto err; + BN_bin2bn(sm2p256_curve_params[3], sizeof(felem_bytearray), x); + BN_bin2bn(sm2p256_curve_params[4], sizeof(felem_bytearray), y); + if (!EC_POINT_set_affine_coordinates(group, generator, x, y, ctx)) + goto err; + if ((pre = sm2p256_pre_comp_new()) == NULL) + goto err; + /* + * if the generator is the standard one, use built-in precomputation + */ + if (0 == EC_POINT_cmp(group, generator, group->generator, ctx)) { + memcpy(pre->g_pre_comp, gmul, sizeof(pre->g_pre_comp)); + goto done; + } + if ((!BN_to_felem(x_tmp, group->generator->X)) || + (!BN_to_felem(y_tmp, group->generator->Y)) || + (!BN_to_felem(z_tmp, group->generator->Z))) + goto err; + felem_shrink(pre->g_pre_comp[0][1][0], x_tmp); + felem_shrink(pre->g_pre_comp[0][1][1], y_tmp); + felem_shrink(pre->g_pre_comp[0][1][2], z_tmp); + /* + * compute 2^64*G, 2^128*G, 2^192*G for the first table, 2^32*G, 2^96*G, + * 2^160*G, 2^224*G for the second one + */ + for (i = 1; i <= 8; i <<= 1) { + point_double_small(pre->g_pre_comp[1][i][0], pre->g_pre_comp[1][i][1], + pre->g_pre_comp[1][i][2], pre->g_pre_comp[0][i][0], + pre->g_pre_comp[0][i][1], + pre->g_pre_comp[0][i][2]); + for (j = 0; j < 31; ++j) { + point_double_small(pre->g_pre_comp[1][i][0], + pre->g_pre_comp[1][i][1], + pre->g_pre_comp[1][i][2], + pre->g_pre_comp[1][i][0], + pre->g_pre_comp[1][i][1], + pre->g_pre_comp[1][i][2]); + } + if (i == 8) + break; + point_double_small(pre->g_pre_comp[0][2 * i][0], + pre->g_pre_comp[0][2 * i][1], + pre->g_pre_comp[0][2 * i][2], + pre->g_pre_comp[1][i][0], pre->g_pre_comp[1][i][1], + pre->g_pre_comp[1][i][2]); + for (j = 0; j < 31; ++j) { + point_double_small(pre->g_pre_comp[0][2 * i][0], + pre->g_pre_comp[0][2 * i][1], + pre->g_pre_comp[0][2 * i][2], + pre->g_pre_comp[0][2 * i][0], + pre->g_pre_comp[0][2 * i][1], + pre->g_pre_comp[0][2 * i][2]); + } + } + for (i = 0; i < 2; i++) { + /* g_pre_comp[i][0] is the point at infinity */ + memset(pre->g_pre_comp[i][0], 0, sizeof(pre->g_pre_comp[i][0])); + /* the remaining multiples */ + /* 2^64*G + 2^128*G resp. 2^96*G + 2^160*G */ + point_add_small(pre->g_pre_comp[i][6][0], pre->g_pre_comp[i][6][1], + pre->g_pre_comp[i][6][2], pre->g_pre_comp[i][4][0], + pre->g_pre_comp[i][4][1], pre->g_pre_comp[i][4][2], + pre->g_pre_comp[i][2][0], pre->g_pre_comp[i][2][1], + pre->g_pre_comp[i][2][2]); + /* 2^64*G + 2^192*G resp. 2^96*G + 2^224*G */ + point_add_small(pre->g_pre_comp[i][10][0], pre->g_pre_comp[i][10][1], + pre->g_pre_comp[i][10][2], pre->g_pre_comp[i][8][0], + pre->g_pre_comp[i][8][1], pre->g_pre_comp[i][8][2], + pre->g_pre_comp[i][2][0], pre->g_pre_comp[i][2][1], + pre->g_pre_comp[i][2][2]); + /* 2^128*G + 2^192*G resp. 2^160*G + 2^224*G */ + point_add_small(pre->g_pre_comp[i][12][0], pre->g_pre_comp[i][12][1], + pre->g_pre_comp[i][12][2], pre->g_pre_comp[i][8][0], + pre->g_pre_comp[i][8][1], pre->g_pre_comp[i][8][2], + pre->g_pre_comp[i][4][0], pre->g_pre_comp[i][4][1], + pre->g_pre_comp[i][4][2]); + /* + * 2^64*G + 2^128*G + 2^192*G resp. 2^96*G + 2^160*G + 2^224*G + */ + point_add_small(pre->g_pre_comp[i][14][0], pre->g_pre_comp[i][14][1], + pre->g_pre_comp[i][14][2], pre->g_pre_comp[i][12][0], + pre->g_pre_comp[i][12][1], pre->g_pre_comp[i][12][2], + pre->g_pre_comp[i][2][0], pre->g_pre_comp[i][2][1], + pre->g_pre_comp[i][2][2]); + for (j = 1; j < 8; ++j) { + /* odd multiples: add G resp. 2^32*G */ + point_add_small(pre->g_pre_comp[i][2 * j + 1][0], + pre->g_pre_comp[i][2 * j + 1][1], + pre->g_pre_comp[i][2 * j + 1][2], + pre->g_pre_comp[i][2 * j][0], + pre->g_pre_comp[i][2 * j][1], + pre->g_pre_comp[i][2 * j][2], + pre->g_pre_comp[i][1][0], + pre->g_pre_comp[i][1][1], + pre->g_pre_comp[i][1][2]); + } + } + make_points_affine(31, &(pre->g_pre_comp[0][1]), tmp_smallfelems); + + done: + SETPRECOMP(group, sm2p256, pre); + pre = NULL; + ret = 1; + + err: + BN_CTX_end(ctx); + EC_POINT_free(generator); +#ifndef FIPS_MODULE + BN_CTX_free(new_ctx); +#endif + EC_sm2p256_pre_comp_free(pre); + return ret; +} + +int ossl_ec_GFp_sm2p256_have_precompute_mult(const EC_GROUP *group) +{ + return HAVEPRECOMP(group, sm2p256); +} diff --git a/openssl/src/crypto/ec/ecp_sm2p256_table.c b/openssl/src/crypto/ec/ecp_sm2p256_table.c deleted file mode 100644 index e19df48f5..000000000 --- a/openssl/src/crypto/ec/ecp_sm2p256_table.c +++ /dev/null @@ -1,16387 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * This is the precomputed table for the code in ecp_sm2p256.c, for the default - * generator. The table consists of 32 subtables, each subtable contains 256 - * affine points. - * subtable 0: 0* (2^0)*G, 1* (2^0)*G, 2* (2^0)*G, ... , 255* (2^0)*G, - * subtable 1: 0* (2^8)*G, 1* (2^8)*G, 2* (2^8)*G, ... , 255* (2^8)*G, - * subtable 2: 0* (2^16)*G, 1* (2^16)*G, 2* (2^16)*G, ... , 255* (2^16)*G, - * ... - * subtable 31: 0*(2^248)*G, 1*(2^248)*G, 2*(2^248)*G, ... , 255*(2^248)*G, - * - * The affine points are encoded as eight uint64's, four for the - * x coordinate and four for the y. Both values are in little-endian order. - */ - -#include - -#if defined(__GNUC__) -__attribute((aligned(4096))) -#elif defined(_MSC_VER) -__declspec(align(4096)) -#elif defined(__SUNPRO_C) -# pragma align 4096(ecp_sm2p256_precomputed) -#endif -extern const BN_ULONG ecp_sm2p256_precomputed[8 * 32 * 256]; -const BN_ULONG ecp_sm2p256_precomputed[8 * 32 * 256] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x715a4589334c74c7, 0x8fe30bbff2660be1, 0x5f9904466a39c994, 0x32c4ae2c1f198119, - 0x02df32e52139f0a0, 0xd0a9877cc62a4740, 0x59bdcee36b692153, 0xbc3736a2f4f6779c, - 0x495c2e1da3f2bd52, 0x9c0dfa08c08a7331, 0x0d58ef57fa73ba4d, 0x56cefd60d7c87c00, - 0x6f780d3a970a23c3, 0x6de84c182f6c8e71, 0x68535ce0f8eaf1bd, 0x31b7e7e6cc8189f6, - 0xe26918f1d0509ebf, 0xa13f6bd945302244, 0xbe2daa8cdb41e24c, 0xa97f7cd4b3c993b4, - 0xaaacdd037458f6e6, 0x7c400ee5cd045292, 0xccc5cec08a72150f, 0x530b5dd88c688ef5, - 0xb21646cd34a0ced5, 0x009a084ad5cc937d, 0x2a81052ff641ed69, 0xc239507105c68324, - 0x7a253666cb66e009, 0x6bee2e96ab8c71fb, 0x35f1294ac0db1968, 0xb1bf7ec4080f3c87, - 0xa575da57cc372a9e, 0x344a417b7fce19db, 0x040e008fdd5eb77a, 0xc749061668652e26, - 0xa6976eff5fbe6480, 0x5006206eb579ff7d, 0x4504c622b51cf38f, 0xf2df5db2d144e945, - 0x2c8b1a1a3c4b0d30, 0x05ff8856a6601689, 0xbb17c93e71f22a31, 0x0927afb57d93483b, - 0xd572103000088f63, 0x81adf1f0855a064d, 0xac1c0ef6ebf26645, 0x150c6b1ab4d1fc7e, - 0xcd27e384d9fcaf15, 0xa80198337744ee78, 0xfdbe86a75c139906, 0xddf092555409c19d, - 0x223b949657e52bc1, 0x037937707d6a49a2, 0x5cd6b6e9c12d2922, 0x847d18ffb38e8706, - 0x675d822ded0bb916, 0xd4ea35d60c1c29bb, 0x3db4333d4e860e64, 0xb9c3faeb4b161071, - 0x3c286da2cfd31a3e, 0x0366a8a03024b3e0, 0x2491d2de9accf2be, 0xc519b309ecf7269c, - 0x173472a58cca247e, 0xfe8d59cb43619e4f, 0x0b4a2444a46a74c5, 0xa27233f3a5959508, - 0x85227940922c02e9, 0xc3a8433140d1ebca, 0x768f7689b210f45f, 0x379e72f63722c924, - 0x84a04814db522756, 0xb2d0d065cd219e32, 0x21666061f65c3e32, 0xd3f94862519621c1, - 0x47d46fb2bab82a14, 0xf6b743f64d1482d1, 0x2ebd57d146dca428, 0x4b9030cf676f6a74, - 0x668c74d78ce20ace, 0x125dcdd589c2ff82, 0x7c1aab770f67f543, 0x04b3cb10c9c6d8e2, - 0x739a8fd805174a4b, 0x0c94816e63c4bc72, 0x4918e5c02e2b0b93, 0x63516355287e39fe, - 0x9c9f042d0279270a, 0x5f7b516c5bdaddad, 0xd89058ca059be46e, 0xbfc2df6bb17f971a, - 0xb5fbf87b64af3c26, 0x73c9fc6fd150aaaa, 0xcef696b67939bd7b, 0xb145513f59636106, - 0x2b834f1d509b9cd0, 0x7bea65c6421e189e, 0xfa804513275f58aa, 0x952072d6ff9c65bd, - 0x03891e105e009b00, 0x58d1434ae934ba75, 0x1a473f8d9748f238, 0xe6bb9804458bb70f, - 0x0a44248a36bb0a07, 0xef22ea28be9d44de, 0x4aaf81826982d748, 0x83b4a4de96a4d70f, - 0x2e663cd4373a2f24, 0x3e707c8a43852949, 0xeeb6d6c6d7e74f8e, 0xe481c0d9ee8a98d4, - 0x461b1bbcb80f829c, 0x3b424f35f0ecce4c, 0x3291676c38d39324, 0xf73b839f13912c1a, - 0x4c3533771db0955e, 0xdc2e788fb170aa14, 0x5ee9fab985c12455, 0x32ec7722695dc7cf, - 0x55645bbc8704fb68, 0xea133cd248a93e25, 0xbbce44ef5db3e419, 0x35648233f554ae51, - 0x4411a9a30eca2046, 0x154b1870a6a65166, 0x6117bca9ce885dd6, 0x04d7ac60f6d975ef, - 0x36969f8b77125e6a, 0xbf5113ab85b5ed44, 0x1c993f01115c57f7, 0xdd18aa4aec26eac4, - 0xe0180d54d4ae2221, 0x6cce6001d7c0b853, 0x3361493f76fac50f, 0x161e5851969da822, - 0x427e33ae8ed9c317, 0xa3448557a24554db, 0x221ea8657781ce71, 0x69fdf8a436fd8e5d, - 0x71e35709341fdcab, 0xeb7b1ad657c155f5, 0x6b0a9658b33b347b, 0xbaa856d1cad68af3, - 0x1323f252e493952a, 0xf7c96c55940a4726, 0xe2e0586bb5dc9419, 0xa68b2ec49d1921d0, - 0xfd4520a63f5cc585, 0xb95f2de69200269e, 0x933ef442fca7a734, 0x96f361a23b3deb99, - 0x0588fdd88abdee94, 0xabbb1ea1417f958d, 0x7575e3cb2fa9092d, 0x08314765e44847eb, - 0x5eeb0e99c2eb61e9, 0x7de3a53937e8a75f, 0x6b443ab436095378, 0xdf922344fa592a98, - 0xe1de5f100950c8ab, 0x5b551c86a8976933, 0x137cbf4e8aa50535, 0x6407be639bc5b738, - 0xf945321148e90344, 0x60fcfd988fff6dec, 0x12a9423fd80dea54, 0x141caee612e7ab07, - 0x0755a352dafa1e1d, 0x3e7307f46fa550d1, 0x1835a4d0f3e5e299, 0xe61120569974c0bc, - 0x3b86cc142850b0f3, 0x02855e18d12379b4, 0xd08fc10e5056e4bc, 0xcfe30aa2c3877e52, - 0x4dbb2fcd069d45db, 0xbb570a9f430ea7f6, 0x15c07a8c04226fe3, 0xac8df677299e8288, - 0x7bd8b7c847718ce5, 0x0e9b9643d766da33, 0x30c43be69b23cc14, 0xaef82cf361db1c71, - 0xd7240a35d1f4ffed, 0xec268911ccb79283, 0x314fa6c814c7bda8, 0xdf8a74f904d1715e, - 0xbcb60a9e82017c30, 0xe1c80e7752bd6499, 0x7a7606ef148aa5fb, 0x0fed448a3f263918, - 0x0263b20d539e4139, 0x7e8581fbbcf5de01, 0xce1564c3df7d67ea, 0x178d1f6bd584afcc, - 0x9b499a7da7820bf9, 0x7021aafc29ac8e86, 0x53da45cec1caada1, 0xf575f34e5f6c63c6, - 0x8e6a9e4fdf895290, 0x73783ef304251e74, 0xbcf26fe238a102c2, 0xa0a1a4fbeb476576, - 0xe54f7f3a4896d9d3, 0xec73068587361b34, 0x0d09c3f281d4c41c, 0x8dfdd7cd150d2564, - 0xce37e84fc9541b4b, 0x02a85a7410754529, 0xd7a1f169475b4ddf, 0xec6f34ceacaa73a0, - 0x73306a0ff0485872, 0xe607f33be20135ea, 0x6f251c63fbbd2927, 0x3c85cf62a744817e, - 0x6cb2f058c747d33f, 0x36f0dfe49113c16e, 0x94ed53ed581d191d, 0x91cb72dc55f84346, - 0x3ea6aace93921f31, 0xf4c600e8b5dfe811, 0xf7efb07682931f88, 0x1957ad4cfffd491e, - 0x9dd4ef421635b9dc, 0xc9816be068be66db, 0x625773cae69bbe05, 0x28221c36398c4c3a, - 0x2995225e66b162ba, 0x64c77a81858d4ca1, 0xab2012c41627cd81, 0x3b4d172d6963510b, - 0xfd23ea021da51c9d, 0x8b1ac02e98d543ce, 0xc315796922bf1343, 0x17a5f74d533ae724, - 0x57d2e81dbb945dfe, 0x7befa4db8c8d3eff, 0xb9038823d9ad0d5f, 0x498470ec6752b2ec, - 0xac995c5066bdaaa5, 0x27d053c0f150fe10, 0xd6b48259b69e8530, 0x08daae840b2a43eb, - 0x2458b9e3c73bf657, 0xf7dc310560bbe83a, 0xc1b8e58150da662d, 0x80cc0a980495af87, - 0xa32641e56024666c, 0x791734cd03e5565c, 0xa6d5c2b5817f2329, 0x25d3debd0950d180, - 0xd39e100303fa10a2, 0x0458895120e208b4, 0xb938c406dd5cb0e1, 0x92d99a70679d61ef, - 0x6d69b1bd07837dc9, 0x451641e8455eedb6, 0x4624f85fe9b7cb83, 0xa1aa7f4a42808908, - 0xc3c8bd75b5fe25de, 0x6e67fd81941c6a60, 0xeb15fccc63694830, 0x3d420dee447499cf, - 0x37a60fce33247b4c, 0x1afddac4a9667094, 0x72e11351388e9d3d, 0x87e13ab96307c971, - 0x217ed12bbc413d35, 0xd7ac6191958240c7, 0xadb32c8ad1d8a615, 0x3ff4e73fac0283e5, - 0x549786448c4f3f4d, 0xbaf30f4638d2231b, 0x46dab2eba04afd78, 0x68126a90b051062b, - 0x5a8ce092c6b7c016, 0xe7fec8984be51251, 0x35ecf208bdec5980, 0x7e72f70a2eaf1bf4, - 0xfa2f46cf90a47d2b, 0xe7579164cc5a233c, 0xcdd2cc00d6ce411e, 0x323159ac22478ac2, - 0x939227b1d05f49ba, 0xa94c472c0089e4b4, 0x7e7a32282a0c2900, 0xe488cf88fe83acfc, - 0x31f8514dc6115cb5, 0x8c641a6cb43ae7da, 0x8b8f45b58a12e6cc, 0x3242e1de1fa494c5, - 0x7dab7b373c8f551b, 0x56ab9bcf11b6f25a, 0xaa27b4380a8333ad, 0xcbf18ef2f229c025, - 0xb7d6af71fb640a75, 0x47a9f7f697a28e0b, 0x6b663188dc931a44, 0x7f404379407b0976, - 0xf500974f78295571, 0xc7652177fdfae41b, 0x860bec0a6c78c5d0, 0x3178d1bb1435c9af, - 0xcd1bb864637cf4f9, 0x10e34d63f48c5ed4, 0x5b075be3c412ad03, 0xe6857f7fc3345385, - 0x1ca28a8ee15f1878, 0x5d72264551d68eae, 0x581dfa7d2b4c09e3, 0x6b61903f2ba03769, - 0x2bed704f6dff2150, 0x3b66f17073149664, 0x2a722a8a3b798a66, 0x376bf9e5403eedb7, - 0x95c769d2fc372cd9, 0x35698578596b792f, 0x2670a8f2e7f7a641, 0xf6f8ca3c58d3089e, - 0x47d58af32a91ddc2, 0x30abb6eab3f02921, 0x490df25e43f73f13, 0xf537773939d38a9b, - 0xf9099fa3212d8fc1, 0x63ae367eb959d9f8, 0x8e2e511cce74be18, 0xbeb6132d28ff4099, - 0xc88b69c57731c07e, 0x52ed6a3d227ff763, 0x52cd70822173c59f, 0xd0d299040f1250d8, - 0x5769c965f01be7a3, 0xf6db21dd26bbf942, 0x648ab14004516e12, 0xabfc349e2f1d2118, - 0x0180bb9d84d7141b, 0x134911d8e4bbaaec, 0x4b92b8f64e804905, 0x897da7bb358b6d21, - 0x38ecb2e51caf0aae, 0x6543e7e5a787f93d, 0xf571032510364cd2, 0x995b4a9a20d88c89, - 0xdd379996725be125, 0xb1df2f7a4fcad9d2, 0xa7b5fa7d47457508, 0x2575c600187cec18, - 0x1e259e3d752b00d1, 0x83580245568efa73, 0xed22d2e0508647f4, 0x84442dfc92fb06ff, - 0x3d24524ac4d2c465, 0x0462308a30d9d3e5, 0x45254a01baae8a2e, 0xd2593661cf6daf12, - 0x5dca599abe435bf0, 0xa169da8afcb3a620, 0x6cf13824af9f6508, 0x416b602161b2bbf6, - 0x5c4b342a0ead84ed, 0x654b3a33c67ed63d, 0xa31dbd4bffd8c960, 0x2f7fa743ade7d8ef, - 0x3e6c057797b29702, 0xd19f608ec8d8fdbd, 0xfc7473793f836beb, 0x3a0b20471be6e94a, - 0xec0ddc2b82cb0747, 0xdaeda7da80dde8ce, 0x98e2f9df22a96604, 0x8ab85f67eec67f06, - 0xe2874ab08c6c6e2b, 0x0445b6381fc45c42, 0x53e29b91b9afd253, 0x0d90c6f4584c9d15, - 0xbcafd803c7fa407d, 0xd4a5aebc3d150787, 0xf0d92d5deb9a8696, 0x516478a2567460c7, - 0xab9d905b209fb8a3, 0x0e93b2f066f5b023, 0x36ecf288427a2ed2, 0xcd8596762e7c1384, - 0x2755c80bb1316dd9, 0x787ab24d5a604fb9, 0xdf25cd6e045b914d, 0x968513d803c0a611, - 0x60f9fd6e5a72901a, 0x9a31b57f2e430478, 0x893c927b9668ddd5, 0xb0992fa258261de1, - 0xf4bfcc77ea4809c9, 0xc56284e868ddcfa0, 0x6f0aea97a43907c7, 0xfbb54c148bd5828b, - 0x18c1df9b3daafa9b, 0xd037d19f80bdadce, 0x06a3849d9204ccb4, 0x3ace4fd8dfca573e, - 0x35eaa6e36461d0d3, 0x17905f2a781de540, 0x2d531dd8552f2765, 0x01eeb7b6a1494475, - 0xb769488b3901a9f3, 0x7460ad9d8bcceeb9, 0x66a2abf1c7753bb8, 0xabd4a0f7fac6deb4, - 0x869252402e6a9f99, 0x109e9f4b61c6f6bd, 0x20498d04f39cdf59, 0xd62fed8ce52682d6, - 0x58f3f064c341c481, 0xaeb47ad53e382513, 0xe63d189ea323c1db, 0x106309c4f4f41c3b, - 0x828cc5c0f20462a9, 0x6d96e2a601b93c0e, 0x4cd09b8e39701ff1, 0xd909bb3be4c02cd8, - 0x5eefa9a3ec3f9511, 0x05977fb39adacec7, 0x5bb65608aa16f24f, 0x10d8c2a3b3396bd8, - 0x4b9fdb48e73da1f4, 0xaf424d81ca4837d3, 0x12977a990acddfbc, 0x30c06b0c4ad8881a, - 0xbdb45a06820f214c, 0x0d2b4dcdd1d72197, 0x6ac7be40ce1c0b24, 0xdffe969d5c0c2037, - 0xc86610f34f66b3df, 0x015c9bf09e75950a, 0xcbde1f5982ace379, 0x6a13800155ef04b7, - 0x1d092851b33025b7, 0x41d95d9e42639ac6, 0x84f5b14011d026db, 0x2de1cdf12e9cb8c0, - 0x866723c1db16de4b, 0xdca3ec1ca731d89c, 0x72976afd2d0f4836, 0xb21e68780bd8ee9a, - 0xaeece0bcd44dbd4b, 0x01b03b987a6324f8, 0xcef0026bcefa2aac, 0x85405981156ecd7d, - 0x123308b4a9c5711a, 0x721dd5dacce5bc3d, 0x26804860af6a7e95, 0x7ccbc3bf87cc4d54, - 0x5995334c3e242616, 0x3acfb676b987c213, 0x145518b80573d4c0, 0xc6ebde08aded9af4, - 0x9a4daa72d104dd8b, 0xdde4dac3cd08710b, 0x57e470f1db64f91b, 0x6c7f036eb374832a, - 0x9f07db54caab8fcf, 0xbc06cf52d8d5f56c, 0x4aea7bee9c0e0ed6, 0x282c76e18b24f8b4, - 0xcd43853968e0f8c3, 0x9cab4fb10369ef34, 0xf5aba10ba06befde, 0x99ff4d8ff311e9a0, - 0xd35aaa101d22c55a, 0xf2b3182557bf0529, 0xdd523fc5310a3ecf, 0x0b3cc5a35c82a6b7, - 0x3a15c7a108af767c, 0x0693ab9cd8951ee4, 0x05f23cff908ed521, 0xba38ae4ed8f30818, - 0x3567e7eb19841fbc, 0xd4f1f91388ce0fbd, 0x7b3440651da95cb5, 0x56b6c91fef0c0213, - 0x0c5516aa79811c1c, 0x8869b952943a29fc, 0xf382d60f67fcc2e2, 0xd3a606c601b5c66f, - 0x0d6b519f677f9e84, 0x3ef8e988f9e9e6c0, 0x8726d00ed296c2b3, 0x16429acf62bb2d8f, - 0x858ddab4a1817e91, 0x6f0b334f51351a23, 0x7f09f1f531213b6f, 0x82a81778d373eb3a, - 0xbed0e457d7a8801d, 0x632446bf2a988560, 0x3f2e4e9a57914a68, 0x45690fae01c59d44, - 0x29dc4c21401d787f, 0x5c4a52f82ad10df3, 0x7048dcd10cdece63, 0x5eee8921f4fcd342, - 0x9ffe119727f734ae, 0xf39e93dc1c8b27ad, 0x73516bbad0f603e4, 0xb6b3462b6a39a55b, - 0x4a69c806e78615c2, 0xe988794843797b0c, 0xa47dcf644f700651, 0x97662389f36ce643, - 0x17eef58bf866b33f, 0x5eb06c4515ff01bd, 0x328b8d67d4ae956f, 0xcd4d9449aea5cac5, - 0xcd382e53ed23ae1f, 0x7b1eb83b8306ace5, 0x15a8ecd063d726ba, 0x403b18162679c055, - 0xd0a5d4ebea56f425, 0xdf882dfcffcc84bc, 0x8d61a1c805ef7074, 0xeb42ebf496a7bd69, - 0x4932828fa1eda64a, 0x9b29f654ca04e515, 0x899b922465789ef1, 0xf37e88764c7ac45c, - 0x5c21a5c0dd61d736, 0x2554d8e82845136b, 0xbad57a13e441c8c8, 0xfbf59fee83ed48c0, - 0xb6dfed771b9c57aa, 0xd2a721c4ca5988de, 0x4a72eef2c3b79c4a, 0x4735ff387ce5d531, - 0x6595b84458e25402, 0x9050829a6000ae69, 0x4e309f2a26856099, 0xa8ed2943cca6016f, - 0x93533941dbcc0ace, 0x841a1067098656b6, 0x30bb10f83e216e8a, 0xb4f7130dc7e9212a, - 0x93ce7fc238b81921, 0x5b7e8e5a8f641f8c, 0x9ddc33deb95a5b71, 0x2f29f4a14bab0c77, - 0xfcfdd7e629b3cadf, 0x773924b26ef7247d, 0xda1da81a16bb9420, 0xd25e0cb35f1f1faa, - 0xc545c1a75e458451, 0xabedaafe76dcc117, 0x429b67f6ae073a8f, 0x67295cb2e7ed1851, - 0x844c475a38b961fa, 0x355386afd494c53d, 0x81c6dcf871dd881b, 0x092af9a79f4170d1, - 0x7d3dce96cbf08554, 0x72eac8accba2b32d, 0xca582b8e15cc89af, 0xc9a3d27ecf09b2f3, - 0x09acc52ed3b84f43, 0xb5b0f39553570b43, 0xe9f8c0f1152682a5, 0xdc532866d1359272, - 0xa830e1dfd3792ce9, 0x6ad5e7da482f3096, 0x0c80e17dba0aaedf, 0x306cfc30b3a85aae, - 0x13970909b51d1a80, 0x14ce48989614edc1, 0xdc467497755cbd57, 0x9660670c8b61b1f5, - 0x63d8896cd021a129, 0xff50d6958c5fb8d2, 0x39ee34cd34b67a5f, 0x8358e926ccc8e30d, - 0x99f8edca8af044d5, 0xd3cbf4330b841d22, 0xd174131b63b02eba, 0x38eb496124b15eeb, - 0x48a22e8cfb012cfc, 0xf5da2e278065962b, 0xff4403f9200acae2, 0xf184e282b6b418f7, - 0xc6b107ca5b20c9f3, 0x26f583a441e0630a, 0x92d0e75229f427bc, 0x4f703d0de9e071bc, - 0x3171ddeaace634c6, 0x12c49c6dc514680a, 0xa7002297639b4906, 0x1700f3d434b737fc, - 0xbe75a9cad6bd2300, 0x748db9d9c1cb48ba, 0xadcafa24a2f2c39b, 0xea6d56b46741bf09, - 0xc30eead02da09851, 0x8577fffe20f3731e, 0x245a5b85a5e0ebf1, 0x18e5eead89732407, - 0x746075b5bfc55131, 0xba4453d3d8e14282, 0xf8a86d159d9a709d, 0xa4f4630217f0e85b, - 0xf04d0d9b0b09c7c9, 0xe9669f06d6c1b268, 0xe2e27fbcfbda045e, 0xc8a35bec05eb827b, - 0xeb9d4eaae174d6f2, 0x03e09a3ad81dd4ff, 0xebda3404ab654a2f, 0x5d39c2892dfb220a, - 0xf1c3399fdb7845ad, 0x7d979bb1bce205b1, 0x89d18134a93a533c, 0x35b91f56afed2092, - 0x7c42374fe8984cb6, 0x9c764897cfffdca0, 0xcaac6a4d50fade17, 0xe0ef4846a887c038, - 0xd82902a52c00d995, 0xc5067a8d1a2f3a41, 0x0b4ce0b7d7daff7d, 0x9230ff23783733a8, - 0xad1951435957726a, 0x2206d08929796e05, 0xd4fe4d4127cba088, 0xa162fb5f8af4185f, - 0x3427a4e96c94f67c, 0xe86e6a15f91af428, 0xb465eb367ca10566, 0xb9e3bb2376a12196, - 0xe445463326fd5391, 0xd2cb5fc8266bef82, 0x39f16938e29cf376, 0xb5e7992af43ff901, - 0x4bd11a57af8b46ab, 0x0c5e337332e53a91, 0x5585aff51f34de17, 0x3324dae6ff8a7d31, - 0x7d1a4a46cbefccd9, 0x27f1086e10c87ee9, 0xc78c20aec8a5fdfe, 0x4da8bbb0f99ffecb, - 0xc62e8ca1d2255a8e, 0x6888bcbdff421c66, 0x4505e94b0635a09e, 0xcd4588f38a097beb, - 0x5dd66e10fd954995, 0xf4c89fe58d2946ee, 0x9a994e3b6e6e8e23, 0xd8a26b12ea5c4d4f, - 0x5ad271ccd78f3f33, 0xcc32ddea6b84a3ba, 0xe06e913d6923d4a2, 0x47d54e91bac13b53, - 0x9c284015294338fd, 0xd5e81534cce44432, 0xcbe70ca22e5c5807, 0x28449ff72f96a915, - 0xdde032503e64423c, 0xf7bdbe2478c4a4d8, 0x8b7b8d331850a22f, 0x6c6d8382d09406fb, - 0xef14705131546e8c, 0xe37d671893d4f2fc, 0x5d8d444d3e8cf4ba, 0x61572c322b045a47, - 0x82d10acd5f5bebce, 0xa1959df1dec34077, 0xe91ea4d68a7640f3, 0xe3b511c42ceef515, - 0x481f41613b55ec09, 0xcfc60e29aa074743, 0x58d29712da50cb69, 0x9912f2f0ae0595ac, - 0x43196eb35d95b713, 0x76e6a7ee0eb0d7f7, 0xa63822a6684c88ec, 0x308d561a3f241d0e, - 0xc044c871fa3ba783, 0x37dae2451b0fed32, 0xcfcc32d3fcaf9df7, 0x8363e2ee5e5c4c5c, - 0xf7ed72d0656c9949, 0xdc3b5a0fe7c9306a, 0x1c69661627293531, 0x9efda4aa036202c1, - 0x062de907d0953ad0, 0xf8041e0c6f8a9ecc, 0x6115b8f3de7a86b0, 0x347ddd33f2a2a793, - 0x8e7bcacc36680035, 0x9fa94d3cdc8750a8, 0x04e8b20daa55606e, 0x5a2280dbdb9b3752, - 0x6e31484dbd771d0e, 0xa5fc6ff489cfc13e, 0xb169aadf11e60e81, 0x2837c25fdadc2e26, - 0xf2030f4225a85995, 0x7807147e3e464e77, 0xf99091981e6bac0a, 0xc3723f3fc316243d, - 0x2741ab72959580d9, 0x1cb62f04ad353b13, 0xd083375dc8f7410b, 0xaed46d363137584e, - 0xa3d1bb4c8eb5d8f2, 0x830282b1dc1d81a1, 0x083108d71d52fa0d, 0x3b0d22596a1813ae, - 0x4267a218068f0d8e, 0x0ad610f0ff4c2d83, 0xa8c734baf1d3ad11, 0xec1fbdf9e74e9729, - 0x776d76940e9089f7, 0x423ce1f3edd46634, 0x47826701c68b519b, 0x3eedfd07b9887154, - 0x68ef1d10a42f776a, 0x6386fcab69688add, 0x36a5dd89d191c3cb, 0x8c4881e5d7393294, - 0x66def5145491bd99, 0x0a91c1fa773c3457, 0x43799ffb08efb028, 0x85cdcfb669092acb, - 0x7f4fc84c4030cbfb, 0x72269f3f2b240663, 0x2fbdd32ac29920a7, 0xbdc4d61a0367ef71, - 0xaa1216a122a659a2, 0xab1da8335c171bb0, 0x3ca0d22c3b3e45c4, 0x0f946baaadeae8d8, - 0xef14e81fdd2b364f, 0x2f55ada2ae0de459, 0x940af90140a2de69, 0xc7e75e9aabf401bc, - 0x47a9ff788e1c3a3a, 0x2ce3f69491e5dce8, 0x81dbf7dde0977adc, 0x42ba0b11851f6dd9, - 0x1b4edf339840cf0e, 0xad8ff5b21132fd4f, 0xd5e31a8a588a6101, 0x1661c89b16294868, - 0x8019be652e5068b1, 0xd91b1fada5f73c41, 0x283898bb911395dd, 0x06ba3fad70685b96, - 0xdefeb106c49760ae, 0xdad52aa98d5e77f0, 0x0b9e3af9140b5ea4, 0x6ea78774ca7519f2, - 0xcb4aa4498abb9d51, 0xa5c5b3f4d0469271, 0x397aadf70746ccb3, 0xc624e8dffabcc782, - 0xcfdc959047b20f62, 0x6839796b357d0be1, 0x2c431e27f83005b7, 0x0569f1c4a75037ca, - 0x709460772e4c876a, 0xeaa568fd7c9f7dd9, 0x4eae8ce3c8dd9b61, 0x8847a12643e7a6f3, - 0x50337833e84a623a, 0x6d0f0e64adf10b27, 0xf3029bda34083597, 0x8204c4bb5cc07376, - 0x5eb6dc8d28db1c54, 0x08f42a900ba5d9db, 0x90744ed74b2b207d, 0x1e9991f233d884f0, - 0xaf5ca8e8bfa880e6, 0xfa43aab6924390d1, 0x4a7d2a178fc16736, 0x5a4aa3bf0525ff25, - 0x00092c75b19c8544, 0x31fb7664db32f0ef, 0x196515334ea72250, 0x053efa8bfc39562f, - 0x9bf9efc1b0242cb9, 0xfdd1f568ad88cea0, 0x5bd701ea2ded6b29, 0xe3ffd2783b7121e1, - 0x6301c5c26b454d79, 0xc4d21c15e28fe4f0, 0xa2a4b664ef38802e, 0x7e23d698181dfdd3, - 0x2a76ce3fcc775c7e, 0xbcf7808774f52159, 0x080f9528ffbf916c, 0x885bebd0441de6f0, - 0x51e6593154e91061, 0xe5233d5e7b348535, 0x473b3b3f184aa822, 0x21151e7b977aaa95, - 0xde51ba8cbee9bd99, 0xae5f23f269f957cd, 0x708d74a0a5a682c7, 0x91097071cb646e5e, - 0x3bc35e10f5963bbc, 0xb426789ae226b25e, 0xef6cad6c7f12beb6, 0xeab10ec96bacab69, - 0x23f3533bb1ac8e06, 0x8a6724bfaa07e6b3, 0x7b8760a2905deae8, 0x56702de289f4e2f0, - 0xdada3f65d4f55bf8, 0x7f8c6d057d31d642, 0xc9b00abdc8495650, 0x1c51e60abc0eccd2, - 0xfec65ffbf9c42481, 0x22e8c26bb26384db, 0x6be43bc89cf09d40, 0xbd6c96655fcfe6a2, - 0xb626341c04311030, 0x905fe7ef57925046, 0x9509529bde2ef3df, 0x7948636bc0cdc7a4, - 0xc30c108a100fec81, 0x6221aa9d592e8828, 0xf3c8215591cebb72, 0xd8bf6ea39c57b9dc, - 0x1d183b59164b8c40, 0x7ea2a786319e78a6, 0xc973b32e93bc2f87, 0x9025624014782cbe, - 0xbd6e444a134eac53, 0xfe0f19b444f2e570, 0xf60bc3c21500767c, 0xa8502a4bd10e123e, - 0xded8e57012f4651f, 0xaeb1b9241eb04d7a, 0xd513f05f555fed42, 0xfa0c380af507ac6a, - 0x65b433a8dfb5ce3d, 0x98999ef103eca182, 0xf56d0853313c64ec, 0x3e0417206422ad1b, - 0xed5571f713fb7ed3, 0x02876cabef2d641d, 0x60cabfa0f8547278, 0x63a185e898896c4a, - 0x0e1ecd9e46466a47, 0xd757ae92795b1541, 0xd9dcd23ac5c15de8, 0x3b3de05121ff3a36, - 0xa11606102e19c39b, 0xd6106c43e6eb5c91, 0x5ab4cccacefee53a, 0x007b8326ebd1926d, - 0x32ba940140487c6d, 0x46e0755a22260568, 0x7790f1c8c0222f19, 0xb2bb4be2c6bded6f, - 0x175f675d63cdbf6e, 0xa1809a9af4da899a, 0xf7138f89ad5d7afb, 0x761b3fb67fd8c602, - 0xa2eca2c4334f453b, 0xf64d9dc500a7a507, 0x924adebceb539c8a, 0xc209a7edb19c905e, - 0x6dbbff18fdc777c1, 0x17fd5c2997f84b5a, 0x73c3bbc097fbb941, 0xf67681a6300cb02c, - 0x5dcfafe1b79d21f7, 0x7d4217fb0d2f24bc, 0x53cc4b66d1bf5b67, 0x8512234b09a632ee, - 0x7afef5d3c566680a, 0xcc16073d5ac4919b, 0x8cc1c7b68bbc832d, 0x50c43f4413c4c58c, - 0x724c733a800256f2, 0xff3978a5dbd4f083, 0x9b2a00ed5c8a3916, 0x69e19793abf7a632, - 0xcecb6edcab1857af, 0xa462893d3c564a29, 0x61c82e8a6cd57f10, 0x3fa79f6ccffe83c8, - 0x364ce350ccad2815, 0x83d7adbfcaf46f89, 0x4c2fd40ac0ad5ce2, 0xd51c2c33a7b276b2, - 0x37328b5094395ce3, 0x70812a15ff5243fe, 0x15ad4a85db620d26, 0xff3f9978e53508fc, - 0x4dbe3930f085dd46, 0xe68954afa8bc18b9, 0x1e20699412fc97b3, 0x2b5bc43566c8182b, - 0xe948a5e26d1d3ddc, 0xa35730ae4e899f73, 0x8187e151ade21342, 0x22ddbfb842707e88, - 0x314b8155d6b6ad5b, 0x73d0d75407a752cb, 0xd9da783ff9fa2b9d, 0xf435d414df4f5076, - 0x1dff839d5c16d42b, 0xd136dc23e40ab6c1, 0xfca6360f0e1c5a33, 0xaf1c58598e7cb56e, - 0x43dcc5d73e585d51, 0x519f6ad4eebd062b, 0x80754ac3c1d740f1, 0xaf1fe551463c5b67, - 0x1d24556ff8ace198, 0x02b91a2134d2ee23, 0x13d1ffba18d54997, 0x5d56aaa246770a3d, - 0x4fbbcd60cdb5494b, 0x848f8dffabbea9b6, 0x42672e40d5a53819, 0x584f13fbeaca5bf3, - 0xf160a9ccab6e11b4, 0x51a73a3a653b6373, 0x63c3bc25ecdf58c7, 0x325cc030e9514373, - 0xe2a6b8f7aa1916bf, 0x9c668a4c33a54663, 0x5931972834a653b2, 0xc5e8ca195ea0274e, - 0xa5a7a5e95eabfb91, 0xdd71f31da6f7c46e, 0x51b21e140a5fcb76, 0x7d4dc704d3c881db, - 0x72bb35a49a20c914, 0x1c426d0b4fe64fef, 0xecfeeaed431bab4b, 0x85ac16eb56493abb, - 0x45c61bdd78abbe40, 0x937b96bb7431209a, 0xaccf7f070abbe8fb, 0xa17a2586c5bab868, - 0x0b40d7846acd93bd, 0xe92a3100a8aa7d86, 0x5a90c85c2b95d141, 0x42b957edf0eac80b, - 0x97b291bad89c9cf3, 0x8164c5fc56ad0b3c, 0xe178efd995023ccd, 0x0035cde42dc2cf15, - 0x6a755592839eaf7a, 0x324a4b4eba667d1a, 0xf5c3460b16ba21a4, 0xde9f478017f5ef6b, - 0x8d31bc90b58a334a, 0x31fbdfe21dc8eb9c, 0x23d8b2d4a678ac83, 0x4dbb4cad0a7e2706, - 0x7c91dba2fa129d98, 0x213e75991c59be79, 0x59f768c230bde513, 0x8c2972d5ff12ff22, - 0xc691e275dd7475c9, 0x25f4ac9004ef9825, 0xfaad8de2af3a5b1a, 0x0460afdc2ee1ae1c, - 0x8f542a7f1830047a, 0x2d9c80739c120316, 0x259137908d220bb6, 0xc1f1ed239eb2e2d6, - 0x52fc97633cb4e187, 0x466c94626a15b643, 0x4a2391acbad2fc28, 0xd85fcc762fbee484, - 0xe746df69c444e173, 0xe716b7df6ff3866f, 0xde7f692c0021274f, 0xb5ff42f5c63a693a, - 0x183370004c000aa6, 0xcbb94a5f3fd9d41a, 0x592117ec50ae9cf2, 0xba6d987fe545a9d9, - 0x7f564fc0813b5d4d, 0xb0c3c5252443ec98, 0xf763d3b20ef393ef, 0x86ba065985aab176, - 0x1089bea456d640dc, 0x572ab615149aaddc, 0xbb18497c0b5e8699, 0x42acc8ecf1858fd0, - 0xc7033a8ea70569dd, 0xf6670a5251e942ca, 0x76f01cc756a45ce9, 0x12192f6c25bb90b7, - 0x5e67d8c073397f3e, 0x5943d66bca82b331, 0x2177d44402e76b82, 0x1e247b0fb5819ef9, - 0x69f15537f160c9db, 0x2b7b74de9915f35c, 0xb028cd637d5f3b3f, 0x730cac8955b4a829, - 0xff62e2a27178e739, 0xd895fd5a4574f298, 0x9245177f0510e27b, 0xf5d500feda9a7127, - 0x073a60f73af7d8b2, 0x36fc50273862bdbb, 0xcf8391afee8b886a, 0xcd304e07207e8fd6, - 0xf71260f3d5316697, 0xe8d71e81ffa497c1, 0xa945c9b42720abae, 0x45e5de0e5987e1ad, - 0xce05418c9fdb0589, 0x94a780beb0eda9d8, 0x5b7ded5404d3fa8c, 0xdbe264a2ee1ad880, - 0x7aebcd67079987f5, 0xf60062c177a57277, 0x9f71be1267cec822, 0x3f901c7848bef8e0, - 0xecebbadd7dbef974, 0x3d01a5999ab31878, 0x9646807f6f1f91ba, 0xd8b30022c4e5f774, - 0xd4f845b74984a74a, 0x9ef317840a4cd193, 0x201c39b8e0cbedc6, 0x19204c0d6c7fab92, - 0x2aff1d698b3e9596, 0x03c3310096cbfdfd, 0x37fb5e89e4ade4a1, 0x73fd021127db2008, - 0x1e56cc63984cbbdf, 0x884aa70a81db42e6, 0x2230ad7167d0a7ee, 0x6eac311bf6f3a5af, - 0xa6bb02fdb15ad981, 0xd53d933ff264dba9, 0xa46551cb65cfac45, 0x666c41a76471a819, - 0x7bd7ba38a851312a, 0x480fa8cd7fa30feb, 0xa5c42ef922662d51, 0x0afa1c9a8be1ab4f, - 0x8d9c9aa214346d55, 0x516094082cdc3e18, 0x16a963ea107997d4, 0x8280001138c15cfc, - 0xe4ab234bb9ff258d, 0x69880d3b6699086d, 0x3d7d3d1e32308eca, 0xbed0774a3daa1b2e, - 0xb6df4485983a76d4, 0xc6fb7d520aee753e, 0xdb74f71648c81d1f, 0xb88725df36319c29, - 0x9d0c477978210a58, 0x841060a628470c02, 0x70ccd8bc370e425c, 0x1634e628b865cc42, - 0x0e037ed34a49d063, 0x53cdc28cf90ff0e6, 0x348b5047fc84dcbc, 0xf79591be4e039c0b, - 0x9e5db80c33f78fb6, 0x69ce92f11c21829d, 0x919dc94a46942844, 0x0d8185a00b9273ca, - 0xbb19136e4077238d, 0xdd9d4ca7441d8075, 0xd51e8bfd157c3dca, 0xcefba865bb345540, - 0x89564e91dd8456bc, 0x1029166c5b13d782, 0xe02cd50a812d2c44, 0x654de0c9dadd2e89, - 0x14709b5a9e1a3379, 0xff2a7b2dc49a6ce6, 0x93bb630b0c412bd8, 0x11f4f036687dd7c0, - 0xee7fad8f341b1c8f, 0xaf83447eca86634e, 0xbd9c1795dc812ce4, 0xb7099de7e683f6bc, - 0x2d39ca5fc097f911, 0xfdff3a8358576b7a, 0x272beaee1c8f9796, 0x726e78ffdbc59ca0, - 0xb67d5853eaa0c878, 0xc03a497efc478336, 0xdc72b0f36a94a20f, 0x15d177b6ad45123a, - 0xb887bb864c795fb3, 0x6566180c005efe01, 0xb3b53dc1f6851821, 0x44699e38ee0a6581, - 0x1c55a93806333642, 0x1fcbc68cd58d1cd6, 0xa244bb60d708e219, 0x07173dff1b5350f7, - 0xb8715e9116baeff0, 0xd3752ba762ba2ca1, 0x86d91965e5a31a56, 0x3c27c4951d076d17, - 0xd41f651c2ade59ba, 0x8f2ca8db21b97bc5, 0x1c2a1efdae15b528, 0x96977c9dcdba8c3a, - 0x91c8ddf3688697af, 0xc2de47e16a2fa935, 0x01d09ee453aeaf90, 0x9dbc205a2b000616, - 0xc7a82e96d594c62e, 0x3317b8eb5b8b73e5, 0xfff7c8b4c3a32096, 0x6453b70094e3358c, - 0x2b1bae699aff8a4e, 0xbf7b4736ad753b49, 0x20f6e0752f5bfc7b, 0x497dd9715e1cd5d4, - 0xa91030923e88ceb4, 0x602ac3f33ced7c4c, 0xbf584ae8dfb8e2d9, 0x3105b41ec3549b6f, - 0x163d3abbe998423f, 0x51234aac25187982, 0x8770f9bcfe97c132, 0x1800daeee2fe2a88, - 0x0f02b3a8243fd4aa, 0x4a536e070e2bc9ae, 0x628c64308bc9c315, 0xfbecf1323c9722d9, - 0x331ad785fc3e1d41, 0x5f5508bdfb62937d, 0x91a3214706923e0c, 0xfc796e8da1017f42, - 0xaafe90f59343d9bc, 0x2d136e33a60bef49, 0x02f9f2c3db9d0229, 0xd4892cd932be8419, - 0x79c72ac302f0d281, 0x8abb8e35b1d94786, 0x0c6de808c0c12a8d, 0xac6d4d8e5b9c2ed0, - 0xcba0baa1708e791d, 0x4bec127f651f20ff, 0x261d3b5a50d67450, 0x0fc781b9f92cd6c3, - 0x383faf53f2a71eb8, 0x0c20e971447ba60d, 0x3bdbd7533c088d5b, 0x56c8661b6d330584, - 0xb60ad35ea1d2263f, 0x737d7418bce8ebe2, 0xdda539ab99ec21ed, 0xe5c1cea1abbce298, - 0x4c92449387476985, 0x9c3be4e72e40685f, 0xf6d9c228a9f85902, 0xa44764c91adc6a9a, - 0x0db0d2ba599a7553, 0xe3788aa1dd74946a, 0xd239baffacd81b2e, 0xef52f85666c340f8, - 0xaaadb6ea9b1c8a45, 0x86fc426284837b81, 0xb33810a5001fa2cf, 0xf990bd17f7e2ecf7, - 0x14aa4676275e6c1c, 0xf19a05d12146d05e, 0xea5f3c7c6d5b9b4f, 0x32efb45c62042da0, - 0x7f430093a67ce3ea, 0xc8f47792c4d16d4a, 0xa7e72976b7f5e237, 0x26550333603f8d3e, - 0x9b8fe810f3c3b225, 0x65e904778694f5c3, 0xe18bde37757c008c, 0x4fc064ba9ccda596, - 0xa6ec138b14e30506, 0x064b8d8f63af9202, 0xc3783b153f753ecd, 0x20315c3032141460, - 0xbe04af9758a4cb35, 0xa7f4476ceb2653a3, 0xc4699159169fac3b, 0x4e2d709e9b1d01cd, - 0x4ec5972ace479262, 0x2fc417acdf559f30, 0x0605a730b11000e5, 0x8b65acbb6a17e365, - 0xd967fbe917266184, 0x1c10125e01f5db0a, 0xbf638c23b61967f9, 0x71b359fac654beb9, - 0x9d83451d64277002, 0xc8777ce755abe4f8, 0xfb4530585d7e824a, 0x63c28165d9737d56, - 0x7d9dbb4c39824fe3, 0x469e8165902bacf8, 0x5fad83c408ef9c40, 0xf8b2d576c30d1026, - 0x29f9f4bea44b1781, 0xac726909624bfe7c, 0xaafd751738177909, 0x90563245ba1c612c, - 0x389ec2a7d7e59ce3, 0x50500c567966d3c3, 0x26b15bf6cb1ad6cc, 0xe9080fe122c631b5, - 0x17f6dabe12e3e5d3, 0xc3f85d05a0f070f8, 0x67d33630af152ed4, 0xdb3b4096c5c332d7, - 0xc3dd07768e70b774, 0xa1e0b82d0516b75c, 0x867ca995ad903a61, 0x4fcd47a488ba6d51, - 0x6c6aba39dfc7d00e, 0xab65e0c18b16269b, 0x09ba717e85654122, 0x07e4cb761fb1b0e5, - 0x9265799c5d55a837, 0xf7e6d513b6437316, 0x94a696b3988e41e6, 0x2607761bf2629a5f, - 0x973e75d9df82ece9, 0x48730461eb3c4d75, 0x5796dc273ed9d51a, 0xf325bafd97b61151, - 0x7007d37e9b53ea4c, 0x3368fe5e05f94087, 0xadd778da036b0736, 0xc31fcbabf6fdf8d7, - 0x35de52e719187698, 0x1d2f3e4087fa3c08, 0x8047f5eb6ee05190, 0xd5add1eda45df387, - 0xc2431d8a98a8fe49, 0xb1c0388fa3162e61, 0x1f0ab07565c55677, 0x7f3b14c57417b083, - 0x1f0ffa4f1abb4291, 0xdfec55dc328de4d0, 0xb792695f0a121172, 0xe684ea8df0457fae, - 0xcef58a827a97e9ba, 0xfd341be16fb9e359, 0xc87a2331a9fb1692, 0x2527b948cee2e7e3, - 0x5e101fbe7514323e, 0x0d06560293e73c10, 0x1829f79b98724d80, 0x0c6ea39a34167a64, - 0x1da654dac1c2b486, 0x4381d7a197d73b93, 0xed2de677dfea2660, 0x98cf929f0489101c, - 0xd6f382504a8d8ea7, 0x483110e79c14ef66, 0xce485d95bbc34776, 0xaec1a6a8430e7289, - 0xcafe578fb670a41a, 0x61243b580f10d832, 0x221c3043ce630b2f, 0x8e8705368c784121, - 0x7963cf963518f155, 0xc21b75f27e48406f, 0x8f2f2bb31db2a418, 0x467a4ce355531acb, - 0xd45bfce4db6ad410, 0x116a3d9dbd9162db, 0x26b549e62d30c113, 0xb4799b40daeb1ca5, - 0xac6d4450ef27b16a, 0x51303f5d166ed3ae, 0x2a7b47acf0255aef, 0x85c10733b0fd7de5, - 0xfa1b80f6fcd647ba, 0xc3b3a75c8e36ba96, 0xb89f7cf587c773bb, 0x2585e4ca847ec1e1, - 0x8f587f5b9b828303, 0xdaf3e553b32110ee, 0x5a080e99f299f28b, 0x60b02561cbf056d9, - 0x819439e181a81c62, 0x68e7551c7255e324, 0x8e47a1bdf4e2642f, 0x30b7c9517b5baffe, - 0x2effcd65da6cf6e5, 0xc0e53b063106b643, 0xd1ac9588459392bf, 0xc3a9b3fe5c4914b8, - 0x6fd4a4e70d53afbc, 0xb16f9f98ee51d7d6, 0xc372f1f76b65fc1c, 0x7ace9609a780c6dc, - 0x4b62ba7528b34440, 0xba37cf920321a2ec, 0xdab82152af4d36ec, 0x359da66c3e405aa0, - 0x32133b6e7e57f6b4, 0xfb739dd61168c5ee, 0xfbfc045b30bf74a2, 0xc1c2002bb95c9009, - 0xfb0e87292c13de33, 0xe6a1ada23c8a0b87, 0x4a4ad963b5c4b22f, 0xed9800b5e15d4c3f, - 0x1d3fd890d8ae7729, 0x46aa7d9d9bae01b2, 0x4f6e3a19e1a8fb67, 0x2b460d0337d72c74, - 0x7be622092baae6c5, 0x8949c3aeda3c2feb, 0xa9d884a9e4704860, 0xfd3dd63e0fd6e075, - 0x5de1715bbc8b024c, 0xaebb8f1195f7516c, 0x3bd0bf7a3f4a5378, 0xddd22dd5c8597960, - 0x94e9199dc8d209b4, 0x5de530f34523777b, 0x51ba8ba0b866a17e, 0xfd55a04a1d22b19c, - 0xcdb238bd4130d0d6, 0xa06aeb43357757f5, 0xa475cc0cc62d4527, 0x6d0e779b6e6a8758, - 0x713405fae15e9778, 0x0f00a2b27568a6a3, 0x62bd58efbe3090fb, 0xf05cdf4c267eb1b6, - 0x5e62d04738221b8e, 0xf5b9900f1295612f, 0xa0e5749d177233cf, 0xc44c5592311f8054, - 0xbd52eb6cdf769fe2, 0xdd0d254454cf2122, 0xb981221ff70b0658, 0x3c36fb55ff496fd5, - 0xf430a57f99390967, 0x0954ace551e02989, 0xaafe4ec82b2d9387, 0x6d9fb14548a4f4bc, - 0x8031a9996ef12134, 0xf6416c9d68835bd2, 0xaffdc09d53a51a0c, 0x19ecea50392e51e3, - 0x34a13752a3b17634, 0xee74fe197f0a21c7, 0x428f82d78e4fdbf6, 0x43d9a86b4ee32b41, - 0x37bcb35d71d05a42, 0x6f9295c16c0084c8, 0x83be0ff5413d3a04, 0x3cd1cc1bba243b00, - 0x9a355c5fe780c65d, 0x96f751125aa49305, 0x4996cbc8f7336602, 0x0c35ae189a910bda, - 0x17aa114cd1e76239, 0x3b9a332aabe8563a, 0xe1275ec7a3241720, 0x7725b43f265d42a2, - 0x652de643921dc113, 0x95074114a43d3ba2, 0x6ac46e80d122a80f, 0x1af28df0f5ddb5d8, - 0xc49b4fcafd637609, 0x60812edd2d354b6f, 0x3491bbfbc599c6e0, 0xfc5c0fada9da1453, - 0xb9d457569b43643a, 0x70ba805435dffd85, 0x128cc37ce3a2d1e3, 0xa5e7f2845bf10011, - 0x5a9599124872a087, 0x6358b4f5deaff08f, 0x2d11e3e6dc803a0d, 0xe028f2b88218e837, - 0x0f71a38a839a4b4b, 0xc92a5c359039daaf, 0xb2cc835b6228f5db, 0x8ae04d563cbc4d7a, - 0x1c1cac7a75e61830, 0xfd2fd5f227d0984a, 0xaa90ea33113594e1, 0xf9e02ffebbe576ff, - 0x5ce891014479b80c, 0x605419be299f8523, 0x83202fb7a4ae2a75, 0xf6e64ff96707e5d7, - 0x3ccaf13d7dd066ce, 0x01aac585c5b5e009, 0x1b4aab8014566d6e, 0x11e7198efab59ba3, - 0x24a323f70fc7120c, 0x5609b08316141155, 0x98b0a7b806547c85, 0xaca41a2e51149415, - 0xcf751f8bd38bf1f5, 0x58ccdf65c8344efe, 0x6c72df390c015ec8, 0x133f9c9c23306d85, - 0x61d1cfe15b558202, 0x6a3c4c3582a5f673, 0x637f9512f4c797a7, 0xd330644463579ff0, - 0xf16e1e34b4ee7ea3, 0x618000e4502dc31c, 0xaa71c457381d0335, 0xf8d920561946d767, - 0x9cecf035b94a3d40, 0x3e4545a17b61d5d5, 0x1c3564aa53e6e73f, 0xc2f4bd3857b00d4b, - 0x211858b81239b300, 0x5e4926ac16c86f0a, 0xe83e64adca66d4c2, 0x63f875c830616302, - 0xc6d8069d2b091649, 0x92f747c0d271c5eb, 0x39fa58b367129527, 0x4984ea9b3a80ff9d, - 0x354da29136527160, 0x997edad9709c86fc, 0xad497e23b94da9bf, 0x7d12cc94bd4d5b76, - 0xcabccf6f82027a0d, 0xed6559dcebfc6c95, 0x2f81892df9e2941b, 0x83c3531b95bc842b, - 0xe48a4ee1f58654e0, 0x05fdbf72d0daa87b, 0x51ad2decf57851ea, 0xd61ba55be8f651a5, - 0xceed919f6700c3e6, 0x763257403bff52ef, 0x61af8a4943b1a111, 0xddaf8c619a405fef, - 0xd722bc84759b7dea, 0x35e5a14698bb02c2, 0xadcf17d7252bb6e3, 0xffe831ce375687b0, - 0x02a7b4f8dba11145, 0xd4e67ff0a1cc75bd, 0xf6bed864cb3c0f74, 0xe379cc57b232b594, - 0xb6599ef34e3d74df, 0xd06b61e56f72748b, 0x2ad1cf0c57a4d409, 0x98afe437bbe1e3aa, - 0x1e21f20f87f1dcfb, 0xaaf3d2c81546a6f4, 0x7d14ff98c8a0d395, 0x3664b61f266003a0, - 0x707b3ab273ddc773, 0xbb486f128f8ecd85, 0x7a396e34185f08f6, 0xa9d40d14282ef537, - 0xad0fa6ce1e0d0138, 0x990146e8ced1b2ef, 0xb73509ea936fd5e5, 0x8408168b79b481aa, - 0x9217fc8086230dd9, 0xaf81fa1a7a00519d, 0xd3c65b61ac9d8d7d, 0x3e83ff6d216d466a, - 0x6393d00840e8fe0b, 0x5195eff7a58ecd18, 0x7b7aabcecde2f2bb, 0x1553aee7976dbbb9, - 0x5d7f749ca175d496, 0x6292f95e5d333f51, 0xffe7f74bb8baf5a7, 0xe9649e0679728c39, - 0xa937ccfa06522d38, 0xa7495d7e37c3a928, 0x974158e812ddc092, 0x9a6e70c82ca0b636, - 0x4ba73f678b2d9ba1, 0x752ba57f558e6a5d, 0x9586e2990a7af7b4, 0xa486d8ada4ae89fd, - 0x15ba224f45406dcb, 0x5854cc00ca213240, 0x12cd7a5d7053add0, 0x960b6b398a21e899, - 0xac81ded4ec6e91d1, 0xe148de8034d93ae0, 0x3084579bb0d44847, 0xdb718742421f4cc0, - 0x7358c2d19a2d549a, 0x929c1c9f9de2d932, 0x269b4d558077ce0b, 0x25fcf5912e113dc7, - 0x04cc755f9b71f92b, 0x54069e1f886a4974, 0x5789c6348dc58a5a, 0x1dd27d9819361398, - 0x7a2c2390a9dead06, 0x836b38bd06ad26e8, 0x735114cacf0abb97, 0x4ecfb2f5999e82c0, - 0x85cd7efe9da98029, 0x88e472a38d18b4ce, 0x6e0a3ee3d8777812, 0xa05e44ebb1425ce7, - 0xeb884cca4ee53297, 0x67a0f982cb5689e9, 0x84fee0ff278b4fa9, 0xa002601e44d5e266, - 0x95fa98bbdae12807, 0x118c212dbb88133f, 0x9ede913100051a91, 0xd354ec4f9b29f406, - 0x032ab62fc51a4ed6, 0xbfa91cfd68c85311, 0xfb98f0ef27647507, 0x5e418f8926b631ad, - 0xc17e3e4fee8c4e5a, 0xc221a1b7acf82d68, 0x67bc9540ebff48f4, 0x390b760628e7982d, - 0x0d9a4b18c65ca2ac, 0xe27166e392c64490, 0x081ae367a842acde, 0xa041f47bfe8bde53, - 0x4992b5f7f17e54af, 0x99c4a493193d843c, 0x932846bd07cfbaba, 0x1de956fd8e865a43, - 0xeba2f84cbff126af, 0xfc5a5703ac806638, 0x47cbc2aece2dfa99, 0x20cbbf78bd8d99e2, - 0x22f23b1804c21205, 0x401b1e0887941f00, 0x4421d0c0c7631a72, 0xb7cdbc7b4156fde4, - 0x61fce30f2abd51d0, 0xbc70fa9f22178665, 0x94516d4d06878320, 0xfb4a042f53e53e96, - 0x34e40b1d511804ac, 0x22f37391e01602d1, 0xc280b1aa78edf92d, 0x0efbce19d6a90f10, - 0x2bd9185734d40dcc, 0x213d47ac1e778360, 0xc63af769b6d25114, 0x13232c903e39fab3, - 0xb6757d5b86fd514f, 0x1a6dd98096301f47, 0x18bab6a726908647, 0x05d536ca761feda8, - 0x15bff4d2b834d102, 0x0ddc2d840252dd3a, 0x766542ad05502aff, 0x109015445ce703d5, - 0xae20cb8c3d9dfea0, 0x46c095b0284fdcde, 0x2aec0ae73dbf67c3, 0x85bd72390c54277b, - 0x85d589367f593a03, 0x8c18f9bb056fff2e, 0xfa2e3c913319abe9, 0x9ff6e4cd79254ae9, - 0x03fc24efd2f3d4b4, 0xb2706372e82111fb, 0x097721ba480b1e14, 0x9e06c158cd3746a9, - 0x683ec04d3612e0bb, 0xed3a15599cbe3da5, 0x807d052bd2d5260c, 0xdd923767b4122b7b, - 0xd186423ef1afb4c9, 0x22ac6c53b4c862e1, 0x13da7b1fca53e5df, 0xdd5251565fab22e4, - 0xa1197271cface62f, 0xc877adf578cd19a2, 0xee960066597841bd, 0x366359a40c773e2d, - 0xaeabb0a3600f13fd, 0xabebe74e161a2d18, 0xd71b9e26fe2b124d, 0xc4662e6e1f313619, - 0xfe173a50c65e8caf, 0x1bec45186302f028, 0xc075ff623f3bb554, 0x4323d08715ed7717, - 0xbb776e2356075217, 0x1de047b5503e9641, 0x2e1bae289a76bdd7, 0xa757dc4e78636ced, - 0x1095069123a4d1f9, 0x3f14e9b797008cda, 0xec63848d50b81ff9, 0xaf3b5cad36a95559, - 0x233f53e5cf01a158, 0x5c8d2abb26b8dcce, 0x6d7537e21ef21ad0, 0x3c97225a9782d11a, - 0xa6d3757ad819677a, 0xbb1692af9785433d, 0xd185a54d16cc6631, 0x39a85ac7b47d54d7, - 0xb1a3cc96aa01e9b6, 0x8e7379d626c41f79, 0xde4e39bf11f63063, 0x3583d9a70f56bdd6, - 0x4a41104b4edc1f13, 0xb32372a1e2403d80, 0x24f4942ad83baa34, 0x851fed6fc80a5dab, - 0xcf66b12331d0de29, 0xd29c8987a3759f4f, 0xb724eca99bd71bd8, 0x998eed25e5462fd7, - 0x18cc0667b23ed309, 0xa1678d75c0f9970e, 0x0e94c4407c882ebc, 0xaff49c696b108ff0, - 0x8b85dc03cf67af83, 0xc3b8971a2c053ba2, 0x51b58872e31237dd, 0x270d8a4bdac2c029, - 0x592315a7ffdee7ab, 0x6f9616538e5248cb, 0xd3852c3c412ced71, 0x627199cb4cea1797, - 0xcce13b89571aae9a, 0x6d89ba4fb0251209, 0x1c9f50ab6c243e6b, 0x8aa0b93b2dc1383c, - 0x7f548b6e1f79e9ab, 0x0cebfb32a3be244f, 0x017d1af1793fd3cd, 0x00ff6670e1a38c31, - 0x36eed14d0e34adc7, 0x3abe441ae4f2a164, 0xb72d2b9e2fb7d8d3, 0x9da2accc7a9055af, - 0x97d125518e51fa97, 0x99c34b9a44826d1f, 0x4f4076a28ddf9638, 0xf9c53f64cc369d29, - 0x511e0048328995c1, 0xa88a11c39be88ceb, 0x2cf0f489fdd207e5, 0x9340574aef831f56, - 0x804be4f3f31293ea, 0xbb4faf64c2a79175, 0x7e1b92775f22058c, 0x64ddc9fad2caaf6a, - 0x0cc102dd78d54fcb, 0x75420b65bd9076d6, 0x43ad0f0faf4e6657, 0x7b7be9498e27286c, - 0x28ebea8ca428e47f, 0xb8261eee41c26b60, 0xfb9699617411b1bf, 0xb2f5a238c8e024f6, - 0x7e6243e5c35f8bd3, 0x9d8a370c1d4933d7, 0xf90035f232f83b9d, 0xde869941089e4792, - 0xbb56c69fc5e94c5d, 0x3c20a31cb2acfee3, 0xec681c5669d1aed4, 0x35f813bbbd63efc8, - 0x230bf3de5360fa81, 0x019ca9307351b1ea, 0x89483d5f294ad0d1, 0x2d26d5c3523a7bb0, - 0xf448b3a734829026, 0xea966ac84c0d84a0, 0xbbc75b0df9cb45b3, 0x223fc92dd877705e, - 0x82116ddf8aa5f946, 0xd833740e0435e67f, 0x3dd62d9029b2c9b5, 0x9d586c02e7d84980, - 0x053549bc36234aa6, 0xb714a37694615d34, 0x310fe9eeb979f658, 0x49195c1ebd4390dd, - 0xf8eca25389e4ad62, 0x3c0aec86657896d5, 0xb4ff852a39a579ed, 0x746e85d2c5b681b8, - 0xa79919f1fed354ee, 0x3b5af8c711d7552e, 0x2ad734a81d085256, 0xace41048241012e7, - 0x136b210fcacc5cba, 0x9030ac05d3bc5ef7, 0xae4581e0662ea88a, 0x9dab74fde9f5e395, - 0x97c99675e85ca95b, 0xa6daedb7fba8027a, 0x627efe55a1a9d222, 0xa1ca5bf7d9bdb3b2, - 0x7f9a4f4ff38af6a5, 0xe6daaa3132c047e9, 0x8ed319a4c49017d8, 0xdecec69b23dde41c, - 0x6252770bd96b69b8, 0x57a7550df92a1734, 0xa515e7cc2adc773d, 0x4a0275ace8116020, - 0x8d457d4f899101da, 0xab405c20ee2cc2e7, 0x3dc8c539033cc8f8, 0x1722b2c04c0bdb46, - 0xa5e9648b2dacd730, 0x7034710480a5bd9a, 0x7eac55fe42fbd17d, 0x09289386d2f4082b, - 0xd1a2f0078ce8def6, 0x1242812623174229, 0x08599f664d91c228, 0xc81c406a7396224e, - 0xd509938218066269, 0xd2343389470438bf, 0x3447fb9f941dae35, 0x004de4a9a48ac636, - 0x27ca90f4689c2301, 0x2f3412defbbc0ce5, 0x2ee5bb659869f6c3, 0x15261632db5911f7, - 0x4199cac2a1336353, 0xebf5478d0e80e1e5, 0x48e4fa2e689db485, 0x9f8fc1c1df2169c7, - 0x58acc70991dedf76, 0x32bd6dd6df76e594, 0xb1294d9a558addef, 0x939f07081590bd58, - 0xe7353289595edd46, 0x6843a58ece448307, 0xa4f54869bbaa00e0, 0x3b74ece02c301fda, - 0xe5c428dbe2f92851, 0xa968e8cd8720a09a, 0x04200b07e3772957, 0xf5ea6a41259fe96b, - 0x51ba8890646974d1, 0x23a5b771eb9ef59a, 0x8ce5a677e27e30fa, 0xcd275c9060dad20b, - 0x3db013fabfb3e7f0, 0x05285a98f8dcb9df, 0xfa1f29b103bbb7ef, 0x99b296a1cf43bdea, - 0xb4a6f3c0a0b38b3d, 0xaeb12c1aeb5f28d8, 0xae2f1f03eb3e3968, 0x9693b9a888df1b69, - 0xbec2bac7d73e3d13, 0x820cdad7cbb99d6b, 0xfe541ea866e19a24, 0xf8e2001b77881b99, - 0x4442c8e030e9d71f, 0x3370b7879f9014e3, 0xb37106c5ed768dff, 0xf5f96b4753b52deb, - 0xeb5b72aa23740779, 0x3c02fd793a6a8c6a, 0xbfb37fd699f63b3d, 0x637caaa95593d047, - 0xcc3db0ced7526e79, 0xda0ba925cfaab89c, 0x0a1e599f26764a93, 0x98619f3e4d9ae02b, - 0x9cfce1cf38a28a84, 0x49e24fd3eeb4444f, 0xc55ba9775274a5d1, 0x4039064d18391462, - 0x49bad4dba89b74f5, 0x87a4287d22a51523, 0x08007c717cdbd135, 0xfde0ad0c0dc31b17, - 0xfca05d8e9d566c4a, 0xe1c22849a715f865, 0x4f77083d9194bd32, 0x04de5434dc8aa50a, - 0x8786c84405db498d, 0x451cf2e8867c0994, 0xf6277144136903e6, 0x3d442f97db571722, - 0x14afeca0645cfaa3, 0x1ca827f4190b6464, 0x9b1a7cb30ee81922, 0xdedbbecf9c1f0335, - 0x7f4eb403f969fea2, 0x5ac4f8a651f1348e, 0x003fc5816243b999, 0xe5fe41ff42ee5925, - 0x973631b960509287, 0x06a13b8162813286, 0x9f9f8709acd52ff3, 0x5c11e094abaece8b, - 0x354a7913bd03c8a2, 0x2eafde865751f313, 0x0b8712a8ef75ec69, 0xe848590e2df8c405, - 0x60770245efcbc022, 0x218656367a1d7070, 0xf44c069e7f97795e, 0x72b0541de7a5879c, - 0x08358548d77ac787, 0x92e2c32dc055d4f8, 0x4cc1abe176a5d932, 0x9ec9aab004d3ff13, - 0x0ee5fd1c3b8aec79, 0x5d8f935675f05c32, 0x68ff2e62e4518e70, 0x501f9f82e0fca152, - 0x842603f19c33c874, 0xc735950dc7e2f996, 0x8c1d99be490e3597, 0x8e42d0e66230fe93, - 0xae329aa1564a80e9, 0x5649e17791c1ef21, 0xd02fb4de5056b33c, 0xaf53eb7e9c69160f, - 0x526587c48aeb958b, 0xe5bd7f689e8f1d01, 0xfa7c125443e9cf4b, 0xda376c156aa7605a, - 0x7ec566537c0424d6, 0x8c1a06af932c661f, 0xcd9125069014bc3c, 0xee08acb4bbfc0cee, - 0xe6501cae883efb79, 0xa6f190b1ce1a8394, 0x2c8f90c4a264b718, 0x34a813d4c676a289, - 0xd9ac28124c66cdb9, 0xa84842ddf899fee5, 0xd6f4dfb2165b1a4a, 0xdd9d2eb71a17b11b, - 0xcd78677ebb392b2d, 0xec292883c27357c7, 0xbce91e11f9830d92, 0x7aa8b68435489adb, - 0x54eb584bcdfea954, 0xb8dd62f763e7e58e, 0xa438f6df8db982d8, 0xca9441de3b544abb, - 0x1deda190c70f9a4a, 0x445bb6ced6490995, 0x76bf6a3ded7b314d, 0x8a66f016dfdfab07, - 0x64a21901e2bbdbac, 0x191ff6bb807fc7f4, 0x86b368995e99d41e, 0x417e6535be219210, - 0x1491867633f53559, 0x0cecb02863af988a, 0x8e4e386e752beffa, 0x4ba58fed077cf4a5, - 0xd777631cb7cc0ef3, 0xe11c0ccd18c90c4f, 0x9343426a5050f38e, 0x55efc291307ee56d, - 0x0fbcae5d59aeb627, 0x288b467aa5bdfbca, 0xc0a392953f680019, 0x48916fc91ba84c06, - 0x4a8b4836ba342ed5, 0x03142a9ea4c0e59d, 0xcf342a86d47380b3, 0x3454da8dc9c8791a, - 0x0260f14f96803fa0, 0x973bbe79a5f164a8, 0x49edc10d9edee956, 0x0cf5cacd8fccfba6, - 0xdb4f8f51bb534076, 0x470b7f9d70824b90, 0x1a2e4d589fbe3c75, 0xcbc03b148e363146, - 0x99ee702383e08652, 0x7d3a7bc2b1d17942, 0xe29dbdde2a789780, 0xd05be133837d8a4a, - 0xf2657c5783404faa, 0xe4c60bef204ebf83, 0x8af681c946077603, 0xa4c30856371c7ea4, - 0xfd8491493be53498, 0x65b4bb5c0b432d37, 0xf36cf84d1e69c2fc, 0x72540952b560db4c, - 0x926fe399b48b06bd, 0xbe7bba0600877983, 0x78756a5857287bcf, 0x39344c4f631fab92, - 0x8938f0d75f41bb00, 0x62950eb07231d85c, 0x151250c44835e534, 0xeaa6153d366040b6, - 0x0399ff3ab9d2029b, 0x4f9291ac66bea780, 0x5538b8774466b317, 0x104a472bddd01ce6, - 0xbb530a2727a7c317, 0x12a408915c974a92, 0x02d8e5447e228765, 0xbb6c944b06c2be01, - 0x860518c2841ca8a0, 0x68edab4683438cfd, 0xd5764f712a25913f, 0x14a6ec6cfa7c3115, - 0x6f0cc123ec798547, 0x19ae22cc367c885f, 0xd7be6d5f449e5951, 0x8555f4f8fd21152f, - 0xf3335605e1495f41, 0xb2ea3183d8e360a9, 0xb6221434c5c90c63, 0xdf215d9c2e93390e, - 0x98519ef2b0af8143, 0x06044c2d81e1edf6, 0x645f2661d1fc17e2, 0x469a5d2269c3b05f, - 0xca70212744447558, 0x07a4eb3b4b8e8cfd, 0xef200622675d49f9, 0xd7024c3ac1a6d90c, - 0x5c54fcec35596bf2, 0x030e135fcb8673a0, 0x9f1cc6537857af20, 0xbf75a9960af6a424, - 0xa1ce52907049fd5f, 0xb85d9f1d69b9e757, 0x85beffbfcc396269, 0x0bdf4af30f73131a, - 0x74e5c58ffc16a8fe, 0xc3cc87ed72fe0e64, 0x604c1bc18d443cba, 0x9a1b8fd3634f1d41, - 0xa33795d2516fad6e, 0x99ac9eb1c2ada062, 0xa2713d9be0bcb94d, 0xfd2f6b2989fdf4ea, - 0x55e33b2805a13f58, 0x2c7bd9290ee9b966, 0x81fd6f83134014ff, 0x179f8894deefcc6f, - 0xc61c8e56d5fdb3c0, 0xa4e5343bc6afdde1, 0x36ad5e9310f6d4eb, 0xd5fd90a79e27cc78, - 0xd8ccd606e4d48e8d, 0x25f66a9cc989fed6, 0x3d0402457fc4bae1, 0x612733dcf3d0d07b, - 0xe38fd4832db05d6c, 0x8e2809b7ea934954, 0x52004f18865f4734, 0xbb8ee10a8dd754c1, - 0x0fd0437b265a01b3, 0xad6498e5df9586e8, 0x8e143ecbe7db3bba, 0x17331ca3a86566a0, - 0x50cf16807d23b56d, 0x7c52f40e61c17c15, 0x70aff594054aca7d, 0xa9005fada5801d62, - 0xc2c62f1740a8a26d, 0xccccb11f991fb387, 0x2f38ade83ab14eac, 0x33a947c235f4d753, - 0xa5625e617031d563, 0x4f964fabfa8b3b58, 0x84b4e9eaead9b63c, 0x6df29966734e4146, - 0x11dd7ce026d0c29a, 0x9963e5fa15f8d521, 0x1e20c4ade063847d, 0xaed43f08f72a9080, - 0x3dd7ce36cef85b37, 0x78e5e46f7874aed5, 0x5ded630f68dab2fa, 0x9e41471fb56fd63b, - 0xda4d8c5b0c650d1c, 0x9939dd39cf15b48c, 0xc4caf8da2e6475b3, 0x783e1a913f0b79cb, - 0xd6131558db68427a, 0x544774b505054d4c, 0xa30cadbbabc33d9d, 0xa8d250afca0b7a81, - 0x36f07dfcde5fd8d0, 0xd25bb4feb8a0b5d6, 0x52cfde1125affcca, 0x95876763d947df39, - 0x916a38f31aa8c326, 0x7fbdbbe0054a44be, 0xf26e36f7c5eb7a26, 0xdb7ba35f9caacbbd, - 0x8de921ce9854cb9c, 0x83c48c8ba5f3ac18, 0xea86332217508916, 0x89d816e3efccd171, - 0xea8314449612a664, 0xf3afd7b04e3e9dcd, 0x0265cf8252d368bc, 0xd9b9f2c5b09ab82e, - 0x1be9d93315e7e8e8, 0x4e263044ea592f12, 0x4390a239830f4ecf, 0x55c249d481ef6a4c, - 0xb263e1e7553222ef, 0x11ea971a5642c42c, 0x7894d600bebd1abb, 0xdd6fb6941031338f, - 0x29568a2c0c94d52d, 0x01315555449edeb9, 0xb8a03366e13184df, 0xaafa5d4593a6277d, - 0x066b638835960be2, 0xf5215adaeb2e42da, 0x020ebe86ab18e836, 0x1999e5c85d17cc7c, - 0xf2d3af28af6c1437, 0x168c7ecd2b437f59, 0xeb96f37a2f474f63, 0x9de6b71e12e31189, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xce9cd33d29a59885, 0x462405f2ec37fddb, 0xe9f95e03de5d9678, 0x32a499f45d6d059c, - 0xb7a7c4ac40438c8e, 0x799106a722c97755, 0xeab902f42b28befb, 0x8fb93ba219f0fa22, - 0x5c27fe8e7eed5abc, 0x839ae78e6885ba35, 0x65b2bdd6b571e366, 0xc21239b06a82a68f, - 0x26426a1c44edf006, 0xa51b087898926399, 0x50dd06b95ff73ea8, 0x350fd6832d5d8a71, - 0xdfffa1d7439e847e, 0x50ef50696acbda82, 0x96e7b1adcbe64349, 0xa661c7598c159dc6, - 0x1d0b10a97d969fc3, 0x9ea34b494de74267, 0x4edea005f36498cc, 0x43ebbab5509fa12e, - 0xd1952984a42c160f, 0x76e1e9656957f8b1, 0x61c022aa2b712591, 0x33fcaa4055c87aeb, - 0xf2f3f62c4902c505, 0x2e04362a29622184, 0xf9a8906e4ce8d974, 0x67a92a7c276d6c81, - 0x2e723cc4046cf6f7, 0x46a58eefe418a4de, 0x54146288022ff301, 0xd97b03e5ce48c366, - 0xffc53f77c7e390f1, 0xb19ed264213168ee, 0x1d1cc4cef51ff370, 0x4b01018471a269bd, - 0xd649cd3f8e77bc01, 0xc1c33dee731a6569, 0x9c3debf67492ddcb, 0xc7aba687e18eb596, - 0xa47f2a85253d642d, 0x15ad913edb07477d, 0xbf47cc37cb52a10c, 0x2aa65065b10f5c5e, - 0x074b5755d6c4c79a, 0xea72e605eff0a4c3, 0x7371a011e2152cae, 0xaf3b1d61e69d2d68, - 0x4aeb6a54a16e2a41, 0xfe9ad97748c873d9, 0xbc8e5496ad667e7f, 0x4ed85c905f5fd5c8, - 0xd8d82279edb41f35, 0x04d8b0d26beb0ef2, 0x9e2f258b581de408, 0x0f29dd95946735ef, - 0x93893418172f05f0, 0x1568bfbb7707cf31, 0xffa942d8c58917b7, 0x4f196e37a79fb5c6, - 0xf7a207f21b7fd527, 0x9c7087c54110e248, 0x34599d521c75b202, 0x6f14aa2f1155ee48, - 0x0364d2d6a4f30f2c, 0xeffc87f7a5c96b93, 0x51e101275d5ce3f9, 0xb6e359d2545e7851, - 0xdb4a8673c9e8f7ac, 0x9d6ff1c9ed137d6e, 0xb0375b28098ab5ca, 0x5f9f838ae885cfc2, - 0x8b18c0d17dc8bd1b, 0xc98f728e4bcfafaa, 0xd49567385a62fc16, 0xa75b453d79e39f2d, - 0x6005cfbaa2142cb5, 0x16a1ea8f9993dcb0, 0x9e1bdd41d5e86290, 0x4e6889b02bb09fcf, - 0xc01af3d14afe6920, 0x2a10fc1dbba3bee8, 0xd5b1b81ec507c78a, 0x80b35ebe105e8a93, - 0x5663a0c61d9a4ab1, 0x29f2ce0a4970da65, 0x3437b0ee967e6486, 0xcc33fd7763b4c8ff, - 0x4c88c518fba76872, 0xd3bc1add7a67898d, 0x25f2c0438f15c0b7, 0x09e3614e693f9607, - 0xbf02ea16f5a094cd, 0x24b69e9ffac17b7a, 0xdc80a29a1d7cda5d, 0x15456aef44d51c3f, - 0x164377b5c09ff307, 0x5d3b7bb6ddd312b6, 0x87f264809e1f23ff, 0xfe227dd14beb4201, - 0xae8c9cf168344e2e, 0x50ff5fb8ddebf65f, 0x5173c819ce1ff7f4, 0x9ed606fef487708b, - 0x7e64c28e59b1d380, 0x5c3b447c6e9107ef, 0x572c36727f886cf3, 0xc9840d1b57464bad, - 0x2c8ff89726b24e47, 0xf892e0865e58c01a, 0xa2087103a26e48f8, 0xfdee2bcd7ef8dd1d, - 0x3b0af71e4d202b3a, 0x5f75bb3c3e51ccf0, 0xc16bd22b307df4a0, 0xa65aefa0ac503128, - 0x8a017bdc76ffc624, 0xaed8345f8ed38aea, 0x9602ca421aa4a41b, 0x528dcb8d80e86035, - 0x9a762a7150726cfd, 0xbdea0abb7806701f, 0xf83220cde8ab6637, 0xb1abb72b46da3b76, - 0x0c949efa3a3e00c6, 0xdf0d1fedee06373c, 0x7c51a6ec518d8988, 0xb2ee6d2cb276bb5d, - 0x688d3e5103671753, 0x697802d6ab4b09e6, 0x89fcd60d36e20fcb, 0x23a80b1141f19b8e, - 0x506ec248de4b8233, 0xef56c410b27be458, 0xed77454335dd6871, 0xc530cee3e23f9387, - 0x030bccd0c4749a78, 0x196533d441a8b1a4, 0x8e70fb05bd371e2a, 0xf11863a6929c5aef, - 0xd592ae9078402490, 0x8a52f050f98e9cbd, 0xf9565b2d0752b3f9, 0xdd3f0411d9208eff, - 0x011861f3bb22f806, 0x8d35f291fe54e90a, 0x8e3072b46642a9e4, 0xef2e5ab5bac38983, - 0x0499e866da526590, 0xd8cec600d216861f, 0x969fe2788866ae79, 0x4e7de95abbff7b21, - 0xab967470419117b1, 0x2a4e69c1efb6e380, 0x2b30525b18f750e6, 0x145d53c14819d8c0, - 0xd62b283f3e9377b3, 0x76aec7693edc9101, 0xf4108695d27d5c15, 0x19925416b6c3bf42, - 0x01ac9f16984d19c8, 0x56edcb1ad89f2722, 0x74eb0717070c195b, 0x1e06eec0e7654599, - 0x3291c3fafc4de6b1, 0x4320ec8397d905b7, 0x928eaac9bc81d1b6, 0x41df3061bce0495d, - 0x7910ab7408f81a8b, 0x11cb4e677c1d2c8b, 0x9433d55ca6cdbdd3, 0x697b5b6b6679dd72, - 0x0a534b67c710c4bf, 0x3b8eb34678a993dc, 0x7e5bb5ab0d399626, 0x584009c610f43d01, - 0xebf90e47110fbcbe, 0xd7ac0e55b5e96822, 0x2fd00b7ae7692bd7, 0x45e1b18bdd7a6c69, - 0xc8516a0e57f9e6b5, 0x6926df2085a6344c, 0xd855d061b9c1b838, 0x48cd0f9525a992f6, - 0x185d5a3e13ee333e, 0x29f384731b7d0d80, 0xa9e5d8e653987dad, 0x3da10316f3ed0572, - 0xc2d82daef8504403, 0x96ef96361517796d, 0x0266355d9cd05ed4, 0x2c7a6bd43d20feba, - 0x78895efb12b32745, 0x3742d880356da9c1, 0x417e34f04029ec81, 0x884472fc8ab3ef39, - 0x940f5146848c871a, 0x8ad19248c6678757, 0x243da57d235ac0ef, 0x28d0fd48abd7624d, - 0x6158403bad70c132, 0xf7ab64a82ff7bba0, 0x52619046950c6bc7, 0xb2fdbac92c4d3946, - 0x083dbad044e357f4, 0x60b68c624410adb9, 0x2e6ebfca8dbfd99c, 0x71548e0d5a7f5c78, - 0x81d02f04aa880e60, 0x6357ccf13d437a75, 0xed94ffeabfbfa03b, 0x8e384a7755249be5, - 0x358d5c7e649055e3, 0xb6d566c2e5a72813, 0x3751781f16cef998, 0x10820cdda14076a3, - 0x3244654a70004f7c, 0xdac80e58c4a1ee30, 0x39987eb1363cade4, 0xa1a250c689823cc4, - 0x9248cbb92367da33, 0x5a18469337b6bb34, 0xa9b533e98d982f2e, 0x78b843c8248b3bd7, - 0x47961e2a98984b8d, 0xc9a169f7c1116104, 0xe97a998bb5afc87d, 0x5a05c03e91324046, - 0x0281590b1e3ec8a6, 0xe973965731f3f574, 0x02ad4d1dc419d93e, 0xe15f6d7e84b76d7c, - 0xe33572ca39e2e64b, 0x4bc831af534fae43, 0x12b5c7ab2c5055b3, 0x01f1761317f91623, - 0x18c1bc9d6eb955e8, 0xa8d3380abbf3ccde, 0x4de7e773238d55cf, 0xe4e87d9489b2ebb5, - 0x13490e826637bef1, 0xbe42eb241fe28bd9, 0xd7217da58121a512, 0x95c6210e3bb74e3d, - 0x7d1ac6eedf3d71c7, 0xc95c09597429a11c, 0x7e4e0784ab24f335, 0x6a0648890192e2aa, - 0x9f2ab43c574a70ee, 0xba220b29f5ab259e, 0xa577d4b85131e5f5, 0x94807b00167be83c, - 0x0886b0099def3843, 0x828919abc4bfbbd1, 0x15eae97bdecfa63d, 0xed521b0bf5c81e03, - 0x187f918c504e6983, 0x27a6e27e62d4871d, 0xd9ffd99800a55bb9, 0x2c25c3bf0e9eda64, - 0x48f921d410a43fcb, 0x28bf5c3cc5d00501, 0x3107de4e939e67b6, 0xd157c16234c09df1, - 0xee14ebe712abf81e, 0xa405ba5ac727f573, 0x1cc7ed1b4c220078, 0x5d0d63ce07e8a984, - 0xafbbb4e4fa95252a, 0x8ab805922a2db691, 0x9f0ee146313a0559, 0xcef92a673919d257, - 0xce5b33395f8ae402, 0x9667edef08e9da82, 0xf40b3cf7bb5dcdf2, 0x94bcbdf184e25cb1, - 0x5dedfc82529ee997, 0x8ee63d6a9e1f97f2, 0x2481ea70aec7978d, 0x1e9f21f31414a9dc, - 0x2fda35696dd6ac60, 0xd4069eddc09b6d7a, 0x6fa6d79e8c61bad5, 0xe9124037debe0315, - 0x875d366b1d3af1b4, 0x4ba41987997eedb7, 0x88991624e9ede339, 0x5cc4dd35a2482721, - 0xb367ccf9a420d963, 0xd8c6563f2d4b937b, 0x47c2fd161e4fff57, 0x48a5aede3a1c7445, - 0x46cc670e381c63ed, 0x5de239585075f497, 0x6ac290995c3d25fc, 0xd64b13a283afe917, - 0xad82e07e9d132b7e, 0xdebe6c3d1234d710, 0xe12dbde426a63b9f, 0x4e3bcdec466c5c1a, - 0x22302f851542552f, 0xd59a0a6d1c0e5ebf, 0x8bb0fcb16e2e5603, 0xb1860e0a5d47bacf, - 0xc2845376cb0a34c7, 0x22fc99b6013828d1, 0x777b72cd26525066, 0x2eadf98f42bd63f3, - 0x039531f19d8d7a7a, 0x0d3d6818069762e8, 0x59cad112f2434602, 0x86f0ef66fab5f4ce, - 0x46b20d18f3e6240b, 0x7abb0087549e08cd, 0x5d8bd59edf06503f, 0xdb7f76320fffdad2, - 0x4c381427bafd5b9a, 0xb899d108ae20db81, 0x2fdf4633fa384d1b, 0x94b107e531fdf6ac, - 0xc9c5bceb7c1bd34f, 0xb3581b658ac0d99a, 0x1f8afbaac6afbc2d, 0x70d6156d43c6450a, - 0x842926ed9f05e1e3, 0x1e6d518a2a71b02a, 0xd5b0888842e7a8cd, 0xd07ad72fd30f6845, - 0x51f592161ae80305, 0x70110841dab4a05d, 0x240da5a951529478, 0x547debe7625e8e50, - 0x02b34cb5e18d3268, 0xb0b8de2466f61cc3, 0xeb320d935ea9087a, 0x2fdc0c69b8b029bf, - 0xf2f28aff0fdbffd9, 0x3b6f9c1c70a1985f, 0x80fe1e3be4166bd7, 0x559e5a50eecf59a3, - 0x1730272bfd21abac, 0xc63ab0c6b68d10bc, 0x2e83d0952ef00657, 0x133bece3df9c86d1, - 0x3a3bfbb20cc11064, 0x3f71cd6fdf036725, 0xf14b04848c890f44, 0xf08282b4dd96bdc7, - 0x156c145f9d3bc14f, 0x35f0e9ef3f260eba, 0x18873d2eae28b782, 0x75bcb82cd079f490, - 0x28326233aa654d60, 0x8a999075a5ac1ff6, 0xaa32d4cf56624d0f, 0xc2f2356730590c62, - 0x2268f26231545ef2, 0x316ccc3d2a182e52, 0x2ede43ce94bc6939, 0xb40f57b078f099da, - 0x326d2b364eae3a73, 0x5f0c296a215e1b16, 0x8163d17b3db4f546, 0xdab0077eed69c17f, - 0x2e5d45958ef8de80, 0xe8c55b5a5d0da008, 0x9402720096064471, 0x3360b6f5e0bb04ee, - 0x5a4ccc060bd22de5, 0x71be88fd4db728df, 0xbb327ad0fe1af6ec, 0x4fc7e1315c502a94, - 0xb6c66ac6efb9a6ac, 0x89b4420f6243dccf, 0x88117f89779f1e8d, 0x1d23bd79a54ada3f, - 0xce21ad66ba034da6, 0xdd77d94bdc916b84, 0xa1184047d8426aa4, 0x308844ff7cccf377, - 0x11acc7444bc1b5f8, 0xc9592730fa52948f, 0xb988fa69c01e0fbb, 0xb191f4e6158c55e8, - 0x54817f8639e3a35f, 0x9bbcb207e9b2e285, 0x022374a826b2e928, 0x94fe1ef3b1b5a317, - 0xfa8a3242ea88bf68, 0x84105d09f940f62c, 0x3d631de06096d16e, 0x3e1643ae068e99bf, - 0x07c632c36e8d1052, 0x3a66f06be5541ee3, 0x93a926f1069add4c, 0xf8d149ffdef30ac2, - 0x6d083df690f8660d, 0x5efc819dd7142b9b, 0xe5360d972feb64f9, 0xe27189bad8a028d1, - 0x6277d7c8bf8825b5, 0x07a62b8dc5ff4875, 0xeaadab7f777ea6c8, 0x4ab26bfad6ab0730, - 0x193fb99a8aff3bdc, 0x44e06f020052fe20, 0x047499fd10c250ad, 0xb6867c613cb94540, - 0x36a583541a5e6dd4, 0x022fe6942417e0c1, 0x557ca9a79a2005fd, 0x23ab3219f130ded3, - 0xcea5619744cdb9d2, 0x90f5e84444e7b67c, 0x999bfdb1b0a00321, 0xb9bbacce16f1e0b9, - 0x7a0199f827b1c6d8, 0xbc5e715932f5b276, 0xe554faaa36ce8d11, 0xc947cc430b199be1, - 0x1cde4e48062b9923, 0x90c84902e022c26e, 0xabfc2e4016fcd0cf, 0x15e54c54b9c6e34d, - 0x4d202a1c72ce809a, 0x0f476ab013304cb0, 0xc89feaa2a608b38b, 0x9d10aa03500a965c, - 0x49e4de560a3e3120, 0x87cda8f9d62d6c30, 0x0ec3cee23d7493c8, 0xf2527e6a66f076af, - 0xd2b598de696d8425, 0x8a90c9b8efb64665, 0xc15499c287eb2305, 0xc90f1e0566bfcc79, - 0x37cec803ceb85aac, 0x169488496f78ef0a, 0x8e8183fb0371602e, 0x9399accf97ada886, - 0x8469f7036c59d099, 0x50d075ac36299216, 0xb9591eb6c07b0ef6, 0xd6afe6ace6a660d3, - 0x62417340f518dac5, 0xcb62885b79111c46, 0x32dfed8cac9cbf18, 0x66126697ef9ec8e7, - 0xcc83dacddbe63414, 0x1e6e36ab7bbbfb5d, 0x2e5f5aa5265a2530, 0xe4f35d17fedd405f, - 0x8d706506aa031bba, 0x157ff59964322be5, 0x42464562e8c99089, 0xf5dbf0b014951b8c, - 0xfe171b5720fa323b, 0x497d8043a2e2177a, 0x41f4eab08a51276d, 0xcf4d14c2bfb42d3c, - 0x7ffdd0933902452f, 0x6c6b6d8c4b0bc757, 0xa7bd93e472a74c65, 0x9e9c4d8c33b0af42, - 0xf231c7c356874f8c, 0xe8ee3ec4868623bd, 0xec55f6e875607c21, 0xeff6cac772fcbbc6, - 0x088be87fa8d71291, 0x65960d4d2a0568d0, 0x60c9c6e644cd577c, 0x85972db8164dcf7f, - 0x5ea8a6c7c3d81e99, 0x6d183a1b547d4790, 0xd276b9b33785056d, 0x443b854cd2e6672d, - 0x1215b60944725a76, 0x5f15c8b7bf72c972, 0xfb6888890cc71a2d, 0x467450209a4c9320, - 0xbb87220766d0bad6, 0x1f7f41122eaeff3d, 0xb7dd3b84a9fdea10, 0xe0bc3b16a557bed6, - 0x6885b15ef2ecb0a7, 0x47c3c8283bcb361c, 0xd490da14f3cefdc2, 0x0b9f219ee1f2479c, - 0xff29426234aeb178, 0xcf397e585ac1f35f, 0x0022192c5d6e802e, 0x2930c2c41e433852, - 0xdfb6b15b2891011a, 0x607848ee9f009643, 0x69b3a438994a9f4c, 0xa6199243c826acf8, - 0xa4b60e9f88d5090b, 0x71cf385eee46a0ca, 0x2507240546a85abe, 0x4cb4b627f6de77f2, - 0x4922262d13a9821c, 0x7a6984a32f85525e, 0xaf18252f1b75dab6, 0xe6a673aa7eef2180, - 0x7806edd756a2852e, 0x74dec606e9c06cb8, 0x972926ff1122abbd, 0x7b2491f56bc8b2c9, - 0xfeddc9ad10304871, 0x623da743b20aa41a, 0xe08e311086fab7df, 0xb8b2ed1c6a474a36, - 0x6842096ac2b31a74, 0xcb13d8e8e42dee17, 0x361770960a4dd20e, 0xd9b4fd6d5e564efd, - 0x2abfc723620697c3, 0xbf0eedfd2929a5da, 0x4ef2fb426b8a4a05, 0x35a2f2aca2672d51, - 0xcac536f65747f730, 0x7f7365571d4c5a5a, 0xaacec4c22044e4ce, 0x9682b3cf40290405, - 0x0c49f6555ac57a70, 0xde00b0c17f90c740, 0x84ecde35c6627a91, 0xd116e096c81a1217, - 0xd2cbdec7f45eef5f, 0x0088d8652b1f612f, 0xdae2318aafaf634d, 0x9327b132136fec65, - 0xfe1264e759b15b5f, 0xc90c6d42f891d08f, 0x50d5c463773c15ff, 0x18fa2a8638055787, - 0x3aa1148f3ae55d80, 0x5345f5058b5b3017, 0x36a05171c9be1eb7, 0x28f8a6383658c5c4, - 0x11bcd6a27b089b3f, 0x97229d7f2042db2c, 0x286a34e0c511a4a2, 0x56f9b355ef304101, - 0x3016625356128194, 0x525e1e9b15bbe6bc, 0xa88492e1ed993063, 0xdc7e0427045bb569, - 0xff92e547c0b776bc, 0x42707325cb24de9a, 0xcde793e6692e32a7, 0x01082a5baf3b265d, - 0x5627ce679c7eecd7, 0xb53af94762d9451e, 0x4ac90a2e46a22b54, 0x0636eb94088b5b2f, - 0xaa978edc05e58c33, 0x4857468d8155351a, 0x30ad1dab04165d7d, 0x55e1ca5c8c5284c1, - 0x7e9ba02bc3ceb39e, 0xf8384e0b448134b1, 0x0ab20284b59fd1ef, 0x4776aefb883f09cb, - 0x65416199a6f94b6e, 0x18d6eafa67511359, 0x5f020b18d63815c8, 0xca1a1c12103458e4, - 0xa62b1a0945f55f60, 0x68fdf99ba234bcd3, 0xddac11ada8cdd0c2, 0xd1a3a037f239de59, - 0x55398c5dcee915c0, 0x9d9e88eba93435d9, 0x7150dd4926c6c641, 0x7adb157d082a2fbe, - 0x7893957747ccd1f4, 0xff85909d1ad379e1, 0x164771f773f27c50, 0x74a7dcf2b5cbcb63, - 0x7ddf755923b878de, 0xdbafef8476cab630, 0x84f62cd6e27eb046, 0x4126b031ed360e54, - 0xb43ef63dea606aaa, 0xd7bedd3e0e1049d3, 0xd073705f4b396251, 0x2f930afd9cff2d29, - 0xab7accde57244b1f, 0x786a149e20894642, 0x2d5da33a4d29f1c7, 0x7db3463918f02528, - 0x35954cee1708079e, 0x1d327e6afd118663, 0xc4a6f3bf68487903, 0x5c96d8e500cfa623, - 0x423b8e0b9bb47fef, 0x202a3278d522c149, 0x286ba25e3af87e2e, 0x604dc3f56947be29, - 0x98b284acacbf5f5e, 0xc3eeddd591232f46, 0x0dc95fe8e31d6cc3, 0x58a478931641abd2, - 0x9f062abef9b339b3, 0x132e29390a003c62, 0xf4d69c29138a3a16, 0x4d64ccdc28febb6e, - 0x727ee4557b15798b, 0x47cb29c8fc1f166a, 0x05bef6ae762e9843, 0x5782085ec487599a, - 0x3ded0df93d96362b, 0xfd897a36895ca7ac, 0xdb7d2909aa7d97f1, 0x49a44a61f94a7546, - 0xb40b1c4bb90de48f, 0x90681a39c7fad17e, 0x441694c44b4d59df, 0xc38a234090b8d470, - 0xa379b9d3c65e7da6, 0x93b08a8b49504714, 0x920002e636d314dc, 0x382d5bbbfefc1d58, - 0xe71517d01d13920e, 0xdc0d7b1f76c2f9ad, 0xd5fb7ef326e107c2, 0xa897c68991634567, - 0x09e96eed6883bb00, 0xf8db4aacaad9fed7, 0x96ad16e864a5798f, 0x56b55259f64a1e1c, - 0xd852d6dce3cabad7, 0x58139456004a3fc5, 0x2cc513c4870cf150, 0x710149c0d3f8d7f8, - 0xf5acb979e22b5067, 0x485233ec0d78802f, 0xa67ffd82be61c5ec, 0x07d2513fe2230711, - 0x93a57e3ee20e5c3d, 0xa7622cca0bd4432a, 0xa9e148d538338925, 0xd15a513060fd750f, - 0xd3194239a1d0e137, 0xebbcae0b6759e946, 0x1fa3e1fe3cf41f7c, 0xeae630c9deb0e975, - 0x2b62de7119fb8de0, 0x817a6537bbfe9ab4, 0x287149ea2036b9ac, 0xfccd33dc0a22d5be, - 0x3e2a52ca23d60c87, 0xec7ecf387f125504, 0xa1419faeb7d2e02e, 0x44db7c65160663db, - 0x57c079ecaa6c357e, 0x4ccebed1238dafb2, 0xd5df0f5abe753cf1, 0x850a51369e5d9b37, - 0x04432b31776fb69a, 0xf06f937911a0c85a, 0xa9b1a7ff5d2427aa, 0xc2857bb49ea2b24b, - 0x370e72e71b0b2ddd, 0xdb2e0e456c00285a, 0x16b20f48bf1f1b7f, 0x6ac9f5e466d30b06, - 0x559930c88603bfd5, 0x14926a510f635e09, 0x38cd43e02646f01d, 0x83e17142d7621622, - 0xf8ff2022c9c1d9ad, 0x16534b9f3eec6fe6, 0xe0e086c796c847a2, 0xb9276bed868e378b, - 0x82c8fafd0f198373, 0xb5da23ae85c3285d, 0xcaff4821c276c17e, 0x535c75d372a282fd, - 0x6a32cbb4f78a360c, 0xe65b410e7821f0e6, 0x9198f50e39456b56, 0x9a789ba684ce9559, - 0xc967df274e5066d8, 0x746cae24a0e1ae07, 0xe6e991a9b0636786, 0x97d116f1a6670ffd, - 0xa3a127a26eb5b4ae, 0x3b5caf42b7e5420b, 0x689c017afae63272, 0xa04e48e8d220fbe3, - 0x60c45400321f042d, 0x945fae85fc70a6a7, 0x20e9615815d23dfd, 0xc08196421f43fa7a, - 0x3dad048a6c684a12, 0xd9dbbdd2f29abae6, 0xc59246bd6811629b, 0x7f7c561749d690bd, - 0x08928468e07155a9, 0x2e1c4537dbb6b874, 0x75897ab6bfca4d1a, 0xdfcee05834957c03, - 0x81ecc8672f2e4278, 0xcd0fead39881f35b, 0xa8320620f96315ec, 0xfe73bc013c2eb892, - 0x9ac8f391f05a117f, 0xa447613413670fe7, 0x531e6e6edad44821, 0xea504168dabb56fc, - 0x6a69fb32be89e497, 0xd929ee788cc241f2, 0xab2fd4162f430fc0, 0x6947bb7dfd9f5f17, - 0x8cfffb87f8dd6367, 0x6b777be20118bc95, 0x9abf37d1c8385483, 0x649671a7d2962159, - 0xdde8243bbf3fbbbf, 0xd58659b8a6a43fe0, 0x59bb58338eff9a74, 0xcc70c2517e3acad3, - 0x5788cf013de85217, 0x726ee59b3a7c2bd3, 0xa7b3f5e24715c959, 0xc8748945bcec97cd, - 0xd65881223fee88f0, 0xa775fc34d29d243f, 0xc2222c04196a6bad, 0xc3b1411a427e140a, - 0x3983758ed8d8c35a, 0xa090791d7ba0faf5, 0x8b2d1f2436617576, 0x52dbd9e0e37e48cf, - 0xce65d7a421b6500c, 0x193f1253747ed001, 0x8d4ec1fcc89fd4e4, 0x20e974073e00bb75, - 0x73edde237f81d22f, 0x2484d1999381472c, 0xe4e406643775db15, 0x856fba731aaf09a1, - 0x9913c932e613b0ce, 0x2640355d2e477c24, 0xd7e3d273749a9bd9, 0x4cc35e3cb495fe0c, - 0x63899ba256026ac0, 0xead5e93a07cecb0e, 0x7c620b2bca7a151b, 0xe1eba6e65f69ce8a, - 0x41ae135d1c88314f, 0xa4fc3ea44c1f2b96, 0x1fb948786c605141, 0x81d0655ab3f7bafb, - 0xdad4848bc8c0baea, 0x7e28fdeaab9a28db, 0x6468dded75dbf8f9, 0x2ef2165301f13228, - 0xeba46cf4a77e67c3, 0x34d613b52a79dce8, 0xec5c2a061b18efc1, 0x4c284c8da29af2be, - 0xe1fa5cc4b5a007f1, 0x08eb9356c35bbfba, 0x6fd8aa3ffa19e22a, 0x4a21328128b2ddcd, - 0xb4fa943ad9e8aede, 0xa789c852122cb2cf, 0xf12d42439c2791b7, 0xb04d0d03d3792385, - 0x65d57acc0ff444a7, 0xc033366873d5c00b, 0xd294b23e78c8602b, 0x59807a80172eed79, - 0xa5020f687ce2a759, 0x87a6f7d0826e755e, 0x61e6285db706bfbf, 0x018c5dee15f328d5, - 0xf7c4eaf042d9c573, 0x11957412ec339d79, 0x52932341cee6c7e2, 0x0403f37e50c6274f, - 0x2f4d893dc949344e, 0xfe2eda82948a93db, 0x41234e80aad710b5, 0xe751ffef185137a7, - 0x9f1024297803752a, 0x56d6c5b90e737b77, 0xfc9c5d74ac9e4cd0, 0x1be479f113df8ed0, - 0xc085038cb762c45a, 0xac5c34eeff1c97b6, 0xccd051a558afa7d5, 0x456a3d85f945b88c, - 0xafc366c43fc7b271, 0xee442112df2648a6, 0x7fd202aa89f7a102, 0x29dde3a2863ed340, - 0x63d764e2743b561c, 0xa6e1d40bef68b20a, 0x9617fa08e6bbc892, 0x987113a5151fa4d3, - 0x284af24ee0967178, 0x6d02a35dc8996421, 0x4beadfd485ffc06c, 0x4eb18c652bef47ab, - 0x82b3ddf40e4dcf81, 0x0cbcaad37611cd7a, 0x465108cfdc386a62, 0x6a9b016cfe84e197, - 0x7e0005b6cbb7995e, 0xcc698421aa37d75a, 0xa79fa31e47591250, 0x6150afdb8404ef70, - 0xd6060c1f885337d5, 0x3257da149e09292a, 0xbfa139d2e1524ff9, 0xa98f4e6f5b1e115b, - 0x3e1730621958afde, 0x0873224814ac4379, 0x3419091555e0d493, 0xab4ebdeae9c8a7d4, - 0xe86f4d35bfb1fc27, 0x44e3c686773a38d0, 0xa2b6367ff6bf5b30, 0x4f3d0cf2c552a529, - 0x4d14357098e94c2e, 0x416f11ccf7f2eb57, 0x9692a1a532ae6bb0, 0xc3250774c95d9480, - 0xda7b21617678c39c, 0x16ff125cbd2b1654, 0x4e227d2371754faa, 0x90fcde2ced84ef10, - 0xa4e305c839503f84, 0xf5239010ae84c37a, 0x668868c0772b9838, 0xe232ba6a67fc12d2, - 0x3555e3866690e574, 0xd7b7d75a3c8340a5, 0xcd92f2c093f274e2, 0xac6682eac53a6b0e, - 0x333a3f31e3d066bf, 0xfea9a401f74c110c, 0x626e1fd4b72687b9, 0x9f5d05bcc859b662, - 0x027a2cd31b9d6e9a, 0x6dfd9e90c1e5eaf3, 0x1b44cd8dd5580a20, 0x76a9a9d66000c35c, - 0xcb9aabe58bde2ef1, 0x4006d33c1942184a, 0xa5dc60b5d1f0bca2, 0x6916fa84ae5ffa7d, - 0xbfb074f9ae3b72cd, 0x8d056bfe84ebc669, 0x064d8640ca0c3517, 0x3dc9cf6b51665764, - 0x1c9513a9badbc9c9, 0x33db88d2dbc8ca00, 0xc43db9d873a0b09e, 0x99750c4e3d8b357e, - 0x5b61235d5f813840, 0x8b8bfea400b86a9f, 0x645fc6002bbf066f, 0x3c14dd5aaab9e933, - 0x11e625a05dcb4f42, 0x40413acca73ff872, 0xcc649da0cb6588f4, 0xe4a1be8969502a32, - 0x82684e864fea4804, 0x985457adef73f41f, 0x7ac9fd484a65e6d6, 0x613e43717fcd9461, - 0x7785044a7ee7622b, 0xe0da6db5d19483d0, 0xe0870995d476d95b, 0x6f602ebcde5a6619, - 0x62494bef713407da, 0x9fc3015d7a120ef0, 0x1f680d8d7e84c29e, 0x2e2d0d2f38548ddf, - 0x2ebddb4b32eeffe4, 0xe5c10c326ada7f58, 0xa03077e57a479144, 0xa16774283cefc675, - 0xf48d75a2df25befd, 0x7700d6fe0e3dc28f, 0xeb8c32dd6db708e7, 0x1b92d562ae031022, - 0xaffd80097081a71e, 0x08b192b0654c08c0, 0x80af12e090c2a085, 0x6e15f183065f4ba3, - 0x15b9fdef5abd3e38, 0x970e865b2ab86e5c, 0x488d35a9b4f4f7ce, 0x0a2ef6a631e71e72, - 0xe93158b8c7ff856e, 0xbfa9021a992f1777, 0x84b13d3b9e4807b7, 0x8f5d6e9faa3cb0ab, - 0x6f3208ef5e009ba5, 0xffc109c0bdef5f8e, 0xb649ab5201216c26, 0x72b176118e94382f, - 0xaf0cbde4d4746d6b, 0x104888f5f0354c16, 0x51544859073fef15, 0x090740f99f3ed7c1, - 0x0663517091cb317b, 0x6485fc30210b0700, 0x8011dfafe1f1e18c, 0x91d95f012c4b662f, - 0x1b483a68320b3b73, 0x5523544c2283006c, 0x37707cd1d1f8220f, 0x2e79faec83d19b25, - 0x48bdc14a2e52f984, 0x92f8de0c70cd069b, 0x5efcdc28edd6a4dc, 0x43d3190e41362848, - 0x02bee6b384a82972, 0x8f321209f897e44a, 0x293875c2b93e01ba, 0xc72525236046ff50, - 0x0872ae9f3e528dce, 0xd2175888b7e174fd, 0x82dbb198782f9669, 0x0831edc494c5b0dd, - 0x2ead0626ce4c116a, 0x7a184a4356e37479, 0xce0f0ba1724b4e24, 0x106c07da4facf8cb, - 0x0c874b32816cf0b7, 0x6793a87df29f9e87, 0x676d1f8f6b2ff7f4, 0x2fc0514c6fc036b0, - 0xf76d7fd1dd6a7d0c, 0x811aaf3755124345, 0x50a77086a2ead4fd, 0xde90b81b71d8b412, - 0x2f134daf44e59f0f, 0x01092545bad475a2, 0x5868e8b26586495c, 0x5091d483dd849151, - 0x2ab3413feece9b22, 0xdedba3fdd59d4c96, 0xeeca1bd4a4c50970, 0x3e313d5ea3e73147, - 0xc218a69c78848011, 0x7c87bb26d8b77446, 0x180d65a9f22d57f7, 0x7f8b7f44bd286ec3, - 0x3e9d5c27cff38454, 0x8b7162045b986129, 0x31eedd8bcf746fce, 0x38e97e8921cc3a6a, - 0x55bc6968a565f7da, 0xf661e9351c4f789f, 0x339c6e48890bd0f8, 0x308afb6f9913b380, - 0xab9d4217e388d4ea, 0x26d737f5aff41841, 0xc0eb3015371604bd, 0xdfc7fa471860ff93, - 0xa5477056042ea7b0, 0xafde29f6ca94c465, 0x4b2ce9a6b22ae061, 0x7345a17f576ed214, - 0xed772a6d23a3587f, 0x380ec2389ec715ba, 0x067d8735ba51d8f1, 0x6ffd1fdb1d391b8a, - 0x796e518b42503b6c, 0x6ed597130f567327, 0x73d5949fe348755d, 0xab24e8c4aadbeaba, - 0xa292c13a36061421, 0x057ebc08577913c5, 0x0c8471df91468267, 0x6b38a78d6861a13b, - 0xce75cd1e0cf35ee0, 0x344b8615f7ea78d4, 0x6a18bbddf52baabb, 0x7ac85e6e9e88d446, - 0x01b45e7f78905045, 0xac3b5ceca7469489, 0xf33ae43bc090ef90, 0x1749bd3d527ec2c7, - 0xa8a792948698d22d, 0xe5a8f633159aa8f1, 0x20b7b6afb81435d3, 0x7f8d8b23236278c7, - 0x858b3801da91a76d, 0x7daf88dc407cebe5, 0xa1561aa8427ee064, 0x21344e335fe454ed, - 0x968439626560a8d8, 0x049f03a5b9b8f5bc, 0x6f71a001487119dd, 0xc0662f8bfae92c62, - 0x02f8f41f021396d7, 0x623a7322f1593bda, 0x8015de4b1811cea7, 0x6617ae9b95508a7b, - 0x42a3df0dcaf35414, 0x24912b7d3d231ea5, 0x22c2ce3755d88cb6, 0xc1e17851d9e42ab1, - 0xa4ab5e0b2a35430c, 0xbdec579c5e68d45f, 0x5ac4bcfddaf74d4e, 0x60a9a380525dd890, - 0xb6cc29b6591628e4, 0x55d2e0a1ac916355, 0x16665a96b6b5974d, 0xd66000fac9ced98e, - 0xb6d6365954b149b7, 0xd9d9671f5b740055, 0xa024242467d2bc8d, 0x9ada14ce72667eaf, - 0x6ec9333668e771ca, 0x10ad27cd8f6e66ac, 0x0a7bccfba5dc4956, 0x0369d0625644c922, - 0x3588cb71586578ca, 0x6289b1cbfd434c6e, 0x7216f0b22c378d27, 0xf6f77a7e0ce470d0, - 0xcec1605249c1cefe, 0x8126962615ab8885, 0xde2595bb920c61a0, 0x806608e592892efc, - 0x3a046ccd796a225d, 0xcae7b48f7a75865e, 0xe30fd98a3a5747d5, 0x134d0b17e5b14307, - 0x2ae5066337335ab3, 0x6dd09ba9dd08399c, 0xc773e49ae400f47a, 0x05a4a6492de5beba, - 0x4b0613c897a4caa1, 0xc60090c47c5b2b8b, 0x8c4ae717922add15, 0xbb7abf4308325930, - 0xf1103730e0217557, 0x0af9477ed669fea5, 0xbdd95510d3e36d1e, 0x72f2405f27e8add0, - 0x6b88b7ba6f338839, 0x62294c34e0fb568e, 0x4863f78bc28ac8de, 0xb8b06f74cbc60218, - 0xb657b6df12121b68, 0x50bff128deffb87a, 0xa184ded9e0cbd3d9, 0x45c9136d623fe4da, - 0x784140ba00e3130c, 0x5d30ba34d8f5636b, 0x2f4da588bd4fab71, 0xff003c8617e5e6a2, - 0xf0ae3587fa914e03, 0x2bd066ea89e927bb, 0x2899fe299a990128, 0x98072110bad8d28b, - 0x8758eba8b3dedfa4, 0x27ae832469fc1304, 0x1d8b9df17a66370d, 0xdcd56424dea40687, - 0xc26dc42e1588e98d, 0xc78e5761337ec2a8, 0x49116d6e54ad59b5, 0x9602db1e3010b517, - 0x6686b3acfd4bc339, 0x8d55136c5d375832, 0xdaf5208d6c4b805f, 0xb754c37adf25a422, - 0x41ce33198e749299, 0x31793d3ec0445729, 0x03eef8e00df89311, 0x7cb6481a8ccaa4d1, - 0x8c5a4aff05730e1c, 0x2b44bf01ed5393d4, 0xdfeb429210077af0, 0x4ad3aa4970bb8478, - 0xda01b1b576b8dea3, 0x342a3ad1ab13fe37, 0xc3921c78a353899d, 0xf618e8ff78ab72b0, - 0xe26064a09865301e, 0xf0fec72de8ea4ab5, 0x538408e5a1a338e0, 0x43a36c104835d967, - 0xefe3d3c3422b362e, 0x54ae5cae7d9ef0f9, 0x87bc19cfecdd5b3e, 0x6676d8d951ab8e97, - 0x2de8f0b36955d437, 0x3574f2a188a3d6c2, 0x0862af497eee8061, 0x5f31bdb4441a7cbc, - 0x7cce8cc27cf8b6de, 0x84fa5081d60a10cb, 0x739c89f9d5b939fa, 0x3351be61be26e7c9, - 0x4c1fefb0cfd35f0c, 0x81ec347ada49b412, 0x64f4745dab3e7da7, 0x1f96aa4f31f50472, - 0x5a2f38b99adfbfb6, 0xf5923ca97bdf41d0, 0x838fb2f2d0b6e43d, 0x40296a14276e444e, - 0x6b01ce971050a62a, 0xd06b41b719ac1f03, 0x4cee947899b0bdb6, 0x965b0e8be5a5fd19, - 0x59fbeb1727c1ed00, 0xf238be92dc76f4db, 0xcab30744c4309ea0, 0xaf3ff93c92e99b3b, - 0xe9c2b719e70ff5f2, 0xf4fc768b4e4aedc5, 0xd4bdff465b183204, 0x32f0e024e904a85a, - 0x61f11634874c3731, 0x9f67d1f1ac4a373b, 0xbc63dcc5bc4974d0, 0xf3a188f98fb9d6a1, - 0xc37a586d80957516, 0x4af287b01af49edb, 0xca8ed96fd8baca28, 0xb0f4693e439d1b8c, - 0xb51fc007d4a45a8e, 0x9b2e8975f5e9f99a, 0xf3adbec92122befc, 0x260fc818cb066d66, - 0x33ec0bbc45b52bbf, 0x0b1fbcf571f84d60, 0x2c9a2ad826f19e91, 0x3b7cba9c4ffa9d3b, - 0xc069b968b5e7cdb2, 0xdc22b78bfc0ade6b, 0xa88edef07f8b18ff, 0xdb20bf85ef5af6dc, - 0x0e5ffa0430481f9a, 0x83b142bf68851cfc, 0x37a908d5aae14d6a, 0xbce64e08c6758a3a, - 0x55f66acf41cfab4e, 0x934fea271b6318b6, 0xb23dcb5220559c2c, 0xc31e5fadef26d1f3, - 0x3c627f4f27941e45, 0xed7a79376878aa56, 0x8f30d91a179c8d27, 0x9cb962cb41226742, - 0x7de88ab9d0af50db, 0xe467a9f8194beff1, 0x99ab87d9d5d90192, 0xf619a6c6fd8509d8, - 0xb9a824dda1dfde21, 0xc49d1a0412ba1bc4, 0xd203d148c34aa6e3, 0xff9d182579b13f32, - 0x5ed100d5391e7d4d, 0xb9c212f8e10fa391, 0xde97d1911a2797c8, 0xb1a878569925b652, - 0x8941e5b050857780, 0x940337c0d178a932, 0xd108df1c1b672142, 0xdb7a25ded63b9be8, - 0xdd3043fb3653ab3f, 0x2cae48504121a428, 0x13ad2085c93f7653, 0xbc53e55f4df9e097, - 0x64d2b22c9ad47305, 0xb0a1a41e3b69e03a, 0x19b32ee3435666c4, 0x3ab07ed5ef6df9ba, - 0x13071cc89c19ef25, 0xbb5cd8ebb929e7f6, 0x28fc1e99fe1c8e3d, 0x78e54ffe69d2e4c3, - 0x2f80728f2e79ea4c, 0x5ab72c0a0790cf26, 0x378cc8d4639310a5, 0x7c379d55fa0eccfd, - 0x68905d438e24f6e9, 0x66eb0750218afca1, 0x312b2c940bdc3e0c, 0x92b7f184fe4d72ca, - 0xa33d497406d3a24b, 0x2dd49616dbb9db9d, 0xe053e70838a7e1ee, 0xe5392c47905cf46d, - 0xee166b9887564c3e, 0x466f388c7d8312ae, 0x82d5377a630f752b, 0xb70b515c0fcb1539, - 0x980583cad70240e6, 0x7a64b37e4457058e, 0x7663eba3961139d3, 0xfd2870e8b9ed3ce0, - 0x4fa4ec6edd9fd91f, 0xd045d710002a8a7f, 0xfa98f3323bd5961b, 0x3bbfb08d87a0e913, - 0x8bbe6ae967c4c642, 0x59a6fa8957947732, 0x978e06d0c6dd8016, 0xe9d31815b99b415d, - 0x9bac1528abcc29f4, 0x0c4d60fe3f19249b, 0x9047cd8a3e9a2d55, 0x3f727ff464c48848, - 0xf8c49b4f602f6d57, 0xf0e15e69dcb2129e, 0xc6d9e531555f5efe, 0x0e153b72d0b17d0f, - 0x8c8200491e0f7dc6, 0x0295a1f4ffbee5e6, 0x4c4c8a208a92dac9, 0x3d29a356f64f411c, - 0xbd5200c4329b68e3, 0xc385103c6ee1bb4e, 0xc90cdb0e9722d2ab, 0x37f96bd3ff544360, - 0xa2b7f87f66420330, 0x282c650f7412776b, 0xd6be6343553f5d07, 0xfbfa37ecab3ba920, - 0x9093c38e022e421a, 0xd00daba995ba9af3, 0x786184e46395932c, 0x8b1a3f0d096a55c4, - 0xe08097ba3f0d8347, 0xa91708d75d825097, 0x33b9922f3a9e3324, 0x7a1a3f0f5c1bfb57, - 0x2de4f194498f249f, 0x0aa491bf06febdda, 0x344765cba30629cb, 0x122b8c12911841f1, - 0x2e2bc1f1458487b4, 0x3160bee6f0ff42f3, 0xece0a30498e1227e, 0xddc05d6dcaa8e697, - 0x34021b369ab3137c, 0xf0ddfc469217ba08, 0x95c458c7b6edb68a, 0x17d03378dc847aa0, - 0x6acb679a11730612, 0xcb93255fd17b3e92, 0x18ad048630c8cdb1, 0x9109b30294d63edc, - 0x3ea11ddf29dd321a, 0xb4686da7ff1ff3bd, 0xbbffc672373ab8b6, 0x2999678aabbc815f, - 0x4d55d6261ee4b32d, 0xffd5fe3b3964bedf, 0x187127a056f937a7, 0x7449351c646463ca, - 0x79e8f46d3a8c691e, 0x6d01e77310af77bc, 0xfd1b310004bcc2e1, 0xb959adcad4ad4a01, - 0xaa172b8db4b99cc2, 0xe8571877e9eaafd0, 0x8bd9e92cc2a0db60, 0x1ae02ea738a7c189, - 0xd13b3cdb32fbea18, 0x31a3795d1b2c5f56, 0x07fcf623e164b459, 0xe5739ffba1887878, - 0x874e2d02a88dd9a7, 0x70e7ffb137b23051, 0x5dd6afeedc75ac46, 0x6d00cbd68f2eb6fe, - 0x6fa0eba48d3df55f, 0x3f796f994c555322, 0xe09f5dd186f1b7d2, 0x669e7e57276ac37b, - 0x0e48ece4da9a5b6b, 0xe560f7d6c8e97e82, 0x9d1893c6331697ed, 0x662775fb14e113d7, - 0xece68a44f6fefab9, 0xf4886037b16ee522, 0x6188769dfbd7dda3, 0xc6c046a54f8a8da0, - 0x5f70a8d4aa1e7ccd, 0xe28aecc280084258, 0xbc8a090cffe8f3ad, 0x3b1ee1bc42c0b788, - 0x0638bddfe099c2b5, 0x7ae78bd275e17ae7, 0xdd9210fce6b2f693, 0xca683d0d4e4ad7e2, - 0x455436b77b6706d7, 0xc0ef0bc07a643a0a, 0xeda596b5e5ae00d6, 0xf5598394e8844916, - 0xd9892fecd00afdc6, 0x2079aadacab2eafb, 0x9811ed2bba446388, 0x4e7af7ffaa0c051d, - 0x995053ec23883ad3, 0x349d014674a49f59, 0x3a0d11f9bcc2381f, 0xce50a144106a31f8, - 0xbf5297e11f98e7ae, 0x8cf32c1f952d0188, 0x845de3bd9d249084, 0xb70a5be01bd770ad, - 0xdef0a6e37f1e961f, 0x796b2dacf8cab916, 0x20bd7f9b4b2675d8, 0xd679f4f80bcce924, - 0xb68d90fb6f77a420, 0xdd2eba4c15cd4e8e, 0x68397d36b33b6d37, 0x9d7760f1261b3b74, - 0x117326c6efc03171, 0x7e4976f063ad90ed, 0x77233ca22983fffc, 0xf7fa08ad152dd606, - 0x903921564f4bc56c, 0x8e1f78ba3d557725, 0x6dbe605a2e751171, 0x8da329f666233c1c, - 0x3751a6b65d1021ab, 0x451a21fd52b66e07, 0x84a6cc4d7f35cfaa, 0xaf824cb94484b447, - 0x8b3cabd3601b93be, 0xdc769724be087150, 0xbf56e590ac606aff, 0x8eb0d5630b034d03, - 0x789d435fdc82ee3d, 0x4758325cac904073, 0xaaa669ad894db943, 0x2f3f918688079296, - 0x3deedb53eea04128, 0xf83034c163b06ecb, 0x38ad6e241bf5f53a, 0x2013b91d9835953d, - 0xfcbbd5befbe29a46, 0x305bc4a171141fda, 0xda70fa95095a4904, 0xe1ac43a2ac6a7df8, - 0x51a515201b6e5b88, 0x9a33d82ee03cd5a2, 0x96706534a6930672, 0x11cc14d0e99866d3, - 0xa1a4240dc2cd9cef, 0xf18d96b1e5aaaddc, 0xeacb9e625cdf22a1, 0xaf7d3de65394493a, - 0x1fcf6e0eb5ddba2d, 0x267689828d28e0b3, 0x9572fb35fe171b19, 0x9b253289d31f6387, - 0x6da6cece06df3934, 0xa581b5f8e75ac10b, 0xe90280eadeb3dc98, 0xa45b2bac115ccb72, - 0x9f15e2acb5aa1cf9, 0xfac1b688ac7c60eb, 0xea86b3ef6c59fe13, 0x4fc9804024bd4e59, - 0x1e5c887ac60a5f9a, 0x928b47141320d79e, 0x653b583525ecb311, 0x8f9fce910a219c0e, - 0x4b244987b1c2a8e6, 0xb242c1cf862ab8ea, 0x62e50f4d928de98d, 0x56f08b01ccfbcdc7, - 0x4df8251ad896053a, 0x18627f5b9a4ddd9a, 0x98658586c866ff8e, 0xdf0634ff5e617695, - 0xe75310b02045a095, 0x5cd458e9b9289fe8, 0x5eb35601eb21d130, 0x0986235b2d2d615c, - 0x1c0eb3e6e7fa4dd2, 0xb5676d2ac98ce775, 0x828713c48fdb7a9a, 0x1e6ba5ea46d8b827, - 0x55f45fb565475bf6, 0x1deff33cd0263fb5, 0x6dc23c61ccd9e86d, 0xcba38e84cdffcc40, - 0x1c5c660b914577a6, 0xc28b29d4c8bb48d9, 0x2e017840ee7a1e06, 0x3cba444b69d454a1, - 0xbbd1962356935010, 0x7d39e14f157354fa, 0x4e80e72a07c2b225, 0x044b668a4b369079, - 0x39028d5270ae290c, 0x6e23c4d118fda9d4, 0x7c0dab87cb651b2e, 0xa3e648fc26c64056, - 0xf17351d4d8f7c465, 0xf14cad35f41263ab, 0x2fc7c4d2bca7dc72, 0x2b45432bd7fe7508, - 0xfa7191ed19e004a3, 0x6b786edcc5f6cc55, 0xdbf8236727dee198, 0x3c003e29b549421b, - 0x5876590c0288fabc, 0x166c62ebff9cf93b, 0xd89c54c1c45546a6, 0x1b54ea4e9b13cdd5, - 0x7fa5fed0b2c5315f, 0x6e45b9c3cee19f81, 0x871646f892eba17d, 0xcea5dcf84572730d, - 0xe7bb4eb75d62b166, 0x1ce7e0d6bc7b6f53, 0xa51c772393f4b21b, 0x590fc72de380397b, - 0x78425d4df0b678a0, 0x091b4cb7a7f25869, 0x022b2f2b6b6a04b3, 0x5f1d985860a5cd44, - 0xc2c3be8e9d18d5d8, 0x0336a189de3984c5, 0x0a8b8d6ae2d5f123, 0x616695f353b97305, - 0x0aa4b1ccc099dad2, 0x4dea8d3f2072fbc9, 0x94fedf3f135469be, 0x25a36f9d0a5c2eda, - 0x884ca2d2a2a50392, 0x74d666f96f176f5d, 0x7067b3b8d1467bc8, 0x57f98089136eeb2e, - 0x0c6854b005740632, 0x0e1730262ebba61b, 0xa21875113dbb8880, 0x951f878921f2699c, - 0xc8e592d8c3bc5762, 0xba04bd4429480419, 0xcfba328c26697ed3, 0xaabf3e38640a4c08, - 0x001593cad60fc6df, 0xbe30bdb572a8813f, 0xbce03eaf1e0c9a81, 0x3f7e61956a620bcc, - 0xd0e6c5a7dad27b50, 0x4bb9568a0529457b, 0xc2fe357ef1e9b3d6, 0x31201f101965aa07, - 0x283604999b54350c, 0xec96fe04ea1f853b, 0x8f1f46a165be71ea, 0x7046c9f9ca7ea047, - 0xa6281e87e93e4cc9, 0x52822bed8f3073de, 0x36c38b3806af71de, 0xa5caba5e4b86e58d, - 0x3c33e98f05ca8aef, 0x95608764928c8d9f, 0x3c48bed4c26f2e47, 0x04c4fe49574bfa1b, - 0xc32faf41087ae788, 0x19c1edf3039ebc39, 0x70d72a99938996f8, 0xe445153545a3b83b, - 0xc752513d9e43eecb, 0xd9c20c0c3bcb8320, 0x9b4dcd562ec821e6, 0xbe57ab3d38708c52, - 0x26bea35483ce18a7, 0x0112feccf72fc168, 0x181f79bbea32cdd3, 0x2bd0d7e80280e860, - 0x7371381b7a503808, 0x1f3d064d0db916b2, 0xa8f6d5c3d861721c, 0x3d8f93423749742a, - 0x716ba0fe4341dc13, 0xe90680d07c8ad686, 0x78bd50d600e1ce40, 0x10b7ce5ec57dc74a, - 0x0dc4bd35a1c7fe3f, 0x54f3e5eeca67c171, 0x8671fdf51d402eaf, 0x3a1bcc17a64164ce, - 0x7285f42935c3a7b4, 0x4be8224229e86812, 0xe8ef906c58386c86, 0xaff6758ad9bbdd4d, - 0x61acaf7da7aa2534, 0x9e3f8f907ae528c9, 0xd8e962e31882a959, 0x3d14c68f0267fa2f, - 0xb2b0cce7628bba28, 0x21466664136be05e, 0xf17466b6187ef3b9, 0x6335ce078b14e4bf, - 0x300341488d42c504, 0xf726f17644755c18, 0x34fff8bb2f1dd7e3, 0xf70e518ca981bfe4, - 0xe20a0243fc734350, 0x2b0289d8d43beacf, 0x66d8dd2b0dd995fd, 0x5df8f8e604d7c0f2, - 0x65560ccccaec3fd9, 0xce8322b081aca0e0, 0xec803b146935ed00, 0x250ab8392aa2dde1, - 0x19551b065b977891, 0xaa6beb91bdc6ea0e, 0x1589bb3fddc82063, 0x454c4796188a746a, - 0x6a0f5b67cb792388, 0xd1eb558953359843, 0xd9f7e1764104646d, 0x1cf5feec652c89a0, - 0xe13e08fcf2ebe0d8, 0x42b9a3cefb087af2, 0xa76674dbeaf4cde5, 0xea207abb1be206e4, - 0xa39b429cb9275f9f, 0xf99bb8a915d7adb9, 0x93d1f942e5e37558, 0xe2ebf6e5f5dbc828, - 0xe6eddb4fb64440c1, 0xa0025751c59a1a21, 0x28c3db6665789e98, 0x0b1c2f06496be89d, - 0x3dab34bf51a01a86, 0x8a10df46dabae133, 0x05308a01daf3d037, 0x64135af9e1bb3061, - 0x566f40f1d16c43e8, 0x88b46276a8b83583, 0x3d05a86da3147eaa, 0x15aa65ceffbc11fb, - 0xaefbe23d3312fc7d, 0xeeb1722be82733fd, 0xbe8cddce70f62f8a, 0xbfa729bb3904ab52, - 0x2d7ca46381ee9a0c, 0x300101afb2cfdf9f, 0x262cac81c986a170, 0xfd4ffc22f5e3933e, - 0xb5b3afab9708e332, 0x0b483445de1a68e0, 0xf045e1b3f78406c2, 0x32a75ca09d0b0069, - 0x034e963ca205f24c, 0x8bc305c6febec1a3, 0x9148545792f0d5cb, 0xfefa0c531dd5369e, - 0xd84baf606b249988, 0x6c1c447febf61f87, 0xc7f69738c1aa99aa, 0x6054ba988df8f4c1, - 0x9cf4a0d410341b9a, 0x94841063e9d663ca, 0xb016f475abd322e9, 0x6690ec37c43a9575, - 0x06e489c152f8aaa9, 0x8c7ad22984bd9cc8, 0x3eae03a13784dc12, 0x49b49ae5d383c81d, - 0x13e8d32675098c23, 0x8af75c309ee4ccea, 0x54df63b27ece43ec, 0x190879d4d18e5d2a, - 0x20b77a8baed118b6, 0x6cd0cfabce914858, 0x1d1e32bd826aa8c3, 0x1d18ecdaa21050c0, - 0x3ed09f3e6cdd97c9, 0x0ccd68fd4d2b1123, 0x23308ea911e8dd55, 0xf2532ff72988cf98, - 0xe12598ee5990c53b, 0x0322b6f226c1d204, 0x857773c13bdb2989, 0x618f03e69b90eef6, - 0xe9200ef9b3e73894, 0xc37391eb526a690a, 0xe473e9f1812150b5, 0x166f3fd63eebc15e, - 0x657c0049d2c5828a, 0x8523010031224ad9, 0xd7d61cc41a7d1ee2, 0xb521743076a7a8c9, - 0xcb091f4163220254, 0xca446fabe91f13ce, 0xda61af0926fd2880, 0x04ac4c0b8ace7426, - 0xfe99be0bbeac5447, 0x5ad06e7d5913302a, 0x3bec8f975d0f38bb, 0xf2972a26bcb0b54c, - 0xa108ba67a4fe5ebd, 0xce1c775b1473f30f, 0x5bc00377233babb2, 0xb0ffa56927c5f5b7, - 0x04f6e46142943963, 0xe95aa93d94e19c6e, 0x9d7ff9c22aac07e0, 0x29ab2b161ec1d8a3, - 0xc1a9cd9374cda333, 0x0cd55948d6584ee7, 0x4ec6af6e22bfe771, 0xa2baafdfb6905f81, - 0xd23b82ea048fad3a, 0x1e430f240835b550, 0xdade0e927bfbb569, 0xe393d7ff83dcbf21, - 0x210dfa4ef9602816, 0x805ecdd29f174819, 0x0fc4374088238f46, 0xdae177b99644414a, - 0xb6fb95f2187e7d47, 0x90c02722bf69083e, 0x41b0708667b5cc93, 0xee3d23be16a045eb, - 0xccac36957daefa83, 0x5a0f061df5c8f53c, 0x8b62c3ef372f0712, 0xa03818d30300ec7c, - 0xff0ec8c1b22048cc, 0x251eb53fa70f47b0, 0x8c6f30f7d9dbca54, 0x866f8302b8aff07c, - 0x4a5db4e64eae4c08, 0x9bdcfc376ef333f7, 0x1af745e1169693ed, 0x1b48739aa4ce4bd6, - 0x5ccee7fc1cc71026, 0xfaceb988f576297f, 0xfcd26e301e063253, 0xef12ce21347bb7c0, - 0x4f60907b1dfb36c9, 0x878768ad4d8163af, 0xf240b167f3e66574, 0x24f9a3abd7c7866f, - 0x9ff0264564d52bc2, 0x270488044eefd3ba, 0xcf6a0ac81825d701, 0xaf1987d4b3df8ec4, - 0x74a48be1f828f882, 0x21a178526b0994dd, 0x2c20153ff3f04cdd, 0x4dda3fbfda00bfd0, - 0x09981aada46e2a2c, 0x11de4ee48ec7b656, 0x8d8c192482d18375, 0xb97dc7eda3ac7c59, - 0x8cde6ba7ebff1608, 0x65f3213ea5897d1d, 0xa7a28f179428eab4, 0x7770d1677ba15e20, - 0xd1f4711a6c3c9e2a, 0x0138c261f90794cb, 0x59c6d34042a7e8a1, 0x4cfbc206eceefb8e, - 0xd75e76504459477f, 0x034c86bf5c4d504e, 0xfec8cbb3fb2500a3, 0xe844946434d03dee, - 0x0297506176c6d1f3, 0x7dd14462ce609dec, 0xfc7394065e43d0ec, 0x7febf3ac4e23e42e, - 0x453af607e5aae17f, 0xade34aa36a078ff6, 0x941acbba4f911c47, 0xc28889b78588451f, - 0x9377e1db1bf91b16, 0xe2f3bb7f789cfdc8, 0xbf772980b8af27de, 0xb4a1d9051567dff1, - 0xd55fdb0162d136fb, 0x1f98e82e8819f5e3, 0x70b319ed316a9bb4, 0x321cf31a4d2176cb, - 0x6833732053f95986, 0x9a084c20f6b44de7, 0xfa3adddde3386ac9, 0x06de7616856fa41e, - 0x2708d37f188c8596, 0xffb1643a48c1b65f, 0x5aad2e7b65939922, 0xb1a98ca7d2d85685, - 0x6a73786354e89d70, 0x86044ab44ff7221e, 0x3b3098953db9829f, 0x7a8c3f24589c1b3e, - 0xf70e6395e4a7f2a3, 0xda8316c9b072784a, 0xc24074f0b7b045f7, 0xf50af41f2e937a93, - 0xc1be65ac7d72eb43, 0x186ec42d3b2b5722, 0xaaa10fb0ef2a699a, 0x9d14c0c74a7ae15a, - 0xb1facecf4b1a792d, 0x529d1701894e61f7, 0xc19b35e560f47292, 0xe3be0cfcbde066c0, - 0xe87aafdfd1e06ccf, 0xb4446148b1bae589, 0xe29b2c9884cf5631, 0x00d0eb074cb53da1, - 0x6841df19ffcb6068, 0x22ce112106430bf3, 0xceadf1c25704f801, 0x32bd6478c4ce23e8, - 0xfb43dabaa3d95bba, 0xd50b0f9b8a09c850, 0xb2a5ff6982c9b048, 0x3f2790135a4261d9, - 0x4d9cc300084cf7c6, 0x1c390af95a4c7e1c, 0x20c8da89b7075665, 0x2af80c175a18b9a7, - 0x67f23fa177e7e364, 0x69d274fd5e6e5882, 0x5f56657e8d053b3c, 0x5531eea234fb69d6, - 0x88aee4d9068a4136, 0xb7a84698b4b38176, 0xe03d1fce530bf7d6, 0x9b45dc0850f2b71c, - 0x01fe0cdc2646ee31, 0x26b0d87ffc788c97, 0x69758f37d534d20f, 0xe2abd9346930877a, - 0xbdc667eae4856e2a, 0x826cb5eaf3ab98ca, 0xd5c2acf8e71c14e1, 0x2a4c5998097ebe05, - 0xe59a97d1066c1753, 0xddd2d8cdafc8d9d0, 0xc71ea4858b8ca61b, 0x46e481c79a7a17ba, - 0x233bdbf899c25a5e, 0xce0bb0e1da4c671e, 0x26709c98575020ce, 0xb0365ba5e17d4c4f, - 0x0b5cc36635c83f5b, 0x2fa8fecb95417480, 0x5960626c6cf95e40, 0xdf62912e62294f80, - 0xb8d360e2a7f09a10, 0x3a578b537d5a2d2a, 0xcc24ef33709c40b2, 0xd099e49ad3a4230e, - 0x7a30564e0b46ef72, 0xb1e4e1466b8d3765, 0x12fa332644c62144, 0xbcb2ea48dade6474, - 0xf00d03cc5536393e, 0x8079a6566115b8ef, 0xa80994168c4aff17, 0x6c430c589134622f, - 0x4c7157cf8523dc53, 0xb44d6e671c0b902f, 0xe56b2caf6c161919, 0x2e3a569676dc1c35, - 0xeb05d4aad1981845, 0x174842eda9f65dcf, 0xa64fe4de015be262, 0x491084a228f0ec07, - 0x9ff9f0164b29d97f, 0xbad5728a7549398e, 0xaf54ac566fad451e, 0x043677a314670fec, - 0xffa4ab0bdd4aa1f1, 0xc4d6a29d3a1fa0d6, 0x805e17a9218b3233, 0xfab659408574c7c1, - 0x85659ae0b18ff3f5, 0xe34ba066a333e1af, 0x7c9a323091ead048, 0x6c753b6c7efd4df6, - 0x9036b7becf58383c, 0x24c418c33be323ec, 0x59ab6fdd36de6204, 0xe3494ef60ce568f2, - 0x518837b001bccaa0, 0x0ee65f83e25487a4, 0xd7f77b6aa00949ef, 0xb51b73b1eb5e833f, - 0x750667f39a1e6fb7, 0xc21c9d7c088b42cc, 0xd62b58bbc641de29, 0xc3594f2c5d0cecc7, - 0xb07d8e7e49b7efdd, 0xd0b6f84ffe48bf08, 0x3a3cff3e0e28f152, 0xb68815e527566783, - 0xc1ba107095544fd4, 0x75d59e7db0202c27, 0x907af32626db452e, 0x91de8678714439b0, - 0x0ce39c7dfc7d1278, 0x082715e9242543ca, 0x75ac6b872b754459, 0x761cdb6c21c2ba97, - 0x3b33d435cccf0f06, 0xe375a89fb1eeadb1, 0xdb658e982ba46f98, 0x6c5ab34405fb6344, - 0xfbea45a2a57d50fc, 0x0d9eeb4bf4a30331, 0xd89d0939e616743d, 0xf92ed97c1bf6a4d3, - 0x83fc97deee2ee851, 0x91a312561b1e183e, 0xa80f05c9750d9912, 0xb942abef1e34ca47, - 0xa6eb110e8d54c0cd, 0x8cef800ba5610d3d, 0xc0eaa3ed67f01c6f, 0x9c887f4aa6a3dcf2, - 0x2f710f3576bbbeeb, 0xf759954a869f116a, 0xdfa63c1b4e1e3ccb, 0x3ce9bc3ceaf60219, - 0x3361da3cd02c4703, 0x5c162bb1aefa0dde, 0x8cac1538fa977431, 0xdcc411b78ab69604, - 0x6c1402989eccbde4, 0x375f3c04c3eb345b, 0x9b8dfdd18afd9d3b, 0xae3627b3d854a5b3, - 0x835f74aaedda7694, 0xfa1bbc390557d04c, 0xc83336bb6d3c624c, 0x99591577cdb28e88, - 0x73403eb5ed98b2fe, 0xaa677e810ca2bcef, 0x2f6c2070be1e29a9, 0x4e2dc1313a13bf9e, - 0xaea50f3ae2812b26, 0xfd869c82ac686554, 0xc1facec414697082, 0x92becf92fed11d82, - 0x01a2831aca6b09c6, 0xf49b0c23be77b2d5, 0x4041339a88c8309f, 0xcd8b4bbcd00b786b, - 0x372f6704d3bd3113, 0x9d0b86b7860b0ca7, 0x0a8dd3a96584fef2, 0xef68e2b06f0e7b16, - 0x81a148824c3c7d64, 0xfa37e89ecf88da10, 0x970f753620820d68, 0xf82efee32ae3b5cc, - 0xcaf9534d43561ba3, 0xb9677e3c242d44d4, 0x1389f742075bf073, 0x3f54145149231244, - 0xe3845de16d4f6674, 0xc313dad0ac90b250, 0xeec5772fc03ba5f2, 0x93885adf6a5eb894, - 0xbf987f72b8075306, 0x558e54e09c99f222, 0x0ea9d020a89c9e80, 0x98c95714cb166a40, - 0xae48c57fce993f80, 0x9c7ce2f1e41092ea, 0x49dd74dcaf1d1e84, 0x308f2951fab60b13, - 0x08ddf58bcd3b7afe, 0x96f17b434a3aae15, 0xd248e6342b72ac90, 0x00375d359528e1ae, - 0xa484189341648560, 0x4b38db273dfc473d, 0x07b62afb161372ba, 0x373167ecac67d5d6, - 0x01d1b8c2761f97bf, 0xad95e2ce55888ecd, 0x2ddf6ac916ba2317, 0x6edd29b3a1305fd1, - 0x42e8a6fadcc18446, 0x36795f64c0279b08, 0x16c4c1dc0eb6ca60, 0x3425956489ba1915, - 0x771e3aa060ee5284, 0x3bb800832802f496, 0x117b2fc9525fcd61, 0x2db90016a39104f9, - 0x0c882393d84ebc4c, 0xc859f1507e65b55a, 0xda0dae082af012a1, 0x81f6fa8452e539eb, - 0x0d727468916ea4ba, 0x2acb757b28da6c75, 0xb2032ade6af010dc, 0xc3c1148299a78de8, - 0x01e83d7da89a30f5, 0x294d1781b6148442, 0xd774c29530a6090b, 0xe7f147867eff82d2, - 0x32f7bbd79f909d45, 0x4a1e8a34f6c1df4b, 0xf06fbb207934a0f6, 0x2c295f9d742a78ad, - 0x429df3eec2a1c458, 0x270bc19383b97e1e, 0x7b8ec1a1c86ae6c3, 0x774ee107d5534fad, - 0x530d73bee0be4374, 0xc00c963fca7de8bd, 0x9441adb5d69b490d, 0x704ec48321a008a0, - 0xe524a8e26a6f7c70, 0x2dcae156d20f2b9e, 0x9c721e397d96eb55, 0x47aa2f04da2c2ddd, - 0x204e1731fda296a1, 0xcb8b8011bf6a03c9, 0x218ed7bb391d33e2, 0xdd9af39b6d6914b6, - 0x3c43f22fabf8a310, 0xef38ac110da24c60, 0x42d6c03c0e60890b, 0x210f84ed87b741e3, - 0xe08701075822e0dd, 0x0360633341883d67, 0x5450c00ce7773e75, 0x64809a1fe7870d99, - 0xf7e6f72d5b654392, 0x75960432031877b9, 0xf49af206643cd7e6, 0xe80b00709de42d66, - 0x16d42b6b13160b52, 0x538b2c6b362eef8a, 0x8fe774d825f6d9f8, 0xfbbf3b881b576d3b, - 0x5add2ba6b49b57bd, 0xdafea301209a66cb, 0x270d18fed17f869a, 0x871749140672a041, - 0x53d087cf3bac84ab, 0x1151272d50e386e2, 0xcf092d2478b829a1, 0xc07caf08a97b4ea5, - 0xab1ceb2a1502085c, 0x15aac6580207d92d, 0x47101df376cac03a, 0xedd4ab721e1b0e00, - 0x17140fea8a510be8, 0x434b187b25907435, 0xca1d171078c27c2d, 0x0f1e2e9f0114d4ee, - 0xad2cc7a64fbcd67c, 0x026e2181ad02ac77, 0x13fd7c5bb7b0f5d1, 0x99157656e6487547, - 0x328fcb418479b204, 0x9205b0c6116a08ad, 0x7f8150625f72a321, 0xa66c201723a86dba, - 0x3d73a06b8fb2ee3a, 0x72420179013c109d, 0x2ed221e562f7c5bb, 0x1b55c5ca650da670, - 0x58846c4eed2ab062, 0x389358ae7baa1442, 0x25cd2e4f0756abea, 0x7f7168818c7ba33a, - 0x8e7974a218acecf4, 0xcebbe834286cfc7e, 0xee2d53b0b95eced3, 0x918bb86bd0b3e798, - 0x01afe454314b6595, 0x2149ac3ca01ba713, 0x743c601766822e84, 0x163333e6fcc41323, - 0x17a7cd8bb733cbdd, 0xc170679b9f4803b7, 0x924a19f12c0d0abb, 0xc95bcea2ef79317c, - 0x41db6301e69261bd, 0x27c5d21412832ef1, 0x7d18503979f82676, 0x866335ecf57780f5, - 0x4111e5de057bc51c, 0xd3845fabe3c63ff8, 0x33fe9498c9134cbe, 0x4cda52b2225b16bf, - 0xac62f35ca0c95a04, 0x5213bef690c0714f, 0x3ce2999b0fd69a3b, 0x29c54a1d54c92d6b, - 0xc78a127f7304a30a, 0xec9dbc4bd6e4b2ee, 0x08b29fe98bf02b3c, 0x2a6141db6012b62a, - 0xdb51746b5d9aa18e, 0xd7fef9b6ef4488e4, 0x785db050c8535121, 0x9ac9129cd2f90988, - 0x4a848db771482a1f, 0x8985052c78340c09, 0x138f8b646d8b688c, 0xc2bfe55f2c927977, - 0x9a2d7a2a8cce6322, 0x9211547f7674f354, 0x3d06e1dfa2ddf61f, 0x9d6424ec71eccdb2, - 0xf0d9f74996ec5024, 0xbcd7005014d395c8, 0x76180646fb79649c, 0xc1d3e3cc67e26375, - 0xcedc412a875df84b, 0x2b0ff3e22f82493a, 0x749067b4eaf521a4, 0x2ad9305ac7048c3e, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x312d5ca67482d023, 0xb3541bd290bed320, 0xb118be112c2b8c73, 0x39d99bbf92f69b5b, - 0x2b875a20cc7cbe33, 0x7f37974671487294, 0x2994db86931d3726, 0xa9c3cd42271b243c, - 0x2f23ee35933e4717, 0x4e296824a48770bf, 0x05572c01d47807c1, 0xcf0c46b0be471f20, - 0xd2eb1f233d544184, 0x8148c9b90080041c, 0xc24c5dfd0952a7d6, 0x7b751753196de8eb, - 0xfef330427abac550, 0x295c63f50b7b1e2c, 0xbc73dfca83df94e8, 0xb8beff9e52a601e3, - 0x0e5bd64ed5fcb2bf, 0x30e8e4eb68530d2e, 0x52e062ef9f25f33f, 0xc60e0dc0bbec8cce, - 0x3cd4e958b1dd35b9, 0x50f857b5a9bfa0a1, 0xa975946e2fee6d98, 0xb0be744f06251460, - 0x1133241b67590148, 0x199e1fb9720c8330, 0xdec7d5392ead9007, 0x232f9df87b8df8c3, - 0x48c0231165b49477, 0x9e6828214fb27fed, 0x7963d93ccb878817, 0x6b34f08826676cbf, - 0xfe1e91b6b6ac2932, 0xf2a6289a1d7fa6b9, 0x6a4065324b89ebec, 0xab0406335c55a0c1, - 0xf249740ced3ce7df, 0x720a88a072183363, 0x09797a286e1ff82a, 0xc643c3409f2ed036, - 0x077764a906b49b52, 0x0c0be7cc9b0601ac, 0xc72812964856cd7c, 0x8e6d1d1612adeda2, - 0x8b63660122a14ef3, 0xbb77167fad558d04, 0x6c0687e55d6fdc1b, 0x5c8a14b9ada3f720, - 0xa9837a1be7d5cc1c, 0x0e9b9b851e8be000, 0xaedf7f77d7d9d23e, 0xacc1a344f673b59e, - 0xfde9da89b2a8dda8, 0xb3cfc7749cbe9aae, 0x0387aeb3a39c0ec9, 0x9a6582348139af0d, - 0x47f019dbcfc35cce, 0xbf2f59a7554eb293, 0x655028fc0b53037f, 0x234a32a7d7b88db8, - 0xdda8dffceff67270, 0x73d3bb89d19ba899, 0x9c22d9b9321b3e22, 0x71ee62c1d06de362, - 0x7abf0efb7398a474, 0x19eeb9bf87b04adf, 0x041a65022f575411, 0x74fdd01facbaaeeb, - 0xa8d6da78b97ba34b, 0x79cd93316e22cc56, 0xee5fceaf8f7e3f09, 0x8b553def10b23433, - 0xb313a697c58128f3, 0x7e800f682a5151e5, 0x720d4f85cb0ff62a, 0xc5bb0f7cc98bff73, - 0x64d1121886f57cb0, 0x6fe272cd86e760fe, 0xb22a2bad9eaf5da8, 0xe101ee90526ebbda, - 0xbc10a4f1234ac302, 0x366392d87bdabba5, 0xec86facc3094c567, 0xd063f1d3a15ca4e7, - 0x4933e1e79ea25542, 0x3ec6e0d97a07d13b, 0xdde4387dd5d7048b, 0xc872294ceed15170, - 0xcf1fc2b88dc3b938, 0x2b22cd0ae1db9a09, 0xe820acfb9d3aba94, 0xf3ee426e2d717cf8, - 0xdf73b5744e40ddec, 0xa3d78de64f41d27b, 0x0bb7970a153b390b, 0x5a97b73fc02ecc76, - 0xf5e825e01a0c0526, 0x24ca17958d5f9bce, 0x1ad0b1b6786050e6, 0xb61b2c8ed004ea2e, - 0x5201c10d0253c063, 0xe205552ad73f348f, 0x4ea7058709d54aec, 0xd5bfc501c6042c7c, - 0x132114c10fd80719, 0x4c46c0e817fc74d9, 0xfd39f91d25e1988a, 0x70a94aa1af7c6fd0, - 0x27d2d577778fa09a, 0xe082c15573d3efab, 0x5cee67e3c2a98563, 0x4a760d4f600cd6ae, - 0x53953646cb1468a1, 0xfae329631b6808c5, 0x0be60bef66bdbccc, 0x76b5dd1e24265fc5, - 0x06f4b5b644608d09, 0x1df17698e506ea02, 0xb8e581555197280b, 0x528d42bdd74af9cc, - 0x05589f0bcb5b0e7e, 0x245926acc4732aca, 0x56d6ab0b0d751eef, 0x22dc58c3cf902884, - 0x2cb3d99efac0aaf1, 0x68434cc011eae7ae, 0x9e7879f5895b0ea8, 0xfca47e35aee48554, - 0x335cc314abb7dc6e, 0x37fa2b11d8291903, 0x3e3f9927c8b95ac5, 0x7596c87ee268453c, - 0x7d878ae25741b216, 0xd7e55f4f3c661959, 0x2880153a576bd510, 0xad1b0c64e7d15ec0, - 0x0e5a5c3682f367b1, 0x0c6b493240d4b3b7, 0x276c7ce6a843b153, 0xca419aed6aa95b8c, - 0xcbf473f5026f80c3, 0xa6facd79a417d6d1, 0x9821411c083c6c8d, 0xd6596e9d24b090cf, - 0x096398c8e680760e, 0x9df16cc77f77bbe4, 0xec87dfff1acdc1ff, 0xab0ebb43021a405f, - 0xb04767572a005839, 0x59c083ac23f9dec8, 0xb26e1a4d34501409, 0x73a91779ffd862ec, - 0xf2572b5602da1ab2, 0x40b448232a7c712d, 0xe3fd6828723db975, 0x331cd783b9c76951, - 0x551a3141cdd4afa0, 0x2a9d34e1d7512de2, 0xa17846cdd1afe710, 0xe8253c71153a383d, - 0xac111dec72748a73, 0x27eb995c4ab2a4e5, 0x62d4217e32e0ae12, 0x1b48d60b95ea496c, - 0xe34d6a8a13c66bfd, 0xb8c44e3f56fe0d8d, 0xee43cd6dec7ede09, 0x511596794c305347, - 0xc6b73dbe48e65803, 0x8f6f2b3ca1c30c4c, 0x8a2766f7d0f8dc57, 0x3b7d4377c505b5b1, - 0x66a3dfbd5eaa3891, 0x84b86daa5bb1be5b, 0x3cc4a67617231b3d, 0x6ff8e897394e5630, - 0x8905dfeade7b2b10, 0xd13e8b0a1c289a16, 0x5c4a2dd7d91806a0, 0xbb37510165a709a6, - 0xeb3f543405a3033b, 0x3486531fb981cc75, 0xb66b4123bbe935cf, 0x19471235b2e5eba6, - 0x8e3e80b14ba9f053, 0x1eac925150d90d9c, 0xbfbafdbb9a4ae0f0, 0x650da8d532acc039, - 0x3ee9d7b69a84c9c8, 0x6b3caf192a958cbc, 0x5258b72debece466, 0x76d5f13566b25dc7, - 0x7fb4290a43974c04, 0x539966aed4a713c7, 0x354e05f48955e8af, 0xcaa89635d4589b4f, - 0x25a90e771ae50873, 0xfbe5a6dad77b363e, 0x5dfa1ef45e4e89b5, 0x850ded439160bc0d, - 0x9d6bcb57fc7ee5f3, 0x20644f05380f107b, 0xf5530ffe5030f7a2, 0x9c3d15a60f400583, - 0xf453224cca482bac, 0x4938b2160cdd6229, 0x56523b710efcf3b2, 0x92e3ff47735809e1, - 0xf0179956c01b834d, 0x885ddb2f283d5bb8, 0xc142a552a9dc6503, 0x8ae9d8d69c0ce536, - 0xba7424aa61225674, 0xda02fb25ac87c6f9, 0x24fa86dd462f2c17, 0xc5cbe5e6a1c27002, - 0xb58af53e273730b9, 0xbc423e3b5b6b423b, 0xbfd482e6f83af2f5, 0xacf809e5edfdedcf, - 0xd397d4904daec822, 0x38dc276c2f305a8d, 0xe0f7fe00bdf9af89, 0xad78e8c89333a14d, - 0x38814f6e19a181b6, 0x01ab2bdcd6721c0d, 0xa4c2cc705383d9c9, 0x3cb3ee9a0aa9a715, - 0xa9b9b6b83cebd9de, 0x1c00bbb7b09ce4a7, 0x27d86fe0b2604ef4, 0x36f4d80b5772cef3, - 0x01b0c8bb000e2d8b, 0x64ce5731055d7d16, 0x5fa750643a153e82, 0xa12191b8ef5cf369, - 0x2c4cfc42e45d9935, 0x3b94396bc848bd4b, 0x9e802678412db686, 0xaf3c12a7ee39f20a, - 0x42e4fed3523edcb4, 0x686e804f132ce3e6, 0xf056bb9cf46cfb73, 0x4b98f1b7b6e3703e, - 0xe1d64e56bef8cd77, 0xe8d79922fa0520ad, 0x5fb12b935ebda7dd, 0x1a202b9abefe3556, - 0x99b81165d1c26fde, 0x93f61afc9cc0d192, 0x5ba564f367170084, 0x6967b93e4513278f, - 0x68036d55ae7e8a0d, 0x100b1d0a89d5196c, 0x02d8c1b2f3a0221b, 0x90aa07065bf579cb, - 0x50230748abb1a5ab, 0x9360177807d84e25, 0xa40b8b480b0c7cdf, 0xe159a74904d949f5, - 0xaecb0353fc9f6cc2, 0x2e17626483dd67ad, 0x295033d20fb07562, 0xac23bd5bf81202c4, - 0x614adf6993b552cb, 0x372c426d9ba8d695, 0x6817d158447c6db0, 0xd711e490bb0a96c9, - 0x42ac67d5d68e5e60, 0xb577d44358cad9a5, 0xeb9088b8b9cc02aa, 0x4eea848c36d01b59, - 0xc39d05204ec54f0e, 0xe9828ebfdfb3fe5b, 0xc5040c3176538a2f, 0x984bd3217b830911, - 0x06693fa6c3603ef0, 0x259c81076f9edc75, 0xe3c16cf9664d3907, 0xd430ee00b2881173, - 0x8530d7db605e5628, 0x42f47a718fbafc99, 0x50a608c2fbed8a9e, 0x9c26bd531ffe3fd9, - 0xe223dbae34e1aea5, 0x5b0b0240280b06b6, 0xd76b9c1ab6331dbb, 0x8976aec739f8bde6, - 0x3b4e1cbdaca0dea9, 0xc08686f40577c408, 0x5ceff49262c3b5ab, 0x8f47c0c2afc9c587, - 0x15289d6b6d62851a, 0x12c6c9dd51627dc8, 0x0b2ebc403d3df551, 0x5aeeab941ef98d4d, - 0xa9f12659a3fe86ab, 0xb6d2b1b9dd255757, 0xb8578a8a29bc39cd, 0x180c6edbe5928616, - 0x05d33444576686c1, 0x2b4c641b4c1e5d89, 0x4e472b011443636e, 0xdc8d4d3357d55249, - 0x127d9600ab065506, 0x7b37bd7b00b6ac5e, 0x78429bffbc743113, 0x3bf46cefb046253e, - 0x9f6984a37b1a8c78, 0xf779d4014a269df3, 0x1e9ad59b99e2b588, 0x0f4ed05c88e7db9a, - 0x8ba3d1d87e794bf1, 0x364018d90031fb40, 0x8331e251cec518b8, 0x4cc30fb8af6a860a, - 0x38382cf1439dbdd7, 0xac04470c04156e11, 0x59f5267affe1403a, 0x5ef24c3e6a8f4170, - 0xdc39a477ef1e3bf0, 0x3fc37d5901bbf073, 0xc427ee1b31d1c02e, 0x4be6b1c312dc89d0, - 0x221fa07be5d8593c, 0x0d4fc8689de97adb, 0x2e17f7d9e3d12cb7, 0xc594ef6d43419f2b, - 0x9116abee38b50c82, 0x51c52fd7fd28b49f, 0xf73f86d0b9552dd9, 0x834feb44ac603498, - 0xa9839fb500f2224f, 0xa24c74e68f5da2b4, 0xc639d19b51e54bb3, 0xe953140e20febd80, - 0x9ff0572a3e88ba86, 0xda4dce319a864fe0, 0x322019c432c6b022, 0x216e04657bf3881a, - 0xd1bdc3697605a32c, 0x6239f065f8a63dd1, 0xe246d650120a72df, 0xeee591b993c7d9e5, - 0xb47a497972bfc831, 0x3788a416a58d5f36, 0x48b2d77e38c2959e, 0x9a2d22f84c6a0dfa, - 0x5bb259df6715569e, 0x2482ebd7239fa03c, 0x7158dc7fd1b04659, 0xb541c0b84e8080e7, - 0xbf1938ff28cf7df3, 0x1d1d5623521871a4, 0x83880b816f22c83c, 0xf2d9cd23933ca820, - 0x02f1bc9202e79f3e, 0xce0b642537878387, 0x6c0eea5c09641d91, 0x359118f2bfba7f36, - 0xd37769614e8bb12b, 0x5661db0649d7d428, 0xba5be9c74f4d9e82, 0xbcb40c8a196f906e, - 0x811c4dacede7cc8c, 0xca2bc0c4eeda9ea5, 0x7b47b3a5b7d12a09, 0x1d38d56d187b5317, - 0xecda6910b28a65ba, 0x42d43b004c8c2f9b, 0xbc561857d386e072, 0x7b6025a93f711eb9, - 0x7edfaf73d1ebf423, 0x493db7c3c21c8ae9, 0xb6cc327fc8fca50c, 0x100d33c71621cb00, - 0xb14e75a7f402bd3e, 0x7f0fd112998e6aa7, 0xfb971b7df0cf0305, 0xcce6fad26a7146b4, - 0xf1b70b605d8e0dc5, 0x58522922a86d9caa, 0x073694e0a6a9a7b8, 0xb2ab94eb5da08701, - 0x15e1b14edc5af741, 0xc18ac460656789ed, 0x681febbc0fcfe064, 0xf718e44338801be7, - 0xe549788c00fb2a14, 0x4ed07586d4719b61, 0x7ce4c1ddf2f6f93b, 0x90d0647fc59e79e0, - 0x75a8d76135a560ff, 0x0576cfeccb2b49dc, 0x6b964bcdf9ee9405, 0x93c8100934fc08e8, - 0x4652fccc60771c39, 0x6e8702f63db7dd1e, 0x2ebbbff348f4f4a8, 0x03ed452f6ce04e55, - 0x9c3dfd4f224f3a9f, 0x3dccb7e0d3b15369, 0x0a2382ee842d4fdc, 0x58676a605f19f272, - 0x80bde8f51637b6e3, 0x9a5136aeb60225a9, 0xcae720b31404842a, 0x2a383cd7509a4082, - 0x07e6194d1a5ffbb6, 0xfee25c0add70b04a, 0xfca6a8da7c0f64a9, 0xda29eede78355989, - 0x10217f2ee22c41bc, 0x4283d723d7e324d9, 0x96b9b0ec5c12add0, 0x3291b6a6ee79a9ac, - 0x535965340cad2fe8, 0xdf08ca2a04135008, 0xf911a3e5180aedd1, 0x74651696b0aa0bbf, - 0xb6cd9282a816c4c9, 0x236a66ab958e3bed, 0xd6a0351a7d05381f, 0xaa2825ee5376f12e, - 0x3385bed07c35853c, 0xf28d9a73f1a15641, 0x2821b03f0dd7586d, 0xe2662af03157ec1d, - 0x8684925475701d69, 0x1a3c3805c3f55e69, 0x3b797b69a2e5e1e5, 0x8fb83d2d42854f7a, - 0x7e85bc3984776a04, 0x9e8535095d074d8e, 0x80438376394bd629, 0x54b70ac619bea9b1, - 0x0707cf21da936b10, 0x89346d57c574d801, 0x7920522e806efb41, 0x6226cc0bd1cc0adf, - 0x21de5073b66ca898, 0x26fd4ccad6e09c04, 0xf2938d59c1cff5ee, 0x3281aa2e788372eb, - 0xb58cd1e3555780df, 0x4985704534857b2c, 0x5da2e64a0061ae75, 0xad9e55b120b3cad2, - 0x59d7c7558a7c0696, 0xa7a250a8b7614398, 0x53e79b60d1996d68, 0xacc82675c77c0360, - 0x9246f8f0d0c21a7a, 0x3b013de96fba5dd6, 0xa4625852462f3c8b, 0x25d407f2854bb16c, - 0x786a22afe48ed932, 0xc2706706d42acd4c, 0x83b13f0cb9957f94, 0x44687b76d0a2c892, - 0x5e8789526efb1a4c, 0x044a0e28299104f2, 0xac5a07253e1599d4, 0xe110dec16f578863, - 0xcd8ad0c903c7fcd0, 0xf737e0335cf99a0c, 0xf7fa1221f164ba0d, 0xbba04ed1b05ed6c9, - 0x783fb7b111149ac7, 0x7d33ff4e090274bf, 0x6388a9746d0050ad, 0xdebc58d7aa14b90d, - 0x0d7f966584b03c9c, 0xef4fa3e1d79b8ea1, 0x62f6b13e45f07436, 0x23d2ce3ee1fee870, - 0x54a6b70450cbcc2a, 0x5a938afef070cae6, 0x4e5d255a50bbe052, 0x52fda18c50742990, - 0x041fd2f6549ada1c, 0x06f6ab85ffd028c5, 0x2517bbdae2b5f33a, 0x153165fbe4f0b6fb, - 0x57b86ae658ca9642, 0x95e59a04bc7fc8f8, 0xfc660809c7f659df, 0xa800b6032603c310, - 0xf09dae04bb02ed1c, 0xd75638af0f171fe5, 0x6c5b388a682d37b9, 0x8b4f44fcd2a2f2a4, - 0x9e4a66ba8cf8f0a8, 0x484ba7bb18b8ca05, 0x317ca915b10e9b9b, 0x0396a9e20c5dfdf8, - 0x076d711d47039bea, 0xbe11a029a8e26cb1, 0xd83adb46bd0f306e, 0x5fa5859d91c4b6ec, - 0xd495c32969586b50, 0xf9dab3b4efd03652, 0xdbf1411db43aef64, 0x053ad2d1fbdbac7e, - 0xcb53b5a673d59299, 0xf16dabf60a5f1620, 0xd599f3a7ac81fe23, 0x084c55c0b13dbb73, - 0x8a37e041c33ff108, 0x9a20507f229901f0, 0x952381708eaaab19, 0xce3e60a911adea9f, - 0xb24d0ea225757a17, 0x7df9c1c3dccee478, 0xa95ce32559f1242a, 0x16a8ca55c0c10c8b, - 0x2a84a5eff3d3aae0, 0x47b1e4e3a6757dae, 0x46276db095d9b343, 0x6ddaaced07e1189d, - 0xc5dc936d36d1ab46, 0x45afba7dfa3da5a7, 0x1ac15182f8f377e2, 0xda59cefb982f5649, - 0xead873adb9cc9d9c, 0xadc1daa6e9526e55, 0xb3eb9982870e6554, 0xa49ecac14ee4bd84, - 0x4d3ead0444662db3, 0x9fa3f51b9cab5713, 0xf1c5ef0d15b5de33, 0xcd1fc13dd78587a6, - 0x5161a0675a9a6edd, 0x7425eb83791c302e, 0x5928da7ada207b7e, 0x6375c8cb9117e707, - 0x940777b1918e6278, 0x7b63f1ef16921659, 0xcf81a9fac69ad6bd, 0xf62087ceb228ce05, - 0xe8d461ba645d55bf, 0xf1c0645c0b541e64, 0x448661994470a165, 0x5de8c01ec2339f21, - 0x78f5d4cc0d9fd908, 0x344cfddad86596f0, 0x87a6ccbe6e597cc4, 0xda79d1a3ab947672, - 0xac1695531ce42d16, 0xaf3ea33b33ed8962, 0x19fc70e1e0d51b56, 0xe275d8dce122ea88, - 0x00c3a18447f6d95b, 0x4918a35167cc533d, 0xc3dd9adfa99c3176, 0x221dcdc52b3d97e0, - 0x3cf7717505485b94, 0xd0b8512cc428e3e5, 0xecafb64ad72469fe, 0xb0ae441963f22554, - 0x107b2ada1915dd7d, 0xbb1058c51d414732, 0x544f2eb9dc624d44, 0xcd55b8020140729a, - 0xd1e8eec02645e66f, 0x9993bd6edd498aef, 0x51059c504807f249, 0xc86d8a3b265aef26, - 0x42bba851517da888, 0xde136def587cf15d, 0xc43449a9e521ebf1, 0xe78c37548b69f458, - 0x1b5505dcb9359706, 0xc132fae79b87596c, 0x66f80ed0f55402d6, 0x990718c8d9b3c3be, - 0xbf4428183b8010a2, 0xec25bf3cc9b033f5, 0xae9baf8a00c13de6, 0x1a86a3fee63d859d, - 0x5f0879d562731087, 0x37e769bdfe25689f, 0x841ec71ddf08daa5, 0x2546e68dbd97ad35, - 0x2ad8cbaced7f6582, 0x55a7017295a07b9b, 0x3811a0bd7fed8668, 0xf14f17b39ee0d520, - 0x30f0a1bd21a950d9, 0x6325a89610d3c33f, 0x3745c78ffe9848dd, 0xcff4aae2c1abfd7a, - 0x24f0779ec0819e25, 0xdde346bcdbd36f17, 0xf0b353d616c9e82c, 0xc43758e13575c0a6, - 0x5c40b054e444ebfd, 0x0ec7a32a3607ae9f, 0x159f1ccdd2ffc5a4, 0xae4fe26a88d957a4, - 0xabbb9a91fd91410e, 0x00a7acdd900d041b, 0x3348fc7ad8d80ff9, 0xcdeefae7f52a08fa, - 0x3c4785569dd936c4, 0x03a7e830f692d0f0, 0x210d33973cc4f5bb, 0x3bf18cc6802f0779, - 0x1085ab5904cc999a, 0x0284a616492747ca, 0x120d948423ed5e6b, 0x2c20131dbc5b7d9e, - 0x0d87b9edc35c9f4b, 0x9ec60dc492d06627, 0x08504d228a86e239, 0x8d3c010ad7f0ef8c, - 0x8564d34dac0c4f5d, 0x176024b99371f730, 0xcbd16f32d8d1ee4e, 0x92f6a6dce381b78b, - 0x2994445875ca9fdd, 0xef229c3bbb957d90, 0x6c2fefee97be83bb, 0x94cbf732af8ec02c, - 0xe76b971d703eb96b, 0x028e42dbf878b9af, 0xee8ff81409b8a3f7, 0xcaa7c608474c5eec, - 0xe0b25f40f8a95229, 0x614d19e395026404, 0xec773dc21daedc04, 0xba48556baae16388, - 0xb0ce57fe2cb3ae70, 0x0c8cd6c46311b484, 0xf025be0fa84e32b2, 0x414956b10d5e09ef, - 0xf82a175185950ed5, 0x79c5cc3172524a1b, 0xec6b599ca612d0ca, 0xde6357473c0de2f6, - 0xf9beebbbbe2d3fdf, 0x64aa78062ab46940, 0x5c33be60d91f9967, 0xb845f47ddb2316bf, - 0xb5217da519e8873a, 0xcfcdf527beb7eca6, 0x8aea0d207833902c, 0x6b9433ca3d4846f4, - 0x9327d9b9ec453ae3, 0x15f331994a71120e, 0x642f32d572770513, 0x0203ce21279a2e0b, - 0xad60b0e8013517b9, 0xb2a87adfe0c45cfb, 0x40756072bf13f93b, 0x6e450f85f2c49750, - 0x2b9ffa2249d4f7b4, 0xc21e70ee6cfe7cdc, 0xc4cdf0e71771da11, 0xde53c5b5f2f1d0d5, - 0xec95592ba613c23a, 0x0a3a93340fd40fd2, 0x03e043c5965dad7e, 0x520f3dc6b0de8694, - 0x7854f2f16b671c87, 0xa3a56d160ed7a7c4, 0x3cc2bee4e4253063, 0x05e13665813d145a, - 0x5cbc21eb53c63872, 0xd1996590eb467b56, 0x667d874b931576b8, 0x1984f1d7dc4d3c12, - 0x80437fd648b16226, 0x516e2eabf31072a9, 0x18951837e2ececd3, 0x3ed1d4a7655326ec, - 0xa5ffe40a6eeea354, 0x5278c3ef75049337, 0xbfdd81c8724948a9, 0x7344b3e9d3fa3840, - 0x332dd7b2a4a06034, 0xbf17ba5e598b60b2, 0x3ed2fba7555b396b, 0x33cfcfc6aa117e45, - 0xef626c2d01ee8ee8, 0x68b83a99d704fff1, 0x14f6a8d183259d84, 0xb0649083cfee42ba, - 0x807e1e81bf7d8906, 0x08ecd1a92907c8a8, 0x30d86f55f18f616c, 0xe5ae9f6604194f19, - 0x2c914dfefed7e32d, 0x0fa9d846863debaf, 0x39d365fefa1b8b66, 0xa6914e2c30fb0836, - 0x209587c1cf285735, 0xbbee81efa7db82f0, 0x69a929a9e03a27b8, 0xe241c119084692c8, - 0x1574d62a36ffa72b, 0xef7a234af30365b5, 0x6e59903b6770870f, 0x574f6a8d9cd025bf, - 0xac12d512d7d9ff83, 0x006226cb3dffb9e8, 0xd10c2e70ed6ca633, 0xaa5611e494a62bb1, - 0x9492e5a3bb91b4a3, 0xa44a1ebd6db1e7d1, 0x6c5b301b349f2e70, 0x86f528c119ff640e, - 0x9379d7e604a6ad12, 0x938cc67f226490f8, 0x01584f8b9159f464, 0x7ab1109eca29cbad, - 0xd1706cb4e50f5b30, 0xa988fcd75bf08f81, 0x80e21af3fcd319b6, 0xdf3c99cb38ae85a6, - 0x106e46ea26b2af54, 0xd25ee32804558f2b, 0xf704bb889cc52e2a, 0x3491b586132b43e3, - 0xe77d908cbc7c89b8, 0x84af26310189048e, 0x9a7165cb6dc883cb, 0x32ac4686c2ac8c6b, - 0x5892839049867c93, 0x185c641dd0102644, 0x80eb59bff6a6c331, 0x0b3c36cf208300d4, - 0xaba0e097a84ecf7b, 0x923f67b087a3c7fe, 0x5995275d3f9dd738, 0x6645c750f4e9f133, - 0xa6a1153815a5dd5f, 0x5d0f9d846a22ca90, 0xfa2bf820de16d38c, 0xdbbd9197ba5309ba, - 0xdfcc51025a2d276a, 0x530d57ac25fbba69, 0xaec92d21327fe30d, 0xbb26b8f6104c6cda, - 0x7b84db2f4d11080f, 0x91883dfb2b54f6ce, 0xbcfa679eb41e1283, 0x7d28333d0034eda4, - 0x11a11b199e55f7cf, 0x970b1c214263e484, 0x39429b421720e3fb, 0xe5d96148e49ee1f3, - 0xd93793be95952616, 0x4d46320649ca0b61, 0x6c6584154d389519, 0xf08b76215515c640, - 0xd64c6f09daf94797, 0x569a38a30dbc9e84, 0x25f46e5d05c5cc83, 0xdbdca940dff6f7d7, - 0x1b8b9f31a6ab1bff, 0xd1d20541294d79bb, 0x0f7da73684c7670b, 0x060cf94a531c6950, - 0x0eb236e888ff6736, 0xc1fcd2670521444b, 0x9beffbb82a6c0633, 0x942e4a635209a344, - 0xb41801e938d0e3d0, 0xbba1b57b7713ca01, 0x25e4b7f3254dbaa3, 0xfc6c59cf5aa6c333, - 0x3127ea1f0afab63c, 0x871d4110ec405664, 0x4569d1bfa03bb9f3, 0x684e0a74d3e0f549, - 0x4608b001ed038410, 0xb2e439be2cd9999c, 0x2337a130dfca21fa, 0x574693f758abd7f6, - 0x8c2d5088df328d29, 0x572f502e601ae05d, 0xea311cd257f8b927, 0x9a2077d5c467c672, - 0x0f1b6f7d49fc02c1, 0x2f0b65736d65933b, 0xbc4996a7cfb754aa, 0x62b82199852c60bd, - 0x9c83f741ae6d713e, 0xfe1cbb2ae64926ac, 0x33711ae6b37a7d12, 0x8c393abee1eed3ad, - 0x0aca6314bef5c746, 0x08ddd91c7eaec4ee, 0x9e6e7abfef89d093, 0x2819563e93a2e4a2, - 0x4cb6022e20ce8e94, 0x1f68f6118b452015, 0x6fe26e1acff61b26, 0x2f00da434955a6ac, - 0x712996f9cd092af8, 0xe240e77a22c5fe28, 0x1fdffb13efd96691, 0x327cb474fc6f86d6, - 0xd4c5730b252f3a33, 0xb317defe5f317a0e, 0x32e25b790a02614f, 0x6a15f06d83a0a1fb, - 0x370b50ff7b77544d, 0x1bf51ea13caaf4f4, 0xa0af8a70e7766e63, 0x0eb0600db8074253, - 0x06281d0a011a79ec, 0x377343894b28db90, 0xb824ceed9d1d3a5d, 0x12b775513f0f66bb, - 0x55d995174e9db042, 0x076f8b1535892daa, 0x721e532252888002, 0x3cd0f01ae6ae5a5f, - 0xff1abf2b85715f68, 0xc481d98ca5b44a3e, 0xabcf5b6d811cc8ca, 0x3deb89d5eaf5f0d1, - 0xf2cdbe0a72a36963, 0x41a189669dd26a66, 0xbb21393d190cbf21, 0x0b8ae3e152ef160a, - 0xf9b3fb205bc60642, 0x97d961a7fcc17019, 0xcc3bd568e2d33583, 0x3f60b2ea2c4a503c, - 0x9fa174fe0f7afc9a, 0xb4efe92d511eb893, 0xa119057aeae74caf, 0xeca9f5f791eef516, - 0x98575edba7c69d1d, 0x195bd0e47e4e36b5, 0x8c23a24d071ee445, 0x70d2d956d0dd835e, - 0x376811b131acfe91, 0x4c79c5fdbbfd0cfb, 0x99fc3d6eb750d007, 0xeba8b3be5f576336, - 0x477e4e59f4fdc9fd, 0x13885032cd58535f, 0xc1b55d3355638eb8, 0x3d2f615f169d3f6a, - 0x843c29f5b6e7535f, 0x512aa178fa51439f, 0xd65dd85db34c0471, 0x6e42d2875d831f3d, - 0x2a8a7955e2b383cb, 0x1001632aca64cd88, 0x34f0d6c7cde3a74e, 0xe33a9ebf866d3399, - 0x29dd439e559298f0, 0x10ce1d7517cc1906, 0xafdea236b8b85bb8, 0x85c0816a13162151, - 0xaaed4331796c682a, 0xe2c8a792ba1e85f4, 0x0366dac07e900e96, 0x5ea6bba0b16c60ea, - 0x3c940f47c0009c22, 0xe1df5ad747f268ab, 0x3cde1e9e8059b1cf, 0x8cb9fc001026e3a3, - 0xe7ae2474d38d316d, 0x82cd58cf7538d495, 0x7dd8a28255163980, 0xd674bd6d9180b886, - 0x034febe527a380fb, 0x34ddc6f42ed7a281, 0xfe26713067c8fc70, 0x374e84b39b68aa65, - 0xf4c3b526011e7357, 0xd2ce72a2106e6c2a, 0xf3054cc94450378c, 0x37e6d57fe0533fdc, - 0x58648d4cf9a256d4, 0xd29a11fca3e174df, 0x36d8649aacde7f9b, 0x94e19f77059063b6, - 0xe5e89b51b00cf9cb, 0x2c751e818855bd7f, 0x0f89e2c826828dc3, 0x7e891fc8e17d9c8a, - 0x3628bc962ddf6430, 0xdb45c304f8da75f3, 0xf7741893d30ae47a, 0xb291fc7dde5f1ec6, - 0xe7fca007afc3cc31, 0xc70bd6ad2123fe11, 0xbe67528ea9794cf7, 0xcdd223f2d2f19621, - 0x2904b24f15703c2e, 0xccc5ee9f45b57712, 0x227d0b633a9b19fa, 0x35843a8569d46491, - 0x020dc9bb26460c21, 0x238403a590796adb, 0x31f0e12c0eb5dbcb, 0xa38a3d0d861dfebb, - 0xde2fbad1f785501b, 0xee2b30c4f324138b, 0x98ab99d5bfdd9c69, 0x861e82db61d94095, - 0xf01365b57887050c, 0x4035ec826c3acf86, 0xa8e0670e3c42f052, 0x3ac90ee28f7a13e3, - 0x6e6851a2e9928cf0, 0x975aa980363ed943, 0x5d5e5a01730bd6a2, 0x7750c0c7bceb983a, - 0xfa45b7f12b26fa8a, 0x498c63f2ac2b04ff, 0x605601a3321dce10, 0x9bee862b986227b6, - 0xf861fbbadbdb630e, 0x7083e40ce4db97f5, 0xdb681adc2d8a7c15, 0xf681a91cdc681b1e, - 0x268fcfa3224d3daa, 0xb4f63c71b72b254f, 0x540d8a4953be95f1, 0xf9e98bf8957993c5, - 0xde4c584383ce966a, 0xdab33cf5146eed4d, 0xae8160cd7430f54c, 0x7e25be30091b190a, - 0x0eeaba5b899bf74c, 0xe4ec82d56334aca7, 0xa578b137c276bfc1, 0x37b2443f3ee0d387, - 0x6201d379be970090, 0x1e60191c8936e4a8, 0xc479e792f67028fd, 0xffc2702a1ec11028, - 0xc01d455992035e85, 0x87d6a3ade7518b6c, 0xc2d3946ec8e21795, 0xe2e4f33c9237cdce, - 0xf59edb1c6442e9b6, 0x58011d6932fc3128, 0xcfa7495bf1747a29, 0x13c8fdb961187b6a, - 0x39d457d928d21f0b, 0x10122608436b48e3, 0x279eddc8503cc1e6, 0x223d709a37ff2861, - 0x8aa47d71f3e49b7f, 0x849c6a677aa4629f, 0x13073ad393e934aa, 0xf988c87ef3ae715b, - 0x82a1ead4d546ab53, 0x84ca560e15e3405d, 0x6a1e83d44e2a2177, 0x5c374cf6d186c23c, - 0xf38e63022563c0db, 0xd9a7bdaea181fa42, 0x1a2e8604ee7cf6ab, 0xbdc7682cb0ebacfa, - 0x45d3f679ccd1894c, 0x0ac2b829ecfc8c60, 0xc2da590f8f0bbf89, 0xc02cf0f17df04fdc, - 0xd32e8bcce7d404d6, 0xd597346447734f5e, 0x8aec16b04d7de9a8, 0xcf22898783c9797b, - 0x04622ce05bc21853, 0x14b3227ba08a7f94, 0xe92db65a356b90a2, 0x73f2030e32461d18, - 0xe4289c18c25e81f4, 0x8e87c9460626edc8, 0x76af04ef0d37c672, 0xfe4ac3f909004738, - 0x17b6adabb26ba928, 0x34cacba2184a050e, 0x59c0d373d5e2f155, 0x43cb03e0bee6e56c, - 0x014a3d4d03442912, 0x4d64cd9f1dd31262, 0x0e3dd0eda3befe30, 0x379341cdfe314be9, - 0x007672a61fc9f3d0, 0x106aec40b8f0b706, 0x72932430dbeccfba, 0x44329422fd5250c7, - 0xc13d9de94066c03e, 0x9606343f6e8fbb39, 0xad0b36a2023b7e9a, 0xd392718f636e3916, - 0xffa661d8473ed057, 0x8e89a93b64866159, 0xe0ac5d9ef1b35489, 0x9a8e5c77f8e23fdc, - 0x0d69d9d700830b4f, 0x9d4de4d3b2893214, 0x421d36b2b977f44a, 0x08441ddfadf7c82b, - 0x73d5e36d031aa64a, 0x0178ea238f4a668d, 0x41086efcd137c9e0, 0x93815bfa246fa7d8, - 0x2c2c7af010e8e8ac, 0x69aa1ecae6702468, 0x86c7df8792f2a816, 0xf9eaf333564db1fc, - 0xaed38689b13d44f9, 0xed31bea13b137101, 0x86c8b2dc72877b7a, 0xdd1ce9c96c69de43, - 0xfccd5d96c27bcf4c, 0x140142a084740d51, 0xead5b31a3bcffff1, 0x3d33179449fd804b, - 0xb72eda6c6a0f089f, 0xaa28cbfb5bfb4675, 0xfc6e4863dc890a0d, 0x2de3edc7d578e5ed, - 0x27b2c659e7a9753f, 0x32a3d8117b48cd4f, 0xca503b515b39f4a0, 0xa677b99f25bc8e20, - 0x451fd81f76994ae0, 0xf3e3a4740bc75b71, 0x80f74b3b5a91f164, 0x7564d53caba94fca, - 0x1bcb503482da6d94, 0xade40b59f1a695c3, 0xd505ab145c91c673, 0x732293f3a00e0f9a, - 0x9b63c01737b24402, 0xaf081eada0ae6d72, 0xd87a89d52c0039df, 0x6adea921fd87a5d0, - 0x92024529e22929d5, 0x89f394700e0310ec, 0xe49a26d982be56b5, 0x61462514a0de0983, - 0xaba899b577b13826, 0xb6804c2d5ca7934e, 0xa07250312b8e0ff6, 0xda96a95d7b3192d5, - 0x3a19d74fbba1a8c1, 0x97a8361c6ade2a61, 0x84b8958e3c77c548, 0x145de3835d14ea35, - 0x22c98fb2c9f41283, 0x638b9625dc156b46, 0x7bef5679d95ad5b4, 0x9e7181a9d72bee1c, - 0x8e3ec5543ab38848, 0xc1715b1b79de41ef, 0x928e894f6770bac1, 0x4b42c5ba57efc7a3, - 0xb9b37daaedd851c6, 0x522d8e3cf65bea04, 0x0d607f65046c1dd8, 0x91950e1c6339cf74, - 0x4ad019797988807f, 0x9a7aeb1551ac64e0, 0x0564d9ec1415e3e1, 0xe74ba9d93c061c1f, - 0x7ce84a7a41eae055, 0x6d4427bfb4f8d6d5, 0xb48105e0cc573a10, 0xc6af6eed024f5e31, - 0xa8c5202618b757a6, 0xcd1dcf8ed7a562cb, 0x9deca935e9bde00d, 0xba1ff4ac95319622, - 0x004373b8a9c3d68d, 0x63c375ae1c9c995b, 0xf30181926ff19e0c, 0x2818354995394ffe, - 0x34834b4b7ed87b64, 0x15e6d05b73e8987e, 0xd2d2323012f37e27, 0x97e9784f4a1e637a, - 0x77ddecd7b4c6470b, 0x27128f0f10179469, 0x2c6cd115d0eba682, 0xd88a0dc9eac90b9b, - 0x4a6d39b8eb620c5e, 0x66f330eadcb921e3, 0x78bf4bd0aa46fd47, 0x6df84089fabdf014, - 0x2a4ab3428a29eed3, 0x525e0c856c41d5ca, 0x1230c714f17a1519, 0x227e27cb5adb4d15, - 0x2eae0a7c7d0302f5, 0x20e2baf1f2326dcb, 0xe5c9ba92458428e2, 0x23827844e6abc7a0, - 0x8fe92f51c3f08cad, 0x04cc291799d15b4c, 0xd769387e1b0938c9, 0x4d92581bdf1c882f, - 0x51bc6114de8f52bf, 0xccf7264fb4fc324a, 0x71604a70ca496270, 0x0b1d85fd198ab5f2, - 0xe5f7c7fd349598f6, 0xf6f1af235d3d170c, 0x411196355f4cb176, 0x062554f27247fb62, - 0xc2a3d2456635d561, 0x3f5549e9ff5c4361, 0xeab782b5e0676c0d, 0x6d75fcfd8c41e5da, - 0x01d22662b2f10946, 0x8dd5609142f058ad, 0x1e4a47028b57429d, 0xdad68252963b2aeb, - 0x5cd07b22bebc724f, 0x301255351476849b, 0x65426039acc55345, 0xc59177fee81f42da, - 0x6106d885bab39822, 0x34ecfa3eec4ee67d, 0xb2ddf76cf67b12bb, 0x7b8390c344454a64, - 0x1073f8c4519fb963, 0x44f8ab2aa3bf3c98, 0xa5f68ec3e120a9b3, 0x3b10cf522e7a1416, - 0x427becea914a8819, 0x5e9983cf1125e4fb, 0x8bda6b21e34d02e7, 0x307192a1daf91a02, - 0x02c86eabb4a1e657, 0x7820ae0495e1f2e9, 0x2b1ee5aef70e365e, 0x717a5ae0d24ccdc3, - 0x44152db6e842cce6, 0x15f0bfd53b68e078, 0x788c4704f9ae91e2, 0xa434ca03f8f80e34, - 0xa8513ce54077b73b, 0x64c986673b9f6def, 0x3599d1f40408508d, 0x830b26b446b88edc, - 0xfc0c6a59fa4dffa5, 0x6fc9e4da6c59de2b, 0xf9cb100ee5572428, 0xccb6e919f73d3e60, - 0x214675187f1cdc33, 0x8b2088fdc031b4ad, 0xd970b242ade60aec, 0x27937b85df7ae47d, - 0x4b6bc7390dd61eea, 0xf50abb734c7e120e, 0x62bb9d4e2efd2b17, 0x8b0d31da97003fb9, - 0xcebfa2bd08acb9c2, 0xa384915e1df256be, 0x0ec9923e8f5ea745, 0x88db66154bdc7917, - 0xb0e856350c814d61, 0x0c1b08a79b522253, 0x037dc23e7dfdea72, 0xb3164158bb21f80d, - 0x0d2d68cba4327b17, 0x742339e8ff2afb3a, 0x6fd375e66e7ba375, 0xe51519a74635669e, - 0x10cb4d17902183ec, 0xbc778da751d2ca1e, 0x0184561788aa47e4, 0x0fffd9ce36b9f404, - 0xa8271a86165723ac, 0x7d58fbf0be388247, 0x26cff02050246451, 0xe2af042d5ac04a20, - 0x54702026939b8f57, 0x77762534a707f66a, 0x926ff890faa6c516, 0xbdfa778f70aa6f24, - 0xe2fe916086a67b33, 0x3e853cd57987fb5c, 0xe20952cd639ed00a, 0x089fbbad7a113eee, - 0xa40a6144b74c535f, 0x6d1b2ff4a72f2edb, 0x63c4273bbbe83cae, 0x522fae25174940ad, - 0xa558168481abde13, 0xf3e923fdf1b1ca34, 0x99705c81b4b80f03, 0x3ac195454c5828a2, - 0xe95a16217036bc11, 0x5134c11249d2c523, 0xf85a2bb87a0ce21f, 0xcca1e989a4c02765, - 0x86c84b987f500f35, 0xc9feffe228f31bb2, 0x77fb1f0b04eb2412, 0x0af90b7cabc65ee4, - 0xd8838f199adb99b3, 0xdcb238c85754b528, 0xd6d940edf6b0c1d3, 0xf656c851eb975350, - 0x3934fb243828a416, 0xa0d3d390ef837e39, 0x55581690e72c86f5, 0x05d914bd4d69ac37, - 0x53bdfaa2b9a62dc3, 0xe851027c34c08a98, 0x8d4229ff81662c6c, 0x54102d1cf4d40ed7, - 0xf17cd34ac6a8b9c1, 0x0876b3c5cdeb7810, 0x9d9bb512170fc7bd, 0xbb31b0417798a847, - 0xd053d6934cc7e2dd, 0xe02cd0b547f57066, 0xda387485f755e14d, 0xd2bde4818b881fdd, - 0x671affc79b6d0250, 0x8b3cb113006f3855, 0xb341c55a23424b0d, 0xbae2022f91afb021, - 0xff91b3fa9fabe199, 0x4e7c062f25283a4f, 0xe44b1538c68184be, 0x4861571f83399798, - 0x319ee1f3ab051cfa, 0xa0f6111ff699a869, 0x233f85463bf1a5e6, 0x4514e911b249798c, - 0x02c9fb3a13794ce3, 0x2eb4d867ea61a127, 0x938a8260beece405, 0x4415470c32a9a19a, - 0x82d46f8959bbf2dc, 0x13265156a5c6101b, 0x9b779587cdeb1ad5, 0x82e740a33d712be0, - 0x271a2a557506ce79, 0xa6766e26cebe6fa3, 0xeda1cd2229de2528, 0x8ed13b11362a4b0c, - 0x5d78aeb1a576ba58, 0x3df1201fc36a4c4a, 0x30d4c24dd02b085d, 0xcb443eaac77ff647, - 0xd6db42d013c48f1a, 0x9e91c7805e80114f, 0x92acd998e05e17cf, 0xf8dbd74f0d426206, - 0xd698890aefc0d121, 0x457907dd93481c05, 0x9e1a93f457bdeb19, 0xd10c479517574c62, - 0x7cc4d0d71a7c535b, 0xa5ed5a0248704d94, 0xce9b6319482a8d22, 0x4835fcdd95ef556a, - 0x1d9a40999e218b59, 0x867c1c02370fe465, 0xb4e5172286fcfa53, 0xb480b78ff0453c47, - 0xce1b9e334908ac46, 0xcf350dc973c5a833, 0x160da9bc5c80aaf4, 0x74caaa8fa2d99fb7, - 0xba8ba8bd5a7aeeec, 0x3e596e838f918187, 0x7d7aaee18267d453, 0x02bac5efec7edc1f, - 0x8b9bf297183c4b3d, 0x8f6bfc78a9f6c5c9, 0x01beaad60d780fc6, 0xa57d153b0dc6393b, - 0x9dc8083875eb49a8, 0x26400eb8b7eaeaa7, 0x01f5abf61cd369c4, 0x11ea5e244dce5053, - 0x26d3443f4fb33168, 0x34a16af07ecd23e1, 0x66761400ab1611f7, 0x2dc645b91bfd522a, - 0x933b1308b15df495, 0x5e90df16d37d7b5c, 0xe36466ab0fc587c1, 0x03f00421184591d1, - 0x4f158b2535b4748f, 0x4b9f578a8c3b78c1, 0x221e0bc705e4e1b6, 0x219cd523e09a9a8b, - 0x0759052be84a1d4c, 0x3f069d517a0e1bc7, 0x2cfe71531e3c7164, 0x165683905990b275, - 0x4a8884ef69f7163c, 0x158fc4744d2304ab, 0x4f756446b8bfb18b, 0x23e3ef0e2beabe6e, - 0x3e5a721ee1e78572, 0x57a0292e73c84428, 0xbc1be1d5001b793b, 0x4a42ca150853cf0a, - 0x094b5fad18671f8b, 0x9151e6247a85e7fd, 0x15041b5c4d970ee2, 0xd53e821cdee9659f, - 0xe61a3ad2adce5555, 0x6e3113dbf809c9ed, 0x80c819cff8d67d01, 0x66f44c6001defd15, - 0x5ac5a8060b13147f, 0xb58d5398b2864652, 0x8fa928b55875178b, 0xd70f80e5508e85ce, - 0x61e2a0810e4964c8, 0xbb2a4e4bc898e759, 0xb84f2e399bc91a9c, 0xf69da9212176a42f, - 0x800afc6aea93b421, 0x9a0cf748400666f8, 0x947748aa5dde387b, 0x55dceacbae68b51d, - 0xfc4f77fd4f549dd8, 0xbb3597b6b777b331, 0xe31fa57535ba0029, 0x6857c665c89a8d06, - 0xf0150f11feef9170, 0x54b42cb9f07e9276, 0x554df4b53a8cb80e, 0x5750dfc7e428f69a, - 0xc297da095fb40b12, 0x15f7cba9c40dea51, 0x52daeedbf036d123, 0x7b84b56fe6097026, - 0x758512826cc8170c, 0x0dfef0c2d9ac150a, 0x2ddbc1d22a157e5e, 0x7c44d77c4fbe999c, - 0x203e9bd9d5c7172f, 0xa3e39aea24d10dd7, 0xaa66abd16c28ed27, 0x40c62461b35bfdec, - 0x29f51d4c4771a668, 0xf8e9b464c1e88f21, 0xf20f7eb7a51894f2, 0x64a8105cc60a0270, - 0xaadbc86039a9a57a, 0xaf3bac98253c1634, 0xac34a6837f41c394, 0x89368860afde70f3, - 0xc680e452edd995c6, 0x14af5144e7e90a6b, 0x3faec54444bb24ee, 0x3e996fc07138593d, - 0x111401694f223132, 0xc328ee018fef0d0b, 0x67a34c1e66db2838, 0x090636d317f38f7f, - 0xbdd43f561114f409, 0x16992724d4e4c63c, 0x0b6d9736e40f08e2, 0xbeb2b8657da7bd2f, - 0x727ddac4999e08f6, 0x0a6f63bbe834abb9, 0xd9816000f5cf3e08, 0xbccdb57930d6d43c, - 0xcfcd975a99327801, 0x87c9a3f4080ffbfb, 0xd36edffaf3bb9e13, 0xf7cff74b58aea5fe, - 0x6b628e2947c14fba, 0x0aec5255c50c7246, 0xc63cd582a038af10, 0x567a466f64645ec8, - 0x17e59a5735c2886b, 0xe4c09a576abaa561, 0xb29c2c3628aa811b, 0x7491d8b4efe015b6, - 0x87ea356bfef10bc3, 0x685fe0cef59576de, 0xa2e3a56d313a52aa, 0xf0377ca878562272, - 0x36c8e2720f8ecb0e, 0x2d590839618b95c0, 0x0e737b5c1b2a5590, 0xaab90d1d39a153c4, - 0xb3a009eda483f5f2, 0x8c33138f36da932a, 0xf8f9ae975dfc6c33, 0x24ee96046fcbb061, - 0xf115d2e091b5a157, 0xa79621534581baab, 0xac4f1c4a7dd3cc5b, 0x5de3eba570a70f97, - 0xda34ce7a78fb6c6c, 0x319dd6c86eb679d6, 0xd944e4e7f16a0980, 0x066b9bcd51b4ad58, - 0x4f015dd14514d799, 0xf7ce1613739ef86c, 0x9eac9075b26ea209, 0x19525f3c11e83ea8, - 0x80b9ec4d3f48ddad, 0x414834b8d74ea823, 0x4468d596ea2dc92d, 0x5e6154c3bb4ba040, - 0xc6615f2b6d6cd575, 0x39b6cbd0ad1a41d9, 0xdb8b6735ed4f63a8, 0xfc74b12b37856fa2, - 0x1ce7a2da5e023bd8, 0xc898964861a0bdb9, 0x1a7def3a52e5fd73, 0x71fd5a09c22b352b, - 0xe29e055d7f878a31, 0x7ee4eae05a5f980c, 0xbc3e4c0cb2e3f896, 0x0e200f5126faf587, - 0x443fd23624783cf3, 0x0d063daae8925bf4, 0x6d9636600f24dc79, 0xba9c51c1b2a1fd6a, - 0x4eaa624b142afe91, 0xa5db5bb7d79cce4a, 0x3a0821d63c40145d, 0x5efa9404a833c594, - 0xa31a689829413c2b, 0xa93210a99aec8bbc, 0xa41ac15944fd678d, 0x97ba4792f50c837b, - 0x678d22b1384d62d5, 0xab11d38cdc067abd, 0xad088f07c162f370, 0x7ab8923503ace33b, - 0x2a171e096a38c16d, 0xe8e4a918b15425bd, 0x023c10a5625c4c72, 0xf56fc8b9a10d2db0, - 0xde556046861f9d65, 0xb0b6ed784f3f25e2, 0x6e9f9f870fc2ed01, 0x7d98a862bb85e6f0, - 0xdf99dd1ff243df98, 0x5bb5494fcaa4f974, 0x06e97960aa88cb5c, 0x9605364b565f3991, - 0x26db4b055418b2b7, 0x5971bb605ca05b97, 0x5efa9b46c5ec8251, 0xcc19263a40c1108c, - 0xd4e051f82679c884, 0x3860e39c2e6ac099, 0x9f7777398b80fa4b, 0x6e509970a9515259, - 0x8cb8f54e7c58feb9, 0x2f441c50ccd87797, 0x4bad1116649bb530, 0xb6e24c757ee4bcc8, - 0x4be79a48d73da723, 0x049663f4e352740f, 0x5a5b8d8aeb2c3879, 0xa13d7469c70e1615, - 0x5d6ff6c2f018648a, 0x4943109c3f64c7e9, 0xab4a80d4c3a8b0bd, 0x459c668c442712da, - 0x8437b69fe40211ac, 0xda0a84df90132140, 0x9e57ed870cf80610, 0x403cf5adbbe5e824, - 0xb6b574995d7fb601, 0x5178bbef3a757156, 0x148bb00a8400075a, 0x3545cf79b132b0a0, - 0x64b4148cd26eb507, 0xc9fc8b60a10d777a, 0x50d80f1f766c5233, 0x8667e5cdd176df0c, - 0xd30309cee4029ce8, 0x24181059a5d75cf9, 0x6576c840210ab758, 0x3224708589d629c6, - 0xcab6bdb24cbea116, 0x3d677bc0729ef9d0, 0x4effc3b80a7e1770, 0x47fb10ee069ef07d, - 0x7f45a58e0468db01, 0xf26a4c91b3a2011f, 0x9c6a00ecc0a7f1d6, 0xda5d4ea072670069, - 0x8ded69b02d321d06, 0x088614fa8e2fcaea, 0xea6adf01b088397d, 0x50099289e503f404, - 0x9cc533462cbd80d1, 0xd2fb1b6524744887, 0x80fea62ed1f5b2f0, 0x619e0ed78125c15a, - 0xf55ac124084da683, 0x762b33239912a421, 0xf2ead76786439989, 0xb2c63fc0eb8b30cb, - 0xd49cf9f4340b76d3, 0xe3d1be168cc36501, 0x1ddabdee490d1807, 0xd649a0f02c664423, - 0x9565d202b62a2a5b, 0xde4a04c44345253f, 0xff70f3d535ad3ca9, 0x4dcfc9b6f4ddba82, - 0x774208266218206c, 0xde0f1ba36a563024, 0x253acffbe28412b0, 0x06a57b105e037b5e, - 0x0cabf140bdd8644d, 0xd475f9a1e64b3a92, 0x0c0acfe2331d4791, 0x52329e48414ba1c1, - 0x3330392197428a21, 0x817403d59e66a296, 0x7083f346a60ff453, 0x563c187652fb58d6, - 0x997c955b860a49cb, 0xa9a1941d98ea6fde, 0x22199aa5b91baf9e, 0x7ea55f084898f7e1, - 0x97c8bf18c327632c, 0xbb951e749c739167, 0xf12d07eb0283c0fa, 0x9a31212bca52f1bb, - 0xda40bd6437eff8d4, 0x605cd0f796beef2b, 0xbbf20eea41a8396f, 0xadac5b64e2d576e7, - 0x571415862dafa633, 0x1b49edcde2a69ebb, 0xf479d41db5eec567, 0xcd62655e63e2f313, - 0xc95c7c7f38807d44, 0x474ff1962b9e4d92, 0x3d38a5cc25213298, 0x86d820f9c5ccd5b2, - 0x3d7fe6f707780d46, 0x7d2c7236d42eb4bd, 0xf3e65af361c855c3, 0x384b7c1909d042e5, - 0x39d5bfdaf5fcfbfe, 0x0172c1b2fbde2741, 0x870c648e0646e842, 0xa700ab1c5503153c, - 0xca2b419b67da219e, 0x84321b37bc28019c, 0xd59db5ef6c6e1c17, 0xc56b270761566fbf, - 0xa90ae4edc70691d3, 0x8d9dfc844f229916, 0xb73c1ed7a2e78281, 0x1cccd9dae4301c2b, - 0x2504359d4f43a45e, 0xfb15354948dd428c, 0xcafb1d2d26f841ca, 0x7db6ae3004e03831, - 0x53db786afd441801, 0x64f3bbf8c2772923, 0x633310c3703a8bd8, 0xc03bb1b54a4d2fd9, - 0xaa2fc3ccce88e7a4, 0x796500b980c2e181, 0x5d66ebe2c5894bad, 0x9fc140ffe8c020df, - 0x23439d0846ba2e74, 0x429cc47a7fe95d8d, 0xbbb605e08beed417, 0x2886d5d8d097dbc2, - 0xe309bf460fc67287, 0x438063b5c13405bc, 0xf188858454709159, 0x3faff66aa8c39b3a, - 0x60c0dc6b6072715f, 0x683eb5c6685d6432, 0x9d5cedb2319ffdd9, 0x585460f7144f3ade, - 0xd1f3b140b125a1d4, 0xeaf5984d10714b78, 0xcf33a8b448f55423, 0x2dbdbc69445010b9, - 0xf0d249b5ffd3cd80, 0x7f21ae4611c4fbdd, 0xd91f1b15d21fb0d3, 0x1a2edbb9e26b41c4, - 0x6a4098c7009ee660, 0xe18ebda4967b923d, 0xe9de10473a3b4d0e, 0x8037c0346e3f265a, - 0x037f8025bed4239d, 0x4373d6150a43388b, 0xdbfcb1d5c757babc, 0x3f1e27fea50a587d, - 0x90d02e1ca2086498, 0x9b65af99df610917, 0x9f441e6d3bcd7cd2, 0x9016a22081462046, - 0xff1bf7cf7f86142a, 0x04a9f693a634ea4c, 0xa84dfbe7d777831f, 0x3a4d218995e1bb77, - 0xf2db40ace1aa604f, 0x42d934de2e01dead, 0x0c30543725afbc82, 0x7efd1e3f2c9c4d72, - 0xffc0f1cab1dfad3a, 0x8f6d224f1c97f995, 0xff659d3c41135121, 0x83767ab5d2b8a44b, - 0x034580d612ab51db, 0xd9b15e604f94c335, 0x583447d1c315d377, 0xce16b25d25d64750, - 0x4b3e599a181db28c, 0xad3bcc61707a1e54, 0x2304aff5e0e1312c, 0x2775dca215bd4649, - 0xa1df72a526105430, 0x8d8c3abac369df24, 0xe0307819b752455d, 0xd66dcaf92ed7e80a, - 0x40957e2b9d8ab1eb, 0xc3ee067d572c1eb8, 0x98949be7038f4312, 0x1e44dac9cefc0a3e, - 0x6790661b469d1e2b, 0xbfeedea8cc2c4156, 0xb1e37698943c97d6, 0x7a9904c8ba563d5d, - 0x4a518a7e6f8b50ca, 0xa69e67019af3149b, 0x092459dc16374acd, 0x5d3b1fff2da6bd7d, - 0x74c841f4443a18db, 0x9bf8b6ee19dd1ee8, 0xa3d9ffc7a1c69b73, 0x8e7a2c7089077647, - 0x85fe44aca6e3cbb3, 0x3c2296d490990de8, 0x8c4ece1c07040dab, 0x7dd19ab8a9baf65a, - 0x232bb32d97595076, 0x90bf58b35d1abfc0, 0x72d0b2907af66b91, 0xc7fa661133f2a951, - 0x6cf30ef3657e7042, 0x508ad0f6160b767f, 0xb19cb4d4e6abfd12, 0x07e9cdc0317565f5, - 0x20d8c1c51f5ef8df, 0xb990b0a1c734ee59, 0xb14410502b26f576, 0x36bb1fd751cfed83, - 0xafbee0f3cfb81fa6, 0x7f69e91a15af53ef, 0x6e78f569b4bb69df, 0x6a70b4b07d88e898, - 0xe95b7e6d6b4eb64f, 0xcbed9d5d310dd85c, 0xbb6dece954c9887c, 0x6288bb12a18f10a6, - 0x98f871c69dd90ab8, 0x1527fb57661aaa3d, 0x9d6b73833146c0d6, 0x0e0e0ac0e377c037, - 0x23d629997ec20f4c, 0x8a742719a656c90f, 0x945cab17ddc141d9, 0xf9cbdfa93c170c50, - 0x9a3403c0e0dd72be, 0x06564cf2e1eca6cd, 0x359d5c3d173da384, 0x7c7a06ae74ab9168, - 0xdad0da229f4faaf0, 0x5f72912b71ac4412, 0x9b8426fd80942e4d, 0xbb6ae1d0a01a2f23, - 0xbe24dd2c41c93254, 0x2e57c1483c75be6e, 0xd2c14fbcd6d21341, 0xc88522e46072ed0a, - 0x8fdaa0991de52916, 0x3dee502b1d002399, 0x74c92cd78ec2c5c0, 0x1661eea3a1b186a5, - 0xc84484d2af19befe, 0x0da060548459bf37, 0xe8b22929416bff3e, 0x0ca0e69218277d58, - 0xfaddd54a08828880, 0xbef6c3256e22f537, 0x0fb0eb69ddf6373f, 0xed8f8b11f2396004, - 0xaab6757a3f66b35c, 0xab8cd32a13a079de, 0x2d325da303d181f5, 0x132a806299c3b9ad, - 0xb58148425cb8398c, 0x0c5801d22e0c67b8, 0xe38d49cb989c5c72, 0xdf415037e862eb11, - 0x5bf664a7203872d6, 0xd72f7d5233759165, 0xb66473fdee2e03c4, 0xfe0f12ae942aa812, - 0xa509d6464f5d25a2, 0x698db37327aee839, 0xc2a1a8b9c9f8e6a8, 0x0742ef7248c88e67, - 0x71891bf81d53c121, 0xc9dfd8b02bd64f57, 0xdce14bdc7d523e47, 0xb0a2792344b192c6, - 0x37b2a9e0a538a493, 0xeddd5c4b3a398c59, 0xdb8646267de3ba13, 0xadc62a4fd9bd339c, - 0x019f20ab87726faf, 0xde4eda70acac27ea, 0xae749ab863b47d6f, 0xa89f9c12da463b46, - 0x234718d959c5d6ae, 0x70d18e4968300128, 0x234e2324ba311a44, 0x160157cd6465b17e, - 0xa2ed85e77d34a81d, 0xe6a1826e6df88f6f, 0x4796738cc89fe26b, 0x57193fab4d47df3d, - 0xc513a1a0c019a30f, 0x9ca46e1dae1b37e2, 0xef7b12ddb8133469, 0x645467cf5ed69ac3, - 0x52e651f4df02b22c, 0xaa14c0549620cb1f, 0x5746a9335322c326, 0x791225455687c137, - 0xeade84f80c4d2ab8, 0x27fd8a9957d484a0, 0x6676fb2094ae84c8, 0x0267241e40e01f18, - 0xcb03582d818b6b85, 0xdfc22a614ff799fe, 0x693a2e4965f135f7, 0xcc6e9ff4574b75cc, - 0x2b076c3e2e0e658c, 0x085b2b881a7bbd1f, 0x30e0a767ceeb5103, 0x3b3aa94c87699d5b, - 0x30a233ce24e700b0, 0x0e4aa886a3fd6a3f, 0xd1d49fcd5a47fafc, 0xf90b19cd50bb9d0e, - 0xdafb3f2fc61ef2fb, 0xc0929c6d15a1e08a, 0x94ad75c2083ceb6f, 0x1dae43bfd7f2fa0c, - 0x740f3b6b9b5b0fbe, 0x2b40125473d372c1, 0xb5805dd230472b06, 0xd99dc4543a1d1e5c, - 0x099c140c76414939, 0x9dcb1f41209b25be, 0x0dcc487ef6842dba, 0x7def1d2646c52fd9, - 0x4bf44b76e1bebff8, 0x4326327426494093, 0x9e82e53ccbd50c13, 0x27903b5100c99a18, - 0xcd872a62857615ae, 0xac02f6e09c382b4f, 0x1734f4c56d1d02cb, 0x12753fc13e7f4dc3, - 0x1125ade5cd3a24f4, 0x261e2a08ba325e04, 0x337ef78a1f2c3fd1, 0x734cec0f1e32e47f, - 0x22fd3f5fcb925a5a, 0xadcb9ed05cc7c52a, 0xb30a65a8b625afb0, 0xf6e4daca5c105d88, - 0xc6d4a4ef1609da5b, 0xa4b06b8b2f7fb711, 0x294ef523beb48c65, 0x9ec40e3b213425e7, - 0x5bf5777933a74d0d, 0xdd19749213d2df32, 0xbe3190f2e3dfdc8b, 0xbe5d440a790d549e, - 0xe400eeb0a514f118, 0x5741a6879bbc382a, 0x131b6d4fcff0efb3, 0xbcdda0efe16a2af8, - 0xdb730b01d3d84057, 0x92f8aef243bdd67e, 0x129fc4e2f8c0a46b, 0xcd627daf16e57d8d, - 0x95ae1e3b0a2b0988, 0x4df621ca8e26543b, 0x98b819943a038ec3, 0x8da8c26bcb8c607c, - 0x81f5b5fb1767a8af, 0x18cf7652ad0f427e, 0x0824cdaf3721a7c9, 0xfadfa49c9bcb2da1, - 0xe88da2ecce4e5b68, 0x344a37ca56402fbf, 0xae28ef992e7dda27, 0xe0e43c2c0a68acf7, - 0x77e3c0f16ead0f68, 0xfe46928a1ca6c4e6, 0x0967c29c03112f00, 0xfaad9c730965f95c, - 0xc4b6cfd8fde6779e, 0x13fe3796b6239956, 0xd8db75bf1dd50b66, 0x187f277a478e21c9, - 0xed51a5cae48a9d28, 0x6a023b8a6945c553, 0xc2ac098cd96d2239, 0xf0c1074c1583d547, - 0xabcfdfc3004a31d9, 0x7ab0a047d0f94263, 0x4cce30e83f8b7c8f, 0x663a1c6b4fb6d597, - 0x4592fd187f3c843c, 0x9d12bc038458b1c0, 0x5d7cc4f792abb369, 0x9fecf17e30b05014, - 0x73a2505aef71ba88, 0xb4166e80b87c4009, 0x238d324793ade58a, 0xab5fc59a86fad05c, - 0x66ec07f0a90d1137, 0x27760f95cbe33b8e, 0xaa3c8d12191363aa, 0xede2f52644c3b422, - 0xfcb0f99471ab3aea, 0x2c80b728669279da, 0x39c5b67ab2a3b4ae, 0xb8ee4498a14f6839, - 0x08fa66794a3823ac, 0xc2202052921c0c43, 0x378813abbd8058ef, 0xe0c6563f2978bdb1, - 0x91344f646817d7d8, 0x88ab2bccc0841f4b, 0x8634d29394de4e7b, 0xd8a77161e46775d6, - 0x4499f660472c4f23, 0x7b24628eac38944f, 0x96383e965bbc9837, 0x780557b07684d9b8, - 0x03afd86f271a0a74, 0x908e396ef9238bad, 0xb92fd06cc0dd940c, 0xbb045293dcf6f988, - 0xab2cdf4583625b3f, 0xe886fd7283f9221d, 0xc4a05a54478e5a42, 0xe51cab325f7bfa13, - 0xf5f723630efbf9ca, 0x099911f715c4793c, 0x27cba6cdec27fb69, 0xbca037b732de76ea, - 0xe236408b59c78a5e, 0x8f9c80b56db4b012, 0x02d77c317fca6c7e, 0x42fca95370eb02db, - 0x239f22536c9ada6d, 0x88f70dfdef1b8f23, 0x191351f07a6816df, 0x30d67ff2d00a7c57, - 0x2a7d09d70585c799, 0x27b765045cdeb865, 0xeeb532c45821e531, 0x2f015a53c55cb2ef, - 0xfebc565b08ecde04, 0x0fc630fab8c46d47, 0x5e7458e16fc98219, 0x5e560e3a3a93d199, - 0x1e966122c84a888d, 0x4e45af24e0b3aa76, 0x3126f4d3acaf8528, 0x41c02305501b89f2, - 0x1775a35e56225425, 0x345bd20bfeeac8e8, 0xc4fd2670a58c14b9, 0x6e0cdba5f4901b88, - 0x87e905b79969eb56, 0x6d47b1da0987c341, 0xd5ac0085d9a525db, 0x42055449dc20810a, - 0xd379eb4956df3aae, 0x57fe735cd956e681, 0xb242177cb4c7d44b, 0x02157e10a51ba192, - 0x2b23472c86f3d42c, 0xdfb752c6e70df66b, 0x440c53749c109ffd, 0x51eb1d3e92ffba77, - 0x7abf2311203c8c7f, 0xf5c9a9906d15869a, 0x328d57d8b3ac203e, 0xc23d4e465c428841, - 0xbb95c5c513f489a5, 0x202167758c1c462c, 0x3641162761b0d42e, 0x8241254f3cd29915, - 0xd19d8567565d6d9b, 0xf121589ddf4b88cb, 0x296e3e46efb12b76, 0x3ab5000a0ea7130e, - 0xd6fd4d1ac31dba80, 0x6a410fdda6e0ce74, 0x658d14f4778cbc1f, 0x5dadb0a8c201adcd, - 0x8d32f17901a55767, 0xbb4b5bce928fd087, 0x7b616feeb27dc0ce, 0x1e49a26d572341e5, - 0xbb0a1d434cdc2c5f, 0x97724782aa0eca61, 0x7f70f10c098eb7d5, 0x17019b0ba4fbcd0a, - 0x4aa0f6b75317ba5e, 0x96c0c0f39e677232, 0xf2b123c940d1052c, 0x2220d9938ddb26ba, - 0x2eef5baba6b8e709, 0x5e857aedda5e98f1, 0xade8b89d75933075, 0x09e1e8fe080477b4, - 0xefd86b2920c320b1, 0xfc424961a7471965, 0xf2c44396b38f9c50, 0x7223d57d683e9333, - 0xc700e0da3238116f, 0x1dea33243fe13842, 0xd323bc16706a43a2, 0xd5db95d2d807f849, - 0x3d1ec16c575f85b2, 0x8c71719ddb7efd7a, 0x58e3f5a328033faa, 0xbac2081f989aae78, - 0x3ade383fb5d34aac, 0x5385831483718ae1, 0xc79389ec2f676712, 0xe627a205855e5dea, - 0x26a76d76ed73255f, 0x3cfcbe8a36e30824, 0x394916cf9f31c655, 0x1ca701b78097ba76, - 0xf334182a175bbf9b, 0x571a6e2f0bda762f, 0xe0101eacf9bbfb5f, 0xc60af282384b5eea, - 0x87b0affb2a2c929d, 0x2f340c404bb2d36f, 0x9e40556fa8dac4b6, 0xdbf1a2674d3671c8, - 0xdfe064f4859e9226, 0xb66ea881ac47e5a6, 0x62e536de8272168c, 0x1a92de4a36277a7e, - 0xc717d118b4d078be, 0x5014c077b69cc9ac, 0x15b105d9921ba0ab, 0xdef33690123b4d4a, - 0x8cf3d8d71634870f, 0x6842a9f72188de60, 0xa3e718f7bc7a263a, 0x4b49711f6646e065, - 0x5b391ae995e14dda, 0x6c42da131020c85e, 0x885845dbaf9bfee6, 0xbf29c0990d319625, - 0x727194da1427313e, 0x4e6c0ddb63378dbf, 0xb818c3b35f5f09aa, 0x5c17db76c9f32a7e, - 0x2b5ecbe9c52798e1, 0x06967fa08ce3073e, 0xa21379807b751fee, 0x367435c845530e3b, - 0xa2e464f7e5c7ff3a, 0x5234f5d5d7180b4d, 0x2cbcdd8acfd9313f, 0xda5fc23c4d95902b, - 0xefe8da391c05112b, 0xdd269a45cd470d6b, 0xeb0c9bd848761aba, 0xa1405ad04a697470, - 0x636979500c7c29a5, 0x7bd3b449007fbebc, 0x7aea4c9ee3ce6489, 0x2b957cd3788659c7, - 0x3cef4ddeb7171144, 0xecfbcaa20bd096a1, 0x8229c85b09c647cf, 0x884c0ccc943ef066, - 0x1093a6b760a2f03e, 0x8721728d4594c03b, 0x1bd6fb8524a81344, 0x0818df4c5fd9dc1b, - 0x01a43170f8cdac92, 0x9ae2a7e7bd7cbfba, 0xe2bb654c0f72abe8, 0x79314e00b59e14a9, - 0xa59a03a0d4c7e5b2, 0xd6edb8057b5f6d24, 0x4dc193a9d7be0475, 0x06effa5a4c80cb10, - 0xd8143f9af1c79f8a, 0x04b7b53a57fbd112, 0x00feb3f77a841900, 0x69df01dde1a6ebd0, - 0xbd11031bd989f0e4, 0xa169069023a4255b, 0xa5e4e1f64f9c4624, 0xa689af56071f9a44, - 0xb55084c28d7b3c89, 0xc34f82b2c720b248, 0xa8cf3432667f3f66, 0x95635f37ae3ab119, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x608a9a79434d1e8c, 0x2c593d2432d34f10, 0xbf48338b0a121bbf, 0xbdb80209762066bb, - 0xda7250f8a3b78fc4, 0xd9bc25a6960b343f, 0x7990e3b9aa443c61, 0x78bda5af8714d018, - 0x42ed9aed861543b3, 0xa6dac9052bcd9fee, 0x9bb46dc7849882b8, 0x802725f4f3ed580b, - 0xb0315cba690f2eeb, 0x7f4cf904fc0f2e5e, 0xda1e50b7d2bf3bb2, 0xf334dddffeb4112c, - 0xc50e5a02e56e0960, 0x396d29a7bb6ae889, 0xfe4956912f0dcbd1, 0x9eb6bf6c22dd781b, - 0x31a6c534f7b0c9de, 0xfcaad155fc1fed8d, 0x27b7fd72331d5b94, 0x050af92fa9cad789, - 0xec71d451b13c9708, 0x78743df2a256b478, 0x8815a3ad839d37fa, 0xd7b698839ea28594, - 0xbca812700133fe46, 0xf19c4a7077c1dc63, 0x1f94635e6cf1bb69, 0x0941fabca137624b, - 0xd3a79ea98f03c84f, 0xde6d7f5e7ca1df15, 0x02340b4849296a77, 0x139ae3f68f1a30c5, - 0xe963af4494d03602, 0xf1a4f89b7741372f, 0xb5aea277697d4eb7, 0xcecb1902dc515c1c, - 0x0d7a333ad43cbc07, 0x62db1ee2c5baf4f2, 0x6398d7179c719e66, 0x319999822c32948e, - 0x5e0872e6e16001ec, 0x8f72d00606aeb35f, 0x74773ef9c19e9966, 0x155391f8465506f6, - 0x6a175ec3429b728f, 0xe48ba68b212a5132, 0x815a5c6a22bf9a02, 0x80682039591c8952, - 0xf55dd43e9187e292, 0xacb66bff14a03d9e, 0x4098424d81ed75bd, 0x11ce2c210b2531f5, - 0x8f32a33b1ca05c53, 0x96c9670b5d0c668a, 0x258ede25feb6dc13, 0xa11c3324ff6c8404, - 0x7de5b199ab720f1f, 0x61196eae48c3086c, 0x18d3c132192b3be3, 0x13cca2bdbebc956e, - 0xad87993e3b85f761, 0x0815ecbee0d1c607, 0x09d262627efc0fae, 0x89014eaa36346b81, - 0x2efce345bce1c558, 0x0fb8c0c7756bee2c, 0xd2bf0a72cef372eb, 0xdc2ad1bd285967d8, - 0x0805988c101b0880, 0xf928c2d3b1296a21, 0x619642865fb1dfec, 0xa3d2a2ad25746021, - 0x0fa48081cce10077, 0xc6470087a128e9a7, 0xfb66e18200b5e018, 0x129c58d559f07a83, - 0x107b747df0fb1826, 0x681bd67ce21e14b3, 0x632f75616a37e290, 0x1a58659b0a478422, - 0xb3d1c8ea386dbaa3, 0x5424a90a3aa7d2ac, 0xefd15d01c30be310, 0x9a383204cb1a6233, - 0x83b000c9a9b595aa, 0xaeb6fe33ad51ca66, 0x6ad5bf1cd1eb028d, 0x5cc2a834a62e0568, - 0xf746fec622b90e3f, 0x29601624f5127ecb, 0xa1e70a78dfe92355, 0x2194998184ce234e, - 0x14e4404bc5efcd7b, 0x7ea02ab05aaedf2a, 0x425028517f46f92d, 0x01104cc534b76bd8, - 0x4f338ce48d793928, 0x293a0fd99bf1d97e, 0xb1ea1e69b178ce9a, 0xa4187b69fd274261, - 0x992b51c1d6f3be34, 0x9dc3655a10c468b7, 0xdbd6911903afd7c5, 0xe100670c46823e24, - 0x945942efa102aeb3, 0x57060b5b80a4fdfb, 0xff442806e7f43b6d, 0x1ee46a9ce512bec9, - 0xddc73216e54b0d05, 0xcc5809c94705ac37, 0x1140395f15871dac, 0x7c74fb167baa67d4, - 0x80309a17ca46821c, 0x753b22adeda2b8be, 0x55bc184648bf4f30, 0x7c64a2315180c771, - 0x62c8a9e4b2f1e313, 0xade18678d283fe2e, 0xdd2087206af3f73e, 0xead4fb5d2ac47ec1, - 0x18da9ec6388a15a7, 0x703b3f01f009c658, 0x9a89c825254ad7ee, 0xa35756332515aac0, - 0x5f7edb2f08f28518, 0x63ec6907bf887946, 0x59b0c9534cd1e6b5, 0xc4c23a8dc34f0b0c, - 0xfffc02afc5ae743f, 0x39655ef696be9f42, 0xb48afe1a91738d9c, 0x37d839098b71a0b5, - 0x6e95a63ae594f1e1, 0x424f13ca3592efc7, 0xcabaf8b84e6852c2, 0x5cfb09a4b596cd35, - 0x1b9c0efa08e7b3f7, 0x15c2f594554d0d55, 0x2b728dde0084a829, 0x4894e21d9e3145f2, - 0x88a25719270a56c7, 0xd830e9cc31ed6946, 0x9b9e60eff3fa1bbf, 0x7d12f1ff15d85622, - 0xca9e8e6407b602b3, 0x48c36da2fd34b907, 0xc7d638605b149cc9, 0xf419ac68a5d61702, - 0xc5fc7eb87c14ac45, 0xc43c592c501f6b10, 0xadabd24ceb218a98, 0xb3560794f78ad22e, - 0xfb111c7618deb415, 0xf00538d84873dc43, 0xd2de375aaf765963, 0xe6e28b7530f0e822, - 0x7f8499b5d2ea33f9, 0x4c74268a37c3081b, 0x12ada4f51e0a7cd6, 0x0c6f700778729f48, - 0x175c5d0981ddc678, 0xa1f38cb975798213, 0x4b19c49252c673ce, 0xf96e2d3bb5d0d64c, - 0x97801059fea55530, 0x40e2524699294634, 0x13cd9c5e049ac8ac, 0x942e2d839d75b9a3, - 0x51ce2f7d85fdc9ac, 0xfe7db2ccf18ee4f1, 0x213c7bac8526962d, 0x457459efe7b25af0, - 0x6679522d2e046f47, 0xda8558e59c459d2f, 0xaef9af97454433d2, 0x2b2b496a939800ca, - 0x95c722d00675a47a, 0x72c60bf6acfa2a10, 0x9065c75f8ea2659b, 0xf52304d93a3d8f53, - 0x9cc0e7e93bdd7c04, 0x2d4e27995eb0e80c, 0x9f70048eeebafbb5, 0x350fe852c2576b67, - 0xa58ddba50e923823, 0x90af9f956c024a17, 0xa73753eb9d548b65, 0x9b4f9a04e4844852, - 0x590e76585e01934e, 0x82afdfb5e36e426d, 0xd96be5219832f8ad, 0xf6ca328666663292, - 0x89b7c3b7f21bcb85, 0x24ba771d4ede3c6e, 0x86166a40aeed789e, 0x30d5570048c48b51, - 0x4c2aa77b5f774278, 0xecc84d80e7232c84, 0x235e9e55a8a79993, 0x996fc08d120187f3, - 0xce3e6dab771fbff6, 0x564cf64d225cc775, 0x082481b092face1d, 0x5c5d45972357b379, - 0xa90a04e1e0a6ef41, 0xd9a8ee400554e0e9, 0x5f66669b58579a2b, 0xf944e536f190e7ea, - 0x51523bdb9c5c96aa, 0x87a6e60cd7c6f194, 0xa33ecf59bb70592d, 0x06cebaa5178782bd, - 0x3f0cdcd108bd2c5b, 0x8268cc378d7609bb, 0x1aeaf0d3be06b98f, 0xdbcd50282707585a, - 0x3018c1f91d5441f0, 0xd080373678061b5d, 0xd96ea551138b21ad, 0xb45864cc46c9354c, - 0x2a11aedc93ca313a, 0x23b82fbc79237e14, 0xea8d120e22cf578c, 0xe62828d0f0377574, - 0xc650b2e087eea48a, 0x8b7c78d94e50a0da, 0x8c730263289ae67f, 0xa068cfefc52a9cd7, - 0xc8c407e04c8e31fd, 0xc1ba041ec488ff38, 0xdcd35790882f7f61, 0xa832eb985fe10d1f, - 0xcf13cff698fba59a, 0x0ac55194a57f8d9b, 0x6785563015bfb864, 0xae279e9f50ca2ee4, - 0xa1602398431a5654, 0xb958a106ea73ca4b, 0x720dc5b4829e9c52, 0xf1d5c74b398003b5, - 0xcdd3501ca2fc0848, 0xe8d7f062cc7baea6, 0xa4d33aeed07af0b9, 0xe2ba25df8c218fc2, - 0x40d2c08bff99c73e, 0x28f7c0d8fc6de66e, 0x8a670299e6b3a53c, 0xe5b872339b4aff82, - 0x1f71ef51abe8e79d, 0xbda4a46a4b56f5cc, 0xaad69d5b5325e79b, 0xb713cc4a90c68b79, - 0xe9c9386f79a6e833, 0x3ac59f2f3aa5fb69, 0x08d14735898a046f, 0xb59b6cd64b379266, - 0x2ccd376e33a3de1d, 0x1ad8509c4375fb04, 0x340fd8dda599ea43, 0xbef38a16f2effb65, - 0xe1e913c9bca57815, 0x705f5866e9813cb3, 0x0a6245b603a2dbe0, 0x8873f33ebc50b0e1, - 0xb85b10860af17b26, 0x0b0c4f3f26a09cc8, 0xa0f03154e647ef18, 0xf39fd72c3fdddd61, - 0x3ab04214f424840a, 0x02d738b97b432318, 0x1556ebf60fc1ad21, 0x4cb3d337e03155bb, - 0x39123b700d0bd59d, 0x601de96212c52eaf, 0x978de0bd51fcef40, 0xcbb2053519bb94bf, - 0x2d738ae9a8b93fba, 0xd64b7f24cba1e9de, 0x852dd487981891fd, 0x31e6b72e95941d87, - 0x85e63bd52b4caa34, 0xa7579e056353a4d0, 0x3dc3cd685e3a9417, 0x5c8fa490c04ef7b8, - 0x6cc6a846e3848110, 0x21e7ac9833e88188, 0x0cdd9d007147f3e0, 0x53cc2db7febf8843, - 0x0e2c13473995cd7f, 0x9c21b66317e1b034, 0x7fbb018a2ad96710, 0xc01126650331d17b, - 0x982c1988f436f67a, 0xb6d8c62e6fb6e884, 0xad15ce3734c73da1, 0x8b12c7fd557ddaa1, - 0x0260352e9a3e574b, 0x1d419ce76a2c9888, 0xf4f29cb804f96dbb, 0x0a18c43fd40e293c, - 0xc9d79c8b22377534, 0x34b6e729afeab15f, 0xb48676f39e2e57f9, 0x50ba0b8d9d312f9b, - 0x21df423437b3f70b, 0xff7a2cb3362b9104, 0x8df3297c49f4a49f, 0x4af26bcfece1fd36, - 0x8d5c33f5b6dfa570, 0x842c1753cc4b6cf8, 0xd56c19ea9ebb8d8e, 0x7a2f2bcf989e2c67, - 0xae0898b24b2de252, 0x5e4d82804feb159a, 0xd1645ba07bb76b31, 0x5cc6e425d166f721, - 0x3fe787236fafc16b, 0xe33eec70e262df3d, 0x8a7e0cb5db9a4e40, 0x0d83be7869143f79, - 0x16882626c377005e, 0x592ca0753e8ce4ae, 0xf79301ab468c62bf, 0xd5d393476472fc97, - 0xcb27d16af5a79249, 0x518c43db4972589e, 0xba0e01f1aaf10c52, 0x3be086b45378bdc6, - 0x446515092324314c, 0x1efc5dd78c46f5da, 0x1086a856f5f5ffca, 0x7802b5c71cc8bb97, - 0xbc181955ceb20c41, 0xa21a8cbf8d54c4a1, 0x46481c8682c0ab42, 0x7810f0262e766ffe, - 0x92e556967c6e2b87, 0xf14ee6d008625bfb, 0x3daeba4b9910d738, 0x5ad74b806914e958, - 0x71544e4e46b1d49a, 0xaf989938ad4a6953, 0x0f415e1c1023466f, 0x92a7f1f706c0081e, - 0x6bf441519dfca363, 0xc3bb09e8bb163112, 0xb1641475736d3306, 0xead5436f5bc6fa71, - 0xf4e6637cdd7f6237, 0x9a81da094c801452, 0x9935d3133070a151, 0x761653a527ba138d, - 0xc847c40bd343b75d, 0xc9fc46ed8b339d23, 0x77ba68b8640bd91e, 0xaeb5531dd794525f, - 0x5e6c240bfae79050, 0x93ef52936d8e8ecf, 0xc1dc8c960104544a, 0x1315846c38a28f28, - 0xe58b3b8e234218e6, 0xce8a06ea6f869b34, 0x89e015f4a4563bb5, 0x60b59181364aa5d8, - 0x436c7944e2feb2a8, 0x3975cdd411bbac23, 0xa0d8b13657801f56, 0x4e98f2ab64664c21, - 0x2ad76a8737f35649, 0x07e4db37d8f9e0d5, 0x9ceb7809ce56703e, 0x567d4c77f65ba72c, - 0x60c034e43db318ab, 0x871ff6da37c8580e, 0x889478b1a0c71e17, 0x1f343de1e7771c9c, - 0xeb13fe79fc6f8d6e, 0x6020b607e8f79343, 0xb84d26392404701a, 0xd2912f982f434f97, - 0xb73b06b011521287, 0x196cb4e081acde62, 0xa14f961f50cac97b, 0x2c2b8754b0808d9c, - 0xf9270ca3ba953bc3, 0xba6f50199c66f325, 0xf1a3a43ecbd5effd, 0x74be4e553a2cac55, - 0xd674c5334fb996b4, 0x7a7a51f9c2e07ea6, 0x66ee9db53ea67989, 0xe9d9e4d14f5093d7, - 0x18bf3dcf04d532a9, 0x4042826ad81fd635, 0x71d80c0e3790aba7, 0x9b2b525966c111f9, - 0x48f7d235b9a3c019, 0xb9d5de2a0de02732, 0xaf6c4de305006e7b, 0x462d8f0b92781def, - 0x462f5f0de669cb28, 0x8551995bb51347fb, 0x392faf06076bbbbb, 0x29ef642daf46a0f2, - 0x93e0b5051f0a361c, 0x3c642d3f3490c255, 0x479fbab342afbc08, 0xb3bd260f4e7647ca, - 0xf15bf68f9e9827ad, 0xda082eb7467edb82, 0xc7297517a4f92466, 0x019858adf6665064, - 0x25c3f6521f2d1953, 0xaf10275760a4a2a4, 0x2ab0a568447fc826, 0xaa028eeb8919d089, - 0xedb97f4d0326c3f1, 0x7ae990159b86f368, 0xc185d9a903c2fc3b, 0x2d5029a68a567805, - 0x2cc476b8cbfad6b6, 0x8624394338dec52e, 0x88ac69953a09849c, 0xe97dc171637c5546, - 0xf99a4086fef67bc3, 0x3a350abeb59bc215, 0xff18ea4ae3663f69, 0xfa75f329654fe2ac, - 0x76c1a4eadd7ae9d4, 0xbdd64d8f3b5aa482, 0x129868d91212af2d, 0x012e387d6a168c1a, - 0x830fdb67468b411f, 0xb91454392a78666e, 0x87795fb959660ede, 0x24bca94f7b7992e8, - 0x80c3f471d6834896, 0xed42811a08dbd3dd, 0x44891be918e30aae, 0x50cb8027f01702f5, - 0x3f7fef1db984a52f, 0x9a90140dea0c0c62, 0x1cc59d2dfe140abd, 0xfeae9cc1710d25ff, - 0x98d7359596662e2b, 0xa9feb58ab1fb8acf, 0xc611ed41c45af4c4, 0xdf8a84b7ee8b0bb9, - 0xa5c231b08b347096, 0xe09d05b7872ed7fb, 0x4dc074ed7c595aa5, 0x6a9fa1a8a0ea72da, - 0xca2988916e23112d, 0x81ec0732a4fa7f94, 0xa33b11bce96c81cd, 0xe2745c04857e8f2f, - 0xf600a8b2ac798498, 0x778d7e145974c51e, 0x609bb5da371925f5, 0xc5452b018ec72ae7, - 0x2b62af148890eb11, 0x2a26b96929d2dc4e, 0x3bbb95336b3ab317, 0x7636bdbf8e5ba028, - 0x6edce23e6a994192, 0xb4a9ad5d60882065, 0xbbe1e21cd6b214c6, 0x9853b71039c77d22, - 0xa270f29f0fa8bdd3, 0x208b04ceff205558, 0xeb969b6d6747c7f4, 0x2239e9c30648be2a, - 0xd3e0bea65f00463d, 0x16ccfa5017574073, 0x58c98bc20b9f3bc8, 0xdf9e9bcbce9ead05, - 0x013000d67c66dd8c, 0xa59c04ed3366d772, 0xcf81fa1968d6cfb4, 0xf2099e95756ef332, - 0xfd291ec48a6171e8, 0x5349d8dbc2c4bfc8, 0x5d211fa98c96cf96, 0xefe798f524138ba0, - 0xca540fce28a63ba3, 0x40f8bf14964c6294, 0xa8af392b73954beb, 0x3fe7099b06334f6e, - 0x804511335a403430, 0xc73ef3705ad6872a, 0x9e71b7137325c13f, 0x5019e321dcb51420, - 0xb432fa57d756cd3d, 0x2b113a42a29c1124, 0xeaa2252b3d6a9f62, 0x75cbdd29e115fb0b, - 0x30a8c6a7c4e7f158, 0xc7d331ddb4f4d608, 0x233cd0224c5de545, 0x47b919f94fcb13fb, - 0xb455b45dff711a97, 0xc6cdee12f4a73224, 0xc5ac0179d6a5743f, 0x204643bbae0b27d5, - 0xcbe67edc6f15f8ad, 0xf777a811a9d93e24, 0xa6b4c2867d4fde9e, 0x50023795ad369e5f, - 0xd9be587fe5f375ac, 0x9e7622651cd1bbe4, 0x951fabd73a500e61, 0x6bcde4c8e2be3531, - 0xe2945dd6e99ef02e, 0x5ba4d61cc73dd774, 0xf137e8ae95b949fb, 0xb38ad2611487f485, - 0xcb052b70995c195b, 0x450f770bb9869a37, 0xe08f01089abe7af2, 0xfd0042781137a61e, - 0x12ace0d979cef5cf, 0x7428ce6f9d38f11e, 0xfd8725deb3134c6e, 0x3ec0da6ac78e277f, - 0x4e3315ccf1a07759, 0xe41d4581793218e3, 0xf275c01d6d660d21, 0x682563f05091551c, - 0x39f82dd7fbdd9e6d, 0x9457bf9198f782cc, 0x8eee65e567a82c66, 0x413a5d226c6fe4eb, - 0xcab2388de3afae0c, 0xdc5c0218f33f7c7a, 0xc35ac1651408378e, 0xfccc79e4653f0ba1, - 0x74b8d979e8808003, 0x1774591b6b85ca99, 0xba01ef1c2200475f, 0x9ccf66e4f718dfe9, - 0x2400924abf23a9b8, 0x87a66ac38eef1ffa, 0xe6c3bd007b64035b, 0x4af7e8b4909c4015, - 0xdbbff4e9fe8b29f6, 0xe5e8d1d5fdddc4ee, 0x2a0f9a4f1f38f47e, 0x68f19b5f466dbc8a, - 0x3966fc2f49af067c, 0xf33a5ab59b9b162a, 0x6ecbd6eb857080b3, 0xccd2e92338f8e731, - 0xe9906cfed6fcd007, 0x870ca3dd4056404d, 0xb662691afb3c2be8, 0x89c33e1c166e5951, - 0x4625749d10cac785, 0xf8944fbf61e59285, 0xd9a7c321b62884be, 0x16bbf8da1ab78fec, - 0x67144c63179830d0, 0xd9aa6781cd3cfe01, 0xbc01f7feb103730a, 0xd11433638ec32752, - 0x938e628dfc459b49, 0x38249a316f5eaae5, 0x1072a30608eb6270, 0xe4235d57a326a411, - 0x00780f87b08af768, 0xf78fc275ff534351, 0xa3ad1f43cf5fbf0d, 0x03a0cbefe0002e61, - 0xfb68776b1b48a894, 0x4aa5c16969deebfa, 0x5c1872eaa846b645, 0x44aac0e88980652f, - 0x3fef962fbe4c12e7, 0xfb468b875ba88bc3, 0x3d83a4ce4655eab5, 0x8b965a68a963159a, - 0xebb0bfcef035a2ad, 0x217f004472e36321, 0x1d07cca6cde5c9f7, 0x7bb7cae9eb566113, - 0xea0869b2472308be, 0x0a3eebe96bfee9fd, 0x2f105f54e8c93133, 0xe4e40f0b5b476708, - 0xb803689d3871504b, 0x3863f72b37302cfb, 0x7f1b85502f27d4b7, 0x1a0cf302379bebec, - 0xabe7211b23ec7f7a, 0xc01bf7e1278265f7, 0xa09bf2071e30a5e4, 0x9dff886c1c0c3633, - 0xab8896cdda4e06df, 0x0eef4eb73b1196f7, 0xf297a71ab86f0836, 0xc0f3e3094da0594c, - 0x9ef19f94a25a169b, 0x5aa90c72990d6ded, 0xf6ce9542bbdfa107, 0xb1db4902a1f74dc5, - 0xcbebf81a2d72f217, 0xbc253cfb83aadcf1, 0x64a859a56ce69cea, 0x282020b595b67d95, - 0x0e31602364f0a5e0, 0x5f58dd640c86894e, 0x06c74dded9fbe184, 0xa1c2ebc141dc6d25, - 0xe2f85b316ee29287, 0xab81a1818c10c578, 0xa3e54718b9eaf9bd, 0x5c497210acb3d1ae, - 0x3a20377aff1cc1e8, 0x51824544f9965ff5, 0x9858ed5bd9b04e31, 0xf108da5837e085c3, - 0x1ccb1839cc9346c2, 0xcfecb428bb384437, 0x87e5c9075c968464, 0x150397c168f47270, - 0xa17dc0382e417be6, 0xd7c8abbc59383711, 0xfc3b1e35f8819750, 0x09bac79c89a067e7, - 0xb73c170284d50b2a, 0xbee697a9eb976a5e, 0xd12319ec233626ca, 0x717ad4c33a129008, - 0x6a3521f2039b54f1, 0x8e086bc86dc8f1dd, 0x0c189baea4261f64, 0xc1e43cda92632e1e, - 0xddad9e0604196dd3, 0xa2a96c6a200c1263, 0x5c855aa2b489b239, 0xb0cbf1c664da5f4c, - 0xf93ffc5e110dae48, 0xac89842612e0bc06, 0x50903c4b612ccdbf, 0x5b1d64322a7ebf82, - 0x47cafa814b146597, 0x9c95eb3c56e09d24, 0xebcff8184b9494e0, 0x7758adf4357f51e5, - 0x9c00f511e5623e29, 0xf216ec2b606c713f, 0xa80bf74de3560d9e, 0x3b5481dccd13442e, - 0x915755287cfae4ff, 0x2170acca33fbcc9d, 0x9c67b4c53ec86779, 0x7ac286ba9548db82, - 0x795989deaf3236d3, 0x6f167d6d8cf81e3f, 0xbdadedaa3e46eb94, 0x88f7655ba658e213, - 0xe530594c8fe113bd, 0xfcf449cc1781f0c8, 0x376b3ac6a83885a5, 0xefb13c5698e83bca, - 0x6a833af63c17201b, 0x05724137cac030c7, 0x8da81a8f90b32228, 0x9cb062a0788e8315, - 0x2ae3f1cacfa393c3, 0xc453cc041335c05d, 0xd76fd7194cd9aa4f, 0x363929586c4cbc9a, - 0x5566b1352ca7a321, 0x611a013e87ebfe6d, 0x4028b352b49c1ac7, 0xe667c72a01b80116, - 0xd789c93bd50bc8b3, 0x4bc22f71930f68d5, 0x9777687fdf4c5442, 0x7a04400dc8bdb759, - 0xa9e8bb815b828078, 0xaa4e565dbd185f30, 0x9fe3d9a92414be9d, 0xeda72d70f6ca51b4, - 0x93b60ccc41a46105, 0x6249b5ff82c04cfd, 0x1c8a8eb273d864a1, 0xe1e6923651139fd1, - 0xfa918c04ed051cf4, 0xa87c658b4a189834, 0xaf336f186ae81986, 0x6f1526aa5e343a1a, - 0xd70a45378d31beff, 0x6af6c73c1470bcf0, 0xee9483c22da542bb, 0xd55eaf0399a18d04, - 0xeb1b0f52ec5b02f3, 0x5e0627f42f29a026, 0x0072b9917409e0be, 0x1a344e5987936eea, - 0x254db4b5a2000345, 0x106ae764ea6840e8, 0xbade868e80f26745, 0x056978bf063f3f8c, - 0x5bd19f09b63802bf, 0x57ce6b2337ad4745, 0xc6455c4dfac81779, 0x9b632e760dbcefd1, - 0xaa9c311c27ea334e, 0xc567d1c14e4a45e8, 0xee98198895d822fa, 0x8b82cace3f6fbbc8, - 0x304571898c12f9d4, 0x207ae49ccfbae74a, 0xc4cc8d50eeda6ec4, 0x7c1439f6be9aefae, - 0x941d7fe0134d3eb9, 0x4a7e69ec94e93e2a, 0xd871360e79262860, 0x3afcbd8201c221c3, - 0x441d549e31e1d3be, 0xfd270e427322177d, 0x178a442c733c231c, 0x404a23e3ab1ec656, - 0x883932ef89cd4cee, 0x07710e29a2be30bf, 0x7e285d52e45ec391, 0x0f37d01edada0675, - 0xbefaded6ac57b0e9, 0xec7168306271d292, 0x86b9a9a71aba7885, 0xe6a0156074600a08, - 0x91c6eff4329a1e52, 0x9fd277a1d17737ac, 0x5b5fa2ce6f2824bc, 0x8f5a05840c3c6599, - 0x738a58a78c96ab00, 0xa6e510a26a60511a, 0xe86dd6420af33f09, 0x4c2e99506072a91a, - 0xe57f62dda8f3be29, 0x8c6949816c564349, 0x3ad39819aba970c2, 0xf10531a4e679a0bb, - 0x1b0d7e9eb5a80a45, 0xf2aa167377196d6e, 0x85ce596e4b33721c, 0x81c63aa0c23d3622, - 0x3d7a24e4773a57a0, 0x6004759d2a680ce2, 0x5cf28c2d0b9559ce, 0x74857bdbd8fcb32b, - 0xfe33145ac8f62984, 0x8847df9dd9435435, 0x4ba9321fba8db826, 0xbd67fc90537277f0, - 0x666e2ba8123c4a57, 0x65991d339df9a8f3, 0x8d23d11334eb5bce, 0x615061690d0442a3, - 0x17c2fef286a7e44a, 0x461e51673c59a401, 0x2680326ab93d7a8e, 0xf6bd26937eb21f01, - 0xe270972890df2902, 0x31bc4acab8ebd209, 0xc084403d249f07c7, 0xd652c8dcc1de5e1b, - 0xff84d6de85862e56, 0xe50c4e7331b20661, 0xee183b6e4924523b, 0x22602531439be3dc, - 0x5b7be93d4daa06d8, 0xc1fe109c1fd7bab7, 0x2532a0096f5019ce, 0xc123fadde78f0b7b, - 0x8690c3dc9b19f91d, 0xa2ababeeab1aaf0d, 0xf2ba9b2718c668ac, 0xb654f36acf14a645, - 0xfed0d9b07d636501, 0x8c973e24f4f992c3, 0xcde2f745b5e60cc3, 0x2b6fca4781f1609f, - 0x86b054ddd476d3d0, 0x52df8d5a79e88877, 0x88c3e4ed88d7da62, 0x527264be33bc44d9, - 0xf6266a46f4e95601, 0x2a005714f658f562, 0xc2fe239b4301ee75, 0x38d5961a52c2f040, - 0x55424b3aeb6fefa0, 0xb0d20f7a57fe433b, 0x03b2fef3f4258e1d, 0x1a0f7ebcf87958bb, - 0xfd11ee47ca2d2a92, 0x6a3dd7cf0e863628, 0x5ad0283cc148db42, 0x0fff5f84446d2bc7, - 0x24d778b7d75d62f0, 0x97b34e5698dd81e5, 0x5911591b95e47fe8, 0xffaaafadf0c5b64c, - 0xc161f9a3f3daacce, 0x47d2373e0d209b30, 0xd7f85656425282d8, 0x6efff0f4ebef1e2f, - 0xb20ce2642b848d31, 0x0966443ba1d39f22, 0x681417e4cadf69d7, 0x12c04e3b6aeeb235, - 0xdc3dc3520b16648d, 0x8d7d6c1925195036, 0x6aefbe41161d6942, 0x345245cc2100c197, - 0xd9b495b6703a6a57, 0x5f00a657793c623c, 0xae69182e67a2b812, 0x48cfaf64f2f005f8, - 0x0cc27936aaf06801, 0xa1e54886e97210b9, 0x7443f6df088846fe, 0x3a7951f3422a23d3, - 0xbe75e47aedc62b57, 0xbd7902817290e1c1, 0x967d6e81eeaa8243, 0x6ed3d2f58fda0a13, - 0x2f0442203ede1718, 0xac3fca312c2fc966, 0x1607bc700caf16f0, 0x738eada17da464f1, - 0xbd7b9846ab0f3b78, 0xf0b6b4236542215c, 0x48c06b368a98cb8c, 0xae17c7588988c7f1, - 0xe56b7d95c32feb28, 0xb8721b3621431dcc, 0xd1bfa46da7e7c36f, 0xe77c4d105a0c6b38, - 0x6f73afb2a1f27d7d, 0x5dc4f91da2759afc, 0x8e076b75f87638ed, 0xefe00cf0d03bde9c, - 0x133118ad38ef980b, 0x12be809f336814cf, 0x810aab1e45906d5b, 0xebe3d94e29c9ae5a, - 0x925ab5f2122348c9, 0x1faf6935db1b9a77, 0x51bc6b2573dbf63e, 0xa95b0a19f1b85e8c, - 0x8de23b643bc89c14, 0xa3f97d11c798dd9c, 0x8837a69769cd8d4f, 0xa47244ca72ff7b87, - 0xb0c533e4b7f23547, 0xc2fbce36a6dd771e, 0xbfb69023ed016f66, 0x2e4d36714fe56a99, - 0x1d85fe2a612b473f, 0xfbe59dcda5367967, 0xccb433d89c80c8bf, 0x8dce6744a5655fbb, - 0xc24ebc0175fefe51, 0x5eb8c54a47af22a8, 0xafb5cecf089ea7f8, 0xdc5870c918ccdc7c, - 0x33a6f27fdc4a3206, 0x41d73a75dc06ca4d, 0x6a6419e96da71205, 0x999c37b930dbfbb3, - 0x2b5ebf2230defe35, 0x151efe74b36e65e5, 0x1ccc34955ced049e, 0xaecfb297e997f82d, - 0x08ec9edf80aac99c, 0xa129dcfa283ca7ef, 0x96a489b83cc51a46, 0x7cf683d1f1df3917, - 0x8266b2db02e21255, 0x3c321e24a0b43ee2, 0x8912a5d8dc9533b3, 0x6d9f6fc7bda39eb6, - 0x1913d44b57537c94, 0xecd321fdb94463f0, 0xb4c49ef3e89c3a11, 0xef9d8347f0aec1f5, - 0xc43bbdc365a157cb, 0x9541f01e569592e3, 0x733e11103925c4e4, 0xebb901892bac587e, - 0x229ddba738ee2980, 0xb5d00bd68626c32c, 0xd3c5a60917d7121f, 0xa72583c7cf2c9f43, - 0x6d3a923ab1f1c1ff, 0xa2a6c2e8786dd943, 0x19e7e9e595d42e4b, 0x8c60eb8c3108e9db, - 0x604d0bcf5c6ca2d2, 0x0d340289428e00a9, 0xcfd14852ebb17ea5, 0xaa94358ffb0785fa, - 0xa0fb2e7588df81fd, 0x2a4ac67f1d2e0211, 0x1156548a61b82c4d, 0xc869c80c5d260cf9, - 0xb8a54a7ee011f4fa, 0xdbbb48a83a5d1789, 0x52350c1540598d23, 0x436636f81f1fd918, - 0xaecf44bd28560801, 0x70c655c498590a0d, 0x6dd16acfeba7b34c, 0x97036ed7039e7c07, - 0xe05d04bad530f434, 0x2141cf07b90b3fe8, 0x8f419170f3fe1fcb, 0xcffccf4cbbec96b3, - 0x35f30af59e2b2197, 0x01e79f4ec45694b2, 0x07113bf5b50ce4f0, 0x47d9dc7f6714609e, - 0x07bdbe8a10e443ed, 0x037bb1a5b6a4005c, 0x68aff22405e75a31, 0xdf097228806c3b73, - 0x867077462b3da42d, 0x411c7842a75276f2, 0x4a2dfe7037277513, 0x0d784416cf160289, - 0x27dfa461a3ad0f03, 0xadc9a372c07b485d, 0xe779f986d748b7ea, 0xa7480dca02e17eb9, - 0x419fd3e9b8171ac3, 0x4d624dd7f5fc148e, 0x0764590377883ca0, 0x3c2fe4169dbc621d, - 0x64e43b21ebf82f89, 0x63440f1c0fadd3fb, 0x2361039510a8f7b3, 0xb9e596c451c66da1, - 0xef9842b445616147, 0x28565c2b425adaf8, 0x476faa5655e7d6a7, 0xa942d36ec15b5e43, - 0x894cc7435ef42a79, 0xef00f69cb29c4821, 0x2497e5478b49b136, 0x80ba2da630cc64c9, - 0xfbafa22eebe4772f, 0x7dc412736a90d624, 0xea644e10747778c8, 0xad43a4cdf251b173, - 0xd054e339036e4d41, 0xbc3965a4a0bee885, 0xa03c7ba47833c97c, 0x4b7426a984e07672, - 0x93c9c1ae88b854f3, 0x32d2e2ab62074dee, 0x96caa408f02f1fd7, 0x7b0a4e7c3d48eb62, - 0xaa810b23a523a784, 0xce9e4d3a997e4411, 0xa87ce6f88219ad25, 0x96174a23ca722a1b, - 0x1561b090f973dfea, 0xa8d751f3dcef8767, 0xecb93f2c57abeb24, 0x84149e36ce67b89a, - 0xd462fed031879b0d, 0x2e0d4ba18bd83a83, 0xabf559b1e9fe0170, 0x3e81199be544c317, - 0x8ca4dd143ea7b816, 0x76f1bad0f3c1718b, 0x09de81f7e31e1881, 0x4a838c103a69fc41, - 0x59a9dfa77936e114, 0xe9ec59621ddf78e6, 0xbd95469f382eca4d, 0xbb35d28a223d9934, - 0x90d097b9e4522328, 0xf11aa0c380119db9, 0x236801b632c2b433, 0xe3b3f41dcbb6eaa7, - 0x3cac132c4836a003, 0x7653656c93f27513, 0xe2f2bfdce77c1e7c, 0x82663f4251310735, - 0x175aed3063345fac, 0x1ff308eb7341156a, 0x93d59783d255facc, 0x37a9cfca45527ef1, - 0x5a57662af3532dcd, 0x9b2544dd8bb019db, 0x5d1b76bf9402bd1a, 0xa850a5a8710a17ca, - 0xe753f1c6abc25522, 0x39427e829e0f7563, 0x56c23dd3dc33c949, 0xc9aea9da238a1a1b, - 0x03c8c80a44dc67f3, 0x2a47dd74be77156f, 0xcb1a0120aa3b2fb0, 0x59bcde83f3e3170a, - 0xd048ba32815fb5a3, 0xa7a786b4ac4c51c3, 0xf95185b7be4b8845, 0x62af94a9a031cfe6, - 0x77432b283aa64dfb, 0xf2e0da1c22e229c0, 0x418af392252e880e, 0xbf341dbf1411dab6, - 0x59b45922a0be16b6, 0xbcd101d35d5f4534, 0x15ee40528a0efeb1, 0x4902f215fd29fa03, - 0x79bae6018464a479, 0xf237bfa1b01ef781, 0xd4bc4215245b2099, 0xc4bcb8434da0633f, - 0xe4b9b46ff68b1a63, 0x084cc4fa7b478bba, 0xdbfb43c9bb90553d, 0x1854086acdba3401, - 0x542b0d4f457ceeee, 0x5f6d380159097c14, 0xab9d0f613972e9a4, 0xb120304b2124b87e, - 0xc1280d6a97715bb4, 0x8de688679e896d90, 0xd75679d8ec457f22, 0xe8d44f6bc287f4c1, - 0x31e0cfb19b6625ea, 0x19398226f07eee92, 0x971454006b90ab0b, 0xaa6ca3b1121bc1ad, - 0x50742e33f11e90de, 0xed74827777f48938, 0x9d72745dcda9bac8, 0x03134a4b6b73fe39, - 0x21d17531ac6438b9, 0x45874a1ad24d4d22, 0x3f85c291154c167c, 0xfe2600ab738d7164, - 0x7fa99054e893c95c, 0xd2c1f6dc4a75184c, 0x92b546af4dd45a0c, 0xfc5e4162c5216cd9, - 0x6018887a88b20581, 0xbac5a1c2fdd3e73f, 0x5063f7655ebb54bd, 0x62f9658bc17b53ae, - 0xed2c4de8a59b191e, 0x5bc716e4383cb96f, 0x37c5831ebeedf433, 0xe8ea1d41b665c4e1, - 0x793280e3c5856bf3, 0x1a55d37cc6538e3a, 0x8a69495d1e111284, 0xa3d27d793d2bd7ed, - 0xb41e9754102fa2de, 0x58104f50dffa7179, 0x96d8546843064086, 0xf37b1e3bea76822b, - 0xdcc671705b5a3cb7, 0x05888dd91458fc01, 0x95ecb161f00150e5, 0x198603b64ce39e13, - 0x141175241f14d7d4, 0x8113bb902dfd9bb6, 0x870f48adb2db9bfa, 0x0f96a2b79946a3c2, - 0x72bd4a0d6d90ff3e, 0x272b928d92f77bef, 0x0a0b1da5195ff2ab, 0xa42bf43e005a3adc, - 0xd72887c317245c97, 0x8ff412407b7b7d38, 0x6dc1366b9704baf9, 0x5ab28781b44a6bb0, - 0xde08e8afe0705271, 0x59018ec8dd107c6e, 0x23127b86eb5933bb, 0x8f252c9dea513d89, - 0xf94f8cd8a0ef16ff, 0x8571488fa8dad6d7, 0xa6332ac371661b12, 0xf5759fa2a32cc373, - 0x0864a02eb4ffb7a2, 0x443f50620d13f422, 0xb66470be66af826f, 0x119c6488308860e2, - 0x31b5cd3b00e70b2a, 0xf841fbf485beec57, 0xce3bf99668978201, 0x3cdbe54b9ca532e7, - 0x50f49494ad43b1b3, 0x7d9842ce1e21a6ab, 0xf1b45f142045f362, 0xb09aef5995c3bbe1, - 0x55ab89a5343cbb21, 0xf3d943413fb2f9af, 0x9888dccb63910725, 0x5f0e316a1065b815, - 0x1630355cb0022f67, 0xa8f59ee27feb08dc, 0x61cb94f0b691057f, 0x36bc72b86f00fe53, - 0xf17c409293e5111b, 0xd178ca13f655a335, 0xe8000f1a0b9f5b6f, 0x5dca46cf1f5345f0, - 0xa77e941a0c54d2b8, 0xae3efaafb6864fb2, 0x3f160f7ad7e9ae32, 0xa9537aca0d491c33, - 0x624834ac98396881, 0x441f0a519925c876, 0x21ff13474ab1240b, 0x706fd5d80ac1e265, - 0xc901ddda038f6cdf, 0xcddd7e37fa743dd7, 0x5dc714a99a8cd7fc, 0x04eaed7bcef1cae1, - 0xf5c924a8b85e2281, 0x402f82058e6b3ed0, 0x0f6dfc077fa73dcf, 0x2400a8f2363a6eb8, - 0x560e936e0ae1406b, 0xa2f0a82123dd2991, 0x100f2d8af55ef752, 0xcc6f89ae83639721, - 0xd6ef1ee313d82bc1, 0x4d81954eee39c9e8, 0xaabfa24191fe7a64, 0x27c0489d3d7efd01, - 0x0a5286eb6f48e334, 0x0285746d50ccfac9, 0x3f2b14533d368cc0, 0x98d7e0f13e795f12, - 0xf22335de050eabaf, 0x3878e7e4bc0c9c16, 0x64c93d1575ea40e4, 0xd42f6a2b3e5ee487, - 0x72b4e0919983e856, 0x0b5ac7c7945130d2, 0x979b6349e6b70958, 0x49bf5c82b2ce2072, - 0x90b080aadcb16cd6, 0x4c7de5706a37ff44, 0x358d3cf65f3b00c9, 0xa715eb3d34e420a4, - 0x7a2eab13cc2bc678, 0x3c9bc5460cd01b7d, 0xba798cfb829221e4, 0xfc57f3e3bfa2073f, - 0xad57dbda53fa602b, 0x456f395c7c207f28, 0xff3b4c90075b86df, 0x1c45e5898dd93a31, - 0x529b16ea2501122e, 0x9d3c6cc7b807e973, 0xa8412dbdbf54f759, 0xda40f05294ec2e55, - 0x188322a73bebc1a4, 0x99e810391c32dda8, 0xf9c0764846f1d4ef, 0x689e48dac3b3436a, - 0x83ca1caea8974592, 0x6fad0a8c722591da, 0x5f02683e25206562, 0xffbdb93a6f2d1c8f, - 0x2463e3e9d63557d3, 0x3d9f04100f8dff7f, 0x570c531c7dc45b67, 0x44a155fc3684d3b7, - 0x5b29cf8b1be60fad, 0x3d4105dceb36b404, 0xb4f81cfbd989e84d, 0x47b91e75d666ab26, - 0x24171bec298da039, 0x84690ec930b9c1c2, 0x65bc55548d5c8f0f, 0xaafd01edd529dc0f, - 0x44c2cd3b5579b1a7, 0x99a289c9c8883f77, 0xf11fa0a0ef303c10, 0xc14fdaf40717c61c, - 0xd170effc6b25cb24, 0x727115dbd9990a2f, 0x3cdfa13faff6ee40, 0x22ca00dd105e8479, - 0xf3be7c10919fbf35, 0xa3894d7cadfafc08, 0x7536dc915f4b2b8b, 0x2458e60e6228a67d, - 0xfd4caa760b4ae4a9, 0x4a9da57d176df538, 0xabc592522a506edb, 0x4ef92c271de4e711, - 0xd6db0527d8b7497d, 0xb7e5619ddf8b7cc7, 0xf893d0a79b3388a5, 0x852839ad38bd17b9, - 0xf09baa3db12d57bc, 0xd9fd9a0427305bae, 0x7867442dea63d046, 0x6a6d074561ccb972, - 0x84a33479af072b54, 0xbf4330c6b6c536fc, 0x1a1110dceb22a162, 0x8911171d1f17e5ed, - 0xfe1e9455d6aab5df, 0xeedb0cb236a1dfb3, 0xea0654ef9fb0561b, 0x31ea999bcad1e537, - 0x4243301692fb8809, 0x956d022e5ea8bfbd, 0x671f3a9ef085d17a, 0x15c521b2856c0893, - 0xc6408fffa0dd5cca, 0x3ee1546afeeff3e7, 0xba43a1ecad1bba77, 0x046d26f414200eee, - 0x8feb68803849c340, 0xa4d39e3c27251ca7, 0x5e3d493642c8a144, 0x43af7a535bc4bba4, - 0x01b5e19b3fa1ee19, 0x576138cae63535d4, 0xd96c524e3a7da081, 0x964fe5fa3eee22e2, - 0x48df1f0d0e6bedbd, 0xb5f3d58460298bb4, 0x83811186c9c83622, 0x9d80d55e653a5331, - 0xbfa91f8b5584a441, 0xf5eb02a63157dcec, 0x8f4c4933113fa951, 0x560537f4432a7c83, - 0x25b21ef54b733ba2, 0x8530feb4edc6fb40, 0xeac3a93d3e044a84, 0xf501bf5468cd10f4, - 0x14b643b45e7afb09, 0xaefb2c65a36190a5, 0xcb6cf016dcb43f37, 0x1d240dcaceed4221, - 0xe1b4dd3e7696dab9, 0xa8b49fa0ec692eed, 0xef36e5e4502ab5cd, 0x252690d0ae40b446, - 0x8797b9745fae913e, 0x825dcddd5a8334c0, 0x273b649db916f10b, 0xc11197b0dd3609a4, - 0x4cabf16659ad19e4, 0x41add448acd1e1a8, 0x80452a199e21b1de, 0x92635f9c0d6ae456, - 0x0f8df04221136585, 0xf428c139e59a5182, 0x63f5e0b28bf2f8f4, 0x630d690c5aac3329, - 0x73f08bed429e4299, 0x02554ed252f82384, 0x004fff81e33e9e16, 0xf74dc8c0cb9ab691, - 0xb87985276c0d8518, 0x6d5a7a703b7e7298, 0xa93703ee90107d81, 0x8b0bc78b3c6d96ab, - 0x72c0ed04ab0edba9, 0x0a83c83b951a9324, 0xd6aed0b33f199f21, 0x1f0fc281858a0050, - 0x97f3e095d91f0565, 0xa0ed66ac8290940e, 0x755615e192c53c62, 0xf64823284b6f834e, - 0x98346468e242bb34, 0x7712f37e43dfda84, 0xd6fe3430ceb08147, 0x6fb5da0dadf486a1, - 0x463d1e72b3212f49, 0xbb98cbb405e63310, 0x8ec705d7dd705f56, 0x5138dea07ba88b33, - 0xe9e46a6f8290db2b, 0x36cc127a16df4e32, 0x9ca3470be2a2b660, 0xad6dc96260def45c, - 0xe95955e6ec5c44cd, 0x0f4d552091cd390f, 0xc4ea772bb6485cd1, 0xe3f89c7ce7c03690, - 0x18816e8b000830f2, 0xce359fed58527838, 0xb597ed19147aaaf1, 0x52fdf464f4869019, - 0x57b6739453ab94f3, 0x7c2d0f82db20bb05, 0x540553d1c449c0fc, 0x2cee868941b8d123, - 0x89712724e4039b6f, 0xf3549a13bb901422, 0x2c521411bfda2af4, 0x96fa1cb8012b97e8, - 0x0a95024c889edf44, 0x0a326b93261bbd00, 0xc07bd4f2fb23bec7, 0x8c1eb9cbe538f8e3, - 0xe629abdf34628234, 0x793a035e035f5138, 0xb5bb1bc01d82c3e4, 0x2b70aa2ca15746b1, - 0xe23887bc9951e0e8, 0xad6f79dfe0a696b4, 0x2656e0edfaffa782, 0xee736f3b2e489357, - 0xdb49a39e52130c89, 0x0ad26c18ecfc7b4c, 0xd40a256acbefcb33, 0x2bb471024ccf7c2c, - 0x7a622d18fab838ee, 0x86efaa814af51d8c, 0x3a7fe61d15be1d36, 0x90c1539a03dd5116, - 0xaba53e9224fd32d4, 0x11f91a35db9e4937, 0x620eb41a74fbbd5d, 0x2a89926d35bb92fd, - 0xe4d6663b2a8a1d00, 0xb62033a32c85e6e9, 0x2e65df1fe455e21f, 0x2c9b9225f1ca50ff, - 0xc089949dc0c1dd72, 0x799b0005ad8777bb, 0xe126b515229b7712, 0x93a30c959b87b378, - 0x674b93edf00c0572, 0x7ed09379591912ed, 0x98bc9e9681543d23, 0x3bf27adf8e323188, - 0x48162b4801ce3384, 0x49056d2eae8031df, 0xd6d2c4e97aee04a2, 0xc8bca618e0669b6b, - 0x206182e924002003, 0xcdd419c25f0fbdba, 0x1b17a8e96490ef09, 0x151814cae7112db0, - 0xffbdd5629276638e, 0x191cc6f20378a4c3, 0x7c28f3f2fadd0eee, 0xcafc3c90b15cf7fb, - 0x2f6df0f71df982e2, 0xaa2e167f56fcc98e, 0xb25a5ea76556c8b9, 0xe9de9325f6211192, - 0x25133961111d78cb, 0xeaa99f5860db3c04, 0x8116d92c2d550408, 0x3aad86e1bf181e1a, - 0xce2f6cbe29dcf264, 0xb21c215da0325a41, 0x96d633cbee22aa1a, 0xd224114a4181542e, - 0xab0456c282c28700, 0x9a311dfd4b40c44f, 0x8004f995e800793a, 0xd7553b24cb8076a7, - 0xe0e061893ffa0ec3, 0x274c0bba1f246526, 0xaf455bf108633658, 0x9de65a5275eaa4ab, - 0x27ddc024c23f97cf, 0xdd95480eddfb8bd0, 0xfa969572a3783005, 0xdd54926fea5feabe, - 0x6a653b74f01154ab, 0xef8fdba5c2fb9e75, 0x5c397a1236d68f3e, 0x8bb1637e3831a237, - 0x3493a4903ea6184e, 0x6c7a9ddd98929ec7, 0x67329437a791e2ef, 0xbf030f1f678f29ae, - 0x102de7adc5d41b37, 0x551bcd6904371b22, 0x5f81f3b29d304a4d, 0x3b88baf0c43f0f76, - 0x71075cf9526c6004, 0x5ce708d5d157f1bb, 0xceb5116bdecbed2e, 0xd35f93be9383c047, - 0xff0382550b9dc9a6, 0x8a431cf71f4f9d7b, 0xf0a683582281d679, 0x8cb5e8c8f738dc0c, - 0x6c37e7c50b1f9483, 0x39e49236e293738b, 0x669d4a9a206297c4, 0xeee562a4cb6dc677, - 0x8c77dba71c1bd14b, 0xd768bd1b76e48502, 0x041a1795f67ddc23, 0x2e337811c1573e29, - 0x1b0687c5b19e5d61, 0xa8c7e68a2c63ca36, 0x6b15c8bb64d89e01, 0x4e254fb2751cdeec, - 0xce3a13c5eb93ea8e, 0x26b10d71bc47a991, 0x61297fbbe1c360e7, 0xb22f4fe57a756f6d, - 0x8d7ef60778d1c065, 0x7c38b5c47ee06d68, 0xd4ec2e2ac5a4a24b, 0xde463dd6aeeff944, - 0x113ad8ab37f91258, 0x629dcbfc65555e0c, 0x377eb2d6f531de8e, 0x137f4d8ab8c5dcce, - 0xdaf4aebd8e3de00a, 0xe9f0cff67f06a67f, 0xccff83896072e3d8, 0xa63ca4aeeb1d4d34, - 0x356a4e89d93cf950, 0x2d2ebf3c785d290c, 0xf9fa8d731e1a2d95, 0x74011c34e4c59f53, - 0xc657c09a2661a0e3, 0xf5cdf9489c8df558, 0x4258b18673921c7a, 0xad75ea27eb35c7f8, - 0xfa53aba1c0b612ce, 0x43df9edf503a8a12, 0xf3df2e8928b7f35f, 0xe126e903af14a8ac, - 0x448c77627d73515d, 0x00c3f486d29e53be, 0x3c31fd93b469ba5e, 0x4fcb20a5a7529543, - 0xa3707c2f4411658b, 0xa7465250261e466a, 0xe67c4c7914d7bd21, 0x1d8ccb0dfcc16691, - 0xf4f4c8c059ed84ef, 0x3655ab2a8f1f1a8a, 0x4735b28c2855bc6b, 0x477c7d01c2a14ddd, - 0xa5118b68602bbb35, 0x0e83dba8356753e9, 0x48f3469c4f8f338c, 0xe23c95b45543b05a, - 0xe3493c2fe45bee6b, 0x2df676fa625f464d, 0x206057079f750f06, 0x72a93734503eb936, - 0xdf5b08b8f461dc59, 0x1cc511da5837c237, 0x6c8c830e31bdc45a, 0x4690d1d8a194d057, - 0x589d3d45c1420dea, 0x7b32800a5e117c4b, 0xbbb0c6a8ac3dd539, 0xd50570ad7bfdf527, - 0xae2179180a7ba703, 0x686d3b7e6d3a11aa, 0x9f0e5531bb4a5a4f, 0x5e45492bad6ab826, - 0x8d95649c36f3fb0b, 0xa7593b34d8ddf337, 0x6a2ef0ea01e66643, 0x151313da52d1040f, - 0xa6caf7cdeac31fed, 0xb906ac61d44f18bd, 0x0091b6aac33a8ac1, 0xd7047e26c6624899, - 0x54bf3e50806b22f1, 0xcc4e0288d4757258, 0x9b24cfacb00d6f8d, 0x7828930badbf3931, - 0xb27dd4bcfa55f214, 0x30c5580dc0f692aa, 0xdcb9a5e151d6949b, 0x95865a720b3fbc26, - 0xc21ca9a9bea3a5ee, 0x0c6200fd1066602d, 0xc0be40ba165812a4, 0xc311e724a599551a, - 0x592feb7846f260ea, 0x1bc94c796e634e0c, 0xe8cb8aae8e677650, 0xc52089c03b3a9ae3, - 0x6d6f5ca4f0e6212a, 0xdba99209feb77993, 0x9c5610d7d3dd2ab9, 0x43d0a23a70c15485, - 0x695864e7c65d617e, 0xcde022e53b32b063, 0x85e9081c648e494a, 0x5437fd9bf3235244, - 0x42c0574c4c69b673, 0xb494111be8da63d5, 0xa09e0f84069c5b03, 0xf608f1f5aa4d8e0d, - 0x4b5917f9af3dbb6f, 0x876db0fa77f6c1d3, 0xa82080840c5efe3e, 0xe4dcd912641a957f, - 0xe52a4f4d4cd110fb, 0x3c000b727b728f6e, 0x9e0adfd7cf7ffb00, 0x55b06df8e913eced, - 0x776290e7e64e17a6, 0x030e32251192a647, 0x914a039902606d6a, 0x614c04f2f0deb2ae, - 0xed1c5c03c896d90c, 0x81d451c79bf48502, 0xcc97a274280d67d9, 0xec0eaf51d75b35cf, - 0x1eb6fcad5ce66f0f, 0xfdaefd72db579b86, 0x8e5aae3324a5e959, 0x3233195e19d91ecf, - 0x30d9620f6fea7fae, 0xf31fd45e3004860b, 0x0c0edcf81fc88418, 0xecc829ee5d5f2bed, - 0x65b48e75e45a5079, 0x7a15e70a8eb6fdc5, 0x0987ab0872508205, 0x3857b2e4fa044ebd, - 0xb509d1cff02ebbed, 0xca16434363ac795a, 0xe72882e693f8c825, 0x035f7e69a46ef4b6, - 0xa9558e56ed4eca61, 0x8d152cb6270a616d, 0xda2357888c287f99, 0x4374954e04e75bf2, - 0xdbecad86354b644b, 0xc1be06c672d19e30, 0x36c91322cc271dc2, 0x9ed40f4909a59093, - 0x5056b2466d24399e, 0x0e6e1f4789296e1a, 0x5f27183cbc931e22, 0xdb880680ffe5d098, - 0x46df0893c2284d92, 0x3e81a40c36286e79, 0x7061fcde186bf809, 0x46393537388d36f9, - 0xaeb2bd0add3e5b59, 0xd92366c6e15b740f, 0xf1b1c65f25df9035, 0xce5c92f99785cfbf, - 0xdf83bb2c68b2a85f, 0x948b9bbffefa68c8, 0x4eb3e3d987a03a58, 0xaade292c62b1a97c, - 0x2b600e8e8c890118, 0xabc706a9da3124b0, 0x614f9aa2cfcfea12, 0x5732c90755bd446e, - 0x673f39744d5738a7, 0x285c0559a6c30318, 0x6029518c6f69a013, 0x0631fc5c12722556, - 0x55e98ba8c797a280, 0xcb7d9260518e3e9b, 0xf0c0cf17cc1499b0, 0x1bf53625e2c03c2f, - 0xd30c73614915df6b, 0x12d79e1116b59d02, 0x00e9c3801043397d, 0xba9176daa2998081, - 0xd04f8c11c1f99664, 0xdf6a14a7663f37d9, 0x13a9caae8f94422b, 0xd321b54959fbc938, - 0xf81f264e83604923, 0x2a39d22599a7d411, 0x185261ce73c33c3f, 0xbed1025475709ec4, - 0x1f99ea29eaafa941, 0xf80fdb450fe0297f, 0x2caab9371690eb0b, 0xb439a767e537a205, - 0x83657401a60c1a73, 0x47b3c98fc95cda94, 0x8c725deeb0f692f4, 0xa4f0badf18755dbf, - 0xee6c4703b1009b33, 0x66cf202576079e5d, 0x2f9d9931b10debec, 0xbb0d315a6d4c4418, - 0x72e8892188ace8ac, 0x4ba59a8981373410, 0x42fe0c0d94f05b83, 0x1025765af27db7e4, - 0x68242ec1235ddd65, 0x333349de465a4214, 0x4796e203a254476a, 0xa9943ec20f7cdb64, - 0xca62d842ebc0bc1c, 0x9efa52dea7fa81b4, 0x7a943750178ee1e2, 0x7e75bc96d11b3282, - 0xf6f691d6e54979d7, 0xff167e6ade6bdacb, 0xe22d517817b501a4, 0xc3f0f2cfae8ad232, - 0x5cff623658d9f984, 0x179b72f35493164e, 0xc9e29ca4125a7e8f, 0xa57b7073b87dfb38, - 0x947111814469da59, 0x4f726c1bf4f19f59, 0x62f2a9b090b93ad9, 0x1c67274f2b1c9240, - 0x9c54bf30fa8275ca, 0xc02a9797ebd83298, 0xa34b7bf6deccd55b, 0xc59bbdf7cb6db6b8, - 0x601025936e76492c, 0x11cf23352115a0ef, 0x07d6307e827758ee, 0xf9119a7342433928, - 0x5bf3cb63519a6cb6, 0xb3aef6f3ce0dad0c, 0x30a98f4ae0d7dbf9, 0xd54ba528feb44aca, - 0xbac183dd40082f35, 0xccc44655fd44514d, 0xfb201d475c95a68a, 0x2bad58d15f1dbae0, - 0x7a67a4e82aa4a661, 0xc9a9043382c08334, 0xa820836c3f04b087, 0x673e3efce460dea1, - 0x186538b4ec679360, 0xd5d4dc5b85b2a834, 0x654fd0a343ae906f, 0x12c4151dbf17b763, - 0x93f373299029a955, 0x8ec35c9846cef520, 0x1996036f503a7706, 0xd37721a4173041eb, - 0x4adb731975ffd143, 0x975404dd1e4ca03b, 0x75be4eb9553a1ba1, 0x38b5d1b974e44a95, - 0xfeca3249460fbfdc, 0x0d911dc52b543fca, 0x76e69494d0ec7c08, 0xbecbeb3b4051293b, - 0x6c7aa918889baa1f, 0x0eed94d0f8c930ca, 0x9500706287a6d0b0, 0xe0670e76961e0080, - 0x4508847afcf55204, 0xab6d8ae202ea29f7, 0x813361acb8697ac7, 0x54213569bacd86c0, - 0x613e224f4cf431b0, 0x8d8b983bbe52e6c4, 0x2a85d5e738aaad72, 0x5bd7abf48694c069, - 0x0248875ab841cbd7, 0xe81843076adccb67, 0x34648c0e6fc49dc8, 0xad13bc6be34bdda4, - 0x5cbd2d2852261889, 0x903b96294f5d2464, 0x990e911bcdc4a2ee, 0x9517753237983fe6, - 0x3f167f8624f36676, 0x5766b690acfb60b7, 0x812ef4067457d618, 0x848c5fa2351cedc9, - 0x575323a752426251, 0x4472209fe40546d4, 0xad1c9247b43652c9, 0x8256d98202673bfc, - 0xe4ca34106e46bd93, 0xbf8445f06e66e51a, 0x1eaba756e2ce11e2, 0x47e8a6dbd3d1d057, - 0x2694b5e8f9948065, 0xd86adbad206f66d6, 0x89ae618fbc42e9d4, 0xb272617d606d715e, - 0xc9eff4d76d9737f1, 0x8580e5817b8e56ba, 0xc054c28cb73416e5, 0xe8418937c30f89d0, - 0x9422058ce6e0f421, 0x8185208436b84427, 0x9e57f903c9d54f4a, 0xa1da3ad20f21720c, - 0x6f3920470364c70b, 0x7267de41e5d7cd47, 0x951cdb18370a99e4, 0xbecd7d1f7e0a62b0, - 0x1a731ced4facc7ec, 0x6a6478589a8fc230, 0x8e6f57b28032df20, 0xea4dd836b8d3fecb, - 0x37e1017a589ddb72, 0xc8f231cea38ca605, 0x6552302293958337, 0x4290b506481e953d, - 0x1ddacde8d684b070, 0x7288e9cf714f3d36, 0xcf5aa4b2ca9b1fec, 0xc612369c24643b43, - 0x4e5d5fafbf2ab94a, 0x53d4e552bc186d0f, 0xa11c79b1e1e701a3, 0x3fa6e9e50d026cef, - 0xe0275611a015bea9, 0xbe071a8805d338e3, 0x777142776a7cbef3, 0xd1ca68784e718622, - 0x7beb0ecd135f983c, 0x46404ce42f971bff, 0x24bf336ebfcc1290, 0xeb43630435a0166b, - 0xec69c4308ec84c37, 0x4c6ce7547f2f641a, 0x5d290951044c819c, 0x1a2e42ee1c7f89c8, - 0x85dcb83be15aed49, 0xfe2b45cb1036204a, 0xb622d1ff29106f98, 0xeb7d494e19eb6cd6, - 0xe061be72d6551a09, 0x0478d97f943e5252, 0xf610cf18809b1702, 0x3135fe3f9fa657db, - 0xb7e65f1cae7b6f46, 0xcda1555e9af90dec, 0x9f976411eb12b0dd, 0x21269e112fe88dc8, - 0xccbf999057a3682b, 0xb974d9bbfaab3c4e, 0x349d10f77d9b91bc, 0xa2160e7f19e30f42, - 0xd41092d2a3d94a98, 0x859ef19d5b10161d, 0xf938cc3c8be4c82e, 0x341efbd6d6fa6bbb, - 0xd10ac4cd23699c9b, 0xf54856acc30418a8, 0x444af70c7605db54, 0x576fb121056e8397, - 0x6121d56419a02a11, 0xbef739c4c0bccda9, 0x6923dfd76c0b2e27, 0x1ba06db8b35f9416, - 0x27770017fae68c1c, 0x352390ad1fdd6738, 0x6dbb47a15c3194f6, 0xb35f51886ffd1260, - 0xcc2c4cba311f074f, 0x7548409855f955f7, 0x264fd1a4e1f5d6f4, 0x1342979385fa6c05, - 0xbaa78427d50336cd, 0xf96f93564a48e068, 0x0e5b0484f96c3068, 0xde7a01ba132e8bef, - 0x6ec689a5348a3c60, 0x7c84e96a97530a60, 0x379bd9287ef5e3de, 0xc4579349937e0771, - 0x79d25571fbb5c63c, 0x508e18fa051c858d, 0xb721ce76f227e74b, 0xca6fd945a03a6af8, - 0x7ee8f804c032dafd, 0xdb95af1f5f696ca4, 0xe092c16ca2e3085c, 0x591ab06e78e27ddc, - 0x03aac4f937aa88bc, 0x7ec6963d125a801d, 0x81712f03e0e452b0, 0xdcadeaa3f229005a, - 0x3810ee5253f86e92, 0x17f6b7bdd79f106d, 0xc311376262ecde17, 0xabe3f50206a1307e, - 0xdac7a2a68bab66d4, 0xbe0914e326910021, 0x1410b04d1c0478ad, 0x7686036003b45a9f, - 0x0b527bd9e826e01e, 0x29992bc289821fb1, 0x753df71ce8994b8d, 0x6a180d92845eb84e, - 0xcfdd13341cd3e83a, 0x562858feae05e64e, 0xc423c5a12ba15860, 0x4bd34f3fb4029ccd, - 0x38500c984d4fad2f, 0x6903a8d89b94a6c0, 0x6170de4c039631c1, 0xb614adf60d0466b3, - 0xb0bfecbe98b4c80f, 0x597a679c82807d59, 0xc3f8fc73da4c2209, 0x3394fdfcba20f57e, - 0xe44b0bc10a99e77d, 0xdb78a47567751de8, 0x9bd751096b243ced, 0xfcdd6e491aa3154c, - 0xf6bca65eef18f0fe, 0xeae9d07fd41bab3d, 0x4a6ffe28426b2afe, 0x5e970e7f3255a78c, - 0xa4e54566b09e2b3c, 0x363ade46e6933ee8, 0xc57c04c214152b1c, 0x2f2e2b2271bab939, - 0x73817b1f5a5f8776, 0xd1f525aea579e536, 0x480980085815aad5, 0xe6d356a140f76bb1, - 0xa8a0bc3070cfd5bd, 0xc31092e66c35e567, 0x79e2da0ab816295d, 0xc04389f7107a96e3, - 0x7c914cdc15635a2c, 0x6b86d21725e768e5, 0x72e2584504af1fef, 0x15538a27564f28bc, - 0x67cc8d243e224aab, 0x40995dcff1645f4a, 0xf107e2b99228d84d, 0xdf4ddf11c7ae4930, - 0xe17a63055d810887, 0x98ef7ac14f6fb93e, 0x161409e104f6e12e, 0x7c2173c58d841e01, - 0x11b28a03054cf9b5, 0x228be8e9cbf13fed, 0x6805bc0f31fd78cb, 0x022e2fd7a2ffda4c, - 0x566be2c9d4d9c96e, 0xf5c8ea76785c246e, 0x3065f7f6bb78a576, 0x7aaa870c28c99114, - 0x3e84c706230e123d, 0x05944ada658193b9, 0xc1e929e56655e872, 0xbffd75e53659461e, - 0x06ea1d837f0c2f83, 0xe6819115bbeb9d55, 0x592560157eb09eec, 0xcff371fb925b9754, - 0x9bbf432d396e6916, 0xf1bb953242efc631, 0xe4323310703d11b8, 0x0381d3f1f3e50141, - 0xf92c494f8d876aef, 0x692ca186b1c08a51, 0xd10b042555097c0f, 0xdf0e666f44cbb618, - 0x1f93c52266618738, 0x627fbe2d5a6b5737, 0xc6bee7fc416ee1a3, 0xd88f7f4c57e05f88, - 0x337ae0e7b9a93815, 0x32c61f4b522cf347, 0x63b6da1447895641, 0x391feb027add0b35, - 0xb78a16bd38368941, 0xa0c1f196eaa0ff18, 0xaf8a07fea53c1e1e, 0x893ee13636aa6c96, - 0x202ea2215318c5ae, 0xb97334a657056882, 0x12a53b107d97d193, 0xc080fbfb06cc7f8b, - 0xc2d8b463f27b0b68, 0x8c87763a9ddb31b7, 0xcfe0e7dc238f6a4e, 0xd23cd37e3da44e82, - 0xa97d4c62dc6223fb, 0x373f8f780172deac, 0x0be49a0bb9d67509, 0xaaad651220db5bd4, - 0x1546f63e2a1f13ba, 0x8f3d0e8a145032fe, 0xd7f5573cbd6ebe37, 0x291088784f8448ac, - 0x85e4255bf05e4245, 0x644a84307cac03fb, 0x859da1e45e2b75ac, 0x9fadcfcf422e761d, - 0x781449ebbc4c70db, 0xc3ea5049297ca356, 0x577d32242cde8204, 0xe250dfcecaca36ad, - 0xedc082116f367a07, 0x8a6cbb131008e725, 0x2aa6cc866474e2e5, 0x95bc97e3c82e320f, - 0x679f4d9dd88929b2, 0x71124e37958792da, 0x4ab3ebbc695f9ebc, 0xab8f80704ecd9297, - 0xde8e3f052dfe3dff, 0xdac780542c7eb6df, 0x6ab7e8884670bbb3, 0xf563589a4dbe1b91, - 0x58d3a86ee0df7d1e, 0xb30316d424975032, 0x74fdf277548e9be7, 0x626330b5625e1996, - 0xee3c0d808e37ca3f, 0x213f9ca29d8138bf, 0xe4930dc245d2c4aa, 0x1a05694e6b9b7c7b, - 0xa39afbd5ee949689, 0x48b3d1325158eb6e, 0xc78c0ab651e45fc6, 0x32ad9eb09f7b2366, - 0x9704e110323b5938, 0xe04b4dce9ed51298, 0xb6d93d21fa6ef7dc, 0xd4d634e8a8ad5dae, - 0x8433cc979885ae49, 0x8bd938a061cf36c5, 0x07bccb3c4b9c1384, 0x14761a82db6c30fa, - 0x04dfdeb63479eb6c, 0xf77d2d8bfe52960b, 0xb7f3a0c3cd3bef0b, 0x2e8188985379b0c1, - 0xa24b9ee3e34e2149, 0xc0f2a7177d23dc5a, 0x5148da475c1c2dcc, 0x74ba16bff2e1cf9a, - 0x8e73279f56ccd257, 0x31d246f73d169729, 0xa6f056d8818f000e, 0xc37c3a81996eb824, - 0x290a699c1c714803, 0xc10b5f1a5fb805ff, 0xf89050dc1eb91fa0, 0xf62f7431d4603290, - 0x72507ef2d4cbcdc9, 0x62e7eecfbeb72623, 0x406d5df2d538e135, 0x236adcd937de320c, - 0xf537ad192958f8b5, 0xe720429625600657, 0x10075841f5c5f6aa, 0x8b7d5431d1cf6e84, - 0xa8c146bdac430bbe, 0x80b8be51018a3445, 0xe50da522a9c721e6, 0xaf2c2ddb5e1cd8d9, - 0x8e7f19aeb5b404a2, 0xb02c9ac19f308180, 0x4f35d045eb853cf7, 0x7740202596e5928c, - 0x3a2be11e275a4e84, 0x5006379559132fb4, 0xb687de7abc399d67, 0xd0cbd5d8f82c85c5, - 0x67585602cdcf5250, 0x2ded7f38995dec22, 0x2f3a5d7c0789ac68, 0x65cc7774b8f71642, - 0xdc8079ed31b82b05, 0x68502ac6f99f6888, 0x3495dcfe810e2476, 0xc975c5a11c563a3e, - 0x284b6119336e009a, 0x0e034b9abee60e89, 0xd3e2f96fc092829c, 0x7edeb6cc9517ac78, - 0x29273a0fdda9623e, 0x476207fecb6c8264, 0xbf23425d202afeb5, 0x1c671b97a879b14a, - 0x64ca2ce5731f0eab, 0x564d544dce025f53, 0xe32f67379640bdb1, 0x5c9e8e6ed89095f8, - 0x7f8fcaa339611d52, 0x24fb9dbb39f416cd, 0xce620a391101d27e, 0x3a1d0e8f082c79ca, - 0x81439c3862231578, 0xe8c877efdcbe869f, 0xcf7e69883a89be75, 0x4191e2288601c14d, - 0xaa868c8c2f552a2d, 0xcf3d6b58b42ef6bc, 0xd37dffaecef7afc8, 0xa9426f8ee9aec446, - 0x8e39f5fc60e903ea, 0x3590984c05ca08be, 0x459f315beda092f6, 0xab1a7e4827b36dfb, - 0x4a89b6136f22a2ed, 0x74aaa2cf359ef6a2, 0x114f48fb69426ced, 0xfc3d76a9d8b9c07e, - 0xf9125e090e05b205, 0x1abf34d821f56432, 0xd85b0267a003a838, 0xe71796e3df88ecb0, - 0x030ec94db5bddcf5, 0x845f2cc307710e17, 0xf038809e87799bab, 0x9e54826b9f6ab1df, - 0xbeecf9ff61fc2a22, 0x65ca36be408e4935, 0x4264689e0e0013e3, 0xbf86fa3472597f13, - 0xf9db9d08a8c3a50d, 0xfa93b5312e9b3004, 0xf30d04e249136076, 0x75b68ce41fd7e2ce, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x68a88405ae53c1e9, 0x51e46707fd558656, 0x71e834cf86896c10, 0x3d251b54e10d581f, - 0x1884d5b0eeb19032, 0xeeaf729853e526fe, 0x5931f6831a8d8c11, 0x87891d33fb98b4d8, - 0xb487b5255ba8a18b, 0xaef785edb2d976c6, 0x8ac084b354d5c019, 0xc714fe0a7e7cb0c9, - 0x272b34204b9086a7, 0x66d421662f652954, 0xbe640374928d3af4, 0x7d3257b3f0553a6e, - 0x4f1d9b4c437a08a5, 0x11ae3d72e0e1e536, 0x6e7924dc694356b5, 0x6e9ecc765f59086c, - 0xde6d6c65d7a5a821, 0xdd931ceaf6a11453, 0x6a175ae7c1e72630, 0xa9bc5474fffd9cf6, - 0xeeaef3b448525ec8, 0x15008c9351c8bd2a, 0x81b3090c5ee3e61c, 0x01767fed76e01382, - 0xb6cfaa3c6b39105c, 0x36e7d0051c456721, 0xdae50f00fa46dbee, 0xe965d41c883da319, - 0xae3791f2103ece26, 0x1ba533b9e8c869f0, 0xa65abccce40c0a4b, 0xfa6915e0b76276a1, - 0x4feb7b7a96c8a36f, 0x7f49425e7e54bbaa, 0x74911044655a6d3f, 0xcf46469a6114d216, - 0xa2f5106e2a65f70a, 0x5bb57bd0c4d8d1ea, 0xf8465cb0f73a0bab, 0x8011828ad5716c05, - 0x2b64be22994f2908, 0x2a247946e673e7a0, 0x87a2f13d788e0d87, 0x0628e8ab5b721f18, - 0xdf60eda1163aa13b, 0x6a3d7af1fa7bc9d2, 0x3f8b712461d47f9b, 0x64363be32bbcb36e, - 0xa0df8815f0f98d15, 0x5e9cd01f29a9b5c2, 0xf5f24fcdea2f8661, 0xd427b3c4db6a676d, - 0x559b2ef1191938c2, 0x0904120044e6619d, 0x9b412237d49e8c08, 0xa9c7b33ecb50d796, - 0xce71beaf6d1a2066, 0x05982d236e3699d4, 0x7ef041f65bdd5dfd, 0x92f71df57e10e1df, - 0xba9fd98804b12e39, 0x3eb4d11bc6597636, 0x74a95561edce0538, 0x364c98dfe633b663, - 0x96331a38044d02d5, 0xea03f85842a6a6e0, 0x7f11713d4acd83df, 0x174cd4862b09b207, - 0x9ac99fd99d293746, 0x829088bda918704e, 0x1e25e23ef32adadb, 0xb36ce3bc545c430e, - 0xca4dbef135fde2cc, 0x7a09bd725ede630f, 0x0a7653425c86d27c, 0x2fefac3ba6c76189, - 0x17f29a0a93b8dca4, 0x087c1e3cdfccc0d3, 0x540dd8131ce016df, 0x34c138326824e27b, - 0xfcb24127d7126e93, 0x9700d6148ea2ce82, 0x7c74c39eb552bf19, 0xd3c48f611dc32059, - 0x4593316f0c9ae54d, 0xc5e9cf2a76cec102, 0x5031874bb39954ef, 0x6277fc4b66783ff4, - 0xf8efc66553113921, 0x922cfa1c2729fc98, 0x87a2bb305b22b9f6, 0x16cc00ce83c32c58, - 0x5a0c119889d5b87e, 0x83386168b33d3709, 0x4725a7bf1ce70257, 0x4a69de1c9053da04, - 0x5a18c5f56e4ff0c5, 0xb7d5bc335d7fec62, 0xee2842a810e1d7f7, 0xf8029fcb04514984, - 0x44097e987d384762, 0x1f670486da26c390, 0x2b21e4e65f6b7249, 0xfa846aee1c1822f3, - 0x671bf2cf4ba94153, 0xb1b00cce1f7b6a0a, 0x866874dec139ab19, 0xe1da0d6b89a9a65a, - 0x45e8480cd67f43d6, 0x472ae5aa72d8290a, 0xd14fb4eebc9b3f1f, 0x05e2dc745167e7c1, - 0x0759c43bb10d7a36, 0x171ced051dc4aeb3, 0x8b8ea8b911fc198b, 0x331b01c9195af0c3, - 0x44b0770ae5e09db6, 0xa5f622fc48e91c17, 0xbda14dc3486cc341, 0xc8cee9fb6ecd485b, - 0x4140fed8ec6d03d6, 0xa94955e3511aa5d7, 0x0d579b4bf974f83d, 0x7507573ecb8d959d, - 0x666fdeacfc793064, 0x6941efa8f1b5ae4e, 0x110daf8ef142d048, 0xd98106b177caeaac, - 0xeee1135cc68e8ada, 0x9a9661094c3b8b59, 0xaaaf6f12b3423c40, 0x08ac7d1e1576e043, - 0xc03c4ac1712d1d66, 0xc14b5354034d9d1b, 0x9b76dc0dcf104c94, 0x9be1a977cb012a82, - 0xc6c1137efe5f27cb, 0x137e94494e276707, 0x877dde2cf9338d7b, 0x518c11d72047a052, - 0x99a4649fd1969cae, 0xe12ad8516f5dfe24, 0x5c7b37f89a048969, 0x154226b3f008c0aa, - 0x658c164a8eebcc68, 0x4dd70568e89ec7e0, 0x4c4b2929b9a9c007, 0x6159bd57c3704eee, - 0xc11792c0c04fc58f, 0x2750e1ec3c5c05ac, 0xa3e8ed0200e723e7, 0x95a8b8b19b536312, - 0xe66ffcc92c3f46cd, 0x91c4fd71f7f83e48, 0x4958d79605512469, 0xe2dab1909168da73, - 0x206ad23714748082, 0x16864ebb5656fb09, 0xcd681f3f66c33fdd, 0x9f823c45f2484ca8, - 0x448cdfde0e625e57, 0xf804766df9c4599a, 0x1aaf20ec9381cc0f, 0x7dc00124d1863e2a, - 0xfa1cff77a4bf3270, 0x8a9b34cf02dd3fcc, 0xe73230cf7e5794d1, 0xb4b6d4a7ceeea784, - 0xdcde4a7169d825e5, 0x2464e3573aae1d86, 0x4fcf1ec8fa532c0e, 0x894e984daec0e65a, - 0x1516b3acdc84f46d, 0x4b5b83c08eccbb45, 0xe9c7852f64a4f99a, 0xc350e55f1a837df7, - 0x1e56f15ac7b78181, 0x19088cc08abdac8e, 0x6270f5f100c2fff3, 0xeb9247065be9cff6, - 0xedc22969d1c97abb, 0xe96361618325e027, 0x0c72b206cc0738f6, 0x3329aaa4346397a1, - 0x8541c9611d60e115, 0xc9c80cbfb4ef3c7c, 0xe9000b60a281ab77, 0x2da579151532c076, - 0xbaca9481a2782c62, 0xce6ee589779b618f, 0xd5cb1989340aaba9, 0xc195090afd1670f1, - 0x2611c813d09b50d5, 0xdc02ca3b2670ad05, 0xdb5a62ee2cad3f18, 0x883f0994bce4ea50, - 0x4638f49f3bb23b52, 0x32c256bf67046a20, 0x29c2aeb570a48f47, 0xcc60711f19e0b32a, - 0x1c75d49f6895f302, 0x8b4bf4d57aed92fc, 0x15b2fedab168372d, 0xc814350db17a4e30, - 0xfde4daa4722af3f3, 0x6aa6e92f01e9e122, 0x6390e377a5bb073b, 0x2799338a6574e8b9, - 0xedd5131cb43fc739, 0x9bdf9b56f2a578d9, 0x150b15947f700c4c, 0x73a15063b16c2cb0, - 0x1a29a94672580154, 0x7bf2e8016a8e7037, 0x63d81bd80cf07515, 0xa71c95e84eb71cb3, - 0xe0ca1c960020fec9, 0xf4365910240825a7, 0x58d457e1bae55016, 0xea58a44649268fda, - 0xc95910b3e9178463, 0x26ca556e98942e1a, 0xd5d9f1db827da3da, 0x9bcd8fbaa47df5c2, - 0x5ab9f787c3c0b49e, 0x9459c256614eef41, 0x03d9c4d007717be2, 0xa65e973a6390e838, - 0xeef59a8e2c6d0400, 0x5903c0e9b3cfb489, 0x01e63ea530389f56, 0x7c7bd36f7c68056c, - 0xede6ca105864a242, 0x12a0924cda935c12, 0xb13fa1ec336f1cc6, 0x3a383db45c2c8df2, - 0xc48689e87fe0bb90, 0x3ad354a4e2041fb4, 0xb43005598d6cc969, 0xe88ce9a3b0a7e8a1, - 0xc8b24787ac4ad2d4, 0x44cf7f2f67646bb5, 0xe58c5d7a31f11361, 0x2be0271f13e8b84c, - 0xd968752ec676c556, 0x2112bedba1fb3fc1, 0x2d138123077fc1b2, 0x6fc0527824c7feb4, - 0xc1ed0b5f443eeb16, 0xc779afc4543bb154, 0x7723661541dc816a, 0x1d6e455db4c51246, - 0xe3f5923d012b4413, 0x3d31546585916da8, 0x77c15b5d5393499a, 0xf735e16b346dddcc, - 0xf350d0d5aaf10353, 0xc2ea2fb3b94dbe71, 0x0653a193e6a96f1c, 0x52f61850bd61acca, - 0xef0090312f32250a, 0xade071ff88a7aebc, 0x2c4143590a305d7a, 0x8d44fcd721daa8e7, - 0xd926d33088977806, 0x936a1ca7a85e29a9, 0x6d064181ddc15b26, 0x3efb8a39359cc3ae, - 0xf092e887ed81d300, 0xb8968b4b8f29e2f0, 0xc6c670166e6d5702, 0x1320e07cc89889f2, - 0xa02d12fec9d43f0d, 0xdb249650c1374ec2, 0x38a542341249795d, 0xa2b6cba8e26057ac, - 0xf57e5cf5c5431e62, 0x7c6eaaaeeb9d8237, 0xa584ff86a6ef5fa6, 0x38fd0ab720a5f83a, - 0x8d09831ac99806c1, 0x810261309bc9e1a2, 0xf2b6c665885111fe, 0x1a2ada0cea61d19d, - 0xed66bf575f3d2195, 0x9720b6d0d916383d, 0x1aa3861ce7c3e810, 0x83c63c49056ba786, - 0x15509e059afb8423, 0xe27fb0ae655af55b, 0xb9ccc95902017ce9, 0x4b99244a08c8a245, - 0x913210b2285dc912, 0x17b2952d51137be5, 0x267165af3028a815, 0x25fc497ba7448a37, - 0xa0c438e81e940b3a, 0x239c637315d9e0fc, 0x43cfb0956874407f, 0x4d925bf477e85263, - 0x8a59448959e6e14c, 0xdd1bfbdc4170ccb3, 0xb60c94c8b4bd266a, 0xe374b6b5dafdfcd2, - 0xafeae0fbd611a889, 0x3a33abd1351447cf, 0x410fd647b3f7f350, 0xd16354bcef59b7b6, - 0x3725b51f0ac33c85, 0xc5261224cf7040c6, 0x2d8d2bd90ea31036, 0x6600bed1ee3ed81f, - 0xdafd1254f4ad0b9b, 0x4547d4bb47ff955b, 0x92923d77dd27f3cb, 0x3fd86bfd2e6d5d54, - 0x3e1370b411e2d71d, 0x12d9f9827a09b233, 0x5c5590c579f8c437, 0x28708dd608f1b261, - 0x635d80066dc9f205, 0x7963478b72042708, 0xb5b8fa60abc809a5, 0x17ad37ebe8b3994b, - 0x1742aa93c8cb3bac, 0x4ac5c070ddce008f, 0x2084a97f81f6e8b7, 0xb3bb55112cedbef3, - 0xc2174a405789a7ee, 0x87f9677d909e4dfc, 0x0b7af2c5df902825, 0xd9a219447a133d35, - 0x28fb55da9cf9b883, 0x31a4cc967647484f, 0x3737d752bbfef251, 0xd31d8d7dc645663f, - 0xffbba32e24f1f14a, 0x686ed5401a22fdee, 0xde531f96923a7ed4, 0xfcc7b3228e4a95b2, - 0x98c7bdaf44a1851b, 0x1f0281277a70c2fa, 0x4107223f6745b389, 0x050305f649a8e72e, - 0xc848f30952aea81b, 0x016233fd3524c5bd, 0x56dda3621f0dc8ec, 0xbf7fb461d4ad8321, - 0x07f13c5900a2f2c2, 0x5a7e401ce67e2fff, 0x591f29c50a1a6a02, 0xd306aad8556b23b4, - 0x1beb83c9a4391776, 0x69878af5ef6b65e5, 0xeba7fc9c9c450800, 0xf66081bb88787c13, - 0x4d8928d2473f1e9b, 0x41eefed7adab87fc, 0x96b2d91c7ac490d6, 0x030f0d32712d7200, - 0x923da55916ad0675, 0x2abb3c0e7a92e6eb, 0xc983d6951262d7cf, 0x966d29eecfb2a585, - 0x807f205a8a263b90, 0x4ab58b3a8888c805, 0x8987eb7da748219e, 0xe24b69cb89abfe2c, - 0xc8b1ed3594476d01, 0x9936f86417a376ab, 0xb9a1e220d4c73b5f, 0xaeabdc57301ed215, - 0xab93f40d8ee8bcd6, 0x18d57cef8d76c4bc, 0x99ceb630ccb5e003, 0x2bf9a013583d0b91, - 0xfecdaf141ba73704, 0x1f1b2727fc548fad, 0x49927d8a552c698d, 0x0e964d3810d60b8a, - 0xf6f22eb5d55407b1, 0x521ca44e9e6f8141, 0x36b72da00c7ac3d2, 0x08bcc8ddbcb8b0ac, - 0x82adce5f095be780, 0x347d5557d97303c6, 0x809fdb82eb872ae3, 0x32542bb150a4c6e6, - 0x94af1390dbd6e6cf, 0xe17b6fc1806c6651, 0x201e35edb4c9506e, 0x2f290f0de69e6bc6, - 0xe4138a63f6c01e17, 0xfbd906a46f6424e8, 0xc7aba38db2f28da4, 0xc8ecf19cbcefb769, - 0xb6a8533d9c7744ac, 0xc2f45bdc87c66b40, 0xe3a786221187073e, 0x1a2e332fe1f49af2, - 0x5c8b724d22463cae, 0xef903c277473111d, 0x2ecfaa108f04eaa6, 0x64021e3fe0bea4ac, - 0x7d9b195fe3f0016a, 0x8349a989cba17e6a, 0x778fa84eb9848b79, 0x7d5b0910a0d1daa5, - 0x099a5a3ed46d0e2c, 0x7605ebaf3b98adf4, 0x44b6712f212e4a0e, 0x7256a2cc104ed9c2, - 0xcab2a696e71328f4, 0x52773cec651b38b8, 0x0c5b020e58a0e705, 0x71739ab7c3e5dd4c, - 0x04bc54f524015657, 0xd80c97aabd974030, 0xfbe0803ba7213b44, 0x885347f8e756d3e7, - 0x7170ddc4ddd41bf2, 0x9f875418829c1f40, 0x320215bf9ca5bd79, 0xf00d3dcfebb21285, - 0xc19c84689053c550, 0xfac80461c1d7f9db, 0xddfc1cf206524809, 0x1caf44508e360343, - 0x1eedc25b6b3b8be1, 0xf91e79c23c591682, 0x8869c24e0574f9c6, 0x84fbba5733bf4eca, - 0xd3ec6e3a79017943, 0x5ac275cee135b8eb, 0x230f3fd1279ed028, 0x7796111faba4ce12, - 0x1c1a08bcd43f1973, 0x88732d942367ab21, 0x01e6f6fa5f065f47, 0xee45eace122d031d, - 0x9c017c86b13f1199, 0xabba94829fab9987, 0x6905db2612d71133, 0xfbc7990ae35b5ebd, - 0xe093a488d0fd85e0, 0x4a5dd52368442e8d, 0x6cfa40e05cb5cebe, 0xfcc30f2ecc5cdfe4, - 0x0aad2f4ba54db069, 0x851eee93183f0058, 0x5674ba1c578d3256, 0x8b3cdf56eeedecf6, - 0x5f34f5352d13953e, 0x9cc34c071ed95111, 0x77cff4d3730fb35e, 0xdab386c79085f993, - 0x0233e4294e5086ef, 0x62f3b2288f05f9a1, 0x2eb73f21143c6de0, 0x95714b93854269ac, - 0xd73eef81cb898ebf, 0xffcf1caa79c9e22e, 0x5c58fbade23d6815, 0x5f04ead32e992f7a, - 0x3d590ea15506cf9a, 0x5c921b706621a15f, 0x92fc778ecf6cf59c, 0xff4dbb69f3f7c5d8, - 0xee275cc8270da58e, 0xb5989ebd08fc22c1, 0xa380338ab1fc4cbc, 0xc27c6ee4fae33d41, - 0x63c3e130e2fcad53, 0x8eb819f33fe7bcbe, 0xd7c6dd9b08dc828a, 0x86d6febb7ae12048, - 0xbcc2c09dfffa0bc1, 0x6d231c8beb49ad65, 0x6e21db7410e88407, 0xdbc4375428836d04, - 0x54b5c871999ddb8d, 0x98847b5cf87371c1, 0x350048d90c58464a, 0xcd88b06d02f97620, - 0x6529d1fca6eac1f2, 0x059eeecec8cb2dfb, 0x41d91847932b05e1, 0x1e7157e1b2cdcef4, - 0xcb20c89fe11fed06, 0x9958e95c96c58c22, 0xcc88b21dc1fb1691, 0xe02419263109b22b, - 0x63dd28130256f643, 0x476760144540adb9, 0x5c5aa45ca95a6dc0, 0x36d60de6a2ebae85, - 0x8ea0e21e999aeef6, 0x33cde8655659711c, 0xa09a6098a27fd968, 0x586793c00628217d, - 0x98e2c995100db5dc, 0xf4f6605b0c4d444b, 0x988c39da3f8866c4, 0x8708f33ea531970b, - 0xd9dd50d2c09fe1a7, 0x64593f962e50f5e2, 0xcdd32fa1e8f5498b, 0xe1f05064b245f00b, - 0x8c61c608aa8291dc, 0x3bfc23d4e79e2638, 0xa17cdcad16680907, 0xbf464ad0f058ab30, - 0x97d38fe2519ccf5a, 0xa83defa9c0ab9691, 0x2a469a3ee422824b, 0x6d62b0ae6b395270, - 0xe3adb450f2d9eb6b, 0x24c8819d67bea704, 0x6a1d3b214030d765, 0x2a1ceffbbff67252, - 0x6f2a0ffde3ddbd5f, 0x108131b0fe3d544e, 0x72e55493d522e42a, 0x51ce07e48936a2e5, - 0xb2fce667b21b5f96, 0x05ef35e3844a5334, 0x0ca52cd5eb3e98b7, 0x2ea9445011d02927, - 0x682d9bff42223dac, 0x9ab62eead52d669a, 0xa71bfefdf7253405, 0x4e1c99333977644c, - 0xd6b72acff8c6d81d, 0x71695a880b0c5ce0, 0xdd8d32c5f1abe8ad, 0x73ac774faf7f7a5b, - 0xba2bb0ac26556d2c, 0x6a8cdefc90555f9e, 0xdab29491a40ab6da, 0x0632fc8aabcaabdf, - 0xd8918803e880fa06, 0xdc937dc4ab6f44e5, 0xb5d82efd01e65f80, 0x7f95393feb9b8660, - 0x4c5bad032cc2623f, 0x9a030c322d633a3b, 0x984e84086b79be3b, 0x5e2c1780eb388af4, - 0xa4d41e7fe5893be9, 0x7c249587365effc5, 0x5d3ee45dad75375b, 0x510abe8f0e5fb860, - 0x6d04173f67c4c29c, 0x42f0d659eae864aa, 0x4decd9d561108165, 0xee4f43427e1a9429, - 0x4927700ce0db95c2, 0x67784819a4540612, 0xfeba597d843be1f9, 0xc7936293f68aace5, - 0xe96d9272c9f96973, 0x20fd99c1bea43974, 0x67d1c6f8a8bf3a48, 0x10e6a50579651104, - 0x9cf2623c1a97e865, 0x0fba96c5f7d4475e, 0x8d963b9baa6bd223, 0xd8a882157909cfb6, - 0x97a217a4714d5fcc, 0xe7cededc7cbbe7d8, 0x539e4f876a685fe2, 0x5bf162fe44c20d78, - 0x8c1c7e2d8f3a35cc, 0x165798ef2d27c906, 0x8ff0ac3d0baea43d, 0x15c6574c6634b946, - 0x8030d54f34e176fd, 0x4cd867028f0a841e, 0x4856fd154d228b15, 0xb23b9d24a8f01d3f, - 0xcc3dc30c0f926156, 0xca9a3d3e69c977a1, 0x9278c6bc92b0c072, 0xb825e59d380aecac, - 0xfc853abf77796adf, 0x576d6f8a66dc0758, 0x1ce36d41d4c04fff, 0xfb795f91b155b44f, - 0x324ef6a30c9d33c9, 0x3eea5f01cab7def9, 0xc20cfacfdac4f956, 0x6f333838edc580fd, - 0x8320bec2fb4ac89e, 0x1c21f074d8bcae96, 0xe861e816c5b2c8ae, 0xb720642b9de41d7e, - 0xfc80b9a54b1c6500, 0x71e0a04d538a944c, 0x22fab65c9968a861, 0x676693e907111a66, - 0x63e6ae888168b7d6, 0x62199b5cc8459a70, 0x832c36acbee483d3, 0x5f7f4cd3ac7a3d01, - 0xb0743b49557e31a8, 0x5a9248e7693808ee, 0xfcb6c30892c2fcdf, 0x470d861e9bc6ab17, - 0x09ba01bd0bcaae85, 0xa9662f2750be63db, 0xec39f037c87cc532, 0x278c7a9e11299954, - 0xaf2bb95712fd1dd0, 0xaae4ec68a753c9b9, 0xbd6debf0d0227fde, 0x9735e7f65b2a5bf6, - 0xc53467e1869f2c5f, 0x8d3d61840239c44c, 0x02c253599c8c0d02, 0xca3a62e38b8cba89, - 0xbb7f2865d464df84, 0x303c156b635ccab7, 0xcfce0b2fa7ec7537, 0x7541c60dc645b46d, - 0xe287edf8277deada, 0x39cfe221e9623cfb, 0xe6f15ee7e8b64dc5, 0x65e37075223bd467, - 0x00191bd826560028, 0x66d2fa787ea37b13, 0x2ccdec590a32f4c7, 0xbaf230752dbf5d2a, - 0x132c05c75c73d62c, 0xc61e235e193dc690, 0xd2c06c30b10cf51b, 0x5c09ed6813408ca9, - 0xd0296a6a8e3d17ea, 0x3d16c66e5901acee, 0xdfdbd806d49e79bf, 0x4e587159a360a03a, - 0x7dd099b6394c1687, 0x62a5371f46b6e907, 0x24f0f7c0a3fa8e4a, 0xc8c4de8477c6ae0a, - 0xbf61ec7e94bfcfe5, 0xd853e18da1ba7535, 0x4690d41e01470cbd, 0xe0859ba0a234387e, - 0x4fbab772cca40a06, 0x91c208b6eaeb0b29, 0x77ccc1604608ef6a, 0xfab204f7d65eba9b, - 0xca3f50775a343ed4, 0x9d5a0d35af2c6151, 0xea85884931f8c2ac, 0xc2461abc3f36528b, - 0xbad58007f5f8d39c, 0xb3a1e6fe12e389d5, 0x018c7201e88fbbe6, 0x210edddd14f0990c, - 0x90479d181530756c, 0x472027681e9299c7, 0xa72cfc66442a2361, 0xc9ddf56011519d6d, - 0x2eb1da33d744b97f, 0xed00a9a951fc2263, 0xb05d5cf9f8039f67, 0x139a3d74b7f6c641, - 0x9023f62b9104ce76, 0x01e76897301b5e2a, 0x9258b7c567d2f3fc, 0xac3df1ea5c4d45a7, - 0x962dc580f6522299, 0x6211d2df146ec64e, 0xa0a30db54e6c3f78, 0xc64a50eed26aab06, - 0x2d5f8cde168ff719, 0xc3f603b25b0d7afb, 0x6a77564d98310a4f, 0x629bfaca7270632e, - 0xe8510b8f0ac97bbe, 0xaaa524dc61ee914b, 0x3f92dbe8cc3a23cf, 0x5904a527491ecc0b, - 0xdec0a8a62f37d4db, 0xd7d023142ed7fec6, 0x487d75f8fa050e1a, 0xb48b067dbae39218, - 0x4ae1e33dbfad74eb, 0x180a1e74ea443245, 0xee6d3e405cf468d7, 0x63b2a55750f19e8d, - 0x6bdcfcf084966d3f, 0x8f4bc9d05b08737f, 0x4c073bddaaafd5e9, 0xb8c7d0e3fff78405, - 0x263d4c6ab8f48525, 0x5c1a61e91536984d, 0xae6c615039505adc, 0x1466872a983a7714, - 0x64afb867a9b983ab, 0x6028ce99ea980ec8, 0x1bde4233183a1ea3, 0x137084c619b97579, - 0x2b5f319849d27ddc, 0x5246cb181bae6792, 0x6c4d680ff2231aa6, 0x400569aded8de516, - 0x167ddb8fafcc58af, 0xddfcba2e8e65a9aa, 0xb621d82f7f7e53f0, 0xdea72947a771dbd5, - 0xb2d295b4603663d7, 0x2ac89daf6c8bc32b, 0xc3df2d217893d5c7, 0x1ec52f38416acb02, - 0xf9622a5a56da178a, 0x669e587f735963fe, 0x685a785f7acb8fca, 0x202b9f331db7293c, - 0x42dcc7afa1cc2c43, 0x9384aeae398f001d, 0x26e51059e3a8d962, 0x034ed2ccf1b040d4, - 0x49d00e77bee4bd76, 0xf6788c4a203471ef, 0x682b4b63c110e26e, 0xcb13003456a4b36b, - 0xdd20575c392f8a1f, 0x8ab3e9ba528a7d74, 0x27b09b5be903e0c8, 0xaa901b2ca5da7521, - 0x239b33b728f8ae54, 0xc44df91f338b5dfe, 0x9e1195bb30d03383, 0x7c60460a98bf637b, - 0x8101f042107e0df5, 0xd7839071462897a3, 0xbe8f3e7dee8c1c69, 0x90ec2174b5d6f6d5, - 0x5bf627049e30447f, 0xd4b71b12a7853826, 0x9e3a92aaf5a6ffaa, 0x589362bb652a86c1, - 0x3cfaed1b97bb01d0, 0x5c1a3f47d2c5a619, 0x84903fb44aa2a1d1, 0x8575fcf52cf99edd, - 0x920f27f9d8352ac8, 0x2cc587f3ed81aa38, 0xc8da15c9fb440a80, 0x1f9eea62486b957f, - 0x16c05590043e2ab9, 0xfa44514256abeb5a, 0xfc148b2b684fadd8, 0x67adb7e0018dc9a2, - 0x3f75a9b2319702d7, 0x748d7166a86cc1e6, 0xf1ea257cb198ccb2, 0x9d2f20e3959f1e79, - 0x0054a40329b34730, 0xf2800309dc4e5622, 0x3e5efd3e65a5670b, 0x3c9f7090ca02f906, - 0x5b3ba8915cc6fa4f, 0x7e9b90e39b3e7f5e, 0xe222341ea4f47b41, 0x999ff3f0ecf13e1f, - 0x7086b48e29d67a40, 0x52a55f492fca67c2, 0xba4a26c8d510e50e, 0x21aa75516560e1ae, - 0x2eb0ae3919d22152, 0xf8f5bf1573837c2c, 0x15b9af0ea1a025dc, 0xf4b2553022db0607, - 0xe0efd93b932d708e, 0xf38683b9ff44e66b, 0xa2b16328bd778d98, 0x1286d5c537322926, - 0x39f0875be9b90433, 0xe0065196cbd861f8, 0x12ee2c7945594680, 0x04e351d73c57dafb, - 0xa602a2723d74c0d1, 0x10267cfc16cfcad5, 0x77e95f28753ce94e, 0x1b4cfe9b64a42517, - 0x9b33589539094cb4, 0xf71f061f8058b246, 0x07e67f3e88ea84ea, 0x9573646d5946bb6e, - 0x17e528ac2200925e, 0x35f3e14e561df567, 0x5760cd7754e245c5, 0x1425f2772d978c75, - 0x1850be44e441f0f5, 0x2ae79541527b250e, 0x90347b425a8f1052, 0x54b483d6bf51c177, - 0x4a3bc6ef5ddec0b0, 0xb382c256b787afdb, 0x14fdb142c1d8c7fe, 0x9645b592804b65e4, - 0x6674748dd3ce6371, 0xc8db18f5d85173fd, 0x43e5895ecb320d01, 0xde27a22f2adc7829, - 0x8a6fa8434065a973, 0xc694f46c9e42eefc, 0x03f5153b4f8d03b5, 0xb2920a7ab09b7360, - 0xac6c0d89aa3375ba, 0xce1357c6de141be8, 0x7a92fb3a5afbe316, 0xb9c06c6748f99f7e, - 0xe2f698dc9466a600, 0xf75fe203aa885410, 0x12f15e085d1c30c5, 0xb0035ae1b35d19ce, - 0x2c9840987fea1dbd, 0x21c87b8c474a53ff, 0x92ad540d46c589e5, 0x5aacd3d96d98883f, - 0xe58926ac3065c818, 0x65fcd82b582c6f77, 0xc828a1bc6aba4ea6, 0xb5937c157934e50a, - 0x1f51624944107f42, 0x7a413edaa22c3a85, 0xfc5d19c3b4aa7d4c, 0x2c8da632b5b3eaf0, - 0x5c8f0d81aa8ae557, 0x154bf81d72b9ee32, 0x423549e63786d4ef, 0xb87e6e325f6c9417, - 0xaef530bda2cf9b4f, 0xda5f2b0074a7242e, 0x6c63e852447c1bde, 0x4679f10571b2bd70, - 0xe54f0616875c662f, 0x1dd0d135386c8165, 0xcd891538eee7d346, 0x1a477b4f503baead, - 0xef891c04184f8c08, 0x8ba7fbdcb0479886, 0xeec534d83ed32f8a, 0x7511b67bade8400c, - 0x258173e0f30e4704, 0xf53eddc6b65be135, 0x1b6701d7b37a938b, 0x7905f1a68684ffa2, - 0xdf4c94f28214991b, 0x1b906c64f1e4068c, 0x38c60fa25ebd9358, 0x5638db78c5a9c639, - 0x5b9cc2830ec95591, 0x7a51120987d1e1e0, 0xebf7cd8bc200a822, 0x8c62f16c5ad42350, - 0xd94902922c304a92, 0xf9b62471a63d738d, 0x6278514fe601ba6c, 0x8273c1b119848464, - 0x59145705ce3a90c0, 0x0863d89499fcf355, 0xfdb429b3f8db3a0a, 0x10e8ddd36729de8e, - 0xc335a2416dbeefaa, 0xf93b90345c4f43ef, 0x702bf5af66c16ae0, 0x9eab7cc8dbb3935f, - 0x5b5f39c499107bff, 0x64799ccfedfa4650, 0x34d72ab4860d94ee, 0x1e3339515d75d17a, - 0xf2d4d2d9bfde7356, 0xadb902f957c6ce9e, 0xf154cc1d9498d24d, 0x1303f7307c59f4b5, - 0x3d3bdc611bd563a9, 0xbb19c5d5934df6e5, 0x45a4911ce13230c2, 0xf4100ebffbdc57dc, - 0x04ec224cf196e56d, 0x535b02eb0bf77f22, 0x901721d87104d044, 0x7a842f8ad299a021, - 0x75284dae9d85ac72, 0xe542e5e7d76499ab, 0xfa67b257d4db06b1, 0x9d31845ddee320ff, - 0x855695a549621583, 0x71bd9794d93a49c6, 0xf7340618a60ed56d, 0x0eff348916c1e55e, - 0xbb80beeb5410cabe, 0xb065dd40238c949f, 0x8a89ad1b1c1288bf, 0xa64b79c5229c2c51, - 0xb864d86ce4539341, 0x1b17265b84dd4394, 0xe885a18c15628ddc, 0x3b5b39aac67d0e9b, - 0xe0f520881827bdb3, 0x659f3e0aadb1b62b, 0xf4d3215e317da4f9, 0xda02faa30f9fa8af, - 0xbb2cd5b70229472e, 0x657524e70fa4d6d9, 0xd153b6ee270b3f01, 0xeec0ae8a56104baa, - 0x327524cec63a8092, 0xcbe599f59941996a, 0x82e3ea97233c32a7, 0x6e4f016fdb703fc8, - 0x2c16712cfeb8e2b4, 0x007ec1be449b999c, 0x4f645df0a1a66d79, 0x7be0d9df08a59ab0, - 0xd73b4049650fae43, 0x98175c902b054038, 0x90fe7aa8f07242f6, 0x2278f1a2a6160c4e, - 0xd9f42f837eb9ffd8, 0x40591660a9667517, 0x82bd841ef3d255d3, 0x38e4a240812d5d4f, - 0xc88fdcab342f8f73, 0xc7398ac206b2d0d7, 0x5fa6c929f8603890, 0xe1aaab335676b822, - 0x9db00ce94c3bbe47, 0x149c8a8ce25f2023, 0x8965cece4bb7aead, 0x2c6ef954f2691f66, - 0x277d0c3c488d034e, 0x013b65cd5ae9041d, 0xf29855129295054d, 0xa1ce2458a684f80b, - 0xfa1f3c969966a1aa, 0x8f71a932834b29c4, 0x81d0c412683ae646, 0x0b7758a483af85db, - 0x05e7c69a32fd38ee, 0xf126b4a0a385d230, 0x9409a460ebb589b4, 0x636e82e234ae43d7, - 0x82dd9ff981b5d96f, 0x4d91221e227d5795, 0x507155e1d597c644, 0x6cd826044733ee90, - 0x2baf2543a4b383db, 0xd5744195291c7203, 0x137a52aba94e7efe, 0x55ef259f455e95fa, - 0x26303478bde0b1ca, 0x1ef6f19b252ee589, 0x43614fb619f4a2b7, 0x5b8f728248511201, - 0x46274c7ca07da664, 0x6caaf0c009c97348, 0x21fe354e60d9e2fe, 0xf3a79f114a28e927, - 0x8c7b2363d408236c, 0xb3631ed744dfd4db, 0x2df81911b998572d, 0x267aeb4158424a55, - 0x6c1a9d2f4b315643, 0x240454c46dc7c52c, 0xeea02b5e11001c49, 0xc823415254f3f948, - 0xd9b944649c0a8e8f, 0x433a15ad7254ab87, 0xb81b23c9a7e1cfd9, 0xc135073100e8f189, - 0x7a273e7519762474, 0xb55194d4b382bee1, 0x830d3ebbd46115e5, 0xca9207c5275147c4, - 0x61ead232637093ca, 0x9e8860c5bcadb459, 0x1f694881cc0b5ffe, 0x617d4c9d2093941f, - 0xa63c48fdb350dc1d, 0x2d986565560a3efe, 0xe23cae26f65ba98e, 0xf419b5bf2ae46568, - 0x10f2053153a2ed44, 0x884797e2a94c7620, 0x62f3f88689a0e5f7, 0x79d78a5af9e66011, - 0x893d687e9883b617, 0x58863c2f53491584, 0xd70902209e0a684f, 0x9bd3440f30bb7f5a, - 0x085cda42b6e75c94, 0xe0d67bb5f5a6f567, 0xf1ce7ce19540d0c2, 0x5a8db6ef336f6f87, - 0xfe01c1f2b957c41c, 0x15400e32858dd78b, 0x6b369c9b233a9b44, 0xbb311bfb3f2a739d, - 0xac95f7fd1e2f5acd, 0xffc103403004607b, 0x5cf5886bb9a166ad, 0x8b8652fbfb2098c9, - 0x81462ed7c0fd5f70, 0xf95e5cec6001d828, 0x8f181b66c1619a4f, 0x3076ca51f3aaf38f, - 0x3407b6de5a8f3d5f, 0x931ccd8d6bf40803, 0x47dbb5ec652d1e7f, 0xfb627469f3fe317b, - 0x449306112092638b, 0x54636bc2c9d1ca6f, 0x8d1c2dc8bed72888, 0x2acbe90f08f1b870, - 0xaec3c9c87fe503e2, 0x03242b7ed3b7f2ad, 0x6e1b9354e353fbea, 0xbb1f06175cd8f84b, - 0xfc477a485e66388c, 0x20dff23b4ba94a81, 0x7e1055c3d104dfc2, 0x687ee6b30aaa866e, - 0x5b7c99dd972c46d9, 0xf27d76e6a4844f8b, 0x40f756542438f0f7, 0x14cea037d4444eec, - 0xb3d22ebcd74f16a6, 0x573a3c8084c9e206, 0x7a9f640dfcf2494f, 0x8fa52ce262841958, - 0x7f6303258a83f0ef, 0x94b12b2ee8076336, 0x5ed50372a4862055, 0xe569b603158a3743, - 0x8912a2edeb93326d, 0x5226ebc15fea282b, 0x81a02d2d37f40ed5, 0xd3c3d79ae09dda77, - 0x0682b6aaf9882c30, 0x2a9fa81cd669e10e, 0x5b63582c5109a602, 0xe96179aa545d77ab, - 0x144f4fdb1b273033, 0xf4414167c7e6c623, 0xd506a64d6e715fa0, 0x2b375347f53db2d1, - 0xfcad4e913d566396, 0x9895479c3f36e6e7, 0x3856782ac89f5ff1, 0x4b1cc042725174f3, - 0xe02d2c940f420776, 0xcb6e96f97d87233b, 0xc0d86a6f563e75a8, 0x94afa2ae948fb45f, - 0xc96ec5bfee3158d3, 0x121e79c3a047f623, 0x78cd1d22c5affda5, 0x22340214e770b339, - 0xa93de594111248de, 0xc027965f0a8e0ee8, 0xa0b4cba44614eb5a, 0x791d6b3dc70e183a, - 0x8939846cab683102, 0x88d8b2f4dfaceb02, 0x808fa981df8f565b, 0x5c703f69c4f11d70, - 0x50dc549197d2dd01, 0x738907194f41c458, 0xd358152d35bf39f3, 0xd7df61065c7c7687, - 0x2ad1ea25e9150244, 0xb41954e82e3ce893, 0xd7eda3a2925d4af0, 0x85283c40dc88cf5d, - 0x8aef1f337f7e747c, 0x2996f94dbe63a81d, 0x7433b18e9494e5e1, 0x6e5263c3d1aa4244, - 0xccccd67d1ef73f16, 0x7a2ea4659603734d, 0xea98ba4ac4240094, 0x3df0eb501b02b971, - 0x02402b37f6d6e2a9, 0xfa8f08d60f245605, 0xb317085fee3fbfc8, 0x309cc8680d99e2a9, - 0x8da0d2c08d600c8a, 0x1abbc6923377248c, 0x0435825f53d2d2b1, 0xd9fcb1c5cae04350, - 0x173cbf10cf117044, 0x1569f7a19a006c66, 0x92fa264103c7c082, 0xf576b231f45fc211, - 0x4afb44d6e1fb3a9b, 0x232ee2b2a4915522, 0x63182a94cbc83c51, 0x67f8b806f9aabe3d, - 0xb82807965c57337f, 0x97ba0fac2e6025c8, 0x5f56c5490df9b7bd, 0x9b5821aefaaaa1ec, - 0x18c5c8ec4f9690c1, 0x6fd2c1e1d9301747, 0x984151c8044a03eb, 0x1cd8de31ccbf66aa, - 0xf244be03330373d3, 0xeb0f4ca97427bdfa, 0x253463caf56af7c7, 0x41192cf32f4b0488, - 0x5de4b902f5a93946, 0xb6397b02e91a7144, 0x749d302e14c79352, 0x9de28617439db971, - 0xf5cb93a751af04bd, 0x43541d2a9f3a3345, 0xd53f539dab61747f, 0x889351b692152e48, - 0x5ceb39c0c664980a, 0x66701f0cfa7d0f9d, 0x458b2806e815e32d, 0xd9b314f1344f131d, - 0xd42ddad5c3b7790b, 0xd5bf9e3ffc554f80, 0xe334bc9a34aac68c, 0xf83c092c90e79d79, - 0xab700f3851cb0e7a, 0xd50cf0da33081223, 0xb2095c1b6222d710, 0x9fd7758d736eaf02, - 0xccc54d6c789f27a1, 0xfa1d132ed500c76b, 0x5af57cb79e542a5a, 0xea2dad9c681995a8, - 0x1152fd728fdaf428, 0x1933d15b370017ce, 0x0c3b8713b3de4524, 0xb6636253c068bc90, - 0x393b4ab1cde3d4f0, 0x9b0ff94658d55fc5, 0xdb74ed2c7e83bff0, 0x825e17aacbe3fd9c, - 0x861e4a10df4a440e, 0x274a42f793ac0aaf, 0x1f896be08b02ae79, 0xad04e0dd5ce3d29b, - 0x21cc491071c39b0e, 0x7208411acd3ebfac, 0x3c0ade46c3dbd92e, 0x3b4fe1e5e8054ba4, - 0x052fab6e6d4e59cb, 0x10e73df4eb1b9339, 0xce9ac8bfc7ea5fc9, 0xb84195f2d80f2543, - 0x666c8b50d1ba647f, 0xe4bc07a528a934b9, 0xcfa2dee3802b1354, 0x7a3bda566e93911e, - 0x02fefa36b7d97d77, 0xad7bd60872af58e0, 0xaf0c12a50af73f72, 0x97136c831016da47, - 0x1f54d98a46137105, 0x6385f1b63f8d5549, 0xdeaf3362594b0e7d, 0xe6fa4ac541a86280, - 0x3d96fa62d610220e, 0xfb06eb2ea51580ac, 0x1b8b0411312bde3d, 0x3536721293601765, - 0x2199d17c542d0491, 0x086842fbb1368f10, 0xf55e0cdd1f39a30c, 0x724100a738703a25, - 0xe3eb3d1bdcc0a70a, 0x4cdde3f643b0f55f, 0xa524e7f0c4712762, 0xa85ba63a01d0b9d7, - 0x59d64b5252374820, 0xb520e3d036a95465, 0x8e2e1ccc8b315067, 0x2ce3e151a791a5dd, - 0x8b2fce0296dabedb, 0x44a0d86a0090a043, 0x8b5df9d21b256ff8, 0x6fa5b7284b5f3b07, - 0xb9f448d4408f663e, 0x21cf7d90ee4ff1ae, 0x2b41ca8e8c9b15c4, 0x7ada58f55afdf730, - 0xe9eb4fbab2e055c6, 0x5ba19c53e98d7fd4, 0x9219a65c80e1225c, 0xd851f26e21f16c00, - 0xb73d37eb3ab42b39, 0x9c75727ddd218189, 0x027a111cf1c205f3, 0x0bd471c980fa0cde, - 0x7324331e708c6e02, 0x88cb30b6ca3c5d56, 0xd2eaeb728a4183f0, 0xf8ed98253559aae7, - 0xfbf90ae24fe2ee1e, 0xa0f4f24f25304eb2, 0xc0e532cca01cdac8, 0xf2689535f1bb5af7, - 0xd2de06b033646bd3, 0xdae72894c62c8a1b, 0xeb537a60ca207b10, 0xb622581fdb02a435, - 0x868d02c3edd440aa, 0x96cd99ffe685fa4a, 0xb909e7c9d944e4f4, 0xe4c0d22238d9252d, - 0x4dea9df25c757b3e, 0x172673fc8e86eb1e, 0x5c3b1ff6ef436727, 0x5a678d066cee6f9a, - 0xd95d1e42832e012d, 0x80a1a9cd4ceafec7, 0x32966bd8b6d43c7f, 0x0b9037c79e9dd795, - 0x96a1015d3ffdd53a, 0x6268c3fb9d2137e7, 0x1cdea59c19d05e43, 0x1b05c6b4f3b3a967, - 0x8de1cc4c51b1c6d7, 0x6fb6aa624499d398, 0x033d2ffeb0e921ef, 0xa74979025fa5cd2d, - 0xfbbf24f65411ca75, 0xcceae9ec139d0c6a, 0xda0665323e453a4f, 0x9ca1317df18c55d0, - 0xcf293e4d9271341a, 0x66804893acd75220, 0x3ddbcf577730243d, 0x356201b348c51a2b, - 0xb950894fa89bcaef, 0x2c7b1a76503ee1ea, 0xf26e19f03fa8d7ec, 0xdeb0f4e8c2c3f6b8, - 0x218fd2169c416c5e, 0x0c2688841e16234b, 0x7bb732b75c9d240c, 0x009e7ad17423c7d6, - 0xa333ffa14bafdbaf, 0x85deed97ab497b48, 0x785247cf054e63a9, 0xe5ea0a98e922e2ad, - 0x848e65940ab1608b, 0xa826472b0a57b5df, 0x4308a118f32331d0, 0x40bfbc8aaa6ab3b6, - 0x57eafd96577cc875, 0x9c51087a194407ab, 0x2c70a410e228885a, 0x374c5c65c4fa9436, - 0x987bf35cd2f37241, 0x4e1e29f7cd0b1c0a, 0x8ed57f7bc704f9a2, 0x246d82f53542725f, - 0x8d9c9329430e4e3c, 0x4517e387fd4ba9c4, 0x24f238d17e54c8ea, 0xa833b9f01d9f3cec, - 0x98e9e4dcca156860, 0xfbbf911efd27e7d0, 0xa61dfdf093b7267b, 0x71ef96d8bfd4fc56, - 0x534d0941a9749043, 0xa2fc80e3957df85c, 0xdce4809338ba0b27, 0x96733b13958a6a98, - 0x1515c3a3e5ec6678, 0x76475102a6e6a382, 0xbd65032a734954a2, 0x6b2bf26209d3d461, - 0x55b8312a516340d0, 0xb40638dd0c824f20, 0x9a9d13f6bacdade7, 0x38ee4ce9cf91ebea, - 0xa3b46b8e8e16063b, 0x381bd493d0135286, 0x05c43fa2ab42cf7e, 0x5a0f643bb5c4b23c, - 0xb8e4d0f3d9af3031, 0x10f4e23e26c6d6ff, 0xa8f3bc02e249bf67, 0x0617ee244efb0d3c, - 0xe1bd0acd083d3ab3, 0x68c4cb0a6096d017, 0xaa7ef58e468fff4c, 0x162429da480351c1, - 0xd27b077798c56b93, 0x034f3a2541d02fc8, 0xfd0b91bbebacefe9, 0x516b0275280f33dd, - 0xff502f62cdd3347c, 0x649acaf6d34486c4, 0xd9fdbc8985332e5c, 0xb4c3983dcc53b60f, - 0x24fde3ececa0a64f, 0xe6411de26fdffdf2, 0x0a284f1c3c7c87f7, 0xda7df903e5a31381, - 0x80f8e52c2c348130, 0xdf8ca81fda5438fe, 0x3034475646f847b3, 0x5a2e3e4d2e72d851, - 0x9ad5d7834672945f, 0x68187f984ff6601c, 0xebbf9d7c6f601688, 0x835bf4b474e52317, - 0x1cf1cd5c7e1592fd, 0xd0264875a1ef5914, 0x50d13fe656a9a65b, 0x88315a08a7a9bc0d, - 0xfef795a3e399e164, 0xbc66d59ebded82b5, 0x4705706128157913, 0xe3d36cfda0338592, - 0x816a614718dbe31c, 0xcfd0f5da4040f1cc, 0xf0e48ab9f135ccd9, 0x6089a225ac961d50, - 0x717ed1cb8a31eb57, 0x0719978575e02008, 0x5dc0099e12028718, 0xfec4e48e30fce0c3, - 0xd8598df282864a50, 0xe6917d204a61cac2, 0x6ee04744efc0a109, 0xb9c568997ec2e7ac, - 0x8157337dea2696ef, 0xf2382785ab8e844a, 0x0e7d138547340712, 0x53c0a964c8f695b0, - 0x4b2a07217f55dbc0, 0xba81997911bdb57a, 0x14c1724385b3ee55, 0xcfa7fbe4e6ea68fb, - 0xf58cb3f615bfb6e7, 0x536211d90ac18715, 0xa0aa2196821e9105, 0xe43c7a8f0f2d2f66, - 0xcea4749cb3493c68, 0x3c519d60f14bc378, 0x625615c48e75184c, 0x2b97defc0038c71f, - 0xa5f674bf87bb0689, 0xad74cdbced6cc23c, 0x5e37d6db1c36b4a9, 0x87ae6ee11c55f1ef, - 0x1a7080365b6f660b, 0x7ffa9ae86b3393c7, 0x89e0094d5179004f, 0x6762096b6c020050, - 0x26e74ce68f776dc6, 0xfea5d61b8ca39718, 0x2435925b416802d9, 0x3ed2b1a778c52581, - 0x215362707b000d9f, 0x5c1199fcd7950a2e, 0x86ea933c7eed563b, 0xa283a0a109824898, - 0xce9749f223fa2d8a, 0xd0ed461fd92f7870, 0x2302ce198617a819, 0xe837580f3607e3c2, - 0xb2f89a91f26d0ddf, 0x2ca0c30e626a7022, 0x342054f2f310f3ee, 0x24e805ddb123c620, - 0xfe141088dbaaaf36, 0x3ab53506b017da5b, 0xff21cec90a2628a4, 0xf48bf1b049339734, - 0x5940716ad9752465, 0x67868406cce0a673, 0xe4058b4ecbfc5c95, 0x5af7babc7e6599cf, - 0x694914d46216e919, 0xe81e1d77f9e99302, 0x0600656063a7efe9, 0x24ed2753dd3e7cb7, - 0xb8fc6c5e14f3dd93, 0x8d7b7e470f27f865, 0xbadd150a4b804ff1, 0xde584745d72b71c7, - 0xcdb625248a3adb6c, 0x411982dd0dc5baec, 0x0859b74cb4eb1400, 0x44e89fbcfc06fb30, - 0x1327169c8bebfc36, 0x141604a1d480815a, 0x5062d3d193397f45, 0x5863e87a3388930c, - 0x46bb8e08f57a646c, 0xb78b8ad506373f94, 0xcbe4ff549cc1fcf5, 0x3306b628d44bbcc6, - 0xe567c7e01062ef5c, 0x81dc47adefd60baf, 0xe120663b7613bb35, 0x04a423aec9374864, - 0x1098eaa7d4ef3452, 0xe1bc2962fdc226b9, 0xa50fbd2669e5e890, 0x8d9886428ebb380c, - 0x45480d115286226d, 0x6beecc9fe7e8b5e3, 0x15fb2be14eccf081, 0xd13cfb44c747327f, - 0x0942457879c35a00, 0x7fba7287dc1e27dc, 0x5d26576f0a913d96, 0x9b1222b33597ff4b, - 0xe36713efd0b042fd, 0xbec94ccee5b452be, 0x45e5f4e913a26b35, 0xfe3f3aa2d49752b2, - 0x9fdd539f44535375, 0x42b0a298336ad01b, 0x399f56deb4f0e8f5, 0x7bf811dd67775d90, - 0xfbedf06cb050247b, 0x9488435e56faeb4f, 0xbe087abdc98f811e, 0xc53a6b5e9d59e6ad, - 0xc57b40704379b2b1, 0xbd3405c6f24fc071, 0x8ff8049195b29686, 0x51452598df604aa4, - 0x69897907e7e99a7d, 0x59b2a44ea560fee5, 0xc53d7a077d91dab7, 0x1c92164545e8bec8, - 0x7cfad2a5ca348c42, 0x048bc6047f3fdc13, 0x9de0123eb0cf42dc, 0xd8ac44081676f5f1, - 0x48dcc9bcec98523d, 0x608cfddf186cc8dd, 0xca363f5618602644, 0xb50dc1160f4c6e18, - 0x29b28c24d49e904c, 0x4381d207a2eebc5b, 0x3d1a6407679d9407, 0x18471c6e42ca77ed, - 0xd0d45054c3095faa, 0x22ed59e0afdc34bc, 0x2a89431a023349fb, 0xba132fd1f97a1e25, - 0x6413cecaeb6fe2bb, 0x1d50605be07e5c88, 0xb02adedc33eb691d, 0x816660e2dfc1950e, - 0x07524dbc2f583bbc, 0x85f4258c2cc4e3b4, 0xff58fed6377f0a65, 0xba3b588dcf68b1e9, - 0xf22669a9564366d8, 0x37990145214004af, 0x9b4a45863e5cf7ff, 0x42301e6270ea0d31, - 0xa5af766e3cbfc5f7, 0xe9b66cb9eb03302f, 0x879d0ac2ac5b1bbc, 0x8b39e3889b96e6e8, - 0x23c1cd2298acc08a, 0xa64ae5423a8035f1, 0x08ddce25138e56a9, 0x91de77ccb5be318c, - 0x18de21697cc9afc2, 0xb4963f35a443e4a4, 0xf8e8b75d3cace7f0, 0xed30eb7183e680c8, - 0x8a9398781da7d10c, 0xc58ce74132f603a9, 0xe4a3e7d5a2d532f9, 0xb77c183be762bec9, - 0x95d533081d7fd1a0, 0x42e9cfd75b760365, 0x727ce748190dfad5, 0x6afe8d5787b2f73c, - 0x120385e1947ca409, 0xbd630494b98b4952, 0x78b71456628d5a39, 0x8d6e335eb773d4a6, - 0x5a39ea26d92f16d5, 0x60c40971eb9649c2, 0xecf6cbe15d5a5cd8, 0xfe96381798d10a59, - 0x22a0c77b1aef6598, 0x85e2f83e29ea7e8e, 0xb0f9ccad71e193b9, 0x653fb1f5874e9633, - 0x9757950901017123, 0xb11ff05b66dac179, 0x6e1402f0cc5d32ff, 0xb398834d33268f54, - 0xadd650b1cdc104a9, 0xfcf86fd3dd750489, 0x3b5ae8b6f8acd850, 0x1bb440cd027766ad, - 0x002d4577042de42b, 0xee1ef14266661baa, 0xfe1525fd7e3d7148, 0x17c6aabd12e5a847, - 0x7658d832c0c61fb9, 0x6a8a6e46a969d42e, 0xb606b8b060f5cefc, 0xf8e1caa96b766bae, - 0xa12fcd3101d6cf14, 0x8d7b9c42e4b38807, 0x63fdcb3eb6b271ec, 0x462bc6072998d73a, - 0xbe671f5b6dc169f0, 0x4ef69196964782b9, 0xd5aa4826f58139d6, 0xc1c32f80f4b189e3, - 0xa0adf716c08bdf32, 0xc7bef5ab7b21c887, 0x54b12e06bdf5167d, 0xdbdf708a33c934e4, - 0x92687138f0941f63, 0x849db988c8595abd, 0x52b8015f79f5b4d0, 0xf7d207f9d6511583, - 0xb496bb2123c57ab1, 0x6e465f0a8a9695a1, 0x4d3d4e536221735d, 0x0772c763ab65c8a4, - 0x563432fa14359423, 0xe4687a2e0294202e, 0x8cbb1009cca4542f, 0x1e873964803d55ea, - 0x05f3904ff9dcf49a, 0x14f2695c1adebbae, 0x44d69434dd7d6e33, 0x51c34e2609736bbc, - 0x4053ea577d8b2281, 0xd096680b6b793f4f, 0x1c54a939b3c2cb2f, 0xac94471c33021ce6, - 0x40b6c60e4cf86bf9, 0x101642b894a1dac4, 0x2418f3cdee1e9bf3, 0x01aca956349cd4df, - 0xc61bdf5eb99b70b6, 0x15f856357c3ff5eb, 0xca038653fa2d7532, 0x68d182819961bdee, - 0xb4ebaf9649520574, 0x33bc86addefac51f, 0xc000be5715aa8f58, 0x93903ab1e2652a1c, - 0x14e0f32d919f5752, 0xdd424885b9718417, 0xb68c3cc207b593f2, 0x4c070f45facd6db0, - 0xf07b0201960fa212, 0x03418093f0f90ec4, 0x30d0a74a7dd522f5, 0xa81e5aafdacdbbe9, - 0x4552442f15aaddc7, 0xb8861366c2c78efd, 0x639f2a7e93ea4e8e, 0x9cb045439e203484, - 0xf7c0e4936d450041, 0xa9a9532ed584efbb, 0x9a3e3fe705013270, 0x7dd873794dfb57b4, - 0xf694e28131a3ffde, 0x9821cde284c9d337, 0xfc5e677aea803367, 0x0cba0e75923b7c63, - 0x3e775ef5779d8542, 0xb0f58795b17172bc, 0x721d1949156e298d, 0xa657b1675375e394, - 0x528b4353a3f3fbfe, 0xb63a21c9b5c62352, 0x576c2756dec31a1c, 0x15fb8c48eada71d8, - 0x2cd3c983a7758377, 0xf8d031f609cac75a, 0xc339942415f1fc30, 0xa259d61be67a62dc, - 0xb4ddd7a0add904a6, 0xbcda3bb2e41e4e40, 0xe230f206f2042801, 0xcf06cf90b25ce853, - 0xcc27e21286f32e8d, 0x73f4159fe1f076a3, 0x5381a41d999c926e, 0x33848e628cc35a8e, - 0xa0cc41e953ff4970, 0xa4b09bd12ea679d7, 0x7638843a9181c476, 0x15c5f2f7ab9e3210, - 0x89c4a176c9cfbe07, 0xe3fcb582d912c8bf, 0x22ef353dc470195a, 0x1d9998e2fa702cb9, - 0x5bcdf14c6d084b44, 0x636989bb7c069885, 0x748244fc7925ba3a, 0xe4c46231619fc8f1, - 0x39e902f04692d5d8, 0xb1057fbda14849d8, 0xd4da22f2ebfb3d61, 0x3ba15c3a205d5aed, - 0xa89eaa6bac803353, 0xc79bffc7b5e2b7d6, 0x0bd84aa7ab9976a2, 0xa17a63bcdc2c414c, - 0x91cd37075f428d77, 0x3aa3172bdce6a7b2, 0x87b5d3a3117e289c, 0x36d3d549f20f8f2b, - 0xb5786f4d02157a69, 0xb8f60336fd297aae, 0xd95081baddc5373b, 0x3af3ae7c2888c51a, - 0xe7bb8eac4e279e84, 0xc70e8ada525e5080, 0xe7facb1f353fbeeb, 0x2e6cf2729b462926, - 0x5ef9e7f338f74938, 0x06d5fc4af2449429, 0xf9b0ae0de7079f00, 0x65da2d7bea4fb163, - 0x4598cf75edcc987b, 0xf1f888304da0e097, 0xd3fc1ff884fe8739, 0x93a3be70679b8f79, - 0x94814abefafad386, 0x803374bb6cf2cf6c, 0x3ddd4ff4c23c2b1e, 0xdd2da3e718369eda, - 0xce0432cbb8819f88, 0x85a2780052dc48c0, 0xd11bb8383c7480dc, 0x1b0cd24812ef98e3, - 0x7f1f86d775e7c50b, 0xfe96402e64d4b997, 0xff82a58217db538b, 0x38a932bb103ef1a9, - 0xc5679f30892e0708, 0xd771edcf443dbd1e, 0xf966cffeccb85765, 0xe5290e52c718e31d, - 0x342363d29fd69964, 0xea7be0ab1c2ecf57, 0xf97ae957d871e0f7, 0x0abd3ff18ee438d0, - 0xb10c669a24245492, 0x73c39a402b012659, 0x2f6d7661c29120fb, 0xf4b897133e8fcec2, - 0xf4c74a715981e5f6, 0xafe55763709497bd, 0x305c049ab643c4d4, 0xb17480bcfe42f540, - 0x985f35e4bd3af61b, 0x1d2e1625e48fd92f, 0xd7fd2694324b3e81, 0x2958f94d77a49fcd, - 0xf75410ce6f3e80aa, 0xeb3af1ba3dbcaa2a, 0x43eaea98bb4f75c8, 0xb917f74fe8618474, - 0x0aca8a3d628be306, 0xfa70b45c0d7a42ea, 0x083855d12eef0c71, 0x64e58826f5edece9, - 0xc77bd2a77bc8c038, 0xed563a4c9a7af32e, 0xb2b3ded2cf70e778, 0x56a2b38c41f63f03, - 0x2335839deb3f4fce, 0xd2fa93ba1e75d341, 0x6efd22a8391d9471, 0x37433e699a7121a4, - 0xe74b02c817375418, 0xb67a1c59279e8c35, 0x5a9b8a05a63fc3c3, 0xed6dcf14ac520b90, - 0x5f76450c671d68ba, 0xd1f63bf6443d7acb, 0x44035d46bfe3ea5f, 0x5d182dc156825af8, - 0x2954163425c01125, 0xd662a677d6a977e4, 0x087c8cd9f2358289, 0x1dbdc745f112fdb3, - 0xf23497c2af80a766, 0xf3ca52929e651349, 0xb3713b768dcc5bf2, 0xdec18944392ffdbe, - 0xa87d7c4c68e7a200, 0x4baeb4782038fbe8, 0x10869dd35773638f, 0xacae9d2a92e81b95, - 0x4b1c2d2388d3d718, 0x419ae5ddec8a3179, 0xc15983e4d426c904, 0x17fb4fc51db3915e, - 0x367dae0c4e1b3e30, 0x12852febb3db40ec, 0x804d2a105a164ff9, 0x5f3b454f997c8d7f, - 0x551d578594efd18e, 0xdac4354132123894, 0x37e9e741ca764308, 0x7fc96e6bc728bd1f, - 0x962c5ef7fe5e2655, 0x484698383f926b0f, 0xbab00a75bb3b7994, 0xb9275c8eb4aa6c31, - 0x0729967323d94d09, 0x639133047a69bcfa, 0x8f62d2d2b3dbd2f1, 0xb11641fbb8e08f7f, - 0x3956d29c26275fe1, 0xb4094332b30ac0b2, 0x30f6f4a6cb8214a9, 0x35e924de91381760, - 0xca20992e0f46eb73, 0xc6ccd5abb63f5992, 0x19e772b7c2e72b27, 0xa394ddaccfcd498f, - 0x40e059259b38b094, 0xe92dc0bff344c255, 0x79834f31d041a068, 0xb482111de857794e, - 0x6a16795c77d762f3, 0x108803688653ccf8, 0xf95784a36a369239, 0x3c62d1ce54080aa6, - 0x95afa0b5df186040, 0x53e6b925bf8bf1d4, 0xa38fb75804e75847, 0xc5f5595d3bca54e8, - 0xe3fcf65fb5daec54, 0x88a685b0c5d56092, 0x050a127412b7b727, 0x4df5585c7cb05433, - 0xc7df3f36495a9750, 0xfbd4e08065d8eca1, 0x676b7201be00a256, 0x2efc46d7e65fbb93, - 0xa6dba7ec408ec36a, 0x9fa4753f84239b78, 0x5679114cf63cfaf9, 0xae9aee7e86c111e3, - 0x2416e0f5ac98a6b1, 0x3891a1fb26eeb795, 0x430e1a501c853583, 0x1dfde13cf4a3aa60, - 0x1e10d3d7bfb6d70b, 0x42ec836acf3c3a26, 0xe59d782da66c97c6, 0xd63d4bcee08123ec, - 0x92465743ea059036, 0x049bc2f80a5f3b73, 0xb90d6bdf8781eccb, 0x60188905eeb43e37, - 0xa16abbe6d4bf732c, 0xc7e8ca84b10ae98a, 0x527e0ef0563ebf75, 0xe2cfec39039380cf, - 0xaaea4f1d34fedcd4, 0xe7058061194e24f6, 0xab1de0726960d928, 0x1acd311cbea5ae0a, - 0x4ca9407e52c5e034, 0xcf53e10f855e8a30, 0x2d3b1b52714ae226, 0xbde90f333f173b10, - 0x211d3bec779e8e91, 0xbfb627137c8fc5a2, 0x6ffe48555ff4a29f, 0xc6a54af4d98a5c75, - 0xc234243d6595ec3b, 0x91eeaaea77dacff1, 0x3597b3dece5779b5, 0x736d231f8ba76735, - 0xfc4a9620e50b0914, 0x2659c1560db2899d, 0xe7692d9ca489af0d, 0xa459c0e8c2f5fb10, - 0xbfe4b4e5f6751730, 0x22dabce499da6110, 0x929984b25b78d8e7, 0x603e37274baa69b1, - 0x790d9e2565f7d39d, 0x9c8036429d29881d, 0x807ea3c2deb895a1, 0x831f387551df3ec3, - 0x296a025686666b9b, 0xedb362c693564732, 0x4957bb38a89fec4d, 0x7f8d9affe10daba0, - 0xc2b83bba55117de1, 0x0e91922cf183470c, 0xce86e813a00d0e24, 0x88cfed84077cdaca, - 0x8d29380f74df203a, 0x4b513e3aa8eaac09, 0x1baa3cda9e9127a1, 0x11716e6a97e603ec, - 0x4ec3c649071e169b, 0xc01840efc4410633, 0x3240d031d06010ce, 0x803a10e29abba413, - 0xfe32b278d4434cf8, 0x6419a23ec2122d7d, 0x09b32ff84d191bf4, 0x39fa3b5134571541, - 0x4cc65ee7a3e15852, 0x73c1e9b62cdc5d0b, 0xae3bc6949fd81c92, 0xc881027cf7a5f072, - 0x33921f5076466b8e, 0x88b7b8c4e1f0b954, 0xd927713061da90c8, 0x295519880617720c, - 0xd29273039ed76bc1, 0xd7fc543789de9d39, 0x85dda2f7c80b67ff, 0xad3dd7cc76193d7f, - 0x64fa9a68b4e64969, 0x18d4a8bb7ddbff0d, 0xa0c75c01cff4036d, 0x6dac61b68cb09d78, - 0x98ad15f5de78478f, 0xd17fe6265171c974, 0x834606efdcd46fac, 0xce272d63e9779091, - 0xf59fb74bbf312a64, 0xa35dac88c54a7f94, 0x7c506804b6c8ddef, 0x9f4299f83166fd4b, - 0x8fdf50639d270c71, 0x0bfd4e1bfc16dd24, 0x5068d1e21b93f10b, 0x396634b6b670c748, - 0xcb569a5aab2e9b63, 0x8e49acdd511066ca, 0x326e0faf97f605f6, 0x36dc5142fad3a6f6, - 0x4b18739d1438793d, 0xce24bcf7c6a0e737, 0x23ab9dacc99d29e8, 0x7ac6c3204b2394e2, - 0xa9b2a399533be59d, 0x85dd82481851c796, 0x81370cb36c842fb3, 0xcb96d602488b0a66, - 0x812cec179c82f682, 0x6c1c969704c068ff, 0xc7a0a7a466b025d3, 0x53e2e03a918291c5, - 0xcae3b7468e15b22f, 0xd864d317265e507a, 0x4455070a16ad3d1f, 0x5271ef291e63645c, - 0xae744d32bd077de8, 0x2bab0a20bc448724, 0x7a7603037defc31b, 0xf1b0d83ee8a784d5, - 0xc5f76fd72eb1ba8e, 0x00ccc8039f52f392, 0xa0551d68ea345642, 0xa63cd992f2b872aa, - 0x70217377f00140b3, 0xe87bf72cf9f7436b, 0x2dd37a4f3a71134f, 0x90649da9b08e73f6, - 0xef789a594789e927, 0xed5736c84d1a9144, 0x539939c978e8b9c0, 0xfa01cc5de74c7bb2, - 0xf8e1a32d576c319a, 0xb09947f4bdedd6ff, 0x2d356a3de988f7c5, 0x75a77333e380515c, - 0x96df85d3cf6969ae, 0xb3924b31f2ac6c91, 0xfd3d6f0b62a3d102, 0xacd27a7995cd700b, - 0xfdb31472477290dc, 0x60b17fee01521f14, 0x738de5136adfdf3a, 0x7c1a26be2b48ef2f, - 0x195e0875b16f2a90, 0x2d2da46515215dce, 0x385cf746c400bc15, 0x671f7853affd8f31, - 0xd1920e68ca62ee19, 0xa0d7a572f70b377c, 0xd3634685a1ffdf35, 0x7e1509fdf89ea8d2, - 0x0dacd804e6d549f9, 0x92b7a0659930a1f5, 0x6ddc4be657b46543, 0x0c357ef54ecc4ab5, - 0x7271925a542bcf29, 0x70fe456ef3e3166d, 0x949b432c58628353, 0x4668339ccd51265f, - 0x971f3d9d3700a7f4, 0x91d63ff7661755cd, 0x2af76fd68d21ae8d, 0xcd15047e7e823dce, - 0x012f2007c3e39a0d, 0x322d256bf75c0230, 0xff4affe8efe1d842, 0xd04e95638db2edc2, - 0x94154b27bdfde557, 0x61b3b7c43b2cc758, 0x222531faf652234d, 0xc355b61c54fa2cfa, - 0xf91565c9f4eac72b, 0xce9d89ce1d742df9, 0x94d2dabe62e8db25, 0x236d2db6f9cc7d1d, - 0x0d295827d74fb3e2, 0x911ba1741054d7cd, 0x143309ae2f1d93b7, 0xd5766f197a45a6c8, - 0xfa2844846bd185af, 0x7bf06f6e678e6c57, 0xe6fc3e728f8f5732, 0xd3d2800f8eb8f0d2, - 0xaa063eae4638785d, 0x20fdc73ad9bcdf95, 0x536ea729d69b9c5f, 0xd9f269eec63d2e2a, - 0x239efdbd200365a5, 0xf2737b81f937a248, 0x6fe629ed55d9f294, 0xc81672814576ec12, - 0xb3ee569e7b224acb, 0x9841e18719b66391, 0xe8f4920d2f3ec8b3, 0x1f2b1c24e0590521, - 0x2156506fdc4d0516, 0x699256830d309e73, 0x71252464ff7e3262, 0xa25a832ef2c25a2a, - 0xb207404de880ef95, 0x45389c6ac80defa6, 0xb3a2f41a82039584, 0xa652df951660f3ad, - 0x5ba15ba20d95e5ad, 0x3d93ce5d8d46056e, 0xf7da323462bc1755, 0x13b7e273159e7d43, - 0x7d1b21f2fb26006e, 0x38ed7c1ca1f46d1f, 0x932569245aa9460a, 0x8ba8a0bdce452a97, - 0xe4397b826f4f4e34, 0x30ca997526bb4425, 0x9d063a689d2fbdfb, 0xd67b2677045d794a, - 0x9a24daee119ddf12, 0x3a81ccb1ee967087, 0x3a5c63064098f1f8, 0x056fc8fbffe74350, - 0xfe4681d2ee112b46, 0x8da1b8edfa2c6a5f, 0x8c00ff94f00d44df, 0x5690bb60f6dbbc88, - 0xc8c92b783cad88b4, 0x5720d5bbfde86969, 0xa34adad079a5cb62, 0xdb54a0de46fb547b, - 0xe1a667286795ff6d, 0xe6bd8745cfd08dac, 0x3719cf0c6b716530, 0x357818b5bcb00420, - 0xe88cdd47a03c2797, 0x6d0b33eaf420065b, 0x7606291325bc807d, 0xc139849c1263cbca, - 0x6b5f1bcab6a21d41, 0x217a33991003460c, 0x3edcbb1377ece9b6, 0x1b8e73053ff35ab0, - 0xb4d2623590a614e3, 0xb13a4c1d79419c02, 0x39f981a92066a2d7, 0x7f0d94f9eade9997, - 0x2cd4e1c04b1f8c4e, 0x189b146a355c1146, 0xdcd88d6104de4e12, 0x43d087f8dd0bff46, - 0x27594c51e6fa040e, 0x7fdd31b5996c7c5c, 0x6b5db846de05df6f, 0xcaab3f9c8c55da5f, - 0x396f113c1562a5fc, 0xc19b63b74aed2ed0, 0x99a4a1e518b25220, 0x4225b3d90312279d, - 0x30e2816c0f54f367, 0x43e1936a7a772dcc, 0xc0cd19ccc21b87d4, 0x314d72bf6d8bb7e1, - 0xb0fd29e477323140, 0x4ec98ede270f202d, 0x3b7af53fea4a0623, 0x8821762529976f6b, - 0xf35dff3813ed954a, 0x722444ec24dfda7c, 0xefb4cb56c829004e, 0x40574b95411632f9, - 0x84c8b1d77f18b7fa, 0xebc9610c3a249238, 0x960731f3b7afa2ba, 0xf61a0a9e13950db4, - 0xaee4fee142d8aea1, 0xbd80b89b4224be1b, 0xe87aa30384a07432, 0x482fe22178851788, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5a8f38d7c83969d3, 0x802e5f11f1fe9d22, 0x971e2233b3706de1, 0x76b6337d587e2bd4, - 0xa06d4dab9d9ac34c, 0xfdd1886ebd1f51ef, 0x753f34e2259078f7, 0x8a15315b0cacf396, - 0x7df39c75450ed822, 0xd4cad8dd05432463, 0x2a63aa2cc7d11117, 0x281a9bae8be8cce4, - 0x0774dfbfc3578f86, 0xb30f0677670e4ad6, 0x9c9c7e0873cca208, 0xaca680550fb6b4ce, - 0x6c89af1a010bbf7a, 0x67ac01e0658d9387, 0x9b6e6d1e14b0454a, 0x97805fcb273874a4, - 0xa01f1cd26887a47f, 0xfd1fcfa70ab32902, 0xa7042e601d1e1a9b, 0xad0ea9e01017a3eb, - 0x6e9bee9009df37d6, 0x0882d06e46d5bbf4, 0x6aa0f3e26f2da5ad, 0x1249478fab46c914, - 0xb9a654c4f58aaa17, 0x4201586ef33c2220, 0xe960e5180d8511d1, 0x446b8c44ad5e8e51, - 0xb3986530d209cab9, 0xec6eb9239e6294cb, 0x6e85cfca53843b58, 0xa67675f0086bd6ab, - 0x10b68155186a319a, 0x05f8a1e9a94b3051, 0x0d9c996c6e481fb2, 0xa24d73b257f7d639, - 0x5fa05b22498059e9, 0xb8695c0809a98199, 0xdcbc496bb42d34c3, 0x8c819661e89bcb02, - 0x3ff9562affd8b702, 0x1e8c09e32d729c9b, 0x1939d65bd3b38fba, 0x871119fc0766d2c6, - 0x8e84a76f246504d9, 0xa0c3a71df353dd88, 0xa19ffa2870180346, 0x656a24e87545ae36, - 0x413da8429ef9893f, 0x05f3ab116db164c3, 0x344896ce075ea7f1, 0xa411c893772aed68, - 0x30297f640172d318, 0xef6784ec9a477091, 0x44ef2ee6203f324f, 0xbddbb72e16f0c0ad, - 0x7d748dfb2957f9bb, 0x17a4fdca60443ba0, 0x67c440e8bc64c48c, 0x58bf85f5bbb9c432, - 0xef308eed0c6b0f53, 0xf405b8138250575e, 0x69d0ab18297bd846, 0xb08aeb90d77f4c09, - 0xb8def247c2d31cd7, 0x905ea96433337d25, 0x57f541d40ddcf9f7, 0x4d5f02a52ce2c2e5, - 0xe48b9993f83fc7a1, 0x04204a46bf0db1b3, 0x1397479620bfa6e1, 0x3f327ccb1a65cb54, - 0x2b52c27fecce31c0, 0x9803ed3a8a9bf103, 0xe4b0508133af5c4b, 0xbf3974f50453edca, - 0x45103933f83e3ff4, 0x6333b523211f9884, 0x8ecb3b2eaec7df08, 0x16576ac0e468b71b, - 0xc8baed988a5df212, 0xb2cddaec863a445f, 0xaa5f8565ca886e80, 0xf0f4a4ff78ac86e7, - 0x4ee4480c11aa0bc3, 0x95ce14f718ed1dd7, 0x6743e3a391c9bdad, 0xd50a217d5797ddf2, - 0xa10c5b3e449fc989, 0x3e1c2822bd392db6, 0x46444223786c240f, 0x7ca67c8fdc61117b, - 0xa0ef242be7370743, 0x14649f37fd36321f, 0xc579ce78f526eafa, 0x4b9e803195fd047f, - 0x329712a3dea2e5f2, 0xe28fcc37f828c360, 0xf77109e010f722bf, 0xfea9211c86989b96, - 0x43c34a4a053fb266, 0x656438a3926650d7, 0xcf2a5fe730c95022, 0x467c0e0e11e8eba7, - 0x69876d6a16560608, 0xa82ddf61798381a0, 0xb429fcf0b6c5a8ec, 0x677ca223b219319e, - 0x2ff3559f66bf2644, 0x111f0da19555f56d, 0xf5952bf6c13c7920, 0xcb90c0292f38776b, - 0x44527e4f7bdadcca, 0xe349d06ec5435a0b, 0x7a8c54bb993b4773, 0xddeb22a41da24827, - 0x1caa15afe0a91cc1, 0x9566f81c81fcadbb, 0xcb204f1770a2b9fc, 0x18fd7e355d8e7341, - 0x42c72db7623698fc, 0x8c509238fcff2f83, 0x132ebc7f799f1e09, 0xc6815e870b13bd80, - 0xa9d39167f9b775d3, 0xb6bbce0669f760f1, 0x007b924931db01c8, 0xe5591a4e0a73ba60, - 0x20b60c96c6ca5770, 0xc412fba9bf8b54fc, 0xbb45ab8ae41c5932, 0x9f9d9f0fa06a6c9c, - 0x597bbd4e4492f2f1, 0x1e7e9e534c373251, 0xe84ac7d44cac8d1e, 0xe52e23c9d9a6d797, - 0xf0ebc7a50049f000, 0xe2085a49d9805ff1, 0x89522e9315dc03a7, 0x9482efdfa99898d5, - 0xcaaa99271d59a2fd, 0xf2b45757725718ab, 0x28b3bb2c718ac7fb, 0xb6abe034e1c0730a, - 0xc0361dd1d3ca1047, 0x3332a34ccfff9ab3, 0x5c3164a35800a6dc, 0xce50182d7221b1a5, - 0x3b0a370b459bfc5e, 0x14e1e321bd2fc7a3, 0xd4cfb094a0ef4d7e, 0x7cf290facc636612, - 0x28da92b1c7245d2b, 0xee440c9382db7ca7, 0x43c4255cf8eef53f, 0x2c746b017a72c9a5, - 0x8189bd8d2e388b3c, 0x67d550f7976bdbf3, 0x17dd654e05f1bcc8, 0x26549bc51624762f, - 0xb5659f35d4019b26, 0xe752652baa8022bc, 0xc18717cae8f7e4f4, 0xf9c226875c1ad55f, - 0x423907cf58a4cda4, 0x11ccc4542d41147b, 0x72dcc4a3cb72b87e, 0x7fa47d2e048b37b4, - 0xd8c52cca81a26ded, 0xc68d9ab477104d5e, 0xf48966245b3e7922, 0xaed2ae4dad6b296d, - 0x2053f2fb4542b014, 0x6ca82c87d680552e, 0xe001ccca4a9061f3, 0xd87cbea55540c14e, - 0x83e3a248e1273cf5, 0xad48e51131ddba12, 0xfefe9a67aa385551, 0x623f96ef8e63f1fc, - 0x79dd24b1c601a870, 0x2407afb34ad7599f, 0x3656065a8e9ebd7a, 0x4bebc12b185c526f, - 0x59ce4a0c01904cb9, 0xde317f727a17c006, 0xf32edf212d30772b, 0x310494c6e448ffa4, - 0x454b1feb93b91386, 0xd88c37b51da79bf3, 0x95055ef4f1ec57c3, 0x25690f62c0a106c2, - 0xd9e1e9b8353e77f7, 0xbdf62b686cfccc17, 0x6bb4600b5dafe3f4, 0x645b4429c72316a4, - 0x5f1b6d13fea0c57e, 0xd49293375bfc4287, 0x8dc9cf452eb315f6, 0x43f02146e402ae92, - 0x64139b4772c651af, 0x76147c1fa134c301, 0x27ec6172ee7701aa, 0x87ff850b5da7c22e, - 0x3303a1be5d9d5a3f, 0x42769b6234557c28, 0x4f12ca2bb28c3b15, 0x515743c347f0255c, - 0x0a16d1ba01c8a420, 0x906d648490444c29, 0xc76f05738039f31b, 0x36dcb048c5e59c4e, - 0x10e38882e9a7cb5a, 0x63bb65256d036222, 0xbf331a8796f9a8c8, 0xc51401129e0cde8c, - 0xf4088affc50183e7, 0x7c81485a28b2b365, 0x9799ebc6565f32ea, 0xf02e9e77fe5fd4f9, - 0xee7435d9fac186b2, 0x2391fefee84317e0, 0xdf3cee7f9f3483d1, 0xe0e36622dae692ea, - 0x2dbc3ccfe6874d28, 0x969b6b2b4f859f06, 0x12527c1cf0e1b0b5, 0xf3e5d5b7413151c1, - 0x0466dd6a317f71ad, 0x713cc3b94342040c, 0x0206084d9da44347, 0x7c4112f4120c66da, - 0x05cdf3a5f2687ebd, 0x9659ec97dd323618, 0x48361608b090d888, 0xe802a74c29d740f9, - 0x2316e647fd34d4a8, 0x936988bf50cb343f, 0x43a65ed5fc0e0e63, 0x1f660b2f34f216a4, - 0xc846e84f339ff9c0, 0xb0a0f58bdb8b3255, 0x58a3d6f0a05181ec, 0x6d850361e04ec675, - 0xbe072db0fd188206, 0x45af81ae1097ce4b, 0x762443e60ba0a6ec, 0x3b81ee490e49afcc, - 0x77b5743e2483a147, 0x0e8ad1158c4ddc8f, 0x842e45257bd9324d, 0x8a092a5d4cd7b6b7, - 0xdd125cef06b384f2, 0xdde39ae47b383de8, 0x4f1a234cb2fe8d6c, 0x0c9ff51c7966e227, - 0x3ecd0b06037bffc0, 0x865259176c28aebd, 0xb78747c05a74a855, 0xed07076603c5b728, - 0x178d9d71fe6dcb52, 0x288e33186289b93e, 0x6f8d5aff98f21e01, 0x5df68436b8887be7, - 0x2708aa6a6b13d411, 0x2522871878c58a40, 0xb1387a348c7e75b6, 0x8b8e1961cc11facf, - 0x7bb26841f5a1200a, 0x20ffc314c2e5382a, 0xed2b2ba4a706e121, 0xf19627f27b4e4f19, - 0x9b9c576226e873ed, 0xbf779d887fdd16bf, 0x927bb722910b05b0, 0x08c06aa16babb3d0, - 0x51340bba2d26140c, 0x7e6ef42585d2db70, 0x66ca5ec0087658a8, 0x19311e73fd5eb1cf, - 0xb73d1ee5a023e750, 0xc28f342fcc290e5a, 0xefd64bc50bd23692, 0xe5c5b0ec895fff99, - 0xc234c5d061b60636, 0x9518e09a68a6940e, 0xeb8d459aa0ee2583, 0xb6f2dacfdef12ec0, - 0x58362c4fe49e0806, 0xb856083b49f57d66, 0x0f5a0605a89738f6, 0x711a322ec291980c, - 0x2d308d25a59d014e, 0x4c4f346d139d3bb5, 0xe70e78bbe59bb1ab, 0x61aefa169367cf04, - 0xc048d5adccf3d1b4, 0x2576956c9110e514, 0x19aacdbe1e1cbb5c, 0x2d43e4f91e103e8b, - 0x7a9ad3342754acf0, 0x9c7b2f45452843a1, 0xc93cd83537076876, 0x919f2bf3faf17f07, - 0x3376c38959213e07, 0x5ed68817877ae98b, 0x7676de6c3a2c5c0d, 0x21ba16cfe41c8d8e, - 0xc364436d4acb7942, 0xc84892401584d42f, 0x5ebfc2fb0bdf0417, 0x74ef38c2081c51c3, - 0x2773b3e1184cff5d, 0x3073f64f41e6030f, 0x653d63b512ad2497, 0x13f7005893cce8f1, - 0xd088db0f22753537, 0x669bfc28cff7d343, 0x24d69e68e4f6421d, 0x696ea37ebcf003ce, - 0x2487163f8aaf0339, 0xd7302fe619c7fb65, 0xa300a75b6c23dc97, 0x794f4106a6716468, - 0xb37caa93463d9e7a, 0xc1f18dec332c899b, 0x1a6df77148a84025, 0x1734ce79d3285195, - 0xae5109918b3a605f, 0x21fc603edf4c4962, 0xc948041b7707a7b3, 0xa793588f1f36214a, - 0xcd2d6b2ab418f995, 0x86b57292a35213cb, 0xb5554abd5022a5da, 0xa74570a84ccb0f12, - 0x38414fda17654f52, 0x3ec3ccbccd924d1e, 0xebb4d22d312c2338, 0x4f141d6fa4f317bc, - 0x53c86326cf34b074, 0x79fce0df3e1f1131, 0x8d5ed8ef6bbbdb84, 0xdaa628a37ff71bc7, - 0x5d9c3cf3cf1e74e6, 0xc5f66eb2c601060f, 0x18bf736b345d4583, 0xd74749c687259084, - 0xa38fe4fbcbfac896, 0x854863f5222f63f3, 0x60de4eb8c2d350a9, 0x9e8b701ff4384ec3, - 0xa6baa14afdb6dc9e, 0x7ed9b30f58ca4371, 0x31c70e5002a41247, 0x07d3fcc787e04d6b, - 0x3f74088dd6b3575c, 0x223a329ff6d0a27b, 0x2b525b9ad8d9e5cc, 0x4023a209f2a42b61, - 0x0cacda59e4a1c5bf, 0x8abaffaafe754ae0, 0x6e0c9a59649076c2, 0x50814fca81ad1624, - 0x2c33bc818a8c71da, 0x5b92cfbb7c5a5bff, 0x4456b8973e1bfb4f, 0x54aa57168ef3c398, - 0xc3d1947f481af26d, 0x505639295fbfa7d9, 0x818343f8e4b3715b, 0x6dcc4a95146ecd24, - 0x19f96eabb6379fe9, 0x906e8407cb3ff96c, 0x7ea2889ee971c4b5, 0x09b9159e8982d450, - 0x7f0153dd5a3e7773, 0xd3481e11947344b4, 0x1d90d830fc20dca7, 0x2fb5449e2c337c2a, - 0x2693a37e1482a6fb, 0x7c47fa8bd018259d, 0x77d8bd52da6f3331, 0x0e9410c70cba0eb4, - 0x33e185ecce5d6ecb, 0x7bc1f5259b192091, 0x40bf012c71063498, 0x2006cddc4c931744, - 0x6356faffb909099c, 0x40bc37c4d27ef756, 0x25bb834160dba004, 0x11062da64c29ce71, - 0xe10b96ed0d6ea6c7, 0x9f4a11d1d4b3d9b1, 0xbe8b6cb9a6420d84, 0xd8b51c38d47ad27c, - 0x1353069334007af3, 0xe0778c8942513212, 0xdbd03f41e283b4b0, 0xc529525241981d0b, - 0xc707c825811c39c2, 0xa225d6315de413c3, 0xa35e67d7bbbd2404, 0xd09da87eb9005f1c, - 0xc59915ab1c0cdf3d, 0x1878fb1b6c3d4095, 0xc66bf3557b7d0768, 0xd262d2f80e19e848, - 0x3bf514dd7a7793e5, 0xe5fbda01c3ea2d31, 0x405659bc5a23b6ec, 0xc876dac24b5fe0c9, - 0x6c0cf720506e3f1a, 0x02530758876ccd0b, 0x4169fb5f48946519, 0xef2168f86dedb9a1, - 0x9072e40c40eadf5b, 0xe46702e0e6b86df5, 0xa4bb530ad84c8dce, 0xd4500beca3d23fd8, - 0x1132f0ca01b2ce12, 0x8967e70952c380fa, 0xefcc2cdd600e711f, 0x49e77942c5f7ec5c, - 0x278cdbcd1815b710, 0xbc2be6a65912e5b3, 0x7dc57eb56897ed38, 0x54cccdbeaa880005, - 0xca029935658d7e87, 0x40d3581995dab3ee, 0x5e145c6c148cd517, 0x3c3072abb35a13fd, - 0x3abdb313a7d27412, 0x28d61f1ae138e577, 0x578cd5ad3b7ffb40, 0x5104bd71526e24b8, - 0x37f50895d3ffe1c7, 0x1777d244fcf50d21, 0xd31e3b0eb559137b, 0xaa9ca1e3e2dfb4b5, - 0xfec8ba5d7d713cbb, 0x0802bfb4b5e508e6, 0x5532a4c4e6c958e3, 0xac0764b60aa4654a, - 0x5db2dc891c3da857, 0x500f3268259916b5, 0xf9f1c5e183bf6579, 0x8e88051966b3590b, - 0xf8514a2114f13889, 0xdfe96cfbf7eda812, 0xb8f0c8ada4891771, 0x303849195b40e0e5, - 0xab12b9f86d64c6db, 0x7c837967f717e36a, 0x6ed11d7f57f699fc, 0xe851c0a451b09b4d, - 0xedac45b6e0fd43a1, 0xeb0e8e82f87f3797, 0x6214d0aef4cc69ae, 0x1fff67c043b8ef86, - 0xab459dd5cbfb4319, 0x40f2a299e72d864e, 0x9a1066002c90d6e9, 0x9d5892fe14861283, - 0xcd7befc7a9bf1771, 0x23f76803e8a6126f, 0x4c2dd1b0208996ac, 0x74b541ea47bbf3aa, - 0x7715056d79688d83, 0x80009d567055bab9, 0x5f9afe349037d0b4, 0x8ae19f35ebacb0c1, - 0xa43402bfa204a95d, 0x268f0d383e9c1b93, 0x1d4e07020801c816, 0x490e29b83c45fbeb, - 0x30e8bd05ff88ab69, 0x1ac9535a65f9a145, 0x080fa85d0fef2686, 0xbbda9395e6d33d56, - 0xfd704455b8317f69, 0x236a9d970f14f33b, 0xa874b98b74cad0bf, 0x661c23055338942d, - 0xc38522030142fc0e, 0xb0666b3cf3de00a9, 0x2bbc83d93c022c02, 0x40328e6352e1f6a4, - 0xb65224ada6e937d9, 0xdf3ece37f2bc5640, 0x258d16a23e6f1a7b, 0x18213da5f118da95, - 0x204e9069a247ae04, 0xcc860fff14d38882, 0x5f6495d2e0dd4d6a, 0xf51382a0c27a4b31, - 0xb14ad7e2af0b5e7d, 0x1465515ec3e8d3ff, 0x1f1ecbc9ff4a65bb, 0xd56bbf8d47594bbb, - 0x00d84f84841fc380, 0x18bc44e5998c7449, 0x1a822ec9feb68448, 0xeb8c07d2883ad17d, - 0xcb6cecbe86169616, 0xea68a9ec6de26ea1, 0xcb59e4e433bde29e, 0x94267594624a02a3, - 0xb22e3ea3e164ec33, 0xa72451ff924f6929, 0xf6ec77a0893c9666, 0x688e46d8f9c4b18d, - 0x7373566eeeb5a523, 0x8cb58b6c7c34c653, 0x586713f10bce5421, 0xf18a85d89927e456, - 0xd1643d38ecfa68be, 0xbfba323f47f0a283, 0x7e68df6426316a9f, 0x284512e196f1534d, - 0x4709dfead3bc3932, 0xad39edca33770372, 0x72c3b5fe34ace038, 0x9bdc1a6a90a0524a, - 0xac4a8b16a0bca00f, 0xbf954e5e019e7290, 0xd933dd98ebc426ab, 0x255525add4fcd6bd, - 0x7a5fe2594fb9fa0b, 0x54241cf2474c2c90, 0x0e0a871d8746203f, 0xe5445d03fbe27d56, - 0x7db2739c87c25739, 0xb1aee3a6c37968f4, 0x2c55a8063a571e07, 0x74f2ae3dcee04210, - 0xad7b7437811f29c7, 0x5e85b5aec474c75f, 0x6c45946b28ea9f04, 0x06b392322121f006, - 0x72f21636e3fc2ae0, 0xf556df5f2d2c097c, 0xafe79584222f947d, 0x613c3c83efaca8fe, - 0xb763d046dc8c7762, 0x5c4691b45d1e1ef2, 0xda5d31692eb923fd, 0xfd291914d730b2a6, - 0x837dccc6b85a3ad4, 0x6e8235314e75bee9, 0xec55cb9f9e9ba698, 0x141f2e0ed7ff330a, - 0x441b3689a00022c1, 0xe6b3af4b35f0e590, 0x237f4a382612dac2, 0xc1c90aaf5db4a716, - 0xec09cdf30fd517e1, 0xe7dbe012bcee6e49, 0xaf644224e25b044c, 0x7de3afb3281108cb, - 0xe33a614b8f0a6a1b, 0xc898ce1a4f685c58, 0x81ccb769d3a234c6, 0x8aba86cf5e2f5b1e, - 0xc04d662dd3f02f50, 0xbe83cf3e09074397, 0x5b446bc638af6b94, 0x7d011d68a2acc7f9, - 0x8e1633f53e3519c1, 0x5732eb7d9c7d8498, 0x92c6cb49de770a30, 0xb01b1a866a0ee488, - 0x4be26820b829c84f, 0x9e41fec37bbc7bf2, 0x667ffdc38d06abad, 0xaee3439f10115c78, - 0xb40bea9328850277, 0x13a974190d983425, 0x85e091ea5e51bc56, 0x9acb103e60362d7e, - 0xed6e667d25fc5028, 0xb404cb49ca842f00, 0xec76e5706af56fae, 0x4eabe8552a3d02b3, - 0x62137632b42b16cf, 0xc1b611bff135b71e, 0x9ed2a71991bdb60a, 0x808289c7fe10ea9c, - 0xc8feeaa13c7f9c94, 0xc48c1cb835476a0b, 0x42fd27c7a9fec5c5, 0x73fd0baba549b816, - 0x7a242770e8249c32, 0x093e8ea4554c5ba4, 0x12059b06b0aa2970, 0x92717a6e7783c3a6, - 0xc629716dfdc2b374, 0xb9b2696bd7a7cdcb, 0xfa32051b4446db9a, 0x1a47ef6c22124454, - 0x8bfaf86efed90edb, 0x9ed8e906be96639d, 0xa480497aa2ec9db9, 0x62bd4168fb61f987, - 0xe57d1eded7404879, 0xbd054fac50b106c4, 0x4f338ea341707b77, 0xff326644b682342f, - 0x738b4861e3fe6150, 0x5a2a74f9960eb4ee, 0x1abbe49f664c95f6, 0x23c51013fe81bd1d, - 0x98595c44c46ce46e, 0xa45d9fcc4978456a, 0x32b5623985087979, 0x042c65da05d6bf8d, - 0xa36eaf84a36716ff, 0x19f88d4d5170c42d, 0xff41b43ffd84d9db, 0xff70026eff81f506, - 0x8e1dadffe084d470, 0xde0e3483646e4a30, 0x78b831fe371279e7, 0x57d212fa85d3f179, - 0xacd0f4a17ba8d7ec, 0x25626805833bfff9, 0x23fbc0a201b94594, 0x44d61f3e0f0ec90c, - 0x4bf165c54ac44612, 0xfeda57e02d889c1e, 0x1c05672eeeac590e, 0x9cc7745a403cf70e, - 0xa26e301fceccc9d3, 0x7c92d5e714294f0f, 0x25a5888684785341, 0x1cd4e8b55757e8d0, - 0x7b4379e33c74d00a, 0xe9e47e935cfdff0c, 0x4420cf14a5a29c7e, 0xd1d925f17bde964a, - 0x2a3a4f0dd47104d6, 0x094fb8df369db2a0, 0x1446eed7c8791b33, 0xc22943f19b3c2819, - 0x5a9067bfadb02a4a, 0x58cd029f630fa3fd, 0x6b80e747a344ee9f, 0x9bbb3ac743fc58e8, - 0x824c576651dd9f6d, 0x244e7073f019f145, 0xb4ab725a5911e094, 0x6d0d4bfa3ebcb5b7, - 0xc62ebc2991eb17f7, 0x966c218b15e332e6, 0x06e7a259c30f2179, 0xfebc0b7f318d3238, - 0xdbcaa9a457f66834, 0x05d70ad551ae7eb9, 0xf133d62bbf0571f2, 0x1d7f4a014995ba3f, - 0x0526136313e8b163, 0x9184c7ecfeec77af, 0xb10c9263ea291425, 0x776b03afe1d01a3c, - 0x054f7b391f74fc7c, 0xce20af538b790e5e, 0xda88cdd75427add7, 0x156f568358b960dc, - 0x3feb26e9c25a9d53, 0x846e9905f3fe80ff, 0xe3ee528ef8cc3502, 0xbba05ac23c64a7ee, - 0xe47f7c43ec2b1d36, 0xa8bd433a9aeae249, 0x70e5a77a4d14a148, 0xb471f597892f7c1d, - 0xc265bd1fffcd8a1e, 0xa26ea2ff459be5bd, 0x42f24eae9cb4846a, 0x1731421e9dacca95, - 0x8a95eab877910a0f, 0x5b46c2849bff6c36, 0x019cc1ae359dba37, 0x989386735920f566, - 0xb4f595fba50afabe, 0xbf5a867bfaf16e54, 0x5956ddbc040967a2, 0xe76d38b66ab5185c, - 0x407083213b60061f, 0x2ff5331f21e84e78, 0x1385f9a2e062f663, 0xe089a5012dfa7836, - 0xd48425899d738154, 0x19a9886481b5fdcc, 0x3d8afb14ec3228c4, 0x6dbf6520721f3975, - 0x529fd4b0a0fd3e95, 0xc12666f91c690f16, 0x013838094571c97b, 0x32b6ea151c258207, - 0xd3081a7593261070, 0xb3bf7fb994a5dec5, 0xd48f08115e60431e, 0x89c8d7b87e75e179, - 0x0c06f1abd7341950, 0x5372de0a0c052f48, 0x042903485cf3cee4, 0x9fbc57cee8606755, - 0x669acbafde5e023e, 0x435ca0c1e7f4563e, 0x81f1df9d33ca246a, 0x64df2aede26ddfaa, - 0x3f205a3a5e1ffe59, 0x7ef7bf95272eca03, 0x16f2db280e97e824, 0xa886d13b9ffae026, - 0xba6b60efe389cc08, 0xe3553897a3c93d7b, 0xd61b32cab765d37f, 0x518045c5336c84d0, - 0xeaba43b306c9bba0, 0x6ac2e9edf51b0d28, 0xfe7d09c07b0458b1, 0x5fab8c351168a5d0, - 0x3f4739df6b6a660e, 0x037059edd077a5ba, 0xbfc4404f56b81e8e, 0x33ba42db8f090883, - 0xf2dc351b24bdf517, 0x2d99b6ee202b117e, 0xf4a72c88bed1ccee, 0x9598993378c910db, - 0x0aafa50fe53120b0, 0xe1191ccf25000935, 0x7e59133c34b8a78e, 0x6b15280d7bb2c9f9, - 0xc93510f40acbacaa, 0x24a90b3d51d866d7, 0x3abd1f613b947a23, 0xc1a0aaf79d38a599, - 0xaa63b2fbe976be34, 0x5132304cacc5dbe7, 0x3c4616ff90a119b2, 0x08ada3d4af83e236, - 0x2e68e44b9de2a20a, 0xebeeced3612399a5, 0xcb0ca7d00fae8fe9, 0xc1d650809aa65ff1, - 0xca82d28873de0eee, 0x705c7d54aff18927, 0x1db14f4ccacb213c, 0xf2183d461a76062e, - 0x24b3a4abad3eba8f, 0x3bd775d4017e7b46, 0xa9999e80ce0d5aae, 0x28223281522e7f20, - 0x952359ae716680e9, 0xae90fc1e6548cd2e, 0xafac577b0390e48a, 0x89bbc1da1365e680, - 0xac581017531d487d, 0x529de57e8531636b, 0xaa319c7c0f14ebba, 0xe85977e3e82a1e7e, - 0xc8dfc65ba8fa441d, 0x0436e9d8cc62da0f, 0x94b538042e33ab95, 0xd8ac919fa145977d, - 0xf8f6c813fc1d8f8d, 0x71aa806c2dfe47bb, 0x386e7e59d9f6272a, 0xc66e0baf95c9f4e2, - 0x8b037a047244c22c, 0x0b9533279f164d0e, 0x1f7631e19f418b98, 0x11df16118ff023fd, - 0xb26d45934baadb3f, 0x1396a135042a20ba, 0x588e04ff0c1cb6a0, 0xecc863ad24dd3f25, - 0x90cd642b8d2d2298, 0x051e49d21c050d18, 0xba8ff0850c4fd327, 0x908fbf331d62f607, - 0x2522a665f93ba253, 0x78ef22d91c5ce31c, 0x0464f0de665ca6a7, 0x7f3d6aa1022e7f4f, - 0x59ee5c01dba4a943, 0x1a811bdb69f7781f, 0xf572f2a04d9ce107, 0x7d0d6d0773bcf042, - 0xfc1fb30c63c6ac31, 0x835003f3d280b7f8, 0x2cb2d8419365dda5, 0x9d66c681d8ba07ef, - 0x2647bb1d49ba2768, 0x5b8de63465bc198d, 0x00a6795b3b51fa03, 0x0d10ea05eb78c0a1, - 0x1fa9bb055e3c5054, 0x289e437ba23e1112, 0x486e80d9c0bf3583, 0x43e18580cc8de7b5, - 0x5bc825de6fdb99f8, 0x7e0dae59fd7c7a82, 0xa270e15266b61e1f, 0xd200c2e375ff89ef, - 0xafbb1c0ce94759df, 0xc08b2ef1a2e9616b, 0x6abc02c7dff91d42, 0x8a4e1a6f3b58204c, - 0x7e55a82dc9ac84ca, 0x022629a8d52258af, 0x39e1fb67b1d5c25e, 0xfd44a13bdfba9111, - 0x9320c444b60d5365, 0x1812473d81c4e7ba, 0xb17145cc882d4160, 0x62965c62b6920226, - 0x452a3a575c9ca854, 0xd171e056eb206966, 0xb89e75c404e7894f, 0x955cd26b7b087efb, - 0xa2229326ca7f8e9d, 0x2cde0e727576f960, 0x65407eb03f603e51, 0xc63f98a0b2f8f695, - 0xbe02efc403b9d018, 0x0d2cf573d88dbc94, 0xd5aee58f43511c68, 0xfc636bea384cd2e7, - 0xc173b35a9f32f84c, 0x79c372aaf51aea95, 0xf08cf8194f84b0bd, 0x9a27134391645d47, - 0xd9c05ac05c41d69c, 0x63295f87db458101, 0x564576229aacac0e, 0xcbe55d5d0aa1f37f, - 0xe8acdfe640f662e2, 0x7f60f951af98676b, 0x4c9f8beea2158934, 0xbc6312069197551a, - 0x2541738968f6c627, 0x4bedb94a827d6b16, 0xe9ab03b64ed56185, 0xd2da92941a1cf040, - 0x1edc93076739af4d, 0x76c50fdb1ab46788, 0xdc9a1960e1adb49c, 0x931e7ac1a013a16e, - 0x850377df32d28d92, 0x601442199e0e8182, 0x8aa6f44707b1fcf4, 0xc6efc079f8949bb1, - 0x682cfd08ab3796c8, 0x6f79f6c65f53c3e6, 0x122b6b77dc6bbbfc, 0x87f2eec4f07132bf, - 0x8514f49ecaebd80d, 0xb4db05bc564a35ca, 0xa162a8d4ee31153a, 0xc9eb645bf966f3e9, - 0xce6cbaf9b862e038, 0x791cb77f15ed7046, 0x6e2cb37ffe7ee712, 0x0227febf9e6a3b61, - 0x6a2128e6959c4679, 0x4f1d813a81424c90, 0xf52fa9df49704172, 0x9fa166d2f65b821f, - 0x6da0943b16ddabc9, 0xf88559bd988e348a, 0xf76f0127896184ff, 0x95252f1ad4422707, - 0x09815eaa25cf5183, 0x46cfc7999ff99e33, 0x62836e6503342db6, 0x4440908e898f418d, - 0x5531d5af9914f1a0, 0x66df770cd09fd3c7, 0x071fe3935e69acd9, 0x0c2d24843199cf74, - 0x07cc5c64a5428620, 0xfbacfe0c1da37fda, 0x5dbee68f873aaeff, 0x04c7243990ebeaf9, - 0x42a2efdc27ff1cc0, 0x3ad2f1d23755e09c, 0x089a6b8ef9f042dc, 0x0bf85598e46f0bd2, - 0xaa2d0bc66c2abfcb, 0x3918667a23a9b953, 0x35f80edf06cbb9a1, 0x19d24700054f857e, - 0xd7ce9a679a9b9233, 0xfbf58fd4f8601491, 0x26274e6dc91be97d, 0xa21aaa9ee84fb5df, - 0x7ed96ec6804c93ca, 0x7467497c6c3eca1a, 0x9071b686d9e94ff3, 0x7c7c05fd0afd2941, - 0xd32265f68cf9c91b, 0x4f69a15be95d241b, 0x6d0e444b739de6c5, 0x887812c45f50f839, - 0xc4066703eb818ca2, 0x0b95ec7b2402ff83, 0x5a849228514e574a, 0x75d4684b6cdb7dd6, - 0x5874cc58a41efdb4, 0xf4c880e9035cc354, 0x28702e2a54d8a168, 0xa88209c224e24e8d, - 0x7ebb3602a8e31f33, 0x9455e2035c054bd6, 0xb01a8668f7185cd8, 0xbd4fdadc2c9eedd2, - 0xa44a6fddd7e3427a, 0x949e31d09cbaa5f7, 0x8774f9a07314121d, 0xafff6fa48d05011d, - 0x10a267c1f4fc9e37, 0xdcfabbe6b20afb6b, 0x14fe314458d82ee2, 0x2108b7b82b6e21da, - 0x4d82823f187b6ab4, 0x5c950756d833e69e, 0x5e42d0d8f3dbc86e, 0x2a2395b30845fcdc, - 0x9fb69e177b3fc478, 0xde37e130dcc17cc4, 0xc5e224e643694a1b, 0xd50175a39f7b2384, - 0xb6152b54dbd3908a, 0x65b6cf1c2f1e5a99, 0x56d63a93000fb88d, 0xbd8d5851b4a9792c, - 0x9bbd779e633b9ba6, 0xc7d0f1b26fae40b2, 0x26898f51edaa4a04, 0x71ff9fdbd6a9bca1, - 0x47c43d2ae7cf3285, 0xfcfc6f8cda2c3c8f, 0x511892a0cfcd7c91, 0xf86d06529830dee7, - 0x39b5e6fd35c1d479, 0xf3d88244ab4dfd20, 0x57625726a2d9718b, 0x9e0b2877d30ca32e, - 0xf0fdf1b1ea5a17d2, 0xb76b8b4c5a4fed9d, 0x3b51e1fce7baca8d, 0x0591f2b2d06e9675, - 0xfe3792673c245535, 0xad6f0291f6393cc4, 0xa7dc3c63d7ba64f6, 0xab76c7fd3fbef7d8, - 0xf6c28386dfaa3992, 0x7ab84be596a6a314, 0x636399d0c840f7f4, 0x3941c136c9e63152, - 0x92af191e855af3a8, 0xba0aab2f37eac87a, 0xb8e137128644abf0, 0x7361b057c5a68dc1, - 0x1d8b8ec4386410b4, 0xd734afeda7e55de1, 0xe53b938ddedfb61e, 0x6df0edab3960396d, - 0x3344ce9a52398f03, 0xe4af4f8b8b19183c, 0x6f3920ceba55ec91, 0x404feb193e820500, - 0x769258611c63daf8, 0x73c2fb7b121d9d11, 0xce0bcc1ada4fb28a, 0x66700335b1880bdb, - 0xdf93464b3b3090b5, 0xa8143f86b03c7530, 0x40cf93ac20e27350, 0x11a6b7b9c9bd8f33, - 0xb11ae12174aaba44, 0x6df5c15d241b1526, 0x26210453901bc0b4, 0x4c09e446d26891a5, - 0xb3af71672117ed27, 0x6263b478fcd29939, 0x731cd17213ad51f4, 0x0f771fb203e87267, - 0x42b2b3d72e154f6b, 0xf6482d16d11beab2, 0xec5563b6a9fbaee2, 0x0a920e157735c24b, - 0x571577d039198db3, 0x919eef1000230663, 0x6a11beef43844123, 0xa9e75a2b9077889a, - 0x561bd98091c73dbb, 0x13cd0132ef8bb3bf, 0xd13e3125037177e4, 0xbb68eeb3fcacdfa8, - 0xca2b44c9fd52e31e, 0xf18789aaec003760, 0xe5fecf334446db4f, 0x73766766c8e467b6, - 0x11a4bf4bf0030644, 0x464443dab5e07bff, 0x879b7edc2bd9fab2, 0x3d6198c188c573aa, - 0xa2dc2394736482b4, 0x95ef17d7909fce82, 0x437f6fe66945163b, 0x896b821d869b79a3, - 0x313091ee36578349, 0x9bb5f920e1a8b634, 0x020f119377514111, 0x2f4e19c9d49ea219, - 0xb54094136171169e, 0x1f56cb7ed51c1f70, 0x325ca7d0ac3ed474, 0xaa880e79900ac4dc, - 0xf92386e8b623bad6, 0xdac9073f5a07eab5, 0x151398e86a7e8397, 0x3f8c7ae3095175c4, - 0x95ed93a654189abf, 0x4dc1b4c37b377ed1, 0x046df154bf4db546, 0xe38bee4f0aaa8020, - 0x4471872210e63387, 0xf8db82de694148e5, 0xda0705a5f4ee7963, 0x653ba9c69aa6296a, - 0xc68e4528934d8353, 0x5c8dde85524fb05a, 0x72ba2a50a3545712, 0x55a64eb6a2918b87, - 0x36e43f19dc610449, 0x5d1d383a4720a999, 0x8af7df437f659fae, 0x64e75e4e8338dc99, - 0xf706793982d889ae, 0x29597f0b95443317, 0x54390ba328167a02, 0x5933eaf4f5f79f29, - 0x0d080e527726af1f, 0xbac6b1cb40a3a50f, 0xc921a49f5689b8d7, 0x8b6adfc772c30612, - 0x45e8721c15c4ba3d, 0xed67b96d9bde4160, 0x48ec4fbcc55c8a25, 0xfe0c43579e09e816, - 0xd6428ca2fa59d0c0, 0x6f34ecd684e1cd8a, 0xa607d15eea7528f5, 0xe72acf0721b567bd, - 0x6813927489c5ed96, 0xe9466b133568fea2, 0x7a11c59b431950f6, 0xdfaffb86826c50e2, - 0x310b389c908702ac, 0xa0835e820d7d2581, 0x2da4480ff1da24d4, 0xfca491fcb67b33cf, - 0x4e1f477159fa8ae9, 0xfd9837b31a14dbd2, 0x53c78d2eeaf597c6, 0xc4f562cdc87aeec8, - 0xd6ae3577eddd97f2, 0x0e466ef38a5a3dc7, 0xf8b57545d0bd7152, 0xcec91bba479b0103, - 0x5f76c2366cd77dfd, 0x01d6dd50adcdd9c6, 0xf3d6e5c217e41c5b, 0x061c765ef74a0c8b, - 0x4e6767db8a1761b4, 0xf6b46bf600122476, 0xdaa86bf838b5a5dc, 0x3af742c020ff68db, - 0x72989e359dd78a6c, 0x6addf861d499b5af, 0xecf97268566a377f, 0xc093bf48a727ace0, - 0x12ea718e66ef41a3, 0x05f36e19a7a83436, 0x731ac21042a99e84, 0xbc9ad524e30c808a, - 0xe1d6534d24bec04f, 0x5cf783fd38065ec7, 0x168f91f83c92e424, 0x374a20fa9ba6924b, - 0xd19dda74b8cc89e1, 0x58e6a9563e8ca842, 0x900936600bdc1457, 0x4bdc076af2d24664, - 0x2ff72053333df7e2, 0x00bb1984c30eeee3, 0xe1c9c3b8d49e7616, 0x13c576c4e216c474, - 0x76b8ef455b215345, 0x8316e7887e9f9ebd, 0xf6f2bb8e9db5ee9b, 0x151a4a7fec7de520, - 0x8ebc57cb66a6061b, 0xaa5a54fa0b3ba61d, 0x2fad664039a69c9f, 0xbd68bcee9235e4f5, - 0xf31c4a0e73026025, 0x151bd5423a5483e2, 0x9d50123c8314233a, 0x3cdfe20d69d5177c, - 0x8355d90500d515e5, 0x4e532429edc16e2f, 0x6660026418cff1bc, 0x7ffed9328ec8224d, - 0x0f7500963d9a9f39, 0x11b0c2f43c91bf13, 0x075b8d5265aef3ff, 0x9acb65e607deb8c6, - 0x3d6bc913babb4390, 0xd6e75a350721e493, 0x40f93a3a6eee0155, 0x0adaf785c8913186, - 0x619c4d83f2e2e30e, 0x3890808f8c0f0ac7, 0x1fbcee0083866432, 0xdcc32c27ffd1aec5, - 0x847d7e75acf2b861, 0x3809337fead1dc44, 0x0eb5e4fb6951638e, 0xdeb7356b149ea4d5, - 0x372fcedb34fb150d, 0xc11d244e475200fb, 0xf8e9549c6ef8bfee, 0x64fe59dc1137167d, - 0x5c941b7b20199cc3, 0xfe96c84c9d460f46, 0xa7be8ea8f6aab575, 0x39ac5c159b2f177b, - 0x2344f7a080e6c2d6, 0x85b97e735dd1dadd, 0x68bfd4867773758d, 0x53146b1cc89d10e4, - 0x0537c51f25856164, 0x296d9a2ba083a835, 0x8f7d7a54fa69bf4b, 0x919d9b99f9a72bf3, - 0xa7ff60209930a276, 0xae8d183f6cf2b9e6, 0x64679511c28551fc, 0x1f211f947d713dcd, - 0x67679a2ffd58b210, 0xa1eac74a1a7f6959, 0xd2846b3cde2674a5, 0xdf5c47e165a51049, - 0x0e0b8d6b210b35c0, 0xabe544e3de2a11bd, 0x8d268cd00033157c, 0xdbf882a0a4e85595, - 0x7dcc08b203d90700, 0xa4ecbd6fac1629d4, 0xee578fd9ee8fbbe9, 0xf9fd2ac49d4e7a8f, - 0xad5dc6bf7496228a, 0xbb8376973dec92a2, 0x2282b8d97d2aab50, 0xecff53b071d25722, - 0x1bfb0a6c1dac26f0, 0x8eeae7cbedd17422, 0xfb233051a240e26e, 0x715885fddcae0f51, - 0x06ba84025dc9a4fb, 0x28a4ae74cc7bb9ce, 0x416b38165c69a9a5, 0x0335c9a572faeb77, - 0x37f404a9c748fb53, 0xbc9a85ec33d837d5, 0xbc1411a6bc24e0b0, 0x2f30cea446f98c52, - 0xa3117eb686385dfa, 0x497cf8b437702912, 0xb4e5265530d89678, 0x919a968fd7d27281, - 0x3d586c14f455bd98, 0xeaa0018a04300f19, 0x9c7aa90cc9480be1, 0x059c7fd3b1c54551, - 0x06a6107c61a9bc11, 0x9ab8d068b4d0367a, 0xac26370b1f488971, 0x5e2b58af4cce77c6, - 0xc0aa0f7392e66780, 0xae5e89edd0ed1861, 0x52a896b9d58b35d3, 0x8bcda04bdf6029a5, - 0xa9be348a16c5bb31, 0xadf1e13e1fe1934c, 0x3aa3382acede5375, 0xccbb661bde129d28, - 0xea3dff45849ed9fe, 0x5e32c5f961468634, 0xe8539d90ba7c70c1, 0x5d16e3ad61e75bc5, - 0x424e32bd67a568c4, 0x83eeeca0a94b6b54, 0x87849dcd93c4cccf, 0xb74e98267119aad2, - 0x2bd329484e38f4e5, 0x9d0f1de5337f0b36, 0xbe58e9be6815a8b5, 0x213ae6d2f4548d8c, - 0x070162a7a3625558, 0x70ff896785c6b109, 0xe978d44adebfcbcd, 0x3f636d9b2e729bf0, - 0x69b897da2c193bce, 0x0782f61b24954551, 0xbf6bedd9a660073d, 0xbc9e855b45aafa87, - 0x0ee8bbc21337226f, 0x6dcbfb9a5dd57fab, 0xdab95f3009aeefcc, 0x1b414043f4a9abb9, - 0xdbedc554ab15a879, 0x3e04d42bb877ae7f, 0x37172ebd734a549e, 0x07d41ace0341712c, - 0xbb09f26b557d43f0, 0x4747166d305c6fce, 0x25f6acfcdb5e9fc4, 0x37c9b4be71a2f8d3, - 0x4947a7dae631f2bd, 0x9e86e6a58c7d96f7, 0x128ee7e03252dc46, 0x556086723aaaa65a, - 0x99534e170fb5ed7a, 0x1249a5430f5d9b37, 0x0aa6e33e8113d4d2, 0xcd312093ad077d3f, - 0xc503bf6c6bdfa6ef, 0xc57367b6f2faecb5, 0xb130efdf07cdc8a9, 0x9635f8c5cade84df, - 0x3abfe4668adfb949, 0x2e511d35df89b99c, 0x2b169eb3b5ee57ae, 0x5087b467ee6ff2ca, - 0x24ff6ea8b59c0cb7, 0x8a03b9600151822b, 0xc6b4091feb0af87a, 0x472020b83d8cf8f3, - 0xc66e299911b3fde2, 0x157861e893bf8da0, 0xfcf30d920288f598, 0x452b2d3a914d51a1, - 0x5e0805791ef14f89, 0x58d919e399c533b5, 0xdc57f04269948bf0, 0xa430cbfdb4dbdd23, - 0x57bde28c7bc18715, 0x03506efd2ada55d0, 0x9ad7bdcd88ae349d, 0x6ab764209fd23309, - 0xaf2d628356b3942b, 0xa65165691fa22bc6, 0x670f5d1d2eb8ec0e, 0x61a5b9e0dd0b9abd, - 0xd1a2fd4132e082f1, 0x7f13d496fd6d49bd, 0x79fef238c5ac037f, 0x559dc1c23a70f87b, - 0xa54e57189ed68722, 0x203bc4f766aa7df6, 0x5cf65f1dd9110bc6, 0x229b2b4fb12a6ee0, - 0x2d420d7ee48f8e50, 0xeb7e8aff399e2fcc, 0x4ad97fdd048bab03, 0x75113873929adf74, - 0x441032f6686d93d9, 0x1d1b3f8a3a700eac, 0x11e824b4205a8ad9, 0x0f67596a124a531f, - 0x09e6a523613e69ef, 0x9e50ddac2bf41ea8, 0x4128bda1f4a531f2, 0x26dc4eb9fdc50950, - 0x6e9de3de8fb94e6f, 0x88d3c585bc5f308a, 0x136c58e5b6a1c549, 0x2b0a8ed634ca5986, - 0x801fe7ef5e9e6200, 0x0966b21fa23ca7bd, 0x8f54bef1e6850b84, 0xdfb90a9326d530e4, - 0x86d364f678ee3e34, 0xd9edbab3d82270de, 0x7d44477cf130e3ef, 0xb6845c99e5b606a4, - 0x9ae8ab71ee87d5c4, 0x168aa1eaec00fa92, 0xdda615718abfdc2b, 0x758e681bc0730818, - 0x5405cd04f975cfa4, 0x635e16099b6191e0, 0x7913112f9fbf2869, 0x63a68348673578c7, - 0x84a724e9eebaf7f8, 0x2b0d14bb9bfafe20, 0xce4a911c5079fe90, 0xab13b105a2b38a84, - 0x0349e6c12adfbec1, 0x2e2eb31ffb75e55d, 0xe9ac3d6679300a8f, 0xc43911d328d7881d, - 0x0a8ef5de52f939a2, 0xb4310516f4d16510, 0xf2fed39097fd3dcf, 0x127567e24159e59b, - 0xb5a5445c42a8633d, 0xf901e287c2d6276a, 0xafc670096e5dfae2, 0xc41b161ce0c2c3e2, - 0x717ef81e8677189b, 0x3ccf9f48c860a3ed, 0x95a504f198e6c414, 0xb3f6a285c436cf10, - 0xb3770e07e1d2ecfd, 0x0f64384831a46362, 0x2c028dfe16339529, 0x4183f12f1f840bfa, - 0x306d031d88e383eb, 0x8d8a4f890e33bccd, 0x0c828285a9d374f5, 0x8b290c4cd8e5278a, - 0x95186d057e5d9602, 0xd19e6a8b6c29dd4f, 0x55ed680ad45a9044, 0xe0bf034973f37a8a, - 0x18b1432cf1dfdcc1, 0xbdf440eabeb0a115, 0x068d070e58829f47, 0x1d8a2cc894bd0f55, - 0x038395bf7bfaaa87, 0x78278b83d4495d00, 0x2f7999decb6be7a7, 0xa678411eb7eb9569, - 0x8a9927bc1ca9ac19, 0xbdd28586f17d7f9e, 0x2c0b6b86f08943d2, 0xdfbcfe3671d1269e, - 0x05b40bc64c6bddf7, 0x785f45d668320adc, 0xa5b3d90424c9003e, 0xe976fb1158449299, - 0x46849e6e305e70e2, 0x6469d13bee0ad7f4, 0xeb287966072ea01b, 0x6d7cdb617f998e4e, - 0x94732e60a03443d6, 0x0d90a95980ff534d, 0xe6c2beb47a6c8986, 0x66d22eb4912c3508, - 0x45ae2c4685e0f4a9, 0x87e039740640a8ed, 0x439d10cf7ee1b7cb, 0x3e5bfddaaad1fdbe, - 0xe3cfb17658bfd01d, 0xb7974424fae704ef, 0x51638743572461da, 0x0dc48307e889ebec, - 0x07e812045160e335, 0x6e4eab3cac4822f5, 0x17927ae77a1f3b9f, 0xea5ed1993fb922ea, - 0x2f5bbe2f5c02b613, 0x045cb8bbb5809d4a, 0x4a4be3af7a00bdb1, 0x84630e81a24f4d57, - 0xd45a177415e71eff, 0xa9f8e70f617562f2, 0xe36b1c8942a06fb2, 0x9296e915ab30c91e, - 0xd82895f195cfffcf, 0xbae2c86ddce7b51e, 0x11f29b4ce4365d98, 0x9aaabeea5bd28c31, - 0x7bf324d98321b34b, 0x7995897e22e37566, 0xc95efad5fd7fa1c2, 0x8c81a4bfeca3718c, - 0xfa072b9d11c206aa, 0x42786b644d3cafa3, 0x8f26c3bd7490aa6a, 0x3d877dfa3692c58f, - 0x068958c95fe61566, 0x6d3a8a70b4a298d9, 0x598678bd4cb8a0fd, 0x78b48b2a3e73ed64, - 0x6a3093ea87cd9951, 0x913c8d4c0b268870, 0x7be8887e87792ad6, 0xef2c9e8b104f4649, - 0x9010892e26058be1, 0x4dc2fbf14d717bed, 0x0e1c143afd83af41, 0x66fb8091e5b8c2e9, - 0x3d3f2a21a0d56f26, 0xc06b78cc9df135ef, 0xbc50f415722f81c7, 0x000064321e57d7e2, - 0x697b8ef04de06b7f, 0x66ac213f321bd8de, 0x598cb36bebe5316f, 0x30f5df2ea314aefd, - 0xfafcba6469eb7647, 0x22bf4d097ddf4e19, 0x75ccf3f5d4524826, 0x95dedc7ee1819def, - 0xaa6837e744b0833a, 0xa759e4bf83e997c1, 0x11d120e7906f8f4b, 0x53b8a64058667d97, - 0x0085507767790df6, 0x69a23b5bd1b0bbfc, 0xe5e8b4b48b0435a9, 0x1c9edecf127f30f8, - 0x36c66e00358953fe, 0x03d69e2816f790f2, 0xcdf8f67ed64fd555, 0x5299c2791c1cd166, - 0xf4f306791b27552e, 0x4447a0df88fd4383, 0x20f4ddf4170f8723, 0x5c03d900444946cd, - 0x9b5c77907aafd14e, 0x7c22d7b11f2e14da, 0xe7854515bf7450c9, 0x137208ee64a36434, - 0xfd27ed7318d7ecb2, 0xc9780e945e333f9c, 0x1d8402599550ca24, 0x80786a12d6c38fcc, - 0x48c68f0bc59280e0, 0xe24a85667f8dc11d, 0x60597bfbaefd4ed9, 0x050173bd7541e005, - 0x7a1aea79da3b7cd2, 0x9243ee1f2b583b42, 0x47d21bf80bfd4d0f, 0xe35d14b1447b1924, - 0x42787d9aa02aa6dc, 0xe902209ce0ee9cd0, 0x5816e6034e082657, 0xfa51f3ac039fdd37, - 0xed67b14bce9b0f15, 0x55bcac07a5155709, 0x30893930574af12b, 0x730bbd04ff750567, - 0x52c147520ad92348, 0x734ca317849817e2, 0xc56559632480a49e, 0xfa9fbc276626346c, - 0x60692110bd413a92, 0x97803993888c535e, 0x868f3fd066c81fef, 0xc6015cce976ba364, - 0x1cd965cc964ef93e, 0x10bc60ef431d985a, 0x3b0115cc9104eb53, 0x338587c191cb1b1a, - 0xf51aaf427b3ca399, 0x9ffbfa8bf1ace60f, 0xf19d48b045de9f9d, 0x87abcbdb20e32ffa, - 0xef9b1d47a064fcaa, 0x84492111223d3688, 0xe9d2c7683419ee37, 0x94e566547fc5ae88, - 0x372bff93ae22fb37, 0x93b1c1d4a19a7e91, 0xe7a314e3d311e3eb, 0xc54386dcc4970404, - 0x266be3285384fd3e, 0xd574257da1aa92e3, 0x7331ebd79d6ba610, 0x2c04c2def0dfc869, - 0x85ce2ebc0f01ca5f, 0x289b9fead4b781c2, 0x4782567254e798b2, 0x4b16165523a317c7, - 0x3db5d38f8fc6e59a, 0xd4cd6e994141d733, 0x891e81a803f1f0be, 0xdeea97782154bef1, - 0x195b3c9318337164, 0x9d0031c8c6cad2b4, 0xca05267fdedccba7, 0xae81f62168fee652, - 0x3472cf0b0dddebbc, 0xa28cfaba3efa4a46, 0xe5747504e62e6871, 0x9c82afee2a061398, - 0x67f720fda968f103, 0xf8b887e1b316cbf6, 0x44d8c036ce887a49, 0xd558dcd93dddce99, - 0x5b51c318bd834af3, 0x8526f60ec2ccc299, 0x87a3eff30affca8e, 0x0ba91e43717b84fb, - 0x3adf44d3760432e1, 0x395ee745ceb5d106, 0xb81032c6c38e5dae, 0xba7aeae0181e96dd, - 0xf854a79456d838ca, 0xe163dcd76c1e4d76, 0xaedbb46659e1b0ad, 0x04376d99900e64ed, - 0x5c52f136f72cd57c, 0xcfb366137e7e15fe, 0x230a2cbbed0df0ec, 0xce0c8b48fb5e5d4c, - 0x1b41bffb8c1a8e46, 0x8621e947d81410af, 0x8565b165356e477a, 0x5f87ba7a836de326, - 0x5eeb50efcc085deb, 0xc62de0b74931b33c, 0xcdff1122250095bc, 0xd3c03b4c7a7a9e77, - 0x416bd0c9da7494c6, 0x367146a26a394e57, 0xe634aee9deb5a5d0, 0x29836afb5b333a94, - 0xcc12406820529fa5, 0x8b8ee237c0dbe52b, 0xea7b598f284c126c, 0x76ad20877e0283d0, - 0xe64d4452a804c184, 0xb9ffa95d4d5adfb9, 0x39fbf340bb589620, 0x0827fa2d6e97bc48, - 0x8e53f04ca8283c1f, 0xf78ae0024a4eb567, 0x4f620bbb975872af, 0x43487ec67891db0c, - 0x390c7518de7f531d, 0x8e13e210b4eee946, 0xc632f48bfd83b5a1, 0xd1fb1b346c4b7778, - 0x599cf3d21841ab57, 0x9cc670eb791beaac, 0x927b41f9d30f5366, 0x7ce93591d7517bc0, - 0xc8f7e93d485bedd3, 0x244fec6d4960ae6f, 0x04214feaafd455d2, 0x5a5c5a85d543c6f0, - 0xc9f68fbeed3747eb, 0x5750452003eff206, 0xced2c7ec38b0998d, 0xa25937dcb42c364d, - 0xd540872e1339e292, 0xabd932ebf08a1de2, 0x167dab5d123704e3, 0x5f75b1928c45638b, - 0xffb4ecc40fbf7362, 0x5dd8340469847c2b, 0xc01a54132fafe9f4, 0x40545ea96cda0969, - 0x877fa2717de5afc8, 0xf8fb4bdb6dc75454, 0x43f62d2c388ce1ea, 0x503f183087dd71c1, - 0x2848d1ed5ab4f509, 0x7e14aa36fa13d5b3, 0x4121804c1e17225b, 0x92c53d20071f0f48, - 0xf0cd147ee18c5048, 0x9b90ac8540018e19, 0xefbfea2de9392093, 0x60b91be6992762db, - 0x142538a433c02a27, 0x61c0dde589e517cb, 0xd7503f62e8f344df, 0x0254a4f0b4db3506, - 0x37bb4a6f6368744b, 0xc25b395c005c463e, 0x19dc39533b04654a, 0x1fc70c2a5b4180dd, - 0xb8d4a7462116e6b9, 0x2d9fc71264ff4afd, 0x6e46a16929cf462c, 0x9671d90ae4eda2a7, - 0xd3c3304b81b8510a, 0x9ce039a2885d5302, 0x7c67dd3b27391e68, 0xdf07fa8f23c10934, - 0xc58d48167de8d287, 0xec9d57035f1c1a8b, 0xdb6cc6bbc0bf75e9, 0x79d5b76e73b7c1a1, - 0x1722754f7c7d937c, 0xeb889450cf1f2aae, 0x1926528f0d42a4f1, 0x7f063bfda112be95, - 0x2464e5bc8e094fa7, 0x62eef100aae9d966, 0x6a338f4abb909862, 0x2e82b366baa2bb7a, - 0xab6cf5bcefbf6ab2, 0x5ba3629fda5c7f7c, 0xdd9e1bf0b7f4822a, 0x42ff2a6eb0d95ac0, - 0xa7cbc5d6c855b232, 0x2b3224668aababb1, 0x7565233455c998b4, 0x43ae43ccf52e25be, - 0x2846d361e6c032e8, 0x03fdbf1c338b572d, 0x25df6aef1feae7fb, 0xef017681e95cf3de, - 0xb2e697747074befd, 0xaf32b9c80a0c3f03, 0x678839067abe41c3, 0x0fdd53151d36f272, - 0x15fc4edaaf4a6fbb, 0xfd16430284e9528b, 0xf3614b96a7a4b04e, 0x45af1acc72453ebd, - 0x43a36616f4fbb89a, 0x7a3b9011b88a273e, 0xa629d90a1c3adcff, 0xb1b369e10c2b48dd, - 0x16a044d6e56a6f11, 0x01bf9e4134253fa2, 0xf7884b87d7114ee0, 0x17ed7f15f448a8f7, - 0x6417a1f9779f788e, 0xdad149d71b9153a2, 0xde0c308202cc9a30, 0xd96a71523e162080, - 0x6324cb30fb28c47a, 0x0922caa86cc2ec59, 0x7cc8951dccb472e4, 0x7f84c9a202c20787, - 0x3557cade4d7096f6, 0xa6d30a9e1f11f8ab, 0x9cbfbe17be102a7b, 0x342d33d62994804a, - 0x1dec2fa6cb5fe821, 0xdca6556377ba773a, 0x7cd8e4913b12a93e, 0x99c25678336273e9, - 0x6077ee7acdf8edf6, 0x7158e061ce06c755, 0x99b5f6ac2da43bcb, 0x62dd6d1adf30d943, - 0x95e3d0fc58d5e827, 0x719ab213bcebc7ae, 0x11ab5f64f2040ed2, 0xda88d59b9687c929, - 0xddf9aef85600125c, 0xaf79cac3f8ba37f9, 0x23c24db74b6a3dd0, 0x77b3175aa1311d22, - 0x3a8ea3914c8b8d78, 0xf04c5489a1709c88, 0x093df9d686d39b17, 0x088e2272b21d3515, - 0xfe6366a5e44ae8af, 0x0eb7839945cd72c2, 0x20f95094ce1bb48d, 0x4745b5566583e0bb, - 0xb311e655b0b3cfbc, 0x1b961ca72b7bf836, 0xf10b78ba77d11558, 0x52fe91498720b153, - 0x67e6285891ae559b, 0x0d9779cbd0049046, 0x983a697dfc334e08, 0x96868c3a92e5249a, - 0xd5149512cd7c4d66, 0x3b0daa00923ea8da, 0xb4a9a6068292ab62, 0x3cb13e42019b22c5, - 0x10d420ce11922098, 0x1ba7fad3a2219313, 0xa67e9a5073bc424f, 0x576258f9b23e21bb, - 0x02ac562bcceb7eb3, 0xe71dff8823b17988, 0x948585759c559f49, 0x7f4d8d1a9467032b, - 0x97cf8e06be507ba1, 0x93724ee59be2d236, 0x72f35d64dab7d6b1, 0xcb009bb946dbffeb, - 0x3246fb862cc771ab, 0x5f46ad9b19fa6e5f, 0x023915751b60fc22, 0x4eb0d10493845cb5, - 0xa7303eaa80b24480, 0x7249baa480eede23, 0xd7384df0f003db4d, 0x8a19ba46f9ff6077, - 0x1f44d8bc105288b6, 0x8778fac5071f13fa, 0x3f240544d30f003b, 0x198ffcea832b2b1e, - 0xc0237367ae54ca59, 0xb6e573c789cf5fbf, 0x39fb4ee76bdb80a9, 0xd3c02a7d864fc368, - 0x657f534754fb7494, 0x587cab36437ed91e, 0x54641d1f93f03b38, 0x1cef325f775da9a0, - 0x898a903cf3baed0d, 0x3116753d4c4fd02b, 0x5a421f8f23e658ef, 0x1bec6ad8c0ae016d, - 0xce11a2be8b4e2daa, 0x24e34a8b68939aeb, 0x3bc8d7b11cf7fd93, 0x887a3bc167708e78, - 0xcfff67c74ccc9f7f, 0x05b75464e08a0f35, 0xff380401e1e15ab8, 0xf9e424b6f064eb4e, - 0x52ec2040ef27d012, 0x41187b8ef53db396, 0xb4487b793a19aec9, 0xc5525628e7cf2705, - 0x5f6cbb1b9dc8304d, 0xaaf545a2c9eac4f0, 0x2a3219a9b3f3a615, 0xd3410d6d12fc954a, - 0xece1b4d443cc24aa, 0xd8091ea31ed0bdcc, 0x0b469c9a4ef07a5f, 0xa33eca769b8a0009, - 0x47f998365db7b461, 0x13f5954c9b87186b, 0x4b7a3fb2d922b667, 0xfc661d6fe148d9dd, - 0xe9e923cabcb019e2, 0xe4fbe73744b47e23, 0xce2c505a6228b5aa, 0xdc36057cd5464408, - 0xb9ac58cddc3b4771, 0xe332ba8ea2e27d2e, 0xa2d82df50e6a9dc8, 0x3884255e884aead3, - 0xa74dd09504b4153b, 0xeb7e5393d7d39b56, 0x6d662fa9481448b3, 0xa9576b2487d57fb5, - 0xe6b7ab871b1ecf34, 0x89012daebcc8366e, 0x26f22a4398912f1c, 0xe0177197638e6d77, - 0x4bc56d7a5fc53ee5, 0xe886fee4458f61a7, 0xd3161c312f11d07e, 0x64bbf81acb0940f9, - 0xe74a618485400980, 0xfac95e92eb109a94, 0x90a6698f6ae1d132, 0x2f3f6f563bee83b6, - 0x3ee0021384dc37e3, 0x6db23230266b871e, 0x4ff691ed02146b08, 0x056113b59714e606, - 0xb139b12bd23e98f9, 0x5c7566de20e925be, 0x9b697910bda9c5bd, 0x462503eb9ffa92d8, - 0xddffaa79064a2511, 0x7481c720773b648c, 0xdc9d99fafb1ce957, 0xb4a25ad1553af1ad, - 0x135ad0d1c6c3726e, 0xb9b09ac29894717f, 0x45238640d7d3cdf2, 0x184082bf7c66ddcd, - 0xc4448ba6c72c0b8c, 0xe4c0b30d6f30213a, 0x0d7b0c070ec059e4, 0x997dd0aa879a700f, - 0xe708690af0a5b2ab, 0xe46e2b9deb405ca9, 0x84cf177586786f4c, 0x8746c5d4669b4f03, - 0x9e8f63b21c30dcf5, 0xf28dd88c0e2df952, 0x8ef7f9ae70cc1b27, 0x7cb39d80ec7dd5ae, - 0x068d076706ae886f, 0x5b20538ef0cb93b6, 0x55a1be3cf5669ca9, 0xf0d9eea3711b5e7c, - 0xfe836b8b1d393808, 0xf92ce56180745a90, 0xa9757b956680c2e8, 0x00f568ab8d2cd1dd, - 0x315ca544b3d35e3a, 0xfd6e2ab416ec0a8f, 0x45e7b5274782bf5f, 0xbdba55ce5084059b, - 0xa41de2ac04d215eb, 0x984b202cae303924, 0x1e8710c28604dfa5, 0x323d2b6253ed5f86, - 0xb5d8379e42a29f81, 0xdb4740ac8d56cd8d, 0x7971017b601c3d9c, 0x71cd3139d8e32a59, - 0x87fcdbffe0c5999d, 0x8395605c664b4186, 0x77ec02bc3b7d4ad3, 0x78d5ad6f4c1063e3, - 0xe28b24b9437aa599, 0x52ae8288bc4693a9, 0xf18de7bee9767d6e, 0x6e55ad05f2cfc66f, - 0x3c6e7c98ffe1ccc0, 0x9821ccdf1c608aed, 0x12ffe74f8780752d, 0xe88fa973860b19a3, - 0x304e3a469702273d, 0x75d16c6d5831594d, 0x4710281b4a96ef96, 0xba1a54451d1fcf83, - 0x7a57af4cd9be48b4, 0x256dffe831f24200, 0x5f6535d5b70ca5fa, 0xfad8bd481074e393, - 0xf786bd40cc03eb1d, 0xe16ea21ce415c679, 0x762085c79be72cc0, 0x57b931e8d7c28501, - 0xfa21443ef4eb0bd7, 0x98c7a0d8e0d47471, 0x12a978f1fa5748c1, 0xfc7b4c8f70130cb4, - 0xfb7493025a145b34, 0xafa60f9dc370c930, 0xbc3f1b24e3ae226a, 0x207f02aa10f5e0f2, - 0x6fe8b366a117a48d, 0x4e4b77088f032817, 0xc05832b23371914c, 0x5eda80353eef5336, - 0x8eeb36bdc4b671da, 0xd80fd6666b625226, 0x42586c95e7e72e76, 0x02e6789403f814d4, - 0xbd988333ef94cc57, 0x393154807bbfe536, 0xddc238b29b027009, 0x3f4db2aa543a7c2e, - 0x122449eb192560d5, 0x04936cbdde017053, 0xdf3d616867318e57, 0x52f3c8b871bcd541, - 0x7ec7efe176c45199, 0x759d9672875cd231, 0xc1dace190b3596d0, 0xce71663e37d84344, - 0xd67aaa73b5cba2dd, 0xab99b279e83c8eab, 0x6f61d6084dd77cda, 0x710d71fbe23a2869, - 0xc0a95aa344ecfa82, 0xce2327e6cec09e9c, 0x22ba865865d49527, 0xabda4ac625b5c663, - 0x639fba1405f3dd28, 0xd17f891563b9ca25, 0xd9108e1465ffd277, 0x8db80a474df18b4f, - 0x09b97d8088df91a0, 0xd75befcdd819bb76, 0x6a52bf5073422746, 0x2c0f5acdb7eff209, - 0x653dc56ad309d0cd, 0xe005c74b266ad71e, 0x52fe1945bc2d26dc, 0xd5a0ee6782c88fd8, - 0x33a90c4ead7cabc3, 0x7eaa0b11710540e3, 0x9ece3e56fb7205fb, 0xd1b24364a776752b, - 0x366873c7aed8528a, 0xdc2adb9291ff0b4e, 0xbba86e4d46bc6982, 0x9004024708a9fc97, - 0x40437d346e642ecc, 0xb5fe0234842e348c, 0x3d8d988b33331c55, 0x4468fed083692609, - 0x69c7924ae7032661, 0x3cb5a272415584cb, 0x71f68ec396df8250, 0x3ecbbeeef5a64f86, - 0xc44c4025fea6df6a, 0x266f36840398ca39, 0x18cdeda71e24d0b5, 0x392e4459569bc044, - 0xd817103eb79d3c5b, 0xeeaf5bd4b34ab48a, 0xb23e366eb02efd08, 0xa76d15efba0ee3aa, - 0x643385dc6631ea5f, 0xf9f3ea4fcc90f774, 0xdf91cbdaea94c9b1, 0x61c24b6e6c529bf8, - 0x2b0ba33a34dd834f, 0x5166437827f0b7f3, 0xb276e6c080cd3ca9, 0xf32f2fd75dc7ef5a, - 0x65301766452a4cf3, 0x9f08589b940806a5, 0x3d5a0448c28d01c8, 0xf8483423db20e1e6, - 0x3b4c41b97a1b9f21, 0xd28c9828667536b2, 0x8b2da57b6eb9c42d, 0x480bac61090498ce, - 0xf672e787a53bffc1, 0x4affdb576f493607, 0x2d0c595865b3777e, 0xb793dc92eab15b00, - 0xfcfc57fb54391e6c, 0xe7a616260ab04681, 0x707521a9cc83bc8f, 0x9030fa3026a0e332, - 0x4aae9d7a5a0f6398, 0xce077c97ede3f843, 0xf11af727b514dc9f, 0xa4d582c2e73aa860, - 0x835218ea5f4f7309, 0x9cea80edeb42204f, 0x1e0a719b74cf86b0, 0x775069db679182ff, - 0xbbb2c53b191f616e, 0x235ab3404d23df0d, 0x786805e2da2a441a, 0xea8615711f8cc4eb, - 0xb7fa48324d531b3d, 0xdcbe2e3711334ea3, 0x395c033f5f1a5ca4, 0x3a48029edae75753, - 0x42bc9029ea0d399d, 0x0de1e43051afcc3e, 0x207da739e84c39d5, 0xe260f451cdc13d34, - 0x292c84ee55da15ee, 0x0f2654d7ca29960d, 0x9927c37d3f2f30af, 0xd2e1ea2c8112b0b6, - 0xdf25c6879e06d79d, 0xfd5b7a727ee9a2c3, 0xce35ee6826e7b941, 0xa9b149e329b90531, - 0xadcbc7772dbc05b6, 0xf356e3d9c1ea96c3, 0x2f4c3f6f9e524c08, 0xf2f9ac8b8c4b312b, - 0x4ef2ba6905d0493e, 0x4920a49717486be9, 0x75231dc0e41522ae, 0xe6fbf92d57aa4d8e, - 0xcb4655f12b362c78, 0x28be05e636e0efc4, 0x12540f7983676ed9, 0x2fb4460db9306f37, - 0x07b2bad10c4b7681, 0x6cc414cf58a6df9b, 0x16e7409a43b1597a, 0x22fa7610217f497a, - 0xa22941424a88b7c7, 0x6cafb5dbaacc680f, 0x7472bf534d96a5ed, 0x5c0b7cb51ea4310c, - 0xc3dae0e5893a81ca, 0x874d074f518128d8, 0xb8325b9efab2c958, 0x43a604dbc2bd0297, - 0xbded34f64fd696ad, 0x93e31bd548e8ba22, 0x7997ba234f736cee, 0x3aa3e18696f9d59d, - 0xa61cd14bfeab77cb, 0xeeef56910e12f7b1, 0x966a66f1e2fa908f, 0x63be60603cb5ad82, - 0x182d4d3ae256dc10, 0xf279c1c86b0b28d3, 0xfc8104a2393a5f36, 0x56c2e5ece6a6386b, - 0xa25c9003753b1995, 0x80f678b3ccd7a54e, 0x54e7eda1ca0dc5d7, 0x1ad85e67a1a6b48a, - 0xe4bbf78aa0140a30, 0x69e14d0475fb8753, 0x3bb8e67f5ea3cfcc, 0x16d3afb3a12bef63, - 0x246915327f73cbc1, 0xb50265f44cfdf8c4, 0x65d23fb3e7fbc190, 0x0dd28999cc30ed6b, - 0x7215ca65469fbbfd, 0x6de8a207f555be62, 0xa77630edec5e31f2, 0x921c1ecfd82b0fb2, - 0xa7bca763df2901b5, 0x2cd4663a28c1f99e, 0x6cb9b5ffa97114c7, 0xf5eaa4c7a653a559, - 0x96584b1424b27936, 0x4b1be5a69c4e1a85, 0xa8e3916bfe6df1dd, 0xd66580508bc2ea8f, - 0x7ea6090fc390e159, 0x16f9d33f1cab5bb6, 0xc892ad2e4564131e, 0xc6bfea2734606cd1, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x46b305b199233b13, 0x8a1ba49ed975a8b7, 0x4aca364f2fee77c0, 0xda471191557e63aa, - 0xf5ea671c9e1c93d6, 0xec2cf231021f3291, 0xeb1c1908821111aa, 0xae1c9039f3f894c5, - 0x817fb54d40646417, 0x39334647d4f0ee09, 0x47388d9c75fb2476, 0x1e78f60135ea2ca2, - 0xf1957d26e8359071, 0xd7629b1cf9001909, 0x7262c5514acc9949, 0x157a64db3bc9efa2, - 0x6bb34a2a3512f3b6, 0xe09e0909bf519eb5, 0x94cac4aa92c3d75f, 0xd361e2d0ac5f3e14, - 0xd4c32942aed8270a, 0xe9688b4f22376316, 0xfcbeecc1955809ac, 0x861a23f6ba21a38f, - 0xc75f6d924f570b6a, 0x6cd1f24d53508ae9, 0x4b970293ecb82a51, 0xb5d008691e149e00, - 0x8ee79db918181531, 0x32736f1148a9dd71, 0x2978c080cc482d2b, 0xe8802fc47a504644, - 0x6b1c2be52a831b63, 0x8ec6c4dd3d408aa7, 0x1d0d174c6096afbd, 0x34bcfa010a1c859a, - 0x40e198db45391cd7, 0x3014c01cb9f272cc, 0x718c74b6968eed96, 0x63f3c5aeb963840b, - 0xa39aadaf366cfbeb, 0x4918f4e5fe42a6f1, 0xcbfb76e0a356fff1, 0xa636553e88abe702, - 0x1466ae8faa831f31, 0xd49b41ed88f763ec, 0xdf15400c2017b48d, 0x0c1db557d692dee8, - 0x9c530d5875e1b88c, 0xcca2bf7e9bbab585, 0x41569f407f8b3593, 0xa04983abf6d97450, - 0x504022c4cd86b957, 0x45d74d8c8fc38f0f, 0xb109f7be7e0c15b2, 0x4c583cb18f752903, - 0x0a86f15c86cc2b3e, 0x0de2dcce6214549c, 0x86e30a834b90fba1, 0x9229c0f4c92946a3, - 0xab54c2d8a79ff8eb, 0xcce8611ccdfe7ec2, 0x2558a52972fed1d4, 0x6ae9483cd1c0f7e2, - 0x6a4dc943cc767105, 0xf4fbe0043c601f17, 0xfb7439bb0bd88f68, 0x255b3bba71d4a8bf, - 0x0e91731e69f8ad65, 0x59ac414a31e6205c, 0x87d9ccbf93bc14fc, 0xbdbdca4f647e33a9, - 0xb337753f2a3a0e65, 0xd624b61fe95fb2cc, 0xf8dee5a47bb63804, 0x33cf9e1ceb76b398, - 0x33c3fab6ce91f7b7, 0xcef03cc3e782489b, 0xe8f0482a4195afb6, 0xfa27604d59add7a1, - 0xd5ec96073bee31be, 0x168935fc5dffb2c7, 0xb006b7b27a8e5f47, 0xf6be155f8d4c9835, - 0x245ef05dc890aef8, 0xdaf2d982d6b89f5e, 0x2ede41607f7c5d5d, 0xba18fbf8892ef2a7, - 0x93110a39e2311358, 0xd225ff1e7ad96881, 0xa5c6172432d398d6, 0xcc19e6943e762e0f, - 0xd8a506c8c9c8c076, 0x48ef0a60f27ad126, 0xcc555cccb80b8cff, 0x385165df597f9fa2, - 0x1d98b5724fd85f58, 0xe8fcea90a29e9aa3, 0x401ef34367c46556, 0xd48e8a83b33bd1dd, - 0xdfbf0306cf45621b, 0x604512c9bf5ad55a, 0x9e9679f774224aa5, 0xb535214d9c9ecebc, - 0xc869e38a17d41550, 0xbd256584e5ee7a09, 0xb20bfb0515fedd37, 0x459cbfba8d57f083, - 0x28ad3c494459c894, 0x32485ecaf48cd85f, 0xdae598e4edc0200b, 0xc87d30085e2a1062, - 0x8dd9f18abc662034, 0x31da0cacaac31d41, 0x217f78d2480da178, 0xd2aea0f2fde82335, - 0x4ca2a068a2f4b3e6, 0x7795789d574640cd, 0x6845173348480a28, 0x7e0f590204c26480, - 0xa64ee4fc016fba8e, 0xa1b6811a60c947d4, 0x53e55d2db6393893, 0x82555ceb8426e35b, - 0x31814ccf6d101797, 0x1de1c9ae8d5a436e, 0x40cc3f65d5b66d1e, 0x09189d437f46c334, - 0x7f613aaebb332521, 0x1c59d41ae3d1d69b, 0x7b1e16cd572700b3, 0x9e949b46f14a6120, - 0xb1574e0d806a8d63, 0x897f5c8360d16595, 0x00e86eb5f5b6c911, 0xfed5858dee4596ed, - 0xcef1da7156c75dcd, 0x32495703a6ab38b4, 0x59c7b52aafc41da2, 0x4562be8af9bd83a7, - 0xe5652a8454e80e5e, 0x09524c8f8015d832, 0xe5582f0c01dcb2c4, 0xa14425a4cded8742, - 0x1d0ac9718982b8aa, 0x2e150f935435957c, 0x6730954d4e84caa7, 0xb0e2f3f0152a07b0, - 0x53b9933a1e044919, 0xc613700d0633324a, 0x5582213e2fcea917, 0xc2daea2814c323ed, - 0x03c9ac5bb579b8f2, 0x7895638b272259d3, 0x7fc98ee6005c542b, 0x4ae8c4714c58c9bb, - 0xce7cf4094653226d, 0x39e95dbb5a00f1fa, 0x1ee2464c5fe5536b, 0xd3f81973dcafece1, - 0x69c04d0e2dc68730, 0x4bdba13ccc250b76, 0x2357a1a9d34ebb44, 0x12dab8604c50e5f1, - 0xee7c020012da6563, 0xeb39e37ce0dc9a9a, 0x1e7502ad31c85fc3, 0x30c281188e67a317, - 0x1f05b49d22c66288, 0xc1baa152e1c07407, 0xdc69fe1c64e35150, 0x1c1d55329f2915f1, - 0x7c7b2dcfa4d0bcb7, 0xc9be58e0814c10a3, 0x285655927622c5cb, 0x414edfaad0e9e751, - 0xab86e2768cc538d5, 0x417631bad2750210, 0xaa7192bfd7c523d2, 0x69663de0b3dca8a1, - 0x900b9bb913d9af83, 0x2345d6d25703d1c8, 0x27569dffae8c9130, 0x5b6c67f1b02e1640, - 0xd2cdee62845a4564, 0xaabef8b8530cda77, 0xf38b90f75947d1bc, 0xb3064ec69712128d, - 0x8abce5ef04709784, 0x1382a33109f46e23, 0xffd1c46f2e4f6fb6, 0x8d527fe4e88bb4f4, - 0x7d5c76c94f2c1e74, 0xe929f25c600a5210, 0x8b8084fee20467ae, 0x371482d21dd60bcf, - 0xf13d68b1f059fd0f, 0x0c6ad1d9efbaef2f, 0xcd62a5fdd41cbd21, 0x0f0ff5b3d6965382, - 0xb449a199a178d1ba, 0x5ff13cf9d08e47e2, 0x8a9d8ab9b10a38b6, 0xc8aba831173a331d, - 0xac4820f9b580f788, 0x391b5cedb78c3192, 0xefdf95f4132e9271, 0x122db5de9c77dcd9, - 0x2b1f92e338221464, 0x94cae058bfada4ee, 0x12d96aa48c95594e, 0x9970e8c92336e2f4, - 0x6e0c1627e840d213, 0xe6695c5a07cf0620, 0xc13a02c81c020152, 0xac6feace1a36e4d4, - 0x36641eeddaf11c9b, 0xc1fdae299545207f, 0x9a30bed713a90ec7, 0x8285d1b852b1ce53, - 0x5855e062afab5372, 0xc00664841530731d, 0xa377e5b2e31c0c81, 0x51627c9b2c9c24fc, - 0x4c4a7f5535b72e62, 0xc5c97618018a6b78, 0x48ff78f3c47b3a4b, 0x46e0158fd39d5d8a, - 0x67729c89f4b9d917, 0xcfa6726f4206c0e0, 0x034305c84e98ed79, 0xc80f88ec461a82d8, - 0x7ad50987c0a4502e, 0xccffb7b2394fc47c, 0xd1ae1f812704d8b0, 0x1ec53e4863b45247, - 0xb55c13b0aadbde20, 0x7e4ece3e3c223ba7, 0x57b2af82a9384b73, 0x1031aaae59047235, - 0x8ec1e251bc0f9f8e, 0x212449636e00f8d7, 0xbed7fc071f74248b, 0xc8c1683ad1187697, - 0x62e8c8303c33b53f, 0xfbae2fa554271510, 0x5d07e84f28ee3ace, 0x93b5381de7def5d9, - 0x6a310d3823ec96ae, 0xb83a0fdb3a98e551, 0x6b39745e3bacb328, 0xb0722479d4763f74, - 0xcdcdf63348685302, 0x6113607831319764, 0x2180ed8c1670171e, 0x98a0d9785fe1c293, - 0x16159ff14d086bbc, 0x61dad0bff93832de, 0x7bd88a50a8559ff0, 0x9965a2be8e22ac52, - 0x23efc863c739102f, 0xf9ddab1c1ba36b6f, 0x846a7a37ce3a7835, 0xbe67bdea7657678c, - 0x9456c52297f99ce9, 0xa612bc3ce21c5c83, 0x8d2c4ea7889063db, 0x739e61490337bd19, - 0xc9f9fad81e5cee44, 0x46fb0f003ecde28d, 0x12cf216dc9161134, 0x601b3c54aa1e938e, - 0xf464f8cb0e66d95b, 0x9537b4fe9ddbb18d, 0x27526076047c55d9, 0xcf1eaaeb3c5a872f, - 0x20b646c7af6a5932, 0xe4d0a886d2906e1e, 0xff8f5f20aa6d9c37, 0x14a56868734ceb95, - 0xedc208f4979a87ad, 0xc9b0839a79a68cdc, 0x6e63cdada241dfa8, 0x1b981fc6a7f7e88b, - 0x175bd19230eb6554, 0x920a7f6e350b8814, 0x159505fffaf65b38, 0x8af774b97b3181eb, - 0x89b05990ce615ca3, 0x179e6085ab74ab1c, 0x69d817eecfed5652, 0xf9ac26045155e15c, - 0xdd469b3e09c54f1b, 0xb51b49fb43185d8f, 0x6b54dd7e08740db6, 0xb7937788f3a17cec, - 0xa65786f2eac45631, 0x22e20f5c41d1d04c, 0x31f85f479e9f49fc, 0xa322c7ae87d04403, - 0xa85942cb8b1730fa, 0xcc14cce102fb6abc, 0x60188cc78c17a73e, 0x9a48ad46f5648752, - 0xe099ee49717d87b0, 0xa24076e16504e2ae, 0x1a4696427e74de20, 0xf5812abadfff5da3, - 0xa1de791800007c92, 0xaf2f06b77466c97f, 0x18030f9a338c7238, 0xe7456432a35173e7, - 0x423a8c70f0d29ec1, 0x9bed157e2d4d921f, 0xfec5d98dad0103d3, 0xf9efa525bbcd6f1d, - 0xff33ef78acc46c6e, 0x443dfa37597ea9be, 0xaa844a3c7ebc60ff, 0x57d21c347d0c2f42, - 0x0dbf4ab9e3d73bb4, 0x07820cea5f61a1e7, 0xfc1bd8323cdd3244, 0x559fb325795f8c06, - 0x07d1d2252e545f61, 0x848deff76a838596, 0x271e63446226ec41, 0xe9584e7d42820b69, - 0xb2df70cee009ed99, 0xea6d24d1c0f6d579, 0xecd8956ecf0d1983, 0x9e75dcc3c8deb72f, - 0xc0558e13c9ddd812, 0x144e52454cca084b, 0x84c46a70b9a5c847, 0xe938881ae60bc298, - 0x295ae853dbabdd59, 0x175088b793c65d77, 0x7b857ee673eb7b22, 0xb9753586ff085f80, - 0xb668cc2ff32d434b, 0xadb11fc26f3982f7, 0xa2c3341d05ebf4ba, 0xdf57794edc9869ee, - 0x595a581a8a4a0e6f, 0x826389bae9c4f7ae, 0xd722754411cca53f, 0x204f7613de14b56e, - 0x9144e9329087c5b8, 0x7ae84f41ad680918, 0xae1a159aadb31d10, 0xd1d015f2ec84dae9, - 0xd5077723398a34e3, 0x2e075e82008a5230, 0x679de60a1790b615, 0x8d9fba6f8a6b06f8, - 0xba2a99e913cb7ffd, 0xecbd50b4f3300914, 0x92ea853378583b94, 0x4cd36c4be5951599, - 0xccb805b6588cc950, 0x29c4e4abd1f4747b, 0xbe5be0bf94e8fa3c, 0xf43759983e03a37e, - 0xba70fb6f23bb15ec, 0xf8e3d7265330c642, 0xac456e0bf3e19559, 0x5d7023725b6ac1dd, - 0xd9de6d20b6d1ce4b, 0x8bc0fe8cc6320167, 0x56207010b3c47f6e, 0x2b611998882e5412, - 0x975717699c5ac653, 0xbc63190b3aaff821, 0x2458ef963aaf1c9c, 0x6802571a7c4d2f05, - 0x8ac5d472c48eaff3, 0xaae4d25ac00ea503, 0x19d82b46d7342719, 0x30abc6fb49941ff9, - 0xad067ba4f18dfa87, 0x58a820457f1a2b80, 0x5fd971ef4266f6e7, 0x5b24b8fcda63738c, - 0x74c1baa8c6c66293, 0x0189d1449d03fb61, 0x93e47cec34421ffd, 0x5982d13d30c9e368, - 0x510538baf2221007, 0xa3eee0b207be764a, 0xedb41842afd5ef6f, 0x8c7a1b0993fc097a, - 0x23392ee02580bea8, 0x25ce2714433240f5, 0x0d570c178a9ea10d, 0x68f49315f24358ed, - 0x0ccc47ed74a59354, 0x86947d6e197bb6aa, 0xdf7f79f89f9692d8, 0x5f140eb3e2c16a19, - 0x2bd22ad861c2d3a8, 0x423be84dd430d0ad, 0x474ae6f01671e376, 0x803e471b277d632b, - 0xb1935fcc1856d823, 0xae6041c83ed60c71, 0xb462726566a8be7c, 0x4ddc24f4c97a294e, - 0x97f704bf2d5b5284, 0x816e07fe582fcc98, 0xf0c9842726c1770e, 0x6ed7bdfba20e46f8, - 0xd550f54c300cc109, 0xafa09a889fb2fc63, 0x7e7a623edd3b7138, 0x404adaeb5cdcc21f, - 0x6510afcbf0122fa6, 0x332e822f28aa0c9b, 0x071eb1ae3f05794e, 0x1b1eb4e8c36bfcd6, - 0x7613c4cf80739bbd, 0x3d83e128c93d5a2a, 0x195935ec6e89e31f, 0x234257cffcfd3075, - 0x65980bae9a299e85, 0x06a40a0bb5153646, 0x57d0af60d4d93ebf, 0xba5ec40da37945cd, - 0xc1ecf15bd6b5e941, 0xdd92a3a90fa0e028, 0xccec70f1dfe86ac6, 0x16bbda36dd6d883f, - 0x50e721ff309b6eef, 0xdbe9a377f230274e, 0xb8859d52cd2fad92, 0xfc933912bd3c180b, - 0x50f209ef0c3f2760, 0xc5351a048873f8a0, 0x9c29b73d53a0afcf, 0xcd2b9f4836e3bf65, - 0x8972c0183b6376e5, 0x095aae83ec4ab329, 0xb09b31aeced75b16, 0x7cd00dc7b28a0209, - 0x723e0914bd69d026, 0x48e8f877ed2cbe09, 0xff503fcb34a926b4, 0x652291ff80e19f94, - 0x36325161afa9548f, 0xb563d1c671911797, 0x35277877eaaecbe8, 0x475d13e97d46c956, - 0xcc2b0cc4a7981b2c, 0x41207b24a924c504, 0x725fcd05cbb070ed, 0xaaab104a011149c7, - 0x29bb905406426f7d, 0xda0ebc5538a0c276, 0x6e101684c723efbe, 0x7cc3334b9ddda35b, - 0x5a643558d09457ce, 0x68de78a40db6bdcf, 0xc2cd1acf7ccd37b3, 0xf5fdcd247a66f6d6, - 0x5cf5baec515b4c37, 0x7518aa64f8368726, 0x3500c5564f603f9f, 0x8ff59f55494c8139, - 0x1223fedf2ec3414c, 0xf348a9ccb164d7c3, 0x757c90ba2c1378ce, 0x3e034e45c946b6b7, - 0x182fd38faf3f7668, 0x8a82b82aa4dd5908, 0x0f8fd6771cc9f529, 0x13082c51cf2a9777, - 0x0280a97a4b62eacd, 0x161a8683478421ce, 0xfa3c321dcd4e9e8f, 0xded11a646d98e267, - 0xff1234f3d4a939bd, 0x3cbc220fcf389b67, 0xf5413bbd872d1ef6, 0xb8b7f35b0b494f49, - 0xdc5c68c04bbd4497, 0xefc1ac6fdc76424d, 0x57376a8be06687e0, 0x099f66f45b04fdc1, - 0x9823d52a2ed7f5be, 0x5ac97ef071ec4f40, 0x3196fcbdc8f9732c, 0xc688b98001728740, - 0xa386f4a40e3a7feb, 0x2d33193993c41d27, 0xf35caf14ae7df18f, 0xdb4d15702b7426e7, - 0x057feeb14b6f2561, 0x03220c5c905f7919, 0x075036c4543ba869, 0xe4768b46150c14e0, - 0x394c842d140ac791, 0x4566b2d0d4b6961f, 0x20d2c57f9ce03869, 0x750d7633290c93c4, - 0xb55e80de05a0567c, 0x53cc7a5c62d38f04, 0x75bed7598d9a8f9f, 0x3aa58b08689dd8a4, - 0x8b541f99d5eedcc5, 0x2367f9329fb8413a, 0x414e59737e6153ed, 0x36d49d6966f1d70c, - 0x63cde5df104078df, 0x6fc5ce71a0d3af29, 0x4757f4ddb99ad9f4, 0xa3c05b9554a89fc1, - 0x6b0f47c330c0805f, 0x2a8a81c4b3bfeca2, 0xd08665d60c6e65f8, 0x4c7e9d4caeba45a7, - 0xf25e2cd451d39408, 0x615c2cd4a3e066e5, 0x0902453115834ff9, 0x5ff5d76798d7b5bb, - 0x167457d16cb7c4d1, 0x3596599914adf034, 0xe1f335255b8dc23c, 0xada032e53ca95b43, - 0xbce021d049b532ed, 0x27382a3373dfba30, 0x8237820085714a64, 0xb55f046299833950, - 0x0c30ed81a86d833b, 0x677f276aaac773fb, 0xa74bc52ef68eb3d3, 0x71da4c245aea4f18, - 0x4fa365c9af6cb475, 0xf5c346c5ac3dba0f, 0xb7a3cb3b99144c32, 0x8de54cb7e74dd513, - 0xddb1aa169b3c4690, 0xdf789ab57e2fd368, 0x085b8088ebc8bedf, 0x3018c1b373d64e9c, - 0xca8d30912f0c9e8e, 0x5d1eebf13d5bc8da, 0x65673099ec87064e, 0xdaf4a1c6148b0c24, - 0xcbfd66fcfc844c67, 0x5ffdea20820b1173, 0x43cf7f951f687d5f, 0xc413b9759a62b056, - 0x0ce14d5ceeaf0ce0, 0x398fec3f4a66e77b, 0x9e91f04c9c39085c, 0x7e3e0f06a7dccd96, - 0x3bb9a48d58e8e333, 0x04aab0933871f4d3, 0x0b05aa83bdebdb80, 0xf05f9c6016029017, - 0xa5a2b04f3ecd115a, 0x8505fdb06fed3ddd, 0xeeb802489812a4c0, 0x5585e8c2bbb4e3b8, - 0x97b8b32233cb898d, 0x95603ac61e866d53, 0xcab9261ab47c7f9e, 0x0bb8a8e259936f00, - 0x69b9b9a168e58239, 0x75e5246fe3d2a10d, 0xaf46236e0ec60a78, 0x6a91de2e194c9c61, - 0xd3266726766fa596, 0xba5ea14792783a2c, 0xfabebe9dd7f781ed, 0xb51156ba15ae9dc5, - 0xb20f501ead3862b4, 0xf2dd5fdb6cc961f7, 0x5f8426b83c824e38, 0x424a739eda53a9cf, - 0x2343e04fb7b1feb6, 0x3b35733927ee86b7, 0x84c3bad372d9842d, 0xa8a830795ff0b3ab, - 0x25bedc46c901f252, 0x502d2b7e050f277a, 0xc382184d30f17fe1, 0x36a5a23597be6717, - 0x2373429f7242764b, 0x2578ad1297d0e6a4, 0xcfb8c405948cd141, 0x8e6b57f03a0d083c, - 0x7a0d352129b0a8bb, 0xe38965c32cb5129a, 0x4814b99a7a164200, 0x8bcc5a9e0874ba04, - 0x793eb99a908f1aea, 0xce8468c8888e2129, 0x7c3f9f05c52f1913, 0xa362e468c7269bcc, - 0xe615f92b5fd3eaaf, 0x2cb8b7b97fa5df0f, 0x3cd44dcf1cd642e8, 0xa2bc70cc3be24e2d, - 0x1d506e9ef8bfb016, 0x562bfb0506200284, 0x0320d6189d372ec4, 0xc084244470d15127, - 0x80f88cdb6c5e35f2, 0x795ecd96eadf2dc4, 0x9170e3eaf62c2be4, 0xe5aab0cdf25137c9, - 0x4b82cd05b0120763, 0xd5de0b714bcc1d47, 0xfe5867e76128cb2e, 0x5604660e35e0bd29, - 0x8a13c1ffb947ad99, 0x7e554cf960de404e, 0x9eee2b723efa6de6, 0x3c69121996560dac, - 0x4b5ccc1a00e01230, 0xbc19e81a920b2956, 0xf9dde3cdf6bdd4f0, 0x6ebdad6f720598fc, - 0x51c42eac52da104d, 0x5999b713130400ec, 0xdb0649cac39c4d9a, 0xce6a20d591da309e, - 0xe1b7635909ab8061, 0x78d8d0cc7be4e519, 0x7449cd0af3fe9ca1, 0x53cd0c08db64e7f6, - 0x249e5874212e6e89, 0xb98711400886208f, 0x0ea0d5e557c305d4, 0xf0aa3d5478fd80dd, - 0xcdab3e71184ab63f, 0x04237f6beb22721f, 0xcb298c39cf00c9db, 0x212f84ef0ed67d22, - 0xec380d4c50fab479, 0xf190ce3fe629ce91, 0x5a363094b7e1c4c1, 0x75c583bc7b91a240, - 0xd08c593d3bbe8d3f, 0x8eb1fe288be9db18, 0xb6e9016bdf25b0a7, 0xc89da07100063f58, - 0xc626face1111628b, 0x043973a3337faa0b, 0x97be7d823ca8f2f1, 0xfb1d42ff44fe15d7, - 0x22b526ef70ddee36, 0x316cb494b50e1258, 0xa5f2409d444f61b0, 0x01a5abd75d1bb5cc, - 0x7774a162f37fb553, 0x55fbe2594c0c9d4e, 0x0952acefda2efe42, 0x6e0a57707814efb9, - 0x97ff9fa6e04546b4, 0xb1180316f83c5cc3, 0xf20a31aa1b7de281, 0x8cf06e265fac4ada, - 0x87b0292ef4f26699, 0x5d621ec7f145d82d, 0xc4d44572389c1f33, 0x695aecee4e0477fa, - 0x2699febeb996bc05, 0xf5fc994b0a8c533f, 0x48d3c074f3a880b1, 0x01eace10adae1893, - 0xba49c1c25141a3a7, 0xc0d5fb6e31ebf576, 0x8b8d1c1348654635, 0x7d7fe10212ebd9f7, - 0xe399472c3e070ddd, 0x1900e1056689f0f7, 0x09017c0e08deebc1, 0x16f4cc43ef3ef63f, - 0x93736eca9511b7e9, 0xb9ab1c9043cedc28, 0x5720a09b37134aae, 0x0159421e16b8acd8, - 0xd48160e03c7ccba2, 0xccf08599cbbd4aff, 0x9bcdd0b51ecc78a7, 0x9c62af63f1b6ecb6, - 0x32eabb88e76fe029, 0xde2f0c5244c4f7c7, 0x5a152124e885ebc1, 0xcd12e01ef65fd26e, - 0x1dad818d52628acc, 0xf53c45144547502e, 0x747f4b0c5c01c9f4, 0x8816f3fe2fb57a52, - 0x9ca7b3bbdbbd53c5, 0x20804234ec08500c, 0xf102e8fe45dc731f, 0x691c42f9cc9ae110, - 0x8d1b4b2fc130f823, 0x2f5f72f101b1ef24, 0x1e42a3eddf059848, 0x1a753bb9b46dc300, - 0xc7d64082a345c2a6, 0x0b14b0a3230c2c7b, 0x934eeeffb51b0ae3, 0x347f7f94e7e5b383, - 0x00ca7bb8f9f245b2, 0xbcc5a5b4b089cfa8, 0x1244df176ddb33f5, 0x2a25c4f32c84947e, - 0x8280667988f1a648, 0x622ce25bef57c04b, 0xa94eca220dde40fc, 0x5c59813073c8bd21, - 0xdc24ae3fe53a2a0a, 0xa348cdebfc371ac0, 0x55e0497c0df2ed7a, 0x95652248e49a4e96, - 0xd61e638efb9c1bd5, 0x1e8c33500df72884, 0x67ccbdf5d6adb48e, 0x621314f82d999679, - 0xe56c54a92c6e0260, 0x19b6bdef4acb14d6, 0x44ac406a0b469cb4, 0xf242f4b66be46c20, - 0x4936fd6b911491fc, 0x7aa867dcf3f4ddc1, 0x465189f73171dac4, 0x601dcc3af287eb1b, - 0x2420ea0fdc9d074c, 0x8dcfde5a530edbf4, 0x06f43f025d2e6a39, 0x60733c1a72d5b55f, - 0x81c9a78888297733, 0x7bf124908c8056f1, 0xd35c5d513467a75f, 0xbcd4dee1b137cb14, - 0xd0bf9e3a9aa00909, 0x7c1d7cf7bf2c6f18, 0x3d10b718c9af4ba2, 0x4670cc7a0203bbce, - 0x3c33e7fb842a3e6a, 0x9b02b918d4a513dc, 0x0c36c084493180bf, 0x4da8af74afe9c2a5, - 0x9cd88724f430f04b, 0xbdd917946f6eca33, 0xcb3d026cd1f31f46, 0x5b6b2919fbc092e1, - 0x057dc9931c1f6181, 0x5ba1d1cee68fc641, 0x173b1418fd9f5ccb, 0x1eb708ccfdd9f875, - 0x4711c82e3134c3cc, 0xd53f67b561a82721, 0xb802a9bd2fa60255, 0xe024652dc0003c28, - 0x9e0f412ee75bc32c, 0xf61982c7f013c25d, 0x1cdc4e3e8e684d66, 0x9b85a76b62a9316b, - 0x9d2a50548d484d4a, 0xf26af6a7399c13a7, 0x0d0e7441152bde3c, 0x8fb034ec62d7c16b, - 0xdefbefb90acfecd7, 0xa1e1e4d30be7226b, 0x2ae71e349bfcb0a2, 0xce801f56cbbf42eb, - 0x608bd12081cad771, 0x5342f9e593be8845, 0x9ae6534e074e7d08, 0x13e22e52b7a90013, - 0x2dc523be27d3b175, 0x9feb30dfd5f974b1, 0xe0d87925eff0a04c, 0x8787eb00a68e4643, - 0xfc0e96ea22e054a9, 0xcc1701190f3a8f02, 0x1c9663f661507b9f, 0x15c1add4655afcbf, - 0xcde838c421301fbf, 0x531637134f09eddf, 0xb405c7df0ae055b0, 0xe8575e6fdcc1edfc, - 0x3cc67be56793a2be, 0xc9903b10aa0c0953, 0xe041456d9bc6dc2f, 0x4ecb8466a9e6a5ee, - 0xc012c11d372707e6, 0xa3e735527bfa82e2, 0x8e0f248474d44fe0, 0x21b3c85df9f222c1, - 0xaa8135ca99ec4fe6, 0xc4ab4a98a9c18770, 0x6fdb152d7f3da26b, 0x644e478a690c6652, - 0x406a534b71272c2e, 0x219ec3eadd7eeef4, 0xd645ca04f2015d41, 0x6a1d1dbbbdbff229, - 0x5b50490756b0431a, 0x38048f8253d17667, 0x3cf48ebeca2fae83, 0x7da044ecafd2f3e2, - 0x118cd24c7c3956e5, 0xea676330ee67c5c6, 0x3201bd94660e5fb2, 0xb8821712ab2eca8c, - 0x1ad5738ba7fbac1d, 0x38652048ae58cffb, 0x4b2a60b28e78a003, 0xf6ebec5b87cd9dda, - 0xed0481c255f75efc, 0xa755193a41878ced, 0x155696b1dd88b1e8, 0xe99c714bce6a5ecf, - 0x00310b9dba0b94b0, 0x10e683cb9df5531f, 0x3381c9e82c7c5653, 0xf19158123cd23661, - 0xe62deed987bf6b6f, 0x639354a80066ec5f, 0xf214d92c6388ea1d, 0x3e3fcc4c7e876b0a, - 0xc18029f88a4d82b9, 0x1261f31df31f7f6d, 0xa5baca538a7a2a8c, 0xbb53e60d8d668aa5, - 0xed1254a8ef8e37df, 0x5b8d9f15a0e49f65, 0x6832feb72f90bfd7, 0x00a9b55b0ff10768, - 0x17f469e7fc370071, 0x5d033452209f5261, 0x44571757e9b1338d, 0xe44a66858543d272, - 0xcde6ac9a1fdf3fbc, 0xe159620970a6e8b5, 0x2a723883831aef9d, 0x17d32eb35149c780, - 0x1c8e7da5236b5bcb, 0x6ae61aab0d2a03db, 0x1972738816cb56cc, 0x74c6def6facc85fb, - 0x8dc97def32eb8748, 0xae3d45da534754cf, 0xea32e5dfa10b0371, 0xf1b5a87d3d043788, - 0xfd34829eb7657af6, 0xc0e566d735030fe0, 0xe4ed08492dc1cd38, 0x07a2a366f63b0431, - 0x012fa7d4a4553daf, 0x9a8e819a42653e9e, 0xaec6852564f7c1a5, 0x14910e6dde51ad8a, - 0x963b1397f8771718, 0x34a9c7d91283e0c1, 0x68fbf6d9b7107aeb, 0xc9fa3124d0942c48, - 0x9043b293db723df5, 0xc06a25dd00bec966, 0x0ba7ed3066a4c192, 0xf21f6f3120636cf6, - 0x4a4c733bd2f0fd67, 0xdf91d6124d1ecde9, 0x70d83d807b2bf6d8, 0x4bc907f66b104180, - 0xf0747da4be3e84dc, 0x52fb089ab0128eaf, 0x8d15ab8af6a35ecf, 0xb3f5506c835f5e18, - 0x41eca80663db97dd, 0xa22569e96f3344c8, 0x83702d22d3425cdc, 0x57d98a416b9e4bd0, - 0xfe9a57998fd66b47, 0x0a80f508e2e1c411, 0x9399ffe916174b20, 0x5a9a65fc00036081, - 0x20b7360f5ef59200, 0x82ad58207b1ace5b, 0x35d053f927d5d565, 0x2f4adb66aaaad0ab, - 0x621c06675a777dfe, 0x6317515f3d12cef6, 0xe3c539d42e48fa98, 0xc7a9b9f7b9de2121, - 0x562bda3588e92c27, 0x08bb56faae7f0927, 0x0d29943c33c501f3, 0xddb015f8d1a3a6d0, - 0x3a8ff95af791664f, 0xcbb7a029dd7e70b3, 0x52e66fee6a176a27, 0xde0e5b78be59b767, - 0x42b00114e4f4e185, 0x76a771849f369b20, 0xb559a92323537530, 0xb306ae0bd20da82f, - 0x1841bb2a99b6bd35, 0x4c18a060f0990eca, 0xfd5e2bc82af69f99, 0x5d828bc3df22ca1d, - 0xf5180ee22eed0763, 0x7d8d3118615280c0, 0xf73d16c82238ac05, 0x55d5da03262b8290, - 0x559fc8fa523d1965, 0xb4e4aeaed96cdff6, 0x270bf0383069b148, 0x7ef1efeb323ea3bd, - 0xf6095dcc94554075, 0x27c1a7f7a0b72a9e, 0xb561f5ebb1646df9, 0xb5c65725e318a6ea, - 0x109cace70c8ed815, 0x00f4eb2c721aada7, 0x7f0a28ef6d1b26b3, 0xc2655ee52f5b677d, - 0x3e620e3aab7256e3, 0xb0fb61983c4128eb, 0x7b90539b606256a5, 0xd6b924598be38c09, - 0xae819c4e7fe39ce7, 0x438df3ceeb4fc5df, 0xd1fd3a72f1264f71, 0x4a1e60a566a753bf, - 0x73c93734250c055a, 0xcd311fc078d3bb47, 0x7a3f16e7138936e0, 0x652626aacb432a5a, - 0x789937ac9417c8b4, 0xdd273e3949e50798, 0xeae4e15f060dd03b, 0x8e585f054e156d00, - 0xb020f5a4b05dd283, 0x05ee937d4f158b9e, 0xb8fc1c1b9dd5c18c, 0x0f183921118c989d, - 0x79962c492209be66, 0xb1574ac6c5175cbc, 0xf27a061ea93f4ee4, 0xa2e04742ddf6ae6f, - 0x65885053523a1473, 0xca9600a03f869d2c, 0x95f21ca21a561408, 0x9b651d6185ce6b05, - 0xf66ceaa88508ccb1, 0x5e20866a1bb1ba9a, 0x55639c33ab8404dc, 0x05de5b80f5bcced9, - 0xbf9e88989fd51b67, 0xf89e8562229e0bee, 0xf06ac14fbfdfdadd, 0x9d483164e8eee731, - 0x2cfa031eaf5e0367, 0xcbb053427ec593ff, 0x6ccf8beb3da032e6, 0x5ca6902f9c153ef8, - 0x929ca463a210630f, 0x7e21c22f2f4e5c7a, 0x142d9e8a58a3091a, 0xf94d3028b18f4c9b, - 0xea85ca6ee1bffc02, 0xb6d311f00255514c, 0x9905bb88b4c011c2, 0xb99d8fbee37f248c, - 0x4ac9c159cb42b938, 0x7cb2bba4134f3a35, 0xf8907bf4b9b6d0d7, 0x9e41a5dea4024d1e, - 0xf14c5783c416c12b, 0x9783b73ffe8a4418, 0xd4aafc1f077a3b17, 0x8b5b3404a17ca755, - 0x6c650fdebe87d5a0, 0x43b9d80e4a6bea2d, 0x64166e200c84323c, 0xa424343b3c2b71b4, - 0xbb4771a4c205a0fd, 0x7d8ef6034410c09b, 0x737838e8c97611d2, 0x0bdd9fcbab2e19bd, - 0xc6353e3fecb963bf, 0x20433d5def7ebd2b, 0x13056025d6a024e3, 0xe327a1f6bce4443e, - 0xa4b4c4adce80bdb3, 0x71aaafd4a690784d, 0xf0140d1edb1867c7, 0x59d6dd39cd8d9e09, - 0xc9f3b09a5ae3d5bf, 0x7a816bff9b9aa336, 0x6ab12141da863b2d, 0x0d57b64cb5461e09, - 0xc9aae96d66691c86, 0x967c5e03ecdcd3ae, 0x75636b9433ca369c, 0xdd9a1a39c46a437f, - 0x6930f4c108740495, 0x49c94a1b63d59f73, 0x43294bcb38f64caf, 0xbb7a2ee1e2943b85, - 0x88dbbb31ab769396, 0x9b3565fcfc61ad16, 0x3f34afa53b3c6023, 0x3c8ae91fc02d38a6, - 0x087eccba051bdd4d, 0x63db4eae0e33007b, 0x3202b108af3fbd4a, 0xc0363662fa576a9f, - 0xd53da44bb9220b59, 0x59bf6fdde0715921, 0xf73710e0ff3946ec, 0x6ae04ed37c52f71b, - 0x4a05073fa1faf0b9, 0xc3d9df90a36bf1e1, 0xee87e93687107ba1, 0xfc4bd84785dc4393, - 0xdafea2a61b5bb2f8, 0x62c175e61d96ed94, 0x69d3fc898d280d57, 0x2e89d9fffac8c2ac, - 0xdd737fbd8c1afcf4, 0xdbbbcb9200fe9dd3, 0x71926cf90848bc5a, 0x06cea2e421ba050a, - 0xd84c7023f25b4b5e, 0x2bb39653b3848d5b, 0x5083b373235c4338, 0xaeba9811b487ce91, - 0xe7cb02fa8e88550f, 0x71fdf6ad42bddad1, 0x561ffb2678f1136f, 0x1a05b9acafe8cede, - 0x5f701d6c17e956e5, 0x99fc79c6201646e2, 0x64ab46f40de6d466, 0x1180bab91a59db34, - 0x42bbb522cbbf4e2e, 0xd41f7b38edc12fe6, 0x9e36edf9f84a1ac6, 0xedb4ebab7f795708, - 0xe7ba4ef211007708, 0x08dcc0ceaf6320b1, 0x23bd6929c12f95ca, 0x2dc244f376bd94b5, - 0x107a2a3bbeb5116c, 0x458717cf7aed9ab2, 0x5edb6cefeb9457f7, 0xf06e7ecb9129d4e1, - 0x84211638eda4201b, 0x29d00e3640aaab78, 0x9b09bedad5d19360, 0x483d60f38a7ec3f9, - 0xa8863a4ac57d5d09, 0xd3c326b3781d71a1, 0x795452e65afbd3fb, 0xaee1ac38808e38d5, - 0x3e3eb84504f8e2e7, 0xb95b025147ee6582, 0x9b9694b5b201ea6b, 0xb6c146a2139dd5d1, - 0x2c0ba6a4d83e9505, 0x29acab595a97577b, 0x471031b8ba182ba4, 0x2479fa7fb7865d63, - 0x9fc666b91716df14, 0x4d65b078a7946f6f, 0x4b1d699d012344d3, 0xd705bc7a68a694a6, - 0xd40fc82cb867c02c, 0x467afe426945ed88, 0x4194bdd9b2396817, 0xb135b5241575f773, - 0xad71005a084efbfe, 0x6bd3545c4e09431e, 0x17ed2f5fe34bec3c, 0xac209abd795c9f48, - 0x154c56f6da55fa90, 0x7ee9d857637c20e9, 0xe081e46d00bb8a78, 0x1552bc5f3ced9ea7, - 0x334cf982461ca7fa, 0x34adcb0f3092d53e, 0xf429287f123826d8, 0x487b3403279e9b9d, - 0x9b2084ce2bc0475f, 0x3cd66eee693483ac, 0xd85fcf770a6f35b8, 0xf1ae32535668e9f1, - 0x4eca60d72af6b8cc, 0x26a67ce44b0d4e45, 0x92ea23a19aa37786, 0xe28d8385ed95add3, - 0x81487ae6521b8118, 0x1f358bbc23d19ce2, 0xae99913a9f903361, 0x28c84170bf75f01c, - 0x3f752f66574a73d5, 0xf526e369597fe122, 0x583784bd6ad03c0c, 0x221bfd49fd11fc25, - 0xf0f0a685d2fcf473, 0x1e8f897e1f281b84, 0x9bdc2f3886c93c6e, 0x3511a7654986e4be, - 0x56889f045b81e280, 0x0a47a9effeac8264, 0x8a7790991c2b3807, 0xd1355610dd4cd735, - 0x7a2f528e0b403529, 0x15aac3f9edee9224, 0xa1485c671ec58c77, 0xc25a6643430c1e63, - 0x4c44a753ceaa1798, 0x672d769f40746201, 0xb2c7d56047a3e1d9, 0xc42e1372c2459b14, - 0xe4cc09049fa893ca, 0x56bf320d4a60f01f, 0x67b984ae2b62446e, 0x8989841fb624803b, - 0x4c1461698aeeba17, 0x677fd64fc36801ae, 0x3e3cd2b553b17ac4, 0xeccc1b8054eaf161, - 0x5da2ab36d63d132e, 0x220dc943ba3c5acc, 0xe262295e005e1a7c, 0xbde5c26f2bae49af, - 0xd09cce1e89b82f62, 0x96923415a8b7767c, 0xaf107ac806051503, 0x45b3140397d45791, - 0x37f2b446d06340ac, 0x828a9b8067931398, 0x87fca05480701cda, 0x91a8cbaa7fdf69ae, - 0xa1a23a5212727570, 0x747c1900a7dfd425, 0x04e18af9ef6c91db, 0xbec2ed735a5d5004, - 0xd600e00ea1b5d17f, 0x851dad8dabd0df7a, 0x89a5110ea729438d, 0xa36ca01355a09cbc, - 0xa44be2dd3ab542e2, 0x8fa73c70368d891d, 0xe2980d946dabb73c, 0x12b24d1a24b08997, - 0x9e7e92642a28d7fb, 0x55c29b46fada374d, 0x13854b8550b5ffde, 0x01a2fbcb401b7a07, - 0x92f0042bcc3f3a1c, 0xb35986ff1d7436e9, 0x35b770a76403fca7, 0x3ad48e8b978b6426, - 0x62a7c170c46f9653, 0x10bce6ef1293250a, 0xeddb8e242c99f456, 0x3608e044a8fbf3f5, - 0x568e6975703a6c09, 0x3362177c4c5af90c, 0xaa5ce0a76b705dcd, 0x053538eb3d9c285f, - 0x3f3c0f2dc47fd2cd, 0xceac960642c1fae6, 0xcad6755cf051427b, 0x3f4111490943282d, - 0xd3e6bfa20fa02904, 0x691eb5d4dbc6b483, 0xee6c43bb9a5f4b56, 0x52e10557bd5afd9f, - 0x49df833bbd60f151, 0x183f847fc8f65820, 0x1e16e3124f9b0640, 0x7811b51f9e56c0b5, - 0xf0e14dacd42b614f, 0x087e61186b59185d, 0x7fb066b612284ad5, 0x302184ddf4779f1d, - 0x40f96a3c739ce405, 0xea2d8dbd4db0a5d4, 0x97f64bc5d554f7a1, 0xfdce68966c1cb359, - 0xafcc35cb0f76f824, 0x7406fae013563362, 0x773f88bf874dacad, 0x5142fc941a2dc4c3, - 0x17f157dad76ccaa2, 0x168b361a94b78d0d, 0x3c83fa7aa96b4017, 0x660f86a687fde823, - 0xa1b7dce1f7e1ccf1, 0x36bec1aac7cc2392, 0xf46fdd7a4b395914, 0xd1ddea46a15c6af8, - 0x5c16e0b4a6c3c82d, 0x5e9539464ccb7bac, 0x1335d48dd77b0587, 0x2001eb13e7a7bcdb, - 0x3a0aa1a8ddac5f1d, 0x5a7666c67bc9a8d8, 0x6607986b70546ff1, 0x9066572b554cf501, - 0x7af3930a27cee503, 0x10084fac6c99033f, 0x8a197d720ada09a2, 0x1ca50121f93e69c4, - 0xa7e232ea097ba009, 0xc374d925ed190ab9, 0xa7b77cbf5e297b86, 0x68626e49aa22269b, - 0xe42537afd526277c, 0x2dc61c40f85bf420, 0x4023266094092ead, 0x8fecbb31dc6a97cc, - 0x1116febe8806c7a9, 0x46815d3a5162db8f, 0xb09f5743edadb9e0, 0x51dcf8932d9f977b, - 0x03ab57de6cc3f43e, 0xf522cd507f97ccda, 0x3fd85a313a213591, 0x0e90ca8e2fa9b045, - 0x08be11774eb6c544, 0x90ccfd20dbab6997, 0xb97f1452fa062994, 0x31ede623225151f9, - 0x9566387d54ee1d13, 0x26b31583a7067927, 0x780dba0338a49c14, 0xe2eaafbc08cd5083, - 0xf688511504ada1a4, 0xa988c26d9e88a043, 0x76c173e61746260b, 0xddcc85f9f28e8608, - 0x5368b74633e6e65f, 0x670ce025c33fe9b3, 0x67d778babf7061e5, 0xdcf0662b36b8db45, - 0x12d28437399b1b8e, 0x8acef556430712e5, 0xbaa6957f82b81482, 0xed2c284db8e0b889, - 0xf611a9ca5f0bdb94, 0xc3f6115ac847f7e8, 0x546128261727deda, 0x703e6afbf86f68d8, - 0x81a7231cc4c0d7dc, 0xc9accd7b4d30f94a, 0x9fa1b50c0a267e9a, 0xdeeb485699482a4a, - 0x7c5289696a96fe59, 0x08b65acf0e2b4ba1, 0xa4be79d0210a62c2, 0x03409e0769a5178d, - 0x53e06d25b4642962, 0xb2e38a474b3fbd0e, 0x8a60af2381a93fad, 0xbe60ab36e0a08e4a, - 0x71071d8cb0becef7, 0x77dc949745a03bae, 0x965c8c2a1ee74900, 0xd52a3839cb96eac9, - 0x3022d76caa2d4af5, 0x1c80b7ac32d82b60, 0x4035434f233232aa, 0xa676acf564260f55, - 0x13a6a12cc0d04b88, 0xb7fe52f5cc16fc94, 0x70c72266ef12ca0e, 0xcf7c392d38b476d5, - 0x8bd08ad474221fe6, 0x0b6fa4f9dfa978eb, 0xfb7f1050e12db233, 0xd13ca04b30e0d76f, - 0xa73cc9130ab43295, 0x6e2466aa8c29fda9, 0xd4ad1e99f85ee20e, 0x67371b6228e8fa44, - 0x635c3233b83c09e8, 0xc3933fb5a27b6335, 0x170688c7ae17983a, 0xc32802c13c5cde3f, - 0xae31772dab5f1fbc, 0x94e2b5a5d3a8676c, 0xbdf06016b12b4412, 0x6bba7668de2d9f11, - 0xbf6c3adc349df560, 0x734ea81429114e35, 0xa22928bc9a78e9b8, 0xb915ac991b12e1df, - 0x0ea8defa80690a39, 0x5c939835031028fa, 0x73ab9fdae205ac96, 0xb6f8e631a4e0c91e, - 0x2515a575c78b5cf0, 0xc6f81e807d60fb63, 0x76d3fbfc462f6c98, 0x945aeff6c0d13727, - 0x6890fbf6b8623ed0, 0x4fac86b641d21f88, 0x9b7322871b21bf53, 0xb35a67ad51b023c0, - 0x348656d01b82fbd4, 0xdb316034359c2efe, 0xfcadfc3ff12c0949, 0x6196f5538bbde0ad, - 0xba4d863810afdf38, 0xb38542d85e3758ee, 0x9f21a7979cdd3ac8, 0x690751a408d1a47d, - 0x44f69635fd4a8a13, 0x3ab8e1128823e440, 0x6b0494f235d4aa92, 0x1ed3a33180423859, - 0x07ece0c6667ae412, 0x34e22c1aa207ba0e, 0x9aa674b5b0941d60, 0x49a9da78ad9a9733, - 0x586c75ccbcca351e, 0xd73a31e4b7db7807, 0xdb21c961eedcb37f, 0x4b08395b1ce05d2b, - 0x320609d4e28cbcc5, 0xf9319d1e006ab08c, 0x08e066bde96b0f7d, 0xb6b4a72ba463c80d, - 0x22cd52e9e724680b, 0xffa16ab3d2ab80ad, 0x16947f8af9c03384, 0xd119383f2ae37006, - 0x527bef3e8b58013a, 0xc055bb2f6ccb762c, 0x19577840e168962a, 0xaead151ec50cc24a, - 0xfd8f18a9d49f4adb, 0x055517213ed2e3df, 0x33aeb89004077685, 0xd1c97977cd82b6d3, - 0x43dd6c28d68ab339, 0x9def65dfeae1fe31, 0xeabdeec14d875b9e, 0x947dfe4c9e36e380, - 0x899100fb32c7a88a, 0x7fd097aa138f31ac, 0x0eacacf74a8df40e, 0x1789e52cbb84742f, - 0xb4bf1ed70043406e, 0x02d630159dd8f0c7, 0x79f5adfe443be4b5, 0x5ab0d66f2d64d816, - 0x15b163afb596c3a6, 0x65f58bc798097afa, 0x683134e6b8ae2d23, 0xf087942fee100cae, - 0x9513c2f9a3c170c1, 0x32000df3d69f809e, 0x6eeb10fa9f30839f, 0x81c65857d0155b2e, - 0x0083c8e65798035d, 0xe4deb91b289fec98, 0xb561296b6307e877, 0x3bc2951983697131, - 0xe52198eccee429df, 0x472efabdd2d5f26b, 0x3a7e39171a53afc7, 0x0bb22f1da1aeb348, - 0x2ed2c3af4a1bbf6c, 0xa9ace381a6b98023, 0x27a60b43b7b02b8a, 0x07e7717219ade8fd, - 0x0f63f2eef17b5251, 0x6285f23b3e7fcaf7, 0x238e194739a1bafd, 0x8fd260a620b0044a, - 0xddb52bd86d75a344, 0xf450b8dbd8283259, 0xe82de9c7abd04644, 0xc2a18b87f565b036, - 0x59b01c7848c8ea2d, 0x5ac7a874966febbf, 0xdf090ed98287f24e, 0x6ea7aa2b9b7cef0e, - 0xc5197416e1287885, 0x3fa796433d0e26ec, 0x0f33685bf9fe476f, 0xf8cb8c84481f469c, - 0x3cf530446aebb793, 0xf7380e07c075d257, 0xeeb14fadb3ef370b, 0x33545046dd6a53b4, - 0x1245e1e1da9fe75b, 0x76c369049dc8be29, 0xdfa77446d69ee161, 0x1540234fb76c6699, - 0xafe6846ed4a81409, 0x6140a2cbfc4188be, 0x110f8f6d6430a752, 0xa4984711a513d8a1, - 0x2a15f090512c368c, 0x912a67c34b0de4f3, 0x586062bdbcb02fae, 0x628c6141fb9038bf, - 0x715422ad0b9379b5, 0x0078a2ce795c53fc, 0x210232c04be1457d, 0x26582c1fdbf672b6, - 0x5caf43acd1baa06e, 0x99b9e9b37b486074, 0x9615534f88ab5350, 0x1d78bd7d331abab8, - 0xea3e8519d96aa0c3, 0xecef7968776b2c3d, 0x38c527d2e7cf3a18, 0x38af9da6fde0beed, - 0x0c04b4e0a21af30d, 0x8cce3157d2e9755e, 0x54c0ad71e1d2b10d, 0x67ed0cf0f1ff059f, - 0x18b942551e73a589, 0xcdfc68c53fb7c340, 0x1613f99449bc546d, 0xafe1fe51cb4c3c63, - 0x26ef54aaac797b19, 0x3d3d302a4504fd20, 0xce90472f2865cd6e, 0x2fb9bd1228b2895b, - 0x094e4de35046e267, 0xe1eea3de5f4a31fa, 0x3f74a57fdb0d0389, 0xf00f674c3a8a9602, - 0xeb2b786f373b7976, 0x1cf4e637d7dda0bb, 0x69e8535bee91f776, 0x60d5887a41c84509, - 0x2ebd9c94c770a221, 0x917a2e35285919ac, 0xd2c45f8ecfba0904, 0xf4490ee59f6dbd36, - 0x89758084112c5427, 0x3ab8d94b38d0a452, 0x0c2250832119ecad, 0x74a6e3be4ea56ec8, - 0xccf08af81abcea36, 0x55768ae293c98e98, 0x92e9a532c3c34582, 0x511930274d2a7740, - 0xab466f90a713b9f8, 0xf3adb12f5c323c90, 0x915ee582e58f18a4, 0x1f20eaa4873040ec, - 0x1b3f7358c743888d, 0xe49119ec6b3823e3, 0x8172d693336a08d2, 0x34e9e46c3b3eafc2, - 0x01005093a09f5141, 0x4037102d139673c7, 0xfe8fa22aec8b26b9, 0x2768b412a3749b4a, - 0x1d7d17cd76e7f0dd, 0xacfa3e3021a83a41, 0xf7142b4a43124a2f, 0x407935ffa2974aa6, - 0x123f50ea462c2196, 0xc9488ae4681d4518, 0xab5968557b9a57da, 0xa83205bf5f46dc05, - 0x151e4eea6953f8b8, 0x33000a3fadfa51df, 0xdcc1ce0daf3fbda5, 0x7d371271bb7b8768, - 0xaf876bf0f638bf8c, 0xfbc27cfeb07a2f5b, 0x356c93140189e8f7, 0x5b8f5bc77b318664, - 0x5b74fbd6d41f1254, 0xae2f6bea698b9e41, 0x94365509202b40f0, 0xc3dca0798c3f22dd, - 0x184b26e4df1c35ed, 0x9539430cd2984773, 0xaf2d69e1477f45b4, 0x86652c5509baf2cf, - 0xa041901ea37a4701, 0x73259eb1554909e3, 0x7a37c6addfed8bcb, 0xd1cb93fdb1488dbf, - 0xdd2d2d57afb62217, 0x4f7c8d1457c435d7, 0xbb7105343bca4b4b, 0xea1d93ef3040ee2f, - 0xb4e82168574cb690, 0xc8357ed16ebb391f, 0x6f1b42fb72669bea, 0xcb108bc000a371d5, - 0xf400d48e14a9d057, 0x2c3da521a133febd, 0x76510424e78badbb, 0x18094fe0197d3b30, - 0x5a272063adc5899a, 0x476278bc4450c187, 0xa02d79a633f6ecbf, 0x3a7242757e766580, - 0x1947479d44813265, 0x446de9f0e6996e95, 0x5f7ca2bc06058fc2, 0x7ed52bdb4d777800, - 0x10640de1c837e875, 0x63c7555a3ce86856, 0x39371a89992b1ed2, 0xdd61d740f69d0528, - 0x161979d08b7b469e, 0xd8facf1dbb5cb36d, 0x5ca558cb6f3034dd, 0xbbd151da22b8ddff, - 0x3547a8f8e49c77ed, 0x51e08bdd847ee961, 0x64102dd21a2152be, 0xc0cd2deefaf4120f, - 0x9e42789d521c9ac7, 0x536c2f5b6f0db765, 0x9c4427544ba98653, 0x8595169f46fab3bf, - 0x9667b31322bba3db, 0x0370aee42b3e731a, 0x30ab9192cb9287f3, 0x11582d84383f8482, - 0x15625b77e5ada1e2, 0x928042df1e154836, 0xfbf5fd6a015014b9, 0x0f9b90fba44bb827, - 0xfab046fc7e1829ba, 0x34c9363ca080cdb7, 0x06eadf16902def24, 0xa2102654698aa05c, - 0xf0a73f3aefed8738, 0x23d3e51fe209e636, 0x37d6fb8b893c58c0, 0xe07f44673aedbff3, - 0x47ddc151fb702677, 0x0a2f193d8fd58539, 0xc437684f68b3b514, 0x504a27aa2d27cb46, - 0x8604f620d7f43527, 0x040efbb8623447ce, 0xe3324d59a59be502, 0xb5eb38fed8d23a19, - 0xfd71153ea417ded8, 0x25a73a99548275f9, 0x0628521041d8288d, 0x088b9d6c1707f315, - 0x092c5f3f9461d28a, 0x1c441af94b125106, 0x8e1905d7315dc4c4, 0xfd23631e22a3c50f, - 0xd3140c6522ebfbc8, 0xc03894d0305c9fe7, 0xfb20b5b25929fb96, 0x18681f8d368322a6, - 0xb6cb24de026cf8fa, 0x97feeceaa42d12bf, 0x06b649a3bbcc36bc, 0x473d8005b19d9fae, - 0x1f8669e2da459978, 0x0b47eee844e2b4ae, 0x08fe0fcc8d26c04c, 0xae0f5ec2480a64fd, - 0xf54f5c3fec69f08c, 0x27aec32b3c19aa0c, 0x0d0b7104db791eee, 0x50238ec8377686bf, - 0xb77cf3c5c66b01b0, 0x609f7d1d1a489756, 0xd7625de72ca98723, 0x6e0760bde39c4935, - 0x36f8bd89ac3d014a, 0x68c6963ae0261adb, 0xdc9523f224927f44, 0x927337fd3519c673, - 0xad11f6e6616da14b, 0x19705660bd8ab87a, 0x8037f0568ab92537, 0xc6183ddd17bf0d2f, - 0xa2cc5590377a0601, 0x1085853d7f72fa2c, 0x5f2e046f9d36cf69, 0x6091dd1f8a812223, - 0x1372e040bced2b5c, 0x7d9d5c60a7a71ce7, 0xbf6d231a4a39d158, 0xa7020e4829c225ed, - 0x2b969d03d96b9d76, 0xee563af36fe0ad12, 0xf6604f946ddabaa7, 0xb2588f41beae22db, - 0x11cffabc946d77ee, 0xba378ad6c75194e0, 0x0418fe7981d35c44, 0xa7b570f6674db09f, - 0xfc26294761b8f856, 0x5a7f0e51374a6c24, 0x66802d9fd51b9ef2, 0x2e6226e99d02e67b, - 0x36be1561d6f4125b, 0x627da10ad8c0e1cb, 0x3d4616f5652cfcfe, 0xbc2d9692eaf05c99, - 0x1cb24fc69d4ec952, 0x314671982680cc10, 0xfcf45fa682a86a45, 0x6439fe94ba8b61f5, - 0xaed728b2bca7f9d0, 0x212a783db05abbcf, 0xaac4f56b360482cf, 0x114fdf72282d8f4a, - 0x530b4e3a901d9517, 0xce042afeb531b3fa, 0x0fd4c9c33154ffb3, 0x2888ce24eba2bf96, - 0xa263dc2e3385dba6, 0xb7e82dc2ea1d7bde, 0x8b09585f6ab3228b, 0x1e4fe421177dcb76, - 0xcd74e6113cd709fe, 0xdb717f11b1dcf79e, 0x1e8dd13f4a1d0818, 0x13d32f111d88bbaa, - 0x64b5b8d00273f993, 0x80f75100463bc42e, 0x5089dcff2d4d1b76, 0xf15ae1228e98608e, - 0x844f04565e35b0eb, 0x8c857578814529f5, 0xc0842119c25a4899, 0xac84f6904a2b4fc3, - 0x1c864312d2e47e56, 0xb38e2eb706df7cb3, 0xab44b0836162dbf3, 0x14ac55ad7b37e1b3, - 0x67abf8ff3a2d659b, 0x708e8c3718f81e83, 0xd79f805950b8a560, 0x0450116dceb2fba8, - 0xc536a47b45f5f398, 0xa461e9532890b089, 0xb32d6f441b4b61b5, 0x5a1780861dcddd46, - 0xc4ca7bb284013428, 0xc8cfefb7cf5de50e, 0x0b15b5793fb03d49, 0x13d314e7501d8833, - 0xe63f33c59d25d607, 0xf0e6fe74d89a48b7, 0x6528436594328d07, 0x0de0dacdfb66d44d, - 0xa225ee29acf7a5d2, 0x1cfe84ae46fd37c2, 0xddddcf50454eb528, 0xa606bae54c111997, - 0x7f95e39cc27b797e, 0x37c10dffb2e36da4, 0xefba67590a452a2a, 0x24f36a6d50708f0a, - 0xb78bf2ace92bf877, 0x3fec905895ec0fb0, 0xa38ce77988a6bf1a, 0xf1f1fb337baaef07, - 0x4a8e9ef8e644a5e6, 0xbdff0256fe306885, 0x6ac192dc2f4993c6, 0x13cd3f6784ac3233, - 0x91940a96e59d04c1, 0x4273e20062ed4087, 0x9f1378ea52e0d1ea, 0x8e426546d52c2f91, - 0xf31484f00b6470cc, 0xb89b037324c03f1c, 0x2a0af557849372be, 0x4d73f4720b0dc5f9, - 0xa579c7e15a3257b7, 0x8b537d0179f81fc3, 0x4bcca4e93851409d, 0x691e09bff3f51f5c, - 0xc404cbe36275c847, 0x803e8928c1d4618f, 0x2857dfc76df09f9d, 0x2082511c02521bad, - 0x7a80546fc700e127, 0x73834ddba4ca8273, 0x9c21dc90e34dc330, 0x6e850dc2c07dd3a1, - 0xcae7c173bf49ac62, 0x561b6d7fe64b7dc8, 0x55e3095725e8538d, 0xc6a38cc9b49c16c7, - 0x4787c16e9097415f, 0xcbec873d5f30b6bb, 0x1362e2e8dd552c4c, 0x07347b16f28848bf, - 0x4abff3cd59f8806a, 0xc1595f0d01f9be68, 0xd9f682dc6c20fbef, 0xcb094ff00fd628df, - 0xc03062b8f46ac1cd, 0x98fd0f94b058832d, 0x7905d2d8bf45dba3, 0xce47aa2fe0d02a87, - 0xf3143779478bd2b7, 0x81751ee8128f1780, 0xacb0aa76cddefcb1, 0x2ee6cfd9f2a43440, - 0x53ffefbd272a3086, 0x5fffd81b0fec83a7, 0xce7be945d837f09d, 0x8b09427ae24cc191, - 0x045f520c4ce9bbeb, 0x3b70e02a6140666c, 0x66e4f17dd37ce310, 0x09e528bf1f6c2b8b, - 0xe1900131060ad79c, 0x0f9b53ab0bfac98b, 0x1e753466524d3bad, 0xadeecf80af1218b2, - 0x21afe1a444c4c9a7, 0x06d24fdbc93604a0, 0x5c654f1ee4d2e2e0, 0xe74e521055985714, - 0x1d9fff9e1fef4bcd, 0xd5998e718fe97799, 0xede690f20c324c1b, 0x058401fc625190fe, - 0xac4fed0b687bf35c, 0x7be364af4b9244c7, 0xea607e7e969fa48c, 0xbd095d6d09a19788, - 0x57f0d1a5f46b4300, 0x5f6b6faa572d0868, 0xe0aa7ab79d4ef9c7, 0x4e3832c192b0c5e4, - 0x3d9d17566e888c01, 0xebe1107cac4cb83e, 0xa3a480e0b4c9c3e4, 0x3d57fbecf657c2ed, - 0x46eb3e0b1ac4507b, 0xa10a0b89879f007b, 0x2a5f1b6482d46045, 0xae0239d6d79e6908, - 0x2b62259273a28b42, 0x1bc75c6518b25c2e, 0x59555f64a8d3b0de, 0x9e9a20942435268d, - 0xfae9f0063baaa4a8, 0x9c16a806248d4dc0, 0xd212cb26b9d3f9d6, 0x5e577dd2289923c4, - 0xb6e79073b15af5c3, 0xafbcb2ac5405a033, 0x2339839bd4f8a4c1, 0x2a2e7d166b297c8f, - 0x8b6e89ab1aed7394, 0x6ed99d00f13096b2, 0x25bd090211d17f99, 0x4a4e8010126211ed, - 0x1e5c31837fc238d6, 0xaa944122d454e91f, 0x2aa72b5dcdd1b8d6, 0x6399ccd1425b21e4, - 0x159ca2479c5164b9, 0xa522bb2089c286a8, 0x1ae365d2dfeda7be, 0xe79c93899c22e9ea, - 0x4e11cafb97461697, 0x16845a838913e28f, 0x310aad1ecf98fc09, 0x346b0d1db5283e5a, - 0x5cee2089a053bad4, 0xd064a21adc95cbbb, 0x4296ad446bc138e3, 0xa70b15caeeb377cf, - 0xff12ffb137a75eec, 0xbc5809bf5c3be822, 0x160b16191a788835, 0x4d13c212a3a0f8a9, - 0x6ba84585fcc589eb, 0x75215949767c65ce, 0x6e9156489208c35c, 0x194b5ef8852fc631, - 0xb1856068167e4051, 0x7c0e0d6085bd19e1, 0x23164380d2307cd8, 0x7d35ed3ed770c37d, - 0x38ea2cce718f0394, 0x608b48bc39e13573, 0x6e533b991629fc8a, 0x63decc3ade931420, - 0xc57c36d4478cd7db, 0xe5503e2f516322a0, 0xa1942a05307f2ef2, 0xab900abb16885cee, - 0x340e74005ade4fac, 0x76879506e9e4cdd8, 0x9e981bdb6dc1043b, 0x9e178c09e7274ed5, - 0x53d96f6ed451b70c, 0xcf64a14bdb465676, 0x47f15cef07394a45, 0x098908a45d0527a4, - 0x8fc16907a8e2f8e7, 0xaa959d41b64bb117, 0xa57742260431f266, 0xe5bcd5e6bed185de, - 0x222d8f044e616333, 0xb268c2f0be438fb6, 0xb1d8db987d971dae, 0x7133678a953a4b86, - 0xe9760ebfc9b99826, 0xed5a36ec1a76d100, 0x698a95beea0bd0e4, 0x50b7f5c790bf1092, - 0xf3e8e504bcd578e2, 0xf48bb6746be9a786, 0xee9e2e15e7eae906, 0x57771414ccf24506, - 0x0e22e623697eb12e, 0x6a5d8f568b8d1a59, 0x682bd158b531af50, 0xebb8484d5a950d32, - 0x747890722d662f0d, 0x557f3faa11523f29, 0xe3ed35294ae6af26, 0xc577ff7280ca78d5, - 0xe5b300474299c20e, 0x3aafe0f366503baa, 0x251fb73e209e33df, 0x3e28885fcc5193ba, - 0xe2dc3ace8bf069cf, 0x9de1d8d1c3561741, 0xc283654c420213a6, 0x4d4d984d42e2488f, - 0x700c66a393b50f5a, 0x568c9adbc3ff805a, 0xa0f0536cf084512e, 0xe84680c13080a9f3, - 0x90c53f6ec992edb5, 0x85a1a16794842d4f, 0x88a6b99807e765b0, 0x60835a754d24f896, - 0x3f48f4c1e3217aec, 0x3b5e362c04c92f64, 0x49e318f7f10b7a46, 0x245cd6e8ccfc37f9, - 0x00b7b86f5c1bc4bb, 0x2e1095296cd9f79a, 0x503586f3f889a8e5, 0x6f35d00da1c28d5d, - 0xc246156fb25355ab, 0x754341daeacfc3cc, 0xd12c618f1c804dda, 0x0a9f70e4706d1c24, - 0xbfeea8baf3ef43c8, 0xf14ad9914f6a0045, 0x790dbfab02f17c9f, 0x3651435f5c0bb594, - 0xd7c854314e6546fe, 0xe9ae6cfc8d33e9a6, 0xc8583921e0b9dc00, 0x54f75a9e71f6c28e, - 0x71a3856a90680903, 0xd61c95f9461913fe, 0x617d1cd7f4f768bd, 0x5d2c6b7f1652a849, - 0x7bc70ca8682e5da5, 0x3eca520d5ab3d67a, 0x4f9394d44e4cba30, 0xaf79ae35effadb0e, - 0x0dad63429ad0abac, 0xfb219bba29ac2004, 0x67ae855148b15c3c, 0xbea45602b1374aac, - 0xf9464f2fca0cece9, 0xebcfead82d7d95ae, 0x05774c8df003f2bc, 0x8c0c4a09493cf1bc, - 0x9c047854c64cbd23, 0x2e9924483b77a110, 0x29ede04b41424905, 0xbc837fb5d80dab4a, - 0xa0735c613e5cfaca, 0x0975af08f3152523, 0x2efecf0aec4d0868, 0xa9f04f1882d0cc17, - 0x41aec4c124552129, 0x9800fd7667ada453, 0x052aaae7548ac653, 0x78701944c6b3bd3e, - 0x1f37e94739892b1c, 0xc122e6e99f26dc6e, 0xec3a405e977e7e0f, 0x9843767bc4c3c146, - 0x5e5dc3eb8ae10540, 0xe6921a46aebf54c4, 0xfa5267004f77dca4, 0x24794de1b7f562ec, - 0x24f7e67cf814e1b4, 0x7178c44a4ef0143a, 0xdb20e37d5db7dab0, 0x4f168878c2eea10c, - 0x2c3c68ec66cd4eb8, 0xba0da78573822639, 0x77e405cdd736b94c, 0x7f2d25a33c75cacc, - 0xf44481015a124206, 0x22edb111738b693f, 0xdde445d68cb38436, 0xa821a541ec95dca0, - 0x69aacbe181f81d0f, 0xe60c433a8d682db8, 0x8431582d40ac17a1, 0xf0785797502bedec, - 0x57d332cebc86298e, 0x6959b43126f369a4, 0xc8c01d95a693f454, 0xa45d8297f27c9b46, - 0x269e248ecacc06db, 0x32f7b96bca8931f3, 0xc21ac71fb1500147, 0x76c337937ba41bd9, - 0xb92973589f24c9f5, 0xd5226b232e0ec3ba, 0xf4b67cfddacda3fb, 0xab212fd9a48c65fc, - 0xca670a3b9be8b001, 0x9a2ff1cb1624df31, 0x0c5cd280d8b5f78e, 0x2cde847c918d085e, - 0x8297c2aac028d2bb, 0xe0018b981de45639, 0xa4edc759c25d387e, 0xac007a1418d81ae9, - 0xca5a5fc597156273, 0x4e96fa5be40fb1d4, 0x9bcb619c530537f5, 0xbb72562572aafa67, - 0x55bc606376a81e2c, 0xd10c63d88febca05, 0x549c31448dbf292b, 0xa707b15a74daec59, - 0xd20ec952fa53c3ff, 0x8c166b34cdc88af4, 0x5d902b7ab99a290c, 0xa91c0764701f4c56, - 0x8b361ce7fd57d5ed, 0x7661bf74866f8d88, 0x22880b057dd7dec4, 0x1b43fc401034fb4b, - 0x9aa6428cfa5a23f1, 0x2e8f7788cba73be6, 0x81cfb0ed9bdcb302, 0xa234782fdaeefa8d, - 0xbc0c322cd5ebcf76, 0xf6fdd4721ad18a8a, 0xbe6d9de8f6d285cc, 0xb8dc477dbee766ce, - 0xcdfeed13b44572c0, 0x0bf68f12cd7f4a5d, 0xd6e9ab1721778ef8, 0x06dc2d509dd36dff, - 0x02941f233b4e7b9a, 0x8dbfcf7b1858ea8d, 0xc814ce259a5b35a5, 0xab57c04047896629, - 0x8e81cf678e536e8e, 0xee1de6c58e90beb8, 0x75bda7defb5dc6d1, 0x097e91d1c058102e, - 0xaecb4b42fe85c5d7, 0xb7d2e2b6dd609a48, 0xef980a63b516e352, 0x7d114af9bb7bdd4a, - 0x674def540b71e45c, 0xac127e4a16adad63, 0x712d888a7db94168, 0x749b34376039d6f7, - 0x792b06b8a6cf2cc2, 0x27d1233cbb14cbbc, 0x607f5f455faa3063, 0x3b5ec7083ca864f8, - 0x3d6b4f84259432d4, 0x3b92e96fa24d9217, 0x55fd7208dabd4dbe, 0x99971a09756698fd, - 0x555327eb5f412f11, 0x27e954832de83f2e, 0xce3335cdea22d03f, 0x154435f5d65c38e3, - 0x14754df4b4f7034a, 0xd8bbe59899ca03ba, 0x1582c6df61e13666, 0x5ca1d0aa2c836859, - 0x8c53a9ce187aefde, 0xc3c3fa830c5fb22c, 0x3c66ef8a503d892a, 0x2d59c893d6247c4d, - 0x22dbb46652471854, 0x9faddbb85e09a14d, 0xd8bccb45f6e6a0ac, 0x23689d009120c0e2, - 0xd510f9a6986bb124, 0xad515b2d6c8f189d, 0xca274cadd82c7283, 0x2e3bad4cc4820f7f, - 0xdda8c3f3d59b70bf, 0xabcb762114bf9b97, 0xb3561c7402d3833e, 0xc80472bcbdd26531, - 0x9c4312cd3c3433eb, 0x43725a824ebe2c6a, 0x3ae684f91eea02d0, 0xb48185acac74505c, - 0x7f4bc143bc8ab9e3, 0xa95e14512a58e75e, 0x6b9457b7ff451cf3, 0x38488d6e467f61bd, - 0x61cce64fc260a9d5, 0x0968b4ed68a0b428, 0x853587bfd42de1d6, 0x626d92e3210271ad, - 0x5b088aa52ffdfe7b, 0xaf75d430888c8b28, 0x05bc40c8f354c01e, 0x3caab0da3beb22cf, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9fef732379de414a, 0x6600847f1838c76c, 0xee8b61b78f6caae8, 0x8b16c28882692b66, - 0x637b94ce46e03218, 0x4a095a74d026b9f9, 0xff2aa6e6f8363863, 0xe5c07e64fbb17864, - 0x28c4a3811251a067, 0x149f38a7986c4eb4, 0xb7111c04a65a0452, 0x3bf79845768e1d1b, - 0x4799f4451a1fa720, 0xe75f63f3698c4796, 0x68bb690a5b08f3de, 0x6083834cd9c1ff0e, - 0x20c3c4fbaba251b0, 0x067c7bdedd6f013b, 0xa18f00c535c6ac31, 0xbc6d15626e922c67, - 0x4f387494da8b4f50, 0xce13e7c4d056a06d, 0x3a23034a052ee7b9, 0xb279297f1e3761cf, - 0xe4eae1446020c076, 0xf3909bd9aa99c3a0, 0xd5e4da44b451ff4f, 0x1dc7a7d49cb8caa4, - 0x821b5665df448835, 0xe9e5c11883be375e, 0x37a32eda61b52416, 0x87f455d74f901759, - 0xa380c9b9f04c76b7, 0xc804a25d87937457, 0x66052d593ebba3e1, 0x87c8ad560b378d64, - 0x9645adf825cb6cee, 0xdd166304d8bddf17, 0xc66e822210e2878d, 0xba763f799afeb5fb, - 0x1e61a9f939351e92, 0xa265dd673c9c9027, 0x7ef019850692931b, 0xb038738417404d2b, - 0x63f1bab3951b958b, 0xd6b5784555a01a45, 0xd4a68fc9939ab88e, 0xa50462b98d424b87, - 0xc0e1ea039102e7af, 0x4d42a6f03addbd51, 0xc6b279c218116a27, 0x20abda844cf047fa, - 0xa5c5dd82895e91c9, 0x468cb72b4f372775, 0xc21bf51cb360479b, 0x617ea7c3560ebd73, - 0x8f41aa242ec6dff0, 0xe4ed121e63656469, 0xa57aa39ef645203f, 0x04b26170917eccb0, - 0xf0ddde3aa4ca36cb, 0x19a649c2293520ab, 0x14c64623b2d91a4a, 0x5d6548975bb88205, - 0xb387c49044dbf484, 0x103137d68995d83e, 0x8537831885090fa4, 0x382a2d3290006bad, - 0x375badb305bcd280, 0x1945e93c37a2e319, 0x89ff8e93c9e75907, 0xa9b085de7fc9e82b, - 0xdb4248b4369d7bb1, 0xa65ecfb136717195, 0xcae3b3ed8d22b809, 0x36816a8a281157a5, - 0x45679767932db635, 0x4f05f81a14cdb212, 0xbd178feacd26a748, 0xd6187c5d1cbad41e, - 0xe8c5cefeb4b2807d, 0xa3e054a4fe1e8e4d, 0x8051e78704e653fb, 0x87318c49cf5eecaf, - 0xf637ff5b63c0138e, 0x3f92dbb745177ee6, 0x805d8fc906af7474, 0xe8ebf957046fffeb, - 0x35f1feab86dd0cce, 0x0af8eebd7ba15f8c, 0x72625d486068c0a2, 0xba1afac055f02dc0, - 0x4d77f1f1930c8b8c, 0x51ea759228c4cc29, 0xa5b44a23a7a2c3c2, 0x31d5b7e72dae6e63, - 0x65f98ccfac0652e9, 0x025c9247f0abe613, 0x1f554763b44be034, 0xe863b9e157bc0478, - 0xfc53353ec40a78d3, 0x38c009b415a4ae63, 0x99a1d14d66b2353d, 0x62bea4be5732cc26, - 0x390bdef4e21d187c, 0xaea8c74a6ef1952e, 0xada2df6888e4b970, 0xd51b11ee6bbb7745, - 0x7cf8b86552e328af, 0x856e262179bf5fc1, 0x12d81ab85c52b97d, 0xced645b221f8f392, - 0xefad7be0febce73d, 0xb7f23c23b144fcda, 0x27abc90aa91c7da2, 0x2ee1908c8f732e6c, - 0x6b084bb56d262647, 0x92b2a663d18b15b5, 0xdbcf358c4d621581, 0x170ddcaf1ca26b25, - 0x12fee11c1f95d502, 0x51cce12a68207a11, 0xbfae824478469bdc, 0x08f49b31507f9715, - 0x0a2144aee2da7430, 0x2f6dbb99029b5aa1, 0xf0ab1af5b1a12fb7, 0x82ef423890fefcb6, - 0x77ea9a61206d517d, 0x5b3e059a2139b395, 0xbc0555a41dbb5257, 0x3cdee8baa209a8e0, - 0xe9bb25c771c04a36, 0x2e8eabdf95627370, 0x394c710c0a848ce0, 0xfe3eaa4ce4bcca14, - 0x4d6b57998af7ca07, 0xbd4fb245a243b83d, 0xb2821b1345a68e2a, 0x4261db3eb338b23e, - 0xbe7ef41e0ee3c8aa, 0x55881f02fbeaeada, 0xc995e317b1b8d0e1, 0x89d907f31f10c727, - 0x9298fed159820ecf, 0x0db6043292cd0014, 0xcdcdfb13b4bf4eef, 0x234df24df94c2038, - 0x353ad8163dc5d2e3, 0x5ecaceff19104fff, 0x32d845e845893b0a, 0x3ca59ff3d8db88b8, - 0xfb8732d99f2408c8, 0x2edb89f356b28e3a, 0xd30b99df878a2434, 0x589bd1d260518126, - 0x056f98d8d5510e63, 0x3559c06c261d8b51, 0x5d4adf00eab0c6cd, 0x776acd6b44560fe9, - 0xf96b5d83df7657ce, 0xa02986f821c647c7, 0x4c55a8151111a945, 0xc1fd6ec0f4d3766b, - 0xab67b3273092125a, 0xce0ef31c5d49c9a0, 0x66e5be8a15c874f8, 0x8d24f695c86ec2ae, - 0x0768681473f84673, 0x6137b98ac1c15b74, 0x2f7f65a88fa5e0a5, 0x338de1bb86ad7496, - 0x1762bd2196284bf1, 0xec2df2926248c5e0, 0xe7dc6193f294be40, 0x412732835815041d, - 0x677dee7699794e49, 0x715fc3dad7ebb2a0, 0x28dd0957510626ec, 0x8de471f9cc7c0251, - 0xeaae6a2fba802b87, 0xaf706c138dd01281, 0xa1b6f02e8c33d126, 0xdcf432b4080a77f4, - 0x38fc91397b30344e, 0xc445c91bf87def1d, 0x8044bdf85aba75af, 0x9ceae19ca32df6fa, - 0x3958ff56cbd28ebf, 0x7905a6e806c57710, 0x25df8f1e88684e99, 0x0d7fb46b0e828bb6, - 0x57f0ce6fa4b42dc9, 0x7bef3041de8a50e3, 0x3841389551395a28, 0x2784340eaaf46fb5, - 0x4d9aa489fba1b683, 0xfea062eddbf326fc, 0x201f08472e12fe57, 0x5b086acdeace4ec2, - 0x4babfc8b18fd88e9, 0xba6934df667af1d2, 0xbde5416689bf13d3, 0xd5b4c826a8e01480, - 0xa46d3b6d93a34a86, 0xf8025464fc9f238e, 0x98b5910a31d9a64e, 0xa9315e9ae8a51a3f, - 0x60aa1d4dbc048af2, 0x9bcc498caa0745a1, 0xf2fc02f1680875ea, 0x843475753f50b39b, - 0x0c792e8d2e6b4a50, 0xd4518002dddf5b25, 0x09b8f6d48f71b80a, 0xb895ffab9abf6316, - 0x1d08095230c14d4c, 0x02165f50740b5a2f, 0xdc30e36684a561aa, 0x6faa8e4cd9b6fb48, - 0x04eba5a776bfea1f, 0xd8a610f555fd3519, 0xe4f6dc46f111f5be, 0xda2ff9ed07f902ef, - 0x12727c15eaa9962e, 0x007c81673a67a3ea, 0x7b7881a0b1a7c603, 0x9157e60be6b57442, - 0x721860d7faefcb29, 0xe885557cd6865d82, 0xab3b5ccd90a1991f, 0x441afc49dbebabb6, - 0xa56f6fd8a5730a5b, 0x866bbe4cc949a1ed, 0x0b9f1b4007fb3ae7, 0xf98cfaaeef289f43, - 0x2b3804d48207607e, 0x36c961d88e728352, 0x58a7a6cc2939ed22, 0x32dd96008fd3b075, - 0xa57d12b422680de0, 0xbed448bb5e9a5703, 0x5f2ac9f14ec70edd, 0xb50a0e15cb7c2beb, - 0x0b8dd681659f68b8, 0x22d46e5b9750d588, 0x078f6f1b85b62eb2, 0x0bdd64205f01ac5f, - 0x57738cd009c1285d, 0xeb9fa56be4c3ac2b, 0x013e2c9cd51e3665, 0xf571d379b67eb64f, - 0xd1e80e990d5df459, 0x968bdb1ebdb0305a, 0x516a8a4123bf0218, 0x67a93227def04119, - 0x130500cc18bbd046, 0x733f717a02901813, 0xc304dd0db3a1f63d, 0x42710dc2da3bb773, - 0x2f68c784ac3b9732, 0x18320ca75d16ca7d, 0xbe69220002af767a, 0x97928eed82d544da, - 0xdf12cde1cda9061c, 0x393b6e43600213e6, 0xfde30bb301aa97ef, 0x7b77d38cb4087918, - 0x39f2620b8c06afd7, 0x09fbf3c5cf1d35e9, 0x81b61a34d6705003, 0xb8607c633c06441f, - 0xd1234a856a4786c3, 0x222b13f3355887ee, 0x4d5887b649a1fb32, 0x13b7d7c810d8acd9, - 0xac43cbf90a62064f, 0x09b6aa8267fdaf23, 0xfc0fdd6fd2a3278f, 0xc1ff55690e60c64a, - 0xd1a1711a1076f592, 0xd09de59a73634829, 0x78ae59d7874d64ca, 0x26f3c52520636097, - 0xab6aed8d2ad5f95f, 0x34f737d2595c421b, 0x848a7e7b251d0671, 0x8fc4be01e1dac188, - 0xc0a1af6e89161355, 0xc3434fe504bd4dba, 0x0127e1834adf3695, 0x4903cee0f08db083, - 0x74f0a13c5948c467, 0x744559a3462c73ca, 0x7278299d145e088c, 0xa57a1ba386a12b8c, - 0x193774a92d4c83a6, 0x2343f12b79368435, 0x300907b9ce23febf, 0xbd76a0ae43d37255, - 0x1569223b374f6747, 0x663732ecae39e5a2, 0xabcaa71d59a37df8, 0x5347f0e321548a82, - 0x07cac5853713c9df, 0xc3a9dc2dd39c069c, 0xad7c5ec78481d09e, 0x9749b01d4f3f4ab2, - 0x3f874504f8903d77, 0x8d2ad854f9ef25a7, 0x3dc7f44653241f47, 0xffca3d3b9574bc4c, - 0xb495e678e28129e4, 0xa4f99e15aea8eee6, 0xba9434e5e8ec5dff, 0x58b8d5f136a63e41, - 0xbb62fc5b75249de2, 0x95252c0c4163e5c4, 0x63b3d6ac6068a87d, 0xbf610fe2a230c694, - 0xfb371e30b6b0800f, 0xbf0c60b8cf46fcd2, 0x5f482ee6fd636349, 0x483cf249ee9c58b6, - 0xabfd792b92c04f04, 0x37ac505a24cd34d8, 0x15b0e1a4bc419a8e, 0xc054d2736de68382, - 0x9664c3eb268e841f, 0x3703a233267fb857, 0xa80503a59dda91ed, 0x9495540e2eb9e856, - 0xa4d325ea30b536a2, 0x5de810e06d6f4314, 0x0eebcea9bf7d972f, 0xa897bd70cf24ef58, - 0x4aa4beccd2e751bb, 0x3dc7bc22436e0c5e, 0x1083c660b40b0701, 0xa429ba1a71506636, - 0xaa9b3d701ba42ca4, 0xd4410dd39174bb6e, 0x4460f2403d1f8189, 0xb74a5b410a66340f, - 0x788fea5eb4b7f315, 0xf5743373cfc43733, 0xfb008910f1f9f045, 0xd95d456bbb7376f1, - 0x98b9e354032613ce, 0x4f5704777390709e, 0x03d18e03e98200b7, 0x07df6cacfc619693, - 0x21a929ae6acdd30a, 0x2bcfecccc49f9dbd, 0x82424cf9b4ae86c3, 0xcd0504d1b894c709, - 0x02937d1875a5f666, 0x8ee35731639ca2cc, 0xb2eb2d22a7c64c59, 0x24635467e1124084, - 0x6781f74adff370f7, 0xc44fdf1749a8037a, 0xf8fa0ede7126334d, 0xd87799babce18a5b, - 0x200d558d4ef6a41b, 0xf3cf45360980d9e2, 0x31c349ea0125bc33, 0x023a3fd256338941, - 0xfad19742c8d871c6, 0x65402f1f04faec05, 0x5811c8ba870e8934, 0xb784c5b428db629d, - 0x9c04c7f3c631c548, 0xe403757813cf60bb, 0x2094d21b9628d5f2, 0x2e3415aa021355b8, - 0x62f586805a6705e8, 0x82fc5c6dcafc6ced, 0x808ed4b119ccd701, 0xedd86f83cc44ccd6, - 0xb598993fe5f73f0d, 0xa25dd22507c4cf32, 0xe7a3dc66eeaece14, 0x60ceafe8f0ed4176, - 0x02387790a955f77b, 0x554b0dee54688c0e, 0x01528024d2330b1b, 0xa065b5c010aaae6c, - 0x65630a0881075bfd, 0xf4d6c3ee80081efe, 0x7ef971d59ce0f60f, 0xffcc24f646da88c1, - 0x5c395a8c7e9417e2, 0x8337287cdf917a4d, 0x24681d122b7e69a9, 0x63fa3be7eec4f8b6, - 0x25389cba26c1a553, 0xb69f6af4aad42b25, 0x6ebf3273b487a241, 0xb63e4a5cda389502, - 0x618c9331736fff99, 0x44458a0d1173de69, 0x633f400283708505, 0x6882cd6263286c0e, - 0xc4b48787b55143d6, 0x80061840d5fdc22b, 0xd6521ab280e4720b, 0xda861a17df5597ca, - 0xb7fe3d834cf3331b, 0x5ad83df37196fb74, 0x67e567e62dc572c3, 0xfea1e44a98cd99cb, - 0x3e894855f605cbf0, 0x9a631232edfa8118, 0x319b51dcf7ac5d94, 0x2b1002ead4c95228, - 0xd5e10fd34c18e746, 0xf7b0a334bd75137d, 0x63659771d0778e8e, 0x87f39eabed1fe825, - 0x46a59f0935abb259, 0x7c6356aa18bbaf11, 0xabcb9bb888fd21ac, 0xcf572c57f4107f01, - 0x4048956956608287, 0x65760b28323a4572, 0x6725e4b3b9faf533, 0xf533038a54477253, - 0x193551bde25ccfd3, 0xb3d16ab8a60ccd81, 0x94a5f0200296a623, 0x04fc5920be1002a9, - 0x30e76362e3e33f4c, 0x1bad458ce0c5572e, 0x1cbb78605a577a28, 0x0ca92a38d5e34620, - 0x7304dd90b6098044, 0x6b2d93f43d2d3564, 0xe0c2cc48d054f173, 0x8d805bc4d62edcf5, - 0xc6d5f0469e3fd469, 0xee3d579f412ffc8d, 0xc3e9b7c1e4b02da7, 0x034e4d3fcdc4b7b4, - 0x7c80f2421549d7b8, 0x000d0947771fb77f, 0x1a230ef572a69f20, 0x1e418e45c28757dd, - 0xc29815a1470ae0b9, 0xb6a726dd9054f3f1, 0xf5e885744cababdb, 0x184d4ba6a8bebad8, - 0x1d0ea6f7fbe7558c, 0xf8cec75e3085ad45, 0x5d0189f64fbc6764, 0xf8061799c9483656, - 0xa2eaa4394c120490, 0x20578b973a93cab7, 0x8444239149b3e985, 0xbe8af5e989f42d60, - 0x32dd624a683b5f86, 0xd4736d9405e9e0a0, 0x870a296be412c177, 0xa4f10b38a633271f, - 0xb5e6bb9a18b6733f, 0x2035d41bc89fa7ca, 0x2e321b63db70fc42, 0xfa4c1192ffdd5b89, - 0x9622f17b3bf0c523, 0x4406bb9af76a8ce8, 0x01f57def82922e6b, 0xe8b38737442c24e1, - 0x3d56ffa85bbb4f59, 0xe30f7432b88014a1, 0xfd2b2922d0551f1d, 0x6d3d0e004158073e, - 0x289fad3dade64100, 0x202ed3ea20f66e13, 0xe9ebb92297569b7d, 0xb3444f3e28413484, - 0xbc6f7c76a5cc73f9, 0x7e1399196e285a02, 0xc9d582ee454b1045, 0xeea3a632383a2dbb, - 0x324611b8ebb85812, 0x20572d18476b668e, 0xa8adb97d4a033b43, 0xe2ae22892ca6d106, - 0x6837e25c5fe3b662, 0xd185d9f4bffac720, 0x4aa3294e68bee123, 0x76dde412cb891e78, - 0xef4c6c79ddfa62cc, 0xb2b478a018a1ff23, 0x3535f790527a5fec, 0xfff10602c871e4c2, - 0x85103d3bccdafc62, 0xba00493118a1ac0e, 0xa76a68f9092ef0ca, 0x8bbed22f125e32e7, - 0xe0e4609827741c24, 0xcd1967cbffd237cf, 0x70e56635c15b6ffc, 0x96efb4f6ecbf4f55, - 0x74262b8f0adaa54e, 0x02a5db2615066c86, 0x8c5a46022c34d6cc, 0xc4bee67758f08a7a, - 0x687e060cd7e3d736, 0xbee0754c13bc47ff, 0x38d9223712dc8571, 0x7f0ddbdc66805a23, - 0x757076d9189fb0c2, 0x38461cca224be48b, 0x97ae19ef42ad2d18, 0x87d63875c9692319, - 0xf9fd145a67c1f1be, 0x29933932e0973fb9, 0x63630891073366a4, 0x28844cfcff118733, - 0xa30bab193b46edef, 0x3591d083f0e740af, 0xbbff7420e363b268, 0x91c6ec36e5f3b88e, - 0xbd5e36423ba6c22e, 0x42240e405fb7cf73, 0xe95d3b3c08b3abcd, 0xf8cefa6e85916c12, - 0xa69b90e9b8ed4140, 0x2637c7c866297b8a, 0x4a5afd1b9106bd36, 0xa93d831890ec4e3f, - 0x15ba9d21108ae928, 0x845a954364e3b3aa, 0xa9398021df698ee3, 0x0c22d039ba9db7a7, - 0x1dc7095ec4dc73f9, 0x6f761db47b77ea9a, 0x838a12a2e1fdfe17, 0xff2c3d3f3ca4590d, - 0xc947213c19ff84ba, 0x8619d4cdd28792f1, 0x77c0f37cff97f9ac, 0x2dca69532efd892f, - 0xba3071f99e822992, 0x9f451fd94b56bb9e, 0x2115dbc4ba1209f3, 0xdbb7681b74f6e69e, - 0x6415ec4b4259e747, 0x593568909be18c54, 0x4ad2a315495334b9, 0xa7fb31b9ad769985, - 0x241f14f42817156a, 0x5cfebc33e3eafcc8, 0x9b434c91cd18a4a3, 0x2a15993048869096, - 0x20350483999fe559, 0x1f3ae7a4d3bee96e, 0x1964f5984e1a9b9e, 0xfb6454a7f0502298, - 0x9795610c1dbe3362, 0x97c02d233b987cd6, 0x50c4a34e92bc39af, 0xfdbf3abeb1e772ad, - 0x916c1f1312876cde, 0x56840db45cd79429, 0x119c29ef39f30269, 0x2944fc2346b645b3, - 0x4a5f59c984210031, 0xa7d86086d179853f, 0x5d49ffd13420d3da, 0x973add78ecf4bde3, - 0x0907646c857dbcc3, 0x8c263361a267986a, 0x7e2949a0fc71620e, 0x141387dee9918325, - 0x2413a74199be80f8, 0x31dfd46bd2155c08, 0xe23bc194d1c65db7, 0xec07b2b8fa95ca86, - 0x6ec72b2195b5d8cb, 0x75410972335ad4e0, 0x4c5b9ba8852ec9eb, 0x983914f7e0d79c86, - 0xdc6b422e6f3f1d80, 0xa3ee8b3305342c49, 0x3299dd69c2b019c0, 0x8639bc747234aed1, - 0x8f95de0765871f47, 0x0ad12b76345f18dc, 0x9bc2680edddcb52d, 0xd423cab8f3fd857e, - 0x7bcd2fbfde6762a1, 0x678229bb6b6c073e, 0x3e2ed9f641a01517, 0x24b895e733fcfe18, - 0x889996d74498c983, 0x11e461ea2cc00fed, 0x706ba390740f0889, 0xf11ca030074160b6, - 0x01e5164bf75a2347, 0x04bd42da11ac8021, 0x661442c8f604087c, 0x5f6077a3c81a62ab, - 0x4ae046b263108978, 0x8929d2ec6d437efb, 0x6f73d5622e467666, 0x465e3a0ba39ff38e, - 0xa1bd0f234881e84f, 0x621afe362ab06727, 0x1b28e2f0a0fe7a4c, 0xe1cb10833ddf3d25, - 0x9b336cf72c35ce09, 0x476d1bad6f14e174, 0xd154f3b38e6e478f, 0x99a33ed98d8e9e47, - 0xa4126b02514cf989, 0xd7adb055f9346b24, 0x3e97fffe373d022b, 0xc29489c58343152c, - 0x50bf4dfe64a906e3, 0x614514e12f1e85c1, 0x3e13113f6b39d15e, 0xfaf36b91e4c230f2, - 0xb629de52c1e2ed13, 0x667da598b9a3411f, 0xd2ab8a244e4db847, 0x866acaccbaee9f2f, - 0x02230d28d934f8e2, 0x2b5301658374ea7e, 0x0fab71421f81b0ac, 0x577242b013f2fa0e, - 0xde7eab69260d9382, 0x1c7a132da2a310c4, 0x0bdd1252e68e07ed, 0xcddcd4e9818a608d, - 0xd74a85b2e44bacbe, 0xbb05cafbb9407260, 0x7ff6eb74a15fad81, 0x43c5be114b667abd, - 0x39e28cb30560d7be, 0x20a09c0e1da7804b, 0x4372ac27af480058, 0xcbf9ae705f471562, - 0xbf8b5a07e5c2f8aa, 0x8e6351d8d5e2ed4d, 0xb3791878fa2bcba2, 0x1e1438e79e8d53ad, - 0xef89e9e1f930c13f, 0x25f035bb1504c5e7, 0x13a1061a3d3b20d3, 0xe04d5d02c47dcb1c, - 0xa895b1994711366d, 0x7c3e2517b54006dc, 0xbd279e703dc6340a, 0xfb8c2421e45648fa, - 0x69f21b883146c396, 0x30761ade5edc805a, 0x7fae01b63c5e9e75, 0xa49bc45a3a12d828, - 0x58761ac1554623e3, 0xd60863c3d017f31b, 0x388cbfca16604022, 0x3684aef4174c3474, - 0x1c41c0e898fd9004, 0x841f102deea2b3b4, 0x5f7c8d1b0ffef836, 0x6e147119ff0fc8c6, - 0x61b4452453c9ff09, 0xb838a7be01b415e2, 0xb72fc03946f8339e, 0x3d3444ef353cab92, - 0x5bbba2374ecab2a2, 0x4d68cd2e21d59bc0, 0xfd6aca5841378cfd, 0x58b0425bb787dfec, - 0x20171a7f29927b5f, 0x10356998303e34b6, 0x83ad7440d1e90ecb, 0x1308b3b699d40af1, - 0xc6bcc1358103a4bc, 0xfbc3f0a9ec6f04e6, 0x077721dc618fbee6, 0xa516d96196bd0ae0, - 0x187308ad161fb1bd, 0xe4cd724e202238e1, 0x1fa733452bf5b3d5, 0x4d3a28c287ff47e6, - 0x2e724d97402803ae, 0x6705fc9ac7210c05, 0x28ff71c27553d7da, 0x801cea05bbd29bf8, - 0xee8d9c565cee5cf0, 0x7da63625d5544aaa, 0xb0b9898b51c28fee, 0x5614515c96500a87, - 0x3e7ec3fcd3c7b8de, 0xb5b6304df70d3bb9, 0x4e51967a36b75a88, 0xed544c3202c19ef8, - 0x80a3a47300250716, 0xe9e0cfa10d17867c, 0x2ad7465bace57593, 0x80f025a95dd7c7f2, - 0xd2ef04d30eca591d, 0xa063dc397ccfb811, 0x9ddcbb74bebbcd8b, 0x88a94374f1c67bfa, - 0x950fff67f2823fcc, 0xbcb1b200cb44304f, 0x0fa29b343dab9b04, 0x6238724563f84e5a, - 0x3f7ec27e515cd3d2, 0xe4d6cbd29fd50ad0, 0x635bc6f0946ed6e4, 0xeceeac0b89981f8f, - 0xb68727738843f0e2, 0x70977a16eb0d5086, 0xf675ff429128e6ac, 0xb2063d29d122d0c8, - 0x78367280611e49a0, 0xfbb53afec9cc5286, 0xed5d984284e865ac, 0xc426e160baa632d5, - 0xda0c7f26b03d30de, 0x0acb933da2e3fc1b, 0x2dcee7ac6260b8ac, 0x12d171567d706e9b, - 0xad58d4d84f10be49, 0x6a03cde7d03cbd85, 0xc0d05aaea248f9ba, 0x05ab9bb6d94beaa6, - 0xf910b641c41567b5, 0x72f4c29142685c56, 0xf16d64e35e0a12d2, 0x479fd9f777a51fe5, - 0x8fee9b0cd5589da0, 0xdc97f62c5f5ddf91, 0x6ac41a9ec61670e2, 0x1be76352443729fa, - 0x8eeb2c2f582b0267, 0xe8b2170cbad11487, 0xac92cf3efb9e8c1c, 0x2f70a7cc43f12256, - 0x87ab2d4206bc3bed, 0xc9b1bd19ca9599d0, 0x6172d7d9fb1716a7, 0x40294a18c6fddb78, - 0xa3c138da881b1932, 0x69c024bc174c8959, 0xf2992df6a1800f1b, 0x6350b03c68bf78ac, - 0x62ab3213f0813641, 0xf998330530ee8e6d, 0xac840221f6421369, 0x7c9e0277daf9efc0, - 0xab7365fe1a2af31b, 0xaff7f31df55030f0, 0xb91c1f713fe013d9, 0xaab51b9cc2377bf4, - 0x6236a954dbe14a3e, 0x2dcc3ef4e86ed5c5, 0x9a83bb951ba13113, 0xcc5f5dfcc8033ca4, - 0x2888fdd747223e81, 0xd89d216bc633c4c4, 0xffc8d99fd45ac8d9, 0xbbe7fedd2ac092bc, - 0x2fc9090510ee3e21, 0xcef1b0191d043894, 0xeab83af140bdedf1, 0x60c74049bf583907, - 0xb1384528425478ab, 0xc760141c34196630, 0xb38ff9a06aa4def1, 0xf56313994f1997e5, - 0x2578331137c0379c, 0xfe7e726097da9ed0, 0x5218a443a9a14871, 0x14e89462ff0e898c, - 0x3a9e469f0513485c, 0x60efc20a6fe192ab, 0xdb26072bbfff240d, 0xbb6c4c4509f1e679, - 0xbd02a35c1bc284c3, 0xe36992d0159c8608, 0x25405aed9d2735fc, 0x6ca396aff2acc338, - 0x03523baee65a6f0f, 0x7a9b225177ff8f19, 0x0ab9eff4f2edf9c0, 0xa99146175cc68007, - 0x8cf2038cde83f0f3, 0x9aaccc2a4ec3f906, 0xd087d8971b893efd, 0x54e8be5a6e99c971, - 0x576e73372fa3050a, 0x5258648fe9b0e237, 0x84cf69a83c3681b5, 0xe97e16e65f202603, - 0x583cc7cd6146944d, 0xf2c1e887fa65f387, 0xb1b274a7554b9a60, 0x456c85111567972c, - 0x2179f035331d7d67, 0x1f727f75d0dfb362, 0x5cc7ca857d97d4ed, 0xa113a416609b636c, - 0x3b1fefd1129c7b2d, 0x02136ec852ce759d, 0xb3c2921541bd0f03, 0xb63e50958a333578, - 0x0455bff44eb99742, 0xed8a38dcb30132d3, 0x09dbc2265ab76f0f, 0x4439225d0138842e, - 0x708636fa22d6cc81, 0xe2b61ed7723f0d2c, 0x82f640fb7a9e0809, 0xef0bc0b0776ccc8f, - 0xd88072fbdcf66737, 0xa4f4a722a57a224c, 0x4b102d19806f7ad3, 0xc1e8f41144a07b50, - 0x48b5b6c09b73042d, 0xd7f9d7855149aa9c, 0xa8bfab5b8fe53870, 0xd40818334b097e43, - 0x5ec751f9277cb4fe, 0xe7321c84a3d63268, 0xa49ddb33e1e24897, 0x9edf0273dc5f5bdf, - 0x371cbed58277a969, 0x580526e03e6cb4be, 0x850e8343c7fa0894, 0x00dadde39b113b5a, - 0xf92c75842fad1fb6, 0xaa9420b9b39801f3, 0xf91d6afb9c5cfa7a, 0x2598726ab5c8b410, - 0xa6fe83f1769ee468, 0xf98d66461a2df6d2, 0x75f8f2befd3f618d, 0x6f3b43d59a63877b, - 0x7019815f3f7f091f, 0x00d5ea8b79035001, 0x550a39ffc686312a, 0x5e70b67b41d5925c, - 0x773d5fbe9a9b106c, 0x3907f1658336e22b, 0xb581ade72a1767e5, 0xb98987bb986200ee, - 0x0de527f1a412b1e1, 0x8301558aad946cf1, 0xbdbdc5d094c91238, 0xbc07fb3c21980226, - 0x60ccb10015a2b0fe, 0xda458504337ea8a2, 0xc2b18b650a26e706, 0xa444c73bd97e7abe, - 0x436cb31f2c2e7692, 0x7488b65d32babb6d, 0xaf0b435fa7777ea8, 0x334f7a71449246ed, - 0x7ff6588146258121, 0x6322cfb8cb44c624, 0x7c41fd7d30ec9c19, 0x0a1b7e851c6fdc1a, - 0xdbe44f5dd47d5aee, 0x7f6492db639a0023, 0xd33b75ee4c230ca7, 0x4302ec8d95af0800, - 0xc02999523cabd7d2, 0x6539b0b2c44e3572, 0xab7be2442cbff7a3, 0x5219602e71ed4467, - 0x63d4d866d6f51f1a, 0xef67e300c694d351, 0x5793494e86588b5c, 0x44a66da0326f144d, - 0x27a528cae343b10b, 0x055faea79a8451e2, 0x5928e4589b8f4308, 0x83e67cba9034953c, - 0x1c22e1a36837affc, 0x27393e3215ede47f, 0x11d5aef3a7030d7a, 0x4685001ab9db497c, - 0x09ab6ba404784ed3, 0xe0e81379427e0fac, 0xb1adbbdfcd29a6a2, 0x6b523991d5d3fd55, - 0x849f890e01046f8b, 0xb76b64b6e4aabd8f, 0x3fba0df733410726, 0x93b30912b88f2df7, - 0x133d2c9358ab0205, 0xe908000a463c886a, 0x1278a8955e86137d, 0x1d42dda914ceec7d, - 0x7e22fe89922a9655, 0x144c3345cdf4c9a7, 0x10f008929273c0c8, 0xd5105f0f71145df3, - 0x0eefda26b8146836, 0x8d5aaeb498b8514e, 0xaac27129ea5cdf7a, 0x3539b8abe5d0173b, - 0x24d2be6b99798d92, 0xbb00a94780acec6d, 0x0db9e5cb566de4fc, 0x3f96b59bb1f78265, - 0x7517d7bd0762d645, 0x9e2dec7ede0a4d4c, 0xa0710a03de8cc0f4, 0x34ec582c98d57d6c, - 0x6b4c1ef15a1902a1, 0x23d448533d568c2f, 0xadb51adec854ac7e, 0x5a49fad59014bb19, - 0x82490c2ca0c68b59, 0xf211e228b24df1bd, 0x8b4c64cd5c1149f2, 0x4fa566b280293697, - 0x2965421ac0c9175d, 0x63804e7fd15ee9c3, 0xa0df2ceaf71391eb, 0x99f318aafe0c0508, - 0x2bf942acb0444871, 0x65a5aedb33ef8f22, 0xa31a8120da0d0f12, 0x72021b1cbc331ee5, - 0x01751d041df1267f, 0x357282823abe1b75, 0xc7e779b369268b83, 0xe826eef79f35bdf3, - 0x53cf452cf37c5d6e, 0x6b67fbe70d8c8b98, 0xfc6668a22db81881, 0x5f10567d90b1a596, - 0xec3402f9bd829b7f, 0x3f46d8d987b34c61, 0xeaec6ddb3ae03509, 0x933a79e5f5a212cf, - 0x8b6ff33074e9e07a, 0xafbe69aca68f88ac, 0x6a98eedc802c3c45, 0x920d891e1279b10f, - 0x52c6b59b2ead195a, 0xcf1fcf310c0e34f6, 0x57edc267e94de68c, 0xbb8b23bfce5ec884, - 0x5bb4792e2b262ec7, 0xf80a0ffc881c66d1, 0x178420ad01233906, 0x73b30338ffe6e388, - 0x859db437bc9eb680, 0x856671d30ed8ac45, 0xa07b1e4e541e65c9, 0xffc26f287b1aa856, - 0x6c150cb36c2fefd6, 0x838ef06a97adf2c9, 0x4adc32828033f45f, 0x6506924144773163, - 0x242bf6b59af8a638, 0x69d69be2d3efb811, 0x063b92b4ab7f384e, 0xa9d7665e499282bc, - 0x0cafe2800a9419a5, 0x2dc6a989a4651d9a, 0x66d3e063a8024bc6, 0x2348c5ad26d87da1, - 0x46689de965067fcb, 0xd22fcc06425a37fe, 0x7d0707145270c92c, 0x70b5d135dc0a67a5, - 0x20bc8ffb9b6f49ad, 0x130e451a4ec90bd6, 0xac947430ee8635da, 0xfd089cc08031e9cf, - 0x25db8e6d71b0da03, 0x38bfeef0c5f0f8e8, 0x9cdb6d55ca5e3905, 0x78d2e38b50dd03ff, - 0xb8bf1b99441abb68, 0xca42ec87e7eae143, 0xd61c53e1abb43c02, 0xf388f7eb83e14b9a, - 0x6164b42fc31a84f1, 0x0126e6c400e35c83, 0xa187aff5248e0e6a, 0xdea8a1e935f117a1, - 0x002926d861971d2f, 0xb6fe415686b0a31d, 0x98c0d5755ebc8513, 0xf815c738ab21d47d, - 0x77219241d086ad7e, 0x5747d5b7373dc644, 0x40f40d1a3bf7a11e, 0x97abd81021596e64, - 0x763ae3cf980fc763, 0x538f09e601b0af96, 0x53d836fd7878bdbf, 0xfd071044951fd869, - 0xfb037fdaa7b409ac, 0x24dd9dd97459ec71, 0x33fdb95b1e4811b7, 0xf5faf3841e6a7ac4, - 0xe86d092b76edc4d7, 0x2854c623e3061c2a, 0x0fb599fd937ca40b, 0x4eef20d9343cd0f1, - 0x26c8b2e3cc2da7e2, 0xeb93781e8dbbbce4, 0xf58538b7173ae8e2, 0xbcc1732e68018add, - 0x8f068b9dcd87aeb9, 0xcb0c2ae33f541f81, 0xcdc817b2e132866e, 0x71efde46046829c1, - 0x9313703558af2c98, 0x019b40abf6ffb62a, 0x2e71f907c2612ab8, 0xd55916e43f0f8dad, - 0x3613a0cbd3247b66, 0xcbc21929b651c8e3, 0x3bf7b6d4328e304d, 0xccb92a98a3feac1f, - 0xac795d805ef52629, 0x3aea792dd673c970, 0xb7a0cbe1039f3ca8, 0xdbab52fd5c14d5a4, - 0x14c81bfba4109ad1, 0xb5874a356aed5f6b, 0x59db68f26dc7c176, 0x31c0d08cfa1b6488, - 0x4df2347994acabc7, 0xe170e07a8650b94c, 0x24a8465def275d33, 0x18a4f018ed4fa268, - 0x59b97ac4464d308e, 0x9e8283d5654fe141, 0x1557b8fedda84496, 0x2158cd9d36d9215c, - 0x9f156744c0acb685, 0xc5906b07b0695d76, 0xa5a98567443c8101, 0x5dea04627814a8a3, - 0xce9f949163b77f66, 0x08b8579b91ec3b7e, 0x3bf1b8c27c90b9c1, 0xb86ea4d1b2658221, - 0x42a8ec970d613a99, 0xd856c9d6a3cf44a2, 0x3541249c994b9814, 0x50c624a99794d479, - 0x51ac98038e4bb1a9, 0x24f4d35bae329b40, 0x8e18f79002951e01, 0x7c12f2b2d2e5d1c2, - 0xa56503d846168ea5, 0x1aa827943a256d45, 0xc8d8d3f44d5c8530, 0x0d9e3ef491208646, - 0x40aecea6994ebec3, 0x3e1f873884c696f0, 0x0a94030375fde069, 0xcdc7b936ed70d10d, - 0xf9e97fa4f8ff300f, 0xe87e7d3870ae6a6e, 0x6431b975d2ca0a82, 0xebe138519be675eb, - 0x08cffa8376b0a3f6, 0x342ffa303b3bd90c, 0x838fb6651ff19c9f, 0x1c625f80ae810167, - 0xaa72752ed0c9969e, 0x686209c30cde7420, 0xfdafb54703d02265, 0xe0c7fde129603896, - 0x87e197da439a2875, 0x3cd074a5ea8cada7, 0x43738d3936ff3acf, 0x607d15d25ea90e73, - 0x176ff04511c7188c, 0xf2a30e44c457a90f, 0x5bab5009f6ebdf0b, 0x471bba65a7548aa0, - 0xe0be5347b0b3c833, 0x82680bf0f4405552, 0x93cbaf577ce1c0ef, 0xb7b0cafdb9e4ec07, - 0x47672f4b601d2d96, 0xe6e00aba4873cddc, 0x9b566e7814f343f0, 0x5d0cfdf2e4f74609, - 0xec304579ed0ba984, 0xf4958117f2ffe6df, 0xe5c6430cc330c952, 0x89a2d1a7cb8b9021, - 0xa0df5cda7327c350, 0xce56fec1c9ed720a, 0x347048f13a7764ec, 0x49545a2e48d1cc13, - 0x6dcbc8b37b715470, 0x305d14c8aa534f13, 0xf28cd936027755f8, 0xd141258f2a3a7ef8, - 0x7eb05ea4fcf6270a, 0xef75f784a6c94950, 0xd8c635a443eef371, 0x83566ba5c4c535f7, - 0xff9a389bcb075a1d, 0x8a3a4d4aec645142, 0x5073ac4a6b78c110, 0xbc28252a5aaa129e, - 0x61357ed3f5632def, 0x3e7403074f500987, 0xd80387f2389e26f6, 0xfd8167081c95fbe0, - 0xa3a59e3d26be9880, 0xc9b80a6f56c5edec, 0x8bd3bf564af9584d, 0x60bcbd6aa1546e50, - 0x92abfaaf21cd0989, 0x8642760b00b74d20, 0x650588b5c5bb38cd, 0xe0bffa237bf11597, - 0xd2bbe6d1470fbfff, 0xb2e6283843a098b7, 0xd87ea5ecef8a3160, 0xa88fcfefcfc50a78, - 0x344ac5ded52bef7c, 0x0097f4908c79d7ce, 0xa354431d491e7e0e, 0x2a05c657362130fe, - 0x0e77259201495e10, 0x3ce5002f63285c91, 0xfac64675d3acc69e, 0xf514420860440002, - 0x1b39dcab8c29c149, 0x17988680f4fbbaf5, 0x6dbc864e5ccb1029, 0xacb819cd1bb19222, - 0xdbd13c2bfbbc98e5, 0x7121042165396941, 0xca010de781bca904, 0x8d1c40e9e07733be, - 0x3e474eaf98e189ee, 0xaf8aaa88bc62a718, 0x5deb4e24dbeb7c4e, 0xdfe4b13c3d7db078, - 0x3a30df45461c8379, 0x3e69ec661e5de183, 0x76ac4a1851c13db6, 0xf3cb0d4974c966fc, - 0xa6644a88d5bfba4c, 0xd840688f80e15aff, 0x0d7adc64d8b3ccc8, 0x7dc204824781df39, - 0xeebfcecf5e4ec802, 0x710d6867d2763202, 0xcf17cb58e9a748cd, 0x9f3de7775c80a6a5, - 0x5e38d52f7269bc30, 0x9a4cc197ff5b795b, 0x88ba66d8ae47dfbf, 0xe8c4ccaf3de36249, - 0x8024d3dcc40031e7, 0x1f44397a585fcb33, 0xf4185ab72a6556e3, 0xc08ffee9c939b9e5, - 0x40118c9efdc86a41, 0x659bae5db968cd1f, 0xe2f419c026037925, 0x206fa830ccb16889, - 0x803c9b8b0bde98d5, 0x10622375cba67f66, 0xd74d2a653001b954, 0xab1311fc41125709, - 0x25e0a7cf20477c7d, 0xa084ce509b78c057, 0xda960101c166ed30, 0x6e3e99632767f4f7, - 0x9b7f73492cd01ee8, 0xf9ddcbbaacacd201, 0x4e299d67b2f15c2b, 0xb6d0d2d06ff812a8, - 0x4050f769c25fed67, 0xd48ee3afcee45de3, 0xdbd3a38851c59076, 0xc339fcf63331cf72, - 0xea1a485912bbc1fc, 0x5f4588f41c35102b, 0x63c9ae9a4efb616f, 0xc617d1f845630275, - 0xd2ec1b2299b914bf, 0xddad170ef0b48ea9, 0xb58149d333e21e88, 0xee2739f4f25738e4, - 0xe475815a18b999f7, 0x229e7d6488411432, 0x442b7fef41a713c5, 0xe79ccbde9a3814c0, - 0xad89f885f9bbc83b, 0xf83ee7fc3a4312b6, 0x63c24d66406f7530, 0xed2d93ecdb9193ec, - 0x04c360f33bd81358, 0x55f12582eefd7cd5, 0x03665f4ba1aa4681, 0x9aac2afd3ae62979, - 0xede1db5a9a92078f, 0x7f649ce48e29ec83, 0x73d715bc0e4a7141, 0xf75f9600690c24a8, - 0x529597ab7ef015be, 0x22a7696c7c1e8439, 0xca56c2561f405e6f, 0x6cbd4a0c5706a954, - 0xf8f97b079dc698a1, 0xbd7ae808cf7a5bef, 0xbcf38705abcf9804, 0x837c0f8411eb3426, - 0x4c9f70764a9beef8, 0x2b8d1614ec49e8b4, 0x942a2484a75ae1a5, 0x1f3a96e04c52e10e, - 0x2a9e6cf2caba51e0, 0xd4f392488bb40f66, 0x3d049f81a14263f7, 0x1efb9779487a692d, - 0xe64bbc741fb90ee5, 0x750cc0a76f252081, 0xfc01af620bb64a6c, 0x03a8bc50607ab4c2, - 0x989669e019c573ad, 0xedc31de4fea1d8ac, 0x88c14a69a48331ac, 0xc4394ea7fd9d06b9, - 0xaa5c37f06d82c133, 0x71bb0ba43cf79e8d, 0xf33e40cc3b14a369, 0x477d5884ac2ec468, - 0x8f0c10cdd55bcfa1, 0x7652d47c7f4b6cc6, 0xa484eb4290e5fc58, 0x0cc42a5346593f10, - 0x05460e3442a86bd0, 0xb93e0ce4852b5cc5, 0x564b2c9ee56fdf4c, 0xc20d817091a1f201, - 0xd0d7526ec2626916, 0xa1c2b8f61a63e38f, 0x0edbb00df5e7ffbd, 0x3dceb1df765be25b, - 0xc4a94f5d8b82ca01, 0xffd3449c9c12138b, 0x340b7d2e8ad00203, 0xef8e8226255e94bb, - 0xdbf7afb91294c5aa, 0xf10f4e2250bd922e, 0xeed3a7f344d7b8bc, 0x26bb405b427e22cd, - 0xdaab9fe34bcaee35, 0x9102466a85464ce9, 0xf07aa034065f910e, 0xd6a588529d7528ed, - 0x9ee581c2fe26be5e, 0x3bd5133eb8cd90d6, 0x3c7ef2a6f11f152a, 0xbb9d0575d9185c23, - 0xd44882bee2c908ce, 0x2b1133213f06e348, 0xbc7a8008e26335c4, 0xeede4a0a5a89b5e2, - 0xf1ff588abd41d9a8, 0xe91defdef7be7f37, 0x23ef7812e603e125, 0x50c488c831074e23, - 0x591f93ff0d0d8691, 0x549aa39567b1b148, 0xbe1912dfa454ab0e, 0xe10f49d0b36c5fb5, - 0x02bb041d4d45c368, 0xe35f0c192d23fa1a, 0x014a42692bc5932d, 0x669e6f2a824f10c9, - 0x87eaab66007bdb59, 0x6a83f986b33d1e00, 0xcad3574a8fa64a47, 0x2dce6e1c5858d170, - 0x02d5b0fee5508e7d, 0xe19356c00df8823a, 0x87d91298cea26403, 0x6ffafa09d4e9b151, - 0xa543dc8d266e52d0, 0x8370e169b7f5eaf1, 0xafad6efd0b675aaa, 0x3e6b92fe3f18c218, - 0x8f825787566de840, 0x0d54e00ba9b44365, 0xd9a6bca439b42e7f, 0xd1836265974fae60, - 0xbb0f622e2ca9794b, 0x441576cc2191ad04, 0xe8de465c5a28a796, 0xbdc5d9843cc5df80, - 0xe46749d009c743f3, 0x4832f2e97fba1bb8, 0x1ee8346d6b1c59af, 0x2fee7325575a744d, - 0xea6ea8bc14f832fa, 0x09921be95b586aa2, 0x9ca27a2e63275cc5, 0xbfadf8dc32b9220e, - 0x20b2e513671afe72, 0xa438f78a19f5720e, 0xb3773e1c0006aaf8, 0x7a3e60ba34b5c447, - 0xa7ed53afc01d385d, 0xbdb87cd72e4de8e7, 0xf37bae30a87c91e7, 0xfd5337e1bd545b4f, - 0xe99c1c8944899014, 0x6c618bf02a04eb9f, 0x9bc39c9a5ed2552b, 0x4f30d0b904156efb, - 0x1fc2f7e2ab3977af, 0x3eea1443727d4873, 0xa16b28e209f068d1, 0xd8ca7a65675a2a13, - 0x9008a855ae6b97a6, 0x4f71c7ceafdbcad6, 0x174bd07a5b10e3f4, 0xdf53300605cf6a46, - 0xd564bbb8ad4cba38, 0x1229a32cb8667476, 0x0aaf0cf013067db8, 0x8279b13f76460e33, - 0x6c9798e58ea1892e, 0x1015dec51527947d, 0x24626c805758e0da, 0xcca536d820d6157e, - 0x779323319b13e5a6, 0x3d34d5274d7ade5e, 0x2580b553862bb9b6, 0xdbbdd72614c162b5, - 0x2b2fe4950434d7f1, 0xc9f6a7b91d0ee020, 0x0ca907d9911e8482, 0x58df292b3b54e54d, - 0x67bf1fd23db1a2c9, 0x14cd54e7ac806031, 0x0880d81c49b33931, 0x62d81bd8523deb8d, - 0x8e0554ff47245685, 0xd1a9d1fc43c7d559, 0x2e484468db0656c3, 0x5a3225a3e3e2a77b, - 0xddb15b9a10a14d14, 0x1c8e87b4624ecbda, 0x1f0c02a737ded2f3, 0x6ef7658d997c8ca3, - 0x22a6fb03325b4a92, 0x99f7b7365f176ade, 0x24baeb560080a886, 0xfc7d201a285f6ab9, - 0x43c2caaaad37d462, 0xdae58c2b758b9534, 0x5a0b0b579cf4fac7, 0xe9f6c9acf902e7e0, - 0xcaba45fe60521b22, 0x1ad5aa47bb7d93c4, 0x9304d47316104f23, 0x4cb665db75d73e8a, - 0x510f27ab7f8e5bba, 0xa3647bff75b32fb3, 0x4b3c688175f26c3d, 0xe595fc88286d01fb, - 0xe20d2f64e2f94c84, 0x85c5faa1d13e8fdf, 0xaa77cfaa03e9b321, 0x94431c6f20289385, - 0x5541bade2ba2805f, 0xdb0c4711eba0f1ce, 0x01b24be6e4277182, 0xd6aa881e8e1ea548, - 0x11be17c668e086a4, 0x7a02fbd33c570880, 0xbd08d491ba1fba11, 0x6ffa057d6550ff7c, - 0xa40746e66af6cebd, 0x4526db2b09c011b5, 0xba50bda98cc246c0, 0x3b82d16101067c7b, - 0xd347ed714cfdf10a, 0xf768a9563f545755, 0x0a94f5e871bde0bf, 0x407b6cae1450d616, - 0x31d5158eb2d1e902, 0xd5b8aeafd4ac2335, 0x8836d90acd1fedd5, 0x771ff10de417a6cf, - 0x781b35bcdd77481c, 0x0e156f8b58d26e28, 0x2a73fbc632bd3cbf, 0x945562a4d6e61fb0, - 0x0c9b97937482504b, 0x8d00db9eca1b5bc5, 0x15cc3cf3a67d1168, 0xe958ad350d2397b0, - 0x5c1110c84a9bee4b, 0x6a53eaf70438ef22, 0x39b8dd62a1a0ae2d, 0xd35629f07e106b01, - 0x5d6e0f6101fbabc8, 0xf88b84ffc7b4b0de, 0x2e93e19f4defe581, 0x8faa6fc837d30f56, - 0x2afe53e336e4d051, 0xaafdbebff4dcabf6, 0x7ab440a7ec5fb90b, 0x6357983e799c2076, - 0x53499a8abeb757f2, 0x021ad2f4de1211ca, 0x73671f7d52c61a0a, 0x30f1c31cd356941c, - 0x6d79743867b0af5e, 0x93440e3a66cbc917, 0xbc4d4f11638949e4, 0xb378722bfd881fe8, - 0x63c24f70e5937e91, 0x9b04c536979896b0, 0x69a0cbdb7451d059, 0x9d10cff70f4d47b1, - 0x6423e9a605dc3b29, 0xee5ed6a1189dd6cd, 0x9ee76ad15d37d6d2, 0x825371b07cf0cba4, - 0xf343c33540dd2f40, 0x167ae2fa35f5c00c, 0xc4bcbc85beb9c9de, 0x176108f0d7da1b6d, - 0xc64b2c9e07a5b776, 0x4c884223a4280018, 0x91670a60f2ea2f9a, 0xac0b2757a357e0de, - 0x416377952afb6320, 0x12257cb160cc54f0, 0x4135242c487456cc, 0x6a5de8f4130a23df, - 0x47ec941026abe431, 0x15a8029cde62e1ac, 0xd68309ef57324141, 0x0031e9caa09accd3, - 0x8e3d59e527212e9f, 0xd158790ac03a3471, 0xb5f590e0a7692d4d, 0xffe1068be1c2c0fa, - 0x5074094a0753688a, 0x90449e3c9a1c45d2, 0xc113df6068ef0b45, 0x838514c108fb9ef8, - 0x2373d9a5116d319e, 0xcff1a7152400ab3f, 0x9df5debee6f93a75, 0x3876465df489bd27, - 0x87f5ff100fc0333b, 0x54c15aa299deacde, 0xe83f537c4cdb8c00, 0x39f8c35044cf4bb8, - 0x240528afcfcac9dc, 0x8d230fd2d5e8b3d2, 0x68491711a442fe56, 0x60914be85c94fbbe, - 0x944b30314eb9acc0, 0x532608b5f5436c42, 0xdcb22c193ede0a09, 0xd0edf5c7de9a1846, - 0x0f0a3efcfe2e4404, 0x791d4bc421a304cd, 0xc324f337e22c2981, 0x89984847a8ed8f52, - 0x5bef9298e67b3701, 0x8f8dbf9a1c108d8e, 0x5ea1867c2a92f67a, 0xe92e37c88957ea4f, - 0xb60b86b02b127eaa, 0x1f4285451b92f882, 0x54b58d6e58f541dd, 0xfa2d804cce156a96, - 0x905f9c4cfef8437e, 0x7a7139153018644b, 0x710fbbf104b1e397, 0x340b67047c7de8b0, - 0xf38fa7d13abe2143, 0x4374414f4ca6aa8e, 0xd280d83f25a7dc3f, 0x7d56917cf067f5e7, - 0x3b1da956532fc65f, 0xae04462c1ddbe350, 0xe852b43eb3a9265a, 0xb1712fb428215f00, - 0xc8f4fa7ec907f065, 0x46e99f27a7ca0dbb, 0x6d13f18fe4a1701b, 0xd78c2e5c11426696, - 0xba077b1a9b30c4a2, 0x1c6286e7d7071d8a, 0x818c17145b5338ed, 0x5ec769fdabfdb6a1, - 0x12405c2ea1a99be1, 0x838f8325f0a20ce8, 0x649e291e073ed8a0, 0x6d59225a4be12c43, - 0x39d58e1e19fffcf0, 0x5f954142442cd1c9, 0x04114814e3c8077e, 0x1e18919041e88178, - 0xe751c607992dfd69, 0x52883c269284ad0d, 0x8d8c1cd11c0bcbe4, 0xffdddf6d42887e0e, - 0xa61626039fbc11f0, 0xa358894101a43d25, 0x15a0e39786dadfea, 0xde52110bb266ffcd, - 0x789cf787c26962bb, 0xa8307193c746d77b, 0x2da298985e57c79b, 0xfb253abeb4af0d38, - 0xa7aa369bc28cb23b, 0xe619dddcb9f67599, 0xcdc779bdb6ea8903, 0x6f7e4a3c3e826c3a, - 0x10dbc2eb9f6577b2, 0x3d0844e07f4e3ebe, 0xacd58279214aab25, 0xcfb3e262c372883d, - 0xebe100b854c85b95, 0x09c1308e870b6293, 0x3ea7b0cd44708a04, 0x7a0019fa0cec4a17, - 0x79d3b45184f2a736, 0x5468159f5b45bad7, 0xb80a87e28892ca36, 0xc6efcaa562dc24c3, - 0x455b590791ad3ff5, 0xf2dc0db5dd191c1e, 0xa2f57ae6ce6c7a24, 0x4185a0add71eebee, - 0xbbb4bbbb764cd586, 0x9ee50324166cf005, 0xac9bc406eb6944f2, 0x47cb136ef0177b40, - 0x90eab1853f9d3817, 0x64159f8f8a7ba82f, 0x0b4f830186dc316b, 0x9f558a730108a8d4, - 0x313ee7d95633a7d6, 0x364418f65f236313, 0x193ba1b06a2aeff4, 0xdef0f7f8a1287c8e, - 0x6b3b0acc6e914d2b, 0x4a304aabe71841f4, 0xa814c767d926c645, 0xa63c960874b007a5, - 0x9fc9ac26c74fae82, 0xc8c539c0e8559b8e, 0xbd6d3449e7cf7b58, 0x9de03081bbe18e5a, - 0xdd3c4c26c7ee2550, 0x2b65d531629f3caf, 0xcc0e0275864d009c, 0x4d7121b1c229f15d, - 0xa14f4423ef690b3b, 0xdf61d843a8db75be, 0x567d5393e91de71a, 0x099ba1bdb7a389e6, - 0x3f88703baf7eff50, 0x1c14d5077fccdbf1, 0x3083ce559861e47f, 0xed4cc8aa155c2421, - 0x3f21ded2b6e86f3c, 0x5b76905eff8d5e20, 0xed7468bdbfbb0f53, 0x40bb7c90942f2f1b, - 0x53afe034040b27f3, 0x2ce0c2a1a793bada, 0x94c033ecbc6e1704, 0xb56d92eb6202e285, - 0xb768af8e184389cf, 0xa523162c0edb9217, 0xe93f9a1e666969dd, 0x1b34ae8a5757d80e, - 0x7ef66dfd633bf2da, 0x489e6399c33a1de2, 0xef247db583920e63, 0xbda35894ccb9ff09, - 0xeb2e51cabb0847d8, 0xaa37dee8bbf2993b, 0xc7afb6930f1ef16a, 0xdd372182c3a8b8f8, - 0x4ebfefe95df3a7b9, 0x089d062711edd073, 0x8e1f6c56434324da, 0xb7ee7e34b4ef3970, - 0x7e09a61052b66d18, 0xade447874d7bb760, 0x09416a6a2f4f3306, 0xa70c4c4824721dea, - 0x88060cb742bf2b70, 0xecae6713dda5240d, 0x176dc8cb5b627afd, 0x668252b6a0cb92ad, - 0xea8dac8df082df12, 0x4f7f3f5175df8fa2, 0x94cbbc3bfb4c23e6, 0x7038bd66dab2b185, - 0x3dc31fc42cbf3765, 0x44c55ec0146650c1, 0xae3b80e29657785e, 0x40735bb3c0e543ef, - 0xde526f1fcc0da510, 0xc1770e33fb14f048, 0xd2d004270fad147c, 0xa0824404e7fbf593, - 0x68808bea1adb89c0, 0x4930fc7f38a0b5e2, 0x717967fdb336cf34, 0xd33aba1357f47743, - 0xa9aeacfa1a8ecd52, 0xbbe36ed730876f8a, 0x606214d70de85541, 0x81b25ffb755622d1, - 0xb2e2782bddc6850c, 0x59943615e04c8024, 0xfdea0d06eb678e2c, 0x3b09d2656ea1417d, - 0x829fc569ff490bc6, 0x0759f0b840a7eea2, 0xd3a4eda95d6ee0e6, 0x674ae465c476ab80, - 0xe70232fc0514a30d, 0xc2d28ae692261222, 0xefda3965c14c1a13, 0xcdfcefe8cfab097a, - 0x5b7b59047b551dd8, 0x90d2de59c7b6af06, 0x915859e7d8e98167, 0x5459fb0dcc16192c, - 0x2d33c6037c93d5b3, 0x7afba33d7da8d473, 0x96eb623eee2792ae, 0xe36a5ff5c6c98259, - 0x2d45baa19f3d2bd7, 0x32825e286e55e33b, 0xe6c72dfbc44a9e39, 0xbd8e3d49f21884ea, - 0x673106cfcb28a738, 0xcdf54e3966b95922, 0x432d0a8cb7850483, 0x373bc56d47437503, - 0x79d997526f61c6c5, 0xfcaca5f0267ef33b, 0x68cf3a29ecc6bd55, 0x0c09f6844ea84d27, - 0x047a3a02c789423c, 0xebe08d874cfec9f8, 0x1893d663191a04dd, 0xb020e8155aa8b9e2, - 0x322de5857ea1f70e, 0xdf3b2f16e5402105, 0x6901782b2075c69a, 0xfb1a15e4788b9a05, - 0x2cd6b928ad50b701, 0xf66428d2342f8be6, 0xfe882ef978a35dee, 0x8bf01e43cef231a2, - 0x67b411e45c3a5fe3, 0x704db88984d316a5, 0x59d2f96fd1bc8f13, 0xa090e2c08753cb73, - 0x341bcce4b9616cd5, 0x2a91b52a6b3ccff5, 0x2882b7a521953d3b, 0x94ae6082a3243af3, - 0x506ae93cba20447c, 0x67756677f2050593, 0x8a6d378deeeaa264, 0x61f9f368706bd9cc, - 0x7ed01ad970ac077c, 0x2e413275c328f2ee, 0x5da8f546c26cfba6, 0x3430e059766beb14, - 0x1cdd081adfa81842, 0x7837570e38fd15a5, 0x2842e6d88ea9ec7b, 0x3103c1e67413964b, - 0x4538a885ce207cef, 0x5e57bd1ff4abb35d, 0xea442e0826c403e6, 0xa21ba01a429f705c, - 0x87038fbc98c6b9d8, 0x77ea999a7a5ad45a, 0x88db71fa5db9088a, 0x955e574d658d2fa2, - 0x70ef36e0a5bbd30b, 0xd63756d8cf58c477, 0xa34e96ae788d6417, 0x5034c33e0458da34, - 0xb03431cc0333ea36, 0x688b2733990a707e, 0x230a6e039ab05d1d, 0xa2590f7479c2c53b, - 0x6a5fb90268a63af7, 0x3f7e9fc6f767e604, 0x00244acd86a98409, 0x69fbf4f853d611b0, - 0x5dc9c1265a332fda, 0x48e06aea43952044, 0xe2533a19deb5ff30, 0x8edf7903d8d91f16, - 0xbf683ce4885c0f30, 0x24ab7739dbb4785b, 0xf84f0434844c3ae1, 0x7a57571faa009fe0, - 0xf1b764eb1f69f518, 0x344ad2ce552f1502, 0xb4bbfa5fba056afb, 0x9a38063cc9316ccc, - 0x257fb13d1ae0ea5c, 0xe051a8d9e9cbd21c, 0xae195969ad6a2d7d, 0xe9b2b3e8899dcf93, - 0xb4c77bcbf642c6ba, 0xdd7ea5a6f7cd4788, 0xef2331fdcfecf414, 0x31f0f3d371d587cc, - 0xbf08ed995a5f8ca7, 0x9e47d0e71e33f20c, 0x72045739700980fe, 0x797c804e72e9a9f1, - 0xe1498b85b9dcb8f0, 0x75fcc08867ef656b, 0x10d111c3dfc1cf06, 0xcce54f66c5ad47a0, - 0x91229d69d5f3dfa1, 0x6b6c3dd4dba8d247, 0x6beaaee41daa60be, 0x07c5a96947ea78a7, - 0x4a8d261a79f55a2a, 0x2eebd53c565300b4, 0x9cea8d0feb6a9da1, 0x9c8b3096abe51517, - 0x00beca9f7662f4d8, 0x8742016cae2f08df, 0xe088eb84b861b1a9, 0x945e04557e7ae8a3, - 0x8bf0ac86f7bd93b6, 0x9796f3c67e2ba1e9, 0x6db54c7d73f031e6, 0xd7685a39b3837a8e, - 0x7869d51b3fbe7b5b, 0x2136bb80b6f94adf, 0x77d593f27de53477, 0x02fc88982fa5fbc3, - 0xebff26da31766a41, 0x0f37d08f485f8e16, 0x00b52ee450384e3f, 0x5f4ba1e7ebab9276, - 0xb381d28a7ef50ce0, 0x56407e1607a2b23b, 0x8712564a0a43f46d, 0xee64d2c21b5be832, - 0xec2d00d71b31027d, 0x036539893a621356, 0xbde22828f3b8bc0c, 0xb5c36b4bae925b1d, - 0xeaefd1171dd1d878, 0x4ddac1434e620296, 0x51a519a7967e5382, 0xe19960915e8fefc4, - 0x2ff2461496f27990, 0x3ed5f03481dc77f2, 0x43c2cea4ae3608aa, 0x3d34386662511820, - 0xb87f3f44d34ab4c7, 0xd6701e8accaa4875, 0x320a0c464cea9710, 0x0f188865de9b089b, - 0x8783d67afe6e3ae5, 0x4e67b56f366ebe44, 0x559154040727a13b, 0xdc1a9424f74883ea, - 0x33625b50c5850bc1, 0xf9f8e555d70078bc, 0x81f377341948117e, 0x558cb4cd13e89012, - 0xa742c995b31b73aa, 0xffaea227143262d7, 0x39882796fcaca466, 0x124132e25d47a7e8, - 0x6d89a6dc37793cc8, 0xc03e275c9be81003, 0xd85c7fa33b00d3ec, 0xe0a18527be8e5ab6, - 0xf10a8199d6f8ff1a, 0x41d12fc74341f158, 0x0262ed3153f47144, 0x962eb7ecb83db07a, - 0x0c725c436d1e53e0, 0x86c4e6bd430d24ec, 0xc4bd7da5a7c1018d, 0xc014dcb56b8503ba, - 0x37181d6be5b32733, 0xa5ed25b2fcb0ce84, 0x4061bc18800ef1b9, 0xb87a58d1cca06867, - 0x5d3fc7848d5a1dca, 0x2f5f04f5d252d140, 0xad31417012bdb581, 0x62e6ef7d668c9986, - 0xc3f4f04e9903645a, 0x0ef7712e898b4497, 0x2249978f9b720dc4, 0x653e38463667a772, - 0x9d814ee832ca02b4, 0xe7ab7e3ae77e2790, 0x70e19f47e0742ba1, 0x7d9e48e011dc63eb, - 0xf20512d42949ad87, 0x8c528d18dde1b37b, 0xd028105e3365cd70, 0x6236dd69990f0d2d, - 0x5f5aecce0c5f86c7, 0x6325fefc48c07732, 0x636f53c055619a0b, 0x089598fe2f75c386, - 0xd5b948d2216f6ec6, 0xb3491c8da2b582fa, 0x32c03a00a29d2b50, 0xe11b5e12e0e48f5c, - 0x105ab17605f845d1, 0xd973949e20816450, 0x7600c92355891f7d, 0x1291d34b06af3081, - 0x5ceda2bad1428906, 0xf05c3bfc4586d2e2, 0xfbc9101200a59f9a, 0x03fafb9c803de789, - 0x315badf62532e34b, 0xa6eb81dc008c47ad, 0x471fab7e1135c1cf, 0x2f4ffa4fd9c8befb, - 0xf6cca2948bca8f87, 0x15362b0c9a434204, 0xd03d6e651246fd66, 0xafa4afc5137ab2dc, - 0x87f924c2fb6e89da, 0x6cdf25c197c99340, 0xa4c19ddc9b91cc52, 0x44a58fe87108b45c, - 0xd15187db2a76f4d8, 0x5d8ee8ef4a14db09, 0x553b5ee2a6a015c4, 0x1a7142bd3ab5b58a, - 0x121b1fee936eb7d0, 0x23cd938ce9dbae44, 0x0720d5e55e9b8ef1, 0x89b3846188c4cd7e, - 0x6e628bf557b837ca, 0x1e16deef024eb4c6, 0xca10bd44c248a316, 0x00b1f4d50753e6c3, - 0x0b2eb02441554571, 0x92149d9565729898, 0x14f7c7657d6b612d, 0x5190e77606e3de0a, - 0x197ca8228b4999f9, 0xb17dc804e60227dd, 0x9b7b8c9f60a64b36, 0x1cfb5c2c88bcc69d, - 0x8ea12eb23ac67f44, 0x08224df34542bd2b, 0x833cc9c84f1e9843, 0x3aad250d667b86b2, - 0xea5ff0333d4865fd, 0x6587f4859efa1f93, 0x19f17fbfb26eaef0, 0x4320ad958f1fabda, - 0x59fd62f570431db9, 0xde197ddc64817f9c, 0x468025723823b1ef, 0xcbdc892f1d0e9eb8, - 0x7eead70a4a27744a, 0xd73d379084d9fef9, 0x65b15882be4cc6d0, 0x16d13b0207aacc90, - 0x3dbfe14350a0231d, 0x5239beed4095a2e1, 0x049f0253c1bd79e3, 0x3a80bc3b75caa01a, - 0x178f3ff3a7dfca1c, 0xd3e3093f3bc654dd, 0x3100ad41ee6b0b0e, 0xce5d66a719fc3587, - 0x1efc393c5ae56b52, 0x67b1e82fa6b26ce2, 0xb1854b4bfe2d8324, 0xdd945864e23ddecb, - 0x68256a4c071eb8be, 0x617b29118cbe5b09, 0x8bb685a12bf7ef6c, 0x32760191db9560f5, - 0x42b8ee06813cc077, 0x1f75bf6a2a9c9ab9, 0xfb9d9261be593be7, 0x79e64fda6930c3c0, - 0x347cb08238ed6347, 0x2aa9e2d7875dbaff, 0x85a02a6558ae9157, 0x0d4536f0f1dbfb94, - 0x07d833605f1669fc, 0xba7af48877114366, 0xab8b9082959761e6, 0x062ede641a5d866a, - 0xe55d2f0f4948bb63, 0x0657ba4f2c0c0010, 0x316278da0f141211, 0x94cdd78c38250035, - 0xeb96eac52b51015b, 0xa22f073c91e0deb8, 0x36a6fb43726642de, 0x525f6de263c5c0c1, - 0xf49478dfb90dcf56, 0x0a77d38c69f40f1f, 0xe93c4bd558d5de0a, 0x383d99eab1ed4b0e, - 0xd3f9924fd86103b2, 0x7869482815b17630, 0xc6a42013a5cb1994, 0xb7c523ae630b6b65, - 0xe0fb8aac618d2945, 0x7fb162a97656262a, 0x8fa3e3f4c6c496a5, 0x22180d485b5888e0, - 0xbb22c7dc20126c19, 0xb4b73350b9b8e5cf, 0xd1c3d0961ac19619, 0x2bd353a2c361df04, - 0xc98d228c96567059, 0x9ac20ee6dd81d01d, 0x9966532dc141caf0, 0x82066f9f62a1d849, - 0xda7ed3a6b79b8f81, 0x6667e3ad7d1320dc, 0xbe19ecdac666ad42, 0x62ed84dc975e5c46, - 0x9be07a8ec67f4210, 0x3309312e68ff91c7, 0xa2ba8a5f6e2bbc12, 0xe0fc982e88137f1a, - 0x5fb4b7bc155c5c41, 0x7d6af5b796ac88e5, 0x0c679fa3ecd435f9, 0xc8e1b9619cafd98c, - 0x5837e249a3a24f1d, 0xd7175b90f68109d6, 0x509a9b2c3cade503, 0x4ee6df4820f7dcdb, - 0x2a08a22249048c76, 0x2e54bbfb2824768e, 0x3872cffea2650bdf, 0xe792a541005d3bd9, - 0x9a53f7cd921238f8, 0x7bcdf18c579755a5, 0x20fe89d41475e231, 0x9a191dd449da2d62, - 0xb1661f7def2cb41f, 0x8dbdfdfa7832c124, 0xc6405afe5c4dc377, 0x1e93fd0c7d481701, - 0xbc8cc115e184e621, 0x174e90e57c0a2e53, 0x9009752d3a9d04ed, 0x2d639f2023fb6042, - 0x139abf98f5b9e899, 0xe054eb324debbd84, 0x74c90f15215a9f01, 0x01f44504554fdf8c, - 0xaea5a3f1c40a6062, 0xc8083d34153976fa, 0xb8f0ccdf9608eb2f, 0x19962765714e7c87, - 0xa5ec37c77a6fead3, 0xbbee8540d09ff300, 0xe3847dfab632e5ae, 0x33d8e83cf33df74b, - 0x21ef53d49a273bf1, 0xb27ac146cfe0f0af, 0xf6a647fdac0c8241, 0xb9795729dea1b6b0, - 0xe23db47b329b6f77, 0xdda420793fe1217c, 0x17fc4f5e926b27c0, 0x749871a3b091d321, - 0x45fb11e75faf3924, 0x97e0962c459f0f7d, 0x1b9755d69aa4cb70, 0xff0f7b03d0508d70, - 0xe3e91dfe6fd7f5b2, 0x635fdf8342771c65, 0x89c9576ff3ceaac6, 0x6a51622275618a41, - 0x4b0cff084fa3175b, 0xe7d03518f230d90c, 0xbdb3cd15c14a5e8f, 0x67da9227ec25517a, - 0x838d4adbaf9f9c43, 0x610ccde46af86423, 0xadc6fd1176fbc113, 0xe6c918f668e88bd8, - 0x752765d5469c1186, 0x74bcd993c4eda4d0, 0x52701883c978a4e9, 0xde489d1e30e3147a, - 0x9a27eaed3b03bd88, 0x5f4613ebbe59b7ef, 0xafb7cfa82305e931, 0x9e540c9a5b93575f, - 0x298a5244af3df777, 0xc651d68ed8484302, 0xa9e04a6de113c901, 0x09b8cf9a5ebcbbf8, - 0x7b89a1fb0648569d, 0x53f34f416945651b, 0x49f1ffc8cf96ade5, 0xa13b5821b5f06601, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xe18bd546b5824517, 0x673891d791caa486, 0xba220b99df9f9a14, 0x95afbd1155c1da54, - 0x8e4450eb334acdcb, 0xc3c7d1898a53f20d, 0x2eee750f4053017c, 0xe8a6d82c517388c2, - 0x152879e935811666, 0xaecd900d995f5ac8, 0x55534f24546a77e4, 0x867897622c279791, - 0xbd0e28c622e2d858, 0x1fe1c1cab00e501d, 0x5ebd909551cd9476, 0x2cd775ebbc39a143, - 0x1cae9a5c417c6efd, 0xfb0594f9f58532a0, 0x5d00b08114c05c30, 0x7343068b3fef5c84, - 0x8c814560bfb86c41, 0xdfa792b712d80bc4, 0x09f638d876ba89b5, 0xb53aece24720d42f, - 0x3caf2a97832d7c67, 0x23187c43d8012d58, 0x96306de157a7b651, 0x057df205404fe736, - 0x40b2c634710fe5f7, 0x30eef6c16d867d51, 0xcedb88fd15e05c87, 0x02170597f1895e0b, - 0x2f345b4530cd3708, 0x579d25a75ba7ad1b, 0x626204e0c0db527b, 0xd5522231d41a4776, - 0xf658d663179e4199, 0x4c45e428ad95e489, 0x8518878c419fa87b, 0x2c95824c089acafa, - 0x15254c0a6c186b71, 0x472d1f63eb8c3a03, 0x56e1743be626c3e7, 0x2f8bc30cf926a3e4, - 0x4a18365a3000dc38, 0x8bcdc5cef13fe024, 0x4051913badc2c2cb, 0xd4f54c934af1047f, - 0xb9a980e28b399623, 0x62c9f9a8be129663, 0x59b9749c4d0b7d38, 0xac4ff401e8727c05, - 0x5672a959c3c6031b, 0x73dc92cd775998b1, 0xceb0eea243f69df0, 0x86e3e5abce990f72, - 0x9651896dfc4d7f55, 0x55ad01318c5c2f52, 0x2bf783f58710f8e9, 0xc1fa2978d883e72b, - 0xd8de09f6d169ee57, 0x6991121f0e335caf, 0x439c72af74d8e3d8, 0xe58d08ee72011d4a, - 0x780a09347573843f, 0x041cf5b6f136675f, 0xe9fc3b8af8b89645, 0xed676a9643c03293, - 0x7272ab22be50f4cf, 0x2b821fb6268a0970, 0xdef49fe985160b9f, 0x13ba3cfdfdec3097, - 0x2f6524b7edb163d1, 0x6098f7058032adf5, 0x359f4d13d9d45a65, 0x23c7e15aadabf5d0, - 0x50af4a9b249b4b4b, 0x61040ac6c40117d8, 0xa89490c4ead00279, 0xb0e4fb0c77688906, - 0x15e7312427fb1326, 0x5566b0626f566469, 0x77028b52f3ad7ef1, 0x747e8d2d4bfc05b5, - 0x7b4654097f7f9a1b, 0x49e4df0c78201451, 0xaea37782a8271ef0, 0x8434941ae272b963, - 0xdbde2e5613783827, 0xcb7fe2bbf9d283c0, 0x46f862e0316512cc, 0x3f79a1c5d342b99d, - 0xb77d718434d42e4a, 0x44c7061af94cdc60, 0x039ece5b115053dd, 0xb34f073238b1f73a, - 0x82f60e8c9fc79727, 0x506db4c02864f2dd, 0x646f9b4123c2171c, 0x03b438befe3d8b73, - 0xfa8a9285175cddce, 0xa75d4a13bbc752cd, 0x556a2f039972d1f2, 0x5066c9914887ce82, - 0xa02443c6f48eb833, 0x6733c4bdd1805a32, 0x4637b560fda5a526, 0xff900f95171abdd9, - 0xe3109d1082d34cde, 0xfaa3c78aaeb009fd, 0x92a88e94f87edd67, 0x963cdeb022d3a10c, - 0x12ca8329d6f3dd82, 0x5573e20305879c23, 0x12da11bc02bb47cc, 0xcc3a812307bb06e1, - 0x64c4bfb42d65d975, 0x0cfd9a9c31167bf2, 0x23f6c5cb65f0e106, 0xc90d6a73299ad66a, - 0xf9207cf22553610b, 0xbd67226bf8c6d865, 0x0780901fb50e9caf, 0xb9c274f85e0ee46a, - 0x3900f865cf894428, 0x088cf6170c0e2541, 0xcd7c2ecabd675de5, 0xd25de699c387e64d, - 0xe42f2edfa1976946, 0x86231740f6922a38, 0x3879338a2dfe6ee9, 0x693b8e9f726fbf54, - 0xc0652f06ab132eb9, 0x1f04836877608ae0, 0x79ef07e45327e776, 0x662856891a9c02b6, - 0xafe7f1917a2b349d, 0x3b47767a3a59cd0d, 0xce1c84d0d1dff396, 0x0b947fde7bbcf13b, - 0x6c9910e54d82bc4d, 0x6b03ceb94b14b6d9, 0x8b8057e881a0b050, 0x4c70742606a8b4bc, - 0xf753a4676eed5bba, 0xbbb21622ce427c01, 0x89bc173cf51175b0, 0xcfb0b8f75d6e9038, - 0x0ed88ac4e793eb6a, 0xf8e6f1d305a59386, 0x265ef750572c0eda, 0xb400c4fd06981276, - 0xd04c2825c6ebca00, 0x5a4e8bf8e4932beb, 0x37289429c7aa1c3e, 0x817f59c7a01c7d2f, - 0xb8224f4653894128, 0xabc0c98c158cc5d5, 0x745a4d62df5ec60e, 0x3ec6cae0ee515eca, - 0x2b07afe1caeaa92f, 0xc356f794cde9a011, 0x2980bb7b494d2227, 0x7932073f512cca32, - 0x2fb594e0425ab0e5, 0x6cc7510ae3fd0bbd, 0xbf3dc80c48c189fb, 0xeea71f5a33d0cfd7, - 0xa87e46e6b00a6163, 0x1155b263b6615e5b, 0xbd01811d777cb68c, 0x664521c46d491d6b, - 0x4cd3dd031a927cad, 0xc0d4b0c30b476fce, 0x4444153632d893e6, 0xa9e72bf573afe5a3, - 0xaf55c371112ecbae, 0xb7256d2863729d55, 0x99934eea7c7aba6a, 0x3949053f99ff16e4, - 0x6cbad34f27bb0c25, 0xc896a8efd813e51f, 0x91b0a2a5ed9d7bbf, 0x92817b604faf5594, - 0x6a87de3f9172672d, 0xf8bc1d2b2e3ec83b, 0x7dc74321261e4794, 0xa434fa8a7d31d0bb, - 0x69400ed4c12908cd, 0xfef3b8a2f38145f2, 0x82f74053941ceb7d, 0xbb5d052b0a7d5c9a, - 0x15d290651e3a746e, 0x001bcbbca6acb6c0, 0x83f1637fbcd0e664, 0x8b1ea6e53fab0653, - 0xb72f92382028b839, 0xfcc53918788388bb, 0x635e04bcc0aaa3ec, 0x1cf4fd75c4b6d2a6, - 0x88260134d443ede4, 0x26ef5d26818a60da, 0x7740340f76528012, 0x1594a4cec116fbc7, - 0x49461554665be9a1, 0x82e9e49750569696, 0x92583728a3f943b6, 0xfd4e3526b222a136, - 0x9fd1545f7f24a025, 0xed5a38641f96925d, 0x995b089a731cbf4c, 0x5435169636b30daf, - 0xf57f17ade62875e2, 0x57e7ac71628c133f, 0x6029b713c752f48d, 0xd50724f25a013f6d, - 0x1929f89308635742, 0xb1252aaebabce0c6, 0xa18fcac2824a7ce8, 0xe1a76f96d7f36558, - 0xd9b90d707d7a694a, 0x251c8e4a9b0cae32, 0x1f95e6096e7eac95, 0xda4693531b91d6ad, - 0x61701f9e51b4b34e, 0x32e1f656ac3ef7f6, 0xf6216f20382a35a0, 0x6a46692948a1c263, - 0x915edcec16a079d2, 0xe46ed15af6128d0c, 0x493e7137e8db44f4, 0x32cbbb71028811c2, - 0xeaf1373b4526f49d, 0x3d87e8c73108ca61, 0xd585e1960e074797, 0x3e94551d7fa5da45, - 0x69d95896130464f8, 0xd331598da49d4785, 0x05d88753abee76b0, 0xf2f3e8cf85a165ca, - 0xb3562cbbfffeaa7e, 0x13682c7215a84b17, 0xc9ee56ef2662e4d0, 0x636f129ad53e4478, - 0x9a49b95f948693ec, 0x53a56df039d29768, 0x276c078b4ba47b0a, 0xf53f09881fbf97ee, - 0x1c28b986b3f903d9, 0xdc014aafa8e4cd80, 0xb8517ff49f2dee13, 0xc9e135ad4429d32c, - 0x60e643a7330d0875, 0xb10b6fe468c29898, 0xd5c79b7bf3073dfe, 0xa2a22806719b0e1f, - 0xb99393500ce302ff, 0xbb237d49a5deeb59, 0x06c1dc92250e99d9, 0x80787053329b43f3, - 0x4b3ebe2b5f1f0061, 0x6f23571e051efc9d, 0x96afdeba57cbb6c2, 0x77b380d5e2101881, - 0x953b255b576c3ac4, 0x5389f935280f8070, 0xc09b873c7c3d36b9, 0x3d8a8f633ff746a5, - 0x03ceed72145fc246, 0x86394b6581fa0c21, 0xccb1a1bedabec619, 0xf04a08e84b8d2993, - 0x1b13b27b557461ab, 0x6e645a1589bb810a, 0x1c2ed089b583881a, 0x26e45622c88c1e0f, - 0xb3daf09f222c1c83, 0xdbd238c6b2574939, 0x2c07cc6c31c77c3e, 0xfbdb95546b98af58, - 0x400fb496fbbbe90b, 0xacf4f5394ae9ad90, 0xb9ce6ed4d338450b, 0x341ee8e9239e15c0, - 0x9c8be6b09c31510c, 0x974d589e2a643e44, 0xa4b951a1df326336, 0xb38706ad8bb69f1a, - 0x333ee89499fb3edb, 0x54977416cd9b9f02, 0xda0a3331423c0797, 0x612e1cec46175cb4, - 0xf07d665723f926f2, 0x067574f584c7498f, 0xf4f80e599194d26f, 0xc8803b771f0b96c8, - 0x0ba754ac1fe3cf8a, 0xb4fc7ac7f2dab617, 0x38f86fcedee681df, 0xaefef1e8f04152d6, - 0x251db0d004e0e2d1, 0xc19f5f9e1cddfe4a, 0xa204c7ee1db9a0e7, 0xb093b6b4ee18f7fb, - 0x3ff1dbc877dd8b82, 0x739b54a86bead240, 0x1c16a9561318942a, 0x99f914cf9865774b, - 0x623a9bd510069be8, 0xd37c3272da1cfa25, 0x22fd93d02b541f5a, 0x27f7c60a36166272, - 0xeaa856e77183e0a6, 0xfc40e1094a67ec4b, 0xcd393283035e44b6, 0x484cb965a67f7bb8, - 0x1bc3e7c2042ec3c9, 0x3fbd32549515a339, 0xd8a3872b686f13c0, 0x960007f3b5ff4467, - 0x40d2743c4f6c9140, 0x3d9132aa1b3e24fd, 0xa8abf7700ad7b23b, 0x569be56992ee0d31, - 0x98667697ab8a3b62, 0xbc0fe604366f63bc, 0x4a198d629d973a8a, 0xf1a92b28f72d2958, - 0x2fffcd6a1ad67448, 0x1218407966041197, 0x89d7d5af7ed22b8b, 0x7e706b3ef8508cd8, - 0x055896d0f0deff9e, 0x3126cdb92918c859, 0x1e38fcb1476a5d9c, 0xd1e3a0c92d4a6861, - 0xc816e68d558e87c2, 0x5e0ff7bdc264a1e6, 0x27d4c47f41a287e4, 0xf8851b68a93af89e, - 0xe28aa64a80995088, 0x01c4689ace7bf460, 0xc6376057e894d337, 0x44a6b1ef48e4866f, - 0xab4a0ab3e8457138, 0x5378e9c758e17968, 0x0da8b846ad8aec47, 0x70c394ced5b48b9d, - 0xf008d7aec8ee2621, 0x7caa8472b19fc8c4, 0x5a374a8881cf3f10, 0x59cf7fe66077854a, - 0x3cc5d1fbd14e6186, 0xf9cbf1ae84470aae, 0xdc36654dbbf843c1, 0xb636e59cbb1caf9a, - 0x2e8f0e8f57134f78, 0xeebb48bb07d8aaf7, 0xd02c82d35fcd8bce, 0xf7209fc26f272249, - 0x9ef2961799f45193, 0x0cac99673d019541, 0x66ec5b58d4d13dd4, 0xdc4d253bc0c86357, - 0x902aa34bf03da786, 0x8d4a5bdb79e91154, 0x22289fcd13fb4ed4, 0xb54b84be62e31ed6, - 0x623cf3764be7e051, 0x2ddfc8887f1c2fe3, 0xcad35095c778dc23, 0x1ef5a2aec3ebf81a, - 0x57c8d17e8088ad63, 0x9ce3f3e19d0c65f6, 0x9eb87f4005374afb, 0x62aee4900ae7d053, - 0x9fdd2ec141e9ed99, 0xd78d632908935cb5, 0xf263af92936ec69e, 0xfa9c4fe8e1af11d7, - 0xb65b8bc66351e430, 0x119462cd62df12dd, 0x4d7a800167f9afbc, 0xad7067efac892403, - 0x0dffc39a2a2090c3, 0x37ecc2813254c280, 0x1f131cde3a6859e1, 0xb743bb732be8e124, - 0x26a86fa4ff72cf20, 0x41b584bfb9c2b7e4, 0xc7cde8d15e69b89b, 0xd5085ed6e44f6326, - 0x5e287fe09bed2b2a, 0x420a2a8eda43c034, 0xc960d172f92b0682, 0xc544bd340165599b, - 0x436b6e509e76a3c7, 0xc0b34f2a44427e26, 0x9a6b7018de666968, 0x97a3c534cccc75f9, - 0xd0d79a847d7fa21a, 0x14288bc14906c587, 0xa7b92684133f244d, 0xbdb69a2750b742dd, - 0x99dde9344a107112, 0xb99f071cf3a17929, 0x20c85b899b4bd960, 0x2edd00ee6dd84b8c, - 0xcb86453b16054562, 0x6f7ee5c79ea6a845, 0x747fa298cad38981, 0x11cca68b0133a5fb, - 0xe4d8a00db3333c05, 0xab431c6116f8dca8, 0xed1505ea4073dcef, 0x335409f0a2277dae, - 0x8e5e47b9d4bda264, 0x9b9097ca230dafec, 0x72575d8fa4af2cff, 0x4622878980eb678c, - 0xa2942327e23b716b, 0x5bf2a1c9b4ff0e2b, 0x904923e99736fcf3, 0xeafe6d8ae6e88e06, - 0xed248c09fc41dd3d, 0x6590a57965c4d144, 0x8bc04b979c4bee3e, 0xf59e2e4dee7ac97b, - 0xe6779aa7c9bbd853, 0xf933266ebbaca6b9, 0x66f41467bcb9ebaa, 0xe3272f54be5de02b, - 0xdcc55dab960b5a64, 0x2e3cf65b0eb17eaf, 0xbfa27c2117ff904c, 0xa36270a7545f0d94, - 0x4cadf2ec850543e1, 0x455ebc849ae79ed4, 0x0da4f967d302677f, 0x96a99b3ec9171212, - 0x74327b504eb18437, 0xe750a910f35d149f, 0x9d3f22c390484945, 0xe3c8fb6e36b5431b, - 0x6eb1f8215f871f3a, 0xcbc7938922f03c2e, 0x6fa5d58905272470, 0xe566cde893782621, - 0x295a17cea223da44, 0x2c8a9f00fb2dfd3e, 0x61e490bd4a311a60, 0x8a9428ed33ea0882, - 0xfef8227ae51d184c, 0xb2ced339d60c5450, 0x9d23ec6a32f6d902, 0x13e82060fe0b6652, - 0x455d6df5fa78ef6c, 0xe7cb917a55dbdf45, 0x6507ef8fdf02fe7b, 0x6e2a02adf8c946e9, - 0x805f31a19e4dc49b, 0x04b6b85ea8c558c5, 0xe31e6b1a9421d638, 0xb1f965f23a5f7d0b, - 0xbe7546cb917661c7, 0x5c66b5af967619b5, 0x0fca6dad373e003d, 0x59f93a519831a61d, - 0x2484d8891b5c707c, 0x6665cc0ebf3da632, 0x3510c9b898f470fb, 0x23b1c8225f2d6094, - 0x1923f05f8ecd0d1b, 0xe81908f48bb274d1, 0x894166690868c8e1, 0x3fa4c72768d63279, - 0xdd8fd5067e000772, 0x1586cd7d09f06edf, 0xe36c52b22e89d16c, 0x7199a2bb8a9f6705, - 0xdfe0c6879c9f2ba1, 0x998bc8cab9ca49e7, 0x694ddb746ef23aba, 0x18382b9d14579d6d, - 0xd80cc85665ae048f, 0xa1fdfbfa8f9f255f, 0x07cb129213f2b614, 0xbc554a6a7c439cd8, - 0x07a8e959f20461fa, 0x945565aa30b11d13, 0x46b0c3ca62d7ff7b, 0x291243400630f896, - 0xcf83cbc56bf0b214, 0x3a760aad877b9b06, 0xdb27be97a848b363, 0x9585fe12c144a86e, - 0x770057e646101d2b, 0xe9a73afcb98545c7, 0xa6017bc9b2273f5f, 0xaa1a5eb8fc80fe9d, - 0xe93a7e929bf6f40e, 0x1642317027d69a82, 0xebd8dee408fc28fc, 0x01b0e3a921261898, - 0x12e08a5d06918211, 0xa02d6febd9f2cf00, 0x3f27b7156337c216, 0x29367efc31ac0de1, - 0x52fef397ac540eef, 0x008dd5184b8a16ce, 0x705d57fa7a7faff2, 0xe0848700bdba073b, - 0x8440e45537f27536, 0x1dcf9924d2a26e94, 0x5e173bb7dfbd9c12, 0xeb2b645d24b3809a, - 0x0e806cb5a1296865, 0x5a8c4085fc136102, 0xa20e28e2fa3354cb, 0xe111d064f962993e, - 0x0ff6afda7f70fcef, 0x2b4bff9cd2c74aa4, 0x2c005309f8654e95, 0xaddc7e914ccf1ae8, - 0x36d61d9964e28344, 0xbd168ec382ef0333, 0xbbe75323161928e2, 0xc17b274b41047d43, - 0x74d1629263047cda, 0x1bf53e662303b27a, 0xe2dbbce865c999f1, 0x6f3451f33815bcf5, - 0xd09babe6e5a1fdd0, 0xe0cbc2b7fa93933e, 0x98bd49718c8a470c, 0xb4516ae0675752de, - 0x491bf517c04dde9e, 0x2c8860a8d0eab6d9, 0x1a1a54b033bbda67, 0x559d850100066bdd, - 0xad3a5f817321142c, 0xd257342c29489c8d, 0x27fde435f7dd094b, 0xe16aa4f9dcecb90b, - 0xb64990b25214136d, 0xb7cfe317c913781a, 0xa71720389e8ec993, 0xe37232ad1fa3d9f1, - 0x9515c778ec4e38d7, 0x3ae6d32891b2d1bc, 0xaefffe61927773d2, 0xf3f00c8a5f3bc213, - 0x4d12e04f6cb148f8, 0x60f267ed8015194b, 0x6cb763015af477d8, 0xa503949c1e0aaa00, - 0x489f491716d0d1c5, 0x7a014a7f95969b07, 0x712d9e3da6974e49, 0x3d83f50a787b0c26, - 0x9c51fe2431804738, 0x095a35c59b965f42, 0x34186c65970c459a, 0x7b3c99092d4f65d9, - 0xd804bf942a3498ef, 0x411cdfddf367f2ab, 0x29e234b76f98924c, 0x9dda740b8d91965a, - 0x7f245bc00cd6c527, 0x5cc1cd33087e714b, 0x1e3f7ca0287674f1, 0x1c567ae30b751b44, - 0x74a28c468eee7344, 0x4a8ae0a87aaa9a4c, 0x4c477d6fbf696803, 0x7a0d08c402ebad58, - 0xa8d71d8dad8f01b5, 0x03e960ce6bbe3f9c, 0xcc15bfb2a9772b12, 0x23c7c4c6bd1f788b, - 0x94473747d9b7d678, 0x1afbdc98fcb131f8, 0xf10014eed69a23bc, 0xeb6072976fa20b19, - 0x21fe4a38d8184865, 0x0161cd1a12e703f2, 0xdfb93652f9798a55, 0x7cd03edaa5ae311d, - 0x7a9a56cff176ea47, 0x0afce8838f171538, 0xf223ee6b05f510ff, 0x46315f54e128a927, - 0x0051b92fe82c365b, 0x2cb3b32ff4b3f07e, 0xc763672637c94ba8, 0x3355a30193fd1509, - 0xcc21a7e88f153cec, 0xdfc5541bd6863410, 0x38095657116bf742, 0x1cabfe5a923387f5, - 0x384280ac9824aca4, 0xe43c92eb8c11ac95, 0x0dac6b3d49476b36, 0x6890eb17aab8f8eb, - 0x22a23eccceab5956, 0x5490902d56763336, 0xf32198fd1e30be76, 0x1f38713b67d35c93, - 0xd70e8457b68f60ed, 0x91679e7a021f0bff, 0x556790f0d3ffd96f, 0x10e638c14f1d169f, - 0xededaee8670cb507, 0x57ecbddeceda2185, 0x85623e5ff08e4184, 0x25a908088ef77237, - 0x42ab939cc2f002e0, 0xc40e512f2ef69d36, 0xbcfdd37a106b953b, 0xb85a4d1a712b4ac3, - 0x41cd568e908b779f, 0x2f341eed1dc39aef, 0xadbb972925fa61de, 0xa5fae87c93290bd7, - 0x38813dad5dfd4df3, 0x75e913d56de5798c, 0x97ac58ed9ba51df8, 0x889f0b469678168d, - 0x1bdc52b60f55b897, 0x0e2fa3fb7fa34143, 0xb6af6ee58471e75c, 0x824932b44482907a, - 0x9c23b1810e2d4748, 0xa2ef8f9b52e2a08e, 0x45bbb5a1c9d72acd, 0xd83a1a96afbde5e7, - 0xbef667cc45ecdf2f, 0xe6458713f0a6a90d, 0x087160acab53bb75, 0xa6e1eb1768d9e9ce, - 0x571ed6e38ac7e3cf, 0xb84c3947f102b45b, 0xbeb00fea95c61d7f, 0x5bc5608cc751fa50, - 0x4329ed6453aa9f5b, 0x5ff5d5f05fc5f1e9, 0xabb817f7b532687d, 0x638507ca97ca8c5e, - 0xc33059193b486299, 0xa89a09152ccac8ac, 0x6915745fce4252bf, 0x22b291dca76ca9d3, - 0xf60a2d08b4410f39, 0x213a2f569c2de403, 0x299b6f6d394813c6, 0x545e9f0818790a6c, - 0x25301e7e3f6bfe9b, 0x740cc41235219c3d, 0xaf1c7c372530f074, 0xbbb6bf6c81316e62, - 0xa3880fa9580915c8, 0xa8e4482a33204d79, 0x0b75f81f149267b0, 0xad1b911dd1d43be8, - 0x2081d5f7f1a66091, 0x118be91329ac7633, 0xd65a4f9959367613, 0xac6c934d9d0debab, - 0xd693bf47c693e5d6, 0x2d83768b82c20e58, 0x316ae1060e702938, 0xe020866bcf5b5dda, - 0xc68ccdf29d438cc6, 0xc2b863339618065e, 0x04af45fe7cfffa9c, 0x15390cb5cae665d9, - 0x9f57c2b120362e1a, 0x8eb9cd2f20c5ac19, 0xc995c3a94d3c3038, 0x6afbb0603ca99800, - 0x1c9a7883e402bf21, 0x7e0be01a537bd296, 0x33fce9189c973d9c, 0x5ead0cf07d88dcad, - 0x87a8e4d37d2ba1c2, 0xdb90c75f183520ea, 0x1a684a8973e11b97, 0xd92db7fc4c92d5f8, - 0xe7900620ba280f13, 0x7652b80a1b21e2be, 0x87f6df43491cd49e, 0xd78da3354387b60c, - 0x05f3ea9003f4b985, 0x5c493c023ff25483, 0x48bfc5e4266eec81, 0x1fdf442a23a785dd, - 0xe44fd5bba3c59c0f, 0xb6e60900083a314d, 0x3441ece04b932eb4, 0x6837d3ba86ba3777, - 0x80f0bb5f2b458247, 0x72d36fce0c1f6739, 0x17c64deb89908b3e, 0xa28cc3f228c913a8, - 0xc71fae72ec3d6c91, 0xf97126dbe5a6bd96, 0xfd48aebfe8010118, 0x3a094ee78c6cc819, - 0x1af5a9edd489a177, 0x56a1777a58911009, 0xb7f9403cb9b14866, 0x8c1d82fbe696de4d, - 0x352848c9bb6154da, 0xab9c227a680fcd9e, 0x8686ad05b6745b9e, 0x93a23421cac06964, - 0x62f3ecbc2c381c88, 0x42ed37fa24b620ca, 0x550008f1c292b43c, 0xbb8e71996c144d40, - 0x0b6f380ddab18992, 0x8ef856b32ad94401, 0x0cd6894577b2899d, 0x9204292a0fb039c0, - 0x88a2386ede01e3b9, 0x83132b498f32a515, 0xb91a4a795a23e1f6, 0x3a22f312c945be7c, - 0x3cdfbfbbf4beba53, 0xd2984702c3fda4a1, 0x7c140ae728d9ebf2, 0x56bf8304ac7d5ed6, - 0x407fcff404cfebb2, 0x7e2af89e1bbcb43c, 0x21a777faa2718aa1, 0x4526017bc9ffffec, - 0xcfd65da89d237ba2, 0x527af1b4af567dce, 0xcc9dc24ae0cdbb07, 0xcea8983ce69e8e0a, - 0x3a211929b4a8acc7, 0xd2bffb9423d7304d, 0x632679e00b749acb, 0x5f6cedfd6e53ec92, - 0x22757f202fa88dac, 0xe555dec632efee53, 0x870b6de91bc79998, 0xe63cebb321de3909, - 0xff61093a070097e8, 0x4010c660ca3e62d1, 0xc315ee19bfb557b7, 0xd9f2725efc6b7ce9, - 0xd964bb99679b3b7b, 0x83fed68c2f4117ae, 0x6875f474437fbea4, 0x295d5e0c22e6f040, - 0xa01a24baba478acb, 0x68bc7bf99a58f889, 0x648f83a84777dc5b, 0x4d55f9ddd4b69d0e, - 0xf9f9e9fe20c4c7c6, 0xd045b7395c45ff21, 0x02c4f4dfcecd74d8, 0x840cdae79a34a601, - 0xdf53cc97520b07e6, 0x3b7655117b62bd6f, 0x5907bb57f1623e9f, 0x948d2cbfe625d752, - 0x48c85e353071320d, 0xfcf3b5cff12839e5, 0x9e0179af837dced6, 0x524f8d75fa202978, - 0x61573d42f3393ced, 0xf0d8dff1d35bb0a4, 0xa121a2be71362cb0, 0x214f813dde7b46fc, - 0x8007b3bceed2cd83, 0x667b707656dbcc04, 0x25f03a56ca0e1b5f, 0x29be5ce7ecc72501, - 0x76fb3cf7867e6329, 0x3aa0ec5df00012bd, 0x4078e11617ec88ae, 0x32ab0aa0a19047d1, - 0x091639e2a3b47847, 0xd6ad7a6e4d066f4e, 0x6af83e46c9a1c27f, 0x987600dfc450923c, - 0xadbd5bb11e8b0e08, 0x56c2e0984311ca95, 0x06c7ba302f16a29b, 0xcb02674e2ecbfe02, - 0x95f070dbe78540d4, 0x81e8b77c786b00aa, 0x893fc5e0e4301407, 0xb9b2fd69f7d01e18, - 0xf7d6b9909d9535be, 0x5bbf759c3064ab00, 0xa7acc9ebf69b8e9b, 0xd91e3e93d8eb856d, - 0x3aa7aa06ce151cf0, 0x3518b46d24b0ccf2, 0xad5c39799b72e52f, 0x1ffcae13653ea7d3, - 0x0b5e797f12be6610, 0x49511362e672baf3, 0x84b097e71ba1f9d0, 0x774c7f38887ad2d3, - 0xebf83855a3688edc, 0x8d4796169bf9047b, 0xc68f0de7d9fad18b, 0x4cd24186e86f46b6, - 0x085f8ab97b5a4851, 0xc69006562cdec21f, 0x8c6b7e791cf6a1fe, 0xa39cfd6078564780, - 0xcfd4a44824deb7a7, 0xfe752f70933d6b2c, 0x7b79a72b5471cb3c, 0x7287c8951cb8bef8, - 0x72646244c30041ec, 0x717a2744188b495c, 0xe7aac4606b5ab9b6, 0x2f554e07de97f229, - 0x1fb4ab13a7ea6134, 0xf57e69197f83b982, 0x6451f0db1328c679, 0xd6e3a770126f8d71, - 0x021bc61de1d1b0ff, 0x30d9847e5ff647db, 0xba94982a71364306, 0x2f996c1d9cf4be43, - 0xd648d542f82c6f47, 0xe669d2973695fa9e, 0x0ec438e67f26acbf, 0x1f611823201e1e29, - 0xac49069f53a3deee, 0x9f9a5ed174e3f382, 0x73dabec68efd651c, 0x148f6924b5647cdb, - 0x94fee701d3aa9227, 0x9bfc0a6f8d893a8d, 0xa6ed74a096c4021f, 0x1d262691c1b78989, - 0xbea9e44bc57b78c6, 0x018c087da9252adf, 0x951a07e693621520, 0x200b6aff42b49da6, - 0x38a4acf6c2f54309, 0x291f264a833b5a5a, 0x28e0b13f8b03f290, 0x51c179f66ef9a918, - 0x4d524503be4c7559, 0xaec857b8eb53b4fe, 0x620a9f76a0c78fdc, 0xf477f80165acc127, - 0xbbca30312d221669, 0xca46490d25a284c9, 0xa9665971300b2bb2, 0x9b7d9ae72b040752, - 0x1ead30430518597e, 0xeb9367e65fc69fc8, 0x32a6df400bfad080, 0xe2a8541d1db445c2, - 0xf6d4783950910d8c, 0x7ae5b848d97a9d3f, 0x41a15f8dd1d1206b, 0xe973b7c141fe98e8, - 0xfb115b9ec872a12a, 0xe7601bc09684ef1b, 0x24435ef8df74d661, 0xe99441197722d6ff, - 0xb4338c5734a2fff2, 0xe2f898a5b58fb483, 0x62198f5031b08137, 0xcb2169c29461edeb, - 0x3dcab0d2dfcb5ac5, 0xded9cba5ff085f28, 0x0aba1da5b01750b9, 0x9fc8c17ea304415f, - 0xa89c37ea335cf789, 0xf6491bc32753eeb6, 0x7b933d8d40a745a8, 0x3d174e4196057c0b, - 0xff24b8c09e1297dd, 0x6928f8b07aa2502d, 0x8ac43a4483311e96, 0xb8b5652d6c412630, - 0xed5c6a3180f32cc8, 0xa146e484e53cc8c0, 0x9a7ee9760fedd270, 0x20d4f06c413a6ba3, - 0x28910a836f46fb81, 0xca35e6b848bd0730, 0xdf547045bc65b05a, 0x1ed7f7a04ca4d6a9, - 0x898d2574b369fe54, 0x12afa3eeca130c38, 0xda985b29e3397857, 0x11bf9040eada0fbd, - 0xa19527e47737e89c, 0x71eaf3ccc2cfc809, 0x50a920d8bc83a60a, 0xebceee8275494bd4, - 0x293e3c48b5f49ae5, 0x698a7705f40348aa, 0xc72cccb043c181d1, 0xb490a5eb0b921b2e, - 0xd767eda079a1700a, 0xd6bf09abaee22197, 0x83e8130da46fcc2b, 0x090d44dd6b923f70, - 0x5394d6c1aa81bf65, 0xdb00797b2f86986f, 0x31ee4452a73b8bdf, 0x580c0f17cb00e446, - 0x74a4883bf6846008, 0xf8c4bd341356376f, 0x49d7978372a3b23f, 0x72f8ed728e85b27b, - 0xdf2c6d077ab1aa7f, 0x69af0581d8cdcb9f, 0x713565f5f5394f51, 0x8b4b3e06520b91fb, - 0x05dae26ed41e675a, 0xf2e754964495de49, 0xfd906a7971c00ade, 0xb7c364511ce398d3, - 0xfd701ecf7e811b45, 0xbbf328ed6f12f8b3, 0x49a09c7d47ead9e4, 0x7e6c7aeac51ad46e, - 0x380ed291e099d818, 0x21d757c28a7afb50, 0xb92a67872779bbf2, 0x22910c5d05323451, - 0x3ae9c6adef04813d, 0x2f3c74a512583030, 0x0dd086c10c4513ed, 0x9da9b5b252e60916, - 0xf5a7673faabf217a, 0xa4d6ba743e0e8261, 0x7a5dca6ff3ae718b, 0xb48d02aa60e9f0d1, - 0xa6dd41c9d1cddef6, 0xbcda594db2e6a62e, 0x0956c95e1e0e91d0, 0x21841a8e6edfb9c2, - 0x0cb241b90d042309, 0x28b1b1f9af25a92c, 0x89c180ac50076ad3, 0x5e33dabd392a9697, - 0x0efdbcb4ed592bf1, 0xde15cb0fea225f4b, 0xf223709b20ebc363, 0x79e1c79f4bf1b37c, - 0x26a157583236dfd9, 0x79960a7ad90bfc31, 0x1d2175c347b55bf4, 0x78b92d8c808ca11c, - 0x3337192a6ae5ac43, 0x23b74f86894916c4, 0x164a4d82a76fd184, 0x45eab54b9bfa7d8c, - 0x864950e763559c9e, 0xd9ff52717eec5891, 0xa0933317fc8c4cae, 0x1495620b622c52da, - 0xae34207a978f85c5, 0xd0178970a2b7488e, 0x08d413df3f936223, 0x21e5422e4e099217, - 0x1ce6c0b37b7cd9fd, 0xbf8b8839736155e0, 0x28ff971ca2c5a063, 0xf50875bd4ac3ca99, - 0x8ef854d982edcb16, 0x1e9bda02e2d85d96, 0xeca1d8cb73d60af0, 0x25c1c5c4ea83fbc7, - 0x0190d4b3d57ff092, 0xe554d97151e2fa52, 0x70b9b73537e1855d, 0xb1f7bccaf3506abe, - 0x01163113d10152de, 0x5f6b0f96721e2854, 0xccb5ead1b489925c, 0x9133eb3624a47cf0, - 0x9a6c7eba68df1594, 0x9eec41ea23fa0b5c, 0x3cdbbdf5b98d8b4c, 0xb985dc8f8e9e0fc7, - 0x8facae28ef0ab490, 0xd0bb5b5cb7a25544, 0x774e2fdd5e2995d6, 0xcbbe218c427ed27e, - 0x4f2d7daa2391e3d6, 0x3d788e433af4a921, 0xb828102fa2eebd1a, 0x69de4e9dc394a368, - 0xdc65089391378696, 0x44e4aad6a81e98cc, 0x2d2148e67ae55af3, 0x8f8b5a64113509c7, - 0x33e4cd7c19e9a8b2, 0x0041c875b8faabc6, 0x09abc607fc52d704, 0xdd1688995509d186, - 0x022c31273c90dfce, 0x7714474a31241496, 0x4c63d3aa19d79170, 0xce5622277d47937c, - 0x5781090df672a571, 0x8640ceeb8cdc5a28, 0xffedddbd765d1100, 0xf6b5a990254a354e, - 0xdba0eb4a6c9f823d, 0xc952b52450f289fd, 0x7029e6e0cb17f410, 0x28c3a52b778d1314, - 0xda2ba226420f463a, 0x1a04b80b08dffb9e, 0x95f2ebe7a0ae3e39, 0x0163895911108709, - 0xe089e1cdc0fceb1b, 0xdd7d58b111125530, 0x308d69c2cfe54ba6, 0xaef42d1c276e31df, - 0x27ea92efa48995f3, 0x1cad0906aa24dfbd, 0xd852a9989db95224, 0x500e04c2890e8bf4, - 0x211015d5dcfeb006, 0x8ad4e805b53e46c7, 0xd29bf99b9471be8e, 0xe3b7a5f5dbc2ddbc, - 0x67441f29b5c26937, 0xbfe1c12d30ead35d, 0x7ae1d51d06a171e1, 0xe70e69cf404c38ff, - 0x341441136f277771, 0x32ba8bc656aff63d, 0x3c9de8b0c111ca0f, 0xd9b2001258ba8fa0, - 0x3eb4a3bd1ac64896, 0x2048ec9cb2765cfd, 0xb51b6d8c486427c0, 0x5685d4b4d1a92825, - 0x9a53ce2d78ab2112, 0x81cc626e75e674dd, 0x213f36e563be6bbd, 0x91cd5cd654a4d04d, - 0xa165c8cd02922d2a, 0x9926685a10d67f53, 0x4ec68ebb001d9396, 0xfcaf0eaa0327828c, - 0x01584791b6e5766d, 0x49d2a1f50f0051e8, 0x89d72e85c8ca7873, 0x3e76491ceec661a0, - 0xf998f8eb67d43c41, 0x193e5ad2adbda372, 0xd81bdb5591a38247, 0xdd636bf82f7474e9, - 0xc9b0a0b8950852d2, 0xad059b527bfbc39c, 0x28afbfd73b562aee, 0x34c6e8c8119e97c6, - 0x46f405766e4ce273, 0x1a323bbe5d794524, 0xed5675d9c7710da3, 0xfabfd4f161960c7e, - 0xb52a3e921929a52d, 0x9e7b55dc5b17861b, 0x8b3f7ec97b34b77f, 0xfa63be6376f6ae1b, - 0xe7a04d4bd8ab7fd6, 0xea4b187503d6e89b, 0x7999b2516bfd00e9, 0xf24d7bd65df15db9, - 0xa2da72be4d541871, 0x4cc78155a3dce4d3, 0x75e337c7ea9447c4, 0x94b95c55b9066eb7, - 0x9931014a671fae5f, 0xc81167293332e36e, 0xf6cf02e94d6352ad, 0x3466e040a2e763f1, - 0xde107bcd0bf2d30c, 0x7bb2a4af8928d195, 0x9fc9909aba44bdc1, 0xb69f09a4836efe5a, - 0xb53a3d487a1de808, 0x5558873b25b08b72, 0x2057632b7f963b6b, 0xc57da58dcc6ae516, - 0x29bf44619269e1c2, 0xd43662524a908590, 0x23981aa0f9d50c68, 0x65712ed531b9fb3c, - 0xa0ca71f9afad9454, 0x73266f87baec3819, 0x8322f0acdd77584f, 0x57190270e5ea1cfa, - 0x59e47eadcf2e94bf, 0x284bf97a77501a3c, 0x1cd585e03585934d, 0xc9eeb40d134a6aee, - 0xbcbfd09b1e3d3f8b, 0xa753d649e6b389bc, 0xb97cbe57c44c8d57, 0xebd8ceb4978b6630, - 0x3eb8f6dcf3137377, 0x15db7bad6001419d, 0x8eca34a517f883e8, 0xa80c5e00d1ceecbe, - 0x60d32b2088b97522, 0xd445c93e31e05e59, 0x23cf0ec5100ccdb8, 0xf950efe33a61ec9e, - 0x8749ba66ed240cb7, 0xdc6ce043323c8678, 0x449e0187155ac2c9, 0xd6647132dcef4d0d, - 0x58f1a152b44a9d7b, 0xf5c3481bb733c774, 0xc02673b31d92101f, 0x6e03cf73738d3803, - 0xb5ed094017928493, 0x4591a1b5d038b220, 0x823f20167f714031, 0x44f289c7cc739233, - 0xc3517c06cc4a1dd5, 0x618dc92f70f899cb, 0x8d404bebfb35b752, 0x42c793ad9a4f30b4, - 0xd6cae9fc01da9a1b, 0x3e513e6c94021465, 0x4a0fbf595b75f99f, 0x947bf2268c6fc1fd, - 0x4bf383dbb2abe156, 0xee9c3b40770fad2b, 0xa0f59bd95a635e4d, 0xf52b99e41857b678, - 0xfb5a63a103491dba, 0xcf2fd58f2a057c9d, 0x21836c7f301c6a0c, 0x9932fd6bb9fa6a4b, - 0xafd35eba99b8a219, 0x7cbc1ee48db98b01, 0x035f281c289308fe, 0x58f1e075aebe9821, - 0x4b82a42cd84a709f, 0x25809ddc3df0586e, 0x7f7e6eca759ce720, 0xf025fb3006d2aab9, - 0xbe47add3d3418422, 0x84f79c9da216bc44, 0xa8657d4f01b2c288, 0xac8a6a9c206ca243, - 0xae8ca2feb16caa6c, 0xf9ca74bbd5ce91fb, 0xe1b09ff423a2850d, 0xfad85195e07244f8, - 0xef83fcd4f7cb17c0, 0x8d7e83d25dac0b14, 0xd13e4b1f6bf0a805, 0xadada711b1546173, - 0x81913681fb6ac41d, 0xf2871a7b1d87aef9, 0x79522c6602f0bcd7, 0xfc19b18797818faa, - 0xbe1c711cc2d07039, 0xe282629748973985, 0xbc0ed52dc0f5538b, 0xf11ee382d6e3f518, - 0x8f0bb5a4b6f910e1, 0x9867be165d4f2ad6, 0xc575fff9787a8d3f, 0xca2e094752cead36, - 0xc72373fb1d63f4be, 0xe303ad00705e6cd5, 0xfec75dfb0e83bda7, 0xcbe692e46a934943, - 0x58cf3ff76e3144a2, 0x770f4e34eca9b3ac, 0x331e5bcce115f60e, 0x33636e39eabb9cf0, - 0xd167e9c3b22c5d13, 0x52b2dcc00cad263f, 0xd159aac9c2d1588a, 0xddbc2dda8f6b856d, - 0xa4688f418c5cfd98, 0xecd38a707141ecf2, 0x5b99b1fd6cd27f19, 0xf40b1dbb2aaab56e, - 0xe67f615837f53a3b, 0x54d867ce32d85657, 0xd1ee4830cfa35d74, 0x72e354e9df5b4988, - 0x7a385a40858aa435, 0x18113b58a89ab050, 0x1f333a14d265293c, 0xc1e37f5a38399a4e, - 0x8ce259d3514e0083, 0x5f3547a7571ba2e0, 0xdfd706a79a21a5e0, 0x210f289b04411773, - 0x2e9a0d971296bf7c, 0xcdc67a85cbf4bc97, 0x29f09b85ad54fabe, 0x664f4c4dff7a1268, - 0xc4620a0890a27127, 0x71bd58f0bb0fb7fd, 0xf6a26656993ae541, 0x7377879994b6fc2b, - 0xb0b45443d0f617a5, 0x612517d9dad3ecec, 0x16d26f2c9c235d4e, 0x1153d58b789118f4, - 0xaf6f680c76b3ec44, 0xcce0ebc72bf5b523, 0x5e40f9b52897e9ea, 0x831d7996d9a3027c, - 0x01de7e6708089e0e, 0x0e1e3f5c0e82c623, 0x9230e985fe74f785, 0xd78fe4a978412519, - 0x05ad9657cf84896c, 0xdded432166286643, 0xfb807e7e346d341f, 0xaa957fab164b1c15, - 0x80445c56cba073ac, 0xd1074995a72e84e5, 0xa420e83d2a9f499c, 0xadd5e1abd6731899, - 0x7769bd9296ad66ec, 0xa52c71f373aa482b, 0x3c16c86bd3b112dd, 0xe2737ba27eeab189, - 0xc939ce7280bc1e22, 0x87ff0a09862a4a82, 0xcbadd2d869b08857, 0x317ccd0dcb0cc9d2, - 0xe8b878c2f628e423, 0xdf362ddd8e4ca6ea, 0x766285614471d688, 0xfb6d60b6fd22d7f7, - 0x5ecb3c6542243487, 0xed91fb7932e1ffea, 0x24742c48a7965792, 0xcffc43f315b5c810, - 0x0525d0a1b3c3c7a9, 0xb766b687a4357748, 0x0842107929f41d8a, 0xa6b7806bb9a09894, - 0xe7779cd546c72cff, 0xc0c2e16e881f088b, 0xbf85da98c666d256, 0x48ae1e38a05c919d, - 0x2cbbe4726a06830b, 0xc8d012ff9b748778, 0xbe027b5965f45972, 0xdcead234ad61e6a5, - 0xa08f0f51edbb24a0, 0x779f93688d90b167, 0x97943e1939c1e88a, 0x6768aab84b63c9a4, - 0xcf1be9701f11cbd8, 0x3f21e00d7ce372d6, 0x0a5e70ea4a3cc8c3, 0x7e2347b44d83866f, - 0x097e66a173356062, 0x7d9de4ae0bbde3b8, 0x9d7b524ce2b3860a, 0x80cb7afa7aa08ab9, - 0x595dbb3428fbc614, 0x0af42c2bfcec99c1, 0x93df7a96d2ac55ce, 0xe27648f38fd342db, - 0x87880f53c2edfa85, 0x980a945e77273118, 0x5673e4979e94410d, 0x23861447a4619069, - 0xe796a35305b956c0, 0xc0accff0a109a8db, 0x7f43cfebacf0e457, 0xe978e94658a0ff70, - 0xcd90e94d71d755ca, 0x09b0acd95c50703d, 0x6fdf2d5dca83bde6, 0xe9c13699610e658c, - 0xaad7420ae61b6769, 0x1f2ab89d7944e7e6, 0xa70abf69c1947b38, 0xb6d6c8db0f22ab0c, - 0x4b6dd52916073e8e, 0x4d9862f21348322d, 0x5932576e5a5f4efe, 0x28695e3c21119829, - 0x6d82a8058882d3b6, 0x754d1643bf6fa7a7, 0x64827dd6ece37ac3, 0xfb8ef96ff4c398f8, - 0x9ea7c45bd67cf7f8, 0xac0891d114108f12, 0xac89c63929bf554c, 0xb508b039ab299f54, - 0x9cf1a0afa921f0ea, 0x48bfcb975d4b3cad, 0x2fc8b7086e2f1ab9, 0xbea3d10ca0f575cb, - 0xb212ccd2106ed95a, 0x606fd99127b13ee9, 0xca214f94e65822aa, 0x442b90107f533033, - 0xede22288037ab4e4, 0xad5d2f2cc2845863, 0xb54c58aa294bc19f, 0x64fde41e6505fba7, - 0x998ad11f5c745541, 0x69a4550ff51ebc57, 0x19fba5a7a7be74a8, 0xf5f0fa08a661c298, - 0xf40aa076ba88bd59, 0xdca082482d972783, 0x1b3ff0657399e318, 0xb47c89e48951d972, - 0x81bd03eeb1517339, 0x44b084c8b5ff3b1e, 0xf6724e1830ccf719, 0x1af2086f81a36f21, - 0x7ba34e4b6b6a4a8e, 0x2e60d0c2a1738204, 0x5e1788470d8f1680, 0x02836d08e0fa153a, - 0xe3e706332ab4d2b7, 0x5039f4805855bc52, 0x6a210847dbb26304, 0xdb6dd17205f20986, - 0x74931e444d50eaae, 0xf70c760c3fcaf1e5, 0xcc4970a1a4bf2469, 0xa4cab62a706d1496, - 0xebe00d58e029e2ea, 0x864d749a6dedff32, 0x2a6b25b25232a563, 0x9c2fa00b04a16745, - 0x01886f87e9c77bb8, 0x3dd339b17db90d85, 0x69b8a2a843bfe53f, 0xb13e523b780f1457, - 0x95f8b60ebd145c65, 0xd8d9273b7262d2c3, 0x98aa8b09c2c98259, 0x5ea01dcfa765891a, - 0x102bbb82dc967944, 0xb11796a0dfaf008c, 0x62cebf15175ccbd7, 0x5ba2e07dd4819ab0, - 0x0bc0fbb17875b8d9, 0xaaf39faf1a8adfab, 0x10c397c689a85097, 0x4b0fbe0160cd4b39, - 0x3e73cd18b11ac513, 0x9112f3c8725ec380, 0xfba8a33d9779e299, 0x03464ea71e8e9dd8, - 0xa32dc67996b69e9d, 0x01f17c631cad0ae9, 0x4245f5fd9e3779f5, 0x4e369e48e4be944f, - 0xcfd0b2c37c2c9fef, 0x849d7367ab8cf2a3, 0xf4e8f6043e82a28c, 0x61a38ce6916f5ed3, - 0x4d9023c0edb38633, 0xa302358aedc0196c, 0xbeaf68928345c24b, 0x3710b2de72474619, - 0x5565a9c1a6017f07, 0xcb2766c721ceb683, 0x3e0cec72f4825305, 0xdc07c1446dd817e2, - 0xab8daa43998223f2, 0x23d687796e5dff7d, 0xbbdc3247859cbcf4, 0x9581741bb3cd2f4e, - 0x3262c17d0fb0dc1a, 0x6bbc3315fedd72be, 0x4391bb88d95c804b, 0xd73f83cbdeaacf19, - 0x2455b44718e24aee, 0x7f63091fdaec4ea7, 0xe20017d153f4d7ca, 0xa88579a1c5ac5eaa, - 0xa26cd14e9c27eb04, 0x91104ee07eb34064, 0xaee6bee4f3d84ac2, 0x2924e97966e4bb26, - 0xca1c03dfb5ec8bec, 0x58f932b48c7d4e33, 0x56ed3ad2817bb309, 0x5526d4beb2afa9d7, - 0x6e562d46af276a19, 0x60010660f238d3a8, 0xaa2a9113c9dfd728, 0x4d25e63c06c9b24f, - 0xb7f9d4332f3ed50e, 0xcda88c284a00a470, 0x443f3a5abe0779e4, 0xf90c8f6759e7d781, - 0xbfa48a58c7778a26, 0xc5aff9afaa9d431b, 0x5d5ef444695e1b85, 0x173536fae1d3bbff, - 0x40a2376f7b2aee0c, 0xa4c8a88571c3831c, 0x0fc7776a78b477d0, 0xb53db5cfc724a976, - 0xfec74eeb8eeee8cd, 0x1fdc3a5f83cd7317, 0x57457802c994454c, 0x336a229a06f51924, - 0xa4dc769355ac019d, 0x071f16a06c2debb4, 0xa942786a68dd89bb, 0x730e2a8b7e479e39, - 0xffbfb11ee010d0df, 0xb1d5c00f6cf4c6d3, 0x28b65774b51bf324, 0x2e4ca633f36943e4, - 0x20c561be1285f412, 0x39b140bb7d2867b2, 0xce181e474f9d41f5, 0x2dab0b582861da8b, - 0x9a35828c0a5c542b, 0xe637aca948de2487, 0x8befe3066a97c336, 0xc799e9cce3e1052e, - 0x6ff709740b966abd, 0xf814c052069b68ba, 0x72d6ca05769f8631, 0x16ec2d500ca7e842, - 0x972c946ed6b8f8d9, 0xa60707cedcc88ab4, 0xca50146d78874dc8, 0x40144e070791998b, - 0x947b7d92fbf8988b, 0x95d659b8bef3a11e, 0x54ca6ff939e9ba9e, 0x3177b7a8bc3dadfc, - 0x069fdf99d75a56ca, 0x6b4bdba415de5f08, 0xe96ebde5c4dff05b, 0xa2a0afaa01a2902b, - 0xa2f7f029a7813c0b, 0x7bfd7d83698bf42f, 0x06ea1a1aa3f464a3, 0xf32406a8e004b526, - 0x9f03878fce26e49e, 0x5692f891b51101cd, 0x267c7396f0d123ed, 0x5ea8fbb788699727, - 0xe655798a5e485512, 0x4591d67ec63b3f28, 0xc6da8554b3c00037, 0x343f2103b4d73398, - 0x33b27c1be87936fb, 0xa1314cdbfd98b49c, 0xfd462bbe17b7b92a, 0x0843a924a18f4ef0, - 0x329ef377d33377bc, 0xf7a40bb43720e4a4, 0xf91df778dea7d018, 0x9c2947760204753e, - 0x9c74abf7e8985b7c, 0x7200540d1cc7b15d, 0xbf2aadcaf041b960, 0x22886400ea99cee3, - 0x1dfba390e2572919, 0xab4a516abb7cf1b5, 0x2a66ce81ba02e775, 0x9a22912f15c6ddbd, - 0x8345793c6b48374b, 0x3095f94432f539e5, 0xcdbaa9c65c233f9a, 0xca58a7855a55746b, - 0x0809baccf28e6d94, 0xc84dabf7ceb7bc09, 0xd3b582c0519a8a0b, 0xb2fd73243d2aa59e, - 0x2f1bf72693e82061, 0x923ea17a16de91e4, 0xea60046381c06635, 0x78525df123e71143, - 0xa041d8555c758dbe, 0xf6ff98dc9eb315eb, 0x5cdcbd91cdaaaec2, 0xf86aab56abf3ef55, - 0x12e9ad338f7710f3, 0xf661b3709a7bcf3b, 0xcd003bd0cdd279a5, 0x77c790ddf751ff57, - 0x2a670ca593c9ce02, 0x0443d06fcd269838, 0x6376595d4c7f9db3, 0x936b8fb00f5ac45a, - 0x380279605615df1e, 0x4886d5c0e60da6c2, 0x5a759f5810af2c1e, 0xa5369c4decafdc06, - 0x3dcbb96561d00bbc, 0xc6d5b973729f0a4f, 0x20938c0eb2051871, 0x2fafe912ba443ba3, - 0xe42fa1f23b8c0883, 0xda65f7bc69e8744a, 0xd40b79a8dbd1f74d, 0xb3534e69531f783b, - 0x3ad14e214682bcc6, 0x1b0a143c93bc9cde, 0x7805b28d8978f134, 0xe2137faf17393008, - 0x235ee610b4cd29d2, 0x08e789036bc2dfaf, 0xf81caae860f96fc7, 0xbee5d915677ea289, - 0xbdf23a89b7f72db1, 0x845f28f70e3de092, 0xf82915656ecd3dda, 0x2cb7c062e6b27321, - 0xf9c9558bdff701ec, 0x42457ac2e3e7e60b, 0xa63676263dbb8ca6, 0xe01a69d91dcdd1f8, - 0x39f726a80ba035b1, 0xf2255afa8014a7b1, 0x9489068a399a5f6c, 0x3e212784c41648a2, - 0xe068dcb680152a55, 0x78c0f65fc610cccf, 0x89832644645cc026, 0xda870d1865f3c185, - 0xcb8779948b3c0e7e, 0x82a9652489dcfb58, 0x21631753f8a6ab46, 0x085edcaeded35877, - 0xea33b3c0bf8b904a, 0x44a2992dd9a1c105, 0x56134a80f16a1ce9, 0xa00e0d5a460b7d8f, - 0x7f805b067de63f0c, 0x0eb44b63d3d80b33, 0xf14c555bbbfa8312, 0x2f81cf1f7a5d42f7, - 0x1e400d68988d0fe4, 0x5b2293e0c62bd4ca, 0xd795de5c204761af, 0x34cf4779930d1c11, - 0xd4304e9f6514a8c5, 0x507d7261cd6534ad, 0xc8e1719a09ba365b, 0x1242af4357fe027b, - 0x68d81a6f7722171c, 0x8fe77861d81fda8c, 0xaa3fc32a99ad5d0b, 0x37353b260bdf9f38, - 0x8a2562bd73ce61b6, 0x7c50da0821ac1666, 0xa3190c9ed9750aba, 0xcbe4832d0505d6e5, - 0xdffa227239ec6826, 0xcfa7375f77c70053, 0x2a528663f2fbb58f, 0x5e5e090ec8527069, - 0xd35455dbc31ca917, 0xf8bac820a6593581, 0xa2908dce958dcc41, 0xfddd36fbe476c2b0, - 0x2edb77073c2ed3a8, 0x065bcc8945f3e1c8, 0xa620708ac3447e76, 0xba4efd9f28bb9d6c, - 0x4551c812eb3b2b5b, 0xf379da238289e094, 0x5db26aabf2565fe0, 0x6f6aca809accedb3, - 0xb28c431d118d1b07, 0xfd4feb1c1645ec8c, 0x15740e5e22f8b5f4, 0xbef0e2c2fe58e9d9, - 0x2a51e436dddef028, 0xd0f534a3ae44a12c, 0xa79f36aa6ed5f875, 0x087214f31674974d, - 0xfe35b77caab2ac69, 0x65d6538cc589bad9, 0x25174bb0bfdb7c22, 0xe2ae25979241d493, - 0x0359e221adf91501, 0xa99d114459f67c0b, 0x1d9642476428fc5a, 0xba412ff0166dcedf, - 0x782df8bada86dbc0, 0x638455627a754177, 0x7decfe4009d21fc4, 0xe4747bb76ffcdc8c, - 0x88181a9baec37747, 0xebd3e3e9e7c1c668, 0x186f014cfeea02a3, 0xe4ea9c583cf8f6c9, - 0x2c2836b08143625b, 0x423b2dbe98c75722, 0x058f650e9ee8103a, 0x76c8ef05a3bc6d3c, - 0x206550a78820c8b6, 0x9f6a3ac23e00cb2c, 0x37007a6433d4d107, 0xbc6f7111e9e811e6, - 0x456077076c19da90, 0x2899d37046960298, 0xb337dc76abbd3b50, 0xc07d96d9f7b1b3d3, - 0x7ebb8d5f41a8fa41, 0x33bcced96db014e1, 0x17c149ad5ed3e974, 0xf5b6abefae6d1f21, - 0xf4828cf0227491d4, 0xf2f36bd31caf108d, 0xaae76a1bcbd4f13d, 0xcc64372112092105, - 0x8d2ec50a3087f3b4, 0x3ea245c8a845a522, 0x5d192f998988881f, 0x420237427c1b03fd, - 0x11d608d5a36b78b2, 0x2b6f62038a61ec79, 0x8f40c6e6d2c3ca6a, 0x8263d245b489336c, - 0xd3e01de65f314199, 0x5ebe7cd6dfd69aef, 0xf17405a9311d6b72, 0xcfd399907d9f4ee8, - 0xe4a459d9f96a4d61, 0xfd46daff42ebfacf, 0x01af169ec019e39e, 0xc47c978743500620, - 0xf6b21eb316ddd39e, 0x926d7f3a56769482, 0x2d62c92e0b426662, 0xa0a376e788f67976, - 0x614d12f3ebf1c74f, 0x25451082d842cd99, 0xb0250cfc219f31d5, 0x86a79ab6d666e151, - 0x1bc145b422612dfa, 0x2070aefc2f7cd298, 0xd22de7d9bdc57814, 0x6193d07ea4df45cb, - 0xc4a4b8b2434fc892, 0x5d8a8d620f57061c, 0x80d0104116f12b3a, 0x326f327d7d91e207, - 0xd728320c75e5f7b8, 0xfe6a0b6f5d44c14a, 0x2892486de709c625, 0x91ef2bc668bb0cbc, - 0xa84536177e6a9787, 0x96cbb8cc93383dd6, 0xccd7dcd65655e697, 0xac700e390c39072f, - 0x925f597642e05435, 0x5ffb06b7f900e947, 0x6cc62ef16ee00425, 0xd8440bb44f93f61a, - 0x51e2a0482fcadcfd, 0x3e2a1a9fa47d5ca2, 0xceb8b19883969df2, 0x46521082bc8356b8, - 0xec1f133865f3fe17, 0xc1f49cabb4eadb07, 0x56d3462e5bf36223, 0xbe8aff1000be3842, - 0x7394d80158fba64f, 0x8e1adf811f76db99, 0x42f8a2d073ae11eb, 0x45f44e93ee82abda, - 0x4cdfbfe1a0fa4e88, 0x204d157766c0cda0, 0x96e7ef6eac7a7489, 0xb66c240b5120aa04, - 0x459dd52e1de2c7da, 0x1148787ae5f0ace5, 0x9a094363c877e3ad, 0xef9d3dc3fb791338, - 0xdc3fededdac9a320, 0x0a519b95ffff95f7, 0x6609cfe87b68080d, 0x62959e94ad519835, - 0xb7c449c9bf8afa9e, 0x11f209063d4507ac, 0x0bc42e57bedd2e37, 0x671d39b96ceecefb, - 0x044527eb7b963fa9, 0x48cafa134788633c, 0x4477cb8fff7ca10b, 0xde5defca1a75ac55, - 0x17f64da2dca59ece, 0x40e45455b805097f, 0xa07a0f46a70b5e7d, 0x818b43fb1c043e80, - 0xa6fd8f6ce731a89b, 0x6fc862c588e31d0e, 0x508e1ffaacf2aeec, 0xf8672dfa2d5adfed, - 0xb15b0337c2d149be, 0x346f3388b9d88bb0, 0x70a366c2303875bc, 0x9723d3339afac94e, - 0xfe29f56c84cfac53, 0x5b499c27bbcae71f, 0xa394db10b60e6c8f, 0x2916ca59cc9f94fa, - 0x25d62f78d10b0ba9, 0xf2e0182e07673ed5, 0xd2f79ad35d80dac8, 0xea98b6de39536144, - 0xa183325edccdda7d, 0xb5ca8984e6a0e4fb, 0xbd1c5938f60adb51, 0x9168e8a373882554, - 0x063302444bd90ad5, 0x4c6034d1f1f6deff, 0x1a5d12d9cec2b480, 0xa4cb87ffa55d1547, - 0x4c91b0904d9959b6, 0x86dc657cbd189900, 0x5647a3c5db24884d, 0x20eb2435e06ed220, - 0xb117ed2935a7b3e1, 0x8ac51bc96760a52d, 0x2b2b1843815e6dbd, 0xe7a760726e738442, - 0xe6cf7a538acd37de, 0xb341fe5f75c487d8, 0xcb4bc0b40a49c051, 0x335e7ec90fd44e40, - 0x549e0eba3f9fca5a, 0x11dfccdd4b2f2135, 0x03b1f8493eb1c4ab, 0xc43ac1f92f3bea12, - 0x7207b9dbd930c04b, 0xe83be577d9ea3b97, 0x2db2c5ec823c5d69, 0x25052ca079416eb8, - 0x7e012ec69c8d306b, 0xe0df3d59d012d1fd, 0x4192e8784437fb9a, 0x234c3ccd5b6fa6f6, - 0x61763488625c823e, 0x5b530687d30a3ac6, 0xddf571a200d5d63e, 0x5c5d6d01dac775a3, - 0x071d25bf0ac833fd, 0xac99778eab986a11, 0xe22baeafae9b7376, 0x97b3d35a0cfb8745, - 0x16533bb6fdd41252, 0x5ffeb31bf3533fab, 0x5b3db782a5b8b24e, 0xa0ffea0041542ede, - 0x9b41bfa159e4a5e9, 0x97e3217feab1c2f4, 0x47d7a8c4a933ef6e, 0x5155c6751b10c6f8, - 0x046cadc9fef5743d, 0xbc0456f5eb89fc86, 0xdd3131aa5b6ec496, 0xc9d6a3f206f05aa8, - 0x18623f8f954cb9df, 0x3b8f9c5f3d1ed9b5, 0xc5a8f5342f06926a, 0xe57908213e457ab7, - 0x5c487de33d56f8e5, 0x54f8b13b9064b6b8, 0x00264d60937363f4, 0xee3ddeb08374c5f9, - 0xc7e6c206f411a3f8, 0x1ddf9fcd745b384b, 0x02ef5819d563118a, 0x86d53ad0cc473760, - 0x47bbb48575c626e6, 0xacbb8c53b8e4c610, 0xfe7960bfdedcf4f8, 0x2956a8e2b9dd1d3b, - 0x64cee52115904e65, 0x4e06e12dc958b26e, 0x349bd01196399be1, 0x7e5b77c9d0892c44, - 0xf68f625588054eb0, 0xc094ad20da36bce8, 0x6d9327f0703e1365, 0x946ff5c7973ea1b9, - 0x5d5a19ce90078f67, 0x58d9ded6888d07dc, 0x0caa982ce77ce912, 0x7d7ffef8672f06ee, - 0x26d7ac5f4918c318, 0x5ff6d11ea8b149b8, 0x422745f95d62c60b, 0x7519f778bed8c1c0, - 0x4149149987bf2535, 0x391b7c6aace48d53, 0x01c0bce3a014ab96, 0x596cbabf34195dfb, - 0xf8ce111179bd8bcd, 0xa777b4d75877272f, 0x093befdb47ddfd4e, 0x7149856f41a9931c, - 0xa0935cacfff4bd7d, 0xc0fa2db6bdc010dd, 0x91def746f9e76fff, 0x65b13ddae95d6231, - 0x2c772a1485bd37d9, 0xcc9d721e369d17fc, 0xcac523860f42618b, 0xeb9f8416c18ba2ac, - 0x027c89351a77021e, 0xb13f8abe9309854b, 0x7ee182fa982e25b3, 0xbb49604afb3121cc, - 0xb6040eb7c4123a2d, 0xd7e52e5cdc2e2bd1, 0xa0edad9f12f5223d, 0xb9c50bb75ab2dae8, - 0x747f7ded6d50dfe6, 0x708954d8088caba8, 0xcf3726075efb1547, 0x91b5316a2e4cae0d, - 0xe413b23e66347117, 0xbce2d2759201dc51, 0x05a9d7785c8ebd4b, 0x6b7c91bf45b24d6b, - 0xd70c42c45d1bb405, 0x4d00c6c3290362d5, 0xa4db882996f01bd7, 0x2ce82c1d8eef8832, - 0x3582aeb9aea7fbfd, 0xb29ddcd2cb0d40e4, 0x9761a3ce2259256f, 0x0d54d64a1e2df8a4, - 0xcb88706194fa177f, 0x586cfe4548edde87, 0x77af635bc8757a2b, 0xc8fc480b27d59ac7, - 0xb65632aaf3b8788f, 0x71b607a1e8b429cf, 0x96dd73fc07fff3fe, 0x961c62fc08815bd4, - 0x32a56d92cec14bd7, 0x28c83b02f1b9782e, 0x6fd8d183d56b6002, 0xc7fb5f7f4ca8c2fa, - 0x16fcbea13080517c, 0xdf2defd02860c8c4, 0xf2f212f0818cdd03, 0x551e1f1073641996, - 0xae8f2a08b501804c, 0xf29d4589b75c6601, 0x8190701da1fe009f, 0xe6468515aa457e8d, - 0x8549cb525b366fec, 0xa34280f850b06a54, 0x72455ab8219eb451, 0xf0eb00528aabe907, - 0x409169ce48cfffbc, 0x1f02650200e492ed, 0x5856e540ebe567f1, 0x3ea9efceba507a73, - 0x8d40538f8be37a4e, 0x01dc014d0501174c, 0x550e9c3fee9cbea2, 0x5141e4a22f54a38a, - 0x3d5134f09bde648e, 0x4376958a1b39eba3, 0x8fdc8e8ffd3bf584, 0x1d4f7fcba86b1b6f, - 0x8baf03d5f3895c74, 0x02468d5d474040d9, 0xa69e9c549377cc35, 0x99cefe759ee9254b, - 0x670772236ddf5e09, 0x8419f6e8c15a913f, 0xc8111c51d2897c7c, 0x47fed3c87e5e830a, - 0xd57b2c2597389496, 0x72e03d9dd7c0432f, 0x9ddfb663cb225a33, 0x644407622a42a19b, - 0x36f1c858a8191ee9, 0x5b55eb9a39c79eb7, 0x205512efc9853ac1, 0x61638bbd871baa91, - 0xc2c5a321972e06ea, 0xc2d50ebb51ccef81, 0x00a8cdc815b2002d, 0x35e39c3d85bb220f, - 0xf7cc216ccebdabc7, 0x034d730c94c6ff38, 0xe68d4736576f5424, 0x657f20ae289cb61a, - 0x7587dbeb4543d01e, 0x4ac370af3aa7a2b8, 0x0608fb7b8b38b542, 0xf3aada3f9f9d4ba6, - 0x16a3a0ade12bfd54, 0xa338857ce1c56ba0, 0xea7364c7d14dfafc, 0xb815bc525eaaf960, - 0xdcf48128ebf48ea1, 0x08c31f79e642f874, 0x96eb3af427796aba, 0xf18bc0fba1acbede, - 0x58a601fdcb5d1c1a, 0xec523f16292e0e80, 0xa38eacf0b885861a, 0xcc3b3d9854fd523b, - 0xf36f918352406b3d, 0x43400fd10760c884, 0x933a5c242df16d71, 0x80c922ce4d62c945, - 0xe8f049073d0403fc, 0x4178783a4d8b1a82, 0x90dc8f6ae6aa23ac, 0x54a9e1b221f0905a, - 0x45ad3abb539bd4e6, 0x66d388a54f923a17, 0x6c4f7cc6183bf8d6, 0x4bcb4ba3f6d7b5da, - 0x4e85c0f645bb886b, 0x19475ceabf6a96df, 0x9189ca576d87ac75, 0xcaf7fd78674c5b85, - 0xf007a6c6864ba792, 0x1b50b2b7974a5c36, 0xd951c27823338cc4, 0xa6cea6e5ae80291c, - 0x127675d7f6ac8ff9, 0xab587be3feda1cab, 0xa16a0333181d992f, 0x2a51c4eb17e3aa06, - 0x14c4d94e333d2a5d, 0x3dca8f448e89f225, 0x409d11a4d5257ee1, 0xf560e29949efa40c, - 0x869d8125b8b24ee1, 0xcf3284de7f4dec37, 0x84a0e1b90eb6b517, 0x55d708ccc7d01e0b, - 0xbe466bed445005c4, 0xc790a6a9ba5bd332, 0x3f262db07b7c9756, 0x13fe360480a7830c, - 0xd27d2bbd7df4d28d, 0x40ee0d80122be9b0, 0x04ee84519a7d3704, 0x14415cfa51e2052a, - 0x11c4c53bce9962c7, 0x6e85b45a40101948, 0x8dc219a91e3888ea, 0x4773b03be2516ed5, - 0x4904a0e3693e7b89, 0x3fbbe0b4d143bec2, 0x64ceab586f528006, 0xef22f2545e9d0b60, - 0x882f8241080b9895, 0x8ad1259eba0ae10b, 0x4db7830c3811f62a, 0x4f556432751e289b, - 0xf86d47e71a3fdfa1, 0x0b2f109ce30bc4e1, 0xe3fe65c18aeddbae, 0x0e8047ba359f1a80, - 0x49839edbdca3f7e6, 0xb131a101abd8ccf7, 0xbb403bf586c22340, 0x8fd364f6acb73ed9, - 0x840e9ce2110c44e3, 0x12f2650abb61eefe, 0x4a7ffc89e18b1c0b, 0x32fce4dedd44a7f7, - 0xa4958aa058687c16, 0xd1b031eafba424c5, 0xf49dedfb91512a01, 0xd987e938b0722694, - 0x001b841d8fe3b34b, 0x64df5fba0961780c, 0x66732ca2329ff525, 0x18735ef5cdfce09a, - 0x4597de07aa64a94f, 0x7bbddd549017eeef, 0x760597fd8d2ed4cf, 0xe3766b93e0d0ac84, - 0x21faa6c988d9aa98, 0x779ebb2b615a4e8a, 0x1dcf5a963461e524, 0xeca7f0dd5bfecd69, - 0x75e7888446c9f52e, 0x06dfb3ad640ba6cd, 0xa7ec5bdb9d7ef026, 0x89b0b2c86f106ec5, - 0xd97c364e226490fd, 0xd2af46e1ea09fc54, 0x8862d5a50b98f8ca, 0x4dd09b09544333d6, - 0x2a9f3e95f0215815, 0x5daa5e77756a3b10, 0x96cf900596833435, 0xaf10588115709a5b, - 0x4ee06b4cfd8fcf9a, 0xb234068040000222, 0x062933989e6d6eb5, 0xf8e240565da11cf9, - 0x660f25687ae4e535, 0x1141f1e1840ef44f, 0x6c8c4da79aeb5ca9, 0x83033dd75132b9d2, - 0x1b06bc2b72c6cd27, 0x2006360a1ed89ba0, 0x8bcb42a2a24e2ae2, 0x27dc556bd78fef02, - 0xf76553dd32fb13c2, 0xcc9bf0a27a440aa3, 0xd459cb73642186d6, 0x8d2b4f094ac320f5, - 0xe6f78bfb8aff5757, 0x45227df690929bcf, 0x834ac2cc4791c761, 0xce2af3ad01721f9c, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xeedca7e0c951d03a, 0x8c3a63c35a3a74c5, 0xfa8a42505e1aca76, 0x9e0b6262463270e7, - 0xbe5f2c0bbc622002, 0x8bcb0324db709e0e, 0x4dfa15cbf974028b, 0x91f6a4e258bd0908, - 0x39c2322b625cf89c, 0x81a6d586b71f7dc4, 0xb776f914608a477b, 0x7b1b066d9fd38da6, - 0x683d887edf2ad19f, 0x932feda5cd02ec7c, 0x39dc8db7ed59a6d1, 0x1cb57620678f4dfb, - 0xf213050bee21fe36, 0x4747fabea4738f11, 0x23b75fd0a4ea2ae4, 0x33551327e0715d43, - 0x05451167e0b948d5, 0xc12b5141c5bffd39, 0x70564fac026ce2f6, 0x1722a54441718a12, - 0xcfc3834ae99902e6, 0x1308d7f7f19fd04b, 0x04bea6c145ae0ee5, 0xccee8241a570c968, - 0x5294c488cb3b0bca, 0xa62c35eb08ae56e3, 0xf2d1591ceb391ed9, 0x282b3d507f2024be, - 0xa735799795f0a66b, 0x5ef6738449ead606, 0x193c3d5407959e22, 0x8b3834706b4ad5b8, - 0x319b1594dcafefe2, 0x2fbaa5cd2f58bbb7, 0x7fe46be9d546f0c5, 0x476ff0befaaec411, - 0x4bf095df325923a7, 0xdb8ad4bc0cf89f23, 0x42e832b62997c678, 0x2406b889805a4d2b, - 0x98c88904cdb09c7e, 0x0ec51b7a3fec5805, 0xe68070ccf539b26b, 0xfe73e57af33e3448, - 0x1119c5aa330e82b4, 0x493b287c20dad20f, 0x4e05506904754617, 0xa94349868339b93d, - 0x458ad7670acaecef, 0x772f0748b7254cef, 0x86354036a9a1bf40, 0xd6cf2ac9982573ae, - 0x23cddbf845a2813f, 0xe6d5b06f5a65211f, 0x4c73b3fc538e06ed, 0xc3373690b62a231c, - 0x04110e90d454550c, 0xc0cd5bb5479e4032, 0xa5d05b8c0ebd2db1, 0x956fd51a130cf00f, - 0xac1d51907f6751be, 0x8c49ef886759567a, 0x5a001a21694ac906, 0x6116187d67a7abfb, - 0x443494d942f9ca00, 0x948fd111db19db29, 0xd016c93d5bd34508, 0x44641b3e3d8cd06b, - 0x1a6d94a8b3e1571b, 0xd63cc3dbd6056d78, 0x3da0287568f7a771, 0xa06c041ecccccc34, - 0xcab283bf2dfab727, 0x09f64659271aa55c, 0xd125465e53f07efd, 0x9c113cb88f8ae022, - 0xa226c83f10155980, 0x538fadc1cd1724e4, 0xb9009dfb0311d9f9, 0x7b111d860f9096dd, - 0xcfce9c0bebe9e8d7, 0x9728bdc759f9a8e9, 0xd7e016f52f7a2549, 0xe7ba45de23b820cf, - 0x0c5b9051c29c819b, 0xe4d2542e5c3e0858, 0xc571ef54c864f690, 0x1fa83e7eb8c1eb18, - 0x1849e47b4b565700, 0x046614d04e7addd9, 0x1876a03d342d6971, 0x1180d1ab8c8e0a47, - 0xb9101255287a0bdc, 0x7f526e55c5f10d1d, 0x91167b055836ae8d, 0x677c0715bac0ae8b, - 0xc9e76104bf4a3734, 0x062646b4678a529d, 0x204e7f5b643b81d3, 0x6c847283a0f9bc1a, - 0xf8e68cb8d4464237, 0x00aff6fc7a134438, 0x0371c005388581b2, 0xc3ea62ecc39ca54b, - 0xe738af197b2825b7, 0xab0840b3a6cac25b, 0xfa9f3ab8c67808ed, 0xca4fb82d078d8a2a, - 0x25be15a56c7a6283, 0xbc1ad2ab28c3f86b, 0xee50972501c4b52f, 0x9d1745ff6cde957f, - 0x052d0ec30cfa7923, 0x0f69c5dae10380ad, 0x6fd6081ca097d00d, 0xee8dfe85babe2e09, - 0xb4d7449f88cdc5fe, 0xcf29d18e4e66171d, 0x556404519b37c8ee, 0x3bbb7d622176d876, - 0x545ef749cbfe19ad, 0x8caa7998c081dda0, 0xc3c961bb7c6c0708, 0x0806d5680dab22ce, - 0x6f47951c99190618, 0x661f469eb80b44f4, 0xf52ced8b05b59fc8, 0x608c961847a71320, - 0xec1300aa3a3a472b, 0xa4fda54a82d5b545, 0x049ee20ae96b6e43, 0xbaaebd683585ae5f, - 0xd7235f034febbfe1, 0xd81135a204422d83, 0xc83aea2248dcde2c, 0x3d7def57d1e02da1, - 0x63692f4f90110d2a, 0x7855033f4dbdaf34, 0xec26ef59de03eecd, 0x801c1c7db0a8467f, - 0x30e96fe14975f925, 0x4120b50d2695e98c, 0xf5ebcbef2360b166, 0x1f594ab777d4a7a5, - 0x34ba90b2aba42d03, 0x4467d0c6c3f9fe3a, 0x3fe794cf89c4fea5, 0x71b2acdbe872de43, - 0x158e424396f789fc, 0xbb01aa76f1729b66, 0xd04eaec6aa61f15d, 0xa618affc32f48127, - 0x9d98b124d223427a, 0x6df87fa6539bfaa2, 0xeba5c6f1adede360, 0xb55526ad159d94bb, - 0x7aba35d95db81a89, 0x0235c7975a847760, 0x1942ff51922470af, 0x96f2a54607e14882, - 0xed35cf97101055cd, 0xd12d5e2f1713c8ce, 0x552cf6757c6ead6c, 0x530bd7ecc0e5d4f6, - 0xf78d450698d2c5ab, 0x1d91b24edc0cbcc8, 0xb935b39163b88df8, 0xfcb0a9066e15f695, - 0xb91cd79b199aad88, 0x61b3b0c46a597610, 0x1c3ff19b69fcd728, 0xc245f87c8c733a2b, - 0x35c5faf1b9df0019, 0x6183cabe3728d4d6, 0x1fc88253ccc491fa, 0xf056f780e7349ecf, - 0x0f7a570aef71c143, 0x405576874b455c40, 0x05715375ba4cfae3, 0x0b70e4db263cdeb3, - 0xaa2953a9a0df767f, 0x1fc7d20ab046cd55, 0x5ce12921da648770, 0xbeaed8e67d1b60fc, - 0x8301d88886d910e7, 0xa56ad65789b2f0de, 0xdd0a56762fa93464, 0xb5beeac158e8c6f9, - 0x3ffd8c534ef8c327, 0xcb0a3826b5a8add9, 0x99c9cd658d10f2a4, 0x2a5760b13ad79b73, - 0x8fdefe73725d0fed, 0x05147743cf9f8f56, 0x0776f762e2e5898d, 0xa3fd095366bf2682, - 0xeac9d6b14da24bd5, 0x482e26613ae4fec6, 0xa86109b8a2eb4966, 0x789d457de61aa75a, - 0xcf10363d8926c0bd, 0x25d3ade5315962e1, 0x7da5e59de8ef9965, 0x03eaf354831897b6, - 0x0b6926bc526bb421, 0xb94beab00fe73ce7, 0x41b92de25f6ba13b, 0xcb781ab3e278f4e5, - 0x3f576c029a60cdaf, 0x454b07952701e9b6, 0x8e2a04764b5e9492, 0x5d15f0eaa3bc23f0, - 0x5d936e8a7a293056, 0xe5d3d1f4dea1ecca, 0x1c69082e7321904a, 0x04879264f0b8c1fd, - 0xc25edc138e526759, 0x7f266ac53d61e887, 0x6f3fecb7fd7479e0, 0x888c8a9c866c39af, - 0xf1bd55c8cd4c767d, 0x31594fce2f509caa, 0x65072bf074b41878, 0x5c6c884cfdfc8afa, - 0x3c62832e7c323d7e, 0x5efd8b019cb5e902, 0xeec6db2bdfe49052, 0x9b3b12789b54e1d9, - 0xef6442d0045a2422, 0xdc9f647d840a6795, 0xd805956f51c0904c, 0xb2265f8b0f255b00, - 0x528eb333c0d18f24, 0x550b165d3eedd9ad, 0xf1f50f6ad334bac1, 0xd0e02e4fdfe7b99a, - 0x9b833d3f6c6934a3, 0x5a30b168cf4e7e94, 0xc8669606b3e5797c, 0x223e2f3d55c98452, - 0x5d297547f60fa049, 0x80d1297610174683, 0x93e471ae279c4e17, 0x379610eae7171e8d, - 0x46edfe107be63eda, 0xd9ff77246253e46a, 0x601411085976302c, 0x15505db63eb6ba38, - 0x55e879d7c002180c, 0x77fd8e0d27c942d5, 0xec36abc90c9d8c40, 0x9179bc137385859d, - 0xaca981cc12043fb7, 0x9655e80a1611f4af, 0x924e54c771400ad6, 0x4b852cf0a9043cf9, - 0x6e14495be643a707, 0x9444e022daee757d, 0xb6013cdec879904f, 0x04c8649861909465, - 0xc28696e5675f5571, 0xec88526c947de72c, 0x2a185c5efe0bbe5b, 0x9095bf03f0f4f45e, - 0xd6994a228069461f, 0x0e2c6b10f6900418, 0xb6aee5b977c75a0c, 0xfe863c2d199ed4e4, - 0x83f6024fce5f78e6, 0x6275c174bd0ade4b, 0xd4a983cd5f8a5e23, 0x62c17ef4e9006ad1, - 0x689ceb571b996ead, 0xdab1c5caf08b595a, 0x49a2cf1a22921753, 0x1567664576587781, - 0x0bf1f5c29714c1fd, 0x396b4f84fcdad2b9, 0xa3c791b3054993ad, 0xe0ec8ba6171bd837, - 0x580439c27c94095a, 0x5bde22eb19cb519f, 0x6c63f3d7074148f0, 0x234bcccb5d8e3685, - 0xe2d8297abf1aeb46, 0x2fb89975e94882c3, 0x34fd8449def296a5, 0x40ef531ff0532f88, - 0x74c2936215c79631, 0x2bea03393b39d0d3, 0xde6d28f55a68015e, 0xdc9ad0c8b709c6a8, - 0xade1c8f2aa701a24, 0x187588d257b6b6d0, 0x44030ce46f9d0ae8, 0xa95749367447e64e, - 0x53ad95fd69f31d93, 0x9dcdaf159f3447fc, 0x1a2dd5f1e6f46453, 0xfa625ec697ee9e6c, - 0xd2cd0d906f71b0c3, 0xdbc1a14eb5430c79, 0xb9b3f1125daa1389, 0x971c9eed96c94700, - 0xb731e271bcdafdc1, 0x6f7c5123e3f9b76c, 0x3332081be60f26ce, 0xbf0b21f05100c86f, - 0x145838fe97158f18, 0x0b476445d7c99fc7, 0x8aae6516078c6b3f, 0x49d1202e75a47892, - 0x19167ecbd5675bc0, 0x28c6d8bf7be63064, 0x07c0876dd0e5d4c2, 0x9901d9bb0475ffa1, - 0xa4b245bc7e67acb6, 0x77d1b38a3cd33147, 0xb3db1c30e10d87a6, 0x20f626f1af66a562, - 0xf7a03f4583d5b299, 0x5c6b4bd3a83718e2, 0x826e63d3a60fa432, 0xbac7825b83995590, - 0x4370af144fd59cc8, 0x718965e31b440927, 0x25894644dd0ec107, 0x71609e626d0bbc25, - 0x23de302c59a13b66, 0xbcec30b40cad1f96, 0xd8a0f4c04171fac1, 0x5c13531581cf313b, - 0xd93b39fe92138f7e, 0xa032afd46848a759, 0x5040f30a775d1d09, 0x715861a47214ca47, - 0x614f7d6f3b54636a, 0x58cecc1be72dae12, 0x27b61d27ae455ce3, 0xdf9fb8ae5a3eb273, - 0xd1454276a1cb05c2, 0xbab191df3e98f797, 0x35684119f85c2482, 0xd241d81e5744408e, - 0x5b9fbd9c5169ff7a, 0x9646038bea34d844, 0x098505f7c378ce9c, 0x6050399903151a93, - 0x47958b99b21a2ee4, 0xf2661e3533608799, 0x0ba5ebb2dfda4358, 0xf8fbe9ad66652fc6, - 0xf5f483584e94f501, 0xe3503fc5e95479fb, 0x1512fc47f82b7a4d, 0xaed3645653345f51, - 0x321b73054c99a5f1, 0x4a2ee1dcec34b3df, 0x43cbce287fbbb582, 0xf8f98e28fdd88ce1, - 0x4fab29f4e6d7a5b4, 0x264395556cb35f0c, 0x0acf890a5017554d, 0x7aa7dc37c13b1848, - 0x2371f1608ba74a63, 0x903fec755e0aee0f, 0x009fe6c15ec4e0a2, 0x824de33c89ee04fa, - 0x46254790c0354ac9, 0xa9f370a62058e8bc, 0x977d768da83afbd3, 0x42d7f4b8d680f84c, - 0x4efa382ac36866bf, 0x41fcd288ca22fbe2, 0x7d9875c548bd222e, 0xc97cb3762ee57adc, - 0xf530770f44070480, 0x1ab2560990052473, 0x3b5fa3ecc1d933d0, 0x93cc208bed0aaf86, - 0x9c39e3d2a7541ad1, 0x5b093106750a5076, 0x67dc7bb4b7361406, 0x53301bd97e564d39, - 0x17d108cdd69ce79a, 0x434eca4b7bda6429, 0x2a4460d967b771d8, 0x0d7817f4662a20e3, - 0x83622d96686c8916, 0x3e917d557d9232d3, 0x213070ddb0829c1e, 0x4e7d79faed8a625a, - 0x7e9e796feabaf30c, 0xf179d8630c7482e9, 0x234d96985e03b774, 0xf4f427c41d28b5b9, - 0xf17fd9da735417ae, 0xbd2282e466cfed71, 0x95a54f8ac0e815f2, 0x7432918e790f17e9, - 0xe9d1542b88091651, 0x923042039dda15a9, 0x2a3a2854d024a9f2, 0x9c02e5ad6e335b22, - 0x10a9d427f6c85f6b, 0xd254826ee7d89670, 0x8a7a681f1ce57917, 0xd23620e65b1cf653, - 0x63e471182233bbd2, 0x86d584dc12f61090, 0xea6953a7f6cc73b8, 0xec78d41058948340, - 0x0c57022c8580ff10, 0x6e8814354d096c00, 0x9630a01139666f3f, 0x9866366fcc49b3bd, - 0x990abf7024e1077d, 0xc67db297519b6601, 0xde1590e766eac3fd, 0x3d9f4fc409db0190, - 0x1d493170c2edda8a, 0xd38dc9c314a5fa62, 0x8b20bdf4fe140929, 0x96e35100c4cccf48, - 0x00667b36850c4a85, 0x7a57c842977eca84, 0xa7cf6d0c0f6518cd, 0xa4a438711a7ac38f, - 0x9013c6c7d9a59c74, 0x28f0d954f072d8ec, 0x9281955ae6b21293, 0xb49438cc63f15971, - 0x5c31a2eee5f672d3, 0x060b3deca98e7dcd, 0x6908830065b682d7, 0x61a8e0567c3bfab9, - 0x3dffc0512a3b734f, 0xdccdf959c9b09edb, 0xd1c6dc1c01a0e87c, 0x610dae53161e0812, - 0x4d2d0edc18dffd1e, 0x70c1094c6d2668be, 0x78228c7595e3408e, 0x830690cb001ece34, - 0x294593f1fe29986f, 0x97750956f6a473c4, 0x65145dc2affe6846, 0x89677373d2f5cf43, - 0xa1868dda2542c805, 0x5f1e8ebcc92bc7a3, 0xec360b46ba8bb93d, 0xe04c7d42453244a0, - 0x1b39f0a4cada1f23, 0xac6927cef5d61d23, 0x158685c1973f929d, 0x22a01a3db8903bcd, - 0x21ef5669f65d2472, 0xdec89fa413e567e7, 0x9916f6cd1bdecf32, 0x04351f33496c2296, - 0x049ffc3f1407dfbb, 0x92fcf36cf3f1e705, 0xcd2319fd6046ca0f, 0x3b30f68c1b0ba08d, - 0x2483ed36c613c02c, 0x915ad00983df3a7a, 0x68bfb60ca70a3cb5, 0xd8b233a32eebf3ca, - 0x91019006ade73bd7, 0xc738f45ed144e990, 0xa98f0082d26c0d4a, 0xf848d9df9a444dce, - 0x0f4ee801e85e01d7, 0x4cfbeb3fd576e939, 0xaaad7bcf23d85984, 0x45daa3a8c693d0fa, - 0xed5b9a172e495e35, 0x76d818e7ded6b691, 0xc821b413eaf96497, 0x4b9ce6e6fc87a184, - 0x6a48a853d344f0a7, 0x94028ba1588eba23, 0x3d01f2fd2107654d, 0xfbeb502b13e7175c, - 0x8fe7521cc21dc1d5, 0x5ff6c77b2251eed1, 0xb1908c4f73fa9d06, 0x8de670b8040f5f1e, - 0x03f6d066065e1e69, 0x74affb51c2b74761, 0x0a569d7c6c31e689, 0xcb311c8d24ec6662, - 0x83e00e002e69dd49, 0x5f6d8a7a8d161c96, 0xc7581248e7a3a423, 0x73c206d05f1ce563, - 0xaf20860f7daf4564, 0x64aa0da623f08bf9, 0x89f825758aba0b9e, 0xcb5633ff07db796b, - 0x609674eeb2a1e504, 0x2cd638565f222b5c, 0x4af04e8da85a09e6, 0x994413b89736c693, - 0x42bcb97e73a04713, 0xd61131b54f09e71e, 0x3206e5a25da6b730, 0xc4f59f4751f3e4c8, - 0xf3f99500de1fca73, 0xabbd971696a2fa86, 0x7ebbb816255f434a, 0x9420e3cb88e90040, - 0x4956f44faf60594a, 0x01e35c097df95500, 0x301b829dcc499373, 0x694cd2176c87c8f0, - 0xd1af81f166d232d4, 0x9e07e5039a5d8008, 0xa5c1c4b6a2f22bca, 0x39bbc86346e5eef3, - 0xed19a035b46b75d5, 0xd302602a109cca35, 0xa62f8859a1692e8e, 0x1b94f9622c7eedb6, - 0xfbed00befc0b0b69, 0xf74d8abbd92bd28f, 0x0f65aa3fed0e01ff, 0xd025f3e2e9ae4bf3, - 0x4e86522ae27336bc, 0x19e7b4b5f426b678, 0x5c474b6fbd411bb9, 0x0b25f842865a3900, - 0xbe10802a2f6985fa, 0xe9c896f6d31f1b04, 0xc815edcb36a8647a, 0xe795534ccec7bc5b, - 0xad8f53989fb55458, 0x176d6c3fc36305ea, 0x604ac92d29021d3d, 0x3b397899b8295249, - 0x3f5421792baddcf8, 0x6e2487532ae82053, 0xe32ca83fb42c54dc, 0x12a5a565b2693842, - 0xdf5bb9c2675c2818, 0x84c049536e24e16a, 0xc693bba9162f56b1, 0xab89ad7daa4fd305, - 0x225cfc06cd31db7e, 0x38c669dda9bbde8d, 0x162a837cd0189451, 0xb7b67b2a61230abe, - 0xd2525f7f077d1b65, 0xb1731ea0f0f55899, 0xfe141c3f4516af80, 0xf13f04dc611f144d, - 0x4ba25c5196746647, 0x2c5d728666c32f4a, 0x0910d64f0593f13a, 0x74fd544879fdf502, - 0x572381a1b4dcdd3e, 0xf42a914c1abe9664, 0x0fa9e4cb9ce67f0d, 0x52b58781b3fb603e, - 0x71520cad1db65117, 0xbf9f981b47935256, 0xaabd3917dd7192c2, 0x2fc98f8310b3b1e6, - 0x763ecf79485a7e7a, 0x2e9184d425436183, 0x798b15b456b24f10, 0xc50f0f553f211a38, - 0xd31a2e17c57d21a9, 0x9d359cb7034dc79f, 0xe0d4d2c6788232f2, 0xe109e83331e89b15, - 0x2db0848c8f42c145, 0x69e209a189669a32, 0xd6b238ceb614f9fe, 0x32780472f981db53, - 0xa04e14afb47d4c31, 0xfa0bdf4c63ae4b87, 0x19016376ab9be09e, 0xb44e9d058b95b62a, - 0xe5d93873934275a4, 0xfc3ed9a1c8c4887d, 0x0654b73ce23e0614, 0x14540550d8dc3961, - 0x95e9a7c132aecaaa, 0x823cc14150348b0d, 0x11afc75053a0e977, 0xbce93af8e490f0cd, - 0xb65330d69b4337f9, 0x5be17a4cfda9a219, 0x85d2599684eba638, 0x63d0fe0ca7b6979e, - 0xa527a5daf110958b, 0x824f38ca7f3026fd, 0x229217507f26346d, 0xb4cf917bb37c2303, - 0x5b2ab24e47560abc, 0x3d3c6c0ee654554f, 0x5128867c9b5adbf9, 0x07d61123b8afc4bc, - 0x5ce8b94eb5063afd, 0xc6a6ceed33496a13, 0x1a6bdfacc9e25b36, 0x6e5a57c5d3067bf7, - 0x036f32cb0e961107, 0x319a4a1cef386689, 0x8d23d0a98654761a, 0x5f4d928fb9de303c, - 0x4fd9348989fe56d1, 0x8bf81ddcc8b68636, 0x840727876f78152a, 0x7af41e9ebf8733f2, - 0x268219578b5dc867, 0x968796ad3a6d0826, 0x998a02a35a0b2f30, 0x623ed046e001f561, - 0x0ff26d66cd9edeb1, 0xd1d59598059ba367, 0x57aed45d052aa0b0, 0x4013d3f12cca9322, - 0x8f84ab1f762ccca1, 0x55d32a97ef39e2dc, 0xb561727f67d7491c, 0x16f112500cd1dd83, - 0xcb3e9a2480036b18, 0x1c85e3a9cfcbcb06, 0x7e6b4a5e720569f6, 0x3aa960beac17014a, - 0x1e07efe5ed7583ce, 0x6f37e1f909f7ad7e, 0xdc3510a4e3ff2bb6, 0x62caf8d624891132, - 0xb6b82d4d5bf20b0e, 0x0718d04ebfa159d6, 0x8fb03d6abdc3c502, 0x2489203e90249ca2, - 0xd65e81d74f1eda6c, 0x0036c8583360bc66, 0xde20fb216d94ea20, 0x5f88e95becb5121a, - 0xb35aab96e20b0ef3, 0x95c467af0f1c18cb, 0x327a852397ad9ef7, 0x16879d7b98361c52, - 0x0ec50bc3457cfaf4, 0x48d6017ef0f013cc, 0xf962704ab5e4e56c, 0x5340e80c374b0d58, - 0x3b3a8659127cab35, 0x09125ffa12f7ec79, 0x08a9ad56710a76ba, 0x6483ddd616389fca, - 0x54645c1924316d5f, 0x729f22451934063e, 0xa7675de5b111d262, 0xbd49f4fb9ed52be3, - 0xbabb549481ac64ce, 0x29f77467e02d4f8c, 0x2c7fb915e461d6b2, 0x2854b544b2ab3da3, - 0x52a817c81ed5eca0, 0xa747859e1e88ca60, 0x2fa81d34d023fbe3, 0xb9dfcd89947b98f3, - 0x4dc60be5ff88a7b6, 0xd5ddeb898454e293, 0x6f66adf6de5c4c9e, 0xc79e508ed3bba1d6, - 0x465e1a49940d8363, 0x8dfe3f6cccb8fd2f, 0xa9ee7869d85e52ac, 0x4ed79e1efad22402, - 0xb355182660c67f4e, 0xca4897da6df223eb, 0xf1c345157a656ee2, 0xb206e81c824f5ea7, - 0x29b8c040f071a234, 0xda174d83dd78a82e, 0xb5fa8a0b95ddf173, 0xdeee03b0a6a1d4b6, - 0xc1106005f8199f51, 0xbf4dc0cbd746b65c, 0x916d314a0a3f1507, 0x5520e7bd2e720374, - 0x9d5ae0058396f203, 0x887387852c63c969, 0xe5ab872c3ef90004, 0xd8d5fb35585f1928, - 0x74dfdb82146c69da, 0x6751b673d3225d6f, 0xf2ac98801b51d515, 0x01bf64b441ad3cd9, - 0x6f1a90518be70bdd, 0x81937c5c9fbc78e4, 0x6b6b66a9c322c6d2, 0x5923d45d78f2b1f6, - 0x71a153dea87ee760, 0x7633cf6769c86aa0, 0x2ffe0c18a2f71d3b, 0xf141d6ff24794911, - 0x606bc90e3daa0cd6, 0x70c6b3e8e41912e1, 0x184d898657e98393, 0x98a3798770e1d251, - 0x98acb0a3f9ffbccf, 0x9a7d28091c1a6565, 0x4f58611755cf196c, 0x3d3c0d93dd24d551, - 0x696cdca52f629ad5, 0xdb92ef6a360779db, 0xe55fbb8b3e05d929, 0xfeb5de5e075d1ef7, - 0x5082480214462f50, 0x8fb42d9f5f8f7f6c, 0xad816d33e83722e4, 0x299ac652f5c6dfd4, - 0xc09f941425e8673a, 0x4a1a70417629ed0d, 0xc06d6f2088feebbb, 0x16e62194b0edcc1a, - 0x9ff09309d5927e98, 0x5dfc6122fb7e6e3b, 0x1323ea44c789a8f2, 0x03d23b37b7f4103b, - 0x7ec3772172fed47a, 0x661df689e073abae, 0xdfa436d5db06a55e, 0xa10c8dd313c66ffe, - 0x93d7596342ba47a4, 0x53ba9f97113c8fe9, 0x608a5562d2fef37d, 0x4a2910ff78c914cf, - 0x6f503f67fb72d363, 0xaddfb8f733b41f23, 0x43a136d5eb1d9e31, 0xdbb9094cc9208ed7, - 0xd521786507fc7464, 0x1d8c66346a280b77, 0xa6299b261217f3d6, 0x8b8e5e6bc1bf7b51, - 0xf10c90ec08eb37cb, 0xc710dc8bc2e454fd, 0xae6e231db40bd41d, 0x4abd666261b4658b, - 0x4f4df38a427b71aa, 0x15db07f309b2aced, 0x8b2621e212752172, 0x331df8b78e724176, - 0x7bd6d185c7ab30c1, 0x6c0508c3eb628f4e, 0x3c9091ebe141ef39, 0xc74218f8104a1bd3, - 0x82c41768a57cee94, 0xa872e76d1983054c, 0x4061ca7cc66b8a1f, 0x02775e41d1afadbb, - 0x9754414efe90675e, 0xdbb16d9430acee66, 0x97698c3898fb3ba1, 0xda5583297383a667, - 0x8caf8fcd27993c66, 0xaeb42ce539c12063, 0xb987751743316e92, 0x3ba61194a1b90e0a, - 0xd7c771b0594a75c7, 0x311cd3c95965a477, 0xb71fa9c317be67af, 0xbe3fe4cf49cfc4d0, - 0xdca3b5968c565d52, 0x2734760dcc663ca6, 0xf850f9f76ce89c6a, 0xda6de7879a382afc, - 0x8ea94bd2bfddaecc, 0xe2d109cc3194f01e, 0x13f51666b74a74bb, 0x5abb96cb14de095c, - 0x9778177974021c0b, 0xb1b31ed058d51815, 0xb7811e965cfcf017, 0x3ee104e5bc9e392a, - 0x4cc01b66eb138fae, 0x33e2569061ba01b2, 0x22671083924e1154, 0xeba6a9b2843a171c, - 0x97771401e7c383de, 0x03f119c6d32895ae, 0xd8cf9e1670f45c0f, 0x581bf7c4c827d53a, - 0x37929df9916e1d3e, 0x0d1cdbc872c54143, 0x4f5a0b0e957667fb, 0xa1f7ee131bdc9476, - 0xa99d5c7f2791c16a, 0x162df6f2ddf44229, 0x7f0a15723ee6b3e5, 0xb89f4d949a4c5a29, - 0x97cfee78f67d9d16, 0xbf95d34d502994a9, 0xf9043171bfa8c476, 0xafcbf7291a637b15, - 0xa44b9953bfc57766, 0x558889c7c822708d, 0xb1094c1c730da66a, 0xf741eed8519766d1, - 0x6c2c899a86b9f01e, 0x5bdb6a47098a63fb, 0x4a2fc524f9d01cb4, 0x391bfb85f7104bb6, - 0x0394ca7f694bfbc1, 0x2082d059e0222a21, 0x33096710a5f35f01, 0xbfa1f455b915dfc3, - 0x5d2213054b495c2c, 0xab47a5129c2706b9, 0xdcbbd5464ac34c37, 0x27988c6e5da28bef, - 0x4a911d95a831b1cc, 0x1dcf408e1d04631d, 0x9ae718519067cf9c, 0x866df4e1f5f2cce7, - 0xcea33a3eb7f4d7bf, 0xd38ef1feb64c1c46, 0x4f049175bf92b6b0, 0xce102b429ebfd959, - 0xb54b18f5ddc40a16, 0x72fc7511b987e566, 0x59f78f9e674cc0bf, 0x83b7892549cde1bc, - 0xaa4816154824aefb, 0x86009e4bad967cfc, 0x48b01b746ec74a62, 0xa62b949a5a9e1881, - 0x6774930b2885bd13, 0xfe90c343acc89e3a, 0xcf995e0de019327b, 0xf7d8fca81e039490, - 0xd9a8e72d5840b430, 0x85554058b69fd547, 0x521e0ebfa0d5cecf, 0xbfb087bdec1024c0, - 0x4e1624b38ad382c5, 0x7295b35c6d54d976, 0x6af2aab549212009, 0xdfbf0c79d082b7e7, - 0x150cad644f708045, 0xcee3efcdb73d8cc2, 0x58e41853c844023e, 0xb43e36eac83a359e, - 0x7f501ca545415131, 0xac0ec5593f4352cd, 0x1da892e6b2bf03f6, 0xed071367713f1687, - 0x03ee02fe7191b0b0, 0xbbeea80f84c0b7f9, 0x15489036a4c7502b, 0xc1b4cfd0a689f4e5, - 0x737c82ea8f986886, 0xc6c314dcfa6aa07e, 0x4404477fb3039e5d, 0x83bd77e5d0e6a2d7, - 0x35689dcaf811b4fd, 0x2c49607406d0e301, 0xa3cb22ce202e618f, 0xd4fa03c35e15cd3d, - 0xbef0315d95556403, 0x78aae6a13ae6c74a, 0xc18f1b0e98a32f5c, 0xc95644d4685abdd0, - 0xd173dc63f67333b1, 0x4ddf840e77184811, 0x41766b763dcfc03d, 0xef2d5b5dc6287ac6, - 0xce131e09945630fd, 0x45d8b4f64a074ff0, 0x9db9b492858a73e2, 0x8a18661007eed33c, - 0x3c13bf69d0129ab0, 0x5aa172168859d905, 0x6f9ae89e7ffd2fa7, 0x68489ac0e049cb5a, - 0xa08f58469ac6f6ab, 0x1205233c4fff0b0c, 0x518d8c2f60e72b9b, 0x3bbcc200c202c7be, - 0x185d7e1843bf49fe, 0x3f3c410fb8af9350, 0xa8268036cff1d81a, 0xb8474f46cfbc6b8c, - 0xdeff0d2828966f89, 0x8b696bcfdbfef8d4, 0x27ca60bdbe31dbce, 0xc98de7283e071089, - 0xeda065307f8c59c0, 0x083809b1ca74fb30, 0xa9ec33b810f54806, 0xe42f82f83e602003, - 0x2ad61666d7a323f0, 0x16276307cf987737, 0x43605ec58cce8bbc, 0xc9d1b6f2fab5b074, - 0x98cfbb789f690a76, 0xed7f9aa0793b8073, 0x838a8eb62319c1ee, 0xdbe1d3c0c7df14c7, - 0xcd2d9fb60576d36d, 0xa3219929a3f84c31, 0x9bf273f053e76922, 0xbd6da3f1cc3c00a6, - 0x1d9693ef2722001d, 0xf88991358f3e38af, 0xd69ce63a779a7a38, 0xd56e3859c923fe4a, - 0xcc908f819f7c94bd, 0x392e225ad433cc70, 0x19a2cc2d5bf7b419, 0x65ae962c83ec4530, - 0xd3c62b59d6ce908f, 0xe54a64a39155e773, 0x4992759134b76db5, 0xf47971a612e7d0a3, - 0xb31a2bc4dd4f36ee, 0xfc3f147d357cb36c, 0xf59c6e329735cdab, 0x6b948f479535add1, - 0x75f8136c448a5b4e, 0x2d24a08c87a5e25d, 0x5db4e1b6da209eb1, 0xf0b61946741d8480, - 0x80301c91820dc0da, 0x34ce9d4fce2b3142, 0x0c79eb766e38320b, 0xc84d59f5d9153064, - 0xf704522f67df3255, 0xfdb28ed3e5eeb5e4, 0x11e52dd58a9f8b89, 0x5f06352b79f16867, - 0xcb3624a0a073b527, 0x0882a16d3019e395, 0x1a045f7a9ab99f83, 0x08bb5ce34028de6b, - 0x58c7439f9c12ce5a, 0xfc4a47ac96f98221, 0xb993b660a80bef06, 0xe986f25392855a9a, - 0x91e97efaef4ee77e, 0x339e3969d4df25b3, 0x6541b68f14e67aac, 0xaa576612d21c6235, - 0x909602d9a39a63c5, 0x3970f6c236df54f6, 0x7ed39295d15c858a, 0x8a36bfa9541078d4, - 0xbab433f16cd9ec51, 0xa3ee596446c059c9, 0x6001622b3db480cc, 0xb377fdd5ec92c01a, - 0xdb999a15aa758978, 0x66f97bb674f8069f, 0x0b69e28eee2aa7aa, 0x366587b5baa8a836, - 0x4f55bfdc9e03fad2, 0xe64aa89f664c6e9a, 0x900094edefc2dc83, 0xa752531dea2e55cb, - 0xde90b931990c3faf, 0xf2c88fc61241733c, 0xfbaed1b4f8cae927, 0xa4fc7a08ba1ac7e7, - 0x57c4b52b1532605b, 0xdd6d151d93420ace, 0x5094b2ab2ea36af8, 0x1cab5493ffaf08c1, - 0xb5f57f5a322f3bec, 0xfb00121c0e86e0c9, 0xf398f1ea352e6a23, 0x54145f197c95b418, - 0x1805a2c0d74e1c2d, 0xf8fd3a7e58cd8408, 0x5ee83d3381ac2202, 0x946855c0eae16a3c, - 0x128d8482e701c84c, 0xd71570e66ede12ce, 0xbc4c8075b5e44e6c, 0x9d1dd08ca1575940, - 0xe56fe27c8de09450, 0x2a1da8c15d9d971c, 0x143d415f2aede9d4, 0xe1bac1dae2b9189c, - 0x5ad7b29c6f7a4853, 0xb9ba71265253858b, 0x339105d5cc13b190, 0xfc8f69baa4fe2f7f, - 0xf871bd72a74a7e26, 0xc59be755aebe6343, 0xb1eb67f9d751c90e, 0xd816671c3c833cd9, - 0x69923c6d42b27b34, 0xf2afe5864c582b82, 0x9ea4ab8ca9addb06, 0xca8a33a96308d88e, - 0x9aaa39d49ed7927f, 0xd7fdc6e094ebb325, 0xe6e63e16a1014974, 0x6f2932f4da08e002, - 0x9a184908852f8495, 0xd087a2c0abba2d1b, 0x898b3675112a9259, 0x68ae771d493ee36d, - 0xaa10abaf62acd23c, 0xc253174ef5844ec2, 0xe058f1c95fb5551f, 0xcfa1c4dfe5a1bb4c, - 0xfddd03b28ab83808, 0xb42a2842345f5fc2, 0x6bfcaf3d7e2be9c5, 0x6a52542e7cec0502, - 0x08bc4bb0f58ac762, 0x108d462884add97b, 0xe0b21aadacc6d5ed, 0x1410c20b566b9c69, - 0x61ea45520230396c, 0x748f100c73dc1836, 0xb33bff810e1566b2, 0xabbc97e0b061d86b, - 0x61a3fd66f19af4b0, 0x431574cd597ea6b4, 0xc7c526fe183b72fd, 0x872facb8e070d346, - 0x29515670c32659ec, 0xd5e319a730ab1253, 0x822a0aa4acd3967f, 0xdc1c6d57500b15b5, - 0x430c440a1a8f0da6, 0xaa2ea0228f03334c, 0xd291c3442b39ef2b, 0xb5a590f4a3696316, - 0x3c8ee9586b63e23f, 0x6f66c6ee971eac36, 0xc386af2c79dfbe92, 0x052a5eb203f8e979, - 0xe08986f8cf8d712c, 0x0e7312c9ce83ef79, 0x4685715de68b9943, 0xcba54fd096d4fdfa, - 0xb114c600cddf6ca7, 0xec802fe022ef290e, 0x48cc89b96baf1405, 0x856fe3c273fd1ddc, - 0xc3f5a1345ea30221, 0xc79ddcf9ad98b32a, 0x9ef0b2ec67942c37, 0x5db6b9bf62d9ee39, - 0x18cf423fc2f183e4, 0xb42c0d925790b338, 0x909344c530b0511e, 0x2c31606eae65b05b, - 0x41b5f0d576434d83, 0x9d6d9fe0d8c91244, 0xd4f9b9d65849396b, 0x626a7654941810a6, - 0xf5ae2ba734ecd9b0, 0xe2fcc14528bbf66f, 0x36271a15ca6ac7f9, 0xb7caf9f39f1e08b1, - 0x7ce5080dcdfacb7d, 0x22d2ae93967675e0, 0x0b6b87f08c53d821, 0x9191eb2d67a3f775, - 0x5d81b8c604d5ed21, 0x8bec6abef22af58e, 0x17d9985759a7d1e7, 0x656ab558c584b803, - 0xf4cf334888ebed3a, 0x3aa42fc704339810, 0x3f9b1d479e7ea337, 0x8d0dd1ca03868cf9, - 0xaf641e168602d056, 0xbf972c5a2d378aad, 0x8c0a1664714ff859, 0x74f370e0a6185f27, - 0xc279189ba1d0391e, 0x8b1322fd07cc0709, 0xc6d4bf2fceeae1a5, 0x7f829806215af5a6, - 0x83ea1a5ce8fd47fe, 0x4338e0d7f07ccde6, 0xd42255ac454211e0, 0x5320b4fc47b37160, - 0x47ebb541b657d84f, 0x37c6b79f9bd885cb, 0x39542d935cf8dce1, 0xed3a4f960f08c63d, - 0xd2d10d23beac7d25, 0xff29153fa8e97250, 0x8f989cfde05f90a9, 0x3fdf0313b2ed64a8, - 0x48b1f8649cd4dbc6, 0xd98004fdd75ece8f, 0x693da7645b3d7752, 0x746ac6f281fa59b2, - 0x84648b7533b1d334, 0x27fe4619f11db66f, 0xaab9849cc1ecd61b, 0xb59264ef426064c9, - 0xefa788ebe8c1b7c2, 0x7f4e4aa02d827dce, 0x63e7dafadfb4e68c, 0x6f2fa99cab3b7756, - 0x093b73eba6d45fe8, 0x6bdb627f072ccde6, 0x158289e847bf3409, 0xcbeb4eb2144bf165, - 0xc096eb59e7176d4d, 0xa452efad7c48b107, 0x470c35fabc02da28, 0x711fe80571ff0583, - 0x9d4fbf024490f208, 0xcfe0def981b77204, 0xbc46b43161a37c75, 0x6f4955a97c62b4ba, - 0x67f99ef9e9c66453, 0x068663152bf57dbf, 0x59d2852af5ce7dda, 0x15ba63b14729a5f3, - 0x16f025fd5088ef73, 0x0f5d55c4126f2ee7, 0x2be56f25e364a8bd, 0x1ea53ece05bbef1c, - 0xcdbc8de2f35744fc, 0x7972ddc5943c0bed, 0xfebd6793ccb1615f, 0x9f855182a99651d4, - 0x37a3b078098483af, 0x670195f11bc92f6f, 0xc864d91c2ea75977, 0x909a3490db009aa0, - 0xa776b75c697b4f35, 0x3697c858549e1c78, 0x70b8d1f10f4433b7, 0xefe112f7cf636588, - 0xe0a0557c1566ac67, 0x4283d26df4eabfb9, 0x2c1ece6be1850f7a, 0x505ff9c375415500, - 0x6fb7be86432b5297, 0x1ef404984fd4bf52, 0x7bc081ab7e098c2f, 0x4cf4733b316baf7c, - 0xc3e14c983f08d70d, 0x180d5d6a5486ed7a, 0x4bbdb7924111dba4, 0x163552b72c662ca6, - 0x56c9fbbcac602d99, 0xab56141d66d078df, 0xc43941b6329faa15, 0x38c0dda0d241fdec, - 0xd9e7d8bdbb7a25f2, 0xe1d5c57102c129b5, 0x5abedf6daf898df3, 0xf951766570b8b7e0, - 0xeb44617c5aa319a1, 0x7c8b1c48a42ba6a2, 0x3575922d1983744a, 0x90a15e63f8859ce3, - 0x0ffc0c9a615f5fbc, 0x4d3d6fc9407943c2, 0x65990f8bd8b03fe8, 0xf4099781f6524543, - 0x5c1727b84cf9d98e, 0x40edff69dd1b3036, 0x49d2393afd383ffb, 0x5257f27e809a9d8e, - 0xa4676d56c9ab12fc, 0xf72d2b8bb1b84b22, 0x40d432fa00d31869, 0xeaf742c1f653dfbf, - 0x382e263397295694, 0x4a97b71b0c3cb02b, 0x3f7f28b621da4c7c, 0x1d5d0b35acc37a74, - 0x1008e34ad7b3589e, 0x2a0684134dd320c5, 0x3699d57b560bb633, 0x600e70cb36e573ee, - 0x93565693d204b289, 0x81158c61693e677d, 0x7c7df1e72ff3ff51, 0x9a2f32101049b427, - 0x3fe9b312d26241aa, 0x14d11bd76bfc0102, 0xa3c6f40d1a6d05ab, 0x9cedd63eb4e1aeba, - 0x7ba51527a389f40f, 0x9b56f7492d1928e1, 0xbe98619d8142627d, 0x4a59f61439eb01e7, - 0xea0b445908c7a153, 0xd220210c666138df, 0x79d68ea400996442, 0xd0387186cc468759, - 0x97203bc1fed798a2, 0x2b71ea290942c128, 0x4002bb079034f2ce, 0x4829b3d8ce0091fd, - 0x8ffa064bfff0bb72, 0x80dd54ef9c1cbb92, 0xef934c45b23cd0db, 0xb3065f98fc7aa1a6, - 0x79aa883c2c4cf989, 0x0b1f281ee7dc2311, 0xc81d492554c3c0d6, 0x147108c737f00fe5, - 0xc2e39e19401e716f, 0x55c5bf20c72e8fe1, 0xe7cb803f6dd8c54a, 0x05b5b9a9aabdd52d, - 0xb543eddf0295d1d7, 0x8c483ace1f3d62f4, 0xb8af82ec97f8c6a5, 0xa3ed7ca97bd2d462, - 0xa200091a2b3c441b, 0xb7951aa443460f06, 0x0dddc5191280b1e1, 0xeda3a94fab249fb4, - 0xa5313c5e4fa3f278, 0x273e7fc64c568053, 0x0293bbf9ca4a208f, 0x8beb212c6580998d, - 0x2f38c061131caef5, 0x75ce1942cf6234bb, 0x3ae9ad33b53817fe, 0xb8a0d8f820bbb7c7, - 0x1eebe62a083b4177, 0xf5df93554ec4c065, 0xa9670e88e2ed8bf4, 0x52cd4ca465e3a1d9, - 0x65f3ad88fb62e131, 0xf6aa413892942cf1, 0xf3f87ef79d466e9e, 0x5d1a2f2130270cbe, - 0x26758076578e7bbe, 0x8e07824fbbbd19ee, 0x0ea72eb7f22e1f28, 0x719e48f5a5052dd0, - 0x534c78cf20c015e2, 0x0bec9d43b33042a9, 0x1bb9f25fd3e67866, 0x129806756b068737, - 0xad843a6b86a7dde0, 0xea5a72059adf7c00, 0x12834ec13423cb05, 0x7bd07bb6266b69bb, - 0x814e1962f6fbae04, 0x93f6fd8ceb51d8c7, 0x92309d88be07b2be, 0xb277d7f8d8768e36, - 0xf9ed8ab32ee6aa8c, 0xabff1260dec23d7e, 0xd751a13757709f17, 0x4d0b15868092e21c, - 0x6a8b8e130c099ec7, 0x5154c2ac8035dda8, 0x2e9ce6ca625fd4c8, 0x20f9c14f2b5341d3, - 0xdfd00390e363fc31, 0xa3bff3f27fcaa882, 0x8afa12e46cea34eb, 0x6d17162cd88fa97d, - 0x46dd1382ec452d84, 0xdd1108b44f660d23, 0x1781a147c7f95f15, 0x81d78c46edc0068c, - 0x2d2e4081f36be050, 0x5b9100ba921f30e7, 0x78eb71f06364c184, 0xcc4030d6bc56e93d, - 0x0d98def85ffc5265, 0x252ae6111d7d718b, 0x34805a3fc6ea2aba, 0xaa40cbb314dbd620, - 0x5015e2cdb9cc8b6d, 0x67ce2287b14ba54d, 0xf9c01a091808107a, 0x4cd8841e1a9cf36f, - 0x9f32d7f32e26b8c4, 0xaddbb5b199cd439b, 0x784e97d804a700de, 0x45e5730f8eae3b46, - 0xd599a15bf5e00c64, 0x6209588210388596, 0x3e4456701ceb2ce8, 0xe41a1ee48e045b32, - 0xc4760085b5c767a9, 0xbace70a4f7c79c8f, 0x229bcd695ce7d27f, 0x366e974495ff877c, - 0xa739b99f1998d192, 0x92cba1444fd1c02f, 0x76bf4afdfbd17d25, 0x9bd27487d02def3e, - 0x0363ee035986a7f8, 0x50e60d1a77074e10, 0x96f974bdc3501c74, 0xa98ea128e1febf5d, - 0xf9b4a7fcc1ecd7a7, 0xb92e054908486aec, 0x588fefd4bc828f1b, 0xcdb443cc321a5f72, - 0xc2f81ce712497f02, 0x2d4bcdca207c96e1, 0x3a0c089cda89e09d, 0x1e236d130a953f4f, - 0xffcba29786d519d8, 0x54d354fe19326abf, 0x7f9ab7cc916018dc, 0xc1679b60b4ccc58a, - 0x82256a4a9155f8fd, 0x8de35c7c3c5e5d4a, 0xe04c46ce141bf8c6, 0xce735fad2ddfeb2f, - 0x08a29884a7e57c83, 0xead838a967b57a86, 0xe920a402a9d07cd5, 0x14fe988497d280ff, - 0x232c06de2bd97b99, 0x5f3da36554ad309b, 0xba6e4d6b0ab1a6ad, 0xaaf6ae4e71cddc26, - 0x1efa1def28c60d60, 0x5cace05e0d74175c, 0x1ff7cf004a23d3a3, 0x8b0e6346e885cf86, - 0x73c3841830b9d951, 0xa621ed616e0452e6, 0x96cfc73b8e22787f, 0xdb95bbb26003785c, - 0x35d183a832119df1, 0xa5bbfc582ed91496, 0x6a307b7b4644e1f2, 0x0b155a0a00ef83f1, - 0x6d6b1df25ab36ed2, 0xba7bcac2bab4059f, 0x9cc1756f9a61959e, 0x526f3e0567a55bdf, - 0xc628890258f3d625, 0x6cdcaec39e4ad5f9, 0x9f8747028056f003, 0x6d203230f0679e7e, - 0x276d80416e38bcba, 0x5fe0c0d21c67fcff, 0x3f1454a1afe20d7c, 0x6a89922a9bf58194, - 0x988fdd0c791d7d4e, 0x21542dc0e89dffc0, 0x95640e0804d9bd42, 0xd293313d73c36243, - 0xd1eb5d623816a2fb, 0x49fd83aede3c4bec, 0xd85efdd79590e04d, 0xfdfab13d5eac8650, - 0x094993dc42f2a545, 0x1e747349426bb19f, 0x334e033d6f60df7f, 0x9f8e4e3057f1247f, - 0x8f1d9fbd47768a9b, 0x3f1b7c050812a0e6, 0xfbd4503879404304, 0x28bdc7902af38c8d, - 0x915d4d73e319d378, 0x4828c458ff2a04e6, 0x0a0e207e2f6c88ef, 0x988bfcc1fadae1fb, - 0x257c02bf75a1cb95, 0x0581739bdf9c2ca5, 0xec82a097c043f402, 0xd6646c69a85212e1, - 0x9b79f56a7454b5b9, 0x8edbe557036ec6a5, 0xb999eb216eb41e15, 0x8b1abecb0bc4d2cb, - 0x46ab884895a3d549, 0x41e6b6ba02bae974, 0x173d6cf00f75c47a, 0x1ae3eeb76f868894, - 0xe72c574d18f2fa98, 0x161264b981ffc1b4, 0xae13b2a780181e14, 0x4a756d841e8b150e, - 0xc7306e678aa3237d, 0xb14601b94eb8b4aa, 0x5d0cac7cfe624945, 0xabbd0b0e4a8df71d, - 0x8a02109cc686e463, 0x7bed4cb0a554a2f2, 0x15ed7cfee9dd866a, 0xd39084344ff4efa8, - 0xcdab05ad641079e1, 0xa9e77164eb9874c2, 0xfc98cfbfb0f63e37, 0x9f3024825425b7cb, - 0xb393dd09e408fa74, 0xbfda28911e6ba757, 0x6cb292dc8ab30c74, 0x800cf9ccd2d1c3e6, - 0x19eb46b92686841a, 0x581a84678f2f0d70, 0x9869e4ffd3ab475f, 0xa325cf03a8359e62, - 0x0be5731491c6916d, 0xb2487e94fe1eae17, 0x13ae351ada4e10a7, 0x7e8574d323398582, - 0xeb2b0cddbe40162d, 0xf49d69374776f02e, 0x251c3da8988a7760, 0xd1de0ba881a00927, - 0x2526f1e0c75f2c1a, 0xb2425247bfd31eda, 0x1ae3f56acc0c962f, 0xc0089e83486deb4b, - 0x695089752ea874f9, 0xdc5ffe2766a018c5, 0x884332e9c9cffad7, 0xa16b9f2b6827be83, - 0x591a5f38d3ca8a7d, 0xf56691b745435e67, 0xa9a8b6b7e97c4d14, 0xc4d578bc81ff6005, - 0xe46d7321d39f76ab, 0x7e88e34bcb8b5695, 0x80a3578057dac824, 0x17677de38154b11b, - 0xd2a301b7ebba64f0, 0x332beb69593b7a1c, 0xa0865c6496e3f53c, 0x552728145e6d3bf1, - 0x570f5d76348c1b0b, 0xf846416a4fe11d8f, 0x8f7c1d0330f56a0a, 0xf19cfba6bfa1d5ea, - 0xa29eec9e9640ba7b, 0x8c6200b98529728b, 0xfb4c1f66decddaed, 0x263a4c57adf58de7, - 0x87555120e2bc958c, 0xfe166d51cb6c4f77, 0x0f62cb402f8829ea, 0xeba42d427bb7e847, - 0x3494d9a11cbfdb1f, 0xf0406b3460b12d21, 0x7e9fbf2a36def8a1, 0x29bc090be9420f39, - 0x41e98e0417a91996, 0x0bc0d14e0d6097f8, 0x20c7daa66c81f80f, 0xebcbced18f806cca, - 0xfb4c1d8d33b0f075, 0x3bae4461c95eb9d7, 0x37e0748642f92d8b, 0xdda9442a59743ba9, - 0xa97a06d3e5f19267, 0xed6347f1fb581ad9, 0x0048e7eebc224cb5, 0xfec026e085a7e955, - 0x0cf80e32977589d1, 0xcab7287374af694f, 0xa22ae317c2d60243, 0x9eae39215e8735e8, - 0xf851968ac74d6986, 0x8cfa77cc1388e514, 0x34a7d3a94edf8cd4, 0x081029dd1605d572, - 0xc359d243cdf3328a, 0xb19c733864084207, 0x3c4a1c0555d3a92f, 0xe904abb0419d0f4a, - 0x4e9768d735e087f5, 0x731610d8ebbedd11, 0xd235d9064f5a96d8, 0x3f6a55a382e19a8b, - 0x086797ebf703b1d5, 0xd0e861affe695688, 0xf6256b2f2ef418b1, 0x94f703276dfae1ea, - 0xa4b406eb29ef47e8, 0x6e084cbbff234678, 0x2b2715b11c1516bb, 0x771ec0fb02d9054f, - 0x9455da5bd449dd65, 0x6273e035afba263f, 0x4ce959fbbc170d97, 0x51fb6c20f2ddf20a, - 0xf6e36e7ff9fb1379, 0xc0d79c0647e31355, 0x6dfdfb649b0e568c, 0x6c12d30e7dfe9ca7, - 0x4715bfcee76b9161, 0xc05ac23ea94d7204, 0xe39ba750bf41b7c8, 0x41f829dc2404e528, - 0x0db8c1fabe47fe40, 0xb9b66a2bfd3e3a9d, 0xd72a79a6b20e5604, 0xf81936d30dc051b7, - 0x3fd94dae0fbcc1d9, 0x9a3464de87d11e54, 0x4243a1dc2d22f602, 0x0c59871503b45ab2, - 0x878aeb23e5638317, 0x5ad4d623499ddc68, 0xdb73e8086660a411, 0xa8deeecdd65a0439, - 0x8b50bd4678e98720, 0xd396a0edf277aea4, 0xfe9dbb444dc30f71, 0xb120a86bdcbd60f6, - 0x7ba5d7608cc6d70b, 0x7f4e541c2d5f1608, 0x5dcd52aeb586bf22, 0x8a01f048b0f1c7ca, - 0x5d01dec5f13874b8, 0x33d8f4945b8d12f4, 0x907d0deb271446c8, 0x2958f10ed33c2f86, - 0xc252c3f8106f046a, 0x20e61cae73996492, 0x78c6630e2e262816, 0xfbfe65e000aa1b0d, - 0x97e9e884bdd47141, 0xf6a8f717dab16ce5, 0xb1dd3f2f52e04539, 0xae40b48e3544b724, - 0x2d5884bfed904926, 0xe63dfdc0acb6ffd7, 0xd404c2abc6c91fa3, 0x3868467a529e43a5, - 0x13a4bee482497153, 0x6378dad607b5c657, 0x033aa9ed88ef8bd5, 0xa59d62d71dee9ec4, - 0xbfdbb309378d5343, 0xaf84483a46289ab0, 0xb05c320187922fc3, 0x22b76e4f7d2b6621, - 0x3cbb43bf565ccb88, 0xfbe1301436b65613, 0xa6d93098c591d6a4, 0x260613f99923f95d, - 0x43f68aef5b308127, 0x8e3cf233603746d6, 0x0c06820a7fb4e02f, 0xe14144cc2ee0ee51, - 0xfc96d00191e4aa6f, 0xf78d9d41f812da43, 0x8ed96d94b2a70090, 0x4b3eb72d0a698ec7, - 0x4c3458c9000031c4, 0x4b6bf7a07284d93d, 0xee606586de67803c, 0x4558ba515b6bf076, - 0x40238b921b756c0f, 0x08636a403d533a12, 0x5f50eadfe50274a5, 0xf244552c897a3c89, - 0x95c3d7a1428204ae, 0xd72614cf3d34fcc7, 0x231aecf96e3c96e4, 0xfdb487fe8de365b1, - 0x0fb06be1dfca0632, 0x09219787e18ce5ba, 0xbfee26916ae30bfa, 0xe13554d7faf99f8f, - 0x82fd8e4dd45a8531, 0x2d0fdf345aa85d46, 0xcda1342ec4173820, 0x4d9ecf51fc83fe80, - 0xc3c4ae9f2827d351, 0x504cd20a0d097754, 0x8fb4e3905eed5435, 0x86565f61010f699c, - 0xbfd0c4e53f88575c, 0xaff209f9955b55df, 0xd2ea2b05c672a186, 0xe208c086bc865942, - 0xfd991a2a9ecbba5b, 0x87bd5cb9cb50d5d8, 0x988359e2f322c826, 0xf4642b3cdd1d3156, - 0xc95709a538afeefa, 0x33aa117ee04b21b3, 0x895a4aa2e1072ad3, 0x7163ca6f2b0661a5, - 0xe3562211332502cb, 0xaa7f033a994be37d, 0x1b918388d13492f3, 0xf8984576bba7c0a7, - 0xfd815a1327f232b2, 0x8f83d03953292c09, 0x2c3725732ca9aa3c, 0xf4d4e427a8096a28, - 0xb7b05a36b1ff9fbb, 0xf2286e083c618f92, 0x1a9a5b30bfb2a2b9, 0x2c40faf1a1226bd0, - 0xc87e2c0a8bf71f63, 0x9d5c98a958797b2a, 0xc4a30d5d101aa1ad, 0xf9dea4cc71073ee0, - 0xe825841bb890e29e, 0x392500642cbfa549, 0xf0ba0c7604ce38d2, 0x953cb89cb8dfdc11, - 0xebcf1d092022e4ad, 0x22e990fd5cfdb481, 0x94113a8b4edf23fd, 0xa5dbefad1a3da796, - 0xa43d3a3233c361c8, 0x23998edb7ed62773, 0x362e89c4f4618f67, 0xfc4329fe6a027c06, - 0xfc50152466d302fb, 0x7f902f9f9f07b9ae, 0xed6a4cbe42a53a0f, 0x54e00a6e97a2319f, - 0xab12abe9d036e730, 0x713286425352350c, 0x359b23ba08c18a8e, 0xc7d0b5e46b4e58d1, - 0xa5bf74b56edfbc9c, 0xf9f982fc5aa95860, 0xcc23a39c01e694af, 0x6ff29156c128743e, - 0xf33d71d07b375d6e, 0xdb595aa83390c13c, 0xe35da7574f0aa3d2, 0x0291829f94abae0f, - 0xd5c0e19cd80bced2, 0xe9ddfc8ce8f595f6, 0x500b94a630688a17, 0xd427108208a9cb09, - 0xaba674fc9abb6014, 0x79fba67ca938a2aa, 0x062cefc6f6ed0fd3, 0x5d26f4ee250cbd7d, - 0xd2edf3266aa0e9ce, 0xb1d0679e227a552c, 0xb45d8d6b363d0e7f, 0x666ad35982bc0452, - 0xe7f427e2f2469769, 0x976987f17ce8f2b9, 0x0f6b01ff8d9da2bd, 0xc6bd42964de88659, - 0x27104ee45a7ce2d6, 0x2af7abc8cb24956d, 0xfab2d53731993674, 0x8ba7519b73de9a6f, - 0xa76fa8bbb1b92cfe, 0x4d6568fc604df256, 0xd93fe94f7b7748ce, 0xbeccad63b1543c34, - 0xd85a64cfd686a511, 0x4bf6994801a6da89, 0x4325374f7ef6656f, 0xd38ff455bf316f22, - 0x8613ecb8d6b05c50, 0x73c262cb6a92eb5b, 0x15904daad10af77d, 0x945286b761e54941, - 0xa68e03cda62e8b07, 0x6b2b256d6d667396, 0x03e630a2525bbd72, 0x0f55826b38d6885f, - 0x8281f02ddb4b4659, 0x21e5a221f0b61cc6, 0x3f48470bd662543a, 0x4e1704f44444fc63, - 0x158fddb513f7d2ce, 0x0fa121f41998da98, 0x24738ad3ed5ff91e, 0x7de86659ad0bede7, - 0xd06fd4711bc7f9bf, 0xa1d074ad36b4f4c9, 0x107630e6cad92dd6, 0xbcd990cd89b84827, - 0x066a0641383552f7, 0xb24c3e0d691f3f3e, 0x4f48aeafbd1342c0, 0x22a32bc19303cedd, - 0xce1cbcb4f20c2a98, 0x885ce40b163cb050, 0x6df486865c97e562, 0xe59c4ab5318f231c, - 0x1ff82d01f235ef77, 0x446237491d021b76, 0xf4696bc1ee8e052c, 0xa63e045bb3c2a016, - 0xa64ca533656880db, 0xd992c692f222ee87, 0x5d294cd30cf4ea20, 0x1af689e0fc98d344, - 0xf58cb5d4ee86e5be, 0x29806654d2d42da2, 0xf5cc0c79765f1c10, 0xf19b19f26bd2c242, - 0xe23a77dfde0a1377, 0x120d5d238ee2f9fb, 0x4c6bb1b7cc8beefc, 0xf60dc61be225f7bd, - 0x98f7b3ee3728182f, 0x2c7dee764021e2d3, 0xcf9b7adb5f746c39, 0xc4eb2a7139f9d90f, - 0x30882bfeb38ea335, 0x01f1cf64fc4e2772, 0xd96a0ab184fd53b6, 0x7a2a8bb8badaa8af, - 0x9c5e3b15e95f9a1b, 0x2d75bfa54fd95422, 0x4a08ba8cb2b436c2, 0xc628a7496bd18beb, - 0x8b4ebf3bcdafb81f, 0x0b76c770f7b691a8, 0x2cfc3d92c6695345, 0x85ef308bc9f89e33, - 0x07e4ceda8d40d737, 0x5f76242d10cd3f3f, 0xf6ed74931e7cc998, 0xa39aa5ac3a25f686, - 0x1f0cf42decf20732, 0xd88f8e3de4414814, 0x3227b21f254676f3, 0xd8796c528b4c5681, - 0x77db4714593ae219, 0x950fd9af6e11507f, 0x5531a4ed6de8e1be, 0xfd557a831ecfdac3, - 0xc966fc4867138a39, 0xd470240d6af46951, 0x15eed74ffd98e899, 0x5fa0557c884720b3, - 0x92e7756e5c143f44, 0x629f09470db890ab, 0x2b4be37fff93b482, 0x03fec57b8de254dc, - 0x26a83dd0595452fa, 0x89c9326c2fe44590, 0xbe9af785b02cca55, 0xd47bc668dd69bb5a, - 0x4c0ec09638a54c40, 0x4211a6a43d70d0b6, 0x140f766e4567a0bd, 0xb559c0ea5ef87b99, - 0xcbe638e4a765fa00, 0x6fa4df7133f21b8c, 0xc4e974f050acc440, 0x53e8a9bcfd868513, - 0x590b8dfc2bfc6b13, 0x4414f990baf099aa, 0x404985c77d3ca692, 0xbdf3c1203a3510e7, - 0xef699f62dae5de45, 0xfad0ed792dc6a5cd, 0x50ccddf6401fe12c, 0xe0b7c8c8cd526df8, - 0x757703ca8caeecef, 0x05a6a61e51cddc85, 0x3093ee2174053b93, 0xe6ff6a6555f43433, - 0x70b242f83bce0a5e, 0xd429e4a699bb6b42, 0xa4ece1f507bd6181, 0xd9efe01cf453f046, - 0xf96fdd1068c01bd5, 0x9d003f6a34e338bf, 0x0ea8a5ce57b1ed4b, 0x7a38f8a1dc7b2218, - 0xbbee019a48049eee, 0x7c6f4aa6720aa908, 0xa23e8849060f939d, 0xbccbeb38c3de88d6, - 0x696e58cb348f2fbf, 0x18fe8a848ae5a532, 0xa4f1f72bd77f0d86, 0x1b4ed6b08f877cb5, - 0xefb8fad3f04aa24f, 0xe6c23841ec01bc8e, 0x0358d076d2ed0520, 0xa7cfc0940d94f7de, - 0x56a27ec0484bffb9, 0x13d622b73d0d70e2, 0x40707216cd97e8e1, 0xf5ef41f52eac5605, - 0x1504eb2ec7976f2a, 0x7498a370267dc0cf, 0x981ec4753123c28a, 0x4a36a5eaeefec56e, - 0x668e5392f4e28d70, 0x1ac7002e137e3908, 0xd014e80473387bd2, 0xf429b32fff7c197c, - 0xd06d479c8870d8c3, 0x5c87c08cba3eb14a, 0xe199092c4bc604eb, 0x9e0a7c9338e25f2d, - 0x9ee1429d4d92c145, 0x84137eaacafa7a14, 0x9c12ea935af6a60c, 0xf02a35a6dcab69c2, - 0x389836696d8b44a2, 0xc8a88ae6887dd62a, 0xbf00e738b934b5f9, 0xbca43c2b3a76c608, - 0x68ce9201508e385e, 0x101031fd5f3c20d6, 0xdbea723f81939d2d, 0xb8c792fbac8d8028, - 0xbd991a9ad71d6fe7, 0xc3180da6243a78db, 0x7bf38b35afab17fd, 0xcd869686c12b9ede, - 0x126a720f79f4081a, 0x9fab19cf026b01ef, 0xbd1cfa4c0a5d896b, 0x9cf88e019bf30173, - 0x1f692d4445a89665, 0xac1585683f863c92, 0x99c074609c8c051a, 0x950ea6bc65e6e416, - 0x90635bd043d57beb, 0x9d13fd7b3c63ffe2, 0xd4a555939b00d03e, 0xb4c0c567ea072cc5, - 0xfaa3c632849b7fe0, 0xdf9da469c7bc8321, 0x94b29edf7e1e296e, 0xaa50a1ed34592b13, - 0x78894331ca9e2f86, 0xe6771a420cd188d9, 0x44acddebc0855dab, 0xc7989b3473730065, - 0x72af8cef7600b8d6, 0xc6670e23cac76768, 0x6afbbc70a9ba9d2c, 0xf2c29ab4eca28ce2, - 0xfc7843facf509dd4, 0x92bd2b64dd601ccd, 0x88954c3207b70848, 0x2caf5438614466b0, - 0x8ce14cb8d94af7d8, 0xfafb9bdd3004a1d3, 0x742ed44e1c099b40, 0xbcb28f0462a98b24, - 0xece851f92cfb28c9, 0x6af702197eae7ea7, 0xf05284a7881358f2, 0x5ffaa1ec67ff59a3, - 0x4622284f6607c08a, 0xdf0c10735c3077d2, 0x47f26dd39bc52266, 0xce5eea3a84308012, - 0x203638fff30b3f45, 0xab67ef64112fff6e, 0x07e71ec03ba44faa, 0x38200d6f68be6b26, - 0xf78de414d2aa86b8, 0x5638894ccb5e2e67, 0xf14bb932e3a3c894, 0x6732a756e70fb2b3, - 0x649e67ea44b1e6b2, 0x42e471ba60bacf31, 0xc00086e02e18f194, 0xa0d6fa5a66d508d1, - 0x950879943e839b27, 0x3a2e94a44603963c, 0xecdb5067d6ac6b83, 0x1ebf2cedc26e361e, - 0xd8f04b89355ccaf6, 0x72058ba72aed16c3, 0x8ca5377c0603aa5d, 0xf5b356ff277b9499, - 0xccf24d0c9cfa7bcc, 0x636c422258a1e9e6, 0xee1cbbb7a3bc5d03, 0x97b79828aecee94b, - 0xc83f35f9d5aab418, 0xfafe35b83d90a1b3, 0xe9293244b9b203e9, 0x3863f148df6f9d77, - 0x8640c9537b787c2e, 0xc5c272812cbf57a8, 0x4592c0deaa544589, 0x65d7761d2f5a95a0, - 0xf506032b1d974808, 0x8ece539ecfa8d692, 0x825a0ec78698714c, 0xfe34e841355fe0a1, - 0xe1a4bc08b5aeedea, 0x61fb11a986cfa642, 0xc78df063156ed08b, 0xe4316c61c0f7419b, - 0x94fde2539985ae18, 0x2527f9ac04dc49e8, 0x4a0fffd55dee63ef, 0x1c445ee2054c438c, - 0x678f93b86a322876, 0x6393970a4bd462fe, 0x7b1f376aca979630, 0xd551dd61120bcc11, - 0xfb3caa8be7478902, 0xc9d138416c25bfbb, 0xbf1ce96c11bfb19b, 0x86766dfa170be6e0, - 0x29d158c4c9a725cd, 0x07ce8d2a727b0eae, 0x838b9537931e4adf, 0xf525290bcb3229b5, - 0x0007a289ca61fb35, 0x3b7962ef6a162ba3, 0xbf92ae436d55ba93, 0x478e257a24b8acf6, - 0x3c732d31cacfa014, 0x9be9e85b32ea867d, 0xf2290937a2d61bfb, 0xfa524df6bbdaf3b6, - 0xaa8e0ae5e5893c11, 0x854c633ee3d6bf73, 0x5c04b295f48de69b, 0x81ac467bdb768c02, - 0xed7e41e3f797606d, 0x4667475d8757f000, 0x4afc108d8af07dc4, 0x8daaee376fc40828, - 0x08c16bd16019293f, 0x95f1aa2a28bcf37a, 0xd3ee6c72241957e7, 0x3d12752087db1dab, - 0x0b89a07e89a30481, 0xfea5b9d9f777e733, 0x2c07d14b23cfbf83, 0x284349d2d50c3a08, - 0xa88767d969846393, 0xd8cd369f05a5ae05, 0x8684e79c0f7c6a93, 0x1f88ebb4b6ee9179, - 0x03c255310876fe85, 0xd442c6e9cd60f0d6, 0x0706b467d9c07d6e, 0x010a4442889adf88, - 0x0fe8ecb5110ce157, 0xd5c8b62e629a1fe1, 0xad70e97ed383c59e, 0x1f4e1c670897af9f, - 0xbf178e34f9e8ac70, 0x78382ae4abdd1341, 0x83ec86d1afeacf06, 0xe4a7cba03e902ba6, - 0x0c3484b9d9258174, 0x6a8d286e9050101d, 0x83b86c81ddfec840, 0x72387f3e3277f91c, - 0xbadd75730d723afc, 0x37d101ef2e81caa5, 0x8b10c02e33098950, 0xe6ac9e54e6c5fccc, - 0xb4cff2e97d24e1d6, 0x5e736e0763ef3ac9, 0xc8c6301c5b92f843, 0xda82f0ee968c0d02, - 0x9fded2098580ef6b, 0xdf8bb1924b849f81, 0x0488bdb1dd6ca3da, 0x6b200fcd1ce32bef, - 0x374ac3b50212b03a, 0xc12de6e38364499d, 0xc16619cbd1088cbb, 0xb0cb13fd873bc392, - 0xe8df456d650dfbc1, 0x7bc2041fede00437, 0xa24e6ffe3a5a216e, 0xb338a180f2da6a8f, - 0x9265138f5fec9e52, 0xb677672209dee73b, 0xf42b12736469c042, 0xb3a47450ab34af25, - 0xf741b25152e1632a, 0x61443720405922bc, 0x24991bdc2605bdc6, 0x0fc7682bb3aad669, - 0xf8caba191b7dc26e, 0xee4222b011292f1d, 0x3749a1349619c89f, 0x2261a98aabdac4e2, - 0x99c9bc8fd4f60e87, 0x254686c5b7969ad1, 0xda5305be3680a8e8, 0x267e986f6e237ec3, - 0x30810c2e9caa2b7a, 0x86c8349a8e7ed5f6, 0xb480e607df2e3f22, 0xefabcbb2a86bef44, - 0xef3926083baa3ccf, 0xc42feefdd9041037, 0x6f490977bd008b7d, 0x3cb1d8001d34f061, - 0x45f194c0f26f6e4f, 0xd0db70cfb5ce6a8d, 0xde1bdcf7c6349644, 0x643a181d8dc6550b, - 0xdfa2d557f50b0c92, 0x28e9b7b500bd7c39, 0x26cf6566595110eb, 0x8b43d54498056cf8, - 0x4eb351ec706dbc9d, 0x77f693ac39fd8c98, 0x63cdfd6e3b2fd027, 0xd3a0e1cfd0733280, - 0xa688846341b509a6, 0x880ff11ea7e0aadb, 0x140f4b7bfff86b1f, 0x46305047913ba35e, - 0x23dfa5fec199f953, 0xd8e86994439fde4d, 0xae0203df0807fa75, 0x68324fc6beabbe28, - 0x5def14d5ddf9fcc2, 0x3cbc9bf0c28a66a0, 0x5dd3a2695d4ba855, 0x21e39a77dc28a0ec, - 0x0bb1c6aaa0782f3a, 0x3796285be4c80121, 0x7265421b2aa8ba42, 0x2f303bd6672b7aea, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x92ec9fcbc11d186f, 0xa82acf836455f395, 0x141a0d74be00af09, 0xee6dffb61a7255a2, - 0x6ea0ffd5c784268c, 0x0d75990fb8f5dd63, 0xeba0e4eec5f89aa0, 0x98d85a17a1e82ea9, - 0x5cb6ccb6690ff870, 0xb979cf0ebf20a814, 0xd56f941bc595c46b, 0xd7fd1913a9f3d5c8, - 0xa53ecca6aebe8ca8, 0x275a25e61667fad6, 0x2afc1e38a8de4e8a, 0x3227742d4ecb16a5, - 0x64e77dd8e7ab72a4, 0xceef1b141bd3fd7b, 0x163f1072ecf92040, 0x28c1ecf47e3a913e, - 0xb47f7540df359169, 0xd95dfc929b5ade69, 0x5ce277d1efc82867, 0xa6719a66ff85aa87, - 0xf1f976428ce3506f, 0x4e784129ed984779, 0xf7f628a70c3c1ea2, 0x89c7f6a5216d2963, - 0x4f7e36d84c0cabc3, 0x800fc714cda7eb1e, 0x5dc5f1cbd59bc028, 0x1895708fbea6d4f8, - 0x45b1478f270fd036, 0xc9c5c5ae101609cb, 0xdab6ce72e2a1463b, 0x1d8a53f480eafa70, - 0x7698237781ac5ca7, 0xe2d4f1e4a7369ec5, 0xd60d5042966647b3, 0x75d663cfb5bb7edf, - 0x3ded53e095b83af9, 0x4793b9caae960871, 0x6bf5a0c201cda80c, 0xcea2dc43cc9c9846, - 0x74aa61989bc5b274, 0x349b8c6c044a65a8, 0x7e7f94853888a8b1, 0x275116c0ba0e40ab, - 0xa8891a981b8fbe88, 0xb5a39fc41facf310, 0xa0e72226a9fb2d53, 0x09f4796d6901f049, - 0xeb700af12f8aaa66, 0x56a53aca18b432b0, 0xa95c6ebf55df3e2e, 0x48b8746ecbbe7501, - 0xd50d13cda7bf5210, 0xe3d6de0fac99e865, 0x8f2a710ce66dadb5, 0x59b796da1e66e383, - 0x8b4124e02a0824e0, 0x0416a768374d565b, 0x1e4d58242da77e4d, 0x070c21bb7bcc63bb, - 0xeed971b575113bcf, 0x17e7c5c733c60757, 0x0141018c2c9dfb2d, 0x78aa32652038bd05, - 0x1b42e42f4e1a4ba1, 0xbcd8a43fc0833a19, 0x1721502546e543d3, 0xb04c540b0311ba20, - 0xf652c459e39fa1a7, 0x9fae857efe593477, 0x05dac0a4b63bf3b2, 0xf4d63bd9a38f2631, - 0xd118e163c04964bc, 0xff8d054d555ac624, 0x3c40c6615783e948, 0xc170ed8c99af1d78, - 0xa41bd5577f51a4b3, 0xc9eb4bd6b1406f90, 0x6da11442c29c3481, 0x998e6e2de585d096, - 0x01afc0bec354fc46, 0x6bae53fd46c17bcf, 0x45bdad66264e16c4, 0x62b77da2d2056c03, - 0x6d3975c05ce31ee4, 0x8d3556c9d022abea, 0xd73101d75c33a66b, 0x721c5c42622eb56f, - 0x435d8b24b689109f, 0x9fcf332013c39a14, 0x51157c4f3197732f, 0x897fef32ef84c965, - 0x539b4e4d13a263eb, 0xde2d8b519280bba9, 0xdab673b83f34a5a1, 0x39327e231361417b, - 0x965e1c0b8cb2890f, 0x95e9780d421eb9f6, 0x5e401c98a0782ee2, 0x7459e3e2685539a6, - 0x02782e569eac3034, 0xc585db7f4c65d96b, 0xc94dd0efe89fc473, 0x15c04a3adf3ab997, - 0x0ea10c76e442b58c, 0xde284afdfb55ce7b, 0x933c0e4c8c9b6788, 0xa68e16599f73dbaf, - 0x373c6fe05cfa8d70, 0x57e018cad4bf2300, 0x02a3fd645f150846, 0xa39be7591f58e298, - 0x7412260f04f5bbca, 0x92592a3c645db889, 0xb335fe62e2f13b40, 0x97c04761e5a16e08, - 0xb3b1c279fd4f3c1a, 0xe93d2e9d59d493c5, 0x3e6830903f5c2186, 0x8af447cbe5ee83fc, - 0xf32a087d959159e9, 0xa5562fb86cb24485, 0x1db9f60d6be4b506, 0xee1e3a7f883c49dc, - 0xb86722e6f6972142, 0x1a62ca6e4cd6618d, 0xf2ea685ffd442c2c, 0x1d1c38fe0be2ca64, - 0xf21e1e76fce80610, 0xc9e759bfe4eb4214, 0x86bfb2bd4920c5ac, 0x4cde595b9ebfa223, - 0xd57ec5b20d8cad09, 0xfa98fb76990d7efa, 0x68f1ac88e8a5ba91, 0x75766195761a8a46, - 0x103f26a8f45e16f0, 0xa593141af399ba7c, 0x1739d91122e49f8d, 0xdfca8cc8c352ce8a, - 0x4fe7e0a4e4090405, 0x53d6bd169822cef9, 0xd0af15b873685f15, 0x74a33df666ada48b, - 0x5406ad022b38f8a8, 0x669cbcd879909cf8, 0x7e41cf5a10efd51d, 0x02dad13eb2fd4d22, - 0x269d6571310de2da, 0xeb50151afb34c857, 0xd004a857c68af74c, 0xe5ad25025ed8f8e0, - 0xd6ac6b3ecd320eb0, 0xad1f36df0cb37994, 0x39ffd36ebf3f022f, 0xdcdabeb1bf165bdd, - 0xb92fb5594333adc5, 0xe5f20b4e862ecc8a, 0x1a133aafc60dca50, 0xb2b8c814ffe03393, - 0x57fd275c316525e7, 0xe5811f95d22fa6ca, 0x87f2e97594e2167f, 0x30e3873a4398aeab, - 0x1c0dae0ef085596d, 0xcc7f09749b369ddc, 0x2a28d5e93a3fcd69, 0xbbf9780743e9ba89, - 0x22790d8d1921e478, 0xd7bdf07c7dd59315, 0xa0c3388ba408b20b, 0x5cea764c67a9cb11, - 0x8eea573f2a73463f, 0xce2356cefb445f98, 0x12549ef755537867, 0x534fec11267dd91b, - 0xda95b1b7f2bd83fb, 0x66439e3acf812dfd, 0x5abe4c448bf48a55, 0x32dc56f5a3dc1830, - 0xfdf1cab3a63772e6, 0x5092921e1944e7c4, 0x29c8cb607bb643f8, 0xa54bbcf28c7f1bd0, - 0x2d5e396f95d2bb9c, 0xf77be0690f7b7cf8, 0x9f3a1770b4d9cfab, 0xc411123fb458fed9, - 0xd51928fe8adf6828, 0x55fdd0707e1d642f, 0xaa1dcc49ff8d2ae6, 0x076d85c09cb1df2d, - 0x74cf1f5572108216, 0x526ca26be7dfce18, 0xdd1d00b37fcf95de, 0x31df0f3094d89ad6, - 0xcb72ea21ab886337, 0x459b150481ff39ed, 0xd5b76762e58f9fef, 0xfe75418efbbea107, - 0x1f9bceaf6ee2b1c3, 0x18d92c664d183b77, 0x09664c0b4ccfb4bf, 0x1f7e4a8a5d4056aa, - 0x8ad1a9db48550927, 0x06e3d3fb8213fcf1, 0xb4e71790cbd80a55, 0x8d6f782fd4cbd51e, - 0x8e6419ff2fded883, 0x3fc0f826b22602b1, 0x5db983f83593e459, 0x2108cb2479a38498, - 0x1d2b2cf8dc1411b0, 0x7564ee0bfc75499b, 0x613eb7bfa1a12f3a, 0xa939f5cdd60fdae8, - 0x7c9eb8472316632a, 0x8b36de01372e6935, 0x4077c04b18c122c3, 0x8e10f578d62b3541, - 0xa30589e84e86912c, 0xb2ca8facd2538cc7, 0x0134bbc09474d1b2, 0xa61d5ee38f0c9cbd, - 0x62926c33a90f3a63, 0xc15a4daf7a1a3804, 0x322c3540b11102e7, 0x5fa0ce4b03f6b9c0, - 0x4c4ddc8e4f6d4192, 0xadba225136fb2d7c, 0xc832d28edf57f24c, 0xe4dd81026eef3ca5, - 0x44c02a2b17ce5fe4, 0x1be6ff7d149a7ba2, 0x33e4ccbb6380f035, 0x982772192bbe32ec, - 0x9c70dbb9d969ae17, 0x71fdad4d9ca95c64, 0x975118228722e8ad, 0xcb148776982701be, - 0x6d5ffc93c6ac6c25, 0x97ec4bfcb3e29fd0, 0x16a6248be081447a, 0xca564efda3921edd, - 0x48bb2bc36f19abca, 0xa3d46a50bec6147c, 0xcafc107410d5d8e3, 0x541346c77632efad, - 0x8dc5cfee5d66948e, 0x41b69047b76466dd, 0x7b098a7bb5f9a93e, 0xcb365d004bd1c525, - 0xe84bd06a7c905de0, 0x58f8d7a88684a88b, 0xe2222ffac1c97f08, 0xfbb64e35e5890f5d, - 0xf6f34495ec09c759, 0x048116b2973b0aa7, 0xd71d4e0ecba6f768, 0x3f61898b5a0c9265, - 0x2049fd988bb4b241, 0xa2d099685b7a5e62, 0x3ee61b9a9a612f7e, 0x69fafb182d97b67f, - 0x755938d08674283f, 0x060182337779a4cc, 0x195276444eea3c28, 0xf20dc31ff0b26b4f, - 0xf344902e63a6aeb0, 0x9056e9dc6b3647c3, 0x54fcbb33c8d4d0de, 0xd27935803df96908, - 0x30f3aa3c0c56c816, 0xd03b1d75a60c2a8a, 0x70919c8d776202a8, 0x14b7150ed919545e, - 0x89f8366cda0c539e, 0x7ae6e603fbcddaad, 0x8e5c4e52848ee528, 0x4eecf115c37852f6, - 0x47d9d3821e7f242c, 0x789414c6a146b33b, 0xd68c075c0781d165, 0xc67f7fa36c422541, - 0xfc0cef71d532090e, 0x3fb7e794237fe526, 0x90ea4097a1a6a893, 0xab339f2f767785cf, - 0xaaeb388a016b272d, 0x90dcdd7e837ca33d, 0xe0deb35a2bea84b6, 0xa5fd4d359dcb662e, - 0xc72e99571d05aad6, 0xb5b5cd2bd1b0b999, 0x8cc8d9efd7738ed4, 0x49ae09fbd0b0b269, - 0x585c4207dbf8a17b, 0xc207eb08b4ea131a, 0xdc9cd45e0d344412, 0x00885d26e7bd30a3, - 0x8ea5e5b8b1deec3c, 0xe7ce5a3fab4e43e1, 0x3f43e98488203221, 0x8d839cd47d36bdfd, - 0x21da96b698a50a30, 0x54577d9319889adb, 0x67a52972ac057689, 0xc0579678500165dd, - 0x1c7453b1bde822a6, 0x192dd52707b73197, 0xac9941fca7f9d763, 0x9f388f25021a3f46, - 0xa4182c9cac58617e, 0x25acbfb2547a5335, 0xf5f26e8970007ebe, 0xff715b7c27aa4976, - 0x77507ce623d4bd10, 0x55d5eebb8e2ae649, 0xcfcbe39cf5dc68af, 0x02be69f08db85585, - 0x658d1851af915e55, 0x7540670fa4fd5ad7, 0xae92b8557e0bba6f, 0x8f76f151d5554f57, - 0xd1597399bc8c6f2a, 0x77e921e62009c019, 0x0a42154320e623f9, 0xef381ff06100f57a, - 0xb2aa841eabe04396, 0x1c364ff8222cb82a, 0xf579cadff2d1d7ef, 0xb9d95dc9b9533ad7, - 0x1359d1b97df7091a, 0x5d539ca02710b668, 0xf20524fd6191f277, 0x704eacb8089633a1, - 0x876ac6916f0f44f2, 0x2a7a2e54aa4c4e9d, 0x4f1bd1d7c30ea717, 0xbfd421ead4ee0fc2, - 0x3fcda7017b0e4aaa, 0xc4793c28aa4772fe, 0x2ac182fe03697128, 0xc1bf656c3e782e96, - 0xb510aa46d5e60ca3, 0x7da6072c8fd709a1, 0x04cc93ee7c8075bd, 0x80a230ef597a1248, - 0xbe2fd9d0368f8efd, 0x4ea4fd12e48248a2, 0xcf185e6051ffbde7, 0x7fbb4760c011e11b, - 0x89cda2e74d658319, 0x46cadb2395f3c118, 0xfa8bce607b3fe6cb, 0xf38efdd4a06de989, - 0x3e6b03ea65788008, 0xe5e0a638c9223597, 0x766101c0aca788ba, 0x121b1292b5f2ae25, - 0x2da51a7121f39bd4, 0x0c8862041c02cb3f, 0xc5913b690bc601fc, 0x421ff23c75f73cdf, - 0x635d30a19b4e0bc2, 0xd5b2f5e0fa40c5e3, 0xfebeb65b4993a843, 0xeab8f0aa806257a7, - 0xd1c717ff288275ee, 0x9e83434cac015192, 0x2a209212014bc128, 0x68b8df0ea435c919, - 0x9c82695c20ec5860, 0x6171ff9a182fe75a, 0xfa6c3b143634b33c, 0x5d1d8ebf2f14f13c, - 0x725b93228e5122b8, 0xfc104ad8526c9c87, 0x8b155d4cc31b6d57, 0xf14cc8e48dd58e10, - 0xab7c6ee7cd75e281, 0xf1f243c204e7e0bf, 0xa1ef2c756582aaa7, 0x73aff865b985923c, - 0x1b92a85b3123dfa8, 0xa608329f529015df, 0x56783c9cff63b3e8, 0xe76c50266c9772aa, - 0xab99b637b8718490, 0x1bd892f22477faad, 0x0f4d6a3d4176024b, 0x8fd2bda2c4cae0dc, - 0xf660a894e5b4d0bb, 0x55813f1e0574479c, 0xb9934ab037354e94, 0x3d806c28d1d36831, - 0x5135c61f431dac8d, 0xf4a836f62d104d83, 0x314a2186b0df4011, 0x2f44edbf0b022417, - 0xaa0d8019f8c30794, 0x1b03b10080d0d62b, 0x6643f358e293c3ef, 0xb8b9c12a062c02f3, - 0x0deba631bf1064eb, 0x4d43944de57c1a89, 0x07e125a5f535331f, 0xe35ea671428cb65f, - 0x05882c6785ea7d90, 0x08b7a0dcdc2a2948, 0x0319d61cccf699ca, 0x9c6037b52bb514c0, - 0xcd30f1c9accfbb58, 0xae05af5390dd11d3, 0xaa3eeaf82b2c5458, 0x49f16dc7c5d0abbe, - 0x652e8cd21cc68ced, 0xb83f47716f879430, 0x60c5a4e61eb261c2, 0x3d4c8a86c2de6c23, - 0xc0da7c2a4f351e9a, 0xd3dee073cbf26b0b, 0x9f1ab8a12cbfa5e5, 0x8aac649848ab6f28, - 0x0c4f20529b94ff3f, 0xfc34643143a78d0b, 0xd8fa1d7e776fa6e0, 0x8c9ac5c0d9a68a5d, - 0x38d7c6efd9130ba1, 0xd82e2f0d623645e3, 0x2b1a6395072d217a, 0xfed85931a886f05c, - 0x99f68c1865d284c7, 0xe3f5a551b9d05f75, 0xdd9a4db563f49c33, 0xa1d1ffd7771aee36, - 0x94543a6fdd8964d8, 0x8dfb3fda7d2a9b08, 0xb5198c2c84bc353d, 0x10ac97166f0addc9, - 0x92b7ee26160db51d, 0x3aff4c3c5bab1fef, 0x4502edc24e81b215, 0x8446e8c85e5c59f4, - 0x5a40102dcc502e87, 0xb5dee15e247bb2be, 0x511920a726801883, 0x737bf5387f53e887, - 0xe469cae14d5fdeb0, 0xbfb312d3d36796f5, 0x39a37b317e907b73, 0xe4695d37b7fc15b8, - 0xd9c0e6e28f82847b, 0x56759569ad5c76b5, 0xdf50db76c7103bea, 0x7ca3d864e7d0629c, - 0x9c7432b05922cc1f, 0x87facc61155e9e7e, 0xe1c780b3a2b1611e, 0x75354ff21102eb82, - 0xe8826cd68121aaba, 0x0f185570dd159f4b, 0x27bcd004c062bfe6, 0xa3cec636da398a60, - 0xed3a6fd0c41ef229, 0xf06dd740cf3a558b, 0xcfa201da0c69ef52, 0xfefc4df2b1df90a7, - 0xe11d719f20f712dc, 0x60e25990c816fff6, 0x96201a1b4d3c2eb7, 0xa26561fe88992962, - 0x9ccc6c20406aa033, 0x14388855b3dbfe81, 0x1e431a663af246b3, 0x0e8f343d2361d697, - 0x86b51e283edd031d, 0x264a4bb511a930af, 0xa55271c6d250b0fd, 0x6be8567d8b5d1093, - 0x63caca8edf0176be, 0xacb0726844e9737e, 0xa8ecb17314a8b05d, 0xb90bde858ad97dba, - 0xe48836c661904bdc, 0x3521752117de9a02, 0x51aa92e8df7f3de6, 0x9ebb2c5f7b59bde9, - 0x050c10c1746eb858, 0xbca6ff69ae16b238, 0x0053e5af09df1c70, 0xaa22625301c11b9a, - 0x6cc52ac87abe13c6, 0x27d0a7aecda5960e, 0x2de6d322ed4b172b, 0xbda4ae4b38294c3c, - 0xcade95e521dfe3ab, 0x525e5e6b58960821, 0xe2721e5e8a72384f, 0xa4debc1c2b07f4b9, - 0xbadf32318c218204, 0xc915b83030bee951, 0x08172c75e8fcd723, 0xc3fd77aaa834c8d7, - 0x4a3759686ddd1a8d, 0xd17459393ef8dfd0, 0x9eb6112e0f059ca5, 0x419a3cb9895b4a54, - 0x92d8a23fa002b93c, 0x955152b75d5b3c9c, 0x46d401e281822e3e, 0xe24f9af9be8958a3, - 0xb06ad44678cbe9cb, 0x462cd7801b72bc90, 0xa35771b132de9d27, 0xe79b219cdf17a7ec, - 0x76273ddbf88735ca, 0x5eecfe2534b22306, 0x9a8de88bcd5cea00, 0x73bec6267c32a070, - 0xb3c2c4d9065139f0, 0xa750a684f0a56256, 0x72f67491e704e87d, 0xcb39c7e03b69c15f, - 0x17e9cb56b64c3bcc, 0xbd8f03ce615f2449, 0x38e140dec06cb9b7, 0x02399eff003f7819, - 0xc0df1523bd1e8e6f, 0xbb00fa8d0dfd8588, 0x26cf82b535dbce55, 0x18d0c4f5c67b8f4a, - 0x82359c461f029e98, 0xa1dea83c053d14e8, 0xb5ed1fba2d9cf977, 0x4997be9dbee46755, - 0xb43be5ce4c38b9fa, 0x8fce7c432394a26a, 0xecd884797fe7a6e2, 0xc61d7381330792d9, - 0x2727d047c973fca6, 0x8c5c17812d17a647, 0xe56b033af58e9694, 0x0332b2b87819276a, - 0xaf149a42177aa270, 0x3de9d0262b38c57a, 0x16b155abef9aaf09, 0x3fe9e602320c2ef1, - 0xcb0577a400cbd8b6, 0x4d123296442db11d, 0x67e679ac4964b001, 0x5f34c6b9a45ddf3a, - 0xc7d3539bd1556a08, 0x8d8110ba11f06d29, 0xd5f068401fc707e3, 0xc3e099bfb9a0eab2, - 0x9972eed53c5a5d1a, 0xd7d4ba223c690a6c, 0x3bd4eaa7c2d60fe3, 0xe166de1d29fa0a89, - 0xf3f200474c5a200b, 0x5963879349e6e255, 0xded17cc891ba2e02, 0x7b04c79adf28f12a, - 0x02f08733e7d80886, 0xd3d235270b116651, 0x3cb8f8e7eb874fe3, 0xdf0b2f95594774ee, - 0x407eecb1592d22f0, 0x2e192f9cda124d3b, 0xc8ba5cb4301ec7ad, 0xeda9ae1ad2421e45, - 0xf7c9c727e2b59387, 0xc3d167ea4e3ce845, 0x20d68094801c5a74, 0x7105578c3ddf8b6c, - 0xc429adef0d65ca05, 0x0f002b2f985666b0, 0xe484e5c9d1dfda15, 0x7163ad03ce85ad75, - 0x1f9e81f22bd40684, 0x3d65468d3413c842, 0xbe06a5306f6d59f9, 0x0331ed2fee31b8b0, - 0x19d0e317b3ddde2b, 0x72544e3179c6a83c, 0x41e80093823ad543, 0x7aea07fac613754b, - 0xa1d728cc54b6df25, 0x7621cee012881b22, 0x14f496a5e17becb8, 0x351c66ae87b016ee, - 0xded046c47c8ee6df, 0xd50567b119abe949, 0x462e2cf8e7479554, 0xcd6e1c527774607a, - 0x59dbac841e619dca, 0x4e2962cdb4db9fed, 0xb1a37c05a7e1ebc6, 0xee495dd50edb59e1, - 0xd0ece16bf34e2f1b, 0xd517ee8f04fb5b38, 0x6fee015023819778, 0x82faf6d788da640f, - 0xb1b591f8b3e6c737, 0x555b707ecebf723d, 0x12a4672f581364c0, 0xff004c74f7f295dc, - 0x314c59afe93acb66, 0x1a1efd9740ce3a52, 0x5d66b29cff054ad0, 0xccfca59f6cbdcb12, - 0xa5c8d13f3f1d4f64, 0xc2fbeadb04273ba4, 0xa1107c88ac9e9fde, 0xd77ba13b26b4c8f3, - 0xfaafa93fb0c24e35, 0xe669513ac456734f, 0xd1291dcb26f5d23d, 0x0f56bf5d37360e81, - 0xd075d264935c616e, 0x5a92c7406fe30ed1, 0xb546ba759462d507, 0xb0c2c3657fe199b5, - 0xb1d41571cb848ef7, 0x7109f513d4cd1ba5, 0xcf919bcd631c931c, 0x898c1d33c5b48074, - 0x5555f330665a1f33, 0x338486167706e878, 0xb74e42e669f42643, 0x0534facdcb57ea06, - 0xaa2ee0cee0138851, 0x1f8c4834b49cf609, 0x7798c2c5c97dc4c0, 0x6436c47274c7e933, - 0x9bd9e3592c7b71e6, 0x3b18dbfc9bf7ac9b, 0xe13b00e5e1aef412, 0x3a15b29fdceb1223, - 0x5fce4b1680639448, 0xe2a0bd9cee8b35b8, 0x4363f5db29cd0f31, 0x90f8369f218e4cb1, - 0x7005a41f049602ee, 0x902a098ca50aa1b9, 0x8d1c81aa0a3ebc85, 0x252c5bb4db3a88ed, - 0x55317f3af06880cf, 0x5fda76c8c7de0726, 0x2b5ddd4a314489de, 0x6e2aea480a18d19c, - 0xa8444229097ec8ec, 0x29e6a246d1f12a1d, 0xbedb4f3d5ec883ff, 0x1b07a387f2763778, - 0x78a5b122c3a61937, 0x98abd6ab57cf893d, 0xc47f8271c0ac60fa, 0xb46155629c766488, - 0x2cd2d28f0d51559a, 0x7e442b9daa1b8889, 0x68ff812fdf8277c0, 0xe9e5199d3e6ecfab, - 0x2697c46ad8796ef5, 0x9282b0f5983e0916, 0x4795caec398cba09, 0x43f20bfb79381939, - 0x5c6f0a151e2f2afd, 0x8453e722d403247f, 0x6142ef16af08567f, 0x8883338f95067375, - 0xa485c3c95837829d, 0xdd71542df12c5b0e, 0x5e1503aa1d94dc5e, 0xab89cde3b700a472, - 0x80a1b2666bc83a8b, 0x8bc14a056e7ff925, 0x913cb47293aaf8ca, 0xa6e0bdad1c3fd2f1, - 0x6fd52b141a249618, 0xee1b3815618dd3fa, 0x27f1999d828b4c3f, 0x20a71df6000d7fb4, - 0xf573b48a2ae60ee1, 0xd86c9f410b7d86fe, 0xa2aec6523eb474fb, 0x1c7ce55ef55bf45e, - 0x81d42d56b2c6c4d9, 0x5ebc5a090d5935eb, 0x4c67729f311b1b2a, 0xdf47c5af78d0cd3e, - 0xe0296ad6afa54f9c, 0x3a90c5189ac4188c, 0xbc3a16f21ce76183, 0x744d4f70497f15a2, - 0x7b54dbc3ad1248a6, 0xfdb2baa345c63b8d, 0xfa66a06424f94de9, 0xda13480398fe97e4, - 0xed1ad5262c7133a8, 0x44a71c0c162235a3, 0xb62c7b01f908010e, 0x2fc8cf1fd19b00fd, - 0x3e905fb36b90c768, 0x5bf098e651f998e5, 0x01f537909059fced, 0x04aaae08626fd701, - 0x58763a277d4a7f5a, 0x9c1026a505db0599, 0x89830d1917f16886, 0x29674702a3b9cfe2, - 0xc6235b01d18d3814, 0x507384791bd9be65, 0x6a7d2d1bc475fd72, 0x2db3709e6d642e5a, - 0x83161c4f921778bf, 0xd78961d296658a57, 0xeabd3f456c870e15, 0x54cda079d87c5bd3, - 0xcaf5de89b56f4c9e, 0x3aab89586eaa7c09, 0x97226f283c1f6e91, 0x63dd7512c5e470ef, - 0x15119322fb7a9c53, 0xe972e17f01e2710e, 0xcaad4d700c5d137c, 0x5e389a8e844a600f, - 0xc8d030172ac318ca, 0x05ce808abc1c499e, 0xc1e7c3bb31350837, 0xd4403f8b3fd3f9dd, - 0x8f60b752b14b47aa, 0xe5d0c292306104e4, 0x08848a3e6cc8b600, 0x12ac166d07883ea9, - 0xea4e6aa0ae570ade, 0xe7192d752817e167, 0x7a4ea8e544e493ba, 0x7d93a8ae89d186c3, - 0x6e3880118c862401, 0xb371faa7aa5ae53c, 0x76aa09eb29fe70ab, 0x9957b772f51334c1, - 0xc9951046d185d7ee, 0x705c521932996b85, 0x698d051ebb062d9e, 0xfa5c4473e2e558e5, - 0xea09bf6807fdc4f8, 0x38c1274e7d8ed381, 0x1e5d067fbe7a39c7, 0x90aaaa61a104739c, - 0x337c5edcd131df51, 0x9a4de380d1da823c, 0xce8b0989787fae45, 0x74bcec3c9c34a864, - 0x84c9fb0b13f759e1, 0x08e551e06bfad8a9, 0x20b613bf659f7c37, 0xf881bec7b4ebb44e, - 0xfd30ba9f1741d72a, 0x21b7e350a9c25f0b, 0xc8254e1ef6038991, 0x286b669065b670ec, - 0xc1b2d9e15abae138, 0x6e2c84100e6c6690, 0x35b06390741bcc17, 0xdfe4fb675f2e4370, - 0xf937d490cfee1dcd, 0x595f44d3ca8700f6, 0x467bc4b755a18c87, 0xc6ea5a208cfa5368, - 0x33101d8256001628, 0x2cc7d058444a171b, 0x3c55b8f9c6fed656, 0x9bc343425e60b490, - 0x7c9ce72065a653ee, 0xbdcc745f06c4479a, 0x1a77a6309b72520e, 0xf195ac44f4ff4444, - 0x6162ac2cd7bd6069, 0xf9cfe2e5e60ab149, 0xca63671a85d33dc4, 0x019a4bdd4ca5d0a9, - 0x94ef5e3d7ed0fa72, 0x380b527200bd288f, 0xe46cba8c810febf6, 0xbcd3701e4d747262, - 0x43bdfdf75683de2c, 0xfd84db74c35351bc, 0x7484952296479867, 0xbbd05e27ac1ae82c, - 0x09a5ff63677948ce, 0x1c16bf210742fb78, 0xe819b3023a72147e, 0x603e827dfadcd057, - 0x9898f6868dee0a5f, 0x946858357590d28e, 0xd5f8d75ed25ee90f, 0xf50809b67a4438a5, - 0x8e45f5f3d9ba2b14, 0x8e10c0087dadfb9c, 0xc93c1110a296ba20, 0x83ad934525e661a1, - 0x438e1acd81540cba, 0x025d47dcb0456151, 0x9c64c1f3e0f538b5, 0xd0c8869af0e46de0, - 0x315a1192f304622c, 0x0ce81115520cfa8c, 0x609443d12bebe0c8, 0x25e00a01757bc1e0, - 0x0ce41c10def57914, 0xda12d9f3a943e197, 0x9286a4f9e679ab77, 0x73c002ca500db876, - 0xf3c022d386013251, 0x4e5f8a2171025111, 0xf2d089cbb543eda7, 0x00119cd17a35a1fc, - 0xc7e24787b52c485d, 0xeae9f71848f26271, 0xd85e2ea6b0aaf778, 0xf4478a7fc40ee2e2, - 0x7f361a5bf1a77d99, 0x03d68d7dc25a1229, 0x4693c7b8a37e01d2, 0xf7d060c9acea088a, - 0x2186f4b24dcf7ae5, 0x81df0953bfd640d0, 0x5b5e6430a51dbfd7, 0xc16c98efad26c6e9, - 0x6ec0161ff0f1f1e7, 0xff28b9bb34a21746, 0x374526fbafdbedd5, 0xbaaebbc2049deb01, - 0x22ce8a71c8acb364, 0x60c8d6391877c40b, 0xada073f73875d7dd, 0x9f31be6311b6772d, - 0x38b199763481966c, 0xd2d8cb7d7ddd2d5a, 0x7113f7fcbfd62e50, 0x6d39f2b5d5c8e9d6, - 0xb02c00f648435960, 0x6c4fc4346cd44fdc, 0xd1353b8ecaf12d8a, 0x8dc5f4627d35a336, - 0xf9d3594ec9935a4e, 0x0b5be26c5c7f7628, 0x7d7d4802d0cd7f41, 0x54932f3506fad863, - 0x9d5f931ed998ad7f, 0x6fca2664d0e360eb, 0x86a45fe21b140ca3, 0xdb776acd850c5b2c, - 0x483cbb36d882c769, 0x6302391a287b3400, 0xa99aa484e510d2ab, 0xdacc2e1c47f6866e, - 0xd22459dad0ba1bef, 0x0d169f97312cd5be, 0x94df81b22af45158, 0xadca34cdfd0236d2, - 0x0f23c710c79ad65b, 0x16286aba54b7b365, 0x239c06757741ee24, 0xa8ba66ac0b74a497, - 0x549671240ff95a22, 0x4e2e27fe50430ba5, 0x817a17de8b3a6a7b, 0x2ae7c35e791b0064, - 0x529f3f64107d4bc5, 0x3e4ec2ae1e97ce0b, 0x37137f4bc94ca307, 0x89f63655fbcf8fde, - 0x01cad2072bf83ba8, 0xcca036821c1c46e2, 0x12c061eef3b6127d, 0xa60035530a9b1ee5, - 0xad3945ccb49e3be6, 0xd4e57682c09f3f3b, 0x14c43e197e0754e0, 0x2dfaecdd72464169, - 0x208571b1bd02ab76, 0x06ee7c8f6c1a97a6, 0x05ac17cd8113f0bb, 0x746329d4bd3a2610, - 0x0ab5267b56289508, 0xf5ea63ab960a5499, 0x8845a8185c5531c8, 0x028f71d7d9d4281f, - 0xc16fda832de219d1, 0xdb315ff727ad6719, 0xd552c6234026850f, 0x602b175fb3f81f6f, - 0x61db21e60f804ccb, 0xa57c5da126cffe4e, 0x273873a94ffbc545, 0x00e6790fc073bd27, - 0xb5cfca4472afb2b9, 0x9e5c7f98214dfc3f, 0x084998669d180719, 0x21d10542fc6258c1, - 0x5f4e91fa4a2c26cc, 0xb32f156e96812048, 0x494e76347bf04c11, 0x176dcbc7c23aab57, - 0xc26d9402c6851fde, 0x193796483062b7c4, 0x90441019ea5a22d2, 0xe557709fbc8a8bfc, - 0x7e9844f04b9e9bf5, 0x39aeb8b9551a4283, 0x0bb73351d16f27b3, 0x1c985a894e8d0ddc, - 0xbad8488d0397c8dd, 0x5139b1591c5a5d74, 0x22d10159e65adb2e, 0x16474f7a122ebecc, - 0xe8f5c65256fc4737, 0x814d402692d42087, 0xe2d20d2e103bc9ea, 0xcb622eed89e9ee45, - 0x652db640afa79a42, 0xd2b667ff95d0158b, 0x55f779c0f3ca1236, 0xaaaf33d79db3cd71, - 0x6c97048e04b0ce4b, 0xd1fcc941bfb958c6, 0x5323363a4ba7cdf4, 0x2212b2211c21bd43, - 0x0b052c222a990b7a, 0x1adf04a5b10d4486, 0x2a8c3e5aa32f3c6d, 0x304320f82613a9d2, - 0xb5ea6ce470908c33, 0x0bc14cced49aa821, 0xaf71720fc74b92f8, 0xe9060a2dc020939c, - 0xafe122062870afa6, 0xb2791c460e54c17a, 0x7e6c97bd0d8e2d3c, 0x4fa632c83b62498d, - 0x87f81c74569347ac, 0x8341229dbd22c9f9, 0x7d86f9d3777175b9, 0x948e272c8e873fea, - 0x5087d53f854e599b, 0x9150d8e82b3204f1, 0x6c138b56bcefe0ac, 0x76910455d3769131, - 0xec413360d4909877, 0xd6c82ab06e245a04, 0x786b763a8c2df863, 0xbbca5c8dffe8d0f6, - 0xeeb2baa4318f4a4c, 0x45ee27584c202dd5, 0xcbf85ab159585587, 0x2f6783bfcfc80177, - 0x282e6d3872e9a74c, 0x211ee409086cfed8, 0x91952f6edd702835, 0x94553c43c2de6b1d, - 0x9b1406ff08d8a329, 0x13c103c35b469b51, 0xff32db78f16e8166, 0xb38ca3043b1d3dc3, - 0x3b46535e21f8b654, 0xf6513ebf1975f057, 0x92289b8ac2a1afa4, 0x203f861a8c40d3cd, - 0xfe3ac691e2698303, 0xbcd193fa507b1bda, 0x691da99d0c832cb8, 0x8db9246298377078, - 0x843c6e1ac7b7daf4, 0x924adbb91bd0d536, 0x945f2faac316be47, 0xd603341875df2d27, - 0x78210f3ec8fd9403, 0x329c3aa1e2914189, 0x730fe9d7d7dcf027, 0xcad54c49682371ca, - 0x288e1e60a7637940, 0x7ce8659d18fd0497, 0xb7af43b951f53d74, 0x3eb5742701686376, - 0x2af2ae55ae302ead, 0xbe2eb3de4052bf41, 0x8271bc3024fd974c, 0x2fac4c3ada675c30, - 0x66c1717075ecd181, 0x6f55e6c692fe9be8, 0xaf28d4bc2cea9734, 0xabcadf5fc11d469a, - 0x38aae8e6751d8c86, 0x368bc397df5fa858, 0x4049c232d5fd0bae, 0xce295d9190db99c4, - 0x8ea8eddbd9841358, 0xa86fc12202d8dcb6, 0x10d36dca46c45f82, 0x73e4e0f23992a8fa, - 0x9f2ce47859376622, 0x7048706310936409, 0xe6c920267343b522, 0x2a391a2758598cfc, - 0x153770cf214040c7, 0xae8979bfc4da0a44, 0x70d08a5e52ff2909, 0xeb45ed1d4a55dd76, - 0x7b63c2c4febe7060, 0x9d9fae40ba131e80, 0x6160f621c80cfb2f, 0x7b71cadd2bbaf7ed, - 0x384fe44e144b1e15, 0xe043ca2720a06220, 0x9f536f5308a1987c, 0x9df59d18a9addddb, - 0x3708a2040abaf01c, 0x72206aa39bdb49e1, 0x3a27e63413ef1a13, 0x1fe99a935f3b6421, - 0x6b28e6e7fa5727ee, 0x339bd08d03cc6368, 0xe7a100aacaf114b6, 0xae7f8462e3485acc, - 0x0a09cb489613a369, 0x9fb9644a44f7f763, 0x2f2a8fc8b07b896e, 0x1e8e1af672303ca4, - 0x60ff8c2f4e694866, 0x8c4ae1db88694e20, 0x9bbd10a657291e1d, 0xa0209b2a95a72cc0, - 0x528bb712440cfe6c, 0x4a90552b726bc344, 0x9c5e533d6ad1a09e, 0xde9586c979576f8c, - 0xc3fb4aa5c3c3a016, 0x35575b710e95be3f, 0xb425184c275d13cb, 0x7e4db6691368f84a, - 0x928f975c5c3d7044, 0xed80b078a435f1b3, 0x807c4ef963511e9a, 0x8af76dd46c565dae, - 0x9862df40069378cb, 0x56684e87f9ca6b01, 0x6e4438c9dd1fea10, 0x1ef3e6bc58e5088b, - 0x163b3efeb210a7aa, 0xf1a23421cf489c3e, 0x1d22df33dba97581, 0xd84099d333f23817, - 0x3f7cb2347ca134ab, 0x45105390b272005d, 0xaab5662be8366bee, 0xb90f9476e1b65e32, - 0x7a573dfc1ae4d1ee, 0x99c5f918b0af652e, 0x3a42e45fb5bf3ac7, 0xb694812df84d30e0, - 0xc2a1dcb79a8488e4, 0x6240741d451e4270, 0xe15273400f3a09f2, 0x951b1322706eb67d, - 0xc411b9dc4377f2c8, 0xf103d4274ecd1d43, 0xb8b0898f39552a65, 0xe94ded4ca5e72349, - 0x17f0aecfe8922abb, 0x045456787be4b6c2, 0x103780c1c335ecd7, 0x556279f2a1e5abac, - 0xcd08ffcf14df5527, 0xfdd84094a180b1f1, 0xea30faf0117413aa, 0xc9f667111e685f95, - 0x3773e2598f695ed6, 0xbc0b1d103ed27a97, 0x51b36c5ba862413a, 0x9fbdffec571c4893, - 0xebc9c1be46648f91, 0xb6edb7843887c04f, 0xa7ce6f2867701e78, 0xb9a9350617e521fa, - 0x812e75ef018f9aa1, 0x7c5a872a99cac922, 0xa5341cca0fc2dda2, 0xe89e815f0a3f006e, - 0x39b5ce55b9ae49f0, 0xbb2e9ab2d05b83c1, 0x605ca0ca691f15e3, 0xbfb6aac2cb7b9d63, - 0x9f89434addebdab5, 0x6289f97b55fe25b1, 0xd493d7f32171208a, 0x6598c7e3a72ed5f5, - 0x3a3dd7e867a0b2a2, 0x31bd932cdf7d3457, 0x899eaf5a85d58a64, 0x46e48de0ea303e45, - 0x1e208e260d85db0a, 0xf585c62c4b677b60, 0xdafb7b015f342bd1, 0x143f841aaeeb933e, - 0x4e57c98754ddf9d8, 0x32ad9d5a201d84fe, 0xbcfb1d9a852225f8, 0xca8fee9f9c79e9f6, - 0x5237e4626a0b7b6f, 0xc808c6b492db16c0, 0xc77cd68995563304, 0xfc9cfdf0e9369edd, - 0x23d52a66404cfc29, 0xa7b2b6e4e750050a, 0x9f160ba2423be3d2, 0xdeccfe881a818248, - 0x4a944652c326b649, 0x700db299489e2eaa, 0xe6de20736d8b94ef, 0x533959c5640b3045, - 0x4a8f2109f488efa5, 0x40530de89f44e647, 0x654f1a0785284c44, 0x16a7eafea7d78f6d, - 0x426056c9203b1697, 0x73b12751e84b9c08, 0x03985cba93691d13, 0x71cf3e4e223ac323, - 0x5dc87db669a97374, 0xc1c21d0e9d8f3e27, 0xae46691357edbfc5, 0x92dda301351ed190, - 0xb96ad9b88f077855, 0x1921e4aee69c0a1d, 0x40177ec46c0c29ac, 0xc9f869cec12fd253, - 0xd7412debf2239a8c, 0xdc89fe5550b1ef03, 0xb7d60581d3ee4cfb, 0xfde08fa2813b22f1, - 0x79033561a71e3767, 0x05df4be385cbed09, 0xaafd150f3b5b2bbd, 0x4b77e95dd4b0dab1, - 0x75dc773ab579436a, 0xd04b9dca0eb8d58a, 0x25725610b0bb9088, 0xbc14eec8ffd61260, - 0x3443e8681ef7316e, 0x34976fec0557118a, 0x1180dcbe6b6dbba7, 0xc58b565ab832bea9, - 0xf738e5fccbaf95f7, 0xaf3ae87f2c220cb4, 0x50387103900e16f6, 0xe856a2ef4986ef31, - 0x89ba2e0032142a28, 0x5957c250698b393f, 0xe71356446e6d6de2, 0x31e4f223acdcf926, - 0x941e6af230d58dd5, 0xfd7ebcae1b213259, 0x99f6503312dad593, 0x5f14119683b53b8c, - 0x91156161f1855e12, 0xa988c81743b2fffc, 0xbf58fb274ca611f9, 0x22c8676cd473265d, - 0xc1703a12d4d04981, 0x3fa0d4fa4dff7a55, 0x197d831e0d22713c, 0x1fddb57be04fdf45, - 0x08662dd0a8847b0e, 0x043ad20b76855ce8, 0xd696ac6c2d481f3f, 0x2b9148a25b35361b, - 0x6e7cfc9a62ff5c9d, 0x5b71fccf789e0ac0, 0xe8d3ee260fc896d4, 0x80272974f836f14f, - 0xeabb5867c35f6ca9, 0xa8255a79f2503033, 0xa28bb1f83fb120e3, 0x5f703c1fa401d781, - 0x714b123ac88d456c, 0x05c54c68ff17c5ba, 0xd64a4b3f1e516c5e, 0x2f7f94bc0d42c213, - 0x1d58c2b779f487b8, 0x2448d9c8c57a76be, 0x07d163c5eb43acab, 0x309d6f0deccc76b4, - 0x96511a683e7fb9c9, 0xa39e4c7b6dfc13c9, 0x4c85db7dc557fb5a, 0x9e266c3e3fb1e1dd, - 0x40834a718db08d57, 0xda7cfd7647829733, 0x8bc970a1b750acaa, 0x395559f7847ee4d3, - 0x5458d840cef87e64, 0x29409fc0c7df5f2a, 0xd2e0e782013cd37f, 0x4c5f40b8474bdf3d, - 0xa7246a6e7a7eb61d, 0xccdbd4b73e6b397f, 0xc5658be7057632b0, 0xfb87dd8214c272cc, - 0xf4e9d623ed79e7b1, 0x706be8099bf421ad, 0x5f590cccc073750e, 0x3c1dc743d2b6e723, - 0x60e000d0eaccb4dd, 0x8b670ec910de6a0d, 0x1d95acd708fbd139, 0x114643933aeb7f30, - 0x5b1a9456c747a1df, 0x6c3d65bfb9eaddca, 0x31433feb1e564c9d, 0x6fd6d6072436dc33, - 0xb4c9362581601820, 0xcfdf5d2aa4c2ebd9, 0xb758e105603ab789, 0xc37adb13fdb4418f, - 0xda45cba27c8acd4f, 0xd624660fddcf471b, 0xc287708b80a31124, 0x04d64f520682b650, - 0x2192ba0917cb504c, 0x202b31d33d95b344, 0x83ee79ad1f91e2a8, 0x075297fd56d43fa0, - 0x994ab39804c78963, 0xdaf4e71acf11719f, 0xcb6c57c4a8700dba, 0x82e7a9ab0561cd02, - 0x1aded4c52766afd7, 0x8413a899ca576d7a, 0xbb5910f3aa40eb0b, 0xb24a4c7d413652c6, - 0x05d25b7444a5ebb3, 0x9a867e74e997b1fa, 0xdb5b9e03b6fe385f, 0x2b95963f5d721b5a, - 0xeab9e46670b8e263, 0xf0df63629fb72877, 0x49f619e2d5e8203b, 0x80f9b9b99191334a, - 0x067917087ec4dfad, 0x6148be814548f734, 0xd7e76199065faa77, 0x8cd44c33225f4247, - 0xb965f2dff8947ad7, 0x638b66f45b99e998, 0xe7348488dfab511a, 0x8f224dad126923c0, - 0x706f0f04922d2714, 0x6ae1d7d62bbb2474, 0x9f0e309ee0498579, 0x287e22de2dc452de, - 0x3996661c3b6f811e, 0x7c8d902d2602ff24, 0x1950ac372328737e, 0x0ff921152757dc60, - 0x7cbfbefbb14f1593, 0x2d557952596605fb, 0x7a9dd18b7fd5c74c, 0xfc823fb54000f963, - 0x378b1a68dcda7786, 0xc97ea36ce340a4df, 0x13976db93457a7f5, 0x02dbe618e9c24f93, - 0xf91f7ef12a79a72c, 0x215df089d2aab3ed, 0xc94aeca03a65bf12, 0x42fca940d8f47f33, - 0x9490ebe3a70ec214, 0x3060b024aa44f987, 0x2e3a58656152fdcb, 0xdac96028bbc5bc0d, - 0x50f96c7a77f6393d, 0x8f26a898ad17c39f, 0x6e22ec34015bc3c2, 0x10fe7c3a9da4f0f4, - 0x7735d8e93d887c8e, 0x9c72490fa970b813, 0x3f688bf1b009a651, 0x0ca8f32a767fbbb2, - 0x8fba22fb50ad2c21, 0x9d03e3216a6fa3b7, 0xda04817036fa7aa3, 0x20ad60c4989893d8, - 0x605d08919fb75f54, 0x451f497ac9c15c32, 0x1978bd4cd8479ca2, 0x2f17735ebe2188ec, - 0xbae9d66721ad7a90, 0xce700d1f2e2d1c2a, 0xd5898877d9c87f2e, 0x7a00ae55a40508f9, - 0xe8b98dfaceff3cd5, 0x7e8b1d6bf0f43d61, 0x2db3db13e904638e, 0x41db614c48b29c99, - 0x0b227c64d5b9fba3, 0x964e7285f8dc461c, 0xc92334a5fd53026a, 0x27511338ac6c71d9, - 0xbe09415c3a8a6416, 0x0455c6d269f628ec, 0xdea4045907fc8c69, 0x4547aa96fdd62769, - 0xb9584047b85ccec9, 0x715b30a35798b7e8, 0x51000d3c93f4bf4c, 0x360d09170121b237, - 0x5d11bfede3511c50, 0xa15fbd78d9bdc30b, 0x21d2774be2807eda, 0x6d2919ce4a8e9390, - 0xa4cd90c5c538bf47, 0xcbaa4ac0cae2203c, 0x4b2ee15900781c82, 0x0fb11b1173a3cf64, - 0xab897dec7c447534, 0x9d092cf18a07c368, 0x6f18ca32934dac59, 0x67026f48949c4f2f, - 0xc689d8f293613bb5, 0x5e67a6fbe213991e, 0x92429b90f6996912, 0xea7f13884772eaed, - 0x39bd11680d44df75, 0x395640f6ecbf36cf, 0x9071ad13a61446de, 0x80ca035932d58927, - 0xb417d18b117ad802, 0xd1f8b12680122c57, 0xcf48c1d4e991bda2, 0xbd1c34b4e4f79789, - 0xf47f965c16bfca22, 0x86fbb0330100afb7, 0xe1a632b40f8df09b, 0x93d793260a887f18, - 0x997579654484af4c, 0xe5420615d98477bb, 0x2956a9f4a8e41621, 0xa5f7291efad9cf7c, - 0xdbce736d7eb64c77, 0x43e246be93393fb6, 0x30016776051ffd83, 0x6973dc4a341d2cdb, - 0xce97ebc49b17d710, 0x640f2b195e23eadf, 0x57d1a7e5940ae014, 0x44f4251d738364cf, - 0x9a677ec483e624ed, 0x29a5c1139a23be02, 0x3ba2272615dd3108, 0xb7493efd927114d1, - 0xf6405a1d83266fef, 0xe37418529ed24265, 0x0e2a242ec600fa03, 0x05aa0bca789d807d, - 0xf34dfa098ba1574d, 0x6d731c6c730fe91e, 0x8a910b24494beafc, 0x17acf76602dd8ea5, - 0x21aef0da1c452807, 0xe1f3bb1c887029f3, 0xdfc2f737c81b88e8, 0x94371e46e34ed838, - 0x9d896791a5d61b9f, 0xb149268a2b7f1241, 0x10ddba716b542d16, 0xc6d066c66df26a7e, - 0xd0b6343e36c8e84f, 0xa38f8389b03fc0fc, 0xdc0c1cdf60d6604b, 0x73bc679d7c846659, - 0x9dbb7cfcc9591c47, 0x9ba66b408ebd59f5, 0x477dbd61ee8df194, 0x03ebb9460ad59c0f, - 0x7e78fc8c934098f5, 0x254bbafc8d6727df, 0xf426ea1ac256c248, 0xcbfc76774c3bdbe7, - 0xbe4f97464aed8b05, 0x186086dc18a5b961, 0xfd9ab4f368c31fd0, 0x00365d1f7db96185, - 0xb5309d35f2f68e23, 0x38c420731150765d, 0xffa055dbc4eb7c42, 0xb198876cf3d38946, - 0x83aa1c989315a1cf, 0x26f1a06a94973086, 0x6451a80c8a46b2a0, 0xc92c2dbe2b5a22f8, - 0xe258f4ba3c5b0809, 0xadc38fad1a6c2c92, 0x9c4fa8cbb62b83da, 0x0f96163e0cbe99e8, - 0x4028d95fb4b42d89, 0x342b1ab6286171a3, 0x4188adbfe7ac8931, 0xbd9f343c9ec2683e, - 0x84e519ba2c1f22ec, 0xa732822ccace90ef, 0x1ecdda3f09d296fb, 0x3237c4ae32e70395, - 0x0f096f53c2cd3f4d, 0x96a34a2ce21efd46, 0xd78db462618caa7c, 0x8b3ababd1ee7cd66, - 0x4085e2aef38e1af5, 0xf57501e5819e22a3, 0xc21b1e1bc1c6bd0d, 0x177ca29689bb8b53, - 0xd97848dfb447eacb, 0x52a7916debed6210, 0x332e2693a49ee362, 0xb03cf0221c7ae54e, - 0x49cb81dfb178d2b6, 0x8add6c72622021aa, 0x4feff3e7e7dfb9db, 0x6d0bc767dc840bcd, - 0xe8c0c052820a2c6f, 0x0bda6f94608365dd, 0x427f3b1d1c11cc6f, 0x3f62b362c60b0777, - 0xd1bfa232852b0b80, 0x264f51d020bb5882, 0x11dc3ba926d1563a, 0x407b4bd5f77e0275, - 0xa81f362bba062636, 0x60f5922210987abb, 0xe78cb2f2b0b1de0f, 0x483135fee7e8bd4b, - 0x91a25225894cfe9f, 0xe0f434b740329067, 0x06a5cbe1f2405ae8, 0x610e9ff0d0c7ebab, - 0x0716a976bea1a48e, 0xa28c12da389f0813, 0x9d5b20ed9e4106ff, 0xdd8e08e716651c8f, - 0x794539c542181c5a, 0x07daecca4cc20e7b, 0xa44694b00220be3d, 0x61b24c38fc468649, - 0xccfd9305776e8880, 0xc095d50b72217018, 0x04a2becf673ccabb, 0xa40c1dfb6102a4f1, - 0x7477bdc152f5a89d, 0x1e56660f2cc4191f, 0x0e073b4725506834, 0x09fc4d6a1ae73446, - 0xe8498661703fa9f0, 0x91efb1a5f0ea4902, 0xa1dda0f0236ef0bb, 0xda5dc6cc75c59a91, - 0x9697ed68c75a4453, 0x6b3e1f9f365e3823, 0x3570e76da5f1e1c2, 0xd417ae9441e31bd3, - 0x7f8c580a2256d7d9, 0x0d479239cdbe2c87, 0x2bef8e771ef43882, 0x7b7523d22c0a740f, - 0xebe043c45f687ccd, 0xc635012134fda0dd, 0x141c34c7dbe90318, 0x248389085a8af2fa, - 0xb5826bafed0a71e4, 0xb32e644e2dadbeaf, 0xd7075804e2015a4e, 0xddd475680b32ad37, - 0xfaa0fec564696df4, 0xbc0bb0a7011009d7, 0x03c4d3aeaeaeddc0, 0xc7bceac25c2f7739, - 0xff4c9b0f7affdf42, 0xf40810a8f11f873a, 0xa6c8b67b547bc894, 0xec8fe88bbdb4aa64, - 0x77e07ae8dbbf0e90, 0x222f30bf842db487, 0xf9dee217ff274d73, 0xbd792ec7d1cd92dd, - 0xf2e6104b97432a68, 0xe57c299dce056260, 0x96f40c86a150e6d0, 0x061e3731cf0a3086, - 0xc802ea5b144929cf, 0x13c59a72845d9cf2, 0xae7b6a693f0ae30c, 0xfc2d83b328288d91, - 0x8b0e4c6f040cce8b, 0x08b7588582b9322b, 0x348c93cff7849da3, 0x1b6e6dfcd6857c48, - 0xc145dd7315effcd9, 0x29fc95209c619a5e, 0x3bf44dec5e195a3c, 0xcb980ecd064dec1b, - 0xe80a9c2d52491604, 0xd2f28159e1b5098d, 0x6329e761fe16f39c, 0x69cad58eb4e920be, - 0xe0d854122380e2bd, 0x0a64c596fc29683b, 0xcde63737a1b99515, 0x1b65338b3860e90f, - 0xf6a6a6c75ae026e3, 0x36445fb703e785e6, 0xa3970982596a5aa4, 0x98112112628b2c64, - 0x0abb071decf7e885, 0xc37b841b7493513e, 0x47c93bc89b5eafe2, 0xbb11895985dbd874, - 0xb982c458617cc54c, 0xe53748353da6c825, 0x932d71b9a4e8930e, 0xf9381f6ac1f750a7, - 0x1e6e882ee3e15530, 0x368f6292b6dce69a, 0xa490a15fd9d89c05, 0xa98cd7a37c317e8d, - 0xd5ec1390eae2e2ee, 0x3434479e98c560d2, 0xadae0de05af244c7, 0xb2efd76e1a9306c5, - 0x5dda21a7c1a852e3, 0xe7df187daf1c9cd9, 0x765e0c125fd634d7, 0xace47294e9bdf8f8, - 0x16f095db8fc43856, 0x6a20d420187468e8, 0x181e3fba8ecc14b4, 0x666fb2f855c6cf43, - 0x1f53aac42aad7c11, 0xdbbfb5431eba8c18, 0xe4177151e01a4fcd, 0x9c7d1dd8aa9c0c28, - 0x89d5ed0575ca5fe1, 0x5e4d2f97f234a3e0, 0x8b2e7b4697a1731a, 0x4470e94e89b57aa8, - 0xe7a0b475662efe5b, 0x40c7e93cce4e7976, 0xbf053864d6249fb6, 0xdf1716b7be57d8b8, - 0xe0e5559aa9e8bd20, 0x3923206500fd552f, 0x9e84edfa318d7f22, 0xbf39ed52afbef286, - 0x14aab6f642d44a74, 0xb0a62fbfc5892235, 0x1a366165f7688490, 0xd2e08073c7bf45ff, - 0xa2d0e4c4c76cb6ba, 0x9f4b26244eeedbd2, 0xaef7d3ea371a5f2a, 0xc332763a928251fb, - 0x27ba32e191791c31, 0xd9c4e9fa0ceae1e1, 0x8a20645922d83bae, 0xe5398283813de7fc, - 0x4bff98a3422693fe, 0x9c039e9d8a8bf724, 0xe749a5cb250e2e39, 0x0fa6eab35f791333, - 0x72a7869640c1e204, 0x7d1ef8afee6dc328, 0xba36a840bf4f1aaa, 0x60b1862617df2f3a, - 0x48311dc258f6eb3c, 0xcd9cb32476b4ce2f, 0xb6dd7305fd97ed1f, 0xdd0ffd076511fa58, - 0xdecd394ca0ce296a, 0xd860373adc891518, 0x5ab2986bc355d49f, 0xee2649f41ab3ca80, - 0x3f541112ffb94f2c, 0x9f2e78aefc4ce245, 0xf400807e090b5c22, 0x3f68634ea47a6666, - 0x0c4092fa24184d24, 0xae83816f3e802aa0, 0xfc6c91cc37752a95, 0xf2459acb362c0bf2, - 0x1bb3e2abd387d84e, 0x80819a92e4d3e560, 0x3816b01fc0bdab99, 0x2607ff572720702b, - 0xf54272b250f3dd35, 0x0ba051fae77a316f, 0xe4a112cf58d2adfd, 0x9c8e263d82687016, - 0x2c9e53f5bbb0833b, 0xf3f5b98151fa8144, 0xe659b47342a503f4, 0xf6ab3a4062fc2b98, - 0xc6636e72271b9f94, 0x93e5b8032d040c1b, 0x60ff003dcb3e3d53, 0xb6ef2a0fb09b4e79, - 0xbcdd177ae7635cf2, 0xecc641394b4b4fbe, 0x3e30ca5f9a396716, 0x8fdf9dbb6fa5eeeb, - 0xe3c26acce8829780, 0x067a655315b7479c, 0x9423f280c13c40e8, 0x591b708cc6886a62, - 0x8467877981665f70, 0x3dd84fb9b0bf9d3c, 0x0619ba0ffa2fb3cd, 0x3c0279daf68bfd95, - 0x66df22b793e24fd5, 0x6aa1fcc8caf415d5, 0xbee73e922bfd1009, 0x8352368d9e053109, - 0xb0af8ad0e8b2cf12, 0x43f3d64291b5d1f8, 0xbe934fa98aec9e4d, 0x0e8e8621f735398d, - 0x57691d31e0aeeed5, 0x6f9fdfb2c77ad2fc, 0x9ff658c7991127d9, 0x2b8a90193bf378cc, - 0x469db0a3b183380f, 0x08afc274aeae39c3, 0x86638543ab7b2090, 0x6d8f4075235936dd, - 0xf314b42740a14d61, 0x7176be562c4bdb32, 0x659466e2de0c46c0, 0x37274c53add732cc, - 0x491218b323b2322e, 0x05d4dfefb3eb3189, 0xbdc5455c5675564a, 0x2b0670d558e56a7f, - 0x005b11ad29d7f580, 0x21a6336d7681cb51, 0x5d299f7ace7e1c44, 0xf376beda0cda1865, - 0xefc432133f963a1b, 0xad2e9711b91f2a27, 0x6e494c89c7655ce8, 0xcd440112bb81592a, - 0x41415fb1da5c87f3, 0x104bcd3cd4198a7e, 0x008415b4e6c607b9, 0x728e143643cdd79d, - 0x3dfabc8980ba1cb8, 0x09b126664710fbb7, 0x9e8d877b93e6562d, 0xcdd85a537bc2f4a4, - 0x92cd1512ab36cac5, 0xfb81157b9ed0fa24, 0xdfce2825f63ac7f5, 0x8eb4949d7c5449c4, - 0x026807bc80cd2f4f, 0x07b36f639a1b51a5, 0xb42c75313d1f6ead, 0x26511a245bfcea41, - 0xb1518a4088630458, 0x3092bf803e823a73, 0xe14047f410cdc10b, 0xd6f6d0e1c197858e, - 0x4477d03908e91b94, 0x3c9fa57383bab117, 0xef76f78b339a789c, 0xf3ee93b50538e108, - 0x3aadd5426afc3f23, 0x2b414eb936128dad, 0x4e3be797dac570ac, 0xc0282665fa849ce0, - 0xa1d8f8bba0cf8ac0, 0x395fd4da82b1374e, 0xb56aeb0a5cb26a75, 0x21642cf1ea575a54, - 0x79502e72c1dad50c, 0xe561df05718ecc20, 0x00d8b260bd8e5374, 0x764ddc5609a5ae5d, - 0x2fea1382d89a30db, 0xd8f4568b81ef610f, 0x76da90fc745a47e5, 0x342a6e541e9d0417, - 0x1a152736b5c5e31a, 0xb350bccf6ed30a47, 0xef9b31897c1cbfa9, 0x62ee149c3721a54b, - 0x7f582bae5d0bd0cd, 0xafae54979aaa48d1, 0xb5fd1870ab9f0b4b, 0x42bda68a81149cf4, - 0xc6dee9e9f076c19d, 0x954bcc961d4de9cf, 0xabd0cbc8e8d9d9a1, 0xb4580790be9b7980, - 0x784452821d47e832, 0xd79df172ffe433ac, 0xbfd4b86c15e55501, 0x7676603bb4880371, - 0x18e6a5d2d5577571, 0x9fab856341d54d6f, 0x12cf2e87eb12db42, 0xae9bfcf91028c85d, - 0xdcc366489da06b88, 0xa3827452f135b9b7, 0x9d163161fdd46ff4, 0xaf67c71f33eab3be, - 0x02ffdfff0f678591, 0x9d678a76110cad89, 0xd38d2486376bd009, 0x6b634baafe4bc3bb, - 0x1951a5ae48d308a4, 0x5b96ac3384adc05c, 0x2f07bc21790aa79a, 0xd276a51447486815, - 0x1b145e9b79d76658, 0x76bba0b4ce5049fa, 0xa2d07ac07770d788, 0xbdc0d5ecedc8e4ef, - 0xf46310c492d77bac, 0xba2c55aa4daa7a2e, 0xe37b6ffc2573072b, 0xf21972bcc9e2c356, - 0xde741bc07222e6af, 0x2061faeb99d66cb9, 0xa6126c03c6991817, 0xad3e4f95e403b0c0, - 0x35760a41b3e85db5, 0x2dfdfbee70a314bb, 0xa163ef61d1ba2da9, 0xf8d8ec41e251c1eb, - 0x76e11618235afebd, 0x438002db719ac14e, 0xfbc1578171521697, 0x96112dd18a6bd3c3, - 0xc1c9ddb7041d0650, 0x9ae7527fd9919dc2, 0xaafdcdfd74e078ec, 0xf71a70d3c52736bd, - 0x4cbe3600ccdb9cf6, 0x1f02a52c59b2be74, 0x5473482c3178312f, 0xc36ba15615dc9803, - 0x7dd8bd231f1fc0da, 0xf9291df737b537fd, 0x16de34376147b0dd, 0x6b1fc8e9ea8e8e81, - 0x5ef81a47717b67c6, 0x7038a9372fcd04db, 0x13126d6d0ba6c131, 0xf6cae068fe5a0c94, - 0x9b0cc4d8a7f98920, 0x979d179fd387e569, 0x8a74214862a9170c, 0x45f7d20fb7766029, - 0xa2f5fe86d87c47e0, 0xc26d6c27793f0d23, 0xe5b540046fb4f388, 0xb5d15f7d9e1b7ee2, - 0xaabfa187ee69f6d3, 0x076027ecd309d4dc, 0xbe08ff62e5a461ec, 0x201b91ad4e52384d, - 0xd312f0f39357d868, 0xde11041766eb6f9e, 0xe3d102b1035adbd5, 0xe3128eded0ba04f1, - 0x951000f1c9fba7c9, 0x39c6ed7edcfc35ed, 0xaea312a98376a855, 0x7a6aa72a5103a212, - 0x26b2c36358189b2f, 0x33d2d991a1b71c58, 0xf2dcc15b58c9a1ef, 0x3b56f5c731986f70, - 0x1a079f0e4bb5d9c5, 0x4bfb48d09919b5c9, 0x8826ed2b6dd3094b, 0x8b064b7caa428f97, - 0x95ef2497ca3b015b, 0x791265a8c4e96e56, 0x073724879c8c6ec1, 0x3bdc9cab5fb4a4ab, - 0xc4b38ae3cef98615, 0x495747e29eb17227, 0x3fa0f98dc4819bee, 0x917c6c66a69f736f, - 0x4fde46ae358b06d8, 0x2eedd383b092b1c6, 0x5d6d20c48404e018, 0x9979f406fa73fc22, - 0xc723fe61334eb916, 0x04cf48a680b3165c, 0x122ab4ba8ba6a967, 0xb0aad2bf607b07f7, - 0xc3e593aee88afdcd, 0x70a3b1dc9d87a3e4, 0xb3525636c03b4ade, 0x7a648a4c6eb5f695, - 0x5535916bf2c041d6, 0x685e4e85678d581a, 0x95d3e30c4cce25b9, 0xf2005369eca9c3fc, - 0x6d22463d8a760895, 0x3ba0b34af5cda4a7, 0xed2111e811b364cd, 0x6d100bf7cd920721, - 0xed7ceb85abd1e184, 0xd872c3786ef49fb8, 0x1d9015f66450d242, 0x6ffe98a91fc5290a, - 0x6486d9b82aac8260, 0xea290975ef381e71, 0x394e1b2f7a6e853a, 0x43d30943b2ca4b31, - 0xb6e7963ead61b1eb, 0x71fea06959c3ddc8, 0x7097db1db429ec19, 0xe4ac4a7af5061bd2, - 0xb46038f19cd05ab6, 0x31f90ae5e51e141b, 0xe819b8468582433e, 0xe147e2935921a50c, - 0xb503656ea67c5c82, 0x9278ad04f1ad89f0, 0xa21387c32b2c7765, 0xa5ea577352aa4184, - 0x44895fd59d61f926, 0x07b4267107b23302, 0xeb2c929fb29062f5, 0xb3606d6fb8042363, - 0x8d771fb68159e975, 0x3a4eca522ae02123, 0x82377672389cef26, 0xc4ffad7d01e0f7e7, - 0xf15c752cf76ad395, 0x2467d844c6e53e5a, 0xd96428a7f814a14d, 0xb876599d58c55752, - 0xef458f4a3216e716, 0xd4e874af35d20d5b, 0x88d444bd9f5ec741, 0x7a61a66f036e5d52, - 0x1a2055a0b4867eb2, 0x61ab001cd8e79ca5, 0xa02458f06b7587d3, 0xfca0656a588757ec, - 0x5bb1d1a62febcc7f, 0x039bab1fe708f85c, 0xa3133f4064f3067c, 0x6c10775c5039c333, - 0x4e34bb1fedb1c239, 0xc7b713d7843ca040, 0x53720d4afdf108d8, 0xe875341166e40bee, - 0x3775cdc55a548ade, 0x373710aacf98d4b1, 0x480ec684828d2846, 0x4acc2a3ce4aed95d, - 0x4416bd9713827a6f, 0xc5657bedc52ca85d, 0x2b7bf9a5c4c5c4c3, 0xe165efe664cea75a, - 0x7e0ebab20683b301, 0x3d5fd2d21542c584, 0xfbbe0d863322e79c, 0x1ea69ae0a21a6355, - 0xfca8f86d86ddd59f, 0x12de1aed13e6e13f, 0x050b0745841ba09d, 0xe7243296aea1a094, - 0x63f1a70a15475146, 0xdeff6afd1ca7f689, 0x08b2b01ca6336ef7, 0x1092b68187d87499, - 0x0f862a5c4e4bb5ca, 0xf789d66611cca518, 0x99dec34affd3840b, 0x4536c12f89790e40, - 0x67d072a17e1ec793, 0xa9ffb5b9073dc52e, 0xe82320cb5e34733b, 0xa57fdafca2ab603e, - 0x69726ebee805108a, 0x71f92f565487d39b, 0xa7d25c200e01ced6, 0x9b6ec8b2df4b5eff, - 0x46bbe1b46f71223a, 0xab58dcf32b3945c6, 0xcfe9a923fa30548f, 0xf17a3e528c1b403f, - 0x54f647fc1285574a, 0xaa2b2938d21b7b78, 0x76cc5022eb531b01, 0x3e79bf70d265e70b, - 0xe77820f85ecf1310, 0xcff89a3e49c990eb, 0x69d85c4d40d8f406, 0xa8017046a69dc12c, - 0xb1bcd418fa762659, 0xec576ddede953559, 0x178dae0125c13de7, 0x29c0771245769404, - 0x4df70525bd2ec33c, 0xdc32bcb3888a33e2, 0x3c48df959e1e86d8, 0x356ad9cd2e9bce13, - 0xc5757c399cef23f8, 0x8c6c15e67e2d8c5b, 0xbce611f2ed2a3f53, 0x08a2430396126062, - 0xecfc7c3e8baed266, 0x7156a42041e4f3e7, 0x3c2be02d9aaa8da0, 0x0269357dc4fca78f, - 0x301e9c823dee0e0c, 0x96c588306ef9e330, 0x452b23c9455e20c4, 0x9829c7c98ea86d06, - 0x1c779a551b3e66b7, 0x9a6e75fee7f1853a, 0x9d555c25ae2c180a, 0x5af4bb86318e2116, - 0x8b3425a634a6f0d8, 0x0885a23babaeb2ad, 0x719e10eadffa24be, 0xa1535e3dda36c28f, - 0xb70f842b5af8435e, 0x37a8d30d13bf843a, 0xda3d76d8ad1b5d61, 0xe2431dc31fd8fac8, - 0x4ed70865c392f94f, 0x89f3596e01245a21, 0xac5d432b604d5ba8, 0x6586dc13b0de01d5, - 0xf247169161ff3f2a, 0x579ffbcaf62ae1c7, 0x1d6a3f1cafc7e1e7, 0x820dad0c1f6dc67e, - 0x4c1f57bb29967bd5, 0xd09a2ca91aa55c6e, 0xaa4151c9de67f6dd, 0x87c3e646d414d653, - 0x634425b3ae877ebc, 0xfe6e67342f746b48, 0xeb8de2f05794ab82, 0x9a446ccbb3c84839, - 0x93349b75a338e5e6, 0x642ba855c2aced4b, 0x79d0571fe5582bb1, 0x334543300e63836f, - 0x20523b929a058022, 0x2e7bcf5ab0756149, 0xb3f520d6d1103c91, 0xfba5755b4feacdf7, - 0xc12bddfe1f253bc5, 0x7acd8cb202b1d9a1, 0x53dfbd82378798e9, 0x4a406a84d253bc2c, - 0x183f0c819b7c29c7, 0x9047354e786c44b5, 0x4a26540b61515bc5, 0xa9015a150c32c3a8, - 0xd1022fa5afe67462, 0xc2d4f0019074f297, 0x13f0a4cd55376b36, 0x1e6afe2e85feb815, - 0x51242e1de5638e78, 0x20cab0ec110561da, 0x73393aa3498cec89, 0x30a6a6d8759c8cd6, - 0x10c7219f12d3f654, 0x736b4bd4aa2643f1, 0xe7be54207fc95aef, 0xfed62eb5c7c877a1, - 0x2f18579c0a5b32d7, 0x954f0c82498d7977, 0x2670fd92b3943d57, 0x80436bcbe40dc0b9, - 0x505dcbb69b69e594, 0xe347bb418fcb7f3a, 0xaaf5b4f8efd9de56, 0x48c2c3ab5c35595e, - 0x44a1241197f6bda3, 0x2ca35f40fb68d85f, 0xd3f023250f3e4297, 0x9ed86826952904b4, - 0x695eacd5ddd9ac84, 0xa94001bc2d23e89f, 0x2c427e95c9ca9897, 0x68650d0077f6d2e6, - 0xab95eaebb03bad61, 0xd74a0c0a60e100ab, 0x5ba2bd69eccfdf63, 0x7ae89826231a2966, - 0xa40c9463cdddd7c6, 0xed228ec6528fb423, 0xd33ddd7e8ed5c5e8, 0xe4cbc63381644900, - 0xebed3fedb59f68b3, 0xc2d7504b8c123cc9, 0xff2b7a600daad47d, 0xff2daf312ae6e8f1, - 0xb631e821a282811a, 0xaaba6a171954114e, 0x2156d5efd072f89c, 0x6cc090fd8ccd6298, - 0x7d83358207a00e60, 0x7ffdd930ed5b393e, 0xbc85f92d10c3ae33, 0x0488e182fe76d6bd, - 0x1c55488b05bef1f2, 0x38d2406c013c2e84, 0x35de9a1970c11810, 0x2b7afe93f81a3d56, - 0x8974a9f85dba4963, 0x5e26a39b700917b0, 0x9a76f45979b13d91, 0x84d7a8c2b330cffc, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9fff63056ed06552, 0x16f7b4c87092131c, 0xb90e9bcfae4dab17, 0xb8125e9e658252fe, - 0x8c25dd13a076c9a5, 0xbc07adf736a65241, 0x6eeb1e51349beaee, 0x26bc797fdb9eaf2e, - 0x2f5669b3ce11a3bb, 0x4fdba2b7c198efe9, 0x2ea8ac251cbd028e, 0x7f9376badf0a7d72, - 0xc5e86cefe0ed6b8e, 0xa49090bcbd540085, 0x62d4801a6355db10, 0xa1aec9f3a4f97c99, - 0x6d4b570bd67ad062, 0xcf1e644453990169, 0x80c17889229bb302, 0xc52a351d68069c14, - 0xa6ecf2981286716c, 0x47db518c30741fdc, 0x2d012d9e8165b952, 0x03db6b0164ff0eab, - 0x96613a86eb60f15b, 0x6de318d4ad689e96, 0x89845897fd3acfe9, 0xf2e7a57016a222f3, - 0x90a6d777d2968fec, 0xababa0d2bcec00a9, 0xa6e827f2cde70f2b, 0x288ff0e0f4bbea8c, - 0x5b6e4586ab850111, 0xd00bee510d6ddd4b, 0x35bb071ac55515ac, 0xd6b7f46fbb52ac47, - 0xb42e6eed275a748c, 0x17e1c0bf17835ca0, 0xc7532e72aa9f8003, 0x29ba4b5952830987, - 0xf1f5a3149fd03037, 0x156fd2793e8e0bab, 0xa4ced5e273813cd8, 0xa1e1175534f851e7, - 0x05f87942dfeb7f8d, 0x4302ccca16515385, 0xed723d4bc59b08f3, 0xb1b1efd6f89f11d7, - 0x6df08d875cae343d, 0xedda1ac9d18dbef7, 0xfa00d38a5c5a7c1c, 0x954ff6e3bf041b58, - 0xe9bd8be7e499a0fe, 0x8006a49904804037, 0x66b6e12c6b1c5172, 0xea98d1b45f4f956d, - 0x6abbaed1f5a7afab, 0x3d7be5124378e684, 0xd0620bca2c096241, 0x1dd75e559f41a0b3, - 0xe57497cc44d3119b, 0x47efa81985fb7a06, 0x6d3f76342a2e3309, 0xb46cb486c0d339a5, - 0xc7dac5f0538f219c, 0xce4bd4db3f948694, 0x014cb2bbcb80867e, 0x50f554ac56480123, - 0x042d8cf693c43868, 0xf8f001f337c85159, 0xe59338a51532d75e, 0xadcb8e25551bc121, - 0x49ef1791f5f2db36, 0x09f71f4f4d410864, 0x7ce2684eac6d0945, 0x9f36b07930708ed0, - 0x0b19aade7024e138, 0xc5eaa9c3c5bcecf1, 0x3f382a80edf41c9d, 0x3d123999223a623a, - 0xca17149f536e4b58, 0x0ef3bbe61b8b4154, 0x9fe3de8a4a096f8a, 0xe240d31c5de91f91, - 0x54f7abb0d90012b2, 0x04b29380d9fc14ab, 0xeb5f2def99af763c, 0x17ce75da5f058922, - 0x2cc4aacf32cc9e79, 0xf58d07461de39059, 0xa6fe23fe175e98e3, 0x9a023f88079a1a53, - 0xd886ee779ebd052a, 0x3e5b9a57ed495d9c, 0x1f3623cb56d3c78d, 0xe060005c487722fe, - 0x3c8d5f406e81615d, 0x5303d43bc8cbc36a, 0xc9bf2977458e6657, 0x4d51f825128a5911, - 0x33ca342ed031b2f9, 0xf665c2bf31d7989a, 0x07cb93953484b9fa, 0xd108efce983fda8c, - 0xcb4c4e27f473ae87, 0x2e08ef3e05a1b1ce, 0x00c30832ca11ee45, 0x6bf2f66edc60519c, - 0xc874317aa8b8e68e, 0x52b929eeabaa14a0, 0xe63c33659b399724, 0xf83bff706f87eaa8, - 0x6256b57275b19f1f, 0xf28505ab38af9596, 0x3bc8ef6a6773b215, 0x90ae2be7a3a9b6d1, - 0xaace2c0e6d8fcca3, 0xd05eb86b2620c364, 0x81ccf3f44063530e, 0xea0cd043440d5f2e, - 0x5cb05893f8ac9cac, 0x3b65c9e05161dd4c, 0xe9065bf23ad831e2, 0xb174d89c538fbff0, - 0x737c6c0931c46098, 0x34e2d54eae146f85, 0x8dd310957e5e8301, 0x4f366f390feebf69, - 0x8be452dde3250824, 0x5ee0d5d41e8e2068, 0xbaf5235908833c7d, 0xa2d51a0244f94155, - 0x3c9c6ee51284293e, 0x9e99e8209b0d6ceb, 0xae91ae643bf62a68, 0x218f2137c7b4d2ea, - 0xb036224a9ea361dc, 0xcd3b72467d671abf, 0xcf83c3699cc590d8, 0x7e1849a5beb578c2, - 0x412a2ab5c929e418, 0x44d556fb61b0fe5d, 0x70730310c9347d2f, 0x9aa3014e040b5bc5, - 0xcb151db8726a2df4, 0xb67041a1ae4022ec, 0xf255f699461d2428, 0x2e0955329cd0a099, - 0x43959a9d57529c37, 0xe0850ff2bdfdf267, 0xf28cd6d637f3379c, 0x1a42d8f50fa07c2c, - 0xc032a7f786aceb7d, 0x4f98709b053a3091, 0x53e93f8427406cd4, 0x7235ab69fabe0947, - 0xd0008cd9b53b8cb3, 0xdc8c3e533794d7be, 0xc6e2baaca5e9e49e, 0x2b5e8fecffedcfcf, - 0x5a03cb5ed5466d69, 0x04a0bb48b9f2d766, 0xa6e0bcf91906fc94, 0x6831c726cf0f70c4, - 0xce7be139ceb013a4, 0x2c186eafe6b0652a, 0x6c7f2ee803e1fb8f, 0x22b588314099b565, - 0xe8633df8d65f98af, 0x3a75f378e79d02af, 0x449064d142620038, 0x76fbbdd44ae367eb, - 0x7ce79605138593b1, 0x4a0c4af1f1c47d0f, 0xc80a65ae9df38419, 0x7bae43063576a3b4, - 0xff13d8c6542b7327, 0x20100e30057152a0, 0x0befbcbc7c08ada8, 0xfb609617a8e026b8, - 0x28ef02216fc3a9a6, 0xc2589b50dbbba5d7, 0xd50d584e9255bc1a, 0x707c09e31b1b5f33, - 0xaa258172f67bd62e, 0xbeea58a36695b84d, 0xc67a93000c0f52f3, 0xcad16445903beaf5, - 0x8153b5e3d0281db3, 0xe31fd3403c11cdd0, 0xfdda8df73d567bad, 0xac489db919a719f0, - 0x83d2bc8bd8e0ebf3, 0xccb8fd776f55a173, 0xb8abc54624ef1e71, 0xf2342bae89e745a8, - 0xcfadd9198c735374, 0xfbd2e115ddf7396f, 0x143d3176f865c6cc, 0x51d134587be1d336, - 0x76efbe8699f73149, 0x78df0de3d73d46a3, 0x19df058861e6a36b, 0x622621bdf84a3abf, - 0x98ebaf3b7f444afb, 0x94cb8297adbc2ad6, 0x67df6af749f1f833, 0xaef23c3c3a2b5a0f, - 0x6f717e8aa86ff80a, 0x1802af8496dcaae5, 0x36d66751c1071163, 0x04d51219daccb2c2, - 0x083f6f6de650c23e, 0xe0a2be9919946909, 0x3e76c4f9beff9441, 0xaa638d8600472e6d, - 0xf54fa672f3713565, 0x068fe5a532e5309a, 0x6782a8dfbec4d79e, 0xf55fb7c29a64195e, - 0xe4cf927096686149, 0x4c0b85fceaf31cab, 0xd893f18d82bc5292, 0xbe7cb406246129d4, - 0x6ecadf9cac9ae262, 0x67cf722a4663fbdc, 0x527e295332a50dfe, 0x3bb2e87b1e8a46bc, - 0x772769469b37047c, 0x1fdf6a85ff10126c, 0xaa319b9e3674cf43, 0x9c853dc70c64ba69, - 0xf0913cc13789fa71, 0x305c8eb03a2faec9, 0x33693dd35a15706a, 0xe23f265df187ac5b, - 0x7b4d5c35fe979fcb, 0x505ac0f400dec5cb, 0x9527ce1bc5e5f0e3, 0x4e4cc7eb6a55a2d2, - 0x78013649cf6c5273, 0xb43dcf8501e9c054, 0x5bfa2caded446e52, 0x21eee541f12ea717, - 0xd820d99ff5c1cd81, 0x34dc3edd0a51e418, 0x0922799f21981cd2, 0xff7687ff28da2a4b, - 0x0786f8627c665e1d, 0x4ad48ee2ebdb9e6a, 0x3f6c31355fb5ab7a, 0xf27593687352ab42, - 0xc556c002772525c2, 0x710bf2d413efef50, 0x070428bddcb5fe15, 0xb69008d569788ac0, - 0x3d2e61ac3c61f03e, 0x808e1e752182050b, 0x98edc6d052cdcdea, 0x4f2524d15e633443, - 0x9c3e4b5bad23fa76, 0x5e66063f9bf92dad, 0x44d552c2b3a746da, 0x8a9dc7a1362ad566, - 0x3fea0ae1c5cc6d3c, 0xfde76b07f981d666, 0x2ec8af109d628916, 0xe699c9e3027133c7, - 0x68429c228a7624ae, 0x2b98e8344dec9055, 0x2fd109acbae8b57d, 0xfa553dd831847525, - 0x7f24a2c356e2e081, 0x165d667dc6e3038f, 0x1924e2a67ec76f95, 0x6606d2cf05e14780, - 0x4549a82d0f9b9987, 0xe8fd72fd398ebe4c, 0x40b9bd22890b0801, 0xade91a571cbfcca4, - 0x998d91a20bf33ea0, 0x58d5377df803e54e, 0xd987c9e03176a770, 0xa736a18b9f91a55d, - 0x845a26ec07a12116, 0xf8811c2541b100db, 0xd99596f395f06cb8, 0x579fd8cb3b9c15b5, - 0x2f526d3f84cbe19c, 0x8b7635dc7030042a, 0xa2152bdeb0b256d0, 0x0cc163ea16760fd9, - 0x813185558748185d, 0x637d54d223be7bea, 0xa1d450591482adf4, 0xee7a9a3033a5a447, - 0x9c74d83877c3eea6, 0x416c3abc0d255eb7, 0x608acb406e6fd8e3, 0x695c2b91d163c73b, - 0xc1fc481a2e53ed04, 0xa1ebfcd69a28d53a, 0xade8bd3f872e82aa, 0x4e1cc2b0f73b40cc, - 0xc6b669c9943663df, 0xa9b7d27a3c7265c4, 0xd4154759ffe4fd10, 0xc6d26030b64716c4, - 0xc4193f18efd0f586, 0xb183a700e3dc81f7, 0x9589d916358822ab, 0x9ddf785e5462c282, - 0xc322cdc08f8fb49c, 0x749161d3f66b16f8, 0x0bc8b22096eb6f6b, 0x9994cf3176525f74, - 0x22dfb2ea8e332160, 0xf98703b6550aeb76, 0x884251bd16288281, 0x8a8136d0339ed5b9, - 0xa09f9c250a303164, 0xd17903f68a42a7c7, 0x9608f7fa9bc473e3, 0x130b63a6751fbdc4, - 0xece962a593657e83, 0x9bc03499057b817c, 0xafddb8387058e06c, 0x20fd27fabf02764b, - 0xce68367a84b23e41, 0x19a01c270872aabd, 0x050c2e5f35911cf2, 0x0d48290908056dd4, - 0xf75f6e273f4cb9c8, 0x5947bf61d4f4b26f, 0x7633490523f4dfe3, 0xa1a5c901b2c43ae4, - 0x240113e3cead4194, 0x7adfbf6e56d0f2b2, 0x8fbbc241e1e8f0df, 0xf537d5cea6b86a5c, - 0xc7e7162632b224dc, 0xcaecc485a8bc94ff, 0x117e6fb42762fc72, 0xb09b910051c37d7c, - 0x14617abe1c20b2fc, 0xa2eed56ba262dfae, 0x4d7443b749b793a1, 0xf9a49c97f03ff980, - 0x81a52cea6e8011d6, 0x52f0def20961df7b, 0xa965f4024a72d675, 0x6d87063d3f808e03, - 0x9165e447a625e892, 0xf431316481e1507c, 0x099956f28a58f74e, 0x89854be970dcd251, - 0x56e58fd75f15d2d9, 0x2e19af097a2e8471, 0x244f9488e3e63ba4, 0xea72d13010d6fad6, - 0x6e87987d1bf2a70f, 0x01bf79f6d9a53638, 0xfe047e0e3da19c26, 0xfd6fee1a809a5676, - 0x8a6c1da89f8ff555, 0x4146e879251a771a, 0x8cf748f79e4eb8b4, 0xb2992bc746a33c37, - 0x0e6335f9cbc0912a, 0xe517caca432b212e, 0x90937b67925f8a4c, 0x605d7a28eeb021da, - 0xdf6eace1e89d3c11, 0x422f81cc954e9b81, 0x523347df21f19c62, 0x48833da49e46a93c, - 0xf0cd96910bb6aa95, 0x5aa425689e99c112, 0x5e7ae2a4eb9ba102, 0x0bbd99327e9cd31f, - 0xd9f290515d0a16af, 0x0aa5602eeb618d74, 0x862ef84123c4fc93, 0x97b1ff397a68b7ea, - 0x58e16e6c61682643, 0x901f5b0366f83ec2, 0x3adc437adffe3a7f, 0x9246aa53f8a74134, - 0x16f72d0fdf3196db, 0x6c4b99463b6894a3, 0x95b23b3842f9b6b4, 0x76615ca0af99963b, - 0x617d2b25b67fa765, 0x1cd29cef9fd64bca, 0x3f93518612457e40, 0x7b66e0fd857eb150, - 0x81f67416cb319528, 0x401b4e96a528106b, 0x2a393ee9660b19ff, 0x64ecbcab3d59ac4d, - 0xbb2dab9255ccac14, 0x05ceaa5872412de4, 0x98a67a78fec304dc, 0x0f8452a74c537c37, - 0xd9ed7c1ca64e61e8, 0x5b50b348a83493d1, 0xde82ea94c5550a3d, 0x947e8bcb780b5e7e, - 0x1ee5c9a9e2b44b92, 0xd7bc9cda779a5df4, 0x78b2b38a17104c60, 0x824b41051ea46aaa, - 0x0e1fd4ff44c69bb9, 0x00117a5a17b6c97c, 0xdfb8a6e612d458c3, 0xb286213354c0d4ad, - 0x62a2cffa6762dba9, 0x5298013edc3d78f1, 0xbb4b454957ec57eb, 0x8f5add22901728d7, - 0x52a8cd17e89e8618, 0x9a8169d9cc7e7aef, 0x9b08cef191a82a66, 0x089da7756cc0e0c9, - 0x1e60986d89b98f02, 0xf7cbb5f3a511ae0c, 0xaf471f027b7a8794, 0x55e55e08c2242914, - 0x99713e7333efb198, 0xaebf60b429af16e9, 0xe97d4e473677d197, 0xdf26db7813321518, - 0x92378410d973166c, 0x2c086b4a98875fa3, 0xd587100b7c2707f9, 0x3f06cc904612d53b, - 0xb5154c8668b9cb28, 0x2a6e464d0f7a538e, 0xcde86bea4e6b6c61, 0x8b4e6d8ee072a8aa, - 0xf4082770f628ed63, 0xf1628f5c8f639ac0, 0xe12156c890fc5aaa, 0xfcd98a8e3db3cd11, - 0x70b886bdbb71323e, 0x591334c42fe35516, 0x4ccec87479c49a4a, 0x9312fe5d25636c42, - 0xbf0be15e1f0d65d1, 0xd04884671aafe2a9, 0x172c5a6559f716a6, 0x2a0d3de8470794b4, - 0x62a9679d0d98691f, 0x4799bed74a75f788, 0x58b1d9cd893c5086, 0x0abd11f76a2d4ae1, - 0xcbdc0bf75a38b630, 0x8727e93672765d8e, 0x6056a2e20265fcbb, 0x45e6609f5e21e9df, - 0xb6aefccc0b00b726, 0xc7564d3056a01f4e, 0x8d7c695e92b84ec6, 0x64a872266cd06204, - 0x44bc993b6e5f4acd, 0xfb334726fdd30657, 0xaa0780a016ede5a0, 0x297e9c7940d6db57, - 0x506c2af2cdf607c9, 0x2c74106c7b796afd, 0x00b758fcda17d6b0, 0x279b06468e816a8d, - 0xe45369e9934961c1, 0x8e4b29457c799729, 0x35c55e87361b3e41, 0x6cd1b014dd7fb686, - 0xfe9fda934eabfe5c, 0x7f57bbca71adf5ae, 0x72ec124b019f8089, 0xbeb2b9ee9b4d4af8, - 0xc6a646d10ed357a6, 0x22579964ff32f607, 0x68231561b7402873, 0xa18ad9f63f0b36b8, - 0x04832342e20876a1, 0xbee5943fa804e547, 0x4406ec8b17f63af7, 0x9d07615aaddd13d6, - 0xadb8fa68fa01522d, 0x390c27a285f21d7f, 0xd6a6312730c9cf8b, 0x8cbfca2994d6b379, - 0x4476b4082c6aa088, 0x02b5ad55beb98175, 0x9424f5bdfb53999a, 0xcdc9897b272d81f6, - 0x99b47cf98729e809, 0xd20844676b22b0fe, 0x41af442119291c6f, 0x3cbde034d6397bb9, - 0x4d8820d5118ffba2, 0xf1663b1dbba3fc1d, 0x739189d5439702e0, 0xaf15b3e45550c2ea, - 0x64d585bc0ca6cdea, 0x87ae19cca8217158, 0x5625c8522ebb325f, 0x801ef90088d8851b, - 0xc03172173ebefc6f, 0xa369e166ecc34af4, 0x8be4f434b7e4c69e, 0xd5e77d909ea9d2d3, - 0x9c6295b568de2401, 0xa22aaa960e9ebc0e, 0x40e3bfc59ca0c3e2, 0x9dd56217fb4c5923, - 0x32dcab906878af52, 0x720322437d6ebed9, 0x823aa90118b3a706, 0xb21c66b32725a46f, - 0xe49a8c1b1ae22f8a, 0x77ac50f69d5efc29, 0x761bd53357037257, 0x9977ec61ebbbf102, - 0x6133137eafb84a6e, 0x5f56651bd232b449, 0x7c5d7f3c5603f0ab, 0x5d0cebd196cbc2c5, - 0x9db8e902c03c80a4, 0x947334ef8f318866, 0x3438b99ef1599c20, 0xae442bf025a5d26f, - 0xede5d5f1dc53f0fb, 0x91f2e0054d264692, 0xafe7ec8cd6b0202e, 0xae1d6e887407dc32, - 0xe4320f849542343e, 0xa8e5ae1b12b37096, 0xde6cc0cc5fee0789, 0x1e8709aa68913788, - 0xe188178375c63d48, 0xd29f3d0d125573e7, 0x8484ee99d4fd88aa, 0xe905e1cb0ce0849f, - 0x07ce9e7059120857, 0xfbfee51f67d556b6, 0x2759569a7ff8c9f7, 0xecd6185945eb780e, - 0x058c6a5247b9d9bb, 0x0abf9ef2ea50708b, 0x940a6eb145ae0907, 0x99c0355247019e83, - 0x9dc961c5ccf21fc7, 0xfd9c9a094d47088b, 0xd58544fa708724fd, 0xce711325e966cb7e, - 0xb5a53a9e83f39e63, 0x5352bcf288153c7f, 0x20244e9ab46a611e, 0x88f9d3f9725a25ba, - 0x82f5b6b1a47d478d, 0x0a8dd8e05e13880e, 0x15caa05baca9b42a, 0x0582e8fbd3af6e9f, - 0xe7c34f70ad5acaf8, 0xc67f44a1660700bf, 0x86df10840eb9616f, 0x972fbe7b6a6e2592, - 0xbb3a9a8cbb994687, 0x4d0fa82cf8451d8b, 0x8e55093312c0d866, 0x4495b5604a155ed7, - 0xe74df3842b1323f4, 0x4bdf23455c482393, 0x7a9bcea31f687d27, 0x5514bcd3938cc06c, - 0x5cf2ab700360159d, 0xaa0d5750b2365b0e, 0xd49871205c927f81, 0x6aa5e5a4b943ecbf, - 0xbc19aad54882556f, 0xb980361691c04cba, 0xf7a589d3441c558d, 0xdc4a3279c16832d6, - 0xb1eed56d6ce19714, 0x9eddf1e1512c8722, 0x4e0059574dc39510, 0x72240f4843b140b1, - 0x5333c92e6cbb77d6, 0xf747d44cbf0c95d3, 0x80f65984a5957cd7, 0x0cb2b08a8c843d46, - 0x607c65a950034731, 0xc1ea51c0353588d4, 0x9f0232364bdbad8d, 0x3ae60690f3071ace, - 0xa24a8985fd98cfda, 0x6159a316f7719249, 0x827abfaeb7b28eda, 0xa6b36013efd0a178, - 0xc80f89a500763bbb, 0x28d98645f7df9918, 0x0e155da1d912051f, 0x176a3ebfe61b52ff, - 0x6b8f5224a181419a, 0x4903a615cb4d2415, 0xb8806e41f2634145, 0xc934e8045b420074, - 0xa4e02a16453e032f, 0x87a1bfba42536bb2, 0xd15ffdcb28c71bf2, 0x8ac28ff6891e13f1, - 0x8cc53d29ad2e0776, 0x20b73cc39c1914d4, 0x8d5cc999454f6268, 0x4301e02401d91af2, - 0x8720c87768a44d72, 0x7d04855b63f35a33, 0x754f85ae33b52ee9, 0x43c05867307b55ab, - 0x39a582e2c90100b8, 0x039d184991097fa4, 0x9beb296d9063a43c, 0x85eb829f960971ee, - 0x167ce75e8513eb1f, 0xb1c5f6044c2e0c5b, 0xc0f3543fbaeb27d2, 0xbe7a3aea58b7df53, - 0x3926aa324027f01a, 0x706d554fc628682b, 0x5a468d6b0eceafbd, 0x20f9f270128b2456, - 0x678c0361e3af38b8, 0x8e0b251d56d9fed9, 0x1dba11620128beae, 0x4b2543d8d7924b2a, - 0x56bbbca83ff47a0e, 0xb7296e1dccf77788, 0xe1d8a4ef3a69fb9e, 0x36e917d5c5d03257, - 0x4e13c8fcc7f923c3, 0x26a3d3c0e0818e16, 0xaa4d46749d29e0c0, 0x329cfe6400c1b95e, - 0xde43e96243e84a5d, 0xf8cfcc93bc35013d, 0x8db86e3ac1b0aa49, 0xc28683c1f9a2e395, - 0xb7e2361c646ac7f6, 0x90e90b6fc8cd7b47, 0xf9dcc995dd264a1b, 0x7776b1ad0532221b, - 0x8d3b80bee584be22, 0xbf02a0f6b593596c, 0x5949ad6370e81b5e, 0xf595994983843344, - 0xb9c3c8afa549a196, 0xcbccb44ee508c4f3, 0x84f91cac0a0f0344, 0xd9da5c8de0b9cd3c, - 0xfc6563da12570137, 0x79a516966238e478, 0x78558cfc20e7d312, 0xf3bfbd22f82dc259, - 0xefaf6a62ea0b0977, 0x7e2585b9d8362644, 0x6849d743fd03cd3b, 0xcfe1bf1e97811c01, - 0x507559e441a0ccfb, 0x319d14bd390216a1, 0x372a82222d9ee3b4, 0x8289dc0cd04a090d, - 0x9749e9439abfbec6, 0x572c1bc13130c8a2, 0xdf35d7c556fe8032, 0x83baf84f38531e5f, - 0xb1f21236040c8ee2, 0xfac1555c52766850, 0x810e4a5d3eb8e57f, 0x8335afc4864db238, - 0x6bc938556129ab4a, 0xe53e6e18e8420e6d, 0x16e8d84e6803a018, 0x62d5bdec8e5fcbf3, - 0xae2b57c86827929e, 0xbf03d1eb0b406ba7, 0xd1b3a633e03ac0d6, 0x1f9e439fb4ec51fa, - 0xb629c3469adeba4a, 0xefa11a93271dcb3b, 0xb6df0abe0924f6d7, 0x3d7cd9d5d7a48680, - 0xe76c2e7b10ff70da, 0x20c82b7924c25a81, 0x599be27ab017c943, 0xb1305ff11b9bd871, - 0xdd791722c5420313, 0xf63d4e7f286b28b3, 0xeae6a8890df45b18, 0x8582907264f4c8bf, - 0x85e5971f9368098a, 0xfec637e1968510fc, 0xb92b3406267ae475, 0xbd112b6183379ab8, - 0x704107677fa9a439, 0x595619008502ecdf, 0x109cc252ee3c2b71, 0x4d67283869a90deb, - 0x58e454367ec946b6, 0xe985406a3fa8a49c, 0xeb690e74fb08d4f1, 0x068b97eabdcc2e3c, - 0xaefa3a903f32dff7, 0x541f669d0d3f9e60, 0x5188ece0f95dcb26, 0x9a6e9834c2e12535, - 0xbaca7da307848aad, 0xdee41f1b2ee7b66f, 0x22db4bf842626fca, 0x95956038f2304e2e, - 0x37743e1f2f6fbe64, 0xaabed7a3d403ac8a, 0x408a2d4deac13698, 0x9cd3fb5e48c52777, - 0xa3fdc85671411185, 0x59c1823600667555, 0xdb56e3611e6b306b, 0xf8fd973cd8866a31, - 0x7331fd81be2c6cc2, 0x49ab1e27eea27211, 0xc68c58d60cb3a585, 0x8231e5193bdcdbf3, - 0x08c4f8349d267c83, 0x3ba9e2a0b6b45664, 0x3f13fc1837145133, 0x83821bfd0072e721, - 0x28de8915b4e1159e, 0xf9ef394c2586f09c, 0xc8f06bf70969d334, 0xd4af5ef796700106, - 0x0f92c4e0228ea439, 0x2829df2c0e48da56, 0x0c91787214d31d99, 0xeb3a315da0bf5ef9, - 0x875c1d45f3279741, 0x2ca30946742fcd4b, 0x4a43ddbc954da493, 0xbd2b788f2cd9fb24, - 0xe50f906f8ffb59fc, 0x1fe75fa82cc37604, 0xb60c41d399887631, 0xeaad8686a74cca3b, - 0x3bf38a88a4a31cd2, 0x82821f5444b5175a, 0xd4ef84b9645665a8, 0xcc3394e720472443, - 0xc3fd2ba530465829, 0x34e97a0ab53ce3cf, 0x8faf199b8a5699b7, 0x7c8a7b99fd979b68, - 0xea6b8faa1444881f, 0xaa1e600cfb60420b, 0x26f47384d70dc6bb, 0x47e8868b7bfe20ce, - 0x328fbdfc969fd182, 0x68bfa18447f1b24a, 0x8a030dd7921d7ca4, 0x73dbcb29329caf83, - 0x9f0fdb9c45b11634, 0xd8d8e88a2c8f46e8, 0x10c2f67486e10ba6, 0x75f6f1826e367956, - 0x1bf576927e586af2, 0x66933963990e8833, 0xb799605ab472c3e6, 0x88a324bc7e8b6162, - 0x7bbd2ac2e43e4b5f, 0x545233c03201e628, 0xec0f87d30badf223, 0x6d2f52461c051a50, - 0xa712e4cdab1ece5d, 0xf3354a689b2b091d, 0x39bc0f8831782b7a, 0x085411c03e52507d, - 0x316a427fbb094db7, 0x6cd2792b3b050926, 0x629fc555b860cc9f, 0xc0156a6007028225, - 0x3ca1e9910bd3138c, 0x5d63254142ea6fba, 0x494a3a9c63c8f776, 0xb26f5ad84210217c, - 0xd199876ea746f5bd, 0x49b14fe820107374, 0x99e407220d210681, 0x3a216a6e679793bb, - 0x8bba7fc95ee699fd, 0x48f7cf2b23cd3b9e, 0x1ec23384148e2681, 0x61e1d210f6441241, - 0x5853b56be2c3e925, 0x1b980acef7ab3cb0, 0x86de1f866c288951, 0x3be462f4ec6cf7da, - 0xceec9ff0c30b6da4, 0x321cc9abb9b974de, 0xe070ae7dd00a0c3c, 0xa9732653b3973301, - 0x990ce16d01770fdc, 0xe2f7edcca107b01b, 0xccad43e030010e24, 0xf6fce5d080908e31, - 0x7bceeb9344b1ec56, 0x38069c19ccfcfd3b, 0x08c6836ca798c6e2, 0x654c8064ac567a41, - 0x87b9513a66c899ba, 0x34f3f3665bc5cdf6, 0x16a6e055957c26d3, 0xde658c38f03f3651, - 0x97199b712e9be779, 0x3855210e438d191b, 0x07c9275d4215c560, 0x391fa47eabc9aaf7, - 0xc4cf37a5a247cab0, 0x492c1bf81981b196, 0x708ea5037560533c, 0xe5be547ead25e33c, - 0x7abb312f97ebc95d, 0x0bfc46916b589cd4, 0xf0cac89061226500, 0xc092019fc767913c, - 0x552605531b64a68d, 0x9c6017cde505474c, 0x1dbc1b2c7e65f2c3, 0x69e20ecca1fc5835, - 0xd4bc910f2a15d1c1, 0x75ba7f4ca84e8b39, 0x78818a6fec3b21de, 0xc30996ac86f1e58e, - 0xc12e27c9e77d0fe3, 0x58b9031a59cb3fd7, 0x925288a6eb9e358b, 0x0a34f57be5b1788e, - 0x6fc1876fe07828e9, 0xa0fcd6e291b3bc04, 0x31cfa3533eea0704, 0x6ad113c7ab419564, - 0xbb65625b13ab5401, 0xb140ce96528604e1, 0x04f590eefb401d6c, 0x3811ec178af6e917, - 0xb843fda8e03bbaa4, 0x9a05fa62e20ca281, 0x597488458f8f366f, 0x26c4efdfde40b2a3, - 0x32e53e9f16484a2e, 0xfcefb96ea6142f4a, 0x40f98d8d0be5b82b, 0x0ff86b23bf6b8758, - 0x7cc2e75280d0145d, 0xdd1fa456b29487e6, 0x510335ec7a7727a1, 0xbebb2c7b31ee4ce2, - 0xa59eb113f5701bcc, 0xe772e89000f76c27, 0x99629970a18ea142, 0x9d1553f3b2a0a401, - 0xab473f6bb149946a, 0x0a14d6af6fe56a76, 0x885ec1fd9ce5f2b4, 0xe975527957020c5f, - 0xceba8ed037ea1059, 0xffb498a2f537816b, 0x2465bdc000ffb3a0, 0x282dbc90fff083c7, - 0x552f9ae55f539cc3, 0x6767141fffbdd05e, 0x2b600d2a6c62e538, 0x46e2653c71395737, - 0xa3eb1e0f992f6fda, 0x10303b4c07ec5c43, 0x0c33668ed412dad8, 0xed6f376afaab5d63, - 0x1816962c3e952801, 0x9ac40cd85ac9b4e2, 0x489e39922c41cbc5, 0x0827d598ac2e21bb, - 0x06d3b76a298c45b2, 0xd2cb9929a46cebfe, 0x4bd020b85ead85cd, 0x848ff6f07a873f9a, - 0xa5c302501af2e55e, 0xd3ff788b079ef8b3, 0xd6dfc3757cf4364b, 0x720e338bf657beab, - 0x999656c2241a5745, 0x715dea9520e9623b, 0xf9e8d40622477b3c, 0x3fbaa7806fc0d3a7, - 0x0fdc1dad6722d0b8, 0xd34cb4a18532eef1, 0xb37ded46b620b0f9, 0xa16b4b4f708b9586, - 0x0fe8fa4777c1a78f, 0x8b9733b78da26be7, 0x4c632811dce909eb, 0x16a093d884efaf56, - 0x239704f1bbe4b757, 0x769a003cba827580, 0x52d240a7a2f003f3, 0x2d6d9112f509c899, - 0xf69f3aea26693bc2, 0xa594c3cf013560a3, 0xc7caca5ba87cf3ac, 0x801342ab9d6a2ff2, - 0xfeec5dcad9c772c4, 0xeee901dbd7d5c5fe, 0xb5472fe8bb252bc9, 0x2f1c2c8037b14915, - 0xb82e0db42c1c9026, 0x47b1e1610d4d7fd1, 0x8a00f0dd1e089dcd, 0x4f9596c1ceb8e7b5, - 0xa02996a478a38feb, 0xfce1793f32e96f5a, 0x5b3fcca31da4e596, 0x596a73f831c875ef, - 0xb65198aa20147148, 0x01c6dbb4445c318b, 0x867426fdd0588d9a, 0xf1e1fb01ab132e53, - 0x975ec7976017aab3, 0x7986011fa500ba15, 0xf50ff024fd34849a, 0xcada091ab329e7e8, - 0xc6366a31fa5ec4ed, 0x8efc2680f0f349c1, 0xda3ccd5a4526e85d, 0xac322f6aee1df187, - 0x4aca5c28ac0e7209, 0xf789d85e58e7d0d5, 0x7d82ed034a4ab206, 0x05269ec75fc7ef10, - 0x34473760efeaaf06, 0x84f49c5880d991c2, 0x82a07f1f1b69807d, 0xca1edeb91bfdefb9, - 0xaebf8371cb58fbb7, 0xd641eb80c05a9ae7, 0xc4c418faf2cb3d6f, 0x4b0955bcee4ced55, - 0x8f651d63e98da60d, 0xdca8d0628c575507, 0xb2df84ba8dcde049, 0x495442052b64bd1a, - 0x1205884daffb23ce, 0xd5e41a03238a4571, 0x187a62eb7f96c5df, 0xefb28c5bf15cd18f, - 0xd8361382ba4949e7, 0x146b048b68fec98c, 0x06234198d4d18f9e, 0x72af49a5d13408f5, - 0x43a4a16444731e00, 0xb67c18fd11e9317c, 0x647094ee94599b74, 0x90ce5aa4ed513535, - 0x3226691a04ff7308, 0xafdaf190f3988cd8, 0x58f3c46e7db9ba88, 0x54b5c41edca0413c, - 0xd0ff3029ff70a694, 0x23a33c7fe5509322, 0x054a78d77dbf528a, 0x27dc97c39c64404f, - 0xa8116ded920b1796, 0xed7ab63c678ca8af, 0x7b404ad6924dea39, 0x823fcdf42d9e8045, - 0x9f7588a978fdf07f, 0x07758e66ef3313a4, 0x3aaa8ac0627ab5f0, 0x2abebff0ff0139aa, - 0x3437f9479ed4c802, 0xdd3b84c4da55d3a9, 0x27b9344a5bb14d45, 0x2e1c6bd73feaec76, - 0xe8380f9b95a6c146, 0x4bf989ad70963cc0, 0x6fba5498ed87d769, 0x5c24288c7af84318, - 0x07d157ee29e54c0a, 0x1df9e390a4802e12, 0x765364ffda4e6813, 0x244cb8e65df37c6c, - 0xa767da3c4207caa1, 0xe29c4a1aed3cf6e7, 0x26266bb56e7bc8da, 0x9657c29419edbf86, - 0xe2c2a2a1cb487df1, 0xbdfe6c6f85cbd3a0, 0x827fc2be36a7bb21, 0x383f45234b681e08, - 0x5e04daacc0d8b5b6, 0x149bcaa40bc3b207, 0xc7bd5011099f6c80, 0x3f9af18ac0be9b3e, - 0xa8f066a5eac34d75, 0x9260cc0ff1c8d34f, 0x69020d7a5361d391, 0x6cc6e3de1e93d6d0, - 0xc70021f33ec302d6, 0x5896bb30a38b28c5, 0x59e52eb04e9eee0c, 0x664916e01ca501e9, - 0x5cd1a736a1ebcee6, 0xbba9f2f3532fe3dc, 0xb535fd248f5f9bbd, 0xd65b2f39a2aac0a7, - 0xad5be0533da27799, 0xb5c846f41e8c75ef, 0x4fc05fe4da228245, 0xeab41a97685a3530, - 0x74e0c29969beb1c9, 0xb59baa39371b7adf, 0x6f0cadab66105bb6, 0x0c43f1cbd897940d, - 0x6e5c89bbb1b2e70b, 0x4f334e81a9fd31a1, 0x5dcaa8633242d778, 0x08d44c3d4b7ead5d, - 0x9d74c73b2833bbc5, 0xad33932bd78586b8, 0x2f6f3a983211c3c5, 0xf8cf4709361d5137, - 0x604725ee8bb33190, 0x3b3d9b919ef4ca05, 0x1b0ae3e23b01abb6, 0x10d14d17405634f9, - 0x469f42edfc65bb99, 0x2b6dcf18e6e2c503, 0x1b5f7e3447648c5f, 0xbfdc777a994b050c, - 0xb54278a4769e039f, 0x2996b8b9ef87fcb8, 0xc83b83c6de49b922, 0x8be8f792de0b07e6, - 0xfaa33c2d2b6de01c, 0xac3e15274a312aa0, 0x9eee85abaf283923, 0x97769f6858985e47, - 0x33c213f7830ff1de, 0xd1d84b843c10d0c6, 0x8b611e2ceb355d24, 0x27e1735eeae23f72, - 0x144a1730cb935c3f, 0x4f14210c9c20a2a5, 0x900cb58ba8ebbcb3, 0x1578d49fe24dbd16, - 0x121aa314ea6d37c6, 0x1f9904b8948c5751, 0xf6c3521e68c74e8e, 0xeca2203e2f9f6847, - 0x4672d5beb197e7a6, 0xdf05b184ad194fe5, 0xf5e3e7b4c9dacc01, 0xed071e5d81abd956, - 0xa85b2e41ae264934, 0xee11dbddd2ee4c93, 0x28ff8ce45b43aed6, 0xf1f694a7b1fb7520, - 0x5f1b92e517bd909c, 0x4e096e8b49275bc1, 0xac9b185d8d34943a, 0x38262c1cd4cef2d1, - 0xcc242753581651ca, 0x3c69f58e17b2fb5c, 0x3614e52193c5386c, 0xd5b66e2ad5f6d3bf, - 0x4a3601d8517a93fd, 0xc08a0db5350b593d, 0xe6a983df7722335c, 0xff27a0bb09e81b4d, - 0x70aca3db006205c7, 0x45af8e2cfe38f335, 0x462c584580741a32, 0x2dfa6a0a7d11075f, - 0x205312678dff6a51, 0xe9dd76eacff76d27, 0x3ca8a3a9a554bd2f, 0x82571d7ace0c0d3c, - 0x05cfbad6e6999caa, 0x3ae0c0490f3a7420, 0x29dc4d9e68ff0e38, 0xe4e3a519d8301fef, - 0xfd8f8e918460e5bd, 0x1a78b52eb830d1eb, 0x360829c56a60b77b, 0x588786d4feb605bc, - 0xd7bf0bccca749119, 0xb8ef7b9b50cd91ff, 0x8c828af8ea8605cd, 0xddebc5f168b6bf6b, - 0xcb12193cf26568b5, 0x7c27b060bda2ff75, 0xca32e999b23db52e, 0x2acb6176f2ecbf6a, - 0xfaca13ef16ca5bb6, 0xb7d5595188edd9fd, 0x28636e651aa71a34, 0xc606cb2caf3d6c75, - 0x38a4a883950f4a53, 0x5f77fb0de3126589, 0xe1bec50c602fbbad, 0xb73da0aedbdfc019, - 0xce782f1afa15cbbc, 0xe6d1d76fa2aead58, 0xd1fd713d55891459, 0xbf91fd37f603eff4, - 0xda70a89a2e041ec7, 0x4598ba9d0e3a83dc, 0x72553c0149ffc263, 0xe8b930afb24b3259, - 0x7998c1a51085c774, 0x08fd65b507f3f68c, 0x70dbc9e22d78a30f, 0x1d222fe6069b655b, - 0x76b4929fd3921cba, 0x953fa38dea759538, 0xdada6ae58251dd97, 0x04c3ffdae12dd29e, - 0xb9a9d8efe3cde8ac, 0xcfd609c8c9f881c2, 0x59722f8f51572edc, 0x1a26fcb6d9fdd73b, - 0x1f149a030a181d2c, 0x1017f491ed999b3b, 0x535c7547db2a1544, 0xa04b28bb86b3ae10, - 0x76f501506607e2d3, 0x7a71bb6a66009ef4, 0xfd03741f3e6fcca3, 0x51d4eec2b52d1b5b, - 0x073541a42e7cd472, 0x85f1a9db461cdbf0, 0xcf86c472c17ee155, 0xdb575f551fbd5811, - 0x6beee48583b86c90, 0x8b7fdc6417104dc3, 0xdeef019d38fe6882, 0x163b50e956d6e77b, - 0x381922066f2e851f, 0x4e6af2df98f18221, 0x5221bef4f5b57c4a, 0xc8d6ea9ce2126155, - 0x232ef1e5df4cdaca, 0xce275d47285a6478, 0x64e8bc6d0a98d3a5, 0x51f57571d1cd6ca0, - 0xe3ccd76179f9941f, 0x1f145a50e5802719, 0x0005acc495c82230, 0xd7f64b316b12d83f, - 0x8cda90ebedc86806, 0x4f736d91c47fe2e9, 0xe27656535e9f6f09, 0xf2b2da697f9c0e77, - 0x5343f8c0c7aaa07f, 0xfc76910d262bd14e, 0x97f8278ce9846137, 0x6b2558b4938b9d36, - 0x226109e730c0631c, 0x9759a6f0eaac1387, 0x8e77b52b67a64c3a, 0xd75c58f243ebccf3, - 0x2a850e97fcc41382, 0xd853f63824a2c4e8, 0x057713d3c0341474, 0x604659ad6ce78e09, - 0x57b5939877e2e998, 0xc226b55d553bfb4e, 0x846f0a7dee794c11, 0x053013a2bf6bebd2, - 0x8bf07927b3f8e3a0, 0x91758fa1c4cb529a, 0x878e0430a4139a5b, 0xd095e358e0df995f, - 0x392373d5b30d775c, 0x3d8fdfa90b41f300, 0xe600905df87b0a2f, 0x499ed1479da075f3, - 0x0e2a55c98a1628b7, 0xc344c05c8173a765, 0xd5d3d0021048a238, 0x17a3ca0efe93c608, - 0x23182f232d34f7fd, 0x4ad2e5000cbd41b5, 0x5bc59f0635ea7078, 0xd01d41195bace36f, - 0xc1978af5e4263ffb, 0xe3ac1f051ac8d86d, 0x54ea33595a7ac7b0, 0xc8b4c9900be1fee6, - 0xaa71a51abdecce95, 0x5709cff0f13c00f8, 0x742cb76c2bcd4db3, 0x8b3b7ea2aaf2343a, - 0x650f17653cc0ba79, 0x742eba9028d92167, 0xbfeec4c2b49ee002, 0x35344b15ab446d09, - 0x6c161402a175da04, 0xb15df1226bb590be, 0xa2bfa36b8dd8fb91, 0x5d2b3abdae96f6d7, - 0xdc4609910830c92c, 0x3162dc69ce707a8e, 0x6f3ddc96e6c83606, 0x00707e47619604ee, - 0xfd970ec2e42d91cc, 0xc7abf0e077db4def, 0x1c62a972cd6005ab, 0x836d5dfa6767cbf2, - 0xefc2d53800ae5d4f, 0xef75efe8ba8f7b13, 0x59a435ffddcd3ec3, 0xbb3ece6dd45a6d55, - 0xdb0523391631fa66, 0xd662a0682a3d75d0, 0x820c6154facd67af, 0x0cfff3b431c7b607, - 0x81497e4176a93d61, 0x6e453865d397d157, 0xedcc715f50507d18, 0x61626982bf818067, - 0xe382f615a1d4ded2, 0xc183af47f16b9e72, 0xeeb28f1cc597b79d, 0x4132b505ab6d63f8, - 0x5c1106abe0dc3374, 0x2101209ed9eee34d, 0xf9fb57c209237d05, 0x1c382b0b64d2c1ef, - 0x54f89881f3a8c0b2, 0x190df8d3a2307b34, 0x6952688f222cdf94, 0xe9c83d0fa508c042, - 0x34bf602074019c5c, 0x58f64bdc0dd56088, 0x56c9a8dea34fa786, 0x867984e5f0926e2d, - 0x454636fd4fecf7b5, 0xcee84aec4a11cad3, 0xacd6f88b2cca9e65, 0x4cddfc4a6e624cd4, - 0xa813b951706cdc1e, 0x3cfa682584240e5e, 0xa1e5ffecdcbd78ec, 0x92d687f1927a8703, - 0xec8066371f9875f0, 0x91e36e446a1e3849, 0xaee35a45e145af4d, 0x3a90c2c7d34ee428, - 0x83d99072f50016db, 0x1b69da37310619d0, 0x3754ac49f343b6d7, 0x711dbd1b3f5f63b3, - 0xee2be08da4b3cc0b, 0xe5c5c366ab9e266b, 0x0b2ca7aec0f9537c, 0x6a1590f73390b0e3, - 0x6e7d1b3bf82987a0, 0xc55c8cbb82a8fa32, 0xfe08f7a4eb594093, 0x53a2926acc0aa38f, - 0x15c80f4181b078ca, 0x5e8e24be9ff1ea84, 0xed58e99bc9606b9e, 0xe3dddc1274d5d571, - 0x52da2316a8530e09, 0x7335e514a196ae1b, 0xab6892ebfc6ed293, 0x50ebd74cd2795e1c, - 0xc13873186756abae, 0x92d32cd9074e5b56, 0xb0d5d0f342e240b5, 0x846ab88374bf93e0, - 0xcd5ed721f2f40277, 0x84dffee26cd3d1c9, 0x68c0437f0c238d06, 0xd5eb1c120781880b, - 0xcc7f8080fe9ef950, 0x888caa55f6d2d73a, 0x7c730ea467066db5, 0xeeae843c589ffcfb, - 0x8a9d642663b883c2, 0xf4431463bac58685, 0x4d6d237a6b954806, 0x0c26287f17a7cb4c, - 0x7d586a62409ab38d, 0x942e28cb654c60d7, 0xc66df1a5f7f63168, 0xbe1ada5199f63c6a, - 0x797bb5be10fb9c20, 0x62fce3883fd0e0c3, 0x6083ea23d0f32003, 0x6a6d7e7c86f66b24, - 0xde1629a883d02b25, 0x5f7c946de31ac3ec, 0xce89ae4a0fd74a02, 0xe111d5ac76925e2d, - 0x8d81d00c03ac3d5b, 0x0c0ed13a0e582272, 0xc54be40e28af5761, 0x4cd0572cb5973d51, - 0xb5d223d400197b15, 0x647784e215daf73a, 0x9948e164299118f5, 0xe27dfc3596385d97, - 0x59a247076fc5c2e7, 0x6eddb858362a7b10, 0x10114433dcf2bdfb, 0x72460c1c2881dc00, - 0xea907ee1a06b3470, 0x3b97bb9d681af6cc, 0xb478183ee431b461, 0xef613c0115afc950, - 0xef7a0adc5462afd9, 0x8faaab7312663ca4, 0x87675cbb7d7a05e9, 0x45382daecabcada0, - 0x17020dcb04fee655, 0xab3ec291d0ab1327, 0xcc49704bbffd9e40, 0xbe97e1652957b6f5, - 0x760422c50de47577, 0xb07df651f5dd48b3, 0xfcf513c4b3744628, 0xccab180b2066918e, - 0x5ce86612c1b03ec1, 0x0a93fff351e3d389, 0x61ffa52886af63a3, 0x01b927a7e38cb0ec, - 0xfd2331330bbe6289, 0xfdeb6206ed6b8a71, 0xbedca7b4ff9ccf34, 0xbad79abb970a032f, - 0x2e1dd3864837bf8c, 0x113c14c085320816, 0xe8945dd407fe99cc, 0x48ed3b11a81267d7, - 0x4a72e9f3159125a4, 0x08117e1aa0cd5298, 0xd70777b224e50038, 0xaf5701cc12048a10, - 0x499165850104b65a, 0xf7947ab62d94668d, 0x22418360cb233699, 0xa1b52ea08cb59ed0, - 0x38239f7ebddcb5ac, 0xd873f0b6db8118e9, 0xfcfef97ddc56fc22, 0xc75b8d1324914611, - 0x51d3bad5d378bf32, 0x3dbfa01e914fdf02, 0x4a449831f36ae56c, 0xc27e816f7de5d9bd, - 0x76018ae54b895f17, 0x1932e8890b604dae, 0xa2809df5b614901c, 0xab31b3e4c7368ed7, - 0x281ea8612b3a1886, 0x902f72fa0a132a06, 0x75da545fed4b0e0f, 0x3afa68ca0476795d, - 0x72e44152fa7c5686, 0x5f550efe5379fd2d, 0x2ac0423ee3ecd87f, 0xe37f5ed6112c7581, - 0x5eed3ab728b6f993, 0xf3ba844362cf42eb, 0xea0fdaf57923e0b0, 0xf9b8cc5885234282, - 0x64f1848af81ce39b, 0x687a4497219c1528, 0xe0f0a906f2cdb1d1, 0x88f9480c70d6d687, - 0xf76db6d594c94361, 0xfee4d5e84c7018a0, 0x922e90e355814206, 0x57006a89d1339e59, - 0x3c0e893b39706490, 0xb6c519d59167f451, 0xf7ddd60edf332d36, 0x968141207d2ec8e4, - 0xc16ab07335814eef, 0x9acfb32f94dbcd5d, 0xc4a834230c8cd921, 0x2c50ea3ee7b38d83, - 0x46ff8f1438c03818, 0xca901081f14675cb, 0xbd55a448e2652b2d, 0x261beffdbb6e993b, - 0x5805af86bf3d34e3, 0xd5ceecc4a9809a1e, 0x9317baa5b16cec63, 0xd85d4df7e7c9b684, - 0xe4177a404f5fd291, 0xbfd99e0c6e7f5450, 0x7bb87eaf42538e06, 0x8403a4e6462d3a34, - 0x2ef3fbbca9a4a317, 0x9a084b3f7e410e06, 0xb143f64210db5f57, 0x02b161d36ba99ce9, - 0xf8f354acb112d000, 0x359f9fa65f2d22c9, 0xdd34e32af13c4b69, 0xb095b2a711eb7b3b, - 0xf5a24e18c5176949, 0x5119c6f441c0cf13, 0x887d88ced2b194f4, 0x8dd038726551f7da, - 0xe9a61bc62999c5b4, 0x7f2d54bdfafbd5dd, 0x73d617d7c2fc57c9, 0x5000622e23ba305b, - 0xd42ae65d55f75afe, 0x9efeaed46a53e9ac, 0x632a3a3b95818193, 0x6e88d3c2c48cff61, - 0xaefb9b9c5a9d4998, 0xda16fd443ebcff9a, 0x902c8b6e66523aa0, 0x0a53d0e35a7d5685, - 0x5f1f719cb4da6424, 0x03b0977e9806e260, 0x702662b8d78dcaea, 0xb24a194446eaf7bb, - 0xbf69873cdc647cb3, 0x39e886bcc1cdcd41, 0x1bc22abce6ba1c5a, 0x315b3955df3fc818, - 0x951d0ab0216085df, 0xd914cc491438ab13, 0x1375a2a767334efb, 0x29402a9cba62afbd, - 0xc889036dfae9746d, 0x917e2984e7be3d7b, 0xaa7451a5624da045, 0xdef2e0b9fc16075e, - 0x954ad6b178b297a3, 0x8243532361710b70, 0xa4729aace0de3b1d, 0x8c7d567099b6d80b, - 0x32052d0327f0aac4, 0x614fc675dff83d6b, 0xdbd83b12a31b637b, 0xdf75f194513651a6, - 0x20662b3e70845ad5, 0x6b5de9c6eadc5262, 0x15729378c91fa0aa, 0x7070517cf138e9bd, - 0xc1aa52e03a868035, 0xd7874f50b539e594, 0x6e7f37260dd86e02, 0xd78886470bc58ba7, - 0xc7b5b6690ce8b117, 0x29dc066aeb14d787, 0xa67c20a533748a8b, 0xf1758f17a0bcca24, - 0xb7ea0fa7b4dcd809, 0x5526d9578ffd1dd7, 0x5d2805c6d98817b2, 0x8bccc935b447a023, - 0x92371bb02f9ea2e0, 0xa464d4de1addff2d, 0xfe7180a21c8e1bd1, 0x36e4c01f8005fd5f, - 0x39da62284cc1172e, 0x9f032d139116f2b1, 0xd9139a80f0184080, 0x5a8d6f60de16720e, - 0xd3ad78b8f469cb73, 0xf1ff00520eacce91, 0x2e4e95d6a8e76042, 0xc68768a17df33be6, - 0xf27fbd831665cb6f, 0xfcbfb1ca57ce6045, 0xed389102a306f86b, 0x204947a7f0de5c06, - 0x1166209c56c79ff8, 0xa3ce3e66419b8f2c, 0x4a3c7c64794f1442, 0xd2632eb5fb662b52, - 0x91ef7e56b696fdb6, 0x7df1b51a117e786e, 0x855ffb90aca755a8, 0x4da23c8af2f54e71, - 0x87c5c88d9dfd2b2f, 0x75fb54142d353363, 0x957e247e8c00ee17, 0xca8e6a442156693e, - 0x68b681ea202918db, 0x237ede2cf15100fa, 0xa495411c16a15f1a, 0xe88b455a0f862e3d, - 0xc3c28e2f1579f24b, 0x67232d6b3e54b666, 0xed11844314a0e60e, 0xc224773722ca60c9, - 0x36dbe697ee2d7dd2, 0x76652e79684d2085, 0xa1cef157fb8ce2cc, 0xd7df29bb7e03110f, - 0x6496c6d22e545314, 0x1338673d1a164490, 0x3db9cf48591705ce, 0x19c760d015f390ab, - 0x34206c7f7c971db6, 0xc5b3e00e35b0f9e2, 0x0f503360f597ff97, 0x35642f97d1d83643, - 0x0010fa1dc824ae7b, 0x0870e6618681e977, 0xcd45a40d55235f6b, 0xd89a1a2de58e9efb, - 0x5991b7db80f3a972, 0xbcfc8b150e0ff97c, 0xe6b012aa9017460f, 0x1353d522bf20c7d2, - 0x6883acb1c086410f, 0x95d1fb26edfbb9eb, 0x2d98dcde791e62f1, 0x9e0a0a7b0f9ee82a, - 0x0fe5d4470ff767c0, 0x73ca1a2274ad36dc, 0xd41f65c3509bfb78, 0x9b1b4c7c32c7ac51, - 0xa3efda8a19edd0e1, 0x45fc9c0576d49384, 0xc99a6dbe6dd38bfb, 0x83e58701a29e8c8e, - 0x515221fe1a012637, 0x4e3ed995e203f23c, 0x312e5b722a9d76e6, 0xc9f0de9a4efa4b56, - 0xae631e518c202bec, 0xef7823fb57828cf0, 0xaaa463b7ccf2362d, 0x815a3425403f95ea, - 0xe6169ead672eb0b5, 0x77066c1b6f196bb0, 0x7c2c2eedf3ac84d2, 0x43f968f2fa0b6337, - 0xc86ff5fa71619540, 0x9790559e56825ae9, 0x76620f49b5e4be1d, 0xbe8b9bdab3ed26d6, - 0x6ec5987738734d7d, 0xafcc34832e8fdd32, 0x3d71b5c721e7fc48, 0x68dee549a6fb714e, - 0x37bf93f4723e0b70, 0x3a3e68a808586d7f, 0xc91149673e33245e, 0x92cac9da577088fe, - 0x6d495e8b3a7c5558, 0xfe2bafdcb54a4734, 0x6f049a91ed1313d9, 0x7e10a03b7132a702, - 0xbe62ea0eb4d7c99f, 0x54a506d50e10cfe4, 0xb009678ad4ada076, 0x6a2c4206f3bec9e9, - 0x347e7375960e2ea6, 0x9e412d3d7f26a136, 0x58d32a80537ff4e8, 0x75125a58fb56c3d1, - 0xe6f15f5abf437d65, 0x0fda7998cc004f14, 0x11c35209411731ac, 0x04ad89fe669905d7, - 0xc19b043f49da2d97, 0xa8e165967245b5d7, 0x1060b3c45890ccd5, 0x86233b724861eaa3, - 0x7a0c95dc3cbc4ed4, 0xbb4f565074f786a2, 0x44c8f157b9cfce9a, 0x249eb2eda6565f95, - 0xd84c660a901e05ca, 0x062a13c2ed42b65f, 0x7492eb250a8aea1d, 0x5a2673df7c810904, - 0x4bc5b0f9a3d1c77b, 0x6779c106b9997281, 0x94edb58126cd1437, 0x40e34bc020d96732, - 0x150166ca4bf4b2c2, 0xfaebbd0975109cdc, 0xc830d8ef909c6dc4, 0x9060b55b38eafd81, - 0x9de246ea380969b7, 0xca5341007c9d8203, 0x1db6eae3952bfc84, 0xf3b5c74c1783f85a, - 0x4d0d7a6b26e7aec3, 0x9cb9e76938a05b3e, 0xb83f9428e7470091, 0xe793c6550e251499, - 0xbdda4e5993c28417, 0x75c92544222d8eb5, 0xca8fe4309f557dac, 0x50259f8d866a551f, - 0x787c306dc946bd81, 0x7b1eb668beb33124, 0xfb8ef11830bee372, 0x2450cc4f6892232e, - 0x85db68d6c6b46990, 0x9ee25c9577212fef, 0x923db814c5d0fbaf, 0xe8c5e6030f68d42a, - 0x31762eec73ab953c, 0x2edf9db4364c2d8a, 0xd89b7cf7d6b2bb5b, 0xc6ea595a0f8d7bd0, - 0x1f49b9b46c540e5f, 0x593280a0014e1853, 0x9df7da33f2c97c44, 0xee4cea1132990811, - 0x68d97d7be0719ef3, 0x4fb07d09d9a71914, 0x6563f8d23a04199b, 0xc9bcb42d511d41e5, - 0xad8518f9119bdbb1, 0xf5eee74d2c083ceb, 0x84339f3983bf3821, 0x77f38bb799e66f72, - 0x6384836213cfe568, 0x31b19272b7d6e282, 0x8ebcb989427274b9, 0x83ad16a35ec01c20, - 0xe40d88ab034bd5be, 0xe5d98304c6b8f902, 0xf3e1b6b085ee44fe, 0xeb69024b8c67c3e7, - 0x871530b5c9c379d4, 0x3a6136e94490a298, 0x74d8ac991c2fd663, 0x56b017fa40f23688, - 0x21d0835b0b4e5d30, 0xa4e6d23c3230da4a, 0x3cf627e6983fb667, 0xe07422ac44e5d0e2, - 0x7ec487eb1cb40a8f, 0xbb8b5c00b214e189, 0x5f7f653fb306e234, 0xacbeae08d54981b6, - 0x8bbf553366650bfe, 0x4fde25e7de1b76f2, 0xe58d16b80d602865, 0x8dd89fab086ba83b, - 0x2d1be0be2e4c8431, 0x3d4a1c578f464e43, 0x8f02467892d9e9d2, 0x07b4664d65392aa9, - 0x9542f599b8b876b8, 0x2a3f6592f6437e2f, 0x2ef9198a465f1f51, 0x52af5475e8457409, - 0x53ee06241ae684ba, 0x35172fed1a2265d5, 0x86f474792f0cb1aa, 0xc59c3b4efce83068, - 0x6a82c258e6a68e0a, 0x2dba97c21c5d9243, 0x3aa5f441543f22b7, 0x233ab0786acf0391, - 0x710c431056a6f7f4, 0x49eebc3f2f077317, 0xd8a39a2a416e30af, 0x109653581fabbd49, - 0x0ace9e201bc67e75, 0xabf33d31ab0e76f4, 0xf981d81ee0baf61f, 0x7f538a85e55e4588, - 0xbc263292bb777014, 0x6badade602f7253d, 0xec1d6c4c67ac5cc5, 0xca1b6f2c171a9e4d, - 0x22c2b608506c3cba, 0xb4caac4cf2e79a01, 0x0d52531f6c218b3b, 0xf8923bc043421983, - 0x4c3d0e0cbc4b1d18, 0x9c58c72bb5f8c5fb, 0x989d65d1b04218a3, 0x84c85da326ffd5f1, - 0xbff6b5fc5b65d3fe, 0x6cb7e8d83f8c24cc, 0x872d9b0af4f66868, 0x2f125c9d067993c6, - 0x51877cc204b45518, 0xb800d0be947d42d8, 0xace70520fc99126b, 0x61a17e75a5336b82, - 0x67cedd49f48619c8, 0x2f9673a9afbb71e6, 0x5073db71260cc292, 0x448c2a1b55570b56, - 0x77da32f818a3b50f, 0x7f36ff2fefa9af64, 0x12cdb18437b3f33a, 0xf247edce222115a6, - 0xa7eca54b74369245, 0x795ccb5735878a5f, 0x38bd04ffbfe19a28, 0x7dc35ba872f21bed, - 0x66d46e400b12aa6c, 0x70df3eecb5bfa9a4, 0x1cc766ca209c6c6d, 0xfbe65164b5959189, - 0x5862439edcaa33ff, 0x9821ae06f831291c, 0x71a863f41e5117fd, 0xc8013b44bddbb7be, - 0x27c83de2ed61eb6a, 0x48321bf0c970d33d, 0x284fe987fdc8d7ef, 0x87d565f1896c1f10, - 0x661fd2074afb3cba, 0xd5ec27d77918adfa, 0xa89f1e0c97599d20, 0xb1b4a89648840dc2, - 0x75b40b8c43eb8a20, 0x9692678dad79eceb, 0xa8c57eb5b490ef78, 0x21268036a1c7034f, - 0x9df3d1b875b65c85, 0xd3cfdacb73acea3f, 0x9899dd26f0aa0d3d, 0x2aac0078858de882, - 0xa8fc21d78c2aa62f, 0xad6d95591710d5ed, 0x2419c6674433e4d9, 0xf0c6456734332146, - 0x9f0de4acb7c1b412, 0x03ec90e1f8bd475d, 0x72120555863283ab, 0x5a89377105536621, - 0x4368b8f2134b88a3, 0xd19d63c5b77ab4d3, 0x15f6da163b056797, 0x0e1e63c4591e597f, - 0x3bceab2ecc5e2c21, 0xe61f0028cdff61d3, 0xbb434bd614517d65, 0x1b3fce3b1b36aece, - 0x8a336878642b503a, 0x526c3bcc62fe601c, 0xdfecea72744eb429, 0x0e5a0be0d29463c3, - 0xe65fdac84f3d49b4, 0xcea1e678f0ef59fc, 0xec5e745b3c0f3eaa, 0x15e3cf8f87f1ddb7, - 0xc92b969c306838dc, 0x88bdd828e0a00bc4, 0xe95aaa92595dc625, 0x50e3ddb8f8fd0ff2, - 0x676dd39c72a6f6e1, 0x70aae36bc2a655a8, 0x951a7addadbec8b3, 0xb7628ee42ea0021d, - 0xa4fdfc540267dc30, 0xa96f05029667e19b, 0xba27caa9677ce9d1, 0xc7493b1160b713de, - 0x320cfd057d6e7ed5, 0x62fdfea4d6c6d807, 0x1d5d8548227f2242, 0xdee0ecde21c3d9ac, - 0x3b13039fbd8fe244, 0x6496af2762d7f528, 0xe9619223d95a990e, 0x5d2d2093255a4a3f, - 0xb4ebca36068f5d86, 0x5ab0a31941cb92ca, 0x6f2f9323abcd6fd9, 0x6f67f1256e879f60, - 0xd6e7ff72b3aada6d, 0x3762b2cd2863c4d7, 0x1d603175b5655667, 0x946bddbc2dc19226, - 0x987de35419a7d933, 0x7d86f2f359a85c2e, 0x450de71132c8869f, 0x72e8dfa0df411c77, - 0x0b6db364959c6fcf, 0xeb0e1888a10aa490, 0xf565e98e643c0641, 0x3f7c448ad5031d81, - 0x79df105e5b5af95b, 0xa1e8abce25cb142f, 0x5a35cbc1d9d3812a, 0x10d611a06edd15e6, - 0x9f77babee4c306ba, 0x3d7d172180c44c6a, 0xd2c4a662d3a5f01f, 0xed7a03824fafa873, - 0xc185ee2dde59d635, 0x99cf0e20c270cfb8, 0x426d2677850798af, 0x1322a9da827e310d, - 0x4196090440dde507, 0x30811e41ad8328d7, 0x7ff0d0c4b74b3202, 0x72bf567558123040, - 0xde14f4f17aa16d75, 0xbdd846139dc8f34a, 0x0549450383d120fa, 0x729c5700047fc5dc, - 0x3c8ae593d1210821, 0x230e086feeac20b0, 0x45ebb71fef0bf7f1, 0x934b9944a3af517f, - 0xa879eb545059240a, 0x751a2b418ea29a18, 0x777d20d7209e9fe0, 0x843000d22df88271, - 0x7085bf26495a94b0, 0x61b03e8a2a054e4a, 0x6a9515af372925f0, 0x0e0773f837758d0f, - 0x12eb6ed4121743ae, 0x378f42a1771e63f7, 0xe027ed7c29fcac28, 0xdac893d266f2e5d4, - 0x9b0d4d31595cb3b8, 0x1506b405f17551c7, 0x493d624887a927a7, 0x662051e349462981, - 0xb64a741968f9eb27, 0xbba798e9c0d9ad1c, 0xdf0625d211feeca7, 0x5a0149772504331f, - 0x15071dd0075d94bb, 0x689f489d738d4144, 0x42ae02215bffa3c6, 0x987ebe22d4daa7c6, - 0xf1cc775c5d41cc97, 0xa0b38fad2cf85e10, 0x84ed45020f1171b2, 0x3cac1ce1730c5e9a, - 0x7db1804767f617cf, 0x217f281d4a4eaa7e, 0xe3d133be49a5b74f, 0x5a1a3d1496576532, - 0xda8c285a5ef97169, 0x14745ce23041bd9d, 0x23f088c890ed51c0, 0x1029b7a723c3e53c, - 0x674bc5e89d6fc5e6, 0x5017a0459a179ec8, 0xe6198bf005593bb1, 0x0170c1be39cc95b3, - 0x468ce14573353f2c, 0x9203f3267670b7e5, 0x9993040e9e285b72, 0xb3033c9f3996e6e8, - 0x0695938e2cf285ab, 0x47dc35edd6c03aa5, 0xe0c0f9d611806230, 0x60f9b86f2bf5aee4, - 0xa33fa8e0c7e94ee0, 0xc94a517a5f02ad57, 0xd628a4b6d9c5dbfb, 0xae4c2ab7c039cd4f, - 0x87728f5fc31b8c3a, 0x3ed6562586f1a3f1, 0x7c1d176d442e7443, 0x5d50ebb54fd740de, - 0x63c018fcd9da2b70, 0xd19decb3c062cd24, 0xc7b7dfad6269b530, 0xdc299db355fa188e, - 0xb1db4e37f434cf2d, 0x168d77ee00d049b2, 0x0620c9eb28ee7514, 0x5098e90c072e2f8c, - 0x53973099e7ba4aca, 0xff1049c572e80f29, 0x7bc9c4319c383680, 0xfff57ad7c397a35e, - 0x04825d90c248c6e1, 0x95b1c05eb2e5d575, 0xf4affd2b3624a0d8, 0xc5210ce0b88cb876, - 0xdb4120df79f2d5aa, 0x5c95632fdf37e5e2, 0xc38e603d3eca6502, 0xd891ca5dd3389dc5, - 0x9032425229f651ea, 0x1531987eade0a9bc, 0xecba3582be2c9a50, 0xb59d49e01aaceade, - 0x06719e86c1d755a0, 0x2ca0ee64963639f6, 0x39dd706d4954ed74, 0x7006848e29e5060e, - 0x01bfdad962e6e9a4, 0x65dbb414cb2e9a02, 0xfbab422aa46297e6, 0xb456c38b6c17a3d6, - 0x63d6505083c9ebea, 0xb20234bf09326171, 0xb7cf62502d4e0214, 0x7ea2412281370864, - 0x4c81b1bb9447b8ab, 0xc3cebd5292277aa2, 0xb15e2827539b0742, 0xbcc1da131cd046be, - 0x8168d8b4b7437723, 0x7ff3d4fd2d4079d5, 0x87025a684d81e2bd, 0x799afc37001114c2, - 0xfe2b709fdb49f0f7, 0x1b91f75559b9ce5c, 0x3a74c17f8a6b86a2, 0x6a4f269c516cac46, - 0x1b1267d908ad7193, 0x1da280637b37aff8, 0xea347f02c6316af1, 0xde7b9c4934391524, - 0x7ffc891a7b2c28d0, 0x4c08ecc51a5d6ea9, 0xad3fd57d07e3d524, 0x1ec017537694e4d8, - 0xb825996259b81d99, 0x74cd9ff02dee6568, 0x001d5e9aece59417, 0x9ccc29d9a60d1c16, - 0xae7fc1716c901a72, 0xa0ca327697e59f5f, 0x851cc907aa906f76, 0x4e4fdf0b5e9ab95b, - 0x39b510ad035a4e2d, 0x8f1d892503124ad7, 0xfc4ef7e24a5bfbce, 0x5257999a00896bca, - 0x825d29b7c00038e2, 0xf82117045f12b2d1, 0x6f4acbaafe27cd0b, 0x686f3c442062a08f, - 0x44aa982b433bde56, 0xb3ca460b8e8e180b, 0xa29ccfdbc571b4be, 0xb430f59defb18a14, - 0x9da759d792ccfd27, 0xbd4c59d878a78764, 0x2afd8783dab73b5c, 0x571d7a4d138c8dbc, - 0x7f15fc05b490bccd, 0xeeedd768aff18a1b, 0x2f0fcae6a2f25f80, 0xa6ca90d5c966995b, - 0x13289e032e6fb38d, 0x8490ea970c0d06c3, 0xf52a61f2011a9447, 0xe91fb6694e53ff99, - 0x84399a0413af13f4, 0x893900a9ff1cd650, 0x5be096e2c2058cf0, 0x66fbf75ae8be8fbf, - 0x5330484ab2a0a22a, 0x5b8e052207fd5775, 0xedd78e9f0c5d30f2, 0x6ecff2fa1502daea, - 0xf75c33b5478ae11b, 0xa74a96a247b52fa6, 0xac5498070996d7d7, 0xe8a8cf79fc2b4473, - 0x8b73199e324e9b9b, 0x349cab8ff252b192, 0xed738ba23742f1a3, 0x6b6e1f8e048894c3, - 0xb07433b2023827d5, 0x4e4cb166f163dbbb, 0x3dbb80b56b3c3231, 0x6c6d3251355e83d2, - 0x03061d1a414bd749, 0x8c61f7ec974cd6ff, 0xb68223c8ab5a894f, 0xe6c52550bf90276e, - 0xfd163238aa076a37, 0x5f40f5eee06af68a, 0xb821c1ff45b876e8, 0x4b1e9aad263d8451, - 0x41680eaca3692281, 0x7d2dada50e6fdbac, 0x0113a31d85c17f4a, 0xa189c5bc75afaff9, - 0x576885b1f6e611fa, 0x7778585cefb260fb, 0x5e71b49148be1cdc, 0x29fab567e18a9bcf, - 0x72fc86e88bea8836, 0xcb37bafd31b208f2, 0xc49af8ee7812a44e, 0xb1e76ea72e945ab2, - 0xce6e9922821e49a1, 0x0d6221ae508e6f29, 0x41342d026a6f7e06, 0xc258f087b6b260f0, - 0xf4d74dc8d0b2fbae, 0xb890ef947e9f0d66, 0xcdcdb785f113bf6c, 0xa252454ffedad13b, - 0xd40f4beb7659e1e6, 0x67aee41fbd7dd50c, 0x896d151ea8db9182, 0x2d34b792ef05111f, - 0xbcca99caaa6f7db9, 0xe775ef2082276b2b, 0x8628144d29e9eecb, 0x499308ff64a4859b, - 0xedcaf1b2452319fc, 0x6ef705e743a46f68, 0x684ec82718a7fbf8, 0x0b9e4c93fe6d6d6a, - 0xbd7d723e4a254236, 0x483504af05e28494, 0xe2f72a26d7ec7b14, 0x8d725a203384bd62, - 0x73ff3255f1b89758, 0x922f9cd027c65750, 0x41baa8bec165e752, 0x57c496c8677274cd, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x9047673fcac14893, 0xf5df5d83bfb58659, 0x0a6230c81642e71a, 0xef14b33800777791, - 0xcf1e99afa3386fca, 0x7ace937791313d53, 0x36fe159b6dcd01bb, 0xc9bc50d02e2b960a, - 0x040bb31b201676b0, 0x0ec2968aea11f66d, 0x2fc408da505cca19, 0x6c832d1443ac40b7, - 0xb559db3ef08bcbd9, 0x4619dc5b7ce37c35, 0xcc8f60bffab8676f, 0xc1bcc30c926719f7, - 0x97818696139b5c07, 0x1352b3710edb750a, 0xae8aa5c49b0bf7c0, 0x5d429ce2082b25cf, - 0x9cfcdb8fc65053d4, 0xd1f5167196f4ee2f, 0x276839e56f27db1d, 0x1d3ad2c90bcd7a33, - 0xdc353dee3a27c81d, 0x2d3b38e3a6976aa3, 0x0813bfaca25c1503, 0x96cc64bb5525f09f, - 0x239651d5384e2ae5, 0xeaec1df9005315cb, 0x7d16c624608eb63b, 0x12e5b07506d70308, - 0xde13703bc0292e87, 0x08731c61c62e4f0a, 0x207d168cd9483f9e, 0x1fd175d47f6b2ac9, - 0xd03f37e8553c8ad1, 0x1c6b60668971f140, 0x1b45dd6e969e03f9, 0xab1ed433d819eb09, - 0x3b85cecbe3df3f1f, 0x7ee7bdaac9e05a8f, 0xa33be189a0c420d3, 0x8d606d1b0bc85b78, - 0xd7d839fa60937c1d, 0xf319d371cec541f0, 0x61c68906cd53aad2, 0x7b0446d5ca7144ac, - 0xd661e4653aead29d, 0xffaeea7b44ecb36a, 0x2ee6e08d79dcbbe8, 0xa3d5b7421ab10e4b, - 0xf5087cb539f798d8, 0x5901915ad5fa200b, 0xe4dec474b6f7d54b, 0x9dfcd9a7ec9a34f0, - 0xf28bc64ac939257d, 0x79808cf9b66df416, 0x7efb5643731ea788, 0x88875d45971ae24a, - 0x4fb8bf9dbbeb3662, 0xe736648df00e1cf6, 0x0ea8079220656459, 0x83c449c0f1ebe005, - 0x5ef0b6a9c5467b79, 0x69403e0541a453b2, 0x4fc2adf12672b1bd, 0x7c21cbec7ce687ec, - 0x2c92997d7c3b7f12, 0x99a46daab6b1f1b0, 0x6e2dffea854385cb, 0x936b02f71563286c, - 0x84284d45fafda110, 0x8c721ce865fb8903, 0x408ae77f74d383f6, 0xa2f2741356c91dcc, - 0x913941dee5b96db0, 0x690159ee0257c2a5, 0x6f8fb7db6432fc87, 0x087439128197cc2f, - 0x6f8b341061eecf61, 0x260fd718bf5c0365, 0xc2e619502f7236ca, 0x6f9a721b229b2184, - 0xc3f6c8141648291d, 0x6e4810fab9dedcaa, 0x0f10a86948a16410, 0xf34948b3e17e9c87, - 0xe87101ea0f9655e7, 0x44002ab99f3bbe06, 0xee96c7010aabaef8, 0xd14a8243cc6af2fb, - 0x003a859b2fcaa60a, 0x3b2dcd4ced18b81a, 0x13953a4a3021bba1, 0x27f7d10393d72815, - 0xe609b75c79a37e29, 0x99cb26cb848eb000, 0x3fd7ac040c35ee00, 0xf9ede507eedb57ea, - 0x7815e8aefd8f34fd, 0xe27d86ece1ebe20d, 0x154851fff7881b22, 0x8032d036a7beec03, - 0xcba350c0bb4b1028, 0x2eba4f9969ade796, 0x58449949fd0d6097, 0x17bf307f509c09d9, - 0x8773671d5065e23d, 0xd0fdfe265cd935a9, 0x721e1720ba1846d8, 0x14ac94bdc7525ee2, - 0xf2e37fcded0a86f7, 0xf992c715b4e8fd95, 0xb84bec2e6ff8206c, 0x97d4c50e0c566aad, - 0x0846292e0aaa92ff, 0xc12509255198cbcf, 0x7e8ba7d812f9e2dd, 0xec377e69daf3d788, - 0x60ac0d9044d11582, 0x4e70869ed02f01e5, 0xd06cd1baea488fc8, 0x0f9c80bec6af19b8, - 0x1193af79d861b588, 0x6b20b1a03ebe0aeb, 0x58204d8f5698176c, 0x490239ba4ea59823, - 0x37fe263d715d646d, 0x45f1d32e98d64039, 0x6b0eb2eabc2463e3, 0xbfd88608648fab65, - 0x5a26e5b192db6e20, 0x945c0433dd1f5e24, 0x5a4f667ccfae0e98, 0x6120ef6b2a01d270, - 0x80170a58acf6217e, 0xd9eb4d58f5a4dc75, 0x1e3cc79cf142ff9a, 0xd72d95fec686c8c8, - 0x664899f524b3c9aa, 0xf3fc08338f8cb900, 0x54b810514e704f2b, 0xe188019e4edca012, - 0x7fd4614f0a6cd109, 0x9b3604bcd2938cce, 0x5e73c984c45c2db8, 0xdc32545c788abef0, - 0x167ff4246298f598, 0xfddee40ab4708649, 0xe6222358a95ba704, 0xbfd3677a87f72d39, - 0x8ac68566d599fabf, 0xdbe08e867eb26d43, 0x4d405de2b1dd6c96, 0xa6a536e0ebe2930e, - 0x4eae5928cfab2920, 0x657f68def3b1c442, 0xdf2796ad0afa8840, 0x9e7c7eec3345bbb8, - 0x1c577ae5ab675c2e, 0x00715e8eb344fee9, 0x1f2086d06307505f, 0xa95665b4a4e623a5, - 0x9b24fc0615ad16c3, 0x4a2f3e2f6bdedec0, 0x3d2b08376e528343, 0xabe8b21c18d5dc71, - 0xadc99d655cc19f51, 0x42eff4a6a1a8599d, 0x043d1f8da90da7ab, 0xd721af08ff1a71af, - 0x93db3595496b1830, 0x9f4ccabc4c96d827, 0xb772df07557cb467, 0x88e079cc16360047, - 0x2a1f96a73a90f4fa, 0xf10db5ebd15ce239, 0x87334237ad7d1bf7, 0xc155cd2cf8059a83, - 0xb55d40740ded930d, 0x06f9098294132396, 0x1736fee40746d597, 0x4ac16bede818e107, - 0xc320f0e02eee3284, 0x2a9c909c50ae29fc, 0x83ecbd32fceec5bd, 0x4ab0fcbe1cb8a7f4, - 0x9a00fb20d47d4ea0, 0x416a8e3524502589, 0x1ae65970f42fec71, 0x5475afa54e43aabe, - 0x764c7049a72b0b56, 0x46916300a697b1c6, 0x2fd02c10bcd0f0f8, 0x49883c08c9341738, - 0xf2b5feb32ae187fe, 0x6ecc5e14c778cb64, 0x3f87fd93e64f5dee, 0xda2182c9580f8267, - 0xe00aa84df80755ab, 0x85fc4189db75915d, 0xe3eeec60bb608a2f, 0xf837a0340af77ebe, - 0xe53dce5ec75c7cd8, 0x50cdbb01b1bff896, 0xb5fcd383265deda7, 0x6628288aa932d676, - 0xb80e0160bc747ce6, 0xf9d6ec305dd13f3c, 0xec776301fb36efe6, 0x038010de21f50046, - 0xa544ebfc9a938ded, 0x732f758186bb6344, 0x97f506b12647a5ae, 0xb7ed31a650a155eb, - 0x7da4d67730b426a7, 0x9c153fb31233e5df, 0x71b406ef7b1b8e6d, 0x549bacb3c315aafb, - 0x6535f70f616d4f7f, 0x5ddc1992e41d46a4, 0x3547e4ae524d4859, 0x064bdc6dfc64f41b, - 0xe01dc74278dc6417, 0xd7db16e4bd4e5bcd, 0xe5d42ec7791d8f71, 0x42b98a92a58db944, - 0x92dae5d3231b4f37, 0x0cd0c91b3cf9dead, 0x41f9c69f0510ff38, 0x516a3aa8df57197d, - 0xddbe0c9059b84803, 0x2c0289ce6ebc6725, 0x96ae36415f4126f4, 0x7c1602bab0f22a23, - 0x378569bfcb1fbc68, 0x9c6a46d8686ecf2b, 0x71fc617cdc9efecb, 0x2a74bf39da0ff7e4, - 0xeee33823afae2f62, 0xfa591bb0dd1b2335, 0x17ab2165d29696a0, 0x4b90400013c938dc, - 0xa9b3d40b9d6110e1, 0xdbad6acb7bf9de11, 0xacb1a1775bc23161, 0x4fba68b236217116, - 0xd97654ce38feba31, 0xaa2e546373c923e7, 0xa44442ef7c540bd5, 0xde7abb2e6ef6d26f, - 0x73e4a47c3b1980b7, 0xb5bd2c144799f4b3, 0xbc6686128484799d, 0x8fed99e39663d286, - 0x30171a6a383e1a23, 0xffd8bd559be558ca, 0xf00910862b7a1470, 0x886470f922b471cf, - 0xf42a90c4d6922f9a, 0xe0e65b5cce61c367, 0x7506a3a58ecb7103, 0x030a3f4f865e9254, - 0xc74576172bac8f38, 0xe07d0ed514139ffc, 0x6ee7360351d5b871, 0xc36e68fb175aa029, - 0xb78fd72588c8ce8c, 0x4d2262ec66fd1c2c, 0x71d087160e59940d, 0x81b315613d76f3da, - 0x2d07fb33f46511e8, 0xaaa1ca308dca5d77, 0xa817cf48b1eb132a, 0xfe0f97f0472a76a2, - 0x0b06cffabd597dc9, 0xbf77aeefdfd14b9c, 0x48762cff0a40e19e, 0x5689eff225edd4a0, - 0xb631345cea83f8d8, 0xe02818f70faddb59, 0x676ab534fe672577, 0x52085247b6eebd1a, - 0xabb2eb1f5d1aaf5d, 0xf261c9da69ab7a98, 0x0a92ed4cf1640a60, 0x2b6b686273a8b9e3, - 0x82f7739d23384a39, 0x089d7ed37d6610b5, 0xf159615161327d6f, 0x42e1afc32d9d5e5e, - 0xf2017bf6018a082b, 0x64b692a5c33e2810, 0xee7c90603920bb75, 0xd57c3489e40b26ef, - 0xd24d0ddfb23a85f0, 0x82a2d2dd8b08f1af, 0xab9f183893c4b3de, 0x18253a427c5ebaa3, - 0x97d6ff753d374466, 0x10832a04f230200c, 0xb4f668dba53bb26f, 0xb14b464fa5fd5f3e, - 0x749d3b51530c8c71, 0x9ad7e8fa4714b195, 0x5ab792dff70b9331, 0x6885e36cda049ef3, - 0x89f6279b46d5b992, 0x06533680e4756496, 0x7b4fac7a4020aba5, 0x61ef8e3c9706c90a, - 0xceb2aaf6bed362ad, 0xcd4fc12e1d88ba50, 0x657f8f0ac852c790, 0xaa75b507f35cd87d, - 0x3f5c255bcc15a840, 0x3c3deb5262ac07c5, 0x98806d16aebe4228, 0x5e3407d96651df66, - 0xd28356e280b59efd, 0x4ffd556a3699e910, 0x5b0145955d6c9bd3, 0x4bb63c3e5627ae71, - 0xe249211d1aea852a, 0x89dbc36b34e29feb, 0xe3587ced0fca9b7d, 0x4c95b345a43e57f5, - 0xd0fa2c0d4c32bf4f, 0x767fe0c9702082fd, 0xfeead6cd028c9df8, 0xaddc351d8a975a0f, - 0xe12a4386968e46a4, 0xb3c8e9b1412b4bd5, 0xa46dbc415163adb9, 0xd8141016c331fc22, - 0x8407b287b8e3ef49, 0x8e030203b6605de3, 0xff92723f624215c8, 0xec0d04f354187b1e, - 0xbc851fa11a5663aa, 0x9916cc8ab0fb1a4a, 0x7e45bd0f23e259f5, 0x2707fd2b2c61ccd8, - 0xb9712ae0570710f3, 0xf5c55bc9e756d801, 0x34a11080cd2b83a8, 0xcd151f3359e1c7d4, - 0xfc64d7784ee7d42c, 0xe6540dc8225418d4, 0x6f2d09e0d522b641, 0xe833e108bf792163, - 0x081808dcd2aad2b7, 0x73f752b43fe33e9c, 0x48f08457f4dd0bc1, 0x1c108a76247aedd6, - 0xf0a38308a452c91b, 0xf5ce33fd5625cd76, 0x17b049b6f178a34a, 0x8fe94b4e4c86bd2c, - 0xc8ae5e3f64f4130c, 0x0b64430df344de64, 0x5c4e3f16d96c2ce4, 0x267a6ec044538071, - 0x19cd5c40f04fee03, 0x190b95907aaa2522, 0x081898ada943bf13, 0x256112ca5bde7cbb, - 0xf3f9ef81364d15c4, 0x079dfdbf65eed65b, 0x2b30006177d32ebf, 0xe584dc9984077384, - 0x41340a167edce5f5, 0x21234cd5b9d08fa2, 0xec37663b6234366a, 0x2e379b41de33d8fc, - 0x6bc76d7b90db7b08, 0x755ae85383da07ea, 0x7d35c83c8e677068, 0xa998c5fd0746bc3e, - 0xdca7a223e6d31ea4, 0xfbfe36d124722355, 0xee5af95c7ece7923, 0x338c1178fa5704bd, - 0xf82b1b1504ae6e3c, 0x89394552c02f6623, 0x6f1c2ce702ab5ffe, 0xb754866aefdacf19, - 0xfcb6fdec00f28435, 0x175080b1bb6a7c4a, 0xa0b548d4b93ec592, 0xda53c7eec16dcc2a, - 0x710e11c460eeb380, 0xa4b5c59f3b1307a2, 0xf796264ffe6e2b25, 0x9c759cfe5815c886, - 0xd765ed041a702d92, 0xdf7db56d650151be, 0xd4e4c6412eaa2086, 0x8bdcaf80a45d5ee7, - 0x375be23e9a328196, 0x872869e151ee76e5, 0x0ae181edc59ab9e7, 0xc851492cafd1f757, - 0x96d5cada42e77ba8, 0xcf2c3d67a7ee8153, 0xd261876db8cb0b94, 0x26e80f5dc8c0e602, - 0xeb5d96b5c0bbb4b6, 0xcdf05b88ea104572, 0x587c877cdb82ff3c, 0x167bd6422f0448f8, - 0x3388cd9e4908ad7e, 0xfd0ef48486aefae5, 0x9a1b9d755ea0fdaa, 0x5f7bd2b427417aa5, - 0x59eebae112c291c7, 0x200d4e72610b7db9, 0x6755b2c42685be9c, 0x75515a0b0f648a7a, - 0x98d502c0deddbaa0, 0xb0561bbcfd5807f5, 0x5b1ca5c48a34f7ee, 0x17325bbdf6c659fa, - 0x69cad76e13d49b97, 0x6db4e139acfc1261, 0x462f0731f240985e, 0xbdd35c2b6ae2bdd9, - 0x38c401746f686e69, 0xa3375aa966f6597b, 0x0331af7ce06cebb6, 0x5409b343e1eb2ae8, - 0xab6d51e69d12c63c, 0x674c2dbe84c5c870, 0x69d394ab58d924c1, 0x3325e8d6d32d659b, - 0xf9a3a84ca9d037c1, 0x7f6b27e2e2878c7f, 0xfc004c1ec135dbb2, 0xa1ffdb8a8ebb8176, - 0x9a04ed00f9cc2e66, 0xb98ff91cbb39c793, 0x1462f86071db87a9, 0xc0eac4f4ea6b30ad, - 0xf2f49134b43e1436, 0xc370f8931936a64f, 0x7b9d852bf8b50d00, 0x4edfd99da790719d, - 0x2ff12982da305ac6, 0xfdd8368d2c0adbde, 0x00115e3ea4be0254, 0xdbffacf4eb40730c, - 0xe38ad75138fdce99, 0xf5e7850d56a4e003, 0x993257196d043658, 0x95503cf966029f67, - 0x23689c9b1a3d574c, 0x2cf177999c9af4ef, 0xae2c32f12205f9d9, 0x574de0e37c6b7c88, - 0x78a8afabe6b2d06f, 0xdd6a2d80a9a7056b, 0x2376f8f9a2b52754, 0x6160c5fee51348b1, - 0x20a318cd7458c874, 0x48f37465fc005859, 0x49b3207280c4bffa, 0x4f8ee277e21892f0, - 0x63dbd578961696c0, 0xdd794100ed20a014, 0x330d304c3afd1f22, 0x9557b6fd1fb1cc3a, - 0x0ae0e85dcd63ea77, 0x0ebcab4dcc00f616, 0xa839e7d9694afd2b, 0x0ceacc614a359de8, - 0xbd19268dade01e56, 0x121b33331be7e9c5, 0x371feea5ebfbac35, 0x13949281c7c94de9, - 0xc3aec0bd5bb9db20, 0x389c0270b5613a39, 0x02fa85f10c1ffd73, 0xa44e64c3e41eea65, - 0xe877d950290abfdb, 0xe49782dd124d9dc9, 0xf3dc8a21032cd01e, 0xf20a9ee77abf34c7, - 0x4aa5177e15fc5f07, 0xaa75ce560be57a82, 0x38dad4f3d2249aca, 0x4e9104384c74d0f3, - 0x00d62508aa60b6d9, 0x496bdf56036e944f, 0x62da10dd30bbd8f9, 0x157861acaccb1e59, - 0xc7686eee005fdbe6, 0x86881063b072906f, 0x267273cb7a569b23, 0xc5a733d9f71596da, - 0x01ecb9e91d932421, 0x1071c7b321f968d8, 0xfcdecb42e044c3f1, 0x762032743215e3e1, - 0xdb9815ac6e56ab28, 0x76518c8356bc0193, 0x7284107f42820171, 0x905390d72adac3cc, - 0x85091fe4b98fe45e, 0x33eaf559079874d3, 0xce151f26e904494b, 0x79a296806ca41c54, - 0x7e735eb3d64fbffc, 0x7d2994b3aace2643, 0x509fdd5b4f5502da, 0x54b5260a2f269c9c, - 0x9c3b3fc85903abb5, 0x5d32c69a124cd715, 0x18ff8d2b8d2600fa, 0x133a04d3b63eb6b4, - 0xf8046076fda2950b, 0x019a9cf0c7cda9a1, 0x4739b4f393663119, 0xd7a9897fa25f61c8, - 0x74bbc610a675656a, 0xbff1f87f963466d9, 0x4f1a104a67167f41, 0xb414235dc1fa0392, - 0xccd3a69d2e5a7db7, 0xd04ef93cb6149ba9, 0x4abb0f17eef19d12, 0x116e38e6bc26d4f5, - 0x6ef4c0d0f7d45dd6, 0xad04036499e68ead, 0x8d36766959ec5776, 0xddf82e7868fed989, - 0xa1c4f95f31d143bd, 0xc7c20431e10374d6, 0x3085cdb88e7dfe7d, 0xcd4e45c5134e814f, - 0x3928df7666729459, 0x952221b37ac24ffa, 0x70eae01d6bc6814c, 0x7bf3b3c926b59b02, - 0x6912711c73c40099, 0x74b7b127c452774c, 0x487de688fa263700, 0xf7f7eb4723f89ee4, - 0xfa0d7ce654cf50b6, 0x173a0fa491298044, 0x3cb431296235b2ff, 0x35a47c0fafa5648e, - 0x6602fbc3f8b4987d, 0xb8e90610fd126dbf, 0xb451552602f208cd, 0xc32399a43a4bbfd3, - 0xe2b28b2d9a657d46, 0x96bf93d367416718, 0x277a2f8c21a5c41f, 0x91cd6817f29e6f56, - 0x9e0ab8d663668355, 0xac4cb913cf9bf0e3, 0x9a714a8ee63544e5, 0x32023aca9a9e7926, - 0x420abbd05557ffb3, 0xbf33f91590d0bcfb, 0x57c3e9d22a2f9893, 0xbef0306deb15abf9, - 0x1d9d1fa99c39614f, 0x389dd8b38f179997, 0xd0c642e3f662f0ab, 0x8ae111b61e4dda14, - 0x6bec8d8f02e2ea38, 0x537114315396945e, 0xa29d05afd17f2a69, 0x28bb1981ebb348c8, - 0x53c74d3d4923a880, 0x81ae4405f813880c, 0xcd7878159be2f9e4, 0xb261482e8b0a77c4, - 0x43b04310c6dca8b1, 0x6c6b72e057dd0587, 0x4fd5323a6cae0894, 0x787a3b7a98d143f9, - 0x33275c30e096fa03, 0xa2f33956b8ed7358, 0x3f5611a025269192, 0x67ccb469b569d3c8, - 0xe4cc519de140e2b9, 0x74d825f1cc0ad969, 0x609882ff7a7d4efe, 0xf3999457451552dd, - 0xec0a9966a38cd73c, 0xdf2e09d5cbe737a4, 0x1bcd6e30dd37b800, 0x9a1d255293e141b2, - 0xf8516f062012b17b, 0xa52dfa41756c393c, 0xd7f16314cb746823, 0xc348497dcbcbd9ca, - 0xe24253cffdab5664, 0xa54495a5e7479269, 0x0e90de2bb508df0d, 0x31f2dd61d0fc886b, - 0xc6899e44469aabe5, 0xb57c7fd64720f1d3, 0xefb6c5bf66f8f4d5, 0x3e1d4a6abe4f6a00, - 0x0b910643b6e239cd, 0x94b0d6a1cdd618c9, 0x093212d61dede103, 0xda593bf228bb8d45, - 0x04d48fb647e19a3e, 0x94a8187683113bb3, 0xe7780efdcccadf47, 0x597892cb5f39e7df, - 0x79a38697afb51437, 0x9ad60aeadb42d0ee, 0x6bbf99cd2c902ed7, 0x8f2684d36c8dcdae, - 0xa8be8718386e785c, 0xb4a0d5d260b1c6af, 0x4d1cf89405cfc52c, 0x0e16a1bd34f26032, - 0x61d3f841354d7b5c, 0x7719e546a276a2cb, 0x0996c7c672760c36, 0x6c9120ccd7aa7f5a, - 0xee5504e1ec78a23a, 0x68461a6d8fd02984, 0xe5806402dc4bb5c2, 0x34c889e638d4459a, - 0xa016cd97a98017e0, 0x58073ac817a9ded5, 0x978db22b7424afbe, 0x534bce48e9dbebe7, - 0xf95605185351fe4e, 0x3c0c6805aca0da01, 0x4a20b5efdbbce17e, 0xbc8e87ccf43cc8d4, - 0xb02cc35858065281, 0x8fe079608ebbd641, 0x287ccd0c84fa59aa, 0x09df7772d487570c, - 0xe2097fd4fc63878b, 0xe3e3b39d50c710b5, 0xfc94fc002df01826, 0x6370c3839c5922b7, - 0x72cac6edbfa07561, 0x21d5d210f2ce876a, 0x08afc84c65087b1e, 0x2b6c3ce9b4facf5d, - 0x79ce72ddddf98ed6, 0x2118a9d5718ea663, 0x9dc1da7fb2d1f121, 0x2ab7d6264e129b63, - 0xeaeedb2970f4cbe2, 0xba3c0339c33a3bed, 0xf313ed94c23a206d, 0xfb93488880bd28ea, - 0x9139c530fcc2190f, 0x459885676868ed7e, 0x0914b7d3bee447ab, 0x361a6a47ff2deda6, - 0x63c272ad2f3a3e8a, 0xeb81d9da896e64b0, 0xfd5255554953db67, 0xab70df9766fb99e3, - 0x3045e540ad6d2bc9, 0x118ff7e43de73ace, 0x132aedb7aa773627, 0x8cbc604da58bc253, - 0xd0865d5c1fa63111, 0x346754e6444d8555, 0x2e9d69fa4cdc8c1a, 0x21c0810fe2e82e89, - 0xd957b1bb7a293fd8, 0xa6c850d75a07e806, 0xf2e73a4313cc652e, 0x5b56b203bffee30b, - 0x69c8718700b12d95, 0x57d7bb6ace06fb70, 0x656dee8f94227e69, 0x74ec5936de02820d, - 0xaab4fbbf903e35b1, 0x77f818762d97d594, 0xbb69d5f758d452ca, 0x908e2df7157ea4b2, - 0x449057d807f41fed, 0xcb6ac2f12aa07545, 0xa8be2722d28a6b95, 0xf8671ea7092e76f6, - 0x925b50a22aa38ccf, 0x624eba124ad5a87d, 0x3ac7b1b2e6460422, 0x5e18e8e6e77342a5, - 0x613ec3c930c8e031, 0x482ec7a3082e0896, 0x0d7a84e5353e1797, 0x5de2a00a176b65d8, - 0x9eb6966a8cee7649, 0xf526ccc038ef43f7, 0xc29369890fc41996, 0x5e9f775597e2bef7, - 0x374644c5e7bc0af6, 0x846bdeada430cec7, 0xc7b4bda64e0666c8, 0xe24917e9d33fe50f, - 0xf584773618b9f9ab, 0x4f73a1bdae884a8e, 0x71fb850cd7d853cc, 0x63ef08ca54b99c44, - 0xf9a85db320a3c22f, 0x6d0627b72d074ef7, 0xdac5d5de845ff721, 0xa2604a8b5869298f, - 0x5682f7eb49f1ba4e, 0xace85e78a76350d8, 0x272f6897df79d9d2, 0x151746d5b9f7dc15, - 0x88741c16d75e7244, 0x32dd2bb82832f4c1, 0x45ce6199563e9ed9, 0x6cf07ead5dd4aacf, - 0x5c6b777802f22691, 0x1798686649e27e14, 0xc4db501c08980bdc, 0xa960ae87a3083375, - 0xccf65a028ff15848, 0x2463757d986554c4, 0x1474027af782b551, 0x8e822f3b45f982aa, - 0xf6f9f02522ed6e31, 0xe1b220310b6a6ce3, 0x128bc35d7f4fba49, 0xc7b3da87a3bdee2f, - 0xe2b570ea49bd9e99, 0x2d5b00c0fd982625, 0x2ecedccd5fa00907, 0x4d3aad864c2e7ed8, - 0x1ede941fe7133473, 0xab3dde2a21fa8df2, 0xcea088a761ec7660, 0x7a012f8a113194fb, - 0x4ba7182851566ef2, 0xbbb9edf4a3175b7c, 0x5db5d154453c5e78, 0xf9d1cf8a354931d4, - 0x683228f881cf7055, 0xbb7942d1a8f673ab, 0x57e6e68be4d29688, 0x7169c1468645bf64, - 0x9ffb798c7d9c6962, 0x3826101e091418f9, 0x136015f47ff06c13, 0x97adcec5738d7b63, - 0x126d676f0d45ce6e, 0xa7fba10da6b5b84c, 0x44bd57eb069e009c, 0x6721f2e295187c9a, - 0x547160cf3182b14c, 0xa807dff9b3a34f95, 0xeb6ca6ce36820742, 0x6bfc1ff1eaf6d6f1, - 0x526093199f0c9889, 0xc9f5cd3ca3f7f257, 0xcff422ea53b0f082, 0x0653e31de2bc9e17, - 0xcdb7feae875f72be, 0xf6ecc1005ff93ac0, 0x2541a0c8e9719898, 0x9b9e0d49bc2b82ac, - 0xd6fb1422cdb47d2a, 0x0c5d5f0a2f33fd1c, 0x816421b5d067b790, 0xeefde0023ac74e4d, - 0x65267f5851f19c76, 0x3a328eb8db150775, 0xf59e0ccd25e19614, 0x2364a745ba132ad7, - 0xfcef6a6fe9d5592a, 0x581669a8cf66f002, 0x904e1e3ad4939ffe, 0x3d6037d71d61f379, - 0xe053b27b815d868c, 0x750b387070f6a4fb, 0xe3e2386ca0a967bd, 0x224faff36b421409, - 0x7a223417d217e426, 0x4966c63d690bd388, 0x596f15dc2d80014e, 0x6ca22012ff7559be, - 0x3632136a72b1135e, 0x19c0c8da9acb2c03, 0xbdbdf8c49f3f2eaf, 0x369a385adec88694, - 0xc966acd4b4c5afaa, 0xb52eaab4552347fb, 0x50e7bc4835e2d0b8, 0x69bb697ae8083795, - 0xad88fa20a5caa602, 0x8ee3ba7ea8763c97, 0x90e91a78284f9a7b, 0xb92bd1bd1e29889e, - 0x1c8828b48978a4e1, 0xf9c2c7065cadd02a, 0x413564a6bf660c8c, 0xc2e026c6fe56397f, - 0xb4c1276da5712e91, 0x64cbcad64f53f498, 0x3cc5f29dac77804f, 0x72deb443ce26da48, - 0x1e13ab533cc1cdc0, 0xb00ba869f81da3dd, 0x27ee4146027d4550, 0x9fbdc389b05bf8ce, - 0x3099d9d080fae68a, 0x431723c2090613d5, 0x4786bee900a827ba, 0xa9590dfce9851b97, - 0xbca3016ba0950b1d, 0xf6ae4147435e92b1, 0x630fd1c0c317a3a4, 0xd5a1c1bb8c8d10e5, - 0xb3bc9fc0575e335f, 0x3862682cecc697af, 0x144a8cd1f8b96183, 0x6d18b1abb0c30772, - 0x7b05f3c6773cfe82, 0x7263348629b14cb6, 0xeed7b96a9aae65d2, 0x5bb0e463b322b047, - 0x7a7913b41b02eaeb, 0x651b0fbe0c92090f, 0xfc22605d223c8669, 0x34747217c2faf57a, - 0x4d21dbedf61116c8, 0x3055d23128bd3de8, 0xe86b221ab4df7c4b, 0xc04e3337fce58071, - 0x0efcf24f8defce26, 0x2c9513b745d883b6, 0xe39c8ac4470fe585, 0xe07c4705b83ae512, - 0x10a605249157f52e, 0x19e813b5e666979e, 0x8b1086bd3fae0b9b, 0xb5508680eb8b0571, - 0x4f0a0d595f383988, 0xb0223823903c49bb, 0x3cb9b655bb49be4e, 0x452cc9f7a2ec2676, - 0xc2bbf19945ddf5b2, 0xfc9a0dd906318117, 0x88b0a6e992207378, 0xe32a4bf390c02620, - 0xf970efcd5d4282aa, 0x7c4296a98ac6fac2, 0xa10493197dc60cfa, 0x7fbdab67a87de5cc, - 0x5a09559ba29ab548, 0xc42b1d1275f2a578, 0xb094b489c57abd00, 0xd93898d383308317, - 0x1677a8c805d04489, 0x3fec2d7b062aef97, 0xea625c8532c3e30e, 0x4cb34b80c3f1508b, - 0xb2484a9621b505ef, 0x5217c30c9cacd22a, 0xd2090c432c4bec76, 0x2194375edf73ff42, - 0x0fde1c4a2fec68fe, 0x9b3667255fb45553, 0xc1ecb44dfd7406cc, 0xc4215ca1dd232ae6, - 0x810fa41a02df5e0f, 0xf8ae08b7a4ce46de, 0x329ae07b7cc049cc, 0x52e3f1cdb3c8a14f, - 0xfcb71af2d630965c, 0x9f1a6cf17e2cc7b1, 0x64a0a49aeff1eca8, 0x0e67a842d9a1bb05, - 0x2bbda8b58c75abf6, 0x46fd1318eb1bb3b7, 0x2f71ec6a93e792b2, 0x519480bf922f29bb, - 0x7f78634cda266a60, 0x1be77643a069608e, 0x93b6168efc30b8da, 0x12aa01a3c1187bbd, - 0x8b1bc8cd7cf0b3e8, 0x34441eddf6a4bd20, 0x83704c39b1ffc847, 0x7d4f596ac3876fcc, - 0x5b50c7c4a0e33784, 0x360ce82614fc6a57, 0x79bedc5e920fe657, 0x93aac902b115df7f, - 0xb6e134fd159f25ee, 0xcfe73871ef4579e9, 0x0d0253d63b1e3ed3, 0x76e8c0db3137846f, - 0xca8877ebc5cd4fc5, 0x4385c08e93083e12, 0x8d19fd140da61ea8, 0xe14762845751c57c, - 0xfaa166142199cc1b, 0xf443e4e3ea1a6127, 0xb30d9f01899d508d, 0xebeecfb2ce48495d, - 0xc82138006faec8d5, 0xd5eada50f9a8a0e8, 0x1148de082947f234, 0x5c581c85b683a06f, - 0x15db1a44bf48f6b1, 0x3a2111ccd233afbe, 0x224683c0065efa3d, 0x7e3832f96d5d01f6, - 0xfb7980a198a060fe, 0xb44362c8168191ce, 0xad7d6b14a8abd7f3, 0xfe5aff274aa1c67a, - 0xb6e60ce8bcb3c3b4, 0x94869d20af096500, 0xa87b1959f91ae852, 0xb425434f366089bb, - 0x667062aac308c3f9, 0xd54425fa3bd92ab1, 0x9339ed5ba656b7ba, 0xe501a85688d26334, - 0x91702e9536d9b8c7, 0xacf2810aa8a9e8b9, 0x3636e2af905c0ba3, 0xae2e82f774d0a2e4, - 0xe73b71f1e79d4854, 0x56ea2e9ae6ac12b6, 0x65b62bb6978c06a6, 0x6f39a9f8a2ad4807, - 0x429ccbbfb8791543, 0xd7c1e15144f4aae6, 0xb0b026ef827314da, 0x0bb918a52be310cc, - 0x64724f99fc51eb2e, 0xb8b0ae5ce45254f9, 0xe297f13836b0a941, 0xc2ea17281d9b1b70, - 0x2c928f985b258445, 0x1cf17bd61f660b40, 0x17f1fc0598bab389, 0x8bc7c5618ee06d53, - 0x2da1fda8dad078e8, 0x80662f2b688cd223, 0x4aaceef3546369f4, 0x737a994a67564597, - 0x4e9ec68c991b35ee, 0x336cc487ad18a998, 0x0deb76bbffaada08, 0xa52dd72403192c7e, - 0x53cea93187cdf6a1, 0x6831b92e592717c6, 0xfa7b796ddf9cb926, 0x341f33f10e90b77d, - 0xb2841f160c260a38, 0xa93ce7aaa860192b, 0xcfa320105e884ed8, 0xe3c4f17745e83cad, - 0xf4f82acae5dab534, 0xda336119c81cab70, 0x14e50724040ff8b5, 0xf0caba759c8e9611, - 0xa52da2bf0fe2205a, 0xae5a5f4ee55136d3, 0xed79904198719de3, 0x3b89edeef6e685ce, - 0xda21aab61282f771, 0xf9a4774787faf43f, 0xb1f9ca2d842e7775, 0x2ace9b8af82881f6, - 0x97b47161652b55bd, 0xbf182b87504504de, 0x915131962a8788a2, 0x1c00b9f2111991d1, - 0x4f45d80c2b7fed5c, 0x7979b7a78ddf8df9, 0xdce7f74513b23f9d, 0x6f90577c84c68ce7, - 0xc879faf1f48d0171, 0xb5b26ef3ed96fc36, 0x8ce49b100d80305b, 0x69294b47e96f6566, - 0x70dc39b7296166a5, 0x465b49f17b8964b8, 0x20f5293d1ff425f1, 0xa69552930b717ed7, - 0x2a4b4e4d3dcba846, 0x42ed588de4a49189, 0x5c661c7ac8de3d98, 0xcdc15e47f4c25186, - 0x21f742361c02d2d4, 0x564890f6f0a39b30, 0xdbfc98f6291afb07, 0x6b84ef3dc87fcfdf, - 0xbf55f953cccc5e63, 0x333fb470f0b933af, 0x884728b501d1e5fe, 0xf0b3740105c69f58, - 0x54ce6d3301d1d086, 0x881bbedb3ae61f5a, 0x0814aaf6af352dcc, 0x10dcb94852329589, - 0x59b909df614401a7, 0x329d882df0303681, 0xd60207cbd60a21f0, 0x89758e996b566f90, - 0x4ba2aaa99953cdbc, 0x9e80205927952ca8, 0x4f486c900d6ac698, 0x90527c8acf39831f, - 0xb1ae20baacf84c6b, 0xcf915493e8c5767d, 0x11a7fead31d59538, 0x846c005bc70163aa, - 0xdbb8c3288118a8bc, 0x5d6fe2aaf1762aa1, 0xf275925d377b6007, 0xee19ca724e7fe5db, - 0xa7a431ea3405d2dd, 0xfde75946217c945f, 0x11b4ca868588fbb3, 0xcaa1675164bea266, - 0x2c29663aefe147d8, 0x970e7171d65af6e3, 0xf5c457f0fb7dcc07, 0xf9a29dd9c3d2d88b, - 0x317cb1d3cc05ace0, 0x1ef7bbb2aedd42f0, 0x975c5a0bc00da175, 0x1d5f9fd0f4c43081, - 0xb196d8f8b941db0c, 0x4e74422ec71c372d, 0xe22eb62050384597, 0xc8057fb3328becb8, - 0xbe4e0e5a5d2563df, 0x74b87e42dcc105b8, 0x2496d077c94dbccd, 0x9513bf4cf67c5145, - 0x77dff44018c40b36, 0x28997be772a0b7b4, 0xb9db161611c52922, 0x5da72dbc41074141, - 0x07e613c7eda72a89, 0x1fb26d9791443c0d, 0x6fa60b83e685c34a, 0x721b73625a18b6d3, - 0x4fd8fcbe8abfc667, 0xbe71be35b0a81a23, 0x8b4ee76679d3bc1e, 0x3ccfb67406faf026, - 0x10ec89c384505fd4, 0x448fb314305f58e7, 0x3dfed3c02cf17c71, 0x6a0c2fe8c05cf678, - 0x6c9adbc4bc741ff1, 0x9b2484d4f8af2e60, 0x82bf4550a51de79e, 0x792ae8aab2d8aa38, - 0xd28d08be14ef2ee4, 0x25cce0cef9ee514e, 0xc4971db048b24d97, 0x08e2f8e98930624e, - 0x41add91310343f13, 0xa785cb576a4c33ed, 0x0be57b8d46d44abf, 0x1e37106e51eee1cf, - 0xcbadd5cc65f65e25, 0xee6fbb2d0136f8f6, 0x1315474f5bbe9b8b, 0x85af2ca4d469732f, - 0x99b61e48bdfcb4a3, 0x02d5ee1c36791fbb, 0x1569fa216a94bb67, 0xb6cb53e494c7ce5e, - 0x2861cabc4fcbf60b, 0x5b192ce5eca8922f, 0x030f6aad9cadcac2, 0x7d4f0544264d4838, - 0x97a00c9291242167, 0x9e79f06af0192a9a, 0x3cedab6937ae681c, 0x1580423bcad5dd63, - 0x2ef7c7670bf23a0c, 0x9d5bd2f1b6e2dbce, 0xdfdd8667bbde784a, 0x02f79c7e8d9ac441, - 0x23885dc9f26355ed, 0x8a38a3134155d591, 0xc7e34bb26866e7a6, 0xc524fd28602dd5c6, - 0x37437aec3224a102, 0xebf061e3b1f64184, 0x97cb33662662fb3b, 0x64d43974b564ffff, - 0xe71484dca0b94ad3, 0x4f43044d56d0a32b, 0x46eae24732f233f5, 0x75c910369aff78ad, - 0x7da6d10b75443d01, 0xb2d76d35c4033d38, 0x67dc4f04f690885c, 0x131568cf02c74672, - 0x52e8dc89161d9d16, 0x3ae7c7a732e52a57, 0x78e1b9697f4e59f3, 0xe8fa1fb7e0213b9b, - 0x0e5b0049c96dd48e, 0x87956e1d8e2cafb7, 0xfd35f963dc92d855, 0xea5b4a4d12abe178, - 0xcab6dd084a22cec7, 0x7ce15cc5d29a81c8, 0x77e589f161c1db9f, 0x7dbc1a6d4eb6a831, - 0xc4cbdcb9d392dbbb, 0x3dd256e4426a08bb, 0x0e2a1efbfc685122, 0x453e49d291a4e930, - 0x2404994f0995a327, 0x0e88f460095331ba, 0xadee07fdc43a3067, 0x4d877bffd7eb019a, - 0xdfa2cbdbf104464d, 0x8d385d79012114d5, 0x81a48bc2fa460db9, 0xafa87103124a8e29, - 0x244c694ded2aa66c, 0x1f1946781cddfd60, 0xaf1cd25e36bb382c, 0x7233672194a63a5c, - 0xa8362d8a3be913e0, 0x44f5badace4c3088, 0x2d390e9563993ef5, 0x5d2fdde2930d6acb, - 0x9596eb7ab5690462, 0x6cd352a166f8741b, 0xf0c8a52be85dbfe7, 0x41ce696fd14931e8, - 0xde7e88219dc73aa0, 0x3580481fcd5929f8, 0x93ff41374fce0eb6, 0xbd532ab91db0e533, - 0x864d1d294c4068c2, 0x332bebcfb1daf918, 0xbdbeb51fe52c2ac9, 0x7e60276b5e61d8ae, - 0x16c60a00a12bb404, 0x963b9d3ea27e24f0, 0x387030fa925ef5aa, 0x52148d71892d7bb7, - 0x82e47efd6f669f44, 0xdaaaeb054f243b94, 0xb38472e0ddd960c5, 0x7cc73f77ba78657c, - 0x86a3885f7fd3c673, 0x0e087c85d93cee23, 0xa212f0e452d5be40, 0xffa4f4cd91b51a9c, - 0x1f6c9b047e0ab63a, 0x903acce001ec5f1d, 0x42cbce91b1b9f1c3, 0xc6e45d5054463e25, - 0xfee11e956d381cd7, 0xeeb4da5dd5ab2ad6, 0x7a0ac2090f09c2b1, 0xaaee43fd7f6c74d7, - 0xd197a3134e3c0181, 0x18c629e2699f7f77, 0x2273479923b3bb79, 0x3759131c568f1ef8, - 0x240b470664175a3d, 0x2e7b2c7ba7e26c59, 0xc7a2b4e1fd6535d7, 0x5647c73c92e3aa7c, - 0xa1ab07bd1a95e94c, 0xbe89e0aa48de9782, 0x2fca24c421653e47, 0xdefa3b60dea5aae4, - 0x06d4a3e1374bf6cf, 0x60465c2b532b1ccd, 0x46d0f782118e3feb, 0x30b226b6eb8247fa, - 0x8140b8a2823759b5, 0xdad89a33a0d02c2d, 0xff6379c997b33234, 0x9145d90c8830f2c3, - 0xc533ff7c8097e665, 0x37d2e089ff79ad90, 0x8182338df9cc5fce, 0xa9f43cc3aa42aa80, - 0xe61e03931606e3df, 0x8b3ee0a38fcf0e3a, 0x8b5078cbf5559735, 0x6109da7d2f05121d, - 0xc955d5711b923fc8, 0xa4baf0c235419a63, 0x094eee1d4adcac6c, 0x39d22478ec6cb36c, - 0xe34397f46adb768d, 0x0723d49866e10282, 0x510b1c2ce93c3762, 0x68084ad1d611771a, - 0x2166a2cca108c67d, 0x8f46da440b793e67, 0xcfe68e151ff413bf, 0x23a65d54ac8809f8, - 0x87e2ad6a5e9a6f52, 0x7b88e1ec0c75a5eb, 0xfc3081463247998e, 0x9189ce148ca1c8df, - 0x540bf3a145fb795b, 0x43c937c1a90116d0, 0x75a699ae4c51c62c, 0x67b39972366a135d, - 0xe4f2cb4036147732, 0xec852a3a7c4d7c4f, 0x91f382e542ddf515, 0xd92ecc8ecdd0c21e, - 0x8ccf49a93fb6c4e5, 0x39b48cb730f5e26f, 0xfdc93778858e46bb, 0xd8c9bfe9cdfb972d, - 0x314ce22df1a38b30, 0x694c3f4fa03b9543, 0xf37aaec4c12c6be0, 0x13bd201e6de19020, - 0x9685e7ac2c83bbf4, 0x3e44dfa771b2105c, 0x9b794fb251eb7d8f, 0xd3344ff200d8de46, - 0x99c002b7e816074b, 0xa2939cd29060f5d7, 0xbc0fd37d91c28eff, 0xc66da1d671234729, - 0xa5521d4258c8b5a6, 0xe9b203cb4f897063, 0x9aed1ed067b8cfea, 0xe14e53b2019f9d9f, - 0x5ef7af36bebf7bfa, 0xefddb95022ed0f51, 0x2d7e1e7780b725d3, 0x182c9faa056cdbc6, - 0x63c1fc15a7bd2a54, 0x516a95f2ba040192, 0x79108d77ef69130b, 0xc7384a218c3a0dd6, - 0x256396790b34da2a, 0x490959200fa176d9, 0xb69eaffaeb202d9b, 0x7ff84d1cf506383a, - 0x50048a2fd1015e1a, 0x520514d2d61449f8, 0x481bc85f1c695827, 0xd8e7aa31a21acb9f, - 0x901fbb7b2edd3c1d, 0x130da25ab14db223, 0x9912c245889ac947, 0x43c69865875b319c, - 0x59615cb0ec2a60c5, 0x8dbacebfca89a39f, 0x06483d9be414e0e6, 0x2325fe656bb05124, - 0xae4847ef71c44a21, 0xf7f7add047a5c5aa, 0x9fa1ac7dfc60725a, 0xe46faa355308cb56, - 0x9780850af3f57f56, 0xa87c3bfe263f38f5, 0x37cff25eac08ebc7, 0x53f287aff127ccf5, - 0xe5f1e3859183fa76, 0x91b83311f18e7619, 0xd6fcbd30444a4e01, 0x0cdfab2c1baf0d4f, - 0x1e17af95943451b4, 0xb24b06a876daec08, 0x1ca0ca381373143a, 0x38312229094b9587, - 0xe8a5d548ac56d69d, 0x2bb4b98778bdf780, 0xe6f6a2cfebd4cd82, 0xc0b74dd8cb51a469, - 0x813e7401c20b5276, 0xda08ff30b49c7cb4, 0xdd87ba9a5a494ea0, 0x25d8dd5a1ed19d9f, - 0xf786b3d56d76ee8b, 0x6337fe12913b8b49, 0xcc048655757ee4cb, 0x8e804f8afb4775d7, - 0x2d6757369b2be13b, 0xf64204708875e7a3, 0x6a0ac8f1f93fabce, 0xd7c867f405658052, - 0x3f8ba8b988ca8bf6, 0xb3580788119552ac, 0x3847213008fa9c35, 0x558c2e3581c5ca18, - 0x37692098334c777a, 0xf1bf582dfed8fe37, 0x47f42df2adde9772, 0x77b172a47e6d6398, - 0xcf679e65c01d4bfd, 0xf6b6db4897a38592, 0x79b67cf3f8e4a556, 0xaaab94e52ccfd931, - 0xa6566b80a61da572, 0xe797ade632065ac8, 0x04bcc71c99e69400, 0xf13fc369713cc54d, - 0x77bbfd1d5628b1dc, 0x782946c1fdad04be, 0xda92eb4fa9f22ca3, 0x6c4e2601d509f661, - 0xc3762d7eb3d89986, 0x8f50130966a6d53c, 0x13d1827323197333, 0x9598c33944f0a49d, - 0xde761bf5946efed2, 0x3ccdf95cd2c975f4, 0x0258ff5ac03da83f, 0x24e77a62babed354, - 0xff657fbe14224a93, 0x0b37a828b9cef567, 0xfea69b14995efa72, 0x3b800c74d47e38be, - 0x0b3455e2c673caca, 0xb7e634e980d9607d, 0xfd23af57858bd5c4, 0xcc149f3e9db9dab4, - 0x5eb0bc53336307c1, 0xd04a63edd46de8f6, 0x747d65a62939041b, 0xa8abf10a80d964c3, - 0xfa57f51ea6de4a6f, 0x00ba7036ed7dc386, 0x4b571ff391c8eb47, 0xd0c7c9fa3c0bc401, - 0x69c6345c903d9daf, 0xfb9d313329ff0b5b, 0xd2c1bf32732b9420, 0xa98a78e331480ef2, - 0x75be0f695208fd68, 0x317380f8724e7679, 0x5fec9037ac0915f5, 0x512f36dac61d3f71, - 0xc1b526d21f12641f, 0x0d0728016f1d597a, 0xcd8f9a2937b1bce8, 0x835a5f03df3c9089, - 0x19ecb49b43cc71e0, 0xdb73bd1fb71682bb, 0xf9edadb8c35dd680, 0xb898f966f9e0be14, - 0xd11cf62c0b0f9da2, 0xa2d567b46c96e19b, 0x5f1a42c3152b0598, 0x8ce0467bc0033b56, - 0x1f10a87eca72ec57, 0xa720b3e18034c4b4, 0xca3288601bd803e5, 0x280718c1b0eb270d, - 0x45e3b3378320e53c, 0x38eb8ff616dc1844, 0xde777c0917d6e048, 0xaaac9b8dac1fdc0b, - 0x1f054466db74363d, 0x3aa721dd755e0f89, 0x30e1aea711a061b2, 0xdf5bebe55cddbf93, - 0x36216e4430e1b438, 0x52c9f096f0eac8e7, 0xd8a841e76298a9d7, 0xfcdec447c5c419c1, - 0x83b63d70444faffc, 0x4cb47c5e4736bc3f, 0x76832cc3b97b9d24, 0xe067ca46b2ae5e20, - 0x8f60c46b8b8277a5, 0x660bb513581819ec, 0x2ced1740f6e6cd00, 0x8f40063d7d49a605, - 0x772df024ebf00c99, 0x17615b186d086035, 0xd6491375b0cc892b, 0xaa407d8cd5ef6418, - 0x51112b32b13cc2bb, 0x0df85d5f1146b42e, 0xd94faa35f3665250, 0xb47663f6e4e1b79a, - 0x60837eee56f62150, 0xd6bcd6609f280fcd, 0x6da80abf4670377f, 0x97d02dbf9b5988c8, - 0x0e12dcd7b0ae11dc, 0x7606baad49b96b2a, 0x1826be922d480024, 0x09baf41a70440458, - 0x062603cc9fcc2750, 0xbc0f406b1b76b0f2, 0x36ac92b0b5841071, 0x0e4d9da31efb6ea6, - 0xdc38eac21cf59f32, 0xb0de4e96651f7c81, 0xe3c090b2df99769f, 0x26639c4c369c4243, - 0xbe07f6ac649612ac, 0x8d4325000f2cf6b9, 0xa89e32a016cf1889, 0xb289048d36228840, - 0xca508eb623feabd7, 0x88e02ec2516c18d0, 0xc2fc805027291101, 0xaa1e81b3a731a75b, - 0xb4c70a59c69a0785, 0xa63eb20b72573d90, 0x7fbc6149926a48e6, 0x20d5f628c85c4d72, - 0xf3c743f2657929b9, 0xeca9e116a681142f, 0x38369a1159e6d895, 0xb4bf146f16878ba8, - 0xb7b68e4616bdec2d, 0xc84101d01e8096f4, 0x675fbe286128ac25, 0xb9edafd8e7ad4f26, - 0x74a20cc86281bd4a, 0x4a466960c5e804b1, 0x8a149687ad2b7285, 0x64b9a06d8ab0f22b, - 0xbb991bdd6eb514b2, 0x4de3fcffcd428b8f, 0x2464484e3fcf6d11, 0x03958b859f2fc0c5, - 0x89a394c6c16f24d3, 0x5b161805fe71982e, 0xc391c4bb906d2b35, 0xcd7241ceec63bb9c, - 0xf1c314f6ac195b52, 0x8995d87e707fa1b7, 0x54baf1d358fd49fc, 0x3ccebc83ec88d7bb, - 0x0f874698aa98cb61, 0xe43d68c2164b1e07, 0xcfe503d8657632eb, 0x428c670ff6126436, - 0x0b9baf2d7a465132, 0x755737d4ed2d549d, 0xa824be4c7bb705e1, 0x95858a2818f035db, - 0x607fdf6d0971b06a, 0x7bba426ee3884444, 0xd7f10a602b1e59bd, 0xa16f3f1d79281aa9, - 0xb62374200379ed8b, 0x6e5fa3e8dda32580, 0xc6273eedbca700da, 0x805f27f6f93c1270, - 0xf6321aea00309f31, 0xc71dccef85b5298e, 0xe646119436e8c4de, 0x9164c94f72d4d384, - 0x55ac60a41c4a98ae, 0x19dc3da7485b4dca, 0x38c4162c44976caf, 0xee7395d903139138, - 0x29c31eca0be80180, 0xc9dca3e61542822d, 0x5df6c961c92088c8, 0xfa80d04cf20097d1, - 0x99f0a9b237f645f2, 0x36aa4b1bf2372ba8, 0xe9c2736c11193517, 0xcd62644a1f66d4f3, - 0x53340648851b2b8c, 0x854a0fea838faef3, 0x13f3ab9f4c6ec43e, 0xb82d4bcd2941cbbb, - 0x26d085d6f2c0263d, 0x65099e06102cc17e, 0x8c6c6422199400db, 0xc6552aabba8c5acd, - 0x1dca32d98243a4b2, 0xbf6e8dc24ce7daf4, 0x9f791fe7dba0fc1e, 0x8ca23ea314d70f90, - 0x3aefa7b9f4901e59, 0xf69e7b03c46c4a8d, 0xd2306d5987d5a14b, 0xacdc21d16202078d, - 0xdf425d3756fb471f, 0xfe8e7bdbb7009a91, 0x4feea68ed3dbdcdb, 0x9193630a62694bbd, - 0xb0fd4b4c920bd48f, 0x373515f1b4992ff6, 0x71924ade1955f80c, 0xef830e145c0397ed, - 0x35af4208735aea5c, 0x1ca603e1582e95c1, 0xe292fc34368ed5da, 0x96dbc7f046abbfff, - 0x7cca2d61ecaaec9e, 0xcdbab55ac0c9bc9f, 0x95621f83d8755585, 0x7660c6def794c0b2, - 0x99da48951632ebfe, 0x387c2fca00e09e73, 0xba761acca450f79f, 0x4b45f361c444fc88, - 0x38f3b92b86abe63d, 0xe44e18b5d92f112b, 0x9a3782fd954fc7cb, 0xc37efa5b713fafb5, - 0x42eeb1b69c8def09, 0x95cf4b8defba9377, 0x0b8ae887246cf645, 0x3e32667467064a73, - 0x34d819c823afb687, 0x19d0f4c3ef8e299c, 0x5e470535d9523234, 0xbf03e5bde23d85a5, - 0xb98a764d0151e671, 0x97f0f80bd63bc9a3, 0x488d3af3e3bb4fce, 0xc4eda71a90d4dafd, - 0x23908d0c7c6081f8, 0x5583c6cde544764e, 0x9c119dbbf8f7d2d6, 0x1dc9d1fd7a040acd, - 0x7fb889a1688fcd70, 0x396a3b776709a8ff, 0x987a8497c7ab08b2, 0x02d9e487a6e0c28b, - 0x830aa7338c3382bf, 0x7795154efdb9c497, 0x9e344af73ddcb490, 0x061cf4d028416c62, - 0x4e59fb66ade94729, 0x2ce7e175f4eda33c, 0xf3d7481866d57df3, 0xe47b85278045db6c, - 0xbd886f72773d9236, 0x0ba218015ae0c02e, 0x621afa8e1bdc9ea7, 0xf25a08ebd5aaf451, - 0xd19dc33146c657fd, 0xb560953b5604cfd8, 0x8c2146bf46350377, 0x9a7ade60831255d5, - 0x402d1e2a680f5b33, 0x3d96f3388d24d1ff, 0xb4db88530a32abdb, 0x974a344238dd4793, - 0xa36736a076cae2fb, 0x1c98f3004c9c0d64, 0x1ef88f44edc13d42, 0x590c42eae2c64122, - 0x25672075b53dfa43, 0x601f7dee29820ba7, 0x5d6e0cb216306637, 0x5f9bbf3d7b7c3fbe, - 0x6795a53b4f3e1737, 0x20002e8aa88cb6b1, 0x1cfb6a48ca9dc709, 0xe24f387a2c6c2a1b, - 0xf7af223d9c79a57c, 0x50612854876564a6, 0xd51c43874406c5e3, 0x77b87dedc58b046f, - 0x6bdbd07949a0e99a, 0x20270594ac02694d, 0xaa1446dd1dca6ee9, 0x0d8fe41a9fbf3c8c, - 0x99be8b1545396d5b, 0x4f39635dd99652b7, 0x43bbfb2bec8bc187, 0xce4789c1e735be90, - 0x36981fa7233aa06c, 0xf5d1e8bfd865213b, 0x057a529eb951bdb6, 0x106fcebcb38d2109, - 0x61d131f362084247, 0xd236b1376dcd11df, 0xf1cca70aad94154e, 0x85d3d9bc0b853958, - 0xc84a3066624cb409, 0xdca5e985a70f9c87, 0x789160ae49f8b5e0, 0x2e4c503cd55eca5e, - 0xf07f6b15f4f89cb4, 0xee03452169ee2f68, 0x7340761cedbaeed6, 0x329306166ad8e41a, - 0x32d4eaecb8b4fad5, 0x0e46204839ec8b8d, 0x67b7ff2bff382159, 0x1407e69b569f826e, - 0x4fe4ac790df30a9f, 0x701117666ce619b3, 0x2a00503d24729ba8, 0x2ffbde4adc505ea1, - 0x0e8e75538d746cf0, 0x100a33595f1ac082, 0x537ebe7016b7e1ba, 0x99949be7136ac56c, - 0x8167bce74c6621a9, 0x2752080060f9ab76, 0x3096a9f374f385ac, 0xd0674caf5a6f1cbb, - 0x27002fed7eaea9b0, 0x1679b5eee6b50b25, 0xedc20d513c2f32ef, 0xf2ba508181c85bdc, - 0x6efb4655eb1043c1, 0xf40aa347947531e8, 0x4dde175551aca7f7, 0x61df266112ca688a, - 0x59b0c837ffb81c6d, 0x16d05f0e45822a65, 0x6daaef7bc9350c4a, 0xaf7a6c87f6393c49, - 0xc514e433645537e7, 0xc3f12aee7e01d0d3, 0xcfdf9722d0028722, 0xd67b6741b2f98b20, - 0x5ca31126d24beb6d, 0xd76635268fa0198b, 0x1595c39a2a50650a, 0x5354c6fdabca5651, - 0x25f36840eb3eccee, 0x7250161b5133ee71, 0x36dcb5f0eff058f6, 0xb5264dfa74b1b0c1, - 0x5e0a94b8e3882037, 0xaccf1a937f141aa1, 0x82dab4e4abb51083, 0x375ab20edfda88bd, - 0xeec79d35ddd69de5, 0x9052f7cd99b2c06d, 0x0ad859c9fc8a9c8e, 0xb6b3d3b20702e798, - 0x135f026dd2fbe058, 0x3eefe45553dd0085, 0xf732de799b8c8c4c, 0xbd4911769e4eebc4, - 0x3b18ab8b4c671c4d, 0x47e1e3589d47dad4, 0xc2a3e3277c608506, 0x56dbdc58c8d56898, - 0xa8357d850096288d, 0xcdba049ef091ad4d, 0x1bc62190aaa1e0b9, 0xe5097f207a3e1ad5, - 0x089ce2191c02708e, 0x6474d72b03828acd, 0x4aa39f11005a1031, 0x05c067e5a388e345, - 0xe9ef6c784ef2267e, 0x751941aba1778cbc, 0x54d94a09de663b4b, 0x2cf11762b4b5a7a9, - 0x49c82dab8abfe606, 0x4ced20152b67af61, 0x3bab3edd19059a1a, 0x2c0076a10bbc2833, - 0x089a9ca50c5e14a0, 0xac6fcd5f9df3311d, 0x567eda14128e22b4, 0xd0226172579a152f, - 0x142a67450a1c4545, 0x681f8901ad715533, 0x0d06f6f9da1ea9d7, 0x661e31b7a718583a, - 0x271b2ebf035077bb, 0x651e09e8240fb017, 0x91f95eb3a1461485, 0x00b44c6d06d9a2a5, - 0x4f19e198b6901ba5, 0x86d580c9ce49ced7, 0x664ddc7e92ac5622, 0x40661be1707fc4ed, - 0x6cf7237eebc4886f, 0xd6f8133e42703588, 0x9e7d06d19972a6f1, 0x09c05542bde0da74, - 0x0fe5a1b6b82799e7, 0xd35efc393ae0f23e, 0x8bc4251286cc1974, 0x297500ea151e4d6e, - 0x3f32a5df428cd8e8, 0xf8bf933bee1b2201, 0x16597f18f267d6ed, 0xe710d43c90673b39, - 0x25f02d8e20850744, 0x2957f8a732b10d76, 0x976af6e29dbde167, 0xb7ce41c9d36d12ac, - 0x20d1505761dc62bd, 0x0dc3ab4900cc5b19, 0xa589e0bcaa247075, 0x531cbd93b7a7c44f, - 0xd50c8e5e438bb1a0, 0x1fd673b978cfb9ac, 0xe80ae2d54787bcdd, 0xb56e14dac275d28b, - 0xe4fbddd66fd370cd, 0x0135b63537932984, 0xee872df90ffec0a0, 0x0859ba0cb6430855, - 0xc5e177664b02af91, 0x5bb286fbe4699249, 0x815884bd47555df9, 0xe7d70ccd2fcd7628, - 0x2a8bdca4ada2a67c, 0x19490274aa037759, 0xaac2c469d76c0089, 0x2d1d5ec1ad0b1373, - 0x0cb208a497828dbe, 0xd24935af0cded5bb, 0xd0dd4c04dde0e35a, 0xd2733ab7a3acf6a9, - 0x0810fb6ec6828899, 0xe978d4cfb5b9a19e, 0xb5700d40a394ef08, 0x3778e05853b99e8d, - 0xb7efec0672a23a8c, 0x21b07b71cbc78e64, 0x9c6fa0dd0b685cae, 0xf8a02f1838c60b49, - 0xef4e0b62f291bc4c, 0x523347cee91766e4, 0x9b8a4c03c8153b2c, 0xdbfa8a07251b1ace, - 0xa561b66923590668, 0x16950e8116f9b7f8, 0xf00c290190d3ff7b, 0xcdbdf6544c4c5ed3, - 0x24453422dc520ee5, 0x4e71767b2be83239, 0x7f46f609661f759f, 0x7574f2bdde733313, - 0x6b7f973a8dba93c0, 0x700e182a24016599, 0xdea646d384cae0d9, 0x9a9357b8a9f54e04, - 0xfe8e415c52016a26, 0x956d0fe2a10d88d2, 0xedf6b31e2e72da87, 0x5d52ebbe1c9b1ba2, - 0xcd871a427910627f, 0xd78db306283b45b7, 0xc1abeb3ae57fb02d, 0x575005bdf0c0f425, - 0x0b80f1f9a0d59a1a, 0x700b4038b771a281, 0x59784779cb70c918, 0xcfe52f015cdf968a, - 0x6e0e2e19724cc02d, 0x7778d905899e2226, 0x5f8fb412a6cb0bc0, 0x931d2d762aefd6a0, - 0x9fb0d1a3ea23a91f, 0xc03890ab5cef7d6e, 0xc119ebdd4948d76b, 0x42a92c0a16a399f7, - 0x79617bf36a202312, 0x0ca2499ed9919300, 0xb360d811a987bccf, 0xeeaac9eccd5507df, - 0xb172bf14cc15504c, 0x440ee0542baf8b80, 0xc3c6e37104a19598, 0xd4e7774647dc769e, - 0xae71865543746ca2, 0x58f89d86cbcb22d1, 0xab811468ccb0e740, 0xff9fcf3e368bca81, - 0x54c4a4393dea48d8, 0xf02ac04aba9ade99, 0x905d161c2188bb90, 0xeaf1a7bebe6f03b5, - 0xf38bca616d7f9bfe, 0x8525c10aa5f50eea, 0x292c54d59f0d9bf1, 0x78f0b1b1f371ef23, - 0x91fa38ecc9dd5ea5, 0x72a6fd65f132602f, 0x8e7483cb1639b1db, 0xf47e80cc11dee983, - 0xd51b087fdbc0610b, 0x22a3eb273c3baa35, 0x2920b1dc0d718938, 0xece5cf9caf6699f0, - 0x40206488fba573b5, 0xeb2cc57dd37bac0d, 0x26eb9b8dda7cdd66, 0xed5cd452e785584a, - 0xcffedc5cd3565527, 0xb79e3deb7a340c8a, 0xe01f37c73729094d, 0x3e5e17ff62f06df4, - 0x5500ed0148226012, 0xc754484b4393c7bf, 0xb7d032f0805520ca, 0xe386b8d51e460132, - 0xdd5e3d0b1f520150, 0xe9296af3a652983b, 0xbe721beea5067b12, 0x5674fb4d39f43784, - 0x99686b64994cd3a7, 0xaf80e8de9b71cc99, 0x0855c005a1d1e765, 0x6f12a6f0c2ed124f, - 0x1deab1710f4dd43c, 0xc7a1695f21a6f5d6, 0xd8ba05ded95d00e5, 0x94fc77f5842f5cff, - 0x7a01dbe295fe2278, 0x3861c5d888a1786e, 0x78136a63e2507e7a, 0x44e19ea5d0be81e3, - 0xcf3638c994ac39cc, 0xca7d29075ed949b8, 0xfffd73ab10719179, 0x0c0786c2004ff8e7, - 0x8e550eae7ea4d2f9, 0x369bbe11b7793d8d, 0xadb9dcf0ded8d287, 0xf6564061966d8df0, - 0x40007e2f668869f4, 0x89ecc996bbd1fa1f, 0x17243ed53072f0aa, 0xde05d87789ecfd1c, - 0xdf3a02709025ebfa, 0xe9af6a5ab217f05c, 0xc9e2e2897c7d4155, 0x2e6a0ed7b78e6949, - 0xf2fd442173d7e5ee, 0x95e551cb26d4818b, 0x78181b7f88fb21ec, 0x8a2306fde58644d4, - 0xc4169ba7990149b1, 0x50b2403cc17f99db, 0x780bd6573b61582a, 0x8916fc82bd75715f, - 0x79f0c0167b1ebf83, 0x5e6c0c6f40587f5b, 0xd617b957d687fc5c, 0xf3b8fd33f6367591, - 0x5aa436da519426ca, 0x56ac7eab2c2564b5, 0x3d07941ef30605e1, 0x9bfff7347fcb4284, - 0x82ced7ce89d608b3, 0xcd11ac44be2a9e55, 0x648ec33bf5d0c40f, 0x6668a6846322d18e, - 0xb4bbd5d87bfb9aa9, 0x9ccf30f91f654829, 0x2e78b20621b64b33, 0x66110106bc7aac37, - 0xd874f7285f04a246, 0x7b3fd95ec30db1b4, 0xc850ef605c8426f8, 0x7815adf69e763e99, - 0x2c58ab20d5ba27cf, 0x7c2fa05f78893615, 0xb0285f230f9b90d2, 0xf4abf33b340da595, - 0x586b16a5c9d7c76a, 0x7675388529313c5d, 0x6adaff4b1bc68c0a, 0xf09fcc58437717e4, - 0xfd9821d19d28a6ba, 0xb3b1202bf4d4d793, 0x92b89e9863cee0bf, 0x1ffce81f602a13b3, - 0x840f25dcae9d589b, 0x8fa8bb0f876110ad, 0xa9b0c1bd8a5e8a2e, 0xd66835574190fcd3, - 0x3cb7cf9fa0a0f07a, 0x186f9eeb4e13e68d, 0xfe803fbf6dfc17be, 0x38aa2e7ca2465215, - 0x30f2cc949c4104c2, 0x8d200731c746319b, 0xb7d7003ae11eda3c, 0x38506c984d3a0309, - 0xe1c6491824c66794, 0xc73e54f2c9c14767, 0x8a7e2ca9d5481151, 0x893fc892a8079667, - 0x433828fdee415bbc, 0x8fa1b4b2d8f7c848, 0x0c5a1011a064f996, 0xca5421353702182e, - 0x43841debb3a70963, 0x19f640bb488a312c, 0xeb69965a01560873, 0x460d3cab23466c13, - 0xb23554017c3d6b78, 0x39602cb82cb04c7e, 0xa3c5272478279b4f, 0x58bdbf706a74ade5, - 0x85ec70da3633d1d9, 0x117dd59b79626827, 0xbe67ede736a0b3d7, 0x9cf5b071c402c1a8, - 0x6a1e84c7bba1ff3f, 0xadaf457198e6753d, 0xe10205f6e8c05cc1, 0xc1c9ac4e3c5d729e, - 0x92582d8f2db2342c, 0x3fb3cd2f8c8b55c3, 0xe7cce95f69d2a33c, 0x3089390ea5879321, - 0x1eb6a8936400eedc, 0x2890ce537fbdbbb1, 0x2cca7dea90ca2584, 0xcbe564c4ff2d3dd8, - 0x027b16242817fbf1, 0x5d7becb82bc3619d, 0xa183e4dce6c2a1e0, 0xa4301207d052a562, - 0xcbbaa5eae852d28e, 0x9539e4fb1daf9525, 0x4d4598d994fb734e, 0xe1d0cad3d78a17b3, - 0x55bfdcdfb65a8aa8, 0xc82555359fdcb348, 0x9135a05a24efb347, 0x70ce9358da719063, - 0xc651f0f2cfc97d43, 0x2a9bd238779e179c, 0xae2fa722861f52ea, 0xb66505d290d90eea, - 0xb0d1a90f4008b121, 0x7cac97ee252cdc27, 0x1d94839f58bcbb4e, 0x11e2527ca193a533, - 0x97907d982dcb5b0c, 0xe2d142bfded54991, 0x77162a02db980573, 0xd90c6642b503159c, - 0x1885adbcf1515aa4, 0xa941777095b695d5, 0x4d698071a17b153b, 0xe11cccee0f7542aa, - 0x22538898df0fb857, 0x65e233a375e76a1a, 0xa079e239db1478b4, 0x7b33d7f0c9c6eb1b, - 0x726f2d8fbf45fff0, 0xfecd76d173ba35fe, 0xb3a8967ef36cfb49, 0xc5d9f703f7d509df, - 0xb3ab36c8eef477fe, 0xe84c68dbb957597d, 0xa7e76322d8b35fa6, 0x88c80bec280e3fe1, - 0x07f3e5108e2d8da8, 0xde9cb039b3762596, 0x1d1acbe0dd3c9748, 0xac6ca3cc453fa9af, - 0x057ac800f2ad54cc, 0x57006358171b2096, 0xe76a98a2e6ea2966, 0x0ed88ac00b15adc8, - 0xd767f633d3649bad, 0xf86f57ff08c0079c, 0xed6a17751802166e, 0x09f5f561ac302ff8, - 0x33dc072b372f64a8, 0xaead1a02891bfd56, 0xce53b1a4834b2253, 0x82fbd8982c006436, - 0xe0fa2e82f60426e4, 0xcdc732dac6e88e41, 0x99bb6a8966973b97, 0xa4bd5446aa480d1b, - 0x500ca9cb7e493ea5, 0xc12d766e94e16b32, 0x0b9ab609d2e2361b, 0xf5e2581360de01e5, - 0xd858383309c178d0, 0x24625a428927e6e9, 0x8afe88fd23e26c5a, 0xaf3e936ff1b0fe3d, - 0x29aea6dc67ddccfd, 0x0cd1d0591ffdc3ec, 0xb7c2e4f87795691d, 0x1db6a33ffbdebf08, - 0xd41c06d2ff34ca2e, 0x24f99d2668765759, 0x3ff4b7ac7a822aea, 0x58eeba464d861fdf, - 0x5eb7381f00826fe4, 0xf9232ef5e3126145, 0x0f1885f10b4a1089, 0xe0ec889277a9d93a, - 0xa524af930dc87100, 0xa9beae59a0cd0399, 0x18367d49595e780c, 0xdc2fe288d3fce039, - 0x1a468c82bfe3c965, 0xbafa8efdea2942d0, 0x089c7e1d53f9b84e, 0x6241ca20b201f56c, - 0x2390d9e811475c76, 0x3512aa181017180b, 0x433437881b926f2b, 0x79d3de4069f86308, - 0xcdf2efc517b28c3a, 0xec1cc28523ef0b46, 0xef94066fded0c024, 0x98650d2aab484a04, - 0xd246289256787cdb, 0xeb76d5341d6a7ff2, 0x1760287a41813922, 0x1dbca9910706b6b7, - 0x7c60d555360aaac3, 0x132c3b511a430052, 0xd8e58af0980516ad, 0xcdbe8409d927895f, - 0xd17618bdd22969e3, 0xd73f3d6b7509db9a, 0xcf2d0613cb90308f, 0x551ed304bbe87e24, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x57012b7623513003, 0xb9d53d3a262981fb, 0xe59152cde65c47e0, 0x9e923f46001294a2, - 0x5c7e821de003ecb4, 0x7a0136fdb5c08b9a, 0x1f739b028ab288d8, 0xba787c839e8e6241, - 0x2ea81495dc7e14ba, 0x4e693ece3030a628, 0xf0dd79876eb1b895, 0x83758ece660b60f0, - 0xec1d55458d59179e, 0x59365825e63ff5be, 0xca9a4796f743eb07, 0xe4abe3091e998f38, - 0x6f16fa584f945f41, 0xb85f2f62f74e53d2, 0x851ba3f9cb61f7dc, 0x44bb2aab08e53136, - 0xe208b02e8d0653e1, 0x870ab2907cf00697, 0x74540958dbd4d8fc, 0xaf3b792717068107, - 0x667f86d56f5bb7ce, 0xbf5e26cd162c0be8, 0x2d6cd78ed4c823ad, 0xdb20a247c92ff2bb, - 0x7daabc09fc5e7192, 0x0919e5db33883db4, 0x2db0f91ddd0c26c7, 0xfbf0b05a0fc80c35, - 0xe93cc23f3a99e52b, 0x66e144d33f88e47a, 0x2a9eb42d1d9d07de, 0xc7731d33dc47a241, - 0xdb87ec6e7a0ad0fc, 0x244e79f4edc06e2f, 0x8501a00fcf08f60d, 0x3dc3f696f724d3eb, - 0x8f30846e89f43c42, 0x7202a854ddd95d05, 0x0c18a89f5f5868a8, 0xff42e0142a9baf2f, - 0xa7e29dc3f192e73b, 0x8d72f26266a19b04, 0x72e5aa54b7f6d00f, 0xe64b10de6cacb419, - 0xa5baf0dc702bd928, 0xe1a0275bec9b219b, 0x166a720a7d501d76, 0x0301e1c365cbaecd, - 0xed4dfc985238d838, 0xa03abd75da120d3d, 0x5cf5b5d32f4958fc, 0x4960fd5fcd324eef, - 0xd7e7a9e831bf22f6, 0xf7f930f56871ba4f, 0x9ff006f604086bad, 0x1b6852a09a32a20b, - 0x855b29d27ecf9a89, 0xd930528872825802, 0xa442f602a66488a2, 0x66573df39f8b7703, - 0x1fa3ddcf2e78ae49, 0x530845061f003d25, 0xff61c31fd81b4f6f, 0xcf22b01a93b89545, - 0x3c0fd53621068e5f, 0xc8cd6bd1d4905028, 0x659dfe6c250302fb, 0x30fb94af9b51de71, - 0xeedbeb7389f46cf3, 0xd0e669dc2bf62310, 0x805b4d45558478d8, 0x786e3686f6554732, - 0xff7ed48b460b496b, 0xd45f4d88f4483bf6, 0xe1e5b82dd4670fc8, 0x48cae852971bcad8, - 0x018db25069d90454, 0xf6437e41c011ba54, 0xdd945f7a880dfa69, 0xaa548af614f01827, - 0x16ca278d860d0793, 0xc239b3de9e6f65d8, 0x59a203aa1d86421c, 0xa6d854080824da11, - 0xae5fb74b636d4112, 0x8e6b01e2a62b751d, 0xa4c73b21e331a967, 0x46cad2db229d92e4, - 0x0463fdc0931c33a8, 0x184cbb249590e13e, 0xa572aab17b604a44, 0xd57e4c28794fc965, - 0x73f148ae7b1b9541, 0xa5f7a4542e0a25ff, 0x848600a5f91ef667, 0x895e7ce4e5a992e0, - 0x7e5cea23f47e9afe, 0xf81d58699d6ab1d2, 0x1d5c044a470f822d, 0xab63aee94922379f, - 0x532b2a346adadc83, 0x00a9045e4aa69554, 0x4735b6bf8b2c2335, 0xdfd4f47cd4dfe6c8, - 0xb168f826d6d9c090, 0xad1e41ae7179a8a1, 0x65899abf5063e125, 0x5a3d2700e66471b9, - 0x393ac11d52e483a4, 0x180a7e1b691e8c35, 0xd650e93ad125ad2b, 0xd96e18039b0ba5af, - 0xe0bb9bb0baa93b6e, 0x239f8b304ea320a3, 0x23f56cf7917489ad, 0x9bfc411d7538e8b9, - 0x093a02e51e95d12f, 0xc7a3e63cd19b8134, 0xb16c5d5bf0b7b2bc, 0xa9a11e907ad6c4c7, - 0xd0b5e64080fbf169, 0x9ca4a0ced302dc75, 0x3ce9fc15eaa5e606, 0xf5bf367dc2db11ce, - 0x8c33b0d4416f2502, 0x7bee44a874bb45a5, 0x9f6280b43737a5ad, 0x5ff81c6bee599166, - 0x5a43ff22d23268f9, 0xa86f555f6e403e43, 0x4d6088ebe31a0f2a, 0xce14669459d57320, - 0x0953d39a06833cc1, 0xb65c0691432845b7, 0xfaae751a908acfa1, 0xe68c92ab15e2f91b, - 0xa324295921f5aacf, 0x69884066a075642f, 0x04efbecc03725374, 0x6f8fd8ddb6347149, - 0x91066ff90e7f6a49, 0xbdd7211ecfd3aa61, 0x7b7492eb9b67e45a, 0xeea0825bb26a3770, - 0x325d4401091b77a6, 0xfdda0a36fe9bcb86, 0xf7b69a0a651ab8d5, 0x46bd511534a5e7c0, - 0xc6d2f8ea03c4b209, 0x95c2e3d055fad576, 0x75b95e2ea2ac61a6, 0xa75f2582ef061a7a, - 0x4fd9d93b8e400bc7, 0x007ada1bde8c15ca, 0x98546ed6b6446554, 0xb913cf73516c4779, - 0x3ce35d2acb0786fb, 0x74eb9fcb49bf4633, 0x79064bdd3c5fad14, 0xc8634902a8f844b7, - 0xbe9e04c7ea046e83, 0xee308bb287e34ced, 0x043c226982253c33, 0xe80d36bb6a7d39d8, - 0x39c205209f258df6, 0xed0e1bcf38c52a46, 0x7567f3b074f33cf4, 0x938dbf723de1eb16, - 0xe430e24bd8fbdf70, 0x47d7065294bf6467, 0xd792f73ca4868280, 0x3decfa6f831010b6, - 0xcec684dbd539bee7, 0xbb4f8817172e7665, 0x55ed72c64a07c6d8, 0x81fa2669c3f132ca, - 0x1162de6fbe8e16ec, 0x66dd10122d3f979c, 0xfe2a7c187aa41419, 0xaec8b041452da475, - 0x8b6d482a30cf8472, 0x3f3f2dc7e2e6c4aa, 0x210b2710945f2807, 0x78ed418c296e9939, - 0x826cb272f991129b, 0xdadc82a245dd3444, 0xf4fdd75856554a84, 0xf930111f1f32b5af, - 0xe075922592834b72, 0x14004ae0e90e7496, 0x624bbd0a6b5721ef, 0x60396b37e330a4dd, - 0x5aeeb7d5e470a347, 0xdf0c0ad51019986e, 0xdf9c50cf239bab57, 0xc7a63486c957cca8, - 0xf8bdd98cee648c3c, 0x281d8364bc6467f1, 0xa8ea359ce68d0c86, 0xbf2bfa9ef8f7228e, - 0xac73b55f0a88ba57, 0x711adbf806f2a375, 0x65ebede00ec0a445, 0x4445d19b1f3e8a49, - 0x2a1ca9421cc192cc, 0x602c0dd9bfef1b50, 0xba06031487f21850, 0xf421fbf58bcc43af, - 0x44c57d9ad72b496c, 0xb95de6f949d2ed95, 0xe8cf11d005d847d3, 0x2c15d4d4c71dc750, - 0xaf91c7e734ce897e, 0xd5b1554ecab1ca4c, 0xe55535ffd08cbfcb, 0x845fe8e460d90b9b, - 0xdb6f0ea18982a569, 0x5338101edfb7566f, 0xdbc4fdd539bf3919, 0xf4f546513f6059f9, - 0xf3ddc95456f90417, 0x9c2652de428c6d7f, 0xb892a77b5ffb40a8, 0x572af6c44579f34b, - 0x6dc40bc71ea154bf, 0xa8790566aba46634, 0x652dfc3f8381a58e, 0xa7509d7b7ca90beb, - 0x14f057af088b5b97, 0x831799f09602183d, 0x2036d3d151165db8, 0x576fe9855865b246, - 0x01e84d4822ea9d75, 0x2fbdaa6beef003d3, 0xb24bbad76466139a, 0x872475faf54d1f11, - 0x4c43012676e12fc8, 0xf531075961f2483d, 0x86433eae7223a4cb, 0x483c40aa18f38a9f, - 0x5abfef21901d5b2d, 0x5d960db5de10e565, 0xafde6275fee61aa5, 0x62533396b8aee586, - 0xa906b40eca1fde62, 0x9a58a16a7564a5d1, 0x47fa05ebf1cad9bc, 0x3b280d37416a5d57, - 0xfa07785c838b20f1, 0xee4312f78e4294d0, 0x8d60a0ff056f34f1, 0x2fbd5e42957e7d87, - 0x1814ebfece7a41d5, 0x42fbb057798b0a7c, 0x353fa598196897b2, 0xfc09d081a6c72e5a, - 0xc8b88c9f7445068d, 0xd659e4cfe6884c7c, 0x57059f24c6c6b875, 0x269fce9992f4b8c3, - 0xc0b817d15d7022fe, 0x1c76c0a649677bb7, 0xa54664d824b1f876, 0xab005c8c1a48250e, - 0xf3778760c5d91193, 0xe3e6a6b38465fbf2, 0xc12dfe9d31cea2b6, 0x90312ec1063fa356, - 0x4ff4389318dc1616, 0xe80680b5c20c6389, 0x9df376b58f45e6df, 0x28162e74a8ac60bb, - 0xc66e10dbfd7f7fc4, 0xbdfb9addd1547e2d, 0x6327fa331e5251cc, 0x2cb1113e753c6fe5, - 0xe9e8c71eee4202db, 0x8d32e09d3d0abe7f, 0xb87281bbec6a056a, 0x8f997578749f1005, - 0x22c766242cb50b0a, 0x75493ca1565cd19f, 0xe679f9c97f214d4b, 0x0849148d17c75c6d, - 0xb0e519d98f2ea7d9, 0x565d9f53516c1da2, 0x204f0bfff68b5310, 0x6e04ed1c0c155c00, - 0xb2391a88969b7a8b, 0x3e31fe5be6d28356, 0xb7e83ca196e9f3db, 0xd97f4ebcd551b175, - 0x71112d36f72a5952, 0x9b650daea6289d5a, 0xfa8740aff1de9206, 0x2adf02d058ad9200, - 0x71f27d2ef6788d25, 0x35d6a4e2b2ab4247, 0x182dd5ec8c753449, 0x80b09c4df1c9c4f1, - 0x8feaf81c2b64039d, 0xb20d97158ff59250, 0x40309695c6be6c40, 0x465de5e5ed205428, - 0xc650abcc2545e4ab, 0x0366c6a58be9245f, 0x63c1d1a29267d6c7, 0x91af1b9819c1198b, - 0x3966eb999eeaeb3b, 0x47bfa36d98b976b6, 0xbfd55fdf23db7115, 0x694fdf60ad4f242c, - 0x4d674c438eb6a791, 0x1a93c54b6206a228, 0x472b44a8758569c7, 0xe591ef7d86b9519b, - 0x073b2f2886af1c12, 0x7444a5e8bad7a5a8, 0xf4dc4018a017811c, 0x5c10af685575195e, - 0xbf7e0d59181d63b0, 0xda84ffb372e72d73, 0x69a8f8e72137f141, 0x8928c7dd580afc5e, - 0x3ea5ac0f49563bd7, 0x9fef5bd4f7091f62, 0xdc5befe242fdab17, 0x7ea1543168470de6, - 0x4d42af84695e301d, 0x3c4a3323e19e5740, 0x3e30e46cdcc25347, 0x3c1b1218d5cfc084, - 0xb98d23c733ab894d, 0x4cd31a7aab1b2dab, 0x4484759d45c90163, 0x4e07297bdbf220e8, - 0x8980b99791d042b2, 0x6ccfb3c4fc39b902, 0xebaf4f10c12d8876, 0x7a502e7870dfc7e5, - 0x673ac34b8fb06e49, 0xbedc9b5a356e3835, 0xf169c3c3655b2843, 0x46441bb30a6d9169, - 0x0cc15712cc2dcc44, 0x7ba5bfa2ce45d3f1, 0xa2b372fa5a6f1665, 0x39de81417e43d829, - 0x2dfcf473778e2535, 0x8d94f3e693e23849, 0xa798beaa31e2f0c8, 0x3b3caa0c395c1fcd, - 0x31483108452ae5df, 0xbb6612034fa9dd21, 0x2fea393a695884a3, 0xae05e43403e3f92f, - 0x8de94abc71a28db1, 0x824ba00e8b16d119, 0xef003e1d844d6927, 0x55530bee2ed65047, - 0x8fa949d85224d14f, 0x0c82cfaae962185d, 0xb9121a587cbd8968, 0x7d6900f64fc472d4, - 0x7d0b4cae00e07e8e, 0xd83d89f6d0d38432, 0xd6eb3d05142d3a40, 0xb16ec50dbaa80824, - 0xd83a4d2a878d4a00, 0x9625a5dce49b6237, 0xbb486ef1c4af5865, 0x3077135a7b4dc4c4, - 0x0cfe3477d05b5b7f, 0xeb9e1cd0d4fffe40, 0x8d5b751fed9c83a0, 0x51855d2023390ba4, - 0xa9d95e25d5df5ec7, 0x10bea39e74a53536, 0x302b6b39ff4fd4d2, 0x62805a9536abb932, - 0xbb4acbc8694c21d2, 0x3e46f37a27eacf4b, 0x9c2279b32886148e, 0xe785a392a7e1abb1, - 0xba1f353c056fa979, 0xed85965049dea9f9, 0x8959c1bbf58ce555, 0xfbfca8b8b78ad553, - 0x25d224d534bcd2e7, 0xdf350385039bb872, 0x1e5330c5c9e6356e, 0x3baaab45a97dac57, - 0x73acc59ad6b4a039, 0x81b72873922f1ec0, 0x86307e881f5dc965, 0x50ab9d93ce618fa7, - 0xb56a82c31236399a, 0x925322cbfb61f582, 0x1c3badbb7d63c638, 0x2c3415e986ede3fc, - 0xb319b9d5ed880ea5, 0x9d6a00efeb105386, 0xad8bf4bf06e083af, 0xa58e07c3bed1a2c4, - 0x517c0632c5a7c6c2, 0x9d89aa8f0af2f949, 0xb1048c53c8c588d4, 0x4098c3833eade047, - 0x10a2a68e10248827, 0x8cbb8b90121a6aa6, 0xf4e667a8bed5e67e, 0xbbba897d8b561fc8, - 0xc0643c049879fb18, 0x345a6854efe2cf1b, 0x3509674116a484e8, 0x2b7b5ead735e2c2e, - 0xc8c0d3246949b122, 0x729a881301fa12b0, 0x5692d6303f1b7053, 0x88b17b15bf5fa163, - 0x0e6ac97a56c78042, 0x87fe29c9566b55e7, 0xc104f8981d0608b5, 0x103f626cfd68ac9f, - 0x77e4fa487b6eb752, 0x5dbbf7adf144e13b, 0xb644e466f7683b5a, 0x6343d5c78025b914, - 0x43e02a79b7372f23, 0x65d044df048c9bff, 0x7816e630e023602c, 0x494d032b44d47bd8, - 0x0909f24cdbd64ac6, 0x9fa1a36938d625f6, 0x4c57370f1dd70004, 0xbc04bb5b6592941d, - 0xdbbacfd279a485f1, 0x23dfef6c200c41e9, 0xcd9df86865565978, 0xfc9d88fa098302fe, - 0x398382a387b63739, 0x9570a9561ab8c57d, 0x6b91e65cd9bae003, 0x3c8eb36ea5968624, - 0xcd48d4b4edcba055, 0x5d3c2f729a4c2f64, 0xae2c851f3ed26fb9, 0x112842dadf8554fd, - 0x6d006bf3fb875a9b, 0x608d849912581cd6, 0xa6c171d124943ccb, 0x759b8a1fb0d19df7, - 0xf0ebd6a1d0e774de, 0x9a5811323349fbc5, 0xa775ad0c30374b51, 0x1be49d5dad4a121d, - 0x3a054173fa304b44, 0x89924789e96b5481, 0x1799b2939875fcfb, 0xffc3444c7d784008, - 0xcdc9d1a42ac08561, 0xa964047feca73809, 0x4b49db547ecfd2cb, 0x67ece48188e13e1f, - 0x3bb94e12d6ae6d60, 0x95e287ad4eb0a2a1, 0xfb6fa2404b6145b0, 0xf9d681ed27914a66, - 0x1d90166d00acfb26, 0x1b50e0977a09573b, 0xf1dacf74cc99285e, 0x0d02482a190e2be9, - 0xa39611e04103197c, 0x9c77efcf9bdede12, 0x7fad23f4a8465660, 0x493d32d9c18d907e, - 0xbf4e30b5998e6421, 0x424bd628fda6fae1, 0x64ac51bcc4dec712, 0x38eb6b9a8e468016, - 0x107ae6a6cba998a7, 0x2aab3be94a810a36, 0xa54bc2682e20252e, 0xec7fbf3060ecfcf5, - 0x10a701da722b558f, 0x3eea5c3074d89347, 0x2cc28b3ab611e713, 0x7b6dc4e5fb8808aa, - 0xe7ceb58d34979139, 0x478b5986f8f1e520, 0xf5af9c876728655b, 0xe3ca60ab855b905f, - 0x1478586a61e31370, 0x5d8156e42757d983, 0xaf3044846eb06235, 0xa01178ba4df87c32, - 0xb216f58359f77c02, 0xbbc82f0cf2721ac8, 0x41199a3f8b8a6e54, 0x08c17d9003a85540, - 0x325986c29a03b666, 0x54706aceeae151eb, 0x6a6e73be7613f0de, 0xbaa075cd5fc17c47, - 0x6fffcbc8add29bf2, 0xfab07d54375d8c1a, 0x20d95b9ac58af7b7, 0x6519f8201f4c7f39, - 0xf5c571ac07e2b34f, 0x958d05a187e1f909, 0x90694ef6d5515b3a, 0x209bdef95df89bf7, - 0xbccb60b34d995690, 0xf529d968f7a341e0, 0xd5932a77e1416627, 0xc1687a9cdb4450b1, - 0xc45e9aa8e15ff353, 0x8e6828f6a3294daa, 0xec76c31c742c1dc0, 0x1f119541cb67862e, - 0xec778c9290654911, 0x0c68fee761ade7f0, 0x57b667c3a7a2abba, 0x156871374fa1634b, - 0xca8722c4142903c8, 0xe0190366cf38ffc1, 0x4984e6b50d387ba5, 0x5d41d9046e573292, - 0x8f32b9d1374449d1, 0x065b8d9c1ce863b9, 0x8e2e98aa837b30e2, 0xf920783d944177bf, - 0xf721f6344f86d4ea, 0xb62fe0633c236b1e, 0xc29c7b8721660ba8, 0xc6ca25205429820d, - 0x000508ec8c3b9ce0, 0x025769fbfac7032f, 0x5bca89e18d0978cd, 0x302fa6dd8d8b97d1, - 0x6dd5279567a46e9f, 0x71f5ad753e205505, 0x71d668e841d58897, 0x48bd5f8f85374404, - 0x3bd378340f649884, 0x06d0ec1fa222a38c, 0xca39527ef3002808, 0xdffb4491bf27164a, - 0x3a7b0724d2f83309, 0x672df76bd93b22ef, 0xf728e8a75454f8b1, 0x10290a3096941c37, - 0x144a4c87c9ee0c0d, 0xedb15fc1ecfcdd39, 0x44df6cf6a26bfeda, 0x997609a6b7b8d23d, - 0xa07429ae1067dc8c, 0xea6e6884374549b7, 0x976984c329042c72, 0x4db88cae728aec1d, - 0xe3368aee461dec8f, 0x4d6a884191038861, 0x3ee2d7ad68c34d12, 0x32285d89f6390344, - 0x27f22908b8e1fbaf, 0x609676b991567d7e, 0x236a39b4f7695282, 0xd32e1e584d8f7fe3, - 0xb74752f975caeea3, 0x9ae7db66c19ed965, 0x5427673a3d58837b, 0x79bfa05291cd96fb, - 0x52cabea1d1632fbc, 0x38458fb7f1823163, 0x2a07dd1f44d2c7f6, 0x55e34f86c2b295d0, - 0xf588af0bbce09ed2, 0x07036247d8fc47c2, 0x8a2d6c2698984d11, 0x451a8ef3629c0182, - 0x11ee1a2ec5de3f57, 0xa764b654996a0332, 0x0fd303499408ca71, 0xf76ffc82fa36022a, - 0x399bae21bdb4c2ad, 0x1b0f6045d840ca15, 0xdfdcdfb6abff1d92, 0x706c14b6ef605b95, - 0x1861b70f36e75d00, 0x12e714d710c6af86, 0x971c7a89ddbff262, 0xf79431f73ebf4854, - 0xee28e53f6ea16654, 0x76f4a1feb1cf4fe2, 0x7b7ea402968141ab, 0xf872c7b5d0af1bfc, - 0x49c73157c1f18301, 0xddc972090426fbc0, 0xab112f1ce5c5a68a, 0x1af2a1c8f135eacb, - 0x57f44acbc78c7606, 0xbdc030802b79a903, 0x83b9ffdd0678937a, 0xee72cc6db82d2b42, - 0xf71a3585b194b3fc, 0x2ef030596cffb6cd, 0x4e5c16be945802e9, 0xdaf07126be4106d7, - 0x459a7ec76e5406f9, 0x079a958227330ac4, 0x900c4ba763dab213, 0x71b93cf3046cdbbf, - 0x0c452a8248b070c2, 0x35fec0790342c6ee, 0x48ac57836e1908c4, 0x073e89c8f4edce34, - 0x6ecb6a2badd62a5e, 0x171f99d2f330561a, 0x01004c473fb69407, 0x98b05f15e492500e, - 0x3c0910bf5b6ecf41, 0x7aedb95863f5b229, 0x76fb9a37e929f2a8, 0xda1075590c673529, - 0x8835dfe275cfdf88, 0xe1dc686664dd9503, 0x1edb23640d30e9f8, 0x78fb09c4715e9f7e, - 0xa4974b3e20a1e356, 0xccd510b9f681cd65, 0x9e9aa92fcfd375cd, 0x31ec608392512a82, - 0x82b79656961c7156, 0x01712a4ba4cf352e, 0xea891ffbabd2af9f, 0x97b7b09c24289115, - 0xbaa151ed16b1ef96, 0x25a075304964bb1e, 0x957a82c5dc4f79af, 0x625908de16e8be15, - 0x8eb34fcd5756d9f3, 0xa47d1f331dde50cc, 0x84702a593a720cac, 0x58e56563577d8779, - 0x722fdb6c49a1959c, 0xaf0926b371747a14, 0xe2ee90798977f1cd, 0x35afba3336a4e6b9, - 0x38779a9f23a60892, 0xc7bbc3a37dc8cdf2, 0xb45a26affe67f713, 0x65bba22ef6b92f64, - 0x14f4c865c57ac7e5, 0xeb3130065542b9c4, 0xcf5ee5db79cedb61, 0x47c5da2bf40292de, - 0x649e3da3ce2b28b2, 0x4e7b71ddf8b23fdd, 0x070f74eacbf4338e, 0x3f15779b4cb5aa16, - 0x2d1c26ec3febfac2, 0xaf38ffa4e5137a83, 0x6847b6f5b48c41bf, 0xcca0dd6e09525ce0, - 0xbe051ee8bdd07ca6, 0xc9acebe544707d9a, 0x8073c472692dbe3c, 0x4f8a5bafe2528d7f, - 0x37a3a2ff47d04bf6, 0x6cff881602b6082b, 0xb951882fa38e937c, 0xed81e065e8bcf06e, - 0x003c6f146c127710, 0x19807ea0423289ad, 0x150cc944daab6fed, 0x74fd3b1ab70c42f4, - 0xcdecac3d9b3faf97, 0x2b14ba4dd141bf46, 0xc4af6686965d2c12, 0xeac952a7729df97d, - 0x9600ebeca9511565, 0xa88c1f6af2576874, 0x4f513fdb0d05fc85, 0xb40c0d9c7283c06a, - 0x88ddb40eeef1afb6, 0x173e397eb5b79971, 0x7d41b69fd846fd0e, 0x6728a6fc0aaf7be4, - 0x80054837d60dd909, 0x6d0e6449c7573a69, 0xc818190280c5cac8, 0x67bade86f03eb7fc, - 0xd6dabd341fdc61d5, 0xc95051e335cbc381, 0x4df99b96bbf06a09, 0xc24e14c8b4e16456, - 0x1f20e21f1d2fa3e5, 0x265cac30f70c904e, 0x7e16fecab711c3e3, 0xe6ccd3dc85a01811, - 0x85d866d0c09325b2, 0x6ef844d4f76df0a1, 0x901caec1727d7865, 0x94a002b720f7a4bc, - 0x939675bfa9117442, 0xfae6eab0a30e5c83, 0x6639cb38a7e57499, 0x04521005e1b08979, - 0x462d2f6560a37643, 0x6aceae0c8969ef73, 0x08adad0af6e8ebc2, 0xb25253da3b4f4bb6, - 0x7b083547a7026971, 0x12310d41e71469a4, 0x1a22c4c36752d5ea, 0xa030a996fcf9d354, - 0x078ad216996ce2a3, 0xa5f65b09fc18d7b8, 0xecb1ab03dabd1cce, 0x4899386ec242ea6b, - 0x587493b7b78a4a16, 0x662b9df844f54def, 0x3750ddef0b1e1f95, 0x87eeebe2bc246de9, - 0x907b9b69c2acee37, 0xb9da549545f357b8, 0x6c89ad5785f60f40, 0xec7eba081691b5f6, - 0xc51b72030a104a6a, 0x7b3a415806a8f0c8, 0xe1082cf323cf4d2f, 0xd4b2b895250267f4, - 0xebbb34c78bf139ef, 0xd4e35fe25fe67f10, 0x61e460da796fba3e, 0x8ba1ae68aea2fc04, - 0xe83334544bd00781, 0x20535b252f1b3652, 0xc8f93c7a0e2f4bf6, 0xdc75ccfa838ce01a, - 0xbece06e00d6b8d96, 0x0d177c4f8ed59e2e, 0x5f234f0537c69e95, 0x0b2b5a729439922a, - 0x4c258f7fe293f38c, 0xdaee0b624ea9e8b0, 0x7bf341615781d045, 0xd2cf4901f284a4aa, - 0xa6ec8a077d72b83e, 0xaf3791798a74360a, 0xa1d80b7df41a9aef, 0xf26860bf40e71dfc, - 0x7eb05ff7a75ed12d, 0xb36850e6bffd810b, 0x3180ee15c93643c1, 0xd83305d22fefb789, - 0xd56e972ffccd9ba2, 0x4ef116b1442570b7, 0x9f99c5bc1890414f, 0xe4c70b1ffbeebc49, - 0xe473c4346e7a98a7, 0x136e3a2afd4228dc, 0xa39a5e60daff0b2c, 0xe0a06630c1ad47d5, - 0xff6b1eb569d417ec, 0x04f70ae08b401696, 0xe4b306869db853b5, 0x8e08b1d39109e01c, - 0xff6667b7603d2379, 0xb647a549a9130efb, 0x3dd94a96bc864514, 0x57538405361d6843, - 0x461a5ced91ac0a04, 0x604c958c81d5675e, 0xc74a8dfee8a238bb, 0x27a228160f096f2d, - 0x7f2fc94314b025f5, 0xeb37df86e0ee3381, 0x086621594fe10cab, 0x66d073c004aed1f2, - 0xd152adb9d6316f4e, 0xefff6a3dc0947650, 0x7ebf6d2128c569c6, 0x5c100a397e31b9b3, - 0x9e50ba0690685af8, 0x0817172fac5c35b0, 0x28fcf6d1f169b78f, 0x7fffe961aa66c2eb, - 0xc0aac6e163eaba45, 0xb84712730e8c77fb, 0x1bd790edd9d4bacc, 0x8a23160c5029a2b6, - 0xc1903cbfe44a4826, 0x81108fb4a609fe6e, 0x375581aae77803a9, 0xbd20609afcc63521, - 0xf97e45fe77987f01, 0xe7e87338da885596, 0xce2a6bdda1369a5e, 0x2183597d9e776ad6, - 0xfed5146f67e00647, 0x1cd3ae2f1c4b23c0, 0xf34eb0dc1fb281f0, 0x47d04d6729e480a4, - 0x2947a779df8d6c72, 0xf27c881da14c3811, 0xde1a3150e4dd9fa5, 0x11c90fc9f610a8a1, - 0x1af6efa06f77b91c, 0x6471a9a272ff2f9e, 0x5afc0a78eb9cce6b, 0x2a0646a9ebf7f6e0, - 0xba4c06ac6e11271e, 0x48218b1bbeec4da8, 0xe0e81070a20349b5, 0xac9ec3fc5d1f33a8, - 0x65886ecaebb85891, 0x38b41336d8f1a009, 0x1aa73c4f953613eb, 0xf4b1d0fa64db429b, - 0xf4b4e653b66ddd04, 0x1c8dd2613fd2d8ff, 0x994b1643b3e2a00c, 0x33854e09328b86a7, - 0x0259880aba215dfd, 0x18bc56fb25b0d6b3, 0x8faa28d7d21f5fff, 0x3bf79f1cb43b4177, - 0x73c225265109b906, 0x31f123ef829a728f, 0x076b6b083e10cc3d, 0x1480eb3c48f62a7c, - 0x34f2dc60a7a289e4, 0xee80358d422c59f8, 0x97b1fdcd8c8eb6d1, 0x678297701b2d1a2e, - 0xfee205f69433c230, 0x2a0e40e1cec20faa, 0x2908246baf2ff684, 0xc85013e1ae99b16b, - 0x73dd3bbd77a4ad77, 0x74b7e1cc8fdc7347, 0xf007b3c37c347ad7, 0x6f64f83d3206d13c, - 0xfb0cb16eef2d1540, 0x5bfeade7ae436926, 0x48eefb48e5e361fd, 0x50d3fc592eb163df, - 0x5c08dfc0b981ade2, 0xa2533b3a08f3ba24, 0x0cfec60ede8290e7, 0xf61b3426d3d5cb11, - 0xe1319f2f15334194, 0xf6d5797982406f2a, 0xd8a0e66549a831c9, 0x0347ea1599d5e455, - 0x4fd9e3920947b6c5, 0x14e44ea4a414cb23, 0x8aed5f43b296777c, 0xef0c03dc7f5b30fd, - 0x7056190689ee6260, 0xf7aacfee33c68a89, 0xd9a29a5a9a3cd216, 0x56568922104a2ec2, - 0xbfb6777c7bcbc789, 0xd3a61554d650d809, 0x9268359a9d9a7ad2, 0x693f0af0bf4d48d7, - 0xa5158b84e520af5a, 0x20b479788d588449, 0x8fb152ac308ce01f, 0x3726147316df7b6f, - 0xdeb3db1af1c75b25, 0xe8de6c73d03189ba, 0xc72de9ce66cca884, 0x2b852d4ae3edbea4, - 0xcaf9ce19a2911a0c, 0x6383b35e01424ff2, 0x1cad8056b68a21a0, 0xd1d18445d48c836c, - 0x7bb0e25f2ac58da7, 0xb1d7d1947e1ced3e, 0x23ed975c7c501fe4, 0x86faeab83ff5d558, - 0x1edd6a8f9cfafc63, 0xef254f7e01b7a1c3, 0xccba349eb30a0509, 0x486b72b0630bb2a7, - 0x422f104f834180b1, 0x4009b93eef760518, 0x81ddb055f428a7ba, 0xcb597e188f5247fb, - 0x2d936aa9e2f74762, 0xe2db4f29ced59401, 0x8788800ffd853c1d, 0x23c5a65c22709d0a, - 0x3d315ed09eb82838, 0x8fe9e5d743f5b40e, 0xf50aff377c1a1657, 0xa4c6c5817e8e30b7, - 0xdffbb3f20e998962, 0xbe0925a47a1a50b7, 0x0a040c2a3704ba24, 0x7bdc6939150a08cd, - 0xf4b3faded34ad6ef, 0xee3f831f905b6604, 0xb67448bea334574e, 0x7dd500a94f29ea47, - 0xf4b372026615c2d7, 0xc289c17384e3603d, 0x70bc80a6e80bc10c, 0xa73e8783c5a0b265, - 0x8501011cf55f741e, 0x892c477c84d26ace, 0x6e121f6229e4667d, 0x61f69ff87ee5a6bb, - 0xab35195078a8de2e, 0x3b46513598fb6df4, 0x2069ce150ad40238, 0x00db239126eb17f6, - 0x06c1ea30d13e1880, 0x2babbf0c0fa6ab4a, 0xe0940fe57336d1cf, 0x6e55abfe899f5a61, - 0xfada8d72ff8197b8, 0x53a54de87cc1599e, 0xf6ef14d9d39b00a3, 0x626cbdaf89cc33c0, - 0xa5fa6e036d7175ad, 0x1bc1858ae5fd6ab4, 0x0227d66f30e4fe8a, 0xe19a194974f70408, - 0xdf433f09afb06f6f, 0x2d6b76386fe81650, 0x3f80496e98ccc40c, 0x32ec4687f9213635, - 0xc33bd5ad8c640251, 0x0eb94c6219eebb02, 0x80f85dcf3dc8a139, 0x922f713aa1d13ea6, - 0x5652c6d5f5f138bb, 0x5533eb06a48fb1c2, 0xa6aa8b2c61115c44, 0x46846298b64eb75a, - 0x5f96b764d7920da9, 0x9cbab8afddd84545, 0x66e72d04aeed6a35, 0x734739602896d038, - 0x104a37cfbfbc58c5, 0x8a1e96286f1691d3, 0x2f7fbe15e4a37f25, 0x148f0de3b08ee080, - 0x4c6de4a2f323e06b, 0xf662e191925a00a6, 0x8c6a93240b4305df, 0x65dff271da8dbe2d, - 0xd4992dd6c574ad76, 0xfb925de7641df566, 0x91d8357176f544fa, 0x4396c1e8c25e43f4, - 0x85f05c8ca4be65ed, 0x1add1c386b776660, 0xc8d8caafd06a993a, 0x3f1d3e15a77c8e00, - 0x14cd10410e9227ba, 0x18956d85fd369ac0, 0xc9fbcb52cee29a5c, 0xafdb40a075270c9f, - 0xe0f637552547e10f, 0x87631c67f08c710f, 0x43cf47409eb5fde5, 0x79cfcd50fa494035, - 0x1dd6db07aa64893a, 0x9eda42304c1ffa7b, 0x3d077ae0f8539e4f, 0xdd1e2075d147dec0, - 0x5e700cea6645dc86, 0x7e71af1eb6e0f5c1, 0x7e1d68721bf5d07b, 0xd29914c67bc31e80, - 0xcdcf94c2f9785d80, 0x3f8c772226087efd, 0x791564b535bd9d7a, 0xb5db0df96b9406af, - 0xa5730fa1b5140b8b, 0x7947229a2ea366ae, 0xd743462c0c42979c, 0x6145b0f9410f9934, - 0xae0981e9c512e1c9, 0x4332419f1c083508, 0x7df9937e124e3ac3, 0x7cdef42df9d95a05, - 0x179e16d09d3a63c6, 0x94d1af2aa68bfe20, 0x5ee19c3900d28727, 0x9cca6e1446d20dd1, - 0x732b6069159c8928, 0x4493c711829c380d, 0x4259108c1506bd07, 0x9d09b96634448b4e, - 0xe75bcea40a0c7348, 0x5c238d2f1ed998e0, 0x4aa894a36202b878, 0x8622d098fe74e6de, - 0xb52f25bf5efab5a2, 0x86a6667676243fcf, 0xecd6f632a9d32b44, 0x907f3be53ab4e8dd, - 0x5dd2d10fe40b7b29, 0x9c2de720e15e1d98, 0x1e07320ab9c63ed7, 0xb6db65cf4825f287, - 0xf6b42e6fe65e9d8c, 0x83f8285fa787ddf7, 0x55d98dee36fbedbf, 0xea51cfd937d4a0a8, - 0x97018c762ed80f76, 0xa0d4e078050c5bf1, 0xfce8ae2c509b5a77, 0x9d2a417e265f2aa8, - 0x1aa973487d323174, 0x47e26ded1e39aaf2, 0xc69954da7af0a139, 0xfcfa529f7f2ef3e0, - 0xd1b0ff1caa3af1d1, 0xdb18c11b891ef5bc, 0x4c975e10e7042356, 0xaf1ff6626b2ff96c, - 0x3110a17a2890ebbc, 0x24511e1ddd26ba31, 0xa29a47a88c278905, 0x7cd3b6884a0d57cc, - 0x84a2a2b5f4f08e19, 0x9bd9410a923669d2, 0x6d7f897ce23b93c8, 0x306467677df3563d, - 0xac8065cfe1af1f5f, 0x38dc36578ac69965, 0xd14dbc2f9cf4d0eb, 0x334c4bc69f958ee5, - 0x605430f1be7d9968, 0x47e8e4a0c3ddd375, 0xceab5e929f0c56f0, 0x2b6d821a04a00bae, - 0xfcdd09e7a3b995c6, 0xe0a5c2b03a57833a, 0x442bc7bb3c61cbbc, 0x8e90a880bbe8fade, - 0xe5e6327702fe0fe1, 0x7a41edc3f4b33923, 0xd6ca1c9e2b0d65d4, 0x919368abc0692833, - 0xe73d6c00b9b5cdaa, 0x5db2a35e6a21b6cc, 0xc14d14b1db1238d4, 0xe1f654bda75eaaad, - 0xe0513db4ec9de2cd, 0x5c87ffd03d270994, 0x9829f0150028cd06, 0xc091edc5ca579832, - 0xf1a8ddcb1c1443c4, 0xb9c6123929f8b96d, 0x6fa16bbf4e6df1d8, 0x8cbb904f6411de06, - 0x9ef60a154811f124, 0x3b94949e80dd5629, 0x48cdae9380489f99, 0xfd5e41b3de3a37fe, - 0xa6d507da8636bffc, 0xb7315d27f9014c0e, 0x9c3ada89880bfcd0, 0x9aa60e3f9c5dc914, - 0x8a7dde8d431afaf1, 0xe0b75f4063b1cc87, 0x869822d38abe70d4, 0x2ee10ef7b293698b, - 0x16067980c0e5669e, 0x42b1f1eb003aa6e1, 0x0b7d73154a0909f4, 0xd34ed6a5f3aa47bf, - 0x3165da0cac593a5c, 0xdf7f280b6ede5fd9, 0x7740916fc0786f7a, 0x033e14d8a5293331, - 0x2f1b96c640ca8462, 0x224ff0cf2deaab18, 0x35b00469e70b37ec, 0x5973d0f91638575b, - 0x1327897a1a7629a8, 0x95855cc1a2b025cd, 0x6b326f11d0f48a17, 0x0e324561272126a6, - 0x0784c01fa6bd5cf1, 0xa3ec62a70128a0cb, 0xc69cda8a5d200aa1, 0x60b7249e70d9065a, - 0x83f07c23cec63ead, 0x4844058f53cc9dc5, 0xbb017718d0224953, 0x52bd8386fd76af7d, - 0xcf35d88d5b3f6952, 0x6809517e7624dea7, 0x8b4b452d03bd25d3, 0x0b05e9591f5ce862, - 0xd76242c79b254fd3, 0x5c6d8928fe27efe0, 0x6c0914f89300e5c8, 0x984af1ebf0022093, - 0x049308388973db9d, 0xc32aa685764a30d0, 0xec6168ae7da684e1, 0x4f43ef4824207f44, - 0xa8432aeee90bf88d, 0x7d2a87252a3c77c8, 0x42bcd0c04fdfd7a5, 0x561952626fde64c1, - 0x8caeeb9f1a7fd79c, 0xf5d2670c4416c5ca, 0x3afe6a17b29f420b, 0xb6f67508b20a8358, - 0x2ea099b4c78fa5f0, 0xcc8b299bcfae3105, 0xcf2d46285f47b4f8, 0x602da559c703deba, - 0xcfef2d0c7dfa1d93, 0xcbec20f8c0f93c01, 0x38eb2dbb318647aa, 0xf5b27326a99df90c, - 0x84400c8b0c53db55, 0x029b85555747cd43, 0x10dc4a0ae9ad3f98, 0xbc96c4c9029f9c9d, - 0x7eea105f81a59439, 0x1ffed4f53bc67fdc, 0xff97d47d04e2e3df, 0xf550949cc5466ed2, - 0x37937d6b36f40cbc, 0x5008a2fff2eeb1cd, 0xf0f89c6a312e5af3, 0xc28d4cd84197f9b1, - 0x98c2abe8976c5a98, 0xf04d7fc86613db76, 0x7f94e9853d777fb1, 0x974bcd3ce97cb7ba, - 0x65d218088eb18ba7, 0x2c58fcbe253e57d6, 0x6ef388b0b976a679, 0x398c98487a2d14c8, - 0x741d365a08cf079a, 0xa64c2412f67fd93a, 0x368fec5512858ab7, 0x8ff779d1c5cef2ed, - 0x662c7873a309dd08, 0xcdb7f33175da152a, 0x49f61e5750179618, 0x9ff5a59e8cfed1ff, - 0xb401a3cb2834cbc7, 0xc8d6d84ecd245244, 0xfb0a423ca54a1ddd, 0x610c0547d464cc17, - 0xe13969d12ce2783f, 0xbbb92c6010b8608d, 0x07922b75cfa9fc26, 0xba8907fc1ae5b9f1, - 0xe155f8b12016b72e, 0x58f87df47ab7104f, 0x75112663d571ebc6, 0xa62bf3f43694c233, - 0x5b162c07782a2c35, 0x0c76c9d6ef3b3db5, 0x10033feace7a97d6, 0x8b6132c705de40a9, - 0x2830b5d1fe249114, 0xc0994db49808f8e4, 0x97ae08cba87b4bc8, 0xc209b4affd71ad62, - 0xb5330119a5765665, 0x93e0a18d8a445291, 0xffb54a38e8291b47, 0x4679e81a466c9bd6, - 0x50a9f41b54a5c5ac, 0x41ce9e2b5117fe7c, 0xbdca22887075d968, 0x7ce3f32fef105f84, - 0x0af59a4dd2a8b6b2, 0x89e2e2bbddda54ac, 0x6fc9ee94d7a6c3e2, 0xe33eb7897c0a189a, - 0xcadd0fceea828ad8, 0xe43aa64c96d6e5cf, 0xaf0c4f1a97615e5b, 0x6df6725084ca1b4d, - 0xab1037dea2fd685f, 0xb146e3aad0496947, 0x70fcbde893aa18cc, 0xdac77166aed6ffcc, - 0xf0d5c6238541f499, 0xdffacf8822a5fece, 0xb5a781a6d3a26087, 0x91e6ab8e182a5664, - 0x65ef06c6e7f58085, 0x758fee4e0037bb24, 0x36c2bb9766b44c4b, 0xfe730b24751e615c, - 0x3a189f2e0a20e417, 0xf67534aff003d7ef, 0xd2248e5a22ff8919, 0x98407f69f63a3587, - 0xab1bdc58ffeaf7b5, 0x08ffcb2837cb833a, 0xa420fd3276ba862e, 0x22547eb44856968e, - 0x83e647ca3d07f368, 0xba6a74b7c4703536, 0xf1bce45e5a796531, 0x4a1ae853df5293e0, - 0xef56854ebedd9602, 0x07b370530c44712b, 0xadf11e023b570738, 0x5b5b8fe8ac45a1d3, - 0xb4ed5b8078a92dea, 0x06c2925bdd492390, 0xda3a0fab8c679adc, 0x6e642df50d13c22f, - 0x69610fcc7f18b1c2, 0x9f53b5f09112f5c2, 0xff70c82ca3e6563a, 0xfecf70c22c17b9dd, - 0x32283eb731206df3, 0x83e48cc878fb47d6, 0x93f9471ccc00eaa1, 0xa4f11979a515f28b, - 0x99a210e39f1baef6, 0x20b417776954512e, 0xa2f95ff6e3949c36, 0x2555c37bb36739bb, - 0x627cc9a5ac81e071, 0x74b86293f7539cfc, 0x7a709be31ce42aa5, 0x4179339c7fad27bd, - 0xa20b8ab12e02425c, 0x42278d2095602980, 0xee63838797a88b40, 0xf9e7b547bf182227, - 0x1c8673b369e8fa80, 0xc33c3256e0c5b0ad, 0x27b1445325e3519f, 0x2d67baf42e451b97, - 0xee03fb130490ec9f, 0x6c300943fca8c65b, 0x72ec73a072dcfc2e, 0x55494394b19ef5e9, - 0xab50e40e21a18122, 0x2119d92f745695a6, 0xd52ff433a5f4e652, 0x4bfea2183e187083, - 0xd5c7afdaf8562648, 0x8b70d174bf9b773c, 0x28519071bd6656ed, 0x43cdf73aabd59a2b, - 0x72119d2ae41cd844, 0x1c23751987ce00f2, 0xffe678f781a9f177, 0x7c0c7af12ed44f20, - 0x8c40c59f2ffc7ae3, 0x5335377913cae3d1, 0x7c42046196f07d48, 0x8db5301e396f4d92, - 0x1ec19ecb003e9460, 0x3a735f22284bf7b2, 0xee8a00452d265f92, 0x513c039175b7f5fb, - 0x17593c3b344554b0, 0x960ac95981c8a1cb, 0xa3fb500f09539d91, 0x2045335476ac5355, - 0xef75f7ac46f1c7e3, 0xdaab1a3469df57f9, 0xccd1e8876d4d914c, 0x3b2933144ac69575, - 0xe5c61d4ee366a51e, 0xab109899ab009683, 0xdd52b1af0300cc0f, 0x3b50d2206dc872a3, - 0xc81a12ae49871df4, 0x4c161cbb9bd52085, 0x33560b58d344d076, 0xcf69534a4ef85b3d, - 0x35206edbefceb527, 0x35ae3c2b23d7d88e, 0xc3d3a23fc8052a33, 0x3c7a4dc3deef96c9, - 0x1c6c26bd2bc2d26d, 0x79fd0a779ed5c0f2, 0x7139d1e7a0fb73ab, 0x95f400a53d2331ea, - 0x3bc0ec8e4b15f06a, 0xaa0850b77573a978, 0xe0ea4d29dfd39b24, 0x1433f346b4205063, - 0x35adb5ad8e055f1f, 0x7d99dd2fd6aa2ff5, 0x183655954736f1b9, 0x6776a81dd8615674, - 0xf56b93e86881c956, 0xa92cf2a6ec8453c9, 0xaeee9d304ac6cab4, 0x7085debc963b6e0d, - 0x9c29adf11395b548, 0x83e40fab023f1263, 0x51169db811013980, 0x67306742f93ad248, - 0xecfff543c6febd74, 0x7c7a366c221005d8, 0x6df75277ae8ddd19, 0x4b14f84c5953da7c, - 0x3c4c744353fe050f, 0xe5f0c54b4816317b, 0x046f5e0b13547d7d, 0x71e5f645009bcf62, - 0xc974991b5cf66e95, 0xb35f04a695694b2a, 0x5a5804eb646d9bbe, 0xe9164997638f4088, - 0x44b020c7c8253352, 0x279b9540f2fb60a7, 0xe5d34b0ad3ddd390, 0xa1903c6fecd5fec9, - 0xeda4f189fc2052f8, 0x19018f3ef771c637, 0x17d9978651671ea6, 0x2ec1badb62c1351d, - 0x5bbb0af8ea63d098, 0x3d37389970d9f381, 0xd920ff7363e39b11, 0xb8e1c51ccdc130c3, - 0x0be866461502c7af, 0xcb1451454f052a70, 0x303dac3f38d4f6cf, 0xf41fc007b2387bc4, - 0xc3883c5f1950eeb6, 0xf2a45e8d88d0aa9a, 0x8a17c49508364a54, 0xcafd3ef0510cb140, - 0xfc03351bf69c94b3, 0xfaa1459a7d127784, 0x95a1ae8f2de269bc, 0x6553cd60c2b04be7, - 0x586d00b187243079, 0x4cfaecfc974424c4, 0x8235a40bf3b4de51, 0xe7229efc1cd0827a, - 0xff2737e40e5a2735, 0x061bef99e5ee36b4, 0x881d9e4dde0d8da7, 0xa7849dbe1551ad19, - 0xc429a96c720e3821, 0x8aaf44c1b0dad3ee, 0x69cb149c2bedd342, 0x66f6a588db2adb9e, - 0xd3dad551e343033e, 0x17d8483e10abe300, 0xb392844983bcd31f, 0xf143d37f5b467187, - 0x794a915a05dd8a6c, 0x0d2da82d2efe48d1, 0x7a36595daa72c78e, 0x70323a0b290c9bc4, - 0xe8b53f89a31adaf2, 0x98fcd57d1d800a09, 0xcfb0322d8567ed37, 0xd77df207093b5097, - 0x4e0987e3687348fc, 0x4c3362de480e121a, 0x0b5f44a2f9bb95b9, 0x8a3a5f7a847e8bcc, - 0xb77525cb3129ba09, 0x1fe4bc8eb613f0e6, 0xcab70d559896e460, 0x34487901e49db96c, - 0x85008ca36752e7b1, 0xfc0ff71b1c78f238, 0x209a32088937e0c5, 0xef913256abda6559, - 0x45bd38a91b07d9fb, 0xf6b05c7346da3e92, 0x19ca569411650ae0, 0xd662e1acb3ba4cbd, - 0xcf3fdfd2e80cd675, 0xabf7cdff6a44063c, 0x37c5916dc264867d, 0x73e3f6949653ea50, - 0xf5488e7815bdff45, 0xf6499a2038c4b88b, 0x8faece05e43b010f, 0xac4b08dfc61644c5, - 0xf21278dadfd45bce, 0x31f5a73c05de6510, 0x3eae03f858ad8ca3, 0xb284730fe003fe98, - 0x1a2da6b830caa083, 0x26ef5b44f6915f78, 0x71f9afef1928ed59, 0xe5926678d02f0700, - 0x313ea3af07e19d31, 0x686e49294c645c83, 0xec34054a45ed9d8e, 0xc2faf8c2e99c3487, - 0xc0dbf0732a2d6b17, 0x312bb2d05d527302, 0x62b7031cea68def9, 0x93204d182e62cb46, - 0x09a88264ecec3cb1, 0x5ea34d5c887b30a8, 0xc1c74e1405e20e7a, 0xbbcbcf82b131ec0c, - 0x7cc4e7f6b83a2207, 0x1f80e6ff0ab1742a, 0xb02ae4660e619b7c, 0xa3d29f04b9112516, - 0x0d404db8f948ea52, 0xe1cc7dfec64d90ff, 0xf912e586eed8ed47, 0x7cf3f1ad0dfdc464, - 0xddcf6663f7011938, 0x1ff8b05251352459, 0x363af9cdfe67678a, 0xcd99ec9801e0411e, - 0xb424f800014e1e4b, 0x4f089ca0fa9faa1f, 0x10c9aca6b2de4e2f, 0x2a67baa55deae676, - 0x1b024dc2546f530b, 0x8d1ab50bf8d796db, 0xe0a2e1059a7f55f6, 0x6413b34e43239782, - 0x4a6bb5110cb6ca86, 0x59f7ec41ea761275, 0x8de0470a24ec7743, 0x9816ec7a9da0cab5, - 0xab9d87720042c080, 0xfd7857f64d9be0b1, 0xee1320f4b293b05c, 0xa2cb0290aff6bc28, - 0x451588bf14950dae, 0x060726bdfb707c2e, 0xacd73a3d82b246e5, 0x7724278a2073f59e, - 0xd4dbf9cfa8dedbad, 0xa317751d0f8667e6, 0xb94217565c0d8cdc, 0xf5136ad8aad1d6a8, - 0x193ce8b42ce6abaa, 0xbd0dcff1991860f4, 0x78d6a189b702b330, 0xb60e8e3a1e332056, - 0x76109c00e6ef6bfd, 0x386f1dd6eeb35ae7, 0x4f6d48b857117f9e, 0xaf315dfcbdbe10b3, - 0xe21f313c65ec1dd5, 0xfa1bb2c06e6934fe, 0xe38ccc8b7dafef87, 0x30b7c4639e9788aa, - 0xeaa495430e860a9a, 0xa48274bf62a1fb6b, 0x36107fad7d86a96e, 0xa6ade127cf4d038a, - 0xd17e3c2ee6ce15cc, 0x4a3722753efdecb7, 0x3630ce08257a9e9e, 0x15a3bb7c62f6ab68, - 0xc3de60931e83fe9e, 0x17d46bbb0ae54913, 0x2051e4773d0725e1, 0x0df39bf1531d9f84, - 0x583d2802634372ab, 0x1f17a415046b94ab, 0x95c33b2d0b871c63, 0x71afac4a1a534621, - 0xa29d49d4a7b95984, 0xeddf0ff1be1de240, 0x09a39d9a4b27ed3d, 0xe6293cd1900a56f9, - 0x27c01d045d3f5a77, 0x66b43f0151889dbe, 0x3e1c61689f950619, 0x92f4b074e27c783f, - 0xea27939121580a2d, 0x8534146fc6625186, 0x403506977b759fb6, 0x9ad768dac735c111, - 0x80f8fe3f3e24e608, 0x82b7351107a57fa5, 0x45409305fcd56b80, 0x3db307370da492e0, - 0xf6808465e018f83a, 0x02e824e37887fe00, 0x29a7f802e285c430, 0x09ffa35c45438f9b, - 0x5209b8423193ad59, 0x643484538c72c576, 0x5313feb27743332d, 0xfa3618dc415769aa, - 0xe1af4add99475938, 0x0ce72b028aca74ed, 0xd7a01986607991f5, 0x8c07f6b7b0836b93, - 0xf1c11dcb9367a128, 0x542e031497465725, 0x1828e7d6c479b14c, 0xca72204cb1a50eda, - 0xeef0670e42800eb2, 0xe7348b42db5955da, 0xe7f6c3a2a92f4e69, 0x84a845cafcb9ec27, - 0xa7eb2d43fc2202d6, 0xaa3f507dc067040c, 0x60b78246b0dbfdbf, 0xaa72e2a520cb49e2, - 0x25135fc22f9d5f5c, 0x2245469fb8411fea, 0xb987de4b931776ff, 0x45ae38a753dd049e, - 0x3b95b0142a76964f, 0xd41d9faa037771d2, 0x7386ebdaf7381652, 0x27676f6106c8d4eb, - 0x78a0d107245604a4, 0x287ff513daaa017c, 0x3d606c135fc984ac, 0x4554b122fa930f9a, - 0x04fda6b0dca62578, 0x6267419fe1b4c69d, 0x93ff60895b281dc4, 0x3f64ede669a66bdc, - 0x83de63aa4aea2819, 0x974b40177f54a639, 0xf4b912ddce00fc66, 0x1ea0339b722e120a, - 0xb2608e9fcba3b1f2, 0x4393cd9ae5bd4a3e, 0xc26e71e42396354c, 0x6bee4735f2af252c, - 0xd14021fa8a788aef, 0x4a8471a43763306d, 0xbeb09e424bfc6f43, 0x27694eb006178793, - 0x3535698169914310, 0x71b6a206914caf27, 0x5d497991cfe55794, 0x4de3dedc1ade58ea, - 0x824c95cbb22e4f0d, 0x0836aac46cc59428, 0x47c5b71bee0dff27, 0x11eb25f13ac83377, - 0x680dc24780802d21, 0x0fcde91d3fd1a0e1, 0x58e17c40eef3cc58, 0x9e5355288ad139ef, - 0x929185c99818f04e, 0xa6e8646ffd3c9efb, 0xdfc0117bc22891c2, 0x04dc7727532b3dd0, - 0x49a1b7b674ac273c, 0x912555f09b9628fb, 0x4f1a9bf7c26e056e, 0x609fb6c2406eeaad, - 0x77e3766b2d2b1bca, 0x8000002d215744ba, 0xcf76dabf6309fd2e, 0x0fa8eb1f62f285ca, - 0xa924d0b0f7475f7e, 0xffb5091bde082684, 0x9202d6e0e0de09cd, 0xc86477d5e9adbd0f, - 0x59a1073747adb6ca, 0x64edc3e7cf58f325, 0xf620ee3d4a357d96, 0x35fd4b69ac4d2fd3, - 0xcad2eaae87315c23, 0x598c31c9a882641c, 0xdbb0b78a3a58ccd3, 0xb50937eac1c9f811, - 0x7132b0bb847f28e4, 0xbb3e344b8c46b750, 0x6a7fef241462754a, 0x026922f6ef9b188a, - 0x6cdfa708ebe6cc06, 0x901fbcd30eb060c8, 0x31a680804129ab54, 0x054fa6e9a521e2cc, - 0x027956170232225b, 0x4f41e5ea9eb932d4, 0x5effe2eef3b7437d, 0x5ea89230fb1f2614, - 0x1f472ba91645f9a9, 0xf395c32c6e3b40b5, 0xfac7c83e6ded30d2, 0x949a55b2a068f65e, - 0x2d8a82000b870af5, 0xaa48e65fdce4863b, 0xddcfe03e3cbd8d3d, 0x612f8b57330599fd, - 0x382cec36c813716e, 0x95055265347241a6, 0x00cd6a8a3358fd07, 0x312ee103e457d733, - 0xb7806fc255e3b975, 0x1b197d06977ec858, 0xebc385df52ebadbd, 0x98c8ec7db11e8eae, - 0x0f9194f33d2f7e54, 0xdf1809aef30d8c07, 0x66802e00ce2c0a77, 0x06a1ef55cf005f04, - 0x3d1eae65dbad4776, 0x9f7a666085bd9295, 0xd31cf67644f81cee, 0x26545440ef0c49d2, - 0x4a7dad778efc1c57, 0x2ae7dd1da3b75e00, 0x161e72bde5e25237, 0xa192f07c516f7eee, - 0x4ddd5609ab38abfc, 0xb49c79d2545673b7, 0x7a5996778212a61c, 0x76eccb4754ac2129, - 0x130ec27bda09bc7a, 0x95101557dee990bf, 0xb3ccdf1aec16f4f8, 0x1f30968d61ac74d8, - 0xce841ad002d6d401, 0x48adad5547489e0d, 0x0fa3e57f72882d99, 0x169f8ab842f26e25, - 0x3f5dab03dbdf25d9, 0xc9283a91ffd4093b, 0xc852704c60158c53, 0x4f1536e8b411c00b, - 0x3d5c5d8bff70352c, 0x4ee2b4a8360a368e, 0xe246a46f86c725eb, 0xfc9469c042ed98d6, - 0x102ab194a220d885, 0x07e90b6446d55a85, 0x2f9992df72d86bf5, 0x396d983c8061f255, - 0xf58d232f68506826, 0x221bd594af273d6d, 0x1811dcaabb79edfd, 0x461a995a04ef8f69, - 0xeb4a74e68dc2e389, 0x5eec3898736ad0ac, 0x4baf35e9972fe792, 0x4dcaae7730651c1d, - 0xb326e776f9604af1, 0xa75afb93f4c9985a, 0x71e11f0aa915b864, 0x07b7873df32cdafa, - 0xef66a0d973ffe5aa, 0x75827192e2620ffb, 0x1e42f2ee73345ea7, 0x9d70213507050491, - 0x2763fff868879f8f, 0x2b708af607c7a3d9, 0x4c8d0c799bde769f, 0x6c8ce8159d0e4f98, - 0x78f8a0d4aedc0313, 0xe6d5524c3f7b0b98, 0x55823a0a905c0d14, 0xf08398fde98190e2, - 0xabc8b56d5eeba4a6, 0xa138fab7c379c3b9, 0xeb788d4172cc145f, 0x3de93dadc6fa12db, - 0xfe490b1ec223485b, 0x6b8b9e4f295fb4c3, 0x976a91f9d764100b, 0x7ccd1dacbfa2f73b, - 0xc493c69380912c95, 0x32e0ac02ef19c39e, 0x1574d945e719219b, 0xd4f1b2892e667795, - 0x12dc8a60d36eb807, 0x5e29d229ad21144f, 0x49208551fcc9ab79, 0x47f122150dd39eec, - 0x22eaa3a2c25170b0, 0x094a4331fb1ee0de, 0x927bb4141d9a5190, 0x5a0d27f208b26298, - 0xea228584361e0a58, 0xd2551b0871780c42, 0xdb1a77245557f532, 0xfa32bac560e67488, - 0x6d99f50a588126f3, 0x7049953f7774a409, 0xd488b2e305eab72c, 0xc204d13b25289f18, - 0x193642c2b386d742, 0xfe2cf4780d5c9295, 0x4a540fa21794e85b, 0xb888434486bedacf, - 0x4db9df92a1974851, 0xb70a374fc2091b24, 0x0b339585a556d452, 0x243fd66313cffb94, - 0xf8ead7e804b0868c, 0x8525956e07a2a426, 0x3ce2d2ef7e8eaede, 0xa52e655586b0ce63, - 0x4a176b21cb855356, 0xfb7ec6621f6740d9, 0xb44537b4f0a43862, 0xfce2c0dfdc2b2605, - 0x551d2297828d007c, 0xf1b04f778cbbba79, 0x026b331bb21bbf56, 0xbaf1f314d1c085af, - 0xbbe0d8872c98a2a4, 0x0b4a3abe8ee63041, 0x18fb743e82ac9bc8, 0xfe186e5df8b073eb, - 0x8ce360995b2a4fce, 0x289c7e5b24a7303f, 0x16bc5e15aa0690a8, 0x968445b5572e3857, - 0xe102d2c7d9f93821, 0xe9360f3d3f5a0f6c, 0xf1c3f26261d348f0, 0xc74ec5d834d335f5, - 0x317ffabe55476d35, 0x3a2aeab6959d4356, 0xdf70dd232a3161f3, 0x46514f7cf6e70152, - 0x1f6ab0283e56477d, 0x1fb8474a056d4584, 0x0fe8b9c832f04c7f, 0x8b597f975ebaa629, - 0x1b93323286895a10, 0xf557ea111e0ad909, 0x974c10fe6dd4c832, 0x014309683aae9632, - 0xc80cc1116b3b1487, 0xa6670f3f80c47dc1, 0xd033ce374d295dcd, 0x365e0300dac3ea8b, - 0x34b1de0b52439586, 0x485980cad9d34276, 0x2732abfd8510fd1f, 0xde8214565686af7e, - 0xaec5254050da3419, 0xe576d161025ed285, 0x9ed77a97867ef425, 0xa3331a4d138e349a, - 0xa2164fc42a09a077, 0x9799f310785d7226, 0x1ff548c2cb69531e, 0x8ee874fbc900ac69, - 0x379310e716f1b967, 0xc8464f3824a188b0, 0x3d74e1587a60428d, 0x425e164b169eccc7, - 0x0d4a4cf40466f244, 0xda5f77ae7f8fdab2, 0x62ec1ccfb9456f69, 0x08cb034b58e273fe, - 0xa81d361463fdf695, 0x079ace3cb10186bc, 0x8c3a19449fb311ce, 0x1a5664f6a2b51a23, - 0x190d4f5bdd472f1d, 0xfa85e03b4f0987cd, 0x4f4315eca842a185, 0xcd8d278445c50ecd, - 0x540462c7ae085a9a, 0x3d56f2f15e8bdd31, 0x9b5be4ba8b1c6382, 0x8e519be4a5ca0772, - 0x58d01433e10bfa9b, 0x0ff463b413a90158, 0x184819f35bbbb1db, 0x5f2b6ab967252de2, - 0x304f86d09d146610, 0x7d1bcb68291b8b3f, 0xb03d28df19da97f9, 0xd94682837fa6fe08, - 0x7696f34b0affe0ec, 0xcbc827e777a3aa81, 0x4f14dfbfac51d5be, 0x1e1326ef581b86f7, - 0xc43c52c16ee3d8d2, 0x279e60852489c309, 0xba5f69d8a99e00b8, 0xbc88e5b3744f8170, - 0xefcf7198c4fae2da, 0x5f09923891a903c6, 0x1c0bb1e1a7684e30, 0xb2f1dd50aa1633c8, - 0xaebece010b5d1131, 0x9e36520d425de096, 0x1de867f14a02a37c, 0x37c49b2d6d5a74bd, - 0x708e41c6041b63bf, 0x2af420f11b22c51c, 0x81f73cdfd3b0552d, 0x58c1395062a1612f, - 0xc40d6153abe1e8dd, 0x88971ceffa5d69ef, 0xf15b6fcc4dec59ef, 0xc86077d72d737b20, - 0xa86b9a30456d5eec, 0x4b8a3e9b42759736, 0x267442d8cc0a7e12, 0xdf4b08371a2ff2a5, - 0xea5a4f2db2a5ab71, 0x4ac640289b9c41a8, 0xd12ea2797153436a, 0x7ba239299abb0c46, - 0x662ce9f937a5c531, 0xbea053e1bb4f5d89, 0xfccc54658c15e02c, 0x420281ff3ec48dd3, - 0xc0f84ad0e973d1a3, 0x74829145a83f48c4, 0x65bd98829c57d3ad, 0xb8027108d7a2fc46, - 0xf4251b738dc47076, 0x2bb283b058eb8a96, 0x5a45d4a37d3d8b7c, 0x8efced6bf027dbdd, - 0x834c3b4c30477970, 0x4d44a429218c9bf8, 0xb0b912b54858e90a, 0x918c86eedc1540c8, - 0x44a79ea3469a323b, 0x5b5d87181df68c16, 0x64fa87ff4904fe78, 0x1790dcd6323b7f19, - 0x82acea9fd5936e11, 0xa6350c5c22664490, 0x3200c84ab9ae3086, 0x8e6c00397e564a9c, - 0x49eddb52aab2f42d, 0xbd495a6f7e84e917, 0xf2f42e00d84a7b82, 0xbfb4c8a1f6d72edd, - 0x50b70f3bf6f193f2, 0xfee06187928bd065, 0x6387e91a00521895, 0x47838bb2607d04dd, - 0x569c187e0658abec, 0xf7b85717e7e6c044, 0xec511d0b8a6db8bf, 0x8039a9bd7502013e, - 0xc7dab95256bc8965, 0xf7f70c5f8d8f1cf1, 0xbb4391faaf02b310, 0x0f840145e9d03f53, - 0x298c350ae294cbc4, 0x6699b62564a80a5d, 0x64e3936e10012fd8, 0x5ece6ea819842f11, - 0x17eb6f99db4e1f0a, 0xabe5916c4e98d287, 0x9ea543543d93e7eb, 0xaead23151d48beab, - 0x06660b3ef9408972, 0xd001fd3f42d254c7, 0x8f00958d80ad8041, 0xde48bc56b1ba0b87, - 0xb4f4367f255cc613, 0xae6b58528daf72d9, 0x129df342d762cd22, 0x4572efcf53433140, - 0xa5b94dbd30fd87c3, 0x62f7fa9ae15abc5d, 0xf1065594dd10c968, 0x0cc02f562919d362, - 0x99bcd97ff2591eee, 0xf4cbd1a3060c07f5, 0xdfce2d01398bd310, 0x97dff7e52c7ddd2e, - 0x2630972534141c7b, 0x88de3c795ac50d3d, 0xf64fbcdb13ef6af2, 0x04fcc2cc0347055a, - 0x73eb0736c5be3746, 0x63190725286553d4, 0x47165e264c03e815, 0x6275e64d6b9567eb, - 0x7c82b83e1aff620e, 0xe5684f63ac5ada94, 0xe2c7ef71feb0df29, 0xe78635f2588ebfa4, - 0x652711d5260bd147, 0x2056b2917aa07836, 0x8c13d3ddce7e31e2, 0x2d7119788a56cde6, - 0x2dd4a43ec1bb6f2f, 0xbc25517fdea5b63a, 0xec7e7144c8795e85, 0xbf95810e90f3753b, - 0x61cf15a19480b91d, 0x6896fc47232c876a, 0x0268323998c69d19, 0xc0f2f17e9ad88206, - 0xbafd903f5458d81d, 0xceddc2979f5dea0d, 0x6dd8a44622973d49, 0xe890cce8bb1f2677, - 0x3fd64fab693c3f61, 0x12c06efd29140492, 0x1f9a0755e07c9e54, 0xe72f78c5f768b18c, - 0x4572469950053ce4, 0xe880b142d80684ae, 0x5388c3ff81da3616, 0x6be6ce601cd1465e, - 0x986f36d9e5fc6814, 0x938d2a9ad959a1b0, 0xa0b6218fa77f352b, 0x7a35ebcdf2630023, - 0x70541e2d66c4d508, 0xfbc9864782a96a65, 0x9adb28ffad168265, 0xf8b920ad8899be4e, - 0x45232bd10610c0ad, 0x65fa338ad5be7bab, 0x35b86cb434fb648b, 0x7478ec88bab2c708, - 0x505e0ed619b6fae5, 0x84ad76c5a761f202, 0x07c362748d11402f, 0xf31e8eac8aee9fd1, - 0x816f2699f15af7ad, 0x07611aaec129bcda, 0x6f604d45b3f16111, 0x7937ffd72328e7d3, - 0x6b0a696ed8dfb2f9, 0x142f4a57df64809b, 0x1c53d7392859733d, 0x6560f0597f7c311c, - 0xd7d9902ef496f60c, 0x02678bcd95a9ff7b, 0xbf63a7986979c18c, 0xeb9918d52813600d, - 0x4cf89b27fdd74535, 0x7bbad9c4038123c3, 0x376200a95c720b12, 0x5173ef1dc916c246, - 0x732db4fdab8c3f31, 0xd431132eaf7da249, 0x21b605bddeebf7aa, 0x62182bbb26089ff8, - 0xf3c12805db628002, 0x8a720fcad8bc1cc0, 0xf2c08e8fad7d8bf2, 0x1b04b6e38cc81cba, - 0x3ed6ce6369067dc6, 0x8880b6ae8810cd1a, 0x8a4ea967c11974ab, 0xcc15eb397ecf22ec, - 0xa5138e0b64ce548a, 0xd248ee7e2552cd90, 0x1ad7bf2f341df1b9, 0xc1710874b23bd5bf, - 0x62e1b4623a9a9c8e, 0xa61f9c08c6f32532, 0x77c79c8716b73519, 0xc99356df93e74bc3, - 0x3d39786fa0c504fd, 0x9a3a758dca40847b, 0xa8d8c3b3acb965c2, 0x8d5b6d8d9fa512a6, - 0x690763471434500f, 0x4fa4d04274a087cd, 0x44f18c850f2312c5, 0xf977796dc27c69d7, - 0x05b6d37ddb002638, 0x6a2ff2c394ea32ab, 0xe98003bb3a79a974, 0xb3d2ad204cb2f040, - 0x6124e2a75553a347, 0x83a93ba6a4cfdb1c, 0xc2dcaa0e23b6c930, 0x3af25c98a1ee7550, - 0x7aa032918ed03837, 0x4301be5d8611769a, 0xcb7a72f5031ba450, 0x03b76735ec2266e2, - 0x70f7e3e285625e2f, 0xec879dcdca1ba27b, 0x048771d348265839, 0xf26eaee52c641522, - 0x26e150b78288646f, 0x056764f9deda116d, 0x296d9afc4d2af749, 0x8da214c276add899, - 0xc789666613aef14e, 0x09cc75bb7cb07ae9, 0xafeb80cfdf65b6ba, 0xc3d1c21eac0b0706, - 0x9c9c69a72b529355, 0xd575f73fd715ca17, 0x9fe21e1d77a498f0, 0x18fc6edf60f01267, - 0x91a54f5881a8c62c, 0x812cc651a9be7ca6, 0x1b5926e60ee20dac, 0xee8b5cb6c8ebbcdd, - 0x89fa637fc7cb0097, 0xb5c93aaacb869135, 0xb4049058bd6af8e8, 0xd9a42499d3ff6f0d, - 0x8542fd6e3114270c, 0x653d900b3f7bf9ad, 0xed059c741952ae2d, 0x0b1b7c09c918e2f5, - 0x14f489a6e3c392fb, 0x13b569699b325bb3, 0x2e7f8c8162215c4c, 0x626ae0b2f7c965c2, - 0xef9db9339a145ac0, 0x83cc0d3b58c3c34d, 0x263af87c9f7fcc04, 0x7471f7870153c709, - 0xdb4a6523c0be463e, 0xb7e548d8a416cb6c, 0xff11a4c68e17aca4, 0xf97692912dc4d01b, - 0xa31b35cbfbcc24c5, 0xc2af0099037a74a9, 0x410ffc39ee031f13, 0xb8f2eba46dcabbe7, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x1e541c70512e770e, 0xf34cfca15ffcd925, 0xbf6badce03ce057d, 0xd328f255dfd4a0d3, - 0xd440845886860fe3, 0x1845dc6fbea5d62f, 0xe44184dc9397c052, 0x421bd496560fcc71, - 0xa963851e3897b4e4, 0xc98faf814d82ccaf, 0x5f7b4d99a83dc58d, 0x0d2b3876bf9b3dec, - 0x6dd3d11c72044dde, 0x3b3f7fa6e895c4b7, 0x25730e7cfff1498e, 0xcbf89220ac5e87bc, - 0x6c715f2c172fd5fc, 0x79e3c02e028c7ba6, 0xafe7a734a1aac2ff, 0xbeb81b5073920251, - 0x3703b400527d1beb, 0x39b034a4f848fb3c, 0xf2b7e93618866ffe, 0x4cbc8379c4c27dad, - 0x5366b036eda202e5, 0x9d6bee6bd8a59fa8, 0x9695b156cf2d4291, 0xb8185dbc153c5e3b, - 0x41100d00ec504d29, 0x3f8d071421221cb3, 0x0518d7024c75c3ca, 0xb0da09ba69535bd0, - 0xc12e743c9ac84d63, 0x344d46ad2d270f31, 0xd6f63ba444fed01e, 0xc45f7c343f880ec5, - 0xbb19f82b6eabf83f, 0x000691f0aa711bc4, 0x8801a326147f95ab, 0xc52742c2e36e07d0, - 0x7a0403889e35a607, 0x3978e2862ed501d9, 0x3a509bc997d58d00, 0x2dbd9e21bc0997b4, - 0x181914fb064c3e10, 0xc4de01a7719e8513, 0xc58695b752a8c875, 0x83b41fcaae89ecf6, - 0xd2e1be83b8a204a6, 0x134836135c212046, 0x619d74ad44aaa43e, 0x9424da5ad22a1051, - 0xd16f73b9ebb6f2ff, 0xf1da2b65a4cb0d43, 0x316ba6ffb2cc31b5, 0x28f86ed52846db32, - 0xf9627435d3de0466, 0x02b61dd62617e30a, 0xf9b733a022dd8d6f, 0x9b39925259549c34, - 0x4e7e4707379080f5, 0xe5c7094057ec3f59, 0xdcb3d9a65c54a538, 0x565d0fc11d5942c4, - 0xf5c1d6252353269b, 0xeaf7fca851a0e2ea, 0xa6e774e639ebbeb6, 0x85cb67edb1d95387, - 0x931780678122e952, 0xf4f0858f7c4710e5, 0x739b608c7212ac02, 0x7051abae72e8e2a6, - 0xaa788e7fca29bb82, 0x4b77c20f0c3ac723, 0xfdd61248c77108e0, 0x6097ee514257c879, - 0xa25eb0e336258923, 0x3843d016cfc38124, 0xf0f92e335e868b83, 0x4463b85908c0dccd, - 0x864aed4f71de4687, 0xea349574f6d69d50, 0xaceb75cf52bc2588, 0x84be6fb4baa41613, - 0xfdf9f4da6ca6261e, 0xfa8cd5c7dabf90b9, 0xb565e764988684d5, 0x0ac6962f80cbf39a, - 0x60b0fd7f587d7270, 0x849ce0c71be03ad5, 0x0389a73c69943371, 0xc3424d63624c9755, - 0x858912bcaeb64bea, 0x1aae596e126501a4, 0x6f5ed7b3905d2ba7, 0x5da907ff6f15471d, - 0xe5515a5baeb526a8, 0xfe1e8a61ae1aad62, 0xa1d0b825b64a38c2, 0xbe32b5d3a9d09d25, - 0x0d16d261e3f9ea7d, 0x1d81601b86bfb34e, 0xc8fff99fe2ad145a, 0xf0dcf2f3afa6f743, - 0x22a617d06715047d, 0x80d7678d2d1fcb92, 0xdce7b2bc4efcb2b9, 0x47ae2b6829e6bc2d, - 0x070864b1a0f3f8eb, 0xab0751a4bc34f5f3, 0x2e05674b68329c2d, 0x33cf5af1b412a5f8, - 0x382d6ccc38b54cac, 0xf2bb38fc47b570b0, 0xc2321ff5c31f3f06, 0x5b549d1e53336c48, - 0x1301e958d0f25d30, 0xbb6effaa77a645a8, 0xc988cd8200f34c38, 0x8f531168cc847d03, - 0xac5ad122bcb6c4e0, 0xdb26ffa1fa9bdcc7, 0x03bb8e59e6aef529, 0x66784deb54136fd9, - 0xd676bdf650a0a25d, 0xd9a74948e1de3fea, 0x48fa60fa804e6aae, 0x9489bea84ba2af2d, - 0x191c19423ce59c21, 0xe523398fe7bd0b5e, 0x815b75a2a53d367d, 0x904b3b41e92c2b24, - 0x45d36fdf654a29fa, 0x66795943313a9923, 0x5511b914a0c020d5, 0x9fed5835f76164ab, - 0x8c16d4dc0d0cee97, 0x2eff99150df844b9, 0x5fd5ddbded52d673, 0x8bfa8a7ab16797e7, - 0x1665ed5c53e3e840, 0x8ecb01dc6340f4fc, 0xfbeb0129ddb00021, 0xb5db4a553eb3a5b2, - 0x55b5d9e505907d20, 0x730acaed80c74591, 0xdb8cce809cb29fb4, 0xe962d819179248e9, - 0xa286762a183282e5, 0xf8eb767d478da550, 0x0071d5e908821fdd, 0x73fe4ae3122d4804, - 0x8b637d8b332eadce, 0x607a69efa1dcadaa, 0x6c37b1bf224b0016, 0x3cd5d2303de0c28a, - 0x72f2528210d2b8fa, 0x992418cfaff23eb1, 0x78cf2f99f81d302c, 0x0cce368c77f764b1, - 0xe1e66abcb2f027f1, 0x200b8fce0e6786cb, 0x265b44b27002e3f1, 0x99f967e9279cc9dd, - 0x8a139592525e0469, 0xdfd34d3f4ef811a8, 0x26926758b0d58e21, 0x454ac56a8e41b191, - 0x6320da54a8fa2c2f, 0xfa646bff105ce816, 0x2bb72bd14ee1bc86, 0x0e3fc6013de2dcf1, - 0xfe8da67fc1ae64a7, 0x4cf4c3dacdabd19b, 0x9306677aba9064b5, 0x931e55531ca3c89b, - 0x4cece606c98a94b1, 0x44e8f727eab00f99, 0xc66a90ab7c35ebd9, 0x5af83923b1f0c894, - 0x3f36eeae51e90d07, 0x7e6f41094ae2bfe9, 0xf01d04b80aebff52, 0xf0610d0e4219c28b, - 0xe262bb76b2619930, 0x11b21ab1da21a0cc, 0xfd3a0fec6c179ec5, 0xcd64afed80a33424, - 0x561ecedb948f3ca5, 0x66b684fb62265ed2, 0x9c440479d77e9863, 0x3e92adb3c92ec938, - 0x92e0c573d198f32c, 0x12b850360ca74e06, 0x19056c53044fb6e3, 0x208b1b50fb916807, - 0x93678a562d39c00a, 0x643e47b841c5f338, 0xb88d0bf1be352dc2, 0x67ba399af8224505, - 0x2b942366edab800f, 0xa5e6ce0493ffc19e, 0x0742466642ea3de8, 0xdf1338d047f5776a, - 0x4744a7b545e9e5c7, 0x389d420224df59a1, 0x75686fff51bd8f08, 0x2090c089a050a741, - 0xe67b650808c4ce00, 0x07b04952ec1620e9, 0x14b2ce4513c7d9c1, 0x9f17b36e5480097a, - 0x01ca00bee1339a64, 0xbaf097e355c4e8f4, 0x7bae613b98876789, 0x1752feb84e4bd147, - 0x74baf4be5949b56f, 0x7718c967362bc28a, 0x3a072e42b9828d2d, 0xb7dc021c09c24b87, - 0x699a36631eafdb0b, 0xc34217d01e827629, 0xaab94763618ee828, 0x3d3bee6a1f27737f, - 0x694034c57ecfb8d2, 0xb916115dab74e9ef, 0x21eef37a366d86c3, 0xc6a293829394d15a, - 0x9179a5024bd44300, 0x9cbfe112e041088c, 0x04354bf412a48283, 0xc27dac51a12632da, - 0x82ac69910a87e305, 0x72b2023aa3f9aea3, 0xac6a98f63a142b49, 0x2f4b25a6ec9885af, - 0x21cecf62f8195955, 0xfbd1d9deee15bb75, 0xc360e27524db145f, 0x31fbf06bd27bbc12, - 0x9895b0f64fbdce52, 0x973abc2ae2b203ce, 0x4c84a9d9a6244af0, 0xaebc14fb688609b9, - 0x13056c00abecd144, 0x9ebc4cc8cf18821d, 0xe7c1de477a371893, 0x471a7c09a30fd8a9, - 0x3c064388aebea5b3, 0xbfdc3693f3cb9953, 0x1d9f687826dbc9e0, 0x932c8254711ee1d7, - 0xf81305ff4d398f77, 0x4a0683867a2802da, 0xfe4dc202a7812901, 0x7bb440636a8b5e58, - 0x7da29b93f58f2f54, 0x6e54b3ae7322ae1f, 0x70825360df68b67a, 0x8646f65fc3ea25f5, - 0x7b6d92d6850d4462, 0xb1e1c3bbe7aed2e6, 0x7f07470fdb23b58a, 0x2c8d110490ffb7f0, - 0xd4d121783aa8bb84, 0xaaed8667c84eb7a3, 0xe94b55b9202a7714, 0xddd07baef0bd7cb1, - 0xdbbdb74cb14e9d6c, 0x0551ebab9ac8013e, 0x64a17501c592a1b2, 0x899c52ff612ed117, - 0x919587c8c016b718, 0xa94cd5d3c4437f1e, 0xa3e72cd1cb0cdc3b, 0x0466487dcef02035, - 0x34f699c32a04884b, 0xb2fb582739d39363, 0xc9aaed0441730d55, 0x8137f29b8b2206dd, - 0x6017a8157622b990, 0x762e5693b88f1237, 0x51d6fd164e7da128, 0x619f3ad7cdcf074e, - 0x7a719d00e60ddfb9, 0x0af3cb60a4c38215, 0x717e2def522e99a4, 0x2954c6a10e360e1b, - 0x5081a8878ec2a0f3, 0x0776e7cb432b61ae, 0xc89b0aa8444e51e3, 0xbf5c4c5be5ecbc62, - 0x66556f5088a1b2b6, 0x99106fa54e848aa3, 0xd6c14c76c50a6b56, 0xc5092b8651fd40ec, - 0x10dbc0a985031712, 0x87e358515c6cab91, 0x8e9e0aed2c144c5c, 0x303be589027aceef, - 0xc1473bb075a8cc1d, 0x81cde03b541a2d77, 0x8888815522d927c0, 0x67933ae8bd8419c6, - 0xc9e3b27be48751e6, 0x976a51f051bee480, 0x79e056d92b2b0f89, 0x6ac7d54931407464, - 0x405d2a4ddbe3bf13, 0x7a7a78b191d38d9f, 0x60015d8ef4ee9a1e, 0x6f07b752b967e788, - 0x8a462078fbe95166, 0xf18a710a6d547d97, 0xd897bf7165091c94, 0x381fe5024f737fdd, - 0xa760eb333ecd9598, 0x3f1d5b1331164393, 0xf8c78942a35ae8db, 0x70d95bde0ec4c165, - 0xab0984717010b18d, 0xa9cb559104d92737, 0xe801738f8f7868f6, 0x4641705b5b111477, - 0x91acbc620c650c11, 0x326160d46caac6bc, 0x32f31e8c001790b8, 0xca0ea62d7824d0d0, - 0x8b867097098d7336, 0x03704c71aab8b803, 0x177c71c080fce233, 0x9155e6c6ee4b3b88, - 0x5cb8ada46c25f7c6, 0x93d470871015ce26, 0x9b324e38d877cfef, 0xea46f0cd940f8bf7, - 0x6cb93d193a796cef, 0xc3044b160736a255, 0x118f1a8d9f6fa813, 0xb4733baa4c03c36b, - 0xfc77250f9a3301c8, 0x8e0db793aeec34fc, 0x7eb088ecaf6bc67f, 0x9d4f1e8613b5fe41, - 0x4ea5eaea136661d4, 0xf118292a09bd90b0, 0x490333abdc30e152, 0x44601924a3acf26b, - 0x5448748c33ee18f8, 0xd8d025fe6fb48c88, 0xb0dd0067a61570f9, 0xbdfe5b3a0290f137, - 0x603cd98838e83b3b, 0x5c4dfe7f628e587c, 0x722fd24e3d633622, 0x147c023fa96d5f54, - 0xbba570222a42ea5e, 0x7a97c38ed935f6a0, 0x975a0dcaa6db654b, 0xfe9e7f6fb8d8aba6, - 0xc1fb4e5ec51b1a1f, 0x85a670ed76a8cc4c, 0x168593cc9eb7745c, 0xfcbe04349d4b7dce, - 0xb946ef3c39139147, 0x82bee50ca4e9b2e0, 0x84b8115624d76a38, 0x23a8edeb092207d7, - 0xbc349393248f5426, 0x855e3a337f38ebab, 0x1431b148d3de671e, 0x3e1e446494742251, - 0xecade1c34edd370a, 0x7c5f1696d2101e00, 0xe581c6c5b22c59f5, 0xe6534943b390bf73, - 0x78473c4948a43672, 0x0ff0d09d9f8fe852, 0x3e81725d3e4af88a, 0x5f0e4f583b588f22, - 0xf8eaf58e13804a0a, 0x1f925f00ff68425c, 0xe4d8037f6120f13f, 0x9ab38384d63189ab, - 0x1f8973744bda0eb4, 0x789a378ab86a4506, 0xd1d01fafdb8c2fea, 0x8fb14f62155006d2, - 0x53aec8d8cac80b74, 0xa33919849998cea1, 0x686a5d3c95b73e7a, 0x4019dacbf2fc12bf, - 0xed2c9aecb3c9f848, 0xb3158da7c7a41857, 0x5fc01dc846509570, 0x1ed6dfcca392b6a8, - 0x2a17523c12db9b17, 0x14c26525dd98e226, 0x9098ff293bf2b4a1, 0xcb6f78be8a08f47d, - 0x20eb9a528480d87f, 0x12fa2e00b3122f98, 0x82ad3b3a2af47935, 0xef8ac4055875b8e7, - 0xd1c0abc003f3bba7, 0x7d91c428252fb6d0, 0xb039686270d5e111, 0x69cdf780af4f4085, - 0x47668f566aee2b72, 0x5301376108c13f5a, 0xc4d41d4388ea1ff7, 0x63cd7c933b0407f1, - 0x1bd44807ad73c679, 0xe5581cf9d8983d63, 0x5c74f87df555eb8c, 0x23c0358535bd2d9e, - 0xc6a7f36d0694d3b0, 0x58b967273df30a88, 0xb9d106194f451669, 0x0cb6df68807af75f, - 0xc652a461784a947b, 0x4eac38d3eaa39b6f, 0x656ce67b35ae7be6, 0x89e83414e23535ea, - 0x9b900fa31bbbbd5d, 0xc6890bc42ebaf04a, 0x60c365ed72edeea0, 0x25e8dda2bf5de7c0, - 0x66da97d8f0982b1b, 0xeadf0738181d59b1, 0xba39b6f4e62a4785, 0x08f59cc2a71b879d, - 0x4e4d2a7ccfe55bdb, 0xc8c044e28c089325, 0x964d502de167403c, 0xf50cfd021f345d33, - 0x6fa9bae7bb6d9b4f, 0x2c11f6a3c6ddad47, 0x41a8d6db61650e69, 0x1b68a05c14b8b6e0, - 0xd4a45339bb1a70ac, 0x739878bebd3aafba, 0x485584d02ac3c425, 0x88cabae8109fc73e, - 0xb305dd89d2b34b57, 0x1d4967992a8220bd, 0xcaa9da90ee842684, 0x8a0af57dfce8dd0f, - 0x74b932c8639c2191, 0xc3c7d09e9016fa08, 0x09980871a6cdd54a, 0xda5743084b31a986, - 0x9c9b9c1550f4d89c, 0xeeb7c995bde780f6, 0x1fadc3944584f289, 0x80d356117ef33496, - 0x8b615e9ebb892d68, 0x12cec03341a06dc8, 0x4338add790ec464d, 0x0e96266c5369605b, - 0xb0ed4061b3827f40, 0xd8817ca7245c7635, 0x30dafcae107ebf59, 0x5961f34b41321ce7, - 0x4e24e74c156d767d, 0xe24579325a9e2b6a, 0x35cd3f2be4e9ba6e, 0x8b6f199f5e5e08d2, - 0x418df8e67d9bea69, 0xfa26895256ddf23f, 0xe9e9e31c77f6071c, 0x2569410986138acb, - 0x68f1866731cd2bcd, 0xde03a73c98486f24, 0xc3c00619f74b70f8, 0x94915df87a3b6b08, - 0xcfa04937bc3da9b7, 0xfcab4fb441553eea, 0x4a90863d67f5f4d7, 0xb70daf5a15d03e6f, - 0x15781d4c262f0054, 0x720763b46eff935b, 0xf12d0d7d936a41b2, 0xe03073fb0f209025, - 0x9750c372edd61d6f, 0x15ea303a88334d66, 0x532b9ed0601a80e1, 0xc2e2d8e88765e7a8, - 0x3ff957613800382e, 0x56e0aa88cf27f7c5, 0x5a2dc464432ea411, 0x5a3a05347ac00522, - 0xd76e41e4970c188b, 0x13ba4ed0621932b1, 0x92f54c29b983025d, 0xaf7df48c8f8014cb, - 0x708cdec450947cff, 0x7ea8fa9f7ac50dcd, 0x299c941ba0a16f4f, 0x5295f6684500d75a, - 0x4fb1750a20e221f8, 0x0f1497f42616c22d, 0xd04cf88db362d7f3, 0x3e856650449349fd, - 0x497441eefd9b798b, 0x3a18c35deb7df212, 0x79c55b05428ae55d, 0xf2884831f02097a0, - 0x27b3bfbb35b57e1b, 0x2f600694febf5957, 0xe4e141044759654c, 0x29020673bd3f18a5, - 0x89e9ff811a8d5305, 0xc111e3e15007c641, 0x7221fdb90e0ed04d, 0xe4b2ec9d81de4ef6, - 0x9d1ccea6cac48e55, 0xc4e3af64e9f634ef, 0x2e2ed293fa14e6cb, 0xdf213806e4f03f70, - 0x530ec1f6f826bf46, 0xdd584cbcaf16d544, 0x5811c4640398dced, 0x42cc4964fb7758ad, - 0xfac016cd76401ca4, 0x30eaa07ca301faa9, 0xee9cbd31c2b796b0, 0xeaad1ca417c73c3f, - 0x2c4784dd7d402790, 0x2c0cb6902bfc34d8, 0xc4df4866d25d574c, 0x05c5c33072bd369d, - 0x0e4b163246aaeca9, 0x7e6ebc1c84f80692, 0x38de40614ca24244, 0x18d79a460f38ef3e, - 0xd91696ed3e41a84d, 0x8676ddf06ddc68cc, 0x26fd1f72bb4dc379, 0x764eb86f0c67e224, - 0xef7a3660faabbfb3, 0xc67d6f21dc99a89a, 0x2acede7067f7f25f, 0x1ac5e5e23be3de71, - 0x08dd05affc1840f2, 0x8a7623470bbdb4d7, 0x03871265fe9c49f6, 0x1c84462ebe457105, - 0xf88c7827c0751197, 0x267ace53518143df, 0xba9e9f783f4bcdf7, 0xda780b4a349d6353, - 0x0a163c29556e5851, 0x3555b66f9e1be79a, 0x461c9f5929ac2787, 0x30f33fbdd2bdc323, - 0xf03b3a769602134f, 0x3da0d451fd76c12b, 0xd0fe782b376e6ea3, 0x9563e3e19fbcc438, - 0xbf21074b6c4d29b7, 0x24e102aa920d1ee6, 0x3d58f94654a91ae6, 0x87e008117d676724, - 0x294a1eeb4f6d6844, 0xb15a658e3de9147b, 0x341d7c4452aa90c0, 0x1ee6251566e20220, - 0x0358076a3c262b22, 0x128a96bc2a9f8116, 0xb6e0f026ef4c89d1, 0x1be88c2ee2d969ac, - 0xa595f30b1822d76a, 0xbaeb4fd81008698c, 0xc83dcc656f957dbf, 0x69105d54572ee642, - 0x8e609c68b0be83a4, 0x63151149865fc266, 0x36d4e3e60b3e64ff, 0x033f0cf8aa007281, - 0xa44fd7064191f1bf, 0xd5b0716a5ef36fa0, 0x1e687ef4a3f0e37d, 0x43ac9acd9bfdc4a0, - 0xc1ce6aba9c1909a2, 0xf4b2d2bc1ca7328a, 0xb509e3cbb5402719, 0x31cd90edb45c85fe, - 0xd0b068c8992fa28d, 0x268763c6441f9098, 0x80808ed8fd86ab6f, 0x62261e5720efe5e1, - 0xe426ef8fc5694249, 0xc2ed11760c34e710, 0x78c00ab617e75d5d, 0x8d6e513f2adf1ab9, - 0x2e706c376047c275, 0xc81487ac75816ce5, 0xb28b0238003f8942, 0x4175b39de9567e82, - 0xfa81d8c78715ffe9, 0x805b2d431ee5994c, 0xdff7f515d4090db7, 0x82e73352f2007555, - 0x41e1fbe052953923, 0xe860774973dcdd7f, 0x0b5c6610cd3cc2b8, 0xf988f184eeeda5ff, - 0x583f66dbc97ffa11, 0xe8547b337c80a947, 0x82e3d89fb588fb44, 0x4ee536a17f027166, - 0x8bac2f4829b54f39, 0x1e6e8fe1bd0210fa, 0x4152f6bdfecefab8, 0x710485bde7a5ca64, - 0x01aae79712cedf14, 0x3f7a5cec651c0c44, 0x5624de7366e6aad3, 0xe980c707f674aaeb, - 0xe529237f2492e3a6, 0x5d6bfe2f93641043, 0xb57376e3b3f32d6d, 0x24afb802cd90c018, - 0x76225270e2521aa4, 0x29e5e51c83767b9d, 0x53b4f32e454e7d87, 0xbb0ca34ae019069f, - 0x25b0b27a4e6cbf90, 0x646d152f46bb9777, 0x8461d5419dc5609f, 0x8f1f31c012e497ba, - 0x67bc6af80e8a2182, 0x7a6565809a50e429, 0x52e66080f7df4620, 0x6ae26b01e38edcdb, - 0x66c12dbe36e03009, 0x0ff0c07cc4d02255, 0xe9fbfef252bcf469, 0x49955d97cc49bfe3, - 0xf70c22eebc6be4a5, 0xe35dfe6c0c5db0a9, 0x852a033da5b1cf99, 0xda5bda9961cfcb7d, - 0x5fae57c1ecceace1, 0x0bf75732dc66de02, 0x58035f5372dc54d9, 0x8a3adc28a5e1a46f, - 0xb8f564e171bcf5de, 0x25427b1b1a83f791, 0xcaf50e714b539d43, 0x14792da8b9ad445d, - 0xc5f5fa532cea0c87, 0xf4d66ea7436663f7, 0xc6afd726b9094860, 0x1219d329d326e009, - 0xd54f1d4d08ca90d5, 0x0d82b638953ff0bc, 0x41f87dcc6a5ae42e, 0x2bb8ea8ea8509038, - 0x6c091f9cf22597aa, 0x9fd68df83d09441d, 0x5cdbd4f614de9d9a, 0xa4196d7615266075, - 0x703a2e1b8e1ccb24, 0x15084ab4ac295a10, 0x67b4b7e4d8ea0940, 0x0481c84ef1dcc43f, - 0xf533fae486d4cc34, 0xfa4cdb26692f0f14, 0xe347cd684e90b1ad, 0x3d2888beab886ca6, - 0xf7535ec6d55ed6f1, 0xdbd7fb4d78ba5275, 0x2173db9633ba2536, 0x6f86641ba0076df4, - 0x5f4053179dfd860b, 0x0fa07aea481ba7cf, 0x8d39df2d19f4d6ce, 0x22441a58ff6fb864, - 0x5f19faaeb072dcbd, 0xcd4d74b2e4aef1d9, 0x651177167ca39d4f, 0x7e6ba7e0a7bc8546, - 0x10b7233502b50231, 0x9253297a13d8eefb, 0x3c2ccc3877b1e054, 0x0199dfe9c6cab929, - 0x665623deef6e6e04, 0xf01626a8e43de6cb, 0xbbb7f6eead316557, 0x85be5103051137a7, - 0x35706c57aeec6494, 0x1b69384e00ca93c3, 0xef49785b21496ec7, 0x1d5b446fb7b18e15, - 0xb9ef53f08e17b87f, 0x3c5ab22030d4766c, 0x00a6813d4d22ad0d, 0xb5a638c70218cfb0, - 0xea9ff995ab0ed45c, 0x3c8d21a64910ef69, 0x73ae3956bf101b66, 0xbeb299763572b968, - 0xf8520ca4bcb1248f, 0x17af46fd102a4eae, 0xbe0804f77ce606bf, 0xdfc296ba8427cc88, - 0x730eacc4a67ce0be, 0xca0b7ffbbe158bb8, 0xd0ffb80de9a3cf2c, 0x91eaf0602b3e4c9d, - 0x1409fddecc86540f, 0x139c7c9d40fff489, 0x3b95f913b542da26, 0xbba01781e551c246, - 0x34a12e73a2f09b0f, 0x32d26d8ffc259dcd, 0x6956370419f9b683, 0x499ce59ccce766ea, - 0x619538bb7c47adf9, 0x42b115dd2bcd974e, 0xf47a11a520cbeaf8, 0x6dd9af5d6249ef76, - 0x7ae9fb5692f1fd45, 0x0e0ced7f9e9a6037, 0x5eec4583f161079a, 0x97c2aba122572df4, - 0x181a48b0ec4c9336, 0xf59a3e8ab1c6c371, 0x0c2f7eca629993c5, 0x035cb54d31bee461, - 0xbdd333ce047150c3, 0xd3d2bb990bc1e6e2, 0xbc1255aeb480c891, 0xddc30ebe83e67e8c, - 0x971d0651990fafaf, 0x935f3215bcbed194, 0xdd6354dd40ec17ec, 0x87441e6354405db1, - 0x006ab909d044f912, 0x801c8231e9c76f8b, 0x009d3998e67d2b5a, 0x8fc6d7565f33ff4f, - 0xd14a25c0593b1567, 0x324506d808c432e1, 0x240e8f9bf4264a15, 0xce7e18cf9f49626b, - 0x3a218641f1ad13e5, 0x430483f1408d59e2, 0xfab57e70bacdc32e, 0xfeb7dafb0ed09d56, - 0x5fe484dcf3e97cd5, 0xde516fd6911c094f, 0x0bac2b10625953a7, 0xc46dd1508da134b4, - 0x494a37851c815066, 0xb998a81107a1e81f, 0xbc172637319d186f, 0x4cb6bf20cdde6dc8, - 0x80e4cb1d264fe753, 0x13a61d9509b44caf, 0x32c1b469779b31be, 0x46feb69582debb34, - 0xe9c4ebf07e0e290e, 0x2ecf63b4ea8245d5, 0xe93978f561e086d7, 0x4b973a34a25a0e65, - 0x3e1fb008cacb8c0d, 0x3c0a1158887aa8af, 0x25ebb10567715d30, 0xd51d4a2a59d97b87, - 0x45eb04e72673c0cb, 0xe32c602c41a0e7d5, 0x395dd07564ceb9b9, 0x94e66615f6a8eac3, - 0x6c75bbade252a0b8, 0xaf21545653fc761a, 0x8c07471ec4564c1b, 0x5dc191b6b6d4032f, - 0x2f7bbb64c2fe332c, 0x59efb2ba5ac391de, 0xd95363ee215db7d5, 0xb705b39227c9c5ca, - 0xf8e4cdad206258e6, 0xd4cca59bd071f967, 0x3d74d6a89245a0e8, 0x3a7dec4842bd8805, - 0x614f3f332074c509, 0x46fdefcb31bf736c, 0x7a854af7bbc99f27, 0xf2820f898b9ecbba, - 0x3bd05cd305e8aa03, 0x4de1c78da5a6529a, 0x6df59be4cdd47d44, 0x0d530106ade8fa10, - 0x6672b5019c037c77, 0x7c714a78d94889ba, 0x90a135c47561d549, 0xa313ba77622226a5, - 0xfe671745bdef3146, 0xc8198f88ef63fe99, 0x0a84f16e8f587794, 0xa236497364aa3626, - 0x5fdf12e527941060, 0x94f564db899d3314, 0x85b73b7fda972f3b, 0x722f67fba576e60b, - 0x251b8fc58ba65458, 0x34d3f41fe3135e74, 0x2ce3dbb91ba91b1c, 0x1f5b2858b370dc71, - 0x069e45ffd4aa04e3, 0x122a6fcca4cb6aae, 0x788ec91eef474018, 0x35e3ee811aa42cda, - 0x5f4af759bc379fee, 0x4f226389be8e1220, 0x98b37df945e6f6f4, 0x5e77c46f24171a60, - 0x28d145da70a66ebf, 0x89a3c010851c6784, 0xbd802dcc381c5645, 0x30c99eaa44e7ded6, - 0xea15f8e3c1f021f9, 0x9b2f5651da33886a, 0x9628db308757c890, 0xd23df148708570d8, - 0x51e0e641e53704a7, 0xc84ad76247fb8471, 0xc3248960fdc52d7f, 0x4a6a7e2cbe1628e6, - 0x0da13848348fb3ca, 0x103abc968aa7a1fd, 0x9266528f01e98b8b, 0xa3167e2de040ce80, - 0x7fee8b9ce123abc8, 0x759d87a3088767b5, 0xefa8c43869808567, 0x64d9a2b071c226c1, - 0x0c8702e8263d9114, 0x0b99982d13e29d94, 0x63478fc36369e90a, 0xca4353a652be1d10, - 0x9391ebdacef57042, 0xd8e5bf557b8a953e, 0x61e81bc66cd61e73, 0x8f6a88a5d7e5f031, - 0xfa69742ee7fa4546, 0x9da43f846afe3b61, 0x80cf658d601765f0, 0x1f1773b83f86c562, - 0x050c0b49d8092a88, 0x416f34fb5b096e6b, 0x33b4c705b0f5a748, 0x9bb1c3b10466ea4b, - 0xffb4fde9d6398349, 0xf2d337814985d67d, 0x6bc1110eb2d9743a, 0x6e33c52f52e2bbd9, - 0xfbc0516a368c6bb0, 0xd0d7c491da0fa9c9, 0xbe6d87aeb5da61b7, 0x869198f6194a509a, - 0xaec75544e0bf1319, 0x1b0027e004b11d34, 0x31f99df8c12f272e, 0x0372651f52707e00, - 0x12d33cff35e698f4, 0xe257b2fe3b3e574c, 0xb98c5048badbc401, 0x5c8488924a5bc662, - 0x8f644cd678be1b42, 0xaab9ca5d98d82a75, 0x1abfdfce32ba2714, 0xdad6a5804b980714, - 0xc0024dc1230eb6d4, 0x0a7c147af4b7802c, 0x6b8283a3c7c0997c, 0x3f4de1de62b2266a, - 0x0bf19c5dc9db0c6e, 0x5841a5e3cf3bd0a1, 0x195ca7f25de69e7f, 0xd74db663b52d2925, - 0xeb31d8e67205c760, 0xc82d4b9782d632ba, 0x0501100a99b3fca2, 0xb5033f2b1f565bb2, - 0xc8ab6b5f87482e62, 0xd181833b98eb36ca, 0xd9ac858f43b0f7d5, 0xffef6f3d770f4218, - 0x24107e4f77627273, 0x06ca7272af4019fb, 0xc54c84eff5466af6, 0x21ed775d63a12c89, - 0x7d87f415338d1917, 0x28bdd18198d422dd, 0x0f707f505110847d, 0xd9b6c99add8a535b, - 0xc6dad3faf9b41bd9, 0x423a969f9dcf6e36, 0xd9a596449673924c, 0xfbe6f174fbfb03d6, - 0x8032a524bb720f1d, 0x515d1b73f9aa3b11, 0xab24f5485776d709, 0xcd2e2f3c5c142296, - 0xce1b8879912a90f6, 0x14daff1635115015, 0xe91bd0080b815e36, 0x8abc88c5c4387d25, - 0x979379c0b201a74c, 0xac3e724239f66bce, 0x8d1bf64cefea0470, 0x470e4d07262c0eab, - 0x7ebdac56f530be86, 0x2d90084f46c19143, 0x4fd7d652f8255f37, 0x529b2e918d6050d5, - 0xbdee549b7b3669cd, 0x7d8846c15f3e6e6d, 0x702b57415dabed3e, 0xec7bceadacce247a, - 0xed0189d08a36b2ae, 0x061af6607feb3c3c, 0x2e41583708927111, 0xa9d2d98e142dfb61, - 0xf01f693ab83b3086, 0x86797984636586cf, 0x16ad7ea60417e597, 0xf54bf08087f83a60, - 0x44dddb7048da95fd, 0x7cd8c9ce9ef8c869, 0xf1e9dc98b13b4da4, 0x180778ceb5e93f60, - 0x65cda16a2ae68d05, 0xa7e3239afe162953, 0x2becd78990cb321b, 0x209d1f7d571b18ba, - 0x3913e4da104a4603, 0xb657b2a908186289, 0x1103a1188d5c753c, 0x69beac8e2e044a15, - 0x1191cf3f6d7a7571, 0x616af591004ce866, 0x562c0d4cb03c7dcb, 0x25e9e69cea598abd, - 0x9c893bf18c34026d, 0x5cd4a026ebbccea1, 0xb84f9d753f5561fb, 0xcedb2cf26617dc20, - 0x25355cfb6ac8eef8, 0x41e89054ba149109, 0xad709449f744a536, 0xeffc0c31d6c1c3f0, - 0x811b8d65e89689af, 0xd3ef14243d457d63, 0xc17b089e086e905c, 0xb7b2d8dfda59e6d7, - 0x39c11e96bafc1cb5, 0x437fb325d5be53c8, 0x044d23f96c61bea1, 0xb9e1b5a25fc39431, - 0x7d3ae7d3c39470c5, 0x7f3ef92beb2a7cc1, 0xfcc4027d0cc683d5, 0xc2d98ec1ba81b000, - 0x605cc703f37f343c, 0x21bd5e86826a8def, 0x3c09ae2a942cd7ad, 0x9d815b4200880d29, - 0xdbf6299819e6b95e, 0xc916e8e2899bbde1, 0xf6404556e8014e83, 0x3786a2d596b9cf5a, - 0x7f5d67c8111d7139, 0x4b622a58f05041f2, 0xc1b00721b471e0e3, 0xbf468eee70943f1f, - 0x93c4490009d794d2, 0x4cf2e5c461a14ca8, 0xe829a2e0321e8070, 0xafc54c729f58bf46, - 0xcba58adb57142efc, 0x24acc6137f140075, 0x183ccc3e95044bed, 0xcb2e690f69b0cd50, - 0xacc6355ee2fe5c90, 0xc7cfa445d7160f5e, 0x248136af915ac344, 0xa0c8d29247db50c4, - 0xe6b663bf799a9e6e, 0xf379be1fd46b5b3d, 0xa3fce043684956a5, 0x8ecd5c722647cff7, - 0x6e00a6c81a3565ef, 0xf2e41efd648dd079, 0x677cd643b842f654, 0x5c1ec690285cc86a, - 0x1dad5afbb90e6bd6, 0x1b8edc0672092aee, 0xffecebe968c2e030, 0xa1106037c9b85a10, - 0x80106dced2497cbb, 0xaf668dc37d149d22, 0x0d6ae0c681727a9d, 0xa32b7e26fcd42569, - 0x7f2d82059d7e51a3, 0x2ab0b823c0b549d1, 0x203e9da862831ad1, 0xbe344bfc646e3a33, - 0x2f8347f299f3c547, 0xdea019d472dc1aa3, 0x929a38d1b206b81d, 0x30ecfda394d4b391, - 0x5c45f4188a995190, 0xa552c144465450e7, 0xaafce7c6e680e7d7, 0xa883aa1aca89589f, - 0xa50304173c3dcefc, 0x3ad1b86e1faad144, 0x536757c5f2f312f9, 0x1bba998c0bb0b863, - 0xd5e2791d5f06f554, 0x264de3bd5d6dfd5f, 0x30af2c0750e55e56, 0x16f1d67f5a9dad2e, - 0xbad5885a539aea9b, 0xecccda8c8f36568f, 0x90fa3986db7bfca8, 0x7f8d61d8c0535b59, - 0x854d191d772cf0a3, 0x112c948e4260443e, 0xab88aa44d854de61, 0x108bce9fbe48618f, - 0xdbf35fc33772ac1a, 0x5ad75c0aba4e1ad7, 0x423c368b23720e68, 0x9bd43dbb2c238ba4, - 0xd32206d188b28e2b, 0x862c50611917763e, 0xf0def702ee240b2a, 0x62e0d22b004c1302, - 0xf7cb3574564354da, 0x035b010237fb29a0, 0xf4e4c357ec476f1d, 0x3d7fb8be522cb386, - 0x5fbeb46aac5e7ec3, 0xfc70b12cf77dbdfd, 0x3df9d6a2eb6b9c48, 0xa2216840b6c51ac9, - 0xf9c5e0caf98fb263, 0xb31104c9ed0a5303, 0x6eee780133412869, 0xb0c8fa7be4e01d38, - 0x93edd2b4065e9ac3, 0x1d2c9d2e2af35007, 0x8efb80f36f437746, 0x3cc721f11b347c2d, - 0x608fd166ca1aa805, 0x00b7c196cd2639e0, 0x3ec65b54a84f9254, 0xebb06c7f88ada62c, - 0x4f92e9afe8db1a51, 0xc51cc40612a0c9d3, 0xcb5919738de5e537, 0xb86620f001559801, - 0x5c7ff320778dd5d8, 0xb297bf16075ecf98, 0xb79f5b56f4c9cfa9, 0xb352f500b37c6308, - 0x79b25e792e799657, 0xfa30a53d9437713a, 0xf92578d2848ae371, 0x6797f7c86291750a, - 0x2738ebab8902e8b0, 0x5d12368d03534383, 0x5844e8c0a65708e3, 0x6bb36fc5dd463c57, - 0x48bf02528b31fb77, 0x10350538b8c4a6ec, 0xe44da9159186884c, 0x5fba2bb34da050db, - 0x396adc63d26a0dbf, 0x5360e439f959b314, 0xc14836cfa19e2490, 0x82f7ed6f01173edf, - 0x046701aae6a3578a, 0xe09510cc79add4eb, 0x2fa01e8924eb5cce, 0xf650e2a079b69e7a, - 0x6ac42f4c81229c6f, 0x4e9f1c2087b6e40f, 0x47a5d10ff7cdfb2f, 0x7a5eaaf33237eea7, - 0x24d1c41d81ece71a, 0x19ee288347fca6b7, 0x7a6c32c1dde60002, 0x089d3d5f47e04a0e, - 0x38f50ba39c99d80d, 0xef588c60412fb536, 0x0f5e6713dadc4a15, 0x97f8addf24537e10, - 0xe42607936aaf1074, 0x18a3e1ac01bda0b2, 0xbae19df96dc1e17c, 0x9fc44fff5701b719, - 0x3739577c572c70ff, 0xec8625d0532a1f8c, 0x948eb18cf5b6d59a, 0xf0704c23bdfe535b, - 0x53284faff6ca181c, 0xa34afe579ec841b6, 0x33349aad9b46748c, 0x2d7f1554691fd165, - 0x307621e8e95def3a, 0x2032fe24e7ba0e95, 0xf8e912374b480d42, 0xcce32cca00a0e645, - 0xed9b55868ce1b444, 0x2720e93103c9f0c8, 0x9dc946f8b6dfc60f, 0x39a6a8ec87622b8f, - 0xcc4c758e5f4cbb1d, 0xf4c3507637c1d9b5, 0x54285447abaf8caa, 0xa17e99de2376a99a, - 0x6a8417d3ed67fbb5, 0xce09a551c62f2407, 0x874221cfc9fb0535, 0xed715c7f335e10a9, - 0x0c5b46ca481ca52c, 0xd018f048faf5071d, 0x34458f6c08a30c4e, 0xcdb7052513e9ed8e, - 0x91e8a125a71ebdcf, 0x8535a14bdb147f44, 0xaec1c8f586f5ac5c, 0xbae0b7bf99a3dff4, - 0x63ec690e40be9c54, 0xff9ed3d27beb7e52, 0xa01869cd441edd74, 0x2d6b6bf959c9670f, - 0x34f27ae68453c98b, 0xf9fb0934921334c2, 0x19aa0432dd347249, 0x4d3dd7a1febf2a74, - 0x9ef327e5f74cfb51, 0x6baefa034ec26115, 0x7140ba3415804359, 0xe2466a3c371414e1, - 0xa5a62195e6923b40, 0x898baf05eb4d9c39, 0x3996b3a0f33dcb62, 0x8a1bb1620eff8950, - 0x1c53be39f33cf50b, 0x257096ad1adff1ed, 0xe3505a4aa666a7b3, 0x013d6134ba7ea3fc, - 0x6bf1c89aaff525d9, 0xa0fc0e736f736000, 0x34e55f3f1f3e392d, 0xc0cf114f54590022, - 0x1883e0a50ef489af, 0x2f2b9a58ed1cac9c, 0x44a12be7ed36f3ff, 0xd6e110a11207cf4c, - 0x9b5d5d222670af7e, 0x12e8c886987f63f3, 0x5e076c5a90cdeec0, 0xa72d9fb919ace8f0, - 0x520519661edf0d2d, 0xf0e6a4a90750a054, 0xbdb2c441fd2a3f74, 0xb872d9d9e4776049, - 0x41d75f0b290c1035, 0x4f06fc001bc708db, 0xe381f601a2b76bef, 0x2629a472fdc84af1, - 0x52ff42a03ecfb4ae, 0xe5dd6ba8acd5122d, 0x1af68b4433003501, 0x6db024179e96ed70, - 0x2b9ad5db44fe10f9, 0xfddcb617d95b943e, 0x02bdcbcf2e147139, 0x44afc5dac6a4caf9, - 0xa009608bddf5fe05, 0x5978803cd0366d87, 0x86a428e6252358df, 0xe49709b8e8db8e9c, - 0xd5deb751baf194fb, 0xd5902cb05ec7ef2c, 0x44787e411c521184, 0xddbcf7d09fc23cb3, - 0x643310603a0802aa, 0x32ccf135e62ce182, 0xcc7df710a3025b2e, 0x611e505be4206f88, - 0x23cc5978137eead9, 0xcbea00fb135ce56c, 0xb29de0e04cd38312, 0x8c1308ad32279a66, - 0xcd3e2459e1ab0aef, 0x216a9c9611b627cd, 0x98bc251f7eca2ad6, 0x9f4f1d5e89d71efa, - 0x36c637858ab5eae5, 0x305c532c1a79e6ab, 0x238457ecb01afc37, 0x5478cf7737698637, - 0xaf3257b917110270, 0xc8da149b7e6f45a4, 0x30186f92727482b3, 0x081f403a4ce80c8c, - 0x42a4b9cd81a07db9, 0x6ee93d3bf6d1657d, 0xf5bd8b3994033b44, 0x8faa1c39e00beca2, - 0x191faa12c20986ef, 0xc4147a8c23752e2e, 0x608d2bdf814e3f01, 0x5d794c4cffca8113, - 0x2f91b82ebbe33b18, 0x3fed2044689fef2a, 0x0ab76806b4f10d56, 0xf787c4a2b26fc3c1, - 0x49f2dbe75d97957b, 0xdc6a7e884df4d2f4, 0xc4675624785eca0a, 0x10cf7afe7804daa3, - 0x4d966435ff491393, 0x61dfac6a9e55e0ee, 0xde3879d94db709a8, 0x9ef0d863662f8257, - 0xf56def24c498c0c5, 0x6ced2b1f7970d8d5, 0x20bb1a43ed5acecd, 0x91ebf0263cf74014, - 0x03a6bdf43e88043a, 0xee49f7dd5a369f39, 0x076dd4bcef99fbb9, 0x8aa06e5947b4699e, - 0xa200f57d8a93d052, 0xc8c84b478dce8e9e, 0x5f80d5ec20ce3167, 0x436a51f5b9d42d73, - 0x5bc0a3233d44b16c, 0x9d4be201e1745983, 0x0a7c2c684f1abc18, 0xfd02f132754047d3, - 0x151abb02a164fd5a, 0x76c41044e19513dc, 0x1a495d8520defda0, 0xdb38bf832f7cfd87, - 0xd5595a0ba7dfac34, 0x7d73721c77151f6c, 0x3d94cdfd9ac740d9, 0x1f30dd516678721a, - 0x1e1ddd998579ec51, 0x2fae0d16b519bb97, 0xff8108ddfcf892cc, 0x33be2d63886e6a4d, - 0x76ec3569e2419dbe, 0x2e8fe6991bd65185, 0xd21fc89f0f9aea76, 0xc4527479fff574b4, - 0x5e6a3350ea9dea45, 0x2a632685da3e2fbd, 0xaeab719fdf07c663, 0x0b753b179fc15019, - 0xf3f5a8a3cdcc902a, 0x65a0bdfef8b1d605, 0x58d68aac1e5ad17d, 0x2e3c5060d4bc3bde, - 0x7dadb3439b6ba82f, 0x5c279953f49f6223, 0xffbc7021e3adddcf, 0x6e4e36dfb1efd724, - 0x6173345379f7ad00, 0x6ace8d0917f57d85, 0x755434794ca3337b, 0x6fddd67da818853b, - 0x92fe6a49aae60643, 0x07a5d11c02798e55, 0xf03e282173c16aba, 0x709895d4a19606f0, - 0xb903efb73967be35, 0x4277847ec748173f, 0xfe00d8baa8381c7d, 0xce4fe4832dbb327f, - 0x66e66ed36b3d29f1, 0x55c15ec18d8f0dab, 0x9b78987c211547b5, 0x7276dd142c3c8179, - 0x8af87b41155573bd, 0x8bc80d818fe6cc62, 0xbc0247b9b0e99905, 0x6953e198813fe3e3, - 0xc57c535afa4b6417, 0xea35384510c44793, 0x9ff7248d6b9f7f18, 0x7edb43a68b26d13e, - 0xd2afec7cabcef686, 0x58e27b88e4999de2, 0x93f507b934ba815f, 0xa3bea24c3b2bcaef, - 0x936bc3cb74ea28fc, 0x1049034976cdfc08, 0x5ffa567755a6f3ec, 0x6f246ef4f0f9b7fd, - 0xd4cb878e65ab4d92, 0x33cbf37816c69315, 0x8578962dcec8b94b, 0x4e8744c1e9b91e04, - 0xbb5f2acc3333654b, 0xead375507d82b21e, 0xc5bd36b91787f7b1, 0x99e2a9abe10c3d48, - 0x5b8f927facf76d9d, 0xebd075a29a58937a, 0x1a2aa08358abd181, 0x3c442d9a564ff679, - 0x4856acae05de0547, 0xf1c3a136c13ac915, 0x4e77e29156ddb629, 0xa22d4c8860b44341, - 0x959b8a4a097e8315, 0x5fb9c1b5a66d0d71, 0x1ccfa9c23b737713, 0xe818493bbc436232, - 0x03376127aee296e1, 0x5715ae4a5df418d1, 0xe3fcb0a8ec00d617, 0x43f9e2669b446dee, - 0x56b55c3f9cc84f4f, 0xe78ca292dff31bcc, 0xed5246042dd588f5, 0x438968aee1f139b0, - 0x08365669ef102388, 0xe04c70e4916e802d, 0xa32c6e76673bef96, 0xd3cae06130e8c2ea, - 0x9daab43cc7cd9f8b, 0x8fd6145c40a4d438, 0xacc6b2f7fbbe3c65, 0xb34e2840bad9c7e0, - 0x0fb4a33846acaf62, 0xc3c3924479179a32, 0x5ad79428d9a5a083, 0x4881d3f99b803cc8, - 0x4b8c584624e818f7, 0x47f6691a0b9e6e8d, 0xd7d8771cdb2f8d3d, 0x1a5a5989b5381fd1, - 0x7998ee467f1efe81, 0xac02d1c33fe261a7, 0x332d5d71e63702ba, 0x922172db372da20f, - 0xbc25fd105af7f302, 0xbaba8cecb5e8961a, 0x358e7bfc84e999eb, 0xe74813bb1f364914, - 0xe16327e58ab2028f, 0xc7f7c76397c927af, 0xd6e11f74422cecc7, 0xcbc56ccfc0173fcb, - 0x740551f173246ce8, 0x0dc4ba18247800f6, 0x7410edeea18dd3bb, 0x37d82984b65d3cd9, - 0x844c40f233cd939f, 0x353e8e2fe3122269, 0xa9e884e357a51aad, 0xad08a43a002abbbd, - 0xcb9651aef059546a, 0xd51369c33efa3902, 0x9bff1f325e7abafa, 0x6edb693ae94065ef, - 0x17b39083db27cf42, 0x74f318c81b61d013, 0x3d379e1bb021575c, 0x42a047b500b09525, - 0xa18c594383a064ab, 0x2fd502802a71cada, 0x2564268a5a6a975a, 0x8492edb0fab8045d, - 0x3afe3fb3b8b16036, 0x193f0d66cbc88695, 0x94f1f2ed33882863, 0xf206f47b7a4febd9, - 0x6d2d77ab4c8d9fcd, 0x722328087827f6cc, 0x6b95796385c74a0e, 0xa57d45ad50382344, - 0x8de7ed4a217234f4, 0xa8e53d32b0158058, 0xeb11b0fe23fb037d, 0x879bef4e87669b43, - 0x148d4f3598ebc21b, 0xe4b566a9e47372cc, 0xda424e7e105c1f8b, 0x1c6ba65609042d78, - 0x932a62b775e356a6, 0x5f3d1de668b71dab, 0x4aa37159a80a7f80, 0xc2e847785eccd0f5, - 0xf505763031cfd353, 0xe54b8c25f891d30a, 0x346bbd0e6285a422, 0xe881092ab69e9319, - 0xad007973bb1caa5c, 0xd6b84c9d3b720764, 0xc35538e1c491f47d, 0xd621863b5dac5502, - 0x05eb6818f66dfcb9, 0x6384db36e3b3fbdf, 0xe6ae277b14a6959d, 0xd73ff1b41b6bfbf3, - 0x6297cef45d246cd8, 0x3cda64e84bebd99b, 0x80c5bff15ee5c7da, 0x342b55b4b739cc78, - 0x8564653e1ed7ea27, 0xd934eb7426ba2305, 0x9b2d6ee44cc82c81, 0xf6c2b89d5c1cca10, - 0x762cf4b1ba5619da, 0x19abda57b2cf51f3, 0xb173158954732121, 0x103909e6fa5a19a1, - 0x93bf27f5b03ba2f8, 0x8af419fa775d0611, 0x680b5a80d88b084b, 0x9d072d85a1c12019, - 0xdc181af3c2eea2ae, 0xa9bfcaa8ebe2c9a2, 0x69758853ddb7efbc, 0xedaab665c71da2f1, - 0x920a1da9d39a47a2, 0xc3a1a5f93ca2df93, 0x86a5a8ca12bda254, 0x6e1ad3ef7f55e4cc, - 0xfaaa66e13edaa7e9, 0x6e5ddd9c96f66c26, 0x2864c17aea2b7e29, 0xbab2ba405f6373ad, - 0xcccfce4b309ab156, 0x537ac8940592bff5, 0xe3c5e009807fc5e2, 0x156cbd7eb477a3f6, - 0xe6f9f10dfb0a0e55, 0x9aa6f4fc5664f2d0, 0xc282d86703523f3e, 0x0d866f100297c057, - 0x96271a76b8b85bec, 0x220ee9c693c7358e, 0xfa47f8b10c93c613, 0xb4fba10fff4390a1, - 0x63165b47063289c9, 0x395f03a242cf4af2, 0xa5b6c713ac181df8, 0x41d2f17efa58d9c7, - 0x50fdf5ec40e2b376, 0x4c77f025d92506e0, 0xa96f6f22cbcfe05d, 0x3c18da2177c52c1a, - 0x48a975ba65a2e1bf, 0x3bb086c0b07b6626, 0x0bc61a75cc2690a8, 0x3f5374633debcb37, - 0x360b219fd9dbb2e1, 0x3e1eed006207d5ef, 0x8a7a98a7b1a1c5c7, 0x28909be1ac7b48ee, - 0xd986a2f79e30fc97, 0x12878ac8e96f4170, 0xaeefc88149995f22, 0xcd0276fe5f26eb9e, - 0xdb5d423e2aaa46bc, 0x1edd91b23397010c, 0xd6aca77eb717fde7, 0xdb18f1b78e1ff7d1, - 0xeea1d95b3177c539, 0xf0cc9c85af103f7b, 0x3cb167ecd3923702, 0xd3f2a5a5b23bc179, - 0x13e21d46f2cb3f65, 0xf07d5cc1f321fd6a, 0x062c86312a4c7f07, 0x615ddff7f2970df4, - 0x65241783135029e0, 0xb101f721763c50e5, 0x4202b8ce089906bd, 0xef5d1570aad6665a, - 0x294db18714531e94, 0x7b6607a9a561c8fc, 0x1f580c18220cd459, 0xb23b9d691f8324b8, - 0x4aafbff9023c8df0, 0xe362350850cd3b5d, 0x8091aadd6467829c, 0xeb9798233a9a81ad, - 0xade8f49c225d73a7, 0x8c5b9908be4ec85f, 0x0fdc05d8b043bbeb, 0x563136f4e1050ce9, - 0x6ab29dcad125cb64, 0x11061ad0651a2362, 0x4c9b7c2933866073, 0x86a704d539e9541f, - 0x98c397bf90168d4a, 0x99ad03023406bb45, 0x4f329c4f82842e91, 0xc02ded3c6964c15c, - 0x6f5a182f0081c8da, 0x7805dd8633f78d42, 0x95cae82533c6d1cd, 0xbe4846508b648550, - 0xa71d8f622b54af50, 0x4baa7b8d2c8863d2, 0x9a089de4fecc8638, 0x9e29146ace0d1f57, - 0x272d9903b4cf59f1, 0x0f58f4628450bc7e, 0x226194b91d99d0a6, 0xa66770bb2b56bd89, - 0xda1d0210505e1cfd, 0xb5ac2893a624ffea, 0x2fb0b85aa830fb2d, 0x6757491e0509aa95, - 0xe4c696434742469d, 0xc0cd6f059d880fc1, 0x307c2a2a6da4e3aa, 0x60dd2949cad0e0bd, - 0x2a5298038acf4ac8, 0x9fc566dcb1bffea2, 0xf65fc70fc6ab132e, 0x45ce107d03829e0a, - 0x33d21576eb7fed70, 0xe57f35d14dfe2f12, 0xe76645d9f3b75b17, 0x6651f5077ccba481, - 0x8b9ae360daee58a8, 0x8552573e0edc7ee9, 0xd80dfe83e6c86e9c, 0xb26e29459e61e79a, - 0x818b7f9a7f93c276, 0x4f8137c31ecc9d70, 0xffb165b23a9b2a70, 0xc25875ddd191789f, - 0xb3625285514bb82f, 0x8f1273bd2346316b, 0xdafeb11694cc28b2, 0x2ec1894813e07f0a, - 0xeee61d38ed1b6aa9, 0xae7c9e7a5cc85d3f, 0x3785e7b4f7742af7, 0xc8b224bed3f745bb, - 0xd4b37d583dbc42b8, 0x8660726ff6f25145, 0x26bd6a404e1804e8, 0xe5ac91cc64a147fe, - 0x3985bccf95351319, 0x999458ac899eb4eb, 0x6a6ac63f7167e315, 0xd9e162a06b4e7036, - 0xe7eff0b30195c8ba, 0x07e74a3a0a9dc0f7, 0x889507bbe7bc4f3b, 0xba92d036f2a7bc57, - 0x5d9156aa97b2d908, 0xc91109d1f306fc43, 0x2b6514c30ace3f3c, 0x16fc59a56fa11ef2, - 0xcc664c2d18b83e82, 0xb297972dd36ca51c, 0x265f2daa31369729, 0x1e3a2f5a31842e10, - 0xdd6ca4635d4aa6d0, 0x4ca6779902250192, 0x27d8228666525d39, 0x45f59acd15482679, - 0x958f894c69386f0f, 0x1abdfb8c1f6e5756, 0xc69c1f045458a346, 0x38b3afe2052f8a2d, - 0x4078d5ee1feba16a, 0xf1cd4e923fd45df3, 0x69dd3722dea6e351, 0x158f3c258af81211, - 0x8d2bda807ebf6c21, 0xb8c1e6c894c26210, 0xcade84233568ff55, 0xe56c96e63d5109b6, - 0x2945b8751c0ba49d, 0x051f802f1fc88fe7, 0x8a75dd211340207e, 0xa8b92d8f2a8707cb, - 0x9644db650cdd2fe4, 0xc3d23cf03977fe01, 0x6313e1bb71dc2ec8, 0x246551ac3d8ec019, - 0x5b4a6c9528601bee, 0xfe75fec1b28c5ee5, 0xf72f66737b86f316, 0x7e641d40b204e42c, - 0x483ff8ea89e82039, 0x561a2303715633a2, 0x46719e9415a1e08c, 0x956efa06711ddc74, - 0x2d35ff2c23b9e65c, 0x4357b87531889ba0, 0x0a07811281787917, 0x2213ba4a2cb9b5e2, - 0x2b4d8bc6c175c597, 0x5be555e25a12c7e7, 0x6e18d7121915f99d, 0x853b74f588396a11, - 0xd093260fbf5f1f40, 0xcb53772dcd478dfa, 0x1302ea7a411dd070, 0x53f5068f0eb39908, - 0xb07cd5c3411acbd9, 0xd387e108ba20ce4f, 0xe8134b53e92f6f88, 0xc91dd324da5ee045, - 0x48acbfaef216ecc6, 0x86497dcff9b3c6b3, 0x112b22cea4bbdf2d, 0x1b70d2ffc21ceb21, - 0xcc158766af268566, 0x5d2032b51cd7da99, 0x4b30323c2d5cfd15, 0x2303a5a2aa2c1a27, - 0xb2f3374f6e67f51c, 0x28d0a0787fa49ecf, 0x9988874205a5fc98, 0xf2533549ef2b440f, - 0x8a7381ee3a0814ce, 0xa583db2a4d46ec8c, 0x2c5014909e04066a, 0x14c2aebbb2a7ff7e, - 0x66c1fb4a678bcca8, 0xf0fbe3adb69a2861, 0x1282af346cbc0204, 0x72b63c7d1b3170c5, - 0x92cb8940e7e6fbed, 0xdcd7146983b485bc, 0xabe59a87e152f788, 0x3dfe716b4775acd2, - 0x0f6aa6d15db1a1a4, 0x06a1b1ab965efdad, 0x20a1fb80b7010acd, 0x44c86034b4b4bcdf, - 0xbc145f8e5305e97c, 0xadaf6b26a18d6219, 0xf5c7291ba26290d7, 0x3ffab8a27a8db052, - 0x693db2ed8607e8b2, 0xc7e23d342e30d934, 0xd3488f4396b3ea6d, 0xa33b65aa667eed1a, - 0x8850ff59d62e729c, 0x0768907c70d5ab7b, 0x164ab173f43c6dea, 0x636ba832f1a21915, - 0xb1c57941918310ce, 0x4273508cf4cad76f, 0x13e1d3a9aab7e837, 0x6e92703feb9dee23, - 0xe10372de74e5f6b0, 0x48df1fe29f56d2da, 0x308607440f57f6c4, 0xff576bcd45645482, - 0x615c242c28a03db3, 0xf1c754bbcb2b5715, 0x0ea7fca5d3598bfd, 0x9fe34bceb9148ecf, - 0x51b8ff98c6e10c7a, 0x8943abec8e83bb57, 0xdc97e28a6f686122, 0x1ce3460a7a2afe2f, - 0x0ee0e9fb4843b9c7, 0xe5e9e9a943d18cd8, 0x640d5b5b25f7d5ab, 0xb81676035309fdd7, - 0x2b4e26aa6399404d, 0x70854bf02df5a358, 0x395d6d4db7f237d5, 0x2850189822c95984, - 0xd7ae9188d35e597e, 0x029b7566aef51ff3, 0xda9c9d207b36db37, 0xdfc9a89c2f358218, - 0x8c2cad92d5a1a985, 0x7c3801339827505e, 0x747302134eae3985, 0xfa4318c56ec0f773, - 0x388c4e7deeac0d5c, 0x394dbb17b16a369a, 0xf005a583ffdc4a27, 0x29954e8940b373b1, - 0x672bdc54378e354a, 0x942242ac97be2ac8, 0xba805662716cc7e7, 0xe5d67e0ddd2b1f0a, - 0x0f38fc56cfce6b30, 0x8935e4ce4b26b1e4, 0x88dfdc890a4e18c2, 0xad2b208bdc49e390, - 0x329eb6236ba0cbef, 0x3895d7a424dc7f8e, 0x8244bac2a36eae52, 0x28c3b33458a663c5, - 0x3e5cf117677841cb, 0xb42f3253c6cd1f86, 0x3e4e870bd9da22df, 0xa96c8ed9a585f0f0, - 0x3d00f92246666fca, 0x3858bc5803e19cd1, 0x4476d8fe1d390e14, 0x0d062e1c57e7e670, - 0x15702155eeed580a, 0xe91f5a8901624989, 0x1e7172ec72a681d9, 0x56b8f3e859603611, - 0xc9a997c286777135, 0x0331adbda5d96c36, 0xba3d91628358f466, 0xbf604ce7cf8e7cfa, - 0x183ec2b2815db517, 0xb83a079b7ddea6b9, 0xc66dd7c82bea6f62, 0x745332073b8c092c, - 0xcc806155f6441760, 0xc04b1806d6841186, 0x27a9ee4401e83103, 0x0d7aa2167cad5bde, - 0x883e309ee57d6571, 0xc0b22063ade52c3a, 0xaffdfdef083b7e3f, 0x01ffe2c38ed30c13, - 0x91aa83258d1ae08d, 0x5268c1ddca3e676a, 0x7f05f24d383aac47, 0x5a9fade0e33fd84e, - 0xac62336b9dc0a58d, 0x08475c00611b82bb, 0x25812e21f474079e, 0x3865ae99daf7d5e6, - 0xef5a1f19529f7238, 0xbf7c32bd1324ef6e, 0xeb7990cdd591354e, 0x0b7e9b3568c9295b, - 0x9b6a819ab48f66df, 0x447fc94665ebcbee, 0x5fd1f03c1820cf4d, 0x55be8c7a1c49002d, - 0x4f8fce645fd9e856, 0xfdd1f3a1179585e7, 0x0ff84dfe5538dff2, 0xd86cfca818fd5e29, - 0x5051a6f4cb8f1852, 0xd83099f031edf12b, 0xdc706daa54d16236, 0x6733c541fc554412, - 0xdbae200a9bcadb00, 0x30d6e2b46d1154d5, 0xfc7636afe105231f, 0xc5efbe0b8eb5c691, - 0x2fa645e788f9185e, 0x91551a05b32dd160, 0xea66cc726f1ed338, 0xe85005c70e5270bd, - 0x5b0282fe6eac99ec, 0x875f17d234da5565, 0x76f1baf856a378f7, 0xacb6eda56d8308da, - 0xa3faebab6fe719e6, 0x3193fee2814307fd, 0x12ade73dbe2e1cfa, 0xffcf5218052b80fe, - 0x2230127dd53288e7, 0xe3a1541c84e73ee9, 0x86e30597eb25fb9b, 0x43ea8e9e03580663, - 0x940aee3fc662a354, 0x8bc6247708cc2014, 0x4441142c9f724343, 0x66ec0c9038c88f34, - 0x2d9da13e78e5dccd, 0x37c374f0e412b225, 0x972e29a2b8d77113, 0xfbe2c8e22810bb1b, - 0x692728c169b58302, 0x366ab521e3db6756, 0xb2eb35735acd6096, 0x32e4b3f16c650eaa, - 0x6e303652947e2566, 0x2586049a9745f841, 0x5c28b4155238afcb, 0x960e7c2aab1acd26, - 0x1249f7ac43cf4ee0, 0x46220bfa88530db0, 0xe0e52f5cb657df54, 0x7a7f9d7615453d49, - 0x218574431ac3d76d, 0x36e51e9795e0ffc5, 0x8aa6663adfbfd091, 0xa116d08d09023191, - 0x51ab88f25e2149bb, 0x52c7435c768ef4a8, 0xe89cb1028057e2d7, 0xbd0543cea8e0fe24, - 0xaddb77fd1c72ce94, 0x93ed9589d3061377, 0x1f47f77afff3106c, 0x87daa2162a22a1b5, - 0x6e3e3527c8082abd, 0x800a6727ffe8a64a, 0x320ac9a4211d027c, 0x884337f730ec7d96, - 0x5370fb653a989feb, 0x8c64ad1fa81f87bb, 0x52d19955969af256, 0xa0a067470800a975, - 0x680c4fec6b788702, 0xfbb999858af0a498, 0x7848d5b86a9a11fe, 0x6939d3440cf72108, - 0xfa401ed34e48c849, 0xa5a38c9eb063adf0, 0x12632c9eb3b1e9ea, 0xba77e48c5fb1cf34, - 0x6f4291d533197e1b, 0xfa44065fa55b5536, 0x0a7ce048e22d4482, 0xcde9174db5f3c081, - 0xe73d378fa5d27c5f, 0x97e219909976313d, 0xaefbcb34d9cbdadd, 0x5f8583bb43266ec8, - 0xcdab1ca34f92926b, 0x575714eb73339bcc, 0x4a419d4a4d4dc3b9, 0xde33777e8e8f3e17, - 0x6044ff9d22d88560, 0x7d8bf7a6cd9a9d2b, 0x1ba6853ea2ea2f98, 0xe209ab75d012d925, - 0xb9a915efda8a090a, 0xeac7d5074c7adeec, 0xc5d31bca566340df, 0xbf77e4e09e458553, - 0x90f0bd0267addb93, 0xb322c412ffdc1721, 0x1ee2b2511ea85175, 0x3a68585b0f064e3c, - 0xf9d6ae07dfa30ec9, 0xe18282eca1ef9f4e, 0x702b952c95aeb29a, 0x111a774e8aa644ba, - 0x3ea15a430715143c, 0x0b81af2c78cbaab8, 0x31f3775db062ce0d, 0x9765017ded5e42ed, - 0x7f60536821ab902a, 0x06c14ed3e061f4bb, 0xcb398f5a59064dfa, 0x3d0d13df88740a02, - 0x51f55de210a2d733, 0x0df3166f909b5024, 0x7a3a913a9bcb153b, 0x4e9761541f28125a, - 0xcc5382c1ae741e9d, 0x1aee13d030dc97bd, 0x77da261184de2386, 0xfd9f5e4201c16cd4, - 0x31b6b8d02137a5a0, 0x243b7d5dd6922fc6, 0xdccd5ed34cfdf97d, 0x4c42a290733d094c, - 0x81cb64b005dffaab, 0xceedf9b0fa40098d, 0x77d66ca2919555ac, 0x5bf6d382bbed5b35, - 0x339d9e61a19ce4a1, 0x2021d282fd8da982, 0x0b22a0520fa29817, 0xa5f34976ae5e6dd6, - 0xbdc5155c0bcce6e1, 0x7e76c4b391f58bfd, 0x1d931d8f70f02efe, 0xd7157b1f9fd21320, - 0x6d615f9ac3752867, 0x0bf5ec331b7113cc, 0x30687c33af4d2792, 0x41fbc90331db3475, - 0xeb877a26450bb3aa, 0xdb4fef64835a5568, 0x1fbbcd70abef7f89, 0xda260446f2420d9d, - 0x4cae9ce772878a80, 0xff2cf3871b4be1d7, 0xa8d4eb3462ee279c, 0x467b706c5974af82, - 0x706555a72cc72f41, 0xcc1930f215f68005, 0x24ed7f635128e3a1, 0x68fd517d0385592f, - 0x6747f565375a2bd5, 0x69298f19e725c64d, 0x3cbb04d5e08cff53, 0x22794035b9e2c5ee, - 0x83cc3a05cda514f0, 0xaee0aa787caa608e, 0xb28b1442d53f59ad, 0xa86e1e51c1d9931e, - 0x795ad288689053e6, 0x31a8b13cad9577c7, 0x1c65400b1135ebb8, 0x5775ad86bbae2ea0, - 0x0584e703f6601337, 0x5a3518e6aa7b4c53, 0x8be31e21ce178141, 0xa67123ae3e53bd62, - 0x376ebedd521ffc5f, 0x6126896b4462aca4, 0xe09fb6a4da052f9c, 0xdada8122c9f201db, - 0x2cb076048551c57d, 0x7d082d365f84ddf6, 0x5add57c23da58b88, 0x0486d2a14560c05b, - 0xd4bebdac1df9b632, 0xefa448508476a8a7, 0xc011ca1dab569b7b, 0x2cfee31c3a41b44b, - 0x602767278f18adb6, 0x09e29c4fab06f23d, 0x1337024e5bf88022, 0xe21c1d17fb2f69f5, - 0x4f73f87e46b83b48, 0xcce3116630c17144, 0x419ac4e68cdad0ef, 0x6277222fdbb2cc81, - 0x21df68ad27035d3f, 0x6988a930239217eb, 0xb5327c50b965b9cb, 0xda106556517d8071, - 0xf221356608e871c5, 0x808265814ca8c000, 0x694921435e917dcd, 0x736ffc7b8d5290cf, - 0xdc222713ae236980, 0x873ae7df3e82c147, 0x952ecddfe474373c, 0x7f359e5b3d37ba48, - 0x97aae065f981e945, 0x9838f7271e38d48b, 0xfa6cb6b6775a2290, 0x34838de76b7be32d, - 0xb66bc528052eb0cc, 0x21328da6a3095607, 0xaad40ee65a661eb5, 0xc993d06eabd017e8, - 0xb63bb345c9d42405, 0xd7bce8254f75985b, 0x01929ccb2840f45a, 0xcf6b5d01a952ce02, - 0xd27ccae375229c65, 0x05a329ddf3a4f801, 0x399d2c0511e26eec, 0xbf0c6ed6ef76c219, - 0xc73c191c0810a910, 0x9aeea91be26c912d, 0x53566c9f2e26508d, 0xbfc9795a52f24a43, - 0x2a223429d28f207d, 0x10c75713c533c581, 0xd957dcf08527955c, 0xfda8a222d361eaf2, - 0x8866391cd948dd4c, 0xb032a7be57b7bf02, 0x729c073e996a61e6, 0xdc59b997046858cd, - 0xe9f91435e50eae92, 0x1778f3ddb80fdaea, 0xab346f663ef8e4cf, 0x9b6f5e5a3b02dc75, - 0x758a0a7ead8ed235, 0x2db90d15c3188e72, 0x50f39b5604ee576a, 0x65716847b8ad6629, - 0xe2aadc7947dfa9a8, 0x87fb38b0108ecb18, 0x0fa90a6125d31363, 0x12976fc4b5de242c, - 0x7e85f33aa17519ee, 0x65c91f8f42b5262c, 0x4a593245a36ad9d7, 0xfeaa9c71cdcf4442, - 0x20adcee064915a4f, 0x6b90a0c5bb62799a, 0x3a845f65e2af0a01, 0x2ddb7957fe6bc200, - 0x1601822c27493a1d, 0x94d23a0648d48a1f, 0xa6dcba0e0fed7693, 0xcb901cbd11e22bd6, - 0x2618b164f2762955, 0xf1fe2f884ce2d364, 0x01793b19600d5a3d, 0x0bc26b926b33ee75, - 0x4b7425346a7e533b, 0x05a4ac22ff302965, 0xae95661ff258f2d8, 0xb61edb9f9710343b, - 0x5da5d7da84112530, 0x2678f20158d5278c, 0x8cbfa163d3b22b8b, 0xd864ab3a2657f88b, - 0xf6b1cf066a96103a, 0x6cd75dc41f6c2f39, 0x3376ba9207423d32, 0x434f12d36a7153ad, - 0xad106df268673835, 0x880233bfd965a543, 0x6b1b21ce97a5de03, 0x61ad8ad7d6ac367c, - 0x88ce317dd953a182, 0x4a4dd162b6f0f5f7, 0xa82d7036142ec365, 0xaa72a4b4f1034a0a, - 0xdbb642fb44debfca, 0x4e8004f207df3da1, 0x3181642a5b29905f, 0x1481e9ba1981eba6, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x07c355675c42fab2, 0x415bc04c0bffe00d, 0xf2f7b28bba0e588c, 0xa78eafea783a3766, - 0x7ba2defd1316e511, 0xcb726b9ceda99eae, 0x35adac35c3c8baf7, 0x9a444260de1e5c0c, - 0x38a5261df7dd51dd, 0x1818f3a3a5a448c3, 0x4a9589257431c6a7, 0x8f6bb5274656878f, - 0xf952d8f932b512c0, 0x9712b336a52eaf07, 0x4d77d00a8060c5c0, 0x8ee0313f86442c4f, - 0xc650ab2105e8ff50, 0x0e39578ea735a125, 0x40fb887db6cf0036, 0x996678697a2c908b, - 0x6fa4f7d521bbf86f, 0x2bb7a675a06ce37c, 0x279b005eee88862c, 0xe468eb075c2ade64, - 0xbfbe3e333acdc97c, 0xb17f3a57be9eab3c, 0x4a322ff5c8867d10, 0x5b0fc19feca71c7b, - 0xf3f5255c51834388, 0x583cfa4a30aeeb4e, 0x81a3f1735b37fd10, 0xebc9e1c5d9a52aa5, - 0x4177bd7ef2e19c29, 0x850b0b3d28278698, 0x515aba5cd0ca9cc4, 0x1b638370496e6fcf, - 0xe7ddf6b5767c33f8, 0x9ebe1ed303f766be, 0x3561dc72a9d2d615, 0x06300d506939c4ed, - 0xc64799108eeb0a28, 0xf0bdc640aa68555f, 0x55b09b540757b2e0, 0xc9f54ca0479fae68, - 0xb0c8ebe4fb2a196c, 0x595fc1120f48328d, 0xbaf7a7ac23cc683a, 0x2ce08eadd56cdedf, - 0x3fc04d1823b876bc, 0x1b1eabd616f7e531, 0xc344ac3cc977e397, 0xfd2e2a4226db234b, - 0x5efac9a8f35fd1f5, 0xe4e2adcacc5c23c2, 0x8f6bcc33172f3e8c, 0xe97d2c45de43d2bf, - 0x61272acbd27b0d63, 0x1d99ad61cacd114c, 0x2f8c6aa1b4263b79, 0xf38173e91d8a52ef, - 0x2b52a08292c6d063, 0xde6cbb15890a6f49, 0xc29e12089f6d3e89, 0x11017bb09b5b3d0a, - 0x604cf274c0dabee2, 0x06281cf056306d71, 0xeb25683416837013, 0x3efdf365065781ed, - 0x4c048b86eebb7307, 0x22ebce902c95ca77, 0x67ff2beff123389f, 0x8e00d97138303f49, - 0x0f9a3971fedf4eff, 0x5cf538931ee9c824, 0x702c8ee3d3692d45, 0xbaf431d3a40e2e35, - 0x1873d77c3cf9da04, 0x804436aaf411acca, 0x78df47745a9c94b4, 0x39193503c95b65ca, - 0x95b5789d6cd3be5e, 0xf26388cf4e58bdf5, 0xcbecbed37ae91a12, 0x15a71279c58642e9, - 0x4ea9f6d4f035b778, 0xfdb71a80117f85cb, 0xa635e2109693e2df, 0x73cc95e6727b8aa0, - 0x93f49bf32e21559d, 0x5f8cbb71816ef6fc, 0xf72717722c62c102, 0xe6d07262e61d08c9, - 0xb2f8025846140897, 0x104d26c40529e9e5, 0x4427cfdc868d1f59, 0xf2f4c776091471e9, - 0xc8ba33a5434fbec6, 0x70d05bea9d943e3f, 0x41f3cd3a4e611642, 0x1be5336513a31c5a, - 0x587c988c859bd018, 0xac8be99c8816d54f, 0x0782f11bdfc8b61b, 0x43e37966f62fe707, - 0xa459ef2ca99eed37, 0x120950a1cd6c4082, 0x0eba43f8769b9b24, 0x093f5b9728079866, - 0x10a31805eb9fe4a3, 0xf5623fda615becba, 0xc969827680a0de5a, 0xe48c5f6bbe8f79ac, - 0xdf7b7603f7c879df, 0xeba5957195a1bce1, 0x2c84c1a809b35a3b, 0xeb743509c03e020e, - 0x82ca882448ba3758, 0x1a701fd756b5bd49, 0x14ebc45f46e2c6db, 0x5fa632b6c7d25918, - 0x6e8d3647a2c533aa, 0xae185268d8a783b8, 0x2b9d21af357aba26, 0xad6761d4d9acdd45, - 0x681572a35d43f08e, 0x7f104649952dce4d, 0xfe03cd3122222823, 0x5e33a7308c03cc0d, - 0xbb9e533bb63410e0, 0xa9f50ed80534405f, 0xc22cccf7ca434395, 0xa63ac45cff253864, - 0xe22fb5ecbb86bc4b, 0x6e8e1c3dea533654, 0xed9ad8f832f74468, 0xe87489c372c1dbc4, - 0x4a774799fdd71a47, 0x4a9a13c3be7a52ff, 0x3578a08d273a3195, 0xb509aa247289a1e9, - 0x2eb496c15e4bb70f, 0xcb97c42bab1f9b4b, 0x313389d7379df07c, 0xe32fe10a752996e0, - 0x833ba3b3d215c15d, 0x6da9d57dd0beb94d, 0x082d2acd84f2987b, 0x1907967226330448, - 0x9f272ccaabfc6431, 0xa88455770988b988, 0x407d6756fea62d99, 0xe82402b6a1551a72, - 0xf987e9c446fbddc6, 0xd96fd5b7e14fdbbd, 0x628bd803224c347c, 0x5be59b172a33656f, - 0x65a156ea78a0e57f, 0x0b34a6c35f1245c3, 0x51a6eeb210f30699, 0x119237f79bb77234, - 0x813b118050376d79, 0x7009ef7af96c4278, 0x5e95553aa30072b6, 0x5a22ce9d81f0f0da, - 0xe77107911afc19c2, 0xd895c84226884c85, 0x0b745d9e0d33a9c4, 0x52fd08ce7d29172c, - 0xa977d0b453f25569, 0x7d0460ad054ac996, 0x2665cadf2721248d, 0x1d8f1fc28b8ebbff, - 0x7da1692e579f1090, 0xe51a12e6cf183f68, 0x8d94b629e0a72b87, 0x9420cacdfead4271, - 0xbabbfca149408490, 0xf5af3b29ec2c4866, 0x8aa6a38bd6d08567, 0x29978736d48e5a59, - 0xc2e91e3517abe42d, 0x864a2302b2014b97, 0x37adb4b01e8b817b, 0xf05a507048d54105, - 0xdf93cc4136da60ad, 0x5321fadac1ad528d, 0x9d71bcbee0ba3d3f, 0x85e85fc20b17181c, - 0x5d2e8eadffa6224d, 0x924e702b4512cc6e, 0x7e4a8fa73b802773, 0xa63913abf4347ff4, - 0xdba52e31e991a09a, 0x712534969197a3b0, 0x5e96017b6a1cbff0, 0x17169280464d4319, - 0x9e48649cdad72f79, 0xc534dcc994d27ba4, 0x854220b067052144, 0x4dfed5000f59165c, - 0x1e8d9c98498f66e3, 0xad9baca8b5bc557c, 0xfcce4f3193440dca, 0x2e9e46de575999d2, - 0x3380e58860c7b1b1, 0xb01093059fd45b69, 0x0c83a1431da204f5, 0x2bf91e140d67f500, - 0xa1ec5dc3e03dc8a5, 0x6484df47f8934bb2, 0xe81baae6fe3f74d0, 0x4d1227530181d2c9, - 0xa856048776c54f39, 0x294025f08c6405b0, 0x02b4099f219e0d5a, 0x40f5655c31c1cda9, - 0x4455445d0f00ccfe, 0x0d6bfc09b49bb1c9, 0xdbf6a5e31fb5e260, 0x2222595835789ecc, - 0x154c92c18cda4e3f, 0x93565048d1be433b, 0xa3f528b7594266d3, 0xeeb6901a479c00f4, - 0xd51f1644b1072c9c, 0xda3b674a45728d9a, 0x89a0e380905e1d54, 0x860142b2ca1b057b, - 0x20d308af97b4a6c4, 0xb9599a5cca0fa37e, 0xf5b304089d405a43, 0xfdbcfbb63667f524, - 0x650cf5ca5d47d37b, 0x2bc4d7e87e3b5f69, 0xc86eecffbb24304b, 0xf2f07e4f74f0a46b, - 0x537538d1aa74fb91, 0xea958783d52a3df8, 0x324b85d9e5cdfa5d, 0x3f773ac0b8d8ba49, - 0x7efbe8ab3de15e88, 0x65db5222f09689ee, 0xcf2320b4c2eeb19a, 0x7b2bd444be20a1e5, - 0x360a2d43a7fa82fb, 0x88b70933b5c438a5, 0xf880e93ed7229a6f, 0xca8578d2197753f8, - 0x4cab1d53a14dfe2a, 0xca10d5abf29d5576, 0x169782b524220f9c, 0x36f84412c14d72c3, - 0x1d7d56517cf7efa3, 0x9643ee22e4edfd1b, 0x10f770e4ff7973be, 0x2a4501b56d1d597e, - 0x0ce73a61093f13c3, 0x288ba7d721a24b07, 0x41bbfc11d40f7e9c, 0x767ff8cf99d9ab7d, - 0x927b5382cd3e1f3f, 0x78edd79277cedf71, 0x9b6c1629bafcb03c, 0xf634e4da4f05d22c, - 0xffaea0499c12af43, 0x8e2bf3b1e8ca23fc, 0x805ede7228784585, 0x91fe25c2b449ef93, - 0x8248706ab8bcab70, 0x141b8580d5762e9b, 0x4daa29811599ff3a, 0x944e016838872565, - 0x6991a864ba8ae4c5, 0x5d32219ca425869e, 0xe684e448d7df0754, 0xef82d2c5363fd631, - 0xff3fad20c90f53eb, 0x08784eb4bf18ad3b, 0xc2047f72f07c06ea, 0xb52b33735ac55c86, - 0xf0537573a3658463, 0x894fd435d8d3d7dc, 0xa2c156b06d940a9f, 0x33235d40d70ab16b, - 0xebc9cdbf4f195745, 0x5ba562c8f83a75b1, 0x1f93fae34bcd7e82, 0x121900cb80b921d7, - 0x7b136207c520a6e5, 0xf960a391e69e634f, 0x13756b54a9c56361, 0xe21aa9129402a215, - 0xd8f6ead1e68763d5, 0x032a7092fbf14856, 0xf1d731f09aed09dc, 0x82c539df5e1925ae, - 0x15939949fc1475d9, 0x54c0a9f1810e85cd, 0xb890c5fa9a602565, 0x606213b450b25b39, - 0x4ceab2aef2643062, 0xfe062d5a55e42963, 0x653b7a7342a33dc0, 0xc38f547585302db7, - 0x1f3c1f71b5716432, 0xe4239ddd68bbb00b, 0x0bb2b79dd4250bb8, 0xa31aacfcecdfcfab, - 0x4631b3ebd8d35ff4, 0x30c023048a8f1058, 0x8e6df92e790bceae, 0x3885e3c85d036784, - 0xc63e3963f96538ac, 0xe454febab149738e, 0x0764ad5867cbc61d, 0x280218082feefab8, - 0xa6c1fc96a5f5739b, 0x9fa17ba32f881eba, 0x5b81234f418dac84, 0x81663bca6cdbb158, - 0xfa59c5e3b61677e3, 0x86a0ee54c5a235e3, 0x94c6422c035598be, 0x6578f84714001e15, - 0xcaba42b552a144e7, 0xb5cf4ce124a64262, 0x2d87eb1356597a10, 0x27773e324db78105, - 0x36ea6baa0ff9c0ca, 0xcd2f5eff3a43385f, 0x91f5420a3717f1e0, 0x4fe3e5b871ef8542, - 0x7e0e9268b7498de6, 0x371ea7ed24463007, 0x504919ec70981d9a, 0x62c6a4bd16fe59ee, - 0xc646ea6aa3a2b03d, 0x5a077f2f74f05a69, 0xcb33377858f403df, 0x832529aa61c5a721, - 0x90dfe467cee0fbe2, 0x20210b47144a75cc, 0x0b1ae992b2db5f6e, 0xb4b8dc39b0234849, - 0xd9a1afcad1702386, 0x0bb296ec2b07d462, 0x3d45dd88dae776c5, 0x51155c5c1fd3bf7f, - 0x44671ff960bafd57, 0xb55121f954d1ce00, 0x4389b94ff4e66537, 0xd6233fbfb29ab8f2, - 0x9f6611c511d46c71, 0x812cbb819c985f32, 0xc317d13502202202, 0xd0100d32ab2e25af, - 0x740f59f3eaee3d95, 0x0bbaf1d3c4ada2f5, 0x3fbbe3f0c3a4fee6, 0xe05d60cbfcba3d0c, - 0x7dc3646bec437eb2, 0xdb7ba9cf8e91eb36, 0x8fd9e7efb69f539d, 0xe4846b1099cdced5, - 0xc89ab850265eea44, 0x428f15e5aaa1f7fc, 0x31b09d06c49fd9ac, 0x6b9676b2fea38953, - 0x4f666f11380a5c24, 0xd5fceaf072de8a09, 0xbe30f3d6392e5e4c, 0x339bcb1751e6aaae, - 0x5a59a592827b4e5c, 0x7ed2e18ab6a9ea9d, 0xeeb6b982701abbfd, 0xaa0d5954c88d5ed1, - 0xcf66cd880d05ec75, 0x42cfd6b82a452678, 0x862b4c8cc8545049, 0xfb6b3b8bd9d06600, - 0xcf8476e93ac4c6e1, 0x6573c3d8bc5eb462, 0x121b0511c09e6b4f, 0xb8c2a8b149569f2b, - 0x054195501710b6ed, 0xc6a301d95f12d596, 0xf3647fe7fa71a154, 0x35c415615b1dc213, - 0x58dfbc6448257ece, 0xe0b0de4d149f1a31, 0x60f6eb813cf3f33e, 0xed0ab8658909395a, - 0x3f660f27fb17f9ba, 0x2f9a57b3855779f7, 0xbc7f38942791ab02, 0xa80f0fc16f1e6520, - 0x25eb3061a5e64884, 0xffe6ab993b893902, 0xf5d4c997ae375324, 0xced84c1907282924, - 0xa118f129900ecd1b, 0x713c1f936858c36b, 0x4b32be890d9f187a, 0x4c5b8d717d24e693, - 0x7466eef52da63196, 0xced99b862954b7d3, 0x341ec59fb5cde422, 0x48044f6490a76e04, - 0xb13de393d45c8f19, 0x3e301c6a5529f642, 0xb754964d9fca33b1, 0x8070d4ae753346c3, - 0xc7d132879aceaf22, 0x27bf4b619df7e7ce, 0x48afea1d5d9dbc12, 0x2ca19fc4be5f637f, - 0x0300e4b37eab631e, 0x991b33e4c712dc6e, 0x9b831a3f8b2f1d58, 0xc4f693eb59f7fae6, - 0xd0c92b5b392d6ad7, 0x5c362abf8d8f416c, 0x0a9d89f12be8cd12, 0xc649a656ca63ce25, - 0x75edc46ea6a39f7c, 0x14672a6aec4b1037, 0x8a3ffa4a87bf9299, 0x086654859e6ee318, - 0x2c60941cca72f33c, 0x97f8d4c2dd2854c9, 0xa21269d0abab0b92, 0xd3904841739f0ba7, - 0x929064dec9b9e751, 0xc5ba0b8bf665e166, 0xb3f4d04be94b1b57, 0x4bb231305763ae2e, - 0x3677f5ef041b3dd3, 0xea71091e90cae7c2, 0x7216fadcd99901ee, 0x68fb0c04b258ae44, - 0xc176deee606b1cd0, 0xf8f96992b2abe13b, 0xb3b822d0195830fc, 0xfbeeefe2f3cdac1a, - 0xf5b092b453ebefb6, 0x9142a8b4bc0600cf, 0x536d707ed12d8f64, 0xa61939cd1c67080b, - 0x174ca8a00e3dd5c2, 0x31dd2bfe5502123b, 0x41b675d73907fc2e, 0x075e60353f1683c4, - 0x7059aa41411f91f3, 0xac391c66cf314a89, 0x1ecd25bfc54b2a15, 0x527af4a4a01e89bf, - 0x6090a22818136231, 0x28db578c9d29653d, 0xc6ba5026c2217f6c, 0x7206aaf2eb528e7a, - 0x94618a7e05b9106d, 0x87cab40fcd9d5178, 0xffa1d3d7ce7f0aaf, 0xb8fa7a4c50fc65ff, - 0x3a9b129d1e3926e0, 0xea75978b12adf712, 0xc8c7339d535f0405, 0x051da5ab572acb02, - 0x6a5fe72e5fca4f9c, 0x84c8759994ec3548, 0x3282fa94e565793a, 0x894fee1c2021d2dd, - 0xc327ff53157ab5eb, 0x495ff167aaf6dffb, 0xa421fa71d4ae79be, 0xfc5a4ab4dc6a4c8d, - 0x7d4c8d1e8b49a21b, 0xf602d260536ca99f, 0x597b8cd0223af094, 0x2a2fdc6c16a115f7, - 0xbf35c485a8483955, 0x4bda25d8e3136ff6, 0xc05328b5dfeca9e4, 0xab044e5aa14b3c09, - 0xe9e1b45b7ecdf9fc, 0x13c13542b624a9ee, 0x13bbdd1d99a8c26f, 0x89ed324b5a61c4d1, - 0x84d74677b200bb7c, 0xaeb67feab084e765, 0x08bd2cff1f7aeff4, 0x1962f234ff964946, - 0x0b71598f3a352366, 0xb3d53df4c3fdde66, 0x9bf0c89627c33a6a, 0xd6254a82379ba58e, - 0x69d7d51f8ea08f3d, 0x3d025cbaecfd3dee, 0x29433c1825e798eb, 0xb78810d216912054, - 0xf49332cb438929f4, 0x1d251b8acd492d1b, 0x0a27d52c54c25411, 0x873c7f9f119945f5, - 0xb5f2901ecfd20343, 0x83735779d5bde027, 0x57ae2bd5938582c8, 0x79b28c12e4b2b528, - 0x2be25f50c0f4b7f5, 0xecfcf074008a21d3, 0x8d2acd4b31871cbb, 0x33308183ecc5e4e1, - 0xba3eb07e964ea665, 0x1b98ccc528363656, 0x167a4df346e596bd, 0x2b5eeaee31df58c3, - 0xf65ad1f5708f1d64, 0xecc45a9374e16ce8, 0x9460cce80439f790, 0xa6b99def7a0c8f76, - 0xe3ccdf10e3c8c1d0, 0x83a71537c937ffd5, 0x8dbe5b99d1b4236f, 0x447712fb9ee19fb6, - 0xa8c54d97bff6ffa7, 0x503d42ab2e813ec3, 0x98a328a75f11fff0, 0x31f4a4d2f4b51626, - 0xfa2f59621b55c727, 0x873268c77407da08, 0xbc092d739bf6717f, 0xd14a4d03eb1bed74, - 0x2fea03c691f9a08b, 0xb79b8c99f7a6a0a4, 0xea9c36cdc6ace8ec, 0x919c66534ee26c75, - 0x49cf948404c6be62, 0x0b85ad377a4e7f12, 0xbc4f637cc8789ae1, 0xd883507a53ce8007, - 0xdc203ef504371570, 0x081e600efa273f98, 0xfa9581421f24bdb8, 0x9ba6071abb4c6787, - 0x6ca17d841eee09be, 0xee16d1e5bded2363, 0xf7535a48e1772500, 0xf51845b4914d0907, - 0xde8aa3a8829c1c94, 0xff0ca8198347cd63, 0xfdadfadb575fe47a, 0xd81c1d94e54e02fb, - 0x246180c975975ff9, 0xf74a905583e268ae, 0x3f4832758a70dfde, 0x37a5b7f242248754, - 0x50f16fd99f754094, 0xb7945668450dc17b, 0xca3c56f582a97371, 0xb492de090622d8b3, - 0x7da03e49d2a58069, 0x95122e3449350cfe, 0xb10001bba01564bb, 0x74a2f09dd90228bd, - 0x44add4182cda37d7, 0xc79160f70e8f2ef8, 0x5935395691c82c0d, 0x7a196906e3cdc4f9, - 0xd609b71ce22c12d2, 0xcf80adcebe87a19a, 0xfddd329521e68110, 0xb923ff8069517f8d, - 0x45912f1d0a5d829c, 0x6171963b9088836e, 0x61802ca1a49b41d1, 0xa027581b5b6c9641, - 0xab1abb4660cdab68, 0xe69d7cff31c506bf, 0x61d4415d05f582fc, 0x87f3a6a022865735, - 0x4d0f9749157a9d02, 0xf3fda77c8e7ddd33, 0xb57deee4fec80d2a, 0x1cc39570bb3d3e8d, - 0xd319ce8f78d517bc, 0xae594278f9b320a8, 0x57b36633f1fd3898, 0x3575de7e454e8712, - 0xe9463a06e0ff1122, 0xf94ca1b51cbc3cd9, 0x5f75407a837bb827, 0x97e1bf1b54005bcb, - 0x4a9a1773dd8585f1, 0x0564290c8316c467, 0x2d18650babed7ff9, 0xd541c01078b89ab6, - 0x77ddb08ba50b3271, 0x8c56d6e4ecb2e0c2, 0x4b930463a8693929, 0xa934eb9718925005, - 0x6ff57d30224f4b07, 0x37c536ef1652eadf, 0x6061e51cac1febba, 0x0aae7b44511111bc, - 0xa9f24dda6511f77a, 0x7e19c9053bd340c9, 0x536e05193aa44d06, 0x27ecc3e976fde127, - 0x2f7fd90b91dcbcbb, 0xbbc1526489b43190, 0xa83c45ea3c86b21c, 0xd360e1924f94c12c, - 0xf501401bd420cbc3, 0x15f00e071309e990, 0x74aac7fdb0fd6aeb, 0xef958d5c7ba3ce11, - 0xc27338015b2c96db, 0xd97e6f37bc6269a4, 0x57442e207ab0d160, 0x7c199d3175e62dbf, - 0x51c9391b43434ddb, 0xc5dd3cbbf0196e96, 0x71106b773d68a77b, 0x5cd10e8f5c231da0, - 0xbeac7af35b880135, 0x9e6b8b9bff5100cd, 0x27f6acb61f6ea89e, 0x06a1f9c54b0bd35a, - 0xf388e07f2a7e5913, 0x9dca48fdc96b077b, 0x944e679752a33b14, 0xb50c3c6ba63c16d8, - 0x98db08bfdf8a6b92, 0xa782f30b5d4e2616, 0xa961adfd68ffc76e, 0x2299c3135a7e57b6, - 0xe90b4c46dba76b98, 0x29ceb64ca5a50a05, 0x37ae4d1bedbc9e12, 0x1bc940b5caa7649f, - 0x2ccd4c936be4500d, 0x8af64be5bf28cf33, 0x70ad6ac769cb27c7, 0x619f51de21c853ce, - 0xd24da5b0ca9b17e4, 0xa7ea3abe3857cbd3, 0xd8f8d06f4f3cb52c, 0x51855699896b8456, - 0x1311047fa2cc62fd, 0x53f18146434f8cd7, 0xa138488f4bf0aec6, 0xe45842a1fd77138a, - 0x9afd87e4c4593df2, 0x8d03154f03437eea, 0x91f187cd9c43b2e4, 0xf2ace27367cac282, - 0xd681710bc087aa9d, 0xfa75c5d8d8c2dbc8, 0x7a9472f0a2fb234c, 0x976bc1c19ebbb386, - 0x40643036296787d2, 0x9e9554d49965a5d7, 0x7e9c83c8e3666804, 0xee7edeabbe4e034b, - 0xc9ab6e54aa53e876, 0x38e3133387c9dca0, 0x2dbc3ea5f51b8038, 0xa85f96d5043e765b, - 0x314f4f1f438c1c1c, 0xc2814797055b0b09, 0x896f440e866bb908, 0x35c0a3d044a4b28f, - 0x1d396280104aa135, 0xaaf643afc9fe366d, 0x6789bfb61abc89cb, 0x8768c654787ac6d8, - 0x999b2f60ce2e09c9, 0xa9f3f84e9e995b3c, 0xd8cc77eb086d73bf, 0x0aa9cbc52a559884, - 0xf569e10d77358644, 0x2d01110fc1345106, 0xee49f1615281dcda, 0x484344a6c8146345, - 0x9d85f2196f0574c5, 0x1b2d5296618a6649, 0x316b3c5b14ebbfc6, 0x0a2e05b3cba4804d, - 0x427beecdbf0d6968, 0x1c7c7ad2defee37d, 0x9193c1aff049e3b6, 0x649f64f95396c262, - 0x1693d43e919f47cb, 0x6112481e72b7052d, 0x9b4c36c12aa0e0b5, 0x2602359dd8069e6a, - 0x682168faf88612ad, 0x4779ea1701a833d8, 0x404b6efce94b4ea4, 0x7868743585b323c9, - 0xf1c557b240831667, 0xa46b4ded73a9a7ed, 0xdd3b00b4a4aa5519, 0x4ee8213ff33accad, - 0xcb296ca84cd03e56, 0xb04cd70cff4d78be, 0x8f2c14ae182fb9e0, 0xa3774b41cd6c4d10, - 0x94f3a26cfb5bbf6e, 0x60dad561801714cf, 0xb37934e75c6bb046, 0x7954dbdab4f89242, - 0xcf5030e1bdb916e7, 0xff2d38a70aa40484, 0xc69afd71cbc56d80, 0x4c67b9d74d7c6e28, - 0x360ca35f01d07f77, 0x5020ecfb3a6f854c, 0x7c4b1b321901b909, 0x92d28bfe21dc2ce4, - 0x73eb188f8d116d42, 0x47862288c6c6bc80, 0x45749d12e5bd5446, 0xef0609cafe8824c4, - 0xb8dec881a5af9e0e, 0x521c4757cfea7080, 0xf01ae16535340db9, 0x7b7fb42c62c788f5, - 0x1a02db39b3f7511a, 0xb6411340eb09a82a, 0x8664ddd5b7a94275, 0xfede2776edc35e88, - 0xef6b78141132ea7d, 0x01636bb7f3148707, 0x74426c4f97b418c9, 0xf18ea401e22ffcf0, - 0xd00261d5833b3f1d, 0xd8608f49bd1fc6ea, 0xc0b5e9ab19e6111b, 0xf33ef5f1f5e79eef, - 0xf360da3b84700ab7, 0x10725acdb3b3c000, 0x23b636a137b8f954, 0x44072441cd9b8ce4, - 0x4607db3007ffc6a5, 0xa6b4959db19ae477, 0x7313ff7f35f33368, 0x1449b39ce55fffb7, - 0xea0d9ffa7c52b4eb, 0xf75aee7c389ccd62, 0xc65a816fd263d6b1, 0x3489c54fc097cdf6, - 0xeac356c3f510a73c, 0x9f11b59949d18ca0, 0x9731acf2f32e100c, 0x381a90b5895ed19a, - 0x66ffa6801f24fb14, 0xff7b2c1826805d51, 0x1d6aec61d3d1a920, 0xe8fb7010eee15dd1, - 0x51627650c042e2f7, 0x09c3a4345d399849, 0x7f7307f0c4e3b823, 0xa1f5a5e41a8f4aa1, - 0xe8a3e536aae3af83, 0xdd789ddc1df0325e, 0x1e3e6092a34f433e, 0xe8880bd3dba2c85a, - 0xf530ec1d4af3a25e, 0x77129d050c65fec5, 0x3f36b9056ac66028, 0x939e85a2ed5622a0, - 0x1885aa46e475de71, 0xd577d25b1305a0d7, 0xc153ae257863f8ad, 0x15990063a8b7f2e7, - 0xf8642dea7eac8cb7, 0xac72333d6d8a64a7, 0x9a3fbe907e1dc8b8, 0x5623fdd919d6a209, - 0x0b477ba3a4d33f12, 0x15eb8b67682dc072, 0xdf78bfb375ce6b0c, 0x157503212f873f52, - 0x1d779bae4a2e2214, 0xf5b763c3a2126cc1, 0x0db66216b3b2cc5f, 0xf5d0b11756bcf1f0, - 0xfd6d6c9f71b7fac6, 0x4d68a5d82964b29d, 0x8ab701d4200e47ca, 0xaa94b0803b3f3013, - 0xd26203a231c2081b, 0xcef69821eebb98fa, 0x097a91f788d69064, 0x2d507cb462361ce6, - 0xd6e924283fb06639, 0x8858a75c1b21cf72, 0xde9b9c572ee5c6a9, 0x83c9e7b91cbc3a21, - 0x14b9391b20e2de17, 0xd500d1af6931370c, 0x8c284559f0af8f37, 0xb6864ea0d5b6b247, - 0x1c530e657e71fcf2, 0xcfe282725f930620, 0xa17013dc143ae448, 0xb4403388fd3b5d6a, - 0xc289aee6d6d27ec6, 0x8bb9b8318166716a, 0xa0e83244ad610a88, 0xaae90e0d8f201f2c, - 0x2526df94de1b947d, 0x6bc49351adb0c090, 0x71876d378229bc05, 0x75521250085bb724, - 0x1a3f265818192b92, 0x8c2cbb8d57571867, 0x946782e2aa3977d2, 0x4bd1292a43a43e82, - 0x79b3ec406e040850, 0xf2aa3affb22f9dd7, 0x6b34d0a0b0639871, 0x2379f306750546b1, - 0xc181ab4c6a076192, 0x43bae37d638fbd74, 0xedc672fcdaae9f71, 0xf694d5b8fb58da7e, - 0x0ad18626dabf46b7, 0xddd6fea5c6cb25c8, 0x1035e0947c98aa5e, 0xea29c7bf6d145428, - 0xa3fbe7bf55c3d175, 0xe0e8f111ab5eae8d, 0x0e6ed5bfd9687fc3, 0x28cffc9882c6d13a, - 0x25f5249a723199ec, 0xd4a3ab18c9dd62b8, 0x060f1a6ef5726b18, 0x404ee876a4daf918, - 0x8f2f2e61017d731e, 0x11177f56286807a7, 0x78ea1a33e2ab2090, 0x122ff745948b4eec, - 0x6d2d191a8828436a, 0x30f793149e8f2d92, 0x2b5f88f5698e241c, 0x93dfcd50e53b36ce, - 0x0c3d9df357b83db5, 0x97b82a1889291278, 0x7ef4c1771a758f11, 0xa8c5e82c296a9b7e, - 0x7a9ee8834538ddbd, 0xa2030ce002f0aab9, 0xb6b3a5018d5a99b7, 0x483266f7561aa58d, - 0xb79c7a87fc1cb226, 0x7edb227e8a208f95, 0x2ad247cbf12442ca, 0xb2b1e890102632a9, - 0x4d73bff89d59935b, 0x995d93552678aa4e, 0xaabf1d3dce82a61f, 0x448dede27bea8ff7, - 0xd363fc77bb7f4dce, 0x7f6e4237403d3f7c, 0xed05209675517bd5, 0xc15a779dada4d7d8, - 0xe5efc52b4a405e5c, 0xc5a0204d0fe5f7e9, 0xd27c4f3c5d099030, 0x2ddc5140f77de47a, - 0x004232086e0fc7a1, 0x005fc09fb5cc6c4f, 0x4131a64934a4488e, 0x0ba0ba1836e8595e, - 0x57c824cdddb6794e, 0x04b16e94fe9aca3f, 0xe972c410cb99f56a, 0x639f43fab3280e2d, - 0xbe1abe315c8c7513, 0x13a1536d603805a8, 0x0125c01650de251a, 0x172ef571f64e8d89, - 0xe9570c6b6ad807d9, 0x54d06b31c88750f5, 0xa34efc63e3e2886f, 0x43450b8e0686af39, - 0xeaebaad8ef9ea28f, 0xfb52d7b4160ad076, 0x7d4f588481a5eff8, 0xf6509c4670b1f84c, - 0x8edc54a7a47ddda8, 0x50dac7fce0cb61c9, 0x34780bb80f0d75aa, 0xeb4e7389128aa491, - 0xe370abffeb18f495, 0x677ae6c03051c241, 0xb52b664f75400499, 0x4cc0c8aa4c3e487e, - 0xae40e7040bc031a6, 0x12fe650170e6e720, 0xbef268accf5c67ed, 0x0701f966ce8edad7, - 0xbd516c4468afac41, 0x8c5b5a82a2dab696, 0x4e0e1eb1795a8ed6, 0x78f6e3ad5a00eee0, - 0xd7cc9b7d5b212454, 0x5598df5d03824d94, 0x9b601143e93f0ebc, 0x590b504570e608b3, - 0x9dbce71fc31c6de4, 0x3bb039a859b0ef86, 0x997258878cdd6030, 0xb978eb476ce42d2e, - 0x55da05f107ef7f5a, 0x6bb0ea38b6d69b7c, 0x6fc71b51b2567bf6, 0xd377adf3b6ac828a, - 0x5ed21ccf2cf7e8a5, 0x3292f14881a89c23, 0xe97e17f63b786aea, 0x8aa80bcd041d5a5c, - 0xd1a7603b8e9fd20e, 0x601189afdd3d9fc3, 0xf250c7a7bd5ab342, 0x7c69c3f9bc4e1ced, - 0x185bc39e80a7a3e7, 0x4396ed5f0b9956cf, 0xd202722c63e08ff5, 0xe584df94ccdc38df, - 0x304251a5c1ee32eb, 0x01b17e0f300ef383, 0x274211af9c33b2a1, 0x4448914d9d6af202, - 0x21e5fb3f4535bd1c, 0x8a093a31449b3bd0, 0x148562d53b596caf, 0x36f4f185af6f5f2c, - 0xbc30ae54754991b5, 0x4d93cdd49636da1c, 0x7a5e728d74ce6ee0, 0xa02058d11467ca07, - 0x16f6127d29a3bbb2, 0xf2843f42ed6706af, 0x8d048506d42ddcb9, 0x68f022052ccb3178, - 0x3f8d96c146f5b28b, 0xc028c6aa4a8b1bb6, 0x2b3d12192a5e576a, 0x293a742544e966bf, - 0xc4e85429678c5dc0, 0xd57523a6d1b67745, 0x26bbee597832de0a, 0x56419cbd009d5d69, - 0x9389892c0f601189, 0xa5a743d56f1da381, 0xcda65e1673f41e0e, 0x1cc593bf5f0c7b17, - 0xc2fafbdb6c9eedc0, 0x3fa14081a06c6396, 0x68734c567acdc2a2, 0x171894be9eaca8f4, - 0x51b55b1b513828a3, 0x5f3421331bac299b, 0x57dcb6f1ef8c9502, 0xe351444ec3f9d1c2, - 0x8167b21ce6985bb0, 0xfe9a2550f12d3db5, 0xa641511ca07b6ffc, 0x05da8d888372e911, - 0x6d41d90394f6e962, 0xe2d097bf48b63895, 0x5cfe19c5ec930260, 0xb6ae3f36932ee5d0, - 0x7f1531ad502e2266, 0x43b790e41636e3cd, 0x7dc6e7bf9258770d, 0x63a33ee8e0aeced8, - 0xf9cc7500871fa7e8, 0x4c7497cf9f83047b, 0xd3fef7148855bcf4, 0x6eccb952443c4ef7, - 0x6ac888065d97c1c8, 0x1a89aff375ae83d0, 0x7785a23644841015, 0x037f6b63b0e6f8b5, - 0x89e4000781faff18, 0x77befe5d372c7edd, 0x1ab197860a57f695, 0x58153fa66faf5663, - 0xe951342e5b79800c, 0xfb511b946416e9a9, 0xfb0c662920dd23fd, 0x8b5a8a49a21f03a0, - 0xd6f1b879c81fbb60, 0xe8f60ead3cd62aca, 0x6e2f2bfb41305942, 0x04dd22fe89a4949a, - 0x41a847ed884b34e0, 0xfb8755857979a98c, 0x02fc30f5ea41b037, 0xba7ed475caebf079, - 0x88b2ee91dc22907c, 0x55a78fccdd43caec, 0xa120011a3eb21496, 0x986299e8ecaad90a, - 0x64ff2e577efea1ee, 0xd7e68c3cd75c6a88, 0xfed1cf367ada46d7, 0xa868bbea2ebd3211, - 0xc5795b5b7ee1a080, 0x01da333367c62f33, 0x05331456ec03e666, 0xff1c165034fd0bfb, - 0xfce9c21ebc30ed86, 0x4731569d73269820, 0x7bb981fb34e1ee6a, 0x117f2735358158bc, - 0x0f73b935bacdfabd, 0xb8642dcdc1e59341, 0x62b5a8639ebfc8ae, 0x0cf342d7239094d2, - 0xeab177cb5bed1cf6, 0x9c9c1d1a07b1cc49, 0xedb2a9ac8bcfda68, 0xdff0ccd208335312, - 0x7de6d0a9eb70571e, 0x9bfc0401809a726f, 0x856f4ca7b861f957, 0x15b2311ad9140289, - 0x81239a118404e0ce, 0x1447dcfd6f8740f8, 0xa58c9a6a6d7cd7d5, 0xd53c895eaaf0d616, - 0x8b6256a3ee46107d, 0x52fcfb6980c3563b, 0x439dc764ebd0bef7, 0x03bacc73eda935e6, - 0xbcc153c9eaadcf20, 0xa60633d86c79c051, 0x3d6c50cd8f52cfa9, 0xd146162e1bdfcc6a, - 0x03aecac15469fb11, 0xc99c26ffd9030ceb, 0x35e9edd2ecbf8c76, 0xf06cc73592bee959, - 0x8017c6f713d7111c, 0x64c9ecaabb7cba07, 0x23b1bdf4433065d1, 0x61fb4d8112e01db1, - 0x15042ffbd3c19589, 0x79f5d043303c20f9, 0x10c918feffcfb206, 0xfc9b81a82969b4fc, - 0xd4ee611f94e4ee75, 0x339373b001e745fa, 0xbccc5a2a97dd33b6, 0xe99965c23907dd33, - 0xa53ee25909211297, 0x37f074e407bd4a63, 0x60e5badcf6a0d5fc, 0x8983e358cc9d8a05, - 0x33dc1c8bec586b64, 0x6447492c217c6389, 0xed89f18255a0dafd, 0xb888dcdd08f08318, - 0x8ec6a6f67be2edea, 0x2954f3e8c14a96b9, 0xe436a39cd53deb38, 0xe1ffd6b9e4651bf9, - 0xeaac449460585ebe, 0x247aa3511ed40e30, 0xedb61d03bc48bcf5, 0x348ed81ed0a2b4af, - 0xa0d8240aba4cf8c1, 0xf2649e21195415ce, 0xd719775990ac82da, 0x6e170832d3aaa08b, - 0x55b38a0c52651c31, 0x270536c77c80f9e4, 0x817eeabc00c86ce8, 0xc1783ec713717889, - 0x4a73b469ab7e547e, 0xdaede620ff51f2fd, 0x2b935030450410c4, 0x4e63bd923e5a213c, - 0xcf43396052ada4c7, 0x65b0a601b176c838, 0xbd75ac1acdee2db6, 0x1485d84a35cc5ced, - 0x3928db777e3e2942, 0x8cea210888ee2355, 0x4a109349b31f2d5b, 0xa8c6572bd9918978, - 0xcc4dbf249e5e0dca, 0x53c20716205bcd80, 0xc4b8cc51d8e303de, 0x941ada77b253fe96, - 0xaf64e9f47f272040, 0x84b4854b134d0e26, 0x3af5f271d1204c81, 0x3788a5ade08693ce, - 0x3d4297593c2e60f7, 0xbb61f02ff444b1b0, 0x9f119bcf3e83c746, 0x281291e6860437b5, - 0xee70ac647b1ef93d, 0x7d0f85557cc68d54, 0xb784debeffe6a18f, 0x003b14e977a56c5c, - 0x3914a79eef4476fa, 0x3df92fe33f938059, 0x0e3e533988c66ca6, 0xda7982728096168b, - 0x18e67e0812dbaf42, 0x004d827b6d9079ad, 0x9e7dc26473112842, 0x46da68611570481f, - 0xe17a1c91d75b78a7, 0x0d3e0efff9035b1c, 0x4f8ef3a70a22f6c3, 0x8cecec8e729e1e20, - 0x201dbdb54a033229, 0x8c5ae49d6970f05f, 0xe5f61d0d00c27431, 0x5721952873a03b65, - 0xff563289cca9c535, 0x13e87627c43623b4, 0x6648335860854f6c, 0x7b7b3a7a5eb4ff7c, - 0xf9efc67c78eb15e6, 0x7c78ebec7c274d7b, 0xa9648c1ef64335ef, 0xa942a5681255d9f5, - 0x1805393c933ca7d6, 0xa983ece8fd1ec82a, 0xe671bd4355f35afa, 0xea0893299ac041ad, - 0x6a2715b0295f402e, 0x56487ce77b30972a, 0x4728f7ed6eba8356, 0xefef13bee61d5b7d, - 0x34d4256e67ddf960, 0x94ccaa12c3e412ec, 0x7ea32b73b566936c, 0x75dea816b4bf00f4, - 0x6bc0504a3f9f8a7d, 0x244453602feb470c, 0x83f2156980a6cca2, 0xa94773870e16bce0, - 0x2a9d2a77dd10abe9, 0x562e54279e4b3edd, 0x6354986f3249fb1e, 0xf65b8665150dab74, - 0xd7cc0465795f5f04, 0x9d23bb539bf16ea5, 0x282543591b54cc29, 0x5b897ac34ff789ae, - 0x242084c41a45765f, 0xbba90b841e9c75d7, 0x2938511d6fb2f070, 0xd89ee3e3d6ccb855, - 0xd1b4a58629cb9735, 0xc269c9a430b24104, 0xf38b4d2a49244eeb, 0xcedfeb6c3f96849b, - 0xef7f3bd1c507cad0, 0x906c8b8802f7e0da, 0xadc92bc8ebac8cc9, 0x4a4a578ad76c4d59, - 0x8c5e6028dde04dae, 0x44bb36da160466cd, 0xcdffbeaf0fa17ab0, 0x3055991b0b524b4f, - 0xb6f45fb682effc1f, 0xfccfd005ffaf49be, 0xb10d7b896fe48a3e, 0x9084ecf8fc3bd73f, - 0x1ccfb3bbab2e5503, 0x5dc751af02df5390, 0xafcddd397c49357d, 0xb0e435f148133e1b, - 0x27fde5bfe6ac821e, 0x30c858fe31232417, 0xfab29a818838f00c, 0x9092d7c52748242b, - 0x14f783eedaf79490, 0xdc8e1067d77e38a2, 0x9fe1accc6731074e, 0x42bc14b746c8b9d0, - 0x1eb0cb7b723a277d, 0x867287214641d236, 0xc2c9d17129f9be1e, 0xb802945975da314f, - 0x5f1ebfcdd121c039, 0xe557d65a0ccf5668, 0xb502e751db58e13c, 0xbc76152d59516b67, - 0x194fb2d2459ce81e, 0xed5ea24e1fd9525b, 0x936a5acbdf8c4962, 0x21dd0d1e2320236a, - 0xa85f41e52490f3ac, 0xf0807a677783e259, 0x76c7c99a9b7bf52e, 0x2edc18fbd505863e, - 0x740d89ab0d64df51, 0xb79bb8aac286f233, 0xd74c11411cdf719b, 0xdc8697ede5026c4b, - 0x609cfc280a2f376a, 0x11cb4a7d13ab29bb, 0x393534be1b3abe59, 0xddb913fccccf1788, - 0x06c30c6cb89e20b6, 0x81812aaa56ddd84f, 0xe80234684bfde488, 0xf2b0a42585e39765, - 0x61995a793132f5f2, 0x0e2d4ab5d1c9f549, 0xce90014e7a1471d1, 0x62f5ef2dfb81c050, - 0x951bfb8dcac29ef0, 0x2aa19d813bb73532, 0x495ad6addded0211, 0xd72ba2db89fa2549, - 0xcae43402853cd08f, 0x222cba94e245919f, 0xb2385e8de71a02f3, 0xa9e670056c7f89c8, - 0xd91ebe804f8a01f4, 0xcb878af7c64b5caa, 0x6d2386037453789c, 0xb73830b8e0fbf3a8, - 0xd0bf573c957440ef, 0x2a054f9b8aa14945, 0x4f54273a2b6ab238, 0x1e38b47d6aea79d9, - 0x9b65f7bcbf25a716, 0x6676703f34cbab1f, 0x49046e2562eb70dc, 0x461778471c103c11, - 0x063f6f192d03e19c, 0x135b04e2ad373563, 0x77ae6afec5a8dbf6, 0xad50801ab90ee4ab, - 0x9fb156edd25288f9, 0xb4ad3052a0cf5470, 0x0f46a9e66acce657, 0x69575566f44dfa0b, - 0x7f2504f991e00973, 0x8a58f288647f3f0f, 0x35553c4b1ba45d4e, 0x3782e5eda3fae009, - 0x09c4dd7ff3dacf87, 0xed3cecc038cb397c, 0xeb94823c2ee57f99, 0x4b14f4001b9a400a, - 0x63956ba83308573b, 0x526cb5b68d705970, 0xb453ac8d3408960f, 0x261979fe4336ae00, - 0x16bd76d36ea4b011, 0xb3271d5953206b10, 0x1857dd60de074176, 0xf4ef9963e48abd83, - 0x2e540c476bfbf6e7, 0xd7737a0ff4da2861, 0x6623e4c263ce8219, 0x3a516557e5ed80af, - 0x0737cdb03d9c7939, 0xf1ab5064cc1b6cc5, 0x91fcd4e0d815928b, 0x96efa390a5382976, - 0xd68dbfa2596186f3, 0x8e2e3c035bea839d, 0x2750f4f4a4ec9ea1, 0xa09f9b1a052e76f6, - 0x4794eedf23256de3, 0xb9e2c6739d3e92fa, 0xf40e01fba736554d, 0xe30504f4047b6636, - 0x1839d1e32f950823, 0x89b29bf43eef802e, 0x2267c949fe13fcc6, 0xca434c9dfe31372b, - 0x16b03c0ff36aff70, 0x885151031bfa6d69, 0xec3e0d9f4acc90a9, 0x579c662d9448e8a3, - 0x16c2d36034f87be4, 0x151bcd465f265da0, 0xe40d5715c435c9d6, 0xca9793aed648e2dc, - 0xe6ee0b88b9335bd7, 0xdae9d64410015f89, 0x9b4c98d2defb01ad, 0x87766dc8f34e1dc9, - 0x0983ddb12e5539a5, 0xa1733cf47b5c03dc, 0xdeb66bb4eec77478, 0xeadfb7ac78060a3a, - 0xbe17875db213cc6f, 0xb9cab450a85110b8, 0x74604bb9c75ace4c, 0x58ce786f564a618f, - 0x2ddb9396e5f30ada, 0x0efce46d9d25d804, 0x2b61552157c59077, 0xd73abc19a5fb7f95, - 0xdbdb82b1ebbb8532, 0xf17278ed8e0b877d, 0x06272e2990cb7c1e, 0xec8acd12768f7584, - 0x071ebc91a8eea51f, 0x865201127828b1ea, 0x869bb9d620026cab, 0x989135227568f878, - 0x44be2a84fd1c346f, 0x7618f7b56b55ef08, 0x23f76fc4cd2ad41b, 0x24c50176830846ce, - 0xddd7c001ba6bdb42, 0xb0a757845c71b695, 0x7eaefb0097d48d4d, 0xdf5dba8df87f6f3e, - 0xaa723d7a973286b2, 0x4c158ca43d859001, 0x3c26f5f83476118e, 0x31f8c964b088d75f, - 0x7442c1388a415599, 0xecc305bd632e4d3f, 0xc867575246a95a44, 0x257ba3cc634f646e, - 0x5fe5980ebb62bc55, 0x43d42c681d9c768b, 0x2f7bd571d25d78a6, 0x32c9cd0b55ebd7ad, - 0x0220eb9ae80b9e97, 0x8c485e5d9b3e3a1e, 0x3aae163461f6c691, 0xcd4c37da9aafbe39, - 0xb83c2eedc37e2b9d, 0xe64b70bba8027046, 0x09b1bb398716985d, 0x2035a34ccce99f5c, - 0x25ea1d19fcd7caea, 0x301a9163a4cc24d1, 0xb6bdf250e95da883, 0xf77979f601a3feab, - 0x2affc9665dbbbe13, 0xe7f2158c68550783, 0xfeff28df311c5bc2, 0xf09e9ee071bf9618, - 0xfd4b477cd7621616, 0xd9f3e0cbe4130e16, 0xae9f7f0e03ab32bd, 0xa38007b1fc1360a1, - 0xe7d223bd3dd2e9be, 0x53a260eace4cfcb0, 0xf5ff1e528c4289ad, 0x5d0c4fd16addc01d, - 0x2bcb91e78966d1e6, 0x9b0e231d41485eea, 0xf238e086c7a00e27, 0x058267116b7e80ba, - 0xf3515639aa8875f4, 0x0272477a6e97c042, 0xc064746238b98665, 0x00ab4a55a169ab2d, - 0xd8251ece9e5ad39e, 0xc5e643fc27343a8e, 0xd9283257a5ed5dc4, 0xa65a57217fb22840, - 0x9ce4ca8904dc3b40, 0xadd3d7d8c8aa1743, 0x58062575954de631, 0x50b70c7254413e08, - 0x732b3341e65cd8eb, 0xb56d0499e7e64ffa, 0xd74e127ed4f02d0d, 0x3c357838ef7eb72f, - 0x452d3e884ff9a34b, 0x3662bcce992aef63, 0x52749ec50f10a796, 0x8b1d1520d846eccb, - 0x19420772d004edcd, 0xa29a0f75e4049d48, 0xc20181d414496d75, 0xd70da1480d7b8ad2, - 0xff3f75c55c8ac488, 0x8dec7fc8efc1aaa6, 0xc89d56f0c3ea6e64, 0x9cc18c5d6d09ed70, - 0xa9c0f5e09d8aba58, 0x1c7faec11920a3e1, 0x00af9a3a778653ab, 0x887843d4c1e2f274, - 0x61107c3047018eda, 0xb3f32750a823c5a5, 0xd7efe3417c1b52af, 0xf8d4ed1f191c9726, - 0x8c6a470e877c778c, 0xe56c0ce0b746d329, 0xf24ec946e95ff2c2, 0x5affc47567a34fa8, - 0x7fc3a2d9be7f6787, 0xdfd2bf7a049a2d02, 0xa4a0663a3bd3aea7, 0x537c3282d7733f3a, - 0x5cb516a93c303db3, 0x0920de507d2ddc6e, 0xd79786e55b17ad14, 0xa4e57b0475ebe6a8, - 0x146622695e0fb756, 0x0a04b4637814a1ce, 0x129a0e718e1f44f1, 0xb88387c9d83a4a4e, - 0xa6b7f7c305d22bad, 0xc7b90d080be6b02a, 0x9cc349bc83e487c0, 0x14e9e417266bacb8, - 0xe1f23fac30799875, 0xf2bdded5177a254e, 0x5e6edef130e1b648, 0x94823736c393537d, - 0x8c6fca92016efa29, 0x43fe8deaac7dc3f3, 0xe2eb2e2e291bb81d, 0x07fa11e76711c23d, - 0x07bef68e0316143e, 0xd2b442eec1f81464, 0x9f7484990ecb4a78, 0xff8d5751fb769e17, - 0xe6495c158172e0fb, 0x765ffe80411e72bf, 0x5af9cea5ceb51228, 0xb6fa635bc1031923, - 0x5d4b0b84c2994105, 0xe0ece7e10903c9d5, 0x370b3964db811f11, 0x26a24152f26d4fe4, - 0x5e8eaeb48975de27, 0x521d43e12056e199, 0xd005e687e78b19da, 0x3ceed3fe542d2e9d, - 0x41115af35fe9b8bb, 0x071f5096e38ad3d9, 0x4a871ac86cf51d90, 0xb713cf3e9dd42bd9, - 0xf4d6a8a18630597b, 0x08333b1ab97ec1c3, 0x8d69fb82b6e4c7a9, 0xcc4c36e2a1865481, - 0x10812cec65496bb5, 0xa6500b88e2fd120c, 0x758d7c8b293b4c07, 0xae374f15a932b919, - 0x6230ad902f478a47, 0xe7325ceedc15b0ae, 0x3e29be7fcfe42bea, 0x8d5fce8ef9db0c07, - 0x196f9a2d8c50a0fd, 0x8c870532c854a494, 0x8596c8f1bfe6e075, 0x14d6d1c83e815e8e, - 0x8b06e8ac21427d8e, 0x1b0e19102d24e35b, 0x45e23b3afaf09185, 0x1e39da14e765d3a1, - 0x45b42fb7d3e81ba6, 0xb11b48aabb05f7c7, 0x73fb9a0c25534447, 0xdc3098db5549111f, - 0x22e86b8b7195350f, 0x0e54598750d5378e, 0x233023ae55d7432b, 0x5b951a96461eb6e1, - 0xfe9b9e0053a35546, 0x5d3d4fb9c5ca5e40, 0x8c65b93adcf92940, 0xac179ad45d451365, - 0x14de668d388aca85, 0xbad45d3c526c5e92, 0x42641b741231d72f, 0x46cd9cb5ce258c15, - 0x5f62e300d24f06b3, 0xd1f5de36ba211fac, 0x750a057b692bea0f, 0xef6e6aa15495f489, - 0x4b96c67d53b44e2f, 0xb08e626af7cbe840, 0xb66b3941eeca88d0, 0x48e713b7114860cf, - 0x5f44978380104f26, 0x612488f1ecbdb42d, 0x467787962994727c, 0xd97b966461428001, - 0x66a65971493e3bf0, 0xa6a09522d9711328, 0x601147e8f1dafd1e, 0x9ca063e18b54d8f5, - 0x7c2537345ede6f98, 0xc1742e0922274735, 0x005cfec0a41e7f85, 0xb7231a99b2dbee60, - 0x8ce87bbd335b5920, 0xeedbbf48d05428c1, 0x3c70f15206a20896, 0xb83fe5040eb3710a, - 0xb39c7391ca89d6a5, 0xfc49b578de256d05, 0x98e93e9aacd42850, 0x3df62a03c7db21a4, - 0x9166ea7dfa21a274, 0x1666421015c1f2b2, 0xa6dbba923a7aaa58, 0x591149e04c9267dd, - 0x6dd9458c9d42d775, 0x181645ac0ff68059, 0x0a7bfc178c68f5f8, 0xa1da2e544afe9139, - 0xc6917ff81cbb2fc8, 0x16318521af4d204f, 0x77615ac41afbb39a, 0xed7a0d43cd5c85a9, - 0x358ee0eda7e41f6c, 0x60f4b4dd83475c96, 0x14e5d47f74a6ce7c, 0x7b2250857de15670, - 0x6b5d20a1ed01fb28, 0x1526022629227e01, 0x83e90d59f7f2625f, 0x3da67a2ee911e928, - 0x21df0c044f36c320, 0xddbddf9ed2bfe0d0, 0x5134f9ccc2a89e31, 0x28ceed5009a7aaac, - 0x527869df0b1e7bad, 0x25d50d318ac2acdd, 0xde8ac7ebfb975fd5, 0x34879005326817e2, - 0x83e44d3a79091231, 0x07be1a3fe55da0af, 0xb575c32dccb444ee, 0x0501f51208bbdfab, - 0xb3468b909b29b88a, 0x699e4973a320dbbc, 0x5210b1128f7ab9b7, 0x52c8c08857dff90f, - 0x4f6d6fb47b042309, 0x7e8c9d86efd3c2a6, 0xbfb36997ecb2de53, 0x0b9fa5a495851970, - 0x4ac1f5e7c09a56d7, 0xa8ca17be5d667e85, 0x87f0799db344b884, 0x4ef0983f97bc9d22, - 0x450d10b35dabed58, 0x9e7a7012b3e13f15, 0x4e5de30bba5fe917, 0x7a782d3848d29398, - 0x6b4ac24428ee9622, 0x23738b0b1b2f3b43, 0x1499af1df676b586, 0x9482d4f67f42cd6c, - 0x87b2a3f71b679c39, 0xa0f1d1a19f0f5977, 0x5fef3176bca4a12a, 0x004f2d57152257fc, - 0xa621032aaf4a4b20, 0x05e160844e065947, 0x30b0309cfdca4d34, 0x29f0ff26d65a5c27, - 0xaa051d322bf45503, 0xe737564c7bca280d, 0x2a7b042869d82934, 0xecc188ab5fcfd093, - 0x74c144405b63cef8, 0x3fd3b7f0032d57d7, 0x01935a79173be745, 0x90b255e54065d987, - 0x7af78e14fe52d1b6, 0x9980250984f7a233, 0xa7cebe6cbd05bd70, 0xe569a780ebe227b7, - 0x0a7d065def620ba7, 0xc7a0c8ecd90c5308, 0xf6e95e3b3523203d, 0x8781c291a3d60dea, - 0x9ac50faa8f1cc56f, 0x6b17d102e2ea3987, 0x604c82db785bb002, 0xfa9f9b8ca507ed5b, - 0x65e992384db10faf, 0xa7edf4762f9647a0, 0xdcf0bb310facf5d4, 0x369418e966c2a1c4, - 0x2b1be9579df703e0, 0x49d8f9d01746dc95, 0x44b8310c23ca6c42, 0x00e5b61e395b2515, - 0x75e57fbb64029398, 0x745a56acedc8709f, 0x6c12521c7476376f, 0x5e65e4feb178c765, - 0x0c9cf19d77135dfd, 0x53cde4229c091035, 0x117fa0b28e31852b, 0xd88c8d7c6adfefbe, - 0x2839e639203c31a4, 0xa22e0e884ea9f046, 0xc84c8028254bde73, 0xf0361cc55860e83f, - 0x1670385f5e1ff11d, 0x933fa94e0bf00932, 0x512ac24448a7dd5f, 0xbc91fc4590ea02d2, - 0x2e819f916004ec7b, 0x3f8ee20d3dfab6b3, 0xee2e25f78fcc19e9, 0x9367230908c17184, - 0x0c47cc9e62def2f6, 0x3b86372b702c23a8, 0xdc875800f6e06e66, 0xac247b3cf9e93140, - 0x12fc066710b9ebc3, 0x40160c16595b0d53, 0x667a781446293639, 0x5b17a76d20a64dee, - 0xc70898985d183d18, 0x3ecc8fbe45117ba8, 0xa04ac07a05c1ac4a, 0xa1b564a3adf4522b, - 0xb40b1ae29b738574, 0x97e783ef55d058a3, 0x2208d331d934f0cd, 0xac0d9d11fc8ecb7e, - 0xe2f40ead55d78b5a, 0x777d44e79aabe146, 0xf51fb7e5e43752a5, 0xbf315e586ffe5373, - 0x7f4f5c24bb729ff0, 0xbecad2e82e507b2c, 0xaef19ea329314c66, 0x01deefae5bcf8955, - 0x46a3e69110ac7ad4, 0x8050cc46e81f6a2e, 0xe23ea3c75dc9f02c, 0xaa0e6be782eb4c4d, - 0xb7e1ad15a41ff532, 0x1abc5e87eb4e196c, 0xffdc6226210dfa5f, 0x85874e082f760f6b, - 0x25b2a77756493766, 0x4163963df3b8ec66, 0xc7724a8a13207099, 0x65b589357a4e0e92, - 0x29e25d6f5e4e5422, 0xa3e7d6ce6e961036, 0x14a11a960952d617, 0x21a19bf9242d583e, - 0x1107290466a96242, 0x58351b8c48285d49, 0x3e6d26ec3ddcd9c5, 0x29d5500236eb724a, - 0x2c6a1ea0d55143e8, 0x99e31f2e416707f0, 0xc8ebad87963d8951, 0xc4defbd455b55d99, - 0x58ce559185905e9b, 0xa9b5e1d0d25e14dd, 0x5b9da8b4baf4f25e, 0x2c103a95a72caa7d, - 0xa2843f742576dc81, 0x9949888b1b7b9502, 0x8205f8d6ca97d2a8, 0xa1869d679dce425f, - 0x5122de965a99939a, 0xc51d62f08c1f99c7, 0x07fb4ae2122e5e35, 0xd2e2a5f4c98dac0d, - 0xbe0fdd1b3a51eef9, 0x281ad85d788fa214, 0x00c0d926b3993439, 0x288a9d31d904ddb4, - 0xa6d18cb275e0ed98, 0xcce970a84b77dfa6, 0x7eaf4b92e235bef9, 0x95d428ff247c20e1, - 0xe46de494ae6c0a30, 0x382763d0efb910e5, 0x03ed5b7cca98eec6, 0xe7eb8c2f4fb28b03, - 0x99e5fcbd6614313a, 0xb2c438da1a500a8a, 0x945aa9d2fde89f2d, 0xc69e7e6f7340e236, - 0xfb3e6ea45c0712fd, 0xf8acdc63754f0e19, 0x4a6cc387115d01ac, 0x525afe02155664cf, - 0x4a55eb886dfa99e3, 0xdc003f306f7cd58d, 0x806056efe0d5d84e, 0x78342603ab82fd58, - 0x57cdafafecb37c0e, 0xcd2f05af513c2a69, 0x50bc5f2009524a66, 0x1fd8b246f1cc16ea, - 0xeea18db1902fcda2, 0x153136dd86636394, 0x7a20f66589fdc6dd, 0x9d6ce6af03186aae, - 0x81ada8a037c9895c, 0x1605ac778c98aa62, 0x2aa197bfdcb71e6d, 0xe909d6a41829bf79, - 0x51d4158e96e89936, 0xc1cc54d7fc94b4e6, 0xc1d7973f169da1ea, 0x584e3896cfbdf7b1, - 0x909fb070df873248, 0x143386edb8373213, 0xa099b7065cd0ec93, 0x734e59befa4195d7, - 0x96cbe07319a1e487, 0xd4f03d2baa527787, 0x2fe65067334e633b, 0xe9f95ceda80b16ea, - 0x08335c5e2d345850, 0x2e2ea894972c4734, 0x97f328f728b15d61, 0x573814bdb4f47c7b, - 0xbc9514f9b3748a0b, 0xc71180ae7e2d0201, 0x66dbcfe583fbac6d, 0xe2e9877c5bb9f60f, - 0x9fd0b2ab762e75bf, 0x8900daff050caec2, 0xb1488bca7f837720, 0x639f157b2b753839, - 0xf318234ebbadfcfc, 0xc774582fd74d8b87, 0x9606dd8f21bd4c9b, 0xb220f74a33e7f96a, - 0x3425d7212ce03db9, 0xa37ccff13458195c, 0x45a9e6b33b0c982b, 0x39c6cb90c80bfe03, - 0x6a8ef71b48fead7e, 0xdf03f474b88c143e, 0xc872829916a7805d, 0xf97a89600d5d6de7, - 0xfbf1cc411d522fd7, 0xabf421ffe7e82caa, 0x54a628ea76f82043, 0xf60a10899be8ba2f, - 0xd97b95671d91b318, 0x960acc11f7e7594f, 0x2e7caaa1429889bf, 0xb45e7efc6c724f28, - 0x77f7c37c1adc34da, 0xbc7fc73e5b0f8129, 0xfbfa375aaff5c54d, 0xaa78dcd469d7f096, - 0xc1f0cca311665cd2, 0x5df787be9b6fa3f0, 0x7ffe5573ea3354b3, 0x2025de5f6d78c68b, - 0x6305b1a7b4ec44f2, 0xb004132e71c67e6c, 0xe518abaa81beaaad, 0xa6070fa55ed3eadf, - 0xdaf3158fde77fc26, 0x17f8926dd445b308, 0xa480d86f3c4d8a1a, 0x6bb4194771bac8b1, - 0xd6b2b7bfb5fc35d2, 0x7fc81d56ced90f06, 0x75a51c64cf9b73e7, 0x673cf1c0e8ee6cde, - 0x0f677310aa4ccf9f, 0xc5469182718d73cb, 0x89a839f82233b598, 0x35b3053a9220a71c, - 0x60d881d5862fcc41, 0x980459f9503d88bc, 0x70e3976fcf6953b1, 0xfc05ae6b5594e694, - 0x48c01ca19fb825de, 0x3ec01f416081aad7, 0xd00145f3ee25ddcc, 0x22e663e4f14d0614, - 0xd1a209f7548e3b9f, 0x7b6cf86d91d9f466, 0x6402f01239accead, 0x90a7943722ce0d12, - 0x6c82f1a848351979, 0xb8a57d4c4fa5f197, 0x3947f05699d72d31, 0x3f4445c7376fc84b, - 0x5ed9fb236598b481, 0x9fcb6aa2208f7e28, 0x398dfb5fdd9bb341, 0x481c2f7617e5de59, - 0x4d644b2e8fd08dda, 0xfcea7a4300d36d87, 0x2a15a5500a8efc9e, 0xba321aaa7ffc4e78, - 0x04957eed8ec2c4b7, 0xe3ed4cc85a3d0a8f, 0xd77aee27926dc584, 0xbb4866537249bafb, - 0xd98f245cac9fbee3, 0x0ed33c207467ca2a, 0xae16c6687065de0a, 0xa86a12759912f08d, - 0xbec866c40e85e90f, 0x843ef7670483e6b0, 0x888bba618169c20d, 0x4101782db5df959e, - 0x05c6c544edb94a92, 0x7f717367731b0b7c, 0x7ddec76a6bf58d9d, 0x1390e7b61529901f, - 0xbcb1483f45d321a4, 0x1fab10cfb63ad2d8, 0xb70b69d64b22b0a0, 0xe69d3b8c7260484b, - 0x23fb0c0d36dc5518, 0x073e444ea4fa6d6a, 0x5aa797f7e5107a3a, 0x6e5f5fc67a2db45e, - 0x43b13259dfe7d5d0, 0xc77f6da16b443f1e, 0x00399828c206510e, 0x833026886da593a8, - 0x618aa9e989feab85, 0x1ea3d63e9721d43a, 0x83ef13f5eb2be60a, 0xf0869f84266fdc7c, - 0x1893f8f2d9ecacf7, 0xb349a3d4c7f40204, 0xf93bf9000eeaedc3, 0x61cc5bd23bb95050, - 0x542b4020c4ebf637, 0x894dbbbe32669a25, 0x29820e01b364c697, 0x3316f4f27d742afc, - 0x2b0fe91dd2c0e672, 0xf7ce562bc947d1b0, 0x94e46bd523fd4009, 0xce9fdf16c0fc722c, - 0x0240d5a6c8c8ff28, 0xa6605b9d2a52b1fc, 0xe8166d6c0b7dd182, 0xa23cf03207fcfe05, - 0x9e3512cbfbba61a4, 0x75cda01631f88c8e, 0x62b2fb7b4757c759, 0xb93103fad033eb39, - 0x39993eb95f7180a8, 0xf5411d11b4796b82, 0xa2d9c3d6bbeaf179, 0x59198832c7119ba3, - 0x2b42d6c9958f7c5a, 0xe69af361fca820e1, 0x1349c9761d3e8f8d, 0x74e7a9c05e11463d, - 0x5364c96723c71c87, 0x69e0194dd87ad533, 0x5b3f2d7adc45b2ad, 0x7dc438b33736bec0, - 0x1109cd7b873c3a1f, 0x5ea30ba931addfac, 0x2c1eb11143af0dfe, 0xf251cbb5b2026337, - 0x0fc5b9a72a7dfa94, 0xd1e030ae5a9071d0, 0xe8618b7b180df05f, 0x481bef31a552e2f8, - 0xb91fea6c59a35f1e, 0xf1fea7bd86ceec41, 0xeddc293cf7da8036, 0x972baf4f04927fcc, - 0x2ba9316fa11bfec8, 0x16ae3dc34341d2db, 0xb16d3ae9308ec0b8, 0xa372c1d885794d27, - 0xe2edef4c27508276, 0xf4fee90e85332b54, 0xd9c74de27a30e4fb, 0x71086d5baa88d300, - 0x227a4b083399f4e9, 0x6e4247a273906ddc, 0xab9a34b4d5ed149e, 0x284e86e4109ca3a8, - 0xc80f836df7049941, 0x56cb47560a6f83e2, 0x54206d439288f978, 0x62e3013a8d88e852, - 0x83e8978fd18cb744, 0x09468ef6a53d4d48, 0xc12490fd3261bc8a, 0x45955f4cedc534f9, - 0x48d186fb23eaf869, 0x5a4e4c517f370885, 0x459ef9b4b991107f, 0xb9cd5e50f562cd41, - 0xbf9955f83d5dd963, 0x2d0c4cdbfd944283, 0x7156ec54e08e1f0b, 0x1adeed7022f0437d, - 0x09d8615d697481ea, 0xc74dce830a16a2a8, 0xf5aa22fbd3c0c790, 0xf0624ae5f43b46d4, - 0xf42b58991d49ff0c, 0xd10d0dcb0f60af96, 0x061d3e5ac83133b3, 0x9c89241192dc2a87, - 0xcebd76dcbcb3afab, 0x5a54b8130eddf810, 0x4521e56e405661c9, 0x52455ff32e7cf7e4, - 0xcb0397b5d97d1681, 0x4553b6496fc73265, 0x310f98d2aec3ea68, 0x9daf2d679f7c4d14, - 0x47de6520216cd7ed, 0x16014a5c9ba65d19, 0x4d9b469b61277599, 0x196e3c0b487439e2, - 0x2cd8d5c921dfe748, 0x74472d3d39305259, 0xb7b7e9bafb492747, 0xc36c5100fcd02fd4, - 0xdcdb0a964626baad, 0x70018645df67d33e, 0xecb7acdadd65fc3e, 0xdc1dd19ac8e788d4, - 0xcb92ffbbfb192096, 0xc3f7f554c98a81de, 0x0a9286e341c21b84, 0xe2bf185ab8e23b13, - 0x0c028faa07916dae, 0x87096727d60a7162, 0xda3b97663f056832, 0xe9ce804bb4ae6075, - 0x5b5d70619428f9d0, 0x22b7fd23ac8e669f, 0xf4af78c412696f97, 0xa5ea97f2db089d3a, - 0x4658769f687bbc64, 0x29cddcfd04f80325, 0x944c540e1d6e78b1, 0x52ff5c972b52a943, - 0xf574dc9ba41126a3, 0xeea83c643431b537, 0x4ecdf281ba093778, 0x0800778d80ce187b, - 0x8967242c6cf4aeae, 0x91878a12eae20c26, 0x14d3a925783d3531, 0x082349d996dbe673, - 0xf2f944e0b56e7eed, 0x76c8fad969f1fe61, 0xce353e721e3b1ac7, 0x2e5afb3083176bc0, - 0xf706dc933b8bb849, 0x14b64232408b4686, 0x0972f0dbfe82803e, 0x530ad214bb15fa15, - 0xb9bb4baecf923b36, 0x0c2308fe46a9036c, 0xd3ac369efd48ae5e, 0xbed68633a72fbba8, - 0x40ae92877a36ce59, 0x8d313c01b40681be, 0x509772c82b825ec3, 0xdbd99f91ee22cf05, - 0x007e605938700115, 0xd524799dda2a677c, 0x59f9c93e8e6c6bfe, 0x54251e197ebac959, - 0x8054f281aa5c8261, 0x45e27e209c6dfced, 0xa3a6609a5b7af6f2, 0xb9097d4608c87968, - 0x9eb6d68825ce0e95, 0x4296d7658d1c7a9f, 0x3da1d4992e00a3a6, 0x285f4d1915bf66f4, - 0xc23d4f6ec69f5df5, 0x234385526d8bdc72, 0xc31c2d6527650e8d, 0x5960f839f9f84405, - 0xd2d1d5eb6c8f280c, 0x89e22f7b7bf0553c, 0x84c846cf494c0fd7, 0x23d055d765a11439, - 0x51bfadfee6670bc4, 0x1870430adf276f27, 0x1e1e360424275c20, 0x78585da488cc69c5, - 0x138a19744eca9f2c, 0x1dc6853290195c14, 0xdab3b91e91f5a896, 0x91f357700abcb584, - 0xe5d8cbef9c8ae189, 0x9b1cb4c95b0291d1, 0x634e31d664ea312e, 0xf0d68cbfbde5e685, - 0x446d0444a77f2659, 0xd834c80f95943138, 0x293b69aa7c18dd06, 0x7d4755e805bbb65d, - 0x0345ee777481ea36, 0xdebe0673faec3473, 0xfb52393a5d82a9c8, 0xc001543558b8649a, - 0x404c4b03ad3e37fe, 0xd2635aa468d84afa, 0x43828f9d5d476802, 0xaae632eee2c42b89, - 0xb3c8165eab686ee6, 0x5b7bab41baff23c1, 0xac24fee632132dc9, 0x972fe29d964cf4e4, - 0x2b94c88fd02d4018, 0xb0bffb2f2a4871d8, 0x0e04c522699204ca, 0x9b2e7ff95ec72ae0, - 0x9cac3b91aa22b72d, 0x2fedd4142ce64a30, 0x646a47d042f20d64, 0x1e01b20f8d57a661, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xeae3d9a9d13a42ed, 0x2b2308f6484e1b38, 0x3db7b24888c21f3a, 0xb692e5b574d55da9, - 0xd186469de295e5ab, 0xdb61ac1773438e6d, 0x5a924f85544926f9, 0xa175051b0f3fb613, - 0x9bf3db7629176ddf, 0xd53e73af2990e129, 0x5caf459b22655702, 0x0b5aa6d9a4f3b961, - 0xf08bf17067ab7ab4, 0xb47d3dfc9919b007, 0xc99b1c7de482791e, 0x01c94fe169f7cd30, - 0xacd67b0589d2fd34, 0xb4301dc813ff5f1a, 0x7207cbaddacc4994, 0xe513884525ac1b73, - 0x696ec6ca650d930e, 0x9b436ba07d628807, 0x4e3037e8148b6294, 0x47d72ba44b125cbf, - 0xe12dc16b1af7b8bb, 0xe462afca53893679, 0x4bac5266256f1881, 0x4bac6898cc267ef7, - 0x9b72c54e44cbb149, 0x91118de437092612, 0xbd2bbf39973dfc2a, 0xf87a708d05995f72, - 0xd89f0aac3015f5f5, 0xe00a4e47a82f6b6b, 0xb14bd14bb72e3df7, 0x6cf0cc6d6a14770c, - 0xcfe82c695515ad85, 0x71c9c10b26bc1b80, 0xada0c8b80b53b4f3, 0x4f3df8081379190e, - 0x06e5f650cfbf0f3a, 0x8a16ca569a88066d, 0x49bce43f1ebf929f, 0x875a1cd73c3ad071, - 0xd42b163d84d8329c, 0x011e689ac046145d, 0x11c16237db99c8b6, 0x14bdaba7279e9fe2, - 0xeddedb381b7873c1, 0x0f11359176f217e1, 0xdb4673bf9c4aa5fe, 0x74a16d739ffc4ef1, - 0x2c5b0a97b9f2aeff, 0x05483fc48cea01ff, 0xdd39518fa66834b0, 0x461c1a0eec9ac2e7, - 0xcf70938cb7fbf571, 0x6582a8c5fdc7cc40, 0x238bd05f48efad95, 0xf2d46f7ff8042939, - 0x9e7d750ad7986412, 0x565d868b0c5e74c8, 0xea0a38bfcddbcf31, 0x279afecd3833af83, - 0xa57386ccfe3412e0, 0x25c81c25518c48b7, 0x8d5453c6a6838e31, 0x0a65eb95c42c541b, - 0xa646cddbdd24ff75, 0xe263d512f8c0d159, 0x78ebdd6cfa1183cd, 0xa7f9608ef943b910, - 0x9cffe26a26df1775, 0x2313d0dd0489f4fa, 0x6e8b2f82ddbef4e9, 0x9a402af8a8256e31, - 0xa089368ccd7a39ff, 0x6945ec04c76f563e, 0x53a0b4c3fd6a4502, 0xf1f26ebb6e19d374, - 0xa52dee4a0d78433b, 0xa2d844fd070cd654, 0x3ab3a6dd4ed7a208, 0xe63dad16b8fcbb50, - 0xe4bcde97a1f3809f, 0xef85e12e9a402e7d, 0x963f671b9048f1f4, 0xc81f1798f0241d04, - 0x6d92fe41489cd529, 0xf75f52b15f3a033d, 0x374e926410a7abf0, 0x9d4408a6f98a7f6a, - 0x8ebae5dead848066, 0xfe07aaa7708c090e, 0xe2495b6f3c03abf5, 0xee10845bc81aa298, - 0xf8220c0c2ad081d8, 0x935a5d993e588f84, 0x33d47f6375269fc8, 0x56e8e7c3d1973583, - 0x8b59c97bf2445a30, 0x2498790a73c714cb, 0xe805b6979bc4fa7e, 0x41675b6f9af4916a, - 0xf79b5a2e2f40973e, 0x27401fcf90693f51, 0x0ef7abc5a44e8726, 0xa15cede03ce22cef, - 0x3ac2f43afc2f1887, 0x127200bc8ba31c41, 0xdb79eda4420607f6, 0x34265f10da810040, - 0xb31f128976abfcad, 0x2222c9521df3d54f, 0xe78865cfc3187eea, 0x7fb86a301c6648b6, - 0x336982ad7caa8893, 0xa5c42f3f65f30a38, 0x71db98d293fb7af8, 0x8d61fceee901f3c4, - 0xe2635a237d4a4505, 0x03e1698bbdd19995, 0x230886c9d850bf5b, 0x990aa9e5392db7f5, - 0x096187496fa443ba, 0xc187fe4fa992ff39, 0xded600baa9ed9f2d, 0xf0e36557f8a1ef39, - 0xa02a76ac60eea7ef, 0x0bcc6a1120649168, 0x28c10de4be120dda, 0x0313dd0bc49db21e, - 0x97ab92b0e07ec222, 0x9d8cce5b8fe9d1af, 0x8a6f218ad2a2ada9, 0xe46eeba85d809220, - 0x26d5ca6b2b854b72, 0x1a4fd16127106e65, 0xdcfc8c1b0de18300, 0xe82e25a8a26335a9, - 0x9bcdc6ad20764e59, 0x9c42f8ffc11ed9ee, 0x6d551f8e4efd74cf, 0x260d04d6bfbcf372, - 0xf16941ac2e8bae8a, 0x407b50c56a914919, 0x9801c33b2311243c, 0xcf826dab0d5ec9c7, - 0x6c2d1ccc08544ed5, 0x68570a7772f71e44, 0xb3855fe0b2b6b6a4, 0x782ddbe2cb51f53f, - 0x9b1d7157876d95cb, 0x31ced7d658cc4bee, 0x456d689c02c19617, 0x6b49082608d49c84, - 0x07aa834bb6514c46, 0x3adee3467da44d26, 0xa8aad9edc660cc56, 0x8876b8fe5fdb1386, - 0xa82924bc3c0c60d3, 0x4f6d695c3a042f80, 0x0c3fce41110e0056, 0xaa59d4fed7ff3418, - 0x4a99bd5518f33374, 0x4f330208d9104e05, 0xe05c9d54056d0f6f, 0x05c3210b869c9c19, - 0x7965ec2b73839b1f, 0x14d103eac5443f16, 0x941a6455b6373b89, 0xc61382ad217c89d3, - 0x82ad6ce15649e268, 0xec13335cc9c6b073, 0x95a8c223c1506e34, 0x99565767586f4f6f, - 0x53b3dd4c689fd872, 0x3096a25afa00b8af, 0xdc5f696a982fde43, 0x96d63e497a7127c1, - 0x1f8866214bbd9e03, 0xf17c38523a33bbb7, 0x0217ac556184040f, 0x2aa040fafe0e56f3, - 0x5413cad8ef31839e, 0x09c2e5b2df55b1ae, 0x372396bb472c7fb3, 0x91c56d4c28dd8b93, - 0xa06da020ce1b2a62, 0xdcd0bf3f3c872aaf, 0xcdc5c69d5a3f245b, 0x807e57cc6a0baa2a, - 0x90734f540cdff7c6, 0xdf39c3c40a9021ef, 0x60b4056df68e6127, 0xd2af8f5df9466f34, - 0x5e5463ba568a7b26, 0x9a541f5c3856a2d6, 0x0c6f26e3634b5d2b, 0x016bff3cd0d42b67, - 0xc57a7cc17eb98dc1, 0x99b1402be7403fda, 0x5e6a81ea77908fda, 0xd37cbf986a6696b6, - 0x0020d97bf5c65da1, 0x4fcb68ad5a292b05, 0x93e8d6472f1ca47d, 0x80f0b95cb631e75d, - 0xc2a5b3b28a5a888e, 0xbce7c6173dab527e, 0x68b3244089195d59, 0x0ce3da229a8f8130, - 0x6d2f307ed2dfc908, 0xa910a03ba7d18994, 0x85f40c97ac573df7, 0x6efcbb55bd0532b8, - 0x025fcc7ff49807a7, 0xf559611d0ec62f8b, 0x4e51fb987486f350, 0xe09d1352c26b6628, - 0x8f07a7c45fd549de, 0x2e36c9029ba77658, 0xa2f2482497f85cc9, 0xf70d4d3326c4ec98, - 0x65375895b7de078b, 0x4c9147353cce21d7, 0x61154c92f4657735, 0x8fb6aaa44ee46557, - 0x1045514a0a3855d7, 0x114d6ee9459a439f, 0x8af859115a8baf60, 0xf4e3d22e7e82994c, - 0xd8cf659bd03b99d5, 0xe762db71cceeb428, 0x1d0a34978cd8d362, 0x1a8b6a23906828f6, - 0x3e9e464a8eb80412, 0xff93b3661c9ef559, 0xd13e8df64a259323, 0x9509b89936670a60, - 0x6e842d6830a603f1, 0xbeb483c1f3dd7550, 0xbbdd4370a5bafa12, 0x01aeb2583a265a0b, - 0xba32891ca0f44f1c, 0x7227649aa72e811f, 0x42ed66730caddb77, 0xa745d2d46bbc0100, - 0x0d4b580ab4e70a73, 0xa14ab011635d537e, 0x6bb951e294a1688d, 0x5666eced4fd6e84f, - 0x50f060247550750c, 0x3184af8c1a9101d3, 0xb2c3fee458d21bf1, 0xd0d0a4af3e854474, - 0xb3aa46f99b617e8b, 0x34a16f76da10c772, 0x5f2ed357c8c70597, 0x0ff5f4444cc3b8fa, - 0x597391b42064c823, 0x2b82bab3f34276f7, 0xc4cd984ae478b494, 0x5c2b940d90bdde7c, - 0xc1dd881bdae2ce9b, 0x031ab1faae316631, 0xe5dc5f10ba1fd7df, 0xa282b69a9cc7e751, - 0x2fb96a738e878366, 0x49abc5d861c71a01, 0x2267bd7e7bbb082b, 0x6b57cfb38727382d, - 0x06caa4e0ce59c049, 0x80eb0d7862b82618, 0x6f1418e0f0034c16, 0xf2d585ee3129da11, - 0xf3a5a16a7a34e29b, 0xfa60ade62b7ebfef, 0x4fc35ca6ed5da985, 0x84ecb08dbb6bc942, - 0xd4191835a70a0cac, 0x1fe50ffdbabfa1af, 0xc7086657edb27d4e, 0xba1186b202677d23, - 0xfcd58c83cdf9573b, 0x796e556329970b59, 0xdf80434a01cdd77c, 0xb9031b96c694b3ea, - 0x10a067a88c56d56a, 0xeb69ac06ddb99c0b, 0xe2be9bfd03af657c, 0x33d01126bc1d6660, - 0x763e22d835975904, 0x678e4b846e6c3988, 0xd0cda63cc6d8db9d, 0xded91ce43c156624, - 0x00b62ae332763d12, 0xab251426c206ddf2, 0x967fc5fda820704b, 0x8583a2c9801f6193, - 0x3fd1489356c21931, 0x1923bb8e361ce9c2, 0x716f1085fe945670, 0xc76e74fd2f96230c, - 0x09066f90c1dd95e8, 0x12ed2e9edb82365c, 0xbaf6731e65f8b612, 0xa0ab605dd3b94ddf, - 0x9e6fcaa6ff165a55, 0x1d443b13fc522cdb, 0x2c2f4b6e9acbe026, 0x7a32f7eb753a5db4, - 0x30d33ce171f75695, 0xa82ef196ebbdf5e7, 0xeecd1be1e9868e60, 0xb3613db04b7c1274, - 0xb08bdbb15b5fab03, 0x766d5986cf0a63bc, 0x145bcc35160ec214, 0xc4dc582870a4865c, - 0x5b66e1ef3cc886f5, 0xca5861514a604952, 0xace6e4548eed8bb9, 0xe7958d13b21dc68e, - 0x37d29685e51062d3, 0xc1d318c6b7fdc6a9, 0xff474de9c462952d, 0xf10977a3ca4c8312, - 0x22a5730981bd9143, 0x20f3fb8d30921bf6, 0x66e3eba30b3187e3, 0x77a1531d0309fb68, - 0xc4e048b2171cf70b, 0x1d55fd8b97364bfe, 0x24913f443d2e59ef, 0x4ebb1a9f77b1b37a, - 0x9f009cd135eb62fe, 0x1ab9f9f9ad320e42, 0x38a953e6dc64b967, 0x60329de5f939ab28, - 0x221ef2c26e0c75c1, 0x2efa1c4f1669dd05, 0xe5297b4e71dd881b, 0x034d751855b27e72, - 0x480906aefd404ee4, 0x2b12e6b5d68f1f36, 0xbee7ac46f05bef43, 0xe4fba571136f8e16, - 0xdb6f6845c99c08de, 0x91a729b2702e8d87, 0xbe688461ffa7eb26, 0x4bdd07f6120a7b54, - 0x8de878a13c97c9d0, 0x03891c4d94872229, 0x11a1e8a0ca1c6eea, 0x58a4d1daa305d656, - 0xd9db45c25fbf0309, 0xe09e528aceec2900, 0x7049affee8ea5660, 0x66e4e8c2cae2e4ec, - 0xa8e24fcfe8d53fa3, 0x4fbe6c8dae1409d4, 0x5018e9c694839ac3, 0x562a7de68c7e1881, - 0x8d658e13ec4022a1, 0xacfc2409402cb30c, 0x8c4c80c111d8de8b, 0x8ad6a83dd6666736, - 0x198cf279ee2bce52, 0x925577459acf31ef, 0xf5a74b800e44c362, 0x203c37dabc0200e5, - 0x5c8ac324ce758ce8, 0x5f64ea62365bef84, 0x138af91003604522, 0x6a0669fa247ac124, - 0x723680ff681991aa, 0x2ae1248a776631a2, 0x30c2f7c42054d7fe, 0x440d965883444c82, - 0xca5a12119d809509, 0x5b96233a88e92e32, 0x3b3f33f5feafc4b1, 0x688d85b6162d56d4, - 0xde618b818073a554, 0x7f0cb30526e29844, 0x2fbd674d4a49403e, 0x8feecfaf7144793a, - 0x60ea472f6d160ab0, 0x701cf35ce48e4da3, 0x6aeae11045b054a4, 0xfbb8c92cd13c3aa1, - 0x4964a58ec91ad855, 0x16cebcab10c4b17b, 0x59c0128319181356, 0x7a97da113c44287a, - 0x050ef79a1a4eebe8, 0xfb7bd4b46ea7f75b, 0x4b517f6ac0ef31f8, 0xf8fcdd1d6cc9556d, - 0x32c04cb56a63abfb, 0xab96665ca411f943, 0x10250247d9332c0c, 0xcb4f0e321c1991f6, - 0x299776d9b4558a60, 0x8e011531a9cf1b9e, 0x33165eff1c645e3a, 0xbf5b572eb948a73b, - 0xa37ebfb891e58dab, 0x1d7bc09268e61cf1, 0x25d8508f797111d1, 0xaba25583aeb7de80, - 0xba9eaf82bf936899, 0x70f784a019f7d817, 0x920ac3630d8b4054, 0xf1a5d10321cca779, - 0x705c2c9025f20c7e, 0x41f7357789c4a593, 0xda64026442efe7e2, 0x844818444ec5077a, - 0xecda697b4b69df11, 0xcd0c0aad893236cd, 0xd4566c3fc5e7de22, 0xb81fee0c0cca4b55, - 0xc6f7aec14085ae58, 0xb4d879f05d4d6b4a, 0xe41bc026babae7df, 0xf0f13a0e01d1214e, - 0x9696d69b0427d127, 0x28b3b5047ba10de2, 0x38631ba6340ef020, 0xca4882ec5bd7603e, - 0x94cc68842fad1e41, 0x366bbd28a7acfdcd, 0xf28d3630cb87ae64, 0x02c2252b6eaef3a2, - 0xaa72f37f327bbef6, 0x191cf174404ee034, 0x99bfc0d4584d5fb9, 0x6fc646b4e6b6fc4e, - 0x22d6f8c4a73e92ae, 0x9ad5cda5d4804d87, 0xadda180b67ed9e35, 0x91272b4ae9a3d440, - 0x57f857b695b82ecf, 0xdabc49b2eb11692a, 0x71373b08c9ffeab6, 0x7c563b6055379455, - 0xb2fc097035ff7434, 0xbb7722ef9d68f875, 0xde1e852915efb28e, 0x48483957299d7659, - 0x51140a40259e55ba, 0x70f30a335279e6b1, 0x08c5f25a35c4d8b3, 0xe0018ba15f902711, - 0xf58d8ef5804a697a, 0x8024142ec7239bbe, 0xe75ef978815644ca, 0x5ba8cc73ad4ec2f5, - 0xeb6d07638fb7ac37, 0x319a996be50b108b, 0x749e64f9086afb21, 0x3074106e11f211d0, - 0x292f73cb5cc0ab0f, 0xfb353e18c4afe472, 0x0a04ef4951360f28, 0x0e208953f5402577, - 0xa8e7acfb3bcc80ad, 0xdaae04e3eab7408b, 0x05fa2bdb41a3299f, 0x4042b061e3cef2b5, - 0x1a2bbfce72308753, 0x989462e2891146c3, 0x620ff72b80078d34, 0x2cdfa69689a1d76a, - 0xcaefc67590a9fffe, 0x14e12331a0d49e4b, 0x3c504e6a928b9921, 0x3388d1f9e7a1db29, - 0x72859f2b9cc8acae, 0x368852a3899eb4b8, 0x3e0719ad45113ecc, 0xa2501a74de7a3473, - 0xa866043ab5b2face, 0x13ffac4c661b51a6, 0x68aba1110150e54f, 0x21d2a7edeccade4c, - 0x941a8a72b96414a4, 0x67c6a441d60083fe, 0x5b2238e90f1cc5c3, 0x760b0c5d2b9249b2, - 0x6076621a81c5de9d, 0x7d3c8db2b101d5ab, 0xd6329e970cea8175, 0xcc49fbf5caa71e18, - 0x781867d35c0bcb98, 0x6375fcfd1fd7cc35, 0x6a7e977ad254f10f, 0xd8fad55bc15f9a08, - 0x8ac95c67078e210e, 0x5792f1a6641e8bdb, 0xd99c4053f6327719, 0xdf23a0bfdd6c8a0e, - 0xb9a6661c005d0101, 0xa60839eafc06591a, 0xafae4c2f63014dff, 0x74e0001d267d05e1, - 0x0c735e460f029981, 0x64de0be874486089, 0xe8bfe428f59e4242, 0x8c40c6a0a7b44cd4, - 0x6f18632171d561fd, 0xd97719a394d22976, 0x2909c39b175f7321, 0x914f25bb8c9cff2c, - 0x82157fd5aa163bdd, 0xd1540ab6edbba78e, 0x415dec6cb0e65326, 0xa8455758d78f7301, - 0xbe92345f14868d3a, 0xe816a57c82535d16, 0xd395966f9dff8e3a, 0xe87578c981d74a43, - 0x09ebc55a32bbae94, 0x91912be6836ecd60, 0x34d326b51e42b410, 0xba7c8dcbd4b80b74, - 0xada3e066764e0258, 0xd17510cd254b4580, 0x6e66b73aac68fc98, 0xaa0a5e539a4fd735, - 0xf96077ba310a49c0, 0x812a2b1f78cfeb48, 0x84c4cd57d90bbf5f, 0xd830d15de0e8db13, - 0xa6a69e07a3b2575a, 0x9561f41b8dc11f86, 0x9b4a5a8cda827d2a, 0x9d4d907159d3d9ad, - 0x7f21cb643597c466, 0xf7d6bea2082d1405, 0xdc07986130ad13f9, 0x32ecaa8353e6e9ad, - 0x284061b34c13d1b5, 0x2077539cd76a12f0, 0x679b013539fb1893, 0xb941b7b488ef8a92, - 0x9a07a0b2ca64e77a, 0x573ff9b67c5521b8, 0x43a04fd427b48d30, 0xac2099118728e36d, - 0xcfd8d8f49b4f9f01, 0xa49f42accb65c8cd, 0x2f75c9cb9ddacd23, 0x095cadd8ed9a94ae, - 0xa0cd4ce9759c4c28, 0xfa96431b2a8d6b3b, 0x918ac0dd109e69ff, 0xca9c57aa266fd2e7, - 0x4ac8c493d767b3b4, 0x2800b4692d687c4f, 0xf0ff7c9794eac8ac, 0xbd55ecda9aa06f1c, - 0xb2a612f75b8e1970, 0xb77262feaaefe0db, 0x47e73caacb0f78ac, 0xe42434dc53e72912, - 0x6382d14927276575, 0xd1ac262861df1026, 0x1d99cc06718c3c47, 0xe717aeb79397a7c7, - 0x0d1e0e4c643c223a, 0x94d618110d1c7864, 0x64501fa2dd7cd4d4, 0xc42964df4caf4746, - 0xf458f6fc93f7e80a, 0xf13f558707d7294b, 0xe6d9a93d888d59cc, 0xf71c601dbf431d97, - 0xa11f3753622d7bef, 0xa9959d6e3203ed0e, 0x03050bd98ddd1fbf, 0x6f999826682aa7b7, - 0xe2d2120b40825036, 0xe8deaec0b325df31, 0xecb8b6a6061b290a, 0xe8aa723fe1b4c95a, - 0x54d162af219109f8, 0x6750c3517199cc41, 0x70a9ace837ddc93a, 0xb5b88206de68e5bc, - 0x0a38b0e43299957e, 0xab1c675e7446cf63, 0x6dd357907b12ce91, 0xc386f7c6ee1cf3e2, - 0x607971a75e3f8f0f, 0x3fdc591d5a1050cf, 0x16df0d0d90454012, 0xd371db3e374d3850, - 0xe8fa49a3ff272c56, 0x69e5ae3a74f13fb3, 0x0c0068a068f88280, 0x49e7101cd3ac2a7a, - 0x9b8a0b0334280280, 0x233252a76815d62d, 0xbdbddcbecf61ff23, 0x764cd6a7911c1dec, - 0x5733225cf238260d, 0x990e20ed9890a21b, 0x21c80647cd3361ce, 0x8b6880854bdd0005, - 0xd5b28b9b75fc85e8, 0x1710c3d83ba0378f, 0xb8d807694f113a0a, 0xfaad51da4a7051b1, - 0xa45db72027d53a30, 0x280e26638a233612, 0xa205b05c1f0ef9e3, 0x5bc38540b011759a, - 0x67d530d9255b5728, 0x5efd86e9048c4540, 0x8ebb9b506f0e99da, 0xb3f51265d01ff5e6, - 0x231c03e635a4e844, 0xef552f15c1eba62b, 0xe0e83d685703e62e, 0x8e5a4ee5c1593a78, - 0x4638afd3fe4e1eeb, 0xbcad9b673a7409be, 0x40eba35f305414e7, 0xd543bdc2ba151468, - 0x25a062fa1b42fad7, 0xf9b13970a7fe29e5, 0xe3efa3e7d6c8e226, 0x6bb2266735ae17c5, - 0x76743e4f11b27f3d, 0x1b5a4216dddcb631, 0xd840f4dfdffb7540, 0x4776cb97421ff2df, - 0x63dfb444db4293cd, 0x2ebddc65342b75c6, 0x8701f2d0f36178fb, 0x1c51959ecb1c5ad4, - 0x2c365c987c655b37, 0xe0f26f81ac37bdfb, 0xf2410bba716c0cde, 0xc0c8b49e466405c2, - 0x3d94e9711367f462, 0xc6fbe99363f49103, 0x8eeaeb3c5d40767c, 0xfef67a8354d69aff, - 0x830817a4fb406e08, 0x568a1dfdecbd95f3, 0x3ee240edfbef6776, 0x45bdda5512f8fc61, - 0x1fad2fd530f449af, 0x301b6791dcb2eb86, 0x478104bb20cb96d3, 0x39d20faf4b5b781d, - 0x9196558cace2739a, 0xd4812e1f6a22e6b5, 0xac22c499b6c9a8ef, 0x99e6f7c809f5f731, - 0x7afe3ad2fd697710, 0x87a708ed71cc19e8, 0x034c371483333c79, 0x49de6d901ebc9e7a, - 0x1aafcf1d0502ef55, 0x96a3e500b60d3e34, 0x26123e766cd34f41, 0x4de99692b8a81684, - 0x7998a64c87e60f74, 0x811d9139677371f7, 0x1a68a2c8f524567d, 0x8250aee9691167d2, - 0xe5f9369c26426b4b, 0x76c48f858762b114, 0x0d146641624dc64f, 0x521a913be1fc9c6e, - 0x4c8ebea64ce4955d, 0x7238f9ac21fade30, 0x4e9da14b37422e94, 0x403f7555150b459d, - 0xcbde6dbb7305cbce, 0xabaa0268da859a50, 0x99180b068ab66140, 0x91474cc85c7fc89d, - 0x66a84ab716d46041, 0xdd6d77cebd4ac464, 0xfadf0321b4b56d0c, 0x83b56ea8fc36cc0c, - 0x5cc59de34fcdd889, 0xf3280af9092a9415, 0x3dc7cb5f8e37a510, 0xf211eb93965882f1, - 0x3412b3727741a29d, 0xc3fe9f78ca6380cd, 0xf7ebb1b8eb800e4d, 0x80d65945e4fa1c4d, - 0x1df178b11b1dc04c, 0xc7aa27cd65da7f9c, 0x7e6a2845b6d84088, 0xdd8240183a74676a, - 0x8db28ba78d32b0b4, 0x0c451d8fe6b73523, 0xb34af14e8e378844, 0xe58d835f32369970, - 0x2094f6549027bf80, 0x6a03e5b67463215f, 0xf8bc1aff1afc7005, 0xd745037f94bb4001, - 0x88575f45379504c7, 0x3b25988a5b8ccb9b, 0x4069cec592d47d53, 0x2cbbf7cc8d353415, - 0x725da4fbbaf9b78f, 0x0242fd24935e9b26, 0xb40971211b5a7b92, 0x297e3bef4d242a9a, - 0x4bb4a0aaaf2857bd, 0x448c3b5030adc265, 0x2e65c5e13d40efd9, 0xd819a45a27b78abd, - 0x58d9f5a4cf282125, 0xa654c71298e046d2, 0xf4f57f544ed44706, 0xc04ae7c953e5d4c6, - 0x48d030f4a4ef0452, 0x2799919496695526, 0xc3cf18f14008c576, 0xf0e1198dc72fce38, - 0x33bbe17c1b99f529, 0xc9a7ef8efe39d005, 0x06f1a1ba2aa15538, 0xfd7a8f59241c23bf, - 0x8dd483497471a821, 0x1a04fa5eac53f608, 0xaf1c812963a96a86, 0x7984ad781f38febd, - 0x95c36feeb6e2b256, 0x52fcd56f8d7e0fdc, 0x9e4a061be97335b3, 0xea8936d1f7c18624, - 0xf66a08aa20bcedb3, 0x5073b3ae90088616, 0xc8d39abd1dcee456, 0xe2d061ddb2fb6000, - 0xadf334e15e7dd882, 0x394bfc5f6be72e4b, 0x6f37881ad32e7953, 0x90a254d5f61f8331, - 0x4478bbe1d7b61ad6, 0x63e473c4f2c7eca1, 0x0714897ff257f53d, 0xd938342812c9e8b1, - 0x139ead73be1afd06, 0x7445de40b1ab3008, 0x9ea93953b9f2b43d, 0x0e1bbd2927301eaa, - 0x51447217ae48146d, 0x5e446d4b40e7a3f1, 0x944e52bdaaee8769, 0xb7a6f1772b22ea35, - 0x33d40c6c67dc0875, 0x553b60f0f097fec9, 0x691d68d913260bdf, 0xf2e78e1e82670a11, - 0x9b367cd9553dc0e9, 0xcb2695f34fc667b7, 0x340610dcd702ef9a, 0xcb10bc9e24d7cb70, - 0x5789a111f099ca50, 0x701987998a9b2e37, 0x391b486d2ed7f931, 0x08e375982f4a7c23, - 0x7d8cc5092821fa8a, 0x10b2fdc43ce64ba0, 0xe37085b7dee94a4b, 0x15cd5cb23dcbe670, - 0xd100fb37993fe9a5, 0x22895b26d28a03a0, 0x3d13bef331a1707a, 0x5b687349fa78f208, - 0x968db6b956f8b7e2, 0xbf9f2de9428b9a52, 0x14d64e2377e6b58e, 0xdd8dc94458a21272, - 0x34a6cd9ed4e28f78, 0x75cfcf333bfb0b2a, 0x4c3d6ad0714029c7, 0x721465d30a5650ab, - 0x8b2e18e8e0a358c9, 0xc326187d48f97cc6, 0x24bb28bca196fd0f, 0x5533d96aea0aa39b, - 0xe27724e58bac553c, 0xf5820c74f70e41ec, 0xbedc74954a9f4e08, 0xabcf7be9eb0a912d, - 0xedba74e0f2e70486, 0x0dcb7e38833016f5, 0x61a3dcc8e2c38f57, 0x05359e0ddc02f380, - 0x9b510b6a935f11c2, 0x9e479f3d5363963f, 0xc135dbc98f45863b, 0x29006795f7081b74, - 0x04986d326e776522, 0x666c2b3f2d17be14, 0x02a796066aa411ba, 0x78357eef70a3154a, - 0x73e8632d15410857, 0xb3df50e681d36f62, 0xc08f06521f695206, 0x1fb5e193538dc744, - 0x31d6b5c335c447f6, 0x0963f588ca8daa48, 0x141df49b2865a791, 0xd241f40c8ff83d47, - 0xab5ee11bd28f58b7, 0x32749715a0fd116b, 0x75c1520d4422fe5e, 0x61d0a89c3e4b2956, - 0xa93a41d719f3f3ad, 0x74ac90f656868ee4, 0x39f40e285e898f38, 0xf2c95e94457de860, - 0x81e8666e83aa12cb, 0xb9a2271ba4eaab64, 0x7dfff681c9491baa, 0xa76af98b24319655, - 0x2dc0ff6fcb0ad706, 0x2f3cb9bac9166509, 0xc8877bd96d8f27e2, 0x8c7ff78e73251053, - 0xded45939cc99d790, 0x01edd811ba3ce20e, 0xf9f776f44e4a355f, 0x76f299d9251a0616, - 0x582f895c076c21a3, 0x1dcf4c3da15e10aa, 0xe3f0944511ecd362, 0x0d5b198cebd8eda7, - 0x4118122e6864e93f, 0xe6bbf73a8589df1a, 0x65dc1bc0548e9907, 0xd52817edf1de216b, - 0x6071d65140975f36, 0x7a79f7dcea535472, 0xafb749a013d9b373, 0x6af8d9d983b3028f, - 0x771ef9f7e17b9955, 0x68aedf678ecc491c, 0x9ba61910b5a45b38, 0x66ab19b350eed8a8, - 0x737bee9b61e2d823, 0x25aeb471f62d0ad9, 0x8aa871eb178f838d, 0x175c6263c5d3641e, - 0xe32714b53b2d7449, 0x2e5d0ee0485c9b67, 0xc80a759e5c221dab, 0x4d3bedb93ca4b3ba, - 0x6c3c8f60eca9b65f, 0x887a2d16f1cf4584, 0xfeefa866087fa1ac, 0x72953eb3757bd02a, - 0xd6b10935c6f1b736, 0x8d2335781bec23e4, 0x10170dcbffdb22d2, 0xb05b009a066a0cf4, - 0xf36acca25726c9fd, 0x57128b8e81753928, 0x5abf006f81e72495, 0xd9cbe1fadd3abc5f, - 0x5a91233b058aff61, 0x22eed41827ef19b9, 0x7b44461a1436f2a5, 0x85c89f30e9e75a3c, - 0xca0a7a1b405af455, 0xfc40a996bbf51c6e, 0xaf7df64ea7df2fe8, 0xc51af6ed5ea9b06b, - 0x63460d443e5a8a37, 0xae957ea45f460226, 0x686a7db719a5c7fe, 0xa60793025c3c01a6, - 0x1bf22a74882f9f3d, 0x602afc7610b8183b, 0x2bcad105e11c1e9c, 0x1763856c20844186, - 0x3256bffe3695c260, 0xcca33e4ba1e3567c, 0x54b6ef8db3b8e2a4, 0x14d0c6c042e314b3, - 0xb698be875dd42c11, 0xb6e212d089400506, 0xd021de2984d31f8b, 0x94f8c8364dbe13ea, - 0xf209cf1332e4bf80, 0x8f334b2802bf7479, 0x5fda4c370a978a8a, 0xa6c28ec6590bf7b2, - 0x093f45e90463369f, 0xd0c80f01c438179c, 0xf272eae1c920c8df, 0x1f051069628aed05, - 0x6d532ea42468c108, 0xc9f7c0b132c6e780, 0xd95637a9ca70d464, 0x6b888561ff4b4461, - 0x1a5d043244d61382, 0xfd12e59fe88a6cad, 0x9e2677d977c87f05, 0x679be9d8941e5116, - 0x334f8739748392f5, 0x7e0c2d86fef6a485, 0xe63fa47138d4e22c, 0xd9ce3c6d7f9ee577, - 0x08f6dcff59decfd1, 0x80ab2b041e1196d2, 0xc4a661c1ece4be28, 0x7e9dc93ca5a397b5, - 0x72abddfe452bc71b, 0x3193bd3dfd5e47cc, 0x9217a031f515c4c4, 0x7b13ca660df16a43, - 0xe9ad98ec1a9680b4, 0xa93cbd4d7136f523, 0x5ca2cc48894d17ad, 0x8b988d4b544a1bdc, - 0x702338808fc3806d, 0xf279ef6111567d95, 0x3053423b63fcb9aa, 0xe9fcdc59b74029fc, - 0xdc017ae6b18f15b2, 0xe7035b2187efd571, 0x4680b7071ed25ecc, 0x14edb94d391b6228, - 0xafba187b8de670c7, 0x6ab1e3851691edb2, 0xd82f0828c8d9e1bc, 0x9e90c1d129f70e9f, - 0x620c1095426d9eb7, 0xabcaff73f6fc12af, 0xf8572ed4d17343b9, 0x3e52f73530d5a11d, - 0xacd3691f4bc76a9e, 0x07a7caef9ed17213, 0xab1bb1a3c642596b, 0x0b2283d501dac666, - 0x7f688fec2165e4d7, 0xca4154a88b7d70f1, 0xac5c9ab32f46b80e, 0x22aab01de72641b0, - 0x3f44fd289f03661e, 0xddfbb3ae0b607592, 0x011b18b0798cba04, 0x2f489b6ba362c59d, - 0x21847c41e792e2ec, 0x11da7d848fcb250f, 0xddcc95ee7dc38bed, 0xe2ac1156dd72b866, - 0x726df81137f116fc, 0xe516c18bebb3e97d, 0x454ad087374cf8a9, 0xabdc72a894db3fbf, - 0xcc36391727815f12, 0x38d720511f6a53d9, 0x4b9cbf969b5518a0, 0x258dfd87d645e507, - 0x2a96eb3e3fc8cd85, 0xe731a82cf05e90b0, 0x07306f29089ff5eb, 0x489024507ff8d603, - 0xbbc45a6c935b9b52, 0xe32aa8cc3e53f09d, 0xbbb67bd44da9deee, 0x1ad13705b64c13d2, - 0x4e4d98e087d9b4cf, 0xae8b79449093e984, 0x59c4e5fa553d80bb, 0x1701bab58a7af1a4, - 0xa4061417a7c9fdfd, 0x4bd7bd7343ef8994, 0x99c4d50bb26095cf, 0x155624a78e4f3a2c, - 0x4992215f101ef1aa, 0x75ee42c932bc470a, 0x5390282afca5b442, 0xb2023c47fe233df4, - 0xd64c1ceca080ef5a, 0x3722b56dad48623f, 0xd87f2a76c439ae24, 0xd0823b29d8628920, - 0x96eb8bd6dad283a5, 0xf818072e58829930, 0xd2dfb229e9cf3ee9, 0x602471d22b439a43, - 0xe4bb37b1346dab0b, 0xcec1ecc625bcfcc0, 0x8ebb7dcae4cc9d60, 0xf6b5dc623778bb89, - 0x1e876c3341133902, 0x6ab18b75f23573f7, 0x9885a43501edb929, 0x550db363fd93b103, - 0x831940cd1697c962, 0x5bc6dfde5c3e7419, 0x26ebae1eed65c5ab, 0xbb57646a2f3a3a8e, - 0x3d7102b309e1719c, 0x2b9f3bb56b023593, 0x6cbca03822826f3d, 0x52f1535cfb177312, - 0xb986f6b15eaca9f2, 0x35a741f2bffdb5c5, 0x7fca371cab594e00, 0xcf7ee8c03c880137, - 0xed61d2c52f6a77da, 0x3050b21711c873ac, 0x7eedf740cc7853af, 0x7d387e25f9c473b6, - 0x195fe31416abcd2f, 0x4eb128074fc308d5, 0xdcaa5856b6a0db43, 0xeb764ca9a9d59ea9, - 0x1da220342d9bd36f, 0xdb93663ceb69eff3, 0x1a54646d49f1a3c6, 0x96499915ac0510b6, - 0x0c8fbc16f7fca893, 0x7fe4c77ab1625270, 0xd3e16bf639e44d4b, 0x4d53062add112a95, - 0x52891b721fce174f, 0x55c3871904a640c9, 0xd8c0cad62bd9f7cc, 0x408859c2597b2262, - 0xfc4b2adbb90ee5f6, 0x415eaacd634f85a1, 0x7133e185fe12c5bd, 0xd4154199c7402aed, - 0x472e22e366ae35c9, 0xf5f6707c24855664, 0x18209dc2dadd4a13, 0x62c3b42bacb3fab9, - 0x2f24d6b80aa57087, 0xe039e7732df76e9c, 0xf6f41cd0ce5036b8, 0x19b639668ed7d35d, - 0xdd75c423081ab649, 0x2be7d21b5d5a9b35, 0x43e3bca3f403b4a1, 0x4c6f940ade0db5d1, - 0x656206c5e5b5d89f, 0x60d2c339ce45c060, 0x27b6d6e155c5d7c5, 0xdce44e520b62e61e, - 0x143b074ccdafe17a, 0x8d436b02a9152ff9, 0xe18a593a0e66cff7, 0x8592e4df8a7b35b0, - 0xc39201f83891ec94, 0xb36af7f64363d1df, 0x9e4a541ff0298d45, 0x795e466c0882bf39, - 0x9c54741515ea8f6f, 0x4c4e3c0c4a303326, 0x6bda83f4f3e6d3cd, 0x668c565a42695f25, - 0x0ca4c56fc33ded6c, 0x2a6e963bf3062570, 0x90c1ef9d1d2be33e, 0x9093479023d1a563, - 0x37b3058ded2dab1b, 0x7a55d35491a5bc97, 0x22d086ff6e80386d, 0xff8b8dee7261f9cf, - 0x7be3abec5b3357b0, 0x7404e86b425bfb60, 0x188397c16c72b924, 0x14eb8615bde786ac, - 0xd234c2ea20481d91, 0xffb3fa8480b57e50, 0x0d9831591b83d7af, 0x38bdfbd19746ff3c, - 0xd0584ac628213de0, 0x2ff661657beb456b, 0x0653c28f2cce397a, 0x7b185b9afbd583e3, - 0xeaf4233af972f141, 0xce25850a7c60bbef, 0x6cd44edd5684f4be, 0x8f0511d59dbd2d14, - 0xd22ebba9001557db, 0xb028d524229b133b, 0x5de2a8c71e0a263e, 0xf80cc182eb5cdf6d, - 0xe39c9556cc433c5b, 0x5ec1b519805f5a54, 0xb799457309a3a5c1, 0xc1397b5c01da4187, - 0xe57a27d50b546a73, 0x5c9fed234b4bdd93, 0x6e2228bcc5a3c247, 0xfcf487edf8fa16cd, - 0xe86e88aa17957a62, 0xa9d62389ef1da2c6, 0x57d3799a8d85bd09, 0x973c185a7177626a, - 0xa542e7b1e985844f, 0xb1d010ecfb8c204b, 0xbb73a7577cb4933d, 0x42b70c4c01cb61c9, - 0x72854295ba34b1f9, 0x9f1be157a2ae9841, 0xecacc6be2e852d96, 0xf020e49d5a2f44bd, - 0x9c1e5c8cb1db7c67, 0x69421db1cd34f59b, 0xe083595e231dfcbb, 0xd453d34c8fdfaaa9, - 0x6e32cc038b107e85, 0xd1dd717d4d0fd2ad, 0x04e525e450038090, 0xd292012c3b978e1f, - 0x84e353973cf7654b, 0xd5d480a6df45f277, 0xf168f0cbe4a9fb00, 0xa4a4ba92bbf857c7, - 0xe954bdaedd7cdf39, 0x52f4f682b93bf065, 0x80ad5ebdce22226e, 0x8a48594769f445d8, - 0x69a2d144e3f8b817, 0xe06b7ff400eb37a8, 0x63fc741bfddb3854, 0x62de466a14d395f8, - 0x9948d98e9c5093a3, 0x3a0a330e8be12fb8, 0x1088e2b79541c7d5, 0xa848620fcd25ff40, - 0xe78a59fc927bd01e, 0x9870072664c53c15, 0xd892a6a74c560062, 0x509140725dfa1da2, - 0x455e1d8ac84a55ae, 0x5aaef818be798b9f, 0xf24bf81a7db4363e, 0x83588e6e380504ca, - 0x298523ea5e8bad0c, 0x53be8ff2729d2471, 0xe34a6099d8d1a5af, 0xf6f0a090edf46777, - 0xbf14daa2d24d5d40, 0x31fcb4f35ff341ed, 0x324310e487323c72, 0x3f59ae8e8ea23b71, - 0x00053b234c3701a6, 0x091f62ecd5141cb8, 0xb4765320fcbaceb5, 0x0de7d7ec2534f26d, - 0x374949c64f91debf, 0xeb1d31dcd3c1ab45, 0x10eaab326304d123, 0x10fc17d27f69e141, - 0x117d00050fbee7f9, 0x2705886402581560, 0x9d86293a0e8f4d9d, 0xd28e00f3014242cc, - 0xb2694feddccc30b9, 0x00e95064fadb18c4, 0x1bb66cfe7d5a1a76, 0x5af5825662b90d28, - 0xa435f18f9b04eafe, 0xba57a338179ba0fb, 0xbf53a57c39ccb396, 0x51350b4ac7878176, - 0x1658b5e3ce3ff3aa, 0x0e2620eb9571f828, 0xd0247ff4ea81721e, 0xb217fd43d43ade7a, - 0x81a418297a9770f7, 0xc931f60d899c9645, 0x35955f76469d9de9, 0xa4fa6954e442719b, - 0x15868e0ea346fbc4, 0xedaa623f9416ea3a, 0x89d69bcc5d8c86fe, 0x910a0a447205294a, - 0xe0cc9c02d7fe16eb, 0x4d15cff7bbb298ce, 0x910789de1e245e99, 0x8dbe3e2b0b599246, - 0xd9d3ef179f407112, 0x00fc0adbbb242239, 0x62658ef8726e86ae, 0x9f31ca8438d94feb, - 0xad9fe1d65577c134, 0x3a42bc2312a981ae, 0x7ff2ef52e962dd90, 0x6bfdbeec8df83b2e, - 0x6517cfa5f68e25c9, 0xeb39453af4da2602, 0xb255fb790901e5e9, 0x73a7b6e85177d5c8, - 0xabc55b5d03809c31, 0x0f68a7398dc0fcd5, 0x73d19b7b9bf2d7a3, 0x2f6e59f5b719e482, - 0x66a420077fad9e19, 0x56b60a2c3a94fac7, 0xc3bdeabdf9c9f3ff, 0x90e25f8538b17780, - 0x350f03199063f157, 0x9cd80475f40ad174, 0xd672b4a5db9ae180, 0x479a54a27e270ba2, - 0x4e5ed7b0d949c187, 0x2dffcaadf4c206aa, 0xd2d28140e5e25609, 0x579b7220574b8ed1, - 0x550248a7bff5b8fb, 0x43f6ec995c99e24e, 0x533c2f4932538704, 0x5c7922caec1d1e76, - 0x12bba099ec3c43bb, 0x03458a7c6ef1e6a9, 0x253af30575c22b74, 0x9243107add038985, - 0xf783272a391d95b7, 0x37f91cd4d7de8aad, 0x178f09ffca14dfa6, 0xfb96c440d1be4855, - 0x62e0c68cde14e517, 0xaf0f8a736ffff955, 0x5b732c4e40b8c04a, 0x00a12931d33ad2a3, - 0xd634097ae2d1790c, 0x0dfc9c68bbada50e, 0x964ba89c4a40befa, 0x6e49f32436e64072, - 0xa201cd81e9d971fa, 0x4d651d3084d2cdc0, 0xbec2bc43f1f5318f, 0x78c85a322a2181a6, - 0xfa6424e8d11779e5, 0x104f05160a0a09ac, 0xd587d4742feb1704, 0x295906d3991e6afb, - 0x644264f277dbfbf9, 0x802bf36168e372f7, 0x873f3ef8d554b8d8, 0x4669e41156f2f028, - 0xe68c242c30de8d0e, 0x008a7640e1a09a32, 0xa168338ae7826261, 0x5366e2ab3401f6a0, - 0xaa0a9caf5730e9c2, 0x51cb178eb7bb4e31, 0xc9b7e7c607e7edf6, 0x8892553095f19ccd, - 0xba8aa8bb50c0074c, 0x11fa65d939c324a3, 0x8ce7ef9838c10553, 0x21a4666c45bdb713, - 0xf2c18a9fdaf9e677, 0xdfeb786d8a1f30ba, 0x62882f5dc4f89c6d, 0x4fb9879dde817281, - 0xe9c022548ed40839, 0xac4b816632aad658, 0xbf4292f20b30cd36, 0x7fad5de659536a89, - 0x9080ac033208c856, 0x610132584e52c2b8, 0x7d33bcd2ea98a946, 0xc61dab2b60a3847c, - 0x7b3f8471a4702f78, 0xaf39027b4e941f0c, 0x294ea8a1f64609e4, 0xf22d574d235fc103, - 0x04d1d4641e229e97, 0xc55dfc4ff7af42aa, 0x29c02f8d5e0bf64e, 0xc74b949ee63ba8bf, - 0x452bac800c69cc45, 0x2ac84fce7db10dba, 0x4819edd55bcb11a8, 0x91eadd288c8c8af6, - 0xadd8adf54c069546, 0x1a2374246693b29a, 0xc086d75c7c2129d7, 0xf5a5dec3e5b927a8, - 0x3a272b0872c87832, 0x553a3eabb88c047b, 0x4d43a938995307e4, 0x17f08fbf1b9901d6, - 0xe0ed1122df136f38, 0xf6eb9d131fc827cd, 0x5801bc12130c5333, 0x5ae4a3a272888093, - 0x7c0a5698d339e33f, 0x93b580b83d5970ff, 0xd7796ec330f5a152, 0x1c87dd3b58e5ee30, - 0x1b34c919b57c56d8, 0x2d8be02d1d162c22, 0xc2a02c0bbe458d40, 0x18f7c40a4a4916b3, - 0x39d3ef756a2b2439, 0x9de0ba8bab42f446, 0x0656d078335cdbbf, 0xdb09b8f348a9898b, - 0x149e13aed09c8d9c, 0xc83deb62cd385bc1, 0xa400f82b7ae309cb, 0xbbca0d37ebc98bbb, - 0xc67bcb23609d299c, 0xe0e9becc7dac976f, 0x18294acfbff04b5c, 0xc6607e3885bdee95, - 0xddbd5e621a3e9c30, 0x6d3020dc030f8439, 0xb8f6df6640ae288c, 0x7d5dded44821da6c, - 0x30af10da52909708, 0x4c32b26320f00b36, 0xaaad1fe447c35e49, 0xf74ceb4df8f025f6, - 0x59fd4626afeec408, 0x899c8838c220b8b5, 0xe549721a84e59a97, 0x2d31625fb2ac6a2d, - 0x24236d521c45b698, 0x09878e523a2a1241, 0x80acd8496b2e9244, 0x9d0ea104b5493bbb, - 0x8c01af30ff9d1f70, 0x3ead7916f9fd9ee1, 0x7108f122ea443986, 0x66992a41bb69479f, - 0xf8242226c1debf73, 0x688d016d716a9ce4, 0x1f7e260bbd32f7a1, 0xe7fecda9ac6623c3, - 0xfba572875a71926b, 0x8d20cded51073183, 0xa73c9a61f144f64c, 0xb6b02f83308c8107, - 0x89b02761718d50fb, 0xf3a517e0a780667b, 0xb7c0d040ca616d53, 0x429c7ed34129cb00, - 0x90677fb30dd259d8, 0xd2381b67fd85c15c, 0xbe1950ca5b37bf2d, 0x03092050ca240232, - 0xec8f36851a434ee6, 0x082ee7de4087803c, 0xafe7c9eaa0f3cca7, 0x09996d59a1c29ca8, - 0x0f5ff43cd40b22a7, 0x60aa3885ad3cde40, 0xe905e74ec3f7e69b, 0xc2b85130fc9e9fe2, - 0xbe81a881840c9383, 0xf2bb1f37828cd9e1, 0x8f4e68228671ea9a, 0x7b3b646dadf43dc9, - 0x6e3b59f12d0d8614, 0xaf795a84f64eee80, 0x43b51fed57b31aa3, 0xc033a7a7d10e0f46, - 0xa84390860ac6ff97, 0xe6ff062b0e92bf21, 0xfb770ef9cf477968, 0x03f4a24b532e19fb, - 0x9ebd43fb08421dab, 0x4aed55cea91de6c0, 0x7ff519ef50f3a92d, 0x67895e7271b8a4b7, - 0x4d8d36fe5eca2399, 0x314e1838c9533761, 0x29e37ae72dffda3a, 0x5380ebfe11e38dbe, - 0xf8b15213c9fea179, 0x76d1e37f22064cb6, 0x20b40168b07eec4d, 0x5dba597ff1c18802, - 0x8c2939d83a159afb, 0x87636dc7be8b98a5, 0xa9a57365e589d981, 0x321b276179688c5c, - 0xd1b157a294e7efcb, 0xbfa0ad06299cc236, 0x97e9254ef3bac482, 0xbe380ac48382c6a6, - 0xde73d9f9b0afab81, 0x49cc7ef3d4455900, 0x97f37c847d7bc885, 0xae6b8dee56ffb77f, - 0x188c4b783e441a1b, 0xa128026853fcdf5d, 0xa93683f4d1088946, 0xc2a6b3419ac829f9, - 0x269d19f36dac94c0, 0xcba5a9d6de4a78bd, 0xbeaa334ecc4c6744, 0x8a87fb040e962fba, - 0x9f8e1cec3925b05e, 0x678a01f0c853070c, 0x56585be043e0031f, 0x1032eced5d6d0c33, - 0x3ee0b9b527203737, 0xb0949485a5c4d58f, 0xdebec6280067d9b5, 0xc397ed6ab32a35de, - 0x5e2053ed19de791d, 0x4b2c2902a0837974, 0x3f71296bdc448cdc, 0xf9ec12c2920c5b2b, - 0xee57ef8d8c176670, 0x2e7d145cae255636, 0xaf68510ad35089de, 0x70497e0fef82ad04, - 0x1cc49affb7d7a055, 0x0c62e60067470136, 0x26ffcae5dec1d1d1, 0xcc02d18868659802, - 0xfb0d7b3b81fff622, 0x693f807ac5ddfe08, 0x87289ef2085fe6be, 0xc777d7497b8ed119, - 0x29db3e2c79125e22, 0x1438c5a362f456f5, 0x3348269099bacaa5, 0xc5c0a47d9e8621e9, - 0x78783f5aa97bef28, 0x5886cdb12b7997c7, 0xaa10250093f522dc, 0x35fcb986ecd664b5, - 0xa0783bbefc5e30fa, 0x30a7036ada65970b, 0x619d98e1f209d3d3, 0x19196ec9a309c998, - 0xb1cb9fada72545b2, 0x8ebed581df93f1a0, 0x1f9f205dde052709, 0x6d813e36a557fbbe, - 0x2ce7f77f13b16a26, 0x97175f9c8ea82cb6, 0x33c22b0be6a9a764, 0xd7e10fe43ae9d5ed, - 0x886650afe20d2a71, 0xc9998b9c2ca3d20c, 0x3ee859962d441789, 0x45dee757099f62ea, - 0x3a9c5abb7c70e839, 0xaf61d66f73c779a9, 0xe95918543cd8928a, 0xe1f14629cfaae394, - 0x8003a2bc1709865c, 0x95a44eaf827a3c95, 0xcec45e7367552c27, 0x6536c677f387bb81, - 0x3934ee1ccf43b5d0, 0xe62ce6d24a7facb6, 0x3ed76bb63d6c5028, 0x3d90a75f01329c8e, - 0x379acd4f05e08e41, 0x6404493be6dde4a3, 0x8b505261042bb981, 0x2dcb10df77d925ef, - 0x977b777b6616b977, 0x8bd0353965bc3717, 0xcb5441e5eede46d7, 0xe14ca2ac16c7b24c, - 0xa3cb63e1b9e863de, 0x75c24abd9d86fd0b, 0xe4475d04bbc640f8, 0x1531d5be930a03cf, - 0x19f80142dacf7ca1, 0x89530fd6a376ec8a, 0x0d0ac7876a8da7e1, 0x82de740fc23c8d80, - 0x076e9087ee963510, 0xed077111085a17e2, 0x8c12f20185eb9302, 0x6480ae00192a67ac, - 0xe9058ddf59416aab, 0x7a1435ba1cbeac27, 0xc7040127b89f1a6a, 0x050c478fae2275e2, - 0x042ecc708efcbf6e, 0x4114a86ac49970de, 0x7cc98420ef3a3f46, 0x0a364094bdef0bae, - 0xbe39e9f934bdc3d0, 0x51a5475be1b9ad66, 0x4adea2bdc2c12df4, 0xac7545b46a41d889, - 0xed80b4efa3f970aa, 0xcb1ff55f78571a1b, 0x6e25150ff62ee58e, 0x3cb24e8fb12d0cc7, - 0x44fd1931f9168ee0, 0x90fd97e1b3a01eab, 0xfd9ba3b64f98e021, 0x45ac3e8a9eae5c82, - 0x6594579c0b5802b3, 0x4baba9361160242e, 0x3e4fb2fece46d763, 0x2b3516b910bd5d56, - 0x2c21b552f527f7c4, 0x75ad9bdefe3d3186, 0x18553dbd04e2ee5f, 0xd2a433b4bb8758ac, - 0xc9b11071ad8cc349, 0x9cd05a4f242bf87e, 0xbcd4f85f7b913725, 0xb942595058223020, - 0xc2f65aad36d90671, 0x4c2e91e591212b9a, 0xed272db0a928de8a, 0xf5b1574c2c390a82, - 0xd324dd969b8460a3, 0xe011aab84a2109db, 0x4d43d4eeade83a51, 0x7a8ce3b89b65073b, - 0x51405f05f32621b1, 0x6358fcf9c3fb6db2, 0xf6d1bcde2858e590, 0x90456d9ca1526de0, - 0x9303fbd188289a39, 0x0a66991693647c13, 0xeb7b48858069e596, 0x62efb1b808534d85, - 0xc169ac4847c42b92, 0xb050218f516eced0, 0xef472d628d6a0793, 0xb793c0f0a8961da4, - 0x57e306ec5d2fa6b0, 0xa4b2a1c3b295ea93, 0x3842197c20001fdf, 0x475c447df1af8e1b, - 0x494d57546f660b89, 0x514111d3cc3af0b2, 0x090dc3de8f4fec76, 0xa211c2353cf45d07, - 0x3c6d9ddd2d5ca403, 0xe481318127965ea1, 0x66fc7acee1a9e075, 0xc220ccc167962426, - 0xc388e5e7ecdcdeee, 0xc498961cb3c1f7e2, 0xf2df37b0a36fad68, 0x16c7978a71f07ca7, - 0xb767ab7d67aa58a3, 0x34c5c715a89fe8b7, 0xd9fdc4bf477df042, 0xd85bb2965b91b1ec, - 0x02bb84af3de9ae73, 0x2762f77af3ecacd3, 0x4cce06b6c135c6a0, 0x3c12ac70269a97c5, - 0xd34264ddf9f481af, 0xda2c20db23a2a951, 0xfb354a55dc9fe1f9, 0xae56613479404fa1, - 0x30a1e736e1e0979b, 0x8a43525dce23083d, 0x1d3d31636fe39681, 0x69b7806c71434192, - 0x5fc3d70cdf811365, 0x2c21baabdd1f3ac2, 0xa4df2734eb4fd3c2, 0x05fae8f60869b250, - 0x24f02cc89eab2850, 0x29def04380f91cda, 0xb52061a1129631a8, 0x19e06c408d77a342, - 0x098020cc1462c046, 0x63ac7fc061bcfe7a, 0xe6e7c64c04929ade, 0x1607be500aca5ac1, - 0x81628b5ba4f20066, 0xc4c5652c6838c436, 0x1050ff4ba2a8d111, 0xdd27940195e099c9, - 0xcb99872e705308fa, 0x50c478b3d94aaccf, 0x924318b014c1011d, 0x25d73860d2404cb3, - 0xdb45831ff6321071, 0xc2f9f19a14af17da, 0x101646ac60be0e81, 0x6dba342ecf5e92e1, - 0x3c99bb088c2e52bd, 0x68ae5303d5f7219c, 0xf90982fd7141decd, 0x566bba4d02652bf7, - 0x24866a4d9dbc9da9, 0x28f37d5ca5341e34, 0x6618cc4d450e2dfa, 0x5bbf2809edf350f1, - 0x80fb47d9e300892e, 0x26fc85aa9cfaa0b0, 0x806ddd37f3cc5468, 0xae490934949759bc, - 0x29a0e655e80b75f5, 0x7779044e1412074f, 0xc35cd2d7e7698fe0, 0x940aef680f79f484, - 0xe64c9f73c67d8fe7, 0xa6a96d63337ec9d4, 0x1f7dd49668eed91b, 0x907899af30155709, - 0x5bf3c5f6643ae96c, 0xd8cbec88672c85cc, 0x2d3f0a08516c4358, 0xfc4b88aacfbbbbd2, - 0xe271f23ad1d18136, 0x8ca732df1ebe539c, 0xb88d898f4ac26b5b, 0xb2e297ba042c14f5, - 0xeb1bf7ee1f77d689, 0x67d86b1cb041690f, 0xe70cc5433f0f5ac1, 0xe79f9f78543cf7dc, - 0x74b777ed83e3a89b, 0xe5269dd49a700ece, 0xae1a3584f55efb5c, 0x15e255dcdd058c2c, - 0x6a3833d5b6a91cf7, 0x8a592239a6847a1e, 0xa8fdb4b5f994874f, 0x7db74c4ec3ae70a5, - 0x82da1665ad0dbbcb, 0x358550daf164d1b2, 0xfad790370f6ca348, 0x0b9222d05d113608, - 0xc62b4011b9fa9cc0, 0xfc5a89149e7d7980, 0x59a250e70d073f20, 0xc7534a359d0e495a, - 0xf9235e314514aca7, 0x3df309ba4adece5f, 0x94849afecf3d7650, 0x5b59c52a60f76f9a, - 0xe6a2cc14e1b59656, 0xfe3178953dff61ad, 0x16c546690297cde7, 0x6f703ea28739349f, - 0xae0a30d7dd13b2a1, 0x249b762670ac916e, 0xffba778d131c907f, 0x292774be907365d2, - 0xfbf2c785fd531803, 0xdb7144d62812ddf1, 0x086570ac2b4a81d5, 0xa41b0a36f452e8c1, - 0x8fa809ec08b7bb05, 0xcae7856976218344, 0x7f356eeacdab41e4, 0xef2c422159ff9be2, - 0xfdc0207c40e5f5c1, 0x0551341564dd7bca, 0xbe53194532aeece8, 0x7d5bbd145520fe79, - 0xae5bfb107dde2247, 0xed61a411d84a58b8, 0x19b8b8819c1b8410, 0x649a81bc82cf90df, - 0x5d452c0891c8bd39, 0x86d74b7f033393c7, 0xe38838b0f87f2bbe, 0x90a3edcd1e94e4b7, - 0x299a990459528c51, 0x90426eb368fdb8af, 0xd9edc9c54888d97f, 0x6eb87744d1a30bca, - 0x9eba72a03688f56b, 0xc7b96fb38387a7e3, 0x84cfab0a6ed5252d, 0x8cdfcaf43ca0ab81, - 0x797034ccc9571108, 0x21c1a13fd92ca367, 0xaca7ff593902e8f6, 0x15fbfcd0036bacac, - 0x09adfc1ca3c546cf, 0x758179fea0eaa984, 0x585d1f20d4dbf915, 0x6b4f211231223287, - 0xfb32baef29d25fe2, 0x7816ef6fa155cd5d, 0x2a3632c1c8815ec1, 0x76a288f890cbddff, - 0xc04f600325c8ef00, 0xe3c2f0db15b92579, 0x0e07bcb64dd36f29, 0x6903e74dbb385788, - 0xd87549c2b040140d, 0xd1cbe4608d3c2c56, 0xe93bf851e1e5e05c, 0xe577354f35b83bf2, - 0x26a8c5dac3779fac, 0xe280558f97f20dc8, 0xdf48ca8728a8e146, 0x1e7cd908656c78e5, - 0xf9d9f43ac88cfc86, 0x20eb69e2e2f5a543, 0x96bdb9587173e733, 0x7dc88c3b886868fd, - 0x255a9ede9c36a293, 0x25e5b0b3ec562058, 0xac2ab486a78a6ebd, 0x7779ec182fb67970, - 0xc22d882dbf9704f8, 0x7f63347bd82bcc95, 0xe2f482c4107004b0, 0x0a651935a19023a1, - 0x82120a197e509f50, 0xd87c60a1583cd604, 0x7445ed6734397276, 0xf890c1bce6517924, - 0x646cb0d51ec15240, 0x06b53d035f09dcd6, 0x51292e6e78851979, 0x4cc297e1e946e7a2, - 0x0b64d261f89ce3ec, 0x6b25a56ac5775ef1, 0xf5e23c6fd86b2bf5, 0xb96323b27b2d24fb, - 0xf42ccb8fb018b1ab, 0x41205bf62eda2be7, 0xf273e3925ea04936, 0x312bd45c54dd095f, - 0x539ae4f37d144f8e, 0xe055c2d890f471bf, 0x322bc70cc8b64969, 0xb5d27aea96e3e954, - 0xfdad41b27cfe635e, 0x66dd8823e2acb883, 0xdf08d026935411b5, 0xe06099b3a6317773, - 0xee3d5f781463b615, 0xc03ca7478a642967, 0x333be962ca1192e2, 0x159775c82fc18c4a, - 0x5bd1422a6e3efa60, 0xcc661898e81b5a97, 0x2d1c4f5a54c8561c, 0xb1d5b773bf94b8bc, - 0x371daea2265021f4, 0x5df627ded4aef5a3, 0xae1d7dfff0fc1cdb, 0x80ccb10a2b5bc7dd, - 0xb9a0ad80a896c618, 0x75461c073d7442cf, 0x60c0c9b5aa4f961a, 0xce4db4d4458b692a, - 0xa824fa89bf5e8d38, 0xdbd0099a53bf3d64, 0xbebb55974efd30ea, 0x61c37fbfad7e34f0, - 0x11d8f3ce9face7e3, 0x8851d3ebd06495d2, 0x7a19e890ebd35619, 0x212971797d8b2904, - 0x1913dd39a3dfb428, 0xfcaf24cd21d80a01, 0x2bbb281cff180403, 0x34854f411b2cc406, - 0xb8eec1c4f20627c0, 0xdbc4ae345a789456, 0x4050e1c23e246ba9, 0x76cf678828aff38d, - 0xf03f1b4d7205c6d9, 0xc041fcbe3976913d, 0xc44a7ee9375aa69b, 0xebd559f5efa3b359, - 0x48a83b4b28aaf168, 0x05b2dd130713a8aa, 0x9b904f9edc334db6, 0x595503bbb221ed7c, - 0xe3ebe979b4a3e3ad, 0x11eb068868a96eaa, 0xe4de461039d622bd, 0xf21d78d78b43e655, - 0x3b928db0ba54d339, 0x328239b1d58c9f66, 0x6e99f0017ada5132, 0x62fcb611dcb4968a, - 0x3b5cc401e04f4a80, 0xa8cbe09b34c8fc1f, 0xeae399dcfb233afd, 0x8ccbe9665c923b2c, - 0xca62b27c66c94b8b, 0xff90cb4e292805ac, 0x8071d31743e07c45, 0xecd44b1a1996e043, - 0xc5fc723c9359de93, 0x97b308f6914e1226, 0x5798117e8ce48975, 0x538ab5cde8cc10a2, - 0xabe5e9bed4f70df6, 0x5cb6f56b7ce580cc, 0x1a61b699b431c9dd, 0x6d07ef9499722502, - 0xfb619effa077ea80, 0x6a65f6f66692be20, 0x84ca63329c9b5c7d, 0xcf60e080f238b6f2, - 0x182020cfb6f0af64, 0x9ce13cf3848df56a, 0x4b19d42fc8e4414c, 0x7852cd707c29b15d, - 0x1545a14427cdfb19, 0x1769086cbdc305a8, 0xa9d084dc8d0eafa3, 0xfc758766798194ad, - 0xde1924aa1b98ce20, 0x3051813e080d36c5, 0x3a8378380b5130f0, 0x060e436c36853dd7, - 0x84effbfbe6f259a0, 0x4029fec9b938d703, 0xaafefc6f4ed00879, 0x26e61ed228c8f334, - 0x46bb30908e6ec690, 0x9036a30ff855f37a, 0x246fa0e830780a40, 0x1e3148c990619e17, - 0x3275517374f6c851, 0x92ecddb667d04f94, 0x9f02f5e8d0696ea4, 0x14a7e9f9e22cc1bb, - 0xdac481c8e06d2063, 0x4088a3a7fb503e3c, 0xd2e25e47d1317d7b, 0xf480e52275a4377b, - 0xa88bb91d75282037, 0x3c922a4034cbe2b0, 0xa649480f027ceb23, 0xb00fc66c74f60b2e, - 0xda85fed905f600f4, 0x05a116e1e532fe5f, 0x4de9e57c6ff20aa6, 0x94f3caeee5ef3481, - 0x958615453ff4475e, 0xebd64a2a3f7f3459, 0x197acd18f8804824, 0x363775a0cdc0aba2, - 0x98afcbacd14fd0fd, 0xf7e5f9d5ab3e7c39, 0xcb41759ba3e2b6c4, 0xb12823b6b9d507b6, - 0x85177cae733b777f, 0x1fe98a584f2537db, 0x486f76cc9698066a, 0x2658ca8e0d8e9603, - 0xecdee992ddc4139c, 0x9a28d18df65ee491, 0x6e7d6b4e2e768616, 0x4eda52d2c12c5a7f, - 0xf26aeaf70ba9dd3a, 0xb59d2deb012adbf5, 0xa27e085b3dda76ba, 0x7b16a2adeb548389, - 0x0ed24c5a1749ddb1, 0xec9318bf4cfb62be, 0x8d4ea42fb97830af, 0x650b46372b02b877, - 0x299289d153d02e20, 0xa7e41325e9d77fe0, 0x361760c3063fbf3d, 0x96a9ea423fe68c49, - 0xe8fd44d7f8b91e82, 0x111c1d531a407126, 0x0dcb976f928d381d, 0x087b953775c1452a, - 0xc0a73410dfc11d92, 0xd54b69d639f333e3, 0x1a8c1420c76a3a98, 0x91f73accb9fb7353, - 0x7c636fb01d60dec4, 0xb36eac90ebc1d673, 0xeb562affee9e58a3, 0x209dac984fb36e69, - 0x91e091e7b1339596, 0x75de4bc6a7b4bfe5, 0x2235d683b14a386b, 0x7335e0785de4d314, - 0xe631c2259d991c64, 0xfb67e7e3dbafc6d9, 0x4b61d8aa333baff3, 0x303d00b884c04c87, - 0x943b02001b6c4a13, 0xe543ec9ff6aa7528, 0x5da2c557318d5fb8, 0x1e05e41a375442f6, - 0x5dce03dc6b002344, 0x11359cd5f96fa74a, 0xb850fa85d4b85477, 0xba7ee8d4143bd4c4, - 0xce2efa66c53faf83, 0x2840147f72ae2116, 0x3dec9bf9f41befe7, 0x8bddc26e45ff09e8, - 0x03ab73b8d0e62f7f, 0xb8aa40498c79b1d2, 0xd8c79eefd6f5c27a, 0x8ab7de447dc664f5, - 0x2a420416dc313ab1, 0x3e3deafabc128491, 0x8e185bfdc5c2a8cc, 0xf282cc033f7bcdd4, - 0xcccf89082c90385a, 0x1d7fdc1228868c60, 0x89dae00609e418c8, 0x077c835aa333f794, - 0xcf9fe52e125bdc37, 0x14d9d89f4e593756, 0x31e63d89e08bce08, 0x7db9365210c6c11d, - 0xf470c372a95870d9, 0xd8dd8bad10dfd890, 0x28dee96f12014edd, 0x5fedac75bc0a2568, - 0x2e484f322f8f7a29, 0xe96ba47d5150a994, 0xfae702516c54b6cc, 0xcd00ffb4968b69ef, - 0xf2801387d1cbc1c3, 0x3e926b195ea9a74a, 0x87531a019cb4216c, 0xa3f489964a8b33df, - 0x7a45efad32c79c95, 0x5a32766fcebd320a, 0xcc5dde2f89a4ca99, 0x6522963dcdcfbbed, - 0xfb152be3a9398734, 0xa63ba08b16021d79, 0xef90a96e970b18d0, 0xbfa1d710ac0427b8, - 0xb61ae59554d63093, 0x163f87d0c0c4f318, 0xbc312daa01cda2c6, 0xc9b3a78684f4ad83, - 0xf3a82e342be4b5ae, 0x48ff0e23fa6038b2, 0x8a7b33da7cef8a32, 0xbaace901a1775ccb, - 0x2c854a50b646fc41, 0x39eb8099c508eda7, 0xf7317e5f5414eada, 0xb193ed2038df83f3, - 0x8efe0432051af048, 0x02c101d8dd9540d1, 0xbbf2c3831874384e, 0x47837934d4de9253, - 0xcd6790c9a868ddd4, 0x11d99b4198b5321e, 0xe6e2e4e3f98cffae, 0x057fc17193714be8, - 0xd96b273470272d0b, 0x37142ed36c337c01, 0x804887cfc5dd884a, 0xe5a51a1c1370aeba, - 0x5bec1b27c5c0616f, 0xf117746732055fd7, 0x09c5ea79121d2521, 0x85c849aedb30d9fa, - 0xdf3224c63b119a0c, 0xd31f4384819538e2, 0x35031dda5ea8686c, 0xcfadb56e708aa990, - 0x63f539864afde9ac, 0x495eae5b295cc38b, 0x81fe22b1b5a14160, 0x32ed23e730eb5635, - 0x5f83f9df55d94400, 0xff1d9c23d996eac9, 0x556b67ddd12f5523, 0x0319f96408d54e19, - 0x0597dda784687bb0, 0x39a9ed179ab1db9d, 0x93276bc94cb2bcf6, 0x4bb054fab8f86e44, - 0xea19f58ccdcedc20, 0x5a106854d6246fe9, 0x358c5b2a4080919c, 0x127c880b950a20fb, - 0x80950cce7df3b6f0, 0x87ebc8e8b809697e, 0xe7ce42f2e5f2633a, 0x859ea7b626aea862, - 0xd532f0945e548d83, 0x10ce7253bc316f1b, 0x4253393feb8d2020, 0xb99f9b3257b4765d, - 0xc25c35242d0570b0, 0x5151b8823483c555, 0xf20d826fe909d993, 0x0ada96a7cab9b3c4, - 0x65fe6001a9abc381, 0xc5c304bf9dc5e12f, 0xf80a47363c7824b6, 0x2e71cf0628ecbdd5, - 0x3ff2e199c6cfdea2, 0xb3fe7a0f24863347, 0xa961cee5ed454914, 0x13478e30f1fe6e20, - 0xf43f81e2532e4e87, 0xa85a84e623c54fbc, 0x28d1915b4b3f8553, 0xe5bbbe79c60c3bf4, - 0xb735ab5eca2c405e, 0xf1a07668b55322bb, 0x181efc4e5d494eb7, 0x5ceec4adf84d94b9, - 0xd43e20a231d9947a, 0x6a30c25560a12054, 0x9ab008d416a27bff, 0xe778308cd4d6fd4f, - 0x3c513f87f5b2eb68, 0xd1e963a48023d363, 0x0edfb89c7b2bb079, 0x9a99d756dbf20093, - 0x9377dd24c802cdbb, 0x1a74d5e46be66d4c, 0xdb606bbeee69097a, 0xb90eb9b495a16636, - 0x4706c9de80b82328, 0x2bc86917c10eb335, 0xd0518f9f6328312a, 0xbe6209a4e13d1aaa, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5eeaf2037a02b1d0, 0xca8b65027ed387f3, 0x9e9f8e89df253623, 0x7550aad5e7da19e2, - 0x7a5e7fe30356eade, 0x7a7b5410beaa3bfa, 0xbda578ce466036bd, 0xc645118444bd9dbf, - 0xfaec353745731e46, 0x920797e8e13c2f56, 0xdc12620198de6780, 0x28589cd499b47598, - 0xdc0edca5d099349a, 0x372cfc338995027f, 0x557688de1f2afaa6, 0x7adb72342f6d425d, - 0x68b6f0d4ecef6241, 0x95be1cf0988f9ab5, 0x569f8ea377c3b0b0, 0x95fa205bb7e82086, - 0xc40073c1792322cf, 0x19e3a17ba29aa431, 0x9679bdddaf9221fa, 0xe481eee23f090904, - 0xe29d414645196b96, 0xfc57cacfcb141119, 0x072b29a8858c5d5e, 0x44f5a90ce661e33b, - 0xeecd5ba3fa43e0d4, 0x757f22c356aa98da, 0x8133c84f4b35fab0, 0x7e18aabb85c39434, - 0x26909ad6d950e696, 0x52b2677e6702196e, 0x3c53a0bfb1bf56be, 0x6ad6c7c6ae6ddd1d, - 0xac2fe72056454dfe, 0x4e23f8dc3bc266bf, 0x3162116aff274ce3, 0x7bcf818b35bc9916, - 0x797e3b24c837621e, 0x97453b1ab6196d40, 0x13db43ffc43102c0, 0xf9aaae5c7b1b3c5d, - 0x8b3122c3f92895b7, 0x7590f81b59616f24, 0x3215724f208ea20f, 0x176f6261aca643c0, - 0xaf06c56820514fb5, 0x00409dee1120c578, 0x8cc4ede5be28df52, 0xe8f0a2934c8e2ae3, - 0xba6f5805252c2ff6, 0xd427de194020c16c, 0xdaf0426c0896c110, 0xba17d2ca1da4ced2, - 0x9f0038ba655d558c, 0x1c22ac1fbd14b10d, 0xa0ec9ab3cb4042a1, 0x0834d15fba605666, - 0x60d66b935c236162, 0x5553f834a5066746, 0x0710de012440f6b2, 0x9552de2f03c56458, - 0x2b13e44b50408ab5, 0x078289dedbaa553d, 0xad5a1ba94959b4e2, 0x7d1a331121907810, - 0x24e8f5924bf92d20, 0x6bf34310b54366a4, 0xe240259191ac89ec, 0x25b06d09ab2c0008, - 0x4c48783281dc79c1, 0x535d8d13dcaadbf7, 0x339e384d00481308, 0x7f0a4164efc48b71, - 0xb3625bbb7bbf5e98, 0x79cadab0bb8f3420, 0x4d56d9914266bf81, 0xf339cdfb17321364, - 0x1b2564bf15e8ff17, 0x762a35e28d4cdbb9, 0x33af2164557efc3f, 0x3de246ee45f1e19d, - 0x502ae5aeb44d69c6, 0x0a8178b4ba1fa347, 0xdb323c7d9133d7c9, 0x2daa04d22f4c4230, - 0xf7eabb257a210830, 0xce8ac7d198b7a0e3, 0xa70c384212a2eb03, 0x963be368b2ffe062, - 0xcad15aa9581185fb, 0x7a426f5dc237b6bd, 0x6a32307d47b1d1a4, 0x5ddfc740b91aa3ee, - 0xc24cc4614bb6c144, 0xe689e25acf622136, 0x879430dc18ce605b, 0xbbe985d41d1a6ac3, - 0xbb0e7b75b16ce3a7, 0xf92c6480df3570b5, 0x0674538ec7484826, 0xfe9ea1b682e05964, - 0x4140f0243c789eb7, 0xe123847e2f8d422d, 0x258f4efe9c478080, 0x0003ae9ffb15d4c8, - 0xa3b406bd4852db65, 0x71f8a4f75ad6aa58, 0xf2a57ee71a030125, 0x3b64bd544691199a, - 0x6cd971ae51ed4129, 0x51f0d3f1255745aa, 0x59d6ecaf87a7da4f, 0xa09ade226345e4ec, - 0xfb2d2a560f1a06e7, 0x4f5c1faa03480ca8, 0x15c32f7070bcbdf6, 0xb383058b764c2187, - 0xaf352e7faa8b5d6d, 0x65a09effa85a4115, 0x2f1bbbb1bba73800, 0xfa563d192e30c20d, - 0xd170f48831cc2211, 0x5f6bd812acfe0007, 0xba0d9d835b742cdf, 0x43c56da24a0fff6e, - 0xb081d3a926f58105, 0xfbcad2f8430fb0b9, 0x65a546fe76c6a4f2, 0x6c07c17d7e78e073, - 0x560be6a40c1f83b0, 0x58c10b9175b135c6, 0x65433e1779e0aaee, 0x8da161d1b4d3bba8, - 0x7a15b22053a89987, 0x6c55d134ed84f56e, 0xec779673da9f96e1, 0x69984895f43c3f0c, - 0x49212d973877961e, 0x4a811a08d7996454, 0xc2dca0b5530ded24, 0xefd3fa2e0392f7d1, - 0xd8c41e7297fa859b, 0x3cffb9f6350cf9fb, 0x0678ccddbc07307f, 0x1db9d71012f2b44b, - 0xc4b2c1f201a48261, 0x09e51e2ee30a9f4d, 0xfe2aaa9243ac44af, 0xa6e23d2d62af1d8f, - 0x2caf09c834e851b6, 0x38f62278a6db7b5f, 0xca34b721933944db, 0x485ed670bd08ea56, - 0x47d3a4c1a20cf846, 0x39f6f42cea1acbc3, 0x5009a708021eead4, 0x23f1bd4ef2074a30, - 0x238a5922e2ab4096, 0x617c1cf217dd3787, 0xb2ed593ce15d0215, 0x1f71dcf4427c2b6e, - 0xda8f2196adc48d60, 0x981ab4395f2cb008, 0x9218fb4f7cda142e, 0x4ced703dbc47a348, - 0x18fa42f9a2fe02ea, 0x78ab5d2f0fe628aa, 0xfe69c0d26584bd68, 0x78cddbc7ab498d1b, - 0xe0cee9541ce8cf8c, 0x2c2e11acc694907f, 0x304fa137fcffaef1, 0xe2f5860f61495c69, - 0x47542da2506cabd8, 0x5b55fcf4aec89ae3, 0x301d4d5b2ce85307, 0xd0b5b592aaedb94e, - 0xd61be5de0855eef0, 0x773a2a1c19e47201, 0x63a027995bbd4e52, 0xab1d53197fc600a6, - 0xff73901afd1e99d6, 0x6bb15bf8fee6fe46, 0x070507d48debfa7f, 0xc94a6a93ecfb0eab, - 0xbf9ae717f1c62c44, 0x2dfdab9fe00e0939, 0x12da36eb398d3b92, 0xe675b8973d173c74, - 0x753782935180cf81, 0x63c5533b29ace698, 0x0d64784bbdc7398d, 0x72055a5d7751c47c, - 0xc86e3bb5c1230050, 0x630c90ba81a1efea, 0xdbe90d4cdbb4da14, 0x15a858a897ac223b, - 0x656cfeab4ae249fa, 0xb6334ac62701482e, 0xbab359b1e7f2e24a, 0x3ba0f8eaf09b979f, - 0x593f7f6aba94d1bb, 0x9d9841de535ac2eb, 0xafc7433103b59734, 0xd3fa5b214d7d27e6, - 0x8149335f9edddb3a, 0xbb9d74d3d8c04ec6, 0xb75c1599de8fe31f, 0xb0b479fbd8c6d867, - 0x46b612579f9c1288, 0x128b9cc4df952cbe, 0x0276f860b064f6ed, 0xb93eaf73646c1969, - 0x1c5b36b967feef64, 0xe8640d0337a7278a, 0xe203499e3ff3916d, 0x6d40f58b24d13e53, - 0xc5c8f6a15a5a2221, 0x673e94a5ec1f92ef, 0x0534233485daca98, 0x5a4618ecb0388401, - 0xb896d4eb55fc4f6d, 0x9a4259c4305e5301, 0xc62ff22b0dcf81cf, 0xc8700ea118916e59, - 0xc1854d1c8f398fcf, 0xb97d8002b04ef0e6, 0x8dc0ecd8cef3900a, 0xb89a80b62c767686, - 0xeacf89e936bc0d88, 0xb9fea68f59311e3d, 0x0ff7e68100959d39, 0x8c54a35bcf08991e, - 0xc7bddd753432dc59, 0x8a83041a8d10d09b, 0xf6160a8df49b1252, 0x35e2ae2c771a4c73, - 0x503696bae45b3d5e, 0x3ed00f4e8e515816, 0x2d04ce2677dd24ec, 0x108563110be70e9a, - 0xcaebaa9e369e863f, 0x9c041ca68ec96783, 0xcb956b59347caa62, 0x2e1896c384860e62, - 0x0130539e4843c098, 0x1ef2ffab71a20354, 0xac9830a2fbda86c0, 0xcb240cdf8a328a96, - 0x2f927d0ca60abecb, 0x7cec871a67bc9f10, 0xeab713c713f7a8fd, 0xde33a967d098c905, - 0x749f7afa4897a7f5, 0xd92199b73c9168fe, 0x3f340624ab1130b3, 0x9d9bec562661029c, - 0x09addb2b0e40d473, 0xe96ca6c1d6df9a5f, 0x38ecd32697ae3467, 0xff8b393ca4344ab7, - 0xe179a1bec318c4e4, 0x2005a1c760173587, 0x2fc0336d272fb7c6, 0x77d3f56e6fc0256e, - 0xacbde97601516d6b, 0xd930df30d8157720, 0x847912d4339e1073, 0x71d98465cfd131b1, - 0xf1278e964592524c, 0x81151cf8494fab52, 0x08fc4934aea825ca, 0x8892fd7a1fa42145, - 0xde2021e4d1d34d6c, 0x8e5df2b060f24541, 0xb5dd3a57713cfea4, 0x9881cbe4ec5d917a, - 0xf875061330ec138b, 0xf06e9b54338cd895, 0xcd2f3af1e1d0405c, 0x460e8ba5ec08fcfc, - 0x6958650d1b47db37, 0xf12460c55f717d5d, 0x2f64554145913b2a, 0x42cac96761c462c9, - 0xb150044af826654e, 0x4f61079e648f2798, 0xbd99f7180e3904f0, 0xd487efee8128ae0f, - 0x55af8274b87c3e13, 0x9163f1911c0a2259, 0xf627433179f51294, 0x2723cf8cf92cca7d, - 0x8df82931c4e77141, 0xef5a3bfca47592e1, 0xc9fb7e981788c2d8, 0x00c4b918efffd1ca, - 0xff1345ca01105bcb, 0xc5eb5c20d4db2972, 0xc9370d522f9b3a30, 0x932e373242223e42, - 0x431e0e659407a72b, 0x9fd4fcbe4663efb7, 0x3fb6726765386bd2, 0xf5951fa2c40745e0, - 0xe37624563cb3291a, 0x76c474b48bdce9a4, 0x1f2c0512fdbf88ee, 0xf760cbf5d1a8bca0, - 0x9252a523a3e73ecc, 0x6b57ecd73237c38d, 0x57d0d29edb486266, 0x2a85ff8ca947c6f5, - 0x5924111a2fb82472, 0x2dd84fe753f3ce1c, 0x5ce6c8f38528f8bd, 0x51691f539ef93684, - 0x52ea41bbc448f6b7, 0xbcad1a41936e4ccd, 0x937ce3e7303e7457, 0x4df46522c103a4ef, - 0x3af22d230d7b7808, 0x4697f51599bf5949, 0x6afe0a3cdde2f921, 0x3c722e95a065c111, - 0xe0df8d2f0926fa29, 0xa722e4cc95eb0ca8, 0xfdf83e8b818f2685, 0xa3551b738ca62fc8, - 0x5014384215e13d3a, 0xbae0d31c81590629, 0x9b18f0dfe00205ef, 0x57eb007db2eb7373, - 0x5965f3b9a42ef1dd, 0x84136aa46e0e611b, 0xe0a6dd667973a800, 0x6fe41b6bf9e25322, - 0x2009795606effa51, 0x6d2caf8507179e8d, 0xcb414f6cfddac304, 0xf67d253a82ff0375, - 0x3881799527dbd22b, 0xbfd142e475860331, 0x56acd86cd8f44d8c, 0x011a281d13bab642, - 0x0eabcaf83dbfa762, 0x0d0987b6fadb141f, 0xc953a428cddcc071, 0x5398d6f198d5ffb4, - 0x26fa9eeff6b0e4d7, 0x9f7e41f0ebd93c8c, 0xf92895405ab02ca2, 0x33d1d0b2fa3cde2e, - 0x5b3c03e34e5b2352, 0x98135c3b03565402, 0xae876b9449f406dd, 0x4040c2909d76a105, - 0xf18a6fb24a96244b, 0x3ea6ae1140dfb301, 0x08288bf34d5b245f, 0xe892396a798e8366, - 0xae6f716d86dab1e6, 0x5868e546abcdf685, 0xaa5eb304f33c0b2a, 0x6fed8c1a5f607ba0, - 0x8fa5566a2e069c1b, 0xf5b3aedf5bdf7c12, 0xb0113b59f7da0a6f, 0x315ceab94456f440, - 0x2a95361b9f5c9199, 0xb7ce76d80c864789, 0xdda2c351047e3340, 0x859b202a079eb325, - 0x273ce471be14dd53, 0x161264d8c6cefbab, 0x3c626c2b8d94c369, 0x95a83a7e5472262f, - 0x06c228cdc0da8d00, 0x1b0d4cf76ac8bd90, 0x2c62752c6b6717a2, 0xb2b03bbb65a7dea5, - 0xbfaa38b6cafc41cc, 0x5f6587decc32793f, 0x8fef79bae47b416d, 0x96188e811ef1749f, - 0x5320405be0faacad, 0x6f784fa69a541271, 0xa7c87f5708ce3ddf, 0x5979054c88b3819e, - 0x0f85a85207e3aa1a, 0x3e8c53f1973cfedb, 0xa196151f63f91ba3, 0x73ae0d64ca0321c3, - 0xd2dd3ec110b346c9, 0xcefc1e4e3407dea9, 0xef15ee0006964ec1, 0x0678e0c795316cf0, - 0x8cd6375c1df3f4f7, 0x9bc3feefaefd3ffa, 0xf749e20576152996, 0x3dbdace8e3688035, - 0x2cd02682affee2ad, 0x9eb5943b35f96102, 0x1078c31f7aecbb24, 0x65b137ded1f65c82, - 0x619e4db6a05b0c96, 0x86cda0064e089f8e, 0xbf5940b931d3e80d, 0x3cdbfdc029d5d42a, - 0x52dacf78045b9dce, 0xd8f67053df9aaff4, 0xac90c52e7c5f8645, 0x3e7fc7023a167496, - 0x88c6b2b6c75d2bb6, 0x9eb91f785a720630, 0x0f33f24724943e7b, 0x5bb49392abefaf6d, - 0x111930d44a0ff4d6, 0xf3dff37d4355b43f, 0x7b2f29102c9570ed, 0xc52c8ab0f1c61779, - 0xb820a20397e0a622, 0x8d8ec67e5c03de03, 0x0c447cf8f013ef5f, 0x71b7884422aa0fc2, - 0xd26bd7ae8fdf11da, 0xc0dd59244640195c, 0xe548c9a448b7f513, 0xfaff24472aa2c228, - 0x4cbaf788c7912e42, 0x1f8721b0fabf3703, 0xe101827ab46df845, 0x188a7d5686c1f705, - 0x5c762fcb847483e3, 0x34220635ee27754f, 0x40f598337b02991d, 0x96d0af5f12200c0b, - 0x8bf65bb2f5edec2c, 0x0eabeb218d852f4b, 0x792313c8ddd05fc2, 0x166126259de6e6c3, - 0xec7ba2ae8797322f, 0x6371206235b63380, 0xf549f255b5a6d6b1, 0xb301e10148578ff1, - 0xf33049a789378d11, 0x45c812e6405ce53d, 0x7e1f41fe812b3efd, 0x02e2b26e126d652a, - 0xc6e2ba1d1a4ffa5d, 0x1ed0706b081dbd0f, 0x721a856b981da04b, 0xc19b097bdb165e75, - 0xb94dfdeec6602d9b, 0xfb6c24399d25aecc, 0xd2f538f34ddb1e37, 0x2f7987955effb3b2, - 0x927b56d288b5ef77, 0x3c150d91bc87c346, 0xb65d43e957354dd7, 0x3e7c3209b3b15dac, - 0x3d753e06551daf28, 0x33534e1b5d0ff276, 0xf60f9a07973d785b, 0xcd4ad5c12fa6cd5c, - 0x95b23f2d9856c081, 0xabac3cfe4aaea4c7, 0x376ed652a04e6901, 0x1cbe024e14355e29, - 0x9341e84cb65501a0, 0x31c92763b5703939, 0x20abf91f59f92dc9, 0x7665918ba30f832b, - 0x89c7fd5a4f7f03b2, 0xf26e96e1c0024fab, 0x972d28e0fd3e6bea, 0xfa8d2182f3b3283a, - 0x70f54ea62c7b0fae, 0x24efbc2110c8ec05, 0x74b5d9202c594510, 0xa87a454931007096, - 0x73a9698e7bad426b, 0xf5b62ba38d12283a, 0x8e04a268c1c71573, 0xac378f106b80fbee, - 0x167feb8d33c17a06, 0x5e3260aad339f737, 0x523ac55693082085, 0x4fc11422457e1622, - 0xf5c2edfbc68a6f15, 0xe15fd21704e6ed53, 0x818a00d0cdd1266a, 0xa983776b0722f26e, - 0xb7f89d07b0e6e659, 0x7661c72888f8c313, 0x436db1baec68ce16, 0x7d2da53ec84ffbe0, - 0x1121fed18b1979e1, 0xb2cac1cdcc5a098c, 0x6be53bbe4f36b502, 0x821c8fa98369ce7b, - 0xeaee3cb046d2b43f, 0x3d58f9b80d3d1485, 0x78d8683dd83d0f98, 0xd9ba9a73ef4534f5, - 0xb64fc4eccc9df73a, 0x6ff682900af66f57, 0xc343638ebdd8a568, 0x9e6687ea8e975ef3, - 0x25850ac066cc63a3, 0x1c5aec82b19c14c2, 0xb4f94c696846783e, 0x07e35f1874e92fb6, - 0x6f1023c79365d1a4, 0x181ed4f60c474992, 0x03f5c9e9d7e02a90, 0x8174fa9c56456fbf, - 0x9e301d0155c0be0d, 0x5c2181e983794c11, 0x3e874c91c8b70229, 0xf0bdc8e3f76e2b7a, - 0xdf1d41b0081e6c66, 0x2d46776f2048797a, 0x292b61777ba2ba30, 0xd5bed711075db78f, - 0x69eaad7f976b74af, 0x1d282e02a0ba1ba4, 0xabac653f19cf50b9, 0x7c638002a183c0dd, - 0x2f097ce88a66a246, 0xf539a9ac8c517e54, 0xb788772781c10c73, 0xa23031f8ab3b5d71, - 0xddb336a522cef187, 0xcc7a46dd6518d4c8, 0x12dfd5c311bbdfe5, 0x2561f06e06abdd78, - 0xb8f12d94933d6e3d, 0x5b53ca0de81aa24b, 0xadc548d9daf48cad, 0xcd4847b6746ca983, - 0x46dbce7718386700, 0xb4456ad98b35aee7, 0x9480ea17ebe3a695, 0x0312370060ad9c3f, - 0xc0005d55562819ab, 0x8405287bf43628ca, 0xba7af7e8b935e385, 0x596978c5ea365935, - 0x0e8d45ca76ff724a, 0x3fe37e3b004bb0ed, 0x570996c752bf9b07, 0x8031e28b9645b0c6, - 0x72e041011f18aacd, 0xd274f9155906c0d1, 0xa7512973424f479b, 0x295fe0bea5cb1fc5, - 0xe2dcf7b831944e98, 0x4971472dada31b79, 0xb8b108a51f33db9e, 0x3dd86a59cf928172, - 0x6641999db5f279cf, 0x96d6a9e22fe95ba5, 0x9c2d35bb100d8685, 0x57d1a61add6f7803, - 0xe467080c3d69eea9, 0x6c64feab01b78a29, 0x6a154f57ac9e7ab4, 0x213bba59972b553c, - 0x5bd55c138ee8033e, 0x4d4479f0f66e40bc, 0x728597dc0598c0d9, 0xdb83c2e6120af490, - 0xff176ecadf752498, 0x03f95eb3204d2f31, 0x95f8040fcdc5c011, 0x59bf1560b998b7d4, - 0xfb3c13ac2cd5e01b, 0x3b92869274cb507f, 0x77e52552770c00f3, 0xc2376cf5756d4642, - 0x1a6dc59affd90662, 0x00d8c4b4c54f2941, 0xd292b22ec8fc6069, 0xe85ec1a58a145e19, - 0x011205142d5b781e, 0x8738c0530c329377, 0xd52ecd870e1d4f30, 0xa49065726f824fba, - 0x9031fb7947933dea, 0x22277a4a66c68570, 0x10291bd1ba4f28dc, 0x7f75bdfb09b47e83, - 0x018862629cccbd99, 0x906012bcb65aaf97, 0x0626af1d6e46525f, 0xc911ed1a03cc7603, - 0xc71ce28d1a1b8e45, 0x1eddfb7dddc2ff34, 0x899eddc795b0ebec, 0x9dae8260c5276884, - 0x88b99c867ae0a35e, 0x26e97fc75d5f49f5, 0x292b7486eaf91979, 0xdf7fffd307ae3124, - 0x26e4323d06c2584b, 0x81e72277caa0ead0, 0x2b6d104a5ae5c1ea, 0x7fbdbc0bea8800ce, - 0x65d890164528eb8e, 0xaf230fc740625739, 0x549d17747a674070, 0x9240f7e52afe1e25, - 0x07d67893ea9d5379, 0x7fab5b711d92bae1, 0x7062289b05ef7b2c, 0x8d4327793198d1fc, - 0x255b45444c9288f2, 0xf21222b04559cf1b, 0xe01b5a49383b9013, 0xb79f00268b639952, - 0xdd88c6a21ecb0aee, 0x93de8c64b2adaf26, 0xe37ee35289b8ec05, 0x870bbaf4cb50ad90, - 0x5f04c87be747745f, 0xc0364274e54a04ae, 0x5aa75c75850a8611, 0x4743406978c05ec7, - 0x8da71a4031640a59, 0x4173d64407e843c5, 0xf9e173dfe991b2f0, 0x6383d9c8755a8ed4, - 0x72dc08302d2db9a0, 0x4a6f61b842d0095e, 0xd68f3877aaef740f, 0xa4dd677e40f60184, - 0x95b37b8e71b5ab1b, 0x8469f4409074de88, 0xca3f5dedadfc2aab, 0x0cc677d58aaee6b5, - 0x21d46beb1d1c7c3b, 0xfd4164330c31c00e, 0x7f569e00d2312530, 0xef99c7e2fa6fd0e5, - 0x0ab228c26b88c9ce, 0x6284a2f0db99ce8f, 0xd525a3e5ea216f75, 0xe2b4b6817b04a78f, - 0x5095b31ec55c8057, 0xc6f143857e96703f, 0x5e49af7f1b15db8c, 0x6b172b6d891552b4, - 0xcffd03a10869f817, 0x29f94cb2b6678105, 0x3871b9f5ff4980c7, 0xf0a1b0b93aa33b1f, - 0xa8dd09795e242508, 0xcaefb344fe157786, 0xc59c91b3414d12bf, 0xadf4e3d96b375e71, - 0x222b8e8682916266, 0x2c0d6633de87ed4b, 0x69dffbda3c96e10c, 0x5a7023f67463771e, - 0xa9464e300fa3dabe, 0x4586e20c1e81f68d, 0x5ef602e40e3ce9f4, 0x105b346e13cb0159, - 0xd41a41b36148b7f6, 0xf97b9da9aae8f18b, 0x622d84d4834b9694, 0x88d71b397ba8f2d6, - 0x2cb7c271389d7684, 0x358d69aa8d8c480c, 0x677ddad92cb1680b, 0xba8e742ddd90977e, - 0xdc2926f589df4e30, 0x3624452ab088e1b0, 0x6d7e07357ca38609, 0x63443779eb0d2d6e, - 0x64657567516d5086, 0xa5c3011801392205, 0xf53755b84cd48317, 0xa4acd51ab9a8f9eb, - 0x0378e32147d1b546, 0x85d4cb74e37b21c8, 0x2209f858ce00c37b, 0xf98cb3222ed30dde, - 0xb41d740ca18ddc82, 0x6ade887ef267200c, 0x0cda70896015d386, 0x4bf3411a364d6d6b, - 0xf8c3018d8136bee3, 0xaa008423de9ac1fe, 0xdf51e499171b706f, 0xb6aeb47d303c7b79, - 0x25102fc57688f92e, 0xfb7ebec253cbeb74, 0xa9fb0013df64226b, 0xd171b1830ec6824f, - 0x8d7d1b38fa6e2a51, 0xec3cbdc70cfd39b5, 0x1b3886546ed20f03, 0xdb084dc0851b220a, - 0x71bbb4655f293dc1, 0xf00ddd64ee909302, 0xfc4d0d65c18bfdee, 0xba9b612e5e39dbad, - 0x5b6862be2894dff8, 0x081c08e04cf49791, 0x50d5f49e937cb580, 0x39bccc7e4cae0bef, - 0x99de5c6a58727a69, 0xd32c41e45a4732b6, 0x904357aa1dd3dd2c, 0xca8a43442ab59df1, - 0x18b7776765b50ad8, 0x98116522e9f676b6, 0x7416b8f7f559367b, 0x533e9ce3cb525713, - 0x0af2515df86a47ec, 0x55e985b886cd6977, 0x6e5dd97c537ebc62, 0xf4f9619a5de6bddd, - 0x392c6625851c4bce, 0x14839dea100a5721, 0x9f3c1f23bea7f176, 0xa406023e7dcfc7fa, - 0xd5316683ac1a38f0, 0x40b8f19361a2a13e, 0xd274cbc897781e4a, 0x7f10b228038b5a69, - 0x68f923ccf3e2d9ef, 0xa8c35ddffd3a5a34, 0x11d27a6c38c426af, 0x535b4754b54bc6e9, - 0xa05d15b42b5ebf98, 0x8b7e965017701a11, 0x679341bb0146610f, 0x01c856425a92f5f9, - 0xc304d16e29966ed2, 0xc93fab3294672de3, 0x8b999daba35dfa80, 0x8a81bb7a90801bae, - 0x1f5027d000b9b554, 0x518f5ba4626a3584, 0xc5bde19251339173, 0xff16f0a26b741677, - 0x9b6c1537d8a84929, 0x18f8e9e092afd450, 0xa0d126f657e05d88, 0x77afebd1e6874759, - 0x6a29429703d4fd67, 0xa5ad221fe9ca0d0b, 0xdd29bed8e9d5ad44, 0xa39742f27c4534a1, - 0xfd08bc94a8e7f457, 0x11c7fc80a59abe1e, 0xdc2b9703135e0825, 0x0898251fd9e2dc95, - 0x53ff8554b2fdd6f6, 0x87e433e6cbde8d68, 0x8fabe397d30f7afa, 0xce26f9c439fdf368, - 0x85632ce9197ca924, 0xcfb5583b7ca94960, 0xd3f4509e53b1706a, 0x044eef09978245a6, - 0x017a8de65560cd2f, 0x3fc139c5472fd220, 0xee0c7a668ddc82cc, 0x4c96a0c47a36354e, - 0x90197192b6a3f776, 0x7e233607c24a3bac, 0x1bfb66406b08686d, 0xde9c5fe80333d2cf, - 0xd984b19133e4b0e4, 0x118cd3e6809bb85f, 0x888dba29cd4a5b46, 0xcd34b780289103c1, - 0x7c9ba07347001de1, 0xf25e13bab9f6b463, 0x25d864d3af66b6c4, 0xe15958c900d74ae8, - 0xfabf7298afd77b8e, 0x29f6b2f6edd141a9, 0x8f56d844ecdd8333, 0xab3156fe53775bf7, - 0x2f09162c87c972d0, 0x67c4b774a1e7070e, 0x82c6abc9d88e9893, 0xc2369c8ceb8f1aac, - 0xc8ca6d1d8c40f7eb, 0x3f0fd7d7ed62b1eb, 0x2e813735e047863c, 0x4c35aacd6e480a88, - 0xc6e2b14fbcb17156, 0x3469b5d0f2a3fa35, 0x0add118247926b24, 0x93e4a7d1f82c45c0, - 0x095fc482186ed81d, 0xd31d330870c445e3, 0xd74fb46dcea122e6, 0x74c2b98117c6011b, - 0x9bfcdfbfd0d8167d, 0xc963814e0d6c5c29, 0xef5d8ac69455dcb4, 0xf7364b71935eff2c, - 0xa013bbb0aec71ae2, 0x5a7eec7f9d5e6c25, 0x85f330dd23d8f896, 0x8ba3022f4c6ebc8b, - 0x2008f21331f4f4d9, 0xa54cdf610a53a1b6, 0x132001f9d56f73fb, 0xc97e9140e444e15e, - 0xa97fe1dc83a98990, 0xf45ff14a47328241, 0x4b34d8aba481e3fd, 0x7a3f1ca10cdf5a40, - 0xdc9b405bbfd0814f, 0x820b788020835e80, 0xf19750e9b757032b, 0xe3ff5d7a3561e968, - 0x1acff0e88cbaf8cb, 0xd123874709288703, 0x2e335930d2b0315b, 0xdf112c63478ea745, - 0xf7402dcb081655b6, 0x0c2f3c0e22a7ca26, 0xcebe2efa42784ee5, 0x04befecf0251265b, - 0x0346f971eda4edf5, 0x7704eca3984d1b93, 0x48a49d83ba73e047, 0x53ae948d8209e59e, - 0x2899f8030b388b34, 0x8afd14cd07ad52eb, 0xd72e461689dd2a0a, 0x83c80ef495210bb1, - 0xc7541330d8c5c226, 0xa0e4034785cb92af, 0xedd64d246fecfdd8, 0xdd0858ccdd335797, - 0x1d60317c0bb8ca73, 0x27ea5ebccc381d09, 0x2af059f7144028ce, 0xbdd15ebc501ec5e5, - 0xb04d20e2f1730c26, 0x8f03fb4992eae9e0, 0xd901f027569d5ce4, 0xd010b43dc13a1190, - 0x09401ac98f37a4f8, 0x990d554c37e32187, 0x22dcb5e4f5d29d11, 0xfe3ae311551f1811, - 0x974fa9c902ac64d3, 0x00dfa29bc2c037e3, 0x82bca47a7266ac71, 0xcb63c63907fa9622, - 0x1e9ce93caa3fafe2, 0xa6fe6ee291ccd614, 0xd896ea1e9cd030c9, 0x89075b2ff2c49992, - 0x0ec3be6bca8af72e, 0xd2be8b2760a32333, 0xc9630112b08fccd6, 0x8bca645b3afabff2, - 0xf5b761eabfc1353a, 0x5a8d4125ab9469f9, 0x585bf2f83efd0f23, 0x378a35fe9ce95967, - 0x11d886e74a1185ef, 0xe15c8e068fb1f6a3, 0x67787e5dacd993d7, 0xff41886508b6140a, - 0x32c3a1ae974aa761, 0xf0db76ecafdaf375, 0xa01f4020c993020b, 0xaad8386efc4dfaec, - 0x5cc6a071f1e478c2, 0x46385d09ef52a7ad, 0x0bb1055867a46e6b, 0xbab089233efed964, - 0x30ca374e4f536059, 0xbc4a84759f5981d5, 0xd0959d3b068451d3, 0x51a57592120c4502, - 0xdbdf49d437e21f94, 0x552a91c7fd2a5313, 0x008c122d577aef93, 0x583db63d1ea8584a, - 0x56aa66633122e785, 0x7b81b9d7a2343de4, 0x85f8862a5bfc2e9b, 0xe0ec5af65ad9a49e, - 0xb827470bc999a2dd, 0xe5a1b6de58cadec2, 0x8bc3210cf522e215, 0xbb1d4508a06389df, - 0x2da65f0c89057c95, 0x0635e2f876eda214, 0x32c089c2e59c4525, 0x65aae2a4fbb75cb3, - 0xd26fe36de090b432, 0xbb3430200684de4d, 0xdf69b803dcf40464, 0x82bb016dfe2bbd19, - 0xf3592cb0d84454d5, 0x4586a81aed947985, 0x69f7b55cceca96b4, 0x6664d540dbbe5254, - 0xd65db541bd74ad45, 0x84282a5b9dfd5137, 0xe770994b4450f34b, 0x30d667eda6a50cfc, - 0xb08aafdfcdccbcc5, 0x0c1d89471f8ce257, 0x78ae466abee996e6, 0x82349f4cd9b4e448, - 0x2e1aef1942946b6b, 0x21cc959c494417cc, 0x5a3264f726f1b308, 0xa558c10a11797c8a, - 0x2470721b786f3ac7, 0xb6b6d6dd0a1feae9, 0x4fe6dc5de6211cb1, 0x4b0cfa132ce7a34b, - 0x560ee61ecc692fb8, 0x6148a4388a215b2f, 0xddbe33f17cf3e036, 0x15f88203797f9c62, - 0x4cce3d6f9c1143dc, 0xef095d16656003c3, 0x280e43ac4b536214, 0x3d45da693d94a2ba, - 0x2159543ae2c0d15f, 0xc973882483403bd2, 0x41b68ed8e5daf821, 0x2cbcbbd05f0188a3, - 0xf31139fae44bf05b, 0x2f489f3c0282cdee, 0x3fd9d215e7426fb0, 0x4251d156dedd0993, - 0xbc1f23f141bd5b0f, 0x98bfe75598aec79d, 0x83be0496aa331d14, 0x242582f87dc5dde6, - 0x9a88dd7f377807ca, 0x3fc9736c1bfaea54, 0x9e989fc51cd840bc, 0x445fc7710201dc55, - 0x25040fa29be52694, 0x83aadc69321cc603, 0x4e22a2405a4daaab, 0xb85096ca58754c23, - 0x7707d0e70f7e275f, 0xaf27e2db7852245e, 0x40fce827ce2de7cc, 0x86acf8e7a6a048a1, - 0xf6711a9b1d4e3217, 0x7db9d2808bca2928, 0x5aad01901988a5a2, 0x7bb5e27bf2642b8b, - 0x091865b808b6d0c0, 0x76630cf009816976, 0x03aa92afc0281927, 0xf091637ddabee31e, - 0x3afdc45a2f8f4d00, 0x6dc84a64ecb8a918, 0x75dc5e62fad91d23, 0x81d78e540402d15e, - 0x3dab1946a77d7cc3, 0x8dee0abf063d9c3d, 0xaa7d434fc8e706ec, 0x8674394379c0c074, - 0x0a5395559686c784, 0x26b398f16f206d31, 0xefcfbd7c85ff8447, 0x9c3c50cc5187c9d6, - 0x90172ab213b4b0d7, 0xda685ea68da7a2cb, 0x50ce5244f2b9c20b, 0xd6386467a8a9ba6f, - 0xebaf29f7bc1108ee, 0x3049d86a1ff85dca, 0x2ae6b41f92b3ac3d, 0xedf8846334a84e8b, - 0xdf7f19e93c6854fb, 0xacf8d6d21e0b585c, 0x17057643b49a9768, 0x9016fe426f7e38d1, - 0xd735d577821e27c2, 0x3f88aeaf125fc3d7, 0x31678ceca42de4db, 0xc541cd062afce3c8, - 0x6f6a499ca04555a1, 0x8373e4fcc7b859b4, 0x2b8c421faea84efd, 0x7f739d875cba07b6, - 0xe2c34bf3dcde3b25, 0xb0cfb9f928d65f51, 0xce2f4da5cc3da70c, 0x5b9e57956c441f5b, - 0xa1e9ca22cbdac599, 0xe0ac43cd1efe8e1d, 0x7a156fed01d6cbb1, 0xfd2d367dbacb8279, - 0x5e641907df28e700, 0x1de69d9e89fa7010, 0xac3321fdb2cbd5d3, 0xf52a596929c4dc2f, - 0xfbd7a21edbf50401, 0x796fdce584a6fa53, 0x732e28288473a473, 0x9bc6363e6c66b961, - 0x831cb7b208b3ab72, 0x6a169a6ceee49468, 0x5df81d9307c4ad59, 0xecff644d460e0d3a, - 0x4dfe0f29184273ca, 0x17c29c36aecb3330, 0x18a122fdca9460a7, 0xc3a65ec6c640729a, - 0x9595af04fc84bc81, 0xf28b2d2cf226bf85, 0xddf1bf088b86c0ea, 0x322676ecae70d255, - 0xbed49ae658d21baa, 0x585f6226b231cde2, 0xd472226b689598c1, 0x6a6582b67fdbaf02, - 0x8e17569c50e63a0c, 0x715159686b5776dd, 0xcc2600963f3d594b, 0x924ae839ae9aeef0, - 0xbb59d75fd24d3a24, 0xc38e56d78e5890d2, 0x8a237600f093df0e, 0x825dfed0e4a3950a, - 0xef86de56c31aad92, 0x1957c4266cb360c2, 0x9ef7125aac7b5e28, 0x86957a61de8b4b91, - 0x90b57e5b0f029756, 0x583792e1f432a3a8, 0x62f6b76543feff00, 0x56795fa689d089fa, - 0x199e7c6b2e05533d, 0xa84baba18250645c, 0x9c4b05d0eb14f1c5, 0x69f15b07b5bf9286, - 0xa469f5a8b6aeea23, 0x6cd5a1b7828f5918, 0xeb55d37d5d4bb786, 0x6d25faa01b471db9, - 0x2b3d22f3d498d787, 0xea3a60ad303d515d, 0x20696f3403e066fa, 0x2301a27198b5c9d2, - 0x695837089fa4a83e, 0x1f00bd7e68c2af70, 0x8b15e276ac37d5df, 0xc10a70e693f7d8a6, - 0xe5ec58f7e48e1050, 0xafae364650dbc86d, 0x86e24480ef984c86, 0x6a9b1050811ec910, - 0x189f57620dbd5512, 0x453dd9db26ea4835, 0x8d0fbdc9ef5f2042, 0xe56f3092fdba13ef, - 0x0fa50771ff18138f, 0x23f9a3d48766c3d3, 0x6467e57eb883edf5, 0x3768ba3637a1ba5a, - 0xd2870bb2874c8cbc, 0x5435128dc1986e5f, 0xe00d72599dbb84d2, 0xf641699da91a8e18, - 0x8cfd9fd630ab72fc, 0xa0ac261257c30dd9, 0x27a05f434a5e7696, 0x4c4aadbe9c4c0355, - 0xeff82e8d2a3723c2, 0xd723bd26bb4ab52f, 0x6dd99f1c7c244761, 0x989e5ea4f6e12bae, - 0x3b3e761c26c126a2, 0xdd043f6c26198e66, 0xa96fcd5eea7cb782, 0x8b473095533ec891, - 0xc0e9bfe4a2505cbe, 0xd48db6a85e1b9c95, 0x9e1a2865bf8742ac, 0x06edeca2523191d5, - 0x22106c276bb43e9d, 0x02f8bd46a9447b1b, 0xc13766ef7a2e0c19, 0x6055ce44db717d92, - 0xe66212413bddb202, 0xefc730f4d649dcfd, 0x2e037829b51b6fb7, 0xa14bb217e444ea8e, - 0x6ae91a0b5271683f, 0xc3a65afebd57e866, 0x4869d6687ba0a1a3, 0xb9f87d438fc900ca, - 0xb020aa83a5bff9bf, 0x5ab0ee52cc38ca6c, 0xaef8c83a2d5e1e7b, 0x23d37909de07935e, - 0x6115722ef9b4bcd6, 0x0e019ee2284ed88e, 0xe9d192940e1c0995, 0xf41cc76fa94b56fd, - 0xe7316977d8f0e1e3, 0x3a3eb032dffcf133, 0x96616d18ed84ffaf, 0xf2449df9bf495f33, - 0x837517b4a33e9233, 0x6b130eccceb16a62, 0xfd524e00fd8b89f5, 0x059681b1b137984c, - 0xec55ec114f985a46, 0xdadcffbee9e93170, 0x8493565ec4298bf8, 0x67663276b7dc8e57, - 0xc611c696a27da8d2, 0x1a140f38a66be0d2, 0xa955d1f148cf4c51, 0xb8049aeb334c5d4f, - 0xfae3195f20a51825, 0x93e82e1b5ad618f7, 0x7bff8157c94302f5, 0xb40cc0bb9347f36b, - 0x0f7d12e529402a21, 0xa2a6f38120935cc8, 0xabffa6b58a8b6d97, 0x110b334586a0effd, - 0x0d2008928878ff00, 0x51ad6b4f06ba05ec, 0xd4b2f05062eb80e7, 0x0668b6741ee47257, - 0x8021a2b19e1303e3, 0x55e75c7193aeffac, 0x8176ddca9f114a47, 0xc57eb42277934adb, - 0xd70f34e05bf32b0b, 0x26b3400898fe9c86, 0x542566861fc57fed, 0xfb6b3f9c59129a6b, - 0x1905e0c8a09d5570, 0xf98d2e87bb76041d, 0x08da2dc8907e3c51, 0xa156a34d4bef0c97, - 0x01ff77040f7a82aa, 0xed4059fc4486e33c, 0x28c0d2c3e8e161a2, 0xf813c406442f805c, - 0x00d252cd601bd475, 0x3f721fc0408e0e0f, 0xa45b140bbaddb6da, 0x2109d08354ebe60e, - 0x46b5fe8432c10ad7, 0xdd00f7f6a0a9cd53, 0xcd2e7488dc60fe66, 0x0503f646f5334888, - 0xd04625d78650200a, 0x9c4e86d7c5dee07b, 0x92bcc468884bb3f5, 0xbffb2cb24b27d36c, - 0x440aab66e17c5596, 0x4427468ae7770e3f, 0x0b92e422d32f73b5, 0x2da47f1060a57327, - 0x861b4ee0a33f490e, 0xe13a977be37ef5ff, 0x135ddfa90e432924, 0xd7b16859f33fd28a, - 0x2cf74ce9dd3984dd, 0x2d0414095551562a, 0xf71809ae7cccde01, 0xd8e9a12cdfe24102, - 0xee8fc3ae2b86b559, 0x6d77ce1ba5d1966a, 0x3d24ede68287ef48, 0xaf3f4e4cdf50402a, - 0xa2f943b82f2dc17f, 0x9676743a3737affe, 0x0386ad14daa79b25, 0x2be1d05b6b59fc37, - 0xfd62b81a8bccc759, 0x6af6920b60b30ba9, 0x15406d48c90c3899, 0xe99260f48f756349, - 0x741215d55a5edbc6, 0xc0922556054dd7e7, 0x3539f0338a3a6a49, 0xb086aaba5736fb5c, - 0xa4e93e79e281dc96, 0xa6d280978f9e6abb, 0xc8ecb766e5c0b9f5, 0xd1debdd51c2c4be7, - 0x0398e82d28b0222d, 0xa1f4e03158939bbe, 0x0dc2fc6d3eae1ad3, 0x525d2374a8884d2a, - 0xb09f3817b5601a95, 0xa69bb7129d67c481, 0xb8365b56c5cde395, 0x5fda8e56c8f96a12, - 0xf0cf05327c18c60a, 0xbaf0c4cf91ae28fc, 0x8c08f1ffd788d990, 0x0a063f0ec21b43df, - 0x6e3d394ef7a8c3a6, 0xba8406488a834a20, 0x1878a78e7eb757d1, 0x54bdb3e9cebd6f0b, - 0x5a59da503711a3de, 0xf33450901d888b75, 0xe2ecfc57ceac7ccd, 0x3607c106a9c3147c, - 0xadd154e3a4c63f11, 0x77ae2042c6a49da3, 0x05ed7f87939bc27a, 0x2b8db4a0b0663763, - 0xa55cecc1e045fc97, 0xb4f731f2e77fb9a0, 0xf0de9477fa8b7716, 0x51d12aa191425444, - 0x9269b47ca7c091e1, 0xb9f3637e5a0cbb24, 0xa6d8dc6afa015fcb, 0x6df02641445b634b, - 0xbc9dea4388f7fbd6, 0xe3bacdf999444007, 0x2466ae8df34cb8df, 0x78b9a7cdcd780fab, - 0x6bd239e7bf358251, 0x5fb3ec79dd3c8150, 0x3479966b02cb5e74, 0x5e058f86ff22035f, - 0xe28a3f5c11ebea83, 0x935df6d1fbf5c9a7, 0x0e7f1b6853bb8b9f, 0x7ae41bbc703f7af0, - 0x905ba15b6c83c3fa, 0xad89bcb8323013f3, 0x85cf1658249f9117, 0xa06ccd00aaf6946c, - 0x00da5de225bd1822, 0xf9cbc394a9b8abe9, 0x19bb8bc731986b94, 0x23e4358bf16dbbe0, - 0xb9f00590075d84ab, 0x34bc1cbf7516d73d, 0x62f4f9c6a57d36c5, 0x45bd14fb72e05088, - 0x424db19529da11af, 0x6272ae67e40f11c3, 0xf328c97dfca0cdf3, 0x741280f70c408847, - 0x7a21ef0a09ca3f74, 0x7bffb6033754bb8d, 0x4bd63970e8e28e68, 0xa2fc3192f9bd4a1c, - 0xca3c94b5bac44842, 0x7da26633e1a69555, 0x98095690e9dfb8d6, 0x7cebf7bd4e03f227, - 0x591fe9f6f7c57987, 0xf67e6bac21ecc8ee, 0xebf64b48bf0b7081, 0x40a57000b8acb1c0, - 0x0f3bdcdb7fd35527, 0xf4b48cd78dd77361, 0x3873a90e1cb47267, 0xc142071811362830, - 0x64d463c7c6f36eab, 0x24cc5c000bb9c4c9, 0x801ab019b494f8fe, 0x60ffdbe3f237b6f6, - 0x0428cb42ba704a79, 0x7e37d7c39bd3be3d, 0x06444497c73b5999, 0xfa58699ee3c77811, - 0xdaba09e23eb7df5f, 0xe02a645340cad7b0, 0x1a706cef7b97acc2, 0x92ae32d7a45396c6, - 0x8642d5f1799697bc, 0x0e8ba43b9aa5a1f9, 0x423df6548c7e71a3, 0x4d74c9f752e307d9, - 0xdb776998cb1c1b60, 0xead5079b71690f1d, 0xbd4785828f035d8a, 0xa58c066600b64be2, - 0x2ca9f21d1bc222e5, 0x6aa6577f4cdcc8d3, 0x0f9e36a998b2ddbd, 0x803eca39b4ec0435, - 0x9611cf589cc10781, 0x4f08217aaf31642b, 0x09677b7ca2444103, 0xd3bd98fbde813c2d, - 0xd0c7155385c10bc2, 0xd6d2a2aa01e517a4, 0x4f2d72cef5c860ee, 0xd79b13d5b67f1ffc, - 0xa06cbb17581bd8d6, 0x52b6e7246a046b7d, 0x10e13e6e35256845, 0xb30aad844e61ee16, - 0x2456fda36541d43a, 0xefef2aafed01363c, 0x4b71f214a218b3c4, 0x36c1716af64cab86, - 0xb837353447ea0af8, 0x3fa4d1428f39fd10, 0xf45d5bf1b114c8c1, 0x49d24519283262da, - 0xe5c69ff535678e18, 0x111559430c0dab4a, 0xf23fd35ed071cc0d, 0x9281b41c73e74f08, - 0x71222410af683092, 0x0021fe2eae1f27bf, 0x3235dc80213a5ba0, 0x9a86c142ba1d556e, - 0x70761373eb5a1973, 0xe79dc5ea92dd3ffb, 0x5e09f65aa4b7b130, 0xa7c0dded1025c808, - 0xaf0dd7639cf34395, 0xab014049b8c36760, 0x6795656e53d283ff, 0xc6a688160bc12356, - 0x491abf9ee5d6eef5, 0xbcd395505ece3fd3, 0x6491967585be1bb0, 0x036da8dbbfe0813d, - 0xf9b0da016f74622d, 0x9ccc375a83725472, 0xde264b42d1acebbd, 0x08b55366f65d8750, - 0x9f33eca6e36eaee1, 0x6c69264dd6b1786e, 0xc8ba6347c5299510, 0xf6be6e1bbc730900, - 0x3d0e71f2aaef8a61, 0x69db7db8cf4a51cf, 0x17b2eb5455877357, 0x9f219fc4c1cf6386, - 0xdaff6fd5a44f25ab, 0x2f09b62a2d707eed, 0x0a713782e514055b, 0x3a5b047015f5e86a, - 0x24a4b24b603121e6, 0x5003a509f76cb109, 0xf75047311c2968be, 0xf5a8923465199c4c, - 0x9fdfa7b1c66a7889, 0x376d243bbe32bd75, 0xd6027ca75658f114, 0xa6fd2641e6891e00, - 0xb43e4c693aea9c70, 0xd45c0ec98ddfdc54, 0x9271020c1d4d1652, 0xeb6768a4eda07cc3, - 0xea1d572b753a9b9c, 0x961e3a2ce62ac9fa, 0x380bec519a20e828, 0x084a5b8078753f1e, - 0x4bfd323ff235614a, 0x7e1826b7e2a4f802, 0x88b494a58469b096, 0x2f27a05c881db5d8, - 0x3bfad091366abdb7, 0x3458c6f0d466f9a7, 0x9c6400d1423a1d87, 0x67109c5acc7c9ec9, - 0x1db3cdff4a7d8a3a, 0x43ae8e8f7e4cbf7e, 0xe70cdb94b4f97a32, 0x838150324d5aedcf, - 0x0011da9fdc8db66b, 0xee412946e8b65e92, 0xa776838cdf5e7498, 0x307d10638537ad0b, - 0xcc9475dbb5fd8d1b, 0x30bd8c901d1cb462, 0x117f0173984e34ec, 0x358225662e17f13f, - 0xe16fb0e348d90dc5, 0x63b6a2edfc4db411, 0x3f02138208d2fa84, 0x807e6d4e08e1bec9, - 0x36ea4817580571da, 0xd80f73dfb1c53d46, 0x0811d27aeb8bd1e0, 0x18cea70472f18b00, - 0xaabcbf3a98956a64, 0xe53cf745e7e224f3, 0x5169a52ee170aa3d, 0x43939fb1d5f917b2, - 0x55e6fa23eb41ce0f, 0x66ef1d5b16bfb849, 0x74bc5078e86ce7be, 0x4612196f83fe0295, - 0x58f3443f9bd95107, 0xe1b6514110acc4a0, 0x02d2f7c45959935d, 0xd1ea3cc94d35a619, - 0xda6bd344c4cf297e, 0xeaa73a676cf3ac31, 0x19902a28fe1a681f, 0x46bbd3d7bf2988e5, - 0x038890e53f17a35d, 0x8e20f907700a5f54, 0xa25548f72721fa0a, 0xb89fe306a395a494, - 0x56dd6976ee8961fb, 0xfccba90313321dd5, 0x278641d837c426da, 0x8a5966c4aca81763, - 0xb6ca52ed6eda47bc, 0x361d523d8b994b85, 0xd3dc0fffe6bf6d55, 0xa793c574f6d82bcf, - 0x0fd982f75634e516, 0x804e4d8a7f32de9a, 0x20b95e9311dd04b5, 0xda8dd325dc323d97, - 0xd3a5fea0b8a7ea87, 0xb9b837e9b1e22dd3, 0x88620be8c24a55bc, 0x7e35eeb871f3808c, - 0x48ed91d621135926, 0x8e2a9442f653a8a7, 0x18252919c34bf7df, 0xbb4b161b875c92fb, - 0xf77c5f1afde5eeff, 0xb0ac415c7a44b1eb, 0x4f0f0d83f2552cfc, 0x96c5656a26c7e49a, - 0x18f65b433071d6af, 0xb21b265cf4716174, 0x4b0af600eade3682, 0xa2cab40510636435, - 0x4e5c932cc68f7a25, 0x865e091044335e2a, 0xb4e818a5fdfc67cd, 0xc3fba0ef0e01108c, - 0xbeb68bcc30d55fa1, 0x89b8a21333a81f26, 0x649d2ff63767230f, 0x18d014adeef9923b, - 0xa488f826bf518963, 0x8c2bf807b6720384, 0xf1cdecbb1d1a0053, 0xe2ffe1ce595470bd, - 0xdbdedd5c2e4b8d22, 0x82319bf31047ef0c, 0xd27eb197bdc40e2d, 0x4e946c842eb930da, - 0x51c1f990cd488231, 0x89e4835cebff8cc7, 0xc664c5adcf0bb5e8, 0xa7d7abc311bfb777, - 0xa809df4fbc098c18, 0x2278ddc92736d640, 0x5bf517f60011cb3b, 0xa8166816ad81f4a3, - 0x677de8fa3e0950fd, 0x70945372dbc8a843, 0x841add8bb5292f82, 0xfc3f92bb85674672, - 0x572dae5ce80170a9, 0xc43b7f5c826982c8, 0x54ef9b0427b8ce97, 0x7c4787a4cff5e845, - 0xf4edd06221bd0781, 0xca1f7ac267f5d972, 0x977444da04544717, 0x01a3bde49b885150, - 0x5a4c6bf8de8eba0c, 0x43d19e11d079d21c, 0xb5ce0f863343552b, 0x24767aa1039c3186, - 0x334726760c256dc2, 0xf2699e81767b882a, 0xe33735a5fce07535, 0x5e7bcd0a0b4833b5, - 0x8a1ebe9f04da7755, 0x4ccdbc7e5ba1a501, 0x6b030d885b50a549, 0xa269e97049eccce4, - 0xfac7a1a65195af42, 0x5a99168aa81548c6, 0x5f5b28dade633e2f, 0x4eaaba4a903cbc36, - 0xaaf3813c6c7bd0d6, 0xb63cf9789d4e3ed1, 0x0d5961b4ea16e73e, 0x7cf838150fda4a8a, - 0xcf0effca9543a364, 0xe6fe2dfb72eb95be, 0xd451efc89643a616, 0x98249eceeca42aa4, - 0x03d3859e983ee29e, 0xa0c5f35563abfa8c, 0x1835c606f4530e4d, 0xd89e23f46dab0703, - 0x097a16956bc41c99, 0x7c1fc2a3c9294712, 0xbd65cc0c0c7aa45b, 0xf4dbbd2ec2de9018, - 0x0b4a95f2fd75ba2f, 0xd22ddb46aea9b5b0, 0x2ffe87e160450eeb, 0x4a160a5bc7dbe57e, - 0x4a3d521fd2faeb35, 0x5163f4ce0dd82b99, 0xd876f3fea50f392d, 0xe3810c9c45dc9719, - 0x7a724a8a7caff212, 0xf510665a4d84f979, 0x6fddb5528b9c7293, 0x25232da0767666f1, - 0x3654883a54cf8251, 0xed1e42157c12aadf, 0x9914fa7100252620, 0x6a5f2ea4f2592cdf, - 0xa19dbb7c5b7a0c2a, 0x8351f228060f8b5e, 0xc9875792c4da3f64, 0xb71f5d2d9544f2a7, - 0xcc2dc9f3f91639dd, 0x7312b20fd1884024, 0x61f5fba2a1ac44e8, 0xe32d9efb6ba7bf41, - 0x8b504089dbde590e, 0x020fd96f94245b4d, 0xa0d1fa5174172893, 0x30e739b7e0139f00, - 0xce5a0c56ffc722c7, 0xf4ab7115c235cd18, 0xbf77b6fb64cd6c72, 0xfbe71e93e4d5aeea, - 0x6fc83810dbd3772f, 0x324665c12c65f909, 0x9acc35913a68c452, 0x7184d85203ed1b67, - 0x6b13cf03ac66113c, 0xf0e187d1d6411359, 0xddf76ff0f076dbf0, 0x478c361636d84f45, - 0x37a18e71e89b819b, 0x642ab6cb83519f78, 0x3761d5b5716ec931, 0xd5f2ba722134b7e1, - 0xb1f7d7734ab04bca, 0x894bb458296adc56, 0x3b177a5954823584, 0x48893f5f044cd931, - 0x0e35f8103d17f4c1, 0x4a771c9a0d87b6e8, 0xe92a4692ca951b33, 0x6b6e08eb8cee9eaa, - 0x4e5f7f611fe218bc, 0xc78e5e746ad2e1b0, 0x61dc19e66c5f2020, 0xedfc2453e712d885, - 0xd6c6a7a116c3f434, 0xfd43b0cf591b1005, 0x2fdb45c9307fe0fc, 0x8d8ff6fa5490d31a, - 0x6fe72823341b9d3f, 0x7325afcd3523fb65, 0x2ed780dfec0b53a1, 0xb9bd9303dee7c441, - 0x90fbf652605b1f69, 0x1cfa8400dd6f0f7e, 0x0eff86965a7803b8, 0xdd8a82dd9896d1d5, - 0x97bab61bdcfa82df, 0xceb692328e69ca54, 0x1fae73657aebca8b, 0x09a2b65035b66593, - 0xfdbf6e82c1457775, 0x9c44dd4207c3556d, 0x1eea947a6b7f498d, 0xde16f234b1569aed, - 0xe30dee5be89ef784, 0x2f5b2d807d72c104, 0x3f926318132a8e34, 0x36d5aeb1a76b309b, - 0xf804d55b16948840, 0xfc0c7d4c1eab6322, 0x7c90c32ed3bfaaca, 0x774319bcd50b813a, - 0xfb4f7321885ebd4f, 0xb8cec5d13d6fab77, 0x5e58ef435ba6ae0e, 0xe45b90ac494722b2, - 0x03b25f7fdf9d623b, 0x27062068d0668e18, 0xdcd6ab1eaf04d664, 0x0da7c297e5de6b4c, - 0x9229827e291691f9, 0x6cb06927990a70c9, 0xcdd20c9aa5eead65, 0x596654e45186f44b, - 0x471f545cc5cc16de, 0x3014fd7e3190d5bf, 0x10ac3bcfb831aeb5, 0xdfc943ad3cb13b79, - 0x262034a90d124eba, 0xcde6c918d3afd160, 0x9548b990234f39bd, 0x1b5ada02ceedfc03, - 0xcee63ee0fde3762a, 0xf3996803f03aa77c, 0x41a35cdab854ef26, 0xae2cfa952e691b6f, - 0x52ad694bbb2ab64e, 0x734801fd0e38bb47, 0xd9f03f6b5c5f9a54, 0x2abb04f54b216858, - 0xa0847afd98da5814, 0xf033bd0e08766395, 0x7079bb31acab72b0, 0xfe5a8e9762f3fa76, - 0x7eb654b3df13cb36, 0x384f87e8554048cc, 0x1a1ee2db5fc3fade, 0x2fb01bf6e736953b, - 0x5657fc2db66b5b90, 0xdb6562d4dbea5830, 0x3d401d6550b7bf37, 0xb1d6b256b2c14a09, - 0x5152cd0d3b516161, 0x081b92b859dca783, 0xe9aa2fb077c1c725, 0x891841f7721bc39d, - 0x96d77cb745ce854f, 0x2833bf83bd7ebf8d, 0x3888d223822eec52, 0xd6d6df3487693d4d, - 0x129d819ddb0eb92f, 0xf43e9ff2b52ffd04, 0x33fe019fc1d6bb2c, 0x7efaead9c3f4eaae, - 0x67590d94cbeeb19f, 0x9b11beda6bfb9aab, 0xcc87b4e6fe37c5b9, 0x7f00133d26c5c9a4, - 0xf14aedec89f95632, 0x555e9c2755e49a92, 0xf37fbd4e5efcf585, 0x1c5fa1fc9e8996f1, - 0xc26a6e208c82f436, 0x3cb64b62590f9850, 0xa2e430772a99f913, 0x80e4db30b6a685e0, - 0x8eb787947bc5acc3, 0x44c94fce35c5d71a, 0x424832600f0cd2a9, 0x47937104efc9cd13, - 0xe8c12bce3b926f3a, 0x6806d5ed2892e6ab, 0x6d3f6ddbc4fc889a, 0x0ca3cb42fb90a983, - 0x2531f5ec154bf791, 0x807416298ffc69f3, 0xa2504d8a1e85097b, 0xbcad351327defea9, - 0x226bce3a141d5c4a, 0xfcd6099518bdf4d1, 0xf1d7469848fa3286, 0x39ca944a4e5beeb9, - 0x365766e125dbf7ef, 0x4923729af4a39330, 0x7ef9b8ef1d820362, 0x46d915bc8cac7217, - 0xfe653ba0cd05973f, 0x26e918c661ece05a, 0x0b5f250c67807f91, 0x04446de867b2d67d, - 0xb8b8e15aac930e2b, 0x127ccad58e203412, 0x9cde938e2afd7691, 0x4a269e6742f9e29e, - 0x626c62b38cef6888, 0x4c2cd6611c363492, 0x67610970909f3b28, 0xca931852350107bf, - 0x03b6074f50f8b5ac, 0xfd95c1157187c48b, 0x8529cd4dac4cbff6, 0x89e6f64c73120f94, - 0xdd85ba4f8c6e40a4, 0x1de26f3101ab596d, 0x3c86925c858ae8ee, 0x8f4ebd29a844ea53, - 0x32cd8c3491420b2b, 0x15797284be363b19, 0x10d48e3e8647803d, 0xb46b8b687e9d07f7, - 0xda65dec41a863152, 0x2aa08ccaf0765bae, 0xaee27fe7bd31421b, 0x71b30176617c9faa, - 0xf56715e8429ff98a, 0xb77de5f0805edcf0, 0x050c241c4689c2c6, 0xca7e62e358420209, - 0xf1dae48caee3a894, 0xb6346a5d18a84267, 0xcceaaef96d8b393f, 0x0f2697c54f4bfa4d, - 0x19361d5c9245630b, 0xe0d245bd737d29fa, 0x1d52512e1913946d, 0x106d4603800628f4, - 0x4da7479d52d7e546, 0x797c4676ab51c438, 0xca5c43912215ac5c, 0xc4db220e685e6d32, - 0xe52c016ec199e5c2, 0x9b7baaf2a8ee7217, 0x9826ab0f838f01db, 0x86cba2271912032b, - 0x56262063552d2b5c, 0x450bb23115c9ab54, 0x55a0de29091b54ca, 0x5342f55e7da91227, - 0xd82e73b723dd264b, 0xa933f9d1a0f67a82, 0x6d277f6a31b49958, 0xcdb3cee9a397f84b, - 0x2f63527af49b1257, 0x23cbeb614c613c62, 0x00164bed84146ba9, 0xa40d2efe8f8055c5, - 0xab261615d5216360, 0xdcd5b45dd0f68918, 0x5e0480ba35f9dea0, 0xf729a9146b75a1c8, - 0x94851a59a5d6d12c, 0x061c60fe1539ddad, 0x48775e0ec16a0c93, 0xf04acea34a515c70, - 0x836388b897cddfdd, 0x8eb2d2bcf8972d77, 0x85e5fa032e97281d, 0xbb6290eac0e1fa2a, - 0xdffbe29d75addbde, 0x5431d19b71b702bd, 0xa5b945a6bbf8da6a, 0xc82e11ad55a720fd, - 0x170def3fa204789b, 0xe787dd4037dde6c4, 0x0992436c117d8c39, 0xc57a79680a2c80c5, - 0xf44d2627cd5eb3cb, 0xaf8c0bbbc5125db2, 0x588ed49ae87f854f, 0x3995c10b20caa137, - 0xc857cf23ef261b4c, 0xf568311d9a422fc6, 0xe50cf5df6749202f, 0xe4c468ad210199c1, - 0x31846b35709ddf04, 0x58aa160c7b0d50c6, 0xea94ae873e4ee8b3, 0x0c8dfc4383ebdad6, - 0x1684c89fbab171df, 0x3b883b7099959c09, 0x047dba0d5406c815, 0x5795ea6dc2dc8a0c, - 0x219872ae9c3f5d79, 0x90ea8b0e0ddfbc74, 0xa44321c7bbc1230e, 0xb52ec17124029687, - 0x82a59fbfcecbf584, 0x3c4008fa3c17e00d, 0x4f4bf18bf5f609e1, 0x48ca9979ddf51b9a, - 0xeeb48d291fd2f6c0, 0x4a60af2a44fb8655, 0x10933220c39017b7, 0x0984db572087fd81, - 0x48ffca03edc8fb5b, 0xce89021953bedb7e, 0xd8a6eaca6ba02589, 0xdb36f70df6cb3d6d, - 0x629c750f1d0a6112, 0xa9652c0d96166ccf, 0xde07a483612f6ad0, 0xc37397912e8d5369, - 0xe5680e9b468ba2bc, 0xd68ec53ed29fdd90, 0xfa5186d2ebb35d3b, 0x371f138509a72205, - 0x448f17d58d9a6982, 0x67a9ca2c02b63e35, 0x9319abcb6970dfe9, 0xe24b30d965a3bc79, - 0x2482990fa14da633, 0xd32c1e4d2a1daa74, 0x0c500f5d0d7a6047, 0x22e47dc2fde5b900, - 0x537691218cbcb017, 0x8f41baae4155dee1, 0x5b79955a9ae9f47c, 0xb13d43be52d77579, - 0x2f8ac58e8478304d, 0x4e0d3845dc5a057b, 0x346f87cd389f03c7, 0x2a8270467253786a, - 0x02d016306f0eb643, 0x665b17110188c8af, 0x0eea6d49bc814866, 0x7634fd52f845a73c, - 0x74f041b3b7936198, 0x544e02bca5e4d0ad, 0xbe9d2fc71cbc0b66, 0xf28362e6d6bcdcba, - 0x5bbed1a027a90ba8, 0x1b38765f7d10f6a6, 0x7d4bfebb137f62cb, 0xf55eaaadab859e15, - 0x53a63d8691512c24, 0x6620bab264cce94a, 0xec8e6cebc0ae96da, 0xa00dfde6d4e760b6, - 0xe8e107f176f23eec, 0xee85fcbceef2d99b, 0x5ef324619c9e86ae, 0x035203484036ccdd, - 0xa00a56ae95798d88, 0xcdb835a27ba6201c, 0xdb5c2e269d1e7c2c, 0x996be50fa8a6fd2a, - 0xc768081deed8c76b, 0x267b83ae8b33a151, 0x280e675bc71b7821, 0x6d250265f629af68, - 0x47dc097804b80f1d, 0x338e97fe29ca9612, 0x08b6ab50a563c217, 0xdce0b5fc55bf9a8b, - 0x28966b5b58e37c49, 0xbb61f381e6dd08f5, 0x984e1f451694c1f3, 0xb063d92809efab8f, - 0xf0cb696ac9cd8d5c, 0x495cead9d9f305df, 0x8542890ccfc7cec9, 0xf8b4ddad6a3470bb, - 0x5b9376d2d70f7170, 0x8354fce1357452fd, 0x8f7396213f4940a0, 0xd017f9544ff07e6b, - 0xc98e216f4345840e, 0x96ec9d0d96ea2af8, 0x608387188fdaa72c, 0x23223ef5dfae2ae4, - 0x821fa908a093f579, 0x994394362025b8d0, 0xe98ccd2b79c3bd28, 0x7da17b7fcb930dd5, - 0x1506ad9e4f0b6132, 0x4db8ed055d7acbda, 0xbf9bbe7a83e70b8d, 0x0b1450bf0faf1a79, - 0x89635ce3b156569c, 0x790fe7894c11d247, 0x31fb3c25274be716, 0xeeaf58e7aeed9b4b, - 0x85756374c2348f61, 0x746960064c140d29, 0xb219fe43d689622c, 0xf6265a1ca4ee2f52, - 0xdf309d0b7e38cb72, 0x177e69796db5285f, 0x8236ef5dd31b8858, 0x97d93a73078961a7, - 0x5bd23396136788f5, 0x6986739ba20166a2, 0x83eb5c797a59245b, 0xf2d3b7276c17ec65, - 0xf8adc62e69e580a8, 0x01cc76f68a24315d, 0x6f610039c8b03937, 0x1b07d2e897508fb4, - 0x26bbbfe5719f9184, 0xac3322b6143ddb44, 0x412541cd74ce1df7, 0x6e3c9e65639f0979, - 0xc64afda0c2e3cf09, 0x6b5eb4837aef853c, 0xfc91fc6ec0cf4d07, 0xb4435cf79ea6d6e1, - 0x8f87aa85856a1ff9, 0x8d5919d0266b42cb, 0x230cd605a3c7f08b, 0x7e65c7d353c3488d, - 0x1fd6b143690312de, 0x46da922b22ac95f1, 0x2342b43ade639535, 0xde9c0a1a076883f3, - 0x60a1756999d15f5f, 0x0f2cdb7e8901efc0, 0xb4f7ed9cf0046990, 0x5b1201b980a2c72a, - 0x0c2ec9f6cdecb1eb, 0x32dbd3b2810c5662, 0x23b753c42668271c, 0xa01640ff8c8739c8, - 0x5a9199bb5728092a, 0x25d2ca26d9c53b26, 0x44f2f8284005afa5, 0xa1db4518f574ba86, - 0xce859416b8c797e0, 0x1c4715a2147b78b9, 0x2373dfdd54680621, 0xf1334b2aa238c807, - 0xfc1447390c542407, 0x93f2cfab65f79aa8, 0x5cb0dbb0b71e1e16, 0x519a51074f2a9f66, - 0x46390e30b8513a05, 0x7c3abc675e702bdf, 0x8eafa9b485b7e4c5, 0xcd2fc931d0dcda64, - 0xc102d501bb1097a2, 0x8dc78457038ae8ea, 0xb63ca7df30fe9f8d, 0x2d8e4ceb5eda157a, - 0x2b77fe5b17d3f27d, 0x2dba3106d8bf046e, 0xb7152efe034ebe33, 0x5c03bf7ece3590f0, - 0x00fd93cc6a097947, 0x53eaccaf4fff8f0a, 0xe8b0a7f8d78cc825, 0xb0a23fad2c489566, - 0x3e7c2cd6784abe6e, 0xff4c39e245cf6f8a, 0x24a1d91d4f294d1a, 0x136eeb7c6d3b6189, - 0x215895ce94547fee, 0x09fe927293b292f0, 0x56e5982ccd4a4293, 0x51e889dff0ee5e43, - 0xf7a70d9382e5e41e, 0xd162f758ef9f0e55, 0xd1c13865c279058c, 0x9a9554e1be0d9f1f, - 0x99e2672f45e7e545, 0xad5b6d2e1c0e41e3, 0x859961fbb4194557, 0x38ff0c816fae2c6b, - 0x2fb6d6c5e82d5a4f, 0x6a0c935043f7b94f, 0xa0a08892ebcec506, 0xd2301ed516ca7a6b, - 0xbfadba7ed810b49a, 0xf0ef116416cb6524, 0xd79def8a7e5ea8a2, 0x0233702c2fa37308, - 0xdef86fa215c6c23f, 0x887d7cbcbe16969f, 0xf686f8fe0f0aa7c3, 0xde872eccb17a94d9, - 0x1d27e081e38efa1c, 0xfe79062cdf55551a, 0x67cc3026b54d63ca, 0x2d301a5b6d848131, - 0xa63be877b21e4889, 0x25bfea0452370cc8, 0x194d5916b50260e3, 0xe59ac425deb6a081, - 0xfca24476093832f1, 0x303d5d88d7167cf0, 0xa6a30243def4103d, 0xbb10b57795c11dcf, - 0xe12e01f0181e6a22, 0x6ea668816ca767db, 0x2f70a589194823ae, 0xf8cd73576c460fb8, - 0x7c08658f0e65b5c3, 0xa13248b96e31bc97, 0x2189ad821c3dcc22, 0x7cc1f02d04e36b93, - 0x4fe14f7dbc2bb07b, 0xb49daf2f1573c7a3, 0xff90e90c59b8a003, 0xf1060688d2ce4ab7, - 0xed13d1e16cb0ba0a, 0x13018d6a51075d3d, 0x784513f3d2b36a95, 0x6aced115f206b8d6, - 0xc279e0c9a8971fbc, 0x77c562d116835fd5, 0x094429f005e55b30, 0x7923ddc3cb1c8697, - 0xdffc48e089a69fb0, 0x38424f42639d649d, 0x34c7541730be264f, 0xb7fa5db0e79cf118, - 0x3adc28a4c6b50c52, 0xce403a6c98e167ca, 0xece839eb995c4f8e, 0xd40200735402aca9, - 0x06ea2142714b6ec5, 0xc856c08811220f1e, 0x7c3a980f7e42ae8f, 0x5b48d9dbc4bfc58c, - 0x9c7b3294c402e28f, 0xc8f981ad2064dc5e, 0xa965b16e5dbf0e89, 0x3c606d0e5c43b1d4, - 0xc7036a98b326d1bf, 0xc03615932c426a1f, 0xb46d6cffdbaadd17, 0x03a92f24d97fed8c, - 0x5b82141391dce7ab, 0x94e6c4f4aa30e9f4, 0x2881be22f97b35ae, 0xd697ea2fec9678a3, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x792a5be3393fbf15, 0xe0b9865761cfdf77, 0x74f1c27eeb3cafe4, 0x77d193fef13edcb9, - 0xe7f7c64cb420b8f8, 0x030a0480b9843eaf, 0x828c2d9d27fd1ef3, 0x8a354c4212ea5ff9, - 0x367fde4141a343ce, 0xb1b932409a6c4f24, 0x204218453911e128, 0x982295afe9c5698b, - 0x634c3c14821e578c, 0xa70197b023a501ca, 0xc239f3196849921e, 0xccf6b6247c8b030c, - 0x8a55e6d30aaddbc2, 0x005a4fb89dd2bc3f, 0x94fc3fbeebd6421f, 0x058e1021bdf4efc6, - 0x5a4191bffe8a18b1, 0xaf49aae1b9b12749, 0xdf9f5622da628b47, 0xca4f5c8da9fe0113, - 0x196a842302bd8d72, 0xa971733f5e59afd9, 0x60bf28c68a13a7f6, 0xfb9a88a4e980844e, - 0x7b6071e0c7fe4c10, 0x336667a3a939ac53, 0x1a1e64860ca2c93b, 0x019e586a940e4203, - 0xd2c212dd676efff5, 0x551ef7906b62c4a7, 0x9b08fc2b5c0082ad, 0x585ec19b057b1eb3, - 0x8adba1a7acecac55, 0xa8c55ddfbfd30db6, 0xa1c4de6e4f2e139e, 0x7a00e079abd74ab8, - 0x28c5a27849a84e93, 0x34a1a9e85d8f8e3b, 0x4743b46cdd5caff1, 0xc3e8943269df5215, - 0x19347096e4ec78e7, 0x901fc8bdf1be1a2b, 0x265337d60231efbc, 0x3962f80e26bbf4cf, - 0x63b6168ffcc81ce1, 0x1ceaea2301ed6ea6, 0x72439c1d3f357986, 0x15ee370d88e3e954, - 0xc993115de2cd12c7, 0x81a69965df63879a, 0xcc11169db946c0c3, 0x5131033b74dba5de, - 0x9abf993c8e6ad907, 0x85a138bab357ab92, 0xc68b86627314f971, 0xdd2568b2cba3ed25, - 0xf1bf2d0dc15b5d86, 0x1c814713f0317df3, 0x22b4aa44aa517656, 0xc3a8ae07c6e96382, - 0x9ff622fee0b777ab, 0x4e1fad864545ad28, 0xa8b8f6eae7299e95, 0xc0c4e8aa2ab89221, - 0xd20536e3402e46ba, 0x89cf88c91ddd525b, 0x88ba3547be633756, 0x2529f82165508a2f, - 0x82c757400c8c7cc4, 0xe77f1dc074b4bd23, 0xeacef2cd0997c63c, 0xec63a55e5c9292a8, - 0x4f90a50c2721e1a1, 0x95d2d0a27bfb1eaa, 0xb54064fa54bcbfee, 0x1b0d49993081d506, - 0x65bcdc0bda212821, 0x3d3d6645ea19bbfe, 0x556df9f4e5d069ef, 0xb72466807e9356ca, - 0x54e583aa13b09f97, 0x5e2cd7ba68b3c108, 0x81b8e989240891ce, 0x34388169b8abe0c8, - 0xd5dfaff6bbfa901e, 0x8f8ab8d46d369b02, 0x624f47338bae9f15, 0xebb6952f2f0a7c63, - 0x416e4e3aadde4bbe, 0x0ca0510fe2e51adc, 0x9a4e23c41756d438, 0x2d1176839c447c20, - 0xd4738f69979559cb, 0xbf283beb16b52156, 0x846accc4f6421b01, 0xf17457a6563d219d, - 0xb558986701f9e3d4, 0xc98475d74fb28d16, 0x63ed9b34e0c46807, 0xa745e38c0872e61c, - 0xa35df7a5b33f8015, 0x3060b130e3860e44, 0x3e8f4dc3af9a745b, 0x8e828ba1e38fc46a, - 0x98781b5fbe6f622d, 0x58d563ee175cb474, 0x40cd85d6e5067de6, 0x5d6298c976cbb7d6, - 0xfc94dd1d0f6ce428, 0x0c3043df87a15bab, 0xd681746bd5c25bcf, 0xcd24ce56d3596f64, - 0xf1bd06e02d0e9ceb, 0xe545e54b912cb0c0, 0x2900d8480070067c, 0x46d082d5514dd43d, - 0x9f5e3af1e8537e18, 0x03d409d6def7b57c, 0x7b1bde65687184c1, 0x87eb10f4723bf59d, - 0xf5910296449b9958, 0x99889dcacf060582, 0xa89f9d3e0717f631, 0xa093d3be317b0e2b, - 0x75e0601112e328a0, 0x662f2d345c75d8a1, 0xa3b620046521d399, 0xa5de5ae77dc1c909, - 0x329b7dc8cbf04401, 0x23c8fce90c829d07, 0xdbf52565f371df62, 0x356065754b271424, - 0x8fd6f37a8caad9df, 0x19e4404e1649b089, 0x0a102d29053f4f61, 0x4782f4a7d89da380, - 0x5963451f926d8232, 0xec7e31d53632c670, 0xad52fa7ae8532254, 0xb470c262133f657f, - 0xc56738319f981550, 0x96ced0155a32215c, 0xbc97f0bc658b2fdf, 0xb5fb35271ed5b579, - 0xd29c5af1dc76c31d, 0xbf30b0cf615d6441, 0x0f025a659aca9e95, 0x88f871bbb22bb89b, - 0x4594185003392d44, 0xe6c6c4ce6df977e9, 0xc39a5d4988fcd5ae, 0xf00ec87a5a8c876d, - 0x8381c6f41756d4d9, 0x3dd13c6886cc9bb3, 0x91464f76d4fc060a, 0x6c644fc76a5904b4, - 0x6bed83f9ea14d925, 0x0205fb0f34b525c6, 0x8aba7a584326a4b7, 0x3e10f89137a6122f, - 0x501f59b670a9089b, 0x0bde45c34be806a0, 0xfc87189ed43208db, 0x0c3faee6871b4b83, - 0x750b95d294e7632d, 0xf1b964e35d8725d6, 0xd31d1f69605ce857, 0xcd1e2ace24dcd8da, - 0xb63a6e1228de0724, 0x185cb465f56afec0, 0xf788a2900b78a3fd, 0x408a72e89af4d709, - 0xbe0107660526154a, 0x33fd09c37aeed3a5, 0x69b8e3b28db955f0, 0xe7fe6175805d39b6, - 0x744ed573387233e9, 0xa9b64c3f1dd03508, 0x555e554f5275a4e1, 0x264a25f2b47d8531, - 0xcfd168daaa93ef05, 0xf2ab7fa3c2548d06, 0x2910b363ec94787c, 0x7490b54cc0a71d0f, - 0x86cb0f8edb515cc2, 0x546b424b3244f3fb, 0x9827b12a810487a3, 0x6b3a8c5cc7369438, - 0x5098e4e9a08b43e9, 0xa2c9d97a0df62e7f, 0xf7edad25e0ba0af3, 0xb6ccbe9a060dac60, - 0x1242c269f1e3377b, 0xbdbb3ad7ced0c9ff, 0x1f20bed33a58b2bd, 0xac9df63dbd2a3079, - 0xf1f092b4c39a5060, 0x00de3784a5b4e862, 0x71a5c5ab598b2ca6, 0x207889e43678018a, - 0xd0ffacdbb0e01bb7, 0x6f3839ec5b4e2faf, 0x6681cef892ef4fdc, 0xf8409472b86970e4, - 0x733f089c75be1db3, 0x15c8ec1a0e1d4425, 0xb90d8336eec7d477, 0x2f88679bf724f69a, - 0x393dad6a5a594020, 0xf7a5a47e66fb2405, 0xc0435cbbaceea75f, 0x117d039d9a3d4785, - 0xb01c2724c3228e8a, 0x496d73c43895d565, 0xbd0ebd7b9940322a, 0x9dc10e360f81d9e0, - 0x18bfe6cfb20a954e, 0x3898f7eff22f4ec2, 0x27d5697d9c3318e7, 0x3362718a64757ce3, - 0x1c17aa35d425211f, 0x3d4eb102f8dbb5c0, 0x5370f062c2ce29b1, 0x6c35621f206772aa, - 0x1890c19ba661a97a, 0xfbc5555dd915c174, 0x4f7d1aa6d1e2bbb8, 0x48d8746b4d3fae77, - 0x05674759c9577439, 0x411ae1015c834416, 0x74ddd92c372c078c, 0x66b4c2e8025b6717, - 0xd2abdfb4b2dc3c05, 0x97c4c47cff7f6bea, 0x5527daa745f9f7b7, 0xae06fc3e501e8f74, - 0x9a09d57a50990866, 0x400341924baa225a, 0xec6ad3c98d998ea3, 0x8602b730777be7a4, - 0xd29dba2ef3dd512f, 0x78024d3301633aef, 0xbe2d7182e20827b2, 0xfd53b638fce5e131, - 0xb98a11d6a3708865, 0x79b0eb4dbb031640, 0xbcfb335a279623ab, 0x70282fe277a29287, - 0x31c26534f25a0a39, 0x5dcbc2b5c4539782, 0xa77770a4c8ab538a, 0xf2198b0235a88739, - 0x1cb187a3df200e84, 0x86b94097d83529a6, 0xa8ef7448bd6606be, 0x4ea9493a375edfd7, - 0x15a8e79e22c7b32d, 0x1438d9c5c646d334, 0xc8ee8e34d502c50f, 0x725bd20c5513807f, - 0x89a771b06a17a588, 0x81d69f253f7cc41f, 0xacd884c266118ae4, 0xac1ea70676873cb2, - 0xf53e2b46bdbac912, 0x2811ca3ffc0f2762, 0x347fbc3584a7c2d8, 0xb10aed0167070c77, - 0x21774d93a59221dc, 0x6ca3f703688715a3, 0x1b68f3c8901c5929, 0x1d612a5823d08c58, - 0x6dc729f448d733bc, 0x219a0cb6b62dc9ca, 0x174743aee3a4acd0, 0x192102d9af0582ac, - 0xb20d65fe74e071bd, 0xc80067fa723d20a5, 0xfd477cb872a2eb60, 0xbb89aaf1f5a5fc01, - 0xf38f8b30ca9239b1, 0x6cba5916fdeb98ee, 0xb0821ad9d933a225, 0x0224ab3e8888843e, - 0x65289c6d98b0f63e, 0x71267143d8ee0d25, 0xdbb6f8a171ec4db9, 0x6a2f0d35f357b2e9, - 0x1204e970df06dfd3, 0x9fc69d3eb7fd3a3a, 0x8f38fcd11f21bbf9, 0x03998e41e1680480, - 0xbc5adbc4ea4d64fc, 0xed0ba739945d975d, 0x43aec39c9034d879, 0xc3c87b57b7380a70, - 0x9c26a847af220d5e, 0x026debbfc8339c0e, 0xac4ccf362833b88d, 0x57ea2e7a9e6257d5, - 0x96ae792741c41198, 0x0a41bd053faf555d, 0x03057dcd45f5989f, 0x6e29ad8e8621908a, - 0x464acd0c7f8e6013, 0xff09fdb95aad8847, 0xf24de6e1b14e662a, 0x7bb480362a048620, - 0x4f1b0f5cd1c2f420, 0xa14739ea691d9d39, 0xe3a47a5661af06a2, 0x884e5c09968b73db, - 0xcb039370f2fd01db, 0x2cf211e5e8ff73c3, 0xf7a5d728146abb18, 0xb0339f224cadf7cd, - 0xea2dae1836e505cf, 0x6fabcae8c21587f8, 0xfe4762ac08d95a90, 0xda8b223411b97906, - 0x9815fe14c7344863, 0x00e640578111bf29, 0x9dce2a787b469340, 0x9c1708d1d4807245, - 0x5789df7757c8fc4b, 0x82bf90b9efbdcb85, 0x450371f74ed30e07, 0xa96cbc0904ef57cf, - 0x5418f4642595fea5, 0xba2ad73beda1dd57, 0x5e1897dde2d931be, 0x33a34bf286a0837f, - 0xeb624c80f32b114d, 0x25babbd4e384cbb0, 0x0c969e174ffd302f, 0x9ae901a4e422cafb, - 0x120f33461a6b7f6f, 0x83c65381fbc8eaf3, 0x0bfa1f4d3e92b560, 0x7753197713f06a5c, - 0x9bc559fe81c2929a, 0xe0c5a2cbbfba70a5, 0x8947e86cbdd63a24, 0xf9c88cf54ab36338, - 0x8a2078fdfd503670, 0xef58447b07144c0d, 0x6db09eac9404bf74, 0xef3f869f25aa6157, - 0x6dcb7ecf980ac81f, 0xba8bb7200a9fd4cd, 0x8f27e042122a872c, 0x201a58cf928a54f8, - 0x57c24e650efa010f, 0xb07e0404e2b4eb33, 0x4ddd78d1d326c7f3, 0xd14f3d5f1fbd8d31, - 0x0e69eb6feaaf7446, 0x69fff3611e5c628d, 0xc14a8f37059ba4dc, 0x6f90dd2e1dd9ce7a, - 0x7348f1755895acb2, 0x723ed4a77771835e, 0xb45f484755b1d5f5, 0x40ed62ce88931aed, - 0xa055750daa5443c2, 0x3984bfd3a991c37a, 0xbbe805d089aafa25, 0x3439f8e8e2c71776, - 0xba0782980666aeac, 0x4977c9f40c6e78fe, 0x9da972c822905be2, 0x1cc1b18d076b915c, - 0xb7856bc20a4ae79e, 0x979bff99001b926c, 0x403f737b1f0ac586, 0xacd98bd75313b3df, - 0x176b12e0e2fdfa15, 0xe1ee26503873ede4, 0x02e9cdc9c4cfc00e, 0xdf1ac0f74aebc19e, - 0x4617954a3548184c, 0x3a56c32636515a87, 0xfbbc6d467c6c52c4, 0x7bf994189fc08185, - 0x0420c267980145ff, 0xfdb732b176558575, 0x9de563258cf5920f, 0x70da934c2072bfa3, - 0x7583c7482a3964d7, 0x8351f2100d77214d, 0xfc94ef3ef611bcf0, 0x85a67d61dd88c32b, - 0xd929ac2272f142c3, 0x23c1da0fb5ee7761, 0xadd7c2a83a9913dc, 0x26ae807a691ce8c9, - 0x416a9751586ad478, 0x137706ff4a7cd74e, 0xaa06b021f9c833ff, 0x3e2ee67e495cf0d2, - 0x73d7dad4f64e8fd1, 0xf28a4eb63a5cf738, 0x873fdd6887dfd1b5, 0xd66166b50b3a1acd, - 0x7171093a0e120d60, 0x40a677bc4c8e449e, 0x7c803b1e18a2f89a, 0x0179804ecf68d5f4, - 0x4609da3c2e94a440, 0x35beef77eff38005, 0xf603bbd4149d6412, 0x83379cc783993a08, - 0x4b702cf005df74d1, 0xd08cbfdd27fcda3d, 0xd602a508ccb7838b, 0x7516a0f1180c3cb6, - 0x552153eb74b0eed7, 0x0de4b173a675b6e0, 0xb75ef52f3c4806de, 0x6b8f76dfd899af4e, - 0xaa499dc53f7949d6, 0x5a075e1f074be86f, 0xfae74003e5da2050, 0x92df03266178a012, - 0x3a88b440b0bff162, 0x5179c690b4959903, 0xb266139aceb7ced5, 0xdea9bbafc141b1d6, - 0xf31ec2fee4caaefb, 0xde844ee0b2dc9ba7, 0x433def21ac60cf37, 0x465a246b167d2c78, - 0xb1a9cc35902e43de, 0x616e27b872330895, 0x6c53095a1aa979cb, 0x66ecc8bb9777eb31, - 0x58eb5c0278f40beb, 0xaefc0ba42dece872, 0x3fd10d81ccbcc40c, 0x39a928ccf43029b0, - 0x5867d9b99e40197c, 0x0db3dda0fe350663, 0xf6119ec60c6d4528, 0x035db20219f5dece, - 0x4849f57cb5109c05, 0x7c4eb77df64068e0, 0x6f8e8c54959b024a, 0x7478f34c00073e92, - 0x55f9c6aee79ec924, 0xd86f5c57e8bc2f2a, 0x364780f78564a476, 0xb861c9defeb445af, - 0xf520b7374e565f5e, 0x3b9361b4f75d25cf, 0xeec5df85042f1165, 0x5ecc8ba003b46f2d, - 0x70ca0dcbc7cd7deb, 0x2a0300aa20cd62e6, 0x1aab9e0c9c1d711b, 0xd3e2c2f631c3e007, - 0xdd0bb7f837109378, 0xa15c2fdde3406ac5, 0x029d65b2df30a5ca, 0x560b234cedf38a39, - 0x28bd894abcaad860, 0xd9ebba00e4c1f507, 0x4066224c6137e027, 0xca8efea0f2a41757, - 0x13f736c3ce6ee223, 0x32eb49c3adc7affe, 0x44bbfe6d8cfcee4c, 0x568772f3516017a2, - 0x1a2b7ee2fc55a5b3, 0xf2faa0e334f215b5, 0x672a1256efa5f7ad, 0xc37f1c0980adced4, - 0xefab304836bf0cd8, 0x9c07651241e37c4b, 0xfaa7d20cd5118bae, 0xe0740fe13dea7dc6, - 0x006d855f3b3b3d5b, 0xe5bf775c45e5bf14, 0xa1fa93f69f63f73a, 0xa6decd1e6dfb3fa5, - 0x2cd325a40f80efdf, 0xf2da15853d106245, 0xc5fabeead66324b7, 0x367670af2f602772, - 0x99ce8c9ee55d9f67, 0xa29b8377a708a449, 0x5725d79d73251d6f, 0xe8e3a87cc59fc27e, - 0xcd3e6564ac274c48, 0x903121c35eb2e79c, 0xed3af55172ce62b7, 0x36912bb7200a6786, - 0x972673c999c66c7f, 0x0be8d1effe4913db, 0x12772004e4e2d758, 0x3ecf707416251f0f, - 0x75b61aa1e1607724, 0x0638fbd2ab0197e1, 0x291c3437f6ae0a9b, 0xe42f40b09a7e9098, - 0x942d68fb64ebd2f6, 0x79f4240f328aea1d, 0xe2271abf21c8ca41, 0xdb988493b9d94647, - 0x3010e4fcb060241d, 0x392efccc80e6626f, 0x12356c871925e6c0, 0x87cad311fabf5ea9, - 0x6187c01c2f0d936f, 0x93218ef76ef4c02f, 0xc1e3afd915b7f57e, 0x349fd39bc31f7ecd, - 0xacf892ba73e72aea, 0x41b7693b053b4e85, 0xc4d7d05fdcb88520, 0x30ceac370b70032e, - 0x09c3a42ff17e9323, 0x5abb94168899583c, 0x63c2a7b1ac98d304, 0x7fab45ae9ebbd230, - 0xe9a20b342b533de1, 0xff0b78a96d9e45e1, 0xb10e635514b86b4c, 0x797d048508d79f41, - 0xa3c2851a823d9eee, 0x5e9e600e44742c68, 0xc48b5eb4496b7d07, 0xc6de54bee07ee459, - 0x670e841a3ed5c5d8, 0x136b1dbd6aa2dee6, 0x841207f83669c6a2, 0xfeb22d3670382bfb, - 0x625a34595eaea27a, 0xf9d33e364ccd3bf0, 0x21b7a15b479fd6cd, 0xfd542bdc41ef9490, - 0xf9289b21a1e81b26, 0x2c6d4c5db79e044e, 0xaf5bca9d14abcdf5, 0x7a089216ec5faa05, - 0x869912803b9325da, 0x5479f0de10affbb0, 0x3ae9004580b298e2, 0x35c632198dda728f, - 0x05b7adda436ea05f, 0xa03a405fdab60ed2, 0x8d503c99e54be05b, 0x20fb51191edd8f41, - 0x85bd460ea01decff, 0x04a171a73caff07f, 0x10942e7d6a01060b, 0xfae470df3e6b3933, - 0x7c282e3c37b2bd21, 0x9eceb6459818f1c8, 0x15733159a986aae9, 0x3607310777a2096e, - 0xc710f3a70b383e88, 0x9232b96f5d1c843b, 0x9515e49eebd26506, 0x50d4202cc9727130, - 0x30eee28d34cabe3e, 0x4d3efa03b42577d9, 0x37a658c199436070, 0xe41cc51711d503bc, - 0x1aae84ae11ef5c70, 0x40be866800e9679f, 0x738860503109f040, 0x600bb48348f44f34, - 0xf425c71d465eb5c6, 0x4eb8c8b825a52db6, 0xa92f0aff88ef6efe, 0x5a86fa024076884e, - 0xa4d5973e92007bc2, 0xb90a665e1f945346, 0xfaaf0ec1f069c61e, 0x5f8d959c0b7eeb05, - 0x20cfc69b225292b6, 0x1335e7fe7fc6457c, 0xcf44c0a7c72ad0c0, 0xd651d1b9c9b311f5, - 0x99da781fe9c59376, 0xa54aa656133e17cb, 0x0bb2443bed0206fa, 0xabb5bc811a70127d, - 0x1701014d7c741131, 0x6d44b302af7af7f9, 0x09b84696e25a51a7, 0xfcd22d401d0841d1, - 0x6a12ac87aab441d2, 0xbb1f95621aa4b759, 0x5285e397155e0a79, 0x23e9acd5c2f645ba, - 0xfcd6829f741d02ed, 0x04aedfc15d4bf8af, 0xa219947e9ddb37e7, 0xc521c64c488f5e22, - 0x44f32333fb78768f, 0xcf45f1ea1b2d5dcc, 0x5f733c5461ea6495, 0x513aa280e9a3250a, - 0xdccf548be26ff5d9, 0x441d8a68d97895bd, 0x4d6ae926e8491cc7, 0x9dc84696f8616d69, - 0x2f7bccaa5ae77cd3, 0xb67ddf4468407227, 0xbdcfad269c17411a, 0xfec7600456f5431e, - 0x6b6f8581e7f1ce9b, 0x0159e94a7323ed81, 0x50963316ec43e50a, 0xc10c7cea1f522d2f, - 0xef1cdff34582be66, 0x8f58b2c2e26e4a3d, 0xcdea67efa5e3ab69, 0xdfc6487811bc457d, - 0xbf3f7cc1bf8372a8, 0xf70918badb2cc005, 0x5df94b98a0f0b331, 0x071110572bb1fa5d, - 0x01b300c46006e80c, 0x00e8332e34df967b, 0x7540b5d324e7283b, 0xa2f0a88c92475d84, - 0xd0f43d473ddfeb5b, 0x835ca1ce2d4a9710, 0xd831fad3b0bbec3e, 0xeaea8d2999cc220f, - 0x901bc30f1fd60d4f, 0xa9d5316b3edd1051, 0xf2e4e027030ed6d2, 0xb7c1abe62f043131, - 0x36ddc8928918efa7, 0x0dffcc44e3040db9, 0x49551d95539913ae, 0xa2314dbe11a10942, - 0x80bc1837eade07d4, 0x8693fd2c9c2f3121, 0xc962bf7a5b11d417, 0x4ab12ed19a30fd71, - 0xcfcd528e4534ecd7, 0xc63bea3b74a61f95, 0x41bde433300c8e37, 0xd8f0fb9a41dcf85d, - 0xe5640f738c9b911e, 0xedc22ec12c7c5689, 0x0be5dee7031924ed, 0xac2b918c979b0ab5, - 0x25cdcbc4dc4fc742, 0xa1297728c0994439, 0x41ce9b07987348e8, 0x65279d0c5f9e9f83, - 0xbfa94e7d8d9daee9, 0x1aebcaba315ddb77, 0xe27f0c61a5a7f821, 0x5e84529a66656553, - 0x576fe072b4908d39, 0x287bd0d94a22619b, 0x40b0afea313a400f, 0x7573db7c7a527001, - 0x0490ca3fffa4fa09, 0x8d61cf0b44bb7bb5, 0x1360552fd37c2d51, 0x2a6437303059cb8b, - 0x4e06f056bd4b7a96, 0x52b54e89ccfff423, 0xf5c681d7e8eeaa6e, 0xc00e758523c5c9cd, - 0x73060a35282ace8e, 0x42e459745cae0466, 0xcb4fd882dee0a454, 0x5ed2253467ba174d, - 0xa2f48e753578f408, 0xb5509e350b5077cb, 0x1cf331fe0b069335, 0x29043d1787072def, - 0xd7766b8f3b86c55b, 0xcdcf97de094d6113, 0x2b996a2c5fe13794, 0xcbf4685735b084b1, - 0xfea915b08b7d80a9, 0xfe3dc6b26c4bb3ad, 0x613e457b12cbbc19, 0x1053edc8ec3baf27, - 0x0d20cd38a32d172f, 0x413fff58a39d85f1, 0xbf8983aece7ab7d3, 0xc21714edf14d5a20, - 0x5ecc396bccf20caa, 0x70114d3098ce312b, 0x2fa6913015ee8773, 0xf2f729d15e9417ef, - 0x496b2bba954a7e47, 0xa06f058f5e59e136, 0x52c23d0c5998b8af, 0x971debf8bdbe41f6, - 0x26a93cdcd8fccdf5, 0xe264a097580af703, 0xf3968c58a3afb70c, 0x312fb8458111f39f, - 0x840655c88919cca6, 0x893a69e7134395a3, 0xf63d6131c7408688, 0xc8afa11949cadc11, - 0x482736d292d253d1, 0xa8fac85a875204b4, 0xedb848dda88006d7, 0x5d6a399524847de1, - 0xda79c4a66d7b2450, 0x6d563a24378367bb, 0x55f7ab36debbaa4c, 0x7755c5b71c997a17, - 0x51164b819ddbd662, 0xdfc29c054b7dbe07, 0x1119cd748bdc5969, 0x89e032b3b1420048, - 0x4596ffd192e5b27d, 0x5c58f38d8475189a, 0xd13fc40dd9f83257, 0xf45f3432a19fcf87, - 0x139c53697c8cfd26, 0x0a5114c8b11505e8, 0x04193e9739554168, 0x48e2afa13477eb16, - 0x1f1212b12eb770ef, 0xad8acb505476bc8c, 0xb57fcd541db4da36, 0xcb89801bf3310dee, - 0x0d10c1a75d402f6b, 0x617a79563e07372e, 0x337a3d0924eb434b, 0x4cfe6091939821cf, - 0xc7ff033ef890e39d, 0x4ecd0ac67a004a17, 0xcc547dac1a8aa6df, 0x88acf1d484c7b4d7, - 0x4e40afcd89b244d3, 0x63b7b2741e97c3c8, 0xc7d7730faa310315, 0x58b0a1990a9c59e6, - 0xec71c91c25a88b82, 0x37d88341efec2a34, 0x94a47e39da399e26, 0x3a0fa17a4ce13bb5, - 0x1000bcc02f387dfe, 0x1d3c4a20002b9a9d, 0x229be278937039a5, 0x2f2304566c56bcfa, - 0x45a52c321f0c2d19, 0xe6468d6c0ddc885d, 0xa3e2da81743b188c, 0xa80b6939ed6966b5, - 0xfa16593e1289eafb, 0x471bc94e16c5b844, 0x2ef88ce9df53568b, 0xcf10b6899903ee13, - 0x93f630a0384248c7, 0x9a53c2755975479e, 0x65afcaac4f5a218f, 0x1d51aef8898f983e, - 0x28eca3e946cdd4b4, 0x2f3cb49a9e8c35e0, 0xd2ee833cfc270f26, 0x7701a72d2fd17de1, - 0x36253a3fd3af3b67, 0xecb7e6dc26317b05, 0x62c235ce6c4f8c0d, 0x7a22cf865420994b, - 0xccfb0e36807e422d, 0x3767477285a9223d, 0x7400338f754a9aa6, 0xad6c63bde8e54f66, - 0x3fd5d508c5063bdb, 0xd3f2ab009b190ab2, 0x8323bf00242dc22c, 0xf339c8ade12afb68, - 0xe4406d6375d68b49, 0xb31a94a654046d4a, 0x917cf1de6478f3ab, 0x44997551ffd7ccf1, - 0xf4c9488c9af2f9bf, 0x15d34cf1e6629471, 0xd95ddefa01fe614c, 0x4239ff0df2fc85d8, - 0x9de1c92194b6567f, 0xd870779ff9d2e234, 0xd62fe5150dec8bd4, 0xe81c8dec94ad6d2d, - 0x23b00bf02d0c9b2e, 0x90c82027644a61a0, 0x27d1921072bb99af, 0xeb26ecf038b05b5a, - 0x12aff77f5462a1cb, 0xd7a416bb141470dd, 0x3cf6359103cd96c8, 0xbe710bcd24b05dda, - 0x2d3623e756731212, 0x38c9734a010eab52, 0x894a4229c5da79dd, 0x1805170f7974270a, - 0x2e6a2ede8aa49da5, 0xa7cd147823a6d499, 0xf7bae122cf2b36a3, 0x8d7dec8fcc7698b7, - 0xfa028c8440085fdc, 0x6481c8fea2b5bd45, 0xa933702eb02556bf, 0xad1972f5b24b24b1, - 0x31598d3542bf86a7, 0x23b94d4216412b9d, 0x7571f10f51155261, 0x80513bbf2a16de31, - 0xad9b3a508e3262ec, 0x90f9e85f7972290d, 0x162e1eb7ddfbae6f, 0x3e016fe2ea3fd7eb, - 0x551a12f98110884d, 0x2d51ccd3109d935d, 0x5892905d36f7ca86, 0x35d9b58b118f6003, - 0x5e42a0eec4930b22, 0x1d344afdd1260cfe, 0xd3478b870dcb049b, 0xb8259dfd40b63699, - 0xb35f34b47f958ea1, 0xba7be49e99f5d76f, 0x6333679f4b6b7c1c, 0x7cf9a8c739d4c66d, - 0x66d4139ca4692cbb, 0x2bc204a32f273b19, 0xc4ad5079c8856721, 0x37174be215e84cb0, - 0xed878eb783d0dcd8, 0x1dd282b6e4d5b345, 0x9f6520b2b44b5246, 0x0c207e064ba9c2a0, - 0x4afe4aa679334e21, 0x5f07f1ad0cbb7fb6, 0xdb53948dbc83ca46, 0xac7bec8e88d25856, - 0x0ee214c18cc2a232, 0xc398e2420880d813, 0xfed4d219d5662f2b, 0x5766ac4c7759e0b6, - 0xcb70fbf45e5d58bd, 0x658aa40fd54e4410, 0x1d834f77a399bb9f, 0xe23320a4d732e2be, - 0x6b0b43ff40bfd761, 0xa4a19dc4dce24920, 0xb042fcd65dbda5d4, 0x19d252f962a1b194, - 0xefd65fb1f0898c76, 0x20f50652431b6703, 0x2d01712ef60b4fdd, 0x54cfa734b4d114b8, - 0xb24db30c3bd0d3f6, 0x466410a08feb08c3, 0x09ed3cbc1c667270, 0x24593cf8e8252fb7, - 0x70316522afdd7296, 0x59b6fea7cfd1406f, 0x683b9c2e69c832f0, 0xbba7cbb88d2a673f, - 0xbce8c82c26cf2b49, 0xa5514d36c4688f69, 0x7ff7ebe9d0a06a3d, 0xb794ab39f020e67b, - 0xb91f8058efc6c234, 0x93bd2de63b881d35, 0xfc78fb974f757b77, 0x3f26d29ff80c3242, - 0x3625c2e07d45f116, 0x23496d77e07c7fef, 0x239b4ae24e34eb82, 0xa9a87837952fffc7, - 0x0cc05f29323fa653, 0x910c46c018ba7e24, 0x19e081de69e3b077, 0x0066a8049737e260, - 0xba3bb8dc687d36b6, 0x08db7833c9f5a4fd, 0x0ab69778399a8543, 0x4dd335ca0d884857, - 0x331992977d1b4312, 0xb290c39a22670abb, 0x5bffe98bc2a19bee, 0x4c45e812b58ba59c, - 0x20783bb0499d51e4, 0x105301259cef9e74, 0x101b003b76db30c9, 0x1099e4bfdb4732f6, - 0x4580208e18424dc3, 0x46df5bd3d0a29ace, 0x11790e37347c5205, 0xd8d98b3088cae035, - 0x37639436e31aacbd, 0x7688541df7a76f66, 0x6648a134ca5dcb99, 0x8471ce41bd7a2bca, - 0x4c1acf423c60372d, 0x019be49c8d197459, 0xc8924c929f4ba11f, 0x6b0d229e52a2e530, - 0xf28ffaa65655a3ec, 0xca25a75661740419, 0xefebb1ea014dbac9, 0x29c16b2189d3fcc1, - 0xfa28e5462b8a3ff9, 0xdb9f9eca4bf3cbed, 0xae18e3f420a6bda3, 0xe1d9e8ffa0085114, - 0xe7267a55e6772f07, 0x74d9d8f458313234, 0x8c60226b74c1f8bc, 0x7ea5e8f6e80e6f4a, - 0x5916337d296cb442, 0x79f52744f43a261e, 0x14055babc07fc8be, 0x0637024b051fcc92, - 0x2f20672871e15e91, 0x75302d87262b00c3, 0x039c886d0beb53b1, 0xb9853810d1f01e4f, - 0xbaf6d7a7fb4b6acf, 0x4b03369eb2decb78, 0xf582ceead56b7c72, 0x2633094a3ae582ac, - 0x51ddadc20bae292b, 0xc550c49675b6847f, 0x4b5d305250e1d64c, 0x75491061bab578e8, - 0xfdcca577cf27d454, 0x178157a19dbd0faf, 0x156dd88a7a4c30bf, 0xaf3a8b0372625887, - 0x38230837f9bc0b22, 0xc7eeead9dc4986ad, 0xe533065a7783734b, 0x1ac07ba902341580, - 0xc8163db1286659bd, 0xd73db1fd518f0d7e, 0x4ab55dbcbc4d7824, 0xb6474e0a9e4938c0, - 0x9eacc501ec684223, 0x2bf9ac9cf960ff72, 0x71f014d02011a383, 0x2bf8248495fc50f9, - 0xd7bdd43a294f7e48, 0x563c4d21e735f1e2, 0x7fe9436218266427, 0x3f5e8196b27b9ea6, - 0x0b4c85695dec9982, 0xcfb053cb78173e1d, 0x70638df3cab016b5, 0xfcd6af2d74f00acd, - 0x960d6345022fce81, 0xe313188a72694f48, 0xfb479bbb50ac1483, 0x37e14a4c495aa5af, - 0x3c7725ac545ace47, 0x5a67d53ea76e32d4, 0x97413b9789724e83, 0x8de4bff41d8545f8, - 0x96cc3f7df061f255, 0x59c811915e7acfaa, 0xd8c4bfce03be4f8d, 0x73f721936df4db30, - 0x977e57457a53360a, 0x9fdb77372f2297da, 0x3da189f2d017d559, 0x7439e0d25a95c309, - 0xc4849ec50666c340, 0xa97dd2b81d1e9ba2, 0xd477750a6dab957b, 0x4136fea9a215d644, - 0x9e24392162d63935, 0xe715385ddea9e64c, 0x26e3d0c39904d9d9, 0x364ea3a888b72c0c, - 0xbd05f01769e3d4ee, 0x709560f7c0cc8aee, 0x693fbdcdcd442e38, 0xc289cf40edfaacb6, - 0xb44e7d3d73a97f29, 0x7f019fb5a7157a2f, 0x595a97170dbc8172, 0x0e248b441525ea08, - 0xab022d256d9d8f16, 0x84b849ef17ebec95, 0x33ddf6efe400cb58, 0x7276d1708a3a6a64, - 0xd261204edaf26da8, 0x34aa572f1e0c5ce8, 0x3c105d6bbb97fcb2, 0x126a75ec64d62a34, - 0x73efd51abe9bf6e1, 0xdfe0decc09a68d22, 0xb665ae663be00db6, 0x0bd941b2a5169b56, - 0xaafb73d63425cb24, 0x9a38a218fd040d37, 0x24422a688ede53bf, 0x5d97b208ba4a6e89, - 0x239ff5d4e7bcc39e, 0x8583f922e7d2abfa, 0x58596cc27800df33, 0xa7c94ef51c685107, - 0x7fa399733e16c966, 0x7ccce7201fee6842, 0x2d4d7affb924e718, 0x4cdcf18eafb5ea97, - 0x466bbafeb936ee72, 0xa5469bb30ceae2db, 0x2f5f475b798f6ad6, 0x70454dea1258212a, - 0xeb2dd1a9fd510c1c, 0x8677968f4f8adc8b, 0xf56a13d33c8da126, 0x777ea641514d41fc, - 0xc72d0861a2489ac6, 0x32ab664f6cc38b10, 0xc1e5349f1c01618a, 0x2e03244a7c81a3cc, - 0x38fc845f4520b9d6, 0x02231d6beb030f3e, 0xb0578413be2e2238, 0x8086e6c650ddc376, - 0x33cb7bb557c0d3d1, 0x97238fd87c085ac2, 0x7bab220f08dc625f, 0x30fae121d810f5d9, - 0x7955ba2f1aa53649, 0x819ba63b8f9e827b, 0x7e49c5a5fdf5ea48, 0x7775d7b7469e4df8, - 0x38623df52b0426a1, 0x258d45cd60049fad, 0xf30dee671a41783d, 0x403f2d3c81ef4bc7, - 0xff3625739397213a, 0x6e8b92774fbbbc02, 0x448079281b428889, 0xe1e3bab0ae59d357, - 0x7653e726b1d2fbe2, 0x2a1b46da95a7b549, 0x98938371b4f5bbb0, 0x8149b9d9e7a207bb, - 0xa57556d20e4b2595, 0xd9136d595f8967ca, 0xd46251f42a9bc7c8, 0x1ef78b30c52e1ef6, - 0xa9d959ab4f8a81b2, 0xf65b15267326829e, 0x8e4b938861e722af, 0x3ee2f387d87c4a77, - 0x594a29dffa2b9ac2, 0xaf6c8aae91412fad, 0x6fe3d5cc3ec3c86c, 0xb9ccb0b9a6c6d6a7, - 0xd7e5c8551444fcdf, 0x7247655e2d3ca100, 0x7d6c4e9f986174dd, 0xeb2fedd83093bd6f, - 0x354ddcdf78d24546, 0x22571dbf38076678, 0x478ee2f5ea063e60, 0x2d4211a8bc439b64, - 0x8bb83ccb377e6e46, 0x47fddfad09b7c24e, 0xb50eb184a254fec3, 0xaec19b6fe6a90c5d, - 0x47fa8f18cc359b2d, 0x5d7ef3b866e45943, 0x7d380c3bd2036f81, 0xc8e41d797a4a4d84, - 0x0f4763e140d1d1fc, 0x1ac7d11f02aaccf3, 0x2d48a2a7214e1994, 0x8c39759399467538, - 0x923513bbe2cba4d1, 0x9eaf1f44053a1f3e, 0x619de0ff829523b0, 0x902be56a99a3ada8, - 0x48deb29f06ca1833, 0x562b1bcf0f0b5569, 0xd7f1a206c7393b07, 0x9b0149adf00287f8, - 0xa9c1f40a5cb6f6f2, 0xb6ca837ae22b87ae, 0x877b598f5428f038, 0xc432a5c1608f4ba1, - 0x8db4e054a77616c5, 0x4b0305dc78fa25a9, 0x9212488e266e5302, 0xd1a7a546897f8245, - 0x3c38cf03fd9a730a, 0xad9b61fbb677db68, 0x1a557562c1b0f977, 0x2935f1411ce0c2e6, - 0x3ce63691e594bf80, 0x877e7fbb643af06c, 0x6d7de6e2ae198bad, 0x7d3a439018521492, - 0x50bb2a88ce7c488f, 0x0f3c36ea786c14d4, 0x911a8e167b0f398a, 0x947d6aac90e7e98e, - 0x56eec24cf0201bfc, 0x4d32a501b38c56c5, 0x5f8258ff9f7d414b, 0x87d52946981007db, - 0x4ddd9fca3646298f, 0x1f32e5bfbe43525e, 0x586dfbdac515d632, 0x497fe51fb3e29b41, - 0x99cdc3883e379bfc, 0x5ebb1e01559cad1c, 0xfa4bbdcee14b3c6e, 0x12e79e767e70a0eb, - 0x23c1ab042d3dcf1f, 0xb0e9d9fd63dc8759, 0x4195819fc7174c94, 0xe9277234771e7974, - 0xde0d54fc77cdc39c, 0x137e272f37555118, 0xadfa2950ee023a52, 0x9b71a88a1f32a8e6, - 0x57c749199e7d7a98, 0x13432a90c4889c20, 0xc88a948357d76dcb, 0x4decf5c94f318ab0, - 0xfa5a9b080436bbb6, 0x92144c858477bc73, 0xc341081b6c58756e, 0x8a3dc11833bf16c0, - 0x8888035dd5187bf1, 0x73c10e56adda9c39, 0x19a4c4f3201f4776, 0x7014dd6af67c0409, - 0x4b8b78888ca93fb4, 0xe5381d091064827f, 0xb5995b7b99a6dc90, 0xfb510e757bb184db, - 0xed1765509ba9b4fa, 0x170fa3d0391e027a, 0x84ebed95b81db988, 0xdeb7ec4ce3fa9da1, - 0xb740b741bcf48059, 0xb9afe962ddf86e97, 0x08d0606647cef9d2, 0xc9373f474bb7c230, - 0x78c5459b5776f32f, 0xeee613071f0189a0, 0x659d276358fc742c, 0x6d3e05b11982c745, - 0xdb959e87f32a4393, 0x2d38d2108238420c, 0x84ab5f42a4ba0842, 0x9a31294a7aa93c6b, - 0x759bb6117961b904, 0x97a525ef2da76afd, 0x1ef794ff4bb9dd8e, 0xa9deb0126cfef6f2, - 0x088a9bf2850843c0, 0xcea9028ad614e03b, 0x63c1ffb902559a3f, 0x358a2bf95dc1ad7c, - 0x195cdac4483cf26a, 0x74d03ad14d08c779, 0x927eea93628cd93d, 0x0ab24cefb8416397, - 0x3925ddb00b01a956, 0xee194861e1c34254, 0xe4289f66a6161cb4, 0x2dc9753a205c7efa, - 0x4244fe30f75fef5a, 0x8a4cbc58db61e665, 0xd146908a91fd8d48, 0x84e846a8efcfb334, - 0x1a5066a5790ecb2c, 0x590e03e7c207daa0, 0xb35b496a69f28674, 0x0d77efba2adb71a0, - 0x38fd010b04b85772, 0x7c5b3564191b4015, 0xc5910e002bd46b08, 0x44bb465e23acc0e8, - 0xcad1956b8514244a, 0xd99b7ed45e60b1b7, 0x6cf2ba5e20d729f4, 0xb417a5f8c5a21e70, - 0x5012dd4b7a08b14c, 0xabd8aff2da3477e5, 0x1c597348d3c6ec74, 0x30a46a5e50aea3a4, - 0x611fd119f597dc8d, 0x36548eeb250d1b68, 0x422dac6b2a00dbca, 0xcb647a9bcb2f7b7e, - 0x82593fb8da9cbb65, 0xdd99e4963f8301f2, 0xaf52f0a51309d203, 0x7cc796c8d48e06d2, - 0xa5d639bf24c2b009, 0x7259096214add330, 0x58679d11851eb054, 0x892b4ca53deb3d85, - 0x7db3fb8e9f84225f, 0x38afb949519f5208, 0x3fd0d0dccbd84b6b, 0xf6e3db08c23a093f, - 0x18b34b6aa3fe697e, 0x5b81323e71a232b0, 0x43018335da66f719, 0xea096df8ee0b1688, - 0xfb257c61574231c8, 0x229a91db450d136d, 0xf439c110d39067c9, 0x96888c7e428d8e50, - 0x555350702bcd0399, 0x63b816d705c22daf, 0x16bc96882d9ed236, 0x30369d6cf925fe42, - 0xbca7625d194733e7, 0xc335b97d3412e382, 0xd05d5c74d1b2448c, 0x4fe7d745c0cbf733, - 0xac97e8f750f93358, 0x98d5b2e049468da8, 0x310f0aec1a8a5842, 0x5cd712c67763aae0, - 0xee44f47193c30854, 0x3e371e52ddb061aa, 0x0db07791481cb284, 0xfe23dced2d0734d2, - 0xdc78b9aa596edac1, 0x717030751920baf1, 0xb10a4e2548d501bd, 0x0a4f71e62ddfac9b, - 0xb68929a8e682a2c6, 0x65169f13afb46301, 0x5a4f06d460f3182b, 0x79802586e381dd14, - 0xcd59d4a5272111b9, 0xd23ceee18802b114, 0xf1869d9f3f049c42, 0x0f7071e1ad897aab, - 0x6e660c265d0c9644, 0x7184155335abd2ba, 0x543fca9d24b12c57, 0xa51236596076fd40, - 0x43816ac63e845bad, 0x077b587e1be4fca3, 0x1bf5cd942845635e, 0xa63e4bca16b21fc1, - 0x2953c3becf8319db, 0xfb349d9734fafc78, 0x607e7f50b49fc991, 0x22f300f1e6f20c8e, - 0x7262bcb4c79f4e40, 0x8c573f4d886e4fa9, 0x842c39c8db28dc60, 0xf5c98d25d044aaa7, - 0xdc7826d20e0c6bd8, 0xf35247d871518d00, 0xc984d6910cf5bfa2, 0x7a93c257644794c6, - 0x109c840e054ccd3d, 0xa6c19534a17e622a, 0x22271c8fb932f752, 0x63c6bd6204885477, - 0x10b8953ed6495d58, 0x8073c5b4654efd81, 0x4feacae32782a6b8, 0x95840c9a6ed9b184, - 0x381fb3d43f4ed0ef, 0xec6fdbfcff7ded2d, 0x57796a5fc76c7e94, 0x9d286b06447dc5cc, - 0xb461c20f63281420, 0x356decd7b1eefc90, 0xfc685a1973bea82c, 0x1f4f66a1f92db9d0, - 0x0de37497ea9ed84a, 0x0032bfd65ec1af2d, 0x69ae22dbd430c1cd, 0xdad0ac4473c35221, - 0xa3f8eba0679fce53, 0x39a0db8660e2540d, 0xe66dbe85cef1a586, 0xeb76bec1aa51d2c0, - 0x03d5f123895faaa5, 0xdd91ec237dfab1b9, 0x5ff6ddf7c4309b97, 0xf9462910805938b4, - 0x0833726aca825fff, 0x3918c87b448ecc12, 0xe56c608a3f70b6f2, 0x2934c13017b6741c, - 0x46dacc829c21d200, 0xd59b7421625c37bc, 0x54f796879abaac32, 0x267bfd6304cef715, - 0x7eff8fa9f5ff3f14, 0xd97c7baceabef25b, 0x4171fa90052e6e56, 0xd321045ed87bf73c, - 0x60964aac050e34ea, 0x40002e37a804e424, 0x8f4e45b783d81878, 0x0af889152a8ae971, - 0x41231e9b47e6ef6d, 0xa5e31d99d2d07356, 0x2e2de4fee18d3b86, 0x159a89fc6aeaa484, - 0x96b1eacbd235697d, 0x92ba2184f4e5633d, 0xd67f5bd2cfcefe44, 0x27dad1c390521a38, - 0x7b0a6ac50ab57e3a, 0x6efe970fe3367c9f, 0x231a9ec699a005ee, 0xc34ff828b1b74d2e, - 0xe2b3d3c48014d757, 0x5251e197c578e46c, 0xdeb6854b9a85d771, 0xb6a98be254f71692, - 0x9a46e9e6624622f1, 0x2c7ffa7fa7235840, 0x22189f63430a823a, 0x7ff2f0c858142bd2, - 0x52795ff1c9ae17a4, 0xde4d942498e8b7e3, 0x7cb5d7f2e10328a3, 0xa448bbf7a8c17e20, - 0x577f6499b4d99223, 0x8ddef7b509722845, 0xfe970766acebc55a, 0xd3d622c47f53ac74, - 0xf2f3b22a941de355, 0x60fec70eae26e7b3, 0xd017249e9bf75f04, 0xe6e426f869da0739, - 0x48588906f7550fbd, 0x0e829fd415a607fa, 0xac5b9e7c3599a169, 0x6b1e1a8ba0266fe7, - 0xb4ace964d53428a0, 0x517725d126e5807b, 0x24217384bf9b084f, 0xb6585a398eb35b63, - 0xc1a494229d7443e8, 0x202a99a9c128690d, 0xfcf687d6e4b5a7df, 0x829d695f9ba0370c, - 0x89dc3320d97881d4, 0xc4c0f07d3dde7cc7, 0x664dbfe261e2f7d2, 0xc77c3359e6fabc99, - 0x59c87221ec4b70af, 0x6d20a2a081179676, 0xc0cafb53ccf7010a, 0xaa3c8bfe0bcabfa4, - 0x88325d6fb2ba544c, 0x9602e7cc414ab3e9, 0x539ffd34092e6235, 0x776cd67c51086b39, - 0xe6655d77e766e148, 0x13a493d95c65e0ae, 0x6aa14a331ddad59f, 0x9cffa275d139a1ca, - 0x931a6ff692076c59, 0x85858d2fd38bd488, 0x6978c30e7b3f6652, 0xbbc28742427aa515, - 0x61a92082b746a311, 0xd7630334b1faee13, 0x15b0dd5757328db8, 0x6e2d125fd75dd9a2, - 0xb6f27d8f4ece315c, 0x4de1179c87d4be5c, 0xf7b7b6d9729a5290, 0xe3b72cf98b54eab8, - 0xf5792b1aa4f0bf17, 0xdc417492e857d9ff, 0x0dccc6bfadf01a80, 0x945089bf7e52e868, - 0xf1699d463708ba66, 0x462828cdb34f7f9b, 0xa7f3f6d54f9da920, 0x9516d46027b10b7a, - 0x0c343c167b23c251, 0xb2b25554824f047c, 0xdd2c9a586c2f5700, 0xed37c196863c0e66, - 0xf3df16be4b01e2c8, 0x4e9f7ecdba2a0bb9, 0x33bfb23aae06f6d8, 0x760da13793f070a4, - 0x92e44999e44d03bf, 0xa605acbdc8d7ad02, 0xfa53e64d37c60cc8, 0xf76bb437943a7bc2, - 0xb972e28772ecd402, 0x2d625234ee5134d1, 0x6f4b87dc99da7215, 0x448be09396b62fa5, - 0x954664682fd59fc5, 0x5b03b21b757379b1, 0x9f739037c7aa3e6e, 0xe90a023124a98984, - 0x0bbde3d156a43996, 0x85d1ded0be7d162c, 0x2c9216200c5b00f4, 0xae6406805e321870, - 0xe78daea854b0d13f, 0xea25f39a5b725d9c, 0x055356c6798b8d0c, 0x6b01174bf52067bf, - 0x8038aa92356b6d14, 0xb6c6ba0490c5a63a, 0x1f7ebadcd62fcf3c, 0xd6481b149292ded2, - 0xfee875dc89e3ab02, 0x3d57b40b96605d1d, 0x3d1e2d9602493ec4, 0x89219d8125302a55, - 0x4279050826315794, 0xb02ea5e07dbdc1ea, 0x572d131a9d665606, 0x14c8f48391d0cea1, - 0xe8f65053a14ce2d3, 0xd9f61297aff37b4d, 0xf9cd776f49e66bcf, 0xb3630a4a6bb6f7f2, - 0xedf103058e3a76a4, 0xe9403a87f39f53dc, 0xf388ce92578d5628, 0x465269d3e2895e4c, - 0x6af59126ad0c7da5, 0xa6a3c5527f6e09f6, 0x9bfcab1fe6bf6601, 0xdd3ca3f09e316163, - 0x8895728d793ed552, 0x45907c854d71696b, 0x1fa5d614a0ff6f74, 0xb62fb03767c55559, - 0x5e2b3542d1ed06ea, 0x157c3ac5db1402a8, 0x88d09ac5e273e713, 0x6eb1c607e8543897, - 0x90debc4f18e30a79, 0x5ad3c56e41faf60e, 0xe92c9aaf5613fd5f, 0x33ae596e32b44bd6, - 0x9979ad97ab7dbf19, 0x4d47774832371f86, 0x764b1303b22f2c33, 0x7128fce519ee9fae, - 0x66c9315be04bdf86, 0x7f396bf9f8e5efa8, 0xfb7fbc7eeb95f502, 0xf569adb7a0bbf8cc, - 0x8b9856f857da722f, 0x1fb7d853c11d173f, 0xcd87963c2812cc60, 0x368d46b904938350, - 0x16f2b878508018bd, 0xd4da93c75c5869ef, 0x21907b6dd35fd4b2, 0x55eb2b110606cf22, - 0x5d5566a4ad062fe9, 0x2935900c18fcd04b, 0x99675186bce59985, 0xd92636f5c3993761, - 0x465349f35aa91d60, 0x36658fb924bf562a, 0x2fa8c4d10174fa33, 0x58e81d740af9b668, - 0x9eb22bc86105853c, 0xb0e6562de4c2bc54, 0xf22b18eddf7d693a, 0x99a6b2ed95f8cfa9, - 0x0f77f7dc54c621db, 0xe163c5b6b6f6e9fa, 0x0e8407c6200e8057, 0xe0636f65210bf612, - 0xa6a531974d2ef38d, 0x7ac625c1900637a9, 0x5d6f50fa441a4eba, 0x56e3fa04b376e81a, - 0xd529cab6ff052116, 0xa2410a8d55d52e63, 0x89cce8bdd7d913a0, 0xab83a42e920f3118, - 0x726a290583bd52f4, 0x458a6ffd5aa93586, 0xbfc833f8ff9e5346, 0x02448958cb4a80de, - 0xdf4e6514e5a47e7f, 0x8436d0a14d535684, 0xbac66d7caef7b742, 0xfe05435deceaa1b5, - 0xd46a06eebcb1271b, 0x26e2b1c43bdf70b5, 0xd21bc8970ac3f78d, 0x2fc2e87e2652b6f0, - 0x3d541feea4f3c37e, 0x6597dbce0fe5017f, 0xc14356e3cd683463, 0x5eeabb4e0bf06531, - 0xd3ef32e1fb3863e5, 0x535aa592799d1c2d, 0x25c6117f070fbf7c, 0xafd562f0ec083a93, - 0xd82ed1052b6be71c, 0x16523ea0b84192e7, 0x602e4df312c83d77, 0xe83945956a725ff6, - 0x13d069d03fb838ec, 0xe6890493587b3223, 0x38c03f8a7daa4ec2, 0x11810327727eea14, - 0x90960451669cb4c1, 0x203611f635ce8605, 0x7a7e4d3729b3472c, 0xb469151de6b06827, - 0x6c48e3dca113edba, 0x94e41904361e617f, 0x1c940d6cb53c9dd1, 0x53a2a79942c9a5d2, - 0xa701005af1cecce2, 0xb023225aa000dd67, 0xefb8fa80d4e56be1, 0xea711747a3296e59, - 0xef4b72273bdcb1c4, 0x2f11b9531b1dd48a, 0x6f1d50da2a886b00, 0xb76e66ab68fb62d3, - 0x4cf7b7f20b3be2ab, 0x7af89c632536cbb4, 0x8c6ddab9cc768fce, 0x8dbb25f39e853ad7, - 0x9f61f001d8752bc8, 0x45600fb62a84d5f7, 0x059e518519667daa, 0x03aa45ef487f47a4, - 0xcd7d7c26565e7c62, 0xaefc4710009d0b91, 0x3d834a4b9c163724, 0xc2c918e604cc7f22, - 0x34f698a1e485f914, 0x5adcd39cd57586c7, 0xe12719b6417793bf, 0x04c54843f1eee7d8, - 0x78ba564029811d09, 0xfebbf8e744128e12, 0x237581da8befd07f, 0xb8db2cd56ffbf7a4, - 0x9ef23b855c418ee6, 0x778ad16200b96964, 0xb0f025cfc0d29d99, 0x27c8ef3baccc72c4, - 0x47f2ec25f5694c2c, 0x470d423e2b4d3a5a, 0x31aa34086feb4d20, 0x2ba8f70b5a7cde0f, - 0x24d5bed193bb5804, 0x97e255875cc6fc96, 0xa39a11689f53e6a5, 0xeaec8f074b446e38, - 0x01dfa0bb63a47dcb, 0x554e4d97c16056cc, 0x9398e033cad1e541, 0x7b6c9e91f3505a8c, - 0x7221929702c918d8, 0xcf55d6b2c67dfbb1, 0x5fe70d9f976c42ab, 0xa9bed7579e53e838, - 0x01bd507aa62a5150, 0x22fca3664b738d20, 0x82a177f82ab647bb, 0x7acb24f9b81467f4, - 0x76461ac9fd7062de, 0x8cae3e4d405f2667, 0xf4ec19941ca53059, 0x20b9ffe98b069d12, - 0xacee7b4b14698acf, 0x9ea4b771af9c2434, 0xeed4289f8cdd9d91, 0xdd5af37c0b24933a, - 0x716c7d63f3a245da, 0xd60dc72ef03b6af1, 0x11a03aa697f51046, 0x0deb14b8a77f9302, - 0x49b0c4d9f3c947e8, 0x5cfb4cefe4f57a22, 0x76d73bf35a6b8ff1, 0xa160c2051e99f0f6, - 0x3b21dd766e08b65c, 0x00db515e5e2024ea, 0x4c0a74cf778db7b8, 0xf788618cb5ca9fac, - 0x628592d29adaa08f, 0xe7d21d3ca609be64, 0x2e5810fbbe92bb7a, 0xbcb12db2d5d1e1cd, - 0x1cc43e73d2b187dc, 0x16b60137230718ca, 0x5372c482d4006d2d, 0x665ef7a68024eb6a, - 0x5b0640fbcfb7ef42, 0x80540eab79334bbc, 0x16ff780efbf5d5a2, 0xf435ea0550d41c31, - 0xfa76e549aacfe33f, 0x1ea145f989db7619, 0xda05b32bab591245, 0x7bb60ca102cc1918, - 0x4abaf1ce5d54c228, 0x296fc9cd89081bc2, 0x406888647cc4fd14, 0xb6b0bc88f4b1642c, - 0x7a02cb09e012a150, 0x44db20f60c122a86, 0x256d5e15de92fdc7, 0x8efccf0a27382f30, - 0xb695c381f35e2d76, 0xb27915f54884799e, 0x5dd7a5bea1ed239d, 0x9117ec76c92a112d, - 0xec2b905b46f17e30, 0xe623e5887a9a07e4, 0x7cb8ae38be23ec7c, 0xa4191903410fdb61, - 0x08e7d13df6e510c5, 0x32d97aa4607f3a0a, 0xe61deace59d38e52, 0xbdd8bea09b2f3255, - 0x415e95012eb83720, 0xeff10b9b2abe83c2, 0x699752f6cff1d303, 0x2439144dbd55a943, - 0x6c942fb25d75033d, 0xacc7ffa0352e0a48, 0xcacae9ddcb5877b4, 0x0a76c550cbe0cd52, - 0x889719362db67195, 0xb101bdd3a00b9de4, 0x31c48242850c2189, 0xb8cedbc72c88b894, - 0x4e465d5fd48ee749, 0xd766f805de5406f2, 0x7ed1021cc3976f2f, 0xa338d0c156883be1, - 0xdf521f5f24c6dd7d, 0xc1eeed65ef39a242, 0xc02631eebc5c96a6, 0x9cc1fbfd628e1829, - 0x7d73216ba0699361, 0x2540317d7b66fb29, 0x63a054452be49dca, 0x7d73742ae0848e88, - 0xb30c2497e6193a12, 0x2682980472f836d7, 0x4305ec0a00594e45, 0x3c8fffd1c0ef0c22, - 0x5e8be6dad5b64fbb, 0xf5ea60f4d282d001, 0x6e9e672773fc023f, 0xe9d74708e6696640, - 0x719f1c395cb08fd3, 0x4e062755f9935277, 0xd58ba0f178758de1, 0xe57887a691e40a3c, - 0x4b6d403d2510a226, 0x6f37aef02045c5a1, 0x008f63675158b1e5, 0x56e0067421054218, - 0xe31c514f72f684b1, 0xbdf6cc0f01470680, 0x67bc87bad09c83b3, 0x22075b949dd1d88b, - 0x86d9a9b0b96deab6, 0x17aeeb734d4eb78a, 0xca242e09e738ae5e, 0x900b79ec6294c63d, - 0x69672c4b972f1817, 0x1a5af20f146ddccc, 0xc2e57d514f17862f, 0xf73624d6efe17e2c, - 0x39430990b4d91983, 0x8b64914dadc0c619, 0x7b8c0544391d2f06, 0xe1e31fc32cde6027, - 0x3f17f49563b47ae2, 0xfb5ee4b36c079387, 0x4e5f836b96ee8618, 0x6483f553d7099c60, - 0x16a3525fd0bd9f5d, 0x11ae2d4141cd7181, 0x4d467dd4036243f8, 0x25382f4820103be7, - 0x04aad65be4933b78, 0xe7c4ed8ae9f5134c, 0x5d017877021c10a5, 0xa79be1811cb4de61, - 0x3e9df7054ad3831d, 0x0caace50fd8e5877, 0xda73ea1f41dc74e1, 0xbf5836056520acb7, - 0x7865e3b0301cdaf4, 0xc0deeb305d874491, 0x606a0ca63031e753, 0xf2a54ed007cbe026, - 0x73779695ebb09152, 0x30fbcf0a4283276b, 0x1e75eb46e4a3f640, 0x50c44ab9a1985f6e, - 0xe8e2fc4184069369, 0x7013d4094a8ce4b8, 0x3f8be01df01c8f56, 0x63cb13d88cd1c779, - 0xb66264ded853d752, 0xb31b29a495ca0c92, 0xb00df6192087df6e, 0xa281d1211faa8302, - 0xcb0747c65238201a, 0x03f13733287a1c13, 0x5b9a642b7f60396c, 0xcb5269137d2a7744, - 0x0bc1ce8eb3f7817e, 0xe9412aa2690b4a93, 0x9c7b6c344f263961, 0xff2f8074abeb6cd1, - 0xd906fb25cfc7c808, 0xe102fad0dd9c736d, 0xcce6097bbe848931, 0x9a549d00790a3373, - 0xd5ac5bb214d477f0, 0xd0e8919415015b0d, 0x37c6bd1b388568fa, 0x87aa0ffc80e394ad, - 0x18fa904f2183f17d, 0x7d3bf175b8289f47, 0xeed05c40748909b0, 0x36266c65ea9acaba, - 0x2dfe6d37337e1551, 0x926aee4b432ed24d, 0x5e47ed4d9dd318ed, 0x57f0130f81eff66e, - 0xad8258aebf497eb8, 0x48bd25078aa02f04, 0xea88eb9d81e10833, 0xaa17e2454f43d389, - 0x03a46df0fd92a69b, 0x01f32bbae2669e22, 0x5ad580a747cd4b55, 0x6058ecda453f6da8, - 0x0cbbd0556fb8a40a, 0xd91df481ba22ef1a, 0x88fca89aaf45bb32, 0x1fa12d9bf4141464, - 0x553159310c51bda2, 0x64906c434627252a, 0xf2d2be89f02099fa, 0x964e47a4c77e6556, - 0xbcbec19714595b28, 0x539ba6f42dad5062, 0x96fac67913e63c90, 0x83e9484efaeeda9a, - 0xfcaf061f65942e48, 0xa9b0a9ff9ae385dc, 0xb79dfce6f8238592, 0x14e3dbcfc512ce7b, - 0xd80468e0229cb23c, 0xddce88d71254c4d2, 0xf643e0d9422ac9d0, 0xe2810d05403fb092, - 0x2a358c9a491aa767, 0xc750c0db3b17e90d, 0x7d69bd6c310e3fb3, 0x84a186b2c6b13b1a, - 0x0f272a11bb195b36, 0x52473d0ca1fecf98, 0xd7e70d382e5dd11d, 0x608da68150d4e38c, - 0xb826eb9cf77f4544, 0xd7b9e81d43cba77b, 0x08aba5637422b57f, 0xb9dc3d8ce71a6116, - 0x3fc1d04e66ac073c, 0x6b4c0fe82bdf31d2, 0x25245b5d00f6ae8f, 0xee3c6d2b6de42506, - 0x9e20b5935d612b40, 0x65a2239589a359ba, 0x243862218db0c132, 0xb3cbda169a4f2560, - 0xd868a40fea9dbf25, 0x59537872a368d366, 0xe3682a3e0bd0fe11, 0x694f50065b928de2, - 0x0180ae2c81389703, 0xd883adf457e692a1, 0xf50876b2c4c9a1d4, 0x70c2464bd9f73a30, - 0x0fcd96cdd2434f6c, 0xf2d9a073dcd75110, 0xd62984d6739c8f0d, 0x18a15515603509d7, - 0x6c80c5f8abc713df, 0x9ab2917c05261cb1, 0xe797726444b2e9d1, 0x3d3bb63e1384c325, - 0xf53b3051901ee282, 0xe871771f78c33328, 0xdca7d82c4157a494, 0x026948364812b649, - 0xf761ba1b501366e2, 0xa470e09eab50a6b9, 0xf340e2b7f59adf5b, 0xcfc2f6931971b38c, - 0x666bafc230cf07ae, 0xa702c163e23dfc7e, 0xcc5560d0113157e9, 0xc840191bead63b3f, - 0x5c1c0c85be00f20b, 0xe7a01c32bb80bc0f, 0x72a14c9f01863b05, 0x86bbe6b6933313f3, - 0xf432861d49913adf, 0x8cd7df8a3545dc79, 0xa705374c617f236f, 0x289459878d34e42c, - 0x89075ecc0513405c, 0x5f1b994f544cc725, 0x97777064cf006d7b, 0xb21c55fed3c331df, - 0x9bf7c774d6d205ce, 0x70c6ac11d30c8d60, 0xa6f75076c3befcc7, 0xea10f55bbba6046c, - 0x14493f2043d6b71e, 0x6f8a5e31147870a2, 0xcdbb9084579014b0, 0x3581034700c347ad, - 0x694c3336c5cc1ef8, 0x06383bb7b56c3c53, 0x2fb979d1bf93a2a1, 0x07f9493af6990c65, - 0x283c2fd2fd2b30ba, 0xb1fd6466fef48699, 0x96f85d1f7af606ba, 0xc43115f321ca5eff, - 0x9e79c155ded2afa4, 0xdeccd382f063730e, 0x74f7a752e25ab419, 0x783a86a0e2d84dfe, - 0x0a3d9dbf5c42bda1, 0xd9fceaee360847b2, 0x5f7b4d21e0858c13, 0xd214cf14399ea948, - 0xc28c589e31388e3f, 0x626a27c3d7587430, 0x5c555b03ff8ff699, 0x3d062a1ebadafed3, - 0x400efd1032187b8f, 0x9b74150fa004ac7f, 0x7f9ef9ef8851c3a3, 0x26a5dd7a22bdaaea, - 0x31d33df9074f03bb, 0xa02cbb077c7a9672, 0x68f3178a5157aebc, 0x4a57132fbfb7ef24, - 0x3cf263f1ba232bec, 0xdfb40cf2e6e7d252, 0x9834b9ecd8eac612, 0x105c09695215cacd, - 0x69d31e2bfbe6edf5, 0x27f4a28cdf40d6d5, 0x66dfa27d9e0b394d, 0xa2d0bf6e8d09cb3b, - 0x084e66bb5a4f69b4, 0x4c37de99f0c0cd00, 0xaa64cf6a8b188876, 0x716d7c852cab6b5f, - 0x99b8bb5b5f2e6bc1, 0x2f5f74985880f613, 0xf94612f1e743418c, 0xb9449cf3d9a9ea11, - 0xdd2447eb385ce8da, 0x1408dbf66bde6567, 0xbf708386fa5dbe37, 0x52e793c08ebc76e3, - 0x519d9f4c3d0e310e, 0x2a4ecb10ac2b04c8, 0x6f8e19c2d57a34b8, 0x3bd02f65bdac6b1e, - 0x9729ad60865b3d8b, 0xa444c0770dc2d599, 0x234c4f2ec29d26a4, 0x7a3040be1d17ca4f, - 0x1ee96841ec8240b7, 0xf0240b6ecc33a501, 0x355800a9a2c90832, 0x8e33565022ffd441, - 0x82433d3e2514768f, 0x60e2a5753349e7d6, 0xdc0841e8a4089800, 0xd86c8a45a30dd6ee, - 0x4b489c3decf4e1fd, 0x08acfd2cee31c580, 0x51a37c7ea8c14013, 0x71a2defe4e8da738, - 0x2545cabc960550a0, 0xe548a32519d70da8, 0x757203a8f4d75521, 0x271c2866bd8be26a, - 0x7a2da670931a4f58, 0xf5eed822dbd791ef, 0xe1166fb06b0f28e9, 0x0bd949d87b079069, - 0x28204e86e655c41c, 0x258a3c6f1673203b, 0x1e59e86d04380244, 0xde1a3a599931170e, - 0x92a01b8fd8e1267e, 0x560eddfa3837f984, 0x521f9b0440e9ba83, 0x813efbb701470505, - 0x8dde80906da578f6, 0xbc0a5fa235181fb0, 0x4f8ee4381a7a273c, 0x0ef7100c1fe9d2ec, - 0xf063e2a2f889afd2, 0x01edd5cc0d3cab72, 0x570bdd2ac22923f6, 0x613686f7c659a6bb, - 0x987052e905889d2a, 0xfe72524e7544dcc7, 0xa4b222bc12c656a2, 0x743bcd539dd0c232, - 0x6b883b6ec2443f30, 0xe638217862a2cfdd, 0x53a6f188b1616900, 0xfa901cb976d9f9b2, - 0x1365db124de8da01, 0x60a7d81f9b91aa83, 0x6e1e066cbb9137d0, 0xc0844ce514ac77c5, - 0xd214bb381e11217a, 0xdb601f662399f610, 0x1fd0160454fdf365, 0xa409ce9dcf7466d4, - 0x0f6632058ee58c94, 0xfa1aa4c0fcbbc5d6, 0x2256ac40d8755bee, 0xe9cbf1cc56bd76d4, - 0x87059414203c36f8, 0xd84ab79422e2288a, 0x74c81d6c8430cfac, 0x3d4f7ce973d3dc57, - 0xe098257b700b1964, 0x0ee9a1f5c03c3fdc, 0x3d48ee56902a14b7, 0xa0264fec88ae359c, - 0xbfb9fb5855ebf9b1, 0x44aa0714d94a6a0a, 0x580a06fa8ea19f0c, 0x269ee7ddec1a0f89, - 0x0daaf7a44494b10a, 0x2703eb785b7fc1cc, 0x45b1e5a01c5e9e3c, 0x1a2e53db41f4b4ba, - 0x706e08a93f73b6b3, 0x2cf6771c1d66f31b, 0x4bc96ebe0a7ae002, 0x83df17b4341f25ce, - 0x41480229c590355e, 0xd802f520fcc7c8c6, 0xfad5b6cec14e9cb6, 0xb1d40ec2370c8a21, - 0x828273b5b2d9763f, 0xa9f818ce69f355a3, 0xa96dae508268e86a, 0x508b13a181d39274, - 0x42eb7ce556563780, 0x0a283934c9125808, 0xe85ed6ce1d904def, 0xc376ba3d26c7f3f9, - 0x9930c4713ef83913, 0x62d98be760c25c1f, 0x2ce24044ba9b4fbd, 0x758f1cf816a7848e, - 0xdc705606f2ec2705, 0x3de9078f6124874f, 0x9b020ebf2ec0f8e7, 0xd9986570b7f1c60a, - 0x17cd3e6303bd4857, 0x7504ac42f974647c, 0x4ebfd2efa9a3c920, 0x68d58c9e02872b03, - 0xa8e0fcfbfd2f6a42, 0x5e8bcbaf671313a3, 0xaa4b40058b612561, 0x76ffb64c8d23cde6, - 0x6a92e17bf0182283, 0x89f3d2d362deb744, 0x44d2dd2c01d19b1f, 0x8acfc93a87797dcb, - 0x9a52bd8c60ed7f9e, 0xd9241858bfd25c5f, 0x43fb099239bb8a1d, 0xefbbb644c4f7ea5c, - 0x534928753089bc5f, 0x0d96aa2b69af172a, 0xc71c0b64dc05447f, 0xd5a207d0b657f938, - 0x6daf7ab9aca5933c, 0x4eb44e34f672de80, 0x8fdb909eab476387, 0xb02c19a8cd7b38ac, - 0x332dc7a68f73298b, 0x0ee6e9a92ddb7d77, 0x68e4ca04ed2a2d7a, 0xd8c9e8078309b149, - 0xc859ef21bdb9790e, 0x61a18792b92635ff, 0x5b2149910043716b, 0xf1bc5799725c1271, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x47cb316daea4446d, 0x1951d98ce3ae6f79, 0xb7148c3ee14ddb8e, 0x529d00790b29995c, - 0x0e6944b71353355e, 0x442df4b06a9d679c, 0xf4d7d8ecf15bce99, 0xc2729812271118b4, - 0xcbec423fd5c2299a, 0x704ba2ea8625b17b, 0xbfc7840f681706ee, 0x33353b5cad9c638a, - 0xc2b65951be6563a6, 0x851b0da694ef4f71, 0x10fb82a89c66e789, 0xe77d6281f2f2c293, - 0x1d4f177acd037e60, 0x3f8cc338e2be8ec2, 0x9b3c04eaf322d105, 0x28efeed79d0b0bcf, - 0x42b8f17f9f28b232, 0x30ded5b11c39b20d, 0xaca5ebe1a69cbb0e, 0xcdcbb3e55222b7f8, - 0x6fff1331ca5d80d0, 0x07743e6c4e604abe, 0x0b72ce3024ce8baf, 0x4cc2563caa3c8421, - 0x65f799cd78856906, 0xc8c8935bdcece665, 0x213ca01f187ee65c, 0x72e7becac6f1ad22, - 0x10f649dc860091a6, 0x143a53ef13efb407, 0xa3948617437489f7, 0xeed84b264ef67c1d, - 0xc3f103cf0dc4c5e2, 0xcd5672e8f36d5cbe, 0x02e7aa140381add5, 0x90f53bb2c1798b4f, - 0xa93b7e2605969c9f, 0xb244078a237bf710, 0x189c129beb9776ee, 0x291383865815f73e, - 0xadd260584c31640d, 0xa807499843f1fe10, 0xfebee385bbb83457, 0xa0e3d3196d896f3f, - 0x5f39077784374edf, 0xd70c57651867dfcc, 0x94bfdb26142812b1, 0xa8e6f6791099039f, - 0x84ceb1ece9923461, 0x91ca82b4f734663e, 0x545f9488d8bb743d, 0x16ac9ea0959b1d71, - 0xec93068e54fe6370, 0x23e8f22996689b71, 0x33740d31eb184703, 0xc84f773116418155, - 0xbd0ba404052c2c4f, 0xd6c051c1eb21b54d, 0x0ac0dd54e06261f1, 0x380245f267754403, - 0xa50b0c8ef0017a3b, 0x2bc6765a017852aa, 0x1a1adecee30de67e, 0xc9c556e56d906cf5, - 0x69196fab4542af68, 0x9fa1700ec6374adc, 0xd35583cf612d0fa6, 0x36f38fc3e8d40eab, - 0x2f1e71d61bc1ead0, 0x724cc10a8a09c43c, 0x042d4dc607b5e38a, 0x41438e3a0dae1e43, - 0x0dc2b6772ea5b922, 0x581f84f233ef2ad3, 0xbe9af0cacb8a9f03, 0x1e2443006ec883f0, - 0x96175ac2efc8e191, 0xa9c7c6f669170786, 0x4288dddb2a127a38, 0x7aa54716c2885182, - 0xfdc4ce25aefd55bf, 0xb295e7dad3dee7b3, 0x15cc19af07873c9a, 0xa7bba9aae9a100e5, - 0xf3ed65fe29a42e3b, 0x1609d3ccd4980df0, 0xb538b1096d1271b6, 0xae96631e92f9ef3e, - 0x2e33eb109f4c082d, 0x83ff675eb63fe29a, 0x3482765c2cf3f6cc, 0x49bcbbb7de41ac68, - 0x4a20c4923377f0fb, 0xed53162ac48e7879, 0x7f20ca8caf99f8b3, 0x8f8f6adbd7f27490, - 0x09a0429ffd36849c, 0xc1e9629b88840e3e, 0x8c13728a7747eb1e, 0xa00874f8321d00b4, - 0x059eef8543a7d067, 0xde7e4d040aad2c4a, 0x099560dbfbed784e, 0xa46808fe07aea14c, - 0x998caefbbd951923, 0x4a079110f47364c0, 0xc8d623516ae34623, 0x30411d5ce9e120e8, - 0xc673cc7c96f31c64, 0x2f3f72499b8ef808, 0x347d54c095cb019c, 0xcd80db5a2fcf8e4b, - 0xcb8163d474bde015, 0x76df0e64c94b8e1d, 0x7caf4002d6b5e32b, 0x918dab05881c3ba4, - 0x85fb5a4e3290e444, 0x6f8ff530cc29ee14, 0x8362006271ab4291, 0x0469b4d3f24ee97f, - 0x23c7129756f2a337, 0x756f56b50fa8d68d, 0x9d0a9d2f85a1342a, 0xed1c6a919e2cb92c, - 0x56a0d97e41cac1e7, 0xed34cfec61a63cd4, 0x6c38deb8413b7e95, 0xfe46eaa74668e061, - 0xff34677a9c214500, 0x55de956f83a1dd96, 0xaf792861304df3a6, 0x4da6d0710faf12d5, - 0x2a5dc50923cfee45, 0xd7aa50a91b7e77a6, 0xfa465360ad5ccf84, 0xa160fb98fd14590e, - 0xf453666b5a6a8234, 0xb470b84f2a78ba87, 0xe333706269ceae4a, 0xd8a7f4503a7e6f4e, - 0xcd638baecf55fc6e, 0x917472517219de08, 0xaab34ad80912cd2f, 0xe1680a485af2a691, - 0xa32b31bbe687ba14, 0xb663dcce34e070a0, 0x27b39a1b925a82f0, 0xc900fb075b7b676d, - 0x594664aa07b0d620, 0xac339c094cc3eb14, 0x53dadcda8e2efcdd, 0xbf3b7b3a9a10d5da, - 0x90262f690ad4579a, 0xe54f32aceca1a63b, 0xf1c3772584ecfa57, 0xb2ae4c75d776d718, - 0x9ccf0f69e3d7556b, 0x52f085ff31cbeb2d, 0xfc45f7287d7285df, 0x2bc6a12c6c003d65, - 0x7e8cc21f50b5f71c, 0x00f98d230908895a, 0x582d2664a34223b4, 0x3eb7c2bef8223a03, - 0xafad7bff8c46cda3, 0x24976da0869dc94b, 0x7f80682972f7f3d3, 0x0a257c22dcf077b2, - 0x7b0097d30dd81fe7, 0x875ea948852008f4, 0xbbae190cb83cd7e8, 0x664bc047360ebacb, - 0xb3063486d9b4ea84, 0x8723129e521ced43, 0xf4dd00f371fb45e3, 0xc68cdd5f48009724, - 0x23a1ce213de11e81, 0xd1c2aea3c75b435d, 0x92f1c67dd1ff86d3, 0x25ed902c664402d7, - 0xd5ccc0d981c86105, 0xef38207fb6b53c9c, 0xc3e2db630efa6c87, 0x1c1ccc2c1bc04c8e, - 0x0fe21e302194d8b9, 0xd2bfcad02e6bda28, 0x4574300cbb38ce15, 0x867f16a835add5fc, - 0x7d884694eebe3ab0, 0xecb149d5705c2f2f, 0xc0d0c28f3f7c1a56, 0x3f4a2adde22e8524, - 0x13254d1970d3371e, 0x3c4352c7145bbc10, 0xb42ae4836483b273, 0x6826beb9c032efe1, - 0xfadddcc70d492dcd, 0x03c498a880bfc6bc, 0xf6e4f6031bce807f, 0x7e7441bdc4b13200, - 0xa3467dbe77c74c92, 0xb90d82bc024a6ac9, 0x11f09ea3584ce63b, 0x3619a167e2607ff7, - 0xefec2f945aeafbcd, 0x2dfd4f62a789d762, 0x55a7515e54f991e1, 0x5061b50a19bd1c72, - 0xd3385157619354b5, 0x14699892fd831a54, 0xe0bf057ab6f655cd, 0x61d28d98ff39a941, - 0xb8d9ee573902a2a9, 0xd782b6aa9a24006c, 0x2c9748b3a91d29d9, 0xaa104d93bc4a5e17, - 0xdd1756dcceb4c938, 0xcd84c8924ed882e1, 0x35873129ef573559, 0x745d64236733f746, - 0x61483f5519e6c605, 0x5df423a8d6cf44df, 0x51d901016c72ff7f, 0xaeb4d808de28e1c5, - 0xff6fc0dbc1adeca2, 0xe262917f973040ec, 0x6961452836b1357b, 0xdf4eb4c6908b2470, - 0xcaa2e62e69883261, 0x2b5053b94e0f50d3, 0xc9c8824c766e90f5, 0x975effcc70b8c73c, - 0xc085364f4cc9401d, 0x9e7b5f9f1032b91c, 0x4fa4cf11f7c79943, 0x1997f1c8d634a144, - 0x36b6205ba9d81aba, 0x44ec41fecfd9fa85, 0x6f29d3728be3c8a2, 0x810ab87a65196835, - 0x3b2fee0196937387, 0x7176f65fb338d2a4, 0x7de602c866a8d715, 0x1949fd7f53364725, - 0x4e503250f063448a, 0x287ef746ffabff92, 0xf1b17d0c50fbb9fb, 0x8e184be143d22139, - 0x82199f86a4d16fa7, 0x347721622b43cdb4, 0x6d332a0a44c016ab, 0x71792bcd04504ff2, - 0xd7c23253f7801b75, 0xffca4691bb4a7f6c, 0x38f974a36cedd305, 0xae5bd756ea76453c, - 0x7f802f51225cd4e8, 0xdf10c32f2bcf3945, 0x236193b2644e05b4, 0xcf541753297c8992, - 0x0d08c9fadb641ca9, 0x1d240820083d4fd7, 0x51aed68225884c12, 0xb343010b93695ddd, - 0xcf8884e73886bd75, 0xf973c8e5421e0496, 0x268acc5398d9fed1, 0xac7acfab2910110c, - 0xbaeaf17ad7fff99b, 0xcef700952a481763, 0xf30c97e9e2688049, 0x968ba91addf95150, - 0xdbab7bcf53eff647, 0xf3a6fff8b4f471a0, 0x05d842e0894e6ae6, 0x45dc59c109c8c876, - 0x45e326285966227d, 0x033cc45b8a557bc8, 0xa7ddd9e723b09cac, 0xa828b283b4661187, - 0x72f19d5293f0be95, 0x6169e406134dc960, 0x6d8151a7239fa753, 0x48d23ea63515a23b, - 0xf11bb3de503ea6bf, 0x5f9ee2d662870eef, 0x60b595c51bb84fe1, 0x3afcc46b4a6f263d, - 0x693a38f9bd48333e, 0xf4ea534f8dfed1f7, 0xe76a7aba7270adc3, 0x6ef7512dbe51dea6, - 0x2ce7df1feef70a0d, 0xf50fd0fa12b2999b, 0x87b2766766a97c83, 0x6e882dde7d6497a7, - 0x298e5298d267376e, 0xc891c72ebc0261b7, 0x5a197cf8ec6868cb, 0x66e48ffd0ff1418d, - 0xd242f03716739628, 0xaaa329db92ce8133, 0xf0dc48352d745444, 0xa3d2f9200b0e221d, - 0xe4009afbcc0d15fd, 0x998d25b96d0de8fd, 0x67cf64f86bf785f9, 0x7b5fc4dbf5735d10, - 0x83df23153143e04b, 0x22a99125c73c5a6d, 0x051c91517fe3de04, 0x13652a71d7e1cd45, - 0xab078084feaf04b6, 0xa2c9b439282ca036, 0x7907ec117f87202e, 0x3be8c40a15315dc0, - 0x72cc683f303b4766, 0x704166544bf83523, 0x7e139628355f78f5, 0xdd09add8ff4e57e5, - 0x7a585378089aa255, 0x79ca69a5678573ec, 0x6f486cce3cfd8c67, 0xfb6b0e2ad41f9683, - 0x828a251c85edeceb, 0x1889b443c4c5f958, 0x0ea11dd07a38aa13, 0xfb8d47f003f0b660, - 0x29d6dcff77376523, 0x53754ce8aeaddeb4, 0x86508b071448c4e7, 0xf792d741169f62b8, - 0x0c7bf325d1724ffa, 0xad1d8a7328b8d159, 0xed25e8104dee693f, 0xaa0c42d660566999, - 0x59dab08a756f7d19, 0xb31f5841447a6f6b, 0xa749e402fc1a6f6d, 0xa74680f8e09221ba, - 0x3721282011e718ab, 0x4e135060f98410ea, 0xa6aa9cd85563caf0, 0x073053dd8ca807c7, - 0x2aa679e5c2401ed4, 0x4b38f9d7d09731da, 0x66bf953a6335c2d3, 0x601b6feb84438214, - 0x4cf1a02e3c0f4c57, 0xb9a4e7bb82ab37a9, 0xe6aa242706f354c9, 0x665edd041e1883b3, - 0x76d1adaf1e01ee9a, 0x4431789abd420625, 0x792a05f7a10a4944, 0x3ca1135ec3d3d39e, - 0xbdbf76ff77fd0207, 0x61042fe5613d9603, 0x4efccade64f769ca, 0xf799a7437f32c6d8, - 0xf09ae0e7caca0f66, 0x960098c6db2a0e99, 0x425313ede42172b6, 0xba24954b4a181acc, - 0xcde435229275b83f, 0xdc3e51b9dde9e591, 0xf0c3bf296161cd97, 0x8a4554ef0a626f30, - 0xefc19b8265aee9f5, 0x00a3156603833e10, 0x93584c3c940ded54, 0xa1475668db068cf1, - 0xebf8fa31471d7ca0, 0x64e3c3cfea5e3eeb, 0x7e03d2e778592fa6, 0xe98265e6de7ab471, - 0xbe9bd3ef6d3067f3, 0xe8cb06bf3d18c58f, 0x80264e09dcb3015e, 0x212c431fb9a5c174, - 0xeba51260e4960b60, 0x4b3d953fc4a7a7fe, 0x379a5a6b0c3a4957, 0x853f6c8faca0f413, - 0x2c629ff34faf1f36, 0x067c4531626d07b0, 0x0db77c91a5a15c32, 0x3ddeb4d1782478f0, - 0x0db951b3cffc2146, 0xf909b2d9ebdf32b8, 0x08da82db8dcf6e58, 0x4028a3e4c597a3ee, - 0xbcd5db4732d38af3, 0x7e432dddc9934aac, 0x493434be916138e6, 0xf1835dd64c822313, - 0x4000b4d6a6ba3295, 0x5f8e7afadd35d2e1, 0xad36e660029e7cf2, 0x4dd64c262b5a578d, - 0xb871bd38de2a76d9, 0x235a4fc21d72498f, 0x627cbc0775bd48f0, 0xea62f8b63620af68, - 0x4f855530302c2354, 0xfa092ad1cb2a1fcc, 0xe4b7c36c7d50017d, 0x9bc32bfdb6a45806, - 0x7d57edf81f70d8db, 0x297e985919f3d59c, 0x276758dd38787959, 0xea14765fbe238fe4, - 0xdd43cbad080c8089, 0x34454eba4d36098b, 0x96a81a8e493b3e5e, 0xfa36f6569e8e5b8a, - 0x632a3634befe270a, 0x932f7e0b969a49f9, 0xe3c655d0c06b08de, 0x60b8f06c6861583c, - 0x39872ae9ab4932d9, 0xd9bbbd35f32c48f5, 0x9e83f41bc1af1c61, 0xf0ad701c188c67f3, - 0xf3480953397816b0, 0x5930c8aeb92b2b05, 0x70318e477f997b41, 0xddd5e12fcd674ffe, - 0x699c40ae0814f288, 0x59986c252ba524cf, 0x4a7000843c4d1ce9, 0x8c0216bff40533b2, - 0x6efc37b594987609, 0x7b8e8659bb21816a, 0x44f8c072d4244980, 0x9763a5a1ffc2284e, - 0x57284699a4b29d76, 0xc5ea66e93bcdb658, 0xa92d621e1b324485, 0x84479b39cd5ea278, - 0xd6e4ddcbd3c649aa, 0xaaacdb7a319656bc, 0x77d5568b1b583f6d, 0x4b7ba20c8e54693b, - 0x4055c17ef7b9df50, 0x2d313395c890f8a8, 0x16a4e5fc3172fefc, 0x980661c87c33184d, - 0x55ffe897ed35ecc6, 0x21c7ba4702409fb4, 0x28f5687b1908190a, 0x235e3c15f38226ee, - 0x0460b23eaa313168, 0x9926bdb8e71c202b, 0xfdb3407cba0b6486, 0xc1b1552443aa19fa, - 0xc6cc5507ac2c5379, 0xeae3d62018ed9df8, 0x1a86f1ce83fdf045, 0x855518254d9a4192, - 0xaea2c4b19b71d093, 0x27fb0175bafafa07, 0x040806d8f6bbd258, 0x3529b0bcfe57f4ed, - 0x6f960cbdc7eb0e16, 0x288efac98d684114, 0x3ba44bb950d24c59, 0x6d2a43a4969f8467, - 0xf9aaf9a10ffda60c, 0x87fd0ee930173869, 0x11cfb125401f56ee, 0x1dcd1dab3cf955ea, - 0xc557d042bb3f72b0, 0xbcca1b773e40fcc2, 0x97af114ceb5c4b3f, 0xa69fc92772b72fbd, - 0x666aba79dfc2dcbf, 0x595a1614bf33dd4d, 0x21b3e745ca12809d, 0x08a014ead0f1c42b, - 0xdf6ec5c128b1a985, 0xc394bb039c7abfcb, 0x327c0cbff595ef3f, 0xe9f2b89edfbde610, - 0xfa2c03d9e372b12a, 0x8c1956db1b0b6f1e, 0x630c871c996008a6, 0x1eb6da14c5c87bd2, - 0x3d0c76dfc273dd35, 0x1852a10138e64305, 0x12cbdcba4f7749b6, 0x9d646b9a39d398dc, - 0x001377cc14f6e69f, 0x694a04d7199f1370, 0x064cabfa1617d659, 0xf2917f4b3e041591, - 0xa31297be87524fa4, 0xa998da9e100e3a42, 0x44aad0fe965ac824, 0xc0301443dcfc5acb, - 0x3f2193b6dba57e46, 0x0bd47082b2578ac3, 0x0f89e338ed507cf8, 0xb8162fc987a33f5d, - 0x3632d1dcf107fc18, 0x6ac325bfe3a0be2f, 0x49f8dc81579733ea, 0x82bbe493bdb803d9, - 0xdfac95e92d79ac8e, 0x4a285f579b518ac1, 0x4f4a4aea08909091, 0x6f433216233e47f6, - 0x924bf19673ee901d, 0x04082aabef5b459b, 0x93c5ba2a98b61546, 0xbecfe21475dab497, - 0x7aabc67b85716bfe, 0x2eda9d697a833bcf, 0xa1464a06b8ecea75, 0x721b825e794aa4cb, - 0xe751041d4e5e5d13, 0xc555e497c0c3077f, 0xf7a9459b707ef6b2, 0xae919c9673d32555, - 0x423d9bd7b4e20617, 0x288aa5523d9b6c48, 0xed9e31a80fdf2bca, 0x9b4bce3410b63370, - 0x6485958b1dd80350, 0x75dacb5228027426, 0x70cd6e76243c99f6, 0xbb19667e246e56fe, - 0x2210273f1c3e358e, 0xa103ee4e62c318eb, 0x2a4fd1aeeb834dea, 0x8f59d7cf5c78d96e, - 0x48373461fc75fa33, 0x09ed4eaa83376b05, 0x6e7d4375f60317ac, 0x7156fcdd0880dc03, - 0xfc1ba0c2f1138649, 0xe4d10a9c057f74fc, 0x334ce263b7898ca1, 0xca290e92f437a928, - 0x45ba0b07cb32c7d1, 0x4269315a8c8c8505, 0x49851998931f4183, 0x43fa4011a70ecd3f, - 0x428f6b5f209719fc, 0xccf1bb0647981d81, 0xbd25180bce544d86, 0x13f3e913e128411e, - 0x75a7920596a35609, 0xb3ee52adfab44c63, 0x01295f8681c67165, 0xc2fa9c41b8eade42, - 0x35c66e3b68f78919, 0x7034b0ee6e342860, 0x4623cd0229761f82, 0x7cb18e1f019c2b02, - 0x50d15c51a7c905bd, 0xd00fa6856e9828cc, 0xc0002e908755cdf0, 0x47750491cabc0271, - 0x6bb6f6172283e05e, 0xb012eab6321c6ec8, 0xac641d597d9019c6, 0x4d648d043a4e8748, - 0xa2375339001b9cf9, 0xada4648d8d0dc3e4, 0x4004d12b0a0d1dd4, 0xd20387e8759a524f, - 0x587c6c770f9cea80, 0xc5dc826b2a921d07, 0x8f0f6b3de14c9fe0, 0xb89a4e419f3c0993, - 0x5fe3947f7de56f90, 0x24184367a51fcca4, 0x9d94b51c92db4dbe, 0xbb2309bd85800234, - 0x8d6fed1c90ee95d5, 0x96eae22ff57eb7ba, 0xf826197f2d07cbc7, 0x48d525e90dcd1623, - 0x6016aa5b6ae100ea, 0x5ff8c63f5b590a1f, 0xb59b2bf14817ed39, 0xa192056432a8d025, - 0xca4cea8599dfb866, 0xc35b85fb1e017e9c, 0x9039f1bef7a4a410, 0x7f1f289608d04c23, - 0x886074a37d8bbf74, 0xa9e2e5faec088d67, 0xe81e0b6aa76f742b, 0xbf00e276db6b7138, - 0xa14c7390c72952c9, 0x39763b6d4c852242, 0x1ca6b5327f716b38, 0xb647d4e3a8116bfa, - 0x3cabbbaa96cdb599, 0xf1f1c103fd93f3e2, 0xbbc5663208574c6e, 0x35257365df41bd73, - 0xb38c0269a2ef7ac6, 0x695bc8fc07c22ad1, 0xf5d65950fce3839e, 0xb6e3ca2d45f8f8a9, - 0x78a57b935d45070c, 0xc4665585fb5c1678, 0x3b59ae4f705a7543, 0xfede23d7b6d5780d, - 0xe8a8d460b43725c3, 0x361d8c8c68440d28, 0x13f356072e5fcdf3, 0x2882d50c04da36bc, - 0x9edc7d2e50db8663, 0x0745565d51454fc1, 0xbaba03de33903404, 0x1424e9fc1b4f9c17, - 0xe843201343e05786, 0x7c2ea99cde59abcb, 0x287299373d7028c6, 0x8ec81191abecc4f8, - 0x7d9cb74299ccb8f1, 0x9ed8d1b243a797c6, 0xcf1c00bf484c3a85, 0x3238b380b4bae9ef, - 0x1487ed49765375b9, 0xa65a82d8fbcf068c, 0xb68be07ecfa34900, 0x02268f3ad4497df0, - 0xc3436e5766042e7d, 0x2f11398b36f348d5, 0xe50b51e4de21adfd, 0x871681cdf4135dbf, - 0x9f3155fa73776c53, 0xb64123f08f711456, 0xa2d0d22f23cc6685, 0x245a1b7088c1a2d7, - 0x819c3a58be83483a, 0x6fc4ffb31c43c956, 0xc879a30ae1be570f, 0xb2f37760c9176243, - 0xe31baf84c38f7595, 0x069c6c9ad922c3ff, 0xf62e1d24c751f9fe, 0x2104c723c93273ca, - 0x6ce809b56520d52c, 0xf91c38465ba2b6f1, 0x093ddb1b427579e1, 0xdc19de78b762460b, - 0x53c69c49c1f9724e, 0x8b0265fca043604f, 0x7f8f89b9bfab5db4, 0x37f63f4fb2487664, - 0x2b3440b91eec4044, 0x9563be670b3cdeb3, 0x0758e285978c74bb, 0xe3105c1f607b3468, - 0x021379fc2361b7d4, 0x37a76eab34f18162, 0x69858692b5445131, 0x428380fe0775e500, - 0xe809432d8e597b44, 0x57c54422f33f6f97, 0xc2ec6cf9977f006c, 0x66fc757e7e4b99bd, - 0xfeec3309152f9925, 0x054ad60e0e7a46ce, 0x9479f4b64541483d, 0x67e6f938c0b64e21, - 0x112b71fd7a9fba7c, 0xbcb4f119fcdb4850, 0xc9d138daa7702b58, 0xac42ebc3175ba914, - 0xaa16df74c28eafe5, 0x000c43a79776d8b9, 0x3fb46e8f755a665c, 0x6c6cb9d59666b7e0, - 0x64e2b6eefba13c34, 0xbc1f817a9285d039, 0xbe2574491afdf18c, 0xd8e7c71bf466fe32, - 0xb7263c83586e2c32, 0x85e9a07da29a1aa8, 0xcc03588a0a8994b0, 0x5b69fa2f454f57c3, - 0xc7cfe9c4a9795ce2, 0x67894d6d49e83794, 0xf533bf76330d98f5, 0x708405f9dcce681a, - 0x60d978cffa081e3c, 0x5be06f645d693a06, 0x1abdeaa0f3301e9d, 0x19dd54b112ba50d6, - 0x52bb9528e5223894, 0x635070fc469a8ad2, 0x411eb7511019ae17, 0xa915001fa159aabf, - 0x259340edc94fa139, 0x424dd6c0b7e58318, 0x99a9b357f0e8a901, 0x0a9a3668a0f2fad8, - 0x91b249baf870b0b6, 0x7d9b2c5bcda0e175, 0x1283b326f609a375, 0x68c0ab09cd0460df, - 0xcc58c383a4880227, 0x3ef37c2a85e73f27, 0xa6c87b71ee14fc41, 0x64936b667b77a39b, - 0xa87cc1c311d57939, 0x0e12cea78550a98a, 0x0edf3416ff006295, 0xeb8f3540f57987d1, - 0x0cb937a5ce8a2188, 0xfaf6b5689ce2320f, 0xbc302819b0376102, 0xee6184df9a45b63e, - 0xdbdad8a918653125, 0x5707ad611d2e6f34, 0x1ec378c9f4db2853, 0x63141c1d21d20498, - 0xebba70461457f020, 0xd9c6c7a94b852e21, 0x174fad4cbdcdd88a, 0x1ce8fdde17bac1e4, - 0x88febbffcf1d3f55, 0x2b602e30c2a37768, 0x05663b30101c7a7c, 0xd442d0cb93ef78e3, - 0x9fa5d43570bd84cf, 0x79397b09174c3fff, 0x40ec092470734a27, 0x30b408ac34025f2b, - 0xd9bc7a63aa62850e, 0xe87d2b829efa4406, 0x6f496234b5e2a055, 0x5021867a710481b3, - 0xb35379d77d6241f6, 0xe56fbd4bf3b02b88, 0xee7be2d64cc67585, 0xfe34042bdaeb413c, - 0xd69badb932f14be8, 0x796633824168aea9, 0xc4d1607a893bfbaa, 0x864df0080dee784a, - 0x69de3fca59ccf044, 0x26bbbeceb6d1aec4, 0x4bef826bd9c2714b, 0x08bf0c4394a7cc10, - 0x295cad5f10bf1038, 0xcfd50867a7deb835, 0x398035d3a934ed5a, 0xd02c0c31f2eeb615, - 0xc8263073e51ae573, 0x9eb3d04c1fee5c9c, 0xf495c0a301c45247, 0x71dd00d967ba79ea, - 0xdd2f22907f57a746, 0xe56d25c8fc9db477, 0x68b020553412a08c, 0x0760ce0cf31b722f, - 0xdb4e6c427ef76516, 0x494a53cee10d34cf, 0x7fd825f55bf4a4be, 0x3b0a3173560135cc, - 0xd082b29647d98c72, 0xe9ca03be70de959e, 0x2fd5f068cc93b311, 0x3ea8fbcf90efea68, - 0xbc977cd1f2635187, 0x62c0f8b4f81a7f5c, 0x166247828fe3c2a4, 0x22c8067c262b640d, - 0xc0232bea1e9e7386, 0x1d0b61512ab417d5, 0x7935e8314da7d05c, 0xfb4529e0a661514e, - 0xa9a5a253553ab284, 0xaf05514dad66a498, 0x6cce42c3b28cd2a2, 0xbf2eb430f2ec5b57, - 0x95c54a03eebe98f5, 0x28101319b8d1e262, 0x75b6661a98fa5c34, 0x37f99e27f124fc79, - 0xf034af90ca8906b2, 0xcd73c94dc7f3fbc3, 0xc64e8e2d02110a9e, 0x89aef42d28f67353, - 0xb257700c28a619bb, 0x3e2482c333d0717d, 0x48206b6b1943c481, 0xee9e03015007ff7c, - 0x83249b9377caf192, 0xa1396245c04896ec, 0xe59924288321090a, 0x9c48279b72be9bfd, - 0x9b0d6e0f436ba245, 0x3fa440fdca7c5927, 0x1a5bcc14f595799c, 0x8edfdeddab469aa0, - 0xc14f62d1be917def, 0x18e4157fab5255f9, 0x2f6454a15e9ff53e, 0xbc62423127ad4365, - 0xf97c76b0f9949566, 0x02e91eafe1a67feb, 0x7d6d192dd561632c, 0xdff1d45d66c66d9a, - 0x803a9199f1a56646, 0x03db75de54454f70, 0xace0868fda2642b2, 0xc8f008c0212c3e95, - 0x9057ee7cc43576e3, 0x26227f6795e29ef3, 0xa3bf923d51c7c2cb, 0x4cde9f0ef5700132, - 0xe71234eb70d03ac1, 0x96391244c1ffda3a, 0xe579efe1283b4825, 0x977ef1846832e63e, - 0xacf7b77d1581a80b, 0x0019b7d5e2d2a3e5, 0xd75e868876250920, 0x44f83a7ef51f6857, - 0x1c7373654e9829fc, 0x3e5b1c3c23c3b5e4, 0xf0f10ee3ab5e0d03, 0xa247caea0f5d4003, - 0x9ff14eed5de0a13e, 0x63f81ade4945da25, 0x35b6385213119c79, 0x3a3277733f095313, - 0x656bd0ef38eb78a8, 0xfcd5186d88a8ce66, 0xe20c27a0fa96d44e, 0x377d8a0bf100eae3, - 0x080c9ae246a0c063, 0xe875149e21d86df2, 0xd347565727c083bd, 0x02f8721aac0487ce, - 0x7752851fa108937f, 0xb081c313f237f377, 0x8b9526db8ba325fa, 0x29c66a68f57918f3, - 0x0512e9646f2c5137, 0x089781692547d02d, 0x6569b9392b5733d7, 0xfa4ccca436aa2d6d, - 0x803bb51c3113cba0, 0x562e69ef558c64e6, 0xab9047f724161343, 0x7f5d285969df4c36, - 0xd15765332d5d48ba, 0xc1974bd021b3b300, 0xe6770281002ba1aa, 0xe6d1b80853359a2c, - 0x590f9dad2c5a0d8a, 0xcf8a37435edd29ef, 0x979d39c9efa9aa8e, 0xf83a4da88bc29f6f, - 0x508a3d5e22aad5cb, 0x68e903ab0d52cb71, 0x312955296d2218a5, 0xd6763edcd49132df, - 0xf2aee76eb4c2b452, 0x0a79a3a469e4eaa9, 0x91d5b16859d5945e, 0x46bde14fea91c2cc, - 0x1c21b0d8a937485e, 0xecfc62c90a9b8d72, 0xdd71a76ac34568e8, 0x744113505aefb078, - 0x8e6ede0695cba7b1, 0x674d23032850e515, 0x141e60f4a0c34b23, 0x1c0bf8bc3a201bda, - 0xb19fb814e9ab8fd3, 0x57437bbeb6c78720, 0xaf050278a1bf823e, 0xaa9a76aa461c0e50, - 0x484002b7cb46a46f, 0xb1268adabea1d8fb, 0x268e3df300a337fc, 0x0d99a7debb2f5128, - 0xf2c9ca9eba2acc4e, 0x98d0bc5f8beeb084, 0xde59ada46aec2ad2, 0x0b9504aab45d0c01, - 0x630bb43177e7a1a6, 0xf96e736e78d863f3, 0xe5230a2bd828a5c4, 0x407299f22d008275, - 0x9d42e7b40e2f79cb, 0x1f036aa3dbd21e14, 0x1fa8808dbfd1dea0, 0xee6f091f5904636d, - 0x91fc99bf63ae1083, 0x7dc884d88c542023, 0x144b12d52dbbe853, 0xda5d9b8e2a24f9eb, - 0x227d286a8798d192, 0x74157aef50711b0e, 0x503b9aa5f947720c, 0x893536b2d5d97949, - 0xf50ad69fef98d74a, 0xa4eb7b76d5315e4a, 0xbe95c7d12ae7e2cc, 0x803543580d9a83b2, - 0xa37514831cba3ed2, 0x6c2d3d6fc6250a2e, 0x2f60b856a9dd9a49, 0x76c814bc1c60a2d3, - 0x2c7ec34fe5a0ed41, 0xa48b89c7e7899f67, 0xb094004b5e9d63a3, 0x52797ae1bd3c94b3, - 0xaf82d4f5608fabd6, 0xa6a2fb39d5c19fa8, 0xf96b3a8bf3625a27, 0x222e9a6aedfba7b3, - 0x49ef45f8dc23045d, 0xdcf70e4663000e41, 0xefc841d124b61dfb, 0x13c09e0bdb90184d, - 0xb8d738e075586921, 0x1e050c769a1ae290, 0x89244fdf1b9551fe, 0x802e84d907f1ada5, - 0xb4fd99152b21929a, 0x5c62f5d7a9b8bd72, 0x3ac3429c56929428, 0x0c9f0c4a4894b478, - 0xfed6c2d8910800bf, 0x8bdc84f7a0efc8a7, 0x42085048639d33e9, 0x9c2081ee94962f32, - 0xe265d303aaece6ff, 0xcbf81d5e472e5d9e, 0x9774628a2c9e1384, 0xd7f1c6020b0b8e61, - 0x8070b09f2236f896, 0x530e6858efe2826a, 0x5dabbe0d21622264, 0xe0ebb13efa5d6f1b, - 0xb89ea0457c6314ee, 0x004930c7e3715faa, 0x88b448d351f918fd, 0x5f482cc1d901b1ab, - 0x572760107bbddfd7, 0x069b90304afacac1, 0x9a99682b54885287, 0x1facd5be2f1294a4, - 0x4f967a5015d1cf1f, 0x76163e0546d7dbe7, 0xfa275d894d4f8193, 0x854b07c578c150cf, - 0x1dc7bfbe799dee0f, 0xf6a3c45e52e8aee5, 0x976e5563275d2520, 0xc13529e66676ae8f, - 0xd265cbd73652709c, 0x4ecdfcdf68dbc49a, 0x19c4e31faf26fa73, 0xca1cd6530166132f, - 0x8cd3dd1d84254cbb, 0x5191689df316289c, 0x59593d62456c45b5, 0xb74f9d844cc6a179, - 0x0f32bf9f98e3092d, 0x1b2bed3ec1bbdfa7, 0xffe0b5caa38d9fb0, 0x9e26f336ce31155e, - 0xf12c954aba567b3e, 0x2165dc08bedda2ec, 0x54eebd0a17a9134f, 0xabae066183916b25, - 0x2710a17e4dbf1e39, 0x31328d6b37ff69a4, 0x68e819fb4d792fa8, 0x0c483e19ab77610f, - 0x9e388c818e3c59aa, 0x0aae6910a3ac698a, 0x310d0ba78f2f071b, 0xdbe5ded5465dd648, - 0x2e4867a24f6e9699, 0x99052eb3aa1ad32e, 0xc0ba8f29086f00b6, 0x46d7d5733f1e8e14, - 0xae852045df608b88, 0xe4f3879f1126a717, 0x06709ee95d812d59, 0x78bc6e0b53513cdd, - 0xec95ded66b4301a8, 0x056ceba1858e168c, 0x0b5404f133e43790, 0xf7246cb79c9d6ea2, - 0x7516a42d63c67451, 0x937a74beeaf5bccb, 0xc6b778e2e4d25993, 0x1a4e27bf1f1313e7, - 0xcd4635a0e871f17b, 0x1ff342b9ccaa4185, 0xf4788faebc4312f2, 0x39356ff42b2a75e5, - 0x0cc3564e52f81b2d, 0x7e8936d18b5540bb, 0xd92432676f54e2ea, 0x8d4e73c126a1db09, - 0x951cee42b521deb2, 0xa8cd89e58c2cdd67, 0xdfcd3c678a43028c, 0xb488d09f6cf4cabd, - 0x187084d1feed6fc5, 0x8b03cf35cf64d01e, 0xcacbefc199a62030, 0xf76f4c783b218eda, - 0x9ebe34d4d778e02d, 0x0ee390b46532b26a, 0x6d4072fa129207b5, 0x1e2b19b67ed19e47, - 0xa29d9eb10ebc7e65, 0x327f6d57cdc159df, 0x8a400df13b4a2298, 0x09b91c47c852c384, - 0x13733539f94c8fb1, 0xd2e36a7fa55e4d00, 0x8bda57cf1729b023, 0x7865c25ea55f1eda, - 0x0712c64eb879d8db, 0xf1f280ce4b527f3d, 0x3343922cd4bc18c6, 0xdb5de50531ee2b70, - 0xea3b957df1bb79c7, 0x348ed5e10ddfcfd8, 0x68fc345d5b6b6f64, 0xd910640aa29331d3, - 0x08854e8ff6f6978f, 0x6d0df657a038a57a, 0x6463a1a30f62899b, 0xbeaa22899e2337c5, - 0xd8e219de34c7bfca, 0x7affcbe16bba9663, 0x274f95cfed85aaa8, 0xb77d2bf0c95125b6, - 0xc07c427fc0acf7a0, 0x3c60bc3399e408e1, 0x8a2eeb621420bccd, 0xec663ad1deba216c, - 0x0ab0dbce71c30f67, 0x6e16b6d532e3a98a, 0x46aa447319102085, 0x27e752c7cac6eef4, - 0xb1f9ed6e05ac9c3d, 0x8bf8b1d23067ae8e, 0xba898030c7d9d7fb, 0x523696173519d106, - 0x86c177f7521e5482, 0x2c1aacd42e24d4ab, 0x5149f959873e7db1, 0xc85a8b10fb5ce953, - 0x0a90bec3758fc3e5, 0x01f1e06f01e0c685, 0xedf59a08f381c59d, 0x92fc8c8ec774cf40, - 0xb6eb9905f9a8c5f2, 0x1f51adfadae73724, 0x1b72b73cf2d83156, 0x9e59df020c098c22, - 0x19ee954df8cf3ade, 0x808e65170e18a8ac, 0xacdb8c9e9aa91ea1, 0x984f66e920bf4e18, - 0x60e0790a9b3da320, 0xa67155461566fedd, 0x5caf6098eec644c5, 0x1914e48b07b150b3, - 0xade8b48fed8d64f5, 0x5d74e9475606486f, 0xca12e488d1eeb59d, 0x6b629604a35b37dd, - 0x223a543e49ae86a1, 0x1ec4f982454bb064, 0xb2e224a2d714aabd, 0x9df5df3f86ef6afe, - 0x2358ad066f298f87, 0x900b627e298f03ad, 0xae423ff79c289af7, 0x1e60a35f906d2729, - 0xd8a87be56265353c, 0x283d220bc656bef1, 0x4e1568642a67e90b, 0x4a372da4090c203e, - 0x930f17fa6d9d9295, 0x008e7cad7ae44963, 0x4def85ed7d6a2684, 0x8ac38810bbb07c1d, - 0xc1eb38ff94e33719, 0xa20452176b0e10fd, 0x8653c016932139a8, 0x65801e9076b0d5a1, - 0xab480895a5783ceb, 0xe4497b4f9cd239b7, 0x9c633934099b5587, 0x9c2ea07e2443ac5c, - 0x1fccd8d242221b62, 0x58637826b3405107, 0x2432b21719c8d05f, 0x3b38cf7af7879264, - 0xa85e7e22226cd4ba, 0x80c15b74cfbcd5fe, 0x4a3ee4bc6fd5a1a4, 0xe4a25f80f79b0ffc, - 0x3fb4e9303e73ac7f, 0x898dfff3e53ad339, 0xe86e0f1bba15eda0, 0x60533e19ba424934, - 0x75a7705ba2031cbf, 0x10f6b25dad4cff20, 0xf3d368ed7224ef8e, 0xfa45adf2440d8ee9, - 0xc81f29d2dfd91b75, 0xb3d103a0f4343748, 0x2531a2141d6b559c, 0x5036701548a751a1, - 0x08954e789cacb630, 0x7b6b11a33cebc0c4, 0xec19b0d5733c051d, 0xb8e99762e92db1a2, - 0xbcf3317f85af2fcf, 0xdf8e19da0cb927eb, 0xa25fe9d344618317, 0x596b4ff3a88ac428, - 0xc07c1bb018a073f0, 0xa93df433beb88c39, 0x3b47c18275f93404, 0x7567bf3e78958bc7, - 0xe904d5f90df577bd, 0x78b3125f1ddb8139, 0x69cc3d0ae81d428a, 0xb66145ae356801a2, - 0xda3082217fe10444, 0x7a0bda5f9e98b7df, 0x74902f50cbb04c2c, 0xf4abf3924010964c, - 0x23638997a9931a3d, 0xaeb063448efe7508, 0xbca88a5517f38213, 0xc81b64dae5874482, - 0x4bdecb1808bd0557, 0x78a0ab6e90584ef5, 0x0f38dd21b6d8087f, 0x800049c26c151625, - 0x495414f1764aac21, 0xcdde8ee0c950786e, 0xf105d0591baa20b5, 0xcbbc516880056d89, - 0x11fcd7f6c4502d8f, 0xe5116eadd640b15a, 0xdf7f734105551453, 0x2747de7d01793454, - 0x59aff3ba6581d1d6, 0x467f129784fc33db, 0x84c70d02d105d5d4, 0x8de340b6e077cf4f, - 0xb9cc74a6854e5659, 0x539dd27bbec75ef8, 0xc44a042b67f5b684, 0x0e45e5bf8ee70f28, - 0x03ce3dd8243a27cf, 0x77c5953b67c0394a, 0xc49e22eb3055d0b2, 0x5e30d1816f6325a6, - 0x88cdfd302a91213d, 0x1bf4594fed605006, 0x4cbf704f19713184, 0xc16eafa4760eb7ce, - 0xd0b3cdb7ee8d6bbb, 0xae557152030015df, 0x56fbdd1e8a7c13c7, 0x8650dc7437cfafea, - 0xe7360958b8190b61, 0x8cdeef985b588f54, 0xebf6b16fdc991a71, 0x8bfaf803dbc82953, - 0x62eb1fc78d40d6c4, 0x34812e41cf0889ea, 0xbbe7890e434b313e, 0x739ffe0d6d4c0227, - 0xd9c229c40dd06ced, 0xadf41a3b9ba14964, 0xa668ea699254ff2a, 0xe9d4a68dafa3ec32, - 0x239099b86b4b5d17, 0xfee2fdb2ec108e49, 0x58e135fe920bea50, 0x08ed44268d416142, - 0x844e2ed4afd40801, 0x08572c2d4c6e8515, 0xc38ce392d0af2414, 0xa274a71febd66711, - 0xc9cdecda07b309d4, 0xddd7bf8d522ff68d, 0xfe2cf92627fc261b, 0x70c9254f57f2c2b8, - 0x4a9fcef9888b9fe6, 0x275c57ab6d6cf167, 0x94fbe062e3ce1366, 0xbb4942007c21eaef, - 0x48ba00be03751ae0, 0xca692e7617f9f867, 0x7730cbcd74c5080d, 0x8d6fecde134f6ac9, - 0x6c807e361adff979, 0xbad7fdb686d0438a, 0x1d559db4a942d85b, 0x5d26e0fa12087d15, - 0xbc07f142baf0c3a8, 0x67400a87453c020f, 0x65cac6ef2cc17db5, 0x80d5d9c8d995d83a, - 0xdbe188cac6b7e851, 0xd09daa1accba128f, 0x995a882a4b043e9e, 0x1652f413c43b61d7, - 0xb6a7c65ea2e614b5, 0xb29476ec31360102, 0xe096ae81dda16d1c, 0x8bcdb96439a5ff52, - 0xfd950222976efb67, 0x241cec3aa3c23931, 0x3c24b775287aec32, 0x0341a7df8309b969, - 0x6c66ed622d1b055e, 0x5f44f761bfce19fc, 0x308ec44109203cb4, 0x59d2ee6c4b88fe73, - 0x65821597cfb45938, 0xb5dca474aae9bc46, 0x905468324a31580f, 0xce19dc288b5d9d97, - 0x27b6cce1420c8b28, 0x5d93367c1f5a5ea3, 0x5bb50d6939c373d7, 0x63eed7cbad5cab44, - 0x327f066591780682, 0x85b61bb1fdf7a633, 0xb9b4d037bae429d2, 0x4805fdbb1a39a24c, - 0xcf59d5e292befde5, 0xe4ae180d0223b3fa, 0xd4a25f4da61cb7e7, 0xb8afaf674cb4b8ba, - 0x5cd11d67be6763f2, 0x0b492d820d840c45, 0x56ba3608bdf83d8b, 0x9f339365fbefd11f, - 0x50a34a8728042fdc, 0x81a4b4d888b3c713, 0x4be8da1fa991b8c8, 0x282257f7f30ba02d, - 0xe87095d558724e62, 0x893db002f003c1a1, 0x3dbd1891517622b6, 0x671bd0e3a63dd5f9, - 0x544acd92d791c56d, 0xd3c07db5378a4171, 0x39cf33e23fb2de61, 0x868824f8718a7769, - 0xe53406291cfe9611, 0x9d48889653ade343, 0x86f89e6654ac444b, 0xff2453ee39ff4bb3, - 0xfaa2cdd362faa89b, 0x52f8212b0730b506, 0xc4da1dcd13540c73, 0x51d8c5d24436493b, - 0x63fa2aa1d7bd87fd, 0xff00aa0e69724743, 0xd3387bd34baed072, 0x5b94668c23c570e7, - 0xeaff4d1dbbb05580, 0xf52056bc66a6c80f, 0xfb0d2da7c8e40470, 0xc913639dd1942f12, - 0xc4a0a52e8e9e71d0, 0x95d0e4c8ab9492f2, 0x167f9184dedd6b7b, 0x8eee4cdb3e5d5b19, - 0x7220cc62e71203bd, 0x17d80b8c21508b10, 0x27f8453c2e3a8538, 0x56ec343a59233409, - 0x54f9e9c686f9fdee, 0xe05fd4b5363da293, 0x183724cf40ccbb0f, 0xad5093e265e449a9, - 0xd6e6cea5c7385cae, 0xdcd39b916a12fca4, 0x2d5667f7b622b7ff, 0x64f604cc21413c38, - 0x35df90a54d628045, 0x20da70e4ffe6882b, 0xb8a62e793ddd8e55, 0x01828b9da66c3f34, - 0xf51126c77c0d21b7, 0x1de0aaa3a0c76071, 0x3565dc664b65fa5e, 0x7eedce211bb44079, - 0xd7b955e20a021331, 0xc3e716e42da8409b, 0x164e615f92d0ed2e, 0x78fc527a7554b384, - 0x663efac6fae91776, 0xec6ad065c4399330, 0xf86994010dbc35e3, 0x62dbc3beec6caff9, - 0xac2df2d420b38533, 0x4c027bd7988b3ff3, 0x2bb2ca5d04585127, 0xdf0572d20c82a968, - 0xf362b1335e82d222, 0xea0d08feaf494906, 0x9eb1cb9f11656b63, 0x85a4640eafd9d016, - 0x93c952ea62a1e2d2, 0x1a24355bb32b19f2, 0x5641f07e5be61c86, 0xc79cb618b6f806ca, - 0x1b4c0aabd9dce0e6, 0x3dddefed24e17f00, 0x0503c6b53be8a126, 0x120b2a2d8c1e66f8, - 0xa270580c86a13e81, 0x5413d15b09d76d7d, 0xbaf4727db5ff2980, 0x44fdd143d20da814, - 0xca73411d8c6fe7b9, 0xb5b01d9e3f74f739, 0x272612acc2ba9b32, 0x51cc5bffae475930, - 0x98211d6c632031b1, 0xd42fadc1ac8c2eb7, 0xbf884a8934c3841b, 0x09c08aaa348c0f91, - 0x1ccfc4714715bb91, 0x57532d159fc84bd3, 0xf3d5eae5ee3199d8, 0x883ca018707a12e1, - 0x1a5f4a23903b7fb3, 0x2d99749c321d832c, 0x79428c48c991c49f, 0x0b288eba0ae89261, - 0x624a40ffbaf45910, 0xbeeca9ba3f2a1b64, 0x205437aa57f81059, 0x8118067e14e2e4d6, - 0xde5b9a4585d06e18, 0x39e0610122d24e94, 0x70c5c13adf4a649a, 0xb99c623fd2b85032, - 0x4899fc05d5d4f8fa, 0x538a90c1a03c3493, 0x69b87fb2b24fba99, 0xf498b5305d37f996, - 0x00ded8f787ee2e5f, 0x72a0150e2ace176f, 0x526e87ea868b98fd, 0x2965a46a31fe83d4, - 0x140babc924206a50, 0x9b077d8c18df4632, 0xe9d279ad9bef721e, 0x526029d9b17e515b, - 0x4466d50c771e178a, 0x6b7eeff2c93d9bd5, 0xb2f22434b4e95b8a, 0x51146e21eb622f69, - 0x60b1d9daf80647d9, 0x3363073029c8ee6f, 0x28153f1581297849, 0x00d71107db9a4a27, - 0xb04564ed9901895a, 0x66cb133bf6abe80e, 0x414c13c6d405c706, 0xbcf45abb4908c354, - 0x752d4b8bf80fb3ce, 0x50a42ffecf806b54, 0xf60c89ff4db3faf8, 0x204ff208ef8ba12f, - 0xeacba888ea889c05, 0x7eec3fdddcacba9b, 0xbabdd5b5fc47ad34, 0x2360b80ddb33e0d2, - 0x5859db730dcd1f1b, 0x731af6f31a8bf130, 0x2d7b4e37da6da9e0, 0x3177c5dbc28b6e1e, - 0x9f6ea1155cc1ac81, 0x20551ed993f28729, 0x0ec39004f9df0a32, 0xc236181679db9ed5, - 0x9ef683ec7dd15253, 0xc9959a3e8fc9cf79, 0xdef8192f6607410d, 0x72e53088b42f5e79, - 0x46fbb5bac6271518, 0x5a33f965044fbb10, 0x5f9cf92d0e48770d, 0x47a6371777a1b216, - 0xe50f0a789c725f3e, 0x4dfe6e2dc179ecbe, 0x25bb61271db9f00d, 0x6723ef7d754cb90c, - 0xdc59aab0a504122d, 0xace4beb091fd2a5b, 0x415cbb93d5b53232, 0x4781d1eb2751ee48, - 0x457a2c2d653c67a1, 0x2686f01568cf6417, 0x7ce35b3aacdda374, 0xddcaeabc80a10a12, - 0x33dfb73c60bd11bb, 0xc4d82c05a1b89855, 0x240f6e5f6045fb14, 0x88c846eccfdb390b, - 0xc0ada53d0e0d6737, 0xcf0e2d848b6414bf, 0xfbf8b6d82e08b14d, 0x92b7cacfd220e1ac, - 0x2b6c05256a8a27e9, 0x2a3db35729add9b0, 0x596e247da113cc22, 0x8494be40f94de427, - 0x178d892376d680c2, 0x9f2584e0e40eeaad, 0x82ed61e42f55d963, 0xa9743a1da41c3838, - 0x6a8d46d78ba82cda, 0x3f71a9f1d67dd999, 0xef28d9de93dbe99c, 0x6d2f4b03e4d753e1, - 0x7dd46cb40e8e8558, 0xd3369177a765e599, 0x22a6c36ff89cd99f, 0x93ea2c16c766efca, - 0xf448b76aa5bfb04a, 0x9cd4a20d061fc730, 0xd6950dccf487a6a4, 0x3b58ff7e1126ceb5, - 0xf7bb0069790e8c24, 0x153eb0ed546daade, 0xc443955940a73eec, 0xf1b9845010d116e6, - 0x7b5f119f4d43026f, 0xa2caaac2fc3009aa, 0x0400bf2231d980f6, 0x889847cf1317f51b, - 0xe780d292154b61f9, 0x09cb12287223e34f, 0x9659f13221c95d23, 0x76b190e0510767d3, - 0x445230143f70843b, 0x55e06d8cfd13f0af, 0x9f7ec0fff5c0b64c, 0xed3fa8beeec7c975, - 0xdb469201a0ac176a, 0xd52f9ce508670427, 0x37808801fff18765, 0x97876b08fb04577f, - 0x4626dff92fe31a87, 0x7e9306c182ae991e, 0x9b4f740552f3ed5d, 0xf260ac16ad5a618e, - 0xb79f39761f2665b1, 0xd06e2b04271b9cf4, 0x3bbfecda3f9ecb03, 0x5baa74707f559a3d, - 0xdfe12e537e5246e2, 0xaa3e307dd21069c0, 0x99c10121bb01ea71, 0x4e1a36ebadd6b0be, - 0xd5cebc177e0978ff, 0x463d5f9e8e729ce4, 0x877824c0cf42266f, 0x021b150ace5e648c, - 0x2946c3678309432d, 0x6d67a3af313d3bd4, 0x586b35e5618421f4, 0xcf24559d339621cc, - 0x902c0ef7e9667b56, 0xd9d3d01377c3422d, 0x30109b717cb04798, 0x495fb55b90e3420d, - 0x37a8a414c01e959a, 0x5c106c49a79c56af, 0x020a7bb04c9d8cbc, 0x04c88903448c4e3b, - 0xd84532e065f6d162, 0xab95d0fef6ec33c1, 0xf5e196fde0d44e80, 0x733020e7424057cd, - 0x780cda11a95588d0, 0xfb5e6fa83fc9696f, 0x1ca62b3478ab680f, 0xa661bc3f6855337c, - 0xa1b870f1f4991c8e, 0xf6598e91f2b24a18, 0x90a0b03f1d5d0ab8, 0xe11f7c52d49b2994, - 0x909c9ee31bee3d84, 0x190022760e06eadd, 0xd19d1b33703cc985, 0x3e03cc571f0f56b2, - 0xc92173187bc107b1, 0xa28844263458eed1, 0xe675aaf903a494d7, 0x4827d6d36a588f5a, - 0x333259248eee45bc, 0xabe9076fd02e3e7e, 0x1f65553f546a405a, 0x23142d6193beb6f4, - 0x34e89bc34e6c5b90, 0x7d71a154edffe99d, 0xbd57d2a4be542b79, 0xe92d9832edc5dbde, - 0x40eaa280dbfe7e8a, 0x5e9702e10b56bf73, 0xb207ae2ee4c55b2f, 0xa58deed5f938254f, - 0xba5aa9f7e10880ee, 0xc592695e86d0831a, 0xc8bd11009279c702, 0xf6a27d3c98a53d69, - 0xbf47f5e427c6e2a5, 0xb8dd49f706529353, 0x9d928374d5e6dcfb, 0x1f8c22b3f4983d29, - 0xe40d64fc483557da, 0x47122186bfc32686, 0x18cbc900da69dda6, 0xf154fa05c947dd3a, - 0x13d790a028c760c0, 0x3c0e94621f5240cf, 0xb01662b26ba5dc98, 0x498ffa76bc80656c, - 0xa94c013464e96ff8, 0x95e058e99f12d947, 0x6cbb1d2722178d3f, 0x67449b0eb00827ad, - 0x9f0f1eadd530a958, 0xca5c8f0e8cb8cea2, 0xee7992e28425454e, 0x41f843a5434a8d81, - 0x1e70d629e89ef62e, 0xe73d7ac161925697, 0x2afcafc126bafa40, 0x839c8541e17787ca, - 0x059b4a1347775dbc, 0xd0e8e8aa2a02b126, 0xba19423ec103d8f8, 0xa269cb5c4b392621, - 0x49e1d81c1715a816, 0xbe8128262c06b176, 0x637c94e2611aa824, 0xe79182e180fbfd1a, - 0xe52499e8db72176f, 0xb14c5c8abd81059e, 0xee85302384c91e38, 0x2415e1e03acb9b3a, - 0xce9b0902f0b16591, 0xea871ee4ff52d791, 0x4dfe7d9f354e97c7, 0xa423aebf855a966c, - 0xfca679f5ab43e7b8, 0x2376f2c256580689, 0x75c007c9cfcd3e6b, 0x007018deb51dd51b, - 0x05741b4392ac9f46, 0x4219f55cad9fa1e0, 0xd34fd83f3a05b018, 0x3258fcb6e2caf401, - 0xd3706a047f3a8bc5, 0x7282e95f1ece0b26, 0x2629595b3329cfff, 0x109bc1edf5a9ec88, - 0x10b6f17cc0fb80a9, 0x021a31935b546561, 0x4a0ce2ffda185806, 0x646a4a57fb11c849, - 0x336eff842bb4d613, 0x91076d5a5e281e69, 0x95ac47d63002f6b2, 0xa689c31c90ad9f20, - 0x47a6f9c6c3039c06, 0xc8e24f56589f6129, 0x1cfb5834ed4854f9, 0x66bd581ff4a5139e, - 0x0a307a46aad4e7a7, 0xc419c108f8fc0006, 0x789d491b2883ce29, 0x9470ed56758d6909, - 0x2a013295b0530545, 0xf84ee893b374ce5e, 0xd33b40c8cf50e563, 0x5d604a36907bf253, - 0xcc86f8a55fd58aca, 0x866e0a4db7bbf60b, 0x06dadde4854de211, 0x4c919fdd2db95152, - 0x203364852994802b, 0xe40f2646072519e8, 0xcfa3733d49a67573, 0x4d43c581968d40ff, - 0x020261c0e38ef90b, 0x07ee52c01ccc54f0, 0x94eadc04211a92d0, 0x085374ae06151475, - 0xa1944c72d1fed7ac, 0x2e61e180ba300666, 0xf66a49c2e672a1c1, 0x1dde22a920be5309, - 0xf306bfdf1649666d, 0xea1962f984ad97e6, 0xebb447fa2e8367f3, 0x2babfed61d5c4c84, - 0x1d85a5d9d87386d8, 0xc25d1bb63f19af12, 0x4d0d3bad766d283c, 0x0cc0b000e0364e46, - 0x3250646806e2a32c, 0xa42b02227b68de06, 0x7d018af3aa367513, 0x4dbe3308e9f529a0, - 0x7f0181b0d10c1a37, 0x3e68e528ca56a94b, 0x69abd2aabb33e533, 0xb4a78f483f9b00f8, - 0xe04772e10656ebf2, 0x08eff26610d421de, 0x7dfca8e4078a34b0, 0x73ad97d7909ac3e0, - 0x87b73920626bf7eb, 0x4c9b95c4cae699c4, 0xdc6799ec60217dbb, 0x818c6d1b6fc5ff4b, - 0x1f74545f51c0328a, 0xd00e5e624e442b79, 0xc3fa2c972c5c1bc3, 0x03a1c2715335c907, - 0x9c490192b66167cb, 0x3438986b5b13c999, 0xfdb9e92495267c5d, 0x01ba8574f26e6e60, - 0xc31bcee671043164, 0xf771215b0ad446ad, 0x1fc1078db0cb7fe1, 0x5e1e5431ef02de38, - 0x7b8cf33274bb42d8, 0x528e850451492ff0, 0xc128a5221dc49829, 0xf6cd5baeb93cc8ac, - 0xdac5bcaa0597a5d9, 0x3c569072c708bca1, 0x4c13c4e69d08e0fb, 0xb123cbb52dbb5188, - 0x7631e917bad7ab7e, 0x6256e54f060aa039, 0xba81baf06ca42a4e, 0x72a0210cd661dd19, - 0x46039cfc6dc4f3a6, 0x7c426366ec1757e8, 0x57b41e2fdf60921c, 0x38d48028b54aa707, - 0xcd9f0dfc8b25c5ec, 0x9bc25c6e29e24a51, 0x3698f35b9332be85, 0xb490da53e89feafb, - 0x2b27a7fd9418aa2d, 0x8c62bd6f0d796721, 0x967ca3257ccfac36, 0x32f8fe66de3e55bf, - 0x1a26dcc5d403afe9, 0xfd52ab81ed6f7465, 0xc75a25a36e48113e, 0x027bb33cec1728b4, - 0x4d7df30f98e42623, 0xa792a4c2eb5e3ec6, 0x89a90212ed111e77, 0x86bfb86e3fa4473a, - 0x30f7ca7a695c430a, 0x8f1a47cf19a6ace0, 0x3d54172fcf52386c, 0x05b18281ee8d51c7, - 0x6b58e01432a2052b, 0xe83fbc359c6829f1, 0x82908594cd40fa05, 0x592d56989165a9e6, - 0x2065db5008ec360e, 0xc1259a1acabcab83, 0x3da4135656c9fd3f, 0x99d67d5003717b6f, - 0x0d246c3b2bf73968, 0x19b418cb701a7428, 0x71c07f7fd3f02724, 0xfde141bcc4117143, - 0xc7ff52916048096e, 0xe6dd6f3e83b03d45, 0xf615d14fe9d236e7, 0xda97ec7be57ebe9a, - 0x558007425a37a097, 0xd7007a0dc16a9c21, 0x88f1eecf2fe8a303, 0x2a41e3159523eda3, - 0xf6bbbfcbe32da0a7, 0xad9f8998d9b0b659, 0xba0dcde66a3fa571, 0x084d1fc4236ba76c, - 0x1e7b0ab173823e0b, 0x27ff05523d74ea43, 0xc2a9da8be6390ef4, 0x881f2154179089f3, - 0xa46b0387598e0173, 0xc6ed5daff74323ba, 0x6340bba4ffecfd2d, 0x802702495095c677, - 0x404c15f9992679e7, 0xbd090123145673ea, 0x0601a806ba4596c3, 0xd0fd3b76a940737d, - 0xf357cca9525daa33, 0x51fd8aee99dcfea1, 0xdf391f8ce0828c67, 0xd90937a8cbcb2e18, - 0x28911afc17aa443d, 0xfc27c92441385ec3, 0x11929641cf493c00, 0xa8807b842e2e7afe, - 0xf24ded425eb56586, 0x1e581f0729804ebc, 0xf841bdff4f815816, 0xccb34b71af841516, - 0x3bdff0d3d2e42637, 0xb77918f006c04d8a, 0xba134e57e3c95927, 0x01cd483888b78ee9, - 0xa8b3a2c5d82f7116, 0xd39ca587ccb58ac5, 0x9c3c662156b4af80, 0xdd98da47c0abb777, - 0xa6f9c510be7176e2, 0x96aa88205667fe12, 0x62bf037b395d6cac, 0x0788888bd5e45b66, - 0x520dff35e0c87278, 0x4bb162251835b4ef, 0x31b1e8cf070aa857, 0xb88dd111e01ea6ca, - 0x1da9d43da36514ac, 0x3070129125e823bf, 0x10db89b25fb49b14, 0x541de82488beefdd, - 0x2374b2f8c136b8dd, 0xdb967534c109c5ca, 0x6ab9ac670f5d95b4, 0x8f671c38cb69560e, - 0xe3b9356e27cae84b, 0x71a26536a94c2178, 0xb981d98bdad90b42, 0xa7737a2bf3b895e1, - 0x7956135fdfe3bb39, 0xbfc4aea536701f00, 0xbfa77621164c27e4, 0xd090e133dcc9f8df, - 0x9cf4b30fe5c77d96, 0x5569faa83b6d7076, 0x365e3af98a952adb, 0xb07d3e52b300ddca, - 0x9620b0d00743ad2a, 0xe74acf8427f122db, 0xb819d02a30fe4a8c, 0x818e8a24f3a1c3c6, - 0x9126518fb26aa458, 0x327c8681c781b247, 0xa5e480050330cc13, 0x3a3f3a00cbcf5ef9, - 0x2b37fb383719ec98, 0xd4ddbc3eb910c43c, 0x71971a79119017cc, 0xfdff1d970413dd04, - 0x339c394269d04642, 0x7f7ab88f0330589a, 0xecad5ca203228a41, 0x8fcae7fd1bea1fbf, - 0x1cf9892a71413d08, 0x89fe2466439f690e, 0xfb8f76bf7f667576, 0xecd74be2ea1f9f6e, - 0x4f1b01a5c3d1056c, 0xf7053907fcc7c4ae, 0x2317293ca18d6da3, 0xccfb652dbc010409, - 0xab0a9a0828c909a2, 0xae9205fefaa950df, 0xd44c1726c3be4ed9, 0x9347813eb6dd28b2, - 0xf0bf0d403449ec2a, 0x7ff2a7fb59e2e1f9, 0xd38fc88e57119cef, 0xc44d7aa91607e906, - 0xca5d7d5b1bd8c0db, 0xb3bb5e61058b6bf7, 0xe969053e20a4f165, 0xc7cef08dda54cbe0, - 0x01f74bf955282c05, 0x95a1135d3f9b6569, 0x4bf5bf0e85cf29cf, 0x7bfa44c5277da909, - 0x7c211995704dd9c0, 0xf671f43049a302f2, 0x0bd5e5c0f14887d0, 0x60c6c1afbf0f0a4e, - 0x6c96f9e252be9e47, 0xa30700f418162677, 0xc9ba6d3b5f661a71, 0x4f21f3066c20c7f9, - 0xd78a03ae9f56d6bd, 0xd41a0d42df6def57, 0x673612fc7ed41543, 0xa5de6c73ca1e839f, - 0x52171e1656560c80, 0x48cddaad2a6f50b8, 0xe6432d42663586b5, 0x7542d8747f11bace, - 0x13878728158d0f2b, 0x7c33ff11c336c8f5, 0x224c42d7ed586107, 0x2f69585137acb76e, - 0x2c3b780b7b49541f, 0xe8c8403f99f1088c, 0x9619786690bdb39b, 0x71e6f780d500942e, - 0xba293f02b7ec7c31, 0x9e775a56163d60ba, 0x889964209259bb1a, 0x5f09ca6197469bf4, - 0x3c0d6198a8d17284, 0xf4b04e61f5f3d83b, 0x660e196e4102dded, 0xad30c74fefd8daec, - 0xa287ca1b53bcd6b3, 0xeb31468467d14f15, 0x1803e0fcfa3759c4, 0x463ed7002995f12b, - 0xeb8d23602d77346e, 0x3c2c740d653afce4, 0x3f84b5b3a47f571e, 0x70a9c8a3e9261dbd, - 0x126271164e64ae28, 0x262886639733289f, 0xc4bb2cd30bd80910, 0xf69ce6565c59e0c1, - 0xd9e6f58aaa8adb62, 0x0ea602be6a2f32f3, 0x7d59cad10a01803e, 0xa157204ca0fd03a8, - 0xaa09f8e8ebe31823, 0xfe569b92ef3ff553, 0x34d437459ce3f123, 0x8229313fb5ff507a, - 0xdbea553ff188298c, 0x7b4d71f99ec8cdf9, 0xa5fc8d9541cee13c, 0x3260fccfe26138f6, - 0xe9cdda0088736ba8, 0xf68d3ed50fa2abab, 0x8120fc7835bc1ad8, 0x07c6e4b9e3efae6b, - 0x8090386af4ab9023, 0x059652bdf4fa9b20, 0x0057e127e3e39eab, 0xd146cffdb9d0e4ac, - 0x7e1b457ec70050f3, 0x34b4e4c8db1ed921, 0x1c37217679829216, 0x8bd2ea275de42cfe, - 0x4f2445358745bfbf, 0xb7fdb22b43d9ed26, 0xcb350fe29f9e7b27, 0xf1dcab7cf1709d57, - 0x47f7b2df9a71cb44, 0xbc973febbff968d3, 0xc67591cf7333eacf, 0x85ece1c4a0e14093, - 0xcc714ab3b6f47531, 0x1bb99cd8a348a85e, 0x247e8301aae6bc8a, 0x9beb36005325bee4, - 0x64d4de5098e14469, 0x351c70426ad04682, 0x210c7a9239815db2, 0x12c000ee7c1725db, - 0xd6189b38a58901f1, 0xa01f8ded5ce6cafe, 0xd64a48963d8b3d4b, 0xb1f751b97be0fe7c, - 0x3d1c7bd96cc79c55, 0x1b5e89e7e32f515d, 0xc54d13d1c351aaff, 0xc32751e95a283af3, - 0x83312af72f9ea759, 0x04b9352e6368117d, 0xb501ccfc2286f90e, 0x35537873b9c2c8ad, - 0x96a73ba3404597d4, 0xd7211d2c8ee19a7b, 0xa115bfa2bb85bdea, 0xb334683bd1a209cd, - 0x111cd3f00fd5acb2, 0x9648cc7a00879c9c, 0x61a3946cd61d2596, 0x3ccc4a4c90eefc85, - 0xc0ebf7602c9807c3, 0xf1db93f9eee10304, 0x2201acebcd536d03, 0x8ff4f3860e1f172c, - 0x1c79f304102349d3, 0x72aa7fb1077e80ee, 0xa0f565e9bc004ad0, 0x96ba74f0773fd5b9, - 0x7ce7fb42c193cb6e, 0xd3ed2362a55b47c6, 0xe2b6606622ae7aca, 0xef6c4d0817dbd7c1, - 0x4be8271e9e449610, 0xae8916f72ce0c551, 0x8beba29b9503edc5, 0xb4ba98a0a9dbc28d, - 0xa25548bdabed562f, 0xc41c2c36d371aed1, 0x4a69e935df646cd6, 0x72512589e78ee5f2, - 0x04a833ce058796ea, 0x04903de9da66c0f5, 0x75d6d83ae8dcd8a3, 0x774631271aff1402, - 0x5d63fb92710323ea, 0xd47245ee28cb1a7e, 0x8bda24eb944f24a0, 0x1217246b7d566412, - 0xe11d8f7c21a84e84, 0xeda8eb5facd06b93, 0xbefa5f580828fcc8, 0xb8e2242c1c58f8f8, - 0xa09d89f069219db3, 0xbf829e263f62c37a, 0x0b99207c6d836e97, 0x3fd8661fc50a3f30, - 0xf404e13fa33c8dda, 0xc2ea108dfec0854d, 0x55760e813c3573ea, 0xf17c94ebad517d29, - 0xaf07337dac11f006, 0xddb9e92d2e8fa483, 0x5fb9dabcacb0d849, 0x8a200380f82c0629, - 0x9fb826f075c11f74, 0x0590d52bea541ffc, 0xb9e3742c51dfd804, 0x9814c6cd967c82e4, - 0x14eaa0cf6ecb1240, 0xa6e2d477fa834ebe, 0x6a3e5aa64f39eca3, 0x0b2a5e0080311d4d, - 0xc2ef7569f975bd24, 0xdae2a608eb817861, 0xb78cf43bad00baef, 0x75f0f5b5e494b67f, - 0x9ac7fa57821a872b, 0xb10f561c071a8b21, 0x43a0ebf2c32f50a3, 0x8d78fafac056ac8e, - 0xb1174a2630d06a98, 0xb26143fa1c8f3ba6, 0x4da0090e9f86d136, 0x55ee567016823d48, - 0xee69fab4f1d2b8f5, 0xa4dbf19aecc373a4, 0x68b3b42e51e967c1, 0x5480d1af12462163, - 0x2ca2ce808aa84a03, 0x0aebbcec43419402, 0x48a850139ab0d603, 0xd2951ecd03b50489, - 0x0672735ac41d421b, 0xabfa2407540437e9, 0x5370686da6e4040d, 0xc3d0b6c8aae9eca6, - 0xf99578fbf67c20a7, 0x1dd2bece9621c649, 0x40a02641d579ac42, 0xae3bd888fc37c8f2, - 0x040493804f45e6a4, 0x24aa374a9e59d9e9, 0x36047abbb144472d, 0x0e255712cceec542, - 0x6ff01230483fef3f, 0x2cff5040a6aa2817, 0xc5368ee7f062baac, 0x68238b4bd643292f, - 0x161fd0339b626b52, 0x36654cdc77fcace3, 0xde55500445eb7d4a, 0xdc34aa47d4b83b2e, - 0xbf58a9e3025dced1, 0x6e324e501436bae3, 0x83a646297eee6d9a, 0xd9ff7f06572c8984, - 0x6032fa2d0044880b, 0x33302f5166bae5cf, 0x707b6340d8d107a4, 0x9e343e319bb58fa8, - 0x4fe8af87987b2fef, 0x56c20cd316e21a80, 0x889819f09a4cbdca, 0x6ab218c485c81c36, - 0x7b0d9905dd239fe9, 0xfeb245572b6a83d9, 0x73293b451411a655, 0x8b9d45ce35e7a7c8, - 0xcf73294aec97abc8, 0x9f2505504d9f41f5, 0x278063c15cffb38d, 0x53844a31ba22cb68, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x834dbff6678337ee, 0xc607e811fef0785a, 0xaaefc62be30a298b, 0xeb5ca335326afad3, - 0x9774fe1384af54a8, 0xca4b6ef5785388b4, 0x1346c82d66f6c642, 0xedcc0c2aaa2d53ce, - 0xb9dd9b739a6a8ba2, 0xbc51e19177104dc9, 0x1a83634213237211, 0x5ff69f51cdcabeea, - 0x6dfbe5b4c0172223, 0x187a6a8b284fc824, 0x33d6aba3aebf41c8, 0xe026b4eb8795d856, - 0x5d020728bd7d86fc, 0xd15ba07e713c1ac8, 0x7b8a85468316aa0b, 0xde53bf4a0921c5d3, - 0x3bacf926314274b1, 0xaa6473f0e489a20e, 0x1a35b2266183f440, 0x27e38367373832ab, - 0xf4b475a052f7f6ac, 0x1319fb286978fff3, 0xefaffd23eb76cb67, 0x4c3514b07e7fc4b2, - 0x6b88c08b50794140, 0xa666ada647622f9a, 0x8dc0fc40a2e7fb96, 0xb557a9504af4ad12, - 0x423066e6cb7c5af7, 0x6d4aa104ef0ddde1, 0xb50e24f20c0fa712, 0x14bd6aaeeec4e1f2, - 0x4c835329efff35e1, 0xbcf4baea96e851fa, 0x9a0f11623885b71d, 0x166707a74e19411f, - 0x932ba9289cd28601, 0x50fb36f017896b35, 0x68237e2f3dfb90e6, 0xba2ea6bc3cd9a39c, - 0xd1ad36cd23fe8f01, 0xac8b2ab8cf0cc4ce, 0xb043ffc82b2b5b63, 0xbd9162720ec3755c, - 0x9b708568bb21cb79, 0x31493b87821d5c5b, 0x22f8418ddc2dd569, 0x992028fedd061736, - 0xd942c46b5191d9ff, 0x2bf067df5511345d, 0x277e70715aa2e38c, 0xc4d3dd2c649895cf, - 0x060a7f6eac0d6b6d, 0xfe7d6270c845cf26, 0x964658cf2ad87cba, 0x467f1493cf5f3ee0, - 0xcd45853b3b56e262, 0x83ec7aa03fb6e673, 0x0efe037b89445388, 0xa1a175362967cf21, - 0xed0b054915382916, 0xed71547e28b9726b, 0xdd8d0423d588f46f, 0xb49596f9186bb489, - 0x7b0bb2912e6e0ad0, 0xb6e3a6b5aaebca7c, 0x5d47bc7c5ec4ccc5, 0x615b74d006bcee76, - 0x37e279b70bdd3234, 0x912cee2fd0d0a4c0, 0xd4360b55635f0474, 0x2e7c64035873a344, - 0x4f33e31a6d46b307, 0x9ad457a3f9f8bfbb, 0x4ae7ce9df7a244de, 0x3c3d8e0eeaca457e, - 0x31e2d107846c72cf, 0x5196826a7c06eb69, 0xcaa792d5214b5b92, 0xe58bd074bb238a83, - 0x3280c94e3325797e, 0x3bb4edce17974073, 0x2e0b3f5245cc5177, 0x23267f698e609923, - 0x6d654eda0fc9a12e, 0x340ac26f7b46d1a5, 0x45deed0e6a65c721, 0xd2d96ca1f382660a, - 0x33384f4b1f2d1ac2, 0xc58b41a266ded10c, 0x1a6eb060519479e3, 0x9c4595ad1a60e9f5, - 0x221b517530317642, 0xacf0afcd376d30c3, 0x08fb0bfc4b86f3e5, 0x747add2f86cd7c23, - 0x7cbbad6fe3e36cf6, 0xcbb66fd9a266e57c, 0x8dc9dc9d0d111b28, 0x031c17cb5d40a944, - 0xa8dfd662ce5bba5b, 0x1c6fc1a282e49bcf, 0x8d3d05c2fc8bdea3, 0x51e25c00b61b8154, - 0xf1625614357f93f9, 0xe1fcbb86270d89a4, 0xaa0204a7d66a66c4, 0x84cc776bfcc8d5a7, - 0xef2b546422529f8f, 0xcb6cf3fe2a406bb9, 0x6566ffbb8d4ee8a2, 0xfd6508248a7f8a5f, - 0xc06fe2d06b12a423, 0x197f3b8ad3dc4452, 0x9c61281388d05818, 0x261ff495f9228b2d, - 0x69d3e0a876abb7a8, 0x132c1433636d1c4e, 0x6539d5f259b4df32, 0x4ce0b6759b214c13, - 0xe330bc9b4ef66167, 0xfcbd3f24a2df334f, 0x048285d14074ce86, 0x1390ab9c7bd9e91b, - 0x0cb9235b41fa84e9, 0xca99e159ae43e247, 0x48c713975061cd6e, 0x71278a093b84c968, - 0x9ccadb3cfaaa12ff, 0x63cdd416fa1af8ab, 0xdbc255d0e5ab5a5d, 0xed3136643f942f1e, - 0x9de8cb41dae09c7c, 0xd2ae9b77f9ab32e2, 0x48d5b61d5802b496, 0x893caa4ec91104bd, - 0x6e9f0d9322bce488, 0xc39a00dc21338bc9, 0xae348034d9903b7a, 0xa5ad87f9646e68d2, - 0x4e7f38d77fe247d2, 0xcb16934588ac9393, 0x1752e6f5c995b0a8, 0xc2bcd27f864eef35, - 0xc549ed814286e7b2, 0x3aef8b12e5ee7ef8, 0xfe1e048b7bcf26af, 0xb95ade8441ab150d, - 0x8509f758fd95f8a0, 0xf10800ad91e63003, 0x4c1329aaa056fc71, 0x3523b1ce29d78dce, - 0x24ed81165923ccf5, 0x59e6b7cebd213931, 0x914ad94282bf8af7, 0x342a752978122c61, - 0x06712352c34318d7, 0x9585bbd9a27819b1, 0x290a347f8d7fa655, 0xaf0aebc1dc20798a, - 0x11d7e55a47627250, 0x5816f7d327b5cea0, 0x4719b3b7d732d136, 0xa0b4bfbda2519db6, - 0x5532d6daa3b75698, 0x8321cc4a81ef90be, 0xf38a78e14ffb463c, 0x1f609818617df89e, - 0x9abfd2de6770b121, 0x7c177385315f17cb, 0xe85df23c7b81d190, 0xf3adb4857b454fc2, - 0xa5444030faede6d3, 0xf0422412d517829e, 0xeb14c0d39ee9045a, 0x0c301ab47fdb44a2, - 0xaca2ee4e6786f072, 0x1d87032d2f334582, 0x345fe5516c32f166, 0x74ee80ae0ebd7960, - 0x4b73b3bc02a6e29f, 0x70b8274f9418ffb8, 0xf01022c3b4042db1, 0x19455881d2b42ac9, - 0xd5edbdb61d20cf9d, 0xa6415bd817d4a3ca, 0x3c62a62b1fa4d1ea, 0x651457f278665943, - 0xc9498001208b83d3, 0x6199028c8200f975, 0x711e5a0ed13ff0c5, 0x1f0285a9d183cc30, - 0xf29675cd0a36ecd7, 0xb7cb5bdbbf4d666f, 0xd7875f55618fd235, 0xecda9ef68c587561, - 0xfe25a100fb188178, 0xa82c75c8c949e3f7, 0x60725c0ea385f9be, 0x4403ee407e4bdb22, - 0x0a3ea67d44343dd9, 0xe556e011299e0ccc, 0xba5798f6e65b0435, 0x73824b5c2b50ad4c, - 0xa09b100821fa66cd, 0xf1eec9f3886384ab, 0xfa208374dd80459c, 0x8e24516c85106c37, - 0x572cd306a60cefd9, 0x096f95aaa9517214, 0xf17697ec1a00392e, 0x17139c3bf7fbb785, - 0x3e680d3d6bed4611, 0x8d757b5a6f01c1fa, 0x11e98a2ed9a755d6, 0x0f0f0b22cd26393a, - 0xac8db9c6f2967ede, 0xc8b6519b6cfaa5eb, 0x47d39eb78777ec37, 0xdec11d458058d973, - 0x6145626a9c4bb00b, 0xef80302ab5dd292d, 0xf8d0111620fe7faf, 0x9432a0d5efeb4ed0, - 0x2fe0a22db22026d3, 0x8ac10fd791ae09ea, 0xf18e48939d42342c, 0x72af9bd2482dd5c9, - 0xe1b2d3d63310861e, 0xc3c6d8d1198c676a, 0xd4ed618ebd765fbe, 0xec1227c11b8ac80b, - 0xb2b5ab35a85945ad, 0x844a44e35868c22c, 0xd89319cb421f4c8c, 0x5daf65e96a41e286, - 0x369e1b4a84bd8b30, 0xc2d183e647dae27d, 0xab9e0dc9d338469b, 0x7316b2e32cac88ee, - 0xd806bcb9524b211d, 0xcddd2192951440d8, 0x77686fafd0d7e10b, 0x791a76820f073947, - 0x7adf1f6eec0b49b7, 0xbfff9310ad564ce3, 0xc5d423f9cec8d505, 0x90987a8e587fffb1, - 0xe8544f0024ad27ef, 0xfb62130b397e7efd, 0x588431f2b1f447a9, 0xead0c17a8556da90, - 0x4745b8f2c7c02ad0, 0x26e60d2ddcdab226, 0x2e68c2ac9a0206ae, 0xffc6e1b678a3045c, - 0xe83fdc122eed9851, 0x27f6147ede88de2f, 0x5c2d53ff1bace11d, 0x1b1a9e6c638453a8, - 0xf72aed1ec0a6af19, 0x230157bf2561a47f, 0xde0424bdb6794fcd, 0x48729471c95f1dcd, - 0xf6a0598b4772caf2, 0x323de456af48b324, 0x92d0a06ddc83cce6, 0x455bb49d0ace2960, - 0xfd6d2430faaafa84, 0x1faf50dd97bfaae1, 0x41559a930f226689, 0xf7853dc5ee820c62, - 0x1827900add846d09, 0x02c6c4831af2abd5, 0x118de37cf6e8d132, 0xf6333484d47de9ca, - 0x4d0a62e7ca2f7f61, 0x85edb0a80d575ece, 0x4b489639624ea886, 0x67b585f1c85a901b, - 0x546d2f88d27b7748, 0xcd424665d509aecc, 0xcab6296ef0cdf1e5, 0x7af0dee910218c3e, - 0x1047a308cc6e03d4, 0x4cd57159c1187ba5, 0x122e80e0f2d577dd, 0x71931ebe058bb764, - 0xac86c688a0df2953, 0x6fe9552c242487ce, 0x0015f05835e26e47, 0xf09f5e9497fc613c, - 0x4b6d9a1187e2a40a, 0x2e56c3543c0de8c1, 0x2e7c09583d44fe58, 0x7ee4a21fa64e639a, - 0x072b357bb1329e36, 0x4b29424b427b85a1, 0x505c2c29f4f1b25a, 0x037837f7d1fa8a61, - 0x4691aa02e724d914, 0xf9612578c5fd5985, 0xe71ac574140e74e8, 0xa387ffa91cf64056, - 0xbb54b110e1493c0e, 0x309232ba427c76c8, 0xe24e570c52a97048, 0xf9e2dff7a6ce5794, - 0x74e5b00b8ca30500, 0xa3c2a5f3a031f239, 0x29f530ab8ecbf939, 0xaf40c098cf39e1aa, - 0xfa62ef0a667201ff, 0xb8aa4fad52d0c41d, 0x4fec2215b16214d6, 0x8a159680b7bbc775, - 0x2570b9daaab74ed8, 0xb2d3427f63c5b3dd, 0x5c20043de0a267da, 0x8d172f5b8c2e54ca, - 0x98e7ce45f5971c47, 0x6f0f2425a354f34b, 0x5ccaf33d8c52e73d, 0xe652926cbf5c3151, - 0xff86e5d00b0bd563, 0xa6f53b99bf8dd888, 0x9b86f5dc3a43e6e4, 0xdd966504239f52b0, - 0xd28cf01b0a2049e2, 0x63b06dbc16a0549c, 0xae3ef317e84e64bc, 0x7bb18aae5a9aaaae, - 0xa256f03bba656034, 0x6f3d836b97e338be, 0x686572916d31cfb9, 0x8af4c2df94db844c, - 0x0f0c1421b1d4b71f, 0xbe2c583dc659000e, 0xf073f0aab986e8eb, 0x00053a1bd6cfc0e8, - 0x582b88988d3b68c8, 0xe68960a141e4dc7a, 0xef0543d68502c491, 0x17a335260135fae0, - 0x7e36dc66bc16dfbf, 0x5499030fd1d3b0d8, 0x5fdc37548e0699d5, 0xf6da855de190effa, - 0x3fd075f61c440456, 0xc96590d9765e3906, 0xc4f219279d340900, 0xb6d7fe8e91abcfae, - 0x8c3de25864b469e8, 0xac84d786a8695195, 0x8503722387943bae, 0x256a74abf7e359b7, - 0xbbfd5dac02ef24cd, 0xc7c5bbe1c6822740, 0x0ffc6c69548aa8a4, 0x5dbfa0fcd7701ae7, - 0x00eb1a55fe2ddc6d, 0xb26d5ae633b92854, 0x28c933684ecad3a2, 0x31da221c04bc72ec, - 0x9ce94ee5ac3475b6, 0x46d0b101875805ea, 0xadf76505b132f324, 0x976c7574b87b87a7, - 0xe68e60bc1c25f907, 0x497c79d5b9438fa7, 0xc0acf4e2598ff65a, 0x5607c17d0bae1b19, - 0x5d12071c66120577, 0x772f87b78cc1c13f, 0x38217f631a9b1b76, 0x741772bf483cd064, - 0xbb6474471dc2d18d, 0x45579fcbf21fd86d, 0x4acd4f32b958fb49, 0xdae690181a8f3e66, - 0xf92bfbea83e0a1d6, 0x1a9adcdc0e368995, 0xe15c3be442c50e1a, 0x567510e046f8db31, - 0x101bf54e0f05183c, 0x380546174e856d8b, 0x6447ecbb57be82e8, 0x6e66ffb820a7ceef, - 0x7188af51791c6fd2, 0x9fcdf63f9ff07fa7, 0x90d87bcf2d2dec3c, 0xb16be6a7ac5c97f3, - 0xe4270ab82aa3ed95, 0xbd850eb7914f3b52, 0x7869e4362642c796, 0xf905a0b439c3977f, - 0x2831caee108c3335, 0x9c568ebe238cf0c5, 0x6d52ac420d69692c, 0xd0ed9966e5063900, - 0x5335c62747b59f47, 0xa18840615f2e1ebc, 0x82b544aac2e82cce, 0x4bdc24c9a5ac853c, - 0xdaa8512081f5d393, 0x296d27e57cc8d0f0, 0xec6f6e11f18be957, 0x1156dfcfdc26f0d5, - 0x949c759ba60bdeb9, 0x301578b4a3f22614, 0x653049722adf5796, 0x8eef8ccc23e6c1b9, - 0x64884fe5ed416e28, 0x41c7326cee44328c, 0xfd820dfc3de01d87, 0xcd15e6e37f55e84a, - 0x3776fc058ff85076, 0xe75b7374681b96c8, 0x67a267873320bb1d, 0xc217b6c69827a6df, - 0x9af2d19cefe29835, 0x4676d16685f1ab32, 0x04d850c49aaee825, 0x8e81b03e42b43487, - 0x21b81bba4881966b, 0x7a803bb31f6c245a, 0x7922420254cc66c3, 0x58ceab50b983ca9f, - 0x1f4fa588e5e53230, 0x0d360325f2b26d72, 0x5a503e16822bafd5, 0x7eb1bb9c18b9f2aa, - 0x0106b60873136d60, 0x76515e04340ee9a4, 0x314a28759b0a911e, 0x1f5b35a1ae4a7f81, - 0x5098dee8c62e27be, 0xc652ae78c3a86eb2, 0xe177af724f517cfc, 0x6da41fd8249423b3, - 0x99f54220dfa36f60, 0x04d3abe5502ed501, 0x055942122f3ff774, 0x0aca15737996a982, - 0x8cdf36e00fb40e84, 0x18e7dbf8829fab49, 0x86cd07b8bad6f876, 0xcc461a88c853fa90, - 0xb2181c38d4b1444d, 0xf27e3e4f88e2c17b, 0x8976b16d247f894d, 0x495417c82c5b5f92, - 0x6168f795746f53ff, 0xab72fcc1829c7431, 0x491c498e83b05d6b, 0xe0566db638d17e28, - 0xf7b7138ecd53e772, 0xce6b2ed9885ff54b, 0x7cb6aad48764ac15, 0x25bb2a692caec4d5, - 0xc1f8a7d86949753d, 0xaced07b867a041ca, 0x4a96286e44c06fca, 0xbb9e643adc5c4148, - 0x82d152060a560804, 0x2b2c83048a2e6c26, 0x4d1503163f7ae25f, 0xec1dcd5e532fc6c3, - 0x2d3ef0e95522871e, 0xb7316f9cb9990570, 0x850d0aedee195dfb, 0x34b1508a402f18d8, - 0xc1f07e202b3c5d95, 0x7e4a56c2ac2a9f14, 0x9bacf077f1422a8d, 0x3aa44fa2e92ae1db, - 0xb1f3701f42d58b6f, 0x9834b6245d36109b, 0x4e73a3eae0a60e12, 0x85cc652e677fd390, - 0xec73079a8347f096, 0xedef0ae1d6c7d829, 0xa3287756760fbf0f, 0xf1224bf6863f556c, - 0x3b51028c6725acfa, 0x351f8a45c88ece8c, 0xa51d86bc6a5e7c7f, 0x2c3c12ffdcc59b15, - 0xe795775769620f48, 0x1704855fb1483f5c, 0x3057d55201bd0a8f, 0x050762232efda61c, - 0x513e7705af5347bc, 0x6fbbdf25614c15ec, 0xd96c6f36ed55400b, 0x6242744f1f95fcc9, - 0x069189ccd8fc1915, 0x15f1cd37ad455d34, 0x98316057fb7653e5, 0x3c26c159e8630d1c, - 0x1bd1dc966a031341, 0x674ee0ea249a94f5, 0x8d22c449fbd78c25, 0x092d89c822cd79f3, - 0x3f637847d7fde301, 0x4567904474b901a8, 0x9d5c10ae67d3379e, 0x896d59db3843a8a9, - 0x8176b2efc6c04118, 0xa55482a741cdf640, 0x89dc5670a2dba5aa, 0x648c3cf4848bf82f, - 0x0466b4398ade15d1, 0x0697924120c74be9, 0xc100086d25a6cc08, 0x448d559904509c90, - 0x142ecfca095a7629, 0x61f1644225154f04, 0x115cf3b7befa7131, 0x37ad7dcca404c5ec, - 0x5cd8439d4aee100e, 0x4a96258495cf42cc, 0x69e2a04cd993ee93, 0x375e23d74ff5b885, - 0x5377e13299113806, 0xc578e11da2b9e7ac, 0x50deaa94950e035b, 0xfee5852e44cddca1, - 0xa41939a880da2a1c, 0xded44ebf39c030f9, 0xda1391a366ff953b, 0xef36b33e02be3811, - 0x1befa81aa41dc12b, 0x95597bf4f67d44cb, 0x7949a4c180749247, 0x3389e0a207502389, - 0x53ec2aac3a724d3c, 0xa53803de17dd2a04, 0x805452524d3001e4, 0x2ff04ed1aa7c45ba, - 0x6dc2e3396061bf4d, 0x89305cf6a85b4c06, 0xebe02688e4ba769b, 0x0b1f19a0e3231b27, - 0xe188402466c68416, 0x127b27d8d97fd131, 0xd215526b20e083ff, 0x4b15fa79982c13b6, - 0x1e2fff2a692cc961, 0xde4e80fe6a827bdd, 0xa0338f7526c97e60, 0x3ffa9d9d5ec555a7, - 0xfebf228c37db2238, 0x687007a8e014237c, 0x7ca5ebe3781dd59a, 0x05b46050d20eeafb, - 0xea8041262852de2b, 0x6eb1607f8539536e, 0xfcc10541992d7bf7, 0x58de42fabfa2c704, - 0xb3cf070910bc7f70, 0x26643556268c3625, 0x1c2713fb9f65be2d, 0x34d7a23cfbe6544b, - 0x3a10c168248255ad, 0xa69d669683a9c14b, 0x460e2af63ae35487, 0xa1b1b0c748504510, - 0x6dd55ab97ac8b270, 0x85d62f5ad1767fd1, 0x274cb6e8a11e6deb, 0xb00364df3099914c, - 0x0f424f5bfecc56db, 0xd82b78fd00f82839, 0xfb2b8ae8167af182, 0x9c441334602b7072, - 0x9eeda2338c7c434a, 0x443beaa029272823, 0x87edeef0ad331149, 0x626e8486b4de5387, - 0x19de0ed5abc37ccd, 0xd9b30adca0118572, 0x9c61dba0f3e40777, 0xa4671fc8caf89109, - 0x4e9e18c8c635a150, 0x2e2d0c7205d5ba0f, 0x8ade9dc025b3fc6b, 0x58f4000f9fbb137a, - 0xab2663bf5c970b19, 0x1c7394e4daa75d1b, 0x154043d9d838c332, 0x353de120ab55b189, - 0x2217c2e0b910676b, 0x3bc824e8341506b7, 0x65348f93a5cc2d82, 0x4b4588813419c367, - 0xe4cbf3a775ec4429, 0xebbdfba4201dd16b, 0xa2e902b0d55a3457, 0x469ebf4b43c255be, - 0x984a36ff70407510, 0xc48845a11ef232a4, 0x6a31ae01406d1cd0, 0x3a0d1019adae9c57, - 0x6b236779eb024380, 0x539d15f2f04df89b, 0x1539bdb3df4cd73a, 0xef9ae1c07447ba06, - 0x3b20c2720d628cd6, 0x00090f31c568b31a, 0x312daaa42a0cdd28, 0x00d5660897ff7b38, - 0x46d450dd16f8222a, 0x03fdb8b1bdc80ff7, 0xbd25a752604b8f31, 0x2d3d23fc78d17578, - 0x249ac8b3b073f884, 0xab776510d4b155c3, 0x0f901a278de859e4, 0x6a8e5c8a5eec0c28, - 0x1a0030d53615ee6c, 0x94e3a655965b8fcb, 0x8c96b15c23a73726, 0x4c5cee55bc2b5b1a, - 0x72158c3b2485e6ee, 0x8868a38b7a94af03, 0x685250359c5cac78, 0x3def9a660c34ccf4, - 0x9fbaf6c9f9408408, 0xb2fcc6b9e5c986b6, 0x5686a961561a9344, 0x01191bd3c49b0202, - 0xb9b2895eb9a191da, 0xfa168213ee3d2207, 0x64fc9c8db8ac88a0, 0xf44b36eb17f01852, - 0xc0cef49547493078, 0x2c8f655a7768c43e, 0xbad3e2a7ba401bc4, 0x8bf489d2110cffec, - 0xec4e5d151606cd49, 0xb6821342a54336b1, 0x19683bb2a6c9a9dc, 0x3b486440a3ea5d5b, - 0x4c255204773a455d, 0x2cda95dff3268edd, 0x9ace7fa71c9486b5, 0x975521dec41922dd, - 0xcc3255ec6161f4ac, 0x966cd574ebc7e117, 0x8c78d71c770f319f, 0x0ce5774d965baab4, - 0x1b198b0c594c5107, 0xda135791f36e3539, 0x3e5986e41675bb65, 0x6e86501df18f963e, - 0xd5984d329555150f, 0x2102a2c95b707d58, 0x90e8c851f2d1734e, 0x0b91ebd3fd842010, - 0x0956e678a70b17cc, 0x6e03dd48ad3bd55b, 0x68a81b6e0c73893d, 0xb568170e153be7e7, - 0x40a8e28ec29d592c, 0xaa5e2794baefa34c, 0xf2dfed7dd66d98a0, 0x8a57fba98085ba98, - 0xaedbec19a010b33a, 0xeeac675444a4322f, 0x952212bf47ea9dcb, 0x2be61cb715d32f11, - 0x7a4575609d071cfa, 0x54bc02a750fa7f5e, 0x0ce48a1c3d7a53ae, 0x3a50772501d6b4f8, - 0xca5aa96020d62c54, 0xf564565df9620f8e, 0x070aa789c9ffac3a, 0xf080d900db2ad43e, - 0x25daa110453abbdb, 0xb87b7dc1caec595c, 0x6a014234634d5814, 0xfe61557a0eb73496, - 0xe83169ba07db2075, 0x02846dbc06c74455, 0x9ac8229bab731248, 0x3da1bee0133b9407, - 0xc720f1dca01a14ca, 0x3b7135df80f4300b, 0x9a4bbf9946b19f02, 0x1b7183c8038b68ab, - 0x1e0c467ada0c6e3d, 0xbbb4f4308ad285f1, 0xfc3b214eb76463a2, 0xae82454aa1cabf8d, - 0x4ff8c559794cd0e8, 0x2bb1e4de7232100a, 0xe10f3086170d8446, 0xa6fe832b273f415f, - 0x59b9984c7fe7acae, 0x5e5dcb093db867c6, 0x9d60938932c6b158, 0x1147767e2329cc51, - 0x3d492aac8647680e, 0x366b49f792ab664a, 0x1460707db39b989b, 0x5d3b5b41096ac61c, - 0x1817e90cae7d7c57, 0x68160afa17a37063, 0xb5249f9034a7cb16, 0xe91c8eb5ccb11456, - 0x944839e78d8ada50, 0x00dbb0ce5bf89ce2, 0xb4048ea3f193a9be, 0x74bd065d8bc757a9, - 0xd48ca43e7ad929d5, 0xaacf25a6b204b722, 0x503b00ca70db1f71, 0x0a2ac26d009638b4, - 0x6ddd514f7b4e9c5f, 0x43c18c776514e4d3, 0xf39ce6e2dae69ee0, 0xb8aa1925bb5886dc, - 0x7ea100a92a6f9ba0, 0x246dfe32fc5efa19, 0x4c76806c03397513, 0x1ffa042587c1c23b, - 0x7341d4d0a2fa3b06, 0x1040b3e0a99df8b8, 0x1706521ba418003f, 0x2890ae49aa03e78c, - 0xf3653f53a27949da, 0xe8aa64b52adc7208, 0x8990ea519fdc7212, 0x4f80300bfeef0cfb, - 0x854b8b126cfa817f, 0xc49ba62dc43ed596, 0x5a848d418950c6ba, 0x50908d394de9e05b, - 0xd8d3a5524bf14697, 0x1b41254644063c7f, 0xaddd324cadb97afe, 0x2ec5436e101b3b8c, - 0x6d4337d3cb2e459d, 0x3acd68f204c01e5b, 0x96e6223566271b9f, 0x4f1eeac7d25a6bea, - 0xdb743f549bc1e70d, 0xb4a80efdc7def233, 0xe135be26c73cc673, 0x54df97a6280761b4, - 0xfdbced3b8f9b8363, 0xf46085d320d572fb, 0xf0a14f022e979bcb, 0x6f9f5ce7ece6b8d3, - 0x90f8cbb98cab0737, 0x8d31bf010dd8c27a, 0x838957587a699c17, 0xfe82ee34a90f81ba, - 0x1a7ffc5b3ce35282, 0x4f9067d32d68682e, 0xac6378eebed261c1, 0xb53f7c37a66c25f0, - 0xd93b47a353a9c0a2, 0xcf78016647e81b0f, 0x5c1951beeb47b6ae, 0x147853f00f5995c1, - 0xcf927312f632def3, 0x49ef4594cfc4b4f8, 0x3e1b752b4d78fa0f, 0x1882a837887df899, - 0x2ea1408527040c68, 0x8be8c480e449cf5a, 0xdbab6ff4ff3b8c3b, 0x7676d17d838f06d9, - 0xc835c62d8d53b69b, 0xbb5a82031e8019ef, 0x4aee0217a3a2e499, 0x5cbc3b8114ba0b2a, - 0x7ca22753677ca0a3, 0x7a05743f1f50c4d7, 0x0cd0593c5dca20da, 0xedfa441184139c49, - 0x05f7712b276fbac8, 0x53c94d4d91b7d7ab, 0xe768f5113cedc320, 0x40e4a1f39bf3f3fd, - 0xa11ed6b9c7830354, 0xad6e232b6e2950c1, 0x7557b94a96a9548a, 0x91ee96da76588e2c, - 0xa5f86bc269acad54, 0xeed58581d0862377, 0x2650b5e576534c3f, 0xc735f9ca03674d14, - 0x2e9b57a5f0aa1e20, 0xaba84b505d0a3730, 0x935ac4d419044d67, 0xe4bada37d4f9df2f, - 0x83137f9e5f08cfb6, 0x485c7339e24a3ea0, 0x941e7aa5ff371dae, 0x6209b70ac3275683, - 0xab6990061d039b3c, 0x60b0a3477805d9f2, 0xf39dce403c7eba94, 0xc02a7b2459869525, - 0xdf8a58ae4cf70e27, 0x443fff84b0cf982e, 0xc507fb4381acf121, 0x9295c6508b89e27d, - 0xe7427e9679ebc213, 0x3febf378ae486fe0, 0x9dcf064b55e74e24, 0x97eb8a701cc3b15f, - 0x11c564d988346bac, 0x7fc0ad9af399e017, 0x92330608fda6205c, 0x2edc32b628056b34, - 0xda07815beafc7cc7, 0xed465ca8dca50b23, 0xcf33cd8c0f78b624, 0xd4b600fb5b08f65e, - 0x3febfbe7b93181c1, 0x7929f3c915ee7a5e, 0x043d494ddd66126a, 0x49b6e73b15414ee4, - 0xa5e5ed21ac6852a3, 0xd91ac9aad73cd4a4, 0x50384c7d4e2cff5d, 0x3a53b3c7faab0565, - 0x412aa0cab833d18d, 0x5903be9720271c18, 0xf4202c1af7c23615, 0xfde760408ed44237, - 0x32b9c84b1da554d3, 0x0177f951666a8993, 0xd59b61f911e59ada, 0xe47dc1b90cb3703b, - 0x0e0490f85b875dd5, 0x9d86d6e11408255c, 0x3a7d916b049c5bfa, 0xb8b601e7bf44e7e1, - 0xfb21753aedd3a954, 0xe743d95287a5b605, 0x72badf2e32652deb, 0x3d0a0bc9adc60839, - 0xdfa7dd25351e1bd1, 0x17db4ba4617dad3b, 0x7f121ad3b18633af, 0x8bd1b3eb98e18893, - 0x6dc1f12b53b0726d, 0x25a12113d063377c, 0x778f2e9632701b2c, 0x6f6db3afb5f84244, - 0xfeaa8c2d77cac2df, 0xccc596b84cbf81be, 0x113a706c9175c4d2, 0xd73be62875063034, - 0xd644b625db2adf13, 0x0e74101715928b12, 0x4b9b45fe86e3e887, 0x326567c1b8eb4400, - 0x6378a2bf177ed627, 0x82796fbd85d466f8, 0xe3b46580cc517238, 0xe27bd5be3ca32760, - 0x00c9906048323c07, 0x00add80864fc3321, 0x5ed46047e6638844, 0x41f5aa0fa8a6de87, - 0x775419afacbef9ad, 0xe25d9addaa6a3d95, 0x929e95bcb16c0f1e, 0xf27402b28442885d, - 0xd77743b89e170d39, 0x607b5dff43c4eed2, 0x0d46fb17f1bceabd, 0xb12dd623ff00b1fc, - 0xf6b0afe670cfa994, 0xdc8f808b367ec7b2, 0xc09fa60c7749d901, 0x0b9a01916af7f190, - 0xad072226300b1aee, 0x0793cdd244feb924, 0x70c0755b697f6c2e, 0x600728576fd0c5e7, - 0x8f2da12931507368, 0xa586fa7ec78c7091, 0x7ae0c124d4ab6b6e, 0x404eb225a730a760, - 0xbd368e5ad8a7be2c, 0xba6600f77f1b5cf8, 0x1a217b8129d409e3, 0xdde2a85066af4a7d, - 0x130195c6a633217d, 0xc587f70874942ce6, 0x75739d13e4e459c1, 0x479b35bf8f835e08, - 0xe2417ce224577eb4, 0x6bedd3f8a71899ff, 0x4fca88e3de10cc40, 0x2bcfd4962dcb6fd5, - 0x3d5cf2da6e7cadf6, 0xab0c642c3a7b98c8, 0x95ae180376833e73, 0x9f9506f5c406ddff, - 0xe410d533d769b795, 0xa27e319307a5929d, 0xb49a403d767df49d, 0xe42ce93f919215f5, - 0x9ad55241f4ec4b0d, 0x3dbf372ec6ed1a03, 0x4e8efbada855a3b4, 0xcab9fbde7690002f, - 0x984f91a9d25ad3f4, 0x7c17d4e02729a547, 0x6b07f0aa543d67de, 0x6f70ab7cbe47badc, - 0x13b12f4d0511bffd, 0xc1de5336a586e4dc, 0x684e9b4a8a835e21, 0x464a2b04ed8226e2, - 0x2eef8b6ab39e32f2, 0xe65c226a5f37b9c6, 0xf46d3c6183c26673, 0xd29a7088de2f10b0, - 0xa50d32bb1355f114, 0xbea9129b0333bd65, 0xe956d7c73007f5a1, 0x544959f678d8bb27, - 0x1c404e70690ef3a8, 0x945e6d5735c0b740, 0x2c3f36f8cbfa8566, 0x2e0426a0b5d80c7a, - 0xcd2ed31ed2587d19, 0x6efacbfbdf6908e6, 0x968750066ad2658f, 0x9d4b3a5c5cd10bcf, - 0x0ecabb0759ef811f, 0x53dc033e2983287c, 0x003d3259d30b0327, 0x1a5c62726c1d6e0d, - 0x5e4313d7849869b9, 0x95fbaf1036cb9382, 0x72e47f25a6eb21e7, 0xa62d5ac89b082f60, - 0x17cffe72cc96c08f, 0x86feab9f56614b99, 0xb33c8a135b06c763, 0x6398d78df950ce9d, - 0x2e0562954ac6e5ab, 0xdd925084d65f5907, 0xd32f4e0eeb030fa7, 0xbfcd5162715654a5, - 0x08d0d6558cd40786, 0x06877bce467241de, 0xb09fce9e48bec539, 0xb52893d15e530b22, - 0x7fa0e7a78a47804c, 0xcaf594d8a1749ab7, 0x00756071f94091e5, 0xe13ae9c4c9393cfb, - 0xb8ff7e4c5c10b0c0, 0xb3d1281051f15c3c, 0xd3bd8d56fc1cc3a7, 0x7821cdb5135216cb, - 0x8d67806f2f6068ad, 0xe61929fe191bc17b, 0xed29a8c799af3450, 0x200c39ec2759f604, - 0xeab78ed42624ea80, 0xa1a2b12173b1b2be, 0x02c6d1d26de912ba, 0xcb982a8d921b8e89, - 0x3fa7806f3b3b9e05, 0x9a593734ada4813c, 0x1f44f40306d6fc30, 0x3817df3ac118db8f, - 0x84fcc3d072f4a566, 0xe6f4a01e19b53f39, 0x84e306d981d74de3, 0xab534a239c1ae202, - 0x3ce23d0d9e41442d, 0x9c088a28ce6cb62c, 0x90c187a63b5492a8, 0x452e8b2a8d85ff34, - 0x3cc54d3ecc9d1d73, 0x79a425cc18b5642e, 0x367e41eadd6ff8ff, 0xbfdcd3d790108551, - 0xe68bbcd85451b6b3, 0xbaee4537d2210c21, 0x9b493d2018d5a2d4, 0x2450fc3db9d80c5f, - 0x30b8b70dbd1f17eb, 0xdd2717995cbbb3c4, 0xc0a6d2d5e288793b, 0xd4f2669e4eef4dca, - 0x655dadbc0c53b597, 0x589c38a66d76e8b6, 0x50dfa6357d52950a, 0x0385dac5ab79f8c3, - 0x987db7d3b56b8688, 0x652d42f469a97dec, 0xd02939ff063ec2b0, 0xb0d8ac67953de4ae, - 0x44a0b43c48e9f981, 0x23511e5c2ad3388a, 0x709d372e8ecedec8, 0x283ec4d4593314df, - 0x3e894d77492fe9f8, 0xd0ff7b33a95c9c4e, 0xb8e679cd2684e7d8, 0x624d4cc29c46e770, - 0x04e71608a7c372d3, 0x71cc2e96e3234b9e, 0x97afcef914a633cf, 0x81457a3c739c894d, - 0x916c16dd94221621, 0x0a419a989362721c, 0x3ca722df05df2d15, 0xa4853cc08db037c6, - 0xe3c69adfcc18d483, 0x565420f5bc003d2e, 0x27d4e078224431d7, 0x51e5da5bba319836, - 0xee5952d6317cee08, 0xd1ab83c77b2f95c4, 0xc4bce189215a2902, 0x304cbf7642eb0c93, - 0x62ce0b72b0531d23, 0xc64b27c0be6fd67d, 0x32f7a47b0100e2ac, 0xb2ffcbb69887a35f, - 0xd0ee092438094101, 0xf87afdd802db2fc0, 0xf2d1d31a968fe123, 0x1e699fab5cd051dd, - 0x888322238612474a, 0xa67e20a6ad4cbe2d, 0x404b6ea6befa95fe, 0x63ee3c57297d9794, - 0x0d07339a8fc99c01, 0x661dbad1385cd640, 0xdb52f8e743865c50, 0x7cb5df36939864ad, - 0x1c4b581c70e03212, 0xba78d2d1d5fcc83c, 0x25c1be8c8b031426, 0x91a96a8a2e0addd7, - 0x30f7ba737e781f08, 0xae48d61ce5900a08, 0xd6017f52c4a41699, 0x679718da675bd100, - 0xfc15a4edf51cf587, 0x750471f8dce79549, 0x7d271c7b283a020c, 0xb3e923e0ff040d70, - 0xe107058e529d20fb, 0x3422fbc8d4c93808, 0xe2ce0bb9f4e72204, 0x52c6a5bae4102705, - 0x5f342f72b78b9900, 0xba6ec81fb4fdc1fe, 0xdba0a728e5f31406, 0xcd5fb8ae906754e2, - 0xdb9ef8d54d6daa80, 0x0a7061451a0c7018, 0xba2e7be8cef73c51, 0xadbb52f7d02d2afa, - 0xeadb948831ff2634, 0x9efb6255516c7273, 0x875b7028de3e6827, 0xbdd4404a1130b225, - 0x0f232bd6b73d4585, 0x0a06504e17e4a42b, 0xa2b73821b1ff1341, 0xcd0eafda8ce7b427, - 0xf073fc120ed84bef, 0x99e15c0002f5d4af, 0x15cd30d512d606a0, 0xd8de5a35e1b83f63, - 0x097fb0d560f8af2a, 0xb68847a954fc5a0c, 0xf1f7fab53790aa9a, 0x31bf1d4ac0029531, - 0xa9c4a2c0e56ecea1, 0x7544f42fdc24eb0a, 0xc7619b45f5b002d7, 0x06cb75e874d88f50, - 0xdbc94a566c79e4ea, 0x2b1c899750e6e87a, 0xec4038ef589b3cff, 0xffbef115cf6591fc, - 0xc3e4dadd296d2015, 0xc8da5b22dd5cc031, 0x4c3d50fd72711d27, 0x9ce85e22362de40b, - 0xa5d9296c7e92889b, 0x8734b5ee57172b93, 0x087da725fac5b439, 0xc4dd30a9676abda9, - 0xa883cb2e03607e5f, 0x872f8800ad4f7503, 0xa62aab2d74d00ec2, 0xaab1922cbd15fa2d, - 0x39b43a4e1596ba05, 0x5274eb52dd67c58a, 0x86a8239bd22b7d01, 0x6dd4d77df49dd5f6, - 0x55cf16da1e91b89f, 0x01607833de80b593, 0x7d190450266c77ec, 0x7a46222a9d994ce6, - 0x6395ee4613c9f79d, 0xd9be6c4f4f572fa7, 0x682aaf993ce8ace1, 0x5e101a104be63691, - 0x583622e6f93f77a7, 0x7b51d0bca39b2b03, 0xdb07337dac6d6037, 0x449c3a8c929c8684, - 0xcb36ecb5e7f727b3, 0xf6924201c7798dfb, 0x5fdc99745e4c0a18, 0x2d74ade2588caa60, - 0x1dd4c9adc0f46df5, 0xdab116dead0e9104, 0x5bb770e46e52df93, 0x5e020c30fc715f47, - 0xf193549b502a7638, 0x81ba8cee1498081c, 0x4b39938e666260d5, 0xd6437be8c6db0b86, - 0x19a5e87d9b5af1e7, 0x9c2403e9a52f77f5, 0xfb19060f59779918, 0xa1e608cf4e692f4e, - 0x2411f2bacddbfaf0, 0xf24ac5e3abcb1c1e, 0x2804bf0cfca53897, 0x18a7b3791cb41652, - 0x89af37439d4bc60e, 0x2576ae9a9b8e5282, 0x7f3379ccf01aa876, 0x86513b2bd4185610, - 0xf826f6d5fd037494, 0x275d4322f851d993, 0x9b140f6b8150721f, 0x28e0c5ef42439749, - 0xfc7f5aeec6ab106e, 0x41019fab9fea7dd3, 0xad20f7d7e5e16c28, 0x733d5e86ab3cd029, - 0xf7f2846847d153a7, 0xb2c38018d6a8a9e6, 0x277992de9c63c07d, 0x5894eb3353a19488, - 0x17612711bce8bdff, 0x749714592288992d, 0x6de6623ef90afdb7, 0xdf5d81ed1b1145a2, - 0x906ca97c98329ea4, 0x84195602c19e1d96, 0x884ec039d958b571, 0x6967d91db9a7791a, - 0x778de6768155aade, 0xd1d16a43aba579e8, 0xc4f77feffd7f5d89, 0xd09b4bfa0c3d0d2d, - 0x47e5a2aa07fed0fc, 0x665c729cada29d0d, 0xd4805c9d34e81edc, 0x2132689f0a3c2e89, - 0xa75ff57444b1ba7f, 0xfe3d1beb159457d4, 0xb2bab2016fcb2fbb, 0x62b4fd302cdaf4dd, - 0xd5073013f0b0384b, 0x7375793ec708c192, 0x98e52236c9b71714, 0x48ec18ee50f8fdf2, - 0x363bfedf438fef4f, 0xd41ea73295b143ad, 0xaa3dbc67edbdae8f, 0x33c8794f9cd4d352, - 0xfaa79ddea95a1cdd, 0x3ee8f98b2dbbe739, 0x7f6dfd48100895d0, 0x1d5fa08d420cab74, - 0xaf3a47a9377385f9, 0xad10af098a65827e, 0x7cc8f9a8b30de56f, 0x65df538488f9705a, - 0x7a324c4c39f6794e, 0x36897230f764d854, 0x7b1a81f7083a441f, 0xc9f6570ad9c366c5, - 0xc60fa4c70c589edb, 0x396e3af190dcd228, 0x0c0f0e93e4ff2ba1, 0xa9fc4563a5e9f69e, - 0xb3cc765f664a39cb, 0x55e8ce2c21a55bb3, 0x3ea4c53934a9ddeb, 0x16fa5a57cd5f0bbe, - 0x175d8804af0c86d2, 0x9cad21f57c7c27fc, 0xca3d6917e0345b60, 0x0bc13cecc94e753b, - 0x1752882a8de0d156, 0xcac3431e2f927733, 0xd760e44df10274ef, 0x62213dd8291736c5, - 0x5a724a8dfd7e1306, 0x2b204cc1b6deb535, 0x6c1af48289ad8cef, 0x65243a03a1c38dbb, - 0x4d3e18e5bac085bb, 0x8601671ab007c492, 0x460fbfc48331d0fd, 0x8b377fd61dc020a9, - 0x25d8b7ff8bf01ea8, 0xf7c1affa2d5a68e2, 0x091b333af226313b, 0x4be4a3c5395f7cde, - 0x3b3b416775b63804, 0xfd505d8c2867f79d, 0x2f38bffd96fd1540, 0xaa4d819c895e7c7b, - 0x79f90665c1475fbe, 0xd1bdf5bd270155f2, 0x8c01a2519be2a7fe, 0xb001c77e19e090b5, - 0x12366b067589f3b5, 0x7aec7a032923e703, 0x04dc44c9163ab998, 0x454ed4843edbeeb3, - 0x4973e5a27e31606c, 0x0605af22b3c1f07a, 0x63a3579d62aa3b21, 0xa587c44740640701, - 0xf67a923d543b3acf, 0xdea79ba02f19a955, 0x97f14f1e4dd36595, 0xb6f855294e02a4a4, - 0xf747fa6bcebca709, 0xcbf0c93a237a9655, 0xa5c3c3c3256360e8, 0xfc0b2727d576c2bc, - 0x2c16af0ec03987ff, 0x71387460588360d6, 0x32343aad274a175c, 0x130f74e06b5bae47, - 0xd1bc1c10338f6afa, 0x2b0f29ff48f8c7c9, 0x135acaeb9b6709ad, 0xd71106f0327d4047, - 0xa3125588679e7b67, 0xdbc415ca13a55a9c, 0x1d6cc04b023bb5d6, 0xc19c83dad308b764, - 0xa3655e25f8872774, 0x97121e6894499b79, 0x285e5d60f5b6fd2c, 0xcfff7fbcb1799ee8, - 0x3df0f25015ab3776, 0x447726f0bc89a254, 0x76aab70446686f13, 0x455eb6627069f6a9, - 0xdc176e711899564c, 0x33fa15bd2502985a, 0x1748fad6ae8bd8c6, 0x0985b152f1adc870, - 0x14272034791c5c6b, 0x27a24f4f5cf627ad, 0x7be960bf97ba3850, 0xa3deef691d10b684, - 0x4520d77812733101, 0x0cf51ccff852cc4f, 0x48e0a830770125e9, 0xf1b8d5cbd6e05065, - 0xf34c63cf466e48b5, 0x25a8863368fee33c, 0x7325ee7a101f114d, 0xedf98c24e579f993, - 0x4c44ed80aba7e9b9, 0xa1e6ca452acd47fc, 0xc06e561726bf6d77, 0xc29d2989bb24db28, - 0xe57fac3e75033e18, 0x38b5d6f4c537eb16, 0x640041934165cf9d, 0xbcf7c377603a3f9a, - 0x49eecec34dfa6f15, 0xdea935be4070e334, 0x2373fb09488f03ce, 0x78e09fc9c679833e, - 0x1e28b660b30aa3b2, 0x2fe2e259dc8a2788, 0x5e6d11eb4f416897, 0xf38388d82ee40044, - 0x3223a284900960e2, 0x2ce794bbbc66c8d2, 0x90cf60375624b940, 0x835d5c070ebcafc0, - 0x77c3cccccf7425b5, 0x239810f33dc17b73, 0x7d5f2806a42c05b4, 0x2b770978eab4eb83, - 0x337dcffa88110f7d, 0xdc0fc7989e405469, 0xf98ce8adc047af17, 0x67fd18cd6e12c86f, - 0x38f2259c56e5fb2f, 0xef26948f22c9019f, 0xd62d756f74a5507e, 0x3c8c59c87cb0cecb, - 0x3a4adc31be44a97a, 0x78f1702517c0223f, 0x499d2e5d6f95f58f, 0x4e645163dfc42bbd, - 0x8c9c21c5f5f9a4fc, 0xf87ee5e93349881a, 0xef48bced3c7bb5a8, 0x0f97c2226b7b66da, - 0x6e6f0128ff05d048, 0xc99facf1a257760d, 0x07dd7485317716ae, 0xeeac9982b61497e8, - 0x4947aa2900d7225b, 0x17fdd310d95b4599, 0xe19527ca9d50b554, 0xc1892cacc26a227e, - 0x31f6fdbc9c4af32d, 0xb2959ab6771adb98, 0xb1571166f2b82dd5, 0xc12abf82b6bbf559, - 0xfe8500135862b66d, 0x083b1c0d2e8063d5, 0xf8562b3b41ca1e2a, 0x5a6cff012816803a, - 0x0ee936df4c0c6c9d, 0x655f53a11f1824a1, 0xbb1af8ac07a9907b, 0x294e6193dc11b2aa, - 0x10c901702b05529e, 0x47621eb90a4fc548, 0x9011a522979346a1, 0x1550828ac6b861e0, - 0x1a16237d1d869111, 0xfca554b19efb0a38, 0xaae4e23d328dcec8, 0x4325701ba1efb861, - 0x143edab4d6c10978, 0xb7eca8125aefef97, 0xb9b96cf742a4d1cd, 0x91f7bd621788d9da, - 0xce65ad539dbacfbb, 0xc07f2cfa648171d6, 0x84380c44755a2bf9, 0xb17c1bd1fb28161e, - 0xb2ead82f204ca51a, 0xb9c519c3deb48075, 0x490e8e273386a4c5, 0x62fe20b896aabdc7, - 0x08150553d4fbb8b7, 0xb63bcefa397b7e1e, 0x8e31d60c9f8f2c1d, 0xf14c82ff9f148b40, - 0x9ba5fe4c96c0ad6f, 0x3a1d23df697e327a, 0xe581a946a5a3c7bb, 0x20c32db2ba2cd082, - 0xa1cbc0a4ef37a9cc, 0xbbb1adffde502e68, 0xb44ea412508f8201, 0x92f0a51aa41c9bbf, - 0x2b1da53e0b137079, 0x05ee0f3c3d481a43, 0x3ad65c87391713c6, 0xcba6e998bd4542f3, - 0x5e32d453b8f2ba75, 0x60da46269bef3bfe, 0x3a86517f5cbcb848, 0x5a99f0da7fa5f345, - 0xed9b38f0fa3e9d3d, 0x15bcf01a4b29770b, 0x8a1286497c0c6f90, 0x6acd5c08efcfa4c0, - 0xdbf02b38010a3cdd, 0x80e2327c2fe4ac7a, 0x855c8b49c2963894, 0x098559a14258ece3, - 0x95624ac44e757a49, 0xbf25d3293ae9cd0a, 0xd53238f6b8f7bb0a, 0x36a968035fd1c8d5, - 0xe051d6484063306f, 0xf53fc03c4723b819, 0x69e876c598e8a5db, 0xfabc1ea5302e31c1, - 0xf26a1ae58fbb95d7, 0x3a822ebab5a0bf93, 0x561c42f41b868f38, 0x18fafa517dd4b6da, - 0x6af57fc438fd45a5, 0x40d86293cd88f629, 0x0ce7365686ca1391, 0x5d8661c1c84d41a9, - 0x79868e0607026b93, 0x30556d27f3ccef6d, 0x15dd62c2eeb804fa, 0xa333732911f5cc36, - 0x9bff54e0807455d3, 0x797998c07ba99ab5, 0x4671f90e861fc580, 0xb590b2677303f12d, - 0x0cbcfd7d3424c39e, 0x2811092e87360f84, 0xc51b1e2455b58896, 0x730e2a6a2617790d, - 0x0c6515a19e5b067b, 0x3403d29e3e84fa67, 0x0ff2a06d1a289f05, 0x0d5274d3c65a773b, - 0x2b5ea8181f91471d, 0xad656587e821b310, 0x5bbc8b600a762e90, 0x24148e977d001783, - 0x3d9fbaa34369f083, 0x3cd5a711da22332f, 0x2b8faa2d06e89b7c, 0x08adb3255fff611c, - 0x1fca63750eaf1c40, 0x414b5d2d73baff49, 0xab3629b5d9263c38, 0xdb7cc3bf44e4cd1d, - 0xc732e590e2c8379a, 0xad424b702ec19d01, 0x43b6b4d978912937, 0x5c15ca4822975fdd, - 0x8abc511954742d74, 0xeebd3a1fb466e087, 0x070099e1d43a246b, 0x9063da354cc04a99, - 0x29c3ac9bde33b592, 0x497b59e128c6497f, 0x73862eb2eec34906, 0xd32d228aaf11dfa5, - 0x2bdf5d9b5bbbcf33, 0x62d768cd0f02d1d5, 0xbec1a308e03900a0, 0xd79e72d438a9c1b5, - 0xb5aff58a8b6c4024, 0x9506625f8cfa4926, 0xab2ca7627b707193, 0xd905f2b202a12e9d, - 0x8154cbfc9d0ada45, 0xe723738498040733, 0x72b89e04543af8db, 0x9499bfa7de4f51e1, - 0x746e3b239d3a6eee, 0x7cb0642f4f625572, 0x8cba0ab60d2a63bf, 0x0548667d48752f05, - 0x151fccc5863b35f6, 0x5b6e1c502fe4ebf3, 0x3dce755562a10890, 0xa0e5b98c27c407a7, - 0x169d7c98c445a565, 0x790f9ece7eed3be7, 0x4b73b8834b16fff1, 0xdda5765e581eafd3, - 0x653cda81bcb801fc, 0x9e3a64620be6b967, 0x20ac4042fcc06134, 0xd7824b9cdfad452a, - 0x6b4adfaacfcf41f5, 0x70b01f29240ccfb4, 0xd02b0694c5f716bf, 0xe18af666ec172a38, - 0xa898f65cff3ff144, 0xd6c1800170c8f098, 0xa3fd80165b09a524, 0x2bced08ff5bf500d, - 0xb683d65fb19f1987, 0xa5ef2ce4f48d31e3, 0x6fecb237491b5724, 0xb8cf7b5ddeb2ae20, - 0x0c214b1dc76c1977, 0xfdfc91d67cb24b57, 0x7f9872d72bfc9f2b, 0x1f0cc2a7782131b3, - 0x993742b329727edf, 0x1a068def328e3011, 0x4843e31a02130818, 0xcbf406399845eac3, - 0xc9d35eac79a19a17, 0xf70045c93eb82f3a, 0x30c75424b05de249, 0x49a7f928af8bdc97, - 0x1d2e19f457339378, 0x5ed8d2b4dde49c63, 0x32287bc276e4dc95, 0xe21aa52523105d92, - 0x44f8e79db5c7d50c, 0x701f9ca833cd4c10, 0xa7a80b865a23d508, 0x5c43db78ca0fe31b, - 0x2052cdde7cd80735, 0x2c50f11c1f8c78c0, 0x21b9ed48028d1acc, 0x70642a59746ff93e, - 0x54b851c4239deb79, 0x82e0e9d8cc53614b, 0x07ec84a6c32b6d23, 0x0636f29ccdcfc4b7, - 0x697504028a739001, 0x0e83edb74e6cb802, 0x13923fd32d0cc9da, 0xd95f3300f556f534, - 0x57ba97946df38723, 0x7501504fdbf7fa10, 0xc20a8e32607b0758, 0xfbda887ca4b987ae, - 0xb220a0d666698947, 0x6010a6ae85bfec59, 0xa2066ba9c2b4f20d, 0x19daa6bef56c872c, - 0x93d824c59800ec4c, 0x1f09e8b96ea4e172, 0x8346c8dfa9c9b740, 0xed174c23b46ae650, - 0x10937a28ad590fd7, 0x6ce1bb2f6f5aac99, 0xa90883edcb654bde, 0x63657e74170bb493, - 0xa35c97782cdac674, 0xb5918c8271a8b61f, 0xf8577e80ef39d4d9, 0xfdd6e6f750fd1f8b, - 0xde2da1449563a106, 0xeb4fa4d6bc38addd, 0xe9a46ba3aa76be8c, 0xcc2d9dbf498c1469, - 0x3edc5f3e9dceb64a, 0xac3b751e1009811b, 0xbdbb04d12a6b5d73, 0xd4ba520b808339f5, - 0x23c4ef1d091b8cec, 0x4041e7a367d6596a, 0x7829385e5a564f91, 0x28794d93e4855add, - 0x5d3a4604b6bef218, 0x20f405e2c7155627, 0x22e91a99ab11cdfc, 0x0a8448bfcdcc81f3, - 0xcc79588cc231e874, 0x0a407f549d7782e9, 0x96c46d234b2b81cc, 0xa383e49fe81c48e2, - 0xa95b1d89e3cbbb8c, 0x31328380629718ef, 0x2a60a04ed9aaf281, 0x5bae2a13687a673b, - 0x046f72d520161fe6, 0x969c870d836ba230, 0xc5b1ea79a08ded5f, 0x3497c5f228081897, - 0xf31239acc18e083c, 0x5cf4113b4c0dab5c, 0x60368758620852ac, 0x360b0ad5b275d591, - 0x89daef9a13bc1e1a, 0x8e056edf1a6291c1, 0xe756641a8d2067aa, 0x119b8ce49a12497f, - 0x794ae2689af4ccb0, 0xb2e8a87f5bf538eb, 0x3fc1b9671758b88f, 0x8e7ffb31776060e9, - 0x8bcc28d529ded9c1, 0x5a3dba33bd858b95, 0xf95ba6cbed8676cb, 0x56d7b831645354d9, - 0xcb97f95e560fd42b, 0x15c975e3a5f02edd, 0x1c8ad4cefe84aab9, 0x89fbb3e5096a1b53, - 0x8a6bd72d19a49336, 0xa8c438674728cce7, 0x6e9bf006367fac77, 0xbe9c0d034da9aa83, - 0x562509d4613f9d50, 0x9ec925306b829745, 0xab206bfad977cd93, 0x53fc240e8ba439d6, - 0xebdb554dc809bfcc, 0x7abe14f5f08a80c5, 0x5ef4b08d48bec6ea, 0xdb5e2d1367fd0c2a, - 0x9eca1168a300f53d, 0xd2c94bc950afc447, 0x4fe87520c63f57e6, 0x2e114f5af5bf6213, - 0x47c01ea3f87daedf, 0x856916951bfecf80, 0xfa14e57dbca223fc, 0x30f29363506f032a, - 0x800324891eb84f0e, 0xef95b9baf0b59993, 0x9e8ed3145f4c16ba, 0x7ca2427b60bd568a, - 0x5d09f57e7aeb0368, 0x7554d0144f997c0c, 0x836c9e97ca75ef79, 0x6fe2b78a492a0bfa, - 0x6c55024659cbbe4c, 0xb223c5125067de96, 0x65f8556a7a0fd7ce, 0xc8dff2d4a543b7b3, - 0x109f0515ff97b76f, 0x1c6d61af9449b16a, 0x6782ba8b5c295541, 0xcd61f36187ef01f2, - 0x8f956bd35209c792, 0x05d59b2b36853ac0, 0x1cac6a7a2c55e244, 0x22fc68aabb238fe4, - 0x93473f8653c4ca78, 0x26d1fe1ae466f878, 0x068541965a0ac6ce, 0x250deb24b76e97db, - 0xce3db7ca3fabbaa9, 0xd325fb7959ae73d1, 0x5396324c7223d623, 0xe4c5dae4503ef770, - 0xbe1cbe9a36116ed9, 0x81396af46c645e30, 0x78eb830518944b0b, 0x0aff10c440185c1c, - 0x673ad80932ea6ca9, 0xf950ff18c9efe0b4, 0xc9c05a63846d071c, 0x6bb9867b8384dbd1, - 0x218533e69beda535, 0xb05399e3783e14db, 0x37349f1c0794650e, 0x90d61522228fc7df, - 0xd6dc715c1a996c40, 0x70cc8908e7ce206e, 0x01a2528c6c7797d1, 0xcd849e4f15fec036, - 0x0344f566333e9491, 0xbc4a737ced22411e, 0x2fb1fe55314d97df, 0x440546ef4c2302fe, - 0x435f749d341964ca, 0x17feb4e8f6a95d71, 0xd8b49c65668f0d16, 0xef7a410529cf8d2c, - 0xc2e8fabab69b3781, 0x870629bef642f07f, 0xcd84fd849dd44e32, 0xfdac194751e9ef10, - 0xdb7d9655142d971f, 0x4584c374c1665006, 0x317690c9f33518f7, 0x86b80f41d7a169de, - 0x98386e92e68433b2, 0x20fa940824a71db8, 0x032ad708792f6f86, 0x7e86f29b3d1cab95, - 0x16ded5992f217d1f, 0xc26aa63754d300ac, 0xd5a8c1e348f2d7f6, 0x0b6b023137759e5c, - 0x751d2a2d6f5ac019, 0x89815fe1f6a6d9b1, 0x4f5fce1a005599f0, 0xdc562c9b86740dbe, - 0x51d47ec6922bb5b9, 0x43adda9c459efbdf, 0x0bc47dc36a530bbc, 0xf2ca1eb4609b70fc, - 0x6899de23f23bfd69, 0x3b91544c8da13ee8, 0xb70470e07809b424, 0x993a0cbafe1c599a, - 0x61a9999f5f79e066, 0x97ce971bb002b6e6, 0x5b6c48609c0ea530, 0xf18a1bc681ff09ab, - 0xeeb6d1704a0de5a8, 0x685dd0856d6d3b8f, 0x58234bfcd7dde628, 0x444288801fbe53e3, - 0x4bdf30ed3e09c0b8, 0x5e9d2940228d7872, 0x971fe18a19c705ed, 0x2ceed9f8e683e599, - 0x2b68f590354f6e24, 0x317d6029b2c1ba67, 0xf4efa17f924c6edf, 0xae99be4459572eb7, - 0x31d3721087d452e4, 0x86b4c1186209780b, 0x625b55e9f7ccd20f, 0x82b8590608ad25e1, - 0xe3174c6e6deefb93, 0xb7c364f9e5d5c2c1, 0xd81d193d68170e51, 0x9b31af9c2e7a6be0, - 0xb264d807799747c5, 0x35406801fa2fe502, 0x3a30d416f8c0ac50, 0x645ab9f50d60fbc6, - 0x7c951203cc6c5a61, 0x16a1bb5122f997b2, 0xb0c5b6b16ce7c525, 0xc5183f8241e0f121, - 0xec10da3b8b523ca2, 0x903a6bf119f8e5a6, 0x6ae88d5be6f20157, 0xf65813ccdde6de57, - 0xd75a9fb2c92003bd, 0xcf4d0a2c050db506, 0xfe135f472cc3ec40, 0x5c78c7bdafc86873, - 0x8794cc8b4128227d, 0x371fba35d5ddef74, 0x53d101f48b8d1be9, 0xdd96cde41c1413d8, - 0x9910c6cac12222f3, 0x4d73acf31d9a8b30, 0x9cad7a253e57f524, 0xb70ef86a6a9fa4a3, - 0xecba7bb8d95ceb02, 0x610805e9d8ac4e8e, 0x2954a349451ff273, 0x87d3dddfc0d2eb25, - 0x31e4cfdc546cd33f, 0x3dd1015bd4b1a9c9, 0x34e6ab7a515cce41, 0xa4f22cf39f1cfc45, - 0xb464ff5eb4eae19f, 0x5c919485e8227c37, 0x60864efe35178354, 0x2428295be8c87610, - 0x737c2d31482e8af7, 0x809459d01441fa58, 0x00739617c28b7fa7, 0x5af1067d2335c563, - 0x24ebd3779985ad1b, 0x644f007d16bf4620, 0x5c72161a4b69fe91, 0x768fb7a25c413376, - 0xd7722256a5fa5afc, 0xa7d33547e6d6817b, 0xf7ded47ff6dbd0c0, 0xa32ceb0b14cb84b4, - 0x34fffc287e08ae84, 0x250b01f07b70541e, 0xc330859c1071b6f2, 0x7d419225744ffdc5, - 0xd6edb96d32a0918e, 0x5ff5e0551e0e5e4d, 0x59fd55d0b6f84d99, 0x9bbf6ce86e759ebd, - 0x8a3091a3af0fdce5, 0x6123b268e3891542, 0xbf3e9a3527b3fc91, 0x83840881c56293f1, - 0x153489e3eebdea4f, 0x74befd70e198c760, 0x480ffa542f801aa4, 0x03f2e70186ba50ed, - 0xe88d875b109c6c53, 0xbd931df445b0d3d4, 0xfc2894ca22edcc5c, 0x1066a5577207be0d, - 0x37f816803f972d9f, 0x175fd13b5ae6bdfe, 0x1dc7e6b9d06d4917, 0xb323f50957f4f1b6, - 0x305421a2fb06ce9e, 0x2547cd68fae353b7, 0x5f4ceda4556087f1, 0x3c3d2b7c2cd7fb49, - 0x8108e47fbab9e200, 0x2547712db36f5f60, 0x612c8d9ed208c410, 0x361f5c6b6de91ed8, - 0x2776ac78c812e4b0, 0xdca590b07b798a46, 0x976c50dff6b6796d, 0x9013f75a77298ca3, - 0x42c8c9fb21116b6f, 0xfcb09b8c8e776827, 0x78e2b35644743a91, 0xa203c7f743f2180c, - 0x1dd91e45d4dd2abc, 0x8a8effbea582ebe6, 0x0a7f2df21d63cc0d, 0xd7dfaf88d996d764, - 0xa2fc94bd9f908309, 0xecc3e36594c3e009, 0xde8aedc2bbd12a78, 0x8fda6164c9487270, - 0x621cd2debe25d21a, 0xd2f595317a56bc92, 0xdc1ecc590b0516d3, 0xccbf0b3338e306d9, - 0xfa020f459ed94817, 0x97c77259ad48fee6, 0x6980cf7631b9b233, 0xba0cec9a661e8849, - 0x8fe7eb6906dd3555, 0xd88d8a5a165232ed, 0x1a9656966730978c, 0x811a4db34e0f6852, - 0x5790a988ab8a063b, 0x0288cdfae5575614, 0x9b673c4021752b39, 0x7a20c2401b85f684, - 0xadffc4e612e05232, 0x1af35ff6cd2825f8, 0x4215da02cd13ded3, 0x47bf1a7ce68b7230, - 0x5f8f8aaea57c55a4, 0x9885d05cd2542364, 0xa945b5e5673ccb8e, 0xbd2b8732ed143f93, - 0xf4a53be4bb45a1b6, 0x292dd9745aad7fb5, 0xeac4450be4da8a97, 0x3e15d59e60484586, - 0x9a544f595bd88ac5, 0x88a4175e62441273, 0x1d4df9115351210d, 0x806dad28cd8fe258, - 0x83e72db86b4193ac, 0x8ac4679a7566924e, 0x1888b6ee4533746f, 0x64980420b3f2959c, - 0x6031b63e5eacaf3f, 0x3400e8ca7d212f10, 0x68c88a3457370b23, 0x2100766c9b127aa6, - 0xea08f8f9c782bbd3, 0x2a663454c175061a, 0xa88fa16f7fc16335, 0xa0784eeed2698503, - 0xbb2df291f2c37992, 0xa0417cf58398ca6d, 0xe7d52653105508b1, 0xeae4336f7b5d6697, - 0x94f4fd20d5a05f6b, 0xeb687655cc318deb, 0xeaa180612ef6d649, 0x1e52b1aaf7f80766, - 0x71d93ee3608ae4ff, 0xc697c5f587aa24ce, 0x6eff5cc55b7a4ca0, 0xfef61f4267121dca, - 0x3f3602e0877fa3f2, 0x5ff7b1d221b405a7, 0x53bc1542e81a3f84, 0x156073fd71cfcc82, - 0xb7e2f343fbf47484, 0x5a08619d28b7f98c, 0x834498e92393e623, 0xc0ac393b79718ac3, - 0xabeeea4895446789, 0x140b529e2b07643e, 0x21ae3f79853aacbb, 0x44ad2d0990f68dad, - 0x13902e913a97d294, 0xcbbd6f12a9fc6507, 0x63a2ffffeb4a2034, 0x61bddff9135935d0, - 0x0fb8f6c35769e0e3, 0x5f99ed6b8583a395, 0xdcb92e3ed8f9d253, 0xd3647f790044bbe0, - 0x764e2cff879ed85c, 0x4c8550964e05e7f9, 0x07fa97575f10767d, 0x122d9b222e50c3c7, - 0x9d02078e35eef7b3, 0xb9a6e8eaffb82479, 0xe90b78bf97d9b963, 0x83bd489d4f7d7334, - 0xffcf4c529c0c40a3, 0x830be4590c8c9525, 0xded4425f0035a025, 0xf27b78518ccb1314, - 0x67368b2aeb5f522a, 0x75fea8d59f53934f, 0x0631508406640e54, 0xf26ff24bbe55470e, - 0xd80d0bd5a8a0fcfe, 0xaa239f6ca5775639, 0xab00f8c9ccc0d7b4, 0x515f7c1d5ebe18b4, - 0x197f11cccb452d66, 0x7d5285f0d9cc14ae, 0xa64fd2740b9fa3de, 0x0994d30bdbd08278, - 0xb0805478044b3598, 0xe9cdc8a7ff34bc8d, 0x19c803ed3e289d3a, 0xeff429199efaf2ac, - 0xe4da296c6cc35a02, 0x7ea6be40e152402b, 0xf2b1ed318dc9f447, 0x2e8bfcf47268e756, - 0x77e1ea429fe24a7b, 0xb2ac6e9f234c57a3, 0xd08a887f6f121134, 0x5f633ceb8fd91839, - 0x80718529cf61952f, 0x7d74aa3b4379ae4d, 0x7d1294787a0f5df6, 0x87e7cedb3d31f444, - 0x7fc9b76a83d0ba5e, 0xbfb3d782a5ccfcd3, 0x3b7a395620ca9d16, 0x4f99980ec5e4554b, - 0x10acfba4908a8078, 0x7d6e371bb8dfaad0, 0x39f4a2fa0321c69a, 0xe2b56aeb88d08466, - 0x75b7ea90c19c7045, 0x83df2887b082603c, 0x9959f1af0b47e63f, 0x1c50672d39e09dd5, - 0x5c5c5736c88a8f51, 0x3597670c1610fcbc, 0x28733ea483c7459a, 0x9c9a51c4a3e89245, - 0xc17411bf2b79ec63, 0xc53824e14bf176d2, 0xba47229691a0748d, 0x9241c732e9f24a1e, - 0x00144891e5056563, 0xb0d9b0c839f4d643, 0x26bc6e0aba6db9c5, 0x4b0b54069a520144, - 0xb6250d4628820dc9, 0x09256eb4654fccdd, 0x8837d07266dab501, 0x86caf9c1ace4b9fc, - 0x772ff27fcff91506, 0xbbc6e07d9e9e56d4, 0xf7f1f269b17bf681, 0xa6e61af74875bce4, - 0x26733027a9f43088, 0x7a411cfc02229ed2, 0x7a333f0ee8b579b9, 0x3f5a2ca63db98dfa, - 0x72847b87f2f08ff0, 0x7c15723f8e3e649e, 0x917c0a3bfdba0cae, 0x6cd7b3bfa47495b1, - 0x83365ec1a892d5ae, 0x8d40fdde02574135, 0x51d90586a2b7730c, 0x88da5fb16420433e, - 0x86ca3f4fb016ad73, 0xb57fd399e0db1e25, 0xd88d62c8e21de074, 0x893cb5fc6392d706, - 0x516353658d4700a8, 0xae1481a2b8649028, 0x88bc3dfa04e2e358, 0xd2585b40c5de1757, - 0xf2525c77121170ce, 0xc786dae485abbcb9, 0xffbc1378494d8be6, 0x4e9c59c64acf357b, - 0x016da700d4de7474, 0x8d526b0ba6594139, 0x91371534804d51f4, 0xeaac0ef283e0bcbe, - 0xc6636fd552a1f7ce, 0xdc7668b39482ac91, 0xc6689d5abc8668ed, 0xecaaa304aa77da3d, - 0x8886c5ab9dc8feb8, 0x4ecf9fcb77dd14e5, 0x6973ce60b7c2dd91, 0x27246b9247843128, - 0x4286b30eaaa530ec, 0x6b245a25bb5d0142, 0x6449b36bdbd9b906, 0x45d525b156707f77, - 0x2430cb07005ae3fc, 0xbd704fbdef7e895e, 0x189eeb8589455477, 0xa1c4f6eaced92cb3, - 0x4f28d2442e7de9a1, 0x6bd1ce0c0590d6da, 0x6b5f5f054f163493, 0x2490ac5c7d69c723, - 0x0aa7bfa912b1e3ab, 0x2fb29033175acf86, 0xde79b2de687da6ae, 0x000f189cc9b3bf5c, - 0xf1a2c129991ff277, 0x59052e36382cd2d7, 0x139a6db0c1ec2a8b, 0x1167f43b985c898f, - 0x8276818b0e5f4119, 0xc35b6ded13821fd8, 0x70aee68f2a45a4ce, 0xf2718867a22778dd, - 0x302df979402bde10, 0x027108603ed9cfa2, 0x5c90161497ddc5f1, 0x5f0c966bbd71d9c6, - 0xfaedb74a312872ff, 0x48e176e0a01893e8, 0xe995889528d6544b, 0xb99fb176e3217bcc, - 0xcd488fd4bc91414d, 0x1a04f6971356e715, 0x2a7e4e4869822eb5, 0xcd246cc619a0dfd5, - 0xd89218f1d65a4779, 0xc800628d28c346c7, 0x994834b81c1dfbb1, 0x0234841ea632eb85, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x56ade753a18128e2, 0x8e0b65a3e6f002a0, 0x0c67d90f6a968afd, 0xf74bfa7fc58d5cef, - 0x2fcbcdfd6cb4f76e, 0xfe7bbd749052577c, 0xdc2979947d1f1167, 0x5481430e884d6ddd, - 0xab41074d3279aaa6, 0xb224d23cf8cc855d, 0xb9775e00e2dc94b7, 0x60fc89c599b38501, - 0xa40b6eaeb852d1b2, 0x5a95acdeba9d0251, 0xca4649001d906779, 0xd2bfce1442a1a126, - 0x31840acd267bd511, 0xd2b7056a7d65bdad, 0x228c05b9d42e77dd, 0xbac4a70243b1599a, - 0x367047cf7735e04d, 0xe12c6d25d4e174ba, 0x45492b671b88fae1, 0x5875282efb76b479, - 0xe563507a4a5c9e86, 0x3ed469fa90a3f7da, 0xd9c1a904dfacbe50, 0xd3a9f9728ec1396e, - 0xdaa67a58d9402a08, 0xa936adef62506d6a, 0xb9c19d615875a3dc, 0x61df4bc427d24570, - 0x708f77c22abf5c35, 0x99a53b0ebdd1c47c, 0xc017233c28a95795, 0x8e9e9ae36e50f6d0, - 0xecb1703545de861e, 0xca636c80b031bc99, 0x6d33d1203e83615c, 0x54502434f10c069b, - 0x7f055ddacbcce497, 0x11a12b3c681554a5, 0x917bd7d938de9953, 0x3e9ef8eb6c549cc2, - 0x3b6fa570719bd91d, 0x27b61e903699897b, 0xa2bbc64d406e0a27, 0x21dfd2028bc627b7, - 0x59487e05274aa1e6, 0x54c162fcdf490ed0, 0x2223a4682c0d0311, 0x0b46ef4a76d1fe4e, - 0x54d5b408d6d77871, 0x3f693afd38b64802, 0x93d46a1ee711c081, 0xf20561696a403b47, - 0xa9f36ab0daa2b5c3, 0x45f19955638cca22, 0x2ccbc1265398a372, 0x2a89eea977260cac, - 0x75334e0b6a180fe2, 0xfb3facba29e5b8e3, 0x5d171f094e85d2e7, 0x49c4cef69ac20dab, - 0x6464b661e403051d, 0x454d9e7d05f74a05, 0x7ef78fa1c632a6d4, 0x4ac5e32c3818c72b, - 0x79ce543f96d4dd60, 0x1fc99a68a448e6eb, 0x1d211886b3852a44, 0xbfb5fdd1b06d96e4, - 0x36e11c944ecd2741, 0x9a800407ffbf461c, 0xdddc2ee912b1e680, 0xccbcca995ec3268e, - 0x8315b2721997d624, 0x68351be7f4458024, 0x40b5dd9561855a6e, 0x95ab53765f070b4b, - 0x3924be3b01f901cf, 0xb8e3abebe9b96b47, 0xf95d138760ebf51e, 0x3507cdcef2af1327, - 0x30ef79c39a3b8f20, 0x90f1bf39a1e10658, 0x19411380430b8fc7, 0x9865e6acc27a9367, - 0x1818da646b991b5f, 0x1fbd9e0f443a754c, 0x35d89ed5c436eadb, 0x00ee6b980b1385bc, - 0x8c6c19977eb7ccc7, 0x7e068d39593efcd6, 0xd94a26e618fd5c51, 0xe260ca63b5bf5457, - 0xdecb1dbe41838c79, 0x5581c3d26a80b0e0, 0xbb464bd48e392236, 0x8189c66021134fec, - 0xa58a2a8805180abf, 0x5d0a0dc03a91e480, 0xeda8323508cf64bc, 0x2053fda089b80372, - 0x1daed6daf35c2fce, 0xbc3015d754bc3708, 0x21aa61e28317a4fd, 0xd32a0b919e399955, - 0xb137815bc69575bf, 0xb34594ea2cbb95c5, 0xb06fc328c7b60dd8, 0xc8ea7378716c00ae, - 0x2472318dd4ae2b4f, 0xd4aea4200f4a6729, 0xcd3b782f7a87bcdb, 0x9be2489086acdb0b, - 0xb4604c5def22147e, 0x34f165eccf1361ec, 0x164973840a35e939, 0x2e704fc6d2a3fc0e, - 0xc897a6b93429783d, 0xb5d6b5765e42dc7a, 0xa5eee9d4a635ff82, 0x06687d1078ba0903, - 0x036e4394ee352a7f, 0xa8133f73ada63db3, 0x1a7c23ce9652df72, 0x61a557d8490c51dd, - 0xb97237b3a2732e60, 0x7293b1e573fc8e86, 0x4f85c8f63d901718, 0x0de1f9b1e25862cd, - 0xd00c305f449e7775, 0x878842cfdd380775, 0x95db0a365e58863c, 0x8f8e221f54f5b82c, - 0xa4881c5f7a77f45d, 0x95881512dcac7276, 0x7b39a0e3181c9835, 0x6da0be3bc6134542, - 0x848bfc3706d132fb, 0x7c4b5de565b85251, 0x58335f12321812ca, 0x8046641c6732093e, - 0xe40a5c97869fdf3a, 0xd2294084a25f336e, 0x53b7c37aeef28ec1, 0x653c0334dc6233a9, - 0x32436f2afb405d59, 0x7512866ea3a5aed3, 0x90cd7c4da140e6f6, 0xb46a69cd9121d655, - 0x8968eed57ddb53b3, 0xc5fd7d554ca0dfdd, 0x7d5f742a14e3fcf8, 0x600dff0cbca7d7c9, - 0x1db48658afec558c, 0xd8fe267f0d81688c, 0xce4e4241375c6041, 0x80cda5842b710cfc, - 0xa58fcb00c7ae1d7c, 0xef8566280bc57dbf, 0xbd9c32afa23ad4e7, 0x68785faf5b7f2bc6, - 0x645a519b44525c75, 0x6d34cb9d094efd4e, 0xe7cd9acc04c993d7, 0xc7daa3f496bdbb51, - 0xddc2584218cd9ab7, 0x07c2c378a73bfaa2, 0xc33bf53f730e0e5d, 0xf986e73befc5f655, - 0x7923f71eaef7757a, 0x866de2974cefa7d1, 0x97646d4adf3f4eb1, 0x3e09482918ed4569, - 0x6434f2ab5a652108, 0xbf76b82783115a21, 0x3d461bd81f0193da, 0xfbce33564d8d64e9, - 0x83646a58249d9437, 0x98e48566a080ce72, 0x73d223e0ac402801, 0xc568e561351ea597, - 0x3e39d6d3fd1b31e3, 0xe20f99671429b295, 0x061189bc2b15bf22, 0xc4ffcbaa121afcf2, - 0x2b05cd3224f394b0, 0x389c5349c00304c4, 0xe15327c9f84fe4dd, 0x6c72ab2e66405dc1, - 0xf3c7c74e6b085a95, 0xbc0b58c231e52c0c, 0x648c5eb6af760e9b, 0x5ab49e76e95c05eb, - 0x75b96f95dec0a89d, 0xe38d3f0c56b3442c, 0xb474968e76a76628, 0xda62edc26468c3fa, - 0x22869acf40771c29, 0x8d1e876ebddc2610, 0x6d8f098662eb66f5, 0xfa61c4a2ad73469a, - 0x163f7d1d4546ee06, 0xc24fa53771941edc, 0x83623bd6c6e2ef76, 0xa97b777755dfffe4, - 0x547f8c03c592449f, 0x4626610bfaa8b1d4, 0xba3e27b95eda58d2, 0xb49aa4438ea5d042, - 0xce93bfe2c4fc48d5, 0x95bf9cd4cd372b02, 0x5db18104b9d49574, 0xb471fe585495822d, - 0x6653add72d8ec8b6, 0xc795382ca0e89b03, 0xc7e90aa5879493b5, 0x1d3b725757785553, - 0xeaab684ca4840157, 0x45489cdc127b0482, 0x0eeade98dd6c5878, 0x076c2ddaca695fd3, - 0xf56f9bfc32779222, 0x425b770e549f8664, 0xf44ff886babda87f, 0x48e78d982d5c775a, - 0x38912341ca87380d, 0x991a4eec59790d60, 0x551f87b5a77b0db3, 0xa75b3a0e6306c016, - 0x70b4a8ead1570af9, 0xc132696edb595393, 0x9ea19890375e1ba5, 0x03b2a571006f96dc, - 0x969e9cf39ac8ad0e, 0x5bb5572e200cc78f, 0x755dcf2785768ded, 0x92fcf34524dc445a, - 0x92fac528d6afdbfd, 0x2afc8177c5fbeda4, 0x080568800d45c6f6, 0xd8303d47cd804386, - 0x7f2fb76f1320b5d4, 0xe69d24a85f64aa2c, 0xaf80e93b91ea4db0, 0x9d9ca6b54709b09c, - 0x4f147df629677af7, 0x3450a1b94a2d6896, 0xa50f9fe2ae92049f, 0xd9c8428d0cd4e7a8, - 0x53aeb2013db86e57, 0x38b9549594385b39, 0x927107fb4135d1a4, 0x41e8a6ed7ffd05be, - 0xa06bf7a3591328ac, 0x4ca14a4c728a309c, 0xbee22f3a71bd8199, 0xc498a0fa84d822a0, - 0xc0892569e58d391c, 0xd6a578e2d93ab827, 0xff2dae24c26f8706, 0xb2aaef75afeb7b5f, - 0x330c2a337328fc7d, 0x1f5f5356264157b0, 0xb2f307c183aa8ce0, 0x137a0f069e9ec063, - 0x56a5481da68341f1, 0x2e843ced41a4b156, 0x4c94c9ea05681b65, 0x15b36d578cb6bf11, - 0xc1570b00402a3d4d, 0x2e794a714e132d9c, 0x16ac1e8b907d2ee4, 0xadf3d0aa0d8b32ba, - 0x48bfd58c4b1067c7, 0x78d126b622c49d4c, 0x80b2922a7123be3d, 0xa668a019514cefcd, - 0xd2172cc2fcd4b978, 0x9e4b1dd4cc9b2e47, 0x22c291c567a73be2, 0x02e5919c4b3d36a7, - 0x397ff3a415816d88, 0x490a83c41b04f0b9, 0x2cf2bdfd24cc53ac, 0x7d788d4942d22778, - 0x9e00a10d6c08e752, 0xeca1303e883d5eaf, 0x793058d7d4f537a6, 0x4d1a48ae67163065, - 0x8777d271272e27e6, 0xea5decd51647e90b, 0xa90c8588f5db968e, 0x3f4f7d18da7349f2, - 0x2750ba6a0b5a6d3a, 0x4abd07407e857301, 0x798a2478cd06cbee, 0x64e9dad00625ef3c, - 0xc98e07ac71dd0d2b, 0xba9067794f64fc4f, 0x33b0cfb9a2a53225, 0xcd4eb913b3315345, - 0x957e1f9b09b09cfc, 0x0d231e09e033c10f, 0xf305de1b555d7536, 0xa7a3c6a3672519f6, - 0x0e47030df79b2560, 0xcec5a920877d68a6, 0xa1c198cb9de40c01, 0xfea46d2a7bbae9d9, - 0x42ae0f90b29a41fa, 0x69c2b5ee4dc853ad, 0x5886b7b244e8f8da, 0xdf3ae3e33e1c2601, - 0xaf6455d3e06d8fa9, 0xad1b931243f798e1, 0xc318250f66e29e28, 0x4ce17036ed8d1dcd, - 0x24e7733f6711b3b6, 0x1b401171145ed99f, 0x8db529b6847b3081, 0xf38d49b2f2268a4b, - 0xac3818ab95c0c7f2, 0x9dc8827bdf8d6025, 0x83e3049ae1b28b07, 0x6946bc44f9dcdc1a, - 0x5090498e7a0a2cc4, 0x20c728652e103d6a, 0xe1ef326900cfd7b7, 0x390c1a247dd6b011, - 0xf9b034b05e136fc4, 0x04afd6c7901b4781, 0x5a7de9253cbe961b, 0xcee994d1835fe30d, - 0x2e8957538afaf2a4, 0x0142b006393b3988, 0x33c35aa8fa7507f9, 0x85b0072c191832cf, - 0xf3d6f769ea7aa350, 0x03f20e5736bba3db, 0xbaaccf20c928a386, 0x4acfbeb7bf20c9e2, - 0xf70870c8079586c2, 0xd30bb2a77b7668c0, 0x8474cf4bfe5b6907, 0xfb437a3a78bbe41a, - 0x3831d80c7292a47b, 0x20d68f0c570243ba, 0xb16843e72f1b6101, 0xe1bde562f3617838, - 0x5ac0ceac72859058, 0xb8c3bbc1a7c592f8, 0x1f73eda1a0aa0362, 0x0cbf0a88b2973c04, - 0x51a8819818af1e53, 0x23d1cc6c5c5902a7, 0x066ded2f9b950cac, 0x909c03b584fcc6df, - 0x42b21bcf08836d4d, 0xc014376a44679fbf, 0xa4e25915bccbd87e, 0x4b983bee020b0ff7, - 0x28b3cc85e5019644, 0x13b41c7d3807e7ea, 0x08eda7486dd03ce5, 0xb7d06e8a66aa9166, - 0x3cdefae3e31e848d, 0x92018365825457f1, 0x7668f204e6104f8e, 0xcb9c3e8cd721c67e, - 0x33ed893d74bcb5c7, 0x863edc504301c648, 0x3b3bb9383b5fa061, 0x7f59e14a87fdb967, - 0x603ebe9ee6d252cb, 0x7242612e277577dc, 0xe8af8f921d690951, 0x2b9a7709a5f20dc5, - 0xe6c7b0e4f0b5e2e4, 0xfea2998e54c74435, 0x7dbb07bd70b2b57e, 0x3452dd18abbda548, - 0x3d11d526d6d90fb1, 0x933197d80ed19016, 0x0f72b20376fc2a69, 0x8fcf74d978eaa1de, - 0x7cb20c4d844c6ec8, 0x3a5e93f246a6ce1a, 0xe2860bf20ace0123, 0xf30bd30bcb85e696, - 0xa6d348d0bba958ff, 0xc00d7812ff88491c, 0x9775cbb5adf55d09, 0xcfd894f9184e97fd, - 0xe9ab298ec0417b69, 0xc72f3fb15e2fede9, 0xc9743f18fbced93f, 0x43b61b8069880ee0, - 0x825189ad88a33454, 0xb218383d275f6aff, 0xb97f5c6fa26d1b63, 0xb4cafe74a9f64b15, - 0xee8087487db34682, 0x041c626fd16bb541, 0xae97356a7fee4cce, 0x06e4343a305b42ec, - 0xc484b8f1319dcbf5, 0x0770593606c406d3, 0x961fd507f6cbf4b3, 0x0db98c3e6489dc33, - 0x10d06b153c864857, 0xed4e4cfb17304d88, 0xf7da72b989ac2076, 0x28194504b2130379, - 0x16f3c70d9642268a, 0x6bbde17657123d77, 0x1776d1027681cb1f, 0xd7bc9e6404472b9c, - 0x7302b960ddeb1536, 0xe86a2ab1265ecfa1, 0xa739c45b70f89619, 0xad5a2b78cff8a8bd, - 0x36040d8e58e9d9f7, 0xe85a01c491b42dfd, 0x7eef716453ccfd91, 0xb0c3d37c2a423a0d, - 0x500b3729475c63dc, 0x267597292bd5cb25, 0x5cffd83768329a89, 0x4373732c04dbd8fe, - 0xed651d4addf29942, 0xa72494b134f14f42, 0x946d44f687ff81c7, 0xd832cc13d732b1cc, - 0x5113577d10b77896, 0x84979de1f59bf743, 0x8674f36c973f2380, 0x880dd79a4d8a136c, - 0x6b74aeee4614443e, 0xf2cd42e402a3f55f, 0xbdc7f2db5c3e0560, 0x8a2ba36de240d185, - 0xf3d2569f6644b2c6, 0xe58d535c54f28116, 0xec6d494814057979, 0xbd58c0151a959ea3, - 0x52410b41b6d257b2, 0x6c0ae9a4a43e7082, 0x8ec381d52579519d, 0x2c7e25997f9bed86, - 0xb6f5df9ba2e90d8f, 0xd909305724bcd54c, 0xedf1cf65f53e4ec6, 0x2d02f22f6e965d6c, - 0xeb92ce0444c65392, 0xbb4bcdaa05d66441, 0x8469e04ce903ff86, 0xe9f8a144b7177708, - 0x6062d2b77f9096f2, 0x8b94e632abfbd988, 0x006dcc4a8b357273, 0x0b52b365cbe8e247, - 0x83e8df34ed500725, 0x60fd84dd1d9dd5e7, 0x8084cd8c5ea4c7c3, 0x20b1eae5c60af630, - 0xc3ded3e993ef93e5, 0x14f0de738fcf3494, 0x30e0b5a99e6897b1, 0x90cde81f228c44e4, - 0x79865ce5f95cdd7e, 0x2fe876ee23e0afe2, 0x85a0cbc260e2a112, 0xb4572a6b95ba6774, - 0xd76a575af98ef0da, 0xbe7fd58e4019fada, 0x9e31546f4480eb77, 0x2207255c4da78544, - 0x4151a1a4e409cc08, 0x24f42c01534d84a3, 0xe00cd214dcfcf70c, 0xbb12e3cf2271e4ce, - 0xdf9234ccc043cf94, 0xd65946c90ef01aae, 0xacf686bdcd04a68c, 0xf886c2f66101fa0c, - 0x21e99356dd3754d0, 0xc1d66e4014890a4b, 0x7f21863e79124044, 0x66c5684baf8f53f9, - 0xfa3a335deeeba46f, 0x7ed01ebd976ba75b, 0xd5f7d4a203f31d90, 0xa7f86f52d24f9a55, - 0xd57277f15fa4ab64, 0x217657d3f174d742, 0x1a222059ed4f8708, 0x02df867792490925, - 0x70e6b96c1d3b9308, 0x4cd1efd51599c084, 0x3e4db2b342ae63be, 0x043591db287b835e, - 0x45b6487afd915989, 0xd481630e213803f7, 0xacb638e76039fb66, 0xf2fd3d7c759b7e2e, - 0x5f053ba171dae8f3, 0x727e8b676acc4351, 0xa0f3cff7e86b30f5, 0x87f1d93d7c5e0908, - 0xdf071bfc0823ce77, 0xfcff6cb801d89da0, 0x162bc7acdbb97075, 0x6edb143c6075d706, - 0x5f40d968d73880bc, 0x9381bf4e3d255711, 0x09666fcf275dd855, 0x02a99f634bb41153, - 0x40ff7e0456e8b2b0, 0xbc3d41ea2a241bd4, 0xf536beff38cd0323, 0xf23284725407737c, - 0x081ac6490e883541, 0x24819dce50295be4, 0x45158ba43e44f2a6, 0xb44eb9760b20402b, - 0xcc827640eff56875, 0x170353d089c82f78, 0x59761458c22a7b80, 0x3c38dbab63a12852, - 0xe8874b8e2439a3f5, 0xe00f6141a0ba8b2d, 0xa95f5f21ccec385c, 0xd7423c3bb39557ca, - 0x48a538fbb3052bbb, 0x1cccc3140a0dd8a5, 0x197544019fbf8294, 0x11c44dae5b97deb3, - 0xd75e7d364a9f85d9, 0x17119f6d11e845ea, 0xc7377ebc4f4f57d3, 0xe5f5a3e4ec0e66fb, - 0xd3f78681f6e9dee7, 0x7769676a2c6f1b64, 0xc383b05c7d2136d3, 0x0544a24860f19dd6, - 0xd9c6039ea1047fbc, 0x1711e7b5695a7844, 0xa0c61bb4157caa5c, 0x136ba9283175f8fb, - 0x8818099c9e918e12, 0x5d6520a7680d300d, 0x87c5c86f623d5222, 0x86faf4f72bca8507, - 0x8e80b76f6bc33a69, 0x7503aff95849cf36, 0x0781c0232d62012a, 0x43f95e8700817154, - 0x03a3074f05f6fd83, 0x83092c8d4060d103, 0x316ff2cfa1cf05bc, 0x66a18b3689c3b485, - 0x8bd44428a3b0cc8b, 0xbe7cead795d16dde, 0xae6ad5a1f1a63a14, 0x9cd2d25e9b86e245, - 0xbea173a04e918892, 0x8ef9e9aca9151c7c, 0x7703940a6d0416cf, 0x0530351f5656447e, - 0x55a7761e78f757cf, 0x21523a3af82472da, 0xe0464a72643d3561, 0x3ec19098aea1acec, - 0x7022f407893c1e09, 0x06d47170ba660013, 0xb03ef6f367bc0a1c, 0x9afa7e5f3723c6a8, - 0x3693f31dec3c6503, 0x61da062779dc24c7, 0x40f46b048c6e18f7, 0xf80496de46cfc92c, - 0xc826e25c17bd4af5, 0x71f62131df10f47a, 0x3ee5902a456b5872, 0xb861d90a571fb997, - 0x6f2bf9f500513def, 0x30cc22f1be87a8fe, 0x040652e6176dc08a, 0xdd474e12ee614d45, - 0x4bb625ac7e0072ca, 0x9c97b75fdf73952e, 0xb2432b3849ff6452, 0x27b819c8a0bbb820, - 0x29012df96e6b71c7, 0x1b93822e39cb3542, 0x4fa971141efcd09d, 0xbe498e4a455777bf, - 0xd41757cf32472a2d, 0x523626f8a1c26b2d, 0x18921b9df745480c, 0x8bee769eba287457, - 0xcb27d05b0b352e39, 0xb1c3d655d7f11ffa, 0x9b2f3ea4d84d19de, 0x6dc61172e8af0e56, - 0x4356445d5413a47e, 0xc8ca751b61fce681, 0x28ec0280a2fce37b, 0x9dba07c74b45b615, - 0x45156b6db2928bc6, 0x12efc082a5c86e12, 0x583fce5489074bc3, 0x203b7cc8d33d983b, - 0x346121c682649c27, 0x76fbf3c0b5b675b2, 0x802e9924d7d45438, 0x3a32b66d726839dc, - 0x2edb4ad1f793f886, 0xb5b4ca71a5053caa, 0xded14dcab8b9e7ed, 0xb0d889b189ed0ef9, - 0x92a4c1606279ac8f, 0x37f69b445b37c897, 0x773898e36ec19c51, 0x2ae2e23ea8d5afee, - 0x7f022a9f834e535c, 0x2bc273111502b755, 0x4be2b6180cbd87e3, 0x34e1045804870c8e, - 0x98dcf58e30eb5463, 0xefa785c718ba2aca, 0x2f55f4418904a8db, 0x00a6ea0afca95b4c, - 0xc63d852930812175, 0xdefd6b5ff3a57c1a, 0x69349d4d1368d59b, 0xa589f7c541f1a189, - 0xda661a7e37dc1f36, 0x3a844d87a50f77d9, 0x4e2aaa3c258e5eb1, 0xcbf59291136311a9, - 0x9bf035cf21318e81, 0x8ae1fee150cecdcd, 0xd25cb7f69d2e6afc, 0x7b42f7a2ca434695, - 0x0c619019f458c37f, 0x574f6d982f5a8858, 0x3e31a283aa4be987, 0x94584af5c4214dc7, - 0x36381d95c5a35b86, 0x179c81ac4524d405, 0xe3d07e84b57eb8da, 0xb150a8283e862816, - 0x6d2fb7e46f4d1366, 0x581fe96536be719f, 0x34c53e9df4fb4113, 0x797586c3ad4562c5, - 0x536be3480fcb5f05, 0xbb7cecd08e9b4253, 0xc67819c60fd14bae, 0xa8f6561b80fafd13, - 0xf8598d9996740fe8, 0xaf1d4af39cff83bd, 0x882532cf59669be9, 0x36e54d04713113c1, - 0x3f3af394a1142c38, 0xadb98c08a17a5de4, 0xc067884114add3c1, 0x5e4c1634ecbc83dc, - 0x7dc574028603991a, 0xd01f12e15ba77a6e, 0x248d2117f5150bc4, 0x9acc2cfe09139956, - 0x68c00775a0314212, 0x20cb68d03b3e1011, 0xb1ae3d3993c8aee6, 0xe1659e5e2834396a, - 0xaf8d28e389021c6d, 0x35fc4d30244e2053, 0xf7811386199e0efb, 0x42a003970b8d3efd, - 0x5c9bb0791c0c51a0, 0x42e9e42c3a96709d, 0x749d1980b24648b5, 0xab63b32a4bfb8b4d, - 0x8d21126c23e57c2b, 0xbefa700750b1ee0a, 0xfcb3a2ac78898891, 0xba08bf5840f3522d, - 0x37930c44e5938de5, 0x6be163407fcabaaa, 0xdc6cc486dc44a915, 0x7be4a191962ca11d, - 0xfeb5963673afe083, 0x5d03f23fd0a5181b, 0x366ad0985d74fd4e, 0xc47a6faadc060059, - 0x78c9ef306c6b5f6d, 0x60f1cb18e74120e9, 0x1f1232ec6e36bc19, 0xdadee09bc0857e11, - 0x886a659de662d2d5, 0x76ea0e8215387a72, 0x270866a5e759853f, 0xec2f8e0cdf1c52c4, - 0xdb957fe531eefd6c, 0x1c4cef001432f7eb, 0x2db63f558f2ab646, 0xcee2727199a74ad7, - 0x908f1fe1be97adc6, 0x0039b476c22d0b34, 0x79ef96620bb04ad7, 0x601f73eec945226a, - 0x75bc5b1c397d17c4, 0x6dcc36bbfd684827, 0xdb6fa85c2b027345, 0x87c56b2baff3df4b, - 0x811d88d9efd75355, 0xfcfc1c1e0849c17c, 0x0fa3e4adaaf81f9d, 0x5c432fb7ee778f37, - 0xbc23d9625983e39e, 0x356089d7dea5675d, 0x895ea85d94c6b249, 0x347449bc949a6ded, - 0x460e3e4af7013d7c, 0xd6f79548bdcb19f7, 0xc6b41cc483a5c1f5, 0x42c6e10799029743, - 0x3657222ac4d5251e, 0xebcaf6b8b38eabf1, 0x980e92eaa8082f31, 0xdfc1d11acef82d97, - 0xbe80c7ba84f5f2cb, 0x3c8523444f1d4419, 0xd4a5901c06160900, 0xf2fd072325fbf418, - 0xb7552e36f39f4147, 0x11cf0a2a7760567f, 0x3198a6418f1425e4, 0x2caf84291274bc0d, - 0xe57aaa42a1436353, 0x0d1f9ba0881960fd, 0xbbaf2df2d692e5e0, 0x5ab2c251192652ce, - 0x7b20a122a443e293, 0x96598365f8a2fbb3, 0xa305ab86b49e3c3b, 0x04884c0f4028aa5b, - 0x89f491d63c4fecda, 0xbc502942e8081415, 0x83ea7a3ce05d18fc, 0x11a163249fab7a72, - 0x112d1cee61e5318d, 0x9e7a093cb612d7f7, 0xdb0264fe1be213c8, 0x3a616071d417950b, - 0xc14651674a6630ad, 0x72e68619e7bc788a, 0x178dffaef8dbe8e5, 0x1444931895f231a4, - 0x1d1810dfc47ab56e, 0x3c3916216e3887a2, 0xa28f2a29606b7825, 0xbbc2f403f39b034c, - 0xbf1d4d13c05f3024, 0x9282d719a9a7f17c, 0x5be7b8ee6eba3e3f, 0x6d0cffc8d789a36f, - 0xff6e5fc3496a65c3, 0x1f01d2c1ebab6ed1, 0x02a92eb674a551e0, 0xf5d607a11bfbae84, - 0x27bcc52900c8125e, 0xd128ae9be721d4eb, 0x9a990d8bc088d989, 0xf3cd14eb9c2693aa, - 0xaffe63eaf21db2ee, 0xe62ef4715a1e203c, 0xb68f67d1294da274, 0xc827ff060bd31224, - 0x607b5a4973af2aca, 0x2c5e1a8b0444c202, 0x1722a565d069a21c, 0xbbf2ddb67a7c6a5a, - 0xb5a8eb205e255128, 0xbf4ed6ee45ce6bd3, 0xea7054634a877ca3, 0x68f1fd386d50e277, - 0xfb26b5713f484a47, 0xe44aa02589a4be16, 0xe6e8a6477f730028, 0x31a49b99b889f50f, - 0x8f6b8f39c1e6ca22, 0xfb02cd89bc28609c, 0x63e17473a9fbf162, 0xd0a72354c5bfc2fa, - 0xfa727684a47a88b1, 0x68da0503e8590112, 0x12f38f0babf64ecd, 0xba99d47442ad8741, - 0xb8dc041d8fdc2b41, 0xa5f6947056b49d61, 0x644681f60dc4021e, 0x34ed5cde691424e8, - 0xc72bf567e52ae403, 0xb0f67ef58df60f5f, 0x0a827bac8c3e862f, 0x39dc49a2b2f42e28, - 0x8ae217f8410308bd, 0x452b5a3f4df88b0f, 0x430f3c1dce6e177a, 0xfa51a42fd5a6d789, - 0x6e5f2afff5f1d257, 0x3c0737c44539fc07, 0x67e72e1cbbdf718f, 0x2ac9d3c84173ce2a, - 0xc0df86e6470da6fc, 0xc810a322c4d8c662, 0x17bb018cdcdc4bbd, 0xc3ab0c624b650319, - 0xdfc3adbd0e6e072c, 0x16729f78f1f62f12, 0x7fe24963a2c4730c, 0x78e48aaf6c4f7b58, - 0x3988d6b4951e8858, 0x7ccbf72642c06a4f, 0x6586fa1255e544e7, 0x61298d3c386ee71e, - 0xb82d0d5facc951b4, 0x0d885d2241292c7a, 0xc3f2a683b76006e1, 0xa373fb96a7febfae, - 0xa550c922be54ff6e, 0x9ef8759c70574dd7, 0xdfcf8ca38f6d40e0, 0x9fdd3c671315f594, - 0x6cead69ae1238cde, 0xfa3a4508465f8b16, 0x4a1a128bd545c3c1, 0x673bf884dd3f6991, - 0xc33cb99eaa67749f, 0xad815e616c52c4f1, 0x32360428bcdb35ad, 0xae915bbf8fdaa611, - 0xfec3ff4d6284b53f, 0xbb038b2b60ccfb5a, 0xe8fb28de4ba0d116, 0xf39a06efa9f275d7, - 0x38c3e2c52bffca35, 0x65e60616323121e7, 0x3d406d524b78bccf, 0x507c0330774bd0e7, - 0x6dbe6a02b4a57bfb, 0xc3f8bf3af89b1525, 0x80ff15c53e327172, 0x76d2a6fa659d042f, - 0xa9ab3383a2f794ee, 0x81112970abd546ed, 0x81e209e9e200ee84, 0x65c9cb20da4f7126, - 0xe32a41034dd6d241, 0xc973e20d592a3fe0, 0x71eafc732b1fcfbb, 0x3f6bfc164e0a7d8f, - 0xefd2798ca89df45d, 0x5b5535f6f29192fd, 0xff35124e65e93af6, 0x9e2d74c162498a97, - 0xf92eb1c1b427a70d, 0x10e8ca21fc7b927a, 0xdaf221031ea41aae, 0xff6af20c56ce0961, - 0x61f134b36a87b97c, 0xce772761bd4daff6, 0x124bbcf04ed56de2, 0xed4082483c18fa57, - 0xf4fab53a52c2c77b, 0x6040dafe70c7c149, 0xe2f9a7d31dd15a65, 0x8879a81d8ddff120, - 0x34d386cde2bdc6b2, 0xaae1402a02bc64ba, 0x0bb9aef985cb1df2, 0x84ee60d82c5a8cb7, - 0xc671d33db0f15f08, 0x35d097f0e8cdd9bc, 0x049bd2759a8002ae, 0xd634a3d2f6acab25, - 0x64ecf0be7b3b3b63, 0x1f23156f16a798ac, 0x5b33ab2701fa7cf6, 0x8c2d3f24c48fcf08, - 0x7ac15fabffdd4c85, 0xa3685f57cd088a4b, 0x65604388be2ec68b, 0xc9b656446f781b15, - 0x9609211701611bf3, 0x8a00148e9ddd6c60, 0x17243a04f64a574a, 0x1d06e5192cfb5155, - 0x23530648d38b5c1c, 0xdc680bc5f4d5ce7d, 0x6b2057306daedc19, 0xfacea2129a1dbe65, - 0xd9edfac05757c699, 0x9597e5290bc9cf0c, 0xd45f1c389d7d4a68, 0x3a8168bf5c24bac9, - 0x1d8987040a891f08, 0x78da3a18478641ae, 0x505cf0582b1df9c6, 0xc109120c8dc7a277, - 0xd2b829f701b192d0, 0xdedc1f5b18478c13, 0xc0dc9b35328517b9, 0xf57b53e08ecb9a80, - 0x23f2e72dbcf41418, 0x7c2c7501fa1b0759, 0xf6b780a85cfed843, 0x111c5c642993ac2f, - 0x11604a9c9dc9c293, 0xfcc32e0e41ff9852, 0xd352c549219bd47c, 0xa8d0dd877ccf98e8, - 0xc4cc66cf00bf4c17, 0x26c309d8c3e788e6, 0xa0098b965ee0aa7b, 0xeb3a30f355728ea7, - 0xf4938284f136b702, 0x25099f1272843603, 0xa664ce435e14038b, 0x381bb3866f5150a5, - 0x39d629dc082590f6, 0x2836ba46c66bb4ad, 0xceb214e850e4ea64, 0xe48036ea3041bc62, - 0x48126646e9701da5, 0x03fcd4b18e5bd998, 0x7fb1262e091477e7, 0x8b3dcf7216b7fa4d, - 0xfabe3fd8f47e9b0e, 0xcbbbe5301fb00a6d, 0x5744e5c66486f32d, 0x04489ede39a8e231, - 0xba61564258672916, 0x0328f9f1e736e2a7, 0x75b09f2418d87ab3, 0x97e85b98cf406e8e, - 0xa59de15997979cf1, 0xd9f24958952c986b, 0x15fc3a1fac92c5ea, 0x31d68ef6f5ad241d, - 0xd97982c709589cec, 0x4f5045e5b2c84704, 0xe54811b322f6321e, 0x711da6047c85112b, - 0x313315e842fa593c, 0xec3d985cff07e762, 0x6994dc0e3dd78dc4, 0xbc9b2dfc0232443a, - 0x99dd46ed39c07d6c, 0xe0c1001690ca7a8e, 0xa117f38e178666bc, 0xdaf9003ebe5a5e44, - 0x728f89ace6ef98fb, 0x068023b8a7ba1fa2, 0xc4e2b3522c5f2e8e, 0x40d5e0bca4ea34ea, - 0x034cc9fb2708b080, 0xd53454e8369ba3d3, 0x6b82ef1502d917df, 0x220753d0530f7877, - 0xdaf1d162696bcb90, 0x7032edee8ea6aed4, 0x8cec33d3fb85bbf6, 0xb0f6f2a3bfcc4a01, - 0x36fc4538b6eba5f7, 0x082b8813d467d91a, 0x55617dca197bb4bc, 0x6219075aec1b5e10, - 0x83247da4d7c91444, 0x8bcdf605d4dfe8d2, 0xf659006120c57e8b, 0xb602840c48c2a29f, - 0x436d6d528ebc98a0, 0xeeec330ad6122438, 0x11cdd0f92e5ea03c, 0x06f80872a9df0544, - 0xead35b186b700328, 0x8d52540a8e65bbd5, 0x95aace2409869caa, 0xfad3a3ec20048cb5, - 0xad333c7252f7cc71, 0x36bb03f340c44872, 0x2a148b6062170c61, 0x21b0f4c1356d5c0e, - 0xd84305411199c017, 0x464afb3395347f9e, 0x6dd693c51eb182a1, 0x775ac58542a276e7, - 0x20971594cf8de26f, 0x71bbffaeb0f3f042, 0x65f7c54a2fc0346b, 0x5aed5bc8679a895c, - 0xe18773e416c9df2f, 0xdd9cea6c8964c35b, 0xee0d2f4a412deeea, 0x5ca868f95594338f, - 0xf9f426a07a8679b9, 0xf36957725e9c53aa, 0xed8cb503dddeb6ee, 0x306d12054aaaa608, - 0x8de3f0666f2c9baa, 0xd61f2ec1b94964a3, 0x73449d5b8808e1ad, 0xc45b5423f0653260, - 0xf3e85d462518bd75, 0x284c2d5849a27e7f, 0xe7271e78c92aab81, 0xe80f69cc31528559, - 0x3038787e093d53c8, 0xe67dfbdc6169287a, 0xf1fe50748f614390, 0x422d24aa8a3bdf3c, - 0x4e8ff618dc55c68d, 0xb88dc2acdd118428, 0x96d1050cf1badc44, 0x284a5e41e2b68b6e, - 0xe39065957804c753, 0x7b5aa004bb709165, 0x444836942d26d62c, 0x515beaebfc3ae606, - 0xbb77364301adabca, 0x4134ab404d545d41, 0x016497b070eee340, 0x0e9b6b637a9a6690, - 0x954fac12e71a88db, 0xbb77bb024d521d86, 0xa1118a21c3ea1f6f, 0xb3945124694de3a1, - 0xa53f48accbb0ac75, 0x9e53cc1e0fc665eb, 0x04752aea2f03141d, 0xe2c81ce4a6f99078, - 0x767b21f7d387bd7f, 0x74e34a760658769c, 0x1ccac6c03c8fa2c4, 0xcee524f5bc7b391d, - 0xfb4111502e3595cd, 0x05576d1d3c1b4c82, 0x8b03029012119e66, 0xd28822a7093dcb7c, - 0xcd82bfe2ee0d7f0f, 0x18ed7363fbf73bc4, 0x9438ce0aa3e4d0a9, 0x308618e834e9e0ab, - 0x912b93cd6a2035e6, 0x6bb2ccee9bc2c489, 0x2f5a2bb33dd0e157, 0x10dd76f0126effcc, - 0x002b04d4639d2197, 0x71c6e39ca0175ec4, 0x1761ddca96410b15, 0x367dc51a39ff1ada, - 0x3dc4f6c2747a4fe1, 0x277667b78a649d78, 0x37a1d00ba343bdab, 0x29377d384be15bd4, - 0xe79b4d3867b685ca, 0x56c90bffbee643d6, 0x7525f208f34e3eb6, 0x0375539a86d8ea89, - 0xe27aa0474b9d7ab6, 0x767175d2947d3311, 0xda72fbe570fa6d96, 0x0ebefaab9aac0f2e, - 0x83ab0a032b5e716c, 0xf65f38ab24a5805a, 0xcee5470ae184684e, 0xf7a885f3c21045c9, - 0x2d8466d39982ba9a, 0xbde9120c864b25bb, 0xa561a9f50ee08b22, 0x4600e3c72779bf2a, - 0x33230446aa3854b4, 0xc25e397e58495557, 0x8e043b773f9e88fd, 0xe2f59e854c3b6c29, - 0xbeae22fd68085c9b, 0x358ad74d95b939eb, 0x6cd6239dd5bc4e4d, 0x278ad4c22bbb7ed0, - 0xa9caecfb86f67c90, 0x9e2116dd016635ba, 0xe4b69bf2e4db4d95, 0xaf13105a367719fa, - 0x16b55864a69ae171, 0xa4e783d3097a9532, 0x867204d6815be317, 0x1fc95302f15a2f9d, - 0x527f975eef50668e, 0xc92c02c6dba67771, 0xfb2c4cfce985ccf2, 0x97beb7f89e212fe4, - 0x9120236a5d645fc5, 0x087f23174065afa9, 0xee551075a4140901, 0xfa6854fa25d4ff2b, - 0x3e33844d53c159ae, 0xa69eb7c14ccbed57, 0x1bd696b9389c314d, 0xa26130d5fd650225, - 0x0f3bb5ddfc8249ab, 0xa39388a60a872abb, 0xab0481883ecedd2e, 0x48978ae9fd6d6d8a, - 0x71b454a019f8bc95, 0xd22b7df2e04587d4, 0x2ba5616a77f1cc2e, 0xf949181ae92fb1e9, - 0x5bf10977534fbf43, 0x284713719c17d419, 0xda93bedcaeb5714e, 0xb124c64f545560dd, - 0xb9d6b8495ac19488, 0x5c2ba005a9332395, 0x3718b22dc2c8e3d7, 0xdebe421c13b66a08, - 0x980d3820cfb24c3c, 0x29b578ca69c577a6, 0x2d9729ad18aa3934, 0xbd4c88e5514cba87, - 0x0c76a74e83270905, 0x5e59c7a679709ab3, 0x9c35dde44d246a24, 0xd5de8d9aa129f3c5, - 0xa3ec60bc4c46a7df, 0x5bd9c8db6e6bda43, 0xf385bf78fc911a65, 0x96e7c50407335cd2, - 0x36b8e6178895fc92, 0x97f44e36fce9f39c, 0xd0fc253652a4647c, 0x11c8d8fc5e2b221f, - 0x3f93cfc2ee897ce9, 0x7e75aee2d5f774d3, 0xa648a2df457d494a, 0x73961d270a40a3fd, - 0xe4996304e1907f66, 0x1427d1b3d99bb75d, 0x8b014aa0d1540406, 0xd3d69926d35647d9, - 0xfa22093a0323765b, 0x45cf74f035eda47a, 0x2bc0eb5a92ebfc48, 0x5b8a6c57fcdc0ad4, - 0x334da2a777b2dcba, 0x7b2a1c8c556200c9, 0x1b5c7354de61b5f0, 0xd07f2c17b0d10d24, - 0xc975bf2abefbbce5, 0xfb71ec0de4eb9379, 0x03c03283e29942d5, 0x07a0c9f62f5b4989, - 0xa32776985d902bbf, 0x90ed6f3048c1f4e3, 0x42b9f9c658decbb4, 0x6fc55e0c175429b1, - 0x3bb7898960abb9dd, 0x33d926a628ce0deb, 0xe05c2e00ed1150e2, 0x5d2c528040e0e30e, - 0x6d15a5f03efeff76, 0x090c6ee678b0d374, 0xe313d45152cc8113, 0x3e764fc2999d0364, - 0xaaae0487ff581f39, 0x8b52dc9b473d9c4e, 0x8eb8744cf3ad1be0, 0xa696d348c15f9a08, - 0x0a23d09bba9842cb, 0xbbd1a9ba14c9308e, 0x8b61b9442292ac65, 0x46d459e65e2518c4, - 0x9d4893063d76bfa6, 0x50a0c234f060e91b, 0x7e77a03666491d30, 0x7d70d32e34132244, - 0xadab72219505d0ef, 0xd1722156c586ef35, 0xe0a87bd791c32090, 0x765794a6952de117, - 0xae8f97f5a835ffc9, 0xeafdf5145e107884, 0x27c86318aa7ba1a0, 0x1b0536fd42926745, - 0xc70e56f45a9aed13, 0x215364d78dc0adda, 0x795cb646752b72e0, 0x09c11a43b6697cff, - 0xf3cb4cfb69ae5108, 0xce6231e09a8f4977, 0x4f257d51b836088a, 0x4c1c485ffed1d5ec, - 0x2d36c6469db8660d, 0x7892335e0ec62383, 0xa4acc1a38f10b6b2, 0x3ebadf7d97b1e06b, - 0x569589fa8e278d36, 0x81f1262ef0cb6087, 0x7132bdbf9c734ee7, 0x4188b0aa38abf80f, - 0xfa90291a2b68b21f, 0x4bc614add1694223, 0x3e052bd548d3753f, 0xea9485d95e9f325c, - 0x51b363e85294ffc1, 0xdf76f6101234846d, 0xb7d46ce4db8e10f2, 0x9a9a23c053f559e7, - 0x79a93ae5d0a6dfdd, 0x31f9b64a65ed99ff, 0x659e14f91ae75c09, 0xb42fbbcdf7175c71, - 0xeb8f43dce46b849c, 0x60f63197d82c0a58, 0x76fa236c56501ea0, 0x667d49ae28dc3702, - 0xf858419d5c621abd, 0xee42168afc1c8e14, 0x04f541a9a7078adc, 0x373257c8be20ff92, - 0xcbdd21cec53ae15d, 0x5cc4f178817d14bd, 0x735185b898d42ec0, 0xf8538c3bd4b48f29, - 0x4ff7e931933e473a, 0xbab8032790b898f0, 0xc0fd9ce521961e4e, 0x05a4e9d76d7865b2, - 0x3dd8c2ded4a70288, 0x3ee53e796548aed6, 0x9e100223969d4adc, 0x7f8853a64077d02a, - 0xe05822d725e41343, 0xeb74d45af6ec17a0, 0xddbc4fe9af3f9cf7, 0x9b958fa7aa573128, - 0x0a6f41c37a0515ae, 0x3fb56560be64d75c, 0x51b9c42ddc151ff6, 0xe377ae8ab80bc45e, - 0x80f01690d6119bef, 0x1f626c08897dbca3, 0x196e2f2efecd6b40, 0x49d26bc4b9dd0799, - 0x2f322ca24d887faa, 0xaadf24aaa0361295, 0x2d0c05ce7793241b, 0x3f451a79540a5fb5, - 0x4ed5f746adc8780f, 0x6a0f7fab948c1d95, 0x5e56be4f7c5cc470, 0x924cc0d6025ce42b, - 0x6809eb6a7bb5d94b, 0x094f446d84e6ab24, 0x3d1bb05295a106d4, 0x3c14f180dde4ba33, - 0x5cb8d84bacaa8ed6, 0x15e14644a099cb4c, 0x84d1bfe4e320e1d9, 0x4c49857689ee186e, - 0x4c8bf94d43013fd5, 0x7a232c3aed100723, 0xcb348f572ef09283, 0xdcb3dac477e70234, - 0x472ae9ae14bf41a6, 0xc5421ecf18539cec, 0xace00d70765787c0, 0x405a1bfd6f7504bf, - 0x805bb6663638aa80, 0x902e12244fdbe97d, 0x513737b2a12982ce, 0xfb69613834342be3, - 0x83685a4028ed7ef0, 0xe89675dba086f929, 0x8e6b05b66d460668, 0x8313f0d9009e099a, - 0xaebeefb0d96dd52b, 0x20cc3c5b8d6e1354, 0xa0d9dbd63830771f, 0x7e8182888c94864a, - 0xdc8b503a7fc36f77, 0xbca1672548d26488, 0x35c43e16f28b357e, 0x2be989b14e1bf90f, - 0xe14fb805b853f9f9, 0xd2c204ab3e1bb67f, 0x0d4dc8a8d827c49a, 0xed8c67127beb2a51, - 0xb34d1828909f6d6e, 0xa8a19888ba503612, 0xdaf5ed8e81ae56ee, 0xd7e774fa74756c2e, - 0x6f8fcb4b6e5bf24e, 0x34bbc8cc36bfc5bb, 0xeb88a054c17628b2, 0x0ecf053fa582f641, - 0x9d618d0492d0492e, 0x910511bdb29e16a3, 0x1e15564e50afd095, 0xf2ff25414b37fc87, - 0xf76f3f63384c92c1, 0x8d7b5b9f6a8f36c2, 0x110a7a89a1d0e85d, 0x4b7ad5ff6073ea67, - 0x377a7c5fe6a13ee6, 0xfc6ef1961257817f, 0xa6a5801e0a26b70b, 0xa069edd56b43e346, - 0xfca3f9127b10830a, 0xad540a4f37d96c53, 0xba71d2ef9baa3fc1, 0xfe47bd05889cfa30, - 0x400c27bbb49d4efb, 0x2f6d33f68be6bc18, 0xd5fe0bcb92a68c0f, 0x85763d41184fd4b7, - 0x3eb3f537d7169da2, 0xf24dd4da8362226d, 0xad39a8373c2b047b, 0x1a914947818d51c5, - 0x9e4cf47494e7e219, 0x68e8dbd631dca0bd, 0x65f4260385afda51, 0xd7fde31c3bae0c70, - 0x09e15e852b6294e9, 0xb56fb360e1d56f95, 0x00d63e13254fdd80, 0x81d9ab96492740e7, - 0xd6de50cb204790c1, 0x42fcc23ddc89e1f7, 0xba1f0e16a0cbd46b, 0x19f2367995541b7c, - 0x00d489423015ddef, 0x6b69961803218aef, 0x73a9d119c961e0df, 0xfbb164e28757a9b7, - 0x88bbc65e242bca94, 0x46c352a98ec37e5b, 0x18ed2227c5538b3a, 0x42e7f6f78bb4097e, - 0x697b7d9680ace2a4, 0x1cf211535b77fd32, 0x681870447a476039, 0xe38265f88bc66a1f, - 0x5c897711af0c64db, 0x267aef382fbadf85, 0x0f908732e297183c, 0x4e0d5f3d6130c259, - 0xc577f90b4d87e142, 0x8722e96c5e9734cb, 0xb27f5e6bc644f54a, 0xf7fa3a21b6159ce8, - 0x227cc23ba6431d48, 0x17de80785a206c11, 0xedea21f2a6638048, 0xbb1c17c6934291d8, - 0x7ebcc63e90804846, 0x39c61a3647ee332f, 0x406c6eba3c3a7ed1, 0x6497884e828124a3, - 0xaea665ab4ed930cb, 0xe7e2d21c9f97c742, 0xab6db030434745df, 0x3fae35126b9abb00, - 0x9f6dd814ab4dbb1f, 0xed10492b737ea1cd, 0x2acb280f03d2775e, 0xd8ec0f034c2b8ca0, - 0x7678ab2c7d11497e, 0x2d464dbd9dd3db12, 0xc630c9201f694773, 0x40024150586f801d, - 0xdc8766c6600108cc, 0x2dad1cccf0600bcb, 0x4ea4e29913689fdb, 0x30a875aa94d0a82c, - 0x0be9100b0f92b8d7, 0x30b2451879623fde, 0xb4b7c47cfd80e13d, 0xf8b2a49c074213ce, - 0x3855d9eeafb5edc1, 0xb660aade5260fb22, 0xc27eb4a2d6d00e37, 0x4aa55dbb5ca4ada2, - 0x2705e64f1edbfcef, 0xf0a80db047c0a903, 0x7c6eb467887265df, 0x72843cea6254eebd, - 0x6fc175a4f41aa822, 0x61851995167b5151, 0x36b6cdeb06fe575f, 0x30f894399f952792, - 0x5a51c071714c642d, 0xa3f0ddabc6e841ed, 0x0ae333da6f7c9138, 0x9149d23a17569341, - 0x1f251b037485bcd0, 0x4a7657876de9c548, 0x2f7c840f497e3a4a, 0xe82d53ae4d668b60, - 0xfbc3b69fad4835ac, 0xec2e4d2333e00798, 0x07d59a3c4c05b3b0, 0xe8beb15d4738fd5b, - 0x85e7d47e09fe3fa2, 0x52d47528276efd0c, 0x316e2ce5ed754cf7, 0x548e858349624e29, - 0xccfa6bbe9ddf9aae, 0x75abd6c57c9d2616, 0x77691a1135204180, 0x7683a4cadb2ea38d, - 0x844387838ae5eace, 0x13017165c9dae4cb, 0xe44226060001d9a2, 0x73f63072200780e4, - 0xb92088b818c7bf3b, 0xc294e7e2ca8074e0, 0x9240748d08f9872a, 0x238967186fa747ac, - 0x0e36621827c344d0, 0x57db097b7138027c, 0xbadc243432fd330d, 0xe648ad40b5cffec9, - 0x5a520eb09c258f68, 0xe9340dfa993e999e, 0xc85338b6b745221d, 0x7a9f0939951e4ad0, - 0x915672c66b84279a, 0x0d7f00e101218fbd, 0x235a264f592f330c, 0xae2ab17a7bbdd156, - 0x8db1be2781eee902, 0x8bda463863d599e4, 0x7a6583e5ac6d5b3d, 0x79fb85d75a6f931f, - 0x6cfc167c61bd90bc, 0x4c59fe228c5e3b70, 0xd136a27e57adcc78, 0x1383bdfd7c856712, - 0xb1b096f5eae05f14, 0xde0df2f5f09b48c2, 0x4fbe190af4a0aff3, 0x404a9dcfa2f3132c, - 0x440bd69ba7f0e941, 0x4112a4e869d3c9f1, 0x5a3afadadf27c0b2, 0xb1fd487973e49718, - 0x1f091ac1ce68af60, 0x25cf30c9ccb135bb, 0x4535df22d73df4af, 0xb256dcc40f763679, - 0xf67b98153b8e9a1a, 0x2e62156fc501d0e4, 0x72b0958c8568f329, 0xcf38352e91ee877c, - 0x9efa70568358057f, 0x8eb1e0ddbf7cce1b, 0x6d3b1fe4f548ba5c, 0x41495bd8bb341615, - 0xa65fe13cc97bc853, 0x62439e25392efbc5, 0x4ad5fdb7d43d0d15, 0x1b83166bd34d2769, - 0x0ccef53fb6a986c3, 0x38f2a61ddc85d349, 0xedc61eb711c6072d, 0x7b68e2b92d2e4d28, - 0xa0c5a4b1cdeba45c, 0xc9c968ccce86bb4c, 0xf35cca9fb837419e, 0x718359452490e96a, - 0xa6b9968d80e3ed0a, 0x89de1801606ea608, 0x7d93ff455cc2ea7d, 0xe1693de6b307ea11, - 0xdf362ef9ed73d130, 0xa5de2330f27f6ff8, 0x0e2f5c9c3b13d7f2, 0x80449279c65cfc5c, - 0x64a79a1fc44289a0, 0x6560595e26b579a2, 0xa4a808b185f7c128, 0x17396ed3ea098d73, - 0xb7ed276eb35ae0b8, 0x564718adddd8fbd3, 0xef296967d40b4583, 0x262396b76f5f644c, - 0xfb22ab9fd0feccb6, 0x90a0e1c08ee0fa68, 0x41f7111a9becc78b, 0x4bc161a92f8a117c, - 0x0d8e50bad529e8be, 0xc7f55ecebe2e13d6, 0x94c42b2186fb5ba0, 0x64a8fb924af7c62a, - 0x75942bd9c47a69b1, 0x45d86924972ddad2, 0x83547a55c0c467bc, 0xe8d8dc0e1bc3c651, - 0x7f3460368e1ad64b, 0x07b24f48518cb541, 0x521c349fb375424b, 0x5292a8f8264bf26d, - 0x29f2133bd56a1710, 0x37e3ebcad4b16220, 0x1a57301801eba217, 0x6c77f1cd37b538dd, - 0xa42259505043aeaa, 0xd7270b9351894d7f, 0x63b781e7e42923a2, 0x7dfaf8840228f6b7, - 0x61dcf785709f5e3f, 0x873a4102953a2777, 0xae40b79bd103fffc, 0xe3b7ba0ab26310b9, - 0x195f1396d39aff4f, 0x7c41298ae5d4355f, 0x46b4d307d48e7953, 0x2c0134482de25522, - 0xb43da8f953be6915, 0x3e9dd2036d89341e, 0x4cc87b4af27e4ec1, 0x75d0db76a7ed5eeb, - 0x9df1701f40009719, 0xfcea3623d01180b5, 0x399dece6854e75a0, 0x0522431059dd69f9, - 0x2b1ffca63dbdb6c5, 0xeac6e1dbe353c735, 0xca61bc69e37f9c4a, 0xa01831c7ab5bf58f, - 0x5bbda493fc993e50, 0xe3007acb4a6d352f, 0x13dbc2cc1fd724c0, 0xef73262aa33d8f90, - 0x565b95d6c6b40fd3, 0x5dc876e3cd9754f7, 0x8f0e047406dfae20, 0x55e5121f21e9c3f9, - 0xe0cb3653aecd6ee4, 0x00cc909cf73a984b, 0x4d80bd1eae46a736, 0x9b7c029eee32cbf8, - 0xb02cd90913b8487f, 0x72743ba806b5b0b5, 0x07286b4b1cb6c3da, 0x943a5bac7bdb0ba6, - 0xfc2598e864e7acaf, 0xe6563063d7c12123, 0xc4c625acffa95a37, 0x5819af45a0685275, - 0x6fec595eabb64ea6, 0xcb34b837cd27ef8c, 0x791011e292a40e5d, 0xbc16d5943fe1a61d, - 0xbc599cedfe3dba2f, 0x0df3a6fd38df2412, 0x569834ae6e3046db, 0x3fc46428416b08b6, - 0xf329551a6da4e7ab, 0x452833bb8731d89c, 0x48fdafda082428d1, 0x7f6c1638e04e154b, - 0x3ce8fdb5d1926d20, 0xcb2869d1e5de87d0, 0xb13be96625cbdc36, 0x360b06c432bee06e, - 0xd6402441ea36f55c, 0x0de7b632b1b7c70a, 0x79144b010a12381e, 0x868c0d8c9e0cb274, - 0xc684860efbb22d0f, 0xbbc60427ef6d296a, 0xbe65f1de4aad12db, 0x02855670cb7c83c6, - 0x49d0b4a47d1bb24c, 0xf117b7a92665d597, 0x6de90bdf04543ac7, 0x65595bdd6faa67d8, - 0x3f5720203378f263, 0x1a739498384b86ef, 0x0684f7b2b0967b8f, 0xd6e15c859611aefe, - 0x3407665905fecc1a, 0x066ddf47b2789fd8, 0xb670d5d5cce56b3f, 0x883ec957316fd9b5, - 0xaa0a4492592e173b, 0x53f41f7f29ac947a, 0xa118d464268a4845, 0xf63b0b5242e5bfd0, - 0x1268f0876d50d3ea, 0x2ac4f9520ce3f3d1, 0x32145025c9c8da33, 0x28cfc4aebd45cf23, - 0x964238de1f31605c, 0x361ad207ee7c32c0, 0x135045472c1be0bc, 0x17d3c78ae101a9e7, - 0xfeea0ded84fe33b6, 0xeb9ecbd11a9e9896, 0x9842c91a51078d8f, 0x5e8a353b3da7e567, - 0x9b7f21b72efe275a, 0x3f38f0472e3892c5, 0x8dfcfdd74564469f, 0xefd39d0ed9171e79, - 0xc80c012dcc99e88c, 0xc10e9c55fff07496, 0xc601b9f44494916f, 0x52e6cc3ef6173f69, - 0xfd97fe9ba2f183da, 0x4e5270f600037b81, 0xbea7b02cd9d4f958, 0xcd98e614be43d9b0, - 0xfb12816c86131076, 0x21382f96bbb867ce, 0xd28ea5970c3bfb7e, 0xf32a92ebe79d7132, - 0x338dc97cdbb6ed81, 0x3aada4613fb2cc7b, 0x96605d49dfa62ba3, 0xaeaed2410ef20d36, - 0x44bb7dfac637c61e, 0x7b4dd0586e58e0db, 0x0e83a6c3eac7f526, 0x7ba75acd2ae8f6cb, - 0xfbc4530ad38ecfbc, 0xa6d3cdf51656bdba, 0x5cd3e96cee8cb6cf, 0xc182ad53b196f315, - 0x27fcbff52d71f5ff, 0x55ce6a5abd61df73, 0xd7f0cadf60014ca7, 0x2297c2086fe8b642, - 0x6a5f1f16418dad54, 0x5c9714f72d215f25, 0xabdab8b3ba505f03, 0xf4f89266c51b6f3a, - 0x99dc9d598c509f6c, 0x4ddf4bdfe6b236fa, 0xd26eaf393a07bd2a, 0x40e5c27cd072c49c, - 0x8bd73a4bca41219e, 0x11a787c2bc1b4c4d, 0xa08161782bcafe1f, 0x188a5377287bb3c7, - 0xe50b84098a490453, 0xd7d49ce0d2bc2b7e, 0xfb7a218d1234555c, 0x0a81bd1769058caa, - 0x2af67cbeb59c7e42, 0xec15ccc03a35001b, 0x26ed94af73f3df30, 0x888fdc168701800c, - 0xca7efe780148f689, 0xbacd5898d54daaa7, 0xc2e3b65616551198, 0x1d424ef36bc4a5c9, - 0x81974c4d38e27566, 0x6f0d0a91274fcf64, 0x44ded70ed19963d0, 0x55ebe2e535a21c91, - 0xd68514174a225c3b, 0x6fba9e0e080bdb2c, 0x56ec7bdaaf17539d, 0x6b982b145c41a56d, - 0xfdf4a16d50d1604c, 0x5a3c3fef97c17968, 0x7e78dcebc3d1a9ea, 0xb3402a4fe7dc1521, - 0xde3bcfe1f6a88d32, 0xe504e4b43ea13532, 0x27518379519e0f43, 0xba295cc50cff4b44, - 0x958e3d05f21956ff, 0xf4666ae5c5421ff7, 0x8f51fbbaf8fbc1ca, 0x699f6371cc51e3af, - 0x208eaa1b896cc758, 0x40a4d27bb49a4a45, 0xe509aaec4f570916, 0xa1add92316c872e9, - 0x280092bee075ab3b, 0x706cb36ad58b7405, 0xaeeeb3e5059fda64, 0x636d6d71b548ca8c, - 0xafd776d27b0958a0, 0x3c4ad3a65dcc785d, 0xd664e9dcc4e93847, 0x546e850f1c6b3bf5, - 0x83acbe62b55ad789, 0xd5853f8222d22104, 0x2b009aad0d75905f, 0xc88e223b63e83406, - 0x64459fd59600011f, 0xb05bf3ad705d8b92, 0xcc5f8c655da127cd, 0xddd328aa252c7181, - 0xc49e18012b8ecb76, 0xcd4d592e8d58b944, 0x16604d9cfadd3511, 0x62b38dcc5f066b75, - 0x99435fdbdeabd4e5, 0x16db9465a7a9f605, 0xaae1023e416d5966, 0xe49a9167391df0b5, - 0xa1c3887b9257deca, 0x06751b5391cd13f0, 0xf83954f9e2ecaac9, 0x51764b2262c0d404, - 0xb53b1489da49ec2f, 0x503483959ca1eaef, 0x0ab7f037a8798c65, 0xc941596e67310997, - 0x2a2ea7d65e950f17, 0x0582a190a8bb84ae, 0xfca8d225fb88362d, 0xaf40046830e8518a, - 0xdcfa77593505fbb6, 0xb421604e1336bce7, 0x448451e84fc65cce, 0x4041ce7b056b25aa, - 0x14a6534b7cc74919, 0x2f91d4a9e462246d, 0x928d78c14997222a, 0x20108cdd35b41171, - 0xd23395dd1662160d, 0x5958d3651fc0ef7b, 0x857c673ea63fd8b2, 0x6fd7528c3ac0abb7, - 0x2080dde6118a10ca, 0x8dcb9171a942ae44, 0x10f7ab722592fccc, 0x6a67ef771154588f, - 0xf0328663bcb0f6f0, 0xe17f2644bf5879e6, 0xa47bab498862a61b, 0x1c850170e66f96d8, - 0x7aff13c20fa91597, 0xe7394ea5ab0b500a, 0x348dc7793493b9d7, 0x39185b41d4e7a358, - 0xaf46c4e9d881095f, 0x3d823680804ced20, 0x92b9b0ab8ea29f20, 0xda47d1829d7668d5, - 0xd1599c3f13af2b25, 0x808b109bd70ed006, 0x7ac11ef869343fc6, 0xfbcd6b17571571c6, - 0xd0c0b92ddb002927, 0xc8817b03990aaaff, 0x2ada4417188674d1, 0xa05ad40022868353, - 0xa52966d2928f67cd, 0x9b87d8042f9067cc, 0xea6b630987773514, 0xab030b812cb8f4c8, - 0x6ebb15ba6d83dc68, 0xd2082096909efa60, 0xff9c89aba8c96073, 0x0786d99ab8b70d5e, - 0x10fded6bb87bf992, 0x226bcd5cfbb3c848, 0xe6ccc5dd19484196, 0xb405740e63ef0d60, - 0xb22849b22ff6a285, 0xd669b7c04da4f423, 0xb2c6505c5787988e, 0xe8b79abbbb858233, - 0xeb54debc9afe99bc, 0xd68754476b34b63a, 0x443083341fc40d5c, 0xe1f7dc35f7a216a7, - 0xac24e7d31570bef3, 0xb135a68dfe265e74, 0x5ffbd3deca3b3e74, 0x4c1b68a928b669bd, - 0x3c2d9003d9816651, 0x4b9a26d8cc1ddaa5, 0x8a569b58675f8061, 0x7a952bb8274deea8, - 0x7c1be4a1932a7ab9, 0x6cf4f6896e1ae35f, 0x22f21412bb9201f7, 0xb2c0fe56e3be263d, - 0x44cf08fda88c8c35, 0xf48ba7cf0b79d648, 0x2a0403abe57f8515, 0xbc397d0def136411, - 0x83bd3eee55b8bfcb, 0x140ce02c4f5670fd, 0x7d912b9bec2eed00, 0x0e190f06f12a7c30, - 0xdaf8a900a7a98a3a, 0xa6956207566ce87e, 0x88691fb6410a5841, 0xdc8d4aa9ec1ea548, - 0xd39e85d301fe6d97, 0x9c8419ae16bbc71f, 0xab7dcdc60d388a93, 0x4dc8f97604248d66, - 0x7433cdebfe640c16, 0xafb69d95e8994424, 0xa2774b5f00c97aa9, 0x1d78fcaa5819e55e, - 0x3a71649dbcd36a0e, 0xa8ed3835424ca467, 0x8ef5e8059487d851, 0x71a888792777cffc, - 0x42238e163378543d, 0x9cfaab57036813ae, 0xfabbd8063379b8b4, 0x5ab7ed84195a537a, - 0x0c98dd2e5c9a3c9d, 0x036ad63a02f931e0, 0x1a53a8926ae3fa5f, 0xefddad35f430f5a7, - 0x7342366d5a15e3b2, 0xf7bb11f4c6286b35, 0x85c4d86c6ab9d775, 0x791319a190b4e90b, - 0x27081da84e2fa544, 0x77ef0be4a10a3c7b, 0xd3bffa533968002a, 0x1befaaa33d4c7d63, - 0x75acc0f01c95cc29, 0xb55299014149a9fe, 0x48d3cbfcac8b0478, 0x6927ba295eeceafe, - 0x5362df06eb960c1b, 0xdc5e1ae76b7bdcef, 0xe4f0b1d708dcdeee, 0xe4f2159331b05e00, - 0x93f928098af5ebdf, 0x94553e32eec78ed9, 0x9a15e822cc48b42e, 0xa0beead4d3d6ce02, - 0x2422eebb7c19b2b2, 0x0ee6259f6b2de3ad, 0x6fcddf1b26bead94, 0xd9a0825484ade174, - 0x751aa874089f4964, 0x0c6c2801b9d0703c, 0x62979b8de23bf3a9, 0xe11c80a8e4d78d45, - 0x83c188c2cbff4ea9, 0x4eff922682ec26c8, 0x11bac6d25b281d7f, 0x60d606a90f1c53ec, - 0xffe4048d78f0d8de, 0xd49c40147272a197, 0xf4dbe5f18e22f4d7, 0x1703be685ca08624, - 0x7663a78c381f34db, 0x88a0709ec1add91a, 0x247e6d8027b93d9c, 0xa23b47fa9381a330, - 0x08f1b30fa359bee6, 0xa550a3c144300385, 0xe769ca32443eb826, 0x36d8af4532aadd15, - 0x915ca3f94a2da07e, 0x5d09f8f4888210c0, 0xb3ead867ebab8ec3, 0x87f30bc384e49041, - 0x8f88cfc0f066eaa5, 0x82a8a43c5a02d4f5, 0x0ad45d7c823d359e, 0x83ca220fc6ce6e6b, - 0x2cc135378e98f6f2, 0x9e6fe6a3ec72e91e, 0xd0f98eefaceade1f, 0x5f80b16d9c0f40bc, - 0x50102d02b483dd00, 0x70263250cc89fdc1, 0x88baec9b8decd192, 0x2c5557d1004259c3, - 0x99c710153fdc5d47, 0xf3ccd27585fe1aa4, 0x9d0c6386a3d802ae, 0x3aaf3266af0729b5, - 0x1d509ce186cc80ea, 0x80cd61a91122327e, 0x58908698a4abc525, 0xb0e785e95b987aeb, - 0xb91ebbf1dfd53347, 0xa36af62c0aac52e1, 0xa360736e08eb5472, 0x08f0f470aebb4472, - 0x68d851e9c1cfde28, 0x88b56565fa1619a5, 0x2111319eb4dabff4, 0xe390aff741b113ce, - 0x53a69f348c4d61aa, 0x899cb31a7814aeee, 0xfa9f4dd122fe2f58, 0x368151697d058f52, - 0x4ed91d69b3196e71, 0xce31c9a4570ac75a, 0xb6b41a335fa8b35c, 0x0b09340c59ae4345, - 0x82ebd25812c14e71, 0x482063a32f61418e, 0x68a3b1baedf8e148, 0x43636f2cd24e2ae0, - 0xd0634b6db577e32c, 0xc365ec9cc73731d0, 0x6526d4b3e9d34a52, 0x49a54587a8525b5e, - 0x6e9f036a54d3dfa8, 0x88ee0f139f953fb7, 0x787878701af37bca, 0x16c500c3ee39ed20, - 0x179afb3fe1acd1a4, 0x7d38730623e0500e, 0xa045143f765624b2, 0x9052d2a14bd506c7, - 0x721e3b9a685bda2e, 0x833dd53b0d1b7f4a, 0x93ea0720c26668f6, 0x8c991671001c4603, - 0x011b756cbda56f63, 0x8ee9a9f3cd32a3c9, 0xa4582d06c3d0da4f, 0xe3f9b7502bd0b86c, - 0x20820bdb3247ae1e, 0xc8c26c035eaba8c2, 0x4f46e323bc76626b, 0xee132e79f106e86b, - 0x6448a694ecc318a7, 0xe3443138d617c6fb, 0x84890ac1807eb3d0, 0xd69bf2d7b38ea61f, - 0xaa5856edc9e31a61, 0x81d5760d2e605c5e, 0x4d9dbb22d274f310, 0x7d39d5056698f828, - 0x1aa90eee61c38f60, 0x056085098104a68a, 0x0cadf40ba2c2312e, 0xbac6656ff63258cb, - 0xacd0e771c41d6729, 0x96a24375f6aaa1fc, 0x3038cb7e075a7f83, 0xab970b5da0ac867f, - 0xc76d55eef495ccce, 0xc0dfc4c6b1861232, 0x814ae212bda82a65, 0x1ed19c1af1e46b04, - 0x053dacbf58cc7b3b, 0xc3fadbe9ba704853, 0x26e912710435d118, 0x648062476983f040, - 0x238041bc8aa1bad6, 0x020fdf0a1780e8b9, 0xb74aae4eb456813e, 0x11e4289c005472b1, - 0xe15a400c6935a7af, 0xc5ac17434ac36548, 0xa90f9c905da686a1, 0x75e3f8bb5bd26776, - 0x8d79c00064f39030, 0xccff9ee408c27523, 0x0b0614aadff25751, 0x9fa420604f7b360e, - 0x609362db416b9623, 0x4968594c9b437401, 0xab2f4383a7c230a4, 0xc73024b6fd58b91a, - 0x2c3aee006137e1dc, 0x164e956a94d1c2cb, 0x060b6ffdf864166c, 0xe376ac0c3b811715, - 0x03c1bb39f23a92cd, 0x9f4a61cb6a8fb912, 0xde3958efb2424782, 0xf253766450e452a9, - 0xa1478b6653ae55f1, 0x1a05267fc33a9448, 0x328006de4733c750, 0xd3e8a45e276e7efd, - 0xfaf9a4ebaf777b59, 0x1c66a26848babbd7, 0x84fa35a85f704cd3, 0x3a6b1545e2cc56eb, - 0xb1c16ae612cb283e, 0x840d882b1bfa9eb4, 0xc51aa8d583872502, 0x456a5977e3acc9f0, - 0xb021d23bf3a2ea77, 0xfc480546f279a784, 0xe86d06b7a23a3bfa, 0x612e7eb71d96dfb4, - 0x93d4aaab8c2cf2c2, 0xfc165ea58879ec8f, 0x15fe6cceed76dd67, 0x46081490fda876a4, - 0xfd3d91f3e6ba2d4c, 0xaf7a5e5beb73ef56, 0xda9a339f21fcd01e, 0x6aac3daa36d4fca4, - 0xdc357f40a2c144ba, 0x6630f8ab67b52802, 0x134162f37024d561, 0x500b82a18091ccee, - 0x4dfadcbc733b5528, 0x9092d3db69d04552, 0xeccb389f5291a63a, 0xc261ab69d83e3bda, - 0xe69b5249311b4f7f, 0x99a9a34582cbb473, 0xc29cd844a15fe400, 0x97e71b560d77a425, - 0x0c1b231b57d9f201, 0x8ee5130e97542777, 0x7e9ce13e1d657b8e, 0xb3ff7ae4dc8dbd78, - 0x8a44ef3c19e1c616, 0x3006f055dc2f7230, 0x1aec071abb5f9372, 0x52a1b15f49df5831, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xfe0c0696af16ea60, 0x67e04e815aca19e5, 0x506e57de98503ade, 0x4982dee131f9309f, - 0x70622bf624c3a9fd, 0x373aba1de3f256b4, 0x7c5e8e1d2586fb41, 0xf2a2d3d383553c98, - 0xd72194082b8395ff, 0x0a2a747b61822e7b, 0x86dfbc1f6cee0e84, 0x54967faba65ff882, - 0xc3c62a22c42b1af5, 0x67506e9d0801b684, 0x2e6f290d53f05c10, 0x39240cf81d08b6bf, - 0x0a9d729277f7aa85, 0x2f5a9a374a41d036, 0x35737e7507542476, 0x14339c981e467701, - 0x2a3dff199f5e5fd7, 0x8d34032b8f7bc7bf, 0xdd745bf0340cddc0, 0x2cb206e831b837e3, - 0x956b8bff8eebf03a, 0x6dc4efb04d210bd1, 0x66599ddebee240cd, 0xe3bb7cb63df3562a, - 0x7ebf2109fda12027, 0x4895bc04eb2981a1, 0xe10c3f08a0503d38, 0xba6173b6bdccf122, - 0x666fb23e95b1190e, 0xc12cca9357c0f953, 0xdf2aa3d99cdb3821, 0x0af8422626850564, - 0x744dca7fef5de19e, 0x65870b797db48569, 0x9a2bd6e0b607efcd, 0x77b4a295df0fbe90, - 0x1e6b66b72b140c33, 0x26d224415fad3c7c, 0x40a028d359132838, 0xa83580bb2053c6ec, - 0xcecaec85efcd1de2, 0xe4deab1d810d4c4a, 0xeda4173ee0aca18e, 0xec8f719d4a39ae8a, - 0x9a194fa318b37662, 0x2b9fae9022158dfa, 0x002ee4e9841eac9b, 0x91088fc017298eb9, - 0xd789b82ed1393a6d, 0x25c00ceac7b9cc74, 0xe983eeab217e7cc4, 0x21f9f830d35858ca, - 0xfa4163d0fcecff6e, 0x5bcd213783de4890, 0xe1b47de54433efeb, 0x63b134b9c0f7625c, - 0x2c65ff9f5f41d91c, 0x943d14762edc5d96, 0xc807eca0203b3b66, 0xc42707329a57157e, - 0x52c918477e5bda41, 0x5baa32aa66a6ea8c, 0x056f3957820bf1f4, 0x45d8eb2627e5d2e6, - 0xb13ba52280332544, 0xefeb08ca7bd150dd, 0x1b7bab22dcd8965a, 0xb739c7122c0eb041, - 0x603542f73bb5a723, 0xc26c1ef66acd27f3, 0x01d89539a0254f10, 0xb37b39a0a93736e2, - 0xc434b77d1aa789cb, 0x071bc9d76627a54e, 0xd6b093b50a1a7d3a, 0x4f640d96bebe32f0, - 0xcc80ea0aec4fb721, 0x2f9b2e77b5999dc4, 0x8d0ecac074f06cdb, 0xe2f8bc656303fd87, - 0x414f6108a5bfdc4b, 0x2045f401498d233f, 0x59c2b3ce689c83cf, 0x63764f4b5efd4f42, - 0x016ecae6a3ee8d07, 0xea91c9ad87c17fd9, 0x5ed09ea73cc9b066, 0x4543c8cac5bd1bff, - 0x00efb5805777b3a2, 0xe152cfec419e9cc6, 0x3b72b71796713ea9, 0xc5f1a8b85bffdeed, - 0xdf7b719b3432b2da, 0xbec9b99071f6c9c4, 0x543ad0c354c21cc1, 0xcac5e9e1ba40e5f0, - 0x5d18660e82ebb52b, 0x2923ff54a443a934, 0x29e9dccf094f1f17, 0x53f9506ae0e8722e, - 0xbcdc48af90c65d0e, 0x70eb49ca8196864b, 0x299d6b2db0bed9fc, 0x9ff63417f21d4dbb, - 0x41cc4df470e341fa, 0xadaf80b9072da0fe, 0x30380882cdff5a2d, 0x203a724ef2cf3d11, - 0xf1b6247a39ce9b19, 0x5f5738a8a1d03e87, 0x90b0e16a84a5a6b0, 0x6a93d44f8801b84f, - 0xedde095eaf3dac7e, 0xbdfd17c9fa86c480, 0x0dc54dbbfb346182, 0x489b77d9d2f1471a, - 0x138a4d1a8c366226, 0x147b1c72102e0468, 0xaefe97255fea946c, 0xac66b96111baae83, - 0x5d9f2078e454286e, 0x8f8e053574d650f5, 0x90d24265e998a42e, 0x8fb6390b553579e6, - 0xc3696cfeded4005e, 0xab0b34410b7f035f, 0xac6c20c5436de821, 0x1ba147b95e0738e8, - 0x324a13024109c931, 0x367b54cbf159471e, 0x9656390469b5e34b, 0x162b67fc16ed9e9a, - 0x37bebe2e7bc7c3a3, 0x114afff4ce3025f6, 0xea5f04430b758140, 0x107475143357ba51, - 0x6990c64e57aa3a34, 0x6b361d5a6b9a7e41, 0x36fd8ccaaebac07e, 0xd22dea4f43bb2e2b, - 0x1fab5d8f2e7c460a, 0x90af3880cc57cabb, 0xef45bc4fed879ba0, 0x8dadaaecaf0b221a, - 0x1359a307e82157e9, 0xcb61fc76e547369e, 0x077044b9ebb11201, 0x77e14e6957b4c579, - 0xe5c903b01076455f, 0x531a64536a27c158, 0x047299b0bc695b21, 0x99a1225bc40063d5, - 0x6d8aedb1447125ba, 0x59a58bb6a5e83ba1, 0xe949e803a53775c9, 0xafe45d48836dab4f, - 0xe541e7b0f0aaf595, 0x33213d8443bd1143, 0xbe57d5655fc42d47, 0x0fca52426da3d296, - 0x42a25d6900f13979, 0x700a881ff48b62bf, 0x53549cade71eee8a, 0xd1cad80c0fcab540, - 0x1a28d60d63d8ea7b, 0xd93225138a1ceea8, 0x1f7a24a8f4c89093, 0xd43f17969e8ecbcf, - 0xeaaabe2d593c7bca, 0x8362d51f678717f4, 0x94eb5ba74a8aad9c, 0x62e9c960ba5773df, - 0x9361ffca9d5a565b, 0xf63b61944e4d9ddf, 0xebb6a8eca1f09ff2, 0xac2fef4ea59d792b, - 0x978feede090b4c3c, 0xc74348511404a2d3, 0xea78c5381b123ffa, 0xe07ba98e73373de8, - 0xf3fd19855ac4b090, 0xc9ea33fab40886dd, 0x033f76c4850f8cb6, 0x59ac9b2ef26ef1f1, - 0xd62cb5bee4db5685, 0x096614019fabe137, 0x023affd0041a82ee, 0x862aa976d8492839, - 0xc7d57c7c95974ff8, 0x063d7855b9c3490d, 0x7c5ae6afdd4e4e5d, 0xafdc17284d046195, - 0xfe8e762126513944, 0x23418cc2b191f9dd, 0x305d322d13054598, 0x928e8bced326018c, - 0x0e38481c20820c38, 0x507e8ea0362eb4d5, 0xf5ab717338b562d2, 0xfbedc27652c590cf, - 0x1f86588785cb8c14, 0x4b722bcc82cc0750, 0xb8993e3ff6c28648, 0x4a2a4299a4b26e81, - 0xb07ca4ada7046cec, 0xc3912167eb55e4b3, 0x6a14688e77e98af4, 0x8316ae85764ea338, - 0x86063ac6cea03907, 0x0867b176232604fa, 0x32456e3111248d47, 0x995bd47cd007bda3, - 0x9dfe41bf19cbab52, 0x4ea048d071fb4ecc, 0x8e451b99aaeb1962, 0xdebb87cf5792246c, - 0x657c5602d5d25c08, 0xe5eaadce21d0950a, 0x2535cf0348ba1f3b, 0xe084c652c4087199, - 0xc94b742108204a06, 0xc9d3d43ecd77f658, 0x11d611f60ce03ab2, 0x52cb41a05ec710b5, - 0x37d1edc7593097e1, 0x734f6bf6f24917eb, 0x1b75851482767a06, 0x51d7f44a22d52d16, - 0xc1bde8949b96ba7d, 0x5fe1818d8a80b7cd, 0x8bcbb20169333b26, 0x11f1f099bb28b15a, - 0xeb78ae13f00d980a, 0x317bf57b15093687, 0xab79d9a29116da02, 0xe88a80db454e4902, - 0xf4c6dbdb51eb0974, 0xb9de939f6000728a, 0xdb2c76550bf03d02, 0xdd02f4a481c24279, - 0x4495fe97fed9b1fe, 0x86ba5c9931909c65, 0xacfb55b66b6384a1, 0x896791d0a7a867ec, - 0xc4aa32f333b60dcf, 0x722f5f7cbb787c45, 0x1b8131da4436b955, 0x1c089511b522c565, - 0xd5094aae96729ec4, 0x6a43885794ca9742, 0x1ac3296c092eb33f, 0x85458fe8c224ba7e, - 0x173e27a1cb03219e, 0x19fe007729b89f54, 0x477ecc86c687dfc8, 0x738c5f9cf472d72d, - 0x40e124e10f49431d, 0xfaed40385cb78a3f, 0xe8f00ea3670d99da, 0x6378917e3dc37f12, - 0x429dc9197c616f03, 0xd26e8203d5a1aa49, 0xcd3cd57fbed18d2d, 0x4b116a91c46ef560, - 0xcc8ac8337e2de47b, 0xdafa613df0716a55, 0x9b035859df238e4f, 0xe799c1a41cd4e632, - 0x0c63141330d17ddf, 0xecdd13ef33a43ad0, 0xd99517e048cc7191, 0x3b56c619543c7012, - 0x177609fa53833e7d, 0x2cb391c61566638c, 0x73daa4adf78e0bcd, 0xd36e6e6458a1d3e9, - 0x77dcbc040605bb54, 0x354fe0358cd70c25, 0x6ecda5d3acc41b2f, 0xf9592022ce846a3d, - 0x3657752289770572, 0xfdefd35b68a30d34, 0xf3b02449888c1a86, 0x0936006c73c13210, - 0x212df8849a1933d1, 0x81406ee8542bf598, 0xd68d13d48e05c3bf, 0x9a0c6e5505b988d5, - 0x2dd184de5977ef43, 0xf8fa939abbe611b9, 0x9ff27d64d4784752, 0xcfaea34b86ac2f52, - 0x19b3af7140cfbcbe, 0x550c22ae24d277e4, 0x2e5d3d785b0afdef, 0xd45630eadd8a5fa8, - 0x832e04eaba267614, 0x4105957c9d20029e, 0xac4bec67eed573a7, 0x629c6456d93e91a5, - 0x17f56c680137c2d2, 0x915b27fcfd99ccf7, 0x7691bfba3d05abcc, 0x94c2ef3f0dd0c512, - 0x1053e8087efeee7c, 0x91b9e6196919f5b4, 0xdc04db132ee1a10f, 0x0731207ab585b4c5, - 0xa5878b0032944593, 0x0f23bbe0b483f2d5, 0x0e0abd6f1332efce, 0x2ddddf3a5ab1119a, - 0x6e69540a9500e647, 0x962f6456902b9b8c, 0x69054b8c34f555b0, 0x9e31574a90c78d8d, - 0xbe627136ff79a922, 0x4fe2dc09443071ba, 0x01a325e72c8a7ac5, 0x2ce819c80a79233f, - 0x78d8789f30d5d0c3, 0xc916bffd5aee7cb1, 0xe91a792af49f55f7, 0xedef20ec6e3f1e9d, - 0xacb01bc1dd4d4b34, 0x2627c73c78950bb2, 0x6cddddfbad727c78, 0x4533bec4100020b9, - 0x4d74af2abfb1c4d6, 0x2a90095eb6de0fcb, 0xfa08a9149fb49ead, 0xd548b3fa0f65f7cc, - 0x17f76d10fdfb2446, 0x4f6db589371230c5, 0x35fd96d0da4bb9ba, 0xe53616eb5bea577f, - 0x0c26bb429cbe9955, 0xadee469a1819289f, 0x5fba77ffd2b926f3, 0x2b2974ae7f275565, - 0x87152d4f145ef8f7, 0xc6d0ad47f3fde1b8, 0xb8621a333fbe6997, 0x1e70238096457ff8, - 0x939034c45d644ce5, 0x95e9b3b26fc10a78, 0x29a62fdb70395689, 0x9b9f480ecde3061e, - 0xb1c0125b53478f56, 0x714a8c3504b5d6cf, 0xd3124546314e9ab6, 0xe80291a7112762c5, - 0xf7c0422f4b7466ff, 0xcd593c68d79ea308, 0xe99cdc3925f857c5, 0x92087b3bcf1cc288, - 0xe14422c453945d9a, 0x79955436452e920b, 0x0db7ff75a64cbe10, 0x98969ba27fa4e17b, - 0x48e6bd608c79d0ee, 0xc4c69bce1fb7c836, 0x374bd53f85919f2d, 0xcc3494698b17fc22, - 0x83f4278fb35710d4, 0xf8abacdb77bf0a18, 0xb0c51de557728489, 0xe2659e5957ad2e83, - 0x708b26e76e91e799, 0xe80fc50c3a6f0245, 0xbc01c0ff6f22f4f8, 0xe1eda51ad0290eb0, - 0x0be4505db5e8d56e, 0x4b7d8a274d9475aa, 0x0e1df4606ba3f943, 0x8c743a37694180ca, - 0x95079fd1ec2e10bd, 0xf2f2ca5db9bd2d39, 0x6b451a304bfc8dbb, 0x61c0a15a7d616855, - 0x00dd985bb4f40e25, 0xecadc9ebd8890197, 0xd64d70a4f025a226, 0x2593844e4eae2383, - 0xea3fe387aef67ba6, 0x452fc56796c54160, 0x365463a8c04539d8, 0x0a9fd0ff080b1ef4, - 0xf0ef7f2b9f43a18d, 0xca41212f026ab8fc, 0x355684e7dffc4691, 0xaeb4eced4d208c1c, - 0xdba01f95ea794bc1, 0x15babfe5995b3525, 0x41a6c9c5f0b7d1d3, 0xc58f9c7d59c3ddd5, - 0x6a65ae80ee64947a, 0x4c23157fd1e283fd, 0x323549f40628033d, 0xef17418741dd2927, - 0xfd30096a9607e30d, 0xefe59e4354e8bad7, 0xf84c3a07557eb186, 0xb4a411b0fb841e4e, - 0xc3f28ef770e5ac4d, 0xb12f40696f8411d2, 0x6d53e23993b8a4a3, 0x84653b833df78b96, - 0xd7abd89069755f14, 0x4d7838e0077a7809, 0xb6b20b74c0842371, 0x6374af05240b49ad, - 0xaf45dc3f4a5cae62, 0x0454910a1a4e6c7b, 0x0cc531a42cc46f8d, 0x68e48ef638c20088, - 0x3d333c89a510ae8f, 0x5fa18d5eeb5045c2, 0x25e2817614aed3d3, 0x9d1e548498f4b4b3, - 0xbdf881a57a2fb458, 0x9709f0079b6375ad, 0x214563e52d0a2364, 0x513b821731adde18, - 0xe0657230d1bd78b1, 0x7d24bd04f9b5def3, 0xb19cde2239faaedb, 0xf28703cb65ab1009, - 0x12540e5681d57a15, 0xd2974fb7aa022b75, 0x3ef73adba37fc448, 0x6d6c71feaacc0948, - 0x062bba6136cd4f70, 0xebbba420b6dcfbae, 0xd9e7d204862be4a7, 0x6825a3fec58ff5db, - 0x4e0ba82cfb9fb6c9, 0x02f96a868aebbae3, 0x9cb467e2472d84dd, 0x9866166dc2c62970, - 0xca2ff48598759b63, 0xc1767f6e6c1f2400, 0xa60cd34743c05674, 0xd1101c43e82d91e3, - 0x06fbc95e3f24b934, 0xc8571d1769d79ddd, 0x7a48c74afe904ecb, 0x9910ad4f081413cf, - 0x892fba786df4d3fc, 0x57f3edd3edc540e2, 0xc27603685713e10e, 0x8fe0aab3f28f3ba0, - 0x14f453694c66c751, 0x80a0c9d4dba726a3, 0x046b37222f30a97e, 0xa7661709777b5ea6, - 0x71edc593736e88fe, 0xd26d67c6cd5fe6e6, 0xfb4e127b67e5c5f0, 0xba4d18984f28af8a, - 0x1c17dd1f9a942c93, 0x44c0be90c46994af, 0xd66f2b96ce390660, 0x26dc6df8911f91bb, - 0x2764f42ebf2e167c, 0xa405053c256f6216, 0x823bc2b7abec3971, 0x3dd61660702c15e2, - 0xaa0d3551c99b7df1, 0x060ae0da77d9e908, 0x24631f2a4dc507e5, 0xf4522a6ad69996f9, - 0x91784be06318cfda, 0xab71c8a1da86ce2c, 0x35604568d285f9d1, 0x1b25b1f75beb25a6, - 0x182fa1c7349178fb, 0xab69dd014a4385b3, 0x0ce77d5173e35147, 0xed6f6b9d66844d93, - 0x1d53b577be90628b, 0xaa3588338e1f0430, 0x03468a18bc1c2745, 0x6181eca9396138cd, - 0x589c73fe52ef21bb, 0xa5a8c3b6c72e36be, 0x6fd89cd44bc12fd8, 0xdd6fc86a5438c55d, - 0x1d75c3f15f4182fd, 0x99a1f954a2b2556c, 0xbca5296415bec8af, 0x2b50e4a4b74959be, - 0x4eb1aa8bcc66c4ec, 0xcfc8082b1c79fa84, 0xf2fb076c571ff104, 0x9f2dcfeaa40f88eb, - 0xcc4f8ac80089e216, 0x619c09be6089adbf, 0x9159dd05b75d1ba9, 0xfb0bf317ca2d662d, - 0xf0a8650a3c7cbe7f, 0x2e446ebb1cacfd05, 0x87249af16fc48312, 0xf38b302e163cab1a, - 0x3fa3000cbc94061d, 0x8054e8821056f182, 0xe956a56868a855b4, 0xcd2c13108730a5c8, - 0x99e43d6960d3bbdf, 0xef9848a17a6e8486, 0x5dc58dabee7ac7c3, 0x372393a414c1a497, - 0xcebbe1301729d06a, 0x2d82486ccffb68c7, 0x508486894f1b735f, 0xb10e55f560398ca0, - 0x0711aa19741156e9, 0x4148dafa30e5a84f, 0x86398a6aa5a8811d, 0xcaaef75a0e1588ee, - 0x55b1c3bfac3642d2, 0xe7ca19336e472e9c, 0x575f36de0ff72504, 0x0ea430d32b430249, - 0xc7ac9fcb03f51ee9, 0x9b21d2c0394e3600, 0xda8f3ea9cf822c97, 0xd758e8694b7a28ef, - 0x0f0afac6e7a8f840, 0x788c9d587a294800, 0xcb9cfd2ea371f939, 0x4a4c82051e9bc24c, - 0x55f291d37babb6f9, 0x9c56d1871ac83c35, 0xde0ef84a22d76c35, 0x99ecc1a6bfc1d241, - 0x9576e0bcd7bc84e4, 0xe9d894b013802475, 0xb371b02ea7382c25, 0xc078413506d09428, - 0x682d6b0c3f7f6ab3, 0x5fe30fd81c442baa, 0xbcdbdd8aa4afc6a6, 0xe12e92fe2b610969, - 0x0d95365ebe905e4a, 0x582decf9aaa4a2a0, 0x09613d823822c216, 0x9f2e75dcbeff816a, - 0x066beec86c0b252f, 0xe8b1c7c36f947916, 0xe972cb4c6a36f197, 0x9ff5bbbc0bcc4ba2, - 0x532c9663f7a9d741, 0xdd614ddb81aa9985, 0xce53907af37525d0, 0xf41c4a87ca653248, - 0x10c0380c0f9cd596, 0x2af623c7cf82ca22, 0x097c10728d0d4d4d, 0xe10c8575ed045d7c, - 0x1725b39fff6934fe, 0xc4dca568230d8d57, 0xb62f747b80ab4eea, 0xcc726630f6338e42, - 0x3fb197787da1705f, 0xe4ca65318a172fde, 0x86bbc89817212b28, 0xc0f0da1e1e8d2db7, - 0x16b2bfd9d4ae4b3a, 0x51b1ac596f804c0e, 0xfe4f70ee06caef22, 0x863954bac19efe7b, - 0xc8210e8b89f2a0c1, 0xb5fada191df32438, 0x2769ec6bc93037e6, 0x9b69d5000bf6e91d, - 0x91ad5d84e67e8f83, 0x2a7b76c699823bec, 0x9f251f2402c8613b, 0x01eb0dab3ca9a765, - 0x10defb0ed6f6a61c, 0x20547d3ae8d8f161, 0x209de984d3741c27, 0xf70df6615d11a6c6, - 0x5c574943e8bd92f4, 0x678828e6bbd5b04a, 0xfbbbeda95325b573, 0x44b2dd2a5d4e8272, - 0x705f0b479911259a, 0x775b1c34a3732e4d, 0x2a4d5ce696d861c5, 0xebf8945742dbbaad, - 0x4e8fcd6c7afa3a22, 0xd1569ed2de95ed38, 0x092126729e703f8d, 0xc8a178da320d0930, - 0x397a8eb8c272ddf9, 0x13b66902415dd6cc, 0x1955c9b1f51a8f81, 0x90b80b8022936e41, - 0x28927478e543712e, 0x83a58f53faf8b9d3, 0x80357065146c0e51, 0xc99dd68c97975917, - 0x8eaea0b78835389e, 0xe5083a63323150df, 0xbf08d905976b4976, 0x3fc6309b1092841a, - 0xda0718f54d715ceb, 0xd6eebbd808b8cf40, 0x87fe5793183a6ddd, 0xcd71162f58f4be90, - 0xbf1838ed40510a9a, 0x7fae6e7bd4feab24, 0x98d6ce71bd140761, 0x110724f62745cdd0, - 0x88e9e8e1b3522865, 0x0dcc639a015cbd1a, 0x5d4b30549bda2077, 0xdc48930f94b63f33, - 0xe2872bdc01f84efe, 0x222286d5c23d4321, 0x358e67a979d9ca7e, 0xc0887cb6d1c8a11a, - 0x582e0875359cb353, 0x48bea9a53fe11c2a, 0xa09a11aaa167123c, 0xf47b7d58a013fc6f, - 0x9b383a7e90d0d662, 0x20f5b0f910eb2bbc, 0x7deed4083e4f4eba, 0x6b4870849e7f292f, - 0x6a1d957c7c08e6ed, 0x737ae55672417c64, 0x42d218e8433375e4, 0xca8febf1cbe0f3d1, - 0xffbfc549366d89d2, 0xc70b8b88e6431522, 0x09d13a7fb8d6e63a, 0xa2135ca8b603555d, - 0xaef2b5b11d1ca07e, 0x9f718178f04b005b, 0xec6a7b1ebc75de70, 0x0a1724943e035d3c, - 0xef69900720b0f1aa, 0x070cc778bff41bd0, 0x9ec404ef6a026b89, 0x2542c310b30b8c65, - 0x94da2bbb622edca5, 0xb3178aa93ecf9e7c, 0x95b8bcf3c2f44439, 0x3cda672a42117d9d, - 0xd580d4ffecec500a, 0xe5bd43dc0e565dfb, 0x1675905804049488, 0x33af3666564f7a39, - 0x0ba335bc84efaef0, 0x797025f44ba7dcdd, 0x8e80b3ad682b501f, 0x6ec0774045055669, - 0x48fe08b9004b0a9d, 0x2704889588e7a8b4, 0x058f9fd38c8c2a51, 0x204f6619de89a7f0, - 0xf719c254158b655b, 0x9e2527b7697ca746, 0x61f57f54717d67bf, 0xc8796acf55cd415f, - 0xa6b25b685e370808, 0xa60ad5602cdd2aea, 0xdd7b16ff5d2517fd, 0x891648def546de90, - 0x9d40ce3c2f88c23b, 0x907d685764c2a79f, 0x5a1f9d94b5f0ad75, 0xca6bc9b0f04e233c, - 0xdef83a1f6e001b6b, 0xff0374579cf9cdc9, 0x37af4e09c3f84f96, 0x4d6cf2fe1bf4aa53, - 0xb01be863d99f03ec, 0x542761d4d4c2c741, 0x60711f5c46fa16fe, 0x99dfa17f2f2a7fad, - 0x6df2989376bc665b, 0x08caadb30cd74c7a, 0xc61018cd2bd8357e, 0xad024cb4c53d1afc, - 0xca389d437c82a550, 0x34bc11507c6384f5, 0xa4a1299caa112381, 0xc148a9dd1e06a3f9, - 0xe50b4421e7cfd8f5, 0xb27eb772f98b90f9, 0x73e663b3749888b3, 0x1879773525d4bd9e, - 0x619e5a17fd2ae475, 0x907a2d2874db9bca, 0xea59efb9876f5e97, 0xc299b41c4690299f, - 0x6b6a37deffda097c, 0x10d44f94e305813e, 0x2bfcc4adfcf6f59b, 0x4aff8b8609f59822, - 0x53000ee16b2a5bba, 0xfa8dddc34439d1b1, 0xa613a2499ddbe560, 0x49e381006024435c, - 0xb0e4ba1e77f2e40e, 0x23f7c4c13ca15860, 0x2d898324268d9a1c, 0x0d3e493be0516160, - 0xbfaf2a45f441d2d9, 0x1d46df51386f06bd, 0x4c2ce95016ac6694, 0x11ce62ed7ed9a1d8, - 0xfb31116ef5eb7133, 0xd02d61a61fb86f22, 0xe648a605f4aaef76, 0xe08d0ccb7d2420f3, - 0x3cb4e2a2be26edaf, 0xdef5e29a52fd24c7, 0x33ed71ad0409a6e8, 0xa7b258b969a2fed4, - 0x19df8c964ceafb8e, 0x1668074ba179b2b1, 0x18b3c4c10bb29521, 0x367b7bf64646d305, - 0x549be257f7341976, 0x60c659a426894098, 0x3de9a8be140001f6, 0x23ceeb49dc4e5d4d, - 0xf6e868b935340fae, 0x958519ed5ccb7b51, 0x389181f254fe31a4, 0x587caa6406fc57e8, - 0x4611a55b4eea5840, 0x07367e67b93e9444, 0x2b57d6be24100066, 0xf0564fa8dccda22c, - 0xbc9150b2a1a6c025, 0xe01f25d87cf3e3f6, 0x9e8b17c6d1a842b5, 0x73fd5f3e6a736099, - 0xdfff4b25e1abcadc, 0xedc02567d1eb6211, 0xb49efb946baecc86, 0xf8489b079b35cc38, - 0xe9998ba9a7b61c0a, 0x672369475fa2229e, 0x3040611b80ac9c43, 0x6b5bdd39a83fad83, - 0x17a13ed4811330e4, 0x2db0ad34fc00f2d9, 0x88e07b0593d5214f, 0x5e3c7374b2913d99, - 0xb8b49e8fb204b30b, 0x2b49f24e4cd04cb1, 0xc3bd42728c485b4a, 0xea92e5ac1e726457, - 0xf8aa43da6555ede0, 0xd48b9155ef1d770f, 0xe7044669d7fe7433, 0x89f7565261094d6d, - 0x75ca2a7e519f6a65, 0xe1932b7841d7cdae, 0x4330594b27c55fa9, 0xb10d94eb38453ce5, - 0x6aca2a7c7fb40917, 0xf389389c0f51a349, 0x211f83fb6222e8ec, 0x3a8ecfd2f687b355, - 0x581cb58bf30cc079, 0xbcec4c9f07c3b4eb, 0xb65ef4eb7e0793a0, 0xc64df68569179fcd, - 0xf2f3cf12493109c6, 0x1a4e19ebbba5a836, 0x908603224b55857a, 0x1d86e50746224ec5, - 0xbe172383597fd86f, 0xe039030e0483b39c, 0xe8e30869504dacce, 0x90ed50587c12c9cb, - 0xe50b29211e960b80, 0xc444e3d39423d1b1, 0xb93a69d722fe2da4, 0x59537f1ea1a79ead, - 0xf2b8b9cede84d785, 0x51b1616733d02516, 0xd83b507b297b29fb, 0x34dcc6f588c34344, - 0xc1ed35d691c15731, 0x3f3ece2cc6316a11, 0x25eea38f418e6110, 0x48f674b98beddc36, - 0x50985064e7173536, 0x13d7b40cd5a38241, 0x9ed88dab010060f4, 0x05c94a58262f2383, - 0x26b3a6b83dc641ac, 0x10697e194e403002, 0x15c2f089cfa17ad0, 0x07792b166becf3a9, - 0xa2175816a3ebabdf, 0x8e78b27cd1b0c8ec, 0x2f91f2daf873c29d, 0xef67aa282235d992, - 0x971fe33ed94afc79, 0xbdf9b59ff40a5fad, 0xb3aeb47af9f06e7f, 0xa03750e7bf7d8913, - 0x92c9de5a87ff4ffc, 0xf2ce1ea92135066a, 0xb70014b5f191a15a, 0xe1dd23ca96959073, - 0xc46b9a2da21177c1, 0x2e5379dbd76bddff, 0x1083ce9c6f98e375, 0xf1ead0d11899ebc2, - 0x0797299ae91aab4f, 0x187b8135a9126746, 0xde887667626991a2, 0xbcb9d76e1d7c42db, - 0x03f5f29c8042961a, 0x68a0335e50ea834b, 0x50da9df53df1184b, 0x5e748f0221fffba3, - 0xe5ad3db9b41b23e2, 0xe6b9cda13ba1121f, 0x50ef38a574132e0e, 0x1848e575096315dd, - 0x405d51036a7f46ed, 0x41f5a58e9fe42312, 0x5c175d2d8310504a, 0xb27f8b2b4579227c, - 0x5d62ecdfa14c886d, 0x9cc43f3dfb855832, 0xebd477fdd2576b33, 0x548639b35fc84e84, - 0x4dfb075af385f77d, 0xd89f09fee64381bf, 0xede8d1cb54c09aaa, 0x3eeb5fa0cb52d24b, - 0x14511ac77f2a4da5, 0xadffdd5ac1298cf6, 0x9788c0e098ceb2fe, 0xb5f75eb3eee49855, - 0x8a583f4edda300b8, 0x7cc1423ffdb48c7e, 0xd49d092b237aeff5, 0x22cb521d5a86c2d1, - 0x25e7ecc8b8eb93db, 0xd8624cb708d59967, 0x6c366db0e411ca9c, 0x1544cdb65879952f, - 0x02ff3b20977b2908, 0x3d891e9856da4347, 0x5a223a272f6930ac, 0x9d023a3f995ee81b, - 0x8c4bbbdcb81d0cf3, 0x703c26feaf2cfa7b, 0xb56bb404cc729853, 0x1670a13e755eaf3a, - 0x3fbb39edac853e55, 0x15f448c1842aecf0, 0xd3af03502e6d6495, 0x46a15766b8480f43, - 0xbfb3b461113f0e6e, 0xb11cc9e9939bffd6, 0x2ecd49c43e440de7, 0x7e088ff1e0496c29, - 0x1b9131f71bb86cda, 0xd1290e39fd89ce3b, 0x8b0d123cb2896a50, 0x46981709eb850953, - 0x1d9a8478d35191a9, 0x6296b525da70f443, 0x1924b949c32035c9, 0x9fa1c690653641bb, - 0xc3e20f20a99c4418, 0x3a1ad7be82507001, 0xfcbe24756a6cf124, 0x6dc14cf96668e2ed, - 0xa7849d04eaca6de5, 0x24c04ba701cdb49f, 0x06488d7fdfc00779, 0x9dc7e03065a80961, - 0x296f1cc01b4fd5f8, 0xe0789f8dcd13e149, 0x1be70cab6119a011, 0x680ed3b4c15903ef, - 0x8d6e1c7b6cbf21d2, 0x51cf5b7129bcda4d, 0x63e564eb2f9badda, 0xede14771c34cb479, - 0x9f32a400489def0a, 0x54a9b968775d340d, 0x3d8f2cf5d3ef7569, 0x6d741ae17755e7e6, - 0x6e130036accc50a4, 0xf321a97b144eedbb, 0x0b218d7c0b8432a2, 0xe27a7ed69125274f, - 0xaba91ce3e2a384e2, 0x38206f6295227690, 0xe15f152f033688f4, 0xb0dcdfc18c077c27, - 0x580de028edc38193, 0xd402986bf5d97812, 0x8df737e116f0e94a, 0x4a1882a9c2b9ccc5, - 0x0841926aa17a1b66, 0xcc73275243da997a, 0xe1eb4e91682e8f16, 0xc2d234cc31b17b1e, - 0x45050f6cf72e988f, 0x67cdccdf8787bcfc, 0x7a9917bb86dda5fd, 0xa6a1adfcc814a06b, - 0x60254ec1069e3f33, 0x5d298d0572bb86a1, 0x38090c8f7031c173, 0xb3e22fd5e4b4d765, - 0x79de384d42c3dbdc, 0x4c1363bbcb0890a1, 0x21ff97a3b636638e, 0x6b2649436e3541e9, - 0x7d237c746fec0597, 0x6c2a8d4884eaccf6, 0x86c83237ad82ca8a, 0x2368b2fe8d391173, - 0x231e32aaf842eb2d, 0x31863f6a4b21bae5, 0x85812a54cb34e40f, 0xb661d5a62457799f, - 0xc41ccdd646284643, 0x40798887853ef30b, 0x77cf7e7006dee320, 0x8e3d2610342ce3cb, - 0x9bae3aadc77e658e, 0xddacaaba30b0ad7b, 0xfbe7a0d86bde9f65, 0xe4c431fef5b404c5, - 0xb0f1818fe5a28f91, 0x40ec268093b282bb, 0xbb9c3515d03aac33, 0x04284ce13ecbc32b, - 0x3efafc8d7e4484b5, 0xb0e9ecf288696de4, 0x53dcbd8362f087ce, 0x95edd2039da74d09, - 0xcd9841503a2de2b8, 0xca119235e1a48983, 0x648401a8ec010a46, 0x05d5833c6dd04e90, - 0xfefba4ba1c26c16c, 0x6d902dc635fb73bd, 0xa749fa1f8e2fb704, 0x6090416c653f598c, - 0x0a05fb7a436519a9, 0xdf438dad8157684b, 0x0392dd61c2a20633, 0xf7c3e258f979d38d, - 0xf8b6f24641257716, 0xef1f38850e2d5b85, 0x6de223a51a95441b, 0x765ef525077b0165, - 0x95cde52e4f032586, 0x030c84e39b53f3b6, 0x5d1e545a035e9d70, 0xdfcb7156afffce7e, - 0x20034070a59ffeaa, 0x06b738a27b34fd50, 0x4557caa68154f25e, 0x9f2d101b4e2eb1f8, - 0x63b48bd4859e9454, 0xb20daaec7761bdab, 0x665d913c3972257d, 0xbebe255fb2a63e15, - 0x8d3379848a2e215f, 0xf13ed2c7d540743b, 0xaffe3edd26a1e251, 0xa134cf0aba0b3fc8, - 0x9054715d3e1642d0, 0xf8eb4d5ad56fb1f7, 0x7088e87e86069439, 0xa854342b78191112, - 0x3e62d3c4deb55ff0, 0xbdcc14007bbbdd71, 0x5c4c5f3c56f9a142, 0xf2dd3b23863e4165, - 0x121cbea4a6c74e35, 0x1ba594fb84bdfb4a, 0x27d3c4b80fae40c5, 0x737ba6c92731ada2, - 0xf758e56511a2fad2, 0x84e8e6c43f696ccd, 0xe64b3f8c716ed3ff, 0x97d2c89c8adc062b, - 0xd8a9178b507c4f23, 0x9956ec6bc3732b67, 0xafca4e3f243529cd, 0xb4fbb09dbb1f8b8c, - 0xed6235608561637d, 0x114cd6fc6e81b4f3, 0xf6995b871c66eeae, 0xc86b62d1bd820e63, - 0xa27a44a27c7602b3, 0x98151c09c0f8f098, 0x525af701c07d5526, 0x845b3c52bdfd9ca5, - 0xa4476a40e617c001, 0x59415a274f87e3bc, 0xed47390f56a18edf, 0x61f5a3fb5243ac97, - 0x2577accf8281d599, 0x2c6e5c44b0f7e6c8, 0x5ea760e5a5417839, 0xf887685c2c51c30d, - 0xebb973b68ba94904, 0xb53b96f406e7a845, 0x9c0141d9586f6708, 0x2372cb9566405dec, - 0xbe81ffe007472e07, 0xd57485a42158db08, 0x246d6b0ba4de9542, 0xd52a3d1668cd6797, - 0x599e0def72521bcb, 0x6f2015bee2c79930, 0x4e48ff8304aa99e4, 0x69abd4d6be31de5b, - 0x6d0668d8cc862225, 0x2107e1c2dc41972e, 0x9064ead282b6143a, 0x9ac81f56f84948ee, - 0xd271d2b576b851c0, 0xb9d6a648b575f305, 0x59e58b8c4faab3f7, 0xd79cf6148e06e2ed, - 0x0adf890935d2ba7b, 0xb64f002019b56c9c, 0x13436b2427cb2138, 0x2deff3278cba08cd, - 0x8776afc11c5fad02, 0xce45e560e3a630f3, 0xe12b4c7868d82f1e, 0x43ee809aafbdeb12, - 0xe1338ad3b27d6ab0, 0x34501882c1b0050a, 0x5314d75ed582809a, 0x62e4c5ffcb21412c, - 0x89d6e1548b84dfe9, 0xb834023527ef16a8, 0x513b6c4e4df76f7b, 0xeaeb9e75fd366691, - 0x566eed9658604f35, 0x914fd437810a29f8, 0x42a6f2bad6e66fee, 0x273efa72614c7952, - 0x2e7bcecd83f1887c, 0x47eb7d7fb7b62934, 0xa9354446b3ee8d5b, 0xbf31d548190626d3, - 0x2917a36f65a9d8e5, 0x41b8ef2a4c475788, 0x4fcf91eb211c1199, 0x382fb895e69ef4e5, - 0xa67c86bf1c09a04d, 0xac3bbe467eaee4ab, 0x4139efbf89cf3a72, 0x75a60bf927d1c5b7, - 0x381ce3c10a587cdd, 0x4f9bd21f98deef12, 0x563f76d8f42f2720, 0x673c32bd844b692a, - 0x71490248bbf44500, 0x64f8afe48b2982f3, 0xde55e33138790831, 0x7e44f567d3e9f792, - 0x86c2b84b957c60c4, 0x50e677c6cfa66ca6, 0xdd1e35bd8c854702, 0xbd7cdc7082d2e585, - 0xa4d115541e1ea5c5, 0xf9c42ba5eda4a6a8, 0x30b37b768a97b9df, 0xf188177d58ecadb9, - 0x301b17f838820361, 0x8c920ac6e4982023, 0x28a32691eb5943c5, 0x6bfbc0421b50ea92, - 0xfa58da76e40a06f6, 0xf6e5fefa4e87fe9e, 0x60b0fd07764de7dd, 0x06467bd66af100f9, - 0x4dbd5de0c0cdecd1, 0xc198e0830ad7ace0, 0xb1d34a15dc21ca3b, 0x1fb46b463e9d501c, - 0xf5b393f672a5be61, 0xb6fc615f0feca44e, 0x5fe8f4b11bfef636, 0x9c2f96f23a545699, - 0x2125f827499f75c3, 0xfb5931977d316f73, 0x530e981579fa759d, 0xdd262499c729dd6f, - 0x9406070586e7a739, 0x4733b0705c8f8934, 0x23083861eb42c82e, 0xf64194ffcd3a9e53, - 0xb5afcf9ec729acbb, 0x4413284cd32070ab, 0xbe89a94ff7c3592d, 0x9afa2d341a40ae54, - 0x3474e0cadd80ac72, 0x4796c588f37a8afd, 0x0f6a2a67df8d8aa2, 0x6f2133418347b5f3, - 0x2d2160bb14e18e3f, 0x5a4a0ce529dd2232, 0x6bb38ac0c784b734, 0x3a02acab3e957bb3, - 0x4b234a8d134b6568, 0xcb5409bcca78ed4d, 0xd1faf6bf0deff0d9, 0x8c5ef01d9ade5900, - 0xc9a3df41ed9d66b0, 0xd5db7b7fdf41805b, 0x1f1a5e47acf60f23, 0x1a8d2b57e749930a, - 0xf1dbd5bdac7e27c1, 0x3222b0ccde01e28a, 0x53a102755e01d5d7, 0x1e53a3daf7cf10aa, - 0x4343a3f1c5988fa4, 0xf5f7af0b90941501, 0x433696e28b1c350f, 0x7f1a3c4a785924c6, - 0xb8c83eccae0633b0, 0x5b9bfc7ef4fcc118, 0x6e920f33bd31d9eb, 0x181cc425c7f44999, - 0xeea5b29b2a38f61e, 0x24e0d3f28a5faf99, 0x5a091873a0bf9a77, 0xd09865c8a2dfa6b2, - 0x7d7e9b6484341bb2, 0xe438337be793b481, 0x4fbbad997f25e4c8, 0xaa0f6b2458eeefd0, - 0xe371650aa4710886, 0x58e99145d5893423, 0x5ecb9f9b684fd9fc, 0x8f760b3ea4f028f5, - 0x563d06c215f2c0b1, 0x791ef3d0fe50c649, 0x9365b9298ff52ee2, 0x41b3e25bfb189d6b, - 0x53b1236f167bc5c1, 0xc1f82c0e18b02f40, 0xff093d103c613b8a, 0x4804668950b19b75, - 0x283e23aa110361a9, 0x75e38ff473b0ec0f, 0x09bd491699e1d2a0, 0x710aa351654364ef, - 0xd3c347e8793ff3fb, 0x011d3a5fe13e613c, 0xf9929f1f027a9b7d, 0xe2f1afb37deeb058, - 0x2aecd9cfd3231c29, 0x391ddd36d82662db, 0x242892e112dca5f6, 0x540af1828f57d0b6, - 0x1299c2a10000b3a5, 0xc1ae9b44078e7d73, 0x733d03c683dd93cb, 0x6e07e9d57f92f7a5, - 0xb766b5823f7b35df, 0xbb8a7effc5da652c, 0x75adb16fce9b4681, 0xabeb12f915153942, - 0x072a56daa586c243, 0xdf63b2bed4ddf894, 0xf69b450451b498b0, 0x7c3e2cc4ef5b482c, - 0x2e31da92c8fb8181, 0x5d7e68fef025454f, 0x5e2a121cebdd4a4d, 0x20a6736e76af0645, - 0x4d2077205687b67d, 0x863094ac6d96ef17, 0x2fee2b70e997d2db, 0xfeefc7b0f0a4940b, - 0xdf5cf6486d5cff53, 0x18d1d6d4f209d95a, 0xdd66772fed6faa37, 0xfe884afab2e16386, - 0xa4f9e17a824db151, 0x0809783e6502b913, 0xae679da17029a35d, 0x8528ad8bee4fa73d, - 0x835f1699225943ab, 0x6be82fb1ea41cf46, 0xf72d79cd5ed44bc4, 0x2dbf179cb3e611d0, - 0xd7c5d9779e177a27, 0xe52f2cd9785f4b8c, 0x25ca2880349ede53, 0x8f452e179c8414e0, - 0x0ec1c02bdd5ea308, 0x8466e11593b6c17a, 0x664503515aded236, 0x94b3597a5b118fc8, - 0xa7d9b75c85091291, 0x5478a0f732597115, 0x9e779746a30dff5d, 0x4d809f865fc1a5af, - 0x8ed3dbd364c89be2, 0x2a91007e1742803a, 0xf5782f25dc4c625f, 0x1691dd122e1a954f, - 0xfe4ca35636467544, 0x89faec0404649698, 0x2d59d0ad031d79df, 0xda28bab16fa6ccc0, - 0x5abf25b177e9c7ed, 0xbb5675ac15df382b, 0x2613d5b6188613a7, 0xbf42ebfd48eea069, - 0xc7efabd1e74c538c, 0xf7a061c898bfd88e, 0xa72c1ce1c66e5aea, 0x2b6e2523697f8a2a, - 0xdc181e273a66976d, 0x67e1b8321596a4be, 0x47b610c557867d27, 0x9ed6d5c8b9d07005, - 0xa00278b16aa9149d, 0x8291565e01274cee, 0xba3ae4ce261194d5, 0x42fb64756ca82d86, - 0xadb3b525b9db1714, 0x7699f13a1ae2d3a5, 0x2f75f93326dda802, 0xa72f256466174d04, - 0x2c027126765db09b, 0xfee8d1ccf8bf559c, 0xd0253579c028e1f8, 0xd0e772e313ba5ac7, - 0xd61c86fb8cd5c05a, 0x0096b73167c79e44, 0x41bd740321d7d35d, 0xb3b9a269b45c46d4, - 0x95f5d2321282b6d6, 0xfba42aa1a975079f, 0x7c6366ca83585cd2, 0x4d9257b6bead31d0, - 0xa7eb3a8052e82c20, 0xa6bb242612e7f42d, 0x908156b5a1b3c0c2, 0x5ab963de9fa184e7, - 0xa042c474c6cfc4b2, 0xfdfbbc40d0bba95d, 0x966f92fc17760083, 0x41ff0a1dc916ca85, - 0x33091cc11d4703b4, 0x8ca400d729c16899, 0x845b6f162398443a, 0x2d7128d600fcd4e0, - 0x0e3e02cb1a15f4cd, 0xf0c25d7990d3d2b5, 0x4348f79dcb3b8419, 0xbb4564b87c73320f, - 0xbfa65673f34bd4cb, 0x586aacd9077bd2b7, 0x3d9aaadb6e2bbb39, 0x0ff223fd6a090454, - 0xc2e8472ab94c538f, 0xac2b10428c1d774f, 0x11f150ebafbe0d6a, 0xeb22935171500230, - 0xc4e00aaee7f32576, 0x5cfafa2a98e5bda4, 0xb7fb2ab020a5a893, 0x7d9ed6f8d176098a, - 0x1c24a7ca529008d5, 0xee476b7a879049b3, 0xb3e7405a408c71e6, 0x1673786d415c6328, - 0x4f218834bb1f36e8, 0x84cdca85e36c968d, 0xd53e60f48ed3a73d, 0x017c5ac42aa0ec2d, - 0x128d9d616e5c1737, 0x96b1704c5c29f3f8, 0x96e84f79dd7c18a5, 0x42e244571b6d28f2, - 0xb4866840e21da423, 0x37259f151e76c0ed, 0xa4a97f5cba8659fd, 0xd9249061631fddca, - 0x086246d6a1440c16, 0xc04414b53d8ad38d, 0xe8bd28b67dbf837a, 0x76b6aeb42803e652, - 0x6540ef7c8140449c, 0x60ac238e69602689, 0xb24cb4c71a76c46f, 0x3368407ce34d1926, - 0x94124d68dc4921db, 0xf0a41f0f2fa7f999, 0xcfba95e1313f6630, 0x32477b6623841b1f, - 0x17645fc84e8d3a8f, 0x30d31ea542f6a937, 0xcc070478e478cdc6, 0xe81f867d5e3d107d, - 0xcdbe21d501785077, 0xbdf9cd3ed8c61668, 0xf2b33997a8e1b19f, 0x58f7e9161f113d6f, - 0x085057db45cf73c7, 0xbcbb2a79cc8405f1, 0x4e9955adbeb8c49a, 0xa81b06483fe6ccae, - 0xf72ee3ecdc41783e, 0xeadea722277d70a5, 0xf3cd15e86eb993df, 0x2a1bff1299088219, - 0x70428b993edbd370, 0x95a4e4f141ddac15, 0x5339c037e664ba75, 0x64992443123898bc, - 0x6b2c080e072fce92, 0x2d1a27214c304b14, 0x07dfdf091069a02c, 0x4dcaf67db789ac68, - 0xca57c5c4c2df63d8, 0x7462afa8f769a6c1, 0xddf6949f1cc15270, 0x902f1a9dd127d5f1, - 0x74f61e475a3783a3, 0xeae8e181e130c9d6, 0x523be69b82f4e501, 0x9c0cc4cbe6231dab, - 0x085fe4c45e8abb81, 0x675f9816fe882ddc, 0x87a765215f129b90, 0x247b734b7c3e010c, - 0xddd60bb0878c24b8, 0xff102dff480103c5, 0xa061b74fddd72001, 0x023a0c969420c32b, - 0xaed5dca6f671b9a5, 0xf714cde311c80712, 0x49bdd42339e05038, 0x2b7ed3345f6e5932, - 0xd2381ad568aacac0, 0x21bd606150fa7eb3, 0x2aa8d3953351fd2e, 0x9e386456e3cdb032, - 0x0f501078a9395bed, 0x0e826621e4576abe, 0xcc599786ca6dcf98, 0x1276a79d5495f597, - 0x398bb8493e2d9b79, 0x6def1dae77b76db1, 0x8fac7487743b24ab, 0x46947d61a1aefd1b, - 0xccbd9756e9277f32, 0xeca045c555d198c3, 0x6570fe5e8b089c69, 0xec92694f50c2ec75, - 0x4e49e3dc17b2275f, 0xc554b865ea361c5c, 0xd8e42c7f805f5704, 0xab89590f65dbf0c8, - 0x4030825603646061, 0xabafe99b4c5f31f5, 0x6d30538b8c06c50d, 0xa4088aaec3c6f505, - 0xf9eba4bbf02d3377, 0x8f05e855e5d1a504, 0x76d474bb9e965faa, 0x670c0c7d0fbeebc1, - 0x1e2f078c86acdb4b, 0xc708f4ba42ce0455, 0x9e516db0703c60a0, 0xd9a4ec1bd7fb1a00, - 0x3b7483a3caa5c287, 0xc2b6d5460575e31b, 0x17c167aa0fd6a584, 0x802d49e9a8bdd8d5, - 0x73c6200b8a1a585a, 0x563ece8855c224a8, 0xeb0a0040edea7efb, 0xb2a2413d768d7d76, - 0xa651f857c7cd14d1, 0xcdd0417468a3ae62, 0x0c1b4b5deb592abd, 0xd63feda402a0fbb3, - 0x890a75599e60e79c, 0x2433fcdb19761a3f, 0x9623e6aa0fa6a035, 0x76d564e4ad912040, - 0x99b49075b232797d, 0xfff7607e45372c4d, 0x6477c180dd6d8829, 0x0a0df16f8e506b61, - 0xbe34bd6a4aa4ba00, 0x54b433d3962fc784, 0x940cdab5293df29c, 0xe8a7712823849dcf, - 0xb610cf0d5e6e36e0, 0x68aa76b26d63d904, 0xb07b95118235ff17, 0xabbe72a41bbf6937, - 0x4335102116de6c95, 0xf18468ad9fc60850, 0xbb109e4a6d63982a, 0xfbf576c7101c19a9, - 0x48c645d2d42580fe, 0xe4fbf2c0003af10f, 0x7ab4d6fae3e17b06, 0x19fa6886e1906a40, - 0x229074673a955a19, 0x5d308012b518b165, 0x67d153a6c5088361, 0x98bad801cce60366, - 0xb45e9247bb364c29, 0x29dbb367a543427d, 0x02332528d45694fc, 0xdaa97eec6762a0ea, - 0xc685c6a545998461, 0xc14af1a65b000a5c, 0xfbc934ac98f97b8a, 0x98988099d629a7d5, - 0x0f35cc4f16125948, 0x3e96c4eecb45828b, 0xd3776b91c76f5733, 0x4e8e260f4c5e38b2, - 0x58ee23fed9b1baf5, 0x9c484fa82ae4c4db, 0xaf8c61dbcc295939, 0xa3712e0577317725, - 0x053d7a7aa60c88f0, 0xb6685f17bf9d3a8f, 0x4bf16f93e649a772, 0xc2045f3a9c720932, - 0x65f05cc39d831146, 0x976134fde4337450, 0x4e2ba081b7e50f42, 0x8e5320c951916a0c, - 0xf71cabc3b97534de, 0xf108ba882e62277f, 0xa3276d81cf830d0a, 0x492ad5c08177ce90, - 0x5dc6863f5b4ca83d, 0xacb7154e38190d42, 0xe688889b2244c780, 0xbf5f200b255aa3e8, - 0xf7b2d764a44b4b29, 0x3ba21e3c0fb76665, 0x92f2c8502a368540, 0x670e239fedbae5f4, - 0x138d8794bb2ca459, 0x7e4d1bce72c28b01, 0x2668de28d037cffe, 0x1d1ee3819c7a30ec, - 0x33c38d38e7dfea3f, 0xbb0972c93799a8ff, 0x6f5446a6ee33af8a, 0x532bec51fd252b1a, - 0x9b366b5cf86dcd33, 0x908d470754de3d9f, 0x8ab904a52cab7663, 0xd038ac05220b1b14, - 0xb5ac4321bc5d24ff, 0x9f92f41e6ca0ecf6, 0x9e7d7bf762790456, 0x45f423a19152927a, - 0xd681f8e0c1b1e681, 0x6ffe25c099487c8d, 0x9f1c31f8d4b7352b, 0x2362078c1f65c57a, - 0x3936cc72ce47aa77, 0x6c933ffcad73be3e, 0x37ed509053bcfe0e, 0x3c9076fdab33279c, - 0x065f743fdc22d071, 0x0d7e85751de9d8d1, 0x280c10b433277fb6, 0x4ddf26c8f1049f57, - 0x5f25cfeeaa3eaf51, 0x4cf4ec3be1c93924, 0x4dcffc17c3ea2177, 0x20b02b1b26328e57, - 0x3646e9de6bc8ad25, 0x786eae906e01fa14, 0x77d72c30daed2b39, 0x974cd08716b770d7, - 0x3c1e6f872fe81c53, 0xbb7f1deaeb43ec79, 0x624b80a122cf6839, 0x49d210183ad3ccfc, - 0x6aabd6e98fc434ca, 0xb36aba7a255bb1d3, 0x9732f2cc61a8ae3e, 0xfcc68c2adaff8706, - 0x1edc8ba4898cb7d1, 0xb5ac15cb166e790a, 0x3351915a051ff1cb, 0x97bcc2f3d801fb2e, - 0x1f5c168df0422395, 0x4cf59e3ba5871480, 0x8c4be52d16de5f1e, 0xdfc3b0c36d609a1e, - 0xe4da2162b0e595f5, 0x5911f0cb5855085e, 0xd00fca7a498bbbe6, 0x58b3d3c2dcb3c6cb, - 0x13eadbb24291aee4, 0xf3c461a0c662f645, 0x7d2350b72d61df99, 0xf8d63b1d105c7988, - 0x7af7b87dfe9d02f6, 0xb5180fd00f01a679, 0xb4d5bca520722f7c, 0x12b6c0b6dcd7b46e, - 0x5eab7bd1edbd5bf1, 0x9b18f206a7f573ee, 0x205d2eb585299908, 0x8b64f5099613105b, - 0xbe74c37875789ff6, 0x92f4c22b04d84c0c, 0xbfe400743d558510, 0x5ece784a4cc21c13, - 0x49e7322371157e61, 0x3e17e47ba80f660d, 0x12e61375f5752dea, 0x9e506f23cc1f4041, - 0x066a4a3a44f86424, 0x0a60b5c5eea42566, 0x5af39fa11e5ea0b2, 0x2c6f192ffbd29655, - 0xc269ddfbe928fcfc, 0x3a96639475a4d2f1, 0xbe2a7b982ee497cb, 0xc074ec700915e3f7, - 0x4d4d450f0260f53f, 0xb6a9c3ca5dd3d39c, 0xbfb5b9ad60102e0f, 0xe386084cddc8eadb, - 0xe295019000245b4d, 0xfaa02e40ea95b332, 0xe5531beb1d40e432, 0x7ee078e1d15fa02b, - 0x015ec6e38c87a50c, 0x954c764681f7a062, 0xaf3c0df0e81f99b3, 0xec80287c9170b38a, - 0x1f5538b9575ad5a9, 0x686db1e4e6aafd36, 0x81c85254fdf4d70b, 0xd3f7562a75c1b114, - 0x8e3aeda89da24bbb, 0x9bd4927db732583a, 0x47f96c3749de4cb6, 0xf557ec01beac7dc1, - 0x89cfff6753e1e71a, 0xcafd46092c7c7a00, 0xbda8d2955d7cdc75, 0xdc658d8ac71ae85a, - 0xf6222763ad444d80, 0x31b3cba10565a282, 0x9471954e8cf3d8fd, 0x9d1d2294213e82c0, - 0xe264caf5902b3520, 0xd3e384827ab1d8f0, 0x22c1cdb484029f4f, 0x53b07c637068ff4d, - 0xf3e2ec54670d8169, 0x1367dec986477371, 0x85a80e8a06169dbb, 0x352b67b9e7248fa0, - 0x41c771850ac6f24c, 0x96a5f41163640ce6, 0x583e94313b442f2d, 0x9119db28bd0c1256, - 0x5e7ebd232265fefb, 0xa5b6b6cd9b3ed589, 0x381a000f59ed6ea2, 0x08c63eb903e0ef04, - 0x52528e0def72dec3, 0x235fa0594c0175fb, 0x443473747d16d6ba, 0xe6ba1816f4e81d71, - 0xc52c269b9df759cb, 0x1a42cc36fce01a2d, 0x3201ec0a611c7f9e, 0x5835848cf56d69a1, - 0x46e7ad4a20a895a6, 0xe7b9943c13a52892, 0x3d2ddb9e8f898c46, 0x90b2467bcfea2c1a, - 0x245c12c39fe5b875, 0x1912f655d8c7f863, 0x713fb253035e6aa7, 0xe93d48b13d7db94f, - 0x2714ab912028dbb7, 0x57058dada56592e1, 0x7904f129190d0c0b, 0x4c77c4691629737e, - 0x11d89ab172ea1d24, 0x752fc3d0c468b9bb, 0x172a94f1862c7721, 0x95125e1fd0b16e01, - 0x4d548681bca4cf31, 0x9a1eeffb6e413e59, 0xbfb83e42ddc1ee03, 0xd1cfc5549279e631, - 0x5cfc8e12b5f0da52, 0x70899ec9bdbc7182, 0x0020e1a5e4687184, 0x6029b4bdee4ebe7b, - 0xae3c575f3d00f559, 0xf9ff6baeca1f92a7, 0x9846f96e736b2582, 0x12407dd9fd565e76, - 0x44ae6dc391154dce, 0x24cfb39f74f62354, 0x423eb39c4ee9507a, 0x525b8648273d5da6, - 0x08c6206cea681dba, 0xc3e38cfa154d3c0b, 0xa2e955c843079d20, 0xde17c095d95633ed, - 0xc3577cd7aeb59135, 0xfff2f1695ae3d076, 0x57da54c4bb73c6b5, 0x4620dadbf64c2728, - 0x9b63ef766d514b98, 0xe5947230656808af, 0x0c4611a8d4595383, 0xde8fe2c3c54ae7fd, - 0x6b251ef515cc66df, 0x4cad0638e91e9fbc, 0x262c2460e8d8f845, 0x831cecbf8f62df38, - 0x2378082b1ae861c7, 0xe84685174ae5e3c0, 0x1f05368123d194bc, 0xc6be55029b5c5214, - 0xa68ee1bd0028df68, 0x1bdf8086e0945a96, 0xbef6757f8747a107, 0x4c6b7675e8f464db, - 0x2529f40830adb2ef, 0x26feb8baac572e33, 0xbe68e6e7a339a7eb, 0x9c9e72b363d369a8, - 0xce8f7c5c3d2d525f, 0x731cddca25ce4433, 0x3360076f92182ec4, 0x337c968ea1de4f95, - 0x3f2f097842ed09c1, 0x93b0f8e164c795b8, 0x15e3e9e28de26ab8, 0xc47c402a0a693551, - 0xb1790c3f2919727d, 0x7127d8ad1970833e, 0xa5bf2e0ef6cb99cd, 0x28a42c14e20f90ce, - 0x4c3d83e5575b8890, 0x50bd3ec7e130a39c, 0x1db891777191da03, 0xf82e3d7fb9265f1a, - 0x8702324f34b7b4e3, 0x1a3e56bbc85bbe5e, 0xa6584b86d64a190a, 0x952c8aba7e6dbaff, - 0x4f6c77460278d46f, 0xf35ee7d808cd80f3, 0x3d41cab38258bb70, 0x9632d8b60b8f8a52, - 0xc687417308791655, 0x7b8518d529a5f7e0, 0x088ab1c978ac7c1f, 0xaf2d05ec2ca77731, - 0x324c49e681c84a00, 0xabb5f914edcf5258, 0x82e56965e67cd033, 0xe08c44305853a661, - 0x58e301fb2ed803d0, 0x12b1726dc24e932c, 0xd3e9d4ce5895615d, 0x5a0aef1e9fd7a325, - 0x0404595bee9e3392, 0x1a705bf411f63ced, 0x9c41dd9d43e92ecf, 0x3e964aaa9e83c657, - 0xdde2974aeefdd469, 0x45e6aeb86aabfdc3, 0xaf810fbf17329e40, 0x0cd9861f2f803dda, - 0xd7e6b5cc4a44a6ac, 0x354c739599ca2989, 0x51dd45c3b7438f45, 0xc18252f61c6e1295, - 0x8f9ed53c0d66a319, 0x641db9fa4aaf3d8e, 0xe3c8e64b4773a518, 0x300f3c820aec5ab8, - 0x3f626c8ade367348, 0xb57b459f87b35fe0, 0x3c55c637f5cf6da0, 0x88ede31e766015c1, - 0x706c31ab43e05f78, 0x57ef70ec3c6f0342, 0x9c3d2061322a9ddb, 0x5d793c8f81b071b7, - 0x753c1350a7b85828, 0x5c28dc7a5be38a29, 0x699beb979af1546a, 0x53a76100107b079e, - 0x9445880c8dc3fbfb, 0xabfcca75384f2bac, 0xb119a0494642b8e8, 0x44aa6fb866e65497, - 0x5d1761d932fbff43, 0x362f71284b71e7e7, 0x5b3351ce6dbed3ed, 0x8fed7aff2664b9e5, - 0xe5faf83dcb1648b9, 0xb55a8bba8ea4e7da, 0xb24005c2b13830ab, 0x36c14e55285b1feb, - 0x87b05f29e74e9990, 0x6b60856392a33d96, 0xce8f177cf9751c8c, 0x8d8ba4626308e8e9, - 0xfa6f947d5f0ff81c, 0x98b564d86cfdaf40, 0xcfc2c17c700b294e, 0xeaafd1c9c9ef2bc7, - 0x0d951c41490af34d, 0xa6709f1abb95f676, 0x4015a6ec249ee4a7, 0xe6889aa48db8d43e, - 0xedaef60e34a7521f, 0x8ebe5259d1a1a3eb, 0x2b440b6f8feb74bb, 0xe0aace7c555fbba6, - 0x3089afd561c53c27, 0x64d363bd2f4298ec, 0xe99fa96b6fe1fd02, 0x9ce8ed7559210412, - 0x6c491230ca6e8547, 0x4652f83544ee70ac, 0xff646a622f77c3a7, 0x7708b53e3b239292, - 0xd4634354abb0f9af, 0x9ceaca26df6b9e8e, 0x85849c997cb431df, 0x483cce1b12db34c8, - 0xed56d12ad5112b79, 0x0d632aadcbbdf80e, 0x70edd711a33737da, 0xacbed8d424da0b52, - 0x211c0f59d2a879b3, 0x5970039203d5bf4f, 0x2aae7111b77aa919, 0x9f80cac3a40ebd61, - 0xadbc1ce2247261d3, 0x8002bde45fedc0d9, 0xbfd89769234c93d2, 0x2c698508b011c283, - 0x64c425a3b8023a58, 0xd6969d1c52da2a18, 0x70071b8522e0ca4e, 0x21870ac7175596ab, - 0xae89ca1f5ab6c092, 0xde0ced8f36a0f775, 0xbb6f308f9560bc9d, 0x7e980ead1f356832, - 0xfb6a447607a48d9b, 0xf8a0a1d2fc93acb9, 0x29896860bc83649d, 0xf2f155d09e48d512, - 0x36bfa958899afce7, 0x9d942fcc07a357b1, 0x94a0f459a2ab1a3d, 0x983ad53f8ab2d697, - 0x944e1ef67c75ac44, 0x836511af3e8ec5f6, 0x109d60cb24c301e9, 0xd1bc77688552287d, - 0xef4806a963941d4d, 0xbfedef941f736b5f, 0x0f91c97aed0f516f, 0xeee2cc0923c1f8cc, - 0x11843c9bd04a416c, 0x92d93d228508268d, 0x0b23a136dc386510, 0xcce870b2746153b7, - 0xb7c72bd49225fde1, 0xbaf5cd7fce0bb71e, 0x686ee19c531ecae9, 0x023dbc7831f4d544, - 0xa228c96d7b84736a, 0x677b06ffce47f427, 0xba412c4fc41b8722, 0x29c9ae86dea73624, - 0x6bc740b124c6172e, 0xfcebb2f4192631f9, 0x32600c5babac82d7, 0xeff09ceca6aec448, - 0x3e7eb96580ed6e7d, 0xd9ef4fa3f2d78ca1, 0xb2320e022ef15db4, 0x69909645bb63b367, - 0xd8797201cc82acf2, 0xdfc629abf4bb0665, 0x88cafab34337abf9, 0x32d40784db22a3bd, - 0x9991f92ea80cafb1, 0xfcd3ab39dec48eb2, 0x881722b0f377788e, 0xa26e7ba7277ab096, - 0xb6984d0d4977db60, 0x5358d5e4bae2f7f2, 0xce309fdf70b72b3b, 0x195368e486236134, - 0xe61181c4c4f8dbb4, 0x93f6a0fba39d6134, 0x548c41e10f7959ed, 0xf2005a32db4d51d4, - 0x5bced3a3f031f115, 0x858bd172800e82e2, 0x616a97f3100952d2, 0x37c4914b8f8c9816, - 0x25455093ecbac731, 0x8baa5f5ee6bad42b, 0xfb1111ef59a11155, 0x1e50a81b18d6e0db, - 0x3d94ad28d3ef76e8, 0x866c224e7ec0f20f, 0x6be1e112b02a461b, 0x312083b8773f0cd4, - 0x99160c2c025b1140, 0xa481d53db41e5067, 0xaffdc1e5ca2a8ed2, 0x2158d3e80bad63cd, - 0xa94ab90317f52729, 0x294be56a2562de0d, 0xeb5516d9e85bf365, 0x478fe9449186883c, - 0xcd20fef03d63ed26, 0x36c9658b5335bbe1, 0x391391658524c7c1, 0x5b4320917f183400, - 0x0f4b45615188af4f, 0xfe285dc241fb5e16, 0x643853c70a444ca1, 0xd514594cb07a64ee, - 0x51148edbb32677f5, 0x1c37c06e46cfa878, 0x0e3b7aec6871af79, 0xf0de443677f5e615, - 0xd5fe161abfea10c5, 0x367a305e5517deae, 0xa74f080cf2ed2671, 0x6f409e1bce6ce74e, - 0x733d12e80cbfa881, 0x8037584f75c3236c, 0xda33e89721b59ee4, 0x56e3f8e837cebf5f, - 0x301901062fa5b975, 0x898698745490101b, 0x885a0d1841454c11, 0xe1c0c3f9f82e76c9, - 0xaa3020c67cf175c0, 0xbf27c1fba255a0a9, 0x7ac04064ff19f5e6, 0x6390cd0e43c7f470, - 0xe13e495aa7e6537b, 0x04ebfa43f7b5823c, 0x2f61bb8db7b59a4b, 0xaa72ec30b87dde00, - 0x24fe5e0cf95ea9c9, 0x592c46275780e21e, 0xe11fdcb30c62926b, 0x33143106d3907c9d, - 0x6294933ea07a23e5, 0x8a7f9b869d3b10fb, 0x7f1b2320b3a2be3e, 0xaebe58151afd9ad4, - 0x62984d47040d26fd, 0x4c92360e2c216f09, 0x19bc360b2f1a84f2, 0x3297c0ce0af56744, - 0x811f05d424c0a63e, 0x29da07bcde62643d, 0x736cc751de1b43a3, 0xc2ec878ff2a49f33, - 0x452ad8b66741c79b, 0xd0ae9d941a26cd7b, 0x8fddbb4473334631, 0x9ec7a45e5982a62d, - 0xb388e4222c02f4ff, 0xaec63a9998e91bc4, 0x1d78f06ab0c9ea18, 0x6cd2c23c240b0c3f, - 0x61800c3b7deb062d, 0x3ce1de65404f195a, 0x488a6fb78b1072d3, 0xb003e57cef41900a, - 0x4e34991f8191ae05, 0x0582ea81d4b2d22a, 0x50f78bd18a906d4b, 0x122ce4c3b0441147, - 0xab8c61866f80578a, 0xee116fa2d21d8e2a, 0x03071fe3e82f599f, 0xfdc6933a55a87d3d, - 0xc9acb0abb2e7d0bd, 0x4636a992c998e1ea, 0xae898d265827c59f, 0x9f6519ed066fe521, - 0xefefe34740949480, 0x56b4645b10fb7c8e, 0xaf6915f671870880, 0x283ea1ee03a6e196, - 0x9d94184b6d405bcb, 0x49cba7cb18fd46e7, 0xac64e174161b3d18, 0xf15fe69ce485fd91, - 0x58a921f1a5ac2f88, 0xc710a1605f42e22a, 0xfde897f12c363fb1, 0x8c6a24ebd99f2bce, - 0xc720ab6a87f510d8, 0xbc203725dbda6a1f, 0xe889bc8ca0c5ac91, 0x52609ff83f8ecb71, - 0xde2724b0a84350f8, 0x220e60cf09b880bb, 0x5c57818f6ed7cfc9, 0xd7b4b48a586e4c70, - 0x2756effa2cf67530, 0x5724337f1629c10b, 0xddd4192046a39a64, 0xa2d065d29210fe58, - 0xdf96720706f727c9, 0xfc27a49e1076eb6c, 0x7c508eebd0c55cf2, 0xe20720a7ddcb83ed, - 0x910631cf6fcf4315, 0xddfe59cbd0f9c089, 0xa9d794a4eec973bf, 0xfe0354f82ca31fa8, - 0x132ce3b1d394f27e, 0x3b5590cdf656ae93, 0xb6da438100c3d7bd, 0xb26093950a48b436, - 0xc467bf890877866b, 0x008ba522d26edd2f, 0xabeee5b109332fd0, 0xbd7d3d60070e79e5, - 0x71cd7d1dc4cc8501, 0xfba071f129b52df4, 0x6c8d2789fc6733db, 0xef2952dd6cf59d8d, - 0x4b1dfb9e62c87e20, 0x78bc1911e5f0caec, 0x0fa5b0ab7e98da31, 0xac30980805f1a0f5, - 0x507f338eed25c6ba, 0x068f2e9acbb91839, 0x424a884edb364c9a, 0x43d9b67230917af0, - 0x4cb315ab287fcd88, 0x7046b76cc408dfe5, 0x9ae2e6170f496aeb, 0x7706980c747efa87, - 0xc140746d8b3daa6f, 0x1beb17c43e5fe37c, 0x9dd84f4d82fbe635, 0x20556f2415762b2a, - 0xb4a5c0c5fcb6e55d, 0x99ba9e7e139fb6da, 0x6298c14758daddbb, 0x8a0a3701d20b2c7b, - 0x086ee5c9a5065444, 0x3adbacedc5f2cbf0, 0x07cec280dca0f2e4, 0x6ac9dfed4154c0a2, - 0xa400eb033937d6c4, 0x28760b2d8b70eb37, 0xad3029a986fa934f, 0xfef995836953e63d, - 0x6387859173112617, 0xe8ac784be4a1b060, 0x97fb07f070c990d2, 0xe32354a3c65a4a51, - 0x44e1d9d0d273799f, 0x76374da83dc5ca52, 0x0569bcf76aab451a, 0x5d1325b97640a6c4, - 0x886fe2490ffea79c, 0xe35142a2be158d9a, 0xaff4035022511e91, 0x74299b2ef9daa1c8, - 0x73683a82ea962377, 0x754fba24c083a599, 0xfe58cb20af5335ef, 0xa8d0b1e85e5aef13, - 0xefbafcd215beb37a, 0x95bb254371b878b1, 0xe5eba9776bd3b9dd, 0x01ec959c32e353c7, - 0xbe6f1ef9b1702e7f, 0x8ee4837082d99ed3, 0x232ace301944e61a, 0xb1241110683b1408, - 0x492c5b8af3eeb207, 0x4eb64aeba3b84739, 0xe77d6c6b50562cd6, 0xc7a6de9d0e95ff22, - 0x7abd08dcf3282ecf, 0xcf02e760005404e0, 0x04f1766827885b8e, 0x8aceb343d9742645, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xc8262abd3b39b9b1, 0xe60bf00edca1e6b3, 0xed4adb11f343aaa6, 0x05e490af8b3900d4, - 0x3a810102329192d7, 0x30b80e9ef889205d, 0xb975451dbdcac43c, 0x193fadade2fc1716, - 0x7725cf740490e669, 0xcb58c73f4c575843, 0x4e44152901cc6310, 0xba982df20859e203, - 0x392a81c3d34d6b1f, 0x814c5f88b1e6070a, 0xaaf3ddff045056ef, 0xcb8953e509890774, - 0x21559725fbe467b0, 0xeee03a4eeddb3cbf, 0x008f9fd6d54586e9, 0xe2ebf5aedf9b6533, - 0x63eb0e035c256495, 0xeb33c8d4335a16da, 0xfe47dea24b430db1, 0x18b13ad29f0e7966, - 0x65e2f4a3fe88d77d, 0x35afc1fbd67965ed, 0xeda0a65d97407175, 0xfb1746af8b1908a9, - 0x64d55d92832fa7f5, 0xa7ad683ec146cc66, 0x5d0db3a88327c5d4, 0x236cc7402fd0d18c, - 0xbc6dc347233a6fb0, 0x24f0cc2ce4959b82, 0x8e1ebd57f22e8a7c, 0x764eeefc376a0d75, - 0x7da2de7f2b5245bf, 0xfa9a1b6c97fef7b6, 0x43a2d194ca046324, 0x5a936de649eacbdb, - 0xc08d1f429c441d62, 0x56b7d209cac1a3b5, 0x59adc69c2d70501f, 0xdbb3b04b7f3ed2e7, - 0x42c3d3ba4dd8234c, 0x98a0b5617c1969bb, 0x0aecd08f2cf1ec0f, 0x8653d196e94f0369, - 0x88e571f559d7845f, 0xab0540a7bb7442f0, 0x310d4ccb2b1e8430, 0xd2af48b75971d8fe, - 0xd535ccb53cf4ae1b, 0xf5b2b105302acb9b, 0x2330e80e4196b8e0, 0x55e486b2c2914062, - 0xf3aab520de57d2b2, 0xc9ca9dd7f9540d11, 0x1fedd079904aae4f, 0xeddf090d6e970f43, - 0x7ad7b5bdec02151f, 0xe49fed5bcf849cc3, 0x6525aabc18dd0a3b, 0x6238795c52ce8b9d, - 0xde12f5df5a19ef02, 0x44a449e6d8292dc2, 0x8d40454bd0bfd955, 0xa0f6b0aad5e81569, - 0x5aefac6f38845398, 0x40a5d11dea55305a, 0xc9df50fe52c1f427, 0x218b467e7568955b, - 0x834015cd5198f21e, 0x447111ce5ae64215, 0x6f11401bfea88aa2, 0xffe75c4113bafe86, - 0xe2b48f7a200a42e5, 0x839dacd4288d1f5a, 0xd9444c95740475ac, 0xdd14e1d1ea3e6462, - 0x45f0cc8e99c6d1da, 0x7a03540fea610c39, 0x909d6b6a0fae0f17, 0x05ed2454bc88c856, - 0x242951361df3f5af, 0x9b3b360a3b24813b, 0x4841163bda66dd99, 0x8aa60cb754cdd5bf, - 0xb8bbb48bca9865a8, 0x10bcacb0162cb6bb, 0xcceda9c1abf2cae7, 0x5ffd91fe98549c00, - 0xca42bd99c5fc953a, 0x5634f2239991960b, 0xe622ea5fd0b738fb, 0x8484833e53c91674, - 0x876b7d7aca23e963, 0x1cde128535b4ab4e, 0x7cedcccdaaace3c7, 0xd5713ea86362684d, - 0x58faeb0f03b702a5, 0xaf13db66966ec9eb, 0xe6716a31058df8ad, 0x66ab4071ca58fd38, - 0x7957cb49d2d19e37, 0xf9617fb72af823d6, 0xa7e332751050048d, 0x395596b56c51a5a7, - 0x6083c6557de3a2a4, 0x5257008061cc54f7, 0x0d689ac705d3154b, 0xf827966fcc134117, - 0xd126ee677e585a44, 0x5bd3bfee40db0664, 0x65f55009d3df7c60, 0xc2d07f1747dea883, - 0xdbc983108fae32c9, 0xab5acb3d7b4cd0c9, 0x7cb8e265b6850ca5, 0x72fded8debdf4359, - 0xc8ca5ebf8d632acf, 0x47af9f7738f79ba6, 0xecc52bceff23a8b2, 0x3b04b1bed94c2b4b, - 0x86f43870a0ceb79e, 0x493fd815769941f5, 0x58df64287742315e, 0xc3657113d8da762b, - 0x35e34df0a3a12ccb, 0xa653c2baacf63006, 0x805dbe73922f50c6, 0xa803dbc03215f571, - 0x93e943e03dc28830, 0x38859e9ca4aafe43, 0x3fe5d938671a4175, 0x6fd1105782d77593, - 0xba8a1b4db0a1307b, 0xb744d84063a767a0, 0xedbbc5d4445de18c, 0x5432da547d5f95ba, - 0x9608786c271ec975, 0x2e3285837baa917b, 0x869ba079b4987aaf, 0x5c16bf24ebf099ec, - 0xf3a739d749306f23, 0xbba0d18a7b0ce464, 0x7ed4280658f8e901, 0x4b0217c7cf425b0a, - 0xf67049e78a2820b5, 0x79280fa60b2a0efb, 0xfa88d40c981afae8, 0xf53469cd20de9365, - 0x0457b41d1d5d6fb3, 0x0a87cabe39247ab2, 0xa8f5af64b89cd6b9, 0x828eac9013a49169, - 0x4a6584dd6e82067f, 0xa6e318d300f7532a, 0xd87ecca6443c119d, 0x8061448dd1e26bde, - 0xbd83f2191486bdcf, 0xedc7050260e2f946, 0x890fa510990bc567, 0xb247ffde5c8d3a31, - 0xa2f99b16f4c23337, 0x15c2261e37411d61, 0x302ec4888ca44c70, 0xaffe8d089f3ef502, - 0x0cefe020a7454197, 0xd1252bdec7b475ce, 0x0b9738a5e2c37a20, 0x0c83c66d0ca05f2f, - 0x5ff2ddf364cf8a8f, 0x4343cce75a1ac5ac, 0x8372369347fe36a0, 0x17ff4de73f32a507, - 0x3e6836867d5e3bcf, 0xd98ce42333478b49, 0xec06c2261c669b0c, 0x60c044763c4fcab2, - 0xda44669bab2ab5e4, 0x7aeefe90f3a14c79, 0x15a8b43c7ad30259, 0xe5180308c2af7125, - 0xc2b9379a066728b6, 0x1e429e21da3a6e68, 0x8ee4dcba644e0304, 0xb60335d8e07c8e10, - 0x4f813281fcff343b, 0x96d9de85005d8c3a, 0x2684dd1b9247d759, 0xa904af25cb3e4e43, - 0x20b767ece53a2ae8, 0x6e3ca764b7779948, 0x708477d973193114, 0xa5d326b737a0e503, - 0x815ad27b52af3a94, 0xf02d10b68d85081d, 0x49cbefabada95380, 0x1ec65dd80d53338e, - 0xec1727216e724b85, 0x8c38c1a29e6396d0, 0xc7db58a979eae9d5, 0x093f44f74a79471b, - 0x332f317b801a95ed, 0x7ea31f1d689e4600, 0x9075b51db8571f7b, 0x56905f2879ba787a, - 0x47df4250c8b6f372, 0x6e016ddd9cca5ba6, 0x3b729f2b191f7f85, 0x7ebf1d2354035578, - 0x74c9f94fe0a76bfe, 0x7cdc07ba9de48b54, 0x6acab0c89e1cbeb5, 0x3db6c5086439d876, - 0xaed7fb6320e18492, 0xe8318a4ce30d2cc0, 0x65ad2db45840ad1b, 0x163f94a3af95a974, - 0xf4f546b7364e3b58, 0xa566fc5fea9d776a, 0x3902f5a8d2395207, 0xfa45c573db9dc4bf, - 0x909ffea0fdf85c69, 0x7e29a46595844de7, 0x61c2bb9b0894211f, 0xfc88b6370d6a9185, - 0xfe65320726a5e176, 0x301d41d3bf9c8e26, 0xe74546c4b2adedd5, 0x71ccf3c726b8e32c, - 0xbf367946f8db52a4, 0xe97732ef63196670, 0xd6b550db7c1e4d14, 0x5f9589b006d2e775, - 0x48e2533301af5f8a, 0x8aae0060bd237052, 0x41e8cef9c466da00, 0x0f266246047c0f12, - 0xdd442b288bfa2253, 0xc306e0532dc61a0a, 0x2711bf71854ea80c, 0x6e427b8703bd8f93, - 0x8a125f8cea261dfd, 0x089d06491e84d00c, 0x181797e9b5e93d4f, 0xe0486a2af3d6582f, - 0x8479be2ce5ab4ad4, 0x2883f8b40eddbd49, 0x680fba8b1618d3c1, 0x2a367b47da9f40d8, - 0x29d9239e48288a3f, 0x321cc7c1b3b8ce0b, 0xb254cd398dd458fc, 0xe3cd431719829723, - 0xfcf6e8cd48a95174, 0xefc88e7100d910dd, 0x48c8b17d4601427c, 0xab2ee81fbf40c9ac, - 0x92d03071c15fff43, 0x13d21338ff8fc415, 0xd1bcc55eec33756a, 0x9837544317c2cdb5, - 0xcd447b5bff31f6db, 0x3f977e5e48aff7b9, 0x05dbec11bd7ae285, 0xb8df878a9c0c7da9, - 0xc4ff5b7bd7197e22, 0x28b26465dc31ed7d, 0xeaefd22fd2ea3c56, 0x22ccf70cd9567ff1, - 0xd092cb564530ad8b, 0x42946234c4a08cfd, 0x14f2837a7488152d, 0x78133c718b217b54, - 0xbb6fec40d774a58a, 0xabb7947810ff2bb3, 0x45515c7c271ce353, 0x806cb20c1da63986, - 0x76d9995f2344a018, 0xb21f1420ef624fd6, 0x44f521c43ed41dba, 0x6f059479b51241a4, - 0x64868f0781f6e6c3, 0x8a67592bcdfb131f, 0xc0411343d76726e8, 0x1133a626e1c936a9, - 0x4e24e6eb21560984, 0x56b602a46df3e8d4, 0xcddf072a40be5dce, 0xf9a9465e87a67582, - 0xf8bcd3f42a6b9df4, 0x292e1bcc1bac0fde, 0x9abf911ec232417a, 0x332f3f0636388b4e, - 0x95cbe58e42520bb7, 0xaaac4fc762c86aaa, 0xcf1074de23e51112, 0x0d920833a729eb0f, - 0x6af154e74e8b135c, 0xa97a2dd50d5b516c, 0xa2d039a4a918e042, 0xec813189d2c95d4f, - 0x43ec93c6c49728d2, 0x675d4f24778bfe14, 0xc57a962d7e4839d5, 0xbce857202c7ab25c, - 0x0df4b462b12167e8, 0xa74ec4209603e47e, 0xf7bbb60bd3e364fa, 0x1059dc45fe259f2e, - 0x9ee1983504ae31ce, 0x3b0c7ffc4213b7f0, 0xbc06fb9dfb7cf49a, 0x00df01dcd325f977, - 0x1d69336863393cf7, 0x50ab7bf1b33ccb45, 0xe6c7869d05b89d11, 0xc939b6abc327d49f, - 0xac92fa5af3dfcaee, 0xad9a6025d859e1e0, 0x015b407fb0061005, 0xc3bcaf4333b73959, - 0xd81c61af6b00a24c, 0x98b5173620d47301, 0xcbb24d874d5448ec, 0x3c23479afce7236f, - 0xd4e09513c51d3e6a, 0xcbbe3ba832d4dc6c, 0x2ecbe4cd103a903a, 0x0af9d12360c1ab56, - 0xbce89100add7f40c, 0xfe0121c129f00b9a, 0x59c371eb5a539b62, 0x367526e5824a9238, - 0xb06a301fd1e41fbe, 0x1008f64a5729fac1, 0x40aa208f561dba5f, 0xa6d5f0ed086a5b30, - 0xf19dea4fc848647f, 0xce6d3402c3c289da, 0xf444dee33a518bb6, 0x7888c884505e3faf, - 0x3459375ed89011c5, 0xa2c72fc9936cb064, 0x931d06964349438c, 0xcffa7cfa11becc6e, - 0x17c4f1b45ad11b4e, 0x6b89f7b164b65282, 0xa85123a88e722924, 0x657aca4c897992f5, - 0xfb10db2725e68b6f, 0x9d2f296d47ca52b3, 0xda7bf1834f9a0687, 0x7bac00138e1ee7b2, - 0xcf099d5a0cf43f4e, 0x02c79a0ef8a1e89f, 0xb2c9f107d6e6f95d, 0x53fc61b3fa69d81b, - 0xeb7498d5a36a7c9d, 0x028b055dbd84c86a, 0x9c1bcbc63d311d1d, 0x8e0daf51ecd1edda, - 0x8b66511e0b6caec3, 0x8d870f8462e282bd, 0x7828ac68282aba19, 0xcf23c72043f337a0, - 0x89773ead20642678, 0x442f43ca81cbca66, 0xe14b137c008855c5, 0xa23578f4bd668ffd, - 0x0b54c97d59344c69, 0x35e3d7bf83b44ba9, 0xc9b2db45c08f020e, 0x14f13318c7848f16, - 0xba67dc4c4fc73e5a, 0x3005dd297ee71e81, 0x95abc79b73250353, 0x30e873f34c51a0d2, - 0x8a6f51caddbabce3, 0x020448deb4588fc4, 0xdd1258c1957e3e51, 0x6f6dda6e29a875d3, - 0x8f27c4f8d1c81fcf, 0x06006b8adaff6205, 0x525c6abbd3a1188e, 0x80246698dc86cb63, - 0x64c2bff746c98f7e, 0x1c814565c06a7f15, 0xefd1a3f664bd9fd8, 0x093dca464b094e6d, - 0xfbaebf4b9e1a0cc6, 0x2e5b8046a170b020, 0x56e40559a96580f3, 0x5afd166cdeae5618, - 0xbc03f598b2bccf26, 0x7131809d54fdc324, 0x978dc5a70c305539, 0xe68f942dd8ccb778, - 0x2bfff105f1029de0, 0x72925e022a362c31, 0xfc16d3fa7fb40ee3, 0x634ae9b82cc0e12b, - 0xc70bafc6e227ed66, 0x860226c77599dad3, 0x7444a24015dbb6ab, 0x685284d99fa0dd40, - 0xa8c8b802d74fdd4f, 0xef91d344e31fb2d4, 0x6f12eb98cdc499f7, 0x9168d7e3464046bf, - 0x93d9db65aca777d1, 0xa1be49215c64302a, 0x2885a0f6966d3fc1, 0xa6a2a73532f66e1a, - 0xd4c6eaf3b5a25eb4, 0xbc874f314c657cdd, 0xd9777f391b27a6fe, 0x3b0648a11a991ec6, - 0xd19e3061acdcf266, 0x9ddd7c6c6e4f06bf, 0xe5d0a051fab2b596, 0xf009eb4191cee75a, - 0xc03ef9a49d8b9e0a, 0xdf42cb1eff347366, 0xa3c92515cb7f4e0f, 0xdf0464e286e5f1c0, - 0x3f73db78018bdc6f, 0xe5f943807bd3a23f, 0x8c235211df0994cf, 0x2cfe072a4e8bdea2, - 0xc8d427bfe56f0932, 0xe5e1ac239e20cac1, 0x083029f74f8c0212, 0xaa3a3ed45a4287e0, - 0x74c6ccb14b430144, 0x4c381cc2e2ddc27e, 0xc4885af1065f2d2c, 0x9586768179db0b09, - 0x89466b20078733ec, 0x4825181d8039ffe9, 0xc87831e1c6d4cbb0, 0x87e034a5741961a5, - 0x29aad84c29f1428c, 0x4b9b21ef6cbc2402, 0xfed029400ade7f5c, 0x814b16c0d0310cf9, - 0x538ba08e935a7427, 0x0e4d59b8e5aa648b, 0xf8dce8be29638c2a, 0x3fbf1001dd155592, - 0xd6081ef4dad8857a, 0xd3e23eb6e8270505, 0xa6cf6761428d3f7c, 0x1e8a9613728bd9e4, - 0x5c88078224adb34e, 0x63984819edb72568, 0xb92397ec15b63ab2, 0x0f867ff3fe5846d6, - 0xc37a596e39b13b5e, 0xe6a376cbc02c1804, 0xf8e312c5e9f281f0, 0x9c38a8541f50f2e1, - 0x881261176e601702, 0xadba05de1ced4c46, 0x83d6d06e8393f1bb, 0xa54d0cc5ced322c7, - 0x50e4ed28b150e074, 0x303d53636ccbecad, 0xab7bbe71bd1fe4f8, 0x5aa844511f3f32a7, - 0xaa1d7b06e930916c, 0x051d3440a772ad0b, 0x7901075e16f8012f, 0x79184a4ebfc0943e, - 0xfdfc3a4f3cd3a849, 0x5cb5691a2c256d9b, 0xce7d8192594dd790, 0x55f27c7ea161da27, - 0x940135ac335fa902, 0x80db23d00d5f82bd, 0x350fa9c3ea6db4e1, 0xd527ca9ffd83dc2e, - 0xc4ddb24bf349bfb0, 0x4f7af90ff0914dc3, 0x2f883972d643960e, 0xb20a54f70bebe428, - 0xb9de59fb8ecd911f, 0xbb95c8d7732b567d, 0x86438156c29305f9, 0x82e32b80d2514700, - 0x4bd94cf9c669a45f, 0x5ab5b7cb35a8babf, 0x9c5b36fd20c00862, 0xd0554b39a8f40dda, - 0xf707f6a43ca75a5a, 0x1a9a99b8f559bcc9, 0xdaa2de48f0fe5d6f, 0x14bd127adbc8a816, - 0xc271625baa62c146, 0x4e64e340f5b53840, 0x385d6908dac9cde0, 0x163df47a03057fe9, - 0xdee2b5d838dc9d2b, 0x5a05142d558c2991, 0xc2392c9bb4f9d5c1, 0x50c02ef42c3ff462, - 0x783e01b44917e215, 0xd602419cc153cb99, 0xdd932aab20c898e8, 0xdf10d6aa0959ad3a, - 0xf9ddb328f5c1a2c8, 0xa19a407437a828f2, 0x16617fc6cafbc39a, 0x2735c1dc69e470da, - 0xddfd58a3ebfee999, 0xbeccc33304da1cb5, 0x9c9f553d8737ca63, 0x051d0761f56c17e9, - 0xfdc01782c17fb9de, 0x3ac70a41eefc8493, 0x3c61ea02eab3125d, 0xddd32725f3661ce2, - 0x3308bac59c80bcc0, 0x611897f580ce5f2b, 0x59661276cb9e8d77, 0xe3835160ebcb5e4d, - 0x121ce9cb7cffccbb, 0x730cd4ca209aa18e, 0xf9b4a030088c8b50, 0xdca207ff26da764b, - 0xc6968dadeda0effd, 0x39a7b133899d0d4a, 0xda0098faebda6483, 0x02b62caeafbe789b, - 0x165f0e314ffeabb5, 0x133f726b07831b93, 0x13d6329715c8f6b4, 0xa5cc40fa78f69991, - 0x7e8adf7afd53c80f, 0xd28b08dcbf47df60, 0xafffc1caf977f320, 0xe362a59206a37580, - 0x5948361c82787066, 0xcf9cd43ec23f2b6e, 0x11b25b15db1507dc, 0x830470024c00c46b, - 0x69169d7705c18274, 0x2cfbe6fa49b4d89e, 0x1315d6a4fe33f124, 0xf55eb7a720e59af1, - 0x61839724995a2533, 0x139e6ee606fe7593, 0xc35fa00db97dab33, 0x421c8be8664f7e0d, - 0x7b71149938068230, 0x2de781c19f52cd7b, 0x5f73ccdfeac85212, 0xddf77578ac96dda0, - 0x1dfaf5f084693ac4, 0x19611464f516bb86, 0x010d84be6d8f15d0, 0xb333b02e51e0b27d, - 0x0357749b81b61e60, 0xbc5d7e234ae42f9a, 0x073fb0028da484cf, 0x7eee16b8dbf80d23, - 0x896c023acbcc551b, 0x7b9eadde5be233d4, 0x2b82bbe948f94018, 0x7a4a53b765b9f3fd, - 0x0090954f291b9eef, 0x900a921f4503f87a, 0xe17f34b5245081d5, 0xd09a1b1da2584768, - 0xe5cce3530b26cce7, 0x373aef871710c7b7, 0xe94c78861c36c7ba, 0xe3c1f24f0a5ea2c4, - 0xb07c4a66c147187a, 0xc30f65a2c415dfe1, 0xfd6fdb3f82a47a15, 0xaf826f25f94543d2, - 0x152fc4744283c954, 0x23259f7f4304d188, 0x90bf7eef8330e747, 0xd4d96859d260025f, - 0xed48f3adfc77212b, 0xbc0fb30c4d95e224, 0x9a430f7ba89a8074, 0xbbffce1fbd79d817, - 0xa8ed6129edcc5cc9, 0xd7ad65c141142c8c, 0x8cce08da486488d9, 0xb89f1b275dba10d9, - 0x68aa1d82c2864477, 0xb0881722bc485b76, 0xae7444b971d53b61, 0xbffc0efd51b676f4, - 0x2fa26ca116dd1796, 0xaa0fcc720ea32863, 0x1a950cbf09155d72, 0x91fe8b751780e2c2, - 0x3d13d82aa4f8c1bc, 0x99710873bb3b92e4, 0x876d1f9b11cbb0b6, 0x0304e7338714503f, - 0x863a8f6e2f934730, 0x693931a0af423f3d, 0xc04a3a35e7752d17, 0x31522c6c563c31f2, - 0x21f944903b44826d, 0x8a43559fe3d1fa47, 0x0adc5f3861f67f1d, 0x4ccbddec9767df4b, - 0x421802efd669a032, 0x72aaf7472d9e9af3, 0xc6e7a6ad415bdb73, 0x95341a1d1396008e, - 0xa3f0bbc3d1b39406, 0x2bbf857e31607f66, 0x6ce1ac8bf06f961d, 0xf2bcf45b995d8d80, - 0x54e40a3490ec870d, 0x0aa305028bc10f9f, 0xcb6e23dd8aa524ab, 0x89ed504e843102a2, - 0xc44f156237de6268, 0x96a1dd4535ac4fe4, 0xc9b5268be5f3f462, 0xded91ddde7655484, - 0xdca6d90e607d7a99, 0xf69c789f8618d96f, 0x37d3374acc11f162, 0x4c8ed7be15f5d3bc, - 0x45d7499ab68c2b9f, 0x3c85a3176518e602, 0xc5228255dd278cf2, 0xf8b0c00d850f0bfc, - 0xf859d9f2be619bd9, 0xd91763b129554710, 0x8ce965d8fce4c3c4, 0x9d8f199b207565ba, - 0x8402ab289268bdc8, 0x1857579e1f375da3, 0xe2556defe2ba50b8, 0xd220c2acf01ead79, - 0x638e50950f2f0992, 0x998f2fba260d197e, 0x7872b743036d708c, 0xbb0d066cbc93e667, - 0x783cf34df5b54bc3, 0x2bf92a590998be10, 0x8407d5b3b17403c2, 0x793ec4b5e96a7bd4, - 0xfad64d79e005de5c, 0xe1ec7d4654583bc8, 0xf76990aabf810fb0, 0x5023c76424773c43, - 0xb29182696b42afea, 0xbcab07ff215f0ba5, 0xa12fdc03e3d1a19e, 0x8f8595eb3a928ec1, - 0x074ddc5b194444f4, 0x70c0fcacafaa9480, 0x83e0e178371d60ed, 0x2e9570fd0cc42529, - 0x492a0233357a5a28, 0xa89a164bd12f8457, 0xd89b3443db0dd2cd, 0x7b17165eee2722e4, - 0xecb4354cd47d260f, 0xd921a0e447dcaf8a, 0x38830949a6843433, 0xf9447ebb37bc149b, - 0xbaf6d7deebf522ce, 0x6224cecde84b64b2, 0x69d60ff41d6a7bfc, 0xdcab06f680908e94, - 0x77f5e3c95351b9ce, 0x475520c75246298a, 0x9266b4df1c61fe76, 0x97015c55bc7e8502, - 0x507e28bf212383f1, 0xe4dce1fbb2212f63, 0xdf60ff51da8bf25a, 0x693d47379fb16217, - 0x0ff9c5b19deb7bbb, 0xb3f2731390cae9e7, 0x24a0a42b48203974, 0x6db7a88f38505171, - 0xda55833025e9293c, 0x99c01baedc1f7808, 0xa5ff89cbf0c51c97, 0x36e87a7337fd2154, - 0xd61451956fc41ade, 0xccf744ba04064690, 0x265e766074582bb8, 0x4dbaab4bc0b50d49, - 0xd14cfb3bcde50d58, 0xcab1725bb6cf3c21, 0x8228ed9ab71d74e7, 0x8869e7da4e2f7dcf, - 0x0e0d9c99c2434d26, 0xf2d9d650471e3665, 0x5ea2caeac7b27954, 0xb3ec2887cc509de5, - 0x940b09171b2611f4, 0x3271b74c068cb2df, 0x51c7437a63f9fa1b, 0xf17bfc2a13f5bf4a, - 0x528efa0a0b29ce97, 0x36202dd7baeb543c, 0x07c3c1a2621310b6, 0x5b8c526656ec3021, - 0xf4bb87a0b8f8165e, 0x9e1f35c47f8f104b, 0xf779bf82f46beb81, 0x7e3a590bb7762349, - 0x17b9a6d328bc4404, 0x26d383c880627f0e, 0x374a0e32bd57683f, 0xddd484552f8eb9b9, - 0x71327b6b7b4d6a11, 0x8bb0ccbdd9ebf893, 0xd2ae7cd1f9f4320a, 0xc38fc9641895f301, - 0x31ba9ee5810819ff, 0x8d7f037da5a7cf4d, 0x13365bc499635048, 0x026d26f28adcd7d3, - 0x848a75680b009f29, 0x1b2a0c16ae8ae71d, 0x0677fde750d81a21, 0xa2ebd26758863c82, - 0x20271da647ac912e, 0x07951936c7cbfc20, 0x4003bfd4ccb5929e, 0xbe82cb93cf5f4fe4, - 0xb35863ae0517b07a, 0xa4eeab0bbedda8db, 0x9a736f1ea3ba0e1b, 0xaf2ed098bf860daf, - 0x47760cd654d8ea3c, 0xd44f528b01739006, 0xd5e9574579407c80, 0x9e3b4a583fb5a3cc, - 0x8272a481a6291159, 0x2645f3fddfa2a9d4, 0xaf246fc054619919, 0x7920b61d9a70155d, - 0xc0ec6f55d82a4518, 0x2edcc831887cb0b5, 0x0040ff0bbb6700a5, 0x0e1d414257b6c7d3, - 0x4d39608340177d8d, 0xffc299f2edef6d8d, 0x8b18ae9033649e75, 0x365a019fe5211386, - 0x91258c69359c0367, 0xe00740d55ccd9819, 0xee12c8065e54940f, 0x8f9d508aa6e31ad5, - 0xcaad6df5ec1c7846, 0x0fb17a85cf0a8292, 0x5ac202bccfaf485e, 0xe97ccbf6c8b69aaf, - 0x7b74907918a17e99, 0xd37266ba63c1d74a, 0xce2031290715de47, 0x3403b73c1727bdda, - 0x1821d6d60f5c2f74, 0x35a07c57b008fa8e, 0xe091c57597b91e50, 0xcfaa6060ae48b66c, - 0x92f1266ef5b8ea0c, 0x808f8429f278d39e, 0x92d273eb0a8ba957, 0x513d57cebab1345d, - 0xb6dbfb612464632d, 0x6adb3a010cd0bf69, 0xb5c2908f1a45ab5b, 0xd456a6f0800b0a10, - 0x6ba1b566ec5853a9, 0xfeb09a4bc1b613e9, 0x11d8d51b09e01ee1, 0xa086ac536a806a31, - 0x00271b2411ea3b16, 0x94b39bea5fb50180, 0x45a04f3ef7e8bcc0, 0x3565e41efe365b6c, - 0x6da5d2a6cb79100d, 0x93c91e2ee5a81139, 0x4c6ada4c38945284, 0xa04702486888b283, - 0x35f2abfb416a8294, 0x29c749e4175e652f, 0x8809977e32235d5f, 0x7ba0eb91e236ef50, - 0x2c25e067e0f8fd7e, 0xf013cc5f815c308b, 0x755740e8829f6e4a, 0x56ddc51b84a07d0a, - 0xedc2af6b380a032e, 0x73c841ade30d7e64, 0xd8757e5de44d001d, 0x438bf6478251b977, - 0x930a28316b236b6d, 0x66d38a492d97694f, 0xd4724d57dde20a95, 0x9d276c6cfa11221d, - 0x6044fa85aabe0930, 0x75c07419d9d0b1f0, 0xc6da5f29b5f1fb31, 0x67dbd9a4d8f1f218, - 0x38499c2cad04abc5, 0x5f99d46c1f41bc8f, 0x91a4806d717258f9, 0xe2207201aefcd8ef, - 0x75f9aceb539d7a13, 0x6a3c120cc4b4edf7, 0x6657b5aa65bf7b07, 0xc5344d4320182039, - 0x4152fe780f341c83, 0x85441680b7ef6ac8, 0xa6b6425d279aee4d, 0x6f310f8727bebb57, - 0x070064f13f2bc818, 0xf961201bcc0d948e, 0x74a0186f6d95845c, 0xe326ba59a72533eb, - 0x06234a0bd0ae3f02, 0x06b1f375f09add05, 0xc74930729dff87b8, 0x7bbe4623b672ea51, - 0x95149656bb78f735, 0xc5a6fc7b747bb2bc, 0x7fadf9e2d3fefd7d, 0xdb58af985690f1cd, - 0xdab71092a013d481, 0x411a96c20ed3ea76, 0xd7775aaf3943a822, 0x95a8c53bb2da5dae, - 0x90c3b1eff76cf6b7, 0x53211922add0adbb, 0x944515b3428ef76a, 0x708fcf0f0eba9774, - 0xfbfd76c4d96e6d51, 0x3c64af3335028c80, 0x696fde98feaf856a, 0x5d981ceb593c006c, - 0x74d6d0f0521531c6, 0x31255e5bc6630ef6, 0xd156025539ce28f0, 0xc53a700a404cce5b, - 0xe2fffa50d96af07c, 0xdd24b7c86cf9dd02, 0xd7354261c373fc6a, 0x1d56f89c8a5c9bc7, - 0x33a7135f02921c7e, 0x3a5319d9b1a1157c, 0x941182356070275e, 0x211c6bf9203e488c, - 0xde48c6118d6a8068, 0x819fb5f989368755, 0x2ff90e9192f90693, 0x9fa86b348e8c6ac7, - 0x4843b8dd2e186336, 0xe39b352e5bc6c40b, 0x44e2593bf8b46f9d, 0x29f3f1297917dbe4, - 0x025bcdbc636f3c7f, 0xb04683cd5aecc72c, 0x7c27b9f84bc42d2b, 0x12481ebb3d7429bb, - 0x638e311128ec1ca1, 0x045ea4af0da3304d, 0x8365a09cc4fc50d9, 0xff5a758aa9bb4d5e, - 0xeb44acf4dc874d35, 0x4b4df65e70d1c43c, 0x2abd42898043c9f0, 0x135335441c426e79, - 0x8b7dce6afb7c92a1, 0x08ab46f36fef1396, 0x16ad4bf8497aed38, 0x3d7a45838852dd3e, - 0xf929bb3a1197c421, 0x12e7430cfc75d34c, 0xc350fae878a51ab7, 0xea92c9f9066605b4, - 0xc2e456b5d293a1e6, 0x82bf1d469e473fb2, 0xdb8a9ef6060b7431, 0x84d7d518a687aaa0, - 0x7511aba6d72ab3e6, 0xf1803a219a0e8541, 0x6671e0b711d16ada, 0x1fe56df3dd806f34, - 0x123e3e99c75584a9, 0xfb2912ed58114083, 0xa567b1be7f0d6ca6, 0x158fd724ae15cd42, - 0xe4e9093dbb2251fc, 0x693066c64a877938, 0x7f0a88da9e49d69e, 0x85fb2e8c884211f4, - 0x16b3703d61b5fc07, 0x10f204ae223a3bb9, 0xcea9b03f769dc6af, 0x26a85ba73d11a754, - 0xa736a3990cd8fba1, 0x3d78f3fc826589c9, 0x6af351aa1d5af8c5, 0x9789da391b4045a6, - 0x7dd80e45368e8f15, 0x55bd8f69c6c4a04f, 0x63701e126f7f7555, 0x0f4eaa6b192e7b68, - 0x3e60f97317c3355c, 0x3d8369696399d954, 0x811db425ca1bd0c3, 0xce60f0ea613717ad, - 0x756b4896c9fce76b, 0x48ff48057e170322, 0x1e674debc605948f, 0xe8ebabc592c96df5, - 0xa0baa2890072593d, 0x3d5ed05ce017bd30, 0x2d3a4321f9e14bb0, 0xd8b6cb333adeca62, - 0x56ef35032f9f8550, 0x14a78e964178dbbe, 0xd22f28cf4282954a, 0xfdb769b92fb886da, - 0x456419eb131cdad0, 0x8bed7ab49a0a975e, 0x47f3344e8cf0bd23, 0x89be455d73df6c65, - 0x6591c78ea6e9cdf5, 0x62f3ac8394595027, 0xfbbd73e89d04aca9, 0x754ad3395cc1703a, - 0x2c631059948a10c6, 0x70444874cc7a5bd7, 0xc42433386e225787, 0xf623d0f29219d8d9, - 0x7863ac3b2da17b77, 0xf974847549705dee, 0xe1311d57f9ff4c29, 0x4d8d8e511d37bc5f, - 0xc710841563d5653c, 0x9104f0ca4619cf27, 0x6c5a6c9cc63dbd38, 0xf206bbd60fb4f8b1, - 0xa46ab141d503ce00, 0xee0b3cb552228988, 0x43ff2198f7e7f130, 0x7acc681a85338892, - 0x30099256bf85ab02, 0xf3a19d21e796b48d, 0x974e069f4642153d, 0x951024e9d8c1c0e0, - 0xcb98162507a57526, 0xba7bfba106f8915c, 0xd689a5916572d2e1, 0xb04d938c0bda12a8, - 0x312ea88fa00e767d, 0x5ceb9f376b2a1458, 0x88ad23a48716e444, 0xb357a4055cb6c755, - 0xefbfa99f519b14aa, 0x05100ea0c2c5bed4, 0xbd5d0ed83008e7f3, 0xbc808bc4f3c5a3be, - 0x043ba08eee404167, 0x48da0a5381c0053d, 0x8e2acfab62d8b2ec, 0xcbb65307060907f2, - 0x8e2e03d00f7c3b46, 0x7bf1342e6c1eb4b7, 0x79326f91ce2f90f9, 0xcf82c22596bf087c, - 0xbab0205f3e37d6d6, 0xcbe0a2bbbbb6da98, 0x88df65781c3eb49b, 0xd3b3af5c9881f577, - 0x5ff9c025b3bfdbf3, 0x1bc44d3f4bde0426, 0x3c7ec7faaed59e19, 0x28b3338b3346ec94, - 0xeb3013c3c63271b3, 0x72cb71dd6446654f, 0x9d77f015f5180a7b, 0x04cd8c08a9a7759e, - 0x68f6648046ced404, 0x005c821877f9a0f8, 0x60ca2c971cce3888, 0xe7f67f893a21c0dd, - 0x2b86e3ea4feca4d5, 0x9f4f45daf48f8733, 0xe2c504d1e437accb, 0x3a1b9b1fd301f763, - 0x130cd4d44d922a42, 0x947d0bbd5de87b2e, 0xb82d109d4d3c2e21, 0x15483db4eaae9858, - 0xce64e28b31523814, 0x3a3fcbebb8d65a17, 0xc366013c9b823c80, 0xca98f47fe9a4b5f0, - 0x0500e33d24b127ed, 0xf311e5b0c89a63d4, 0x5fa7bd3523d869c4, 0x39177c3c2a86174d, - 0x92cecf2593315805, 0x167d95526c5d5c8e, 0x538da689aa9a8bed, 0x5ec455f011991ea4, - 0x19088f17d2bc0219, 0xc9a4a3cfb111acfa, 0x3f188df22fc58014, 0xc7931b5dbd7e673a, - 0x7920b3afdf7677e2, 0xe081f8fa266d4c5d, 0x0b6945311135e64e, 0x333e25be4f7f644b, - 0xf26668cc1020fcc3, 0xa657a29ebb251907, 0xb5a87f8ea7c160b4, 0x0ac7cdf80bed35d4, - 0xaf99003f68236072, 0x594f8bd296dfbbfa, 0x347603df94903d61, 0x6a76802e6e50b807, - 0xafda83c584773d4b, 0xe67c98523866f9e3, 0x65a1d20b0f3f30cc, 0x1ff89acf370ee8a0, - 0x703f8476e5aca25d, 0x85113e014c2cf1ed, 0x158f0d836d09f6b1, 0x222ccb0cb5b73fc0, - 0x9d91168520866ba1, 0x01c832c22023edf5, 0x821b9950192046c8, 0x29e2a78b9bf4d684, - 0x839a1fe01a24c654, 0xdd125c5554c65caa, 0x2dd60754202b1373, 0x78ee3d9e93237add, - 0xad1a23444173f4c2, 0xdadaacd0cc99129d, 0x951faf617bd18088, 0x6d5fb5c0072ebe96, - 0x46bbab0a7e9d3066, 0x7331eb5fd19cc432, 0x672b8bedaa55f6d5, 0xbb856f1736d7a84e, - 0x4997d67db772a9cd, 0xd9636095aa677b67, 0x4f19a6d90385d33a, 0xabde735cb2ecb18d, - 0x2b4021056160cb51, 0xb915d9d52c68f2d6, 0xe11bfc79b430fed3, 0x01ef451db7d1d49e, - 0xc8152fbf2fad5342, 0xe08deba32a30d49c, 0xea8f4b3cd2a6a5f3, 0x8e9ebf6876efc454, - 0xd9cc3adddaa306b3, 0x4761850006eb24b6, 0x9d7125e6fb08f8ff, 0xce982936473f4f56, - 0x0afa1f01b49afad5, 0xd4939e7cd20890d2, 0x1184d45f45a0bb7c, 0x81eb0f8215cd5958, - 0x924c56059ba64c6c, 0xfe93259a01693ec2, 0x1d4b1b17843f0caf, 0x9c5146fa7cf9c947, - 0x8d3b9b3112a98622, 0x1cd73990a7fcf454, 0x41869db4247aed96, 0x69b5e8f181202ba6, - 0x2bc3e697d630def3, 0x2679557706f067f1, 0x49826fe4040975c8, 0x82b0d2f2af7c8210, - 0xd10b7e6a658239cc, 0x46670b29242387cc, 0xf35390ad05300ce6, 0x62575308ff2a442a, - 0x9862525d43d07bc8, 0xc9c46092a5ab0841, 0xdbcb53b9d2c6090b, 0x4ce9e3cc3a3460fc, - 0x0b0ac10e618d4ffc, 0xf23e6d76006d6dc9, 0x16232cf717d0fb84, 0xbe2556867f968aab, - 0xe0376d7c1f7db74a, 0x7a8b4197f3995d69, 0x52ab45c5e1afa41e, 0xa8a5171cae8d9040, - 0x688fb5f37054de63, 0x945c0472b76db91a, 0xcfd4aa2875e188cd, 0x252624a7ec793b96, - 0x8bb4f81976db4f82, 0x506aaa3ed069ef0c, 0xa87f85e7cae641ff, 0x91a44d5c2b24e4b2, - 0x5c0d96bdc7e06d49, 0x29634968c7d9390a, 0x475bab5a9de0a68d, 0xd860c14528fe7869, - 0x93c9cf184cc29ddb, 0x9999df6af81beaf6, 0x0f8e5eba505b5eb4, 0x1bf1c5ec9e022bbc, - 0x8da274bbc2cfa328, 0x64b6b450c00936de, 0x1aa683efe64bbafb, 0x65374861c906cd92, - 0xfc7dc9a96e2f9af3, 0xcb1c8f379b8aa99c, 0xa828a12cd72b6f46, 0xf4a8cb20c5c81808, - 0xe1c8509c894f9658, 0x27f6c8a8f029f1fd, 0xcf642e9d8b676293, 0xbc8981ceeead4ab9, - 0x9ee89a2a0d428070, 0x1ffdc8c438926f89, 0x79ee8615877f02fc, 0xafc33efde621d1a3, - 0x9cca0c08750a4abe, 0xdecb33b434cb80f8, 0xe4433ef13b1ce5dd, 0xc5cb21343f2b9ae8, - 0x09e77af5da3b4236, 0xf98ba451dd7a41bd, 0x1fe44edf92767723, 0xfb48c3791d4a4537, - 0x728a41972d39cbe2, 0xf92fd3f52f2401c0, 0xabb622c744100316, 0xaf474266fae92641, - 0x4f78d2718be6689b, 0xfe80ee10a758727e, 0x3901c7b96362545c, 0xd315a1d07b59f441, - 0x36cc885bced72e02, 0xe6e0837d1daae373, 0x20e3a2f455e93f17, 0xe61e085fb412968d, - 0x9855ac8513613ba2, 0x76914016c0ddea45, 0x83378d7f4181aa02, 0x12cd6c7d26d449ae, - 0x0610b41d77e39f52, 0x152b56ad03066fd0, 0x54b406161d4ce266, 0x835e05978ace9c3f, - 0xd9a4b6d98ba0781d, 0x0d2d0a2851b77d75, 0xd3e7afe8539341de, 0x007639151eecb098, - 0xaeed0e267b68f0fc, 0xa902da2e13b658fb, 0x430f488dcf8581f0, 0x2c924174f82ff2ea, - 0xd264cebe790b82e4, 0x7375956fa720560e, 0x287fa0a7f2956d33, 0x9a023aa5b62371d3, - 0x2a96ef68776c87c7, 0x21978bd44c54c70f, 0xe4dc757743ebb080, 0x6d97494df9e5f95c, - 0x066d726c54c3fec4, 0xd71d6552a27ea73e, 0xddd3df8123a035c9, 0x827449c03c292121, - 0x5a82a72ce82b1b6f, 0xf84d27daafd73fc3, 0x6f2a181866577e6d, 0x9f6bc2cb89320c69, - 0xcf8be96b5b1fa761, 0x33277a29b8a3be29, 0x0e09cdbe68c2e7bc, 0x34d99fe2a7cd7dc3, - 0x848b83123a91f836, 0xe0885e2b4367b50a, 0xd70e37311d410a95, 0x7a2b7f6eb15757d9, - 0x6cc4c1ded49786a3, 0x37d1ce40640e48e8, 0x0d631fa500c5c6ec, 0x88c3573ffd522b5e, - 0x46a7b1f0c7e053b5, 0xf5353d8418b27b87, 0x4f1a08fa84a5cb18, 0x3124e26187e67b8c, - 0x39231c72ad9bc640, 0x93adc2157aa4cb4a, 0xf1400ca0f7289ab2, 0xe2d93855eeb6c591, - 0x3bdecc4177c634b9, 0xb1d04d7a5810c581, 0xa011ae3e7aa7c62c, 0x99c6863a6476445a, - 0xa2e5d9811e704882, 0x660a9bfd7c032c91, 0x1299aed8f59fe22c, 0x0faa651d92dea160, - 0xec626cb62d59846f, 0x0b78fd17c18130f7, 0x5bf19b4b6ada6f7a, 0x335da108ae82a51b, - 0xbac0d6dcd378160a, 0x8b85ba5a5fcd56ce, 0x8ccad6332b2520c7, 0x9193101383dff7f9, - 0x692381021347d487, 0xb99d5c4555fc9656, 0x9f9d653012e0b9a8, 0xf0fec44ee8e3a9e8, - 0xafbc161e857aede1, 0xc861d5974b8ec38f, 0xd35b0e08153d8227, 0x1329c83c17ea2f98, - 0xb37671b74069beae, 0xd1dba3d86b20b392, 0x0ede977a030e37bb, 0x6132d7d961238ff0, - 0xb61237b9d7d9cb04, 0xacbc31af0de7beef, 0x6ecf35bddf10953e, 0x72495d1023e562cc, - 0x0d59e2bc0464e025, 0x4689875c9c97e731, 0x0000e9094f280665, 0xf388e71061169357, - 0x27eec6f2c98fa92e, 0xacec9f264e0819e9, 0xfa76a40a7b409599, 0x073336202745e229, - 0xb9fb60354815573f, 0x07d7f6f5aa902412, 0x35953135b3c055b3, 0x38bee00310bfb808, - 0x13bd214be1e618f3, 0x2fbc9d9d27bcf92f, 0x3bb7966ad020dc2b, 0xb6672cb57125b5a2, - 0xe208b4552b0ee534, 0x8c25b40fd81e44a1, 0xe35944eddb676495, 0xe765de9c752482d7, - 0x7d8e098e615521af, 0x1e400c10cb5cf038, 0x551fe78c7a7a13bf, 0x468e05a0dfec2dca, - 0x0a4b468d0a3e4676, 0x8d718c5c609eb027, 0x4b0d9fe48bbf3e1d, 0x0874133f690566dc, - 0xff1d5ded64da5c55, 0x532a6d4e1959ca49, 0x9562bdbfa20c5df6, 0xf41c5bb19c152155, - 0x3bc72a305a9c46a3, 0x6ca44362c39d311d, 0x4fbc94e7f381f018, 0x7fb00a1121aed419, - 0x4929ba12480ba73e, 0xd13324d7e1ecb90b, 0x33df7e886e2b8426, 0x620e25127b0a75b3, - 0x833658ad16d27bdd, 0x5eb53842c1dc1abd, 0xb526e71233ef1429, 0x3a01c24633d0d32a, - 0xff6ab33b12e559da, 0x3d1fa997fefb14a4, 0x58fdf931e359fdd1, 0xdae87d0a877dbdc9, - 0x2b559fe28d79ebe3, 0xe761e9ddd8c4a297, 0xe9a63c7d7eaf12d2, 0xc9ed537205e1ac10, - 0x2bde0b4feda7729c, 0xe4f18ada76311439, 0x82b892592510d7b3, 0x5d8cee05c472072b, - 0x18a53fec6f337fe2, 0x60275333f197418d, 0x0db0fe9efbf0f37c, 0x6a5c6d2ea787ffa2, - 0x5187c6d316964188, 0x4ebf4b3f9dca23dd, 0xfddbe23de6c38804, 0xa298cb8bf95f36de, - 0x8f7987fbc3935c12, 0xe24b1156c0f2742a, 0x4b32e9ff75677667, 0x15c6cdf36f12d3c3, - 0xac5270a3318c6a4b, 0x651dd822ff6ebfd1, 0xc68b2e08b497a6c1, 0xc43ded5938b8c0a4, - 0xf2bba9ac4118fda6, 0xc29a8bd9582cb066, 0x4fad577274bd735f, 0x2da00fc13f14f256, - 0x97525069c9edccc0, 0xf59363b070f0d092, 0x503aa30f7355d890, 0x6e1d4d291bea2222, - 0xa1105233cd692269, 0xd9fd7d8fe5e61214, 0xe5e1571a78898fe7, 0x321e99d9ea1d3029, - 0xbfe388d8f873ad2a, 0x00806248e903f63d, 0x6b0fe4197bb3d19d, 0xe755e681146a2096, - 0x4fff7adc0eee3390, 0xaae3391991cc36fd, 0xdd3d7148b5206969, 0xc4a5e397b9c72cbc, - 0xf340ebca0150c739, 0xbdd6a499bcb6c7fa, 0x945c46991d9fc470, 0x079d6b60b7bd8480, - 0x1c92c3ad0a66b8ad, 0x5567f02ecefed2df, 0x439d96ca947a0be9, 0xe68a0f6d367ff86d, - 0xe3f371d72cf2a428, 0xf4dd42b7e6c0ec21, 0x2d4650fdc0f4d7a9, 0x2674142c9d9da001, - 0xbcf940b8522ffe38, 0xc5b71b5807bcc397, 0xdc5584bdab79d857, 0xa8ac6b2b067fbdab, - 0xb689adda503612b7, 0x085e4044fff668c4, 0x9caf0f0920a0f708, 0xb28f7a871c89193c, - 0x754770da5c773557, 0x598997cc5214c327, 0xcc51d6e04cfa2851, 0xd4108de786c72f74, - 0xd96aac81b6fd6e92, 0x7b15c51f843356e4, 0xc589928a613c40a8, 0x9315701d92158198, - 0x501876d8c9c016c6, 0x02875d52d0545f9b, 0xf485a603855b1fd1, 0x9f87176ca08ea6eb, - 0x18069187e6dc5585, 0xd32b6e850541be76, 0x2d765922b364139f, 0x65cff3de48eb11ee, - 0xcb04c6e4975b9f85, 0x86762558d035dd2c, 0xbc809d7d0bad9f7a, 0x2bf396ada69b8569, - 0xc1b04349a8ccc91a, 0x1a09328bcbac921e, 0x826127cae83ba7bf, 0xc1fd60bf480975d6, - 0x17820de74cace639, 0x0f13f947749d3830, 0x241358cc5bd02e8a, 0x57da67775b3fc5cb, - 0x648b99295b8b84fe, 0x07d7555282349ac0, 0xe5f56d2fd5bf113b, 0x1bad498690192b1e, - 0x025df4fe038e041a, 0x0ae4c5327a963888, 0x74de8a9e7d0f8c2e, 0x4b5a994417ccb071, - 0x7ef8f747a525c5d6, 0xd3f325f295a3a1ce, 0xa4b6e5a83c3f6da1, 0xd6a580809ba36ffc, - 0xbd99c925d2324ad7, 0xffc5021a4bc6bada, 0xe266a3277325c8b0, 0x06d1a245d4ec6e16, - 0x893365ae33bf88ea, 0x190739b7e49ed2e0, 0xa80e6022c6e7f182, 0x5822c7a0ca206110, - 0xcbb0a7b35798df3c, 0x546f121c6ace3652, 0x90305fd56f089f36, 0xbdb96bc252013109, - 0x2e55f4ba753401d3, 0x48b91e63882d57e5, 0x64d47e9672073ebb, 0x0bc4c6cbb16107e6, - 0xfdc4061d5e34c8c6, 0x89f8c2f10c0a492e, 0xb1ae3535e2e81b23, 0x199aef7a8ba0633e, - 0x9eea1f7d6e9eb3d9, 0x1ee5fdd29e2fb04c, 0xb2cfc080773b2382, 0x9c77da06576f817e, - 0x2603a12c53ee968f, 0xe0c29056ad915f84, 0xb0b80a92453b4294, 0xf12288da4465ed5d, - 0x05e6449fc4d591ba, 0x72642c50c1f353d4, 0xa3e75aac1bd91301, 0x5ea6875dc4945a6f, - 0x4e8db7ce56ee10c5, 0x63d882bb454cac46, 0x7fae3c5c5cde9baf, 0x05a2ad4d91188440, - 0x7739e9e364e430b8, 0x8b27483b3f72f77e, 0x7a84d9ff9da81d01, 0x7fe7a8f25a6507f6, - 0x104a09d66ff8aec3, 0x221fccccee3bf002, 0x8f456c566864b491, 0x2a38bc19bba083e3, - 0xca4df86ed7f6331d, 0xa915c2e70a21c466, 0x1b0a6310b07198f4, 0x880205f37b38db00, - 0xabfd5dc088f28724, 0xb902944c8843a56f, 0xea21cd710b4b0941, 0xa08aedf5bdc929d6, - 0x13fdca5dfad83c9b, 0x25be3d6caf6fb5c9, 0xce657651331f107f, 0xbb8fee7065e57110, - 0xe5deef7727e9ea0d, 0x85a4e89986b1b846, 0x07055eff7acbc66e, 0x85f8c65f27b69203, - 0x1d51cc06a246676b, 0x6da38092ae38a8b4, 0x347cdb985732cde8, 0x4626d721dd731972, - 0xeafebbc5fb211c7c, 0x87b8470bf46d19ca, 0xbcbd4d982c98e52d, 0x33914aa83fe154c5, - 0x8544d451754a6d58, 0xcfaf07cc2412f46b, 0x5ec12a7822086fdf, 0x04700feb313994f7, - 0xc4f0313beeb0edf1, 0xfe7b5b03e1edf878, 0xc44e597dcee44416, 0xf6a4732fc93cb6c2, - 0x70d20a4137d58e15, 0xb1ec5d5d9abb923b, 0xfce08de99a26e4f6, 0x3a651d5a87af493b, - 0xec72cf6367eb6c89, 0x2c2a91d3a2244247, 0x0c06629cf90ce339, 0xd990235af69a95a7, - 0x94c66316a1895e82, 0x643ede73df79d9e2, 0xe9a1b799a742e55b, 0xd3013304870339ac, - 0x9a38898b6872ad4c, 0x1d8a6521f5a39304, 0xf11fb450b68f1874, 0xae6de3dc2407fdb6, - 0xc831ba678cc7716b, 0x1abb10a06d30dd78, 0x32c9fd5a1984af48, 0x0774fb36a08321fb, - 0x17eca8e7c1831760, 0x2d3058f5a8e11c20, 0x24e07ec632d50292, 0x07a60b2fc2bfc887, - 0x8aca454fe34d5c6a, 0x1433b480d7346a6a, 0x0b8e726318d6b7be, 0xbf64250506fef9de, - 0xfd52082d0dd17caf, 0xfaba2ef781cd6a7a, 0x42170fe9288c52b0, 0x7212bae00badbd14, - 0x373943bcc9127baa, 0x06ac6ef6146dd0ac, 0xbe3349c937eed4a2, 0xed1c6046dc5416f4, - 0x5eae655572e10bf1, 0x1351db63fb13efe9, 0x66e9dccb6b27bbab, 0x8095b4e20c66e765, - 0x7a7cc44bb67d697b, 0x9c5892c5b383733b, 0x5e4acb76cef38e33, 0x9abfebea5de73782, - 0x49b68c4b47b2cd98, 0xf15cb68197be9d40, 0x5e6f77d022d111ff, 0xc2bbf876c1f27342, - 0x6f50665c0980d3e5, 0x83465806192f276a, 0xf3c54e0ddaabf129, 0x0b672acd1a1ea896, - 0x8d1cec1cbebd9da6, 0x1a52837f5a372f01, 0x2faba87e32c2062c, 0x74a14286eb70ed23, - 0x47918a2a0769603a, 0x4547f51b7fb424ae, 0xb33be6d28027942e, 0x8e44b22119dc78b0, - 0x54f950241fec7316, 0x9246c6dbe38825c4, 0x3483b15a04262936, 0x563ca7bce7202412, - 0x00ae5b00bcad778d, 0xa90c95a7afbfcc8a, 0xa586965961a78056, 0xc86d7ca2c6f20029, - 0x1cbeafe9f9ecb985, 0xc076be90209c7e33, 0xae19068cd6d9f825, 0x886f507cc9eec63e, - 0x97e563026294f8d8, 0x56dc8679b6a592ca, 0x8c6d4f3a5c878f1b, 0xa10da022f7bd8bdf, - 0xb8fc5994d531d673, 0x90d6c11921577f88, 0xf17823fb973df077, 0x5067b9a780c2b4af, - 0x69a32d12ef5410b4, 0x0e45276fb4b7a126, 0xc6e974204bbfe249, 0x572d95bc0cdcc601, - 0xd09b0e69f0758cf4, 0x8eda90918d18b26c, 0x6a0e4f119e5dab83, 0x6c58e5dad45ca8fd, - 0x3d5cbe91ef0adff4, 0x225784ec557eb6ae, 0x4d5a0c9bc888b68e, 0x47f3fd705131fb28, - 0x2e7b6fc01f5f1ff2, 0xaf8957acc909a0bb, 0xed49ddafcbf79b6f, 0xca89d0432a68bfab, - 0x7f4fd500fd5dc725, 0xd029e34f012ff05e, 0xaa43c4a0cbf2d862, 0x0dd9c348c4a10af5, - 0xf3bb564a13990178, 0x61cd5392089fe43d, 0x064b2b9ec49f6376, 0x7a77053e241cf322, - 0x0b0bab12bdf3e0c2, 0x266c4ac6c7b9fe69, 0x442ae1889137211c, 0xf23dece399bd36c0, - 0xd172bd1e5633899d, 0x70950cfa8c8d4b6d, 0xfba42c430cf138dc, 0x646fc4021f2c7d91, - 0x7fa825edef3a9667, 0x4ebfc87c386f3a37, 0xe41f57cba74fd040, 0x5e8b0f3c6fb5df91, - 0x27d4be4c2be3b874, 0x623a2e90a791a3ca, 0xff1b644bf8e0fd91, 0x50817140186def61, - 0x0883ad49f5e4196b, 0xc79625d083449f59, 0xdba819843a5296c8, 0xfda3c04733d456f2, - 0xbd3899cc21144b45, 0x688fc6e4964183c8, 0xaf02290f6ecbe50d, 0xeb11a01b5fd41e5d, - 0x6522873bb28b527c, 0x6ef8c0b10f97aead, 0x5c3ebc27407850eb, 0xa1fd513b2e3ef020, - 0xe14d489a06a49bba, 0x1fd2a8a1bc1d81f7, 0xfca58cd4ed95a112, 0xf02cdf9df724aff4, - 0x729b5217df995cbc, 0x590c118548cbe445, 0xba85bcc5e6f03f72, 0x8d519f8a82458e9c, - 0x137dfa0262d2dc0a, 0x366c3ff3266b9d8b, 0xa40d53a0e14b4c8e, 0x3cdb908bec30b72d, - 0x85217ea75d29bb86, 0x831252473c340533, 0xb7413eb2333b4655, 0xc7e2f0bd5bbefbfd, - 0x75f2bc68d5eca377, 0x3f76ce965180f95d, 0x744307274250190d, 0x9b71e67608a0db0c, - 0xf6587e89b22eb31c, 0x5cd1d79551cccba7, 0x0cae07b81be35ad8, 0x7cf57ddb13b8a35c, - 0xe080bfef828ad648, 0x25c495491887efa4, 0x8f0ab2d3735f106b, 0xb79480ed1cb74039, - 0x1beed6bdb4fe45e9, 0x73e21cc0af3b6527, 0x74972fba1628207a, 0xdac8bdbf762bc34b, - 0x1f67156057b93027, 0x1f7d9ae945c351c1, 0xeee82e575dcef68f, 0x2e6e9afe066955ae, - 0xf1b80f947a919f54, 0xc7cac7e9ea0deda2, 0x01b90ba9287df438, 0x254345d52c896fa9, - 0x4fd93a5c442b881d, 0x597ccc01d842569c, 0x8ef6d00892e39c44, 0xcb72872409e97a26, - 0x94e62a1fe5f29368, 0xf40650bbfa1b916d, 0x6a9997aa13bffbf4, 0xd4eb2bc552fb90bd, - 0x12ec524718bbc05e, 0x1a04b054139da5a2, 0x3a7411de644fe3f9, 0x85116ae61cb39ac6, - 0x32d377f4f77a2adf, 0x2bbd9a82dbdc5fca, 0x3b403aeb6647a7aa, 0xb1242a7e1d54971a, - 0x8a1a12ce640f536a, 0x84cc00462b2e9a05, 0x425bda357cc6b80b, 0xb79d2f30bd32f162, - 0xd6d84aae9eaf237e, 0xedba93ab2686f5d6, 0xc3ae4c1be1332f21, 0xa62438304e19083e, - 0xa67396573a33a04c, 0x01551985bb53d0d4, 0x8d34936482e27d4d, 0xd402537bd54c1a9b, - 0x7156c31916a277e8, 0xf74e523cbe729c55, 0xe26a9d052770fa14, 0x8e69234fcb340f75, - 0x81d3df4bdca859c0, 0x89bf13eeec7b420c, 0x65b053bae5a53d9d, 0x1a67e49172692fe9, - 0x0f45280e81499413, 0xc51d0f0155760914, 0x0a5b252532e4ab49, 0x0a2dbbdfb738c5fa, - 0xf74c88e8b66d9c8f, 0x38a582534e62b441, 0xf3dd3f3230a5908f, 0x39191a50a489823b, - 0x88400571618a4fa5, 0x72525c9833876892, 0xdc9cf45181bcbe4f, 0x25dd68c151a9b9f9, - 0x81822c3a0f8c122f, 0xf7f60d3887d0c3de, 0xdd31955783566f9d, 0xf36eb6b8f8426c34, - 0xd984d34e1b023be5, 0x42fc4944ce9c6a53, 0x07d3c94ae15d36d6, 0x872305ce57b403f8, - 0xc40441aa91af1357, 0x89704975bb11ed3a, 0x58a8d8a371fcd221, 0x07f2700ef9ac4c2e, - 0x36a18f8297bfd4d4, 0xf1242a374307f169, 0xda186903ba4a6586, 0x7fee365f8f7a71e6, - 0xaa1512a0322ac060, 0x845fac97497eefcc, 0x388a56b002461ba5, 0x1a12cc2c9a191065, - 0xd64dcd6876b63d7a, 0x58287eca4eba17bc, 0x6ab5bd86c45f84d6, 0x238956db4225d442, - 0x6e6d52642b1383b0, 0x5fc7aff410d71819, 0xf18194fb89ac8823, 0xff6e2ff25cb5a049, - 0x255aaaeb81e47d09, 0x07532a160bf92e0f, 0x0c7f7b51f62de239, 0x15e9a0f3ee90da89, - 0x595937bb8cb074ba, 0x70bcf22d0f3423e4, 0x66a30d62710e8367, 0x925f2734559f5d0a, - 0x0f427ac20056a06c, 0x94f1d2dd9061f3d1, 0x35370d6ba793f72b, 0x1a177efe275108e1, - 0x02808a69be95d5b4, 0x99bdb888d68204c3, 0x20ede4d884601290, 0xc29ee24e32bf260b, - 0x6f4e51ee9373eb76, 0x91d710f82ba44dfd, 0x010c106fcfbd4c06, 0x86d7d5e841f4375f, - 0xa75657b1f7c2e3a5, 0xa011de1a131a9851, 0x0a1b7c2086ce6760, 0xb32680d6582d6b69, - 0xdbb6b14b8d826704, 0xd55ed88184e9be71, 0x379a238613b9a92c, 0x6f36257ba6a78708, - 0x823af9f73afa8159, 0x4b609fbad69f1234, 0xaa0bb117ab301fc9, 0xcf0fe0055aeb6f54, - 0xf729426bb380f099, 0x878519050169b573, 0x45c986af1c498099, 0xce4e0dc3a12b51bb, - 0x82920776a5135fb6, 0x336f34c4f1aace7e, 0x9e4f738169518155, 0x39b7c9f32f575dd4, - 0xc5735c782fbf8223, 0xea51cece535d0d15, 0xe472d99968a3cc94, 0xfff8033f3adac524, - 0xa46b453910b0be8a, 0x7ef14fb4eac9a562, 0xa987d6a6d7178e21, 0xcca1d4455ca9c3ea, - 0xfe6990bbbfe37f4d, 0x9d8837ead7cfceca, 0xcd5d5804efcaea7a, 0x5d72786dbe6ab8a6, - 0xf1b0d2f1158638af, 0x5b79e508ed9e4f8e, 0xb307af9641e8779d, 0x1aecc0bbf5482d2b, - 0x9de4d831ee050ff0, 0x4dff44d028f0e5c2, 0xd2735b010b7b4cd6, 0xb2fbc9e6fc5c272b, - 0xfc80d65bdd52def7, 0x2143aa1f838ee365, 0x4192d91fa0481fe6, 0x706dc068d535e3c1, - 0xf77a44ce6ab54d15, 0x4bd995e78c224550, 0x7769696268eff391, 0x1fdf16ae5aa64b85, - 0x90af33ff6768ab0a, 0x6d2c3da47455e27d, 0x085cc726ab2d0bdc, 0x0f978107f6f51755, - 0x3fed3f76c846379c, 0xbcb2f9f060cd8b08, 0x6c73cfd3640aba02, 0x4ddfd1986e18efad, - 0xe5e2f79152d49baf, 0x3415efd12db9ea17, 0xdee5c7c8fa3144b9, 0x9138424a9548604a, - 0x83f73b79cff747b7, 0xfef3c9808834cd05, 0xd9504e020f456baf, 0x74d1284ed6ab10a8, - 0x98bd9d1af8d2356f, 0xe525efb0f0d4ecf8, 0xf3fb5de4cb3fed51, 0x9e7f6d9fce3b70a5, - 0xfae98c1f96b97307, 0x00a801def0bde569, 0x2b6a1148b301159d, 0xa8c3f64882f4dea4, - 0xbacad82ccef76184, 0x52d5d6a821732eea, 0xfc2ff309b92eb9dc, 0xbccd367e49083131, - 0x66b794783fdb0f8e, 0xc2d820d1d492a582, 0xec37cad13dad924a, 0xacef598e41a999ff, - 0xef518d3cda6bc7f4, 0x04d6e2872da2e7e9, 0x6d8b5dfa8957e78b, 0x6b29d5b66e122bb0, - 0x6e1f4be77afd6623, 0x743c3707f3858573, 0xd40f6152f6c4e8ce, 0xb3c10b801bac7d72, - 0x3ce6e3fed7d20eb2, 0x8e9fa1b72de70b40, 0xb5ec1b1c5dd29e8d, 0x8d6f178064892870, - 0x93d654cf6063b59b, 0x9a121664f44af21b, 0xf353c6e144ccc5cb, 0xf548320b74c70ebf, - 0x13fe62657d6fce64, 0x5170e60b78758e22, 0x1261a3afd4acc7d8, 0xa5deced974a2397c, - 0x5eb98ad8bc64d5e9, 0xb90d438bdc1b9859, 0xc8605d2bf744812d, 0x5c0e98b7e0d20e0a, - 0x550bf6348f8c4330, 0x3859a5cd209073c3, 0xdad955138b3a6bd7, 0xbb6322c076003df9, - 0xfdff4477d2d5ded4, 0xb442cf00915c2740, 0xa071f2acb80639e8, 0x3ffe61d14d206519, - 0x6bd82656b0701c7a, 0xd310f63801a783dd, 0x301b224c9311aee4, 0x9a408a07ffde6aad, - 0xb106fd4bd2ded631, 0x3034de3c4c252161, 0x15ee40e3f3e14f00, 0x835e5de9a7e01f5c, - 0xd379566157d5f104, 0x246da89d16b9934e, 0xb72fc03c12ba20d0, 0xd43f4f9f8e9fbaab, - 0x1ee83c2386f38551, 0xca8c69e4f4a1300c, 0xac7ce7219648f61a, 0x64d5ea940163a06a, - 0xd524518fc3024c5b, 0xa4e01c7b590b62ca, 0x4f371e72d2e67e4e, 0x559e18e77bb9c385, - 0x0802931d7a61b9aa, 0x87430cfb9b406e32, 0x53bcd2b77c137eeb, 0x3c3f29c59d78bfa0, - 0x1488ea2299a0c16c, 0x1b67ec24d1c23881, 0x401b68c41713671c, 0x03884fba97532717, - 0x395155e749a570f0, 0x59d1b525242c5c60, 0xa387c0448894846d, 0x0e6f2ebcc795c726, - 0x0281c8e35c14099d, 0xe6f13ef226874322, 0xe2db24ac4523cf64, 0x7c39fec0d5ee32b9, - 0xe4066008ba725cc9, 0xfbe6d527a8e3f0ad, 0x8d3ddb07faab1f10, 0x41d07f91e53b24c8, - 0x16270b7efec4c76f, 0x0c637d63c0514ab5, 0xdabb68bc780bf0c6, 0x26e40c9efa148830, - 0x0dc02761d306c500, 0x87234baa133165aa, 0x6a1f4a09c9a2cb21, 0xeed87e5684a9332e, - 0x207941e4ae3c5cba, 0xf7029d43fa3dab28, 0xbcd5661eaaf151a1, 0xde71efef4854dbbb, - 0x7097a48b6369c877, 0x84504d96f09773de, 0x781d554ca4eefd73, 0x9f90b15c76996be9, - 0x3cb11140412cd42c, 0xce9542b523eda067, 0x0473019dd214aeb1, 0xbd4bde9d7bdda27f, - 0xfd67e71907f96410, 0xea2ed09261efd8db, 0x52cf94f287e3209a, 0x7740fd2c63f5dcfe, - 0x8fd01dd7be5576ed, 0x09ac409ad55d8ee3, 0x94c124a6596e4465, 0x67505c07c0b5a8d9, - 0xc9ae1cb21f3c90d6, 0x440e7ed19ee2f710, 0x3ea217f51eda3592, 0xf083632528cdb80b, - 0x26c1c467dbec5679, 0x37c47d541631dd42, 0x4bf3ea283f9bffa3, 0x6dee2dd3b306f2ad, - 0xc06a216062850c39, 0x47d2089da7357819, 0x8e8b629338e5e152, 0x2837bcb2562091be, - 0xa239cafdfeee8f33, 0x73c7b224793be241, 0xfaf7a533fc90d7af, 0x4a16ca9517c8120c, - 0x94f6c3da5e7019a0, 0x8bf0194fc37e1c23, 0x5b2992b83f28938e, 0xff72c2a0f75e6b40, - 0x80383e172d67e512, 0x07f73796df794a8a, 0x25546c2f1726eca5, 0xe5cb191a66566231, - 0x1234251e0ecb2d4a, 0xeff762c1b19b3190, 0x3dc580a41a845c4f, 0xd2f801965f73cdea, - 0xf9b71118b3e240f2, 0xc0c2afaf351e5fdf, 0x8c1e22d426611894, 0xd74fced7e9f128b1, - 0xbe1e10f33f23403c, 0xaff6a9579b1bcbef, 0xa7ab39abada2dd1c, 0x6a327f527feb3d3c, - 0xea7c13879fcd5d92, 0xa3be11c2c79249c1, 0x534a3c1dcb413a07, 0x5413aef65c157047, - 0x7632e3f187aff728, 0x092e3f72a5555ec9, 0x5d21cffb11658aeb, 0x552ebe4dfdbe7798, - 0xb1ab16ee92c98ec7, 0x059c1a2f640eb003, 0x943930c73f90d599, 0x3593d1c9af168924, - 0x8683ee6607005191, 0x24948f955f28f1b4, 0xf69bac73113fc911, 0x8efdba6546dc2eb7, - 0xcf0839cd7ac348c9, 0x0b98a2d3c2555bb8, 0xb23067a0a8124614, 0x2fbf6cbdf113974f, - 0xaba1121357b658f4, 0x080c55eae9a647d3, 0x49415854e654ec8d, 0xb983fb2b8d01efa7, - 0x26b04c5ee68d8d74, 0x4a1c0499ce1fc48b, 0xdcbda5c00bfee0a6, 0xdb41bada10a48e4c, - 0xf2ea7a18400ad131, 0xc44018a75b744731, 0xecd4b447fcd6fdc5, 0x86bef9e04d60e47f, - 0x0e75df7a09d572a9, 0x0bfe42f84acf8a10, 0xcb67dccb0f1c00f2, 0x46d05bcb24fee1c6, - 0x7ee1803861a504c3, 0xbc618d71d09e5db4, 0x47f1e5da06d87f7b, 0x641d8a27f1f9f431, - 0x3cb5aae32466f0ec, 0xa02d3a5fff233538, 0x4f9cb3671cea8a23, 0xe8b10b3d7f69944b, - 0x1b42331e241a32e2, 0xbeaca9961723b1b1, 0xc7f26bd2800cbed5, 0x1d62937f40d730c1, - 0x694446061af804ad, 0x9172fb75d2f78122, 0x297ed99b697844f5, 0x60964079587f3ff5, - 0x2a9d2521e5991458, 0xf3015c1a419a857f, 0x6e2d211729cfa8cf, 0x512b1972dc78cead, - 0x4960b77bd97e2276, 0x959c13f93e9381c8, 0x456a1d79f4bc172d, 0x1fbc9cfd9d16322c, - 0xc5776bdda7cf4e11, 0x1e3f7f64152a497e, 0x2a8bce59cc66a064, 0xb843dc70f3a6a43b, - 0x456ba6d109d96250, 0xce8a401d1b900d0b, 0x0bb586909ea06350, 0x7152221c9cb1537d, - 0x50e22a11f5d30b9a, 0x4350c3f15f488564, 0xd802baf8537f98b1, 0x572db9b49f9029e9, - 0xaf00f8ea4bf9ba25, 0x0fc801d5257d3566, 0x9b898768be614f92, 0xdad7381a68467491, - 0xe7f56c36b654e247, 0x7d192a43080643cb, 0xd372c8e421433007, 0x2043137ca90f3a9f, - 0x307df50e51f45538, 0xd1f81a27f318ce30, 0xc1787fce99113738, 0xe4cacc5fd7867e40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xad8bc68ce031d616, 0x16888d8ee4003187, 0x44c0757f3bb8b600, 0x793fae7af0164245, - 0x210cd042973f333b, 0x08666ff52dbd25f9, 0x65c5b129f5f7ad5d, 0xe03d7a8d19b3219a, - 0xe0d98d04c44fcaa4, 0xf99439ee8752bcf9, 0xf5b9b6e8c71a0e10, 0x713f9fa797f986cc, - 0x9a67c3986d98a43b, 0x484e8d2758a09283, 0x6d0952de13455120, 0xf9937540698be490, - 0xc5b0fd8fe13eb912, 0xf512ce57673bcb32, 0x6e164408eb8f345f, 0xccc69dcd48c3cdd5, - 0xf0c920e6e0afa3cc, 0xe842857399981e91, 0x48ad23d074346f66, 0x8fe410fe8bc07929, - 0x023899bb9ea6a3f6, 0xe20fd0d565070c45, 0xbd15dddf6c80c013, 0x6ebca33ef486d172, - 0x5906b47b84a9d6c8, 0x51682f2843d0c4f4, 0xc968add0f5e0c2a7, 0xa5cfb9d08a1967c5, - 0x86d17d3b96e91b7f, 0xb6567758af97c825, 0xa87529701d98df45, 0xd9dc6e749df9d26d, - 0x9d079949b4480750, 0x0338a928aa10fe48, 0xcf434b3d2afddb6c, 0xe3c1f5298dc39f10, - 0x78e9a0846c4f6621, 0x3d274e75732ef3fe, 0x8cdb67d9e4366dcb, 0x930f3f5a6902449e, - 0x7252041991d9679a, 0xcd60e915b231c490, 0x4da119217b6ec062, 0xdf1609130bc19e60, - 0xbfa14d5f24823e15, 0x934a9377a9a12764, 0x557f263e69a1666c, 0x18ff26e261aa8846, - 0xc134ace5dc841e80, 0x075865b1de84e3e4, 0x1e28a52e7e84ecba, 0x42f5a659c1d05dd6, - 0x870d9541158c9176, 0x769f45e17527d450, 0xa74509d7328f6de2, 0x6bae6f172ae5297f, - 0xbaece7117891400f, 0x191f2080e989523d, 0xe5bf7d9851a2c974, 0x507c65e03b7de2d6, - 0x93dff2dee8f46ca7, 0xe4767a2b717d4b12, 0x8645aa11aab86959, 0x1b0e754dfcebd22a, - 0xb8b0221a346522c8, 0xdb1334e2f2425854, 0xaa074f9bc5144ad8, 0xfc418dd2d8ac14fb, - 0x59f9668458f26776, 0xc1985681d89bf549, 0xfbeac674878c3f08, 0x8d6ff97eb9f16443, - 0x8b61657c91ae5545, 0x059dd8a912f861cf, 0xda62378dbeb83d03, 0x9af0fa2861888892, - 0x4f2d40859a0076f1, 0x21cc77c9a754fe8c, 0x6a32db7dfb459ef5, 0x1ce4edcaa2855f23, - 0xb261e2a233654ee6, 0x6ba0441d353dec43, 0x8e669cfc18ae9f3d, 0x03efdcc14f0181d9, - 0xe13ce19d8f16a473, 0x1ac01ae30accd9ec, 0x46496b50a607e9f0, 0xef63affc7b636211, - 0xcc86e37c8bcacb41, 0xfe74cb479a38d888, 0xaafe0aade7d6f3f4, 0x21c8ca1917d1a4f4, - 0x0b15de89b65b39d0, 0x03a0f7294e38bec0, 0x1d791616a7cc9625, 0x405f3612f5c91abc, - 0x2d7c7b5b0c8d001b, 0xace57829fcbbdb16, 0x9bae7a0e2a85d001, 0x931d1002fdd010d3, - 0x5c3802feaccac564, 0xaf21c8242cc36942, 0xa753592d67660b38, 0x3d8d776f81851f57, - 0xf6a6521f0f22bdc2, 0x29b399ca9fa723ff, 0xd81b74c1bc4bac45, 0x5cd3d2a38e220c05, - 0x037f9aed755146f7, 0x8e5ad27d824a8b9d, 0xfed8715a9d32a3c4, 0xd89fa46c4af1623f, - 0xc1fe24754e9175dc, 0x1063cdbae163e8b3, 0x7ba01f134500caa9, 0x06179734b0e5ac8a, - 0x7249439236b0df30, 0xef2d9625c7b9e3bb, 0x971cea093b4ae42d, 0xab05d3e2133af6f3, - 0x625b6a5cad0ea260, 0x820ae79552d27add, 0xf2e97f663b17d5f8, 0x39506e1c3fc013b7, - 0x191f324163cda313, 0x9f2c0bd8e38193a4, 0xb94e3e059cc12b5e, 0xd6fbb8a180999f20, - 0xc37df2f0690eb305, 0xc823d9bf21849579, 0x268eab19769f0e8f, 0xdcf93ea059620f4b, - 0x8e8869a8a78eedef, 0x11957cd8d173b5cc, 0x0a349326b5f1b856, 0xa81bff5757a5bdc1, - 0xb9560179f44ae2cc, 0xba90572797b35863, 0xf0209d906ae871c4, 0xcd8ed6da40fb6bca, - 0x72456cd3cd2b7c8d, 0xbfd522396c8afc1a, 0xd66265e4366a02e1, 0xdecd0077e670e8fc, - 0xc08896c3391d9745, 0x6c80ae0f13b4a431, 0xc1fa6cfb400bd57a, 0xedb88922879c63fc, - 0x6ef6b77820fa3f63, 0xca311016bf65a345, 0x95e6a68042d8b988, 0xa2e9404ee478a29e, - 0xc7e323de9fea4709, 0x4b3982d7fe4b6995, 0x4164ba666304cf04, 0x95f63593b70010ee, - 0x1825c0737e7696f9, 0xdc16f46922949f9f, 0x0816da250604c637, 0xf92ad91faa10194b, - 0xef048d890f461c33, 0xedc5164f3c020c34, 0x79d0fb8ae2cfeeca, 0x8abb4ef43d8100cb, - 0x9ebd569148bd4cab, 0xbb918e4bfb053e38, 0xa751f211bc4cd877, 0x8b04d897be741a0d, - 0x80764af4bcbbb271, 0x99008678ff5788b9, 0xc8c7841e7a08c504, 0x2148f446482e76fb, - 0x436d727584079a9d, 0x1b5e228a73e325dc, 0xbd2534706156a568, 0x07c11faff2ea99cb, - 0x61781e958be19b01, 0x03d384aa323790d9, 0xcfd9b8311a8fec96, 0x36eeaeb0fd58adca, - 0xb0ee8e87939c0dc0, 0xfbbe1111ac50cbf0, 0xb73663a5c09328e8, 0x3479b3c2b9b83324, - 0x47ea9590bdfc6a52, 0x215b1a58397a980d, 0xb960b40e4d453d6e, 0x48f8036f37a5f313, - 0x4acb94f16c968f93, 0xd28ebeb00cbdb595, 0x835f9af487184cc5, 0xc68ac73d514f66e9, - 0xeddc70db04724292, 0x1f14683248943b06, 0x0e46637d4357bfdf, 0xf529abf54245f486, - 0x57cea528a94e7085, 0x27184a175b192fad, 0x7295d93cb6ab4615, 0x97074eeb2fdfefde, - 0x7dc7c36053661a5e, 0xbcbb863815ffaf57, 0xa836b132b417d9b9, 0xfaf8f382e81f47b0, - 0xaf1380ffeb9a66b2, 0x78ca5acd02c0025b, 0x06121e5380c216a9, 0x0f20aeb213cf6976, - 0x9444e52ff201a5ba, 0x124e01e767777ce4, 0x6f7f84e409cfa0e5, 0xd0a914edb7831260, - 0x92657616096c1bd5, 0xa2f1edeaf3218fc0, 0x1d2f23410b1b99f0, 0x006b688790c68464, - 0x48262de13a1f9e17, 0xb1257e94ce377cf5, 0xb32132b862293f74, 0xa885444e6d9788f5, - 0x47f46d5e29bd1f46, 0x6b35ddd815e0614b, 0x4dd2adf4dec20404, 0xa61fcddb8aca9021, - 0xd18f376e7f0c6e22, 0x49c0de42986bc333, 0xcf58c053772fcc75, 0xce7467d3036697ab, - 0xff8cf62d701b3740, 0x4812908ffa6193c9, 0x511af45040e6d04c, 0xe27a49b05469cd2e, - 0x65a40175a5f675b4, 0x113634eb76ca0920, 0xab7c35edb3d83a85, 0x42b97f246360e041, - 0xec46f227e51d0264, 0x6d89de8ac4065de3, 0x9694f16073fc54a0, 0x18319970f67c715e, - 0x3c30e74aa69e1d76, 0xbe3abf39d7943514, 0x1823f8168e5fba40, 0x9f81041c6754eaed, - 0x91e35b0beae11188, 0x522f53d831fdba2c, 0x08b03697abc449c0, 0x5c3932444df85d1d, - 0xc61faa71a5a4886f, 0xbe9adec82d320086, 0x7cf8c37c7f510fc9, 0xe90d874d5a24100f, - 0xba9a1b32028e4856, 0x1ef41a9042a73f31, 0x99f0a584d3da9a2a, 0x4204dc9a785c4bd3, - 0x2eed1db8a6d24b4d, 0xa047dc4d25f9009a, 0x97d1f744bc68ce6f, 0x980bb2d27ce06fcc, - 0x259c137f5642dda6, 0xb947348b80db5caa, 0x624b4a5f435e7574, 0xea02bb4c5aed06ee, - 0x4b03746f59e42106, 0x5523879bb7572b55, 0xc7572f70a70f3781, 0xd884d71c8dfc0f02, - 0xa158d1080140ae6e, 0x9528770b139a9263, 0x585e169ae1e92db4, 0xd924489320d810c2, - 0x09388fb6b783b910, 0x50373d28896b92e1, 0xbacd0a0c6667f018, 0xe1e02acd2f28685c, - 0x60b54c1ad833d9f3, 0x31eee6f40afd126c, 0x0ee768cec793cb04, 0xed40050d5aa849c0, - 0xd6f44aa739a4e55a, 0xb01e72322b243c91, 0xd2918a7e36bdec80, 0x579d876e0a74ec33, - 0x66f115b3f97ae770, 0xa61d42657348d631, 0x3fa2b248a7f25789, 0x24235d1d1e7cb0fb, - 0x027a9c67959777d1, 0xbd000c805dda2d7b, 0xe1e5799a19603cb6, 0xf70b93c568893ccb, - 0x81eb233bcc3af284, 0xd2f0b28c41338818, 0xae50af1f82d93202, 0x764428fb4bf481ea, - 0x934b2918e0dd3951, 0x37cc7248aa9286b7, 0x8c4c4da9a19fe021, 0xdfb1d156fc1c3358, - 0x802ed9b76fea0968, 0x2602ad628b498333, 0x3ca12fc3e172190a, 0x981247e38983901b, - 0xa456846a3713a0a9, 0x4bf8b9c069f118a5, 0x4976bfe5352ba8e7, 0xe73f3efc3f9ef9ab, - 0xec19dd6cdd423bfa, 0xd2cef2c12552fc65, 0xb4e2f036e4add7dc, 0x1ca87755c6db3231, - 0x7fbd7cbce56acc93, 0x960d4b4ab8959775, 0xa7398542d36068b4, 0xd7040b66fd0057d7, - 0xb41b2b776c0234dc, 0x00b8ddbe778c18c6, 0x6b5e43944ecf68f5, 0x043b6b7ccf259465, - 0xc992b3653201d6c0, 0x0dc80b5d845567fd, 0x17a054fbe99a4489, 0xde87f6b1c38f83ee, - 0x36b3d2b536dfb35a, 0x2d15ba37444a5596, 0xa3afa817eeeb111f, 0xe5ab367506dede69, - 0x6d4f2ae42e147a5f, 0x5133df19dffe067c, 0xc459088e06c4b37c, 0xe0f149c9cc380b4a, - 0x9e32e4e2e7e7d0ac, 0x7709ec21db32156f, 0xfcb636cafa34624d, 0x5a14e436e1000c72, - 0x19975d2bd3d5e206, 0xa95f63a7e82068bf, 0x6373566711af4fd1, 0x052c972732b53592, - 0x1f20ab824f02fcf1, 0x2b169e6a8a8684df, 0x27388136ea2662ef, 0x56aec80cbb5c0e06, - 0x8f52e5c931b8f736, 0x416caaeadbf1d56c, 0xfe2aa53c41581446, 0xd549a265ec64decc, - 0x3df3a3e78869081d, 0x1d9cf882c23f01e7, 0xd720eacc96fd81c9, 0x125c052ef30e0f67, - 0x0270dbc43d1dbccb, 0xd07ab6d7177da3fd, 0x1072ccda35e696b8, 0xd4210ac66eb921d5, - 0x14c622910a017dea, 0x8c703e672b73d136, 0x4942a8e05cf6d5c0, 0xc72f661479bf185b, - 0xd5e5391a1ce19300, 0x0c8b5a12ed183891, 0x66645fd0ea45f259, 0xf30647ba7010d636, - 0xf75f5f4a2b1c94b7, 0x13391d4b9fd4798f, 0xf947a6b8800e5af2, 0x8af76f7fd5576751, - 0x0343e4b0a9852c2c, 0x0d5fc25728852d0b, 0xead2c5d671a91e35, 0x7f02275863b5d10d, - 0x2b45395f8ce30452, 0x1dc8f0ab1c9c564b, 0x55684629f1770b59, 0x4627cba9cad4037e, - 0x2d97f53b8f497527, 0xd5a6c8c83f899451, 0x8961977fc3c651b4, 0x6d792a973d645dda, - 0x9062e41e207eb827, 0x67968b238c32b264, 0x59928bdff804eb7a, 0x2cca31672d9f17af, - 0x647e9d4da66ca99d, 0x9ebb6fac229b0340, 0x84b634f9f8bde3d2, 0xf68ba53aeb7b9f51, - 0x415d44b181557d97, 0x596797b8e464be37, 0x44938d128fa8ccd7, 0xf5cf473d1f61e4ae, - 0x9830dc9a384c3d65, 0xe363c2d278cc4bff, 0x5e8c02ef5550ac53, 0xb9e2e955a90a0f59, - 0xcdb368765853fc03, 0x142ac6ad816f1d5d, 0xd3b729517d50130f, 0xbb5b0d2f4c167b11, - 0x0837a92ac2069d9b, 0xc392729c7f788596, 0x5c1a24a2a79191df, 0xa1d7d3f974c93d3e, - 0x596365646f44c7b6, 0x4115f27372425ba0, 0x0cf7ec0c4a28b2f3, 0x00431bcee95314f0, - 0x1ee7e74940765d87, 0x68850b73062be66d, 0x4b0a1761b7f7a66d, 0x2a1ecb67c725d90d, - 0x44eeca98805bf112, 0x7c615b2923d661bd, 0x23924b532509e387, 0xfe0f3b3629fa3e9f, - 0x72e77e5f02f75116, 0x6eed54e4f9952cd1, 0xc946db9f3d82172a, 0x6caac55e98fca10d, - 0xa86e7cbbd4a4e429, 0x95faa7b647e709d2, 0xba5e19a694e9eb16, 0x2f4f6b22afeadd03, - 0xb48e925f1af02b65, 0x578e3c423bce8be6, 0x82256d2edf79d6da, 0x62e180765b54012e, - 0x661bd7992681bb8a, 0x0526128540824191, 0x7a7d279ecd4720f3, 0xbc0c263b22256981, - 0x3f406ce2a77e381e, 0x496e92879b9a78a5, 0x4002487382b384d5, 0xebd513c7a9dbc170, - 0x5af0df1289d0e48f, 0xc851cf5cbc4ce0b9, 0xb039e21b260def5a, 0xdb915088bae00294, - 0x90a875849f005fd0, 0x216e2fac1c859da7, 0x95b7031f134469dd, 0x660c5100a280d466, - 0x504ba3af9bcf6cd9, 0xb57b2c794654164f, 0x00c5ae5b818acbc3, 0x98e283c39b38ed6a, - 0x134cb409cef51d13, 0x772bc6983458f2d4, 0xe5999bb11fd45f26, 0x01893928027e4cbf, - 0xc20aee9edbc83df3, 0x676f4b2caa556f49, 0xaa87d88fb2dafffc, 0x9dff1c74fcca5623, - 0xd84eba8cffba03d4, 0xdc003a3bac5c49a8, 0xd77811a9fd7f69fd, 0xb31593ad280c6f0f, - 0x31c2d67670be6936, 0x52822e373d880148, 0xc161f514d9209431, 0x5679c20a803d5df0, - 0x53f06d16bf3d188d, 0x06c08f0b862ae58c, 0xd4aa960d534bf0fe, 0x84d452bfd4d1caf5, - 0xfa40006321382333, 0x286388ea172b96ae, 0x24eec962b7911486, 0xfaf78faf90150a68, - 0x9c0cf33db4113808, 0x761b7a94ea344ea5, 0x9f95abbaf774dac6, 0x572bc6d43c9c7e4b, - 0x54a89ff5e8f941fe, 0xc0fedfd448b0198b, 0xa9c3f21bf0cbdd96, 0x82efd318bfcffe6d, - 0x055db5dc6be3be75, 0xd157cc8249d8d28c, 0x6429d4902cce3fe3, 0xb7559d1000b3a9e6, - 0x7358db9ac82b1e00, 0x52c983b6da12029b, 0x33feab193ce08b26, 0xa42a05d86c60246e, - 0xeb4e817547ca65d3, 0x10741b657eed1773, 0x96d65bb9125f95c3, 0xa0ee0469fe77b14a, - 0x6dda3d8c1d0875af, 0xe5520eccf49b85f5, 0xfaebc73b9e83f56b, 0xa63b277bd28a2489, - 0x33a76ce9f7b33c2a, 0x73aa71ad66ad532f, 0xdc08c5325792bffb, 0x8d16c863c37ae721, - 0x947c7690d47e706f, 0x1b2d1a8433ec72b5, 0x3e0e28c490e364f5, 0xf44aaa572092af6a, - 0xcdfe7fd6d7b2427b, 0xcf85e74409db9efe, 0x140c309240d11c22, 0xf274d6098b1b8ecb, - 0x11216b3eb0ca9d68, 0x8e4fb722e1b82d7b, 0xb68c828a981d087d, 0x5cd379c71a770be9, - 0x212e322f5552af36, 0xe41877e8135862de, 0xdd95fd498a65861c, 0x2fe79c930e2bf1be, - 0x6bb58ead5208e88b, 0x84bb2ea8ff4ba5e4, 0x045bde9b3da4f5dd, 0xb3c080baa26576a9, - 0xa51be0cd90dad08e, 0xbd5a1df5166cfb5a, 0x6807231c14297d62, 0x326b3ba573a3a7c0, - 0x7ada34558b345c98, 0x8da1a70b3c91d7ff, 0xa576bcea83daeed7, 0x8af65985a3e4a39c, - 0x04780f77127c0d9b, 0x2fdf5e19f86bee48, 0xd2c8fa482c3d3980, 0x9fce9d031f9fe625, - 0xc06934d908203598, 0x278aa815d2f04a87, 0xeedc89c3b72cef0a, 0x94efd5c0f0990ca5, - 0xcc309fa90e373da9, 0x719748c91f73c72c, 0x56836802b043a589, 0x67c3c26d885609cc, - 0x5ca6a547d73a6d54, 0x0716b5cb4b763b4d, 0x58f41f6d24e53eea, 0x40b1417a73fa9b24, - 0xf0fbd86da3cc638e, 0xf4ee73920d5b697e, 0x695a260fc195f439, 0x98879f92c0378921, - 0x8eae6a15ff2c7042, 0x5587d772bb83194f, 0xda0bd09a839c0460, 0x4583cbb276c1fbf5, - 0xfe60f4e2b090b65f, 0xd5017c68bef1558a, 0xed413f19061e1245, 0x7221a8b4d87f8e58, - 0x5383d8b7c376cb31, 0xf0d901e229b86680, 0xd9c934ff118cabbe, 0xfb32d7a16b2abafb, - 0xe0f5036303b629e3, 0xe0ae0818dc0fb75c, 0xe4eea3431935c676, 0x1205a565da0786f8, - 0x808382b0b9bfde29, 0xf110e3e298971c57, 0x2c12e65e5c359f44, 0x750f21db8dc849e0, - 0xa6036536df854af6, 0xf5096a448d108c7c, 0xdf91f300d48a2133, 0x88b0876fbaefb5c6, - 0x22973f2784cf620d, 0x5f3bfe4a0b197417, 0x3b6fc0e1543118da, 0x94ff4f25472baa32, - 0xeb2c2d9df5449cb1, 0x6e3f95a226ca9468, 0x9ef0e6ef443ad247, 0x42013a842d493fc1, - 0xcd0e2dd5f5f1e47e, 0xc4ac44ce934a7606, 0x8cce1ef7b72e1661, 0x1e2090bf5284d333, - 0x02bd7bb6d5d3108e, 0x663042c8f7884df5, 0x81acb949cb0009ba, 0xdad17497d48f86f1, - 0xebe8c481d34f156c, 0x99335e8f78a6c0e9, 0x11d4c662ad95fe2a, 0xa69f6ff236ec2d3d, - 0x588bd689a4995927, 0xa95eb190bc4ef82d, 0xbe029150cab44569, 0x419b5cdd35d217c0, - 0x66e45390ef0a21d7, 0xe0fe295db23dfcf2, 0x514b122a536563ed, 0xcf1cbfa50828d353, - 0xcb2fed5c50464661, 0x1724e8268c696a1a, 0x5645f53e0db58b90, 0xeed3f47bb83a3190, - 0x860b49f7a0737f36, 0x2ec8c337654f18a0, 0xe07f01e61fdd13cf, 0x7e3e2d46d86472b7, - 0x9bb6cc0992bbba91, 0xda45841774b93f91, 0x6ca92e4c7e7fc8ec, 0x0499187cb0958c77, - 0xa2a4d0b22e7eb061, 0xab73cb0b3fd65ad1, 0x2ca185e3167271a9, 0xadcb19066ee85f31, - 0x0765fc451543a137, 0x9d39f713d2e4dc9d, 0x2b87b6287716a6ff, 0x55e150ecc40fcb1c, - 0xa56a62692372f042, 0xa044dbbe8c82a4cc, 0xb3d27452cc341b39, 0xa308233beb7e33c0, - 0xcf19a57901481ab5, 0x23867e0f96854bc1, 0x6f1da8842dd26999, 0x3c3a6042c375f9df, - 0xa66949b5f2abac01, 0xf5300524d83b9ba5, 0x73be8b3ad663f6b1, 0xd65d774e0ee07de2, - 0x57a7d7d0efdc36f6, 0x3f94831eb87e8e32, 0xa8ddd8a004c38936, 0x4202544447480294, - 0xcb76473a4ba3482e, 0xa597a56d1ec3df7b, 0x3aa241f6bafb75d1, 0x4336cf805669782d, - 0x123f3f62d5dda8c0, 0x656ce3d7ff630946, 0x8d398514fa96d663, 0xd07b57fb3fde1cad, - 0x028aed43d73bfb55, 0xc7d42c07b12c1c92, 0x88c91c0cf517df79, 0xc206369595cc4802, - 0x4828c60bf1613a12, 0x1b557465a9699f9d, 0xa39c8e842d8c8632, 0x59356131f19d62ed, - 0x1f999eab87c8d648, 0x551dc01264170f3f, 0xba7c75ad4e0e997e, 0xdfbf5763c418f1cf, - 0xacb6ba72b92c4c0b, 0xa701a9ff921d878e, 0x5d4290df833d54bd, 0xf67588f723d8b368, - 0x815e3eb30ff43292, 0x3b6b61223569d20b, 0x8edb036030e40034, 0x3804384da0da7346, - 0xb72ae0ff3f91c51d, 0xf1e7955f14121458, 0x84b0dffb02a38d5e, 0x2a538b2ebfd25a74, - 0x2ea2cc4b0317de28, 0xc3426c050e51b31f, 0xdcd9c0969f9863df, 0x10bbae42ee46577c, - 0x652c0e21b4e198f7, 0x5a5e26415dbfb4ec, 0x361b6a27ee5bac4f, 0x527c5a34450666d8, - 0x2628bc6d0f2af7f7, 0xc69185f49bb3b14c, 0xe7b7ea7422ccac07, 0xebf9f0934bdfd6b1, - 0xbb0c28b52a166e10, 0xd43c310b58bc085d, 0x49db070b995420a0, 0x12c4bb44b72b4c3a, - 0xfd7990c5cfa76aa1, 0xefeb2ed084769e79, 0xf1f4ca6ceda6adc6, 0x217d0cc743227012, - 0x9ee9506325a0bf19, 0xafb33245ec46f4b3, 0x9171c3e5713da8c2, 0x32065311ea3f3f5b, - 0xb17af92889a84d36, 0x8bb41b3a81768e0a, 0x4b4a60e09c16dfce, 0x6f82165af73f76fe, - 0xef28729cd3131be1, 0x2e2e6b3467ef084d, 0x034eba547990f823, 0xc7e9ff1731b0face, - 0xbfd2cbcd881602ad, 0xe6515e43ae8fec21, 0x193365c6787acd9c, 0x34d09409aa93c5cd, - 0xe7ac515c76670d81, 0x1155a946a38b0178, 0x0448ff9f5e5a175c, 0x1af998870fc39f41, - 0xdd0f17b6b849e7a0, 0x044287276c664788, 0x4c35b4674eef9358, 0x4d9ecacd482a1946, - 0xacfb090c2ca95b72, 0x0b2d9d9fc1404add, 0x3f56dfbde50e0478, 0x1e930fcc78a0d1a6, - 0x6090bcb793adf6d5, 0x5bceb13646e5cb9a, 0xc37fb75ec42335ab, 0x247164ab0b2e88c1, - 0x856ae1836d71fb54, 0xfa32f8960074a709, 0xb6053031639a0fda, 0x5db909b6a11f088f, - 0xc6b333f265327961, 0x64aa39e7ee9e02be, 0xe5c435676b5aedc0, 0x38bbab7846288185, - 0x3b3af4b084e93c21, 0xf9ba84cff5e4aab2, 0xfe7b8ab181cefe8c, 0x7970dc93a3abb53f, - 0x83662e4927744e7b, 0x825fe03b9a39c1f6, 0xa05afbc2eb85b459, 0xec0d6a0eda7bac3b, - 0x630e0b9a9e7e008f, 0xf162baf5a1de096a, 0x14c43cb8eea326dc, 0x0b1cce81a0f117fd, - 0x5ed2783cba7d1fc7, 0x15698133c8343049, 0xe5e64e5b2accc011, 0xff1fa8847f3f321e, - 0xd6c53b4f215db9b9, 0x37865188dc8871f6, 0x55918f66965e734e, 0x8cd0d6a2688b99c2, - 0x1825c9a54a8514fa, 0x06ad2bd8f5c11b89, 0xc7980f347e2806f7, 0x7575451f3f3f0fb3, - 0xd25afd14d64f2cda, 0x47c5774db2fc9014, 0xe3cf2b311f5fc1be, 0xbe613f1918577d15, - 0xebbc1afbaf183651, 0x8e1fd7a7c86e8672, 0xb3551d30e3aca559, 0x2af0f7548a87b3df, - 0xbe86a18f403d02d9, 0xbe7667739c710577, 0xf3ae8fc8dd1befc2, 0xcda11092ef0b8609, - 0xc0a532d2d4f1f0bc, 0x438d0b24f3e6d3bb, 0x92641fbec03ae1b3, 0xad30feb4d3d29961, - 0x5e04da55f5f1a227, 0x34c61cb61635808a, 0xbc97fb19484a326f, 0x2b0bc4ff2f23eab5, - 0x798aae3d827bff80, 0xd1885e665da2621c, 0xf30d937c90320f25, 0x4d5ad66bf1b28a35, - 0xed6a868c23ec404b, 0xe8042695fe29b19d, 0x07b03728e8ed6e81, 0x4427d1f371db1eeb, - 0x4b84d0d54530ccd5, 0xee960ecc37b42fa7, 0x0a53fea51938ea2d, 0x9b1620ccd742d967, - 0x0b01eaec56cc58b0, 0x775c2b6d99d128da, 0xe35df337a955f20b, 0x96ec13abe7e5cc1f, - 0x5a6c1ee338c87984, 0xe129606ae99103b7, 0xef680a415460db66, 0x7f5293053ba27e72, - 0xf638efad29a447c2, 0x3bec133c80241fca, 0x36515db203bd7793, 0xdb01c6b4c0155744, - 0x2ba45d681117ed79, 0xe5e25d1e32c2e6a5, 0xf8ef6e90e36f98a3, 0xd45f066251de4cca, - 0x43ceb28f4452539e, 0x8170acaed9cbfafd, 0x2d016ebb4448cf79, 0xf8528150e4ee0763, - 0x673a7c1561a5aba1, 0x3278e4229586de9d, 0x67e3db5aa73cdf6d, 0xdb5a987c1d1ab41e, - 0x6ebf8a10cbc2913b, 0x0c3bb0a4ca4ae5df, 0x21e5aa3e65736300, 0x6be3938b60f630a9, - 0x6d53fedc3470fe29, 0x60485160f7f2c84a, 0xd599d230abbc9f26, 0xf3194a36f28ae487, - 0xd1b54a66a34555e7, 0x3b24e4c77328bb0e, 0xdff3dde865908b7e, 0xba877ad12633a697, - 0x1a73236cba236611, 0xc50e1c1338fdc6d2, 0x3e676c15ab5bacf8, 0x0a13a19241fc80df, - 0x98b39a2da38de97b, 0xecff5a4f94d07f67, 0x502a193da0fe8a8a, 0xdcba794ad7cb47d7, - 0x58ec84f47c4a94f1, 0xedcb97c2404c0a22, 0x4bb35b16bed2f825, 0x58085d4b92b66c3d, - 0xf3fbebb14e87ab6d, 0x0cfac81a4d5b0675, 0xb986e8513ffc2451, 0x0c5bc57857b7026a, - 0xd504e39688e8853b, 0x468e16d8d493c2a9, 0x446730d5fa67cce8, 0x8cc8c2791c14af9a, - 0x95de43cf49c45409, 0x85280f7c319780ec, 0xa619fda30b6a4930, 0xb88b4f9004e84468, - 0x8b1b1fba47fc6239, 0xf48df5305e64c312, 0x05ba6dc628306108, 0x226e27184d3c78c8, - 0x7372d88d3015ad75, 0xdd1fb12c65cfd3df, 0xa0f6d036b40364a1, 0x7773aac7ca85180d, - 0x0272f3538a8f42dd, 0x80db715bd1134cbf, 0xb710fefd2e74fbc9, 0x8563c90564c7353a, - 0xe5baabd1cf1b8ec6, 0x10e8f1e06810f65f, 0x4b05dc2bc0370ef5, 0x6e78e87101d2c4d8, - 0x6daa03b3b84c8cdb, 0x8255562694d15e69, 0x35516f8ae84b1823, 0x5fadde6dcca820b6, - 0x89827d11b0129f76, 0x803bb1bb939615ba, 0x94449644695d491c, 0x62879d0a6c67d02f, - 0x83921456d321eb63, 0x94de208245486925, 0x514682786c0242b5, 0xff4e0bc3e3465c44, - 0x001eb9110f02970b, 0x9d360f599bae38ff, 0xdca594d4ce6a6df3, 0x25dd74b6db2252e8, - 0x791cc122b538d11b, 0xdc475a2a188a8e6c, 0xac5414876828a3a7, 0x9384d2648fd8bb42, - 0x9bd1ced03e8a9013, 0xb7e804c2b086596d, 0x537ee55721f8ac87, 0xc4c849855b10ce0b, - 0x75ce9cc0422e4f19, 0x600366cee14d194a, 0x1556464a8608f8b5, 0x3f1c56d6092f32dd, - 0xa024860f9f213420, 0x2dc5da2f9b3c7f89, 0x22f2d5f433e406f4, 0x0a80fcd6b13e8541, - 0x288e07a2e2186e94, 0xa82804c33f92344f, 0xc11110a2cbead10c, 0x36bb0044cfdbf959, - 0xe121b026364b0d77, 0x5248a4cea833d9d4, 0x21271c6ba7350b3b, 0xd29f31eccb510604, - 0x57873f2d9604b8fa, 0x974a949bb0541e48, 0xe0ae231ee67d0870, 0x4ee35448c7362f9c, - 0x126cad157654ff51, 0x1ccf52e4c4634434, 0x291710a5b54983e2, 0x44c3ff00c76576ce, - 0xca0cd02f2e42ffea, 0x6d85405eeb23daa6, 0xbb313a2e755ff0f0, 0x953676d0a48fcae6, - 0x39084f920edd856c, 0x087599fad019da46, 0x1ca7da072010fad9, 0x2c964d52914ed2ff, - 0xd7905aa2f8fc6157, 0x6bf893fa451f8844, 0x9d388406f8112422, 0x711450f546fc6ee8, - 0xb428b3ffa79db352, 0x4dc8d369ac4599ad, 0xe1d90ba6129bf63e, 0x2c7d57ab645b862c, - 0xff30037b58b3d985, 0xeda9e62245f8cb34, 0x84a240366d02d6e4, 0xd995c2811d587334, - 0x2705fefbb208958b, 0x0a7d125cd8a004aa, 0x7d305afd992a0111, 0xd667ea21b4cadab6, - 0x88c4700949920f46, 0x59795a204dc41eae, 0x4b24ce02a24fac2b, 0xea454c6fbc3b1457, - 0x09b004d5725aefb5, 0xd19793592a0fc855, 0x2e5b8dc2d557c990, 0x8e3405542277c423, - 0x83a0804974c0842c, 0xc7edb27fe946d380, 0x120b645f99c87dd2, 0x52644cd25dbf0b69, - 0x2fc2bdf0c6fb3393, 0xff874e321641ce96, 0x2f03c261d8510302, 0xaa8941b91888fc8d, - 0xd5e1b83119e35d3e, 0x49a459b8e91f0e3f, 0x8b36de1c045386fd, 0x6c1a7bbeb803f2bf, - 0xac3696f05e24d9b9, 0xd56c41879e52a0a8, 0xf1188c1c5d640e4e, 0x70f6b11ebce4c7cb, - 0x5eacef67cf1ce2a7, 0x0cc7933561adac50, 0xfe7dc27823ccefe5, 0xd381e0a3b1ba6682, - 0x84b57bee78dae1c2, 0x49f9312c092803fc, 0x632a029375baba94, 0x03affd661a6415fd, - 0x7ffe21dd89984ccb, 0xc23919d46bbb0f66, 0x93f149875f525ab7, 0x3f7a99bf356764b3, - 0xf30dcec0978be581, 0x372a8b6b270286ef, 0xf9b92a3d25679768, 0xcc6124949183090f, - 0x0bd22be8d1d110c4, 0x68f5d5297dbc876a, 0x5c61049caead8d6d, 0x0583700371020f27, - 0xdb23f82f0d94a924, 0xc83a244b0d66658a, 0xf0721b693f685cf5, 0xbd99742d09d9a1e9, - 0x8ce83c7ff91d8877, 0x9c00ad5e4899872c, 0x7a9ce66c977b536e, 0x75005368c68334d3, - 0xeb1c6260de15a27f, 0x40aae370f0f5b1b3, 0xb84e3670acb3add9, 0x525edffa86187c78, - 0x758ba461594b6f0f, 0xc6d6f9c6d394e553, 0xf651cfc061361a39, 0xe11be7a5e7b291f6, - 0xb9b93ef86b643a01, 0x4d6bd93757908c49, 0x767754b9b6ce6c8b, 0x9be6f441e1bb0192, - 0x5a38abb5a5d848f2, 0x486c6a23974ae781, 0xb328508ba7307455, 0x7d9ce9c86b03ea7e, - 0x03bb58e6bdfa39d6, 0xfd20b608c59dfec5, 0x581c9daa35477b59, 0xfdd477c393dd8830, - 0x60aecdbdf34a5271, 0x4933253867cb51bd, 0x114ddd56b20e015c, 0xa8ee8ac24d4eaabc, - 0x1541495c64bf3bd6, 0x265c15e7bae82684, 0x10a6103349cd7723, 0xcf99d8469e6d2937, - 0xd157ff74e147db1f, 0x12b2d9ad340d0e35, 0x485142e395af3f5a, 0xc873c8c19b0a085f, - 0x976f607d11961faf, 0x3c44ccffb04e6c1c, 0xa3a8d7462123b8f9, 0x3b35fe642cba0482, - 0x666fed4b4d03f9cb, 0x518e6a9db0f7a65b, 0x059759fe417e4ad7, 0xd30e9734359aa77c, - 0xa763eb7b845b2c24, 0xaa07616ec1108086, 0x0b49c7da3890f553, 0xc1e46ce4e82e390d, - 0x7a3bc2c88cd84737, 0x1647337316aa17fd, 0xdf7e7af50ad76557, 0x096fe7b29faa33d2, - 0x52debf3c9fac977a, 0xdb1cc3d929b56bb5, 0xc4fbd2fef8a00ce7, 0xcdb4f656b768aaa8, - 0x1bf02c90e4574541, 0xb6872d129f6f0a57, 0x9ae23b95316671c7, 0xde22caf792ff7099, - 0xc61644e8b95f0c8f, 0x6eb676237d5673d4, 0x3d478b893bf4aaaa, 0x04c381d407510043, - 0xc26a456a19076a6c, 0xb0516879ba1f99ab, 0x9dbd57aa34f1842c, 0xe42ad2d58cb5589b, - 0xb9c5d97cca386925, 0x053c7be2b7358ba3, 0x49c3b14c0b8d318b, 0xe1d48901d0c8ca55, - 0xccf46702aa8f4cc6, 0xff9d22b54eb3ed9d, 0x57c550ecb4d4a63a, 0x3ea67ef30102b03c, - 0x11b3996f6df40987, 0x3db9e0131e7c88b7, 0xf2e8675174439d60, 0xd1d82795d81adc48, - 0xf1e745c4e45c0029, 0xea97ed9e69d0e9d4, 0xa3a5a5685bfc1f94, 0x16438c326caf5e44, - 0x9ba950d62e00cdf1, 0x84f21b4ede6177ca, 0x0641db881a953f0c, 0x010765b3fc469da1, - 0x658c2f0f84e0a5e4, 0x35de32d87bc7a595, 0xe6860706ad7a7430, 0xe0e58c2800595328, - 0x5afab22a18fc3c4e, 0x951efdf5cd92e5e9, 0xf08bbb084bfed598, 0xc8864d6ec9331ca5, - 0xa912a95ed9285f0c, 0xf127c60ce343bd86, 0xf6dd5a34fdbdef49, 0x735a23b9a57a5fff, - 0x2baf471c81515ddb, 0xee3500e0532d6df4, 0xfe093f860f193143, 0xf6e9c62b5b347205, - 0x51ca076f8b36d961, 0x5503ea9faf7c2917, 0x48d5cb93912880fe, 0x67944b5c28ad739c, - 0x079f1973e99f18c5, 0x3892e1ed0ae8f3c1, 0x487eded21df64f9d, 0xf4536971aee2ddde, - 0x8ce1bfd3d68ad04f, 0x786e5b62e53c46e0, 0x98bac7191444f53d, 0x66839dbab2c6d02b, - 0x7c8bce319bb15e84, 0x2675b8e555f8af55, 0xe61fd18dc23b9d86, 0x1ed0ec1e61de209a, - 0xf68a86bc64aeb2c9, 0x5ae5ae6b1972d6a7, 0x1531025fad24912d, 0x05eda620fb6a77ce, - 0xbd70160d482d321d, 0x6b87c10c6910a764, 0xfdca6f279c438eb1, 0x8afd9cdd1e60adcf, - 0x9702de073113eb68, 0xedf26d2ebc49a6ff, 0xa0c7aaf785faa386, 0x701723acf7e4300c, - 0xa1f4fe88355e2f86, 0xc931578c32c7b4f5, 0x57e6c85a9eea7a78, 0xfdb7febb7084c1b1, - 0x15b27578256d5710, 0x75df5a9f946389d2, 0x6a10786c27f60f67, 0xd13db34fc32cca10, - 0x2ddd37827eb5e2cb, 0x4153e6262c799542, 0xbca50b3546f5162f, 0x8cfa17d08a59bf18, - 0x76555e06f01fa2df, 0xf9b915a9b1c8cdc4, 0x6d7c7ef629a15f35, 0x659da55c6c53254e, - 0x5779ffb3cb86887b, 0x807cc7f441b4b187, 0x008ce572e9e1ca5d, 0x9ab1cf7e050e2255, - 0x38a06ee1a595643e, 0x24f7f022f39b7528, 0xada6a3005da8c730, 0x10eb7cd0a3f86b40, - 0x632772584dafdb86, 0xab23c5ca2bc3c0c1, 0xeaba26809e095806, 0x75cc973b7e82dcc6, - 0x59caf380e8699998, 0x38347a8bb5f3e007, 0xa1f4c243428dd443, 0x520c1afd2915aaf1, - 0x731ed8e7507e9e12, 0x683c6d72461618bb, 0x0f761e848bbce97d, 0x890e7aa52d6e908c, - 0x8a795b05ef1f4b21, 0x146f25796b7c0b6c, 0xa9db46dd8a0eed1f, 0x86176b1287061de4, - 0x96cc303d723bd1f8, 0x8b77fea1d5d87212, 0x74d592acdba619f9, 0x2d51243334d9dd2a, - 0x66a34394276e11ca, 0x7f9c203e8e62a3cb, 0xc4eb5a154d8514a9, 0xfc896dd3cc50a2ac, - 0xbf0d7fd967da230c, 0xdf25506739d3c42a, 0x97871d8005f68e8d, 0xefcc7e6e7456285c, - 0xdbe5ca6f2e123719, 0x20cbc8e38bc268bb, 0xd17d079abc4d7986, 0x5668ac37d32df205, - 0xa047c0b5bcd21805, 0xb73f7e026d741cb2, 0x1bc6c2d2611696a3, 0x4fe69fed9f7056e1, - 0x3c01109d7ab2ec7d, 0x83a40f93e8901b39, 0x87004ae5a176dd1f, 0x8b9ae7935207a7cf, - 0x2c35bfdffb4e3c5d, 0x1ca5fa81c416e79e, 0xc96bbcde55b79ccb, 0x4b5dbc610429081c, - 0x5da5a050c67b8c89, 0x12f4468a10d7aa16, 0xbbba2ddae73f1330, 0x24de15d7fe7d64d1, - 0x55d32d5f59ac4032, 0x8ad2de93498c9c22, 0x15097bca5e06c066, 0xa1c41297956f998f, - 0x24585f555ddca960, 0x26a9a02e79ff6d01, 0x29974316af6cd918, 0x8f0c4e124227e62a, - 0xc06837915350aba4, 0xdc61db2b0feb4690, 0x6762534cca16def6, 0x844ce6d7d93804e1, - 0x1531677b40dfca94, 0x463b1052a9120281, 0x82021d8457aedc1e, 0xefd1d5d5a714aa55, - 0x988abd81956603d5, 0x8bba91a1b8b66aba, 0xc0c7179ee212a7b0, 0xa1b368cecedcc421, - 0xa107b55e2e15deb2, 0x89a8653d8070c4b2, 0x27a00795381198d1, 0xe842780d7cf406d6, - 0xa05af3d6f2ae3110, 0x874a4a5442978569, 0xcb283b2826a350b7, 0xb3a34521c47005b6, - 0xf5b85e588f6ae236, 0xa633e9da76dbac1b, 0x018f63a3c4206f0c, 0xe902487c55744dc7, - 0x9a64faf59c91b256, 0x27261ab2566b825a, 0x505fb59927a9d6f2, 0x9e8ac9e82fac08c1, - 0x0980f389db2233b2, 0x4aad46944c35475e, 0x582ace01bf9a07e9, 0xe31d992497f30151, - 0x4e5e841705cd8030, 0xc0ebd307bcb517d0, 0x6e00144bf7aa6465, 0x68bf3a5d23e2a451, - 0x7775b165d058fd4d, 0x333b2ef38a83028a, 0x8a8e5ed3b5f05bc5, 0xe2b6cb51765dffd2, - 0x254311881cd29b2c, 0xf3e08788eaa8299d, 0xfe5413491e2f9aad, 0x17f1de18d622d5a7, - 0xb77ef53de3685c16, 0x96a90813f2c9e5ca, 0xb66e2a1d31d0ea27, 0xa30b7cda6415c2ea, - 0xea5d8c0332e1169a, 0xdac565fab2953579, 0xd368a5b63678591a, 0x3e9e94082e323aab, - 0x8c65aff4a4b66459, 0x3986d2a068f5bc73, 0x7c2f61afb2b21833, 0x5a56379711b8bd38, - 0xaf81029472abee3a, 0x711b34e460250cd0, 0xb0626dfc43885ac7, 0xca1b7764f06a0e7b, - 0xa88bdc11bec8c039, 0x6cd4af4ecc6bc58b, 0xa14a30f68ee44183, 0x18c4246a0e9579f0, - 0x51f0752969f39d8b, 0xe2dc6dea0854fde6, 0xf0822b4869a62980, 0x3d4ee71e92a2b5b3, - 0xb01aab5a138b940e, 0xbf20d0ae9c2d98f1, 0x7d439cf6cad03c1f, 0x68a5b02dfd97e70e, - 0xae1559a31e86242a, 0x47129fc295561089, 0x42b5ef30a05654fb, 0xfb0490fbcdd7fc43, - 0x3af0f2bfce2c8106, 0x50acf99de12de893, 0x45554a7abebdef93, 0xa95cea3ec444a518, - 0x361c6d1dbf532dad, 0x104ba9c52139eb46, 0xd82f036ef8e62972, 0xe3bf5c43b7cfe015, - 0x6e4ecb4b010852eb, 0x294695c26bd5a19b, 0x7545f7614335d132, 0x6bbe738e8e9cd014, - 0x437874873d59fae3, 0x1a642f8ff3bf2eaf, 0xd0975b777abd7af4, 0xdb98c39c2472a4fe, - 0x8904a249bab3eb24, 0xfb2e065542a87ce3, 0xaeffd20a61cec0cb, 0x69352f71eff9ab35, - 0x603d8a901f0d3fca, 0xbb4f0c5d7d456262, 0x2638ee897cd3f789, 0xb4a77784050d703c, - 0x5be6b108913c26fb, 0x38be35371d2e2d02, 0x3837339f8809893e, 0x60d10df11dc812a4, - 0xe5a7b893ad28009e, 0x2e78ba212bf34abf, 0x66070574fa6deb06, 0x9e3932d414c774ad, - 0x2e3e575ffc95a337, 0x4b651f093407b19f, 0x1967f4679905d9eb, 0x2b26bdf1217ccd30, - 0x6312156c94f7c56a, 0x0e356ff20fec666e, 0x5bb08f48bd9bfa31, 0xfdf99d8a3a77b220, - 0xb200f24929040510, 0x9309fa8e0df272bb, 0x9338c39fc45c5fcc, 0x4165eef972184767, - 0x921f3ce1d2570738, 0xdf8324955796ad2e, 0x19a1337b134e19ed, 0xe40f67a824378062, - 0x3c77af462d31210a, 0x49f14f7fcf191d35, 0x0dc138d24cca183a, 0x12bc60e66d823150, - 0xfb428169c8ee1542, 0xe90d2708848d8bb1, 0xd0031495d83fb68a, 0xe00271cf727f0415, - 0x86e194f43cad9db6, 0x223cf8e7d9f87d20, 0x75595e1d6bf9e28c, 0x8f61389183d032de, - 0x5943da7b05a5a4f0, 0xcf7a54e90e09ff27, 0xdd3b65b0ad2d430b, 0x9cc67a371a5fbe54, - 0xb3dbc3d93e875c10, 0xa55af470429455ec, 0x26ea48a9a533fbe5, 0x03bc4bb30bea7531, - 0x6fa557845602c634, 0xfb790ac11b09bef6, 0xd7b00d2addeec181, 0x1d7ef2bcaaa182ec, - 0xfeccd8fef0ffed48, 0x46ae488e9af37784, 0x3f0a65bd8ddac848, 0xeda8a54a84c7db77, - 0x5acd96e73e7cccd5, 0x91df8b942d79b7d6, 0x2fd509eb680af888, 0xe53823d0022deb79, - 0xc7b4c0fec49335d1, 0xd4adfdab79cdf41f, 0x9b1e670c2a1f3603, 0x3e42b951239549a2, - 0xa77dc1f3dfcf047e, 0x3f81ba6e39d4db2e, 0xb7cb754a92addfb7, 0xf2b043daef6f5c51, - 0x231421640f8e58c9, 0x666c2ae3eaea7738, 0xbd2eae1c92e84044, 0xbc82baa71a5cd08a, - 0x63cc08fd2831be84, 0x9407bb8e33ce1bfb, 0xd5733d072d893568, 0x0e045b14954bc9fd, - 0xce18e8a1add59adf, 0x7dbdb10c0781af0c, 0x601997a369228acc, 0x16203d7dac717474, - 0x5f8c28eca6f4d222, 0x6aeb5a543e0b2f6c, 0x76b40c4ca52d8c43, 0xd8123a4713d1b774, - 0x9142e92ef17f74f7, 0x2be60926ab4a9926, 0xde0c7008ff03659d, 0x547682034efec13d, - 0xf64a67ae20ba35a9, 0x97d5f0fbc727c4c1, 0x84821519687ff379, 0x9a6fcf054e5c2c2c, - 0x0256dc58143168be, 0xf9de3cfd7d836bae, 0x934f6044e158fe7a, 0x09f61790a7f8327f, - 0xbf7d3b660142d860, 0xd82816a085c84281, 0xaf00df2b0db3616c, 0x824755f83e04021f, - 0x3d86bc2c4d0eb6e7, 0x0377e26be02b1600, 0xfac4b33dce67a2c8, 0xf6a077f29cf87bf4, - 0xdaa2a520f9c1b590, 0x21ed4a43778043dc, 0x099e007b1046184f, 0x07332ba0a13f3ce7, - 0x4dd02f8ff369c6df, 0xdc858b7247d6e624, 0xa67f179b0eae79ae, 0xcb191725966f0b8d, - 0x04013d4b7ae563ad, 0xbb94ebe8fb101cd1, 0x4a27e47bfc6c6ce6, 0x584171cc5c54c6c5, - 0xfeb9ec8af722a5c0, 0x631e0498e08aa19f, 0x5ff0bc40bcfe4e6d, 0x00eab65e934b3ace, - 0x70e3545c7a09de77, 0x5c432d4ba52db20a, 0x45ece20df417435e, 0xbac8f7e7e8433d7f, - 0xdcb8666820aaa473, 0x9da1391c21e1d570, 0xce86cf70d05b341a, 0x835c179aa3e57355, - 0x2e5392587e60aaaf, 0xa897b6a47218b85a, 0x552215f524f03907, 0x6d4f55b9f3b7dad6, - 0x7658380b330da9db, 0x2a6397611b8cb577, 0x756598a3d805c879, 0x63185d6e2f900fa7, - 0x15298bb8136cc7ac, 0x4939bb3f98030099, 0x4329fbfbc10ca4aa, 0xb8854f11deb03811, - 0x25116328c87dd8c9, 0x681c6aa288ab5edd, 0x6bdf1f47a59bd3ad, 0x60c8e05aad3afb74, - 0x60b4d735eb27e581, 0x30c6edf7375209fc, 0x7b4bc33f37871021, 0x016902edc9bc3d4b, - 0x87598b818cc76647, 0xe4d2f0d4ce273d50, 0x9e60271ff852edbc, 0x65f4772c2c5153e4, - 0x984d23216681ac49, 0x06315c9293173b76, 0x6b81394dab386cce, 0xcb9a60979248f9e7, - 0xa3bcbf70092e356e, 0x8e485e0ae55e2fbf, 0x3deba59867354745, 0xed8e19c8f5aeb007, - 0x0fc5366b2844516a, 0xf0787092414ac62c, 0xfd0dc6a79a391bdd, 0x00b632d5e29bc851, - 0xe9435fdd6c02ed80, 0x18b0465ce1d34702, 0x91d93fa72dda5b98, 0xcb00429aae5ba1c1, - 0xcca3a4bfd8eec3f0, 0x152aff9333f6422d, 0x307e9b2b210a7286, 0x4330534cbce43e1c, - 0xa24b4e7eb62f81ea, 0x9feae875867d079a, 0x22c2f6554fb1f052, 0xe44b6b16df285e74, - 0x76c4981976eba103, 0xb2e660a177a1b405, 0x6765bc0744443970, 0x1adb8ce12c39ffe0, - 0xe73e595890260e15, 0xef3a51c2a323cc3f, 0xd4680780ce067e2b, 0x18bdc1e0138d7718, - 0xe58e8de7143e3dbb, 0x7a8fe5d6b18b8bcc, 0xf9b15f1a5ac71cac, 0x07e6729aeb96a9f7, - 0xebf3da95e0c2d599, 0xc1240e762cb02eeb, 0xf1352359c1db98ea, 0xdc3b799e91f222ef, - 0x0a4d60448a7ca598, 0x36810d0b875f5dc6, 0xdfd28e6b4b9fd2c0, 0xd3ba54564d5ceb58, - 0xacec80baadb1492a, 0x6114be69ad9b9002, 0x03553a9f3128a0fb, 0x7463e4dbe892d00c, - 0xd693a80228b36e22, 0x80751f47d3daf342, 0xffbb1ec857b15526, 0x5639c6e72ae1560f, - 0xdae0414cec888f69, 0x011e3bf269741587, 0xd19166d49a925650, 0x3bc6ef116ce11941, - 0xda70f8081cb4fc75, 0xe66f884df3f2912c, 0x8dd83fb94706d96c, 0xa04c4efdea6afa89, - 0x69d0435761a09e82, 0x35cc269068f3fa4d, 0xb925e452526146e9, 0x22b352d5a5002135, - 0x792cec1110b2a765, 0x4c6a1614e4dae95c, 0x79719e22b04a95e2, 0xe22052eae2e13e5a, - 0x6a7238a52539f0d7, 0x8a0f377a8e204b3e, 0x7788bad972992e58, 0x6f875733387b5b83, - 0x9257972a0b488987, 0xb40caaae724fa6e8, 0xb2d89e6ae6c5ee7c, 0x6104435a21b77bb0, - 0x986ddbbf979af5df, 0xecaf6bdc61e33be7, 0x45e9c729615d5b65, 0x0892befab55ce136, - 0xb6141f6aa2337dcb, 0xc3c5db89f28ce416, 0xd3398989e215a87e, 0xf1f63b86bfa1536d, - 0xe7625f16c187257c, 0x9c9a7ef9eeecc07f, 0xfac5050af18c13cf, 0xebb43abff6e713be, - 0xe3a33a4cbd01c317, 0x21ada937140005de, 0x55244e0f39004739, 0x8f2b4448160c66f1, - 0x83d16ebd9b4ddb77, 0x8623d713704f5fbb, 0x9045f1883af96844, 0x6a7760a60ceaeb7e, - 0x8cfd062676b52592, 0x9d6fb66a4eb53b1a, 0x43903450a16fca9a, 0xa5533fc9625a2f5f, - 0x8c26c4b0bbada818, 0x63ac76bece4bff58, 0x3be3f7db72d94029, 0x50e2b42d8280b8e1, - 0x6494f650fce2bce9, 0xb65dd994c7cef393, 0xe5fe3c505a85fe6d, 0x2b03581820340cca, - 0xc20396501b5ca00a, 0xe1587cb0bfd49c4e, 0xe178c083281e65fa, 0x92cd0b48330aed3c, - 0xe2399d0402b58a4e, 0xe929b086ef597f58, 0xe11b42bb6f353499, 0xfaf742bd50e2744e, - 0x6ee8fc37c316f8f6, 0x5f2ce60fb02caa99, 0xdf103c3067f122eb, 0x004ec267f0ac8bb6, - 0xaa71d703fe199442, 0x48be9e28144bda1c, 0xd2898ac22d18aae1, 0xfcd42d82b02a4c8a, - 0xbd4d6c6b0b283c9c, 0xc98b7dd982b5f484, 0x4bf931753115ed26, 0xe22a764b2c76b4fe, - 0xf86b96c66554f3b4, 0x24df9fc44385ef01, 0x6bdda1f7ef1c0aed, 0x701008900e5cc3be, - 0x8b27319fd8205491, 0x92ce807f58bb3a62, 0x60637ee6f010b3aa, 0x43e049437ff6f8e1, - 0x726b27bf47cd47e6, 0xb18f23d71ca363a8, 0xf8515afef6427f29, 0x303fb1eca31d4943, - 0x31306e225b7f9eb5, 0x138b5d0f4fafe917, 0xb5e9b3bdc5dea9ce, 0x88a344eb8ce710ec, - 0x12d52a808d02623c, 0x06a1aec77d5bd9f9, 0xe67a93d41aa7dc20, 0x519a5ee1f9ee711f, - 0xa486c12923a46f8c, 0x6d14147f822fc06d, 0xb1685c72147a28e8, 0xaa83f0ce41f43423, - 0x9725ee6cc0a2a926, 0x836085dd3014cf05, 0x3d2223a0977ccefb, 0x4360915e79383583, - 0x831998dbcd91734f, 0x8e3dc8dd1cb2a3fb, 0x6dd3ee8715c570af, 0xd330fd8bc3ab1a3d, - 0xb00c4cb20e975a08, 0xc7db11d82ef74ebe, 0x2ed9126cf16130b1, 0x2d52a173eb0fb798, - 0x003bec1d9d57bcc2, 0x49bc1305f0775089, 0xa1a924c41d5721ee, 0xe8e63e150f50ff3f, - 0xf96b021b86a31a22, 0x011bff0f684e6e81, 0xa0179117b795724f, 0x1197a07ea06b8de7, - 0x4521943b918d7fbb, 0xeb5ba9904ab5f361, 0xfd821bd37cb8c2f1, 0x8b931ef1c2135448, - 0xfd46452625bdd7f3, 0xf0b1231e5db45953, 0x063574d0352432d1, 0x82ea03ace1b40a2f, - 0x05e5ded3e4b1e8d8, 0x929ed8e193f210b5, 0xcafa3501c9d23979, 0x951940d8e3a69d8b, - 0x696e947938774ded, 0x147979682a18ef8b, 0x64cfb7ba23444509, 0xfe3dbd4a2678c8b4, - 0x5d50486a2943ce7d, 0xf7222fea75f07257, 0xc140e5b0c1f25859, 0x8604d373d2765a8b, - 0x29602fab895949d6, 0x1b6e14a6bce7fa30, 0x19f61b45c5355e32, 0xb895fa9656ce89de, - 0xb8afc49f571ec0d4, 0xe56969e30152e02e, 0x74539c7af5f50e6f, 0x5a7ea420d31a7d3c, - 0xac582548335763ce, 0xa8b12e35676c0de8, 0xda82520b12336b18, 0x04499654b13b0bd1, - 0xdfe6468c8753864d, 0x7ee5695d4f872b1c, 0x08f7461dc0dd0cd9, 0x267b013c86732a1b, - 0x06478f206462df47, 0x02dc7f8ec0e99c1d, 0xbafa7e776f12806f, 0x58213eda30327e00, - 0xe2892ee9f1cb11a9, 0xddca2f7c6e0792b9, 0x90271461bd8d71e9, 0x08683789d66544ee, - 0x2dbca0d9eb26342e, 0x43d46010227659cd, 0xcfae45ea00b3fdbc, 0x5ecbf2bf17ea65c3, - 0x791554c64f37f26d, 0x3d0d26f6df9c5b63, 0xa657556ed490b7f6, 0x3d1d22e7da2362a5, - 0xb60639be2f7b4a77, 0xadae7f0f837ddf1b, 0xb4953abd1e95ebb0, 0x20e5d84844963b91, - 0xf2cf0830be795c11, 0x30ab128456d11b0e, 0x40c80975c19caad9, 0xd7e530bdc51d92d7, - 0x2c6744c46292101f, 0xd530741d49685f1e, 0x331a35db27d5022d, 0x3b7a6be9d0ab3676, - 0x35338d814da4c6e5, 0x35e88f4d275bee8b, 0xf64d96f442824de7, 0x2755956dc495c4dd, - 0x185a318a3c24c1b2, 0x771c72dd813e5632, 0x4554313faf5bd65d, 0x847a8bf9e5193ca7, - 0x9aa1341676d7d991, 0xd8bdc4abd11cab4e, 0x2e57e96d27e923e3, 0xd8b1b5d20bf1b4ff, - 0xbf30ab6ce0d2143e, 0x4e8c28e0dad171b9, 0xa0bcf5e015d8f27b, 0x435fe4979471817e, - 0x52884ad5233cd7fd, 0x5c02de59b26eac44, 0xf17fa5ff710ae52a, 0x20f140aabdefbd1c, - 0x0898a3f42f549cbd, 0x563ac7b705d77b96, 0xc38e813a9afd035d, 0x0b08441064a508e9, - 0x55de827191737ed3, 0x8de5b2fa9b2f21c4, 0xb3ea94080702e6fd, 0xeaec40606e224e73, - 0x1cae950c1d06e95f, 0xfce0e0d0093d8102, 0x0e5dbff0ae87da7d, 0x4f21bd58804289eb, - 0xe2c283f5aaeaeb91, 0x353e0250bbfbddd3, 0x5d86e1a7e8183fb9, 0xcaf06c277668bd19, - 0xaf2f9b92c119e1e6, 0xbb60620eef4aa9fc, 0x6db6bb70cb9db9d9, 0xa7ca3a32bff2b1b2, - 0xcdae6d6ccc43b15b, 0xc4ba5a3ddea54e1f, 0x99e2b599ca42092e, 0x7a8a32283309b008, - 0x5a0ea54020c0a459, 0x02db74b093dc5e5e, 0xb28c6c32d5fb07c4, 0x56a6b5b384e75d15, - 0x5efd05aeb6a3f34b, 0xfd3f3480f1bdad11, 0x1c7f54350aa49ccf, 0x4339ee25899d5fff, - 0x4d77212ffa386f54, 0xa4c6f10b0a3a4b3d, 0x8d70f861fdd9fe0b, 0x52eab4e040c0215e, - 0x17d442065e8e1bfc, 0x0fd6868858fd9a05, 0x30f7448e8c12a773, 0xb3c655c9df11500c, - 0x6399bb856719b704, 0x75c2a6af0f47e933, 0x2e6234d056502aaa, 0x5e405d9550b05a5b, - 0xeb94a7cff8bb575a, 0x2c23da687104266f, 0x336dbcd7d56bc0d6, 0xb8ca1e01cd5d7a9e, - 0x52c172be3fd87df7, 0x0896e6254d54a0f9, 0x15dd274ddfc4b099, 0x484148e6682e3aeb, - 0xf06cdb9a724589df, 0xeca9e17eab4b653c, 0x4b6ef73fbedfc887, 0x10180202b532c92d, - 0x8fc9498669fb35e6, 0x2624a886af623c94, 0x7131c6ff6dad97d5, 0x20c3da4608e7b58f, - 0x1e36bbfaefd0d616, 0xe3dce9f218f7de78, 0xfa62e34531c11f2c, 0x62e189e841d14ca8, - 0xf266d6edb307b251, 0x823c5c686e4e2144, 0x7e1d19e32ac6d5fd, 0x8eff8862f0ba61d4, - 0x40557acad81c5c85, 0xd13ff4963daf07cb, 0x185fa2eb0a6545d4, 0xd223823ea11c8595, - 0x63603fc351d70950, 0xc38c270331d65bd6, 0xa1005c55b7c59e27, 0x01686651817df2ae, - 0x4a435deaea063512, 0x1547641a364d81a3, 0x52a35d16f423e7d4, 0x788b49cbf5a86371, - 0x2ed717ce8ea3bb68, 0x49831307ef96d5ed, 0x22d4a8bfa35149c9, 0x64b8c5abe78a7813, - 0x99f41728c951f7e8, 0x94655356db786cf5, 0x6d393de9cb352d9e, 0xabca521211eaa2ac, - 0x5c9bfd8e9e150418, 0x6ab3e4d2e2610278, 0xe7d488e7a045a8a3, 0x131991eb5ecfc408, - 0x28d1a1a57baf509a, 0xae3fb89bbe60b7ff, 0x14c5bcadcc6f4769, 0x952a3c5169ac2569, - 0xdb2f099ea0861b46, 0xbe022ed894c5b38d, 0x545934f1d4bf79f1, 0x73a6b6ec394c0805, - 0x5d85ecd69cc6f5cb, 0xbc90a891c69324a5, 0x55b66c8eaed01af2, 0x285259c3e2d60b9e, - 0x5e611a230231fda8, 0x147ba77bef350276, 0x16f990e0f8681661, 0xde95ba8fc66d5d0c, - 0x8fa314cda13610a3, 0x9264158ddb36607a, 0x9b176053d750df1a, 0x594dce946810afd8, - 0x045e0e5bd92aaeb2, 0xb5a6333fb9e1cd1a, 0x61a9b5d1f1054cb1, 0x2eb4384401c4f708, - 0x0e61add50688a190, 0xfb4e68d0c9bfb741, 0xf1426640ae78dfbc, 0x43b450cf78c3efcf, - 0x45644cc65dc64dd6, 0x5adfa9ebfd8ced15, 0xfb6afd5c95267112, 0x48975ca21a6a21a4, - 0xc3023ad35bdadf40, 0x7a683c8b12b6ebd8, 0x98e151f584b46c3c, 0x02840ed66e9915fd, - 0x9f8bc359298655ed, 0x49725e906b54f56a, 0x107b268889f72253, 0x78c9e5eafa37c510, - 0xf33494a6426d569c, 0x750d1b4a23910b0b, 0x04d5423f82998d72, 0xaf985f33bb5d6804, - 0x7fcb45cacbc16cb4, 0x27ad6742f80c9f3e, 0xade1f456973e698d, 0x07c4dde77b32eda3, - 0xe95e908d4892c97d, 0x61bd34943fe2774a, 0x95dae59d3009298b, 0xfcd159a901d0a8a3, - 0xeb022e747a9bba47, 0x384e8b297a3600cc, 0x614dbf4f2c3ff5c0, 0x71439c824f6be963, - 0xb82ca378956c1673, 0x2830d7f787fca139, 0xfe96bcf4d7d2ab68, 0x45a7018a030b7ee3, - 0x836a637e366416bd, 0x9aa6a0de17c80088, 0x5578ceeeb2e9f332, 0xef71f48296b7068e, - 0xb8231b2aa5c6d089, 0xcd84693dd819222f, 0xcdd94d170a1ed0bb, 0x68c6e4295209886b, - 0xc65a86a352df4657, 0x9c9f4c89d36d6329, 0xea85e9ad10f8d063, 0x60955b1303ce25c2, - 0xe7c6f4801122d0c3, 0x45b9d4c62cea42db, 0x824b2dc9257f4ad7, 0xc2a29d4ec50e5120, - 0x08796a094d641258, 0x0d20756d86101a11, 0xfa01f478e25490e1, 0xae1e2bbda20fc36a, - 0x30ec95c3629389ec, 0xf49941b37d4dc502, 0x95b6137bae3bbff5, 0x8479e65674fa2841, - 0xc366d8ae98e23d62, 0x0560b2a85f99fadb, 0x0645943ffe0ebb50, 0x97ff6530ce63222f, - 0xffcc588fcd88b6b5, 0x0b5d088d1af64948, 0x13a4125fcbed8d98, 0x60eec3dc5351f514, - 0x0f4fdab56c12f196, 0xf42b6479dfe890d7, 0x49ced3b2a341892a, 0x6f489e11c4f2f563, - 0xa3e7906e0800ea4c, 0x6d6ce56e88b19da8, 0xad7accea715a24ad, 0xdae8b66df39afab4, - 0x649143bfb795a97d, 0x620bf33c23fd9963, 0x6760b747c5c7026a, 0x093b0d9a44468ed3, - 0x4425f8c50a663eda, 0x19ee9facb7a5b74a, 0x645fd5fd07216130, 0xe5f9d7f2ee47ce18, - 0xaa98b91bd0b2a13a, 0x06542db41630a300, 0x4969e39a29867149, 0x9dcc7e7456ec3395, - 0x2c373c23325f2b6a, 0x5404c9305a2b2365, 0xe00d26e935b40505, 0xb66338b1af315770, - 0x565eb7b409b3a5da, 0x2b895a2ccfacba17, 0x17276deeb23bb4d1, 0xd5d0c79382703267, - 0x3282097e99075f11, 0xa075a1ad9d811f39, 0x70ef1c6f2a10ec8d, 0x42dabc23f06a0488, - 0x2596ad1992db6801, 0xb96d38841788db1a, 0xcc6405924429a4e6, 0x3eb9b87096205aaa, - 0x7df08c803e4ea5f4, 0x455beb421eeee810, 0x2045728a45dc9123, 0x4f5ad6647403ae33, - 0x794059f72018feee, 0x3623fe1f4bc38149, 0xfa7c41de263a0667, 0xcd67a892af10d084, - 0xc3770c298a41e6fe, 0x4c856e881f137f08, 0xc4f850de61c69d68, 0xfa0ef5da5e0539a2, - 0x52076b42ccbad387, 0x2041523de945fa64, 0xbbbcae5cca85cac5, 0xae6a3c5423df4d8a, - 0x5eac296c0b7ee422, 0x11a4ec37c9d225ec, 0x98ee358ac76d052f, 0xa28bd0c395c89ed3, - 0xd790c36dd0f858c7, 0xc0a6f2b80ba105d1, 0x3464b85f05e22b66, 0x76063c06a40c6be2, - 0xb4e50777cff89564, 0x0f16860d5228ceae, 0x84ee4126f94448c1, 0x08936ff6b202b4aa, - 0x8657c747aa2655fc, 0x689f283d988eed5f, 0xc30482562f3a2497, 0x2ddd7409d8c53e7c, - 0x6772e5c5cc7731fe, 0xb144d9ae05e5e7d3, 0xeda599f039b6ca7b, 0xfd4ce13f59103c23, - 0x1f415ec754b3b784, 0xa2f249ada8e84e11, 0x3c3814571b8077c2, 0x45f0e06bf4dcf786, - 0x0137aded796ad680, 0xe81d6f0ade7421b8, 0x7044af319237f16c, 0xb20ea0bc8da3e3aa, - 0x718581e2c2c6fff1, 0x56a5bccf33ab1dbe, 0x6a7109754d358441, 0x1f3091bfffe0e552, - 0xfb9c72c0573a9fe1, 0xe49d088dc97d638c, 0x9f58d4ff9feb64dd, 0xbb95c9734fbeede8, - 0xf328627a96fbe049, 0x024e3775f352b912, 0x1826dc75c812ad15, 0x22bc4ef23a8ead2b, - 0xc6898e3ca143d8a6, 0x94f41df2677b8cf8, 0x2816c04f2838f793, 0x300d2137c9e52ac1, - 0xa2b5820e482887a2, 0x1fcfdb00fecf0e83, 0xc6dc035b1eff5cf5, 0x9a66e973c2a50111, - 0xccad368dafc50c76, 0xeed5dffa03676bb7, 0xede1402311b0ecf1, 0xdf8b95b550e6bebf, - 0x44710f41283dc354, 0xb13dd2f027179d2d, 0x8102e7c3a00223db, 0xa3cee3cb3c32071e, - 0x7893d3f6cc4ebf05, 0x7f7f6308c650a2f1, 0x3f542132b84a03d2, 0x5bbe5ac7ece0d1a9, - 0x29d78ebcc7d9ab23, 0x9d2bacb0c558d7eb, 0xdff8f6a1bd37fa84, 0xc72d5ffe40dcee26, - 0x98b1693c2aff0012, 0x1f2a0722120a168e, 0x4c6595728dd8fc73, 0x70de0300097bee65, - 0x76d0309a33a87e66, 0x08e1ceeb8efb47c3, 0x0c3097e50bbe270b, 0x4eb51a4f91d5d151, - 0x30eaf0365ea1a728, 0xaf9cbd611b5d74c7, 0x8503807f474ab221, 0x39ee66d06e1bc996, - 0xcc085e60f175ad60, 0xacc0c7b02698b851, 0x4cd0084ec00f1b11, 0xf3df5a3606e90c56, - 0x07ad715b6952f865, 0x269f242f382bced9, 0xdfb5c5d90c6929d5, 0x608b84ff2a4e1a06, - 0x3e9239859f0f7035, 0xff114f8c1ef228a6, 0xb35287c251b69394, 0xb40b54262db2cc1a, - 0x6e1a99d05c74ff98, 0x889584b38ce876fc, 0x8c8c20bc8da2af0b, 0xc27391d07edbed3d, - 0x4f6863ba6d1088ee, 0xc0485a0a35384c90, 0xddba95542221ae27, 0x590a650f7e97ec48, - 0x9543e7a8c791b3e5, 0xc21f5f28180a2d17, 0x9de580670fd28fb9, 0x618a5f8d2ab7024f, - 0xae63ca1b7c841494, 0xe33d4b521cec8147, 0x5e71605ae0ed7feb, 0x88750edacfd6fbb7, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6ba80d6d9e595bef, 0xbf74e3a2dcea2b33, 0x6caf1defaf37aec3, 0x05fb7d6f85a9d77e, - 0x6324953a900b2d09, 0xb41d83e7132852e7, 0x1e1dd0e57108c827, 0xee4afcb8f9f4ebb0, - 0xde4f506ef4f7f569, 0x8ca73bc863181420, 0x2ee9d0741b938ba9, 0x285680cb732d977a, - 0xdcae1f50c2e5aa55, 0xd082e3bd3bd721c9, 0x3b8262e5eb074dd8, 0x430ccf7a349bf3c5, - 0x697bbfdc594fa341, 0x360f0f6280b4e319, 0xc417b8b41aa1502d, 0xb0c70506afcbacf9, - 0x2ebb284c2f7f32a2, 0x08e9d3878c44042d, 0x1dba1d5347e2faff, 0x3e15b1c1438709e1, - 0x5901c4f1954d7afb, 0x2367e97ca9f9451e, 0xdf4a3d76b632e2cc, 0x9ff330659054b186, - 0x4962ed3d807bb4ef, 0x76af4a9cbed1d494, 0x379f297db24f6299, 0x5db0c4d44c0115c4, - 0x66d0f0b8adb1b594, 0xd73a91265710164e, 0x44f7e33ee7de32ac, 0x97c0c025b33e1902, - 0x2954c3cdd11cc6fb, 0xd5cb87117b75347c, 0x0eb11501d2585808, 0xbe7587e40c17e2f5, - 0x57918e44b4715a33, 0x04b76edd2c9a3f01, 0xfc52614ae3119f30, 0xa14c520f005cce09, - 0xd0a6a276f9bfe270, 0xf960fb0536c682a7, 0xf8ffd7a93d177870, 0x87e4dab2cde67a33, - 0xe397dc97e31f18fe, 0xa9d27661d2b7098b, 0x1b57d7beb4dab663, 0x3dcffc0011459cd3, - 0x968ccc7975670966, 0xb1110a7e0778599a, 0x0392af40057fe183, 0x01136010174bf091, - 0xab90fd6921c759fc, 0xe19a8ef47a385cc0, 0x7388de94ba9aca10, 0xf3ffb4d5b1314d97, - 0xda177f11ca6cc2e0, 0xaa6ee679e0d72c12, 0xf48a9b5ab93fa5b3, 0x1163a065305498dc, - 0x71d9ed185cbf8ee1, 0xe87cf67567f0ab1e, 0x9262ead9284d7808, 0x2cb5d43cd596a7b8, - 0x5fef752a106e3eaf, 0x18060962e036100c, 0xd6cbd3ae55b346df, 0x0a2ef6708035fc5a, - 0x2314520ef630546d, 0x2bd78448443bf776, 0x8b907c80bcb43477, 0xbd3a07eae299b7c7, - 0x84faaa9b70723518, 0xc54afe037cd43d70, 0x87abf08000fdd0d8, 0x65d13f1a33b81459, - 0x2262d86765467952, 0x5ee71bd0c12ae50a, 0x17a82729fcc74a77, 0xab3774283b961f61, - 0x9aac1588549837e4, 0xbfa9c930db51210d, 0xaf140f711bb37d3d, 0x00929a771e94853b, - 0x82313d4b1a9bb050, 0xd013b38b73dd57e9, 0xb0d8b279df7e043e, 0x53eb6734aee485a7, - 0x564f78bada79f4c9, 0x5017cc7648649561, 0xc169197a6df80d80, 0xe5cdbe9a4cab5a03, - 0x9428d2611c978d99, 0x97cc5f8faf299830, 0x3cdf07da75870abb, 0xa5076d9caaca215b, - 0x2e01d129c157aca8, 0xeb4870bec5a157f0, 0x01cc4c5f5e615a08, 0xa1790f82cac8affc, - 0x221788e13f7c4c91, 0x58d7e502280efa4a, 0x255a7cce431061d2, 0xa5ce56923ad9043d, - 0xa233dc77ee419864, 0x347aed9c799aa53b, 0x11ca654570b96d0a, 0x5656778f6294c9da, - 0x4b9410948d5e136b, 0x574b9aad45e34a96, 0x9ec281430d14c61f, 0xaa0d009e0d893d73, - 0xaaad5bf3a355e88a, 0x138b3f41b169c060, 0x0460703ddcfbc08d, 0x86f7d7c990072ad1, - 0x603860a22919cc22, 0x07f17c2fbeca03d4, 0x18bfe3ecc7aadae6, 0xf25ed5d952ce0638, - 0xec89d8c7fba547bb, 0x37e39422c2849133, 0x423d6041f4e5d85d, 0x8850748882eab882, - 0x2077225973c19257, 0x3fa0008a78b59d74, 0x1bcf59341e13c65d, 0x51ca4da47b61b55e, - 0x47c19afc9603c0b3, 0x5ba14de8d485b715, 0x3ccd7164ed35e57c, 0xfb77018d829e711c, - 0xde9c444da65ddd34, 0x519fed98329f3adb, 0x738ec0b6e7c9e6e3, 0xb741a50753bc8bc7, - 0x7c13508c3bf06c67, 0xa91519399bc5492f, 0x88341b6772c135f9, 0x3065b1c1f1fd7536, - 0x6fe3556135384ea8, 0xad089435b3320bb5, 0xf1a1863876ab5200, 0x90f87630152eec81, - 0x2be5eaf04af33dda, 0x5da11256b39e9539, 0x385634c5f66b956c, 0x84e8df37503e1fd9, - 0x69c4f0e31c3d4e1a, 0xbdb98796e88b888b, 0x8f638815e8234df5, 0x83963fd83cacb98f, - 0xbc2b7733f159cfac, 0x88111730b6e68f79, 0x58d51ea9b3fe539b, 0xa077e6ab425b0826, - 0x4131b5b06290358b, 0x85cc3ab76fa87388, 0xca017932cbb7778f, 0x0c94e0dcc0d8b2d5, - 0x36374ac1d45d1bfb, 0x5ffceb267182f45b, 0x02624993b4fe8022, 0x3f05baf855fc313a, - 0x4db56368dd60cf56, 0x8849033f25770438, 0x74173a7daffafd4d, 0xf8988d85f165d92b, - 0x148145e8afe98fc8, 0x633bc57b8bd5210d, 0xe2446cecb15f2baa, 0x9e69418dfb3485b2, - 0x4135ec1a2f0b17ab, 0x778593414bf40793, 0x6af7214224bfd27c, 0xd307526d9139380d, - 0x4d29f65dc1ab9c76, 0x65a9dc5afe3c159f, 0x29c8cef9e24c2646, 0xd4bc9355a2287256, - 0x61bf74dc014398b0, 0xf33e6e159ac74c7b, 0x7752d745ff8c2a68, 0x0606581ed90d0c86, - 0x83fe2220e057e157, 0x9fffba45d2fd9109, 0x1814aecfab9f35ad, 0x8df551f24bd41a39, - 0x89072bad58aa6fdc, 0x9d8329821db69f78, 0xbec9701d31174300, 0x7caf2e14e5faa0b8, - 0x3beda2c3b3ae8a47, 0xc1edabc8e44ac9a0, 0x1c6971f03d250493, 0x19af641011b39386, - 0x6f6e17744515ce52, 0xd9224cdf41c9721b, 0x504bf630d5593f05, 0xe043b20fa5ce526b, - 0xf8447964c63b631f, 0x18ce408244819977, 0x4ad7ba658904332e, 0xc5cff7f1a689dd48, - 0x40d0e3bb2b49f922, 0x9084c0a8eaf63362, 0x5ccfee1b98df32ba, 0xd2e3ea2a37f0fd67, - 0xe9be4a66e542cfe5, 0x42c4df27b443c8dc, 0x0af533eb454bb47f, 0x684f114b5e0d2c0a, - 0xeb0650ad3e892fb3, 0x32352378c6c759b2, 0x0afa2bb9d4d45468, 0x3cbd99511fb6a29c, - 0xd34d415ad8fb679f, 0x29c0352811f53cea, 0x7b6e5c9c1d2fd319, 0xb423700d6474da31, - 0x5c7317918e592810, 0x212b5dc5c8d87c68, 0x2c1e6cd6e0bb127e, 0xc2103e097b708a7d, - 0xc3084eeddbb20e43, 0x061a9159fc390d92, 0x7f3f1498c5022245, 0xda19f6bc9fa1e420, - 0x8bdedbcc2d410fbb, 0x6589cbfa6efe036e, 0xbafd48471ec3bfcd, 0x6c1c3eee7589445f, - 0x09bf9ea6f2ccd49d, 0x2bc8ed181fae376d, 0x190e40a0aa7efaa0, 0x784a42a0f02406b0, - 0x534af0489194b5b9, 0x5bd021701cf0fb60, 0xaeb421979d02745b, 0x8c3a8f28184b18d3, - 0x615990d325528f71, 0x77d61f56ef48dab6, 0x121b39ecdd0fcb2d, 0x86b73fd8ad7d4bf6, - 0xdcd8ade86269454b, 0xeb7ca8d2fe4974d3, 0x7ad07dd231b7b389, 0x817b19ec11aa92a1, - 0xb5ada6e94fcc0dc7, 0x1833b9bdcd69d646, 0x6f7908b696ef6f69, 0x5e5816f77dff6cf8, - 0x681b74d54d2780fb, 0xae8fe4c24fc5e2f4, 0xb1dd6bd193a137ca, 0x015b345f2274364e, - 0x1aec139421778339, 0x2645c6889d8180e4, 0x73075dced32cc5aa, 0x58acc87f97a76aaf, - 0xa77dec0805de5eb9, 0x154f9371f5a3194f, 0xd7258e65736002ba, 0xc1d4fc6e0a60cca3, - 0xe4872570338cf073, 0xb4b666d0cbb49848, 0x9d3e75f761317e8f, 0xa3333be64d2ed7c4, - 0xfd294b6252d1d1b9, 0x9cd734ec41cafb17, 0x4cb3f8ade6c1e3c1, 0x54316712841fe1e1, - 0xabec61bb7799f721, 0x691b7d7fad3649db, 0x069961e6a1b63e59, 0x91ae6eec5f7a5882, - 0xe35f05562e000119, 0xf2f909f2fc30d30d, 0x1363be8b9888dbd1, 0x8d90183a3e1d0a86, - 0x92f3c37b65e68cde, 0xd6919899978214e3, 0xe2d717e45d36709b, 0xc77e7633fe1f66bc, - 0x8dd07e422b331fed, 0x403dc270560e1f44, 0xc75ddad670e464ee, 0x3d2102b9f8f07ada, - 0x9863ddfe594b94d0, 0x7e445ca34bc5e70f, 0xd6cffef8b2c07883, 0x0d87aab233dbf06b, - 0xe3bfee82c3f869bf, 0x527c27a656008480, 0xd1836b07c4c3af36, 0x99eb569cd86f57fa, - 0xde43cf4bbb769d8c, 0xa151a30545424711, 0x1be851934129b601, 0x8e1f369b314e232c, - 0xbe7018e9c85f8cea, 0x4b92adc66da7ef5a, 0x978807220192b66a, 0x39df3cfb1d7bd73a, - 0x8284e06ba40d72a4, 0x75bf7732890881dc, 0x6d2a498bbe19415e, 0xfb664e541825573b, - 0x976a952d5f4ce437, 0xe1ba110a02b3cd62, 0xabd205c4ddb02df7, 0xb3e5f0aa4b9ea9fe, - 0xceb156780c88e260, 0xd01f2786b33c6a8b, 0x03de408ce1049efb, 0x149ee09fb6252d3b, - 0xbb7ea066696241a2, 0x1b0f001e4ea6fad8, 0x27cb4d7bc8043bc1, 0x084b64ee849cce66, - 0x37d4cbc31864ae4d, 0x3c7e9f77b16f0208, 0x64f8b04523d7d7c1, 0xacd75b3f0bc656be, - 0x18586afbde497301, 0x176759e3e788e938, 0x0b23cf2f02291758, 0xc3918a77b0c472da, - 0x69cd54c2b22bf285, 0xaaade807f33c799f, 0xe78a725e72ea8f46, 0x0c92b013490ea332, - 0xfc462d3e5f60bf28, 0xdd7bec6880b45067, 0x18137cce039085c3, 0x31fffc94ab12acad, - 0x0cf3e2b5b8af26c1, 0xd3758616f1ae8d83, 0x1369db9dc16b4213, 0x52b7a52bf17eef23, - 0x71503135a5d4502e, 0x08d4295066556b28, 0xa7120ab63857ac4f, 0xfeaf84a2b3d9825d, - 0x194678ac4eb0823e, 0x910da1e33e5b9cce, 0x1312dc0fa35d96fc, 0xca5242da171f343d, - 0x6159c17ecbc7669f, 0x163b12e29b80b8b2, 0x167b8bbe8346c2c0, 0xc0773e549be95e05, - 0xa83c52b546c249b8, 0x7ad674c9bf5f8b8a, 0x84909b12c5801b83, 0x499edbc0dfc90b2f, - 0x175626366fe5cdf3, 0x9ed024f0dc0460b5, 0x7e5b2ad761d07816, 0x64257e7c1a66ce70, - 0x8df9810108b6862a, 0xb407a22e3b001fff, 0xacb5e817f4798f70, 0xda454b80a60598d5, - 0x7b9e33f535d1dc38, 0xfb73e396fdfc7977, 0x6feac162d336892d, 0x53a13b856555b289, - 0x72ffd8c7ae166eeb, 0x59756694c17474db, 0x02694cdcbd6819bc, 0x7d8931535ef516b3, - 0x563cc4be6cb70abe, 0xe8071a717b6385b8, 0x8a69ea8a5ae0fa38, 0x8b54a6b06fdff099, - 0x5dc231c9a6d9e8f6, 0xef2ae32a3c91b24c, 0xc2680a6b8dd4a71e, 0x8c0addfcbb4b9df7, - 0x6d985dff061bbc73, 0xbdf46ee1d160b3a9, 0x5580fa38dfcd336e, 0x48a953367c6b9719, - 0x819ce7e0d7f6a332, 0x8fd88cda98092ea4, 0x4c2d0be98db0410a, 0x45ec5331c5012a8c, - 0xcb26f46b50cc776b, 0x9160c00ad454116c, 0x250fd86321584324, 0x4b56071e8db09fa8, - 0x407e529b9bf5ab00, 0x33367f04537f7109, 0xc11f9fed6c01261d, 0x961fef17d19b79de, - 0xbf95ed6e1b87a599, 0xbe4caf3d8dbda966, 0x8b3f2f6d68e43a0d, 0x67a82743cbc96a8c, - 0x01ea2ad03dc62deb, 0x4dd3723cbce1fa40, 0x0a069e1998b7dd58, 0x070b93a8ad85b8d7, - 0xf8c81b3c6cb1dd49, 0xb0bcfcb6ec0ca057, 0x4bfbebc2351f428b, 0xf595fa6d4054003b, - 0xd34e29f482c8e04c, 0xa5617a0ca711f06c, 0x5d1a9af1a6b0e97b, 0xf503c3501774bacd, - 0xd8f090492d15b114, 0x7cbc66982960e088, 0x51f2b9387991b35c, 0xf3cb07ba49b25908, - 0x2892880393de5846, 0x4e300aa2d084a47d, 0xc79a84d2339e48a4, 0x4638ed418b76c15b, - 0xc0fc8b084fa762fe, 0x3723d840b65e6505, 0x8497993d5a01d1c2, 0xabe096827f18c1a7, - 0xb9adf761351376e8, 0x35a0d7504a8b1e10, 0xef2e6b32283a77ec, 0xfad2f136100748e4, - 0x644ea41c9e698e24, 0x1968015936792a4a, 0x9ea1cedae07c5469, 0x4864ac682e3ce51a, - 0x611ae22d395d280d, 0x83ce6aa8216ff3b0, 0xa1991bdf6ae86e3e, 0x56e791d68bf3f2ac, - 0xe19ba907a64772fb, 0x75c5a4733b31d752, 0xda6df88f1788144e, 0x026bbeb995429cf9, - 0x1f6d1248511abbf9, 0x9de4f443ab074134, 0xefe1c258d5911d0e, 0xb6dc66bf94b999d8, - 0xef55eb299f980c90, 0xc2d20fc8b1f90833, 0x94b961c0b4614d50, 0x7aabe6fbde19818e, - 0xd5775875ec2478b2, 0x3dbc0cdbace727d6, 0x9ecb4a32f5eedc0a, 0x97066199c0f193ff, - 0x7dcc5b63f10ec589, 0xcc343d343d52fd84, 0x1f60a17cbc276775, 0xa7ade1bdc6748d64, - 0x600b5e5badf0abfc, 0x1a40a916dae8efd1, 0x459321785cbbc22e, 0x47e23995b59e79d1, - 0xef58d8f372fda762, 0x1a1013b064ce0dca, 0xbfca49c6c05f325e, 0x976e9a96a82fa37c, - 0xc90b74de984948b0, 0x59604ae4e8421b43, 0xdc5a52a7b0df5f2a, 0x5217f5d834f56ecf, - 0x71a6814dacbb73de, 0x0d52623b4e07aef7, 0x150f13dac5b6d1e8, 0x7a2aaad6e2c8d87c, - 0xfdf67e73f84c5e6e, 0xef49f49024030f38, 0xdd2a86ce0bc94230, 0x537a1124bf405617, - 0x415af0bec9af7057, 0x668a9e81379b39e9, 0x0bf7230894b3aebd, 0x59a933b7344f4e2c, - 0x8f63ad8abbcfb01e, 0x1bafe18804a66ed4, 0x9a34109d3bb4a200, 0xe7abf0dd43df16cb, - 0xeeaf9a2b53e5cc5c, 0xd23360ce09b0707b, 0x5133eda8143af28e, 0x5fb02ee11d346f2a, - 0x2d16ead5593f8950, 0x97051b0d8c33bafb, 0x5c79340b195a15fc, 0xb592c5974b7d317f, - 0xd07ad581a6a34d97, 0xaae011e7632567c5, 0xa91067ba576ea8b8, 0x3526a8f75406cfce, - 0x1a547f3ec5223cdc, 0x96be2ef067ee60da, 0x89f2d6dd15226d14, 0x98b77b17fdac3a4b, - 0xb1d8b9937251d353, 0x722470cecc426467, 0xcded3e014730fe09, 0xbc47664cdde3d0e2, - 0xb589437570c105e4, 0x66a2fc59a2b19a6d, 0x4cda8e9acb289ceb, 0x439c69e86392c491, - 0x2691dc6670365132, 0x2432318bf89b5120, 0xd2895f272fae2bd2, 0xf351b96eab33aa4d, - 0x41e3a8b67917914b, 0xe858d1b0af9c8f12, 0x486a21f52a1fa878, 0x5f1d0be89bbadf66, - 0x11b3fadf38459f57, 0x55ac1fe54f76264d, 0x1ff96a738f57aaf9, 0x9a8bb1fd5a344961, - 0xcf7fb0413911881e, 0xc19bb798789472fd, 0xbec0e34ca2f4baba, 0x1e0ef06e5fdff850, - 0x9b2008caed1198e1, 0x4ff9fe3e9646fba8, 0xb34d6ed466dede7b, 0xa47a473934a29144, - 0x3a0f9e52749f8376, 0xd16e2b50c98a8829, 0x07db409dadc05655, 0xba814b693d5c8cb5, - 0x95ab401906a566be, 0x83655c4bd7f0ee89, 0xf98e2d54a6cdea90, 0xb77169096020c8e6, - 0xbbdcc5fcd61f675e, 0x07f5d94dffebbde7, 0xeda568bdab379402, 0xdc851894cf611c3e, - 0xc04f36b7630cb051, 0xdfa360df8a28031b, 0x6eded21d69dc730c, 0x88bda3c31218a19a, - 0x017334c871e3e30b, 0x7fc594fc0ad29359, 0x1e1784e864d01c88, 0xe496ed538b5541d6, - 0x6bbf165cb96e1fea, 0xe4758602a9b4555a, 0x0b24e044232f7577, 0x9b87c65c7721e089, - 0x429dc32b0267a05a, 0x6082e4d3d3edb268, 0x4a77d47da7e98295, 0x0c768f3669a0a032, - 0x14858d4678d530f2, 0x123c70f29c0b69cf, 0x50d103c6fa518c90, 0xd1aa915a3992b7a4, - 0x195ff5862e27d5bc, 0x9649d1d851b05a00, 0x5dc55fa5a2a60a04, 0x743e71de252ef060, - 0xc81df09ca90c3525, 0x23e3733c1b9ab4dc, 0x7c98202b29a35ed2, 0xece34e68743e4b16, - 0x0ecfa166d16d1eee, 0x31b77aacd04adedb, 0xec0a7ffa4edb87ce, 0xb83834a90361c9ac, - 0x3e294d944bda3caa, 0xdff637a3652d5b93, 0xae425d263e9cc999, 0xf6b4cf748b42d8ac, - 0x05857d057c8e7a15, 0x5a49a66e979f35ef, 0x9385e97cd46a9a20, 0xcdce10b63eb6183a, - 0x4368a964f22f002f, 0x6aa20b55bae2b618, 0xc41c44d8ca2d080d, 0x16b9a7375218cd0e, - 0xb30ea60999e72bda, 0x361ef27b809be1bc, 0xf2e4da0735754088, 0x6f9d7f5407674963, - 0x2a6979dbf009d406, 0xc1fcdf27342d6f43, 0x0f0b56a965c5be27, 0x62b1574fe2a52f2a, - 0x5c37624e153d0793, 0xbecf6bb888f91ea5, 0x9fc392da79852e1c, 0x24b6ef9e4683e1b4, - 0x354e59fa39c81319, 0x7d4296b051c38b90, 0x940d8cf33c2c135d, 0xf73182fa73938ace, - 0xbd6041810097c64f, 0x0a2b47c4bd877d46, 0xbe8d4d8a3ea43524, 0x37fcfea59664274b, - 0x1adffc6a9de447f5, 0xe0ad0cfb9cf9ab27, 0xd1fa176c030451d7, 0x107a993bd31a30a9, - 0xe69919f8ef78e669, 0xdde2088d5065c479, 0x1195c973f1c9194f, 0xf599bbb08c1dc8a3, - 0x7c2644c762eb4da2, 0xde7141c70326e72d, 0x73b84ac8e4f5986b, 0x0506978d60544185, - 0xd3a025af51855a7a, 0x4255d3d263d8eb1f, 0x4f78a275d25983d9, 0x733ee25406452183, - 0xbfec3455a1ba14e5, 0x24de877bd3b9d41d, 0x93ab8958d14b11e8, 0x300dc0f105b37ce7, - 0x27fe1b19f06dbd19, 0x4573779b41989ea8, 0xcca0898939579291, 0x0f5958995476c263, - 0x68dbf21f62c28bb5, 0x4b6b8ad1dd0735ff, 0xe13c2c92799090d4, 0x535df82035c6c642, - 0x2d17f2f0193af29d, 0xf16a3e183415b731, 0x33fa9ba186cef653, 0x7bd33de594f046dd, - 0x4df415b6655b2c45, 0x26d53e986edcb178, 0x1ab99c3dbc37ea80, 0xded6f35cd35c63c8, - 0x77161d063149b571, 0x43e56bb16e53d05c, 0xf8b48d8380704f3e, 0x37a7a5760f09aad7, - 0x2d1862e85b490f58, 0xd006065cc7f6f8df, 0x2037189b8f34a654, 0xd9beaf1401605d9e, - 0x517264f6c64c0286, 0x72dd43b5b5c1e9e8, 0x57295382bd92a099, 0xa15a205324044aca, - 0xc9966948eecb5e23, 0x6335159bbe55a098, 0x07c8355e785fb8b8, 0xbd84473ea8950b18, - 0xe3fe2b40f168c1f7, 0x81e144eef9e64518, 0x4c020634c82fbc84, 0x2ce0fc7a5992dd7d, - 0x5dd951fc6c0389dd, 0x70aeeb6da8160cda, 0x636cdd4d184d8508, 0xc021ccef61a28386, - 0x3d215d34a8cd6bba, 0xcf8f3a8fb69a8b60, 0x49e98a03c4ff877d, 0x326cfb4aa1c7d5db, - 0x6ba0ce471943c7f2, 0xa52f6dd3f97e0058, 0x0c7f9c01ae6fd446, 0x3113a25e97321534, - 0xbf6bb6e70ab83dcc, 0x2b3775e21e458b5a, 0x6665c42bc3aa7613, 0xb110c199efe30aba, - 0xdffd56f480052717, 0x45ae47969e523beb, 0x5308ac188cb02c57, 0xf11cbaa3c2dd249a, - 0x1d11cdafc80a443c, 0xbda1d9d57ffad7c7, 0x5d33b39069883fcf, 0x6fde77086b2c5e92, - 0x088cd180cd90704d, 0xc2f6bf4ae94c72a1, 0xfd481187bc85536d, 0x0af99190a55fd5ed, - 0xc5fdc1f1e5c20805, 0x24ae9582231f95d9, 0xbe798af004ac67a8, 0xf10787cebeaaae54, - 0x30ba9c0484908244, 0xbabf0e78ac882179, 0xef78e16b1ace6436, 0xba69426b1f8a1075, - 0xa7882c0eff80d61a, 0x1e9e1ac22806995e, 0x96e8f313987ccfd7, 0x2b3254b81b80f4b4, - 0x602b00a5b4371310, 0x0b3b4bbd673c27ff, 0xb5adb3cca0f90ac6, 0x22d956d19295d906, - 0xc294068109ac92a2, 0xba1c7a257678b8fe, 0x4cfe1e8785bcdc18, 0x5fadeee2434752f8, - 0xf429ca362fc2dc6b, 0x1207e8810638c05a, 0x4295dd26e6189c98, 0xe4f5e7b7e18e57a4, - 0xe4f3f6c62b5286b4, 0x1a3f17dd2927fb17, 0x9350ae5157e1f4b2, 0xc0b76ff6b76451f2, - 0x792c0579f64b8cff, 0x511f3e4429ebacac, 0x780fb50a362959d4, 0xa21d6a961cbdc259, - 0x6d06939fd21cb76d, 0xef3d12ad1ab0c754, 0xc2455c7b4503dc47, 0x06e7f9f95ed770f4, - 0x7aac8a14f60bfb60, 0x5d4af428aa85c6c9, 0xbaf01f75432d70c8, 0xd7e39a298d8ec048, - 0x2098fcb7a60825d5, 0x9a87e12b49fdc04c, 0xa6c31269ca73e86e, 0x90e3285d7b03bbe5, - 0xce9d2d615dd342a3, 0xad8297590eac8dde, 0x495980f6fe34be0d, 0xcaa5216e9fcef019, - 0xdb4a206d9d7106ec, 0xe210f6d8083765d4, 0x8c880e4ae81842f4, 0x211ce5aac7f7f3a0, - 0x9112ab34b9522e0a, 0x9c28088d9eb361af, 0x169d1768bbe7b4ef, 0x2fdf29cca38bfc5f, - 0x1d7c91b2e2e6aeec, 0x18cb4f3ba20a92c1, 0xd86ef8ebb024fcdc, 0xf2a3c4f9b7d6444f, - 0xa868d00ad2b90d7f, 0x8045276c3277cfc1, 0x6a1e3d32b12b321f, 0x5f3a136868460cbf, - 0x8e5cc1e146647434, 0xc991453fc181e5bb, 0x2e53ef63da828af6, 0x88f741e5a442ff88, - 0xaee8f09c8f05ea07, 0x596aa430ebeed2ca, 0xfbf67ddb9f061e4b, 0xb8f4b112569e9ff0, - 0x19579d5a908ca61a, 0x1ee4e2714e04e371, 0x341abab6e1667c4a, 0x886dbc62a9e95f26, - 0xa03bd6a07108a693, 0x0f07dffa1a7c985b, 0x724569988ede3fd2, 0xe8e1b7818f4113d2, - 0xfbcafa9d88429a77, 0xc908e3e00ec06a71, 0xc519e73c73f90302, 0xdf8ddf47287cc5ff, - 0x9e7f880cc7f98496, 0x95f2d8842e9e99ba, 0xd0076ab4ce17e596, 0x382e71aab3c9a83e, - 0x6f618f30dc4265eb, 0x190c084cdc0b0614, 0xd91cd0ec703704d0, 0x7fcb7a5c5848656c, - 0xee5c5478b015f182, 0x746f1e5cb44bafd1, 0x7483822b70ea8589, 0xf80d49c4312e97ac, - 0x308d3deb37e98bd7, 0xee04c3c99f407bed, 0x9ba26239e4ef141c, 0xa6e7ed32a6b3d96b, - 0xb8348e9ae2cd9796, 0xb39f344320a221c9, 0x7a6ceac688fc4cc5, 0xaa96fbce5c1ed18b, - 0xcee2f16ed46a7d99, 0x31e7ae31a48ffb4c, 0x9ff98769517a0989, 0xf6e9441b173099f5, - 0x25d5c846d4ba79c1, 0xda39a1aa5ddd7cd0, 0x7d71729bd17ec9df, 0xa84231893cddee69, - 0x7d4e468681b07e3c, 0xfc071c8c3f0ba196, 0x2e5a99defb5c68c5, 0x2d35f21c6fa1d0a5, - 0x5077483a6c12015a, 0xc3e8a1d7f00d9147, 0xb39f48cbb562688f, 0x6c8b3f2bd9052b04, - 0xd2dd4b6d2a637143, 0xa5a86984482b9441, 0x16d54d0d4be141aa, 0x89338cd58c592888, - 0x3261a79be7003b15, 0x6a0f74434f838bcb, 0x6118d957e43f3e58, 0xa5687096724754e4, - 0x0e2937f091e7cdca, 0xf878457176b3cebe, 0x9bf13aeadb8358a1, 0x1563119d6e137817, - 0x0a16cb9fe9c158f0, 0x8c9e2b9011b7bc29, 0x97bcbb7d85a23b53, 0x97ffe6dd3e49237c, - 0x9fcb7245137261eb, 0x1981e3be0378778f, 0xed7df5ae39fbdf9c, 0xdbb5046a8ad7e05c, - 0x687391e185ac838d, 0xa6584128ed9e7ecd, 0x2643d0821805f321, 0x78fd9a64756804ec, - 0x57916a68fdf22f96, 0x87a8978001e0cbc4, 0x99b3a28474a6e68d, 0x3278c23584a55e63, - 0x6fc81c4ffa68a849, 0x0e413ff83f642522, 0xd6ba6e5cff3232c6, 0xac76405b3efffb30, - 0xa24f106e80fdfa89, 0x90df16eb42d104bb, 0x71c1a2bce2a20710, 0xc6a1620114a601ee, - 0xccdfc2decf2a961e, 0x639b571a21f1f77b, 0x850b60cc6e7e5b26, 0xb93b218bee04bd99, - 0x4ba0be61e5f586a6, 0xe2bd70101bae2aed, 0xfd28688758c19284, 0x65607c31960b1b97, - 0x0248e8bed0ac1e3b, 0xfac95427a2af24a8, 0xbe8d3195eff1aeb5, 0xda37c7fb32c4e085, - 0x338a21479043125b, 0x1a5afebc028baea9, 0xab1d9b3f5d3848b0, 0x964ff1e50dedbca3, - 0x5f8f982301dc9620, 0xd950fe06b70f6ab5, 0x99453c215a5ee807, 0x707cba2b712d0c12, - 0xd3dc8ab577afe3d4, 0xc0a682ab35bf2dbf, 0xea37aff5571d6bb1, 0x5ee6c34e007a28cb, - 0x4b5b1d79fd98284a, 0x822d754b38b8b971, 0x0c58464704335b4b, 0xb59d3572d5d70976, - 0x77fdae2d786e9730, 0xceb737037f3f689e, 0x27715ec99f1c4718, 0x169cf9d90c1b7c96, - 0xebcc6bab0add59f2, 0xf3068f6756fcf6cd, 0x44c8d5c16925ad21, 0x4ae40bbff2836d10, - 0x5ed26c79c6894d1c, 0xd6d9cb24b5445d2e, 0x35f49992e3674662, 0x550a3e3183de8caf, - 0x14575ecf312f6c3a, 0x4902bbb5d436ccc8, 0x79d90aacb1c5db86, 0x349298bf333ec128, - 0x10c92d95d00e1080, 0x32f98f50437a60b6, 0x85ee0c76f589dd8c, 0x4c8dd45f0ee18010, - 0x90e535ac678b6066, 0xf0f4fa6f097d4eeb, 0x14a40d5d115405bf, 0x1a5f89969eb4c39b, - 0xdecfc7c1bd22a838, 0x65ecdb952ba11722, 0x97efe5bb1c7a55a3, 0x8468c31291aad10d, - 0x490a5987bb1f6536, 0x22012bc3e765bf38, 0xb9a63fc565461c7b, 0x1f9d2e9d6ff7e86d, - 0x4464d41a80992204, 0x51ef238973d3604f, 0x74a5217d1edf96ec, 0xe09234009589049e, - 0x078b6b135f5c8575, 0x9e1549e93dd2250c, 0xb55f4812840fb5bf, 0x2a05c905593ce544, - 0xb54c2a7b4e916c12, 0x0d08f2c6f4636b06, 0x37f55f9546ef7ede, 0x0d05f19062cc21f1, - 0xc8649fc34ef55198, 0x47f411cda73fbc9c, 0xc143a975348550d2, 0x0d6d11fb99ab6f9e, - 0xdb847977f02c3f88, 0xc0df5a1063f70200, 0x1c906205134ae5df, 0x056dfb52d1f2151a, - 0xa8571383f72ce77e, 0xc7285f9e0abde084, 0xe74a678b0d52bd70, 0xf3b941d3701ec43d, - 0x0221b3ee521e33d1, 0x8f78416953fbe7c7, 0x3f095970131dab05, 0xab02c595fa7dce3e, - 0x350bdb13d3345db7, 0x1d1e9faa98554776, 0xa0d755ebe1325e61, 0x96b4b8ea60bc7b96, - 0xa938363122b4b39c, 0xb647220e1e7b59c3, 0xd4e63ef78dee82b9, 0x7c61ea473c315e36, - 0xc9913da68c9f7723, 0x3065f49ef4f805d8, 0xd1e408903b859219, 0x2f73187c39eb5d26, - 0xbcd7cc8c1da12e4d, 0x00d068f5aab9f85c, 0x43e48da4c6f24ff6, 0x81cefefde94e3abc, - 0x4ac04676c3162c9a, 0x505b0142e8d2028a, 0xcd811e43379d30ae, 0x0713956fa95904f4, - 0x1d3d1d5ce01ddd00, 0xcd0c5d8fb10e8896, 0x4c270dffba49db55, 0x3731517035254a6d, - 0x05b940151d93bf6b, 0x4cd91f0d1ebfdf60, 0xd2e637c2791950bd, 0x4695f8a6d412145d, - 0x3906e6a624fe15ef, 0xc29a53adc610ddfb, 0x7f066dd3a658ff3e, 0x60311e4d48848db5, - 0xd47ea7e7a6ba1bc0, 0x4f03aa0d4ee2dad1, 0x724eb1be1435e017, 0x15dffb3ad0b750f3, - 0x6a91da7e0e008f1a, 0x40aa3ace9100fbc0, 0x9cbd4086091a3f85, 0x17ba584e0214c146, - 0x06c24e245254d6cd, 0xbe2c1d8b8c235a77, 0x206b3f09969170a3, 0xac977b008785c187, - 0x52b2b50353d60b91, 0x7ff8fe55ce83a453, 0xc9758cbcea5e7b39, 0x0ff9e7c40b9f7fdd, - 0x4500b3204d3374fd, 0x7af188393e2d5f53, 0xd8e0dd4ec438bf8d, 0x974d9f9cda9406f4, - 0xea35cdf9a40a90ce, 0xee10560f7dd2f150, 0x6629dbdf77fe4018, 0x28b927b1ba6f71ff, - 0xa111e6873af1eaaf, 0x60c82b469096c9ed, 0xff2940a348b0d625, 0x6d853ed1ccd49a11, - 0xc533dd13ecc13f1d, 0x77f01ef16f95fa16, 0xb7af7f6dcc5249eb, 0x287b454eb6a75498, - 0x2070dce28ddcf2a7, 0x0c0ea2b77da9a685, 0x1fcab14666be4281, 0x5d5c1d37a7a774b0, - 0x576682624c84ddae, 0xcef34a956261d3d8, 0x84579a45668f0178, 0x89fba785feb9d7c6, - 0x38dfdde663ed4da3, 0x5c4cb15599dfc575, 0xa725ffbba4491f3b, 0x01d1b2e28ca7cbd3, - 0x01c3bccfe3c78014, 0x1b2ba952c699754a, 0x61e213b454fbf9f2, 0x864d9cb7f8c4ed1a, - 0x9f426ce566f2951b, 0xaa923ac6493ea189, 0xde61cfbf1792697b, 0x3fcde0fc67b6009d, - 0x9b08bdd596e2bd90, 0x57c8fbffb35a39a9, 0x54c351e7fd70163e, 0x1faaa586e173abdc, - 0x04f910e4e7d69cd2, 0x07c071378d356a7c, 0x9834386202ba5e69, 0x6b2275603cd339f1, - 0xf5e78709463472d8, 0xa7f475c34ad30da6, 0xf35c7ff1e38b5608, 0xd502de26e4bbfa5d, - 0x8e423dbe27ac78be, 0x0c703a06c6d2664b, 0x55ca004466eca6a1, 0xd8a953a9e694e7e7, - 0x638f18160a8f3e09, 0xd22991066106550b, 0x8e95e027d492f3a0, 0x1bd4ade3bb2fb84b, - 0x73d70f9489419393, 0x3a94ebbf6d2b92a4, 0xf516b3cd3b8e6b8a, 0x5a0821c7b1c9ed11, - 0x7d2b7bf62dba06ae, 0x8efcdc8a324a2b4c, 0x8633a222c0f00f3c, 0x148df8497afeef8a, - 0xbe93e42f449636b2, 0x8f124e96421c3efe, 0x56bace5507de2b87, 0xd1fd04e2f9bbc971, - 0x47aadabe3a4f41a3, 0x7bcd91bc772b4a4b, 0x47a1e7426559557a, 0x967fdd038942d9ec, - 0x5b9ba0986c50fc26, 0x8bd8e205b3d3e98c, 0xc18f6b02f86a9e28, 0x66d2672fb1119cb3, - 0x1191ac04b3ce431e, 0x709d4cf205ba79f9, 0x033b5f01a2854c1f, 0x62b1eec32bacb556, - 0xd54189085ad817dc, 0xe40f6ae0c51d2823, 0x0403c762e71c819b, 0x5695716631f224ca, - 0x03b6e20ed00e8800, 0x5440717037f13054, 0x8201a8a6b6ed4196, 0xc4768d3063d2fb65, - 0x789f92947c37db28, 0xbec6369c1f6ac419, 0xd13ac76d7f47678d, 0x0312b75c56b322a8, - 0x9edc6892171af139, 0x6e44f0d4b361d904, 0xd185f4cdc16ac922, 0x7602089aed899873, - 0x3659cae46334b308, 0x2d1f5d845cd56a39, 0x7a6d7931516eb2e7, 0x1c938bd46b2a3ac8, - 0xf8841354211159db, 0x8eaa70ad8b33805e, 0x8eacedbdfc4543c1, 0xbd5d70c64c63f6b6, - 0xd479809b1823f5f4, 0x6db286e9ce50a70b, 0x5922145d43ddbef8, 0xa2d366591b25847f, - 0x6e9b88694832aac5, 0x0647dfd325b775aa, 0x22a0044e46fa35b6, 0x2f876c203426e894, - 0x17989d50a40a6dac, 0x6869250c30dc012f, 0x3fd959f3906e1eec, 0x9985a136336e9382, - 0xf42e3e7d291d048c, 0x17a43ef715612957, 0xb5a7c4806a4bfe2e, 0x884995127035712f, - 0xb168aa05e48f32ee, 0x328ab9cd3ab0cb67, 0x334b17bbac8b1784, 0x1781f2f118cb83e8, - 0x5f64a67775a16648, 0xa6a00f503301cf74, 0x3ad542b9f92ee37c, 0x479b17d4542c184f, - 0x873830e421ad76b7, 0x3119ac128c572c13, 0x772daf77c54845a8, 0x35a08f7b3f5469ed, - 0xdb4796bef1f7f708, 0x27de394f50df1b5d, 0x170c5eb348df71a2, 0x8da0f6934e65739b, - 0x454907bc420523d1, 0x654653c403e4c330, 0xd60bec91c0f2d796, 0x4df6189e94e8af68, - 0xff1704b2c163b530, 0x3c546403a588c79d, 0xcb815f3510c411da, 0x9cc67a0f87badb31, - 0x00e102723cf0cfa1, 0x3e4d9889d7a9ada3, 0xd856c12c04401bbb, 0x3cbb37d86cf80ad2, - 0x385fbd0fa962a72b, 0x8aa1c2e852d51eef, 0x3111771a81fea7e1, 0x2adbab164473c104, - 0xc79454d92778e3a7, 0xd4de86bee7c53763, 0xf975471361ce1f74, 0xff53b221b3340749, - 0x21c83d6a4b54cdb5, 0x6f8217a1c20ac318, 0xe41b38db06925dd3, 0xa70b66b85969b519, - 0xfec59d517772568a, 0xf151ab8d6c4e577a, 0xf147490e4b13c538, 0x7362b37ab42d2a8f, - 0xb776b9fdfbf04c99, 0x5de66b8d2c03572f, 0xd6aafea40f87458e, 0x50c838e3af626690, - 0xf92754cb549ec5d9, 0xaf9d481240fadba6, 0x26c818e944293d97, 0xb73faeb6ee337b70, - 0x14a8b1295a1aec48, 0x5177599efef02c4b, 0x2584d2a2c3313190, 0x4d3bff9aab7b565d, - 0x8e2a71470acc72fd, 0xd36a72e7740822d2, 0xb670a2f584dbb89c, 0x5206e78416cb3b6a, - 0x87c751919169b51f, 0x02997755004343a6, 0x9b8d3c7b41487c20, 0x3e0bd6111b4287d1, - 0xccdcd0f680dbae4c, 0x31ceb43bb8dffc08, 0xc1bf34d7d63b41d3, 0x459fe030df6cc3c9, - 0xcaa0cc4d70c910b9, 0x6bef2516b746ca3c, 0x24e20744bbf36ef8, 0x239aeb3f444d052f, - 0xc0949583958f8633, 0x33bd9a430f2b54a2, 0xdd8a959f21a0a9b7, 0x52ef00ae7a39d4f7, - 0xa8ab3d1aefc29fdf, 0xb8cc7dd5dc46bba9, 0x31c38ef5fd21c21a, 0x6886b4f5ab487343, - 0xfc216082aaed970b, 0xecf3637845f087f7, 0x65c3686003ecc7e2, 0xb9ed68b04e297ce9, - 0x3f36ec19770da1fa, 0x756fdb017f19419d, 0xfe7db08b77ae2973, 0x6fbe12090ebcf0e7, - 0x852c9d3598dcb8d6, 0x7042d7771aa740ff, 0x9c72bb6d176f07df, 0x7e5761cfb120b3c1, - 0x57260961a8e5cd97, 0xca2e120a46c0989f, 0x9d05b33c91d76ea6, 0x451455bfb30a62e7, - 0xbf8d6b4687cbe7b1, 0xe089d47fd306ab87, 0xac2e597653ecdca5, 0xdc191d79888e1c34, - 0xd5c1fe3719ff0e62, 0x9b874f912007d3e3, 0x7d739e45faf33f90, 0xe705a51907f8200b, - 0xabd796266778603f, 0x572310f379a4589e, 0xa143998fe4c05fc4, 0xeac45d302e06c866, - 0xe2ee86749e4e6991, 0x16ffc457eee9a18b, 0x87e56c7b5cd614fd, 0x27f49d71d0a53a96, - 0x22df607982c3e0f2, 0xb21c4082ec7fa358, 0x1f4c16b8ca537bcd, 0xab40398be4f0a96c, - 0x2048b050f84d4f35, 0x4182c7fc26ba92cc, 0x6c8d3881e1bc1fc7, 0x7fe4ae7b21df29b1, - 0x350a2fbb61d8ff5f, 0x7a7240c60b396dbb, 0xc84b2544a083ebf1, 0x556b858ec60cd574, - 0x0f7f68c821d23fea, 0x09447e107b0e9f94, 0xe30659c66ad5d8c3, 0x4c43069ed13b5a07, - 0x5c1f813f7c9d3ab8, 0xbb649c86aca37d86, 0xdab549708c14a550, 0xb9f38b094e34323f, - 0xf4a680b33c3522e7, 0x56db76c4e7c812c5, 0x653591396a07baa1, 0x83c19a8b38a9f3ca, - 0x85eeb9ec1f7d4199, 0x841915b55ae12a3c, 0x4a8d75e8eef4e33f, 0xc02423681dc148da, - 0xd0ae5d2a15a25dde, 0x00adcefc2ec88483, 0x319f849928cf5003, 0x3f2c9abf91085d16, - 0x5809d7ef7e72ca32, 0x4892ba2af9bc255d, 0x0ba79e9cc5cd7d99, 0xa93543ebf7b852f2, - 0xae487de0e939465a, 0x0b9625993f4b2952, 0xa5e345e9a12ea00d, 0x7ff0a7932cf8837e, - 0xc911fa2173cd5c82, 0x1907daf2d9f9617c, 0x952005b028e08022, 0x01d0f6fa16aaf76b, - 0xa98e3a84b2f03427, 0xd649bf140dd0f6a4, 0x92e109d9db0885fe, 0xc4c1e4852ae19f0f, - 0xfd4244f6fd8b1ec7, 0x382e374e5224ea47, 0x3de7152457229a61, 0x3080f3348e599bed, - 0x8af858c87152ae5f, 0xc62cbda2cbda1e01, 0x816525c4418de8f3, 0x7df97929b652bd61, - 0x1647e46dce394b19, 0x218aae56b7e15b4f, 0x001b9c655ad7cd50, 0xcf9a95ece9ab6e9a, - 0x44f0bc945369c81d, 0xbf743cc9e195663d, 0xc0f806f1dd906bad, 0xd8dbd59384217dde, - 0xc333c0eea88ea071, 0x1b2bfea0c7310511, 0x86b2a3e9bbedd577, 0xe363225f2fe5ca52, - 0xc5fa61175ae57b1f, 0xe55fd5731b4a1745, 0x30aa73768dad2e60, 0x1fbb005063322af9, - 0x833db5e1bbaa87fc, 0x908ae9ea6486c431, 0x7b87ee310160a515, 0xe5c147d2d79a6c9f, - 0x37f0901244b0a59c, 0xddef6946863a4230, 0x726de91defc5a40d, 0xad4420715257d6ae, - 0x38ed303408196fc5, 0xa2cbc9b536bf63c8, 0x1d1b52d387f77e53, 0xb32db47c6ce5209a, - 0x13de4726a12f9544, 0x897a078286e0ffb5, 0x3640bc23dea72e0f, 0x117526606fc31f4a, - 0x9ba81c370783934d, 0x85e3cd3a79fa4cd7, 0x02f050388afed867, 0x184d024ece089815, - 0x68be8b4fbb403d33, 0xa9ccd3e106aebcdf, 0xe804d56e32de327f, 0xcfea5ba43cf97c00, - 0x3b2522c75b9869d1, 0xe256fb75c3704b5f, 0x555196c7203e7592, 0xb68baad0b1156488, - 0xd7a0ad3478e65456, 0x1533b4a622069d25, 0x1d01e90bd43d667f, 0x2b76f7ee53a96564, - 0x341dc0f1b6809549, 0x2a5632463915f239, 0x99b45c62bf1506b4, 0xe431ce42b3641656, - 0x0263925ff02479ba, 0xb9aedadf7d92d95b, 0x6142d5eb839bb3c9, 0x6f0c8c61b99b510c, - 0xd604ee367f1386da, 0xc3e8ffb4e8f2140d, 0x284a3574d5a88d1f, 0x5aa963bb5d548d2f, - 0x5c77bb44c914e590, 0x3400e170c0e907ba, 0xa1c24c927a2434d4, 0x4ed55484a833aec6, - 0x0cf5bb44dd1854fd, 0x6111efb00898de1d, 0x9e58e0ecf23c020a, 0xde15067dc2ed809b, - 0xaad4332534a80399, 0xd964c2393c45eb08, 0xbe26363b47873262, 0x1403dc1b34de805e, - 0x73b022c3eed8b9a7, 0x3198dc8118838200, 0xa7df6e3903bdfa0b, 0x55dfe1daa7693f57, - 0x4153a2bd15dc3c95, 0xe0db00ebd7102aaa, 0xd809d0c16b562d4e, 0x33f68c07455f948b, - 0x0ff2ad37ab775ffa, 0xf45eaf4aab728258, 0xd61bda838549e4a6, 0x91a25a3c25d43a9e, - 0x584fd60356ee5394, 0x78b7f90f16b82bcf, 0x51742ba9fa7084c8, 0x735a63c338033963, - 0x88519260196f8f88, 0x5741e5baf090f319, 0x7fe464b6b7487b42, 0x94096736834f17c8, - 0xf341ccc2256ef239, 0x9b32e0ca197a5a7e, 0xc222e8e3a91d7a9c, 0x14abbf6529b4f9ab, - 0xb5c5a433996fb92a, 0xf790340bb517e4ab, 0xdceec1394e5ebf41, 0xe87b35ef8f510bbd, - 0x8af38f8b1c0a165d, 0x0d93e9597ef01aa8, 0xb69089c00aacb168, 0x86df700cd1d7082c, - 0xd2cc9d867312fa29, 0xd3c91a8a6d8485c6, 0x200fc36e63bb45d0, 0xf9e3b0449e9408c1, - 0xcb8cc66cfd8b5eb5, 0xa0f2147892e7cdec, 0x405a3d9078bcee38, 0xd3e41c996f792a6c, - 0xdf0315ace36a07b3, 0xbef02c951a7fa0d7, 0xad501d9618ce8a1f, 0x1f4716a11c295aff, - 0xea69d7e73cac91b8, 0xc812d38905c60b2b, 0x4999ba140e86cf85, 0x16086d394098f396, - 0x28cce63cb472467c, 0x0333d54a83335732, 0x3fb5798c542c686e, 0x9b3eea69ff0b269c, - 0x1dfb9b8bfcd62808, 0xbffc11ba2c2cd453, 0x84fa1263dafb3ba3, 0x5dd4047e7d1b6536, - 0xfeee8e4c06dfd4af, 0xd0574c9826b91535, 0x0eddfc4713c847d9, 0xe3a211fd760c6c0b, - 0x96cb56932021d5e7, 0x1245962341804b43, 0x203aa3e5851c269a, 0xd2dab9454ffaea14, - 0x745407c6245e9d87, 0x3652768d77ec8385, 0xc898d68999756ef6, 0x5e40d61be1cbead1, - 0xe8d130a4227acd73, 0xfd8349c75e36cdbb, 0xacc5db39985866ac, 0xab2b759b740c480d, - 0x11c6cc7766427fed, 0x8a32a32e0121eae5, 0xf44f9272d4735d53, 0x2399d575b3fd249d, - 0x5be74f2440959d9d, 0xb4f528db8ac82df6, 0xa0d3ae1172ad86ed, 0x0dfcc68e485456b6, - 0x4bb9dd3d2a555047, 0x97f564156ab0e66e, 0x54c70edadc373694, 0x819e46d346e33eb4, - 0x621f4ba4d3a7ef90, 0xc960c804439fcb76, 0x80d076c77a7f6d3c, 0x7b207004393316f1, - 0x8bfcb18140b35e85, 0xe9fd8a1dc3c56eb4, 0x9528a818a22772c8, 0x115b10779b531fb9, - 0xfbab9c458778f238, 0xfd5fa694b61e34bb, 0xa113d9ff58515007, 0x1aed06b9ef841e05, - 0x6f81fa597ae350d6, 0x40cdd9ddf41d44e0, 0xd099413b93858fee, 0xaf6ca1fb12d510e1, - 0xc8c64c59a739efd9, 0x2ea053d3921d53d7, 0x1ab67abe81295eae, 0xf6b366a7ec12c9ad, - 0xde8d18aa3756b150, 0x89f02aa607ae4b4e, 0x52a4dd43b3ddb14c, 0xfbbe8a7d19ab512b, - 0x6c167a64efb1a243, 0xccfac4abe42fd439, 0x3c9595103e05a3f6, 0x0950b9df1e207108, - 0x6a03d09c32dea7a4, 0x5e88eb0d10e08e4f, 0xda0f548719fdc820, 0x5f37e1cd685fb228, - 0x4bbe370def21cd23, 0xb03fd7d70a5899eb, 0x9c846b472498b2bd, 0x44742c1c9f04780a, - 0x0225a479c5e82e45, 0xd9ab3efee0944892, 0x2b93901e09ae4973, 0x63ae5dcf5d76bd06, - 0xaaf9cd361ec18062, 0x0a9de070f631d8ca, 0x65a3f2e4083aac65, 0x802e4410dff9f5ac, - 0x933e1c8552c8d36b, 0x77242c4459fe96b1, 0x6929173bb109da96, 0x4e28787618c0119c, - 0x4e0641d70ae0c076, 0xe062b0cb2b5d5057, 0x66e86231a30efdbb, 0xa5224c68eb491ed0, - 0xee23a5a0e5168837, 0xad13f7f50bae58c7, 0x6255f18b49565ec8, 0xdb6c284dd7468269, - 0x7522197396a6ce0b, 0x74bd7023b732ad5e, 0x209bacc789da78f4, 0xb1a5d83bd2272465, - 0x64ba935c4cdac539, 0x9cd8f52f323cd65c, 0xd7befe83a1cf9b21, 0x7c9ab780e37e70d4, - 0xaccd3539ad967c9e, 0xe58e2e5872d112fe, 0x8df0850bf00a00c8, 0x29670c2ea3485f6c, - 0x7280ba22637d6758, 0x63dd8ef2f10d76e6, 0xc617dd1817288bb1, 0x2ba3f593bf81fcd7, - 0xb69dc3f9ec1ab163, 0x3536787d7beb5418, 0xf4b8ba3815762e6e, 0x062389dcb87920c0, - 0x3f39c52850f34b56, 0xca223251be5b2be3, 0x39db9670d6fa5924, 0x50d41ba8ee8792a9, - 0x4b98c18850bdbf66, 0xeaf31c1976874c17, 0xeab28fb2ee7d9101, 0x530155a69be22942, - 0x579ff0e03fdd7554, 0xd58b588abcab598f, 0x049c172d092268d5, 0x9ff22f3b94946989, - 0x5a69791413e434d4, 0xd1d1807ea7fe76a0, 0x41d1397dc454cfa3, 0xdec2fc5f824c5420, - 0xe07533eb58e3713f, 0x7d1c546f4612ed84, 0x3f4fb8bb23e75459, 0x196b2138f87e931a, - 0x669436452dcd4c73, 0x14243b4526afef43, 0x0146bdcbe1d909bb, 0x71215ed142e9f830, - 0x0ab820250a76c669, 0xeb8ed1dd4be5f41f, 0x32fd2b77f4d1aadd, 0xa86b380e1e8d2ea7, - 0x1674ef4edc54a2fd, 0x058c633280af995d, 0x86690f059447030c, 0xb6b6ffc387dde5e0, - 0x29eb633f897abcef, 0xf193a57229da4b06, 0xfaaad5ad5d8fe969, 0x4d7f6dbcb2e629ef, - 0x40982c877a2b19b6, 0x36be19582fe7768e, 0x97b45b12154ae2ef, 0x5b608cac9c8def94, - 0x8be5e7201e276d9a, 0xa2bcbb964ed449e2, 0xdc91b2b2618debc1, 0xed8e41c8922df1e9, - 0xe4f2c8e39e784b30, 0xa1110773ed8469ac, 0x804a7b72e40b7588, 0x77303caa04aff440, - 0x13bc78bad5baefae, 0xe1e2a3debb636ad0, 0x9de81c54ca0a5cb5, 0x235f08d4c5e790a7, - 0x8a87b164e2b0a8bb, 0xe66fefef0a7c6659, 0x40ea1993063a88c0, 0x846835f9ede4f8c8, - 0x3ffb4e60a5479445, 0x5f9e7ba458f7fc20, 0x71fdee06d321fc2b, 0x258533b53ef06dc8, - 0xa047ac673e94c324, 0x549634696280c1ee, 0x9c3bcc2ae57c1d59, 0x95d568720240e50f, - 0x756c82aa16220d52, 0x38f2b02c9f5f56b1, 0x6d332eedb7aa2b76, 0xd72628fb270bd114, - 0x2f641f6144329682, 0x5bc5588cf218afeb, 0xb20bc520b403382c, 0x1a2022f2b79a1fab, - 0x6609297faf341f8a, 0x29f2089bcc31f0ce, 0x82b978abb7cb46ef, 0x55bdd936d6688a53, - 0x55edf1199d86b7b6, 0xc12a6e907661e9ce, 0xc3e3f7ba1ed737d7, 0xea0fdfcdbffebcc4, - 0x9d7bac8653cf3ff0, 0x2b5fad18ffb37042, 0xac108be56cfdd26b, 0x7ffcd37c0309e203, - 0x1f14d7be0a24eea3, 0x470012824b447d7a, 0x706c45e5895c300f, 0xe90b7a2d44d0e521, - 0xfd8f29ac3919f6b0, 0x2f55fbaa424b512f, 0x1333a55d59e80098, 0xce970ffc0a52f139, - 0x1e1a1a8b3c9d6d81, 0xc004504e92f2c6c5, 0x9f3eb3504e689aea, 0x4fb5b57ddec22072, - 0x0f485e8ac6d456c0, 0x62c92e701548d5dc, 0x2f3203f5ac47def4, 0xde0598028ac19753, - 0x4133c6596aac7e05, 0xdafa4399a0ce47e2, 0xc6538bcbcc4bad6f, 0x5b397fc52fbcab1f, - 0x84e2e02302ed81b8, 0x40b8dbc5e94933fb, 0x6986bc7ad444511e, 0x438deca4bcdd9aa1, - 0x230d1e595346d5af, 0x3e3d6eb2e40a9fe8, 0xc2bafb69cb478d64, 0x68f9e2cd55df7c56, - 0xb3b1e5bf16e3805e, 0x66ab9f4e0c0a447d, 0xc7cbd2a5a9c31b3c, 0x6da100323d6a43d9, - 0x0095438529bcdafc, 0x27df57bc0bd080e8, 0x61f68ba946773e75, 0x0bb4016ee2b05731, - 0x3e99f8affa6730f9, 0x35f8cd06113f8140, 0x692be58c881f1497, 0x6850fcb71f4e3437, - 0x0dd0c4f5da457363, 0x45d54b3bebdf7783, 0x6fb1cb19eeeceab6, 0x5ab1e795cd1213fb, - 0x76b7cde88be3f857, 0x454b65e885c475e8, 0x253ab99c7ad47c3e, 0xfa781d50ecaa1eed, - 0x7d4b3481c8f29237, 0x9877551da3c559b1, 0x991294f242661e29, 0xd4f7a6253641e4e9, - 0xbb013a4c535e74c3, 0x0d9da78470983d54, 0x8560f7df7c140fe7, 0xe247b7b8cc1280e1, - 0x1ef70420ebf1dad2, 0xf1cc1ea551d17d93, 0x736eaffaa3f61fd1, 0x2c5b7f21a0ee02f8, - 0xe00d3051d43521a6, 0x6a8aee9197979abc, 0xca8be1d72fb24080, 0x2c83c30b00d928f6, - 0x8282c3797babc781, 0x811935570dd3a6ef, 0xa1bf60537720163d, 0x5ceec0878e299016, - 0xd95db7fad1a992bf, 0x75275a08984fea7b, 0xf3a64beca260f9b1, 0x06ae7891eacf8ddd, - 0xe74a604b26ec114b, 0xe5e8795494a413b3, 0xa4c7dc355c1f1c0c, 0xfafd10bbe1c22887, - 0xb4d4744e6df0729f, 0x706badb38a285796, 0x25dcd7aecbf62530, 0x84983d6091531ed7, - 0xf9a91c1746e986ed, 0x8c3c48d20594095b, 0x48382b612ead5e5b, 0xac0de8dfd0e34903, - 0x94fe970253308518, 0xfeba7a72c06ca7b5, 0x6e439bd87fb1dc6a, 0x59659aa0e780335e, - 0x500bf820162e6fc4, 0x3ab4e4adbbfeb856, 0x43981f470d82f41a, 0x771674b177628e2e, - 0x075eab258ab653c2, 0xaee2882a943ff301, 0x1b27473ca58ffaaa, 0x737c21968599133b, - 0x25d792ddf94f79e6, 0x2dfdc6525cc00029, 0x24aec78423bf5c57, 0x5b106eedb81f364b, - 0x2fd8db7d3c2b13ec, 0x5f7f95fadc9783ce, 0x1799e120f077f6c9, 0x1aa459a91f6a36c3, - 0xf3321cacb1511dcd, 0x1bceeac6187df713, 0x410ee9b376f30452, 0x793609400365981c, - 0x876e75d887f2f101, 0xdbc1484e8fb8706c, 0x75aa05b8f6ed9191, 0x14d2f3a1ca95c6bd, - 0xcddeb10130d22eb7, 0x52ed9ad38503b576, 0xfafe57cc176c0403, 0x8170cdce34667c11, - 0xacd06bc0780ef149, 0x195a1f0cd6ea9ff1, 0x713a88609d46960d, 0x93934526a00eb72e, - 0xb86b22c03e712846, 0x7219b6bea25d35c8, 0x4bdd36e3c2978c6d, 0xa0cba610e2905b16, - 0x298ac7d04908ad60, 0x46428b07c44e6887, 0x76d803bab3397cdb, 0x3684fa6529248b83, - 0x1eb3a76fe775d12c, 0x9c58c59842629fc6, 0x4da36a6d1bdcbe5f, 0x159e5f58f8a3b540, - 0x7b453db9123b626f, 0x4e301fd6246c0cfb, 0x7aa6f2679d795e91, 0x11ac21f461c06e01, - 0x65e30767660afaf2, 0xed53849379ea1b7a, 0x1faa814c84551f29, 0xd086aabf74a7f5f4, - 0x1fc2286d10b4e812, 0xf3b4ac72eec0cf4e, 0xd1b91e287d541729, 0xf15cf8717068fe3f, - 0x844f54c7cdbe429a, 0x822d30345754b8d9, 0xceeb383983678b1e, 0x98d230d30024ec0e, - 0xa3b5999b9314c56c, 0x8f3fdc8755b19754, 0xd3f3fadcca90872a, 0x204493e688d23400, - 0x4d93155dbfd67387, 0x52603a37b2a7d72d, 0xd3dd90b6faf2a91c, 0xd86ba08d63f3bfa9, - 0x297ea48709c5dcc8, 0xc5a60f70c232627c, 0x81f2c85651f8ab9d, 0xf242317d5d72fc58, - 0x40fc056098555f8a, 0xf92573b1f80b7be1, 0x348d64773f64aae0, 0x4d6ba134b1b1e2f6, - 0x524890475a4832b5, 0x4264218ca3f77aaa, 0x5e4505757bc44459, 0xb9c2ec758a2900cf, - 0x761e865824383c84, 0x7eb27b21e816f477, 0x654a05f40b3945bb, 0x1c0351ed82cff701, - 0x081e61762e7b4247, 0x20c7b3c613052d4b, 0xe890d639976d834e, 0x0e010c137d82a90c, - 0xf3c4669c734d3405, 0x2a69e3054051c51d, 0xd26532d6fe62c58e, 0x4e032b14c32861e9, - 0x2b05bcbeade9c8e9, 0xf5325c27fd2186ff, 0x09155cd5996fe687, 0x0741672c5c9d8e02, - 0x19e9663e9782c659, 0x4a17e9acfb832692, 0xb7db1d6020534ddb, 0x4eb80a906def8ede, - 0x6235ddf37fab84c5, 0x90b7ed200ff636cb, 0xdb567ffa21f84109, 0x6a6f290a5339dcbd, - 0x5a310ddebda79471, 0xbf4270200b033ddb, 0x5a9c0ad6cb2dc8bb, 0x578461089fb75ee1, - 0xad30333392c9f417, 0x6b2d7fba9f09604b, 0x9e3ddc5cbe2cbf08, 0x55c3d2f1464d096b, - 0x307cbb8246bf8dcd, 0x3730b0f1012eb0d5, 0x0c8393a4d48895a9, 0xb117032d9c0dbd50, - 0x3e7dc4b693dd38c9, 0x2852d037799efc60, 0x802884a5a4f90bc7, 0x0ff140f62396f353, - 0x76c938109c8916bb, 0x0866bdf44a64b752, 0x7afda533ecbad279, 0x6a4891e0cb73986e, - 0x27ee88fe6402e8d6, 0x8e09ad5d27313a71, 0x1387b8f864e2a0c3, 0xfff6cca7e3e6a376, - 0x7135ebe4f69c5ae3, 0x29b0156c4aa766a4, 0x2d3a01bab5ee2721, 0x10e90946003beae1, - 0x222f554e8299dccf, 0x9b8edaa2559591cf, 0x83c130b08b685cfc, 0x6ccd28cf1865ec9a, - 0x404b9658d953ae89, 0x2a5c33d1a1c23689, 0x53013a1504ebcc19, 0xa2c33e30eb913f3c, - 0xb8b25004e9250f7d, 0xc58b720152e893ae, 0x0697bca1562723ca, 0xf2acbc9931d28b99, - 0xbc4d734f9eadfdd3, 0x4309a532d1f69b88, 0xb2ea7225a0dd1d66, 0xfc3e6525acd73353, - 0xe63ecd7e8ab7f786, 0x9a02b18fbe3dfd55, 0x00009e77589a68f3, 0xb05757942afd3573, - 0xb7604baf88a47fe8, 0x203c3469ed5fb3db, 0x9e285f0d1415d4d0, 0x54e57b00875d662c, - 0x68df74370c2264c7, 0x2a88510cfab58a2a, 0x25b0428b18986c2a, 0xefc221b10e5bca0f, - 0xba3bb182af5671a0, 0xb95605adcb6d287e, 0xdd6b836c2f8159db, 0xfaf7a301afc80cff, - 0x00d957f240e7ad38, 0xf70524efec05c34c, 0x86369e947b9dc9b7, 0x31323c4ce232cb02, - 0xc2cf47c2dafe18ab, 0xf441fa74cdf35fb5, 0xe02032f4930e8935, 0x1ba3ff67f39590ea, - 0x684b7e869c2cd13d, 0xecbdfb9243e0cd6b, 0xa34dc40982f070a5, 0xdfcd4ee911b130a8, - 0x7b318b977c5da342, 0xfbc0cc23ea1ce086, 0x90823289dc113374, 0x5e9e417cdcf0cfa2, - 0x4da5c4737fab7524, 0x9c287bacfc92866f, 0x803886d6f883ea99, 0x07aba60c09a22b4b, - 0xdb9cc27dbf1ba1d8, 0x0579aa9a02ae92cb, 0x7a80e3e3727eb657, 0xeccba0498386d34a, - 0x6f6ee44287bbad01, 0x2c16efac48d23ad8, 0x1d89636be65cc278, 0xd8d311af06f78da6, - 0x6f1f62a014fe163f, 0x7536c21aa75e5122, 0x64e8c6ed73bf033a, 0x526371c1c07eec0f, - 0x7f3e19b04884c9c9, 0x3fbcc4a06f8f0009, 0xbdbd45be1939cde4, 0xac3a9c988a0f219b, - 0x95ed7eef5e4dad56, 0x0afb30ade0fa5713, 0x5cfa36408822abcd, 0x021d64e96ed444c8, - 0xffa99bac71612c8f, 0xf7c90e3b17b50c50, 0xa95c4897cf07440d, 0x0c9995adf85a584b, - 0x2cd77ba5e4a26dfd, 0x95560587993de26c, 0x661e8e35c7c049f1, 0x87bada3f76e09e5d, - 0xfeee24b8d6e97f36, 0xb1004deac829b857, 0xeb51027b4e72dcf6, 0x47860343d28b40fc, - 0x288408f94b075f3a, 0x8093f95fdf799172, 0xf1a6cc6f4b74bee4, 0x6ac2ce913693a5ea, - 0x85226f249026d3fb, 0x7677fb2d1451f3a5, 0x6fd32728fe4ae65d, 0x3187d5104949ed30, - 0x3fa34a1e953e5cae, 0xd2872367a58450b0, 0x9252b32f5e423646, 0x7bc65cf4e72d0b2d, - 0x5c92da9cc6f4fb49, 0xf417cd08687a6d74, 0x891a8116385721e3, 0x7367ff0ce20bad6f, - 0x8f81184748ec07a2, 0xe97916a44a9035f0, 0xabd7335b3f8db96b, 0xf435d70af3b6ec7e, - 0x7ef5a8abc69362c3, 0x8112c67d568db8d0, 0x9d8f9140a936e6ee, 0x01841533a6945cbf, - 0x0c88dd7ee278fbe4, 0x2675add415581159, 0x66a84a1a8598a0c0, 0xa655b21e7fb9bde9, - 0x1c6a4f5b346b4e03, 0x0d812b378e45123a, 0xb76dc272c7d4ffc8, 0xf2fa977bf6236c99, - 0xe53a4e10f1127e58, 0x3685fb8f6a99fdbb, 0xa6cdfb6f3eb88059, 0xeb0d9b9231464f11, - 0x58c1ad3d9099ce81, 0x68d7ddd698709e76, 0x988e555847be1260, 0xba9795332cbedbaa, - 0x7f7932085c3bb30c, 0x835b299205f22a7e, 0x29be38eb1a3a2008, 0x2f6b5276b7793d24, - 0x013f0b7aa3cf0b4c, 0x793f3a0bf3b8c1bf, 0xfc166342d101c56f, 0xc0067de1923a26d9, - 0x87b84789dac694ba, 0x5dba43616bd0c68a, 0x17c92e7b7c1372c4, 0xda5a18166c0d87dd, - 0x4d2fda4ffca3c1e4, 0x61e35569b3052d11, 0xcde9b16f4ac4b15a, 0x3b8eb91483c7abc6, - 0x3bae3b780fd988df, 0x25f386da6b64771a, 0xa270b9754a842d23, 0xfaf0660efba4af20, - 0xc52f6e9b0d930016, 0x420b2276433f1084, 0xeb5203326985af94, 0xab62277e98ea4331, - 0x5c1579336d815e8f, 0x11c3c74b023835d8, 0x4a169e6fe0fe362e, 0x9d26ddbcc2f915af, - 0xf6e858d89a72de23, 0xcbf04f7caf05be7e, 0xba6de0e0749a0377, 0x5244e8573b6d4135, - 0x9fc9d91f87abccdd, 0x18e9cededb1bde1a, 0xfcc448978e7beaa2, 0xfb1d8e002ba9b852, - 0xa04666ebadb8f65d, 0xb67843a6c5c9e9c6, 0x32d9887dd3028b92, 0x51381050ea0522d9, - 0xf6b755dd6dd15607, 0x4fac0b7a04780a83, 0x6bcecee5115e0c0a, 0x002d65301846c3e4, - 0x242389b56256aed2, 0x5493b33727337f7e, 0x6e4c246ccf9ae62e, 0x580f6c2fbd84f52c, - 0x78feab157b0c258f, 0x34df24b5eeab2b27, 0x88a87bf6216abf60, 0x9d75525b86165d14, - 0x65ad4eff45c09e06, 0x3e68b27026312dc0, 0xb04ecff45e11562b, 0x1aa3a3e0b27ee625, - 0xe5d705c27a1be709, 0x5802b259ed6e3d67, 0x040d594d0436264d, 0x70d6d5a40e1ea48c, - 0xd67d0b9f6a24da15, 0x8ba9036ff8348d09, 0xa25b0d5a27f37e79, 0xd1e33fc657dfe209, - 0x980ebf4cf7c83d9b, 0xe68de6bc5af00815, 0x2e74c43566954678, 0x7497764591a81a86, - 0x5578a453c7d7168c, 0xdaaf0a3b33e9195b, 0x28ef49c68282e47f, 0x4d2bcd04792ad340, - 0x5e7abbc03dfdc56e, 0xa5ef9a77cb92e794, 0x918c7e08cc44da38, 0x443bd4a7add01449, - 0xb1689cc0293e59b8, 0x376ae4eb7e63898c, 0x48b6439b5155247c, 0x9b6bd1bb5fad621f, - 0xbbca2b0db8ef483f, 0x60839b79b1fd8077, 0x53896df7b49fc2ba, 0xedbadd81ad675287, - 0x439c07002d93c773, 0xbc0b135fdae0071b, 0xd16c36ea859fa311, 0x9a2f59f18e5a336b, - 0xd7f022cd2acfab8b, 0x565c4ada530783cb, 0xff8fc630bef02782, 0x97e6f8513cc49eaf, - 0xfd34ef4ee526a2b1, 0xdaf614da8bf00162, 0x004c0fcafc0e7e5d, 0x44f15551a0edfb62, - 0x56bc0f68be7399ac, 0x1d4b074edf46748e, 0xb3660b2bf82cf330, 0xac1fbaecaf897e55, - 0x85bea8578d688ec1, 0xf8384402ee5ed763, 0xd3808926506f4282, 0x71a32c9f12d502ff, - 0xa824d97dfff89f18, 0x13ca3a0f002eb2ca, 0x5b59298466e0c7b8, 0x5979470d17f26aae, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x527bc06b69d3c6c3, 0x05498eba956c0576, 0x8af60ddc108e2cd7, 0x525cab2ffab7d9e4, - 0x876279cb60382bbc, 0x35f80465ca2bea54, 0x1cb75708b47262b8, 0x454d184a787480a8, - 0xf84cee287c5d5d24, 0x77b9ee8ab8465bff, 0x8b0705bdc161999a, 0x99cccc004bef1a29, - 0x76d9a4b18d584e51, 0x934e1591e4c767d4, 0x0d68e5ddaff0b1c9, 0xb95e68b04147c852, - 0x58ec6f70a9ec2425, 0x30540079823bc9a0, 0xe475cd6a3fca0cdc, 0xa37d28da4967f14f, - 0x7fce8638fecdff45, 0x98187280ee74ea16, 0x4f9d2fa1df01d9de, 0x6a542cea7cd3b94e, - 0xda3fd31eacb4c0be, 0x6c4897a56975e65b, 0xce21ba7b2f3782cd, 0x87feecc2b2fb1245, - 0x20edf5aa560d4a58, 0x0bedfb01609cde9e, 0x2ec53f5943829dc3, 0x01fa61eebd049076, - 0xaa0ecd8ea37992cf, 0xce2080d9d9d3d2cf, 0xd2d4662d9a071cda, 0xe1fdf7f5d37a481e, - 0xc9c2a730fe3a36b5, 0x30e52565fc9dabfd, 0x38571f188d4dd4f3, 0x66efc528e471f9a7, - 0x8dd25558c49bd972, 0x2312737577df3097, 0x614b3979db2b79ed, 0x6bad0b0b5cb516d1, - 0xe49bf01be371b050, 0x6d4164ba4d12f7a4, 0x7bdb13874255f3ef, 0x9a1d5a1ce1ebdee7, - 0x52a3b0638af27831, 0xce52b48c4468b91d, 0x4c89e8b6b417334b, 0x225d500b3d251ddb, - 0x5602211f8c27777b, 0x58329b879ccd2b34, 0xbb737d59a15cc10b, 0x1179d6efaf894ee1, - 0xb5eeb1f1d16038a4, 0x8b79ae4a94af87ac, 0xc0fe0599d51b7e3f, 0x2e90a083f45e5a2e, - 0xd3bedc90fcbc932d, 0x6fe5e7d09db47405, 0x737e67a05d097232, 0xe424d7cc9afd00a7, - 0x4b6f985d28921434, 0x94273884f6c42b59, 0x37929f69bf344e72, 0x4f8e4cae2ddbd892, - 0x1c13fc4ab1ba0452, 0x88e9564d129f32a2, 0xdea8e01400a0b7c3, 0x3b1d1ca457e316d8, - 0xf5b4363728eefb64, 0x35635f3c2b39dea6, 0x7c0397b5de6b77ad, 0x143b1c0bd134e41a, - 0x32f3f8cfc2aa48cf, 0x6af26650ef730f75, 0xf10dddfce6747e81, 0x415d62e6a698ef4b, - 0x4d2c202348a8a8a6, 0x0b9c9c3f78594116, 0x0db3ea52e0967922, 0x66289f1221fe3c2d, - 0x5f2ed12125c16a53, 0xb31ce59ce7678573, 0xf4f18e32934e41b4, 0x5a5d53e319a3ce76, - 0x9b8cc6650616d74d, 0x38e6432ce39488ec, 0xb19c9ef2ae396ddb, 0xb240e3a5bfa3d7b1, - 0x3c2e79d6d1af706d, 0xf3ddf33a537353a5, 0x2f70b6a05b16bbfa, 0xc3277c04cd96252c, - 0xac440931c14b63cc, 0x5a381e36c812809d, 0x44a8f07ce4a8e4af, 0x520a0030035dc4fb, - 0xc39989af8c6aaf5c, 0x618be06597fe1d11, 0xadfb79ff2b1c9e67, 0xd1efe42505a354f4, - 0x2d4693f22624687b, 0x3131b4d208b77967, 0x7dd88f28e16288a4, 0xad9a91a1ad60567d, - 0x4fc0d5898af0f4a8, 0xabfbbe028dab6f38, 0x1c7ebf59ad529a72, 0xa1103fd248ef6655, - 0x5232e43b1b8b48eb, 0x487c5caaba72aac7, 0x6df179800409a351, 0x9dc5b62ec041e68d, - 0x8e8cb63d9a3e6802, 0x2ee477de98105c0a, 0xfc9b63e302e6a88f, 0x77be699081abbc6a, - 0x311aa96418b534f0, 0x7139373ede15c937, 0xe77f3d855a40f67d, 0xffd549aec9539c04, - 0xb43f7f6627b3ec9c, 0xe1893de70ffff7c1, 0x9c68f5bba7a538fc, 0x37b323311467365a, - 0x446fc45b43101c8f, 0x2302cc8eb20d1fd4, 0xa4df959b36b02eeb, 0xd61c9c6005732146, - 0x3a7d5c663e11a288, 0x97d687ad2b43268f, 0xd8b72b2ea8ac0f9c, 0xb3a1bec5c3d64d51, - 0x2fe0437f832d326e, 0xb4f782627f8ffc0f, 0xc802630f134e8ff1, 0xa976247a20f9c3e8, - 0x8172015e2e9e84a1, 0xd05f43440d213683, 0xb7ae8bfd397ed087, 0xef617cbe5bb6d51a, - 0x86f1dc4a06b1258f, 0x3c16f88d60ced74d, 0x2ad2a19b2d60da1d, 0xbdbcc0afd0284409, - 0x8cc31202ba4c5635, 0xe921f17ab3a71f0f, 0xe00af9c780df137b, 0x1dbde3b846422082, - 0x22003ccfb7df4afa, 0x6fe17aed18a1eef5, 0x8b4d08c3a7b01bf7, 0x97441d7435875eec, - 0xb1c09ec91a0dbb15, 0xd1224b3004b77c71, 0x337ae492f4dcb3b6, 0x979f3c259b268cc0, - 0xe597eaeefdfb62c9, 0x5780d8b5f29fce50, 0x292c118b87872cd5, 0x49c3014d7c02ce01, - 0x6d2859332e630e36, 0x1ffa8fca890df4e2, 0xf5d942b3b521be9c, 0x41fed73ac018fd14, - 0x0c3874f45a860f01, 0xc1742055f2f655b9, 0xe787aa619dacfc1d, 0xbdffc41d11e879b0, - 0xfec8cbfdbd023c03, 0xcef0bb9260290a3b, 0x30066ff9090e8cd4, 0x41b8763b7d2665e3, - 0xe8d514e544250487, 0xef5f63c03f53e873, 0x2aa4d86daf3a82a5, 0x2a6b8ecfb6be9fb9, - 0x20f2aa6503d9c3ee, 0x26bb05bd46f6a75c, 0x70f09e251c036bcb, 0x1651d9ebdd99c00e, - 0x0a1efc3158005c9f, 0x2630cc5633dea383, 0x6b2e4fd9fa9900c9, 0x9a0f20e2b0d0ef6c, - 0xe0a375cb130c235b, 0x026f0324789b19bc, 0xb00adffdc53e4679, 0x309d682a20e3b4cc, - 0x9a708e3803f132b1, 0xea307336668667aa, 0xe585ecbf32853f75, 0x3820f32d1c51a74c, - 0x7b7c03ef41076761, 0x4437d4af98bdd255, 0x51ed16da4a4337e2, 0xac9cea2dedc3d3c6, - 0x2c32b0c90a84806b, 0x7a2b1a27732003d5, 0x627849d214aebc79, 0xd90ecaa8b13dd4bf, - 0xa3fd31d0eb60c816, 0x80ef80ebb334320f, 0x6964641ce74227d2, 0x44708ac91167da63, - 0xe5d108bbcc0be99b, 0x619d65a3a484a5c9, 0x63b9095fdbbc3e94, 0xa2dfd29403dabcb6, - 0x32cdc14d5889265d, 0xd01527102d9be1de, 0x6b01b03562a3ac7d, 0x3d9bcf388d72f405, - 0xb185073ad4ee70e9, 0x5303cf55725cc965, 0xd02bae7f388f29cd, 0xef6c95d804001c34, - 0xd08f063051125a7e, 0xbd16db472737ab4b, 0x9b4a2c87f779a94d, 0x3bcd4e2c70023b97, - 0x939d65f7a94ad0fa, 0xa3bf3a1ffcd9df8a, 0x53bc2cd7b0826a03, 0x24463ef66fc58c3b, - 0x0c061c57dc62192e, 0xd56d417f8fa3cb2c, 0x4c747df3f421bbcd, 0xd3fafbb0488feba6, - 0x862e61b3e28e9f16, 0x9d25259e46096eca, 0x0cb19435d177d2c3, 0x7d37ec1d69645729, - 0x34a79b7571da5923, 0xe0f2857ded0db07f, 0x757b6ff5f0906697, 0xa4ee31d02c7b44d0, - 0xde417bda5b474733, 0x7ad4a3959064991b, 0x7c649d87b5f11656, 0x9d31b799f11c6bc4, - 0x9ea74b7436a56472, 0x9d4a9afbeb310aef, 0x79a11186ca6828fb, 0x9b9eac9af245a8ea, - 0x06f3bc8df2bb1c0a, 0x9da08420a9cab483, 0x203c4fe107bd2a26, 0x99bb32564db646cd, - 0x3881af645f9f6cce, 0xc59683c28446933a, 0x099b304810100d8e, 0x48f970c9ec4111d3, - 0x71902bdef39d81ae, 0xe96f41d09f1da163, 0xbdca152b8a6092dc, 0x98c30847f6fe6f8d, - 0xc5d858548e798a87, 0x2f330d523af3e613, 0x729a2b4a351ab44d, 0x4dc5b7810c1a167d, - 0x3416bf916f8996ed, 0x70c3ddb3af0a1040, 0xda9d0e77c3ffd501, 0xb13663b5ac9be58f, - 0xedbaa88f41805287, 0x5dbcdd2c26e202a8, 0x88d3c69477d3ced3, 0xf94c7a8429467651, - 0xdb0f9095b0ef2f4e, 0x33c25f9ac74fce4f, 0x8789b42cd793113c, 0x0615cee2a18a66ec, - 0xacd19f05ce522ddc, 0x8d193217e004bf70, 0xde304b40ed93a7b6, 0x9950230705069326, - 0xa39ac2bdecf5a35b, 0x5c584231369edaf7, 0xf14caef9ead95f57, 0x7a1fe266d14ff9e5, - 0x2ffa4aa05dd20078, 0xfc1640c5f8ad649f, 0xc2c842b7ca977417, 0xb8f6f8bed57d1787, - 0xd7951b44dbdfa1e1, 0xe7d5ee18b668c5bd, 0xc508b80eb87843d1, 0x0ada489d87cef66e, - 0xfbc5d7fdf5b65dd2, 0x4cec150191658ece, 0xa4ed2415f11ca6b6, 0xcef97a762fb315b8, - 0x5e5d86fcc6530c63, 0xcd06aba91c549d8b, 0xb1291a37dd276637, 0x2399703400bafb43, - 0x0682672a99c5dc78, 0x5856670349296bd1, 0x175d7f91e968ca54, 0xa501df19ce173123, - 0xf30d6ed491cf98b3, 0x0634a8607a78df7f, 0x4a52aa5d0a90ce55, 0x8a7b1ea23bad88a6, - 0xb9df4fe2a8b0eb33, 0x2a3b088dc66146b7, 0x782355254bb4cd81, 0x0f7ab7ba0b6bfa0d, - 0x01a20ece85199c31, 0xd66a138b603bdd8c, 0x6f2c291883dd1141, 0x7308e11eb374ee45, - 0x4e5c546f815a23de, 0xc27ab6f752764394, 0x823ba591cdf6e231, 0x4f43e0c2373040ec, - 0x3fdc79fa1d0d34fd, 0x0939aaf3112b4ce2, 0x3ee2baa29c0aba26, 0x67dde73e9a732bd9, - 0x3d45d85d529d9376, 0xfab8626835b069b2, 0x7356b6a4fda352e3, 0x0a3ec98792397666, - 0x54eb17aaf4d92424, 0xc8f1bb8291fff65d, 0x057baa1bfb3c80bb, 0x0ec74efd9948e6a6, - 0xe139d7e63bc3a6cd, 0x01d34c5372f3990b, 0x8998d23238cb0964, 0x1eac71a1a4cef3f3, - 0xfa512da72508ff54, 0x5ad4ec1e22378f8b, 0xa88356fe6c2b667d, 0x783c683b7f9a4593, - 0xc6adbe981c29c9f4, 0xa62b8e0264006eb3, 0x21a302897c333094, 0xd9e68cf252a89173, - 0xd413a38eb3333de7, 0xaf2665b565a8efe7, 0xc7edcc1879a55733, 0x8efcc1a446668e9f, - 0x9a505f0eab1ac263, 0xb158989a9b5cd50d, 0xe531fe0d9c6f3093, 0xf2ce8f9cbee3b7a1, - 0xd0aaa22862a8592e, 0xe2bb580ba6053c7e, 0xa065b762acec45c3, 0x15a64f28a583b910, - 0xa60c5ab64dc9cc33, 0xc7be61a23432778c, 0x3b93f9b996c66db1, 0x637a384392bf619f, - 0xaed5d351a2470bbd, 0xd4822cfaa39fb5ec, 0xe03f6dacb4e71524, 0x7291f8774c2c0771, - 0x28be310786b27725, 0x687de67c79ba1c34, 0x2e2d7a155a4ab615, 0xb4ff6ea11dc22e4c, - 0xcfa326bd4e1fba66, 0xaab46c8994735d1f, 0x74c80df08305ae81, 0xe0b65f217cf29acd, - 0x887bfec93ba5402f, 0xac7717726e3f643e, 0x1fb48d9513758835, 0x277d80216264480d, - 0x568b4d85bb69c481, 0x8d333b15ae871fbf, 0x29a005373d6d6cbe, 0x4237fc6ab33de076, - 0x9d85920a19b30958, 0x1947a72ebb36b9dd, 0xa43c432ccdaa7a02, 0xa1d857534f215d41, - 0x4281a176456ba54d, 0x28771e2f0619e83a, 0xa1b3ccacae92c6ae, 0x991750a9ac1db45f, - 0xc18e1b1aa427ff52, 0x1a971ee527f13956, 0x897af3bdc37d52d7, 0x1daef12cd556ab4d, - 0x4649ceda24efde75, 0x979285dbf715b358, 0x9731c1fc6d83a666, 0x056f3b07f3d3e02f, - 0x5d5e0273972a3f01, 0x0d44493db5ab35e7, 0x5b652d9fdc5c8d53, 0x9132c09ebfa009c7, - 0xe8cd596505e4c886, 0x5bfdfb654539873e, 0x710eb30619195b01, 0x5c1c8c1d318208ee, - 0x46be5759fa2611cb, 0x8a2ff5105f01334e, 0xab52c31aac3714e7, 0x63e8640b013d0b57, - 0xc4286c611347a206, 0xba757ce138460497, 0x16c7f4e4106a1bbf, 0xc7944db997763be8, - 0x0f82326a0e08580c, 0x4bb0daa0a7928719, 0x5372c25318ac2da3, 0x3c499e8db4091448, - 0x964ba0e347cdf444, 0x9f9861f80307b682, 0x7e3d70eb721f9f28, 0xc09af98c860ac6cd, - 0x59ee754953f413a9, 0x8f873d6d20f3ee0f, 0x748256c0b8ef7e2f, 0x5cd3c8a23d365f2d, - 0x64168676872d4589, 0xc3f0b1504ac8f909, 0x1fab77143ef9297d, 0x27ce95e651e80c40, - 0x6bf0a4a149beee8e, 0xe5f4318173181810, 0xf9e1109af278b5d6, 0x20350cc1c70e3f40, - 0x62814c97d7b67216, 0xe98a02d9b5573677, 0x775053e204087f2e, 0x26c1ab69939b6abd, - 0x502904a8938fc3ae, 0x3418acfcab8d005a, 0x68edff9ba48c045c, 0xfab82881219a0a72, - 0xd9d743bdafc699a8, 0xda25e6e5ebb0a0a9, 0x3efa707ba9fb191d, 0x6dcf9201194a26c7, - 0x2500d29cb2567e28, 0xbe93dc5dce629d19, 0xa5d8fc73ff70e757, 0x6da16566c731fbb6, - 0x03054492fa748acb, 0xa3f691b5e6ebb6ab, 0xe70e9e259ab09c3e, 0x1c9af3d7a721f52e, - 0xc5ee7aaa7a72b6d7, 0x9a76746e0eaab44b, 0x61754d335583a8d5, 0x1b6602136624b5e2, - 0x0d5ff67d2fca3d30, 0x44d121ee1d0f59dd, 0x6f520d90d0a2b85e, 0x90a6f6ba497adc51, - 0xb5fb5999391b3633, 0x9782a37e84cda07e, 0x94332745fa6758e1, 0x6785cdf3f86f2435, - 0x585873707c2adfc3, 0x296aa37e739afdd6, 0x7a252c526b67611e, 0xcd1d3495be0d19ec, - 0x22e02c0dd9f7b691, 0x68f1f0a5c2d3ee29, 0xf8c429b337ceb8be, 0x4183459867716757, - 0xe05ab2f5df16e74e, 0x7baaddfa1548d180, 0xfb783d7474d89a33, 0x2cfa83c5a876f786, - 0xb6873b3e0adb43f3, 0x80f2e14096f518a0, 0x1e62c8ada3ed7c29, 0x9afa34ac874b131f, - 0x2a5379cf2de87ef5, 0xd9d7ce33e21f6e62, 0x9677b81462106d7a, 0xbc95b85c79d4a7d0, - 0x0f14cc12685d6ce1, 0xc917d85c1074389b, 0x66c1290a1ce76d03, 0xa4befc1343b5db1f, - 0xfa9ceffab2c18101, 0xdbcc1250b6c50ac2, 0x224e9bd15162e30d, 0x5dea04407e7d8316, - 0x82d7881d81c4c5b9, 0x16cdbe057b87fdf8, 0x0cf6f0b979e33697, 0xa3c2fc254738ebe7, - 0x12bcd78e112f7abc, 0x352ac1ff2c7a7123, 0x4f428cd0af990470, 0xe793219287307feb, - 0x77acadc62d5625d0, 0x5679cd42e147b893, 0xdf64902d5c436273, 0xe6e2df1fb3f0b493, - 0xc23ec5a0a58c8a50, 0x71a7737077d1f4ad, 0xa3984db89049a293, 0x6cfc4569a97b33d1, - 0x0f7773cbec0078c0, 0x223a7415eb8c1b89, 0x86d916b56ea44ef6, 0x34177c8a67558ca1, - 0xe8d52d3e5b4800ec, 0xd25202d17c0c45f3, 0xd66b923b0041372d, 0x31c23e908ba0fb86, - 0xf25b3029c6d0c0c8, 0xdcd5dd8e6fb8270f, 0x745fe13f3e1842b8, 0xbde17e0db3bafe0c, - 0xdfa217c57eb8444e, 0x71acee25fec0bd8b, 0x2fa5d687fdbe62dc, 0xf7d485376ef9981a, - 0x3f890b66aeeb2227, 0xaee43f31a783c84f, 0xe27cadee8fd181f3, 0xb0b6ce9a4bfcde8f, - 0xc99b2afdb62b106b, 0x081286cbb9226043, 0xcd8fa3fce78027bd, 0xa502b7ed60ea56aa, - 0xafb10aa5949ccdd8, 0xe5345259edfb1ccc, 0x0cbf812d9a49e31b, 0xe1c8740757a23930, - 0x67eee9e82b5605e9, 0x858f5b24945a88bd, 0x22915e49fe42c35a, 0x776ed5d365290cbd, - 0xe2f5b20a25c5b7b8, 0xed5cbcc0e4f7c8b7, 0xf46acf4dace016f9, 0x4dedbbe548f230dd, - 0x71026831c9ca9f95, 0x3a4043da86b54849, 0x64970176b1397c16, 0xbef5a640ec11823a, - 0x84554c053ffb7d8b, 0xdfe4fc206a0146d8, 0xa2ac8e5ea0d196c0, 0x999c853beb7db32f, - 0x5bc34c93ba98ddb9, 0x633888fa2686d4be, 0x99a275407547d6bd, 0xc211e8ce31ff5777, - 0x1f3110d37470afd2, 0xb1d45087e444f060, 0x4480ee2ddef2efb2, 0xe51c319cba5e1993, - 0x9d114110951c462d, 0x63936cd37c3c3b50, 0xc3ecc17873090db4, 0xf024f6771a17d9fc, - 0x105b19bebdd07017, 0x4fb78ec3c1e9222d, 0x11431281659ffe9f, 0x5e3bcb1615f4f364, - 0xc8f42915bb6b8d8d, 0xf5a411fd12c1e14c, 0xc47d9b6815aeceaf, 0xb19f4aa7d1d4d66f, - 0xeebd588fa66843b0, 0xf7c733fc8ec9b0c0, 0x1934a8661c55f1a9, 0x3276587f1448ea7c, - 0x228177458445cf0e, 0xa32c25de9a4ea2f1, 0x3bfe9c90c3115084, 0x3d78811bb5be1e15, - 0x2073a64f87bb4565, 0xc0a18a91126bf02a, 0x1db1d41b531cea01, 0xe2d389d643e173e3, - 0x9821cb570db7c548, 0xe3b0325dc849a73a, 0x1c2cd6e05e22bfc2, 0xa94a97ae1c0668e4, - 0x4c86f87d1e1568a6, 0xe25a2b104439d8c3, 0x394905bccc5d7940, 0x30b38f18c0cb980a, - 0xff7470608859a262, 0xff435b62eb2f0138, 0x57ff9b8b524e20a1, 0x2bcf4dceccebe38b, - 0xcc048ce8441d87f4, 0x3bd9751020a2e8be, 0xbf8f55662bae10db, 0x3d36646d0bd072fb, - 0xfda298673ac05f6b, 0xfa37e9f5351a0d0b, 0x3a6a0e3a9ee687ea, 0x877f92c95aa3079b, - 0xc81dd4ed48f9a625, 0x741eed7861aef93b, 0x2dcdb8a805578a59, 0x458d27f03b58adc1, - 0xd635a7ca054a251e, 0xbdc1674249144311, 0x2fe3f014f0f6f6e8, 0x05bc37174ad163fc, - 0x95db2dd0081b7445, 0x35fcd01628839f9b, 0x4f2a2d1b581bc1fb, 0xc2251baf1a9e5f58, - 0x3e04d8d25b5266cb, 0x41138154d072046e, 0x7fe056b5c1bc9513, 0x7ec5a07020b22bce, - 0x599c13ed2a7198da, 0x038e68ce14fc8bf8, 0xd50d729fd0e00631, 0xd122bab3bcdfbd18, - 0x5af5c2d77db32720, 0xe5d7f81391dc86fb, 0x0d60adc5f879032f, 0xa594d6aecbaf9736, - 0x1bf486ac72542727, 0x9457e2c474c0695a, 0xb6427a0201c5c638, 0x8fb3c636bfa5b923, - 0x273b12889b0f39e5, 0x7d7bfcdeafa25cf5, 0x4b36ec8a73006ecb, 0x6132cf6c703d7580, - 0x19c5777695518a22, 0xffee3e21e6b4a0ad, 0x6a7870c1b0f0b8cc, 0x1e6ba492e63d0670, - 0xf46ef85d587fe708, 0x15fe18b56e4c6fa8, 0x2ed5bfba95894de9, 0x6af9cb3574c6bab0, - 0x43bd7c15fe15b065, 0xec2223819a1cfc13, 0x79c2f96447c8bbdd, 0x7931a49be0dd0bcc, - 0xa3a0c7ca0e5ccc0c, 0xab2b105ad13a8391, 0x565244297abe4c4e, 0xcbd09ff9b3cd3d2f, - 0xa750f1c48256041d, 0xffd6f2633bdc1448, 0xa10efbda94fa8afa, 0xa8aad983e48ae447, - 0x0d82b6442d0e1548, 0x2ece640b636c167a, 0x733d867aa721cbdc, 0xe003cf5e3a8798e0, - 0xd30896d8fc429f87, 0x39309fbf77bfb1c4, 0x26751cf478e45667, 0x66dc87a964e236f2, - 0xcbebca317c508f8d, 0x18e901268fe63a44, 0x99b51bbcff6315b6, 0x5c2517277008524e, - 0xa95a2dee52129a65, 0x69219caa898b34b1, 0x5a6dc81cd41157f2, 0xbf939234128ee31f, - 0xd06cf207b05ab7f3, 0xf4f60d74579613c8, 0x203182a119a9e526, 0x280c29473ec2da51, - 0xe869aedda9b6a5ea, 0x25ecd3ff423da68a, 0x6eb3718e8ab5bb82, 0x363bca98469f0068, - 0x9243fb703f70aa1c, 0xee7f87bf0c757a1a, 0x1ae471a5a5cb4379, 0x3efa6b1039bea9f6, - 0x6168ec831ad60b0e, 0xbe85d9203cf84b20, 0x4d6ea3677b78f4fb, 0x3906dce6b289b401, - 0x53842b626e854a5b, 0x69d93f4c6ad0e5dd, 0x071265768d63cecf, 0xa8390d8f06e355a1, - 0x2910bf70117fb7bc, 0xe76adf6121406162, 0x77c4b695bc1b3065, 0x8fc4e9bfb1f51822, - 0x3d141c6eaa68dbe6, 0x115213ec59ea2196, 0xf2fa9e26a52c9ca8, 0xf600d0a51df92483, - 0x0d395a559281c13a, 0xb45d9fd230857d1b, 0x14405a8b22d97873, 0x7cb5eac97bfbdf1a, - 0xad5da91dc4e32dc5, 0x6e214e456bee2bf7, 0xbe0b243dcc577c11, 0x059d031812907dd4, - 0x5d599b864e91c843, 0x99393c1d4eadfc4f, 0x293852f2a107ccc6, 0xbebd6bf03e8e99e8, - 0x222a5b46c6381218, 0xbfb9fb27d91b0178, 0xf810f449d77a6c97, 0x6594a2951257571c, - 0x5f86c179e62516b4, 0x8e48a62f34ca380b, 0x149077d9e47c6b07, 0xf9ef284bf1c657e2, - 0x3ccf8e1352839ce5, 0x9f0d5a1df00701b1, 0x6f26e54fe5824faf, 0x792da0de9275f232, - 0x3d32b968b445c3a3, 0xa927e99372923261, 0x0daad8741d4d6c0e, 0x5f8d00e5901ae972, - 0x492a438492fb034e, 0xc43c3fc1c583dd06, 0x56ade41c5fcdd732, 0x14cb33efba06863e, - 0x86c7ee1ad9db42f8, 0xeabbb93f5d15060a, 0xc5d1d11d33bec8bc, 0x671e6aa0a3a17b93, - 0x0db30e06053d6445, 0x091246a470107037, 0x66b310406988516a, 0x581f5b9a9418315a, - 0xcbb24ea3808f1863, 0xafe75a4a83b6d617, 0xc3c132b2b0b4ed25, 0x731247d53e5f57ce, - 0x228ee29f5bc51be7, 0x66e77aee4a2410a7, 0x9254c1b4f9a3f522, 0x6d417b09c5eed63c, - 0x3cf9277e25ab8ff4, 0x66a75ed5cc5fedc9, 0x292d395977873bea, 0xc74073eacdbfc346, - 0x3acef694411969e0, 0x2f55288792760230, 0x088e0f6459578f05, 0xe97720072ccd9a62, - 0x54d0b5264b9287d6, 0x993b4d9d5aafc437, 0xe0006e04113b3ee7, 0xcf50ccac329a48f6, - 0xac17389d6784a05a, 0xfe5a896d9696aa85, 0x2dca2a0a5089a151, 0xe28d29badbe727ae, - 0x556a462bb6d28d31, 0xd6fabed32e06edbf, 0x6e224f74f9eea43e, 0x52fa24c251f07fda, - 0x8c20bc0fbfc9b805, 0x6be9ac5728f4f9fd, 0x996a6bcd227701fc, 0xb10eb6eff56e04e5, - 0x482db2853cf6ffcc, 0x02dd5e5c1d5d7186, 0x4efd89f842627b97, 0xd2c8e4a1c8482724, - 0x8656bad25748fc43, 0x6e4cf755efb7633a, 0x801afad3b760bfb9, 0x068352987e9c10a5, - 0xad51fcc88e9db195, 0xcb4eb33e1f780e03, 0x3a64518af6488a6a, 0xe5a705b274325fd4, - 0xa9b12f8d2f2c7280, 0x5283d148a71a8dc6, 0xc747b8bf3154c53e, 0xeca849400d4c1bdf, - 0x328601757d57687b, 0xac4abdfc00693aff, 0x0209408a2c500dc8, 0xfa41265c53d8173e, - 0x9fb75e4a22e38677, 0xd3a559b66e7d61ec, 0xf8dc074c7e0e08b7, 0x200350b52f2bd6ef, - 0xfaa9877213a52229, 0xce77a2d78791fca8, 0xc78e9f6daa775bfc, 0xa08ce0a4eb935900, - 0x38d5d9b58698c909, 0xad7710b070bfd52c, 0x5c2dae5ab048acba, 0x91391b7704630003, - 0x9536712574789929, 0x1bef759fa113c332, 0xac7c2445df4d7f2a, 0xffa878d2fcfc9a5e, - 0xc8d7a709c11ab5f4, 0x8df7ef2a6db2b6fc, 0xea7e95becbe17eb6, 0x55f72f7799e23115, - 0x55d151e34a68c790, 0x2551ddc3f4444254, 0x7338172f7bdf3c65, 0x95c4a77798bcf593, - 0x5cf97bfbc3d6f2c7, 0x1f0cd1cd12915de3, 0xd516638f8cc27db2, 0x09d5b7be24b9ccc4, - 0xc5d8b280d979e177, 0x70dd07ecb190f616, 0xe76ba3c522ef4482, 0x357aa4fd6916cacb, - 0x53050fa6a32ee871, 0xf172ce8332dd0204, 0xd733a292e775fc21, 0x4b51c958dab75658, - 0x8996c7ae5e7352e8, 0x2db81c567e16f3e0, 0x8d7c5a26cac22c9a, 0xb98cb515ae16d486, - 0x4d34f494ab8ea6ad, 0x937c83862a023362, 0x7414fcf8574d7254, 0x4dd2182c280f90c5, - 0xc387952524a804ef, 0x76104ef75ab620d0, 0x988319647e7f084c, 0xe51fff124fa86f99, - 0x6ad763d4e42e75c5, 0x6e23f19379eb6040, 0x60b75a472d3e11d7, 0x18596587b023b235, - 0x1fc4d53128da27f2, 0x4b11b1f76fe97a2d, 0x6f257c9dbf970c2f, 0xaea2f5dcfbf2367b, - 0x1f3053998380d862, 0xd63aac0f03f2a1e9, 0x59a7d72da32cf025, 0x554c55ccc1a24c84, - 0x4e727eaa44ba7d9f, 0xcd758345521536a5, 0x69d1aced2477b337, 0x146938c83d6c2396, - 0xe8bfcefb5d732d25, 0x34fffecaf92eb30b, 0x8220f7d787d1b84a, 0xab38b08d772a57c0, - 0x40a859994765c098, 0x8f3f26ba662f79f4, 0x0037dcc078b2c32d, 0xd0b5a5e8f03668b4, - 0xcd6aa35798e5cce3, 0xdaa33745466b1636, 0x01a9aae54257039e, 0xbb72e38b8aab0f9e, - 0x3cdf21b0e37e66dc, 0xb965b6ce6f84cf7e, 0x28ae8f59d520c25b, 0xaab81d89bdb16d6c, - 0x6e96d6a1b5a96284, 0xee9ca4e9ed1fa05c, 0x8d4e4c2b631278e4, 0x203325fdb69b4e87, - 0x4c53296525ab130d, 0x9b4a1d24d55fd789, 0x755ce32b7c9aed6e, 0xc1a72061245c0f48, - 0xb7948605da7adbb9, 0x648963256efe7a0d, 0xa574c49e8da06f6a, 0x25b5c626c4d5da5e, - 0x606b165e226c65f3, 0x626112748099675b, 0x22a5d4315eb038a2, 0xb8ed494ab96dc793, - 0xc4799090539c7f43, 0xd93f13390b787819, 0xc547c1d7977662b6, 0xd603ff58f9b8db55, - 0x1cfacba4b2f3c831, 0xbd2669f54da6c106, 0xcb3cb9411efc75ea, 0x1e7f2ec7eebc7e1f, - 0x67b02deebbf183f0, 0x86579965f743b3c6, 0x8e2259a28d5228b5, 0xccbc6a98ab906574, - 0x9521b7eec6184d70, 0x4113c66bc8ab0ccf, 0x8d66ab3d13bbe880, 0xc3b7c0c8f36446fd, - 0x448a0444cb2585a4, 0x1e2b3c2e4c75d06d, 0x2a15f2c73c09e402, 0x8b0b1945162be9b5, - 0x5ff11045e3355859, 0x19114e5948c21560, 0xeada146fbadb091a, 0x6ce3c6e16c5fe265, - 0xf3827c766e951bc5, 0x1d1091738d6a55e0, 0x97f1ea6a17bec836, 0x4cd9420c9229c3ea, - 0xb22170e67978b0ca, 0x5be0c0088b8ca810, 0x0f15c9e4b6685984, 0x74cb033982ae2667, - 0x5c6c5a2d85065882, 0x7b109bb09655debf, 0x56bacc5bc5db2767, 0xef8babab6935664c, - 0x1eb15c0bee6c8b37, 0x258b0c1bf2b5bbb4, 0xd95ccad6fa92715d, 0x9577ee75b742cee6, - 0x81e2e6c555e2fbf5, 0x951928cf0d18f816, 0x6686c6e16aadf2f6, 0x7270cd96ddc8d3b9, - 0x47a17f2a6a722bb8, 0x668c0f727f83413c, 0x028cb35e0f176eb0, 0x7fbcfbb4b4d5e41e, - 0xb5d7a1255a097cc2, 0x37d956f36b107390, 0x87217dd542a6dc8d, 0x2d41a33b7127cfa7, - 0x55b72ce72ffa38da, 0x337dcaac4e15c6dd, 0x49bc3d7cba07b97e, 0x88005d9da7656738, - 0x947342869d8d029b, 0x91cbc8367e34f809, 0x55b8db39221e2249, 0x2bb38401062a3235, - 0xb902b2f07aa34f7e, 0xd8b72b22d939af67, 0x2a45efb3ce1f4904, 0x0e96f91aa6a83508, - 0xc17d7fbb5d3f7e75, 0x24c926a9673b6433, 0x7f79fce7c4bfbd36, 0x61c40b11b705af07, - 0x6c938680b55032ad, 0x2592aa14ecce2357, 0x8d1328c0ad3d6c40, 0xbde590cec54c5bed, - 0xa53436b02b65a3b3, 0x984341eba2b81eb9, 0x76330da141c01f06, 0xf5da3bf1e7a5d161, - 0xbec5455b891f80ad, 0x7f1db5bb91c87745, 0x88cd3e7ab50f7d0e, 0x052b1991b79493ad, - 0xb9af59f2da249721, 0x97dd819304dd04c9, 0x51731eae5f755b36, 0xcad7db0033ea4a8e, - 0xd43d57f2ff724f21, 0x2ef70267d48f4c99, 0x80061ea7d7e473ca, 0xbc828327fc5c8f43, - 0xc6dc39660ef1610b, 0x2f06ed5f218ce49d, 0x855ca0d286723c56, 0x1fc15a495254af05, - 0x408d0964e363a528, 0xc9dfcdd5e57692e4, 0xf39897882de50c1c, 0xb9d27aca194ba134, - 0x5e7c667e7890cd9a, 0xf3efa39b825dc8d9, 0x3982e744260a5bfa, 0xea9a954644a2c3a2, - 0x79c9828626cbb23b, 0x89afbe946ec766f5, 0xf3c829aaeaba4732, 0x7bc93a3f35dcd86e, - 0x3a6b34a97cebc1aa, 0xba1f3c317bd99cbb, 0xcd6afba53d872970, 0x994f1023afb18744, - 0x24aa8402756b8ef4, 0x15537dd5dfd090e1, 0xd715a49071f00b2f, 0xdb83972a7c93542f, - 0xe36a9ef7516f82d4, 0x6296323fad0ec190, 0xcc41210f982f19be, 0x878e1df888833969, - 0xb8740bdc558f456b, 0x378da7088ce5e194, 0xaba48f7959e0d795, 0x14e27c831d3c00f6, - 0x9a723a12eb96e7a7, 0xf162f30e5a4eafc0, 0x0420a802f463f507, 0x395b4998de937a15, - 0x915ab99eaa1d64ba, 0x8e51d6cdc5f08500, 0x66fac167407acd4a, 0xd446f72d250cf47c, - 0xf8e996c937040556, 0x965ab458322fdcb2, 0x4a21d0a2b9411d1d, 0x20dc3a017e6b3e61, - 0x143a57f20804d010, 0x0d18f09a537888a1, 0x085914296f7ceef0, 0x2bd03eec6509d5f1, - 0xf7e46db26c867fad, 0xba1bb8847942a78f, 0x8ecd8e94d93474c8, 0x4fc2a6483f3c69a4, - 0xfacf50ae1b9cbe8d, 0x50e7adb3c10ce9c9, 0x301342529c99245f, 0xd7fccd9353dddfab, - 0xbf8ec2995d1782ba, 0xf50851db7123d0cc, 0x01cedc0aef339df5, 0xf449ecb0bbd52e30, - 0x2b64ad7c106c1dc5, 0x5b8f9311b3c37f99, 0x5dfc3b21d5cc618d, 0xf8bba7582be99cec, - 0xfea5b1d70088a207, 0xbf4387daffdda71f, 0xbe642814a59c852d, 0xa800adcc819b4f63, - 0x50c85746c7972d31, 0xcf62ea5252aa76f6, 0x893752345030149e, 0x4004881dcde9db6b, - 0x17bd165759cd9722, 0x72f5b515d47d4854, 0xfc42f097c47cad49, 0x7c10782c3481ef1d, - 0xfb8a78bbd857ed49, 0x6f376d8eec792bf8, 0xb4c1ac43e5b7c22d, 0x74d79b2e4a21f15c, - 0xb3c3e8d0a3f371fc, 0x19c70d69cddb3cb7, 0x6180731a9612db16, 0x2df588dab10fc8ca, - 0xae6532cd45781c2e, 0xd7d9a97e37075155, 0x43537b879c9882e2, 0x3f553455fcae8e4e, - 0x6451b39af00fd9ad, 0x7c6459da8da3152b, 0x85bc5561ed572d02, 0x3d643ebfe8e98079, - 0x66a7ff664ae39ef7, 0x7827c861a062334a, 0xbf6ac96466c0e1e8, 0x9333ca5f77cfbaf9, - 0xa6a8a0f7ddbfc9d6, 0x15651954bb81f270, 0x47147d111e836534, 0x8beacc5102a8636a, - 0xa4e8cb060f2ae617, 0x4237cf2711441880, 0xe0022904e9057f4b, 0x63ddb809a138b8ff, - 0xa2da3cfb4619c2c2, 0xd81dd11b63a7aa8e, 0xe5540f596069dfdd, 0xe8c65ed2bd24ea43, - 0x4ca8233f2368ac44, 0x52df9e9f4beed4d8, 0x1b608e948d7c75fc, 0xc99967ae4af26f8a, - 0x20f0a795b48eeb13, 0x74b4f7593b3e3a1b, 0xeb5315ab19b99e9a, 0xdad283cea201f472, - 0xad9102d4dd92d388, 0x50837cea9b218186, 0xc92dd8107da087e2, 0xf2afb2cfbcd2f884, - 0xb0eb202c890e2ea7, 0x68a8cfc2567c0c4a, 0xed78716c901616a1, 0x749353579283fc04, - 0x01eeae3c5730eb52, 0x16946d4d52ddc894, 0x32bca2dd63be576a, 0xfad6c15e6733d3a3, - 0x21386bd7228ee954, 0x0a577a44c8bc48c6, 0xb0aea714f6825855, 0x3dfd69500ebf69b9, - 0x93a4d7ddff220417, 0x6c394468d5c8df40, 0xb8c18e05d3a98655, 0x2886d5b932afdcb0, - 0x6dbbb721582bf13d, 0x99f3666a10acc9ee, 0xd02f80f5954d97d3, 0x00bef487eaa9aa1b, - 0x0b9d42dea35c8e77, 0x7e29d2d65fd7a78b, 0xddbfeaa5b7d3b144, 0x8da38cf3ed0164e3, - 0x530f5569d641f597, 0x8ff59ac675ac19fb, 0x9468047a51b3c4e5, 0x05ec9ae8bd1c0971, - 0xa8a4ebbb7c57eb9b, 0x922ab899c9e92da3, 0x3bc812008c6e7f09, 0xaefce01fa66062ef, - 0xc6da99e1b7f68c7e, 0xea8f0657b7158777, 0xe3b9ff40f58fc677, 0x6f7eb115bb419fb0, - 0x8acbe7780818721c, 0x9589e01afbcd168d, 0xaeb54fc322878191, 0xfc22adf19237d5c3, - 0x5de940bc2dfb41ea, 0x37308ec12997a976, 0x26dc7ab698781c97, 0xff1e1e39167c4c79, - 0xaa13fe6ceb679607, 0x33ffdd57fff3d635, 0xc5c065b004e3e81b, 0x90076e37cfbb3b27, - 0x7bb5d81f2aeb1ece, 0xf8c80e617de6d08c, 0x4df7e54424ab5191, 0xa1c818c14756c25c, - 0xe16943c1ead721cf, 0xb1e15bcda65aa4e7, 0x5223d58f8d9286c2, 0xc8824a19895b029c, - 0x924aa58f1122dcd8, 0xdc66f927110fa933, 0x2da5fbba7ea356fe, 0x5b5d6d52aec9e5d9, - 0xbafd2dac6df100df, 0x9ffa8e11519ac74a, 0x7a8fcd20fd86090e, 0x014e3c549a8bd7da, - 0xf936d1140a10ed26, 0xd25cbe0ade57fb93, 0xd01c7f8e9d52047c, 0xdb72e278bc88c9eb, - 0x6de3b137d58228a3, 0x8932f53a1a18063c, 0x174a67defe216e60, 0xbd29536ee3cc9b16, - 0xae5ab163f66151d5, 0xe6e44d1aad038562, 0x2037e479e9acb15b, 0xcdf3c5ef6a714e19, - 0xa73e5d5f650ad914, 0xe0f6af922aa502f4, 0x0a6dfbd7222bee59, 0x5505ea6fdb4105f8, - 0x476c5c4875358769, 0x35ad6f007244ab4b, 0xf0b58cbaa43984a3, 0x4b43f4118ae22902, - 0x889f1f2554440066, 0x5f40b4173e0db3c3, 0x99b239b7253ab36f, 0x5ee1c0b2bfe1168a, - 0x04e4ccb360e270ef, 0x6e154346cd1e1d95, 0x71189b05bfcf95cc, 0xb82cb59ec38400ff, - 0xe80de52b0da51cbb, 0x2fb8274a266307d7, 0x221c4c5f77531d89, 0xbdb1bd58a5a5ffc0, - 0xfd1d2fdb9d695dc2, 0x378764a220131abd, 0x8180ed4fddd87ad7, 0xfd225366cc3f6418, - 0xbbe49008706e0fc6, 0x8f33da0ff8eaf6fc, 0xce22d2bc75082c2b, 0x86531526c62eb6f5, - 0xcbcc76bf847eb1a2, 0xeee60924fdbce5d5, 0x65fc3e82a509e88b, 0xf4979eab3658bf42, - 0x2af994c859b7fd99, 0xdbb892dbf9373baa, 0x57979a42491cf1a3, 0xb48aa89de6651963, - 0xf11c0f9fa424238e, 0x97f8e205ef06c082, 0xf2a118fa1367e5b6, 0x1ef41dec2f857eaa, - 0xe1eed5042b8f337f, 0xaea45849643da220, 0x35746d201a2764f4, 0x5cc21b8b28008b76, - 0xe1ebfb9bd0cc9ede, 0x10f165081eedae57, 0xb0a71a8c90256317, 0x4666c1cd4ac2898e, - 0x68bf3a48ffecb3d4, 0x3317507434db3003, 0x61ba7ebdf94b518d, 0x41a518093ed93fcc, - 0x1f0539464db2b670, 0xa3312a11e34177ea, 0x4153b75f5e2bd449, 0x0c93aafc1497a9c9, - 0xfe64520bef8b6c1a, 0x2ef3b69dc1b950eb, 0xc50bcff31323c55d, 0x137b9df0a41bb83c, - 0xf8e4e98d30582e1e, 0xb5ae72f9f43be8ca, 0xbdf3f8e1a74335ef, 0xc87d092b772b41c0, - 0x4cb852a1e3f5b5c6, 0xc7d7b951e854d86a, 0xb0411ebe46765c16, 0x568eda2c24758408, - 0x9f8908e6198d5652, 0xd468571e05276c24, 0xbac62185cb3b365d, 0xc7fca94dfa7403d7, - 0x7ac4208ffe930e32, 0xea6097ad7725dc7e, 0xa237def29953e68f, 0x2a0f537b4b4af124, - 0x1f83cf7028ea592b, 0xbd9a9e944cfbd5be, 0x4ba1867f6cba1f75, 0xfb78ad72b0219be3, - 0xd15e52aae258e86b, 0x1045e5d48cca55c5, 0xe13a7dbad01a7cbf, 0x82f199510e27c08a, - 0x5edb7aa78eed597e, 0xf01a1d727e07a609, 0xbf012b768ce03f91, 0x96f65507238cfad5, - 0x8af234918d123905, 0x57917e837e8cd0f7, 0x6288c4124f07bfaa, 0x5c2b52d1ac020b07, - 0xede50a7eb7ad4e0d, 0x7306494f6df89616, 0x1ffac5c14a635424, 0x40d92e3c4f9e7253, - 0x965447ba3f573fc6, 0x99ac297b70f9d72b, 0xf0e20f2b03170156, 0x0c801f40cc1df451, - 0xbd5bafdc652de461, 0xb099ae2dbaf93d9a, 0xab24028e1b8172b0, 0xd7ef34c5340db7b7, - 0xfd5ed5a0549f7d09, 0x05cfb5d7621950f5, 0xbd9d9de82b5656d2, 0x952e13c8559173eb, - 0xae66eb2052370a9d, 0xe871c6fcc9a75f32, 0x23fa6877d29b6221, 0xae2cef849717be34, - 0xbe9dad289da5e2d0, 0x8d3f46860877fac0, 0x136e6d4ace20c93f, 0x5b0d9e220f209e99, - 0x0c0eb9608d75e9b6, 0x30fa13e5b070252e, 0x87d13223efccc859, 0xd71a27c23cc63c85, - 0x4b5b83c6871f5cfa, 0x74f73383a108dd5a, 0x1ca54d5d2f768044, 0x1f8fbacc27067794, - 0xd642d2f8356c7a49, 0x9e6689b2ef3802f1, 0x85e616cbe8c19dea, 0x8b489aa9dc687936, - 0xdc0715eafd33ab71, 0x53d2ca40ea4a728f, 0x9ee351b1217ae867, 0x00eaa589c55c056f, - 0x4397eba9f5e52757, 0x3547cbfb489e393b, 0x9b824502e15c3449, 0x0d5639d3e43cbcb9, - 0x9ea36df677ecf972, 0xed10fc839cac702e, 0x134228dd9bdab488, 0xc0e4458c71785f32, - 0xe10d06f73d07ff19, 0xfabf3352891381e0, 0x84d3fd91e1331f1f, 0x4e3b838d0bc45076, - 0x812b0f10637c7483, 0xea53484dc3b88ebf, 0xea69d601cb027ccb, 0xb49a40c94f1de4d2, - 0x2e9e687201084bb2, 0x3a5dd49aefd6427e, 0xe6404dd01c3aeeaa, 0x0d62e010d2788934, - 0xf9670006c5e67689, 0xcdf6ca7209300d95, 0x494196c2f168b56c, 0xec9145c61b32a618, - 0xdd75c5986799aeff, 0xad348129b9ffba03, 0x6d9f0a815c90a142, 0x9077173b158b6145, - 0x7c208588f257f28b, 0xd97337ac54508243, 0x4d757e4b9a316189, 0x20578fa448c1f64a, - 0x603c38468d73d4bb, 0x6e3d2c01600e8143, 0xf22ad145b1a6366c, 0xaaccbccc2d34fd7b, - 0x84a6b01db9a3bf93, 0x23e8cd1d659f2137, 0x5f684d4e486810bd, 0x0991db2ea3c08827, - 0x02fac0c908599741, 0xae9043198fbf88a8, 0x4b38052c0ee8eb27, 0x5ee085336ca7b73c, - 0x934c785c648c9306, 0x0d8ffe812c23de8a, 0x8e7ae5bf93b0a735, 0xdeaa1dd1945ab6f4, - 0x2357f2ff8be97053, 0x7fe6c7392febe11d, 0x089d5fa95b2519f3, 0x8c6ee97646323f24, - 0xad6b7ed099b311db, 0x38431c703502c400, 0x6e11cbc2f4e4270d, 0x8263bfb1dafeb41a, - 0xad7097c0cbcb7b42, 0xd6fcbea1016b0d14, 0x2be2f83ddd35cb17, 0xa85a65cfa7e8f192, - 0xa6bc82bf4a403279, 0x34952e64df73c0bc, 0xec0493fef156ef40, 0x8332cc58646fe8fd, - 0xad9ed5b9436010d6, 0x8af139efb2484d89, 0x24fdf989b7e2a8f7, 0xaefcb67beb38aab4, - 0x71e1ad9080d3dc45, 0x6e33032124026a46, 0xd6cc72ca6003d27e, 0x8c32cdaacb2e6281, - 0x4b2e1e5ae4305092, 0x4dbac277fa4bc906, 0x3d0b60b3d5f23f6b, 0x3e9b4ea5bd200fe9, - 0x9c7b5c3cd2aa2828, 0x0ef020eb51bfa9fa, 0xc6700679a120f2a4, 0x7b8df6e053728241, - 0x7ade720fcf2ddba1, 0xb12102b57d2aa2d2, 0x2e6d676006d4b171, 0xef1fe437ca6b8caa, - 0x6666700703ff9641, 0x8fe91eb2585c9c9d, 0x46933ca24b693444, 0x69519edeedea3b3e, - 0x159b1789ff0cffb7, 0xa1232c8260a26fc4, 0x44b5f96671c1dcdf, 0x5cdb823b21875cce, - 0xb208c8cd7f6cbb39, 0x0ae5c6a881afce06, 0x3a341dfd3f7b669e, 0x40f9281816036d8d, - 0xe4b70ddf94a68a37, 0xf5fd85195b1b2f97, 0x024a921bd04eed31, 0x6bf328ee0efadd1b, - 0xb9e19213a20c1fae, 0x7e1879d7b06d2086, 0x38f7ac6ac026e465, 0x59b72a5f13c192f8, - 0x833b4daa85eec5d1, 0x3648b527328425da, 0x33cd4c074d3e95f5, 0x21b4f87fb462d872, - 0x5a962facadccf050, 0xc60fa211e753dcda, 0xe9e673c033892942, 0x7303aa9b19e6608a, - 0xc73d98e2230bd6cf, 0x68231928d2abade9, 0x1427a7dbcc2ee787, 0x9e9e41797e8c613c, - 0x9715d1d10cfa7cd2, 0x7960165a64a60a61, 0xcdf740436a708000, 0x70e3f0e9994e744a, - 0x5816aeb7c3ecdaed, 0x9d8915ff37fc4a96, 0x4cb51143803b0a93, 0x7a0adcc16f9b1beb, - 0x422062b638359f16, 0x344bc6c4e290073c, 0xeda0cc0c24fdbb34, 0x3a9c81462b59ceb2, - 0xbd82308bf76a1cd9, 0x74c452c2f029d410, 0xbb9dc5f4e7ada533, 0x63dff9d3e5693396, - 0x58c88044a48038b2, 0x9900478c78615eba, 0xfde9b14287390053, 0x82d22f6b5d9c9bf0, - 0x0f41023071a2e1ba, 0xf3bf6dc000b70e71, 0x74547dbfc56609b0, 0x4410714add5110eb, - 0xfae57be8fbc5d666, 0x21b066345b096623, 0xa0188239d8a0f016, 0x7568d7e32b286fc1, - 0x3612f13ce609212f, 0x9137d545ca6e2fc5, 0x8fa6748e2d703dc6, 0x1dcc4f600f50b63d, - 0x434b32baca4b9484, 0xbed9c6df88b107a7, 0xa238542a67605cd9, 0x42d499c67535490d, - 0xbed7c65ab167feb9, 0x0e2d9b421a200965, 0x18ea4b9e911e4142, 0x58d20e2293e56bc5, - 0x275db3d28c78265e, 0x06ee512cba71d0f4, 0x9688deb9e97c2294, 0xb97cdb8d1b81e956, - 0x67533d6e3deb86d8, 0x0524878a81ed676b, 0x7062d73f7e8ee778, 0x6a053b1361985336, - 0x4ad3280659f49a98, 0xe1f40dda848afe59, 0xcd569106951cf523, 0xa5718b489c89b788, - 0x94ff359b11447e7b, 0x4a5ad3ab9fbae86f, 0xfa5bd91dc159103a, 0x2b2fe43aace7f9da, - 0x5514a060df98226c, 0x0ee7482c6b7cd226, 0x5b12ad7f4a36e091, 0x6d763d570c7a3d16, - 0x80ac0f088c0325f2, 0x92ba5f82b6ade82b, 0xf3917866c2ef6a0b, 0xcea7882f6cfadd58, - 0x3bbe413305e214b9, 0xad9f1a3fd51eec00, 0xbb66d0285f8a4e93, 0x29ee68ed090df847, - 0x6dfc631ccc8e10f4, 0x7639d254615bceb7, 0xe5c388004e58239f, 0x863bfe9b5f7d3898, - 0x1e724a8ba7bb7b59, 0x9c721ed7711f4b62, 0xc9535c044a2f9353, 0xa396cfdeedfc4143, - 0x30b4f246a536fd7d, 0xf350cf5ad26720e8, 0x9a43e38f265b3210, 0x9d33aeab3cd6950d, - 0xec95778e77b20800, 0x46dd1ebf2a696aee, 0x7e74a3c113d7f40e, 0x9a4a291cf13ea136, - 0x735aa38987dbf7d2, 0x0b5a69ed9980680c, 0xe27cf6e9b5288bc3, 0x505b1d074b66a583, - 0x13d0040502eb80ce, 0xfb17e5b2fdffbccd, 0xc30ca9574989c0c3, 0x10bf726e578c21fb, - 0xc668acc2ee45fdc4, 0x7cdf5587f0143433, 0x48a17f1c3ef78e1a, 0xcbb1dd26a47d624f, - 0xda31856fcb739d6c, 0x8d591f5a29646617, 0x21c8c466f12845b9, 0xb593ecf8d16f8adc, - 0xba909c56dabc7af9, 0xe966e5a47abde7a2, 0xf731925114fa06e0, 0x6a14a4d6b46e3c0e, - 0x88ce236a2599f99d, 0xad9a2d3df024558b, 0x40d6179c58ca29bf, 0x4b0932da0dd79185, - 0xd8f6ec18852f5a03, 0xa16c134263b8f026, 0x5d91f18c5e0a5162, 0xe38cf0c0336aa0dd, - 0xf6bfc0a4857be21c, 0x0fd63faef86280e6, 0x93d381e9d184fbc2, 0xb2667fdc4f709422, - 0x89df29085ca1ba5b, 0x0bbf611999981c79, 0x356ed9536ef97eac, 0xd967c57787227b3d, - 0x149abe628f7fe7dd, 0xee992b9d12a4172d, 0x0c5f17bc3dce1223, 0x9f70f68a68b92fa2, - 0x2903b10ffe33fc51, 0xa8ecf8ea96093e6f, 0x9190bb471b454c77, 0x8d1d35335edccce9, - 0xb899815177efe522, 0xd45d27cf8128c439, 0xe79629cbdd6010b3, 0xba52ba65a52e4eb3, - 0x26ed87e226c9fb22, 0xdbe0db4bbd7519a5, 0x120eca287dee888d, 0xd2f7fc69830e0a60, - 0xfb4dfcfa77057d31, 0x8215b70677827952, 0x06bb138db0cfe875, 0x056268c0e706ee90, - 0xefac511a4f798498, 0x945d8cd21628b345, 0xc1c069f8685affe4, 0x512e6a44a73706b5, - 0x59cdd380757b532b, 0x562699032d619472, 0x4d55692214f22193, 0xb70f0af2ef39a86f, - 0x19822b3ef7cd1b8b, 0x87454fcd5ed8d1a8, 0x80d6b306505941e3, 0x437b678d1a8c14c0, - 0x3931b1d8a7904c06, 0x7d8eb32de4e18644, 0x6eebe9166c3000c3, 0x2d65c42904528a44, - 0xffb63149f3af2acf, 0xa799a35c59525162, 0xbdd8b1ece3a34828, 0xa4d72877704b1477, - 0x3f1344ca5311ed96, 0xe75df7afb57a747a, 0x186833ac2da1d112, 0x1e8029bdaa8cfa85, - 0x8fe6ecf4e4b06cbe, 0xb42eca36d6586a20, 0xc5f7cad8441e1b74, 0x4eee86f5eab8d53e, - 0x946fe279bc9bf63f, 0x90856bf5a1e9969e, 0x6dc098e4b03cec48, 0xf84d203b36ae5fc0, - 0xba31832016f094bc, 0x5c729b422dbb0659, 0x877e3e457e6eddd0, 0xd2a188ffd5153ff5, - 0x4e96bd6737eceaca, 0x6b4bdd29ea50144d, 0xccb766eef0541baa, 0x76e156bf39152cad, - 0xbc434b89559cc4a5, 0x825e9a3fd791535d, 0xdc8f26c7ea6567df, 0x9af7a54a7c039c8a, - 0x306f0dc701253fdf, 0xfbc8b0199055a8c1, 0xb1707d865415160f, 0xb11a95123f3c9a42, - 0x2e44ef9663591598, 0x1e1084a35a63e375, 0xee388b73b265a7b1, 0x23918627e715d581, - 0x07f4ad1f562237fb, 0x1a7e603f47e9de7f, 0x34100cb488cec8e8, 0x14992b0b9851e44c, - 0xd61cc16d8d3b1db6, 0x1dd8d2be1c65fde1, 0x1db08942bdf50541, 0xb75500f5335eb49d, - 0xfecd8765d392c105, 0x6c563d8b0abb13a5, 0x120eeada2764bd89, 0x91872c22a548436e, - 0x56e12938f16e7dce, 0xf13c7ef4929060ba, 0xd7172a3b90e0f85e, 0xa617c41f39e63509, - 0x287c5dc005087a09, 0xeacd597984c56f35, 0x4ed7886dad9c575c, 0x31aa01a94964900a, - 0x2f7bf1056e6c19ad, 0xaba4d401ed742250, 0x905cf49670844e54, 0x617becba85fc7c70, - 0xdfc98e481a34de4c, 0xafcd5e00cfd6d0d9, 0xf4cf2023fe4d3101, 0x9fdfc9ab3294d05d, - 0x6ca764d5908ff10e, 0x59b50dad3cd67c34, 0x4f13da29fed25d90, 0xdf6c93d97a1aa2d2, - 0xbc29df97a920e4ed, 0x08d0ee19d5376900, 0x2f6b88fe55c5acea, 0x8a08cab54c4718e2, - 0xed7e1ab21899dc6c, 0x872acd7cbe9da8dd, 0xb3fc05cd5b6239ac, 0x4a524c829e8b23ea, - 0xed311acc11e08933, 0xb0f16d23a74fe645, 0x980e444b682a9845, 0xb95f4095519c3fe2, - 0xf111b31a1797605a, 0xcc2bfe99ee65a4ad, 0x03b53f9089864673, 0x1f2bceb8115f0458, - 0x4e670e0c7f4d2d1a, 0x1cfad714abab0391, 0x2d677e826d1ebd00, 0xf840c088d8ef282c, - 0xfb0c3c0e9769fd5a, 0x201b6dfe79d1ee44, 0xd8f9f10c925b7ee5, 0x83228c30a0856857, - 0xe35a8122e51098b4, 0xfc140ab95b2604bc, 0x82e8faa5e23d1010, 0x0760654ee3570bcd, - 0x22f7d6e10b111fd7, 0xe849a767a5a3cd5e, 0xf46158fcba11c40b, 0xfbbd38059171926a, - 0x3ecf7588d4238dc2, 0x447b2fd50683cf2b, 0x52e4afd72a9f8f33, 0xb212a4902a93b17f, - 0x4c7a70627fec40cb, 0xdb2b1fbbeee580d9, 0x6228b281eec43834, 0x88845030c4d62857, - 0x087142235b2ceb9d, 0xd0506be613196d48, 0x872ba6e4502cdc85, 0xb6a2962ac13ac070, - 0xcccd77a783ae13a1, 0x3239ab4fd8df11a9, 0x3b1a873b8b26d62f, 0x3576b85fb65c83c5, - 0xf8abc5cd0c082b33, 0xa980c105bb5299d8, 0xcf0cd05aff348fea, 0xaf526ddf5ec270ee, - 0xa4cead7465e20916, 0xbc637b01ead98c50, 0xbac9cbdc4c482b35, 0x39dddd1b3988635a, - 0xad3876744b0ea9a1, 0x83bf558eca06888c, 0xef2e72d65b9ce4ba, 0x419b75f71e947881, - 0x08874f0049fb581c, 0xd65a8e40701e3ac5, 0x57aacb192b4002af, 0xea3ac58fbbb9ed30, - 0xda5e297cc254896b, 0xe90ae3f457c395ce, 0x0df4f2c5b3d4b9b7, 0xd5c350b298f21b11, - 0x22ddba3fe4701a1e, 0x5a7bf3c7b6cb444c, 0xbfc160dc45d38eb1, 0x52929b4348d726c7, - 0xf37d6d1f0cf0ab25, 0x821d3f162035f8b5, 0xcd1c396a30578748, 0xcdebae816ff01552, - 0xb19b6f7e3cba80d2, 0xba82705fab694f1b, 0xd7797d62f7df8249, 0x8ed4a406c4785f8c, - 0x4e14379aedaf8243, 0xfb257acddd3d523f, 0x7f70354b662cba01, 0xee5d70f638ed8382, - 0x0bb7591922631fc9, 0x871e89090fda1ef0, 0x34cc11bc4df9203c, 0x0609cd069806ba74, - 0x907484be16d75f4e, 0x22dce1b9c0288aac, 0xff7464dcbea5cb6b, 0x326ba60336161e63, - 0xc64da7370fd95088, 0xc8737effa3264203, 0x8698c6b9f4f99cf2, 0xfd6ebe008b3ccf8e, - 0x5e536d093cb845f9, 0xb4368342b45da646, 0xf8159613936d64e7, 0xf7f14e83b5e3e6e2, - 0x9e51f76ac4be0d7e, 0x527ca003cafef960, 0x39a6790b7ae53b0c, 0x8dbfeb13365e9637, - 0xc8595b330d552033, 0xc98c4f63bf04ad42, 0xde3b0ac250098871, 0xd5a0f86c5ab092c1, - 0xcd7b9fdd1a1655d1, 0xfd2f6c0d722db96d, 0x0afdd5f171b67772, 0x691c1873ce370856, - 0x9597215d545d8697, 0x454910c21bf1561b, 0xfb75a0a5164a5ea7, 0x21f28799dd9a8b10, - 0x2d0c36bd60ee44dc, 0xd3eb3d1e0ddb837d, 0x25a599c2eb8b22e2, 0xe6d8a5030cc1f15c, - 0xabb88ca189f02f72, 0x7d48376cbed985a8, 0xfd16679909a7900a, 0x29678d861be670bd, - 0x251ba1e01a47e8b3, 0x27cd303bc25f6210, 0x377288ece6dfa687, 0xd0a78cf4b48da76d, - 0x6140135d3c46b1a4, 0x959aeef6bbfd2687, 0x00c23da7c036ac33, 0xd87049989ff2a05a, - 0xd484146ed334d5cc, 0xf9aa593bdf9e8e33, 0xf8b35acde11d27a4, 0xbe5a247a6479c46e, - 0x1d7a9b2a0ef2d588, 0x13123b10dcbb133b, 0x5a93f7cea88bb29c, 0x0ad93302ef02e498, - 0x95ede844de9db014, 0xad8b7e46715c06ae, 0x873b8aa7983cdd71, 0x3d8469afde978a58, - 0x16a715d99083fb20, 0xd81fd4dbfb5345ac, 0x157a3d58119df0fb, 0x0070be0f86a6d6c4, - 0xb3a4b324ee87c381, 0xb4701afc5a2e1800, 0x8e4fa12ea1327071, 0x7bc9e55f46be6feb, - 0x9b7fba62714509da, 0xabf96fe3c4f94389, 0x2434bf57e0923c13, 0xf039814c7b5fbbc0, - 0xf62948f9d12de500, 0xe27a9c61ff29c363, 0x4c158bd0282ffcaa, 0xf005469c290be1a7, - 0x713255c977a11599, 0x3371a889ef8b93dc, 0x246ce79e9f5f9749, 0x256df3e3f93e4174, - 0x40570986a175394e, 0xbeb3184ca60427e3, 0x46eece3a96893bfe, 0x8319c48c5ae02fec, - 0x7e2f43ee60890de1, 0x680bade6e38e15f4, 0xc8bb01f781a549b4, 0x59d75a2f91ef9639, - 0xb6208cf32a5f5c2a, 0xbad59f75e07e26ec, 0xd8c040a0ca9f913a, 0xd96b1c39084540ba, - 0x49012aafafb0749a, 0x8316954f99465a40, 0xcc4616b7727811a7, 0x27d9ae5ba5aec630, - 0x62d7b918ab412f03, 0x01a999661c887da1, 0x182564ffc63dbb91, 0x8d220637d3c436a9, - 0x85de11d68f30f5af, 0xe8c3e2c18bca1e89, 0x5f2f253f2acfe2ee, 0x94a9f4cfb8f79236, - 0x06bdbe4b8a6caf87, 0x4d160a65b7ccaad0, 0x226ce5ada73f7c50, 0x0e327927e201e6c2, - 0x03478649d0dff484, 0x11294335e800e7a0, 0x591b89f8b03cfa8e, 0xb4c68c23eceb52d1, - 0x647c7ce1da2788c1, 0x913cb47d1ecf5d86, 0xef3e13a2f80e779d, 0x564ce4592ced1cd2, - 0x9b5e8d79a9b2769f, 0xab18d22e35694aed, 0x3f99bad99192d8c2, 0x0d12d6ba73754d2a, - 0x89c06bdb767233b0, 0x64faca09ef9f9792, 0x9e4d4787c52968d6, 0xadebff40c7245fd9, - 0x33a2430318f35c43, 0x9cc8b34cfc9b18ef, 0x64f66e7bfc30492c, 0x7c7c1c9533a95733, - 0x7ca0283d6019227e, 0x41e804495a7a30a2, 0xef8900296ab2c56a, 0x9be1a18369abfe69, - 0xc141427e8065408d, 0x8ba938d17cb402ff, 0x8a730af8ea1a8d52, 0xbfbbb67cbfd25bad, - 0xa23d505fc7d538c5, 0x0f05a46b6fdf1548, 0x89991b74878d21c4, 0x92fe6a0d6efdb62f, - 0x52378e523d38481b, 0x621da85e57e0771b, 0xaf5017b1fbe3acb8, 0x63dbab1dd3bafbc9, - 0x02f26a18db45556d, 0x112bca10cb100927, 0xe3910e1321b14eff, 0x7e2e657408cc6d45, - 0x5adbc1bf88ebef9e, 0xbccedc4a955706b9, 0x18350d8f6ef3653f, 0x0021550d0b29abee, - 0x8de975b8b953ab6a, 0x8b37133f46c0ee73, 0xeedcf73332c92ddc, 0x29ff849ec9632497, - 0x7c399ef612b95b16, 0x3d37cb300302f0d9, 0x0a3d8d316e6da2d8, 0x80cff5e89a293584, - 0x1b65ad23d2905210, 0x39e574471588b3b5, 0x0f2b362f319677a9, 0x7e33fda99fed03bc, - 0x91209738fba939f5, 0x332ec2936a144f15, 0x6b8aae9928f7d164, 0x186310cab5a6ce88, - 0xa02353b054cfc664, 0xfb90b6168c8a553e, 0x7783772e52f7bed6, 0x1ca6b1d2d93fd27c, - 0x2919c3c85d108c07, 0xe50d434dece73c49, 0xa67492acea15d718, 0x7d7f9e709625c657, - 0x3d92c6863fd40be4, 0xdfc59bc210e0e430, 0x4e76c24ed1d234cd, 0x49cbb6d2ea0f33f8, - 0xbcdf8d9a5e042e86, 0xd14e6f875815b512, 0xbf5e2074e8aeec02, 0x9e02a20e5131b05f, - 0x360ad4df7af6f624, 0xa93dd841e726c1e0, 0x76de1c577c038586, 0x9185f1fe9589a03f, - 0x392bc110b8dce0a7, 0x058d1b660f84067f, 0x6b85f1db5877d118, 0x4ff8a4af2c16aae3, - 0x7b4bb03622f7310a, 0x0a04cb78d9d7533f, 0xb7025901a6c083ac, 0x0b09966aefb3cb40, - 0xd0e69307f85e5c3a, 0x84e56a2ad975f315, 0x8a0e30ae3683730c, 0x398d1481c8f77458, - 0xd168765a21f564e7, 0xa6c0cda290d3a7de, 0xdaaea91538af2b40, 0x58a9d0e6ebeccce4, - 0x2c98c34b8707e751, 0xcb705de2df843d4b, 0x6abcb4977bd679cc, 0x66eae07f7ca3eb7e, - 0x0198d015228079ed, 0x1f5af0dcab0a961d, 0x11171b4bd9ea0411, 0x07509e5a234f7ca1, - 0xc6da9747a5e44c1d, 0xef58d859cb214cdf, 0x5e27bc81c39da0f0, 0x510ecaf8e6923bd4, - 0xf6d24bc7952b0266, 0xcb616d5c4ccaee5e, 0x964d0f42ab767e42, 0x6426a976385cfa3b, - 0x48c52b3e35b2a6ab, 0x7a83ce3559e8de74, 0x3a49a3a6807ec94e, 0x58b9b654692327e6, - 0xe5feac449fb7d844, 0xad68b5d2ab072006, 0xc6828f6826ca0097, 0x1088bcf33bd145d9, - 0xf7b02d61cb8d6e4c, 0xb6f8ccf41fc5a80f, 0x1db28e40b480db66, 0x80726481766d44d5, - 0x49caba6f8edd2328, 0xaa5e72ff6eb5a5cf, 0x89555ec7aea4890d, 0xa1d57437d7b99790, - 0x1f3d0083172c3dfa, 0xcf66d1c8b3050d0a, 0x9975e2903831f484, 0xc1663598ab4d535b, - 0x6f6c4a52295a3728, 0xa66d60c70ab4182f, 0xa7fb03c5913b78bf, 0x2f5144c88969eb37, - 0x171b8686ba0efb70, 0x3ee576b610f5f132, 0xf40dfe346f5b4bc8, 0xccf9058d902b3ec9, - 0x64ec951cf4aca808, 0xe2ed49d1c923ad14, 0xfbe06a225608e407, 0x6cd20ca28c0aa562, - 0xbe32262bb89e936d, 0xae28a8f9f3f56bbe, 0x2dd2dd181ef5124e, 0xa8f554eaa13efd64, - 0x41df04024af6c920, 0xb7e5210444f123b5, 0x4fcc36ac1b1be891, 0x3d8454480e77be6b, - 0xb57b992e101397d0, 0xc910450eb88abe75, 0xc215c72ee826a992, 0x75a5704b615c0864, - 0x74c35f9f426d57fd, 0xf02e1d3fbf344cb2, 0xebd59577da4067b1, 0x5f5d93684b2b5de3, - 0x13af3958f84f50a0, 0x35555eb8d0003ecd, 0x228e26aec68b1949, 0x51dd64cd057ecc84, - 0x9e0452f462ba789e, 0x48bcf742566d525b, 0x93e5514a8fefb005, 0xab5c500fd2ba2ef4, - 0x5a3b9b17e4a87c4c, 0x29a61751806998bc, 0x7fe6c019f39a68e4, 0xa24a22d788d14c29, - 0x83e7d3c298215759, 0xdc2b1c6d6bef4347, 0x640d51d84ed56834, 0xd613ca709f6ccb13, - 0x7f13198374c22a80, 0x909a5872238daaad, 0x4b96a56dd7ab3686, 0x73c265dc5bcc576b, - 0xaa303446b9af8b68, 0x9d0446cda0695515, 0xcfb11aaccb26125d, 0x141b6afe3b7922e8, - 0x7b39e5a5a9b5268d, 0x1574570a0fe3edc4, 0x7fb5c1976e9a8b0a, 0x89f99b08d73cd1c4, - 0xc264beffcd584506, 0x2c27c54de04b3e61, 0x9331d520fea42719, 0x08cebc837ffbe441, - 0xac4ea5401c06c775, 0x1d80118aab7d109d, 0x80bad2d976263da0, 0x6e9f6c33dcbee03d, - 0xabf37d960728fd5e, 0xe6611f6c26fe5ded, 0x44d3d9454eb9ed43, 0x15960550a922ab98, - 0x7000f8c46d6680ad, 0x446570f1e3da0705, 0x060c5aa8eb4f8180, 0x40729a6edace1b71, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x110319a95b6c0546, 0x3de75ee7ad2c63c7, 0xa55db193d9966ef6, 0x65681ac6ea84de14, - 0xa21c06e5c3d161cf, 0xbd77bd34b807e312, 0x6ecd2eb89a476d4e, 0x1aedda80c5247e06, - 0x30a4a00cd12bd5ee, 0x844d1c312e6ad357, 0x61e29c8f12d8b877, 0x1564f89cef169c28, - 0x94ea53df2559ef28, 0x87591f659b385833, 0x96020e68a0aa321e, 0x0c0786905578830b, - 0x474e8d4dc943f8ca, 0x6951da06938af9c0, 0xbb2551baf39271d8, 0x53743fda6c7e7bc5, - 0xfc1cf9e1c6e575d1, 0x086d344e8c362e2b, 0xf29ad418fefbcb54, 0xe6166484464442eb, - 0x4d9d6cf095bc1964, 0xc0e26677d220887e, 0xd257ae6bde3e236c, 0xefe8286d6b0efd42, - 0x2ec3a9684fdb8f08, 0x89ee7ca150dcc6f3, 0x4ef9b84b1dee73d5, 0xbbfd0787b5ae48f1, - 0x00f1f4ddb610551b, 0xe3c9488bf3d66bd6, 0xa9ba7f78920faf7c, 0x26b4801f47f97442, - 0x395c5644b51f20ab, 0x7097a1ade545045a, 0xb2f89a9fe0a81bf7, 0x2047b4070bbd5d02, - 0x8705ddb479d0a0a6, 0x290370f5f485ad96, 0x493f0d4bf5612cbe, 0x94a99f31ce11fff2, - 0xf1bb362434fe8bbc, 0x48339dd3fb3dab83, 0xdd96502b120981b4, 0xdcdebeee2c6f8f13, - 0x37d791bc702a1b76, 0x7667d65ef60e4108, 0xa7b79581c3398b5a, 0x67b2b4a621f222dd, - 0xf9f8423ce54edaa5, 0x207c467a2f73064a, 0x4ba0699a72fe5c39, 0x55539a1a69a299f5, - 0xcca96677c9b59f77, 0x2a0671e2b3c16897, 0x8c9e9ba0fcdbc326, 0xa1a49549856825f9, - 0xed96169b72244c80, 0x00384f7470a47820, 0x0623e51567c6cb7b, 0x9c2c40db6fb3ec1f, - 0x514f93bec06ea16d, 0xd68ce1bb75abd6f0, 0x9b1ec28c5c38fade, 0x9d67e2d2811908f2, - 0xafef80aede3321cd, 0x0982e3ca67380c73, 0xeac4d40fb33e8b23, 0xb4717eef350dbbf3, - 0x22c319e86acba70f, 0x28310c6c2d084a63, 0xf118502a2c00711a, 0x60843fd6a0a4b9d7, - 0x52e0d1d6d2f9975b, 0xb975ec91b91b3f31, 0x1b6f2d486b0e072a, 0x1cbb83e02c2a442f, - 0x9e97156b8ea23152, 0xb593e9f1619e9ba0, 0xd8a1664be8e9b64f, 0xe12da827362803f3, - 0x2d8a60d1441639b8, 0xea95fd5c0b5f753f, 0x96a38f9e1db193e3, 0xb024439632a98302, - 0x09fbf7a611973521, 0x56d7a4054e8aeb66, 0x7e5cd659bd7ead26, 0x3ac8eaaf28246ebb, - 0x2a64e860bfce66c0, 0xa490f83d6b1c4775, 0xb7edd37afb40f152, 0x37dd7f2da1e888ec, - 0x709f7e4b98a66e37, 0xa154918cfd64e042, 0xf52c7407bffead12, 0x43f5a97c6bffd86c, - 0x8bd80f9f83feee9a, 0x7eb952f9868847b8, 0x093034bb60335484, 0xad312651e22c3545, - 0xbeb2363426eb872b, 0x09552e5f6b515d67, 0x22a98a1553178c51, 0x6573c89364823566, - 0x770a4302490659ea, 0x34e4a775b8e7f38d, 0x6fbcfe12a404d2d8, 0xd8a07ef95fc29fff, - 0xf5ca702d67a98fc9, 0xbb717017fe665cd7, 0x99583727643ddfea, 0x0c3be40c19d36ed5, - 0xe24291ad8ad63bd5, 0xfadd3f7476aa3359, 0x975fa1dcc4f8ab61, 0x9f3025fbe03e80d9, - 0x43e595021faa54c7, 0xc311927829447a71, 0x751cbef4b9fd1ea7, 0xecb35d16ee4e539d, - 0xe111dfd9f78ecf4f, 0x679b22875f6d1bf3, 0x9f4249e0fe378d58, 0x8563e4ed075f6430, - 0xc0234db57951f1b7, 0x21b3e68128237636, 0x230df1bf74f0737c, 0x97010ecea703a2e3, - 0xe2f067a9956ac970, 0xafdf3db052e79e14, 0x8be8e8fef8a1c6d6, 0xbce086769f0ec5f0, - 0x05ac24f03c7a7f05, 0x3e6b65599cf1af49, 0xc136c60cbe18c03a, 0xcc5745ac48dbe0bc, - 0xc9ffc840bde2604b, 0xb6ae15f81e3c79ed, 0x465b537d59ad1b43, 0xcd53aa284ab22b4a, - 0xff00a8d8a1a8c189, 0x6d454f5019427629, 0xa6a30ebc559fa049, 0xe271a62d4678ff95, - 0x68ef8f7ff1b68e74, 0xb315c752a55dc46d, 0xd82f0ac159a3b6c2, 0x4b17b0430b24a6c5, - 0x254c32d96e97bb48, 0xef591ee884639000, 0x3a6792b9e3fa9bf5, 0x5154330d9f41393d, - 0x53d49b1acb2340ef, 0x82df9760b4bf7757, 0x37a8c3b57990092f, 0x7339c7a57930092f, - 0xe9ced756e229bf35, 0x955a54d41fb46ad5, 0x961c98922d3470b9, 0x401b23059b91574b, - 0xab3bb86f100c8e5a, 0x9a001529884642df, 0x204f2c00b195f6b4, 0x174f5427f2aaa41a, - 0x9bc50a52adeb52b4, 0x1eafc961abc16e83, 0x70615054da1af7cf, 0x018caf327b1891fa, - 0xee324b01319f3885, 0x24adcd1390e351eb, 0xfea08d11c004259d, 0x7a3679f87b8ee14d, - 0x12fc0553c507c957, 0x524884d7cd5d52bd, 0x70f5ee7e645b1fdd, 0x3065ffbbb2d60d77, - 0xba6daf2dee220e45, 0x24fb3351a1c7c6c8, 0xa7c5bf7a7b0acb0b, 0xa060c84f7f595853, - 0xed365a25a7889d40, 0xd99d9e7764205688, 0x1300b50c2a2eeea9, 0x760304140cff20a1, - 0x74aa860784ee79e7, 0x5702724d1de5f04d, 0xfd4e4235e00a9c48, 0xc12e003ec6b198a0, - 0x24bb2ba9fa0c6013, 0x4960f35c8c9de7a8, 0x6a5e3e5cb561fccd, 0xc22bc01e8146a936, - 0x5ec053a0f3f2f6f7, 0xc2b5e1343efe15e9, 0x35d9ec5075fc4cc9, 0x9eaaaa42252870d1, - 0x3039a50e5cbf950f, 0x97c9bf6c78a317e8, 0x3400db7a38dfb73e, 0x0c0cfd0b4fa96622, - 0x5b3898b23f1a7833, 0x1da402fc06aafd02, 0xcc1f37b76ce63eb5, 0x0bf88fc97df3ac20, - 0x4f288fd1cad2f418, 0x96d594cdafa24404, 0x09a33a2be5ed0c98, 0x9d4c9d4de4f7189b, - 0xfdc0255d4159e21a, 0x7c9f478b2b7ea9cb, 0x28711134b56e81ef, 0x2d6df6f259330bde, - 0x38447df1ea9f7eed, 0x48b72a97ed270576, 0xf3849eb60c5705cb, 0x03a32521ae37f919, - 0x8a793893be8b0cbe, 0x462d7e27b8b19630, 0x266fd8c18ff4fa6a, 0x9fe6e4b19ac0dcc3, - 0xdad9099c51afe9a2, 0x50e135cbb7dbe1e8, 0x8daf8a4820b35030, 0x7564c6514a0885b0, - 0x4cf0f4aadbb8010b, 0x5128b02fc3e21b18, 0xb96cf535981d3025, 0x2e74060a2516cfa8, - 0x05f5cbe87cd27ca8, 0xae1bc8fbfbe436cf, 0x43bd81273f0cea82, 0xd9b085eb3a215276, - 0x73e650eb7554bce9, 0x7d325b72389623cb, 0xd15f3096f17bce8b, 0xd9c15be44f34389f, - 0x8ac13f9b6f43de6f, 0x887a30e9470f18b2, 0x5f769821aae879b1, 0xf280c8951f19602f, - 0x3e07cd0af83503de, 0x1fde7e30725fb5b6, 0x7cca20d9e29d8307, 0x8fffe1cecca6f4cf, - 0x0e96533140b1c03a, 0x27521921a570556b, 0x763e261a444d4bf1, 0xef7c6f0a1c73cf19, - 0xba5c17da207c5e55, 0xd94beb4736962825, 0x26c73fc1417163ec, 0x65ca40af3f1badfd, - 0x3945050e1d6d14b9, 0x20ba1acc9e03d215, 0x26ded295d5d26305, 0x8e3c49e723de06ba, - 0xfb9cf3c292422f6a, 0x1e67cba58b6cc50d, 0x481588f66f2ad506, 0xd4a3fe73ffb80a43, - 0x68d68162ad57faaa, 0x893bd6c2b9f933a6, 0xcb4ffc06bea9e1ba, 0x0b1a8960495d7bf2, - 0x83a2d3ed275bed45, 0x6b24389fe489dab4, 0x359d5e1998d87182, 0x22b8e0524afbdc48, - 0x05ed3991a123e1d9, 0xba88b3be44360264, 0x4ce936986eb198ea, 0x77d5e7bb260ebddf, - 0x79c00ff65391c7f1, 0xfb3ce3d7486372ee, 0x9763dcc36e735225, 0xcc629b15630fe816, - 0xb6ee20d289f03746, 0x7368d902e063eda9, 0x3b0953e035c217e2, 0x3fa40b903f1df986, - 0xb4dc8e707fafaf8c, 0x43033a13a49247bf, 0x32dd6ce0847ec239, 0x2fd19a0ac9bd898f, - 0xacdbcbcf10febd4c, 0xfb90cdbb33661676, 0xccf1158d73f27c3a, 0xbd0f187bfa6837ca, - 0x9ccdd6c4dba52a59, 0x11ecba7ace49ed1f, 0xb35f43f97e969960, 0x877c9b37dba99a63, - 0x125c5a7117a2844b, 0x37bf4c204e810ca4, 0x303f0a3458bd1ad3, 0xdb2836671b32b28c, - 0x093898dae862ab78, 0x44edc71daebfc852, 0xdd87e80a8f6114f8, 0xb0d90fa567386cc7, - 0x25fba30982ae9955, 0x99e7c9040816c1dc, 0x38f366726d89feb9, 0x6e863bc2677f8873, - 0x13f871bd46d9626f, 0xd77e20e96f77fe2b, 0x66698ff04f2667d5, 0x82af26eb54ef8f0d, - 0xe799b72855015bce, 0xa38a86beb25590f0, 0x48c65d41b6fbce8f, 0x021ee22d7e621edc, - 0x197650638f23b4fd, 0x74b6330c4b041641, 0x0fd6452eca0702f1, 0x55e7d64e0e67a5f8, - 0x925be9a6e6bc59ad, 0x0b12e087229bfdaa, 0x613dd9811f0c831a, 0xafa19e96bf66542a, - 0x3a16818b956f7181, 0xcf9e07130bf24b2b, 0x013eb736a53ec147, 0x7e8707148a1d6c5a, - 0xbb28599d8a2c49b5, 0xc55c82f0361e18e2, 0x538dba514193fd89, 0x2e4273e73221570b, - 0xe86dead30ef27765, 0x4f99a56ff63cf24b, 0x16c38e98f64847de, 0x3ad02ccacfabde7e, - 0x670ec8811b9e9f55, 0x5a623b5c649fe0c8, 0x8390e02f133bcab0, 0x99dc90535d84fbc4, - 0xedcf55a769c573ad, 0x1ea66fb89d65e267, 0x6f1e9722f203fa9f, 0x2a620fb56ac77659, - 0xa9db271c065a19be, 0x97ff3bfa5c0563a0, 0x3c8fcdf749994a8c, 0x9e1bd3400aec31d1, - 0x3e6153f1d00d71de, 0x23fe340dba5c5d7d, 0x7bc34717ba57eee7, 0x51f07556d6d12779, - 0xb2db91b19e6ec9dc, 0x9466d79552fac32a, 0x5a7ef43d677c4226, 0xe4b3e3b44b4008d2, - 0x0d650ce5584def21, 0x55185b47b7ba1784, 0xd559b9ae107894ad, 0xe25ecde7bc6ce65f, - 0xdeef0fa9cecd9db5, 0x6e83194e36776b1a, 0x23ca5fbac8088d3b, 0x2ea2b5e13955a069, - 0x83ae0d51d534a204, 0x7db51fb2d27b92f7, 0x8066b0e1c195e2cf, 0x2ea2a715dcf5ca7a, - 0xf3aa79a2e4e8ba9a, 0xc9e38a2426ba1005, 0xe6965a4e3f8c3a3e, 0x723976df4e5d47ae, - 0x8b53c335d68e8dfd, 0x16e57a539c129b55, 0xa2ab04fe3170a748, 0xb1f3c9bf0408f6d5, - 0x79e4059b0ad702f4, 0x32cf27c54be88d73, 0x3701750a39367a6a, 0xe107278ee64f3777, - 0x82107d39acf8345d, 0x7804bf75a9abb466, 0x4b78494248485962, 0x3e06cd87b792262d, - 0x38d3bdab42c330c0, 0xeed432b27288d4bf, 0xb6d6d44763007956, 0xaf8c9eeddf473e74, - 0xb7ad16ba172dd0f2, 0x1d78b0319c0fbad3, 0xeaf749543d8717fb, 0xd24602a17c78960d, - 0x728b6c7da04edec4, 0x38328347df161db4, 0x7baaee2508e2f1f4, 0xcd1b1ce2fbd61c26, - 0x0d5ad712f8b27fe9, 0xd75eadcaf9c9ce9d, 0x359a7246144bf2fa, 0x90e9b12f9ba6dbb8, - 0xef9fa02db4067ee3, 0x02c4328216137e5e, 0x136bf05dd39fdfe0, 0x26345a89e17f1005, - 0x9ea317660a5000ff, 0x25b84f34fee962c5, 0x1608ed4a6ec81f05, 0x9300248eebc0086f, - 0x39829b8b250e37a3, 0xa7e9c104a193c7b2, 0xd3e05273f27f8db4, 0x1e4c54e9bebc29ef, - 0x9d7f591184200ee3, 0x4c039e2892a7cbf2, 0x0bc29b12b4b45494, 0xeeb6623ddb019322, - 0xf85026fe0cf55d8d, 0x42699ebad4565675, 0xbb3bc82a7c8e6894, 0x87d0f8d61d6eca87, - 0x33e20ddce045878e, 0xa18f7e74f6eb1aa5, 0x51af767f6bb63021, 0x143f36cddb813795, - 0x87caf66b88a3dd2c, 0xc29d9ca3c25b1abc, 0xde5c49f0a6ee9c9d, 0xf797a55522464f00, - 0x3204a86122ff2c39, 0x3316d28dd56d06fa, 0xadeb7ba08568b21d, 0x1bcdc72db4fbd094, - 0x5308b407e9c32143, 0x1a16e976c19ef57d, 0x79bff62950bb1f94, 0xb85768ce270bf229, - 0x5dbb3ae0026ed7d8, 0xef7e01126be6171b, 0xf2b127e1e38de0e5, 0x5472312c1b0cd21e, - 0xba48fd9c791e53cd, 0xe35ad4186a19d2a1, 0xd7a74e2b56551011, 0xaff43a397a6ed374, - 0xd2977eee69f3d7f4, 0xb1ec6685b70724b3, 0xf5a5805d928ad164, 0x0bb527a23002211a, - 0x240ba1fada8ee3af, 0xae8a0630c6671269, 0x0eb261ca9528fa58, 0x63dd02fc4e07b182, - 0x6e3237e5d46cc1ca, 0x495a763c6450a04b, 0x40f0f95effb96cae, 0x35d59542f938d603, - 0x219c756a60c2c33c, 0x3828648b7743df50, 0x6db50505b1222518, 0x175552b0602d682b, - 0x28b3852575fdeaa0, 0xf8f97bce8254197f, 0xa48c0b7b013589d4, 0x99d7202a722b9709, - 0x96f1f01b64d20533, 0x48471bec7ad43876, 0x2fca4d92d9f43e93, 0x2f461e135ef6efdf, - 0x8a84d4d507ace2cc, 0xe3bc43e1babd954d, 0xf880bf5505d73c78, 0x78ced931cc2a0b03, - 0x425975bc20eb0883, 0x536475c56d9582ad, 0x8b8300360c2310b1, 0xe4457d037678d3d8, - 0x4af41d427b43367d, 0x3d5788627db6e926, 0xff8b7da71dcbc874, 0xf1554f065042da98, - 0x90ef8cc83fd9f422, 0xce850a58e678518c, 0x2e31d8eb8b674b38, 0x027648b12c94f197, - 0xd80b3941a2a70b58, 0x3654bf0ab070a0c5, 0x2b277fa9933b4a89, 0x49b9ea19df437907, - 0x7ac1518756b6a72f, 0x79badbc1b40341a8, 0x1e7ca2f9792024d3, 0xa099b8e844792982, - 0x0ee137d167a96c11, 0x1b9f30559cf261d2, 0x144f3b47a60cd6c6, 0xb128ba3d4eff62f7, - 0x608ca6efd1691bb4, 0x1153d87d1b67355d, 0xc77d4b730cd67776, 0x477e0841a45a8a5c, - 0x71eda8ea81bfc348, 0x634959cabe3979c9, 0x1ffe7efb1943a971, 0x4a966f288927f153, - 0x303896d3129fb724, 0x26959cc3ac3dc8b6, 0x63403dad3af72ef1, 0x50529b68193a57dc, - 0x0a14819461c95883, 0x0f1c646acbf19e6b, 0x23988d3b3747c7ed, 0x241eccf5dbd612a2, - 0xd0efd3431b78e44a, 0xed9bc36f39b48af9, 0xd25f4c7bf17cd7ab, 0xb5164bc81c624134, - 0x990b4af88fd60824, 0x6ba893ecc7abd11a, 0x00b9d3bbb913c31a, 0x42d093dad1fb1522, - 0x3aff13397fb9a484, 0x0fca2853e571bbc6, 0x69a86bb74c5e8c0b, 0xe1260f194f624baa, - 0x09eed3be1e939c55, 0x1cc395fd73c4f85e, 0xfd9831c61a1c8ee6, 0xcd5c43b61e98bd49, - 0xcd990827d6716892, 0xa71f19004f5baaf4, 0xa6139ed032311a9b, 0x6bd781f7fc6f7b4b, - 0x40ec6b89009c6bbf, 0xb8b3612b6b8ebbc8, 0x2f995907f3a3eb8f, 0x5bc415e0821c1a01, - 0x0499c1c74c710498, 0x48746df1a6b7a6c9, 0xb8e1052a6de9064d, 0x9f61fc0ac37ec3fa, - 0xeb9707de7eac303b, 0xb52014d7f9391f4a, 0x58b2ed1b9532e44b, 0xb266fba1389c7af6, - 0x4fc53e4600a39d84, 0x2938a29922a33b49, 0x350fb8d6e38bf2b7, 0x579b8423e18018f0, - 0xc36d41b91081acf5, 0x98f934fd5da1e3a6, 0x12459fe348c32ab4, 0xb9ba720cc1be26b7, - 0x8eb9da20c03c7cd1, 0xcd439f1345770d2e, 0x3b87ed27ca34c540, 0x861889f2f1ccad76, - 0xc34ed859426f9453, 0x25db7d650eba6ca2, 0xa141e81a82614fed, 0xdfdd8eb056dddc80, - 0x3e095042ed05e98f, 0x783814eb104be3fb, 0xc651a3d56231f8c9, 0xe22b241d0eedf472, - 0x2230f0e0e0d45616, 0x2616bacaa4495cc4, 0xa8a8d3fd94f9a2f3, 0xe1fb9e13f5a6c0cb, - 0x4052358b230f23db, 0xce27e292a37425bf, 0xc9053046b8eaa47d, 0x715e25a4415aee7b, - 0xf1edfd790b81a143, 0xf68774111690ece3, 0xfe1259c258f0d552, 0xca4fb3e35d331e53, - 0x9d33ebb7c91eaf7e, 0x8b4fcf188794ebd6, 0x7928f1e8dc854f26, 0xfec66009031bab16, - 0xf4bc9957009f8e30, 0x8141edf76690735b, 0x0181ca3c2a52c997, 0xd64b87476bcfab99, - 0x04af88ba50b37c00, 0xe41e3184c38acc14, 0x302d1875730389a2, 0x7661aeed338ab638, - 0x394b285871294789, 0xaaa250990d92fa92, 0x8848a72fc6b57d12, 0x21088cde630faa9f, - 0xe8db6e6becdae418, 0x89d1ca14edfa1616, 0x068fefb1d93db112, 0xf5988a2fb6655a27, - 0xad5fc9c91a2ca6f0, 0x1a0bdfe4d06ccdb6, 0xcbb0e5025d1c32af, 0xcedd0229ff1a0c95, - 0xffdbf85613e523b7, 0x040c75cd5e116b18, 0x0a2af9c449a1ed7f, 0x89a1a510f8d63b1e, - 0x1be539bf8cdc9be2, 0xc91a7d097d2701b0, 0xa90a4b905a31309d, 0x5c5838ec4841ea4b, - 0x2acc1afb603cbe8b, 0x6fe96bad45c7d0da, 0x7e3c8ba1fa8a7bb2, 0xe351182578ad5da7, - 0xdce62e9114f87332, 0x246a9e9a19cb5a59, 0xc7a6ced89ecae528, 0x778c068fb401dfc2, - 0x19e24a0d62ac7cbf, 0x2f55f70e6c8caf67, 0x9b5c37294b67cb34, 0x5e69630f280c0f1c, - 0x2f0e2e918f199527, 0xb07c7ceb286ecf2c, 0x3430aa947a8a1acb, 0xc8084d8eeea6bc76, - 0xd3897685cd8c939f, 0x6d79a9f0a2d49581, 0x6cf2664a5e612805, 0x6cffea406ebea79f, - 0xe9f6ba9224bf2f3e, 0x8c171b09bec05027, 0x5d375da4bbac1194, 0x962ad2d49c986811, - 0xb809c54d29fc4f73, 0x274eb5b328273039, 0x7fe7f99e93c2ef46, 0xd162721dba9ff92d, - 0x8921d4c29918b280, 0x03268b8d337ce637, 0x4f6417eaa2a9ebdc, 0x1d66a055d5ca69d8, - 0x2d448d23a5c48196, 0xf3f6d07ee780c1a4, 0x1d963b408fe45063, 0xdd97b6322537c4f3, - 0x06e5abbd1fd71b62, 0x129f4beb5271a1da, 0x616a1e5ed786c88d, 0x613130f50bafe890, - 0xe4c17e1e18dc754d, 0x247d30e5e31fa670, 0xaf9c2e9cc9458e00, 0x794d4f75da8fcaf7, - 0xb298150e3855007d, 0xc8e2b8453d2896dc, 0x338ac13d16b55bbf, 0x6fc2e8949909d118, - 0x26c388aaf3ea0edf, 0xca979a2cbfc9042f, 0x1520f3979b3fc0bb, 0x7884661d27bdf882, - 0xa297894ec79232af, 0x85a9a8fc9f32eca1, 0x68649ce690c6341c, 0x3aa4de65cce17d20, - 0x47362f95931e2500, 0x954113cab95f2bb2, 0x111c6bb28d00b431, 0x66ebd5098f7b3733, - 0xb77f1a04a2356cfe, 0x11612886b8d1f410, 0xfcd8e097e11cbd23, 0xce7c77f431a417d7, - 0xe25ec1a8457472ab, 0x2b681c67742e69a3, 0x125808f196cc7c15, 0x7d27390646b39f48, - 0x17b9793d9b10854f, 0x32866e834bb1a9cf, 0x10846bcc19136873, 0xd5723701ed8d72a4, - 0x7f8846fc62033e1b, 0xd8f4c982190926df, 0x8eb015abca047d0a, 0xd21bbff9f31dbb7b, - 0x7280d63f54adf001, 0xb4b86a6c7fe7efe7, 0x79f7e892eeece54c, 0x19cf9fa60d55e625, - 0xa7124541e310ba31, 0x990c8c92c74a04b2, 0xbf5d40b41ca138d8, 0x607688fd0df8133c, - 0x1d2b1ac1dc499ecb, 0x32f123187ff61728, 0xdcf1277a5e09a776, 0xab712c95ae3481ba, - 0x6291f5d23ad26d9d, 0xc3a5327c7e015c39, 0xbe6e75f36ce7e8bc, 0xe18e868d74d1b509, - 0x6a85273d0452f186, 0x2cafaba620e67bc7, 0x73b73276117394f6, 0xced802aea9cee93e, - 0xe69c54e0eac00fab, 0xb44f5bb52388ef5d, 0x4e1a04b816c202eb, 0x6b7eca28df55e6f0, - 0xfb2b50f46a13c3fe, 0x963d18ecf6f82752, 0x3daa48129569eea4, 0xa6938a8727530fbc, - 0x90796990c2f98037, 0xfdcb4a2ce9611b99, 0xc1775d37e9f26b7b, 0xc0792e38194b2df8, - 0x464b82a63cfae5dd, 0x155899c28bf5ea1a, 0x92af09ac2d2d75ad, 0xa1b85444dc654ab8, - 0x8e5cb662be49f9a0, 0x54d36758fee4b81c, 0x3233b1820ebcdfc6, 0x71edc2f7ea1c168d, - 0x1ae81524f6ce36a4, 0x148cc829eb55e3a4, 0x1f1c9cbd77a50c41, 0x23d12bc2d78b8fd8, - 0x5f263dbae425fad9, 0xbf8a3f7106707318, 0x06e885d8b362b18d, 0x01c46d8127fd83d8, - 0x0e33276cb9bc3249, 0xfb5b4557d643a683, 0xa888556ef33cf2c0, 0x64c402231ac0756f, - 0x2b8189260f5fb10b, 0x794769ffa26eff72, 0x239ffe9cbaba00df, 0x51d4a0eed81d6018, - 0xfa341c8f1f102350, 0xa9ed425399332191, 0x8c0e275b6e4c8fa1, 0x1a0d76413c7af2bd, - 0x99a6522c0a7e4738, 0xa3e2eae9222e597e, 0x14e9a543da3a9855, 0x99bcec4a7eb07774, - 0x2b4bdcc8f3e89781, 0x5b445c1239901983, 0x8d760e86af0bb2dc, 0x18552411cef5d249, - 0x660eb657953c1a8b, 0xe169f74ca937d7f8, 0xfe21e8ba64c069a0, 0xe5a682ed777f9f4b, - 0xcbdf593bb033a217, 0xcd646e3f07326bf2, 0x0d098a7c404bf1c6, 0x90f1325681fd11e9, - 0xab271f1a27b21af0, 0x8f722260df412ac1, 0x61142bb39567b2da, 0x5702ab9ef092ae41, - 0x78a92be57261f236, 0x1e77ed4f8a11f266, 0x4ad7bf7c7d6b17dc, 0xabedbc57856ebb4a, - 0x7d18c28eb0361b1d, 0x58581cc08f09b65c, 0xdad12c1dbb2c8dbb, 0x69d8e48d3ec27b9e, - 0x9d361c482bbc3064, 0x19977638a1d959b1, 0xa287ca1ca96c84a7, 0xc9944d8a430c784b, - 0x8e97d6610f9deb42, 0xb99cc4c70b2f96f2, 0x4a1f07c07a53b9ac, 0x12f1e64e68882c69, - 0x9adcb5bb88d871ff, 0x2117cd026b609ca2, 0x7f9623dd7302a097, 0x7043eb9c8730e7a3, - 0x860162362d15d293, 0x1463607d930ef3b5, 0x9718216ad0c4d016, 0x001b351a86086a7d, - 0x06d00eef514699aa, 0xf6df9927a56af0b9, 0xf20e6713b17ca988, 0x70b876bdd4474b68, - 0x443bf18d9b3b3448, 0xb48dff93baffc7d1, 0x9401a4ab10ea9c5c, 0xeb6718f8fad04e77, - 0xd1ef1072112c20a1, 0x8fbcc8a0c50ac761, 0x75f0b40f2e341b77, 0x40778f298ca7ec59, - 0x82e88c9af6c4e7f4, 0x28ab3c216360636d, 0xf6d914d72baafe4f, 0xbdb539357040da2a, - 0xb1d4fe75d99f04ba, 0xabcbf861a50d3b75, 0x24d378cc13a14fba, 0x6e45545afdd39345, - 0x82a0b6fcd67c8341, 0x448c4f679800a85b, 0x05ed3b3e9c75a765, 0x3dfb2d1ba40431df, - 0xa2c9eeb56c803c9c, 0xf13e87eb913ed01f, 0xea47c5ae94e6fad9, 0x89ee601c9af0d8b7, - 0xd218d431d63e3695, 0xe2787d5a187ef71d, 0xf475c61e60bad04b, 0x1cc66fb035840585, - 0x6d824991555dbf6b, 0x35d5fbf1d4475ddf, 0x8f9888f9a6f2dfe9, 0x34ed4699fd8079ee, - 0xa57a8df8f633eee3, 0x9da207e3c2d68517, 0xeadcd5e0df7cf0e8, 0x03c3e5764f631089, - 0x0d492f9eb763e3f5, 0x8032957d2d6fabf8, 0xdd5355001ec864b0, 0x8587c358edbf82b6, - 0xbb19e4648dde4ed0, 0x6faf32e7ede1ec9b, 0xf12b04b4e8de2743, 0xc75d0d9394899fcf, - 0xd38ee9aecbfe8dbc, 0x409503645e539352, 0xe444a13cc5b877e7, 0xe9f5c3daa337a73f, - 0x0ba025884d79a250, 0xea7b4c4442e65d5c, 0x5f45387e569210fe, 0xa3ce1dd17157abe6, - 0x991bfe1e8114973e, 0x7b2cbbc18f702aff, 0x54441a039152629d, 0xdbebaafcb8bdd6f9, - 0xb22f22281b8cfa99, 0x26093fbdad9c1511, 0x93fb44a8c2b356c6, 0x1a4b2c94cd8faba2, - 0x5ce6a34d594d8fce, 0x69fdaf3ac4170a59, 0x6ee78bed3d6014dc, 0x022e7eafbc46d660, - 0x90726daa100b53cd, 0xcf2b2b5c2a971eaf, 0xdce45127fc8e9f79, 0xcd3b479108737ae0, - 0xf36d00bac24c90c2, 0x4747a1b077a7a32d, 0xc1f380bc4cf822c3, 0x3bb345ca291389bb, - 0xb75b33f0b656ab44, 0x4b52c76233d7bd8f, 0x3de014d76905ee81, 0x8570148c2da4443a, - 0xa68a37c0a34b9058, 0x32b24e784d05ca25, 0x5a3572d4e3eeb615, 0x50507ec52c5f6bd7, - 0xe006b00db10a6e1d, 0x8a35b746b9776f3c, 0xae5a28316bcc99e5, 0x88e3ec3d55483f3d, - 0x3eb937761f714ed0, 0x0521ea3b43318b49, 0x47639db12c1562d8, 0x02eba2791aac7ce6, - 0xda4d524ad56e4661, 0x6bf13132ab8558ab, 0x8b7d0a43d914bdb1, 0x6652eb0c58775be7, - 0x60275dc415e86b10, 0xdec080cf9220521c, 0xd10f787a12539eba, 0xfe3cf879085ec78b, - 0xfc8676950158553a, 0xe0740582315d4e2c, 0x9f9429aaa4f6359f, 0xf93bf7d8efea289f, - 0xb7b66c63b0f6760a, 0x3a05c33cf07f8ad9, 0x46f19879c89cbb3e, 0xba1a1535fec0ceb3, - 0x4c4c32a9d19fb1af, 0x22371b04dc14860b, 0x9d4d6d4d45bd8544, 0x5e2055b1a81546bd, - 0x77e6f32884f4cc7c, 0xb1648731f6ce6989, 0x3c4416504f7a850e, 0xf17339eb208dfe43, - 0xc10ac0b0652d566d, 0x801d0c0db0db4d3f, 0x67b6be6cfd9697bb, 0x6570e30819eb99ea, - 0xcf888b694ac2084d, 0xec0a0e9b3c2f9f5c, 0x5eb152d48869b6f4, 0x8b909bced54d8146, - 0x78ebf06af63d9125, 0x524c42d3eb35b54e, 0xebf444e2ca6a6b19, 0x87e43654876dcb23, - 0x4a4f4f3194b47b68, 0x6f9b264d728e743c, 0x969a1e63691801ca, 0x29d9144449d04464, - 0x82d6f5c2c7e12651, 0xc3fe192d52781bf8, 0x48312b06917ed27b, 0xc9c056ba2ba0592f, - 0xf70ed7a55864fa1e, 0xc8b1354c79ce3df9, 0x37a7694d595135ef, 0xf8e2c49c0c77a6e0, - 0x7d4710ef6a6745ea, 0xae7854899c31df41, 0xf36a5a835573ea3f, 0x6b3441d18d143020, - 0x07209cae3b5aaee3, 0x33dcb09702b5785f, 0xc724e6f1d66e8e22, 0xc1f46ac4c8852f06, - 0x52439ccbb96d293a, 0xf787ce97919f42c6, 0x6a7e974b5bf8d41a, 0xb11f2a3053d6e41d, - 0x1655c18045b717c3, 0x0410d2dbc4db0e85, 0x10bd39d4404d90d4, 0xc55296723f907f91, - 0x7d6213a90727f867, 0xe6922d1bb7f406b9, 0x8646b1861158623c, 0x61798eab120547ca, - 0x2a0f0c9f26eb00a7, 0x9849da6425d2f644, 0x19a6e9bbf397a3fd, 0x0149a75d9da0d4da, - 0xa1788b609420f4a0, 0x8c46e8ac02eb7d0d, 0x16de65945fc50bf1, 0xb69f6bb14048b6ef, - 0x5ca7a0919d7ec266, 0xbeeb05019bd875ec, 0xadf6b65913e9c7fc, 0xd2ce2f1b5db204f4, - 0xe238406912bf2c3d, 0x9fcd9c4b91e5ecc5, 0x12129796fdd574e6, 0x9b78dad975ca7e13, - 0xa7be6610805eddc7, 0x2ea7f691c6c88d4f, 0x5a66b183391cc748, 0xb3e65537c324b8bf, - 0x974693c4d4f1dbac, 0xda9d4385532b06dd, 0x0c0c2e7c0da7f019, 0x71ddbf5d8e4c35e2, - 0xc87351a8ece53efc, 0xb66f26091bc8c7af, 0x1e1c17478802dd62, 0x0c5141a9244a0843, - 0x0d85c8e0b7e01815, 0x4c776ac5f801258c, 0xa9d9df96ada36a7f, 0x1439c03a067aa679, - 0x289e9102a30520f1, 0xb1afc31844b842ae, 0xd39425d4ed153d69, 0x82bdb8eba98081a3, - 0x62fb70d2dca1a9a0, 0x61d55787b4ce0069, 0x92fa2b089567232d, 0x493c675753a03fe2, - 0x52426e2801f74190, 0x6f824e851f8fed37, 0x9372c372c69cb2e4, 0x60f70c9ec3c852e6, - 0x9ee3cacf8d5b4c2b, 0xcecefbe7c00f8eea, 0x68e7774eeb1b36c5, 0x90540710ae8428ea, - 0xcc7a9f46f495c7d0, 0xd4289c5192d5e7ff, 0x86b7372849c2b18a, 0x64e9c06e3cc7b6b6, - 0x0b21182f4b218168, 0x9596bbac9d929fdb, 0x1d7a7cf3270e3ccb, 0xf24d27f6854fd9ba, - 0x23fb17683c743421, 0xdeaae8226f9512f0, 0x458d7a5d250cdab0, 0x9858e8a64170c589, - 0x6b3a1e28b47ffb05, 0x15a8c2ced16360eb, 0xb2757d4efde9bbf7, 0x27ca3f1cafeff321, - 0x3e4f3587732add52, 0x4fdf006f481b3f57, 0xee616b126d3fc494, 0x1eb155e313b028c9, - 0x645b4679b2ec1585, 0xb195f3c3d5457354, 0x3ce0d2f914f694fd, 0x3d94b056aad122fa, - 0x056d61e3b8d18d7c, 0xbf8d5b91bc123da5, 0xa49848baf6d7212f, 0xcbfac25657dc0cab, - 0x8533c05378ab7f3a, 0xb4bc46d2a55019a9, 0x8960c9c57886f6c6, 0x43202db8103d3050, - 0xb317fd83e76c5bc1, 0x97825381c692d394, 0xa1a167c3f7f04b5d, 0x19be84a49a463879, - 0x484f5fb5d2794fc8, 0x47fd97a3883a5849, 0x1b1d2533e25e8997, 0xd3e98fc3cc7067c3, - 0x23fca5fbf1e4c938, 0x0430d318d8a400f3, 0x99eb9798dea8a36e, 0x879b9ae16531aedf, - 0xd566f23dc0cb27c8, 0x5afadce88d6eca8a, 0xe76247299a02984a, 0x264ea2f4eb4191f3, - 0x0f3f589d11d2a760, 0x89d11822c16c1591, 0x0fab106f59daeece, 0x1a706f952bd35207, - 0x1c06cb35f05ba264, 0x18ec9cf6a0a14818, 0x7bfe55dfb9b9d3a6, 0x12fa1fa930f80864, - 0x86173b648134a51d, 0x541c9cddacde91fd, 0x8853409833784363, 0xed412d19bedf9f1a, - 0xf2543ef9f980e929, 0x7cd02bed8cdfa416, 0xd5d19b79ff350acd, 0xfb51cf3d38c506b3, - 0x63f071f34359c75a, 0xf0c9601537283015, 0x82bbfa98eb5258c6, 0x1cd6a004580c3e08, - 0x0a404f149012240a, 0x390e4a043a5cefd7, 0x9672a29dad946085, 0x6364d767bb0d2a49, - 0xf74fb9a4a8df90a0, 0xcf9d8c3c25e1b6e2, 0xb82869928052b70b, 0xa406d152892662b3, - 0x5fcb892ed4199179, 0x509e7133f78d741e, 0x0ec7270858f825f1, 0xfe048744df2c06f3, - 0x057bbf6f0dd184d0, 0x8bd04bc333ac97d8, 0xbc5dfc613b2cbe0d, 0x3d2d9eb53bc78cb0, - 0x73ff26c278a51d2c, 0x07fcdbc6f763ae39, 0x189fe5193d589c6c, 0x11a4df75972c470d, - 0x0a4036f2de416123, 0xdc263dd70926f870, 0x448b3ba744c0cfe1, 0xeb49071980023db7, - 0x95737fdeb0f420cd, 0x750aab47237f3026, 0x8aca80174399a1fc, 0x19d730a05c33d3b6, - 0xac431d0064c8ff3d, 0x5b0653acc6050788, 0xeaa9af6f454e5383, 0xdd6a1f2a2beb13e3, - 0xa2e37e6153485663, 0x3de114e49a1a2ffd, 0xe0c574c22f05de5e, 0x697069d335fd6e07, - 0x772ce2f43ce3f89f, 0xfd3d6b2ecb2441ef, 0x7699aeb825fe54b3, 0x9aefd297c53b8015, - 0x2f3c312bc7f4b412, 0x23dbadef44f81a30, 0x47c4f4177ec4699f, 0x8bd58c7c36d6ed10, - 0x8969a84245ca0721, 0xe23ffa03e26d3719, 0x3c1c525bec5ee938, 0xac75267b3a3cee74, - 0xa19504bbd4bc31f3, 0x6d4247c373eaff78, 0xdc7aa971520357bb, 0x56898824cd5c9ed0, - 0x6cf6d0911e33f7cd, 0x3cf22d1c6f6dd933, 0x8f0ecd76e4964f3d, 0xceb558a39331a32f, - 0x80d3c2c59d7c714d, 0x848e1cc82bf69421, 0x2ac66fab51a0c235, 0x00f2ee7a73efceb3, - 0x4ddda78cb8b9eb8d, 0xf8993ab13ac4c555, 0xbdbba817fc0c67b6, 0x10682d84dd401197, - 0x00591d27b16e82c8, 0xa2abd332545aff96, 0x4662e16afc17966d, 0x967f2fcc88a8780e, - 0xfb94e72f94e37584, 0x4999cae7c1bfd0cf, 0x9b0529980c4f60c9, 0xf379adad23875b83, - 0xf4f93a1a90254c8b, 0x64dde87b0c86e213, 0x995c109a35e78d47, 0x5832984c3b2d05fa, - 0xf34776ccdf3f5758, 0x4f47895d3b9f4bac, 0x4992f5c3b3feeecd, 0x6c9e9a529a17f0e1, - 0xaa3c6bb83427261a, 0x412d42eec1c2eaaf, 0xe982b25a3c9ff0d9, 0x39e2cda6088c4499, - 0xd8d92b0e4ecada0d, 0xc721f579e91d5c63, 0x009ecec67e0eb399, 0xf10cf96d915fc0ad, - 0x597b338cb7b3e5d6, 0x8d6f40582957a2df, 0xb5e7029d7f6be5c2, 0x0e959da6aba2b9ba, - 0xe16975d343bccfc4, 0x5548f52383ccd02e, 0x6d7b1f7de3b3abbd, 0x0d9bbd9144915048, - 0xd803f904d44761c2, 0x3c8b1c6b2fae3cd7, 0x97d6caed61bdc77c, 0xb116f85b8bd8290e, - 0xe6248aa330187251, 0x36d64a9771824fb9, 0x6bd5c57cd10d6a19, 0xf5c97ba000d171cc, - 0xcaf422ccd00cd215, 0x90a6d701a5b16449, 0xaa8ec0290e4e16c0, 0xd0c8eb8429c5ff27, - 0xa8a52ce48ab19f75, 0x9a51bf7cd09c4b4c, 0x269f3b2a1d7606d2, 0xc8f1ff20e931594b, - 0xe8ea90c4596aed98, 0x098eb7f5d52def56, 0x9770178ff0976786, 0xc3d9af42a5a27bf3, - 0xfe8d29c1d884c898, 0xc9b8e9f067976cf7, 0x4c11492a10797016, 0x98d031ca66acccce, - 0xdb91da8176b3e331, 0x93d2dac7abb6a778, 0xbaae4d13a6d671dd, 0x9f5dc711451c1957, - 0xfeb01dde69dc884e, 0xb99321b6d9fca684, 0x0713e6be6d0bb2f6, 0xbbf396226ebe9c60, - 0xb20dd8a252bdd2ff, 0x120c7f806c9f1cf0, 0x85ffb423bb2b5ded, 0x3986b5aa974cd543, - 0xc8e99a9b2da0a0d0, 0xdf76afc5068ea879, 0xd8bc5a9f2589cd5b, 0x4424e7d368da66ad, - 0x53f8f345f87d82ff, 0xbcb767220c91bb87, 0x2d4d872cea95d9ae, 0x7e6a40cdf5d087bb, - 0x8db504dbe6e58280, 0x56195a98ac172805, 0xe48e2eb2f8048658, 0x6921958bc72ecc22, - 0x35c084366dc3fce2, 0x8bba65485bd94c33, 0xa39c0b1df7c7024e, 0x53c8ec87e8899e7a, - 0xabc78d1ec32222bb, 0x6b114e0aec857542, 0x72d3b5a7851a7993, 0xa01dc576208f2cfb, - 0x476915c473db3dab, 0x4dfb5548e9249f39, 0xcd9ee84a47d184b2, 0x17cabe30a776eae5, - 0x4befaaf414c9551c, 0x031ab4022e44db4c, 0x916e5ec186f0857e, 0x9dfc2372dcd1171c, - 0x40eda17991376373, 0xf34691c16ffd6559, 0x66b629e024ea5db6, 0x280e7be4b8f62bcc, - 0xb5d9884b48c02127, 0x2de021745f40fe79, 0xb45901c7840ebb9c, 0xb53280372d6bc2c2, - 0xcfc53a1266ec7244, 0x39cc7b12a9ddf8e7, 0x58649ed4be9ea050, 0x12d7ab4a8198cf44, - 0x6973f7c5c6ef64e0, 0x1368f3da6bdc472c, 0x654e95ca90f05dba, 0x8f979d8ae22841b0, - 0xc56ab715731272b4, 0x9adc5ead710dfafe, 0x682e8399214c88e3, 0xb325d13468c69042, - 0x758d9f23566f9bb5, 0x2d0993bb4f5d1994, 0x7df3fe051d6edf30, 0x9ff4ad6e06ab4553, - 0x7dfe899f561d29bd, 0xc1054498e1e5e6c4, 0xfc71ab80774d7d82, 0xe579bb4912dc0ced, - 0xdb2fb117ff86a65a, 0x1617621c014e3f26, 0x4a69e84d9fda8e66, 0x493055694f623c5d, - 0xd84607bb04066cc3, 0x22dffad0636cdd37, 0x12ca7f1200ce7ba9, 0xde56cc168b7f3d75, - 0x9a6dc5d68643e0a7, 0x1c443203d186ab2b, 0xaaef9ca1515cf8b2, 0xf1c6bf0d044a7d32, - 0xe04a3aaf4bc12934, 0xd1004d056a4ff84e, 0xf252eb32ab81f2c9, 0x601fcd963d062a73, - 0x6d9d8cb8746f6f92, 0x04aa49471adccc4d, 0x2848099ec7cbcb9c, 0x8b16eff76553b211, - 0x52b9dd61d1c38466, 0x26c0b08f1d0227ae, 0xee7abfdae21da540, 0xe0f3192cb020bace, - 0x0743f8c95d38d277, 0xfb802bcefcafecd6, 0x0a2a0ca5ff49099c, 0xe40a08f3e0bcc317, - 0x98bd7602dea8cfde, 0x181d54e11bca6b75, 0xc5a39b67104a3af5, 0x90982592e10be0d0, - 0xbe06bd7e72a8666d, 0x8778679344febf08, 0x42c137760a75cddb, 0x04b7850ea632f095, - 0x0d3c7772abeab924, 0xfa06cc90492c35cc, 0xd899b5aa1b634544, 0x7c84e27e9442217a, - 0xba3322a38eb86971, 0xe322aada108f745c, 0x21617e1c051aafd7, 0xd7adb59d40b532cb, - 0x6643601da79f8c3a, 0x7f78fe8fb3e1b110, 0x88f8d6c76139cba4, 0x051c706c0e5d6695, - 0xeeacc64c5be615b7, 0x5c27cbb0b2e60b9e, 0xb210e47c3c0dc4b5, 0xe68c49831ecb525a, - 0x3777d04d80c788cb, 0xd3c67d5a07e6590e, 0xc0b6161863747a84, 0x53002902fcdb927a, - 0x7081ff57df0267ba, 0x043a177490275cf2, 0x6621eb121f577d49, 0x844435827a9852da, - 0xd6e869b6c620ad63, 0x6220aa25fc0e4973, 0xccfa27c9bbcdbfb7, 0x73421b013ce76fed, - 0x9f8485080428c4f2, 0x59e90f79faf6329a, 0x9f61511c9a83cb76, 0x7fbec6bf0816863c, - 0xa0db5359ac781e07, 0xdb4a943960cbac1f, 0xb5a7f15ab81c436d, 0x9e415d395dc7ca18, - 0x9679b91768db92c4, 0x6a9e64071a0607fe, 0x880a6f8c7d41f8a9, 0x07f137c2d62a5e26, - 0xc9fdbec73b54ccd5, 0x8b430f607e612d2c, 0xc03744a51c5db831, 0xe7151522840e42c0, - 0x854520897e2ff7ba, 0x5890a1f9a6ee3587, 0x53ede4885dacaa0a, 0xdffac7398a97e25d, - 0x1d99a0592f5bbf4a, 0x5198be0f80d2f512, 0x372feadb269aa79d, 0xec7a9e2824e09dc0, - 0x8f5d8b96588babe6, 0x7803c8bbc39dfba1, 0xf7c3c4ebcdbcf077, 0x7c0ff8a9eecdb3db, - 0x71fdec2b6d34cfd9, 0x166557a3116e44f3, 0x86865eac4971d4d7, 0x961abf7ebaab6822, - 0xb0ddefab749077bf, 0x7a15c1ea0398b9fa, 0xb82bfc6c2af133f8, 0x4ed11d58475c6d7f, - 0x6e9289a1ff83882a, 0x7162a7f2596d356d, 0xcb486a7a2282825e, 0xacf7bab1d0287509, - 0x03ddf0070ae707fa, 0xd00fe1074d175bf4, 0xe0cd02b04cf93ed8, 0x90db138c945c4dff, - 0x6201bb5c6201360b, 0xb508e18ad9436c32, 0xfafe06826d9cb12b, 0xc49569fc9d56e3cc, - 0x17f9ecc8c72e58b1, 0x5feb264a1f383741, 0xf93a263639383df9, 0xf5c8ff2e0e2ed807, - 0x9472359bb886b557, 0x8b3412066cada5a3, 0xb1f60b0025c9d86d, 0x20434439b032ce3a, - 0x933ce92422b299eb, 0x8816d115d26d6d06, 0xed6346e72cf204b8, 0xe23645eed0f84037, - 0xfd86bcf14cf45748, 0xc7bc1b953bd6a71c, 0xe8e83b5564926fa5, 0x46d0ef1ff564cc43, - 0xf293ecc10bbc960b, 0x729432a8bf180dd6, 0xe642b77ef52a449b, 0xd8baab4b085585f5, - 0xc2019a4920d65413, 0x4da3d59c9cc5fc63, 0x0ae4a1ee66ee11dc, 0x6cd20702647fbbb0, - 0x046252e9c1be9cd3, 0x2a9cb692cbff87f1, 0x5d13edcf919a738e, 0xf91a1b3d7a0852ea, - 0x889e3e4b357766df, 0xeeaa64780a091dd8, 0x54bbb5355a145d9d, 0x57f4e2a6a317a96d, - 0x64b9eb764b0a1b32, 0x6da0e1a1d752e0fe, 0x60d2e309c53d7260, 0xffd3541e0adaf6f2, - 0x9c93d8349db1b40f, 0x335ac00b1b54c676, 0x616516766c88f062, 0x1438b7b48dba49c4, - 0x55b7a6ba3c79ea56, 0xc77317577b714e92, 0x9acaac22095ad972, 0xfddf82596867b42e, - 0x4357316745aa8210, 0xff488b052b8c60ad, 0xfaeb36520895e718, 0xa20197062f5c96be, - 0x273b0dd1fe669f32, 0x2071b5962216f425, 0xa0e463b79e0092b3, 0x5427ecc31c7d200a, - 0xab827c7ae99bee39, 0xaf8cb5a8a26b47ed, 0x91c143284d1908ad, 0xfdfa3f02fad2c27b, - 0x5a46304cab5dd072, 0xf82bb55be1c7b864, 0x0f5438c56693f19f, 0xa6ff08eaf673b6cc, - 0x617eff54dcbce725, 0x356e97654a23f704, 0x6987e45a16bb9151, 0x4861abce800476fa, - 0xa04c84c016629e4f, 0x10eb51e9fc46163a, 0x64c3094bd29aa86c, 0x8f313d126d2f212f, - 0x46d64bdf5dcdbf30, 0x9651428cf144f140, 0x080b77612c44dca8, 0xa279e848969d8e9a, - 0x06872e7172b063f7, 0xaba536032bba613a, 0x64f141e4f3477687, 0xc2150ba6b4c4aacd, - 0x2392684d54242668, 0xff055b89220da899, 0x1f633c3a15da8c68, 0x2152a91d528107ca, - 0x4d93fcb17b2a98dd, 0x75076bd1012faf75, 0x63d2bffef401c11f, 0x34208a163b5a0316, - 0x5c95f2acc210dd4e, 0x7d5844d81bbf67c7, 0xdb2f118269251a00, 0xd36d4b28c933c960, - 0x26e41181c299cf98, 0x2b8e4dc69845a927, 0x0d1e68f53150dee3, 0x11838d8af186083d, - 0xb55a022c7b69f09f, 0xc5f3f8cefc1d5484, 0x5ec8156234b9dd9b, 0x0ed9e2df39cc097f, - 0x5459fc2dc0f701cf, 0xb80a271ed121264a, 0xfbe6cfdd215b6902, 0x8f0c44793089029b, - 0x6ea2105c64393c65, 0xa5607481ab348d4a, 0x0dc2e04c759a0585, 0xfa94243536deff9c, - 0xc22645d7b8e933aa, 0xad3659d222d45736, 0x15d9fcafe1f45845, 0x2a34069cd26480d8, - 0x1cc0759070b715d2, 0x5dc180f075a2f59b, 0xad6ab98985ac3539, 0xffa3d82cc30ef61f, - 0x15bee23e4eba9854, 0x910824491b076a96, 0xee17851290203708, 0xc7d913720711adfb, - 0xab93619bb50c9f32, 0x52e0e73bebbe126f, 0xebe0a4ad83137864, 0xc052cec7354ebdff, - 0x9edd2e0d9158bdea, 0x545e232c848b7d72, 0x544e402c077d5fbb, 0xfb540f7080e8b6f1, - 0xb84ecd3108d17aa7, 0x5075c34d601f458b, 0x4963cce92e716406, 0x152b438adbe1dd1e, - 0xc1f8163163176e26, 0x628ff06ec0a95b45, 0x4560a386d04c832c, 0x55440de4a8d1c17d, - 0xdaccf26635073d96, 0x5f1cf8cfc9196f97, 0x032d6f13d2767961, 0x168595dfd3132374, - 0xe651cccbe16c2ce4, 0xffe6877a39407587, 0xc9e3cb8cf4914b23, 0xa5381c415a766ced, - 0x791a9520fc3b5cfb, 0x71c32bef5f0dd413, 0x05bc8c89f185dc7a, 0xd13576b14e2d3d92, - 0x3b91ebc306475991, 0x701af90c7966b979, 0xb5e84b1f436ce0a5, 0xfcfd998822fb82f8, - 0x2c9170dc1c13a137, 0x319f467aa45e6def, 0xf36d78022c680f40, 0x9044dbd3a295e05a, - 0x3a9ab2fd56716fc9, 0xc729c5ee1fc78cf8, 0x35c177fd1ab170dc, 0xb6b8c312ce538ae2, - 0x35a66ff7542daf8b, 0x5f6a3d2435261aec, 0xede8cd7d28717aa8, 0xab356889f1266bb1, - 0xd095b01716ca519a, 0xbc5073a677e4a853, 0xd6ba234972a3bf68, 0xd74e77aebdf622d4, - 0x50cfc14a66b19e09, 0x63082fe99cda4876, 0x09b4fd89b22dc52c, 0x842ea1e2b46e84f1, - 0xf776a8ae38ea15ff, 0xb458969036357958, 0xbeb3f08f3ad39c18, 0x7e441baa45687f4e, - 0xef8e9561b141df79, 0x51f1ea482103fd9d, 0x1701f914c5645ecd, 0xe5030e3d24f4af36, - 0xb898115b10d9f42e, 0xfba3c47b005eb12d, 0xc3407a4d45199512, 0x755a4efbbe95b8d4, - 0xc80df759e4761f21, 0x47e5e61d774ffd2c, 0x053729f38ae7577e, 0xba7b9e33de0c4566, - 0x7c144707dbcb0e0a, 0x1aaa46ac7a8bdfa9, 0x2589fc07a0b64c39, 0x3d7fdccbe4f8fe3b, - 0x33b4bd8c1a018515, 0x61dc8da5d8b7e66e, 0xd898bfb6afcca781, 0x3450d7c98026bdf0, - 0x3b0384f7a81fd0c5, 0x0a20296e97484b95, 0x39598dfd67b0472c, 0x49b0131fb54f24dc, - 0x1b66f7839669f7f5, 0xa2bd09cfb243c376, 0xf36207b52ec4824c, 0xb7578827c8750ecb, - 0xe917bd040e7c7fa5, 0x08baf34706d3ad7e, 0x913f9e916e7bba86, 0x0e4b353c2ee53376, - 0xb08972c457f18ebf, 0x8551ad7aeb0a2edc, 0x419d5ad82efb4b7a, 0x36c93ca62dd02fd5, - 0xc86798799972f164, 0x6db8e39b5a108654, 0x8be5f492da6eb5a1, 0x71e842d23f36bc55, - 0x04e94eb2d0810779, 0x7d6676554aecf981, 0x7cb74ff7bc010216, 0x6b6b218fdcff77b0, - 0x39aa991c6ae9e163, 0xbd153e0aed086903, 0x7d9b898051e358bc, 0x05b30f5d8480ce63, - 0xc8428dd45a80a1ff, 0xd27ed65f608c009a, 0x63a1c865c3118bae, 0x20f7acab9dc8eadb, - 0xc88fa87d29a249f6, 0xd4e591537708c2e5, 0x519c071da0cdffa3, 0x93b78a4ac9b7563e, - 0x6428b449c9070251, 0x28ffbbdd1d9db43e, 0x50947d97ba4320ec, 0x0f184d0c306858a7, - 0xff94f8f35a1586dd, 0x34acc6e206aa4860, 0x39dc1cc1fcf417f5, 0x085af7e502ff1fbc, - 0xddb8c54f41f2045a, 0xe343345e3ce33b8b, 0x996a5c697f68deff, 0x01ddc3bbb092db10, - 0x140f7cc7b0b1714f, 0xbd14c3d073b704af, 0x7925f4a7086558c0, 0xc48aa8865f5e8d4b, - 0xbffd568d8624a33a, 0x4cff6a435f92fdf6, 0x95fa56aed1001a8d, 0xa3c97016170f16f8, - 0xf3c81155a160c5aa, 0x3334d22ba61c4477, 0xfb9888ceece64eb6, 0x198b0bf4da3de4cb, - 0x6251d076d3129716, 0xc03c84b2562561e7, 0x7b375fca53825da9, 0x52a8cc16cb8d01e7, - 0x1b364eae2fed1e0c, 0x834ec87832127220, 0x95afbba2a2caa926, 0xd0fc8c6b810402b4, - 0xa43035a643602510, 0x354dbfaa190f71fa, 0x9afe1b556308f12a, 0x46c29ea2883895d9, - 0x4b6e1d42eb836307, 0x303476b422b798a4, 0x9cb51d2eb38c579e, 0xac8582b7550ba59d, - 0x63d5df574c1365a4, 0x1e446e8c4f7e3801, 0xc18f94bddee2092f, 0x5e4076292f693a41, - 0x9d9d6cc24f3a9866, 0x41cc84ce86b60707, 0x404b88d16f175222, 0x8251afb8cbaabedf, - 0x6f7e27755bcd1180, 0xf67a8f7770a4a6b8, 0xc291ca0add72fd6c, 0x6cb285de74b4768d, - 0x1274f094551a799c, 0x3f13dada535aa835, 0xc6d5ab45ff84909b, 0x31500a87211c204b, - 0xf970a3b85851f629, 0xa3636cb1d6ace4c4, 0x0fa72345021d3e75, 0x771ae73703475498, - 0x2ae644c30d7ab882, 0xee528ba92347c105, 0x9aad23d1b663d8e9, 0x761bc50ccd50440a, - 0x05b528d0b47d19db, 0x59fb7fd9c792bea0, 0xe9cee2d0a3d5dbb1, 0x586f0e6cc71b96fd, - 0x5b52baa2a3188661, 0xfb8b6ab325556dbd, 0xa9adfe73dc0962c0, 0x905a21f9c782597c, - 0x5b50d3779ae80640, 0xe650b77731009357, 0x49ecfd55f47d87a7, 0xb66f166bc3d78c80, - 0xc0afc3578fc32b11, 0xa6cd8cb23583229f, 0x44458c8444535edb, 0xb5d6c94570f072fa, - 0xbbc3ab9160974def, 0x33208248430d8494, 0xb057ee9b4e660300, 0x697cd4c0cecc5d94, - 0xa0416cf1c4ffae0b, 0x46f8f803edaf63da, 0x3f813712c9134718, 0x8515f28c818ac523, - 0xcaa251396a117942, 0x4920bbb45f67bf8f, 0x61b7b684ffd1395c, 0xe3a5d90a026194d4, - 0xe891c7cc404ee80f, 0x0901cf0718e84e38, 0xef1ada01d9d06815, 0x0693e25e4a2aadfd, - 0x65633e0d7500d474, 0x3d9e595326784697, 0x2327758056be78b3, 0x2b328d8432540701, - 0x009c721ea7a97bab, 0x6a48dd96b7f63b32, 0x5e6c429198ad01e6, 0x61e6a996433d3d25, - 0x06d1b4ff459b87a6, 0x4a17b114519a1632, 0x5922f46e5d5315ab, 0x7919fd3ef80ebe22, - 0xfd3248f90c99c984, 0xca0f31142c83b4ca, 0x72fdece85aad2221, 0xc3d5655b64bd54eb, - 0x65c4897d74d599cf, 0xd8c86c2a6a9624bc, 0x885010b9644f4408, 0x5c3840e7dec1ca07, - 0x3174d6db559e38a8, 0xb0256c555b40f8e5, 0xd2c7eac96947ecb3, 0x78ee146d5b56b282, - 0x3bba5babd2723a8d, 0x81a8e7e398388781, 0x2996535034cd151a, 0x961b3f3f5412ae8f, - 0xa3814d33381ef16a, 0xedaad662b82ac4dc, 0x6b925224c0f0a61a, 0x0a2b706cccb663ee, - 0xf06bdd4f0b052caa, 0x4bf353d2a6350a97, 0x02bcd33a866d9899, 0x20ae090e4b7d3173, - 0xff5155189b326025, 0xbc3229d5903e4657, 0xcc37ce7209d5db85, 0x73e88df224c36785, - 0x5bd0a0e01864dd25, 0xf607dc98b37442fd, 0x1dff97fae8e4404f, 0x81c70b6d9db0b322, - 0x00d9134c6ffc8b9e, 0xe4eb944fce9b875f, 0x32808a4657fad9dc, 0x5f71996376c959a8, - 0x7b17693ff3b5e780, 0x51ba2211a419c020, 0xdbb909fa40439dbd, 0xf8c9f446f75f0e8b, - 0xdb803df36516465b, 0x5d5ac9bf102f979b, 0xa02325776de37ab0, 0xb4f5520942fd5f42, - 0x2bb61b22a24f0e4b, 0x341f45f2d3122dc4, 0xb30ea44e6438755a, 0x49388a4980678ced, - 0xc33958744fec3a60, 0xd4660b8ad08ce771, 0x1972c91dd3567f34, 0x63492ce44a13a450, - 0x09742cacc3ef32ed, 0x84b2287dbdde1666, 0xae7a3711b52d0f42, 0x2920bcf98359441e, - 0x186018f6eab2a8d1, 0xd2915d2f8abe974b, 0xfb46a65bd08271a4, 0xc677bc1151b29dcb, - 0x40e3e362912bdcbf, 0xa45fdcfba95d613c, 0x4270d1bb365ee95f, 0xc16bfb0a66b65514, - 0x6af3694273aa7276, 0x8d984ee80d7b769f, 0xb2635235c694a035, 0x51671d63cd2459c4, - 0xe6c1504bd405de58, 0x74921f519222c49d, 0x5cb88d62fa7e1ab0, 0xbb8083e455f2f4e7, - 0x45151d668e9b8501, 0x320b255c9b61a105, 0xdd559ca7ea0ad55b, 0x34eef4162777c072, - 0x2d96a767e5be3cb4, 0x93f0a8c18d68735a, 0x92f13af222db468f, 0x5ac52e54eb60e548, - 0x33c4ac2bc2e1552a, 0xb4be3572c7c113fb, 0xc10ccaf1b077745b, 0x4207bf19fdd10eb2, - 0x3ae711936662ad6f, 0xa6944e9d12e195e2, 0x02bc1105c987a053, 0x391d5b42cac78e27, - 0xb6bde226a8ab169f, 0xa7e57c5541a9e6de, 0x793f979da9826a40, 0x0c2732011360f20c, - 0xc34415631eb5d360, 0x8bc5ee0230b3288f, 0xed90c0881de5b236, 0xad3c72d76342c3d8, - 0x06de1864cf1baf3c, 0x9836aa45861b183b, 0xb6a90970dd65e50e, 0xc03e55bf57fe6913, - 0x6d3ea5cf74e5cb89, 0x7b78163c5d4fc463, 0xe148ab8dd5d52eb6, 0x313edfe77dc71927, - 0x8030de85fa41a4b9, 0x8d1295d4b2dc2515, 0xebafe202d41c143c, 0x4a364c36aba048b9, - 0xc772de2572eb8f1b, 0x7ba918c139683c0f, 0x6c8d04fb8a3e9a9b, 0x6a787cf9258b749b, - 0x4d5c74e35bf634f3, 0x64800cf884364dc6, 0xe560f2118c9bb646, 0xcfa0c2a0a116173e, - 0xc5ec3923387c4994, 0x2d1facc538efdeb7, 0xdaff44f59d3f530e, 0x6193c962d40d8caf, - 0x3e63dfb476920052, 0xdfb354de97719a8d, 0xa1757d4f20ab1640, 0x902c9c6e6cc6a8af, - 0xac5c8bb08614421e, 0xc66285953ec95022, 0x025235d3b7417059, 0x623223b9a631c70f, - 0x7d4b2a3d19398efc, 0x3812298ecfe14eb2, 0xa0066ca00926e97a, 0x6284ade8cbe633ce, - 0x8d60a91d5aa91cfa, 0xf77973124dc532ca, 0xec228dad9d5eda7b, 0x4df3d692b95f83e3, - 0x63e72f390c8510e4, 0xdb1723f12927cbb1, 0x032b6431c12c6900, 0x2f34af1ba9624820, - 0x08afec13156c0341, 0x7e8918de80649a08, 0x40cb8e710f454b0b, 0x2adc124d60644bda, - 0xf72135f1b6b1cc9e, 0x9f4078a17f43d432, 0xa6279e85b370326d, 0xbb9e3004e6b5167f, - 0x87605a6772b0da4f, 0xa2616ce75bfca97e, 0xc5ec058e89b86b36, 0x3a92300cd299b629, - 0x0b5206628344b14e, 0x7f91a60d27308938, 0x3daecec1d891f9c4, 0x9f49c05d5f64dad6, - 0x93337e5f1caac6d1, 0x0bb40238c1bccf4f, 0xd8ab7f53c16ae336, 0x3357a0b9da9a85bc, - 0xcdd3846346650e16, 0x32d115ac58b8ed9f, 0xbe20f197313148e3, 0x382a6fe68b3a3b1f, - 0x1249de8024bd607e, 0x41f2c50284b8fbbe, 0xa105a4f494e4b4b9, 0xad8b0c8d9128c599, - 0x96c68c77d4bc0093, 0x09ca4ec166d6ae62, 0x6b6d1e18b3181c00, 0x8ad1b7b621a2fcf0, - 0xf8412f3bd851d24d, 0x8c9c2d1104b28f0f, 0x44adc8070a94be46, 0x05a5bebea59839b5, - 0x322b7deaddba32e1, 0x173ae323b060515a, 0x297a5c234e021eef, 0xacbc3611d47c6466, - 0xab59caa33317d2d0, 0xd0c2f627f89a6090, 0x026b2a1492531a33, 0xbc56e3c8b41dd004, - 0x6344b023b9d66c08, 0x9d7753b5f234b5f5, 0xfccb753d9cbdcf4a, 0x0f627ff682e41bab, - 0xe4e6392450ee4536, 0x974c5123f2a03706, 0x8bfa5bc8a7336c4c, 0xe341903389f8e756, - 0x5dbfcfda5572a102, 0x0996213240cc196b, 0x7872c8c268bf6508, 0xf7da9d899b88181b, - 0xa3a1e4c0ea4820e7, 0xccf0b52517cd994d, 0x5dc305d4a0d0cfbe, 0xfcbbd98b147fa46e, - 0xc59f8b29dfcc068a, 0x34115b975758f270, 0x2446e43acd2296ae, 0x11d106625b13164a, - 0xde7ab19f5e0839e6, 0xc5780f4579d1b9d2, 0xaaad0e63634b0673, 0x039df84f68de43d8, - 0xf1d66f944a893afe, 0xafa2109dceee753f, 0xa5a67af7fcd4a2c2, 0xca31ae7449ff36d7, - 0x78c1d76084463259, 0xde7c8a29cc9f3a04, 0x30ed6b9d28af4fb9, 0x62df82db1faeae1a, - 0xdf9a60f1fab60989, 0x2da3f791892b6536, 0x80bb450d424891aa, 0xcde3c3efa07a9f7b, - 0x9d6fff64d49c069d, 0xdef5ba81e77897bb, 0xe21d24ab76f82b90, 0x1c36f4860adc0293, - 0x2ad5cfe6704282e9, 0x692f873e1e9fb3a6, 0x732c363af56a06fe, 0x39ed3fcdaf01daa3, - 0xe041b35e855212ab, 0xdc746d5ea6bede79, 0x91e64c3cac9b1f7c, 0x2db0d4e3b83b4e27, - 0xae9a2420e43e2b1a, 0xa42e488d354bd011, 0xb8ef32e364709eb4, 0xa702983b2a70282b, - 0xe10f821759269aa9, 0x35d8496186e9cb93, 0x92426b4c47bbe851, 0x9bb27db1d18c72ba, - 0x08a1b1e4b234817a, 0x3ca40f8687493c15, 0x0bcb23b2d2bec7ed, 0xc36f9b81133bf276, - 0x3761ed796dbff31b, 0x06e093b27b16f0c1, 0x33d2dd92a0b68045, 0xeb589a61a6a086ef, - 0xe728eb240b3dfb98, 0x6fe904ddb72b5fa5, 0x1996290dc158a0e5, 0x3964f7ffaa3f3350, - 0xed7af5a73bf87da5, 0x2470421be6f82151, 0xbd004640e9a83e15, 0xf739e86e89920583, - 0x92e8d07e0375f3bf, 0xd2245da82ecca8d5, 0x74305486804c08af, 0xa8cddfad39aba17b, - 0x6c0d47448174734c, 0x379db8d5c4b25552, 0x45baa48f033ab5b3, 0x769584cf1644d5f2, - 0x8e4ac24e353acd60, 0x686e520926a1fe88, 0x59215612d838e715, 0x7ed026a62e6a0add, - 0x1fdcac8bc892b921, 0x9c27ab3f47981e3c, 0x1b90df6def4b8ff2, 0x063fd55105716d4a, - 0x9441164376b35ae1, 0x8bf705f9d70f7370, 0x1085dd0b3fd1d960, 0x308b35b29f535069, - 0x6ba2fd7616f8bd52, 0x489a6cc785c6b4b1, 0x12b38374796763ad, 0xc806e42a0fceef0e, - 0x42fd2f1739fd8540, 0x5a1db40c7f427894, 0x4057db8a1b992fa9, 0x1a01fdf24532de16, - 0x01f82a12929604f8, 0xe6145432c0bd380e, 0xfd53df2a8ad84115, 0x793683b9ba2c3efc, - 0xcf6e2b77fff5d528, 0xca8e7a2781af7203, 0x13f8b8c7cdde11ee, 0xd0eee90f372f6d34, - 0xf9477e0d18efc3c4, 0x55348a428b9f753a, 0x79ba454ba030db31, 0x3898a7ca933c5663, - 0xeec85533074803b9, 0x511dc788bda99e63, 0x01bd86f9b139362b, 0x07f55df507e3c79d, - 0xf4261fc6102f7601, 0x42165109dcd8d54e, 0x78e6acf5abc62c58, 0xc770ce401d243630, - 0x671321fb1f60a2a7, 0xb5191a1938a91ea6, 0x06c010a7c390ffa3, 0x6759d0f60d2a0c8e, - 0x5d6c089fce483892, 0xdd4fc558ab2d7579, 0xdaf9a23b7792d777, 0xcbeb4e159450dcbf, - 0x57d7fbc13f138747, 0x9b34072b9e346d57, 0x9ff3f24b2af02a25, 0xe371e95e3a3ce4bf, - 0x054762efebd2967b, 0xd21043fba73cd7e5, 0x05406e9ed7d11663, 0xdfbd9d419f692af4, - 0xc28d85f200b3f78f, 0x4a3e6a4efa8de514, 0x1e4928341d014968, 0x173082e207cc749e, - 0x71143a655f24b6c1, 0x5a563894bcf3ef9c, 0xc0d77d9ac032cd9c, 0x61997f46188ad284, - 0x737cebb678533d30, 0x384d8ac86e9b7627, 0xf312d6733d10ff12, 0x25d10e508a8afc64, - 0x702df6253fbc6a73, 0x33415da46f2058bb, 0x358ae90ea23e2149, 0x905d301d360451a4, - 0x5d2f490eabde671c, 0xa40911a3265e9330, 0xdf87f10ac0ca22fc, 0x379a032402ebbc89, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x21ac3df849be2a1f, 0x11006e9fc51d112f, 0x9151aa584775c857, 0x5159d218ba04a8d9, - 0x98b7d1a925fd1866, 0x8f4753cafc2ad9d8, 0x8eb91ec1569c05a9, 0x4abbd1ae27e13f11, - 0xf099e607dbea8b56, 0x45384e961066aade, 0xe812ce3a6e619c13, 0x4ddb9dbb5aef9ba2, - 0x306430fa89d1e30a, 0x36c524282680bef0, 0x9ad0572140eac595, 0x81388541730ed3ca, - 0x599284607e63a320, 0xb17054f5dc39f1db, 0xa0017494131fa68d, 0x8438f43b78f5607c, - 0x95671dc71eb2d95a, 0x984865d97dc93cdf, 0x72f88b417b192b32, 0x5b912b480b982393, - 0xfce040e21ca55c33, 0xab6f10b872047a34, 0x43f589a2592ab80a, 0x9264ac8f97590f12, - 0xec44df1abadf2712, 0x6b80ff17d90f3dd9, 0x138d0c147946cc5c, 0xbdf858ef996cddc9, - 0x3f2f7f3811a4468f, 0x0bfbd8a915460a49, 0xd2155b5e930a0c15, 0x16a8deb827a1b15c, - 0x386a8078861228bb, 0xf969ccef8ae31064, 0xd4268ac23398213a, 0xf5ca6571b5ba306f, - 0x5c4e46c6e4873662, 0x2d6e3a54a2e97a8e, 0x7871bf3fa82e4677, 0x6ccf55fd51d27779, - 0x9101056d3018cf94, 0x0ecb1b58431ead27, 0x57240d7cad2abeac, 0xe311e77a16bfc4cf, - 0x3881910ed7de30ed, 0xcf3c4a226111418a, 0x83d9eeed2cb367f2, 0xcfa71e64af69ecea, - 0xd0287a51b44632a2, 0x85a4a266db33c8ab, 0x40216ed10dcaa137, 0x3a4eff89f90bd880, - 0x79a279de0bea72dd, 0xa6428b200ea515f7, 0xdba42e51081cb40a, 0x5d9054b9d413c536, - 0x388989d5cb47917a, 0x7c658a06055b1956, 0x8c6d5283990b8605, 0xd9a8060f12b6aeac, - 0xb0f2872d38c0bfad, 0x8e6f680d01c817cc, 0x4eb0f3eb37f1f35b, 0x5eaba40975eaf6c5, - 0x924cce1a72157b76, 0x43333f09ca6fbc3c, 0x8fd90c3acf4ea4bd, 0x51b5c74a6d522b6d, - 0x3199666f7304b16d, 0x486801ffb471b6e4, 0x0309e4180b375750, 0x5818a3e192b6e8da, - 0x23d0d3d6a6533245, 0x65a9785b77cc1dad, 0xd7d3da1059a9e09f, 0x7c0cb95463de4933, - 0xfcc4bbeba9ef2e64, 0xacc9b3747bef80c8, 0x91181da581d35a64, 0x1e48a1c42fb5f0a1, - 0x4228fc1ca178598d, 0x3a591a8e915da1a2, 0x344a3737275d25ab, 0x96689e13e0760278, - 0x06c26ecc4c97412e, 0xbbb5728183665b4e, 0xdbb7708c08effea6, 0xd1114a66b4185674, - 0xca4dfbc08b39b9f4, 0xf207bc96b40a9c13, 0xb184c8725f7f5b41, 0xbc6401524c7eb3f6, - 0x682dcd9ce36cf20b, 0xceac96bd3ed88b12, 0x015469f41f8d602b, 0x98c7a7e74459d421, - 0x37767be63030817d, 0xe4291d84f86ae3a5, 0xf2a4a22e323e55c2, 0x7c842d953bd38911, - 0x2841414a33c27393, 0x72d07a561a685fc5, 0x494783d68e30ceae, 0xdc58d9589104349c, - 0x296b5a7e240c1970, 0x1a93083f79534bf9, 0x766b9784fecda8ec, 0x1c512440b2bbac11, - 0xdd4b2df21d13f31b, 0x6e18149022b22c44, 0xca6b7937b7493478, 0x30391aa6c1b7437e, - 0x69b3e0cd8b3e0f68, 0x1ec280fc0eecf43d, 0x966f9bb34fcfe237, 0xd6bdd94aa249a0b3, - 0x4092088842f3ac99, 0xe3ecc9acae294025, 0x5bee2a0bc7fa0105, 0xc3615fcd71c08a10, - 0xd892e421a97d2bfe, 0xfadd8fb71f27510f, 0x67795f6db4b4355d, 0x402c9b3764f2feca, - 0x95cc76beef54455f, 0x8d7a14304ae208eb, 0xce81941ca5abbbc8, 0xfe571f6d3a3e715c, - 0x577353c3145840a3, 0x9277cb2f9c563b2f, 0x82e913fc88da4d23, 0x6c35b3688ddd30e2, - 0xfdaaf702046e057b, 0x34a5aec17dbf58bd, 0xefa12a78c1577177, 0xe7fb8cbc58b75a90, - 0x30de2f053f17c411, 0xcba1cf8cd568c856, 0xec2225771b9b9a66, 0xd1b0740a6e8c9962, - 0xe968d62b39c33a94, 0x2e860c2e752d0ae9, 0xe20b336e84c6657f, 0x12c2b5641a6afba4, - 0x7107f1b48f664c72, 0x53b8c8d4757803be, 0x942338c07a4cdefc, 0xee51af29af3a2b1f, - 0x6fb07d2c46a28941, 0xb3c22ada0fa6d123, 0x9da280eb103340cc, 0xb5fef6f2f1c9b7eb, - 0x71c947b678343e0b, 0x18ea992c5d3eb871, 0x313dfc4d8bea3313, 0x1bc5ffcb0090251d, - 0xa961c4452d1dcb71, 0xb1d123c52db37183, 0x196b6f1b4b39e1fe, 0x7ee3b00d208e9f69, - 0x65ed143b7234c8b4, 0xff30230550eb65cb, 0xc4e8de83f64fde99, 0x7f5bb3dfe5fbf92e, - 0x59b73583a5c0f665, 0xd70866ac91ff1cd3, 0xcc3dd1169f7b1f9c, 0x25f51ac247c2b225, - 0xf048dafb6271fd48, 0xec7899fb6ca75dcf, 0x93e3bfc7ac0edbfc, 0xf7dd0b07ef3f5b1f, - 0x81ce8aa5f438915d, 0x688e93ca31282cbb, 0x95c48ae1f2a00226, 0xd2d4bf5c437b4a5b, - 0xdd0c94f0044df901, 0x1e2f50a512a82ddc, 0x5697942f4803adc1, 0x31a944b3e8cdaa52, - 0xec8a74383680cdb1, 0xd2d24c9b34d49d65, 0x78f909eb185c1379, 0x15abf0ad46e065ab, - 0xe33cbca8b2f27747, 0x6fc28650078d57da, 0x1ed7d995572410ea, 0x59b0d11327301079, - 0xe92d296089e739aa, 0x4eb8637d6ae414ca, 0x7e1b3244e5a19d44, 0x44afe2f0a4c09dac, - 0xb6d2359c23e71cbe, 0xfc8ca2a6fe9d4fc7, 0x5400eb9a2543036f, 0xd20d7b9bd49fb670, - 0x8a80e16d8708b973, 0x1a9e476c68f4dd89, 0x8613d15cd2a271ca, 0x84b33c2dcdcd03b8, - 0x095c8a49654e0005, 0xe1c4877cfc2c3678, 0x575f5945610b798d, 0xe7ea478b6c34aa16, - 0x566dd8202d8f340e, 0x9704f735b9c3a64d, 0xf574ded4c8b60bc9, 0x1e55a3068f0e8b9a, - 0x6bb65cdcd62f3b2a, 0x2afca05a5f6bacf4, 0x5655970eb3b34c61, 0x2dda72cdac000457, - 0xd8abc54c07566f67, 0x39f4e4c899323fe9, 0xe6da1b9879aaacb3, 0x4f911c56e61e746e, - 0xb4ab4829d854f6ba, 0xade133209888d74c, 0xf049730717e84e23, 0xe2105867e69295d5, - 0x4d619257d2db28df, 0x549a6f8646b83818, 0xa5568977fdfdfc5d, 0x421dbffc9e3d38e4, - 0x139e7474d3cda417, 0xab644ee10e3b03b8, 0xab6d924a4cf66915, 0x0ba87ac05971d7ea, - 0x6766061027100721, 0x888a4d27842894d0, 0x89faeb50c0304e1e, 0x7a8384e1102c9c16, - 0x0681d0be7256ddbf, 0x8f95989ebd25cf8c, 0x9162107dbc58f3cb, 0x9f9c9f43e242e3fc, - 0x61a65cb1c14e3a32, 0xcdeb9b5d8bcefde2, 0x7baf2fc411ca5d2e, 0x95e0b2ba6c60af12, - 0x13871bee84a09667, 0xc513cf9e3122d9e5, 0x443b7755126e6d81, 0x47c55e76275ce0e2, - 0xc2532851c947ead5, 0xb42880597af094a5, 0x9e910ead5794f90c, 0x9c37e826e5a9bf46, - 0x323bca9df03743f3, 0x9367a0cc4496bdd3, 0xe79595a9f66cf485, 0xe8e7f7f2e8136a32, - 0xa02b7774600cb39a, 0xc848bf11e455bb0f, 0x4eb247d1955d7285, 0x62391861c9b997a4, - 0xd6e410dc5bbaea78, 0xfbebd415d173b1a6, 0x69780669ebb04711, 0xe8fd67598cfd94e3, - 0x1a51282269cd2012, 0xb627f55fba10fab4, 0x33ec0d96f74f8fb3, 0x7b34ef95636a933c, - 0x0613997a7940836b, 0x8747f93575ecd08b, 0xb7b6a468727aeed0, 0xd5ad2044c756a707, - 0x5b6b5552e2e50bc5, 0x1fa1ca40857d52a8, 0x32594c7a0f80ea46, 0xdf5b99af45ccad05, - 0x82ef5b989f91033e, 0xdb09b55dfc8653ab, 0x1cc13fe83e85d86f, 0xbc9d8b07f5e870d8, - 0x67c244de80b80226, 0xaabb7ccd4dd72cfa, 0x4ea108ec908ce18b, 0x971ea1ba0923a18f, - 0x6a6e8dfce54aeeb6, 0x9de10f64dae56514, 0x4abc0ee92ca798cd, 0x15f4cd025ae0432b, - 0xb9309ccd00b7b46a, 0x4918b3da432d686c, 0x5213ed869b8c8e28, 0x1f26651b2ed59c81, - 0xf87358871cacd263, 0xd2a542f9d6349ff2, 0x4d9e6c06181e8c91, 0x55d6821d9c2fda30, - 0x13564a56f6f24410, 0x6b08fd3d98df1e9b, 0xf404100b72799fc2, 0xb4c573429072e967, - 0x2363ad90e162314e, 0xe6028b199010f2ca, 0xe1e2f78fa29a4199, 0x55946c1a97f5c6c2, - 0x4dee6e96fff0267f, 0x3a9740e2bd7dd152, 0xb227b96a7038ac40, 0x59f5635c603e7a7f, - 0xee8d4f8742cd7c1d, 0xc5b1d0fa16e4205a, 0x4431a690e1cd3ead, 0x7d0493b3bf08d300, - 0xe539a6a09316482c, 0xb62effa68ccaeef2, 0xf72ddfaa61ff2ff1, 0x3bba9df5aab9cd06, - 0x598978f191d8b434, 0xa8b25c953c573143, 0xb8b9c8e2bbb96324, 0x3aafada00a747ecf, - 0x1b00fcb434038db4, 0x65a67483e752e368, 0x4edfe141004ca3f2, 0xe8d56deedc854b14, - 0x9b856bf78d099cbc, 0xb3bb7dcfc30fa492, 0x759958e3cbe4a347, 0xea7fd813830efd2a, - 0x695299836ebef661, 0x357d8355852fdf49, 0x0609642eabe12212, 0x1cba0e6948b4b37a, - 0xee65fdb9653cfcd1, 0x5a7e2afd709be9ab, 0xff768f5e03c00c32, 0x76c64d874ff38db9, - 0xd862c3b43c625e71, 0xa6bef207e52bf16e, 0x205c4964bb8b4dcb, 0xcb0f815b4281afa4, - 0x7319aa13cd0bc59a, 0x78ba6bbfb6f31928, 0x9714b16b5cc6a559, 0xec2c5d283d41eecf, - 0x904fc9b8a3609744, 0xca7b8bd21efdfc5f, 0xb9a71a6648de8ae0, 0x37fe90a0b68e594f, - 0xfe54cc1194adffcf, 0x6b971248c56b966e, 0x0d3dd551c9cddf5b, 0x475eb1dfc6edd9c8, - 0x72191586f807d3e2, 0x100c04d4fdcab2f3, 0xcf5982e4db3dbfb7, 0x72cb36cbede8a3e6, - 0xaaae782366a6df77, 0x9c0f7404fab86da4, 0xbaf3e4ffd697d620, 0x1ea1319d0b5ae315, - 0xedead1882ee55bc8, 0xac886f76095e37a0, 0x3c9fa4e34df74a87, 0x2b0ced3ceb88d596, - 0xd17aeddc7e2b06fb, 0xee80027d862ad1ca, 0x9e4e9e096a4c6e6e, 0x7feb261cb7e906ee, - 0xcfb4fd8d157c9c37, 0x4bbf5f01c169fa45, 0xa227db71193629f6, 0xf4278fc8ef940025, - 0xa0bbc33cfecdd276, 0x9eef97c7513fac4b, 0x6a5160539eea48ff, 0x334b8c7829f5cc9c, - 0xa09cd479d39f6624, 0xf699ad13403afeea, 0x6c48781d2ad5d691, 0x9255296ae9e507d2, - 0xdfaf7175b5c294f1, 0x93a1a8969963a917, 0x045535d13e98d652, 0xcccc28b180a80535, - 0x4b2dbcf12f850813, 0xf949d3c4dc708766, 0x6634b59e43b0f3a9, 0x61ae157f8eb9c45f, - 0x88a15f26815cac28, 0x62e28894b1ebe16d, 0xc1f0591b965f815a, 0xa1b82b79cbb5a95b, - 0xdd49a88e258525ad, 0x20742cadf95abd6f, 0x396091e873bb2a41, 0xba850dcf89c6d6ca, - 0x27ba660067976236, 0x2d87dac3fe8e3a4b, 0xd3ed4bb4daab95c5, 0x2ccd903d96fcae74, - 0xb2aa2518db87a7cd, 0xb7c1fc4715107ed0, 0x3cbf7c46aaa59d91, 0x399e42b18cfc9004, - 0x5b0168830cadb5d1, 0x4ef4c13df5499a07, 0x6438f49a8a794d3b, 0x23f278286504b223, - 0xeebdf54d0b9c84cb, 0x54d227d196ad4de0, 0x99d18c97be02f426, 0xbf0204952a18da33, - 0x6b9c24054c542958, 0xfa4931f9a8360643, 0x6cbaedb0393aee4b, 0xb44424a94a20e589, - 0x239b638bd234d58b, 0x5bd6d4ca4294f078, 0x7379e2fbedc1dde1, 0x265fb61573a8e1da, - 0xa95167eb27526360, 0x46b3237f729309df, 0xe3df48631b693084, 0xdb5bfde8b4bfae5b, - 0x23a6059b2b8d16ee, 0xf26486a658a03c0e, 0x64ce96378d313981, 0xe0d4755cf099e1da, - 0xfcd556652e4eba0c, 0xbc87e55b0ae8e5a8, 0x5377ade7a2bf6946, 0x777ef8251035b6ab, - 0x2bc471672f1da3b8, 0x337d13210adb5175, 0xff1c3c885fc99cd2, 0x5b0a7c65baf42d71, - 0x87e2d46431daca8e, 0x5b2813a3d8956acd, 0x0a70d69d91545646, 0xfefefb24bd3fb6f7, - 0xdff1d6280b080d9f, 0x8e42779b7938507d, 0xd97f149fb05140d0, 0x9c1edac5345184f1, - 0x47208009f08f0158, 0x785b6ab75dfb3abb, 0x92b872ba76f0e709, 0x40eb3233b005962c, - 0xc941ddc854aed1ac, 0xc94269a2381d9960, 0x9d58a7017d399f4c, 0x4ce10cf998d7ca78, - 0xb5f363f9aa82ba14, 0xd006ca37a2e8187d, 0x4bbedb0718201ce4, 0x0e56431df371a6d9, - 0x12cb0af2c3e39967, 0x1b28f0c321079617, 0xc4555ea30c9ec07e, 0xe78e5e6f3ee8e8a9, - 0xa2012c734ec96c41, 0x5a6db63f8c0fe25d, 0x2be4c45cb8f49a48, 0xa3f73761a93c2c22, - 0x4b9236b12e234d06, 0x2434f626e37a4ee5, 0x72b3c39452249313, 0x34df52c2cbe0473f, - 0x0da8e50e5ce318f8, 0xc08f5626106c9898, 0xda75e029a89f3074, 0x22132b1242442740, - 0x93252b144ccad48b, 0xdf375ebc1c3c5a46, 0x9050ad57212206aa, 0xe955e4728140ce1b, - 0x3ca172c073005cde, 0x276db6cd77d84ccb, 0xd3102e58f319a24b, 0xf14ff494ba8f62df, - 0xdfbdae8d076f0b40, 0x3632d1fcbdb9df1c, 0x2fa334e160e911b1, 0xc00c22ee2ecca96c, - 0x359c51489a63686e, 0xb7be9faa04f68f36, 0x045baf2ecf2b5219, 0xc89f9117c1edb726, - 0x9980e7cc490e3460, 0x9d583c192423ece7, 0xc4cb452fbb290438, 0x320d12d474ff5af2, - 0x1b6d16f068a309c7, 0x8d9cd5f266520731, 0xcd15e057eef65045, 0x4972e1bc94622a87, - 0x147f31b0d67a6374, 0xe4a98c59c80c3fb3, 0x817d6ec99c6d017c, 0x4eddfc12c918e59b, - 0xc821c1cca40da27c, 0xadc124ae549dfb7d, 0x754c880c6e36e4c6, 0x6df0ceae7a6623f8, - 0x39eac06111be893d, 0x63053090fb23d45c, 0x945f37fe9d2b3dba, 0x4e6d4a186a4a021b, - 0x2c29cd2f941fd695, 0x3845a49612c3f6c1, 0x8ce30f2d46f4abfd, 0x72279b205bd83dee, - 0x3b2d65df04701257, 0x8bcc9e109aba7a8e, 0x979b3da62e9bdd7a, 0xa7884c8f3de1d90a, - 0x992ec05c4ce73824, 0xcdffd7f22b03e951, 0x59ceca0bab0a7276, 0x554197cc221e8b69, - 0xb2a7dd6af58123bb, 0x1eaca5b58b81a3fb, 0x1920cca62746013f, 0x9e97ad3b49991bb9, - 0x54b7a5b046fa43b2, 0xafa18b8f535e6ff5, 0x4a6d985e1d6e3626, 0x42b09720a8fdc7a2, - 0xac40492eb2642eba, 0x69f8e57e562aca23, 0xa03b7863f26d04c9, 0x413d8bda24cfd3f8, - 0x3341569c1052989c, 0xe3fea2dfda634ee4, 0x7c41ef8524632677, 0xa07c6f80c5f4b74f, - 0x35299c485df5cb5d, 0x29154b40988b4e79, 0x8bb9a8c00d7880a8, 0x85ec0b5c253db0dc, - 0x97026cccc239b2b3, 0x33b71c4be2615e25, 0x7ec91debeaf4a487, 0x726d405f12f35cc4, - 0x4c47706dd9b3ef31, 0xc3fe0db91dea86f5, 0x3700721fd69d90b6, 0xb056a9cd2b271639, - 0x6f1d462a3b5b68c4, 0x78c58fa3aa2f9db3, 0x4f876b8a8c267787, 0xd2d38183c3bf177e, - 0x666028816ea29066, 0xd28e9ee135572af4, 0xd3c48259d2c2aa2c, 0x67f8dc6a217459ef, - 0x18bb970f89059601, 0x8c2640cf4ee6cbaf, 0xc63935ecee4ea3cd, 0xbfc70aa25ade5843, - 0x2acebb4189db05bc, 0xac2e08bd79f2f6c3, 0x68d9948c7044e59f, 0x0d2af72628aeba8d, - 0xa444bfcf703ddd2b, 0x927dcc5a2e117877, 0xcdb292b9dc8012e7, 0x60487702844ae092, - 0xbe1b6df255829fb3, 0x6161c36c92773523, 0x111db80eb7b6d60e, 0xb3afa170ddd84e05, - 0x04953b4cd5845fa8, 0x156d810a427f8a19, 0xed3ce37735965ffa, 0x069a462de8f654e2, - 0xafa654b443f8b081, 0x16666166080bfb17, 0x0d4eebd7d45eac39, 0x9022b33c853a256e, - 0x031a5ddd4a0749c7, 0x90f8ddfe03dea0be, 0x30744723fd7491c9, 0x1497c1b71c3a06c1, - 0x01283becfc4eefae, 0xf6b25d8db65cda59, 0x0e7a10dbdcd1ade5, 0x98841b4acda11a06, - 0x2749512e58b4ef6d, 0xef27a64c97b8f9a3, 0x1a7483f705bada71, 0x2a39820b01aeef50, - 0xaaab549f7d36bd3d, 0x0d554016d019a036, 0xe5a29cdfe2f47af8, 0x4941042580e2a95a, - 0x7526aa5ede922c73, 0xd083c7c8c0d8372c, 0x3d47910e571b8861, 0xd95c0eb613accc2c, - 0xfec268d8f9605d94, 0xb8d5eab93f7905fd, 0x1a75d14921b9ffb9, 0x7cea4aa15221be88, - 0xca0e5749d48277d0, 0xdfdfdbf59cf29ef7, 0x95189b60d07d1474, 0xf8865d10d3baccfc, - 0x9cc1cbb229f6c8e2, 0x9595d4ba24880826, 0x34e75ca88564ad4c, 0x9113900b38c26dd4, - 0xe81845cd548dfd33, 0x8772335e8f19e877, 0x27c92c2fb6669b12, 0xc5f7d7ca22ef476b, - 0x44a3719f36a059ff, 0x26610e49e9a741b8, 0x8eb0eb6a8ba6e542, 0xc137c8fa00b8145c, - 0xccf1f207d4331d23, 0x8706c85eb7146072, 0xaba44ced38bb1607, 0x7864451b2b1815cf, - 0x89d8983fa1f85907, 0xda0843cd1e212ad0, 0x00df8b62400214ab, 0xf9b16f63293e6774, - 0xfaf91800764e36cd, 0x6381d1f3802b0b9c, 0x00afef107a37baf3, 0x2286cf55119b331d, - 0x3e012d6033ba1063, 0x5d5fa60fabbe905a, 0x7825ee253157fbf8, 0x902ea09371b6e2b9, - 0x91b0411e30f3512e, 0xe78aa9c1b8f3d771, 0xa2688331f3b6ec17, 0x8440a9c231ddff04, - 0xed5c44e872e4696a, 0xb5295b037ec03f94, 0xf87792e544223823, 0xe9e9391b23826bc0, - 0x9bfcff10e5abed2e, 0x8efe784608de1bd8, 0x5365f61fa65568b5, 0x52b2054450063eee, - 0xdcbf463c78564296, 0xc2989570f55e8a53, 0x7e164e017b693f2f, 0xdde91125fd199861, - 0x2a169a7b33adc793, 0x9f4e17192850c3ea, 0x19c01991a707f3b0, 0x9c4e5092e8da0cec, - 0x3100e654d7d4a249, 0xda10dcc1374b8044, 0x9b7cc6aa9c377cda, 0x97eff108c6bd796a, - 0x470314e8d8b0f58c, 0xb23a8010eab286a4, 0xebd3adcc33105292, 0xb99c683797b076b3, - 0x63e7e0e675123a07, 0x517fef9b5537e714, 0x7e8957b0640f4547, 0x5c94e0cc35eacf7d, - 0x813110af9bb46694, 0x7af6126991893f78, 0x6fe50a35222607e7, 0x3dc8e5ee6ad06e34, - 0xfdf9411cc087c4ec, 0xe961afd929aa5316, 0x6ee5bfc05fc85cde, 0x80b78e685fe58fe2, - 0x2750e6b2e564b34b, 0xe54c5db1c9f00a11, 0xb7c92ef053a746f6, 0x8460c0b8047fadc5, - 0x3796f74d054b9020, 0x8ed62ca604f8ee3a, 0xdde74f43dc569e41, 0x1224e3fbc2f281a6, - 0x5e6550e648308f35, 0xbe224a6d14f817d9, 0x8c66d9c4b530758b, 0x3fec9a2d7d94e25e, - 0x14fa9d602e49a830, 0xda6536f360c3c933, 0x35b0fa2792798d15, 0xbf946fe0a1583d45, - 0xcb79ad96d674f0dd, 0x661eb3690f104f75, 0xa2f11f19ada9e561, 0xfe0e2c5ad05472ba, - 0xa978f5671b4c59ab, 0x8b8599584f359911, 0x5b5803331ba72a7f, 0x67c5a9fc6846ad44, - 0x55cf92c9b3598ac2, 0x2fb4148956f3b54d, 0x8c247989debde614, 0x7b40299c2cf4334c, - 0x680e91a0c779746e, 0x08e936fae53ee1a1, 0x8877287b86405689, 0xe8daf80895108c6b, - 0xd0c19835bd036162, 0x51141c4dc658bca7, 0x8913710d56e6b819, 0x6cb02a87e51da8c8, - 0x23e17b406a1290f3, 0x386cd4f648409b85, 0x8dcfbcbbd38d713d, 0x78f847960ada5253, - 0x3faaea970310df87, 0x2f1e84b2f2b4c8b7, 0x0eafeb4ad016c8e3, 0xdc881ff9f98a1cd9, - 0xefaab6bfbf8fa49d, 0x4cc4c92169fbcb43, 0x454014b534738f1a, 0x63a74e9c50989367, - 0x1a173a3dfbe7a4e5, 0x86ef30562fe559ac, 0xa638aaaeadba274e, 0xfe5dfc36d3e4cbf1, - 0x715b1369228faf13, 0xe856be2d0de525a7, 0xb7ea0284f33e7841, 0xf7963585639da36d, - 0x31db99a4ff6f1d9b, 0x665860017bd8a15a, 0xab655caf17544756, 0x03f9212052600ae4, - 0xbb4ddf712b253186, 0x77c4fbc3efb13d1c, 0xbf368924070fde5e, 0x796aee4ef778a395, - 0x0971eb60ed054b7f, 0xc671c3c60cdd4bee, 0x8c08840a316131be, 0xf5c13ddf06062f8f, - 0x8df18a849f9369ae, 0x29ad21b365b01dea, 0x36ac5ce99019e2a1, 0x3a3bb68a08bc6981, - 0x135a2bc2075de743, 0x0de6a6ac5fa70270, 0x08f4fbb4b8418ac1, 0xe092ced6d356925a, - 0x772ed44d9a6eab36, 0x2d857ea9d439ca06, 0xbf2dcbf2d1df94b3, 0xaaeee055ae75cf95, - 0xf62a302dfb56f24b, 0x75c5bec0923a7780, 0xd1f6a6f26f21a31d, 0xe868605d120636d3, - 0x7c12fb15b7220a0d, 0xfda84930d2dfebee, 0x6f83c02670970162, 0x793eac7e9f363558, - 0x901390babccbb454, 0x20bffc51851def23, 0x9527490f16494961, 0xb2012ffe8191c0a4, - 0xc707a8b31453e67c, 0x374e0368c43e166e, 0x8e9a37064cd0ecb7, 0x5fa8ef1c5783eebb, - 0x16b4d54ffd14ae23, 0xbbfda4b197b382df, 0x6f13e2da174ec49d, 0xee772bfa125c3189, - 0x3ca6f2d30e8c8146, 0x3dd4f96ba1c42ca4, 0xf7ed939e78391929, 0xa9d81922c673f321, - 0x48f6bee82965b2b3, 0xca2b71e056e62128, 0x42395b5f148b7905, 0xa62eae214e9a1d3e, - 0x90131a03e4a9f674, 0x190cb7d67e54fde0, 0x18c699d5a63f59a3, 0xa4faca83614d5550, - 0x5f6eef67f9ffdee2, 0xe4e457ff3d2b285d, 0xc047f1e7fba34515, 0x14370ed3b08e85d5, - 0xd8f69603f76a6f09, 0x564b702469c91d58, 0xc82a7f8cbce2ee55, 0xa255ae297e115b1f, - 0xce60738027e3eb96, 0xda416beffa2b9f9a, 0xc4c89c08801ebcb4, 0x9d3694aa40026cc5, - 0xc1c5b7dd8ae1690b, 0x59fa23e2892c46e9, 0xf02e6d01f4e00197, 0x4059e93278269334, - 0xcd61c1ee7dcc9fbd, 0x02ebb082c19ea031, 0xf9a418652ea070c7, 0xc8282f5fa3aeecf4, - 0xf83d1529021eef9b, 0x4580daef02ca20aa, 0x0233a9451f8c9f9e, 0x28107ad50409c8c0, - 0x0ed145f4e0319521, 0x529c135f4894247d, 0x0091ff02e142a10c, 0xb79aaf4c4855b7c1, - 0xe54b9f4cde954c02, 0x0295c469e37ebc46, 0x21f1c7056ed6b4a1, 0x4a51e84d34d5f031, - 0x0cb45eef82fbce13, 0x9766f8f46ea85ac6, 0x29c47696b2d03d47, 0x1305069cd4d7154b, - 0x899c5a533eb9a969, 0x062155c8548c5e58, 0x7b44c801621c375b, 0x6b72e3ce3516a236, - 0xf5da6b0dc95ee5e8, 0xea412791ac16fe01, 0x2cffd6b41bfbce69, 0x754a5257d5f2959e, - 0x812c51b554bb5891, 0x1589c0a360e2afca, 0x388228e7c284e90b, 0x944481a3bbac0331, - 0xb882241941c8062a, 0xd54e8baf4c72c3c9, 0x0c03a4b532e9a5b4, 0xd07b394027c1ac7a, - 0xb9f9857465df5c43, 0xbfdc439fcaa733a5, 0x00889ba3e98c01ea, 0xedece1f087061699, - 0xb81ebf8d4f961483, 0xbce411cf673d9d04, 0x76dc8b9981c20b40, 0xd26b9084b3305290, - 0x7c71da491805787b, 0x9b40856ca294e56d, 0x3f332e97225b020a, 0x6c2af72c8bda1718, - 0x51ee043970d68c9e, 0xcf69f93018e32e55, 0x91f243d48ae302bf, 0x6409f031e0519398, - 0xbd1fd105e0510cd6, 0x85e9d81f2e188fd3, 0x6b7c90e0aea463fa, 0x1541d0bbe69582c2, - 0x2c52dc4271c14e78, 0xf6943cf6ea4677ff, 0x5a58a9d8a9068c56, 0x3b21caa165e3521e, - 0x456c8b84896684d3, 0x3b4322388cdc72e1, 0xc94ce2664a2cb632, 0xe8c26b05a7bb7eb7, - 0x1c9d33b830e3a50d, 0x20390236af20b4b6, 0x4b604a89ac5f8e30, 0xa181c958d24b3890, - 0x1385d32e323744ba, 0x7577bc7072a560a7, 0x37e35a74d2326010, 0x468111dd7791c9ad, - 0xdac7402c34267857, 0xb72d12e0f290cd9b, 0x28618d52aca16823, 0x306eb3d980fee69c, - 0xef95d0886c479189, 0x3de5a56421aba09d, 0xc697b7d967cf5b54, 0x4f7e5aeae01bfe71, - 0x2ee5808cb4819b87, 0x19acabf7778021f4, 0x2624d7853746c06d, 0xc41f3a5b4a1bca94, - 0x7fe94f5d12053cdb, 0xd77c0c073ead1a80, 0xad89dec69e090bb9, 0x517ab3742902ae0c, - 0xc5143414b516cbf5, 0xad5800b1b6800acc, 0x48c44439d5f67184, 0x2b0c74b3de95ffb0, - 0xb2ceedc9a6f928e9, 0x27b6e56f05821fad, 0x2145bbf190f94e8d, 0x534bec5098366bd5, - 0x02c167c2194a6786, 0x3440dfcdb724b86e, 0xf5b3ef49089859ac, 0x4a16ec9907637947, - 0x55dccf60c580c4eb, 0x0fc4e0f5fe828680, 0xcbdc05ef0a086fa5, 0x69b1c6b943224200, - 0x10f32a975fa6a5a4, 0x6b57ef8280af169b, 0x42c9ec9b324b19ab, 0x88e1acc19315c2f9, - 0x290c1ea0c28618af, 0x5ee0f6082fc7d5c3, 0xa993988ff40638b8, 0x7d1a5d01d0ee2964, - 0x6f02d7a589eb3578, 0x7a34b9a164eb28c9, 0x3599bf6edc21b5f6, 0x42420084fa399e36, - 0xa55e6e3afb2591aa, 0xbd5cd8fd28d3d989, 0x692e982394e315de, 0x9c711b111f1da811, - 0x8b8a87c8b60797e7, 0x6f1e7b2e3d5d1517, 0x8a3fe3054d83148f, 0xf551a9540f13b3e3, - 0x72e76d96b89a8d4c, 0x7e3cc3f4c60385fc, 0x39c14978c7acc3b0, 0x0af55aae5bd863f7, - 0xaa30ef33dba4b3b9, 0x749828315dcafdb4, 0x475b3cfbe92cc03e, 0x67e72d4e823879a8, - 0xc9ee6f2c391929e7, 0xe18798a7d3c5aba8, 0xe807406fb30b67c6, 0xa61395aa639377d6, - 0xc29a73a2e8ab5af4, 0x5aaa55fccaf9d6c2, 0x967f04f118ad59d4, 0x17e31a7b0b58ad39, - 0x7cadef89e2b22c08, 0x0b276527a22d022c, 0xe11abc1bc3172ef7, 0x057b13c90951fa9e, - 0x70567928f3f80acf, 0x827dcf38b17df714, 0x2e64548f4b4c73fc, 0xaca68bbddd78e838, - 0xd5d5988bc8dc5e91, 0x042d45dc6223f976, 0x662e4de40d33e3ce, 0xdb7459f3c4fd5935, - 0x3110a17d14505d6e, 0x617d1352d87a5b35, 0x7c4685b33b223c27, 0x451e4d7a9152d8b8, - 0x1b65cecf8266a056, 0x5a45f9b0cd470d2b, 0x7d12941567cff145, 0xee1a3030777220c9, - 0x8e713d8beb663932, 0x4c6d8a061646bf2d, 0x9943de3068e54256, 0x5306bd826d06817d, - 0x150391330aef4eee, 0x911cddcaee00ebd8, 0x0635e0d0750de675, 0x0adb29d4c80b250a, - 0xbaa91b7bec14b6a3, 0x8bb251c4d626fdcf, 0xf3d93aa25b681937, 0xacee38b966ca8b8d, - 0x983afcf415e75dd0, 0xae57978afe3b7e84, 0x4fcea029c8c9e70c, 0x529baf14b98df6c4, - 0xba514be5eb3a23ca, 0xa7e0a13409099a78, 0xff6aebbbed579b2d, 0x0b144185e3d7c1b0, - 0xf11d74de9b1744f0, 0xa3e72be7187b1f46, 0x7504be394c45d09d, 0x83a2c8cc8a987a73, - 0x21c88b62d048776e, 0xadbde7f160d8c96d, 0x1dc8c52612577f79, 0xfbe9d2063442f7de, - 0xf5c473ec4a3fe6d5, 0xf31c5744e4170c19, 0xd4aac28d7905d189, 0xed45fcdc435fb588, - 0x97ab84c6236dc214, 0x5baf333780b21044, 0xfab05aa501a125c2, 0xecb022dc9ea08637, - 0xc7850a4da97998eb, 0x4c4f746292b87a75, 0xe68d9b8cb70327b3, 0x4f29c30fa4cd9080, - 0xd5f1e14b34fb1cda, 0xfd9ed31664a562df, 0xdacac110bc98391a, 0xbaa5b57e447a4a5f, - 0xa15730a1af8b9e30, 0xe5a655fba44d3c6c, 0x38529cec7f7c5e53, 0x83f47e73d444f65c, - 0x66ac5328bbc88282, 0x3e08698fa6f524a4, 0x90ec419fc541cf92, 0xec025eba8e545284, - 0x89ea3c5ac193e861, 0xf3e4e77ad74c5d24, 0x8a45df9502f4003f, 0x3cb8ca44c1701b26, - 0x2f7992a09552b596, 0xedd0c10d9ed880e9, 0x38b31f377d894555, 0xf298863b648a67b7, - 0xc1b870a7881b4f3e, 0x002674622a007ba5, 0x8092ad3244ed8148, 0x49ee2f6041b60776, - 0x632cf22888cc6dfc, 0xdba2959c64c16441, 0xc18b9a7b7be100f1, 0x6afe9be632112731, - 0xd6249417887b40a9, 0x0466b36f06b5a76b, 0xe7f0217708de6e0d, 0x166766578aa6bbc5, - 0xf7ab61b7a320b0e4, 0x07f942adbec4c7c0, 0xe7d1baa76a6c79cf, 0x328f25d5461bfbfc, - 0xb0e7869b3ecc4a2f, 0xf0947671eccac059, 0x1027a538087dea64, 0x1ba54441964d0a18, - 0x112db5f8d0e525cb, 0x2dc74cd3dd390033, 0xa12ae55e476839e8, 0x7132edf4304f89a5, - 0x6f00f4f6bc1c532b, 0x9af5b63e28331f5d, 0x3360d6a27304643d, 0x5941fc15dc93e55a, - 0x47a9241412672474, 0x0c4b9803622f4504, 0x0b5daa66678758c6, 0xf22c6cd02ed7ae9b, - 0x6c514a6331f9ad4e, 0xa3431ebc50ab68a1, 0x6f07ea6161fce9bd, 0x199c974715836554, - 0x04f3d15744f3cfbf, 0xcfc9b0c3824b76ef, 0x7c56af924c58a9ea, 0xba495e117970487d, - 0x25a7818abca760f2, 0x9a62797f2110d02a, 0x92646e1554e85c87, 0x6f9cb81486614150, - 0x468a07af46392cf5, 0xa3542cd002bef2c3, 0x54c5626c350ae580, 0xad4efc1fcfb98fe0, - 0x61a4801272ea40ed, 0x5837a739b88ec779, 0x53a4ae9741a27cd3, 0x1d65fce509baff3e, - 0x5147c4fa5aeb7abb, 0x0b91f3f1f6e4435a, 0xbc3100b284051308, 0x1ec2364d5f04bbce, - 0x4c1b4e17cf442a67, 0xacef09e475c7076c, 0xb42f7b67a91451a2, 0xb0b32efc85e68d40, - 0x5cd3c6921bd36f6e, 0x78c6c2d211fe14d7, 0x89663735f0fbb4f6, 0xe127e2a54c54c43f, - 0x111851c5e8b91353, 0xe19b80e61db2fbe6, 0x6ee301c0dab99f51, 0x90131120af1ea480, - 0xd0693289c2114cf4, 0x3ceaf348e2f52531, 0x01bb1ceb5fa67725, 0xfd6bf6946c05eace, - 0xdd8a988a014454b8, 0x0217f301a694921c, 0x677a83e3244ed994, 0x0a16f3ada3c3fe64, - 0xf273ac5541ed53fc, 0x99db037268853a76, 0xada181f2c03fdf1e, 0x15ef5e2b886dd6d4, - 0x19c56f4bdc451ded, 0x34d628483964ad21, 0x6e7f6f99484f1859, 0x3beb6c144845d786, - 0xaa7e9cdf3259b85b, 0x9e4d55291089acb5, 0xb3ce99d00c83d5f5, 0x3bdcc1d081f7b2f6, - 0x6f14e984708daaf9, 0x15e47d9d55c469d2, 0x1979e42fea2e3ca8, 0xbf9c6764d98bd4d2, - 0xdd767e710fb7d72f, 0xfb2a35a931454177, 0xcce476626a9fab9c, 0x8da6e86c9c774840, - 0x71a7f61f1656315c, 0x283fda98d127c7c0, 0x54d455e56378ea1d, 0x4614d2efcb6a15d6, - 0x59d4ea4c7b66027c, 0xfa401e892b8b3eeb, 0x0212b994aaeb7bce, 0x95e3713db591f828, - 0x7382420743d258f3, 0x69e365f2a41573fc, 0x557a5d8e2556518e, 0x46a1a9c2081a196b, - 0x526c8ef17421e4d6, 0x2978001336b6304f, 0xb055c728ee7d0ddd, 0xb482b8aadee7aecb, - 0x05c52e33c5e77a6f, 0x4a6247890e638a37, 0x255e52ac97b3d039, 0xaae35dfea1c75855, - 0x93552dd3f0fdb4ba, 0xfdcc3971e642120f, 0x91b1eef789b48648, 0x154787ca45dda206, - 0x722182f2639f7f18, 0xe3c3f7d3ef340f80, 0x7984022b21091573, 0xf7592b1149094ae3, - 0xeed6e38050d828ce, 0xebe7883eeb1eeaa5, 0x0588d7a1f8092279, 0x93771b4d3d8753bb, - 0x50e6f57bacf1753e, 0xcb512b90080a388a, 0x2b9aa10d3865578e, 0xd1f8d58cbbbbc8dc, - 0x0f1280f8121a4b34, 0x1320ddbe6468761e, 0x077475763e80bbf3, 0x3bddb635e1e10fb6, - 0x7ef40a13dc2fbcbb, 0x445a68774f3da827, 0x1a2454abb2b6cbf6, 0x10c01bcb7919d310, - 0x201a20c4f4ca33a5, 0xc35759e4477d5fcb, 0xafc29909aef70ac0, 0xe6b5b973d3bbb3ca, - 0x378102c9200fe5f0, 0x6e13607bea534c2e, 0x3489bea419ca61fd, 0x0a49dd8a953163e7, - 0xecfb6fe7651dcb91, 0x99e44e6d0e2641a8, 0x4cedd2b9dd5a3d3f, 0xbc09992842bf5116, - 0xd7337adbd8b5528e, 0x175813ec91ebdaff, 0x255c853bcb02fcba, 0x18a7e8f2688cebe4, - 0xec643c4e1bdbb99f, 0x416043b43391809e, 0xaa42198b6781ee85, 0x794540f865959da1, - 0x0050a8422f325acd, 0xda319ea82d4bd78d, 0xd02927fb87760e85, 0xbf5a7c04f13b35f4, - 0x4862bfbee6ab8190, 0xd1e459af39a7f4ac, 0x690f801d7dfff662, 0x6b57c814f57774ed, - 0xf1272d6c178150b1, 0x6b3dda4f9dcd3780, 0x488de59a4383b448, 0x0fad58d6016ef5b0, - 0x8f0419619fe5204d, 0x7d869ca214609e96, 0x83276a8178bdb643, 0x5efbd3f2882da912, - 0x024f5e85fc7c2fa2, 0xaa0c6acb744d482e, 0x3b08e30cde85af91, 0x0aa1c6a5f3ec49b7, - 0x1289c3b746490bdd, 0x280b0cfa9af3db3e, 0x0c313d24a7a6a454, 0x6b31f2aa6071f47a, - 0x8ec97f55532f752b, 0x123e614c49147c44, 0x047660c108bd9483, 0xfbc13db0ee398ce5, - 0xfa92467554571c2c, 0xa929cfd749ca4c68, 0x2a231fb294d7766f, 0x7e10fa8fc05ae42d, - 0x855e9e1ef643127e, 0x2aa4bf00bb360a2f, 0xa14b049ba80ade5f, 0xe7eac84ba0aecdfc, - 0xd2ad4a50fab5a98c, 0xc42e1289b7647656, 0xe0bc0c30ce0970cf, 0xcb702c8fb76311c1, - 0x111207795af0dddc, 0xb8cb9ed0cb21792b, 0xb1a41255f269cde1, 0x61bbc3a3c9176ec9, - 0x3e1d0cdd46f3ec84, 0xda74cb1b02d7058f, 0x2f0c7b9660ff64e3, 0xb1148b7899c83449, - 0xf0a44b4326f2efeb, 0xc84a100ffc441418, 0x57632cbc5beef647, 0x94e1a841204e9034, - 0x7f62ab66e60f764a, 0x9c6d14f1b1876d88, 0x49ab70f3525fe230, 0xeecf215953528f18, - 0xda0579aa12bab08d, 0xc34c4f69d74c0f26, 0x033037838762c843, 0xefcdcc077132b157, - 0x913883d318bcf844, 0x6f9b30e1a4bac00c, 0xa7861ade2e441a35, 0x622c5901979efc24, - 0xceb39a59ca37bad5, 0x3570963f05be6fd4, 0x62ad4895e02bf3a3, 0x7eba6f82b82dc832, - 0x95234a1b8fc71c39, 0xb6f09f13565b8a89, 0xdb57430808d951ce, 0xe4a4248118cf6ea3, - 0xc84cc632a2ae7bba, 0x22f475ac2508c468, 0x68cc3d1497393995, 0xe72aa6df700f15e9, - 0x6ac873c2b8bcb1d5, 0x3f6734836ae30a65, 0x37e5f02daeae3d13, 0xd39167dda66565a3, - 0xa153094a12d166cd, 0xe4c7603edda285bd, 0x923cf3f0a5b5ee0a, 0xf9fb2dd192954e48, - 0x306ac270c5b723e0, 0x8606c830122c2958, 0xe7aecac688a53701, 0xbf6dc059943218fd, - 0xa4535b9dd962ff5e, 0x25571849af3f9777, 0x4c9e418f6ca9e2c6, 0x3572d9f3f3f65cca, - 0xc0bbcdbc341a2058, 0xbd7b0453c63dbffc, 0x4f7a9ff3d29166e9, 0xd1de349e5a6b4d75, - 0x5fcb68a64bde964c, 0xc395d181d82b71f2, 0xac00398a33f51378, 0xf4a6337e47bf20f0, - 0xc64cb0b05163d0a9, 0x30bc5d8cee58e70a, 0x12a3ac6e4c40930c, 0x5d48a5fe9a94d7b8, - 0x0f54c0b4ec777fe2, 0x380a23e1f7d7708c, 0x5ad927039604c51f, 0x60d473456a62508c, - 0xe56dc76ca61fd1fe, 0xaa4433ddf04c3580, 0x0b4a5a54ac14ab10, 0x4f1d86d7edfb852d, - 0xa95fe9a70df1aeae, 0x885a5c7b28159a1d, 0x4409daae13a3a78f, 0xaea8705bcc2fd398, - 0xd02e202cd1edce96, 0x919bd3c74de1be9e, 0x1c2d6aa4d55419ba, 0xdf8a9c4a721b162b, - 0x73c3040c0fd1717a, 0x68b2eae14d78b77f, 0x105f8b2555d5bff1, 0xf9b8e9f43e2f7970, - 0x77f0a45d7862aa91, 0x26102398472d8203, 0x83c8f5a302078aeb, 0x4a781ef65e962e8b, - 0x3ce33d2a4b603fe1, 0x72a4b9d499c2e7de, 0xb58dbf4808d242aa, 0x66a8243e268c513b, - 0x12c2c8966fc7890a, 0xb56c2ffaa78f8762, 0x2ae7b38cd0435a50, 0x3d5e4d97ae34f69b, - 0x5fc317613d6bc59d, 0x6bf9af035c90894f, 0xc98ded5cd8befc15, 0x133678cbfcc1ef34, - 0x32cae0f08227a937, 0xaa3140622b9540f0, 0x4ae39fe55c80fa24, 0x9edfe7a36d6320cd, - 0x00ef4cbcb1e07d2c, 0x1295bb6e2c2a59af, 0x0fe5d0db1f1af0fa, 0xf6b309dfd7ebf79d, - 0x3d24f10e338eaa66, 0x4c77759419ae9087, 0x15fa2a1105528d06, 0x24b3de51ad631e56, - 0x6eac8f01e1ba4776, 0xfe6b5c12a342fe57, 0xda05caeacd2cbf3c, 0xe2491bf0f2121019, - 0x6d937814add0adb5, 0xd427ee433c9f55d0, 0xc98c3644102e4fc1, 0xba570518303ff9e4, - 0x8c778b744d7441d7, 0x6077c1fbdbbdd700, 0x6169ecc5d5b4d4bf, 0x3dfc98b1f0949933, - 0xe377495735c9968a, 0xe70e207f0734f8a8, 0x9f54ca53d6b1cd12, 0x14ae2af4e905f636, - 0x92f3d2649f629922, 0x1f0bc9353a1737ea, 0x14bb26ceead999eb, 0x438d3ae4330827ef, - 0x9d893a3f43533ad5, 0x6bbeb15d949d61b7, 0x0c39ce1075d3cbcd, 0x2366e5079088bff5, - 0x7f6570316888ca19, 0x19a9a32a062f4857, 0x4db801529d58a828, 0xe869d47ccbafcc63, - 0x995b788df0bb049a, 0x314d9467b0daf893, 0xbea1d1cd39fe207b, 0x41461bb994987ac9, - 0xc81440601f679d5f, 0x78bd9800ee5152c6, 0x5a5e9aad6755b22c, 0x159a2f65e39e4ad5, - 0x49427dd743a2fce7, 0xa16e0382d1ed0213, 0xb21ce1fed4a274cf, 0x845a5bad82144817, - 0x79c5470c70c1eb59, 0x16acabdfc6a24be9, 0xc228c144fffcd57a, 0xdeb2ebdc7bacadd1, - 0xa3bb31927e5fde99, 0x3627a2fc82dfda3d, 0x73b8730e93fdb147, 0x18daa73a840536c1, - 0x9818c222439b187c, 0xca6e21a4818c0a64, 0x48ab7a57cd0368dc, 0x29eb7e992ca89a45, - 0x986bb8f35ac73648, 0x8866033fdbf7efaf, 0x896a3f633808768b, 0x2824b27d3d4d12d9, - 0x77072444404f2f05, 0xc9765da8b44b4613, 0x9fa80e27aa9885f6, 0x200e50695055a1d1, - 0xa7fb47fdb7f3a1c4, 0x8b9b2358593dd320, 0xd41209b3b80b3276, 0xd2e7955bee72e392, - 0x23edfec0bed3dadd, 0x4cb470b56c34dfc7, 0x1e765ec4457559a1, 0xf89ce5a0dd65ac34, - 0xf7a814af0e779265, 0xca0724665937b162, 0xb5c114f720da5865, 0x5039a5b398dd42fb, - 0xe631a2259417f092, 0x4a17a751dc03a6d0, 0xfe60a8996914ff4f, 0xabe38daff37c6c12, - 0x257310e525eb4868, 0x2342c32ff4a66fab, 0xf461cfd108d72fc6, 0xf847c991519c331e, - 0x6f3433283b04d04b, 0x5d7a855b88c018bf, 0x139145254a8f665c, 0xa6c1a24cbdc643c1, - 0x4a0441fc756e9799, 0x3db67e8dda3a79bd, 0x8dcd0701383d00c8, 0xc09fde2b46ad746f, - 0xa7fd5f14a3be5172, 0x88aa7886946856bf, 0x6df41ce7f2093138, 0xaebb49c4b56cbf43, - 0x3db9dc96995cc9a7, 0x31b29a8c93c6da00, 0x4c63e4e477e1fb4f, 0x756b55ea39677a4f, - 0xb8edcc84bea0d5a1, 0xb0b0229a783ea671, 0x3b1ec4ae00aa9e5c, 0x80457adcf0ce90d1, - 0x85d604401c90a83b, 0xdb8183c10cbd58e4, 0x9b3cf795a40aaa7a, 0x479ad85728e3a916, - 0x94e91095d71dd760, 0xe905fa435df7bb42, 0xdb275065f61ee060, 0xd04c1b09a4319bdf, - 0x7cdec327135f6f09, 0xe68c2f63a87d1b7d, 0x87478d642a12c5f1, 0x8f2a981b281068e7, - 0xcf60eea69c88dea3, 0x6980a7dd7de75f9d, 0x7eb9e6ee86698335, 0x885fd803d339b9c3, - 0x3ace87b25e8defed, 0xd3c675f1a403299a, 0x82e63c157961bd64, 0x7141d349bf919aa3, - 0x6ef6e294cdc0550c, 0xe4a9e3d2555a5746, 0x2e87176c56f54ac5, 0xc97c150cacebfcc7, - 0x9b20e243d62f850e, 0x5286942fbacf03d3, 0x53cc0db87fea94ae, 0xf548b3ae31e2b97f, - 0x5da07b3fdd12e21f, 0x6bedb1df14ce3b9d, 0x93ae358a4946553e, 0x94abc7ff4018f266, - 0x10abe00593af976b, 0x8057ec6dc812681d, 0x63925f1f331d907d, 0xebd681a7bb8d1256, - 0x5aa08be9fa1d22e0, 0x0f7ea64c96cf0511, 0x662562c82444806e, 0x4ae40a185dbd94c1, - 0x9d3968e70f7ba938, 0x5f881f4a69721b71, 0x9091afb47f1b512d, 0xf7885ac9bf893ecf, - 0x33cb1efe88f4d452, 0xfd6113510645ae34, 0xd37cf694de44ff2f, 0xeb5ff280b7b11493, - 0xcb954699e164ceb2, 0x97d1464ad7cfcad5, 0x75f290b14e9c4726, 0x4112a619d5b25dd5, - 0x076f2fa85fd65473, 0x7bbe73e8f419814c, 0x778de35af34e527e, 0x1a588070b12e536d, - 0xfb36490f6427819c, 0x8e24ff52578492f1, 0xde221a9a3026c51f, 0x77d72bdccddb6857, - 0x35697b1a24ba8234, 0xaefdac5fc1c3d958, 0x8eaf2c832839aa17, 0xb5cc27d67ed90a76, - 0xc73310b2c4e00bef, 0x5159a37641e5ec0a, 0x3182e3f260bf07d6, 0xbbb56b448c9d909b, - 0xf73189b5636e24f4, 0xbb3b173f41640926, 0x976ff682fb1b3f55, 0x2ea50d94e738d099, - 0x9ededcaf185445db, 0x7aa537f8a79c24e5, 0xb2ac6d3a25657e17, 0xf837f424093d2361, - 0x73e345286fc44a9e, 0xf868deb80e1a7a5e, 0xfde4398805fe75d0, 0x80db7d0d57c4e8af, - 0x7021163973fd5347, 0x84b36f66ef5feb8e, 0xa756852ebe22fb12, 0x3d044cc3dfcab0e2, - 0x287226e432edc672, 0x26592697e340cf47, 0x5cc9d88b4d858f8e, 0xcccb174a1c7f6a72, - 0x503507386ff70d37, 0x297a3e743917c986, 0x1a5b0a4f5686c445, 0xa45334749d8715db, - 0x36c7ac287ef8fbe4, 0x60f6e17a89441b01, 0xbc664b82d970df38, 0xef310c6a90f90c40, - 0xe32dc9525f01207b, 0x96b7beb6f0ece79d, 0xe5819290148e39e5, 0x5a1d4782c194fa5f, - 0x528674f75431c382, 0x14875842d441b90d, 0x183034ef2ecae4ee, 0xb28ea719751dd32c, - 0x1f53d63e71615578, 0xf7385084c196adea, 0x24ced28a3528d72e, 0x0b539c0474e73a18, - 0x406beee4cc30cf2c, 0x2a38db5c2d45b770, 0xea4b145581c1dcdc, 0xafd6cddc4e5cf985, - 0x93c6774ca1dfbd40, 0xe61e141648c87739, 0x6d39e6eca3f2a644, 0x6ecf7424c4308e32, - 0x7b71ed72291a491d, 0x90e6c51bfbbf6ce4, 0x3d05c872ce7bda6c, 0x4c865775e74ff6f7, - 0xc9f695d365a87f7b, 0x5fc248b4fcda9fcc, 0xbcee29e64430ef84, 0xaf02531e001b8d4f, - 0x1a04b8d35573e8f1, 0xdb47fb8313b3d49e, 0x48df404200d17dfd, 0xf16a4fc5232de06d, - 0xcf00570cecaf5c28, 0x41ba3aea8d05ec3a, 0x3fee2096b4f1e029, 0x7d28d3a8c1daec1e, - 0x8b3b47a7a28115c7, 0xd4c3075c68002329, 0x844b730f369a111c, 0x21bb5898d33f51a5, - 0xa62b1ef268c00447, 0xe727403bab841fae, 0x78d25b5004a4dcce, 0x565bf6f3795b923b, - 0x58a9f405f55241f7, 0x590adcb520583a96, 0xcc6c883e8228557c, 0xdfddeb10a3731c54, - 0x29fa09f69fe66f6a, 0x59b37831a61702db, 0x28d9ed51555be8d1, 0x471d87feb3e123fb, - 0xfd23eed100957851, 0x633abb187658ad43, 0xd4d83b733dc086a9, 0x81565f012cd7b96d, - 0x727f8d648ee93eaf, 0xe5d2397531695575, 0x9b6180ce82e530b1, 0x93b5c4b8c0bd8cce, - 0xe36f427f893d512c, 0x75b74068f8fe7c2e, 0x663d1d5bcf823797, 0xc45211dc5271fb64, - 0x27fd105f22d7a9a0, 0x2120cc1ff804fa99, 0x01c4eaf68cd50c30, 0xf3d63dd18c547eab, - 0x98b49bafaca46cdc, 0x099191ef087e5545, 0xddd9167c31646df7, 0x55417e531e60c515, - 0x0bb2b7e61f5dfd50, 0x9ffd798287e92b16, 0xd2a341d52c7f6aeb, 0x4c1462a5da3e4ce1, - 0xa6bdb1b41d3c9224, 0x7e90c77f90059805, 0x06447f59b69919d0, 0x2c5e56261e9da87a, - 0xf5e63f4d13f96441, 0x6d991ec2c957e11a, 0x6b402feab2cfbaf2, 0x67412cef64ad2155, - 0xc18a1bbb883f29bf, 0x03b6ab1ae6fb62f7, 0xb23835b313cc11c0, 0x5fc27c2d75a5bb69, - 0x0803fc11649bbe32, 0xa181d87ffb77a4d8, 0x82321b5b1be382d6, 0xa79904127816e1a6, - 0xc67869956a520b51, 0x8606851c143fb46d, 0x3aeedf77f25a0e7f, 0x05623e2e88893acd, - 0xba9be34681a00aed, 0x510a7c318915f478, 0x9b61527f1d7f94b3, 0x9c4ead5c347b6281, - 0x5af6375f64e16861, 0x7d336bc7695c13d1, 0x4d8e5a85c1bd6c05, 0x4830ca3038a5a314, - 0x65b73446240056b1, 0xddfc3dd607ef0270, 0x16f87ce461c68554, 0xaaf43ac6f1476c61, - 0xa891dca0e4062659, 0x8732f710ab382b04, 0x25e1d761aee96d87, 0xabd56f434376b5eb, - 0x43ccf312ba0ad5c1, 0x808dcba81b1ebd80, 0x249e3bc54ce246a5, 0x684e6d453f524b4f, - 0x83e048ce4d1bdcb8, 0xb66553a0f90baedb, 0x910890b413f69024, 0xb2ef70e79c560fb3, - 0x6823b6091db9f135, 0x96cc7a0fb8f367a3, 0x65f692a165b834da, 0xe7357c521d500e96, - 0x684d5ac71cbf032a, 0xd9a726f0ce3279dc, 0xad34f430047f3e36, 0x1d4e372a2ceed63e, - 0xbe4fc281bb82918d, 0xfd5ce0a392151110, 0xd46f2e1af9d76627, 0x0772a938a7378281, - 0x76971f49ec497b86, 0xf5a9547f605254a1, 0x32c780dbcf09df39, 0x580bd05ffae7f505, - 0xd8db1a102d081647, 0x7e63fd7c0699cfac, 0x4c15a7e5aef41cec, 0xda6840b001aeaf49, - 0x2d7e7539a1ac0d53, 0x6709a1ec9f201f96, 0x4779adbaf87dc4f2, 0xc4910409a803bb1f, - 0x685a89f09c35b393, 0xe2306a7d3a7c6aa6, 0x7b0a675bcb05e19a, 0xdd1fed233a6fd0a1, - 0x90882c06940d9bc3, 0x2ae3c98822de6d03, 0x9be6c1a8fbda82f2, 0xe259032abe46b69d, - 0xe1b8d946a284fa8e, 0x93677e4b02cb3596, 0x1f729a4aac5802b4, 0x24ab3dd0ddcf61d0, - 0x34dc4891e3578e6f, 0x6e7471a4a2e77333, 0xbc26ba6a02827cd6, 0xfceb2490b6833c87, - 0x6434a18937675fff, 0x2306a59b7976a86c, 0xbaef50e7fa55f20b, 0x2bddb7618f4c7e90, - 0x4205652fad6ea252, 0x264a2e6aa709916c, 0xa030a2c473d2f71a, 0xf31098854b234ad6, - 0x804e7e9f0c4a9dab, 0x382832339273df40, 0xfcc46b0eb1b54cc5, 0xa94cc451abe97a93, - 0x8cece7870b88971c, 0x1e0c65caa7355471, 0x6c2264cf60776fb2, 0xeaf5ec18a27dbc7a, - 0x36477829fee42bc6, 0x3056362277e6f5dc, 0xb07e5390006c9f5e, 0x9d6aa4bec613d69f, - 0x268dd94e8fb79ee6, 0x58afa579f432c409, 0xf5f36134b3b328e8, 0x99447c50341f803f, - 0x82828834cdc3add8, 0x7ae4367dce33a745, 0xc85c0cc8651aae9d, 0x15081c914ed15242, - 0xdf41a015d2b684ea, 0x9bad6cc1df3997fb, 0x93e10b0ab4068641, 0x8e4d26b88036589e, - 0x4c1df3e9bf0cc443, 0x335d7d62bba635c3, 0xd7b39ac67f3a5935, 0x243bbb7233236e29, - 0x80465d5e7244e221, 0x23f093893f182ece, 0xc2a2aa79b83975bd, 0x711ae8288a3cdf5b, - 0xc959654b62dca92d, 0x408a97d7f15b8cd1, 0x264c96670697c117, 0x63ef135bc7d7532c, - 0x37ed42d2731c7ec0, 0x366455125f4c63f7, 0x666f6303c7b1910a, 0x99b2a2f1e8eba822, - 0x332842ee8cdad108, 0xb58a2ecfc378d561, 0x69bf4f4a98008521, 0xbff90048249012db, - 0xa2e4554d1b9e4a25, 0xbb07ad8f3bf961aa, 0x40af195fc56a3ab7, 0x981307c43341ee4e, - 0x21e6eb7706840f90, 0xca7a7303e985a647, 0xf4ab87438f192f41, 0x093e065e3dad6176, - 0xf7b929bd821b8464, 0x136d600c36b2b23f, 0xca95afbaa23cef2a, 0xe46211c3fc13d863, - 0x360430c9fdfaf387, 0x2edd4fdacc2f6704, 0x245895363cceaade, 0xbfec514249e068be, - 0x1d1eaa5789b43591, 0x81a3f10e930f7da9, 0x87c2e797d57b80d6, 0xd7cb8fb719b6ac5d, - 0xabeb64a4f08de2c6, 0x389eeb7bdc71c216, 0xa9d1de42e78fc9db, 0xe716c1a768c4391b, - 0x825e785d7b4941ea, 0xd1c1007fc6a658fe, 0xca0bfe4d92467c1f, 0x0daf88eb94455bdd, - 0xd7288d84279b589b, 0xa5dd65fa9eb77bec, 0x4d0e0d5896729f21, 0x9dfb76e96118c61c, - 0xb41181165f4939fd, 0x1541ae11df2e6ef5, 0x515cbbdfb11861e7, 0x54cf610e2cd0189b, - 0xe68e58490be5fb81, 0x91566e23c6c0e15d, 0xb5ed267c72127a68, 0x402ff2176df40b2e, - 0x14abefb696515907, 0x80d1b3d0f8526cc2, 0xf078f94be5f119a9, 0x57f8219881be23e1, - 0x74b8daa4a985e2bd, 0xf9579ffd3a01b100, 0x0337c6c555fc3d43, 0xd3cbc8bbb27c436e, - 0xd395e79e10fd7654, 0x3f3f73369153aaff, 0x00415fa2a8938b98, 0xfb8ef65e9907957a, - 0xdd8217849f074fa9, 0xdee3e443edee128c, 0x9a43103ae2eefd45, 0x2fc61b2679635213, - 0x1b3b701744cac761, 0x387cf8f7acdebcef, 0x306877d49c587760, 0x69115e0c0d900da1, - 0x4c23d443d2113f7d, 0x4998c38cf44c7211, 0x7a02cee634de414b, 0xddf010eda93d1dad, - 0x2d4256dd0a4763f2, 0x0eaa4afe6f0d38f4, 0x2dcec3fcaca44df0, 0x5fa70aadd027dccd, - 0x24267637668b3a26, 0xc30d6cf7147c87f2, 0xbe3d31ec89132555, 0x45d3ed4cb50db47b, - 0xad5e6c0a0b985944, 0x8562f4082d0d7388, 0x13bdde9b86f374cb, 0xe10df32ffdfb197e, - 0x4a01208d155e03ce, 0xb85730405595415f, 0x76354565b6ccc8fb, 0xb589a6405defb154, - 0xb8fc5b0cb8bc029b, 0x04865bf0797fd5e0, 0x59a438ee3bdab029, 0x3b52de9a1cf94cc9, - 0xf58ab22ba1d89784, 0x547266c2fbab1853, 0xe15689513baeeed9, 0x4ddf5229d633aca5, - 0x38646771ab65b1d1, 0xafbb9bb3ea018b0f, 0xeb56ad2c2d50e8d2, 0x70ecdbbc7ee922a2, - 0x227f2bf25fa07344, 0x36fd2aa7ac5d8c4b, 0xc2c14545f0a96fb8, 0x8ce7ebdf46f9f473, - 0x9763f8a62f730f7d, 0x11a0e5e06a99c20d, 0xa72c057c92b7530b, 0x35272c86ff82856c, - 0x71482c2b95dbb91a, 0x64c469d7e4a1da83, 0x895b31ddf670da00, 0x61dd19f531f5d885, - 0x3c02a1d9382b3ef5, 0x4628bbc17b4fe39a, 0xfe9aa810181effab, 0x7cd0922b124bf8fd, - 0x3f93894200d40e3a, 0x30d2a6bf3b52538e, 0x0fb14f4cac1e1ffe, 0xb953c821546b2143, - 0xdd8b4506149068e6, 0xd57bc5eacb03af1b, 0x45f96d441626cabc, 0x97aa316053e54219, - 0x1b4d6d2ef20e9f55, 0x98ad5b306a6a2094, 0x29e4b088df202cf0, 0x510b57b23a97051d, - 0x156f2be9295ed207, 0x9f54fdd531b3778b, 0xe70ef32e7ed45e76, 0x468c7348ab182005, - 0x7fb1dd47eae3edf8, 0x4a20f621de4a086e, 0x8ec1b6c8210efc16, 0x629f5e3d7ea79c59, - 0x837ecabe18b75cb9, 0x0ac2f9f6a116ccb3, 0x70f073ba100d9fa1, 0x1c5a6910476abb97, - 0xfb190e79db88578f, 0x2585e6a3eea83575, 0x7993e0b2e8a85816, 0xd7be0787824251e3, - 0xe03bde2e1cede8db, 0xce7934868d62b6be, 0x706ecc60842afbfb, 0xe4fa53e5f474b9d8, - 0x297cd577bf71458f, 0x4a4ae8469d4bd015, 0x89dedd171e4d6f15, 0xfb4563b3350d2c92, - 0x2716d5c1f73ab89e, 0x30e25a3a39dac00f, 0xd879595968ef785c, 0x3ef7788d52aade61, - 0x4fab8ad8b94c4fa0, 0x7affd9374f63587c, 0x5adc1438c60fb54c, 0xc0175fa648b7c9f8, - 0xe939faf7b1204165, 0xfc6c6aa4dcaae3cc, 0xc7a27c5d484a2ac2, 0xa8b935a322f8e744, - 0xf862b34c173cf93b, 0x980a2808ef5e9ecc, 0xe8e65145cf6c4035, 0x75eb874701b88a58, - 0x8e2c85045e7f2024, 0x36b069c8f2bda647, 0xd80a239d51d069ef, 0xc5c907ba6799ac3a, - 0x3534074c86ea3300, 0x00ecdad887b9ae88, 0x307c5e97b55f2de7, 0x23385bca91fb1a85, - 0x48a0cc149bf75c6d, 0x345f59f8074fbf04, 0x5aa0113a1523322e, 0x3a24139cc262ae2d, - 0x4c6f1321ccda02dc, 0x9ae78ab74154a2c5, 0xd11642326e2d2c9a, 0x294b84a0ec1177ed, - 0xeb2ce1ff386cf5b9, 0x0c5939e52eb3aff4, 0x0a90e4b3cef160d1, 0x800b5da8c86ba6c7, - 0x3dfaed35e82f4d30, 0x13a9abafe6b76632, 0x203f6306c3687a14, 0xf49f9699980df042, - 0x4f22932970e2c0c1, 0x43f7f6b6b2e794e8, 0xe2954459fbb8b92a, 0x22ece43749bd2884, - 0x59ccd7b681884e8c, 0x93111d3f52bfaf66, 0xee4b51712ed8a9cd, 0x3403866299e16e26, - 0x787c4918316cc74a, 0xcbbccf7906cc0000, 0xfabc43fe8fea98c0, 0x6baddb48137d5176, - 0xd65d27f59ce13144, 0x6ac3f12084f61776, 0x60d9b1c8a1a6ab89, 0x728d2f0646610f4a, - 0x4da8c28efa55282d, 0x818b23a8e3210ce5, 0x80f47f197bcb1196, 0xab9da675877fa668, - 0xfd8f509188637d3d, 0x88d585a30c4f1703, 0x5ce0fd781c66e7ec, 0xeee4b15d9fcf086e, - 0x49554d2149204eaf, 0x659da7d938223f68, 0x44f46a369292e9fd, 0xbc5762993f04e418, - 0x273284874a109851, 0x3737ab99a012e255, 0x2da06c53b726d70d, 0x3c7738d1b3b2cb48, - 0x9adf781f2213f2f6, 0x5b2b7a06cbff1c6c, 0xb41bfae69e9a9f3b, 0xe30ceacb385567bf, - 0x1edc93b08c78e26c, 0xc3afa196f4c051b2, 0xb1994ff603a07713, 0xf26e2e3a6c4fa6b5, - 0x857ea2100d3c9b7c, 0x45254d96be94d0b9, 0x4707a9d678a5a777, 0xc534beda0013c7be, - 0x5175dc8fa0edede8, 0x974efa8118a30c8f, 0xa4b631773c5ed5cb, 0xb8e606825007bae7, - 0xf18b18a6b97c8a54, 0x46f54f955be77492, 0xad9318fd9b2e74b5, 0xe2a4f606744bea4d, - 0x19bc6d62f86f27fe, 0x69684cfd957366dc, 0x0fc62f70fd9d1662, 0xcacc9d43d5e28772, - 0x7b075f2746355b96, 0xb25e7d1389c0dd77, 0x2e5c66c352d7c963, 0xcc76df5844d61fda, - 0x7f7a48ea108b1ddb, 0xe3781c94c62e39cf, 0x3a88189ceb798a26, 0x6c13240ce0db20c2, - 0x7cfd7ba68755a879, 0x1ef032f9dee3dc62, 0x5fb8ccda58c5cedc, 0x06eebedc2c016eb5, - 0xb1320f7f8f58620a, 0x3d2f3ee3e399afad, 0x5e9292f25a98fbcb, 0x6a731d39e9c14695, - 0xc32e2d6db8ad1d81, 0x0451dda8a19cdf3d, 0x08ade497c5caaa57, 0x502ee7a88a098f68, - 0x91244eb2fafcdcb8, 0x1706a6ee4e6e1905, 0x439dc2fd8d221d55, 0xc3fb4daa8a667870, - 0x6c3d4773da4b71ef, 0xaceeb2355d2c8764, 0x65e31b44972978a9, 0x5203c6b3403d1476, - 0x294f985280a75ef0, 0xe0f05d033f3aec7d, 0x3df31c683eeb8e8c, 0x2b7f3ec3ee59ba19, - 0xc880257415790c63, 0x246a714b11d6ebde, 0x307ce6213362cb45, 0xad806edac73940df, - 0xda796e87178f5289, 0x12c226be6da63315, 0xc97c1710522c697b, 0xd1b01966d04140b0, - 0xf6655e1f1d97f72d, 0x5bce31404a447d9a, 0xba4d3914278c2faa, 0x9670838c424ce830, - 0x52b5c62cc6ebf634, 0xc43e529e663eed69, 0x547c02f6646cb610, 0x26c1bfbe8aa69b85, - 0x1c9739315eae8850, 0x8c49a01368a15cdd, 0xd0e8f187387ef047, 0x03cb7684c0d6dba9, - 0x2b0879cef29c7ec1, 0x07d5b417a2eb5a44, 0xc8297b7929d31957, 0x04991b9ba75e2730, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x801c271e7760ec85, 0xa9317a6387a7a094, 0x520750b4cd1399ce, 0xcbc645dac7eee0bf, - 0xc4d68b0734f26b66, 0xaa42b0d47a3f3d5d, 0x2de39faa76c0d21a, 0x9938ceea34083e10, - 0x0083e12c6a0c51ac, 0xdeadc65894b44bfd, 0xca49a6ceef79a7ba, 0xfb7ebd678c0e36f7, - 0x5a0f75fc070875b3, 0x227e99f533e682e4, 0x3bc9852bf551ad9c, 0x94dc4e40ed690fc8, - 0xebd8e4f176f926ea, 0xc087a11c407aa178, 0x887bdf9de7565be0, 0xe0107bb51e81df12, - 0xb95d477fcd70d71a, 0x80904471f409cc36, 0x0dbb32d4590bdd46, 0x7080bee6624471a7, - 0xfd1df16870c5cb5b, 0x6ac49087fbf54857, 0xe40ab6ebeba46040, 0x7fe5386402befd18, - 0xc0cac70db6df8b98, 0x4b801585455d24ca, 0x5e15599cc3dc2305, 0xdab3e4a3e9686e15, - 0x83535b89d1a88792, 0x74bb1d99e7d7b42d, 0x3381ffab9acaef22, 0x2e035748bf97cdf3, - 0x5e424cc9a29d600a, 0x56c48d25a2e2f9b6, 0x11e6c24b7c57f64d, 0x82b63d7c7a225724, - 0x3d9297af144f5a93, 0x35cf96d8dcbe2fe9, 0x08bce0dc28fc3d63, 0x8146f953fc8ffd80, - 0xfaff0d76f6bbd0fd, 0xe7bfd75363659936, 0x8cfcfeb30dbd4401, 0x417f478acac6e0cf, - 0x135cb8046a496905, 0x8d5c60b74c982576, 0xa3903c43f9357b73, 0x4c687bf2484209d7, - 0x29aa80595c3c5422, 0xa9511b845f7eb2d8, 0xc102fdb4939a4438, 0x2306699ea83ccde1, - 0x6cbc2b983e1e1356, 0x8a1788b60c50bb85, 0x856700d0b3a6e5c4, 0x326db9b3c0404f94, - 0xf8a8b9784beb4290, 0xd0d605f7226a5bbe, 0x13188b88bad882c3, 0x80cc3a5cbab6d0dc, - 0xeaedfff98a4a86c6, 0x6a7aafb9f4829d14, 0x37506d8da844ab44, 0x9dd17dad6878028e, - 0x6c9619c300ed6521, 0x977271bb7ac3030b, 0x7dc0be0affa6c3f1, 0x4b79ba5856a5f181, - 0x11a6eb62037e1726, 0x32087e79cd06cdcd, 0x3e2371d5a604a170, 0x936fde062b3bb82b, - 0x571d24242bcb5d6c, 0xb5229dd650166f29, 0x1eadebdd04f53b1b, 0x523e4111ce815ced, - 0xf14a9f6516e160a8, 0x68d5af443ad0b479, 0xee53cf0d3f337165, 0xe9b4d8cc24708700, - 0xde5ed331df5af850, 0x6605a0902bdbf269, 0x70f621a554c0c70c, 0xb8b520b4f7f22732, - 0x84f4028582ea3d35, 0x81f8238fdc176fa6, 0xc020ca137b2922b9, 0x85ab53d7c41a1750, - 0xb3779c155b5acbfd, 0x89f74b1e9bf1febe, 0x9e99aedd615ef803, 0xb075f5df1359a25e, - 0xcba89d5aa007c355, 0x31f018852a6a536e, 0x0b6d968de79c3de7, 0xf204fd6fba23ae36, - 0x659c4483a1f56842, 0xe7048f0a742187e5, 0x9f0b8acb091e118c, 0x4281e98cd663b5e1, - 0xd966bef81e8c9703, 0x6d36a66a4d0d6ba3, 0xceef2e5814c40d6f, 0x93488666c713b175, - 0xd7dcb0d89732cd0e, 0xd7f88074f17064e1, 0x24cc2008d6388a44, 0x42696228cbe00b27, - 0x5da1ac82b08861b9, 0xc5cbc0ac8f18fb39, 0xb36fb764a940b85f, 0x840627546374fc07, - 0x6417b7f62435b1ce, 0x993fc0b967a23151, 0xb18f0a67180748ef, 0xb578092b7e77c876, - 0xa8d2601ec184fd3e, 0xed83db1bc70c3a49, 0x7a4c7e3296545da1, 0x6869c695f4f88657, - 0x9ecb4569d08a3a68, 0x6c5378a73f00dfd2, 0x05b4855ad4eb47c4, 0x568b230c0ff47c9e, - 0x707163eadf36e9d2, 0xe545bcceaba855c5, 0xbe070296ca521a8b, 0x656e6597d0ad3c26, - 0x82249f4726b54220, 0x25d7bf8bc003c402, 0x1dbca96147046985, 0x62ccd5be08b041f0, - 0xde3efd885b817e2f, 0x18c59744f7002832, 0xdc5edb93f5b368f0, 0xf04767b1dfb6d04e, - 0x20502afe7b38fc79, 0x694df0a5a842089d, 0x1bbdb871220f4bbc, 0x90acfa1af1c08b94, - 0x87cf16ad7c9cb2e4, 0x8934cf0a126eb764, 0x7f14dd61cb79cda0, 0x63022932268f9e2f, - 0xae8457d58b15b8c3, 0x9bae10e47d4ee327, 0xe41d6a74e2ddae51, 0xab19d62f90d56c48, - 0x3f59062a172855ab, 0x3413e1781f474745, 0x05bb12f2c0f34c98, 0x7fe9a13866610a9b, - 0x98c117de6b1c1447, 0x8cb31db8f85e2776, 0x7fa266a4b9ea45c3, 0x1e318becef88b641, - 0xafbf9038a4aa272f, 0xbbc5ba3de18d6535, 0x6b608f02beb10593, 0x65bf674003a43368, - 0xdd9ac75c3cf459e3, 0x0c7aa7cc5179320b, 0x1e7bacd8b8c52186, 0xc214d31edaeb27e4, - 0x6e3e1237baf2531b, 0x0be3e2fff1c3cbee, 0xf5eab3703ff565e1, 0x52d1ffcc8ea6d213, - 0x018fa2471bfbe9b9, 0xe211607243a0ffbb, 0x5cd09e8cb272fcc2, 0x00e039891daf5f7e, - 0xfc81110efca8aeb1, 0x7247e7d90969891d, 0xc60df4302845fc1a, 0xc983ad076c8a8411, - 0xb3b189f52cce4043, 0xcd5b07c93f67cd8c, 0x765d81f0c4c595fc, 0x77eb7dd4d2ed83b2, - 0x3457057796f912fd, 0x353c7747122bac3e, 0x19afe95b9fad0cc8, 0xf4398cb00b315b6e, - 0x570f4a850a1c1bd6, 0xa7acf5534a027bf3, 0x9208a7f04dbc1c5b, 0xb7caf3d84168e7cb, - 0xec3b8e3e4819f98c, 0x53de17187b86d985, 0x4f2f4660426222d0, 0x7d7844dd3a934a31, - 0x6710d84af7c821f5, 0x2ce48f4b79a06f29, 0xf2af15309c1a6383, 0x3cd204772bacfbde, - 0x4e8355f1c49f7953, 0xc1fcedb8e1211c1c, 0xaebd3e6b08f0fa9f, 0x5eac318a0a74ff8b, - 0xd0e0d3c8d4061249, 0xc5ec7c0e42614936, 0x1a1042abdbbfb37f, 0x1f90fc92256c1fd7, - 0xfda090ced6d764d7, 0xe0d11f9d6995f565, 0xff3be31dd37f82a3, 0x8b50155591069860, - 0x3fc5a2611174caf5, 0x6f93b1349e0469f5, 0xe250e357b9c102fb, 0x868509c9a23dfcaa, - 0xc31084953a321735, 0xd850265b971488d0, 0x816383c81938ded9, 0x59210f123cf9c713, - 0xc33f66ead557a2e9, 0xa558d0a86004c6fa, 0xccd332676294427f, 0x90c72d84934671da, - 0x2417a86ce3357267, 0xb5f91da54099ae1f, 0xcdaa8c1e74ee956e, 0xbd9d9b847e8afa4f, - 0x561e9a1ba25e8862, 0xc7ea7bd87f600396, 0x0c3e854d064ee3e2, 0x471e60b68bfd33d3, - 0x96ed8217de0fe7e2, 0xa80f406568173da3, 0xc4875e346009eb58, 0x98693ec8728dc3ee, - 0x343f9e4c69cae886, 0x70cc96a005ca2203, 0x8b6e531e5a8f1bce, 0xf34e5455751b6043, - 0xc1f256bd26ed80a2, 0xe4ca10ca16050e6d, 0x7a7596c36c905983, 0x212e6e3efa4a2259, - 0xcde182a041db2ff3, 0xef4c89adf7efe659, 0xa0f011e28ef150a6, 0x285085b50a7d9542, - 0x3af77b22a8fe128e, 0xe3b44771f46a4d09, 0x7fc4bb0dac7649b6, 0x1c064bd74397d42b, - 0x8ba74560d7001cfa, 0x063e822e4d8473b6, 0x494ba670dcb6f415, 0xb6bab3b73e0b4b99, - 0x07c49745a0bf8c0b, 0xa34dc14ee78c10d5, 0xbe06a7e1904dd3ee, 0x1037929facb9ea5e, - 0xfbd58788639e1c3c, 0xf301d6173d6915b3, 0xac306ee25193ddc3, 0xb2f2f195fe00cc28, - 0x653d7089c27e5ca4, 0xdc9cee1a35f56af4, 0x7bee13e3e2f337ae, 0xfc2ea276aefde915, - 0x86090e01fc9c1a9d, 0x0a53766ee5970b96, 0x9fc2f875815ffcc6, 0x163d4f8a490f2c40, - 0x0b2c410f75d18d37, 0x0e80d686c7601454, 0x2abd8edca1a87738, 0x61412c318ee87d77, - 0xf143ad18a09a9f86, 0xd9d81d9b03cd4cf7, 0x7cef6f3947843799, 0x91be54120ff5eab7, - 0xb2b26353f693045d, 0x0b4e56f98f13c1d1, 0x391d33169c91ac96, 0x9c01476addf99c1b, - 0xad6efcdfd52bd13b, 0x47412f40bf9fa89e, 0xb24cac335f07ba23, 0xbe9340384e6d149d, - 0x64952c8361b80745, 0x4d2b58326d1143e7, 0xe0285b6075261014, 0xa731378b287cdbd2, - 0x96f55253e34150df, 0xfab883020e987cf4, 0xfc463a95d63a73e5, 0xa26c8deb200ede49, - 0xa7cf9dce88667450, 0x52860504cd5c93e5, 0xd87c3d7a45932cd8, 0x433f032dceb0fc6b, - 0x875121590716a5bd, 0x4803b55d327d4fe7, 0xb6e6f39e6469e555, 0xa8684dee46960db6, - 0x4eed134c0a1f402d, 0xdfd7e4573ec5ce18, 0x9fa8c8a2627d9bbd, 0x617bf2fb3e3f169b, - 0xafd991f83e54454d, 0xdc84de07a35e6806, 0xb524da7c3edb37ca, 0x3dcc5c799950adce, - 0x96b73fc380cf2abf, 0x078acc97d354748d, 0xf7be1aa7e172ae88, 0x4aa421e56226f386, - 0x520a1618bc986407, 0x1d38912202329d2b, 0x10ceb5bcdd43f752, 0x4e0a4b88c8d83adc, - 0x3bf741c07e71f718, 0x131dd7a2bfdbb87f, 0x3bee148666ce6840, 0xf90edad8acbd9356, - 0x5a50a244e050f905, 0x4b1a9c864414b83d, 0x129050fa897c37b5, 0x6b035b1aeff5aefb, - 0x3771fc2e3ec65a19, 0xe424fa7246f4a628, 0xdc19da85a654cef7, 0x225ab3e5c93c4bfd, - 0x8b8e97f49c398658, 0x4b1578e17f4bf3b1, 0x110a12aa2e4d47a1, 0xd389d22d4ff9b5c7, - 0xa81e689d87f4540f, 0xbbf9bd5159f3994f, 0x458bd33dba5d8397, 0x534740957a701e9b, - 0x2bcf063b9990b309, 0x0617f7c7bd9089cd, 0x110d61beb89913d9, 0x8142fd7b63c3aff6, - 0xc6bb3438a85a4c18, 0xd1dba4afac7b46db, 0x839908a8bbc3a88f, 0x2bb12cb33d8c5b0c, - 0x28d6cf03f86430db, 0xa557d5cd203f72c5, 0x8c575d8a0b394b08, 0xff61e6eaa72cbb99, - 0x09bd093eada6c6f5, 0xb7377a1f3cf6f67e, 0xd558221107427496, 0x592b31e4d29a75d9, - 0x20d295b1963610ae, 0x08a4ffee12ed8fa7, 0xb86ed87ba9af012f, 0xcbccfcf501609504, - 0x8e741e681d0a1af3, 0x4b995cba014dad68, 0xed3d914714a001bd, 0xff44f2a6690e66a4, - 0x62306cf9322008cb, 0xc3fafaa2e01344df, 0x94d6da4ea7475710, 0x0860dcb584394991, - 0x0ab4e583303e1723, 0x69e449bc973e9164, 0x21bdd58a35e582fe, 0x0b5b30a5fe7d78ae, - 0x7a5c23dc865d07d7, 0x3a0f5c2593bf0d92, 0x93d406d674297a2b, 0xa38a98664ed99e03, - 0x87ffa5989a6896c3, 0x2ed5f62e9ea813d3, 0x8cd0b84dbd68c775, 0x7809069a33b28712, - 0x8466bbcd4413bb35, 0xe388324510467f5b, 0x25b955787bbde7c4, 0x75c27a6c60c461c0, - 0x3b8166432171549b, 0x3a823205668aba47, 0x36c293e582ff04b2, 0x264385f202952d87, - 0xfaab9340864e5c7e, 0x6d889d48e7f4e9c1, 0x8900259ac79f5222, 0xb547b653c88d6149, - 0xd6afdcd3897edfda, 0xe0f05f188bc9f655, 0xb13ce1f9db944da3, 0x87ccd90e72541b7d, - 0xc4f26039c63541e9, 0x02522c719f731706, 0xb4ffb8102d74a0c2, 0x2f84055a785f2326, - 0x29f01817e165ba0a, 0x39c495ee91ceb5b0, 0xd05a01ae7909a649, 0x1a1bb475c1c81462, - 0x7c74bc001ba89d2a, 0x84f5eed2799549ef, 0x4b4891ab4ff50c82, 0x7c89ff42f33bc5b8, - 0x2e5733a2b9f479b1, 0xe4ca58a6a18eff3c, 0xc770e23326aa05f1, 0x907e56fc1f9dcd22, - 0x8d62690c91367bce, 0x34ab1f13d58af911, 0xdbea5abe5cd0b816, 0x573d7096d5f52619, - 0x3eba638c69cc6586, 0x10fef37c1ca2e36e, 0x1259ff3417dc0789, 0x9f1247833f666b93, - 0xd425788e8ac8e18f, 0x913a1cb7130b48a7, 0xaeb3ddd2e8f21e3f, 0x8c5bfbf8e5c0a598, - 0x033484b783f30b83, 0x22698d990b374556, 0x52e2e7105d2f3ab3, 0x64190954c06b25ef, - 0x3e440b92e777dd21, 0xf9acb64cf302f6c2, 0x35a20cc8f356ce7c, 0x7df5171bec201407, - 0x7da508199981e454, 0xa1ec54120d410a2f, 0x339bb539a1750e19, 0xf8850620faa02fa8, - 0xe3714747038bbd48, 0x46a842b4bf3f0d5c, 0x1b3e57f6cac51921, 0xcc5beb16db336af2, - 0xc86f751325d07547, 0x15399118652c8725, 0x37fc1bd5a9eff988, 0x6fbff63ae0a20136, - 0xda6ef7d1e983ec66, 0x44e6b7a75666e745, 0x2dfcc47fd4cb7094, 0x0400478c07bc04fc, - 0x109984dbf506d4b5, 0x9044cf0ed0174bc1, 0x0440099246bc3ad4, 0xf0cf29f287278527, - 0xc65b53c18d85a65f, 0x0961ec15f81ee12c, 0x6dbbb4380a26a235, 0xae5c1761a61bdd6c, - 0x755e3b1630f34aa5, 0xb867374f0df468c1, 0xba16b2309f3e1ce7, 0x3ee62d136ee53d27, - 0x1fedf6d83176aa3a, 0xd51876c98a6f5771, 0x8d980a8ecfcc6af2, 0xd34206b5a061378d, - 0x86dcd9866f98b08e, 0xced6def3b2d4500b, 0xdc77e739018b80db, 0x16c88fca00d9200a, - 0x9f75aea1aba9256d, 0x172cd9e704c0af91, 0xea475961bdb03480, 0xebd93dbd622eccf3, - 0xe26bd41b28eaf120, 0x2180345482630955, 0x6afa80fece323eb4, 0xe25c89dff6d2ea21, - 0xce7934bc6a3aa6ad, 0x263814f9dd35999d, 0xc8ef558fc1240120, 0x7a5f27db9737987a, - 0xde981cc7e21695d8, 0x3129bd5505e8ce4b, 0xfae92b71673212f7, 0xb20496404d6e5c9e, - 0xe2b8f73728358c66, 0x2f33ef89062feb01, 0x3e56665ac9d6b54e, 0x8e51e86d9d72c4ed, - 0x45ff92456332cff0, 0xfeb563ee1e3c873c, 0x50bad587d42b749e, 0xd6a15371501d7081, - 0x19508a14f85f29b1, 0x74f0f985c8aef013, 0x46a779120204c6c0, 0xca00627addb31111, - 0x3b50e7231ba0acfc, 0xb1ac10744a392550, 0x6165fb05e9b2adb0, 0x29ee771c760735d1, - 0x1f6229b0efb15179, 0x378c982cad71217b, 0x09b9f9a9b5d9aa0b, 0x87730ca8f2fbef4d, - 0x6abd36334bdf9306, 0x8d7c2423e05c88f7, 0x53f6c0a6d64ac0bf, 0x329cd7a3b84adbb9, - 0x40235aee38bce5f6, 0x529bdbdd5e74d45f, 0xe0929b3e013a7a96, 0x7fa92cc321f99afe, - 0x47f3fcb6df5b2ae3, 0xadc9c92d3cfeda33, 0xa066ed40bd648fae, 0x0be1af2c320bd8c0, - 0xfbe10effade36b6b, 0x1b76f6727a759aa6, 0x580e252687431546, 0x203ae67bd15a9c98, - 0x2fda7132c1ef2906, 0x3099644e33e3e357, 0xec9d605a50861234, 0xd15d5c155537366c, - 0x33435ec10b85858a, 0x3ada635a9b080c26, 0xf3daee34226fbadc, 0x4169ce251eca04ae, - 0xdfd9dbd01ae1ab55, 0x8286f7c1c03c0ad0, 0x48373008c65b2a1b, 0xb3bcce22b638eb9d, - 0x09fcbfb2e2f722b0, 0xbff9d93aeb68c885, 0x50158c31924e47b8, 0xdf45e27734c91512, - 0x776c0814dade5516, 0x7db74e0618a9719e, 0xdc87aa046837e9de, 0x472ba09c24871f8c, - 0x2277ff1f5d5e63b7, 0xcf54b92bc186fe08, 0xf299681617929875, 0xc3ed1361d1da3d5c, - 0xf53156800386f2e9, 0x8d5d5a1c697e9f1e, 0x749c5cc0ea0df5b9, 0x7309dd3831f0de8b, - 0x417c2e6d818a7bce, 0xc69d54c9c8ae0632, 0x273c297c50004193, 0xed2a2f085888bc8c, - 0x395520913bffc09b, 0x3eed80d0f50c3048, 0x25b9099f2fda0e92, 0x6e36f9a521b7eae7, - 0x0dd3be978dfbceff, 0x5f0f64f8bb1991d1, 0x4e56d18b58bc7d98, 0x379868a8ec621302, - 0x26391fd3442a9989, 0xdc09c1617c295986, 0x301d690cfdac4dde, 0x5f73bbe01aac5d93, - 0xfa1d13edfc5b2ae8, 0xc48de27e576bfdeb, 0x66705d34b617f976, 0xe7001a9cabf8447e, - 0x1fb75d41de6598e7, 0x20a3dd4c53d71bbc, 0x3b0b5e12293ce057, 0x4add38f98989f5f7, - 0x7cc9beea66794cd2, 0xcd9b5ee7bec63649, 0x22464ceb180b9aa7, 0x5feab28127b2d474, - 0x45e2c29d030a55a7, 0x28a3bb0b4a01ef1a, 0x4c827ea69a951eaa, 0x230a7f2275c4d7cc, - 0x248bd6136811f6de, 0x3686339bc9bba600, 0xc6f049640a7e2279, 0xc4c82adfdc05b2e5, - 0x1a92cd8e300fd810, 0xd963dbc752e66e72, 0xab6213eec4e484fd, 0x625d6c18e0842db6, - 0x8231fc966fb9dddf, 0x7a335f0cd0bde9d1, 0x5958a7f218b545aa, 0x9bef1b58614e962d, - 0x5e6cbd175b72f0d8, 0xd9b5c67689142bd3, 0x30db97a59d952a7c, 0x76c682147f4ecc23, - 0xbf94b3e33f2fa069, 0x0782a62a5295fcbe, 0x4c13c22294676139, 0xe951b830ac104a75, - 0x9f239a1b3b905a94, 0x068301fb5f36cee4, 0xd72d66df3119e0c3, 0x5d96df06fa1d4b8a, - 0x863493dc5857f51d, 0x8c69467176f3d612, 0x5a66b5ff5bd73dab, 0xb8a965f931e7fe6c, - 0xb210a1fd24961b3b, 0xa63b6c125fe69945, 0x6206b098675c14f6, 0x99f82a0a3c2a1594, - 0x441bd372dc231074, 0x474c18cac4394d6a, 0xfa79786e7b691b7a, 0xc96d8147f3f21693, - 0xd642484c754e3abc, 0x18e53d06671cae01, 0xc948aafbbc419aa1, 0x43463e4d8ad9f102, - 0x78ae9be8905e5d8b, 0x9438264fa598b545, 0xb54762bc69a8b53a, 0x73f5e1c7304bfb6a, - 0x9a7c9e7500eb2303, 0x82b21eb66f7e3114, 0xc89311ff5429aecb, 0x04540d61fdcfd68f, - 0x0e687e812899afd5, 0x1edc4760d7d29274, 0xc9a2ec79dfc1b6a6, 0x808910d1e64bf954, - 0x0cf017773c366a0b, 0x33b512aa4f45ac82, 0xe7a3898e1450ccb4, 0xde10ebeb9d600f47, - 0x452bc4a1cc1931d7, 0xc4f8c33a2ba7cd9f, 0x680abf39fb259d30, 0x2ff5411005ca3d83, - 0x28ad488b36e4f7fc, 0xf32a0490438764a5, 0xc96b02cddfbb9826, 0xc4f46927e5734f46, - 0x5b5587b52aa68d89, 0x7300f8700f901ea5, 0x94365730fce49a3f, 0xbea377678daaf17f, - 0x6c6b19d6a3e81017, 0x987360e366328761, 0xef33dda5cd54f4ca, 0x257a04cdeb3639bd, - 0xb7003e018652ede2, 0xa10105f8f7d9cbe9, 0x7717a158347b73ed, 0x12392b607dcafaec, - 0xfe7146542d8de310, 0xfdaf79533f8aad37, 0x6799d6c0e34bcae3, 0x0bb8bb4af76c2deb, - 0x2e9e21904cccc003, 0x1316143a18046be3, 0x908ab828987f2f14, 0xa00ce308c1224dec, - 0x6e24e678db7357b4, 0xdd2d8478a942efd3, 0x5bb3c38cfeb78389, 0x34fafb46b3574009, - 0x66d06c7720ca007a, 0x4d0e8d4c4b5c805d, 0xa55595eb32ddd2e8, 0x7dc0cfc18f9c5e61, - 0xd166f00c6508ecd5, 0x2b2d5f9512df613e, 0x56ef13345a471148, 0x4da4517efc823011, - 0x8e53fdd527170d8a, 0x72a480e08de8729b, 0x3aebc0d716bab895, 0xca2ac40cc90fd573, - 0xa46d3f679a56eb6d, 0xfb71952e3c2af8ef, 0xf1ae5ea3cea789ca, 0x6e5520149af38cad, - 0x06cd44ac96246581, 0xd58a1082669eb147, 0x7791071bc345eb2f, 0x1fb37536a562b04b, - 0xa0c77cbfb68d50d8, 0x6835846de1c9eb79, 0x1c10f1b85e836c53, 0x4e5a6fac2ae4fbf4, - 0xcbea171f48e79989, 0xb8685330695139fe, 0x14985504b498ca34, 0x9a5a11c74c6e8c54, - 0x7f09c543205f0002, 0xfed94fc034159367, 0x43d880d95b61c2b0, 0x9230cc07c96cb9c8, - 0x342cdd7459da425c, 0x32b4197eb933ee08, 0x28fcf27efff09b89, 0xa220b1eadd974b70, - 0x838812b638aabb32, 0x8e3a37348b3fb4d6, 0x0c35a4b39654b654, 0xa9a1eb64d8173a2f, - 0x2ac377ac64aed9ba, 0x3347ec252311c9e2, 0xb3382258e77236e5, 0xe072bfb48557edff, - 0xa5df62b59c594627, 0xadaa3c35bfa86bbd, 0x050ac640e852ddcf, 0xdba35b9b79b5fcad, - 0x9ce7bcc219bcb7bc, 0xb295e0bfe3c1edf6, 0xa2f5f402b2dd57b5, 0xaa3202932e2186d9, - 0xf8c6ca1cde13e4a1, 0x1252d584a0265b4f, 0x3b2cf48bd7b521f4, 0x7a06b32bc4122e54, - 0xd012909a2daea6f5, 0x7bdf7e5da331f9bc, 0xff1927e7e1230da4, 0x7747f4e7414b814b, - 0x6708cf474a52e781, 0x5523956705f3261a, 0x035a77f43cec317a, 0xf75c1e79856955d2, - 0x351cc7cfbd592f70, 0xeb5db1943e8b940e, 0x209e79b715e9700a, 0x27e29b356aca1fb1, - 0x4bd7ca2ec64a05ab, 0xbe41375e0bbc294c, 0x1d70f15f9abe25ca, 0x30d0b1e74678bd69, - 0xeb471cdedd77695f, 0x4760e450a6c66968, 0x6d9162272ef2c297, 0xac921106513423a2, - 0x9b2f16392ac09ed3, 0x37e739f111cbe42b, 0x6143d85026de83a7, 0xf7270f1126030471, - 0x26690424713ff5f6, 0xe957a81e9a84ee4c, 0x698a9e98d6b98ce7, 0xb9b983da4c2d45a6, - 0x958d8bfe275777a5, 0x82dc33c3d6baad37, 0xcc037786296886a2, 0x475fa6058d42c976, - 0x4dd874421124f333, 0xd723862290301623, 0x379e5111f5bf9046, 0xee84a1db6e690dd9, - 0x7a3e286c84a854ae, 0x57c81a25c49515a9, 0xd63afd2743073fb7, 0x612f5111658c7daa, - 0x8b9df687ae7b6b48, 0x86f05d4dd543e18e, 0x774bc579ff6b6405, 0xfe45708eb1a6ccb0, - 0xa5d4e4364ef9c041, 0x04c88fbb1aa9c547, 0xd0e4bf7c8d39e62c, 0x41f860fbe8fce06a, - 0x2de83ccbfd1ac094, 0xc3ce2f80e0fce8e9, 0x86ed9b1a0c8def0b, 0x116e0144426adfb8, - 0xfd78921be5167fd5, 0xcd83d92b52136cdc, 0xabfad61c1539352f, 0x3e4cd550df4043f8, - 0x5754cfb87bd4f227, 0x2c24b4ec109bcad3, 0xa1dd0710b71ea5a6, 0x43e638a5f818fecd, - 0x12f318e7df523095, 0x33f7a07a7c723304, 0x2635a30153ae275d, 0x37c7df094a48d164, - 0xa8d6c8f952858308, 0x7f4623a6053b2556, 0x0dd25ae2a758fb51, 0x1d469fe8a5bde333, - 0x6763d1c9618f39dd, 0x041bec6279c756b0, 0x7a741d8733805fe8, 0x4d70defd6b327324, - 0x88c065a17ad19c3c, 0x24ed0da3a5edc777, 0xaf5e60604c85b16e, 0x65f88eb2a454f6fa, - 0xb13574b2d728ef34, 0x60c9bc022368155f, 0x0d9bca5081e50bf2, 0x610485d31289b338, - 0x397e225f601bb365, 0xbd1aa086053a48be, 0xf4572d22db8f125b, 0x4aad485f7aac325c, - 0xcf276d52ddc33b55, 0x637bcf12ed43fe58, 0xaaa35161486f1b86, 0xf5ac5165e47df597, - 0x0da32536b5e5c35a, 0x0ff43cff5de2a856, 0x872e23dd3f25435f, 0xa303a8f7b2697030, - 0x8934848e9d8fec01, 0xdcbe55dbc5eb13c4, 0xb1f412461b133130, 0x66f665175f207669, - 0x0678c27358912884, 0x4de84690f1312d37, 0xad88091c91f53094, 0x429a22c72a2a488d, - 0xb3a4122829259000, 0x002f99695410243b, 0xffdc89e0ab42822c, 0x08c94a3a89d895df, - 0x063b74450488a225, 0x1730c89ef2f3de37, 0xf0c1f0f99550778d, 0x454db280618eea7e, - 0x787deb2187ba379d, 0xfde199c7d7eec3b5, 0x09649f3da517d5db, 0x2fb9e6e4e51d5516, - 0xf2c97f9d3eb6c6c5, 0xe2fb63af84d5a623, 0x9fbda64c6baa62a2, 0xced24060270e75af, - 0x70cc2f97b7558f74, 0xf08d621c66630cf7, 0x3600fc580309138c, 0x50245e7c2236c75f, - 0xcc74a3ff00906ecb, 0x482b1a4de69b4a39, 0x6ba59e50f4c19a87, 0xebddec17fd4939e3, - 0x363600ddc796d9e5, 0xaae7d65a334c1678, 0x4ee0028825c61b6f, 0x144793541f59a194, - 0xfa61931e11726228, 0x19d42d170c4eac15, 0xddf5b943c269b683, 0x8a3fb2e8a4af6c1f, - 0x9e06ff8c378712d3, 0x0a64f84f4cb296e8, 0xf0b946488100011b, 0xa2f24aed45e4033a, - 0x060d6c531f4fa9da, 0x0dcdcca148160e2e, 0x7b15fdc0e51c056e, 0x8571c8893169662b, - 0xebdca8961db24d92, 0x592dfcaf7ae4d2cc, 0x453df6cc16dccdcf, 0xe0e4c14eabd00b9b, - 0x30e174c27ddac36b, 0xacd1b94f0a59c9e8, 0xe58b3c59fa53ff33, 0x5fbf90292afb7c92, - 0xab25d318867b753e, 0x2c3c121c9f5219c0, 0xa03e51efcba647b8, 0x538764f41e2bd44b, - 0xffa2883817f3e65a, 0x1dfd469cf7f0d3fa, 0xa73aa154a882e76e, 0xca4e32f72a101a58, - 0x84705f46b57d38d1, 0x6779821636393ab9, 0x241f6bf6246a91c6, 0x6c919a90dfd6b402, - 0x523f289e909d90d1, 0xcfa7170f9a779f7c, 0x5809befbc232c78b, 0x2313fec016554c73, - 0xd44b7f39b8b4149c, 0x76420e0d84c7d8e0, 0xa9f52f04db9045b9, 0xbee49bb2a137a8cc, - 0x59e2ed92623a86e2, 0x43f6b5ffc7c67e4c, 0x835a3c38e8d34b17, 0x1a2f600c8211912a, - 0x87ea59d910626e7d, 0x031990ad6f95c7a6, 0x50501101fb6f57c9, 0xa7415b33eaf73746, - 0x91a2bd5712227951, 0x864ea8083ca1cc96, 0xc6ff13e16598819d, 0x4f5d6fa10ab62f67, - 0x42cfab2790418130, 0x24b17b76bae4fe40, 0xa602eb62474a626d, 0xe424a91b2c023212, - 0x0d009a5d28175fee, 0xdf4bdbde09ed8f18, 0xe8062ad2f530d710, 0x6a175a1ad4f8eb59, - 0xade5db9504791e41, 0xf3de46739a868e3b, 0xb9218e693ae35116, 0x55d5c03b6b44aa63, - 0x62538d876da70dac, 0x8258bd53598bca63, 0x61e5d9970dc18902, 0x9bd7437e106e5908, - 0x2600057796ad5ada, 0x9e92d79f24ffd81c, 0xc6255ea184b4f129, 0x7fea5339a1b5c6a8, - 0x98b1e237f2cd6ab4, 0x44bfc24bea0dfebe, 0x310c92f4472c9afa, 0xc13672b884667927, - 0x1a19e9a878735364, 0xe626fe223a731e41, 0xa71e30710daae338, 0xea56117241f53416, - 0x4df1b81b833bca9a, 0x1a833c73b68db9c5, 0xc40118ee55ac2bf0, 0xb569ad6d9fd4d209, - 0x1c46476562b2334b, 0x9824ecf42f735560, 0xa8396b244e7be945, 0xf659aa22d2aec99e, - 0x032a375c2f764cc5, 0xe99222b7d5fd6609, 0x8af2b9338f51dfe1, 0xdc14cd6a9cf9182e, - 0xc2aac8a5fd198d63, 0x11520c8a1d7b42fa, 0x4f5c957aee98ba2b, 0xf8c73221ec3056d2, - 0x863ae3fcd4926155, 0xdfa1d3551bde5a0e, 0xe7ccd7cc5159743f, 0xfff12d0cda6e8215, - 0xa4123a90279ae8b1, 0x8c188230a016c60c, 0xa36d77d73b89772e, 0x58e28b702e0fee8b, - 0xd1abfa4e92028f34, 0x1d92d71df471f4b8, 0xeedeb23f952c8bdf, 0xa9110a198c3f0aa4, - 0x63ae1f52441530c5, 0x063f04d37e85a8ee, 0xd38a5441fbe5d452, 0xae240924ec5617b6, - 0xe96cf335e8968a36, 0x65a76bd10aa4298f, 0x4b58e24f57236a44, 0x7ecb4af94882339b, - 0xcaa7d4b52545f79c, 0x8d4adb8a0e6a8593, 0x99311288301c0d8e, 0x621496bb3f558f0f, - 0xa6b390072cb8f3b2, 0x9e9ec60586486e9f, 0xda2cf64ba06ee61a, 0xf0df488a0f065f61, - 0x9d6d2eb6acd75845, 0xb19b59801bec0938, 0x935f1ebc7af8cab9, 0xc82533ad41af0c33, - 0x317ef2b08867d1eb, 0xc09e3b6abf8ae567, 0x05ba1f408101d52f, 0xc3a0fb61de785a66, - 0x27313066704697bc, 0xfb7e7c218d400da7, 0x5da9da2ee881f599, 0xac9e0f8d72b31399, - 0xeff3c69fde28739a, 0xeb87b4b643590737, 0x45826f833feddc0b, 0x9dcaa555d756a8b5, - 0xa18d387235d3b972, 0xf2a8790ef300f856, 0x4dfef664672e214a, 0xd4febea00eef6475, - 0x9a97c7d97ac67707, 0xfb3f5e2d15d3d9f8, 0xe52754c8da1677bb, 0x43e36f42ccaa4933, - 0x9613d6d82e183609, 0xb6835fca18c1ce9c, 0x7bd7ef4d73b8d9f9, 0x6d00b20b29962b07, - 0x5e3cca4a840d0717, 0x659cdfaf7882c9fc, 0xdba27789f4758c32, 0xe8d0398ae794ebb1, - 0xb9c4d1a94dc4adda, 0x861f452c6552ed27, 0x820360b94da62afb, 0x8115f483cb399862, - 0x237884e299398899, 0x33de616ba36f5232, 0x8a49c6e6e6dac91c, 0x852a28316d07e48e, - 0xa042ecc71f823cd1, 0xe81facf64860d209, 0x5ba68b6b4f32a1e4, 0xc99102cb20c610c9, - 0x1126ece1b950e891, 0xbf149ada30137605, 0x52381e330f946f93, 0x95132a0e38acf477, - 0x7fd1c98b811bde3a, 0xddf18b3542ea4c9b, 0x802db7c1feee204e, 0x245775138833095e, - 0x40ff552ac4bc5d32, 0x0cdfda4b0638bba2, 0xa606d196cbb8c694, 0xd9d95fcdff5f7451, - 0x8b5557d6533b69a6, 0x8e88f9850cc2f852, 0x684a9e695b242115, 0x4876b7d3347a81f8, - 0xcbafb295bee2b83f, 0x0a5484a1c7780fcf, 0x379a84716b85f9c4, 0x5c3635f9756d00ac, - 0xd24deca07b61797a, 0xccd37c60cc9e3e8a, 0x94a31d31fc2789f7, 0x8214df5b2bbd2f68, - 0x6254213323cab4e5, 0x94be7e8dff8e5634, 0xbe1bd0b7871af504, 0x240c21bf719f816f, - 0x01c3c63066908fb3, 0x1d9e0ccef998da12, 0x5f766521e12bce37, 0xb270c10142ecfa23, - 0x014e0f358be85d68, 0x54f2b42c59fe38c0, 0xc24b58e3a51e7da1, 0x30592effa54aa2de, - 0x4143d2f2ad13d11a, 0xc1ded44911227f5e, 0x1fa501b45c6a370c, 0xfe8cd4bdf58908af, - 0x795f0aef03db7b7e, 0x94c58119d9e05de1, 0x06e5834300d7b908, 0xc824aa02f4fa9a03, - 0x7268ffd3c9b4c154, 0x55098bb851a26978, 0x0b6cd048964279b0, 0x78f9250b5cb5b86d, - 0xbe0bc33281e04428, 0xa3b4c7292a9637d5, 0xe5bd5144205fee15, 0xb999f068bde150bc, - 0x731695a867103c9b, 0x4c5dedda2a413479, 0x19860c4c68cf56db, 0x10613609c0e7ef69, - 0x28b6782fc15876cc, 0xe167c7892707727f, 0xcbdc49ef9d4aed08, 0x7aaf5d233ad39d1c, - 0xba1a083965d2a7a9, 0x7233a9b59210d680, 0x9868e5deaa900af7, 0xfffe4e1b775f163b, - 0xd5b11cc522bc04a3, 0x675d09684778a5d6, 0xf7f6fa6de487cf75, 0x700153765f780edf, - 0x5463ce19cff4f6d8, 0x3e00c8bf52f800b5, 0xb5b968ab19460284, 0xa26ebbe50f7bb10d, - 0x0ca3a0a275b9894a, 0xef81eaaa40281ebb, 0xc47ffb9125125c63, 0xd9ae3b433f7b0852, - 0x6ecbde3e063ecd03, 0x2caf0e1e1f13b480, 0x8637378dbc3f8ae7, 0xfcd393580ea5f21b, - 0xb9aab423906d70f4, 0x9ca2c07ef99df892, 0xf52843628826fd30, 0xaba37ed84ba81250, - 0x43d56fba2a12d0f4, 0xa3617a11a6973362, 0x380b8b016da5c85d, 0xb4a2e1ddc5d5f348, - 0x2a1a903ac4c79209, 0x3035042671bc6454, 0x03555c165408e963, 0x154c97f28413db39, - 0xd6d50dee18186c78, 0xa6e5dfa031222c0d, 0x3aed9b26e827673c, 0x64a6f8f56f2db5fb, - 0x889a7fa8d0f04dcb, 0xb8f4e5cf13109b35, 0x1e63f37590369cef, 0x559fa563efb5ca3f, - 0x9aa15574406f0536, 0x4910b6462567d60d, 0x666eab8030bf51b6, 0x478c428ba59e8b51, - 0x6b385f255d12ed39, 0xaf97b8d21d1c242f, 0xd6b9f9426574b3ed, 0x5490ff7f4159cda1, - 0x1ed0fb9337e39679, 0xae5fe635e1458915, 0xc0c8610cf6ccf6ea, 0x338268ac72c33b40, - 0xfa6b4a1f79eaa19a, 0xc94a08daa8b626f5, 0x330a64fec8dfd1c9, 0xcee9885dde949eeb, - 0x6c7a90b78545cf2f, 0x8c361abf391bd857, 0x8b2d258c91707521, 0xd1251b7eb6c92cd5, - 0xe18cb1350fe9e506, 0x3c6f264c64e021e6, 0xa904d9fcf9e98505, 0xe2f3179398d9e255, - 0x41fc05718e0efdca, 0x18694512f11e6f5c, 0x796d7d6de3c56cce, 0x8750c809f3187256, - 0x27b1ffe3b720d5a6, 0x549f16728cf85624, 0xf1e46a57f86fc22a, 0x2aa19c51650117bb, - 0x6ebb3384994de433, 0x8013e7b71d225d4a, 0x98f71e1ecb1d7e05, 0x5d3a68e0d70dd717, - 0x1016dd44033e6e58, 0x45655a280dd2ffc0, 0xac445c188cb83fa5, 0xee8794bf6e86eea5, - 0xe0b701793960373d, 0x0b5ea77784410900, 0xa559e735132de5aa, 0xf500e32b260c885a, - 0xb1e8576c036a135a, 0xccd8a322cb9b5454, 0x9c08dee4318a6c7e, 0x9c66212d00c5ed65, - 0xfaa161b2f0d8a84e, 0xd661eafe830a544e, 0xb2fbe19e4a001e57, 0xc69ddc3849e5a8ce, - 0x0202a97a4550a1cd, 0xbb5201ab15502834, 0xcc024394d4c14584, 0x41011d88f2574bee, - 0x6643243206eed0e6, 0x11556256cafd8582, 0x70f8253dec0926bf, 0xe8471cd32cc6c2ba, - 0x10e92be519994072, 0xa804e4eca3282925, 0x3504ccb577bc7866, 0x9578d853e6ab1234, - 0x1baa928dfd2a1894, 0x0543d2f849ac5c03, 0x251f607478d7e630, 0x39522e3fa80f8d35, - 0x018851a699b8154a, 0x00a01a798478e8e2, 0x4364dc7af98d9beb, 0xa20d40c41cba6f59, - 0x5cc492bd0042692c, 0xecbf19fce779da87, 0xff74a5fdf5b8acb6, 0x31f86a5c6add803e, - 0xda9e08a088049e35, 0x5587aff23b2058be, 0x14cb4446f2d1c3d1, 0x4671dc16e06bb9a2, - 0xda2538d25708e607, 0x909f4e64bbe4f9f6, 0xfd972f9b74acb689, 0xee5c9af65935af84, - 0xe6e1785416c52875, 0x5d02fc7e10041ffb, 0xae1673cb2a647c3a, 0xe1fe785a4c0b0ad4, - 0x083a329b97d0baf0, 0x23264b5a053a2ae5, 0x043cd21d02a08b4a, 0x706962b35833d46f, - 0x8abdf4b88ee771f3, 0xcb69e30641684dcb, 0x730daab85ccd3253, 0x30dc6048d99e65d3, - 0xe7874e20649af92b, 0xe396d9ca45aea0ae, 0x75250cd13887dad4, 0xc89d2dd53ec7465f, - 0x4f399644b67db281, 0x4ac371648c34c3da, 0xd857f45288b347e0, 0xea3ba39c166ae08e, - 0x3859af1f086cd7f3, 0x8293a3c9d61a3fe7, 0x3e7b025fa147730e, 0x224737783a94211c, - 0x32cb4a211cf7972f, 0x4276f35837e10451, 0xe052dadd1e347823, 0x85a1a94f1098c375, - 0x0827e7e5e855019a, 0x833080395f045293, 0x5e2e70b19bd1c5a7, 0xeb1097f7683526cc, - 0xc86ecefa804337ef, 0x4bde63709aadc777, 0x22a684087e4a8010, 0x0f8695cb99ca5e01, - 0x1e560d8b67f7bc74, 0xefe51845ba56e760, 0xd030f86ed9ee6ad8, 0x2ee42f15b048b3cc, - 0x3fd00c265c84f2f4, 0x85d4627a151bea4c, 0x3a6f606a482f1859, 0xd2e3fa4f1cf5b0f1, - 0x716849a30aede00d, 0x08b19859432b9752, 0xaabdd1ed72739964, 0xbc4daceec4df66dc, - 0x6b831158978777ac, 0x8328d232ba98a4a6, 0xb7f8f781388b7de4, 0x2c14cadf52e0388a, - 0xd2266b6da5a1683f, 0xfb0e7886b1fa2b99, 0xe2da049dc11f10c3, 0x0d43c3d03eda76d0, - 0x33e15a035ac05914, 0x2c19b3adc49c9176, 0x75185435bf9e2a5b, 0xcf281166491aba61, - 0x00ca1f0aefeef585, 0x1745814208184064, 0x451ba9f6b4f49551, 0x638cd6654f397015, - 0x0212ced2ce4c98eb, 0xbc05e6d2b2708cdf, 0x19c1f2140d929467, 0x3203eadda12e3174, - 0x6f67ef9cf81497f5, 0x9e402adeb5f1457e, 0x454b53629e60af41, 0x8023cf7104e9933e, - 0x36def0cc437a53e3, 0x8458074605a77a48, 0x77f87ba28e26b8c5, 0x3d542cc239167542, - 0x31be47f87d56c70f, 0x726f8c81e2c7f4dc, 0xafc7a0d93c66af34, 0xb61f0b62253e2d3f, - 0xbd773fa717890681, 0x40138543ce601c2d, 0xcf73e06624188758, 0x391dc28fc9ff2f4c, - 0x922a4f626675a851, 0x569478df62af0ddf, 0x89a7d06f32a02059, 0x625f2b869ced41c6, - 0xe1c1ff7d11d3cde1, 0xa9d48a4810e5c51a, 0x3c134a91e675bfa9, 0x09d8c7ed98218c47, - 0x12194e8028971e69, 0x55603e2de2ffb98b, 0xa925cc47c9928d9a, 0x80f9848802d50eb1, - 0x2c6ecfc2540cd507, 0xf2487cea7b1498d5, 0x66ba7bb95700ead9, 0x521d45543be69284, - 0x2d8d3561e66f22a1, 0x591e96366b295a83, 0x81a1ff8a05a7e5b2, 0x028ec9d94da3609b, - 0x28ac6c61fc61bfd9, 0x8250708c4ef07485, 0xc8125f1ab284b359, 0x13e6952a62e7bbde, - 0x623c56de1fd89c9d, 0x4e20acc54aaed848, 0x4ef457dbf0efecf5, 0x41e38ef446680dda, - 0x6197eb1528b376db, 0x13b39482fde8d77e, 0x109750a3314c391c, 0x9c70b0ce5f5ec51b, - 0xf453be341849e91b, 0xb8396b449edb9546, 0x0022e468baf82338, 0xad9ee0903f4929e3, - 0x5ec1234cb41517a5, 0xd141db281f004390, 0xc15625feef7ec18a, 0xa90066e00ea03aa8, - 0x1e047bd6711613e2, 0xd793c2026c0f2e60, 0x7abf117905229c8e, 0x226a709515315f70, - 0x5f9684d3c3612a1e, 0x562a30fb77554921, 0x97ead97b7b7e6b22, 0xe5cff3adda1cd02b, - 0x655f1af80fa15186, 0x7345face5c4482e2, 0x180c9c070c3ca83b, 0x1b658d13d49f8ac3, - 0xab58549b16a3887f, 0xa73ae60ffef6fa1d, 0x04115aff4d29c200, 0x26283a1bcf4ceac1, - 0x6fc3ae3305646025, 0x19dbb964e49be7d3, 0x2776df09550c1a39, 0xe91513f837b19cab, - 0xd20087dd46911281, 0x4a4569bb7e3ca076, 0xdf3d518e5697b2e3, 0xb1bece4faea8ac79, - 0x876fc66d101a515d, 0x0a1c0d78416692ad, 0x3886cdc1366045f4, 0x8b7647f44d9eaf43, - 0xc5a86f03d26d553b, 0xfc12ab5d91cee813, 0x7ad9d3998f6b0af1, 0xc778bfef06183622, - 0x9eb86932a37673bf, 0x8edb9abfc8f54c5d, 0x1bca142da9f38a19, 0x36b464c1d0fd37ba, - 0xee60f33d53fc2464, 0x727a8248f31ad112, 0x024ef6fd4bd75356, 0xfd90b916b572fb55, - 0x57078fb4b066503c, 0x1eb207a666c15a2e, 0xa785b22920436534, 0x77ac6e3d2bd96215, - 0xe630ee0b3ad5b288, 0x56f37915a453023c, 0x1c3de3dfcb8b4eb9, 0x479eadfe07352d21, - 0x76b48fdf728d1cfd, 0x74f2e71cd052f72b, 0x161747d06180c1f3, 0x054f22b764b05d94, - 0xa6b35c301d4c4d00, 0xad118c64d632af3b, 0x9156789f021745b9, 0xacaa151ffe95587b, - 0x93ba011a15d3f552, 0x6c38c7596e86f2e1, 0x4f2786b8c8dce81d, 0xe2f87565bdbfe0e0, - 0x29107eafedb6981b, 0x2fdc28728c80c606, 0xbe20c8463783b13a, 0xaa06cb808ce0037b, - 0xf3aa9ce4103e6f79, 0x907a939887d86dc7, 0x677178e8bced5b57, 0x5b1582a63dec5a4b, - 0x7d26d21504c621d7, 0x6ed2f3a34e3799a3, 0x53339e316d3008d2, 0xb4d11dacf011839b, - 0x4e7e388eabb29a89, 0x49097f4c623bcc8e, 0x722b6dde507cc24d, 0xa5326b4078cbb89e, - 0x491ad8c6e249dc3f, 0x43711c8337922bbb, 0xc5b51efea0b486c7, 0xf1ce1df772b22fd1, - 0xc403cd336be0b1e9, 0xb9146efa2db9a078, 0x61bb5b6afb229709, 0xfbbc021cfe25a799, - 0x1cfed07a04862332, 0x1a49b9af69536029, 0xb3ed4ad3993f88f8, 0x946802c4952d88dd, - 0x269a05fab8481f81, 0xedd3b5db65035079, 0x0ae491c1baca615a, 0x52000e1f174c924e, - 0x7f9192ba11bb2b27, 0xc97c35caf0344d14, 0x231de837bfd5221e, 0x5da764e851f442d3, - 0x76e065794601d62a, 0xe1b37c191213c10e, 0xc5e1856069aab680, 0x972a3f8b240d392e, - 0xa2fd25f6c1955f72, 0x54fb3eac001a7ad0, 0xba1531ba6d2f3936, 0x3fd58d0818ab0313, - 0x8c414597e15eb94d, 0x0ee41b52e0159459, 0x602b8a0189bd4ea8, 0x2c716b2e60c86107, - 0xb338446de8a48989, 0x0690883dc5920ded, 0xa40ab53477cf4ed2, 0xd576b3a92294ca76, - 0xbed9c2154aae2812, 0x8d37a9be2e47bb8f, 0xa20d4891d9027792, 0xe6eebe8f14af176f, - 0x4771974c22c03dcb, 0x9720809ad1da6154, 0xe5fb9cd6e3efb18c, 0xef80997d1a7f437c, - 0xb8f86924e3d9e2c8, 0x177fb4fb05173157, 0x099d6e7b998a734a, 0x2ab47ac36b48e367, - 0xfa6500074c71bf62, 0xc2e6b4ce767b0496, 0x01e55f7fa760326d, 0x7a97a7e56a954d88, - 0xbe2962d62a04185d, 0x888839de9820e529, 0x500edf47e0e04041, 0x3796395868aac21c, - 0xfb2aeb98e1e48f94, 0x6693dd65032fe96d, 0xe1bf07d6d6732db2, 0x7c64fca529adf93e, - 0x40bf58f1183b2a12, 0x1187d9a74a52d59a, 0xab34438b0f6bffa1, 0xf427b44b819e7d76, - 0x9dc6868428366ac7, 0x66173e587002f51d, 0xb299339d9a0b9f50, 0x78461075d04e91f4, - 0x465ed038aaa73c96, 0xb3c4734c25c18a56, 0xb13e363299925311, 0x4048bae92300aa84, - 0xa1f77346edfc2a3a, 0xae238dd794966cc6, 0xf228a2d5eb39a25c, 0xc4e8feb578a120c2, - 0x2c4f662b8052270c, 0x41bcfaa72d541fa1, 0x66a9895b5620df83, 0xef0e0f55ddbb933b, - 0x3ab039cbe0b7eb21, 0xbcb69bdfe916a98c, 0xe5cec9971be6d5c0, 0x2e5f5d48b7830410, - 0x5c6fbdf4606ddaff, 0xb2fcc8d0a5418f55, 0x8dea46cb2f0fef24, 0x8975970884dc4a75, - 0x73e366243a52b8b2, 0x24668c6f451f2fbf, 0x7e5627df587a9f29, 0x6d682aaac228c369, - 0x5279032048195017, 0xcc2dcde3d451a194, 0x7ec68ecc12ca1241, 0xec761a369113fac4, - 0xacca32093e839f34, 0xd8e85def68c7b937, 0x37f896381ba769a8, 0xf53ef3339c67293c, - 0xa3408ce70191acb9, 0xd07a6e2b61716155, 0x4522f7c85c5dcec9, 0x8015a5fbbd326d12, - 0xaed5dc14824341bc, 0x9df90292f8fae0c6, 0x4f6679d9b7698702, 0x1ec7b4e8e7ad950c, - 0x89780c6aaf7939ea, 0x2c273f2bb96c26ae, 0xb037e946828bdd31, 0x80abc571eed2c562, - 0x503c3f2050e44ecd, 0xf7f7942ebd014e83, 0x42a5e67bbcef697b, 0x1ddc59698b3d3dc2, - 0xb54f90bba83cc76b, 0x7d9864348962f362, 0x7a2d1117ac3d4a02, 0x85d76aeb682f984d, - 0x6b4c425d72fe783b, 0x19a9e12d770d0064, 0x34ea92d722ec8647, 0x88f2ed9ad464bb43, - 0xec027a664da1889b, 0xba6d671f92531026, 0x00d90c2a74ad1012, 0x533ecd35aad00cd6, - 0xd23c644e9360654a, 0x25c9448301ddb8a7, 0x3ad3bee2779d25d7, 0x181ce62da74ede33, - 0xb6ecb4c35bc8d42f, 0xbcfdc622ece12ffa, 0xa1870fb047a3d4e1, 0xd3eb9c4716f80738, - 0xd2a412096a3f4488, 0x01297abb74569639, 0x769264a279c90f13, 0xc751ed7184ab9a56, - 0x5240164dcf1c6fa9, 0xe83fb0f787d3e5d9, 0xce5e2eb53ea7f0a5, 0x874b28415ec70e9c, - 0xd6649614b59e5c36, 0xd6d63200186762bc, 0x22910b08da546a69, 0xb07de692d1daa480, - 0x46af88c708d45013, 0x9a9dede7382d8f36, 0xb8428155c6efd2fa, 0xb4580e336ddf42b6, - 0x6891205dabde1484, 0x760dfae37a143dbc, 0x03a36c4aa4199645, 0x7e65d0b89f655d95, - 0x9c218ad8ef9c78e6, 0xc3369f3491c2c9d2, 0x3cb96fd63384a50b, 0xc0dbad06f5f4bc0f, - 0xe71fe7ca546abf1e, 0x5a4405b477462888, 0x42bb56f29e04e9c6, 0x09f10f91138f4c39, - 0xd93803bc63a92c9b, 0xb915d6d8cf821bc0, 0xc65f626b8ff54aa3, 0x7f5c919eb7fbda62, - 0x65911c7a02cf283a, 0x73281cfd071190d4, 0x5d6e2943e10ae9bb, 0xef429df4a7f04db5, - 0xc0c656a699ed6d16, 0x0622d0792b9b3a21, 0xf5384ec459862cfa, 0xa7a9e4e92aa38ff8, - 0x2218759e25514541, 0x4f9f3a01de4986ef, 0xb10f98ae9866c810, 0x685b59dbf3114465, - 0x0b09e8ae7e3d2529, 0xf56ffa1452d773a1, 0xce55a6f6bc2bf562, 0x99bbd97c36355558, - 0x496831093a3e15a0, 0x8f9caec5345bbd63, 0xc0d86a417b911e03, 0xa33f6c31a74cc5a3, - 0x8ebc655438425902, 0xe381c4f7764373c0, 0xf55a1c9e527a2731, 0x823ec8c368aab1c6, - 0x3d56702357bfba4c, 0xe719e916b1d2c3cb, 0x05076d29150fb0ef, 0x34913ad9987f7c5d, - 0x3abe27794c37e45a, 0x6f98dcfb78cc87a4, 0x96a42c41305964e3, 0xe7c6a9fb11397a92, - 0xb23fe0b3755b644e, 0x38f3db604088ef31, 0x62ae789848c8c9d3, 0x239aa9413fbe563f, - 0x1dbbfd965acd3149, 0x25a1dc234de6fe2b, 0x709c33f91148c1c9, 0xadbe865b9f892a9b, - 0xadf74b56f106fa98, 0x91d26ecc00e05256, 0xc3328631fc94ebd4, 0x6f0179946661ce19, - 0xbc2cb204ab3bf7b7, 0xfe03a40b16bc4d16, 0x0f9176898d8f6153, 0xd6f4b68bcde84855, - 0x126dbedfd951547e, 0x8db675af0b95a7a9, 0x32e98cb64eac5dbe, 0x11423778266edeb6, - 0xa4f5d36923d2d92f, 0x27307061f552af7a, 0x0979aa7278c25258, 0xd538aa39ba8eb09e, - 0x3e333d60df10d31c, 0x1232e63b4e72d129, 0xff67f7208d677e79, 0x61201f03653e4de1, - 0x4409efdefe1a5420, 0x07227698fd2e0f4c, 0x0cabb47437416f3d, 0x1ae37b275ba4339a, - 0xd2d03031b8a6cb6e, 0x1ec07582dc32dd8f, 0xc6ddad706c85e52e, 0x2c719a6f0f892b07, - 0xca68bc87d3dca8e9, 0x20825984b6a7f4e8, 0x86349eb3c86d3f4c, 0x68c5a19b46738c9e, - 0x6c32fb552d91eecf, 0x7f5104999c712589, 0xdd20f345061021a4, 0x53db0e051a2691af, - 0x3420ea96e2b8707a, 0x53d4113a642189b6, 0xdcf6aa13a8620074, 0xbaceb1794477e8bf, - 0xc22e146650c567e8, 0x02d3c79ea404eae6, 0x6ff893a218106f2d, 0xd85d85a21739ea6c, - 0x100181abae3103a1, 0x032a3d5647aa66c1, 0xa556adf0c6446d9a, 0x0de36eaa1eedc0ec, - 0x3a6f5c82b6598293, 0x1ad09dd8f1f6a844, 0xc6012f0773e42d82, 0x0062e12c4b44c1cc, - 0x87a9b95a1246440f, 0x23bfb761f9409acc, 0xd80e34e1b0d0691b, 0xecac8111ea8f5acf, - 0xd69635d36870dcd8, 0xc6b136fc4ab64dec, 0xde37f8fb3ebda559, 0xf552b729252b4783, - 0x3f0950b5f68f7b48, 0xdeb07e8ae69d32ba, 0x63e66a9b127dd194, 0xae9e7170d5b2c12b, - 0xe56bf1ce883f8487, 0x1cdfdf0b63131f94, 0x9713659daca9e4fd, 0xf35734314e55e4be, - 0xc9e3b2643ebc2705, 0x355c1b09c368b25b, 0xc97cbb39df605ad8, 0x232291c66727d056, - 0x3eb39cda80c763ca, 0x7f2e4f127124dc35, 0x8f019ebbd1d82951, 0x90183ff36c13f27d, - 0x9cbda72f0537ea9a, 0xf0d1f96cb23654d0, 0xc1ec494c524b54ff, 0x4c563293f3ba922b, - 0xda9a3be3eb9a451f, 0x387582963ed394f9, 0x20a5000297ada968, 0xfe6ba93f9974c0c1, - 0x1105ea348744a8ba, 0x044a2085ba1e3a04, 0x72fc7523a1248809, 0x91588900b5471948, - 0x076a127be3cb2ac2, 0xf9069daa30d7828b, 0xe68ab5ef90c4ca66, 0xb24b4ef407ffebc1, - 0x0bbac86fa86e1615, 0xb21f73a153dfbb47, 0xb3cee6626d2e2294, 0xba166715653d2f0c, - 0x963babd45885f69f, 0xb79f1aa5d6b0279a, 0x4ae5916dbffd1b0c, 0x2f2f10f0c93fd6ad, - 0xbb34c9021619c092, 0x5511965d670d7036, 0x774cfac30a443bd8, 0xe66fd0d24a0658a5, - 0xee0dfc83cb040225, 0x8c633674dea5fd96, 0x59c00dbb213ed125, 0x2fea9af77622c77d, - 0xf3147a5b6c624209, 0xf515673029f99236, 0xf620be1322c847ed, 0x4ed126fde6a16377, - 0x669f15cf97f240f4, 0xe3496c8fafa842ac, 0xc83eaf3dce2da5b2, 0xdf6b5b3929b13b69, - 0x08f758972fb36325, 0x664e33d6b27cbae8, 0xc7a217b682d4facb, 0xcc34790025415f2b, - 0xa2c7ff1db20fbdaf, 0x3b8f8b680bc0588e, 0xfabdaeb9e9f35a5f, 0x9e46351eb59d85e0, - 0xd78a16b202bedb2d, 0xed428ba2725cd9d2, 0x8245f3e27d204362, 0x03e8419aa2670231, - 0x2819776aed364db1, 0x434e086a524af0e2, 0x5b890b46beeeb636, 0x9faab818988901ac, - 0x19ce5a720bcefda3, 0x668c6d504768d9c8, 0x938690724942bf44, 0xaf83e7d6b86fb97d, - 0xe237de1e926ba148, 0x63860d49e5b910b5, 0x8f31699c7045927e, 0xa6b7a6839a3ee5a5, - 0x172b2f39c88f6689, 0x2dcbb436d4c18bf2, 0xa3cd26016b3cde5c, 0xb97a6331729d1060, - 0x8c490834b4a8f011, 0x392c2deb71099e99, 0xcc26f008a6be7cf1, 0xb8b12ac4ce7c46cf, - 0x8dba8e9a44bd5384, 0x56fe2cd736e0b3e1, 0x781aef2b6842f9b8, 0xf93be6a99c87fc02, - 0x33c90e8ada400a19, 0x46e21ccfd03e5b83, 0x7c7e398e3611a9f4, 0xe2fe19a99002ca5d, - 0xdb9f7d911f337f1c, 0x9cd7cf5b5eaa3f8c, 0x2d905bedbdfd232b, 0x8b4dc17c91f912de, - 0x3aff094c26b5d588, 0x9600083d4afcd5a6, 0x9b5fd8d2d9412eef, 0xbdec0efdca269862, - 0xa5eeae604c8fca53, 0xe72f6db4e787ab70, 0xcf24eb18ba0b357c, 0xaf7c0b8c008818bf, - 0x3544569f11e18d8e, 0xe32f8363072f33f0, 0x479791ebd27756fe, 0x6dd2e11cb4133c40, - 0x2dbaec795927129e, 0x3ff3d18a6ef14166, 0xb9a885c175f77a7e, 0x6a9d193c55e56b3d, - 0x957190afff254fc4, 0xa267ecf33e716bda, 0xb0491a1f20aae7fa, 0x56a2653559317f64, - 0x5826bd7c4e824440, 0xcd38c5a39cf973de, 0x332b63f3edb2aab6, 0xab72c7d360075cfa, - 0xb59998dd9b46dfba, 0xa783864a68bb5951, 0xab3dc539b2e9f181, 0xdc6ace5b06cd2351, - 0xaa44c2b701f76bd5, 0x9570ef871d2c5671, 0x3abc1779dc4a1d91, 0xbe3f6dfdf397e236, - 0x8782703585e2402d, 0x40e0fae0369869be, 0x4372d02c3135fb71, 0x6990bc9942e34a90, - 0x9fe66fb1c398a25e, 0xd689259d8dbd9378, 0xc4bf8980c5e7899b, 0xa01436636760501a, - 0x7ea1e356af9ea92c, 0xce8aeb37255ba5af, 0x6054970044f75e20, 0x8f4ce8693aae56ba, - 0x2a2f54e3388c4166, 0x54aa2e98c47dc653, 0xa0401a1a6e50720e, 0x2be689184fb13ce6, - 0x934418f886377da4, 0xbc10847d7450b5d3, 0x2de67e260d1d21c3, 0x68451ffd44a4bf9f, - 0x30620b7318ff2f88, 0xfe73fccc80811845, 0x32eac98006d06412, 0x9695e22be28cb1fe, - 0xc0626be28e463065, 0x1d978f6ca8c1cb8d, 0x21c6a55cdf7de27c, 0x83108de4e95fd62a, - 0x43783f9693797980, 0xf00e8699ea8e0229, 0x961a83d2c37dec5d, 0xc13696499c774c7a, - 0x78cd0c371e1c457e, 0x407141d9790b5889, 0xa1609588b3e6b50e, 0x75105adfdc9b6ef6, - 0xe24585a05b278786, 0x4d95b02c0bc78d64, 0xa4e8a0add35cb044, 0x7f23f5be2787bfac, - 0x0a51a1924e4c0d82, 0x5b728185d7091af4, 0x78737260f43f4845, 0x4c23e562ac375339, - 0xaf9b1793dbf4e0cf, 0xde4e145d0f853896, 0x26c49f78b4e07403, 0x812102c6f16c0568, - 0x2ea7878a5663df55, 0xba66060ef6fb8d73, 0xdce96610cbc8e0e8, 0x488a0069ee3caba7, - 0xd5ffa8ba232d1439, 0xbc9c448692a7c421, 0x0dd00b8277a68b73, 0x355b52f0a5ce41f4, - 0xc4a8f9ed0380b160, 0x38e229e3b5ec0812, 0x601cc1b86d6ffe62, 0x688bc39703febabc, - 0x0f95a8c0ea5bed6f, 0x369ed8a1331494f1, 0xb1e4b427765eb4aa, 0x7a01668ea959449c, - 0x6073e6f15b1446aa, 0xc8c9e953bf0eaf4b, 0x0c194f14aa7846a2, 0xbf7ae10412664c59, - 0x694764ed3535dee2, 0x869b82f11065e11f, 0x1ac2299b6309224a, 0x1dd45d7176902db9, - 0xdecbaaa8fbfc0c20, 0xb699d6bf06d1fdfa, 0xbd7e4a1c8201a815, 0x46c1ca2a33d49629, - 0x271aa363bbb576ed, 0xef7fe423e9c97d86, 0x76ae31116e2d2966, 0x2bee96ad31fb00fc, - 0x71adf27aa7a6e1ee, 0x8c2cac353289422b, 0xb2378b3c1b99088f, 0x22991fbc87efbadf, - 0xd2d55186ffad6744, 0x05fe6be9606bda8b, 0xf86715945edcf68d, 0x015ff53fd82cf7bd, - 0x78d45eda5d37cfe9, 0x20abf5c21f84e8e9, 0xae73a1a15302c7ec, 0x9cb82f6ee82858b3, - 0x7f4f31ca941c61f1, 0x6b94557960d67c71, 0x31d0dfe8fc8944a1, 0x50f12a7628f2ea2d, - 0x2eaa5d4023623ec1, 0xeb1e499efbb32689, 0xaea21d3decfed5ec, 0x8944e826b1371175, - 0x95489f59c62a5848, 0xc3d31e459152ed34, 0xb0267f97a063595e, 0xa43eacdf7f650897, - 0x4d8cdcad2e8302f4, 0x1b55dd7ba996e99b, 0xade8c5d0ea304b28, 0x61ab212e3c84d64a, - 0xdb4534e0db75e5d4, 0xeeaa189d892ebf1d, 0xcba9c304824b6f3c, 0xc3a22b38c6c67eb5, - 0x8d5149ec4cdf3b41, 0xe25c78efdc648ed2, 0x06d5daa21960b7df, 0x169e8302da0f77de, - 0x77cb37a7ebdb8a2d, 0x6e68ea33dcad0eae, 0xfa3e08ce93f169be, 0xc757932177c4016a, - 0xe00a619a181b2150, 0x396470f5df5e8912, 0xa91a5e539cc898b2, 0x5768db8e917e934e, - 0x11e9daa7821b464c, 0x3911d856f15c3685, 0xce4cd7e6e2db81ba, 0x36b089acc742f065, - 0xa911a3fcf08e2b92, 0xafd509abdba05dcc, 0xe007d15d9ed3fa0e, 0xc0d7f1c43e5ea14a, - 0xce78a4bd34d58ba8, 0xa35ded7167645f5f, 0x16776dc4a2aacc15, 0x2d63bcb9c14d3e53, - 0x15fe78017cedb5af, 0x758a9cdd4c333287, 0x1246e841e73bdb0f, 0x17d22d203071b16b, - 0x664b2f1c121c85ea, 0xa274ef7d881ec1c9, 0xddaf37e0b89fb8b2, 0x47062aeb8fc193a9, - 0xcfb6b81ccc2e29e3, 0x91077bbb58ae3a8a, 0xe58a0a785deffa69, 0xd4e27c60bf9029c3, - 0xc489ca68daeeb151, 0x309bafe347a5b2b4, 0xebd9628dead55acc, 0x7985f049869fd476, - 0x51af7cee0d1854bd, 0xa26ab3e79440d934, 0xa294562d9cc0cae4, 0x6de3119419a0bf0b, - 0x45650c8f9fb6390d, 0x416c211c0199a699, 0x4c41fc045bf7c9b2, 0x7d1f3fceaaba3c90, - 0x553372cf673b2802, 0x98c5d72cef3cb60f, 0x3dd999174c21f06e, 0xa715b4c71e850056, - 0x1bf0fe75f44ba42b, 0x7569bf028bfd4afe, 0xc88dfb797cc611d4, 0x9d7eef787dc6c5f9, - 0xb983bef48c06dcce, 0x465c982534c2d6b4, 0xad82bccbb97eaad5, 0xab99c583f0e3533f, - 0x079b2cc30740b636, 0xc243f006c7ea6992, 0x6a47bc4cebfff325, 0x4f279336f7f37af7, - 0x35da4a7c18178644, 0x63d0e77ebe46d719, 0xcb3e316da4e2e649, 0xc2fced014d8d2ab3, - 0x3430b00e2fd590e0, 0x3e8bd9d4001f1250, 0xd5be1451c928d1b3, 0xa284a5fd14fb64b7, - 0xcaffe00098b69352, 0x676661d83b036624, 0x907e4945c93225cc, 0xe248376164a715d2, - 0x1b9f3e247e7ab874, 0x75b927b5c60ea8c5, 0xd7fc4b0586e2ceb1, 0x2960f3f477ef6b3f, - 0x9956715c9d9456c3, 0x2e6fd60508af45fe, 0x853fda99042ff206, 0xacaa0423fa213c47, - 0xed87c3f6fbd9670b, 0x30caa54de8318f69, 0xdcf48ff054741406, 0x625f6bce3a0f32e4, - 0xbd39dc1e51a31090, 0x8eefca612265be7f, 0xad1e144841a7c787, 0x26f22f048d3e4739, - 0xc41d817310fdb810, 0xa00876d1eecc42c8, 0xb12d734872ec4526, 0x7472acac9ba6cd87, - 0xd496db95a5cc9ba4, 0xeb27edb7979d560b, 0x0a7e9ce68b2726c8, 0xadaca9774f879e4e, - 0x2ca874c613a30479, 0xda7d8e8e44e6f7a7, 0xaf7f2cfc3e7c6d16, 0x8de4aaed8ab941e7, - 0x87fe8b1f8412783b, 0x072e41dcdc89be3e, 0x90a2ca3a210e8393, 0x1dfe6d2b30fedba5, - 0x296b29862b793233, 0x855a7cbde0441fee, 0xe8550c0f53709a59, 0xe05202a4c8c2ee37, - 0x19080e1fd23feafd, 0x9477af7f684d75b6, 0x7ace23526a936769, 0xea9fc1a2c973db32, - 0x4767a23de768ceb4, 0xa85cc028e989dbcf, 0x9a4e4859eea2ade6, 0x14a1fdafe526f87f, - 0x1012c1e66b5d88c3, 0xfa73c66aaa01ad68, 0xee8eed95879ccc63, 0xdba806dda672fb62, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x568a5ada3d11fc00, 0xf16449014eb881a4, 0xfdb9a3a516062f82, 0x1eb2cc06c3a45f29, - 0x0551f4d6abb5a6b8, 0x7ac9d46537ca1cc5, 0xb1d327b4a4225f64, 0x2fe98d3ccbf07cf9, - 0x01ab2eed229b638a, 0x2d304352532157ce, 0x20a9e010c3e91cb9, 0x5b8a8c44c0b64697, - 0x439e51a156619c9f, 0x206ff7d6d7d38d10, 0x7c86d28241f5a31e, 0x0135897178a06022, - 0x803717c70bbd66f7, 0x70c7484ad9affc20, 0xa72f25c94c6c4e87, 0x3295f1837e9def4c, - 0x5a15a75e2a6ea548, 0x7a3ab24802462236, 0xa97406a52525fa97, 0x51f79023e3da2274, - 0x9f7be26831a750d2, 0xbf1eb51f9ff53dcb, 0x6b94b9838a900bda, 0x0690d6c697e4ed01, - 0xeb57310031239f7e, 0xc5e298b06574c7ae, 0x95ce7db9f7f02d7d, 0xccfc0328eac5ff5b, - 0xa5f2ba3335da361c, 0x76119ccf521cf5dd, 0x4b960103950903b3, 0xcbd165baec1b0d7b, - 0x3c107efb3644114f, 0x589b67b3a82455ca, 0x782cd45fe886f2ad, 0x93c3cc70cd80b778, - 0xeeeec1163e049cdc, 0x61e301fa4dd0cfe0, 0x1b5f9da885473c21, 0xdc1fe500535ab720, - 0xed20578accb83bc0, 0x61164536f8cad65e, 0x221e2c931390db07, 0xf904d426f61441d0, - 0x09cb112d880868b6, 0x48bf8767193d9bd4, 0x87a9c7aed7fbc606, 0x3b1304a237b1eab0, - 0xa7985c5452f6ce7e, 0x8884978b04f38903, 0x03abb5acbccebb5a, 0xcc7ef88f290aeb0f, - 0x9b4c78c486cba52a, 0xa2506b0510eaf0b7, 0xe522db17e374e49c, 0xdd1792e3aea9045e, - 0x882f1a964c88d5f8, 0x8a69a48cb23046e3, 0xd18cbf42cca7f759, 0x3a323e8c41b1588f, - 0x50d9a918871c494d, 0x0e149ce2fc437479, 0xa4625a1e2e3d7563, 0xe301aa6487651ef8, - 0x4720b1dfe57b07ca, 0x834c5274326e8620, 0x8e0de9c83e63fbf2, 0xb2e87a0a21a791bb, - 0x862dc8edb3ed53d9, 0x1ca8b34167be6fd6, 0xfe4d90d98efb6113, 0xd2d2690dc0daf693, - 0xa94dd4f992450db4, 0x47061a80019416c9, 0xf7ec4060827640b5, 0xc50f330288f2b8d2, - 0xded3541bc7013d3b, 0xae0872a63b62a51a, 0xf40e98ee53f12982, 0xe4763a5315234be7, - 0x90795978778dc380, 0xcfc8e6d0bb335bbf, 0x61c1503ed7c707da, 0x5e40d98579f2b455, - 0x2cc1fdff5c7a0462, 0xfd3fb8b6ffeadddd, 0x428577bd8edf1d42, 0x44b858d55fe3a2a9, - 0xfbcb0d9bab96c68b, 0xbe6be29273d7c9ee, 0x1d5d4d083f0971ef, 0x21356921dfcad62d, - 0x75dd51640467743b, 0xd21c9b4bee046531, 0xae0c0b45daee208a, 0x969723003200f0a4, - 0x16c5102ba9441ddb, 0xb90cdfd1b84b4d1b, 0x2fd6229db9ddc7ce, 0xa2e12c5689773733, - 0xaa50d03dfa1d4895, 0x7ee6e127b64229a6, 0x544b9c199cee8008, 0xbee1b3074e369c52, - 0xd95c712928b3fe09, 0x81677ae9c37a1420, 0xf8ee18b3dd2e4be6, 0x052d036927e04327, - 0x52452d0a461bb497, 0x5d3522f0b6c0f164, 0xb5c85c8d45e81752, 0x6b1deadfa5b55505, - 0x8eb084732935f50d, 0x8a5ad3e8d147b7bd, 0xe98c512914b8e039, 0x15ead3915feb8973, - 0xf9da8a2649db95ee, 0x0b3c802e4a00e8e9, 0xcf20a269b0f0c1de, 0x95c78d4ab5a752bf, - 0x135e46638d1b9a10, 0xce9e26da382c72b5, 0x5eb49349c86480fc, 0xd2b89c178927a8dd, - 0x18148b5af51d8282, 0x53310a3e5c09f72c, 0xaa3f5a57ea32ddc9, 0x95d00f2bab6a7816, - 0x1fe0a0371f87b3de, 0x2c2cfa4e665d2b42, 0xe044ba27f75b9bc6, 0xeb8f8c274f270a2d, - 0x6737f7e959c72894, 0xfad6a91b4a85cac3, 0x11439511cb50af28, 0xbefd92ca9f23148d, - 0x97c9297516003b41, 0x17fcc7038f6dfcb0, 0xa70fde4dc7f7e62e, 0x40d2e6371483c45c, - 0x2315e9aa327e8699, 0x536b540938a2a1ff, 0xb0dc0deb1c90f653, 0x26d7f36d5d7739b1, - 0xd48fa67d69407d44, 0x0d3b576dc5ebe830, 0x56c14e24ae356432, 0x367e0d0208fe79ce, - 0x89d0d108da330ea9, 0x8a35ca4f69e5f47f, 0x2c3603a5d8c8b62b, 0x0d6531523b7199e4, - 0xf7f918445e106a3a, 0x2a6529127debf1af, 0xabb7cad07a47e6c4, 0xf6af83171e34f627, - 0x038000fb2107289b, 0x6288e840ae379857, 0x032fc4cbd4edc911, 0xe93b4e69b409517e, - 0x2e1777c356e42e9e, 0xf2ecf6bd76058cf0, 0x7f01f1386b62ec89, 0x5b6e4126b125d292, - 0x7bec9fc94b601039, 0xe276224c94bf3e9d, 0xc4638f80747e4b8e, 0x80e657eb2cade2b1, - 0xeff49724d697e8a2, 0xda463e4ddd1e207e, 0x86d2b11e1664642e, 0xf6bc2c86cf31e4f0, - 0x6d2c0c400539a7cf, 0x6f4465bf7be39f39, 0x4278a5cdf515731c, 0x4b199f4191800018, - 0x6d2d3d308a429152, 0x2ec35f1caf45076d, 0xfaf6c3bcc842cd44, 0xa9c2d67a38eabd3e, - 0x785a10f6c189e76b, 0xbb365396b7fa07c4, 0xf14f6152b0728b18, 0x1c43d45daa46b17e, - 0x2ad55363ccebf6bf, 0x615ee4ad3878120f, 0xf752ce4465f79c02, 0xace1f0c9a53ff92e, - 0xe47ad69dd19f724f, 0x0ff88c2a5c663813, 0x207a142146536943, 0xbf3250861dc875c0, - 0xf3dc43de59f39b40, 0x523c480b8837a567, 0x130bfc1263cc409d, 0xdb7c6acbf3f704ee, - 0x68dfbeb2dd08df31, 0x056af3434b3c06ee, 0x47e7d39bd65b3291, 0x9c7c4aa3253634fa, - 0x680365fec4bc3512, 0xfb221356dd185c08, 0x7965842e5b887f04, 0x6d173903e3e16974, - 0xb741530501bc5c87, 0xd22d02546551d16d, 0x0c359d176e31bdb9, 0xfa42a081fa22c752, - 0xd35fc73da6b4957c, 0x57df7d193ced18b9, 0xcc05a496ad8cf7a9, 0xdb967409f70d431e, - 0x143e6a1f60904d17, 0xd43525a9c8b07391, 0x8f4c2172708366ed, 0x4d13fc90683b7306, - 0x40ff9c1630544cbb, 0xdd991d73e81a6e28, 0x1040dac94a993a07, 0x00b993a3f93e5791, - 0x511ce1b7bbcf810b, 0x32e2e0fd162876b1, 0x6e0f67bd35c807dc, 0x17d9e8b2d0a3fb69, - 0xfa493ece987557d8, 0xa400e8f4c9399aba, 0x13a19cfc0e3c0e3f, 0xb05368add2f05413, - 0xd5673f30ee46a39a, 0x357c820f62ddaae1, 0xc8d2865af5d10488, 0xdfc83cc45ab0454a, - 0xa976acb64c3dac03, 0x12c2add5b2e9fb67, 0x7a854f12a1041d28, 0x81ba41f7da2a9d0b, - 0x6e0c69322f03dc3d, 0x664989d53113974f, 0xb7a12dd99a5c1285, 0x9402b517f30eb11a, - 0x613ccd93ffff9a9b, 0x5570cddfa6e44922, 0x3c09c35ad52fc26d, 0xaaa902239697e3a6, - 0x2016a1b23ae3e7b1, 0xeeab4de64804ed97, 0x3cbfff1d346a0bea, 0xe114fc93514f1a81, - 0x674c2170a25a08a3, 0xf3badc0da3e54b38, 0xc38b9c44e04ac730, 0x0151b3acbe897bbb, - 0x87504ff42f7638cf, 0xfaa4a51515334b93, 0xbee748492ca76285, 0x1c7c0f92837088a0, - 0xaab8e574d421c17d, 0x7c012e5e742a38f1, 0xb97f335169aeec67, 0x2e9e831ce06fd29e, - 0x15bd714585a3c789, 0xcc270c1148326744, 0xbef7f5a1ff83174d, 0xed3d065c5a920bb0, - 0x38e0eecc923efe7a, 0xf2d4116a2ecd997b, 0x24d24edb9e613418, 0xc8dcdde3a08166c2, - 0xe05d31e24455b29e, 0x0afcf39628ff4213, 0x13aabb60f3007e6f, 0x16eb01a5bc429df1, - 0x716f6a1793c27c7c, 0xba2e02d8dec203b8, 0x4e3dba3457a1ddf8, 0x1a1f9cca94a5130c, - 0x8e8777e0f23ccaba, 0x7c0dbc444ee74691, 0x977237da8d645254, 0x4ac5424d3453c8e4, - 0x1a0b0277316ffd7d, 0x82fad9e118c513bd, 0x311876235da57cb4, 0x49c1183b71ecdb7e, - 0x229457a630851a10, 0xabf9eb7b0b9a81d3, 0x577e29dfbe1de31b, 0x1faedb746b3b16b1, - 0xd278fdbda5b46ac1, 0x7bff6563704f611f, 0xb57aba036f15d656, 0x26f2c278f543490c, - 0xb55e3062ed28ab5d, 0xa374c66894536f55, 0x5b38c7f50ea9d26c, 0x17c1e0b07968e36a, - 0x245aefd23200a60d, 0x6645f9b5cca8727c, 0x956fa31b8a4f978b, 0x63b99479b94867fb, - 0x9217864588b07288, 0x117bfc55c11f9376, 0x41c017c2829709da, 0xdc06e610e5421c66, - 0xaad1096b0e5d5c5a, 0x66f40baf58f84b6a, 0x1c300520fa117666, 0x2436b89d9d507f6d, - 0x73232016a74a26f8, 0x896fb9428da1372a, 0x78bc871241e5fd45, 0x94ab5f0a02a92a6a, - 0x6587b017f16a5b65, 0x60fa6d768eae50d4, 0x5635cd9075fbc8b2, 0x94bdc332c2856d75, - 0xc0d57fb2fa613bc1, 0x45e1d880c78e6cc6, 0x7a5ee2782c84b7c4, 0x2c846372a3c6ac02, - 0x160789b90db38a4c, 0xcc9726043ad1abda, 0x0d3830aa52bb130c, 0xd8097db94ad8a632, - 0xc856fe4b8e6bd8fb, 0x40e3ccf0ce65f9fe, 0xae858551eca04751, 0xab08be4905de0af1, - 0x0930a13d7cd91fd3, 0x97b7fa2cc060cecd, 0xc51694861649ff0c, 0x2af4cdc1952c5072, - 0x84f408173cb819d8, 0xd318aa469433c319, 0x435f96907c6be9ed, 0x70c7e4a5c3d67a1b, - 0xb55f13204e9badf0, 0xbf727b8d66860bb4, 0x1e0e0609d55cd7a0, 0x383df4fc376a2d4e, - 0x52a61376dbc181b5, 0xc446e72917f08eff, 0xd3e1bfd3d97a6f90, 0x5693d7e6cf9273ad, - 0x03c2d2e9fb94d7ca, 0x0f353bdac603a6d6, 0x0259df557a6113b9, 0x5714d4c6c5c94ac2, - 0xe41838aa75048b06, 0x3b6300f67c3bf5c4, 0xcb94178080bceb9f, 0xbcfe58b6bd75e77b, - 0x3e767c0e5edb5f94, 0x5142b6027f0abf90, 0x7a2fce6606e4ea72, 0xb8883bc5c491f2f8, - 0xdff19aa3a7cb078e, 0x6f69e961c01e7f56, 0x358dd38483bc56a7, 0xaa491c66f37613cd, - 0x8de0653bcabb5895, 0x0edc940ea2cff7bd, 0xca93755c778daec7, 0x92246204fcd4692b, - 0xb7183bec951ff4fc, 0xe82213001a581dc4, 0x634c45809962da6d, 0xe2d47c9fbf0f7968, - 0x072c6777a00778e8, 0xafad56d2c9cd20cd, 0x7f2aeccd1244eea2, 0x2f1b34c1d345f3c7, - 0x70766f74f5c7b9ae, 0x0d243d5b24940a1d, 0x374c546246c33dae, 0x07fe4ae2f07fd678, - 0x342823bd878bb983, 0xcf94ed0c1ff5abe7, 0x9a3a86af9b45dd03, 0x90dd6126c526c984, - 0x928bc6388f9cfb78, 0xf560fab5b9ef0749, 0xcae5aca6dca1f46a, 0x68976dc6152d301b, - 0x7938c6477511fd89, 0x51662e3bc5868c3c, 0x59d745d27816cd7c, 0xedc4e332d47ecd2b, - 0x8b8e87fdb17fdc54, 0x2874164201ab78c7, 0xc59b674f4507b731, 0x606e93cfd3b6d286, - 0x22b57d12521cc858, 0x5b1fbd81166d3211, 0xe0a7f249914d3890, 0x815e8a2844dbc69b, - 0x34e2c5c5ff77fcb8, 0xcd8c54a7650b585c, 0x5d4315174e304734, 0x765bbfce436c35f1, - 0x1d2e154b018d9786, 0xa355c053e93edcf2, 0x66422042fa6d290f, 0x704c8e58d16e6d89, - 0x3e522e2e28409e3c, 0x389f8e0aca07e027, 0xc572c58b8d810ce4, 0x8bb40c5c6c3a4181, - 0x5372fb927c306006, 0x2d67ee5830726fc0, 0xd536cd7f1d5bb6ce, 0x6b039a952883f4ec, - 0x9c39a9d8cc92c4ac, 0xd62064bda20ddd8a, 0x43b7de9dd7a20289, 0xb1e0161c3de9f421, - 0x580d6742c338ffb4, 0x130c02d6c4275e04, 0xf7027b65b6d6cc46, 0xc5ef278d19d66835, - 0xea58b381072287a4, 0x34295a21bdc78520, 0xc57bc1ba1456c77c, 0x87d6ca822cbd5e45, - 0x530bc0f3b37f68f7, 0x4a44501f50d1fd7c, 0xdece26346222a260, 0x83acf8229679d0a5, - 0x9dd7c64ed7dd9584, 0x75e9b801447a0630, 0x027c09f873db8ce0, 0xa28f69e02f37118e, - 0xb64754be4d199151, 0x9366522c3949f29f, 0xa13e0fe933233635, 0x38cc943d4c06146f, - 0x910f04c7cb58d5f8, 0x3e46518f515b1efb, 0x5f8099b758a52b85, 0xe0538a05b373845e, - 0x76bc68c969b80e5f, 0x6fdf367ead909aa6, 0x561b424b8c0bbfad, 0x8cbdaf30f6fbed8b, - 0x6a7d978eaa94a1fe, 0x5a269897d7b4d939, 0x24ecd3f1f0f5d149, 0xa67b1e183131ab80, - 0x61529e8005e10158, 0xec36127cfc747e94, 0x4dbd99faeb681112, 0xc3ddd4a167fa8ce8, - 0x467d85f044746067, 0x2e95477d94ff8c4e, 0x435b7a878b147f05, 0xb3b50d57e7980989, - 0xc2405b6479cc48bf, 0x5d81cc39dcd370c0, 0xa66173555f209e90, 0x7ac46a495ff5f86c, - 0x2dd5aa47fcccee04, 0xdac5035a159e4a1c, 0xa08ce6f435ea2138, 0x973357f72cb0a882, - 0xeb6c66d2a2d112c6, 0x5e89bafcecff1fd8, 0xc09b8fc60291242f, 0x7229f7de698fc5a3, - 0xa66f31dee25efc42, 0x96f6c9754983751a, 0x10316df05d8e8ef0, 0x2910e1c85a2914b9, - 0xa27d923ec237c8a8, 0x8323033cf2f378aa, 0xc6dded3a80497d38, 0xe431dce168dae71f, - 0xb58f76c041899528, 0x228647448ba5ddcd, 0xf0d46ea492196cc7, 0x62af12c26d31e980, - 0x9568ee72cd94c04f, 0x5349d4b6d107a08d, 0xf3377ab8357c1899, 0x4aebdc049cad0799, - 0xbca44c38888a2337, 0xad899409759ab96c, 0x680f96eb6fa8478a, 0xb6cb7f032676a5ff, - 0x85278ae2c9bac2fe, 0xdc892364fb579341, 0x3449955456861e23, 0x6249a5c106ca1787, - 0x85b8c2004a8ab612, 0x92529c209273421a, 0x07417f13f5f384d6, 0xf1099d62a77c9ff1, - 0x11d520e3d5e470d8, 0xd450e8e904a82851, 0xd09b1919adfacd3e, 0xd547ab1a516a8653, - 0x1a7dec308fdddae2, 0x13e305357f76c503, 0x27e230a6b9f8e822, 0x5d2a1d84cc901f25, - 0x357bd3d0874d9f6e, 0xb9dfeef9a194dc10, 0x9f040c17b480340f, 0x746fe39c52b0ec39, - 0x0dc2cf20ed9cd658, 0x41c3e53f0ea41a33, 0xdcc5f3db4e932614, 0x08ad92992cb66b4d, - 0xb53fa2c451116954, 0xe540f08d0dbc36e3, 0x9d571f462e502a0c, 0xe741cd3863c72962, - 0x7fc65ebcd9451625, 0x2a432ab221b9a940, 0xabab2a365cb953d8, 0x4c9a1ba1b6c0006c, - 0x9db2fe467025b1a1, 0xb5f1712e204e9fda, 0xf6a2e2d0a385513c, 0xf7663b8dd5bea33e, - 0x133f65320faae97c, 0xa602c6591ddca577, 0xf8378cc957222a40, 0xbb7925b3e37a5567, - 0x8827ca3ce14c4a76, 0x508045ec84ca6e63, 0x12a0291d018fe360, 0x92e6dcd32818975a, - 0xc0ed04512a291d8b, 0xe3be890913e754b8, 0x759dfc7f8cfffba5, 0xf8af08fad1d3ae26, - 0xde1af1b18509801b, 0x49e59b008bc5f611, 0x62f0ef15d167bff8, 0x50df9a13e749aa8d, - 0x3efda9f48fbf5ee9, 0x00ab4f41161fc9ac, 0x6318719beb3e30e6, 0x8aa9e73bdfad44f8, - 0x6287c6d8cfe6c799, 0xd1bc94273f5c0c13, 0xccf9ac0c27fa017b, 0x56c80f813fd9b394, - 0xe13710ff6b88cd66, 0x235c31fd08d63660, 0xa26ce49ca2b574a9, 0xf49fd2738d4ba6a4, - 0xbd9e09b8e06762ae, 0xfd696b50e7838481, 0x67b7c5c3fa0fcbea, 0x9521f66a69c4e898, - 0xdea6fe11d189df84, 0xbfa27e0317c0c24e, 0xde58be706d56d981, 0x482054ef38d997e7, - 0x92079fe545389db7, 0x32538a67db0fc2de, 0xd85766d17a5405bf, 0xf8e3b7e8a4c14f78, - 0xea72d901fca708a5, 0x168acf99837277ad, 0x6a668a1cf1b00a7b, 0x9b1b71145a809a11, - 0x656d61f15c515b03, 0x7388ee7f9e0f6e0d, 0x58ba4c117a9fd1aa, 0x367aa6b744c79653, - 0xf568e012649a77f9, 0x55a3be3b037f9604, 0xc426cb02d9b8334a, 0xb22e949290645a83, - 0x4417e99a3dcc2f07, 0x412a9a9c196fd6df, 0x4b8b952ed7a56cf7, 0x4513e04652fe7df4, - 0xe33eea33ff76092c, 0xb9c8b62383173232, 0x0acdd5523dbc72ba, 0xce104666bc704baf, - 0xa04c4e8ba98d009d, 0x3a1362c09b1be4e0, 0x3ef154c19c3da67e, 0x7a89bd53b111db5f, - 0xea336c550650701b, 0x6521ecdfbefa279e, 0xe847536db0df47a9, 0x0d22c1ffbe1c6423, - 0x32d1e7bef5c711dd, 0x0a5de790f764b939, 0x701529fac7d67384, 0xf87d60b0a85db33d, - 0x20599a1867a8d02f, 0xbffbd34d9be7d711, 0xfc4f95858be57155, 0x5936ed2737109e74, - 0x0fa8e0487cde4ace, 0x5650dc264d3ff821, 0x03cdf26ce9fae6d8, 0x6482c6272ddcc27f, - 0x59fed6a2e9dee32d, 0xdfd07329678f7edf, 0x03681202739f20fb, 0x02a2cc78381c5b94, - 0x06883e9ed5d43b34, 0x8a4b819e013f839a, 0x3c0699e315b56264, 0xba3cf5ec20cd57e4, - 0x35ca2c6f7f99c727, 0x484116606272b76c, 0xb9443fc1dd3f8c37, 0x7b08a364ac7b12b9, - 0x039369ee98cb3599, 0x321a6f6d5b932e4d, 0x2bdfa647ba4e284b, 0xc741ca8990a24ff3, - 0x6089177e29fe9bb7, 0xde2584a18abc1875, 0x4487ec60739ce95c, 0x6d892fb1074df203, - 0xd0cae2eb3535e603, 0xa64dcebc9e306c85, 0xff10015fe6694b1f, 0x1356029a976c2f29, - 0x89b15501665df15f, 0xbace4cef8ce7e12b, 0x85b75e2526357354, 0x41fa8dfa145d8db3, - 0x38758caeee196bfe, 0x6ff3a7a41d79dc7b, 0xb445e1f4215d03d5, 0x0afb0954bc0afb6d, - 0x762c7fc56d35ebd9, 0xa97a318a84efecd7, 0xb3a42290b2e49b7d, 0xa9fca826a8a247a5, - 0x5073476b1b625d57, 0xf9bcd68cf79cdb38, 0x5dd1fe47a31c2bab, 0x9a6d1c97cec9e570, - 0x9e502dd6e931cffc, 0x8c2f35e9b23b3f45, 0x8584c9c3282af82e, 0xd364b4d44623fde0, - 0x1cad7f9c6022e019, 0x6328d205c86e1cce, 0x74a0f384023d2b19, 0x6350049a7d30cf50, - 0x3abeffe3e641dc37, 0xb1982d27c9b9b133, 0x83381772749cbb67, 0xeb5050c7f7acc2b7, - 0xb9c0894d1a2f9e3f, 0x86270204cf47bef5, 0x928359d3b73eb69c, 0xb61022ae57ef1c54, - 0xb2e85b983fa28a74, 0xf1b26a64a2556298, 0x706ea3745ebe0c2d, 0x6bdf137cee4b1ca7, - 0x82ff6dbf0735f107, 0x3bb7be2a37ed279c, 0xbdedfbd2f533665b, 0x7e19acfc3b7ab1bf, - 0x55e30e0aa52f2fff, 0x36a31a4b7a59b650, 0x9be3a1589752495e, 0x5b120348cc395b73, - 0x01436a918587a535, 0xa9a89f52ff3d30bc, 0x189b23bf2b7468f2, 0xd75ace8752346e35, - 0xea915da0631516bb, 0x664e316ad84c5692, 0x4cc62476b32b1c60, 0x3aabb3b700b43caf, - 0xfbc51abc084500bd, 0xd6b8397f0e81ac8f, 0x4dc7afcfe9d37b5d, 0x15628037f268127e, - 0xfe9788a1b188edf9, 0x5b14fb987dfdd348, 0xab5f7757050e025f, 0x91963e0a2d10c60a, - 0x18b22e86edc3df6d, 0x5bb0484d9b9d72ce, 0xf25061ddf0909ada, 0x9c9b887220fe051e, - 0xd247e7ccfcf68352, 0xa0f219759454d79a, 0x6776e4fb90cbd66e, 0x0da20acfb5feff2a, - 0x5e4a4791e87dadbc, 0x19473ff9f7516917, 0x5b3bc3911a9a52d0, 0x087e7f095566e796, - 0x03a0f41e6242bb41, 0x5877cc6220573d3b, 0x48dc498b265140c4, 0x604a27b73bb80544, - 0xbd02edece9139cf0, 0x8c6f50f3d032c038, 0x1f4d2b691d625a3f, 0x8ad337ee55db64ca, - 0xa04f6a3f7e828667, 0xbbf93014591bc241, 0x334860806c9bf75a, 0x70acdc497d529d77, - 0x63b1c30947407564, 0xd922bb904a6e198c, 0x0f67f19c39c6abff, 0xcccab9aec6e45502, - 0x3e2e7b93fadca229, 0x532a14e4bc2e5383, 0xb62ae76373e6ef85, 0xa6da03440554f234, - 0x856d0d9620b0eef6, 0x0c6f6ca3a2189406, 0xae27a2779d68d4ff, 0x68e5795483388277, - 0x7ef8c35bf6d94d57, 0xc2dcf5ead80d4256, 0xc3dfc1ab7f07e2de, 0x3a689ef56e711c2a, - 0xc9880f226c8da978, 0x8c5e7dd2c97a283f, 0x6dd8b5cc6e174d93, 0xe12162e5098bc885, - 0x0b7656f641468bf3, 0xfc1d89e93a5ac4a9, 0xb80c7a4945c89348, 0x661c068cdb02a8fd, - 0x3449d52bd875961c, 0xefc615255c840f2b, 0x14581bcabaebf4b8, 0xdeeb0a1d6846196e, - 0x6504718cb718d2a6, 0x7458c833b16ee8a4, 0x46af57755e4a3042, 0xd9003ece8d48e164, - 0x8b497234bd94380d, 0xf6a3d8a16d023dd0, 0x8a8a434f755a507b, 0xcf4cd1e191858dea, - 0x83943d4a184f3f6a, 0xb6de61627dfad18d, 0xba24472e8e19330a, 0x81e4a70713be9107, - 0xa7431d750b31bd92, 0x1ecbe4d27f9c6a0b, 0x05af46f9b59db433, 0xa330ce3a95348166, - 0x50b05d276030dd52, 0x59c83f1b9a572afa, 0xb798b41f2f62f6c6, 0x886b1f3377bee660, - 0x247a361319fe3cc1, 0x1ed0bfbba66110e8, 0xb30d1712172942dd, 0x35ff6d0d87f5f362, - 0x602ecbd54d91ac0b, 0x6027d9f46849a24b, 0xca2d1a0bf21bcf9f, 0x94cf2758ecb9560f, - 0x7c75a404c13809a3, 0x6851fe12538a55d7, 0x5167534be9365c09, 0xce747962fed11088, - 0x1abeee155636014a, 0xd53549d52a88a696, 0xe5fdc7637c001165, 0xfa53a788d9104c3a, - 0x3f7192826530fb6d, 0xfbac8773a3fa881d, 0x9c73c0dbad46939e, 0xf7c86971b9bd653d, - 0x8ca4bacb98b89d8e, 0xa2711e64bc8964e9, 0xdac128b1251c3374, 0x808887d3587f457d, - 0x0578e0fcff6951ab, 0xa4067c9563cfea10, 0x2438ef1fc2662bc7, 0x2be6da2b0c8a6df7, - 0x6cb6515b6d72f6ae, 0x1465ac34c245d295, 0x7b6ad40d18849950, 0xe4d4ed7932535e69, - 0x958ca955742edfea, 0x41b76779856421b7, 0x5144234feb710ac2, 0x3ddf687c6aa5afde, - 0x5cf58ac857a9ebc1, 0x0b446b91fd3f3dac, 0x30bf4fec9aceb927, 0x03e1746e379c38c2, - 0x1ead3d7506ef0858, 0xde2169e1ff2e3f07, 0xa04e0e9bd63ae7d4, 0xd723ebcff9888de5, - 0x07395f4c1f59bb54, 0xf3417285af6c1cf5, 0xcc20b97d1e3cad3f, 0xd523b663f3b96bed, - 0xc04d01fe94b6ecb9, 0x737b157fe94a9517, 0x9ae1017eff9e95fe, 0xeedf2b6e5d248ab1, - 0x9fea27df0f34066a, 0x9495cc69f5897be5, 0x9e7a431095deb9b6, 0x976e9bdec5665954, - 0x6774bb6ea3a41010, 0xb5a3ff40d6976e39, 0x868a2cb72bc2aa24, 0x1327f8d580167076, - 0xff516b4eaff277b9, 0x86acef212449afe8, 0x5b05b8d828ebe8f4, 0x2a1086f215d30fc6, - 0xb088fcdcff756dc1, 0xd1320bbf14d15d34, 0xc1b54cf7a23aaab7, 0x76df4bcaa135e3af, - 0x22394843026fa14e, 0x239717369b5a6f5b, 0x7a65b8de810c53a1, 0x4ea996b735048fbb, - 0xc2790bf34d3aaeb1, 0x42b358c25644bf6f, 0x41806749bf33656e, 0xfcabb6b360a938a1, - 0xa209fc69b32a7250, 0x540ecff21f17461e, 0x34237448e52bd3c5, 0x86d0fb69cd1686fb, - 0xa24cd197524dd615, 0xfcfe89574a078b08, 0xb384bb449a2ccd00, 0x9260df4a6cdfbb60, - 0x628f4a903b89bfc7, 0x8c3ae895cd89ce78, 0x52bc23bb6dd7371f, 0x94bad6cceb5770b4, - 0x72db9342c4d96ea5, 0xdaada6078904893c, 0x73aa770755161063, 0x6200c141f5ce49de, - 0x44dc29a5fcf908b1, 0xf3bf9022243a5674, 0x53e3e214a144a50d, 0x6c4d7f9f9678ab2b, - 0xc983b2c47125dbda, 0xb84386319377606f, 0xd72d378cef9714b3, 0xdcbf0d8b0d005988, - 0x1bd5a00cf3810641, 0x6e81dfde423dfd05, 0x9c0ffc6a4669cef5, 0x5a07bc6f5fc7cad9, - 0x3c05a87171029383, 0xa6bb1ecd838d25ac, 0x2ca784a482e11dbc, 0xfca25c22f3e0a05d, - 0x0e7562182788f19c, 0xbc3bfc8fef428b33, 0xde769ed1b9596ac0, 0x3b2c557d379aeb41, - 0x3016c4d1f3503870, 0xf7e1e512f711b676, 0xb94a56bfdc097835, 0xcfa602904d20a6dc, - 0xa15719e90ddf9aeb, 0xe582939b10f07880, 0xb7ed3569791178bb, 0x0e3237ac0ddb87c1, - 0xf7091dad6cc2486b, 0x0457d27d35347a88, 0xd84c0af5df0f78b9, 0xc507c767296db091, - 0x3414fb0a26a72e28, 0xe08d8337e5469de6, 0xc8ef52fac6373441, 0x4d60213033451980, - 0x140543bfc5b5e8b2, 0xc1a558c89b6128f1, 0x910967cf8bf96cca, 0x6c7b45bff49ee439, - 0x1bcbf912b7a1cc1b, 0xfe47f2cde4ddb093, 0x34e1c207dce941ea, 0x7de7624a7b052a90, - 0x87913e672762ed9a, 0xd0061b05e28ce326, 0x08571cba21a427f5, 0xa6c037bc60f316cd, - 0x7b978502f079dee5, 0x901b9e1856a5037d, 0x70b2b4b6095f179c, 0x4e6a4dc22229ad67, - 0x97ae112ec3412f26, 0xdf314ea54317a921, 0x7b735d2260e33b25, 0x6bdaee40c7d82257, - 0xce5d31f2c79a204e, 0x5cf8f91b91da5ba2, 0xf5cf0377826d5d98, 0x1fef1277e9d1378e, - 0x4bb8bd681ff3d08e, 0x49862a3bbb913e2a, 0x8d43f8638871fd9c, 0x64ecbf9e0183abe7, - 0x13912c5d134d9a31, 0x629f1bf407f061d8, 0x6b504da625ff4a7e, 0x22a815d4363bd5fa, - 0x74045039b41eaa4a, 0xe92de7096c064de1, 0x3a3172130c23a5d0, 0xd127ca1040853087, - 0x3b330584a5dd63d1, 0x84e8dc94ef904232, 0x960072fa3eee3a0d, 0x5fc1519b6cc0a909, - 0xa116e3319e3131b5, 0x037cbfc41de366bf, 0xc2050e43acd7d87f, 0x42ea8aab9757c117, - 0x40b18c5dfe34d6ba, 0xd97869840f1e6c00, 0xb52027a297d0e333, 0x00f22ab0511f6cfa, - 0xe3537bb92658644b, 0x92613f78d00f4fe4, 0xec3b6b6e0dc85ec0, 0xd0268caf36469a59, - 0xcb2ccbf84bdb1f8d, 0x491d365afb90a3ab, 0xc4a66a1e6d7bb2b8, 0x57c80d2995a125f2, - 0xf5a434e48e0fa362, 0x006787fb608338a0, 0x11cd5e41042da958, 0x5bed15f11ff66fa2, - 0x2ea03b577df6667e, 0x13559234530e9bc3, 0x2dec044a47812776, 0x7880017de9bffae2, - 0xa1b0fcb4bc9de2d9, 0xaf37057bf722eac9, 0x2b89536c5b66b802, 0xaaf82082b9a6f757, - 0xae68a25ceab89cc4, 0xef27d927885febad, 0x5f0314a4808d44ca, 0x89b6ec34f65ce39b, - 0x43c1affac5922c55, 0x988016f7505df38c, 0x1f5f19c9b8c2df48, 0x267b299dd8a52172, - 0x912902c65159e640, 0x596d08e3aed3a5ab, 0x2466f25ccfa34ab0, 0xdbb56011729db861, - 0x16a011f378994c8f, 0x751c09a4583b9274, 0xd2f83628687e76f6, 0xf0ef063e40cfd4f3, - 0x4560584cf0fbe4f2, 0xfc0a671fa2c404a0, 0x97a9d9eaea08539f, 0x968fed869ad68bba, - 0x2b2a3e3a89797288, 0xdf291180cee6e9a8, 0x199ee4f6cc77f2f3, 0xb1b77cac840aac3f, - 0xfbcc922f9e5ee166, 0xb1a9d8bb9f3fedbe, 0x0fc73db4740e9bc2, 0xbab46a53c130a547, - 0x68fe84dbdf5211f0, 0x1f0d4340dfad0169, 0x1f89eeef1a392174, 0xec72a0c8991e3e82, - 0x73bfba62d6f72057, 0xf2170c6cf8c3a83f, 0x07949703eef32fce, 0x48b78d1069935b23, - 0x689bb1fda12bde4d, 0x3f90dc8c95759623, 0x554d254249f90104, 0xe922bba7d3ad3b2b, - 0xa869765eeeecf4dd, 0xf54c7e3ed567c4b3, 0x6a5d1a21d56a968c, 0xd7fd2ec024b7614b, - 0x6c2f7285684ffc04, 0xbb0d8a86cef640e6, 0xa919dadbeef1a357, 0x426bc7bc63477dea, - 0xa5246f31a056bb0e, 0x2e1e6ec4d8171bfb, 0x93acb646499dab68, 0x5b6919d4a6001ac0, - 0x96b39bc9c58e87ba, 0x6aa7086f1aa196a0, 0x965b854972c81648, 0xa2e812f20d195d10, - 0x8b3e95bd9d2ae66f, 0xaf800082eb3d5abf, 0xbf5100c6eb0265d5, 0x4ff246e3a43817b6, - 0x78f0fb470cea9b84, 0x3007a3f9f7f08b54, 0x9a92649c21c82053, 0x0ff1927b87a6afc4, - 0x19fe00b920e728e9, 0x0ac0a280535ec4fd, 0x411d37fd5bdec7b7, 0xfd52538c0bfd3f84, - 0xde0ac8247803e43c, 0x03cb6a8efec2e6e8, 0x6e1063fe1a824b28, 0x43684a136a1db68b, - 0x92e6963267461951, 0x4094e184df0588b6, 0x7626c7aca82e41ff, 0xee108c77a3eb8de8, - 0x4da6a71a157e489a, 0x5783c0f67c0a8852, 0xaac779307443dc14, 0xd9b6dfa78ddc092a, - 0xf950cb2f056adb8e, 0x0f70b18908caeef7, 0xf13de4a8beb86726, 0x5f4545bc7b51cd96, - 0x7ab617c6534f50e9, 0x25bfe7e25ee5d4f4, 0x19a57c3e0180dca9, 0x40676b7e934792c8, - 0x1818c7eb11804f13, 0x7b5ab6e7e95e1f14, 0xa459876df80f93de, 0x01bf28031e8c22bd, - 0x421fdaf5336e90ba, 0xeb858b705df8ac3b, 0x10ab78a7779d7692, 0xbf497bedfae0e10c, - 0x3a87d33617fca31f, 0xc335ec215617f624, 0x156cacfdd063499d, 0xc1825c7f89a77381, - 0x046b9232524ffa0c, 0x94c3db6bed91641b, 0xdd3cf98679aafe14, 0xec1a2a4bb8403931, - 0xbd292a821da2777d, 0xca6efe1421ee7bfb, 0xb7d3f8057cab37d2, 0x6e5114316bc8803c, - 0x4e5f0f6a9bdb923f, 0x0efad2edbb720428, 0xa3ebf4468768b1d6, 0x4e3932aecaae53de, - 0x0f2ee639ee2f50b2, 0x6d258c55990155ab, 0x6617a49f13670727, 0xda3c877ba24b3704, - 0xff05c1e8788023c3, 0x8fdb437fb4ff505c, 0x402dc3af9258bc83, 0x4c9f5babe515574c, - 0x315161857abc74e6, 0xf5239a2f51211fcb, 0x2d13454115b5dcf5, 0x4c9d5ffd636c44d9, - 0x759679fb3820b4ac, 0x6157d00b7e6be154, 0xe7ae41546892b795, 0x2c129445c1bc6ce6, - 0x69edab403fc67076, 0x1633797e5fa2d6fd, 0x972b26e1820c4f39, 0xb14df5f6ede81514, - 0x8b374e08489b3598, 0x9df9dffad20e99a2, 0x4bdd9477400f326b, 0xeefdb2242c5368f1, - 0xd50871baf86f3187, 0x468d924244856f79, 0xc5048f63816024b7, 0x29f11355098c5e8d, - 0x36fdf0ed9f3604b8, 0x33e044cc51b35e04, 0x030c9e99789cb0f3, 0x93e85531eee55b77, - 0x4caf72a73ea01e49, 0x54b8fe828b159c0f, 0x1b78893763bb428a, 0x26576065bf049985, - 0x7089ff9a5d93d2ef, 0xe55948711b3fc9aa, 0x73ee71b859068064, 0x90596ff35dff30e4, - 0xa8b90cc2bfddf48b, 0x238d343441bcb113, 0x3dcb3d08bf90e6ac, 0x475b494996961dc3, - 0x0bdaebd1b22a85fa, 0x487716ca1c030583, 0x3411613568e786be, 0x0c6272b1c032c8d9, - 0xebacbfb4bf7efaf1, 0xe5c019e988639c65, 0x7cadf975d350951c, 0xbf5890e724bd6f79, - 0xa8ce9ac12918a414, 0xaac7ca874abbcc34, 0x9e5b9c884c77a6ee, 0xaca47bb2a372804e, - 0x6bfafc38534e7a73, 0x8aa20df62d0785d4, 0xc354596c4804a8a0, 0x11f64b990a8e944e, - 0xd048b5edc61772e2, 0x55c5570e01dded8a, 0xa81666f3006afa8c, 0xdd1b5adb8f94fe73, - 0x19e522704d5190d8, 0x3d377668e8e93a72, 0xe8d7120b56bd395f, 0x92df61afd56a9acb, - 0x604289e7d323fe6e, 0xbc22caacb84f6b17, 0x25b54ee1c9c6a480, 0x3bb29eeb03d50cf9, - 0x40f24e1cb5921858, 0x09067142a033d3f7, 0x44aca7ea2b1ed06e, 0x83869c5472ff8689, - 0x5fc247e80ca1392e, 0x9ff46511c371a0c7, 0xd829de1e2e3c888a, 0xe54b8788b4621f79, - 0xbe0ac46c5ba02c0c, 0xace826a5cfe6b9f2, 0x6996113fd32f4092, 0xf96eadf18884d3af, - 0x77e6a1bf5e6da0d0, 0x180d2bbcd9fe5cf0, 0xafb431c82f533609, 0xe874af2f75b9e653, - 0xbe30a4bbe39caade, 0x53254adecdb0a0e3, 0xa77383a42af53bbd, 0x28a83612e56c0b50, - 0x2b434bd134d6d9fb, 0xcc9c99ce66301479, 0x038b1b2a55061329, 0x8777d17523b3db6d, - 0x15e4b78242f5dce9, 0x8dad3e679a208d27, 0x5ae20a736d20e296, 0x0dac101afd7d7873, - 0x8f14a36195941590, 0xa7f8f9e8c9d7b836, 0x38d153caa947ad3a, 0xb477be0e68dcba8c, - 0x6dadb71862cc8db8, 0xc17c847232eede89, 0xb5bec1fcce8c82c8, 0x35995be12cc2d38d, - 0xfe24d0ac6b54cb61, 0xc47c2da2e09a394b, 0xb4205c9468473d85, 0x9e24d966c7adc5b2, - 0x90c948dcd32aa680, 0xa70ad2c006e8c8a8, 0x94e246c515a91c56, 0xc1d79e9b3140be0d, - 0x41559ef5d90a9bb6, 0x187ba99f9851ae01, 0xe6309320fd3d822c, 0xb883a69994887b04, - 0xd06120a5ffb062e7, 0xde5a295348048260, 0x1b6b2b063210df98, 0xdea44e2478955075, - 0xa5c1add5f2089fa6, 0xd99dbc2da1ac2065, 0x615f687ed624f363, 0x8bd2210d82211f52, - 0xbb888ca79264ad96, 0x2eec574464ddeb01, 0x6f9c9a8dcd5ded65, 0x5d08f82a2e5b5a4c, - 0x904eead81b4ab8d6, 0x9361e2e265189bd0, 0xa13663e8ea34c362, 0x7c71f4f0fbf66013, - 0xc6a26a5080e782f3, 0xa114c705bf573de6, 0x08241a66faecd24b, 0x48d1c855eea49ef3, - 0x992554d13c859243, 0x297cdbe3273f0378, 0x336256c4155629c9, 0x42dceb3e91e8e464, - 0xf4235b6d098e7c20, 0x40fb73730e14e31c, 0x246465622bcfb22e, 0x9bc424a1939a5a6b, - 0x81b96309f7a48438, 0x7ce2554aced69cad, 0x2f43e3792cd67327, 0xc133df1044ed8ae7, - 0xdf6694c0ff01d3f6, 0xa9f8f681a8cf4e2d, 0x6281794be68dd87d, 0xbdf4b6a12873d4e4, - 0x5656fb3fabba5200, 0x0c526cce582a1a80, 0x1c14dc018d39406b, 0x613066a5d182e00e, - 0x32b97c1994f5afd3, 0xa6c2765e015abcf1, 0xc29b24c6fa88cb29, 0x6dc38dae844cc570, - 0x42e89d5dcfa952e8, 0xc270c5393e2b706f, 0x9765840c4c7ee345, 0x36901f913045a6c3, - 0xfc89a26a56f0d15f, 0xa3b90e55647ae7c1, 0xeaa7d9ac0ec235ef, 0xb5cdf0652da81c4e, - 0x400add737d0af027, 0x19c1133b09f8a755, 0x9051e7ab527a9eba, 0x15c1cb5a3666c0cd, - 0xc081a2b9fc1722a0, 0x7cd59aa9e21c75de, 0x41dbe9df0d0d7da7, 0x9cdba46dd1ed97bf, - 0xf8d2805bac5f265e, 0xb4db0cb43360f86c, 0x7856f20ddb5ea18d, 0x52469ac04673bb58, - 0xe1d1285a0eaff331, 0x7a56a5801deec2bd, 0x6ce54d35ac3e6c24, 0x76166f682a0c0bb8, - 0x13b0231b473f0f42, 0x37ad4c5c3dd7f41c, 0xd2917573078a9177, 0x1482dd3506717799, - 0xc25189313bd9aab6, 0xfe897c04673b89d0, 0x4289db8739cd2141, 0xb2e8ffe059173e83, - 0xde6518ad3263fe2c, 0x0b95c1e5528edd16, 0x738babcc430cd2c0, 0x4867b783baf44565, - 0x6b91e16e3e2f1e54, 0x36def7d01fe5549d, 0xc77bb054fcac8e88, 0x3b734208f5a74c13, - 0xfc9fb0ee355fe37f, 0x1d709883a4eac78c, 0xbae05dfdfa33724a, 0xf88ef75c2f849690, - 0x1507cb09be47d347, 0x148414e4d18e9dbf, 0xd74f9956764ee015, 0x1db91c80924ec5fa, - 0xb1c64da89445c47c, 0x6a2ce728104174cf, 0x02797fb80c55c177, 0x4e1c7a0eb1ec8beb, - 0x01c3061d8c4840de, 0x964c13e6fec21fac, 0x84aa25758dc15434, 0x30a319eaff9f6663, - 0x2c50885b603cd4ff, 0xb0be8a89da713a8f, 0xaca02b0db40572fa, 0x0ff8f36386d46f79, - 0xf5e55796a1f7aef8, 0xffec31a4a7266f7c, 0x79d01c7dfd735c6f, 0x255c4a0b089122d4, - 0xc9c3f1ee32953bdd, 0x30d95b3c0d78c4f5, 0x3a6015bbb61ef46f, 0xaf66029648dd1d68, - 0xb64ef57abbbd0e06, 0xd63cb049a4758732, 0x76ea17bd1d4210ec, 0x9fa2752f4894b56c, - 0x25c8846c10f8d55b, 0x0dfab296cf979189, 0xdf21f4b98c009e68, 0x8435a137f960b35f, - 0x5d7eaf3b707d9400, 0xd8111ae74dada1f1, 0x5ab8226aa7da47ba, 0x6b10ea654beaf7b2, - 0x784369f5aaf64200, 0xcf142951a084ff54, 0x512befc4573d0c4b, 0xbbdc099e7a6a65a2, - 0xea0436889eaf06e0, 0xa366c8b8a54ec18e, 0x67e70ad8773aa6d4, 0x1cdcddf0acddeea4, - 0x92bcf3c771cb6125, 0x1e332fa0690063bb, 0x18df0d46a5928472, 0x387ac829666fbd13, - 0xae5b4cb847b8b867, 0xfee71fb8c24d53ef, 0xcda3b5e4667156a6, 0xdf87fe30b575305a, - 0x35e254eb0ee12877, 0x2cad705906f0df75, 0xe3c80d6fed67696c, 0xed46a7fe556be8ad, - 0x6d8953dc010aa60f, 0x070441d8fd47d744, 0xa213b43ac64d036f, 0x893e5d6fee4ef015, - 0xe75b3d2b0cb2a7ba, 0x2883df2c1dafee8e, 0x794c6e8be46eb8f8, 0x93bed1e7351a39fd, - 0x1a14724246e2c570, 0x08545d4903bc372c, 0x194f4aa832981ba4, 0xd625b8a2814f8584, - 0x5b6274a3c48714f7, 0xeb2667e1047ca5a5, 0xc57170180b53ed9c, 0x134009b98a2ac155, - 0x946d346d1517bb1b, 0x9e35fbecdd94a50b, 0x0995116db467030c, 0x28e2b1860a96b857, - 0x5b678d488ef00e16, 0xe8826c8c3bcce1fe, 0xaeaeecdfe68bb3ca, 0x5e788088f54f4819, - 0xb4b336291bf85317, 0xabaa05d858c826eb, 0xef4843261009cdab, 0xbd875991eca352f8, - 0xee03701aea6f2c57, 0x4c31ecc7c06937f7, 0x59bfb40953bad9c9, 0x7ab89a63367ba78e, - 0x7bac880a27f357ad, 0x501306ce46aaff31, 0x8938e42f27f447d0, 0x9056450e67259fcd, - 0x0fea68db6e205d05, 0x47b30c045aa83856, 0x644a43721793c28f, 0x20320ed633c848ea, - 0x09a884710925a32e, 0xd739104d0d299654, 0xc5933acd834e50ae, 0x225f8ca6aeab4149, - 0x0ceec188bf00786d, 0x85ca981a1af35a4c, 0x196b19b897409e0d, 0xafa55465ab6137c7, - 0x0205c6c4aa9b11c5, 0xf7907f349e428b1e, 0xda1b37a73d8b70f7, 0x02c348db7df7f4cc, - 0x371b1e154f1d1684, 0x3d5add9cb8475499, 0x69d9a57f95d59449, 0x3cfac6962e9c28e1, - 0x4778a67f68b79e5a, 0x893c88409ef7e0da, 0x9378f43b73bdeef6, 0x2f9c8ebf31f7b642, - 0x08e42d70e6f3ea28, 0xe9be5adae098f8eb, 0xf51249ba1faa8dc7, 0x6c3585be5ec259ec, - 0x70269600f9911f5a, 0x7ea9af5c4f07a942, 0x075b6fc5f75ea390, 0xb21a6e60dfa29682, - 0x8c50146f804b1b9f, 0x0464aa73c73e8905, 0x0f49eb00487ad130, 0xd930e7e00697ec39, - 0x5167cc479646f250, 0x0d7abc736b7cdf87, 0xdb739ad6072cde57, 0x069b0f39ea3a1416, - 0x445a18743fea407f, 0x046e9c093fe700e1, 0xf8e752ee3da3a8c6, 0xf863a70808a74450, - 0xa2eb8916cda92cbb, 0x43e3a08c285ced75, 0xa52753033f922225, 0x163e1727f769e03f, - 0xedfeae114f39c095, 0xd02157263b636a69, 0xb53e411f438dc707, 0xede4308ec4ac50d4, - 0xb995c92e7e4bb094, 0x3c7e41f442461026, 0x83c450fb07c11807, 0xb07f4dc10c5dfb21, - 0x1ab433298f8a3408, 0x4c1a42953124a517, 0x868d013a56dce7a8, 0x87f0149aa6e3dc96, - 0xe4c68bc17dc5384b, 0xf0da45f665f997e2, 0xe1da31eb3f34c5ec, 0xb6a7d7e229e99c3a, - 0xccf77b129fa25ad3, 0xbfbf7f3962b81df4, 0x0c80539e69718dbb, 0x25d5bc9245d1ccb7, - 0xd068eafaa29a8ac9, 0xe47bb3e3ea8dbddb, 0x5c8186558455768e, 0x77c9798ddab39717, - 0x0c2fb364e736d720, 0x43e5200174a5879f, 0x90a641366f6b258d, 0x650f3cd224809799, - 0x3b3cf5f97c1ff1f6, 0x546d3eaad3b6e6ca, 0xd25eaa01b5fa4e8c, 0xd7be540c2775be94, - 0x08273d6257920d92, 0xe85d0ee8d94f2602, 0xe5368f9178b01c5c, 0x08ab0975fc6d7f66, - 0x36f958744416bbc5, 0x9290415ccc2a10da, 0xd1537fcde590f42f, 0xc16de6457a5585c5, - 0xe62ad2b95fa69dac, 0x50af2a68eb8b5056, 0x084fb06391c15569, 0xb4522963fbf9660c, - 0x5fe4f1d78ad59f5b, 0x838b1c9179758ddb, 0x55af2eb06635e7ea, 0xfe4adf4dc194a152, - 0x24127542657b265d, 0x132bec02a05a6b26, 0x5d08eaac00b41636, 0x765883bac67b1dba, - 0x0fdbc4c078ae73af, 0xb641ef36e19644c4, 0x502deafe3ba8b93c, 0x0cb7b96365d59bfc, - 0xaa1785724cd42afd, 0xb79df658520bfd8d, 0x8b4ed0a874202b43, 0x896e717dd161eff6, - 0x0515b453d9459832, 0xf2f94c55e3500a7c, 0xfa32f25066199447, 0x0bf427f233b9124c, - 0x84d39f8df58025fa, 0x8714b1d7dc8b3f83, 0xc18084fa9e8b1eb8, 0x721026b690eea7e4, - 0x049f529623a55ee5, 0xe0090e39c816f015, 0x36bb3779466142a2, 0x7b8535433fb93791, - 0x9c57bc9d1c208d31, 0x7548447ef77b5d0d, 0xd90049461be45f89, 0xbb9d3bd5ff3d2900, - 0xd6b50c63b25bef21, 0xd0ae157cdbefa488, 0xa3b90f2e43d5551d, 0x074c430aea06db50, - 0xa11703bce84a7072, 0xe9618cc289c4ecb9, 0xd00304e8d40dd9b8, 0xd55e9e038ad5314a, - 0x23bf04ec22e743ff, 0x3d516e9382f736a1, 0x0d381a166b1b4ca1, 0x24c9ca98c7b9ca53, - 0xe1c1d830fa7f5ba8, 0x5b675ca5f7bb21c9, 0x9fd1a02cd577544e, 0x6cf499feb06aea50, - 0x977dc00c65fd956c, 0x93679772865e409b, 0xaca4e42af914f858, 0x9ee092f92cdb1345, - 0x68ac7ca88ac06caf, 0x0af3d8c4dd40c1f9, 0xe7951de435c8283c, 0x97687faea51325ff, - 0xe1cc8f3ac35801e6, 0x25ed078bde8dddcc, 0x16ac207d74f0a358, 0xcf7de2add4517fe2, - 0x5443b9f7505c49a0, 0x7d485b7a4446761d, 0xde2d75f54692f462, 0x3b7ee4b97a19300d, - 0x2dcd3f8deab62f53, 0x10f64eccd4d5661a, 0x158c7a2c76d00abc, 0x7e12b5bcfe939b35, - 0xc8a419393bea5e40, 0x2c58e14cf1381f14, 0x64af1d99ee4e06c5, 0xca6fd43dfe3acf31, - 0x5fd63bab39eb3c43, 0xaa91f35280f7d9ea, 0xae1bb10c5927526b, 0x8a54be4ac20f48cc, - 0x0ae8b4a0bad91751, 0xeb67ff75ee124346, 0x1515546d1e979c55, 0x14f427ea08579062, - 0x417221524b32603d, 0x0c07be6363dc5696, 0x3069a027fefdbfcd, 0xde0a8a07c2fee3e2, - 0xc59db4df32b08ff7, 0x10fe9fe70a741496, 0x40ce7d70825e87ef, 0xd4ee68a360d4de49, - 0x23be82ca13ce0224, 0x4818b61cbd591346, 0x6d6bf93712ce7464, 0x623efd3623df3fcc, - 0x49803c72d33329fa, 0xf804762425597656, 0xf394c10d792761ca, 0x9e34c2d7f44c7d24, - 0x2befd74db9298810, 0x9425c0dd65402b81, 0x7da3ce88b6b1b56a, 0x1f8c171d276e9577, - 0x9f9910e7da773bfc, 0xa5171b7d3ad1037f, 0xbc6e9c797714726a, 0xb6e24b1ec9a08a17, - 0x1c96639e95980a01, 0xa3752e0d705a9fc7, 0x1fe5201d22672176, 0x95516019d82e11a9, - 0x6215f02966725e56, 0x659d490acaa494c7, 0xe667c001be396dc2, 0x6e280443baf82c2e, - 0x5da036f17f4748c4, 0x350e657a1651a233, 0x3057b08db6472855, 0x0e1ac9d88435a912, - 0x495aa2f5cc734dc1, 0xdcc0be4798631546, 0xed132b2c7b16ab1f, 0x26167f1e138c72f8, - 0xf1cf0c3c0ba50842, 0x7ddcbad78acbdfda, 0xd759ec70663d78d7, 0x9871251848dfa580, - 0x4691d4d7ef3eebc5, 0x52d9962c3fe7e406, 0x546a76323ef9c028, 0xf81639a7f4892429, - 0x6f47b71fbd2ef211, 0xa9aa1eb5c554759a, 0x5de7c7e8482deb45, 0xd4556bc5748dc84e, - 0x720022c1ac31fc1a, 0x8f3117ca2a0325ed, 0x21f1e02fde7327ba, 0xa3591f78e8811dde, - 0x041ca47c58c36a1d, 0x0ca3868fba651592, 0xfaafa93e97b50c6e, 0xad2f81dc4e030785, - 0x121a888d3fbe356f, 0xf1f665886adfd844, 0x5cd23fccb276cdb2, 0x7cafe4410eb8964b, - 0x302a7431516bbfe9, 0x1d9559d87f515393, 0x547b2aa97dabfbed, 0x81f4f876b3516f3c, - 0xe455da3cc9d8fbe0, 0x1e43664ba279f7ca, 0xdaba600180ab0825, 0xcf02371835bd53a6, - 0x8246c1368ec1bdfa, 0x10ba6cd521dae891, 0x181526480d0b44b3, 0x69341bf7381904d8, - 0x5e67552e790493ab, 0x0edb4212b05013d0, 0xc3775c491aef3cc3, 0x5926e8b08d74e5a3, - 0x0d360774a0bac8c2, 0x656cf785c1ee5c06, 0xa8cc674b4d56cf35, 0x908a613d6468bcbb, - 0x0b9edbebba23c8b2, 0xd2c9f982c2573f1b, 0xda0cf29a4144c5ae, 0xec20e2a2edf03a76, - 0x780675bf2762c334, 0xd48b43519b83a6fb, 0x161211289d9e759d, 0x8b4b66c077a71214, - 0xb4a3921c4f804865, 0xa27239bb7b5b400d, 0x16f2a7087f051c3e, 0xb5991a243b380571, - 0xe5f45026aff515ca, 0xfc037e5923a7bf8f, 0xbac2994a59bef6f6, 0xf0101337c165affb, - 0x2569833cfa264583, 0x940b8b0728f5d134, 0xdace65921960a40d, 0x6d97a08ec536e3ac, - 0x4d5df696f6657d1d, 0x2e01488e31fd69c1, 0x403bf7aad9fcc776, 0xc9ef410832f57057, - 0x100857d549fdadc4, 0xe6be79ee6164bf1f, 0xb1da3093ade11eba, 0x713fc73f8abdf80f, - 0xafeda1d704aa6be3, 0xfb185d6153b90797, 0x4263e3770a2e015d, 0x1b1bff41aa946c1e, - 0xde832c9b976e57f4, 0x2174681cf8d5188f, 0x1db8756313eddc8c, 0xd6077ccc4237901e, - 0xc4d82fc3af0a71b6, 0x9e410189c061f42c, 0x0bc1db01c9b5736e, 0xf3aff6b75046cdfa, - 0x5a5b799c8332d991, 0x29e61f65c3a30e2a, 0x8fdb0fb6359909a5, 0xbca4d75d1696fbe0, - 0xf081da38d0ab1f14, 0x2182a269101bfdfd, 0x11cef22081aca576, 0x284032f083e6ba2b, - 0x6f7ab311b66c126b, 0x78d9b20cd3c2f7b1, 0xd28adc93b1aef8fe, 0xebf4740b50657292, - 0xf138efea7d362793, 0x4b18c0cac0220dc7, 0x4a89f79b064e0773, 0xaedbeb1e815005e7, - 0x1469009125e8d770, 0x07147fcb58b01491, 0xdc7fc77dea85fa3a, 0xc6907260d9d9f1a6, - 0x342d1f2a45dde648, 0x39fc314db048f4ed, 0x03c154254e5a8325, 0x88c9804d83488d1b, - 0x7a313aa8bf35d9cf, 0x99e90306963483fd, 0x67fa5dbb3f919a77, 0x15e0ca3d7ceae94a, - 0x3b028350feba5ccb, 0x403324d8635462d0, 0x1a852cb0546d4e2d, 0xe3cd388d14232de3, - 0xa74f832cf5e103b8, 0x759018226ffe3a9b, 0xbae0641b6d9af50b, 0x88af72c72a1735b9, - 0x5a5e3d32754fce68, 0x732aea64c428ddfb, 0x5cdcc140659bb0c7, 0xdb39fbdad503ea86, - 0x0e32561516957cda, 0xf8b53d3547794eb5, 0x3903f263c3345c54, 0x95d3257f0813f011, - 0x69eb601369497929, 0x48a20bd9a9887f43, 0x297544f583a73551, 0x17e674489f5fe1c1, - 0xb4f94fdabcd4437f, 0x469306dcaade1f45, 0x4cca8bc8cea5d7ed, 0xb1e7152e7f812741, - 0x78f0b4a3d381c90a, 0x21129b9bf3c19485, 0xa68b422f6b7b5fec, 0xfdfdd1c380283fb5, - 0xb3ec690dbe654d44, 0xec02ab714b796f70, 0x53ac4a2792e55174, 0xbe89038f89d8c86a, - 0x1637a3045b672422, 0xafa0df471fb15049, 0x86aa975c79b4d651, 0xcec9b5dbd8e3bf68, - 0x172ee2e480ddabd3, 0x9e1eefce76c3ed06, 0x1316041826e0925e, 0xf97d6a6c9a680f42, - 0x8a57d9591bedc610, 0x24b5e20eb2ab3950, 0x0fae09f6727ca9a5, 0xa08b5adfb61f0ec4, - 0x85acb4736927841c, 0xffb27a2605983fe7, 0xdf185bffed6955ef, 0xffd95ddb9dae5f98, - 0x98b5e8c6350d660d, 0xffa09cc75b34d34b, 0xf8316acf61a85578, 0xd99da1a593625494, - 0xae721eef6ce7e3fd, 0x6e9e8f8a0ace30a6, 0xe310371ca0632214, 0xf011c7362b59c96a, - 0x0a2ab5bfdb8579e2, 0x26b735ba43772908, 0x67333142f56ba2e2, 0xf04e6ea0fcb036b2, - 0x5cff96ee4f7d2244, 0x1e63f10d32fa09c7, 0x345ef5049d946c83, 0x4755ab79b772bbda, - 0x7360ed00b0bd0e98, 0x0f28ce27f81e1146, 0x74f9c03f439ee23e, 0x5bbaf7949ab913e1, - 0x760006e8cf8a2d95, 0x73c72c68976f4109, 0xf3c2c188d1fbf78e, 0x7a84450f71b21405, - 0x5983b48f4db210c7, 0x6f812aa59a7d0920, 0x6a8d46113fc21984, 0x416e18941b374686, - 0x1a2c50d13012687b, 0xd3ea3ea214919d51, 0xb2a4d2fec7145208, 0x0bf1c295fb5ac741, - 0xee4852b85bd0c569, 0x5380f776724dbfce, 0x61b1fbb727b42137, 0x13b7317880b932cc, - 0x837b8f8051d4929d, 0x6e99b631f89130d7, 0xbd4a1cdd8857c699, 0x1cc4a20351f73f00, - 0x57638d9c60b719dc, 0xf6689f77f6f83171, 0xb4df983a0b0bf70f, 0xf59ae1562ba18dae, - 0x963bfccccfafc1cf, 0xdb4d2f74d57b2701, 0xaf4a34f60ef9d5ea, 0x6a65ff0450cbcc47, - 0xdbd8fbbb0098cad4, 0x5ae79b4296dd9f06, 0x02c3d3db41e352be, 0x3a5a04821d141f3a, - 0xe85831b1ac7aabdb, 0x109762a12d49bb88, 0x216d9eff75fb3424, 0x80289341c16292cd, - 0xc72ccb89114baf96, 0x7fe7717e3583dd3a, 0xa017fd4711a2aace, 0x7e0e66bffc736322, - 0xe9fee601f71953df, 0x87bf9aa7e70e55e3, 0x139b3a8011c7ce57, 0x91a2961d63429fdb, - 0xa8b1e5baaf9682f3, 0x778afe6dc054e7e0, 0xf8cd7ac9fbc68977, 0x53c1b40ca51890df, - 0xd288bf668e9f8f5e, 0x261ad8d7b74100b2, 0xb995a82fc8fa14c0, 0x2377d33f0c1ebf4a, - 0x464466bf2a4f7889, 0x5ea7d2dbed54a6f3, 0x8fc81bfc30701d06, 0xb03c5eeef6d9491f, - 0x98337168f7de7c59, 0x91b42b56dadc2f25, 0x7a1f5eaea3222afa, 0xd10464548858fb61, - 0x1418513fe2ed3b85, 0x55c29995c9bc85ed, 0x685bb0b9fe5e7831, 0xa55dae869a227f6b, - 0x4c23c6d4b18dc335, 0x7efb3cb3192db0b2, 0x58d3b08a0c5a6a9c, 0x39ccf8172529eff4, - 0xfa65e19f0a212f2a, 0x6a6538e8335de044, 0x799537d940f69da0, 0x4ef5db64ced9f4aa, - 0xdb0c01d97a675467, 0xbf8102c9d27a81df, 0x178a385775b7e807, 0xc0f5708e3b9d4604, - 0x18ff1ba6cd444785, 0x17c889fc54ae7388, 0xfed408c5202f3662, 0x0a4e39e0b18f6f00, - 0x8a3edf99a8455688, 0x6546ea1100d94444, 0x9c2dd7ca0c519b79, 0xd84353ccd06f0f84, - 0x11d184111eb9f9d1, 0x5a5b6e1cc5897e70, 0x25236e1b4b253360, 0x18b1e14ee642f13d, - 0x4b55a28e2f0bf78c, 0xd900cd65e3e040c1, 0xab342a44b9cff5ba, 0x4e1abb4c8332da69, - 0xca2c506136d968d0, 0x2bd3ea0f4aa52317, 0x6b6fabd4014072ae, 0x4bb7efd95ab6b5bd, - 0x9a4d42e391d35275, 0x7d5248093261990d, 0x459e358ea86a3b82, 0xb93f5513a8ef5aae, - 0xabfeefba99aadb53, 0x6311aa0de20538d6, 0x0a866343fa38d1c2, 0x3edd188b6daaa7cc, - 0x5bb5d9e52aa226a4, 0x91b984d169a77eef, 0xd963dd6d25886c27, 0x8c42c296156a8578, - 0x7c06357db744c2ad, 0x63c7554162da41aa, 0xf60042505cd32c4f, 0xfd0d00c0e5d8d81c, - 0xb2a86e3e9c492521, 0xa3761921215fe3f9, 0x9a7e85a3f9830186, 0x610455c30b627990, - 0x1480519d41b78c8c, 0xacf5fae042f389e1, 0x4e3c71c6a0922b5c, 0x7144a645ef671b9d, - 0x7c9caebc6485951c, 0x74f005c17e7643a3, 0xe12c423b59593205, 0xf0747ef10b12aa39, - 0x958474cf7aa92777, 0x680739c8ad1b6487, 0x17fb4af6687d0fb9, 0x456782f50a62da25, - 0xe6479a255195bd23, 0xd52f7da3dbb62661, 0x43ba9e45d21224c7, 0xa00c85b2f0353eb9, - 0x00cc9095afde7f59, 0xb5720c23d85b29bf, 0x98c9aab154ac077a, 0x6bbe875a6c28e02e, - 0xd927a64287dcb9e9, 0x88ea4d58e11c8631, 0x47484145e8216bd5, 0xc621e68a8bf5706c, - 0xa771bf92bf2cb846, 0x760e8dea5262a69d, 0x3e3e22a211886726, 0x679fba6dc925f014, - 0x47776e3fd7fb991e, 0x3426b5650095eac3, 0x05489588d1c7be72, 0x1bb06ce09a1147f5, - 0x68784d5b1ecb2f94, 0x3b1fca86e2fd4e86, 0x720b553fe5aa85ef, 0x9b9e9c18c7469115, - 0x12e6f7437b91fbec, 0xc2b352a9f774933c, 0xd660ac3a93ca1aa3, 0xa21a8212990eb4a7, - 0x2eeadcd367965ab9, 0xe8504b240f220d76, 0xc414c61ae2489c32, 0x343505b24e615a9a, - 0xc80504b4b188e4ce, 0x23b4e13ee270db69, 0x9b4a29960eea772f, 0x05f1b1bec2012b2b, - 0x9cb5917147ab5e55, 0x9b2e7cf39e67e0b0, 0xce142e9a567f1a3d, 0x492036154bc9385c, - 0xc7633ace53e058e5, 0x6ff55315764fb252, 0xb547c490d737bfc1, 0x3ba0dd8c4f1d62ea, - 0x98d378f883e9e4fb, 0x3a48637f37dab652, 0x469f122001321551, 0x2fedfc843a1ccc06, - 0x9c36b82c6956d6ca, 0x5c33b52e160ec8dc, 0x8892d331b30c4545, 0x6ac22c6dfe8ec1e2, - 0xe81ab043d7b9f339, 0x978ebc89af6c6d27, 0xcc346445a73fe790, 0x7260dc1a53e29432, - 0x8069447d490b73e6, 0x359fd59be64742c7, 0xd9d800290e0080c6, 0xa07c6ec91e964b92, - 0x9595e6d861f38dea, 0x7dcfaf54f9275cb7, 0xfcb0b80701e1fa64, 0xf5111f306a9eeb92, - 0xee63cbb405cd4c37, 0xfc53d1887aa00127, 0xc214cc1ec26961af, 0xf2308026c802f880, - 0xba076367e293be28, 0x857c09b243c05633, 0x4fb26b28a1c5ad8a, 0xeb5b349b0deefe63, - 0xcdb91e9c2fe6c1b6, 0x20a245ba4d49730b, 0xdbfbe0326ff70ba5, 0x2d330e5c629b4218, - 0xe642be090ce4a911, 0xb97a6999000ea74f, 0xd051cabe64a9ca18, 0xd3586499ae672f44, - 0x0d5d896178b808d7, 0x200ec59f48d0b2d1, 0xe382623ffb86a970, 0xccf0553cb78282df, - 0x7f5b621be4082b5c, 0x486caa27a0fb880b, 0x80de1725e7c20169, 0xb0dfbca1c882cfee, - 0xc987b1f4d1de4bac, 0xcfa100b3a53b4f59, 0xde0f923a549285dc, 0xef1c52eb27330c7f, - 0x0d378305373b8cd2, 0x5f37b8588fe833df, 0x9b74dbe60118b62a, 0xe17599fa643ffb7d, - 0x9eb91d4a40f91903, 0x199a5943f8253942, 0x097da93dfc8a0b50, 0xf21ee9d435c5c4f9, - 0x8e1e62d9bab403bb, 0x6824104aa8fbdbea, 0x1804a2a0ba867ab4, 0x01c40077c55f9600, - 0xa711afc0d8277a6c, 0x1d8cccde407bae46, 0xb1624a5794a1922a, 0x0d395ad2b703fd31, - 0x0fc26ea87fc887ef, 0x53fbee0f7bf1d96e, 0x6777b7a2732e76ab, 0xfb4c5b6f3cd92e28, - 0xfe5ab944959fb043, 0x3cda7eb55ef8de50, 0x38f3435cae705175, 0x471b3e4925bf69eb, - 0xc2f29b5df092023b, 0x495117f926ad4843, 0xf9f95bd69f2c275b, 0x0ed098292c3460aa, - 0x07aef0ebd3a20b35, 0x8823a9c27ae892d6, 0x83a0e3380a36e1c9, 0xc1015c4896835b13, - 0xe531d415e69369d1, 0x1cee39b21177c203, 0xaf918dcccce417cc, 0x2af0045ad0f4a672, - 0x94322d44bab8067b, 0xaa4330dd359fefaa, 0x40ba7e116b3d9b3f, 0x57695ba81d5261a3, - 0x7673fbbde34d5f50, 0xa899c1790c552c20, 0xebb9a6b8ea1ea32c, 0x04a686f0cab349c7, - 0x2052275a7b642aef, 0x2116b627ba1f9531, 0xa750e257524f611c, 0xa364cb782487ef46, - 0xcbb879ded6a5b1f3, 0xbadc046237e53da1, 0x4be9ea6946674874, 0x513be119cc49fdef, - 0xd0c9907476489ad2, 0xb0a365a535acc4f8, 0x62f1dc3e1f94aeb4, 0x40b06b62e1829fb1, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xca542cdb412698f2, 0x4fe1352a149847fd, 0x253c6ab7ff1fe06b, 0x38920a6f5a7b3a7a, - 0xc203996f363b0362, 0x3c81172c1bbaefb7, 0xc8ed2c81f0946ee4, 0xcb3ff6929a5b190b, - 0x4895710e25ffbc49, 0xebcdf1a88a09b25c, 0x35b2118c1435c20a, 0x3ba2e4a4b73dfcee, - 0xc2a4ded28bff63b6, 0xe61e12701546657d, 0x8f9f2e9cc01a8e8d, 0x18fe777d842a4f3a, - 0x9354840eb4c6b0be, 0x9580b03bef7770a5, 0x15de050c15dfedfb, 0x722f0d54118855be, - 0x8863cc5cdbb80a35, 0xaa3b92204364a650, 0x3dfe40446d3d7a57, 0xafcfedd19bfe7094, - 0x107b4dfaa9e1ebbb, 0xf7ee4d8ac4c3d95f, 0x3672ef04d269ad96, 0xbf822abfd1ee162c, - 0x5aa76cc7b0d35ffa, 0x069afdbc39a0a204, 0x7e734908f3d1a9ae, 0x10c4def6fdb04a51, - 0x621767e85818784b, 0x3f077a9cef83a6aa, 0x3a4f6bd64bc047ff, 0xefc2556901e43847, - 0x2ca1842052079a67, 0xa64d538c318f4de8, 0x4d01b97221fa6ce0, 0x2c864d4bd2089e8b, - 0x271f45a9b403ad9b, 0xe85ee41a993e2bde, 0x8fdadd5ccead598f, 0xb7617ff66bc46ed8, - 0x019379f0caa3154d, 0xb6b2dcd0226711b6, 0x4ee064fd1922c930, 0xfc24496c13763d21, - 0x12686aa52b53cf21, 0xab1f72444b0a41b0, 0x43973298a330308f, 0x2b8fca4a350510ae, - 0x347df266db8b818d, 0x05e64769b5b46f78, 0x7e4fe50bcffe5021, 0x86afcbf747b0e06d, - 0x705802926b26329e, 0x5300a07e90521769, 0x548991830e03a58f, 0x6dcd63c828a15e52, - 0x17d6fb7d1b87ebcb, 0x5d9acfc636497674, 0x2cb1261378d4e408, 0x8b6e8f44c354ae5f, - 0x197ef54f3e12bff9, 0xf4113093d8f79dcf, 0x13b80cb3995534bd, 0x569d1bf30b0fa502, - 0x6430cf726836e5b0, 0x37416f13cabdd815, 0x8ff2dd8ff02f2f3e, 0x24f12eae9ffbbd14, - 0x3f67f8b7cd8fe85a, 0x4bd8afebc5171990, 0x298f2ce406588ece, 0x94f408558495097a, - 0xc6e61fca75dd4ca8, 0x317ab11d1dca6f51, 0xb59ec9819c0ad1b6, 0x2984fba6501365b9, - 0xbf96ec9b6a31c438, 0xacdd91a1bcc69e04, 0xed867b8438b6bca2, 0xaf82c6c039ab82bf, - 0x32939903881b4d3f, 0x6dd20ca0b1b5f101, 0xce9ed6af845f2bd9, 0xf2785284257bb443, - 0xf0ee36539251aa08, 0x8793ce0c732eca08, 0x1c7d5ba42f94144c, 0x9bf23b4911ed9ee6, - 0x8a0ad0b4f553cb88, 0x733229bd5fcb40ce, 0x133b9a981721963f, 0xe33b1e3526647681, - 0x7b65cdf437c85fe4, 0x578540579a49545c, 0x43e19c331c054ccf, 0x92d2e7bd80c7147c, - 0xd2d531cfbb27f2d9, 0xd57feaf66c2b28c7, 0x75796460aa352884, 0xf4f120c36c2dbae5, - 0x9ad43fcc44d3f45e, 0xf5d38b66d62cb6d6, 0xdadfee87d465b416, 0xaefc2efa00e9f86c, - 0x029d696c59e2b923, 0xec76f5dc830aae30, 0x57927a620d08b48e, 0x7efe6418968ddec3, - 0x3d0fef913c572198, 0x91bde403ca8f6de9, 0x6b92638b515b2fa0, 0x91c2fbeb1b9c2fda, - 0xf73cb79442d2a860, 0x5bfa950b9e174ad0, 0x28ae705b9ffa78c2, 0x7c865847d0ff854f, - 0x0f96d3f38986420b, 0xa48d049f96a84471, 0x1de174439a27bd9d, 0x0df68ea8e350b163, - 0xfad286b116126eb1, 0x83fcc047257596e1, 0x13b83ebfed772458, 0x8212268d3fcdf59c, - 0x873ec8347618de7e, 0xcacfd64794de34f7, 0xa304ddcc95a06c83, 0x2054171501db5a99, - 0x318e5adeeeac144c, 0x4b18d7be3e724965, 0xb788a118c9d7b8f4, 0xdf2eb71646e07d78, - 0x6d2a0e9de16e21ff, 0xf6f57b1b9b2177e5, 0x37cd2d09c4671edc, 0x6ecf09859efc225d, - 0xdb61ef4d0d57d7cc, 0xc40b48bc7dc7270f, 0x8f1d12e55f717841, 0x1fb9fd48c8acfa82, - 0x3a6cbaf0b992ba43, 0x47b9e4d679e7d89a, 0xbe62a0680083e6bb, 0x021aaeb6bbfc29d5, - 0xaa94a8104d39d973, 0xb5d3e8cf62ba3f86, 0xbdadd0b34005543f, 0x00c0bf98e12e1623, - 0x0e9f85157ded4102, 0x1ac277a70ef62372, 0x10897f3039068712, 0xc5c7c4517de3a2c5, - 0x14eb10e0edfc5b70, 0xd7b6e5cae42d74f1, 0x2b51d550b91638e8, 0x2ea372748bdff08c, - 0xdfe62dbb40877573, 0x444eaae83d4b4546, 0x6eecd0b396593df1, 0x6dc6a5e0e8e35123, - 0xac56fc243c23829c, 0x866097dd526b74d1, 0x629175c1c1783058, 0xd9f040c7d2f66d09, - 0x7252253f6339a75d, 0xbb6829c08e074f3e, 0x9a3586118f393b5b, 0xfd27c71c68996f74, - 0x5d10b33a097c0996, 0x06ed99e18b84f700, 0x1085a782307d3f61, 0xeb8c90d62375e292, - 0x434dacd0d2954a7f, 0x12f6341e203e26bf, 0xdf9168be72dad9a5, 0x17057cb44bd78cc4, - 0xc31debe76ccc90fc, 0xf77fce074c8caa92, 0xb9445e94de3b8bdc, 0xd800a15c6e7452ea, - 0xed96f03d68976027, 0xc4d7c6e3af6e8da6, 0xc5b413e4b47fea69, 0x7c4cbaed97d0de9d, - 0xf1900c8de14774fc, 0x41426091e5716f61, 0x09ccb84fc40086ae, 0x1cc35e9bf2074776, - 0x0a275cf337d59d4b, 0x3974a4298c37468f, 0x95b5ecd44da403d8, 0xc521b62e2d5f614d, - 0x8d03224243b244e2, 0xa10d5958a4793089, 0xbc1c5737c81e2047, 0x1bc86bd13dee8d31, - 0x34d4e55e8e586b30, 0x5bead4a4439efdd1, 0xc4b2899b5d73d5a4, 0x9bd56aef624255af, - 0x0075ba8b8e782b46, 0x31b7001e8972101b, 0xf36e5c0a64c75a01, 0xeeaa9536595276d6, - 0x115a9594b45d709c, 0x44bdc8c3ce74b728, 0xc088f6b75307b5db, 0x4ac2ee2ffe352c74, - 0xa97b9403f302ade2, 0x99aba346d50e1b64, 0xb0a27d8ecad68e1f, 0xd5c3bce8bf88e176, - 0x6238fa7472fc15f8, 0x0193fb0658cfe5cd, 0x6903c93374fab7d2, 0x18d7e2053ec6afef, - 0xfcd6cb5e5b6c23ee, 0x80659e2041881669, 0x51a1aa99f6d0557a, 0xb937a825ea1c421e, - 0x98f0aa8673807776, 0xeabf7256be842ddf, 0xb6eb56b64f61213e, 0x04c765b283cc018f, - 0x181c5c7929caed11, 0x7e730cbba3f24128, 0x2da688955cb1424d, 0xf7ad0cd131bef18e, - 0x1506f350b53fa5f0, 0x8b017838727c83ff, 0xd97cc85a86a0dd5b, 0x5fee32732e1e4302, - 0xe11eaf20d6abb238, 0xa97d2abc66144aab, 0xc217722da3f9d46a, 0x51557cd7c90a6777, - 0xa62e3b5cffb65d08, 0xbf54efa2120417af, 0xcd7badfd7b702402, 0xfe31d5804efcd7a8, - 0xc6368d5cadcda961, 0xf59a434768c640f6, 0xe16793a753d53c41, 0x0d3576489244922c, - 0x13506238a50eb516, 0x44e4da075a34d5e8, 0x9b84a7383bec0045, 0x3674d61ee7e8705c, - 0xa75b6a55bcf296f7, 0xd66564c9f1de1bb9, 0x95e83737db190399, 0x0770b79ac94ff9c3, - 0x137d8a91cbd41a4e, 0xb9e3e3b0a6e51450, 0x50086ce0297950c5, 0xd6533927d85dc99c, - 0xc57500e690f3d693, 0x2b01b28edb06f16a, 0xc6c35505f54f0469, 0xee6c8a873f429ef6, - 0x7d0608e7c0543a50, 0x38d9a23c9156d124, 0x1205602e6fee8755, 0x43ce9325c1c760b2, - 0x21c155576dc245ad, 0x363e2170e80e4653, 0x786bd19dc5f25811, 0x5044fd592ebe33f4, - 0x8ab1c2ffc39f0bb2, 0xabe586efc3d51956, 0x20b2b427464224d9, 0x0c87d8d88d9545cb, - 0x9cac3fd265d1625d, 0x9891a23725ea25c9, 0x66ffdd52250741eb, 0xb2c64e0cffefb8a1, - 0x05ff99eb1e80c2aa, 0x127cec655c54c936, 0xb2f409127ac3b569, 0x116010d2dbf26f92, - 0xdb0188dc239ce264, 0xab36c117dae86d0d, 0x88e61d55ece6046c, 0x80724e50916fa23e, - 0x31849d054fc4c74e, 0xc670b886cde40afb, 0x945839817418daa4, 0x06094f085a390f10, - 0x32683e18b94cb90f, 0x13ef55fbb6944cbd, 0xca62067197fc843f, 0xc08b5eb082383aad, - 0x4e38f0fbc8038433, 0x421e6f15ab233e2f, 0x86386b85a6ab0e84, 0xc2f40c50e0690bf9, - 0xcbbb0b2d762f75ac, 0x8b09a1dd2bd56014, 0xa8fbdd5233605a58, 0xc118b665a0637e2e, - 0x09701d3c74682da1, 0x163682ee82212dd7, 0x2c232ca5b26da21d, 0x547ab2a1bc8f7cc9, - 0xac198be0512f7d20, 0x7357e9fd307959d0, 0xd567a106b7c0f5bd, 0xdede6eb186aabb16, - 0xb211d1a9434d4d7c, 0xca9b3ac5e5da10a8, 0x3fb66834e62859f2, 0x79d1146dbb9f90ca, - 0x6a33910ea8c55129, 0x6de8d6ec461559ea, 0x97abfa119f429a89, 0x9de3d9abaa2cc297, - 0xeb945077311c0012, 0xfbeae2f7f25ac082, 0x96dcb71bc60e51c2, 0xf118fb542fd24e4e, - 0x78b32042a085ec8f, 0x54d8ed659192580b, 0x8491f7c6f1800046, 0x7bf2be50587e2a09, - 0x2d53b8eb5462f7b1, 0x36854119cf4b622b, 0x22caf19cec8bc08d, 0x94b5d4232a87e751, - 0x8f81507ae939e121, 0x30bbc69ac82212e5, 0xa42c08a88b6b766c, 0xfff9f329b607a353, - 0xea741e3c69cd2624, 0x70c14ee756989c78, 0xb3d582694e0e7611, 0x1b92a851488944ab, - 0xf139973acc591897, 0x64e8f0a050a51e0b, 0x111f4fe1f6109dda, 0xed5b7498d25606ef, - 0x3379a56371b73e60, 0x127ece581b4d8b72, 0x200c404016f88733, 0xcbb09d18513fd83b, - 0x4dd4e41bf7ff149c, 0xaef71150b6bb4d3f, 0x803526f19ea4eb9a, 0x86da6c0f9a5837a0, - 0x0e43c47ca4251efa, 0x528ba07c02caebd2, 0xb86bff5360b7d3e0, 0x23ac5180b3a95349, - 0x4dfae89639b0ab50, 0xc95fe2ba17d790b3, 0xaa48bc568d569677, 0x2ee1cea326184b80, - 0x63c255d06574a857, 0x15735e3ab6f36147, 0xd81949e52cb1755c, 0x1f81179b6a0353d0, - 0x19d5e9f915fa1858, 0x0a58a1a805728add, 0x8f978ae803aeabf1, 0x1248cd87a505da9b, - 0xbafb8459fa942df4, 0x728329d64013de33, 0x8b30f66510d8d73c, 0xf8b8f2c1735e4f01, - 0x49303f6a119220c6, 0x166a747d9d88f916, 0x3ff37cfe1e29e0a4, 0x0e95d685b7a21484, - 0x3960a2fbc61b07d8, 0xc929f9bf454d7df4, 0x3282129d4295cffb, 0xe79d981b672565f0, - 0x7827ab08d65eeaec, 0x78443d7c51db14f8, 0xa4a4dcb4d6839df7, 0xcd2d881119219b8c, - 0x0fea66a899b8161a, 0x6b2589fee55fc23a, 0xb6ad6c0155532284, 0xebdb1454f4af90a5, - 0xae3519840292d1ec, 0x6ed0c81d39fcf9b4, 0xade582463481c4b8, 0xfc44336057ad370c, - 0xc3ff360705b5bbd8, 0x153da7a62e8d7327, 0xb6359dcd0e256812, 0x9224fa49d95d2026, - 0x0094863e268530b9, 0x6efc1da7f5a6bf8c, 0x630d58f1bf94d603, 0x2eb084ad3d8cf835, - 0x07f945a2098705c7, 0xdf98d91ee25b1dbe, 0xf842bcf61f58c063, 0xd4738b4f8d6f583c, - 0x190cd81accdb8cc9, 0xf55feab78d0bc07e, 0xdabde2bab5c1564e, 0xbca9f9f3f35dce81, - 0xdba3be7723e54176, 0x2d950165b245c062, 0x9f6362d8e215cc04, 0x5bff2cb0a2bb93c9, - 0x173545d3287366c9, 0x9031a75685128ee2, 0x3c090269906b7b2f, 0x856e74e3a3ae3c44, - 0x1dcd734b48884f43, 0xf2023762625b45f5, 0x5866b7a6d275b688, 0x7f29417affb6a6c5, - 0x47566d33f6b0f081, 0xab1f9b2707276a2b, 0x8dd1f1bac1160a3d, 0x40ce5c896adaeb26, - 0xf6bdbc8902691e49, 0xdba94a25ada41840, 0x15984af0862225da, 0x40572febd8dd75c1, - 0xbe61829c065dec68, 0x267203788af36c05, 0x47546315b704e9de, 0xf21d43800ac647e3, - 0x9ebe53234afed8d1, 0x2bda6f6656dbf042, 0x0f8bd727c21a79a7, 0x88723293822f2d09, - 0xfb06991bcb45d108, 0xe6f14dc1ba19da20, 0x486ca02e47aa0175, 0x781fb0b433535498, - 0x30e3e14c3968e795, 0x2d149602709817eb, 0x12c98249aa4e87f1, 0x28ad459a6781b897, - 0x72856390164c256e, 0x86635d696eb7c609, 0x80a6b2d16e9aa9a9, 0xe809fdf74284de92, - 0x75cb37b4c63378a4, 0xe1c6aa91d569422a, 0x5b5b80aa06b28c32, 0xcfcbdb4bba662e57, - 0x2390b6068198b4a6, 0xb16193f07607a5c6, 0xaf2160f11896b536, 0x063d67de09590776, - 0xe25e62c3c5d11bae, 0xfadc804cdc39b9ee, 0xee360ba293e2c645, 0x10f715fd445808eb, - 0x4134c9b9e56af9ca, 0x2a3acb81a167f014, 0x64e31c8eb1b4044a, 0x0c2f9be4775c4de5, - 0xe9525bbef12e52f6, 0x42ece0c99a3cb49e, 0xf01ff8d3cc135b4f, 0x017af1916a235a55, - 0xfb6e8f4e7551eb5d, 0xc58beacabb45be6f, 0x77793e4973600f88, 0x89fbf9939bb6c8f3, - 0xdbfe19cacb4267ab, 0xe5ccb6ea49751abc, 0xb89454f51281f80b, 0x5dc8f49b0cce3a94, - 0xc82abc9a6fb024c3, 0x9419e2710009de15, 0x7c2a3546c1c45a28, 0x69abd0296368ae15, - 0x8abf7e07389f9810, 0x314dba595bd7bca6, 0xf3a4bff3bdc066cc, 0x2e7d81a6bca963a1, - 0x915f1335ff0cdcd8, 0xa21f582338707546, 0x0213931226428eab, 0x178b4509d70c9a9c, - 0xe49decc943c93f88, 0xa321327448013ccd, 0x05ecb36cb3bd8d8a, 0xdbee1a9f76bfa274, - 0x09c6ad3fcb4aa292, 0x0990d619071787e4, 0x4574986ddb2f6df7, 0x50f81b52aad7396f, - 0xb7a324f62ab9c972, 0xa55fbfe48b6c91a8, 0x087fc60d4057f909, 0x4830bdd4ced35346, - 0x2cefc2482fd07229, 0xdf36eb0c47598ccf, 0x5c334f62b2c0a6af, 0xffa33dd5f8573feb, - 0x768202a17f2458b9, 0xbbff35584af5190a, 0xebfc0313228dbb41, 0x5961861780176fb3, - 0xb64f401e44df426d, 0xef26313fc040a9c4, 0xcd30b333da08ef6c, 0x4055d40d836058a6, - 0xf5c9482014fa1612, 0x9db0fb44509c0b42, 0x5f67683efe7bec9a, 0x6c00e33642ef399f, - 0x68b64050729d50c2, 0x98658a31ac74c9a6, 0x0fac0957602b15f4, 0x0471d04ef3e8b99a, - 0x42f5968bc25946f8, 0x1bf3ecd0d48bf648, 0x692232d5cdde7f32, 0x7f3accead8de4db8, - 0xc89454a9a00e0ce2, 0x48a3a385e9384751, 0x6a8d8ac180574b04, 0x8daf933b08e7a3d8, - 0x0614b6526a4f9335, 0xa685994539540c98, 0x0cd02e6b27b1bdaa, 0x6a8f3e56d3493671, - 0xa446c0a40ce7fa36, 0xc4d38bb9377106bc, 0x22d9b40eefece45a, 0x24dfea061770f319, - 0x3682d43d9e6acbee, 0x942770aae0ddc133, 0x26fc12a3e06fda22, 0x241ee9a530d815c1, - 0x76948e593c5f673a, 0x6e01659071d81d9c, 0xa27bc40353043e00, 0x13382352d181f82f, - 0xc12930ded68136e5, 0x82617dcc69f59e4d, 0x98047379ddd48eb2, 0xb68d3a5b7e41b441, - 0x83ba4d004b58671a, 0x48a9ff1d39b11592, 0xff2b31e0e41acd76, 0x9812a3bcacbba48f, - 0x7ae252b654c75f93, 0x5b21c92cc2d804d6, 0x1a5690931200c763, 0xc5883ef42e6003ca, - 0xed2e2ec83e4f5e29, 0xaeea91523734edbc, 0x59435bab3af17f37, 0x680ca8ea3d4c7717, - 0xfdc4501d915e7e06, 0xf85e4d35718b829f, 0x79a38d1d00d26f30, 0x6e13412726d655e8, - 0x2d33517560e85824, 0x7d75c23eca415688, 0x1c6fa64779c0f0c2, 0x241583ea38d8e6ae, - 0xe3506a3b2cd29483, 0x2735b0ae612c0c98, 0xd1f92f0210154fea, 0x6b6879478362463a, - 0xe7363375ce994758, 0x58cf74270bd60333, 0x3bfeb9312c30e5be, 0xf7007c76b5e2166e, - 0x2cd448787e559029, 0x947e87ebca861a11, 0xb733ed110fd885c3, 0xf995b9bb41e7a875, - 0x8174e88e3753fbf4, 0xa0aa7b452c5f7728, 0xee58cbdc163c5d06, 0x76041444fee7668e, - 0x4590bb69a1d726c8, 0x3eabacb57bfab13f, 0x7dbbf2caedfde20e, 0xe0abf86b8d4e2992, - 0xf4a5ad330b852898, 0x0efb9dd678a25583, 0x6b3452fd8fcb0235, 0x927cc48b883c8223, - 0xccffc692d2c931d2, 0x60b4bb1ff16dc08a, 0xdf90b0af11485aa7, 0xac8534e8f65a5e16, - 0xaae849df3309931b, 0xdd741e2909d1f3a7, 0x61e5ad18543c4570, 0x17aaea62b07b2fbe, - 0x5a1c652afcb897e2, 0x5d8ce4e0fb478381, 0x306f2724cb358cde, 0xb9618e69b272df95, - 0xc50f972362c0b896, 0xbe24c6ab4428c6c3, 0xaefa08b4981df8b0, 0x7aa7948f968ed4a3, - 0x7360ee23ccdb9bbc, 0xc3c46d28eda07e9a, 0x4df27e8932ce905e, 0xbebb4106419c4753, - 0x3e655f66cdbe390a, 0x8b8cf7ce28e69722, 0xf5dba7cd58f45bd1, 0xcc93695a4e3e6770, - 0x13d9bd5b97d04dbd, 0x4af9fc4b5859b15c, 0x8d76802c17a65522, 0x1fc5a23adfbe934d, - 0xa90840c37ba00cf0, 0x94efd81dfb222d52, 0x2b25b83c4251293f, 0xc7c0f30962aafe4a, - 0x577c2bd98de8193c, 0xeab7376deb5b30af, 0xf5f475c788780d03, 0xab255d58f5e540ec, - 0xb0b74b76b090ecf4, 0x0da626e359051e2a, 0x20ad1eb96e558789, 0x69c7e6ab03031a96, - 0x1185df19dd325fda, 0x75b8672156c80e8e, 0xa7d2bff5e1d61a7f, 0x594ac8c5dd0b8740, - 0xf2b1d4ee0474f6b3, 0xe5ccae5f99c9d3f5, 0xc2414fc5edb03443, 0xeb3cae2727007f97, - 0x09659277335914ed, 0xf2b436c00758eb47, 0x1ba939696f569cf5, 0x9ab7cc2c54f14cad, - 0xa26d3db8f552c0c2, 0x0edf2d2d65c93cd1, 0x189440474eaa6554, 0x20c09ec98b6b0361, - 0x58641ac31dd71959, 0x0ef2dfb2dfff77e6, 0x4ff3bd9980f2250e, 0xfd567957cc291983, - 0x95d70c47fda7a12b, 0x0cf016143c39f108, 0xf2b24d81e08a5891, 0x36856a3a9128658a, - 0xfbd34ea25dd9ccbe, 0x089f4019546ab76c, 0x1cdd49dd02a94e3b, 0x0035ee58b7a0c22e, - 0xa51efae6e7d7f46c, 0x09bfeb3e647708ef, 0xf69b13ad1c4db7ce, 0x2159b438ecab21d1, - 0x6633c665ee69894a, 0xbf5ef6a780924aac, 0x96ec0ed946877a18, 0x8e3761db8c422725, - 0x48c655f96542b2a5, 0x8892187473d2c244, 0x75a6ee7a77450d5b, 0x5526059c2e25af00, - 0x7494729b7c6ef1cd, 0x896fa4aabdc51516, 0x199bb9c27ad4fa24, 0xa60207c24311fdc6, - 0x198e52bdd066b6aa, 0x613e8de7ecabd481, 0xef288b5957f1b392, 0xf2e0cfaa0973f834, - 0x0232bde74c0b5c9a, 0xa0d103e117a48d37, 0xb7ed9cd6736bd5ab, 0x5c90f7b28746db46, - 0xc562150857485d1c, 0xe1f62f202bf8f8c3, 0x512a009983bd196e, 0x4f19f7d92fc95681, - 0x0ea9a9c5fd9bb708, 0x11af5e653f35510d, 0x37a1106d875cae6b, 0x7076163612b12ac5, - 0x1ee1404ee9b490b9, 0xa9046dc945106ec0, 0xdd713f83b7859c48, 0xffb9fefb281beccf, - 0x2b566f1497a16fc0, 0xc7223e8a18b4d7fc, 0xec4449a14858a2a0, 0xd29ecf117899038a, - 0x324b3ff394ec1485, 0x7f2906de505f280b, 0xeef288159d2264ec, 0x85dfe9b3f9dad94e, - 0xe621f1273379d408, 0xb8dce25d03787e7d, 0x9d374dee62bfd534, 0xacdaa8f73d1d1557, - 0xd2ba808b409dd3da, 0xedb90c73fad31c76, 0x2efe6e05d258ed58, 0xb655e6e2ce0934dc, - 0x248b7a999d7e8a2b, 0xb9981aa22064e91c, 0x9472355b5e0a5ca4, 0x5a29215cf279182d, - 0x24d8a12d48f6be15, 0xa07a2b5bd877983c, 0x839e06c59cec3d0f, 0x6282156f61a91fa2, - 0x9beb7a65ba82d3ee, 0x77630b01f0383216, 0xb326910c1d0a4022, 0xa4f4cd36ab1d3316, - 0xf4f5d62fda6f2d94, 0xced8e9bf22296312, 0xaa8df61ea31aaf62, 0x2e1d4567d585569a, - 0x557b28a2d6527810, 0x42e96860b7d12b9e, 0x0315898de306506d, 0xec2506492dbe3307, - 0xb01a66ac8046a09f, 0x9fa552e01b6da177, 0x0859d132693b1a39, 0x4ee835c331eecb61, - 0x2aa97cd7d615c3fb, 0xe93d661decf44479, 0x3d7dca951befb89d, 0xe962aa4515ffb160, - 0x0da9d7026c2da22a, 0x1cd7a55fd12ef9d8, 0x59b614e552dd4e58, 0x13746e7ae220b691, - 0x4fc3b4d7914c690d, 0x8ce7f18c3b521f59, 0x5efc0e445d17f956, 0x05f78722453854d3, - 0x023d80e01844f5f2, 0x440297cd76cfbff1, 0x5f6e31f1a2c8ef44, 0x8e55ef16f2ae20e0, - 0x7641a1f3c2c1739d, 0x5c1647fe253a769f, 0xd67d51e03fe22596, 0x8080116ea94bca94, - 0x7f0d9d2b28904537, 0x76a546bb8f44d549, 0x77cf867d6ff10ff5, 0x595c629e3dac6e80, - 0x72897264ab1795e5, 0x6596e3f4ea36f4e9, 0xb23d471b5c60b2d9, 0xf72c2f6f987a64b3, - 0x313757eeac051c5a, 0x01ebdfd01ccef1e0, 0xa3013edb4eed4b3f, 0xc0c53405c261f40c, - 0x9de0ad40ead43585, 0x2a01e3ee9840ad8b, 0x222316a54e72862c, 0x008ab59a7939ba83, - 0xafdac7626e1214a4, 0xae1f97acf03f4e5a, 0x643851f70105d89a, 0x20c97b38c9b7edde, - 0x84b610b97dbf9b7d, 0xdd2e6e907145f083, 0x90772e49e590687c, 0x21fa461c0d55bb29, - 0xa89ba9fec9c119e9, 0x2200577c6d2b0028, 0x6d2e0d621be52090, 0x719b87af6209ad28, - 0xbb68ebba5c94b6f7, 0xc50018abcb2c2e95, 0x6bff705f0cb55340, 0x021e0e6efede77a5, - 0x8458806ec8ad213a, 0x744d441220bcc135, 0x4dcb18b732218a97, 0x810964a35a1a7054, - 0xa60c20e070ac0c7e, 0xfe0b5d813a87bc03, 0xb317d5ada1293804, 0xd247f1bb7a6bd79e, - 0x35d4d7627d2845b9, 0x18194abc3f8e889f, 0x8a4fc7259a38ea72, 0xab9c0963d0f279c6, - 0x31fd8954d3abc2bd, 0x7bdea8d86f45ee46, 0x790b2dd070d476d3, 0x7e84733265449ae4, - 0xda279b5c4107bda8, 0x9ef646f55daec08a, 0x08d6238182ab3a3d, 0xd115ea53861dd161, - 0x46fd37826ed49f5b, 0x8bf13396330a2d58, 0xdb242489c6c7baf2, 0x5e425c41cb56bd7f, - 0xed3709b31b1b5137, 0x1eaa0a137cd3ff9c, 0xfa70c183b069f825, 0xa6ebb73039531411, - 0x0d635666ebadc09b, 0x32fb28967f49b3f9, 0x2f3a89a0bb0c7bef, 0x39a201e9252b9f35, - 0xe299b45fa22a6696, 0x5759442df4532210, 0xd2f334bef314a896, 0x14bfbf5ec56a0356, - 0xd03a5b56a21f684e, 0x2535e76d44aa3359, 0x9f10b274ba7e68fd, 0x2870f846956d3f03, - 0xb98b13b9e69dd7d3, 0x8f2b658e456d8df9, 0x8da93e9e6aa01979, 0xb4e036323883ee18, - 0x434d0198206a2f77, 0x7d5e8bc029d0ed33, 0xaba8d55c204720d3, 0xa43ba25296139e33, - 0x45b49d723ab649be, 0x78827d0f2d26aa73, 0x3c8d7c52e5db845a, 0xc69a10eb65d5088e, - 0x44d8e672769b7154, 0xa09d79fec39bc791, 0xb6e422577f97a75b, 0xae39daf45d318882, - 0x1f333fb03546763c, 0x8991ff3d1a1702cf, 0x6fd2aad213e7870f, 0xd52164abe50a8b58, - 0x41cbdda07406ee28, 0x5765efe56cfb2406, 0xfd173b286426200e, 0xca95b6946febe63f, - 0xd92cc847dd8e1c91, 0x290d3e39b97ba2a1, 0xf259fdc019e36b09, 0xee24204c090124b0, - 0xeddc179e981e7f43, 0xd04aece1caf65c0a, 0xa6c52d7d54f66f36, 0x14edb8b3c3ca38e1, - 0x84adc30c6d57261b, 0x3e078e1bbbd18dc3, 0x05c807755e414ff6, 0x69ee344b7614302c, - 0xd5ddaf802c8616a7, 0xf24562388184a930, 0xdc6774c6bded223d, 0x36c9dd9edb6d374c, - 0x8a69fc135d2c3736, 0xe06fbef0c8c1de40, 0x6ad36ba8e15969b7, 0xedbfea7ff81f734c, - 0x2c04d6c97a52ec0f, 0x77ce8f388c90ba2c, 0xc5b9037802ae6bc2, 0xf3be49dd89902490, - 0xa5562fd8acdf7fba, 0x6e786496659c3e3a, 0xaa73fbc10b15ca9b, 0xab3808f20bb96618, - 0xd5af76a2696a45b3, 0xa9a9ba59f5c8d4a1, 0xba4f37cad3000e47, 0xf452fac57a712dbc, - 0x6ec63c64ddd05c5a, 0x5ed1ccc4120930a9, 0xc525e9242b7b6b11, 0x05e46865cc9b178d, - 0x6cb6167e4cf54c5a, 0x767ffaff8420b992, 0x4db251da2d63b1fd, 0x5d85ce4e4747b9c5, - 0xb4c1a526b754e697, 0x20caaf3a153c90d9, 0xa8ce1f3bfadfb338, 0x6975a5cb8c7f2fc3, - 0x64ced6e78a4f4a07, 0x86f58968e9bd7c1b, 0xe1be94ea65e267c3, 0x9cce66441c3373db, - 0xf9d5fc615fbb4590, 0xfece86266e56c223, 0xba6e9cfe5f4b53b2, 0x542820d4a60d4cbd, - 0x353089047bfd66a6, 0x16acfef8a66c5fa0, 0x54234b7996e14f59, 0x178b4eb1cae70d15, - 0x343eb676d7d77221, 0xc2d7fa2fe58a8b70, 0x1e934adcc8a15216, 0x48e5ba7391de0ea4, - 0x71e5e77974415212, 0x8e916fb9f2bd7d54, 0xf9b462aa2de3037e, 0x237c437ddcf6acbd, - 0x79962fc4c49cf427, 0x5fd207fcd9a45592, 0xf5494f2536311c9b, 0xb4caf5b520a95cdf, - 0x7963708cf2d50dc3, 0x269c2c9cda4c3442, 0x6dc0a13cd28478e9, 0xb9a999db6710d907, - 0x965d39ddd6a6862f, 0x990cac6e42f9bcd7, 0xc2b604dc4aa811c7, 0x4dadd75970bcc69d, - 0xb0375275889d4aa2, 0x6222e7797d0a7848, 0x8f37688cc15be1a7, 0xd5441064f503da0f, - 0x12a97b9b4d2e0b9f, 0xcfeeaf5959bbe111, 0xc9fee47f3b82f3f2, 0x5a5232b90f558f65, - 0x2bcbe0a8534bfca9, 0x9e9ad7503df2212e, 0x5db3a8c53c2622cc, 0x1b5c9fdf31bd1a1c, - 0xa42d32d7a37d8406, 0x4c3f5c4a1cc699ba, 0x7a7ad82e4998ab6c, 0x58d345aba5163341, - 0x95960ccccbca8c07, 0x3b5de6a1a8631f9a, 0xef51f440afbcbbcf, 0xf2ed17b0e516fd90, - 0xef47edbd84a346a7, 0x3239fb7d8727d26a, 0xd2cbb17be874b979, 0x398fa455fce8f9a9, - 0x2debfb3897614c22, 0x523133997e7a3508, 0x0e56bbab921e2630, 0xe332c5f953da83a5, - 0x90899d3af1c96cba, 0x3b3c2187b1487928, 0xd06e9ff717df6d0b, 0xe38ba14fc6573ef8, - 0xe40e3e521b700b13, 0xfcf6b32e574402d3, 0xa9d721a580d91ba3, 0x2e8dc55132619fab, - 0x3690b8f5a263aa03, 0xfe6993e227543a5c, 0x3b33dd70f464e221, 0xb8fed1b3cbcb275f, - 0xe193a91a8ed73a7c, 0x01477f905eb3c8d2, 0xaaebd3a73936ede5, 0xd002195bb8eab2e9, - 0x458b56b3bf8499b2, 0xee8442fce3064224, 0x294014c6cf1921e2, 0xe5a74db989152da0, - 0x4449a0d75367ece6, 0x7adb5e646178e7ab, 0x29226d7478671086, 0x420b14b1e0facb4a, - 0xba6ec1fe108cfe02, 0x2c8a3356b997a16c, 0x350a3d0cdaf7f94a, 0xc9460fd5b75d22e1, - 0x515834ba4aaaaf6d, 0x0445886ec969cd2d, 0x7bcb1ad709e49082, 0x3dddec714da63eef, - 0xcd73e4b630eb18ad, 0x0692ffd2899e3912, 0xc034c7d5730f633a, 0xf985ad589d81ffbf, - 0xd5fc86c14fd27e82, 0xd18ef06f0105a277, 0x04c63c5931ed15fc, 0xa6804830cd9f2331, - 0x4e839a923de2bf15, 0x080fa3bac1dc50ed, 0x4b5e5abd3b0693f0, 0x5e0e00b70da4e695, - 0x00c5a8ea9312f9fa, 0xcda9881c1e8a6674, 0x397536e380ab329e, 0x1ed53f8762969718, - 0x9f442101f508916f, 0x77304d1a0f514347, 0x22886e73d8e3b0d2, 0xf8c1d26a63a4f264, - 0xfe37193ade6c9f52, 0x47f0c10fe1ee9e62, 0x94cc0bd6413f5940, 0x001e2c362dbecc5f, - 0xad91f11fa67f55d3, 0x05ebe3bbc72b1d1a, 0x0d035d69733baf72, 0x1df08ddd7e93829b, - 0x48bb387a1529db42, 0xa3fa4bfbdaeda1eb, 0x881158cfe779f44a, 0xdcb53eb5b07c0513, - 0x63d0aca20746e1aa, 0x083d8d22bc547380, 0xf0ab2ad4e5fd9181, 0x571adb13e629a820, - 0x489f5d32e996f0eb, 0x423071d932bf8fb9, 0xec98205eccad710d, 0xfa02d3bc59186241, - 0x8d45f25ce5f1ef62, 0x9ea3d1fc4963524a, 0xe07560a0fcd601cc, 0x9a99ba576a2a6135, - 0x27ce68c01b275294, 0xc291163d8e5cb923, 0x15517c943cf06d8a, 0xf758b0bfc7db6973, - 0x6fa389a0609cb2e3, 0x5d467e3177544945, 0xba0b036d40002949, 0xff464bd941433e41, - 0xb7ebb887f127f6b5, 0x9b6f9a4fc4e670e2, 0x86a7547d4cabe301, 0x1fa00e960145b926, - 0xa1e65d78c56faf2d, 0xddfd0e51260915d7, 0xb2e365f0095af899, 0x87657f8d29a78178, - 0xd7a64b017a17e8d4, 0x804feb49f35e33b4, 0x7af40e23399d65d1, 0x3b3afc51f4fcfb9c, - 0x8f64ee6f157bfa6f, 0x6af3b85b5db3fe81, 0x39fcb1fe7def4e6a, 0xdf2e084a2110ff8a, - 0x1e2ef6971f16e0e5, 0x0d24a5c000094a14, 0xb9e2d4847f937ea7, 0x93196a75e502a5d5, - 0xabdb3f094b23a038, 0x5347cb2d3d7d3dac, 0x2fb4d960a261523d, 0xda116d25859c76d0, - 0x7eb64241df444db4, 0x352b982fc4f91e56, 0x1b8574d08f582a81, 0xd85547c95de83669, - 0xfc2b976a8e971611, 0x1b6f101599067293, 0x70a87c99a2e40254, 0x5e3ca88b20e6491c, - 0xad9bd115056fc1ac, 0x0d4d6d12fc7f30ce, 0x56f7636abd73d903, 0xeeb85919ce21fc5a, - 0x40ad65eba458f594, 0x9e02e1513085c280, 0xad4907e90ef70e47, 0x6fe78e839d396975, - 0x6df2cf92869528d1, 0xa2a6ae23bf37526e, 0x0971102407dfef53, 0x1f22d89ffae23bd7, - 0x90e278c8b777d35a, 0x92a326a7ec229a9a, 0xcae88775c31eeb98, 0xbc17096954d3381a, - 0x7104c74b9b70c9c7, 0xebae7d1bdb2a294b, 0xd5ab0b4044b7fc3e, 0x37afc964464381be, - 0xaaf57874040bf849, 0x83526e97d73af243, 0xc2ecf434ba31a3e1, 0x392de1defd7a7a37, - 0xe805974cc87b393f, 0xebb86ab291bde06a, 0xe02e490d7869aff1, 0x146715cb379858f0, - 0x6405cd9d81221023, 0x59ace0405c1fd269, 0x67e024da544a4d4a, 0x3f493b53be229d45, - 0x32c9b0675929bdcc, 0x0f2f9b6e2b366815, 0x37e4fedf112c7ce6, 0x00099c88d92eb67f, - 0x2f6ab5fd34501946, 0xdf7b1d2a8b6ff8c6, 0xeb7af02dd2fdeb72, 0xe560d7c57385bd39, - 0x90213486a5f4e792, 0xa191d1c295c89a0e, 0xdfef91829df74a84, 0x883d2d9feff83efc, - 0x7140b6afebbeaaed, 0x9fb857898d9fc352, 0x82415f5b7b99437e, 0x807b6d1a13b0c261, - 0x01bce4a93c1a8f05, 0x7418c0f662ae29cf, 0x5b1186423e23ae67, 0xad057894d5aba800, - 0xca5f8a94f7181736, 0xa0b7892d0af40287, 0x7f7e8f9cc43bc05a, 0x234082c603f5ea36, - 0x58f1ab4049e1c04a, 0xfa5887b9ad27f7bc, 0xae8afb83072054dd, 0x0e8c560c1cc3464d, - 0x26d98f4113e90970, 0xe5328645ac5a1526, 0xed80b99145ea1378, 0x439baed52198b536, - 0x8830e93aac0319f7, 0x47aaf5a2763b74c1, 0x77ab909ce9b868da, 0x2d0a42cc9fa1b27b, - 0xf1d4c25697ab06fe, 0x9f6f2e0a3add1ee0, 0x2e1ba1ae77ece607, 0xf14fc2742cedfddb, - 0x4b6723bb9502cf02, 0x37173334ee97501c, 0xf059b30105754744, 0x0aee0a9eaceb15a4, - 0x1857f0588bc36324, 0x0d242677361d5d4c, 0x8584ec5264d19375, 0x4439b6a3f09dce90, - 0x5c5139426334f49e, 0x87370c15b7592af2, 0xfa4ed67401389d2a, 0x10df13e997a37a10, - 0x8592eccecf7a9de4, 0x6897f8a94cf77113, 0xe2db77c6ae0731bc, 0xbe78750195fcaadf, - 0x843f78a6af3c8859, 0x3369c71114eb7762, 0x3f5683f3270e44f4, 0xdbd6a1a8a2a76619, - 0xb52ccb4b8c2fb4e8, 0xfe2f60c18b9bb11a, 0x5701a268aabb951c, 0x16df69f23d900017, - 0x181aec189bb0ef6f, 0x46ade44faea07f86, 0xe30d7a4026b75b3b, 0xcba3fe87c310bbb6, - 0x5e76276fb66609ca, 0x6efd398f13bcd754, 0xc1b9ad5d6905f589, 0xba160ae57ee181fb, - 0xa14140106b763bf8, 0x194ea43677f34e30, 0xc667c5114eec40ce, 0x49789068a92d1446, - 0x84d0c53a0f1363df, 0x3feb5ff45623f3dd, 0x56e81185c273c3f4, 0x69b6e9a3ed5c4d6d, - 0x16e655338066c3ea, 0xd96c3912f94fd4e3, 0x936509c46dc217cd, 0xb6828dc765f15219, - 0x0f18bd16c04a259a, 0xef2f55fb080016f5, 0x7bb463d37a955682, 0xbcf256277cad00aa, - 0x716b49fe3c7447ad, 0x419c16ecb35136c3, 0x5415fbdb428e1a07, 0x47d134c6100b8d27, - 0x5be942918fc274cd, 0xc56a9dd95cfb1f85, 0x3adfe008ea36b851, 0x46bfde4354870c20, - 0x155b8df6c4b8a2a1, 0x46bbca4cbe7ab4a2, 0x392ae1f3b74805b4, 0xa786e69bba907b03, - 0x3e6a07a8dd58d574, 0x0b52af34c7a11851, 0x8aa5e6a774aab7f1, 0xe7c73a7101b60a4e, - 0x54c3e9bf07e3eaca, 0x12acee59db65c0d2, 0x40e2b6c5891a34de, 0x62fb0b67e499dec7, - 0x37827e3b0da6e0a9, 0x9cf8178c0c3b0de3, 0x37a20ef39d785a86, 0x1ec9f673fac6d41e, - 0x9c8d57b4ad98a48c, 0x11be0cc0cf83abaa, 0xf23b110987a85d84, 0xb9a8f66123d74b47, - 0xd5701ea405389567, 0xcc6876764a52fda3, 0xd246b8d958ea0547, 0x261f87580478a9f8, - 0x81a6c68775ff4e83, 0xfb740135b59ee494, 0x5788f8d4aff26410, 0xa78e99330c0f39a5, - 0x1b8a8eacd0524a68, 0xe61a6d89e3d0bf28, 0x052fd9439ca22709, 0xce8968884ef74aa3, - 0x867f054c610327f2, 0x4f194ef2f886d6a1, 0x1c138a9f339e7c55, 0xda7111e73c988965, - 0xda9eabaacad26fc9, 0x85b12cd7082e95d6, 0x870b154f5dbfa19d, 0x5e0584aa87fcc667, - 0x4630a466e2c2f646, 0x0a16bcc533f31bf7, 0xcab259648142ab32, 0x43e563f3f19dcdb2, - 0xa7491bed48e1a102, 0xd821ce02c0b736cb, 0xae1a097b0577e13a, 0xb7c598754ffe976b, - 0xf87feb4f3e6a9e17, 0x7a121514bb5e58ae, 0x6fdbe1a4e8a5ea2b, 0xa635c475909a83b7, - 0x58b84497a777f88e, 0x882bbfac3a103d9c, 0x11d3a1037a19ae74, 0xa8f50f5db2f9793b, - 0x6a9374b8499dbde8, 0x2f56917d5a3efbc5, 0x15aaa4936d6ccef4, 0xf267cda6eec7de1f, - 0x2e26423634e430ee, 0x4073363ffe09f1b0, 0x63a51c2a59c36938, 0xaf9a4135a748e4b6, - 0x7668f07677a55764, 0x919765086a157e10, 0x0fa426c1fdf9ed17, 0x8adc92f83385f03c, - 0x2e8757c67bd90b2e, 0xe9dfbcea514936b6, 0x044257b30abd3484, 0xcafb7610ce69f914, - 0x8f0b505705fc226a, 0x3db43bae46f2259a, 0xe135386227c8aeb6, 0xf94407a87071abc6, - 0x4f44f59283b2c631, 0x3d1c43aee4a54dcf, 0xd95a4ce3ddb58ee3, 0xebb7b58598a496dc, - 0x134f392a6647c3a2, 0xb9b31a231abeb2ec, 0xd00dc23b8389d61e, 0x62fa45c1d22af99a, - 0xffca039284fa66e0, 0x62aaf428eb30a910, 0xe2bfcd617a80267c, 0x8bbe5a78c96bd308, - 0x57ca8f41c25ca19e, 0x75f77472e5bae7f9, 0x5b86fecf23357438, 0xe3fdd099fc5a02e8, - 0x1359bc21501aecca, 0xfdb398dbcdf8e7d4, 0xe6546fc40a1ff1c5, 0xb26bad98c677e07a, - 0x83d308cdbb85b87c, 0xfa4f9145bdcc3d0b, 0xe468dfbe2d4ba8ad, 0xe653641ae1c545f3, - 0xe88e1bb2b60cb491, 0x3491384a2df23957, 0xbf8832f794d9316d, 0x694ff13ea2c65a1b, - 0xa3a5655986764018, 0x1cef0cab66214f40, 0x98152f736bf9eacd, 0x1107c6dd2bc5451f, - 0x14697bac801c14e1, 0x65d7d9e446a9c064, 0x51000a346a500c68, 0x1a585cc0934137b6, - 0x2cf2441f8a1505c8, 0x1b12a69ca1487b98, 0x49d770d40bb1f6cb, 0xe2638905db0d2c6e, - 0xeff329b752ec53af, 0x563cec571428aa22, 0xa578a53e8ff5a2f6, 0xf8463f7479a3f4c3, - 0x376bddfc21915b24, 0x978ebbc4fa34f93d, 0x0dbdb92e91939e97, 0xe80c9a80a50540d1, - 0x4b9603c3cd742132, 0x8f521552b319dcdc, 0x531a619dfae39f54, 0xf445095415959a45, - 0x5ce75d4315dfdb9d, 0x52b4426c9876f0e4, 0xdfc0b9790860bf94, 0x90f6862ffa5f4476, - 0xa3a5990c442d3b0a, 0x68348653cbb8815c, 0x7e5a6f1db30c07cd, 0x0a139f2d83e7b73f, - 0x170d349b1f32455d, 0x13b400ad88593a3b, 0x547ebd2bebda62a5, 0xa93088d09fdad479, - 0x467099acdc3f14ae, 0xf756f149a056c976, 0x07b6cb3f6840923a, 0xd34b187ba04003db, - 0xcf94df4d0730b5f1, 0x71020802bf84a0d5, 0x48a2a06e0cf33230, 0x0c28769b785c4831, - 0x124e7a51a12ada47, 0xde5f3e4bffb5af8c, 0xb6e69b87ffde7173, 0x6294cff861f6e697, - 0xebb4c003b89c6963, 0x65603abd917c1f5a, 0x52c96dddf46d9d44, 0x7a64a2d1496746ca, - 0xc9dafe3bf3b56b58, 0x9f62fdd2006cc2df, 0x893d571f975b3017, 0xfe2a41bfc0cc119f, - 0x3ab341941b5c7f3d, 0xeff1e81829d718e8, 0x6a1b9394171bc4e4, 0xcaee7d319ab6101e, - 0x50431a13d323b103, 0xc249dba239e82650, 0xd9d58a2964fc0428, 0x7e2bd2c45eb1db7e, - 0x3a4feb66a7849267, 0xbd367f590532fe30, 0x14fa726b7bc21780, 0xf6afe897a3e48b2e, - 0x7cee020297670959, 0x15be94fed44d6cc5, 0xc93ba7bd1e4d9989, 0x190c40915487fe04, - 0xf87349a71968613f, 0xf3be0d04519c7089, 0x7112722eb374f90a, 0x9354b39a6a9eff54, - 0xe4e715db38af8d87, 0x91fc81aa7b1e7e6c, 0xd6be530dfc32c144, 0x95b38bc42a98cc55, - 0x6864340363796f53, 0x1e4ab2e5e2586154, 0x7238c14778427b8d, 0x127baf8a81802bc5, - 0x71dc19261aa94a9b, 0x7eddc56338fdaa65, 0xff14c4c90f1119dd, 0x109c87ae5843fe67, - 0xb1b4ae93b4661a23, 0xf1fb7819a0756819, 0xeb93220394529a64, 0x594295b0063f12ff, - 0x171cec42566f3fc0, 0x73f8c43392bc82e6, 0x403d3926ed42a199, 0x971af5cd1475410f, - 0x1f0b9af15c6c64b5, 0x015982d1e455c8c2, 0xe205c3b93b4c1e04, 0x61aca977984ed798, - 0xcc18424aa9f8a486, 0x8c727eaecd3d18b2, 0xaa58d7d42eefa20d, 0xeca1f676454746e0, - 0x138a9393e57eb68e, 0xd45df69342378b64, 0x12db66de05caa0f1, 0xb808bc9e5685694e, - 0x11b7fd2f923bc101, 0x8cdc1897c7653012, 0x867dc9e3701c1094, 0x244483d4595a12c7, - 0x0a8fc852bd2323ee, 0x600e716e58381826, 0xfdd7f6b3902cd24b, 0xbf9ca3ec9bb96c04, - 0xfed1603ffec2c511, 0x63cb4ff3e63cf0e0, 0xebd3738648695187, 0x2043f75469a6905c, - 0x0b6e81dd2d8217d5, 0x2d48da5dec78b14c, 0x9ce6fa6545d4d807, 0x39158c2a69a9ff27, - 0xab5f7d00d37caaea, 0xb195ca1a18b476a0, 0x215bddb0b84eee7c, 0x75d3ef8363a0d388, - 0x2088ab196cd55f04, 0x37fde1657d587f18, 0xe78737679807fb8c, 0xa665b99486e6fdef, - 0x871b8dda0c2b7146, 0x4ea57fa53a9de660, 0x8b7becb250b711f4, 0xc2a23f1e79eb1212, - 0xea81404d16c921fa, 0x49636a0d4789c880, 0xb238eb1a3983e8af, 0x62a640c5fe793255, - 0xb5cf281434ef3d59, 0xae2101903ac510fd, 0xb7c21600defeaca7, 0xf0c9aac47cd67d9d, - 0x3e11099413f2bbfc, 0xabb20112d8701c7b, 0xadefee5b8d593c77, 0x1945df112e96b4e1, - 0xbb0c6aca887223ab, 0x4f7fc757e43cf788, 0x8f6cac56b106c181, 0x9ea047aac5c53aae, - 0x7863ba25bc0002f1, 0x0c5e6d17ace75606, 0xcdeebaf9104b4067, 0xd7dac4ffdf8c2905, - 0xd5c41d2f3ca9613a, 0x6bf9d113c08a433f, 0xd8c4c80184056c1d, 0x6c78ed3270b9b6d8, - 0xf26042e3b8fd823a, 0xc83a5fb453520ef6, 0x3cafcffecc15e491, 0xb524bb96c2947401, - 0x20983e5ee74f6a39, 0xad82b0eefa94e02b, 0xf1a29e2f6b92a996, 0xf6777a3be2517ab5, - 0xf8e4991f03e2b986, 0x3dc5714139c599ec, 0xbfa8eb24918e24e7, 0x903e1859346a4c96, - 0xebb8ee69bf1bff9f, 0xacb63ff98f273443, 0xe655024ced97b0d8, 0xaf7c8e9c3f80c20d, - 0x17d16d12a8cc94ae, 0x2d7241e8a63f9dbb, 0x3bad111dbcef470b, 0xddb43bac81e231a9, - 0xca63be00df85463a, 0x13c443f3f2ebe302, 0x83b1ae7f852e447e, 0x2565241e0229bd4a, - 0x2a1998b95ef75215, 0xf0b72abf5afcd5f8, 0xed0d5ff3c83ec4d0, 0x88887505fae32502, - 0x281a8aae1a865cc5, 0x4d63d8bab4e5f8a5, 0x03d7a249297593f8, 0xfa0a54931d901885, - 0x03f3fb7e306903ea, 0x194e2ce31af7f7f6, 0x5bbaafad9b17c94e, 0x712dbf9dab181251, - 0xab65ee1fd0784fd1, 0x9a39336c8b6e210d, 0xb2f20378ffeffcb5, 0xe7f0e60fb2caa498, - 0x1e68de0090dd83d6, 0x6cc1a8e86c4853d9, 0x53fb424a8e77b2c1, 0xda4d3149515adb1d, - 0x35ebeacb5b531b56, 0x2dde9ca9ab6f87e9, 0x184dd222857feb94, 0x2c55f28df31c3a3b, - 0xb5d6e441c28db030, 0xc5c70433507739e4, 0xa3d84a5df31e3c34, 0xba6574fda74db917, - 0x86ee3300ada0b382, 0xd52440a78d087e66, 0xc940b4ab46d1adaf, 0x06959f9f48a7cb46, - 0x9c61ce0c4a169ba5, 0x58bf42533151c1d4, 0xf30c5aa9c4eb272f, 0x6abd22760402947e, - 0x7c807415c0175a5e, 0x88272de46d8d7728, 0xc250b67f242c77da, 0xca4d065e326c1ae4, - 0xf0995fc38ea7bcc5, 0xe7d7390d1dd21c35, 0x4cbc1eec0a144561, 0x77bb6a58776aa0b8, - 0x50d51620d1aafed0, 0xa8f75699dac2b6c7, 0x1ce624835a5de460, 0x41101b81b8e97163, - 0x5b6584f07c8b2866, 0x776b86e3e698dc02, 0x4a17a3a874cb3160, 0xe6048956f3a7c6cd, - 0xbd3fc7b8cd7307e0, 0x4afb7d1dafcaf706, 0xc10cce67bb6bbdf5, 0x3f89edb5ed7fb283, - 0x05f2cc676569bfb8, 0x1076e29528e11d90, 0xc65db64f34a750b6, 0xf34ffeb63c1e0d7f, - 0x2e5f461e52bd7dc4, 0xbbb0799e8293da5a, 0x42a0dce540509869, 0x191d477e9e280ee4, - 0xdc21b16644658471, 0xc52b43795ed61add, 0xaaaaca7f6c6c1eee, 0x480d72d3111dc6ff, - 0x55bbbc36698d2aa4, 0x1f04df2e9708873f, 0x9426cc3670a1b15e, 0x524a21a76af5056a, - 0x150ae6fd07f93e40, 0x69b28ba3d7d14982, 0xb49172191743de27, 0x756bbadbe2c33e55, - 0xf30e5591ad44518a, 0x04d1bc3ffc1d7e46, 0xd8ab2e8efdf0bdc0, 0xbff538d1e0ef7851, - 0x83898ea51c964108, 0xdf1e2668fac26182, 0xa76c253c399f7402, 0x5389735e5cbd4bfd, - 0x30a8a5274e40bd27, 0x8247144331a44303, 0x22bb3c9341735c64, 0x3dc19739e6a1819d, - 0xed4cd0c59d08a812, 0x646d0f00fb547502, 0x2d041c0229c141ba, 0x4afc31929b1ea2ad, - 0x90195a5becda81b2, 0x2cc50b1cac7102f3, 0xa141965dcbf3960a, 0x30347121e789c1a9, - 0x991eb3964f651fd1, 0x026a009275b8d959, 0xfe715d97852014c6, 0x3c353c8272a7e04f, - 0xe80542cd0ce8a08e, 0xa4e049a8bbe081e7, 0xecc3dc4dbede7a4a, 0xfdb3b51cae71c5a6, - 0x0dfbae3b2393c2a5, 0xb257270aad21f33a, 0xfc8c55c002dffd63, 0xa5634df41735fc85, - 0x06ee058d20495933, 0xd557443dd34bf962, 0xddb172bfeb877e51, 0x8a2d836c9252857f, - 0xd98f4f19f6ff4d29, 0xb2ffd315cd977958, 0x4392581d1ddcc256, 0x981953a1152168a0, - 0x11676218b22c47bb, 0x437e936355c99c69, 0xce3e67a67f975d0f, 0xd94ac478182baf6d, - 0x1e72a8712c0e1094, 0x526ffc973d59728a, 0xe8de4ff3d76782cd, 0xe051fe503e9d1304, - 0x00c1e4b0de42feed, 0xa21819370e4abeda, 0x975bc579bf5c5b37, 0x232e5d34a24842df, - 0xfbaca81adaee82de, 0x0f447fd3dcc25a8e, 0x26a04d5eec2dcf88, 0x7ce27c2abc257339, - 0xfcee65f724d136b2, 0xb6e7b6dcc42c3b62, 0xd11e605f09ababfa, 0x7d724ac35f2767d8, - 0x2bcd6d76ac3feb3e, 0x4d97a36d94eadfda, 0x61b6d3637f5cce92, 0xd90cd9ebf0cd9feb, - 0x5944c4d774209e69, 0x2b8c199f2f9cd449, 0xcfdbd3bb25e499d2, 0x5c22ee4e195f0248, - 0xee253c7bdb5e65eb, 0xd8d43ae6715b2ea2, 0xb2868617f42ca4ff, 0x6690857cc463533f, - 0x005b0303a02f27ee, 0x410969717e432ef5, 0xa650b351025b2680, 0xceb8881349eb377d, - 0x9a3d969ca087b1b1, 0x8da924954df86b2e, 0x60074a7ac5554acb, 0xe210859fef7c36a0, - 0x1b403176d0454963, 0xb238b8d4c4553aaf, 0x172e037bfec7d771, 0x129d0c3f75da68c7, - 0x0881ccce752db1dd, 0xd3b243280608f1ef, 0x1913635cd239a5ed, 0x6c994870809b7bbc, - 0x344680ef8fd418bf, 0x46f3feb544e7b9b8, 0x86d9c4f8dae38753, 0x5a9fb822e5ef113e, - 0x59d33eaa0e348c8d, 0x66ba5244432972e4, 0xa99ec36e26df8e47, 0x674ff09bf5ce2d5b, - 0x3a0c57d06eee0d67, 0x71c97b05f6c5866d, 0x80dd5fa0bf35be4c, 0x13524362ae5f1db4, - 0x17dce6e2ae6820ff, 0xc881e83f32221729, 0x7f5553ad2965ac59, 0x8647c7f22907e715, - 0x62ef4d3badeb588c, 0x29ecb0c1b24fc1a2, 0xbfe650e4982806f2, 0x2cd405077c358cff, - 0x65c2bbe0c43cd919, 0x0e8f35ec30b0bc01, 0x08244de0e4e3a95c, 0x844f73b1f30bacfb, - 0xdb85da7657a84717, 0x2f22e93ba6318885, 0xe312b7fc62d993db, 0x7fc93c4904c74f53, - 0x2f68fc80f5f115a5, 0x1e8919565eb5687c, 0x4d5204cd5a96b1db, 0x28d3056029c34e40, - 0x842e8b14f5725d38, 0xed6e80af169ff711, 0xe36e479519579a2b, 0xf2d9ed21b2570d6e, - 0x65d2b078ed0712eb, 0x3a42fc74b41cacc1, 0x7b1f0e56c4b12cdd, 0x4cfac05dd835bea4, - 0x5fe32b621ccb8d95, 0x6059259384eac85c, 0x2bc0b612cc22e559, 0x1d31040fcd81859d, - 0xaea4e13318cd9fb0, 0x3b53bfb970e86aa3, 0xb2dd16f3bcf7596c, 0xe7537b4c8f0c9501, - 0x8a0f824d41213f42, 0xd922ed29c715edbf, 0x79244f20ee30de36, 0xcf523cfba46abf89, - 0x896368c1d0be7766, 0x311fd2a367d5082e, 0x8176e7225aea51a9, 0x1757d632d7d98eb7, - 0x38c7203051aefe7a, 0x63428c43985e119e, 0xefc79a08daf39ad8, 0x545c1d858784bdb1, - 0xf70b98ef0e6eb20b, 0xd5ac8a2229277e40, 0x7764cdd6e4c4b4ba, 0xc856ac44d0224b44, - 0xf8c2bafe6cafc3ea, 0x2284959771587488, 0x6d5dadd0470fb8b5, 0x9701a89343c7411d, - 0xedebc1114abe1724, 0x091852c6f3fc18d7, 0x5eb0a16d80c5f46c, 0xab9316be0fbcfc4e, - 0x805f75e838096030, 0xa1788001b4618dd2, 0x6a506e1266276f20, 0x7384e4321763b344, - 0xfed32314e35ed692, 0x1c90e8a85342fb36, 0xc1d600ab95893d69, 0x62e6ecc3dd269d32, - 0x291d4115260d0e27, 0xa4596ff520f8da42, 0xd063d8ae070fd080, 0x44520a9da928c6ee, - 0xf8530d4ac6992d42, 0xa2bdc07d30cebdc5, 0xc34a56a262d383c6, 0xaade45e868d8e145, - 0x9e77a8400a2a2ce9, 0xc5dd100e3a463dd0, 0x39d97c7485c9c11b, 0x90e8791c86b9e9e0, - 0xa42815ecba6f6cb0, 0x08a6bf14e6377cc0, 0x243e8ba548085447, 0x76e86de95445e177, - 0xc0036c9fd957e7b2, 0x637526fa8d6ac90b, 0xedf07599e940d1d3, 0x41ae5ef4fcad2bb6, - 0xa22dd183c1156b76, 0xea799fbe273e62a3, 0xf5c44c7dd4742e12, 0x71f88c2d35182e4b, - 0xcba84e538201f7da, 0x91a5dcc5aa8df96c, 0xf559c92da6f130b8, 0x816259eb813ef7ec, - 0x921f650abd50e426, 0x9044bc34edcc5427, 0x3ed63258145ded72, 0xed4b39eedb53e444, - 0x42ec8eeecd95e973, 0xd8914c22e58535dd, 0x18c89b389ca25e60, 0x835c44d99f443f4c, - 0x7aba8cd0ddc0bd5a, 0x9aa1ef2f0e6d2226, 0x49aeea7c96e39f00, 0x0c946447e87ba4b9, - 0x6729ca0cb0a7e0d8, 0x20a4833a45b2fdc6, 0xcc5d7491d083ba51, 0x3ad921c0da791eeb, - 0x401df7dbd7d6a812, 0xdbfe46a6aca199e6, 0x2c6fadc847580f40, 0x1542a5c83753e9b5, - 0x1caa4a2cadf60bea, 0x8781a33a0d88f683, 0xc63a4e33769877e0, 0x9c49622dc5f2ca9d, - 0xdcface9923dd98c9, 0x108587d7ace67548, 0x73c785aa0079ed28, 0xd2182bf8ebe7cd17, - 0x68202163b7f22d01, 0xaafdb8cea21bc3f1, 0xbe6fb61ba5009f2b, 0xf92a4a43c49c515a, - 0xe3565c359e17b346, 0x98c8711a15d4cf01, 0xc3502aa8d2ae83f7, 0x84b1c7694d8ef559, - 0xd635f8af5398e3a5, 0x3d7585a1fcdfbbb5, 0x1a0ccfb505725777, 0x3e6b0354a9d27efe, - 0x872455abf0543021, 0x172969f5082f8a37, 0xf3f4d526164f46d5, 0x51357ae9b6792730, - 0x6d8015674bc66fab, 0x85335775a6e43bb8, 0x7bb30c057cf0e078, 0x1559affa39309efe, - 0x9a3590f9a9b550f8, 0x68079f0ccbe8ad58, 0xedf4e5b5adf66113, 0x7912aba0d953f662, - 0xc5580366906f02c1, 0xe404d793a1e8fa8e, 0x35a026bb95a255ce, 0x847c7a2231883ab9, - 0x566495915cc2f375, 0x23fb0e28199220f1, 0xa2e5b935732ecf02, 0xb0c52902952b53ed, - 0xf79872ef461fdfaf, 0x5494638881d2fc86, 0x715d786d7da41de7, 0x95516bbecbfa6167, - 0x10ff08a5e22020d9, 0x87e29f27ac6abfed, 0x4b90993591d821fb, 0xf189c6737184285f, - 0x108993c4fcc1c5eb, 0xa369dfbe7b6e6871, 0x35d8b0f514abb1d7, 0x73ed1ad3274276d3, - 0x40ad8585fedcbfbb, 0x5057cb690e51c590, 0xb0d2d2eb9b130af8, 0x32b160707ed2b251, - 0x16568fbb4e7232fb, 0x8dc55110ab248167, 0xbd050e4500865356, 0xa397e5ac5ba2a576, - 0xd025614cd01cc80d, 0x5f06d34865819cb5, 0x45c43c3e117c235f, 0x9eed55756efbcc43, - 0xa408304fb8305397, 0x00bd449bb5e55004, 0xbbcae7ca38aa6225, 0x0d4ef39450c97860, - 0x0cf7cedf683bb133, 0xfff8476a0384a059, 0xe98fb76051b87594, 0x0cd95d0b48b2ddef, - 0x9a53aa2693419d79, 0xcf63d8a4ae17f623, 0x56017663a4d4584b, 0xc070a807db611a4c, - 0x0acfffea81936050, 0xa4dd81e3c2500ab6, 0x133a72803fc0fd86, 0x0c1d5640fcc6d4eb, - 0x9a905acdecd70898, 0x0a832ab0dc91189a, 0xad295fcf00d94c2c, 0x1a7d49153a349d45, - 0x7bf859d4478d2bc4, 0xa23d3e001e178dcc, 0x5a19288292119724, 0x7324693c833afcd9, - 0x568880c28891762b, 0x791109be7b52d10e, 0xa639fb506e789c6f, 0x68a7fcf408bc1b29, - 0x3d4834b0e7b89b03, 0x80378c31b3cb28f1, 0x0b686e2354f0f916, 0x50c7be955debe0e7, - 0x73ae40212a7ff22f, 0xa63e52b3efcf14d7, 0xd95fa3f1f0167326, 0x551304b3e157a9bb, - 0xc07121222b861230, 0xbd0513345ac59964, 0x8d12404259d7ef1a, 0x963711fc56b526a5, - 0xb3b1e9c18fe23068, 0x8063d9b4990cc33c, 0x679270ac2da3d484, 0x248441b37fd4c6c7, - 0xdbafabe2989e9b16, 0xf511c53516d8f4c6, 0x643f704d00cdba12, 0x8ff4091defe80837, - 0x1d8a18c8e427a49b, 0x3868df864ea6dfe9, 0x580e1923973b6b74, 0xba56b3d778c7a078, - 0x420b4960650dae71, 0x433623cc7f8913d8, 0xbb704a1372680a88, 0x105fc46e6c8b393e, - 0x6418ce16bcad0851, 0x6351985bffe1f8ba, 0x0409ad4d28d6ccf1, 0xd9147ae04d7a214c, - 0xb2c1162a16ffd179, 0xddcd005b7a95897d, 0xfae577761799ce0b, 0x4fc64d6ceceb7197, - 0xbb8d887b360dd9b3, 0x8add2612d289471a, 0x025ce64f77577eef, 0x7f33a4942616d6ba, - 0x74bafa991fc1ffbd, 0xccf760eda80ef0f6, 0xd6c8749143d46d4b, 0x6f08c4d37213cc69, - 0xc290b8ef99f67952, 0xb38641440c0279e5, 0x60e2b64e7674eeee, 0x8de2f124f6efdf5f, - 0x9e563a789655d2b1, 0xdb35e26d1dccc985, 0xc2309cf23ff53f61, 0x9495c85069a0bab4, - 0x37d0e273fac205f1, 0x675f44fcf7866825, 0x5203cc412fb3bbb2, 0xa142a8c605554b80, - 0x26fb5f9ade3f170a, 0x2cd6cb6debc1a7ac, 0xe304c2a6a6add3aa, 0xf47e80b7f8e858d3, - 0x80fe2acfa8fd4476, 0xdc60de24b5a6fc19, 0xd298272486d3f8ce, 0x0e047237a77f922e, - 0xc4745b35122cc657, 0x06487cb1997bc45d, 0x64e0e41aa72cf6e9, 0x2d02282f99d75823, - 0xf81ac1e1d44032ca, 0xf15a03bea3dbed4e, 0xe7c4c1f3346ebf80, 0xf086ac1c39268bec, - 0xba0940095584636f, 0x12b274c13fe5bd4e, 0x8bdaa80dfa6d4998, 0x534e403a0cd2be2b, - 0x5a3433edfd3918b5, 0x770caf904ba076ff, 0x8557b50ff773270e, 0x0184af3ef759e12c, - 0x9a19a234321c833d, 0x7042a8398d4aba7d, 0x5ddc7ea76b0596d0, 0x0801ee3dbaec1ecb, - 0xc7b58665a699438b, 0xd08c488c2e397d7a, 0x547d5d536ec45a9a, 0x89c3c359310e93ae, - 0x82d394074769230e, 0x331adf7180bff49f, 0x3150d7a6b5d9d7fd, 0x42d603bae42b075b, - 0xea6d476bc8779b31, 0x66596427885a80fd, 0xeff1a6307eeb6cba, 0xec651c71b80590b1, - 0x423709523bf0ea29, 0x80f0da57ff550b66, 0x0829b599cbd38cde, 0x22fcc23ca86c6d36, - 0x14b5678161dc44d1, 0x05a745a52be98f4d, 0x8b93ec98870f8b90, 0xd775534901bf7758, - 0xdba89dbbf023f4c6, 0x3d53832454e2ac1f, 0x9f04e6f8558f3d53, 0x97860cb0d21f03c6, - 0x4e86931d816a4dd4, 0x6c4042bad608c7c4, 0x4ed848a0213e5856, 0x0b542408eed49bc5, - 0x336ed821111c8856, 0x87f61959f2811ec3, 0xdc00afb8c213b638, 0x71a0bcc12922f5ad, - 0x3399c168d6098256, 0x001f7bdd3310eb8b, 0xeb7a52be3d826686, 0x4b6fd5d4d8fa5b75, - 0x84e06a9ce249fb6b, 0xe98f2f20e139be1a, 0x3bc7736c7928e64a, 0xf5be36ff11a04d6a, - 0x0bbaaa244658445d, 0x4131dd1909f86a87, 0x30be054a628ba5ac, 0x769b20fdf83c137a, - 0xde68f5981ee65f3c, 0xf533d0a2dfb31792, 0x2c111775770b918e, 0x8b30515fc7f8dd2b, - 0x7c589985e06b31fe, 0xa83de698c86d7e89, 0x89373f7d9a6e7297, 0xa2d12871e280ae3f, - 0xd4f3b17d43d8256c, 0x9cc26b2ad483ba76, 0xd61867b2dadb4dee, 0x8f39db56af4a87d4, - 0x92f8bc892b51faba, 0x41dfa0537b565d27, 0xa178d6bdd3c12f37, 0xec5ba2b64a160d67, - 0x766d2432e1fd1294, 0xcbccfbe1cee8ec26, 0xbecca3d1443b58a4, 0x88c4861eb8f2278a, - 0x8bb5bae190557746, 0x3b63c532468dd64d, 0x1aa2c322d145d43b, 0x96b2a74655ee197f, - 0xefd3f233bc177187, 0x16fb653c3c6eabfb, 0xf4cc9f36d64f4ba2, 0xfa6ba7a73ead99d7, - 0xd79e4c6ed4e7c060, 0x778fdc5920d7584a, 0xfd630d6782d4075c, 0xaa2a9f34b8b855e9, - 0x772011a98a6fc073, 0x25fc7a0eb613e652, 0xe63467ab90cee061, 0x3b6538372b9fe7b1, - 0xbb8132158fd258a6, 0xfe86da623f19c422, 0x5811290e8b89e54e, 0x75a8bf15a08cb3e5 -}; diff --git a/openssl/src/crypto/ec/ecp_smpl.c b/openssl/src/crypto/ec/ecp_smpl.c index 112a4f0a2..bde8cad34 100644 --- a/openssl/src/crypto/ec/ecp_smpl.c +++ b/openssl/src/crypto/ec/ecp_smpl.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -9,7 +9,7 @@ */ /* - * ECDSA low-level APIs are deprecated for public use, but still ok for + * ECDSA low level APIs are deprecated for public use, but still ok for * internal use. */ #include "internal/deprecated.h" @@ -171,7 +171,7 @@ int ossl_ec_GFp_simple_group_set_curve(EC_GROUP *group, /* group->a */ if (!BN_nnmod(tmp_a, a, p, ctx)) goto err; - if (group->meth->field_encode != NULL) { + if (group->meth->field_encode) { if (!group->meth->field_encode(group, group->a, tmp_a, ctx)) goto err; } else if (!BN_copy(group->a, tmp_a)) @@ -180,7 +180,7 @@ int ossl_ec_GFp_simple_group_set_curve(EC_GROUP *group, /* group->b */ if (!BN_nnmod(group->b, b, p, ctx)) goto err; - if (group->meth->field_encode != NULL) + if (group->meth->field_encode) if (!group->meth->field_encode(group, group->b, group->b, ctx)) goto err; @@ -209,7 +209,7 @@ int ossl_ec_GFp_simple_group_get_curve(const EC_GROUP *group, BIGNUM *p, } if (a != NULL || b != NULL) { - if (group->meth->field_decode != NULL) { + if (group->meth->field_decode) { if (ctx == NULL) { ctx = new_ctx = BN_CTX_new_ex(group->libctx); if (ctx == NULL) @@ -258,7 +258,7 @@ int ossl_ec_GFp_simple_group_check_discriminant(const EC_GROUP *group, if (ctx == NULL) { ctx = new_ctx = BN_CTX_new_ex(group->libctx); if (ctx == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } } @@ -271,7 +271,7 @@ int ossl_ec_GFp_simple_group_check_discriminant(const EC_GROUP *group, if (order == NULL) goto err; - if (group->meth->field_decode != NULL) { + if (group->meth->field_decode) { if (!group->meth->field_decode(group, a, group->a, ctx)) goto err; if (!group->meth->field_decode(group, b, group->b, ctx)) @@ -440,7 +440,7 @@ int ossl_ec_GFp_simple_get_Jprojective_coordinates_GFp(const EC_GROUP *group, BN_CTX *new_ctx = NULL; int ret = 0; - if (group->meth->field_decode != NULL) { + if (group->meth->field_decode != 0) { if (ctx == NULL) { ctx = new_ctx = BN_CTX_new_ex(group->libctx); if (ctx == NULL) @@ -529,7 +529,7 @@ int ossl_ec_GFp_simple_point_get_affine_coordinates(const EC_GROUP *group, /* transform (X, Y, Z) into (x, y) := (X/Z^2, Y/Z^3) */ - if (group->meth->field_decode != NULL) { + if (group->meth->field_decode) { if (!group->meth->field_decode(group, Z, point->Z, ctx)) goto err; Z_ = Z; @@ -538,7 +538,7 @@ int ossl_ec_GFp_simple_point_get_affine_coordinates(const EC_GROUP *group, } if (BN_is_one(Z_)) { - if (group->meth->field_decode != NULL) { + if (group->meth->field_decode) { if (x != NULL) { if (!group->meth->field_decode(group, x, point->X, ctx)) goto err; @@ -563,7 +563,7 @@ int ossl_ec_GFp_simple_point_get_affine_coordinates(const EC_GROUP *group, goto err; } - if (group->meth->field_encode == NULL) { + if (group->meth->field_encode == 0) { /* field_sqr works on standard representation */ if (!group->meth->field_sqr(group, Z_2, Z_1, ctx)) goto err; @@ -582,7 +582,7 @@ int ossl_ec_GFp_simple_point_get_affine_coordinates(const EC_GROUP *group, } if (y != NULL) { - if (group->meth->field_encode == NULL) { + if (group->meth->field_encode == 0) { /* * field_mul works on standard representation */ @@ -1275,7 +1275,7 @@ int ossl_ec_GFp_simple_points_make_affine(const EC_GROUP *group, size_t num, ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); goto err; } - if (group->meth->field_encode != NULL) { + if (group->meth->field_encode != 0) { /* * In the Montgomery case, we just turned R*H (representing H) into * 1/(R*H), but we need R*(1/H) (representing 1/H); i.e. we need to @@ -1376,7 +1376,7 @@ int ossl_ec_GFp_simple_field_sqr(const EC_GROUP *group, BIGNUM *r, const BIGNUM /*- * Computes the multiplicative inverse of a in GF(p), storing the result in r. - * If a is zero (or equivalent), you'll get an EC_R_CANNOT_INVERT error. + * If a is zero (or equivalent), you'll get a EC_R_CANNOT_INVERT error. * Since we don't have a Mont structure here, SCA hardening is with blinding. * NB: "a" must be in _decoded_ form. (i.e. field_decode must precede.) */ @@ -1423,8 +1423,8 @@ int ossl_ec_GFp_simple_field_inv(const EC_GROUP *group, BIGNUM *r, /*- * Apply randomization of EC point projective coordinates: * - * (X, Y, Z) = (lambda^2*X, lambda^3*Y, lambda*Z) - * lambda = [1, group->field) + * (X, Y ,Z ) = (lambda^2*X, lambda^3*Y, lambda*Z) + * lambda = [1,group->field) * */ int ossl_ec_GFp_simple_blind_coordinates(const EC_GROUP *group, EC_POINT *p, @@ -1438,7 +1438,7 @@ int ossl_ec_GFp_simple_blind_coordinates(const EC_GROUP *group, EC_POINT *p, lambda = BN_CTX_get(ctx); temp = BN_CTX_get(ctx); if (temp == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto end; } diff --git a/openssl/src/crypto/ec/ecx_backend.c b/openssl/src/crypto/ec/ecx_backend.c index d21c03784..2ab7611be 100644 --- a/openssl/src/crypto/ec/ecx_backend.c +++ b/openssl/src/crypto/ec/ecx_backend.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -110,16 +110,22 @@ ECX_KEY *ossl_ecx_key_dup(const ECX_KEY *key, int selection) { ECX_KEY *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NULL; + } + + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + OPENSSL_free(ret); + return NULL; + } ret->libctx = key->libctx; - ret->haspubkey = 0; + ret->haspubkey = key->haspubkey; ret->keylen = key->keylen; ret->type = key->type; - - if (!CRYPTO_NEW_REF(&ret->references, 1)) - goto err; + ret->references = 1; if (key->propq != NULL) { ret->propq = OPENSSL_strdup(key->propq); @@ -127,26 +133,21 @@ ECX_KEY *ossl_ecx_key_dup(const ECX_KEY *key, int selection) goto err; } - if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0 - && key->haspubkey == 1) { + if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) memcpy(ret->pubkey, key->pubkey, sizeof(ret->pubkey)); - ret->haspubkey = 1; - } if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0 && key->privkey != NULL) { - if (ossl_ecx_key_allocate_privkey(ret) == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + if (ossl_ecx_key_allocate_privkey(ret) == NULL) goto err; - } memcpy(ret->privkey, key->privkey, ret->keylen); } return ret; err: - CRYPTO_FREE_REF(&ret->references); ossl_ecx_key_free(ret); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return NULL; } @@ -185,7 +186,7 @@ ECX_KEY *ossl_ecx_key_op(const X509_ALGOR *palg, key = ossl_ecx_key_new(libctx, KEYNID2TYPE(id), 1, propq); if (key == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } pubkey = key->pubkey; @@ -195,7 +196,7 @@ ECX_KEY *ossl_ecx_key_op(const X509_ALGOR *palg, } else { privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } if (op == KEY_OP_KEYGEN) { diff --git a/openssl/src/crypto/ec/ecx_key.c b/openssl/src/crypto/ec/ecx_key.c index ba725eb57..dcec26c2e 100644 --- a/openssl/src/crypto/ec/ecx_key.c +++ b/openssl/src/crypto/ec/ecx_key.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,13 +9,7 @@ #include #include -#include #include "crypto/ecx.h" -#include "internal/common.h" /* for ossl_assert() */ - -#ifdef S390X_EC_ASM -# include "s390x_arch.h" -#endif ECX_KEY *ossl_ecx_key_new(OSSL_LIB_CTX *libctx, ECX_KEY_TYPE type, int haspubkey, const char *propq) @@ -42,21 +36,20 @@ ECX_KEY *ossl_ecx_key_new(OSSL_LIB_CTX *libctx, ECX_KEY_TYPE type, int haspubkey break; } ret->type = type; - - if (!CRYPTO_NEW_REF(&ret->references, 1)) - goto err; + ret->references = 1; if (propq != NULL) { ret->propq = OPENSSL_strdup(propq); if (ret->propq == NULL) goto err; } + + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) + goto err; return ret; err: - if (ret != NULL) { - OPENSSL_free(ret->propq); - CRYPTO_FREE_REF(&ret->references); - } + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -68,7 +61,7 @@ void ossl_ecx_key_free(ECX_KEY *key) if (key == NULL) return; - CRYPTO_DOWN_REF(&key->references, &i); + CRYPTO_DOWN_REF(&key->references, &i, key->lock); REF_PRINT_COUNT("ECX_KEY", key); if (i > 0) return; @@ -76,7 +69,7 @@ void ossl_ecx_key_free(ECX_KEY *key) OPENSSL_free(key->propq); OPENSSL_secure_clear_free(key->privkey, key->keylen); - CRYPTO_FREE_REF(&key->references); + CRYPTO_THREAD_lock_free(key->lock); OPENSSL_free(key); } @@ -89,7 +82,7 @@ int ossl_ecx_key_up_ref(ECX_KEY *key) { int i; - if (CRYPTO_UP_REF(&key->references, &i) <= 0) + if (CRYPTO_UP_REF(&key->references, &i, key->lock) <= 0) return 0; REF_PRINT_COUNT("ECX_KEY", key); @@ -103,61 +96,3 @@ unsigned char *ossl_ecx_key_allocate_privkey(ECX_KEY *key) return key->privkey; } - -int ossl_ecx_compute_key(ECX_KEY *peer, ECX_KEY *priv, size_t keylen, - unsigned char *secret, size_t *secretlen, size_t outlen) -{ - if (priv == NULL - || priv->privkey == NULL - || peer == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_KEY); - return 0; - } - - if (!ossl_assert(keylen == X25519_KEYLEN - || keylen == X448_KEYLEN)) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); - return 0; - } - - if (secret == NULL) { - *secretlen = keylen; - return 1; - } - if (outlen < keylen) { - ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); - return 0; - } - - if (keylen == X25519_KEYLEN) { -#ifdef S390X_EC_ASM - if (OPENSSL_s390xcap_P.pcc[1] - & S390X_CAPBIT(S390X_SCALAR_MULTIPLY_X25519)) { - if (s390x_x25519_mul(secret, peer->pubkey, priv->privkey) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); - return 0; - } - } else -#endif - if (ossl_x25519(secret, priv->privkey, peer->pubkey) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); - return 0; - } - } else { -#ifdef S390X_EC_ASM - if (OPENSSL_s390xcap_P.pcc[1] - & S390X_CAPBIT(S390X_SCALAR_MULTIPLY_X448)) { - if (s390x_x448_mul(secret, peer->pubkey, priv->privkey) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); - return 0; - } - } else -#endif - if (ossl_x448(secret, priv->privkey, peer->pubkey) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); - return 0; - } - } - *secretlen = keylen; - return 1; -} diff --git a/openssl/src/crypto/ec/ecx_meth.c b/openssl/src/crypto/ec/ecx_meth.c index 6c445f912..72244f1f9 100644 --- a/openssl/src/crypto/ec/ecx_meth.c +++ b/openssl/src/crypto/ec/ecx_meth.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -39,13 +39,15 @@ static int ecx_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) } penc = OPENSSL_memdup(ecxkey->pubkey, KEYLEN(pkey)); - if (penc == NULL) + if (penc == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; + } if (!X509_PUBKEY_set0_param(pk, OBJ_nid2obj(pkey->ameth->pkey_id), V_ASN1_UNDEF, NULL, penc, KEYLEN(pkey))) { OPENSSL_free(penc); - ERR_raise(ERR_LIB_EC, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } return 1; @@ -113,14 +115,14 @@ static int ecx_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) penclen = i2d_ASN1_OCTET_STRING(&oct, &penc); if (penclen < 0) { - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(pkey->ameth->pkey_id), 0, V_ASN1_UNDEF, NULL, penc, penclen)) { OPENSSL_clear_free(penc, penclen); - ERR_raise(ERR_LIB_EC, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); return 0; } @@ -136,7 +138,7 @@ static int ecx_bits(const EVP_PKEY *pkey) { if (IS25519(pkey->ameth->pkey_id)) { return X25519_BITS; - } else if (ISX448(pkey->ameth->pkey_id)) { + } else if(ISX448(pkey->ameth->pkey_id)) { return X448_BITS; } else { return ED448_BITS; @@ -390,7 +392,7 @@ static int ecx_generic_import_from(const OSSL_PARAM params[], void *vpctx, pctx->propquery); if (ecx == NULL) { - ERR_raise(ERR_LIB_DH, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return 0; } @@ -559,23 +561,17 @@ static int ecd_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, return 2; } -static int ecd_item_sign(X509_ALGOR *alg1, X509_ALGOR *alg2, int nid) -{ - /* Note that X509_ALGOR_set0(..., ..., V_ASN1_UNDEF, ...) cannot fail */ - /* Set algorithms identifiers */ - (void)X509_ALGOR_set0(alg1, OBJ_nid2obj(nid), V_ASN1_UNDEF, NULL); - if (alg2 != NULL) - (void)X509_ALGOR_set0(alg2, OBJ_nid2obj(nid), V_ASN1_UNDEF, NULL); - /* Algorithm identifiers set: carry on as normal */ - return 3; -} - static int ecd_item_sign25519(EVP_MD_CTX *ctx, const ASN1_ITEM *it, const void *asn, X509_ALGOR *alg1, X509_ALGOR *alg2, ASN1_BIT_STRING *str) { - return ecd_item_sign(alg1, alg2, NID_ED25519); + /* Set algorithms identifiers */ + X509_ALGOR_set0(alg1, OBJ_nid2obj(NID_ED25519), V_ASN1_UNDEF, NULL); + if (alg2) + X509_ALGOR_set0(alg2, OBJ_nid2obj(NID_ED25519), V_ASN1_UNDEF, NULL); + /* Algorithm identifiers set: carry on as normal */ + return 3; } static int ecd_sig_info_set25519(X509_SIG_INFO *siginf, const X509_ALGOR *alg, @@ -591,7 +587,12 @@ static int ecd_item_sign448(EVP_MD_CTX *ctx, const ASN1_ITEM *it, X509_ALGOR *alg1, X509_ALGOR *alg2, ASN1_BIT_STRING *str) { - return ecd_item_sign(alg1, alg2, NID_ED448); + /* Set algorithm identifier */ + X509_ALGOR_set0(alg1, OBJ_nid2obj(NID_ED448), V_ASN1_UNDEF, NULL); + if (alg2 != NULL) + X509_ALGOR_set0(alg2, OBJ_nid2obj(NID_ED448), V_ASN1_UNDEF, NULL); + /* Algorithm identifier set: carry on as normal */ + return 3; } static int ecd_sig_info_set448(X509_SIG_INFO *siginf, const X509_ALGOR *alg, @@ -807,11 +808,6 @@ static int pkey_ecd_digestsign25519(EVP_MD_CTX *ctx, unsigned char *sig, { const ECX_KEY *edkey = evp_pkey_get_legacy(EVP_MD_CTX_get_pkey_ctx(ctx)->pkey); - if (edkey == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_KEY); - return 0; - } - if (sig == NULL) { *siglen = ED25519_SIGSIZE; return 1; @@ -821,10 +817,8 @@ static int pkey_ecd_digestsign25519(EVP_MD_CTX *ctx, unsigned char *sig, return 0; } - if (ossl_ed25519_sign(sig, tbs, tbslen, edkey->pubkey, edkey->privkey, - 0, 0, 0, - NULL, 0, - NULL, NULL) == 0) + if (ossl_ed25519_sign(sig, tbs, tbslen, edkey->pubkey, edkey->privkey, NULL, + NULL) == 0) return 0; *siglen = ED25519_SIGSIZE; return 1; @@ -836,11 +830,6 @@ static int pkey_ecd_digestsign448(EVP_MD_CTX *ctx, unsigned char *sig, { const ECX_KEY *edkey = evp_pkey_get_legacy(EVP_MD_CTX_get_pkey_ctx(ctx)->pkey); - if (edkey == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_KEY); - return 0; - } - if (sig == NULL) { *siglen = ED448_SIGSIZE; return 1; @@ -851,7 +840,7 @@ static int pkey_ecd_digestsign448(EVP_MD_CTX *ctx, unsigned char *sig, } if (ossl_ed448_sign(edkey->libctx, sig, tbs, tbslen, edkey->pubkey, - edkey->privkey, NULL, 0, 0, edkey->propq) == 0) + edkey->privkey, NULL, 0, edkey->propq) == 0) return 0; *siglen = ED448_SIGSIZE; return 1; @@ -863,17 +852,10 @@ static int pkey_ecd_digestverify25519(EVP_MD_CTX *ctx, const unsigned char *sig, { const ECX_KEY *edkey = evp_pkey_get_legacy(EVP_MD_CTX_get_pkey_ctx(ctx)->pkey); - if (edkey == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_KEY); - return 0; - } - if (siglen != ED25519_SIGSIZE) return 0; return ossl_ed25519_verify(tbs, tbslen, sig, edkey->pubkey, - 0, 0, 0, - NULL, 0, edkey->libctx, edkey->propq); } @@ -883,16 +865,11 @@ static int pkey_ecd_digestverify448(EVP_MD_CTX *ctx, const unsigned char *sig, { const ECX_KEY *edkey = evp_pkey_get_legacy(EVP_MD_CTX_get_pkey_ctx(ctx)->pkey); - if (edkey == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_KEY); - return 0; - } - if (siglen != ED448_SIGSIZE) return 0; return ossl_ed448_verify(edkey->libctx, tbs, tbslen, sig, edkey->pubkey, - NULL, 0, 0, edkey->propq); + NULL, 0, edkey->propq); } static int pkey_ecd_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) @@ -948,7 +925,7 @@ static int s390x_pkey_ecx_keygen25519(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) unsigned char *privkey = NULL, *pubkey; if (key == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -956,7 +933,7 @@ static int s390x_pkey_ecx_keygen25519(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -991,7 +968,7 @@ static int s390x_pkey_ecx_keygen448(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) unsigned char *privkey = NULL, *pubkey; if (key == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -999,7 +976,7 @@ static int s390x_pkey_ecx_keygen448(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -1040,7 +1017,7 @@ static int s390x_pkey_ecd_keygen25519(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) int rv; if (key == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -1048,7 +1025,7 @@ static int s390x_pkey_ecd_keygen25519(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -1106,7 +1083,7 @@ static int s390x_pkey_ecd_keygen448(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) int rv; if (key == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -1114,7 +1091,7 @@ static int s390x_pkey_ecd_keygen448(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_EC, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); goto err; } @@ -1163,10 +1140,12 @@ static int s390x_pkey_ecx_derive25519(EVP_PKEY_CTX *ctx, unsigned char *key, { const unsigned char *privkey, *pubkey; - if (!validate_ecx_derive(ctx, key, keylen, &privkey, &pubkey) - || (key != NULL - && s390x_x25519_mul(key, privkey, pubkey) == 0)) + if (!validate_ecx_derive(ctx, key, keylen, &privkey, &pubkey)) return 0; + + if (key != NULL) + return s390x_x25519_mul(key, pubkey, privkey); + *keylen = X25519_KEYLEN; return 1; } @@ -1176,10 +1155,12 @@ static int s390x_pkey_ecx_derive448(EVP_PKEY_CTX *ctx, unsigned char *key, { const unsigned char *privkey, *pubkey; - if (!validate_ecx_derive(ctx, key, keylen, &privkey, &pubkey) - || (key != NULL - && s390x_x448_mul(key, pubkey, privkey) == 0)) + if (!validate_ecx_derive(ctx, key, keylen, &privkey, &pubkey)) return 0; + + if (key != NULL) + return s390x_x448_mul(key, pubkey, privkey); + *keylen = X448_KEYLEN; return 1; } @@ -1199,11 +1180,6 @@ static int s390x_pkey_ecd_digestsign25519(EVP_MD_CTX *ctx, const ECX_KEY *edkey = evp_pkey_get_legacy(EVP_MD_CTX_get_pkey_ctx(ctx)->pkey); int rc; - if (edkey == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_KEY); - return 0; - } - if (sig == NULL) { *siglen = ED25519_SIGSIZE; return 1; @@ -1244,11 +1220,6 @@ static int s390x_pkey_ecd_digestsign448(EVP_MD_CTX *ctx, const ECX_KEY *edkey = evp_pkey_get_legacy(EVP_MD_CTX_get_pkey_ctx(ctx)->pkey); int rc; - if (edkey == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_KEY); - return 0; - } - if (sig == NULL) { *siglen = ED448_SIGSIZE; return 1; @@ -1291,11 +1262,6 @@ static int s390x_pkey_ecd_digestverify25519(EVP_MD_CTX *ctx, } param; const ECX_KEY *edkey = evp_pkey_get_legacy(EVP_MD_CTX_get_pkey_ctx(ctx)->pkey); - if (edkey == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_KEY); - return 0; - } - if (siglen != ED25519_SIGSIZE) return 0; @@ -1323,11 +1289,6 @@ static int s390x_pkey_ecd_digestverify448(EVP_MD_CTX *ctx, } param; const ECX_KEY *edkey = evp_pkey_get_legacy(EVP_MD_CTX_get_pkey_ctx(ctx)->pkey); - if (edkey == NULL) { - ERR_raise(ERR_LIB_EC, EC_R_INVALID_KEY); - return 0; - } - if (siglen != ED448_SIGSIZE) return 0; diff --git a/openssl/src/crypto/ec/gen/darwin_arm64/ecp_nistz256-armv8.S b/openssl/src/crypto/ec/gen/darwin_arm64/ecp_nistz256-armv8.S index bc30b5f5e..8bbfa9ea7 100644 --- a/openssl/src/crypto/ec/gen/darwin_arm64/ecp_nistz256-armv8.S +++ b/openssl/src/crypto/ec/gen/darwin_arm64/ecp_nistz256-armv8.S @@ -2395,7 +2395,7 @@ LordK: .align 6 _ecp_nistz256_to_mont: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2411,7 +2411,7 @@ _ecp_nistz256_to_mont: ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2420,7 +2420,7 @@ _ecp_nistz256_to_mont: .align 4 _ecp_nistz256_from_mont: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2436,7 +2436,7 @@ _ecp_nistz256_from_mont: ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2446,7 +2446,7 @@ _ecp_nistz256_from_mont: .align 4 _ecp_nistz256_mul_mont: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2461,7 +2461,7 @@ _ecp_nistz256_mul_mont: ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2470,7 +2470,7 @@ _ecp_nistz256_mul_mont: .align 4 _ecp_nistz256_sqr_mont: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2484,7 +2484,7 @@ _ecp_nistz256_sqr_mont: ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2494,7 +2494,7 @@ _ecp_nistz256_sqr_mont: .align 4 _ecp_nistz256_add: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2508,7 +2508,7 @@ _ecp_nistz256_add: bl __ecp_nistz256_add ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2517,7 +2517,7 @@ _ecp_nistz256_add: .align 4 _ecp_nistz256_div_by_2: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2529,7 +2529,7 @@ _ecp_nistz256_div_by_2: bl __ecp_nistz256_div_by_2 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2538,7 +2538,7 @@ _ecp_nistz256_div_by_2: .align 4 _ecp_nistz256_mul_by_2: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2554,7 +2554,7 @@ _ecp_nistz256_mul_by_2: bl __ecp_nistz256_add // ret = a+a // 2*a ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2563,7 +2563,7 @@ _ecp_nistz256_mul_by_2: .align 4 _ecp_nistz256_mul_by_3: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2590,7 +2590,7 @@ _ecp_nistz256_mul_by_3: bl __ecp_nistz256_add // ret += a // 2*a+a=3*a ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2600,7 +2600,7 @@ _ecp_nistz256_mul_by_3: .align 4 _ecp_nistz256_sub: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2612,7 +2612,7 @@ _ecp_nistz256_sub: bl __ecp_nistz256_sub_from ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -2621,7 +2621,7 @@ _ecp_nistz256_sub: .align 4 _ecp_nistz256_neg: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2636,7 +2636,7 @@ _ecp_nistz256_neg: bl __ecp_nistz256_sub_from ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -3014,7 +3014,7 @@ __ecp_nistz256_div_by_2: .align 5 _ecp_nistz256_point_double: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -3149,14 +3149,14 @@ Ldouble_shortcut: ldp x19,x20,[x29,#16] ldp x21,x22,[x29,#32] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .globl _ecp_nistz256_point_add .align 5 _ecp_nistz256_point_add: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -3403,14 +3403,14 @@ Ladd_done: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .globl _ecp_nistz256_point_add_affine .align 5 _ecp_nistz256_point_add_affine: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -3609,7 +3609,7 @@ _ecp_nistz256_point_add_affine: ldp x23,x24,[x29,#48] ldp x25,x26,[x29,#64] ldp x29,x30,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret //////////////////////////////////////////////////////////////////////// @@ -3619,8 +3619,6 @@ _ecp_nistz256_point_add_affine: .align 4 _ecp_nistz256_ord_mul_mont: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-64]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -3829,8 +3827,6 @@ _ecp_nistz256_ord_mul_mont: .align 4 _ecp_nistz256_ord_sqr_mont: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-64]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -4019,8 +4015,6 @@ Loop_ord_sqr: .align 4 _ecp_nistz256_scatter_w5: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -4083,8 +4077,6 @@ _ecp_nistz256_scatter_w5: .align 4 _ecp_nistz256_gather_w5: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -4162,8 +4154,6 @@ _ecp_nistz256_gather_w5: .align 4 _ecp_nistz256_scatter_w7: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -4208,8 +4198,6 @@ Loop_scatter_w7: .align 4 _ecp_nistz256_gather_w7: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/openssl/src/crypto/ec/gen/darwin_arm64/ecp_sm2p256-armv8.S b/openssl/src/crypto/ec/gen/darwin_arm64/ecp_sm2p256-armv8.S deleted file mode 100644 index 802363602..000000000 --- a/openssl/src/crypto/ec/gen/darwin_arm64/ecp_sm2p256-armv8.S +++ /dev/null @@ -1,826 +0,0 @@ -#include "arm_arch.h" - -.text - -.align 5 -// The polynomial p -Lpoly: -.quad 0xffffffffffffffff,0xffffffff00000000,0xffffffffffffffff,0xfffffffeffffffff -// The order of polynomial n -Lord: -.quad 0x53bbf40939d54123,0x7203df6b21c6052b,0xffffffffffffffff,0xfffffffeffffffff -// (p + 1) / 2 -Lpoly_div_2: -.quad 0x8000000000000000,0xffffffff80000000,0xffffffffffffffff,0x7fffffff7fffffff -// (n + 1) / 2 -Lord_div_2: -.quad 0xa9ddfa049ceaa092,0xb901efb590e30295,0xffffffffffffffff,0x7fffffff7fffffff - -// void bn_rshift1(BN_ULONG *a); -.globl _bn_rshift1 - -.align 5 -_bn_rshift1: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x0] - ldp x9,x10,[x0,#16] - - // Right shift - extr x7,x8,x7,#1 - extr x8,x9,x8,#1 - extr x9,x10,x9,#1 - lsr x10,x10,#1 - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - ret - - -// void bn_sub(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -.globl _bn_sub - -.align 5 -_bn_sub: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Subtraction - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbc x10,x10,x14 - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - ret - - -// void ecp_sm2p256_div_by_2(BN_ULONG *r,const BN_ULONG *a); -.globl _ecp_sm2p256_div_by_2 - -.align 5 -_ecp_sm2p256_div_by_2: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - - // Save the least significant bit - mov x3,x7 - - // Right shift 1 - extr x7,x8,x7,#1 - extr x8,x9,x8,#1 - extr x9,x10,x9,#1 - lsr x10,x10,#1 - - // Load mod - adr x2,Lpoly_div_2 - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Parity check - tst x3,#1 - csel x11,xzr,x11,eq - csel x12,xzr,x12,eq - csel x13,xzr,x13,eq - csel x14,xzr,x14,eq - - // Add - adds x7,x7,x11 - adcs x8,x8,x12 - adcs x9,x9,x13 - adc x10,x10,x14 - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret - - -// void ecp_sm2p256_div_by_2_mod_ord(BN_ULONG *r,const BN_ULONG *a); -.globl _ecp_sm2p256_div_by_2_mod_ord - -.align 5 -_ecp_sm2p256_div_by_2_mod_ord: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - - // Save the least significant bit - mov x3,x7 - - // Right shift 1 - extr x7,x8,x7,#1 - extr x8,x9,x8,#1 - extr x9,x10,x9,#1 - lsr x10,x10,#1 - - // Load mod - adr x2,Lord_div_2 - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Parity check - tst x3,#1 - csel x11,xzr,x11,eq - csel x12,xzr,x12,eq - csel x13,xzr,x13,eq - csel x14,xzr,x14,eq - - // Add - adds x7,x7,x11 - adcs x8,x8,x12 - adcs x9,x9,x13 - adc x10,x10,x14 - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret - - -// void ecp_sm2p256_mul_by_3(BN_ULONG *r,const BN_ULONG *a); -.globl _ecp_sm2p256_mul_by_3 - -.align 5 -_ecp_sm2p256_mul_by_3: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - - // 2*a - adds x7,x7,x7 - adcs x8,x8,x8 - adcs x9,x9,x9 - adcs x10,x10,x10 - adcs x15,xzr,xzr - - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Sub polynomial - adr x2,Lpoly - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbcs x10,x10,x14 - sbcs x15,x15,xzr - - csel x7,x7,x3,cs - csel x8,x8,x4,cs - csel x9,x9,x5,cs - csel x10,x10,x6,cs - eor x15,x15,x15 - - // 3*a - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - adds x7,x7,x11 - adcs x8,x8,x12 - adcs x9,x9,x13 - adcs x10,x10,x14 - adcs x15,xzr,xzr - - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Sub polynomial - adr x2,Lpoly - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbcs x10,x10,x14 - sbcs x15,x15,xzr - - csel x7,x7,x3,cs - csel x8,x8,x4,cs - csel x9,x9,x5,cs - csel x10,x10,x6,cs - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - ret - - -// void ecp_sm2p256_add(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); -.globl _ecp_sm2p256_add - -.align 5 -_ecp_sm2p256_add: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Addition - adds x7,x7,x11 - adcs x8,x8,x12 - adcs x9,x9,x13 - adcs x10,x10,x14 - adc x15,xzr,xzr - - // Load polynomial - adr x2,Lpoly - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Backup Addition - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Sub polynomial - subs x3,x3,x11 - sbcs x4,x4,x12 - sbcs x5,x5,x13 - sbcs x6,x6,x14 - sbcs x15,x15,xzr - - // Select based on carry - csel x7,x7,x3,cc - csel x8,x8,x4,cc - csel x9,x9,x5,cc - csel x10,x10,x6,cc - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret - - -// void ecp_sm2p256_sub(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); -.globl _ecp_sm2p256_sub - -.align 5 -_ecp_sm2p256_sub: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Subtraction - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbcs x10,x10,x14 - sbc x15,xzr,xzr - - // Load polynomial - adr x2,Lpoly - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Backup subtraction - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Add polynomial - adds x3,x3,x11 - adcs x4,x4,x12 - adcs x5,x5,x13 - adcs x6,x6,x14 - tst x15,x15 - - // Select based on carry - csel x7,x7,x3,eq - csel x8,x8,x4,eq - csel x9,x9,x5,eq - csel x10,x10,x6,eq - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret - - -// void ecp_sm2p256_sub_mod_ord(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); -.globl _ecp_sm2p256_sub_mod_ord - -.align 5 -_ecp_sm2p256_sub_mod_ord: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Subtraction - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbcs x10,x10,x14 - sbc x15,xzr,xzr - - // Load polynomial - adr x2,Lord - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Backup subtraction - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Add polynomial - adds x3,x3,x11 - adcs x4,x4,x12 - adcs x5,x5,x13 - adcs x6,x6,x14 - tst x15,x15 - - // Select based on carry - csel x7,x7,x3,eq - csel x8,x8,x4,eq - csel x9,x9,x5,eq - csel x10,x10,x6,eq - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret - - -.macro RDC - // a = | s7 | ... | s0 |, where si are 64-bit quantities - // = |a15|a14| ... |a1|a0|, where ai are 32-bit quantities - // | s7 | s6 | s5 | s4 | - // | a15 | a14 | a13 | a12 | a11 | a10 | a9 | a8 | - // | s3 | s2 | s1 | s0 | - // | a7 | a6 | a5 | a4 | a3 | a2 | a1 | a0 | - // ================================================= - // | a8 | a11 | a10 | a9 | a8 | 0 | s4 | (+) - // | a9 | a15 | s6 | a11 | 0 | a10 | a9 | (+) - // | a10 | 0 | a14 | a13 | a12 | 0 | s5 | (+) - // | a11 | 0 | s7 | a13 | 0 | a12 | a11 | (+) - // | a12 | 0 | s7 | a13 | 0 | s6 | (+) - // | a12 | 0 | 0 | a15 | a14 | 0 | a14 | a13 | (+) - // | a13 | 0 | 0 | 0 | a15 | 0 | a14 | a13 | (+) - // | a13 | 0 | 0 | 0 | 0 | 0 | s7 | (+) - // | a14 | 0 | 0 | 0 | 0 | 0 | s7 | (+) - // | a14 | 0 | 0 | 0 | 0 | 0 | 0 | a15 | (+) - // | a15 | 0 | 0 | 0 | 0 | 0 | 0 | a15 | (+) - // | a15 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | (+) - // | s7 | 0 | 0 | 0 | 0 | 0 | 0 | (+) - // | 0 | 0 | 0 | 0 | 0 | a8 | 0 | 0 | (-) - // | 0 | 0 | 0 | 0 | 0 | a9 | 0 | 0 | (-) - // | 0 | 0 | 0 | 0 | 0 | a13 | 0 | 0 | (-) - // | 0 | 0 | 0 | 0 | 0 | a14 | 0 | 0 | (-) - // | U[7]| U[6]| U[5]| U[4]| U[3]| U[2]| U[1]| U[0]| - // | V[3] | V[2] | V[1] | V[0] | - - // 1. 64-bit addition - // t2=s6+s7+s7 - adds x5,x13,x14 - adcs x4,xzr,xzr - adds x5,x5,x14 - adcs x4,x4,xzr - // t3=s4+s5+t2 - adds x6,x11,x5 - adcs x15,x4,xzr - adds x6,x6,x12 - adcs x15,x15,xzr - // sum - adds x7,x7,x6 - adcs x8,x8,x15 - adcs x9,x9,x5 - adcs x10,x10,x14 - adcs x3,xzr,xzr - adds x10,x10,x4 - adcs x3,x3,xzr - - stp x7,x8,[sp,#32] - stp x9,x10,[sp,#48] - - // 2. 64-bit to 32-bit spread - mov x4,#0xffffffff - mov x7,x11 - mov x8,x12 - mov x9,x13 - mov x10,x14 - and x7,x7,x4 // a8 - and x8,x8,x4 // a10 - and x9,x9,x4 // a12 - and x10,x10,x4 // a14 - lsr x11,x11,#32 // a9 - lsr x12,x12,#32 // a11 - lsr x13,x13,#32 // a13 - lsr x14,x14,#32 // a15 - - // 3. 32-bit addition - add x4,x10,x9 // t1 <- a12 + a14 - add x5,x14,x13 // t2 <- a13 + a15 - add x6,x7,x11 // t3 <- a8 + a9 - add x15,x10,x8 // t4 <- a10 + a14 - add x14,x14,x12 // a15 <- a11 + a15 - add x9,x5,x4 // a12 <- a12 + a13 + a14 + a15 - add x8,x8,x9 // a10 <- a10 + a12 + a13 + a14 + a15 - add x8,x8,x9 // a10 <- a10 + 2*(a12 + a13 + a14 + a15) - add x8,x8,x6 // a10 <- a8 + a9 + a10 + 2*(a12 + a13 + a14 + a15) - add x8,x8,x12 // a10 <- a8 + a9 + a10 + a11 + 2*(a12 + a13 + a14 + a15) - add x9,x9,x13 // a12 <- a12 + 2*a13 + a14 + a15 - add x9,x9,x12 // a12 <- a11 + a12 + 2*a13 + a14 + a15 - add x9,x9,x7 // a12 <- a8 + a11 + a12 + 2*a13 + a14 + a15 - add x6,x6,x10 // t3 <- a8 + a9 + a14 - add x6,x6,x13 // t3 <- a8 + a9 + a13 + a14 - add x11,x11,x5 // a9 <- a9 + a13 + a15 - add x12,x12,x11 // a11 <- a9 + a11 + a13 + a15 - add x12,x12,x5 // a11 <- a9 + a11 + 2*(a13 + a15) - add x4,x4,x15 // t1 <- a10 + a12 + 2*a14 - - // U[0] s5 a9 + a11 + 2*(a13 + a15) - // U[1] t1 a10 + a12 + 2*a14 - // U[2] -t3 a8 + a9 + a13 + a14 - // U[3] s2 a8 + a11 + a12 + 2*a13 + a14 + a15 - // U[4] s4 a9 + a13 + a15 - // U[5] t4 a10 + a14 - // U[6] s7 a11 + a15 - // U[7] s1 a8 + a9 + a10 + a11 + 2*(a12 + a13 + a14 + a15) - - // 4. 32-bit to 64-bit - lsl x7,x4,#32 - extr x4,x9,x4,#32 - extr x9,x15,x9,#32 - extr x15,x8,x15,#32 - lsr x8,x8,#32 - - // 5. 64-bit addition - adds x12,x12,x7 - adcs x4,x4,xzr - adcs x11,x11,x9 - adcs x14,x14,x15 - adcs x3,x3,x8 - - // V[0] s5 - // V[1] t1 - // V[2] s4 - // V[3] s7 - // carry t0 - // sub t3 - - // 5. Process s0-s3 - ldp x7,x8,[sp,#32] - ldp x9,x10,[sp,#48] - // add with V0-V3 - adds x7,x7,x12 - adcs x8,x8,x4 - adcs x9,x9,x11 - adcs x10,x10,x14 - adcs x3,x3,xzr - // sub with t3 - subs x8,x8,x6 - sbcs x9,x9,xzr - sbcs x10,x10,xzr - sbcs x3,x3,xzr - - // 6. MOD - // First Mod - lsl x4,x3,#32 - subs x5,x4,x3 - - adds x7,x7,x3 - adcs x8,x8,x5 - adcs x9,x9,xzr - adcs x10,x10,x4 - - // Last Mod - // return y - p if y > p else y - mov x11,x7 - mov x12,x8 - mov x13,x9 - mov x14,x10 - - adr x3,Lpoly - ldp x4,x5,[x3] - ldp x6,x15,[x3,#16] - - adcs x16,xzr,xzr - - subs x7,x7,x4 - sbcs x8,x8,x5 - sbcs x9,x9,x6 - sbcs x10,x10,x15 - sbcs x16,x16,xzr - - csel x7,x7,x11,cs - csel x8,x8,x12,cs - csel x9,x9,x13,cs - csel x10,x10,x14,cs - -.endm - -// void ecp_sm2p256_mul(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -.globl _ecp_sm2p256_mul - -.align 5 -_ecp_sm2p256_mul: - AARCH64_SIGN_LINK_REGISTER - // Store scalar registers - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - stp x16,x17,[sp,#16] - stp x19,x20,[sp,#64] - - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - -// ### multiplication ### - // ======================== - // s3 s2 s1 s0 - // * s7 s6 s5 s4 - // ------------------------ - // + s0 s0 s0 s0 - // * * * * - // s7 s6 s5 s4 - // s1 s1 s1 s1 - // * * * * - // s7 s6 s5 s4 - // s2 s2 s2 s2 - // * * * * - // s7 s6 s5 s4 - // s3 s3 s3 s3 - // * * * * - // s7 s6 s5 s4 - // ------------------------ - // s7 s6 s5 s4 s3 s2 s1 s0 - // ======================== - -// ### s0*s4 ### - mul x16,x7,x11 - umulh x5,x7,x11 - -// ### s1*s4 + s0*s5 ### - mul x3,x8,x11 - umulh x4,x8,x11 - adds x5,x5,x3 - adcs x6,x4,xzr - - mul x3,x7,x12 - umulh x4,x7,x12 - adds x5,x5,x3 - adcs x6,x6,x4 - adcs x15,xzr,xzr - -// ### s2*s4 + s1*s5 + s0*s6 ### - mul x3,x9,x11 - umulh x4,x9,x11 - adds x6,x6,x3 - adcs x15,x15,x4 - - mul x3,x8,x12 - umulh x4,x8,x12 - adds x6,x6,x3 - adcs x15,x15,x4 - adcs x17,xzr,xzr - - mul x3,x7,x13 - umulh x4,x7,x13 - adds x6,x6,x3 - adcs x15,x15,x4 - adcs x17,x17,xzr - -// ### s3*s4 + s2*s5 + s1*s6 + s0*s7 ### - mul x3,x10,x11 - umulh x4,x10,x11 - adds x15,x15,x3 - adcs x17,x17,x4 - adcs x19,xzr,xzr - - mul x3,x9,x12 - umulh x4,x9,x12 - adds x15,x15,x3 - adcs x17,x17,x4 - adcs x19,x19,xzr - - mul x3,x8,x13 - umulh x4,x8,x13 - adds x15,x15,x3 - adcs x17,x17,x4 - adcs x19,x19,xzr - - mul x3,x7,x14 - umulh x4,x7,x14 - adds x15,x15,x3 - adcs x17,x17,x4 - adcs x19,x19,xzr - -// ### s3*s5 + s2*s6 + s1*s7 ### - mul x3,x10,x12 - umulh x4,x10,x12 - adds x17,x17,x3 - adcs x19,x19,x4 - adcs x20,xzr,xzr - - mul x3,x9,x13 - umulh x4,x9,x13 - adds x17,x17,x3 - adcs x19,x19,x4 - adcs x20,x20,xzr - - mul x3,x8,x14 - umulh x4,x8,x14 - adds x11,x17,x3 - adcs x19,x19,x4 - adcs x20,x20,xzr - -// ### s3*s6 + s2*s7 ### - mul x3,x10,x13 - umulh x4,x10,x13 - adds x19,x19,x3 - adcs x20,x20,x4 - adcs x17,xzr,xzr - - mul x3,x9,x14 - umulh x4,x9,x14 - adds x12,x19,x3 - adcs x20,x20,x4 - adcs x17,x17,xzr - -// ### s3*s7 ### - mul x3,x10,x14 - umulh x4,x10,x14 - adds x13,x20,x3 - adcs x14,x17,x4 - - mov x7,x16 - mov x8,x5 - mov x9,x6 - mov x10,x15 - - // result of mul: s7 s6 s5 s4 s3 s2 s1 s0 - -// ### Reduction ### - RDC - - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - // Restore scalar registers - ldp x16,x17,[sp,#16] - ldp x19,x20,[sp,#64] - ldp x29,x30,[sp],#80 - - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_sm2p256_sqr(BN_ULONG *r, const BN_ULONG *a); -.globl _ecp_sm2p256_sqr - -.align 5 - -_ecp_sm2p256_sqr: - AARCH64_SIGN_LINK_REGISTER - // Store scalar registers - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - stp x16,x17,[sp,#16] - stp x19,x20,[sp,#64] - - // Load inputs - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - -// ### square ### - // ======================== - // s7 s6 s5 s4 - // * s7 s6 s5 s4 - // ------------------------ - // + s4 s4 s4 s4 - // * * * * - // s7 s6 s5 s4 - // s5 s5 s5 s5 - // * * * * - // s7 s6 s5 s4 - // s6 s6 s6 s6 - // * * * * - // s7 s6 s5 s4 - // s7 s7 s7 s7 - // * * * * - // s7 s6 s5 s4 - // ------------------------ - // s7 s6 s5 s4 s3 s2 s1 s0 - // ======================== - -// ### s4*s5 ### - mul x8,x11,x12 - umulh x9,x11,x12 - -// ### s4*s6 ### - mul x3,x13,x11 - umulh x10,x13,x11 - adds x9,x9,x3 - adcs x10,x10,xzr - -// ### s4*s7 + s5*s6 ### - mul x3,x14,x11 - umulh x4,x14,x11 - adds x10,x10,x3 - adcs x7,x4,xzr - - mul x3,x13,x12 - umulh x4,x13,x12 - adds x10,x10,x3 - adcs x7,x7,x4 - adcs x5,xzr,xzr - -// ### s5*s7 ### - mul x3,x14,x12 - umulh x4,x14,x12 - adds x7,x7,x3 - adcs x5,x5,x4 - -// ### s6*s7 ### - mul x3,x14,x13 - umulh x4,x14,x13 - adds x5,x5,x3 - adcs x6,x4,xzr - -// ### 2*(t3,t2,s0,s3,s2,s1) ### - adds x8,x8,x8 - adcs x9,x9,x9 - adcs x10,x10,x10 - adcs x7,x7,x7 - adcs x5,x5,x5 - adcs x6,x6,x6 - adcs x15,xzr,xzr - -// ### s4*s4 ### - mul x16,x11,x11 - umulh x17,x11,x11 - -// ### s5*s5 ### - mul x11,x12,x12 - umulh x12,x12,x12 - -// ### s6*s6 ### - mul x3,x13,x13 - umulh x4,x13,x13 - -// ### s7*s7 ### - mul x19,x14,x14 - umulh x20,x14,x14 - - adds x8,x8,x17 - adcs x9,x9,x11 - adcs x10,x10,x12 - adcs x7,x7,x3 - adcs x5,x5,x4 - adcs x6,x6,x19 - adcs x15,x15,x20 - - mov x11,x7 - mov x7,x16 - mov x12,x5 - mov x13,x6 - mov x14,x15 - - // result of mul: s7 s6 s5 s4 s3 s2 s1 s0 - -// ### Reduction ### - RDC - - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - // Restore scalar registers - ldp x16,x17,[sp,#16] - ldp x19,x20,[sp,#64] - ldp x29,x30,[sp],#80 - - AARCH64_VALIDATE_LINK_REGISTER - ret - diff --git a/openssl/src/crypto/ec/gen/linux_arm64/ecp_nistz256-armv8.S b/openssl/src/crypto/ec/gen/linux_arm64/ecp_nistz256-armv8.S index fe01319a3..e7230f81f 100644 --- a/openssl/src/crypto/ec/gen/linux_arm64/ecp_nistz256-armv8.S +++ b/openssl/src/crypto/ec/gen/linux_arm64/ecp_nistz256-armv8.S @@ -2395,7 +2395,7 @@ ecp_nistz256_precomputed: .type ecp_nistz256_to_mont,%function .align 6 ecp_nistz256_to_mont: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2411,7 +2411,7 @@ ecp_nistz256_to_mont: ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont @@ -2420,7 +2420,7 @@ ecp_nistz256_to_mont: .type ecp_nistz256_from_mont,%function .align 4 ecp_nistz256_from_mont: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2436,7 +2436,7 @@ ecp_nistz256_from_mont: ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont @@ -2446,7 +2446,7 @@ ecp_nistz256_from_mont: .type ecp_nistz256_mul_mont,%function .align 4 ecp_nistz256_mul_mont: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2461,7 +2461,7 @@ ecp_nistz256_mul_mont: ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont @@ -2470,7 +2470,7 @@ ecp_nistz256_mul_mont: .type ecp_nistz256_sqr_mont,%function .align 4 ecp_nistz256_sqr_mont: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -2484,7 +2484,7 @@ ecp_nistz256_sqr_mont: ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont @@ -2494,7 +2494,7 @@ ecp_nistz256_sqr_mont: .type ecp_nistz256_add,%function .align 4 ecp_nistz256_add: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2508,7 +2508,7 @@ ecp_nistz256_add: bl __ecp_nistz256_add ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_add,.-ecp_nistz256_add @@ -2517,7 +2517,7 @@ ecp_nistz256_add: .type ecp_nistz256_div_by_2,%function .align 4 ecp_nistz256_div_by_2: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2529,7 +2529,7 @@ ecp_nistz256_div_by_2: bl __ecp_nistz256_div_by_2 ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2 @@ -2538,7 +2538,7 @@ ecp_nistz256_div_by_2: .type ecp_nistz256_mul_by_2,%function .align 4 ecp_nistz256_mul_by_2: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2554,7 +2554,7 @@ ecp_nistz256_mul_by_2: bl __ecp_nistz256_add // ret = a+a // 2*a ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2 @@ -2563,7 +2563,7 @@ ecp_nistz256_mul_by_2: .type ecp_nistz256_mul_by_3,%function .align 4 ecp_nistz256_mul_by_3: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2590,7 +2590,7 @@ ecp_nistz256_mul_by_3: bl __ecp_nistz256_add // ret += a // 2*a+a=3*a ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3 @@ -2600,7 +2600,7 @@ ecp_nistz256_mul_by_3: .type ecp_nistz256_sub,%function .align 4 ecp_nistz256_sub: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2612,7 +2612,7 @@ ecp_nistz256_sub: bl __ecp_nistz256_sub_from ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_sub,.-ecp_nistz256_sub @@ -2621,7 +2621,7 @@ ecp_nistz256_sub: .type ecp_nistz256_neg,%function .align 4 ecp_nistz256_neg: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -2636,7 +2636,7 @@ ecp_nistz256_neg: bl __ecp_nistz256_sub_from ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_neg,.-ecp_nistz256_neg @@ -3014,7 +3014,7 @@ __ecp_nistz256_div_by_2: .type ecp_nistz256_point_double,%function .align 5 ecp_nistz256_point_double: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -3149,14 +3149,14 @@ ecp_nistz256_point_double: ldp x19,x20,[x29,#16] ldp x21,x22,[x29,#32] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_point_double,.-ecp_nistz256_point_double .globl ecp_nistz256_point_add .type ecp_nistz256_point_add,%function .align 5 ecp_nistz256_point_add: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -3403,14 +3403,14 @@ ecp_nistz256_point_add: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_point_add,.-ecp_nistz256_point_add .globl ecp_nistz256_point_add_affine .type ecp_nistz256_point_add_affine,%function .align 5 ecp_nistz256_point_add_affine: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -3609,7 +3609,7 @@ ecp_nistz256_point_add_affine: ldp x23,x24,[x29,#48] ldp x25,x26,[x29,#64] ldp x29,x30,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine //////////////////////////////////////////////////////////////////////// @@ -3619,8 +3619,6 @@ ecp_nistz256_point_add_affine: .type ecp_nistz256_ord_mul_mont,%function .align 4 ecp_nistz256_ord_mul_mont: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-64]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -3829,8 +3827,6 @@ ecp_nistz256_ord_mul_mont: .type ecp_nistz256_ord_sqr_mont,%function .align 4 ecp_nistz256_ord_sqr_mont: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-64]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -4019,8 +4015,6 @@ ecp_nistz256_ord_sqr_mont: .type ecp_nistz256_scatter_w5,%function .align 4 ecp_nistz256_scatter_w5: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -4083,8 +4077,6 @@ ecp_nistz256_scatter_w5: .type ecp_nistz256_gather_w5,%function .align 4 ecp_nistz256_gather_w5: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -4162,8 +4154,6 @@ ecp_nistz256_gather_w5: .type ecp_nistz256_scatter_w7,%function .align 4 ecp_nistz256_scatter_w7: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -4208,8 +4198,6 @@ ecp_nistz256_scatter_w7: .type ecp_nistz256_gather_w7,%function .align 4 ecp_nistz256_gather_w7: - AARCH64_VALID_CALL_TARGET - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/openssl/src/crypto/ec/gen/linux_arm64/ecp_sm2p256-armv8.S b/openssl/src/crypto/ec/gen/linux_arm64/ecp_sm2p256-armv8.S deleted file mode 100644 index 844ff68c6..000000000 --- a/openssl/src/crypto/ec/gen/linux_arm64/ecp_sm2p256-armv8.S +++ /dev/null @@ -1,826 +0,0 @@ -#include "arm_arch.h" -.arch armv8-a -.text - -.align 5 -// The polynomial p -.Lpoly: -.quad 0xffffffffffffffff,0xffffffff00000000,0xffffffffffffffff,0xfffffffeffffffff -// The order of polynomial n -.Lord: -.quad 0x53bbf40939d54123,0x7203df6b21c6052b,0xffffffffffffffff,0xfffffffeffffffff -// (p + 1) / 2 -.Lpoly_div_2: -.quad 0x8000000000000000,0xffffffff80000000,0xffffffffffffffff,0x7fffffff7fffffff -// (n + 1) / 2 -.Lord_div_2: -.quad 0xa9ddfa049ceaa092,0xb901efb590e30295,0xffffffffffffffff,0x7fffffff7fffffff - -// void bn_rshift1(BN_ULONG *a); -.globl bn_rshift1 -.type bn_rshift1,%function -.align 5 -bn_rshift1: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x0] - ldp x9,x10,[x0,#16] - - // Right shift - extr x7,x8,x7,#1 - extr x8,x9,x8,#1 - extr x9,x10,x9,#1 - lsr x10,x10,#1 - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - ret -.size bn_rshift1,.-bn_rshift1 - -// void bn_sub(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -.globl bn_sub -.type bn_sub,%function -.align 5 -bn_sub: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Subtraction - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbc x10,x10,x14 - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - ret -.size bn_sub,.-bn_sub - -// void ecp_sm2p256_div_by_2(BN_ULONG *r,const BN_ULONG *a); -.globl ecp_sm2p256_div_by_2 -.type ecp_sm2p256_div_by_2,%function -.align 5 -ecp_sm2p256_div_by_2: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - - // Save the least significant bit - mov x3,x7 - - // Right shift 1 - extr x7,x8,x7,#1 - extr x8,x9,x8,#1 - extr x9,x10,x9,#1 - lsr x10,x10,#1 - - // Load mod - adr x2,.Lpoly_div_2 - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Parity check - tst x3,#1 - csel x11,xzr,x11,eq - csel x12,xzr,x12,eq - csel x13,xzr,x13,eq - csel x14,xzr,x14,eq - - // Add - adds x7,x7,x11 - adcs x8,x8,x12 - adcs x9,x9,x13 - adc x10,x10,x14 - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret -.size ecp_sm2p256_div_by_2,.-ecp_sm2p256_div_by_2 - -// void ecp_sm2p256_div_by_2_mod_ord(BN_ULONG *r,const BN_ULONG *a); -.globl ecp_sm2p256_div_by_2_mod_ord -.type ecp_sm2p256_div_by_2_mod_ord,%function -.align 5 -ecp_sm2p256_div_by_2_mod_ord: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - - // Save the least significant bit - mov x3,x7 - - // Right shift 1 - extr x7,x8,x7,#1 - extr x8,x9,x8,#1 - extr x9,x10,x9,#1 - lsr x10,x10,#1 - - // Load mod - adr x2,.Lord_div_2 - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Parity check - tst x3,#1 - csel x11,xzr,x11,eq - csel x12,xzr,x12,eq - csel x13,xzr,x13,eq - csel x14,xzr,x14,eq - - // Add - adds x7,x7,x11 - adcs x8,x8,x12 - adcs x9,x9,x13 - adc x10,x10,x14 - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret -.size ecp_sm2p256_div_by_2_mod_ord,.-ecp_sm2p256_div_by_2_mod_ord - -// void ecp_sm2p256_mul_by_3(BN_ULONG *r,const BN_ULONG *a); -.globl ecp_sm2p256_mul_by_3 -.type ecp_sm2p256_mul_by_3,%function -.align 5 -ecp_sm2p256_mul_by_3: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - - // 2*a - adds x7,x7,x7 - adcs x8,x8,x8 - adcs x9,x9,x9 - adcs x10,x10,x10 - adcs x15,xzr,xzr - - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Sub polynomial - adr x2,.Lpoly - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbcs x10,x10,x14 - sbcs x15,x15,xzr - - csel x7,x7,x3,cs - csel x8,x8,x4,cs - csel x9,x9,x5,cs - csel x10,x10,x6,cs - eor x15,x15,x15 - - // 3*a - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - adds x7,x7,x11 - adcs x8,x8,x12 - adcs x9,x9,x13 - adcs x10,x10,x14 - adcs x15,xzr,xzr - - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Sub polynomial - adr x2,.Lpoly - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbcs x10,x10,x14 - sbcs x15,x15,xzr - - csel x7,x7,x3,cs - csel x8,x8,x4,cs - csel x9,x9,x5,cs - csel x10,x10,x6,cs - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - ret -.size ecp_sm2p256_mul_by_3,.-ecp_sm2p256_mul_by_3 - -// void ecp_sm2p256_add(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); -.globl ecp_sm2p256_add -.type ecp_sm2p256_add,%function -.align 5 -ecp_sm2p256_add: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Addition - adds x7,x7,x11 - adcs x8,x8,x12 - adcs x9,x9,x13 - adcs x10,x10,x14 - adc x15,xzr,xzr - - // Load polynomial - adr x2,.Lpoly - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Backup Addition - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Sub polynomial - subs x3,x3,x11 - sbcs x4,x4,x12 - sbcs x5,x5,x13 - sbcs x6,x6,x14 - sbcs x15,x15,xzr - - // Select based on carry - csel x7,x7,x3,cc - csel x8,x8,x4,cc - csel x9,x9,x5,cc - csel x10,x10,x6,cc - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret -.size ecp_sm2p256_add,.-ecp_sm2p256_add - -// void ecp_sm2p256_sub(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); -.globl ecp_sm2p256_sub -.type ecp_sm2p256_sub,%function -.align 5 -ecp_sm2p256_sub: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Subtraction - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbcs x10,x10,x14 - sbc x15,xzr,xzr - - // Load polynomial - adr x2,.Lpoly - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Backup subtraction - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Add polynomial - adds x3,x3,x11 - adcs x4,x4,x12 - adcs x5,x5,x13 - adcs x6,x6,x14 - tst x15,x15 - - // Select based on carry - csel x7,x7,x3,eq - csel x8,x8,x4,eq - csel x9,x9,x5,eq - csel x10,x10,x6,eq - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret -.size ecp_sm2p256_sub,.-ecp_sm2p256_sub - -// void ecp_sm2p256_sub_mod_ord(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); -.globl ecp_sm2p256_sub_mod_ord -.type ecp_sm2p256_sub_mod_ord,%function -.align 5 -ecp_sm2p256_sub_mod_ord: - AARCH64_VALID_CALL_TARGET - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Subtraction - subs x7,x7,x11 - sbcs x8,x8,x12 - sbcs x9,x9,x13 - sbcs x10,x10,x14 - sbc x15,xzr,xzr - - // Load polynomial - adr x2,.Lord - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - - // Backup subtraction - mov x3,x7 - mov x4,x8 - mov x5,x9 - mov x6,x10 - - // Add polynomial - adds x3,x3,x11 - adcs x4,x4,x12 - adcs x5,x5,x13 - adcs x6,x6,x14 - tst x15,x15 - - // Select based on carry - csel x7,x7,x3,eq - csel x8,x8,x4,eq - csel x9,x9,x5,eq - csel x10,x10,x6,eq - - // Store results - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - ret -.size ecp_sm2p256_sub_mod_ord,.-ecp_sm2p256_sub_mod_ord - -.macro RDC - // a = | s7 | ... | s0 |, where si are 64-bit quantities - // = |a15|a14| ... |a1|a0|, where ai are 32-bit quantities - // | s7 | s6 | s5 | s4 | - // | a15 | a14 | a13 | a12 | a11 | a10 | a9 | a8 | - // | s3 | s2 | s1 | s0 | - // | a7 | a6 | a5 | a4 | a3 | a2 | a1 | a0 | - // ================================================= - // | a8 | a11 | a10 | a9 | a8 | 0 | s4 | (+) - // | a9 | a15 | s6 | a11 | 0 | a10 | a9 | (+) - // | a10 | 0 | a14 | a13 | a12 | 0 | s5 | (+) - // | a11 | 0 | s7 | a13 | 0 | a12 | a11 | (+) - // | a12 | 0 | s7 | a13 | 0 | s6 | (+) - // | a12 | 0 | 0 | a15 | a14 | 0 | a14 | a13 | (+) - // | a13 | 0 | 0 | 0 | a15 | 0 | a14 | a13 | (+) - // | a13 | 0 | 0 | 0 | 0 | 0 | s7 | (+) - // | a14 | 0 | 0 | 0 | 0 | 0 | s7 | (+) - // | a14 | 0 | 0 | 0 | 0 | 0 | 0 | a15 | (+) - // | a15 | 0 | 0 | 0 | 0 | 0 | 0 | a15 | (+) - // | a15 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | (+) - // | s7 | 0 | 0 | 0 | 0 | 0 | 0 | (+) - // | 0 | 0 | 0 | 0 | 0 | a8 | 0 | 0 | (-) - // | 0 | 0 | 0 | 0 | 0 | a9 | 0 | 0 | (-) - // | 0 | 0 | 0 | 0 | 0 | a13 | 0 | 0 | (-) - // | 0 | 0 | 0 | 0 | 0 | a14 | 0 | 0 | (-) - // | U[7]| U[6]| U[5]| U[4]| U[3]| U[2]| U[1]| U[0]| - // | V[3] | V[2] | V[1] | V[0] | - - // 1. 64-bit addition - // t2=s6+s7+s7 - adds x5,x13,x14 - adcs x4,xzr,xzr - adds x5,x5,x14 - adcs x4,x4,xzr - // t3=s4+s5+t2 - adds x6,x11,x5 - adcs x15,x4,xzr - adds x6,x6,x12 - adcs x15,x15,xzr - // sum - adds x7,x7,x6 - adcs x8,x8,x15 - adcs x9,x9,x5 - adcs x10,x10,x14 - adcs x3,xzr,xzr - adds x10,x10,x4 - adcs x3,x3,xzr - - stp x7,x8,[sp,#32] - stp x9,x10,[sp,#48] - - // 2. 64-bit to 32-bit spread - mov x4,#0xffffffff - mov x7,x11 - mov x8,x12 - mov x9,x13 - mov x10,x14 - and x7,x7,x4 // a8 - and x8,x8,x4 // a10 - and x9,x9,x4 // a12 - and x10,x10,x4 // a14 - lsr x11,x11,#32 // a9 - lsr x12,x12,#32 // a11 - lsr x13,x13,#32 // a13 - lsr x14,x14,#32 // a15 - - // 3. 32-bit addition - add x4,x10,x9 // t1 <- a12 + a14 - add x5,x14,x13 // t2 <- a13 + a15 - add x6,x7,x11 // t3 <- a8 + a9 - add x15,x10,x8 // t4 <- a10 + a14 - add x14,x14,x12 // a15 <- a11 + a15 - add x9,x5,x4 // a12 <- a12 + a13 + a14 + a15 - add x8,x8,x9 // a10 <- a10 + a12 + a13 + a14 + a15 - add x8,x8,x9 // a10 <- a10 + 2*(a12 + a13 + a14 + a15) - add x8,x8,x6 // a10 <- a8 + a9 + a10 + 2*(a12 + a13 + a14 + a15) - add x8,x8,x12 // a10 <- a8 + a9 + a10 + a11 + 2*(a12 + a13 + a14 + a15) - add x9,x9,x13 // a12 <- a12 + 2*a13 + a14 + a15 - add x9,x9,x12 // a12 <- a11 + a12 + 2*a13 + a14 + a15 - add x9,x9,x7 // a12 <- a8 + a11 + a12 + 2*a13 + a14 + a15 - add x6,x6,x10 // t3 <- a8 + a9 + a14 - add x6,x6,x13 // t3 <- a8 + a9 + a13 + a14 - add x11,x11,x5 // a9 <- a9 + a13 + a15 - add x12,x12,x11 // a11 <- a9 + a11 + a13 + a15 - add x12,x12,x5 // a11 <- a9 + a11 + 2*(a13 + a15) - add x4,x4,x15 // t1 <- a10 + a12 + 2*a14 - - // U[0] s5 a9 + a11 + 2*(a13 + a15) - // U[1] t1 a10 + a12 + 2*a14 - // U[2] -t3 a8 + a9 + a13 + a14 - // U[3] s2 a8 + a11 + a12 + 2*a13 + a14 + a15 - // U[4] s4 a9 + a13 + a15 - // U[5] t4 a10 + a14 - // U[6] s7 a11 + a15 - // U[7] s1 a8 + a9 + a10 + a11 + 2*(a12 + a13 + a14 + a15) - - // 4. 32-bit to 64-bit - lsl x7,x4,#32 - extr x4,x9,x4,#32 - extr x9,x15,x9,#32 - extr x15,x8,x15,#32 - lsr x8,x8,#32 - - // 5. 64-bit addition - adds x12,x12,x7 - adcs x4,x4,xzr - adcs x11,x11,x9 - adcs x14,x14,x15 - adcs x3,x3,x8 - - // V[0] s5 - // V[1] t1 - // V[2] s4 - // V[3] s7 - // carry t0 - // sub t3 - - // 5. Process s0-s3 - ldp x7,x8,[sp,#32] - ldp x9,x10,[sp,#48] - // add with V0-V3 - adds x7,x7,x12 - adcs x8,x8,x4 - adcs x9,x9,x11 - adcs x10,x10,x14 - adcs x3,x3,xzr - // sub with t3 - subs x8,x8,x6 - sbcs x9,x9,xzr - sbcs x10,x10,xzr - sbcs x3,x3,xzr - - // 6. MOD - // First Mod - lsl x4,x3,#32 - subs x5,x4,x3 - - adds x7,x7,x3 - adcs x8,x8,x5 - adcs x9,x9,xzr - adcs x10,x10,x4 - - // Last Mod - // return y - p if y > p else y - mov x11,x7 - mov x12,x8 - mov x13,x9 - mov x14,x10 - - adr x3,.Lpoly - ldp x4,x5,[x3] - ldp x6,x15,[x3,#16] - - adcs x16,xzr,xzr - - subs x7,x7,x4 - sbcs x8,x8,x5 - sbcs x9,x9,x6 - sbcs x10,x10,x15 - sbcs x16,x16,xzr - - csel x7,x7,x11,cs - csel x8,x8,x12,cs - csel x9,x9,x13,cs - csel x10,x10,x14,cs - -.endm - -// void ecp_sm2p256_mul(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); -.globl ecp_sm2p256_mul -.type ecp_sm2p256_mul,%function -.align 5 -ecp_sm2p256_mul: - AARCH64_SIGN_LINK_REGISTER - // Store scalar registers - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - stp x16,x17,[sp,#16] - stp x19,x20,[sp,#64] - - // Load inputs - ldp x7,x8,[x1] - ldp x9,x10,[x1,#16] - ldp x11,x12,[x2] - ldp x13,x14,[x2,#16] - -// ### multiplication ### - // ======================== - // s3 s2 s1 s0 - // * s7 s6 s5 s4 - // ------------------------ - // + s0 s0 s0 s0 - // * * * * - // s7 s6 s5 s4 - // s1 s1 s1 s1 - // * * * * - // s7 s6 s5 s4 - // s2 s2 s2 s2 - // * * * * - // s7 s6 s5 s4 - // s3 s3 s3 s3 - // * * * * - // s7 s6 s5 s4 - // ------------------------ - // s7 s6 s5 s4 s3 s2 s1 s0 - // ======================== - -// ### s0*s4 ### - mul x16,x7,x11 - umulh x5,x7,x11 - -// ### s1*s4 + s0*s5 ### - mul x3,x8,x11 - umulh x4,x8,x11 - adds x5,x5,x3 - adcs x6,x4,xzr - - mul x3,x7,x12 - umulh x4,x7,x12 - adds x5,x5,x3 - adcs x6,x6,x4 - adcs x15,xzr,xzr - -// ### s2*s4 + s1*s5 + s0*s6 ### - mul x3,x9,x11 - umulh x4,x9,x11 - adds x6,x6,x3 - adcs x15,x15,x4 - - mul x3,x8,x12 - umulh x4,x8,x12 - adds x6,x6,x3 - adcs x15,x15,x4 - adcs x17,xzr,xzr - - mul x3,x7,x13 - umulh x4,x7,x13 - adds x6,x6,x3 - adcs x15,x15,x4 - adcs x17,x17,xzr - -// ### s3*s4 + s2*s5 + s1*s6 + s0*s7 ### - mul x3,x10,x11 - umulh x4,x10,x11 - adds x15,x15,x3 - adcs x17,x17,x4 - adcs x19,xzr,xzr - - mul x3,x9,x12 - umulh x4,x9,x12 - adds x15,x15,x3 - adcs x17,x17,x4 - adcs x19,x19,xzr - - mul x3,x8,x13 - umulh x4,x8,x13 - adds x15,x15,x3 - adcs x17,x17,x4 - adcs x19,x19,xzr - - mul x3,x7,x14 - umulh x4,x7,x14 - adds x15,x15,x3 - adcs x17,x17,x4 - adcs x19,x19,xzr - -// ### s3*s5 + s2*s6 + s1*s7 ### - mul x3,x10,x12 - umulh x4,x10,x12 - adds x17,x17,x3 - adcs x19,x19,x4 - adcs x20,xzr,xzr - - mul x3,x9,x13 - umulh x4,x9,x13 - adds x17,x17,x3 - adcs x19,x19,x4 - adcs x20,x20,xzr - - mul x3,x8,x14 - umulh x4,x8,x14 - adds x11,x17,x3 - adcs x19,x19,x4 - adcs x20,x20,xzr - -// ### s3*s6 + s2*s7 ### - mul x3,x10,x13 - umulh x4,x10,x13 - adds x19,x19,x3 - adcs x20,x20,x4 - adcs x17,xzr,xzr - - mul x3,x9,x14 - umulh x4,x9,x14 - adds x12,x19,x3 - adcs x20,x20,x4 - adcs x17,x17,xzr - -// ### s3*s7 ### - mul x3,x10,x14 - umulh x4,x10,x14 - adds x13,x20,x3 - adcs x14,x17,x4 - - mov x7,x16 - mov x8,x5 - mov x9,x6 - mov x10,x15 - - // result of mul: s7 s6 s5 s4 s3 s2 s1 s0 - -// ### Reduction ### - RDC - - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - // Restore scalar registers - ldp x16,x17,[sp,#16] - ldp x19,x20,[sp,#64] - ldp x29,x30,[sp],#80 - - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_sm2p256_mul,.-ecp_sm2p256_mul - -// void ecp_sm2p256_sqr(BN_ULONG *r, const BN_ULONG *a); -.globl ecp_sm2p256_sqr -.type ecp_sm2p256_sqr,%function -.align 5 - -ecp_sm2p256_sqr: - AARCH64_SIGN_LINK_REGISTER - // Store scalar registers - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - stp x16,x17,[sp,#16] - stp x19,x20,[sp,#64] - - // Load inputs - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - -// ### square ### - // ======================== - // s7 s6 s5 s4 - // * s7 s6 s5 s4 - // ------------------------ - // + s4 s4 s4 s4 - // * * * * - // s7 s6 s5 s4 - // s5 s5 s5 s5 - // * * * * - // s7 s6 s5 s4 - // s6 s6 s6 s6 - // * * * * - // s7 s6 s5 s4 - // s7 s7 s7 s7 - // * * * * - // s7 s6 s5 s4 - // ------------------------ - // s7 s6 s5 s4 s3 s2 s1 s0 - // ======================== - -// ### s4*s5 ### - mul x8,x11,x12 - umulh x9,x11,x12 - -// ### s4*s6 ### - mul x3,x13,x11 - umulh x10,x13,x11 - adds x9,x9,x3 - adcs x10,x10,xzr - -// ### s4*s7 + s5*s6 ### - mul x3,x14,x11 - umulh x4,x14,x11 - adds x10,x10,x3 - adcs x7,x4,xzr - - mul x3,x13,x12 - umulh x4,x13,x12 - adds x10,x10,x3 - adcs x7,x7,x4 - adcs x5,xzr,xzr - -// ### s5*s7 ### - mul x3,x14,x12 - umulh x4,x14,x12 - adds x7,x7,x3 - adcs x5,x5,x4 - -// ### s6*s7 ### - mul x3,x14,x13 - umulh x4,x14,x13 - adds x5,x5,x3 - adcs x6,x4,xzr - -// ### 2*(t3,t2,s0,s3,s2,s1) ### - adds x8,x8,x8 - adcs x9,x9,x9 - adcs x10,x10,x10 - adcs x7,x7,x7 - adcs x5,x5,x5 - adcs x6,x6,x6 - adcs x15,xzr,xzr - -// ### s4*s4 ### - mul x16,x11,x11 - umulh x17,x11,x11 - -// ### s5*s5 ### - mul x11,x12,x12 - umulh x12,x12,x12 - -// ### s6*s6 ### - mul x3,x13,x13 - umulh x4,x13,x13 - -// ### s7*s7 ### - mul x19,x14,x14 - umulh x20,x14,x14 - - adds x8,x8,x17 - adcs x9,x9,x11 - adcs x10,x10,x12 - adcs x7,x7,x3 - adcs x5,x5,x4 - adcs x6,x6,x19 - adcs x15,x15,x20 - - mov x11,x7 - mov x7,x16 - mov x12,x5 - mov x13,x6 - mov x14,x15 - - // result of mul: s7 s6 s5 s4 s3 s2 s1 s0 - -// ### Reduction ### - RDC - - stp x7,x8,[x0] - stp x9,x10,[x0,#16] - - // Restore scalar registers - ldp x16,x17,[sp,#16] - ldp x19,x20,[sp,#64] - ldp x29,x30,[sp],#80 - - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_sm2p256_sqr,.-ecp_sm2p256_sqr diff --git a/openssl/src/crypto/ec/gen/linux_ia32/ecp_nistz256-x86.S b/openssl/src/crypto/ec/gen/linux_ia32/ecp_nistz256-x86.S index f822079f8..68a56e2b5 100644 --- a/openssl/src/crypto/ec/gen/linux_ia32/ecp_nistz256-x86.S +++ b/openssl/src/crypto/ec/gen/linux_ia32/ecp_nistz256-x86.S @@ -2387,11 +2387,7 @@ ecp_nistz256_precomputed: .align 16 ecp_nistz256_mul_by_2: .L_ecp_nistz256_mul_by_2_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2411,11 +2407,7 @@ ecp_nistz256_mul_by_2: .align 16 ecp_nistz256_mul_by_3: .L_ecp_nistz256_mul_by_3_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2441,11 +2433,7 @@ ecp_nistz256_mul_by_3: .align 16 ecp_nistz256_div_by_2: .L_ecp_nistz256_div_by_2_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2462,11 +2450,7 @@ ecp_nistz256_div_by_2: .type _ecp_nistz256_div_by_2,@function .align 16 _ecp_nistz256_div_by_2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl (%esi),%ebp xorl %edx,%edx movl 4(%esi),%ebx @@ -2550,11 +2534,7 @@ _ecp_nistz256_div_by_2: .align 16 ecp_nistz256_add: .L_ecp_nistz256_add_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2572,11 +2552,7 @@ ecp_nistz256_add: .type _ecp_nistz256_add,@function .align 16 _ecp_nistz256_add: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl (%esi),%eax movl 4(%esi),%ebx movl 8(%esi),%ecx @@ -2654,11 +2630,7 @@ _ecp_nistz256_add: .align 16 ecp_nistz256_sub: .L_ecp_nistz256_sub_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2676,11 +2648,7 @@ ecp_nistz256_sub: .type _ecp_nistz256_sub,@function .align 16 _ecp_nistz256_sub: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl (%esi),%eax movl 4(%esi),%ebx movl 8(%esi),%ecx @@ -2739,11 +2707,7 @@ _ecp_nistz256_sub: .align 16 ecp_nistz256_neg: .L_ecp_nistz256_neg_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2772,11 +2736,7 @@ ecp_nistz256_neg: .type _picup_eax,@function .align 16 _picup_eax: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl (%esp),%eax ret .size _picup_eax,.-_picup_eax @@ -2785,11 +2745,7 @@ _picup_eax: .align 16 ecp_nistz256_to_mont: .L_ecp_nistz256_to_mont_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2813,11 +2769,7 @@ ecp_nistz256_to_mont: .align 16 ecp_nistz256_from_mont: .L_ecp_nistz256_from_mont_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2841,11 +2793,7 @@ ecp_nistz256_from_mont: .align 16 ecp_nistz256_mul_mont: .L_ecp_nistz256_mul_mont_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2869,11 +2817,7 @@ ecp_nistz256_mul_mont: .align 16 ecp_nistz256_sqr_mont: .L_ecp_nistz256_sqr_mont_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2895,11 +2839,7 @@ ecp_nistz256_sqr_mont: .type _ecp_nistz256_mul_mont,@function .align 16 _ecp_nistz256_mul_mont: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - andl $83886080,%eax cmpl $83886080,%eax jne .L004mul_mont_ialu @@ -3797,11 +3737,7 @@ _ecp_nistz256_mul_mont: .align 16 ecp_nistz256_scatter_w5: .L_ecp_nistz256_scatter_w5_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -3835,11 +3771,7 @@ ecp_nistz256_scatter_w5: .align 16 ecp_nistz256_gather_w5: .L_ecp_nistz256_gather_w5_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -3934,11 +3866,7 @@ ecp_nistz256_gather_w5: .align 16 ecp_nistz256_scatter_w7: .L_ecp_nistz256_scatter_w7_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -3970,11 +3898,7 @@ ecp_nistz256_scatter_w7: .align 16 ecp_nistz256_gather_w7: .L_ecp_nistz256_gather_w7_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -4189,11 +4113,7 @@ ecp_nistz256_gather_w7: .align 16 ecp_nistz256_point_double: .L_ecp_nistz256_point_double_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -4320,11 +4240,7 @@ ecp_nistz256_point_double: .align 16 ecp_nistz256_point_add: .L_ecp_nistz256_point_add_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -4838,11 +4754,7 @@ ecp_nistz256_point_add: .align 16 ecp_nistz256_point_add_affine: .L_ecp_nistz256_point_add_affine_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/ec/gen/windows_ia32/ecp_nistz256-x86.asm b/openssl/src/crypto/ec/gen/windows_ia32/ecp_nistz256-x86.asm index 5c4c572b1..96071a300 100644 --- a/openssl/src/crypto/ec/gen/windows_ia32/ecp_nistz256-x86.asm +++ b/openssl/src/crypto/ec/gen/windows_ia32/ecp_nistz256-x86.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/ec/local.h b/openssl/src/crypto/ec/local.h new file mode 100644 index 000000000..c53884bca --- /dev/null +++ b/openssl/src/crypto/ec/local.h @@ -0,0 +1,96 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +int x25519_fe64_eligible(void); +void x25519_fe64_mul(fe64 h, const fe64 f, const fe64 g); +void x25519_fe64_sqr(fe64 h, const fe64 f); +void x25519_fe64_mul121666(fe64 h, fe64 f); +void x25519_fe64_add(fe64 h, const fe64 f, const fe64 g); +void x25519_fe64_sub(fe64 h, const fe64 f, const fe64 g); +void x25519_fe64_tobytes(uint8_t *s, const fe64 f); +void x25519_fe51_mul(fe51 h, const fe51 f, const fe51 g); +void x25519_fe51_sqr(fe51 h, const fe51 f); +void x25519_fe51_mul121666(fe51 h, fe51 f); + +/* Modular add: res = a+b mod P */ +void ecp_nistz256_add(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +/* Modular mul by 2: res = 2*a mod P */ +void ecp_nistz256_mul_by_2(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +/* Modular mul by 3: res = 3*a mod P */ +void ecp_nistz256_mul_by_3(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); + +/* Modular div by 2: res = a/2 mod P */ +void ecp_nistz256_div_by_2(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +/* Modular sub: res = a-b mod P */ +void ecp_nistz256_sub(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +/* Modular neg: res = -a mod P */ +void ecp_nistz256_neg(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]); +/* Montgomery mul: res = a*b*2^-256 mod P */ +void ecp_nistz256_mul_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +/* Montgomery sqr: res = a*a*2^-256 mod P */ +void ecp_nistz256_sqr_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +/* Convert a number from Montgomery domain, by multiplying with 1 */ +void ecp_nistz256_from_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG in[P256_LIMBS]); +/* Convert a number to Montgomery domain, by multiplying with 2^512 mod P*/ +void ecp_nistz256_to_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG in[P256_LIMBS]); +/* Functions that perform constant time access to the precomputed tables */ +void ecp_nistz256_scatter_w5(P256_POINT *val, + const P256_POINT *in_t, int idx); +void ecp_nistz256_gather_w5(P256_POINT *val, + const P256_POINT *in_t, int idx); +void ecp_nistz256_scatter_w7(P256_POINT_AFFINE *val, + const P256_POINT_AFFINE *in_t, int idx); +void ecp_nistz256_gather_w7(P256_POINT_AFFINE *val, + const P256_POINT_AFFINE *in_t, int idx); + +/* Precomputed tables for the default generator */ +extern const PRECOMP256_ROW ecp_nistz256_precomputed[37]; + +#ifndef ECP_NISTZ256_REFERENCE_IMPLEMENTATION +void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a); +void ecp_nistz256_point_add(P256_POINT *r, + const P256_POINT *a, const P256_POINT *b); +void ecp_nistz256_point_add_affine(P256_POINT *r, + const P256_POINT *a, + const P256_POINT_AFFINE *b); +#endif + +#if defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_AMD64) || defined(_M_X64) || \ + defined(__powerpc64__) || defined(_ARCH_PP64) || \ + defined(__aarch64__) +/* + * Montgomery mul modulo Order(P): res = a*b*2^-256 mod Order(P) + */ +void ecp_nistz256_ord_mul_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +void ecp_nistz256_ord_sqr_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + BN_ULONG rep); +#endif + +DECLARE_ASN1_FUNCTIONS(EC_PRIVATEKEY) +DECLARE_ASN1_ENCODE_FUNCTIONS_name(EC_PRIVATEKEY, EC_PRIVATEKEY) diff --git a/openssl/src/crypto/eia3/eia3.c b/openssl/src/crypto/eia3/eia3.c new file mode 100644 index 000000000..643c12e1b --- /dev/null +++ b/openssl/src/crypto/eia3/eia3.c @@ -0,0 +1,127 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include + +#ifndef OPENSSL_NO_ZUC + +# include "crypto/zuc.h" +# include "eia3_local.h" + +static ossl_inline uint8_t GET_BIT(const unsigned char *data, uint32_t i) +{ + return (data[i / 8] & (1 << (7 - (i % 8)))) ? 1 : 0; +} + +size_t EIA3_ctx_size(void) +{ + return sizeof(struct eia3_context); +} + +int EIA3_Init(EIA3_CTX *ctx, const unsigned char key[EVP_ZUC_KEY_SIZE], const unsigned char iv[5]) +{ + ZUC_KEY *zk = &ctx->zk; + uint32_t count = 0; + uint32_t bearer = 0; + uint32_t direction = 0; + + memset(ctx, 0, sizeof(EIA3_CTX)); + + zk->k = key; + + /* + * This is a lazy approach: we 'borrow' the 'iv' parameter + * to use it as a place of transfer the EEA3 iv params - + * count, bearer and direction. + * + * count is 32 bits, bearer is 5 bits and direction is 1 + * bit so we read the first 38 bits of iv. And the whole + * iv is set to 5 bytes (40 bits). + */ + if (iv != NULL) { + count = ((long)iv[0] << 24) | (iv[1] << 16) | (iv[2] << 8) | iv[3]; + bearer = (iv[4] & 0xF8) >> 3; + direction = (iv[4] & 0x4) >> 2; + } + + zk->iv[0] = (count >> 24) & 0xFF; + zk->iv[1] = (count >> 16) & 0xFF; + zk->iv[2] = (count >> 8) & 0xFF; + zk->iv[3] = count & 0xFF; + + zk->iv[4] = (bearer << 3) & 0xF8; + zk->iv[5] = zk->iv[6] = zk->iv[7] = 0; + + zk->iv[8] = ((count >> 24) & 0xFF) ^ ((direction & 1) << 7); + zk->iv[9] = (count >> 16) & 0xFF; + zk->iv[10] = (count >> 8) & 0xFF; + zk->iv[11] = count & 0xFF; + + zk->iv[12] = zk->iv[4]; + zk->iv[13] = zk->iv[5]; + zk->iv[14] = zk->iv[6] ^ ((direction & 1) << 7); + zk->iv[15] = zk->iv[7]; + + ZUC_init(zk); + + return 1; +} + +int EIA3_Update(EIA3_CTX *ctx, const unsigned char *inp, size_t len) +{ + ZUC_KEY *zk = &ctx->zk; + size_t i, length = len * 8; + + for (i = 0; i < length; i++) { + if ((ctx->num + i/8 + 4) >= zk->keystream_len) { + if (!ZUC_generate_keystream(zk)) + return 0; + } + + if (GET_BIT(inp, i)) + ctx->T ^= ZUC_keystream_get_word(zk, i); + } + + ctx->length += length; + ctx->num += len; + + return 1; +} + +void EIA3_Final(EIA3_CTX *ctx, unsigned char out[EIA3_DIGEST_SIZE]) +{ + ZUC_KEY *zk = &ctx->zk; + size_t L = (ctx->length + 64 + 31) / 32, last; + uint32_t mac; + + if ((ctx->length / 8 + 4) >= zk->keystream_len) { + if (!ZUC_generate_keystream(zk)) + return; + } + + ctx->T ^= ZUC_keystream_get_word(zk, ctx->length); + + last = (L - 1) * 32; + if ((last / 8 + 4) >= zk->keystream_len) { + if (!ZUC_generate_keystream(zk)) + return; + } + + mac = ctx->T ^ ZUC_keystream_get_word(zk, (L - 1) * 32); + + out[0] = (uint8_t)(mac >> 24) & 0xFF; + out[1] = (uint8_t)(mac >> 16) & 0xFF; + out[2] = (uint8_t)(mac >> 8) & 0xFF; + out[3] = (uint8_t)mac & 0xFF; + + ZUC_destroy_keystream(&ctx->zk); +} +#endif diff --git a/openssl/src/crypto/eia3/eia3_local.h b/openssl/src/crypto/eia3/eia3_local.h new file mode 100644 index 000000000..e29bb24f8 --- /dev/null +++ b/openssl/src/crypto/eia3/eia3_local.h @@ -0,0 +1,25 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef OPENSSL_NO_ZUC + +# include +# include +# include + +# include "crypto/zuc.h" + +struct eia3_context { + ZUC_KEY zk; + size_t num; + size_t length; /* The bits of the input message */ + uint32_t T; +}; + +#endif diff --git a/openssl/src/crypto/encode_decode/decoder_lib.c b/openssl/src/crypto/encode_decode/decoder_lib.c index 2e74816ee..a802aeb42 100644 --- a/openssl/src/crypto/encode_decode/decoder_lib.c +++ b/openssl/src/crypto/encode_decode/decoder_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,7 +18,6 @@ #include #include "internal/bio.h" #include "internal/provider.h" -#include "internal/namemap.h" #include "crypto/decoder.h" #include "encoder_local.h" #include "internal/e_os.h" @@ -224,8 +223,14 @@ OSSL_DECODER_INSTANCE *ossl_decoder_instance_new(OSSL_DECODER *decoder, return 0; } - if ((decoder_inst = OPENSSL_zalloc(sizeof(*decoder_inst))) == NULL) + if ((decoder_inst = OPENSSL_zalloc(sizeof(*decoder_inst))) == NULL) { + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE); return 0; + } + if (!OSSL_DECODER_up_ref(decoder)) { + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_INTERNAL_ERROR); + goto err; + } prov = OSSL_DECODER_get0_provider(decoder); libctx = ossl_provider_libctx(prov); @@ -240,7 +245,6 @@ OSSL_DECODER_INSTANCE *ossl_decoder_instance_new(OSSL_DECODER *decoder, /* The "input" property is mandatory */ prop = ossl_property_find_property(props, libctx, "input"); decoder_inst->input_type = ossl_property_get_string_value(libctx, prop); - decoder_inst->input_type_id = 0; if (decoder_inst->input_type == NULL) { ERR_raise_data(ERR_LIB_OSSL_DECODER, ERR_R_INVALID_PROPERTY_DEFINITION, "the mandatory 'input' property is missing " @@ -257,10 +261,6 @@ OSSL_DECODER_INSTANCE *ossl_decoder_instance_new(OSSL_DECODER *decoder, = ossl_property_get_string_value(libctx, prop); } - if (!OSSL_DECODER_up_ref(decoder)) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_INTERNAL_ERROR); - goto err; - } decoder_inst->decoder = decoder; decoder_inst->decoderctx = decoderctx; return decoder_inst; @@ -281,37 +281,6 @@ void ossl_decoder_instance_free(OSSL_DECODER_INSTANCE *decoder_inst) } } -OSSL_DECODER_INSTANCE *ossl_decoder_instance_dup(const OSSL_DECODER_INSTANCE *src) -{ - OSSL_DECODER_INSTANCE *dest; - const OSSL_PROVIDER *prov; - void *provctx; - - if ((dest = OPENSSL_zalloc(sizeof(*dest))) == NULL) - return NULL; - - *dest = *src; - if (!OSSL_DECODER_up_ref(dest->decoder)) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_INTERNAL_ERROR); - goto err; - } - prov = OSSL_DECODER_get0_provider(dest->decoder); - provctx = OSSL_PROVIDER_get0_provider_ctx(prov); - - dest->decoderctx = dest->decoder->newctx(provctx); - if (dest->decoderctx == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_INTERNAL_ERROR); - OSSL_DECODER_free(dest->decoder); - goto err; - } - - return dest; - - err: - OPENSSL_free(dest); - return NULL; -} - int ossl_decoder_ctx_add_decoder_inst(OSSL_DECODER_CTX *ctx, OSSL_DECODER_INSTANCE *di) { @@ -320,7 +289,7 @@ int ossl_decoder_ctx_add_decoder_inst(OSSL_DECODER_CTX *ctx, if (ctx->decoder_insts == NULL && (ctx->decoder_insts = sk_OSSL_DECODER_INSTANCE_new_null()) == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE); return 0; } @@ -374,8 +343,6 @@ int OSSL_DECODER_CTX_add_decoder(OSSL_DECODER_CTX *ctx, OSSL_DECODER *decoder) struct collect_extra_decoder_data_st { OSSL_DECODER_CTX *ctx; const char *output_type; - int output_type_id; - /* * 0 to check that the decoder's input type is the same as the decoder name * 1 to check that the decoder's input type differs from the decoder name @@ -391,9 +358,8 @@ static void collect_all_decoders(OSSL_DECODER *decoder, void *arg) { STACK_OF(OSSL_DECODER) *skdecoders = arg; - if (OSSL_DECODER_up_ref(decoder) - && !sk_OSSL_DECODER_push(skdecoders, decoder)) - OSSL_DECODER_free(decoder); + if (OSSL_DECODER_up_ref(decoder)) + sk_OSSL_DECODER_push(skdecoders, decoder); } static void collect_extra_decoder(OSSL_DECODER *decoder, void *arg) @@ -403,7 +369,7 @@ static void collect_extra_decoder(OSSL_DECODER *decoder, void *arg) const OSSL_PROVIDER *prov = OSSL_DECODER_get0_provider(decoder); void *provctx = OSSL_PROVIDER_get0_provider_ctx(prov); - if (ossl_decoder_fast_is_a(decoder, data->output_type, &data->output_type_id)) { + if (OSSL_DECODER_is_a(decoder, data->output_type)) { void *decoderctx = NULL; OSSL_DECODER_INSTANCE *di = NULL; @@ -446,9 +412,8 @@ static void collect_extra_decoder(OSSL_DECODER *decoder, void *arg) switch (data->type_check) { case IS_SAME: /* If it differs, this is not a decoder to add for now. */ - if (!ossl_decoder_fast_is_a(decoder, - OSSL_DECODER_INSTANCE_get_input_type(di), - &di->input_type_id)) { + if (!OSSL_DECODER_is_a(decoder, + OSSL_DECODER_INSTANCE_get_input_type(di))) { ossl_decoder_instance_free(di); OSSL_TRACE_BEGIN(DECODER) { BIO_printf(trc_out, @@ -459,9 +424,8 @@ static void collect_extra_decoder(OSSL_DECODER *decoder, void *arg) break; case IS_DIFFERENT: /* If it's the same, this is not a decoder to add for now. */ - if (ossl_decoder_fast_is_a(decoder, - OSSL_DECODER_INSTANCE_get_input_type(di), - &di->input_type_id)) { + if (OSSL_DECODER_is_a(decoder, + OSSL_DECODER_INSTANCE_get_input_type(di))) { ossl_decoder_instance_free(di); OSSL_TRACE_BEGIN(DECODER) { BIO_printf(trc_out, @@ -537,7 +501,7 @@ int OSSL_DECODER_CTX_add_extra(OSSL_DECODER_CTX *ctx, skdecoders = sk_OSSL_DECODER_new_null(); if (skdecoders == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE); return 0; } OSSL_DECODER_do_all_provided(libctx, collect_all_decoders, skdecoders); @@ -569,7 +533,6 @@ int OSSL_DECODER_CTX_add_extra(OSSL_DECODER_CTX *ctx, data.output_type = OSSL_DECODER_INSTANCE_get_input_type(decoder_inst); - data.output_type_id = 0; for (j = 0; j < numdecoders; j++) collect_extra_decoder(sk_OSSL_DECODER_value(skdecoders, j), @@ -779,11 +742,10 @@ static int decoder_process(const OSSL_PARAM params[], void *arg) (void *)new_data.ctx, LEVEL, rv); } OSSL_TRACE_END(DECODER); + data->flag_construct_called = 1; ok = (rv > 0); - if (ok) { - data->flag_construct_called = 1; + if (ok) goto end; - } } /* The constructor didn't return success */ @@ -854,7 +816,7 @@ static int decoder_process(const OSSL_PARAM params[], void *arg) } if ((cbio = ossl_core_bio_new_from_bio(bio)) == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE); goto end; } @@ -904,8 +866,7 @@ static int decoder_process(const OSSL_PARAM params[], void *arg) * |new_input_type| holds the value of the "input-type" parameter * for the decoder we're currently considering. */ - if (decoder != NULL && !ossl_decoder_fast_is_a(decoder, new_input_type, - &new_decoder_inst->input_type_id)) { + if (decoder != NULL && !OSSL_DECODER_is_a(decoder, new_input_type)) { OSSL_TRACE_BEGIN(DECODER) { BIO_printf(trc_out, "(ctx %p) %s [%u] the input type doesn't match the name of the previous decoder (%p), skipping...\n", @@ -918,8 +879,13 @@ static int decoder_process(const OSSL_PARAM params[], void *arg) /* * If the previous decoder gave us a data type, we check to see * if that matches the decoder we're currently considering. + * + * Note: when data type is EC, maybe wrapper for SM2, so we try both EC + * and SM2 decoders. */ - if (data_type != NULL && !OSSL_DECODER_is_a(new_decoder, data_type)) { + if (data_type != NULL && !OSSL_DECODER_is_a(new_decoder, data_type) + && !(OPENSSL_strcasecmp(data_type, "EC") == 0 + && OSSL_DECODER_is_a(new_decoder, "SM2"))) { OSSL_TRACE_BEGIN(DECODER) { BIO_printf(trc_out, "(ctx %p) %s [%u] the previous decoder's data type doesn't match the name of the considered decoder, skipping...\n", diff --git a/openssl/src/crypto/encode_decode/decoder_meth.c b/openssl/src/crypto/encode_decode/decoder_meth.c index 2e70e8aa3..cb53343c5 100644 --- a/openssl/src/crypto/encode_decode/decoder_meth.c +++ b/openssl/src/crypto/encode_decode/decoder_meth.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,7 +17,6 @@ #include "internal/provider.h" #include "crypto/decoder.h" #include "encoder_local.h" -#include "crypto/context.h" /* * Decoder can have multiple names, separated with colons in a name string @@ -29,13 +28,15 @@ static OSSL_DECODER *ossl_decoder_new(void) { OSSL_DECODER *decoder = NULL; - if ((decoder = OPENSSL_zalloc(sizeof(*decoder))) == NULL) - return NULL; - if (!CRYPTO_NEW_REF(&decoder->base.refcnt, 1)) { + if ((decoder = OPENSSL_zalloc(sizeof(*decoder))) == NULL + || (decoder->base.lock = CRYPTO_THREAD_lock_new()) == NULL) { OSSL_DECODER_free(decoder); + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE); return NULL; } + decoder->base.refcnt = 1; + return decoder; } @@ -43,7 +44,7 @@ int OSSL_DECODER_up_ref(OSSL_DECODER *decoder) { int ref = 0; - CRYPTO_UP_REF(&decoder->base.refcnt, &ref); + CRYPTO_UP_REF(&decoder->base.refcnt, &ref, decoder->base.lock); return 1; } @@ -54,16 +55,35 @@ void OSSL_DECODER_free(OSSL_DECODER *decoder) if (decoder == NULL) return; - CRYPTO_DOWN_REF(&decoder->base.refcnt, &ref); + CRYPTO_DOWN_REF(&decoder->base.refcnt, &ref, decoder->base.lock); if (ref > 0) return; OPENSSL_free(decoder->base.name); ossl_property_free(decoder->base.parsed_propdef); ossl_provider_free(decoder->base.prov); - CRYPTO_FREE_REF(&decoder->base.refcnt); + CRYPTO_THREAD_lock_free(decoder->base.lock); OPENSSL_free(decoder); } +/* Permanent decoder method store, constructor and destructor */ +static void decoder_store_free(void *vstore) +{ + ossl_method_store_free(vstore); +} + +static void *decoder_store_new(OSSL_LIB_CTX *ctx) +{ + return ossl_method_store_new(ctx); +} + + +static const OSSL_LIB_CTX_METHOD decoder_store_method = { + /* We want decoder_store to be cleaned up before the provider store */ + OSSL_LIB_CTX_METHOD_PRIORITY_2, + decoder_store_new, + decoder_store_free, +}; + /* Data to be passed through ossl_method_construct() */ struct decoder_data_st { OSSL_LIB_CTX *libctx; @@ -100,29 +120,8 @@ static void dealloc_tmp_decoder_store(void *store) /* Get the permanent decoder store */ static OSSL_METHOD_STORE *get_decoder_store(OSSL_LIB_CTX *libctx) { - return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DECODER_STORE_INDEX); -} - -static int reserve_decoder_store(void *store, void *data) -{ - struct decoder_data_st *methdata = data; - - if (store == NULL - && (store = get_decoder_store(methdata->libctx)) == NULL) - return 0; - - return ossl_method_lock_store(store); -} - -static int unreserve_decoder_store(void *store, void *data) -{ - struct decoder_data_st *methdata = data; - - if (store == NULL - && (store = get_decoder_store(methdata->libctx)) == NULL) - return 0; - - return ossl_method_unlock_store(store); + return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DECODER_STORE_INDEX, + &decoder_store_method); } /* Get decoder methods from a store, or put one in */ @@ -211,11 +210,8 @@ void *ossl_decoder_from_algorithm(int id, const OSSL_ALGORITHM *algodef, return NULL; } decoder->base.algodef = algodef; - if ((decoder->base.parsed_propdef - = ossl_parse_property(libctx, algodef->property_definition)) == NULL) { - OSSL_DECODER_free(decoder); - return NULL; - } + decoder->base.parsed_propdef + = ossl_parse_property(libctx, algodef->property_definition); for (; fns->function_id != 0; fns++) { switch (fns->function_id) { @@ -338,34 +334,43 @@ static void free_decoder(void *method) /* Fetching support. Can fetch by numeric identity or by name */ static OSSL_DECODER * -inner_ossl_decoder_fetch(struct decoder_data_st *methdata, +inner_ossl_decoder_fetch(struct decoder_data_st *methdata, int id, const char *name, const char *properties) { OSSL_METHOD_STORE *store = get_decoder_store(methdata->libctx); OSSL_NAMEMAP *namemap = ossl_namemap_stored(methdata->libctx); const char *const propq = properties != NULL ? properties : ""; void *method = NULL; - int unsupported, id; + int unsupported = 0; if (store == NULL || namemap == NULL) { ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_PASSED_INVALID_ARGUMENT); return NULL; } - id = name != NULL ? ossl_namemap_name2num(namemap, name) : 0; + /* + * If we have been passed both an id and a name, we have an + * internal programming error. + */ + if (!ossl_assert(id == 0 || name == NULL)) { + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_INTERNAL_ERROR); + return NULL; + } + + if (id == 0 && name != NULL) + id = ossl_namemap_name2num(namemap, name); /* * If we haven't found the name yet, chances are that the algorithm to * be fetched is unsupported. */ - unsupported = id == 0; + if (id == 0) + unsupported = 1; if (id == 0 || !ossl_method_store_cache_get(store, NULL, id, propq, &method)) { OSSL_METHOD_CONSTRUCT_METHOD mcm = { get_tmp_decoder_store, - reserve_decoder_store, - unreserve_decoder_store, get_decoder_from_store, put_decoder_in_store, construct_decoder, @@ -408,7 +413,7 @@ inner_ossl_decoder_fetch(struct decoder_data_st *methdata, ERR_raise_data(ERR_LIB_OSSL_DECODER, code, "%s, Name (%s : %d), Properties (%s)", ossl_lib_ctx_get_descriptor(methdata->libctx), - name == NULL ? "" : name, id, + name = NULL ? "" : name, id, properties == NULL ? "" : properties); } @@ -423,28 +428,22 @@ OSSL_DECODER *OSSL_DECODER_fetch(OSSL_LIB_CTX *libctx, const char *name, methdata.libctx = libctx; methdata.tmp_store = NULL; - method = inner_ossl_decoder_fetch(&methdata, name, properties); + method = inner_ossl_decoder_fetch(&methdata, 0, name, properties); dealloc_tmp_decoder_store(methdata.tmp_store); return method; } -int ossl_decoder_store_cache_flush(OSSL_LIB_CTX *libctx) -{ - OSSL_METHOD_STORE *store = get_decoder_store(libctx); - - if (store != NULL) - return ossl_method_store_cache_flush_all(store); - return 1; -} - -int ossl_decoder_store_remove_all_provided(const OSSL_PROVIDER *prov) +OSSL_DECODER *ossl_decoder_fetch_by_number(OSSL_LIB_CTX *libctx, int id, + const char *properties) { - OSSL_LIB_CTX *libctx = ossl_provider_libctx(prov); - OSSL_METHOD_STORE *store = get_decoder_store(libctx); + struct decoder_data_st methdata; + void *method; - if (store != NULL) - return ossl_method_store_remove_all_provided(store, prov); - return 1; + methdata.libctx = libctx; + methdata.tmp_store = NULL; + method = inner_ossl_decoder_fetch(&methdata, id, NULL, properties); + dealloc_tmp_decoder_store(methdata.tmp_store); + return method; } /* @@ -513,24 +512,6 @@ int OSSL_DECODER_is_a(const OSSL_DECODER *decoder, const char *name) return 0; } -static int resolve_name(OSSL_DECODER *decoder, const char *name) -{ - OSSL_LIB_CTX *libctx = ossl_provider_libctx(decoder->base.prov); - OSSL_NAMEMAP *namemap = ossl_namemap_stored(libctx); - - return ossl_namemap_name2num(namemap, name); -} - -int ossl_decoder_fast_is_a(OSSL_DECODER *decoder, const char *name, int *id_cache) -{ - int id = *id_cache; - - if (id <= 0) - *id_cache = id = resolve_name(decoder, name); - - return id > 0 && ossl_decoder_get_number(decoder) == id; -} - struct do_one_data_st { void (*user_fn)(OSSL_DECODER *decoder, void *arg); void *user_arg; @@ -553,7 +534,7 @@ void OSSL_DECODER_do_all_provided(OSSL_LIB_CTX *libctx, methdata.libctx = libctx; methdata.tmp_store = NULL; - (void)inner_ossl_decoder_fetch(&methdata, NULL, NULL /* properties */); + (void)inner_ossl_decoder_fetch(&methdata, 0, NULL, NULL /* properties */); data.user_fn = user_fn; data.user_arg = user_arg; @@ -623,7 +604,9 @@ OSSL_DECODER_CTX *OSSL_DECODER_CTX_new(void) { OSSL_DECODER_CTX *ctx; - ctx = OPENSSL_zalloc(sizeof(*ctx)); + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE); + return ctx; } diff --git a/openssl/src/crypto/encode_decode/decoder_pkey.c b/openssl/src/crypto/encode_decode/decoder_pkey.c index eb1be1c98..b5535a834 100644 --- a/openssl/src/crypto/encode_decode/decoder_pkey.c +++ b/openssl/src/crypto/encode_decode/decoder_pkey.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,11 +17,7 @@ #include #include "crypto/evp.h" #include "crypto/decoder.h" -#include "crypto/evp/evp_local.h" -#include "crypto/lhash.h" #include "encoder_local.h" -#include "internal/namemap.h" -#include "internal/sizes.h" int OSSL_DECODER_CTX_set_passphrase(OSSL_DECODER_CTX *ctx, const unsigned char *kstr, @@ -154,11 +150,7 @@ static int decoder_construct_pkey(OSSL_DECODER_INSTANCE *decoder_inst, import_data.keymgmt = keymgmt; import_data.keydata = NULL; - if (data->selection == 0) - /* import/export functions do not tolerate 0 selection */ - import_data.selection = OSSL_KEYMGMT_SELECT_ALL; - else - import_data.selection = data->selection; + import_data.selection = data->selection; /* * No need to check for errors here, the value of @@ -203,83 +195,53 @@ static void decoder_clean_pkey_construct_arg(void *construct_data) } } -struct collect_data_st { - OSSL_LIB_CTX *libctx; - OSSL_DECODER_CTX *ctx; - - const char *keytype; /* the keytype requested, if any */ - int keytype_id; /* if keytype_resolved is set, keymgmt name_id; else 0 */ - int sm2_id; /* if keytype_resolved is set and EC, SM2 name_id; else 0 */ - int total; /* number of matching results */ - char error_occurred; - char keytype_resolved; +static void collect_name(const char *name, void *arg) +{ + STACK_OF(OPENSSL_CSTRING) *names = arg; - STACK_OF(EVP_KEYMGMT) *keymgmts; -}; + sk_OPENSSL_CSTRING_push(names, name); +} -static void collect_decoder_keymgmt(EVP_KEYMGMT *keymgmt, OSSL_DECODER *decoder, - void *provctx, struct collect_data_st *data) +static void collect_keymgmt(EVP_KEYMGMT *keymgmt, void *arg) { - void *decoderctx = NULL; - OSSL_DECODER_INSTANCE *di = NULL; + STACK_OF(EVP_KEYMGMT) *keymgmts = arg; - /* - * We already checked the EVP_KEYMGMT is applicable in check_keymgmt so we - * don't check it again here. - */ - - if (keymgmt->name_id != decoder->base.id) - /* Mismatch is not an error, continue. */ + if (!EVP_KEYMGMT_up_ref(keymgmt) /* ref++ */) return; - - if ((decoderctx = decoder->newctx(provctx)) == NULL) { - data->error_occurred = 1; + if (sk_EVP_KEYMGMT_push(keymgmts, keymgmt) <= 0) { + EVP_KEYMGMT_free(keymgmt); /* ref-- */ return; } +} - if ((di = ossl_decoder_instance_new(decoder, decoderctx)) == NULL) { - decoder->freectx(decoderctx); - data->error_occurred = 1; - return; - } - - OSSL_TRACE_BEGIN(DECODER) { - BIO_printf(trc_out, - "(ctx %p) Checking out decoder %p:\n" - " %s with %s\n", - (void *)data->ctx, (void *)decoder, - OSSL_DECODER_get0_name(decoder), - OSSL_DECODER_get0_properties(decoder)); - } OSSL_TRACE_END(DECODER); - - if (!ossl_decoder_ctx_add_decoder_inst(data->ctx, di)) { - ossl_decoder_instance_free(di); - data->error_occurred = 1; - return; - } +struct collect_decoder_data_st { + STACK_OF(OPENSSL_CSTRING) *names; + OSSL_DECODER_CTX *ctx; - ++data->total; -} + int total; + unsigned int error_occurred:1; +}; static void collect_decoder(OSSL_DECODER *decoder, void *arg) { - struct collect_data_st *data = arg; - STACK_OF(EVP_KEYMGMT) *keymgmts = data->keymgmts; - int i, end_i; - EVP_KEYMGMT *keymgmt; - const OSSL_PROVIDER *prov; - void *provctx; + struct collect_decoder_data_st *data = arg; + size_t i, end_i; + const OSSL_PROVIDER *prov = OSSL_DECODER_get0_provider(decoder); + void *provctx = OSSL_PROVIDER_get0_provider_ctx(prov); if (data->error_occurred) return; - prov = OSSL_DECODER_get0_provider(decoder); - provctx = OSSL_PROVIDER_get0_provider_ctx(prov); + if (data->names == NULL) { + data->error_occurred = 1; + return; + } /* - * Either the caller didn't give us a selection, or if they did, the decoder - * must tell us if it supports that selection to be accepted. If the decoder - * doesn't have |does_selection|, it's seen as taking anything. + * Either the caller didn't give a selection, or if they did, + * the decoder must tell us if it supports that selection to + * be accepted. If the decoder doesn't have |does_selection|, + * it's seen as taking anything. */ if (decoder->does_selection != NULL && !decoder->does_selection(provctx, data->ctx->selection)) @@ -294,101 +256,69 @@ static void collect_decoder(OSSL_DECODER *decoder, void *arg) OSSL_DECODER_get0_properties(decoder)); } OSSL_TRACE_END(DECODER); - end_i = sk_EVP_KEYMGMT_num(keymgmts); - for (i = 0; i < end_i; ++i) { - keymgmt = sk_EVP_KEYMGMT_value(keymgmts, i); + end_i = sk_OPENSSL_CSTRING_num(data->names); + for (i = 0; i < end_i; i++) { + const char *name = sk_OPENSSL_CSTRING_value(data->names, i); - collect_decoder_keymgmt(keymgmt, decoder, provctx, data); - if (data->error_occurred) - return; - } -} + if (OSSL_DECODER_is_a(decoder, name)) { + void *decoderctx = NULL; + OSSL_DECODER_INSTANCE *di = NULL; -/* - * Is this EVP_KEYMGMT applicable given the key type given in the call to - * ossl_decoder_ctx_setup_for_pkey (if any)? - */ -static int check_keymgmt(EVP_KEYMGMT *keymgmt, struct collect_data_st *data) -{ - /* If no keytype was specified, everything matches. */ - if (data->keytype == NULL) - return 1; + if ((decoderctx = decoder->newctx(provctx)) == NULL) { + data->error_occurred = 1; + return; + } + if ((di = ossl_decoder_instance_new(decoder, decoderctx)) == NULL) { + decoder->freectx(decoderctx); + data->error_occurred = 1; + return; + } - if (!data->keytype_resolved) { - /* We haven't cached the IDs from the keytype string yet. */ - OSSL_NAMEMAP *namemap = ossl_namemap_stored(data->libctx); - data->keytype_id = ossl_namemap_name2num(namemap, data->keytype); + OSSL_TRACE_BEGIN(DECODER) { + BIO_printf(trc_out, + "(ctx %p) Checking out decoder %p:\n" + " %s with %s\n", + (void *)data->ctx, (void *)decoder, + OSSL_DECODER_get0_name(decoder), + OSSL_DECODER_get0_properties(decoder)); + } OSSL_TRACE_END(DECODER); - /* - * If keytype is a value ambiguously used for both EC and SM2, - * collect the ID for SM2 as well. - */ - if (data->keytype_id != 0 - && (strcmp(data->keytype, "id-ecPublicKey") == 0 - || strcmp(data->keytype, "1.2.840.10045.2.1") == 0)) - data->sm2_id = ossl_namemap_name2num(namemap, "SM2"); + if (!ossl_decoder_ctx_add_decoder_inst(data->ctx, di)) { + ossl_decoder_instance_free(di); + data->error_occurred = 1; + return; + } + data->total++; - /* - * If keytype_id is zero the name was not found, but we still - * set keytype_resolved to avoid trying all this again. - */ - data->keytype_resolved = 1; + /* Success */ + return; + } } - /* Specified keytype could not be resolved, so nothing matches. */ - if (data->keytype_id == 0) - return 0; - - /* Does not match the keytype specified, so skip. */ - if (keymgmt->name_id != data->keytype_id - && keymgmt->name_id != data->sm2_id) - return 0; - - return 1; -} - -static void collect_keymgmt(EVP_KEYMGMT *keymgmt, void *arg) -{ - struct collect_data_st *data = arg; - - if (!check_keymgmt(keymgmt, data)) - return; - - /* - * We have to ref EVP_KEYMGMT here because in the success case, - * data->keymgmts is referenced by the constructor we register in the - * OSSL_DECODER_CTX. The registered cleanup function - * (decoder_clean_pkey_construct_arg) unrefs every element of the stack and - * frees it. - */ - if (!EVP_KEYMGMT_up_ref(keymgmt)) - return; - - if (sk_EVP_KEYMGMT_push(data->keymgmts, keymgmt) <= 0) { - EVP_KEYMGMT_free(keymgmt); - data->error_occurred = 1; - } + /* Decoder not suitable - but not a fatal error */ + data->error_occurred = 0; } -/* - * This function does the actual binding of decoders to the OSSL_DECODER_CTX. It - * searches for decoders matching 'keytype', which is a string like "RSA", "DH", - * etc. If 'keytype' is NULL, decoders for all keytypes are bound. - */ -static int ossl_decoder_ctx_setup_for_pkey(OSSL_DECODER_CTX *ctx, - const char *keytype, - OSSL_LIB_CTX *libctx, - const char *propquery) +int ossl_decoder_ctx_setup_for_pkey(OSSL_DECODER_CTX *ctx, + EVP_PKEY **pkey, const char *keytype, + OSSL_LIB_CTX *libctx, + const char *propquery) { - int ok = 0; struct decoder_pkey_data_st *process_data = NULL; - struct collect_data_st collect_data = { NULL }; - STACK_OF(EVP_KEYMGMT) *keymgmts = NULL; + STACK_OF(OPENSSL_CSTRING) *names = NULL; + const char *input_type = ctx->start_input_type; + const char *input_structure = ctx->input_structure; + int ok = 0; + int is_ec = 0; + int i, end; - OSSL_TRACE_BEGIN(DECODER) { - const char *input_type = ctx->start_input_type; - const char *input_structure = ctx->input_structure; + if (keytype != NULL + && (strcmp(keytype, "id-ecPublicKey") == 0 + || strcmp(keytype, "1.2.840.10045.2.1") == 0 + || OPENSSL_strcasecmp(keytype, "EC") == 0)) + is_ec = 1; + OSSL_TRACE_BEGIN(DECODER) { BIO_printf(trc_out, "(ctx %p) Looking for decoders producing %s%s%s%s%s%s\n", (void *)ctx, @@ -400,66 +330,81 @@ static int ossl_decoder_ctx_setup_for_pkey(OSSL_DECODER_CTX *ctx, input_structure != NULL ? input_structure : ""); } OSSL_TRACE_END(DECODER); - /* Allocate data. */ - if ((process_data = OPENSSL_zalloc(sizeof(*process_data))) == NULL) - goto err; - if ((propquery != NULL - && (process_data->propq = OPENSSL_strdup(propquery)) == NULL)) - goto err; - - /* Allocate our list of EVP_KEYMGMTs. */ - keymgmts = sk_EVP_KEYMGMT_new_null(); - if (keymgmts == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB); + if ((process_data = OPENSSL_zalloc(sizeof(*process_data))) == NULL + || (propquery != NULL + && (process_data->propq = OPENSSL_strdup(propquery)) == NULL) + || (process_data->keymgmts = sk_EVP_KEYMGMT_new_null()) == NULL + || (names = sk_OPENSSL_CSTRING_new_null()) == NULL) { + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE); goto err; } - process_data->object = NULL; - process_data->libctx = libctx; + process_data->object = (void **)pkey; + process_data->libctx = libctx; process_data->selection = ctx->selection; - process_data->keymgmts = keymgmts; - /* - * Enumerate all keymgmts into a stack. - * - * We could nest EVP_KEYMGMT_do_all_provided inside - * OSSL_DECODER_do_all_provided or vice versa but these functions become - * bottlenecks if called repeatedly, which is why we collect the - * EVP_KEYMGMTs into a stack here and call both functions only once. - * - * We resolve the keytype string to a name ID so we don't have to resolve it - * multiple times, avoiding repeated calls to EVP_KEYMGMT_is_a, which is a - * performance bottleneck. However, we do this lazily on the first call to - * collect_keymgmt made by EVP_KEYMGMT_do_all_provided, rather than do it - * upfront, as this ensures that the names for all loaded providers have - * been registered by the time we try to resolve the keytype string. - */ - collect_data.ctx = ctx; - collect_data.libctx = libctx; - collect_data.keymgmts = keymgmts; - collect_data.keytype = keytype; - EVP_KEYMGMT_do_all_provided(libctx, collect_keymgmt, &collect_data); + /* First, find all keymgmts to form goals */ + EVP_KEYMGMT_do_all_provided(libctx, collect_keymgmt, + process_data->keymgmts); - if (collect_data.error_occurred) - goto err; - - /* Enumerate all matching decoders. */ - OSSL_DECODER_do_all_provided(libctx, collect_decoder, &collect_data); + /* Then, we collect all the keymgmt names */ + end = sk_EVP_KEYMGMT_num(process_data->keymgmts); + for (i = 0; i < end; i++) { + EVP_KEYMGMT *keymgmt = sk_EVP_KEYMGMT_value(process_data->keymgmts, i); - if (collect_data.error_occurred) - goto err; + /* + * If the key type is given by the caller, we only use the matching + * KEYMGMTs, otherwise we use them all. + * Note: special case SM2 here because SM2 can be wrapped by EC with + * the EC OID and EC pem header. So if we see EC OID or literal, we try + * both key types. + */ + if (keytype == NULL + || EVP_KEYMGMT_is_a(keymgmt, keytype) + || (is_ec && EVP_KEYMGMT_is_a(keymgmt, "SM2"))) { + if (!EVP_KEYMGMT_names_do_all(keymgmt, collect_name, names)) { + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_INTERNAL_ERROR); + goto err; + } + } + } OSSL_TRACE_BEGIN(DECODER) { + end = sk_OPENSSL_CSTRING_num(names); BIO_printf(trc_out, - "(ctx %p) Got %d decoders producing keys\n", - (void *)ctx, collect_data.total); + " Found %d keytypes (possibly with duplicates)", + end); + for (i = 0; i < end; i++) + BIO_printf(trc_out, "%s%s", + i == 0 ? ": " : ", ", + sk_OPENSSL_CSTRING_value(names, i)); + BIO_printf(trc_out, "\n"); } OSSL_TRACE_END(DECODER); /* - * Finish initializing the decoder context. If one or more decoders matched - * above then the number of decoders attached to the OSSL_DECODER_CTX will - * be nonzero. Else nothing was found and we do nothing. + * Finally, find all decoders that have any keymgmt of the collected + * keymgmt names */ + { + struct collect_decoder_data_st collect_decoder_data = { NULL, }; + + collect_decoder_data.names = names; + collect_decoder_data.ctx = ctx; + OSSL_DECODER_do_all_provided(libctx, + collect_decoder, &collect_decoder_data); + sk_OPENSSL_CSTRING_free(names); + names = NULL; + + if (collect_decoder_data.error_occurred) + goto err; + + OSSL_TRACE_BEGIN(DECODER) { + BIO_printf(trc_out, + "(ctx %p) Got %d decoders producing keys\n", + (void *)ctx, collect_decoder_data.total); + } OSSL_TRACE_END(DECODER); + } + if (OSSL_DECODER_CTX_get_num_decoders(ctx) != 0) { if (!OSSL_DECODER_CTX_set_construct(ctx, decoder_construct_pkey) || !OSSL_DECODER_CTX_set_construct_data(ctx, process_data) @@ -473,268 +418,9 @@ static int ossl_decoder_ctx_setup_for_pkey(OSSL_DECODER_CTX *ctx, ok = 1; err: decoder_clean_pkey_construct_arg(process_data); - return ok; -} - -/* Only const here because deep_copy requires it */ -static EVP_KEYMGMT *keymgmt_dup(const EVP_KEYMGMT *keymgmt) -{ - if (!EVP_KEYMGMT_up_ref((EVP_KEYMGMT *)keymgmt)) - return NULL; - - return (EVP_KEYMGMT *)keymgmt; -} - -/* - * Duplicates a template OSSL_DECODER_CTX that has been setup for an EVP_PKEY - * operation and sets up the duplicate for a new operation. - * It does not duplicate the pwdata on the assumption that this does not form - * part of the template. That is set up later. - */ -static OSSL_DECODER_CTX * -ossl_decoder_ctx_for_pkey_dup(OSSL_DECODER_CTX *src, - EVP_PKEY **pkey, - const char *input_type, - const char *input_structure) -{ - OSSL_DECODER_CTX *dest; - struct decoder_pkey_data_st *process_data_src, *process_data_dest = NULL; - - if (src == NULL) - return NULL; - - if ((dest = OSSL_DECODER_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - return NULL; - } - - if (!OSSL_DECODER_CTX_set_input_type(dest, input_type) - || !OSSL_DECODER_CTX_set_input_structure(dest, input_structure)) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - goto err; - } - dest->selection = src->selection; - - if (src->decoder_insts != NULL) { - dest->decoder_insts - = sk_OSSL_DECODER_INSTANCE_deep_copy(src->decoder_insts, - ossl_decoder_instance_dup, - ossl_decoder_instance_free); - if (dest->decoder_insts == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - goto err; - } - } - - if (!OSSL_DECODER_CTX_set_construct(dest, - OSSL_DECODER_CTX_get_construct(src))) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - goto err; - } - - process_data_src = OSSL_DECODER_CTX_get_construct_data(src); - if (process_data_src != NULL) { - process_data_dest = OPENSSL_zalloc(sizeof(*process_data_dest)); - if (process_data_dest == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB); - goto err; - } - if (process_data_src->propq != NULL) { - process_data_dest->propq = OPENSSL_strdup(process_data_src->propq); - if (process_data_dest->propq == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB); - goto err; - } - } - - if (process_data_src->keymgmts != NULL) { - process_data_dest->keymgmts - = sk_EVP_KEYMGMT_deep_copy(process_data_src->keymgmts, - keymgmt_dup, - EVP_KEYMGMT_free); - if (process_data_dest->keymgmts == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_EVP_LIB); - goto err; - } - } - - process_data_dest->object = (void **)pkey; - process_data_dest->libctx = process_data_src->libctx; - process_data_dest->selection = process_data_src->selection; - if (!OSSL_DECODER_CTX_set_construct_data(dest, process_data_dest)) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - goto err; - } - process_data_dest = NULL; - } - - if (!OSSL_DECODER_CTX_set_cleanup(dest, - OSSL_DECODER_CTX_get_cleanup(src))) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - goto err; - } - - return dest; - err: - if (process_data_dest != NULL) { - OPENSSL_free(process_data_dest->propq); - sk_EVP_KEYMGMT_pop_free(process_data_dest->keymgmts, EVP_KEYMGMT_free); - OPENSSL_free(process_data_dest); - } - OSSL_DECODER_CTX_free(dest); - return NULL; -} - -typedef struct { - char *input_type; - char *input_structure; - char *keytype; - int selection; - char *propquery; - OSSL_DECODER_CTX *template; -} DECODER_CACHE_ENTRY; - -DEFINE_LHASH_OF_EX(DECODER_CACHE_ENTRY); - -typedef struct { - CRYPTO_RWLOCK *lock; - LHASH_OF(DECODER_CACHE_ENTRY) *hashtable; -} DECODER_CACHE; - -static void decoder_cache_entry_free(DECODER_CACHE_ENTRY *entry) -{ - if (entry == NULL) - return; - OPENSSL_free(entry->input_type); - OPENSSL_free(entry->input_structure); - OPENSSL_free(entry->keytype); - OPENSSL_free(entry->propquery); - OSSL_DECODER_CTX_free(entry->template); - OPENSSL_free(entry); -} - -static unsigned long decoder_cache_entry_hash(const DECODER_CACHE_ENTRY *cache) -{ - unsigned long hash = 17; - - hash = (hash * 23) - + (cache->propquery == NULL - ? 0 : ossl_lh_strcasehash(cache->propquery)); - hash = (hash * 23) - + (cache->input_structure == NULL - ? 0 : ossl_lh_strcasehash(cache->input_structure)); - hash = (hash * 23) - + (cache->input_type == NULL - ? 0 : ossl_lh_strcasehash(cache->input_type)); - hash = (hash * 23) - + (cache->keytype == NULL - ? 0 : ossl_lh_strcasehash(cache->keytype)); - - hash ^= cache->selection; - - return hash; -} - -static ossl_inline int nullstrcmp(const char *a, const char *b, int casecmp) -{ - if (a == NULL || b == NULL) { - if (a == NULL) { - if (b == NULL) - return 0; - else - return 1; - } else { - return -1; - } - } else { - if (casecmp) - return OPENSSL_strcasecmp(a, b); - else - return strcmp(a, b); - } -} - -static int decoder_cache_entry_cmp(const DECODER_CACHE_ENTRY *a, - const DECODER_CACHE_ENTRY *b) -{ - int cmp; - - if (a->selection != b->selection) - return (a->selection < b->selection) ? -1 : 1; - - cmp = nullstrcmp(a->keytype, b->keytype, 1); - if (cmp != 0) - return cmp; - - cmp = nullstrcmp(a->input_type, b->input_type, 1); - if (cmp != 0) - return cmp; - - cmp = nullstrcmp(a->input_structure, b->input_structure, 1); - if (cmp != 0) - return cmp; - - cmp = nullstrcmp(a->propquery, b->propquery, 0); - - return cmp; -} - -void *ossl_decoder_cache_new(OSSL_LIB_CTX *ctx) -{ - DECODER_CACHE *cache = OPENSSL_malloc(sizeof(*cache)); - - if (cache == NULL) - return NULL; - - cache->lock = CRYPTO_THREAD_lock_new(); - if (cache->lock == NULL) { - OPENSSL_free(cache); - return NULL; - } - cache->hashtable = lh_DECODER_CACHE_ENTRY_new(decoder_cache_entry_hash, - decoder_cache_entry_cmp); - if (cache->hashtable == NULL) { - CRYPTO_THREAD_lock_free(cache->lock); - OPENSSL_free(cache); - return NULL; - } - - return cache; -} - -void ossl_decoder_cache_free(void *vcache) -{ - DECODER_CACHE *cache = (DECODER_CACHE *)vcache; - - lh_DECODER_CACHE_ENTRY_doall(cache->hashtable, decoder_cache_entry_free); - lh_DECODER_CACHE_ENTRY_free(cache->hashtable); - CRYPTO_THREAD_lock_free(cache->lock); - OPENSSL_free(cache); -} - -/* - * Called whenever a provider gets activated/deactivated. In that case the - * decoders that are available might change so we flush our cache. - */ -int ossl_decoder_cache_flush(OSSL_LIB_CTX *libctx) -{ - DECODER_CACHE *cache - = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DECODER_CACHE_INDEX); - - if (cache == NULL) - return 0; + sk_OPENSSL_CSTRING_free(names); - - if (!CRYPTO_THREAD_write_lock(cache->lock)) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - return 0; - } - - lh_DECODER_CACHE_ENTRY_doall(cache->hashtable, decoder_cache_entry_free); - lh_DECODER_CACHE_ENTRY_flush(cache->hashtable); - - CRYPTO_THREAD_unlock(cache->lock); - return 1; + return ok; } OSSL_DECODER_CTX * @@ -745,135 +431,33 @@ OSSL_DECODER_CTX_new_for_pkey(EVP_PKEY **pkey, OSSL_LIB_CTX *libctx, const char *propquery) { OSSL_DECODER_CTX *ctx = NULL; - OSSL_PARAM decoder_params[] = { - OSSL_PARAM_END, - OSSL_PARAM_END - }; - DECODER_CACHE *cache - = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DECODER_CACHE_INDEX); - DECODER_CACHE_ENTRY cacheent, *res, *newcache = NULL; - - if (cache == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - return NULL; - } - if (propquery != NULL) - decoder_params[0] = OSSL_PARAM_construct_utf8_string(OSSL_DECODER_PARAM_PROPERTIES, - (char *)propquery, 0); - - /* It is safe to cast away the const here */ - cacheent.input_type = (char *)input_type; - cacheent.input_structure = (char *)input_structure; - cacheent.keytype = (char *)keytype; - cacheent.selection = selection; - cacheent.propquery = (char *)propquery; - - if (!CRYPTO_THREAD_read_lock(cache->lock)) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB); + + if ((ctx = OSSL_DECODER_CTX_new()) == NULL) { + ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_MALLOC_FAILURE); return NULL; } - /* First see if we have a template OSSL_DECODER_CTX */ - res = lh_DECODER_CACHE_ENTRY_retrieve(cache->hashtable, &cacheent); - - if (res == NULL) { - /* - * There is no template so we will have to construct one. This will be - * time consuming so release the lock and we will later upgrade it to a - * write lock. - */ - CRYPTO_THREAD_unlock(cache->lock); - - if ((ctx = OSSL_DECODER_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - return NULL; - } + OSSL_TRACE_BEGIN(DECODER) { + BIO_printf(trc_out, + "(ctx %p) Looking for %s decoders with selection %d\n", + (void *)ctx, keytype, selection); + BIO_printf(trc_out, " input type: %s, input structure: %s\n", + input_type, input_structure); + } OSSL_TRACE_END(DECODER); + if (OSSL_DECODER_CTX_set_input_type(ctx, input_type) + && OSSL_DECODER_CTX_set_input_structure(ctx, input_structure) + && OSSL_DECODER_CTX_set_selection(ctx, selection) + && ossl_decoder_ctx_setup_for_pkey(ctx, pkey, keytype, + libctx, propquery) + && OSSL_DECODER_CTX_add_extra(ctx, libctx, propquery)) { OSSL_TRACE_BEGIN(DECODER) { - BIO_printf(trc_out, - "(ctx %p) Looking for %s decoders with selection %d\n", - (void *)ctx, keytype, selection); - BIO_printf(trc_out, " input type: %s, input structure: %s\n", - input_type, input_structure); + BIO_printf(trc_out, "(ctx %p) Got %d decoders\n", + (void *)ctx, OSSL_DECODER_CTX_get_num_decoders(ctx)); } OSSL_TRACE_END(DECODER); - - if (OSSL_DECODER_CTX_set_input_type(ctx, input_type) - && OSSL_DECODER_CTX_set_input_structure(ctx, input_structure) - && OSSL_DECODER_CTX_set_selection(ctx, selection) - && ossl_decoder_ctx_setup_for_pkey(ctx, keytype, libctx, propquery) - && OSSL_DECODER_CTX_add_extra(ctx, libctx, propquery) - && (propquery == NULL - || OSSL_DECODER_CTX_set_params(ctx, decoder_params))) { - OSSL_TRACE_BEGIN(DECODER) { - BIO_printf(trc_out, "(ctx %p) Got %d decoders\n", - (void *)ctx, OSSL_DECODER_CTX_get_num_decoders(ctx)); - } OSSL_TRACE_END(DECODER); - } else { - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_OSSL_DECODER_LIB); - OSSL_DECODER_CTX_free(ctx); - return NULL; - } - - newcache = OPENSSL_zalloc(sizeof(*newcache)); - if (newcache == NULL) { - OSSL_DECODER_CTX_free(ctx); - return NULL; - } - - if (input_type != NULL) { - newcache->input_type = OPENSSL_strdup(input_type); - if (newcache->input_type == NULL) - goto err; - } - if (input_structure != NULL) { - newcache->input_structure = OPENSSL_strdup(input_structure); - if (newcache->input_structure == NULL) - goto err; - } - if (keytype != NULL) { - newcache->keytype = OPENSSL_strdup(keytype); - if (newcache->keytype == NULL) - goto err; - } - if (propquery != NULL) { - newcache->propquery = OPENSSL_strdup(propquery); - if (newcache->propquery == NULL) - goto err; - } - newcache->selection = selection; - newcache->template = ctx; - - if (!CRYPTO_THREAD_write_lock(cache->lock)) { - ctx = NULL; - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB); - goto err; - } - res = lh_DECODER_CACHE_ENTRY_retrieve(cache->hashtable, &cacheent); - if (res == NULL) { - (void)lh_DECODER_CACHE_ENTRY_insert(cache->hashtable, newcache); - if (lh_DECODER_CACHE_ENTRY_error(cache->hashtable)) { - ctx = NULL; - ERR_raise(ERR_LIB_OSSL_DECODER, ERR_R_CRYPTO_LIB); - goto err; - } - } else { - /* - * We raced with another thread to construct this and lost. Free - * what we just created and use the entry from the hashtable instead - */ - decoder_cache_entry_free(newcache); - ctx = res->template; - } - } else { - ctx = res->template; + return ctx; } - ctx = ossl_decoder_ctx_for_pkey_dup(ctx, pkey, input_type, input_structure); - CRYPTO_THREAD_unlock(cache->lock); - - return ctx; - err: - decoder_cache_entry_free(newcache); OSSL_DECODER_CTX_free(ctx); return NULL; } diff --git a/openssl/src/crypto/encode_decode/encoder_lib.c b/openssl/src/crypto/encode_decode/encoder_lib.c index d1e3aca7a..7a55c7ab9 100644 --- a/openssl/src/crypto/encode_decode/encoder_lib.c +++ b/openssl/src/crypto/encode_decode/encoder_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -59,11 +59,6 @@ int OSSL_ENCODER_to_bio(OSSL_ENCODER_CTX *ctx, BIO *out) return 0; } - if (ctx->cleanup == NULL || ctx->construct == NULL) { - ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_INIT_FAIL); - return 0; - } - return encoder_process(&data) > 0; } @@ -198,8 +193,10 @@ static OSSL_ENCODER_INSTANCE *ossl_encoder_instance_new(OSSL_ENCODER *encoder, return 0; } - if ((encoder_inst = OPENSSL_zalloc(sizeof(*encoder_inst))) == NULL) + if ((encoder_inst = OPENSSL_zalloc(sizeof(*encoder_inst))) == NULL) { + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE); return 0; + } if (!OSSL_ENCODER_up_ref(encoder)) { ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_INTERNAL_ERROR); @@ -262,7 +259,7 @@ static int ossl_encoder_ctx_add_encoder_inst(OSSL_ENCODER_CTX *ctx, if (ctx->encoder_insts == NULL && (ctx->encoder_insts = sk_OSSL_ENCODER_INSTANCE_new_null()) == NULL) { - ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE); return 0; } @@ -525,7 +522,7 @@ static int encoder_process(struct encoder_process_data_st *data) OSSL_TRACE_BEGIN(ENCODER) { BIO_printf(trc_out, - "[%d] Skipping because recursion level %d failed\n", + "[%d] Skipping because recusion level %d failed\n", data->level, new_data.level); } OSSL_TRACE_END(ENCODER); } diff --git a/openssl/src/crypto/encode_decode/encoder_local.h b/openssl/src/crypto/encode_decode/encoder_local.h index 91e601aea..c1885ffc7 100644 --- a/openssl/src/crypto/encode_decode/encoder_local.h +++ b/openssl/src/crypto/encode_decode/encoder_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,6 +25,7 @@ struct ossl_endecode_base_st { OSSL_PROPERTY_LIST *parsed_propdef; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; }; struct ossl_encoder_st { @@ -107,7 +108,6 @@ struct ossl_decoder_instance_st { void *decoderctx; /* Never NULL */ const char *input_type; /* Never NULL */ const char *input_structure; /* May be NULL */ - int input_type_id; unsigned int flag_input_structure_was_set : 1; }; @@ -162,6 +162,3 @@ const OSSL_PROPERTY_LIST * ossl_decoder_parsed_properties(const OSSL_DECODER *decoder); const OSSL_PROPERTY_LIST * ossl_encoder_parsed_properties(const OSSL_ENCODER *encoder); - -int ossl_decoder_fast_is_a(OSSL_DECODER *decoder, - const char *name, int *id_cache); diff --git a/openssl/src/crypto/encode_decode/encoder_meth.c b/openssl/src/crypto/encode_decode/encoder_meth.c index adf34bbb9..eb8fd2f45 100644 --- a/openssl/src/crypto/encode_decode/encoder_meth.c +++ b/openssl/src/crypto/encode_decode/encoder_meth.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,7 +17,6 @@ #include "internal/provider.h" #include "crypto/encoder.h" #include "encoder_local.h" -#include "crypto/context.h" /* * Encoder can have multiple names, separated with colons in a name string @@ -29,13 +28,15 @@ static OSSL_ENCODER *ossl_encoder_new(void) { OSSL_ENCODER *encoder = NULL; - if ((encoder = OPENSSL_zalloc(sizeof(*encoder))) == NULL) - return NULL; - if (!CRYPTO_NEW_REF(&encoder->base.refcnt, 1)) { + if ((encoder = OPENSSL_zalloc(sizeof(*encoder))) == NULL + || (encoder->base.lock = CRYPTO_THREAD_lock_new()) == NULL) { OSSL_ENCODER_free(encoder); + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE); return NULL; } + encoder->base.refcnt = 1; + return encoder; } @@ -43,7 +44,7 @@ int OSSL_ENCODER_up_ref(OSSL_ENCODER *encoder) { int ref = 0; - CRYPTO_UP_REF(&encoder->base.refcnt, &ref); + CRYPTO_UP_REF(&encoder->base.refcnt, &ref, encoder->base.lock); return 1; } @@ -54,16 +55,35 @@ void OSSL_ENCODER_free(OSSL_ENCODER *encoder) if (encoder == NULL) return; - CRYPTO_DOWN_REF(&encoder->base.refcnt, &ref); + CRYPTO_DOWN_REF(&encoder->base.refcnt, &ref, encoder->base.lock); if (ref > 0) return; OPENSSL_free(encoder->base.name); ossl_property_free(encoder->base.parsed_propdef); ossl_provider_free(encoder->base.prov); - CRYPTO_FREE_REF(&encoder->base.refcnt); + CRYPTO_THREAD_lock_free(encoder->base.lock); OPENSSL_free(encoder); } +/* Permanent encoder method store, constructor and destructor */ +static void encoder_store_free(void *vstore) +{ + ossl_method_store_free(vstore); +} + +static void *encoder_store_new(OSSL_LIB_CTX *ctx) +{ + return ossl_method_store_new(ctx); +} + + +static const OSSL_LIB_CTX_METHOD encoder_store_method = { + /* We want encoder_store to be cleaned up before the provider store */ + OSSL_LIB_CTX_METHOD_PRIORITY_2, + encoder_store_new, + encoder_store_free, +}; + /* Data to be passed through ossl_method_construct() */ struct encoder_data_st { OSSL_LIB_CTX *libctx; @@ -100,29 +120,8 @@ static void dealloc_tmp_encoder_store(void *store) /* Get the permanent encoder store */ static OSSL_METHOD_STORE *get_encoder_store(OSSL_LIB_CTX *libctx) { - return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_ENCODER_STORE_INDEX); -} - -static int reserve_encoder_store(void *store, void *data) -{ - struct encoder_data_st *methdata = data; - - if (store == NULL - && (store = get_encoder_store(methdata->libctx)) == NULL) - return 0; - - return ossl_method_lock_store(store); -} - -static int unreserve_encoder_store(void *store, void *data) -{ - struct encoder_data_st *methdata = data; - - if (store == NULL - && (store = get_encoder_store(methdata->libctx)) == NULL) - return 0; - - return ossl_method_unlock_store(store); + return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_ENCODER_STORE_INDEX, + &encoder_store_method); } /* Get encoder methods from a store, or put one in */ @@ -211,11 +210,8 @@ static void *encoder_from_algorithm(int id, const OSSL_ALGORITHM *algodef, return NULL; } encoder->base.algodef = algodef; - if ((encoder->base.parsed_propdef - = ossl_parse_property(libctx, algodef->property_definition)) == NULL) { - OSSL_ENCODER_free(encoder); - return NULL; - } + encoder->base.parsed_propdef + = ossl_parse_property(libctx, algodef->property_definition); for (; fns->function_id != 0; fns++) { switch (fns->function_id) { @@ -348,34 +344,43 @@ static void free_encoder(void *method) /* Fetching support. Can fetch by numeric identity or by name */ static OSSL_ENCODER * -inner_ossl_encoder_fetch(struct encoder_data_st *methdata, +inner_ossl_encoder_fetch(struct encoder_data_st *methdata, int id, const char *name, const char *properties) { OSSL_METHOD_STORE *store = get_encoder_store(methdata->libctx); OSSL_NAMEMAP *namemap = ossl_namemap_stored(methdata->libctx); const char *const propq = properties != NULL ? properties : ""; void *method = NULL; - int unsupported, id; + int unsupported = 0; if (store == NULL || namemap == NULL) { ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_INVALID_ARGUMENT); return NULL; } - id = name != NULL ? ossl_namemap_name2num(namemap, name) : 0; + /* + * If we have been passed both an id and a name, we have an + * internal programming error. + */ + if (!ossl_assert(id == 0 || name == NULL)) { + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_INTERNAL_ERROR); + return NULL; + } + + if (id == 0) + id = ossl_namemap_name2num(namemap, name); /* * If we haven't found the name yet, chances are that the algorithm to * be fetched is unsupported. */ - unsupported = id == 0; + if (id == 0) + unsupported = 1; if (id == 0 || !ossl_method_store_cache_get(store, NULL, id, propq, &method)) { OSSL_METHOD_CONSTRUCT_METHOD mcm = { get_tmp_encoder_store, - reserve_encoder_store, - unreserve_encoder_store, get_encoder_from_store, put_encoder_in_store, construct_encoder, @@ -417,7 +422,7 @@ inner_ossl_encoder_fetch(struct encoder_data_st *methdata, ERR_raise_data(ERR_LIB_OSSL_ENCODER, code, "%s, Name (%s : %d), Properties (%s)", ossl_lib_ctx_get_descriptor(methdata->libctx), - name == NULL ? "" : name, id, + name = NULL ? "" : name, id, properties == NULL ? "" : properties); } @@ -432,28 +437,22 @@ OSSL_ENCODER *OSSL_ENCODER_fetch(OSSL_LIB_CTX *libctx, const char *name, methdata.libctx = libctx; methdata.tmp_store = NULL; - method = inner_ossl_encoder_fetch(&methdata, name, properties); + method = inner_ossl_encoder_fetch(&methdata, 0, name, properties); dealloc_tmp_encoder_store(methdata.tmp_store); return method; } -int ossl_encoder_store_cache_flush(OSSL_LIB_CTX *libctx) +OSSL_ENCODER *ossl_encoder_fetch_by_number(OSSL_LIB_CTX *libctx, int id, + const char *properties) { - OSSL_METHOD_STORE *store = get_encoder_store(libctx); - - if (store != NULL) - return ossl_method_store_cache_flush_all(store); - return 1; -} - -int ossl_encoder_store_remove_all_provided(const OSSL_PROVIDER *prov) -{ - OSSL_LIB_CTX *libctx = ossl_provider_libctx(prov); - OSSL_METHOD_STORE *store = get_encoder_store(libctx); + struct encoder_data_st methdata; + void *method; - if (store != NULL) - return ossl_method_store_remove_all_provided(store, prov); - return 1; + methdata.libctx = libctx; + methdata.tmp_store = NULL; + method = inner_ossl_encoder_fetch(&methdata, id, NULL, properties); + dealloc_tmp_encoder_store(methdata.tmp_store); + return method; } /* @@ -544,7 +543,7 @@ void OSSL_ENCODER_do_all_provided(OSSL_LIB_CTX *libctx, methdata.libctx = libctx; methdata.tmp_store = NULL; - (void)inner_ossl_encoder_fetch(&methdata, NULL, NULL /* properties */); + (void)inner_ossl_encoder_fetch(&methdata, 0, NULL, NULL /* properties */); data.user_fn = user_fn; data.user_arg = user_arg; @@ -607,7 +606,9 @@ OSSL_ENCODER_CTX *OSSL_ENCODER_CTX_new(void) { OSSL_ENCODER_CTX *ctx; - ctx = OPENSSL_zalloc(sizeof(*ctx)); + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE); + return ctx; } diff --git a/openssl/src/crypto/encode_decode/encoder_pkey.c b/openssl/src/crypto/encode_decode/encoder_pkey.c index 29060c5f9..3a24317cf 100644 --- a/openssl/src/crypto/encode_decode/encoder_pkey.c +++ b/openssl/src/crypto/encode_decode/encoder_pkey.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,7 +17,6 @@ #include #include "internal/provider.h" #include "internal/property.h" -#include "internal/namemap.h" #include "crypto/evp.h" #include "encoder_local.h" @@ -73,7 +72,6 @@ int OSSL_ENCODER_CTX_set_passphrase_cb(OSSL_ENCODER_CTX *ctx, struct collected_encoder_st { STACK_OF(OPENSSL_CSTRING) *names; - int *id_names; const char *output_structure; const char *output_type; @@ -87,42 +85,41 @@ struct collected_encoder_st { static void collect_encoder(OSSL_ENCODER *encoder, void *arg) { struct collected_encoder_st *data = arg; - const OSSL_PROVIDER *prov; + size_t i, end_i; if (data->error_occurred) return; data->error_occurred = 1; /* Assume the worst */ - prov = OSSL_ENCODER_get0_provider(encoder); - /* - * collect_encoder() is called in two passes, one where the encoders - * from the same provider as the keymgmt are looked up, and one where - * the other encoders are looked up. |data->flag_find_same_provider| - * tells us which pass we're in. - */ - if ((data->keymgmt_prov == prov) == data->flag_find_same_provider) { + if (data->names == NULL) + return; + + end_i = sk_OPENSSL_CSTRING_num(data->names); + for (i = 0; i < end_i; i++) { + const char *name = sk_OPENSSL_CSTRING_value(data->names, i); + const OSSL_PROVIDER *prov = OSSL_ENCODER_get0_provider(encoder); void *provctx = OSSL_PROVIDER_get0_provider_ctx(prov); - int i, end_i = sk_OPENSSL_CSTRING_num(data->names); - int match; - - for (i = 0; i < end_i; i++) { - if (data->flag_find_same_provider) - match = (data->id_names[i] == encoder->base.id); - else - match = OSSL_ENCODER_is_a(encoder, - sk_OPENSSL_CSTRING_value(data->names, i)); - if (!match - || (encoder->does_selection != NULL - && !encoder->does_selection(provctx, data->ctx->selection)) - || (data->keymgmt_prov != prov - && encoder->import_object == NULL)) - continue; - - /* Only add each encoder implementation once */ - if (OSSL_ENCODER_CTX_add_encoder(data->ctx, encoder)) - break; - } + + /* + * collect_encoder() is called in two passes, one where the encoders + * from the same provider as the keymgmt are looked up, and one where + * the other encoders are looked up. |data->flag_find_same_provider| + * tells us which pass we're in. + */ + if ((data->keymgmt_prov == prov) != data->flag_find_same_provider) + continue; + + if (!OSSL_ENCODER_is_a(encoder, name) + || (encoder->does_selection != NULL + && !encoder->does_selection(provctx, data->ctx->selection)) + || (data->keymgmt_prov != prov + && encoder->import_object == NULL)) + continue; + + /* Only add each encoder implementation once */ + if (OSSL_ENCODER_CTX_add_encoder(data->ctx, encoder)) + break; } data->error_occurred = 0; /* All is good now */ @@ -230,8 +227,7 @@ static int ossl_encoder_ctx_setup_for_pkey(OSSL_ENCODER_CTX *ctx, struct construct_data_st *data = NULL; const OSSL_PROVIDER *prov = NULL; OSSL_LIB_CTX *libctx = NULL; - int ok = 0, i, end; - OSSL_NAMEMAP *namemap; + int ok = 0; if (!ossl_assert(ctx != NULL) || !ossl_assert(pkey != NULL)) { ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_PASSED_NULL_PARAMETER); @@ -247,8 +243,10 @@ static int ossl_encoder_ctx_setup_for_pkey(OSSL_ENCODER_CTX *ctx, struct collected_encoder_st encoder_data; struct collected_names_st keymgmt_data; - if ((data = OPENSSL_zalloc(sizeof(*data))) == NULL) + if ((data = OPENSSL_zalloc(sizeof(*data))) == NULL) { + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE); goto err; + } /* * Select the first encoder implementations in two steps. @@ -256,7 +254,7 @@ static int ossl_encoder_ctx_setup_for_pkey(OSSL_ENCODER_CTX *ctx, */ keymgmt_data.names = sk_OPENSSL_CSTRING_new_null(); if (keymgmt_data.names == NULL) { - ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE); goto err; } @@ -273,27 +271,7 @@ static int ossl_encoder_ctx_setup_for_pkey(OSSL_ENCODER_CTX *ctx, encoder_data.error_occurred = 0; encoder_data.keymgmt_prov = prov; encoder_data.ctx = ctx; - encoder_data.id_names = NULL; - /* - * collect_encoder() is called many times, and for every call it converts all encoder_data.names - * into namemap ids if it calls OSSL_ENCODER_is_a(). We cache the ids here instead, - * and can use them for encoders with the same provider as the keymgmt. - */ - namemap = ossl_namemap_stored(libctx); - end = sk_OPENSSL_CSTRING_num(encoder_data.names); - if (end > 0) { - encoder_data.id_names = OPENSSL_malloc(end * sizeof(int)); - if (encoder_data.id_names == NULL) { - sk_OPENSSL_CSTRING_free(keymgmt_data.names); - goto err; - } - for (i = 0; i < end; ++i) { - const char *name = sk_OPENSSL_CSTRING_value(keymgmt_data.names, i); - - encoder_data.id_names[i] = ossl_namemap_name2num(namemap, name); - } - } /* * Place the encoders with the a different provider as the keymgmt * last (the chain is processed in reverse order) @@ -308,10 +286,9 @@ static int ossl_encoder_ctx_setup_for_pkey(OSSL_ENCODER_CTX *ctx, encoder_data.flag_find_same_provider = 1; OSSL_ENCODER_do_all_provided(libctx, collect_encoder, &encoder_data); - OPENSSL_free(encoder_data.id_names); sk_OPENSSL_CSTRING_free(keymgmt_data.names); if (encoder_data.error_occurred) { - ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE); goto err; } } @@ -358,7 +335,7 @@ OSSL_ENCODER_CTX *OSSL_ENCODER_CTX_new_for_pkey(const EVP_PKEY *pkey, } if ((ctx = OSSL_ENCODER_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_OSSL_ENCODER_LIB); + ERR_raise(ERR_LIB_OSSL_ENCODER, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/engine/eng_ctrl.c b/openssl/src/crypto/engine/eng_ctrl.c index f1da9b23b..5d7e15634 100644 --- a/openssl/src/crypto/engine/eng_ctrl.c +++ b/openssl/src/crypto/engine/eng_ctrl.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -127,15 +127,20 @@ static int int_ctrl_helper(ENGINE *e, int cmd, long i, void *p, int ENGINE_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void)) { - int ctrl_exists; - + int ctrl_exists, ref_exists; if (e == NULL) { ERR_raise(ERR_LIB_ENGINE, ERR_R_PASSED_NULL_PARAMETER); return 0; } - + if (!CRYPTO_THREAD_write_lock(global_engine_lock)) + return 0; + ref_exists = ((e->struct_ref > 0) ? 1 : 0); + CRYPTO_THREAD_unlock(global_engine_lock); ctrl_exists = ((e->ctrl == NULL) ? 0 : 1); - + if (!ref_exists) { + ERR_raise(ERR_LIB_ENGINE, ENGINE_R_NO_REFERENCE); + return 0; + } /* * Intercept any "root-level" commands before trying to hand them on to * ctrl() handlers. diff --git a/openssl/src/crypto/engine/eng_dyn.c b/openssl/src/crypto/engine/eng_dyn.c index cc3a2b0aa..6d402927c 100644 --- a/openssl/src/crypto/engine/eng_dyn.c +++ b/openssl/src/crypto/engine/eng_dyn.c @@ -159,11 +159,13 @@ static int dynamic_set_data_ctx(ENGINE *e, dynamic_data_ctx **ctx) dynamic_data_ctx *c = OPENSSL_zalloc(sizeof(*c)); int ret = 0; - if (c == NULL) + if (c == NULL) { + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return 0; + } c->dirs = sk_OPENSSL_STRING_new_null(); if (c->dirs == NULL) { - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); goto end; } c->DYNAMIC_F1 = "v_check"; @@ -355,11 +357,13 @@ static int dynamic_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void)) } { char *tmp_str = OPENSSL_strdup(p); - if (tmp_str == NULL) + if (tmp_str == NULL) { + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return 0; + } if (!sk_OPENSSL_STRING_push(ctx->dirs, tmp_str)) { OPENSSL_free(tmp_str); - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return 0; } } diff --git a/openssl/src/crypto/engine/eng_err.c b/openssl/src/crypto/engine/eng_err.c index 17c1b7d00..cf097082c 100644 --- a/openssl/src/crypto/engine/eng_err.c +++ b/openssl/src/crypto/engine/eng_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -72,6 +72,8 @@ static const ERR_STRING_DATA ENGINE_str_reasons[] = { "unimplemented cipher"}, {ERR_PACK(ERR_LIB_ENGINE, 0, ENGINE_R_UNIMPLEMENTED_DIGEST), "unimplemented digest"}, + {ERR_PACK(ERR_LIB_ENGINE, 0, ENGINE_R_UNIMPLEMENTED_ECP_METH), + "unimplemented ecp meth"}, {ERR_PACK(ERR_LIB_ENGINE, 0, ENGINE_R_UNIMPLEMENTED_PUBLIC_KEY_METHOD), "unimplemented public key method"}, {ERR_PACK(ERR_LIB_ENGINE, 0, ENGINE_R_VERSION_INCOMPATIBILITY), diff --git a/openssl/src/crypto/engine/eng_fat.c b/openssl/src/crypto/engine/eng_fat.c index 0cf27715c..18ea4b16a 100644 --- a/openssl/src/crypto/engine/eng_fat.c +++ b/openssl/src/crypto/engine/eng_fat.c @@ -36,6 +36,10 @@ int ENGINE_set_default(ENGINE *e, unsigned int flags) #endif if ((flags & ENGINE_METHOD_RAND) && !ENGINE_set_default_RAND(e)) return 0; +#ifndef OPENSSL_NO_BN_METHOD + if ((flags & ENGINE_METHOD_BN) && !ENGINE_set_default_bn_meth(e)) + return 0; +#endif if ((flags & ENGINE_METHOD_PKEY_METHS) && !ENGINE_set_default_pkey_meths(e)) return 0; @@ -74,6 +78,10 @@ static int int_def_cb(const char *alg, int len, void *arg) *pflags |= ENGINE_METHOD_PKEY_METHS; else if (strncmp(alg, "PKEY_ASN1", len) == 0) *pflags |= ENGINE_METHOD_PKEY_ASN1_METHS; +#ifndef OPENSSL_NO_BN_METHOD + else if (strncmp(alg, "BN", len) == 0) + *pflags |= ENGINE_METHOD_BN; +#endif else return 0; return 1; @@ -103,10 +111,14 @@ int ENGINE_register_complete(ENGINE *e) #endif #ifndef OPENSSL_NO_EC ENGINE_register_EC(e); + ENGINE_register_ecp_meths(e); #endif ENGINE_register_RAND(e); ENGINE_register_pkey_meths(e); ENGINE_register_pkey_asn1_meths(e); +#ifndef OPENSSL_NO_BN_METHOD + ENGINE_register_bn_meth(e); +#endif return 1; } diff --git a/openssl/src/crypto/engine/eng_init.c b/openssl/src/crypto/engine/eng_init.c index 0ac91ff5e..a1b9917f4 100644 --- a/openssl/src/crypto/engine/eng_init.c +++ b/openssl/src/crypto/engine/eng_init.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,7 +14,7 @@ #include "eng_local.h" /* - * Initialise an engine type for use (or up its functional reference count if + * Initialise a engine type for use (or up its functional reference count if * it's already in use). This version is only used internally. */ int engine_unlocked_init(ENGINE *e) @@ -28,16 +28,11 @@ int engine_unlocked_init(ENGINE *e) */ to_return = e->init(e); if (to_return) { - int ref; - /* * OK, we return a functional reference which is also a structural * reference. */ - if (!CRYPTO_UP_REF(&e->struct_ref, &ref)) { - e->finish(e); - return 0; - } + e->struct_ref++; e->funct_ref++; ENGINE_REF_PRINT(e, 0, 1); ENGINE_REF_PRINT(e, 1, 1); @@ -46,7 +41,7 @@ int engine_unlocked_init(ENGINE *e) } /* - * Free a functional reference to an engine type. This version is only used + * Free a functional reference to a engine type. This version is only used * internally. */ int engine_unlocked_finish(ENGINE *e, int unlock_for_handlers) @@ -91,8 +86,7 @@ int ENGINE_init(ENGINE *e) return 0; } if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { - /* Maybe this should be raised in do_engine_lock_init() */ - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return 0; } if (!CRYPTO_THREAD_write_lock(global_engine_lock)) diff --git a/openssl/src/crypto/engine/eng_lib.c b/openssl/src/crypto/engine/eng_lib.c index 412363fa3..528520010 100644 --- a/openssl/src/crypto/engine/eng_lib.c +++ b/openssl/src/crypto/engine/eng_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -28,20 +28,14 @@ ENGINE *ENGINE_new(void) { ENGINE *ret; - if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { - /* Maybe this should be raised in do_engine_lock_init() */ - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); - return 0; - } - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) - return NULL; - if (!CRYPTO_NEW_REF(&ret->struct_ref, 1)) { - OPENSSL_free(ret); + if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init) + || (ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return NULL; } + ret->struct_ref = 1; ENGINE_REF_PRINT(ret, 0, 1); if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_ENGINE, ret, &ret->ex_data)) { - CRYPTO_FREE_REF(&ret->struct_ref); OPENSSL_free(ret); return NULL; } @@ -80,7 +74,10 @@ int engine_free_util(ENGINE *e, int not_locked) if (e == NULL) return 1; - CRYPTO_DOWN_REF(&e->struct_ref, &i); + if (not_locked) + CRYPTO_DOWN_REF(&e->struct_ref, &i, global_engine_lock); + else + i = --e->struct_ref; ENGINE_REF_PRINT(e, 0, -1); if (i > 0) return 1; @@ -96,7 +93,6 @@ int engine_free_util(ENGINE *e, int not_locked) e->destroy(e); engine_remove_dynamic_id(e, not_locked); CRYPTO_free_ex_data(CRYPTO_EX_INDEX_ENGINE, e, &e->ex_data); - CRYPTO_FREE_REF(&e->struct_ref); OPENSSL_free(e); return 1; } @@ -129,40 +125,35 @@ static ENGINE_CLEANUP_ITEM *int_cleanup_item(ENGINE_CLEANUP_CB *cb) { ENGINE_CLEANUP_ITEM *item; - if ((item = OPENSSL_malloc(sizeof(*item))) == NULL) + if ((item = OPENSSL_malloc(sizeof(*item))) == NULL) { + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return NULL; + } item->cb = cb; return item; } -int engine_cleanup_add_first(ENGINE_CLEANUP_CB *cb) +void engine_cleanup_add_first(ENGINE_CLEANUP_CB *cb) { ENGINE_CLEANUP_ITEM *item; if (!int_cleanup_check(1)) - return 0; + return; item = int_cleanup_item(cb); - if (item != NULL) { - if (sk_ENGINE_CLEANUP_ITEM_insert(cleanup_stack, item, 0)) - return 1; - OPENSSL_free(item); - } - return 0; + if (item) + sk_ENGINE_CLEANUP_ITEM_insert(cleanup_stack, item, 0); } -int engine_cleanup_add_last(ENGINE_CLEANUP_CB *cb) +void engine_cleanup_add_last(ENGINE_CLEANUP_CB *cb) { ENGINE_CLEANUP_ITEM *item; - if (!int_cleanup_check(1)) - return 0; + return; item = int_cleanup_item(cb); if (item != NULL) { - if (sk_ENGINE_CLEANUP_ITEM_push(cleanup_stack, item) > 0) - return 1; - OPENSSL_free(item); + if (sk_ENGINE_CLEANUP_ITEM_push(cleanup_stack, item) <= 0) + OPENSSL_free(item); } - return 0; } /* The API function that performs all cleanup */ diff --git a/openssl/src/crypto/engine/eng_list.c b/openssl/src/crypto/engine/eng_list.c index a2c151d64..04c73c762 100644 --- a/openssl/src/crypto/engine/eng_list.c +++ b/openssl/src/crypto/engine/eng_list.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -58,7 +58,6 @@ static int engine_list_add(ENGINE *e) { int conflict = 0; ENGINE *iterator = NULL; - int ref; if (e == NULL) { ERR_raise(ERR_LIB_ENGINE, ERR_R_PASSED_NULL_PARAMETER); @@ -73,43 +72,32 @@ static int engine_list_add(ENGINE *e) ERR_raise(ERR_LIB_ENGINE, ENGINE_R_CONFLICTING_ENGINE_ID); return 0; } - - /* - * Having the engine in the list assumes a structural reference. - */ - if (!CRYPTO_UP_REF(&e->struct_ref, &ref)) { - ERR_raise(ERR_LIB_ENGINE, ENGINE_R_INTERNAL_LIST_ERROR); - return 0; - } - ENGINE_REF_PRINT(e, 0, 1); if (engine_list_head == NULL) { /* We are adding to an empty list. */ - if (engine_list_tail != NULL) { - CRYPTO_DOWN_REF(&e->struct_ref, &ref); + if (engine_list_tail) { ERR_raise(ERR_LIB_ENGINE, ENGINE_R_INTERNAL_LIST_ERROR); return 0; } + engine_list_head = e; + e->prev = NULL; /* * The first time the list allocates, we should register the cleanup. */ - if (!engine_cleanup_add_last(engine_list_cleanup)) { - CRYPTO_DOWN_REF(&e->struct_ref, &ref); - ERR_raise(ERR_LIB_ENGINE, ENGINE_R_INTERNAL_LIST_ERROR); - return 0; - } - engine_list_head = e; - e->prev = NULL; + engine_cleanup_add_last(engine_list_cleanup); } else { /* We are adding to the tail of an existing list. */ if ((engine_list_tail == NULL) || (engine_list_tail->next != NULL)) { - CRYPTO_DOWN_REF(&e->struct_ref, &ref); ERR_raise(ERR_LIB_ENGINE, ENGINE_R_INTERNAL_LIST_ERROR); return 0; } engine_list_tail->next = e; e->prev = engine_list_tail; } - + /* + * Having the engine in the list assumes a structural reference. + */ + e->struct_ref++; + ENGINE_REF_PRINT(e, 0, 1); /* However it came to be, e is the last item in the list. */ engine_list_tail = e; e->next = NULL; @@ -231,8 +219,7 @@ ENGINE *ENGINE_get_first(void) ENGINE *ret; if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { - /* Maybe this should be raised in do_engine_lock_init() */ - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return NULL; } @@ -240,12 +227,7 @@ ENGINE *ENGINE_get_first(void) return NULL; ret = engine_list_head; if (ret) { - int ref; - - if (!CRYPTO_UP_REF(&ret->struct_ref, &ref)) { - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); - return NULL; - } + ret->struct_ref++; ENGINE_REF_PRINT(ret, 0, 1); } CRYPTO_THREAD_unlock(global_engine_lock); @@ -257,8 +239,7 @@ ENGINE *ENGINE_get_last(void) ENGINE *ret; if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { - /* Maybe this should be raised in do_engine_lock_init() */ - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return NULL; } @@ -266,12 +247,7 @@ ENGINE *ENGINE_get_last(void) return NULL; ret = engine_list_tail; if (ret) { - int ref; - - if (!CRYPTO_UP_REF(&ret->struct_ref, &ref)) { - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); - return NULL; - } + ret->struct_ref++; ENGINE_REF_PRINT(ret, 0, 1); } CRYPTO_THREAD_unlock(global_engine_lock); @@ -290,13 +266,8 @@ ENGINE *ENGINE_get_next(ENGINE *e) return NULL; ret = e->next; if (ret) { - int ref; - /* Return a valid structural reference to the next ENGINE */ - if (!CRYPTO_UP_REF(&ret->struct_ref, &ref)) { - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); - return NULL; - } + ret->struct_ref++; ENGINE_REF_PRINT(ret, 0, 1); } CRYPTO_THREAD_unlock(global_engine_lock); @@ -316,13 +287,8 @@ ENGINE *ENGINE_get_prev(ENGINE *e) return NULL; ret = e->prev; if (ret) { - int ref; - /* Return a valid structural reference to the next ENGINE */ - if (!CRYPTO_UP_REF(&ret->struct_ref, &ref)) { - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); - return NULL; - } + ret->struct_ref++; ENGINE_REF_PRINT(ret, 0, 1); } CRYPTO_THREAD_unlock(global_engine_lock); @@ -412,8 +378,7 @@ ENGINE *ENGINE_by_id(const char *id) ENGINE_load_builtin_engines(); if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { - /* Maybe this should be raised in do_engine_lock_init() */ - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return NULL; } @@ -437,13 +402,7 @@ ENGINE *ENGINE_by_id(const char *id) iterator = cp; } } else { - int ref; - - if (!CRYPTO_UP_REF(&iterator->struct_ref, &ref)) { - CRYPTO_THREAD_unlock(global_engine_lock); - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); - return NULL; - } + iterator->struct_ref++; ENGINE_REF_PRINT(iterator, 0, 1); } } @@ -480,6 +439,6 @@ int ENGINE_up_ref(ENGINE *e) ERR_raise(ERR_LIB_ENGINE, ERR_R_PASSED_NULL_PARAMETER); return 0; } - CRYPTO_UP_REF(&e->struct_ref, &i); + CRYPTO_UP_REF(&e->struct_ref, &i, global_engine_lock); return 1; } diff --git a/openssl/src/crypto/engine/eng_local.h b/openssl/src/crypto/engine/eng_local.h index 24920973e..03a86299c 100644 --- a/openssl/src/crypto/engine/eng_local.h +++ b/openssl/src/crypto/engine/eng_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -31,8 +31,8 @@ extern CRYPTO_RWLOCK *global_engine_lock; (void *)(e), (isfunct ? "funct" : "struct"), \ ((isfunct) \ ? ((e)->funct_ref - (diff)) \ - : (eng_struct_ref(e) - (diff))), \ - ((isfunct) ? (e)->funct_ref : eng_struct_ref(e)), \ + : ((e)->struct_ref - (diff))), \ + ((isfunct) ? (e)->funct_ref : (e)->struct_ref), \ (OPENSSL_FILE), (OPENSSL_LINE)) /* @@ -46,8 +46,8 @@ typedef struct st_engine_cleanup_item { ENGINE_CLEANUP_CB *cb; } ENGINE_CLEANUP_ITEM; DEFINE_STACK_OF(ENGINE_CLEANUP_ITEM) -int engine_cleanup_add_first(ENGINE_CLEANUP_CB *cb); -int engine_cleanup_add_last(ENGINE_CLEANUP_CB *cb); +void engine_cleanup_add_first(ENGINE_CLEANUP_CB *cb); +void engine_cleanup_add_last(ENGINE_CLEANUP_CB *cb); /* We need stacks of ENGINEs for use in eng_table.c */ DEFINE_STACK_OF(ENGINE) @@ -156,14 +156,6 @@ struct engine_st { typedef struct st_engine_pile ENGINE_PILE; -DEFINE_LHASH_OF_EX(ENGINE_PILE); - -static ossl_unused ossl_inline int eng_struct_ref(ENGINE *e) -{ - int res; - - CRYPTO_GET_REF(&e->struct_ref, &res); - return res; -} +DEFINE_LHASH_OF(ENGINE_PILE); #endif /* OSSL_CRYPTO_ENGINE_ENG_LOCAL_H */ diff --git a/openssl/src/crypto/engine/eng_openssl.c b/openssl/src/crypto/engine/eng_openssl.c index 8b39e3dec..91656e6b8 100644 --- a/openssl/src/crypto/engine/eng_openssl.c +++ b/openssl/src/crypto/engine/eng_openssl.c @@ -450,8 +450,10 @@ static int ossl_hmac_init(EVP_PKEY_CTX *ctx) { OSSL_HMAC_PKEY_CTX *hctx; - if ((hctx = OPENSSL_zalloc(sizeof(*hctx))) == NULL) + if ((hctx = OPENSSL_zalloc(sizeof(*hctx))) == NULL) { + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return 0; + } hctx->ktmp.type = V_ASN1_OCTET_STRING; hctx->ctx = HMAC_CTX_new(); if (hctx->ctx == NULL) { diff --git a/openssl/src/crypto/engine/eng_pkey.c b/openssl/src/crypto/engine/eng_pkey.c index d18d837e6..6e6d6df35 100644 --- a/openssl/src/crypto/engine/eng_pkey.c +++ b/openssl/src/crypto/engine/eng_pkey.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/engine/eng_rdrand.c b/openssl/src/crypto/engine/eng_rdrand.c index b3ece7bd9..f46a51459 100644 --- a/openssl/src/crypto/engine/eng_rdrand.c +++ b/openssl/src/crypto/engine/eng_rdrand.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,12 +20,6 @@ #include #include -#if defined(__has_feature) -# if __has_feature(memory_sanitizer) -# include -# endif -#endif - #if (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || \ defined(_M_AMD64) || defined (_M_X64)) && defined(OPENSSL_CPUID_OBJ) @@ -38,16 +32,6 @@ static int get_random_bytes(unsigned char *buf, int num) return 0; } -# if defined(__has_feature) -# if __has_feature(memory_sanitizer) - /* - * MemorySanitizer fails to understand asm and produces false positive - * use-of-uninitialized-value warnings. - */ - __msan_unpoison(buf, num); -# endif -# endif - return (size_t)num == OPENSSL_ia32_rdrand_bytes(buf, (size_t)num); } diff --git a/openssl/src/crypto/engine/eng_table.c b/openssl/src/crypto/engine/eng_table.c index 9dc3144bb..a8209d9e7 100644 --- a/openssl/src/crypto/engine/eng_table.c +++ b/openssl/src/crypto/engine/eng_table.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -93,12 +93,9 @@ int engine_table_register(ENGINE_TABLE **table, ENGINE_CLEANUP_CB *cleanup, added = 1; if (!int_table_check(table, 1)) goto end; - /* The cleanup callback needs to be added */ - if (added && !engine_cleanup_add_first(cleanup)) { - lh_ENGINE_PILE_free(&(*table)->piles); - *table = NULL; - goto end; - } + if (added) + /* The cleanup callback needs to be added */ + engine_cleanup_add_first(cleanup); while (num_nids--) { tmplate.nid = *nids; fnd = lh_ENGINE_PILE_retrieve(&(*table)->piles, &tmplate); @@ -204,10 +201,8 @@ ENGINE *ossl_engine_table_select(ENGINE_TABLE **table, int nid, ENGINE_PILE tmplate, *fnd = NULL; int initres, loop = 0; -#ifndef OPENSSL_NO_AUTOLOAD_CONFIG /* Load the config before trying to check if engines are available */ OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); -#endif if (!(*table)) { OSSL_TRACE3(ENGINE_TABLE, diff --git a/openssl/src/crypto/engine/tb_asnmth.c b/openssl/src/crypto/engine/tb_asnmth.c index c74fc4700..bd65ede2f 100644 --- a/openssl/src/crypto/engine/tb_asnmth.c +++ b/openssl/src/crypto/engine/tb_asnmth.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -196,23 +196,16 @@ const EVP_PKEY_ASN1_METHOD *ENGINE_pkey_asn1_find_str(ENGINE **pe, fstr.len = len; if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { - /* Maybe this should be raised in do_engine_lock_init() */ - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE); return NULL; } - if (!CRYPTO_THREAD_read_lock(global_engine_lock)) + if (!CRYPTO_THREAD_write_lock(global_engine_lock)) return NULL; engine_table_doall(pkey_asn1_meth_table, look_str_cb, &fstr); /* If found obtain a structural reference to engine */ - if (fstr.e != NULL) { - int ref; - - if (!CRYPTO_UP_REF(&fstr.e->struct_ref, &ref)) { - CRYPTO_THREAD_unlock(global_engine_lock); - ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); - return NULL; - } + if (fstr.e) { + fstr.e->struct_ref++; ENGINE_REF_PRINT(fstr.e, 0, 1); } *pe = fstr.e; diff --git a/openssl/src/crypto/engine/tb_bnmeth.c b/openssl/src/crypto/engine/tb_bnmeth.c new file mode 100644 index 000000000..2204f7c04 --- /dev/null +++ b/openssl/src/crypto/engine/tb_bnmeth.c @@ -0,0 +1,175 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* We need to use some deprecated APIs */ +#define OPENSSL_SUPPRESS_DEPRECATED + +#include "eng_local.h" + +static ENGINE_TABLE *bn_meth_table = NULL; +static int bn_meth_ex_data_idx = -1; +static const int dummy_nid = 1; + +typedef struct bn_meth_data_ctx_st { + const BN_METHOD *bn_meth; +} bn_meth_data_ctx; + +static void bn_meth_data_ctx_free_func(void *parent, void *ptr, + CRYPTO_EX_DATA *ad, int idx, long argl, + void *argp) +{ + if (ptr) { + bn_meth_data_ctx *ctx = (bn_meth_data_ctx *)ptr; + OPENSSL_free(ctx); + } +} + +static int bn_meth_set_data_ctx(ENGINE *e, bn_meth_data_ctx **ctx) +{ + int ret = 1; + bn_meth_data_ctx *c; + + if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { + ENGINEerr(ENGINE_F_BN_METH_SET_DATA_CTX, ERR_R_INTERNAL_ERROR); + return 0; + } + + c = OPENSSL_zalloc(sizeof(*c)); + if (c == NULL) { + ENGINEerr(ENGINE_F_BN_METH_SET_DATA_CTX, ERR_R_MALLOC_FAILURE); + return 0; + } + + if (!CRYPTO_THREAD_write_lock(global_engine_lock)) { + OPENSSL_free(c); + return 0; + } + + if ((*ctx = (bn_meth_data_ctx *)ENGINE_get_ex_data(e, bn_meth_ex_data_idx)) + == NULL) { + /* Good, we're the first */ + ret = ENGINE_set_ex_data(e, bn_meth_ex_data_idx, c); + if (ret) { + *ctx = c; + c = NULL; + } + } + CRYPTO_THREAD_unlock(global_engine_lock); + OPENSSL_free(c); + return ret; +} + +/* + * This function retrieves the context structure from an ENGINE's "ex_data", + * or if it doesn't exist yet, sets it up. + */ +static bn_meth_data_ctx *bn_meth_get_data_ctx(ENGINE *e) +{ + bn_meth_data_ctx *ctx; + + if (bn_meth_ex_data_idx < 0) { + /* + * Create and register the ENGINE ex_data, and associate our "free" + * function with it to ensure any allocated contexts get freed when + * an ENGINE goes underground. + */ + int new_idx = ENGINE_get_ex_new_index(0, NULL, NULL, NULL, + bn_meth_data_ctx_free_func); + if (new_idx == -1) { + ENGINEerr(ENGINE_F_BN_METH_GET_DATA_CTX, ENGINE_R_NO_INDEX); + return NULL; + } + + if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { + ENGINEerr(ENGINE_F_BN_METH_GET_DATA_CTX, ERR_R_INTERNAL_ERROR); + return NULL; + } + + if (!CRYPTO_THREAD_write_lock(global_engine_lock)) + return NULL; + + /* Avoid a race by checking again inside this lock */ + if (bn_meth_ex_data_idx < 0) { + /* Good, someone didn't beat us to it */ + bn_meth_ex_data_idx = new_idx; + new_idx = -1; + } + CRYPTO_THREAD_unlock(global_engine_lock); + /* + * In theory we could "give back" the index here if (new_idx>-1), but + * it's not possible and wouldn't gain us much if it were. + */ + } + ctx = (bn_meth_data_ctx *)ENGINE_get_ex_data(e, bn_meth_ex_data_idx); + /* Check if the context needs to be created */ + if ((ctx == NULL) && !bn_meth_set_data_ctx(e, &ctx)) + /* "set_data" will set errors if necessary */ + return NULL; + return ctx; +} + +static void engine_unregister_all_bn_meth(void) +{ + engine_table_cleanup(&bn_meth_table); +} + +void ENGINE_unregister_bn_meth(ENGINE *e) +{ + engine_table_unregister(&bn_meth_table, e); +} + +int ENGINE_register_bn_meth(ENGINE *e) +{ + if (ENGINE_get_bn_meth(e)) + return engine_table_register(&bn_meth_table, + engine_unregister_all_bn_meth, e, + &dummy_nid, 1, 0); + return 1; +} + +void ENGINE_register_all_bn_meth(void) +{ + ENGINE *e; + + for (e = ENGINE_get_first(); e; e = ENGINE_get_next(e)) + ENGINE_register_bn_meth(e); +} + +int ENGINE_set_default_bn_meth(ENGINE *e) +{ + if (ENGINE_get_bn_meth(e)) + return engine_table_register(&bn_meth_table, + engine_unregister_all_bn_meth, e, + &dummy_nid, 1, 1); + return 1; +} + +ENGINE *ENGINE_get_default_bn_meth(void) +{ + return ossl_engine_table_select(&bn_meth_table, dummy_nid, + OPENSSL_FILE, OPENSSL_LINE); +} + +const BN_METHOD *ENGINE_get_bn_meth(ENGINE *e) +{ + bn_meth_data_ctx *ctx = bn_meth_get_data_ctx(e); + if (ctx == NULL) + return NULL; + return ctx->bn_meth; +} + +int ENGINE_set_bn_meth(ENGINE *e, const BN_METHOD *bn_meth) +{ + bn_meth_data_ctx *ctx = bn_meth_get_data_ctx(e); + if (ctx == NULL) + return 0; + ctx->bn_meth = bn_meth; + return 1; +} + diff --git a/openssl/src/crypto/engine/tb_ecpmeth.c b/openssl/src/crypto/engine/tb_ecpmeth.c new file mode 100644 index 000000000..06160d911 --- /dev/null +++ b/openssl/src/crypto/engine/tb_ecpmeth.c @@ -0,0 +1,204 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* We need to use some deprecated APIs */ +#define OPENSSL_SUPPRESS_DEPRECATED + +#include "eng_local.h" + +static ENGINE_TABLE *ecpmeth_table = NULL; +static int ecpmeth_ex_data_idx = -1; + +typedef struct ecpmeth_data_ctx_st { + /* ecpmeth handling is via this callback */ + ENGINE_ECP_METHS_PTR ecpmeths; +} ecpmeth_data_ctx; + +static void ecpmeth_data_ctx_free_func(void *parent, void *ptr, + CRYPTO_EX_DATA *ad, int idx, long argl, + void *argp) +{ + if (ptr) { + ecpmeth_data_ctx *ctx = (ecpmeth_data_ctx *)ptr; + OPENSSL_free(ctx); + } +} + +static int ecpmeth_set_data_ctx(ENGINE *e, ecpmeth_data_ctx **ctx) +{ + int ret = 1; + ecpmeth_data_ctx *c; + + if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { + ENGINEerr(ENGINE_F_ECPMETH_SET_DATA_CTX, ERR_R_INTERNAL_ERROR); + return 0; + } + + c = OPENSSL_zalloc(sizeof(*c)); + if (c == NULL) { + ENGINEerr(ENGINE_F_ECPMETH_SET_DATA_CTX, ERR_R_MALLOC_FAILURE); + return 0; + } + + if (!CRYPTO_THREAD_write_lock(global_engine_lock)) { + OPENSSL_free(c); + return 0; + } + + if ((*ctx = (ecpmeth_data_ctx *)ENGINE_get_ex_data(e, ecpmeth_ex_data_idx)) + == NULL) { + /* Good, we're the first */ + ret = ENGINE_set_ex_data(e, ecpmeth_ex_data_idx, c); + if (ret) { + *ctx = c; + c = NULL; + } + } + CRYPTO_THREAD_unlock(global_engine_lock); + OPENSSL_free(c); + return ret; +} + +/* + * This function retrieves the context structure from an ENGINE's "ex_data", + * or if it doesn't exist yet, sets it up. + */ +static ecpmeth_data_ctx *ecpmeth_get_data_ctx(ENGINE *e) +{ + ecpmeth_data_ctx *ctx; + + if (ecpmeth_ex_data_idx < 0) { + /* + * Create and register the ENGINE ex_data, and associate our "free" + * function with it to ensure any allocated contexts get freed when + * an ENGINE goes underground. + */ + int new_idx = ENGINE_get_ex_new_index(0, NULL, NULL, NULL, + ecpmeth_data_ctx_free_func); + if (new_idx == -1) { + ENGINEerr(ENGINE_F_ECPMETH_GET_DATA_CTX, ENGINE_R_NO_INDEX); + return NULL; + } + + if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) { + ENGINEerr(ENGINE_F_ECPMETH_GET_DATA_CTX, ERR_R_INTERNAL_ERROR); + return NULL; + } + + if (!CRYPTO_THREAD_write_lock(global_engine_lock)) + return NULL; + + /* Avoid a race by checking again inside this lock */ + if (ecpmeth_ex_data_idx < 0) { + /* Good, someone didn't beat us to it */ + ecpmeth_ex_data_idx = new_idx; + new_idx = -1; + } + CRYPTO_THREAD_unlock(global_engine_lock); + /* + * In theory we could "give back" the index here if (new_idx>-1), but + * it's not possible and wouldn't gain us much if it were. + */ + } + ctx = (ecpmeth_data_ctx *)ENGINE_get_ex_data(e, ecpmeth_ex_data_idx); + /* Check if the context needs to be created */ + if ((ctx == NULL) && !ecpmeth_set_data_ctx(e, &ctx)) + /* "set_data" will set errors if necessary */ + return NULL; + return ctx; +} + +static void engine_unregister_all_ecp_meths(void) +{ + engine_table_cleanup(&ecpmeth_table); +} + +void ENGINE_unregister_ecp_meths(ENGINE *e) +{ + engine_table_unregister(&ecpmeth_table, e); +} + +int ENGINE_register_ecp_meths(ENGINE *e) +{ + ENGINE_ECP_METHS_PTR fn = ENGINE_get_ecp_meths(e); + if (fn) { + const int *cids; + int num_cids = fn(e, NULL, &cids, 0); + if (num_cids > 0) + return engine_table_register(&ecpmeth_table, + engine_unregister_all_ecp_meths, e, + cids, num_cids, 0); + } + return 1; +} + +void ENGINE_register_all_ecp_meths(void) +{ + ENGINE *e; + + for (e = ENGINE_get_first(); e; e = ENGINE_get_next(e)) + ENGINE_register_ecp_meths(e); +} + +int ENGINE_set_default_ecp_meths(ENGINE *e) +{ + ENGINE_ECP_METHS_PTR fn = ENGINE_get_ecp_meths(e); + if (fn) { + const int *cids; + int num_cids = fn(e, NULL, &cids, 0); + if (num_cids > 0) + return engine_table_register(&ecpmeth_table, + engine_unregister_all_ecp_meths, e, + cids, num_cids, 1); + } + return 1; +} + +/* + * Exposed API function to get a functional reference from the implementation + * table (ie. try to get a functional reference from the tabled structural + * references). + */ +ENGINE *ENGINE_get_ecp_meth_engine(int curve_id) +{ + return ossl_engine_table_select(&ecpmeth_table, curve_id, + OPENSSL_FILE, OPENSSL_LINE); +} + +/* Obtains an EC_KEY implementation from an ENGINE functional reference */ +const EC_POINT_METHOD *ENGINE_get_ecp_meth(ENGINE *e, int curve_id) +{ + const EC_POINT_METHOD *ret; + ENGINE_ECP_METHS_PTR fn = ENGINE_get_ecp_meths(e); + if (!fn || !fn(e, &ret, NULL, curve_id)) { + ENGINEerr(ENGINE_F_ENGINE_GET_ECP_METH, ENGINE_R_UNIMPLEMENTED_ECP_METH); + return NULL; + } + return ret; +} + +/* Gets the ecp_meths callback from an ENGINE structure */ +ENGINE_ECP_METHS_PTR ENGINE_get_ecp_meths(ENGINE *e) +{ + ecpmeth_data_ctx *ctx = ecpmeth_get_data_ctx(e); + if (ctx == NULL) + return NULL; + return ctx->ecpmeths; +} + +/* Sets the ecp_meths callback in an ENGINE structure */ +int ENGINE_set_ecp_meths(ENGINE *e, ENGINE_ECP_METHS_PTR f) +{ + ecpmeth_data_ctx *ctx = ecpmeth_get_data_ctx(e); + if (ctx == NULL) + return 0; + ctx->ecpmeths = f; + return 1; +} + diff --git a/openssl/src/crypto/err/err.c b/openssl/src/crypto/err/err.c index b95182d70..4c740862f 100644 --- a/openssl/src/crypto/err/err.c +++ b/openssl/src/crypto/err/err.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -29,10 +29,9 @@ /* Forward declaration in case it's not published because of configuration */ ERR_STATE *ERR_get_state(void); -#ifndef OPENSSL_NO_ERR static int err_load_strings(const ERR_STRING_DATA *str); -#endif +static void ERR_STATE_free(ERR_STATE *s); #ifndef OPENSSL_NO_ERR static ERR_STRING_DATA ERR_str_libraries[] = { {ERR_PACK(ERR_LIB_NONE, 0, 0), "unknown library"}, @@ -143,19 +142,15 @@ static int set_err_thread_local; static CRYPTO_THREAD_LOCAL err_thread_local; static CRYPTO_ONCE err_string_init = CRYPTO_ONCE_STATIC_INIT; -static CRYPTO_RWLOCK *err_string_lock = NULL; +static CRYPTO_RWLOCK *err_string_lock; -#ifndef OPENSSL_NO_ERR static ERR_STRING_DATA *int_err_get_item(const ERR_STRING_DATA *); -#endif /* * The internal state */ -#ifndef OPENSSL_NO_ERR static LHASH_OF(ERR_STRING_DATA) *int_error_hash = NULL; -#endif static int int_err_library_number = ERR_LIB_USER; typedef enum ERR_GET_ACTION_e { @@ -167,7 +162,6 @@ static unsigned long get_error_values(ERR_GET_ACTION g, const char **func, const char **data, int *flags); -#ifndef OPENSSL_NO_ERR static unsigned long err_string_data_hash(const ERR_STRING_DATA *a) { unsigned long ret, l; @@ -196,18 +190,17 @@ static ERR_STRING_DATA *int_err_get_item(const ERR_STRING_DATA *d) return p; } -#endif -void OSSL_ERR_STATE_free(ERR_STATE *state) +static void ERR_STATE_free(ERR_STATE *s) { int i; - if (state == NULL) + if (s == NULL) return; for (i = 0; i < ERR_NUM_ERRORS; i++) { - err_clear(state, i, 1); + err_clear(s, i, 1); } - CRYPTO_free(state, OPENSSL_FILE, OPENSSL_LINE); + OPENSSL_free(s); } DEFINE_RUN_ONCE_STATIC(do_err_strings_init) @@ -217,7 +210,6 @@ DEFINE_RUN_ONCE_STATIC(do_err_strings_init) err_string_lock = CRYPTO_THREAD_lock_new(); if (err_string_lock == NULL) return 0; -#ifndef OPENSSL_NO_ERR int_error_hash = lh_ERR_STRING_DATA_new(err_string_data_hash, err_string_data_cmp); if (int_error_hash == NULL) { @@ -225,7 +217,6 @@ DEFINE_RUN_ONCE_STATIC(do_err_strings_init) err_string_lock = NULL; return 0; } -#endif return 1; } @@ -235,13 +226,10 @@ void err_cleanup(void) CRYPTO_THREAD_cleanup_local(&err_thread_local); CRYPTO_THREAD_lock_free(err_string_lock); err_string_lock = NULL; -#ifndef OPENSSL_NO_ERR lh_ERR_STRING_DATA_free(int_error_hash); int_error_hash = NULL; -#endif } -#ifndef OPENSSL_NO_ERR /* * Legacy; pack in the library. */ @@ -266,7 +254,6 @@ static int err_load_strings(const ERR_STRING_DATA *str) CRYPTO_THREAD_unlock(err_string_lock); return 1; } -#endif int ossl_err_load_ERR_strings(void) { @@ -282,31 +269,24 @@ int ossl_err_load_ERR_strings(void) int ERR_load_strings(int lib, ERR_STRING_DATA *str) { -#ifndef OPENSSL_NO_ERR if (ossl_err_load_ERR_strings() == 0) return 0; err_patch(lib, str); err_load_strings(str); -#endif - return 1; } int ERR_load_strings_const(const ERR_STRING_DATA *str) { -#ifndef OPENSSL_NO_ERR if (ossl_err_load_ERR_strings() == 0) return 0; err_load_strings(str); -#endif - return 1; } int ERR_unload_strings(int lib, ERR_STRING_DATA *str) { -#ifndef OPENSSL_NO_ERR if (!RUN_ONCE(&err_string_init, do_err_strings_init)) return 0; @@ -319,14 +299,14 @@ int ERR_unload_strings(int lib, ERR_STRING_DATA *str) for (; str->error; str++) (void)lh_ERR_STRING_DATA_delete(int_error_hash, str); CRYPTO_THREAD_unlock(err_string_lock); -#endif return 1; } void err_free_strings_int(void) { - /* obsolete */ + if (!RUN_ONCE(&err_string_init, do_err_strings_init)) + return; } /********************************************************/ @@ -551,8 +531,7 @@ void ossl_err_string_int(unsigned long e, const char *func, } #endif if (rs == NULL) { - BIO_snprintf(rsbuf, sizeof(rsbuf), "reason(%lu)", - r & ~(ERR_RFLAGS_MASK << ERR_RFLAGS_OFFSET)); + BIO_snprintf(rsbuf, sizeof(rsbuf), "reason(%lu)", r); rs = rsbuf; } @@ -585,7 +564,6 @@ char *ERR_error_string(unsigned long e, char *ret) const char *ERR_lib_error_string(unsigned long e) { -#ifndef OPENSSL_NO_ERR ERR_STRING_DATA d, *p; unsigned long l; @@ -597,9 +575,6 @@ const char *ERR_lib_error_string(unsigned long e) d.error = ERR_PACK(l, 0, 0); p = int_err_get_item(&d); return ((p == NULL) ? NULL : p->string); -#else - return NULL; -#endif } #ifndef OPENSSL_NO_DEPRECATED_3_0 @@ -611,7 +586,6 @@ const char *ERR_func_error_string(unsigned long e) const char *ERR_reason_error_string(unsigned long e) { -#ifndef OPENSSL_NO_ERR ERR_STRING_DATA d, *p = NULL; unsigned long l, r; @@ -636,9 +610,6 @@ const char *ERR_reason_error_string(unsigned long e) p = int_err_get_item(&d); } return ((p == NULL) ? NULL : p->string); -#else - return NULL; -#endif } static void err_delete_thread_state(void *unused) @@ -648,7 +619,7 @@ static void err_delete_thread_state(void *unused) return; CRYPTO_THREAD_set_local(&err_thread_local, NULL); - OSSL_ERR_STATE_free(state); + ERR_STATE_free(state); } #ifndef OPENSSL_NO_DEPRECATED_1_1_0 @@ -688,15 +659,14 @@ ERR_STATE *ossl_err_get_state_int(void) if (!CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE*)-1)) return NULL; - state = OSSL_ERR_STATE_new(); - if (state == NULL) { + if ((state = OPENSSL_zalloc(sizeof(*state))) == NULL) { CRYPTO_THREAD_set_local(&err_thread_local, NULL); return NULL; } if (!ossl_init_thread_start(NULL, NULL, err_delete_thread_state) || !CRYPTO_THREAD_set_local(&err_thread_local, state)) { - OSSL_ERR_STATE_free(state); + ERR_STATE_free(state); CRYPTO_THREAD_set_local(&err_thread_local, NULL); return NULL; } @@ -831,11 +801,10 @@ void ERR_add_error_vdata(int num, va_list args) i = es->top; /* - * If err_data is allocated already, reuse the space. + * If err_data is allocated already, re-use the space. * Otherwise, allocate a small new buffer. */ - if ((es->err_data_flags[i] & flags) == flags - && ossl_assert(es->err_data[i] != NULL)) { + if ((es->err_data_flags[i] & flags) == flags) { str = es->err_data[i]; size = es->err_data_size[i]; @@ -877,6 +846,61 @@ void ERR_add_error_vdata(int num, va_list args) OPENSSL_free(str); } +int ERR_set_mark(void) +{ + ERR_STATE *es; + + es = ossl_err_get_state_int(); + if (es == NULL) + return 0; + + if (es->bottom == es->top) + return 0; + es->err_marks[es->top]++; + return 1; +} + +int ERR_pop_to_mark(void) +{ + ERR_STATE *es; + + es = ossl_err_get_state_int(); + if (es == NULL) + return 0; + + while (es->bottom != es->top + && es->err_marks[es->top] == 0) { + err_clear(es, es->top, 0); + es->top = es->top > 0 ? es->top - 1 : ERR_NUM_ERRORS - 1; + } + + if (es->bottom == es->top) + return 0; + es->err_marks[es->top]--; + return 1; +} + +int ERR_clear_last_mark(void) +{ + ERR_STATE *es; + int top; + + es = ossl_err_get_state_int(); + if (es == NULL) + return 0; + + top = es->top; + while (es->bottom != top + && es->err_marks[top] == 0) { + top = top > 0 ? top - 1 : ERR_NUM_ERRORS - 1; + } + + if (es->bottom == top) + return 0; + es->err_marks[top]--; + return 1; +} + void err_clear_last_constant_time(int clear) { ERR_STATE *es; diff --git a/openssl/src/crypto/err/err_all.c b/openssl/src/crypto/err/err_all.c index 86b609a55..55aa2b8db 100644 --- a/openssl/src/crypto/err/err_all.c +++ b/openssl/src/crypto/err/err_all.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -85,9 +85,7 @@ int ossl_err_load_crypto_strings(void) # ifndef OPENSSL_NO_ENGINE || ossl_err_load_ENGINE_strings() == 0 # endif -# ifndef OPENSSL_NO_HTTP || ossl_err_load_HTTP_strings() == 0 -# endif # ifndef OPENSSL_NO_OCSP || ossl_err_load_OCSP_strings() == 0 # endif diff --git a/openssl/src/crypto/err/err_local.h b/openssl/src/crypto/err/err_local.h index c5c5bf45b..d4e19dff2 100644 --- a/openssl/src/crypto/err/err_local.h +++ b/openssl/src/crypto/err/err_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,7 +7,6 @@ * https://www.openssl.org/source/license.html */ -#include #include #include @@ -57,18 +56,14 @@ static ossl_inline void err_set_debug(ERR_STATE *es, size_t i, OPENSSL_free(es->err_file[i]); if (file == NULL || file[0] == '\0') es->err_file[i] = NULL; - else if ((es->err_file[i] = CRYPTO_malloc(strlen(file) + 1, - NULL, 0)) != NULL) - /* We cannot use OPENSSL_strdup due to possible recursion */ - strcpy(es->err_file[i], file); - + else + es->err_file[i] = OPENSSL_strdup(file); es->err_line[i] = line; OPENSSL_free(es->err_func[i]); if (fn == NULL || fn[0] == '\0') es->err_func[i] = NULL; - else if ((es->err_func[i] = CRYPTO_malloc(strlen(fn) + 1, - NULL, 0)) != NULL) - strcpy(es->err_func[i], fn); + else + es->err_func[i] = OPENSSL_strdup(fn); } static ossl_inline void err_set_data(ERR_STATE *es, size_t i, diff --git a/openssl/src/crypto/err/err_mark.c b/openssl/src/crypto/err/err_mark.c deleted file mode 100644 index cb01a1f4f..000000000 --- a/openssl/src/crypto/err/err_mark.c +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2003-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#define OSSL_FORCE_ERR_STATE - -#include -#include "err_local.h" - -int ERR_set_mark(void) -{ - ERR_STATE *es; - - es = ossl_err_get_state_int(); - if (es == NULL) - return 0; - - if (es->bottom == es->top) - return 0; - es->err_marks[es->top]++; - return 1; -} - -int ERR_pop(void) -{ - ERR_STATE *es; - - es = ossl_err_get_state_int(); - if (es == NULL || es->bottom == es->top) - return 0; - - err_clear(es, es->top, 0); - es->top = es->top > 0 ? es->top - 1 : ERR_NUM_ERRORS - 1; - return 1; -} - -int ERR_pop_to_mark(void) -{ - ERR_STATE *es; - - es = ossl_err_get_state_int(); - if (es == NULL) - return 0; - - while (es->bottom != es->top - && es->err_marks[es->top] == 0) { - err_clear(es, es->top, 0); - es->top = es->top > 0 ? es->top - 1 : ERR_NUM_ERRORS - 1; - } - - if (es->bottom == es->top) - return 0; - es->err_marks[es->top]--; - return 1; -} - -int ERR_count_to_mark(void) -{ - ERR_STATE *es; - int count = 0, top; - - es = ossl_err_get_state_int(); - if (es == NULL) - return 0; - - top = es->top; - while (es->bottom != top - && es->err_marks[top] == 0) { - ++count; - top = top > 0 ? top - 1 : ERR_NUM_ERRORS - 1; - } - - return count; -} - -int ERR_clear_last_mark(void) -{ - ERR_STATE *es; - int top; - - es = ossl_err_get_state_int(); - if (es == NULL) - return 0; - - top = es->top; - while (es->bottom != top - && es->err_marks[top] == 0) { - top = top > 0 ? top - 1 : ERR_NUM_ERRORS - 1; - } - - if (es->bottom == top) - return 0; - es->err_marks[top]--; - return 1; -} - diff --git a/openssl/src/crypto/err/err_save.c b/openssl/src/crypto/err/err_save.c deleted file mode 100644 index 1994c26ce..000000000 --- a/openssl/src/crypto/err/err_save.c +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#define OSSL_FORCE_ERR_STATE - -#include -#include "err_local.h" - -/* - * Save and restore error state. - * We are using CRYPTO_zalloc(.., NULL, 0) instead of OPENSSL_malloc() in - * these functions to prevent mem alloc error loop. - */ - -ERR_STATE *OSSL_ERR_STATE_new(void) -{ - return CRYPTO_zalloc(sizeof(ERR_STATE), NULL, 0); -} - -void OSSL_ERR_STATE_save(ERR_STATE *es) -{ - size_t i; - ERR_STATE *thread_es; - - if (es == NULL) - return; - - for (i = 0; i < ERR_NUM_ERRORS; i++) - err_clear(es, i, 1); - - thread_es = ossl_err_get_state_int(); - if (thread_es == NULL) - return; - - memcpy(es, thread_es, sizeof(*es)); - /* Taking over the pointers, just clear the thread state. */ - memset(thread_es, 0, sizeof(*thread_es)); -} - -void OSSL_ERR_STATE_save_to_mark(ERR_STATE *es) -{ - size_t i, j, count; - int top; - ERR_STATE *thread_es; - - if (es == NULL) - return; - - thread_es = ossl_err_get_state_int(); - if (thread_es == NULL) { - for (i = 0; i < ERR_NUM_ERRORS; ++i) - err_clear(es, i, 1); - - es->top = es->bottom = 0; - return; - } - - /* Determine number of errors we are going to move. */ - for (count = 0, top = thread_es->top; - thread_es->bottom != top - && thread_es->err_marks[top] == 0; - ++count) - top = top > 0 ? top - 1 : ERR_NUM_ERRORS - 1; - - /* Move the errors, preserving order. */ - for (i = 0, j = top; i < count; ++i) { - j = (j + 1) % ERR_NUM_ERRORS; - - err_clear(es, i, 1); - - /* Move the error entry to the given ERR_STATE. */ - es->err_flags[i] = thread_es->err_flags[j]; - es->err_marks[i] = 0; - es->err_buffer[i] = thread_es->err_buffer[j]; - es->err_data[i] = thread_es->err_data[j]; - es->err_data_size[i] = thread_es->err_data_size[j]; - es->err_data_flags[i] = thread_es->err_data_flags[j]; - es->err_file[i] = thread_es->err_file[j]; - es->err_line[i] = thread_es->err_line[j]; - es->err_func[i] = thread_es->err_func[j]; - - thread_es->err_flags[j] = 0; - thread_es->err_buffer[j] = 0; - thread_es->err_data[j] = NULL; - thread_es->err_data_size[j] = 0; - thread_es->err_data_flags[j] = 0; - thread_es->err_file[j] = NULL; - thread_es->err_line[j] = 0; - thread_es->err_func[j] = NULL; - } - - if (i > 0) { - thread_es->top = top; - /* If we moved anything, es's stack always starts at [0]. */ - es->top = i - 1; - es->bottom = ERR_NUM_ERRORS - 1; - } else { - /* Didn't move anything - empty stack */ - es->top = es->bottom = 0; - } - - /* Erase extra space as a precaution. */ - for (; i < ERR_NUM_ERRORS; ++i) - err_clear(es, i, 1); -} - -void OSSL_ERR_STATE_restore(const ERR_STATE *es) -{ - size_t i; - ERR_STATE *thread_es; - - if (es == NULL || es->bottom == es->top) - return; - - thread_es = ossl_err_get_state_int(); - if (thread_es == NULL) - return; - - for (i = (size_t)es->bottom; i != (size_t)es->top;) { - size_t top; - - i = (i + 1) % ERR_NUM_ERRORS; - if ((es->err_flags[i] & ERR_FLAG_CLEAR) != 0) - continue; - - err_get_slot(thread_es); - top = thread_es->top; - err_clear(thread_es, top, 0); - - thread_es->err_flags[top] = es->err_flags[i]; - thread_es->err_buffer[top] = es->err_buffer[i]; - - err_set_debug(thread_es, top, es->err_file[i], es->err_line[i], - es->err_func[i]); - - if (es->err_data[i] != NULL && es->err_data_size[i] != 0) { - void *data; - size_t data_sz = es->err_data_size[i]; - - data = CRYPTO_malloc(data_sz, NULL, 0); - if (data != NULL) { - memcpy(data, es->err_data[i], data_sz); - err_set_data(thread_es, top, data, data_sz, - es->err_data_flags[i] | ERR_TXT_MALLOCED); - } - } else { - err_clear_data(thread_es, top, 0); - } - } -} diff --git a/openssl/src/crypto/ess/ess_lib.c b/openssl/src/crypto/ess/ess_lib.c index ff174470d..65444d383 100644 --- a/openssl/src/crypto/ess/ess_lib.c +++ b/openssl/src/crypto/ess/ess_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -29,38 +29,28 @@ ESS_SIGNING_CERT *OSSL_ESS_signing_cert_new_init(const X509 *signcert, ESS_SIGNING_CERT *sc; int i; - if ((sc = ESS_SIGNING_CERT_new()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + if ((sc = ESS_SIGNING_CERT_new()) == NULL) goto err; - } if (sc->cert_ids == NULL - && (sc->cert_ids = sk_ESS_CERT_ID_new_null()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + && (sc->cert_ids = sk_ESS_CERT_ID_new_null()) == NULL) goto err; - } if ((cid = ESS_CERT_ID_new_init(signcert, set_issuer_serial)) == NULL - || !sk_ESS_CERT_ID_push(sc->cert_ids, cid)) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + || !sk_ESS_CERT_ID_push(sc->cert_ids, cid)) goto err; - } for (i = 0; i < sk_X509_num(certs); ++i) { X509 *cert = sk_X509_value(certs, i); - if ((cid = ESS_CERT_ID_new_init(cert, 1)) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); - goto err; - } - if (!sk_ESS_CERT_ID_push(sc->cert_ids, cid)) { - ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + if ((cid = ESS_CERT_ID_new_init(cert, 1)) == NULL + || !sk_ESS_CERT_ID_push(sc->cert_ids, cid)) goto err; - } } return sc; err: ESS_SIGNING_CERT_free(sc); ESS_CERT_ID_free(cid); + ERR_raise(ERR_LIB_ESS, ERR_R_MALLOC_FAILURE); return NULL; } @@ -71,53 +61,38 @@ static ESS_CERT_ID *ESS_CERT_ID_new_init(const X509 *cert, GENERAL_NAME *name = NULL; unsigned char cert_sha1[SHA_DIGEST_LENGTH]; - if ((cid = ESS_CERT_ID_new()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + if ((cid = ESS_CERT_ID_new()) == NULL) goto err; - } - if (!X509_digest(cert, EVP_sha1(), cert_sha1, NULL)) { - ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + if (!X509_digest(cert, EVP_sha1(), cert_sha1, NULL)) goto err; - } - if (!ASN1_OCTET_STRING_set(cid->hash, cert_sha1, SHA_DIGEST_LENGTH)) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if (!ASN1_OCTET_STRING_set(cid->hash, cert_sha1, SHA_DIGEST_LENGTH)) goto err; - } /* Setting the issuer/serial if requested. */ if (!set_issuer_serial) return cid; if (cid->issuer_serial == NULL - && (cid->issuer_serial = ESS_ISSUER_SERIAL_new()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + && (cid->issuer_serial = ESS_ISSUER_SERIAL_new()) == NULL) goto err; - } - if ((name = GENERAL_NAME_new()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if ((name = GENERAL_NAME_new()) == NULL) goto err; - } name->type = GEN_DIRNAME; - if ((name->d.dirn = X509_NAME_dup(X509_get_issuer_name(cert))) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + if ((name->d.dirn = X509_NAME_dup(X509_get_issuer_name(cert))) == NULL) goto err; - } - if (!sk_GENERAL_NAME_push(cid->issuer_serial->issuer, name)) { - ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + if (!sk_GENERAL_NAME_push(cid->issuer_serial->issuer, name)) goto err; - } name = NULL; /* Ownership is lost. */ ASN1_INTEGER_free(cid->issuer_serial->serial); - if ((cid->issuer_serial->serial - = ASN1_INTEGER_dup(X509_get0_serialNumber(cert))) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if ((cid->issuer_serial->serial = + ASN1_INTEGER_dup(X509_get0_serialNumber(cert))) == NULL) goto err; - } return cid; err: GENERAL_NAME_free(name); ESS_CERT_ID_free(cid); + ERR_raise(ERR_LIB_ESS, ERR_R_MALLOC_FAILURE); return NULL; } @@ -131,32 +106,22 @@ ESS_SIGNING_CERT_V2 *OSSL_ESS_signing_cert_v2_new_init(const EVP_MD *hash_alg, ESS_SIGNING_CERT_V2 *sc; int i; - if ((sc = ESS_SIGNING_CERT_V2_new()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + if ((sc = ESS_SIGNING_CERT_V2_new()) == NULL) goto err; - } cid = ESS_CERT_ID_V2_new_init(hash_alg, signcert, set_issuer_serial); - if (cid == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + if (cid == NULL) goto err; - } - if (!sk_ESS_CERT_ID_V2_push(sc->cert_ids, cid)) { - ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + if (!sk_ESS_CERT_ID_V2_push(sc->cert_ids, cid)) goto err; - } cid = NULL; for (i = 0; i < sk_X509_num(certs); ++i) { X509 *cert = sk_X509_value(certs, i); - if ((cid = ESS_CERT_ID_V2_new_init(hash_alg, cert, 1)) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + if ((cid = ESS_CERT_ID_V2_new_init(hash_alg, cert, 1)) == NULL) goto err; - } - if (!sk_ESS_CERT_ID_V2_push(sc->cert_ids, cid)) { - ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + if (!sk_ESS_CERT_ID_V2_push(sc->cert_ids, cid)) goto err; - } cid = NULL; } @@ -164,6 +129,7 @@ ESS_SIGNING_CERT_V2 *OSSL_ESS_signing_cert_v2_new_init(const EVP_MD *hash_alg, err: ESS_SIGNING_CERT_V2_free(sc); ESS_CERT_ID_V2_free(cid); + ERR_raise(ERR_LIB_ESS, ERR_R_MALLOC_FAILURE); return NULL; } @@ -179,71 +145,52 @@ static ESS_CERT_ID_V2 *ESS_CERT_ID_V2_new_init(const EVP_MD *hash_alg, memset(hash, 0, sizeof(hash)); - if ((cid = ESS_CERT_ID_V2_new()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + if ((cid = ESS_CERT_ID_V2_new()) == NULL) goto err; - } if (!EVP_MD_is_a(hash_alg, SN_sha256)) { alg = X509_ALGOR_new(); - if (alg == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if (alg == NULL) goto err; - } X509_ALGOR_set_md(alg, hash_alg); - if (alg->algorithm == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if (alg->algorithm == NULL) goto err; - } cid->hash_alg = alg; alg = NULL; } else { cid->hash_alg = NULL; } - if (!X509_digest(cert, hash_alg, hash, &hash_len)) { - ERR_raise(ERR_LIB_ESS, ERR_R_X509_LIB); + if (!X509_digest(cert, hash_alg, hash, &hash_len)) goto err; - } - if (!ASN1_OCTET_STRING_set(cid->hash, hash, hash_len)) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if (!ASN1_OCTET_STRING_set(cid->hash, hash, hash_len)) goto err; - } if (!set_issuer_serial) return cid; - if ((cid->issuer_serial = ESS_ISSUER_SERIAL_new()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ESS_LIB); + if ((cid->issuer_serial = ESS_ISSUER_SERIAL_new()) == NULL) goto err; - } - if ((name = GENERAL_NAME_new()) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if ((name = GENERAL_NAME_new()) == NULL) goto err; - } name->type = GEN_DIRNAME; - if ((name->d.dirn = X509_NAME_dup(X509_get_issuer_name(cert))) == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if ((name->d.dirn = X509_NAME_dup(X509_get_issuer_name(cert))) == NULL) goto err; - } - if (!sk_GENERAL_NAME_push(cid->issuer_serial->issuer, name)) { - ERR_raise(ERR_LIB_ESS, ERR_R_CRYPTO_LIB); + if (!sk_GENERAL_NAME_push(cid->issuer_serial->issuer, name)) goto err; - } name = NULL; /* Ownership is lost. */ ASN1_INTEGER_free(cid->issuer_serial->serial); cid->issuer_serial->serial = ASN1_INTEGER_dup(X509_get0_serialNumber(cert)); - if (cid->issuer_serial->serial == NULL) { - ERR_raise(ERR_LIB_ESS, ERR_R_ASN1_LIB); + if (cid->issuer_serial->serial == NULL) goto err; - } return cid; err: X509_ALGOR_free(alg); GENERAL_NAME_free(name); ESS_CERT_ID_V2_free(cid); + ERR_raise(ERR_LIB_ESS, ERR_R_MALLOC_FAILURE); return NULL; } @@ -346,7 +293,7 @@ int OSSL_ESS_check_signing_certs(const ESS_SIGNING_CERT *ss, int i, ret; if (require_signing_cert && ss == NULL && ssv2 == NULL) { - ERR_raise(ERR_LIB_ESS, ESS_R_MISSING_SIGNING_CERTIFICATE_ATTRIBUTE); + ERR_raise(ERR_LIB_CMS, ESS_R_MISSING_SIGNING_CERTIFICATE_ATTRIBUTE); return -1; } if (n_v1 == 0 || n_v2 == 0) { diff --git a/openssl/src/crypto/evp/asymcipher.c b/openssl/src/crypto/evp/asymcipher.c index d22ab2a01..b7784c899 100644 --- a/openssl/src/crypto/evp/asymcipher.c +++ b/openssl/src/crypto/evp/asymcipher.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -189,7 +189,7 @@ static int evp_pkey_asym_cipher_init(EVP_PKEY_CTX *ctx, int operation, ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); return -2; } - switch (ctx->operation) { + switch(ctx->operation) { case EVP_PKEY_OP_ENCRYPT: if (ctx->pmeth->encrypt_init == NULL) return 1; @@ -298,38 +298,25 @@ int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, return ctx->pmeth->decrypt(ctx, out, outlen, in, inlen); } -/* decrypt to new buffer of dynamic size, checking any pre-determined size */ -int evp_pkey_decrypt_alloc(EVP_PKEY_CTX *ctx, unsigned char **outp, - size_t *outlenp, size_t expected_outlen, - const unsigned char *in, size_t inlen) -{ - if (EVP_PKEY_decrypt(ctx, NULL, outlenp, in, inlen) <= 0 - || (*outp = OPENSSL_malloc(*outlenp)) == NULL) - return -1; - if (EVP_PKEY_decrypt(ctx, *outp, outlenp, in, inlen) <= 0 - || *outlenp == 0 - || (expected_outlen != 0 && *outlenp != expected_outlen)) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); - OPENSSL_clear_free(*outp, *outlenp); - *outp = NULL; - return 0; - } - return 1; -} static EVP_ASYM_CIPHER *evp_asym_cipher_new(OSSL_PROVIDER *prov) { EVP_ASYM_CIPHER *cipher = OPENSSL_zalloc(sizeof(EVP_ASYM_CIPHER)); - if (cipher == NULL) + if (cipher == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } - if (!CRYPTO_NEW_REF(&cipher->refcnt, 1)) { + cipher->lock = CRYPTO_THREAD_lock_new(); + if (cipher->lock == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); OPENSSL_free(cipher); return NULL; } cipher->prov = prov; ossl_provider_up_ref(prov); + cipher->refcnt = 1; return cipher; } @@ -344,7 +331,7 @@ static void *evp_asym_cipher_from_algorithm(int name_id, int gparamfncnt = 0, sparamfncnt = 0; if ((cipher = evp_asym_cipher_new(prov)) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } @@ -457,12 +444,12 @@ void EVP_ASYM_CIPHER_free(EVP_ASYM_CIPHER *cipher) if (cipher == NULL) return; - CRYPTO_DOWN_REF(&cipher->refcnt, &i); + CRYPTO_DOWN_REF(&cipher->refcnt, &i, cipher->lock); if (i > 0) return; OPENSSL_free(cipher->type_name); ossl_provider_free(cipher->prov); - CRYPTO_FREE_REF(&cipher->refcnt); + CRYPTO_THREAD_lock_free(cipher->lock); OPENSSL_free(cipher); } @@ -470,7 +457,7 @@ int EVP_ASYM_CIPHER_up_ref(EVP_ASYM_CIPHER *cipher) { int ref = 0; - CRYPTO_UP_REF(&cipher->refcnt, &ref); + CRYPTO_UP_REF(&cipher->refcnt, &ref, cipher->lock); return 1; } diff --git a/openssl/src/crypto/evp/bio_b64.c b/openssl/src/crypto/evp/bio_b64.c index 8700315a6..81d2609c3 100644 --- a/openssl/src/crypto/evp/bio_b64.c +++ b/openssl/src/crypto/evp/bio_b64.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -39,8 +39,8 @@ typedef struct b64_struct { int start; /* have we started decoding yet? */ int cont; /* <= 0 when finished */ EVP_ENCODE_CTX *base64; - unsigned char buf[EVP_ENCODE_LENGTH(B64_BLOCK_SIZE) + 10]; - unsigned char tmp[B64_BLOCK_SIZE]; + char buf[EVP_ENCODE_LENGTH(B64_BLOCK_SIZE) + 10]; + char tmp[B64_BLOCK_SIZE]; } BIO_B64_CTX; static const BIO_METHOD methods_b64 = { @@ -58,6 +58,7 @@ static const BIO_METHOD methods_b64 = { b64_callback_ctrl, }; + const BIO_METHOD *BIO_f_base64(void) { return &methods_b64; @@ -67,8 +68,10 @@ static int b64_new(BIO *bi) { BIO_B64_CTX *ctx; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } ctx->cont = 1; ctx->start = 1; @@ -87,7 +90,6 @@ static int b64_new(BIO *bi) static int b64_free(BIO *a) { BIO_B64_CTX *ctx; - if (a == NULL) return 0; @@ -115,7 +117,7 @@ static int b64_read(BIO *b, char *out, int outl) ctx = (BIO_B64_CTX *)BIO_get_data(b); next = BIO_next(b); - if (ctx == NULL || next == NULL) + if ((ctx == NULL) || (next == NULL)) return 0; BIO_clear_retry_flags(b); @@ -183,10 +185,11 @@ static int b64_read(BIO *b, char *out, int outl) * We need to scan, a line at a time until we have a valid line if we * are starting. */ - if (ctx->start && (BIO_get_flags(b) & BIO_FLAGS_BASE64_NO_NL) != 0) { + if (ctx->start && (BIO_get_flags(b) & BIO_FLAGS_BASE64_NO_NL)) { + /* ctx->start=1; */ ctx->tmp_len = 0; } else if (ctx->start) { - q = p = ctx->tmp; + q = p = (unsigned char *)ctx->tmp; num = 0; for (j = 0; j < i; j++) { if (*(q++) != '\n') @@ -203,12 +206,16 @@ static int b64_read(BIO *b, char *out, int outl) continue; } - k = EVP_DecodeUpdate(ctx->base64, ctx->buf, &num, p, q - p); - if (k <= 0 && num == 0 && ctx->start) { + k = EVP_DecodeUpdate(ctx->base64, + (unsigned char *)ctx->buf, + &num, p, q - p); + if ((k <= 0) && (num == 0) && (ctx->start)) EVP_DecodeInit(ctx->base64); - } else { - if (p != ctx->tmp) { - i -= p - ctx->tmp; + else { + if (p != (unsigned char *) + &(ctx->tmp[0])) { + i -= (p - (unsigned char *) + &(ctx->tmp[0])); for (x = 0; x < i; x++) ctx->tmp[x] = p[x]; } @@ -220,12 +227,12 @@ static int b64_read(BIO *b, char *out, int outl) } /* we fell off the end without starting */ - if (j == i && num == 0) { + if ((j == i) && (num == 0)) { /* * Is this is one long chunk?, if so, keep on reading until a * new line. */ - if (p == ctx->tmp) { + if (p == (unsigned char *)&(ctx->tmp[0])) { /* Check buffer full */ if (i == B64_BLOCK_SIZE) { ctx->tmp_nl = 1; @@ -242,7 +249,7 @@ static int b64_read(BIO *b, char *out, int outl) } else { ctx->tmp_len = 0; } - } else if (i < B64_BLOCK_SIZE && ctx->cont > 0) { + } else if ((i < B64_BLOCK_SIZE) && (ctx->cont > 0)) { /* * If buffer isn't full and we can retry then restart to read in * more data. @@ -250,11 +257,12 @@ static int b64_read(BIO *b, char *out, int outl) continue; } - if ((BIO_get_flags(b) & BIO_FLAGS_BASE64_NO_NL) != 0) { + if (BIO_get_flags(b) & BIO_FLAGS_BASE64_NO_NL) { int z, jj; jj = i & ~3; /* process per 4 */ - z = EVP_DecodeBlock(ctx->buf, ctx->tmp, jj); + z = EVP_DecodeBlock((unsigned char *)ctx->buf, + (unsigned char *)ctx->tmp, jj); if (jj > 2) { if (ctx->tmp[jj - 1] == '=') { z--; @@ -275,8 +283,9 @@ static int b64_read(BIO *b, char *out, int outl) } i = z; } else { - i = EVP_DecodeUpdate(ctx->base64, ctx->buf, &ctx->buf_len, - ctx->tmp, i); + i = EVP_DecodeUpdate(ctx->base64, + (unsigned char *)ctx->buf, &ctx->buf_len, + (unsigned char *)ctx->tmp, i); ctx->tmp_len = 0; } /* @@ -311,7 +320,7 @@ static int b64_read(BIO *b, char *out, int outl) } /* BIO_clear_retry_flags(b); */ BIO_copy_next_retry(b); - return ret == 0 ? ret_code : ret; + return ((ret == 0) ? ret_code : ret); } static int b64_write(BIO *b, const char *in, int inl) @@ -324,7 +333,7 @@ static int b64_write(BIO *b, const char *in, int inl) ctx = (BIO_B64_CTX *)BIO_get_data(b); next = BIO_next(b); - if (ctx == NULL || next == NULL) + if ((ctx == NULL) || (next == NULL)) return 0; BIO_clear_retry_flags(b); @@ -357,13 +366,13 @@ static int b64_write(BIO *b, const char *in, int inl) ctx->buf_off = 0; ctx->buf_len = 0; - if (in == NULL || inl <= 0) + if ((in == NULL) || (inl <= 0)) return 0; while (inl > 0) { - n = inl > B64_BLOCK_SIZE ? B64_BLOCK_SIZE : inl; + n = (inl > B64_BLOCK_SIZE) ? B64_BLOCK_SIZE : inl; - if ((BIO_get_flags(b) & BIO_FLAGS_BASE64_NO_NL) != 0) { + if (BIO_get_flags(b) & BIO_FLAGS_BASE64_NO_NL) { if (ctx->tmp_len > 0) { OPENSSL_assert(ctx->tmp_len <= 3); n = 3 - ctx->tmp_len; @@ -378,7 +387,8 @@ static int b64_write(BIO *b, const char *in, int inl) if (ctx->tmp_len < 3) break; ctx->buf_len = - EVP_EncodeBlock(ctx->buf, ctx->tmp, ctx->tmp_len); + EVP_EncodeBlock((unsigned char *)ctx->buf, + (unsigned char *)ctx->tmp, ctx->tmp_len); OPENSSL_assert(ctx->buf_len <= (int)sizeof(ctx->buf)); OPENSSL_assert(ctx->buf_len >= ctx->buf_off); /* @@ -395,15 +405,17 @@ static int b64_write(BIO *b, const char *in, int inl) } n -= n % 3; ctx->buf_len = - EVP_EncodeBlock(ctx->buf, (unsigned char *)in, n); + EVP_EncodeBlock((unsigned char *)ctx->buf, + (const unsigned char *)in, n); OPENSSL_assert(ctx->buf_len <= (int)sizeof(ctx->buf)); OPENSSL_assert(ctx->buf_len >= ctx->buf_off); ret += n; } } else { - if (!EVP_EncodeUpdate(ctx->base64, ctx->buf, &ctx->buf_len, - (unsigned char *)in, n)) - return ret == 0 ? -1 : ret; + if (!EVP_EncodeUpdate(ctx->base64, + (unsigned char *)ctx->buf, &ctx->buf_len, + (unsigned char *)in, n)) + return ((ret == 0) ? -1 : ret); OPENSSL_assert(ctx->buf_len <= (int)sizeof(ctx->buf)); OPENSSL_assert(ctx->buf_len >= ctx->buf_off); ret += n; @@ -417,7 +429,7 @@ static int b64_write(BIO *b, const char *in, int inl) i = BIO_write(next, &(ctx->buf[ctx->buf_off]), n); if (i <= 0) { BIO_copy_next_retry(b); - return ret == 0 ? i : ret; + return ((ret == 0) ? i : ret); } OPENSSL_assert(i <= n); n -= i; @@ -440,7 +452,7 @@ static long b64_ctrl(BIO *b, int cmd, long num, void *ptr) ctx = (BIO_B64_CTX *)BIO_get_data(b); next = BIO_next(b); - if (ctx == NULL || next == NULL) + if ((ctx == NULL) || (next == NULL)) return 0; switch (cmd) { @@ -459,8 +471,8 @@ static long b64_ctrl(BIO *b, int cmd, long num, void *ptr) case BIO_CTRL_WPENDING: /* More to write in buffer */ OPENSSL_assert(ctx->buf_len >= ctx->buf_off); ret = ctx->buf_len - ctx->buf_off; - if (ret == 0 && ctx->encode != B64_NONE - && EVP_ENCODE_CTX_num(ctx->base64) != 0) + if ((ret == 0) && (ctx->encode != B64_NONE) + && (EVP_ENCODE_CTX_num(ctx->base64) != 0)) ret = 1; else if (ret <= 0) ret = BIO_ctrl(next, cmd, num, ptr); @@ -481,8 +493,9 @@ static long b64_ctrl(BIO *b, int cmd, long num, void *ptr) } if (BIO_get_flags(b) & BIO_FLAGS_BASE64_NO_NL) { if (ctx->tmp_len != 0) { - ctx->buf_len = EVP_EncodeBlock(ctx->buf, - ctx->tmp, ctx->tmp_len); + ctx->buf_len = EVP_EncodeBlock((unsigned char *)ctx->buf, + (unsigned char *)ctx->tmp, + ctx->tmp_len); ctx->buf_off = 0; ctx->tmp_len = 0; goto again; @@ -490,13 +503,13 @@ static long b64_ctrl(BIO *b, int cmd, long num, void *ptr) } else if (ctx->encode != B64_NONE && EVP_ENCODE_CTX_num(ctx->base64) != 0) { ctx->buf_off = 0; - EVP_EncodeFinal(ctx->base64, ctx->buf, &(ctx->buf_len)); + EVP_EncodeFinal(ctx->base64, + (unsigned char *)ctx->buf, &(ctx->buf_len)); /* push out the bytes */ goto again; } /* Finally flush the underlying BIO */ ret = BIO_ctrl(next, cmd, num, ptr); - BIO_copy_next_retry(b); break; case BIO_C_DO_STATE_MACHINE: diff --git a/openssl/src/crypto/evp/bio_enc.c b/openssl/src/crypto/evp/bio_enc.c index ffe4b5bb0..2d52c48d1 100644 --- a/openssl/src/crypto/evp/bio_enc.c +++ b/openssl/src/crypto/evp/bio_enc.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -65,8 +65,10 @@ static int enc_new(BIO *bi) { BIO_ENC_CTX *ctx; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } ctx->cipher = EVP_CIPHER_CTX_new(); if (ctx->cipher == NULL) { @@ -132,10 +134,6 @@ static int enc_read(BIO *b, char *out, int outl) } blocksize = EVP_CIPHER_CTX_get_block_size(ctx->cipher); - - if (blocksize == 0) - return 0; - if (blocksize == 1) blocksize = 0; @@ -301,7 +299,6 @@ static long enc_ctrl(BIO *b, int cmd, long num, void *ptr) int i; EVP_CIPHER_CTX **c_ctx; BIO *next; - int pend; ctx = BIO_get_data(b); next = BIO_next(b); @@ -337,14 +334,8 @@ static long enc_ctrl(BIO *b, int cmd, long num, void *ptr) /* do a final write */ again: while (ctx->buf_len != ctx->buf_off) { - pend = ctx->buf_len - ctx->buf_off; i = enc_write(b, NULL, 0); - /* - * i should never be > 0 here because we didn't ask to write any - * new data. We stop if we get an error or we failed to make any - * progress writing pending data. - */ - if (i < 0 || (ctx->buf_len - ctx->buf_off) == pend) + if (i < 0) return i; } @@ -364,7 +355,6 @@ static long enc_ctrl(BIO *b, int cmd, long num, void *ptr) /* Finally flush the underlying BIO */ ret = BIO_ctrl(next, cmd, num, ptr); - BIO_copy_next_retry(b); break; case BIO_C_GET_CIPHER_STATUS: ret = (long)ctx->ok; diff --git a/openssl/src/crypto/evp/bio_ok.c b/openssl/src/crypto/evp/bio_ok.c index 2aa1ed755..97e67fcb6 100644 --- a/openssl/src/crypto/evp/bio_ok.c +++ b/openssl/src/crypto/evp/bio_ok.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -132,8 +132,10 @@ static int ok_new(BIO *bi) { BIO_OK_CTX *ctx; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } ctx->cont = 1; ctx->sigio = 1; @@ -372,7 +374,6 @@ static long ok_ctrl(BIO *b, int cmd, long num, void *ptr) /* Finally flush the underlying BIO */ ret = BIO_ctrl(next, cmd, num, ptr); - BIO_copy_next_retry(b); break; case BIO_C_DO_STATE_MACHINE: BIO_clear_retry_flags(b); diff --git a/openssl/src/crypto/evp/c_allc.c b/openssl/src/crypto/evp/c_allc.c index c74b3dcd7..4bcd4db69 100644 --- a/openssl/src/crypto/evp/c_allc.c +++ b/openssl/src/crypto/evp/c_allc.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -61,24 +61,6 @@ void openssl_add_all_ciphers_int(void) # endif #endif -#ifndef OPENSSL_NO_IDEA - EVP_add_cipher(EVP_idea_ecb()); - EVP_add_cipher(EVP_idea_cfb()); - EVP_add_cipher(EVP_idea_ofb()); - EVP_add_cipher(EVP_idea_cbc()); - EVP_add_cipher_alias(SN_idea_cbc, "IDEA"); - EVP_add_cipher_alias(SN_idea_cbc, "idea"); -#endif - -#ifndef OPENSSL_NO_SEED - EVP_add_cipher(EVP_seed_ecb()); - EVP_add_cipher(EVP_seed_cfb()); - EVP_add_cipher(EVP_seed_ofb()); - EVP_add_cipher(EVP_seed_cbc()); - EVP_add_cipher_alias(SN_seed_cbc, "SEED"); - EVP_add_cipher_alias(SN_seed_cbc, "seed"); -#endif - #ifndef OPENSSL_NO_SM4 EVP_add_cipher(EVP_sm4_ecb()); EVP_add_cipher(EVP_sm4_cbc()); @@ -87,41 +69,8 @@ void openssl_add_all_ciphers_int(void) EVP_add_cipher(EVP_sm4_ctr()); EVP_add_cipher_alias(SN_sm4_cbc, "SM4"); EVP_add_cipher_alias(SN_sm4_cbc, "sm4"); -#endif - -#ifndef OPENSSL_NO_RC2 - EVP_add_cipher(EVP_rc2_ecb()); - EVP_add_cipher(EVP_rc2_cfb()); - EVP_add_cipher(EVP_rc2_ofb()); - EVP_add_cipher(EVP_rc2_cbc()); - EVP_add_cipher(EVP_rc2_40_cbc()); - EVP_add_cipher(EVP_rc2_64_cbc()); - EVP_add_cipher_alias(SN_rc2_cbc, "RC2"); - EVP_add_cipher_alias(SN_rc2_cbc, "rc2"); - EVP_add_cipher_alias(SN_rc2_cbc, "rc2-128"); - EVP_add_cipher_alias(SN_rc2_64_cbc, "rc2-64"); - EVP_add_cipher_alias(SN_rc2_40_cbc, "rc2-40"); -#endif - -#ifndef OPENSSL_NO_BF - EVP_add_cipher(EVP_bf_ecb()); - EVP_add_cipher(EVP_bf_cfb()); - EVP_add_cipher(EVP_bf_ofb()); - EVP_add_cipher(EVP_bf_cbc()); - EVP_add_cipher_alias(SN_bf_cbc, "BF"); - EVP_add_cipher_alias(SN_bf_cbc, "bf"); - EVP_add_cipher_alias(SN_bf_cbc, "blowfish"); -#endif - -#ifndef OPENSSL_NO_CAST - EVP_add_cipher(EVP_cast5_ecb()); - EVP_add_cipher(EVP_cast5_cfb()); - EVP_add_cipher(EVP_cast5_ofb()); - EVP_add_cipher(EVP_cast5_cbc()); - EVP_add_cipher_alias(SN_cast5_cbc, "CAST"); - EVP_add_cipher_alias(SN_cast5_cbc, "cast"); - EVP_add_cipher_alias(SN_cast5_cbc, "CAST-cbc"); - EVP_add_cipher_alias(SN_cast5_cbc, "cast-cbc"); + EVP_add_cipher(EVP_sm4_gcm()); + EVP_add_cipher(EVP_sm4_ccm()); #endif #ifndef OPENSSL_NO_RC5 @@ -149,7 +98,6 @@ void openssl_add_all_ciphers_int(void) EVP_add_cipher(EVP_aes_128_wrap()); EVP_add_cipher_alias(SN_id_aes128_wrap, "aes128-wrap"); EVP_add_cipher(EVP_aes_128_wrap_pad()); - EVP_add_cipher_alias(SN_id_aes128_wrap_pad, "aes128-wrap-pad"); EVP_add_cipher_alias(SN_aes_128_cbc, "AES128"); EVP_add_cipher_alias(SN_aes_128_cbc, "aes128"); EVP_add_cipher(EVP_aes_192_ecb()); @@ -167,7 +115,6 @@ void openssl_add_all_ciphers_int(void) EVP_add_cipher(EVP_aes_192_wrap()); EVP_add_cipher_alias(SN_id_aes192_wrap, "aes192-wrap"); EVP_add_cipher(EVP_aes_192_wrap_pad()); - EVP_add_cipher_alias(SN_id_aes192_wrap_pad, "aes192-wrap-pad"); EVP_add_cipher_alias(SN_aes_192_cbc, "AES192"); EVP_add_cipher_alias(SN_aes_192_cbc, "aes192"); EVP_add_cipher(EVP_aes_256_ecb()); @@ -186,78 +133,12 @@ void openssl_add_all_ciphers_int(void) EVP_add_cipher(EVP_aes_256_wrap()); EVP_add_cipher_alias(SN_id_aes256_wrap, "aes256-wrap"); EVP_add_cipher(EVP_aes_256_wrap_pad()); - EVP_add_cipher_alias(SN_id_aes256_wrap_pad, "aes256-wrap-pad"); EVP_add_cipher_alias(SN_aes_256_cbc, "AES256"); EVP_add_cipher_alias(SN_aes_256_cbc, "aes256"); EVP_add_cipher(EVP_aes_128_cbc_hmac_sha1()); EVP_add_cipher(EVP_aes_256_cbc_hmac_sha1()); EVP_add_cipher(EVP_aes_128_cbc_hmac_sha256()); EVP_add_cipher(EVP_aes_256_cbc_hmac_sha256()); -#ifndef OPENSSL_NO_ARIA - EVP_add_cipher(EVP_aria_128_ecb()); - EVP_add_cipher(EVP_aria_128_cbc()); - EVP_add_cipher(EVP_aria_128_cfb()); - EVP_add_cipher(EVP_aria_128_cfb1()); - EVP_add_cipher(EVP_aria_128_cfb8()); - EVP_add_cipher(EVP_aria_128_ctr()); - EVP_add_cipher(EVP_aria_128_ofb()); - EVP_add_cipher(EVP_aria_128_gcm()); - EVP_add_cipher(EVP_aria_128_ccm()); - EVP_add_cipher_alias(SN_aria_128_cbc, "ARIA128"); - EVP_add_cipher_alias(SN_aria_128_cbc, "aria128"); - EVP_add_cipher(EVP_aria_192_ecb()); - EVP_add_cipher(EVP_aria_192_cbc()); - EVP_add_cipher(EVP_aria_192_cfb()); - EVP_add_cipher(EVP_aria_192_cfb1()); - EVP_add_cipher(EVP_aria_192_cfb8()); - EVP_add_cipher(EVP_aria_192_ctr()); - EVP_add_cipher(EVP_aria_192_ofb()); - EVP_add_cipher(EVP_aria_192_gcm()); - EVP_add_cipher(EVP_aria_192_ccm()); - EVP_add_cipher_alias(SN_aria_192_cbc, "ARIA192"); - EVP_add_cipher_alias(SN_aria_192_cbc, "aria192"); - EVP_add_cipher(EVP_aria_256_ecb()); - EVP_add_cipher(EVP_aria_256_cbc()); - EVP_add_cipher(EVP_aria_256_cfb()); - EVP_add_cipher(EVP_aria_256_cfb1()); - EVP_add_cipher(EVP_aria_256_cfb8()); - EVP_add_cipher(EVP_aria_256_ctr()); - EVP_add_cipher(EVP_aria_256_ofb()); - EVP_add_cipher(EVP_aria_256_gcm()); - EVP_add_cipher(EVP_aria_256_ccm()); - EVP_add_cipher_alias(SN_aria_256_cbc, "ARIA256"); - EVP_add_cipher_alias(SN_aria_256_cbc, "aria256"); -#endif - -#ifndef OPENSSL_NO_CAMELLIA - EVP_add_cipher(EVP_camellia_128_ecb()); - EVP_add_cipher(EVP_camellia_128_cbc()); - EVP_add_cipher(EVP_camellia_128_cfb()); - EVP_add_cipher(EVP_camellia_128_cfb1()); - EVP_add_cipher(EVP_camellia_128_cfb8()); - EVP_add_cipher(EVP_camellia_128_ofb()); - EVP_add_cipher_alias(SN_camellia_128_cbc, "CAMELLIA128"); - EVP_add_cipher_alias(SN_camellia_128_cbc, "camellia128"); - EVP_add_cipher(EVP_camellia_192_ecb()); - EVP_add_cipher(EVP_camellia_192_cbc()); - EVP_add_cipher(EVP_camellia_192_cfb()); - EVP_add_cipher(EVP_camellia_192_cfb1()); - EVP_add_cipher(EVP_camellia_192_cfb8()); - EVP_add_cipher(EVP_camellia_192_ofb()); - EVP_add_cipher_alias(SN_camellia_192_cbc, "CAMELLIA192"); - EVP_add_cipher_alias(SN_camellia_192_cbc, "camellia192"); - EVP_add_cipher(EVP_camellia_256_ecb()); - EVP_add_cipher(EVP_camellia_256_cbc()); - EVP_add_cipher(EVP_camellia_256_cfb()); - EVP_add_cipher(EVP_camellia_256_cfb1()); - EVP_add_cipher(EVP_camellia_256_cfb8()); - EVP_add_cipher(EVP_camellia_256_ofb()); - EVP_add_cipher_alias(SN_camellia_256_cbc, "CAMELLIA256"); - EVP_add_cipher_alias(SN_camellia_256_cbc, "camellia256"); - EVP_add_cipher(EVP_camellia_128_ctr()); - EVP_add_cipher(EVP_camellia_192_ctr()); - EVP_add_cipher(EVP_camellia_256_ctr()); -#endif #ifndef OPENSSL_NO_CHACHA EVP_add_cipher(EVP_chacha20()); @@ -265,4 +146,8 @@ void openssl_add_all_ciphers_int(void) EVP_add_cipher(EVP_chacha20_poly1305()); # endif #endif + +#ifndef OPENSSL_NO_ZUC + EVP_add_cipher(EVP_eea3()); +#endif } diff --git a/openssl/src/crypto/evp/c_alld.c b/openssl/src/crypto/evp/c_alld.c index f7d62bd2e..2c80951c4 100644 --- a/openssl/src/crypto/evp/c_alld.c +++ b/openssl/src/crypto/evp/c_alld.c @@ -16,9 +16,6 @@ void openssl_add_all_digests_int(void) { -#ifndef OPENSSL_NO_MD4 - EVP_add_digest(EVP_md4()); -#endif #ifndef OPENSSL_NO_MD5 EVP_add_digest(EVP_md5()); EVP_add_digest_alias(SN_md5, "ssl3-md5"); @@ -27,29 +24,14 @@ void openssl_add_all_digests_int(void) EVP_add_digest(EVP_sha1()); EVP_add_digest_alias(SN_sha1, "ssl3-sha1"); EVP_add_digest_alias(SN_sha1WithRSAEncryption, SN_sha1WithRSA); -#if !defined(OPENSSL_NO_MDC2) && !defined(OPENSSL_NO_DES) - EVP_add_digest(EVP_mdc2()); -#endif -#ifndef OPENSSL_NO_RMD160 - EVP_add_digest(EVP_ripemd160()); - EVP_add_digest_alias(SN_ripemd160, "ripemd"); - EVP_add_digest_alias(SN_ripemd160, "rmd160"); -#endif EVP_add_digest(EVP_sha224()); EVP_add_digest(EVP_sha256()); EVP_add_digest(EVP_sha384()); EVP_add_digest(EVP_sha512()); EVP_add_digest(EVP_sha512_224()); EVP_add_digest(EVP_sha512_256()); -#ifndef OPENSSL_NO_WHIRLPOOL - EVP_add_digest(EVP_whirlpool()); -#endif #ifndef OPENSSL_NO_SM3 EVP_add_digest(EVP_sm3()); -#endif -#ifndef OPENSSL_NO_BLAKE2 - EVP_add_digest(EVP_blake2b512()); - EVP_add_digest(EVP_blake2s256()); #endif EVP_add_digest(EVP_sha3_224()); EVP_add_digest(EVP_sha3_256()); diff --git a/openssl/src/crypto/evp/cmeth_lib.c b/openssl/src/crypto/evp/cmeth_lib.c index 41a1bade2..a806ec5f9 100644 --- a/openssl/src/crypto/evp/cmeth_lib.c +++ b/openssl/src/crypto/evp/cmeth_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -46,10 +46,10 @@ EVP_CIPHER *EVP_CIPHER_meth_dup(const EVP_CIPHER *cipher) if ((to = EVP_CIPHER_meth_new(cipher->nid, cipher->block_size, cipher->key_len)) != NULL) { - CRYPTO_REF_COUNT refcnt = to->refcnt; + CRYPTO_RWLOCK *lock = to->lock; memcpy(to, cipher, sizeof(*to)); - to->refcnt = refcnt; + to->lock = lock; to->origin = EVP_ORIG_METH; } return to; diff --git a/openssl/src/crypto/evp/ctrl_params_translate.c b/openssl/src/crypto/evp/ctrl_params_translate.c index 54e589054..c767c3164 100644 --- a/openssl/src/crypto/evp/ctrl_params_translate.c +++ b/openssl/src/crypto/evp/ctrl_params_translate.c @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -387,7 +387,7 @@ static int default_fixup_args(enum state state, { int ret; - if ((ret = default_check(state, translation, ctx)) <= 0) + if ((ret = default_check(state, translation, ctx)) < 0) return ret; switch (state) { @@ -458,9 +458,11 @@ static int default_fixup_args(enum state state, if (ctx->p2 != NULL) { if (ctx->action_type == SET) { ctx->buflen = BN_num_bytes(ctx->p2); - if ((ctx->allocated_buf - = OPENSSL_malloc(ctx->buflen)) == NULL) + if ((ctx->allocated_buf = + OPENSSL_malloc(ctx->buflen)) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } if (BN_bn2nativepad(ctx->p2, ctx->allocated_buf, ctx->buflen) < 0) { OPENSSL_free(ctx->allocated_buf); @@ -634,8 +636,8 @@ static int default_fixup_args(enum state state, ctx->p2, ctx->sz); case OSSL_PARAM_OCTET_STRING: return OSSL_PARAM_get_octet_string(ctx->params, - &ctx->p2, ctx->sz, - (size_t *)&ctx->p1); + ctx->p2, ctx->sz, + &ctx->sz); case OSSL_PARAM_OCTET_PTR: return OSSL_PARAM_get_octet_ptr(ctx->params, ctx->p2, &ctx->sz); @@ -683,7 +685,7 @@ static int default_fixup_args(enum state state, return OSSL_PARAM_set_octet_string(ctx->params, ctx->p2, size); case OSSL_PARAM_OCTET_PTR: - return OSSL_PARAM_set_octet_ptr(ctx->params, *(void **)ctx->p2, + return OSSL_PARAM_set_octet_ptr(ctx->params, ctx->p2, size); default: ERR_raise_data(ERR_LIB_EVP, ERR_R_UNSUPPORTED, @@ -693,9 +695,6 @@ static int default_fixup_args(enum state state, translation->param_data_type); return 0; } - } else if (state == PRE_PARAMS_TO_CTRL && ctx->action_type == GET) { - if (translation->param_data_type == OSSL_PARAM_OCTET_PTR) - ctx->p2 = &ctx->bufp; } } /* Any other combination is simply pass-through */ @@ -782,7 +781,7 @@ static int fix_cipher_md(enum state state, if (state == POST_CTRL_TO_PARAMS && ctx->action_type == GET) { /* - * Here's how we reuse |ctx->orig_p2| that was set in the + * Here's how we re-use |ctx->orig_p2| that was set in the * PRE_CTRL_TO_PARAMS state above. */ *(void **)ctx->orig_p2 = @@ -1137,7 +1136,6 @@ static int fix_ec_paramgen_curve_nid(enum state state, const struct translation_st *translation, struct translation_ctx_st *ctx) { - char *p2 = NULL; int ret; if ((ret = default_check(state, translation, ctx)) <= 0) @@ -1150,25 +1148,13 @@ static int fix_ec_paramgen_curve_nid(enum state state, if (state == PRE_CTRL_TO_PARAMS) { ctx->p2 = (char *)OBJ_nid2sn(ctx->p1); ctx->p1 = 0; - } else if (state == PRE_PARAMS_TO_CTRL) { - /* - * We're translating from params to ctrl and setting the curve name. - * The ctrl function needs it to be a NID, but meanwhile, we need - * space to get the curve name from the param. |ctx->name_buf| is - * sufficient for that. - * The double indirection is necessary for default_fixup_args()'s - * call of OSSL_PARAM_get_utf8_string() to be done correctly. - */ - p2 = ctx->name_buf; - ctx->p2 = &p2; - ctx->sz = sizeof(ctx->name_buf); } if ((ret = default_fixup_args(state, translation, ctx)) <= 0) return ret; if (state == PRE_PARAMS_TO_CTRL) { - ctx->p1 = OBJ_sn2nid(p2); + ctx->p1 = OBJ_sn2nid(ctx->p2); ctx->p2 = NULL; } @@ -1658,64 +1644,6 @@ static int get_payload_public_key(enum state state, return ret; } -static int get_payload_public_key_ec(enum state state, - const struct translation_st *translation, - struct translation_ctx_st *ctx) -{ -#ifndef OPENSSL_NO_EC - EVP_PKEY *pkey = ctx->p2; - const EC_KEY *eckey = EVP_PKEY_get0_EC_KEY(pkey); - BN_CTX *bnctx; - const EC_POINT *point; - const EC_GROUP *ecg; - BIGNUM *x = NULL; - BIGNUM *y = NULL; - int ret = 0; - - ctx->p2 = NULL; - - if (eckey == NULL) { - ERR_raise(ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEY_TYPE); - return 0; - } - - bnctx = BN_CTX_new_ex(ossl_ec_key_get_libctx(eckey)); - if (bnctx == NULL) - return 0; - - point = EC_KEY_get0_public_key(eckey); - ecg = EC_KEY_get0_group(eckey); - - /* Caller should have requested a BN, fail if not */ - if (ctx->params->data_type != OSSL_PARAM_UNSIGNED_INTEGER) - goto out; - - x = BN_CTX_get(bnctx); - y = BN_CTX_get(bnctx); - if (y == NULL) - goto out; - - if (!EC_POINT_get_affine_coordinates(ecg, point, x, y, bnctx)) - goto out; - - if (strncmp(ctx->params->key, OSSL_PKEY_PARAM_EC_PUB_X, 2) == 0) - ctx->p2 = x; - else if (strncmp(ctx->params->key, OSSL_PKEY_PARAM_EC_PUB_Y, 2) == 0) - ctx->p2 = y; - else - goto out; - - /* Return the payload */ - ret = default_fixup_args(state, translation, ctx); -out: - BN_CTX_free(bnctx); - return ret; -#else - ERR_raise(ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEY_TYPE); - return 0; -#endif -} - static int get_payload_bn(enum state state, const struct translation_st *translation, struct translation_ctx_st *ctx, const BIGNUM *bn) @@ -1842,8 +1770,7 @@ static int get_rsa_payload_n(enum state state, { const BIGNUM *bn = NULL; - if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA - && EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA_PSS) + if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA) return 0; bn = RSA_get0_n(EVP_PKEY_get0_RSA(ctx->p2)); @@ -1856,8 +1783,7 @@ static int get_rsa_payload_e(enum state state, { const BIGNUM *bn = NULL; - if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA - && EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA_PSS) + if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA) return 0; bn = RSA_get0_e(EVP_PKEY_get0_RSA(ctx->p2)); @@ -1870,8 +1796,7 @@ static int get_rsa_payload_d(enum state state, { const BIGNUM *bn = NULL; - if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA - && EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA_PSS) + if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA) return 0; bn = RSA_get0_d(EVP_PKEY_get0_RSA(ctx->p2)); @@ -1971,8 +1896,7 @@ static int get_rsa_payload_coefficient(enum state state, const struct translation_st *translation, \ struct translation_ctx_st *ctx) \ { \ - if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA \ - && EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA_PSS) \ + if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA) \ return 0; \ return get_rsa_payload_factor(state, translation, ctx, n - 1); \ } @@ -1983,8 +1907,7 @@ static int get_rsa_payload_coefficient(enum state state, const struct translation_st *translation, \ struct translation_ctx_st *ctx) \ { \ - if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA \ - && EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA_PSS) \ + if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA) \ return 0; \ return get_rsa_payload_exponent(state, translation, ctx, \ n - 1); \ @@ -1996,8 +1919,7 @@ static int get_rsa_payload_coefficient(enum state state, const struct translation_st *translation, \ struct translation_ctx_st *ctx) \ { \ - if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA \ - && EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA_PSS) \ + if (EVP_PKEY_get_base_id(ctx->p2) != EVP_PKEY_RSA) \ return 0; \ return get_rsa_payload_coefficient(state, translation, ctx, \ n - 1); \ @@ -2218,46 +2140,6 @@ static const struct translation_st evp_pkey_ctx_translations[] = { EVP_PKEY_CTRL_GET_EC_KDF_UKM, NULL, NULL, OSSL_EXCHANGE_PARAM_KDF_UKM, OSSL_PARAM_OCTET_PTR, NULL }, - /*- - * SM2 - * == - */ - { SET, EVP_PKEY_SM2, 0, EVP_PKEY_OP_PARAMGEN | EVP_PKEY_OP_KEYGEN, - EVP_PKEY_CTRL_EC_PARAM_ENC, "ec_param_enc", NULL, - OSSL_PKEY_PARAM_EC_ENCODING, OSSL_PARAM_UTF8_STRING, fix_ec_param_enc }, - { SET, EVP_PKEY_SM2, 0, EVP_PKEY_OP_PARAMGEN | EVP_PKEY_OP_KEYGEN, - EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID, "ec_paramgen_curve", NULL, - OSSL_PKEY_PARAM_GROUP_NAME, OSSL_PARAM_UTF8_STRING, - fix_ec_paramgen_curve_nid }, - /* - * EVP_PKEY_CTRL_EC_ECDH_COFACTOR and EVP_PKEY_CTRL_EC_KDF_TYPE are used - * both for setting and getting. The fixup function has to handle this... - */ - { NONE, EVP_PKEY_SM2, 0, EVP_PKEY_OP_DERIVE, - EVP_PKEY_CTRL_EC_ECDH_COFACTOR, "ecdh_cofactor_mode", NULL, - OSSL_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE, OSSL_PARAM_INTEGER, - fix_ecdh_cofactor }, - { NONE, EVP_PKEY_SM2, 0, EVP_PKEY_OP_DERIVE, - EVP_PKEY_CTRL_EC_KDF_TYPE, NULL, NULL, - OSSL_EXCHANGE_PARAM_KDF_TYPE, OSSL_PARAM_UTF8_STRING, fix_ec_kdf_type }, - { SET, EVP_PKEY_SM2, 0, EVP_PKEY_OP_DERIVE, - EVP_PKEY_CTRL_EC_KDF_MD, "ecdh_kdf_md", NULL, - OSSL_EXCHANGE_PARAM_KDF_DIGEST, OSSL_PARAM_UTF8_STRING, fix_md }, - { GET, EVP_PKEY_SM2, 0, EVP_PKEY_OP_DERIVE, - EVP_PKEY_CTRL_GET_EC_KDF_MD, NULL, NULL, - OSSL_EXCHANGE_PARAM_KDF_DIGEST, OSSL_PARAM_UTF8_STRING, fix_md }, - { SET, EVP_PKEY_SM2, 0, EVP_PKEY_OP_DERIVE, - EVP_PKEY_CTRL_EC_KDF_OUTLEN, NULL, NULL, - OSSL_EXCHANGE_PARAM_KDF_OUTLEN, OSSL_PARAM_UNSIGNED_INTEGER, NULL }, - { GET, EVP_PKEY_SM2, 0, EVP_PKEY_OP_DERIVE, - EVP_PKEY_CTRL_GET_EC_KDF_OUTLEN, NULL, NULL, - OSSL_EXCHANGE_PARAM_KDF_OUTLEN, OSSL_PARAM_UNSIGNED_INTEGER, NULL }, - { SET, EVP_PKEY_SM2, 0, EVP_PKEY_OP_DERIVE, - EVP_PKEY_CTRL_EC_KDF_UKM, NULL, NULL, - OSSL_EXCHANGE_PARAM_KDF_UKM, OSSL_PARAM_OCTET_STRING, NULL }, - { GET, EVP_PKEY_SM2, 0, EVP_PKEY_OP_DERIVE, - EVP_PKEY_CTRL_GET_EC_KDF_UKM, NULL, NULL, - OSSL_EXCHANGE_PARAM_KDF_UKM, OSSL_PARAM_OCTET_PTR, NULL }, /*- * RSA * === @@ -2310,7 +2192,7 @@ static const struct translation_st evp_pkey_ctx_translations[] = { OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST, OSSL_PARAM_UTF8_STRING, fix_md }, /* * The "rsa_oaep_label" ctrl_str expects the value to always be hex. - * This is accommodated by default_fixup_args() above, which mimics that + * This is accomodated by default_fixup_args() above, which mimics that * expectation for any translation item where |ctrl_str| is NULL and * |ctrl_hexstr| is non-NULL. */ @@ -2319,13 +2201,7 @@ static const struct translation_st evp_pkey_ctx_translations[] = { OSSL_ASYM_CIPHER_PARAM_OAEP_LABEL, OSSL_PARAM_OCTET_STRING, NULL }, { GET, EVP_PKEY_RSA, 0, EVP_PKEY_OP_TYPE_CRYPT, EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL, NULL, NULL, - OSSL_ASYM_CIPHER_PARAM_OAEP_LABEL, OSSL_PARAM_OCTET_PTR, NULL }, - - { SET, EVP_PKEY_RSA, 0, EVP_PKEY_OP_TYPE_CRYPT, - EVP_PKEY_CTRL_RSA_IMPLICIT_REJECTION, NULL, - "rsa_pkcs1_implicit_rejection", - OSSL_ASYM_CIPHER_PARAM_IMPLICIT_REJECTION, OSSL_PARAM_UNSIGNED_INTEGER, - NULL }, + OSSL_ASYM_CIPHER_PARAM_OAEP_LABEL, OSSL_PARAM_OCTET_STRING, NULL }, { SET, EVP_PKEY_RSA_PSS, 0, EVP_PKEY_OP_TYPE_GEN, EVP_PKEY_CTRL_MD, "rsa_pss_keygen_md", NULL, @@ -2339,10 +2215,10 @@ static const struct translation_st evp_pkey_ctx_translations[] = { { SET, EVP_PKEY_RSA, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN, EVP_PKEY_CTRL_RSA_KEYGEN_BITS, "rsa_keygen_bits", NULL, OSSL_PKEY_PARAM_RSA_BITS, OSSL_PARAM_UNSIGNED_INTEGER, NULL }, - { SET, EVP_PKEY_RSA, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN, + { SET, EVP_PKEY_RSA, 0, EVP_PKEY_OP_KEYGEN, EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP, "rsa_keygen_pubexp", NULL, OSSL_PKEY_PARAM_RSA_E, OSSL_PARAM_UNSIGNED_INTEGER, NULL }, - { SET, EVP_PKEY_RSA, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN, + { SET, EVP_PKEY_RSA, 0, EVP_PKEY_OP_KEYGEN, EVP_PKEY_CTRL_RSA_KEYGEN_PRIMES, "rsa_keygen_primes", NULL, OSSL_PKEY_PARAM_RSA_PRIMES, OSSL_PARAM_UNSIGNED_INTEGER, NULL }, @@ -2431,12 +2307,8 @@ static const struct translation_st evp_pkey_ctx_translations[] = { */ { SET, EVP_PKEY_X25519, EVP_PKEY_X25519, EVP_PKEY_OP_KEYGEN, -1, NULL, NULL, OSSL_PKEY_PARAM_GROUP_NAME, OSSL_PARAM_UTF8_STRING, fix_group_ecx }, - { SET, EVP_PKEY_X25519, EVP_PKEY_X25519, EVP_PKEY_OP_PARAMGEN, -1, NULL, NULL, - OSSL_PKEY_PARAM_GROUP_NAME, OSSL_PARAM_UTF8_STRING, fix_group_ecx }, { SET, EVP_PKEY_X448, EVP_PKEY_X448, EVP_PKEY_OP_KEYGEN, -1, NULL, NULL, OSSL_PKEY_PARAM_GROUP_NAME, OSSL_PARAM_UTF8_STRING, fix_group_ecx }, - { SET, EVP_PKEY_X448, EVP_PKEY_X448, EVP_PKEY_OP_PARAMGEN, -1, NULL, NULL, - OSSL_PKEY_PARAM_GROUP_NAME, OSSL_PARAM_UTF8_STRING, fix_group_ecx }, }; static const struct translation_st evp_pkey_translations[] = { @@ -2458,12 +2330,6 @@ static const struct translation_st evp_pkey_translations[] = { OSSL_PKEY_PARAM_PUB_KEY, 0 /* no data type, let get_payload_public_key() handle that */, get_payload_public_key }, - { GET, -1, -1, -1, 0, NULL, NULL, - OSSL_PKEY_PARAM_EC_PUB_X, OSSL_PARAM_UNSIGNED_INTEGER, - get_payload_public_key_ec }, - { GET, -1, -1, -1, 0, NULL, NULL, - OSSL_PKEY_PARAM_EC_PUB_Y, OSSL_PARAM_UNSIGNED_INTEGER, - get_payload_public_key_ec }, /* DH and DSA */ { GET, -1, -1, -1, 0, NULL, NULL, @@ -2654,7 +2520,7 @@ lookup_translation(struct translation_st *tmpl, tmpl->ctrl_hexstr = ctrl_hexstr; } else if (tmpl->param_key != NULL) { /* - * Search criteria that originates from an OSSL_PARAM setter or + * Search criteria that originates from a OSSL_PARAM setter or * getter. * * Ctrls were fundamentally bidirectional, with only the ctrl @@ -2855,7 +2721,6 @@ static int evp_pkey_ctx_setget_params_to_ctrl(EVP_PKEY_CTX *pctx, if (translation->fixup_args != NULL) fixup = translation->fixup_args; ctx.action_type = translation->action_type; - ctx.ctrl_cmd = translation->ctrl_num; } ctx.pctx = pctx; ctx.params = params; diff --git a/openssl/src/crypto/evp/digest.c b/openssl/src/crypto/evp/digest.c index ab670a8f4..de9a1dcda 100644 --- a/openssl/src/crypto/evp/digest.c +++ b/openssl/src/crypto/evp/digest.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -110,7 +110,7 @@ EVP_MD_CTX *evp_md_ctx_new_ex(EVP_PKEY *pkey, const ASN1_OCTET_STRING *id, if ((ctx = EVP_MD_CTX_new()) == NULL || (pctx = EVP_PKEY_CTX_new_from_pkey(libctx, pkey, propq)) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); goto err; } @@ -141,20 +141,6 @@ void EVP_MD_CTX_free(EVP_MD_CTX *ctx) OPENSSL_free(ctx); } -int evp_md_ctx_free_algctx(EVP_MD_CTX *ctx) -{ - if (ctx->algctx != NULL) { - if (!ossl_assert(ctx->digest != NULL)) { - ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); - return 0; - } - if (ctx->digest->freectx != NULL) - ctx->digest->freectx(ctx->algctx); - ctx->algctx = NULL; - } - return 1; -} - static int evp_md_init_internal(EVP_MD_CTX *ctx, const EVP_MD *type, const OSSL_PARAM params[], ENGINE *impl) { @@ -181,8 +167,17 @@ static int evp_md_init_internal(EVP_MD_CTX *ctx, const EVP_MD *type, } #endif - EVP_MD_CTX_clear_flags(ctx, EVP_MD_CTX_FLAG_CLEANED - | EVP_MD_CTX_FLAG_FINALISED); + EVP_MD_CTX_clear_flags(ctx, EVP_MD_CTX_FLAG_CLEANED); + + if (ctx->algctx != NULL) { + if (!ossl_assert(ctx->digest != NULL)) { + ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); + return 0; + } + if (ctx->digest->freectx != NULL) + ctx->digest->freectx(ctx->algctx); + ctx->algctx = NULL; + } if (type != NULL) { ctx->reqdigest = type; @@ -202,20 +197,21 @@ static int evp_md_init_internal(EVP_MD_CTX *ctx, const EVP_MD *type, * previous handle, re-querying for an ENGINE, and having a * reinitialisation, when it may all be unnecessary. */ - if (ctx->engine != NULL - && ctx->digest != NULL - && type->type == ctx->digest->type) + if (ctx->engine && ctx->digest && + (type == NULL || (type->type == ctx->digest->type))) goto skip_to_init; - /* - * Ensure an ENGINE left lying around from last time is cleared (the - * previous check attempted to avoid this if the same ENGINE and - * EVP_MD could be used). - */ - ENGINE_finish(ctx->engine); - ctx->engine = NULL; + if (type != NULL) { + /* + * Ensure an ENGINE left lying around from last time is cleared (the + * previous check attempted to avoid this if the same ENGINE and + * EVP_MD could be used). + */ + ENGINE_finish(ctx->engine); + ctx->engine = NULL; + } - if (impl == NULL) + if (type != NULL && impl == NULL) tmpimpl = ENGINE_get_digest_engine(type->type); #endif @@ -223,20 +219,13 @@ static int evp_md_init_internal(EVP_MD_CTX *ctx, const EVP_MD *type, * If there are engines involved or EVP_MD_CTX_FLAG_NO_INIT is set then we * should use legacy handling for now. */ - if (impl != NULL -#if !defined(OPENSSL_NO_ENGINE) - || ctx->engine != NULL -# if !defined(FIPS_MODULE) + if (ctx->engine != NULL + || impl != NULL +#if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE) || tmpimpl != NULL -# endif #endif || (ctx->flags & EVP_MD_CTX_FLAG_NO_INIT) != 0 - || (type != NULL && type->origin == EVP_ORIG_METH) - || (type == NULL && ctx->digest != NULL - && ctx->digest->origin == EVP_ORIG_METH)) { - /* If we were using provided hash before, cleanup algctx */ - if (!evp_md_ctx_free_algctx(ctx)) - return 0; + || type->origin == EVP_ORIG_METH) { if (ctx->digest == ctx->fetched_digest) ctx->digest = NULL; EVP_MD_free(ctx->fetched_digest); @@ -247,15 +236,6 @@ static int evp_md_init_internal(EVP_MD_CTX *ctx, const EVP_MD *type, cleanup_old_md_data(ctx, 1); /* Start of non-legacy code below */ - if (ctx->digest == type) { - if (!ossl_assert(type->prov != NULL)) { - ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); - return 0; - } - } else { - if (!evp_md_ctx_free_algctx(ctx)) - return 0; - } if (type->prov == NULL) { #ifdef FIPS_MODULE @@ -278,6 +258,11 @@ static int evp_md_init_internal(EVP_MD_CTX *ctx, const EVP_MD *type, #endif } + if (ctx->algctx != NULL && ctx->digest != NULL && ctx->digest != type) { + if (ctx->digest->freectx != NULL) + ctx->digest->freectx(ctx->algctx); + ctx->algctx = NULL; + } if (type->prov != NULL && ctx->fetched_digest != type) { if (!EVP_MD_up_ref((EVP_MD *)type)) { ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); @@ -343,8 +328,10 @@ static int evp_md_init_internal(EVP_MD_CTX *ctx, const EVP_MD *type, if (!(ctx->flags & EVP_MD_CTX_FLAG_NO_INIT) && type->ctx_size) { ctx->update = type->update; ctx->md_data = OPENSSL_zalloc(type->ctx_size); - if (ctx->md_data == NULL) + if (ctx->md_data == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } } } #if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE) @@ -388,11 +375,6 @@ int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *data, size_t count) if (count == 0) return 1; - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR); - return 0; - } - if (ctx->pctx != NULL && EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx->pctx) && ctx->pctx->op.sig.algctx != NULL) { @@ -454,32 +436,16 @@ int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *isize) if (ctx->digest->prov == NULL) goto legacy; - if (ctx->digest->gettable_ctx_params != NULL) { - OSSL_PARAM params[] = { OSSL_PARAM_END, OSSL_PARAM_END }; - - params[0] = OSSL_PARAM_construct_size_t(OSSL_DIGEST_PARAM_SIZE, - &mdsize); - if (!EVP_MD_CTX_get_params(ctx, params)) - return 0; - } - if (ctx->digest->dfinal == NULL) { ERR_raise(ERR_LIB_EVP, EVP_R_FINAL_ERROR); return 0; } - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_FINAL_ERROR); - return 0; - } - ret = ctx->digest->dfinal(ctx->algctx, md, &size, mdsize); - ctx->flags |= EVP_MD_CTX_FLAG_FINALISED; - if (isize != NULL) { if (size <= UINT_MAX) { - *isize = (unsigned int)size; + *isize = (int)size; } else { ERR_raise(ERR_LIB_EVP, EVP_R_FINAL_ERROR); ret = 0; @@ -502,7 +468,6 @@ int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *isize) return ret; } -/* This is a one shot operation */ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t size) { int ret = 0; @@ -522,24 +487,12 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t size) return 0; } - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_FINAL_ERROR); - return 0; - } - - /* - * For backward compatibility we pass the XOFLEN via a param here so that - * older providers can use the supplied value. Ideally we should have just - * used the size passed into ctx->digest->dfinal(). - */ params[i++] = OSSL_PARAM_construct_size_t(OSSL_DIGEST_PARAM_XOFLEN, &size); params[i++] = OSSL_PARAM_construct_end(); - if (EVP_MD_CTX_set_params(ctx, params) >= 0) + if (EVP_MD_CTX_set_params(ctx, params) > 0) ret = ctx->digest->dfinal(ctx->algctx, md, &size, size); - ctx->flags |= EVP_MD_CTX_FLAG_FINALISED; - return ret; legacy: @@ -559,38 +512,6 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t size) return ret; } -/* EVP_DigestSqueeze() can be called multiple times */ -int EVP_DigestSqueeze(EVP_MD_CTX *ctx, unsigned char *md, size_t size) -{ - if (ctx->digest == NULL) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_NULL_ALGORITHM); - return 0; - } - - if (ctx->digest->prov == NULL) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_OPERATION); - return 0; - } - - if (ctx->digest->dsqueeze == NULL) { - ERR_raise(ERR_LIB_EVP, EVP_R_METHOD_NOT_SUPPORTED); - return 0; - } - - return ctx->digest->dsqueeze(ctx->algctx, md, &size, size); -} - -EVP_MD_CTX *EVP_MD_CTX_dup(const EVP_MD_CTX *in) -{ - EVP_MD_CTX *out = EVP_MD_CTX_new(); - - if (out != NULL && !EVP_MD_CTX_copy_ex(out, in)) { - EVP_MD_CTX_free(out); - out = NULL; - } - return out; -} - int EVP_MD_CTX_copy(EVP_MD_CTX *out, const EVP_MD_CTX *in) { EVP_MD_CTX_reset(out); @@ -694,8 +615,10 @@ int EVP_MD_CTX_copy_ex(EVP_MD_CTX *out, const EVP_MD_CTX *in) out->md_data = tmp_buf; else { out->md_data = OPENSSL_malloc(out->digest->ctx_size); - if (out->md_data == NULL) + if (out->md_data == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } } memcpy(out->md_data, in->md_data, out->digest->ctx_size); } @@ -837,7 +760,7 @@ int EVP_MD_CTX_get_params(EVP_MD_CTX *ctx, OSSL_PARAM params[]) return pctx->op.sig.signature->get_ctx_md_params(pctx->op.sig.algctx, params); - if (ctx->digest != NULL && ctx->digest->get_ctx_params != NULL) + if (ctx->digest != NULL && ctx->digest->get_params != NULL) return ctx->digest->get_ctx_params(ctx->algctx, params); return 0; @@ -937,9 +860,13 @@ EVP_MD *evp_md_new(void) { EVP_MD *md = OPENSSL_zalloc(sizeof(*md)); - if (md != NULL && !CRYPTO_NEW_REF(&md->refcnt, 1)) { - OPENSSL_free(md); - return NULL; + if (md != NULL) { + md->lock = CRYPTO_THREAD_lock_new(); + if (md->lock == NULL) { + OPENSSL_free(md); + return NULL; + } + md->refcnt = 1; } return md; } @@ -1012,7 +939,7 @@ static void *evp_md_from_algorithm(int name_id, /* EVP_MD_fetch() will set the legacy NID if available */ if ((md = evp_md_new()) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } @@ -1059,12 +986,6 @@ static void *evp_md_from_algorithm(int name_id, fncnt++; } break; - case OSSL_FUNC_DIGEST_SQUEEZE: - if (md->dsqueeze == NULL) { - md->dsqueeze = OSSL_FUNC_digest_squeeze(fns); - fncnt++; - } - break; case OSSL_FUNC_DIGEST_DIGEST: if (md->digest == NULL) md->digest = OSSL_FUNC_digest_digest(fns); @@ -1108,7 +1029,7 @@ static void *evp_md_from_algorithm(int name_id, break; } } - if ((fncnt != 0 && fncnt != 5 && fncnt != 6) + if ((fncnt != 0 && fncnt != 5) || (fncnt == 0 && md->digest == NULL)) { /* * In order to be a consistent set of functions we either need the @@ -1158,7 +1079,7 @@ int EVP_MD_up_ref(EVP_MD *md) int ref = 0; if (md->origin == EVP_ORIG_DYNAMIC) - CRYPTO_UP_REF(&md->refcnt, &ref); + CRYPTO_UP_REF(&md->refcnt, &ref, md->lock); return 1; } @@ -1169,7 +1090,7 @@ void EVP_MD_free(EVP_MD *md) if (md == NULL || md->origin != EVP_ORIG_DYNAMIC) return; - CRYPTO_DOWN_REF(&md->refcnt, &i); + CRYPTO_DOWN_REF(&md->refcnt, &i, md->lock); if (i > 0) return; evp_md_free_int(md); diff --git a/openssl/src/crypto/evp/e_aes.c b/openssl/src/crypto/evp/e_aes.c index 10abb7d52..52b9e87c1 100644 --- a/openssl/src/crypto/evp/e_aes.c +++ b/openssl/src/crypto/evp/e_aes.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,7 +8,7 @@ */ /* - * This file uses the low-level AES functions (which are deprecated for + * This file uses the low level AES functions (which are deprecated for * non-internal use) in order to implement the EVP AES ciphers. */ #include "internal/deprecated.h" @@ -146,21 +146,20 @@ static int aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, { int ret, mode; EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx); - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } mode = EVP_CIPHER_CTX_get_mode(ctx); if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) { - ret = aesni_set_decrypt_key(key, keylen, &dat->ks.ks); + ret = aesni_set_decrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) aesni_decrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) aesni_cbc_encrypt : NULL; } else { - ret = aesni_set_encrypt_key(key, keylen, &dat->ks.ks); + ret = aesni_set_encrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) aesni_encrypt; if (mode == EVP_CIPH_CBC_MODE) dat->stream.cbc = (cbc128_f) aesni_cbc_encrypt; @@ -224,19 +223,12 @@ static int aesni_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { - EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX, ctx); - - if (iv == NULL && key == NULL) + EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx); + if (!iv && !key) return 1; - if (key) { - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } - aesni_set_encrypt_key(key, keylen, &gctx->ks.ks); + aesni_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) aesni_encrypt); gctx->ctr = (ctr128_f) aesni_ctr32_encrypt_blocks; /* @@ -270,19 +262,14 @@ static int aesni_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, { EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx); - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; if (key) { /* The key is two half length keys in reality */ - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx); - const int bytes = keylen / 2; + const int bytes = EVP_CIPHER_CTX_get_key_length(ctx) / 2; const int bits = bytes * 8; - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } /* * Verify that the two keys are different. * @@ -328,18 +315,11 @@ static int aesni_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,ctx); - - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; - - if (key != NULL) { - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } - aesni_set_encrypt_key(key, keylen, &cctx->ks.ks); + if (key) { + aesni_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &cctx->ks.ks); CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, &cctx->ks, (block128_f) aesni_encrypt); cctx->str = enc ? (ccm128_f) aesni_ccm64_encrypt_blocks : @@ -362,25 +342,19 @@ static int aesni_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,ctx); - - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; - - if (key != NULL) { - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } + if (key) { do { /* * We set both the encrypt and decrypt key here because decrypt * needs both. We could possibly optimise to remove setting the * decrypt for an encryption operation. */ - aesni_set_encrypt_key(key, keylen, &octx->ksenc.ks); - aesni_set_decrypt_key(key, keylen, &octx->ksdec.ks); + aesni_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksenc.ks); + aesni_set_decrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksdec.ks); if (!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc.ks, &octx->ksdec.ks, (block128_f) aesni_encrypt, @@ -478,10 +452,6 @@ static int aes_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, mode = EVP_CIPHER_CTX_get_mode(ctx); bits = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - if (bits <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) { ret = 0; @@ -577,16 +547,10 @@ static int aes_t4_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx); - - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; if (key) { - const int bits = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (bits <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } + int bits = EVP_CIPHER_CTX_get_key_length(ctx) * 8; aes_t4_set_encrypt_key(key, bits, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) aes_t4_encrypt); @@ -639,14 +603,9 @@ static int aes_t4_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, if (key) { /* The key is two half length keys in reality */ - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx); - const int bytes = keylen / 2; + const int bytes = EVP_CIPHER_CTX_get_key_length(ctx) / 2; const int bits = bytes * 8; - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } /* * Verify that the two keys are different. * @@ -711,17 +670,10 @@ static int aes_t4_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,ctx); - - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; - - if (key != NULL) { - const int bits = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (bits <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } + if (key) { + int bits = EVP_CIPHER_CTX_get_key_length(ctx) * 8; aes_t4_set_encrypt_key(key, bits, &cctx->ks.ks); CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, &cctx->ks, (block128_f) aes_t4_encrypt); @@ -744,25 +696,19 @@ static int aes_t4_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,ctx); - - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; - - if (key != NULL) { - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } + if (key) { do { /* * We set both the encrypt and decrypt key here because decrypt * needs both. We could possibly optimise to remove setting the * decrypt for an encryption operation. */ - aes_t4_set_encrypt_key(key, keylen, &octx->ksenc.ks); - aes_t4_set_decrypt_key(key, keylen, &octx->ksdec.ks); + aes_t4_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksenc.ks); + aes_t4_set_decrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksdec.ks); if (!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc.ks, &octx->ksdec.ks, (block128_f) aes_t4_encrypt, @@ -885,6 +831,8 @@ typedef struct { /* KMO-AES parameter block - end */ } kmo; unsigned int fc; + + int res; } S390X_AES_OFB_CTX; typedef struct { @@ -901,6 +849,8 @@ typedef struct { /* KMF-AES parameter block - end */ } kmf; unsigned int fc; + + int res; } S390X_AES_CFB_CTX; typedef struct { @@ -1023,10 +973,6 @@ static int s390x_aes_ecb_init_key(EVP_CIPHER_CTX *ctx, S390X_AES_ECB_CTX *cctx = EVP_C_DATA(S390X_AES_ECB_CTX, ctx); const int keylen = EVP_CIPHER_CTX_get_key_length(ctx); - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } cctx->fc = S390X_AES_FC(keylen); if (!enc) cctx->fc |= S390X_DECRYPT; @@ -1053,17 +999,10 @@ static int s390x_aes_ofb_init_key(EVP_CIPHER_CTX *ctx, const int keylen = EVP_CIPHER_CTX_get_key_length(ctx); const int ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } - if (ivlen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_IV_LENGTH); - return 0; - } memcpy(cctx->kmo.param.cv, iv, ivlen); memcpy(cctx->kmo.param.k, key, keylen); cctx->fc = S390X_AES_FC(keylen); + cctx->res = 0; return 1; } @@ -1073,7 +1012,7 @@ static int s390x_aes_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, S390X_AES_OFB_CTX *cctx = EVP_C_DATA(S390X_AES_OFB_CTX, ctx); const int ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); unsigned char *iv = EVP_CIPHER_CTX_iv_noconst(ctx); - int n = ctx->num; + int n = cctx->res; int rem; memcpy(cctx->kmo.param.cv, iv, ivlen); @@ -1106,7 +1045,7 @@ static int s390x_aes_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } memcpy(iv, cctx->kmo.param.cv, ivlen); - ctx->num = n; + cctx->res = n; return 1; } @@ -1119,19 +1058,12 @@ static int s390x_aes_cfb_init_key(EVP_CIPHER_CTX *ctx, const int keylen = EVP_CIPHER_CTX_get_key_length(ctx); const int ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } - if (ivlen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_IV_LENGTH); - return 0; - } cctx->fc = S390X_AES_FC(keylen); cctx->fc |= 16 << 24; /* 16 bytes cipher feedback */ if (!enc) cctx->fc |= S390X_DECRYPT; + cctx->res = 0; memcpy(cctx->kmf.param.cv, iv, ivlen); memcpy(cctx->kmf.param.k, key, keylen); return 1; @@ -1145,18 +1077,10 @@ static int s390x_aes_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const int enc = EVP_CIPHER_CTX_is_encrypting(ctx); const int ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); unsigned char *iv = EVP_CIPHER_CTX_iv_noconst(ctx); - int n = ctx->num; + int n = cctx->res; int rem; unsigned char tmp; - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } - if (ivlen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_IV_LENGTH); - return 0; - } memcpy(cctx->kmf.param.cv, iv, ivlen); while (n && len) { tmp = *in; @@ -1191,7 +1115,7 @@ static int s390x_aes_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } memcpy(iv, cctx->kmf.param.cv, ivlen); - ctx->num = n; + cctx->res = n; return 1; } @@ -1204,14 +1128,6 @@ static int s390x_aes_cfb8_init_key(EVP_CIPHER_CTX *ctx, const int keylen = EVP_CIPHER_CTX_get_key_length(ctx); const int ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } - if (ivlen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_IV_LENGTH); - return 0; - } cctx->fc = S390X_AES_FC(keylen); cctx->fc |= 1 << 24; /* 1 byte cipher feedback */ if (!enc) @@ -1469,8 +1385,10 @@ static int s390x_aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) if (gctx->iv != c->iv) OPENSSL_free(gctx->iv); - if ((gctx->iv = OPENSSL_malloc(len)) == NULL) + if ((gctx->iv = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } } /* Add padding. */ memset(gctx->iv + arg, 0, len - arg - 8); @@ -1586,8 +1504,10 @@ static int s390x_aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) } else { len = S390X_gcm_ivpadlen(gctx->ivlen); - if ((gctx_out->iv = OPENSSL_malloc(len)) == NULL) + if ((gctx_out->iv = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(gctx_out->iv, gctx->iv, len); } @@ -1613,11 +1533,6 @@ static int s390x_aes_gcm_init_key(EVP_CIPHER_CTX *ctx, if (key != NULL) { keylen = EVP_CIPHER_CTX_get_key_length(ctx); - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } - memcpy(&gctx->kma.param.k, key, keylen); gctx->fc = S390X_AES_FC(keylen); @@ -1665,7 +1580,7 @@ static int s390x_aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, * communication to fail after 2^64 - 1 keys. We do this on the encrypting * side only. */ - if (enc && ++gctx->tls_enc_records == 0) { + if (ctx->encrypt && ++gctx->tls_enc_records == 0) { ERR_raise(ERR_LIB_EVP, EVP_R_TOO_MANY_RECORDS); goto err; } @@ -2024,11 +1939,6 @@ static int s390x_aes_ccm_init_key(EVP_CIPHER_CTX *ctx, if (key != NULL) { keylen = EVP_CIPHER_CTX_get_key_length(ctx); - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } - cctx->aes.ccm.fc = S390X_AES_FC(keylen); memcpy(cctx->aes.ccm.kmac_param.k, key, keylen); @@ -2239,7 +2149,7 @@ static int s390x_aes_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) if (!enc || !cctx->aes.ccm.tag_set) return 0; - if (arg < cctx->aes.ccm.m) + if(arg < cctx->aes.ccm.m) return 0; memcpy(ptr, cctx->aes.ccm.kmac_param.icv.b, cctx->aes.ccm.m); @@ -2405,19 +2315,15 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, { int ret, mode; EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx); - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } mode = EVP_CIPHER_CTX_get_mode(ctx); if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) { #ifdef HWAES_CAPABLE if (HWAES_CAPABLE) { - ret = HWAES_set_decrypt_key(key, keylen, &dat->ks.ks); + ret = HWAES_set_decrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) HWAES_decrypt; dat->stream.cbc = NULL; # ifdef HWAES_cbc_encrypt @@ -2428,21 +2334,27 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, #endif #ifdef BSAES_CAPABLE if (BSAES_CAPABLE && mode == EVP_CIPH_CBC_MODE) { - ret = AES_set_decrypt_key(key, keylen, &dat->ks.ks); + ret = AES_set_decrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) AES_decrypt; dat->stream.cbc = (cbc128_f) ossl_bsaes_cbc_encrypt; } else #endif #ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - ret = vpaes_set_decrypt_key(key, keylen, &dat->ks.ks); + ret = vpaes_set_decrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) vpaes_decrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) vpaes_cbc_encrypt : NULL; } else #endif { - ret = AES_set_decrypt_key(key, keylen, &dat->ks.ks); + ret = AES_set_decrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) AES_decrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) AES_cbc_encrypt : NULL; @@ -2450,7 +2362,9 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, } else #ifdef HWAES_CAPABLE if (HWAES_CAPABLE) { - ret = HWAES_set_encrypt_key(key, keylen, &dat->ks.ks); + ret = HWAES_set_encrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) HWAES_encrypt; dat->stream.cbc = NULL; # ifdef HWAES_cbc_encrypt @@ -2468,21 +2382,25 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, #endif #ifdef BSAES_CAPABLE if (BSAES_CAPABLE && mode == EVP_CIPH_CTR_MODE) { - ret = AES_set_encrypt_key(key, keylen, &dat->ks.ks); + ret = AES_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) AES_encrypt; dat->stream.ctr = (ctr128_f) ossl_bsaes_ctr32_encrypt_blocks; } else #endif #ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - ret = vpaes_set_encrypt_key(key, keylen, &dat->ks.ks); + ret = vpaes_set_encrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) vpaes_encrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) vpaes_cbc_encrypt : NULL; } else #endif { - ret = AES_set_encrypt_key(key, keylen, &dat->ks.ks); + ret = AES_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &dat->ks.ks); dat->block = (block128_f) AES_encrypt; dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? (cbc128_f) AES_cbc_encrypt : NULL; @@ -2672,8 +2590,10 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) if ((arg > EVP_MAX_IV_LENGTH) && (arg > gctx->ivlen)) { if (gctx->iv != c->iv) OPENSSL_free(gctx->iv); - if ((gctx->iv = OPENSSL_malloc(arg)) == NULL) + if ((gctx->iv = OPENSSL_malloc(arg)) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } } gctx->ivlen = arg; return 1; @@ -2772,8 +2692,10 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) if (gctx->iv == c->iv) gctx_out->iv = out->iv; else { - if ((gctx_out->iv = OPENSSL_malloc(gctx->ivlen)) == NULL) + if ((gctx_out->iv = OPENSSL_malloc(gctx->ivlen)) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(gctx_out->iv, gctx->iv, gctx->ivlen); } return 1; @@ -2789,21 +2711,13 @@ static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx); - - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; - - if (key != NULL) { - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } + if (key) { do { #ifdef HWAES_CAPABLE if (HWAES_CAPABLE) { - HWAES_set_encrypt_key(key, keylen, &gctx->ks.ks); + HWAES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) HWAES_encrypt); # ifdef HWAES_ctr32_encrypt_blocks @@ -2816,7 +2730,7 @@ static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, #endif #ifdef BSAES_CAPABLE if (BSAES_CAPABLE) { - AES_set_encrypt_key(key, keylen, &gctx->ks.ks); + AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) AES_encrypt); gctx->ctr = (ctr128_f) ossl_bsaes_ctr32_encrypt_blocks; @@ -2825,7 +2739,7 @@ static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, #endif #ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - vpaes_set_encrypt_key(key, keylen, &gctx->ks.ks); + vpaes_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) vpaes_encrypt); gctx->ctr = NULL; @@ -2834,7 +2748,7 @@ static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, #endif (void)0; /* terminate potentially open 'else' */ - AES_set_encrypt_key(key, keylen, &gctx->ks.ks); + AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) AES_encrypt); #ifdef AES_CTR_ASM @@ -2889,7 +2803,7 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, * communication to fail after 2^64 - 1 keys. We do this on the encrypting * side only. */ - if (EVP_CIPHER_CTX_is_encrypting(ctx) && ++gctx->tls_enc_records == 0) { + if (ctx->encrypt && ++gctx->tls_enc_records == 0) { ERR_raise(ERR_LIB_EVP, EVP_R_TOO_MANY_RECORDS); goto err; } @@ -2898,20 +2812,18 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, * Set IV from start of buffer or generate IV and write to start of * buffer. */ - if (EVP_CIPHER_CTX_ctrl(ctx, - EVP_CIPHER_CTX_is_encrypting(ctx) ? - EVP_CTRL_GCM_IV_GEN : EVP_CTRL_GCM_SET_IV_INV, + if (EVP_CIPHER_CTX_ctrl(ctx, ctx->encrypt ? EVP_CTRL_GCM_IV_GEN + : EVP_CTRL_GCM_SET_IV_INV, EVP_GCM_TLS_EXPLICIT_IV_LEN, out) <= 0) goto err; /* Use saved AAD */ - if (CRYPTO_gcm128_aad(&gctx->gcm, EVP_CIPHER_CTX_buf_noconst(ctx), - gctx->tls_aad_len)) + if (CRYPTO_gcm128_aad(&gctx->gcm, ctx->buf, gctx->tls_aad_len)) goto err; /* Fix buffer and length to point to payload */ in += EVP_GCM_TLS_EXPLICIT_IV_LEN; out += EVP_GCM_TLS_EXPLICIT_IV_LEN; len -= EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN; - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { + if (ctx->encrypt) { /* Encrypt payload */ if (gctx->ctr) { size_t bulk = 0; @@ -2990,11 +2902,9 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, goto err; } /* Retrieve tag */ - CRYPTO_gcm128_tag(&gctx->gcm, EVP_CIPHER_CTX_buf_noconst(ctx), - EVP_GCM_TLS_TAG_LEN); + CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, EVP_GCM_TLS_TAG_LEN); /* If tag mismatch wipe buffer */ - if (CRYPTO_memcmp(EVP_CIPHER_CTX_buf_noconst(ctx), in + len, - EVP_GCM_TLS_TAG_LEN)) { + if (CRYPTO_memcmp(ctx->buf, in + len, EVP_GCM_TLS_TAG_LEN)) { OPENSSL_cleanse(out, len); goto err; } @@ -3051,7 +2961,7 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, * where setting the IV externally is the only option available. */ if (!gctx->iv_set) { - if (!EVP_CIPHER_CTX_is_encrypting(ctx) || !aes_gcm_iv_generate(gctx, 0)) + if (!ctx->encrypt || !aes_gcm_iv_generate(gctx, 0)) return -1; CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen); gctx->iv_set = 1; @@ -3066,7 +2976,7 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, if (out == NULL) { if (CRYPTO_gcm128_aad(&gctx->gcm, in, len)) return -1; - } else if (EVP_CIPHER_CTX_is_encrypting(ctx)) { + } else if (ctx->encrypt) { if (gctx->ctr) { size_t bulk = 0; #if defined(AES_GCM_ASM) @@ -3157,17 +3067,15 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } return len; } else { - if (!EVP_CIPHER_CTX_is_encrypting(ctx)) { + if (!ctx->encrypt) { if (gctx->taglen < 0) return -1; - if (CRYPTO_gcm128_finish(&gctx->gcm, - EVP_CIPHER_CTX_buf_noconst(ctx), - gctx->taglen) != 0) + if (CRYPTO_gcm128_finish(&gctx->gcm, ctx->buf, gctx->taglen) != 0) return -1; gctx->iv_set = 0; return 0; } - CRYPTO_gcm128_tag(&gctx->gcm, EVP_CIPHER_CTX_buf_noconst(ctx), 16); + CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, 16); gctx->taglen = 16; /* Don't reuse the IV */ gctx->iv_set = 0; @@ -3183,9 +3091,9 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, BLOCK_CIPHER_custom(NID_aes, 128, 1, 12, gcm, GCM, EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS) -BLOCK_CIPHER_custom(NID_aes, 192, 1, 12, gcm, GCM, + BLOCK_CIPHER_custom(NID_aes, 192, 1, 12, gcm, GCM, EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS) -BLOCK_CIPHER_custom(NID_aes, 256, 1, 12, gcm, GCM, + BLOCK_CIPHER_custom(NID_aes, 256, 1, 12, gcm, GCM, EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS) static int aes_xts_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) @@ -3220,20 +3128,15 @@ static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, { EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx); - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; - if (key != NULL) { + if (key) { do { /* The key is two half length keys in reality */ - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx); - const int bytes = keylen / 2; + const int bytes = EVP_CIPHER_CTX_get_key_length(ctx) / 2; const int bits = bytes * 8; - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } /* * Verify that the two keys are different. * @@ -3372,7 +3275,7 @@ static int aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, | EVP_CIPH_CUSTOM_COPY) BLOCK_CIPHER_custom(NID_aes, 128, 1, 16, xts, XTS, XTS_FLAGS) -BLOCK_CIPHER_custom(NID_aes, 256, 1, 16, xts, XTS, XTS_FLAGS) + BLOCK_CIPHER_custom(NID_aes, 256, 1, 16, xts, XTS, XTS_FLAGS) static int aes_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) { @@ -3428,7 +3331,7 @@ static int aes_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) case EVP_CTRL_AEAD_SET_IVLEN: arg = 15 - arg; - /* fall through */ + /* fall thru */ case EVP_CTRL_CCM_SET_L: if (arg < 2 || arg > 8) return 0; @@ -3479,21 +3382,15 @@ static int aes_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,ctx); - - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; - - if (key != NULL) { - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } + if (key) do { #ifdef HWAES_CAPABLE if (HWAES_CAPABLE) { - HWAES_set_encrypt_key(key, keylen, &cctx->ks.ks); + HWAES_set_encrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &cctx->ks.ks); CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, &cctx->ks, (block128_f) HWAES_encrypt); @@ -3504,7 +3401,9 @@ static int aes_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, #endif #ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - vpaes_set_encrypt_key(key, keylen, &cctx->ks.ks); + vpaes_set_encrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &cctx->ks.ks); CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, &cctx->ks, (block128_f) vpaes_encrypt); cctx->str = NULL; @@ -3512,14 +3411,14 @@ static int aes_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, break; } #endif - AES_set_encrypt_key(key, keylen, &cctx->ks.ks); + AES_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &cctx->ks.ks); CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, &cctx->ks, (block128_f) AES_encrypt); cctx->str = NULL; cctx->key_set = 1; } while (0); - } - if (iv != NULL) { + if (iv) { memcpy(ctx->iv, iv, 15 - cctx->L); cctx->iv_set = 1; } @@ -3674,16 +3573,12 @@ static int aes_wrap_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, if (iv == NULL && key == NULL) return 1; if (key != NULL) { - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } if (EVP_CIPHER_CTX_is_encrypting(ctx)) - AES_set_encrypt_key(key, keylen, &wctx->ks.ks); + AES_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &wctx->ks.ks); else - AES_set_decrypt_key(key, keylen, &wctx->ks.ks); + AES_set_decrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &wctx->ks.ks); if (iv == NULL) wctx->iv = NULL; } @@ -3911,17 +3806,9 @@ static int aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,ctx); - - if (iv == NULL && key == NULL) + if (!iv && !key) return 1; - - if (key != NULL) { - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } + if (key) { do { /* * We set both the encrypt and decrypt key here because decrypt @@ -3930,8 +3817,10 @@ static int aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, */ # ifdef HWAES_CAPABLE if (HWAES_CAPABLE) { - HWAES_set_encrypt_key(key, keylen, &octx->ksenc.ks); - HWAES_set_decrypt_key(key, keylen, &octx->ksdec.ks); + HWAES_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksenc.ks); + HWAES_set_decrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksdec.ks); if (!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc.ks, &octx->ksdec.ks, (block128_f) HWAES_encrypt, @@ -3944,8 +3833,12 @@ static int aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, # endif # ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { - vpaes_set_encrypt_key(key, keylen, &octx->ksenc.ks); - vpaes_set_decrypt_key(key, keylen, &octx->ksdec.ks); + vpaes_set_encrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksenc.ks); + vpaes_set_decrypt_key(key, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksdec.ks); if (!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc.ks, &octx->ksdec.ks, (block128_f) vpaes_encrypt, @@ -3955,8 +3848,10 @@ static int aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, break; } # endif - AES_set_encrypt_key(key, keylen, &octx->ksenc.ks); - AES_set_decrypt_key(key, keylen, &octx->ksdec.ks); + AES_set_encrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksenc.ks); + AES_set_decrypt_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &octx->ksdec.ks); if (!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc.ks, &octx->ksdec.ks, (block128_f) AES_encrypt, @@ -4007,7 +3902,7 @@ static int aes_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, if (in != NULL) { /* - * Need to ensure we are only passing full blocks to low-level OCB + * Need to ensure we are only passing full blocks to low level OCB * routines. We do it here rather than in EVP_EncryptUpdate/ * EVP_DecryptUpdate because we need to pass full blocks of AAD too * and those routines don't support that diff --git a/openssl/src/crypto/evp/e_aes_cbc_hmac_sha1.c b/openssl/src/crypto/evp/e_aes_cbc_hmac_sha1.c index 8843c8ae1..4941f98e6 100644 --- a/openssl/src/crypto/evp/e_aes_cbc_hmac_sha1.c +++ b/openssl/src/crypto/evp/e_aes_cbc_hmac_sha1.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -72,16 +72,15 @@ static int aesni_cbc_hmac_sha1_init_key(EVP_CIPHER_CTX *ctx, { EVP_AES_HMAC_SHA1 *key = data(ctx); int ret; - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } if (enc) - ret = aesni_set_encrypt_key(inkey, keylen, &key->ks); + ret = aesni_set_encrypt_key(inkey, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &key->ks); else - ret = aesni_set_decrypt_key(inkey, keylen, &key->ks); + ret = aesni_set_decrypt_key(inkey, + EVP_CIPHER_CTX_get_key_length(ctx) * 8, + &key->ks); SHA1_Init(&key->head); /* handy when benchmarking */ key->tail = key->head; @@ -497,12 +496,6 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, # if defined(STITCHED_DECRYPT_CALL) unsigned char tail_iv[AES_BLOCK_SIZE]; int stitch = 0; - const int keylen = EVP_CIPHER_CTX_get_key_length(ctx); - - if (keylen <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY_LENGTH); - return 0; - } # endif if ((key->aux.tls_aad[plen - 4] << 8 | key->aux.tls_aad[plen - 3]) @@ -520,7 +513,7 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return 0; # if defined(STITCHED_DECRYPT_CALL) - if (len >= 1024 && keylen == 32) { + if (len >= 1024 && ctx->key_len == 32) { /* decrypt last block */ memcpy(tail_iv, in + len - 2 * AES_BLOCK_SIZE, AES_BLOCK_SIZE); @@ -741,7 +734,7 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return ret; } else { # if defined(STITCHED_DECRYPT_CALL) - if (len >= 1024 && keylen == 32) { + if (len >= 1024 && ctx->key_len == 32) { if (sha_off %= SHA_CBLOCK) blocks = (len - 3 * SHA_CBLOCK) / SHA_CBLOCK; else diff --git a/openssl/src/crypto/evp/e_aria.c b/openssl/src/crypto/evp/e_aria.c deleted file mode 100644 index 5a894fbb9..000000000 --- a/openssl/src/crypto/evp/e_aria.c +++ /dev/null @@ -1,786 +0,0 @@ -/* - * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/deprecated.h" - -#include "internal/cryptlib.h" -#ifndef OPENSSL_NO_ARIA -# include -# include -# include -# include "crypto/aria.h" -# include "crypto/evp.h" -# include "crypto/modes.h" -# include "evp_local.h" - -/* ARIA subkey Structure */ -typedef struct { - ARIA_KEY ks; -} EVP_ARIA_KEY; - -/* ARIA GCM context */ -typedef struct { - union { - OSSL_UNION_ALIGN; - ARIA_KEY ks; - } ks; /* ARIA subkey to use */ - int key_set; /* Set if key initialised */ - int iv_set; /* Set if an iv is set */ - GCM128_CONTEXT gcm; - unsigned char *iv; /* Temporary IV store */ - int ivlen; /* IV length */ - int taglen; - int iv_gen; /* It is OK to generate IVs */ - int tls_aad_len; /* TLS AAD length */ -} EVP_ARIA_GCM_CTX; - -/* ARIA CCM context */ -typedef struct { - union { - OSSL_UNION_ALIGN; - ARIA_KEY ks; - } ks; /* ARIA key schedule to use */ - int key_set; /* Set if key initialised */ - int iv_set; /* Set if an iv is set */ - int tag_set; /* Set if tag is valid */ - int len_set; /* Set if message length set */ - int L, M; /* L and M parameters from RFC3610 */ - int tls_aad_len; /* TLS AAD length */ - CCM128_CONTEXT ccm; - ccm128_f str; -} EVP_ARIA_CCM_CTX; - -/* The subkey for ARIA is generated. */ -static int aria_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - int ret; - int mode = EVP_CIPHER_CTX_get_mode(ctx); - - if (enc || (mode != EVP_CIPH_ECB_MODE && mode != EVP_CIPH_CBC_MODE)) - ret = ossl_aria_set_encrypt_key(key, - EVP_CIPHER_CTX_get_key_length(ctx) * 8, - EVP_CIPHER_CTX_get_cipher_data(ctx)); - else - ret = ossl_aria_set_decrypt_key(key, - EVP_CIPHER_CTX_get_key_length(ctx) * 8, - EVP_CIPHER_CTX_get_cipher_data(ctx)); - if (ret < 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_ARIA_KEY_SETUP_FAILED); - return 0; - } - return 1; -} - -static void aria_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const ARIA_KEY *key, - unsigned char *ivec, const int enc) -{ - - if (enc) - CRYPTO_cbc128_encrypt(in, out, len, key, ivec, - (block128_f) ossl_aria_encrypt); - else - CRYPTO_cbc128_decrypt(in, out, len, key, ivec, - (block128_f) ossl_aria_encrypt); -} - -static void aria_cfb128_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const ARIA_KEY *key, - unsigned char *ivec, int *num, const int enc) -{ - - CRYPTO_cfb128_encrypt(in, out, length, key, ivec, num, enc, - (block128_f) ossl_aria_encrypt); -} - -static void aria_cfb1_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const ARIA_KEY *key, - unsigned char *ivec, int *num, const int enc) -{ - CRYPTO_cfb128_1_encrypt(in, out, length, key, ivec, num, enc, - (block128_f) ossl_aria_encrypt); -} - -static void aria_cfb8_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const ARIA_KEY *key, - unsigned char *ivec, int *num, const int enc) -{ - CRYPTO_cfb128_8_encrypt(in, out, length, key, ivec, num, enc, - (block128_f) ossl_aria_encrypt); -} - -static void aria_ecb_encrypt(const unsigned char *in, unsigned char *out, - const ARIA_KEY *key, const int enc) -{ - ossl_aria_encrypt(in, out, key); -} - -static void aria_ofb128_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const ARIA_KEY *key, - unsigned char *ivec, int *num) -{ - CRYPTO_ofb128_encrypt(in, out, length, key, ivec, num, - (block128_f) ossl_aria_encrypt); -} - -IMPLEMENT_BLOCK_CIPHER(aria_128, ks, aria, EVP_ARIA_KEY, - NID_aria_128, 16, 16, 16, 128, - 0, aria_init_key, NULL, - EVP_CIPHER_set_asn1_iv, - EVP_CIPHER_get_asn1_iv, - NULL) -IMPLEMENT_BLOCK_CIPHER(aria_192, ks, aria, EVP_ARIA_KEY, - NID_aria_192, 16, 24, 16, 128, - 0, aria_init_key, NULL, - EVP_CIPHER_set_asn1_iv, - EVP_CIPHER_get_asn1_iv, - NULL) -IMPLEMENT_BLOCK_CIPHER(aria_256, ks, aria, EVP_ARIA_KEY, - NID_aria_256, 16, 32, 16, 128, - 0, aria_init_key, NULL, - EVP_CIPHER_set_asn1_iv, - EVP_CIPHER_get_asn1_iv, - NULL) - -# define IMPLEMENT_ARIA_CFBR(ksize,cbits) \ - IMPLEMENT_CFBR(aria,aria,EVP_ARIA_KEY,ks,ksize,cbits,16,0) -IMPLEMENT_ARIA_CFBR(128,1) -IMPLEMENT_ARIA_CFBR(192,1) -IMPLEMENT_ARIA_CFBR(256,1) -IMPLEMENT_ARIA_CFBR(128,8) -IMPLEMENT_ARIA_CFBR(192,8) -IMPLEMENT_ARIA_CFBR(256,8) - -# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ -static const EVP_CIPHER aria_##keylen##_##mode = { \ - nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \ - flags|EVP_CIPH_##MODE##_MODE, \ - EVP_ORIG_GLOBAL, \ - aria_init_key, \ - aria_##mode##_cipher, \ - NULL, \ - sizeof(EVP_ARIA_KEY), \ - NULL,NULL,NULL,NULL }; \ -const EVP_CIPHER *EVP_aria_##keylen##_##mode(void) \ -{ return &aria_##keylen##_##mode; } - -static int aria_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - int n = EVP_CIPHER_CTX_get_num(ctx); - unsigned int num; - EVP_ARIA_KEY *dat = EVP_C_DATA(EVP_ARIA_KEY, ctx); - - if (n < 0) - return 0; - num = (unsigned int)n; - - CRYPTO_ctr128_encrypt(in, out, len, &dat->ks, ctx->iv, - EVP_CIPHER_CTX_buf_noconst(ctx), &num, - (block128_f) ossl_aria_encrypt); - EVP_CIPHER_CTX_set_num(ctx, num); - return 1; -} - -BLOCK_CIPHER_generic(NID_aria, 128, 1, 16, ctr, ctr, CTR, 0) -BLOCK_CIPHER_generic(NID_aria, 192, 1, 16, ctr, ctr, CTR, 0) -BLOCK_CIPHER_generic(NID_aria, 256, 1, 16, ctr, ctr, CTR, 0) - -/* Authenticated cipher modes (GCM/CCM) */ - -/* increment counter (64-bit int) by 1 */ -static void ctr64_inc(unsigned char *counter) -{ - int n = 8; - unsigned char c; - - do { - --n; - c = counter[n]; - ++c; - counter[n] = c; - if (c) - return; - } while (n); -} - -static int aria_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - int ret; - EVP_ARIA_GCM_CTX *gctx = EVP_C_DATA(EVP_ARIA_GCM_CTX, ctx); - - if (!iv && !key) - return 1; - if (key) { - ret = ossl_aria_set_encrypt_key(key, - EVP_CIPHER_CTX_get_key_length(ctx) * 8, - &gctx->ks.ks); - CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, - (block128_f) ossl_aria_encrypt); - if (ret < 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_ARIA_KEY_SETUP_FAILED); - return 0; - } - - /* - * If we have an iv can set it directly, otherwise use saved IV. - */ - if (iv == NULL && gctx->iv_set) - iv = gctx->iv; - if (iv) { - CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen); - gctx->iv_set = 1; - } - gctx->key_set = 1; - } else { - /* If key set use IV, otherwise copy */ - if (gctx->key_set) - CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen); - else - memcpy(gctx->iv, iv, gctx->ivlen); - gctx->iv_set = 1; - gctx->iv_gen = 0; - } - return 1; -} - -static int aria_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) -{ - EVP_ARIA_GCM_CTX *gctx = EVP_C_DATA(EVP_ARIA_GCM_CTX, c); - - switch (type) { - case EVP_CTRL_INIT: - gctx->key_set = 0; - gctx->iv_set = 0; - gctx->ivlen = EVP_CIPHER_get_iv_length(c->cipher); - gctx->iv = c->iv; - gctx->taglen = -1; - gctx->iv_gen = 0; - gctx->tls_aad_len = -1; - return 1; - - case EVP_CTRL_GET_IVLEN: - *(int *)ptr = gctx->ivlen; - return 1; - - case EVP_CTRL_AEAD_SET_IVLEN: - if (arg <= 0) - return 0; - /* Allocate memory for IV if needed */ - if ((arg > EVP_MAX_IV_LENGTH) && (arg > gctx->ivlen)) { - if (gctx->iv != c->iv) - OPENSSL_free(gctx->iv); - if ((gctx->iv = OPENSSL_malloc(arg)) == NULL) - return 0; - } - gctx->ivlen = arg; - return 1; - - case EVP_CTRL_AEAD_SET_TAG: - if (arg <= 0 || arg > 16 || EVP_CIPHER_CTX_is_encrypting(c)) - return 0; - memcpy(EVP_CIPHER_CTX_buf_noconst(c), ptr, arg); - gctx->taglen = arg; - return 1; - - case EVP_CTRL_AEAD_GET_TAG: - if (arg <= 0 || arg > 16 || !EVP_CIPHER_CTX_is_encrypting(c) - || gctx->taglen < 0) - return 0; - memcpy(ptr, EVP_CIPHER_CTX_buf_noconst(c), arg); - return 1; - - case EVP_CTRL_GCM_SET_IV_FIXED: - /* Special case: -1 length restores whole IV */ - if (arg == -1) { - memcpy(gctx->iv, ptr, gctx->ivlen); - gctx->iv_gen = 1; - return 1; - } - /* - * Fixed field must be at least 4 bytes and invocation field at least - * 8. - */ - if ((arg < 4) || (gctx->ivlen - arg) < 8) - return 0; - if (arg) - memcpy(gctx->iv, ptr, arg); - if (EVP_CIPHER_CTX_is_encrypting(c) - && RAND_bytes(gctx->iv + arg, gctx->ivlen - arg) <= 0) - return 0; - gctx->iv_gen = 1; - return 1; - - case EVP_CTRL_GCM_IV_GEN: - if (gctx->iv_gen == 0 || gctx->key_set == 0) - return 0; - CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen); - if (arg <= 0 || arg > gctx->ivlen) - arg = gctx->ivlen; - memcpy(ptr, gctx->iv + gctx->ivlen - arg, arg); - /* - * Invocation field will be at least 8 bytes in size and so no need - * to check wrap around or increment more than last 8 bytes. - */ - ctr64_inc(gctx->iv + gctx->ivlen - 8); - gctx->iv_set = 1; - return 1; - - case EVP_CTRL_GCM_SET_IV_INV: - if (gctx->iv_gen == 0 || gctx->key_set == 0 - || EVP_CIPHER_CTX_is_encrypting(c)) - return 0; - memcpy(gctx->iv + gctx->ivlen - arg, ptr, arg); - CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen); - gctx->iv_set = 1; - return 1; - - case EVP_CTRL_AEAD_TLS1_AAD: - /* Save the AAD for later use */ - if (arg != EVP_AEAD_TLS1_AAD_LEN) - return 0; - memcpy(EVP_CIPHER_CTX_buf_noconst(c), ptr, arg); - gctx->tls_aad_len = arg; - { - unsigned int len = - EVP_CIPHER_CTX_buf_noconst(c)[arg - 2] << 8 - | EVP_CIPHER_CTX_buf_noconst(c)[arg - 1]; - /* Correct length for explicit IV */ - if (len < EVP_GCM_TLS_EXPLICIT_IV_LEN) - return 0; - len -= EVP_GCM_TLS_EXPLICIT_IV_LEN; - /* If decrypting correct for tag too */ - if (!EVP_CIPHER_CTX_is_encrypting(c)) { - if (len < EVP_GCM_TLS_TAG_LEN) - return 0; - len -= EVP_GCM_TLS_TAG_LEN; - } - EVP_CIPHER_CTX_buf_noconst(c)[arg - 2] = len >> 8; - EVP_CIPHER_CTX_buf_noconst(c)[arg - 1] = len & 0xff; - } - /* Extra padding: tag appended to record */ - return EVP_GCM_TLS_TAG_LEN; - - case EVP_CTRL_COPY: - { - EVP_CIPHER_CTX *out = ptr; - EVP_ARIA_GCM_CTX *gctx_out = EVP_C_DATA(EVP_ARIA_GCM_CTX, out); - if (gctx->gcm.key) { - if (gctx->gcm.key != &gctx->ks) - return 0; - gctx_out->gcm.key = &gctx_out->ks; - } - if (gctx->iv == c->iv) - gctx_out->iv = out->iv; - else { - if ((gctx_out->iv = OPENSSL_malloc(gctx->ivlen)) == NULL) - return 0; - memcpy(gctx_out->iv, gctx->iv, gctx->ivlen); - } - return 1; - } - - default: - return -1; - - } -} - -static int aria_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_ARIA_GCM_CTX *gctx = EVP_C_DATA(EVP_ARIA_GCM_CTX, ctx); - int rv = -1; - - /* Encrypt/decrypt must be performed in place */ - if (out != in - || len < (EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN)) - return -1; - /* - * Set IV from start of buffer or generate IV and write to start of - * buffer. - */ - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CIPHER_CTX_is_encrypting(ctx) ? - EVP_CTRL_GCM_IV_GEN : EVP_CTRL_GCM_SET_IV_INV, - EVP_GCM_TLS_EXPLICIT_IV_LEN, out) <= 0) - goto err; - /* Use saved AAD */ - if (CRYPTO_gcm128_aad(&gctx->gcm, EVP_CIPHER_CTX_buf_noconst(ctx), - gctx->tls_aad_len)) - goto err; - /* Fix buffer and length to point to payload */ - in += EVP_GCM_TLS_EXPLICIT_IV_LEN; - out += EVP_GCM_TLS_EXPLICIT_IV_LEN; - len -= EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN; - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { - /* Encrypt payload */ - if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, len)) - goto err; - out += len; - /* Finally write tag */ - CRYPTO_gcm128_tag(&gctx->gcm, out, EVP_GCM_TLS_TAG_LEN); - rv = len + EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN; - } else { - /* Decrypt */ - if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, len)) - goto err; - /* Retrieve tag */ - CRYPTO_gcm128_tag(&gctx->gcm, EVP_CIPHER_CTX_buf_noconst(ctx), - EVP_GCM_TLS_TAG_LEN); - /* If tag mismatch wipe buffer */ - if (CRYPTO_memcmp(EVP_CIPHER_CTX_buf_noconst(ctx), in + len, - EVP_GCM_TLS_TAG_LEN)) { - OPENSSL_cleanse(out, len); - goto err; - } - rv = len; - } - - err: - gctx->iv_set = 0; - gctx->tls_aad_len = -1; - return rv; -} - -static int aria_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_ARIA_GCM_CTX *gctx = EVP_C_DATA(EVP_ARIA_GCM_CTX, ctx); - - /* If not set up, return error */ - if (!gctx->key_set) - return -1; - - if (gctx->tls_aad_len >= 0) - return aria_gcm_tls_cipher(ctx, out, in, len); - - if (!gctx->iv_set) - return -1; - if (in) { - if (out == NULL) { - if (CRYPTO_gcm128_aad(&gctx->gcm, in, len)) - return -1; - } else if (EVP_CIPHER_CTX_is_encrypting(ctx)) { - if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, len)) - return -1; - } else { - if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, len)) - return -1; - } - return len; - } - if (!EVP_CIPHER_CTX_is_encrypting(ctx)) { - if (gctx->taglen < 0) - return -1; - if (CRYPTO_gcm128_finish(&gctx->gcm, - EVP_CIPHER_CTX_buf_noconst(ctx), - gctx->taglen) != 0) - return -1; - gctx->iv_set = 0; - return 0; - } - CRYPTO_gcm128_tag(&gctx->gcm, EVP_CIPHER_CTX_buf_noconst(ctx), 16); - gctx->taglen = 16; - /* Don't reuse the IV */ - gctx->iv_set = 0; - return 0; -} - -static int aria_gcm_cleanup(EVP_CIPHER_CTX *ctx) -{ - EVP_ARIA_GCM_CTX *gctx = EVP_C_DATA(EVP_ARIA_GCM_CTX, ctx); - - if (gctx->iv != ctx->iv) - OPENSSL_free(gctx->iv); - - return 1; -} - -static int aria_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - int ret; - EVP_ARIA_CCM_CTX *cctx = EVP_C_DATA(EVP_ARIA_CCM_CTX, ctx); - - if (!iv && !key) - return 1; - - if (key) { - ret = ossl_aria_set_encrypt_key(key, - EVP_CIPHER_CTX_get_key_length(ctx) * 8, - &cctx->ks.ks); - CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, - &cctx->ks, (block128_f) ossl_aria_encrypt); - if (ret < 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_ARIA_KEY_SETUP_FAILED); - return 0; - } - cctx->str = NULL; - cctx->key_set = 1; - } - if (iv) { - memcpy(ctx->iv, iv, 15 - cctx->L); - cctx->iv_set = 1; - } - return 1; -} - -static int aria_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) -{ - EVP_ARIA_CCM_CTX *cctx = EVP_C_DATA(EVP_ARIA_CCM_CTX, c); - - switch (type) { - case EVP_CTRL_INIT: - cctx->key_set = 0; - cctx->iv_set = 0; - cctx->L = 8; - cctx->M = 12; - cctx->tag_set = 0; - cctx->len_set = 0; - cctx->tls_aad_len = -1; - return 1; - - case EVP_CTRL_GET_IVLEN: - *(int *)ptr = 15 - cctx->L; - return 1; - - case EVP_CTRL_AEAD_TLS1_AAD: - /* Save the AAD for later use */ - if (arg != EVP_AEAD_TLS1_AAD_LEN) - return 0; - memcpy(EVP_CIPHER_CTX_buf_noconst(c), ptr, arg); - cctx->tls_aad_len = arg; - { - uint16_t len = - EVP_CIPHER_CTX_buf_noconst(c)[arg - 2] << 8 - | EVP_CIPHER_CTX_buf_noconst(c)[arg - 1]; - /* Correct length for explicit IV */ - if (len < EVP_CCM_TLS_EXPLICIT_IV_LEN) - return 0; - len -= EVP_CCM_TLS_EXPLICIT_IV_LEN; - /* If decrypting correct for tag too */ - if (!EVP_CIPHER_CTX_is_encrypting(c)) { - if (len < cctx->M) - return 0; - len -= cctx->M; - } - EVP_CIPHER_CTX_buf_noconst(c)[arg - 2] = len >> 8; - EVP_CIPHER_CTX_buf_noconst(c)[arg - 1] = len & 0xff; - } - /* Extra padding: tag appended to record */ - return cctx->M; - - case EVP_CTRL_CCM_SET_IV_FIXED: - /* Sanity check length */ - if (arg != EVP_CCM_TLS_FIXED_IV_LEN) - return 0; - /* Just copy to first part of IV */ - memcpy(c->iv, ptr, arg); - return 1; - - case EVP_CTRL_AEAD_SET_IVLEN: - arg = 15 - arg; - /* fall through */ - case EVP_CTRL_CCM_SET_L: - if (arg < 2 || arg > 8) - return 0; - cctx->L = arg; - return 1; - case EVP_CTRL_AEAD_SET_TAG: - if ((arg & 1) || arg < 4 || arg > 16) - return 0; - if (EVP_CIPHER_CTX_is_encrypting(c) && ptr) - return 0; - if (ptr) { - cctx->tag_set = 1; - memcpy(EVP_CIPHER_CTX_buf_noconst(c), ptr, arg); - } - cctx->M = arg; - return 1; - - case EVP_CTRL_AEAD_GET_TAG: - if (!EVP_CIPHER_CTX_is_encrypting(c) || !cctx->tag_set) - return 0; - if (!CRYPTO_ccm128_tag(&cctx->ccm, ptr, (size_t)arg)) - return 0; - cctx->tag_set = 0; - cctx->iv_set = 0; - cctx->len_set = 0; - return 1; - - case EVP_CTRL_COPY: - { - EVP_CIPHER_CTX *out = ptr; - EVP_ARIA_CCM_CTX *cctx_out = EVP_C_DATA(EVP_ARIA_CCM_CTX, out); - if (cctx->ccm.key) { - if (cctx->ccm.key != &cctx->ks) - return 0; - cctx_out->ccm.key = &cctx_out->ks; - } - return 1; - } - - default: - return -1; - } -} - -static int aria_ccm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_ARIA_CCM_CTX *cctx = EVP_C_DATA(EVP_ARIA_CCM_CTX, ctx); - CCM128_CONTEXT *ccm = &cctx->ccm; - - /* Encrypt/decrypt must be performed in place */ - if (out != in || len < (EVP_CCM_TLS_EXPLICIT_IV_LEN + (size_t)cctx->M)) - return -1; - /* If encrypting set explicit IV from sequence number (start of AAD) */ - if (EVP_CIPHER_CTX_is_encrypting(ctx)) - memcpy(out, EVP_CIPHER_CTX_buf_noconst(ctx), - EVP_CCM_TLS_EXPLICIT_IV_LEN); - /* Get rest of IV from explicit IV */ - memcpy(ctx->iv + EVP_CCM_TLS_FIXED_IV_LEN, in, - EVP_CCM_TLS_EXPLICIT_IV_LEN); - /* Correct length value */ - len -= EVP_CCM_TLS_EXPLICIT_IV_LEN + cctx->M; - if (CRYPTO_ccm128_setiv(ccm, ctx->iv, 15 - cctx->L, - len)) - return -1; - /* Use saved AAD */ - CRYPTO_ccm128_aad(ccm, EVP_CIPHER_CTX_buf_noconst(ctx), - cctx->tls_aad_len); - /* Fix buffer to point to payload */ - in += EVP_CCM_TLS_EXPLICIT_IV_LEN; - out += EVP_CCM_TLS_EXPLICIT_IV_LEN; - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { - if (cctx->str ? CRYPTO_ccm128_encrypt_ccm64(ccm, in, out, len, cctx->str) - : CRYPTO_ccm128_encrypt(ccm, in, out, len)) - return -1; - if (!CRYPTO_ccm128_tag(ccm, out + len, cctx->M)) - return -1; - return len + EVP_CCM_TLS_EXPLICIT_IV_LEN + cctx->M; - } else { - if (cctx->str ? !CRYPTO_ccm128_decrypt_ccm64(ccm, in, out, len, cctx->str) - : !CRYPTO_ccm128_decrypt(ccm, in, out, len)) { - unsigned char tag[16]; - if (CRYPTO_ccm128_tag(ccm, tag, cctx->M)) { - if (!CRYPTO_memcmp(tag, in + len, cctx->M)) - return len; - } - } - OPENSSL_cleanse(out, len); - return -1; - } -} - -static int aria_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_ARIA_CCM_CTX *cctx = EVP_C_DATA(EVP_ARIA_CCM_CTX, ctx); - CCM128_CONTEXT *ccm = &cctx->ccm; - - /* If not set up, return error */ - if (!cctx->key_set) - return -1; - - if (cctx->tls_aad_len >= 0) - return aria_ccm_tls_cipher(ctx, out, in, len); - - /* EVP_*Final() doesn't return any data */ - if (in == NULL && out != NULL) - return 0; - - if (!cctx->iv_set) - return -1; - - if (!out) { - if (!in) { - if (CRYPTO_ccm128_setiv(ccm, ctx->iv, 15 - cctx->L, len)) - return -1; - cctx->len_set = 1; - return len; - } - /* If have AAD need message length */ - if (!cctx->len_set && len) - return -1; - CRYPTO_ccm128_aad(ccm, in, len); - return len; - } - - /* The tag must be set before actually decrypting data */ - if (!EVP_CIPHER_CTX_is_encrypting(ctx) && !cctx->tag_set) - return -1; - - /* If not set length yet do it */ - if (!cctx->len_set) { - if (CRYPTO_ccm128_setiv(ccm, ctx->iv, 15 - cctx->L, len)) - return -1; - cctx->len_set = 1; - } - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { - if (cctx->str ? CRYPTO_ccm128_encrypt_ccm64(ccm, in, out, len, cctx->str) - : CRYPTO_ccm128_encrypt(ccm, in, out, len)) - return -1; - cctx->tag_set = 1; - return len; - } else { - int rv = -1; - if (cctx->str ? !CRYPTO_ccm128_decrypt_ccm64(ccm, in, out, len, - cctx->str) : - !CRYPTO_ccm128_decrypt(ccm, in, out, len)) { - unsigned char tag[16]; - if (CRYPTO_ccm128_tag(ccm, tag, cctx->M)) { - if (!CRYPTO_memcmp(tag, EVP_CIPHER_CTX_buf_noconst(ctx), - cctx->M)) - rv = len; - } - } - if (rv == -1) - OPENSSL_cleanse(out, len); - cctx->iv_set = 0; - cctx->tag_set = 0; - cctx->len_set = 0; - return rv; - } -} - -#define aria_ccm_cleanup NULL - -#define ARIA_AUTH_FLAGS (EVP_CIPH_FLAG_DEFAULT_ASN1 \ - | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \ - | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \ - | EVP_CIPH_CUSTOM_COPY | EVP_CIPH_FLAG_AEAD_CIPHER \ - | EVP_CIPH_CUSTOM_IV_LENGTH) - -#define BLOCK_CIPHER_aead(keylen,mode,MODE) \ -static const EVP_CIPHER aria_##keylen##_##mode = { \ - NID_aria_##keylen##_##mode, \ - 1, keylen/8, 12, \ - ARIA_AUTH_FLAGS|EVP_CIPH_##MODE##_MODE, \ - EVP_ORIG_GLOBAL, \ - aria_##mode##_init_key, \ - aria_##mode##_cipher, \ - aria_##mode##_cleanup, \ - sizeof(EVP_ARIA_##MODE##_CTX), \ - NULL,NULL,aria_##mode##_ctrl,NULL }; \ -const EVP_CIPHER *EVP_aria_##keylen##_##mode(void) \ -{ return (EVP_CIPHER*)&aria_##keylen##_##mode; } - -BLOCK_CIPHER_aead(128, gcm, GCM) -BLOCK_CIPHER_aead(192, gcm, GCM) -BLOCK_CIPHER_aead(256, gcm, GCM) - -BLOCK_CIPHER_aead(128, ccm, CCM) -BLOCK_CIPHER_aead(192, ccm, CCM) -BLOCK_CIPHER_aead(256, ccm, CCM) - -#endif diff --git a/openssl/src/crypto/evp/e_bf.c b/openssl/src/crypto/evp/e_bf.c deleted file mode 100644 index 2aeda2ecf..000000000 --- a/openssl/src/crypto/evp/e_bf.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * BF low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "internal/cryptlib.h" -#ifndef OPENSSL_NO_BF -# include -# include "crypto/evp.h" -# include -# include -# include "evp_local.h" - -static int bf_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc); - -typedef struct { - BF_KEY ks; -} EVP_BF_KEY; - -# define data(ctx) EVP_C_DATA(EVP_BF_KEY,ctx) - -IMPLEMENT_BLOCK_CIPHER(bf, ks, BF, EVP_BF_KEY, NID_bf, 8, 16, 8, 64, - EVP_CIPH_VARIABLE_LENGTH, bf_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, NULL) - -static int bf_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - int len = EVP_CIPHER_CTX_get_key_length(ctx); - - if (len < 0) - return 0; - BF_set_key(&data(ctx)->ks, len, key); - return 1; -} - -#endif diff --git a/openssl/src/crypto/evp/e_camellia.c b/openssl/src/crypto/evp/e_camellia.c deleted file mode 100644 index cb69516bd..000000000 --- a/openssl/src/crypto/evp/e_camellia.c +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include - -#include -#include -#include -#include -#include -#include "crypto/evp.h" -#include "crypto/modes.h" -#include "crypto/cmll_platform.h" -#include "evp_local.h" - -static int camellia_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc); - -/* Camellia subkey Structure */ -typedef struct { - CAMELLIA_KEY ks; - block128_f block; - union { - cbc128_f cbc; - ctr128_f ctr; - } stream; -} EVP_CAMELLIA_KEY; - -#define MAXBITCHUNK ((size_t)1<<(sizeof(size_t)*8-4)) - -/* Attribute operation for Camellia */ -#define data(ctx) EVP_C_DATA(EVP_CAMELLIA_KEY,ctx) - -#if defined(AES_ASM) && (defined(__sparc) || defined(__sparc__)) -/* ---------^^^ this is not a typo, just a way to detect that - * assembler support was in general requested... */ -# include "crypto/sparc_arch.h" - -static int cmll_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - int ret, mode, bits; - EVP_CAMELLIA_KEY *dat = - (EVP_CAMELLIA_KEY *)EVP_CIPHER_CTX_get_cipher_data(ctx); - - mode = EVP_CIPHER_CTX_get_mode(ctx); - bits = EVP_CIPHER_CTX_get_key_length(ctx) * 8; - - cmll_t4_set_key(key, bits, &dat->ks); - - if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) - && !enc) { - ret = 0; - dat->block = (block128_f) cmll_t4_decrypt; - switch (bits) { - case 128: - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? - (cbc128_f) cmll128_t4_cbc_decrypt : NULL; - break; - case 192: - case 256: - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? - (cbc128_f) cmll256_t4_cbc_decrypt : NULL; - break; - default: - ret = -1; - } - } else { - ret = 0; - dat->block = (block128_f) cmll_t4_encrypt; - switch (bits) { - case 128: - if (mode == EVP_CIPH_CBC_MODE) - dat->stream.cbc = (cbc128_f) cmll128_t4_cbc_encrypt; - else if (mode == EVP_CIPH_CTR_MODE) - dat->stream.ctr = (ctr128_f) cmll128_t4_ctr32_encrypt; - else - dat->stream.cbc = NULL; - break; - case 192: - case 256: - if (mode == EVP_CIPH_CBC_MODE) - dat->stream.cbc = (cbc128_f) cmll256_t4_cbc_encrypt; - else if (mode == EVP_CIPH_CTR_MODE) - dat->stream.ctr = (ctr128_f) cmll256_t4_ctr32_encrypt; - else - dat->stream.cbc = NULL; - break; - default: - ret = -1; - } - } - - if (ret < 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_CAMELLIA_KEY_SETUP_FAILED); - return 0; - } - - return 1; -} - -# define cmll_t4_cbc_cipher camellia_cbc_cipher -static int cmll_t4_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len); - -# define cmll_t4_ecb_cipher camellia_ecb_cipher -static int cmll_t4_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len); - -# define cmll_t4_ofb_cipher camellia_ofb_cipher -static int cmll_t4_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len); - -# define cmll_t4_cfb_cipher camellia_cfb_cipher -static int cmll_t4_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len); - -# define cmll_t4_cfb8_cipher camellia_cfb8_cipher -static int cmll_t4_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len); - -# define cmll_t4_cfb1_cipher camellia_cfb1_cipher -static int cmll_t4_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len); - -# define cmll_t4_ctr_cipher camellia_ctr_cipher -static int cmll_t4_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len); - -# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ -static const EVP_CIPHER cmll_t4_##keylen##_##mode = { \ - nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \ - flags|EVP_CIPH_##MODE##_MODE, \ - EVP_ORIG_GLOBAL, \ - cmll_t4_init_key, \ - cmll_t4_##mode##_cipher, \ - NULL, \ - sizeof(EVP_CAMELLIA_KEY), \ - NULL,NULL,NULL,NULL }; \ -static const EVP_CIPHER camellia_##keylen##_##mode = { \ - nid##_##keylen##_##nmode,blocksize, \ - keylen/8,ivlen, \ - flags|EVP_CIPH_##MODE##_MODE, \ - EVP_ORIG_GLOBAL, \ - camellia_init_key, \ - camellia_##mode##_cipher, \ - NULL, \ - sizeof(EVP_CAMELLIA_KEY), \ - NULL,NULL,NULL,NULL }; \ -const EVP_CIPHER *EVP_camellia_##keylen##_##mode(void) \ -{ return SPARC_CMLL_CAPABLE?&cmll_t4_##keylen##_##mode:&camellia_##keylen##_##mode; } - -#else - -# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \ -static const EVP_CIPHER camellia_##keylen##_##mode = { \ - nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \ - flags|EVP_CIPH_##MODE##_MODE, \ - EVP_ORIG_GLOBAL, \ - camellia_init_key, \ - camellia_##mode##_cipher, \ - NULL, \ - sizeof(EVP_CAMELLIA_KEY), \ - NULL,NULL,NULL,NULL }; \ -const EVP_CIPHER *EVP_camellia_##keylen##_##mode(void) \ -{ return &camellia_##keylen##_##mode; } - -#endif - -#define BLOCK_CIPHER_generic_pack(nid,keylen,flags) \ - BLOCK_CIPHER_generic(nid,keylen,16,16,cbc,cbc,CBC,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ - BLOCK_CIPHER_generic(nid,keylen,16,0,ecb,ecb,ECB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ - BLOCK_CIPHER_generic(nid,keylen,1,16,ofb128,ofb,OFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ - BLOCK_CIPHER_generic(nid,keylen,1,16,cfb128,cfb,CFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ - BLOCK_CIPHER_generic(nid,keylen,1,16,cfb1,cfb1,CFB,flags) \ - BLOCK_CIPHER_generic(nid,keylen,1,16,cfb8,cfb8,CFB,flags) \ - BLOCK_CIPHER_generic(nid, keylen, 1, 16, ctr, ctr, CTR, flags) - -/* The subkey for Camellia is generated. */ -static int camellia_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - int ret, mode; - EVP_CAMELLIA_KEY *dat = EVP_C_DATA(EVP_CAMELLIA_KEY, ctx); - - ret = Camellia_set_key(key, EVP_CIPHER_CTX_get_key_length(ctx) * 8, - &dat->ks); - if (ret < 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_CAMELLIA_KEY_SETUP_FAILED); - return 0; - } - - mode = EVP_CIPHER_CTX_get_mode(ctx); - if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) - && !enc) { - dat->block = (block128_f) Camellia_decrypt; - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? - (cbc128_f) Camellia_cbc_encrypt : NULL; - } else { - dat->block = (block128_f) Camellia_encrypt; - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? - (cbc128_f) Camellia_cbc_encrypt : NULL; - } - - return 1; -} - -static int camellia_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_CAMELLIA_KEY *dat = EVP_C_DATA(EVP_CAMELLIA_KEY, ctx); - - if (dat->stream.cbc) - (*dat->stream.cbc) (in, out, len, &dat->ks, ctx->iv, - EVP_CIPHER_CTX_is_encrypting(ctx)); - else if (EVP_CIPHER_CTX_is_encrypting(ctx)) - CRYPTO_cbc128_encrypt(in, out, len, &dat->ks, ctx->iv, dat->block); - else - CRYPTO_cbc128_decrypt(in, out, len, &dat->ks, ctx->iv, dat->block); - - return 1; -} - -static int camellia_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - size_t bl = EVP_CIPHER_CTX_get_block_size(ctx); - size_t i; - EVP_CAMELLIA_KEY *dat = EVP_C_DATA(EVP_CAMELLIA_KEY, ctx); - - if (len < bl) - return 1; - - for (i = 0, len -= bl; i <= len; i += bl) - (*dat->block) (in + i, out + i, &dat->ks); - - return 1; -} - -static int camellia_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_CAMELLIA_KEY *dat = EVP_C_DATA(EVP_CAMELLIA_KEY, ctx); - - int num = EVP_CIPHER_CTX_get_num(ctx); - CRYPTO_ofb128_encrypt(in, out, len, &dat->ks, ctx->iv, &num, dat->block); - EVP_CIPHER_CTX_set_num(ctx, num); - return 1; -} - -static int camellia_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_CAMELLIA_KEY *dat = EVP_C_DATA(EVP_CAMELLIA_KEY, ctx); - - int num = EVP_CIPHER_CTX_get_num(ctx); - CRYPTO_cfb128_encrypt(in, out, len, &dat->ks, ctx->iv, &num, - EVP_CIPHER_CTX_is_encrypting(ctx), dat->block); - EVP_CIPHER_CTX_set_num(ctx, num); - return 1; -} - -static int camellia_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_CAMELLIA_KEY *dat = EVP_C_DATA(EVP_CAMELLIA_KEY, ctx); - - int num = EVP_CIPHER_CTX_get_num(ctx); - CRYPTO_cfb128_8_encrypt(in, out, len, &dat->ks, ctx->iv, &num, - EVP_CIPHER_CTX_is_encrypting(ctx), dat->block); - EVP_CIPHER_CTX_set_num(ctx, num); - return 1; -} - -static int camellia_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_CAMELLIA_KEY *dat = EVP_C_DATA(EVP_CAMELLIA_KEY, ctx); - - if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS)) { - int num = EVP_CIPHER_CTX_get_num(ctx); - CRYPTO_cfb128_1_encrypt(in, out, len, &dat->ks, ctx->iv, &num, - EVP_CIPHER_CTX_is_encrypting(ctx), - dat->block); - EVP_CIPHER_CTX_set_num(ctx, num); - return 1; - } - - while (len >= MAXBITCHUNK) { - int num = EVP_CIPHER_CTX_get_num(ctx); - CRYPTO_cfb128_1_encrypt(in, out, MAXBITCHUNK * 8, &dat->ks, - ctx->iv, &num, - EVP_CIPHER_CTX_is_encrypting(ctx), - dat->block); - EVP_CIPHER_CTX_set_num(ctx, num); - len -= MAXBITCHUNK; - out += MAXBITCHUNK; - in += MAXBITCHUNK; - } - if (len) { - int num = EVP_CIPHER_CTX_get_num(ctx); - CRYPTO_cfb128_1_encrypt(in, out, len * 8, &dat->ks, - ctx->iv, &num, - EVP_CIPHER_CTX_is_encrypting(ctx), - dat->block); - EVP_CIPHER_CTX_set_num(ctx, num); - } - - return 1; -} - -static int camellia_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - int snum = EVP_CIPHER_CTX_get_num(ctx); - unsigned int num; - EVP_CAMELLIA_KEY *dat = EVP_C_DATA(EVP_CAMELLIA_KEY, ctx); - - if (snum < 0) - return 0; - num = snum; - if (dat->stream.ctr) - CRYPTO_ctr128_encrypt_ctr32(in, out, len, &dat->ks, ctx->iv, - EVP_CIPHER_CTX_buf_noconst(ctx), - &num, - dat->stream.ctr); - else - CRYPTO_ctr128_encrypt(in, out, len, &dat->ks, ctx->iv, - EVP_CIPHER_CTX_buf_noconst(ctx), &num, - dat->block); - EVP_CIPHER_CTX_set_num(ctx, num); - return 1; -} - -BLOCK_CIPHER_generic_pack(NID_camellia, 128, 0) - BLOCK_CIPHER_generic_pack(NID_camellia, 192, 0) - BLOCK_CIPHER_generic_pack(NID_camellia, 256, 0) diff --git a/openssl/src/crypto/evp/e_cast.c b/openssl/src/crypto/evp/e_cast.c deleted file mode 100644 index 5e9be2dc7..000000000 --- a/openssl/src/crypto/evp/e_cast.c +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * CAST low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "internal/cryptlib.h" - -#ifndef OPENSSL_NO_CAST -# include -# include -# include "crypto/evp.h" -# include -# include "evp_local.h" - -static int cast_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc); - -typedef struct { - CAST_KEY ks; -} EVP_CAST_KEY; - -# define data(ctx) EVP_C_DATA(EVP_CAST_KEY,ctx) - -IMPLEMENT_BLOCK_CIPHER(cast5, ks, CAST, EVP_CAST_KEY, - NID_cast5, 8, CAST_KEY_LENGTH, 8, 64, - EVP_CIPH_VARIABLE_LENGTH, cast_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, NULL) - -static int cast_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - int keylen = EVP_CIPHER_CTX_get_key_length(ctx); - - if (keylen <= 0) - return 0; - CAST_set_key(&data(ctx)->ks, keylen, key); - return 1; -} - -#endif diff --git a/openssl/src/crypto/evp/e_chacha20_poly1305.c b/openssl/src/crypto/evp/e_chacha20_poly1305.c index 731c1a1dc..18e1c0b5a 100644 --- a/openssl/src/crypto/evp/e_chacha20_poly1305.c +++ b/openssl/src/crypto/evp/e_chacha20_poly1305.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -55,7 +55,7 @@ static int chacha_init_key(EVP_CIPHER_CTX *ctx, return 1; } -static int chacha_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, +static int chacha_cipher(EVP_CIPHER_CTX * ctx, unsigned char *out, const unsigned char *inp, size_t len) { EVP_CHACHA_KEY *key = data(ctx); @@ -239,7 +239,7 @@ static int chacha20_poly1305_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, actx->len.text = plen; if (plen) { - if (EVP_CIPHER_CTX_is_encrypting(ctx)) + if (ctx->encrypt) ctr = xor128_encrypt_n_pad(out, in, ctr, plen); else ctr = xor128_decrypt_n_pad(out, in, ctr, plen); @@ -263,7 +263,7 @@ static int chacha20_poly1305_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, actx->len.aad = EVP_AEAD_TLS1_AAD_LEN; actx->len.text = plen; - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { + if (ctx->encrypt) { for (i = 0; i < plen; i++) { out[i] = ctr[i] ^= in[i]; } @@ -297,7 +297,7 @@ static int chacha20_poly1305_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, actx->len.aad = EVP_AEAD_TLS1_AAD_LEN; actx->len.text = plen; - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { + if (ctx->encrypt) { ChaCha20_ctr32(out, in, plen, actx->key.key.d, actx->key.counter); Poly1305_Update(POLY1305_ctx(actx), out, plen); } else { @@ -340,12 +340,12 @@ static int chacha20_poly1305_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, Poly1305_Update(POLY1305_ctx(actx), tohash, tohash_len); OPENSSL_cleanse(buf, buf_len); - Poly1305_Final(POLY1305_ctx(actx), - EVP_CIPHER_CTX_is_encrypting(ctx) ? actx->tag : tohash); + Poly1305_Final(POLY1305_ctx(actx), ctx->encrypt ? actx->tag + : tohash); actx->tls_payload_length = NO_TLS_PAYLOAD_LENGTH; - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { + if (ctx->encrypt) { memcpy(out, actx->tag, POLY1305_BLOCK_SIZE); } else { if (CRYPTO_memcmp(tohash, in, POLY1305_BLOCK_SIZE)) { @@ -408,7 +408,7 @@ static int chacha20_poly1305_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, else if (len != plen + POLY1305_BLOCK_SIZE) return -1; - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { /* plaintext */ + if (ctx->encrypt) { /* plaintext */ chacha_cipher(ctx, out, in, plen); Poly1305_Update(POLY1305_ctx(actx), out, plen); in += plen; @@ -463,12 +463,12 @@ static int chacha20_poly1305_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, Poly1305_Update(POLY1305_ctx(actx), temp, POLY1305_BLOCK_SIZE); } - Poly1305_Final(POLY1305_ctx(actx), - EVP_CIPHER_CTX_is_encrypting(ctx) ? actx->tag : temp); + Poly1305_Final(POLY1305_ctx(actx), ctx->encrypt ? actx->tag + : temp); actx->mac_inited = 0; if (in != NULL && len != plen) { /* tls mode */ - if (EVP_CIPHER_CTX_is_encrypting(ctx)) { + if (ctx->encrypt) { memcpy(out, actx->tag, POLY1305_BLOCK_SIZE); } else { if (CRYPTO_memcmp(temp, in, POLY1305_BLOCK_SIZE)) { @@ -477,7 +477,7 @@ static int chacha20_poly1305_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } } } - else if (!EVP_CIPHER_CTX_is_encrypting(ctx)) { + else if (!ctx->encrypt) { if (CRYPTO_memcmp(temp, actx->tag, actx->tag_len)) return -1; } @@ -498,7 +498,7 @@ static int chacha20_poly1305_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, { EVP_CHACHA_AEAD_CTX *actx = aead_data(ctx); - switch (type) { + switch(type) { case EVP_CTRL_INIT: if (actx == NULL) actx = ctx->cipher_data @@ -561,8 +561,7 @@ static int chacha20_poly1305_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, return 1; case EVP_CTRL_AEAD_GET_TAG: - if (arg <= 0 || arg > POLY1305_BLOCK_SIZE || - !EVP_CIPHER_CTX_is_encrypting(ctx)) + if (arg <= 0 || arg > POLY1305_BLOCK_SIZE || !ctx->encrypt) return 0; memcpy(ptr, actx->tag, arg); return 1; @@ -578,7 +577,7 @@ static int chacha20_poly1305_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, len = aad[EVP_AEAD_TLS1_AAD_LEN - 2] << 8 | aad[EVP_AEAD_TLS1_AAD_LEN - 1]; aad = actx->tls_aad; - if (!EVP_CIPHER_CTX_is_encrypting(ctx)) { + if (!ctx->encrypt) { if (len < POLY1305_BLOCK_SIZE) return 0; len -= POLY1305_BLOCK_SIZE; /* discount attached tag */ diff --git a/openssl/src/crypto/evp/e_des.c b/openssl/src/crypto/evp/e_des.c index 6eb49c033..653a9bf94 100644 --- a/openssl/src/crypto/evp/e_des.c +++ b/openssl/src/crypto/evp/e_des.c @@ -34,20 +34,6 @@ typedef struct { } stream; } EVP_DES_KEY; -# if defined(AES_ASM) && (defined(__sparc) || defined(__sparc__)) -/* ----------^^^ this is not a typo, just a way to detect that - * assembler support was in general requested... */ -# include "crypto/sparc_arch.h" - -# define SPARC_DES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_DES) - -void des_t4_key_expand(const void *key, DES_key_schedule *ks); -void des_t4_cbc_encrypt(const void *inp, void *out, size_t len, - const DES_key_schedule *ks, unsigned char iv[8]); -void des_t4_cbc_decrypt(const void *inp, void *out, size_t len, - const DES_key_schedule *ks, unsigned char iv[8]); -# endif - static int des_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc); static int des_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr); @@ -149,8 +135,7 @@ static int des_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl) { size_t n, chunk = EVP_MAXCHUNK / 8; - unsigned char c[1]; - unsigned char d[1] = { 0 }; /* Appease Coverity */ + unsigned char c[1], d[1]; if (inl < chunk) chunk = inl; @@ -214,17 +199,6 @@ static int des_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, EVP_DES_KEY *dat = (EVP_DES_KEY *) EVP_CIPHER_CTX_get_cipher_data(ctx); dat->stream.cbc = NULL; -# if defined(SPARC_DES_CAPABLE) - if (SPARC_DES_CAPABLE) { - int mode = EVP_CIPHER_CTX_get_mode(ctx); - - if (mode == EVP_CIPH_CBC_MODE) { - des_t4_key_expand(key, &dat->ks.ks); - dat->stream.cbc = enc ? des_t4_cbc_encrypt : des_t4_cbc_decrypt; - return 1; - } - } -# endif DES_set_key_unchecked(deskey, EVP_CIPHER_CTX_get_cipher_data(ctx)); return 1; } diff --git a/openssl/src/crypto/evp/e_des3.c b/openssl/src/crypto/evp/e_des3.c index c0bc7fdd8..1a109d11a 100644 --- a/openssl/src/crypto/evp/e_des3.c +++ b/openssl/src/crypto/evp/e_des3.c @@ -37,20 +37,6 @@ typedef struct { # define ks2 ks.ks[1] # define ks3 ks.ks[2] -# if defined(AES_ASM) && (defined(__sparc) || defined(__sparc__)) -/* ---------^^^ this is not a typo, just a way to detect that - * assembler support was in general requested... */ -# include "crypto/sparc_arch.h" - -# define SPARC_DES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_DES) - -void des_t4_key_expand(const void *key, DES_key_schedule *ks); -void des_t4_ede3_cbc_encrypt(const void *inp, void *out, size_t len, - const DES_key_schedule ks[3], unsigned char iv[8]); -void des_t4_ede3_cbc_decrypt(const void *inp, void *out, size_t len, - const DES_key_schedule ks[3], unsigned char iv[8]); -# endif - static int des_ede_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc); @@ -165,8 +151,7 @@ static int des_ede3_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl) { size_t n; - unsigned char c[1]; - unsigned char d[1] = { 0 }; /* Appease Coverity */ + unsigned char c[1], d[1]; if (!EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS)) inl *= 8; @@ -229,20 +214,6 @@ static int des_ede_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, DES_EDE_KEY *dat = data(ctx); dat->stream.cbc = NULL; -# if defined(SPARC_DES_CAPABLE) - if (SPARC_DES_CAPABLE) { - int mode = EVP_CIPHER_CTX_get_mode(ctx); - - if (mode == EVP_CIPH_CBC_MODE) { - des_t4_key_expand(&deskey[0], &dat->ks1); - des_t4_key_expand(&deskey[1], &dat->ks2); - memcpy(&dat->ks3, &dat->ks1, sizeof(dat->ks1)); - dat->stream.cbc = enc ? des_t4_ede3_cbc_encrypt : - des_t4_ede3_cbc_decrypt; - return 1; - } - } -# endif DES_set_key_unchecked(&deskey[0], &dat->ks1); DES_set_key_unchecked(&deskey[1], &dat->ks2); memcpy(&dat->ks3, &dat->ks1, sizeof(dat->ks1)); @@ -256,20 +227,6 @@ static int des_ede3_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, DES_EDE_KEY *dat = data(ctx); dat->stream.cbc = NULL; -# if defined(SPARC_DES_CAPABLE) - if (SPARC_DES_CAPABLE) { - int mode = EVP_CIPHER_CTX_get_mode(ctx); - - if (mode == EVP_CIPH_CBC_MODE) { - des_t4_key_expand(&deskey[0], &dat->ks1); - des_t4_key_expand(&deskey[1], &dat->ks2); - des_t4_key_expand(&deskey[2], &dat->ks3); - dat->stream.cbc = enc ? des_t4_ede3_cbc_encrypt : - des_t4_ede3_cbc_decrypt; - return 1; - } - } -# endif DES_set_key_unchecked(&deskey[0], &dat->ks1); DES_set_key_unchecked(&deskey[1], &dat->ks2); DES_set_key_unchecked(&deskey[2], &dat->ks3); @@ -387,6 +344,8 @@ static int des_ede3_wrap(EVP_CIPHER_CTX *ctx, unsigned char *out, static int des_ede3_wrap_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl) { + if (in == NULL || inl == 0) + return 0; /* * Sanity check input length: we typically only wrap keys so EVP_MAXCHUNK * is more than will ever be needed. Also input length must be a multiple diff --git a/openssl/src/crypto/evp/e_eea3.c b/openssl/src/crypto/evp/e_eea3.c new file mode 100644 index 000000000..cf5a68c88 --- /dev/null +++ b/openssl/src/crypto/evp/e_eea3.c @@ -0,0 +1,155 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include "internal/deprecated.h" + +#include "internal/cryptlib.h" + +#ifndef OPENSSL_NO_ZUC + +# include +# include + +# include "crypto/zuc.h" +# include "crypto/evp.h" + +typedef struct { + ZUC_KEY zk; /* working key */ +} EVP_EEA3_KEY; + +# define data(ctx) ((EVP_EEA3_KEY *)EVP_CIPHER_CTX_get_cipher_data(ctx)) + +static int eea3_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc); +static int eea3_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t inl); +static int eea3_cleanup(EVP_CIPHER_CTX *ctx); + +static const EVP_CIPHER zuc_128_eea3_cipher = { + NID_zuc_128_eea3, + 1, /* block_size */ + ZUC_KEY_SIZE, /* key_len */ + ZUC_CTR_SIZE, /* iv_len, 128-bit counter in the context */ + EVP_CIPH_VARIABLE_LENGTH, + EVP_ORIG_GLOBAL, + eea3_init_key, + eea3_cipher, + eea3_cleanup, + sizeof(EVP_EEA3_KEY), + NULL, + NULL, + NULL, + NULL +}; + +const EVP_CIPHER *EVP_eea3(void) +{ + return &zuc_128_eea3_cipher; +} + +static int eea3_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + EVP_EEA3_KEY *ek = data(ctx); + ZUC_KEY *zk = &ek->zk; + uint32_t count; + uint32_t bearer; + uint32_t direction; + + zk->k = key; + + /* + * This is a lazy approach: we 'borrow' the 'iv' parameter + * to use it as a place of transfer the EEA3 iv params - + * count, bearer and direction. + * + * count is 32 bits, bearer is 5 bits and direction is 1 + * bit so we read the first 38 bits of iv. And the whole + * iv is set to 5 bytes (40 bits). + */ + + /* IV is a 'must' */ + if (iv == NULL) + return 0; + + count = ((long)iv[0] << 24) | (iv[1] << 16) | (iv[2] << 8) | iv[3]; + bearer = (iv[4] & 0xF8) >> 3; + direction = (iv[4] & 0x4) >> 2; + + zk->iv[0] = (count >> 24) & 0xFF; + zk->iv[1] = (count >> 16) & 0xFF; + zk->iv[2] = (count >> 8) & 0xFF; + zk->iv[3] = count & 0xFF; + + zk->iv[4] = ((bearer << 3) | ((direction & 1) << 2)) & 0xFC; + zk->iv[5] = zk->iv[6] = zk->iv[7] = 0; + + zk->iv[8] = zk->iv[0]; + zk->iv[9] = zk->iv[1]; + zk->iv[10] = zk->iv[2]; + zk->iv[11] = zk->iv[3]; + zk->iv[12] = zk->iv[4]; + zk->iv[13] = zk->iv[5]; + zk->iv[14] = zk->iv[6]; + zk->iv[15] = zk->iv[7]; + + zk->keystream_len = 0; + zk->inited = 0; + + ZUC_init(zk); + + return 1; +} + +static int eea3_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t inl) +{ + EVP_EEA3_KEY *ek = data(ctx); + ZUC_KEY *zk = &ek->zk; + unsigned int i, k, n, num = EVP_CIPHER_CTX_num(ctx); + + if (num >= zk->keystream_len && !ZUC_generate_keystream(zk)) + return 0; + + n = zk->L * sizeof(uint32_t); + + /* + * EEA3 is based on 'bits', but we can only handle 'bytes'. + * + * So we choose to output a final whole byte, even if there are some + * bits at the end of the input. Those trailing bits in the last byte + * should be discarded by caller. + */ + for (i = 0; i < inl; i++) { + k = num + i; + if (k >= zk->keystream_len) { + if (!ZUC_generate_keystream(zk)) + return 0; + } + + out[i] = in[i] ^ zk->keystream[k % n]; + } + + num += inl; + + /* num always points to next key byte to use */ + EVP_CIPHER_CTX_set_num(ctx, num); + + return 1; +} + +static int eea3_cleanup(EVP_CIPHER_CTX *ctx) +{ + EVP_EEA3_KEY *key = data(ctx); + + ZUC_destroy_keystream(&key->zk); + + return 1; +} +#endif diff --git a/openssl/src/crypto/evp/e_idea.c b/openssl/src/crypto/evp/e_idea.c deleted file mode 100644 index 93da93823..000000000 --- a/openssl/src/crypto/evp/e_idea.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * IDEA low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include -#include "internal/cryptlib.h" - -#ifndef OPENSSL_NO_IDEA -# include -# include -# include "crypto/evp.h" -# include -# include "evp_local.h" - -/* Can't use IMPLEMENT_BLOCK_CIPHER because IDEA_ecb_encrypt is different */ - -typedef struct { - IDEA_KEY_SCHEDULE ks; -} EVP_IDEA_KEY; - -static int idea_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc); - -/* - * NB IDEA_ecb_encrypt doesn't take an 'encrypt' argument so we treat it as a - * special case - */ - -static int idea_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t inl) -{ - BLOCK_CIPHER_ecb_loop() - IDEA_ecb_encrypt(in + i, out + i, &EVP_C_DATA(EVP_IDEA_KEY, ctx)->ks); - return 1; -} - -BLOCK_CIPHER_func_cbc(idea, IDEA, EVP_IDEA_KEY, ks) -BLOCK_CIPHER_func_ofb(idea, IDEA, 64, EVP_IDEA_KEY, ks) -BLOCK_CIPHER_func_cfb(idea, IDEA, 64, EVP_IDEA_KEY, ks) - -BLOCK_CIPHER_defs(idea, IDEA_KEY_SCHEDULE, NID_idea, 8, 16, 8, 64, - 0, idea_init_key, NULL, - EVP_CIPHER_set_asn1_iv, EVP_CIPHER_get_asn1_iv, NULL) - -static int idea_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - if (!enc) { - if (EVP_CIPHER_CTX_get_mode(ctx) == EVP_CIPH_OFB_MODE) - enc = 1; - else if (EVP_CIPHER_CTX_get_mode(ctx) == EVP_CIPH_CFB_MODE) - enc = 1; - } - if (enc) - IDEA_set_encrypt_key(key, &EVP_C_DATA(EVP_IDEA_KEY, ctx)->ks); - else { - IDEA_KEY_SCHEDULE tmp; - - IDEA_set_encrypt_key(key, &tmp); - IDEA_set_decrypt_key(&tmp, &EVP_C_DATA(EVP_IDEA_KEY, ctx)->ks); - OPENSSL_cleanse((unsigned char *)&tmp, sizeof(IDEA_KEY_SCHEDULE)); - } - return 1; -} - -#endif diff --git a/openssl/src/crypto/evp/e_old.c b/openssl/src/crypto/evp/e_old.c index e9c9f2211..1bd95f49d 100644 --- a/openssl/src/crypto/evp/e_old.c +++ b/openssl/src/crypto/evp/e_old.c @@ -13,20 +13,11 @@ /* * Define some deprecated functions, so older programs don't crash and burn - * too quickly. On Windows and VMS, these will never be used, since + * too quickly. On Windows, these will never be used, since * functions and variables in shared libraries are selected by entry point * location, not by name. */ -#ifndef OPENSSL_NO_BF -# undef EVP_bf_cfb -const EVP_CIPHER *EVP_bf_cfb(void); -const EVP_CIPHER *EVP_bf_cfb(void) -{ - return EVP_bf_cfb64(); -} -#endif - #ifndef OPENSSL_NO_DES # undef EVP_des_cfb const EVP_CIPHER *EVP_des_cfb(void); @@ -50,33 +41,6 @@ const EVP_CIPHER *EVP_des_ede_cfb(void) } #endif -#ifndef OPENSSL_NO_IDEA -# undef EVP_idea_cfb -const EVP_CIPHER *EVP_idea_cfb(void); -const EVP_CIPHER *EVP_idea_cfb(void) -{ - return EVP_idea_cfb64(); -} -#endif - -#ifndef OPENSSL_NO_RC2 -# undef EVP_rc2_cfb -const EVP_CIPHER *EVP_rc2_cfb(void); -const EVP_CIPHER *EVP_rc2_cfb(void) -{ - return EVP_rc2_cfb64(); -} -#endif - -#ifndef OPENSSL_NO_CAST -# undef EVP_cast5_cfb -const EVP_CIPHER *EVP_cast5_cfb(void); -const EVP_CIPHER *EVP_cast5_cfb(void) -{ - return EVP_cast5_cfb64(); -} -#endif - #ifndef OPENSSL_NO_RC5 # undef EVP_rc5_32_12_16_cfb const EVP_CIPHER *EVP_rc5_32_12_16_cfb(void); diff --git a/openssl/src/crypto/evp/e_rc2.c b/openssl/src/crypto/evp/e_rc2.c deleted file mode 100644 index ffeb17fb1..000000000 --- a/openssl/src/crypto/evp/e_rc2.c +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RC2 low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "internal/cryptlib.h" - -#ifndef OPENSSL_NO_RC2 - -# include -# include -# include "crypto/evp.h" -# include -# include "evp_local.h" - -static int rc2_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc); -static int rc2_meth_to_magic(EVP_CIPHER_CTX *ctx); -static int rc2_magic_to_meth(int i); -static int rc2_set_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type); -static int rc2_get_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type); -static int rc2_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr); - -typedef struct { - int key_bits; /* effective key bits */ - RC2_KEY ks; /* key schedule */ -} EVP_RC2_KEY; - -# define data(ctx) EVP_C_DATA(EVP_RC2_KEY,ctx) - -IMPLEMENT_BLOCK_CIPHER(rc2, ks, RC2, EVP_RC2_KEY, NID_rc2, - 8, - RC2_KEY_LENGTH, 8, 64, - EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT, - rc2_init_key, NULL, - rc2_set_asn1_type_and_iv, rc2_get_asn1_type_and_iv, - rc2_ctrl) -# define RC2_40_MAGIC 0xa0 -# define RC2_64_MAGIC 0x78 -# define RC2_128_MAGIC 0x3a -static const EVP_CIPHER r2_64_cbc_cipher = { - NID_rc2_64_cbc, - 8, 8 /* 64 bit */ , 8, - EVP_CIPH_CBC_MODE | EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT, - EVP_ORIG_GLOBAL, - rc2_init_key, - rc2_cbc_cipher, - NULL, - sizeof(EVP_RC2_KEY), - rc2_set_asn1_type_and_iv, - rc2_get_asn1_type_and_iv, - rc2_ctrl, - NULL -}; - -static const EVP_CIPHER r2_40_cbc_cipher = { - NID_rc2_40_cbc, - 8, 5 /* 40 bit */ , 8, - EVP_CIPH_CBC_MODE | EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT, - EVP_ORIG_GLOBAL, - rc2_init_key, - rc2_cbc_cipher, - NULL, - sizeof(EVP_RC2_KEY), - rc2_set_asn1_type_and_iv, - rc2_get_asn1_type_and_iv, - rc2_ctrl, - NULL -}; - -const EVP_CIPHER *EVP_rc2_64_cbc(void) -{ - return &r2_64_cbc_cipher; -} - -const EVP_CIPHER *EVP_rc2_40_cbc(void) -{ - return &r2_40_cbc_cipher; -} - -static int rc2_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - RC2_set_key(&data(ctx)->ks, EVP_CIPHER_CTX_get_key_length(ctx), - key, data(ctx)->key_bits); - return 1; -} - -static int rc2_meth_to_magic(EVP_CIPHER_CTX *e) -{ - int i; - - if (EVP_CIPHER_CTX_ctrl(e, EVP_CTRL_GET_RC2_KEY_BITS, 0, &i) <= 0) - return 0; - if (i == 128) - return RC2_128_MAGIC; - else if (i == 64) - return RC2_64_MAGIC; - else if (i == 40) - return RC2_40_MAGIC; - else - return 0; -} - -static int rc2_magic_to_meth(int i) -{ - if (i == RC2_128_MAGIC) - return 128; - else if (i == RC2_64_MAGIC) - return 64; - else if (i == RC2_40_MAGIC) - return 40; - else { - ERR_raise(ERR_LIB_EVP, EVP_R_UNSUPPORTED_KEY_SIZE); - return 0; - } -} - -static int rc2_get_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type) -{ - long num = 0; - int i = 0; - int key_bits; - unsigned int l; - unsigned char iv[EVP_MAX_IV_LENGTH]; - - if (type != NULL) { - l = EVP_CIPHER_CTX_get_iv_length(c); - OPENSSL_assert(l <= sizeof(iv)); - i = ASN1_TYPE_get_int_octetstring(type, &num, iv, l); - if (i != (int)l) - return -1; - key_bits = rc2_magic_to_meth((int)num); - if (!key_bits) - return -1; - if (i > 0 && !EVP_CipherInit_ex(c, NULL, NULL, NULL, iv, -1)) - return -1; - if (EVP_CIPHER_CTX_ctrl(c, EVP_CTRL_SET_RC2_KEY_BITS, key_bits, - NULL) <= 0 - || EVP_CIPHER_CTX_set_key_length(c, key_bits / 8) <= 0) - return -1; - } - return i; -} - -static int rc2_set_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type) -{ - long num; - int i = 0, j; - - if (type != NULL) { - num = rc2_meth_to_magic(c); - j = EVP_CIPHER_CTX_get_iv_length(c); - i = ASN1_TYPE_set_int_octetstring(type, num, c->oiv, j); - } - return i; -} - -static int rc2_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) -{ - switch (type) { - case EVP_CTRL_INIT: - data(c)->key_bits = EVP_CIPHER_CTX_get_key_length(c) * 8; - return 1; - - case EVP_CTRL_GET_RC2_KEY_BITS: - *(int *)ptr = data(c)->key_bits; - return 1; - - case EVP_CTRL_SET_RC2_KEY_BITS: - if (arg > 0) { - data(c)->key_bits = arg; - return 1; - } - return 0; -# ifdef PBE_PRF_TEST - case EVP_CTRL_PBE_PRF_NID: - *(int *)ptr = NID_hmacWithMD5; - return 1; -# endif - - default: - return -1; - } -} - -#endif diff --git a/openssl/src/crypto/evp/e_seed.c b/openssl/src/crypto/evp/e_seed.c deleted file mode 100644 index 65ddb5733..000000000 --- a/openssl/src/crypto/evp/e_seed.c +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * SEED low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include -#include -#include -#include "crypto/evp.h" -#include "evp_local.h" - -static int seed_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc); - -typedef struct { - SEED_KEY_SCHEDULE ks; -} EVP_SEED_KEY; - -IMPLEMENT_BLOCK_CIPHER(seed, ks, SEED, EVP_SEED_KEY, NID_seed, - 16, 16, 16, 128, EVP_CIPH_FLAG_DEFAULT_ASN1, - seed_init_key, 0, 0, 0, 0) - -static int seed_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, - const unsigned char *iv, int enc) -{ - SEED_set_key(key, &EVP_C_DATA(EVP_SEED_KEY, ctx)->ks); - return 1; -} diff --git a/openssl/src/crypto/evp/e_sm4.c b/openssl/src/crypto/evp/e_sm4.c index eeb4fd8e0..a3d2a5b22 100644 --- a/openssl/src/crypto/evp/e_sm4.c +++ b/openssl/src/crypto/evp/e_sm4.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2017 Ribose Inc. All Rights Reserved. * Ported from Ribose contributions from Botan. * @@ -13,10 +13,13 @@ #include "internal/cryptlib.h" #ifndef OPENSSL_NO_SM4 +# include # include +# include # include # include "crypto/sm4.h" # include "crypto/evp.h" +# include "crypto/modes.h" # include "crypto/sm4_platform.h" # include "evp_local.h" @@ -76,17 +79,6 @@ static int sm4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, dat->stream.ecb = (ecb128_f) HWSM4_ecb_encrypt; # endif } else -#endif -#ifdef VPSM4_CAPABLE - if (VPSM4_CAPABLE) { - vpsm4_set_decrypt_key(key, &dat->ks.ks); - dat->block = (block128_f) vpsm4_decrypt; - dat->stream.cbc = NULL; - if (mode == EVP_CIPH_CBC_MODE) - dat->stream.cbc = (cbc128_f) vpsm4_cbc_encrypt; - else if (mode == EVP_CIPH_ECB_MODE) - dat->stream.ecb = (ecb128_f) vpsm4_ecb_encrypt; - } else #endif { dat->block = (block128_f) ossl_sm4_decrypt; @@ -115,19 +107,6 @@ static int sm4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, # endif (void)0; /* terminate potentially open 'else' */ } else -#endif -#ifdef VPSM4_CAPABLE - if (VPSM4_CAPABLE) { - vpsm4_set_encrypt_key(key, &dat->ks.ks); - dat->block = (block128_f) vpsm4_encrypt; - dat->stream.cbc = NULL; - if (mode == EVP_CIPH_CBC_MODE) - dat->stream.cbc = (cbc128_f) vpsm4_cbc_encrypt; - else if (mode == EVP_CIPH_ECB_MODE) - dat->stream.ecb = (ecb128_f) vpsm4_ecb_encrypt; - else if (mode == EVP_CIPH_CTR_MODE) - dat->stream.ctr = (ctr128_f) vpsm4_ctr32_encrypt_blocks; - } else #endif { dat->block = (block128_f) ossl_sm4_encrypt; @@ -224,4 +203,641 @@ static int sm4_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, } DEFINE_BLOCK_CIPHERS(NID_sm4, 0) + +# define BLOCK_CIPHER_custom(nid,blocksize,ivlen,mode,MODE,flags) \ +static const EVP_CIPHER sm4_##mode = { \ + nid##_##mode,blocksize, 128/8, ivlen, \ + flags|EVP_CIPH_##MODE##_MODE, \ + EVP_ORIG_GLOBAL, \ + sm4_##mode##_init, \ + sm4_##mode##_cipher, \ + sm4_##mode##_cleanup, \ + sizeof(EVP_SM4_##MODE##_CTX), \ + NULL,NULL,sm4_##mode##_ctrl,NULL }; \ +const EVP_CIPHER *EVP_sm4_##mode(void) \ +{ return &sm4_##mode; } + +typedef struct { + SM4_KEY ks; /* SM4 key schedule to use */ + int key_set; /* Set if key initialized */ + int iv_set; /* Set if an iv is set */ + GCM128_CONTEXT gcm; + unsigned char *iv; /* Temporary IV store */ + int ivlen; /* IV length */ + int taglen; + int iv_gen; /* It is OK to generate IVs */ + int tls_aad_len; /* TLS AAD length */ + ctr128_f ctr; +} EVP_SM4_GCM_CTX; + +typedef struct { + SM4_KEY ks; /* SM4 key schedule to use */ + int key_set; /* Set if key initialized */ + int iv_set; /* Set if an iv is set */ + int tag_set; /* Set if tag is valid */ + int len_set; /* Set if message length set */ + int L, M; /* L and M parameters from RFC3610 */ + int tls_aad_len; /* TLS AAD length */ + CCM128_CONTEXT ccm; + ccm128_f str; +} EVP_SM4_CCM_CTX; + +static int sm4_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr); +static int sm4_gcm_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc); +static int sm4_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); +static int sm4_gcm_cleanup(EVP_CIPHER_CTX *c); + +static int sm4_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr); +static int sm4_ccm_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc); +static int sm4_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len); +static int sm4_ccm_cleanup(EVP_CIPHER_CTX *c); + +/* increment counter (64-bit int) by 1 */ +static void ctr64_inc(unsigned char *counter) +{ + int n = 8; + unsigned char c; + + do { + --n; + c = counter[n]; + ++c; + counter[n] = c; + if (c) + return; + } while (n); +} + +static int sm4_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) +{ + EVP_SM4_GCM_CTX *gctx = EVP_C_DATA(EVP_SM4_GCM_CTX,c); + + switch (type) { + case EVP_CTRL_INIT: + gctx->key_set = 0; + gctx->iv_set = 0; + gctx->ivlen = EVP_CIPHER_iv_length(c->cipher); + gctx->iv = c->iv; + gctx->taglen = -1; + gctx->iv_gen = 0; + gctx->tls_aad_len = -1; + return 1; + + case EVP_CTRL_GET_IVLEN: + *(int *)ptr = gctx->ivlen; + return 1; + + case EVP_CTRL_AEAD_SET_IVLEN: + if (arg <= 0) + return 0; + /* Allocate memory for IV if needed */ + if ((arg > EVP_MAX_IV_LENGTH) && (arg > gctx->ivlen)) { + if (gctx->iv != c->iv) + OPENSSL_free(gctx->iv); + if ((gctx->iv = OPENSSL_malloc(arg)) == NULL) + return 0; + } + gctx->ivlen = arg; + return 1; + + case EVP_CTRL_AEAD_SET_TAG: + if (arg <= 0 || arg > 16 || c->encrypt) + return 0; + memcpy(c->buf, ptr, arg); + gctx->taglen = arg; + return 1; + + case EVP_CTRL_AEAD_GET_TAG: + if (arg <= 0 || arg > 16 || !c->encrypt || gctx->taglen < 0) + return 0; + memcpy(ptr, c->buf, arg); + return 1; + + case EVP_CTRL_GCM_SET_IV_FIXED: + /* Special case: -1 length restores whole IV */ + if (arg == -1) { + memcpy(gctx->iv, ptr, gctx->ivlen); + gctx->iv_gen = 1; + return 1; + } + /* + * Fixed field must be at least 4 bytes and invocation field at least + * 8. + */ + if ((arg < 4) || (gctx->ivlen - arg) < 8) + return 0; + if (arg) + memcpy(gctx->iv, ptr, arg); + if (c->encrypt && RAND_bytes(gctx->iv + arg, gctx->ivlen - arg) <= 0) + return 0; + gctx->iv_gen = 1; + return 1; + + case EVP_CTRL_GCM_IV_GEN: + if (gctx->iv_gen == 0 || gctx->key_set == 0) + return 0; + CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen); + if (arg <= 0 || arg > gctx->ivlen) + arg = gctx->ivlen; + memcpy(ptr, gctx->iv + gctx->ivlen - arg, arg); + /* + * Invocation field will be at least 8 bytes in size and so no need + * to check wrap around or increment more than last 8 bytes. + */ + ctr64_inc(gctx->iv + gctx->ivlen - 8); + gctx->iv_set = 1; + return 1; + + case EVP_CTRL_GCM_SET_IV_INV: + if (gctx->iv_gen == 0 || gctx->key_set == 0 || c->encrypt) + return 0; + memcpy(gctx->iv + gctx->ivlen - arg, ptr, arg); + CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen); + gctx->iv_set = 1; + return 1; + + case EVP_CTRL_AEAD_TLS1_AAD: + /* Save the AAD for later use */ + if (arg != EVP_AEAD_TLS1_AAD_LEN) + return 0; + memcpy(c->buf, ptr, arg); + gctx->tls_aad_len = arg; + { + unsigned int len = c->buf[arg - 2] << 8 | c->buf[arg - 1]; + /* Correct length for explicit IV */ + if (len < EVP_GCM_TLS_EXPLICIT_IV_LEN) + return 0; + len -= EVP_GCM_TLS_EXPLICIT_IV_LEN; + /* If decrypting correct for tag too */ + if (!c->encrypt) { + if (len < EVP_GCM_TLS_TAG_LEN) + return 0; + len -= EVP_GCM_TLS_TAG_LEN; + } + c->buf[arg - 2] = len >> 8; + c->buf[arg - 1] = len & 0xff; + } + /* Extra padding: tag appended to record */ + return EVP_GCM_TLS_TAG_LEN; + + case EVP_CTRL_COPY: + { + EVP_CIPHER_CTX *out = ptr; + EVP_SM4_GCM_CTX *gctx_out = EVP_C_DATA(EVP_SM4_GCM_CTX,out); + + if (gctx->gcm.key) { + if (gctx->gcm.key != &gctx->ks) + return 0; + gctx_out->gcm.key = &gctx_out->ks; + } + if (gctx->iv == c->iv) + gctx_out->iv = out->iv; + else { + if ((gctx_out->iv = OPENSSL_malloc(gctx->ivlen)) == NULL) { + return 0; + } + memcpy(gctx_out->iv, gctx->iv, gctx->ivlen); + } + return 1; + } + case EVP_CTRL_AEAD_SET_MAC_KEY: + /* no-op */ + return 1; + default: + return -1; + } + return 1; +} + +static int sm4_gcm_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + EVP_SM4_GCM_CTX *gctx = EVP_C_DATA(EVP_SM4_GCM_CTX,ctx); + + if (iv == NULL && key == NULL) + return 1; + if (key) { + do { +# ifdef HWSM4_CAPABLE + if (HWSM4_CAPABLE) { + HWSM4_set_encrypt_key(key, &gctx->ks); + CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, + (block128_f)HWSM4_encrypt); +# ifdef HWSM4_ctr32_encrypt_blocks + gctx->ctr = (ctr128_f)HWSM4_ctr32_encrypt_blocks; +# else /* HWSM4_ctr32_encrypt_blocks */ + gctx->ctr = (ctr128_f)NULL; +# endif + break; + } else +# endif /* HWSM4_CAPABLE */ + (void)0; /* terminate potentially open 'else' */ + + ossl_sm4_set_key(key, &gctx->ks); + CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, + (block128_f)ossl_sm4_encrypt); + gctx->ctr = NULL; + } while (0); + + /* + * If we have an iv can set it directly, otherwise use saved IV. + */ + if (iv == NULL && gctx->iv_set) + iv = gctx->iv; + if (iv) { + CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen); + gctx->iv_set = 1; + } + gctx->key_set = 1; + } else { + /* If key set use IV, otherwise copy */ + if (gctx->key_set) + CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen); + else + memcpy(gctx->iv, iv, gctx->ivlen); + gctx->iv_set = 1; + gctx->iv_gen = 0; + } + return 1; +} + +/* + * Handle TLS GCM packet format. This consists of the last portion of the IV + * followed by the payload and finally the tag. On encrypt generate IV, + * encrypt payload and write the tag. On verify retrieve IV, decrypt payload + * and verify tag. + */ + +static int sm4_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_SM4_GCM_CTX *gctx = EVP_C_DATA(EVP_SM4_GCM_CTX,ctx); + int rv = -1; + /* Encrypt/decrypt must be performed in place */ + if (out != in + || len < (EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN)) + return -1; + /* + * Set IV from start of buffer or generate IV and write to start of + * buffer. + */ + if (EVP_CIPHER_CTX_ctrl(ctx, ctx->encrypt ? EVP_CTRL_GCM_IV_GEN + : EVP_CTRL_GCM_SET_IV_INV, + EVP_GCM_TLS_EXPLICIT_IV_LEN, out) <= 0) + goto err; + /* Use saved AAD */ + if (CRYPTO_gcm128_aad(&gctx->gcm, ctx->buf, gctx->tls_aad_len)) + goto err; + /* Fix buffer and length to point to payload */ + in += EVP_GCM_TLS_EXPLICIT_IV_LEN; + out += EVP_GCM_TLS_EXPLICIT_IV_LEN; + len -= EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN; + if (ctx->encrypt) { + /* Encrypt payload */ + if (gctx->ctr) { + size_t bulk = 0; + if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm, + in + bulk, + out + bulk, + len - bulk, gctx->ctr)) + goto err; + } else { + size_t bulk = 0; + if (CRYPTO_gcm128_encrypt(&gctx->gcm, + in + bulk, out + bulk, len - bulk)) + goto err; + } + out += len; + /* Finally write tag */ + CRYPTO_gcm128_tag(&gctx->gcm, out, EVP_GCM_TLS_TAG_LEN); + rv = len + EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN; + } else { + /* Decrypt */ + if (gctx->ctr) { + size_t bulk = 0; + if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm, + in + bulk, + out + bulk, + len - bulk, gctx->ctr)) + goto err; + } else { + size_t bulk = 0; + if (CRYPTO_gcm128_decrypt(&gctx->gcm, + in + bulk, out + bulk, len - bulk)) + goto err; + } + /* Retrieve tag */ + CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, EVP_GCM_TLS_TAG_LEN); + /* If tag mismatch wipe buffer */ + if (CRYPTO_memcmp(ctx->buf, in + len, EVP_GCM_TLS_TAG_LEN)) { + OPENSSL_cleanse(out, len); + goto err; + } + rv = len; + } + + err: + gctx->iv_set = 0; + gctx->tls_aad_len = -1; + return rv; +} + +static int sm4_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_SM4_GCM_CTX *gctx = EVP_C_DATA(EVP_SM4_GCM_CTX,ctx); + + /* If not set up, return error */ + if (!gctx->key_set) + return -1; + + if (gctx->tls_aad_len >= 0) + return sm4_gcm_tls_cipher(ctx, out, in, len); + + if (!gctx->iv_set) + return -1; + + if (in != NULL) { + if (out == NULL) { + if (CRYPTO_gcm128_aad(&gctx->gcm, in, len)) + return -1; + } else if (ctx->encrypt) { + if (gctx->ctr != NULL) { + if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm, in, out, len, gctx->ctr)) + return -1; + } else { + if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, len)) + return -1; + } + } else { + if (gctx->ctr != NULL) { + if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm, in, out, len, gctx->ctr)) + return -1; + } else { + if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, len)) + return -1; + } + } + return len; + } else { + if (!ctx->encrypt) { + if (gctx->taglen < 0) + return -1; + if (CRYPTO_gcm128_finish(&gctx->gcm, ctx->buf, gctx->taglen) != 0) + return -1; + gctx->iv_set = 0; + return 0; + } + CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, 16); + gctx->taglen = 16; + /* Don't reuse the IV */ + gctx->iv_set = 0; + return 0; + } +} + +static int sm4_gcm_cleanup(EVP_CIPHER_CTX *c) +{ + EVP_SM4_GCM_CTX *gctx = EVP_C_DATA(EVP_SM4_GCM_CTX, c); + const unsigned char *iv; + + if (gctx == NULL) + return 0; + + iv = EVP_CIPHER_CTX_iv(c); + if (iv != gctx->iv) + OPENSSL_free(gctx->iv); + + OPENSSL_cleanse(gctx, sizeof(*gctx)); + return 1; +} + +static int sm4_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) +{ + EVP_SM4_CCM_CTX *cctx = EVP_C_DATA(EVP_SM4_CCM_CTX,c); + + switch (type) { + case EVP_CTRL_INIT: + cctx->key_set = 0; + cctx->iv_set = 0; + cctx->L = 8; + cctx->M = 12; + cctx->tag_set = 0; + cctx->len_set = 0; + cctx->tls_aad_len = -1; + return 1; + case EVP_CTRL_GET_IVLEN: + *(int *)ptr = 15 - cctx->L; + return 1; + case EVP_CTRL_AEAD_TLS1_AAD: + /* Save the AAD for later use */ + if (arg != EVP_AEAD_TLS1_AAD_LEN) + return 0; + memcpy(EVP_CIPHER_CTX_buf_noconst(c), ptr, arg); + cctx->tls_aad_len = arg; + { + uint16_t len = + EVP_CIPHER_CTX_buf_noconst(c)[arg - 2] << 8 + | EVP_CIPHER_CTX_buf_noconst(c)[arg - 1]; + + /* Correct length for explicit IV */ + if (len < EVP_CCM_TLS_EXPLICIT_IV_LEN) + return 0; + len -= EVP_CCM_TLS_EXPLICIT_IV_LEN; + /* If decrypting correct for tag too */ + if (!EVP_CIPHER_CTX_encrypting(c)) { + if (len < cctx->M) + return 0; + len -= cctx->M; + } + EVP_CIPHER_CTX_buf_noconst(c)[arg - 2] = len >> 8; + EVP_CIPHER_CTX_buf_noconst(c)[arg - 1] = len & 0xff; + } + /* Extra padding: tag appended to record */ + return cctx->M; + + case EVP_CTRL_CCM_SET_IV_FIXED: + /* Sanity check length */ + if (arg != EVP_CCM_TLS_FIXED_IV_LEN) + return 0; + /* Just copy to first part of IV */ + memcpy(EVP_CIPHER_CTX_iv_noconst(c), ptr, arg); + return 1; + + case EVP_CTRL_AEAD_SET_IVLEN: + arg = 15 - arg; + /* fall thru */ + case EVP_CTRL_CCM_SET_L: + if (arg < 2 || arg > 8) + return 0; + cctx->L = arg; + return 1; + + case EVP_CTRL_AEAD_SET_TAG: + if ((arg & 1) || arg < 4 || arg > 16) + return 0; + if (EVP_CIPHER_CTX_encrypting(c) && ptr) + return 0; + if (ptr) { + cctx->tag_set = 1; + memcpy(EVP_CIPHER_CTX_buf_noconst(c), ptr, arg); + } + cctx->M = arg; + return 1; + + case EVP_CTRL_AEAD_GET_TAG: + if (!EVP_CIPHER_CTX_encrypting(c) || !cctx->tag_set) + return 0; + if (!CRYPTO_ccm128_tag(&cctx->ccm, ptr, (size_t)arg)) + return 0; + cctx->tag_set = 0; + cctx->iv_set = 0; + cctx->len_set = 0; + return 1; + + case EVP_CTRL_COPY: + { + EVP_CIPHER_CTX *out = ptr; + EVP_SM4_CCM_CTX *cctx_out = EVP_C_DATA(EVP_SM4_CCM_CTX,out); + + if (cctx->ccm.key) { + if (cctx->ccm.key != &cctx->ks) + return 0; + cctx_out->ccm.key = &cctx_out->ks; + } + return 1; + } + + default: + return -1; + + } +} + +static int sm4_ccm_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, + const unsigned char *iv, int enc) +{ + EVP_SM4_CCM_CTX *cctx = EVP_C_DATA(EVP_SM4_CCM_CTX,ctx); + + if (iv == NULL && key == NULL) + return 1; + if (key != NULL) + do { +# ifdef HWSM4_CAPABLE + if (HWSM4_CAPABLE) { + HWSM4_set_encrypt_key(key, &cctx->ks); + CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, + &cctx->ks, (block128_f)HWSM4_encrypt); + cctx->str = NULL; + cctx->key_set = 1; + break; + } else +# endif /* HWSM4_CAPABLE */ + (void)0; /* terminate potentially open 'else' */ + + ossl_sm4_set_key(key, &cctx->ks); + CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, + &cctx->ks, (block128_f)ossl_sm4_encrypt); + cctx->str = NULL; + cctx->key_set = 1; + } while (0); + if (iv != NULL) { + memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), iv, 15 - cctx->L); + cctx->iv_set = 1; + } + return 1; +} + +static int sm4_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, + const unsigned char *in, size_t len) +{ + EVP_SM4_CCM_CTX *cctx = EVP_C_DATA(EVP_SM4_CCM_CTX,ctx); + CCM128_CONTEXT *ccm = &cctx->ccm; + + /* If not set up, return error */ + if (!cctx->key_set) + return -1; + + /* EVP_*Final() doesn't return any data */ + if (in == NULL && out != NULL) + return 0; + + if (!cctx->iv_set) + return -1; + + if (out == NULL) { + if (in == NULL) { + if (CRYPTO_ccm128_setiv(ccm, EVP_CIPHER_CTX_iv_noconst(ctx), + 15 - cctx->L, len)) + return -1; + cctx->len_set = 1; + return len; + } + /* If have AAD need message length */ + if (!cctx->len_set && len) + return -1; + CRYPTO_ccm128_aad(ccm, in, len); + return len; + } + + /* The tag must be set before actually decrypting data */ + if (!EVP_CIPHER_CTX_encrypting(ctx) && !cctx->tag_set) + return -1; + + /* If not set length yet do it */ + if (!cctx->len_set) { + if (CRYPTO_ccm128_setiv(ccm, EVP_CIPHER_CTX_iv_noconst(ctx), + 15 - cctx->L, len)) + return -1; + cctx->len_set = 1; + } + if (EVP_CIPHER_CTX_encrypting(ctx)) { + if (cctx->str != NULL ? + CRYPTO_ccm128_encrypt_ccm64(ccm, in, out, len, + cctx->str) : + CRYPTO_ccm128_encrypt(ccm, in, out, len)) + return -1; + cctx->tag_set = 1; + return len; + } else { + int rv = -1; + + if (cctx->str != NULL ? !CRYPTO_ccm128_decrypt_ccm64(ccm, in, out, len, + cctx->str) : + !CRYPTO_ccm128_decrypt(ccm, in, out, len)) { + unsigned char tag[16]; + if (CRYPTO_ccm128_tag(ccm, tag, cctx->M)) { + if (!CRYPTO_memcmp(tag, EVP_CIPHER_CTX_buf_noconst(ctx), + cctx->M)) + rv = len; + } + } + if (rv == -1) + OPENSSL_cleanse(out, len); + cctx->iv_set = 0; + cctx->tag_set = 0; + cctx->len_set = 0; + return rv; + } + +} + +static int sm4_ccm_cleanup(EVP_CIPHER_CTX *c) +{ + return 1; +} + +#define CUSTOM_FLAGS (EVP_CIPH_FLAG_DEFAULT_ASN1 \ + | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \ + | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \ + | EVP_CIPH_CUSTOM_COPY | EVP_CIPH_CUSTOM_IV_LENGTH) + +BLOCK_CIPHER_custom(NID_sm4, 1, 12, gcm, GCM, + EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS) +BLOCK_CIPHER_custom(NID_sm4, 1, 12, ccm, CCM, + EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS) #endif diff --git a/openssl/src/crypto/evp/ec_ctrl.c b/openssl/src/crypto/evp/ec_ctrl.c index c1cf221a0..404358ab9 100644 --- a/openssl/src/crypto/evp/ec_ctrl.c +++ b/openssl/src/crypto/evp/ec_ctrl.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -283,9 +283,7 @@ int EVP_PKEY_CTX_get0_ecdh_kdf_ukm(EVP_PKEY_CTX *ctx, unsigned char **pukm) */ int EVP_PKEY_CTX_set_ec_paramgen_curve_nid(EVP_PKEY_CTX *ctx, int nid) { - int keytype = nid == EVP_PKEY_SM2 ? EVP_PKEY_SM2 : EVP_PKEY_EC; - - return EVP_PKEY_CTX_ctrl(ctx, keytype, EVP_PKEY_OP_TYPE_GEN, + return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_EC, EVP_PKEY_OP_TYPE_GEN, EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID, nid, NULL); } diff --git a/openssl/src/crypto/evp/evp_enc.c b/openssl/src/crypto/evp/evp_enc.c index c289b2f7b..cd1093147 100644 --- a/openssl/src/crypto/evp/evp_enc.c +++ b/openssl/src/crypto/evp/evp_enc.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,12 +24,9 @@ #include "internal/cryptlib.h" #include "internal/provider.h" #include "internal/core.h" -#include "internal/safe_math.h" #include "crypto/evp.h" #include "evp_local.h" -OSSL_SAFE_MATH_SIGNED(int, int) - int EVP_CIPHER_CTX_reset(EVP_CIPHER_CTX *ctx) { if (ctx == NULL) @@ -46,7 +43,6 @@ int EVP_CIPHER_CTX_reset(EVP_CIPHER_CTX *ctx) if (ctx->fetched_cipher != NULL) EVP_CIPHER_free(ctx->fetched_cipher); memset(ctx, 0, sizeof(*ctx)); - ctx->iv_len = -1; return 1; @@ -65,20 +61,39 @@ int EVP_CIPHER_CTX_reset(EVP_CIPHER_CTX *ctx) ENGINE_finish(ctx->engine); #endif memset(ctx, 0, sizeof(*ctx)); - ctx->iv_len = -1; return 1; } -EVP_CIPHER_CTX *EVP_CIPHER_CTX_new(void) +#ifndef OPENSSL_NO_EVP_CIPHER_API_COMPAT + +/* compatibility: for lua ffi */ + +# ifdef EVP_CIPHER_CTX_init +# undef EVP_CIPHER_CTX_init +# endif + +# ifdef EVP_CIPHER_CTX_cleanup +# undef EVP_CIPHER_CTX_cleanup +# endif + +void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *ctx); +int EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *ctx); + +void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *ctx) +{ + EVP_CIPHER_CTX_reset(ctx); +} + +int EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *ctx) { - EVP_CIPHER_CTX *ctx; + return EVP_CIPHER_CTX_reset(ctx); +} - ctx = OPENSSL_zalloc(sizeof(EVP_CIPHER_CTX)); - if (ctx == NULL) - return NULL; +#endif - ctx->iv_len = -1; - return ctx; +EVP_CIPHER_CTX *EVP_CIPHER_CTX_new(void) +{ + return OPENSSL_zalloc(sizeof(EVP_CIPHER_CTX)); } void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *ctx) @@ -99,7 +114,6 @@ static int evp_cipher_init_internal(EVP_CIPHER_CTX *ctx, #if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE) ENGINE *tmpimpl = NULL; #endif - /* * enc == 1 means we are encrypting. * enc == 0 means we are decrypting. @@ -144,10 +158,7 @@ static int evp_cipher_init_internal(EVP_CIPHER_CTX *ctx, #if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE) || tmpimpl != NULL #endif - || impl != NULL - || (cipher != NULL && cipher->origin == EVP_ORIG_METH) - || (cipher == NULL && ctx->cipher != NULL - && ctx->cipher->origin == EVP_ORIG_METH)) { + || impl != NULL) { if (ctx->cipher == ctx->fetched_cipher) ctx->cipher = NULL; EVP_CIPHER_free(ctx->fetched_cipher); @@ -159,12 +170,11 @@ static int evp_cipher_init_internal(EVP_CIPHER_CTX *ctx, * (legacy code) */ if (cipher != NULL && ctx->cipher != NULL) { - if (ctx->cipher->cleanup != NULL && !ctx->cipher->cleanup(ctx)) - return 0; OPENSSL_clear_free(ctx->cipher_data, ctx->cipher->ctx_size); ctx->cipher_data = NULL; } + /* Start of non-legacy code below */ /* Ensure a context left lying around from last time is cleared */ @@ -200,19 +210,12 @@ static int evp_cipher_init_internal(EVP_CIPHER_CTX *ctx, #endif } - if (!ossl_assert(cipher->prov != NULL)) { - ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); - return 0; - } - - if (cipher != ctx->fetched_cipher) { + if (cipher->prov != NULL) { if (!EVP_CIPHER_up_ref((EVP_CIPHER *)cipher)) { ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); return 0; } EVP_CIPHER_free(ctx->fetched_cipher); - /* Coverity false positive, the reference counting is confusing it */ - /* coverity[use_after_free] */ ctx->fetched_cipher = (EVP_CIPHER *)cipher; } ctx->cipher = cipher; @@ -248,12 +251,12 @@ static int evp_cipher_init_internal(EVP_CIPHER_CTX *ctx, OSSL_PARAM *q = param_lens; const OSSL_PARAM *p; - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_KEYLEN); + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_KEYLEN); if (p != NULL) memcpy(q++, p, sizeof(*q)); /* - * Note that OSSL_CIPHER_PARAM_AEAD_IVLEN is a synonym for + * Note that OSSL_CIPHER_PARAM_AEAD_IVLEN is a synomym for * OSSL_CIPHER_PARAM_IVLEN so both are covered here. */ p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_IVLEN); @@ -354,6 +357,7 @@ static int evp_cipher_init_internal(EVP_CIPHER_CTX *ctx, ctx->cipher_data = OPENSSL_zalloc(ctx->cipher->ctx_size); if (ctx->cipher_data == NULL) { ctx->cipher = NULL; + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } } else { @@ -363,7 +367,7 @@ static int evp_cipher_init_internal(EVP_CIPHER_CTX *ctx, /* Preserve wrap enable flag, zero everything else */ ctx->flags &= EVP_CIPHER_CTX_FLAG_WRAP_ALLOW; if (ctx->cipher->flags & EVP_CIPH_CTRL_INIT) { - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_INIT, 0, NULL) <= 0) { + if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_INIT, 0, NULL)) { ctx->cipher = NULL; ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); return 0; @@ -536,19 +540,7 @@ int EVP_DecryptInit_ex2(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, #ifdef PTRDIFF_T # undef PTRDIFF_T #endif -#if defined(OPENSSL_SYS_VMS) && __INITIAL_POINTER_SIZE==64 -/* - * Then we have VMS that distinguishes itself by adhering to - * sizeof(size_t)==4 even in 64-bit builds, which means that - * difference between two pointers might be truncated to 32 bits. - * In the context one can even wonder how comparison for - * equality is implemented. To be on the safe side we adhere to - * PTRDIFF_T even for comparison for equality. - */ -# define PTRDIFF_T uint64_t -#else -# define PTRDIFF_T size_t -#endif +#define PTRDIFF_T size_t int ossl_is_partially_overlapping(const void *ptr1, const void *ptr2, int len) { @@ -571,7 +563,7 @@ static int evp_EncryptDecryptUpdate(EVP_CIPHER_CTX *ctx, int i, j, bl, cmpl = inl; if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS)) - cmpl = safe_div_round_up_int(cmpl, 8, NULL); + cmpl = (cmpl + 7) / 8; bl = ctx->cipher->block_size; @@ -662,7 +654,7 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, size_t soutl, inl_ = (size_t)inl; int blocksize; - if (ossl_likely(outl != NULL)) { + if (outl != NULL) { *outl = 0; } else { ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER); @@ -670,22 +662,22 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, } /* Prevent accidental use of decryption context when encrypting */ - if (ossl_unlikely(!ctx->encrypt)) { + if (!ctx->encrypt) { ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_OPERATION); return 0; } - if (ossl_unlikely(ctx->cipher == NULL)) { + if (ctx->cipher == NULL) { ERR_raise(ERR_LIB_EVP, EVP_R_NO_CIPHER_SET); return 0; } - if (ossl_unlikely(ctx->cipher->prov == NULL)) + if (ctx->cipher->prov == NULL) goto legacy; blocksize = ctx->cipher->block_size; - if (ossl_unlikely(ctx->cipher->cupdate == NULL || blocksize < 1)) { + if (ctx->cipher->cupdate == NULL || blocksize < 1) { ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR); return 0; } @@ -694,7 +686,7 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, inl_ + (size_t)(blocksize == 1 ? 0 : blocksize), in, inl_); - if (ossl_likely(ret)) { + if (ret) { if (soutl > INT_MAX) { ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR); return 0; @@ -811,7 +803,7 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, size_t soutl, inl_ = (size_t)inl; int blocksize; - if (ossl_likely(outl != NULL)) { + if (outl != NULL) { *outl = 0; } else { ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER); @@ -819,21 +811,21 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, } /* Prevent accidental use of encryption context when decrypting */ - if (ossl_unlikely(ctx->encrypt)) { + if (ctx->encrypt) { ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_OPERATION); return 0; } - if (ossl_unlikely(ctx->cipher == NULL)) { + if (ctx->cipher == NULL) { ERR_raise(ERR_LIB_EVP, EVP_R_NO_CIPHER_SET); return 0; } - if (ossl_unlikely(ctx->cipher->prov == NULL)) + if (ctx->cipher->prov == NULL) goto legacy; blocksize = EVP_CIPHER_CTX_get_block_size(ctx); - if (ossl_unlikely(ctx->cipher->cupdate == NULL || blocksize < 1)) { + if (ctx->cipher->cupdate == NULL || blocksize < 1) { ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR); return 0; } @@ -841,7 +833,7 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, inl_ + (size_t)(blocksize == 1 ? 0 : blocksize), in, inl_); - if (ossl_likely(ret)) { + if (ret) { if (soutl > INT_MAX) { ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR); return 0; @@ -857,7 +849,7 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, b = ctx->cipher->block_size; if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS)) - cmpl = safe_div_round_up_int(cmpl, 8, NULL); + cmpl = (cmpl + 7) / 8; if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) { if (b == 1 && ossl_is_partially_overlapping(out, in, cmpl)) { @@ -1044,7 +1036,7 @@ int EVP_CIPHER_CTX_set_key_length(EVP_CIPHER_CTX *c, int keylen) if (c->cipher->prov != NULL) { int ok; OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END }; - size_t len; + size_t len = keylen; if (EVP_CIPHER_CTX_get_key_length(c) == keylen) return 1; @@ -1057,13 +1049,9 @@ int EVP_CIPHER_CTX_set_key_length(EVP_CIPHER_CTX *c, int keylen) } params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_KEYLEN, &len); - if (!OSSL_PARAM_set_int(params, keylen)) - return 0; ok = evp_do_ciph_ctx_setparams(c->cipher, c->algctx, params); - if (ok <= 0) - return 0; - c->key_len = keylen; - return 1; + + return ok > 0 ? 1 : 0; } /* Code below to be removed when legacy support is dropped. */ @@ -1123,13 +1111,7 @@ int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, void *ptr) switch (type) { case EVP_CTRL_SET_KEY_LENGTH: - if (arg < 0) - return 0; - if (ctx->key_len == arg) - /* Skip calling into provider if unchanged. */ - return 1; params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_KEYLEN, &sz); - ctx->key_len = -1; break; case EVP_CTRL_RAND_KEY: /* Used by DES */ set_params = 0; @@ -1153,18 +1135,13 @@ int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, void *ptr) case EVP_CTRL_AEAD_SET_IVLEN: if (arg < 0) return 0; - if (ctx->iv_len == arg) - /* Skip calling into provider if unchanged. */ - return 1; params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_IVLEN, &sz); - ctx->iv_len = -1; break; case EVP_CTRL_CCM_SET_L: if (arg < 2 || arg > 8) return 0; sz = 15 - arg; params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_IVLEN, &sz); - ctx->iv_len = -1; break; case EVP_CTRL_AEAD_SET_IV_FIXED: params[0] = OSSL_PARAM_construct_octet_string( @@ -1217,13 +1194,6 @@ int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, void *ptr) if (ret <= 0) goto end; return sz; -#ifndef OPENSSL_NO_RC2 - case EVP_CTRL_GET_RC2_KEY_BITS: - set_params = 0; /* Fall thru */ - case EVP_CTRL_SET_RC2_KEY_BITS: - params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_RC2_KEYBITS, &sz); - break; -#endif /* OPENSSL_NO_RC2 */ #if !defined(OPENSSL_NO_MULTIBLOCK) case EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE: params[0] = OSSL_PARAM_construct_size_t( @@ -1328,27 +1298,9 @@ int EVP_CIPHER_get_params(EVP_CIPHER *cipher, OSSL_PARAM params[]) int EVP_CIPHER_CTX_set_params(EVP_CIPHER_CTX *ctx, const OSSL_PARAM params[]) { - int r = 0; - const OSSL_PARAM *p; - - if (ctx->cipher != NULL && ctx->cipher->set_ctx_params != NULL) { - r = ctx->cipher->set_ctx_params(ctx->algctx, params); - if (r > 0) { - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_KEYLEN); - if (p != NULL && !OSSL_PARAM_get_int(p, &ctx->key_len)) { - r = 0; - ctx->key_len = -1; - } - } - if (r > 0) { - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_IVLEN); - if (p != NULL && !OSSL_PARAM_get_int(p, &ctx->iv_len)) { - r = 0; - ctx->iv_len = -1; - } - } - } - return r; + if (ctx->cipher != NULL && ctx->cipher->set_ctx_params != NULL) + return ctx->cipher->set_ctx_params(ctx->algctx, params); + return 0; } int EVP_CIPHER_CTX_get_params(EVP_CIPHER_CTX *ctx, OSSL_PARAM params[]) @@ -1444,17 +1396,6 @@ int EVP_CIPHER_CTX_rand_key(EVP_CIPHER_CTX *ctx, unsigned char *key) #endif /* FIPS_MODULE */ } -EVP_CIPHER_CTX *EVP_CIPHER_CTX_dup(const EVP_CIPHER_CTX *in) -{ - EVP_CIPHER_CTX *out = EVP_CIPHER_CTX_new(); - - if (out != NULL && !EVP_CIPHER_CTX_copy(out, in)) { - EVP_CIPHER_CTX_free(out); - out = NULL; - } - return out; -} - int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in) { if ((in == NULL) || (in->cipher == NULL)) { @@ -1506,6 +1447,7 @@ int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in) out->cipher_data = OPENSSL_malloc(in->cipher->ctx_size); if (out->cipher_data == NULL) { out->cipher = NULL; + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } memcpy(out->cipher_data, in->cipher_data, in->cipher->ctx_size); @@ -1524,9 +1466,13 @@ EVP_CIPHER *evp_cipher_new(void) { EVP_CIPHER *cipher = OPENSSL_zalloc(sizeof(EVP_CIPHER)); - if (cipher != NULL && !CRYPTO_NEW_REF(&cipher->refcnt, 1)) { - OPENSSL_free(cipher); - return NULL; + if (cipher != NULL) { + cipher->lock = CRYPTO_THREAD_lock_new(); + if (cipher->lock == NULL) { + OPENSSL_free(cipher); + return NULL; + } + cipher->refcnt = 1; } return cipher; } @@ -1571,7 +1517,7 @@ static void *evp_cipher_from_algorithm(const int name_id, int fnciphcnt = 0, fnctxcnt = 0; if ((cipher = evp_cipher_new()) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } @@ -1726,7 +1672,7 @@ int EVP_CIPHER_up_ref(EVP_CIPHER *cipher) int ref = 0; if (cipher->origin == EVP_ORIG_DYNAMIC) - CRYPTO_UP_REF(&cipher->refcnt, &ref); + CRYPTO_UP_REF(&cipher->refcnt, &ref, cipher->lock); return 1; } @@ -1734,7 +1680,7 @@ void evp_cipher_free_int(EVP_CIPHER *cipher) { OPENSSL_free(cipher->type_name); ossl_provider_free(cipher->prov); - CRYPTO_FREE_REF(&cipher->refcnt); + CRYPTO_THREAD_lock_free(cipher->lock); OPENSSL_free(cipher); } @@ -1745,7 +1691,7 @@ void EVP_CIPHER_free(EVP_CIPHER *cipher) if (cipher == NULL || cipher->origin != EVP_ORIG_DYNAMIC) return; - CRYPTO_DOWN_REF(&cipher->refcnt, &i); + CRYPTO_DOWN_REF(&cipher->refcnt, &i, cipher->lock); if (i > 0) return; evp_cipher_free_int(cipher); diff --git a/openssl/src/crypto/evp/evp_err.c b/openssl/src/crypto/evp/evp_err.c index 42dd7e400..eb1881438 100644 --- a/openssl/src/crypto/evp/evp_err.c +++ b/openssl/src/crypto/evp/evp_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,16 +17,12 @@ static const ERR_STRING_DATA EVP_str_reasons[] = { {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_AES_KEY_SETUP_FAILED), "aes key setup failed"}, - {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_ARIA_KEY_SETUP_FAILED), - "aria key setup failed"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_BAD_ALGORITHM_NAME), "bad algorithm name"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_BAD_DECRYPT), "bad decrypt"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_BAD_KEY_LENGTH), "bad key length"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_BUFFER_TOO_SMALL), "buffer too small"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_CACHE_CONSTANTS_FAILED), "cache constants failed"}, - {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_CAMELLIA_KEY_SETUP_FAILED), - "camellia key setup failed"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_CANNOT_GET_PARAMETERS), "cannot get parameters"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_CANNOT_SET_PARAMETERS), @@ -65,6 +61,8 @@ static const ERR_STRING_DATA EVP_str_reasons[] = { {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_EXPECTING_A_ECX_KEY), "expecting an ecx key"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_EXPECTING_A_EC_KEY), "expecting an ec key"}, + {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_EXPECTING_A_EIA3_KEY), + "expecting a eia3 key"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_EXPECTING_A_POLY1305_KEY), "expecting a poly1305 key"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_EXPECTING_A_SIPHASH_KEY), @@ -160,16 +158,12 @@ static const ERR_STRING_DATA EVP_str_reasons[] = { "unable to lock context"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNABLE_TO_SET_CALLBACKS), "unable to set callbacks"}, - {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNKNOWN_BITS), "unknown bits"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNKNOWN_CIPHER), "unknown cipher"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNKNOWN_DIGEST), "unknown digest"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNKNOWN_KEY_TYPE), "unknown key type"}, - {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNKNOWN_MAX_SIZE), "unknown max size"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNKNOWN_OPTION), "unknown option"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNKNOWN_PBE_ALGORITHM), "unknown pbe algorithm"}, - {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNKNOWN_SECURITY_BITS), - "unknown security bits"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNSUPPORTED_ALGORITHM), "unsupported algorithm"}, {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNSUPPORTED_CIPHER), "unsupported cipher"}, diff --git a/openssl/src/crypto/evp/evp_fetch.c b/openssl/src/crypto/evp/evp_fetch.c index 2067c4062..c126ea177 100644 --- a/openssl/src/crypto/evp/evp_fetch.c +++ b/openssl/src/crypto/evp/evp_fetch.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,12 +17,30 @@ #include "internal/core.h" #include "internal/provider.h" #include "internal/namemap.h" -#include "crypto/decoder.h" +#include "internal/property.h" #include "crypto/evp.h" /* evp_local.h needs it */ #include "evp_local.h" #define NAME_SEPARATOR ':' +static void evp_method_store_free(void *vstore) +{ + ossl_method_store_free(vstore); +} + +static void *evp_method_store_new(OSSL_LIB_CTX *ctx) +{ + return ossl_method_store_new(ctx); +} + + +static const OSSL_LIB_CTX_METHOD evp_method_store_method = { + /* We want evp_method_store to be cleaned up before the provider store */ + OSSL_LIB_CTX_METHOD_PRIORITY_2, + evp_method_store_new, + evp_method_store_free, +}; + /* Data to be passed through ossl_method_construct() */ struct evp_method_data_st { OSSL_LIB_CTX *libctx; @@ -61,29 +79,8 @@ static void *get_tmp_evp_method_store(void *data) static OSSL_METHOD_STORE *get_evp_method_store(OSSL_LIB_CTX *libctx) { - return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_EVP_METHOD_STORE_INDEX); -} - -static int reserve_evp_method_store(void *store, void *data) -{ - struct evp_method_data_st *methdata = data; - - if (store == NULL - && (store = get_evp_method_store(methdata->libctx)) == NULL) - return 0; - - return ossl_method_lock_store(store); -} - -static int unreserve_evp_method_store(void *store, void *data) -{ - struct evp_method_data_st *methdata = data; - - if (store == NULL - && (store = get_evp_method_store(methdata->libctx)) == NULL) - return 0; - - return ossl_method_unlock_store(store); + return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_EVP_METHOD_STORE_INDEX, + &evp_method_store_method); } /* @@ -123,7 +120,7 @@ static void *get_evp_method_from_store(void *store, const OSSL_PROVIDER **prov, { struct evp_method_data_st *methdata = data; void *method = NULL; - int name_id; + int name_id = 0; uint32_t meth_id; /* @@ -240,7 +237,8 @@ static void destruct_evp_method(void *method, void *data) static void * inner_evp_generic_fetch(struct evp_method_data_st *methdata, OSSL_PROVIDER *prov, int operation_id, - const char *name, const char *properties, + int name_id, const char *name, + const char *properties, void *(*new_method)(int name_id, const OSSL_ALGORITHM *algodef, OSSL_PROVIDER *prov), @@ -252,7 +250,7 @@ inner_evp_generic_fetch(struct evp_method_data_st *methdata, const char *const propq = properties != NULL ? properties : ""; uint32_t meth_id = 0; void *method = NULL; - int unsupported, name_id; + int unsupported = 0; if (store == NULL || namemap == NULL) { ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_INVALID_ARGUMENT); @@ -268,8 +266,18 @@ inner_evp_generic_fetch(struct evp_method_data_st *methdata, return NULL; } + /* + * If we have been passed both a name_id and a name, we have an + * internal programming error. + */ + if (!ossl_assert(name_id == 0 || name == NULL)) { + ERR_raise(ERR_LIB_EVP, ERR_R_INTERNAL_ERROR); + return NULL; + } + /* If we haven't received a name id yet, try to get one for the name */ - name_id = name != NULL ? ossl_namemap_name2num(namemap, name) : 0; + if (name_id == 0 && name != NULL) + name_id = ossl_namemap_name2num(namemap, name); /* * If we have a name id, calculate a method id with evp_method_id(). @@ -288,14 +296,13 @@ inner_evp_generic_fetch(struct evp_method_data_st *methdata, * If we haven't found the name yet, chances are that the algorithm to * be fetched is unsupported. */ - unsupported = name_id == 0; + if (name_id == 0) + unsupported = 1; if (meth_id == 0 || !ossl_method_store_cache_get(store, prov, meth_id, propq, &method)) { OSSL_METHOD_CONSTRUCT_METHOD mcm = { get_tmp_evp_method_store, - reserve_evp_method_store, - unreserve_evp_method_store, get_evp_method_from_store, put_evp_method_in_store, construct_evp_method, @@ -318,26 +325,13 @@ inner_evp_generic_fetch(struct evp_method_data_st *methdata, * there is a correct name_id and meth_id, since those have * already been calculated in get_evp_method_from_store() and * put_evp_method_in_store() above. - * Note that there is a corner case here, in which, if a user - * passes a name of the form name1:name2:..., then the construction - * will create a method against all names, but the lookup will fail - * as ossl_namemap_name2num treats the name string as a single name - * rather than introducing new features where in the EVP__fetch - * parses the string and queries for each, return an error. */ if (name_id == 0) name_id = ossl_namemap_name2num(namemap, name); - if (name_id == 0) { - ERR_raise_data(ERR_LIB_EVP, ERR_R_FETCH_FAILED, - "Algorithm %s cannot be found", name); - free_method(method); - method = NULL; - } else { - meth_id = evp_method_id(name_id, operation_id); - if (meth_id != 0) - ossl_method_store_cache_set(store, prov, meth_id, propq, - method, up_ref_method, free_method); - } + meth_id = evp_method_id(name_id, operation_id); + if (name_id != 0) + ossl_method_store_cache_set(store, prov, meth_id, propq, + method, up_ref_method, free_method); } /* @@ -376,7 +370,34 @@ void *evp_generic_fetch(OSSL_LIB_CTX *libctx, int operation_id, methdata.libctx = libctx; methdata.tmp_store = NULL; method = inner_evp_generic_fetch(&methdata, NULL, operation_id, - name, properties, + 0, name, properties, + new_method, up_ref_method, free_method); + dealloc_tmp_evp_method_store(methdata.tmp_store); + return method; +} + +/* + * evp_generic_fetch_by_number() is special, and only returns methods for + * already known names, i.e. it refuses to work if no name_id can be found + * (it's considered an internal programming error). + * This is meant to be used when one method needs to fetch an associated + * method. + */ +void *evp_generic_fetch_by_number(OSSL_LIB_CTX *libctx, int operation_id, + int name_id, const char *properties, + void *(*new_method)(int name_id, + const OSSL_ALGORITHM *algodef, + OSSL_PROVIDER *prov), + int (*up_ref_method)(void *), + void (*free_method)(void *)) +{ + struct evp_method_data_st methdata; + void *method; + + methdata.libctx = libctx; + methdata.tmp_store = NULL; + method = inner_evp_generic_fetch(&methdata, NULL, operation_id, + name_id, NULL, properties, new_method, up_ref_method, free_method); dealloc_tmp_evp_method_store(methdata.tmp_store); return method; @@ -402,28 +423,18 @@ void *evp_generic_fetch_from_prov(OSSL_PROVIDER *prov, int operation_id, methdata.libctx = ossl_provider_libctx(prov); methdata.tmp_store = NULL; method = inner_evp_generic_fetch(&methdata, prov, operation_id, - name, properties, + 0, name, properties, new_method, up_ref_method, free_method); dealloc_tmp_evp_method_store(methdata.tmp_store); return method; } -int evp_method_store_cache_flush(OSSL_LIB_CTX *libctx) +int evp_method_store_flush(OSSL_LIB_CTX *libctx) { OSSL_METHOD_STORE *store = get_evp_method_store(libctx); if (store != NULL) - return ossl_method_store_cache_flush_all(store); - return 1; -} - -int evp_method_store_remove_all_provided(const OSSL_PROVIDER *prov) -{ - OSSL_LIB_CTX *libctx = ossl_provider_libctx(prov); - OSSL_METHOD_STORE *store = get_evp_method_store(libctx); - - if (store != NULL) - return ossl_method_store_remove_all_provided(store, prov); + return ossl_method_store_flush_cache(store, 1); return 1; } @@ -436,7 +447,6 @@ static int evp_set_parsed_default_properties(OSSL_LIB_CTX *libctx, OSSL_PROPERTY_LIST **plp = ossl_ctx_global_properties(libctx, loadconfig); if (plp != NULL && store != NULL) { - int ret; #ifndef FIPS_MODULE char *propstr = NULL; size_t strsz; @@ -470,12 +480,8 @@ static int evp_set_parsed_default_properties(OSSL_LIB_CTX *libctx, #endif ossl_property_free(*plp); *plp = def_prop; - - ret = ossl_method_store_cache_flush_all(store); -#ifndef FIPS_MODULE - ossl_decoder_cache_flush(libctx); -#endif - return ret; + if (store != NULL) + return ossl_method_store_flush_cache(store, 0); } ERR_raise(ERR_LIB_EVP, ERR_R_INTERNAL_ERROR); return 0; @@ -519,7 +525,7 @@ static int evp_default_properties_merge(OSSL_LIB_CTX *libctx, const char *propq, pl2 = ossl_property_merge(pl1, *plp); ossl_property_free(pl1); if (pl2 == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } if (!evp_set_parsed_default_properties(libctx, pl2, 0, 0)) { @@ -571,8 +577,10 @@ char *evp_get_global_properties_str(OSSL_LIB_CTX *libctx, int loadconfig) } propstr = OPENSSL_malloc(sz); - if (propstr == NULL) + if (propstr == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } if (ossl_property_list_to_string(libctx, *plp, propstr, sz) == 0) { ERR_raise(ERR_LIB_EVP, ERR_R_INTERNAL_ERROR); OPENSSL_free(propstr); @@ -609,7 +617,7 @@ void evp_generic_do_all(OSSL_LIB_CTX *libctx, int operation_id, methdata.libctx = libctx; methdata.tmp_store = NULL; - (void)inner_evp_generic_fetch(&methdata, NULL, operation_id, NULL, NULL, + (void)inner_evp_generic_fetch(&methdata, NULL, operation_id, 0, NULL, NULL, new_method, up_ref_method, free_method); data.operation_id = operation_id; diff --git a/openssl/src/crypto/evp/evp_key.c b/openssl/src/crypto/evp/evp_key.c index 7ef94e44e..607d45ee2 100644 --- a/openssl/src/crypto/evp/evp_key.c +++ b/openssl/src/crypto/evp/evp_key.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -88,7 +88,7 @@ int EVP_BytesToKey(const EVP_CIPHER *type, const EVP_MD *md, nkey = EVP_CIPHER_get_key_length(type); niv = EVP_CIPHER_get_iv_length(type); OPENSSL_assert(nkey <= EVP_MAX_KEY_LENGTH); - OPENSSL_assert(niv >= 0 && niv <= EVP_MAX_IV_LENGTH); + OPENSSL_assert(niv <= EVP_MAX_IV_LENGTH); if (data == NULL) return nkey; diff --git a/openssl/src/crypto/evp/evp_lib.c b/openssl/src/crypto/evp/evp_lib.c index be95668c7..0b3cad005 100644 --- a/openssl/src/crypto/evp/evp_lib.c +++ b/openssl/src/crypto/evp/evp_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,7 +24,6 @@ #include #include #include "crypto/evp.h" -#include "crypto/cryptlib.h" #include "internal/provider.h" #include "evp_local.h" @@ -81,12 +80,8 @@ int evp_cipher_param_to_asn1_ex(EVP_CIPHER_CTX *c, ASN1_TYPE *type, evp_cipher_aead_asn1_params *asn1_params) { int ret = -1; /* Assume the worst */ - const EVP_CIPHER *cipher; + const EVP_CIPHER *cipher = c->cipher; - if (c == NULL || c->cipher == NULL) - goto err; - - cipher = c->cipher; /* * For legacy implementations, we detect custom AlgorithmIdentifier * parameter handling by checking if the function pointer @@ -176,12 +171,8 @@ int evp_cipher_asn1_to_param_ex(EVP_CIPHER_CTX *c, ASN1_TYPE *type, evp_cipher_aead_asn1_params *asn1_params) { int ret = -1; /* Assume the worst */ - const EVP_CIPHER *cipher; - - if (c == NULL || c->cipher == NULL) - goto err; + const EVP_CIPHER *cipher = c->cipher; - cipher = c->cipher; /* * For legacy implementations, we detect custom AlgorithmIdentifier * parameter handling by checking if there the function pointer @@ -217,7 +208,7 @@ int evp_cipher_asn1_to_param_ex(EVP_CIPHER_CTX *c, ASN1_TYPE *type, break; default: - ret = EVP_CIPHER_get_asn1_iv(c, type) >= 0 ? 1 : -1; + ret = EVP_CIPHER_get_asn1_iv(c, type); } } else if (cipher->prov != NULL) { OSSL_PARAM params[3], *p = params; @@ -238,7 +229,6 @@ int evp_cipher_asn1_to_param_ex(EVP_CIPHER_CTX *c, ASN1_TYPE *type, ret = -2; } -err: if (ret == -2) ERR_raise(ERR_LIB_EVP, EVP_R_UNSUPPORTED_CIPHER); else if (ret <= 0) @@ -289,12 +279,6 @@ int EVP_CIPHER_get_type(const EVP_CIPHER *cipher) switch (nid) { - case NID_rc2_cbc: - case NID_rc2_64_cbc: - case NID_rc2_40_cbc: - - return NID_rc2_cbc; - case NID_rc4: case NID_rc4_40: @@ -396,7 +380,7 @@ int evp_cipher_cache_constants(EVP_CIPHER *cipher) int EVP_CIPHER_get_block_size(const EVP_CIPHER *cipher) { - return (cipher == NULL) ? 0 : cipher->block_size; + return cipher->block_size; } int EVP_CIPHER_CTX_get_block_size(const EVP_CIPHER_CTX *ctx) @@ -412,9 +396,6 @@ int EVP_CIPHER_impl_ctx_size(const EVP_CIPHER *e) int EVP_Cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, unsigned int inl) { - if (ctx == NULL || ctx->cipher == NULL) - return 0; - if (ctx->cipher->prov != NULL) { /* * If the provided implementation has a ccipher function, we use it, @@ -427,9 +408,6 @@ int EVP_Cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, size_t outl = 0; size_t blocksize = EVP_CIPHER_CTX_get_block_size(ctx); - if (blocksize == 0) - return 0; - if (ctx->cipher->ccipher != NULL) ret = ctx->cipher->ccipher(ctx->algctx, out, &outl, inl + (blocksize == 1 ? 0 : blocksize), @@ -469,7 +447,7 @@ EVP_CIPHER *EVP_CIPHER_CTX_get1_cipher(EVP_CIPHER_CTX *ctx) { EVP_CIPHER *cipher; - if (ctx == NULL || ctx->cipher == NULL) + if (ctx == NULL) return NULL; cipher = (EVP_CIPHER *)ctx->cipher; if (!EVP_CIPHER_up_ref(cipher)) @@ -484,7 +462,7 @@ int EVP_CIPHER_CTX_is_encrypting(const EVP_CIPHER_CTX *ctx) unsigned long EVP_CIPHER_get_flags(const EVP_CIPHER *cipher) { - return cipher == NULL ? 0 : cipher->flags; + return cipher->flags; } void *EVP_CIPHER_CTX_get_app_data(const EVP_CIPHER_CTX *ctx) @@ -514,46 +492,28 @@ void *EVP_CIPHER_CTX_set_cipher_data(EVP_CIPHER_CTX *ctx, void *cipher_data) int EVP_CIPHER_get_iv_length(const EVP_CIPHER *cipher) { - return (cipher == NULL) ? 0 : cipher->iv_len; + return cipher->iv_len; } int EVP_CIPHER_CTX_get_iv_length(const EVP_CIPHER_CTX *ctx) { - if (ctx->cipher == NULL) - return 0; + int rv, len = EVP_CIPHER_get_iv_length(ctx->cipher); + size_t v = len; + OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END }; - if (ctx->iv_len < 0) { - int rv, len = EVP_CIPHER_get_iv_length(ctx->cipher); - size_t v = len; - OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END }; - - if (ctx->cipher->get_ctx_params != NULL) { - params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_IVLEN, - &v); - rv = evp_do_ciph_ctx_getparams(ctx->cipher, ctx->algctx, params); - if (rv > 0) { - if (OSSL_PARAM_modified(params) - && !OSSL_PARAM_get_int(params, &len)) - return -1; - } else if (rv != EVP_CTRL_RET_UNSUPPORTED) { - return -1; - } - } - /* Code below to be removed when legacy support is dropped. */ - else if ((EVP_CIPHER_get_flags(ctx->cipher) - & EVP_CIPH_CUSTOM_IV_LENGTH) != 0) { - rv = EVP_CIPHER_CTX_ctrl((EVP_CIPHER_CTX *)ctx, EVP_CTRL_GET_IVLEN, - 0, &len); - if (rv <= 0) - return -1; - } - /*- - * Casting away the const is annoying but required here. We need to - * cache the result for performance reasons. - */ - ((EVP_CIPHER_CTX *)ctx)->iv_len = len; + params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_IVLEN, &v); + rv = evp_do_ciph_ctx_getparams(ctx->cipher, ctx->algctx, params); + if (rv == EVP_CTRL_RET_UNSUPPORTED) + goto legacy; + return rv != 0 ? (int)v : -1; + /* Code below to be removed when legacy support is dropped. */ +legacy: + if ((EVP_CIPHER_get_flags(ctx->cipher) & EVP_CIPH_CUSTOM_IV_LENGTH) != 0) { + rv = EVP_CIPHER_CTX_ctrl((EVP_CIPHER_CTX *)ctx, EVP_CTRL_GET_IVLEN, + 0, &len); + return (rv == 1) ? len : -1; } - return ctx->iv_len; + return len; } int EVP_CIPHER_CTX_get_tag_length(const EVP_CIPHER_CTX *ctx) @@ -620,7 +580,7 @@ int EVP_CIPHER_CTX_get_updated_iv(EVP_CIPHER_CTX *ctx, void *buf, size_t len) params[0] = OSSL_PARAM_construct_octet_string(OSSL_CIPHER_PARAM_UPDATED_IV, buf, len); - return evp_do_ciph_ctx_getparams(ctx->cipher, ctx->algctx, params) > 0; + return evp_do_ciph_ctx_getparams(ctx->cipher, ctx->algctx, params); } int EVP_CIPHER_CTX_get_original_iv(EVP_CIPHER_CTX *ctx, void *buf, size_t len) @@ -629,7 +589,7 @@ int EVP_CIPHER_CTX_get_original_iv(EVP_CIPHER_CTX *ctx, void *buf, size_t len) params[0] = OSSL_PARAM_construct_octet_string(OSSL_CIPHER_PARAM_IV, buf, len); - return evp_do_ciph_ctx_getparams(ctx->cipher, ctx->algctx, params) > 0; + return evp_do_ciph_ctx_getparams(ctx->cipher, ctx->algctx, params); } unsigned char *EVP_CIPHER_CTX_buf_noconst(EVP_CIPHER_CTX *ctx) @@ -637,6 +597,16 @@ unsigned char *EVP_CIPHER_CTX_buf_noconst(EVP_CIPHER_CTX *ctx) return ctx->buf; } +int EVP_CIPHER_CTX_get_buf_len(const EVP_CIPHER_CTX *ctx) +{ + return ctx->buf_len; +} + +int EVP_CIPHER_CTX_get_final_used(const EVP_CIPHER_CTX *ctx) +{ + return ctx->final_used; +} + int EVP_CIPHER_CTX_get_num(const EVP_CIPHER_CTX *ctx) { int ok; @@ -670,44 +640,28 @@ int EVP_CIPHER_get_key_length(const EVP_CIPHER *cipher) int EVP_CIPHER_CTX_get_key_length(const EVP_CIPHER_CTX *ctx) { - if (ctx->key_len <= 0 && ctx->cipher->prov != NULL) { - int ok; - OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END }; - size_t len; + int ok; + size_t v = ctx->key_len; + OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END }; - params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_KEYLEN, &len); - ok = evp_do_ciph_ctx_getparams(ctx->cipher, ctx->algctx, params); - if (ok <= 0) - return EVP_CTRL_RET_UNSUPPORTED; + params[0] = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_KEYLEN, &v); + ok = evp_do_ciph_ctx_getparams(ctx->cipher, ctx->algctx, params); - /*- - * The if branch should never be taken since EVP_MAX_KEY_LENGTH is - * less than INT_MAX but best to be safe. - * - * Casting away the const is annoying but required here. We need to - * cache the result for performance reasons. - */ - if (!OSSL_PARAM_get_int(params, &((EVP_CIPHER_CTX *)ctx)->key_len)) - return -1; - ((EVP_CIPHER_CTX *)ctx)->key_len = (int)len; - } - return ctx->key_len; + return ok != 0 ? (int)v : EVP_CTRL_RET_UNSUPPORTED; } int EVP_CIPHER_get_nid(const EVP_CIPHER *cipher) { - return (cipher == NULL) ? NID_undef : cipher->nid; + return cipher->nid; } int EVP_CIPHER_CTX_get_nid(const EVP_CIPHER_CTX *ctx) { - return EVP_CIPHER_get_nid(ctx->cipher); + return ctx->cipher->nid; } int EVP_CIPHER_is_a(const EVP_CIPHER *cipher, const char *name) { - if (cipher == NULL) - return 0; if (cipher->prov != NULL) return evp_is_a(cipher->prov, cipher->name_id, NULL, name); return evp_is_a(NULL, 0, EVP_CIPHER_get0_name(cipher), name); @@ -762,8 +716,6 @@ int EVP_CIPHER_get_mode(const EVP_CIPHER *cipher) int EVP_MD_is_a(const EVP_MD *md, const char *name) { - if (md == NULL) - return 0; if (md->prov != NULL) return evp_is_a(md->prov, md->name_id, NULL, name); return evp_is_a(NULL, 0, EVP_MD_get0_name(md), name); @@ -870,10 +822,10 @@ EVP_MD *EVP_MD_meth_dup(const EVP_MD *md) return NULL; if ((to = EVP_MD_meth_new(md->type, md->pkey_type)) != NULL) { - CRYPTO_REF_COUNT refcnt = to->refcnt; + CRYPTO_RWLOCK *lock = to->lock; memcpy(to, md, sizeof(*to)); - to->refcnt = refcnt; + to->lock = lock; to->origin = EVP_ORIG_METH; } return to; @@ -883,7 +835,7 @@ void evp_md_free_int(EVP_MD *md) { OPENSSL_free(md->type_name); ossl_provider_free(md->prov); - CRYPTO_FREE_REF(&md->refcnt); + CRYPTO_THREAD_lock_free(md->lock); OPENSSL_free(md); } @@ -1146,8 +1098,6 @@ int EVP_CIPHER_CTX_test_flags(const EVP_CIPHER_CTX *ctx, int flags) return (ctx->flags & flags); } -#if !defined(FIPS_MODULE) - int EVP_PKEY_CTX_set_group_name(EVP_PKEY_CTX *ctx, const char *name) { OSSL_PARAM params[] = { OSSL_PARAM_END, OSSL_PARAM_END }; @@ -1244,5 +1194,3 @@ EVP_PKEY *EVP_PKEY_Q_keygen(OSSL_LIB_CTX *libctx, const char *propq, va_end(args); return ret; } - -#endif /* !defined(FIPS_MODULE) */ diff --git a/openssl/src/crypto/evp/evp_local.h b/openssl/src/crypto/evp/evp_local.h index 8c1ff35cf..d9e1ca997 100644 --- a/openssl/src/crypto/evp/evp_local.h +++ b/openssl/src/crypto/evp/evp_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2000-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -46,7 +46,6 @@ struct evp_cipher_ctx_st { /* FIXME: Should this even exist? It appears unused */ void *app_data; /* application stuff */ int key_len; /* May change for variable length cipher */ - int iv_len; /* IV length */ unsigned long flags; /* Various flags */ void *cipher_data; /* per EVP data */ int final_used; @@ -95,12 +94,11 @@ struct evp_keymgmt_st { int id; /* libcrypto internal */ int name_id; - /* NID for the legacy alg if there is one */ - int legacy_alg; char *type_name; const char *description; OSSL_PROVIDER *prov; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; /* Constructor(s), destructor, information */ OSSL_FUNC_keymgmt_new_fn *new; @@ -129,10 +127,8 @@ struct evp_keymgmt_st { /* Import and export routines */ OSSL_FUNC_keymgmt_import_fn *import; OSSL_FUNC_keymgmt_import_types_fn *import_types; - OSSL_FUNC_keymgmt_import_types_ex_fn *import_types_ex; OSSL_FUNC_keymgmt_export_fn *export; OSSL_FUNC_keymgmt_export_types_fn *export_types; - OSSL_FUNC_keymgmt_export_types_ex_fn *export_types_ex; OSSL_FUNC_keymgmt_dup_fn *dup; } /* EVP_KEYMGMT */ ; @@ -142,6 +138,7 @@ struct evp_keyexch_st { const char *description; OSSL_PROVIDER *prov; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_keyexch_newctx_fn *newctx; OSSL_FUNC_keyexch_init_fn *init; @@ -161,6 +158,7 @@ struct evp_signature_st { const char *description; OSSL_PROVIDER *prov; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_signature_newctx_fn *newctx; OSSL_FUNC_signature_sign_init_fn *sign_init; @@ -195,6 +193,7 @@ struct evp_asym_cipher_st { const char *description; OSSL_PROVIDER *prov; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_asym_cipher_newctx_fn *newctx; OSSL_FUNC_asym_cipher_encrypt_init_fn *encrypt_init; @@ -215,6 +214,7 @@ struct evp_kem_st { const char *description; OSSL_PROVIDER *prov; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_kem_newctx_fn *newctx; OSSL_FUNC_kem_encapsulate_init_fn *encapsulate_init; @@ -227,8 +227,6 @@ struct evp_kem_st { OSSL_FUNC_kem_gettable_ctx_params_fn *gettable_ctx_params; OSSL_FUNC_kem_set_ctx_params_fn *set_ctx_params; OSSL_FUNC_kem_settable_ctx_params_fn *settable_ctx_params; - OSSL_FUNC_kem_auth_encapsulate_init_fn *auth_encapsulate_init; - OSSL_FUNC_kem_auth_decapsulate_init_fn *auth_decapsulate_init; } /* EVP_KEM */; int PKCS5_v2_PBKDF2_keyivgen(EVP_CIPHER_CTX *ctx, const char *pass, @@ -271,6 +269,13 @@ void *evp_generic_fetch(OSSL_LIB_CTX *ctx, int operation_id, OSSL_PROVIDER *prov), int (*up_ref_method)(void *), void (*free_method)(void *)); +void *evp_generic_fetch_by_number(OSSL_LIB_CTX *ctx, int operation_id, + int name_id, const char *properties, + void *(*new_method)(int name_id, + const OSSL_ALGORITHM *algodef, + OSSL_PROVIDER *prov), + int (*up_ref_method)(void *), + void (*free_method)(void *)); void *evp_generic_fetch_from_prov(OSSL_PROVIDER *prov, int operation_id, const char *name, const char *properties, void *(*new_method)(int name_id, @@ -293,6 +298,9 @@ void evp_generic_do_all(OSSL_LIB_CTX *libctx, int operation_id, /* Internal fetchers for method types that are to be combined with others */ EVP_KEYMGMT *evp_keymgmt_fetch_by_number(OSSL_LIB_CTX *ctx, int name_id, const char *properties); +EVP_KEYMGMT *evp_keymgmt_fetch_from_prov(OSSL_PROVIDER *prov, + const char *name, + const char *properties); EVP_SIGNATURE *evp_signature_fetch_from_prov(OSSL_PROVIDER *prov, const char *name, const char *properties); diff --git a/openssl/src/crypto/evp/evp_pbe.c b/openssl/src/crypto/evp/evp_pbe.c index 9153ecfaa..7e3c29f9c 100644 --- a/openssl/src/crypto/evp/evp_pbe.c +++ b/openssl/src/crypto/evp/evp_pbe.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,12 +33,8 @@ struct evp_pbe_st { static STACK_OF(EVP_PBE_CTL) *pbe_algs; static const EVP_PBE_CTL builtin_pbe[] = { - {EVP_PBE_TYPE_OUTER, NID_pbeWithMD2AndDES_CBC, - NID_des_cbc, NID_md2, PKCS5_PBE_keyivgen, PKCS5_PBE_keyivgen_ex}, {EVP_PBE_TYPE_OUTER, NID_pbeWithMD5AndDES_CBC, NID_des_cbc, NID_md5, PKCS5_PBE_keyivgen, PKCS5_PBE_keyivgen_ex}, - {EVP_PBE_TYPE_OUTER, NID_pbeWithSHA1AndRC2_CBC, - NID_rc2_64_cbc, NID_sha1, PKCS5_PBE_keyivgen, PKCS5_PBE_keyivgen_ex}, {EVP_PBE_TYPE_OUTER, NID_id_pbkdf2, -1, -1, PKCS5_v2_PBKDF2_keyivgen}, @@ -50,17 +46,9 @@ static const EVP_PBE_CTL builtin_pbe[] = { NID_des_ede3_cbc, NID_sha1, PKCS12_PBE_keyivgen, &PKCS12_PBE_keyivgen_ex}, {EVP_PBE_TYPE_OUTER, NID_pbe_WithSHA1And2_Key_TripleDES_CBC, NID_des_ede_cbc, NID_sha1, PKCS12_PBE_keyivgen, &PKCS12_PBE_keyivgen_ex}, - {EVP_PBE_TYPE_OUTER, NID_pbe_WithSHA1And128BitRC2_CBC, - NID_rc2_cbc, NID_sha1, PKCS12_PBE_keyivgen, &PKCS12_PBE_keyivgen_ex}, - {EVP_PBE_TYPE_OUTER, NID_pbe_WithSHA1And40BitRC2_CBC, - NID_rc2_40_cbc, NID_sha1, PKCS12_PBE_keyivgen, &PKCS12_PBE_keyivgen_ex}, {EVP_PBE_TYPE_OUTER, NID_pbes2, -1, -1, PKCS5_v2_PBE_keyivgen, &PKCS5_v2_PBE_keyivgen_ex}, - {EVP_PBE_TYPE_OUTER, NID_pbeWithMD2AndRC2_CBC, - NID_rc2_64_cbc, NID_md2, PKCS5_PBE_keyivgen, PKCS5_PBE_keyivgen_ex}, - {EVP_PBE_TYPE_OUTER, NID_pbeWithMD5AndRC2_CBC, - NID_rc2_64_cbc, NID_md5, PKCS5_PBE_keyivgen, PKCS5_PBE_keyivgen_ex}, {EVP_PBE_TYPE_OUTER, NID_pbeWithSHA1AndDES_CBC, NID_des_cbc, NID_sha1, PKCS5_PBE_keyivgen, PKCS5_PBE_keyivgen_ex}, @@ -72,15 +60,6 @@ static const EVP_PBE_CTL builtin_pbe[] = { {EVP_PBE_TYPE_PRF, NID_hmacWithSHA256, -1, NID_sha256, 0}, {EVP_PBE_TYPE_PRF, NID_hmacWithSHA384, -1, NID_sha384, 0}, {EVP_PBE_TYPE_PRF, NID_hmacWithSHA512, -1, NID_sha512, 0}, - {EVP_PBE_TYPE_PRF, NID_id_HMACGostR3411_94, -1, NID_id_GostR3411_94, 0}, - {EVP_PBE_TYPE_PRF, NID_id_tc26_hmac_gost_3411_2012_256, -1, - NID_id_GostR3411_2012_256, 0}, - {EVP_PBE_TYPE_PRF, NID_id_tc26_hmac_gost_3411_2012_512, -1, - NID_id_GostR3411_2012_512, 0}, - {EVP_PBE_TYPE_PRF, NID_hmac_sha3_224, -1, NID_sha3_224, 0}, - {EVP_PBE_TYPE_PRF, NID_hmac_sha3_256, -1, NID_sha3_256, 0}, - {EVP_PBE_TYPE_PRF, NID_hmac_sha3_384, -1, NID_sha3_384, 0}, - {EVP_PBE_TYPE_PRF, NID_hmac_sha3_512, -1, NID_sha3_512, 0}, {EVP_PBE_TYPE_PRF, NID_hmacWithSHA512_224, -1, NID_sha512_224, 0}, {EVP_PBE_TYPE_PRF, NID_hmacWithSHA512_256, -1, NID_sha512_256, 0}, #ifndef OPENSSL_NO_SM3 @@ -143,7 +122,7 @@ int EVP_PBE_CipherInit_ex(ASN1_OBJECT *pbe_obj, const char *pass, int passlen, md = md_fetch = EVP_MD_fetch(libctx, OBJ_nid2sn(md_nid), propq); /* Fallback to legacy method */ if (md == NULL) - md = EVP_get_digestbynid(md_nid); + EVP_get_digestbynid(md_nid); if (md == NULL) { (void)ERR_clear_last_mark(); @@ -199,17 +178,15 @@ static int pbe_cmp(const EVP_PBE_CTL *const *a, const EVP_PBE_CTL *const *b) int EVP_PBE_alg_add_type(int pbe_type, int pbe_nid, int cipher_nid, int md_nid, EVP_PBE_KEYGEN *keygen) { - EVP_PBE_CTL *pbe_tmp = NULL; + EVP_PBE_CTL *pbe_tmp; if (pbe_algs == NULL) { pbe_algs = sk_EVP_PBE_CTL_new(pbe_cmp); - if (pbe_algs == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_CRYPTO_LIB); + if (pbe_algs == NULL) goto err; - } } - if ((pbe_tmp = OPENSSL_zalloc(sizeof(*pbe_tmp))) == NULL) + if ((pbe_tmp = OPENSSL_malloc(sizeof(*pbe_tmp))) == NULL) goto err; pbe_tmp->pbe_type = pbe_type; @@ -219,13 +196,13 @@ int EVP_PBE_alg_add_type(int pbe_type, int pbe_nid, int cipher_nid, pbe_tmp->keygen = keygen; if (!sk_EVP_PBE_CTL_push(pbe_algs, pbe_tmp)) { - ERR_raise(ERR_LIB_EVP, ERR_R_CRYPTO_LIB); + OPENSSL_free(pbe_tmp); goto err; } return 1; err: - OPENSSL_free(pbe_tmp); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } @@ -259,8 +236,6 @@ int EVP_PBE_find_ex(int type, int pbe_nid, int *pcnid, int *pmnid, pbelu.pbe_nid = pbe_nid; if (pbe_algs != NULL) { - /* Ideally, this would be done under lock */ - sk_EVP_PBE_CTL_sort(pbe_algs); i = sk_EVP_PBE_CTL_find(pbe_algs, &pbelu); pbetmp = sk_EVP_PBE_CTL_value(pbe_algs, i); } diff --git a/openssl/src/crypto/evp/evp_pkey.c b/openssl/src/crypto/evp/evp_pkey.c index a4505a9d0..8f3f15037 100644 --- a/openssl/src/crypto/evp/evp_pkey.c +++ b/openssl/src/crypto/evp/evp_pkey.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,7 +15,6 @@ #include #include #include "internal/provider.h" -#include "internal/sizes.h" #include "crypto/asn1.h" #include "crypto/evp.h" #include "crypto/x509.h" @@ -33,7 +32,7 @@ EVP_PKEY *evp_pkcs82pkey_legacy(const PKCS8_PRIV_KEY_INFO *p8, OSSL_LIB_CTX *lib return NULL; if ((pkey = EVP_PKEY_new()) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } @@ -74,13 +73,6 @@ EVP_PKEY *EVP_PKCS82PKEY_ex(const PKCS8_PRIV_KEY_INFO *p8, OSSL_LIB_CTX *libctx, int selection; size_t len; OSSL_DECODER_CTX *dctx = NULL; - const ASN1_OBJECT *algoid = NULL; - char keytype[OSSL_MAX_NAME_SIZE]; - - if (p8 == NULL - || !PKCS8_pkey_get0(&algoid, NULL, NULL, NULL, p8) - || !OBJ_obj2txt(keytype, sizeof(keytype), algoid, 0)) - return NULL; if ((encoded_len = i2d_PKCS8_PRIV_KEY_INFO(p8, &encoded_data)) <= 0 || encoded_data == NULL) @@ -90,20 +82,7 @@ EVP_PKEY *EVP_PKCS82PKEY_ex(const PKCS8_PRIV_KEY_INFO *p8, OSSL_LIB_CTX *libctx, len = encoded_len; selection = EVP_PKEY_KEYPAIR | EVP_PKEY_KEY_PARAMETERS; dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "DER", "PrivateKeyInfo", - keytype, selection, libctx, propq); - - if (dctx != NULL && OSSL_DECODER_CTX_get_num_decoders(dctx) == 0) { - OSSL_DECODER_CTX_free(dctx); - - /* - * This could happen if OBJ_obj2txt() returned a text OID and the - * decoder has not got that OID as an alias. We fall back to a NULL - * keytype - */ - dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "DER", "PrivateKeyInfo", - NULL, selection, libctx, propq); - } - + NULL, selection, libctx, propq); if (dctx == NULL || !OSSL_DECODER_from_data(dctx, &p8_data, &len)) /* try legacy */ @@ -151,7 +130,7 @@ PKCS8_PRIV_KEY_INFO *EVP_PKEY2PKCS8(const EVP_PKEY *pkey) } else { p8 = PKCS8_PRIV_KEY_INFO_new(); if (p8 == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/evp/evp_rand.c b/openssl/src/crypto/evp/evp_rand.c index 50334042a..0db755e06 100644 --- a/openssl/src/crypto/evp/evp_rand.c +++ b/openssl/src/crypto/evp/evp_rand.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -27,6 +27,7 @@ struct evp_rand_st { char *type_name; const char *description; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *refcnt_lock; const OSSL_DISPATCH *dispatch; OSSL_FUNC_rand_newctx_fn *newctx; @@ -46,8 +47,6 @@ struct evp_rand_st { OSSL_FUNC_rand_get_ctx_params_fn *get_ctx_params; OSSL_FUNC_rand_set_ctx_params_fn *set_ctx_params; OSSL_FUNC_rand_verify_zeroization_fn *verify_zeroization; - OSSL_FUNC_rand_get_seed_fn *get_seed; - OSSL_FUNC_rand_clear_seed_fn *clear_seed; } /* EVP_RAND */ ; static int evp_rand_up_ref(void *vrand) @@ -56,7 +55,7 @@ static int evp_rand_up_ref(void *vrand) int ref = 0; if (rand != NULL) - return CRYPTO_UP_REF(&rand->refcnt, &ref); + return CRYPTO_UP_REF(&rand->refcnt, &ref, rand->refcnt_lock); return 1; } @@ -67,12 +66,12 @@ static void evp_rand_free(void *vrand) if (rand == NULL) return; - CRYPTO_DOWN_REF(&rand->refcnt, &ref); + CRYPTO_DOWN_REF(&rand->refcnt, &ref, rand->refcnt_lock); if (ref > 0) return; OPENSSL_free(rand->type_name); ossl_provider_free(rand->prov); - CRYPTO_FREE_REF(&rand->refcnt); + CRYPTO_THREAD_lock_free(rand->refcnt_lock); OPENSSL_free(rand); } @@ -80,13 +79,12 @@ static void *evp_rand_new(void) { EVP_RAND *rand = OPENSSL_zalloc(sizeof(*rand)); - if (rand == NULL) - return NULL; - - if (!CRYPTO_NEW_REF(&rand->refcnt, 1)) { + if (rand == NULL + || (rand->refcnt_lock = CRYPTO_THREAD_lock_new()) == NULL) { OPENSSL_free(rand); return NULL; } + rand->refcnt = 1; return rand; } @@ -126,7 +124,7 @@ static void *evp_rand_from_algorithm(int name_id, #endif if ((rand = evp_rand_new()) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } rand->name_id = name_id; @@ -238,16 +236,6 @@ static void *evp_rand_from_algorithm(int name_id, fnzeroizecnt++; #endif break; - case OSSL_FUNC_RAND_GET_SEED: - if (rand->get_seed != NULL) - break; - rand->get_seed = OSSL_FUNC_rand_get_seed(fns); - break; - case OSSL_FUNC_RAND_CLEAR_SEED: - if (rand->clear_seed != NULL) - break; - rand->clear_seed = OSSL_FUNC_rand_clear_seed(fns); - break; } } /* @@ -317,7 +305,7 @@ const char *EVP_RAND_get0_description(const EVP_RAND *rand) int EVP_RAND_is_a(const EVP_RAND *rand, const char *name) { - return rand != NULL && evp_is_a(rand->prov, rand->name_id, NULL, name); + return evp_is_a(rand->prov, rand->name_id, NULL, name); } const OSSL_PROVIDER *EVP_RAND_get0_provider(const EVP_RAND *rand) @@ -332,11 +320,11 @@ int EVP_RAND_get_params(EVP_RAND *rand, OSSL_PARAM params[]) return 1; } -int EVP_RAND_CTX_up_ref(EVP_RAND_CTX *ctx) +static int evp_rand_ctx_up_ref(EVP_RAND_CTX *ctx) { int ref = 0; - return CRYPTO_UP_REF(&ctx->refcnt, &ref); + return CRYPTO_UP_REF(&ctx->refcnt, &ref, ctx->refcnt_lock); } EVP_RAND_CTX *EVP_RAND_CTX_new(EVP_RAND *rand, EVP_RAND_CTX *parent) @@ -351,16 +339,15 @@ EVP_RAND_CTX *EVP_RAND_CTX_new(EVP_RAND *rand, EVP_RAND_CTX *parent) } ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) - return NULL; - if (!CRYPTO_NEW_REF(&ctx->refcnt, 1)) { + if (ctx == NULL || (ctx->refcnt_lock = CRYPTO_THREAD_lock_new()) == NULL) { OPENSSL_free(ctx); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } if (parent != NULL) { - if (!EVP_RAND_CTX_up_ref(parent)) { + if (!evp_rand_ctx_up_ref(parent)) { ERR_raise(ERR_LIB_EVP, ERR_R_INTERNAL_ERROR); - CRYPTO_FREE_REF(&ctx->refcnt); + CRYPTO_THREAD_lock_free(ctx->refcnt_lock); OPENSSL_free(ctx); return NULL; } @@ -370,15 +357,16 @@ EVP_RAND_CTX *EVP_RAND_CTX_new(EVP_RAND *rand, EVP_RAND_CTX *parent) if ((ctx->algctx = rand->newctx(ossl_provider_ctx(rand->prov), parent_ctx, parent_dispatch)) == NULL || !EVP_RAND_up_ref(rand)) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); rand->freectx(ctx->algctx); - CRYPTO_FREE_REF(&ctx->refcnt); + CRYPTO_THREAD_lock_free(ctx->refcnt_lock); OPENSSL_free(ctx); EVP_RAND_CTX_free(parent); return NULL; } ctx->meth = rand; ctx->parent = parent; + ctx->refcnt = 1; return ctx; } @@ -390,14 +378,14 @@ void EVP_RAND_CTX_free(EVP_RAND_CTX *ctx) if (ctx == NULL) return; - CRYPTO_DOWN_REF(&ctx->refcnt, &ref); + CRYPTO_DOWN_REF(&ctx->refcnt, &ref, ctx->refcnt_lock); if (ref > 0) return; parent = ctx->parent; ctx->meth->freectx(ctx->algctx); ctx->algctx = NULL; EVP_RAND_free(ctx->meth); - CRYPTO_FREE_REF(&ctx->refcnt); + CRYPTO_THREAD_lock_free(ctx->refcnt_lock); OPENSSL_free(ctx); EVP_RAND_CTX_free(parent); } @@ -692,59 +680,3 @@ int EVP_RAND_verify_zeroization(EVP_RAND_CTX *ctx) evp_rand_unlock(ctx); return res; } - -int evp_rand_can_seed(EVP_RAND_CTX *ctx) -{ - return ctx->meth->get_seed != NULL; -} - -static size_t evp_rand_get_seed_locked(EVP_RAND_CTX *ctx, - unsigned char **buffer, - int entropy, - size_t min_len, size_t max_len, - int prediction_resistance, - const unsigned char *adin, - size_t adin_len) -{ - if (ctx->meth->get_seed != NULL) - return ctx->meth->get_seed(ctx->algctx, buffer, - entropy, min_len, max_len, - prediction_resistance, - adin, adin_len); - return 0; -} - -size_t evp_rand_get_seed(EVP_RAND_CTX *ctx, - unsigned char **buffer, - int entropy, size_t min_len, size_t max_len, - int prediction_resistance, - const unsigned char *adin, size_t adin_len) -{ - int res; - - if (!evp_rand_lock(ctx)) - return 0; - res = evp_rand_get_seed_locked(ctx, - buffer, - entropy, min_len, max_len, - prediction_resistance, - adin, adin_len); - evp_rand_unlock(ctx); - return res; -} - -static void evp_rand_clear_seed_locked(EVP_RAND_CTX *ctx, - unsigned char *buffer, size_t b_len) -{ - if (ctx->meth->clear_seed != NULL) - ctx->meth->clear_seed(ctx->algctx, buffer, b_len); -} - -void evp_rand_clear_seed(EVP_RAND_CTX *ctx, - unsigned char *buffer, size_t b_len) -{ - if (!evp_rand_lock(ctx)) - return; - evp_rand_clear_seed_locked(ctx, buffer, b_len); - evp_rand_unlock(ctx); -} diff --git a/openssl/src/crypto/evp/exchange.c b/openssl/src/crypto/evp/exchange.c index d9eed1cea..8eb13ad5d 100644 --- a/openssl/src/crypto/evp/exchange.c +++ b/openssl/src/crypto/evp/exchange.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,15 +22,20 @@ static EVP_KEYEXCH *evp_keyexch_new(OSSL_PROVIDER *prov) { EVP_KEYEXCH *exchange = OPENSSL_zalloc(sizeof(EVP_KEYEXCH)); - if (exchange == NULL) + if (exchange == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } - if (!CRYPTO_NEW_REF(&exchange->refcnt, 1)) { + exchange->lock = CRYPTO_THREAD_lock_new(); + if (exchange->lock == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); OPENSSL_free(exchange); return NULL; } exchange->prov = prov; ossl_provider_up_ref(prov); + exchange->refcnt = 1; return exchange; } @@ -44,7 +49,7 @@ static void *evp_keyexch_from_algorithm(int name_id, int fncnt = 0, sparamfncnt = 0, gparamfncnt = 0; if ((exchange = evp_keyexch_new(prov)) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } @@ -145,12 +150,12 @@ void EVP_KEYEXCH_free(EVP_KEYEXCH *exchange) if (exchange == NULL) return; - CRYPTO_DOWN_REF(&exchange->refcnt, &i); + CRYPTO_DOWN_REF(&exchange->refcnt, &i, exchange->lock); if (i > 0) return; OPENSSL_free(exchange->type_name); ossl_provider_free(exchange->prov); - CRYPTO_FREE_REF(&exchange->refcnt); + CRYPTO_THREAD_lock_free(exchange->lock); OPENSSL_free(exchange); } @@ -158,7 +163,7 @@ int EVP_KEYEXCH_up_ref(EVP_KEYEXCH *exchange) { int ref = 0; - CRYPTO_UP_REF(&exchange->refcnt, &ref); + CRYPTO_UP_REF(&exchange->refcnt, &ref, exchange->lock); return 1; } @@ -327,11 +332,7 @@ int EVP_PKEY_derive_init_ex(EVP_PKEY_CTX *ctx, const OSSL_PARAM params[]) /* No more legacy from here down to legacy: */ - /* A Coverity false positive with up_ref/down_ref and free */ - /* coverity[use_after_free] */ ctx->op.kex.exchange = exchange; - /* A Coverity false positive with up_ref/down_ref and free */ - /* coverity[deref_arg] */ ctx->op.kex.algctx = exchange->newctx(ossl_provider_ctx(exchange->prov)); if (ctx->op.kex.algctx == NULL) { /* The provider key can stay in the cache */ @@ -419,8 +420,6 @@ int EVP_PKEY_derive_set_peer_ex(EVP_PKEY_CTX *ctx, EVP_PKEY *peer, EVP_KEYMGMT_get0_name(ctx->keymgmt), ctx->propquery); if (tmp_keymgmt != NULL) - /* A Coverity issue with up_ref/down_ref and free */ - /* coverity[pass_freed_arg] */ provkey = evp_pkey_export_to_provider(peer, ctx->libctx, &tmp_keymgmt, ctx->propquery); EVP_KEYMGMT_free(tmp_keymgmt_tofree); @@ -551,8 +550,7 @@ const char *EVP_KEYEXCH_get0_description(const EVP_KEYEXCH *keyexch) int EVP_KEYEXCH_is_a(const EVP_KEYEXCH *keyexch, const char *name) { - return keyexch != NULL - && evp_is_a(keyexch->prov, keyexch->name_id, NULL, name); + return evp_is_a(keyexch->prov, keyexch->name_id, NULL, name); } void EVP_KEYEXCH_do_all_provided(OSSL_LIB_CTX *libctx, diff --git a/openssl/src/crypto/evp/kdf_lib.c b/openssl/src/crypto/evp/kdf_lib.c index 1093aac29..8177626ae 100644 --- a/openssl/src/crypto/evp/kdf_lib.c +++ b/openssl/src/crypto/evp/kdf_lib.c @@ -31,7 +31,7 @@ EVP_KDF_CTX *EVP_KDF_CTX_new(EVP_KDF *kdf) if (ctx == NULL || (ctx->algctx = kdf->newctx(ossl_provider_ctx(kdf->prov))) == NULL || !EVP_KDF_up_ref(kdf)) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); if (ctx != NULL) kdf->freectx(ctx->algctx); OPENSSL_free(ctx); @@ -60,12 +60,14 @@ EVP_KDF_CTX *EVP_KDF_CTX_dup(const EVP_KDF_CTX *src) return NULL; dst = OPENSSL_malloc(sizeof(*dst)); - if (dst == NULL) + if (dst == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } memcpy(dst, src, sizeof(*dst)); if (!EVP_KDF_up_ref(dst->meth)) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); OPENSSL_free(dst); return NULL; } @@ -95,7 +97,7 @@ const char *EVP_KDF_get0_description(const EVP_KDF *kdf) int EVP_KDF_is_a(const EVP_KDF *kdf, const char *name) { - return kdf != NULL && evp_is_a(kdf->prov, kdf->name_id, NULL, name); + return evp_is_a(kdf->prov, kdf->name_id, NULL, name); } const OSSL_PROVIDER *EVP_KDF_get0_provider(const EVP_KDF *kdf) diff --git a/openssl/src/crypto/evp/kdf_meth.c b/openssl/src/crypto/evp/kdf_meth.c index 5ee36b2b4..94af5d40a 100644 --- a/openssl/src/crypto/evp/kdf_meth.c +++ b/openssl/src/crypto/evp/kdf_meth.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,7 +22,7 @@ static int evp_kdf_up_ref(void *vkdf) EVP_KDF *kdf = (EVP_KDF *)vkdf; int ref = 0; - CRYPTO_UP_REF(&kdf->refcnt, &ref); + CRYPTO_UP_REF(&kdf->refcnt, &ref, kdf->lock); return 1; } @@ -34,12 +34,12 @@ static void evp_kdf_free(void *vkdf) if (kdf == NULL) return; - CRYPTO_DOWN_REF(&kdf->refcnt, &ref); + CRYPTO_DOWN_REF(&kdf->refcnt, &ref, kdf->lock); if (ref > 0) return; OPENSSL_free(kdf->type_name); ossl_provider_free(kdf->prov); - CRYPTO_FREE_REF(&kdf->refcnt); + CRYPTO_THREAD_lock_free(kdf->lock); OPENSSL_free(kdf); } @@ -48,10 +48,11 @@ static void *evp_kdf_new(void) EVP_KDF *kdf = NULL; if ((kdf = OPENSSL_zalloc(sizeof(*kdf))) == NULL - || !CRYPTO_NEW_REF(&kdf->refcnt, 1)) { + || (kdf->lock = CRYPTO_THREAD_lock_new()) == NULL) { OPENSSL_free(kdf); return NULL; } + kdf->refcnt = 1; return kdf; } @@ -64,7 +65,7 @@ static void *evp_kdf_from_algorithm(int name_id, int fnkdfcnt = 0, fnctxcnt = 0; if ((kdf = evp_kdf_new()) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } kdf->name_id = name_id; diff --git a/openssl/src/crypto/evp/kem.c b/openssl/src/crypto/evp/kem.c index f96012ccf..7594888b9 100644 --- a/openssl/src/crypto/evp/kem.c +++ b/openssl/src/crypto/evp/kem.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,13 +18,13 @@ #include "evp_local.h" static int evp_kem_init(EVP_PKEY_CTX *ctx, int operation, - const OSSL_PARAM params[], EVP_PKEY *authkey) + const OSSL_PARAM params[]) { int ret = 0; EVP_KEM *kem = NULL; EVP_KEYMGMT *tmp_keymgmt = NULL; const OSSL_PROVIDER *tmp_prov = NULL; - void *provkey = NULL, *provauthkey = NULL; + void *provkey = NULL; const char *supported_kem = NULL; int iter; @@ -40,10 +40,7 @@ static int evp_kem_init(EVP_PKEY_CTX *ctx, int operation, ERR_raise(ERR_LIB_EVP, EVP_R_NO_KEY_SET); goto err; } - if (authkey != NULL && authkey->type != ctx->pkey->type) { - ERR_raise(ERR_LIB_EVP, EVP_R_DIFFERENT_KEY_TYPES); - return 0; - } + /* * Try to derive the supported kem from |ctx->keymgmt|. */ @@ -117,26 +114,16 @@ static int evp_kem_init(EVP_PKEY_CTX *ctx, int operation, * same property query as when fetching the kem method. * With the keymgmt we found (if we did), we try to export |ctx->pkey| * to it (evp_pkey_export_to_provider() is smart enough to only actually + * export it if |tmp_keymgmt| is different from |ctx->pkey|'s keymgmt) */ tmp_keymgmt_tofree = tmp_keymgmt = evp_keymgmt_fetch_from_prov((OSSL_PROVIDER *)tmp_prov, EVP_KEYMGMT_get0_name(ctx->keymgmt), ctx->propquery); - if (tmp_keymgmt != NULL) { + if (tmp_keymgmt != NULL) provkey = evp_pkey_export_to_provider(ctx->pkey, ctx->libctx, &tmp_keymgmt, ctx->propquery); - if (provkey != NULL && authkey != NULL) { - provauthkey = evp_pkey_export_to_provider(authkey, ctx->libctx, - &tmp_keymgmt, - ctx->propquery); - if (provauthkey == NULL) { - EVP_KEM_free(kem); - ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); - goto err; - } - } - } if (tmp_keymgmt == NULL) EVP_KEYMGMT_free(tmp_keymgmt_tofree); } @@ -157,28 +144,20 @@ static int evp_kem_init(EVP_PKEY_CTX *ctx, int operation, switch (operation) { case EVP_PKEY_OP_ENCAPSULATE: - if (provauthkey != NULL && kem->auth_encapsulate_init != NULL) { - ret = kem->auth_encapsulate_init(ctx->op.encap.algctx, provkey, - provauthkey, params); - } else if (provauthkey == NULL && kem->encapsulate_init != NULL) { - ret = kem->encapsulate_init(ctx->op.encap.algctx, provkey, params); - } else { + if (kem->encapsulate_init == NULL) { ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); ret = -2; goto err; } + ret = kem->encapsulate_init(ctx->op.encap.algctx, provkey, params); break; case EVP_PKEY_OP_DECAPSULATE: - if (provauthkey != NULL && kem->auth_decapsulate_init != NULL) { - ret = kem->auth_decapsulate_init(ctx->op.encap.algctx, provkey, - provauthkey, params); - } else if (provauthkey == NULL && kem->encapsulate_init != NULL) { - ret = kem->decapsulate_init(ctx->op.encap.algctx, provkey, params); - } else { + if (kem->decapsulate_init == NULL) { ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); ret = -2; goto err; } + ret = kem->decapsulate_init(ctx->op.encap.algctx, provkey, params); break; default: ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); @@ -199,17 +178,9 @@ static int evp_kem_init(EVP_PKEY_CTX *ctx, int operation, return ret; } -int EVP_PKEY_auth_encapsulate_init(EVP_PKEY_CTX *ctx, EVP_PKEY *authpriv, - const OSSL_PARAM params[]) -{ - if (authpriv == NULL) - return 0; - return evp_kem_init(ctx, EVP_PKEY_OP_ENCAPSULATE, params, authpriv); -} - int EVP_PKEY_encapsulate_init(EVP_PKEY_CTX *ctx, const OSSL_PARAM params[]) { - return evp_kem_init(ctx, EVP_PKEY_OP_ENCAPSULATE, params, NULL); + return evp_kem_init(ctx, EVP_PKEY_OP_ENCAPSULATE, params); } int EVP_PKEY_encapsulate(EVP_PKEY_CTX *ctx, @@ -238,15 +209,7 @@ int EVP_PKEY_encapsulate(EVP_PKEY_CTX *ctx, int EVP_PKEY_decapsulate_init(EVP_PKEY_CTX *ctx, const OSSL_PARAM params[]) { - return evp_kem_init(ctx, EVP_PKEY_OP_DECAPSULATE, params, NULL); -} - -int EVP_PKEY_auth_decapsulate_init(EVP_PKEY_CTX *ctx, EVP_PKEY *authpub, - const OSSL_PARAM params[]) -{ - if (authpub == NULL) - return 0; - return evp_kem_init(ctx, EVP_PKEY_OP_DECAPSULATE, params, authpub); + return evp_kem_init(ctx, EVP_PKEY_OP_DECAPSULATE, params); } int EVP_PKEY_decapsulate(EVP_PKEY_CTX *ctx, @@ -275,15 +238,20 @@ static EVP_KEM *evp_kem_new(OSSL_PROVIDER *prov) { EVP_KEM *kem = OPENSSL_zalloc(sizeof(EVP_KEM)); - if (kem == NULL) + if (kem == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } - if (!CRYPTO_NEW_REF(&kem->refcnt, 1)) { + kem->lock = CRYPTO_THREAD_lock_new(); + if (kem->lock == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); OPENSSL_free(kem); return NULL; } kem->prov = prov; ossl_provider_up_ref(prov); + kem->refcnt = 1; return kem; } @@ -297,7 +265,7 @@ static void *evp_kem_from_algorithm(int name_id, const OSSL_ALGORITHM *algodef, int gparamfncnt = 0, sparamfncnt = 0; if ((kem = evp_kem_new(prov)) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } @@ -320,12 +288,6 @@ static void *evp_kem_from_algorithm(int name_id, const OSSL_ALGORITHM *algodef, kem->encapsulate_init = OSSL_FUNC_kem_encapsulate_init(fns); encfncnt++; break; - case OSSL_FUNC_KEM_AUTH_ENCAPSULATE_INIT: - if (kem->auth_encapsulate_init != NULL) - break; - kem->auth_encapsulate_init = OSSL_FUNC_kem_auth_encapsulate_init(fns); - encfncnt++; - break; case OSSL_FUNC_KEM_ENCAPSULATE: if (kem->encapsulate != NULL) break; @@ -338,12 +300,6 @@ static void *evp_kem_from_algorithm(int name_id, const OSSL_ALGORITHM *algodef, kem->decapsulate_init = OSSL_FUNC_kem_decapsulate_init(fns); decfncnt++; break; - case OSSL_FUNC_KEM_AUTH_DECAPSULATE_INIT: - if (kem->auth_decapsulate_init != NULL) - break; - kem->auth_decapsulate_init = OSSL_FUNC_kem_auth_decapsulate_init(fns); - decfncnt++; - break; case OSSL_FUNC_KEM_DECAPSULATE: if (kem->decapsulate != NULL) break; @@ -392,21 +348,19 @@ static void *evp_kem_from_algorithm(int name_id, const OSSL_ALGORITHM *algodef, } } if (ctxfncnt != 2 - || (encfncnt != 0 && encfncnt != 2 && encfncnt != 3) - || (decfncnt != 0 && decfncnt != 2 && decfncnt != 3) - || (encfncnt != decfncnt) + || (encfncnt != 0 && encfncnt != 2) + || (decfncnt != 0 && decfncnt != 2) + || (encfncnt != 2 && decfncnt != 2) || (gparamfncnt != 0 && gparamfncnt != 2) || (sparamfncnt != 0 && sparamfncnt != 2)) { /* * In order to be a consistent set of functions we must have at least - * a set of context functions (newctx and freectx) as well as a pair - * (or triplet) of "kem" functions: - * (encapsulate_init, (and/or auth_encapsulate_init), encapsulate) or - * (decapsulate_init, (and/or auth_decapsulate_init), decapsulate). - * set_ctx_params and settable_ctx_params are optional, but if one of - * them is present then the other one must also be present. The same - * applies to get_ctx_params and gettable_ctx_params. - * The dupctx function is optional. + * a set of context functions (newctx and freectx) as well as a pair of + * "kem" functions: (encapsulate_init, encapsulate) or + * (decapsulate_init, decapsulate). set_ctx_params and settable_ctx_params are + * optional, but if one of them is present then the other one must also + * be present. The same applies to get_ctx_params and + * gettable_ctx_params. The dupctx function is optional. */ ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_PROVIDER_FUNCTIONS); goto err; @@ -425,12 +379,12 @@ void EVP_KEM_free(EVP_KEM *kem) if (kem == NULL) return; - CRYPTO_DOWN_REF(&kem->refcnt, &i); + CRYPTO_DOWN_REF(&kem->refcnt, &i, kem->lock); if (i > 0) return; OPENSSL_free(kem->type_name); ossl_provider_free(kem->prov); - CRYPTO_FREE_REF(&kem->refcnt); + CRYPTO_THREAD_lock_free(kem->lock); OPENSSL_free(kem); } @@ -438,7 +392,7 @@ int EVP_KEM_up_ref(EVP_KEM *kem) { int ref = 0; - CRYPTO_UP_REF(&kem->refcnt, &ref); + CRYPTO_UP_REF(&kem->refcnt, &ref, kem->lock); return 1; } @@ -467,7 +421,7 @@ EVP_KEM *evp_kem_fetch_from_prov(OSSL_PROVIDER *prov, const char *algorithm, int EVP_KEM_is_a(const EVP_KEM *kem, const char *name) { - return kem != NULL && evp_is_a(kem->prov, kem->name_id, NULL, name); + return evp_is_a(kem->prov, kem->name_id, NULL, name); } int evp_kem_get_number(const EVP_KEM *kem) diff --git a/openssl/src/crypto/evp/keymgmt_lib.c b/openssl/src/crypto/evp/keymgmt_lib.c index 9ed0ba3ca..82aa771c3 100644 --- a/openssl/src/crypto/evp/keymgmt_lib.c +++ b/openssl/src/crypto/evp/keymgmt_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -34,7 +34,7 @@ int evp_keymgmt_util_try_import(const OSSL_PARAM params[], void *arg) /* Just in time creation of keydata */ if (data->keydata == NULL) { if ((data->keydata = evp_keymgmt_newdata(data->keymgmt)) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } delete_on_error = 1; @@ -93,8 +93,7 @@ int evp_keymgmt_util_export(const EVP_PKEY *pk, int selection, export_cb, export_cbarg); } -void *evp_keymgmt_util_export_to_provider(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, - int selection) +void *evp_keymgmt_util_export_to_provider(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt) { struct evp_keymgmt_util_try_import_data_st import_data; OP_CACHE_ELEM *op; @@ -128,7 +127,7 @@ void *evp_keymgmt_util_export_to_provider(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, */ if (pk->dirty_cnt == pk->dirty_cnt_copy) { /* If this key is already exported to |keymgmt|, no more to do */ - op = evp_keymgmt_util_find_operation_cache(pk, keymgmt, selection); + op = evp_keymgmt_util_find_operation_cache(pk, keymgmt); if (op != NULL && op->keymgmt != NULL) { void *ret = op->keydata; @@ -158,13 +157,13 @@ void *evp_keymgmt_util_export_to_provider(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, /* Setup for the export callback */ import_data.keydata = NULL; /* evp_keymgmt_util_try_import will create it */ import_data.keymgmt = keymgmt; - import_data.selection = selection; + import_data.selection = OSSL_KEYMGMT_SELECT_ALL; /* * The export function calls the callback (evp_keymgmt_util_try_import), * which does the import for us. If successful, we're done. */ - if (!evp_keymgmt_util_export(pk, selection, + if (!evp_keymgmt_util_export(pk, OSSL_KEYMGMT_SELECT_ALL, &evp_keymgmt_util_try_import, &import_data)) /* If there was an error, bail out */ return NULL; @@ -174,7 +173,7 @@ void *evp_keymgmt_util_export_to_provider(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, return NULL; } /* Check to make sure some other thread didn't get there first */ - op = evp_keymgmt_util_find_operation_cache(pk, keymgmt, selection); + op = evp_keymgmt_util_find_operation_cache(pk, keymgmt); if (op != NULL && op->keydata != NULL) { void *ret = op->keydata; @@ -194,11 +193,10 @@ void *evp_keymgmt_util_export_to_provider(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, * operation cache. In that case, we know that |i| is zero. */ if (pk->dirty_cnt != pk->dirty_cnt_copy) - evp_keymgmt_util_clear_operation_cache(pk); + evp_keymgmt_util_clear_operation_cache(pk, 0); /* Add the new export to the operation cache */ - if (!evp_keymgmt_util_cache_keydata(pk, keymgmt, import_data.keydata, - selection)) { + if (!evp_keymgmt_util_cache_keydata(pk, keymgmt, import_data.keydata)) { CRYPTO_THREAD_unlock(pk->lock); evp_keymgmt_freedata(keymgmt, import_data.keydata); return NULL; @@ -219,19 +217,22 @@ static void op_cache_free(OP_CACHE_ELEM *e) OPENSSL_free(e); } -int evp_keymgmt_util_clear_operation_cache(EVP_PKEY *pk) +int evp_keymgmt_util_clear_operation_cache(EVP_PKEY *pk, int locking) { if (pk != NULL) { + if (locking && pk->lock != NULL && !CRYPTO_THREAD_write_lock(pk->lock)) + return 0; sk_OP_CACHE_ELEM_pop_free(pk->operation_cache, op_cache_free); pk->operation_cache = NULL; + if (locking && pk->lock != NULL) + CRYPTO_THREAD_unlock(pk->lock); } return 1; } OP_CACHE_ELEM *evp_keymgmt_util_find_operation_cache(EVP_PKEY *pk, - EVP_KEYMGMT *keymgmt, - int selection) + EVP_KEYMGMT *keymgmt) { int i, end = sk_OP_CACHE_ELEM_num(pk->operation_cache); OP_CACHE_ELEM *p; @@ -239,22 +240,17 @@ OP_CACHE_ELEM *evp_keymgmt_util_find_operation_cache(EVP_PKEY *pk, /* * A comparison and sk_P_CACHE_ELEM_find() are avoided to not cause * problems when we've only a read lock. - * A keymgmt is a match if the |keymgmt| pointers are identical or if the - * provider and the name ID match */ for (i = 0; i < end; i++) { p = sk_OP_CACHE_ELEM_value(pk->operation_cache, i); - if ((p->selection & selection) == selection - && (keymgmt == p->keymgmt - || (keymgmt->name_id == p->keymgmt->name_id - && keymgmt->prov == p->keymgmt->prov))) + if (keymgmt == p->keymgmt) return p; } return NULL; } -int evp_keymgmt_util_cache_keydata(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, - void *keydata, int selection) +int evp_keymgmt_util_cache_keydata(EVP_PKEY *pk, + EVP_KEYMGMT *keymgmt, void *keydata) { OP_CACHE_ELEM *p = NULL; @@ -270,7 +266,6 @@ int evp_keymgmt_util_cache_keydata(EVP_PKEY *pk, EVP_KEYMGMT *keymgmt, return 0; p->keydata = keydata; p->keymgmt = keymgmt; - p->selection = selection; if (!EVP_KEYMGMT_up_ref(keymgmt)) { OPENSSL_free(p); @@ -396,8 +391,7 @@ int evp_keymgmt_util_match(EVP_PKEY *pk1, EVP_PKEY *pk2, int selection) ok = 1; if (keydata1 != NULL) { tmp_keydata = - evp_keymgmt_util_export_to_provider(pk1, keymgmt2, - selection); + evp_keymgmt_util_export_to_provider(pk1, keymgmt2); ok = (tmp_keydata != NULL); } if (ok) { @@ -417,8 +411,7 @@ int evp_keymgmt_util_match(EVP_PKEY *pk1, EVP_PKEY *pk2, int selection) ok = 1; if (keydata2 != NULL) { tmp_keydata = - evp_keymgmt_util_export_to_provider(pk2, keymgmt1, - selection); + evp_keymgmt_util_export_to_provider(pk2, keymgmt1); ok = (tmp_keydata != NULL); } if (ok) { diff --git a/openssl/src/crypto/evp/keymgmt_meth.c b/openssl/src/crypto/evp/keymgmt_meth.c index e3bec60ab..fb999c7fd 100644 --- a/openssl/src/crypto/evp/keymgmt_meth.c +++ b/openssl/src/crypto/evp/keymgmt_meth.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,34 +21,17 @@ static void *keymgmt_new(void) { EVP_KEYMGMT *keymgmt = NULL; - if ((keymgmt = OPENSSL_zalloc(sizeof(*keymgmt))) == NULL) - return NULL; - if (!CRYPTO_NEW_REF(&keymgmt->refcnt, 1)) { + if ((keymgmt = OPENSSL_zalloc(sizeof(*keymgmt))) == NULL + || (keymgmt->lock = CRYPTO_THREAD_lock_new()) == NULL) { EVP_KEYMGMT_free(keymgmt); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } - return keymgmt; -} - -#ifndef FIPS_MODULE -static void help_get_legacy_alg_type_from_keymgmt(const char *keytype, - void *arg) -{ - int *type = arg; - - if (*type == NID_undef) - *type = evp_pkey_name2type(keytype); -} -static int get_legacy_alg_type_from_keymgmt(const EVP_KEYMGMT *keymgmt) -{ - int type = NID_undef; + keymgmt->refcnt = 1; - EVP_KEYMGMT_names_do_all(keymgmt, help_get_legacy_alg_type_from_keymgmt, - &type); - return type; + return keymgmt; } -#endif static void *keymgmt_from_algorithm(int name_id, const OSSL_ALGORITHM *algodef, @@ -59,7 +42,6 @@ static void *keymgmt_from_algorithm(int name_id, int setparamfncnt = 0, getparamfncnt = 0; int setgenparamfncnt = 0; int importfncnt = 0, exportfncnt = 0; - int importtypesfncnt = 0, exporttypesfncnt = 0; if ((keymgmt = keymgmt_new()) == NULL) return NULL; @@ -171,20 +153,10 @@ static void *keymgmt_from_algorithm(int name_id, break; case OSSL_FUNC_KEYMGMT_IMPORT_TYPES: if (keymgmt->import_types == NULL) { - if (importtypesfncnt == 0) - importfncnt++; - importtypesfncnt++; + importfncnt++; keymgmt->import_types = OSSL_FUNC_keymgmt_import_types(fns); } break; - case OSSL_FUNC_KEYMGMT_IMPORT_TYPES_EX: - if (keymgmt->import_types_ex == NULL) { - if (importtypesfncnt == 0) - importfncnt++; - importtypesfncnt++; - keymgmt->import_types_ex = OSSL_FUNC_keymgmt_import_types_ex(fns); - } - break; case OSSL_FUNC_KEYMGMT_EXPORT: if (keymgmt->export == NULL) { exportfncnt++; @@ -193,20 +165,10 @@ static void *keymgmt_from_algorithm(int name_id, break; case OSSL_FUNC_KEYMGMT_EXPORT_TYPES: if (keymgmt->export_types == NULL) { - if (exporttypesfncnt == 0) - exportfncnt++; - exporttypesfncnt++; + exportfncnt++; keymgmt->export_types = OSSL_FUNC_keymgmt_export_types(fns); } break; - case OSSL_FUNC_KEYMGMT_EXPORT_TYPES_EX: - if (keymgmt->export_types_ex == NULL) { - if (exporttypesfncnt == 0) - exportfncnt++; - exporttypesfncnt++; - keymgmt->export_types_ex = OSSL_FUNC_keymgmt_export_types_ex(fns); - } - break; } } /* @@ -238,13 +200,19 @@ static void *keymgmt_from_algorithm(int name_id, if (prov != NULL) ossl_provider_up_ref(prov); -#ifndef FIPS_MODULE - keymgmt->legacy_alg = get_legacy_alg_type_from_keymgmt(keymgmt); -#endif - return keymgmt; } +EVP_KEYMGMT *evp_keymgmt_fetch_by_number(OSSL_LIB_CTX *ctx, int name_id, + const char *properties) +{ + return evp_generic_fetch_by_number(ctx, + OSSL_OP_KEYMGMT, name_id, properties, + keymgmt_from_algorithm, + (int (*)(void *))EVP_KEYMGMT_up_ref, + (void (*)(void *))EVP_KEYMGMT_free); +} + EVP_KEYMGMT *evp_keymgmt_fetch_from_prov(OSSL_PROVIDER *prov, const char *name, const char *properties) @@ -269,7 +237,7 @@ int EVP_KEYMGMT_up_ref(EVP_KEYMGMT *keymgmt) { int ref = 0; - CRYPTO_UP_REF(&keymgmt->refcnt, &ref); + CRYPTO_UP_REF(&keymgmt->refcnt, &ref, keymgmt->lock); return 1; } @@ -280,12 +248,12 @@ void EVP_KEYMGMT_free(EVP_KEYMGMT *keymgmt) if (keymgmt == NULL) return; - CRYPTO_DOWN_REF(&keymgmt->refcnt, &ref); + CRYPTO_DOWN_REF(&keymgmt->refcnt, &ref, keymgmt->lock); if (ref > 0) return; OPENSSL_free(keymgmt->type_name); ossl_provider_free(keymgmt->prov); - CRYPTO_FREE_REF(&keymgmt->refcnt); + CRYPTO_THREAD_lock_free(keymgmt->lock); OPENSSL_free(keymgmt); } @@ -299,11 +267,6 @@ int evp_keymgmt_get_number(const EVP_KEYMGMT *keymgmt) return keymgmt->name_id; } -int evp_keymgmt_get_legacy_alg(const EVP_KEYMGMT *keymgmt) -{ - return keymgmt->legacy_alg; -} - const char *EVP_KEYMGMT_get0_description(const EVP_KEYMGMT *keymgmt) { return keymgmt->description; @@ -316,8 +279,7 @@ const char *EVP_KEYMGMT_get0_name(const EVP_KEYMGMT *keymgmt) int EVP_KEYMGMT_is_a(const EVP_KEYMGMT *keymgmt, const char *name) { - return keymgmt != NULL - && evp_is_a(keymgmt->prov, keymgmt->name_id, NULL, name); + return evp_is_a(keymgmt->prov, keymgmt->name_id, NULL, name); } void EVP_KEYMGMT_do_all_provided(OSSL_LIB_CTX *libctx, @@ -375,7 +337,7 @@ void *evp_keymgmt_gen_init(const EVP_KEYMGMT *keymgmt, int selection, } int evp_keymgmt_gen_set_template(const EVP_KEYMGMT *keymgmt, void *genctx, - void *templ) + void *template) { /* * It's arguable if we actually should return success in this case, as @@ -385,7 +347,7 @@ int evp_keymgmt_gen_set_template(const EVP_KEYMGMT *keymgmt, void *genctx, */ if (keymgmt->gen_set_template == NULL) return 1; - return keymgmt->gen_set_template(genctx, templ); + return keymgmt->gen_set_template(genctx, template); } int evp_keymgmt_gen_set_params(const EVP_KEYMGMT *keymgmt, void *genctx, @@ -415,7 +377,7 @@ void *evp_keymgmt_gen(const EVP_KEYMGMT *keymgmt, void *genctx, void evp_keymgmt_gen_cleanup(const EVP_KEYMGMT *keymgmt, void *genctx) { - if (keymgmt->gen_cleanup != NULL) + if (keymgmt->gen != NULL) keymgmt->gen_cleanup(genctx); } @@ -502,10 +464,6 @@ int evp_keymgmt_import(const EVP_KEYMGMT *keymgmt, void *keydata, const OSSL_PARAM *evp_keymgmt_import_types(const EVP_KEYMGMT *keymgmt, int selection) { - void *provctx = ossl_provider_ctx(EVP_KEYMGMT_get0_provider(keymgmt)); - - if (keymgmt->import_types_ex != NULL) - return keymgmt->import_types_ex(provctx, selection); if (keymgmt->import_types == NULL) return NULL; return keymgmt->import_types(selection); @@ -522,10 +480,6 @@ int evp_keymgmt_export(const EVP_KEYMGMT *keymgmt, void *keydata, const OSSL_PARAM *evp_keymgmt_export_types(const EVP_KEYMGMT *keymgmt, int selection) { - void *provctx = ossl_provider_ctx(EVP_KEYMGMT_get0_provider(keymgmt)); - - if (keymgmt->export_types_ex != NULL) - return keymgmt->export_types_ex(provctx, selection); if (keymgmt->export_types == NULL) return NULL; return keymgmt->export_types(selection); diff --git a/openssl/src/crypto/evp/legacy_blake2.c b/openssl/src/crypto/evp/legacy_blake2.c deleted file mode 100644 index 5a5f05044..000000000 --- a/openssl/src/crypto/evp/legacy_blake2.c +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "crypto/evp.h" -#include "prov/blake2.h" /* diverse BLAKE2 macros */ -#include "legacy_meth.h" - -/* - * Local hack to adapt the BLAKE2 init functions to what the - * legacy function signatures demand. - */ -static int blake2s_init(BLAKE2S_CTX *C) -{ - BLAKE2S_PARAM P; - - ossl_blake2s_param_init(&P); - return ossl_blake2s_init(C, &P); -} -static int blake2b_init(BLAKE2B_CTX *C) -{ - BLAKE2B_PARAM P; - - ossl_blake2b_param_init(&P); - return ossl_blake2b_init(C, &P); -} -#define blake2s_update ossl_blake2s_update -#define blake2b_update ossl_blake2b_update -#define blake2s_final ossl_blake2s_final -#define blake2b_final ossl_blake2b_final - -IMPLEMENT_LEGACY_EVP_MD_METH_LC(blake2s_int, blake2s) -IMPLEMENT_LEGACY_EVP_MD_METH_LC(blake2b_int, blake2b) - -static const EVP_MD blake2b_md = { - NID_blake2b512, - 0, - BLAKE2B_DIGEST_LENGTH, - 0, - EVP_ORIG_GLOBAL, - LEGACY_EVP_MD_METH_TABLE(blake2b_int_init, blake2b_int_update, - blake2b_int_final, NULL, BLAKE2B_BLOCKBYTES), -}; - -const EVP_MD *EVP_blake2b512(void) -{ - return &blake2b_md; -} - -static const EVP_MD blake2s_md = { - NID_blake2s256, - 0, - BLAKE2S_DIGEST_LENGTH, - 0, - EVP_ORIG_GLOBAL, - LEGACY_EVP_MD_METH_TABLE(blake2s_int_init, blake2s_int_update, - blake2s_int_final, NULL, BLAKE2S_BLOCKBYTES), -}; - -const EVP_MD *EVP_blake2s256(void) -{ - return &blake2s_md; -} diff --git a/openssl/src/crypto/evp/legacy_md2.c b/openssl/src/crypto/evp/legacy_md2.c deleted file mode 100644 index 72cc99ad7..000000000 --- a/openssl/src/crypto/evp/legacy_md2.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD2 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "crypto/evp.h" -#include "legacy_meth.h" - -IMPLEMENT_LEGACY_EVP_MD_METH(md2, MD2) - -static const EVP_MD md2_md = { - NID_md2, - NID_md2WithRSAEncryption, - MD2_DIGEST_LENGTH, - 0, - EVP_ORIG_GLOBAL, - LEGACY_EVP_MD_METH_TABLE(md2_init, md2_update, md2_final, NULL, MD2_BLOCK) -}; - -const EVP_MD *EVP_md2(void) -{ - return &md2_md; -} diff --git a/openssl/src/crypto/evp/legacy_md4.c b/openssl/src/crypto/evp/legacy_md4.c deleted file mode 100644 index 4bc852b52..000000000 --- a/openssl/src/crypto/evp/legacy_md4.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD4 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "crypto/evp.h" -#include "legacy_meth.h" - -IMPLEMENT_LEGACY_EVP_MD_METH(md4, MD4) - -static const EVP_MD md4_md = { - NID_md4, - NID_md4WithRSAEncryption, - MD4_DIGEST_LENGTH, - 0, - EVP_ORIG_GLOBAL, - LEGACY_EVP_MD_METH_TABLE(md4_init, md4_update, md4_final, NULL, MD4_CBLOCK), -}; - -const EVP_MD *EVP_md4(void) -{ - return &md4_md; -} diff --git a/openssl/src/crypto/evp/legacy_mdc2.c b/openssl/src/crypto/evp/legacy_mdc2.c deleted file mode 100644 index 317d87c61..000000000 --- a/openssl/src/crypto/evp/legacy_mdc2.c +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MDC2 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "crypto/evp.h" -#include "legacy_meth.h" - -IMPLEMENT_LEGACY_EVP_MD_METH(mdc2, MDC2) - -static const EVP_MD mdc2_md = { - NID_mdc2, - NID_mdc2WithRSA, - MDC2_DIGEST_LENGTH, - 0, - EVP_ORIG_GLOBAL, - LEGACY_EVP_MD_METH_TABLE(mdc2_init, mdc2_update, mdc2_final, NULL, - MDC2_BLOCK), -}; - -const EVP_MD *EVP_mdc2(void) -{ - return &mdc2_md; -} diff --git a/openssl/src/crypto/evp/legacy_ripemd.c b/openssl/src/crypto/evp/legacy_ripemd.c deleted file mode 100644 index 1fa1ebc04..000000000 --- a/openssl/src/crypto/evp/legacy_ripemd.c +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RIPEMD160 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "crypto/evp.h" -#include "legacy_meth.h" - -IMPLEMENT_LEGACY_EVP_MD_METH(ripe, RIPEMD160) - -static const EVP_MD ripemd160_md = { - NID_ripemd160, - NID_ripemd160WithRSA, - RIPEMD160_DIGEST_LENGTH, - 0, - EVP_ORIG_GLOBAL, - LEGACY_EVP_MD_METH_TABLE(ripe_init, ripe_update, ripe_final, NULL, - RIPEMD160_CBLOCK), -}; - -const EVP_MD *EVP_ripemd160(void) -{ - return &ripemd160_md; -} diff --git a/openssl/src/crypto/evp/legacy_sha.c b/openssl/src/crypto/evp/legacy_sha.c index 38423ff54..3859286ee 100644 --- a/openssl/src/crypto/evp/legacy_sha.c +++ b/openssl/src/crypto/evp/legacy_sha.c @@ -37,8 +37,7 @@ static int nm##_update(EVP_MD_CTX *ctx, const void *data, size_t count) \ } \ static int nm##_final(EVP_MD_CTX *ctx, unsigned char *md) \ { \ - KECCAK1600_CTX *kctx = EVP_MD_CTX_get0_md_data(ctx); \ - return fn##_final(kctx, md, kctx->md_size); \ + return fn##_final(md, EVP_MD_CTX_get0_md_data(ctx)); \ } #define IMPLEMENT_LEGACY_EVP_MD_METH_SHAKE(nm, fn, tag) \ static int nm##_init(EVP_MD_CTX *ctx) \ @@ -72,11 +71,7 @@ static int sha1_int_ctrl(EVP_MD_CTX *ctx, int cmd, int p1, void *p2) static int shake_ctrl(EVP_MD_CTX *evp_ctx, int cmd, int p1, void *p2) { - KECCAK1600_CTX *ctx; - - if (evp_ctx == NULL) - return 0; - ctx = evp_ctx->md_data; + KECCAK1600_CTX *ctx = evp_ctx->md_data; switch (cmd) { case EVP_MD_CTRL_XOF_LEN: diff --git a/openssl/src/crypto/evp/legacy_wp.c b/openssl/src/crypto/evp/legacy_wp.c deleted file mode 100644 index 3976ff73f..000000000 --- a/openssl/src/crypto/evp/legacy_wp.c +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Whirlpool low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "crypto/evp.h" -#include "legacy_meth.h" - -IMPLEMENT_LEGACY_EVP_MD_METH(wp, WHIRLPOOL) - -static const EVP_MD whirlpool_md = { - NID_whirlpool, - 0, - WHIRLPOOL_DIGEST_LENGTH, - 0, - EVP_ORIG_GLOBAL, - LEGACY_EVP_MD_METH_TABLE(wp_init, wp_update, wp_final, NULL, - WHIRLPOOL_BBLOCK / 8), -}; - -const EVP_MD *EVP_whirlpool(void) -{ - return &whirlpool_md; -} diff --git a/openssl/src/crypto/evp/local.h b/openssl/src/crypto/evp/local.h new file mode 100644 index 000000000..315e3b690 --- /dev/null +++ b/openssl/src/crypto/evp/local.h @@ -0,0 +1,212 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +int vpaes_set_encrypt_key(const unsigned char *userKey, int bits, + AES_KEY *key); +int vpaes_set_decrypt_key(const unsigned char *userKey, int bits, + AES_KEY *key); + +void vpaes_encrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void vpaes_decrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); + +void vpaes_cbc_encrypt(const unsigned char *in, + unsigned char *out, + size_t length, + const AES_KEY *key, unsigned char *ivec, int enc); + +void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t length, const AES_KEY *key, + unsigned char ivec[16], int enc); +void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + const unsigned char ivec[16]); +void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out, + size_t len, const AES_KEY *key1, + const AES_KEY *key2, const unsigned char iv[16]); +void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out, + size_t len, const AES_KEY *key1, + const AES_KEY *key2, const unsigned char iv[16]); + +void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, + const unsigned char ivec[AES_BLOCK_SIZE]); + +void AES_xts_encrypt(const unsigned char *inp, unsigned char *out, size_t len, + const AES_KEY *key1, const AES_KEY *key2, + const unsigned char iv[16]); +void AES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len, + const AES_KEY *key1, const AES_KEY *key2, + const unsigned char iv[16]); + +int aesni_set_encrypt_key(const unsigned char *userKey, int bits, + AES_KEY *key); +int aesni_set_decrypt_key(const unsigned char *userKey, int bits, + AES_KEY *key); + +void aesni_encrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void aesni_decrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); + +void aesni_ecb_encrypt(const unsigned char *in, + unsigned char *out, + size_t length, const AES_KEY *key, int enc); +void aesni_cbc_encrypt(const unsigned char *in, + unsigned char *out, + size_t length, + const AES_KEY *key, unsigned char *ivec, int enc); + +void aesni_ocb_encrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const void *key, + size_t start_block_num, + unsigned char offset_i[16], + const unsigned char L_[][16], + unsigned char checksum[16]); +void aesni_ocb_decrypt(const unsigned char *in, unsigned char *out, + size_t blocks, const void *key, + size_t start_block_num, + unsigned char offset_i[16], + const unsigned char L_[][16], + unsigned char checksum[16]); + +void aesni_ctr32_encrypt_blocks(const unsigned char *in, + unsigned char *out, + size_t blocks, + const void *key, const unsigned char *ivec); + +void aesni_xts_encrypt(const unsigned char *in, + unsigned char *out, + size_t length, + const AES_KEY *key1, const AES_KEY *key2, + const unsigned char iv[16]); + +void aesni_xts_decrypt(const unsigned char *in, + unsigned char *out, + size_t length, + const AES_KEY *key1, const AES_KEY *key2, + const unsigned char iv[16]); + +void aesni_ccm64_encrypt_blocks(const unsigned char *in, + unsigned char *out, + size_t blocks, + const void *key, + const unsigned char ivec[16], + unsigned char cmac[16]); + +void aesni_ccm64_decrypt_blocks(const unsigned char *in, + unsigned char *out, + size_t blocks, + const void *key, + const unsigned char ivec[16], + unsigned char cmac[16]); + +size_t aesni_gcm_encrypt(const unsigned char *in, + unsigned char *out, + size_t len, + const void *key, unsigned char ivec[16], u64 *Xi); +size_t aesni_gcm_decrypt(const unsigned char *in, + unsigned char *out, + size_t len, + const void *key, unsigned char ivec[16], u64 *Xi); +void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in, + size_t len); + +size_t aes_gcm_enc_128_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext, + uint64_t *Xi, unsigned char ivec[16], const void *key); +size_t aes_gcm_enc_192_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext, + uint64_t *Xi, unsigned char ivec[16], const void *key); +size_t aes_gcm_enc_256_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext, + uint64_t *Xi, unsigned char ivec[16], const void *key); +size_t aes_gcm_dec_128_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext, + uint64_t *Xi, unsigned char ivec[16], const void *key); +size_t aes_gcm_dec_192_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext, + uint64_t *Xi, unsigned char ivec[16], const void *key); +size_t aes_gcm_dec_256_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext, + uint64_t *Xi, unsigned char ivec[16], const void *key); +void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len); + +int aes_v8_set_encrypt_key(const unsigned char *userKey, const int bits, + AES_KEY *key); +int aes_v8_set_decrypt_key(const unsigned char *userKey, const int bits, + AES_KEY *key); +void aes_v8_encrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void aes_v8_decrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void aes_v8_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t length, const AES_KEY *key, + unsigned char *ivec, const int enc); +void aes_v8_ecb_encrypt(const unsigned char *in, unsigned char *out, + size_t length, const AES_KEY *key, + const int enc); +void aes_v8_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t len, const void *key, + const unsigned char ivec[16]); +void aes_v8_xts_encrypt(const unsigned char *inp, unsigned char *out, + size_t len, const AES_KEY *key1, + const AES_KEY *key2, const unsigned char iv[16]); +void aes_v8_xts_decrypt(const unsigned char *inp, unsigned char *out, + size_t len, const AES_KEY *key1, + const AES_KEY *key2, const unsigned char iv[16]); + +void sha1_multi_block(SHA1_MB_CTX *, const HASH_DESC *, int); +void aesni_multi_cbc_encrypt(CIPH_DESC *, void *, int); + +void sha256_multi_block(SHA256_MB_CTX *, const HASH_DESC *, int); +void aesni_multi_cbc_encrypt(CIPH_DESC *, void *, int); + +void *xor128_encrypt_n_pad(void *out, const void *inp, void *otp, size_t len); +void *xor128_decrypt_n_pad(void *out, const void *inp, void *otp, size_t len); + +void aesni_cbc_encrypt(const unsigned char *in, + unsigned char *out, + size_t length, + const AES_KEY *key, unsigned char *ivec, int enc); + +int aesni_cbc_sha256_enc(const void *inp, void *out, size_t blocks, + const AES_KEY *key, unsigned char iv[16], + SHA256_CTX *ctx, const void *in0); + +void aesni_cbc_sha1_enc(const void *inp, void *out, size_t blocks, + const AES_KEY *key, unsigned char iv[16], + SHA_CTX *ctx, const void *in0); + +void aesni256_cbc_sha1_dec(const void *inp, void *out, size_t blocks, + const AES_KEY *key, unsigned char iv[16], + SHA_CTX *ctx, const void *in0); + +void rc4_md5_enc(RC4_KEY *key, const void *in0, void *out, + MD5_CTX *ctx, const void *inp, size_t blocks); + +int sm4_v8_set_encrypt_key(const unsigned char *userKey, SM4_KEY *key); +int sm4_v8_set_decrypt_key(const unsigned char *userKey, SM4_KEY *key); +void sm4_v8_encrypt(const unsigned char *in, unsigned char *out, + const SM4_KEY *key); +void sm4_v8_decrypt(const unsigned char *in, unsigned char *out, + const SM4_KEY *key); +void sm4_v8_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t length, const SM4_KEY *key, + unsigned char *ivec, const int enc); +void sm4_v8_ecb_encrypt(const unsigned char *in, unsigned char *out, + size_t length, const SM4_KEY *key, + const int enc); +void sm4_v8_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t len, const void *key, + const unsigned char ivec[16]); + +size_t SHA3_absorb(uint64_t A[5][5], const unsigned char *inp, size_t len, + size_t r); +void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r); + diff --git a/openssl/src/crypto/evp/m_sigver.c b/openssl/src/crypto/evp/m_sigver.c index 3a979f4bd..76a6814b4 100644 --- a/openssl/src/crypto/evp/m_sigver.c +++ b/openssl/src/crypto/evp/m_sigver.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -51,8 +51,15 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, void *provkey = NULL; int ret, iter, reinit = 1; - if (!evp_md_ctx_free_algctx(ctx)) - return 0; + if (ctx->algctx != NULL) { + if (!ossl_assert(ctx->digest != NULL)) { + ERR_raise(ERR_LIB_EVP, EVP_R_INITIALIZATION_ERROR); + return 0; + } + if (ctx->digest->freectx != NULL) + ctx->digest->freectx(ctx->algctx); + ctx->algctx = NULL; + } if (ctx->pctx == NULL) { reinit = 0; @@ -64,8 +71,6 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, if (ctx->pctx == NULL) return 0; - EVP_MD_CTX_clear_flags(ctx, EVP_MD_CTX_FLAG_FINALISED); - locpctx = ctx->pctx; ERR_set_mark(); @@ -234,7 +239,7 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, * This might be requested by a later call to EVP_MD_CTX_get0_md(). * In that case the "explicit fetch" rules apply for that * function (as per man pages), i.e. the ref count is not updated - * so the EVP_MD should not be used beyond the lifetime of the + * so the EVP_MD should not be used beyound the lifetime of the * EVP_MD_CTX. */ ctx->fetched_digest = EVP_MD_fetch(locpctx->libctx, mdname, props); @@ -403,11 +408,6 @@ int EVP_DigestSignUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize) { EVP_PKEY_CTX *pctx = ctx->pctx; - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR); - return 0; - } - if (pctx == NULL || pctx->operation != EVP_PKEY_OP_SIGNCTX || pctx->op.sig.algctx == NULL @@ -438,11 +438,6 @@ int EVP_DigestVerifyUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize) { EVP_PKEY_CTX *pctx = ctx->pctx; - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR); - return 0; - } - if (pctx == NULL || pctx->operation != EVP_PKEY_OP_VERIFYCTX || pctx->op.sig.algctx == NULL @@ -474,12 +469,7 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret, size_t *siglen) { int sctx = 0, r = 0; - EVP_PKEY_CTX *dctx = NULL, *pctx = ctx->pctx; - - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_FINAL_ERROR); - return 0; - } + EVP_PKEY_CTX *dctx, *pctx = ctx->pctx; if (pctx == NULL || pctx->operation != EVP_PKEY_OP_SIGNCTX @@ -487,19 +477,18 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret, || pctx->op.sig.signature == NULL) goto legacy; - if (sigret != NULL && (ctx->flags & EVP_MD_CTX_FLAG_FINALISE) == 0) { - /* try dup */ - dctx = EVP_PKEY_CTX_dup(pctx); - if (dctx != NULL) - pctx = dctx; - } - r = pctx->op.sig.signature->digest_sign_final(pctx->op.sig.algctx, + if (sigret == NULL || (ctx->flags & EVP_MD_CTX_FLAG_FINALISE) != 0) + return pctx->op.sig.signature->digest_sign_final(pctx->op.sig.algctx, + sigret, siglen, + sigret == NULL ? 0 : *siglen); + dctx = EVP_PKEY_CTX_dup(pctx); + if (dctx == NULL) + return 0; + + r = dctx->op.sig.signature->digest_sign_final(dctx->op.sig.algctx, sigret, siglen, - sigret == NULL ? 0 : *siglen); - if (dctx == NULL && sigret != NULL) - ctx->flags |= EVP_MD_CTX_FLAG_FINALISED; - else - EVP_PKEY_CTX_free(dctx); + *siglen); + EVP_PKEY_CTX_free(dctx); return r; legacy: @@ -517,10 +506,9 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret, if (pctx->pmeth->flags & EVP_PKEY_FLAG_SIGCTX_CUSTOM) { if (sigret == NULL) return pctx->pmeth->signctx(pctx, sigret, siglen, ctx); - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISE) != 0) { + if (ctx->flags & EVP_MD_CTX_FLAG_FINALISE) r = pctx->pmeth->signctx(pctx, sigret, siglen, ctx); - ctx->flags |= EVP_MD_CTX_FLAG_FINALISED; - } else { + else { dctx = EVP_PKEY_CTX_dup(pctx); if (dctx == NULL) return 0; @@ -581,23 +569,15 @@ int EVP_DigestSign(EVP_MD_CTX *ctx, unsigned char *sigret, size_t *siglen, { EVP_PKEY_CTX *pctx = ctx->pctx; - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_FINAL_ERROR); - return 0; - } - if (pctx != NULL && pctx->operation == EVP_PKEY_OP_SIGNCTX && pctx->op.sig.algctx != NULL && pctx->op.sig.signature != NULL) { - if (pctx->op.sig.signature->digest_sign != NULL) { - if (sigret != NULL) - ctx->flags |= EVP_MD_CTX_FLAG_FINALISED; + if (pctx->op.sig.signature->digest_sign != NULL) return pctx->op.sig.signature->digest_sign(pctx->op.sig.algctx, sigret, siglen, sigret == NULL ? 0 : *siglen, tbs, tbslen); - } } else { /* legacy */ if (ctx->pctx->pmeth != NULL && ctx->pctx->pmeth->digestsign != NULL) @@ -616,12 +596,7 @@ int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sig, int r = 0; unsigned int mdlen = 0; int vctx = 0; - EVP_PKEY_CTX *dctx = NULL, *pctx = ctx->pctx; - - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_FINAL_ERROR); - return 0; - } + EVP_PKEY_CTX *dctx, *pctx = ctx->pctx; if (pctx == NULL || pctx->operation != EVP_PKEY_OP_VERIFYCTX @@ -629,18 +604,16 @@ int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sig, || pctx->op.sig.signature == NULL) goto legacy; - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISE) == 0) { - /* try dup */ - dctx = EVP_PKEY_CTX_dup(pctx); - if (dctx != NULL) - pctx = dctx; - } - r = pctx->op.sig.signature->digest_verify_final(pctx->op.sig.algctx, - sig, siglen); + if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISE) != 0) + return pctx->op.sig.signature->digest_verify_final(pctx->op.sig.algctx, + sig, siglen); + dctx = EVP_PKEY_CTX_dup(pctx); if (dctx == NULL) - ctx->flags |= EVP_MD_CTX_FLAG_FINALISED; - else - EVP_PKEY_CTX_free(dctx); + return 0; + + r = dctx->op.sig.signature->digest_verify_final(dctx->op.sig.algctx, + sig, siglen); + EVP_PKEY_CTX_free(dctx); return r; legacy: @@ -660,10 +633,9 @@ int EVP_DigestVerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sig, else vctx = 0; if (ctx->flags & EVP_MD_CTX_FLAG_FINALISE) { - if (vctx) { + if (vctx) r = pctx->pmeth->verifyctx(pctx, sig, siglen, ctx); - ctx->flags |= EVP_MD_CTX_FLAG_FINALISED; - } else + else r = EVP_DigestFinal_ex(ctx, md, &mdlen); } else { EVP_MD_CTX *tmp_ctx = EVP_MD_CTX_new(); @@ -690,21 +662,14 @@ int EVP_DigestVerify(EVP_MD_CTX *ctx, const unsigned char *sigret, { EVP_PKEY_CTX *pctx = ctx->pctx; - if ((ctx->flags & EVP_MD_CTX_FLAG_FINALISED) != 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_FINAL_ERROR); - return 0; - } - if (pctx != NULL && pctx->operation == EVP_PKEY_OP_VERIFYCTX && pctx->op.sig.algctx != NULL && pctx->op.sig.signature != NULL) { - if (pctx->op.sig.signature->digest_verify != NULL) { - ctx->flags |= EVP_MD_CTX_FLAG_FINALISED; + if (pctx->op.sig.signature->digest_verify != NULL) return pctx->op.sig.signature->digest_verify(pctx->op.sig.algctx, sigret, siglen, tbs, tbslen); - } } else { /* legacy */ if (ctx->pctx->pmeth != NULL && ctx->pctx->pmeth->digestverify != NULL) diff --git a/openssl/src/crypto/evp/mac_lib.c b/openssl/src/crypto/evp/mac_lib.c index c6b021fcd..24fdb35c8 100644 --- a/openssl/src/crypto/evp/mac_lib.c +++ b/openssl/src/crypto/evp/mac_lib.c @@ -23,15 +23,16 @@ EVP_MAC_CTX *EVP_MAC_CTX_new(EVP_MAC *mac) { EVP_MAC_CTX *ctx = OPENSSL_zalloc(sizeof(EVP_MAC_CTX)); - if (ctx != NULL) { - ctx->meth = mac; - if ((ctx->algctx = mac->newctx(ossl_provider_ctx(mac->prov))) == NULL - || !EVP_MAC_up_ref(mac)) { + if (ctx == NULL + || (ctx->algctx = mac->newctx(ossl_provider_ctx(mac->prov))) == NULL + || !EVP_MAC_up_ref(mac)) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); + if (ctx != NULL) mac->freectx(ctx->algctx); - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); - OPENSSL_free(ctx); - ctx = NULL; - } + OPENSSL_free(ctx); + ctx = NULL; + } else { + ctx->meth = mac; } return ctx; } @@ -55,12 +56,14 @@ EVP_MAC_CTX *EVP_MAC_CTX_dup(const EVP_MAC_CTX *src) return NULL; dst = OPENSSL_malloc(sizeof(*dst)); - if (dst == NULL) + if (dst == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } *dst = *src; if (!EVP_MAC_up_ref(dst->meth)) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); OPENSSL_free(dst); return NULL; } @@ -223,7 +226,7 @@ const char *EVP_MAC_get0_description(const EVP_MAC *mac) int EVP_MAC_is_a(const EVP_MAC *mac, const char *name) { - return mac != NULL && evp_is_a(mac->prov, mac->name_id, NULL, name); + return evp_is_a(mac->prov, mac->name_id, NULL, name); } int EVP_MAC_names_do_all(const EVP_MAC *mac, diff --git a/openssl/src/crypto/evp/mac_meth.c b/openssl/src/crypto/evp/mac_meth.c index a3e7a0220..85fe7704f 100644 --- a/openssl/src/crypto/evp/mac_meth.c +++ b/openssl/src/crypto/evp/mac_meth.c @@ -1,5 +1,5 @@ /* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,7 +21,7 @@ static int evp_mac_up_ref(void *vmac) EVP_MAC *mac = vmac; int ref = 0; - CRYPTO_UP_REF(&mac->refcnt, &ref); + CRYPTO_UP_REF(&mac->refcnt, &ref, mac->lock); return 1; } @@ -33,12 +33,12 @@ static void evp_mac_free(void *vmac) if (mac == NULL) return; - CRYPTO_DOWN_REF(&mac->refcnt, &ref); + CRYPTO_DOWN_REF(&mac->refcnt, &ref, mac->lock); if (ref > 0) return; OPENSSL_free(mac->type_name); ossl_provider_free(mac->prov); - CRYPTO_FREE_REF(&mac->refcnt); + CRYPTO_THREAD_lock_free(mac->lock); OPENSSL_free(mac); } @@ -47,10 +47,13 @@ static void *evp_mac_new(void) EVP_MAC *mac = NULL; if ((mac = OPENSSL_zalloc(sizeof(*mac))) == NULL - || !CRYPTO_NEW_REF(&mac->refcnt, 1)) { + || (mac->lock = CRYPTO_THREAD_lock_new()) == NULL) { evp_mac_free(mac); return NULL; } + + mac->refcnt = 1; + return mac; } @@ -63,7 +66,7 @@ static void *evp_mac_from_algorithm(int name_id, int fnmaccnt = 0, fnctxcnt = 0; if ((mac = evp_mac_new()) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; } mac->name_id = name_id; diff --git a/openssl/src/crypto/evp/p5_crpt2.c b/openssl/src/crypto/evp/p5_crpt2.c index 356173902..8e3fccb21 100644 --- a/openssl/src/crypto/evp/p5_crpt2.c +++ b/openssl/src/crypto/evp/p5_crpt2.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -159,7 +159,7 @@ int PKCS5_v2_PBE_keyivgen_ex(EVP_CIPHER_CTX *ctx, const char *pass, int passlen, /* Fixup cipher based on AlgorithmIdentifier */ if (!EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, en_de)) goto err; - if (EVP_CIPHER_asn1_to_param(ctx, pbe2->encryption->parameter) <= 0) { + if (EVP_CIPHER_asn1_to_param(ctx, pbe2->encryption->parameter) < 0) { ERR_raise(ERR_LIB_EVP, EVP_R_CIPHER_PARAMETER_ERROR); goto err; } @@ -231,16 +231,13 @@ int PKCS5_v2_PBKDF2_keyivgen_ex(EVP_CIPHER_CTX *ctx, const char *pass, goto err; } - (void)ERR_set_mark(); prfmd = prfmd_fetch = EVP_MD_fetch(libctx, OBJ_nid2sn(hmac_md_nid), propq); if (prfmd == NULL) prfmd = EVP_get_digestbynid(hmac_md_nid); if (prfmd == NULL) { - (void)ERR_clear_last_mark(); ERR_raise(ERR_LIB_EVP, EVP_R_UNSUPPORTED_PRF); goto err; } - (void)ERR_pop_to_mark(); if (kdf->salt->type != V_ASN1_OCTET_STRING) { ERR_raise(ERR_LIB_EVP, EVP_R_UNSUPPORTED_SALT_TYPE); diff --git a/openssl/src/crypto/evp/p_dec.c b/openssl/src/crypto/evp/p_dec.c index 29ea3f5fb..7b33edecd 100644 --- a/openssl/src/crypto/evp/p_dec.c +++ b/openssl/src/crypto/evp/p_dec.c @@ -22,19 +22,15 @@ int EVP_PKEY_decrypt_old(unsigned char *key, const unsigned char *ek, int ekl, EVP_PKEY *priv) { int ret = -1; - RSA *rsa = NULL; if (EVP_PKEY_get_id(priv) != EVP_PKEY_RSA) { ERR_raise(ERR_LIB_EVP, EVP_R_PUBLIC_KEY_NOT_RSA); goto err; } - rsa = evp_pkey_get0_RSA_int(priv); - if (rsa == NULL) - goto err; - ret = - RSA_private_decrypt(ekl, ek, key, rsa, RSA_PKCS1_PADDING); + RSA_private_decrypt(ekl, ek, key, evp_pkey_get0_RSA_int(priv), + RSA_PKCS1_PADDING); err: return ret; } diff --git a/openssl/src/crypto/evp/p_enc.c b/openssl/src/crypto/evp/p_enc.c index 64e675145..d4db59516 100644 --- a/openssl/src/crypto/evp/p_enc.c +++ b/openssl/src/crypto/evp/p_enc.c @@ -22,19 +22,15 @@ int EVP_PKEY_encrypt_old(unsigned char *ek, const unsigned char *key, int key_len, EVP_PKEY *pubk) { int ret = 0; - RSA *rsa = NULL; if (EVP_PKEY_get_id(pubk) != EVP_PKEY_RSA) { ERR_raise(ERR_LIB_EVP, EVP_R_PUBLIC_KEY_NOT_RSA); goto err; } - rsa = evp_pkey_get0_RSA_int(pubk); - if (rsa == NULL) - goto err; - ret = - RSA_public_encrypt(key_len, key, ek, rsa, RSA_PKCS1_PADDING); + RSA_public_encrypt(key_len, key, ek, evp_pkey_get0_RSA_int(pubk), + RSA_PKCS1_PADDING); err: return ret; } diff --git a/openssl/src/crypto/evp/p_lib.c b/openssl/src/crypto/evp/p_lib.c index b7377751b..3450c7b33 100644 --- a/openssl/src/crypto/evp/p_lib.c +++ b/openssl/src/crypto/evp/p_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -68,11 +68,7 @@ int EVP_PKEY_get_bits(const EVP_PKEY *pkey) if (pkey->ameth != NULL && pkey->ameth->pkey_bits != NULL) size = pkey->ameth->pkey_bits(pkey); } - if (size <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_UNKNOWN_BITS); - return 0; - } - return size; + return size < 0 ? 0 : size; } int EVP_PKEY_get_security_bits(const EVP_PKEY *pkey) @@ -84,11 +80,7 @@ int EVP_PKEY_get_security_bits(const EVP_PKEY *pkey) if (pkey->ameth != NULL && pkey->ameth->pkey_security_bits != NULL) size = pkey->ameth->pkey_security_bits(pkey); } - if (size <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_UNKNOWN_SECURITY_BITS); - return 0; - } - return size; + return size < 0 ? 0 : size; } int EVP_PKEY_save_parameters(EVP_PKEY *pkey, int mode) @@ -347,16 +339,9 @@ int EVP_PKEY_eq(const EVP_PKEY *a, const EVP_PKEY *b) if (a == NULL || b == NULL) return 0; - if (a->keymgmt != NULL || b->keymgmt != NULL) { - int selection = SELECT_PARAMETERS; - - if (evp_keymgmt_util_has((EVP_PKEY *)a, OSSL_KEYMGMT_SELECT_PUBLIC_KEY) - && evp_keymgmt_util_has((EVP_PKEY *)b, OSSL_KEYMGMT_SELECT_PUBLIC_KEY)) - selection |= OSSL_KEYMGMT_SELECT_PUBLIC_KEY; - else - selection |= OSSL_KEYMGMT_SELECT_KEYPAIR; - return evp_pkey_cmp_any(a, b, selection); - } + if (a->keymgmt != NULL || b->keymgmt != NULL) + return evp_pkey_cmp_any(a, b, (SELECT_PARAMETERS + | OSSL_KEYMGMT_SELECT_KEYPAIR)); /* All legacy keys */ if (a->type != b->type) @@ -450,12 +435,12 @@ static EVP_PKEY *new_raw_key_int(OSSL_LIB_CTX *libctx, pkey = EVP_PKEY_new(); if (pkey == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } if (!pkey_set_type(pkey, e, nidtype, strtype, -1, NULL)) { - /* ERR_raise(ERR_LIB_EVP, ...) already called */ + /* EVPerr already called */ goto err; } @@ -695,6 +680,26 @@ int EVP_PKEY_set_type_str(EVP_PKEY *pkey, const char *str, int len) return pkey_set_type(pkey, NULL, EVP_PKEY_NONE, str, len, NULL); } +int EVP_PKEY_set_alias_type(EVP_PKEY *pkey, int type) +{ + if (pkey->type == type) { + return 1; /* it already is that type */ + } + + /* + * The application is requesting to alias this to a different pkey type, + * but not one that resolves to the base type. + */ + if (EVP_PKEY_get_id(pkey) > EVP_PKEY_NONE && + EVP_PKEY_type(type) != EVP_PKEY_base_id(pkey)) { + EVPerr(EVP_F_EVP_PKEY_SET_ALIAS_TYPE, EVP_R_UNSUPPORTED_ALGORITHM); + return 0; + } + + pkey->type = type; + return 1; +} + # ifndef OPENSSL_NO_ENGINE int EVP_PKEY_set1_engine(EVP_PKEY *pkey, ENGINE *e) { @@ -725,13 +730,11 @@ static void detect_foreign_key(EVP_PKEY *pkey) { switch (pkey->type) { case EVP_PKEY_RSA: - case EVP_PKEY_RSA_PSS: pkey->foreign = pkey->pkey.rsa != NULL && ossl_rsa_is_foreign(pkey->pkey.rsa); break; # ifndef OPENSSL_NO_EC case EVP_PKEY_SM2: - break; case EVP_PKEY_EC: pkey->foreign = pkey->pkey.ec != NULL && ossl_ec_key_is_foreign(pkey->pkey.ec); @@ -850,6 +853,20 @@ const unsigned char *EVP_PKEY_get0_siphash(const EVP_PKEY *pkey, size_t *len) } # endif +# ifndef OPENSSL_NO_ZUC +const unsigned char *EVP_PKEY_get0_eia3(const EVP_PKEY *pkey, size_t *len) +{ + ASN1_OCTET_STRING *os = NULL; + if (pkey->type != EVP_PKEY_EIA3) { + EVPerr(EVP_F_EVP_PKEY_GET0_EIA3, EVP_R_EXPECTING_A_EIA3_KEY); + return NULL; + } + os = EVP_PKEY_get0(pkey); + *len = os->length; + return os->data; +} +# endif + # ifndef OPENSSL_NO_DSA static DSA *evp_pkey_get0_DSA_int(const EVP_PKEY *pkey) { @@ -882,7 +899,7 @@ DSA *EVP_PKEY_get1_DSA(EVP_PKEY *pkey) } # endif /* OPENSSL_NO_DSA */ -# ifndef OPENSSL_NO_ECX +# ifndef OPENSSL_NO_EC static const ECX_KEY *evp_pkey_get0_ECX_KEY(const EVP_PKEY *pkey, int type) { if (EVP_PKEY_get_base_id(pkey) != type) { @@ -911,7 +928,7 @@ IMPLEMENT_ECX_VARIANT(X448) IMPLEMENT_ECX_VARIANT(ED25519) IMPLEMENT_ECX_VARIANT(ED448) -# endif /* OPENSSL_NO_ECX */ +# endif # if !defined(OPENSSL_NO_DH) && !defined(OPENSSL_NO_DEPRECATED_3_0) @@ -1056,10 +1073,11 @@ const char *evp_pkey_type2name(int type) int EVP_PKEY_is_a(const EVP_PKEY *pkey, const char *name) { - if (pkey == NULL) - return 0; - if (pkey->keymgmt == NULL) - return pkey->type == evp_pkey_name2type(name); + if (pkey->keymgmt == NULL) { + int type = evp_pkey_name2type(name); + + return pkey->type == type; + } return EVP_KEYMGMT_is_a(pkey->keymgmt, name); } @@ -1084,7 +1102,6 @@ int EVP_PKEY_can_sign(const EVP_PKEY *pkey) if (pkey->keymgmt == NULL) { switch (EVP_PKEY_get_base_id(pkey)) { case EVP_PKEY_RSA: - case EVP_PKEY_RSA_PSS: return 1; # ifndef OPENSSL_NO_DSA case EVP_PKEY_DSA: @@ -1140,11 +1157,8 @@ static int print_set_indent(BIO **out, int *pop_f_prefix, long *saved_indent, *saved_indent = (i < 0 ? 0 : i); if (BIO_set_indent(*out, indent) <= 0) { - BIO *prefbio = BIO_new(BIO_f_prefix()); - - if (prefbio == NULL) + if ((*out = BIO_push(BIO_new(BIO_f_prefix()), *out)) == NULL) return 0; - *out = BIO_push(prefbio, *out); *pop_f_prefix = 1; } if (BIO_set_indent(*out, indent) <= 0) { @@ -1209,7 +1223,7 @@ int EVP_PKEY_print_public(BIO *out, const EVP_PKEY *pkey, int EVP_PKEY_print_private(BIO *out, const EVP_PKEY *pkey, int indent, ASN1_PCTX *pctx) { - return print_pkey(pkey, out, indent, EVP_PKEY_PRIVATE_KEY, NULL, + return print_pkey(pkey, out, indent, EVP_PKEY_KEYPAIR, NULL, (pkey->ameth != NULL ? pkey->ameth->priv_print : NULL), pctx); } @@ -1335,8 +1349,6 @@ static int evp_pkey_asn1_ctrl(EVP_PKEY *pkey, int op, int arg1, void *arg2) int EVP_PKEY_get_default_digest_nid(EVP_PKEY *pkey, int *pnid) { - if (pkey == NULL) - return 0; return evp_pkey_asn1_ctrl(pkey, ASN1_PKEY_CTRL_DEFAULT_MD_NID, 0, pnid); } @@ -1387,9 +1399,7 @@ int EVP_PKEY_digestsign_supports_digest(EVP_PKEY *pkey, OSSL_LIB_CTX *libctx, int EVP_PKEY_set1_encoded_public_key(EVP_PKEY *pkey, const unsigned char *pub, size_t publen) { - if (pkey == NULL) - return 0; - if (evp_pkey_is_provided(pkey)) + if (pkey != NULL && evp_pkey_is_provided(pkey)) return EVP_PKEY_set_octet_string_param(pkey, OSSL_PKEY_PARAM_ENCODED_PUBLIC_KEY, @@ -1408,11 +1418,8 @@ size_t EVP_PKEY_get1_encoded_public_key(EVP_PKEY *pkey, unsigned char **ppub) { int rv; - if (pkey == NULL) - return 0; - if (evp_pkey_is_provided(pkey)) { + if (pkey != NULL && evp_pkey_is_provided(pkey)) { size_t return_size = OSSL_PARAM_UNMODIFIED; - unsigned char *buf; /* * We know that this is going to fail, but it will give us a size @@ -1424,18 +1431,14 @@ size_t EVP_PKEY_get1_encoded_public_key(EVP_PKEY *pkey, unsigned char **ppub) if (return_size == OSSL_PARAM_UNMODIFIED) return 0; - *ppub = NULL; - buf = OPENSSL_malloc(return_size); - if (buf == NULL) + *ppub = OPENSSL_malloc(return_size); + if (*ppub == NULL) return 0; if (!EVP_PKEY_get_octet_string_param(pkey, OSSL_PKEY_PARAM_ENCODED_PUBLIC_KEY, - buf, return_size, NULL)) { - OPENSSL_free(buf); + *ppub, return_size, NULL)) return 0; - } - *ppub = buf; return return_size; } @@ -1454,32 +1457,31 @@ EVP_PKEY *EVP_PKEY_new(void) { EVP_PKEY *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } ret->type = EVP_PKEY_NONE; ret->save_type = EVP_PKEY_NONE; - - if (!CRYPTO_NEW_REF(&ret->references, 1)) - goto err; + ret->references = 1; ret->lock = CRYPTO_THREAD_lock_new(); if (ret->lock == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_CRYPTO_LIB); + EVPerr(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } #ifndef FIPS_MODULE ret->save_parameters = 1; if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_EVP_PKEY, ret, &ret->ex_data)) { - ERR_raise(ERR_LIB_EVP, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } #endif return ret; err: - CRYPTO_FREE_REF(&ret->references); CRYPTO_THREAD_lock_free(ret->lock); OPENSSL_free(ret); return NULL; @@ -1548,6 +1550,7 @@ static int pkey_set_type(EVP_PKEY *pkey, ENGINE *e, int type, const char *str, ameth = EVP_PKEY_asn1_find_str(eptr, str, len); else if (type != EVP_PKEY_NONE) ameth = EVP_PKEY_asn1_find(eptr, type); + # ifndef OPENSSL_NO_ENGINE if (pkey == NULL && eptr != NULL) ENGINE_finish(e); @@ -1599,7 +1602,15 @@ static int pkey_set_type(EVP_PKEY *pkey, ENGINE *e, int type, const char *str, if (type == EVP_PKEY_NONE) pkey->type = ameth->pkey_id; } else { - pkey->type = EVP_PKEY_KEYMGMT; + /* + * Note: ameth for SM2 is NULL due to ASN1_PKEY_ALIAS flag. + * The key type of SM2 pkey maybe used by legacy functions, such as + * EVP_PKEY_get0_EC_KEY, so we set it. + */ + if (keymgmt != NULL && EVP_KEYMGMT_is_a(keymgmt, "SM2")) + pkey->type = EVP_PKEY_SM2; + else + pkey->type = EVP_PKEY_KEYMGMT; } # ifndef OPENSSL_NO_ENGINE if (eptr == NULL && e != NULL && !ENGINE_init(e)) { @@ -1669,7 +1680,7 @@ int EVP_PKEY_up_ref(EVP_PKEY *pkey) { int i; - if (CRYPTO_UP_REF(&pkey->references, &i) <= 0) + if (CRYPTO_UP_REF(&pkey->references, &i, pkey->lock) <= 0) return 0; REF_PRINT_COUNT("EVP_PKEY", pkey); @@ -1771,7 +1782,7 @@ void evp_pkey_free_legacy(EVP_PKEY *x) static void evp_pkey_free_it(EVP_PKEY *x) { /* internal function; x is never NULL */ - evp_keymgmt_util_clear_operation_cache(x); + evp_keymgmt_util_clear_operation_cache(x, 1); #ifndef FIPS_MODULE evp_pkey_free_legacy(x); #endif @@ -1792,7 +1803,7 @@ void EVP_PKEY_free(EVP_PKEY *x) if (x == NULL) return; - CRYPTO_DOWN_REF(&x->references, &i); + CRYPTO_DOWN_REF(&x->references, &i, x->lock); REF_PRINT_COUNT("EVP_PKEY", x); if (i > 0) return; @@ -1802,7 +1813,6 @@ void EVP_PKEY_free(EVP_PKEY *x) CRYPTO_free_ex_data(CRYPTO_EX_INDEX_EVP_PKEY, x, &x->ex_data); #endif CRYPTO_THREAD_lock_free(x->lock); - CRYPTO_FREE_REF(&x->references); #ifndef FIPS_MODULE sk_X509_ATTRIBUTE_pop_free(x->attributes, X509_ATTRIBUTE_free); #endif @@ -1820,11 +1830,7 @@ int EVP_PKEY_get_size(const EVP_PKEY *pkey) size = pkey->ameth->pkey_size(pkey); #endif } - if (size <= 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_UNKNOWN_MAX_SIZE); - return 0; - } - return size; + return size < 0 ? 0 : size; } const char *EVP_PKEY_get0_description(const EVP_PKEY *pkey) @@ -1847,7 +1853,6 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, { EVP_KEYMGMT *allocated_keymgmt = NULL; EVP_KEYMGMT *tmp_keymgmt = NULL; - int selection = OSSL_KEYMGMT_SELECT_ALL; void *keydata = NULL; int check; @@ -1888,7 +1893,7 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, if (ctx == NULL) goto end; - allocated_keymgmt = tmp_keymgmt = ctx->keymgmt; + tmp_keymgmt = ctx->keymgmt; ctx->keymgmt = NULL; EVP_PKEY_CTX_free(ctx); } @@ -1909,22 +1914,13 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, if (pk->ameth->dirty_cnt(pk) == pk->dirty_cnt_copy) { if (!CRYPTO_THREAD_read_lock(pk->lock)) goto end; - op = evp_keymgmt_util_find_operation_cache(pk, tmp_keymgmt, - selection); + op = evp_keymgmt_util_find_operation_cache(pk, tmp_keymgmt); /* * If |tmp_keymgmt| is present in the operation cache, it means * that export doesn't need to be redone. In that case, we take * token copies of the cached pointers, to have token success - * values to return. It is possible (e.g. in a no-cached-fetch - * build), for op->keymgmt to be a different pointer to tmp_keymgmt - * even though the name/provider must be the same. In other words - * the keymgmt instance may be different but still equivalent, i.e. - * same algorithm/provider instance - but we make the simplifying - * assumption that the keydata can be used with either keymgmt - * instance. Not doing so introduces significant complexity and - * probably requires refactoring - since we would have to ripple - * the change in keymgmt instance up the call chain. + * values to return. */ if (op != NULL && op->keymgmt != NULL) { keydata = op->keydata; @@ -1963,7 +1959,7 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, if (!CRYPTO_THREAD_write_lock(pk->lock)) goto end; if (pk->ameth->dirty_cnt(pk) != pk->dirty_cnt_copy - && !evp_keymgmt_util_clear_operation_cache(pk)) { + && !evp_keymgmt_util_clear_operation_cache(pk, 0)) { CRYPTO_THREAD_unlock(pk->lock); evp_keymgmt_freedata(tmp_keymgmt, keydata); keydata = NULL; @@ -1973,7 +1969,7 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, EVP_KEYMGMT_free(tmp_keymgmt); /* refcnt-- */ /* Check to make sure some other thread didn't get there first */ - op = evp_keymgmt_util_find_operation_cache(pk, tmp_keymgmt, selection); + op = evp_keymgmt_util_find_operation_cache(pk, tmp_keymgmt); if (op != NULL && op->keymgmt != NULL) { void *tmp_keydata = op->keydata; @@ -1984,8 +1980,7 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, } /* Add the new export to the operation cache */ - if (!evp_keymgmt_util_cache_keydata(pk, tmp_keymgmt, keydata, - selection)) { + if (!evp_keymgmt_util_cache_keydata(pk, tmp_keymgmt, keydata)) { CRYPTO_THREAD_unlock(pk->lock); evp_keymgmt_freedata(tmp_keymgmt, keydata); keydata = NULL; @@ -2000,7 +1995,7 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, } #endif /* FIPS_MODULE */ - keydata = evp_keymgmt_util_export_to_provider(pk, tmp_keymgmt, selection); + keydata = evp_keymgmt_util_export_to_provider(pk, tmp_keymgmt); end: /* @@ -2011,10 +2006,8 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, if (keydata == NULL) tmp_keymgmt = NULL; - if (keymgmt != NULL && tmp_keymgmt != NULL) { + if (keymgmt != NULL) *keymgmt = tmp_keymgmt; - allocated_keymgmt = NULL; - } EVP_KEYMGMT_free(allocated_keymgmt); return keydata; @@ -2023,8 +2016,6 @@ void *evp_pkey_export_to_provider(EVP_PKEY *pk, OSSL_LIB_CTX *libctx, #ifndef FIPS_MODULE int evp_pkey_copy_downgraded(EVP_PKEY **dest, const EVP_PKEY *src) { - EVP_PKEY *allocpkey = NULL; - if (!ossl_assert(dest != NULL)) return 0; @@ -2055,9 +2046,9 @@ int evp_pkey_copy_downgraded(EVP_PKEY **dest, const EVP_PKEY *src) /* Make sure we have a clean slate to copy into */ if (*dest == NULL) { - allocpkey = *dest = EVP_PKEY_new(); + *dest = EVP_PKEY_new(); if (*dest == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } } else { @@ -2083,7 +2074,7 @@ int evp_pkey_copy_downgraded(EVP_PKEY **dest, const EVP_PKEY *src) EVP_PKEY_CTX_new_from_pkey(libctx, *dest, NULL); if (pctx == NULL) - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); if (pctx != NULL && evp_keymgmt_export(keymgmt, keydata, @@ -2104,10 +2095,6 @@ int evp_pkey_copy_downgraded(EVP_PKEY **dest, const EVP_PKEY *src) } } - if (allocpkey != NULL) { - EVP_PKEY_free(allocpkey); - *dest = NULL; - } return 0; } @@ -2140,7 +2127,7 @@ void *evp_pkey_get_legacy(EVP_PKEY *pk) return ret; if (!evp_pkey_copy_downgraded(&tmp_copy, pk)) - goto err; + return NULL; if (!CRYPTO_THREAD_write_lock(pk->lock)) goto err; @@ -2203,14 +2190,7 @@ int EVP_PKEY_get_bn_param(const EVP_PKEY *pkey, const char *key_name, goto err; ret = OSSL_PARAM_get_BN(params, bn); err: - if (buf != NULL) { - if (OSSL_PARAM_modified(params)) - OPENSSL_clear_free(buf, buf_sz); - else - OPENSSL_free(buf); - } else if (OSSL_PARAM_modified(params)) { - OPENSSL_cleanse(buffer, params[0].data_size); - } + OPENSSL_free(buf); return ret; } @@ -2406,10 +2386,10 @@ int EVP_PKEY_get_params(const EVP_PKEY *pkey, OSSL_PARAM params[]) { if (pkey != NULL) { if (evp_pkey_is_provided(pkey)) - return evp_keymgmt_get_params(pkey->keymgmt, pkey->keydata, params) > 0; + return evp_keymgmt_get_params(pkey->keymgmt, pkey->keydata, params); #ifndef FIPS_MODULE else if (evp_pkey_is_legacy(pkey)) - return evp_pkey_get_params_to_ctrl(pkey, params) > 0; + return evp_pkey_get_params_to_ctrl(pkey, params); #endif } ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_KEY); diff --git a/openssl/src/crypto/evp/p_open.c b/openssl/src/crypto/evp/p_open.c index 8630553e7..b08f27164 100644 --- a/openssl/src/crypto/evp/p_open.c +++ b/openssl/src/crypto/evp/p_open.c @@ -34,7 +34,7 @@ int EVP_OpenInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type, return 1; if ((pctx = EVP_PKEY_CTX_new(priv, NULL)) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } @@ -42,13 +42,15 @@ int EVP_OpenInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type, || EVP_PKEY_decrypt(pctx, NULL, &keylen, ek, ekl) <= 0) goto err; - if ((key = OPENSSL_malloc(keylen)) == NULL) + if ((key = OPENSSL_malloc(keylen)) == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; + } if (EVP_PKEY_decrypt(pctx, key, &keylen, ek, ekl) <= 0) goto err; - if (EVP_CIPHER_CTX_set_key_length(ctx, keylen) <= 0 + if (!EVP_CIPHER_CTX_set_key_length(ctx, keylen) || !EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv)) goto err; diff --git a/openssl/src/crypto/evp/p_seal.c b/openssl/src/crypto/evp/p_seal.c index 94c8462ab..475082d43 100644 --- a/openssl/src/crypto/evp/p_seal.c +++ b/openssl/src/crypto/evp/p_seal.c @@ -15,6 +15,7 @@ #include #include #include +#include int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type, unsigned char **ek, int *ekl, unsigned char *iv, @@ -58,7 +59,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type, pctx = EVP_PKEY_CTX_new_from_pkey(libctx, pubk[i], NULL); if (pctx == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } diff --git a/openssl/src/crypto/evp/p_sign.c b/openssl/src/crypto/evp/p_sign.c index e5555281a..8e430f470 100644 --- a/openssl/src/crypto/evp/p_sign.c +++ b/openssl/src/crypto/evp/p_sign.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,14 +33,12 @@ int EVP_SignFinal_ex(EVP_MD_CTX *ctx, unsigned char *sigret, EVP_MD_CTX *tmp_ctx = EVP_MD_CTX_new(); if (tmp_ctx == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } rv = EVP_MD_CTX_copy_ex(tmp_ctx, ctx); if (rv) rv = EVP_DigestFinal_ex(tmp_ctx, m, &m_len); - else - rv = EVP_DigestFinal_ex(ctx, m, &m_len); EVP_MD_CTX_free(tmp_ctx); if (!rv) return 0; diff --git a/openssl/src/crypto/evp/p_verify.c b/openssl/src/crypto/evp/p_verify.c index 02db143d1..e5667afb7 100644 --- a/openssl/src/crypto/evp/p_verify.c +++ b/openssl/src/crypto/evp/p_verify.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -31,14 +31,12 @@ int EVP_VerifyFinal_ex(EVP_MD_CTX *ctx, const unsigned char *sigbuf, EVP_MD_CTX *tmp_ctx = EVP_MD_CTX_new(); if (tmp_ctx == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } rv = EVP_MD_CTX_copy_ex(tmp_ctx, ctx); if (rv) rv = EVP_DigestFinal_ex(tmp_ctx, m, &m_len); - else - rv = EVP_DigestFinal_ex(ctx, m, &m_len); EVP_MD_CTX_free(tmp_ctx); if (!rv) return 0; diff --git a/openssl/src/crypto/evp/pmeth_gn.c b/openssl/src/crypto/evp/pmeth_gn.c index 74cca96f4..8e4940ed5 100644 --- a/openssl/src/crypto/evp/pmeth_gn.c +++ b/openssl/src/crypto/evp/pmeth_gn.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -145,7 +145,7 @@ int EVP_PKEY_generate(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey) *ppkey = allocated_pkey = EVP_PKEY_new(); if (*ppkey == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return -1; } @@ -153,7 +153,7 @@ int EVP_PKEY_generate(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey) goto legacy; /* - * Assigning gentmp to ctx->keygen_info is something our legacy + * Asssigning gentmp to ctx->keygen_info is something our legacy * implementations do. Because the provider implementations aren't * allowed to reach into our EVP_PKEY_CTX, we need to provide similar * space for backward compatibility. It's ok that we attach a local @@ -378,7 +378,7 @@ int EVP_PKEY_fromdata(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey, int selection, allocated_pkey = *ppkey = EVP_PKEY_new(); if (*ppkey == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return -1; } diff --git a/openssl/src/crypto/evp/pmeth_lib.c b/openssl/src/crypto/evp/pmeth_lib.c index 0a561323f..26d3e902d 100644 --- a/openssl/src/crypto/evp/pmeth_lib.c +++ b/openssl/src/crypto/evp/pmeth_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -27,6 +27,7 @@ #ifndef FIPS_MODULE # include "crypto/asn1.h" #endif +#include "crypto/ctype.h" #include "crypto/evp.h" #include "crypto/dh.h" #include "crypto/ec.h" @@ -66,9 +67,11 @@ static pmeth_fn standard_methods[] = { # ifndef OPENSSL_NO_DH ossl_dhx_pkey_method, # endif -# ifndef OPENSSL_NO_ECX +# ifndef OPENSSL_NO_EC ossl_ecx25519_pkey_method, ossl_ecx448_pkey_method, +# endif +# ifndef OPENSSL_NO_EC ossl_ed25519_pkey_method, ossl_ed448_pkey_method, # endif @@ -126,13 +129,33 @@ EVP_PKEY_METHOD *EVP_PKEY_meth_new(int id, int flags) EVP_PKEY_METHOD *pmeth; pmeth = OPENSSL_zalloc(sizeof(*pmeth)); - if (pmeth == NULL) + if (pmeth == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } pmeth->pkey_id = id; pmeth->flags = flags | EVP_PKEY_FLAG_DYNAMIC; return pmeth; } + +static void help_get_legacy_alg_type_from_keymgmt(const char *keytype, + void *arg) +{ + int *type = arg; + + if (*type == NID_undef) + *type = evp_pkey_name2type(keytype); +} + +static int get_legacy_alg_type_from_keymgmt(const EVP_KEYMGMT *keymgmt) +{ + int type = NID_undef; + + EVP_KEYMGMT_names_do_all(keymgmt, help_get_legacy_alg_type_from_keymgmt, + &type); + return type; +} #endif /* FIPS_MODULE */ int evp_pkey_ctx_state(const EVP_PKEY_CTX *ctx) @@ -229,11 +252,10 @@ static EVP_PKEY_CTX *int_ctx_new(OSSL_LIB_CTX *libctx, */ if (e != NULL) pmeth = ENGINE_get_pkey_meth(e, id); - else -# endif /* OPENSSL_NO_ENGINE */ - if (pkey != NULL && pkey->foreign) + else if (pkey != NULL && pkey->foreign) pmeth = EVP_PKEY_meth_find(id); else +# endif app_pmeth = pmeth = evp_pkey_meth_find_added_by_application(id); /* END legacy */ @@ -270,7 +292,7 @@ static EVP_PKEY_CTX *int_ctx_new(OSSL_LIB_CTX *libctx, * directly. */ if (keymgmt != NULL) { - int tmp_id = evp_keymgmt_get_legacy_alg(keymgmt); + int tmp_id = get_legacy_alg_type_from_keymgmt(keymgmt); if (tmp_id != NID_undef) { if (id == -1) { @@ -295,6 +317,8 @@ static EVP_PKEY_CTX *int_ctx_new(OSSL_LIB_CTX *libctx, ERR_raise(ERR_LIB_EVP, EVP_R_UNSUPPORTED_ALGORITHM); } else { ret = OPENSSL_zalloc(sizeof(*ret)); + if (ret == NULL) + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); } #if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODULE) @@ -458,8 +482,10 @@ EVP_PKEY_CTX *EVP_PKEY_CTX_dup(const EVP_PKEY_CTX *pctx) } # endif rctx = OPENSSL_zalloc(sizeof(*rctx)); - if (rctx == NULL) + if (rctx == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } if (pctx->pkey != NULL) EVP_PKEY_up_ref(pctx->pkey); @@ -484,11 +510,8 @@ EVP_PKEY_CTX *EVP_PKEY_CTX_dup(const EVP_PKEY_CTX *pctx) if (pctx->op.kex.algctx != NULL) { if (!ossl_assert(pctx->op.kex.exchange != NULL)) goto err; - - if (pctx->op.kex.exchange->dupctx != NULL) - rctx->op.kex.algctx - = pctx->op.kex.exchange->dupctx(pctx->op.kex.algctx); - + rctx->op.kex.algctx + = pctx->op.kex.exchange->dupctx(pctx->op.kex.algctx); if (rctx->op.kex.algctx == NULL) { EVP_KEYEXCH_free(rctx->op.kex.exchange); rctx->op.kex.exchange = NULL; @@ -505,11 +528,8 @@ EVP_PKEY_CTX *EVP_PKEY_CTX_dup(const EVP_PKEY_CTX *pctx) if (pctx->op.sig.algctx != NULL) { if (!ossl_assert(pctx->op.sig.signature != NULL)) goto err; - - if (pctx->op.sig.signature->dupctx != NULL) - rctx->op.sig.algctx - = pctx->op.sig.signature->dupctx(pctx->op.sig.algctx); - + rctx->op.sig.algctx + = pctx->op.sig.signature->dupctx(pctx->op.sig.algctx); if (rctx->op.sig.algctx == NULL) { EVP_SIGNATURE_free(rctx->op.sig.signature); rctx->op.sig.signature = NULL; @@ -526,11 +546,8 @@ EVP_PKEY_CTX *EVP_PKEY_CTX_dup(const EVP_PKEY_CTX *pctx) if (pctx->op.ciph.algctx != NULL) { if (!ossl_assert(pctx->op.ciph.cipher != NULL)) goto err; - - if (pctx->op.ciph.cipher->dupctx != NULL) - rctx->op.ciph.algctx - = pctx->op.ciph.cipher->dupctx(pctx->op.ciph.algctx); - + rctx->op.ciph.algctx + = pctx->op.ciph.cipher->dupctx(pctx->op.ciph.algctx); if (rctx->op.ciph.algctx == NULL) { EVP_ASYM_CIPHER_free(rctx->op.ciph.cipher); rctx->op.ciph.cipher = NULL; @@ -547,11 +564,8 @@ EVP_PKEY_CTX *EVP_PKEY_CTX_dup(const EVP_PKEY_CTX *pctx) if (pctx->op.encap.algctx != NULL) { if (!ossl_assert(pctx->op.encap.kem != NULL)) goto err; - - if (pctx->op.encap.kem->dupctx != NULL) - rctx->op.encap.algctx - = pctx->op.encap.kem->dupctx(pctx->op.encap.algctx); - + rctx->op.encap.algctx + = pctx->op.encap.kem->dupctx(pctx->op.encap.algctx); if (rctx->op.encap.algctx == NULL) { EVP_KEM_free(rctx->op.encap.kem); rctx->op.encap.kem = NULL; @@ -601,13 +615,13 @@ int EVP_PKEY_meth_add0(const EVP_PKEY_METHOD *pmeth) { if (app_pkey_methods == NULL) { app_pkey_methods = sk_EVP_PKEY_METHOD_new(pmeth_cmp); - if (app_pkey_methods == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_CRYPTO_LIB); + if (app_pkey_methods == NULL){ + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } } if (!sk_EVP_PKEY_METHOD_push(app_pkey_methods, pmeth)) { - ERR_raise(ERR_LIB_EVP, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; } sk_EVP_PKEY_METHOD_sort(app_pkey_methods); @@ -853,7 +867,7 @@ int evp_pkey_ctx_set_params_strict(EVP_PKEY_CTX *ctx, OSSL_PARAM *params) for (p = params; p->key != NULL; p++) { /* Check the ctx actually understands this parameter */ - if (OSSL_PARAM_locate_const(settable, p->key) == NULL) + if (OSSL_PARAM_locate_const(settable, p->key) == NULL ) return -2; } } @@ -876,9 +890,9 @@ int evp_pkey_ctx_get_params_strict(EVP_PKEY_CTX *ctx, OSSL_PARAM *params) const OSSL_PARAM *gettable = EVP_PKEY_CTX_gettable_params(ctx); const OSSL_PARAM *p; - for (p = params; p->key != NULL; p++) { + for (p = params; p->key != NULL; p++ ) { /* Check the ctx actually understands this parameter */ - if (OSSL_PARAM_locate_const(gettable, p->key) == NULL) + if (OSSL_PARAM_locate_const(gettable, p->key) == NULL ) return -2; } } @@ -1002,71 +1016,6 @@ static int evp_pkey_ctx_set1_octet_string(EVP_PKEY_CTX *ctx, int fallback, return EVP_PKEY_CTX_set_params(ctx, octet_string_params); } -static int evp_pkey_ctx_add1_octet_string(EVP_PKEY_CTX *ctx, int fallback, - const char *param, int op, int ctrl, - const unsigned char *data, - int datalen) -{ - OSSL_PARAM os_params[2]; - unsigned char *info = NULL; - size_t info_len = 0; - size_t info_alloc = 0; - int ret = 0; - - if (ctx == NULL || (ctx->operation & op) == 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED); - /* Uses the same return values as EVP_PKEY_CTX_ctrl */ - return -2; - } - - /* Code below to be removed when legacy support is dropped. */ - if (fallback) - return EVP_PKEY_CTX_ctrl(ctx, -1, op, ctrl, datalen, (void *)(data)); - /* end of legacy support */ - - if (datalen < 0) { - ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_LENGTH); - return 0; - } else if (datalen == 0) { - return 1; - } - - /* Get the original value length */ - os_params[0] = OSSL_PARAM_construct_octet_string(param, NULL, 0); - os_params[1] = OSSL_PARAM_construct_end(); - - if (!EVP_PKEY_CTX_get_params(ctx, os_params)) - return 0; - - /* Older provider that doesn't support getting this parameter */ - if (os_params[0].return_size == OSSL_PARAM_UNMODIFIED) - return evp_pkey_ctx_set1_octet_string(ctx, fallback, param, op, ctrl, data, datalen); - - info_alloc = os_params[0].return_size + datalen; - if (info_alloc == 0) - return 0; - info = OPENSSL_zalloc(info_alloc); - if (info == NULL) - return 0; - info_len = os_params[0].return_size; - - os_params[0] = OSSL_PARAM_construct_octet_string(param, info, info_alloc); - - /* if we have data, then go get it */ - if (info_len > 0) { - if (!EVP_PKEY_CTX_get_params(ctx, os_params)) - goto error; - } - - /* Copy the input data */ - memcpy(&info[info_len], data, datalen); - ret = EVP_PKEY_CTX_set_params(ctx, os_params); - - error: - OPENSSL_clear_free(info, info_alloc); - return ret; -} - int EVP_PKEY_CTX_set1_tls1_prf_secret(EVP_PKEY_CTX *ctx, const unsigned char *sec, int seclen) { @@ -1117,7 +1066,7 @@ int EVP_PKEY_CTX_set1_hkdf_key(EVP_PKEY_CTX *ctx, int EVP_PKEY_CTX_add1_hkdf_info(EVP_PKEY_CTX *ctx, const unsigned char *info, int infolen) { - return evp_pkey_ctx_add1_octet_string(ctx, ctx->op.kex.algctx == NULL, + return evp_pkey_ctx_set1_octet_string(ctx, ctx->op.kex.algctx == NULL, OSSL_KDF_PARAM_INFO, EVP_PKEY_OP_DERIVE, EVP_PKEY_CTRL_HKDF_INFO, @@ -1251,12 +1200,77 @@ int EVP_PKEY_CTX_set_kem_op(EVP_PKEY_CTX *ctx, const char *op) return EVP_PKEY_CTX_set_params(ctx, params); } +int evp_pkey_ctx_set1_id_prov(EVP_PKEY_CTX *ctx, const void *id, int len) +{ + OSSL_PARAM params[2], *p = params; + int ret; + + if (!EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx)) { + ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED); + /* Uses the same return values as EVP_PKEY_CTX_ctrl */ + return -2; + } + + *p++ = OSSL_PARAM_construct_octet_string(OSSL_PKEY_PARAM_DIST_ID, + /* + * Cast away the const. This is + * read only so should be safe + */ + (void *)id, (size_t)len); + *p++ = OSSL_PARAM_construct_end(); + + ret = evp_pkey_ctx_set_params_strict(ctx, params); + if (ret == -2) + ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED); + return ret; +} + int EVP_PKEY_CTX_set1_id(EVP_PKEY_CTX *ctx, const void *id, int len) { return EVP_PKEY_CTX_ctrl(ctx, -1, -1, EVP_PKEY_CTRL_SET1_ID, (int)len, (void*)(id)); } +static int get1_id_data(EVP_PKEY_CTX *ctx, void *id, size_t *id_len) +{ + int ret; + void *tmp_id = NULL; + OSSL_PARAM params[2], *p = params; + + if (!EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx)) { + ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED); + /* Uses the same return values as EVP_PKEY_CTX_ctrl */ + return -2; + } + + *p++ = OSSL_PARAM_construct_octet_ptr(OSSL_PKEY_PARAM_DIST_ID, + &tmp_id, 0); + *p++ = OSSL_PARAM_construct_end(); + + ret = evp_pkey_ctx_get_params_strict(ctx, params); + if (ret == -2) { + ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED); + } else if (ret > 0) { + size_t tmp_id_len = params[0].return_size; + + if (id != NULL) + memcpy(id, tmp_id, tmp_id_len); + if (id_len != NULL) + *id_len = tmp_id_len; + } + return ret; +} + +int evp_pkey_ctx_get1_id_prov(EVP_PKEY_CTX *ctx, void *id) +{ + return get1_id_data(ctx, id, NULL); +} + +int evp_pkey_ctx_get1_id_len_prov(EVP_PKEY_CTX *ctx, size_t *id_len) +{ + return get1_id_data(ctx, NULL, id_len); +} + int EVP_PKEY_CTX_get1_id(EVP_PKEY_CTX *ctx, void *id) { return EVP_PKEY_CTX_ctrl(ctx, -1, -1, EVP_PKEY_CTRL_GET1_ID, 0, (void*)id); @@ -1472,13 +1486,17 @@ static int evp_pkey_ctx_store_cached_data(EVP_PKEY_CTX *ctx, evp_pkey_ctx_free_cached_data(ctx, cmd, name); if (name != NULL) { ctx->cached_parameters.dist_id_name = OPENSSL_strdup(name); - if (ctx->cached_parameters.dist_id_name == NULL) + if (ctx->cached_parameters.dist_id_name == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } } if (data_len > 0) { ctx->cached_parameters.dist_id = OPENSSL_memdup(data, data_len); - if (ctx->cached_parameters.dist_id == NULL) + if (ctx->cached_parameters.dist_id == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } } ctx->cached_parameters.dist_id_set = 1; ctx->cached_parameters.dist_id_len = data_len; diff --git a/openssl/src/crypto/evp/signature.c b/openssl/src/crypto/evp/signature.c index c05eb78b5..49f40c8ce 100644 --- a/openssl/src/crypto/evp/signature.c +++ b/openssl/src/crypto/evp/signature.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,16 +22,20 @@ static EVP_SIGNATURE *evp_signature_new(OSSL_PROVIDER *prov) { EVP_SIGNATURE *signature = OPENSSL_zalloc(sizeof(EVP_SIGNATURE)); - if (signature == NULL) + if (signature == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return NULL; + } - if (!CRYPTO_NEW_REF(&signature->refcnt, 1)) { + signature->lock = CRYPTO_THREAD_lock_new(); + if (signature->lock == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); OPENSSL_free(signature); return NULL; } - signature->prov = prov; ossl_provider_up_ref(prov); + signature->refcnt = 1; return signature; } @@ -47,7 +51,7 @@ static void *evp_signature_from_algorithm(int name_id, int gparamfncnt = 0, sparamfncnt = 0, gmdparamfncnt = 0, smdparamfncnt = 0; if ((signature = evp_signature_new(prov)) == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); goto err; } @@ -279,12 +283,12 @@ void EVP_SIGNATURE_free(EVP_SIGNATURE *signature) if (signature == NULL) return; - CRYPTO_DOWN_REF(&signature->refcnt, &i); + CRYPTO_DOWN_REF(&signature->refcnt, &i, signature->lock); if (i > 0) return; OPENSSL_free(signature->type_name); ossl_provider_free(signature->prov); - CRYPTO_FREE_REF(&signature->refcnt); + CRYPTO_THREAD_lock_free(signature->lock); OPENSSL_free(signature); } @@ -292,7 +296,7 @@ int EVP_SIGNATURE_up_ref(EVP_SIGNATURE *signature) { int ref = 0; - CRYPTO_UP_REF(&signature->refcnt, &ref); + CRYPTO_UP_REF(&signature->refcnt, &ref, signature->lock); return 1; } @@ -323,8 +327,7 @@ EVP_SIGNATURE *evp_signature_fetch_from_prov(OSSL_PROVIDER *prov, int EVP_SIGNATURE_is_a(const EVP_SIGNATURE *signature, const char *name) { - return signature != NULL - && evp_is_a(signature->prov, signature->name_id, NULL, name); + return evp_is_a(signature->prov, signature->name_id, NULL, name); } int evp_signature_get_number(const EVP_SIGNATURE *signature) @@ -399,8 +402,8 @@ static int evp_pkey_signature_init(EVP_PKEY_CTX *ctx, int operation, int iter; if (ctx == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER); - return -1; + ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); + return -2; } evp_pkey_ctx_free_old_ops(ctx); @@ -630,8 +633,8 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, int ret; if (ctx == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER); - return -1; + ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); + return -2; } if (ctx->operation != EVP_PKEY_OP_SIGN) { @@ -642,11 +645,6 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, if (ctx->op.sig.algctx == NULL) goto legacy; - if (ctx->op.sig.signature->sign == NULL) { - ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; - } - ret = ctx->op.sig.signature->sign(ctx->op.sig.algctx, sig, siglen, (sig == NULL) ? 0 : *siglen, tbs, tbslen); @@ -679,8 +677,8 @@ int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, int ret; if (ctx == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER); - return -1; + ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); + return -2; } if (ctx->operation != EVP_PKEY_OP_VERIFY) { @@ -691,11 +689,6 @@ int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, if (ctx->op.sig.algctx == NULL) goto legacy; - if (ctx->op.sig.signature->verify == NULL) { - ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; - } - ret = ctx->op.sig.signature->verify(ctx->op.sig.algctx, sig, siglen, tbs, tbslen); @@ -727,8 +720,8 @@ int EVP_PKEY_verify_recover(EVP_PKEY_CTX *ctx, int ret; if (ctx == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER); - return -1; + ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); + return -2; } if (ctx->operation != EVP_PKEY_OP_VERIFYRECOVER) { @@ -739,11 +732,6 @@ int EVP_PKEY_verify_recover(EVP_PKEY_CTX *ctx, if (ctx->op.sig.algctx == NULL) goto legacy; - if (ctx->op.sig.signature->verify_recover == NULL) { - ERR_raise(ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; - } - ret = ctx->op.sig.signature->verify_recover(ctx->op.sig.algctx, rout, routlen, (rout == NULL ? 0 : *routlen), diff --git a/openssl/src/crypto/ex_data.c b/openssl/src/crypto/ex_data.c index 0412f38e9..40223f06e 100644 --- a/openssl/src/crypto/ex_data.c +++ b/openssl/src/crypto/ex_data.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,10 +26,8 @@ int ossl_do_ex_data_init(OSSL_LIB_CTX *ctx) * Return the EX_CALLBACKS from the |ex_data| array that corresponds to * a given class. On success, *holds the lock.* * The |global| parameter is assumed to be non null (checked by the caller). - * If |read| is 1 then a read lock is obtained. Otherwise it is a write lock. */ -static EX_CALLBACKS *get_and_lock(OSSL_EX_DATA_GLOBAL *global, int class_index, - int read) +static EX_CALLBACKS *get_and_lock(OSSL_EX_DATA_GLOBAL *global, int class_index) { EX_CALLBACKS *ip; @@ -46,14 +44,8 @@ static EX_CALLBACKS *get_and_lock(OSSL_EX_DATA_GLOBAL *global, int class_index, return NULL; } - if (read) { - if (!CRYPTO_THREAD_read_lock(global->ex_data_lock)) - return NULL; - } else { - if (!CRYPTO_THREAD_write_lock(global->ex_data_lock)) - return NULL; - } - + if (!CRYPTO_THREAD_write_lock(global->ex_data_lock)) + return NULL; ip = &global->ex_data[class_index]; return ip; } @@ -120,7 +112,7 @@ int ossl_crypto_free_ex_index_ex(OSSL_LIB_CTX *ctx, int class_index, int idx) if (global == NULL) return 0; - ip = get_and_lock(global, class_index, 0); + ip = get_and_lock(global, class_index); if (ip == NULL) return 0; @@ -161,7 +153,7 @@ int ossl_crypto_get_ex_new_index_ex(OSSL_LIB_CTX *ctx, int class_index, if (global == NULL) return -1; - ip = get_and_lock(global, class_index, 0); + ip = get_and_lock(global, class_index); if (ip == NULL) return -1; @@ -171,16 +163,16 @@ int ossl_crypto_get_ex_new_index_ex(OSSL_LIB_CTX *ctx, int class_index, * "app_data" routines use ex_data index zero. See RT 3710. */ if (ip->meth == NULL || !sk_EX_CALLBACK_push(ip->meth, NULL)) { - sk_EX_CALLBACK_free(ip->meth); - ip->meth = NULL; - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto err; } } a = (EX_CALLBACK *)OPENSSL_malloc(sizeof(*a)); - if (a == NULL) + if (a == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto err; + } a->argl = argl; a->argp = argp; a->new_func = new_func; @@ -189,7 +181,7 @@ int ossl_crypto_get_ex_new_index_ex(OSSL_LIB_CTX *ctx, int class_index, a->priority = priority; if (!sk_EX_CALLBACK_push(ip->meth, NULL)) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); OPENSSL_free(a); goto err; } @@ -229,7 +221,7 @@ int ossl_crypto_new_ex_data_ex(OSSL_LIB_CTX *ctx, int class_index, void *obj, if (global == NULL) return 0; - ip = get_and_lock(global, class_index, 1); + ip = get_and_lock(global, class_index); if (ip == NULL) return 0; @@ -247,8 +239,10 @@ int ossl_crypto_new_ex_data_ex(OSSL_LIB_CTX *ctx, int class_index, void *obj, } CRYPTO_THREAD_unlock(global->ex_data_lock); - if (mx > 0 && storage == NULL) + if (mx > 0 && storage == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } for (i = 0; i < mx; i++) { if (storage[i] != NULL && storage[i]->new_func != NULL) { ptr = CRYPTO_get_ex_data(ad, i); @@ -290,7 +284,7 @@ int CRYPTO_dup_ex_data(int class_index, CRYPTO_EX_DATA *to, if (global == NULL) return 0; - ip = get_and_lock(global, class_index, 1); + ip = get_and_lock(global, class_index); if (ip == NULL) return 0; @@ -311,8 +305,10 @@ int CRYPTO_dup_ex_data(int class_index, CRYPTO_EX_DATA *to, if (mx == 0) return 1; - if (storage == NULL) + if (storage == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } /* * Make sure the ex_data stack is at least |mx| elements long to avoid * issues in the for loop that follows; so go get the |mx|'th element @@ -377,7 +373,7 @@ void CRYPTO_free_ex_data(int class_index, void *obj, CRYPTO_EX_DATA *ad) if (global == NULL) goto err; - ip = get_and_lock(global, class_index, 1); + ip = get_and_lock(global, class_index); if (ip == NULL) goto err; @@ -444,7 +440,7 @@ int ossl_crypto_alloc_ex_data_intern(int class_index, void *obj, if (global == NULL) return 0; - ip = get_and_lock(global, class_index, 1); + ip = get_and_lock(global, class_index); if (ip == NULL) return 0; f = sk_EX_CALLBACK_value(ip->meth, idx); @@ -472,14 +468,14 @@ int CRYPTO_set_ex_data(CRYPTO_EX_DATA *ad, int idx, void *val) if (ad->sk == NULL) { if ((ad->sk = sk_void_new_null()) == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; } } for (i = sk_void_num(ad->sk); i <= idx; ++i) { if (!sk_void_push(ad->sk, NULL)) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; } } diff --git a/openssl/src/crypto/ffc/ffc_backend.c b/openssl/src/crypto/ffc/ffc_backend.c index c12a88148..9a013d95d 100644 --- a/openssl/src/crypto/ffc/ffc_backend.c +++ b/openssl/src/crypto/ffc/ffc_backend.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,6 +24,9 @@ int ossl_ffc_params_fromdata(FFC_PARAMS *ffc, const OSSL_PARAM params[]) BIGNUM *p = NULL, *q = NULL, *g = NULL, *j = NULL; int i; + if (ffc == NULL) + return 0; + prm = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_GROUP_NAME); if (prm != NULL) { /* @@ -36,7 +39,7 @@ int ossl_ffc_params_fromdata(FFC_PARAMS *ffc, const OSSL_PARAM params[]) if (prm->data_type != OSSL_PARAM_UTF8_STRING || prm->data == NULL || (group = ossl_ffc_name_to_dh_named_group(prm->data)) == NULL - || !ossl_ffc_named_group_set(ffc, group)) + || !ossl_ffc_named_group_set_pqg(ffc, group)) #endif goto err; } @@ -73,8 +76,9 @@ int ossl_ffc_params_fromdata(FFC_PARAMS *ffc, const OSSL_PARAM params[]) } prm = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_FFC_SEED); if (prm != NULL) { - if (prm->data_type != OSSL_PARAM_OCTET_STRING - || !ossl_ffc_params_set_seed(ffc, prm->data, prm->data_size)) + if (prm->data_type != OSSL_PARAM_OCTET_STRING) + goto err; + if (!ossl_ffc_params_set_seed(ffc, prm->data, prm->data_size)) goto err; } prm = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_FFC_VALIDATE_PQ); @@ -107,10 +111,11 @@ int ossl_ffc_params_fromdata(FFC_PARAMS *ffc, const OSSL_PARAM params[]) if (p1 != NULL) { if (p1->data_type != OSSL_PARAM_UTF8_STRING) goto err; - props = p1->data; } - ossl_ffc_set_digest(ffc, prm->data, props); + if (!ossl_ffc_set_digest(ffc, prm->data, props)) + goto err; } + ossl_ffc_params_set0_pqg(ffc, p, q, g); ossl_ffc_params_set0_j(ffc, j); return 1; diff --git a/openssl/src/crypto/ffc/ffc_dh.c b/openssl/src/crypto/ffc/ffc_dh.c index df07e173b..9a7e99cff 100644 --- a/openssl/src/crypto/ffc/ffc_dh.c +++ b/openssl/src/crypto/ffc/ffc_dh.c @@ -13,18 +13,16 @@ #ifndef OPENSSL_NO_DH -# define FFDHE(sz, keylength) { \ +# define FFDHE(sz) { \ SN_ffdhe##sz, NID_ffdhe##sz, \ sz, \ - keylength, \ &ossl_bignum_ffdhe##sz##_p, &ossl_bignum_ffdhe##sz##_q, \ &ossl_bignum_const_2, \ } -# define MODP(sz, keylength) { \ +# define MODP(sz) { \ SN_modp_##sz, NID_modp_##sz, \ sz, \ - keylength, \ &ossl_bignum_modp_##sz##_p, &ossl_bignum_modp_##sz##_q, \ &ossl_bignum_const_2 \ } @@ -32,15 +30,14 @@ # define RFC5114(name, uid, sz, tag) { \ name, uid, \ sz, \ - 0, \ &ossl_bignum_dh##tag##_p, &ossl_bignum_dh##tag##_q, \ &ossl_bignum_dh##tag##_g \ } #else -# define FFDHE(sz, keylength) { SN_ffdhe##sz, NID_ffdhe##sz } -# define MODP(sz, keylength) { SN_modp_##sz, NID_modp_##sz } +# define FFDHE(sz) { SN_ffdhe##sz, NID_ffdhe##sz } +# define MODP(sz) { SN_modp_##sz, NID_modp_##sz } # define RFC5114(name, uid, sz, tag) { name, uid } #endif @@ -50,32 +47,26 @@ struct dh_named_group_st { int uid; #ifndef OPENSSL_NO_DH int32_t nbits; - int keylength; const BIGNUM *p; const BIGNUM *q; const BIGNUM *g; #endif }; -/* - * The private key length values are taken from RFC7919 with the values for - * MODP primes given the same lengths as the equivalent FFDHE. - * The MODP 1536 value is approximated. - */ static const DH_NAMED_GROUP dh_named_groups[] = { - FFDHE(2048, 225), - FFDHE(3072, 275), - FFDHE(4096, 325), - FFDHE(6144, 375), - FFDHE(8192, 400), + FFDHE(2048), + FFDHE(3072), + FFDHE(4096), + FFDHE(6144), + FFDHE(8192), #ifndef FIPS_MODULE - MODP(1536, 200), + MODP(1536), #endif - MODP(2048, 225), - MODP(3072, 275), - MODP(4096, 325), - MODP(6144, 375), - MODP(8192, 400), + MODP(2048), + MODP(3072), + MODP(4096), + MODP(6144), + MODP(8192), /* * Additional dh named groups from RFC 5114 that have a different g. * The uid can be any unique identifier. @@ -143,13 +134,6 @@ const char *ossl_ffc_named_group_get_name(const DH_NAMED_GROUP *group) } #ifndef OPENSSL_NO_DH -int ossl_ffc_named_group_get_keylength(const DH_NAMED_GROUP *group) -{ - if (group == NULL) - return 0; - return group->keylength; -} - const BIGNUM *ossl_ffc_named_group_get_q(const DH_NAMED_GROUP *group) { if (group == NULL) @@ -157,14 +141,13 @@ const BIGNUM *ossl_ffc_named_group_get_q(const DH_NAMED_GROUP *group) return group->q; } -int ossl_ffc_named_group_set(FFC_PARAMS *ffc, const DH_NAMED_GROUP *group) +int ossl_ffc_named_group_set_pqg(FFC_PARAMS *ffc, const DH_NAMED_GROUP *group) { if (ffc == NULL || group == NULL) return 0; ossl_ffc_params_set0_pqg(ffc, (BIGNUM *)group->p, (BIGNUM *)group->q, (BIGNUM *)group->g); - ffc->keylength = group->keylength; /* flush the cached nid, The DH layer is responsible for caching */ ffc->nid = NID_undef; diff --git a/openssl/src/crypto/ffc/ffc_key_generate.c b/openssl/src/crypto/ffc/ffc_key_generate.c index cb895f2ab..c18f349ee 100644 --- a/openssl/src/crypto/ffc/ffc_key_generate.c +++ b/openssl/src/crypto/ffc/ffc_key_generate.c @@ -25,11 +25,11 @@ int ossl_ffc_generate_private_key(BN_CTX *ctx, const FFC_PARAMS *params, int ret = 0, qbits = BN_num_bits(params->q); BIGNUM *m, *two_powN = NULL; - /* Deal with the edge cases where the value of N and/or s is not set */ - if (s == 0) - goto err; + /* Deal with the edge case where the value of N is not set */ if (N == 0) - N = params->keylength ? params->keylength : 2 * s; + N = qbits; + if (s == 0) + s = N / 2; /* Step (2) : check range of N */ if (N < 2 * s || N > qbits) diff --git a/openssl/src/crypto/ffc/ffc_key_validate.c b/openssl/src/crypto/ffc/ffc_key_validate.c index a4a2a58e9..442303e4b 100644 --- a/openssl/src/crypto/ffc/ffc_key_validate.c +++ b/openssl/src/crypto/ffc/ffc_key_validate.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,7 +26,7 @@ int ossl_ffc_validate_public_key_partial(const FFC_PARAMS *params, *ret = 0; if (params == NULL || pub_key == NULL || params->p == NULL) { *ret = FFC_ERROR_PASSED_NULL_PARAM; - return 1; + return 0; } ctx = BN_CTX_new_ex(NULL); @@ -39,14 +39,18 @@ int ossl_ffc_validate_public_key_partial(const FFC_PARAMS *params, if (tmp == NULL || !BN_set_word(tmp, 1)) goto err; - if (BN_cmp(pub_key, tmp) <= 0) + if (BN_cmp(pub_key, tmp) <= 0) { *ret |= FFC_ERROR_PUBKEY_TOO_SMALL; + goto err; + } /* Step(1): Verify pub_key <= p-2 */ if (BN_copy(tmp, params->p) == NULL || !BN_sub_word(tmp, 1)) goto err; - if (BN_cmp(pub_key, tmp) >= 0) + if (BN_cmp(pub_key, tmp) >= 0) { *ret |= FFC_ERROR_PUBKEY_TOO_LARGE; + goto err; + } ok = 1; err: if (ctx != NULL) { @@ -69,7 +73,7 @@ int ossl_ffc_validate_public_key(const FFC_PARAMS *params, if (!ossl_ffc_validate_public_key_partial(params, pub_key, ret)) return 0; - if (*ret == 0 && params->q != NULL) { + if (params->q != NULL) { ctx = BN_CTX_new_ex(NULL); if (ctx == NULL) goto err; @@ -80,8 +84,10 @@ int ossl_ffc_validate_public_key(const FFC_PARAMS *params, if (tmp == NULL || !BN_mod_exp(tmp, pub_key, params->q, params->p, ctx)) goto err; - if (!BN_is_one(tmp)) + if (!BN_is_one(tmp)) { *ret |= FFC_ERROR_PUBKEY_INVALID; + goto err; + } } ok = 1; diff --git a/openssl/src/crypto/ffc/ffc_params.c b/openssl/src/crypto/ffc/ffc_params.c index 680f85ffa..073f661c7 100644 --- a/openssl/src/crypto/ffc/ffc_params.c +++ b/openssl/src/crypto/ffc/ffc_params.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -75,6 +75,9 @@ void ossl_ffc_params_set0_j(FFC_PARAMS *d, BIGNUM *j) int ossl_ffc_params_set_seed(FFC_PARAMS *params, const unsigned char *seed, size_t seedlen) { + if (params == NULL) + return 0; + if (params->seed != NULL) { if (params->seed == seed) return 1; @@ -122,10 +125,11 @@ void ossl_ffc_params_enable_flags(FFC_PARAMS *params, unsigned int flags, params->flags &= ~flags; } -void ossl_ffc_set_digest(FFC_PARAMS *params, const char *alg, const char *props) +int ossl_ffc_set_digest(FFC_PARAMS *params, const char *alg, const char *props) { params->mdname = alg; params->mdprops = props; + return 1; } int ossl_ffc_params_set_validate_params(FFC_PARAMS *params, @@ -178,8 +182,6 @@ int ossl_ffc_params_copy(FFC_PARAMS *dst, const FFC_PARAMS *src) || !ffc_bn_cpy(&dst->j, src->j)) return 0; - dst->mdname = src->mdname; - dst->mdprops = src->mdprops; OPENSSL_free(dst->seed); dst->seedlen = src->seedlen; if (src->seed != NULL) { @@ -194,7 +196,6 @@ int ossl_ffc_params_copy(FFC_PARAMS *dst, const FFC_PARAMS *src) dst->h = src->h; dst->gindex = src->gindex; dst->flags = src->flags; - dst->keylength = src->keylength; return 1; } @@ -210,6 +211,9 @@ int ossl_ffc_params_todata(const FFC_PARAMS *ffc, OSSL_PARAM_BLD *bld, { int test_flags; + if (ffc == NULL) + return 0; + if (ffc->p != NULL && !ossl_param_build_set_bn(bld, params, OSSL_PKEY_PARAM_FFC_P, ffc->p)) return 0; diff --git a/openssl/src/crypto/ffc/ffc_params_generate.c b/openssl/src/crypto/ffc/ffc_params_generate.c index 8294fbec3..f0601e164 100644 --- a/openssl/src/crypto/ffc/ffc_params_generate.c +++ b/openssl/src/crypto/ffc/ffc_params_generate.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -324,12 +324,12 @@ static int generate_q_fips186_4(BN_CTX *ctx, BIGNUM *q, const EVP_MD *evpmd, /* find q */ for (;;) { - if (!BN_GENCB_call(cb, 0, m++)) + if(!BN_GENCB_call(cb, 0, m++)) goto err; /* A.1.1.2 Step (5) : generate seed with size seed_len */ if (generate_seed - && RAND_bytes_ex(libctx, seed, seedlen, 0) <= 0) + && RAND_bytes_ex(libctx, seed, seedlen, 0) < 0) goto err; /* * A.1.1.2 Step (6) AND @@ -435,7 +435,7 @@ static int generate_q_fips186_2(BN_CTX *ctx, BIGNUM *q, const EVP_MD *evpmd, } if (r != 0) goto err; /* Exit if error */ - /* Try another iteration if it wasn't prime - was in old code.. */ + /* Try another iteration if it wasnt prime - was in old code.. */ generate_seed = 1; } err: @@ -621,7 +621,7 @@ int ossl_ffc_params_FIPS186_4_gen_verify(OSSL_LIB_CTX *libctx, p = params->p; q = params->q; goto g_only; - /* otherwise fall through to validate p & q */ + /* otherwise fall thru to validate p & q */ } /* p & q will be used for generation and validation */ @@ -673,7 +673,7 @@ int ossl_ffc_params_FIPS186_4_gen_verify(OSSL_LIB_CTX *libctx, * A.1.1.3 Step (10) * n = floor(L / hash_outlen) - 1 */ - n = (L - 1) / (mdsize << 3); + n = (L - 1 ) / (mdsize << 3); /* Calculate 2^(L-1): Used in step A.1.1.2 Step (11.3) */ if (!BN_lshift(test, BN_value_one(), L - 1)) @@ -688,9 +688,9 @@ int ossl_ffc_params_FIPS186_4_gen_verify(OSSL_LIB_CTX *libctx, *res = FFC_CHECK_Q_MISMATCH; goto err; } - if (!BN_GENCB_call(cb, 2, 0)) + if(!BN_GENCB_call(cb, 2, 0)) goto err; - if (!BN_GENCB_call(cb, 3, 0)) + if(!BN_GENCB_call(cb, 3, 0)) goto err; memcpy(seed_tmp, seed, seedlen); @@ -922,7 +922,7 @@ int ossl_ffc_params_FIPS186_2_gen_verify(OSSL_LIB_CTX *libctx, p = params->p; q = params->q; goto g_only; - /* otherwise fall through to validate p and q */ + /* otherwise fall thru to validate p and q */ } use_random_seed = (seed_in == NULL); diff --git a/openssl/src/crypto/gen/darwin_arm64/arm64cpuid.S b/openssl/src/crypto/gen/darwin_arm64/arm64cpuid.S index b3d076474..ebc6a62b3 100644 --- a/openssl/src/crypto/gen/darwin_arm64/arm64cpuid.S +++ b/openssl/src/crypto/gen/darwin_arm64/arm64cpuid.S @@ -7,7 +7,6 @@ .globl __armv7_neon_probe __armv7_neon_probe: - AARCH64_VALID_CALL_TARGET orr v15.16b, v15.16b, v15.16b ret @@ -15,7 +14,6 @@ __armv7_neon_probe: .globl __armv7_tick __armv7_tick: - AARCH64_VALID_CALL_TARGET #ifdef __APPLE__ mrs x0, CNTPCT_EL0 #else @@ -27,7 +25,6 @@ __armv7_tick: .globl __armv8_aes_probe __armv8_aes_probe: - AARCH64_VALID_CALL_TARGET aese v0.16b, v0.16b ret @@ -35,7 +32,6 @@ __armv8_aes_probe: .globl __armv8_sha1_probe __armv8_sha1_probe: - AARCH64_VALID_CALL_TARGET sha1h s0, s0 ret @@ -43,7 +39,6 @@ __armv8_sha1_probe: .globl __armv8_sha256_probe __armv8_sha256_probe: - AARCH64_VALID_CALL_TARGET sha256su0 v0.4s, v0.4s ret @@ -51,7 +46,6 @@ __armv8_sha256_probe: .globl __armv8_pmull_probe __armv8_pmull_probe: - AARCH64_VALID_CALL_TARGET pmull v0.1q, v0.1d, v0.1d ret @@ -59,7 +53,6 @@ __armv8_pmull_probe: .globl __armv8_sm4_probe __armv8_sm4_probe: - AARCH64_VALID_CALL_TARGET .long 0xcec08400 // sm4e v0.4s, v0.4s ret @@ -67,39 +60,13 @@ __armv8_sm4_probe: .globl __armv8_sha512_probe __armv8_sha512_probe: - AARCH64_VALID_CALL_TARGET .long 0xcec08000 // sha512su0 v0.2d,v0.2d ret -.globl __armv8_eor3_probe - -__armv8_eor3_probe: - AARCH64_VALID_CALL_TARGET -.long 0xce010800 // eor3 v0.16b, v0.16b, v1.16b, v2.16b - ret - - -.globl __armv8_sve_probe - -__armv8_sve_probe: - AARCH64_VALID_CALL_TARGET -.long 0x04a03000 // eor z0.d,z0.d,z0.d - ret - - -.globl __armv8_sve2_probe - -__armv8_sve2_probe: - AARCH64_VALID_CALL_TARGET -.long 0x04e03400 // xar z0.d,z0.d,z0.d - ret - - .globl __armv8_cpuid_probe __armv8_cpuid_probe: - AARCH64_VALID_CALL_TARGET mrs x0, midr_el1 ret @@ -107,7 +74,6 @@ __armv8_cpuid_probe: .globl __armv8_sm3_probe __armv8_sm3_probe: - AARCH64_VALID_CALL_TARGET .long 0xce63c004 // sm3partw1 v4.4s, v0.4s, v3.4s ret @@ -116,7 +82,6 @@ __armv8_sm3_probe: .align 5 _OPENSSL_cleanse: - AARCH64_VALID_CALL_TARGET cbz x1,Lret // len==0? cmp x1,#15 b.hi Lot // len>15 @@ -148,7 +113,6 @@ Laligned: .align 4 _CRYPTO_memcmp: - AARCH64_VALID_CALL_TARGET eor w3,w3,w3 cbz x2,Lno_data // len==0? cmp x2,#16 @@ -177,98 +141,3 @@ Lno_data: lsr w0,w0,#31 ret - -.globl __armv8_rng_probe - -__armv8_rng_probe: - AARCH64_VALID_CALL_TARGET - mrs x0, s3_3_c2_c4_0 // rndr - mrs x0, s3_3_c2_c4_1 // rndrrs - ret - -// Fill buffer with Randomly Generated Bytes -// inputs: char * in x0 - Pointer to buffer -// size_t in x1 - Number of bytes to write to buffer -// outputs: size_t in x0 - Number of bytes successfully written to buffer -.globl _OPENSSL_rndr_asm - -.align 4 -_OPENSSL_rndr_asm: - AARCH64_VALID_CALL_TARGET - mov x2,xzr - mov x3,xzr - -.align 4 -Loop_rndr: - cmp x1,#0 - b.eq .rndr_done - mov x3,xzr - mrs x3,s3_3_c2_c4_0 - b.eq .rndr_done - - cmp x1,#8 - b.lt Loop_single_byte_rndr - - str x3,[x0] - add x0,x0,#8 - add x2,x2,#8 - subs x1,x1,#8 - b.ge Loop_rndr - -.align 4 -Loop_single_byte_rndr: - strb w3,[x0] - lsr x3,x3,#8 - add x2,x2,#1 - add x0,x0,#1 - subs x1,x1,#1 - b.gt Loop_single_byte_rndr - -.align 4 -.rndr_done: - mov x0,x2 - ret - -// Fill buffer with Randomly Generated Bytes -// inputs: char * in x0 - Pointer to buffer -// size_t in x1 - Number of bytes to write to buffer -// outputs: size_t in x0 - Number of bytes successfully written to buffer -.globl _OPENSSL_rndrrs_asm - -.align 4 -_OPENSSL_rndrrs_asm: - AARCH64_VALID_CALL_TARGET - mov x2,xzr - mov x3,xzr - -.align 4 -Loop_rndrrs: - cmp x1,#0 - b.eq .rndrrs_done - mov x3,xzr - mrs x3,s3_3_c2_c4_1 - b.eq .rndrrs_done - - cmp x1,#8 - b.lt Loop_single_byte_rndrrs - - str x3,[x0] - add x0,x0,#8 - add x2,x2,#8 - subs x1,x1,#8 - b.ge Loop_rndrrs - -.align 4 -Loop_single_byte_rndrrs: - strb w3,[x0] - lsr x3,x3,#8 - add x2,x2,#1 - add x0,x0,#1 - subs x1,x1,#1 - b.gt Loop_single_byte_rndrrs - -.align 4 -.rndrrs_done: - mov x0,x2 - ret - diff --git a/openssl/src/crypto/gen/darwin_arm64/params_idx.c b/openssl/src/crypto/gen/darwin_arm64/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/darwin_arm64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/darwin_x64/params_idx.c b/openssl/src/crypto/gen/darwin_x64/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/darwin_x64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/linux_arm/armv4cpuid.S b/openssl/src/crypto/gen/linux_arm/armv4cpuid.S index 01e910bb0..2984c8b03 100644 --- a/openssl/src/crypto/gen/linux_arm/armv4cpuid.S +++ b/openssl/src/crypto/gen/linux_arm/armv4cpuid.S @@ -268,4 +268,5 @@ atomic_add_spinlock: .word 0 #endif - +.comm OPENSSL_armcap_P,4,4 +.hidden OPENSSL_armcap_P diff --git a/openssl/src/crypto/gen/linux_arm/params_idx.c b/openssl/src/crypto/gen/linux_arm/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/linux_arm/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/linux_arm64/arm64cpuid.S b/openssl/src/crypto/gen/linux_arm64/arm64cpuid.S index 5cc56673f..86067da80 100644 --- a/openssl/src/crypto/gen/linux_arm64/arm64cpuid.S +++ b/openssl/src/crypto/gen/linux_arm64/arm64cpuid.S @@ -7,7 +7,6 @@ .globl _armv7_neon_probe .type _armv7_neon_probe,%function _armv7_neon_probe: - AARCH64_VALID_CALL_TARGET orr v15.16b, v15.16b, v15.16b ret .size _armv7_neon_probe,.-_armv7_neon_probe @@ -15,7 +14,6 @@ _armv7_neon_probe: .globl _armv7_tick .type _armv7_tick,%function _armv7_tick: - AARCH64_VALID_CALL_TARGET #ifdef __APPLE__ mrs x0, CNTPCT_EL0 #else @@ -27,7 +25,6 @@ _armv7_tick: .globl _armv8_aes_probe .type _armv8_aes_probe,%function _armv8_aes_probe: - AARCH64_VALID_CALL_TARGET aese v0.16b, v0.16b ret .size _armv8_aes_probe,.-_armv8_aes_probe @@ -35,7 +32,6 @@ _armv8_aes_probe: .globl _armv8_sha1_probe .type _armv8_sha1_probe,%function _armv8_sha1_probe: - AARCH64_VALID_CALL_TARGET sha1h s0, s0 ret .size _armv8_sha1_probe,.-_armv8_sha1_probe @@ -43,7 +39,6 @@ _armv8_sha1_probe: .globl _armv8_sha256_probe .type _armv8_sha256_probe,%function _armv8_sha256_probe: - AARCH64_VALID_CALL_TARGET sha256su0 v0.4s, v0.4s ret .size _armv8_sha256_probe,.-_armv8_sha256_probe @@ -51,7 +46,6 @@ _armv8_sha256_probe: .globl _armv8_pmull_probe .type _armv8_pmull_probe,%function _armv8_pmull_probe: - AARCH64_VALID_CALL_TARGET pmull v0.1q, v0.1d, v0.1d ret .size _armv8_pmull_probe,.-_armv8_pmull_probe @@ -59,47 +53,20 @@ _armv8_pmull_probe: .globl _armv8_sm4_probe .type _armv8_sm4_probe,%function _armv8_sm4_probe: - AARCH64_VALID_CALL_TARGET -.inst 0xcec08400 // sm4e v0.4s, v0.4s +.long 0xcec08400 // sm4e v0.4s, v0.4s ret .size _armv8_sm4_probe,.-_armv8_sm4_probe .globl _armv8_sha512_probe .type _armv8_sha512_probe,%function _armv8_sha512_probe: - AARCH64_VALID_CALL_TARGET -.inst 0xcec08000 // sha512su0 v0.2d,v0.2d +.long 0xcec08000 // sha512su0 v0.2d,v0.2d ret .size _armv8_sha512_probe,.-_armv8_sha512_probe -.globl _armv8_eor3_probe -.type _armv8_eor3_probe,%function -_armv8_eor3_probe: - AARCH64_VALID_CALL_TARGET -.inst 0xce010800 // eor3 v0.16b, v0.16b, v1.16b, v2.16b - ret -.size _armv8_eor3_probe,.-_armv8_eor3_probe - -.globl _armv8_sve_probe -.type _armv8_sve_probe,%function -_armv8_sve_probe: - AARCH64_VALID_CALL_TARGET -.inst 0x04a03000 // eor z0.d,z0.d,z0.d - ret -.size _armv8_sve_probe,.-_armv8_sve_probe - -.globl _armv8_sve2_probe -.type _armv8_sve2_probe,%function -_armv8_sve2_probe: - AARCH64_VALID_CALL_TARGET -.inst 0x04e03400 // xar z0.d,z0.d,z0.d - ret -.size _armv8_sve2_probe,.-_armv8_sve2_probe - .globl _armv8_cpuid_probe .type _armv8_cpuid_probe,%function _armv8_cpuid_probe: - AARCH64_VALID_CALL_TARGET mrs x0, midr_el1 ret .size _armv8_cpuid_probe,.-_armv8_cpuid_probe @@ -107,8 +74,7 @@ _armv8_cpuid_probe: .globl _armv8_sm3_probe .type _armv8_sm3_probe,%function _armv8_sm3_probe: - AARCH64_VALID_CALL_TARGET -.inst 0xce63c004 // sm3partw1 v4.4s, v0.4s, v3.4s +.long 0xce63c004 // sm3partw1 v4.4s, v0.4s, v3.4s ret .size _armv8_sm3_probe,.-_armv8_sm3_probe @@ -116,7 +82,6 @@ _armv8_sm3_probe: .type OPENSSL_cleanse,%function .align 5 OPENSSL_cleanse: - AARCH64_VALID_CALL_TARGET cbz x1,.Lret // len==0? cmp x1,#15 b.hi .Lot // len>15 @@ -148,7 +113,6 @@ OPENSSL_cleanse: .type CRYPTO_memcmp,%function .align 4 CRYPTO_memcmp: - AARCH64_VALID_CALL_TARGET eor w3,w3,w3 cbz x2,.Lno_data // len==0? cmp x2,#16 @@ -177,98 +141,3 @@ CRYPTO_memcmp: lsr w0,w0,#31 ret .size CRYPTO_memcmp,.-CRYPTO_memcmp - -.globl _armv8_rng_probe -.type _armv8_rng_probe,%function -_armv8_rng_probe: - AARCH64_VALID_CALL_TARGET - mrs x0, s3_3_c2_c4_0 // rndr - mrs x0, s3_3_c2_c4_1 // rndrrs - ret -.size _armv8_rng_probe,.-_armv8_rng_probe -// Fill buffer with Randomly Generated Bytes -// inputs: char * in x0 - Pointer to buffer -// size_t in x1 - Number of bytes to write to buffer -// outputs: size_t in x0 - Number of bytes successfully written to buffer -.globl OPENSSL_rndr_asm -.type OPENSSL_rndr_asm,%function -.align 4 -OPENSSL_rndr_asm: - AARCH64_VALID_CALL_TARGET - mov x2,xzr - mov x3,xzr - -.align 4 -.Loop_rndr: - cmp x1,#0 - b.eq .rndr_done - mov x3,xzr - mrs x3,s3_3_c2_c4_0 - b.eq .rndr_done - - cmp x1,#8 - b.lt .Loop_single_byte_rndr - - str x3,[x0] - add x0,x0,#8 - add x2,x2,#8 - subs x1,x1,#8 - b.ge .Loop_rndr - -.align 4 -.Loop_single_byte_rndr: - strb w3,[x0] - lsr x3,x3,#8 - add x2,x2,#1 - add x0,x0,#1 - subs x1,x1,#1 - b.gt .Loop_single_byte_rndr - -.align 4 -.rndr_done: - mov x0,x2 - ret -.size OPENSSL_rndr_asm,.-OPENSSL_rndr_asm -// Fill buffer with Randomly Generated Bytes -// inputs: char * in x0 - Pointer to buffer -// size_t in x1 - Number of bytes to write to buffer -// outputs: size_t in x0 - Number of bytes successfully written to buffer -.globl OPENSSL_rndrrs_asm -.type OPENSSL_rndrrs_asm,%function -.align 4 -OPENSSL_rndrrs_asm: - AARCH64_VALID_CALL_TARGET - mov x2,xzr - mov x3,xzr - -.align 4 -.Loop_rndrrs: - cmp x1,#0 - b.eq .rndrrs_done - mov x3,xzr - mrs x3,s3_3_c2_c4_1 - b.eq .rndrrs_done - - cmp x1,#8 - b.lt .Loop_single_byte_rndrrs - - str x3,[x0] - add x0,x0,#8 - add x2,x2,#8 - subs x1,x1,#8 - b.ge .Loop_rndrrs - -.align 4 -.Loop_single_byte_rndrrs: - strb w3,[x0] - lsr x3,x3,#8 - add x2,x2,#1 - add x0,x0,#1 - subs x1,x1,#1 - b.gt .Loop_single_byte_rndrrs - -.align 4 -.rndrrs_done: - mov x0,x2 - ret -.size OPENSSL_rndrrs_asm,.-OPENSSL_rndrrs_asm diff --git a/openssl/src/crypto/gen/linux_arm64/params_idx.c b/openssl/src/crypto/gen/linux_arm64/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/linux_arm64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/linux_ia32/params_idx.c b/openssl/src/crypto/gen/linux_ia32/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/linux_ia32/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/linux_ia32/x86cpuid.S b/openssl/src/crypto/gen/linux_ia32/x86cpuid.S index d4f2ec09e..b826117f7 100644 --- a/openssl/src/crypto/gen/linux_ia32/x86cpuid.S +++ b/openssl/src/crypto/gen/linux_ia32/x86cpuid.S @@ -4,11 +4,7 @@ .align 16 OPENSSL_ia32_cpuid: .L_OPENSSL_ia32_cpuid_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -153,11 +149,7 @@ OPENSSL_ia32_cpuid: .align 16 OPENSSL_rdtsc: .L_OPENSSL_rdtsc_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - xorl %eax,%eax xorl %edx,%edx call .L009PIC_me_up @@ -175,11 +167,7 @@ OPENSSL_rdtsc: .align 16 OPENSSL_instrument_halt: .L_OPENSSL_instrument_halt_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - call .L011PIC_me_up .L011PIC_me_up: popl %ecx @@ -212,11 +200,7 @@ OPENSSL_instrument_halt: .align 16 OPENSSL_far_spin: .L_OPENSSL_far_spin_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushfl popl %eax btl $9,%eax @@ -244,11 +228,7 @@ OPENSSL_far_spin: .align 16 OPENSSL_wipe_cpu: .L_OPENSSL_wipe_cpu_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - xorl %eax,%eax xorl %edx,%edx call .L015PIC_me_up @@ -280,11 +260,7 @@ OPENSSL_wipe_cpu: .align 16 OPENSSL_atomic_add: .L_OPENSSL_atomic_add_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%edx movl 8(%esp),%ecx pushl %ebx @@ -304,11 +280,7 @@ OPENSSL_atomic_add: .align 16 OPENSSL_cleanse: .L_OPENSSL_cleanse_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%edx movl 8(%esp),%ecx xorl %eax,%eax @@ -346,11 +318,7 @@ OPENSSL_cleanse: .align 16 CRYPTO_memcmp: .L_CRYPTO_memcmp_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %esi pushl %edi movl 12(%esp),%esi @@ -380,11 +348,7 @@ CRYPTO_memcmp: .align 16 OPENSSL_instrument_bus: .L_OPENSSL_instrument_bus_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -433,11 +397,7 @@ OPENSSL_instrument_bus: .align 16 OPENSSL_instrument_bus2: .L_OPENSSL_instrument_bus2_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -499,11 +459,7 @@ OPENSSL_instrument_bus2: .align 16 OPENSSL_ia32_rdrand_bytes: .L_OPENSSL_ia32_rdrand_bytes_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %edi pushl %ebx xorl %eax,%eax @@ -547,11 +503,7 @@ OPENSSL_ia32_rdrand_bytes: .align 16 OPENSSL_ia32_rdseed_bytes: .L_OPENSSL_ia32_rdseed_bytes_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %edi pushl %ebx xorl %eax,%eax diff --git a/openssl/src/crypto/gen/linux_loong64/loongarch64cpuid.s b/openssl/src/crypto/gen/linux_loong64/loongarch64cpuid.s deleted file mode 100644 index b8ba7d3e8..000000000 --- a/openssl/src/crypto/gen/linux_loong64/loongarch64cpuid.s +++ /dev/null @@ -1,69 +0,0 @@ -################################################################################ -# int CRYPTO_memcmp(const void * in_a, const void * in_b, size_t len) -################################################################################ -.text -.balign 16 -.globl CRYPTO_memcmp -.type CRYPTO_memcmp,@function -CRYPTO_memcmp: - li.d $r12,0 - beqz $r6,2f # len == 0 -1: - ld.bu $r13,$r4,0 - ld.bu $r14,$r5,0 - addi.d $r4,$r4,1 - addi.d $r5,$r5,1 - addi.d $r6,$r6,-1 - xor $r13,$r13,$r14 - or $r12,$r12,$r13 - blt $r0,$r6,1b -2: - move $r4,$r12 - jr $r1 -################################################################################ -# void OPENSSL_cleanse(void *ptr, size_t len) -################################################################################ -.text -.balign 16 -.globl OPENSSL_cleanse -.type OPENSSL_cleanse,@function -OPENSSL_cleanse: - beqz $r5,2f # len == 0, return - srli.d $r12,$r5,4 - bnez $r12,3f # len > 15 - -1: # Store <= 15 individual bytes - st.b $r0,$r4,0 - addi.d $r4,$r4,1 - addi.d $r5,$r5,-1 - bnez $r5,1b -2: - jr $r1 - -3: # Store individual bytes until we are aligned - andi $r12,$r4,0x7 - beqz $r12,4f - st.b $r0,$r4,0 - addi.d $r4,$r4,1 - addi.d $r5,$r5,-1 - b 3b - -4: # Store aligned dwords - li.d $r13,8 -4: - st.d $r0,$r4,0 - addi.d $r4,$r4,8 - addi.d $r5,$r5,-8 - bge $r5,$r13,4b # if len>=8 loop - bnez $r5,1b # if len<8 and len != 0, store remaining bytes - jr $r1 -################################################################################ -# uint32_t OPENSSL_rdtsc(void) -################################################################################ -.text -.balign 16 -.globl OPENSSL_rdtsc -.type OPENSSL_rdtsc,@function -OPENSSL_rdtsc: - rdtimel.w $r4,$r0 - jr $r1 diff --git a/openssl/src/crypto/gen/linux_loong64/params_idx.c b/openssl/src/crypto/gen/linux_loong64/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/linux_loong64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/linux_mips64/params_idx.c b/openssl/src/crypto/gen/linux_mips64/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/linux_mips64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/linux_ppc64/params_idx.c b/openssl/src/crypto/gen/linux_ppc64/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/linux_ppc64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/linux_ppc64/ppccpuid.s b/openssl/src/crypto/gen/linux_ppc64/ppccpuid.s index f8c10ec9c..b6017a4aa 100644 --- a/openssl/src/crypto/gen/linux_ppc64/ppccpuid.s +++ b/openssl/src/crypto/gen/linux_ppc64/ppccpuid.s @@ -64,20 +64,6 @@ OPENSSL_madd300_probe: .long 0 .byte 0,12,0x14,0,0,0,0,0 -.globl OPENSSL_brd31_probe -.type OPENSSL_brd31_probe,@function -.align 4 -OPENSSL_brd31_probe: -.localentry OPENSSL_brd31_probe,0 - - xor 0,0,0 - .long 0x7C030176 - blr -.long 0 -.byte 0,12,0x14,0,0,0,0,0 -.size OPENSSL_brd31_probe,.-OPENSSL_brd31_probe - - .globl OPENSSL_wipe_cpu .type OPENSSL_wipe_cpu,@function .align 4 diff --git a/openssl/src/crypto/gen/linux_riscv64/params_idx.c b/openssl/src/crypto/gen/linux_riscv64/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/linux_riscv64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/linux_riscv64/riscv64cpuid.s b/openssl/src/crypto/gen/linux_riscv64/riscv64cpuid.s deleted file mode 100644 index c36585d98..000000000 --- a/openssl/src/crypto/gen/linux_riscv64/riscv64cpuid.s +++ /dev/null @@ -1,70 +0,0 @@ -################################################################################ -# int CRYPTO_memcmp(const void * in_a, const void * in_b, size_t len) -################################################################################ -.text -.balign 16 -.globl CRYPTO_memcmp -.type CRYPTO_memcmp,@function -CRYPTO_memcmp: - li t0,0 - beqz a2,2f # len == 0 -1: - lbu t1,0(a0) - lbu t2,0(a1) - addi a0,a0,1 - addi a1,a1,1 - addi a2,a2,-1 - xor t1,t1,t2 - or t0,t0,t1 - bgtz a2,1b -2: - mv a0,t0 - ret -################################################################################ -# void OPENSSL_cleanse(void *ptr, size_t len) -################################################################################ -.text -.balign 16 -.globl OPENSSL_cleanse -.type OPENSSL_cleanse,@function -OPENSSL_cleanse: - beqz a1,2f # len == 0, return - srli t0,a1,4 - bnez t0,3f # len > 15 - -1: # Store <= 15 individual bytes - sb x0,0(a0) - addi a0,a0,1 - addi a1,a1,-1 - bnez a1,1b -2: - ret - -3: # Store individual bytes until we are aligned - andi t0,a0,0x7 - beqz t0,4f - sb x0,0(a0) - addi a0,a0,1 - addi a1,a1,-1 - j 3b - -4: # Store aligned dwords - li t1,8 -4: - sd x0,0(a0) - addi a0,a0,8 - addi a1,a1,-8 - bge a1,t1,4b # if len>=8 loop - bnez a1,1b # if len<8 and len != 0, store remaining bytes - ret -################################################################################ -# size_t riscv_vlen_asm(void) -# Return VLEN (i.e. the length of a vector register in bits). -.p2align 3 -.globl riscv_vlen_asm -.type riscv_vlen_asm,@function -riscv_vlen_asm: - csrr a0, 0xc22 - slli a0, a0, 3 - ret -.size riscv_vlen_asm,.-riscv_vlen_asm diff --git a/openssl/src/crypto/gen/linux_x64/params_idx.c b/openssl/src/crypto/gen/linux_x64/params_idx.c deleted file mode 100644 index 6227108d3..000000000 --- a/openssl/src/crypto/gen/linux_x64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by Makefile from ../../openssl/crypto/params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/windows_arm64/params_idx.c b/openssl/src/crypto/gen/windows_arm64/params_idx.c deleted file mode 100644 index 523df5ff8..000000000 --- a/openssl/src/crypto/gen/windows_arm64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by makefile from ..\..\openssl\crypto\params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/windows_ia32/params_idx.c b/openssl/src/crypto/gen/windows_ia32/params_idx.c deleted file mode 100644 index 523df5ff8..000000000 --- a/openssl/src/crypto/gen/windows_ia32/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by makefile from ..\..\openssl\crypto\params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/gen/windows_ia32/x86cpuid.asm b/openssl/src/crypto/gen/windows_ia32/x86cpuid.asm index 31f23515e..fc920ed46 100644 --- a/openssl/src/crypto/gen/windows_ia32/x86cpuid.asm +++ b/openssl/src/crypto/gen/windows_ia32/x86cpuid.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/gen/windows_x64/params_idx.c b/openssl/src/crypto/gen/windows_x64/params_idx.c deleted file mode 100644 index 523df5ff8..000000000 --- a/openssl/src/crypto/gen/windows_x64/params_idx.c +++ /dev/null @@ -1,2714 +0,0 @@ -/* - * WARNING: do not edit! - * Generated by makefile from ..\..\openssl\crypto\params_idx.c.in - * - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - - -#include "internal/e_os.h" -#include "internal/param_names.h" -#include - -/* Machine generated TRIE -- generated by util/perl/OpenSSL/paramnames.pm */ -int ossl_param_find_pidx(const char *s) -{ - switch(s[0]) { - default: - break; - case 'a': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("vp-info", s + 2) == 0) - return PIDX_KDF_PARAM_X942_ACVPINFO; - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_ARGON2_AD; - } - break; - case 'e': - if (strcmp("ad", s + 2) == 0) - return PIDX_CIPHER_PARAM_AEAD; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("id_param", s + 4) == 0) - return PIDX_CIPHER_PARAM_ALGORITHM_ID_PARAMS; - break; - case 'i': - if (strcmp("d-absent", s + 4) == 0) - return PIDX_DIGEST_PARAM_ALGID_ABSENT; - break; - case 'o': - if (strcmp("rithm-id", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_ALGORITHM_ID; - } - break; - case 'i': - if (strcmp("as", s + 3) == 0) - return PIDX_STORE_PARAM_ALIAS; - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_A; - } - break; - case 'b': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("sis-type", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_CHAR2_TYPE; - break; - case 'i': - if (strcmp("ts", s + 2) == 0) - return PIDX_PKEY_PARAM_BITS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'k': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("size", s + 6) == 0) - return PIDX_MAC_PARAM_BLOCK_SIZE; - break; - case '_': - if (strcmp("padding", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING; - break; - case 's': - if (strcmp("ize", s + 6) == 0) - return PIDX_DIGEST_PARAM_BLOCK_SIZE; - } - } - } - } - break; - case 'u': - if (strcmp("ildinfo", s + 2) == 0) - return PIDX_PROV_PARAM_BUILDINFO; - break; - case '\0': - return PIDX_PKEY_PARAM_EC_B; - } - break; - case 'c': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_C_ROUNDS; - break; - case 'e': - if (strcmp("kalg", s + 2) == 0) - return PIDX_KDF_PARAM_CEK_ALG; - break; - case 'i': - if (strcmp("pher", s + 2) == 0) - return PIDX_ALG_PARAM_CIPHER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("actor", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_COFACTOR; - break; - case 'n': - switch(s[3]) { - default: - break; - case 's': - if (strcmp("tant", s + 4) == 0) - return PIDX_KDF_PARAM_CONSTANT; - break; - case 't': - if (strcmp("ext-string", s + 4) == 0) - return PIDX_SIGNATURE_PARAM_CONTEXT_STRING; - } - } - break; - case 't': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '_': - if (strcmp("mode", s + 4) == 0) - return PIDX_CIPHER_PARAM_CTS_MODE; - break; - case '\0': - return PIDX_CIPHER_PARAM_CTS; - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'm': - switch(s[6]) { - default: - break; - case '-': - if (strcmp("iv", s + 7) == 0) - return PIDX_CIPHER_PARAM_CUSTOM_IV; - break; - case '\0': - return PIDX_MAC_PARAM_CUSTOM; - } - } - } - } - } - } - break; - case 'd': - switch(s[1]) { - default: - break; - case '-': - if (strcmp("rounds", s + 2) == 0) - return PIDX_MAC_PARAM_D_ROUNDS; - break; - case 'a': - switch(s[2]) { - default: - break; - case 't': - switch(s[3]) { - default: - break; - case 'a': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 's': - if (strcmp("tructure", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_STRUCTURE; - break; - case 't': - if (strcmp("ype", s + 6) == 0) - return PIDX_OBJECT_PARAM_DATA_TYPE; - } - break; - case '\0': - return PIDX_OBJECT_PARAM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("oded-from-explicit", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS; - break; - case 'f': - if (strcmp("ault-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_DEFAULT_DIGEST; - break; - case 's': - if (strcmp("c", s + 3) == 0) - return PIDX_OBJECT_PARAM_DESC; - } - break; - case 'h': - if (strcmp("kem-ikm", s + 2) == 0) - return PIDX_PKEY_PARAM_DHKEM_IKM; - break; - case 'i': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 's': - switch(s[5]) { - default: - break; - case 't': - switch(s[6]) { - default: - break; - case '-': - switch(s[7]) { - default: - break; - case 'n': - if (strcmp("oinit", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_NOINIT; - break; - case 'o': - if (strcmp("neshot", s + 8) == 0) - return PIDX_MAC_PARAM_DIGEST_ONESHOT; - break; - case 'p': - if (strcmp("rops", s + 8) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS; - break; - case 's': - if (strcmp("ize", s + 8) == 0) - return PIDX_PKEY_PARAM_DIGEST_SIZE; - } - break; - case '\0': - return PIDX_STORE_PARAM_DIGEST; - } - } - } - } - break; - case 's': - if (strcmp("tid", s + 3) == 0) - return PIDX_PKEY_PARAM_DIST_ID; - } - break; - case 'r': - if (strcmp("bg-no-trunc-md", s + 2) == 0) - return PIDX_PROV_PARAM_DRBG_TRUNC_DIGEST; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_D; - } - break; - case 'e': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("rly_clean", s + 2) == 0) - return PIDX_KDF_PARAM_EARLY_CLEAN; - break; - case 'c': - if (strcmp("dh-cofactor-mode", s + 2) == 0) - return PIDX_EXCHANGE_PARAM_EC_ECDH_COFACTOR_MODE; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'o': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("d-pub-key", s + 6) == 0) - return PIDX_PKEY_PARAM_ENCODED_PUBLIC_KEY; - break; - case 'i': - if (strcmp("ng", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_ENCODING; - } - } - break; - case 'r': - if (strcmp("ypt-level", s + 4) == 0) - return PIDX_ENCODER_PARAM_ENCRYPT_LEVEL; - } - break; - case 'g': - if (strcmp("ine", s + 3) == 0) - return PIDX_ALG_PARAM_ENGINE; - break; - case 't': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'o': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'y': - switch(s[7]) { - default: - break; - case '_': - if (strcmp("required", s + 8) == 0) - return PIDX_DRBG_PARAM_ENTROPY_REQUIRED; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_ENTROPY; - } - } - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_E; - break; - case 'x': - if (strcmp("pect", s + 2) == 0) - return PIDX_STORE_PARAM_EXPECT; - } - break; - case 'f': - switch(s[1]) { - default: - break; - case 'i': - switch(s[2]) { - default: - break; - case 'e': - if (strcmp("ld-type", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_FIELD_TYPE; - break; - case 'n': - if (strcmp("gerprint", s + 3) == 0) - return PIDX_STORE_PARAM_FINGERPRINT; - } - } - break; - case 'g': - switch(s[1]) { - default: - break; - case 'e': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_GENERATE; - } - break; - case 'o': - if (strcmp("r", s + 8) == 0) - return PIDX_PKEY_PARAM_EC_GENERATOR; - } - } - } - } - } - } - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_GINDEX; - break; - case 'r': - switch(s[2]) { - default: - break; - case 'o': - switch(s[3]) { - default: - break; - case 'u': - switch(s[4]) { - default: - break; - case 'p': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("check", s + 6) == 0) - return PIDX_PKEY_PARAM_EC_GROUP_CHECK_TYPE; - break; - case '\0': - return PIDX_PKEY_PARAM_GROUP_NAME; - } - } - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_G; - } - break; - case 'h': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("s-randkey", s + 2) == 0) - return PIDX_CIPHER_PARAM_HAS_RAND_KEY; - break; - case 'i': - if (strcmp("ndex", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_H; - } - break; - case 'i': - switch(s[1]) { - default: - break; - case 'd': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_PKCS12_ID; - } - break; - case 'k': - if (strcmp("me", s + 2) == 0) - return PIDX_KEM_PARAM_IKME; - break; - case 'm': - if (strcmp("plicit-rejection", s + 2) == 0) - return PIDX_PKEY_PARAM_IMPLICIT_REJECTION; - break; - case 'n': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("lude-public", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_INCLUDE_PUBLIC; - break; - case 'f': - if (strcmp("o", s + 3) == 0) - return PIDX_PASSPHRASE_PARAM_INFO; - break; - case 'p': - if (strcmp("ut-type", s + 3) == 0) - return PIDX_STORE_PARAM_INPUT_TYPE; - break; - case 's': - if (strcmp("tance", s + 3) == 0) - return PIDX_SIGNATURE_PARAM_INSTANCE; - } - break; - case 't': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'r': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("tion", s + 5) == 0) - return PIDX_GEN_PARAM_ITERATION; - break; - case '\0': - return PIDX_KDF_PARAM_ITER; - } - } - } - break; - case 'v': - switch(s[2]) { - default: - break; - case 'l': - if (strcmp("en", s + 3) == 0) - return PIDX_CIPHER_PARAM_IVLEN; - break; - case '\0': - return PIDX_MAC_PARAM_IV; - } - } - break; - case 'j': - switch(s[1]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_COFACTOR; - } - break; - case 'k': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K2; - } - break; - case '3': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_PP_K3; - } - break; - case 'a': - if (strcmp("t", s + 2) == 0) - return PIDX_SIGNATURE_PARAM_KAT; - break; - case 'd': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case 's': - switch(s[9]) { - default: - break; - case 't': - switch(s[10]) { - default: - break; - case '-': - if (strcmp("props", s + 11) == 0) - return PIDX_EXCHANGE_PARAM_KDF_DIGEST_PROPS; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_KDF_DIGEST; - } - } - } - } - } - } - break; - case 'o': - if (strcmp("utlen", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_OUTLEN; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_TYPE; - break; - case 'u': - if (strcmp("km", s + 5) == 0) - return PIDX_EXCHANGE_PARAM_KDF_UKM; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'y': - switch(s[3]) { - default: - break; - case 'b': - if (strcmp("its", s + 4) == 0) - return PIDX_CIPHER_PARAM_RC2_KEYBITS; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_KEYLEN; - break; - case '\0': - return PIDX_MAC_PARAM_KEY; - } - } - } - break; - case 'l': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("el", s + 3) == 0) - return PIDX_KDF_PARAM_LABEL; - break; - case 'n': - if (strcmp("es", s + 3) == 0) - return PIDX_KDF_PARAM_ARGON2_LANES; - } - } - break; - case 'm': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'k': - if (strcmp("ey", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_MAC_KEY; - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_KDF_PARAM_MAC_SIZE; - break; - case '\0': - return PIDX_ALG_PARAM_MAC; - } - break; - case 'n': - if (strcmp("datory-digest", s + 3) == 0) - return PIDX_PKEY_PARAM_MANDATORY_DIGEST; - break; - case 'x': - switch(s[3]) { - default: - break; - case '-': - if (strcmp("size", s + 4) == 0) - return PIDX_PKEY_PARAM_MAX_SIZE; - break; - case '_': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("dinlen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_ADINLEN; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("rly_data", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA; - break; - case 'n': - if (strcmp("tropylen", s + 6) == 0) - return PIDX_DRBG_PARAM_MAX_ENTROPYLEN; - } - break; - case 'f': - if (strcmp("rag_len", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_NONCELEN; - break; - case 'p': - if (strcmp("erslen", s + 5) == 0) - return PIDX_DRBG_PARAM_MAX_PERSLEN; - break; - case 'r': - if (strcmp("equest", s + 5) == 0) - return PIDX_RAND_PARAM_MAX_REQUEST; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MAX_LENGTH; - break; - case 'm': - if (strcmp("em_bytes", s + 4) == 0) - return PIDX_KDF_PARAM_SCRYPT_MAXMEM; - } - } - break; - case 'e': - if (strcmp("mcost", s + 2) == 0) - return PIDX_KDF_PARAM_ARGON2_MEMCOST; - break; - case 'g': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'd': - if (strcmp("igest", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_DIGEST; - break; - case 'p': - if (strcmp("roperties", s + 6) == 0) - return PIDX_PKEY_PARAM_MGF1_PROPERTIES; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_MASKGENFUNC; - } - } - break; - case 'i': - switch(s[2]) { - default: - break; - case 'c': - if (strcmp("alg", s + 3) == 0) - return PIDX_DIGEST_PARAM_MICALG; - break; - case 'n': - switch(s[3]) { - default: - break; - case '_': - switch(s[4]) { - default: - break; - case 'e': - if (strcmp("ntropylen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_ENTROPYLEN; - break; - case 'n': - if (strcmp("oncelen", s + 5) == 0) - return PIDX_DRBG_PARAM_MIN_NONCELEN; - } - break; - case 'i': - if (strcmp("um_length", s + 4) == 0) - return PIDX_DRBG_PARAM_MIN_LENGTH; - } - } - break; - case 'o': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case '\0': - return PIDX_LIBSSL_RECORD_LAYER_PARAM_MODE; - } - break; - case 'u': - if (strcmp("le-filename", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_MODULE_FILENAME; - } - } - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_M; - } - break; - case 'n': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("me", s + 2) == 0) - return PIDX_STORE_PARAM_ISSUER; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'c': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '-': - if (strcmp("type", s + 6) == 0) - return PIDX_SIGNATURE_PARAM_NONCE_TYPE; - break; - case '\0': - return PIDX_KDF_PARAM_HMACDRBG_NONCE; - } - } - } - } - break; - case 'u': - if (strcmp("m", s + 2) == 0) - return PIDX_CIPHER_PARAM_NUM; - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_N; - } - break; - case 'o': - switch(s[1]) { - default: - break; - case 'a': - if (strcmp("ep-label", s + 2) == 0) - return PIDX_ASYM_CIPHER_PARAM_OAEP_LABEL; - break; - case 'p': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'n': - if (strcmp("ssl-version", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_VERSION; - break; - case 'r': - if (strcmp("ation", s + 4) == 0) - return PIDX_KEM_PARAM_OPERATION; - } - break; - case 't': - if (strcmp("ions", s + 3) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_OPTIONS; - } - break; - case 'r': - if (strcmp("der", s + 2) == 0) - return PIDX_PKEY_PARAM_EC_ORDER; - } - break; - case 'p': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_P2; - } - break; - case 'a': - switch(s[2]) { - default: - break; - case 'd': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'm': - if (strcmp("ode", s + 5) == 0) - return PIDX_PKEY_PARAM_PAD_MODE; - break; - case 't': - if (strcmp("ype", s + 5) == 0) - return PIDX_DIGEST_PARAM_PAD_TYPE; - } - break; - case 'd': - if (strcmp("ing", s + 4) == 0) - return PIDX_CIPHER_PARAM_PADDING; - break; - case '\0': - return PIDX_EXCHANGE_PARAM_PAD; - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'y': - switch(s[5]) { - default: - break; - case 'u': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYUINFO; - break; - case 'v': - if (strcmp("-info", s + 6) == 0) - return PIDX_KDF_PARAM_X942_PARTYVINFO; - } - } - } - break; - case 's': - if (strcmp("s", s + 3) == 0) - return PIDX_KDF_PARAM_PASSWORD; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PBITS; - break; - case 'c': - if (strcmp("ounter", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_PCOUNTER; - break; - case 'k': - if (strcmp("cs5", s + 2) == 0) - return PIDX_KDF_PARAM_PKCS5; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'i': - if (strcmp("nt-format", s + 3) == 0) - return PIDX_PKEY_PARAM_EC_POINT_CONVERSION_FORMAT; - break; - case 't': - if (strcmp("ential", s + 3) == 0) - return PIDX_GEN_PARAM_POTENTIAL; - } - break; - case 'r': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("iction_resistance", s + 4) == 0) - return PIDX_DRBG_PARAM_PREDICTION_RESISTANCE; - break; - case 'f': - if (strcmp("ix", s + 4) == 0) - return PIDX_KDF_PARAM_PREFIX; - } - break; - case 'i': - switch(s[3]) { - default: - break; - case 'm': - if (strcmp("es", s + 4) == 0) - return PIDX_PKEY_PARAM_RSA_PRIMES; - break; - case 'v': - switch(s[4]) { - default: - break; - case '_': - if (strcmp("len", s + 5) == 0) - return PIDX_PKEY_PARAM_DH_PRIV_LEN; - break; - case '\0': - return PIDX_PKEY_PARAM_PRIV_KEY; - } - } - break; - case 'o': - switch(s[3]) { - default: - break; - case 'p': - if (strcmp("erties", s + 4) == 0) - return PIDX_STORE_PARAM_PROPERTIES; - break; - case 'v': - if (strcmp("ider-name", s + 4) == 0) - return PIDX_PROV_PARAM_CORE_PROV_NAME; - } - } - break; - case 'u': - if (strcmp("b", s + 2) == 0) - return PIDX_PKEY_PARAM_PUB_KEY; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_P; - } - break; - case 'q': - switch(s[1]) { - default: - break; - case '1': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q1; - } - break; - case '2': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_Q2; - } - break; - case 'b': - if (strcmp("its", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_QBITS; - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_Q; - break; - case 'x': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_X; - } - break; - case 'y': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_PUB_Y; - } - } - break; - case 'r': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'n': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case 'k': - if (strcmp("ey", s + 5) == 0) - return PIDX_CIPHER_PARAM_RANDOM_KEY; - break; - case 'o': - if (strcmp("m_data", s + 5) == 0) - return PIDX_DRBG_PARAM_RANDOM_DATA; - } - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case 'd': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'a': - if (strcmp("head", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD; - break; - case 'b': - if (strcmp("uffer_len", s + 6) == 0) - return PIDX_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN; - } - } - } - break; - case 'f': - if (strcmp("erence", s + 3) == 0) - return PIDX_OBJECT_PARAM_REFERENCE; - break; - case 's': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case '_': - switch(s[7]) { - default: - break; - case 'c': - if (strcmp("ounter", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_COUNTER; - break; - case 'r': - if (strcmp("equests", s + 8) == 0) - return PIDX_DRBG_PARAM_RESEED_REQUESTS; - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case 'm': - switch(s[10]) { - default: - break; - case 'e': - switch(s[11]) { - default: - break; - case '_': - if (strcmp("interval", s + 12) == 0) - return PIDX_DRBG_PARAM_RESEED_TIME_INTERVAL; - break; - case '\0': - return PIDX_DRBG_PARAM_RESEED_TIME; - } - } - } - } - } - } - } - } - } - } - break; - case 'o': - if (strcmp("unds", s + 2) == 0) - return PIDX_CIPHER_PARAM_ROUNDS; - break; - case 's': - switch(s[2]) { - default: - break; - case 'a': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - switch(s[5]) { - default: - break; - case 'o': - switch(s[6]) { - default: - break; - case 'e': - switch(s[7]) { - default: - break; - case 'f': - switch(s[8]) { - default: - break; - case 'f': - switch(s[9]) { - default: - break; - case 'i': - switch(s[10]) { - default: - break; - case 'c': - switch(s[11]) { - default: - break; - case 'i': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'n': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case '1': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT1; - } - break; - case '2': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT2; - } - break; - case '3': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT3; - } - break; - case '4': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT4; - } - break; - case '5': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT5; - } - break; - case '6': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT6; - } - break; - case '7': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT7; - } - break; - case '8': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT8; - } - break; - case '9': - switch(s[16]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_COEFFICIENT; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'd': - if (strcmp("erive-from-pq", s + 5) == 0) - return PIDX_PKEY_PARAM_RSA_DERIVE_FROM_PQ; - break; - case 'e': - switch(s[5]) { - default: - break; - case 'x': - switch(s[6]) { - default: - break; - case 'p': - switch(s[7]) { - default: - break; - case 'o': - switch(s[8]) { - default: - break; - case 'n': - switch(s[9]) { - default: - break; - case 'e': - switch(s[10]) { - default: - break; - case 'n': - switch(s[11]) { - default: - break; - case 't': - switch(s[12]) { - default: - break; - case '1': - switch(s[13]) { - default: - break; - case '0': - switch(s[14]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT1; - } - break; - case '2': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT2; - } - break; - case '3': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT3; - } - break; - case '4': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT4; - } - break; - case '5': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT5; - } - break; - case '6': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT6; - } - break; - case '7': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT7; - } - break; - case '8': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT8; - } - break; - case '9': - switch(s[13]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_EXPONENT; - } - } - } - } - } - } - } - } - break; - case 'f': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'o': - switch(s[9]) { - default: - break; - case 'r': - switch(s[10]) { - default: - break; - case '1': - switch(s[11]) { - default: - break; - case '0': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR10; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR1; - } - break; - case '2': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR2; - } - break; - case '3': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR3; - } - break; - case '4': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR4; - } - break; - case '5': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR5; - } - break; - case '6': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR6; - } - break; - case '7': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR7; - } - break; - case '8': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR8; - } - break; - case '9': - switch(s[11]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR9; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_FACTOR; - } - } - } - } - } - } - } - } - } - break; - case '\0': - return PIDX_KDF_PARAM_SCRYPT_R; - } - break; - case 's': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'f': - if (strcmp("eprime-generator", s + 3) == 0) - return PIDX_PKEY_PARAM_DH_GENERATOR; - break; - case 'l': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'l': - if (strcmp("en", s + 5) == 0) - return PIDX_SIGNATURE_PARAM_PSS_SALTLEN; - break; - case '\0': - return PIDX_MAC_PARAM_SALT; - } - } - break; - case 'v': - if (strcmp("e-parameters", s + 3) == 0) - return PIDX_ENCODER_PARAM_SAVE_PARAMETERS; - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 'c': - switch(s[3]) { - default: - break; - case 'r': - if (strcmp("et", s + 4) == 0) - return PIDX_KDF_PARAM_SECRET; - break; - case 'u': - switch(s[4]) { - default: - break; - case 'r': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'y': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'b': - if (strcmp("its", s + 10) == 0) - return PIDX_PKEY_PARAM_SECURITY_BITS; - break; - case 'c': - if (strcmp("hecks", s + 10) == 0) - return PIDX_PROV_PARAM_SECURITY_CHECKS; - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("d", s + 3) == 0) - return PIDX_PKEY_PARAM_FFC_SEED; - break; - case 'r': - if (strcmp("ial", s + 3) == 0) - return PIDX_STORE_PARAM_SERIAL; - break; - case 's': - if (strcmp("sion_id", s + 3) == 0) - return PIDX_KDF_PARAM_SSHKDF_SESSION_ID; - } - break; - case 'i': - if (strcmp("ze", s + 2) == 0) - return PIDX_MAC_PARAM_SIZE; - break; - case 'p': - if (strcmp("eed", s + 2) == 0) - return PIDX_CIPHER_PARAM_SPEED; - break; - case 's': - if (strcmp("l3-ms", s + 2) == 0) - return PIDX_DIGEST_PARAM_SSL3_MS; - break; - case 't': - switch(s[2]) { - default: - break; - case '-': - switch(s[3]) { - default: - break; - case 'd': - if (strcmp("esc", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_DESC; - break; - case 'p': - if (strcmp("hase", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_PHASE; - break; - case 't': - if (strcmp("ype", s + 4) == 0) - return PIDX_PROV_PARAM_SELF_TEST_TYPE; - } - break; - case 'a': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case 'e': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_RAND_PARAM_STATE; - } - break; - case 'u': - if (strcmp("s", s + 5) == 0) - return PIDX_PROV_PARAM_STATUS; - } - } - break; - case 'r': - switch(s[3]) { - default: - break; - case 'e': - switch(s[4]) { - default: - break; - case 'a': - if (strcmp("m_mac", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC; - break; - case 'n': - if (strcmp("gth", s + 5) == 0) - return PIDX_RAND_PARAM_STRENGTH; - } - } - } - break; - case 'u': - switch(s[2]) { - default: - break; - case 'b': - if (strcmp("ject", s + 3) == 0) - return PIDX_STORE_PARAM_SUBJECT; - break; - case 'p': - switch(s[3]) { - default: - break; - case 'p': - switch(s[4]) { - default: - break; - case '-': - switch(s[5]) { - default: - break; - case 'p': - switch(s[6]) { - default: - break; - case 'r': - if (strcmp("ivinfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PRIVINFO; - break; - case 'u': - if (strcmp("binfo", s + 7) == 0) - return PIDX_KDF_PARAM_X942_SUPP_PUBINFO; - } - } - } - } - } - } - break; - case 't': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'g': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_CIPHER_PARAM_AEAD_TAGLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TAG; - } - } - break; - case 'e': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case 't': - switch(s[4]) { - default: - break; - case '_': - switch(s[5]) { - default: - break; - case 'e': - if (strcmp("ntropy", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_ENTROPY; - break; - case 'n': - if (strcmp("once", s + 6) == 0) - return PIDX_RAND_PARAM_TEST_NONCE; - } - } - } - } - break; - case 'h': - if (strcmp("reads", s + 2) == 0) - return PIDX_KDF_PARAM_THREADS; - break; - case 'l': - switch(s[2]) { - default: - break; - case 's': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'c': - if (strcmp("lient-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION; - break; - case 'd': - if (strcmp("ata-size", s + 5) == 0) - return PIDX_MAC_PARAM_TLS_DATA_SIZE; - break; - case 'g': - switch(s[5]) { - default: - break; - case 'r': - switch(s[6]) { - default: - break; - case 'o': - switch(s[7]) { - default: - break; - case 'u': - switch(s[8]) { - default: - break; - case 'p': - switch(s[9]) { - default: - break; - case '-': - switch(s[10]) { - default: - break; - case 'a': - if (strcmp("lg", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_ALG; - break; - case 'i': - switch(s[11]) { - default: - break; - case 'd': - switch(s[12]) { - default: - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_ID; - } - break; - case 's': - if (strcmp("-kem", s + 12) == 0) - return PIDX_CAPABILITY_TLS_GROUP_IS_KEM; - } - break; - case 'n': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'm': - switch(s[13]) { - default: - break; - case 'e': - switch(s[14]) { - default: - break; - case '-': - if (strcmp("internal", s + 15) == 0) - return PIDX_CAPABILITY_TLS_GROUP_NAME_INTERNAL; - break; - case '\0': - return PIDX_CAPABILITY_TLS_GROUP_NAME; - } - } - } - } - break; - case 's': - if (strcmp("ec-bits", s + 11) == 0) - return PIDX_CAPABILITY_TLS_GROUP_SECURITY_BITS; - } - } - } - } - } - } - break; - case 'm': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 'c': - switch(s[7]) { - default: - break; - case '-': - if (strcmp("size", s + 8) == 0) - return PIDX_CIPHER_PARAM_TLS_MAC_SIZE; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS_MAC; - } - break; - case 'x': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MAX_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MAX_TLS; - } - } - } - break; - case 'i': - switch(s[6]) { - default: - break; - case 'n': - switch(s[7]) { - default: - break; - case '-': - switch(s[8]) { - default: - break; - case 'd': - if (strcmp("tls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_GROUP_MIN_DTLS; - break; - case 't': - if (strcmp("ls", s + 9) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_MIN_TLS; - } - } - } - break; - case 'u': - if (strcmp("lti", s + 6) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK; - } - break; - case 'n': - if (strcmp("egotiated-version", s + 5) == 0) - return PIDX_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION; - break; - case 's': - switch(s[5]) { - default: - break; - case 'i': - switch(s[6]) { - default: - break; - case 'g': - switch(s[7]) { - default: - break; - case 'a': - switch(s[8]) { - default: - break; - case 'l': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '-': - switch(s[11]) { - default: - break; - case 'c': - if (strcmp("ode-point", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_CODE_POINT; - break; - case 'h': - switch(s[12]) { - default: - break; - case 'a': - switch(s[13]) { - default: - break; - case 's': - switch(s[14]) { - default: - break; - case 'h': - switch(s[15]) { - default: - break; - case '-': - switch(s[16]) { - default: - break; - case 'n': - if (strcmp("ame", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_NAME; - break; - case 'o': - if (strcmp("id", s + 17) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_HASH_OID; - } - } - } - } - } - break; - case 'i': - if (strcmp("ana-name", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_IANA_NAME; - break; - case 'k': - switch(s[12]) { - default: - break; - case 'e': - switch(s[13]) { - default: - break; - case 'y': - switch(s[14]) { - default: - break; - case 't': - switch(s[15]) { - default: - break; - case 'y': - switch(s[16]) { - default: - break; - case 'p': - switch(s[17]) { - default: - break; - case 'e': - switch(s[18]) { - default: - break; - case '-': - if (strcmp("oid", s + 19) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE_OID; - break; - case '\0': - return PIDX_CAPABILITY_TLS_SIGALG_KEYTYPE; - } - } - } - } - } - } - } - break; - case 'n': - if (strcmp("ame", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_NAME; - break; - case 'o': - if (strcmp("id", s + 12) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_OID; - break; - case 's': - switch(s[12]) { - default: - break; - case 'e': - if (strcmp("c-bits", s + 13) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SECURITY_BITS; - break; - case 'i': - switch(s[13]) { - default: - break; - case 'g': - switch(s[14]) { - default: - break; - case '-': - switch(s[15]) { - default: - break; - case 'n': - if (strcmp("ame", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_NAME; - break; - case 'o': - if (strcmp("id", s + 16) == 0) - return PIDX_CAPABILITY_TLS_SIGALG_SIG_OID; - } - } - } - } - } - } - } - } - } - } - } - break; - case 'v': - if (strcmp("ersion", s + 5) == 0) - return PIDX_CIPHER_PARAM_TLS_VERSION; - } - break; - case '1': - switch(s[4]) { - default: - break; - case '-': - if (strcmp("prf-ems-check", s + 5) == 0) - return PIDX_PROV_PARAM_TLS1_PRF_EMS_CHECK; - break; - case 'm': - switch(s[5]) { - default: - break; - case 'u': - switch(s[6]) { - default: - break; - case 'l': - switch(s[7]) { - default: - break; - case 't': - switch(s[8]) { - default: - break; - case 'i': - switch(s[9]) { - default: - break; - case '_': - switch(s[10]) { - default: - break; - case 'a': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'd': - switch(s[13]) { - default: - break; - case 'p': - if (strcmp("acklen", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD_PACKLEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_AAD; - } - } - } - break; - case 'e': - switch(s[11]) { - default: - break; - case 'n': - switch(s[12]) { - default: - break; - case 'c': - switch(s[13]) { - default: - break; - case 'i': - if (strcmp("n", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_IN; - break; - case 'l': - if (strcmp("en", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC_LEN; - break; - case '\0': - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_ENC; - } - } - } - break; - case 'i': - if (strcmp("nterleave", s + 11) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_INTERLEAVE; - break; - case 'm': - switch(s[11]) { - default: - break; - case 'a': - switch(s[12]) { - default: - break; - case 'x': - switch(s[13]) { - default: - break; - case 'b': - if (strcmp("ufsz", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_BUFSIZE; - break; - case 's': - if (strcmp("ndfrag", s + 14) == 0) - return PIDX_CIPHER_PARAM_TLS1_MULTIBLOCK_MAX_SEND_FRAGMENT; - } - } - } - } - } - } - } - } - } - } - break; - case 'a': - switch(s[4]) { - default: - break; - case 'a': - switch(s[5]) { - default: - break; - case 'd': - switch(s[6]) { - default: - break; - case 'p': - if (strcmp("ad", s + 7) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD; - break; - case '\0': - return PIDX_CIPHER_PARAM_AEAD_TLS1_AAD; - } - } - } - break; - case 'i': - switch(s[4]) { - default: - break; - case 'v': - switch(s[5]) { - default: - break; - case 'f': - if (strcmp("ixed", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED; - break; - case 'g': - if (strcmp("en", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN; - break; - case 'i': - if (strcmp("nv", s + 6) == 0) - return PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV; - } - } - break; - case 't': - if (strcmp("ree", s + 4) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_TLSTREE; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_EC_CHAR2_TP_BASIS; - } - break; - case 'y': - if (strcmp("pe", s + 2) == 0) - return PIDX_PKEY_PARAM_FFC_TYPE; - } - break; - case 'u': - switch(s[1]) { - default: - break; - case 'k': - if (strcmp("m", s + 2) == 0) - return PIDX_KDF_PARAM_UKM; - break; - case 'p': - if (strcmp("dated-iv", s + 2) == 0) - return PIDX_CIPHER_PARAM_UPDATED_IV; - break; - case 's': - switch(s[2]) { - default: - break; - case 'e': - switch(s[3]) { - default: - break; - case '-': - switch(s[4]) { - default: - break; - case 'b': - if (strcmp("its", s + 5) == 0) - return PIDX_CIPHER_PARAM_USE_BITS; - break; - case 'c': - if (strcmp("ofactor-flag", s + 5) == 0) - return PIDX_PKEY_PARAM_USE_COFACTOR_FLAG; - break; - case 'k': - if (strcmp("eybits", s + 5) == 0) - return PIDX_KDF_PARAM_X942_USE_KEYBITS; - break; - case 'l': - switch(s[5]) { - default: - break; - case '\0': - return PIDX_KDF_PARAM_KBKDF_USE_L; - } - break; - case 's': - if (strcmp("eparator", s + 5) == 0) - return PIDX_KDF_PARAM_KBKDF_USE_SEPARATOR; - } - break; - case '_': - switch(s[4]) { - default: - break; - case 'd': - if (strcmp("erivation_function", s + 5) == 0) - return PIDX_DRBG_PARAM_USE_DF; - break; - case 'e': - if (strcmp("tm", s + 5) == 0) - return PIDX_LIBSSL_RECORD_LAYER_PARAM_USE_ETM; - } - } - } - } - break; - case 'v': - switch(s[1]) { - default: - break; - case 'a': - switch(s[2]) { - default: - break; - case 'l': - switch(s[3]) { - default: - break; - case 'i': - switch(s[4]) { - default: - break; - case 'd': - switch(s[5]) { - default: - break; - case 'a': - switch(s[6]) { - default: - break; - case 't': - switch(s[7]) { - default: - break; - case 'e': - switch(s[8]) { - default: - break; - case '-': - switch(s[9]) { - default: - break; - case 'g': - switch(s[10]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_FFC_VALIDATE_G; - } - break; - case 'l': - if (strcmp("egacy", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_LEGACY; - break; - case 'p': - if (strcmp("q", s + 10) == 0) - return PIDX_PKEY_PARAM_FFC_VALIDATE_PQ; - } - } - } - } - } - } - } - } - break; - case 'e': - if (strcmp("rsion", s + 2) == 0) - return PIDX_PROV_PARAM_VERSION; - } - break; - case 'x': - switch(s[1]) { - default: - break; - case 'c': - if (strcmp("ghash", s + 2) == 0) - return PIDX_KDF_PARAM_SSHKDF_XCGHASH; - break; - case 'o': - switch(s[2]) { - default: - break; - case 'f': - switch(s[3]) { - default: - break; - case 'l': - if (strcmp("en", s + 4) == 0) - return PIDX_DIGEST_PARAM_XOFLEN; - break; - case '\0': - return PIDX_MAC_PARAM_XOF; - } - } - break; - case 'p': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XP; - } - break; - case 'q': - switch(s[2]) { - default: - break; - case '1': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ1; - } - break; - case '2': - switch(s[3]) { - default: - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ2; - } - break; - case '\0': - return PIDX_PKEY_PARAM_RSA_TEST_XQ; - } - break; - case 't': - if (strcmp("s_standard", s + 2) == 0) - return PIDX_CIPHER_PARAM_XTS_STANDARD; - } - } - return -1; -} - -/* End of TRIE */ diff --git a/openssl/src/crypto/hpke/hpke.c b/openssl/src/crypto/hpke/hpke.c deleted file mode 100644 index a53488d9e..000000000 --- a/openssl/src/crypto/hpke/hpke.c +++ /dev/null @@ -1,1463 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* An OpenSSL-based HPKE implementation of RFC9180 */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "internal/hpke_util.h" -#include "internal/nelem.h" -#include "internal/common.h" - -/* default buffer size for keys and internal buffers we use */ -#define OSSL_HPKE_MAXSIZE 512 - -/* Define HPKE labels from RFC9180 in hex for EBCDIC compatibility */ -/* "HPKE" - "suite_id" label for section 5.1 */ -static const char OSSL_HPKE_SEC51LABEL[] = "\x48\x50\x4b\x45"; -/* "psk_id_hash" - in key_schedule_context */ -static const char OSSL_HPKE_PSKIDHASH_LABEL[] = "\x70\x73\x6b\x5f\x69\x64\x5f\x68\x61\x73\x68"; -/* "info_hash" - in key_schedule_context */ -static const char OSSL_HPKE_INFOHASH_LABEL[] = "\x69\x6e\x66\x6f\x5f\x68\x61\x73\x68"; -/* "base_nonce" - base nonce calc label */ -static const char OSSL_HPKE_NONCE_LABEL[] = "\x62\x61\x73\x65\x5f\x6e\x6f\x6e\x63\x65"; -/* "exp" - internal exporter secret generation label */ -static const char OSSL_HPKE_EXP_LABEL[] = "\x65\x78\x70"; -/* "sec" - external label for exporting secret */ -static const char OSSL_HPKE_EXP_SEC_LABEL[] = "\x73\x65\x63"; -/* "key" - label for use when generating key from shared secret */ -static const char OSSL_HPKE_KEY_LABEL[] = "\x6b\x65\x79"; -/* "secret" - for generating shared secret */ -static const char OSSL_HPKE_SECRET_LABEL[] = "\x73\x65\x63\x72\x65\x74"; - -/** - * @brief sender or receiver context - */ -struct ossl_hpke_ctx_st -{ - OSSL_LIB_CTX *libctx; /* library context */ - char *propq; /* properties */ - int mode; /* HPKE mode */ - OSSL_HPKE_SUITE suite; /* suite */ - const OSSL_HPKE_KEM_INFO *kem_info; - const OSSL_HPKE_KDF_INFO *kdf_info; - const OSSL_HPKE_AEAD_INFO *aead_info; - EVP_CIPHER *aead_ciph; - int role; /* sender(0) or receiver(1) */ - uint64_t seq; /* aead sequence number */ - unsigned char *shared_secret; /* KEM output, zz */ - size_t shared_secretlen; - unsigned char *key; /* final aead key */ - size_t keylen; - unsigned char *nonce; /* aead base nonce */ - size_t noncelen; - unsigned char *exportersec; /* exporter secret */ - size_t exporterseclen; - char *pskid; /* PSK stuff */ - unsigned char *psk; - size_t psklen; - EVP_PKEY *authpriv; /* sender's authentication private key */ - unsigned char *authpub; /* auth public key */ - size_t authpublen; - unsigned char *ikme; /* IKM for sender deterministic key gen */ - size_t ikmelen; -}; - -/** - * @brief check if KEM uses NIST curve or not - * @param kem_id is the externally supplied kem_id - * @return 1 for NIST curves, 0 for other - */ -static int hpke_kem_id_nist_curve(uint16_t kem_id) -{ - const OSSL_HPKE_KEM_INFO *kem_info; - - kem_info = ossl_HPKE_KEM_INFO_find_id(kem_id); - return kem_info != NULL && kem_info->groupname != NULL; -} - -/** - * @brief wrapper to import NIST curve public key as easily as x25519/x448 - * @param libctx is the context to use - * @param propq is a properties string - * @param gname is the curve groupname - * @param buf is the binary buffer with the (uncompressed) public value - * @param buflen is the length of the private key buffer - * @return a working EVP_PKEY * or NULL - * - * Note that this could be a useful function to make public in - * future, but would likely require a name change. - */ -static EVP_PKEY *evp_pkey_new_raw_nist_public_key(OSSL_LIB_CTX *libctx, - const char *propq, - const char *gname, - const unsigned char *buf, - size_t buflen) -{ - OSSL_PARAM params[2]; - EVP_PKEY *ret = NULL; - EVP_PKEY_CTX *cctx = EVP_PKEY_CTX_new_from_name(libctx, "EC", propq); - - params[0] = OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_GROUP_NAME, - (char *)gname, 0); - params[1] = OSSL_PARAM_construct_end(); - if (cctx == NULL - || EVP_PKEY_paramgen_init(cctx) <= 0 - || EVP_PKEY_CTX_set_params(cctx, params) <= 0 - || EVP_PKEY_paramgen(cctx, &ret) <= 0 - || EVP_PKEY_set1_encoded_public_key(ret, buf, buflen) != 1) { - EVP_PKEY_CTX_free(cctx); - EVP_PKEY_free(ret); - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return NULL; - } - EVP_PKEY_CTX_free(cctx); - return ret; -} - -/** - * @brief do the AEAD decryption - * @param hctx is the context to use - * @param iv is the initialisation vector - * @param aad is the additional authenticated data - * @param aadlen is the length of the aad - * @param ct is the ciphertext buffer - * @param ctlen is the ciphertext length (including tag). - * @param pt is the output buffer - * @param ptlen input/output, better be big enough on input, exact on output - * @return 1 on success, 0 otherwise - */ -static int hpke_aead_dec(OSSL_HPKE_CTX *hctx, const unsigned char *iv, - const unsigned char *aad, size_t aadlen, - const unsigned char *ct, size_t ctlen, - unsigned char *pt, size_t *ptlen) -{ - int erv = 0; - EVP_CIPHER_CTX *ctx = NULL; - int len = 0; - size_t taglen; - - taglen = hctx->aead_info->taglen; - if (ctlen <= taglen || *ptlen < ctlen - taglen) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - /* Create and initialise the context */ - if ((ctx = EVP_CIPHER_CTX_new()) == NULL) - return 0; - /* Initialise the decryption operation. */ - if (EVP_DecryptInit_ex(ctx, hctx->aead_ciph, NULL, NULL, NULL) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, - hctx->noncelen, NULL) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - /* Initialise key and IV */ - if (EVP_DecryptInit_ex(ctx, NULL, NULL, hctx->key, iv) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - /* Provide AAD. */ - if (aadlen != 0 && aad != NULL) { - if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - } - if (EVP_DecryptUpdate(ctx, pt, &len, ct, ctlen - taglen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - *ptlen = len; - if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, - taglen, (void *)(ct + ctlen - taglen))) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - /* Finalise decryption. */ - if (EVP_DecryptFinal_ex(ctx, pt + len, &len) <= 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - erv = 1; - -err: - if (erv != 1) - OPENSSL_cleanse(pt, *ptlen); - EVP_CIPHER_CTX_free(ctx); - return erv; -} - -/** - * @brief do AEAD encryption as per the RFC - * @param hctx is the context to use - * @param iv is the initialisation vector - * @param aad is the additional authenticated data - * @param aadlen is the length of the aad - * @param pt is the plaintext buffer - * @param ptlen is the length of pt - * @param ct is the output buffer - * @param ctlen input/output, needs space for tag on input, exact on output - * @return 1 for success, 0 otherwise - */ -static int hpke_aead_enc(OSSL_HPKE_CTX *hctx, const unsigned char *iv, - const unsigned char *aad, size_t aadlen, - const unsigned char *pt, size_t ptlen, - unsigned char *ct, size_t *ctlen) -{ - int erv = 0; - EVP_CIPHER_CTX *ctx = NULL; - int len; - size_t taglen = 0; - unsigned char tag[EVP_MAX_AEAD_TAG_LENGTH]; - - taglen = hctx->aead_info->taglen; - if (*ctlen <= taglen || ptlen > *ctlen - taglen) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (!ossl_assert(taglen <= sizeof(tag))) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - /* Create and initialise the context */ - if ((ctx = EVP_CIPHER_CTX_new()) == NULL) - return 0; - /* Initialise the encryption operation. */ - if (EVP_EncryptInit_ex(ctx, hctx->aead_ciph, NULL, NULL, NULL) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, - hctx->noncelen, NULL) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - /* Initialise key and IV */ - if (EVP_EncryptInit_ex(ctx, NULL, NULL, hctx->key, iv) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - /* Provide any AAD data. */ - if (aadlen != 0 && aad != NULL) { - if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - } - if (EVP_EncryptUpdate(ctx, ct, &len, pt, ptlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - *ctlen = len; - /* Finalise the encryption. */ - if (EVP_EncryptFinal_ex(ctx, ct + len, &len) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - *ctlen += len; - /* Get tag. Not a duplicate so needs to be added to the ciphertext */ - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, taglen, tag) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - memcpy(ct + *ctlen, tag, taglen); - *ctlen += taglen; - erv = 1; - -err: - if (erv != 1) - OPENSSL_cleanse(ct, *ctlen); - EVP_CIPHER_CTX_free(ctx); - return erv; -} - -/** - * @brief check mode is in-range and supported - * @param mode is the caller's chosen mode - * @return 1 for good mode, 0 otherwise - */ -static int hpke_mode_check(unsigned int mode) -{ - switch (mode) { - case OSSL_HPKE_MODE_BASE: - case OSSL_HPKE_MODE_PSK: - case OSSL_HPKE_MODE_AUTH: - case OSSL_HPKE_MODE_PSKAUTH: - break; - default: - return 0; - } - return 1; -} - -/** - * @brief check if a suite is supported locally - * @param suite is the suite to check - * @return 1 for good, 0 otherwise - */ -static int hpke_suite_check(OSSL_HPKE_SUITE suite, - const OSSL_HPKE_KEM_INFO **kem_info, - const OSSL_HPKE_KDF_INFO **kdf_info, - const OSSL_HPKE_AEAD_INFO **aead_info) -{ - const OSSL_HPKE_KEM_INFO *kem_info_; - const OSSL_HPKE_KDF_INFO *kdf_info_; - const OSSL_HPKE_AEAD_INFO *aead_info_; - - /* check KEM, KDF and AEAD are supported here */ - if ((kem_info_ = ossl_HPKE_KEM_INFO_find_id(suite.kem_id)) == NULL) - return 0; - if ((kdf_info_ = ossl_HPKE_KDF_INFO_find_id(suite.kdf_id)) == NULL) - return 0; - if ((aead_info_ = ossl_HPKE_AEAD_INFO_find_id(suite.aead_id)) == NULL) - return 0; - - if (kem_info != NULL) - *kem_info = kem_info_; - if (kdf_info != NULL) - *kdf_info = kdf_info_; - if (aead_info != NULL) - *aead_info = aead_info_; - - return 1; -} - -/* - * @brief randomly pick a suite - * @param libctx is the context to use - * @param propq is a properties string - * @param suite is the result - * @return 1 for success, 0 otherwise - */ -static int hpke_random_suite(OSSL_LIB_CTX *libctx, - const char *propq, - OSSL_HPKE_SUITE *suite) -{ - const OSSL_HPKE_KEM_INFO *kem_info = NULL; - const OSSL_HPKE_KDF_INFO *kdf_info = NULL; - const OSSL_HPKE_AEAD_INFO *aead_info = NULL; - - /* random kem, kdf and aead */ - kem_info = ossl_HPKE_KEM_INFO_find_random(libctx); - if (kem_info == NULL) - return 0; - suite->kem_id = kem_info->kem_id; - kdf_info = ossl_HPKE_KDF_INFO_find_random(libctx); - if (kdf_info == NULL) - return 0; - suite->kdf_id = kdf_info->kdf_id; - aead_info = ossl_HPKE_AEAD_INFO_find_random(libctx); - if (aead_info == NULL) - return 0; - suite->aead_id = aead_info->aead_id; - return 1; -} - -/* - * @brief tell the caller how big the ciphertext will be - * - * AEAD algorithms add a tag for data authentication. - * Those are almost always, but not always, 16 octets - * long, and who knows what will be true in the future. - * So this function allows a caller to find out how - * much data expansion they will see with a given suite. - * - * "enc" is the name used in RFC9180 for the encapsulated - * public value of the sender, who calls OSSL_HPKE_seal(), - * that is sent to the recipient, who calls OSSL_HPKE_open(). - * - * @param suite is the suite to be used - * @param enclen points to what will be enc length - * @param clearlen is the length of plaintext - * @param cipherlen points to what will be ciphertext length (including tag) - * @return 1 for success, 0 otherwise - */ -static int hpke_expansion(OSSL_HPKE_SUITE suite, - size_t *enclen, - size_t clearlen, - size_t *cipherlen) -{ - const OSSL_HPKE_AEAD_INFO *aead_info = NULL; - const OSSL_HPKE_KEM_INFO *kem_info = NULL; - - if (cipherlen == NULL || enclen == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (hpke_suite_check(suite, &kem_info, NULL, &aead_info) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - *cipherlen = clearlen + aead_info->taglen; - *enclen = kem_info->Nenc; - return 1; -} - -/* - * @brief expand and XOR the 64-bit unsigned seq with (nonce) buffer - * @param ctx is the HPKE context - * @param buf is the buffer for the XOR'd seq and nonce - * @param blen is the size of buf - * @return 0 for error, otherwise blen - */ -static size_t hpke_seqnonce2buf(OSSL_HPKE_CTX *ctx, - unsigned char *buf, size_t blen) -{ - size_t i; - uint64_t seq_copy; - - if (ctx == NULL || blen < sizeof(seq_copy) || blen != ctx->noncelen) - return 0; - seq_copy = ctx->seq; - memset(buf, 0, blen); - for (i = 0; i < sizeof(seq_copy); i++) { - buf[blen - i - 1] = seq_copy & 0xff; - seq_copy >>= 8; - } - for (i = 0; i < blen; i++) - buf[i] ^= ctx->nonce[i]; - return blen; -} - -/* - * @brief call the underlying KEM to encap - * @param ctx is the OSSL_HPKE_CTX - * @param enc is a buffer for the sender's ephemeral public value - * @param enclen is the size of enc on input, number of octets used on output - * @param pub is the recipient's public value - * @param publen is the length of pub - * @return 1 for success, 0 for error - */ -static int hpke_encap(OSSL_HPKE_CTX *ctx, unsigned char *enc, size_t *enclen, - const unsigned char *pub, size_t publen) -{ - int erv = 0; - OSSL_PARAM params[3], *p = params; - size_t lsslen = 0, lenclen = 0; - EVP_PKEY_CTX *pctx = NULL; - EVP_PKEY *pkR = NULL; - const OSSL_HPKE_KEM_INFO *kem_info = NULL; - - if (ctx == NULL || enc == NULL || enclen == NULL || *enclen == 0 - || pub == NULL || publen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->shared_secret != NULL) { - /* only run the KEM once per OSSL_HPKE_CTX */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - kem_info = ossl_HPKE_KEM_INFO_find_id(ctx->suite.kem_id); - if (kem_info == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - if (hpke_kem_id_nist_curve(ctx->suite.kem_id) == 1) { - pkR = evp_pkey_new_raw_nist_public_key(ctx->libctx, ctx->propq, - kem_info->groupname, - pub, publen); - } else { - pkR = EVP_PKEY_new_raw_public_key_ex(ctx->libctx, - kem_info->keytype, - ctx->propq, pub, publen); - } - if (pkR == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - pctx = EVP_PKEY_CTX_new_from_pkey(ctx->libctx, pkR, ctx->propq); - if (pctx == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KEM_PARAM_OPERATION, - OSSL_KEM_PARAM_OPERATION_DHKEM, - 0); - if (ctx->ikme != NULL) { - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KEM_PARAM_IKME, - ctx->ikme, ctx->ikmelen); - } - *p = OSSL_PARAM_construct_end(); - if (ctx->mode == OSSL_HPKE_MODE_AUTH - || ctx->mode == OSSL_HPKE_MODE_PSKAUTH) { - if (EVP_PKEY_auth_encapsulate_init(pctx, ctx->authpriv, - params) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - } else { - if (EVP_PKEY_encapsulate_init(pctx, params) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - } - lenclen = *enclen; - if (EVP_PKEY_encapsulate(pctx, NULL, &lenclen, NULL, &lsslen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (lenclen > *enclen) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - ctx->shared_secret = OPENSSL_malloc(lsslen); - if (ctx->shared_secret == NULL) - goto err; - ctx->shared_secretlen = lsslen; - if (EVP_PKEY_encapsulate(pctx, enc, enclen, ctx->shared_secret, - &ctx->shared_secretlen) != 1) { - ctx->shared_secretlen = 0; - OPENSSL_free(ctx->shared_secret); - ctx->shared_secret = NULL; - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - erv = 1; - -err: - EVP_PKEY_CTX_free(pctx); - EVP_PKEY_free(pkR); - return erv; -} - -/* - * @brief call the underlying KEM to decap - * @param ctx is the OSSL_HPKE_CTX - * @param enc is a buffer for the sender's ephemeral public value - * @param enclen is the length of enc - * @param priv is the recipient's private value - * @return 1 for success, 0 for error - */ -static int hpke_decap(OSSL_HPKE_CTX *ctx, - const unsigned char *enc, size_t enclen, - EVP_PKEY *priv) -{ - int erv = 0; - EVP_PKEY_CTX *pctx = NULL; - EVP_PKEY *spub = NULL; - OSSL_PARAM params[2], *p = params; - size_t lsslen = 0; - - if (ctx == NULL || enc == NULL || enclen == 0 || priv == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->shared_secret != NULL) { - /* only run the KEM once per OSSL_HPKE_CTX */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - pctx = EVP_PKEY_CTX_new_from_pkey(ctx->libctx, priv, ctx->propq); - if (pctx == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KEM_PARAM_OPERATION, - OSSL_KEM_PARAM_OPERATION_DHKEM, - 0); - *p = OSSL_PARAM_construct_end(); - if (ctx->mode == OSSL_HPKE_MODE_AUTH - || ctx->mode == OSSL_HPKE_MODE_PSKAUTH) { - const OSSL_HPKE_KEM_INFO *kem_info = NULL; - - kem_info = ossl_HPKE_KEM_INFO_find_id(ctx->suite.kem_id); - if (kem_info == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (hpke_kem_id_nist_curve(ctx->suite.kem_id) == 1) { - spub = evp_pkey_new_raw_nist_public_key(ctx->libctx, ctx->propq, - kem_info->groupname, - ctx->authpub, - ctx->authpublen); - } else { - spub = EVP_PKEY_new_raw_public_key_ex(ctx->libctx, - kem_info->keytype, - ctx->propq, - ctx->authpub, - ctx->authpublen); - } - if (spub == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (EVP_PKEY_auth_decapsulate_init(pctx, spub, params) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - } else { - if (EVP_PKEY_decapsulate_init(pctx, params) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - } - if (EVP_PKEY_decapsulate(pctx, NULL, &lsslen, enc, enclen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - ctx->shared_secret = OPENSSL_malloc(lsslen); - if (ctx->shared_secret == NULL) - goto err; - if (EVP_PKEY_decapsulate(pctx, ctx->shared_secret, &lsslen, - enc, enclen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - ctx->shared_secretlen = lsslen; - erv = 1; - -err: - EVP_PKEY_CTX_free(pctx); - EVP_PKEY_free(spub); - if (erv == 0) { - OPENSSL_free(ctx->shared_secret); - ctx->shared_secret = NULL; - ctx->shared_secretlen = 0; - } - return erv; -} - -/* - * @brief do "middle" of HPKE, between KEM and AEAD - * @param ctx is the OSSL_HPKE_CTX - * @param info is a buffer for the added binding information - * @param infolen is the length of info - * @return 0 for error, 1 for success - * - * This does all the HPKE extracts and expands as defined in RFC9180 - * section 5.1, (badly termed there as a "key schedule") and sets the - * ctx fields for the shared_secret, nonce, key and exporter_secret - */ -static int hpke_do_middle(OSSL_HPKE_CTX *ctx, - const unsigned char *info, size_t infolen) -{ - int erv = 0; - size_t ks_contextlen = OSSL_HPKE_MAXSIZE; - unsigned char ks_context[OSSL_HPKE_MAXSIZE]; - size_t halflen = 0; - size_t pskidlen = 0; - const OSSL_HPKE_AEAD_INFO *aead_info = NULL; - const OSSL_HPKE_KDF_INFO *kdf_info = NULL; - size_t secretlen = OSSL_HPKE_MAXSIZE; - unsigned char secret[OSSL_HPKE_MAXSIZE]; - EVP_KDF_CTX *kctx = NULL; - unsigned char suitebuf[6]; - const char *mdname = NULL; - - /* only let this be done once */ - if (ctx->exportersec != NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - if (ossl_HPKE_KEM_INFO_find_id(ctx->suite.kem_id) == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - aead_info = ossl_HPKE_AEAD_INFO_find_id(ctx->suite.aead_id); - if (aead_info == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - kdf_info = ossl_HPKE_KDF_INFO_find_id(ctx->suite.kdf_id); - if (kdf_info == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - mdname = kdf_info->mdname; - /* create key schedule context */ - memset(ks_context, 0, sizeof(ks_context)); - ks_context[0] = (unsigned char)(ctx->mode % 256); - ks_contextlen--; /* remaining space */ - halflen = kdf_info->Nh; - if ((2 * halflen) > ks_contextlen) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - /* check a psk was set if in that mode */ - if (ctx->mode == OSSL_HPKE_MODE_PSK - || ctx->mode == OSSL_HPKE_MODE_PSKAUTH) { - if (ctx->psk == NULL || ctx->psklen == 0 || ctx->pskid == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - } - kctx = ossl_kdf_ctx_create("HKDF", mdname, ctx->libctx, ctx->propq); - if (kctx == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - pskidlen = (ctx->psk == NULL ? 0 : strlen(ctx->pskid)); - /* full suite details as per RFC9180 sec 5.1 */ - suitebuf[0] = ctx->suite.kem_id / 256; - suitebuf[1] = ctx->suite.kem_id % 256; - suitebuf[2] = ctx->suite.kdf_id / 256; - suitebuf[3] = ctx->suite.kdf_id % 256; - suitebuf[4] = ctx->suite.aead_id / 256; - suitebuf[5] = ctx->suite.aead_id % 256; - /* Extract and Expand variously... */ - if (ossl_hpke_labeled_extract(kctx, ks_context + 1, halflen, - NULL, 0, OSSL_HPKE_SEC51LABEL, - suitebuf, sizeof(suitebuf), - OSSL_HPKE_PSKIDHASH_LABEL, - (unsigned char *)ctx->pskid, pskidlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (ossl_hpke_labeled_extract(kctx, ks_context + 1 + halflen, halflen, - NULL, 0, OSSL_HPKE_SEC51LABEL, - suitebuf, sizeof(suitebuf), - OSSL_HPKE_INFOHASH_LABEL, - (unsigned char *)info, infolen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - ks_contextlen = 1 + 2 * halflen; - secretlen = kdf_info->Nh; - if (secretlen > OSSL_HPKE_MAXSIZE) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (ossl_hpke_labeled_extract(kctx, secret, secretlen, - ctx->shared_secret, ctx->shared_secretlen, - OSSL_HPKE_SEC51LABEL, - suitebuf, sizeof(suitebuf), - OSSL_HPKE_SECRET_LABEL, - ctx->psk, ctx->psklen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (ctx->suite.aead_id != OSSL_HPKE_AEAD_ID_EXPORTONLY) { - /* we only need nonce/key for non export AEADs */ - ctx->noncelen = aead_info->Nn; - ctx->nonce = OPENSSL_malloc(ctx->noncelen); - if (ctx->nonce == NULL) - goto err; - if (ossl_hpke_labeled_expand(kctx, ctx->nonce, ctx->noncelen, - secret, secretlen, OSSL_HPKE_SEC51LABEL, - suitebuf, sizeof(suitebuf), - OSSL_HPKE_NONCE_LABEL, - ks_context, ks_contextlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - ctx->keylen = aead_info->Nk; - ctx->key = OPENSSL_malloc(ctx->keylen); - if (ctx->key == NULL) - goto err; - if (ossl_hpke_labeled_expand(kctx, ctx->key, ctx->keylen, - secret, secretlen, OSSL_HPKE_SEC51LABEL, - suitebuf, sizeof(suitebuf), - OSSL_HPKE_KEY_LABEL, - ks_context, ks_contextlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - } - ctx->exporterseclen = kdf_info->Nh; - ctx->exportersec = OPENSSL_malloc(ctx->exporterseclen); - if (ctx->exportersec == NULL) - goto err; - if (ossl_hpke_labeled_expand(kctx, ctx->exportersec, ctx->exporterseclen, - secret, secretlen, OSSL_HPKE_SEC51LABEL, - suitebuf, sizeof(suitebuf), - OSSL_HPKE_EXP_LABEL, - ks_context, ks_contextlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - erv = 1; - -err: - OPENSSL_cleanse(ks_context, OSSL_HPKE_MAXSIZE); - OPENSSL_cleanse(secret, OSSL_HPKE_MAXSIZE); - EVP_KDF_CTX_free(kctx); - return erv; -} - -/* - * externally visible functions from below here, API documentation is - * in doc/man3/OSSL_HPKE_CTX_new.pod to avoid duplication - */ - -OSSL_HPKE_CTX *OSSL_HPKE_CTX_new(int mode, OSSL_HPKE_SUITE suite, int role, - OSSL_LIB_CTX *libctx, const char *propq) -{ - OSSL_HPKE_CTX *ctx = NULL; - const OSSL_HPKE_KEM_INFO *kem_info; - const OSSL_HPKE_KDF_INFO *kdf_info; - const OSSL_HPKE_AEAD_INFO *aead_info; - - if (hpke_mode_check(mode) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return NULL; - } - if (hpke_suite_check(suite, &kem_info, &kdf_info, &aead_info) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return NULL; - } - if (role != OSSL_HPKE_ROLE_SENDER && role != OSSL_HPKE_ROLE_RECEIVER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) - return NULL; - ctx->libctx = libctx; - if (propq != NULL) { - ctx->propq = OPENSSL_strdup(propq); - if (ctx->propq == NULL) - goto err; - } - if (suite.aead_id != OSSL_HPKE_AEAD_ID_EXPORTONLY) { - ctx->aead_ciph = EVP_CIPHER_fetch(libctx, aead_info->name, propq); - if (ctx->aead_ciph == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_FETCH_FAILED); - goto err; - } - } - ctx->role = role; - ctx->mode = mode; - ctx->suite = suite; - ctx->kem_info = kem_info; - ctx->kdf_info = kdf_info; - ctx->aead_info = aead_info; - return ctx; - - err: - EVP_CIPHER_free(ctx->aead_ciph); - OPENSSL_free(ctx); - return NULL; -} - -void OSSL_HPKE_CTX_free(OSSL_HPKE_CTX *ctx) -{ - if (ctx == NULL) - return; - EVP_CIPHER_free(ctx->aead_ciph); - OPENSSL_free(ctx->propq); - OPENSSL_clear_free(ctx->exportersec, ctx->exporterseclen); - OPENSSL_free(ctx->pskid); - OPENSSL_clear_free(ctx->psk, ctx->psklen); - OPENSSL_clear_free(ctx->key, ctx->keylen); - OPENSSL_clear_free(ctx->nonce, ctx->noncelen); - OPENSSL_clear_free(ctx->shared_secret, ctx->shared_secretlen); - OPENSSL_clear_free(ctx->ikme, ctx->ikmelen); - EVP_PKEY_free(ctx->authpriv); - OPENSSL_free(ctx->authpub); - - OPENSSL_free(ctx); - return; -} - -int OSSL_HPKE_CTX_set1_psk(OSSL_HPKE_CTX *ctx, - const char *pskid, - const unsigned char *psk, size_t psklen) -{ - if (ctx == NULL || pskid == NULL || psk == NULL || psklen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (psklen > OSSL_HPKE_MAX_PARMLEN) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (psklen < OSSL_HPKE_MIN_PSKLEN) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (strlen(pskid) > OSSL_HPKE_MAX_PARMLEN) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (strlen(pskid) == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->mode != OSSL_HPKE_MODE_PSK - && ctx->mode != OSSL_HPKE_MODE_PSKAUTH) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - /* free previous values if any */ - OPENSSL_clear_free(ctx->psk, ctx->psklen); - ctx->psk = OPENSSL_memdup(psk, psklen); - if (ctx->psk == NULL) - return 0; - ctx->psklen = psklen; - OPENSSL_free(ctx->pskid); - ctx->pskid = OPENSSL_strdup(pskid); - if (ctx->pskid == NULL) { - OPENSSL_clear_free(ctx->psk, ctx->psklen); - ctx->psk = NULL; - ctx->psklen = 0; - return 0; - } - return 1; -} - -int OSSL_HPKE_CTX_set1_ikme(OSSL_HPKE_CTX *ctx, - const unsigned char *ikme, size_t ikmelen) -{ - if (ctx == NULL || ikme == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (ikmelen == 0 || ikmelen > OSSL_HPKE_MAX_PARMLEN) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->role != OSSL_HPKE_ROLE_SENDER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - OPENSSL_clear_free(ctx->ikme, ctx->ikmelen); - ctx->ikme = OPENSSL_memdup(ikme, ikmelen); - if (ctx->ikme == NULL) - return 0; - ctx->ikmelen = ikmelen; - return 1; -} - -int OSSL_HPKE_CTX_set1_authpriv(OSSL_HPKE_CTX *ctx, EVP_PKEY *priv) -{ - if (ctx == NULL || priv == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (ctx->mode != OSSL_HPKE_MODE_AUTH - && ctx->mode != OSSL_HPKE_MODE_PSKAUTH) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->role != OSSL_HPKE_ROLE_SENDER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - EVP_PKEY_free(ctx->authpriv); - ctx->authpriv = EVP_PKEY_dup(priv); - if (ctx->authpriv == NULL) - return 0; - return 1; -} - -int OSSL_HPKE_CTX_set1_authpub(OSSL_HPKE_CTX *ctx, - const unsigned char *pub, size_t publen) -{ - int erv = 0; - EVP_PKEY *pubp = NULL; - unsigned char *lpub = NULL; - size_t lpublen = 0; - const OSSL_HPKE_KEM_INFO *kem_info = NULL; - - if (ctx == NULL || pub == NULL || publen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (ctx->mode != OSSL_HPKE_MODE_AUTH - && ctx->mode != OSSL_HPKE_MODE_PSKAUTH) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->role != OSSL_HPKE_ROLE_RECEIVER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - /* check the value seems like a good public key for this kem */ - kem_info = ossl_HPKE_KEM_INFO_find_id(ctx->suite.kem_id); - if (kem_info == NULL) - return 0; - if (hpke_kem_id_nist_curve(ctx->suite.kem_id) == 1) { - pubp = evp_pkey_new_raw_nist_public_key(ctx->libctx, ctx->propq, - kem_info->groupname, - pub, publen); - } else { - pubp = EVP_PKEY_new_raw_public_key_ex(ctx->libctx, - kem_info->keytype, - ctx->propq, - pub, publen); - } - if (pubp == NULL) { - /* can happen based on external input - buffer value may be garbage */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - /* - * extract out the public key in encoded form so we - * should be fine even if given compressed form - */ - lpub = OPENSSL_malloc(OSSL_HPKE_MAXSIZE); - if (lpub == NULL) - goto err; - if (EVP_PKEY_get_octet_string_param(pubp, - OSSL_PKEY_PARAM_ENCODED_PUBLIC_KEY, - lpub, OSSL_HPKE_MAXSIZE, &lpublen) - != 1) { - OPENSSL_free(lpub); - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - /* free up old value */ - OPENSSL_free(ctx->authpub); - ctx->authpub = lpub; - ctx->authpublen = lpublen; - erv = 1; - -err: - EVP_PKEY_free(pubp); - return erv; -} - -int OSSL_HPKE_CTX_get_seq(OSSL_HPKE_CTX *ctx, uint64_t *seq) -{ - if (ctx == NULL || seq == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - *seq = ctx->seq; - return 1; -} - -int OSSL_HPKE_CTX_set_seq(OSSL_HPKE_CTX *ctx, uint64_t seq) -{ - if (ctx == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - /* - * We disallow senders from doing this as it's dangerous - * Receivers are ok to use this, as no harm should ensue - * if they go wrong. - */ - if (ctx->role == OSSL_HPKE_ROLE_SENDER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - ctx->seq = seq; - return 1; -} - -int OSSL_HPKE_encap(OSSL_HPKE_CTX *ctx, - unsigned char *enc, size_t *enclen, - const unsigned char *pub, size_t publen, - const unsigned char *info, size_t infolen) -{ - int erv = 1; - size_t minenc = 0; - - if (ctx == NULL || enc == NULL || enclen == NULL || *enclen == 0 - || pub == NULL || publen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->role != OSSL_HPKE_ROLE_SENDER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (infolen > OSSL_HPKE_MAX_INFOLEN) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (infolen > 0 && info == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - minenc = OSSL_HPKE_get_public_encap_size(ctx->suite); - if (minenc == 0 || minenc > *enclen) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->shared_secret != NULL) { - /* only allow one encap per OSSL_HPKE_CTX */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - if (hpke_encap(ctx, enc, enclen, pub, publen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - /* - * note that the info is not part of the context as it - * only needs to be used once here so doesn't need to - * be stored - */ - erv = hpke_do_middle(ctx, info, infolen); - return erv; -} - -int OSSL_HPKE_decap(OSSL_HPKE_CTX *ctx, - const unsigned char *enc, size_t enclen, - EVP_PKEY *recippriv, - const unsigned char *info, size_t infolen) -{ - int erv = 1; - size_t minenc = 0; - - if (ctx == NULL || enc == NULL || enclen == 0 || recippriv == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->role != OSSL_HPKE_ROLE_RECEIVER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (infolen > OSSL_HPKE_MAX_INFOLEN) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (infolen > 0 && info == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - minenc = OSSL_HPKE_get_public_encap_size(ctx->suite); - if (minenc == 0 || minenc > enclen) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->shared_secret != NULL) { - /* only allow one encap per OSSL_HPKE_CTX */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - erv = hpke_decap(ctx, enc, enclen, recippriv); - if (erv != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - /* - * note that the info is not part of the context as it - * only needs to be used once here so doesn't need to - * be stored - */ - erv = hpke_do_middle(ctx, info, infolen); - return erv; -} - -int OSSL_HPKE_seal(OSSL_HPKE_CTX *ctx, - unsigned char *ct, size_t *ctlen, - const unsigned char *aad, size_t aadlen, - const unsigned char *pt, size_t ptlen) -{ - unsigned char seqbuf[OSSL_HPKE_MAX_NONCELEN]; - size_t seqlen = 0; - - if (ctx == NULL || ct == NULL || ctlen == NULL || *ctlen == 0 - || pt == NULL || ptlen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->role != OSSL_HPKE_ROLE_SENDER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if ((ctx->seq + 1) == 0) { /* wrap around imminent !!! */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - if (ctx->key == NULL || ctx->nonce == NULL) { - /* need to have done an encap first, info can be NULL */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - seqlen = hpke_seqnonce2buf(ctx, seqbuf, sizeof(seqbuf)); - if (seqlen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - if (hpke_aead_enc(ctx, seqbuf, aad, aadlen, pt, ptlen, ct, ctlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - OPENSSL_cleanse(seqbuf, sizeof(seqbuf)); - return 0; - } else { - ctx->seq++; - } - OPENSSL_cleanse(seqbuf, sizeof(seqbuf)); - return 1; -} - -int OSSL_HPKE_open(OSSL_HPKE_CTX *ctx, - unsigned char *pt, size_t *ptlen, - const unsigned char *aad, size_t aadlen, - const unsigned char *ct, size_t ctlen) -{ - unsigned char seqbuf[OSSL_HPKE_MAX_NONCELEN]; - size_t seqlen = 0; - - if (ctx == NULL || pt == NULL || ptlen == NULL || *ptlen == 0 - || ct == NULL || ctlen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->role != OSSL_HPKE_ROLE_RECEIVER) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if ((ctx->seq + 1) == 0) { /* wrap around imminent !!! */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - if (ctx->key == NULL || ctx->nonce == NULL) { - /* need to have done an encap first, info can be NULL */ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - seqlen = hpke_seqnonce2buf(ctx, seqbuf, sizeof(seqbuf)); - if (seqlen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - if (hpke_aead_dec(ctx, seqbuf, aad, aadlen, ct, ctlen, pt, ptlen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - OPENSSL_cleanse(seqbuf, sizeof(seqbuf)); - return 0; - } - ctx->seq++; - OPENSSL_cleanse(seqbuf, sizeof(seqbuf)); - return 1; -} - -int OSSL_HPKE_export(OSSL_HPKE_CTX *ctx, - unsigned char *secret, size_t secretlen, - const unsigned char *label, size_t labellen) -{ - int erv = 0; - EVP_KDF_CTX *kctx = NULL; - unsigned char suitebuf[6]; - const char *mdname = NULL; - const OSSL_HPKE_KDF_INFO *kdf_info = NULL; - - if (ctx == NULL || secret == NULL || secretlen == 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (labellen > OSSL_HPKE_MAX_PARMLEN) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (labellen > 0 && label == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ctx->exportersec == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return 0; - } - kdf_info = ossl_HPKE_KDF_INFO_find_id(ctx->suite.kdf_id); - if (kdf_info == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - mdname = kdf_info->mdname; - kctx = ossl_kdf_ctx_create("HKDF", mdname, ctx->libctx, ctx->propq); - if (kctx == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return 0; - } - /* full suiteid as per RFC9180 sec 5.3 */ - suitebuf[0] = ctx->suite.kem_id / 256; - suitebuf[1] = ctx->suite.kem_id % 256; - suitebuf[2] = ctx->suite.kdf_id / 256; - suitebuf[3] = ctx->suite.kdf_id % 256; - suitebuf[4] = ctx->suite.aead_id / 256; - suitebuf[5] = ctx->suite.aead_id % 256; - erv = ossl_hpke_labeled_expand(kctx, secret, secretlen, - ctx->exportersec, ctx->exporterseclen, - OSSL_HPKE_SEC51LABEL, - suitebuf, sizeof(suitebuf), - OSSL_HPKE_EXP_SEC_LABEL, - label, labellen); - EVP_KDF_CTX_free(kctx); - if (erv != 1) - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return erv; -} - -int OSSL_HPKE_keygen(OSSL_HPKE_SUITE suite, - unsigned char *pub, size_t *publen, EVP_PKEY **priv, - const unsigned char *ikm, size_t ikmlen, - OSSL_LIB_CTX *libctx, const char *propq) -{ - int erv = 0; /* Our error return value - 1 is success */ - EVP_PKEY_CTX *pctx = NULL; - EVP_PKEY *skR = NULL; - const OSSL_HPKE_KEM_INFO *kem_info = NULL; - OSSL_PARAM params[3], *p = params; - - if (pub == NULL || publen == NULL || *publen == 0 || priv == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (hpke_suite_check(suite, &kem_info, NULL, NULL) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if ((ikmlen > 0 && ikm == NULL) - || (ikmlen == 0 && ikm != NULL) - || ikmlen > OSSL_HPKE_MAX_PARMLEN) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - if (hpke_kem_id_nist_curve(suite.kem_id) == 1) { - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_GROUP_NAME, - (char *)kem_info->groupname, 0); - pctx = EVP_PKEY_CTX_new_from_name(libctx, "EC", propq); - } else { - pctx = EVP_PKEY_CTX_new_from_name(libctx, kem_info->keytype, propq); - } - if (pctx == NULL - || EVP_PKEY_keygen_init(pctx) <= 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (ikm != NULL) - *p++ = OSSL_PARAM_construct_octet_string(OSSL_PKEY_PARAM_DHKEM_IKM, - (char *)ikm, ikmlen); - *p = OSSL_PARAM_construct_end(); - if (EVP_PKEY_CTX_set_params(pctx, params) <= 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - if (EVP_PKEY_generate(pctx, &skR) <= 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - EVP_PKEY_CTX_free(pctx); - pctx = NULL; - if (EVP_PKEY_get_octet_string_param(skR, OSSL_PKEY_PARAM_ENCODED_PUBLIC_KEY, - pub, *publen, publen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - *priv = skR; - erv = 1; - -err: - if (erv != 1) - EVP_PKEY_free(skR); - EVP_PKEY_CTX_free(pctx); - return erv; -} - -int OSSL_HPKE_suite_check(OSSL_HPKE_SUITE suite) -{ - return hpke_suite_check(suite, NULL, NULL, NULL); -} - -int OSSL_HPKE_get_grease_value(const OSSL_HPKE_SUITE *suite_in, - OSSL_HPKE_SUITE *suite, - unsigned char *enc, size_t *enclen, - unsigned char *ct, size_t ctlen, - OSSL_LIB_CTX *libctx, const char *propq) -{ - OSSL_HPKE_SUITE chosen; - size_t plen = 0; - const OSSL_HPKE_KEM_INFO *kem_info = NULL; - const OSSL_HPKE_AEAD_INFO *aead_info = NULL; - EVP_PKEY *fakepriv = NULL; - - if (enc == NULL || enclen == 0 - || ct == NULL || ctlen == 0 || suite == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (suite_in == NULL) { - /* choose a random suite */ - if (hpke_random_suite(libctx, propq, &chosen) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - } else { - chosen = *suite_in; - } - if (hpke_suite_check(chosen, &kem_info, NULL, &aead_info) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - *suite = chosen; - /* make sure room for tag and one plaintext octet */ - if (aead_info->taglen >= ctlen) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - /* publen */ - plen = kem_info->Npk; - if (plen > *enclen) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - /* - * In order for our enc to look good for sure, we generate and then - * delete a real key for that curve - bit OTT but it ensures we do - * get the encoding right (e.g. 0x04 as 1st octet for NIST curves in - * uncompressed form) and that the value really does map to a point on - * the relevant curve. - */ - if (OSSL_HPKE_keygen(chosen, enc, enclen, &fakepriv, NULL, 0, - libctx, propq) != 1) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - EVP_PKEY_free(fakepriv); - if (RAND_bytes_ex(libctx, ct, ctlen, 0) <= 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - goto err; - } - return 1; -err: - return 0; -} - -int OSSL_HPKE_str2suite(const char *str, OSSL_HPKE_SUITE *suite) -{ - return ossl_hpke_str2suite(str, suite); -} - -size_t OSSL_HPKE_get_ciphertext_size(OSSL_HPKE_SUITE suite, size_t clearlen) -{ - size_t enclen = 0; - size_t cipherlen = 0; - - if (hpke_expansion(suite, &enclen, clearlen, &cipherlen) != 1) - return 0; - return cipherlen; -} - -size_t OSSL_HPKE_get_public_encap_size(OSSL_HPKE_SUITE suite) -{ - size_t enclen = 0; - size_t cipherlen = 0; - size_t clearlen = 16; - - if (hpke_expansion(suite, &enclen, clearlen, &cipherlen) != 1) - return 0; - return enclen; -} - -size_t OSSL_HPKE_get_recommended_ikmelen(OSSL_HPKE_SUITE suite) -{ - const OSSL_HPKE_KEM_INFO *kem_info = NULL; - - if (hpke_suite_check(suite, &kem_info, NULL, NULL) != 1) - return 0; - if (kem_info == NULL) - return 0; - - return kem_info->Nsk; -} diff --git a/openssl/src/crypto/hpke/hpke_util.c b/openssl/src/crypto/hpke/hpke_util.c deleted file mode 100644 index a9d86a935..000000000 --- a/openssl/src/crypto/hpke/hpke_util.c +++ /dev/null @@ -1,528 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "crypto/ecx.h" -#include "crypto/rand.h" -#include "internal/hpke_util.h" -#include "internal/packet.h" -#include "internal/nelem.h" -#include "internal/common.h" - -/* - * Delimiter used in OSSL_HPKE_str2suite - */ -#define OSSL_HPKE_STR_DELIMCHAR ',' - -/* - * table with identifier and synonym strings - * right now, there are 4 synonyms for each - a name, a hex string - * a hex string with a leading zero and a decimal string - more - * could be added but that seems like enough - */ -typedef struct { - uint16_t id; - char *synonyms[4]; -} synonymttab_t; - -/* max length of string we'll try map to a suite */ -#define OSSL_HPKE_MAX_SUITESTR 38 - -/* Define HPKE labels from RFC9180 in hex for EBCDIC compatibility */ -/* ASCII: "HPKE-v1", in hex for EBCDIC compatibility */ -static const char LABEL_HPKEV1[] = "\x48\x50\x4B\x45\x2D\x76\x31"; - -/* - * Note that if additions are made to the set of IANA codepoints - * and the tables below, corresponding additions should also be - * made to the synonymtab tables a little further down so that - * OSSL_HPKE_str2suite() continues to function correctly. - * - * The canonical place to check for IANA registered codepoints - * is: https://www.iana.org/assignments/hpke/hpke.xhtml - */ - -/* - * @brief table of KEMs - * See RFC9180 Section 7.1 "Table 2 KEM IDs" - */ -static const OSSL_HPKE_KEM_INFO hpke_kem_tab[] = { -#ifndef OPENSSL_NO_EC - { OSSL_HPKE_KEM_ID_P256, "EC", OSSL_HPKE_KEMSTR_P256, - LN_sha256, SHA256_DIGEST_LENGTH, 65, 65, 32, 0xFF }, - { OSSL_HPKE_KEM_ID_P384, "EC", OSSL_HPKE_KEMSTR_P384, - LN_sha384, SHA384_DIGEST_LENGTH, 97, 97, 48, 0xFF }, - { OSSL_HPKE_KEM_ID_P521, "EC", OSSL_HPKE_KEMSTR_P521, - LN_sha512, SHA512_DIGEST_LENGTH, 133, 133, 66, 0x01 }, -# ifndef OPENSSL_NO_ECX - { OSSL_HPKE_KEM_ID_X25519, OSSL_HPKE_KEMSTR_X25519, NULL, - LN_sha256, SHA256_DIGEST_LENGTH, - X25519_KEYLEN, X25519_KEYLEN, X25519_KEYLEN, 0x00 }, - { OSSL_HPKE_KEM_ID_X448, OSSL_HPKE_KEMSTR_X448, NULL, - LN_sha512, SHA512_DIGEST_LENGTH, - X448_KEYLEN, X448_KEYLEN, X448_KEYLEN, 0x00 } -# endif -#else - { OSSL_HPKE_KEM_ID_RESERVED, NULL, NULL, NULL, 0, 0, 0, 0, 0x00 } -#endif -}; - -/* - * @brief table of AEADs - * See RFC9180 Section 7.2 "Table 3 KDF IDs" - */ -static const OSSL_HPKE_AEAD_INFO hpke_aead_tab[] = { - { OSSL_HPKE_AEAD_ID_AES_GCM_128, LN_aes_128_gcm, 16, 16, - OSSL_HPKE_MAX_NONCELEN }, - { OSSL_HPKE_AEAD_ID_AES_GCM_256, LN_aes_256_gcm, 16, 32, - OSSL_HPKE_MAX_NONCELEN }, -#if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305) - { OSSL_HPKE_AEAD_ID_CHACHA_POLY1305, LN_chacha20_poly1305, 16, 32, - OSSL_HPKE_MAX_NONCELEN }, -#endif - { OSSL_HPKE_AEAD_ID_EXPORTONLY, NULL, 0, 0, 0 } -}; - -/* - * @brief table of KDFs - * See RFC9180 Section 7.3 "Table 5 AEAD IDs" - */ -static const OSSL_HPKE_KDF_INFO hpke_kdf_tab[] = { - { OSSL_HPKE_KDF_ID_HKDF_SHA256, LN_sha256, SHA256_DIGEST_LENGTH }, - { OSSL_HPKE_KDF_ID_HKDF_SHA384, LN_sha384, SHA384_DIGEST_LENGTH }, - { OSSL_HPKE_KDF_ID_HKDF_SHA512, LN_sha512, SHA512_DIGEST_LENGTH } -}; - -/** - * Synonym tables for KEMs, KDFs and AEADs: idea is to allow - * mapping strings to suites with a little flexibility in terms - * of allowing a name or a couple of forms of number (for - * the IANA codepoint). If new IANA codepoints are allocated - * then these tables should be updated at the same time as the - * others above. - * - * The function to use these is ossl_hpke_str2suite() further down - * this file and shouldn't need modification so long as the table - * sizes (i.e. allow exactly 4 synonyms) don't change. - */ -static const synonymttab_t kemstrtab[] = { - {OSSL_HPKE_KEM_ID_P256, - {OSSL_HPKE_KEMSTR_P256, "0x10", "0x10", "16" }}, - {OSSL_HPKE_KEM_ID_P384, - {OSSL_HPKE_KEMSTR_P384, "0x11", "0x11", "17" }}, - {OSSL_HPKE_KEM_ID_P521, - {OSSL_HPKE_KEMSTR_P521, "0x12", "0x12", "18" }}, -# ifndef OPENSSL_NO_ECX - {OSSL_HPKE_KEM_ID_X25519, - {OSSL_HPKE_KEMSTR_X25519, "0x20", "0x20", "32" }}, - {OSSL_HPKE_KEM_ID_X448, - {OSSL_HPKE_KEMSTR_X448, "0x21", "0x21", "33" }} -# endif -}; -static const synonymttab_t kdfstrtab[] = { - {OSSL_HPKE_KDF_ID_HKDF_SHA256, - {OSSL_HPKE_KDFSTR_256, "0x1", "0x01", "1"}}, - {OSSL_HPKE_KDF_ID_HKDF_SHA384, - {OSSL_HPKE_KDFSTR_384, "0x2", "0x02", "2"}}, - {OSSL_HPKE_KDF_ID_HKDF_SHA512, - {OSSL_HPKE_KDFSTR_512, "0x3", "0x03", "3"}} -}; -static const synonymttab_t aeadstrtab[] = { - {OSSL_HPKE_AEAD_ID_AES_GCM_128, - {OSSL_HPKE_AEADSTR_AES128GCM, "0x1", "0x01", "1"}}, - {OSSL_HPKE_AEAD_ID_AES_GCM_256, - {OSSL_HPKE_AEADSTR_AES256GCM, "0x2", "0x02", "2"}}, - {OSSL_HPKE_AEAD_ID_CHACHA_POLY1305, - {OSSL_HPKE_AEADSTR_CP, "0x3", "0x03", "3"}}, - {OSSL_HPKE_AEAD_ID_EXPORTONLY, - {OSSL_HPKE_AEADSTR_EXP, "ff", "0xff", "255"}} -}; - -/* Return an object containing KEM constants associated with a EC curve name */ -const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_curve(const char *curve) -{ - int i, sz = OSSL_NELEM(hpke_kem_tab); - - for (i = 0; i < sz; ++i) { - const char *group = hpke_kem_tab[i].groupname; - - if (group == NULL) - group = hpke_kem_tab[i].keytype; - if (OPENSSL_strcasecmp(curve, group) == 0) - return &hpke_kem_tab[i]; - } - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CURVE); - return NULL; -} - -const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_id(uint16_t kemid) -{ - int i, sz = OSSL_NELEM(hpke_kem_tab); - - /* - * this check can happen if we're in a no-ec build and there are no - * KEMS available - */ - if (kemid == OSSL_HPKE_KEM_ID_RESERVED) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CURVE); - return NULL; - } - for (i = 0; i != sz; ++i) { - if (hpke_kem_tab[i].kem_id == kemid) - return &hpke_kem_tab[i]; - } - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CURVE); - return NULL; -} - -const OSSL_HPKE_KEM_INFO *ossl_HPKE_KEM_INFO_find_random(OSSL_LIB_CTX *ctx) -{ - uint32_t rval = 0; - int err = 0; - size_t sz = OSSL_NELEM(hpke_kem_tab); - - rval = ossl_rand_uniform_uint32(ctx, sz, &err); - return (err == 1 ? NULL : &hpke_kem_tab[rval]); -} - -const OSSL_HPKE_KDF_INFO *ossl_HPKE_KDF_INFO_find_id(uint16_t kdfid) -{ - int i, sz = OSSL_NELEM(hpke_kdf_tab); - - for (i = 0; i != sz; ++i) { - if (hpke_kdf_tab[i].kdf_id == kdfid) - return &hpke_kdf_tab[i]; - } - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KDF); - return NULL; -} - -const OSSL_HPKE_KDF_INFO *ossl_HPKE_KDF_INFO_find_random(OSSL_LIB_CTX *ctx) -{ - uint32_t rval = 0; - int err = 0; - size_t sz = OSSL_NELEM(hpke_kdf_tab); - - rval = ossl_rand_uniform_uint32(ctx, sz, &err); - return (err == 1 ? NULL : &hpke_kdf_tab[rval]); -} - -const OSSL_HPKE_AEAD_INFO *ossl_HPKE_AEAD_INFO_find_id(uint16_t aeadid) -{ - int i, sz = OSSL_NELEM(hpke_aead_tab); - - for (i = 0; i != sz; ++i) { - if (hpke_aead_tab[i].aead_id == aeadid) - return &hpke_aead_tab[i]; - } - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_AEAD); - return NULL; -} - -const OSSL_HPKE_AEAD_INFO *ossl_HPKE_AEAD_INFO_find_random(OSSL_LIB_CTX *ctx) -{ - uint32_t rval = 0; - int err = 0; - /* the minus 1 below is so we don't pick the EXPORTONLY codepoint */ - size_t sz = OSSL_NELEM(hpke_aead_tab) - 1; - - rval = ossl_rand_uniform_uint32(ctx, sz, &err); - return (err == 1 ? NULL : &hpke_aead_tab[rval]); -} - -static int kdf_derive(EVP_KDF_CTX *kctx, - unsigned char *out, size_t outlen, int mode, - const unsigned char *salt, size_t saltlen, - const unsigned char *ikm, size_t ikmlen, - const unsigned char *info, size_t infolen) -{ - int ret; - OSSL_PARAM params[5], *p = params; - - *p++ = OSSL_PARAM_construct_int(OSSL_KDF_PARAM_MODE, &mode); - if (salt != NULL) - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_SALT, - (char *)salt, saltlen); - if (ikm != NULL) - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_KEY, - (char *)ikm, ikmlen); - if (info != NULL) - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_INFO, - (char *)info, infolen); - *p = OSSL_PARAM_construct_end(); - ret = EVP_KDF_derive(kctx, out, outlen, params) > 0; - if (!ret) - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); - return ret; -} - -int ossl_hpke_kdf_extract(EVP_KDF_CTX *kctx, - unsigned char *prk, size_t prklen, - const unsigned char *salt, size_t saltlen, - const unsigned char *ikm, size_t ikmlen) -{ - return kdf_derive(kctx, prk, prklen, EVP_KDF_HKDF_MODE_EXTRACT_ONLY, - salt, saltlen, ikm, ikmlen, NULL, 0); -} - -/* Common code to perform a HKDF expand */ -int ossl_hpke_kdf_expand(EVP_KDF_CTX *kctx, - unsigned char *okm, size_t okmlen, - const unsigned char *prk, size_t prklen, - const unsigned char *info, size_t infolen) -{ - return kdf_derive(kctx, okm, okmlen, EVP_KDF_HKDF_MODE_EXPAND_ONLY, - NULL, 0, prk, prklen, info, infolen); -} - -/* - * See RFC 9180 Section 4 LabelExtract() - */ -int ossl_hpke_labeled_extract(EVP_KDF_CTX *kctx, - unsigned char *prk, size_t prklen, - const unsigned char *salt, size_t saltlen, - const char *protocol_label, - const unsigned char *suiteid, size_t suiteidlen, - const char *label, - const unsigned char *ikm, size_t ikmlen) -{ - int ret = 0; - size_t label_hpkev1len = 0; - size_t protocol_labellen = 0; - size_t labellen = 0; - size_t labeled_ikmlen = 0; - unsigned char *labeled_ikm = NULL; - WPACKET pkt; - - label_hpkev1len = strlen(LABEL_HPKEV1); - protocol_labellen = strlen(protocol_label); - labellen = strlen(label); - labeled_ikmlen = label_hpkev1len + protocol_labellen - + suiteidlen + labellen + ikmlen; - labeled_ikm = OPENSSL_malloc(labeled_ikmlen); - if (labeled_ikm == NULL) - return 0; - - /* labeled_ikm = concat("HPKE-v1", suiteid, label, ikm) */ - if (!WPACKET_init_static_len(&pkt, labeled_ikm, labeled_ikmlen, 0) - || !WPACKET_memcpy(&pkt, LABEL_HPKEV1, label_hpkev1len) - || !WPACKET_memcpy(&pkt, protocol_label, protocol_labellen) - || !WPACKET_memcpy(&pkt, suiteid, suiteidlen) - || !WPACKET_memcpy(&pkt, label, labellen) - || !WPACKET_memcpy(&pkt, ikm, ikmlen) - || !WPACKET_get_total_written(&pkt, &labeled_ikmlen) - || !WPACKET_finish(&pkt)) { - ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); - goto end; - } - - ret = ossl_hpke_kdf_extract(kctx, prk, prklen, salt, saltlen, - labeled_ikm, labeled_ikmlen); -end: - WPACKET_cleanup(&pkt); - OPENSSL_cleanse(labeled_ikm, labeled_ikmlen); - OPENSSL_free(labeled_ikm); - return ret; -} - -/* - * See RFC 9180 Section 4 LabelExpand() - */ -int ossl_hpke_labeled_expand(EVP_KDF_CTX *kctx, - unsigned char *okm, size_t okmlen, - const unsigned char *prk, size_t prklen, - const char *protocol_label, - const unsigned char *suiteid, size_t suiteidlen, - const char *label, - const unsigned char *info, size_t infolen) -{ - int ret = 0; - size_t label_hpkev1len = 0; - size_t protocol_labellen = 0; - size_t labellen = 0; - size_t labeled_infolen = 0; - unsigned char *labeled_info = NULL; - WPACKET pkt; - - label_hpkev1len = strlen(LABEL_HPKEV1); - protocol_labellen = strlen(protocol_label); - labellen = strlen(label); - labeled_infolen = 2 + okmlen + prklen + label_hpkev1len - + protocol_labellen + suiteidlen + labellen + infolen; - labeled_info = OPENSSL_malloc(labeled_infolen); - if (labeled_info == NULL) - return 0; - - /* labeled_info = concat(okmlen, "HPKE-v1", suiteid, label, info) */ - if (!WPACKET_init_static_len(&pkt, labeled_info, labeled_infolen, 0) - || !WPACKET_put_bytes_u16(&pkt, okmlen) - || !WPACKET_memcpy(&pkt, LABEL_HPKEV1, label_hpkev1len) - || !WPACKET_memcpy(&pkt, protocol_label, protocol_labellen) - || !WPACKET_memcpy(&pkt, suiteid, suiteidlen) - || !WPACKET_memcpy(&pkt, label, labellen) - || !WPACKET_memcpy(&pkt, info, infolen) - || !WPACKET_get_total_written(&pkt, &labeled_infolen) - || !WPACKET_finish(&pkt)) { - ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); - goto end; - } - - ret = ossl_hpke_kdf_expand(kctx, okm, okmlen, - prk, prklen, labeled_info, labeled_infolen); -end: - WPACKET_cleanup(&pkt); - OPENSSL_free(labeled_info); - return ret; -} - -/* Common code to create a HKDF ctx */ -EVP_KDF_CTX *ossl_kdf_ctx_create(const char *kdfname, const char *mdname, - OSSL_LIB_CTX *libctx, const char *propq) -{ - EVP_KDF *kdf; - EVP_KDF_CTX *kctx = NULL; - - kdf = EVP_KDF_fetch(libctx, kdfname, propq); - if (kdf == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_FETCH_FAILED); - return NULL; - } - kctx = EVP_KDF_CTX_new(kdf); - EVP_KDF_free(kdf); - if (kctx != NULL && mdname != NULL) { - OSSL_PARAM params[3], *p = params; - - if (mdname != NULL) - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST, - (char *)mdname, 0); - if (propq != NULL) - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_PROPERTIES, - (char *)propq, 0); - *p = OSSL_PARAM_construct_end(); - if (EVP_KDF_CTX_set_params(kctx, params) <= 0) { - EVP_KDF_CTX_free(kctx); - return NULL; - } - } - return kctx; -} - -/* - * @brief look for a label into the synonym tables, and return its id - * @param st is the string value - * @param synp is the synonyms labels array - * @param arrsize is the previous array size - * @return 0 when not found, else the matching item id. - */ -static uint16_t synonyms_name2id(const char *st, const synonymttab_t *synp, - size_t arrsize) -{ - size_t i, j; - - for (i = 0; i < arrsize; ++i) { - for (j = 0; j < OSSL_NELEM(synp[i].synonyms); ++j) { - if (OPENSSL_strcasecmp(st, synp[i].synonyms[j]) == 0) - return synp[i].id; - } - } - return 0; -} - -/* - * @brief map a string to a HPKE suite based on synonym tables - * @param str is the string value - * @param suite is the resulting suite - * @return 1 for success, otherwise failure - */ -int ossl_hpke_str2suite(const char *suitestr, OSSL_HPKE_SUITE *suite) -{ - uint16_t kem = 0, kdf = 0, aead = 0; - char *st = NULL, *instrcp = NULL; - size_t inplen; - int labels = 0, result = 0; - int delim_count = 0; - - if (suitestr == NULL || suitestr[0] == 0x00 || suite == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - inplen = OPENSSL_strnlen(suitestr, OSSL_HPKE_MAX_SUITESTR); - if (inplen >= OSSL_HPKE_MAX_SUITESTR) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - /* - * we don't want a delimiter at the end of the string; - * strtok_r/s() doesn't care about that, so we should - */ - if (suitestr[inplen - 1] == OSSL_HPKE_STR_DELIMCHAR) - return 0; - /* We want exactly two delimiters in the input string */ - for (st = (char *)suitestr; *st != '\0'; st++) { - if (*st == OSSL_HPKE_STR_DELIMCHAR) - delim_count++; - } - if (delim_count != 2) - return 0; - - /* Duplicate `suitestr` to allow its parsing */ - instrcp = OPENSSL_memdup(suitestr, inplen + 1); - if (instrcp == NULL) - goto fail; - - /* See if it contains a mix of our strings and numbers */ - st = instrcp; - - while (st != NULL && labels < 3) { - char *cp = strchr(st, OSSL_HPKE_STR_DELIMCHAR); - - /* add a NUL like strtok would if we're not at the end */ - if (cp != NULL) - *cp = '\0'; - - /* check if string is known or number and if so handle appropriately */ - if (labels == 0 - && (kem = synonyms_name2id(st, kemstrtab, - OSSL_NELEM(kemstrtab))) == 0) - goto fail; - else if (labels == 1 - && (kdf = synonyms_name2id(st, kdfstrtab, - OSSL_NELEM(kdfstrtab))) == 0) - goto fail; - else if (labels == 2 - && (aead = synonyms_name2id(st, aeadstrtab, - OSSL_NELEM(aeadstrtab))) == 0) - goto fail; - - if (cp == NULL) - st = NULL; - else - st = cp + 1; - ++labels; - } - if (st != NULL || labels != 3) - goto fail; - suite->kem_id = kem; - suite->kdf_id = kdf; - suite->aead_id = aead; - result = 1; - -fail: - OPENSSL_free(instrcp); - return result; -} diff --git a/openssl/src/crypto/http/http_client.c b/openssl/src/crypto/http/http_client.c index 9309954ef..cd9919a0c 100644 --- a/openssl/src/crypto/http/http_client.c +++ b/openssl/src/crypto/http/http_client.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright Siemens AG 2018-2020 * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -20,10 +20,10 @@ #include #include #include -#include #include "internal/sockets.h" -#include "internal/common.h" /* for ossl_assert() */ +#include "internal/cryptlib.h" /* for ossl_assert() */ +#define HAS_PREFIX(str, prefix) (strncmp(str, prefix, sizeof(prefix) - 1) == 0) #define HTTP_PREFIX "HTTP/" #define HTTP_VERSION_PATT "1." /* allow 1.x */ #define HTTP_VERSION_STR_LEN sizeof(HTTP_VERSION_PATT) /* == strlen("1.0") */ @@ -51,12 +51,11 @@ struct ossl_http_req_ctx_st { void *upd_arg; /* Optional arg for update callback function */ int use_ssl; /* Use HTTPS */ char *proxy; /* Optional proxy name or URI */ - char *server; /* Optional server hostname */ + char *server; /* Optional server host name */ char *port; /* Optional server port */ - BIO *mem; /* Mem BIO holding request header or response */ + BIO *mem; /* Memory BIO holding request/response header */ BIO *req; /* BIO holding the request provided by caller */ int method_POST; /* HTTP method is POST (else GET) */ - int text; /* Request content type is (likely) text */ char *expected_ct; /* Optional expected Content-Type */ int expect_asn1; /* Response must be ASN.1-encoded */ unsigned char *pos; /* Current position sending data */ @@ -67,7 +66,6 @@ struct ossl_http_req_ctx_st { time_t max_time; /* Maximum end time of current transfer, or 0 */ time_t max_total_time; /* Maximum end time of total transfer, or 0 */ char *redirection_url; /* Location obtained from HTTP status 301/302 */ - size_t max_hdr_lines; /* Max. number of http hdr lines, or 0 */ }; /* HTTP states */ @@ -76,18 +74,16 @@ struct ossl_http_req_ctx_st { #define OHS_ERROR (0 | OHS_NOREAD) /* Error condition */ #define OHS_ADD_HEADERS (1 | OHS_NOREAD) /* Adding header lines to request */ #define OHS_WRITE_INIT (2 | OHS_NOREAD) /* 1st call: ready to start send */ -#define OHS_WRITE_HDR1 (3 | OHS_NOREAD) /* Request header to be sent */ -#define OHS_WRITE_HDR (4 | OHS_NOREAD) /* Request header being sent */ -#define OHS_WRITE_REQ (5 | OHS_NOREAD) /* Request content being sent */ -#define OHS_FLUSH (6 | OHS_NOREAD) /* Request being flushed */ +#define OHS_WRITE_HDR (3 | OHS_NOREAD) /* Request header being sent */ +#define OHS_WRITE_REQ (4 | OHS_NOREAD) /* Request contents being sent */ +#define OHS_FLUSH (5 | OHS_NOREAD) /* Request being flushed */ #define OHS_FIRSTLINE 1 /* First line of response being read */ #define OHS_HEADERS 2 /* MIME headers of response being read */ -#define OHS_HEADERS_ERROR 3 /* MIME headers of resp. being read after error */ -#define OHS_REDIRECT 4 /* MIME headers being read, expecting Location */ -#define OHS_ASN1_HEADER 5 /* ASN1 sequence header (tag+length) being read */ -#define OHS_ASN1_CONTENT 6 /* ASN1 content octets being read */ -#define OHS_ASN1_DONE (7 | OHS_NOREAD) /* ASN1 content read completed */ -#define OHS_STREAM (8 | OHS_NOREAD) /* HTTP content stream to be read */ +#define OHS_REDIRECT 3 /* MIME headers being read, expecting Location */ +#define OHS_ASN1_HEADER 4 /* ASN1 sequence header (tag+length) being read */ +#define OHS_ASN1_CONTENT 5 /* ASN1 content octets being read */ +#define OHS_ASN1_DONE (6 | OHS_NOREAD) /* ASN1 content read completed */ +#define OHS_STREAM (7 | OHS_NOREAD) /* HTTP content stream to be read */ /* Low-level HTTP API implementation */ @@ -107,7 +103,6 @@ OSSL_HTTP_REQ_CTX *OSSL_HTTP_REQ_CTX_new(BIO *wbio, BIO *rbio, int buf_size) rctx->buf = OPENSSL_malloc(rctx->buf_size); rctx->wbio = wbio; rctx->rbio = rbio; - rctx->max_hdr_lines = OSSL_HTTP_DEFAULT_MAX_RESP_HDR_LINES; if (rctx->buf == NULL) { OPENSSL_free(rctx); return NULL; @@ -169,8 +164,7 @@ void OSSL_HTTP_REQ_CTX_set_max_response_length(OSSL_HTTP_REQ_CTX *rctx, /* * Create request line using |rctx| and |path| (or "/" in case |path| is NULL). - * Server name (and optional port) must be given if and only if - * a plain HTTP proxy is used and |path| does not begin with 'http://'. + * Server name (and port) must be given if and only if plain HTTP proxy is used. */ int OSSL_HTTP_REQ_CTX_set_request_line(OSSL_HTTP_REQ_CTX *rctx, int method_POST, const char *server, const char *port, @@ -199,17 +193,11 @@ int OSSL_HTTP_REQ_CTX_set_request_line(OSSL_HTTP_REQ_CTX *rctx, int method_POST, return 0; } - /* Make sure path includes a forward slash (abs_path) */ - if (path == NULL) { + /* Make sure path includes a forward slash */ + if (path == NULL) path = "/"; - } else if (HAS_PREFIX(path, "http://")) { /* absoluteURI for proxy use */ - if (server != NULL) { - ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - } else if (path[0] != '/' && BIO_printf(rctx->mem, "/") <= 0) { + if (path[0] != '/' && BIO_printf(rctx->mem, "/") <= 0) return 0; - } /* * Add (the rest of) the path and the HTTP version, * which is fixed to 1.0 for straightforward implementation of keep-alive @@ -278,10 +266,7 @@ int OSSL_HTTP_REQ_CTX_set_expected(OSSL_HTTP_REQ_CTX *rctx, static int set1_content(OSSL_HTTP_REQ_CTX *rctx, const char *content_type, BIO *req) { - long req_len = 0; -#ifndef OPENSSL_NO_STDIO - FILE *fp = NULL; -#endif + long req_len; if (rctx == NULL || (req == NULL && content_type != NULL)) { ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER); @@ -301,47 +286,18 @@ static int set1_content(OSSL_HTTP_REQ_CTX *rctx, return 0; } - if (content_type == NULL) { - rctx->text = 1; /* assuming text by default, used just for tracing */ - } else { - if (OPENSSL_strncasecmp(content_type, "text/", 5) == 0) - rctx->text = 1; - if (BIO_printf(rctx->mem, "Content-Type: %s\r\n", content_type) <= 0) - return 0; - } - - /* - * BIO_CTRL_INFO yields the data length at least for memory BIOs, but for - * file-based BIOs it gives the current position, which is not what we need. - */ - if (BIO_method_type(req) == BIO_TYPE_FILE) { -#ifndef OPENSSL_NO_STDIO - if (BIO_get_fp(req, &fp) == 1 && fseek(fp, 0, SEEK_END) == 0) { - req_len = ftell(fp); - (void)fseek(fp, 0, SEEK_SET); - } else { - fp = NULL; - } -#endif - } else { - req_len = BIO_ctrl(req, BIO_CTRL_INFO, 0, NULL); - /* - * Streaming BIOs likely will not support querying the size at all, - * and we assume we got a correct value if req_len > 0. - */ - } - if (( -#ifndef OPENSSL_NO_STDIO - fp != NULL /* definitely correct req_len */ || -#endif - req_len > 0) - && BIO_printf(rctx->mem, "Content-Length: %ld\r\n", req_len) < 0) + if (content_type != NULL + && BIO_printf(rctx->mem, "Content-Type: %s\r\n", content_type) <= 0) return 0; - if (!BIO_up_ref(req)) - return 0; - rctx->req = req; - return 1; + /* streaming BIO may not support querying size */ + if (((req_len = BIO_ctrl(req, BIO_CTRL_INFO, 0, NULL)) <= 0 + || BIO_printf(rctx->mem, "Content-Length: %ld\r\n", req_len) > 0) + && BIO_up_ref(req)) { + rctx->req = req; + return 1; + } + return 0; } int OSSL_HTTP_REQ_CTX_set1_req(OSSL_HTTP_REQ_CTX *rctx, const char *content_type, @@ -357,16 +313,6 @@ int OSSL_HTTP_REQ_CTX_set1_req(OSSL_HTTP_REQ_CTX *rctx, const char *content_type return res; } -void OSSL_HTTP_REQ_CTX_set_max_response_hdr_lines(OSSL_HTTP_REQ_CTX *rctx, - size_t count) -{ - if (rctx == NULL) { - ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER); - return; - } - rctx->max_hdr_lines = count; -} - static int add1_headers(OSSL_HTTP_REQ_CTX *rctx, const STACK_OF(CONF_VALUE) *headers, const char *host) { @@ -432,10 +378,10 @@ static int parse_http_line1(char *line, int *found_keep_alive) int i, retcode, err; char *code, *reason, *end; - if (!CHECK_AND_SKIP_PREFIX(line, HTTP_PREFIX_VERSION)) + if (!HAS_PREFIX(line, HTTP_PREFIX_VERSION)) goto err; /* above HTTP 1.0, connection persistence is the default */ - *found_keep_alive = *line > '0'; + *found_keep_alive = line[strlen(HTTP_PREFIX_VERSION)] > '0'; /* Skip to first whitespace (past protocol info) */ for (code = line; *code != '\0' && !ossl_isspace(*code); code++) @@ -507,17 +453,13 @@ static int parse_http_line1(char *line, int *found_keep_alive) static int check_set_resp_len(OSSL_HTTP_REQ_CTX *rctx, size_t len) { - if (rctx->max_resp_len != 0 && len > rctx->max_resp_len) { + if (rctx->max_resp_len != 0 && len > rctx->max_resp_len) ERR_raise_data(ERR_LIB_HTTP, HTTP_R_MAX_RESP_LEN_EXCEEDED, "length=%zu, max=%zu", len, rctx->max_resp_len); - return 0; - } - if (rctx->resp_len != 0 && rctx->resp_len != len) { + if (rctx->resp_len != 0 && rctx->resp_len != len) ERR_raise_data(ERR_LIB_HTTP, HTTP_R_INCONSISTENT_CONTENT_LENGTH, "ASN.1 length=%zu, Content-Length=%zu", len, rctx->resp_len); - return 0; - } rctx->resp_len = len; return 1; } @@ -544,12 +486,10 @@ static int may_still_retry(time_t max_time, int *ptimeout) int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) { int i, found_expected_ct = 0, found_keep_alive = 0; - int got_text = 1; long n; size_t resp_len; const unsigned char *p; char *buf, *key, *value, *line_end = NULL; - size_t resp_hdr_lines = 0; if (rctx == NULL) { ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER); @@ -569,7 +509,7 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) } else { (void)ERR_set_mark(); n = BIO_gets(rctx->rbio, buf, rctx->buf_size); - if (n == -2) { /* some BIOs, such as SSL, do not support "gets" */ + if (n == -2) { /* unsupported method */ (void)ERR_pop_to_mark(); n = BIO_get_line(rctx->rbio, buf, rctx->buf_size); } else { @@ -597,35 +537,27 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) } rctx->state = OHS_WRITE_INIT; - /* fall through */ + /* fall thru */ case OHS_WRITE_INIT: rctx->len_to_send = BIO_get_mem_data(rctx->mem, &rctx->pos); - rctx->state = OHS_WRITE_HDR1; + rctx->state = OHS_WRITE_HDR; - /* fall through */ - case OHS_WRITE_HDR1: + /* fall thru */ case OHS_WRITE_HDR: /* Copy some chunk of data from rctx->mem to rctx->wbio */ case OHS_WRITE_REQ: /* Copy some chunk of data from rctx->req to rctx->wbio */ if (rctx->len_to_send > 0) { - size_t sz; - - if (!BIO_write_ex(rctx->wbio, rctx->pos, rctx->len_to_send, &sz)) { + i = BIO_write(rctx->wbio, rctx->pos, rctx->len_to_send); + if (i <= 0) { if (BIO_should_retry(rctx->wbio)) return -1; rctx->state = OHS_ERROR; return 0; } - if (OSSL_TRACE_ENABLED(HTTP) && rctx->state == OHS_WRITE_HDR1) - OSSL_TRACE(HTTP, "Sending request: [\n"); - OSSL_TRACE_STRING(HTTP, rctx->state != OHS_WRITE_REQ || rctx->text, - rctx->state != OHS_WRITE_REQ, rctx->pos, sz); - if (rctx->state == OHS_WRITE_HDR1) - rctx->state = OHS_WRITE_HDR; - rctx->pos += sz; - rctx->len_to_send -= sz; + rctx->pos += i; + rctx->len_to_send -= i; goto next_io; } if (rctx->state == OHS_WRITE_HDR) { @@ -635,7 +567,7 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) if (rctx->req != NULL && !BIO_eof(rctx->req)) { n = BIO_read(rctx->req, rctx->buf, rctx->buf_size); if (n <= 0) { - if (BIO_should_retry(rctx->req)) + if (BIO_should_retry(rctx->rbio)) return -1; ERR_raise(ERR_LIB_HTTP, HTTP_R_FAILED_READING_DATA); return 0; @@ -644,11 +576,9 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) rctx->len_to_send = n; goto next_io; } - if (OSSL_TRACE_ENABLED(HTTP)) - OSSL_TRACE(HTTP, "]\n"); rctx->state = OHS_FLUSH; - /* fall through */ + /* fall thru */ case OHS_FLUSH: i = BIO_flush(rctx->wbio); @@ -695,14 +625,6 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) return 0; } - resp_hdr_lines++; - if (rctx->max_hdr_lines != 0 && rctx->max_hdr_lines < resp_hdr_lines) { - ERR_raise(ERR_LIB_HTTP, HTTP_R_RESPONSE_TOO_MANY_HDRLINES); - OSSL_TRACE(HTTP, "Received too many headers\n"); - rctx->state = OHS_ERROR; - return 0; - } - /* Don't allow excessive lines */ if (n == rctx->buf_size) { ERR_raise(ERR_LIB_HTTP, HTTP_R_RESPONSE_LINE_TOO_LONG); @@ -710,13 +632,6 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) return 0; } - /* dump all response header lines */ - if (OSSL_TRACE_ENABLED(HTTP)) { - if (rctx->state == OHS_FIRSTLINE) - OSSL_TRACE(HTTP, "Received response header: [\n"); - OSSL_TRACE1(HTTP, "%s", buf); - } - /* First line */ if (rctx->state == OHS_FIRSTLINE) { switch (parse_http_line1(buf, &found_keep_alive)) { @@ -733,8 +648,8 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) /* redirection is not supported/recommended for POST */ /* fall through */ default: - rctx->state = OHS_HEADERS_ERROR; - goto next_line; /* continue parsing and reporting header */ + rctx->state = OHS_ERROR; + goto next_line; } } key = buf; @@ -755,27 +670,15 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) rctx->redirection_url = value; return 0; } - if (OPENSSL_strcasecmp(key, "Content-Type") == 0) { - got_text = OPENSSL_strncasecmp(value, "text/", 5) == 0; - if (rctx->state == OHS_HEADERS - && rctx->expected_ct != NULL) { - const char *semicolon; - - if (OPENSSL_strcasecmp(rctx->expected_ct, value) != 0 - /* ignore past ';' unless expected_ct contains ';' */ - && (strchr(rctx->expected_ct, ';') != NULL - || (semicolon = strchr(value, ';')) == NULL - || (size_t)(semicolon - value) != strlen(rctx->expected_ct) - || OPENSSL_strncasecmp(rctx->expected_ct, value, - semicolon - value) != 0)) { - ERR_raise_data(ERR_LIB_HTTP, - HTTP_R_UNEXPECTED_CONTENT_TYPE, - "expected=%s, actual=%s", - rctx->expected_ct, value); - return 0; - } - found_expected_ct = 1; + if (rctx->state == OHS_HEADERS && rctx->expected_ct != NULL + && OPENSSL_strcasecmp(key, "Content-Type") == 0) { + if (OPENSSL_strcasecmp(rctx->expected_ct, value) != 0) { + ERR_raise_data(ERR_LIB_HTTP, HTTP_R_UNEXPECTED_CONTENT_TYPE, + "expected=%s, actual=%s", + rctx->expected_ct, value); + return 0; } + found_expected_ct = 1; } /* https://tools.ietf.org/html/rfc7230#section-6.3 Persistence */ @@ -804,10 +707,6 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) } if (*p != '\0') /* not end of headers */ goto next_line; - if (OSSL_TRACE_ENABLED(HTTP)) - OSSL_TRACE(HTTP, "]\n"); - - resp_hdr_lines = 0; if (rctx->keep_alive != 0 /* do not let server initiate keep_alive */ && !found_keep_alive /* otherwise there is no change */) { @@ -819,22 +718,8 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx) rctx->keep_alive = 0; } - if (rctx->state == OHS_HEADERS_ERROR) { - if (OSSL_TRACE_ENABLED(HTTP)) { - int printed_final_nl = 0; - - OSSL_TRACE(HTTP, "Received error response body: [\n"); - while ((n = BIO_read(rctx->rbio, rctx->buf, rctx->buf_size)) > 0 - || (OSSL_sleep(100), BIO_should_retry(rctx->rbio))) { - OSSL_TRACE_STRING(HTTP, got_text, 1, rctx->buf, n); - if (n > 0) - printed_final_nl = rctx->buf[n - 1] == '\n'; - } - OSSL_TRACE1(HTTP, "%s]\n", printed_final_nl ? "" : "\n"); - (void)printed_final_nl; /* avoid warning unless enable-trace */ - } + if (rctx->state == OHS_ERROR) return 0; - } if (rctx->expected_ct != NULL && !found_expected_ct) { ERR_raise_data(ERR_LIB_HTTP, HTTP_R_MISSING_CONTENT_TYPE, @@ -1067,7 +952,7 @@ OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port, if (bio_update_fn != NULL) { BIO *orig_bio = cbio; - cbio = (*bio_update_fn)(cbio, arg, 1 /* connect */, use_ssl != 0); + cbio = (*bio_update_fn)(cbio, arg, 1 /* connect */, use_ssl); if (cbio == NULL) { if (bio == NULL) /* cbio was not provided by caller */ BIO_free_all(orig_bio); @@ -1204,12 +1089,13 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy, const char *expected_ct, int expect_asn1, size_t max_resp_len, int timeout) { - char *current_url; + char *current_url, *redirection_url = NULL; int n_redirs = 0; char *host; char *port; char *path; int use_ssl; + OSSL_HTTP_REQ_CTX *rctx; BIO *resp = NULL; time_t max_time = timeout > 0 ? time(NULL) + timeout : 0; @@ -1221,9 +1107,6 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy, return NULL; for (;;) { - OSSL_HTTP_REQ_CTX *rctx; - char *redirection_url; - if (!OSSL_HTTP_parse_url(current_url, &use_ssl, NULL /* user */, &host, &port, NULL /* port_num */, &path, NULL, NULL)) break; @@ -1232,19 +1115,16 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy, use_ssl, bio, rbio, bio_update_fn, arg, buf_size, timeout); new_rpath: - redirection_url = NULL; if (rctx != NULL) { if (!OSSL_HTTP_set1_request(rctx, path, headers, NULL /* content_type */, NULL /* req */, expected_ct, expect_asn1, max_resp_len, -1 /* use same max time (timeout) */, - 0 /* no keep_alive */)) { + 0 /* no keep_alive */)) OSSL_HTTP_REQ_CTX_free(rctx); - rctx = NULL; - } else { + else resp = OSSL_HTTP_exchange(rctx, &redirection_url); - } } OPENSSL_free(path); if (resp == NULL && redirection_url != NULL) { @@ -1255,14 +1135,6 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy, current_url = redirection_url; if (*redirection_url == '/') { /* redirection to same server */ path = OPENSSL_strdup(redirection_url); - if (path == NULL) { - OPENSSL_free(host); - OPENSSL_free(port); - (void)OSSL_HTTP_close(rctx, 1); - BIO_free(resp); - OPENSSL_free(current_url); - return NULL; - } goto new_rpath; } OPENSSL_free(host); @@ -1461,15 +1333,15 @@ int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port, continue; /* Check for HTTP/1.x */ - mbufp = mbuf; - if (!CHECK_AND_SKIP_PREFIX(mbufp, HTTP_PREFIX)) { + if (!HAS_PREFIX(mbuf, HTTP_PREFIX) != 0) { ERR_raise(ERR_LIB_HTTP, HTTP_R_HEADER_PARSE_ERROR); BIO_printf(bio_err, "%s: HTTP CONNECT failed, non-HTTP response\n", prog); /* Wrong protocol, not even HTTP, so stop reading headers */ goto end; } - if (!HAS_PREFIX(mbufp, HTTP_VERSION_PATT)) { + mbufp = mbuf + strlen(HTTP_PREFIX); + if (!HAS_PREFIX(mbufp, HTTP_VERSION_PATT) != 0) { ERR_raise(ERR_LIB_HTTP, HTTP_R_RECEIVED_WRONG_HTTP_VERSION); BIO_printf(bio_err, "%s: HTTP CONNECT failed, bad HTTP version %.*s\n", @@ -1480,8 +1352,6 @@ int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port, /* RFC 7231 4.3.6: any 2xx status code is valid */ if (!HAS_PREFIX(mbufp, " 2")) { - if (ossl_isspace(*mbufp)) - mbufp++; /* chop any trailing whitespace */ while (read_len > 0 && ossl_isspace(mbuf[read_len - 1])) read_len--; @@ -1500,7 +1370,7 @@ int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port, do { /* * This does not necessarily catch the case when the full - * HTTP response came in more than a single TCP message. + * HTTP response came in in more than a single TCP message. */ read_len = BIO_gets(fbio, mbuf, BUF_SIZE); } while (read_len > 2); diff --git a/openssl/src/crypto/http/http_err.c b/openssl/src/crypto/http/http_err.c index 22c2b40e6..332ad926d 100644 --- a/openssl/src/crypto/http/http_err.c +++ b/openssl/src/crypto/http/http_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,9 +12,7 @@ #include #include "crypto/httperr.h" -#ifndef OPENSSL_NO_HTTP - -# ifndef OPENSSL_NO_ERR +#ifndef OPENSSL_NO_ERR static const ERR_STRING_DATA HTTP_str_reasons[] = { {ERR_PACK(ERR_LIB_HTTP, 0, HTTP_R_ASN1_LEN_EXCEEDS_MAX_RESP_LEN), @@ -57,8 +55,6 @@ static const ERR_STRING_DATA HTTP_str_reasons[] = { "response line too long"}, {ERR_PACK(ERR_LIB_HTTP, 0, HTTP_R_RESPONSE_PARSE_ERROR), "response parse error"}, - {ERR_PACK(ERR_LIB_HTTP, 0, HTTP_R_RESPONSE_TOO_MANY_HDRLINES), - "response too many hdrlines"}, {ERR_PACK(ERR_LIB_HTTP, 0, HTTP_R_RETRY_TIMEOUT), "retry timeout"}, {ERR_PACK(ERR_LIB_HTTP, 0, HTTP_R_SERVER_CANCELED_CONNECTION), "server canceled connection"}, @@ -74,16 +70,13 @@ static const ERR_STRING_DATA HTTP_str_reasons[] = { {0, NULL} }; -# endif +#endif int ossl_err_load_HTTP_strings(void) { -# ifndef OPENSSL_NO_ERR +#ifndef OPENSSL_NO_ERR if (ERR_reason_error_string(HTTP_str_reasons[0].error) == NULL) ERR_load_strings_const(HTTP_str_reasons); -# endif +#endif return 1; } -#else -NON_EMPTY_TRANSLATION_UNIT -#endif diff --git a/openssl/src/crypto/http/http_lib.c b/openssl/src/crypto/http/http_lib.c index cd0e25c85..bd9c096b9 100644 --- a/openssl/src/crypto/http/http_lib.c +++ b/openssl/src/crypto/http/http_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,13 +22,6 @@ static void init_pstring(char **pstr) } } -static void init_pint(int *pint) -{ - if (pint != NULL) { - *pint = 0; - } -} - static int copy_substring(char **dest, const char *start, const char *end) { return dest == NULL @@ -61,7 +54,6 @@ int OSSL_parse_url(const char *url, char **pscheme, char **puser, char **phost, init_pstring(puser); init_pstring(phost); init_pstring(pport); - init_pint(pport_num); init_pstring(ppath); init_pstring(pfrag); init_pstring(pquery); @@ -91,9 +83,9 @@ int OSSL_parse_url(const char *url, char **pscheme, char **puser, char **phost, else host = p; - /* parse hostname/address as far as needed here */ + /* parse host name/address as far as needed here */ if (host[0] == '[') { - /* IPv6 literal, which may include ':' */ + /* ipv6 literal, which may include ':' */ host_end = strchr(host + 1, ']'); if (host_end == NULL) goto parse_err; @@ -118,7 +110,7 @@ int OSSL_parse_url(const char *url, char **pscheme, char **puser, char **phost, port = ++p; /* remaining port spec handling is also done for the default values */ /* make sure a decimal port number is given */ - if (sscanf(port, "%u", &portnum) <= 0 || portnum > 65535) { + if (!sscanf(port, "%u", &portnum) || portnum > 65535) { ERR_raise_data(ERR_LIB_HTTP, HTTP_R_INVALID_PORT_NUMBER, "%s", port); goto err; } @@ -261,9 +253,9 @@ static int use_proxy(const char *no_proxy, const char *server) * compatible with other HTTP client implementations like wget, curl and git */ if (no_proxy == NULL) - no_proxy = ossl_safe_getenv("no_proxy"); + no_proxy = getenv("no_proxy"); if (no_proxy == NULL) - no_proxy = ossl_safe_getenv(OPENSSL_NO_PROXY); + no_proxy = getenv(OPENSSL_NO_PROXY); if (no_proxy != NULL) found = strstr(no_proxy, server); @@ -283,9 +275,10 @@ const char *OSSL_HTTP_adapt_proxy(const char *proxy, const char *no_proxy, * compatible with other HTTP client implementations like wget, curl and git */ if (proxy == NULL) - proxy = ossl_safe_getenv(use_ssl ? "https_proxy" : "http_proxy"); + proxy = getenv(use_ssl ? "https_proxy" : "http_proxy"); if (proxy == NULL) - proxy = ossl_safe_getenv(use_ssl ? OPENSSL_HTTP_PROXY : OPENSSL_HTTPS_PROXY); + proxy = getenv(use_ssl ? OPENSSL_HTTP_PROXY : + OPENSSL_HTTPS_PROXY); if (proxy == NULL || *proxy == '\0' || !use_proxy(no_proxy, server)) return NULL; diff --git a/openssl/src/crypto/idea/i_cbc.c b/openssl/src/crypto/idea/i_cbc.c deleted file mode 100644 index e9608e82f..000000000 --- a/openssl/src/crypto/idea/i_cbc.c +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * IDEA low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include -#include "idea_local.h" - -void IDEA_cbc_encrypt(const unsigned char *in, unsigned char *out, - long length, IDEA_KEY_SCHEDULE *ks, unsigned char *iv, - int encrypt) -{ - register unsigned long tin0, tin1; - register unsigned long tout0, tout1, xor0, xor1; - register long l = length; - unsigned long tin[2]; - - if (encrypt) { - n2l(iv, tout0); - n2l(iv, tout1); - iv -= 8; - for (l -= 8; l >= 0; l -= 8) { - n2l(in, tin0); - n2l(in, tin1); - tin0 ^= tout0; - tin1 ^= tout1; - tin[0] = tin0; - tin[1] = tin1; - IDEA_encrypt(tin, ks); - tout0 = tin[0]; - l2n(tout0, out); - tout1 = tin[1]; - l2n(tout1, out); - } - if (l != -8) { - n2ln(in, tin0, tin1, l + 8); - tin0 ^= tout0; - tin1 ^= tout1; - tin[0] = tin0; - tin[1] = tin1; - IDEA_encrypt(tin, ks); - tout0 = tin[0]; - l2n(tout0, out); - tout1 = tin[1]; - l2n(tout1, out); - } - l2n(tout0, iv); - l2n(tout1, iv); - } else { - n2l(iv, xor0); - n2l(iv, xor1); - iv -= 8; - for (l -= 8; l >= 0; l -= 8) { - n2l(in, tin0); - tin[0] = tin0; - n2l(in, tin1); - tin[1] = tin1; - IDEA_encrypt(tin, ks); - tout0 = tin[0] ^ xor0; - tout1 = tin[1] ^ xor1; - l2n(tout0, out); - l2n(tout1, out); - xor0 = tin0; - xor1 = tin1; - } - if (l != -8) { - n2l(in, tin0); - tin[0] = tin0; - n2l(in, tin1); - tin[1] = tin1; - IDEA_encrypt(tin, ks); - tout0 = tin[0] ^ xor0; - tout1 = tin[1] ^ xor1; - l2nn(tout0, tout1, out, l + 8); - xor0 = tin0; - xor1 = tin1; - } - l2n(xor0, iv); - l2n(xor1, iv); - } - tin0 = tin1 = tout0 = tout1 = xor0 = xor1 = 0; - tin[0] = tin[1] = 0; -} - -void IDEA_encrypt(unsigned long *d, IDEA_KEY_SCHEDULE *key) -{ - register IDEA_INT *p; - register unsigned long x1, x2, x3, x4, t0, t1, ul; - - x2 = d[0]; - x1 = (x2 >> 16); - x4 = d[1]; - x3 = (x4 >> 16); - - p = &(key->data[0][0]); - - E_IDEA(0); - E_IDEA(1); - E_IDEA(2); - E_IDEA(3); - E_IDEA(4); - E_IDEA(5); - E_IDEA(6); - E_IDEA(7); - - x1 &= 0xffff; - idea_mul(x1, x1, *p, ul); - p++; - - t0 = x3 + *(p++); - t1 = x2 + *(p++); - - x4 &= 0xffff; - idea_mul(x4, x4, *p, ul); - - d[0] = (t0 & 0xffff) | ((x1 & 0xffff) << 16); - d[1] = (x4 & 0xffff) | ((t1 & 0xffff) << 16); -} diff --git a/openssl/src/crypto/idea/i_cfb64.c b/openssl/src/crypto/idea/i_cfb64.c deleted file mode 100644 index afea89edf..000000000 --- a/openssl/src/crypto/idea/i_cfb64.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * IDEA low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include -#include "idea_local.h" - -/* - * The input and output encrypted as though 64bit cfb mode is being used. - * The extra state information to record how much of the 64bit block we have - * used is contained in *num; - */ - -void IDEA_cfb64_encrypt(const unsigned char *in, unsigned char *out, - long length, IDEA_KEY_SCHEDULE *schedule, - unsigned char *ivec, int *num, int encrypt) -{ - register unsigned long v0, v1, t; - register int n = *num; - register long l = length; - unsigned long ti[2]; - unsigned char *iv, c, cc; - - if (n < 0) { - *num = -1; - return; - } - - iv = (unsigned char *)ivec; - if (encrypt) { - while (l--) { - if (n == 0) { - n2l(iv, v0); - ti[0] = v0; - n2l(iv, v1); - ti[1] = v1; - IDEA_encrypt((unsigned long *)ti, schedule); - iv = (unsigned char *)ivec; - t = ti[0]; - l2n(t, iv); - t = ti[1]; - l2n(t, iv); - iv = (unsigned char *)ivec; - } - c = *(in++) ^ iv[n]; - *(out++) = c; - iv[n] = c; - n = (n + 1) & 0x07; - } - } else { - while (l--) { - if (n == 0) { - n2l(iv, v0); - ti[0] = v0; - n2l(iv, v1); - ti[1] = v1; - IDEA_encrypt((unsigned long *)ti, schedule); - iv = (unsigned char *)ivec; - t = ti[0]; - l2n(t, iv); - t = ti[1]; - l2n(t, iv); - iv = (unsigned char *)ivec; - } - cc = *(in++); - c = iv[n]; - iv[n] = cc; - *(out++) = c ^ cc; - n = (n + 1) & 0x07; - } - } - v0 = v1 = ti[0] = ti[1] = t = c = cc = 0; - *num = n; -} diff --git a/openssl/src/crypto/idea/i_ecb.c b/openssl/src/crypto/idea/i_ecb.c deleted file mode 100644 index 6304e6cac..000000000 --- a/openssl/src/crypto/idea/i_ecb.c +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * IDEA low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include -#include "idea_local.h" -#include - -const char *IDEA_options(void) -{ - return "idea(int)"; -} - -void IDEA_ecb_encrypt(const unsigned char *in, unsigned char *out, - IDEA_KEY_SCHEDULE *ks) -{ - unsigned long l0, l1, d[2]; - - n2l(in, l0); - d[0] = l0; - n2l(in, l1); - d[1] = l1; - IDEA_encrypt(d, ks); - l0 = d[0]; - l2n(l0, out); - l1 = d[1]; - l2n(l1, out); - l0 = l1 = d[0] = d[1] = 0; -} diff --git a/openssl/src/crypto/idea/i_ofb64.c b/openssl/src/crypto/idea/i_ofb64.c deleted file mode 100644 index a6a149764..000000000 --- a/openssl/src/crypto/idea/i_ofb64.c +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * IDEA low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include -#include "idea_local.h" - -/* - * The input and output encrypted as though 64bit ofb mode is being used. - * The extra state information to record how much of the 64bit block we have - * used is contained in *num; - */ -void IDEA_ofb64_encrypt(const unsigned char *in, unsigned char *out, - long length, IDEA_KEY_SCHEDULE *schedule, - unsigned char *ivec, int *num) -{ - register unsigned long v0, v1, t; - register int n = *num; - register long l = length; - unsigned char d[8]; - register char *dp; - unsigned long ti[2]; - unsigned char *iv; - int save = 0; - - if (n < 0) { - *num = -1; - return; - } - - iv = (unsigned char *)ivec; - n2l(iv, v0); - n2l(iv, v1); - ti[0] = v0; - ti[1] = v1; - dp = (char *)d; - l2n(v0, dp); - l2n(v1, dp); - while (l--) { - if (n == 0) { - IDEA_encrypt((unsigned long *)ti, schedule); - dp = (char *)d; - t = ti[0]; - l2n(t, dp); - t = ti[1]; - l2n(t, dp); - save++; - } - *(out++) = *(in++) ^ d[n]; - n = (n + 1) & 0x07; - } - if (save) { - v0 = ti[0]; - v1 = ti[1]; - iv = (unsigned char *)ivec; - l2n(v0, iv); - l2n(v1, iv); - } - t = v0 = v1 = ti[0] = ti[1] = 0; - *num = n; -} diff --git a/openssl/src/crypto/idea/i_skey.c b/openssl/src/crypto/idea/i_skey.c deleted file mode 100644 index 7564bce5b..000000000 --- a/openssl/src/crypto/idea/i_skey.c +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * IDEA low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include -#include "idea_local.h" - -static IDEA_INT inverse(unsigned int xin); -void IDEA_set_encrypt_key(const unsigned char *key, IDEA_KEY_SCHEDULE *ks) -{ - int i; - register IDEA_INT *kt, *kf, r0, r1, r2; - - kt = &(ks->data[0][0]); - n2s(key, kt[0]); - n2s(key, kt[1]); - n2s(key, kt[2]); - n2s(key, kt[3]); - n2s(key, kt[4]); - n2s(key, kt[5]); - n2s(key, kt[6]); - n2s(key, kt[7]); - - kf = kt; - kt += 8; - for (i = 0; i < 6; i++) { - r2 = kf[1]; - r1 = kf[2]; - *(kt++) = ((r2 << 9) | (r1 >> 7)) & 0xffff; - r0 = kf[3]; - *(kt++) = ((r1 << 9) | (r0 >> 7)) & 0xffff; - r1 = kf[4]; - *(kt++) = ((r0 << 9) | (r1 >> 7)) & 0xffff; - r0 = kf[5]; - *(kt++) = ((r1 << 9) | (r0 >> 7)) & 0xffff; - r1 = kf[6]; - *(kt++) = ((r0 << 9) | (r1 >> 7)) & 0xffff; - r0 = kf[7]; - *(kt++) = ((r1 << 9) | (r0 >> 7)) & 0xffff; - r1 = kf[0]; - if (i >= 5) - break; - *(kt++) = ((r0 << 9) | (r1 >> 7)) & 0xffff; - *(kt++) = ((r1 << 9) | (r2 >> 7)) & 0xffff; - kf += 8; - } -} - -void IDEA_set_decrypt_key(IDEA_KEY_SCHEDULE *ek, IDEA_KEY_SCHEDULE *dk) -{ - int r; - register IDEA_INT *fp, *tp, t; - - tp = &(dk->data[0][0]); - fp = &(ek->data[8][0]); - for (r = 0; r < 9; r++) { - *(tp++) = inverse(fp[0]); - *(tp++) = ((int)(0x10000L - fp[2]) & 0xffff); - *(tp++) = ((int)(0x10000L - fp[1]) & 0xffff); - *(tp++) = inverse(fp[3]); - if (r == 8) - break; - fp -= 6; - *(tp++) = fp[4]; - *(tp++) = fp[5]; - } - - tp = &(dk->data[0][0]); - t = tp[1]; - tp[1] = tp[2]; - tp[2] = t; - - t = tp[49]; - tp[49] = tp[50]; - tp[50] = t; -} - -/* taken directly from the 'paper' I'll have a look at it later */ -static IDEA_INT inverse(unsigned int xin) -{ - long n1, n2, q, r, b1, b2, t; - - if (xin == 0) - b2 = 0; - else { - n1 = 0x10001; - n2 = xin; - b2 = 1; - b1 = 0; - - do { - r = (n1 % n2); - q = (n1 - r) / n2; - if (r == 0) { - if (b2 < 0) - b2 = 0x10001 + b2; - } else { - n1 = n2; - n2 = r; - t = b2; - b2 = b1 - q * b2; - b1 = t; - } - } while (r != 0); - } - return (IDEA_INT)b2; -} diff --git a/openssl/src/crypto/idea/idea_local.h b/openssl/src/crypto/idea/idea_local.h deleted file mode 100644 index e6ada85d8..000000000 --- a/openssl/src/crypto/idea/idea_local.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#define idea_mul(r,a,b,ul) \ -ul=(unsigned long)a*b; \ -if (ul != 0) \ - { \ - r=(ul&0xffff)-(ul>>16); \ - r-=((r)>>16); \ - } \ -else \ - r=(-(int)a-b+1); /* assuming a or b is 0 and in range */ - -/* NOTE - c is not incremented as per n2l */ -#define n2ln(c,l1,l2,n) { \ - c+=n; \ - l1=l2=0; \ - switch (n) { \ - case 8: l2 =((unsigned long)(*(--(c)))) ; \ - /* fall through */ \ - case 7: l2|=((unsigned long)(*(--(c))))<< 8; \ - /* fall through */ \ - case 6: l2|=((unsigned long)(*(--(c))))<<16; \ - /* fall through */ \ - case 5: l2|=((unsigned long)(*(--(c))))<<24; \ - /* fall through */ \ - case 4: l1 =((unsigned long)(*(--(c)))) ; \ - /* fall through */ \ - case 3: l1|=((unsigned long)(*(--(c))))<< 8; \ - /* fall through */ \ - case 2: l1|=((unsigned long)(*(--(c))))<<16; \ - /* fall through */ \ - case 1: l1|=((unsigned long)(*(--(c))))<<24; \ - } \ - } - -/* NOTE - c is not incremented as per l2n */ -#define l2nn(l1,l2,c,n) { \ - c+=n; \ - switch (n) { \ - case 8: *(--(c))=(unsigned char)(((l2) )&0xff); \ - /* fall through */ \ - case 7: *(--(c))=(unsigned char)(((l2)>> 8)&0xff); \ - /* fall through */ \ - case 6: *(--(c))=(unsigned char)(((l2)>>16)&0xff); \ - /* fall through */ \ - case 5: *(--(c))=(unsigned char)(((l2)>>24)&0xff); \ - /* fall through */ \ - case 4: *(--(c))=(unsigned char)(((l1) )&0xff); \ - /* fall through */ \ - case 3: *(--(c))=(unsigned char)(((l1)>> 8)&0xff); \ - /* fall through */ \ - case 2: *(--(c))=(unsigned char)(((l1)>>16)&0xff); \ - /* fall through */ \ - case 1: *(--(c))=(unsigned char)(((l1)>>24)&0xff); \ - } \ - } - -#undef n2l -#define n2l(c,l) (l =((unsigned long)(*((c)++)))<<24L, \ - l|=((unsigned long)(*((c)++)))<<16L, \ - l|=((unsigned long)(*((c)++)))<< 8L, \ - l|=((unsigned long)(*((c)++)))) - -#undef l2n -#define l2n(l,c) (*((c)++)=(unsigned char)(((l)>>24L)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16L)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8L)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff)) - -#undef s2n -#define s2n(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8L)&0xff)) - -#undef n2s -#define n2s(c,l) (l =((IDEA_INT)(*((c)++)))<< 8L, \ - l|=((IDEA_INT)(*((c)++))) ) - - -#define E_IDEA(num) \ - x1&=0xffff; \ - idea_mul(x1,x1,*p,ul); p++; \ - x2+= *(p++); \ - x3+= *(p++); \ - x4&=0xffff; \ - idea_mul(x4,x4,*p,ul); p++; \ - t0=(x1^x3)&0xffff; \ - idea_mul(t0,t0,*p,ul); p++; \ - t1=(t0+(x2^x4))&0xffff; \ - idea_mul(t1,t1,*p,ul); p++; \ - t0+=t1; \ - x1^=t1; \ - x4^=t0; \ - ul=x2^t0; /* do the swap to x3 */ \ - x2=x3^t1; \ - x3=ul; diff --git a/openssl/src/crypto/info.c b/openssl/src/crypto/info.c index 9ef9ee470..5c6b4983f 100644 --- a/openssl/src/crypto/info.c +++ b/openssl/src/crypto/info.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -45,10 +45,10 @@ DEFINE_RUN_ONCE_STATIC(init_info_strings) BIO_snprintf(ossl_cpu_info_str, sizeof(ossl_cpu_info_str), CPUINFO_PREFIX "OPENSSL_ia32cap=0x%llx:0x%llx", - (unsigned long long)OPENSSL_ia32cap_P[0] | - (unsigned long long)OPENSSL_ia32cap_P[1] << 32, - (unsigned long long)OPENSSL_ia32cap_P[2] | - (unsigned long long)OPENSSL_ia32cap_P[3] << 32); + (long long)OPENSSL_ia32cap_P[0] | + (long long)OPENSSL_ia32cap_P[1] << 32, + (long long)OPENSSL_ia32cap_P[2] | + (long long)OPENSSL_ia32cap_P[3] << 32); if ((env = getenv("OPENSSL_ia32cap")) != NULL) BIO_snprintf(ossl_cpu_info_str + strlen(ossl_cpu_info_str), sizeof(ossl_cpu_info_str) - strlen(ossl_cpu_info_str), @@ -131,15 +131,11 @@ DEFINE_RUN_ONCE_STATIC(init_info_strings) #ifdef OPENSSL_RAND_SEED_NONE add_seeds_string("none"); #endif -#ifdef OPENSSL_RAND_SEED_RDTSC - add_seeds_string("rdtsc"); +#ifdef OPENSSL_RAND_SEED_RTDSC + add_seeds_string("stdsc"); #endif #ifdef OPENSSL_RAND_SEED_RDCPU -# ifdef __aarch64__ - add_seeds_string("rndr ( rndrrs rndr )"); -# else add_seeds_string("rdrand ( rdseed rdrand )"); -# endif #endif #ifdef OPENSSL_RAND_SEED_LIBRANDOM add_seeds_string("C-library-random"); @@ -182,8 +178,6 @@ const char *OPENSSL_info(int t) case OPENSSL_INFO_DIR_FILENAME_SEPARATOR: #if defined(_WIN32) return "\\"; -#elif defined(__VMS) - return ""; #else /* Assume POSIX */ return "/"; #endif diff --git a/openssl/src/crypto/init.c b/openssl/src/crypto/init.c index 07bcf8395..f4df1d983 100644 --- a/openssl/src/crypto/init.c +++ b/openssl/src/crypto/init.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -44,9 +44,6 @@ struct ossl_init_stop_st { }; static OPENSSL_INIT_STOP *stop_handlers = NULL; -/* Guards access to the optsdone variable on platforms without atomics */ -static CRYPTO_RWLOCK *optsdone_lock = NULL; -/* Guards simultaneous INIT_LOAD_CONFIG calls with non-NULL settings */ static CRYPTO_RWLOCK *init_lock = NULL; static CRYPTO_THREAD_LOCAL in_init_config_local; @@ -61,10 +58,8 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_base) ossl_malloc_setup_failures(); #endif - if ((optsdone_lock = CRYPTO_THREAD_lock_new()) == NULL - || (init_lock = CRYPTO_THREAD_lock_new()) == NULL) + if ((init_lock = CRYPTO_THREAD_lock_new()) == NULL) goto err; - OPENSSL_cpuid_setup(); if (!ossl_init_thread()) @@ -78,8 +73,6 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_base) err: OSSL_TRACE(INIT, "ossl_init_base failed!\n"); - CRYPTO_THREAD_lock_free(optsdone_lock); - optsdone_lock = NULL; CRYPTO_THREAD_lock_free(init_lock); init_lock = NULL; @@ -97,19 +90,17 @@ static int win32atexit(void) DEFINE_RUN_ONCE_STATIC(ossl_init_register_atexit) { -#ifndef OPENSSL_NO_ATEXIT -# ifdef OPENSSL_INIT_DEBUG +#ifdef OPENSSL_INIT_DEBUG fprintf(stderr, "OPENSSL_INIT: ossl_init_register_atexit()\n"); -# endif -# ifndef OPENSSL_SYS_UEFI -# if defined(_WIN32) && !defined(__BORLANDC__) +#endif +#ifndef OPENSSL_SYS_UEFI +# if defined(_WIN32) && !defined(__BORLANDC__) /* We use _onexit() in preference because it gets called on DLL unload */ if (_onexit(win32atexit) == NULL) return 0; -# else +# else if (atexit(OPENSSL_cleanup) != 0) return 0; -# endif # endif #endif @@ -179,7 +170,7 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_load_crypto_nodelete) } static CRYPTO_ONCE load_crypto_strings = CRYPTO_ONCE_STATIC_INIT; - +static int load_crypto_strings_inited = 0; DEFINE_RUN_ONCE_STATIC(ossl_init_load_crypto_strings) { int ret = 1; @@ -188,15 +179,9 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_load_crypto_strings) * pulling in all the error strings during static linking */ #if !defined(OPENSSL_NO_ERR) && !defined(OPENSSL_NO_AUTOERRINIT) - void *err; - - if (!err_shelve_state(&err)) - return 0; - OSSL_TRACE(INIT, "ossl_err_load_crypto_strings()\n"); ret = ossl_err_load_crypto_strings(); - - err_unshelve_state(err); + load_crypto_strings_inited = 1; #endif return ret; } @@ -383,8 +368,6 @@ void OPENSSL_cleanup(void) } stop_handlers = NULL; - CRYPTO_THREAD_lock_free(optsdone_lock); - optsdone_lock = NULL; CRYPTO_THREAD_lock_free(init_lock); init_lock = NULL; @@ -398,10 +381,6 @@ void OPENSSL_cleanup(void) #ifndef OPENSSL_NO_COMP OSSL_TRACE(INIT, "OPENSSL_cleanup: ossl_comp_zlib_cleanup()\n"); ossl_comp_zlib_cleanup(); - OSSL_TRACE(INIT, "OPENSSL_cleanup: ossl_comp_brotli_cleanup()\n"); - ossl_comp_brotli_cleanup(); - OSSL_TRACE(INIT, "OPENSSL_cleanup: ossl_comp_zstd_cleanup()\n"); - ossl_comp_zstd_cleanup(); #endif if (async_inited) { @@ -409,6 +388,11 @@ void OPENSSL_cleanup(void) async_deinit(); } + if (load_crypto_strings_inited) { + OSSL_TRACE(INIT, "OPENSSL_cleanup: err_free_strings_int()\n"); + err_free_strings_int(); + } + /* * Note that cleanup order is important: * - ossl_rand_cleanup_int could call an ENGINE's RAND cleanup function so @@ -477,7 +461,7 @@ int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings) uint64_t tmp; int aloaddone = 0; - /* Applications depend on 0 being returned when cleanup was already done */ + /* Applications depend on 0 being returned when cleanup was already done */ if (stopped) { if (!(opts & OPENSSL_INIT_BASE_ONLY)) ERR_raise(ERR_LIB_CRYPTO, ERR_R_INIT_FAIL); @@ -487,7 +471,7 @@ int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings) /* * We ignore failures from this function. It is probably because we are * on a platform that doesn't support lockless atomic loads (we may not - * have created optsdone_lock yet so we can't use it). This is just an + * have created init_lock yet so we can't use it). This is just an * optimisation to skip the full checks in this function if we don't need * to, so we carry on regardless in the event of failure. * @@ -524,12 +508,12 @@ int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings) return 1; /* - * optsdone_lock should definitely be set up now, so we can now repeat the + * init_lock should definitely be set up now, so we can now repeat the * same check from above but be sure that it will work even on platforms * without lockless CRYPTO_atomic_load */ if (!aloaddone) { - if (!CRYPTO_atomic_load(&optsdone, &tmp, optsdone_lock)) + if (!CRYPTO_atomic_load(&optsdone, &tmp, init_lock)) return 0; if ((tmp & opts) == opts) return 1; @@ -659,7 +643,7 @@ int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings) } #endif - if (!CRYPTO_atomic_or(&optsdone, opts, &tmp, optsdone_lock)) + if (!CRYPTO_atomic_or(&optsdone, opts, &tmp, init_lock)) return 0; return 1; @@ -672,26 +656,28 @@ int OPENSSL_atexit(void (*handler)(void)) #if !defined(OPENSSL_USE_NODELETE)\ && !defined(OPENSSL_NO_PINSHARED) { -# if defined(DSO_WIN32) && !defined(_WIN32_WCE) - HMODULE handle = NULL; - BOOL ret; union { void *sym; void (*func)(void); } handlersym; handlersym.func = handler; - - /* - * We don't use the DSO route for WIN32 because there is a better - * way - */ - ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS - | GET_MODULE_HANDLE_EX_FLAG_PIN, - handlersym.sym, &handle); - - if (!ret) - return 0; +# if defined(DSO_WIN32) && !defined(_WIN32_WCE) + { + HMODULE handle = NULL; + BOOL ret; + + /* + * We don't use the DSO route for WIN32 because there is a better + * way + */ + ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS + | GET_MODULE_HANDLE_EX_FLAG_PIN, + handlersym.sym, &handle); + + if (!ret) + return 0; + } # elif !defined(DSO_NONE) /* * Deliberately leak a reference to the handler. This will force the @@ -699,28 +685,26 @@ int OPENSSL_atexit(void (*handler)(void)) * atexit handler. If -znodelete has been used then this is * unnecessary. */ - DSO *dso = NULL; - union { - void *sym; - void (*func)(void); - } handlersym; - - handlersym.func = handler; - - ERR_set_mark(); - dso = DSO_dsobyaddr(handlersym.sym, DSO_FLAG_NO_UNLOAD_ON_FREE); - /* See same code above in ossl_init_base() for an explanation. */ - OSSL_TRACE1(INIT, - "atexit: obtained DSO reference? %s\n", - (dso == NULL ? "No!" : "Yes.")); - DSO_free(dso); - ERR_pop_to_mark(); + { + DSO *dso = NULL; + + ERR_set_mark(); + dso = DSO_dsobyaddr(handlersym.sym, DSO_FLAG_NO_UNLOAD_ON_FREE); + /* See same code above in ossl_init_base() for an explanation. */ + OSSL_TRACE1(INIT, + "atexit: obtained DSO reference? %s\n", + (dso == NULL ? "No!" : "Yes.")); + DSO_free(dso); + ERR_pop_to_mark(); + } # endif } #endif - if ((newhand = OPENSSL_malloc(sizeof(*newhand))) == NULL) + if ((newhand = OPENSSL_malloc(sizeof(*newhand))) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } newhand->handler = handler; newhand->next = stop_handlers; diff --git a/openssl/src/crypto/initthread.c b/openssl/src/crypto/initthread.c index e4d830d7f..1bdaeda9f 100644 --- a/openssl/src/crypto/initthread.c +++ b/openssl/src/crypto/initthread.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,6 @@ #include "crypto/cryptlib.h" #include "prov/providercommon.h" #include "internal/thread_once.h" -#include "crypto/context.h" #ifdef FIPS_MODULE #include "prov/provider_ctx.h" @@ -27,7 +26,7 @@ * * The FIPS provider tells libcrypto about which threads it is interested in * by calling "c_thread_start" which is a function pointer created during - * provider initialisation (i.e. OSSL_provider_init). + * provider initialisation (i.e. OSSL_init_provider). */ extern OSSL_FUNC_core_thread_start_fn *c_thread_start; #endif @@ -249,16 +248,7 @@ void ossl_ctx_thread_stop(OSSL_LIB_CTX *ctx) #else -static void ossl_arg_thread_stop(void *arg); - -/* Register the current thread so that we are informed if it gets stopped */ -int ossl_thread_register_fips(OSSL_LIB_CTX *libctx) -{ - return c_thread_start(FIPS_get_core_handle(libctx), ossl_arg_thread_stop, - libctx); -} - -void *ossl_thread_event_ctx_new(OSSL_LIB_CTX *libctx) +static void *thread_event_ossl_ctx_new(OSSL_LIB_CTX *libctx) { THREAD_EVENT_HANDLER **hands = NULL; CRYPTO_THREAD_LOCAL *tlocal = OPENSSL_zalloc(sizeof(*tlocal)); @@ -266,7 +256,7 @@ void *ossl_thread_event_ctx_new(OSSL_LIB_CTX *libctx) if (tlocal == NULL) return NULL; - if (!CRYPTO_THREAD_init_local(tlocal, NULL)) { + if (!CRYPTO_THREAD_init_local(tlocal, NULL)) { goto err; } @@ -277,16 +267,6 @@ void *ossl_thread_event_ctx_new(OSSL_LIB_CTX *libctx) if (!CRYPTO_THREAD_set_local(tlocal, hands)) goto err; - /* - * We should ideally call ossl_thread_register_fips() here. This function - * is called during the startup of the FIPS provider and we need to ensure - * that the main thread is registered to receive thread callbacks in order - * to free |hands| that we allocated above. However we are too early in - * the FIPS provider initialisation that FIPS_get_core_handle() doesn't work - * yet. So we defer this to the main provider OSSL_provider_init_int() - * function. - */ - return tlocal; err: OPENSSL_free(hands); @@ -294,11 +274,17 @@ void *ossl_thread_event_ctx_new(OSSL_LIB_CTX *libctx) return NULL; } -void ossl_thread_event_ctx_free(void *tlocal) +static void thread_event_ossl_ctx_free(void *tlocal) { OPENSSL_free(tlocal); } +static const OSSL_LIB_CTX_METHOD thread_event_ossl_ctx_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + thread_event_ossl_ctx_new, + thread_event_ossl_ctx_free, +}; + static void ossl_arg_thread_stop(void *arg) { ossl_ctx_thread_stop((OSSL_LIB_CTX *)arg); @@ -308,7 +294,8 @@ void ossl_ctx_thread_stop(OSSL_LIB_CTX *ctx) { THREAD_EVENT_HANDLER **hands; CRYPTO_THREAD_LOCAL *local - = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_THREAD_EVENT_HANDLER_INDEX); + = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_THREAD_EVENT_HANDLER_INDEX, + &thread_event_ossl_ctx_method); if (local == NULL) return; @@ -376,7 +363,8 @@ int ossl_init_thread_start(const void *index, void *arg, * OSSL_LIB_CTX gets informed about thread stop events individually. */ CRYPTO_THREAD_LOCAL *local - = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_THREAD_EVENT_HANDLER_INDEX); + = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_THREAD_EVENT_HANDLER_INDEX, + &thread_event_ossl_ctx_method); #else /* * Outside of FIPS mode the list of THREAD_EVENT_HANDLERs is unique per @@ -398,7 +386,8 @@ int ossl_init_thread_start(const void *index, void *arg, * libcrypto to tell us about later thread stop events. c_thread_start * is a callback to libcrypto defined in fipsprov.c */ - if (!ossl_thread_register_fips(ctx)) + if (!c_thread_start(FIPS_get_core_handle(ctx), ossl_arg_thread_stop, + ctx)) return 0; } #endif diff --git a/openssl/src/crypto/lhash/lh_stats.c b/openssl/src/crypto/lhash/lh_stats.c index ea0a3252a..ba4d4ea89 100644 --- a/openssl/src/crypto/lhash/lh_stats.c +++ b/openssl/src/crypto/lhash/lh_stats.c @@ -7,8 +7,6 @@ * https://www.openssl.org/source/license.html */ -#define OPENSSL_SUPPRESS_DEPRECATED - #include #include #include @@ -23,7 +21,6 @@ #include "lhash_local.h" # ifndef OPENSSL_NO_STDIO -# ifndef OPENSSL_NO_DEPRECATED_3_1 void OPENSSL_LH_stats(const OPENSSL_LHASH *lh, FILE *fp) { BIO *bp; @@ -59,15 +56,9 @@ void OPENSSL_LH_node_usage_stats(const OPENSSL_LHASH *lh, FILE *fp) OPENSSL_LH_node_usage_stats_bio(lh, bp); BIO_free(bp); } -# endif + # endif -# ifndef OPENSSL_NO_DEPRECATED_3_1 -/* - * These functions are implemented as separate static functions as they are - * called from the stdio functions above and calling deprecated functions will - * generate a warning. - */ void OPENSSL_LH_stats_bio(const OPENSSL_LHASH *lh, BIO *out) { BIO_printf(out, "num_items = %lu\n", lh->num_items); @@ -124,4 +115,3 @@ void OPENSSL_LH_node_usage_stats_bio(const OPENSSL_LHASH *lh, BIO *out) (int)((total % lh->num_nodes) * 100 / lh->num_nodes), (int)(total / n_used), (int)((total % n_used) * 100 / n_used)); } -# endif diff --git a/openssl/src/crypto/lhash/lhash.c b/openssl/src/crypto/lhash/lhash.c index e0234ccbf..1cd988f01 100644 --- a/openssl/src/crypto/lhash/lhash.c +++ b/openssl/src/crypto/lhash/lhash.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -44,28 +44,18 @@ static int expand(OPENSSL_LHASH *lh); static void contract(OPENSSL_LHASH *lh); static OPENSSL_LH_NODE **getrn(OPENSSL_LHASH *lh, const void *data, unsigned long *rhash); -OPENSSL_LHASH *OPENSSL_LH_set_thunks(OPENSSL_LHASH *lh, - OPENSSL_LH_HASHFUNCTHUNK hw, - OPENSSL_LH_COMPFUNCTHUNK cw, - OPENSSL_LH_DOALL_FUNC_THUNK daw, - OPENSSL_LH_DOALL_FUNCARG_THUNK daaw) -{ - - if (lh == NULL) - return NULL; - lh->compw = cw; - lh->hashw = hw; - lh->daw = daw; - lh->daaw = daaw; - return lh; -} - OPENSSL_LHASH *OPENSSL_LH_new(OPENSSL_LH_HASHFUNC h, OPENSSL_LH_COMPFUNC c) { OPENSSL_LHASH *ret; - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) { + /* + * Do not set the error code, because the ERR code uses LHASH + * and we want to avoid possible endless error loop. + * ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); + */ return NULL; + } if ((ret->b = OPENSSL_zalloc(sizeof(*ret->b) * MIN_NODES)) == NULL) goto err; ret->comp = ((c == NULL) ? (OPENSSL_LH_COMPFUNC)strcmp : c); @@ -184,11 +174,8 @@ void *OPENSSL_LH_retrieve(OPENSSL_LHASH *lh, const void *data) } static void doall_util_fn(OPENSSL_LHASH *lh, int use_arg, - OPENSSL_LH_DOALL_FUNC_THUNK wfunc, OPENSSL_LH_DOALL_FUNC func, - OPENSSL_LH_DOALL_FUNCARG func_arg, - OPENSSL_LH_DOALL_FUNCARG_THUNK wfunc_arg, - void *arg) + OPENSSL_LH_DOALL_FUNCARG func_arg, void *arg) { int i; OPENSSL_LH_NODE *a, *n; @@ -205,9 +192,9 @@ static void doall_util_fn(OPENSSL_LHASH *lh, int use_arg, while (a != NULL) { n = a->next; if (use_arg) - wfunc_arg(a->data, arg, func_arg); + func_arg(a->data, arg); else - wfunc(a->data, func); + func(a->data); a = n; } } @@ -215,29 +202,12 @@ static void doall_util_fn(OPENSSL_LHASH *lh, int use_arg, void OPENSSL_LH_doall(OPENSSL_LHASH *lh, OPENSSL_LH_DOALL_FUNC func) { - if (lh == NULL) - return; - - doall_util_fn(lh, 0, lh->daw, func, (OPENSSL_LH_DOALL_FUNCARG)NULL, - (OPENSSL_LH_DOALL_FUNCARG_THUNK)NULL, NULL); -} - -void OPENSSL_LH_doall_arg(OPENSSL_LHASH *lh, - OPENSSL_LH_DOALL_FUNCARG func, void *arg) -{ - if (lh == NULL) - return; - - doall_util_fn(lh, 1, (OPENSSL_LH_DOALL_FUNC_THUNK)NULL, - (OPENSSL_LH_DOALL_FUNC)NULL, func, lh->daaw, arg); + doall_util_fn(lh, 0, func, (OPENSSL_LH_DOALL_FUNCARG)0, NULL); } -void OPENSSL_LH_doall_arg_thunk(OPENSSL_LHASH *lh, - OPENSSL_LH_DOALL_FUNCARG_THUNK daaw, - OPENSSL_LH_DOALL_FUNCARG fn, void *arg) +void OPENSSL_LH_doall_arg(OPENSSL_LHASH *lh, OPENSSL_LH_DOALL_FUNCARG func, void *arg) { - doall_util_fn(lh, 1, (OPENSSL_LH_DOALL_FUNC_THUNK)NULL, - (OPENSSL_LH_DOALL_FUNC)NULL, fn, daaw, arg); + doall_util_fn(lh, 1, (OPENSSL_LH_DOALL_FUNC)0, func, arg); } static int expand(OPENSSL_LHASH *lh) @@ -294,14 +264,14 @@ static void contract(OPENSSL_LHASH *lh) n = OPENSSL_realloc(lh->b, (unsigned int)(sizeof(OPENSSL_LH_NODE *) * lh->pmax)); if (n == NULL) { - /* fputs("realloc error in lhash", stderr); */ + /* fputs("realloc error in lhash",stderr); */ lh->error++; - } else { - lh->b = n; + return; } lh->num_alloc_nodes /= 2; lh->pmax /= 2; lh->p = lh->pmax - 1; + lh->b = n; } else lh->p--; @@ -322,32 +292,24 @@ static OPENSSL_LH_NODE **getrn(OPENSSL_LHASH *lh, { OPENSSL_LH_NODE **ret, *n1; unsigned long hash, nn; + OPENSSL_LH_COMPFUNC cf; - if (lh->hashw != NULL) - hash = lh->hashw(data, lh->hash); - else - hash = lh->hash(data); - + hash = (*(lh->hash)) (data); *rhash = hash; nn = hash % lh->pmax; if (nn < lh->p) nn = hash % lh->num_alloc_nodes; + cf = lh->comp; ret = &(lh->b[(int)nn]); for (n1 = *ret; n1 != NULL; n1 = n1->next) { if (n1->hash != hash) { ret = &(n1->next); continue; } - - if (lh->compw != NULL) { - if (lh->compw(n1->data, data, lh->comp) == 0) - break; - } else { - if (lh->comp(n1->data, data) == 0) - break; - } + if (cf(n1->data, data) == 0) + break; ret = &(n1->next); } return ret; @@ -382,37 +344,18 @@ unsigned long OPENSSL_LH_strhash(const char *c) return (ret >> 16) ^ ret; } -/* - * Case insensitive string hashing. - * - * The lower/upper case bit is masked out (forcing all letters to be capitals). - * The major side effect on non-alpha characters is mapping the symbols and - * digits into the control character range (which should be harmless). - * The duplication (with respect to the hash value) of printable characters - * are that '`', '{', '|', '}' and '~' map to '@', '[', '\', ']' and '^' - * respectively (which seems tolerable). - * - * For EBCDIC, the alpha mapping is to lower case, most symbols go to control - * characters. The only duplication is '0' mapping to '^', which is better - * than for ASCII. - */ unsigned long ossl_lh_strcasehash(const char *c) { unsigned long ret = 0; long n; unsigned long v; int r; -#if defined(CHARSET_EBCDIC) && !defined(CHARSET_EBCDIC_TEST) - const long int case_adjust = ~0x40; -#else - const long int case_adjust = ~0x20; -#endif if (c == NULL || *c == '\0') return ret; for (n = 0x100; *c != '\0'; n += 0x100) { - v = n | (case_adjust & *c); + v = n | ossl_tolower(*c); r = (int)((v >> 2) ^ v) & 0x0f; /* cast to uint64_t to avoid 32 bit shift of 32 bit value */ ret = (ret << r) | (unsigned long)((uint64_t)ret >> (32 - r)); diff --git a/openssl/src/crypto/lhash/lhash_local.h b/openssl/src/crypto/lhash/lhash_local.h index 8da63b9ad..088ac94d2 100644 --- a/openssl/src/crypto/lhash/lhash_local.h +++ b/openssl/src/crypto/lhash/lhash_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,10 +20,6 @@ struct lhash_st { OPENSSL_LH_NODE **b; OPENSSL_LH_COMPFUNC comp; OPENSSL_LH_HASHFUNC hash; - OPENSSL_LH_HASHFUNCTHUNK hashw; - OPENSSL_LH_COMPFUNCTHUNK compw; - OPENSSL_LH_DOALL_FUNC_THUNK daw; - OPENSSL_LH_DOALL_FUNCARG_THUNK daaw; unsigned int num_nodes; unsigned int num_alloc_nodes; unsigned int p; diff --git a/openssl/src/crypto/local.h b/openssl/src/crypto/local.h new file mode 100644 index 000000000..731fdfa17 --- /dev/null +++ b/openssl/src/crypto/local.h @@ -0,0 +1,30 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +void _armv7_neon_probe(void); +void _armv8_aes_probe(void); +void _armv8_sha1_probe(void); +void _armv8_sha256_probe(void); +void _armv8_pmull_probe(void); +# ifdef __aarch64__ +void _armv8_sm3_probe(void); +void _armv8_sm4_probe(void); +void _armv8_sha512_probe(void); +unsigned int _armv8_cpuid_probe(void); +# endif +uint32_t _armv7_tick(void); + +uint32_t OPENSSL_rdtsc(void); + +IA32CAP OPENSSL_ia32_cpuid(unsigned int *); +extern char ossl_cpu_info_str[]; diff --git a/openssl/src/crypto/loongarch_arch.h b/openssl/src/crypto/loongarch_arch.h deleted file mode 100644 index c7fd42df1..000000000 --- a/openssl/src/crypto/loongarch_arch.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#ifndef OSSL_CRYPTO_LOONGARCH_ARCH_H -# define OSSL_CRYPTO_LOONGARCH_ARCH_H - -# ifndef __ASSEMBLER__ -extern unsigned int OPENSSL_loongarch_hwcap_P; -# endif - -# define LOONGARCH_HWCAP_LSX (1 << 4) -# define LOONGARCH_HWCAP_LASX (1 << 5) - -#endif diff --git a/openssl/src/crypto/loongarchcap.c b/openssl/src/crypto/loongarchcap.c deleted file mode 100644 index 2123fd9c0..000000000 --- a/openssl/src/crypto/loongarchcap.c +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#include -#include "loongarch_arch.h" - -unsigned int OPENSSL_loongarch_hwcap_P = 0; - -void OPENSSL_cpuid_setup(void) -{ - OPENSSL_loongarch_hwcap_P = getauxval(AT_HWCAP); -} diff --git a/openssl/src/crypto/md2/md2_dgst.c b/openssl/src/crypto/md2/md2_dgst.c deleted file mode 100644 index 0b0033543..000000000 --- a/openssl/src/crypto/md2/md2_dgst.c +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD2 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include -#include -#include - -/* - * Implemented from RFC1319 The MD2 Message-Digest Algorithm - */ - -#define UCHAR unsigned char - -static void md2_block(MD2_CTX *c, const unsigned char *d); -/* - * The magic S table - I have converted it to hex since it is basically just - * a random byte string. - */ -static const MD2_INT S[256] = { - 0x29, 0x2E, 0x43, 0xC9, 0xA2, 0xD8, 0x7C, 0x01, - 0x3D, 0x36, 0x54, 0xA1, 0xEC, 0xF0, 0x06, 0x13, - 0x62, 0xA7, 0x05, 0xF3, 0xC0, 0xC7, 0x73, 0x8C, - 0x98, 0x93, 0x2B, 0xD9, 0xBC, 0x4C, 0x82, 0xCA, - 0x1E, 0x9B, 0x57, 0x3C, 0xFD, 0xD4, 0xE0, 0x16, - 0x67, 0x42, 0x6F, 0x18, 0x8A, 0x17, 0xE5, 0x12, - 0xBE, 0x4E, 0xC4, 0xD6, 0xDA, 0x9E, 0xDE, 0x49, - 0xA0, 0xFB, 0xF5, 0x8E, 0xBB, 0x2F, 0xEE, 0x7A, - 0xA9, 0x68, 0x79, 0x91, 0x15, 0xB2, 0x07, 0x3F, - 0x94, 0xC2, 0x10, 0x89, 0x0B, 0x22, 0x5F, 0x21, - 0x80, 0x7F, 0x5D, 0x9A, 0x5A, 0x90, 0x32, 0x27, - 0x35, 0x3E, 0xCC, 0xE7, 0xBF, 0xF7, 0x97, 0x03, - 0xFF, 0x19, 0x30, 0xB3, 0x48, 0xA5, 0xB5, 0xD1, - 0xD7, 0x5E, 0x92, 0x2A, 0xAC, 0x56, 0xAA, 0xC6, - 0x4F, 0xB8, 0x38, 0xD2, 0x96, 0xA4, 0x7D, 0xB6, - 0x76, 0xFC, 0x6B, 0xE2, 0x9C, 0x74, 0x04, 0xF1, - 0x45, 0x9D, 0x70, 0x59, 0x64, 0x71, 0x87, 0x20, - 0x86, 0x5B, 0xCF, 0x65, 0xE6, 0x2D, 0xA8, 0x02, - 0x1B, 0x60, 0x25, 0xAD, 0xAE, 0xB0, 0xB9, 0xF6, - 0x1C, 0x46, 0x61, 0x69, 0x34, 0x40, 0x7E, 0x0F, - 0x55, 0x47, 0xA3, 0x23, 0xDD, 0x51, 0xAF, 0x3A, - 0xC3, 0x5C, 0xF9, 0xCE, 0xBA, 0xC5, 0xEA, 0x26, - 0x2C, 0x53, 0x0D, 0x6E, 0x85, 0x28, 0x84, 0x09, - 0xD3, 0xDF, 0xCD, 0xF4, 0x41, 0x81, 0x4D, 0x52, - 0x6A, 0xDC, 0x37, 0xC8, 0x6C, 0xC1, 0xAB, 0xFA, - 0x24, 0xE1, 0x7B, 0x08, 0x0C, 0xBD, 0xB1, 0x4A, - 0x78, 0x88, 0x95, 0x8B, 0xE3, 0x63, 0xE8, 0x6D, - 0xE9, 0xCB, 0xD5, 0xFE, 0x3B, 0x00, 0x1D, 0x39, - 0xF2, 0xEF, 0xB7, 0x0E, 0x66, 0x58, 0xD0, 0xE4, - 0xA6, 0x77, 0x72, 0xF8, 0xEB, 0x75, 0x4B, 0x0A, - 0x31, 0x44, 0x50, 0xB4, 0x8F, 0xED, 0x1F, 0x1A, - 0xDB, 0x99, 0x8D, 0x33, 0x9F, 0x11, 0x83, 0x14, -}; - -const char *MD2_options(void) -{ - if (sizeof(MD2_INT) == 1) - return "md2(char)"; - else - return "md2(int)"; -} - -int MD2_Init(MD2_CTX *c) -{ - c->num = 0; - memset(c->state, 0, sizeof(c->state)); - memset(c->cksm, 0, sizeof(c->cksm)); - memset(c->data, 0, sizeof(c->data)); - return 1; -} - -int MD2_Update(MD2_CTX *c, const unsigned char *data, size_t len) -{ - register UCHAR *p; - - if (len == 0) - return 1; - - p = c->data; - if (c->num != 0) { - if ((c->num + len) >= MD2_BLOCK) { - memcpy(&(p[c->num]), data, MD2_BLOCK - c->num); - md2_block(c, c->data); - data += (MD2_BLOCK - c->num); - len -= (MD2_BLOCK - c->num); - c->num = 0; - /* drop through and do the rest */ - } else { - memcpy(&(p[c->num]), data, len); - /* data+=len; */ - c->num += (int)len; - return 1; - } - } - /* - * we now can process the input data in blocks of MD2_BLOCK chars and - * save the leftovers to c->data. - */ - while (len >= MD2_BLOCK) { - md2_block(c, data); - data += MD2_BLOCK; - len -= MD2_BLOCK; - } - memcpy(p, data, len); - c->num = (int)len; - return 1; -} - -static void md2_block(MD2_CTX *c, const unsigned char *d) -{ - register MD2_INT t, *sp1, *sp2; - register int i, j; - MD2_INT state[48]; - - sp1 = c->state; - sp2 = c->cksm; - j = sp2[MD2_BLOCK - 1]; - for (i = 0; i < 16; i++) { - state[i] = sp1[i]; - state[i + 16] = t = d[i]; - state[i + 32] = (t ^ sp1[i]); - j = sp2[i] ^= S[t ^ j]; - } - t = 0; - for (i = 0; i < 18; i++) { - for (j = 0; j < 48; j += 8) { - t = state[j + 0] ^= S[t]; - t = state[j + 1] ^= S[t]; - t = state[j + 2] ^= S[t]; - t = state[j + 3] ^= S[t]; - t = state[j + 4] ^= S[t]; - t = state[j + 5] ^= S[t]; - t = state[j + 6] ^= S[t]; - t = state[j + 7] ^= S[t]; - } - t = (t + i) & 0xff; - } - memcpy(sp1, state, 16 * sizeof(MD2_INT)); - OPENSSL_cleanse(state, 48 * sizeof(MD2_INT)); -} - -int MD2_Final(unsigned char *md, MD2_CTX *c) -{ - int i, v; - register UCHAR *cp; - register MD2_INT *p1, *p2; - - cp = c->data; - p1 = c->state; - p2 = c->cksm; - v = MD2_BLOCK - c->num; - for (i = c->num; i < MD2_BLOCK; i++) - cp[i] = (UCHAR) v; - - md2_block(c, cp); - - for (i = 0; i < MD2_BLOCK; i++) - cp[i] = (UCHAR) p2[i]; - md2_block(c, cp); - - for (i = 0; i < 16; i++) - md[i] = (UCHAR) (p1[i] & 0xff); - OPENSSL_cleanse(c, sizeof(*c)); - return 1; -} diff --git a/openssl/src/crypto/md2/md2_one.c b/openssl/src/crypto/md2/md2_one.c deleted file mode 100644 index f9be69273..000000000 --- a/openssl/src/crypto/md2/md2_one.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD2 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "internal/cryptlib.h" -#include - -/* - * This is a separate file so that #defines in cryptlib.h can map my MD - * functions to different names - */ - -unsigned char *MD2(const unsigned char *d, size_t n, unsigned char *md) -{ - MD2_CTX c; - static unsigned char m[MD2_DIGEST_LENGTH]; - - if (md == NULL) - md = m; - if (!MD2_Init(&c)) - return NULL; -#ifndef CHARSET_EBCDIC - MD2_Update(&c, d, n); -#else - { - char temp[1024]; - unsigned long chunk; - - while (n > 0) { - chunk = (n > sizeof(temp)) ? sizeof(temp) : n; - ebcdic2ascii(temp, d, chunk); - MD2_Update(&c, temp, chunk); - n -= chunk; - d += chunk; - } - } -#endif - MD2_Final(md, &c); - OPENSSL_cleanse(&c, sizeof(c)); /* Security consideration */ - return md; -} diff --git a/openssl/src/crypto/md4/md4_dgst.c b/openssl/src/crypto/md4/md4_dgst.c deleted file mode 100644 index aefe6a3a3..000000000 --- a/openssl/src/crypto/md4/md4_dgst.c +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD4 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include "md4_local.h" - -/* - * Implemented from RFC1186 The MD4 Message-Digest Algorithm - */ - -#define INIT_DATA_A (unsigned long)0x67452301L -#define INIT_DATA_B (unsigned long)0xefcdab89L -#define INIT_DATA_C (unsigned long)0x98badcfeL -#define INIT_DATA_D (unsigned long)0x10325476L - -int MD4_Init(MD4_CTX *c) -{ - memset(c, 0, sizeof(*c)); - c->A = INIT_DATA_A; - c->B = INIT_DATA_B; - c->C = INIT_DATA_C; - c->D = INIT_DATA_D; - return 1; -} - -#ifndef md4_block_data_order -# ifdef X -# undef X -# endif -void md4_block_data_order(MD4_CTX *c, const void *data_, size_t num) -{ - const unsigned char *data = data_; - register unsigned MD32_REG_T A, B, C, D, l; -# ifndef MD32_XARRAY - /* See comment in crypto/sha/sha_local.h for details. */ - unsigned MD32_REG_T XX0, XX1, XX2, XX3, XX4, XX5, XX6, XX7, - XX8, XX9, XX10, XX11, XX12, XX13, XX14, XX15; -# define X(i) XX##i -# else - MD4_LONG XX[MD4_LBLOCK]; -# define X(i) XX[i] -# endif - - A = c->A; - B = c->B; - C = c->C; - D = c->D; - - for (; num--;) { - (void)HOST_c2l(data, l); - X(0) = l; - (void)HOST_c2l(data, l); - X(1) = l; - /* Round 0 */ - R0(A, B, C, D, X(0), 3, 0); - (void)HOST_c2l(data, l); - X(2) = l; - R0(D, A, B, C, X(1), 7, 0); - (void)HOST_c2l(data, l); - X(3) = l; - R0(C, D, A, B, X(2), 11, 0); - (void)HOST_c2l(data, l); - X(4) = l; - R0(B, C, D, A, X(3), 19, 0); - (void)HOST_c2l(data, l); - X(5) = l; - R0(A, B, C, D, X(4), 3, 0); - (void)HOST_c2l(data, l); - X(6) = l; - R0(D, A, B, C, X(5), 7, 0); - (void)HOST_c2l(data, l); - X(7) = l; - R0(C, D, A, B, X(6), 11, 0); - (void)HOST_c2l(data, l); - X(8) = l; - R0(B, C, D, A, X(7), 19, 0); - (void)HOST_c2l(data, l); - X(9) = l; - R0(A, B, C, D, X(8), 3, 0); - (void)HOST_c2l(data, l); - X(10) = l; - R0(D, A, B, C, X(9), 7, 0); - (void)HOST_c2l(data, l); - X(11) = l; - R0(C, D, A, B, X(10), 11, 0); - (void)HOST_c2l(data, l); - X(12) = l; - R0(B, C, D, A, X(11), 19, 0); - (void)HOST_c2l(data, l); - X(13) = l; - R0(A, B, C, D, X(12), 3, 0); - (void)HOST_c2l(data, l); - X(14) = l; - R0(D, A, B, C, X(13), 7, 0); - (void)HOST_c2l(data, l); - X(15) = l; - R0(C, D, A, B, X(14), 11, 0); - R0(B, C, D, A, X(15), 19, 0); - /* Round 1 */ - R1(A, B, C, D, X(0), 3, 0x5A827999L); - R1(D, A, B, C, X(4), 5, 0x5A827999L); - R1(C, D, A, B, X(8), 9, 0x5A827999L); - R1(B, C, D, A, X(12), 13, 0x5A827999L); - R1(A, B, C, D, X(1), 3, 0x5A827999L); - R1(D, A, B, C, X(5), 5, 0x5A827999L); - R1(C, D, A, B, X(9), 9, 0x5A827999L); - R1(B, C, D, A, X(13), 13, 0x5A827999L); - R1(A, B, C, D, X(2), 3, 0x5A827999L); - R1(D, A, B, C, X(6), 5, 0x5A827999L); - R1(C, D, A, B, X(10), 9, 0x5A827999L); - R1(B, C, D, A, X(14), 13, 0x5A827999L); - R1(A, B, C, D, X(3), 3, 0x5A827999L); - R1(D, A, B, C, X(7), 5, 0x5A827999L); - R1(C, D, A, B, X(11), 9, 0x5A827999L); - R1(B, C, D, A, X(15), 13, 0x5A827999L); - /* Round 2 */ - R2(A, B, C, D, X(0), 3, 0x6ED9EBA1L); - R2(D, A, B, C, X(8), 9, 0x6ED9EBA1L); - R2(C, D, A, B, X(4), 11, 0x6ED9EBA1L); - R2(B, C, D, A, X(12), 15, 0x6ED9EBA1L); - R2(A, B, C, D, X(2), 3, 0x6ED9EBA1L); - R2(D, A, B, C, X(10), 9, 0x6ED9EBA1L); - R2(C, D, A, B, X(6), 11, 0x6ED9EBA1L); - R2(B, C, D, A, X(14), 15, 0x6ED9EBA1L); - R2(A, B, C, D, X(1), 3, 0x6ED9EBA1L); - R2(D, A, B, C, X(9), 9, 0x6ED9EBA1L); - R2(C, D, A, B, X(5), 11, 0x6ED9EBA1L); - R2(B, C, D, A, X(13), 15, 0x6ED9EBA1L); - R2(A, B, C, D, X(3), 3, 0x6ED9EBA1L); - R2(D, A, B, C, X(11), 9, 0x6ED9EBA1L); - R2(C, D, A, B, X(7), 11, 0x6ED9EBA1L); - R2(B, C, D, A, X(15), 15, 0x6ED9EBA1L); - - A = c->A += A; - B = c->B += B; - C = c->C += C; - D = c->D += D; - } -} -#endif diff --git a/openssl/src/crypto/md4/md4_local.h b/openssl/src/crypto/md4/md4_local.h deleted file mode 100644 index 66aa5e403..000000000 --- a/openssl/src/crypto/md4/md4_local.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include - -void md4_block_data_order(MD4_CTX *c, const void *p, size_t num); - -#define DATA_ORDER_IS_LITTLE_ENDIAN - -#define HASH_LONG MD4_LONG -#define HASH_CTX MD4_CTX -#define HASH_CBLOCK MD4_CBLOCK -#define HASH_UPDATE MD4_Update -#define HASH_TRANSFORM MD4_Transform -#define HASH_FINAL MD4_Final -#define HASH_MAKE_STRING(c,s) do { \ - unsigned long ll; \ - ll=(c)->A; (void)HOST_l2c(ll,(s)); \ - ll=(c)->B; (void)HOST_l2c(ll,(s)); \ - ll=(c)->C; (void)HOST_l2c(ll,(s)); \ - ll=(c)->D; (void)HOST_l2c(ll,(s)); \ - } while (0) -#define HASH_BLOCK_DATA_ORDER md4_block_data_order - -#include "crypto/md32_common.h" - -/*- -#define F(x,y,z) (((x) & (y)) | ((~(x)) & (z))) -#define G(x,y,z) (((x) & (y)) | ((x) & ((z))) | ((y) & ((z)))) -*/ - -/* - * As pointed out by Wei Dai, the above can be simplified to the code - * below. Wei attributes these optimizations to Peter Gutmann's SHS code, - * and he attributes it to Rich Schroeppel. - */ -#define F(b,c,d) ((((c) ^ (d)) & (b)) ^ (d)) -#define G(b,c,d) (((b) & (c)) | ((b) & (d)) | ((c) & (d))) -#define H(b,c,d) ((b) ^ (c) ^ (d)) - -#define R0(a,b,c,d,k,s,t) { \ - a+=((k)+(t)+F((b),(c),(d))); \ - a=ROTATE(a,s); }; - -#define R1(a,b,c,d,k,s,t) { \ - a+=((k)+(t)+G((b),(c),(d))); \ - a=ROTATE(a,s); }; - -#define R2(a,b,c,d,k,s,t) { \ - a+=((k)+(t)+H((b),(c),(d))); \ - a=ROTATE(a,s); }; diff --git a/openssl/src/crypto/md4/md4_one.c b/openssl/src/crypto/md4/md4_one.c deleted file mode 100644 index 36fa88dd4..000000000 --- a/openssl/src/crypto/md4/md4_one.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD4 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include - -#ifdef CHARSET_EBCDIC -# include -#endif - -unsigned char *MD4(const unsigned char *d, size_t n, unsigned char *md) -{ - MD4_CTX c; - static unsigned char m[MD4_DIGEST_LENGTH]; - - if (md == NULL) - md = m; - if (!MD4_Init(&c)) - return NULL; -#ifndef CHARSET_EBCDIC - MD4_Update(&c, d, n); -#else - { - char temp[1024]; - unsigned long chunk; - - while (n > 0) { - chunk = (n > sizeof(temp)) ? sizeof(temp) : n; - ebcdic2ascii(temp, d, chunk); - MD4_Update(&c, temp, chunk); - n -= chunk; - d += chunk; - } - } -#endif - MD4_Final(md, &c); - OPENSSL_cleanse(&c, sizeof(c)); /* security consideration */ - return md; -} diff --git a/openssl/src/crypto/md5/gen/darwin_arm64/md5-aarch64.S b/openssl/src/crypto/md5/gen/darwin_arm64/md5-aarch64.S deleted file mode 100644 index efbee7419..000000000 --- a/openssl/src/crypto/md5/gen/darwin_arm64/md5-aarch64.S +++ /dev/null @@ -1,677 +0,0 @@ -#include "arm_arch.h" - -.text -.globl _ossl_md5_block_asm_data_order - -_ossl_md5_block_asm_data_order: - AARCH64_VALID_CALL_TARGET - // Save all callee-saved registers - stp x19,x20,[sp,#-80]! - stp x21,x22,[sp,#16] - stp x23,x24,[sp,#32] - stp x25,x26,[sp,#48] - stp x27,x28,[sp,#64] - - ldp w10, w11, [x0, #0] // Load MD5 state->A and state->B - ldp w12, w13, [x0, #8] // Load MD5 state->C and state->D -.align 5 -ossl_md5_blocks_loop: - eor x17, x12, x13 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - and x16, x17, x11 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - ldp w15, w20, [x1] // Load 2 words of input data0 M[0],M[1] - ldp w3, w21, [x1, #8] // Load 2 words of input data0 M[2],M[3] -#ifdef __AARCH64EB__ - rev w15, w15 - rev w20, w20 - rev w3, w3 - rev w21, w21 -#endif - eor x14, x16, x13 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x9, #0xa478 // Load lower half of constant 0xd76aa478 - movk x9, #0xd76a, lsl #16 // Load upper half of constant 0xd76aa478 - add w8, w10, w15 // Add dest value - add w7, w8, w9 // Add constant 0xd76aa478 - add w6, w7, w14 // Add aux function result - ror w6, w6, #25 // Rotate left s=7 bits - eor x5, x11, x12 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w4, w11, w6 // Add X parameter round 1 A=FF(A, B, C, D, 0xd76aa478, s=7, M[0]) - and x8, x5, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x17, x8, x12 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x16, #0xb756 // Load lower half of constant 0xe8c7b756 - movk x16, #0xe8c7, lsl #16 // Load upper half of constant 0xe8c7b756 - add w9, w13, w20 // Add dest value - add w7, w9, w16 // Add constant 0xe8c7b756 - add w14, w7, w17 // Add aux function result - ror w14, w14, #20 // Rotate left s=12 bits - eor x6, x4, x11 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w5, w4, w14 // Add X parameter round 1 D=FF(D, A, B, C, 0xe8c7b756, s=12, M[1]) - and x8, x6, x5 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x9, x8, x11 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x16, #0x70db // Load lower half of constant 0x242070db - movk x16, #0x2420, lsl #16 // Load upper half of constant 0x242070db - add w7, w12, w3 // Add dest value - add w17, w7, w16 // Add constant 0x242070db - add w14, w17, w9 // Add aux function result - ror w14, w14, #15 // Rotate left s=17 bits - eor x6, x5, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w8, w5, w14 // Add X parameter round 1 C=FF(C, D, A, B, 0x242070db, s=17, M[2]) - and x7, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x16, x7, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x9, #0xceee // Load lower half of constant 0xc1bdceee - movk x9, #0xc1bd, lsl #16 // Load upper half of constant 0xc1bdceee - add w14, w11, w21 // Add dest value - add w6, w14, w9 // Add constant 0xc1bdceee - add w7, w6, w16 // Add aux function result - ror w7, w7, #10 // Rotate left s=22 bits - eor x17, x8, x5 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w9, w8, w7 // Add X parameter round 1 B=FF(B, C, D, A, 0xc1bdceee, s=22, M[3]) - ldp w14, w22, [x1, #16] // Load 2 words of input data0 M[4],M[5] - ldp w7, w23, [x1, #24] // Load 2 words of input data0 M[6],M[7] -#ifdef __AARCH64EB__ - rev w14, w14 - rev w22, w22 - rev w7, w7 - rev w23, w23 -#endif - and x16, x17, x9 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x16, x5 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x16, #0xfaf // Load lower half of constant 0xf57c0faf - movk x16, #0xf57c, lsl #16 // Load upper half of constant 0xf57c0faf - add w17, w4, w14 // Add dest value - add w16, w17, w16 // Add constant 0xf57c0faf - add w4, w16, w6 // Add aux function result - ror w4, w4, #25 // Rotate left s=7 bits - eor x16, x9, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w17, w9, w4 // Add X parameter round 1 A=FF(A, B, C, D, 0xf57c0faf, s=7, M[4]) - and x16, x16, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x16, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x4, #0xc62a // Load lower half of constant 0x4787c62a - movk x4, #0x4787, lsl #16 // Load upper half of constant 0x4787c62a - add w16, w5, w22 // Add dest value - add w16, w16, w4 // Add constant 0x4787c62a - add w5, w16, w6 // Add aux function result - ror w5, w5, #20 // Rotate left s=12 bits - eor x4, x17, x9 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w19, w17, w5 // Add X parameter round 1 D=FF(D, A, B, C, 0x4787c62a, s=12, M[5]) - and x6, x4, x19 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x5, x6, x9 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x4, #0x4613 // Load lower half of constant 0xa8304613 - movk x4, #0xa830, lsl #16 // Load upper half of constant 0xa8304613 - add w6, w8, w7 // Add dest value - add w8, w6, w4 // Add constant 0xa8304613 - add w4, w8, w5 // Add aux function result - ror w4, w4, #15 // Rotate left s=17 bits - eor x6, x19, x17 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w8, w19, w4 // Add X parameter round 1 C=FF(C, D, A, B, 0xa8304613, s=17, M[6]) - and x5, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x4, x5, x17 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x6, #0x9501 // Load lower half of constant 0xfd469501 - movk x6, #0xfd46, lsl #16 // Load upper half of constant 0xfd469501 - add w9, w9, w23 // Add dest value - add w5, w9, w6 // Add constant 0xfd469501 - add w9, w5, w4 // Add aux function result - ror w9, w9, #10 // Rotate left s=22 bits - eor x6, x8, x19 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w4, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0xfd469501, s=22, M[7]) - ldp w5, w24, [x1, #32] // Load 2 words of input data0 M[8],M[9] - ldp w16, w25, [x1, #40] // Load 2 words of input data0 M[10],M[11] -#ifdef __AARCH64EB__ - rev w5, w5 - rev w24, w24 - rev w16, w16 - rev w25, w25 -#endif - and x9, x6, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x9, x19 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x9, #0x98d8 // Load lower half of constant 0x698098d8 - movk x9, #0x6980, lsl #16 // Load upper half of constant 0x698098d8 - add w17, w17, w5 // Add dest value - add w9, w17, w9 // Add constant 0x698098d8 - add w17, w9, w6 // Add aux function result - ror w17, w17, #25 // Rotate left s=7 bits - eor x9, x4, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w6, w4, w17 // Add X parameter round 1 A=FF(A, B, C, D, 0x698098d8, s=7, M[8]) - and x17, x9, x6 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x9, x17, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x17, #0xf7af // Load lower half of constant 0x8b44f7af - movk x17, #0x8b44, lsl #16 // Load upper half of constant 0x8b44f7af - add w19, w19, w24 // Add dest value - add w17, w19, w17 // Add constant 0x8b44f7af - add w19, w17, w9 // Add aux function result - ror w19, w19, #20 // Rotate left s=12 bits - eor x9, x6, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w17, w6, w19 // Add X parameter round 1 D=FF(D, A, B, C, 0x8b44f7af, s=12, M[9]) - and x9, x9, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x9, x9, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x11, #0x5bb1 // Load lower half of constant 0xffff5bb1 - movk x11, #0xffff, lsl #16 // Load upper half of constant 0xffff5bb1 - add w8, w8, w16 // Add dest value - add w8, w8, w11 // Add constant 0xffff5bb1 - add w8, w8, w9 // Add aux function result - ror w8, w8, #15 // Rotate left s=17 bits - eor x9, x17, x6 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w8, w17, w8 // Add X parameter round 1 C=FF(C, D, A, B, 0xffff5bb1, s=17, M[10]) - and x9, x9, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x9, x9, x6 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x11, #0xd7be // Load lower half of constant 0x895cd7be - movk x11, #0x895c, lsl #16 // Load upper half of constant 0x895cd7be - add w4, w4, w25 // Add dest value - add w4, w4, w11 // Add constant 0x895cd7be - add w9, w4, w9 // Add aux function result - ror w9, w9, #10 // Rotate left s=22 bits - eor x4, x8, x17 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w9, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0x895cd7be, s=22, M[11]) - ldp w11, w26, [x1, #48] // Load 2 words of input data0 M[12],M[13] - ldp w12, w27, [x1, #56] // Load 2 words of input data0 M[14],M[15] -#ifdef __AARCH64EB__ - rev w11, w11 - rev w26, w26 - rev w12, w12 - rev w27, w27 -#endif - and x4, x4, x9 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x4, x4, x17 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x19, #0x1122 // Load lower half of constant 0x6b901122 - movk x19, #0x6b90, lsl #16 // Load upper half of constant 0x6b901122 - add w6, w6, w11 // Add dest value - add w6, w6, w19 // Add constant 0x6b901122 - add w4, w6, w4 // Add aux function result - ror w4, w4, #25 // Rotate left s=7 bits - eor x6, x9, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w4, w9, w4 // Add X parameter round 1 A=FF(A, B, C, D, 0x6b901122, s=7, M[12]) - and x6, x6, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x6, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x19, #0x7193 // Load lower half of constant 0xfd987193 - movk x19, #0xfd98, lsl #16 // Load upper half of constant 0xfd987193 - add w17, w17, w26 // Add dest value - add w17, w17, w19 // Add constant 0xfd987193 - add w17, w17, w6 // Add aux function result - ror w17, w17, #20 // Rotate left s=12 bits - eor x6, x4, x9 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w17, w4, w17 // Add X parameter round 1 D=FF(D, A, B, C, 0xfd987193, s=12, M[13]) - and x6, x6, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x6, x9 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x13, #0x438e // Load lower half of constant 0xa679438e - movk x13, #0xa679, lsl #16 // Load upper half of constant 0xa679438e - add w8, w8, w12 // Add dest value - add w8, w8, w13 // Add constant 0xa679438e - add w8, w8, w6 // Add aux function result - ror w8, w8, #15 // Rotate left s=17 bits - eor x6, x17, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w8, w17, w8 // Add X parameter round 1 C=FF(C, D, A, B, 0xa679438e, s=17, M[14]) - and x6, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x6, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x13, #0x821 // Load lower half of constant 0x49b40821 - movk x13, #0x49b4, lsl #16 // Load upper half of constant 0x49b40821 - add w9, w9, w27 // Add dest value - add w9, w9, w13 // Add constant 0x49b40821 - add w9, w9, w6 // Add aux function result - ror w9, w9, #10 // Rotate left s=22 bits - bic x6, x8, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w9, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0x49b40821, s=22, M[15]) - and x13, x9, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x2562 // Load lower half of constant 0xf61e2562 - movk x13, #0xf61e, lsl #16 // Load upper half of constant 0xf61e2562 - add w4, w4, w20 // Add dest value - add w4, w4, w13 // Add constant 0xf61e2562 - add w4, w4, w6 // Add aux function result - ror w4, w4, #27 // Rotate left s=5 bits - bic x6, x9, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xf61e2562, s=5, M[1]) - and x13, x4, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xb340 // Load lower half of constant 0xc040b340 - movk x13, #0xc040, lsl #16 // Load upper half of constant 0xc040b340 - add w17, w17, w7 // Add dest value - add w17, w17, w13 // Add constant 0xc040b340 - add w17, w17, w6 // Add aux function result - ror w17, w17, #23 // Rotate left s=9 bits - bic x6, x4, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xc040b340, s=9, M[6]) - and x13, x17, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x5a51 // Load lower half of constant 0x265e5a51 - movk x13, #0x265e, lsl #16 // Load upper half of constant 0x265e5a51 - add w8, w8, w25 // Add dest value - add w8, w8, w13 // Add constant 0x265e5a51 - add w8, w8, w6 // Add aux function result - ror w8, w8, #18 // Rotate left s=14 bits - bic x6, x17, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0x265e5a51, s=14, M[11]) - and x13, x8, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xc7aa // Load lower half of constant 0xe9b6c7aa - movk x13, #0xe9b6, lsl #16 // Load upper half of constant 0xe9b6c7aa - add w9, w9, w15 // Add dest value - add w9, w9, w13 // Add constant 0xe9b6c7aa - add w9, w9, w6 // Add aux function result - ror w9, w9, #12 // Rotate left s=20 bits - bic x6, x8, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0xe9b6c7aa, s=20, M[0]) - and x13, x9, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x105d // Load lower half of constant 0xd62f105d - movk x13, #0xd62f, lsl #16 // Load upper half of constant 0xd62f105d - add w4, w4, w22 // Add dest value - add w4, w4, w13 // Add constant 0xd62f105d - add w4, w4, w6 // Add aux function result - ror w4, w4, #27 // Rotate left s=5 bits - bic x6, x9, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xd62f105d, s=5, M[5]) - and x13, x4, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x1453 // Load lower half of constant 0x2441453 - movk x13, #0x244, lsl #16 // Load upper half of constant 0x2441453 - add w17, w17, w16 // Add dest value - add w17, w17, w13 // Add constant 0x2441453 - add w17, w17, w6 // Add aux function result - ror w17, w17, #23 // Rotate left s=9 bits - bic x6, x4, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0x2441453, s=9, M[10]) - and x13, x17, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xe681 // Load lower half of constant 0xd8a1e681 - movk x13, #0xd8a1, lsl #16 // Load upper half of constant 0xd8a1e681 - add w8, w8, w27 // Add dest value - add w8, w8, w13 // Add constant 0xd8a1e681 - add w8, w8, w6 // Add aux function result - ror w8, w8, #18 // Rotate left s=14 bits - bic x6, x17, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0xd8a1e681, s=14, M[15]) - and x13, x8, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xfbc8 // Load lower half of constant 0xe7d3fbc8 - movk x13, #0xe7d3, lsl #16 // Load upper half of constant 0xe7d3fbc8 - add w9, w9, w14 // Add dest value - add w9, w9, w13 // Add constant 0xe7d3fbc8 - add w9, w9, w6 // Add aux function result - ror w9, w9, #12 // Rotate left s=20 bits - bic x6, x8, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0xe7d3fbc8, s=20, M[4]) - and x13, x9, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xcde6 // Load lower half of constant 0x21e1cde6 - movk x13, #0x21e1, lsl #16 // Load upper half of constant 0x21e1cde6 - add w4, w4, w24 // Add dest value - add w4, w4, w13 // Add constant 0x21e1cde6 - add w4, w4, w6 // Add aux function result - ror w4, w4, #27 // Rotate left s=5 bits - bic x6, x9, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0x21e1cde6, s=5, M[9]) - and x13, x4, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x7d6 // Load lower half of constant 0xc33707d6 - movk x13, #0xc337, lsl #16 // Load upper half of constant 0xc33707d6 - add w17, w17, w12 // Add dest value - add w17, w17, w13 // Add constant 0xc33707d6 - add w17, w17, w6 // Add aux function result - ror w17, w17, #23 // Rotate left s=9 bits - bic x6, x4, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xc33707d6, s=9, M[14]) - and x13, x17, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xd87 // Load lower half of constant 0xf4d50d87 - movk x13, #0xf4d5, lsl #16 // Load upper half of constant 0xf4d50d87 - add w8, w8, w21 // Add dest value - add w8, w8, w13 // Add constant 0xf4d50d87 - add w8, w8, w6 // Add aux function result - ror w8, w8, #18 // Rotate left s=14 bits - bic x6, x17, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0xf4d50d87, s=14, M[3]) - and x13, x8, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x14ed // Load lower half of constant 0x455a14ed - movk x13, #0x455a, lsl #16 // Load upper half of constant 0x455a14ed - add w9, w9, w5 // Add dest value - add w9, w9, w13 // Add constant 0x455a14ed - add w9, w9, w6 // Add aux function result - ror w9, w9, #12 // Rotate left s=20 bits - bic x6, x8, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0x455a14ed, s=20, M[8]) - and x13, x9, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xe905 // Load lower half of constant 0xa9e3e905 - movk x13, #0xa9e3, lsl #16 // Load upper half of constant 0xa9e3e905 - add w4, w4, w26 // Add dest value - add w4, w4, w13 // Add constant 0xa9e3e905 - add w4, w4, w6 // Add aux function result - ror w4, w4, #27 // Rotate left s=5 bits - bic x6, x9, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xa9e3e905, s=5, M[13]) - and x13, x4, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xa3f8 // Load lower half of constant 0xfcefa3f8 - movk x13, #0xfcef, lsl #16 // Load upper half of constant 0xfcefa3f8 - add w17, w17, w3 // Add dest value - add w17, w17, w13 // Add constant 0xfcefa3f8 - add w17, w17, w6 // Add aux function result - ror w17, w17, #23 // Rotate left s=9 bits - bic x6, x4, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xfcefa3f8, s=9, M[2]) - and x13, x17, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x2d9 // Load lower half of constant 0x676f02d9 - movk x13, #0x676f, lsl #16 // Load upper half of constant 0x676f02d9 - add w8, w8, w23 // Add dest value - add w8, w8, w13 // Add constant 0x676f02d9 - add w8, w8, w6 // Add aux function result - ror w8, w8, #18 // Rotate left s=14 bits - bic x6, x17, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0x676f02d9, s=14, M[7]) - and x13, x8, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x4c8a // Load lower half of constant 0x8d2a4c8a - movk x13, #0x8d2a, lsl #16 // Load upper half of constant 0x8d2a4c8a - add w9, w9, w11 // Add dest value - add w9, w9, w13 // Add constant 0x8d2a4c8a - add w9, w9, w6 // Add aux function result - eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w9, w9, #12 // Rotate left s=20 bits - movz x10, #0x3942 // Load lower half of constant 0xfffa3942 - add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0x8d2a4c8a, s=20, M[12]) - movk x10, #0xfffa, lsl #16 // Load upper half of constant 0xfffa3942 - add w4, w4, w22 // Add dest value - eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) - add w4, w4, w10 // Add constant 0xfffa3942 - add w4, w4, w6 // Add aux function result - ror w4, w4, #28 // Rotate left s=4 bits - eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x10, #0xf681 // Load lower half of constant 0x8771f681 - add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xfffa3942, s=4, M[5]) - movk x10, #0x8771, lsl #16 // Load upper half of constant 0x8771f681 - add w17, w17, w5 // Add dest value - eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) - add w17, w17, w10 // Add constant 0x8771f681 - add w17, w17, w6 // Add aux function result - eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w17, w17, #21 // Rotate left s=11 bits - movz x13, #0x6122 // Load lower half of constant 0x6d9d6122 - add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0x8771f681, s=11, M[8]) - movk x13, #0x6d9d, lsl #16 // Load upper half of constant 0x6d9d6122 - add w8, w8, w25 // Add dest value - eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) - add w8, w8, w13 // Add constant 0x6d9d6122 - add w8, w8, w6 // Add aux function result - ror w8, w8, #16 // Rotate left s=16 bits - eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x13, #0x380c // Load lower half of constant 0xfde5380c - add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0x6d9d6122, s=16, M[11]) - movk x13, #0xfde5, lsl #16 // Load upper half of constant 0xfde5380c - add w9, w9, w12 // Add dest value - eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) - add w9, w9, w13 // Add constant 0xfde5380c - add w9, w9, w6 // Add aux function result - eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w9, w9, #9 // Rotate left s=23 bits - movz x10, #0xea44 // Load lower half of constant 0xa4beea44 - add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xfde5380c, s=23, M[14]) - movk x10, #0xa4be, lsl #16 // Load upper half of constant 0xa4beea44 - add w4, w4, w20 // Add dest value - eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) - add w4, w4, w10 // Add constant 0xa4beea44 - add w4, w4, w6 // Add aux function result - ror w4, w4, #28 // Rotate left s=4 bits - eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x10, #0xcfa9 // Load lower half of constant 0x4bdecfa9 - add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xa4beea44, s=4, M[1]) - movk x10, #0x4bde, lsl #16 // Load upper half of constant 0x4bdecfa9 - add w17, w17, w14 // Add dest value - eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) - add w17, w17, w10 // Add constant 0x4bdecfa9 - add w17, w17, w6 // Add aux function result - eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w17, w17, #21 // Rotate left s=11 bits - movz x13, #0x4b60 // Load lower half of constant 0xf6bb4b60 - add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0x4bdecfa9, s=11, M[4]) - movk x13, #0xf6bb, lsl #16 // Load upper half of constant 0xf6bb4b60 - add w8, w8, w23 // Add dest value - eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) - add w8, w8, w13 // Add constant 0xf6bb4b60 - add w8, w8, w6 // Add aux function result - ror w8, w8, #16 // Rotate left s=16 bits - eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x13, #0xbc70 // Load lower half of constant 0xbebfbc70 - add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0xf6bb4b60, s=16, M[7]) - movk x13, #0xbebf, lsl #16 // Load upper half of constant 0xbebfbc70 - add w9, w9, w16 // Add dest value - eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) - add w9, w9, w13 // Add constant 0xbebfbc70 - add w9, w9, w6 // Add aux function result - eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w9, w9, #9 // Rotate left s=23 bits - movz x10, #0x7ec6 // Load lower half of constant 0x289b7ec6 - add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xbebfbc70, s=23, M[10]) - movk x10, #0x289b, lsl #16 // Load upper half of constant 0x289b7ec6 - add w4, w4, w26 // Add dest value - eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) - add w4, w4, w10 // Add constant 0x289b7ec6 - add w4, w4, w6 // Add aux function result - ror w4, w4, #28 // Rotate left s=4 bits - eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x10, #0x27fa // Load lower half of constant 0xeaa127fa - add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0x289b7ec6, s=4, M[13]) - movk x10, #0xeaa1, lsl #16 // Load upper half of constant 0xeaa127fa - add w17, w17, w15 // Add dest value - eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) - add w17, w17, w10 // Add constant 0xeaa127fa - add w17, w17, w6 // Add aux function result - eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w17, w17, #21 // Rotate left s=11 bits - movz x13, #0x3085 // Load lower half of constant 0xd4ef3085 - add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0xeaa127fa, s=11, M[0]) - movk x13, #0xd4ef, lsl #16 // Load upper half of constant 0xd4ef3085 - add w8, w8, w21 // Add dest value - eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) - add w8, w8, w13 // Add constant 0xd4ef3085 - add w8, w8, w6 // Add aux function result - ror w8, w8, #16 // Rotate left s=16 bits - eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x13, #0x1d05 // Load lower half of constant 0x4881d05 - add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0xd4ef3085, s=16, M[3]) - movk x13, #0x488, lsl #16 // Load upper half of constant 0x4881d05 - add w9, w9, w7 // Add dest value - eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) - add w9, w9, w13 // Add constant 0x4881d05 - add w9, w9, w6 // Add aux function result - eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w9, w9, #9 // Rotate left s=23 bits - movz x10, #0xd039 // Load lower half of constant 0xd9d4d039 - add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0x4881d05, s=23, M[6]) - movk x10, #0xd9d4, lsl #16 // Load upper half of constant 0xd9d4d039 - add w4, w4, w24 // Add dest value - eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) - add w4, w4, w10 // Add constant 0xd9d4d039 - add w4, w4, w6 // Add aux function result - ror w4, w4, #28 // Rotate left s=4 bits - eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x10, #0x99e5 // Load lower half of constant 0xe6db99e5 - add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xd9d4d039, s=4, M[9]) - movk x10, #0xe6db, lsl #16 // Load upper half of constant 0xe6db99e5 - add w17, w17, w11 // Add dest value - eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) - add w17, w17, w10 // Add constant 0xe6db99e5 - add w17, w17, w6 // Add aux function result - eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w17, w17, #21 // Rotate left s=11 bits - movz x13, #0x7cf8 // Load lower half of constant 0x1fa27cf8 - add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0xe6db99e5, s=11, M[12]) - movk x13, #0x1fa2, lsl #16 // Load upper half of constant 0x1fa27cf8 - add w8, w8, w27 // Add dest value - eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) - add w8, w8, w13 // Add constant 0x1fa27cf8 - add w8, w8, w6 // Add aux function result - ror w8, w8, #16 // Rotate left s=16 bits - eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x13, #0x5665 // Load lower half of constant 0xc4ac5665 - add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0x1fa27cf8, s=16, M[15]) - movk x13, #0xc4ac, lsl #16 // Load upper half of constant 0xc4ac5665 - add w9, w9, w3 // Add dest value - eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) - add w9, w9, w13 // Add constant 0xc4ac5665 - add w9, w9, w6 // Add aux function result - ror w9, w9, #9 // Rotate left s=23 bits - movz x6, #0x2244 // Load lower half of constant 0xf4292244 - movk x6, #0xf429, lsl #16 // Load upper half of constant 0xf4292244 - add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xc4ac5665, s=23, M[2]) - add w4, w4, w15 // Add dest value - orn x13, x9, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w4, w4, w6 // Add constant 0xf4292244 - eor x6, x8, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w4, w4, w6 // Add aux function result - ror w4, w4, #26 // Rotate left s=6 bits - movz x6, #0xff97 // Load lower half of constant 0x432aff97 - movk x6, #0x432a, lsl #16 // Load upper half of constant 0x432aff97 - add w4, w9, w4 // Add X parameter round 4 A=II(A, B, C, D, 0xf4292244, s=6, M[0]) - orn x10, x4, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w17, w17, w23 // Add dest value - eor x10, x9, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w17, w17, w6 // Add constant 0x432aff97 - add w6, w17, w10 // Add aux function result - ror w6, w6, #22 // Rotate left s=10 bits - movz x17, #0x23a7 // Load lower half of constant 0xab9423a7 - movk x17, #0xab94, lsl #16 // Load upper half of constant 0xab9423a7 - add w6, w4, w6 // Add X parameter round 4 D=II(D, A, B, C, 0x432aff97, s=10, M[7]) - add w8, w8, w12 // Add dest value - orn x10, x6, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w8, w17 // Add constant 0xab9423a7 - eor x17, x4, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w8, w17 // Add aux function result - ror w8, w8, #17 // Rotate left s=15 bits - movz x17, #0xa039 // Load lower half of constant 0xfc93a039 - movk x17, #0xfc93, lsl #16 // Load upper half of constant 0xfc93a039 - add w8, w6, w8 // Add X parameter round 4 C=II(C, D, A, B, 0xab9423a7, s=15, M[14]) - orn x13, x8, x4 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w9, w22 // Add dest value - eor x13, x6, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w9, w17 // Add constant 0xfc93a039 - add w17, w9, w13 // Add aux function result - ror w17, w17, #11 // Rotate left s=21 bits - movz x9, #0x59c3 // Load lower half of constant 0x655b59c3 - movk x9, #0x655b, lsl #16 // Load upper half of constant 0x655b59c3 - add w17, w8, w17 // Add X parameter round 4 B=II(B, C, D, A, 0xfc93a039, s=21, M[5]) - add w4, w4, w11 // Add dest value - orn x13, x17, x6 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w4, w9 // Add constant 0x655b59c3 - eor x4, x8, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w9, w4 // Add aux function result - ror w9, w9, #26 // Rotate left s=6 bits - movz x4, #0xcc92 // Load lower half of constant 0x8f0ccc92 - movk x4, #0x8f0c, lsl #16 // Load upper half of constant 0x8f0ccc92 - add w9, w17, w9 // Add X parameter round 4 A=II(A, B, C, D, 0x655b59c3, s=6, M[12]) - orn x10, x9, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w6, w6, w21 // Add dest value - eor x10, x17, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w4, w6, w4 // Add constant 0x8f0ccc92 - add w6, w4, w10 // Add aux function result - ror w6, w6, #22 // Rotate left s=10 bits - movz x4, #0xf47d // Load lower half of constant 0xffeff47d - movk x4, #0xffef, lsl #16 // Load upper half of constant 0xffeff47d - add w6, w9, w6 // Add X parameter round 4 D=II(D, A, B, C, 0x8f0ccc92, s=10, M[3]) - add w8, w8, w16 // Add dest value - orn x10, x6, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w8, w4 // Add constant 0xffeff47d - eor x4, x9, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w8, w4 // Add aux function result - ror w8, w8, #17 // Rotate left s=15 bits - movz x4, #0x5dd1 // Load lower half of constant 0x85845dd1 - movk x4, #0x8584, lsl #16 // Load upper half of constant 0x85845dd1 - add w8, w6, w8 // Add X parameter round 4 C=II(C, D, A, B, 0xffeff47d, s=15, M[10]) - orn x10, x8, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w17, w20 // Add dest value - eor x17, x6, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w15, w4 // Add constant 0x85845dd1 - add w4, w15, w17 // Add aux function result - ror w4, w4, #11 // Rotate left s=21 bits - movz x15, #0x7e4f // Load lower half of constant 0x6fa87e4f - movk x15, #0x6fa8, lsl #16 // Load upper half of constant 0x6fa87e4f - add w17, w8, w4 // Add X parameter round 4 B=II(B, C, D, A, 0x85845dd1, s=21, M[1]) - add w4, w9, w5 // Add dest value - orn x9, x17, x6 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w4, w15 // Add constant 0x6fa87e4f - eor x4, x8, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w15, w4 // Add aux function result - ror w9, w9, #26 // Rotate left s=6 bits - movz x15, #0xe6e0 // Load lower half of constant 0xfe2ce6e0 - movk x15, #0xfe2c, lsl #16 // Load upper half of constant 0xfe2ce6e0 - add w4, w17, w9 // Add X parameter round 4 A=II(A, B, C, D, 0x6fa87e4f, s=6, M[8]) - orn x9, x4, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w6, w6, w27 // Add dest value - eor x9, x17, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w6, w15 // Add constant 0xfe2ce6e0 - add w6, w15, w9 // Add aux function result - ror w6, w6, #22 // Rotate left s=10 bits - movz x9, #0x4314 // Load lower half of constant 0xa3014314 - movk x9, #0xa301, lsl #16 // Load upper half of constant 0xa3014314 - add w15, w4, w6 // Add X parameter round 4 D=II(D, A, B, C, 0xfe2ce6e0, s=10, M[15]) - add w6, w8, w7 // Add dest value - orn x7, x15, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w6, w9 // Add constant 0xa3014314 - eor x9, x4, x7 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w6, w8, w9 // Add aux function result - ror w6, w6, #17 // Rotate left s=15 bits - movz x7, #0x11a1 // Load lower half of constant 0x4e0811a1 - movk x7, #0x4e08, lsl #16 // Load upper half of constant 0x4e0811a1 - add w8, w15, w6 // Add X parameter round 4 C=II(C, D, A, B, 0xa3014314, s=15, M[6]) - orn x9, x8, x4 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w6, w17, w26 // Add dest value - eor x17, x15, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w6, w7 // Add constant 0x4e0811a1 - add w7, w9, w17 // Add aux function result - ror w7, w7, #11 // Rotate left s=21 bits - movz x6, #0x7e82 // Load lower half of constant 0xf7537e82 - movk x6, #0xf753, lsl #16 // Load upper half of constant 0xf7537e82 - add w9, w8, w7 // Add X parameter round 4 B=II(B, C, D, A, 0x4e0811a1, s=21, M[13]) - add w17, w4, w14 // Add dest value - orn x7, x9, x15 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w14, w17, w6 // Add constant 0xf7537e82 - eor x4, x8, x7 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w17, w14, w4 // Add aux function result - ror w17, w17, #26 // Rotate left s=6 bits - movz x6, #0xf235 // Load lower half of constant 0xbd3af235 - movk x6, #0xbd3a, lsl #16 // Load upper half of constant 0xbd3af235 - add w7, w9, w17 // Add X parameter round 4 A=II(A, B, C, D, 0xf7537e82, s=6, M[4]) - orn x14, x7, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w4, w15, w25 // Add dest value - eor x17, x9, x14 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w4, w6 // Add constant 0xbd3af235 - add w16, w15, w17 // Add aux function result - ror w16, w16, #22 // Rotate left s=10 bits - movz x14, #0xd2bb // Load lower half of constant 0x2ad7d2bb - movk x14, #0x2ad7, lsl #16 // Load upper half of constant 0x2ad7d2bb - add w4, w7, w16 // Add X parameter round 4 D=II(D, A, B, C, 0xbd3af235, s=10, M[11]) - add w6, w8, w3 // Add dest value - orn x15, x4, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w17, w6, w14 // Add constant 0x2ad7d2bb - eor x16, x7, x15 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w17, w16 // Add aux function result - ror w8, w8, #17 // Rotate left s=15 bits - movz x3, #0xd391 // Load lower half of constant 0xeb86d391 - movk x3, #0xeb86, lsl #16 // Load upper half of constant 0xeb86d391 - add w14, w4, w8 // Add X parameter round 4 C=II(C, D, A, B, 0x2ad7d2bb, s=15, M[2]) - orn x6, x14, x7 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w9, w24 // Add dest value - eor x17, x4, x6 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w16, w15, w3 // Add constant 0xeb86d391 - add w8, w16, w17 // Add aux function result - ror w8, w8, #11 // Rotate left s=21 bits - ldp w6, w15, [x0] // Reload MD5 state->A and state->B - ldp w5, w9, [x0, #8] // Reload MD5 state->C and state->D - add w3, w14, w8 // Add X parameter round 4 B=II(B, C, D, A, 0xeb86d391, s=21, M[9]) - add w13, w4, w9 // Add result of MD5 rounds to state->D - add w12, w14, w5 // Add result of MD5 rounds to state->C - add w10, w7, w6 // Add result of MD5 rounds to state->A - add w11, w3, w15 // Add result of MD5 rounds to state->B - stp w12, w13, [x0, #8] // Store MD5 states C,D - stp w10, w11, [x0] // Store MD5 states A,B - add x1, x1, #64 // Increment data pointer - subs w2, w2, #1 // Decrement block counter - b.ne ossl_md5_blocks_loop - - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - ldp x25,x26,[sp,#48] - ldp x27,x28,[sp,#64] - ldp x19,x20,[sp],#80 - ret - diff --git a/openssl/src/crypto/md5/gen/linux_arm64/md5-aarch64.S b/openssl/src/crypto/md5/gen/linux_arm64/md5-aarch64.S deleted file mode 100644 index 67e477fd5..000000000 --- a/openssl/src/crypto/md5/gen/linux_arm64/md5-aarch64.S +++ /dev/null @@ -1,677 +0,0 @@ -#include "arm_arch.h" - -.text -.globl ossl_md5_block_asm_data_order -.type ossl_md5_block_asm_data_order,@function -ossl_md5_block_asm_data_order: - AARCH64_VALID_CALL_TARGET - // Save all callee-saved registers - stp x19,x20,[sp,#-80]! - stp x21,x22,[sp,#16] - stp x23,x24,[sp,#32] - stp x25,x26,[sp,#48] - stp x27,x28,[sp,#64] - - ldp w10, w11, [x0, #0] // .Load MD5 state->A and state->B - ldp w12, w13, [x0, #8] // .Load MD5 state->C and state->D -.align 5 -ossl_md5_blocks_loop: - eor x17, x12, x13 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - and x16, x17, x11 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - ldp w15, w20, [x1] // .Load 2 words of input data0 M[0],M[1] - ldp w3, w21, [x1, #8] // .Load 2 words of input data0 M[2],M[3] -#ifdef __AARCH64EB__ - rev w15, w15 - rev w20, w20 - rev w3, w3 - rev w21, w21 -#endif - eor x14, x16, x13 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x9, #0xa478 // .Load lower half of constant 0xd76aa478 - movk x9, #0xd76a, lsl #16 // .Load upper half of constant 0xd76aa478 - add w8, w10, w15 // Add dest value - add w7, w8, w9 // Add constant 0xd76aa478 - add w6, w7, w14 // Add aux function result - ror w6, w6, #25 // Rotate left s=7 bits - eor x5, x11, x12 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w4, w11, w6 // Add X parameter round 1 A=FF(A, B, C, D, 0xd76aa478, s=7, M[0]) - and x8, x5, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x17, x8, x12 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x16, #0xb756 // .Load lower half of constant 0xe8c7b756 - movk x16, #0xe8c7, lsl #16 // .Load upper half of constant 0xe8c7b756 - add w9, w13, w20 // Add dest value - add w7, w9, w16 // Add constant 0xe8c7b756 - add w14, w7, w17 // Add aux function result - ror w14, w14, #20 // Rotate left s=12 bits - eor x6, x4, x11 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w5, w4, w14 // Add X parameter round 1 D=FF(D, A, B, C, 0xe8c7b756, s=12, M[1]) - and x8, x6, x5 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x9, x8, x11 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x16, #0x70db // .Load lower half of constant 0x242070db - movk x16, #0x2420, lsl #16 // .Load upper half of constant 0x242070db - add w7, w12, w3 // Add dest value - add w17, w7, w16 // Add constant 0x242070db - add w14, w17, w9 // Add aux function result - ror w14, w14, #15 // Rotate left s=17 bits - eor x6, x5, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w8, w5, w14 // Add X parameter round 1 C=FF(C, D, A, B, 0x242070db, s=17, M[2]) - and x7, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x16, x7, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x9, #0xceee // .Load lower half of constant 0xc1bdceee - movk x9, #0xc1bd, lsl #16 // .Load upper half of constant 0xc1bdceee - add w14, w11, w21 // Add dest value - add w6, w14, w9 // Add constant 0xc1bdceee - add w7, w6, w16 // Add aux function result - ror w7, w7, #10 // Rotate left s=22 bits - eor x17, x8, x5 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w9, w8, w7 // Add X parameter round 1 B=FF(B, C, D, A, 0xc1bdceee, s=22, M[3]) - ldp w14, w22, [x1, #16] // .Load 2 words of input data0 M[4],M[5] - ldp w7, w23, [x1, #24] // .Load 2 words of input data0 M[6],M[7] -#ifdef __AARCH64EB__ - rev w14, w14 - rev w22, w22 - rev w7, w7 - rev w23, w23 -#endif - and x16, x17, x9 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x16, x5 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x16, #0xfaf // .Load lower half of constant 0xf57c0faf - movk x16, #0xf57c, lsl #16 // .Load upper half of constant 0xf57c0faf - add w17, w4, w14 // Add dest value - add w16, w17, w16 // Add constant 0xf57c0faf - add w4, w16, w6 // Add aux function result - ror w4, w4, #25 // Rotate left s=7 bits - eor x16, x9, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w17, w9, w4 // Add X parameter round 1 A=FF(A, B, C, D, 0xf57c0faf, s=7, M[4]) - and x16, x16, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x16, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x4, #0xc62a // .Load lower half of constant 0x4787c62a - movk x4, #0x4787, lsl #16 // .Load upper half of constant 0x4787c62a - add w16, w5, w22 // Add dest value - add w16, w16, w4 // Add constant 0x4787c62a - add w5, w16, w6 // Add aux function result - ror w5, w5, #20 // Rotate left s=12 bits - eor x4, x17, x9 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w19, w17, w5 // Add X parameter round 1 D=FF(D, A, B, C, 0x4787c62a, s=12, M[5]) - and x6, x4, x19 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x5, x6, x9 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x4, #0x4613 // .Load lower half of constant 0xa8304613 - movk x4, #0xa830, lsl #16 // .Load upper half of constant 0xa8304613 - add w6, w8, w7 // Add dest value - add w8, w6, w4 // Add constant 0xa8304613 - add w4, w8, w5 // Add aux function result - ror w4, w4, #15 // Rotate left s=17 bits - eor x6, x19, x17 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w8, w19, w4 // Add X parameter round 1 C=FF(C, D, A, B, 0xa8304613, s=17, M[6]) - and x5, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x4, x5, x17 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x6, #0x9501 // .Load lower half of constant 0xfd469501 - movk x6, #0xfd46, lsl #16 // .Load upper half of constant 0xfd469501 - add w9, w9, w23 // Add dest value - add w5, w9, w6 // Add constant 0xfd469501 - add w9, w5, w4 // Add aux function result - ror w9, w9, #10 // Rotate left s=22 bits - eor x6, x8, x19 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w4, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0xfd469501, s=22, M[7]) - ldp w5, w24, [x1, #32] // .Load 2 words of input data0 M[8],M[9] - ldp w16, w25, [x1, #40] // .Load 2 words of input data0 M[10],M[11] -#ifdef __AARCH64EB__ - rev w5, w5 - rev w24, w24 - rev w16, w16 - rev w25, w25 -#endif - and x9, x6, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x9, x19 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x9, #0x98d8 // .Load lower half of constant 0x698098d8 - movk x9, #0x6980, lsl #16 // .Load upper half of constant 0x698098d8 - add w17, w17, w5 // Add dest value - add w9, w17, w9 // Add constant 0x698098d8 - add w17, w9, w6 // Add aux function result - ror w17, w17, #25 // Rotate left s=7 bits - eor x9, x4, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w6, w4, w17 // Add X parameter round 1 A=FF(A, B, C, D, 0x698098d8, s=7, M[8]) - and x17, x9, x6 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x9, x17, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x17, #0xf7af // .Load lower half of constant 0x8b44f7af - movk x17, #0x8b44, lsl #16 // .Load upper half of constant 0x8b44f7af - add w19, w19, w24 // Add dest value - add w17, w19, w17 // Add constant 0x8b44f7af - add w19, w17, w9 // Add aux function result - ror w19, w19, #20 // Rotate left s=12 bits - eor x9, x6, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w17, w6, w19 // Add X parameter round 1 D=FF(D, A, B, C, 0x8b44f7af, s=12, M[9]) - and x9, x9, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x9, x9, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x11, #0x5bb1 // .Load lower half of constant 0xffff5bb1 - movk x11, #0xffff, lsl #16 // .Load upper half of constant 0xffff5bb1 - add w8, w8, w16 // Add dest value - add w8, w8, w11 // Add constant 0xffff5bb1 - add w8, w8, w9 // Add aux function result - ror w8, w8, #15 // Rotate left s=17 bits - eor x9, x17, x6 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w8, w17, w8 // Add X parameter round 1 C=FF(C, D, A, B, 0xffff5bb1, s=17, M[10]) - and x9, x9, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x9, x9, x6 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x11, #0xd7be // .Load lower half of constant 0x895cd7be - movk x11, #0x895c, lsl #16 // .Load upper half of constant 0x895cd7be - add w4, w4, w25 // Add dest value - add w4, w4, w11 // Add constant 0x895cd7be - add w9, w4, w9 // Add aux function result - ror w9, w9, #10 // Rotate left s=22 bits - eor x4, x8, x17 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w9, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0x895cd7be, s=22, M[11]) - ldp w11, w26, [x1, #48] // .Load 2 words of input data0 M[12],M[13] - ldp w12, w27, [x1, #56] // .Load 2 words of input data0 M[14],M[15] -#ifdef __AARCH64EB__ - rev w11, w11 - rev w26, w26 - rev w12, w12 - rev w27, w27 -#endif - and x4, x4, x9 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x4, x4, x17 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x19, #0x1122 // .Load lower half of constant 0x6b901122 - movk x19, #0x6b90, lsl #16 // .Load upper half of constant 0x6b901122 - add w6, w6, w11 // Add dest value - add w6, w6, w19 // Add constant 0x6b901122 - add w4, w6, w4 // Add aux function result - ror w4, w4, #25 // Rotate left s=7 bits - eor x6, x9, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w4, w9, w4 // Add X parameter round 1 A=FF(A, B, C, D, 0x6b901122, s=7, M[12]) - and x6, x6, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x6, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x19, #0x7193 // .Load lower half of constant 0xfd987193 - movk x19, #0xfd98, lsl #16 // .Load upper half of constant 0xfd987193 - add w17, w17, w26 // Add dest value - add w17, w17, w19 // Add constant 0xfd987193 - add w17, w17, w6 // Add aux function result - ror w17, w17, #20 // Rotate left s=12 bits - eor x6, x4, x9 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w17, w4, w17 // Add X parameter round 1 D=FF(D, A, B, C, 0xfd987193, s=12, M[13]) - and x6, x6, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x6, x9 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x13, #0x438e // .Load lower half of constant 0xa679438e - movk x13, #0xa679, lsl #16 // .Load upper half of constant 0xa679438e - add w8, w8, w12 // Add dest value - add w8, w8, w13 // Add constant 0xa679438e - add w8, w8, w6 // Add aux function result - ror w8, w8, #15 // Rotate left s=17 bits - eor x6, x17, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) - add w8, w17, w8 // Add X parameter round 1 C=FF(C, D, A, B, 0xa679438e, s=17, M[14]) - and x6, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) - eor x6, x6, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) - movz x13, #0x821 // .Load lower half of constant 0x49b40821 - movk x13, #0x49b4, lsl #16 // .Load upper half of constant 0x49b40821 - add w9, w9, w27 // Add dest value - add w9, w9, w13 // Add constant 0x49b40821 - add w9, w9, w6 // Add aux function result - ror w9, w9, #10 // Rotate left s=22 bits - bic x6, x8, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w9, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0x49b40821, s=22, M[15]) - and x13, x9, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x2562 // .Load lower half of constant 0xf61e2562 - movk x13, #0xf61e, lsl #16 // .Load upper half of constant 0xf61e2562 - add w4, w4, w20 // Add dest value - add w4, w4, w13 // Add constant 0xf61e2562 - add w4, w4, w6 // Add aux function result - ror w4, w4, #27 // Rotate left s=5 bits - bic x6, x9, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xf61e2562, s=5, M[1]) - and x13, x4, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xb340 // .Load lower half of constant 0xc040b340 - movk x13, #0xc040, lsl #16 // .Load upper half of constant 0xc040b340 - add w17, w17, w7 // Add dest value - add w17, w17, w13 // Add constant 0xc040b340 - add w17, w17, w6 // Add aux function result - ror w17, w17, #23 // Rotate left s=9 bits - bic x6, x4, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xc040b340, s=9, M[6]) - and x13, x17, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x5a51 // .Load lower half of constant 0x265e5a51 - movk x13, #0x265e, lsl #16 // .Load upper half of constant 0x265e5a51 - add w8, w8, w25 // Add dest value - add w8, w8, w13 // Add constant 0x265e5a51 - add w8, w8, w6 // Add aux function result - ror w8, w8, #18 // Rotate left s=14 bits - bic x6, x17, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0x265e5a51, s=14, M[11]) - and x13, x8, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xc7aa // .Load lower half of constant 0xe9b6c7aa - movk x13, #0xe9b6, lsl #16 // .Load upper half of constant 0xe9b6c7aa - add w9, w9, w15 // Add dest value - add w9, w9, w13 // Add constant 0xe9b6c7aa - add w9, w9, w6 // Add aux function result - ror w9, w9, #12 // Rotate left s=20 bits - bic x6, x8, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0xe9b6c7aa, s=20, M[0]) - and x13, x9, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x105d // .Load lower half of constant 0xd62f105d - movk x13, #0xd62f, lsl #16 // .Load upper half of constant 0xd62f105d - add w4, w4, w22 // Add dest value - add w4, w4, w13 // Add constant 0xd62f105d - add w4, w4, w6 // Add aux function result - ror w4, w4, #27 // Rotate left s=5 bits - bic x6, x9, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xd62f105d, s=5, M[5]) - and x13, x4, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x1453 // .Load lower half of constant 0x2441453 - movk x13, #0x244, lsl #16 // .Load upper half of constant 0x2441453 - add w17, w17, w16 // Add dest value - add w17, w17, w13 // Add constant 0x2441453 - add w17, w17, w6 // Add aux function result - ror w17, w17, #23 // Rotate left s=9 bits - bic x6, x4, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0x2441453, s=9, M[10]) - and x13, x17, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xe681 // .Load lower half of constant 0xd8a1e681 - movk x13, #0xd8a1, lsl #16 // .Load upper half of constant 0xd8a1e681 - add w8, w8, w27 // Add dest value - add w8, w8, w13 // Add constant 0xd8a1e681 - add w8, w8, w6 // Add aux function result - ror w8, w8, #18 // Rotate left s=14 bits - bic x6, x17, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0xd8a1e681, s=14, M[15]) - and x13, x8, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xfbc8 // .Load lower half of constant 0xe7d3fbc8 - movk x13, #0xe7d3, lsl #16 // .Load upper half of constant 0xe7d3fbc8 - add w9, w9, w14 // Add dest value - add w9, w9, w13 // Add constant 0xe7d3fbc8 - add w9, w9, w6 // Add aux function result - ror w9, w9, #12 // Rotate left s=20 bits - bic x6, x8, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0xe7d3fbc8, s=20, M[4]) - and x13, x9, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xcde6 // .Load lower half of constant 0x21e1cde6 - movk x13, #0x21e1, lsl #16 // .Load upper half of constant 0x21e1cde6 - add w4, w4, w24 // Add dest value - add w4, w4, w13 // Add constant 0x21e1cde6 - add w4, w4, w6 // Add aux function result - ror w4, w4, #27 // Rotate left s=5 bits - bic x6, x9, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0x21e1cde6, s=5, M[9]) - and x13, x4, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x7d6 // .Load lower half of constant 0xc33707d6 - movk x13, #0xc337, lsl #16 // .Load upper half of constant 0xc33707d6 - add w17, w17, w12 // Add dest value - add w17, w17, w13 // Add constant 0xc33707d6 - add w17, w17, w6 // Add aux function result - ror w17, w17, #23 // Rotate left s=9 bits - bic x6, x4, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xc33707d6, s=9, M[14]) - and x13, x17, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xd87 // .Load lower half of constant 0xf4d50d87 - movk x13, #0xf4d5, lsl #16 // .Load upper half of constant 0xf4d50d87 - add w8, w8, w21 // Add dest value - add w8, w8, w13 // Add constant 0xf4d50d87 - add w8, w8, w6 // Add aux function result - ror w8, w8, #18 // Rotate left s=14 bits - bic x6, x17, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0xf4d50d87, s=14, M[3]) - and x13, x8, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x14ed // .Load lower half of constant 0x455a14ed - movk x13, #0x455a, lsl #16 // .Load upper half of constant 0x455a14ed - add w9, w9, w5 // Add dest value - add w9, w9, w13 // Add constant 0x455a14ed - add w9, w9, w6 // Add aux function result - ror w9, w9, #12 // Rotate left s=20 bits - bic x6, x8, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0x455a14ed, s=20, M[8]) - and x13, x9, x17 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xe905 // .Load lower half of constant 0xa9e3e905 - movk x13, #0xa9e3, lsl #16 // .Load upper half of constant 0xa9e3e905 - add w4, w4, w26 // Add dest value - add w4, w4, w13 // Add constant 0xa9e3e905 - add w4, w4, w6 // Add aux function result - ror w4, w4, #27 // Rotate left s=5 bits - bic x6, x9, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xa9e3e905, s=5, M[13]) - and x13, x4, x8 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0xa3f8 // .Load lower half of constant 0xfcefa3f8 - movk x13, #0xfcef, lsl #16 // .Load upper half of constant 0xfcefa3f8 - add w17, w17, w3 // Add dest value - add w17, w17, w13 // Add constant 0xfcefa3f8 - add w17, w17, w6 // Add aux function result - ror w17, w17, #23 // Rotate left s=9 bits - bic x6, x4, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xfcefa3f8, s=9, M[2]) - and x13, x17, x9 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x2d9 // .Load lower half of constant 0x676f02d9 - movk x13, #0x676f, lsl #16 // .Load upper half of constant 0x676f02d9 - add w8, w8, w23 // Add dest value - add w8, w8, w13 // Add constant 0x676f02d9 - add w8, w8, w6 // Add aux function result - ror w8, w8, #18 // Rotate left s=14 bits - bic x6, x17, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0x676f02d9, s=14, M[7]) - and x13, x8, x4 // Aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - orr x6, x6, x13 // End aux function round 2 G(x,y,z)=((x&z)|(~z&y)) - movz x13, #0x4c8a // .Load lower half of constant 0x8d2a4c8a - movk x13, #0x8d2a, lsl #16 // .Load upper half of constant 0x8d2a4c8a - add w9, w9, w11 // Add dest value - add w9, w9, w13 // Add constant 0x8d2a4c8a - add w9, w9, w6 // Add aux function result - eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w9, w9, #12 // Rotate left s=20 bits - movz x10, #0x3942 // .Load lower half of constant 0xfffa3942 - add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0x8d2a4c8a, s=20, M[12]) - movk x10, #0xfffa, lsl #16 // .Load upper half of constant 0xfffa3942 - add w4, w4, w22 // Add dest value - eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) - add w4, w4, w10 // Add constant 0xfffa3942 - add w4, w4, w6 // Add aux function result - ror w4, w4, #28 // Rotate left s=4 bits - eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x10, #0xf681 // .Load lower half of constant 0x8771f681 - add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xfffa3942, s=4, M[5]) - movk x10, #0x8771, lsl #16 // .Load upper half of constant 0x8771f681 - add w17, w17, w5 // Add dest value - eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) - add w17, w17, w10 // Add constant 0x8771f681 - add w17, w17, w6 // Add aux function result - eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w17, w17, #21 // Rotate left s=11 bits - movz x13, #0x6122 // .Load lower half of constant 0x6d9d6122 - add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0x8771f681, s=11, M[8]) - movk x13, #0x6d9d, lsl #16 // .Load upper half of constant 0x6d9d6122 - add w8, w8, w25 // Add dest value - eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) - add w8, w8, w13 // Add constant 0x6d9d6122 - add w8, w8, w6 // Add aux function result - ror w8, w8, #16 // Rotate left s=16 bits - eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x13, #0x380c // .Load lower half of constant 0xfde5380c - add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0x6d9d6122, s=16, M[11]) - movk x13, #0xfde5, lsl #16 // .Load upper half of constant 0xfde5380c - add w9, w9, w12 // Add dest value - eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) - add w9, w9, w13 // Add constant 0xfde5380c - add w9, w9, w6 // Add aux function result - eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w9, w9, #9 // Rotate left s=23 bits - movz x10, #0xea44 // .Load lower half of constant 0xa4beea44 - add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xfde5380c, s=23, M[14]) - movk x10, #0xa4be, lsl #16 // .Load upper half of constant 0xa4beea44 - add w4, w4, w20 // Add dest value - eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) - add w4, w4, w10 // Add constant 0xa4beea44 - add w4, w4, w6 // Add aux function result - ror w4, w4, #28 // Rotate left s=4 bits - eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x10, #0xcfa9 // .Load lower half of constant 0x4bdecfa9 - add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xa4beea44, s=4, M[1]) - movk x10, #0x4bde, lsl #16 // .Load upper half of constant 0x4bdecfa9 - add w17, w17, w14 // Add dest value - eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) - add w17, w17, w10 // Add constant 0x4bdecfa9 - add w17, w17, w6 // Add aux function result - eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w17, w17, #21 // Rotate left s=11 bits - movz x13, #0x4b60 // .Load lower half of constant 0xf6bb4b60 - add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0x4bdecfa9, s=11, M[4]) - movk x13, #0xf6bb, lsl #16 // .Load upper half of constant 0xf6bb4b60 - add w8, w8, w23 // Add dest value - eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) - add w8, w8, w13 // Add constant 0xf6bb4b60 - add w8, w8, w6 // Add aux function result - ror w8, w8, #16 // Rotate left s=16 bits - eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x13, #0xbc70 // .Load lower half of constant 0xbebfbc70 - add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0xf6bb4b60, s=16, M[7]) - movk x13, #0xbebf, lsl #16 // .Load upper half of constant 0xbebfbc70 - add w9, w9, w16 // Add dest value - eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) - add w9, w9, w13 // Add constant 0xbebfbc70 - add w9, w9, w6 // Add aux function result - eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w9, w9, #9 // Rotate left s=23 bits - movz x10, #0x7ec6 // .Load lower half of constant 0x289b7ec6 - add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xbebfbc70, s=23, M[10]) - movk x10, #0x289b, lsl #16 // .Load upper half of constant 0x289b7ec6 - add w4, w4, w26 // Add dest value - eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) - add w4, w4, w10 // Add constant 0x289b7ec6 - add w4, w4, w6 // Add aux function result - ror w4, w4, #28 // Rotate left s=4 bits - eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x10, #0x27fa // .Load lower half of constant 0xeaa127fa - add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0x289b7ec6, s=4, M[13]) - movk x10, #0xeaa1, lsl #16 // .Load upper half of constant 0xeaa127fa - add w17, w17, w15 // Add dest value - eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) - add w17, w17, w10 // Add constant 0xeaa127fa - add w17, w17, w6 // Add aux function result - eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w17, w17, #21 // Rotate left s=11 bits - movz x13, #0x3085 // .Load lower half of constant 0xd4ef3085 - add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0xeaa127fa, s=11, M[0]) - movk x13, #0xd4ef, lsl #16 // .Load upper half of constant 0xd4ef3085 - add w8, w8, w21 // Add dest value - eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) - add w8, w8, w13 // Add constant 0xd4ef3085 - add w8, w8, w6 // Add aux function result - ror w8, w8, #16 // Rotate left s=16 bits - eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x13, #0x1d05 // .Load lower half of constant 0x4881d05 - add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0xd4ef3085, s=16, M[3]) - movk x13, #0x488, lsl #16 // .Load upper half of constant 0x4881d05 - add w9, w9, w7 // Add dest value - eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) - add w9, w9, w13 // Add constant 0x4881d05 - add w9, w9, w6 // Add aux function result - eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w9, w9, #9 // Rotate left s=23 bits - movz x10, #0xd039 // .Load lower half of constant 0xd9d4d039 - add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0x4881d05, s=23, M[6]) - movk x10, #0xd9d4, lsl #16 // .Load upper half of constant 0xd9d4d039 - add w4, w4, w24 // Add dest value - eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) - add w4, w4, w10 // Add constant 0xd9d4d039 - add w4, w4, w6 // Add aux function result - ror w4, w4, #28 // Rotate left s=4 bits - eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x10, #0x99e5 // .Load lower half of constant 0xe6db99e5 - add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xd9d4d039, s=4, M[9]) - movk x10, #0xe6db, lsl #16 // .Load upper half of constant 0xe6db99e5 - add w17, w17, w11 // Add dest value - eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) - add w17, w17, w10 // Add constant 0xe6db99e5 - add w17, w17, w6 // Add aux function result - eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) - ror w17, w17, #21 // Rotate left s=11 bits - movz x13, #0x7cf8 // .Load lower half of constant 0x1fa27cf8 - add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0xe6db99e5, s=11, M[12]) - movk x13, #0x1fa2, lsl #16 // .Load upper half of constant 0x1fa27cf8 - add w8, w8, w27 // Add dest value - eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) - add w8, w8, w13 // Add constant 0x1fa27cf8 - add w8, w8, w6 // Add aux function result - ror w8, w8, #16 // Rotate left s=16 bits - eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) - movz x13, #0x5665 // .Load lower half of constant 0xc4ac5665 - add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0x1fa27cf8, s=16, M[15]) - movk x13, #0xc4ac, lsl #16 // .Load upper half of constant 0xc4ac5665 - add w9, w9, w3 // Add dest value - eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) - add w9, w9, w13 // Add constant 0xc4ac5665 - add w9, w9, w6 // Add aux function result - ror w9, w9, #9 // Rotate left s=23 bits - movz x6, #0x2244 // .Load lower half of constant 0xf4292244 - movk x6, #0xf429, lsl #16 // .Load upper half of constant 0xf4292244 - add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xc4ac5665, s=23, M[2]) - add w4, w4, w15 // Add dest value - orn x13, x9, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w4, w4, w6 // Add constant 0xf4292244 - eor x6, x8, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w4, w4, w6 // Add aux function result - ror w4, w4, #26 // Rotate left s=6 bits - movz x6, #0xff97 // .Load lower half of constant 0x432aff97 - movk x6, #0x432a, lsl #16 // .Load upper half of constant 0x432aff97 - add w4, w9, w4 // Add X parameter round 4 A=II(A, B, C, D, 0xf4292244, s=6, M[0]) - orn x10, x4, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w17, w17, w23 // Add dest value - eor x10, x9, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w17, w17, w6 // Add constant 0x432aff97 - add w6, w17, w10 // Add aux function result - ror w6, w6, #22 // Rotate left s=10 bits - movz x17, #0x23a7 // .Load lower half of constant 0xab9423a7 - movk x17, #0xab94, lsl #16 // .Load upper half of constant 0xab9423a7 - add w6, w4, w6 // Add X parameter round 4 D=II(D, A, B, C, 0x432aff97, s=10, M[7]) - add w8, w8, w12 // Add dest value - orn x10, x6, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w8, w17 // Add constant 0xab9423a7 - eor x17, x4, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w8, w17 // Add aux function result - ror w8, w8, #17 // Rotate left s=15 bits - movz x17, #0xa039 // .Load lower half of constant 0xfc93a039 - movk x17, #0xfc93, lsl #16 // .Load upper half of constant 0xfc93a039 - add w8, w6, w8 // Add X parameter round 4 C=II(C, D, A, B, 0xab9423a7, s=15, M[14]) - orn x13, x8, x4 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w9, w22 // Add dest value - eor x13, x6, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w9, w17 // Add constant 0xfc93a039 - add w17, w9, w13 // Add aux function result - ror w17, w17, #11 // Rotate left s=21 bits - movz x9, #0x59c3 // .Load lower half of constant 0x655b59c3 - movk x9, #0x655b, lsl #16 // .Load upper half of constant 0x655b59c3 - add w17, w8, w17 // Add X parameter round 4 B=II(B, C, D, A, 0xfc93a039, s=21, M[5]) - add w4, w4, w11 // Add dest value - orn x13, x17, x6 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w4, w9 // Add constant 0x655b59c3 - eor x4, x8, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w9, w4 // Add aux function result - ror w9, w9, #26 // Rotate left s=6 bits - movz x4, #0xcc92 // .Load lower half of constant 0x8f0ccc92 - movk x4, #0x8f0c, lsl #16 // .Load upper half of constant 0x8f0ccc92 - add w9, w17, w9 // Add X parameter round 4 A=II(A, B, C, D, 0x655b59c3, s=6, M[12]) - orn x10, x9, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w6, w6, w21 // Add dest value - eor x10, x17, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w4, w6, w4 // Add constant 0x8f0ccc92 - add w6, w4, w10 // Add aux function result - ror w6, w6, #22 // Rotate left s=10 bits - movz x4, #0xf47d // .Load lower half of constant 0xffeff47d - movk x4, #0xffef, lsl #16 // .Load upper half of constant 0xffeff47d - add w6, w9, w6 // Add X parameter round 4 D=II(D, A, B, C, 0x8f0ccc92, s=10, M[3]) - add w8, w8, w16 // Add dest value - orn x10, x6, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w8, w4 // Add constant 0xffeff47d - eor x4, x9, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w8, w4 // Add aux function result - ror w8, w8, #17 // Rotate left s=15 bits - movz x4, #0x5dd1 // .Load lower half of constant 0x85845dd1 - movk x4, #0x8584, lsl #16 // .Load upper half of constant 0x85845dd1 - add w8, w6, w8 // Add X parameter round 4 C=II(C, D, A, B, 0xffeff47d, s=15, M[10]) - orn x10, x8, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w17, w20 // Add dest value - eor x17, x6, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w15, w4 // Add constant 0x85845dd1 - add w4, w15, w17 // Add aux function result - ror w4, w4, #11 // Rotate left s=21 bits - movz x15, #0x7e4f // .Load lower half of constant 0x6fa87e4f - movk x15, #0x6fa8, lsl #16 // .Load upper half of constant 0x6fa87e4f - add w17, w8, w4 // Add X parameter round 4 B=II(B, C, D, A, 0x85845dd1, s=21, M[1]) - add w4, w9, w5 // Add dest value - orn x9, x17, x6 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w4, w15 // Add constant 0x6fa87e4f - eor x4, x8, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w15, w4 // Add aux function result - ror w9, w9, #26 // Rotate left s=6 bits - movz x15, #0xe6e0 // .Load lower half of constant 0xfe2ce6e0 - movk x15, #0xfe2c, lsl #16 // .Load upper half of constant 0xfe2ce6e0 - add w4, w17, w9 // Add X parameter round 4 A=II(A, B, C, D, 0x6fa87e4f, s=6, M[8]) - orn x9, x4, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w6, w6, w27 // Add dest value - eor x9, x17, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w6, w15 // Add constant 0xfe2ce6e0 - add w6, w15, w9 // Add aux function result - ror w6, w6, #22 // Rotate left s=10 bits - movz x9, #0x4314 // .Load lower half of constant 0xa3014314 - movk x9, #0xa301, lsl #16 // .Load upper half of constant 0xa3014314 - add w15, w4, w6 // Add X parameter round 4 D=II(D, A, B, C, 0xfe2ce6e0, s=10, M[15]) - add w6, w8, w7 // Add dest value - orn x7, x15, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w6, w9 // Add constant 0xa3014314 - eor x9, x4, x7 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w6, w8, w9 // Add aux function result - ror w6, w6, #17 // Rotate left s=15 bits - movz x7, #0x11a1 // .Load lower half of constant 0x4e0811a1 - movk x7, #0x4e08, lsl #16 // .Load upper half of constant 0x4e0811a1 - add w8, w15, w6 // Add X parameter round 4 C=II(C, D, A, B, 0xa3014314, s=15, M[6]) - orn x9, x8, x4 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w6, w17, w26 // Add dest value - eor x17, x15, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w9, w6, w7 // Add constant 0x4e0811a1 - add w7, w9, w17 // Add aux function result - ror w7, w7, #11 // Rotate left s=21 bits - movz x6, #0x7e82 // .Load lower half of constant 0xf7537e82 - movk x6, #0xf753, lsl #16 // .Load upper half of constant 0xf7537e82 - add w9, w8, w7 // Add X parameter round 4 B=II(B, C, D, A, 0x4e0811a1, s=21, M[13]) - add w17, w4, w14 // Add dest value - orn x7, x9, x15 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w14, w17, w6 // Add constant 0xf7537e82 - eor x4, x8, x7 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w17, w14, w4 // Add aux function result - ror w17, w17, #26 // Rotate left s=6 bits - movz x6, #0xf235 // .Load lower half of constant 0xbd3af235 - movk x6, #0xbd3a, lsl #16 // .Load upper half of constant 0xbd3af235 - add w7, w9, w17 // Add X parameter round 4 A=II(A, B, C, D, 0xf7537e82, s=6, M[4]) - orn x14, x7, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w4, w15, w25 // Add dest value - eor x17, x9, x14 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w4, w6 // Add constant 0xbd3af235 - add w16, w15, w17 // Add aux function result - ror w16, w16, #22 // Rotate left s=10 bits - movz x14, #0xd2bb // .Load lower half of constant 0x2ad7d2bb - movk x14, #0x2ad7, lsl #16 // .Load upper half of constant 0x2ad7d2bb - add w4, w7, w16 // Add X parameter round 4 D=II(D, A, B, C, 0xbd3af235, s=10, M[11]) - add w6, w8, w3 // Add dest value - orn x15, x4, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w17, w6, w14 // Add constant 0x2ad7d2bb - eor x16, x7, x15 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w8, w17, w16 // Add aux function result - ror w8, w8, #17 // Rotate left s=15 bits - movz x3, #0xd391 // .Load lower half of constant 0xeb86d391 - movk x3, #0xeb86, lsl #16 // .Load upper half of constant 0xeb86d391 - add w14, w4, w8 // Add X parameter round 4 C=II(C, D, A, B, 0x2ad7d2bb, s=15, M[2]) - orn x6, x14, x7 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) - add w15, w9, w24 // Add dest value - eor x17, x4, x6 // End aux function round 4 I(x,y,z)=((~z|x)^y) - add w16, w15, w3 // Add constant 0xeb86d391 - add w8, w16, w17 // Add aux function result - ror w8, w8, #11 // Rotate left s=21 bits - ldp w6, w15, [x0] // Reload MD5 state->A and state->B - ldp w5, w9, [x0, #8] // Reload MD5 state->C and state->D - add w3, w14, w8 // Add X parameter round 4 B=II(B, C, D, A, 0xeb86d391, s=21, M[9]) - add w13, w4, w9 // Add result of MD5 rounds to state->D - add w12, w14, w5 // Add result of MD5 rounds to state->C - add w10, w7, w6 // Add result of MD5 rounds to state->A - add w11, w3, w15 // Add result of MD5 rounds to state->B - stp w12, w13, [x0, #8] // Store MD5 states C,D - stp w10, w11, [x0] // Store MD5 states A,B - add x1, x1, #64 // Increment data pointer - subs w2, w2, #1 // Decrement block counter - b.ne ossl_md5_blocks_loop - - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - ldp x25,x26,[sp,#48] - ldp x27,x28,[sp,#64] - ldp x19,x20,[sp],#80 - ret - diff --git a/openssl/src/crypto/md5/gen/linux_ia32/md5-586.S b/openssl/src/crypto/md5/gen/linux_ia32/md5-586.S index 7e96dc647..5814f893a 100644 --- a/openssl/src/crypto/md5/gen/linux_ia32/md5-586.S +++ b/openssl/src/crypto/md5/gen/linux_ia32/md5-586.S @@ -4,11 +4,7 @@ .align 16 ossl_md5_block_asm_data_order: .L_ossl_md5_block_asm_data_order_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %esi pushl %edi movl 12(%esp),%edi diff --git a/openssl/src/crypto/md5/gen/linux_loong64/md5-loongarch64.S b/openssl/src/crypto/md5/gen/linux_loong64/md5-loongarch64.S deleted file mode 100644 index 4b1192dd9..000000000 --- a/openssl/src/crypto/md5/gen/linux_loong64/md5-loongarch64.S +++ /dev/null @@ -1,772 +0,0 @@ -.text - -.globl ossl_md5_block_asm_data_order -.type ossl_md5_block_asm_data_order function -ossl_md5_block_asm_data_order: - # $r4 = arg #1 (ctx, MD5_CTX pointer) - # $r5 = arg #2 (ptr, data pointer) - # $r6 = arg #3 (nbr, number of 16-word blocks to process) - beqz $r6,.Lend # cmp nbr with 0, jmp if nbr == 0 - - # ptr is '$r5' - # end is '$r7' - slli.d $r12,$r6,6 - add.d $r7,$r5,$r12 - - # A is '$r8' - # B is '$r9' - # C is '$r10' - # D is '$r11' - ld.w $r8,$r4,0 # a4 = ctx->A - ld.w $r9,$r4,4 # a5 = ctx->B - ld.w $r10,$r4,8 # a6 = ctx->C - ld.w $r11,$r4,12 # a7 = ctx->D - -# BEGIN of loop over 16-word blocks -.align 6 -.Lloop: - # save old values of A, B, C, D - move $r15,$r8 - move $r16,$r9 - move $r17,$r10 - move $r18,$r11 - - preld 0,$r5,0 - preld 0,$r5,64 - ld.w $r12,$r5,0 /* (NEXT STEP) X[0] */ - xor $r13,$r10,$r11 /* y ^ z */ - add.w $r14,$r8,$r12 /* dst + X[k] */ - lu12i.w $r20,-166230 /* load bits [31:12] of constant */ - and $r13,$r9,$r13 /* x & ... */ - ori $r20,$r20,1144 /* load bits [11:0] of constant */ - xor $r13,$r11,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,1*4 /* (NEXT STEP) X[1] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[1] */ - rotri.w $r8,$r8,32-7 /* dst <<< s */ - xor $r13,$r9,$r10 /* (NEXT STEP) y ^ z */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-95109 /* load bits [31:12] of constant */ - and $r13,$r8,$r13 /* x & ... */ - ori $r20,$r20,1878 /* load bits [11:0] of constant */ - xor $r13,$r10,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,2*4 /* (NEXT STEP) X[2] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[2] */ - rotri.w $r11,$r11,32-12 /* dst <<< s */ - xor $r13,$r8,$r9 /* (NEXT STEP) y ^ z */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,147975 /* load bits [31:12] of constant */ - and $r13,$r11,$r13 /* x & ... */ - ori $r20,$r20,219 /* load bits [11:0] of constant */ - xor $r13,$r9,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,3*4 /* (NEXT STEP) X[3] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[3] */ - rotri.w $r10,$r10,32-17 /* dst <<< s */ - xor $r13,$r11,$r8 /* (NEXT STEP) y ^ z */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-255012 /* load bits [31:12] of constant */ - and $r13,$r10,$r13 /* x & ... */ - ori $r20,$r20,3822 /* load bits [11:0] of constant */ - xor $r13,$r8,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,4*4 /* (NEXT STEP) X[4] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[4] */ - rotri.w $r9,$r9,32-22 /* dst <<< s */ - xor $r13,$r10,$r11 /* (NEXT STEP) y ^ z */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-43072 /* load bits [31:12] of constant */ - and $r13,$r9,$r13 /* x & ... */ - ori $r20,$r20,4015 /* load bits [11:0] of constant */ - xor $r13,$r11,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,5*4 /* (NEXT STEP) X[5] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[5] */ - rotri.w $r8,$r8,32-7 /* dst <<< s */ - xor $r13,$r9,$r10 /* (NEXT STEP) y ^ z */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,292988 /* load bits [31:12] of constant */ - and $r13,$r8,$r13 /* x & ... */ - ori $r20,$r20,1578 /* load bits [11:0] of constant */ - xor $r13,$r10,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,6*4 /* (NEXT STEP) X[6] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[6] */ - rotri.w $r11,$r11,32-12 /* dst <<< s */ - xor $r13,$r8,$r9 /* (NEXT STEP) y ^ z */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-359676 /* load bits [31:12] of constant */ - and $r13,$r11,$r13 /* x & ... */ - ori $r20,$r20,1555 /* load bits [11:0] of constant */ - xor $r13,$r9,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,7*4 /* (NEXT STEP) X[7] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[7] */ - rotri.w $r10,$r10,32-17 /* dst <<< s */ - xor $r13,$r11,$r8 /* (NEXT STEP) y ^ z */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-11159 /* load bits [31:12] of constant */ - and $r13,$r10,$r13 /* x & ... */ - ori $r20,$r20,1281 /* load bits [11:0] of constant */ - xor $r13,$r8,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,8*4 /* (NEXT STEP) X[8] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[8] */ - rotri.w $r9,$r9,32-22 /* dst <<< s */ - xor $r13,$r10,$r11 /* (NEXT STEP) y ^ z */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,432137 /* load bits [31:12] of constant */ - and $r13,$r9,$r13 /* x & ... */ - ori $r20,$r20,2264 /* load bits [11:0] of constant */ - xor $r13,$r11,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,9*4 /* (NEXT STEP) X[9] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[9] */ - rotri.w $r8,$r8,32-7 /* dst <<< s */ - xor $r13,$r9,$r10 /* (NEXT STEP) y ^ z */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-478129 /* load bits [31:12] of constant */ - and $r13,$r8,$r13 /* x & ... */ - ori $r20,$r20,1967 /* load bits [11:0] of constant */ - xor $r13,$r10,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,10*4 /* (NEXT STEP) X[10] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[10] */ - rotri.w $r11,$r11,32-12 /* dst <<< s */ - xor $r13,$r8,$r9 /* (NEXT STEP) y ^ z */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-11 /* load bits [31:12] of constant */ - and $r13,$r11,$r13 /* x & ... */ - ori $r20,$r20,2993 /* load bits [11:0] of constant */ - xor $r13,$r9,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,11*4 /* (NEXT STEP) X[11] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[11] */ - rotri.w $r10,$r10,32-17 /* dst <<< s */ - xor $r13,$r11,$r8 /* (NEXT STEP) y ^ z */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-485939 /* load bits [31:12] of constant */ - and $r13,$r10,$r13 /* x & ... */ - ori $r20,$r20,1982 /* load bits [11:0] of constant */ - xor $r13,$r8,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,12*4 /* (NEXT STEP) X[12] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[12] */ - rotri.w $r9,$r9,32-22 /* dst <<< s */ - xor $r13,$r10,$r11 /* (NEXT STEP) y ^ z */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,440577 /* load bits [31:12] of constant */ - and $r13,$r9,$r13 /* x & ... */ - ori $r20,$r20,290 /* load bits [11:0] of constant */ - xor $r13,$r11,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,13*4 /* (NEXT STEP) X[13] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[13] */ - rotri.w $r8,$r8,32-7 /* dst <<< s */ - xor $r13,$r9,$r10 /* (NEXT STEP) y ^ z */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-9849 /* load bits [31:12] of constant */ - and $r13,$r8,$r13 /* x & ... */ - ori $r20,$r20,403 /* load bits [11:0] of constant */ - xor $r13,$r10,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,14*4 /* (NEXT STEP) X[14] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[14] */ - rotri.w $r11,$r11,32-12 /* dst <<< s */ - xor $r13,$r8,$r9 /* (NEXT STEP) y ^ z */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-366700 /* load bits [31:12] of constant */ - and $r13,$r11,$r13 /* x & ... */ - ori $r20,$r20,910 /* load bits [11:0] of constant */ - xor $r13,$r9,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,15*4 /* (NEXT STEP) X[15] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[15] */ - rotri.w $r10,$r10,32-17 /* dst <<< s */ - xor $r13,$r11,$r8 /* (NEXT STEP) y ^ z */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,301888 /* load bits [31:12] of constant */ - and $r13,$r10,$r13 /* x & ... */ - ori $r20,$r20,2081 /* load bits [11:0] of constant */ - xor $r13,$r8,$r13 /* z ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,1*4 /* (NEXT STEP) X[1] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[1] */ - rotri.w $r9,$r9,32-22 /* dst <<< s */ - move $r12,$r11 /* (NEXT ROUND) $r12 = z' (copy of z) */ - nor $r13,$r0,$r11 /* (NEXT ROUND) $r13 = not z' (copy of not z) */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-40478 /* load bits [31:12] of Constant */ - and $r12,$r9,$r12 /* x & z */ - ori $r20,$r20,1378 /* load bits [11:0] of Constant */ - and $r13,$r10,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,6*4 /* (NEXT STEP) X[6] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[6] */ - rotri.w $r8,$r8,32-5 /* dst <<< s */ - move $r12,$r10 /* (NEXT STEP) z' = $r10 */ - nor $r13,$r0,$r10 /* (NEXT STEP) not z' = not $r10 */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-261109 /* load bits [31:12] of Constant */ - and $r12,$r8,$r12 /* x & z */ - ori $r20,$r20,832 /* load bits [11:0] of Constant */ - and $r13,$r9,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,11*4 /* (NEXT STEP) X[11] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[11] */ - rotri.w $r11,$r11,32-9 /* dst <<< s */ - move $r12,$r9 /* (NEXT STEP) z' = $r9 */ - nor $r13,$r0,$r9 /* (NEXT STEP) not z' = not $r9 */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,157157 /* load bits [31:12] of Constant */ - and $r12,$r11,$r12 /* x & z */ - ori $r20,$r20,2641 /* load bits [11:0] of Constant */ - and $r13,$r8,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,0*4 /* (NEXT STEP) X[0] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[0] */ - rotri.w $r10,$r10,32-14 /* dst <<< s */ - move $r12,$r8 /* (NEXT STEP) z' = $r8 */ - nor $r13,$r0,$r8 /* (NEXT STEP) not z' = not $r8 */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-91284 /* load bits [31:12] of Constant */ - and $r12,$r10,$r12 /* x & z */ - ori $r20,$r20,1962 /* load bits [11:0] of Constant */ - and $r13,$r11,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,5*4 /* (NEXT STEP) X[5] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[5] */ - rotri.w $r9,$r9,32-20 /* dst <<< s */ - move $r12,$r11 /* (NEXT STEP) z' = $r11 */ - nor $r13,$r0,$r11 /* (NEXT STEP) not z' = not $r11 */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-171279 /* load bits [31:12] of Constant */ - and $r12,$r9,$r12 /* x & z */ - ori $r20,$r20,93 /* load bits [11:0] of Constant */ - and $r13,$r10,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,10*4 /* (NEXT STEP) X[10] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[10] */ - rotri.w $r8,$r8,32-5 /* dst <<< s */ - move $r12,$r10 /* (NEXT STEP) z' = $r10 */ - nor $r13,$r0,$r10 /* (NEXT STEP) not z' = not $r10 */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,9281 /* load bits [31:12] of Constant */ - and $r12,$r8,$r12 /* x & z */ - ori $r20,$r20,1107 /* load bits [11:0] of Constant */ - and $r13,$r9,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,15*4 /* (NEXT STEP) X[15] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[15] */ - rotri.w $r11,$r11,32-9 /* dst <<< s */ - move $r12,$r9 /* (NEXT STEP) z' = $r9 */ - nor $r13,$r0,$r9 /* (NEXT STEP) not z' = not $r9 */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-161250 /* load bits [31:12] of Constant */ - and $r12,$r11,$r12 /* x & z */ - ori $r20,$r20,1665 /* load bits [11:0] of Constant */ - and $r13,$r8,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,4*4 /* (NEXT STEP) X[4] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[4] */ - rotri.w $r10,$r10,32-14 /* dst <<< s */ - move $r12,$r8 /* (NEXT STEP) z' = $r8 */ - nor $r13,$r0,$r8 /* (NEXT STEP) not z' = not $r8 */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-99009 /* load bits [31:12] of Constant */ - and $r12,$r10,$r12 /* x & z */ - ori $r20,$r20,3016 /* load bits [11:0] of Constant */ - and $r13,$r11,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,9*4 /* (NEXT STEP) X[9] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[9] */ - rotri.w $r9,$r9,32-20 /* dst <<< s */ - move $r12,$r11 /* (NEXT STEP) z' = $r11 */ - nor $r13,$r0,$r11 /* (NEXT STEP) not z' = not $r11 */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,138780 /* load bits [31:12] of Constant */ - and $r12,$r9,$r12 /* x & z */ - ori $r20,$r20,3558 /* load bits [11:0] of Constant */ - and $r13,$r10,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,14*4 /* (NEXT STEP) X[14] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[14] */ - rotri.w $r8,$r8,32-5 /* dst <<< s */ - move $r12,$r10 /* (NEXT STEP) z' = $r10 */ - nor $r13,$r0,$r10 /* (NEXT STEP) not z' = not $r10 */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-248976 /* load bits [31:12] of Constant */ - and $r12,$r8,$r12 /* x & z */ - ori $r20,$r20,2006 /* load bits [11:0] of Constant */ - and $r13,$r9,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,3*4 /* (NEXT STEP) X[3] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[3] */ - rotri.w $r11,$r11,32-9 /* dst <<< s */ - move $r12,$r9 /* (NEXT STEP) z' = $r9 */ - nor $r13,$r0,$r9 /* (NEXT STEP) not z' = not $r9 */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-45744 /* load bits [31:12] of Constant */ - and $r12,$r11,$r12 /* x & z */ - ori $r20,$r20,3463 /* load bits [11:0] of Constant */ - and $r13,$r8,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,8*4 /* (NEXT STEP) X[8] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[8] */ - rotri.w $r10,$r10,32-14 /* dst <<< s */ - move $r12,$r8 /* (NEXT STEP) z' = $r8 */ - nor $r13,$r0,$r8 /* (NEXT STEP) not z' = not $r8 */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,284065 /* load bits [31:12] of Constant */ - and $r12,$r10,$r12 /* x & z */ - ori $r20,$r20,1261 /* load bits [11:0] of Constant */ - and $r13,$r11,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,13*4 /* (NEXT STEP) X[13] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[13] */ - rotri.w $r9,$r9,32-20 /* dst <<< s */ - move $r12,$r11 /* (NEXT STEP) z' = $r11 */ - nor $r13,$r0,$r11 /* (NEXT STEP) not z' = not $r11 */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-352706 /* load bits [31:12] of Constant */ - and $r12,$r9,$r12 /* x & z */ - ori $r20,$r20,2309 /* load bits [11:0] of Constant */ - and $r13,$r10,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,2*4 /* (NEXT STEP) X[2] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[2] */ - rotri.w $r8,$r8,32-5 /* dst <<< s */ - move $r12,$r10 /* (NEXT STEP) z' = $r10 */ - nor $r13,$r0,$r10 /* (NEXT STEP) not z' = not $r10 */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-12550 /* load bits [31:12] of Constant */ - and $r12,$r8,$r12 /* x & z */ - ori $r20,$r20,1016 /* load bits [11:0] of Constant */ - and $r13,$r9,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,7*4 /* (NEXT STEP) X[7] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[7] */ - rotri.w $r11,$r11,32-9 /* dst <<< s */ - move $r12,$r9 /* (NEXT STEP) z' = $r9 */ - nor $r13,$r0,$r9 /* (NEXT STEP) not z' = not $r9 */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,423664 /* load bits [31:12] of Constant */ - and $r12,$r11,$r12 /* x & z */ - ori $r20,$r20,729 /* load bits [11:0] of Constant */ - and $r13,$r8,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,12*4 /* (NEXT STEP) X[12] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[12] */ - rotri.w $r10,$r10,32-14 /* dst <<< s */ - move $r12,$r8 /* (NEXT STEP) z' = $r8 */ - nor $r13,$r0,$r8 /* (NEXT STEP) not z' = not $r8 */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-470364 /* load bits [31:12] of Constant */ - and $r12,$r10,$r12 /* x & z */ - ori $r20,$r20,3210 /* load bits [11:0] of Constant */ - and $r13,$r11,$r13 /* y & (not z) */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - or $r13,$r12,$r13 /* (y & (not z)) | (x & z) */ - ld.w $r12,$r5,5*4 /* (NEXT STEP) X[5] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[5] */ - rotri.w $r9,$r9,32-20 /* dst <<< s */ - xor $r13,$r10,$r11 /* (NEXT ROUND) $r13 = y ^ z */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-93 /* load bits [31:12] of Constant */ - xor $r13,$r9,$r13 /* x ^ ... */ - ori $r20,$r20,2370 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,8*4 /* (NEXT STEP) X[8] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[8] */ - rotri.w $r8,$r8,32-4 /* dst <<< s */ - xor $r13,$r9,$r10 /* (NEXT STEP) y ^ z */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-493793 /* load bits [31:12] of Constant */ - xor $r13,$r8,$r13 /* x ^ ... */ - ori $r20,$r20,1665 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,11*4 /* (NEXT STEP) X[11] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[11] */ - rotri.w $r11,$r11,32-11 /* dst <<< s */ - xor $r13,$r8,$r9 /* (NEXT STEP) y ^ z */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,448982 /* load bits [31:12] of Constant */ - xor $r13,$r11,$r13 /* x ^ ... */ - ori $r20,$r20,290 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,14*4 /* (NEXT STEP) X[14] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[14] */ - rotri.w $r10,$r10,32-16 /* dst <<< s */ - xor $r13,$r11,$r8 /* (NEXT STEP) y ^ z */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-8621 /* load bits [31:12] of Constant */ - xor $r13,$r10,$r13 /* x ^ ... */ - ori $r20,$r20,2060 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,1*4 /* (NEXT STEP) X[1] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[1] */ - rotri.w $r9,$r9,32-23 /* dst <<< s */ - xor $r13,$r10,$r11 /* (NEXT STEP) y ^ z */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-373778 /* load bits [31:12] of Constant */ - xor $r13,$r9,$r13 /* x ^ ... */ - ori $r20,$r20,2628 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,4*4 /* (NEXT STEP) X[4] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[4] */ - rotri.w $r8,$r8,32-4 /* dst <<< s */ - xor $r13,$r9,$r10 /* (NEXT STEP) y ^ z */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,310764 /* load bits [31:12] of Constant */ - xor $r13,$r8,$r13 /* x ^ ... */ - ori $r20,$r20,4009 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,7*4 /* (NEXT STEP) X[7] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[7] */ - rotri.w $r11,$r11,32-11 /* dst <<< s */ - xor $r13,$r8,$r9 /* (NEXT STEP) y ^ z */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-37964 /* load bits [31:12] of Constant */ - xor $r13,$r11,$r13 /* x ^ ... */ - ori $r20,$r20,2912 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,10*4 /* (NEXT STEP) X[10] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[10] */ - rotri.w $r10,$r10,32-16 /* dst <<< s */ - xor $r13,$r11,$r8 /* (NEXT STEP) y ^ z */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-267269 /* load bits [31:12] of Constant */ - xor $r13,$r10,$r13 /* x ^ ... */ - ori $r20,$r20,3184 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,13*4 /* (NEXT STEP) X[13] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[13] */ - rotri.w $r9,$r9,32-23 /* dst <<< s */ - xor $r13,$r10,$r11 /* (NEXT STEP) y ^ z */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,166327 /* load bits [31:12] of Constant */ - xor $r13,$r9,$r13 /* x ^ ... */ - ori $r20,$r20,3782 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,0*4 /* (NEXT STEP) X[0] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[0] */ - rotri.w $r8,$r8,32-4 /* dst <<< s */ - xor $r13,$r9,$r10 /* (NEXT STEP) y ^ z */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-87534 /* load bits [31:12] of Constant */ - xor $r13,$r8,$r13 /* x ^ ... */ - ori $r20,$r20,2042 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,3*4 /* (NEXT STEP) X[3] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[3] */ - rotri.w $r11,$r11,32-11 /* dst <<< s */ - xor $r13,$r8,$r9 /* (NEXT STEP) y ^ z */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-176397 /* load bits [31:12] of Constant */ - xor $r13,$r11,$r13 /* x ^ ... */ - ori $r20,$r20,133 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,6*4 /* (NEXT STEP) X[6] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[6] */ - rotri.w $r10,$r10,32-16 /* dst <<< s */ - xor $r13,$r11,$r8 /* (NEXT STEP) y ^ z */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,18561 /* load bits [31:12] of Constant */ - xor $r13,$r10,$r13 /* x ^ ... */ - ori $r20,$r20,3333 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,9*4 /* (NEXT STEP) X[9] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[9] */ - rotri.w $r9,$r9,32-23 /* dst <<< s */ - xor $r13,$r10,$r11 /* (NEXT STEP) y ^ z */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-156339 /* load bits [31:12] of Constant */ - xor $r13,$r9,$r13 /* x ^ ... */ - ori $r20,$r20,57 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,12*4 /* (NEXT STEP) X[12] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[12] */ - rotri.w $r8,$r8,32-4 /* dst <<< s */ - xor $r13,$r9,$r10 /* (NEXT STEP) y ^ z */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-102983 /* load bits [31:12] of Constant */ - xor $r13,$r8,$r13 /* x ^ ... */ - ori $r20,$r20,2533 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,15*4 /* (NEXT STEP) X[15] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[15] */ - rotri.w $r11,$r11,32-11 /* dst <<< s */ - xor $r13,$r8,$r9 /* (NEXT STEP) y ^ z */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,129575 /* load bits [31:12] of Constant */ - xor $r13,$r11,$r13 /* x ^ ... */ - ori $r20,$r20,3320 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,2*4 /* (NEXT STEP) X[2] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[2] */ - rotri.w $r10,$r10,32-16 /* dst <<< s */ - xor $r13,$r11,$r8 /* (NEXT STEP) y ^ z */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-243003 /* load bits [31:12] of Constant */ - xor $r13,$r10,$r13 /* x ^ ... */ - ori $r20,$r20,1637 /* load bits [11:0] of Constant */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,0*4 /* (NEXT STEP) X[0] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[0] */ - rotri.w $r9,$r9,32-23 /* dst <<< s */ - nor $r13,$r0,$r11 /* (NEXT ROUND) $r13 = not z */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-48494 /* load bits [31:12] of Constant */ - or $r13,$r9,$r13 /* x | ... */ - ori $r20,$r20,580 /* load bits [11:0] of Constant */ - xor $r13,$r10,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,7*4 /* (NEXT STEP) X[7] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[7] */ - rotri.w $r8,$r8,32-6 /* dst <<< s */ - nor $r13,$r0,$r10 /* (NEXT STEP) not z' = not $r10 */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,275119 /* load bits [31:12] of Constant */ - or $r13,$r8,$r13 /* x | ... */ - ori $r20,$r20,3991 /* load bits [11:0] of Constant */ - xor $r13,$r9,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,14*4 /* (NEXT STEP) X[14] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[14] */ - rotri.w $r11,$r11,32-10 /* dst <<< s */ - nor $r13,$r0,$r9 /* (NEXT STEP) not z' = not $r9 */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-345790 /* load bits [31:12] of Constant */ - or $r13,$r11,$r13 /* x | ... */ - ori $r20,$r20,935 /* load bits [11:0] of Constant */ - xor $r13,$r8,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,5*4 /* (NEXT STEP) X[5] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[5] */ - rotri.w $r10,$r10,32-15 /* dst <<< s */ - nor $r13,$r0,$r8 /* (NEXT STEP) not z' = not $r8 */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-14022 /* load bits [31:12] of Constant */ - or $r13,$r10,$r13 /* x | ... */ - ori $r20,$r20,57 /* load bits [11:0] of Constant */ - xor $r13,$r11,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,12*4 /* (NEXT STEP) X[12] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[12] */ - rotri.w $r9,$r9,32-21 /* dst <<< s */ - nor $r13,$r0,$r11 /* (NEXT STEP) not z' = not $r11 */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,415157 /* load bits [31:12] of Constant */ - or $r13,$r9,$r13 /* x | ... */ - ori $r20,$r20,2499 /* load bits [11:0] of Constant */ - xor $r13,$r10,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,3*4 /* (NEXT STEP) X[3] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[3] */ - rotri.w $r8,$r8,32-6 /* dst <<< s */ - nor $r13,$r0,$r10 /* (NEXT STEP) not z' = not $r10 */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-462644 /* load bits [31:12] of Constant */ - or $r13,$r8,$r13 /* x | ... */ - ori $r20,$r20,3218 /* load bits [11:0] of Constant */ - xor $r13,$r9,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,10*4 /* (NEXT STEP) X[10] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[10] */ - rotri.w $r11,$r11,32-10 /* dst <<< s */ - nor $r13,$r0,$r9 /* (NEXT STEP) not z' = not $r9 */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-257 /* load bits [31:12] of Constant */ - or $r13,$r11,$r13 /* x | ... */ - ori $r20,$r20,1149 /* load bits [11:0] of Constant */ - xor $r13,$r8,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,1*4 /* (NEXT STEP) X[1] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[1] */ - rotri.w $r10,$r10,32-15 /* dst <<< s */ - nor $r13,$r0,$r8 /* (NEXT STEP) not z' = not $r8 */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-501691 /* load bits [31:12] of Constant */ - or $r13,$r10,$r13 /* x | ... */ - ori $r20,$r20,3537 /* load bits [11:0] of Constant */ - xor $r13,$r11,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,8*4 /* (NEXT STEP) X[8] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[8] */ - rotri.w $r9,$r9,32-21 /* dst <<< s */ - nor $r13,$r0,$r11 /* (NEXT STEP) not z' = not $r11 */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,457351 /* load bits [31:12] of Constant */ - or $r13,$r9,$r13 /* x | ... */ - ori $r20,$r20,3663 /* load bits [11:0] of Constant */ - xor $r13,$r10,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,15*4 /* (NEXT STEP) X[15] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[15] */ - rotri.w $r8,$r8,32-6 /* dst <<< s */ - nor $r13,$r0,$r10 /* (NEXT STEP) not z' = not $r10 */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-7474 /* load bits [31:12] of Constant */ - or $r13,$r8,$r13 /* x | ... */ - ori $r20,$r20,1760 /* load bits [11:0] of Constant */ - xor $r13,$r9,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,6*4 /* (NEXT STEP) X[6] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[6] */ - rotri.w $r11,$r11,32-10 /* dst <<< s */ - nor $r13,$r0,$r9 /* (NEXT STEP) not z' = not $r9 */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,-380908 /* load bits [31:12] of Constant */ - or $r13,$r11,$r13 /* x | ... */ - ori $r20,$r20,788 /* load bits [11:0] of Constant */ - xor $r13,$r8,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,13*4 /* (NEXT STEP) X[13] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[13] */ - rotri.w $r10,$r10,32-15 /* dst <<< s */ - nor $r13,$r0,$r8 /* (NEXT STEP) not z' = not $r8 */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,319617 /* load bits [31:12] of Constant */ - or $r13,$r10,$r13 /* x | ... */ - ori $r20,$r20,417 /* load bits [11:0] of Constant */ - xor $r13,$r11,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,4*4 /* (NEXT STEP) X[4] */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r14,$r8,$r12 /* (NEXT STEP) dst + X[4] */ - rotri.w $r9,$r9,32-21 /* dst <<< s */ - nor $r13,$r0,$r11 /* (NEXT STEP) not z' = not $r11 */ - add.w $r9,$r9,$r10 /* dst += x */ - lu12i.w $r20,-35529 /* load bits [31:12] of Constant */ - or $r13,$r9,$r13 /* x | ... */ - ori $r20,$r20,3714 /* load bits [11:0] of Constant */ - xor $r13,$r10,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,11*4 /* (NEXT STEP) X[11] */ - add.w $r8,$r19,$r13 /* dst += ... */ - add.w $r14,$r11,$r12 /* (NEXT STEP) dst + X[11] */ - rotri.w $r8,$r8,32-6 /* dst <<< s */ - nor $r13,$r0,$r10 /* (NEXT STEP) not z' = not $r10 */ - add.w $r8,$r8,$r9 /* dst += x */ - lu12i.w $r20,-273489 /* load bits [31:12] of Constant */ - or $r13,$r8,$r13 /* x | ... */ - ori $r20,$r20,565 /* load bits [11:0] of Constant */ - xor $r13,$r9,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,2*4 /* (NEXT STEP) X[2] */ - add.w $r11,$r19,$r13 /* dst += ... */ - add.w $r14,$r10,$r12 /* (NEXT STEP) dst + X[2] */ - rotri.w $r11,$r11,32-10 /* dst <<< s */ - nor $r13,$r0,$r9 /* (NEXT STEP) not z' = not $r9 */ - add.w $r11,$r11,$r8 /* dst += x */ - lu12i.w $r20,175485 /* load bits [31:12] of Constant */ - or $r13,$r11,$r13 /* x | ... */ - ori $r20,$r20,699 /* load bits [11:0] of Constant */ - xor $r13,$r8,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - ld.w $r12,$r5,9*4 /* (NEXT STEP) X[9] */ - add.w $r10,$r19,$r13 /* dst += ... */ - add.w $r14,$r9,$r12 /* (NEXT STEP) dst + X[9] */ - rotri.w $r10,$r10,32-15 /* dst <<< s */ - nor $r13,$r0,$r8 /* (NEXT STEP) not z' = not $r8 */ - add.w $r10,$r10,$r11 /* dst += x */ - lu12i.w $r20,-83859 /* load bits [31:12] of Constant */ - or $r13,$r10,$r13 /* x | ... */ - ori $r20,$r20,913 /* load bits [11:0] of Constant */ - xor $r13,$r11,$r13 /* y ^ ... */ - add.w $r19,$r14,$r20 /* dst + X[k] + Const */ - add.w $r8,$r15,$r8 /* (NEXT LOOP) add old value of A */ - add.w $r9,$r19,$r13 /* dst += ... */ - add.w $r11,$r18,$r11 /* (NEXT LOOP) add old value of D */ - rotri.w $r9,$r9,32-21 /* dst <<< s */ - addi.d $r5,$r5,64 /* (NEXT LOOP) ptr += 64 */ - add.w $r9,$r9,$r10 /* dst += x */ - # add old values of B, C - add.w $r9,$r16,$r9 - add.w $r10,$r17,$r10 - - bltu $r5,$r7,.Lloop # jmp if ptr < end - - st.w $r8,$r4,0 # ctx->A = A - st.w $r9,$r4,4 # ctx->B = B - st.w $r10,$r4,8 # ctx->C = C - st.w $r11,$r4,12 # ctx->D = D - -.Lend: - jr $r1 -.size ossl_md5_block_asm_data_order,.-ossl_md5_block_asm_data_order diff --git a/openssl/src/crypto/md5/gen/windows_ia32/md5-586.asm b/openssl/src/crypto/md5/gen/windows_ia32/md5-586.asm index 9dd4b8146..2f7a1fa74 100644 --- a/openssl/src/crypto/md5/gen/windows_ia32/md5-586.asm +++ b/openssl/src/crypto/md5/gen/windows_ia32/md5-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/md5/md5_dgst.c b/openssl/src/crypto/md5/md5_dgst.c index 913b1ea5c..0df344b27 100644 --- a/openssl/src/crypto/md5/md5_dgst.c +++ b/openssl/src/crypto/md5/md5_dgst.c @@ -36,7 +36,7 @@ int MD5_Init(MD5_CTX *c) return 1; } -#ifndef md5_block_data_order +#ifndef ASM_md5_block_data_order # ifdef X # undef X # endif diff --git a/openssl/src/crypto/md5/md5_local.h b/openssl/src/crypto/md5/md5_local.h index fab8bb9da..9b16761ce 100644 --- a/openssl/src/crypto/md5/md5_local.h +++ b/openssl/src/crypto/md5/md5_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,19 +9,24 @@ #include #include +#include #include #include #ifdef MD5_ASM # if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || \ - defined(_M_X64) || defined(__aarch64__) || \ - (defined(__loongarch__) && __loongarch_grlen == 64) + defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64) +# ifdef md5_block_data_order +# undef md5_block_data_order +# endif # define md5_block_data_order ossl_md5_block_asm_data_order +# define ASM_md5_block_data_order # elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64) +# ifdef md5_block_data_order +# undef md5_block_data_order +# endif # define md5_block_data_order ossl_md5_block_asm_data_order -# elif defined(__sparc) || defined(__sparc__) -# define md5_block_data_order ossl_md5_block_asm_data_order +# define ASM_md5_block_data_order # endif #endif diff --git a/openssl/src/crypto/mdc2/mdc2_one.c b/openssl/src/crypto/mdc2/mdc2_one.c deleted file mode 100644 index cb978263e..000000000 --- a/openssl/src/crypto/mdc2/mdc2_one.c +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD2 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "internal/cryptlib.h" -#include - -unsigned char *MDC2(const unsigned char *d, size_t n, unsigned char *md) -{ - MDC2_CTX c; - static unsigned char m[MDC2_DIGEST_LENGTH]; - - if (md == NULL) - md = m; - if (!MDC2_Init(&c)) - return NULL; - MDC2_Update(&c, d, n); - MDC2_Final(md, &c); - OPENSSL_cleanse(&c, sizeof(c)); /* security consideration */ - return md; -} diff --git a/openssl/src/crypto/mdc2/mdc2dgst.c b/openssl/src/crypto/mdc2/mdc2dgst.c deleted file mode 100644 index 607f9fc73..000000000 --- a/openssl/src/crypto/mdc2/mdc2dgst.c +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD2 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include -#include -#include - -#undef c2l -#define c2l(c,l) (l =((DES_LONG)(*((c)++))) , \ - l|=((DES_LONG)(*((c)++)))<< 8L, \ - l|=((DES_LONG)(*((c)++)))<<16L, \ - l|=((DES_LONG)(*((c)++)))<<24L) - -#undef l2c -#define l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8L)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16L)&0xff), \ - *((c)++)=(unsigned char)(((l)>>24L)&0xff)) - -static void mdc2_body(MDC2_CTX *c, const unsigned char *in, size_t len); -int MDC2_Init(MDC2_CTX *c) -{ - c->num = 0; - c->pad_type = 1; - memset(&(c->h[0]), 0x52, MDC2_BLOCK); - memset(&(c->hh[0]), 0x25, MDC2_BLOCK); - return 1; -} - -int MDC2_Update(MDC2_CTX *c, const unsigned char *in, size_t len) -{ - size_t i, j; - - i = c->num; - if (i != 0) { - if (len < MDC2_BLOCK - i) { - /* partial block */ - memcpy(&(c->data[i]), in, len); - c->num += (int)len; - return 1; - } else { - /* filled one */ - j = MDC2_BLOCK - i; - memcpy(&(c->data[i]), in, j); - len -= j; - in += j; - c->num = 0; - mdc2_body(c, &(c->data[0]), MDC2_BLOCK); - } - } - i = len & ~((size_t)MDC2_BLOCK - 1); - if (i > 0) - mdc2_body(c, in, i); - j = len - i; - if (j > 0) { - memcpy(&(c->data[0]), &(in[i]), j); - c->num = (int)j; - } - return 1; -} - -static void mdc2_body(MDC2_CTX *c, const unsigned char *in, size_t len) -{ - register DES_LONG tin0, tin1; - register DES_LONG ttin0, ttin1; - DES_LONG d[2], dd[2]; - DES_key_schedule k; - unsigned char *p; - size_t i; - - for (i = 0; i < len; i += 8) { - c2l(in, tin0); - d[0] = dd[0] = tin0; - c2l(in, tin1); - d[1] = dd[1] = tin1; - c->h[0] = (c->h[0] & 0x9f) | 0x40; - c->hh[0] = (c->hh[0] & 0x9f) | 0x20; - - DES_set_odd_parity(&c->h); - DES_set_key_unchecked(&c->h, &k); - DES_encrypt1(d, &k, 1); - - DES_set_odd_parity(&c->hh); - DES_set_key_unchecked(&c->hh, &k); - DES_encrypt1(dd, &k, 1); - - ttin0 = tin0 ^ dd[0]; - ttin1 = tin1 ^ dd[1]; - tin0 ^= d[0]; - tin1 ^= d[1]; - - p = c->h; - l2c(tin0, p); - l2c(ttin1, p); - p = c->hh; - l2c(ttin0, p); - l2c(tin1, p); - } -} - -int MDC2_Final(unsigned char *md, MDC2_CTX *c) -{ - unsigned int i; - int j; - - i = c->num; - j = c->pad_type; - if ((i > 0) || (j == 2)) { - if (j == 2) - c->data[i++] = 0x80; - memset(&(c->data[i]), 0, MDC2_BLOCK - i); - mdc2_body(c, c->data, MDC2_BLOCK); - } - memcpy(md, (char *)c->h, MDC2_BLOCK); - memcpy(&(md[MDC2_BLOCK]), (char *)c->hh, MDC2_BLOCK); - return 1; -} diff --git a/openssl/src/crypto/mem.c b/openssl/src/crypto/mem.c index eef116570..4ccb49a76 100644 --- a/openssl/src/crypto/mem.c +++ b/openssl/src/crypto/mem.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,14 +14,42 @@ #include #include #include +#ifndef OPENSSL_NO_CRYPTO_MDEBUG_COUNT +# if defined(__linux__) +# include +# define MALLOC_SIZE(s) malloc_usable_size(s) +# elif defined(__APPLE__) +# include +# define MALLOC_SIZE(s) malloc_size(s) +# else +# define MALLOC_SIZE(s) (*((size_t*)(s)-1)) +# endif +#endif /* * the following pointers may be changed as long as 'allow_customize' is set */ static int allow_customize = 1; + +#ifndef OPENSSL_NO_CRYPTO_MDEBUG_COUNT +# define MDEBUG_COUNT_PTR_SET(ret, val) (*(ret) = (val)) + +static size_t mdebug_size_total = 0; +static size_t mdebug_count_total = 0; + +static void *CRYPTO_MDEBUG_COUNT_malloc(size_t num, const char *file, int line); +static void *CRYPTO_MDEBUG_COUNT_realloc(void *str, size_t num, const char *file, + int line); +static void CRYPTO_MDEBUG_COUNT_free(void *str, const char *file, int line); + +static CRYPTO_malloc_fn malloc_impl = CRYPTO_MDEBUG_COUNT_malloc; +static CRYPTO_realloc_fn realloc_impl = CRYPTO_MDEBUG_COUNT_realloc; +static CRYPTO_free_fn free_impl = CRYPTO_MDEBUG_COUNT_free; +#else static CRYPTO_malloc_fn malloc_impl = CRYPTO_malloc; static CRYPTO_realloc_fn realloc_impl = CRYPTO_realloc; static CRYPTO_free_fn free_impl = CRYPTO_free; +#endif #if !defined(OPENSSL_NO_CRYPTO_MDEBUG) && !defined(FIPS_MODULE) # include "internal/tsan_assist.h" @@ -81,6 +109,16 @@ void CRYPTO_get_mem_functions(CRYPTO_malloc_fn *malloc_fn, *free_fn = free_impl; } +#ifndef OPENSSL_NO_CRYPTO_MDEBUG_COUNT +void CRYPTO_get_mem_counts(int *count, size_t *size) +{ + if (count != NULL) + MDEBUG_COUNT_PTR_SET(count, mdebug_count_total); + if (size != NULL) + MDEBUG_COUNT_PTR_SET(size, mdebug_size_total); +} +#endif + #if !defined(OPENSSL_NO_CRYPTO_MDEBUG) && !defined(FIPS_MODULE) void CRYPTO_get_alloc_counts(int *mcount, int *rcount, int *fcount) { @@ -100,9 +138,6 @@ void CRYPTO_get_alloc_counts(int *mcount, int *rcount, int *fcount) * or 100;100@25;0 * This means 100 mallocs succeed, then next 100 fail 25% of the time, and * all remaining (count is zero) succeed. - * The failure percentge can have 2 digits after the comma. For example: - * 0@0.01 - * This means 0.01% of all allocations will fail. */ static void parseit(void) { @@ -115,27 +150,26 @@ static void parseit(void) /* Get the count (atol will stop at the @ if there), and percentage */ md_count = atol(md_failstring); atsign = strchr(md_failstring, '@'); - md_fail_percent = atsign == NULL ? 0 : (int)(atof(atsign + 1) * 100 + 0.5); + md_fail_percent = atsign == NULL ? 0 : atoi(atsign + 1); if (semi != NULL) md_failstring = semi; } /* - * Windows doesn't have random() and srandom(), but it has rand() and srand(). + * Windows doesn't have random(), but it has rand() * Some rand() implementations aren't good, but we're not * dealing with secure randomness here. */ # ifdef _WIN32 # define random() rand() -# define srandom(seed) srand(seed) # endif /* * See if the current malloc should fail. */ static int shouldfail(void) { - int roll = (int)(random() % 10000); + int roll = (int)(random() % 100); int shoulditfail = roll < md_fail_percent; # ifndef _WIN32 /* suppressed on Windows as POSIX-like file descriptors are non-inheritable */ @@ -169,23 +203,81 @@ void ossl_malloc_setup_failures(void) parseit(); if ((cp = getenv("OPENSSL_MALLOC_FD")) != NULL) md_tracefd = atoi(cp); - if ((cp = getenv("OPENSSL_MALLOC_SEED")) != NULL) - srandom(atoi(cp)); } #endif -void *CRYPTO_malloc(size_t num, const char *file, int line) +#ifndef OPENSSL_NO_CRYPTO_MDEBUG_COUNT +static void *CRYPTO_MDEBUG_COUNT_malloc(size_t num, const char *file, int line) { - void *ptr; + void *ret = NULL; + size_t mem_size; - INCREMENT(malloc_count); - if (malloc_impl != CRYPTO_malloc) { - ptr = malloc_impl(num, file, line); - if (ptr != NULL || num == 0) - return ptr; - goto err; + if (num == 0) + return NULL; + + ret = malloc(num); + + if (ret == NULL) + return NULL; + + mem_size = MALLOC_SIZE(ret); + mdebug_size_total += mem_size; + mdebug_count_total++; + + return ret; +} + +static void *CRYPTO_MDEBUG_COUNT_realloc(void *str, size_t num, const char *file, + int line) +{ + size_t mem_size; + void *ret = NULL; + + if (str == NULL) + return CRYPTO_MDEBUG_COUNT_malloc(num, file, line); + + if (num == 0) { + CRYPTO_MDEBUG_COUNT_free(str, file, line); + return NULL; } + mem_size = MALLOC_SIZE(str); + mdebug_size_total -= mem_size; + mdebug_count_total--; + + ret = realloc(str, num); + if (ret == NULL) + return NULL; + + mem_size = MALLOC_SIZE(ret); + mdebug_size_total += mem_size; + mdebug_count_total++; + + return ret; +} + +static void CRYPTO_MDEBUG_COUNT_free(void *str, const char *file, int line) +{ + size_t mem_size; + + if (str == NULL) + return; + + mem_size = MALLOC_SIZE(str); + mdebug_size_total -= mem_size; + mdebug_count_total--; + + free(str); + return; +} +#endif + +void *CRYPTO_malloc(size_t num, const char *file, int line) +{ + INCREMENT(malloc_count); + if (malloc_impl != CRYPTO_malloc) + return malloc_impl(num, file, line); + if (num == 0) return NULL; @@ -199,20 +291,7 @@ void *CRYPTO_malloc(size_t num, const char *file, int line) allow_customize = 0; } - ptr = malloc(num); - if (ptr != NULL) - return ptr; - err: - /* - * ossl_err_get_state_int() in err.c uses CRYPTO_zalloc(num, NULL, 0) for - * ERR_STATE allocation. Prevent mem alloc error loop while reporting error. - */ - if (file != NULL || line != 0) { - ERR_new(); - ERR_set_debug(file, line, NULL); - ERR_set_error(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE, NULL); - } - return NULL; + return malloc(num); } void *CRYPTO_zalloc(size_t num, const char *file, int line) @@ -220,6 +299,7 @@ void *CRYPTO_zalloc(size_t num, const char *file, int line) void *ret; ret = CRYPTO_malloc(num, file, line); + FAILTEST(); if (ret != NULL) memset(ret, 0, num); @@ -232,6 +312,7 @@ void *CRYPTO_realloc(void *str, size_t num, const char *file, int line) if (realloc_impl != CRYPTO_realloc) return realloc_impl(str, num, file, line); + FAILTEST(); if (str == NULL) return CRYPTO_malloc(num, file, line); @@ -240,7 +321,6 @@ void *CRYPTO_realloc(void *str, size_t num, const char *file, int line) return NULL; } - FAILTEST(); return realloc(str, num); } @@ -309,12 +389,12 @@ int CRYPTO_set_mem_debug(int flag) int CRYPTO_mem_debug_push(const char *info, const char *file, int line) { (void)info; (void)file; (void)line; - return 0; + return -1; } int CRYPTO_mem_debug_pop(void) { - return 0; + return -1; } void CRYPTO_mem_debug_malloc(void *addr, size_t num, int flag, diff --git a/openssl/src/crypto/mem_sec.c b/openssl/src/crypto/mem_sec.c index 269c7dcb6..53acd22c0 100644 --- a/openssl/src/crypto/mem_sec.c +++ b/openssl/src/crypto/mem_sec.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2004-2014, Akamai Technologies. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -17,28 +17,12 @@ */ #include "internal/e_os.h" #include -#include #include #ifndef OPENSSL_NO_SECURE_MEMORY # if defined(_WIN32) # include -# if defined(WINAPI_FAMILY_PARTITION) -# if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) -/* - * While VirtualLock is available under the app partition (e.g. UWP), - * the headers do not define the API. Define it ourselves instead. - */ -WINBASEAPI -BOOL -WINAPI -VirtualLock( - _In_ LPVOID lpAddress, - _In_ SIZE_T dwSize - ); -# endif -# endif # endif # include # include @@ -66,18 +50,6 @@ VirtualLock( # include # include #endif -#ifndef HAVE_MADVISE -# if defined(MADV_DONTDUMP) -# define HAVE_MADVISE 1 -# else -# define HAVE_MADVISE 0 -# endif -#endif -#if HAVE_MADVISE -# undef NO_MADVISE -#else -# define NO_MADVISE -#endif #define CLEAR(p, s) OPENSSL_cleanse(p, s) #ifndef PAGE_SIZE @@ -154,27 +126,18 @@ int CRYPTO_secure_malloc_initialized(void) void *CRYPTO_secure_malloc(size_t num, const char *file, int line) { #ifndef OPENSSL_NO_SECURE_MEMORY - void *ret = NULL; + void *ret; size_t actual_size; - int reason = CRYPTO_R_SECURE_MALLOC_FAILURE; if (!secure_mem_initialized) { return CRYPTO_malloc(num, file, line); } - if (!CRYPTO_THREAD_write_lock(sec_malloc_lock)) { - reason = ERR_R_CRYPTO_LIB; - goto err; - } + if (!CRYPTO_THREAD_write_lock(sec_malloc_lock)) + return NULL; ret = sh_malloc(num); actual_size = ret ? sh_actual_size(ret) : 0; secure_mem_used += actual_size; CRYPTO_THREAD_unlock(sec_malloc_lock); - err: - if (ret == NULL && (file != NULL || line != 0)) { - ERR_new(); - ERR_set_debug(file, line, NULL); - ERR_set_error(ERR_LIB_CRYPTO, reason, NULL); - } return ret; #else return CRYPTO_malloc(num, file, line); @@ -260,17 +223,11 @@ int CRYPTO_secure_allocated(const void *ptr) size_t CRYPTO_secure_used(void) { - size_t ret = 0; - #ifndef OPENSSL_NO_SECURE_MEMORY - if (!CRYPTO_THREAD_read_lock(sec_malloc_lock)) - return 0; - - ret = secure_mem_used; - - CRYPTO_THREAD_unlock(sec_malloc_lock); + return secure_mem_used; +#else + return 0; #endif /* OPENSSL_NO_SECURE_MEMORY */ - return ret; } size_t CRYPTO_secure_actual_size(void *ptr) @@ -585,7 +542,7 @@ static int sh_init(size_t size, size_t minsize) if (mlock(sh.arena, sh.arena_size) < 0) ret = 2; #endif -#ifndef NO_MADVISE +#ifdef MADV_DONTDUMP if (madvise(sh.arena, sh.arena_size, MADV_DONTDUMP) < 0) ret = 2; #endif diff --git a/openssl/src/crypto/modes/cfb128.c b/openssl/src/crypto/modes/cfb128.c index 2d37bcb58..384464ddb 100644 --- a/openssl/src/crypto/modes/cfb128.c +++ b/openssl/src/crypto/modes/cfb128.c @@ -17,6 +17,10 @@ typedef size_t size_t_aX __attribute((__aligned__(1))); typedef size_t size_t_aX; #endif +#if defined(__GNUC__) && (__GNUC__ > 6) +# pragma GCC diagnostic ignored "-Wstringop-overflow=" +#endif + /* * The input and output encrypted as though 128bit cfb mode is being used. * The extra state information to record how much of the 128bit block we have diff --git a/openssl/src/crypto/modes/gcm128.c b/openssl/src/crypto/modes/gcm128.c index f8901ed07..52865215a 100644 --- a/openssl/src/crypto/modes/gcm128.c +++ b/openssl/src/crypto/modes/gcm128.c @@ -1,5 +1,5 @@ /* - * Copyright 2010-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2010-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -27,12 +27,6 @@ typedef size_t size_t_aX; # define PUTU32(p,v) *(u32 *)(p) = BSWAP4(v) #endif -/* RISC-V uses C implementation as a fallback. */ -#if defined(__riscv) -# define INCLUDE_C_GMULT_4BIT -# define INCLUDE_C_GHASH_4BIT -#endif - #define PACK(s) ((size_t)(s)<<(sizeof(size_t)*8-16)) #define REDUCE1BIT(V) do { \ if (sizeof(size_t)==8) { \ @@ -48,9 +42,6 @@ typedef size_t size_t_aX; } while(0) /*- - * - * NOTE: TABLE_BITS and all non-4bit implementations have been removed in 3.1. - * * Even though permitted values for TABLE_BITS are 8, 4 and 1, it should * never be set to 8. 8 is effectively reserved for testing purposes. * TABLE_BITS>1 are lookup-table-driven implementations referred to as @@ -84,8 +75,150 @@ typedef size_t size_t_aX; * * Value of 1 is not appropriate for performance reasons. */ +#if TABLE_BITS==8 -static void gcm_init_4bit(u128 Htable[16], const u64 H[2]) +static void gcm_init_8bit(u128 Htable[256], u64 H[2]) +{ + int i, j; + u128 V; + + Htable[0].hi = 0; + Htable[0].lo = 0; + V.hi = H[0]; + V.lo = H[1]; + + for (Htable[128] = V, i = 64; i > 0; i >>= 1) { + REDUCE1BIT(V); + Htable[i] = V; + } + + for (i = 2; i < 256; i <<= 1) { + u128 *Hi = Htable + i, H0 = *Hi; + for (j = 1; j < i; ++j) { + Hi[j].hi = H0.hi ^ Htable[j].hi; + Hi[j].lo = H0.lo ^ Htable[j].lo; + } + } +} + +static void gcm_gmult_8bit(u64 Xi[2], const u128 Htable[256]) +{ + u128 Z = { 0, 0 }; + const u8 *xi = (const u8 *)Xi + 15; + size_t rem, n = *xi; + DECLARE_IS_ENDIAN; + static const size_t rem_8bit[256] = { + PACK(0x0000), PACK(0x01C2), PACK(0x0384), PACK(0x0246), + PACK(0x0708), PACK(0x06CA), PACK(0x048C), PACK(0x054E), + PACK(0x0E10), PACK(0x0FD2), PACK(0x0D94), PACK(0x0C56), + PACK(0x0918), PACK(0x08DA), PACK(0x0A9C), PACK(0x0B5E), + PACK(0x1C20), PACK(0x1DE2), PACK(0x1FA4), PACK(0x1E66), + PACK(0x1B28), PACK(0x1AEA), PACK(0x18AC), PACK(0x196E), + PACK(0x1230), PACK(0x13F2), PACK(0x11B4), PACK(0x1076), + PACK(0x1538), PACK(0x14FA), PACK(0x16BC), PACK(0x177E), + PACK(0x3840), PACK(0x3982), PACK(0x3BC4), PACK(0x3A06), + PACK(0x3F48), PACK(0x3E8A), PACK(0x3CCC), PACK(0x3D0E), + PACK(0x3650), PACK(0x3792), PACK(0x35D4), PACK(0x3416), + PACK(0x3158), PACK(0x309A), PACK(0x32DC), PACK(0x331E), + PACK(0x2460), PACK(0x25A2), PACK(0x27E4), PACK(0x2626), + PACK(0x2368), PACK(0x22AA), PACK(0x20EC), PACK(0x212E), + PACK(0x2A70), PACK(0x2BB2), PACK(0x29F4), PACK(0x2836), + PACK(0x2D78), PACK(0x2CBA), PACK(0x2EFC), PACK(0x2F3E), + PACK(0x7080), PACK(0x7142), PACK(0x7304), PACK(0x72C6), + PACK(0x7788), PACK(0x764A), PACK(0x740C), PACK(0x75CE), + PACK(0x7E90), PACK(0x7F52), PACK(0x7D14), PACK(0x7CD6), + PACK(0x7998), PACK(0x785A), PACK(0x7A1C), PACK(0x7BDE), + PACK(0x6CA0), PACK(0x6D62), PACK(0x6F24), PACK(0x6EE6), + PACK(0x6BA8), PACK(0x6A6A), PACK(0x682C), PACK(0x69EE), + PACK(0x62B0), PACK(0x6372), PACK(0x6134), PACK(0x60F6), + PACK(0x65B8), PACK(0x647A), PACK(0x663C), PACK(0x67FE), + PACK(0x48C0), PACK(0x4902), PACK(0x4B44), PACK(0x4A86), + PACK(0x4FC8), PACK(0x4E0A), PACK(0x4C4C), PACK(0x4D8E), + PACK(0x46D0), PACK(0x4712), PACK(0x4554), PACK(0x4496), + PACK(0x41D8), PACK(0x401A), PACK(0x425C), PACK(0x439E), + PACK(0x54E0), PACK(0x5522), PACK(0x5764), PACK(0x56A6), + PACK(0x53E8), PACK(0x522A), PACK(0x506C), PACK(0x51AE), + PACK(0x5AF0), PACK(0x5B32), PACK(0x5974), PACK(0x58B6), + PACK(0x5DF8), PACK(0x5C3A), PACK(0x5E7C), PACK(0x5FBE), + PACK(0xE100), PACK(0xE0C2), PACK(0xE284), PACK(0xE346), + PACK(0xE608), PACK(0xE7CA), PACK(0xE58C), PACK(0xE44E), + PACK(0xEF10), PACK(0xEED2), PACK(0xEC94), PACK(0xED56), + PACK(0xE818), PACK(0xE9DA), PACK(0xEB9C), PACK(0xEA5E), + PACK(0xFD20), PACK(0xFCE2), PACK(0xFEA4), PACK(0xFF66), + PACK(0xFA28), PACK(0xFBEA), PACK(0xF9AC), PACK(0xF86E), + PACK(0xF330), PACK(0xF2F2), PACK(0xF0B4), PACK(0xF176), + PACK(0xF438), PACK(0xF5FA), PACK(0xF7BC), PACK(0xF67E), + PACK(0xD940), PACK(0xD882), PACK(0xDAC4), PACK(0xDB06), + PACK(0xDE48), PACK(0xDF8A), PACK(0xDDCC), PACK(0xDC0E), + PACK(0xD750), PACK(0xD692), PACK(0xD4D4), PACK(0xD516), + PACK(0xD058), PACK(0xD19A), PACK(0xD3DC), PACK(0xD21E), + PACK(0xC560), PACK(0xC4A2), PACK(0xC6E4), PACK(0xC726), + PACK(0xC268), PACK(0xC3AA), PACK(0xC1EC), PACK(0xC02E), + PACK(0xCB70), PACK(0xCAB2), PACK(0xC8F4), PACK(0xC936), + PACK(0xCC78), PACK(0xCDBA), PACK(0xCFFC), PACK(0xCE3E), + PACK(0x9180), PACK(0x9042), PACK(0x9204), PACK(0x93C6), + PACK(0x9688), PACK(0x974A), PACK(0x950C), PACK(0x94CE), + PACK(0x9F90), PACK(0x9E52), PACK(0x9C14), PACK(0x9DD6), + PACK(0x9898), PACK(0x995A), PACK(0x9B1C), PACK(0x9ADE), + PACK(0x8DA0), PACK(0x8C62), PACK(0x8E24), PACK(0x8FE6), + PACK(0x8AA8), PACK(0x8B6A), PACK(0x892C), PACK(0x88EE), + PACK(0x83B0), PACK(0x8272), PACK(0x8034), PACK(0x81F6), + PACK(0x84B8), PACK(0x857A), PACK(0x873C), PACK(0x86FE), + PACK(0xA9C0), PACK(0xA802), PACK(0xAA44), PACK(0xAB86), + PACK(0xAEC8), PACK(0xAF0A), PACK(0xAD4C), PACK(0xAC8E), + PACK(0xA7D0), PACK(0xA612), PACK(0xA454), PACK(0xA596), + PACK(0xA0D8), PACK(0xA11A), PACK(0xA35C), PACK(0xA29E), + PACK(0xB5E0), PACK(0xB422), PACK(0xB664), PACK(0xB7A6), + PACK(0xB2E8), PACK(0xB32A), PACK(0xB16C), PACK(0xB0AE), + PACK(0xBBF0), PACK(0xBA32), PACK(0xB874), PACK(0xB9B6), + PACK(0xBCF8), PACK(0xBD3A), PACK(0xBF7C), PACK(0xBEBE) + }; + + while (1) { + Z.hi ^= Htable[n].hi; + Z.lo ^= Htable[n].lo; + + if ((u8 *)Xi == xi) + break; + + n = *(--xi); + + rem = (size_t)Z.lo & 0xff; + Z.lo = (Z.hi << 56) | (Z.lo >> 8); + Z.hi = (Z.hi >> 8); + if (sizeof(size_t) == 8) + Z.hi ^= rem_8bit[rem]; + else + Z.hi ^= (u64)rem_8bit[rem] << 32; + } + + if (IS_LITTLE_ENDIAN) { +# ifdef BSWAP8 + Xi[0] = BSWAP8(Z.hi); + Xi[1] = BSWAP8(Z.lo); +# else + u8 *p = (u8 *)Xi; + u32 v; + v = (u32)(Z.hi >> 32); + PUTU32(p, v); + v = (u32)(Z.hi); + PUTU32(p + 4, v); + v = (u32)(Z.lo >> 32); + PUTU32(p + 8, v); + v = (u32)(Z.lo); + PUTU32(p + 12, v); +# endif + } else { + Xi[0] = Z.hi; + Xi[1] = Z.lo; + } +} + +# define GCM_MUL(ctx) gcm_gmult_8bit(ctx->Xi.u,ctx->Htable) + +#elif TABLE_BITS==4 + +static void gcm_init_4bit(u128 Htable[16], u64 H[2]) { u128 V; # if defined(OPENSSL_SMALL_FOOTPRINT) @@ -156,7 +289,7 @@ static void gcm_init_4bit(u128 Htable[16], const u64 H[2]) # endif } -# if !defined(GHASH_ASM) || defined(INCLUDE_C_GMULT_4BIT) +# ifndef GHASH_ASM static const size_t rem_4bit[16] = { PACK(0x0000), PACK(0x1C20), PACK(0x3840), PACK(0x2460), PACK(0x7080), PACK(0x6CA0), PACK(0x48C0), PACK(0x54E0), @@ -231,9 +364,6 @@ static void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16]) } } -# endif - -# if !defined(GHASH_ASM) || defined(INCLUDE_C_GHASH_4BIT) # if !defined(OPENSSL_SMALL_FOOTPRINT) /* * Streamed gcm_mult_4bit, see CRYPTO_gcm128_[en|de]crypt for @@ -250,6 +380,7 @@ static void gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], size_t rem, nlo, nhi; DECLARE_IS_ENDIAN; +# if 1 do { cnt = 15; nlo = ((const u8 *)Xi)[15]; @@ -291,6 +422,100 @@ static void gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], Z.hi ^= Htable[nlo].hi; Z.lo ^= Htable[nlo].lo; } +# else + /* + * Extra 256+16 bytes per-key plus 512 bytes shared tables + * [should] give ~50% improvement... One could have PACK()-ed + * the rem_8bit even here, but the priority is to minimize + * cache footprint... + */ + u128 Hshr4[16]; /* Htable shifted right by 4 bits */ + u8 Hshl4[16]; /* Htable shifted left by 4 bits */ + static const unsigned short rem_8bit[256] = { + 0x0000, 0x01C2, 0x0384, 0x0246, 0x0708, 0x06CA, 0x048C, 0x054E, + 0x0E10, 0x0FD2, 0x0D94, 0x0C56, 0x0918, 0x08DA, 0x0A9C, 0x0B5E, + 0x1C20, 0x1DE2, 0x1FA4, 0x1E66, 0x1B28, 0x1AEA, 0x18AC, 0x196E, + 0x1230, 0x13F2, 0x11B4, 0x1076, 0x1538, 0x14FA, 0x16BC, 0x177E, + 0x3840, 0x3982, 0x3BC4, 0x3A06, 0x3F48, 0x3E8A, 0x3CCC, 0x3D0E, + 0x3650, 0x3792, 0x35D4, 0x3416, 0x3158, 0x309A, 0x32DC, 0x331E, + 0x2460, 0x25A2, 0x27E4, 0x2626, 0x2368, 0x22AA, 0x20EC, 0x212E, + 0x2A70, 0x2BB2, 0x29F4, 0x2836, 0x2D78, 0x2CBA, 0x2EFC, 0x2F3E, + 0x7080, 0x7142, 0x7304, 0x72C6, 0x7788, 0x764A, 0x740C, 0x75CE, + 0x7E90, 0x7F52, 0x7D14, 0x7CD6, 0x7998, 0x785A, 0x7A1C, 0x7BDE, + 0x6CA0, 0x6D62, 0x6F24, 0x6EE6, 0x6BA8, 0x6A6A, 0x682C, 0x69EE, + 0x62B0, 0x6372, 0x6134, 0x60F6, 0x65B8, 0x647A, 0x663C, 0x67FE, + 0x48C0, 0x4902, 0x4B44, 0x4A86, 0x4FC8, 0x4E0A, 0x4C4C, 0x4D8E, + 0x46D0, 0x4712, 0x4554, 0x4496, 0x41D8, 0x401A, 0x425C, 0x439E, + 0x54E0, 0x5522, 0x5764, 0x56A6, 0x53E8, 0x522A, 0x506C, 0x51AE, + 0x5AF0, 0x5B32, 0x5974, 0x58B6, 0x5DF8, 0x5C3A, 0x5E7C, 0x5FBE, + 0xE100, 0xE0C2, 0xE284, 0xE346, 0xE608, 0xE7CA, 0xE58C, 0xE44E, + 0xEF10, 0xEED2, 0xEC94, 0xED56, 0xE818, 0xE9DA, 0xEB9C, 0xEA5E, + 0xFD20, 0xFCE2, 0xFEA4, 0xFF66, 0xFA28, 0xFBEA, 0xF9AC, 0xF86E, + 0xF330, 0xF2F2, 0xF0B4, 0xF176, 0xF438, 0xF5FA, 0xF7BC, 0xF67E, + 0xD940, 0xD882, 0xDAC4, 0xDB06, 0xDE48, 0xDF8A, 0xDDCC, 0xDC0E, + 0xD750, 0xD692, 0xD4D4, 0xD516, 0xD058, 0xD19A, 0xD3DC, 0xD21E, + 0xC560, 0xC4A2, 0xC6E4, 0xC726, 0xC268, 0xC3AA, 0xC1EC, 0xC02E, + 0xCB70, 0xCAB2, 0xC8F4, 0xC936, 0xCC78, 0xCDBA, 0xCFFC, 0xCE3E, + 0x9180, 0x9042, 0x9204, 0x93C6, 0x9688, 0x974A, 0x950C, 0x94CE, + 0x9F90, 0x9E52, 0x9C14, 0x9DD6, 0x9898, 0x995A, 0x9B1C, 0x9ADE, + 0x8DA0, 0x8C62, 0x8E24, 0x8FE6, 0x8AA8, 0x8B6A, 0x892C, 0x88EE, + 0x83B0, 0x8272, 0x8034, 0x81F6, 0x84B8, 0x857A, 0x873C, 0x86FE, + 0xA9C0, 0xA802, 0xAA44, 0xAB86, 0xAEC8, 0xAF0A, 0xAD4C, 0xAC8E, + 0xA7D0, 0xA612, 0xA454, 0xA596, 0xA0D8, 0xA11A, 0xA35C, 0xA29E, + 0xB5E0, 0xB422, 0xB664, 0xB7A6, 0xB2E8, 0xB32A, 0xB16C, 0xB0AE, + 0xBBF0, 0xBA32, 0xB874, 0xB9B6, 0xBCF8, 0xBD3A, 0xBF7C, 0xBEBE + }; + /* + * This pre-processing phase slows down procedure by approximately + * same time as it makes each loop spin faster. In other words + * single block performance is approximately same as straightforward + * "4-bit" implementation, and then it goes only faster... + */ + for (cnt = 0; cnt < 16; ++cnt) { + Z.hi = Htable[cnt].hi; + Z.lo = Htable[cnt].lo; + Hshr4[cnt].lo = (Z.hi << 60) | (Z.lo >> 4); + Hshr4[cnt].hi = (Z.hi >> 4); + Hshl4[cnt] = (u8)(Z.lo << 4); + } + + do { + for (Z.lo = 0, Z.hi = 0, cnt = 15; cnt; --cnt) { + nlo = ((const u8 *)Xi)[cnt]; + nlo ^= inp[cnt]; + nhi = nlo >> 4; + nlo &= 0xf; + + Z.hi ^= Htable[nlo].hi; + Z.lo ^= Htable[nlo].lo; + + rem = (size_t)Z.lo & 0xff; + + Z.lo = (Z.hi << 56) | (Z.lo >> 8); + Z.hi = (Z.hi >> 8); + + Z.hi ^= Hshr4[nhi].hi; + Z.lo ^= Hshr4[nhi].lo; + Z.hi ^= (u64)rem_8bit[rem ^ Hshl4[nhi]] << 48; + } + + nlo = ((const u8 *)Xi)[0]; + nlo ^= inp[0]; + nhi = nlo >> 4; + nlo &= 0xf; + + Z.hi ^= Htable[nlo].hi; + Z.lo ^= Htable[nlo].lo; + + rem = (size_t)Z.lo & 0xf; + + Z.lo = (Z.hi << 60) | (Z.lo >> 4); + Z.hi = (Z.hi >> 4); + + Z.hi ^= Htable[nhi].hi; + Z.lo ^= Htable[nhi].lo; + Z.hi ^= ((u64)rem_8bit[rem << 4]) << 48; +# endif if (IS_LITTLE_ENDIAN) { # ifdef BSWAP8 @@ -312,11 +537,7 @@ static void gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], Xi[0] = Z.hi; Xi[1] = Z.lo; } - - inp += 16; - /* Block size is 128 bits so len is a multiple of 16 */ - len -= 16; - } while (len > 0); + } while (inp += 16, len -= 16); } # endif # else @@ -325,9 +546,9 @@ void gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); # endif -# define GCM_MUL(ctx) ctx->funcs.gmult(ctx->Xi.u,ctx->Htable) +# define GCM_MUL(ctx) gcm_gmult_4bit(ctx->Xi.u,ctx->Htable) # if defined(GHASH_ASM) || !defined(OPENSSL_SMALL_FOOTPRINT) -# define GHASH(ctx,in,len) ctx->funcs.ghash((ctx)->Xi.u,(ctx)->Htable,in,len) +# define GHASH(ctx,in,len) gcm_ghash_4bit((ctx)->Xi.u,(ctx)->Htable,in,len) /* * GHASH_CHUNK is "stride parameter" missioned to mitigate cache trashing * effect. In other words idea is to hash data while it's still in L1 cache @@ -336,12 +557,77 @@ void gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], const u8 *inp, # define GHASH_CHUNK (3*1024) # endif -#if (defined(GHASH_ASM) || defined(OPENSSL_CPUID_OBJ)) +#else /* TABLE_BITS */ + +static void gcm_gmult_1bit(u64 Xi[2], const u64 H[2]) +{ + u128 V, Z = { 0, 0 }; + long X; + int i, j; + const long *xi = (const long *)Xi; + DECLARE_IS_ENDIAN; + + V.hi = H[0]; /* H is in host byte order, no byte swapping */ + V.lo = H[1]; + + for (j = 0; j < 16 / sizeof(long); ++j) { + if (IS_LITTLE_ENDIAN) { + if (sizeof(long) == 8) { +# ifdef BSWAP8 + X = (long)(BSWAP8(xi[j])); +# else + const u8 *p = (const u8 *)(xi + j); + X = (long)((u64)GETU32(p) << 32 | GETU32(p + 4)); +# endif + } else { + const u8 *p = (const u8 *)(xi + j); + X = (long)GETU32(p); + } + } else + X = xi[j]; + + for (i = 0; i < 8 * sizeof(long); ++i, X <<= 1) { + u64 M = (u64)(X >> (8 * sizeof(long) - 1)); + Z.hi ^= V.hi & M; + Z.lo ^= V.lo & M; + + REDUCE1BIT(V); + } + } + + if (IS_LITTLE_ENDIAN) { +# ifdef BSWAP8 + Xi[0] = BSWAP8(Z.hi); + Xi[1] = BSWAP8(Z.lo); +# else + u8 *p = (u8 *)Xi; + u32 v; + v = (u32)(Z.hi >> 32); + PUTU32(p, v); + v = (u32)(Z.hi); + PUTU32(p + 4, v); + v = (u32)(Z.lo >> 32); + PUTU32(p + 8, v); + v = (u32)(Z.lo); + PUTU32(p + 12, v); +# endif + } else { + Xi[0] = Z.hi; + Xi[1] = Z.lo; + } +} + +# define GCM_MUL(ctx) gcm_gmult_1bit(ctx->Xi.u,ctx->H.u) + +#endif + +#if TABLE_BITS==4 && (defined(GHASH_ASM) || defined(OPENSSL_CPUID_OBJ)) # if !defined(I386_ONLY) && \ (defined(__i386) || defined(__i386__) || \ defined(__x86_64) || defined(__x86_64__) || \ defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64)) # define GHASH_ASM_X86_OR_64 +# define GCM_FUNCREF_4BIT void gcm_init_clmul(u128 Htable[16], const u64 Xi[2]); void gcm_gmult_clmul(u64 Xi[2], const u128 Htable[16]); @@ -369,10 +655,11 @@ void gcm_gmult_4bit_x86(u64 Xi[2], const u128 Htable[16]); void gcm_ghash_4bit_x86(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); # endif -# elif defined(__arm__) || defined(__arm) || defined(__aarch64__) || defined(_M_ARM64) +# elif defined(__arm__) || defined(__arm) || defined(__aarch64__) # include "arm_arch.h" # if __ARM_MAX_ARCH__>=7 # define GHASH_ASM_ARM +# define GCM_FUNCREF_4BIT # define PMULL_CAPABLE (OPENSSL_armcap_P & ARMV8_PMULL) # if defined(__arm__) || defined(__arm) # define NEON_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON) @@ -386,214 +673,25 @@ void gcm_gmult_v8(u64 Xi[2], const u128 Htable[16]); void gcm_ghash_v8(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); # endif -# elif defined(__sparc__) || defined(__sparc) -# include "crypto/sparc_arch.h" -# define GHASH_ASM_SPARC -void gcm_init_vis3(u128 Htable[16], const u64 Xi[2]); -void gcm_gmult_vis3(u64 Xi[2], const u128 Htable[16]); -void gcm_ghash_vis3(u64 Xi[2], const u128 Htable[16], const u8 *inp, - size_t len); -# elif defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__POWERPC__) || defined(_ARCH_PPC)) +# elif defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC)) # include "crypto/ppc_arch.h" # define GHASH_ASM_PPC +# define GCM_FUNCREF_4BIT void gcm_init_p8(u128 Htable[16], const u64 Xi[2]); void gcm_gmult_p8(u64 Xi[2], const u128 Htable[16]); void gcm_ghash_p8(u64 Xi[2], const u128 Htable[16], const u8 *inp, size_t len); -# elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 -# include "crypto/riscv_arch.h" -# define GHASH_ASM_RV64I -/* Zbc/Zbkc (scalar crypto with clmul) based routines. */ -void gcm_init_rv64i_zbc(u128 Htable[16], const u64 Xi[2]); -void gcm_init_rv64i_zbc__zbb(u128 Htable[16], const u64 Xi[2]); -void gcm_init_rv64i_zbc__zbkb(u128 Htable[16], const u64 Xi[2]); -void gcm_gmult_rv64i_zbc(u64 Xi[2], const u128 Htable[16]); -void gcm_gmult_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16]); -void gcm_ghash_rv64i_zbc(u64 Xi[2], const u128 Htable[16], - const u8 *inp, size_t len); -void gcm_ghash_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16], - const u8 *inp, size_t len); -/* zvkb/Zvbc (vector crypto with vclmul) based routines. */ -void gcm_init_rv64i_zvkb_zvbc(u128 Htable[16], const u64 Xi[2]); -void gcm_gmult_rv64i_zvkb_zvbc(u64 Xi[2], const u128 Htable[16]); -void gcm_ghash_rv64i_zvkb_zvbc(u64 Xi[2], const u128 Htable[16], - const u8 *inp, size_t len); -/* Zvkg (vector crypto with vgmul.vv and vghsh.vv). */ -void gcm_init_rv64i_zvkg(u128 Htable[16], const u64 Xi[2]); -void gcm_init_rv64i_zvkg_zvkb(u128 Htable[16], const u64 Xi[2]); -void gcm_gmult_rv64i_zvkg(u64 Xi[2], const u128 Htable[16]); -void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16], - const u8 *inp, size_t len); # endif #endif -static void gcm_get_funcs(struct gcm_funcs_st *ctx) -{ - /* set defaults -- overridden below as needed */ - ctx->ginit = gcm_init_4bit; -#if !defined(GHASH_ASM) - ctx->gmult = gcm_gmult_4bit; -#else - ctx->gmult = NULL; -#endif -#if !defined(GHASH_ASM) && !defined(OPENSSL_SMALL_FOOTPRINT) - ctx->ghash = gcm_ghash_4bit; -#else - ctx->ghash = NULL; -#endif - -#if defined(GHASH_ASM_X86_OR_64) -# if !defined(GHASH_ASM_X86) || defined(OPENSSL_IA32_SSE2) - /* x86_64 */ - if (OPENSSL_ia32cap_P[1] & (1 << 1)) { /* check PCLMULQDQ bit */ - if (((OPENSSL_ia32cap_P[1] >> 22) & 0x41) == 0x41) { /* AVX+MOVBE */ - ctx->ginit = gcm_init_avx; - ctx->gmult = gcm_gmult_avx; - ctx->ghash = gcm_ghash_avx; - } else { - ctx->ginit = gcm_init_clmul; - ctx->gmult = gcm_gmult_clmul; - ctx->ghash = gcm_ghash_clmul; - } - return; - } -# endif -# if defined(GHASH_ASM_X86) - /* x86 only */ -# if defined(OPENSSL_IA32_SSE2) - if (OPENSSL_ia32cap_P[0] & (1 << 25)) { /* check SSE bit */ - ctx->gmult = gcm_gmult_4bit_mmx; - ctx->ghash = gcm_ghash_4bit_mmx; - return; - } -# else - if (OPENSSL_ia32cap_P[0] & (1 << 23)) { /* check MMX bit */ - ctx->gmult = gcm_gmult_4bit_mmx; - ctx->ghash = gcm_ghash_4bit_mmx; - return; - } -# endif - ctx->gmult = gcm_gmult_4bit_x86; - ctx->ghash = gcm_ghash_4bit_x86; - return; -# else - /* x86_64 fallback defaults */ - ctx->gmult = gcm_gmult_4bit; - ctx->ghash = gcm_ghash_4bit; - return; -# endif -#elif defined(GHASH_ASM_ARM) - /* ARM defaults */ - ctx->gmult = gcm_gmult_4bit; - ctx->ghash = gcm_ghash_4bit; -# ifdef PMULL_CAPABLE - if (PMULL_CAPABLE) { - ctx->ginit = (gcm_init_fn)gcm_init_v8; - ctx->gmult = gcm_gmult_v8; - ctx->ghash = gcm_ghash_v8; - } -# elif defined(NEON_CAPABLE) - if (NEON_CAPABLE) { - ctx->ginit = gcm_init_neon; - ctx->gmult = gcm_gmult_neon; - ctx->ghash = gcm_ghash_neon; - } +#ifdef GCM_FUNCREF_4BIT +# undef GCM_MUL +# define GCM_MUL(ctx) (*gcm_gmult_p)(ctx->Xi.u,ctx->Htable) +# ifdef GHASH +# undef GHASH +# define GHASH(ctx,in,len) (*gcm_ghash_p)(ctx->Xi.u,ctx->Htable,in,len) # endif - return; -#elif defined(GHASH_ASM_SPARC) - /* SPARC defaults */ - ctx->gmult = gcm_gmult_4bit; - ctx->ghash = gcm_ghash_4bit; - if (OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) { - ctx->ginit = gcm_init_vis3; - ctx->gmult = gcm_gmult_vis3; - ctx->ghash = gcm_ghash_vis3; - } - return; -#elif defined(GHASH_ASM_PPC) - /* PowerPC does not define GHASH_ASM; defaults set above */ - if (OPENSSL_ppccap_P & PPC_CRYPTO207) { - ctx->ginit = gcm_init_p8; - ctx->gmult = gcm_gmult_p8; - ctx->ghash = gcm_ghash_p8; - } - return; -#elif defined(GHASH_ASM_RV64I) - /* RISCV defaults */ - ctx->gmult = gcm_gmult_4bit; - ctx->ghash = gcm_ghash_4bit; - - if (RISCV_HAS_ZVKG() && riscv_vlen() >= 128) { - if (RISCV_HAS_ZVKB()) - ctx->ginit = gcm_init_rv64i_zvkg_zvkb; - else - ctx->ginit = gcm_init_rv64i_zvkg; - ctx->gmult = gcm_gmult_rv64i_zvkg; - ctx->ghash = gcm_ghash_rv64i_zvkg; - } else if (RISCV_HAS_ZVKB() && RISCV_HAS_ZVBC() && riscv_vlen() >= 128) { - ctx->ginit = gcm_init_rv64i_zvkb_zvbc; - ctx->gmult = gcm_gmult_rv64i_zvkb_zvbc; - ctx->ghash = gcm_ghash_rv64i_zvkb_zvbc; - } else if (RISCV_HAS_ZBC()) { - if (RISCV_HAS_ZBKB()) { - ctx->ginit = gcm_init_rv64i_zbc__zbkb; - ctx->gmult = gcm_gmult_rv64i_zbc__zbkb; - ctx->ghash = gcm_ghash_rv64i_zbc__zbkb; - } else if (RISCV_HAS_ZBB()) { - ctx->ginit = gcm_init_rv64i_zbc__zbb; - ctx->gmult = gcm_gmult_rv64i_zbc; - ctx->ghash = gcm_ghash_rv64i_zbc; - } else { - ctx->ginit = gcm_init_rv64i_zbc; - ctx->gmult = gcm_gmult_rv64i_zbc; - ctx->ghash = gcm_ghash_rv64i_zbc; - } - } - return; -#elif defined(GHASH_ASM) - /* all other architectures use the generic names */ - ctx->gmult = gcm_gmult_4bit; - ctx->ghash = gcm_ghash_4bit; - return; #endif -} - -void ossl_gcm_init_4bit(u128 Htable[16], const u64 H[2]) -{ - struct gcm_funcs_st funcs; - - gcm_get_funcs(&funcs); - funcs.ginit(Htable, H); -} - -void ossl_gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16]) -{ - struct gcm_funcs_st funcs; - - gcm_get_funcs(&funcs); - funcs.gmult(Xi, Htable); -} - -void ossl_gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], - const u8 *inp, size_t len) -{ - struct gcm_funcs_st funcs; - u64 tmp[2]; - size_t i; - - gcm_get_funcs(&funcs); - if (funcs.ghash != NULL) { - funcs.ghash(Xi, Htable, inp, len); - } else { - /* Emulate ghash if needed */ - for (i = 0; i < len; i += 16) { - memcpy(tmp, &inp[i], sizeof(tmp)); - Xi[0] ^= tmp[0]; - Xi[1] ^= tmp[1]; - funcs.gmult(Xi, Htable); - } - } -} void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx, void *key, block128_f block) { @@ -619,9 +717,81 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx, void *key, block128_f block) ctx->H.u[1] = lo; #endif } - - gcm_get_funcs(&ctx->funcs); - ctx->funcs.ginit(ctx->Htable, ctx->H.u); +#if TABLE_BITS==8 + gcm_init_8bit(ctx->Htable, ctx->H.u); +#elif TABLE_BITS==4 +# if defined(GHASH) +# define CTX__GHASH(f) (ctx->ghash = (f)) +# else +# define CTX__GHASH(f) (ctx->ghash = NULL) +# endif +# if defined(GHASH_ASM_X86_OR_64) +# if !defined(GHASH_ASM_X86) || defined(OPENSSL_IA32_SSE2) + if (OPENSSL_ia32cap_P[1] & (1 << 1)) { /* check PCLMULQDQ bit */ + if (((OPENSSL_ia32cap_P[1] >> 22) & 0x41) == 0x41) { /* AVX+MOVBE */ + gcm_init_avx(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_avx; + CTX__GHASH(gcm_ghash_avx); + } else { + gcm_init_clmul(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_clmul; + CTX__GHASH(gcm_ghash_clmul); + } + return; + } +# endif + gcm_init_4bit(ctx->Htable, ctx->H.u); +# if defined(GHASH_ASM_X86) /* x86 only */ +# if defined(OPENSSL_IA32_SSE2) + if (OPENSSL_ia32cap_P[0] & (1 << 25)) { /* check SSE bit */ +# else + if (OPENSSL_ia32cap_P[0] & (1 << 23)) { /* check MMX bit */ +# endif + ctx->gmult = gcm_gmult_4bit_mmx; + CTX__GHASH(gcm_ghash_4bit_mmx); + } else { + ctx->gmult = gcm_gmult_4bit_x86; + CTX__GHASH(gcm_ghash_4bit_x86); + } +# else + ctx->gmult = gcm_gmult_4bit; + CTX__GHASH(gcm_ghash_4bit); +# endif +# elif defined(GHASH_ASM_ARM) +# ifdef PMULL_CAPABLE + if (PMULL_CAPABLE) { + gcm_init_v8(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_v8; + CTX__GHASH(gcm_ghash_v8); + } else +# endif +# ifdef NEON_CAPABLE + if (NEON_CAPABLE) { + gcm_init_neon(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_neon; + CTX__GHASH(gcm_ghash_neon); + } else +# endif + { + gcm_init_4bit(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_4bit; + CTX__GHASH(gcm_ghash_4bit); + } +# elif defined(GHASH_ASM_PPC) + if (OPENSSL_ppccap_P & PPC_CRYPTO207) { + gcm_init_p8(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_p8; + CTX__GHASH(gcm_ghash_p8); + } else { + gcm_init_4bit(ctx->Htable, ctx->H.u); + ctx->gmult = gcm_gmult_4bit; + CTX__GHASH(gcm_ghash_4bit); + } +# else + gcm_init_4bit(ctx->Htable, ctx->H.u); +# endif +# undef CTX__GHASH +#endif } void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const unsigned char *iv, @@ -629,6 +799,9 @@ void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const unsigned char *iv, { DECLARE_IS_ENDIAN; unsigned int ctr; +#ifdef GCM_FUNCREF_4BIT + void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult; +#endif ctx->len.u[0] = 0; /* AAD length */ ctx->len.u[1] = 0; /* message length */ @@ -717,6 +890,13 @@ int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const unsigned char *aad, size_t i; unsigned int n; u64 alen = ctx->len.u[0]; +#ifdef GCM_FUNCREF_4BIT + void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult; +# ifdef GHASH + void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16], + const u8 *inp, size_t len) = ctx->ghash; +# endif +#endif if (ctx->len.u[1]) return -2; @@ -775,6 +955,13 @@ int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, u64 mlen = ctx->len.u[1]; block128_f block = ctx->block; void *key = ctx->key; +#ifdef GCM_FUNCREF_4BIT + void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult; +# if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT) + void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16], + const u8 *inp, size_t len) = ctx->ghash; +# endif +#endif mlen += len; if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len)) @@ -1000,6 +1187,13 @@ int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, u64 mlen = ctx->len.u[1]; block128_f block = ctx->block; void *key = ctx->key; +#ifdef GCM_FUNCREF_4BIT + void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult; +# if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT) + void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16], + const u8 *inp, size_t len) = ctx->ghash; +# endif +#endif mlen += len; if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len)) @@ -1235,6 +1429,13 @@ int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, size_t i; u64 mlen = ctx->len.u[1]; void *key = ctx->key; +# ifdef GCM_FUNCREF_4BIT + void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult; +# ifdef GHASH + void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16], + const u8 *inp, size_t len) = ctx->ghash; +# endif +# endif mlen += len; if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len)) @@ -1389,6 +1590,13 @@ int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, size_t i; u64 mlen = ctx->len.u[1]; void *key = ctx->key; +# ifdef GCM_FUNCREF_4BIT + void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult; +# ifdef GHASH + void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16], + const u8 *inp, size_t len) = ctx->ghash; +# endif +# endif mlen += len; if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len)) @@ -1544,6 +1752,13 @@ int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const unsigned char *tag, DECLARE_IS_ENDIAN; u64 alen = ctx->len.u[0] << 3; u64 clen = ctx->len.u[1] << 3; +#ifdef GCM_FUNCREF_4BIT + void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult; +# if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT) + void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16], + const u8 *inp, size_t len) = ctx->ghash; +# endif +#endif #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT) u128 bitlen; diff --git a/openssl/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8-unroll8_64.S b/openssl/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8-unroll8_64.S deleted file mode 100644 index 549f32d45..000000000 --- a/openssl/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8-unroll8_64.S +++ /dev/null @@ -1,8487 +0,0 @@ -#include "arm_arch.h" - -#if __ARM_MAX_ARCH__>=8 - -.text -.globl _unroll8_eor3_aes_gcm_enc_128_kernel - -.align 4 -_unroll8_eor3_aes_gcm_enc_128_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, L128_enc_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - mov x5, x9 - ld1 { v0.16b}, [x16] //CTR block 0 - - sub x5, x5, #1 //byte_len - 1 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - ldr q27, [x8, #160] //load rk10 - - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - add x5, x5, x0 - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - - aese v7.16b, v26.16b //AES block 8k+15 - round 9 - aese v5.16b, v26.16b //AES block 8k+13 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - - add x4, x0, x1, lsr #3 //end_input_ptr - cmp x0, x5 //check if we have <= 8 blocks - b.ge L128_enc_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext - cmp x0, x5 //check if we have <= 8 blocks - -.long 0xce006d08 //eor3 v8.16b, v8.16b, v0.16b, v27.16b //AES block 0 - result - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - -.long 0xce016d29 //eor3 v9.16b, v9.16b, v1.16b, v27.16b //AES block 1 - result - stp q8, q9, [x2], #32 //AES block 0, 1 - store result - - rev32 v1.16b, v30.16b //CTR block 9 -.long 0xce056dad //eor3 v13.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result - add v30.4s, v30.4s, v31.4s //CTR block 9 - -.long 0xce026d4a //eor3 v10.16b, v10.16b, v2.16b, v27.16b //AES block 2 - result -.long 0xce066dce //eor3 v14.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result -.long 0xce046d8c //eor3 v12.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - -.long 0xce036d6b //eor3 v11.16b, v11.16b, v3.16b, v27.16b //AES block 3 - result -.long 0xce076def //eor3 v15.16b, v15.16b, v7.16b,v27.16b //AES block 7 - result - stp q10, q11, [x2], #32 //AES block 2, 3 - store result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - stp q12, q13, [x2], #32 //AES block 4, 5 - store result - - stp q14, q15, [x2], #32 //AES block 6, 7 - store result - - rev32 v4.16b, v30.16b //CTR block 12 - add v30.4s, v30.4s, v31.4s //CTR block 12 - b.ge L128_enc_prepretail //do prepretail - -L128_enc_main_loop: //main loop start - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - rev64 v8.16b, v8.16b //GHASH block 8k - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - ldp q26, q27, [x8, #0] //load rk0, rk1 - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev32 v7.16b, v30.16b //CTR block 8k+15 - - rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - rev64 v10.16b, v10.16b //GHASH block 8k+2 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h3l | h3h - ext v25.16b, v25.16b, v25.16b, #8 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b,v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - ldp q28, q26, [x8, #32] //load rk2, rk3 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - - rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h1l | h1h - ext v22.16b, v22.16b, v22.16b, #8 - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - ldp q26, q27, [x8, #96] //load rk6, rk7 - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - ldr d16, [x10] //MODULO - load modulo constant - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - rev32 v20.16b, v30.16b //CTR block 8k+16 - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - ldp q28, q26, [x8, #128] //load rk8, rk9 -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - rev32 v22.16b, v30.16b //CTR block 8k+17 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load plaintext - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ldr q27, [x8, #160] //load rk10 - - ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - rev32 v23.16b, v30.16b //CTR block 8k+18 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load plaintext - rev32 v25.16b, v30.16b //CTR block 8k+19 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - - cmp x0, x5 //LOOP CONTROL -.long 0xce046d8c //eor3 v12.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result - aese v7.16b, v26.16b //AES block 8k+15 - round 9 - - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - -.long 0xce026d4a //eor3 v10.16b, v10.16b, v2.16b, v27.16b //AES block 8k+10 - result - - mov v2.16b, v23.16b //CTR block 8k+18 - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - - rev32 v4.16b, v30.16b //CTR block 8k+20 - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - -.long 0xce076def //eor3 v15.16b, v15.16b, v7.16b, v27.16b //AES block 7 - result - aese v5.16b, v26.16b //AES block 8k+13 - round 9 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - -.long 0xce016d29 //eor3 v9.16b, v9.16b, v1.16b, v27.16b //AES block 8k+9 - result -.long 0xce036d6b //eor3 v11.16b, v11.16b, v3.16b, v27.16b //AES block 8k+11 - result - mov v3.16b, v25.16b //CTR block 8k+19 - - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment -.long 0xce056dad //eor3 v13.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result - mov v1.16b, v22.16b //CTR block 8k+17 - -.long 0xce006d08 //eor3 v8.16b, v8.16b, v0.16b, v27.16b //AES block 8k+8 - result - mov v0.16b, v20.16b //CTR block 8k+16 - stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result - - stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result -.long 0xce066dce //eor3 v14.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result - - stp q12, q13, [x2], #32 //AES block 8k+12, 8k+13 - store result -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - - stp q14, q15, [x2], #32 //AES block 8k+14, 8k+15 - store result - b.lt L128_enc_main_loop - -L128_enc_prepretail: //PREPRETAIL - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v8.16b, v8.16b //GHASH block 8k - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h6k | h5k - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - rev64 v10.16b, v10.16b //GHASH block 8k+2 - eor v8.16b, v8.16b, v19.16b //PRE 1 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - ldp q26, q27, [x8, #0] //load rk0, rk1 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - - rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) - rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - rev32 v7.16b, v30.16b //CTR block 8k+15 - - rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h1l | h1h - ext v22.16b, v22.16b, v22.16b, #8 - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - ldp q26, q27, [x8, #96] //load rk6, rk7 - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - ldr d16, [x10] //MODULO - load modulo constant - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 -.long 0xce114a73 //eor3 v19.16b, v19.16b, v17.16b, v18.16b //MODULO - fold into low - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - - ldr q27, [x8, #160] //load rk10 - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - aese v5.16b, v26.16b //AES block 8k+13 - round 9 - - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - aese v7.16b, v26.16b //AES block 8k+15 - round 9 -L128_enc_tail: //TAIL - - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - ldr q8, [x0], #16 //AES block 8k+8 - load plaintext - - mov v29.16b, v27.16b - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - -.long 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - cmp x5, #112 - b.gt L128_enc_blocks_more_than_7 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - movi v17.8b, #0 - - cmp x5, #96 - sub v30.4s, v30.4s, v31.4s - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v2.16b - mov v2.16b, v1.16b - - movi v19.8b, #0 - movi v18.8b, #0 - b.gt L128_enc_blocks_more_than_6 - - mov v7.16b, v6.16b - cmp x5, #80 - - sub v30.4s, v30.4s, v31.4s - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v1.16b - b.gt L128_enc_blocks_more_than_5 - - cmp x5, #64 - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - mov v4.16b, v1.16b - b.gt L128_enc_blocks_more_than_4 - - mov v7.16b, v6.16b - sub v30.4s, v30.4s, v31.4s - mov v6.16b, v5.16b - - mov v5.16b, v1.16b - cmp x5, #48 - b.gt L128_enc_blocks_more_than_3 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - mov v6.16b, v1.16b - - cmp x5, #32 - ldr q24, [x3, #96] //load h4k | h3k - b.gt L128_enc_blocks_more_than_2 - - cmp x5, #16 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v1.16b - b.gt L128_enc_blocks_more_than_1 - - ldr q21, [x3, #48] //load h2k | h1k - sub v30.4s, v30.4s, v31.4s - b L128_enc_blocks_less_than_1 -L128_enc_blocks_more_than_7: //blocks left > 7 - st1 { v9.16b}, [x2], #16 //AES final-7 block - store result - - rev64 v8.16b, v9.16b //GHASH final-7 block - ldr q9, [x0], #16 //AES final-6 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - -.long 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low -L128_enc_blocks_more_than_6: //blocks left > 6 - - st1 { v9.16b}, [x2], #16 //AES final-6 block - store result - - rev64 v8.16b, v9.16b //GHASH final-6 block - ldr q9, [x0], #16 //AES final-5 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - -.long 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high -L128_enc_blocks_more_than_5: //blocks left > 5 - - st1 { v9.16b}, [x2], #16 //AES final-5 block - store result - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - ldr q9, [x0], #16 //AES final-4 block - load plaintext - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - -.long 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - movi v16.8b, #0 //suppress further partial tag feed in - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid -L128_enc_blocks_more_than_4: //blocks left > 4 - - st1 { v9.16b}, [x2], #16 //AES final-4 block - store result - - rev64 v8.16b, v9.16b //GHASH final-4 block - - ldr q9, [x0], #16 //AES final-3 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - -.long 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid -L128_enc_blocks_more_than_3: //blocks left > 3 - - st1 { v9.16b}, [x2], #16 //AES final-3 block - store result - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - ldr q24, [x3, #96] //load h4k | h3k - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - - ldr q9, [x0], #16 //AES final-2 block - load plaintext - - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - -.long 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high -L128_enc_blocks_more_than_2: //blocks left > 2 - - st1 { v9.16b}, [x2], #16 //AES final-2 block - store result - - rev64 v8.16b, v9.16b //GHASH final-2 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-1 block - load plaintext - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - movi v16.8b, #0 //suppress further partial tag feed in - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid -.long 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result - - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low -L128_enc_blocks_more_than_1: //blocks left > 1 - - st1 { v9.16b}, [x2], #16 //AES final-1 block - store result - - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-1 block - ldr q9, [x0], #16 //AES final block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - movi v16.8b, #0 //suppress further partial tag feed in - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid -.long 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result - - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - - ldr q21, [x3, #48] //load h2k | h1k - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low -L128_enc_blocks_less_than_1: //blocks left <= 1 - - rev32 v30.16b, v30.16b - str q30, [x16] //store the updated counter - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - mvn x6, xzr //temp0_x = 0xffffffffffffffff - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - mvn x7, xzr //temp1_x = 0xffffffffffffffff - cmp x1, #64 - - csel x13, x7, x6, lt - csel x14, x6, xzr, lt - - mov v0.d[1], x14 - mov v0.d[0], x13 //ctr0b is mask for last block - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - - rev64 v8.16b, v9.16b //GHASH final block - - bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - st1 { v9.16b}, [x2] //store all 16B - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v16.d[0], v8.d[1] //GHASH final block - mid - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - mov x0, x9 - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -L128_enc_ret: - mov w0, #0x0 - ret - -.globl _unroll8_eor3_aes_gcm_dec_128_kernel - -.align 4 -_unroll8_eor3_aes_gcm_dec_128_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, L128_dec_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - mov x5, x9 - ld1 { v0.16b}, [x16] //CTR block 0 - - ldp q26, q27, [x8, #0] //load rk0, rk1 - sub x5, x5, #1 //byte_len - 1 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - - rev32 v30.16b, v0.16b //set up reversed counter - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - - rev32 v7.16b, v30.16b //CTR block 7 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - add x5, x5, x0 - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - - aese v0.16b, v26.16b //AES block 0 - round 9 - aese v1.16b, v26.16b //AES block 1 - round 9 - aese v6.16b, v26.16b //AES block 6 - round 9 - - ldr q27, [x8, #160] //load rk10 - aese v4.16b, v26.16b //AES block 4 - round 9 - aese v3.16b, v26.16b //AES block 3 - round 9 - - aese v2.16b, v26.16b //AES block 2 - round 9 - aese v5.16b, v26.16b //AES block 5 - round 9 - aese v7.16b, v26.16b //AES block 7 - round 9 - - add x4, x0, x1, lsr #3 //end_input_ptr - cmp x0, x5 //check if we have <= 8 blocks - b.ge L128_dec_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext - -.long 0xce006d00 //eor3 v0.16b, v8.16b, v0.16b, v27.16b //AES block 0 - result -.long 0xce016d21 //eor3 v1.16b, v9.16b, v1.16b, v27.16b //AES block 1 - result - stp q0, q1, [x2], #32 //AES block 0, 1 - store result - - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext - - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 - ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext - -.long 0xce036d63 //eor3 v3.16b, v11.16b, v3.16b, v27.16b //AES block 3 - result -.long 0xce026d42 //eor3 v2.16b, v10.16b, v2.16b, v27.16b //AES block 2 - result - stp q2, q3, [x2], #32 //AES block 2, 3 - store result - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - -.long 0xce066dc6 //eor3 v6.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - -.long 0xce046d84 //eor3 v4.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result -.long 0xce056da5 //eor3 v5.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result - stp q4, q5, [x2], #32 //AES block 4, 5 - store result - -.long 0xce076de7 //eor3 v7.16b, v15.16b, v7.16b, v27.16b //AES block 7 - result - stp q6, q7, [x2], #32 //AES block 6, 7 - store result - rev32 v4.16b, v30.16b //CTR block 12 - - cmp x0, x5 //check if we have <= 8 blocks - add v30.4s, v30.4s, v31.4s //CTR block 12 - b.ge L128_dec_prepretail //do prepretail - -L128_dec_main_loop: //main loop start - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - rev64 v8.16b, v8.16b //GHASH block 8k - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - rev64 v14.16b, v14.16b //GHASH block 8k+6 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v10.16b, v10.16b //GHASH block 8k+2 - rev64 v12.16b, v12.16b //GHASH block 8k+4 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - rev32 v7.16b, v30.16b //CTR block 8k+15 - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - ldp q28, q26, [x8, #32] //load rk2, rk3 - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - - ldp q27, q28, [x8, #64] //load rk4, rk5 - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - ldr d16, [x10] //MODULO - load modulo constant -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - rev32 v20.16b, v30.16b //CTR block 8k+16 -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - rev32 v22.16b, v30.16b //CTR block 8k+17 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext - - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - rev32 v23.16b, v30.16b //CTR block 8k+18 - - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - ldr q27, [x8, #160] //load rk10 - - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - - aese v7.16b, v26.16b //AES block 8k+15 - round 9 - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - rev32 v25.16b, v30.16b //CTR block 8k+19 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - aese v5.16b, v26.16b //AES block 8k+13 - round 9 -.long 0xce016d21 //eor3 v1.16b, v9.16b, v1.16b, v27.16b //AES block 8k+9 - result - -.long 0xce006d00 //eor3 v0.16b, v8.16b, v0.16b, v27.16b //AES block 8k+8 - result -.long 0xce076de7 //eor3 v7.16b, v15.16b, v7.16b, v27.16b //AES block 8k+15 - result -.long 0xce066dc6 //eor3 v6.16b, v14.16b, v6.16b, v27.16b //AES block 8k+14 - result - -.long 0xce026d42 //eor3 v2.16b, v10.16b, v2.16b, v27.16b //AES block 8k+10 - result - stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result - mov v1.16b, v22.16b //CTR block 8k+17 - -.long 0xce046d84 //eor3 v4.16b, v12.16b, v4.16b, v27.16b //AES block 8k+12 - result -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - mov v0.16b, v20.16b //CTR block 8k+16 - -.long 0xce036d63 //eor3 v3.16b, v11.16b, v3.16b, v27.16b //AES block 8k+11 - result - cmp x0, x5 //LOOP CONTROL - stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result - -.long 0xce056da5 //eor3 v5.16b, v13.16b, v5.16b, v27.16b //AES block 8k+13 - result - mov v2.16b, v23.16b //CTR block 8k+18 - - stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result - rev32 v4.16b, v30.16b //CTR block 8k+20 - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - - stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result - mov v3.16b, v25.16b //CTR block 8k+19 - b.lt L128_dec_main_loop - -L128_dec_prepretail: //PREPRETAIL - rev64 v11.16b, v11.16b //GHASH block 8k+3 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - rev64 v8.16b, v8.16b //GHASH block 8k - - rev64 v10.16b, v10.16b //GHASH block 8k+2 - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - rev64 v12.16b, v12.16b //GHASH block 8k+4 - - rev64 v14.16b, v14.16b //GHASH block 8k+6 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - rev32 v7.16b, v30.16b //CTR block 8k+15 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - ldp q28, q26, [x8, #32] //load rk2, rk3 -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - ldp q27, q28, [x8, #64] //load rk4, rk5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - ldp q26, q27, [x8, #96] //load rk6, rk7 -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - - ldr d16, [x10] //MODULO - load modulo constant - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ldp q28, q26, [x8, #128] //load rk8, rk9 - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - ldr q27, [x8, #160] //load rk10 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - aese v5.16b, v26.16b //AES block 8k+13 - round 9 - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - aese v7.16b, v26.16b //AES block 8k+15 - round 9 - -L128_dec_tail: //TAIL - - mov v29.16b, v27.16b - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - - cmp x5, #112 - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - -.long 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result - b.gt L128_dec_blocks_more_than_7 - - cmp x5, #96 - mov v7.16b, v6.16b - movi v19.8b, #0 - - movi v17.8b, #0 - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v2.16b - mov v2.16b, v1.16b - - movi v18.8b, #0 - sub v30.4s, v30.4s, v31.4s - b.gt L128_dec_blocks_more_than_6 - - cmp x5, #80 - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v1.16b - b.gt L128_dec_blocks_more_than_5 - - cmp x5, #64 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt L128_dec_blocks_more_than_4 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v1.16b - cmp x5, #48 - b.gt L128_dec_blocks_more_than_3 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - cmp x5, #32 - - ldr q24, [x3, #96] //load h4k | h3k - mov v6.16b, v1.16b - b.gt L128_dec_blocks_more_than_2 - - cmp x5, #16 - - mov v7.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt L128_dec_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b L128_dec_blocks_less_than_1 -L128_dec_blocks_more_than_7: //blocks left > 7 - rev64 v8.16b, v9.16b //GHASH final-7 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - ldr q9, [x0], #16 //AES final-6 block - load ciphertext - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - st1 { v12.16b}, [x2], #16 //AES final-7 block - store result -.long 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid -L128_dec_blocks_more_than_6: //blocks left > 6 - - rev64 v8.16b, v9.16b //GHASH final-6 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - ldr q9, [x0], #16 //AES final-5 block - load ciphertext - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - st1 { v12.16b}, [x2], #16 //AES final-6 block - store result - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid -.long 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result -L128_dec_blocks_more_than_5: //blocks left > 5 - - rev64 v8.16b, v9.16b //GHASH final-5 block - - ldr q9, [x0], #16 //AES final-4 block - load ciphertext - st1 { v12.16b}, [x2], #16 //AES final-5 block - store result - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - -.long 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - movi v16.8b, #0 //suppress further partial tag feed in - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high -L128_dec_blocks_more_than_4: //blocks left > 4 - - rev64 v8.16b, v9.16b //GHASH final-4 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - ldr q9, [x0], #16 //AES final-3 block - load ciphertext - - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high - - st1 { v12.16b}, [x2], #16 //AES final-4 block - store result - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - -.long 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid -L128_dec_blocks_more_than_3: //blocks left > 3 - - st1 { v12.16b}, [x2], #16 //AES final-3 block - store result - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - ldr q24, [x3, #96] //load h4k | h3k - - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - - ldr q9, [x0], #16 //AES final-2 block - load ciphertext - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - movi v16.8b, #0 //suppress further partial tag feed in -.long 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid -L128_dec_blocks_more_than_2: //blocks left > 2 - - rev64 v8.16b, v9.16b //GHASH final-2 block - - st1 { v12.16b}, [x2], #16 //AES final-2 block - store result - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - ldr q9, [x0], #16 //AES final-1 block - load ciphertext - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low - -.long 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high -L128_dec_blocks_more_than_1: //blocks left > 1 - - st1 { v12.16b}, [x2], #16 //AES final-1 block - store result - rev64 v8.16b, v9.16b //GHASH final-1 block - - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - - ldr q9, [x0], #16 //AES final block - load ciphertext - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high - ldr q21, [x3, #48] //load h2k | h1k - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid -.long 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result - - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid -L128_dec_blocks_less_than_1: //blocks left <= 1 - - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - mvn x6, xzr //temp0_x = 0xffffffffffffffff - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - mvn x7, xzr //temp1_x = 0xffffffffffffffff - - csel x13, x7, x6, lt - csel x14, x6, xzr, lt - - mov v0.d[1], x14 - mov v0.d[0], x13 //ctr0b is mask for last block - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - - rev64 v8.16b, v9.16b //GHASH final block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - ins v16.d[0], v8.d[1] //GHASH final block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - - bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - st1 { v12.16b}, [x2] //store all 16B - - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up - -.long 0xce115652 //eor3 v18.16b, v18.16b, v17.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.long 0xce124673 //eor3 v19.16b, v19.16b, v18.16b, v17.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - rev32 v30.16b, v30.16b - - str q30, [x16] //store the updated counter - - mov x0, x9 - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret -L128_dec_ret: - mov w0, #0x0 - ret - -.globl _unroll8_eor3_aes_gcm_enc_192_kernel - -.align 4 -_unroll8_eor3_aes_gcm_enc_192_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, L192_enc_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - mov x5, x9 - ld1 { v0.16b}, [x16] //CTR block 0 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - sub x5, x5, #1 //byte_len - 1 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - add x5, x5, x0 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - - add x4, x0, x1, lsr #3 //end_input_ptr - cmp x0, x5 //check if we have <= 8 blocks - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 9 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - ldp q27, q28, [x8, #160] //load rk10, rk11 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 9 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 9 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 9 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 9 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 14 - round 10 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 11 - round 10 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 9 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 13 - round 10 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 12 - round 10 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 10 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 15 - round 10 - - aese v6.16b, v28.16b //AES block 14 - round 11 - aese v3.16b, v28.16b //AES block 11 - round 11 - - aese v4.16b, v28.16b //AES block 12 - round 11 - aese v7.16b, v28.16b //AES block 15 - round 11 - ldr q26, [x8, #192] //load rk12 - - aese v1.16b, v28.16b //AES block 9 - round 11 - aese v5.16b, v28.16b //AES block 13 - round 11 - - aese v2.16b, v28.16b //AES block 10 - round 11 - aese v0.16b, v28.16b //AES block 8 - round 11 - b.ge L192_enc_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext - -.long 0xce006908 //eor3 v8.16b, v8.16b, v0.16b, v26.16b //AES block 0 - result - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - -.long 0xce03696b //eor3 v11.16b, v11.16b, v3.16b, v26.16b //AES block 3 - result -.long 0xce016929 //eor3 v9.16b, v9.16b, v1.16b, v26.16b //AES block 1 - result - - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 -.long 0xce04698c //eor3 v12.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result - -.long 0xce0569ad //eor3 v13.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result -.long 0xce0769ef //eor3 v15.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result - stp q8, q9, [x2], #32 //AES block 0, 1 - store result - -.long 0xce02694a //eor3 v10.16b, v10.16b, v2.16b, v26.16b //AES block 2 - result - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - - stp q10, q11, [x2], #32 //AES block 2, 3 - store result - cmp x0, x5 //check if we have <= 8 blocks - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 -.long 0xce0669ce //eor3 v14.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result - - stp q12, q13, [x2], #32 //AES block 4, 5 - store result - - rev32 v4.16b, v30.16b //CTR block 12 - stp q14, q15, [x2], #32 //AES block 6, 7 - store result - add v30.4s, v30.4s, v31.4s //CTR block 12 - - b.ge L192_enc_prepretail //do prepretail - -L192_enc_main_loop: //main loop start - rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) - ldp q26, q27, [x8, #0] //load rk0, rk1 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - rev64 v8.16b, v8.16b //GHASH block 8k - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev64 v11.16b, v11.16b //GHASH block 8k+3 - rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - rev32 v7.16b, v30.16b //CTR block 8k+15 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) - - rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - ldp q28, q26, [x8, #128] //load rk8, rk9 - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - ldr d16, [x10] //MODULO - load modulo constant -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - ldp q27, q28, [x8, #160] //load rk10, rk11 - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - rev32 v20.16b, v30.16b //CTR block 8k+16 - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - rev32 v22.16b, v30.16b //CTR block 8k+17 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - ldr q26, [x8, #192] //load rk12 - ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext - - aese v4.16b, v28.16b //AES block 8k+12 - round 11 -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load plaintext - aese v2.16b, v28.16b //AES block 8k+10 - round 11 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - - rev32 v23.16b, v30.16b //CTR block 8k+18 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - aese v5.16b, v28.16b //AES block 8k+13 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - - aese v7.16b, v28.16b //AES block 8k+15 - round 11 - aese v0.16b, v28.16b //AES block 8k+8 - round 11 -.long 0xce04698c //eor3 v12.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result - - aese v6.16b, v28.16b //AES block 8k+14 - round 11 - aese v3.16b, v28.16b //AES block 8k+11 - round 11 - aese v1.16b, v28.16b //AES block 8k+9 - round 11 - - rev32 v25.16b, v30.16b //CTR block 8k+19 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 -.long 0xce0769ef //eor3 v15.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result - -.long 0xce02694a //eor3 v10.16b, v10.16b, v2.16b, v26.16b //AES block 8k+10 - result -.long 0xce006908 //eor3 v8.16b, v8.16b, v0.16b, v26.16b //AES block 8k+8 - result - mov v2.16b, v23.16b //CTR block 8k+18 - -.long 0xce016929 //eor3 v9.16b, v9.16b, v1.16b, v26.16b //AES block 8k+9 - result - mov v1.16b, v22.16b //CTR block 8k+17 - stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.long 0xce0669ce //eor3 v14.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result - mov v0.16b, v20.16b //CTR block 8k+16 - rev32 v4.16b, v30.16b //CTR block 8k+20 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 -.long 0xce0569ad //eor3 v13.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - -.long 0xce03696b //eor3 v11.16b, v11.16b, v3.16b, v26.16b //AES block 8k+11 - result - mov v3.16b, v25.16b //CTR block 8k+19 - - stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result - - stp q12, q13, [x2], #32 //AES block 8k+12, 8k+13 - store result - - cmp x0, x5 //LOOP CONTROL - stp q14, q15, [x2], #32 //AES block 8k+14, 8k+15 - store result - b.lt L192_enc_main_loop - -L192_enc_prepretail: //PREPRETAIL - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldp q26, q27, [x8, #0] //load rk0, rk1 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v8.16b //GHASH block 8k - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev32 v7.16b, v30.16b //CTR block 8k+15 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) - rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - ldp q27, q28, [x8, #64] //load rk4, rk5 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - ldr d16, [x10] //MODULO - load modulo constant - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - ldp q27, q28, [x8, #160] //load rk10, rk11 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ldr q26, [x8, #192] //load rk12 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - - aese v1.16b, v28.16b //AES block 8k+9 - round 11 - aese v7.16b, v28.16b //AES block 8k+15 - round 11 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v3.16b, v28.16b //AES block 8k+11 - round 11 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - aese v2.16b, v28.16b //AES block 8k+10 - round 11 - aese v0.16b, v28.16b //AES block 8k+8 - round 11 - - aese v6.16b, v28.16b //AES block 8k+14 - round 11 - aese v4.16b, v28.16b //AES block 8k+12 - round 11 - aese v5.16b, v28.16b //AES block 8k+13 - round 11 - -L192_enc_tail: //TAIL - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - - ldr q8, [x0], #16 //AES block 8k+8 - l3ad plaintext - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - - mov v29.16b, v26.16b - - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - cmp x5, #112 - -.long 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - b.gt L192_enc_blocks_more_than_7 - - cmp x5, #96 - mov v7.16b, v6.16b - movi v17.8b, #0 - - mov v6.16b, v5.16b - movi v19.8b, #0 - sub v30.4s, v30.4s, v31.4s - - mov v5.16b, v4.16b - mov v4.16b, v3.16b - mov v3.16b, v2.16b - - mov v2.16b, v1.16b - movi v18.8b, #0 - b.gt L192_enc_blocks_more_than_6 - - mov v7.16b, v6.16b - cmp x5, #80 - - mov v6.16b, v5.16b - mov v5.16b, v4.16b - mov v4.16b, v3.16b - - mov v3.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt L192_enc_blocks_more_than_5 - - cmp x5, #64 - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v1.16b - b.gt L192_enc_blocks_more_than_4 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v1.16b - - sub v30.4s, v30.4s, v31.4s - cmp x5, #48 - b.gt L192_enc_blocks_more_than_3 - - mov v7.16b, v6.16b - mov v6.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - - ldr q24, [x3, #96] //load h4k | h3k - cmp x5, #32 - b.gt L192_enc_blocks_more_than_2 - - sub v30.4s, v30.4s, v31.4s - - cmp x5, #16 - mov v7.16b, v1.16b - b.gt L192_enc_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b L192_enc_blocks_less_than_1 -L192_enc_blocks_more_than_7: //blocks left > 7 - st1 { v9.16b}, [x2], #16 //AES final-7 block - store result - - rev64 v8.16b, v9.16b //GHASH final-7 block - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - - ldr q9, [x0], #16 //AES final-6 block - load plaintext - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid -.long 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result -L192_enc_blocks_more_than_6: //blocks left > 6 - - st1 { v9.16b}, [x2], #16 //AES final-6 block - store result - - rev64 v8.16b, v9.16b //GHASH final-6 block - - ldr q9, [x0], #16 //AES final-5 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low -.long 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid -L192_enc_blocks_more_than_5: //blocks left > 5 - - st1 { v9.16b}, [x2], #16 //AES final-5 block - store result - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - - ldr q9, [x0], #16 //AES final-4 block - load plaintext - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - -.long 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid -L192_enc_blocks_more_than_4: //blocks left > 4 - - st1 { v9.16b}, [x2], #16 //AES final-4 block - store result - - rev64 v8.16b, v9.16b //GHASH final-4 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-3 block - load plaintext - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid -.long 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result -L192_enc_blocks_more_than_3: //blocks left > 3 - - ldr q24, [x3, #96] //load h4k | h3k - st1 { v9.16b}, [x2], #16 //AES final-3 block - store result - - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - - ldr q9, [x0], #16 //AES final-2 block - load plaintext - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - -.long 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high -L192_enc_blocks_more_than_2: //blocks left > 2 - - st1 { v9.16b}, [x2], #16 //AES final-2 block - store result - - rev64 v8.16b, v9.16b //GHASH final-2 block - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-1 block - load plaintext - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid -.long 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result -L192_enc_blocks_more_than_1: //blocks left > 1 - - ldr q22, [x3, #64] //load h1l | h1h - ext v22.16b, v22.16b, v22.16b, #8 - st1 { v9.16b}, [x2], #16 //AES final-1 block - store result - - rev64 v8.16b, v9.16b //GHASH final-1 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - - ldr q9, [x0], #16 //AES final block - load plaintext - ldr q21, [x3, #48] //load h2k | h1k - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - -.long 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high -L192_enc_blocks_less_than_1: //blocks left <= 1 - - mvn x6, xzr //temp0_x = 0xffffffffffffffff - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - mvn x7, xzr //temp1_x = 0xffffffffffffffff - - csel x13, x7, x6, lt - csel x14, x6, xzr, lt - - mov v0.d[1], x14 - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - mov v0.d[0], x13 //ctr0b is mask for last block - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - - rev64 v8.16b, v9.16b //GHASH final block - bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - - st1 { v9.16b}, [x2] //store all 16B - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v16.d[0], v8.d[1] //GHASH final block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - rev32 v30.16b, v30.16b - - str q30, [x16] //store the updated counter -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - - mov x0, x9 //return sizes - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -L192_enc_ret: - mov w0, #0x0 - ret - -.globl _unroll8_eor3_aes_gcm_dec_192_kernel - -.align 4 -_unroll8_eor3_aes_gcm_dec_192_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, L192_dec_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - mov x5, x9 - ld1 { v0.16b}, [x16] //CTR block 0 - ld1 { v19.16b}, [x3] - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - - sub x5, x5, #1 //byte_len - 1 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - - add x4, x0, x1, lsr #3 //end_input_ptr - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 9 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - - ldp q27, q28, [x8, #160] //load rk10, rk11 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 9 - add x5, x5, x0 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 9 - - cmp x0, x5 //check if we have <= 8 blocks - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 9 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 9 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 9 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 10 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 10 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 10 - ldr q26, [x8, #192] //load rk12 - - aese v0.16b, v28.16b //AES block 0 - round 11 - aese v1.16b, v28.16b //AES block 1 - round 11 - aese v4.16b, v28.16b //AES block 4 - round 11 - - aese v6.16b, v28.16b //AES block 6 - round 11 - aese v5.16b, v28.16b //AES block 5 - round 11 - aese v7.16b, v28.16b //AES block 7 - round 11 - - aese v2.16b, v28.16b //AES block 2 - round 11 - aese v3.16b, v28.16b //AES block 3 - round 11 - b.ge L192_dec_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext - -.long 0xce016921 //eor3 v1.16b, v9.16b, v1.16b, v26.16b //AES block 1 - result -.long 0xce006900 //eor3 v0.16b, v8.16b, v0.16b, v26.16b //AES block 0 - result - stp q0, q1, [x2], #32 //AES block 0, 1 - store result - - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 -.long 0xce036963 //eor3 v3.16b, v11.16b, v3.16b, v26.16b //AES block 3 - result - -.long 0xce026942 //eor3 v2.16b, v10.16b, v2.16b, v26.16b //AES block 2 - result - stp q2, q3, [x2], #32 //AES block 2, 3 - store result - ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - -.long 0xce046984 //eor3 v4.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - -.long 0xce0569a5 //eor3 v5.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result - stp q4, q5, [x2], #32 //AES block 4, 5 - store result - cmp x0, x5 //check if we have <= 8 blocks - -.long 0xce0669c6 //eor3 v6.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result -.long 0xce0769e7 //eor3 v7.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result - rev32 v4.16b, v30.16b //CTR block 12 - - add v30.4s, v30.4s, v31.4s //CTR block 12 - stp q6, q7, [x2], #32 //AES block 6, 7 - store result - b.ge L192_dec_prepretail //do prepretail - -L192_dec_main_loop: //main loop start - rev64 v9.16b, v9.16b //GHASH block 8k+1 - ldp q26, q27, [x8, #0] //load rk0, rk1 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - rev64 v8.16b, v8.16b //GHASH block 8k - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v12.16b, v12.16b //GHASH block 8k+4 - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - rev32 v7.16b, v30.16b //CTR block 8k+15 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - rev64 v10.16b, v10.16b //GHASH block 8k+2 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - rev64 v14.16b, v14.16b //GHASH block 8k+6 - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - ldp q28, q26, [x8, #128] //load rk8, rk9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - ldr d16, [x10] //MODULO - load modulo constant - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - rev32 v20.16b, v30.16b //CTR block 8k+16 - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - ldp q27, q28, [x8, #160] //load rk10, rk11 - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext - - rev32 v22.16b, v30.16b //CTR block 8k+17 - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext - - rev32 v23.16b, v30.16b //CTR block 8k+18 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - ldr q26, [x8, #192] //load rk12 - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - - aese v0.16b, v28.16b //AES block 8k+8 - round 11 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - aese v1.16b, v28.16b //AES block 8k+9 - round 11 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v6.16b, v28.16b //AES block 8k+14 - round 11 - aese v3.16b, v28.16b //AES block 8k+11 - round 11 - -.long 0xce006900 //eor3 v0.16b, v8.16b, v0.16b, v26.16b //AES block 8k+8 - result - rev32 v25.16b, v30.16b //CTR block 8k+19 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - - aese v4.16b, v28.16b //AES block 8k+12 - round 11 - aese v2.16b, v28.16b //AES block 8k+10 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - - aese v7.16b, v28.16b //AES block 8k+15 - round 11 - aese v5.16b, v28.16b //AES block 8k+13 - round 11 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - -.long 0xce016921 //eor3 v1.16b, v9.16b, v1.16b, v26.16b //AES block 8k+9 - result - stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result -.long 0xce036963 //eor3 v3.16b, v11.16b, v3.16b, v26.16b //AES block 8k+11 - result - -.long 0xce026942 //eor3 v2.16b, v10.16b, v2.16b, v26.16b //AES block 8k+10 - result -.long 0xce0769e7 //eor3 v7.16b, v15.16b, v7.16b, v26.16b //AES block 8k+15 - result - stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result - -.long 0xce0569a5 //eor3 v5.16b, v13.16b, v5.16b, v26.16b //AES block 8k+13 - result -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - mov v3.16b, v25.16b //CTR block 8k+19 - -.long 0xce046984 //eor3 v4.16b, v12.16b, v4.16b, v26.16b //AES block 8k+12 - result - stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result - cmp x0, x5 //LOOP CONTROL - -.long 0xce0669c6 //eor3 v6.16b, v14.16b, v6.16b, v26.16b //AES block 8k+14 - result - stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result - mov v0.16b, v20.16b //CTR block 8k+16 - - mov v1.16b, v22.16b //CTR block 8k+17 - mov v2.16b, v23.16b //CTR block 8k+18 - - rev32 v4.16b, v30.16b //CTR block 8k+20 - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - b.lt L192_dec_main_loop - -L192_dec_prepretail: //PREPRETAIL - ldp q26, q27, [x8, #0] //load rk0, rk1 - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v8.16b //GHASH block 8k - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - rev32 v7.16b, v30.16b //CTR block 8k+15 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - rev64 v13.16b, v13.16b //GHASH block 8k+5 - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - - rev64 v15.16b, v15.16b //GHASH block 8k+7 - -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - rev64 v12.16b, v12.16b //GHASH block 8k+4 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - - rev64 v14.16b, v14.16b //GHASH block 8k+6 - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - ldp q28, q26, [x8, #128] //load rk8, rk9 - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - - ldr d16, [x10] //MODULO - load modulo constant -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - ldp q27, q28, [x8, #160] //load rk10, rk11 - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ldr q26, [x8, #192] //load rk12 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - - aese v0.16b, v28.16b //AES block 8k+8 - round 11 -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - aese v5.16b, v28.16b //AES block 8k+13 - round 11 - - aese v2.16b, v28.16b //AES block 8k+10 - round 11 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - - aese v6.16b, v28.16b //AES block 8k+14 - round 11 - aese v4.16b, v28.16b //AES block 8k+12 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v3.16b, v28.16b //AES block 8k+11 - round 11 - aese v1.16b, v28.16b //AES block 8k+9 - round 11 - aese v7.16b, v28.16b //AES block 8k+15 - round 11 - -L192_dec_tail: //TAIL - - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - - mov v29.16b, v26.16b - - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - -.long 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result - cmp x5, #112 - b.gt L192_dec_blocks_more_than_7 - - mov v7.16b, v6.16b - movi v17.8b, #0 - sub v30.4s, v30.4s, v31.4s - - mov v6.16b, v5.16b - mov v5.16b, v4.16b - mov v4.16b, v3.16b - - cmp x5, #96 - movi v19.8b, #0 - mov v3.16b, v2.16b - - mov v2.16b, v1.16b - movi v18.8b, #0 - b.gt L192_dec_blocks_more_than_6 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v1.16b - - sub v30.4s, v30.4s, v31.4s - cmp x5, #80 - b.gt L192_dec_blocks_more_than_5 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - mov v4.16b, v1.16b - cmp x5, #64 - - sub v30.4s, v30.4s, v31.4s - b.gt L192_dec_blocks_more_than_4 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v1.16b - cmp x5, #48 - b.gt L192_dec_blocks_more_than_3 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - cmp x5, #32 - - mov v6.16b, v1.16b - ldr q24, [x3, #96] //load h4k | h3k - b.gt L192_dec_blocks_more_than_2 - - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v1.16b - cmp x5, #16 - b.gt L192_dec_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b L192_dec_blocks_less_than_1 -L192_dec_blocks_more_than_7: //blocks left > 7 - rev64 v8.16b, v9.16b //GHASH final-7 block - - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - ldr q9, [x0], #16 //AES final-6 block - load ciphertext - - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - st1 { v12.16b}, [x2], #16 //AES final-7 block - store result - -.long 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid - movi v16.8b, #0 //suppress further partial tag feed in -L192_dec_blocks_more_than_6: //blocks left > 6 - - rev64 v8.16b, v9.16b //GHASH final-6 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-5 block - load ciphertext - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - st1 { v12.16b}, [x2], #16 //AES final-6 block - store result -.long 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low -L192_dec_blocks_more_than_5: //blocks left > 5 - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - - ldr q9, [x0], #16 //AES final-4 block - load ciphertext - - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - movi v16.8b, #0 //suppress further partial tag feed in - st1 { v12.16b}, [x2], #16 //AES final-5 block - store result - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid -.long 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result -L192_dec_blocks_more_than_4: //blocks left > 4 - - rev64 v8.16b, v9.16b //GHASH final-4 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - - ldr q9, [x0], #16 //AES final-3 block - load ciphertext - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - st1 { v12.16b}, [x2], #16 //AES final-4 block - store result - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - -.long 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high -L192_dec_blocks_more_than_3: //blocks left > 3 - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-3 block - ldr q9, [x0], #16 //AES final-2 block - load ciphertext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high - movi v16.8b, #0 //suppress further partial tag feed in - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - - st1 { v12.16b}, [x2], #16 //AES final-3 block - store result - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid -.long 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - ldr q24, [x3, #96] //load h4k | h3k - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid -L192_dec_blocks_more_than_2: //blocks left > 2 - - rev64 v8.16b, v9.16b //GHASH final-2 block - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - ldr q9, [x0], #16 //AES final-1 block - load ciphertext - - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low - st1 { v12.16b}, [x2], #16 //AES final-2 block - store result - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid -.long 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result -L192_dec_blocks_more_than_1: //blocks left > 1 - - rev64 v8.16b, v9.16b //GHASH final-1 block - ldr q9, [x0], #16 //AES final block - load ciphertext - ldr q22, [x3, #64] //load h1l | h1h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - ldr q21, [x3, #48] //load h2k | h1k - - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - st1 { v12.16b}, [x2], #16 //AES final-1 block - store result - - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - -.long 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high -L192_dec_blocks_less_than_1: //blocks left <= 1 - - rev32 v30.16b, v30.16b - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - str q30, [x16] //store the updated counter - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - mvn x6, xzr //temp0_x = 0xffffffffffffffff - - and x1, x1, #127 //bit_length %= 128 - - mvn x7, xzr //temp1_x = 0xffffffffffffffff - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - - csel x13, x7, x6, lt - csel x14, x6, xzr, lt - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - - mov v0.d[1], x14 - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - - mov v0.d[0], x13 //ctr0b is mask for last block - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - - rev64 v8.16b, v9.16b //GHASH final block - - st1 { v12.16b}, [x2] //store all 16B - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v16.d[0], v8.d[1] //GHASH final block - mid - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - - eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up - -.long 0xce115652 //eor3 v18.16b, v18.16b, v17.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.long 0xce124673 //eor3 v19.16b, v19.16b, v18.16b, v17.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - - mov x0, x9 - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -L192_dec_ret: - mov w0, #0x0 - ret - -.globl _unroll8_eor3_aes_gcm_enc_256_kernel - -.align 4 -_unroll8_eor3_aes_gcm_enc_256_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, L256_enc_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - ld1 { v0.16b}, [x16] //CTR block 0 - - mov x5, x9 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - sub x5, x5, #1 //byte_len - 1 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - add x5, x5, x0 - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - ldp q27, q28, [x8, #160] //load rk10, rk11 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 9 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 9 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 9 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 9 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 10 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 10 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 9 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 10 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 10 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 11 - ldp q26, q27, [x8, #192] //load rk12, rk13 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 11 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 11 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 11 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 11 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 11 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 11 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 11 - - add v30.4s, v30.4s, v31.4s //CTR block 7 - ldr q28, [x8, #224] //load rk14 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 12 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 12 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 12 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 12 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 12 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 12 - - aese v2.16b, v27.16b //AES block 2 - round 13 - aese v1.16b, v27.16b //AES block 1 - round 13 - aese v4.16b, v27.16b //AES block 4 - round 13 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 12 - - aese v0.16b, v27.16b //AES block 0 - round 13 - aese v5.16b, v27.16b //AES block 5 - round 13 - - aese v6.16b, v27.16b //AES block 6 - round 13 - aese v7.16b, v27.16b //AES block 7 - round 13 - aese v3.16b, v27.16b //AES block 3 - round 13 - - add x4, x0, x1, lsr #3 //end_input_ptr - cmp x0, x5 //check if we have <= 8 blocks - b.ge L256_enc_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext - -.long 0xce007108 //eor3 v8.16b, v8.16b, v0.16b, v28.16b //AES block 0 - result - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - -.long 0xce017129 //eor3 v9.16b, v9.16b, v1.16b, v28.16b //AES block 1 - result -.long 0xce03716b //eor3 v11.16b, v11.16b, v3.16b, v28.16b //AES block 3 - result - - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 - ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext -.long 0xce02714a //eor3 v10.16b, v10.16b, v2.16b, v28.16b //AES block 2 - result - cmp x0, x5 //check if we have <= 8 blocks - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - stp q8, q9, [x2], #32 //AES block 0, 1 - store result - - stp q10, q11, [x2], #32 //AES block 2, 3 - store result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - -.long 0xce04718c //eor3 v12.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result - -.long 0xce0771ef //eor3 v15.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result -.long 0xce0671ce //eor3 v14.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result -.long 0xce0571ad //eor3 v13.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result - - stp q12, q13, [x2], #32 //AES block 4, 5 - store result - rev32 v4.16b, v30.16b //CTR block 12 - - stp q14, q15, [x2], #32 //AES block 6, 7 - store result - add v30.4s, v30.4s, v31.4s //CTR block 12 - b.ge L256_enc_prepretail //do prepretail - -L256_enc_main_loop: //main loop start - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - rev64 v8.16b, v8.16b //GHASH block 8k - - rev64 v12.16b, v12.16b //GHASH block 8k+4 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - rev32 v7.16b, v30.16b //CTR block 8k+15 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - eor v8.16b, v8.16b, v19.16b //PRE 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - rev64 v14.16b, v14.16b //GHASH block 8k+6 - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - ldp q27, q28, [x8, #64] //load rk4, rk5 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - ldp q26, q27, [x8, #96] //load rk6, rk7 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - - ldp q28, q26, [x8, #128] //load rk8, rk9 - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - - ldp q27, q28, [x8, #160] //load rk10, rk11 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - ldr d16, [x10] //MODULO - load modulo constant - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - - ldp q26, q27, [x8, #192] //load rk12, rk13 - rev32 v20.16b, v30.16b //CTR block 8k+16 - - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 - rev32 v22.16b, v30.16b //CTR block 8k+17 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 - ldr q28, [x8, #224] //load rk14 - aese v7.16b, v27.16b //AES block 8k+15 - round 13 - - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 - ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext - aese v2.16b, v27.16b //AES block 8k+10 - round 13 - aese v4.16b, v27.16b //AES block 8k+12 - round 13 - - rev32 v23.16b, v30.16b //CTR block 8k+18 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - aese v5.16b, v27.16b //AES block 8k+13 - round 13 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 - aese v3.16b, v27.16b //AES block 8k+11 - round 13 - cmp x0, x5 //LOOP CONTROL - -.long 0xce02714a //eor3 v10.16b, v10.16b, v2.16b, v28.16b //AES block 8k+10 - result - rev32 v25.16b, v30.16b //CTR block 8k+19 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - - aese v0.16b, v27.16b //AES block 8k+8 - round 13 - aese v6.16b, v27.16b //AES block 8k+14 - round 13 -.long 0xce0571ad //eor3 v13.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result - - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v1.16b, v27.16b //AES block 8k+9 - round 13 - -.long 0xce04718c //eor3 v12.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result - rev32 v4.16b, v30.16b //CTR block 8k+20 -.long 0xce03716b //eor3 v11.16b, v11.16b, v3.16b, v28.16b //AES block 8k+11 - result - - mov v3.16b, v25.16b //CTR block 8k+19 -.long 0xce017129 //eor3 v9.16b, v9.16b, v1.16b, v28.16b //AES block 8k+9 - result -.long 0xce007108 //eor3 v8.16b, v8.16b, v0.16b, v28.16b //AES block 8k+8 - result - - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result - mov v2.16b, v23.16b //CTR block 8k+18 - -.long 0xce0771ef //eor3 v15.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result -.long 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low - stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result - -.long 0xce0671ce //eor3 v14.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result - mov v1.16b, v22.16b //CTR block 8k+17 - stp q12, q13, [x2], #32 //AES block 4, 5 - store result - - stp q14, q15, [x2], #32 //AES block 6, 7 - store result - mov v0.16b, v20.16b //CTR block 8k+16 - b.lt L256_enc_main_loop - -L256_enc_prepretail: //PREPRETAIL - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldp q26, q27, [x8, #0] //load rk0, rk1 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v10.16b, v10.16b //GHASH block 8k+2 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - rev64 v13.16b, v13.16b //GHASH block 8k+5 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - rev32 v7.16b, v30.16b //CTR block 8k+15 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - rev64 v8.16b, v8.16b //GHASH block 8k - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - ldp q28, q26, [x8, #32] //load rk2, rk3 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - eor v8.16b, v8.16b, v19.16b //PRE 1 - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - rev64 v14.16b, v14.16b //GHASH block 8k+6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - rev64 v12.16b, v12.16b //GHASH block 8k+4 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - ldp q28, q26, [x8, #128] //load rk8, rk9 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - - ldp q27, q28, [x8, #160] //load rk10, rk11 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - ldr d16, [x10] //MODULO - load modulo constant - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 - - ldp q26, q27, [x8, #192] //load rk12, rk13 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 - ldr q28, [x8, #224] //load rk14 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 - aese v0.16b, v27.16b //AES block 8k+8 - round 13 - -.long 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low - aese v5.16b, v27.16b //AES block 8k+13 - round 13 - aese v1.16b, v27.16b //AES block 8k+9 - round 13 - - aese v3.16b, v27.16b //AES block 8k+11 - round 13 - aese v4.16b, v27.16b //AES block 8k+12 - round 13 - aese v7.16b, v27.16b //AES block 8k+15 - round 13 - - aese v2.16b, v27.16b //AES block 8k+10 - round 13 - aese v6.16b, v27.16b //AES block 8k+14 - round 13 -L256_enc_tail: //TAIL - - ldp q24, q25, [x3, #192] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - - ldr q8, [x0], #16 //AES block 8k+8 - load plaintext - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - mov v29.16b, v28.16b - - cmp x5, #112 -.long 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result - b.gt L256_enc_blocks_more_than_7 - - movi v19.8b, #0 - mov v7.16b, v6.16b - movi v17.8b, #0 - - mov v6.16b, v5.16b - mov v5.16b, v4.16b - mov v4.16b, v3.16b - - mov v3.16b, v2.16b - sub v30.4s, v30.4s, v31.4s - mov v2.16b, v1.16b - - movi v18.8b, #0 - cmp x5, #96 - b.gt L256_enc_blocks_more_than_6 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - cmp x5, #80 - - mov v5.16b, v4.16b - mov v4.16b, v3.16b - mov v3.16b, v1.16b - - sub v30.4s, v30.4s, v31.4s - b.gt L256_enc_blocks_more_than_5 - - mov v7.16b, v6.16b - sub v30.4s, v30.4s, v31.4s - - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - cmp x5, #64 - mov v4.16b, v1.16b - b.gt L256_enc_blocks_more_than_4 - - cmp x5, #48 - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt L256_enc_blocks_more_than_3 - - cmp x5, #32 - mov v7.16b, v6.16b - ldr q24, [x3, #96] //load h4k | h3k - - mov v6.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt L256_enc_blocks_more_than_2 - - mov v7.16b, v1.16b - - sub v30.4s, v30.4s, v31.4s - cmp x5, #16 - b.gt L256_enc_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b L256_enc_blocks_less_than_1 -L256_enc_blocks_more_than_7: //blocks left > 7 - st1 { v9.16b}, [x2], #16 //AES final-7 block - store result - - rev64 v8.16b, v9.16b //GHASH final-7 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-6 block - load plaintext - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid -.long 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low -L256_enc_blocks_more_than_6: //blocks left > 6 - - st1 { v9.16b}, [x2], #16 //AES final-6 block - store result - - rev64 v8.16b, v9.16b //GHASH final-6 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - ldr q9, [x0], #16 //AES final-5 block - load plaintext - - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid -.long 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high -L256_enc_blocks_more_than_5: //blocks left > 5 - - st1 { v9.16b}, [x2], #16 //AES final-5 block - store result - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - - ldr q9, [x0], #16 //AES final-4 block - load plaintext - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid -.long 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result -L256_enc_blocks_more_than_4: //blocks left > 4 - - st1 { v9.16b}, [x2], #16 //AES final-4 block - store result - - rev64 v8.16b, v9.16b //GHASH final-4 block - - ldr q9, [x0], #16 //AES final-3 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - -.long 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high -L256_enc_blocks_more_than_3: //blocks left > 3 - - st1 { v9.16b}, [x2], #16 //AES final-3 block - store result - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - ldr q24, [x3, #96] //load h4k | h3k - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - ldr q9, [x0], #16 //AES final-2 block - load plaintext - - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - -.long 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low -L256_enc_blocks_more_than_2: //blocks left > 2 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - - st1 { v9.16b}, [x2], #16 //AES final-2 block - store result - - rev64 v8.16b, v9.16b //GHASH final-2 block - ldr q9, [x0], #16 //AES final-1 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high -.long 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high - - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low -L256_enc_blocks_more_than_1: //blocks left > 1 - - st1 { v9.16b}, [x2], #16 //AES final-1 block - store result - - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-1 block - ldr q9, [x0], #16 //AES final block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - -.long 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high - - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - - ldr q21, [x3, #48] //load h2k | h1k - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid -L256_enc_blocks_less_than_1: //blocks left <= 1 - - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - mvn x6, xzr //temp0_x = 0xffffffffffffffff - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - mvn x7, xzr //temp1_x = 0xffffffffffffffff - - csel x14, x6, xzr, lt - csel x13, x7, x6, lt - - mov v0.d[0], x13 //ctr0b is mask for last block - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - mov v0.d[1], x14 - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - - rev64 v8.16b, v9.16b //GHASH final block - - rev32 v30.16b, v30.16b - bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - str q30, [x16] //store the updated counter - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - st1 { v9.16b}, [x2] //store all 16B - - ins v16.d[0], v8.d[1] //GHASH final block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.long 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - mov x0, x9 //return sizes - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -L256_enc_ret: - mov w0, #0x0 - ret - -.globl _unroll8_eor3_aes_gcm_dec_256_kernel - -.align 4 -_unroll8_eor3_aes_gcm_dec_256_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, L256_dec_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - ld1 { v0.16b}, [x16] //CTR block 0 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - mov x5, x9 - - sub x5, x5, #1 //byte_len - 1 - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 9 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - ldp q27, q28, [x8, #160] //load rk10, rk11 - add x4, x0, x1, lsr #3 //end_input_ptr - add x5, x5, x0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 9 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 9 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 9 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 9 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 9 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 10 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 10 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 10 - ldp q26, q27, [x8, #192] //load rk12, rk13 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 11 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 11 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 11 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 11 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 11 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 11 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 11 - ldr q28, [x8, #224] //load rk14 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 12 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 12 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 12 - - cmp x0, x5 //check if we have <= 8 blocks - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 12 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 12 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 12 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 12 - - aese v5.16b, v27.16b //AES block 5 - round 13 - aese v1.16b, v27.16b //AES block 1 - round 13 - aese v2.16b, v27.16b //AES block 2 - round 13 - - aese v0.16b, v27.16b //AES block 0 - round 13 - aese v4.16b, v27.16b //AES block 4 - round 13 - aese v6.16b, v27.16b //AES block 6 - round 13 - - aese v3.16b, v27.16b //AES block 3 - round 13 - aese v7.16b, v27.16b //AES block 7 - round 13 - b.ge L256_dec_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext - cmp x0, x5 //check if we have <= 8 blocks - -.long 0xce017121 //eor3 v1.16b, v9.16b, v1.16b, v28.16b //AES block 1 - result -.long 0xce007100 //eor3 v0.16b, v8.16b, v0.16b, v28.16b //AES block 0 - result - stp q0, q1, [x2], #32 //AES block 0, 1 - store result - - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 -.long 0xce037163 //eor3 v3.16b, v11.16b, v3.16b, v28.16b //AES block 3 - result - -.long 0xce0571a5 //eor3 v5.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result - -.long 0xce047184 //eor3 v4.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 - -.long 0xce027142 //eor3 v2.16b, v10.16b, v2.16b, v28.16b //AES block 2 - result - stp q2, q3, [x2], #32 //AES block 2, 3 - store result - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - -.long 0xce0671c6 //eor3 v6.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - stp q4, q5, [x2], #32 //AES block 4, 5 - store result - -.long 0xce0771e7 //eor3 v7.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result - stp q6, q7, [x2], #32 //AES block 6, 7 - store result - - rev32 v4.16b, v30.16b //CTR block 12 - add v30.4s, v30.4s, v31.4s //CTR block 12 - b.ge L256_dec_prepretail //do prepretail - -L256_dec_main_loop: //main loop start - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldp q26, q27, [x8, #0] //load rk0, rk1 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - rev64 v8.16b, v8.16b //GHASH block 8k - - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - rev64 v12.16b, v12.16b //GHASH block 8k+4 - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - rev32 v7.16b, v30.16b //CTR block 8k+15 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v14.16b, v14.16b //GHASH block 8k+6 - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - - ldp q27, q28, [x8, #160] //load rk10, rk11 - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - rev32 v20.16b, v30.16b //CTR block 8k+16 - ldr d16, [x10] //MODULO - load modulo constant - - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 - ldp q26, q27, [x8, #192] //load rk12, rk13 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 - -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - rev32 v22.16b, v30.16b //CTR block 8k+17 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 - - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 - - rev32 v23.16b, v30.16b //CTR block 8k+18 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 - - ldr q28, [x8, #224] //load rk14 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 - - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext - aese v1.16b, v27.16b //AES block 8k+9 - round 13 - aese v2.16b, v27.16b //AES block 8k+10 - round 13 - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext - aese v0.16b, v27.16b //AES block 8k+8 - round 13 - aese v5.16b, v27.16b //AES block 8k+13 - round 13 - - rev32 v25.16b, v30.16b //CTR block 8k+19 -.long 0xce027142 //eor3 v2.16b, v10.16b, v2.16b, v28.16b //AES block 8k+10 - result -.long 0xce017121 //eor3 v1.16b, v9.16b, v1.16b, v28.16b //AES block 8k+9 - result - - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - aese v7.16b, v27.16b //AES block 8k+15 - round 13 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v4.16b, v27.16b //AES block 8k+12 - round 13 - -.long 0xce0571a5 //eor3 v5.16b, v13.16b, v5.16b, v28.16b //AES block 8k+13 - result -.long 0xce007100 //eor3 v0.16b, v8.16b, v0.16b, v28.16b //AES block 8k+8 - result - aese v3.16b, v27.16b //AES block 8k+11 - round 13 - - stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result - mov v0.16b, v20.16b //CTR block 8k+16 -.long 0xce047184 //eor3 v4.16b, v12.16b, v4.16b, v28.16b //AES block 8k+12 - result - -.long 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low -.long 0xce037163 //eor3 v3.16b, v11.16b, v3.16b, v28.16b //AES block 8k+11 - result - stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result - - mov v3.16b, v25.16b //CTR block 8k+19 - mov v2.16b, v23.16b //CTR block 8k+18 - aese v6.16b, v27.16b //AES block 8k+14 - round 13 - - mov v1.16b, v22.16b //CTR block 8k+17 - stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result -.long 0xce0771e7 //eor3 v7.16b, v15.16b, v7.16b, v28.16b //AES block 8k+15 - result - -.long 0xce0671c6 //eor3 v6.16b, v14.16b, v6.16b, v28.16b //AES block 8k+14 - result - rev32 v4.16b, v30.16b //CTR block 8k+20 - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - - cmp x0, x5 //LOOP CONTROL - stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result - b.lt L256_dec_main_loop - -L256_dec_prepretail: //PREPRETAIL - ldp q26, q27, [x8, #0] //load rk0, rk1 - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v12.16b, v12.16b //GHASH block 8k+4 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - rev32 v6.16b, v30.16b //CTR block 8k+14 - rev64 v8.16b, v8.16b //GHASH block 8k - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - rev32 v7.16b, v30.16b //CTR block 8k+15 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - eor v8.16b, v8.16b, v19.16b //PRE 1 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - rev64 v14.16b, v14.16b //GHASH block 8k+6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - -.long 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - rev64 v13.16b, v13.16b //GHASH block 8k+5 - -.long 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - - ldp q28, q26, [x8, #128] //load rk8, rk9 - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 -.long 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - ldp q27, q28, [x8, #160] //load rk10, rk11 -.long 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low -.long 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - -.long 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high -.long 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - ldr d16, [x10] //MODULO - load modulo constant - -.long 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - -.long 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - ldp q26, q27, [x8, #192] //load rk12, rk13 - - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 - -.long 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - aese v3.16b, v27.16b //AES block 8k+11 - round 13 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 - ldr q28, [x8, #224] //load rk14 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 - - aese v4.16b, v27.16b //AES block 8k+12 - round 13 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 - - aese v6.16b, v27.16b //AES block 8k+14 - round 13 - aese v2.16b, v27.16b //AES block 8k+10 - round 13 - aese v1.16b, v27.16b //AES block 8k+9 - round 13 - - aese v5.16b, v27.16b //AES block 8k+13 - round 13 -.long 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v7.16b, v27.16b //AES block 8k+15 - round 13 - aese v0.16b, v27.16b //AES block 8k+8 - round 13 -L256_dec_tail: //TAIL - - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - cmp x5, #112 - - ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - mov v29.16b, v28.16b - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - -.long 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - b.gt L256_dec_blocks_more_than_7 - - mov v7.16b, v6.16b - sub v30.4s, v30.4s, v31.4s - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - mov v4.16b, v3.16b - movi v19.8b, #0 - - movi v17.8b, #0 - movi v18.8b, #0 - mov v3.16b, v2.16b - - cmp x5, #96 - mov v2.16b, v1.16b - b.gt L256_dec_blocks_more_than_6 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - cmp x5, #80 - sub v30.4s, v30.4s, v31.4s - - mov v4.16b, v3.16b - mov v3.16b, v1.16b - b.gt L256_dec_blocks_more_than_5 - - cmp x5, #64 - mov v7.16b, v6.16b - sub v30.4s, v30.4s, v31.4s - - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - mov v4.16b, v1.16b - b.gt L256_dec_blocks_more_than_4 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - cmp x5, #48 - - mov v6.16b, v5.16b - mov v5.16b, v1.16b - b.gt L256_dec_blocks_more_than_3 - - ldr q24, [x3, #96] //load h4k | h3k - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - - cmp x5, #32 - mov v6.16b, v1.16b - b.gt L256_dec_blocks_more_than_2 - - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v1.16b - cmp x5, #16 - b.gt L256_dec_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b L256_dec_blocks_less_than_1 -L256_dec_blocks_more_than_7: //blocks left > 7 - rev64 v8.16b, v9.16b //GHASH final-7 block - ldr q9, [x0], #16 //AES final-6 block - load ciphertext - st1 { v12.16b}, [x2], #16 //AES final-7 block - store result - - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid -.long 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid -L256_dec_blocks_more_than_6: //blocks left > 6 - - rev64 v8.16b, v9.16b //GHASH final-6 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - ldr q9, [x0], #16 //AES final-5 block - load ciphertext - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - st1 { v12.16b}, [x2], #16 //AES final-6 block - store result - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - -.long 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high -L256_dec_blocks_more_than_5: //blocks left > 5 - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - - ldr q9, [x0], #16 //AES final-4 block - load ciphertext - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - st1 { v12.16b}, [x2], #16 //AES final-5 block - store result - - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high -.long 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid - movi v16.8b, #0 //suppress further partial tag feed in -L256_dec_blocks_more_than_4: //blocks left > 4 - - rev64 v8.16b, v9.16b //GHASH final-4 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - ldr q9, [x0], #16 //AES final-3 block - load ciphertext - - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - st1 { v12.16b}, [x2], #16 //AES final-4 block - store result - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid -.long 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result -L256_dec_blocks_more_than_3: //blocks left > 3 - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - ldr q9, [x0], #16 //AES final-2 block - load ciphertext - ldr q24, [x3, #96] //load h4k | h3k - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - st1 { v12.16b}, [x2], #16 //AES final-3 block - store result - -.long 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid -L256_dec_blocks_more_than_2: //blocks left > 2 - - rev64 v8.16b, v9.16b //GHASH final-2 block - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q9, [x0], #16 //AES final-1 block - load ciphertext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - st1 { v12.16b}, [x2], #16 //AES final-2 block - store result -.long 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high -L256_dec_blocks_more_than_1: //blocks left > 1 - - rev64 v8.16b, v9.16b //GHASH final-1 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - ldr q9, [x0], #16 //AES final block - load ciphertext - st1 { v12.16b}, [x2], #16 //AES final-1 block - store result - - ldr q21, [x3, #48] //load h2k | h1k - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - -.long 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid -L256_dec_blocks_less_than_1: //blocks left <= 1 - - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - mvn x6, xzr //temp0_x = 0xffffffffffffffff - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - rev32 v30.16b, v30.16b - str q30, [x16] //store the updated counter - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - mvn x7, xzr //temp1_x = 0xffffffffffffffff - - csel x14, x6, xzr, lt - csel x13, x7, x6, lt - - mov v0.d[0], x13 //ctr0b is mask for last block - mov v0.d[1], x14 - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - - rev64 v8.16b, v9.16b //GHASH final block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v16.d[0], v8.d[1] //GHASH final block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - st1 { v12.16b}, [x2] //store all 16B - - eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up - - eor v21.16b, v17.16b, v21.16b //MODULO - fold into mid - eor v18.16b, v18.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - - ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - eor v19.16b, v19.16b, v17.16b //MODULO - fold into low - - eor v19.16b, v19.16b, v18.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - mov x0, x9 - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -L256_dec_ret: - mov w0, #0x0 - ret - -.byte 65,69,83,32,71,67,77,32,109,111,100,117,108,101,32,102,111,114,32,65,82,77,118,56,44,32,83,80,68,88,32,66,83,68,45,51,45,67,108,97,117,115,101,32,98,121,32,60,120,105,97,111,107,97,110,103,46,113,105,97,110,64,97,114,109,46,99,111,109,62,0 -.align 2 -.align 2 -#endif diff --git a/openssl/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8_64.S b/openssl/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8_64.S index 293dd7f8a..777c55f9f 100644 --- a/openssl/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8_64.S +++ b/openssl/src/crypto/modes/gen/darwin_arm64/aes-gcm-armv8_64.S @@ -7,7 +7,6 @@ .align 4 _aes_gcm_enc_128_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, L128_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -20,36 +19,28 @@ _aes_gcm_enc_128_kernel: stp d14, d15, [sp, #96] ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif ldp x13, x14, [x8, #160] //load rk10 -#ifdef __AARCH64EB__ - ror x13, x13, #32 - ror x14, x14, #32 -#endif + ld1 {v11.16b}, [x3] ext v11.16b, v11.16b, v11.16b, #8 rev64 v11.16b, v11.16b lsr x5, x1, #3 //byte_len mov x15, x5 - ld1 {v18.4s}, [x8], #16 //load rk0 + ldr q27, [x8, #144] //load rk9 add x4, x0, x1, lsr #3 //end_input_ptr sub x5, x5, #1 //byte_len - 1 lsr x12, x11, #32 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif + fmov d1, x10 //CTR block 1 rev w12, w12 //rev_ctr32 add w12, w12, #1 //increment rev_ctr32 orr w11, w11, w11 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q18, [x8, #0] //load rk0 rev w9, w12 //CTR block 1 add w12, w12, #1 //CTR block 1 @@ -69,33 +60,30 @@ _aes_gcm_enc_128_kernel: rev w9, w12 //CTR block 3 orr x9, x11, x9, lsl #32 //CTR block 3 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q19, [x8, #16] //load rk1 add w12, w12, #1 //CTR block 3 fmov v3.d[1], x9 //CTR block 3 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q20, [x8, #32] //load rk2 aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q26, [x8, #128] //load rk8 aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q21, [x8, #48] //load rk3 aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 @@ -103,11 +91,11 @@ _aes_gcm_enc_128_kernel: aese v0.16b, v19.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q24, [x8, #96] //load rk6 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q25, [x8, #112] //load rk7 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 @@ -115,14 +103,12 @@ _aes_gcm_enc_128_kernel: aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q23, [x8, #80] //load rk5 aese v1.16b, v20.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 @@ -139,7 +125,7 @@ _aes_gcm_enc_128_kernel: aese v2.16b, v21.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q22, [x8, #64] //load rk4 aese v3.16b, v21.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 @@ -222,25 +208,13 @@ _aes_gcm_enc_128_kernel: b.ge L128_enc_tail //handle tail ldp x6, x7, [x0, #0] //AES block 0 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + ldp x21, x22, [x0, #32] //AES block 2 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + ldp x19, x20, [x0, #16] //AES block 1 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + ldp x23, x24, [x0, #48] //AES block 3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + eor x6, x6, x13 //AES block 0 - round 10 low eor x7, x7, x14 //AES block 0 - round 10 high @@ -305,10 +279,6 @@ _aes_gcm_enc_128_kernel: L128_enc_main_loop: //main loop start ldp x23, x24, [x0, #48] //AES block 4k+3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free) rev64 v6.16b, v6.16b //GHASH block 4k+2 (t0, t1, and t2 free) @@ -343,10 +313,7 @@ L128_enc_main_loop: //main loop start pmull2 v28.1q, v5.2d, v14.2d //GHASH block 4k+1 - high eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid ldp x6, x7, [x0, #0] //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + aese v0.16b, v19.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 1 rev w9, w12 //CTR block 4k+8 @@ -428,10 +395,7 @@ L128_enc_main_loop: //main loop start aese v1.16b, v23.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 5 ldp x19, x20, [x0, #16] //AES block 4k+5 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + aese v3.16b, v21.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 3 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid @@ -439,10 +403,7 @@ L128_enc_main_loop: //main loop start aese v0.16b, v23.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 5 ldp x21, x22, [x0, #32] //AES block 4k+6 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + pmull v31.1q, v9.1d, v8.1d //MODULO - top 64b align with mid eor v11.16b, v11.16b, v29.16b //GHASH block 4k+3 - low @@ -751,10 +712,7 @@ L128_enc_tail: //TAIL sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldp x6, x7, [x0], #16 //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + cmp x5, #48 ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag @@ -792,10 +750,7 @@ L128_enc_blocks_more_than_3: //blocks left > 3 st1 { v5.16b}, [x2], #16 //AES final-3 block - store result ldp x6, x7, [x0], #16 //AES final-2 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-3 block eor v4.16b, v4.16b, v8.16b //feed in partial tag @@ -824,10 +779,7 @@ L128_enc_blocks_more_than_2: //blocks left > 2 rev64 v4.16b, v5.16b //GHASH final-2 block ldp x6, x7, [x0], #16 //AES final-1 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor v4.16b, v4.16b, v8.16b //feed in partial tag eor x6, x6, x13 //AES final-1 block - round 10 low @@ -861,10 +813,7 @@ L128_enc_blocks_more_than_1: //blocks left > 1 rev64 v4.16b, v5.16b //GHASH final-1 block ldp x6, x7, [x0], #16 //AES final block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor v4.16b, v4.16b, v8.16b //feed in partial tag eor x7, x7, x14 //AES final block - round 10 high @@ -927,11 +876,9 @@ L128_enc_blocks_less_than_1: //blocks left <= 1 ld1 { v18.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored eor v8.8b, v8.8b, v4.8b //GHASH final block - mid -#ifndef __AARCH64EB__ + rev w9, w12 -#else - mov w9, w12 -#endif + pmull2 v20.1q, v4.2d, v12.2d //GHASH final block - high pmull v8.1q, v8.1d, v16.1d //GHASH final block - mid @@ -990,7 +937,6 @@ L128_enc_ret: .align 4 _aes_gcm_dec_128_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, L128_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -1005,29 +951,20 @@ _aes_gcm_dec_128_kernel: lsr x5, x1, #3 //byte_len mov x15, x5 ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #160] //load rk10 -#ifdef __AARCH64EB__ - ror x14, x14, 32 - ror x13, x13, 32 -#endif + sub x5, x5, #1 //byte_len - 1 - ld1 {v18.4s}, [x8], #16 //load rk0 + ldr q18, [x8, #0] //load rk0 and x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) ld1 { v0.16b}, [x16] //special case vector load initial counter so we can start first AES block as quickly as possible ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif + lsr x12, x11, #32 fmov d2, x10 //CTR block 2 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q19, [x8, #16] //load rk1 orr w11, w11, w11 rev w12, w12 //rev_ctr32 @@ -1039,7 +976,7 @@ _aes_gcm_dec_128_kernel: rev w9, w12 //CTR block 1 orr x9, x11, x9, lsl #32 //CTR block 1 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q20, [x8, #32] //load rk2 add w12, w12, #1 //CTR block 1 fmov v1.d[1], x9 //CTR block 1 @@ -1062,19 +999,19 @@ _aes_gcm_dec_128_kernel: aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q21, [x8, #48] //load rk3 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q24, [x8, #96] //load rk6 aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q25, [x8, #112] //load rk7 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q22, [x8, #64] //load rk4 aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 @@ -1084,6 +1021,7 @@ _aes_gcm_dec_128_kernel: aese v1.16b, v20.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 + ldp x13, x14, [x8, #160] //load rk10 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 @@ -1093,7 +1031,7 @@ _aes_gcm_dec_128_kernel: aese v0.16b, v21.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q23, [x8, #80] //load rk5 aese v1.16b, v21.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 @@ -1103,7 +1041,7 @@ _aes_gcm_dec_128_kernel: aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q27, [x8, #144] //load rk9 aese v1.16b, v22.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 @@ -1114,12 +1052,11 @@ _aes_gcm_dec_128_kernel: aese v2.16b, v21.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v0.16b, v22.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q26, [x8, #128] //load rk8 aese v1.16b, v23.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 @@ -1136,9 +1073,8 @@ _aes_gcm_dec_128_kernel: aese v2.16b, v23.16b aesmc v2.16b, v2.16b //AES block 2 - round 5 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v3.16b, v23.16b aesmc v3.16b, v3.16b //AES block 3 - round 5 @@ -1156,9 +1092,7 @@ _aes_gcm_dec_128_kernel: trn1 v8.2d, v12.2d, v13.2d //h2h | h1h ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif trn2 v16.2d, v12.2d, v13.2d //h2l | h1l add x5, x5, x0 @@ -1200,10 +1134,12 @@ _aes_gcm_dec_128_kernel: eor v17.16b, v17.16b, v9.16b //h4k | h3k b.ge L128_dec_tail //handle tail - ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0 - load ciphertext; AES block 1 - load ciphertext + ldr q5, [x0, #16] //AES block 1 - load ciphertext + + ldr q4, [x0, #0] //AES block 0 - load ciphertext eor v1.16b, v5.16b, v1.16b //AES block 1 - result - ld1 {v6.16b}, [x0], #16 //AES block 2 - load ciphertext + ldr q6, [x0, #32] //AES block 2 - load ciphertext eor v0.16b, v4.16b, v0.16b //AES block 0 - result rev64 v4.16b, v4.16b //GHASH block 0 @@ -1211,9 +1147,10 @@ _aes_gcm_dec_128_kernel: orr x9, x11, x9, lsl #32 //CTR block 4 add w12, w12, #1 //CTR block 4 - ld1 {v7.16b}, [x0], #16 //AES block 3 - load ciphertext + ldr q7, [x0, #48] //AES block 3 - load ciphertext rev64 v5.16b, v5.16b //GHASH block 1 + add x0, x0, #64 //AES input_ptr update mov x19, v1.d[0] //AES block 1 - mov low mov x20, v1.d[1] //AES block 1 - mov high @@ -1228,9 +1165,7 @@ _aes_gcm_dec_128_kernel: fmov v0.d[1], x9 //CTR block 4 rev w9, w12 //CTR block 5 eor x19, x19, x13 //AES block 1 - round 10 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif + fmov d1, x10 //CTR block 5 add w12, w12, #1 //CTR block 5 orr x9, x11, x9, lsl #32 //CTR block 5 @@ -1242,19 +1177,10 @@ _aes_gcm_dec_128_kernel: orr x9, x11, x9, lsl #32 //CTR block 6 eor x20, x20, x14 //AES block 1 - round 10 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif eor x6, x6, x13 //AES block 0 - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v2.16b, v6.16b, v2.16b //AES block 2 - result eor x7, x7, x14 //AES block 0 - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif stp x6, x7, [x2], #16 //AES block 0 - store result stp x19, x20, [x2], #16 //AES block 1 - store result @@ -1322,14 +1248,9 @@ L128_dec_main_loop: //main loop start aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 1 eor x23, x23, x13 //AES block 4k+3 - round 10 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + pmull v30.1q, v30.1d, v17.1d //GHASH block 4k+1 - mid eor x22, x22, x14 //AES block 4k+2 - round 10 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif mov d31, v6.d[1] //GHASH block 4k+2 - mid aese v0.16b, v19.16b @@ -1367,9 +1288,7 @@ L128_dec_main_loop: //main loop start pmull2 v31.1q, v31.2d, v16.2d //GHASH block 4k+2 - mid eor x24, x24, x14 //AES block 4k+3 - round 10 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif + aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 2 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid @@ -1377,9 +1296,7 @@ L128_dec_main_loop: //main loop start aese v1.16b, v23.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 5 eor x21, x21, x13 //AES block 4k+2 - round 10 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif + aese v0.16b, v23.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 5 movi v8.8b, #0xc2 @@ -1401,7 +1318,7 @@ L128_dec_main_loop: //main loop start pmull v30.1q, v30.1d, v16.1d //GHASH block 4k+3 - mid eor v9.16b, v9.16b, v4.16b //GHASH block 4k+3 - high - ld1 {v4.16b}, [x0], #16 //AES block 4k+3 - load ciphertext + ldr q4, [x0, #0] //AES block 4k+4 - load ciphertext aese v1.16b, v25.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 7 @@ -1428,7 +1345,7 @@ L128_dec_main_loop: //main loop start rev w9, w12 //CTR block 4k+8 pmull v31.1q, v9.1d, v8.1d //MODULO - top 64b align with mid - ld1 {v5.16b}, [x0], #16 //AES block 4k+4 - load ciphertext + ldr q5, [x0, #16] //AES block 4k+5 - load ciphertext ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment aese v0.16b, v27.16b //AES block 4k+4 - round 9 @@ -1446,7 +1363,7 @@ L128_dec_main_loop: //main loop start aese v3.16b, v23.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 5 - ld1 {v6.16b}, [x0], #16 //AES block 4k+5 - load ciphertext + ldr q6, [x0, #32] //AES block 4k+6 - load ciphertext add w12, w12, #1 //CTR block 4k+8 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid @@ -1454,10 +1371,11 @@ L128_dec_main_loop: //main loop start aese v2.16b, v25.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 7 - ld1 {v7.16b}, [x0], #16 //AES block 4k+6 - load ciphertext + ldr q7, [x0, #48] //AES block 4k+3 - load ciphertext aese v3.16b, v24.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 6 + add x0, x0, #64 //AES input_ptr update rev64 v5.16b, v5.16b //GHASH block 4k+5 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid @@ -1482,15 +1400,11 @@ L128_dec_main_loop: //main loop start aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 8 eor x7, x7, x14 //AES block 4k+4 - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor v11.16b, v11.16b, v8.16b //MODULO - fold into low mov x20, v1.d[1] //AES block 4k+5 - mov high eor x6, x6, x13 //AES block 4k+4 - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif + eor v2.16b, v6.16b, v2.16b //AES block 4k+6 - result mov x19, v1.d[0] //AES block 4k+5 - mov low add w12, w12, #1 //CTR block 4k+9 @@ -1507,15 +1421,9 @@ L128_dec_main_loop: //main loop start add w12, w12, #1 //CTR block 4k+10 eor x20, x20, x14 //AES block 4k+5 - round 10 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif stp x6, x7, [x2], #16 //AES block 4k+4 - store result eor x19, x19, x13 //AES block 4k+5 - round 10 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif stp x19, x20, [x2], #16 //AES block 4k+5 - store result orr x9, x11, x9, lsl #32 //CTR block 4k+10 @@ -1620,14 +1528,9 @@ L128_dec_prepretail: //PREPRETAIL aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 2 eor x23, x23, x13 //AES block 4k+3 - round 10 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + pmull v30.1q, v30.1d, v16.1d //GHASH block 4k+3 - mid eor x21, x21, x13 //AES block 4k+2 - round 10 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif eor v11.16b, v11.16b, v29.16b //GHASH block 4k+3 - low aese v2.16b, v21.16b @@ -1700,9 +1603,7 @@ L128_dec_prepretail: //PREPRETAIL pmull v8.1q, v10.1d, v8.1d //MODULO - mid 64b align with low eor x24, x24, x14 //AES block 4k+3 - round 10 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif + aese v2.16b, v25.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 7 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment @@ -1720,9 +1621,7 @@ L128_dec_prepretail: //PREPRETAIL aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 8 eor x22, x22, x14 //AES block 4k+2 - round 10 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif + aese v0.16b, v27.16b //AES block 4k+4 - round 9 stp x21, x22, [x2], #16 //AES block 4k+2 - store result @@ -1746,14 +1645,9 @@ L128_dec_tail: //TAIL cmp x5, #48 eor x7, x7, x14 //AES block 4k+4 - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag eor x6, x6, x13 //AES block 4k+4 - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif b.gt L128_dec_blocks_more_than_3 mov v3.16b, v2.16b @@ -1797,14 +1691,9 @@ L128_dec_blocks_more_than_3: //blocks left > 3 movi v8.8b, #0 //suppress further partial tag feed in eor x7, x7, x14 //AES final-2 block - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + pmull v10.1q, v22.1d, v10.1d //GHASH final-3 block - mid eor x6, x6, x13 //AES final-2 block - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif L128_dec_blocks_more_than_2: //blocks left > 2 rev64 v4.16b, v5.16b //GHASH final-2 block @@ -1830,18 +1719,12 @@ L128_dec_blocks_more_than_2: //blocks left > 2 pmull v22.1q, v22.1d, v17.1d //GHASH final-2 block - mid eor x6, x6, x13 //AES final-1 block - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v11.16b, v11.16b, v21.16b //GHASH final-2 block - low eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid eor x7, x7, x14 //AES final-1 block - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif L128_dec_blocks_more_than_1: //blocks left > 1 rev64 v4.16b, v5.16b //GHASH final-1 block @@ -1872,13 +1755,8 @@ L128_dec_blocks_more_than_1: //blocks left > 1 eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high eor x7, x7, x14 //AES final block - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES final block - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid L128_dec_blocks_less_than_1: //blocks left <= 1 @@ -1924,11 +1802,7 @@ L128_dec_blocks_less_than_1: //blocks left <= 1 bic x4, x4, x9 //mask out low existing bytes and x6, x6, x9 -#ifndef __AARCH64EB__ rev w9, w12 -#else - mov w9, w12 -#endif eor v10.16b, v10.16b, v8.16b //GHASH final block - mid movi v8.8b, #0xc2 @@ -1983,7 +1857,6 @@ L128_dec_ret: .align 4 _aes_gcm_enc_192_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, L192_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -1996,26 +1869,18 @@ _aes_gcm_enc_192_kernel: stp d14, d15, [sp, #96] ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #192] //load rk12 -#ifdef __AARCH64EB__ - ror x13, x13, #32 - ror x14, x14, #32 -#endif - ld1 {v18.4s}, [x8], #16 //load rk0 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q23, [x8, #80] //load rk5 + + ldr q22, [x8, #64] //load rk4 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q26, [x8, #128] //load rk8 lsr x12, x11, #32 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q24, [x8, #96] //load rk6 orr w11, w11, w11 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q25, [x8, #112] //load rk7 rev w12, w12 //rev_ctr32 add w12, w12, #1 //increment rev_ctr32 @@ -2039,13 +1904,15 @@ _aes_gcm_enc_192_kernel: rev w9, w12 //CTR block 3 orr x9, x11, x9, lsl #32 //CTR block 3 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q18, [x8, #0] //load rk0 fmov v3.d[1], x9 //CTR block 3 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q21, [x8, #48] //load rk3 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldp x13, x14, [x8, #192] //load rk12 + + ldr q19, [x8, #16] //load rk1 aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 @@ -2055,38 +1922,35 @@ _aes_gcm_enc_192_kernel: aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q29, [x8, #176] //load rk11 aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif + aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q20, [x8, #32] //load rk2 aese v0.16b, v19.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 - ld1 {v28.4s}, [x8], #16 //load rk10 + ldr q28, [x8, #160] //load rk10 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 - ld1 {v29.4s}, [x8], #16 //load rk11 + ldr q27, [x8, #144] //load rk9 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 @@ -2143,9 +2007,8 @@ _aes_gcm_enc_192_kernel: aese v2.16b, v24.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif + aese v1.16b, v24.16b aesmc v1.16b, v1.16b //AES block 1 - round 6 @@ -2225,26 +2088,13 @@ _aes_gcm_enc_192_kernel: rev w9, w12 //CTR block 4 ldp x6, x7, [x0, #0] //AES block 0 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + orr x9, x11, x9, lsl #32 //CTR block 4 ldp x21, x22, [x0, #32] //AES block 2 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + ldp x23, x24, [x0, #48] //AES block 3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + ldp x19, x20, [x0, #16] //AES block 1 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif add x0, x0, #64 //AES input_ptr update cmp x0, x5 //check if we have <= 8 blocks @@ -2316,10 +2166,7 @@ L192_enc_main_loop: //main loop start aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 0 ldp x19, x20, [x0, #16] //AES block 4k+5 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + ext v11.16b, v11.16b, v11.16b, #8 //PRE 0 fmov d3, x10 //CTR block 4k+3 rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free) @@ -2331,17 +2178,11 @@ L192_enc_main_loop: //main loop start pmull2 v30.1q, v5.2d, v14.2d //GHASH block 4k+1 - high rev64 v7.16b, v7.16b //GHASH block 4k+3 (t0, t1, t2 and t3 free) ldp x21, x22, [x0, #32] //AES block 4k+6 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 0 ldp x23, x24, [x0, #48] //AES block 4k+3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + pmull v31.1q, v5.1d, v14.1d //GHASH block 4k+1 - low eor v4.16b, v4.16b, v11.16b //PRE 1 @@ -2434,10 +2275,7 @@ L192_enc_main_loop: //main loop start aese v1.16b, v22.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 4 ldp x6, x7, [x0, #0] //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + aese v0.16b, v24.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 6 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+2 - low @@ -2802,10 +2640,7 @@ L192_enc_tail: //TAIL sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldp x6, x7, [x0], #16 //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor x6, x6, x13 //AES block 4k+4 - round 12 low eor x7, x7, x14 //AES block 4k+4 - round 12 high @@ -2842,10 +2677,7 @@ L192_enc_blocks_more_than_3: //blocks left > 3 st1 { v5.16b}, [x2], #16 //AES final-3 block - store result ldp x6, x7, [x0], #16 //AES final-2 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-3 block eor x6, x6, x13 //AES final-2 block - round 12 low @@ -2876,10 +2708,7 @@ L192_enc_blocks_more_than_2: //blocks left > 2 rev64 v4.16b, v5.16b //GHASH final-2 block ldp x6, x7, [x0], #16 //AES final-1 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor v4.16b, v4.16b, v8.16b //feed in partial tag eor x7, x7, x14 //AES final-1 block - round 12 high @@ -2910,10 +2739,7 @@ L192_enc_blocks_more_than_1: //blocks left > 1 st1 { v5.16b}, [x2], #16 //AES final-1 block - store result ldp x6, x7, [x0], #16 //AES final block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-1 block eor x6, x6, x13 //AES final block - round 12 low @@ -2945,11 +2771,7 @@ L192_enc_blocks_more_than_1: //blocks left > 1 L192_enc_blocks_less_than_1: //blocks left <= 1 ld1 { v18.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored -#ifndef __AARCH64EB__ rev w9, w12 -#else - mov w9, w12 -#endif and x1, x1, #127 //bit_length %= 128 sub x1, x1, #128 //bit_length -= 128 @@ -3041,7 +2863,6 @@ L192_enc_ret: .align 4 _aes_gcm_dec_192_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, L192_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -3055,22 +2876,14 @@ _aes_gcm_dec_192_kernel: add x4, x0, x1, lsr #3 //end_input_ptr ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #192] //load rk12 -#ifdef __AARCH64EB__ - ror x13, x13, #32 - ror x14, x14, #32 -#endif + ld1 { v0.16b}, [x16] //special case vector load initial counter so we can start first AES block as quickly as possible - ld1 {v18.4s}, [x8], #16 //load rk0 + ldr q18, [x8, #0] //load rk0 lsr x5, x1, #3 //byte_len mov x15, x5 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q20, [x8, #32] //load rk2 lsr x12, x11, #32 orr w11, w11, w11 @@ -3080,7 +2893,7 @@ _aes_gcm_dec_192_kernel: fmov d1, x10 //CTR block 1 add w12, w12, #1 //increment rev_ctr32 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q19, [x8, #16] //load rk1 aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 @@ -3088,7 +2901,7 @@ _aes_gcm_dec_192_kernel: add w12, w12, #1 //CTR block 1 orr x9, x11, x9, lsl #32 //CTR block 1 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q21, [x8, #48] //load rk3 fmov v1.d[1], x9 //CTR block 1 rev w9, w12 //CTR block 2 @@ -3106,57 +2919,54 @@ _aes_gcm_dec_192_kernel: fmov v3.d[1], x9 //CTR block 3 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q26, [x8, #128] //load rk8 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q29, [x8, #176] //load rk11 aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif + aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 + ldp x13, x14, [x8, #192] //load rk12 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q28, [x8, #160] //load rk10 aese v0.16b, v21.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q27, [x8, #144] //load rk9 aese v1.16b, v20.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q25, [x8, #112] //load rk7 aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q22, [x8, #64] //load rk4 aese v2.16b, v21.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 @@ -3174,7 +2984,7 @@ _aes_gcm_dec_192_kernel: aese v0.16b, v22.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 - ld1 {v28.4s}, [x8], #16 //load rk10 + ldr q23, [x8, #80] //load rk5 aese v1.16b, v22.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 @@ -3189,7 +2999,7 @@ _aes_gcm_dec_192_kernel: aese v0.16b, v23.16b aesmc v0.16b, v0.16b //AES block 0 - round 5 - ld1 {v29.4s}, [x8], #16 //load rk11 + ldr q24, [x8, #96] //load rk6 aese v1.16b, v23.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 @@ -3276,13 +3086,17 @@ _aes_gcm_dec_192_kernel: aese v0.16b, v29.16b //AES block 0 - round 11 b.ge L192_dec_tail //handle tail - ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0,1 - load ciphertext + ldr q5, [x0, #16] //AES block 1 - load ciphertext + + ldr q4, [x0, #0] //AES block 0 - load ciphertext eor v1.16b, v5.16b, v1.16b //AES block 1 - result eor v0.16b, v4.16b, v0.16b //AES block 0 - result rev w9, w12 //CTR block 4 - ld1 {v6.16b, v7.16b}, [x0], #32 //AES block 2,3 - load ciphertext + ldr q7, [x0, #48] //AES block 3 - load ciphertext + + ldr q6, [x0, #32] //AES block 2 - load ciphertext mov x19, v1.d[0] //AES block 1 - mov low @@ -3294,35 +3108,27 @@ _aes_gcm_dec_192_kernel: mov x7, v0.d[1] //AES block 0 - mov high rev64 v4.16b, v4.16b //GHASH block 0 + add x0, x0, #64 //AES input_ptr update fmov d0, x10 //CTR block 4 rev64 v5.16b, v5.16b //GHASH block 1 cmp x0, x5 //check if we have <= 8 blocks eor x19, x19, x13 //AES block 1 - round 12 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif fmov v0.d[1], x9 //CTR block 4 rev w9, w12 //CTR block 5 orr x9, x11, x9, lsl #32 //CTR block 5 fmov d1, x10 //CTR block 5 eor x20, x20, x14 //AES block 1 - round 12 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif + add w12, w12, #1 //CTR block 5 fmov v1.d[1], x9 //CTR block 5 eor x6, x6, x13 //AES block 0 - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif + rev w9, w12 //CTR block 6 eor x7, x7, x14 //AES block 0 - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + stp x6, x7, [x2], #16 //AES block 0 - store result orr x9, x11, x9, lsl #32 //CTR block 6 @@ -3385,9 +3191,7 @@ L192_dec_main_loop: //main loop start aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 2 eor x22, x22, x14 //AES block 4k+2 - round 12 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 1 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid @@ -3404,9 +3208,7 @@ L192_dec_main_loop: //main loop start pmull v4.1q, v4.1d, v17.1d //GHASH block 4k+1 - mid eor v11.16b, v11.16b, v31.16b //GHASH block 4k+1 - low eor x21, x21, x13 //AES block 4k+2 - round 12 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif + aese v1.16b, v22.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 4 @@ -3508,18 +3310,16 @@ L192_dec_main_loop: //main loop start aese v2.16b, v24.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 6 - ld1 {v4.16b}, [x0], #16 //AES block 4k+4 - load ciphertext + ldr q6, [x0, #32] //AES block 4k+6 - load ciphertext aese v3.16b, v24.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 6 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up pmull v31.1q, v9.1d, v8.1d //MODULO - top 64b align with mid - ld1 {v5.16b}, [x0], #16 //AES block 4k+5 - load ciphertext + ldr q7, [x0, #48] //AES block 4k+7 - load ciphertext eor x23, x23, x13 //AES block 4k+3 - round 12 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + aese v2.16b, v25.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 7 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment @@ -3533,10 +3333,10 @@ L192_dec_main_loop: //main loop start aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 8 - ld1 {v6.16b}, [x0], #16 //AES block 4k+6 - load ciphertext + ldr q4, [x0, #0] //AES block 4k+4 - load ciphertext aese v1.16b, v29.16b //AES block 4k+5 - round 11 - ld1 {v7.16b}, [x0], #16 //AES block 4k+7 - load ciphertext + ldr q5, [x0, #16] //AES block 4k+5 - load ciphertext rev w9, w12 //CTR block 4k+8 aese v3.16b, v26.16b @@ -3547,13 +3347,11 @@ L192_dec_main_loop: //main loop start aesmc v2.16b, v2.16b //AES block 4k+6 - round 9 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid + add x0, x0, #64 //AES input_ptr update cmp x0, x5 //LOOP CONTROL eor v0.16b, v4.16b, v0.16b //AES block 4k+4 - result eor x24, x24, x14 //AES block 4k+3 - round 12 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif eor v1.16b, v5.16b, v1.16b //AES block 4k+5 - result aese v2.16b, v28.16b @@ -3586,28 +3384,18 @@ L192_dec_main_loop: //main loop start rev w9, w12 //CTR block 4k+9 eor x6, x6, x13 //AES block 4k+4 - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif orr x9, x11, x9, lsl #32 //CTR block 4k+9 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low fmov d1, x10 //CTR block 4k+9 add w12, w12, #1 //CTR block 4k+9 eor x19, x19, x13 //AES block 4k+5 - round 12 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif + fmov v1.d[1], x9 //CTR block 4k+9 rev w9, w12 //CTR block 4k+10 eor x20, x20, x14 //AES block 4k+5 - round 12 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif + eor x7, x7, x14 //AES block 4k+4 - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif stp x6, x7, [x2], #16 //AES block 4k+4 - store result eor v11.16b, v11.16b, v10.16b //MODULO - fold into low @@ -3661,29 +3449,18 @@ L192_dec_prepretail: //PREPRETAIL pmull v31.1q, v5.1d, v14.1d //GHASH block 4k+1 - low eor x24, x24, x14 //AES block 4k+3 - round 12 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif fmov v3.d[1], x9 //CTR block 4k+7 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 2 eor x21, x21, x13 //AES block 4k+2 - round 12 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif + pmull2 v30.1q, v5.2d, v14.2d //GHASH block 4k+1 - high eor x22, x22, x14 //AES block 4k+2 - round 12 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid pmull v10.1q, v8.1d, v10.1d //GHASH block 4k - mid eor x23, x23, x13 //AES block 4k+3 - round 12 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif stp x21, x22, [x2], #16 //AES block 4k+2 - store result rev64 v7.16b, v7.16b //GHASH block 4k+3 @@ -3873,13 +3650,8 @@ L192_dec_tail: //TAIL cmp x5, #48 eor x7, x7, x14 //AES block 4k+4 - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES block 4k+4 - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif b.gt L192_dec_blocks_more_than_3 movi v11.8b, #0 @@ -3923,16 +3695,10 @@ L192_dec_blocks_more_than_3: //blocks left > 3 pmull2 v9.1q, v4.2d, v15.2d //GHASH final-3 block - high eor x6, x6, x13 //AES final-2 block - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif movi v8.8b, #0 //suppress further partial tag feed in pmull v10.1q, v22.1d, v10.1d //GHASH final-3 block - mid eor x7, x7, x14 //AES final-2 block - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif L192_dec_blocks_more_than_2: //blocks left > 2 rev64 v4.16b, v5.16b //GHASH final-2 block @@ -3962,13 +3728,8 @@ L192_dec_blocks_more_than_2: //blocks left > 2 eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high eor x7, x7, x14 //AES final-1 block - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES final-1 block - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid L192_dec_blocks_more_than_1: //blocks left > 1 @@ -3999,13 +3760,9 @@ L192_dec_blocks_more_than_1: //blocks left > 1 movi v8.8b, #0 //suppress further partial tag feed in eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low eor x7, x7, x14 //AES final block - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES final block - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif + eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid L192_dec_blocks_less_than_1: //blocks left <= 1 @@ -4032,11 +3789,8 @@ L192_dec_blocks_less_than_1: //blocks left <= 1 orr x6, x6, x4 mov v0.d[1], x10 -#ifndef __AARCH64EB__ + rev w9, w12 -#else - mov w9, w12 -#endif and v5.16b, v5.16b, v0.16b //possibly partial last block has zeroes in highest bits str w9, [x16, #12] //store the updated counter @@ -4109,7 +3863,6 @@ L192_dec_ret: .align 4 _aes_gcm_enc_256_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, L256_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -4125,22 +3878,14 @@ _aes_gcm_enc_256_kernel: lsr x5, x1, #3 //byte_len mov x15, x5 ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #224] //load rk14 -#ifdef __AARCH64EB__ - ror x13, x13, #32 - ror x14, x14, #32 -#endif + ld1 { v0.16b}, [x16] //special case vector load initial counter so we can start first AES block as quickly as possible sub x5, x5, #1 //byte_len - 1 - ld1 {v18.4s}, [x8], #16 //load rk0 + ldr q18, [x8, #0] //load rk0 and x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q25, [x8, #112] //load rk7 add x5, x5, x0 lsr x12, x11, #32 @@ -4160,14 +3905,14 @@ _aes_gcm_enc_256_kernel: orr x9, x11, x9, lsl #32 //CTR block 1 add w12, w12, #1 //CTR block 1 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q19, [x8, #16] //load rk1 fmov v1.d[1], x9 //CTR block 1 rev w9, w12 //CTR block 2 add w12, w12, #1 //CTR block 2 orr x9, x11, x9, lsl #32 //CTR block 2 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q20, [x8, #32] //load rk2 fmov v2.d[1], x9 //CTR block 2 rev w9, w12 //CTR block 3 @@ -4180,53 +3925,50 @@ _aes_gcm_enc_256_kernel: aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q21, [x8, #48] //load rk3 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q24, [x8, #96] //load rk6 aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q23, [x8, #80] //load rk5 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q31, [x8, #208] //load rk13 aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q22, [x8, #64] //load rk4 aese v1.16b, v20.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif + aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q30, [x8, #192] //load rk12 aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif + aese v1.16b, v21.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 - ld1 {v28.4s}, [x8], #16 //load rk10 + ldr q29, [x8, #176] //load rk11 aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 - ld1 {v29.4s}, [x8], #16 //load rk11 + ldr q26, [x8, #128] //load rk8 aese v2.16b, v21.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 @@ -4234,6 +3976,7 @@ _aes_gcm_enc_256_kernel: aese v0.16b, v21.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 + ldp x13, x14, [x8, #224] //load rk14 aese v3.16b, v21.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 @@ -4271,17 +4014,16 @@ _aes_gcm_enc_256_kernel: aese v3.16b, v24.16b aesmc v3.16b, v3.16b //AES block 3 - round 6 - ld1 {v30.4s}, [x8], #16 //load rk12 + ldr q27, [x8, #144] //load rk9 aese v0.16b, v24.16b aesmc v0.16b, v0.16b //AES block 0 - round 6 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v2.16b, v24.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 - ld1 {v31.4s}, [x8], #16 //load rk13 + ldr q28, [x8, #160] //load rk10 aese v1.16b, v25.16b aesmc v1.16b, v1.16b //AES block 1 - round 7 @@ -4370,26 +4112,13 @@ _aes_gcm_enc_256_kernel: b.ge L256_enc_tail //handle tail ldp x19, x20, [x0, #16] //AES block 1 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + rev w9, w12 //CTR block 4 ldp x6, x7, [x0, #0] //AES block 0 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + ldp x23, x24, [x0, #48] //AES block 3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + ldp x21, x22, [x0, #32] //AES block 2 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif add x0, x0, #64 //AES input_ptr update eor x19, x19, x13 //AES block 1 - round 14 low @@ -4472,17 +4201,11 @@ L256_enc_main_loop: //main loop start aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 1 ldp x23, x24, [x0, #48] //AES block 4k+7 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 1 ldp x21, x22, [x0, #32] //AES block 4k+6 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 2 eor v4.16b, v4.16b, v11.16b //PRE 1 @@ -4592,10 +4315,7 @@ L256_enc_main_loop: //main loop start aese v3.16b, v24.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 6 ldp x19, x20, [x0, #16] //AES block 4k+5 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 8 mov d4, v7.d[1] //GHASH block 4k+3 - mid @@ -4632,10 +4352,7 @@ L256_enc_main_loop: //main loop start aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 8 ldp x6, x7, [x0, #0] //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 10 shl d8, d8, #56 //mod_constant @@ -4997,10 +4714,7 @@ L256_enc_tail: //TAIL ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldp x6, x7, [x0], #16 //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor x6, x6, x13 //AES block 4k+4 - round 14 low eor x7, x7, x14 //AES block 4k+4 - round 14 high @@ -5035,10 +4749,7 @@ L256_enc_blocks_more_than_3: //blocks left > 3 st1 { v5.16b}, [x2], #16 //AES final-3 block - store result ldp x6, x7, [x0], #16 //AES final-2 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-3 block eor x6, x6, x13 //AES final-2 block - round 14 low @@ -5067,10 +4778,7 @@ L256_enc_blocks_more_than_2: //blocks left > 2 st1 { v5.16b}, [x2], #16 //AES final-2 block - store result ldp x6, x7, [x0], #16 //AES final-1 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-2 block eor x6, x6, x13 //AES final-1 block - round 14 low @@ -5106,10 +4814,7 @@ L256_enc_blocks_more_than_1: //blocks left > 1 rev64 v4.16b, v5.16b //GHASH final-1 block ldp x6, x7, [x0], #16 //AES final block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor v4.16b, v4.16b, v8.16b //feed in partial tag movi v8.8b, #0 //suppress further partial tag feed in @@ -5170,11 +4875,7 @@ L256_enc_blocks_less_than_1: //blocks left <= 1 pmull2 v20.1q, v4.2d, v12.2d //GHASH final block - high mov d8, v4.d[1] //GHASH final block - mid -#ifndef __AARCH64EB__ rev w9, w12 -#else - mov w9, w12 -#endif pmull v21.1q, v4.1d, v12.1d //GHASH final block - low @@ -5234,7 +4935,6 @@ L256_enc_ret: .align 4 _aes_gcm_dec_256_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, L256_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -5249,29 +4949,21 @@ _aes_gcm_dec_256_kernel: lsr x5, x1, #3 //byte_len mov x15, x5 ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #224] //load rk14 -#ifdef __AARCH64EB__ - ror x14, x14, #32 - ror x13, x13, #32 -#endif - ld1 {v18.4s}, [x8], #16 //load rk0 + + ldr q26, [x8, #128] //load rk8 sub x5, x5, #1 //byte_len - 1 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q25, [x8, #112] //load rk7 and x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) add x4, x0, x1, lsr #3 //end_input_ptr - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q24, [x8, #96] //load rk6 lsr x12, x11, #32 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q23, [x8, #80] //load rk5 orr w11, w11, w11 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q21, [x8, #48] //load rk3 add x5, x5, x0 rev w12, w12 //rev_ctr32 @@ -5296,44 +4988,39 @@ _aes_gcm_dec_256_kernel: rev w9, w12 //CTR block 3 orr x9, x11, x9, lsl #32 //CTR block 3 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q18, [x8, #0] //load rk0 fmov v3.d[1], x9 //CTR block 3 add w12, w12, #1 //CTR block 3 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q22, [x8, #64] //load rk4 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q31, [x8, #208] //load rk13 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q19, [x8, #16] //load rk1 aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q20, [x8, #32] //load rk2 aese v0.16b, v19.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 + ldp x13, x14, [x8, #224] //load rk14 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 @@ -5343,21 +5030,20 @@ _aes_gcm_dec_256_kernel: aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 - ld1 {v28.4s}, [x8], #16 //load rk10 + ldr q27, [x8, #144] //load rk9 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 - ld1 {v29.4s}, [x8], #16 //load rk11 + ldr q30, [x8, #192] //load rk12 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 - ld1 {v30.4s}, [x8], #16 //load rk12 + ldr q28, [x8, #160] //load rk10 aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 @@ -5440,7 +5126,7 @@ _aes_gcm_dec_256_kernel: aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 8 - ld1 {v31.4s}, [x8], #16 //load rk13 + ldr q29, [x8, #176] //load rk11 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 9 @@ -5505,7 +5191,9 @@ _aes_gcm_dec_256_kernel: aese v0.16b, v31.16b //AES block 0 - round 13 b.ge L256_dec_tail //handle tail - ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0,1 - load ciphertext + ldr q4, [x0, #0] //AES block 0 - load ciphertext + + ldr q5, [x0, #16] //AES block 1 - load ciphertext rev w9, w12 //CTR block 4 @@ -5513,7 +5201,7 @@ _aes_gcm_dec_256_kernel: eor v1.16b, v5.16b, v1.16b //AES block 1 - result rev64 v5.16b, v5.16b //GHASH block 1 - ld1 {v6.16b}, [x0], #16 //AES block 2 - load ciphertext + ldr q7, [x0, #48] //AES block 3 - load ciphertext mov x7, v0.d[1] //AES block 0 - mov high @@ -5533,32 +5221,22 @@ _aes_gcm_dec_256_kernel: orr x9, x11, x9, lsl #32 //CTR block 5 mov x20, v1.d[1] //AES block 1 - mov high eor x7, x7, x14 //AES block 0 - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES block 0 - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif stp x6, x7, [x2], #16 //AES block 0 - store result fmov d1, x10 //CTR block 5 - ld1 {v7.16b}, [x0], #16 //AES block 3 - load ciphertext + ldr q6, [x0, #32] //AES block 2 - load ciphertext + add x0, x0, #64 //AES input_ptr update fmov v1.d[1], x9 //CTR block 5 rev w9, w12 //CTR block 6 add w12, w12, #1 //CTR block 6 eor x19, x19, x13 //AES block 1 - round 14 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif orr x9, x11, x9, lsl #32 //CTR block 6 eor x20, x20, x14 //AES block 1 - round 14 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif stp x19, x20, [x2], #16 //AES block 1 - store result eor v2.16b, v6.16b, v2.16b //AES block 2 - result @@ -5609,9 +5287,7 @@ L256_dec_main_loop: //main loop start aese v0.16b, v21.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 3 eor x22, x22, x14 //AES block 4k+2 - round 14 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 1 mov d10, v17.d[1] //GHASH block 4k - mid @@ -5623,9 +5299,7 @@ L256_dec_main_loop: //main loop start aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 0 eor x21, x21, x13 //AES block 4k+2 - round 14 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif + aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 2 stp x21, x22, [x2], #16 //AES block 4k+2 - store result @@ -5640,14 +5314,9 @@ L256_dec_main_loop: //main loop start pmull v10.1q, v8.1d, v10.1d //GHASH block 4k - mid eor x23, x23, x13 //AES block 4k+3 - round 14 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + pmull v8.1q, v5.1d, v14.1d //GHASH block 4k+1 - low eor x24, x24, x14 //AES block 4k+3 - round 14 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif eor v9.16b, v9.16b, v4.16b //GHASH block 4k+1 - high aese v2.16b, v22.16b @@ -5768,7 +5437,7 @@ L256_dec_main_loop: //main loop start aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 9 - ld1 {v4.16b}, [x0], #16 //AES block 4k+4 - load ciphertext + ldr q4, [x0, #0] //AES block 4k+4 - load ciphertext aese v0.16b, v31.16b //AES block 4k+4 - round 13 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment @@ -5779,7 +5448,7 @@ L256_dec_main_loop: //main loop start aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 9 - ld1 {v5.16b}, [x0], #16 //AES block 4k+5 - load ciphertext + ldr q5, [x0, #16] //AES block 4k+5 - load ciphertext aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 8 @@ -5795,11 +5464,11 @@ L256_dec_main_loop: //main loop start aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 9 - ld1 {v6.16b}, [x0], #16 //AES block 4k+6 - load ciphertext + ldr q7, [x0, #48] //AES block 4k+7 - load ciphertext aese v1.16b, v30.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 12 - ld1 {v7.16b}, [x0], #16 //AES block 4k+7 - load ciphertext + ldr q6, [x0, #32] //AES block 4k+6 - load ciphertext aese v2.16b, v29.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 11 @@ -5810,6 +5479,7 @@ L256_dec_main_loop: //main loop start eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid aese v1.16b, v31.16b //AES block 4k+5 - round 13 + add x0, x0, #64 //AES input_ptr update mov x6, v0.d[0] //AES block 4k+4 - mov low aese v2.16b, v30.16b @@ -5831,13 +5501,8 @@ L256_dec_main_loop: //main loop start add w12, w12, #1 //CTR block 4k+9 eor x6, x6, x13 //AES block 4k+4 - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor x7, x7, x14 //AES block 4k+4 - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + mov x20, v1.d[1] //AES block 4k+5 - mov high eor v2.16b, v6.16b, v2.16b //AES block 4k+6 - result eor v11.16b, v11.16b, v8.16b //MODULO - fold into low @@ -5858,15 +5523,9 @@ L256_dec_main_loop: //main loop start rev64 v5.16b, v5.16b //GHASH block 4k+5 eor x20, x20, x14 //AES block 4k+5 - round 14 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif stp x6, x7, [x2], #16 //AES block 4k+4 - store result eor x19, x19, x13 //AES block 4k+5 - round 14 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif stp x19, x20, [x2], #16 //AES block 4k+5 - store result rev64 v4.16b, v4.16b //GHASH block 4k+4 @@ -6073,15 +5732,11 @@ L256_dec_prepretail: //PREPRETAIL aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 10 eor x22, x22, x14 //AES block 4k+2 - round 14 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif + aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 10 eor x23, x23, x13 //AES block 4k+3 - round 14 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + aese v2.16b, v29.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 11 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid @@ -6093,18 +5748,12 @@ L256_dec_prepretail: //PREPRETAIL aese v1.16b, v29.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 11 eor x21, x21, x13 //AES block 4k+2 - round 14 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif aese v2.16b, v30.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 12 pmull v8.1q, v10.1d, v8.1d //MODULO - mid 64b align with low eor x24, x24, x14 //AES block 4k+3 - round 14 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif aese v3.16b, v29.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 11 @@ -6145,14 +5794,8 @@ L256_dec_tail: //TAIL cmp x5, #48 eor x6, x6, x13 //AES block 4k+4 - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor x7, x7, x14 //AES block 4k+4 - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif b.gt L256_dec_blocks_more_than_3 sub w12, w12, #1 @@ -6200,15 +5843,9 @@ L256_dec_blocks_more_than_3: //blocks left > 3 pmull v10.1q, v22.1d, v10.1d //GHASH final-3 block - mid eor x6, x6, x13 //AES final-2 block - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif pmull v11.1q, v4.1d, v15.1d //GHASH final-3 block - low eor x7, x7, x14 //AES final-2 block - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif L256_dec_blocks_more_than_2: //blocks left > 2 rev64 v4.16b, v5.16b //GHASH final-2 block @@ -6236,15 +5873,9 @@ L256_dec_blocks_more_than_2: //blocks left > 2 eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high eor x6, x6, x13 //AES final-1 block - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid eor x7, x7, x14 //AES final-1 block - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif L256_dec_blocks_more_than_1: //blocks left > 1 stp x6, x7, [x2], #16 //AES final-1 block - store result @@ -6272,18 +5903,13 @@ L256_dec_blocks_more_than_1: //blocks left > 1 pmull2 v22.1q, v22.2d, v16.2d //GHASH final-1 block - mid eor x6, x6, x13 //AES final block - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif + eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid eor x7, x7, x14 //AES final block - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif L256_dec_blocks_less_than_1: //blocks left <= 1 and x1, x1, #127 //bit_length %= 128 @@ -6309,11 +5935,7 @@ L256_dec_blocks_less_than_1: //blocks left <= 1 mov v0.d[1], x10 bic x4, x4, x9 //mask out low existing bytes -#ifndef __AARCH64EB__ rev w9, w12 -#else - mov w9, w12 -#endif bic x5, x5, x10 //mask out high existing bytes diff --git a/openssl/src/crypto/modes/gen/darwin_arm64/ghashv8-armx.S b/openssl/src/crypto/modes/gen/darwin_arm64/ghashv8-armx.S index 0abd64983..200c0d031 100644 --- a/openssl/src/crypto/modes/gen/darwin_arm64/ghashv8-armx.S +++ b/openssl/src/crypto/modes/gen/darwin_arm64/ghashv8-armx.S @@ -7,7 +7,6 @@ .align 4 _gcm_init_v8: - AARCH64_VALID_CALL_TARGET ld1 {v17.2d},[x1] //load input H movi v19.16b,#0xe1 shl v19.2d,v19.2d,#57 //0xc2.0 @@ -83,115 +82,26 @@ _gcm_init_v8: pmull v5.1q,v5.1d,v19.1d eor v18.16b,v18.16b,v2.16b eor v4.16b,v4.16b,v7.16b - eor v23.16b, v0.16b,v18.16b //H^3 - eor v25.16b,v5.16b,v4.16b //H^4 - - ext v16.16b,v23.16b, v23.16b,#8 //Karatsuba pre-processing - ext v17.16b,v25.16b,v25.16b,#8 - ext v18.16b,v22.16b,v22.16b,#8 - eor v16.16b,v16.16b,v23.16b - eor v17.16b,v17.16b,v25.16b - eor v18.16b,v18.16b,v22.16b - ext v24.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed - st1 {v23.2d,v24.2d,v25.2d},[x0],#48 //store Htable[3..5] - - //calculate H^5 and H^6 - pmull v0.1q,v22.1d, v23.1d - pmull v5.1q,v23.1d,v23.1d - pmull2 v2.1q,v22.2d, v23.2d - pmull2 v7.1q,v23.2d,v23.2d - pmull v1.1q,v16.1d,v18.1d - pmull v6.1q,v16.1d,v16.1d + eor v20.16b, v0.16b,v18.16b //H^3 + eor v22.16b,v5.16b,v4.16b //H^4 - ext v16.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing - ext v17.16b,v5.16b,v7.16b,#8 - eor v18.16b,v0.16b,v2.16b - eor v1.16b,v1.16b,v16.16b - eor v4.16b,v5.16b,v7.16b - eor v6.16b,v6.16b,v17.16b - eor v1.16b,v1.16b,v18.16b - pmull v18.1q,v0.1d,v19.1d //1st phase - eor v6.16b,v6.16b,v4.16b - pmull v4.1q,v5.1d,v19.1d - - ins v2.d[0],v1.d[1] - ins v7.d[0],v6.d[1] - ins v1.d[1],v0.d[0] - ins v6.d[1],v5.d[0] - eor v0.16b,v1.16b,v18.16b - eor v5.16b,v6.16b,v4.16b - - ext v18.16b,v0.16b,v0.16b,#8 //2nd phase - ext v4.16b,v5.16b,v5.16b,#8 - pmull v0.1q,v0.1d,v19.1d - pmull v5.1q,v5.1d,v19.1d - eor v18.16b,v18.16b,v2.16b - eor v4.16b,v4.16b,v7.16b - eor v26.16b,v0.16b,v18.16b //H^5 - eor v28.16b,v5.16b,v4.16b //H^6 - - ext v16.16b,v26.16b, v26.16b,#8 //Karatsuba pre-processing - ext v17.16b,v28.16b,v28.16b,#8 - ext v18.16b,v22.16b,v22.16b,#8 - eor v16.16b,v16.16b,v26.16b - eor v17.16b,v17.16b,v28.16b - eor v18.16b,v18.16b,v22.16b - ext v27.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed - st1 {v26.2d,v27.2d,v28.2d},[x0],#48 //store Htable[6..8] - - //calculate H^7 and H^8 - pmull v0.1q,v22.1d,v26.1d - pmull v5.1q,v22.1d,v28.1d - pmull2 v2.1q,v22.2d,v26.2d - pmull2 v7.1q,v22.2d,v28.2d - pmull v1.1q,v16.1d,v18.1d - pmull v6.1q,v17.1d,v18.1d - - ext v16.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing - ext v17.16b,v5.16b,v7.16b,#8 - eor v18.16b,v0.16b,v2.16b - eor v1.16b,v1.16b,v16.16b - eor v4.16b,v5.16b,v7.16b - eor v6.16b,v6.16b,v17.16b - eor v1.16b,v1.16b,v18.16b - pmull v18.1q,v0.1d,v19.1d //1st phase - eor v6.16b,v6.16b,v4.16b - pmull v4.1q,v5.1d,v19.1d - - ins v2.d[0],v1.d[1] - ins v7.d[0],v6.d[1] - ins v1.d[1],v0.d[0] - ins v6.d[1],v5.d[0] - eor v0.16b,v1.16b,v18.16b - eor v5.16b,v6.16b,v4.16b - - ext v18.16b,v0.16b,v0.16b,#8 //2nd phase - ext v4.16b,v5.16b,v5.16b,#8 - pmull v0.1q,v0.1d,v19.1d - pmull v5.1q,v5.1d,v19.1d - eor v18.16b,v18.16b,v2.16b - eor v4.16b,v4.16b,v7.16b - eor v29.16b,v0.16b,v18.16b //H^7 - eor v31.16b,v5.16b,v4.16b //H^8 - - ext v16.16b,v29.16b,v29.16b,#8 //Karatsuba pre-processing - ext v17.16b,v31.16b,v31.16b,#8 - eor v16.16b,v16.16b,v29.16b - eor v17.16b,v17.16b,v31.16b - ext v30.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed - st1 {v29.2d,v30.2d,v31.2d},[x0] //store Htable[9..11] + ext v16.16b,v20.16b, v20.16b,#8 //Karatsuba pre-processing + ext v17.16b,v22.16b,v22.16b,#8 + eor v16.16b,v16.16b,v20.16b + eor v17.16b,v17.16b,v22.16b + ext v21.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed + st1 {v20.2d,v21.2d,v22.2d},[x0] //store Htable[3..5] ret .globl _gcm_gmult_v8 .align 4 _gcm_gmult_v8: - AARCH64_VALID_CALL_TARGET ld1 {v17.2d},[x0] //load Xi movi v19.16b,#0xe1 ld1 {v20.2d,v21.2d},[x1] //load twisted H, ... shl v19.2d,v19.2d,#57 -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v17.16b,v17.16b #endif ext v3.16b,v17.16b,v17.16b,#8 @@ -216,7 +126,7 @@ _gcm_gmult_v8: eor v18.16b,v18.16b,v2.16b eor v0.16b,v0.16b,v18.16b -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v0.16b,v0.16b #endif ext v0.16b,v0.16b,v0.16b,#8 @@ -228,7 +138,6 @@ _gcm_gmult_v8: .align 4 _gcm_ghash_v8: - AARCH64_VALID_CALL_TARGET cmp x3,#64 b.hs Lgcm_ghash_v8_4x ld1 {v0.2d},[x0] //load [rotated] Xi @@ -254,14 +163,14 @@ _gcm_ghash_v8: ext v0.16b,v0.16b,v0.16b,#8 //rotate Xi ld1 {v16.2d},[x2],#16 //load [rotated] I[0] shl v19.2d,v19.2d,#57 //compose 0xc2.0 constant -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v16.16b,v16.16b rev64 v0.16b,v0.16b #endif ext v3.16b,v16.16b,v16.16b,#8 //rotate I[0] b.lo Lodd_tail_v8 //x3 was less than 32 ld1 {v17.2d},[x2],x12 //load [rotated] I[1] -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v17.16b,v17.16b #endif ext v7.16b,v17.16b,v17.16b,#8 @@ -293,13 +202,13 @@ Loop_mod2x_v8: eor v18.16b,v0.16b,v2.16b eor v1.16b,v1.16b,v17.16b ld1 {v17.2d},[x2],x12 //load [rotated] I[i+3] -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v16.16b,v16.16b #endif eor v1.16b,v1.16b,v18.16b pmull v18.1q,v0.1d,v19.1d //1st phase of reduction -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v17.16b,v17.16b #endif ins v2.d[0],v1.d[1] @@ -349,7 +258,7 @@ Lodd_tail_v8: eor v0.16b,v0.16b,v18.16b Ldone_v8: -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v0.16b,v0.16b #endif ext v0.16b,v0.16b,v0.16b,#8 @@ -368,7 +277,7 @@ Lgcm_ghash_v8_4x: shl v19.2d,v19.2d,#57 //compose 0xc2.0 constant ld1 {v4.2d,v5.2d,v6.2d,v7.2d},[x2],#64 -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v0.16b,v0.16b rev64 v5.16b,v5.16b rev64 v6.16b,v6.16b @@ -412,7 +321,7 @@ Loop4x: eor v16.16b,v4.16b,v0.16b ld1 {v4.2d,v5.2d,v6.2d,v7.2d},[x2],#64 ext v3.16b,v16.16b,v16.16b,#8 -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v5.16b,v5.16b rev64 v6.16b,v6.16b rev64 v7.16b,v7.16b @@ -495,7 +404,7 @@ Lthree: eor v1.16b,v1.16b,v17.16b ld1 {v4.2d,v5.2d,v6.2d},[x2] eor v1.16b,v1.16b,v18.16b -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v5.16b,v5.16b rev64 v6.16b,v6.16b rev64 v4.16b,v4.16b @@ -547,7 +456,7 @@ Ltwo: eor v1.16b,v1.16b,v17.16b ld1 {v4.2d,v5.2d},[x2] eor v1.16b,v1.16b,v18.16b -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v5.16b,v5.16b rev64 v4.16b,v4.16b #endif @@ -590,7 +499,7 @@ Lone: eor v1.16b,v1.16b,v17.16b ld1 {v4.2d},[x2] eor v1.16b,v1.16b,v18.16b -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v4.16b,v4.16b #endif @@ -630,7 +539,7 @@ Ldone4x: eor v0.16b,v0.16b,v18.16b ext v0.16b,v0.16b,v0.16b,#8 -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v0.16b,v0.16b #endif st1 {v0.2d},[x0] //write out Xi diff --git a/openssl/src/crypto/modes/gen/darwin_x64/aes-gcm-avx512.s b/openssl/src/crypto/modes/gen/darwin_x64/aes-gcm-avx512.s deleted file mode 100644 index 0bd40f01e..000000000 --- a/openssl/src/crypto/modes/gen/darwin_x64/aes-gcm-avx512.s +++ /dev/null @@ -1,136062 +0,0 @@ - -.globl _ossl_vaes_vpclmulqdq_capable - -.p2align 5 -_ossl_vaes_vpclmulqdq_capable: - movq _OPENSSL_ia32cap_P+8(%rip),%rcx - - movq $6600291188736,%rdx - xorl %eax,%eax - andq %rdx,%rcx - cmpq %rdx,%rcx - cmoveq %rcx,%rax - .byte 0xf3,0xc3 - -.text -.globl _ossl_aes_gcm_init_avx512 - -.p2align 5 -_ossl_aes_gcm_init_avx512: - -.byte 243,15,30,250 - vpxorq %xmm16,%xmm16,%xmm16 - - - movl 240(%rdi),%eax - cmpl $9,%eax - je L$aes_128_0 - cmpl $11,%eax - je L$aes_192_0 - cmpl $13,%eax - je L$aes_256_0 - jmp L$exit_aes_0 -.p2align 5 -L$aes_128_0: - vpxorq 0(%rdi),%xmm16,%xmm16 - - vaesenc 16(%rdi),%xmm16,%xmm16 - - vaesenc 32(%rdi),%xmm16,%xmm16 - - vaesenc 48(%rdi),%xmm16,%xmm16 - - vaesenc 64(%rdi),%xmm16,%xmm16 - - vaesenc 80(%rdi),%xmm16,%xmm16 - - vaesenc 96(%rdi),%xmm16,%xmm16 - - vaesenc 112(%rdi),%xmm16,%xmm16 - - vaesenc 128(%rdi),%xmm16,%xmm16 - - vaesenc 144(%rdi),%xmm16,%xmm16 - - vaesenclast 160(%rdi),%xmm16,%xmm16 - jmp L$exit_aes_0 -.p2align 5 -L$aes_192_0: - vpxorq 0(%rdi),%xmm16,%xmm16 - - vaesenc 16(%rdi),%xmm16,%xmm16 - - vaesenc 32(%rdi),%xmm16,%xmm16 - - vaesenc 48(%rdi),%xmm16,%xmm16 - - vaesenc 64(%rdi),%xmm16,%xmm16 - - vaesenc 80(%rdi),%xmm16,%xmm16 - - vaesenc 96(%rdi),%xmm16,%xmm16 - - vaesenc 112(%rdi),%xmm16,%xmm16 - - vaesenc 128(%rdi),%xmm16,%xmm16 - - vaesenc 144(%rdi),%xmm16,%xmm16 - - vaesenc 160(%rdi),%xmm16,%xmm16 - - vaesenc 176(%rdi),%xmm16,%xmm16 - - vaesenclast 192(%rdi),%xmm16,%xmm16 - jmp L$exit_aes_0 -.p2align 5 -L$aes_256_0: - vpxorq 0(%rdi),%xmm16,%xmm16 - - vaesenc 16(%rdi),%xmm16,%xmm16 - - vaesenc 32(%rdi),%xmm16,%xmm16 - - vaesenc 48(%rdi),%xmm16,%xmm16 - - vaesenc 64(%rdi),%xmm16,%xmm16 - - vaesenc 80(%rdi),%xmm16,%xmm16 - - vaesenc 96(%rdi),%xmm16,%xmm16 - - vaesenc 112(%rdi),%xmm16,%xmm16 - - vaesenc 128(%rdi),%xmm16,%xmm16 - - vaesenc 144(%rdi),%xmm16,%xmm16 - - vaesenc 160(%rdi),%xmm16,%xmm16 - - vaesenc 176(%rdi),%xmm16,%xmm16 - - vaesenc 192(%rdi),%xmm16,%xmm16 - - vaesenc 208(%rdi),%xmm16,%xmm16 - - vaesenclast 224(%rdi),%xmm16,%xmm16 - jmp L$exit_aes_0 -L$exit_aes_0: - - vpshufb SHUF_MASK(%rip),%xmm16,%xmm16 - - vmovdqa64 %xmm16,%xmm2 - vpsllq $1,%xmm16,%xmm16 - vpsrlq $63,%xmm2,%xmm2 - vmovdqa %xmm2,%xmm1 - vpslldq $8,%xmm2,%xmm2 - vpsrldq $8,%xmm1,%xmm1 - vporq %xmm2,%xmm16,%xmm16 - - vpshufd $36,%xmm1,%xmm2 - vpcmpeqd TWOONE(%rip),%xmm2,%xmm2 - vpand POLY(%rip),%xmm2,%xmm2 - vpxorq %xmm2,%xmm16,%xmm16 - - vmovdqu64 %xmm16,336(%rsi) - vshufi32x4 $0x00,%ymm16,%ymm16,%ymm4 - vmovdqa %ymm4,%ymm3 - - vpclmulqdq $0x11,%ymm4,%ymm3,%ymm0 - vpclmulqdq $0x00,%ymm4,%ymm3,%ymm1 - vpclmulqdq $0x01,%ymm4,%ymm3,%ymm2 - vpclmulqdq $0x10,%ymm4,%ymm3,%ymm3 - vpxorq %ymm2,%ymm3,%ymm3 - - vpsrldq $8,%ymm3,%ymm2 - vpslldq $8,%ymm3,%ymm3 - vpxorq %ymm2,%ymm0,%ymm0 - vpxorq %ymm1,%ymm3,%ymm3 - - - - vmovdqu64 POLY2(%rip),%ymm2 - - vpclmulqdq $0x01,%ymm3,%ymm2,%ymm1 - vpslldq $8,%ymm1,%ymm1 - vpxorq %ymm1,%ymm3,%ymm3 - - - - vpclmulqdq $0x00,%ymm3,%ymm2,%ymm1 - vpsrldq $4,%ymm1,%ymm1 - vpclmulqdq $0x10,%ymm3,%ymm2,%ymm3 - vpslldq $4,%ymm3,%ymm3 - - vpternlogq $0x96,%ymm1,%ymm0,%ymm3 - - vmovdqu64 %xmm3,320(%rsi) - vinserti64x2 $1,%xmm16,%ymm3,%ymm4 - vmovdqa64 %ymm4,%ymm5 - - vpclmulqdq $0x11,%ymm3,%ymm4,%ymm0 - vpclmulqdq $0x00,%ymm3,%ymm4,%ymm1 - vpclmulqdq $0x01,%ymm3,%ymm4,%ymm2 - vpclmulqdq $0x10,%ymm3,%ymm4,%ymm4 - vpxorq %ymm2,%ymm4,%ymm4 - - vpsrldq $8,%ymm4,%ymm2 - vpslldq $8,%ymm4,%ymm4 - vpxorq %ymm2,%ymm0,%ymm0 - vpxorq %ymm1,%ymm4,%ymm4 - - - - vmovdqu64 POLY2(%rip),%ymm2 - - vpclmulqdq $0x01,%ymm4,%ymm2,%ymm1 - vpslldq $8,%ymm1,%ymm1 - vpxorq %ymm1,%ymm4,%ymm4 - - - - vpclmulqdq $0x00,%ymm4,%ymm2,%ymm1 - vpsrldq $4,%ymm1,%ymm1 - vpclmulqdq $0x10,%ymm4,%ymm2,%ymm4 - vpslldq $4,%ymm4,%ymm4 - - vpternlogq $0x96,%ymm1,%ymm0,%ymm4 - - vmovdqu64 %ymm4,288(%rsi) - - vinserti64x4 $1,%ymm5,%zmm4,%zmm4 - - - vshufi64x2 $0x00,%zmm4,%zmm4,%zmm3 - vmovdqa64 %zmm4,%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm0 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm1 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm2 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm2,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm2 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm2,%zmm0,%zmm0 - vpxorq %zmm1,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm2 - - vpclmulqdq $0x01,%zmm4,%zmm2,%zmm1 - vpslldq $8,%zmm1,%zmm1 - vpxorq %zmm1,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm2,%zmm1 - vpsrldq $4,%zmm1,%zmm1 - vpclmulqdq $0x10,%zmm4,%zmm2,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm1,%zmm0,%zmm4 - - vmovdqu64 %zmm4,224(%rsi) - vshufi64x2 $0x00,%zmm4,%zmm4,%zmm3 - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm0 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm1 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm2 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm2,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm2 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm2,%zmm0,%zmm0 - vpxorq %zmm1,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm2 - - vpclmulqdq $0x01,%zmm5,%zmm2,%zmm1 - vpslldq $8,%zmm1,%zmm1 - vpxorq %zmm1,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm2,%zmm1 - vpsrldq $4,%zmm1,%zmm1 - vpclmulqdq $0x10,%zmm5,%zmm2,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm1,%zmm0,%zmm5 - - vmovdqu64 %zmm5,160(%rsi) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm0 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm1 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm2 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm2,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm2 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm2,%zmm0,%zmm0 - vpxorq %zmm1,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm2 - - vpclmulqdq $0x01,%zmm4,%zmm2,%zmm1 - vpslldq $8,%zmm1,%zmm1 - vpxorq %zmm1,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm2,%zmm1 - vpsrldq $4,%zmm1,%zmm1 - vpclmulqdq $0x10,%zmm4,%zmm2,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm1,%zmm0,%zmm4 - - vmovdqu64 %zmm4,96(%rsi) - vzeroupper -L$abort_init: - .byte 0xf3,0xc3 - - -.globl _ossl_aes_gcm_setiv_avx512 - -.p2align 5 -_ossl_aes_gcm_setiv_avx512: - -L$setiv_seh_begin: -.byte 243,15,30,250 - pushq %rbx - -L$setiv_seh_push_rbx: - pushq %rbp - -L$setiv_seh_push_rbp: - pushq %r12 - -L$setiv_seh_push_r12: - pushq %r13 - -L$setiv_seh_push_r13: - pushq %r14 - -L$setiv_seh_push_r14: - pushq %r15 - -L$setiv_seh_push_r15: - - - - - - - - - - - leaq 0(%rsp),%rbp - -L$setiv_seh_setfp: - -L$setiv_seh_prolog_end: - subq $820,%rsp - andq $(-64),%rsp - cmpq $12,%rcx - je iv_len_12_init_IV - vpxor %xmm2,%xmm2,%xmm2 - movq %rdx,%r10 - movq %rcx,%r11 - orq %r11,%r11 - jz L$_CALC_AAD_done_1 - - xorq %rbx,%rbx - vmovdqa64 SHUF_MASK(%rip),%zmm16 - -L$_get_AAD_loop48x16_1: - cmpq $768,%r11 - jl L$_exit_AAD_loop48x16_1 - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - testq %rbx,%rbx - jnz L$_skip_hkeys_precomputation_2 - - vmovdqu64 288(%rsi),%zmm1 - vmovdqu64 %zmm1,704(%rsp) - - vmovdqu64 224(%rsi),%zmm9 - vmovdqu64 %zmm9,640(%rsp) - - - vshufi64x2 $0x00,%zmm9,%zmm9,%zmm9 - - vmovdqu64 160(%rsi),%zmm10 - vmovdqu64 %zmm10,576(%rsp) - - vmovdqu64 96(%rsi),%zmm12 - vmovdqu64 %zmm12,512(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,448(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,384(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,320(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,256(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,192(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,128(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,64(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,0(%rsp) -L$_skip_hkeys_precomputation_2: - movq $1,%rbx - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 0(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 64(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 128(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 192(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 256(%r10),%zmm11 - vmovdqu64 320(%r10),%zmm3 - vmovdqu64 384(%r10),%zmm4 - vmovdqu64 448(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 256(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 320(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 384(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 448(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 512(%r10),%zmm11 - vmovdqu64 576(%r10),%zmm3 - vmovdqu64 640(%r10),%zmm4 - vmovdqu64 704(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 512(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 576(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 640(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 704(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm6,%xmm9,%xmm2 - - subq $768,%r11 - je L$_CALC_AAD_done_1 - - addq $768,%r10 - jmp L$_get_AAD_loop48x16_1 - -L$_exit_AAD_loop48x16_1: - - cmpq $512,%r11 - jl L$_less_than_32x16_1 - - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - testq %rbx,%rbx - jnz L$_skip_hkeys_precomputation_3 - - vmovdqu64 288(%rsi),%zmm1 - vmovdqu64 %zmm1,704(%rsp) - - vmovdqu64 224(%rsi),%zmm9 - vmovdqu64 %zmm9,640(%rsp) - - - vshufi64x2 $0x00,%zmm9,%zmm9,%zmm9 - - vmovdqu64 160(%rsi),%zmm10 - vmovdqu64 %zmm10,576(%rsp) - - vmovdqu64 96(%rsi),%zmm12 - vmovdqu64 %zmm12,512(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,448(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,384(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,320(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,256(%rsp) -L$_skip_hkeys_precomputation_3: - movq $1,%rbx - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 256(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 320(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 384(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 448(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 256(%r10),%zmm11 - vmovdqu64 320(%r10),%zmm3 - vmovdqu64 384(%r10),%zmm4 - vmovdqu64 448(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 512(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 576(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 640(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 704(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm6,%xmm9,%xmm2 - - subq $512,%r11 - je L$_CALC_AAD_done_1 - - addq $512,%r10 - jmp L$_less_than_16x16_1 - -L$_less_than_32x16_1: - cmpq $256,%r11 - jl L$_less_than_16x16_1 - - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 96(%rsi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 160(%rsi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 224(%rsi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 288(%rsi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm6,%xmm9,%xmm2 - - subq $256,%r11 - je L$_CALC_AAD_done_1 - - addq $256,%r10 - -L$_less_than_16x16_1: - - leaq byte64_len_to_mask_table(%rip),%r12 - leaq (%r12,%r11,8),%r12 - - - addl $15,%r11d - shrl $4,%r11d - cmpl $2,%r11d - jb L$_AAD_blocks_1_1 - je L$_AAD_blocks_2_1 - cmpl $4,%r11d - jb L$_AAD_blocks_3_1 - je L$_AAD_blocks_4_1 - cmpl $6,%r11d - jb L$_AAD_blocks_5_1 - je L$_AAD_blocks_6_1 - cmpl $8,%r11d - jb L$_AAD_blocks_7_1 - je L$_AAD_blocks_8_1 - cmpl $10,%r11d - jb L$_AAD_blocks_9_1 - je L$_AAD_blocks_10_1 - cmpl $12,%r11d - jb L$_AAD_blocks_11_1 - je L$_AAD_blocks_12_1 - cmpl $14,%r11d - jb L$_AAD_blocks_13_1 - je L$_AAD_blocks_14_1 - cmpl $15,%r11d - je L$_AAD_blocks_15_1 -L$_AAD_blocks_16_1: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%zmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 96(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 160(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 224(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm9,%zmm11,%zmm1 - vpternlogq $0x96,%zmm10,%zmm3,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm12,%zmm11,%zmm7 - vpternlogq $0x96,%zmm13,%zmm3,%zmm8 - vmovdqu64 288(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm5,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm5,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm5,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm5,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_15_1: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%zmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 112(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 176(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 240(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 304(%rsi),%ymm15 - vinserti64x2 $2,336(%rsi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm5,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm5,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm5,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm5,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_14_1: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%ymm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %ymm16,%ymm5,%ymm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 128(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 192(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 256(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 320(%rsi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm5,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm5,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm5,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm5,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_13_1: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%xmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %xmm16,%xmm5,%xmm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 144(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 208(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 272(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 336(%rsi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm5,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm5,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm5,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm5,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_12_1: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 160(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 224(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 288(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_11_1: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 176(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 240(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 304(%rsi),%ymm15 - vinserti64x2 $2,336(%rsi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_10_1: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%ymm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %ymm16,%ymm4,%ymm4 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 192(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 256(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 320(%rsi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm4,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm4,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm4,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm4,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_9_1: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%xmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %xmm16,%xmm4,%xmm4 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 208(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 272(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 336(%rsi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm4,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm4,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm4,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm4,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_8_1: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 224(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 288(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_7_1: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 240(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 304(%rsi),%ymm15 - vinserti64x2 $2,336(%rsi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_6_1: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%ymm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %ymm16,%ymm3,%ymm3 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 256(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 320(%rsi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm3,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm3,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm3,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm3,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_5_1: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%xmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %xmm16,%xmm3,%xmm3 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 272(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 336(%rsi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm3,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm3,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm3,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm3,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_4_1: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 288(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_3_1: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 304(%rsi),%ymm15 - vinserti64x2 $2,336(%rsi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_2_1: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%ymm11{%k1}{z} - vpshufb %ymm16,%ymm11,%ymm11 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 320(%rsi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm11,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm11,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm11,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm11,%ymm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp L$_CALC_AAD_done_1 -L$_AAD_blocks_1_1: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%xmm11{%k1}{z} - vpshufb %xmm16,%xmm11,%xmm11 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 336(%rsi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm11,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm11,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm11,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm11,%xmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - -L$_CALC_AAD_done_1: - movq %rcx,%r10 - shlq $3,%r10 - vmovq %r10,%xmm3 - - - vpxorq %xmm2,%xmm3,%xmm2 - - vmovdqu64 336(%rsi),%xmm1 - - vpclmulqdq $0x11,%xmm1,%xmm2,%xmm11 - vpclmulqdq $0x00,%xmm1,%xmm2,%xmm3 - vpclmulqdq $0x01,%xmm1,%xmm2,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm2,%xmm2 - vpxorq %xmm4,%xmm2,%xmm2 - - vpsrldq $8,%xmm2,%xmm4 - vpslldq $8,%xmm2,%xmm2 - vpxorq %xmm4,%xmm11,%xmm11 - vpxorq %xmm3,%xmm2,%xmm2 - - - - vmovdqu64 POLY2(%rip),%xmm4 - - vpclmulqdq $0x01,%xmm2,%xmm4,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm2,%xmm2 - - - - vpclmulqdq $0x00,%xmm2,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm2,%xmm4,%xmm2 - vpslldq $4,%xmm2,%xmm2 - - vpternlogq $0x96,%xmm3,%xmm11,%xmm2 - - vpshufb SHUF_MASK(%rip),%xmm2,%xmm2 - jmp skip_iv_len_12_init_IV -iv_len_12_init_IV: - - vmovdqu8 ONEf(%rip),%xmm2 - movq %rdx,%r11 - movl $0x0000000000000fff,%r10d - kmovq %r10,%k1 - vmovdqu8 (%r11),%xmm2{%k1} -skip_iv_len_12_init_IV: - vmovdqu %xmm2,%xmm1 - - - movl 240(%rdi),%r10d - cmpl $9,%r10d - je L$aes_128_4 - cmpl $11,%r10d - je L$aes_192_4 - cmpl $13,%r10d - je L$aes_256_4 - jmp L$exit_aes_4 -.p2align 5 -L$aes_128_4: - vpxorq 0(%rdi),%xmm1,%xmm1 - - vaesenc 16(%rdi),%xmm1,%xmm1 - - vaesenc 32(%rdi),%xmm1,%xmm1 - - vaesenc 48(%rdi),%xmm1,%xmm1 - - vaesenc 64(%rdi),%xmm1,%xmm1 - - vaesenc 80(%rdi),%xmm1,%xmm1 - - vaesenc 96(%rdi),%xmm1,%xmm1 - - vaesenc 112(%rdi),%xmm1,%xmm1 - - vaesenc 128(%rdi),%xmm1,%xmm1 - - vaesenc 144(%rdi),%xmm1,%xmm1 - - vaesenclast 160(%rdi),%xmm1,%xmm1 - jmp L$exit_aes_4 -.p2align 5 -L$aes_192_4: - vpxorq 0(%rdi),%xmm1,%xmm1 - - vaesenc 16(%rdi),%xmm1,%xmm1 - - vaesenc 32(%rdi),%xmm1,%xmm1 - - vaesenc 48(%rdi),%xmm1,%xmm1 - - vaesenc 64(%rdi),%xmm1,%xmm1 - - vaesenc 80(%rdi),%xmm1,%xmm1 - - vaesenc 96(%rdi),%xmm1,%xmm1 - - vaesenc 112(%rdi),%xmm1,%xmm1 - - vaesenc 128(%rdi),%xmm1,%xmm1 - - vaesenc 144(%rdi),%xmm1,%xmm1 - - vaesenc 160(%rdi),%xmm1,%xmm1 - - vaesenc 176(%rdi),%xmm1,%xmm1 - - vaesenclast 192(%rdi),%xmm1,%xmm1 - jmp L$exit_aes_4 -.p2align 5 -L$aes_256_4: - vpxorq 0(%rdi),%xmm1,%xmm1 - - vaesenc 16(%rdi),%xmm1,%xmm1 - - vaesenc 32(%rdi),%xmm1,%xmm1 - - vaesenc 48(%rdi),%xmm1,%xmm1 - - vaesenc 64(%rdi),%xmm1,%xmm1 - - vaesenc 80(%rdi),%xmm1,%xmm1 - - vaesenc 96(%rdi),%xmm1,%xmm1 - - vaesenc 112(%rdi),%xmm1,%xmm1 - - vaesenc 128(%rdi),%xmm1,%xmm1 - - vaesenc 144(%rdi),%xmm1,%xmm1 - - vaesenc 160(%rdi),%xmm1,%xmm1 - - vaesenc 176(%rdi),%xmm1,%xmm1 - - vaesenc 192(%rdi),%xmm1,%xmm1 - - vaesenc 208(%rdi),%xmm1,%xmm1 - - vaesenclast 224(%rdi),%xmm1,%xmm1 - jmp L$exit_aes_4 -L$exit_aes_4: - - vmovdqu %xmm1,32(%rsi) - - - vpshufb SHUF_MASK(%rip),%xmm2,%xmm2 - vmovdqu %xmm2,0(%rsi) - cmpq $256,%rcx - jbe L$skip_hkeys_cleanup_5 - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %zmm0,0(%rsp) - vmovdqa64 %zmm0,64(%rsp) - vmovdqa64 %zmm0,128(%rsp) - vmovdqa64 %zmm0,192(%rsp) - vmovdqa64 %zmm0,256(%rsp) - vmovdqa64 %zmm0,320(%rsp) - vmovdqa64 %zmm0,384(%rsp) - vmovdqa64 %zmm0,448(%rsp) - vmovdqa64 %zmm0,512(%rsp) - vmovdqa64 %zmm0,576(%rsp) - vmovdqa64 %zmm0,640(%rsp) - vmovdqa64 %zmm0,704(%rsp) -L$skip_hkeys_cleanup_5: - vzeroupper - leaq (%rbp),%rsp - - popq %r15 - - popq %r14 - - popq %r13 - - popq %r12 - - popq %rbp - - popq %rbx - -L$abort_setiv: - .byte 0xf3,0xc3 -L$setiv_seh_end: - - -.globl _ossl_aes_gcm_update_aad_avx512 - -.p2align 5 -_ossl_aes_gcm_update_aad_avx512: - -L$ghash_seh_begin: -.byte 243,15,30,250 - pushq %rbx - -L$ghash_seh_push_rbx: - pushq %rbp - -L$ghash_seh_push_rbp: - pushq %r12 - -L$ghash_seh_push_r12: - pushq %r13 - -L$ghash_seh_push_r13: - pushq %r14 - -L$ghash_seh_push_r14: - pushq %r15 - -L$ghash_seh_push_r15: - - - - - - - - - - - leaq 0(%rsp),%rbp - -L$ghash_seh_setfp: - -L$ghash_seh_prolog_end: - subq $820,%rsp - andq $(-64),%rsp - vmovdqu64 64(%rdi),%xmm14 - movq %rsi,%r10 - movq %rdx,%r11 - orq %r11,%r11 - jz L$_CALC_AAD_done_6 - - xorq %rbx,%rbx - vmovdqa64 SHUF_MASK(%rip),%zmm16 - -L$_get_AAD_loop48x16_6: - cmpq $768,%r11 - jl L$_exit_AAD_loop48x16_6 - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - testq %rbx,%rbx - jnz L$_skip_hkeys_precomputation_7 - - vmovdqu64 288(%rdi),%zmm1 - vmovdqu64 %zmm1,704(%rsp) - - vmovdqu64 224(%rdi),%zmm9 - vmovdqu64 %zmm9,640(%rsp) - - - vshufi64x2 $0x00,%zmm9,%zmm9,%zmm9 - - vmovdqu64 160(%rdi),%zmm10 - vmovdqu64 %zmm10,576(%rsp) - - vmovdqu64 96(%rdi),%zmm12 - vmovdqu64 %zmm12,512(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,448(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,384(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,320(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,256(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,192(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,128(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,64(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,0(%rsp) -L$_skip_hkeys_precomputation_7: - movq $1,%rbx - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 0(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 64(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 128(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 192(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 256(%r10),%zmm11 - vmovdqu64 320(%r10),%zmm3 - vmovdqu64 384(%r10),%zmm4 - vmovdqu64 448(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 256(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 320(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 384(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 448(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 512(%r10),%zmm11 - vmovdqu64 576(%r10),%zmm3 - vmovdqu64 640(%r10),%zmm4 - vmovdqu64 704(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 512(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 576(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 640(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 704(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm6,%xmm9,%xmm14 - - subq $768,%r11 - je L$_CALC_AAD_done_6 - - addq $768,%r10 - jmp L$_get_AAD_loop48x16_6 - -L$_exit_AAD_loop48x16_6: - - cmpq $512,%r11 - jl L$_less_than_32x16_6 - - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - testq %rbx,%rbx - jnz L$_skip_hkeys_precomputation_8 - - vmovdqu64 288(%rdi),%zmm1 - vmovdqu64 %zmm1,704(%rsp) - - vmovdqu64 224(%rdi),%zmm9 - vmovdqu64 %zmm9,640(%rsp) - - - vshufi64x2 $0x00,%zmm9,%zmm9,%zmm9 - - vmovdqu64 160(%rdi),%zmm10 - vmovdqu64 %zmm10,576(%rsp) - - vmovdqu64 96(%rdi),%zmm12 - vmovdqu64 %zmm12,512(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,448(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,384(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,320(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,256(%rsp) -L$_skip_hkeys_precomputation_8: - movq $1,%rbx - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 256(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 320(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 384(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 448(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 256(%r10),%zmm11 - vmovdqu64 320(%r10),%zmm3 - vmovdqu64 384(%r10),%zmm4 - vmovdqu64 448(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 512(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 576(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 640(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 704(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm6,%xmm9,%xmm14 - - subq $512,%r11 - je L$_CALC_AAD_done_6 - - addq $512,%r10 - jmp L$_less_than_16x16_6 - -L$_less_than_32x16_6: - cmpq $256,%r11 - jl L$_less_than_16x16_6 - - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 96(%rdi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 160(%rdi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 224(%rdi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 288(%rdi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm6,%xmm9,%xmm14 - - subq $256,%r11 - je L$_CALC_AAD_done_6 - - addq $256,%r10 - -L$_less_than_16x16_6: - - leaq byte64_len_to_mask_table(%rip),%r12 - leaq (%r12,%r11,8),%r12 - - - addl $15,%r11d - shrl $4,%r11d - cmpl $2,%r11d - jb L$_AAD_blocks_1_6 - je L$_AAD_blocks_2_6 - cmpl $4,%r11d - jb L$_AAD_blocks_3_6 - je L$_AAD_blocks_4_6 - cmpl $6,%r11d - jb L$_AAD_blocks_5_6 - je L$_AAD_blocks_6_6 - cmpl $8,%r11d - jb L$_AAD_blocks_7_6 - je L$_AAD_blocks_8_6 - cmpl $10,%r11d - jb L$_AAD_blocks_9_6 - je L$_AAD_blocks_10_6 - cmpl $12,%r11d - jb L$_AAD_blocks_11_6 - je L$_AAD_blocks_12_6 - cmpl $14,%r11d - jb L$_AAD_blocks_13_6 - je L$_AAD_blocks_14_6 - cmpl $15,%r11d - je L$_AAD_blocks_15_6 -L$_AAD_blocks_16_6: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%zmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 96(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 160(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 224(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm9,%zmm11,%zmm1 - vpternlogq $0x96,%zmm10,%zmm3,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm12,%zmm11,%zmm7 - vpternlogq $0x96,%zmm13,%zmm3,%zmm8 - vmovdqu64 288(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm5,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm5,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm5,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm5,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_15_6: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%zmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 112(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 176(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 240(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 304(%rdi),%ymm15 - vinserti64x2 $2,336(%rdi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm5,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm5,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm5,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm5,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_14_6: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%ymm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %ymm16,%ymm5,%ymm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 128(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 192(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 256(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 320(%rdi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm5,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm5,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm5,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm5,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_13_6: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%xmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %xmm16,%xmm5,%xmm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 144(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 208(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 272(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 336(%rdi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm5,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm5,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm5,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm5,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_12_6: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 160(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 224(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 288(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_11_6: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 176(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 240(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 304(%rdi),%ymm15 - vinserti64x2 $2,336(%rdi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_10_6: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%ymm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %ymm16,%ymm4,%ymm4 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 192(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 256(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 320(%rdi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm4,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm4,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm4,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm4,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_9_6: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%xmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %xmm16,%xmm4,%xmm4 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 208(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 272(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 336(%rdi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm4,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm4,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm4,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm4,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_8_6: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 224(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 288(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_7_6: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 240(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 304(%rdi),%ymm15 - vinserti64x2 $2,336(%rdi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_6_6: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%ymm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %ymm16,%ymm3,%ymm3 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 256(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 320(%rdi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm3,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm3,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm3,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm3,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_5_6: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%xmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %xmm16,%xmm3,%xmm3 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 272(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 336(%rdi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm3,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm3,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm3,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm3,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_4_6: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 288(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_3_6: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 304(%rdi),%ymm15 - vinserti64x2 $2,336(%rdi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_2_6: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%ymm11{%k1}{z} - vpshufb %ymm16,%ymm11,%ymm11 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 320(%rdi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm11,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm11,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm11,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm11,%ymm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp L$_CALC_AAD_done_6 -L$_AAD_blocks_1_6: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%xmm11{%k1}{z} - vpshufb %xmm16,%xmm11,%xmm11 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 336(%rdi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm11,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm11,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm11,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm11,%xmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - -L$_CALC_AAD_done_6: - vmovdqu64 %xmm14,64(%rdi) - cmpq $256,%rdx - jbe L$skip_hkeys_cleanup_9 - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %zmm0,0(%rsp) - vmovdqa64 %zmm0,64(%rsp) - vmovdqa64 %zmm0,128(%rsp) - vmovdqa64 %zmm0,192(%rsp) - vmovdqa64 %zmm0,256(%rsp) - vmovdqa64 %zmm0,320(%rsp) - vmovdqa64 %zmm0,384(%rsp) - vmovdqa64 %zmm0,448(%rsp) - vmovdqa64 %zmm0,512(%rsp) - vmovdqa64 %zmm0,576(%rsp) - vmovdqa64 %zmm0,640(%rsp) - vmovdqa64 %zmm0,704(%rsp) -L$skip_hkeys_cleanup_9: - vzeroupper - leaq (%rbp),%rsp - - popq %r15 - - popq %r14 - - popq %r13 - - popq %r12 - - popq %rbp - - popq %rbx - -L$exit_update_aad: - .byte 0xf3,0xc3 -L$ghash_seh_end: - - -.globl _ossl_aes_gcm_encrypt_avx512 - -.p2align 5 -_ossl_aes_gcm_encrypt_avx512: - -L$encrypt_seh_begin: -.byte 243,15,30,250 - pushq %rbx - -L$encrypt_seh_push_rbx: - pushq %rbp - -L$encrypt_seh_push_rbp: - pushq %r12 - -L$encrypt_seh_push_r12: - pushq %r13 - -L$encrypt_seh_push_r13: - pushq %r14 - -L$encrypt_seh_push_r14: - pushq %r15 - -L$encrypt_seh_push_r15: - - - - - - - - - - - leaq 0(%rsp),%rbp - -L$encrypt_seh_setfp: - -L$encrypt_seh_prolog_end: - subq $1588,%rsp - andq $(-64),%rsp - - - movl 240(%rdi),%eax - cmpl $9,%eax - je L$aes_gcm_encrypt_128_avx512 - cmpl $11,%eax - je L$aes_gcm_encrypt_192_avx512 - cmpl $13,%eax - je L$aes_gcm_encrypt_256_avx512 - xorl %eax,%eax - jmp L$exit_gcm_encrypt -.p2align 5 -L$aes_gcm_encrypt_128_avx512: - orq %r8,%r8 - je L$_enc_dec_done_10 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je L$_partial_block_done_11 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge L$_no_extra_mask_11 - subq %r13,%r12 -L$_no_extra_mask_11: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm3,%xmm14,%xmm14 - cmpq $0,%r13 - jl L$_partial_incomplete_11 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp L$_enc_dec_done_11 - -L$_partial_incomplete_11: - addq %r8,(%rdx) - movq %r8,%r11 - -L$_enc_dec_done_11: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -L$_partial_block_done_11: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je L$_enc_dec_done_10 - cmpq $256,%r8 - jbe L$_message_below_equal_16_blocks_10 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_12 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_12 -L$_next_16_overflow_12: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_12: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_13 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -L$_skip_hkeys_precomputation_13: - cmpq $512,%r8 - jb L$_message_below_32_blocks_10 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_14 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_14 -L$_next_16_overflow_14: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_14: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_15 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -L$_skip_hkeys_precomputation_15: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb L$_no_more_big_nblocks_10 -L$_encrypt_big_nblocks_10: - cmpb $240,%r15b - jae L$_16_blocks_overflow_16 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_16 -L$_16_blocks_overflow_16: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_16: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_17 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_17 -L$_16_blocks_overflow_17: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_17: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_18 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_18 -L$_16_blocks_overflow_18: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_18: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae L$_encrypt_big_nblocks_10 - -L$_no_more_big_nblocks_10: - - cmpq $512,%r8 - jae L$_encrypt_32_blocks_10 - - cmpq $256,%r8 - jae L$_encrypt_16_blocks_10 -L$_encrypt_0_blocks_ghash_32_10: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_19 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_19 - jb L$_last_num_blocks_is_7_1_19 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_19 - jb L$_last_num_blocks_is_11_9_19 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_19 - ja L$_last_num_blocks_is_16_19 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_19 - jmp L$_last_num_blocks_is_13_19 - -L$_last_num_blocks_is_11_9_19: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_19 - ja L$_last_num_blocks_is_11_19 - jmp L$_last_num_blocks_is_9_19 - -L$_last_num_blocks_is_7_1_19: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_19 - jb L$_last_num_blocks_is_3_1_19 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_19 - je L$_last_num_blocks_is_6_19 - jmp L$_last_num_blocks_is_5_19 - -L$_last_num_blocks_is_3_1_19: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_19 - je L$_last_num_blocks_is_2_19 -L$_last_num_blocks_is_1_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_20 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_20 - -L$_16_blocks_overflow_20: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_20: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_21 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_21 -L$_small_initial_partial_block_21: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_21 -L$_small_initial_compute_done_21: -L$_after_reduction_21: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_2_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_22 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_22 - -L$_16_blocks_overflow_22: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_22: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_23 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_23 -L$_small_initial_partial_block_23: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_23: - - orq %r8,%r8 - je L$_after_reduction_23 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_23: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_3_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_24 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_24 - -L$_16_blocks_overflow_24: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_24: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_25 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_25 -L$_small_initial_partial_block_25: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_25: - - orq %r8,%r8 - je L$_after_reduction_25 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_25: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_4_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_26 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_26 - -L$_16_blocks_overflow_26: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_26: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_27 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_27 -L$_small_initial_partial_block_27: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_27: - - orq %r8,%r8 - je L$_after_reduction_27 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_27: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_5_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_28 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_28 - -L$_16_blocks_overflow_28: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_28: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_29 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_29 -L$_small_initial_partial_block_29: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_29: - - orq %r8,%r8 - je L$_after_reduction_29 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_29: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_6_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_30 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_30 - -L$_16_blocks_overflow_30: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_30: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_31 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_31 -L$_small_initial_partial_block_31: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_31: - - orq %r8,%r8 - je L$_after_reduction_31 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_31: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_7_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_32 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_32 - -L$_16_blocks_overflow_32: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_32: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_33 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_33 -L$_small_initial_partial_block_33: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_33: - - orq %r8,%r8 - je L$_after_reduction_33 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_33: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_8_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_34 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_34 - -L$_16_blocks_overflow_34: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_34: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_35 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_35 -L$_small_initial_partial_block_35: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_35: - - orq %r8,%r8 - je L$_after_reduction_35 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_35: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_9_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_36 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_36 - -L$_16_blocks_overflow_36: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_36: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_37 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_37 -L$_small_initial_partial_block_37: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_37: - - orq %r8,%r8 - je L$_after_reduction_37 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_37: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_10_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_38 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_38 - -L$_16_blocks_overflow_38: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_38: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_39 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_39 -L$_small_initial_partial_block_39: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_39: - - orq %r8,%r8 - je L$_after_reduction_39 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_39: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_11_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_40 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_40 - -L$_16_blocks_overflow_40: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_40: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_41 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_41 -L$_small_initial_partial_block_41: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_41: - - orq %r8,%r8 - je L$_after_reduction_41 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_41: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_12_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_42 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_42 - -L$_16_blocks_overflow_42: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_42: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_43 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_43 -L$_small_initial_partial_block_43: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_43: - - orq %r8,%r8 - je L$_after_reduction_43 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_43: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_13_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_44 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_44 - -L$_16_blocks_overflow_44: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_44: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_45 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_45 -L$_small_initial_partial_block_45: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_45: - - orq %r8,%r8 - je L$_after_reduction_45 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_45: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_14_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_46 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_46 - -L$_16_blocks_overflow_46: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_46: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_47 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_47 -L$_small_initial_partial_block_47: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_47: - - orq %r8,%r8 - je L$_after_reduction_47 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_47: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_15_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_48 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_48 - -L$_16_blocks_overflow_48: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_48: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_49 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_49 -L$_small_initial_partial_block_49: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_49: - - orq %r8,%r8 - je L$_after_reduction_49 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_49: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_16_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_50 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_50 - -L$_16_blocks_overflow_50: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_50: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_51: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_51: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_51: - jmp L$_last_blocks_done_19 -L$_last_num_blocks_is_0_19: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_19: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_10 -L$_encrypt_32_blocks_10: - cmpb $240,%r15b - jae L$_16_blocks_overflow_52 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_52 -L$_16_blocks_overflow_52: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_52: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_53 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_53 -L$_16_blocks_overflow_53: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_53: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_54 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_54 - jb L$_last_num_blocks_is_7_1_54 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_54 - jb L$_last_num_blocks_is_11_9_54 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_54 - ja L$_last_num_blocks_is_16_54 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_54 - jmp L$_last_num_blocks_is_13_54 - -L$_last_num_blocks_is_11_9_54: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_54 - ja L$_last_num_blocks_is_11_54 - jmp L$_last_num_blocks_is_9_54 - -L$_last_num_blocks_is_7_1_54: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_54 - jb L$_last_num_blocks_is_3_1_54 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_54 - je L$_last_num_blocks_is_6_54 - jmp L$_last_num_blocks_is_5_54 - -L$_last_num_blocks_is_3_1_54: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_54 - je L$_last_num_blocks_is_2_54 -L$_last_num_blocks_is_1_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_55 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_55 - -L$_16_blocks_overflow_55: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_55: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_56 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_56 -L$_small_initial_partial_block_56: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_56 -L$_small_initial_compute_done_56: -L$_after_reduction_56: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_2_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_57 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_57 - -L$_16_blocks_overflow_57: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_57: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_58 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_58 -L$_small_initial_partial_block_58: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_58: - - orq %r8,%r8 - je L$_after_reduction_58 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_58: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_3_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_59 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_59 - -L$_16_blocks_overflow_59: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_59: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_60 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_60 -L$_small_initial_partial_block_60: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_60: - - orq %r8,%r8 - je L$_after_reduction_60 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_60: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_4_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_61 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_61 - -L$_16_blocks_overflow_61: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_61: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_62 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_62 -L$_small_initial_partial_block_62: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_62: - - orq %r8,%r8 - je L$_after_reduction_62 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_62: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_5_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_63 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_63 - -L$_16_blocks_overflow_63: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_63: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_64 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_64 -L$_small_initial_partial_block_64: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_64: - - orq %r8,%r8 - je L$_after_reduction_64 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_64: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_6_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_65 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_65 - -L$_16_blocks_overflow_65: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_65: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_66 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_66 -L$_small_initial_partial_block_66: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_66: - - orq %r8,%r8 - je L$_after_reduction_66 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_66: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_7_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_67 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_67 - -L$_16_blocks_overflow_67: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_67: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_68 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_68 -L$_small_initial_partial_block_68: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_68: - - orq %r8,%r8 - je L$_after_reduction_68 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_68: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_8_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_69 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_69 - -L$_16_blocks_overflow_69: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_69: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_70 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_70 -L$_small_initial_partial_block_70: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_70: - - orq %r8,%r8 - je L$_after_reduction_70 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_70: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_9_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_71 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_71 - -L$_16_blocks_overflow_71: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_71: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_72 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_72 -L$_small_initial_partial_block_72: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_72: - - orq %r8,%r8 - je L$_after_reduction_72 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_72: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_10_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_73 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_73 - -L$_16_blocks_overflow_73: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_73: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_74 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_74 -L$_small_initial_partial_block_74: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_74: - - orq %r8,%r8 - je L$_after_reduction_74 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_74: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_11_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_75 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_75 - -L$_16_blocks_overflow_75: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_75: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_76 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_76 -L$_small_initial_partial_block_76: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_76: - - orq %r8,%r8 - je L$_after_reduction_76 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_76: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_12_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_77 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_77 - -L$_16_blocks_overflow_77: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_77: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_78 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_78 -L$_small_initial_partial_block_78: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_78: - - orq %r8,%r8 - je L$_after_reduction_78 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_78: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_13_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_79 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_79 - -L$_16_blocks_overflow_79: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_79: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_80 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_80 -L$_small_initial_partial_block_80: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_80: - - orq %r8,%r8 - je L$_after_reduction_80 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_80: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_14_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_81 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_81 - -L$_16_blocks_overflow_81: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_81: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_82 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_82 -L$_small_initial_partial_block_82: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_82: - - orq %r8,%r8 - je L$_after_reduction_82 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_82: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_15_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_83 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_83 - -L$_16_blocks_overflow_83: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_83: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_84 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_84 -L$_small_initial_partial_block_84: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_84: - - orq %r8,%r8 - je L$_after_reduction_84 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_84: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_16_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_85 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_85 - -L$_16_blocks_overflow_85: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_85: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_86: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_86: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_86: - jmp L$_last_blocks_done_54 -L$_last_num_blocks_is_0_54: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_54: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_10 -L$_encrypt_16_blocks_10: - cmpb $240,%r15b - jae L$_16_blocks_overflow_87 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_87 -L$_16_blocks_overflow_87: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_87: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_88 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_88 - jb L$_last_num_blocks_is_7_1_88 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_88 - jb L$_last_num_blocks_is_11_9_88 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_88 - ja L$_last_num_blocks_is_16_88 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_88 - jmp L$_last_num_blocks_is_13_88 - -L$_last_num_blocks_is_11_9_88: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_88 - ja L$_last_num_blocks_is_11_88 - jmp L$_last_num_blocks_is_9_88 - -L$_last_num_blocks_is_7_1_88: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_88 - jb L$_last_num_blocks_is_3_1_88 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_88 - je L$_last_num_blocks_is_6_88 - jmp L$_last_num_blocks_is_5_88 - -L$_last_num_blocks_is_3_1_88: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_88 - je L$_last_num_blocks_is_2_88 -L$_last_num_blocks_is_1_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_89 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_89 - -L$_16_blocks_overflow_89: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_89: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_90 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_90 -L$_small_initial_partial_block_90: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_90 -L$_small_initial_compute_done_90: -L$_after_reduction_90: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_2_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_91 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_91 - -L$_16_blocks_overflow_91: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_91: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_92 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_92 -L$_small_initial_partial_block_92: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_92: - - orq %r8,%r8 - je L$_after_reduction_92 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_92: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_3_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_93 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_93 - -L$_16_blocks_overflow_93: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_93: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_94 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_94 -L$_small_initial_partial_block_94: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_94: - - orq %r8,%r8 - je L$_after_reduction_94 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_94: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_4_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_95 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_95 - -L$_16_blocks_overflow_95: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_95: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_96 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_96 -L$_small_initial_partial_block_96: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_96: - - orq %r8,%r8 - je L$_after_reduction_96 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_96: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_5_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_97 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_97 - -L$_16_blocks_overflow_97: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_97: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_98 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_98 -L$_small_initial_partial_block_98: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_98: - - orq %r8,%r8 - je L$_after_reduction_98 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_98: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_6_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_99 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_99 - -L$_16_blocks_overflow_99: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_99: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_100 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_100 -L$_small_initial_partial_block_100: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_100: - - orq %r8,%r8 - je L$_after_reduction_100 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_100: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_7_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_101 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_101 - -L$_16_blocks_overflow_101: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_101: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_102 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_102 -L$_small_initial_partial_block_102: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_102: - - orq %r8,%r8 - je L$_after_reduction_102 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_102: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_8_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_103 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_103 - -L$_16_blocks_overflow_103: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_103: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_104 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_104 -L$_small_initial_partial_block_104: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_104: - - orq %r8,%r8 - je L$_after_reduction_104 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_104: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_9_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_105 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_105 - -L$_16_blocks_overflow_105: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_105: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_106 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_106 -L$_small_initial_partial_block_106: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_106: - - orq %r8,%r8 - je L$_after_reduction_106 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_106: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_10_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_107 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_107 - -L$_16_blocks_overflow_107: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_107: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_108 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_108 -L$_small_initial_partial_block_108: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_108: - - orq %r8,%r8 - je L$_after_reduction_108 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_108: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_11_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_109 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_109 - -L$_16_blocks_overflow_109: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_109: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_110 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_110 -L$_small_initial_partial_block_110: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_110: - - orq %r8,%r8 - je L$_after_reduction_110 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_110: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_12_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_111 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_111 - -L$_16_blocks_overflow_111: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_111: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_112 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_112 -L$_small_initial_partial_block_112: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_112: - - orq %r8,%r8 - je L$_after_reduction_112 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_112: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_13_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_113 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_113 - -L$_16_blocks_overflow_113: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_113: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_114 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_114 -L$_small_initial_partial_block_114: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_114: - - orq %r8,%r8 - je L$_after_reduction_114 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_114: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_14_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_115 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_115 - -L$_16_blocks_overflow_115: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_115: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_116 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_116 -L$_small_initial_partial_block_116: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_116: - - orq %r8,%r8 - je L$_after_reduction_116 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_116: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_15_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_117 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_117 - -L$_16_blocks_overflow_117: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_117: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_118 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_118 -L$_small_initial_partial_block_118: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_118: - - orq %r8,%r8 - je L$_after_reduction_118 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_118: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_16_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_119 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_119 - -L$_16_blocks_overflow_119: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_119: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_120: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_120: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_120: - jmp L$_last_blocks_done_88 -L$_last_num_blocks_is_0_88: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_88: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_10 - -L$_message_below_32_blocks_10: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_121 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -L$_skip_hkeys_precomputation_121: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_122 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_122 - jb L$_last_num_blocks_is_7_1_122 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_122 - jb L$_last_num_blocks_is_11_9_122 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_122 - ja L$_last_num_blocks_is_16_122 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_122 - jmp L$_last_num_blocks_is_13_122 - -L$_last_num_blocks_is_11_9_122: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_122 - ja L$_last_num_blocks_is_11_122 - jmp L$_last_num_blocks_is_9_122 - -L$_last_num_blocks_is_7_1_122: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_122 - jb L$_last_num_blocks_is_3_1_122 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_122 - je L$_last_num_blocks_is_6_122 - jmp L$_last_num_blocks_is_5_122 - -L$_last_num_blocks_is_3_1_122: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_122 - je L$_last_num_blocks_is_2_122 -L$_last_num_blocks_is_1_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_123 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_123 - -L$_16_blocks_overflow_123: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_123: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_124 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_124 -L$_small_initial_partial_block_124: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_124 -L$_small_initial_compute_done_124: -L$_after_reduction_124: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_2_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_125 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_125 - -L$_16_blocks_overflow_125: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_125: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_126 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_126 -L$_small_initial_partial_block_126: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_126: - - orq %r8,%r8 - je L$_after_reduction_126 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_126: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_3_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_127 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_127 - -L$_16_blocks_overflow_127: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_127: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_128 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_128 -L$_small_initial_partial_block_128: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_128: - - orq %r8,%r8 - je L$_after_reduction_128 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_128: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_4_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_129 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_129 - -L$_16_blocks_overflow_129: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_129: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_130 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_130 -L$_small_initial_partial_block_130: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_130: - - orq %r8,%r8 - je L$_after_reduction_130 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_130: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_5_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_131 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_131 - -L$_16_blocks_overflow_131: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_131: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_132 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_132 -L$_small_initial_partial_block_132: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_132: - - orq %r8,%r8 - je L$_after_reduction_132 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_132: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_6_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_133 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_133 - -L$_16_blocks_overflow_133: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_133: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_134 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_134 -L$_small_initial_partial_block_134: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_134: - - orq %r8,%r8 - je L$_after_reduction_134 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_134: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_7_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_135 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_135 - -L$_16_blocks_overflow_135: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_135: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_136 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_136 -L$_small_initial_partial_block_136: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_136: - - orq %r8,%r8 - je L$_after_reduction_136 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_136: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_8_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_137 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_137 - -L$_16_blocks_overflow_137: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_137: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_138 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_138 -L$_small_initial_partial_block_138: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_138: - - orq %r8,%r8 - je L$_after_reduction_138 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_138: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_9_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_139 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_139 - -L$_16_blocks_overflow_139: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_139: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_140 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_140 -L$_small_initial_partial_block_140: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_140: - - orq %r8,%r8 - je L$_after_reduction_140 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_140: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_10_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_141 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_141 - -L$_16_blocks_overflow_141: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_141: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_142 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_142 -L$_small_initial_partial_block_142: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_142: - - orq %r8,%r8 - je L$_after_reduction_142 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_142: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_11_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_143 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_143 - -L$_16_blocks_overflow_143: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_143: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_144 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_144 -L$_small_initial_partial_block_144: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_144: - - orq %r8,%r8 - je L$_after_reduction_144 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_144: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_12_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_145 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_145 - -L$_16_blocks_overflow_145: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_145: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_146 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_146 -L$_small_initial_partial_block_146: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_146: - - orq %r8,%r8 - je L$_after_reduction_146 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_146: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_13_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_147 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_147 - -L$_16_blocks_overflow_147: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_147: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_148 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_148 -L$_small_initial_partial_block_148: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_148: - - orq %r8,%r8 - je L$_after_reduction_148 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_148: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_14_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_149 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_149 - -L$_16_blocks_overflow_149: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_149: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_150 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_150 -L$_small_initial_partial_block_150: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_150: - - orq %r8,%r8 - je L$_after_reduction_150 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_150: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_15_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_151 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_151 - -L$_16_blocks_overflow_151: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_151: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_152 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_152 -L$_small_initial_partial_block_152: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_152: - - orq %r8,%r8 - je L$_after_reduction_152 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_152: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_16_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_153 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_153 - -L$_16_blocks_overflow_153: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_153: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_154: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_154: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_154: - jmp L$_last_blocks_done_122 -L$_last_num_blocks_is_0_122: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_122: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_10 - -L$_message_below_equal_16_blocks_10: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je L$_small_initial_num_blocks_is_8_155 - jl L$_small_initial_num_blocks_is_7_1_155 - - - cmpq $12,%r12 - je L$_small_initial_num_blocks_is_12_155 - jl L$_small_initial_num_blocks_is_11_9_155 - - - cmpq $16,%r12 - je L$_small_initial_num_blocks_is_16_155 - cmpq $15,%r12 - je L$_small_initial_num_blocks_is_15_155 - cmpq $14,%r12 - je L$_small_initial_num_blocks_is_14_155 - jmp L$_small_initial_num_blocks_is_13_155 - -L$_small_initial_num_blocks_is_11_9_155: - - cmpq $11,%r12 - je L$_small_initial_num_blocks_is_11_155 - cmpq $10,%r12 - je L$_small_initial_num_blocks_is_10_155 - jmp L$_small_initial_num_blocks_is_9_155 - -L$_small_initial_num_blocks_is_7_1_155: - cmpq $4,%r12 - je L$_small_initial_num_blocks_is_4_155 - jl L$_small_initial_num_blocks_is_3_1_155 - - cmpq $7,%r12 - je L$_small_initial_num_blocks_is_7_155 - cmpq $6,%r12 - je L$_small_initial_num_blocks_is_6_155 - jmp L$_small_initial_num_blocks_is_5_155 - -L$_small_initial_num_blocks_is_3_1_155: - - cmpq $3,%r12 - je L$_small_initial_num_blocks_is_3_155 - cmpq $2,%r12 - je L$_small_initial_num_blocks_is_2_155 - - - - - -L$_small_initial_num_blocks_is_1_155: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_156 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_156 -L$_small_initial_partial_block_156: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp L$_after_reduction_156 -L$_small_initial_compute_done_156: -L$_after_reduction_156: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_2_155: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_157 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_157 -L$_small_initial_partial_block_157: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_157: - - orq %r8,%r8 - je L$_after_reduction_157 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_157: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_3_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_158 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_158 -L$_small_initial_partial_block_158: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_158: - - orq %r8,%r8 - je L$_after_reduction_158 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_158: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_4_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_159 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_159 -L$_small_initial_partial_block_159: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_159: - - orq %r8,%r8 - je L$_after_reduction_159 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_159: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_5_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %xmm29,%xmm3,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_160 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_160 -L$_small_initial_partial_block_160: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_160: - - orq %r8,%r8 - je L$_after_reduction_160 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_160: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_6_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %ymm29,%ymm3,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_161 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_161 -L$_small_initial_partial_block_161: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_161: - - orq %r8,%r8 - je L$_after_reduction_161 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_161: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_7_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_162 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_162 -L$_small_initial_partial_block_162: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_162: - - orq %r8,%r8 - je L$_after_reduction_162 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_162: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_8_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_163 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_163 -L$_small_initial_partial_block_163: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_163: - - orq %r8,%r8 - je L$_after_reduction_163 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_163: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_9_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %xmm29,%xmm4,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_164 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_164 -L$_small_initial_partial_block_164: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_164: - - orq %r8,%r8 - je L$_after_reduction_164 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_164: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_10_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %ymm29,%ymm4,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_165 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_165 -L$_small_initial_partial_block_165: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_165: - - orq %r8,%r8 - je L$_after_reduction_165 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_165: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_11_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_166 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_166 -L$_small_initial_partial_block_166: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_166: - - orq %r8,%r8 - je L$_after_reduction_166 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_166: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_12_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_167 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_167 -L$_small_initial_partial_block_167: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_167: - - orq %r8,%r8 - je L$_after_reduction_167 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_167: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_13_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %xmm29,%xmm5,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_168 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_168 -L$_small_initial_partial_block_168: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_168: - - orq %r8,%r8 - je L$_after_reduction_168 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_168: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_14_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %ymm29,%ymm5,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_169 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_169 -L$_small_initial_partial_block_169: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_169: - - orq %r8,%r8 - je L$_after_reduction_169 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_169: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_15_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_170 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_170 -L$_small_initial_partial_block_170: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_170: - - orq %r8,%r8 - je L$_after_reduction_170 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_170: - jmp L$_small_initial_blocks_encrypted_155 -L$_small_initial_num_blocks_is_16_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_171: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_171: - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_171: -L$_small_initial_blocks_encrypted_155: -L$_ghash_done_10: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -L$_enc_dec_done_10: - jmp L$exit_gcm_encrypt -.p2align 5 -L$aes_gcm_encrypt_192_avx512: - orq %r8,%r8 - je L$_enc_dec_done_172 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je L$_partial_block_done_173 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge L$_no_extra_mask_173 - subq %r13,%r12 -L$_no_extra_mask_173: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm3,%xmm14,%xmm14 - cmpq $0,%r13 - jl L$_partial_incomplete_173 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp L$_enc_dec_done_173 - -L$_partial_incomplete_173: - addq %r8,(%rdx) - movq %r8,%r11 - -L$_enc_dec_done_173: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -L$_partial_block_done_173: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je L$_enc_dec_done_172 - cmpq $256,%r8 - jbe L$_message_below_equal_16_blocks_172 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_174 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_174 -L$_next_16_overflow_174: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_174: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_175 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -L$_skip_hkeys_precomputation_175: - cmpq $512,%r8 - jb L$_message_below_32_blocks_172 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_176 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_176 -L$_next_16_overflow_176: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_176: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_177 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -L$_skip_hkeys_precomputation_177: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb L$_no_more_big_nblocks_172 -L$_encrypt_big_nblocks_172: - cmpb $240,%r15b - jae L$_16_blocks_overflow_178 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_178 -L$_16_blocks_overflow_178: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_178: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_179 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_179 -L$_16_blocks_overflow_179: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_179: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_180 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_180 -L$_16_blocks_overflow_180: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_180: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae L$_encrypt_big_nblocks_172 - -L$_no_more_big_nblocks_172: - - cmpq $512,%r8 - jae L$_encrypt_32_blocks_172 - - cmpq $256,%r8 - jae L$_encrypt_16_blocks_172 -L$_encrypt_0_blocks_ghash_32_172: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_181 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_181 - jb L$_last_num_blocks_is_7_1_181 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_181 - jb L$_last_num_blocks_is_11_9_181 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_181 - ja L$_last_num_blocks_is_16_181 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_181 - jmp L$_last_num_blocks_is_13_181 - -L$_last_num_blocks_is_11_9_181: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_181 - ja L$_last_num_blocks_is_11_181 - jmp L$_last_num_blocks_is_9_181 - -L$_last_num_blocks_is_7_1_181: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_181 - jb L$_last_num_blocks_is_3_1_181 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_181 - je L$_last_num_blocks_is_6_181 - jmp L$_last_num_blocks_is_5_181 - -L$_last_num_blocks_is_3_1_181: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_181 - je L$_last_num_blocks_is_2_181 -L$_last_num_blocks_is_1_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_182 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_182 - -L$_16_blocks_overflow_182: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_182: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_183 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_183 -L$_small_initial_partial_block_183: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_183 -L$_small_initial_compute_done_183: -L$_after_reduction_183: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_2_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_184 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_184 - -L$_16_blocks_overflow_184: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_184: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_185 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_185 -L$_small_initial_partial_block_185: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_185: - - orq %r8,%r8 - je L$_after_reduction_185 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_185: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_3_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_186 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_186 - -L$_16_blocks_overflow_186: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_186: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_187 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_187 -L$_small_initial_partial_block_187: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_187: - - orq %r8,%r8 - je L$_after_reduction_187 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_187: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_4_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_188 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_188 - -L$_16_blocks_overflow_188: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_188: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_189 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_189 -L$_small_initial_partial_block_189: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_189: - - orq %r8,%r8 - je L$_after_reduction_189 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_189: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_5_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_190 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_190 - -L$_16_blocks_overflow_190: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_190: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_191 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_191 -L$_small_initial_partial_block_191: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_191: - - orq %r8,%r8 - je L$_after_reduction_191 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_191: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_6_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_192 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_192 - -L$_16_blocks_overflow_192: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_192: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_193 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_193 -L$_small_initial_partial_block_193: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_193: - - orq %r8,%r8 - je L$_after_reduction_193 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_193: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_7_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_194 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_194 - -L$_16_blocks_overflow_194: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_194: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_195 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_195 -L$_small_initial_partial_block_195: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_195: - - orq %r8,%r8 - je L$_after_reduction_195 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_195: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_8_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_196 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_196 - -L$_16_blocks_overflow_196: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_196: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_197 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_197 -L$_small_initial_partial_block_197: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_197: - - orq %r8,%r8 - je L$_after_reduction_197 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_197: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_9_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_198 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_198 - -L$_16_blocks_overflow_198: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_198: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_199 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_199 -L$_small_initial_partial_block_199: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_199: - - orq %r8,%r8 - je L$_after_reduction_199 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_199: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_10_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_200 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_200 - -L$_16_blocks_overflow_200: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_200: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_201 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_201 -L$_small_initial_partial_block_201: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_201: - - orq %r8,%r8 - je L$_after_reduction_201 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_201: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_11_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_202 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_202 - -L$_16_blocks_overflow_202: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_202: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_203 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_203 -L$_small_initial_partial_block_203: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_203: - - orq %r8,%r8 - je L$_after_reduction_203 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_203: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_12_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_204 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_204 - -L$_16_blocks_overflow_204: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_204: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_205 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_205 -L$_small_initial_partial_block_205: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_205: - - orq %r8,%r8 - je L$_after_reduction_205 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_205: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_13_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_206 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_206 - -L$_16_blocks_overflow_206: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_206: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_207 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_207 -L$_small_initial_partial_block_207: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_207: - - orq %r8,%r8 - je L$_after_reduction_207 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_207: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_14_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_208 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_208 - -L$_16_blocks_overflow_208: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_208: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_209 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_209 -L$_small_initial_partial_block_209: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_209: - - orq %r8,%r8 - je L$_after_reduction_209 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_209: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_15_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_210 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_210 - -L$_16_blocks_overflow_210: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_210: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_211 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_211 -L$_small_initial_partial_block_211: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_211: - - orq %r8,%r8 - je L$_after_reduction_211 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_211: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_16_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_212 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_212 - -L$_16_blocks_overflow_212: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_212: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_213: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_213: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_213: - jmp L$_last_blocks_done_181 -L$_last_num_blocks_is_0_181: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_181: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_172 -L$_encrypt_32_blocks_172: - cmpb $240,%r15b - jae L$_16_blocks_overflow_214 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_214 -L$_16_blocks_overflow_214: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_214: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_215 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_215 -L$_16_blocks_overflow_215: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_215: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_216 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_216 - jb L$_last_num_blocks_is_7_1_216 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_216 - jb L$_last_num_blocks_is_11_9_216 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_216 - ja L$_last_num_blocks_is_16_216 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_216 - jmp L$_last_num_blocks_is_13_216 - -L$_last_num_blocks_is_11_9_216: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_216 - ja L$_last_num_blocks_is_11_216 - jmp L$_last_num_blocks_is_9_216 - -L$_last_num_blocks_is_7_1_216: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_216 - jb L$_last_num_blocks_is_3_1_216 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_216 - je L$_last_num_blocks_is_6_216 - jmp L$_last_num_blocks_is_5_216 - -L$_last_num_blocks_is_3_1_216: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_216 - je L$_last_num_blocks_is_2_216 -L$_last_num_blocks_is_1_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_217 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_217 - -L$_16_blocks_overflow_217: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_217: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_218 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_218 -L$_small_initial_partial_block_218: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_218 -L$_small_initial_compute_done_218: -L$_after_reduction_218: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_2_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_219 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_219 - -L$_16_blocks_overflow_219: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_219: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_220 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_220 -L$_small_initial_partial_block_220: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_220: - - orq %r8,%r8 - je L$_after_reduction_220 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_220: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_3_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_221 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_221 - -L$_16_blocks_overflow_221: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_221: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_222 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_222 -L$_small_initial_partial_block_222: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_222: - - orq %r8,%r8 - je L$_after_reduction_222 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_222: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_4_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_223 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_223 - -L$_16_blocks_overflow_223: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_223: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_224 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_224 -L$_small_initial_partial_block_224: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_224: - - orq %r8,%r8 - je L$_after_reduction_224 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_224: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_5_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_225 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_225 - -L$_16_blocks_overflow_225: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_225: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_226 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_226 -L$_small_initial_partial_block_226: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_226: - - orq %r8,%r8 - je L$_after_reduction_226 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_226: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_6_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_227 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_227 - -L$_16_blocks_overflow_227: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_227: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_228 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_228 -L$_small_initial_partial_block_228: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_228: - - orq %r8,%r8 - je L$_after_reduction_228 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_228: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_7_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_229 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_229 - -L$_16_blocks_overflow_229: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_229: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_230 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_230 -L$_small_initial_partial_block_230: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_230: - - orq %r8,%r8 - je L$_after_reduction_230 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_230: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_8_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_231 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_231 - -L$_16_blocks_overflow_231: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_231: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_232 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_232 -L$_small_initial_partial_block_232: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_232: - - orq %r8,%r8 - je L$_after_reduction_232 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_232: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_9_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_233 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_233 - -L$_16_blocks_overflow_233: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_233: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_234 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_234 -L$_small_initial_partial_block_234: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_234: - - orq %r8,%r8 - je L$_after_reduction_234 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_234: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_10_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_235 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_235 - -L$_16_blocks_overflow_235: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_235: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_236 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_236 -L$_small_initial_partial_block_236: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_236: - - orq %r8,%r8 - je L$_after_reduction_236 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_236: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_11_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_237 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_237 - -L$_16_blocks_overflow_237: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_237: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_238 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_238 -L$_small_initial_partial_block_238: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_238: - - orq %r8,%r8 - je L$_after_reduction_238 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_238: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_12_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_239 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_239 - -L$_16_blocks_overflow_239: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_239: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_240 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_240 -L$_small_initial_partial_block_240: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_240: - - orq %r8,%r8 - je L$_after_reduction_240 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_240: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_13_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_241 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_241 - -L$_16_blocks_overflow_241: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_241: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_242 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_242 -L$_small_initial_partial_block_242: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_242: - - orq %r8,%r8 - je L$_after_reduction_242 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_242: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_14_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_243 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_243 - -L$_16_blocks_overflow_243: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_243: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_244 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_244 -L$_small_initial_partial_block_244: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_244: - - orq %r8,%r8 - je L$_after_reduction_244 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_244: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_15_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_245 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_245 - -L$_16_blocks_overflow_245: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_245: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_246 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_246 -L$_small_initial_partial_block_246: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_246: - - orq %r8,%r8 - je L$_after_reduction_246 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_246: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_16_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_247 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_247 - -L$_16_blocks_overflow_247: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_247: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_248: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_248: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_248: - jmp L$_last_blocks_done_216 -L$_last_num_blocks_is_0_216: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_216: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_172 -L$_encrypt_16_blocks_172: - cmpb $240,%r15b - jae L$_16_blocks_overflow_249 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_249 -L$_16_blocks_overflow_249: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_249: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_250 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_250 - jb L$_last_num_blocks_is_7_1_250 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_250 - jb L$_last_num_blocks_is_11_9_250 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_250 - ja L$_last_num_blocks_is_16_250 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_250 - jmp L$_last_num_blocks_is_13_250 - -L$_last_num_blocks_is_11_9_250: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_250 - ja L$_last_num_blocks_is_11_250 - jmp L$_last_num_blocks_is_9_250 - -L$_last_num_blocks_is_7_1_250: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_250 - jb L$_last_num_blocks_is_3_1_250 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_250 - je L$_last_num_blocks_is_6_250 - jmp L$_last_num_blocks_is_5_250 - -L$_last_num_blocks_is_3_1_250: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_250 - je L$_last_num_blocks_is_2_250 -L$_last_num_blocks_is_1_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_251 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_251 - -L$_16_blocks_overflow_251: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_251: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_252 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_252 -L$_small_initial_partial_block_252: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_252 -L$_small_initial_compute_done_252: -L$_after_reduction_252: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_2_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_253 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_253 - -L$_16_blocks_overflow_253: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_253: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_254 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_254 -L$_small_initial_partial_block_254: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_254: - - orq %r8,%r8 - je L$_after_reduction_254 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_254: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_3_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_255 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_255 - -L$_16_blocks_overflow_255: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_255: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_256 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_256 -L$_small_initial_partial_block_256: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_256: - - orq %r8,%r8 - je L$_after_reduction_256 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_256: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_4_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_257 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_257 - -L$_16_blocks_overflow_257: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_257: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_258 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_258 -L$_small_initial_partial_block_258: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_258: - - orq %r8,%r8 - je L$_after_reduction_258 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_258: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_5_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_259 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_259 - -L$_16_blocks_overflow_259: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_259: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_260 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_260 -L$_small_initial_partial_block_260: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_260: - - orq %r8,%r8 - je L$_after_reduction_260 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_260: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_6_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_261 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_261 - -L$_16_blocks_overflow_261: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_261: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_262 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_262 -L$_small_initial_partial_block_262: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_262: - - orq %r8,%r8 - je L$_after_reduction_262 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_262: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_7_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_263 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_263 - -L$_16_blocks_overflow_263: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_263: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_264 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_264 -L$_small_initial_partial_block_264: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_264: - - orq %r8,%r8 - je L$_after_reduction_264 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_264: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_8_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_265 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_265 - -L$_16_blocks_overflow_265: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_265: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_266 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_266 -L$_small_initial_partial_block_266: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_266: - - orq %r8,%r8 - je L$_after_reduction_266 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_266: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_9_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_267 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_267 - -L$_16_blocks_overflow_267: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_267: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_268 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_268 -L$_small_initial_partial_block_268: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_268: - - orq %r8,%r8 - je L$_after_reduction_268 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_268: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_10_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_269 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_269 - -L$_16_blocks_overflow_269: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_269: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_270 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_270 -L$_small_initial_partial_block_270: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_270: - - orq %r8,%r8 - je L$_after_reduction_270 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_270: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_11_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_271 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_271 - -L$_16_blocks_overflow_271: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_271: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_272 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_272 -L$_small_initial_partial_block_272: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_272: - - orq %r8,%r8 - je L$_after_reduction_272 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_272: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_12_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_273 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_273 - -L$_16_blocks_overflow_273: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_273: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_274 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_274 -L$_small_initial_partial_block_274: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_274: - - orq %r8,%r8 - je L$_after_reduction_274 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_274: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_13_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_275 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_275 - -L$_16_blocks_overflow_275: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_275: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_276 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_276 -L$_small_initial_partial_block_276: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_276: - - orq %r8,%r8 - je L$_after_reduction_276 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_276: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_14_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_277 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_277 - -L$_16_blocks_overflow_277: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_277: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_278 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_278 -L$_small_initial_partial_block_278: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_278: - - orq %r8,%r8 - je L$_after_reduction_278 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_278: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_15_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_279 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_279 - -L$_16_blocks_overflow_279: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_279: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_280 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_280 -L$_small_initial_partial_block_280: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_280: - - orq %r8,%r8 - je L$_after_reduction_280 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_280: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_16_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_281 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_281 - -L$_16_blocks_overflow_281: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_281: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_282: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_282: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_282: - jmp L$_last_blocks_done_250 -L$_last_num_blocks_is_0_250: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_250: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_172 - -L$_message_below_32_blocks_172: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_283 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -L$_skip_hkeys_precomputation_283: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_284 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_284 - jb L$_last_num_blocks_is_7_1_284 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_284 - jb L$_last_num_blocks_is_11_9_284 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_284 - ja L$_last_num_blocks_is_16_284 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_284 - jmp L$_last_num_blocks_is_13_284 - -L$_last_num_blocks_is_11_9_284: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_284 - ja L$_last_num_blocks_is_11_284 - jmp L$_last_num_blocks_is_9_284 - -L$_last_num_blocks_is_7_1_284: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_284 - jb L$_last_num_blocks_is_3_1_284 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_284 - je L$_last_num_blocks_is_6_284 - jmp L$_last_num_blocks_is_5_284 - -L$_last_num_blocks_is_3_1_284: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_284 - je L$_last_num_blocks_is_2_284 -L$_last_num_blocks_is_1_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_285 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_285 - -L$_16_blocks_overflow_285: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_285: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_286 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_286 -L$_small_initial_partial_block_286: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_286 -L$_small_initial_compute_done_286: -L$_after_reduction_286: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_2_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_287 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_287 - -L$_16_blocks_overflow_287: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_287: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_288 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_288 -L$_small_initial_partial_block_288: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_288: - - orq %r8,%r8 - je L$_after_reduction_288 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_288: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_3_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_289 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_289 - -L$_16_blocks_overflow_289: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_289: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_290 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_290 -L$_small_initial_partial_block_290: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_290: - - orq %r8,%r8 - je L$_after_reduction_290 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_290: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_4_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_291 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_291 - -L$_16_blocks_overflow_291: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_291: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_292 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_292 -L$_small_initial_partial_block_292: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_292: - - orq %r8,%r8 - je L$_after_reduction_292 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_292: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_5_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_293 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_293 - -L$_16_blocks_overflow_293: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_293: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_294 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_294 -L$_small_initial_partial_block_294: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_294: - - orq %r8,%r8 - je L$_after_reduction_294 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_294: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_6_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_295 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_295 - -L$_16_blocks_overflow_295: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_295: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_296 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_296 -L$_small_initial_partial_block_296: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_296: - - orq %r8,%r8 - je L$_after_reduction_296 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_296: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_7_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_297 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_297 - -L$_16_blocks_overflow_297: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_297: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_298 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_298 -L$_small_initial_partial_block_298: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_298: - - orq %r8,%r8 - je L$_after_reduction_298 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_298: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_8_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_299 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_299 - -L$_16_blocks_overflow_299: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_299: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_300 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_300 -L$_small_initial_partial_block_300: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_300: - - orq %r8,%r8 - je L$_after_reduction_300 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_300: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_9_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_301 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_301 - -L$_16_blocks_overflow_301: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_301: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_302 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_302 -L$_small_initial_partial_block_302: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_302: - - orq %r8,%r8 - je L$_after_reduction_302 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_302: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_10_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_303 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_303 - -L$_16_blocks_overflow_303: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_303: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_304 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_304 -L$_small_initial_partial_block_304: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_304: - - orq %r8,%r8 - je L$_after_reduction_304 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_304: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_11_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_305 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_305 - -L$_16_blocks_overflow_305: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_305: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_306 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_306 -L$_small_initial_partial_block_306: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_306: - - orq %r8,%r8 - je L$_after_reduction_306 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_306: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_12_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_307 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_307 - -L$_16_blocks_overflow_307: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_307: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_308 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_308 -L$_small_initial_partial_block_308: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_308: - - orq %r8,%r8 - je L$_after_reduction_308 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_308: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_13_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_309 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_309 - -L$_16_blocks_overflow_309: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_309: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_310 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_310 -L$_small_initial_partial_block_310: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_310: - - orq %r8,%r8 - je L$_after_reduction_310 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_310: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_14_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_311 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_311 - -L$_16_blocks_overflow_311: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_311: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_312 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_312 -L$_small_initial_partial_block_312: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_312: - - orq %r8,%r8 - je L$_after_reduction_312 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_312: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_15_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_313 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_313 - -L$_16_blocks_overflow_313: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_313: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_314 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_314 -L$_small_initial_partial_block_314: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_314: - - orq %r8,%r8 - je L$_after_reduction_314 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_314: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_16_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_315 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_315 - -L$_16_blocks_overflow_315: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_315: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_316: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_316: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_316: - jmp L$_last_blocks_done_284 -L$_last_num_blocks_is_0_284: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_284: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_172 - -L$_message_below_equal_16_blocks_172: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je L$_small_initial_num_blocks_is_8_317 - jl L$_small_initial_num_blocks_is_7_1_317 - - - cmpq $12,%r12 - je L$_small_initial_num_blocks_is_12_317 - jl L$_small_initial_num_blocks_is_11_9_317 - - - cmpq $16,%r12 - je L$_small_initial_num_blocks_is_16_317 - cmpq $15,%r12 - je L$_small_initial_num_blocks_is_15_317 - cmpq $14,%r12 - je L$_small_initial_num_blocks_is_14_317 - jmp L$_small_initial_num_blocks_is_13_317 - -L$_small_initial_num_blocks_is_11_9_317: - - cmpq $11,%r12 - je L$_small_initial_num_blocks_is_11_317 - cmpq $10,%r12 - je L$_small_initial_num_blocks_is_10_317 - jmp L$_small_initial_num_blocks_is_9_317 - -L$_small_initial_num_blocks_is_7_1_317: - cmpq $4,%r12 - je L$_small_initial_num_blocks_is_4_317 - jl L$_small_initial_num_blocks_is_3_1_317 - - cmpq $7,%r12 - je L$_small_initial_num_blocks_is_7_317 - cmpq $6,%r12 - je L$_small_initial_num_blocks_is_6_317 - jmp L$_small_initial_num_blocks_is_5_317 - -L$_small_initial_num_blocks_is_3_1_317: - - cmpq $3,%r12 - je L$_small_initial_num_blocks_is_3_317 - cmpq $2,%r12 - je L$_small_initial_num_blocks_is_2_317 - - - - - -L$_small_initial_num_blocks_is_1_317: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_318 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_318 -L$_small_initial_partial_block_318: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp L$_after_reduction_318 -L$_small_initial_compute_done_318: -L$_after_reduction_318: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_2_317: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_319 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_319 -L$_small_initial_partial_block_319: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_319: - - orq %r8,%r8 - je L$_after_reduction_319 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_319: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_3_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_320 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_320 -L$_small_initial_partial_block_320: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_320: - - orq %r8,%r8 - je L$_after_reduction_320 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_320: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_4_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_321 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_321 -L$_small_initial_partial_block_321: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_321: - - orq %r8,%r8 - je L$_after_reduction_321 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_321: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_5_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %xmm29,%xmm3,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_322 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_322 -L$_small_initial_partial_block_322: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_322: - - orq %r8,%r8 - je L$_after_reduction_322 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_322: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_6_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %ymm29,%ymm3,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_323 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_323 -L$_small_initial_partial_block_323: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_323: - - orq %r8,%r8 - je L$_after_reduction_323 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_323: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_7_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_324 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_324 -L$_small_initial_partial_block_324: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_324: - - orq %r8,%r8 - je L$_after_reduction_324 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_324: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_8_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_325 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_325 -L$_small_initial_partial_block_325: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_325: - - orq %r8,%r8 - je L$_after_reduction_325 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_325: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_9_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %xmm29,%xmm4,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_326 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_326 -L$_small_initial_partial_block_326: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_326: - - orq %r8,%r8 - je L$_after_reduction_326 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_326: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_10_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %ymm29,%ymm4,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_327 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_327 -L$_small_initial_partial_block_327: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_327: - - orq %r8,%r8 - je L$_after_reduction_327 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_327: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_11_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_328 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_328 -L$_small_initial_partial_block_328: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_328: - - orq %r8,%r8 - je L$_after_reduction_328 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_328: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_12_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_329 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_329 -L$_small_initial_partial_block_329: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_329: - - orq %r8,%r8 - je L$_after_reduction_329 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_329: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_13_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %xmm29,%xmm5,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_330 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_330 -L$_small_initial_partial_block_330: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_330: - - orq %r8,%r8 - je L$_after_reduction_330 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_330: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_14_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %ymm29,%ymm5,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_331 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_331 -L$_small_initial_partial_block_331: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_331: - - orq %r8,%r8 - je L$_after_reduction_331 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_331: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_15_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_332 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_332 -L$_small_initial_partial_block_332: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_332: - - orq %r8,%r8 - je L$_after_reduction_332 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_332: - jmp L$_small_initial_blocks_encrypted_317 -L$_small_initial_num_blocks_is_16_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_333: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_333: - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_333: -L$_small_initial_blocks_encrypted_317: -L$_ghash_done_172: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -L$_enc_dec_done_172: - jmp L$exit_gcm_encrypt -.p2align 5 -L$aes_gcm_encrypt_256_avx512: - orq %r8,%r8 - je L$_enc_dec_done_334 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je L$_partial_block_done_335 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge L$_no_extra_mask_335 - subq %r13,%r12 -L$_no_extra_mask_335: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm3,%xmm14,%xmm14 - cmpq $0,%r13 - jl L$_partial_incomplete_335 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp L$_enc_dec_done_335 - -L$_partial_incomplete_335: - addq %r8,(%rdx) - movq %r8,%r11 - -L$_enc_dec_done_335: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -L$_partial_block_done_335: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je L$_enc_dec_done_334 - cmpq $256,%r8 - jbe L$_message_below_equal_16_blocks_334 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_336 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_336 -L$_next_16_overflow_336: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_336: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 208(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 224(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_337 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -L$_skip_hkeys_precomputation_337: - cmpq $512,%r8 - jb L$_message_below_32_blocks_334 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_338 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_338 -L$_next_16_overflow_338: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_338: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 208(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 224(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_339 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -L$_skip_hkeys_precomputation_339: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb L$_no_more_big_nblocks_334 -L$_encrypt_big_nblocks_334: - cmpb $240,%r15b - jae L$_16_blocks_overflow_340 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_340 -L$_16_blocks_overflow_340: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_340: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_341 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_341 -L$_16_blocks_overflow_341: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_341: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_342 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_342 -L$_16_blocks_overflow_342: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_342: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae L$_encrypt_big_nblocks_334 - -L$_no_more_big_nblocks_334: - - cmpq $512,%r8 - jae L$_encrypt_32_blocks_334 - - cmpq $256,%r8 - jae L$_encrypt_16_blocks_334 -L$_encrypt_0_blocks_ghash_32_334: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_343 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_343 - jb L$_last_num_blocks_is_7_1_343 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_343 - jb L$_last_num_blocks_is_11_9_343 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_343 - ja L$_last_num_blocks_is_16_343 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_343 - jmp L$_last_num_blocks_is_13_343 - -L$_last_num_blocks_is_11_9_343: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_343 - ja L$_last_num_blocks_is_11_343 - jmp L$_last_num_blocks_is_9_343 - -L$_last_num_blocks_is_7_1_343: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_343 - jb L$_last_num_blocks_is_3_1_343 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_343 - je L$_last_num_blocks_is_6_343 - jmp L$_last_num_blocks_is_5_343 - -L$_last_num_blocks_is_3_1_343: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_343 - je L$_last_num_blocks_is_2_343 -L$_last_num_blocks_is_1_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_344 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_344 - -L$_16_blocks_overflow_344: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_344: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_345 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_345 -L$_small_initial_partial_block_345: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_345 -L$_small_initial_compute_done_345: -L$_after_reduction_345: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_2_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_346 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_346 - -L$_16_blocks_overflow_346: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_346: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_347 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_347 -L$_small_initial_partial_block_347: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_347: - - orq %r8,%r8 - je L$_after_reduction_347 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_347: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_3_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_348 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_348 - -L$_16_blocks_overflow_348: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_348: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_349 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_349 -L$_small_initial_partial_block_349: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_349: - - orq %r8,%r8 - je L$_after_reduction_349 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_349: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_4_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_350 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_350 - -L$_16_blocks_overflow_350: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_350: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_351 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_351 -L$_small_initial_partial_block_351: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_351: - - orq %r8,%r8 - je L$_after_reduction_351 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_351: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_5_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_352 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_352 - -L$_16_blocks_overflow_352: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_352: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_353 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_353 -L$_small_initial_partial_block_353: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_353: - - orq %r8,%r8 - je L$_after_reduction_353 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_353: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_6_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_354 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_354 - -L$_16_blocks_overflow_354: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_354: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_355 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_355 -L$_small_initial_partial_block_355: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_355: - - orq %r8,%r8 - je L$_after_reduction_355 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_355: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_7_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_356 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_356 - -L$_16_blocks_overflow_356: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_356: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_357 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_357 -L$_small_initial_partial_block_357: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_357: - - orq %r8,%r8 - je L$_after_reduction_357 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_357: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_8_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_358 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_358 - -L$_16_blocks_overflow_358: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_358: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_359 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_359 -L$_small_initial_partial_block_359: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_359: - - orq %r8,%r8 - je L$_after_reduction_359 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_359: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_9_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_360 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_360 - -L$_16_blocks_overflow_360: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_360: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_361 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_361 -L$_small_initial_partial_block_361: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_361: - - orq %r8,%r8 - je L$_after_reduction_361 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_361: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_10_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_362 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_362 - -L$_16_blocks_overflow_362: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_362: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_363 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_363 -L$_small_initial_partial_block_363: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_363: - - orq %r8,%r8 - je L$_after_reduction_363 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_363: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_11_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_364 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_364 - -L$_16_blocks_overflow_364: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_364: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_365 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_365 -L$_small_initial_partial_block_365: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_365: - - orq %r8,%r8 - je L$_after_reduction_365 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_365: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_12_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_366 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_366 - -L$_16_blocks_overflow_366: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_366: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_367 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_367 -L$_small_initial_partial_block_367: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_367: - - orq %r8,%r8 - je L$_after_reduction_367 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_367: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_13_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_368 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_368 - -L$_16_blocks_overflow_368: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_368: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_369 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_369 -L$_small_initial_partial_block_369: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_369: - - orq %r8,%r8 - je L$_after_reduction_369 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_369: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_14_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_370 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_370 - -L$_16_blocks_overflow_370: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_370: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_371 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_371 -L$_small_initial_partial_block_371: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_371: - - orq %r8,%r8 - je L$_after_reduction_371 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_371: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_15_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_372 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_372 - -L$_16_blocks_overflow_372: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_372: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_373 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_373 -L$_small_initial_partial_block_373: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_373: - - orq %r8,%r8 - je L$_after_reduction_373 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_373: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_16_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_374 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_374 - -L$_16_blocks_overflow_374: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_374: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_375: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_375: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_375: - jmp L$_last_blocks_done_343 -L$_last_num_blocks_is_0_343: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_343: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_334 -L$_encrypt_32_blocks_334: - cmpb $240,%r15b - jae L$_16_blocks_overflow_376 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_376 -L$_16_blocks_overflow_376: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_376: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_377 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_377 -L$_16_blocks_overflow_377: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_377: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_378 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_378 - jb L$_last_num_blocks_is_7_1_378 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_378 - jb L$_last_num_blocks_is_11_9_378 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_378 - ja L$_last_num_blocks_is_16_378 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_378 - jmp L$_last_num_blocks_is_13_378 - -L$_last_num_blocks_is_11_9_378: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_378 - ja L$_last_num_blocks_is_11_378 - jmp L$_last_num_blocks_is_9_378 - -L$_last_num_blocks_is_7_1_378: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_378 - jb L$_last_num_blocks_is_3_1_378 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_378 - je L$_last_num_blocks_is_6_378 - jmp L$_last_num_blocks_is_5_378 - -L$_last_num_blocks_is_3_1_378: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_378 - je L$_last_num_blocks_is_2_378 -L$_last_num_blocks_is_1_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_379 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_379 - -L$_16_blocks_overflow_379: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_379: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_380 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_380 -L$_small_initial_partial_block_380: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_380 -L$_small_initial_compute_done_380: -L$_after_reduction_380: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_2_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_381 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_381 - -L$_16_blocks_overflow_381: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_381: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_382 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_382 -L$_small_initial_partial_block_382: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_382: - - orq %r8,%r8 - je L$_after_reduction_382 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_382: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_3_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_383 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_383 - -L$_16_blocks_overflow_383: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_383: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_384 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_384 -L$_small_initial_partial_block_384: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_384: - - orq %r8,%r8 - je L$_after_reduction_384 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_384: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_4_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_385 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_385 - -L$_16_blocks_overflow_385: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_385: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_386 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_386 -L$_small_initial_partial_block_386: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_386: - - orq %r8,%r8 - je L$_after_reduction_386 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_386: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_5_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_387 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_387 - -L$_16_blocks_overflow_387: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_387: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_388 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_388 -L$_small_initial_partial_block_388: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_388: - - orq %r8,%r8 - je L$_after_reduction_388 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_388: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_6_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_389 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_389 - -L$_16_blocks_overflow_389: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_389: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_390 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_390 -L$_small_initial_partial_block_390: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_390: - - orq %r8,%r8 - je L$_after_reduction_390 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_390: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_7_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_391 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_391 - -L$_16_blocks_overflow_391: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_391: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_392 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_392 -L$_small_initial_partial_block_392: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_392: - - orq %r8,%r8 - je L$_after_reduction_392 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_392: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_8_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_393 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_393 - -L$_16_blocks_overflow_393: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_393: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_394 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_394 -L$_small_initial_partial_block_394: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_394: - - orq %r8,%r8 - je L$_after_reduction_394 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_394: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_9_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_395 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_395 - -L$_16_blocks_overflow_395: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_395: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_396 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_396 -L$_small_initial_partial_block_396: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_396: - - orq %r8,%r8 - je L$_after_reduction_396 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_396: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_10_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_397 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_397 - -L$_16_blocks_overflow_397: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_397: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_398 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_398 -L$_small_initial_partial_block_398: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_398: - - orq %r8,%r8 - je L$_after_reduction_398 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_398: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_11_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_399 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_399 - -L$_16_blocks_overflow_399: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_399: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_400 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_400 -L$_small_initial_partial_block_400: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_400: - - orq %r8,%r8 - je L$_after_reduction_400 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_400: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_12_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_401 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_401 - -L$_16_blocks_overflow_401: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_401: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_402 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_402 -L$_small_initial_partial_block_402: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_402: - - orq %r8,%r8 - je L$_after_reduction_402 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_402: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_13_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_403 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_403 - -L$_16_blocks_overflow_403: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_403: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_404 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_404 -L$_small_initial_partial_block_404: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_404: - - orq %r8,%r8 - je L$_after_reduction_404 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_404: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_14_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_405 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_405 - -L$_16_blocks_overflow_405: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_405: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_406 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_406 -L$_small_initial_partial_block_406: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_406: - - orq %r8,%r8 - je L$_after_reduction_406 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_406: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_15_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_407 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_407 - -L$_16_blocks_overflow_407: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_407: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_408 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_408 -L$_small_initial_partial_block_408: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_408: - - orq %r8,%r8 - je L$_after_reduction_408 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_408: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_16_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_409 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_409 - -L$_16_blocks_overflow_409: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_409: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_410: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_410: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_410: - jmp L$_last_blocks_done_378 -L$_last_num_blocks_is_0_378: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_378: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_334 -L$_encrypt_16_blocks_334: - cmpb $240,%r15b - jae L$_16_blocks_overflow_411 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_411 -L$_16_blocks_overflow_411: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_411: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_412 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_412 - jb L$_last_num_blocks_is_7_1_412 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_412 - jb L$_last_num_blocks_is_11_9_412 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_412 - ja L$_last_num_blocks_is_16_412 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_412 - jmp L$_last_num_blocks_is_13_412 - -L$_last_num_blocks_is_11_9_412: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_412 - ja L$_last_num_blocks_is_11_412 - jmp L$_last_num_blocks_is_9_412 - -L$_last_num_blocks_is_7_1_412: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_412 - jb L$_last_num_blocks_is_3_1_412 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_412 - je L$_last_num_blocks_is_6_412 - jmp L$_last_num_blocks_is_5_412 - -L$_last_num_blocks_is_3_1_412: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_412 - je L$_last_num_blocks_is_2_412 -L$_last_num_blocks_is_1_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_413 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_413 - -L$_16_blocks_overflow_413: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_413: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_414 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_414 -L$_small_initial_partial_block_414: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_414 -L$_small_initial_compute_done_414: -L$_after_reduction_414: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_2_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_415 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_415 - -L$_16_blocks_overflow_415: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_415: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_416 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_416 -L$_small_initial_partial_block_416: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_416: - - orq %r8,%r8 - je L$_after_reduction_416 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_416: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_3_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_417 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_417 - -L$_16_blocks_overflow_417: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_417: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_418 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_418 -L$_small_initial_partial_block_418: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_418: - - orq %r8,%r8 - je L$_after_reduction_418 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_418: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_4_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_419 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_419 - -L$_16_blocks_overflow_419: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_419: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_420 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_420 -L$_small_initial_partial_block_420: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_420: - - orq %r8,%r8 - je L$_after_reduction_420 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_420: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_5_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_421 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_421 - -L$_16_blocks_overflow_421: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_421: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_422 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_422 -L$_small_initial_partial_block_422: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_422: - - orq %r8,%r8 - je L$_after_reduction_422 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_422: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_6_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_423 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_423 - -L$_16_blocks_overflow_423: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_423: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_424 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_424 -L$_small_initial_partial_block_424: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_424: - - orq %r8,%r8 - je L$_after_reduction_424 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_424: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_7_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_425 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_425 - -L$_16_blocks_overflow_425: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_425: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_426 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_426 -L$_small_initial_partial_block_426: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_426: - - orq %r8,%r8 - je L$_after_reduction_426 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_426: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_8_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_427 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_427 - -L$_16_blocks_overflow_427: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_427: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_428 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_428 -L$_small_initial_partial_block_428: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_428: - - orq %r8,%r8 - je L$_after_reduction_428 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_428: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_9_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_429 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_429 - -L$_16_blocks_overflow_429: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_429: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_430 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_430 -L$_small_initial_partial_block_430: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_430: - - orq %r8,%r8 - je L$_after_reduction_430 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_430: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_10_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_431 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_431 - -L$_16_blocks_overflow_431: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_431: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_432 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_432 -L$_small_initial_partial_block_432: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_432: - - orq %r8,%r8 - je L$_after_reduction_432 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_432: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_11_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_433 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_433 - -L$_16_blocks_overflow_433: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_433: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_434 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_434 -L$_small_initial_partial_block_434: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_434: - - orq %r8,%r8 - je L$_after_reduction_434 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_434: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_12_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_435 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_435 - -L$_16_blocks_overflow_435: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_435: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_436 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_436 -L$_small_initial_partial_block_436: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_436: - - orq %r8,%r8 - je L$_after_reduction_436 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_436: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_13_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_437 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_437 - -L$_16_blocks_overflow_437: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_437: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_438 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_438 -L$_small_initial_partial_block_438: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_438: - - orq %r8,%r8 - je L$_after_reduction_438 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_438: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_14_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_439 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_439 - -L$_16_blocks_overflow_439: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_439: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_440 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_440 -L$_small_initial_partial_block_440: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_440: - - orq %r8,%r8 - je L$_after_reduction_440 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_440: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_15_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_441 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_441 - -L$_16_blocks_overflow_441: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_441: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_442 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_442 -L$_small_initial_partial_block_442: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_442: - - orq %r8,%r8 - je L$_after_reduction_442 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_442: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_16_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_443 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_443 - -L$_16_blocks_overflow_443: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_443: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_444: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_444: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_444: - jmp L$_last_blocks_done_412 -L$_last_num_blocks_is_0_412: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_412: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_334 - -L$_message_below_32_blocks_334: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_445 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -L$_skip_hkeys_precomputation_445: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_446 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_446 - jb L$_last_num_blocks_is_7_1_446 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_446 - jb L$_last_num_blocks_is_11_9_446 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_446 - ja L$_last_num_blocks_is_16_446 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_446 - jmp L$_last_num_blocks_is_13_446 - -L$_last_num_blocks_is_11_9_446: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_446 - ja L$_last_num_blocks_is_11_446 - jmp L$_last_num_blocks_is_9_446 - -L$_last_num_blocks_is_7_1_446: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_446 - jb L$_last_num_blocks_is_3_1_446 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_446 - je L$_last_num_blocks_is_6_446 - jmp L$_last_num_blocks_is_5_446 - -L$_last_num_blocks_is_3_1_446: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_446 - je L$_last_num_blocks_is_2_446 -L$_last_num_blocks_is_1_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_447 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_447 - -L$_16_blocks_overflow_447: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_447: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_448 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_448 -L$_small_initial_partial_block_448: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_448 -L$_small_initial_compute_done_448: -L$_after_reduction_448: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_2_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_449 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_449 - -L$_16_blocks_overflow_449: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_449: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_450 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_450 -L$_small_initial_partial_block_450: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_450: - - orq %r8,%r8 - je L$_after_reduction_450 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_450: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_3_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_451 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_451 - -L$_16_blocks_overflow_451: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_451: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_452 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_452 -L$_small_initial_partial_block_452: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_452: - - orq %r8,%r8 - je L$_after_reduction_452 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_452: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_4_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_453 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_453 - -L$_16_blocks_overflow_453: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_453: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_454 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_454 -L$_small_initial_partial_block_454: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_454: - - orq %r8,%r8 - je L$_after_reduction_454 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_454: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_5_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_455 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_455 - -L$_16_blocks_overflow_455: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_455: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_456 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_456 -L$_small_initial_partial_block_456: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_456: - - orq %r8,%r8 - je L$_after_reduction_456 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_456: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_6_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_457 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_457 - -L$_16_blocks_overflow_457: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_457: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_458 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_458 -L$_small_initial_partial_block_458: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_458: - - orq %r8,%r8 - je L$_after_reduction_458 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_458: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_7_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_459 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_459 - -L$_16_blocks_overflow_459: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_459: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_460 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_460 -L$_small_initial_partial_block_460: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_460: - - orq %r8,%r8 - je L$_after_reduction_460 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_460: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_8_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_461 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_461 - -L$_16_blocks_overflow_461: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_461: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_462 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_462 -L$_small_initial_partial_block_462: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_462: - - orq %r8,%r8 - je L$_after_reduction_462 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_462: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_9_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_463 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_463 - -L$_16_blocks_overflow_463: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_463: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_464 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_464 -L$_small_initial_partial_block_464: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_464: - - orq %r8,%r8 - je L$_after_reduction_464 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_464: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_10_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_465 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_465 - -L$_16_blocks_overflow_465: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_465: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_466 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_466 -L$_small_initial_partial_block_466: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_466: - - orq %r8,%r8 - je L$_after_reduction_466 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_466: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_11_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_467 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_467 - -L$_16_blocks_overflow_467: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_467: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_468 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_468 -L$_small_initial_partial_block_468: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_468: - - orq %r8,%r8 - je L$_after_reduction_468 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_468: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_12_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_469 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_469 - -L$_16_blocks_overflow_469: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_469: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_470 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_470 -L$_small_initial_partial_block_470: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_470: - - orq %r8,%r8 - je L$_after_reduction_470 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_470: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_13_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_471 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_471 - -L$_16_blocks_overflow_471: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_471: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_472 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_472 -L$_small_initial_partial_block_472: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_472: - - orq %r8,%r8 - je L$_after_reduction_472 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_472: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_14_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_473 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_473 - -L$_16_blocks_overflow_473: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_473: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_474 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_474 -L$_small_initial_partial_block_474: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_474: - - orq %r8,%r8 - je L$_after_reduction_474 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_474: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_15_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_475 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_475 - -L$_16_blocks_overflow_475: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_475: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_476 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_476 -L$_small_initial_partial_block_476: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_476: - - orq %r8,%r8 - je L$_after_reduction_476 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_476: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_16_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_477 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_477 - -L$_16_blocks_overflow_477: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_477: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_478: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_478: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_478: - jmp L$_last_blocks_done_446 -L$_last_num_blocks_is_0_446: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_446: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_334 - -L$_message_below_equal_16_blocks_334: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je L$_small_initial_num_blocks_is_8_479 - jl L$_small_initial_num_blocks_is_7_1_479 - - - cmpq $12,%r12 - je L$_small_initial_num_blocks_is_12_479 - jl L$_small_initial_num_blocks_is_11_9_479 - - - cmpq $16,%r12 - je L$_small_initial_num_blocks_is_16_479 - cmpq $15,%r12 - je L$_small_initial_num_blocks_is_15_479 - cmpq $14,%r12 - je L$_small_initial_num_blocks_is_14_479 - jmp L$_small_initial_num_blocks_is_13_479 - -L$_small_initial_num_blocks_is_11_9_479: - - cmpq $11,%r12 - je L$_small_initial_num_blocks_is_11_479 - cmpq $10,%r12 - je L$_small_initial_num_blocks_is_10_479 - jmp L$_small_initial_num_blocks_is_9_479 - -L$_small_initial_num_blocks_is_7_1_479: - cmpq $4,%r12 - je L$_small_initial_num_blocks_is_4_479 - jl L$_small_initial_num_blocks_is_3_1_479 - - cmpq $7,%r12 - je L$_small_initial_num_blocks_is_7_479 - cmpq $6,%r12 - je L$_small_initial_num_blocks_is_6_479 - jmp L$_small_initial_num_blocks_is_5_479 - -L$_small_initial_num_blocks_is_3_1_479: - - cmpq $3,%r12 - je L$_small_initial_num_blocks_is_3_479 - cmpq $2,%r12 - je L$_small_initial_num_blocks_is_2_479 - - - - - -L$_small_initial_num_blocks_is_1_479: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_480 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_480 -L$_small_initial_partial_block_480: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp L$_after_reduction_480 -L$_small_initial_compute_done_480: -L$_after_reduction_480: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_2_479: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_481 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_481 -L$_small_initial_partial_block_481: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_481: - - orq %r8,%r8 - je L$_after_reduction_481 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_481: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_3_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_482 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_482 -L$_small_initial_partial_block_482: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_482: - - orq %r8,%r8 - je L$_after_reduction_482 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_482: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_4_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_483 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_483 -L$_small_initial_partial_block_483: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_483: - - orq %r8,%r8 - je L$_after_reduction_483 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_483: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_5_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %xmm29,%xmm3,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_484 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_484 -L$_small_initial_partial_block_484: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_484: - - orq %r8,%r8 - je L$_after_reduction_484 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_484: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_6_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %ymm29,%ymm3,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_485 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_485 -L$_small_initial_partial_block_485: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_485: - - orq %r8,%r8 - je L$_after_reduction_485 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_485: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_7_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_486 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_486 -L$_small_initial_partial_block_486: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_486: - - orq %r8,%r8 - je L$_after_reduction_486 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_486: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_8_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_487 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_487 -L$_small_initial_partial_block_487: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_487: - - orq %r8,%r8 - je L$_after_reduction_487 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_487: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_9_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %xmm29,%xmm4,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_488 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_488 -L$_small_initial_partial_block_488: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_488: - - orq %r8,%r8 - je L$_after_reduction_488 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_488: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_10_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %ymm29,%ymm4,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_489 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_489 -L$_small_initial_partial_block_489: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_489: - - orq %r8,%r8 - je L$_after_reduction_489 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_489: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_11_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_490 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_490 -L$_small_initial_partial_block_490: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_490: - - orq %r8,%r8 - je L$_after_reduction_490 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_490: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_12_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_491 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_491 -L$_small_initial_partial_block_491: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_491: - - orq %r8,%r8 - je L$_after_reduction_491 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_491: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_13_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %xmm29,%xmm5,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_492 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_492 -L$_small_initial_partial_block_492: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_492: - - orq %r8,%r8 - je L$_after_reduction_492 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_492: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_14_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %ymm29,%ymm5,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_493 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_493 -L$_small_initial_partial_block_493: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_493: - - orq %r8,%r8 - je L$_after_reduction_493 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_493: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_15_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_494 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_494 -L$_small_initial_partial_block_494: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_494: - - orq %r8,%r8 - je L$_after_reduction_494 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_494: - jmp L$_small_initial_blocks_encrypted_479 -L$_small_initial_num_blocks_is_16_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_495: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_495: - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_495: -L$_small_initial_blocks_encrypted_479: -L$_ghash_done_334: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -L$_enc_dec_done_334: - jmp L$exit_gcm_encrypt -L$exit_gcm_encrypt: - cmpq $256,%r8 - jbe L$skip_hkeys_cleanup_496 - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %zmm0,0(%rsp) - vmovdqa64 %zmm0,64(%rsp) - vmovdqa64 %zmm0,128(%rsp) - vmovdqa64 %zmm0,192(%rsp) - vmovdqa64 %zmm0,256(%rsp) - vmovdqa64 %zmm0,320(%rsp) - vmovdqa64 %zmm0,384(%rsp) - vmovdqa64 %zmm0,448(%rsp) - vmovdqa64 %zmm0,512(%rsp) - vmovdqa64 %zmm0,576(%rsp) - vmovdqa64 %zmm0,640(%rsp) - vmovdqa64 %zmm0,704(%rsp) -L$skip_hkeys_cleanup_496: - vzeroupper - leaq (%rbp),%rsp - - popq %r15 - - popq %r14 - - popq %r13 - - popq %r12 - - popq %rbp - - popq %rbx - - .byte 0xf3,0xc3 -L$encrypt_seh_end: - - -.globl _ossl_aes_gcm_decrypt_avx512 - -.p2align 5 -_ossl_aes_gcm_decrypt_avx512: - -L$decrypt_seh_begin: -.byte 243,15,30,250 - pushq %rbx - -L$decrypt_seh_push_rbx: - pushq %rbp - -L$decrypt_seh_push_rbp: - pushq %r12 - -L$decrypt_seh_push_r12: - pushq %r13 - -L$decrypt_seh_push_r13: - pushq %r14 - -L$decrypt_seh_push_r14: - pushq %r15 - -L$decrypt_seh_push_r15: - - - - - - - - - - - leaq 0(%rsp),%rbp - -L$decrypt_seh_setfp: - -L$decrypt_seh_prolog_end: - subq $1588,%rsp - andq $(-64),%rsp - - - movl 240(%rdi),%eax - cmpl $9,%eax - je L$aes_gcm_decrypt_128_avx512 - cmpl $11,%eax - je L$aes_gcm_decrypt_192_avx512 - cmpl $13,%eax - je L$aes_gcm_decrypt_256_avx512 - xorl %eax,%eax - jmp L$exit_gcm_decrypt -.p2align 5 -L$aes_gcm_decrypt_128_avx512: - orq %r8,%r8 - je L$_enc_dec_done_497 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je L$_partial_block_done_498 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - - vmovdqa64 %xmm0,%xmm6 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge L$_no_extra_mask_498 - subq %r13,%r12 -L$_no_extra_mask_498: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpand %xmm0,%xmm6,%xmm6 - vpshufb SHUF_MASK(%rip),%xmm6,%xmm6 - vpshufb %xmm5,%xmm6,%xmm6 - vpxorq %xmm6,%xmm14,%xmm14 - cmpq $0,%r13 - jl L$_partial_incomplete_498 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp L$_enc_dec_done_498 - -L$_partial_incomplete_498: - addq %r8,(%rdx) - movq %r8,%r11 - -L$_enc_dec_done_498: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -L$_partial_block_done_498: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je L$_enc_dec_done_497 - cmpq $256,%r8 - jbe L$_message_below_equal_16_blocks_497 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_499 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_499 -L$_next_16_overflow_499: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_499: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_500 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -L$_skip_hkeys_precomputation_500: - cmpq $512,%r8 - jb L$_message_below_32_blocks_497 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_501 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_501 -L$_next_16_overflow_501: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_501: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_502 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -L$_skip_hkeys_precomputation_502: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb L$_no_more_big_nblocks_497 -L$_encrypt_big_nblocks_497: - cmpb $240,%r15b - jae L$_16_blocks_overflow_503 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_503 -L$_16_blocks_overflow_503: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_503: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_504 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_504 -L$_16_blocks_overflow_504: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_504: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_505 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_505 -L$_16_blocks_overflow_505: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_505: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae L$_encrypt_big_nblocks_497 - -L$_no_more_big_nblocks_497: - - cmpq $512,%r8 - jae L$_encrypt_32_blocks_497 - - cmpq $256,%r8 - jae L$_encrypt_16_blocks_497 -L$_encrypt_0_blocks_ghash_32_497: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_506 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_506 - jb L$_last_num_blocks_is_7_1_506 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_506 - jb L$_last_num_blocks_is_11_9_506 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_506 - ja L$_last_num_blocks_is_16_506 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_506 - jmp L$_last_num_blocks_is_13_506 - -L$_last_num_blocks_is_11_9_506: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_506 - ja L$_last_num_blocks_is_11_506 - jmp L$_last_num_blocks_is_9_506 - -L$_last_num_blocks_is_7_1_506: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_506 - jb L$_last_num_blocks_is_3_1_506 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_506 - je L$_last_num_blocks_is_6_506 - jmp L$_last_num_blocks_is_5_506 - -L$_last_num_blocks_is_3_1_506: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_506 - je L$_last_num_blocks_is_2_506 -L$_last_num_blocks_is_1_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_507 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_507 - -L$_16_blocks_overflow_507: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_507: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_508 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_508 -L$_small_initial_partial_block_508: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_508 -L$_small_initial_compute_done_508: -L$_after_reduction_508: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_2_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_509 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_509 - -L$_16_blocks_overflow_509: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_509: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_510 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_510 -L$_small_initial_partial_block_510: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_510: - - orq %r8,%r8 - je L$_after_reduction_510 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_510: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_3_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_511 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_511 - -L$_16_blocks_overflow_511: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_511: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_512 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_512 -L$_small_initial_partial_block_512: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_512: - - orq %r8,%r8 - je L$_after_reduction_512 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_512: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_4_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_513 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_513 - -L$_16_blocks_overflow_513: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_513: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_514 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_514 -L$_small_initial_partial_block_514: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_514: - - orq %r8,%r8 - je L$_after_reduction_514 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_514: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_5_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_515 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_515 - -L$_16_blocks_overflow_515: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_515: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_516 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_516 -L$_small_initial_partial_block_516: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_516: - - orq %r8,%r8 - je L$_after_reduction_516 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_516: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_6_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_517 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_517 - -L$_16_blocks_overflow_517: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_517: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_518 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_518 -L$_small_initial_partial_block_518: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_518: - - orq %r8,%r8 - je L$_after_reduction_518 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_518: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_7_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_519 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_519 - -L$_16_blocks_overflow_519: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_519: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_520 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_520 -L$_small_initial_partial_block_520: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_520: - - orq %r8,%r8 - je L$_after_reduction_520 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_520: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_8_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_521 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_521 - -L$_16_blocks_overflow_521: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_521: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_522 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_522 -L$_small_initial_partial_block_522: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_522: - - orq %r8,%r8 - je L$_after_reduction_522 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_522: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_9_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_523 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_523 - -L$_16_blocks_overflow_523: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_523: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_524 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_524 -L$_small_initial_partial_block_524: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_524: - - orq %r8,%r8 - je L$_after_reduction_524 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_524: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_10_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_525 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_525 - -L$_16_blocks_overflow_525: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_525: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_526 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_526 -L$_small_initial_partial_block_526: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_526: - - orq %r8,%r8 - je L$_after_reduction_526 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_526: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_11_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_527 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_527 - -L$_16_blocks_overflow_527: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_527: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_528 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_528 -L$_small_initial_partial_block_528: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_528: - - orq %r8,%r8 - je L$_after_reduction_528 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_528: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_12_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_529 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_529 - -L$_16_blocks_overflow_529: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_529: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_530 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_530 -L$_small_initial_partial_block_530: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_530: - - orq %r8,%r8 - je L$_after_reduction_530 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_530: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_13_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_531 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_531 - -L$_16_blocks_overflow_531: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_531: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_532 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_532 -L$_small_initial_partial_block_532: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_532: - - orq %r8,%r8 - je L$_after_reduction_532 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_532: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_14_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_533 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_533 - -L$_16_blocks_overflow_533: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_533: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_534 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_534 -L$_small_initial_partial_block_534: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_534: - - orq %r8,%r8 - je L$_after_reduction_534 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_534: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_15_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_535 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_535 - -L$_16_blocks_overflow_535: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_535: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_536 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_536 -L$_small_initial_partial_block_536: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_536: - - orq %r8,%r8 - je L$_after_reduction_536 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_536: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_16_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_537 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_537 - -L$_16_blocks_overflow_537: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_537: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_538: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_538: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_538: - jmp L$_last_blocks_done_506 -L$_last_num_blocks_is_0_506: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_506: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_497 -L$_encrypt_32_blocks_497: - cmpb $240,%r15b - jae L$_16_blocks_overflow_539 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_539 -L$_16_blocks_overflow_539: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_539: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_540 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_540 -L$_16_blocks_overflow_540: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_540: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_541 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_541 - jb L$_last_num_blocks_is_7_1_541 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_541 - jb L$_last_num_blocks_is_11_9_541 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_541 - ja L$_last_num_blocks_is_16_541 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_541 - jmp L$_last_num_blocks_is_13_541 - -L$_last_num_blocks_is_11_9_541: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_541 - ja L$_last_num_blocks_is_11_541 - jmp L$_last_num_blocks_is_9_541 - -L$_last_num_blocks_is_7_1_541: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_541 - jb L$_last_num_blocks_is_3_1_541 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_541 - je L$_last_num_blocks_is_6_541 - jmp L$_last_num_blocks_is_5_541 - -L$_last_num_blocks_is_3_1_541: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_541 - je L$_last_num_blocks_is_2_541 -L$_last_num_blocks_is_1_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_542 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_542 - -L$_16_blocks_overflow_542: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_542: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_543 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_543 -L$_small_initial_partial_block_543: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_543 -L$_small_initial_compute_done_543: -L$_after_reduction_543: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_2_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_544 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_544 - -L$_16_blocks_overflow_544: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_544: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_545 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_545 -L$_small_initial_partial_block_545: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_545: - - orq %r8,%r8 - je L$_after_reduction_545 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_545: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_3_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_546 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_546 - -L$_16_blocks_overflow_546: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_546: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_547 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_547 -L$_small_initial_partial_block_547: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_547: - - orq %r8,%r8 - je L$_after_reduction_547 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_547: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_4_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_548 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_548 - -L$_16_blocks_overflow_548: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_548: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_549 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_549 -L$_small_initial_partial_block_549: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_549: - - orq %r8,%r8 - je L$_after_reduction_549 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_549: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_5_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_550 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_550 - -L$_16_blocks_overflow_550: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_550: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_551 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_551 -L$_small_initial_partial_block_551: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_551: - - orq %r8,%r8 - je L$_after_reduction_551 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_551: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_6_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_552 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_552 - -L$_16_blocks_overflow_552: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_552: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_553 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_553 -L$_small_initial_partial_block_553: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_553: - - orq %r8,%r8 - je L$_after_reduction_553 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_553: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_7_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_554 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_554 - -L$_16_blocks_overflow_554: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_554: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_555 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_555 -L$_small_initial_partial_block_555: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_555: - - orq %r8,%r8 - je L$_after_reduction_555 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_555: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_8_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_556 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_556 - -L$_16_blocks_overflow_556: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_556: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_557 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_557 -L$_small_initial_partial_block_557: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_557: - - orq %r8,%r8 - je L$_after_reduction_557 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_557: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_9_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_558 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_558 - -L$_16_blocks_overflow_558: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_558: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_559 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_559 -L$_small_initial_partial_block_559: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_559: - - orq %r8,%r8 - je L$_after_reduction_559 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_559: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_10_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_560 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_560 - -L$_16_blocks_overflow_560: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_560: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_561 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_561 -L$_small_initial_partial_block_561: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_561: - - orq %r8,%r8 - je L$_after_reduction_561 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_561: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_11_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_562 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_562 - -L$_16_blocks_overflow_562: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_562: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_563 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_563 -L$_small_initial_partial_block_563: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_563: - - orq %r8,%r8 - je L$_after_reduction_563 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_563: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_12_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_564 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_564 - -L$_16_blocks_overflow_564: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_564: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_565 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_565 -L$_small_initial_partial_block_565: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_565: - - orq %r8,%r8 - je L$_after_reduction_565 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_565: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_13_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_566 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_566 - -L$_16_blocks_overflow_566: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_566: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_567 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_567 -L$_small_initial_partial_block_567: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_567: - - orq %r8,%r8 - je L$_after_reduction_567 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_567: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_14_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_568 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_568 - -L$_16_blocks_overflow_568: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_568: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_569 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_569 -L$_small_initial_partial_block_569: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_569: - - orq %r8,%r8 - je L$_after_reduction_569 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_569: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_15_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_570 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_570 - -L$_16_blocks_overflow_570: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_570: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_571 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_571 -L$_small_initial_partial_block_571: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_571: - - orq %r8,%r8 - je L$_after_reduction_571 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_571: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_16_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_572 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_572 - -L$_16_blocks_overflow_572: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_572: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_573: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_573: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_573: - jmp L$_last_blocks_done_541 -L$_last_num_blocks_is_0_541: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_541: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_497 -L$_encrypt_16_blocks_497: - cmpb $240,%r15b - jae L$_16_blocks_overflow_574 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_574 -L$_16_blocks_overflow_574: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_574: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_575 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_575 - jb L$_last_num_blocks_is_7_1_575 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_575 - jb L$_last_num_blocks_is_11_9_575 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_575 - ja L$_last_num_blocks_is_16_575 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_575 - jmp L$_last_num_blocks_is_13_575 - -L$_last_num_blocks_is_11_9_575: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_575 - ja L$_last_num_blocks_is_11_575 - jmp L$_last_num_blocks_is_9_575 - -L$_last_num_blocks_is_7_1_575: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_575 - jb L$_last_num_blocks_is_3_1_575 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_575 - je L$_last_num_blocks_is_6_575 - jmp L$_last_num_blocks_is_5_575 - -L$_last_num_blocks_is_3_1_575: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_575 - je L$_last_num_blocks_is_2_575 -L$_last_num_blocks_is_1_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_576 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_576 - -L$_16_blocks_overflow_576: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_576: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_577 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_577 -L$_small_initial_partial_block_577: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_577 -L$_small_initial_compute_done_577: -L$_after_reduction_577: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_2_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_578 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_578 - -L$_16_blocks_overflow_578: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_578: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_579 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_579 -L$_small_initial_partial_block_579: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_579: - - orq %r8,%r8 - je L$_after_reduction_579 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_579: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_3_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_580 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_580 - -L$_16_blocks_overflow_580: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_580: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_581 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_581 -L$_small_initial_partial_block_581: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_581: - - orq %r8,%r8 - je L$_after_reduction_581 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_581: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_4_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_582 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_582 - -L$_16_blocks_overflow_582: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_582: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_583 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_583 -L$_small_initial_partial_block_583: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_583: - - orq %r8,%r8 - je L$_after_reduction_583 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_583: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_5_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_584 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_584 - -L$_16_blocks_overflow_584: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_584: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_585 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_585 -L$_small_initial_partial_block_585: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_585: - - orq %r8,%r8 - je L$_after_reduction_585 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_585: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_6_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_586 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_586 - -L$_16_blocks_overflow_586: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_586: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_587 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_587 -L$_small_initial_partial_block_587: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_587: - - orq %r8,%r8 - je L$_after_reduction_587 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_587: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_7_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_588 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_588 - -L$_16_blocks_overflow_588: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_588: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_589 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_589 -L$_small_initial_partial_block_589: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_589: - - orq %r8,%r8 - je L$_after_reduction_589 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_589: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_8_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_590 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_590 - -L$_16_blocks_overflow_590: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_590: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_591 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_591 -L$_small_initial_partial_block_591: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_591: - - orq %r8,%r8 - je L$_after_reduction_591 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_591: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_9_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_592 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_592 - -L$_16_blocks_overflow_592: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_592: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_593 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_593 -L$_small_initial_partial_block_593: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_593: - - orq %r8,%r8 - je L$_after_reduction_593 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_593: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_10_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_594 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_594 - -L$_16_blocks_overflow_594: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_594: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_595 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_595 -L$_small_initial_partial_block_595: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_595: - - orq %r8,%r8 - je L$_after_reduction_595 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_595: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_11_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_596 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_596 - -L$_16_blocks_overflow_596: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_596: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_597 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_597 -L$_small_initial_partial_block_597: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_597: - - orq %r8,%r8 - je L$_after_reduction_597 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_597: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_12_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_598 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_598 - -L$_16_blocks_overflow_598: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_598: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_599 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_599 -L$_small_initial_partial_block_599: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_599: - - orq %r8,%r8 - je L$_after_reduction_599 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_599: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_13_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_600 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_600 - -L$_16_blocks_overflow_600: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_600: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_601 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_601 -L$_small_initial_partial_block_601: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_601: - - orq %r8,%r8 - je L$_after_reduction_601 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_601: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_14_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_602 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_602 - -L$_16_blocks_overflow_602: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_602: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_603 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_603 -L$_small_initial_partial_block_603: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_603: - - orq %r8,%r8 - je L$_after_reduction_603 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_603: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_15_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_604 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_604 - -L$_16_blocks_overflow_604: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_604: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_605 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_605 -L$_small_initial_partial_block_605: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_605: - - orq %r8,%r8 - je L$_after_reduction_605 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_605: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_16_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_606 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_606 - -L$_16_blocks_overflow_606: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_606: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_607: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_607: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_607: - jmp L$_last_blocks_done_575 -L$_last_num_blocks_is_0_575: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_575: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_497 - -L$_message_below_32_blocks_497: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_608 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -L$_skip_hkeys_precomputation_608: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_609 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_609 - jb L$_last_num_blocks_is_7_1_609 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_609 - jb L$_last_num_blocks_is_11_9_609 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_609 - ja L$_last_num_blocks_is_16_609 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_609 - jmp L$_last_num_blocks_is_13_609 - -L$_last_num_blocks_is_11_9_609: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_609 - ja L$_last_num_blocks_is_11_609 - jmp L$_last_num_blocks_is_9_609 - -L$_last_num_blocks_is_7_1_609: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_609 - jb L$_last_num_blocks_is_3_1_609 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_609 - je L$_last_num_blocks_is_6_609 - jmp L$_last_num_blocks_is_5_609 - -L$_last_num_blocks_is_3_1_609: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_609 - je L$_last_num_blocks_is_2_609 -L$_last_num_blocks_is_1_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_610 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_610 - -L$_16_blocks_overflow_610: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_610: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_611 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_611 -L$_small_initial_partial_block_611: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_611 -L$_small_initial_compute_done_611: -L$_after_reduction_611: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_2_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_612 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_612 - -L$_16_blocks_overflow_612: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_612: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_613 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_613 -L$_small_initial_partial_block_613: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_613: - - orq %r8,%r8 - je L$_after_reduction_613 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_613: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_3_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_614 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_614 - -L$_16_blocks_overflow_614: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_614: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_615 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_615 -L$_small_initial_partial_block_615: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_615: - - orq %r8,%r8 - je L$_after_reduction_615 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_615: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_4_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_616 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_616 - -L$_16_blocks_overflow_616: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_616: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_617 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_617 -L$_small_initial_partial_block_617: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_617: - - orq %r8,%r8 - je L$_after_reduction_617 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_617: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_5_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_618 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_618 - -L$_16_blocks_overflow_618: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_618: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_619 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_619 -L$_small_initial_partial_block_619: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_619: - - orq %r8,%r8 - je L$_after_reduction_619 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_619: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_6_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_620 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_620 - -L$_16_blocks_overflow_620: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_620: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_621 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_621 -L$_small_initial_partial_block_621: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_621: - - orq %r8,%r8 - je L$_after_reduction_621 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_621: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_7_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_622 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_622 - -L$_16_blocks_overflow_622: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_622: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_623 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_623 -L$_small_initial_partial_block_623: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_623: - - orq %r8,%r8 - je L$_after_reduction_623 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_623: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_8_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_624 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_624 - -L$_16_blocks_overflow_624: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_624: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_625 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_625 -L$_small_initial_partial_block_625: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_625: - - orq %r8,%r8 - je L$_after_reduction_625 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_625: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_9_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_626 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_626 - -L$_16_blocks_overflow_626: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_626: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_627 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_627 -L$_small_initial_partial_block_627: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_627: - - orq %r8,%r8 - je L$_after_reduction_627 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_627: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_10_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_628 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_628 - -L$_16_blocks_overflow_628: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_628: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_629 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_629 -L$_small_initial_partial_block_629: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_629: - - orq %r8,%r8 - je L$_after_reduction_629 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_629: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_11_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_630 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_630 - -L$_16_blocks_overflow_630: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_630: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_631 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_631 -L$_small_initial_partial_block_631: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_631: - - orq %r8,%r8 - je L$_after_reduction_631 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_631: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_12_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_632 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_632 - -L$_16_blocks_overflow_632: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_632: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_633 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_633 -L$_small_initial_partial_block_633: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_633: - - orq %r8,%r8 - je L$_after_reduction_633 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_633: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_13_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_634 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_634 - -L$_16_blocks_overflow_634: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_634: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_635 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_635 -L$_small_initial_partial_block_635: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_635: - - orq %r8,%r8 - je L$_after_reduction_635 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_635: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_14_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_636 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_636 - -L$_16_blocks_overflow_636: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_636: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_637 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_637 -L$_small_initial_partial_block_637: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_637: - - orq %r8,%r8 - je L$_after_reduction_637 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_637: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_15_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_638 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_638 - -L$_16_blocks_overflow_638: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_638: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_639 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_639 -L$_small_initial_partial_block_639: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_639: - - orq %r8,%r8 - je L$_after_reduction_639 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_639: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_16_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_640 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_640 - -L$_16_blocks_overflow_640: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_640: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_641: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_641: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_641: - jmp L$_last_blocks_done_609 -L$_last_num_blocks_is_0_609: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_609: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_497 - -L$_message_below_equal_16_blocks_497: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je L$_small_initial_num_blocks_is_8_642 - jl L$_small_initial_num_blocks_is_7_1_642 - - - cmpq $12,%r12 - je L$_small_initial_num_blocks_is_12_642 - jl L$_small_initial_num_blocks_is_11_9_642 - - - cmpq $16,%r12 - je L$_small_initial_num_blocks_is_16_642 - cmpq $15,%r12 - je L$_small_initial_num_blocks_is_15_642 - cmpq $14,%r12 - je L$_small_initial_num_blocks_is_14_642 - jmp L$_small_initial_num_blocks_is_13_642 - -L$_small_initial_num_blocks_is_11_9_642: - - cmpq $11,%r12 - je L$_small_initial_num_blocks_is_11_642 - cmpq $10,%r12 - je L$_small_initial_num_blocks_is_10_642 - jmp L$_small_initial_num_blocks_is_9_642 - -L$_small_initial_num_blocks_is_7_1_642: - cmpq $4,%r12 - je L$_small_initial_num_blocks_is_4_642 - jl L$_small_initial_num_blocks_is_3_1_642 - - cmpq $7,%r12 - je L$_small_initial_num_blocks_is_7_642 - cmpq $6,%r12 - je L$_small_initial_num_blocks_is_6_642 - jmp L$_small_initial_num_blocks_is_5_642 - -L$_small_initial_num_blocks_is_3_1_642: - - cmpq $3,%r12 - je L$_small_initial_num_blocks_is_3_642 - cmpq $2,%r12 - je L$_small_initial_num_blocks_is_2_642 - - - - - -L$_small_initial_num_blocks_is_1_642: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm6,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_643 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_643 -L$_small_initial_partial_block_643: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp L$_after_reduction_643 -L$_small_initial_compute_done_643: -L$_after_reduction_643: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_2_642: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm6,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_644 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_644 -L$_small_initial_partial_block_644: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_644: - - orq %r8,%r8 - je L$_after_reduction_644 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_644: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_3_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_645 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_645 -L$_small_initial_partial_block_645: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_645: - - orq %r8,%r8 - je L$_after_reduction_645 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_645: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_4_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_646 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_646 -L$_small_initial_partial_block_646: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_646: - - orq %r8,%r8 - je L$_after_reduction_646 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_646: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_5_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %xmm29,%xmm7,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_647 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_647 -L$_small_initial_partial_block_647: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_647: - - orq %r8,%r8 - je L$_after_reduction_647 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_647: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_6_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %ymm29,%ymm7,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_648 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_648 -L$_small_initial_partial_block_648: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_648: - - orq %r8,%r8 - je L$_after_reduction_648 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_648: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_7_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_649 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_649 -L$_small_initial_partial_block_649: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_649: - - orq %r8,%r8 - je L$_after_reduction_649 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_649: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_8_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_650 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_650 -L$_small_initial_partial_block_650: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_650: - - orq %r8,%r8 - je L$_after_reduction_650 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_650: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_9_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %xmm29,%xmm10,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_651 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_651 -L$_small_initial_partial_block_651: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_651: - - orq %r8,%r8 - je L$_after_reduction_651 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_651: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_10_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %ymm29,%ymm10,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_652 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_652 -L$_small_initial_partial_block_652: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_652: - - orq %r8,%r8 - je L$_after_reduction_652 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_652: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_11_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_653 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_653 -L$_small_initial_partial_block_653: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_653: - - orq %r8,%r8 - je L$_after_reduction_653 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_653: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_12_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_654 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_654 -L$_small_initial_partial_block_654: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_654: - - orq %r8,%r8 - je L$_after_reduction_654 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_654: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_13_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %xmm29,%xmm11,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_655 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_655 -L$_small_initial_partial_block_655: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_655: - - orq %r8,%r8 - je L$_after_reduction_655 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_655: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_14_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %ymm29,%ymm11,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_656 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_656 -L$_small_initial_partial_block_656: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_656: - - orq %r8,%r8 - je L$_after_reduction_656 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_656: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_15_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_657 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_657 -L$_small_initial_partial_block_657: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_657: - - orq %r8,%r8 - je L$_after_reduction_657 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_657: - jmp L$_small_initial_blocks_encrypted_642 -L$_small_initial_num_blocks_is_16_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_658: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_658: - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_658: -L$_small_initial_blocks_encrypted_642: -L$_ghash_done_497: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -L$_enc_dec_done_497: - jmp L$exit_gcm_decrypt -.p2align 5 -L$aes_gcm_decrypt_192_avx512: - orq %r8,%r8 - je L$_enc_dec_done_659 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je L$_partial_block_done_660 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - - vmovdqa64 %xmm0,%xmm6 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge L$_no_extra_mask_660 - subq %r13,%r12 -L$_no_extra_mask_660: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpand %xmm0,%xmm6,%xmm6 - vpshufb SHUF_MASK(%rip),%xmm6,%xmm6 - vpshufb %xmm5,%xmm6,%xmm6 - vpxorq %xmm6,%xmm14,%xmm14 - cmpq $0,%r13 - jl L$_partial_incomplete_660 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp L$_enc_dec_done_660 - -L$_partial_incomplete_660: - addq %r8,(%rdx) - movq %r8,%r11 - -L$_enc_dec_done_660: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -L$_partial_block_done_660: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je L$_enc_dec_done_659 - cmpq $256,%r8 - jbe L$_message_below_equal_16_blocks_659 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_661 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_661 -L$_next_16_overflow_661: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_661: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_662 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -L$_skip_hkeys_precomputation_662: - cmpq $512,%r8 - jb L$_message_below_32_blocks_659 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_663 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_663 -L$_next_16_overflow_663: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_663: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_664 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -L$_skip_hkeys_precomputation_664: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb L$_no_more_big_nblocks_659 -L$_encrypt_big_nblocks_659: - cmpb $240,%r15b - jae L$_16_blocks_overflow_665 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_665 -L$_16_blocks_overflow_665: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_665: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_666 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_666 -L$_16_blocks_overflow_666: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_666: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_667 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_667 -L$_16_blocks_overflow_667: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_667: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae L$_encrypt_big_nblocks_659 - -L$_no_more_big_nblocks_659: - - cmpq $512,%r8 - jae L$_encrypt_32_blocks_659 - - cmpq $256,%r8 - jae L$_encrypt_16_blocks_659 -L$_encrypt_0_blocks_ghash_32_659: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_668 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_668 - jb L$_last_num_blocks_is_7_1_668 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_668 - jb L$_last_num_blocks_is_11_9_668 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_668 - ja L$_last_num_blocks_is_16_668 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_668 - jmp L$_last_num_blocks_is_13_668 - -L$_last_num_blocks_is_11_9_668: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_668 - ja L$_last_num_blocks_is_11_668 - jmp L$_last_num_blocks_is_9_668 - -L$_last_num_blocks_is_7_1_668: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_668 - jb L$_last_num_blocks_is_3_1_668 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_668 - je L$_last_num_blocks_is_6_668 - jmp L$_last_num_blocks_is_5_668 - -L$_last_num_blocks_is_3_1_668: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_668 - je L$_last_num_blocks_is_2_668 -L$_last_num_blocks_is_1_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_669 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_669 - -L$_16_blocks_overflow_669: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_669: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_670 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_670 -L$_small_initial_partial_block_670: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_670 -L$_small_initial_compute_done_670: -L$_after_reduction_670: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_2_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_671 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_671 - -L$_16_blocks_overflow_671: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_671: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_672 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_672 -L$_small_initial_partial_block_672: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_672: - - orq %r8,%r8 - je L$_after_reduction_672 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_672: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_3_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_673 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_673 - -L$_16_blocks_overflow_673: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_673: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_674 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_674 -L$_small_initial_partial_block_674: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_674: - - orq %r8,%r8 - je L$_after_reduction_674 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_674: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_4_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_675 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_675 - -L$_16_blocks_overflow_675: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_675: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_676 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_676 -L$_small_initial_partial_block_676: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_676: - - orq %r8,%r8 - je L$_after_reduction_676 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_676: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_5_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_677 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_677 - -L$_16_blocks_overflow_677: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_677: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_678 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_678 -L$_small_initial_partial_block_678: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_678: - - orq %r8,%r8 - je L$_after_reduction_678 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_678: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_6_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_679 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_679 - -L$_16_blocks_overflow_679: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_679: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_680 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_680 -L$_small_initial_partial_block_680: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_680: - - orq %r8,%r8 - je L$_after_reduction_680 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_680: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_7_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_681 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_681 - -L$_16_blocks_overflow_681: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_681: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_682 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_682 -L$_small_initial_partial_block_682: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_682: - - orq %r8,%r8 - je L$_after_reduction_682 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_682: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_8_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_683 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_683 - -L$_16_blocks_overflow_683: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_683: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_684 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_684 -L$_small_initial_partial_block_684: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_684: - - orq %r8,%r8 - je L$_after_reduction_684 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_684: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_9_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_685 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_685 - -L$_16_blocks_overflow_685: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_685: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_686 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_686 -L$_small_initial_partial_block_686: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_686: - - orq %r8,%r8 - je L$_after_reduction_686 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_686: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_10_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_687 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_687 - -L$_16_blocks_overflow_687: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_687: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_688 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_688 -L$_small_initial_partial_block_688: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_688: - - orq %r8,%r8 - je L$_after_reduction_688 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_688: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_11_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_689 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_689 - -L$_16_blocks_overflow_689: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_689: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_690 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_690 -L$_small_initial_partial_block_690: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_690: - - orq %r8,%r8 - je L$_after_reduction_690 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_690: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_12_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_691 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_691 - -L$_16_blocks_overflow_691: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_691: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_692 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_692 -L$_small_initial_partial_block_692: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_692: - - orq %r8,%r8 - je L$_after_reduction_692 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_692: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_13_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_693 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_693 - -L$_16_blocks_overflow_693: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_693: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_694 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_694 -L$_small_initial_partial_block_694: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_694: - - orq %r8,%r8 - je L$_after_reduction_694 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_694: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_14_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_695 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_695 - -L$_16_blocks_overflow_695: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_695: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_696 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_696 -L$_small_initial_partial_block_696: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_696: - - orq %r8,%r8 - je L$_after_reduction_696 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_696: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_15_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_697 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_697 - -L$_16_blocks_overflow_697: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_697: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_698 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_698 -L$_small_initial_partial_block_698: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_698: - - orq %r8,%r8 - je L$_after_reduction_698 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_698: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_16_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_699 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_699 - -L$_16_blocks_overflow_699: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_699: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_700: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_700: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_700: - jmp L$_last_blocks_done_668 -L$_last_num_blocks_is_0_668: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_668: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_659 -L$_encrypt_32_blocks_659: - cmpb $240,%r15b - jae L$_16_blocks_overflow_701 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_701 -L$_16_blocks_overflow_701: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_701: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_702 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_702 -L$_16_blocks_overflow_702: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_702: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_703 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_703 - jb L$_last_num_blocks_is_7_1_703 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_703 - jb L$_last_num_blocks_is_11_9_703 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_703 - ja L$_last_num_blocks_is_16_703 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_703 - jmp L$_last_num_blocks_is_13_703 - -L$_last_num_blocks_is_11_9_703: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_703 - ja L$_last_num_blocks_is_11_703 - jmp L$_last_num_blocks_is_9_703 - -L$_last_num_blocks_is_7_1_703: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_703 - jb L$_last_num_blocks_is_3_1_703 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_703 - je L$_last_num_blocks_is_6_703 - jmp L$_last_num_blocks_is_5_703 - -L$_last_num_blocks_is_3_1_703: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_703 - je L$_last_num_blocks_is_2_703 -L$_last_num_blocks_is_1_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_704 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_704 - -L$_16_blocks_overflow_704: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_704: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_705 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_705 -L$_small_initial_partial_block_705: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_705 -L$_small_initial_compute_done_705: -L$_after_reduction_705: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_2_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_706 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_706 - -L$_16_blocks_overflow_706: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_706: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_707 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_707 -L$_small_initial_partial_block_707: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_707: - - orq %r8,%r8 - je L$_after_reduction_707 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_707: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_3_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_708 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_708 - -L$_16_blocks_overflow_708: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_708: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_709 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_709 -L$_small_initial_partial_block_709: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_709: - - orq %r8,%r8 - je L$_after_reduction_709 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_709: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_4_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_710 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_710 - -L$_16_blocks_overflow_710: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_710: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_711 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_711 -L$_small_initial_partial_block_711: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_711: - - orq %r8,%r8 - je L$_after_reduction_711 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_711: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_5_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_712 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_712 - -L$_16_blocks_overflow_712: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_712: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_713 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_713 -L$_small_initial_partial_block_713: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_713: - - orq %r8,%r8 - je L$_after_reduction_713 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_713: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_6_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_714 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_714 - -L$_16_blocks_overflow_714: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_714: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_715 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_715 -L$_small_initial_partial_block_715: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_715: - - orq %r8,%r8 - je L$_after_reduction_715 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_715: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_7_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_716 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_716 - -L$_16_blocks_overflow_716: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_716: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_717 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_717 -L$_small_initial_partial_block_717: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_717: - - orq %r8,%r8 - je L$_after_reduction_717 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_717: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_8_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_718 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_718 - -L$_16_blocks_overflow_718: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_718: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_719 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_719 -L$_small_initial_partial_block_719: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_719: - - orq %r8,%r8 - je L$_after_reduction_719 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_719: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_9_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_720 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_720 - -L$_16_blocks_overflow_720: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_720: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_721 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_721 -L$_small_initial_partial_block_721: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_721: - - orq %r8,%r8 - je L$_after_reduction_721 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_721: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_10_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_722 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_722 - -L$_16_blocks_overflow_722: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_722: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_723 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_723 -L$_small_initial_partial_block_723: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_723: - - orq %r8,%r8 - je L$_after_reduction_723 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_723: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_11_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_724 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_724 - -L$_16_blocks_overflow_724: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_724: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_725 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_725 -L$_small_initial_partial_block_725: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_725: - - orq %r8,%r8 - je L$_after_reduction_725 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_725: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_12_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_726 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_726 - -L$_16_blocks_overflow_726: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_726: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_727 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_727 -L$_small_initial_partial_block_727: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_727: - - orq %r8,%r8 - je L$_after_reduction_727 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_727: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_13_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_728 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_728 - -L$_16_blocks_overflow_728: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_728: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_729 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_729 -L$_small_initial_partial_block_729: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_729: - - orq %r8,%r8 - je L$_after_reduction_729 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_729: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_14_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_730 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_730 - -L$_16_blocks_overflow_730: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_730: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_731 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_731 -L$_small_initial_partial_block_731: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_731: - - orq %r8,%r8 - je L$_after_reduction_731 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_731: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_15_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_732 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_732 - -L$_16_blocks_overflow_732: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_732: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_733 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_733 -L$_small_initial_partial_block_733: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_733: - - orq %r8,%r8 - je L$_after_reduction_733 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_733: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_16_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_734 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_734 - -L$_16_blocks_overflow_734: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_734: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_735: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_735: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_735: - jmp L$_last_blocks_done_703 -L$_last_num_blocks_is_0_703: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_703: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_659 -L$_encrypt_16_blocks_659: - cmpb $240,%r15b - jae L$_16_blocks_overflow_736 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_736 -L$_16_blocks_overflow_736: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_736: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_737 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_737 - jb L$_last_num_blocks_is_7_1_737 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_737 - jb L$_last_num_blocks_is_11_9_737 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_737 - ja L$_last_num_blocks_is_16_737 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_737 - jmp L$_last_num_blocks_is_13_737 - -L$_last_num_blocks_is_11_9_737: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_737 - ja L$_last_num_blocks_is_11_737 - jmp L$_last_num_blocks_is_9_737 - -L$_last_num_blocks_is_7_1_737: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_737 - jb L$_last_num_blocks_is_3_1_737 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_737 - je L$_last_num_blocks_is_6_737 - jmp L$_last_num_blocks_is_5_737 - -L$_last_num_blocks_is_3_1_737: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_737 - je L$_last_num_blocks_is_2_737 -L$_last_num_blocks_is_1_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_738 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_738 - -L$_16_blocks_overflow_738: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_738: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_739 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_739 -L$_small_initial_partial_block_739: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_739 -L$_small_initial_compute_done_739: -L$_after_reduction_739: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_2_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_740 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_740 - -L$_16_blocks_overflow_740: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_740: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_741 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_741 -L$_small_initial_partial_block_741: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_741: - - orq %r8,%r8 - je L$_after_reduction_741 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_741: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_3_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_742 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_742 - -L$_16_blocks_overflow_742: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_742: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_743 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_743 -L$_small_initial_partial_block_743: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_743: - - orq %r8,%r8 - je L$_after_reduction_743 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_743: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_4_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_744 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_744 - -L$_16_blocks_overflow_744: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_744: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_745 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_745 -L$_small_initial_partial_block_745: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_745: - - orq %r8,%r8 - je L$_after_reduction_745 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_745: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_5_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_746 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_746 - -L$_16_blocks_overflow_746: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_746: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_747 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_747 -L$_small_initial_partial_block_747: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_747: - - orq %r8,%r8 - je L$_after_reduction_747 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_747: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_6_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_748 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_748 - -L$_16_blocks_overflow_748: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_748: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_749 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_749 -L$_small_initial_partial_block_749: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_749: - - orq %r8,%r8 - je L$_after_reduction_749 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_749: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_7_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_750 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_750 - -L$_16_blocks_overflow_750: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_750: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_751 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_751 -L$_small_initial_partial_block_751: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_751: - - orq %r8,%r8 - je L$_after_reduction_751 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_751: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_8_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_752 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_752 - -L$_16_blocks_overflow_752: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_752: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_753 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_753 -L$_small_initial_partial_block_753: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_753: - - orq %r8,%r8 - je L$_after_reduction_753 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_753: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_9_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_754 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_754 - -L$_16_blocks_overflow_754: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_754: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_755 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_755 -L$_small_initial_partial_block_755: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_755: - - orq %r8,%r8 - je L$_after_reduction_755 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_755: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_10_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_756 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_756 - -L$_16_blocks_overflow_756: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_756: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_757 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_757 -L$_small_initial_partial_block_757: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_757: - - orq %r8,%r8 - je L$_after_reduction_757 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_757: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_11_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_758 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_758 - -L$_16_blocks_overflow_758: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_758: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_759 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_759 -L$_small_initial_partial_block_759: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_759: - - orq %r8,%r8 - je L$_after_reduction_759 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_759: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_12_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_760 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_760 - -L$_16_blocks_overflow_760: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_760: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_761 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_761 -L$_small_initial_partial_block_761: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_761: - - orq %r8,%r8 - je L$_after_reduction_761 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_761: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_13_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_762 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_762 - -L$_16_blocks_overflow_762: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_762: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_763 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_763 -L$_small_initial_partial_block_763: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_763: - - orq %r8,%r8 - je L$_after_reduction_763 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_763: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_14_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_764 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_764 - -L$_16_blocks_overflow_764: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_764: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_765 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_765 -L$_small_initial_partial_block_765: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_765: - - orq %r8,%r8 - je L$_after_reduction_765 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_765: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_15_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_766 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_766 - -L$_16_blocks_overflow_766: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_766: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_767 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_767 -L$_small_initial_partial_block_767: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_767: - - orq %r8,%r8 - je L$_after_reduction_767 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_767: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_16_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_768 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_768 - -L$_16_blocks_overflow_768: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_768: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_769: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_769: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_769: - jmp L$_last_blocks_done_737 -L$_last_num_blocks_is_0_737: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_737: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_659 - -L$_message_below_32_blocks_659: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_770 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -L$_skip_hkeys_precomputation_770: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_771 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_771 - jb L$_last_num_blocks_is_7_1_771 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_771 - jb L$_last_num_blocks_is_11_9_771 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_771 - ja L$_last_num_blocks_is_16_771 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_771 - jmp L$_last_num_blocks_is_13_771 - -L$_last_num_blocks_is_11_9_771: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_771 - ja L$_last_num_blocks_is_11_771 - jmp L$_last_num_blocks_is_9_771 - -L$_last_num_blocks_is_7_1_771: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_771 - jb L$_last_num_blocks_is_3_1_771 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_771 - je L$_last_num_blocks_is_6_771 - jmp L$_last_num_blocks_is_5_771 - -L$_last_num_blocks_is_3_1_771: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_771 - je L$_last_num_blocks_is_2_771 -L$_last_num_blocks_is_1_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_772 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_772 - -L$_16_blocks_overflow_772: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_772: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_773 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_773 -L$_small_initial_partial_block_773: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_773 -L$_small_initial_compute_done_773: -L$_after_reduction_773: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_2_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_774 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_774 - -L$_16_blocks_overflow_774: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_774: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_775 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_775 -L$_small_initial_partial_block_775: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_775: - - orq %r8,%r8 - je L$_after_reduction_775 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_775: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_3_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_776 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_776 - -L$_16_blocks_overflow_776: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_776: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_777 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_777 -L$_small_initial_partial_block_777: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_777: - - orq %r8,%r8 - je L$_after_reduction_777 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_777: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_4_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_778 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_778 - -L$_16_blocks_overflow_778: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_778: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_779 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_779 -L$_small_initial_partial_block_779: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_779: - - orq %r8,%r8 - je L$_after_reduction_779 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_779: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_5_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_780 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_780 - -L$_16_blocks_overflow_780: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_780: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_781 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_781 -L$_small_initial_partial_block_781: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_781: - - orq %r8,%r8 - je L$_after_reduction_781 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_781: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_6_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_782 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_782 - -L$_16_blocks_overflow_782: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_782: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_783 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_783 -L$_small_initial_partial_block_783: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_783: - - orq %r8,%r8 - je L$_after_reduction_783 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_783: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_7_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_784 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_784 - -L$_16_blocks_overflow_784: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_784: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_785 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_785 -L$_small_initial_partial_block_785: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_785: - - orq %r8,%r8 - je L$_after_reduction_785 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_785: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_8_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_786 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_786 - -L$_16_blocks_overflow_786: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_786: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_787 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_787 -L$_small_initial_partial_block_787: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_787: - - orq %r8,%r8 - je L$_after_reduction_787 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_787: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_9_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_788 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_788 - -L$_16_blocks_overflow_788: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_788: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_789 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_789 -L$_small_initial_partial_block_789: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_789: - - orq %r8,%r8 - je L$_after_reduction_789 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_789: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_10_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_790 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_790 - -L$_16_blocks_overflow_790: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_790: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_791 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_791 -L$_small_initial_partial_block_791: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_791: - - orq %r8,%r8 - je L$_after_reduction_791 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_791: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_11_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_792 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_792 - -L$_16_blocks_overflow_792: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_792: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_793 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_793 -L$_small_initial_partial_block_793: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_793: - - orq %r8,%r8 - je L$_after_reduction_793 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_793: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_12_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_794 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_794 - -L$_16_blocks_overflow_794: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_794: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_795 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_795 -L$_small_initial_partial_block_795: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_795: - - orq %r8,%r8 - je L$_after_reduction_795 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_795: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_13_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_796 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_796 - -L$_16_blocks_overflow_796: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_796: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_797 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_797 -L$_small_initial_partial_block_797: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_797: - - orq %r8,%r8 - je L$_after_reduction_797 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_797: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_14_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_798 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_798 - -L$_16_blocks_overflow_798: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_798: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_799 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_799 -L$_small_initial_partial_block_799: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_799: - - orq %r8,%r8 - je L$_after_reduction_799 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_799: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_15_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_800 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_800 - -L$_16_blocks_overflow_800: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_800: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_801 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_801 -L$_small_initial_partial_block_801: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_801: - - orq %r8,%r8 - je L$_after_reduction_801 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_801: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_16_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_802 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_802 - -L$_16_blocks_overflow_802: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_802: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_803: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_803: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_803: - jmp L$_last_blocks_done_771 -L$_last_num_blocks_is_0_771: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_771: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_659 - -L$_message_below_equal_16_blocks_659: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je L$_small_initial_num_blocks_is_8_804 - jl L$_small_initial_num_blocks_is_7_1_804 - - - cmpq $12,%r12 - je L$_small_initial_num_blocks_is_12_804 - jl L$_small_initial_num_blocks_is_11_9_804 - - - cmpq $16,%r12 - je L$_small_initial_num_blocks_is_16_804 - cmpq $15,%r12 - je L$_small_initial_num_blocks_is_15_804 - cmpq $14,%r12 - je L$_small_initial_num_blocks_is_14_804 - jmp L$_small_initial_num_blocks_is_13_804 - -L$_small_initial_num_blocks_is_11_9_804: - - cmpq $11,%r12 - je L$_small_initial_num_blocks_is_11_804 - cmpq $10,%r12 - je L$_small_initial_num_blocks_is_10_804 - jmp L$_small_initial_num_blocks_is_9_804 - -L$_small_initial_num_blocks_is_7_1_804: - cmpq $4,%r12 - je L$_small_initial_num_blocks_is_4_804 - jl L$_small_initial_num_blocks_is_3_1_804 - - cmpq $7,%r12 - je L$_small_initial_num_blocks_is_7_804 - cmpq $6,%r12 - je L$_small_initial_num_blocks_is_6_804 - jmp L$_small_initial_num_blocks_is_5_804 - -L$_small_initial_num_blocks_is_3_1_804: - - cmpq $3,%r12 - je L$_small_initial_num_blocks_is_3_804 - cmpq $2,%r12 - je L$_small_initial_num_blocks_is_2_804 - - - - - -L$_small_initial_num_blocks_is_1_804: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm6,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_805 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_805 -L$_small_initial_partial_block_805: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp L$_after_reduction_805 -L$_small_initial_compute_done_805: -L$_after_reduction_805: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_2_804: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm6,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_806 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_806 -L$_small_initial_partial_block_806: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_806: - - orq %r8,%r8 - je L$_after_reduction_806 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_806: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_3_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_807 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_807 -L$_small_initial_partial_block_807: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_807: - - orq %r8,%r8 - je L$_after_reduction_807 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_807: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_4_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_808 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_808 -L$_small_initial_partial_block_808: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_808: - - orq %r8,%r8 - je L$_after_reduction_808 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_808: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_5_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %xmm29,%xmm7,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_809 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_809 -L$_small_initial_partial_block_809: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_809: - - orq %r8,%r8 - je L$_after_reduction_809 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_809: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_6_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %ymm29,%ymm7,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_810 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_810 -L$_small_initial_partial_block_810: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_810: - - orq %r8,%r8 - je L$_after_reduction_810 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_810: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_7_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_811 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_811 -L$_small_initial_partial_block_811: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_811: - - orq %r8,%r8 - je L$_after_reduction_811 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_811: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_8_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_812 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_812 -L$_small_initial_partial_block_812: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_812: - - orq %r8,%r8 - je L$_after_reduction_812 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_812: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_9_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %xmm29,%xmm10,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_813 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_813 -L$_small_initial_partial_block_813: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_813: - - orq %r8,%r8 - je L$_after_reduction_813 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_813: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_10_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %ymm29,%ymm10,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_814 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_814 -L$_small_initial_partial_block_814: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_814: - - orq %r8,%r8 - je L$_after_reduction_814 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_814: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_11_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_815 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_815 -L$_small_initial_partial_block_815: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_815: - - orq %r8,%r8 - je L$_after_reduction_815 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_815: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_12_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_816 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_816 -L$_small_initial_partial_block_816: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_816: - - orq %r8,%r8 - je L$_after_reduction_816 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_816: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_13_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %xmm29,%xmm11,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_817 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_817 -L$_small_initial_partial_block_817: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_817: - - orq %r8,%r8 - je L$_after_reduction_817 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_817: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_14_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %ymm29,%ymm11,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_818 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_818 -L$_small_initial_partial_block_818: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_818: - - orq %r8,%r8 - je L$_after_reduction_818 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_818: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_15_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_819 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_819 -L$_small_initial_partial_block_819: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_819: - - orq %r8,%r8 - je L$_after_reduction_819 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_819: - jmp L$_small_initial_blocks_encrypted_804 -L$_small_initial_num_blocks_is_16_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_820: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_820: - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_820: -L$_small_initial_blocks_encrypted_804: -L$_ghash_done_659: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -L$_enc_dec_done_659: - jmp L$exit_gcm_decrypt -.p2align 5 -L$aes_gcm_decrypt_256_avx512: - orq %r8,%r8 - je L$_enc_dec_done_821 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je L$_partial_block_done_822 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - - vmovdqa64 %xmm0,%xmm6 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge L$_no_extra_mask_822 - subq %r13,%r12 -L$_no_extra_mask_822: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpand %xmm0,%xmm6,%xmm6 - vpshufb SHUF_MASK(%rip),%xmm6,%xmm6 - vpshufb %xmm5,%xmm6,%xmm6 - vpxorq %xmm6,%xmm14,%xmm14 - cmpq $0,%r13 - jl L$_partial_incomplete_822 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp L$_enc_dec_done_822 - -L$_partial_incomplete_822: - addq %r8,(%rdx) - movq %r8,%r11 - -L$_enc_dec_done_822: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -L$_partial_block_done_822: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je L$_enc_dec_done_821 - cmpq $256,%r8 - jbe L$_message_below_equal_16_blocks_821 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_823 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_823 -L$_next_16_overflow_823: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_823: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 208(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 224(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_824 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -L$_skip_hkeys_precomputation_824: - cmpq $512,%r8 - jb L$_message_below_32_blocks_821 - - - - cmpb $240,%r15b - jae L$_next_16_overflow_825 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp L$_next_16_ok_825 -L$_next_16_overflow_825: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -L$_next_16_ok_825: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 208(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 224(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_826 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -L$_skip_hkeys_precomputation_826: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb L$_no_more_big_nblocks_821 -L$_encrypt_big_nblocks_821: - cmpb $240,%r15b - jae L$_16_blocks_overflow_827 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_827 -L$_16_blocks_overflow_827: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_827: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_828 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_828 -L$_16_blocks_overflow_828: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_828: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_829 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_829 -L$_16_blocks_overflow_829: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_829: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae L$_encrypt_big_nblocks_821 - -L$_no_more_big_nblocks_821: - - cmpq $512,%r8 - jae L$_encrypt_32_blocks_821 - - cmpq $256,%r8 - jae L$_encrypt_16_blocks_821 -L$_encrypt_0_blocks_ghash_32_821: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_830 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_830 - jb L$_last_num_blocks_is_7_1_830 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_830 - jb L$_last_num_blocks_is_11_9_830 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_830 - ja L$_last_num_blocks_is_16_830 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_830 - jmp L$_last_num_blocks_is_13_830 - -L$_last_num_blocks_is_11_9_830: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_830 - ja L$_last_num_blocks_is_11_830 - jmp L$_last_num_blocks_is_9_830 - -L$_last_num_blocks_is_7_1_830: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_830 - jb L$_last_num_blocks_is_3_1_830 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_830 - je L$_last_num_blocks_is_6_830 - jmp L$_last_num_blocks_is_5_830 - -L$_last_num_blocks_is_3_1_830: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_830 - je L$_last_num_blocks_is_2_830 -L$_last_num_blocks_is_1_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_831 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_831 - -L$_16_blocks_overflow_831: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_831: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_832 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_832 -L$_small_initial_partial_block_832: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_832 -L$_small_initial_compute_done_832: -L$_after_reduction_832: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_2_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_833 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_833 - -L$_16_blocks_overflow_833: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_833: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_834 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_834 -L$_small_initial_partial_block_834: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_834: - - orq %r8,%r8 - je L$_after_reduction_834 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_834: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_3_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_835 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_835 - -L$_16_blocks_overflow_835: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_835: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_836 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_836 -L$_small_initial_partial_block_836: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_836: - - orq %r8,%r8 - je L$_after_reduction_836 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_836: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_4_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_837 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_837 - -L$_16_blocks_overflow_837: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_837: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_838 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_838 -L$_small_initial_partial_block_838: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_838: - - orq %r8,%r8 - je L$_after_reduction_838 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_838: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_5_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_839 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_839 - -L$_16_blocks_overflow_839: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_839: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_840 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_840 -L$_small_initial_partial_block_840: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_840: - - orq %r8,%r8 - je L$_after_reduction_840 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_840: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_6_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_841 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_841 - -L$_16_blocks_overflow_841: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_841: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_842 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_842 -L$_small_initial_partial_block_842: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_842: - - orq %r8,%r8 - je L$_after_reduction_842 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_842: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_7_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_843 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_843 - -L$_16_blocks_overflow_843: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_843: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_844 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_844 -L$_small_initial_partial_block_844: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_844: - - orq %r8,%r8 - je L$_after_reduction_844 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_844: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_8_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_845 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_845 - -L$_16_blocks_overflow_845: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_845: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_846 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_846 -L$_small_initial_partial_block_846: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_846: - - orq %r8,%r8 - je L$_after_reduction_846 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_846: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_9_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_847 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_847 - -L$_16_blocks_overflow_847: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_847: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_848 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_848 -L$_small_initial_partial_block_848: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_848: - - orq %r8,%r8 - je L$_after_reduction_848 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_848: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_10_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_849 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_849 - -L$_16_blocks_overflow_849: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_849: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_850 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_850 -L$_small_initial_partial_block_850: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_850: - - orq %r8,%r8 - je L$_after_reduction_850 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_850: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_11_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_851 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_851 - -L$_16_blocks_overflow_851: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_851: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_852 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_852 -L$_small_initial_partial_block_852: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_852: - - orq %r8,%r8 - je L$_after_reduction_852 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_852: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_12_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_853 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_853 - -L$_16_blocks_overflow_853: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_853: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_854 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_854 -L$_small_initial_partial_block_854: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_854: - - orq %r8,%r8 - je L$_after_reduction_854 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_854: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_13_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_855 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_855 - -L$_16_blocks_overflow_855: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_855: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_856 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_856 -L$_small_initial_partial_block_856: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_856: - - orq %r8,%r8 - je L$_after_reduction_856 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_856: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_14_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_857 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_857 - -L$_16_blocks_overflow_857: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_857: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_858 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_858 -L$_small_initial_partial_block_858: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_858: - - orq %r8,%r8 - je L$_after_reduction_858 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_858: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_15_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_859 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_859 - -L$_16_blocks_overflow_859: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_859: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_860 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_860 -L$_small_initial_partial_block_860: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_860: - - orq %r8,%r8 - je L$_after_reduction_860 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_860: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_16_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_861 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_861 - -L$_16_blocks_overflow_861: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_861: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_862: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_862: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_862: - jmp L$_last_blocks_done_830 -L$_last_num_blocks_is_0_830: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_830: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_821 -L$_encrypt_32_blocks_821: - cmpb $240,%r15b - jae L$_16_blocks_overflow_863 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_863 -L$_16_blocks_overflow_863: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_863: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae L$_16_blocks_overflow_864 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_864 -L$_16_blocks_overflow_864: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_864: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_865 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_865 - jb L$_last_num_blocks_is_7_1_865 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_865 - jb L$_last_num_blocks_is_11_9_865 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_865 - ja L$_last_num_blocks_is_16_865 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_865 - jmp L$_last_num_blocks_is_13_865 - -L$_last_num_blocks_is_11_9_865: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_865 - ja L$_last_num_blocks_is_11_865 - jmp L$_last_num_blocks_is_9_865 - -L$_last_num_blocks_is_7_1_865: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_865 - jb L$_last_num_blocks_is_3_1_865 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_865 - je L$_last_num_blocks_is_6_865 - jmp L$_last_num_blocks_is_5_865 - -L$_last_num_blocks_is_3_1_865: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_865 - je L$_last_num_blocks_is_2_865 -L$_last_num_blocks_is_1_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_866 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_866 - -L$_16_blocks_overflow_866: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_866: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_867 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_867 -L$_small_initial_partial_block_867: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_867 -L$_small_initial_compute_done_867: -L$_after_reduction_867: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_2_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_868 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_868 - -L$_16_blocks_overflow_868: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_868: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_869 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_869 -L$_small_initial_partial_block_869: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_869: - - orq %r8,%r8 - je L$_after_reduction_869 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_869: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_3_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_870 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_870 - -L$_16_blocks_overflow_870: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_870: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_871 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_871 -L$_small_initial_partial_block_871: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_871: - - orq %r8,%r8 - je L$_after_reduction_871 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_871: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_4_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_872 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_872 - -L$_16_blocks_overflow_872: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_872: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_873 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_873 -L$_small_initial_partial_block_873: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_873: - - orq %r8,%r8 - je L$_after_reduction_873 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_873: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_5_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_874 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_874 - -L$_16_blocks_overflow_874: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_874: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_875 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_875 -L$_small_initial_partial_block_875: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_875: - - orq %r8,%r8 - je L$_after_reduction_875 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_875: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_6_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_876 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_876 - -L$_16_blocks_overflow_876: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_876: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_877 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_877 -L$_small_initial_partial_block_877: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_877: - - orq %r8,%r8 - je L$_after_reduction_877 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_877: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_7_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_878 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_878 - -L$_16_blocks_overflow_878: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_878: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_879 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_879 -L$_small_initial_partial_block_879: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_879: - - orq %r8,%r8 - je L$_after_reduction_879 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_879: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_8_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_880 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_880 - -L$_16_blocks_overflow_880: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_880: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_881 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_881 -L$_small_initial_partial_block_881: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_881: - - orq %r8,%r8 - je L$_after_reduction_881 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_881: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_9_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_882 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_882 - -L$_16_blocks_overflow_882: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_882: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_883 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_883 -L$_small_initial_partial_block_883: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_883: - - orq %r8,%r8 - je L$_after_reduction_883 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_883: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_10_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_884 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_884 - -L$_16_blocks_overflow_884: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_884: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_885 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_885 -L$_small_initial_partial_block_885: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_885: - - orq %r8,%r8 - je L$_after_reduction_885 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_885: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_11_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_886 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_886 - -L$_16_blocks_overflow_886: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_886: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_887 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_887 -L$_small_initial_partial_block_887: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_887: - - orq %r8,%r8 - je L$_after_reduction_887 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_887: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_12_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_888 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_888 - -L$_16_blocks_overflow_888: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_888: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_889 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_889 -L$_small_initial_partial_block_889: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_889: - - orq %r8,%r8 - je L$_after_reduction_889 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_889: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_13_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_890 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_890 - -L$_16_blocks_overflow_890: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_890: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_891 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_891 -L$_small_initial_partial_block_891: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_891: - - orq %r8,%r8 - je L$_after_reduction_891 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_891: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_14_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_892 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_892 - -L$_16_blocks_overflow_892: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_892: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_893 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_893 -L$_small_initial_partial_block_893: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_893: - - orq %r8,%r8 - je L$_after_reduction_893 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_893: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_15_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_894 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_894 - -L$_16_blocks_overflow_894: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_894: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_895 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_895 -L$_small_initial_partial_block_895: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_895: - - orq %r8,%r8 - je L$_after_reduction_895 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_895: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_16_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_896 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_896 - -L$_16_blocks_overflow_896: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_896: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_897: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_897: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_897: - jmp L$_last_blocks_done_865 -L$_last_num_blocks_is_0_865: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_865: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_821 -L$_encrypt_16_blocks_821: - cmpb $240,%r15b - jae L$_16_blocks_overflow_898 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_898 -L$_16_blocks_overflow_898: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_898: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_899 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_899 - jb L$_last_num_blocks_is_7_1_899 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_899 - jb L$_last_num_blocks_is_11_9_899 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_899 - ja L$_last_num_blocks_is_16_899 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_899 - jmp L$_last_num_blocks_is_13_899 - -L$_last_num_blocks_is_11_9_899: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_899 - ja L$_last_num_blocks_is_11_899 - jmp L$_last_num_blocks_is_9_899 - -L$_last_num_blocks_is_7_1_899: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_899 - jb L$_last_num_blocks_is_3_1_899 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_899 - je L$_last_num_blocks_is_6_899 - jmp L$_last_num_blocks_is_5_899 - -L$_last_num_blocks_is_3_1_899: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_899 - je L$_last_num_blocks_is_2_899 -L$_last_num_blocks_is_1_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_900 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_900 - -L$_16_blocks_overflow_900: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_900: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_901 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_901 -L$_small_initial_partial_block_901: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_901 -L$_small_initial_compute_done_901: -L$_after_reduction_901: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_2_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_902 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_902 - -L$_16_blocks_overflow_902: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_902: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_903 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_903 -L$_small_initial_partial_block_903: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_903: - - orq %r8,%r8 - je L$_after_reduction_903 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_903: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_3_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_904 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_904 - -L$_16_blocks_overflow_904: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_904: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_905 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_905 -L$_small_initial_partial_block_905: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_905: - - orq %r8,%r8 - je L$_after_reduction_905 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_905: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_4_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_906 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_906 - -L$_16_blocks_overflow_906: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_906: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_907 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_907 -L$_small_initial_partial_block_907: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_907: - - orq %r8,%r8 - je L$_after_reduction_907 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_907: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_5_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_908 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_908 - -L$_16_blocks_overflow_908: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_908: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_909 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_909 -L$_small_initial_partial_block_909: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_909: - - orq %r8,%r8 - je L$_after_reduction_909 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_909: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_6_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_910 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_910 - -L$_16_blocks_overflow_910: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_910: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_911 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_911 -L$_small_initial_partial_block_911: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_911: - - orq %r8,%r8 - je L$_after_reduction_911 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_911: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_7_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_912 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_912 - -L$_16_blocks_overflow_912: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_912: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_913 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_913 -L$_small_initial_partial_block_913: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_913: - - orq %r8,%r8 - je L$_after_reduction_913 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_913: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_8_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_914 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_914 - -L$_16_blocks_overflow_914: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_914: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_915 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_915 -L$_small_initial_partial_block_915: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_915: - - orq %r8,%r8 - je L$_after_reduction_915 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_915: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_9_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_916 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_916 - -L$_16_blocks_overflow_916: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_916: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_917 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_917 -L$_small_initial_partial_block_917: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_917: - - orq %r8,%r8 - je L$_after_reduction_917 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_917: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_10_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_918 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_918 - -L$_16_blocks_overflow_918: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_918: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_919 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_919 -L$_small_initial_partial_block_919: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_919: - - orq %r8,%r8 - je L$_after_reduction_919 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_919: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_11_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_920 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_920 - -L$_16_blocks_overflow_920: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_920: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_921 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_921 -L$_small_initial_partial_block_921: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_921: - - orq %r8,%r8 - je L$_after_reduction_921 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_921: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_12_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_922 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_922 - -L$_16_blocks_overflow_922: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_922: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_923 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_923 -L$_small_initial_partial_block_923: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_923: - - orq %r8,%r8 - je L$_after_reduction_923 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_923: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_13_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_924 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_924 - -L$_16_blocks_overflow_924: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_924: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_925 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_925 -L$_small_initial_partial_block_925: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_925: - - orq %r8,%r8 - je L$_after_reduction_925 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_925: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_14_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_926 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_926 - -L$_16_blocks_overflow_926: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_926: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_927 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_927 -L$_small_initial_partial_block_927: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_927: - - orq %r8,%r8 - je L$_after_reduction_927 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_927: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_15_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_928 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_928 - -L$_16_blocks_overflow_928: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_928: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_929 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_929 -L$_small_initial_partial_block_929: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_929: - - orq %r8,%r8 - je L$_after_reduction_929 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_929: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_16_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_930 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_930 - -L$_16_blocks_overflow_930: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_930: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_931: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_931: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_931: - jmp L$_last_blocks_done_899 -L$_last_num_blocks_is_0_899: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_899: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_821 - -L$_message_below_32_blocks_821: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz L$_skip_hkeys_precomputation_932 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -L$_skip_hkeys_precomputation_932: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je L$_last_num_blocks_is_0_933 - - cmpl $8,%r10d - je L$_last_num_blocks_is_8_933 - jb L$_last_num_blocks_is_7_1_933 - - - cmpl $12,%r10d - je L$_last_num_blocks_is_12_933 - jb L$_last_num_blocks_is_11_9_933 - - - cmpl $15,%r10d - je L$_last_num_blocks_is_15_933 - ja L$_last_num_blocks_is_16_933 - cmpl $14,%r10d - je L$_last_num_blocks_is_14_933 - jmp L$_last_num_blocks_is_13_933 - -L$_last_num_blocks_is_11_9_933: - - cmpl $10,%r10d - je L$_last_num_blocks_is_10_933 - ja L$_last_num_blocks_is_11_933 - jmp L$_last_num_blocks_is_9_933 - -L$_last_num_blocks_is_7_1_933: - cmpl $4,%r10d - je L$_last_num_blocks_is_4_933 - jb L$_last_num_blocks_is_3_1_933 - - cmpl $6,%r10d - ja L$_last_num_blocks_is_7_933 - je L$_last_num_blocks_is_6_933 - jmp L$_last_num_blocks_is_5_933 - -L$_last_num_blocks_is_3_1_933: - - cmpl $2,%r10d - ja L$_last_num_blocks_is_3_933 - je L$_last_num_blocks_is_2_933 -L$_last_num_blocks_is_1_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae L$_16_blocks_overflow_934 - vpaddd %xmm28,%xmm2,%xmm0 - jmp L$_16_blocks_ok_934 - -L$_16_blocks_overflow_934: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -L$_16_blocks_ok_934: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_935 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_935 -L$_small_initial_partial_block_935: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp L$_after_reduction_935 -L$_small_initial_compute_done_935: -L$_after_reduction_935: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_2_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae L$_16_blocks_overflow_936 - vpaddd %ymm28,%ymm2,%ymm0 - jmp L$_16_blocks_ok_936 - -L$_16_blocks_overflow_936: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -L$_16_blocks_ok_936: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_937 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_937 -L$_small_initial_partial_block_937: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_937: - - orq %r8,%r8 - je L$_after_reduction_937 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_937: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_3_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae L$_16_blocks_overflow_938 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_938 - -L$_16_blocks_overflow_938: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_938: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_939 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_939 -L$_small_initial_partial_block_939: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_939: - - orq %r8,%r8 - je L$_after_reduction_939 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_939: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_4_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae L$_16_blocks_overflow_940 - vpaddd %zmm28,%zmm2,%zmm0 - jmp L$_16_blocks_ok_940 - -L$_16_blocks_overflow_940: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -L$_16_blocks_ok_940: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_941 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_941 -L$_small_initial_partial_block_941: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_941: - - orq %r8,%r8 - je L$_after_reduction_941 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_941: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_5_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae L$_16_blocks_overflow_942 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp L$_16_blocks_ok_942 - -L$_16_blocks_overflow_942: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -L$_16_blocks_ok_942: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_943 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_943 -L$_small_initial_partial_block_943: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_943: - - orq %r8,%r8 - je L$_after_reduction_943 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_943: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_6_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae L$_16_blocks_overflow_944 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp L$_16_blocks_ok_944 - -L$_16_blocks_overflow_944: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -L$_16_blocks_ok_944: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_945 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_945 -L$_small_initial_partial_block_945: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_945: - - orq %r8,%r8 - je L$_after_reduction_945 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_945: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_7_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae L$_16_blocks_overflow_946 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_946 - -L$_16_blocks_overflow_946: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_946: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_947 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_947 -L$_small_initial_partial_block_947: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_947: - - orq %r8,%r8 - je L$_after_reduction_947 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_947: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_8_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae L$_16_blocks_overflow_948 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp L$_16_blocks_ok_948 - -L$_16_blocks_overflow_948: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -L$_16_blocks_ok_948: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_949 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_949 -L$_small_initial_partial_block_949: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_949: - - orq %r8,%r8 - je L$_after_reduction_949 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_949: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_9_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae L$_16_blocks_overflow_950 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp L$_16_blocks_ok_950 - -L$_16_blocks_overflow_950: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -L$_16_blocks_ok_950: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_951 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_951 -L$_small_initial_partial_block_951: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_951: - - orq %r8,%r8 - je L$_after_reduction_951 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_951: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_10_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae L$_16_blocks_overflow_952 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp L$_16_blocks_ok_952 - -L$_16_blocks_overflow_952: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -L$_16_blocks_ok_952: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_953 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_953 -L$_small_initial_partial_block_953: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_953: - - orq %r8,%r8 - je L$_after_reduction_953 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_953: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_11_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae L$_16_blocks_overflow_954 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_954 - -L$_16_blocks_overflow_954: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_954: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_955 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_955 -L$_small_initial_partial_block_955: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_955: - - orq %r8,%r8 - je L$_after_reduction_955 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_955: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_12_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae L$_16_blocks_overflow_956 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp L$_16_blocks_ok_956 - -L$_16_blocks_overflow_956: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -L$_16_blocks_ok_956: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_957 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_957 -L$_small_initial_partial_block_957: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_957: - - orq %r8,%r8 - je L$_after_reduction_957 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_957: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_13_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae L$_16_blocks_overflow_958 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp L$_16_blocks_ok_958 - -L$_16_blocks_overflow_958: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -L$_16_blocks_ok_958: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_959 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_959 -L$_small_initial_partial_block_959: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_959: - - orq %r8,%r8 - je L$_after_reduction_959 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_959: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_14_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae L$_16_blocks_overflow_960 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp L$_16_blocks_ok_960 - -L$_16_blocks_overflow_960: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -L$_16_blocks_ok_960: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_961 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_961 -L$_small_initial_partial_block_961: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_961: - - orq %r8,%r8 - je L$_after_reduction_961 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_961: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_15_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae L$_16_blocks_overflow_962 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_962 - -L$_16_blocks_overflow_962: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_962: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_963 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_963 -L$_small_initial_partial_block_963: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_963: - - orq %r8,%r8 - je L$_after_reduction_963 - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_963: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_16_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae L$_16_blocks_overflow_964 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp L$_16_blocks_ok_964 - -L$_16_blocks_overflow_964: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -L$_16_blocks_ok_964: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_965: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_965: - vpxorq %xmm7,%xmm14,%xmm14 -L$_after_reduction_965: - jmp L$_last_blocks_done_933 -L$_last_num_blocks_is_0_933: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -L$_last_blocks_done_933: - vpshufb %xmm29,%xmm2,%xmm2 - jmp L$_ghash_done_821 - -L$_message_below_equal_16_blocks_821: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je L$_small_initial_num_blocks_is_8_966 - jl L$_small_initial_num_blocks_is_7_1_966 - - - cmpq $12,%r12 - je L$_small_initial_num_blocks_is_12_966 - jl L$_small_initial_num_blocks_is_11_9_966 - - - cmpq $16,%r12 - je L$_small_initial_num_blocks_is_16_966 - cmpq $15,%r12 - je L$_small_initial_num_blocks_is_15_966 - cmpq $14,%r12 - je L$_small_initial_num_blocks_is_14_966 - jmp L$_small_initial_num_blocks_is_13_966 - -L$_small_initial_num_blocks_is_11_9_966: - - cmpq $11,%r12 - je L$_small_initial_num_blocks_is_11_966 - cmpq $10,%r12 - je L$_small_initial_num_blocks_is_10_966 - jmp L$_small_initial_num_blocks_is_9_966 - -L$_small_initial_num_blocks_is_7_1_966: - cmpq $4,%r12 - je L$_small_initial_num_blocks_is_4_966 - jl L$_small_initial_num_blocks_is_3_1_966 - - cmpq $7,%r12 - je L$_small_initial_num_blocks_is_7_966 - cmpq $6,%r12 - je L$_small_initial_num_blocks_is_6_966 - jmp L$_small_initial_num_blocks_is_5_966 - -L$_small_initial_num_blocks_is_3_1_966: - - cmpq $3,%r12 - je L$_small_initial_num_blocks_is_3_966 - cmpq $2,%r12 - je L$_small_initial_num_blocks_is_2_966 - - - - - -L$_small_initial_num_blocks_is_1_966: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm6,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_967 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_967 -L$_small_initial_partial_block_967: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp L$_after_reduction_967 -L$_small_initial_compute_done_967: -L$_after_reduction_967: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_2_966: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm6,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_968 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_968 -L$_small_initial_partial_block_968: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_968: - - orq %r8,%r8 - je L$_after_reduction_968 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_968: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_3_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_969 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_969 -L$_small_initial_partial_block_969: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_969: - - orq %r8,%r8 - je L$_after_reduction_969 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_969: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_4_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_970 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_970 -L$_small_initial_partial_block_970: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_970: - - orq %r8,%r8 - je L$_after_reduction_970 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_970: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_5_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %xmm29,%xmm7,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_971 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_971 -L$_small_initial_partial_block_971: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_971: - - orq %r8,%r8 - je L$_after_reduction_971 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_971: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_6_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %ymm29,%ymm7,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_972 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_972 -L$_small_initial_partial_block_972: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_972: - - orq %r8,%r8 - je L$_after_reduction_972 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_972: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_7_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_973 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_973 -L$_small_initial_partial_block_973: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_973: - - orq %r8,%r8 - je L$_after_reduction_973 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_973: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_8_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_974 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_974 -L$_small_initial_partial_block_974: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_974: - - orq %r8,%r8 - je L$_after_reduction_974 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_974: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_9_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %xmm29,%xmm10,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_975 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_975 -L$_small_initial_partial_block_975: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_975: - - orq %r8,%r8 - je L$_after_reduction_975 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_975: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_10_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %ymm29,%ymm10,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_976 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_976 -L$_small_initial_partial_block_976: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_976: - - orq %r8,%r8 - je L$_after_reduction_976 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_976: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_11_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_977 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_977 -L$_small_initial_partial_block_977: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_977: - - orq %r8,%r8 - je L$_after_reduction_977 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_977: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_12_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_978 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_978 -L$_small_initial_partial_block_978: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_978: - - orq %r8,%r8 - je L$_after_reduction_978 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_978: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_13_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %xmm29,%xmm11,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_979 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_979 -L$_small_initial_partial_block_979: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_979: - - orq %r8,%r8 - je L$_after_reduction_979 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_979: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_14_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %ymm29,%ymm11,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_980 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_980 -L$_small_initial_partial_block_980: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_980: - - orq %r8,%r8 - je L$_after_reduction_980 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_980: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_15_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl L$_small_initial_partial_block_981 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp L$_small_initial_compute_done_981 -L$_small_initial_partial_block_981: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_981: - - orq %r8,%r8 - je L$_after_reduction_981 - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_981: - jmp L$_small_initial_blocks_encrypted_966 -L$_small_initial_num_blocks_is_16_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -L$_small_initial_partial_block_982: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -L$_small_initial_compute_done_982: - vpxorq %xmm13,%xmm14,%xmm14 -L$_after_reduction_982: -L$_small_initial_blocks_encrypted_966: -L$_ghash_done_821: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -L$_enc_dec_done_821: - jmp L$exit_gcm_decrypt -L$exit_gcm_decrypt: - cmpq $256,%r8 - jbe L$skip_hkeys_cleanup_983 - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %zmm0,0(%rsp) - vmovdqa64 %zmm0,64(%rsp) - vmovdqa64 %zmm0,128(%rsp) - vmovdqa64 %zmm0,192(%rsp) - vmovdqa64 %zmm0,256(%rsp) - vmovdqa64 %zmm0,320(%rsp) - vmovdqa64 %zmm0,384(%rsp) - vmovdqa64 %zmm0,448(%rsp) - vmovdqa64 %zmm0,512(%rsp) - vmovdqa64 %zmm0,576(%rsp) - vmovdqa64 %zmm0,640(%rsp) - vmovdqa64 %zmm0,704(%rsp) -L$skip_hkeys_cleanup_983: - vzeroupper - leaq (%rbp),%rsp - - popq %r15 - - popq %r14 - - popq %r13 - - popq %r12 - - popq %rbp - - popq %rbx - - .byte 0xf3,0xc3 -L$decrypt_seh_end: - - -.globl _ossl_aes_gcm_finalize_avx512 - -.p2align 5 -_ossl_aes_gcm_finalize_avx512: - -.byte 243,15,30,250 - vmovdqu 336(%rdi),%xmm2 - vmovdqu 32(%rdi),%xmm3 - vmovdqu 64(%rdi),%xmm4 - - - cmpq $0,%rsi - je L$_partial_done_984 - - vpclmulqdq $0x11,%xmm2,%xmm4,%xmm0 - vpclmulqdq $0x00,%xmm2,%xmm4,%xmm16 - vpclmulqdq $0x01,%xmm2,%xmm4,%xmm17 - vpclmulqdq $0x10,%xmm2,%xmm4,%xmm4 - vpxorq %xmm17,%xmm4,%xmm4 - - vpsrldq $8,%xmm4,%xmm17 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm17,%xmm0,%xmm0 - vpxorq %xmm16,%xmm4,%xmm4 - - - - vmovdqu64 POLY2(%rip),%xmm17 - - vpclmulqdq $0x01,%xmm4,%xmm17,%xmm16 - vpslldq $8,%xmm16,%xmm16 - vpxorq %xmm16,%xmm4,%xmm4 - - - - vpclmulqdq $0x00,%xmm4,%xmm17,%xmm16 - vpsrldq $4,%xmm16,%xmm16 - vpclmulqdq $0x10,%xmm4,%xmm17,%xmm4 - vpslldq $4,%xmm4,%xmm4 - - vpternlogq $0x96,%xmm16,%xmm0,%xmm4 - -L$_partial_done_984: - vmovq 56(%rdi),%xmm5 - vpinsrq $1,48(%rdi),%xmm5,%xmm5 - vpsllq $3,%xmm5,%xmm5 - - vpxor %xmm5,%xmm4,%xmm4 - - vpclmulqdq $0x11,%xmm2,%xmm4,%xmm0 - vpclmulqdq $0x00,%xmm2,%xmm4,%xmm16 - vpclmulqdq $0x01,%xmm2,%xmm4,%xmm17 - vpclmulqdq $0x10,%xmm2,%xmm4,%xmm4 - vpxorq %xmm17,%xmm4,%xmm4 - - vpsrldq $8,%xmm4,%xmm17 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm17,%xmm0,%xmm0 - vpxorq %xmm16,%xmm4,%xmm4 - - - - vmovdqu64 POLY2(%rip),%xmm17 - - vpclmulqdq $0x01,%xmm4,%xmm17,%xmm16 - vpslldq $8,%xmm16,%xmm16 - vpxorq %xmm16,%xmm4,%xmm4 - - - - vpclmulqdq $0x00,%xmm4,%xmm17,%xmm16 - vpsrldq $4,%xmm16,%xmm16 - vpclmulqdq $0x10,%xmm4,%xmm17,%xmm4 - vpslldq $4,%xmm4,%xmm4 - - vpternlogq $0x96,%xmm16,%xmm0,%xmm4 - - vpshufb SHUF_MASK(%rip),%xmm4,%xmm4 - vpxor %xmm4,%xmm3,%xmm3 - -L$_return_T_984: - vmovdqu %xmm3,64(%rdi) -L$abort_finalize: - .byte 0xf3,0xc3 - - -.globl _ossl_gcm_gmult_avx512 -.private_extern _ossl_gcm_gmult_avx512 - -.p2align 5 -_ossl_gcm_gmult_avx512: - -.byte 243,15,30,250 - vmovdqu64 (%rdi),%xmm1 - vmovdqu64 336(%rsi),%xmm2 - - vpclmulqdq $0x11,%xmm2,%xmm1,%xmm3 - vpclmulqdq $0x00,%xmm2,%xmm1,%xmm4 - vpclmulqdq $0x01,%xmm2,%xmm1,%xmm5 - vpclmulqdq $0x10,%xmm2,%xmm1,%xmm1 - vpxorq %xmm5,%xmm1,%xmm1 - - vpsrldq $8,%xmm1,%xmm5 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm5,%xmm3,%xmm3 - vpxorq %xmm4,%xmm1,%xmm1 - - - - vmovdqu64 POLY2(%rip),%xmm5 - - vpclmulqdq $0x01,%xmm1,%xmm5,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm1,%xmm1 - - - - vpclmulqdq $0x00,%xmm1,%xmm5,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm5,%xmm1 - vpslldq $4,%xmm1,%xmm1 - - vpternlogq $0x96,%xmm4,%xmm3,%xmm1 - - vmovdqu64 %xmm1,(%rdi) - vzeroupper -L$abort_gmult: - .byte 0xf3,0xc3 - - -.data -.p2align 4 -POLY:.quad 0x0000000000000001, 0xC200000000000000 - -.p2align 6 -POLY2: -.quad 0x00000001C2000000, 0xC200000000000000 -.quad 0x00000001C2000000, 0xC200000000000000 -.quad 0x00000001C2000000, 0xC200000000000000 -.quad 0x00000001C2000000, 0xC200000000000000 - -.p2align 4 -TWOONE:.quad 0x0000000000000001, 0x0000000100000000 - - - -.p2align 6 -SHUF_MASK: -.quad 0x08090A0B0C0D0E0F, 0x0001020304050607 -.quad 0x08090A0B0C0D0E0F, 0x0001020304050607 -.quad 0x08090A0B0C0D0E0F, 0x0001020304050607 -.quad 0x08090A0B0C0D0E0F, 0x0001020304050607 - -.p2align 4 -SHIFT_MASK: -.quad 0x0706050403020100, 0x0f0e0d0c0b0a0908 - -ALL_F: -.quad 0xffffffffffffffff, 0xffffffffffffffff - -ZERO: -.quad 0x0000000000000000, 0x0000000000000000 - -.p2align 4 -ONE: -.quad 0x0000000000000001, 0x0000000000000000 - -.p2align 4 -ONEf: -.quad 0x0000000000000000, 0x0100000000000000 - -.p2align 6 -ddq_add_1234: -.quad 0x0000000000000001, 0x0000000000000000 -.quad 0x0000000000000002, 0x0000000000000000 -.quad 0x0000000000000003, 0x0000000000000000 -.quad 0x0000000000000004, 0x0000000000000000 - -.p2align 6 -ddq_add_5678: -.quad 0x0000000000000005, 0x0000000000000000 -.quad 0x0000000000000006, 0x0000000000000000 -.quad 0x0000000000000007, 0x0000000000000000 -.quad 0x0000000000000008, 0x0000000000000000 - -.p2align 6 -ddq_add_4444: -.quad 0x0000000000000004, 0x0000000000000000 -.quad 0x0000000000000004, 0x0000000000000000 -.quad 0x0000000000000004, 0x0000000000000000 -.quad 0x0000000000000004, 0x0000000000000000 - -.p2align 6 -ddq_add_8888: -.quad 0x0000000000000008, 0x0000000000000000 -.quad 0x0000000000000008, 0x0000000000000000 -.quad 0x0000000000000008, 0x0000000000000000 -.quad 0x0000000000000008, 0x0000000000000000 - -.p2align 6 -ddq_addbe_1234: -.quad 0x0000000000000000, 0x0100000000000000 -.quad 0x0000000000000000, 0x0200000000000000 -.quad 0x0000000000000000, 0x0300000000000000 -.quad 0x0000000000000000, 0x0400000000000000 - -.p2align 6 -ddq_addbe_4444: -.quad 0x0000000000000000, 0x0400000000000000 -.quad 0x0000000000000000, 0x0400000000000000 -.quad 0x0000000000000000, 0x0400000000000000 -.quad 0x0000000000000000, 0x0400000000000000 - -.p2align 6 -byte_len_to_mask_table: -.value 0x0000, 0x0001, 0x0003, 0x0007 -.value 0x000f, 0x001f, 0x003f, 0x007f -.value 0x00ff, 0x01ff, 0x03ff, 0x07ff -.value 0x0fff, 0x1fff, 0x3fff, 0x7fff -.value 0xffff - -.p2align 6 -byte64_len_to_mask_table: -.quad 0x0000000000000000, 0x0000000000000001 -.quad 0x0000000000000003, 0x0000000000000007 -.quad 0x000000000000000f, 0x000000000000001f -.quad 0x000000000000003f, 0x000000000000007f -.quad 0x00000000000000ff, 0x00000000000001ff -.quad 0x00000000000003ff, 0x00000000000007ff -.quad 0x0000000000000fff, 0x0000000000001fff -.quad 0x0000000000003fff, 0x0000000000007fff -.quad 0x000000000000ffff, 0x000000000001ffff -.quad 0x000000000003ffff, 0x000000000007ffff -.quad 0x00000000000fffff, 0x00000000001fffff -.quad 0x00000000003fffff, 0x00000000007fffff -.quad 0x0000000000ffffff, 0x0000000001ffffff -.quad 0x0000000003ffffff, 0x0000000007ffffff -.quad 0x000000000fffffff, 0x000000001fffffff -.quad 0x000000003fffffff, 0x000000007fffffff -.quad 0x00000000ffffffff, 0x00000001ffffffff -.quad 0x00000003ffffffff, 0x00000007ffffffff -.quad 0x0000000fffffffff, 0x0000001fffffffff -.quad 0x0000003fffffffff, 0x0000007fffffffff -.quad 0x000000ffffffffff, 0x000001ffffffffff -.quad 0x000003ffffffffff, 0x000007ffffffffff -.quad 0x00000fffffffffff, 0x00001fffffffffff -.quad 0x00003fffffffffff, 0x00007fffffffffff -.quad 0x0000ffffffffffff, 0x0001ffffffffffff -.quad 0x0003ffffffffffff, 0x0007ffffffffffff -.quad 0x000fffffffffffff, 0x001fffffffffffff -.quad 0x003fffffffffffff, 0x007fffffffffffff -.quad 0x00ffffffffffffff, 0x01ffffffffffffff -.quad 0x03ffffffffffffff, 0x07ffffffffffffff -.quad 0x0fffffffffffffff, 0x1fffffffffffffff -.quad 0x3fffffffffffffff, 0x7fffffffffffffff -.quad 0xffffffffffffffff diff --git a/openssl/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8-unroll8_64.S b/openssl/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8-unroll8_64.S deleted file mode 100644 index a53d209f2..000000000 --- a/openssl/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8-unroll8_64.S +++ /dev/null @@ -1,8487 +0,0 @@ -#include "arm_arch.h" - -#if __ARM_MAX_ARCH__>=8 -.arch armv8-a+crypto -.text -.globl unroll8_eor3_aes_gcm_enc_128_kernel -.type unroll8_eor3_aes_gcm_enc_128_kernel,%function -.align 4 -unroll8_eor3_aes_gcm_enc_128_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, .L128_enc_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - mov x5, x9 - ld1 { v0.16b}, [x16] //CTR block 0 - - sub x5, x5, #1 //byte_len - 1 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - ldr q27, [x8, #160] //load rk10 - - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - add x5, x5, x0 - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - - aese v7.16b, v26.16b //AES block 8k+15 - round 9 - aese v5.16b, v26.16b //AES block 8k+13 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - - add x4, x0, x1, lsr #3 //end_input_ptr - cmp x0, x5 //check if we have <= 8 blocks - b.ge .L128_enc_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext - cmp x0, x5 //check if we have <= 8 blocks - -.inst 0xce006d08 //eor3 v8.16b, v8.16b, v0.16b, v27.16b //AES block 0 - result - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - -.inst 0xce016d29 //eor3 v9.16b, v9.16b, v1.16b, v27.16b //AES block 1 - result - stp q8, q9, [x2], #32 //AES block 0, 1 - store result - - rev32 v1.16b, v30.16b //CTR block 9 -.inst 0xce056dad //eor3 v13.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result - add v30.4s, v30.4s, v31.4s //CTR block 9 - -.inst 0xce026d4a //eor3 v10.16b, v10.16b, v2.16b, v27.16b //AES block 2 - result -.inst 0xce066dce //eor3 v14.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result -.inst 0xce046d8c //eor3 v12.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - -.inst 0xce036d6b //eor3 v11.16b, v11.16b, v3.16b, v27.16b //AES block 3 - result -.inst 0xce076def //eor3 v15.16b, v15.16b, v7.16b,v27.16b //AES block 7 - result - stp q10, q11, [x2], #32 //AES block 2, 3 - store result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - stp q12, q13, [x2], #32 //AES block 4, 5 - store result - - stp q14, q15, [x2], #32 //AES block 6, 7 - store result - - rev32 v4.16b, v30.16b //CTR block 12 - add v30.4s, v30.4s, v31.4s //CTR block 12 - b.ge .L128_enc_prepretail //do prepretail - -.L128_enc_main_loop: //main loop start - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - rev64 v8.16b, v8.16b //GHASH block 8k - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - ldp q26, q27, [x8, #0] //load rk0, rk1 - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev32 v7.16b, v30.16b //CTR block 8k+15 - - rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - rev64 v10.16b, v10.16b //GHASH block 8k+2 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h3l | h3h - ext v25.16b, v25.16b, v25.16b, #8 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b,v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - ldp q28, q26, [x8, #32] //load rk2, rk3 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - - rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h1l | h1h - ext v22.16b, v22.16b, v22.16b, #8 - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - ldp q26, q27, [x8, #96] //load rk6, rk7 - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - ldr d16, [x10] //MODULO - load modulo constant - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - rev32 v20.16b, v30.16b //CTR block 8k+16 - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - ldp q28, q26, [x8, #128] //load rk8, rk9 -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - rev32 v22.16b, v30.16b //CTR block 8k+17 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load plaintext - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ldr q27, [x8, #160] //load rk10 - - ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - rev32 v23.16b, v30.16b //CTR block 8k+18 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load plaintext - rev32 v25.16b, v30.16b //CTR block 8k+19 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - - cmp x0, x5 //.LOOP CONTROL -.inst 0xce046d8c //eor3 v12.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result - aese v7.16b, v26.16b //AES block 8k+15 - round 9 - - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - -.inst 0xce026d4a //eor3 v10.16b, v10.16b, v2.16b, v27.16b //AES block 8k+10 - result - - mov v2.16b, v23.16b //CTR block 8k+18 - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - - rev32 v4.16b, v30.16b //CTR block 8k+20 - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - -.inst 0xce076def //eor3 v15.16b, v15.16b, v7.16b, v27.16b //AES block 7 - result - aese v5.16b, v26.16b //AES block 8k+13 - round 9 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - -.inst 0xce016d29 //eor3 v9.16b, v9.16b, v1.16b, v27.16b //AES block 8k+9 - result -.inst 0xce036d6b //eor3 v11.16b, v11.16b, v3.16b, v27.16b //AES block 8k+11 - result - mov v3.16b, v25.16b //CTR block 8k+19 - - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment -.inst 0xce056dad //eor3 v13.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result - mov v1.16b, v22.16b //CTR block 8k+17 - -.inst 0xce006d08 //eor3 v8.16b, v8.16b, v0.16b, v27.16b //AES block 8k+8 - result - mov v0.16b, v20.16b //CTR block 8k+16 - stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result - - stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result -.inst 0xce066dce //eor3 v14.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result - - stp q12, q13, [x2], #32 //AES block 8k+12, 8k+13 - store result -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - - stp q14, q15, [x2], #32 //AES block 8k+14, 8k+15 - store result - b.lt .L128_enc_main_loop - -.L128_enc_prepretail: //PREPRETAIL - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v8.16b, v8.16b //GHASH block 8k - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h6k | h5k - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - rev64 v10.16b, v10.16b //GHASH block 8k+2 - eor v8.16b, v8.16b, v19.16b //PRE 1 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - ldp q26, q27, [x8, #0] //load rk0, rk1 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - - rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) - rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - rev32 v7.16b, v30.16b //CTR block 8k+15 - - rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h1l | h1h - ext v22.16b, v22.16b, v22.16b, #8 - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - ldp q26, q27, [x8, #96] //load rk6, rk7 - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - ldr d16, [x10] //MODULO - load modulo constant - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 -.inst 0xce114a73 //eor3 v19.16b, v19.16b, v17.16b, v18.16b //MODULO - fold into low - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - - ldr q27, [x8, #160] //load rk10 - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - aese v5.16b, v26.16b //AES block 8k+13 - round 9 - - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - aese v7.16b, v26.16b //AES block 8k+15 - round 9 -.L128_enc_tail: //TAIL - - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - ldr q8, [x0], #16 //AES block 8k+8 - load plaintext - - mov v29.16b, v27.16b - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - -.inst 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - cmp x5, #112 - b.gt .L128_enc_blocks_more_than_7 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - movi v17.8b, #0 - - cmp x5, #96 - sub v30.4s, v30.4s, v31.4s - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v2.16b - mov v2.16b, v1.16b - - movi v19.8b, #0 - movi v18.8b, #0 - b.gt .L128_enc_blocks_more_than_6 - - mov v7.16b, v6.16b - cmp x5, #80 - - sub v30.4s, v30.4s, v31.4s - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v1.16b - b.gt .L128_enc_blocks_more_than_5 - - cmp x5, #64 - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - mov v4.16b, v1.16b - b.gt .L128_enc_blocks_more_than_4 - - mov v7.16b, v6.16b - sub v30.4s, v30.4s, v31.4s - mov v6.16b, v5.16b - - mov v5.16b, v1.16b - cmp x5, #48 - b.gt .L128_enc_blocks_more_than_3 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - mov v6.16b, v1.16b - - cmp x5, #32 - ldr q24, [x3, #96] //load h4k | h3k - b.gt .L128_enc_blocks_more_than_2 - - cmp x5, #16 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v1.16b - b.gt .L128_enc_blocks_more_than_1 - - ldr q21, [x3, #48] //load h2k | h1k - sub v30.4s, v30.4s, v31.4s - b .L128_enc_blocks_less_than_1 -.L128_enc_blocks_more_than_7: //blocks left > 7 - st1 { v9.16b}, [x2], #16 //AES final-7 block - store result - - rev64 v8.16b, v9.16b //GHASH final-7 block - ldr q9, [x0], #16 //AES final-6 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - -.inst 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low -.L128_enc_blocks_more_than_6: //blocks left > 6 - - st1 { v9.16b}, [x2], #16 //AES final-6 block - store result - - rev64 v8.16b, v9.16b //GHASH final-6 block - ldr q9, [x0], #16 //AES final-5 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - -.inst 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high -.L128_enc_blocks_more_than_5: //blocks left > 5 - - st1 { v9.16b}, [x2], #16 //AES final-5 block - store result - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - ldr q9, [x0], #16 //AES final-4 block - load plaintext - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - -.inst 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - movi v16.8b, #0 //suppress further partial tag feed in - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid -.L128_enc_blocks_more_than_4: //blocks left > 4 - - st1 { v9.16b}, [x2], #16 //AES final-4 block - store result - - rev64 v8.16b, v9.16b //GHASH final-4 block - - ldr q9, [x0], #16 //AES final-3 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - -.inst 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid -.L128_enc_blocks_more_than_3: //blocks left > 3 - - st1 { v9.16b}, [x2], #16 //AES final-3 block - store result - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - ldr q24, [x3, #96] //load h4k | h3k - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - - ldr q9, [x0], #16 //AES final-2 block - load plaintext - - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - -.inst 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high -.L128_enc_blocks_more_than_2: //blocks left > 2 - - st1 { v9.16b}, [x2], #16 //AES final-2 block - store result - - rev64 v8.16b, v9.16b //GHASH final-2 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-1 block - load plaintext - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - movi v16.8b, #0 //suppress further partial tag feed in - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid -.inst 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result - - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low -.L128_enc_blocks_more_than_1: //blocks left > 1 - - st1 { v9.16b}, [x2], #16 //AES final-1 block - store result - - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-1 block - ldr q9, [x0], #16 //AES final block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - movi v16.8b, #0 //suppress further partial tag feed in - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid -.inst 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result - - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - - ldr q21, [x3, #48] //load h2k | h1k - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low -.L128_enc_blocks_less_than_1: //blocks left <= 1 - - rev32 v30.16b, v30.16b - str q30, [x16] //store the updated counter - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - mvn x6, xzr //temp0_x = 0xffffffffffffffff - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - mvn x7, xzr //temp1_x = 0xffffffffffffffff - cmp x1, #64 - - csel x13, x7, x6, lt - csel x14, x6, xzr, lt - - mov v0.d[1], x14 - mov v0.d[0], x13 //ctr0b is mask for last block - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - - rev64 v8.16b, v9.16b //GHASH final block - - bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - st1 { v9.16b}, [x2] //store all 16B - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v16.d[0], v8.d[1] //GHASH final block - mid - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - mov x0, x9 - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -.L128_enc_ret: - mov w0, #0x0 - ret -.size unroll8_eor3_aes_gcm_enc_128_kernel,.-unroll8_eor3_aes_gcm_enc_128_kernel -.globl unroll8_eor3_aes_gcm_dec_128_kernel -.type unroll8_eor3_aes_gcm_dec_128_kernel,%function -.align 4 -unroll8_eor3_aes_gcm_dec_128_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, .L128_dec_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - mov x5, x9 - ld1 { v0.16b}, [x16] //CTR block 0 - - ldp q26, q27, [x8, #0] //load rk0, rk1 - sub x5, x5, #1 //byte_len - 1 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - - rev32 v30.16b, v0.16b //set up reversed counter - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - - rev32 v7.16b, v30.16b //CTR block 7 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - add x5, x5, x0 - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - - aese v0.16b, v26.16b //AES block 0 - round 9 - aese v1.16b, v26.16b //AES block 1 - round 9 - aese v6.16b, v26.16b //AES block 6 - round 9 - - ldr q27, [x8, #160] //load rk10 - aese v4.16b, v26.16b //AES block 4 - round 9 - aese v3.16b, v26.16b //AES block 3 - round 9 - - aese v2.16b, v26.16b //AES block 2 - round 9 - aese v5.16b, v26.16b //AES block 5 - round 9 - aese v7.16b, v26.16b //AES block 7 - round 9 - - add x4, x0, x1, lsr #3 //end_input_ptr - cmp x0, x5 //check if we have <= 8 blocks - b.ge .L128_dec_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext - -.inst 0xce006d00 //eor3 v0.16b, v8.16b, v0.16b, v27.16b //AES block 0 - result -.inst 0xce016d21 //eor3 v1.16b, v9.16b, v1.16b, v27.16b //AES block 1 - result - stp q0, q1, [x2], #32 //AES block 0, 1 - store result - - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext - - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 - ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext - -.inst 0xce036d63 //eor3 v3.16b, v11.16b, v3.16b, v27.16b //AES block 3 - result -.inst 0xce026d42 //eor3 v2.16b, v10.16b, v2.16b, v27.16b //AES block 2 - result - stp q2, q3, [x2], #32 //AES block 2, 3 - store result - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - -.inst 0xce066dc6 //eor3 v6.16b, v14.16b, v6.16b, v27.16b //AES block 6 - result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - -.inst 0xce046d84 //eor3 v4.16b, v12.16b, v4.16b, v27.16b //AES block 4 - result -.inst 0xce056da5 //eor3 v5.16b, v13.16b, v5.16b, v27.16b //AES block 5 - result - stp q4, q5, [x2], #32 //AES block 4, 5 - store result - -.inst 0xce076de7 //eor3 v7.16b, v15.16b, v7.16b, v27.16b //AES block 7 - result - stp q6, q7, [x2], #32 //AES block 6, 7 - store result - rev32 v4.16b, v30.16b //CTR block 12 - - cmp x0, x5 //check if we have <= 8 blocks - add v30.4s, v30.4s, v31.4s //CTR block 12 - b.ge .L128_dec_prepretail //do prepretail - -.L128_dec_main_loop: //main loop start - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - rev64 v8.16b, v8.16b //GHASH block 8k - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - rev64 v14.16b, v14.16b //GHASH block 8k+6 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v10.16b, v10.16b //GHASH block 8k+2 - rev64 v12.16b, v12.16b //GHASH block 8k+4 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - rev32 v7.16b, v30.16b //CTR block 8k+15 - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - ldp q28, q26, [x8, #32] //load rk2, rk3 - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - - ldp q27, q28, [x8, #64] //load rk4, rk5 - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - ldr d16, [x10] //MODULO - load modulo constant -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - rev32 v20.16b, v30.16b //CTR block 8k+16 -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - rev32 v22.16b, v30.16b //CTR block 8k+17 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext - - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - rev32 v23.16b, v30.16b //CTR block 8k+18 - - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - ldr q27, [x8, #160] //load rk10 - - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - - aese v7.16b, v26.16b //AES block 8k+15 - round 9 - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - rev32 v25.16b, v30.16b //CTR block 8k+19 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - aese v5.16b, v26.16b //AES block 8k+13 - round 9 -.inst 0xce016d21 //eor3 v1.16b, v9.16b, v1.16b, v27.16b //AES block 8k+9 - result - -.inst 0xce006d00 //eor3 v0.16b, v8.16b, v0.16b, v27.16b //AES block 8k+8 - result -.inst 0xce076de7 //eor3 v7.16b, v15.16b, v7.16b, v27.16b //AES block 8k+15 - result -.inst 0xce066dc6 //eor3 v6.16b, v14.16b, v6.16b, v27.16b //AES block 8k+14 - result - -.inst 0xce026d42 //eor3 v2.16b, v10.16b, v2.16b, v27.16b //AES block 8k+10 - result - stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result - mov v1.16b, v22.16b //CTR block 8k+17 - -.inst 0xce046d84 //eor3 v4.16b, v12.16b, v4.16b, v27.16b //AES block 8k+12 - result -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - mov v0.16b, v20.16b //CTR block 8k+16 - -.inst 0xce036d63 //eor3 v3.16b, v11.16b, v3.16b, v27.16b //AES block 8k+11 - result - cmp x0, x5 //.LOOP CONTROL - stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result - -.inst 0xce056da5 //eor3 v5.16b, v13.16b, v5.16b, v27.16b //AES block 8k+13 - result - mov v2.16b, v23.16b //CTR block 8k+18 - - stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result - rev32 v4.16b, v30.16b //CTR block 8k+20 - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - - stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result - mov v3.16b, v25.16b //CTR block 8k+19 - b.lt .L128_dec_main_loop - -.L128_dec_prepretail: //PREPRETAIL - rev64 v11.16b, v11.16b //GHASH block 8k+3 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - rev64 v8.16b, v8.16b //GHASH block 8k - - rev64 v10.16b, v10.16b //GHASH block 8k+2 - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - rev64 v12.16b, v12.16b //GHASH block 8k+4 - - rev64 v14.16b, v14.16b //GHASH block 8k+6 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - rev32 v7.16b, v30.16b //CTR block 8k+15 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - ldp q28, q26, [x8, #32] //load rk2, rk3 -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - ldp q27, q28, [x8, #64] //load rk4, rk5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - ldp q26, q27, [x8, #96] //load rk6, rk7 -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - - ldr d16, [x10] //MODULO - load modulo constant - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ldp q28, q26, [x8, #128] //load rk8, rk9 - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - ldr q27, [x8, #160] //load rk10 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - - aese v6.16b, v26.16b //AES block 8k+14 - round 9 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - aese v2.16b, v26.16b //AES block 8k+10 - round 9 - - aese v3.16b, v26.16b //AES block 8k+11 - round 9 - aese v5.16b, v26.16b //AES block 8k+13 - round 9 - aese v0.16b, v26.16b //AES block 8k+8 - round 9 - - aese v4.16b, v26.16b //AES block 8k+12 - round 9 - aese v1.16b, v26.16b //AES block 8k+9 - round 9 - aese v7.16b, v26.16b //AES block 8k+15 - round 9 - -.L128_dec_tail: //TAIL - - mov v29.16b, v27.16b - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - - cmp x5, #112 - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - -.inst 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result - b.gt .L128_dec_blocks_more_than_7 - - cmp x5, #96 - mov v7.16b, v6.16b - movi v19.8b, #0 - - movi v17.8b, #0 - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v2.16b - mov v2.16b, v1.16b - - movi v18.8b, #0 - sub v30.4s, v30.4s, v31.4s - b.gt .L128_dec_blocks_more_than_6 - - cmp x5, #80 - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v1.16b - b.gt .L128_dec_blocks_more_than_5 - - cmp x5, #64 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt .L128_dec_blocks_more_than_4 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v1.16b - cmp x5, #48 - b.gt .L128_dec_blocks_more_than_3 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - cmp x5, #32 - - ldr q24, [x3, #96] //load h4k | h3k - mov v6.16b, v1.16b - b.gt .L128_dec_blocks_more_than_2 - - cmp x5, #16 - - mov v7.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt .L128_dec_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b .L128_dec_blocks_less_than_1 -.L128_dec_blocks_more_than_7: //blocks left > 7 - rev64 v8.16b, v9.16b //GHASH final-7 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - ldr q9, [x0], #16 //AES final-6 block - load ciphertext - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - st1 { v12.16b}, [x2], #16 //AES final-7 block - store result -.inst 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid -.L128_dec_blocks_more_than_6: //blocks left > 6 - - rev64 v8.16b, v9.16b //GHASH final-6 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - ldr q9, [x0], #16 //AES final-5 block - load ciphertext - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - st1 { v12.16b}, [x2], #16 //AES final-6 block - store result - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid -.inst 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result -.L128_dec_blocks_more_than_5: //blocks left > 5 - - rev64 v8.16b, v9.16b //GHASH final-5 block - - ldr q9, [x0], #16 //AES final-4 block - load ciphertext - st1 { v12.16b}, [x2], #16 //AES final-5 block - store result - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - -.inst 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - movi v16.8b, #0 //suppress further partial tag feed in - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high -.L128_dec_blocks_more_than_4: //blocks left > 4 - - rev64 v8.16b, v9.16b //GHASH final-4 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - ldr q9, [x0], #16 //AES final-3 block - load ciphertext - - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high - - st1 { v12.16b}, [x2], #16 //AES final-4 block - store result - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - -.inst 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid -.L128_dec_blocks_more_than_3: //blocks left > 3 - - st1 { v12.16b}, [x2], #16 //AES final-3 block - store result - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - ldr q24, [x3, #96] //load h4k | h3k - - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - - ldr q9, [x0], #16 //AES final-2 block - load ciphertext - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - movi v16.8b, #0 //suppress further partial tag feed in -.inst 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid -.L128_dec_blocks_more_than_2: //blocks left > 2 - - rev64 v8.16b, v9.16b //GHASH final-2 block - - st1 { v12.16b}, [x2], #16 //AES final-2 block - store result - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - ldr q9, [x0], #16 //AES final-1 block - load ciphertext - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low - -.inst 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high -.L128_dec_blocks_more_than_1: //blocks left > 1 - - st1 { v12.16b}, [x2], #16 //AES final-1 block - store result - rev64 v8.16b, v9.16b //GHASH final-1 block - - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - - ldr q9, [x0], #16 //AES final block - load ciphertext - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high - ldr q21, [x3, #48] //load h2k | h1k - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid -.inst 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result - - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid -.L128_dec_blocks_less_than_1: //blocks left <= 1 - - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - mvn x6, xzr //temp0_x = 0xffffffffffffffff - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - mvn x7, xzr //temp1_x = 0xffffffffffffffff - - csel x13, x7, x6, lt - csel x14, x6, xzr, lt - - mov v0.d[1], x14 - mov v0.d[0], x13 //ctr0b is mask for last block - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - - rev64 v8.16b, v9.16b //GHASH final block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - ins v16.d[0], v8.d[1] //GHASH final block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - - bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - st1 { v12.16b}, [x2] //store all 16B - - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up - -.inst 0xce115652 //eor3 v18.16b, v18.16b, v17.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.inst 0xce124673 //eor3 v19.16b, v19.16b, v18.16b, v17.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - rev32 v30.16b, v30.16b - - str q30, [x16] //store the updated counter - - mov x0, x9 - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret -.L128_dec_ret: - mov w0, #0x0 - ret -.size unroll8_eor3_aes_gcm_dec_128_kernel,.-unroll8_eor3_aes_gcm_dec_128_kernel -.globl unroll8_eor3_aes_gcm_enc_192_kernel -.type unroll8_eor3_aes_gcm_enc_192_kernel,%function -.align 4 -unroll8_eor3_aes_gcm_enc_192_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, .L192_enc_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - mov x5, x9 - ld1 { v0.16b}, [x16] //CTR block 0 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - sub x5, x5, #1 //byte_len - 1 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - add x5, x5, x0 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - - add x4, x0, x1, lsr #3 //end_input_ptr - cmp x0, x5 //check if we have <= 8 blocks - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 9 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - ldp q27, q28, [x8, #160] //load rk10, rk11 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 9 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 9 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 9 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 9 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 14 - round 10 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 11 - round 10 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 9 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 13 - round 10 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 12 - round 10 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 10 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 15 - round 10 - - aese v6.16b, v28.16b //AES block 14 - round 11 - aese v3.16b, v28.16b //AES block 11 - round 11 - - aese v4.16b, v28.16b //AES block 12 - round 11 - aese v7.16b, v28.16b //AES block 15 - round 11 - ldr q26, [x8, #192] //load rk12 - - aese v1.16b, v28.16b //AES block 9 - round 11 - aese v5.16b, v28.16b //AES block 13 - round 11 - - aese v2.16b, v28.16b //AES block 10 - round 11 - aese v0.16b, v28.16b //AES block 8 - round 11 - b.ge .L192_enc_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext - -.inst 0xce006908 //eor3 v8.16b, v8.16b, v0.16b, v26.16b //AES block 0 - result - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - -.inst 0xce03696b //eor3 v11.16b, v11.16b, v3.16b, v26.16b //AES block 3 - result -.inst 0xce016929 //eor3 v9.16b, v9.16b, v1.16b, v26.16b //AES block 1 - result - - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 -.inst 0xce04698c //eor3 v12.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result - -.inst 0xce0569ad //eor3 v13.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result -.inst 0xce0769ef //eor3 v15.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result - stp q8, q9, [x2], #32 //AES block 0, 1 - store result - -.inst 0xce02694a //eor3 v10.16b, v10.16b, v2.16b, v26.16b //AES block 2 - result - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - - stp q10, q11, [x2], #32 //AES block 2, 3 - store result - cmp x0, x5 //check if we have <= 8 blocks - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 -.inst 0xce0669ce //eor3 v14.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result - - stp q12, q13, [x2], #32 //AES block 4, 5 - store result - - rev32 v4.16b, v30.16b //CTR block 12 - stp q14, q15, [x2], #32 //AES block 6, 7 - store result - add v30.4s, v30.4s, v31.4s //CTR block 12 - - b.ge .L192_enc_prepretail //do prepretail - -.L192_enc_main_loop: //main loop start - rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) - ldp q26, q27, [x8, #0] //load rk0, rk1 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - rev64 v8.16b, v8.16b //GHASH block 8k - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev64 v11.16b, v11.16b //GHASH block 8k+3 - rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - rev32 v7.16b, v30.16b //CTR block 8k+15 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) - - rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - ldp q28, q26, [x8, #128] //load rk8, rk9 - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - ldr d16, [x10] //MODULO - load modulo constant -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - ldp q27, q28, [x8, #160] //load rk10, rk11 - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - rev32 v20.16b, v30.16b //CTR block 8k+16 - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - rev32 v22.16b, v30.16b //CTR block 8k+17 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - ldr q26, [x8, #192] //load rk12 - ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext - - aese v4.16b, v28.16b //AES block 8k+12 - round 11 -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load plaintext - aese v2.16b, v28.16b //AES block 8k+10 - round 11 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - - rev32 v23.16b, v30.16b //CTR block 8k+18 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - aese v5.16b, v28.16b //AES block 8k+13 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - - aese v7.16b, v28.16b //AES block 8k+15 - round 11 - aese v0.16b, v28.16b //AES block 8k+8 - round 11 -.inst 0xce04698c //eor3 v12.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result - - aese v6.16b, v28.16b //AES block 8k+14 - round 11 - aese v3.16b, v28.16b //AES block 8k+11 - round 11 - aese v1.16b, v28.16b //AES block 8k+9 - round 11 - - rev32 v25.16b, v30.16b //CTR block 8k+19 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 -.inst 0xce0769ef //eor3 v15.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result - -.inst 0xce02694a //eor3 v10.16b, v10.16b, v2.16b, v26.16b //AES block 8k+10 - result -.inst 0xce006908 //eor3 v8.16b, v8.16b, v0.16b, v26.16b //AES block 8k+8 - result - mov v2.16b, v23.16b //CTR block 8k+18 - -.inst 0xce016929 //eor3 v9.16b, v9.16b, v1.16b, v26.16b //AES block 8k+9 - result - mov v1.16b, v22.16b //CTR block 8k+17 - stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.inst 0xce0669ce //eor3 v14.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result - mov v0.16b, v20.16b //CTR block 8k+16 - rev32 v4.16b, v30.16b //CTR block 8k+20 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 -.inst 0xce0569ad //eor3 v13.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - -.inst 0xce03696b //eor3 v11.16b, v11.16b, v3.16b, v26.16b //AES block 8k+11 - result - mov v3.16b, v25.16b //CTR block 8k+19 - - stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result - - stp q12, q13, [x2], #32 //AES block 8k+12, 8k+13 - store result - - cmp x0, x5 //.LOOP CONTROL - stp q14, q15, [x2], #32 //AES block 8k+14, 8k+15 - store result - b.lt .L192_enc_main_loop - -.L192_enc_prepretail: //PREPRETAIL - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldp q26, q27, [x8, #0] //load rk0, rk1 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v8.16b //GHASH block 8k - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev32 v7.16b, v30.16b //CTR block 8k+15 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - rev64 v13.16b, v13.16b //GHASH block 8k+5 (t0, t1, t2 and t3 free) - rev64 v14.16b, v14.16b //GHASH block 8k+6 (t0, t1, and t2 free) - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - ldp q27, q28, [x8, #64] //load rk4, rk5 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - rev64 v12.16b, v12.16b //GHASH block 8k+4 (t0, t1, and t2 free) - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 (t0, t1, t2 and t3 free) - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - ldr d16, [x10] //MODULO - load modulo constant - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ext v29.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - ldp q27, q28, [x8, #160] //load rk10, rk11 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ldr q26, [x8, #192] //load rk12 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - - aese v1.16b, v28.16b //AES block 8k+9 - round 11 - aese v7.16b, v28.16b //AES block 8k+15 - round 11 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v3.16b, v28.16b //AES block 8k+11 - round 11 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - aese v2.16b, v28.16b //AES block 8k+10 - round 11 - aese v0.16b, v28.16b //AES block 8k+8 - round 11 - - aese v6.16b, v28.16b //AES block 8k+14 - round 11 - aese v4.16b, v28.16b //AES block 8k+12 - round 11 - aese v5.16b, v28.16b //AES block 8k+13 - round 11 - -.L192_enc_tail: //TAIL - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - - ldr q8, [x0], #16 //AES block 8k+8 - l3ad plaintext - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - - mov v29.16b, v26.16b - - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - cmp x5, #112 - -.inst 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - b.gt .L192_enc_blocks_more_than_7 - - cmp x5, #96 - mov v7.16b, v6.16b - movi v17.8b, #0 - - mov v6.16b, v5.16b - movi v19.8b, #0 - sub v30.4s, v30.4s, v31.4s - - mov v5.16b, v4.16b - mov v4.16b, v3.16b - mov v3.16b, v2.16b - - mov v2.16b, v1.16b - movi v18.8b, #0 - b.gt .L192_enc_blocks_more_than_6 - - mov v7.16b, v6.16b - cmp x5, #80 - - mov v6.16b, v5.16b - mov v5.16b, v4.16b - mov v4.16b, v3.16b - - mov v3.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt .L192_enc_blocks_more_than_5 - - cmp x5, #64 - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v1.16b - b.gt .L192_enc_blocks_more_than_4 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v1.16b - - sub v30.4s, v30.4s, v31.4s - cmp x5, #48 - b.gt .L192_enc_blocks_more_than_3 - - mov v7.16b, v6.16b - mov v6.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - - ldr q24, [x3, #96] //load h4k | h3k - cmp x5, #32 - b.gt .L192_enc_blocks_more_than_2 - - sub v30.4s, v30.4s, v31.4s - - cmp x5, #16 - mov v7.16b, v1.16b - b.gt .L192_enc_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b .L192_enc_blocks_less_than_1 -.L192_enc_blocks_more_than_7: //blocks left > 7 - st1 { v9.16b}, [x2], #16 //AES final-7 block - store result - - rev64 v8.16b, v9.16b //GHASH final-7 block - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - - ldr q9, [x0], #16 //AES final-6 block - load plaintext - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid -.inst 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result -.L192_enc_blocks_more_than_6: //blocks left > 6 - - st1 { v9.16b}, [x2], #16 //AES final-6 block - store result - - rev64 v8.16b, v9.16b //GHASH final-6 block - - ldr q9, [x0], #16 //AES final-5 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low -.inst 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid -.L192_enc_blocks_more_than_5: //blocks left > 5 - - st1 { v9.16b}, [x2], #16 //AES final-5 block - store result - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - - ldr q9, [x0], #16 //AES final-4 block - load plaintext - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - -.inst 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid -.L192_enc_blocks_more_than_4: //blocks left > 4 - - st1 { v9.16b}, [x2], #16 //AES final-4 block - store result - - rev64 v8.16b, v9.16b //GHASH final-4 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-3 block - load plaintext - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid -.inst 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result -.L192_enc_blocks_more_than_3: //blocks left > 3 - - ldr q24, [x3, #96] //load h4k | h3k - st1 { v9.16b}, [x2], #16 //AES final-3 block - store result - - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - - ldr q9, [x0], #16 //AES final-2 block - load plaintext - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - -.inst 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high -.L192_enc_blocks_more_than_2: //blocks left > 2 - - st1 { v9.16b}, [x2], #16 //AES final-2 block - store result - - rev64 v8.16b, v9.16b //GHASH final-2 block - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-1 block - load plaintext - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid -.inst 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result -.L192_enc_blocks_more_than_1: //blocks left > 1 - - ldr q22, [x3, #64] //load h1l | h1h - ext v22.16b, v22.16b, v22.16b, #8 - st1 { v9.16b}, [x2], #16 //AES final-1 block - store result - - rev64 v8.16b, v9.16b //GHASH final-1 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - - ldr q9, [x0], #16 //AES final block - load plaintext - ldr q21, [x3, #48] //load h2k | h1k - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - -.inst 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high -.L192_enc_blocks_less_than_1: //blocks left <= 1 - - mvn x6, xzr //temp0_x = 0xffffffffffffffff - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - mvn x7, xzr //temp1_x = 0xffffffffffffffff - - csel x13, x7, x6, lt - csel x14, x6, xzr, lt - - mov v0.d[1], x14 - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - mov v0.d[0], x13 //ctr0b is mask for last block - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - - rev64 v8.16b, v9.16b //GHASH final block - bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - - st1 { v9.16b}, [x2] //store all 16B - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v16.d[0], v8.d[1] //GHASH final block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - rev32 v30.16b, v30.16b - - str q30, [x16] //store the updated counter -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - - mov x0, x9 //return sizes - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -.L192_enc_ret: - mov w0, #0x0 - ret -.size unroll8_eor3_aes_gcm_enc_192_kernel,.-unroll8_eor3_aes_gcm_enc_192_kernel -.globl unroll8_eor3_aes_gcm_dec_192_kernel -.type unroll8_eor3_aes_gcm_dec_192_kernel,%function -.align 4 -unroll8_eor3_aes_gcm_dec_192_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, .L192_dec_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - mov x5, x9 - ld1 { v0.16b}, [x16] //CTR block 0 - ld1 { v19.16b}, [x3] - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - - sub x5, x5, #1 //byte_len - 1 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - - add x4, x0, x1, lsr #3 //end_input_ptr - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 9 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - - ldp q27, q28, [x8, #160] //load rk10, rk11 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 9 - add x5, x5, x0 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 9 - - cmp x0, x5 //check if we have <= 8 blocks - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 9 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 9 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 9 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 10 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 10 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 10 - ldr q26, [x8, #192] //load rk12 - - aese v0.16b, v28.16b //AES block 0 - round 11 - aese v1.16b, v28.16b //AES block 1 - round 11 - aese v4.16b, v28.16b //AES block 4 - round 11 - - aese v6.16b, v28.16b //AES block 6 - round 11 - aese v5.16b, v28.16b //AES block 5 - round 11 - aese v7.16b, v28.16b //AES block 7 - round 11 - - aese v2.16b, v28.16b //AES block 2 - round 11 - aese v3.16b, v28.16b //AES block 3 - round 11 - b.ge .L192_dec_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext - -.inst 0xce016921 //eor3 v1.16b, v9.16b, v1.16b, v26.16b //AES block 1 - result -.inst 0xce006900 //eor3 v0.16b, v8.16b, v0.16b, v26.16b //AES block 0 - result - stp q0, q1, [x2], #32 //AES block 0, 1 - store result - - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 -.inst 0xce036963 //eor3 v3.16b, v11.16b, v3.16b, v26.16b //AES block 3 - result - -.inst 0xce026942 //eor3 v2.16b, v10.16b, v2.16b, v26.16b //AES block 2 - result - stp q2, q3, [x2], #32 //AES block 2, 3 - store result - ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - -.inst 0xce046984 //eor3 v4.16b, v12.16b, v4.16b, v26.16b //AES block 4 - result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - -.inst 0xce0569a5 //eor3 v5.16b, v13.16b, v5.16b, v26.16b //AES block 5 - result - stp q4, q5, [x2], #32 //AES block 4, 5 - store result - cmp x0, x5 //check if we have <= 8 blocks - -.inst 0xce0669c6 //eor3 v6.16b, v14.16b, v6.16b, v26.16b //AES block 6 - result -.inst 0xce0769e7 //eor3 v7.16b, v15.16b, v7.16b, v26.16b //AES block 7 - result - rev32 v4.16b, v30.16b //CTR block 12 - - add v30.4s, v30.4s, v31.4s //CTR block 12 - stp q6, q7, [x2], #32 //AES block 6, 7 - store result - b.ge .L192_dec_prepretail //do prepretail - -.L192_dec_main_loop: //main loop start - rev64 v9.16b, v9.16b //GHASH block 8k+1 - ldp q26, q27, [x8, #0] //load rk0, rk1 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - rev64 v8.16b, v8.16b //GHASH block 8k - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v12.16b, v12.16b //GHASH block 8k+4 - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - rev32 v7.16b, v30.16b //CTR block 8k+15 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - rev64 v10.16b, v10.16b //GHASH block 8k+2 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - rev64 v14.16b, v14.16b //GHASH block 8k+6 - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - ldp q28, q26, [x8, #128] //load rk8, rk9 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - ldr d16, [x10] //MODULO - load modulo constant - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - rev32 v20.16b, v30.16b //CTR block 8k+16 - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - ldp q27, q28, [x8, #160] //load rk10, rk11 - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext - - rev32 v22.16b, v30.16b //CTR block 8k+17 - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext - - rev32 v23.16b, v30.16b //CTR block 8k+18 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - ldr q26, [x8, #192] //load rk12 - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - - aese v0.16b, v28.16b //AES block 8k+8 - round 11 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - aese v1.16b, v28.16b //AES block 8k+9 - round 11 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v6.16b, v28.16b //AES block 8k+14 - round 11 - aese v3.16b, v28.16b //AES block 8k+11 - round 11 - -.inst 0xce006900 //eor3 v0.16b, v8.16b, v0.16b, v26.16b //AES block 8k+8 - result - rev32 v25.16b, v30.16b //CTR block 8k+19 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - - aese v4.16b, v28.16b //AES block 8k+12 - round 11 - aese v2.16b, v28.16b //AES block 8k+10 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - - aese v7.16b, v28.16b //AES block 8k+15 - round 11 - aese v5.16b, v28.16b //AES block 8k+13 - round 11 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - -.inst 0xce016921 //eor3 v1.16b, v9.16b, v1.16b, v26.16b //AES block 8k+9 - result - stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result -.inst 0xce036963 //eor3 v3.16b, v11.16b, v3.16b, v26.16b //AES block 8k+11 - result - -.inst 0xce026942 //eor3 v2.16b, v10.16b, v2.16b, v26.16b //AES block 8k+10 - result -.inst 0xce0769e7 //eor3 v7.16b, v15.16b, v7.16b, v26.16b //AES block 8k+15 - result - stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result - -.inst 0xce0569a5 //eor3 v5.16b, v13.16b, v5.16b, v26.16b //AES block 8k+13 - result -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - mov v3.16b, v25.16b //CTR block 8k+19 - -.inst 0xce046984 //eor3 v4.16b, v12.16b, v4.16b, v26.16b //AES block 8k+12 - result - stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result - cmp x0, x5 //.LOOP CONTROL - -.inst 0xce0669c6 //eor3 v6.16b, v14.16b, v6.16b, v26.16b //AES block 8k+14 - result - stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result - mov v0.16b, v20.16b //CTR block 8k+16 - - mov v1.16b, v22.16b //CTR block 8k+17 - mov v2.16b, v23.16b //CTR block 8k+18 - - rev32 v4.16b, v30.16b //CTR block 8k+20 - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - b.lt .L192_dec_main_loop - -.L192_dec_prepretail: //PREPRETAIL - ldp q26, q27, [x8, #0] //load rk0, rk1 - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v8.16b //GHASH block 8k - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - rev32 v7.16b, v30.16b //CTR block 8k+15 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - rev64 v13.16b, v13.16b //GHASH block 8k+5 - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - - rev64 v15.16b, v15.16b //GHASH block 8k+7 - -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - rev64 v12.16b, v12.16b //GHASH block 8k+4 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - - rev64 v14.16b, v14.16b //GHASH block 8k+6 - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - ldp q28, q26, [x8, #128] //load rk8, rk9 - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - - ldr d16, [x10] //MODULO - load modulo constant -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - ldp q27, q28, [x8, #160] //load rk10, rk11 - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ldr q26, [x8, #192] //load rk12 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - - aese v0.16b, v28.16b //AES block 8k+8 - round 11 -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - aese v5.16b, v28.16b //AES block 8k+13 - round 11 - - aese v2.16b, v28.16b //AES block 8k+10 - round 11 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - - aese v6.16b, v28.16b //AES block 8k+14 - round 11 - aese v4.16b, v28.16b //AES block 8k+12 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v3.16b, v28.16b //AES block 8k+11 - round 11 - aese v1.16b, v28.16b //AES block 8k+9 - round 11 - aese v7.16b, v28.16b //AES block 8k+15 - round 11 - -.L192_dec_tail: //TAIL - - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - - mov v29.16b, v26.16b - - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - -.inst 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result - cmp x5, #112 - b.gt .L192_dec_blocks_more_than_7 - - mov v7.16b, v6.16b - movi v17.8b, #0 - sub v30.4s, v30.4s, v31.4s - - mov v6.16b, v5.16b - mov v5.16b, v4.16b - mov v4.16b, v3.16b - - cmp x5, #96 - movi v19.8b, #0 - mov v3.16b, v2.16b - - mov v2.16b, v1.16b - movi v18.8b, #0 - b.gt .L192_dec_blocks_more_than_6 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - mov v4.16b, v3.16b - mov v3.16b, v1.16b - - sub v30.4s, v30.4s, v31.4s - cmp x5, #80 - b.gt .L192_dec_blocks_more_than_5 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - mov v4.16b, v1.16b - cmp x5, #64 - - sub v30.4s, v30.4s, v31.4s - b.gt .L192_dec_blocks_more_than_4 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v1.16b - cmp x5, #48 - b.gt .L192_dec_blocks_more_than_3 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - cmp x5, #32 - - mov v6.16b, v1.16b - ldr q24, [x3, #96] //load h4k | h3k - b.gt .L192_dec_blocks_more_than_2 - - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v1.16b - cmp x5, #16 - b.gt .L192_dec_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b .L192_dec_blocks_less_than_1 -.L192_dec_blocks_more_than_7: //blocks left > 7 - rev64 v8.16b, v9.16b //GHASH final-7 block - - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - ldr q9, [x0], #16 //AES final-6 block - load ciphertext - - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - st1 { v12.16b}, [x2], #16 //AES final-7 block - store result - -.inst 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid - movi v16.8b, #0 //suppress further partial tag feed in -.L192_dec_blocks_more_than_6: //blocks left > 6 - - rev64 v8.16b, v9.16b //GHASH final-6 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-5 block - load ciphertext - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - st1 { v12.16b}, [x2], #16 //AES final-6 block - store result -.inst 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low -.L192_dec_blocks_more_than_5: //blocks left > 5 - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - - ldr q9, [x0], #16 //AES final-4 block - load ciphertext - - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - movi v16.8b, #0 //suppress further partial tag feed in - st1 { v12.16b}, [x2], #16 //AES final-5 block - store result - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid -.inst 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result -.L192_dec_blocks_more_than_4: //blocks left > 4 - - rev64 v8.16b, v9.16b //GHASH final-4 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - - ldr q9, [x0], #16 //AES final-3 block - load ciphertext - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - st1 { v12.16b}, [x2], #16 //AES final-4 block - store result - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - -.inst 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high -.L192_dec_blocks_more_than_3: //blocks left > 3 - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-3 block - ldr q9, [x0], #16 //AES final-2 block - load ciphertext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high - movi v16.8b, #0 //suppress further partial tag feed in - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - - st1 { v12.16b}, [x2], #16 //AES final-3 block - store result - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid -.inst 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - ldr q24, [x3, #96] //load h4k | h3k - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid -.L192_dec_blocks_more_than_2: //blocks left > 2 - - rev64 v8.16b, v9.16b //GHASH final-2 block - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - ldr q9, [x0], #16 //AES final-1 block - load ciphertext - - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low - st1 { v12.16b}, [x2], #16 //AES final-2 block - store result - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid -.inst 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result -.L192_dec_blocks_more_than_1: //blocks left > 1 - - rev64 v8.16b, v9.16b //GHASH final-1 block - ldr q9, [x0], #16 //AES final block - load ciphertext - ldr q22, [x3, #64] //load h1l | h1h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - ldr q21, [x3, #48] //load h2k | h1k - - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - st1 { v12.16b}, [x2], #16 //AES final-1 block - store result - - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - -.inst 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high -.L192_dec_blocks_less_than_1: //blocks left <= 1 - - rev32 v30.16b, v30.16b - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - str q30, [x16] //store the updated counter - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - mvn x6, xzr //temp0_x = 0xffffffffffffffff - - and x1, x1, #127 //bit_length %= 128 - - mvn x7, xzr //temp1_x = 0xffffffffffffffff - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - - csel x13, x7, x6, lt - csel x14, x6, xzr, lt - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - - mov v0.d[1], x14 - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - - mov v0.d[0], x13 //ctr0b is mask for last block - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - - rev64 v8.16b, v9.16b //GHASH final block - - st1 { v12.16b}, [x2] //store all 16B - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v16.d[0], v8.d[1] //GHASH final block - mid - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - - eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up - -.inst 0xce115652 //eor3 v18.16b, v18.16b, v17.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.inst 0xce124673 //eor3 v19.16b, v19.16b, v18.16b, v17.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - - mov x0, x9 - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -.L192_dec_ret: - mov w0, #0x0 - ret -.size unroll8_eor3_aes_gcm_dec_192_kernel,.-unroll8_eor3_aes_gcm_dec_192_kernel -.globl unroll8_eor3_aes_gcm_enc_256_kernel -.type unroll8_eor3_aes_gcm_enc_256_kernel,%function -.align 4 -unroll8_eor3_aes_gcm_enc_256_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, .L256_enc_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - ld1 { v0.16b}, [x16] //CTR block 0 - - mov x5, x9 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - sub x5, x5, #1 //byte_len - 1 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - - add x5, x5, x0 - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - ldp q27, q28, [x8, #160] //load rk10, rk11 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 9 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 9 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 9 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 9 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 10 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 10 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 9 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 10 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 10 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 11 - ldp q26, q27, [x8, #192] //load rk12, rk13 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 11 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 11 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 11 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 11 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 11 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 11 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 11 - - add v30.4s, v30.4s, v31.4s //CTR block 7 - ldr q28, [x8, #224] //load rk14 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 12 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 12 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 12 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 12 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 12 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 12 - - aese v2.16b, v27.16b //AES block 2 - round 13 - aese v1.16b, v27.16b //AES block 1 - round 13 - aese v4.16b, v27.16b //AES block 4 - round 13 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 12 - - aese v0.16b, v27.16b //AES block 0 - round 13 - aese v5.16b, v27.16b //AES block 5 - round 13 - - aese v6.16b, v27.16b //AES block 6 - round 13 - aese v7.16b, v27.16b //AES block 7 - round 13 - aese v3.16b, v27.16b //AES block 3 - round 13 - - add x4, x0, x1, lsr #3 //end_input_ptr - cmp x0, x5 //check if we have <= 8 blocks - b.ge .L256_enc_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load plaintext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load plaintext - -.inst 0xce007108 //eor3 v8.16b, v8.16b, v0.16b, v28.16b //AES block 0 - result - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 - -.inst 0xce017129 //eor3 v9.16b, v9.16b, v1.16b, v28.16b //AES block 1 - result -.inst 0xce03716b //eor3 v11.16b, v11.16b, v3.16b, v28.16b //AES block 3 - result - - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 - ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext -.inst 0xce02714a //eor3 v10.16b, v10.16b, v2.16b, v28.16b //AES block 2 - result - cmp x0, x5 //check if we have <= 8 blocks - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - stp q8, q9, [x2], #32 //AES block 0, 1 - store result - - stp q10, q11, [x2], #32 //AES block 2, 3 - store result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - -.inst 0xce04718c //eor3 v12.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result - -.inst 0xce0771ef //eor3 v15.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result -.inst 0xce0671ce //eor3 v14.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result -.inst 0xce0571ad //eor3 v13.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result - - stp q12, q13, [x2], #32 //AES block 4, 5 - store result - rev32 v4.16b, v30.16b //CTR block 12 - - stp q14, q15, [x2], #32 //AES block 6, 7 - store result - add v30.4s, v30.4s, v31.4s //CTR block 12 - b.ge .L256_enc_prepretail //do prepretail - -.L256_enc_main_loop: //main loop start - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - rev64 v8.16b, v8.16b //GHASH block 8k - - rev64 v12.16b, v12.16b //GHASH block 8k+4 - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - rev32 v7.16b, v30.16b //CTR block 8k+15 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - eor v8.16b, v8.16b, v19.16b //PRE 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - rev64 v14.16b, v14.16b //GHASH block 8k+6 - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - ldp q27, q28, [x8, #64] //load rk4, rk5 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - ldp q26, q27, [x8, #96] //load rk6, rk7 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - - ldp q28, q26, [x8, #128] //load rk8, rk9 - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - - ldp q27, q28, [x8, #160] //load rk10, rk11 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - ldr d16, [x10] //MODULO - load modulo constant - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - - ldp q26, q27, [x8, #192] //load rk12, rk13 - rev32 v20.16b, v30.16b //CTR block 8k+16 - - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load plaintext - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 - rev32 v22.16b, v30.16b //CTR block 8k+17 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 - ldr q28, [x8, #224] //load rk14 - aese v7.16b, v27.16b //AES block 8k+15 - round 13 - - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load plaintext - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 - ldp q12, q13, [x0], #32 //AES block 4, 5 - load plaintext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load plaintext - aese v2.16b, v27.16b //AES block 8k+10 - round 13 - aese v4.16b, v27.16b //AES block 8k+12 - round 13 - - rev32 v23.16b, v30.16b //CTR block 8k+18 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - aese v5.16b, v27.16b //AES block 8k+13 - round 13 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 - aese v3.16b, v27.16b //AES block 8k+11 - round 13 - cmp x0, x5 //.LOOP CONTROL - -.inst 0xce02714a //eor3 v10.16b, v10.16b, v2.16b, v28.16b //AES block 8k+10 - result - rev32 v25.16b, v30.16b //CTR block 8k+19 - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - - aese v0.16b, v27.16b //AES block 8k+8 - round 13 - aese v6.16b, v27.16b //AES block 8k+14 - round 13 -.inst 0xce0571ad //eor3 v13.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result - - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v1.16b, v27.16b //AES block 8k+9 - round 13 - -.inst 0xce04718c //eor3 v12.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result - rev32 v4.16b, v30.16b //CTR block 8k+20 -.inst 0xce03716b //eor3 v11.16b, v11.16b, v3.16b, v28.16b //AES block 8k+11 - result - - mov v3.16b, v25.16b //CTR block 8k+19 -.inst 0xce017129 //eor3 v9.16b, v9.16b, v1.16b, v28.16b //AES block 8k+9 - result -.inst 0xce007108 //eor3 v8.16b, v8.16b, v0.16b, v28.16b //AES block 8k+8 - result - - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - stp q8, q9, [x2], #32 //AES block 8k+8, 8k+9 - store result - mov v2.16b, v23.16b //CTR block 8k+18 - -.inst 0xce0771ef //eor3 v15.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result -.inst 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low - stp q10, q11, [x2], #32 //AES block 8k+10, 8k+11 - store result - -.inst 0xce0671ce //eor3 v14.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result - mov v1.16b, v22.16b //CTR block 8k+17 - stp q12, q13, [x2], #32 //AES block 4, 5 - store result - - stp q14, q15, [x2], #32 //AES block 6, 7 - store result - mov v0.16b, v20.16b //CTR block 8k+16 - b.lt .L256_enc_main_loop - -.L256_enc_prepretail: //PREPRETAIL - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldp q26, q27, [x8, #0] //load rk0, rk1 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v10.16b, v10.16b //GHASH block 8k+2 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - rev64 v13.16b, v13.16b //GHASH block 8k+5 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - rev32 v7.16b, v30.16b //CTR block 8k+15 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - rev64 v8.16b, v8.16b //GHASH block 8k - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - ldp q28, q26, [x8, #32] //load rk2, rk3 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - eor v8.16b, v8.16b, v19.16b //PRE 1 - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - rev64 v14.16b, v14.16b //GHASH block 8k+6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - rev64 v12.16b, v12.16b //GHASH block 8k+4 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - ldp q28, q26, [x8, #128] //load rk8, rk9 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - - ldp q27, q28, [x8, #160] //load rk10, rk11 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - ldr d16, [x10] //MODULO - load modulo constant - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 - - ldp q26, q27, [x8, #192] //load rk12, rk13 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 - ldr q28, [x8, #224] //load rk14 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 - aese v0.16b, v27.16b //AES block 8k+8 - round 13 - -.inst 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low - aese v5.16b, v27.16b //AES block 8k+13 - round 13 - aese v1.16b, v27.16b //AES block 8k+9 - round 13 - - aese v3.16b, v27.16b //AES block 8k+11 - round 13 - aese v4.16b, v27.16b //AES block 8k+12 - round 13 - aese v7.16b, v27.16b //AES block 8k+15 - round 13 - - aese v2.16b, v27.16b //AES block 8k+10 - round 13 - aese v6.16b, v27.16b //AES block 8k+14 - round 13 -.L256_enc_tail: //TAIL - - ldp q24, q25, [x3, #192] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - - ldr q8, [x0], #16 //AES block 8k+8 - load plaintext - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - mov v29.16b, v28.16b - - cmp x5, #112 -.inst 0xce007509 //eor3 v9.16b, v8.16b, v0.16b, v29.16b //AES block 8k+8 - result - b.gt .L256_enc_blocks_more_than_7 - - movi v19.8b, #0 - mov v7.16b, v6.16b - movi v17.8b, #0 - - mov v6.16b, v5.16b - mov v5.16b, v4.16b - mov v4.16b, v3.16b - - mov v3.16b, v2.16b - sub v30.4s, v30.4s, v31.4s - mov v2.16b, v1.16b - - movi v18.8b, #0 - cmp x5, #96 - b.gt .L256_enc_blocks_more_than_6 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - cmp x5, #80 - - mov v5.16b, v4.16b - mov v4.16b, v3.16b - mov v3.16b, v1.16b - - sub v30.4s, v30.4s, v31.4s - b.gt .L256_enc_blocks_more_than_5 - - mov v7.16b, v6.16b - sub v30.4s, v30.4s, v31.4s - - mov v6.16b, v5.16b - mov v5.16b, v4.16b - - cmp x5, #64 - mov v4.16b, v1.16b - b.gt .L256_enc_blocks_more_than_4 - - cmp x5, #48 - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt .L256_enc_blocks_more_than_3 - - cmp x5, #32 - mov v7.16b, v6.16b - ldr q24, [x3, #96] //load h4k | h3k - - mov v6.16b, v1.16b - sub v30.4s, v30.4s, v31.4s - b.gt .L256_enc_blocks_more_than_2 - - mov v7.16b, v1.16b - - sub v30.4s, v30.4s, v31.4s - cmp x5, #16 - b.gt .L256_enc_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b .L256_enc_blocks_less_than_1 -.L256_enc_blocks_more_than_7: //blocks left > 7 - st1 { v9.16b}, [x2], #16 //AES final-7 block - store result - - rev64 v8.16b, v9.16b //GHASH final-7 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ldr q9, [x0], #16 //AES final-6 block - load plaintext - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid -.inst 0xce017529 //eor3 v9.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low -.L256_enc_blocks_more_than_6: //blocks left > 6 - - st1 { v9.16b}, [x2], #16 //AES final-6 block - store result - - rev64 v8.16b, v9.16b //GHASH final-6 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - ldr q9, [x0], #16 //AES final-5 block - load plaintext - - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid -.inst 0xce027529 //eor3 v9.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high -.L256_enc_blocks_more_than_5: //blocks left > 5 - - st1 { v9.16b}, [x2], #16 //AES final-5 block - store result - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - - ldr q9, [x0], #16 //AES final-4 block - load plaintext - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid -.inst 0xce037529 //eor3 v9.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result -.L256_enc_blocks_more_than_4: //blocks left > 4 - - st1 { v9.16b}, [x2], #16 //AES final-4 block - store result - - rev64 v8.16b, v9.16b //GHASH final-4 block - - ldr q9, [x0], #16 //AES final-3 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - -.inst 0xce047529 //eor3 v9.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high -.L256_enc_blocks_more_than_3: //blocks left > 3 - - st1 { v9.16b}, [x2], #16 //AES final-3 block - store result - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - ldr q24, [x3, #96] //load h4k | h3k - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - ldr q9, [x0], #16 //AES final-2 block - load plaintext - - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - -.inst 0xce057529 //eor3 v9.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - movi v16.8b, #0 //suppress further partial tag feed in - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low -.L256_enc_blocks_more_than_2: //blocks left > 2 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - - st1 { v9.16b}, [x2], #16 //AES final-2 block - store result - - rev64 v8.16b, v9.16b //GHASH final-2 block - ldr q9, [x0], #16 //AES final-1 block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high -.inst 0xce067529 //eor3 v9.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high - - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low -.L256_enc_blocks_more_than_1: //blocks left > 1 - - st1 { v9.16b}, [x2], #16 //AES final-1 block - store result - - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-1 block - ldr q9, [x0], #16 //AES final block - load plaintext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - -.inst 0xce077529 //eor3 v9.16b, v9.16b, v7.16b, v29.16b //AES final block - result - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high - - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - - ldr q21, [x3, #48] //load h2k | h1k - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid -.L256_enc_blocks_less_than_1: //blocks left <= 1 - - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - mvn x6, xzr //temp0_x = 0xffffffffffffffff - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - mvn x7, xzr //temp1_x = 0xffffffffffffffff - - csel x14, x6, xzr, lt - csel x13, x7, x6, lt - - mov v0.d[0], x13 //ctr0b is mask for last block - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - mov v0.d[1], x14 - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - - rev64 v8.16b, v9.16b //GHASH final block - - rev32 v30.16b, v30.16b - bif v9.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - str q30, [x16] //store the updated counter - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - st1 { v9.16b}, [x2] //store all 16B - - ins v16.d[0], v8.d[1] //GHASH final block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - -.inst 0xce115673 //eor3 v19.16b, v19.16b, v17.16b, v21.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - mov x0, x9 //return sizes - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -.L256_enc_ret: - mov w0, #0x0 - ret -.size unroll8_eor3_aes_gcm_enc_256_kernel,.-unroll8_eor3_aes_gcm_enc_256_kernel -.globl unroll8_eor3_aes_gcm_dec_256_kernel -.type unroll8_eor3_aes_gcm_dec_256_kernel,%function -.align 4 -unroll8_eor3_aes_gcm_dec_256_kernel: - AARCH64_VALID_CALL_TARGET - cbz x1, .L256_dec_ret - stp d8, d9, [sp, #-80]! - lsr x9, x1, #3 - mov x16, x4 - mov x8, x5 - stp d10, d11, [sp, #16] - stp d12, d13, [sp, #32] - stp d14, d15, [sp, #48] - mov x5, #0xc200000000000000 - stp x5, xzr, [sp, #64] - add x10, sp, #64 - - ld1 { v0.16b}, [x16] //CTR block 0 - - mov x15, #0x100000000 //set up counter increment - movi v31.16b, #0x0 - mov v31.d[1], x15 - mov x5, x9 - - sub x5, x5, #1 //byte_len - 1 - - rev32 v30.16b, v0.16b //set up reversed counter - - add v30.4s, v30.4s, v31.4s //CTR block 0 - - rev32 v1.16b, v30.16b //CTR block 1 - add v30.4s, v30.4s, v31.4s //CTR block 1 - - rev32 v2.16b, v30.16b //CTR block 2 - add v30.4s, v30.4s, v31.4s //CTR block 2 - ldp q26, q27, [x8, #0] //load rk0, rk1 - - rev32 v3.16b, v30.16b //CTR block 3 - add v30.4s, v30.4s, v31.4s //CTR block 3 - - rev32 v4.16b, v30.16b //CTR block 4 - add v30.4s, v30.4s, v31.4s //CTR block 4 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 0 - - rev32 v5.16b, v30.16b //CTR block 5 - add v30.4s, v30.4s, v31.4s //CTR block 5 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 0 - - rev32 v6.16b, v30.16b //CTR block 6 - add v30.4s, v30.4s, v31.4s //CTR block 6 - - rev32 v7.16b, v30.16b //CTR block 7 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 0 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 0 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 1 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 1 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 1 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 1 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 1 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 1 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 1 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 2 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 2 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 2 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 2 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 2 - ldp q27, q28, [x8, #64] //load rk4, rk5 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 3 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 3 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 3 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 3 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 3 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 3 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 3 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 3 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 4 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 4 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 4 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 4 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 4 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 4 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 5 - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 5 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 5 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 5 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 5 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 6 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 6 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 6 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 7 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 7 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 7 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 7 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 7 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 7 - - and x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 8 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 8 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 8 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 8 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 8 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 9 - - ld1 { v19.16b}, [x3] - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - ldp q27, q28, [x8, #160] //load rk10, rk11 - add x4, x0, x1, lsr #3 //end_input_ptr - add x5, x5, x0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 9 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 9 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 9 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 9 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 9 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 4 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 7 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 5 - round 10 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 1 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 2 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 0 - round 10 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 6 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 3 - round 10 - ldp q26, q27, [x8, #192] //load rk12, rk13 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 0 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 7 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 7 - round 11 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 3 - round 11 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 1 - round 11 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 5 - round 11 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 4 - round 11 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 2 - round 11 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 6 - round 11 - ldr q28, [x8, #224] //load rk14 - - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 1 - round 12 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 4 - round 12 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 5 - round 12 - - cmp x0, x5 //check if we have <= 8 blocks - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 3 - round 12 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 2 - round 12 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 6 - round 12 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 0 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 7 - round 12 - - aese v5.16b, v27.16b //AES block 5 - round 13 - aese v1.16b, v27.16b //AES block 1 - round 13 - aese v2.16b, v27.16b //AES block 2 - round 13 - - aese v0.16b, v27.16b //AES block 0 - round 13 - aese v4.16b, v27.16b //AES block 4 - round 13 - aese v6.16b, v27.16b //AES block 6 - round 13 - - aese v3.16b, v27.16b //AES block 3 - round 13 - aese v7.16b, v27.16b //AES block 7 - round 13 - b.ge .L256_dec_tail //handle tail - - ldp q8, q9, [x0], #32 //AES block 0, 1 - load ciphertext - - ldp q10, q11, [x0], #32 //AES block 2, 3 - load ciphertext - - ldp q12, q13, [x0], #32 //AES block 4, 5 - load ciphertext - - ldp q14, q15, [x0], #32 //AES block 6, 7 - load ciphertext - cmp x0, x5 //check if we have <= 8 blocks - -.inst 0xce017121 //eor3 v1.16b, v9.16b, v1.16b, v28.16b //AES block 1 - result -.inst 0xce007100 //eor3 v0.16b, v8.16b, v0.16b, v28.16b //AES block 0 - result - stp q0, q1, [x2], #32 //AES block 0, 1 - store result - - rev32 v0.16b, v30.16b //CTR block 8 - add v30.4s, v30.4s, v31.4s //CTR block 8 -.inst 0xce037163 //eor3 v3.16b, v11.16b, v3.16b, v28.16b //AES block 3 - result - -.inst 0xce0571a5 //eor3 v5.16b, v13.16b, v5.16b, v28.16b //AES block 5 - result - -.inst 0xce047184 //eor3 v4.16b, v12.16b, v4.16b, v28.16b //AES block 4 - result - rev32 v1.16b, v30.16b //CTR block 9 - add v30.4s, v30.4s, v31.4s //CTR block 9 - -.inst 0xce027142 //eor3 v2.16b, v10.16b, v2.16b, v28.16b //AES block 2 - result - stp q2, q3, [x2], #32 //AES block 2, 3 - store result - - rev32 v2.16b, v30.16b //CTR block 10 - add v30.4s, v30.4s, v31.4s //CTR block 10 - -.inst 0xce0671c6 //eor3 v6.16b, v14.16b, v6.16b, v28.16b //AES block 6 - result - - rev32 v3.16b, v30.16b //CTR block 11 - add v30.4s, v30.4s, v31.4s //CTR block 11 - stp q4, q5, [x2], #32 //AES block 4, 5 - store result - -.inst 0xce0771e7 //eor3 v7.16b, v15.16b, v7.16b, v28.16b //AES block 7 - result - stp q6, q7, [x2], #32 //AES block 6, 7 - store result - - rev32 v4.16b, v30.16b //CTR block 12 - add v30.4s, v30.4s, v31.4s //CTR block 12 - b.ge .L256_dec_prepretail //do prepretail - -.L256_dec_main_loop: //main loop start - rev32 v5.16b, v30.16b //CTR block 8k+13 - ldp q26, q27, [x8, #0] //load rk0, rk1 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v9.16b, v9.16b //GHASH block 8k+1 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - - rev32 v6.16b, v30.16b //CTR block 8k+14 - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - rev64 v8.16b, v8.16b //GHASH block 8k - - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - rev64 v12.16b, v12.16b //GHASH block 8k+4 - rev64 v11.16b, v11.16b //GHASH block 8k+3 - - rev32 v7.16b, v30.16b //CTR block 8k+15 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - ldp q28, q26, [x8, #32] //load rk2, rk3 - - eor v8.16b, v8.16b, v19.16b //PRE 1 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - - ldp q26, q27, [x8, #96] //load rk6, rk7 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - rev64 v13.16b, v13.16b //GHASH block 8k+5 - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v14.16b, v14.16b //GHASH block 8k+6 - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - ldp q28, q26, [x8, #128] //load rk8, rk9 - - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - - ldp q27, q28, [x8, #160] //load rk10, rk11 - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - - ldp q8, q9, [x0], #32 //AES block 8k+8, 8k+9 - load ciphertext - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - rev32 v20.16b, v30.16b //CTR block 8k+16 - ldr d16, [x10] //MODULO - load modulo constant - - add v30.4s, v30.4s, v31.4s //CTR block 8k+16 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 - ldp q26, q27, [x8, #192] //load rk12, rk13 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 - -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - rev32 v22.16b, v30.16b //CTR block 8k+17 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 - - ldp q10, q11, [x0], #32 //AES block 8k+10, 8k+11 - load ciphertext - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 - add v30.4s, v30.4s, v31.4s //CTR block 8k+17 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 - - rev32 v23.16b, v30.16b //CTR block 8k+18 - add v30.4s, v30.4s, v31.4s //CTR block 8k+18 - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 - - ldr q28, [x8, #224] //load rk14 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 - - ldp q12, q13, [x0], #32 //AES block 8k+12, 8k+13 - load ciphertext - aese v1.16b, v27.16b //AES block 8k+9 - round 13 - aese v2.16b, v27.16b //AES block 8k+10 - round 13 - - ldp q14, q15, [x0], #32 //AES block 8k+14, 8k+15 - load ciphertext - aese v0.16b, v27.16b //AES block 8k+8 - round 13 - aese v5.16b, v27.16b //AES block 8k+13 - round 13 - - rev32 v25.16b, v30.16b //CTR block 8k+19 -.inst 0xce027142 //eor3 v2.16b, v10.16b, v2.16b, v28.16b //AES block 8k+10 - result -.inst 0xce017121 //eor3 v1.16b, v9.16b, v1.16b, v28.16b //AES block 8k+9 - result - - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - aese v7.16b, v27.16b //AES block 8k+15 - round 13 - - add v30.4s, v30.4s, v31.4s //CTR block 8k+19 - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v4.16b, v27.16b //AES block 8k+12 - round 13 - -.inst 0xce0571a5 //eor3 v5.16b, v13.16b, v5.16b, v28.16b //AES block 8k+13 - result -.inst 0xce007100 //eor3 v0.16b, v8.16b, v0.16b, v28.16b //AES block 8k+8 - result - aese v3.16b, v27.16b //AES block 8k+11 - round 13 - - stp q0, q1, [x2], #32 //AES block 8k+8, 8k+9 - store result - mov v0.16b, v20.16b //CTR block 8k+16 -.inst 0xce047184 //eor3 v4.16b, v12.16b, v4.16b, v28.16b //AES block 8k+12 - result - -.inst 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low -.inst 0xce037163 //eor3 v3.16b, v11.16b, v3.16b, v28.16b //AES block 8k+11 - result - stp q2, q3, [x2], #32 //AES block 8k+10, 8k+11 - store result - - mov v3.16b, v25.16b //CTR block 8k+19 - mov v2.16b, v23.16b //CTR block 8k+18 - aese v6.16b, v27.16b //AES block 8k+14 - round 13 - - mov v1.16b, v22.16b //CTR block 8k+17 - stp q4, q5, [x2], #32 //AES block 8k+12, 8k+13 - store result -.inst 0xce0771e7 //eor3 v7.16b, v15.16b, v7.16b, v28.16b //AES block 8k+15 - result - -.inst 0xce0671c6 //eor3 v6.16b, v14.16b, v6.16b, v28.16b //AES block 8k+14 - result - rev32 v4.16b, v30.16b //CTR block 8k+20 - add v30.4s, v30.4s, v31.4s //CTR block 8k+20 - - cmp x0, x5 //.LOOP CONTROL - stp q6, q7, [x2], #32 //AES block 8k+14, 8k+15 - store result - b.lt .L256_dec_main_loop - -.L256_dec_prepretail: //PREPRETAIL - ldp q26, q27, [x8, #0] //load rk0, rk1 - rev32 v5.16b, v30.16b //CTR block 8k+13 - add v30.4s, v30.4s, v31.4s //CTR block 8k+13 - - rev64 v12.16b, v12.16b //GHASH block 8k+4 - ldr q21, [x3, #144] //load h6k | h5k - ldr q24, [x3, #192] //load h8k | h7k - - rev32 v6.16b, v30.16b //CTR block 8k+14 - rev64 v8.16b, v8.16b //GHASH block 8k - add v30.4s, v30.4s, v31.4s //CTR block 8k+14 - - ext v19.16b, v19.16b, v19.16b, #8 //PRE 0 - ldr q23, [x3, #176] //load h7l | h7h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #208] //load h8l | h8h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v9.16b, v9.16b //GHASH block 8k+1 - - rev32 v7.16b, v30.16b //CTR block 8k+15 - rev64 v10.16b, v10.16b //GHASH block 8k+2 - ldr q20, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 0 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 0 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 0 - - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 0 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 0 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 0 - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 1 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 0 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 0 - - ldp q28, q26, [x8, #32] //load rk2, rk3 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 1 - eor v8.16b, v8.16b, v19.16b //PRE 1 - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 1 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 1 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 1 - - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 1 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 1 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 1 - - pmull2 v16.1q, v9.2d, v23.2d //GHASH block 8k+1 - high - trn1 v18.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - pmull v19.1q, v8.1d, v25.1d //GHASH block 8k - low - - rev64 v11.16b, v11.16b //GHASH block 8k+3 - pmull v23.1q, v9.1d, v23.1d //GHASH block 8k+1 - low - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 2 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 2 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 2 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 2 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 2 - pmull2 v17.1q, v8.2d, v25.2d //GHASH block 8k - high - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 2 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 3 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 3 - rev64 v14.16b, v14.16b //GHASH block 8k+6 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 3 - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 2 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 3 - - pmull2 v29.1q, v10.2d, v22.2d //GHASH block 8k+2 - high - trn2 v8.2d, v9.2d, v8.2d //GHASH block 8k, 8k+1 - mid - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 2 - - ldp q27, q28, [x8, #64] //load rk4, rk5 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 3 - pmull2 v9.1q, v11.2d, v20.2d //GHASH block 8k+3 - high - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 3 - eor v17.16b, v17.16b, v16.16b //GHASH block 8k+1 - high - eor v8.16b, v8.16b, v18.16b //GHASH block 8k, 8k+1 - mid - - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 3 - pmull v22.1q, v10.1d, v22.1d //GHASH block 8k+2 - low - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 3 - -.inst 0xce1d2631 //eor3 v17.16b, v17.16b, v29.16b, v9.16b //GHASH block 8k+2, 8k+3 - high - trn1 v29.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - trn2 v10.2d, v11.2d, v10.2d //GHASH block 8k+2, 8k+3 - mid - - pmull2 v18.1q, v8.2d, v24.2d //GHASH block 8k - mid - pmull v20.1q, v11.1d, v20.1d //GHASH block 8k+3 - low - eor v19.16b, v19.16b, v23.16b //GHASH block 8k+1 - low - - pmull v24.1q, v8.1d, v24.1d //GHASH block 8k+1 - mid - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 4 - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 4 - -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+2, 8k+3 - low - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 4 - - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 4 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 4 - eor v18.16b, v18.16b, v24.16b //GHASH block 8k+1 - mid - - eor v10.16b, v10.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 5 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 4 - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 5 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 4 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 4 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 5 - pmull2 v29.1q, v10.2d, v21.2d //GHASH block 8k+2 - mid - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 5 - - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 5 - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 5 - pmull v21.1q, v10.1d, v21.1d //GHASH block 8k+3 - mid - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 5 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 5 - ldp q26, q27, [x8, #96] //load rk6, rk7 - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v15.16b, v15.16b //GHASH block 8k+7 - rev64 v13.16b, v13.16b //GHASH block 8k+5 - -.inst 0xce157652 //eor3 v18.16b, v18.16b, v21.16b, v29.16b //GHASH block 8k+2, 8k+3 - mid - - trn1 v16.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 6 - ldr q21, [x3, #48] //load h2k | h1k - ldr q24, [x3, #96] //load h4k | h3k - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 6 - - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 6 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 6 - - pmull2 v8.1q, v12.2d, v25.2d //GHASH block 8k+4 - high - pmull2 v10.1q, v13.2d, v23.2d //GHASH block 8k+5 - high - pmull v25.1q, v12.1d, v25.1d //GHASH block 8k+4 - low - - trn2 v12.2d, v13.2d, v12.2d //GHASH block 8k+4, 8k+5 - mid - pmull v23.1q, v13.1d, v23.1d //GHASH block 8k+5 - low - trn1 v13.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 7 - pmull2 v11.1q, v14.2d, v22.2d //GHASH block 8k+6 - high - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 6 - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 6 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 6 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 6 - - ldp q28, q26, [x8, #128] //load rk8, rk9 - pmull v22.1q, v14.1d, v22.1d //GHASH block 8k+6 - low - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 7 - - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 7 - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 7 - - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 7 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 7 -.inst 0xce082a31 //eor3 v17.16b, v17.16b, v8.16b, v10.16b //GHASH block 8k+4, 8k+5 - high - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 7 - trn2 v14.2d, v15.2d, v14.2d //GHASH block 8k+6, 8k+7 - mid - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 7 - - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 8 - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 8 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 8 - - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 8 - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 8 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 8 - - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 8 - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 9 - eor v12.16b, v12.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 9 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 9 - eor v14.16b, v14.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 9 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 9 - pmull2 v16.1q, v12.2d, v24.2d //GHASH block 8k+4 - mid - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 8 - pmull v24.1q, v12.1d, v24.1d //GHASH block 8k+5 - mid - pmull2 v12.1q, v15.2d, v20.2d //GHASH block 8k+7 - high - - pmull2 v13.1q, v14.2d, v21.2d //GHASH block 8k+6 - mid - pmull v21.1q, v14.1d, v21.1d //GHASH block 8k+7 - mid - pmull v20.1q, v15.1d, v20.1d //GHASH block 8k+7 - low - - ldp q27, q28, [x8, #160] //load rk10, rk11 -.inst 0xce195e73 //eor3 v19.16b, v19.16b, v25.16b, v23.16b //GHASH block 8k+4, 8k+5 - low -.inst 0xce184252 //eor3 v18.16b, v18.16b, v24.16b, v16.16b //GHASH block 8k+4, 8k+5 - mid - - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 9 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 9 - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 9 - -.inst 0xce0b3231 //eor3 v17.16b, v17.16b, v11.16b, v12.16b //GHASH block 8k+6, 8k+7 - high -.inst 0xce165273 //eor3 v19.16b, v19.16b, v22.16b, v20.16b //GHASH block 8k+6, 8k+7 - low - ldr d16, [x10] //MODULO - load modulo constant - -.inst 0xce153652 //eor3 v18.16b, v18.16b, v21.16b, v13.16b //GHASH block 8k+6, 8k+7 - mid - - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 10 - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 10 - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 10 - - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 10 - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 10 - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 10 - -.inst 0xce114e52 //eor3 v18.16b, v18.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 10 - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 10 - ldp q26, q27, [x8, #192] //load rk12, rk13 - - ext v21.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 11 - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 11 - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 11 - - pmull v29.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 11 - - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 11 - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 11 - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 11 - - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 11 - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b //AES block 8k+11 - round 12 - -.inst 0xce1d5652 //eor3 v18.16b, v18.16b, v29.16b, v21.16b //MODULO - fold into mid - - aese v3.16b, v27.16b //AES block 8k+11 - round 13 - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b //AES block 8k+10 - round 12 - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b //AES block 8k+14 - round 12 - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b //AES block 8k+12 - round 12 - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b //AES block 8k+15 - round 12 - - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b //AES block 8k+8 - round 12 - ldr q28, [x8, #224] //load rk14 - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b //AES block 8k+9 - round 12 - - aese v4.16b, v27.16b //AES block 8k+12 - round 13 - ext v21.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b //AES block 8k+13 - round 12 - - aese v6.16b, v27.16b //AES block 8k+14 - round 13 - aese v2.16b, v27.16b //AES block 8k+10 - round 13 - aese v1.16b, v27.16b //AES block 8k+9 - round 13 - - aese v5.16b, v27.16b //AES block 8k+13 - round 13 -.inst 0xce154673 //eor3 v19.16b, v19.16b, v21.16b, v17.16b //MODULO - fold into low - add v30.4s, v30.4s, v31.4s //CTR block 8k+15 - - aese v7.16b, v27.16b //AES block 8k+15 - round 13 - aese v0.16b, v27.16b //AES block 8k+8 - round 13 -.L256_dec_tail: //TAIL - - ext v16.16b, v19.16b, v19.16b, #8 //prepare final partial tag - sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process - cmp x5, #112 - - ldr q9, [x0], #16 //AES block 8k+8 - load ciphertext - - ldp q24, q25, [x3, #192] //load h8k | h7k - ext v25.16b, v25.16b, v25.16b, #8 - mov v29.16b, v28.16b - - ldp q20, q21, [x3, #128] //load h5l | h5h - ext v20.16b, v20.16b, v20.16b, #8 - -.inst 0xce00752c //eor3 v12.16b, v9.16b, v0.16b, v29.16b //AES block 8k+8 - result - ldp q22, q23, [x3, #160] //load h6l | h6h - ext v22.16b, v22.16b, v22.16b, #8 - ext v23.16b, v23.16b, v23.16b, #8 - b.gt .L256_dec_blocks_more_than_7 - - mov v7.16b, v6.16b - sub v30.4s, v30.4s, v31.4s - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - mov v4.16b, v3.16b - movi v19.8b, #0 - - movi v17.8b, #0 - movi v18.8b, #0 - mov v3.16b, v2.16b - - cmp x5, #96 - mov v2.16b, v1.16b - b.gt .L256_dec_blocks_more_than_6 - - mov v7.16b, v6.16b - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - cmp x5, #80 - sub v30.4s, v30.4s, v31.4s - - mov v4.16b, v3.16b - mov v3.16b, v1.16b - b.gt .L256_dec_blocks_more_than_5 - - cmp x5, #64 - mov v7.16b, v6.16b - sub v30.4s, v30.4s, v31.4s - - mov v6.16b, v5.16b - - mov v5.16b, v4.16b - mov v4.16b, v1.16b - b.gt .L256_dec_blocks_more_than_4 - - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - cmp x5, #48 - - mov v6.16b, v5.16b - mov v5.16b, v1.16b - b.gt .L256_dec_blocks_more_than_3 - - ldr q24, [x3, #96] //load h4k | h3k - sub v30.4s, v30.4s, v31.4s - mov v7.16b, v6.16b - - cmp x5, #32 - mov v6.16b, v1.16b - b.gt .L256_dec_blocks_more_than_2 - - sub v30.4s, v30.4s, v31.4s - - mov v7.16b, v1.16b - cmp x5, #16 - b.gt .L256_dec_blocks_more_than_1 - - sub v30.4s, v30.4s, v31.4s - ldr q21, [x3, #48] //load h2k | h1k - b .L256_dec_blocks_less_than_1 -.L256_dec_blocks_more_than_7: //blocks left > 7 - rev64 v8.16b, v9.16b //GHASH final-7 block - ldr q9, [x0], #16 //AES final-6 block - load ciphertext - st1 { v12.16b}, [x2], #16 //AES final-7 block - store result - - ins v18.d[0], v24.d[1] //GHASH final-7 block - mid - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-7 block - mid -.inst 0xce01752c //eor3 v12.16b, v9.16b, v1.16b, v29.16b //AES final-6 block - result - - pmull2 v17.1q, v8.2d, v25.2d //GHASH final-7 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-7 block - mid - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v19.1q, v8.1d, v25.1d //GHASH final-7 block - low - pmull v18.1q, v27.1d, v18.1d //GHASH final-7 block - mid -.L256_dec_blocks_more_than_6: //blocks left > 6 - - rev64 v8.16b, v9.16b //GHASH final-6 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - ldr q9, [x0], #16 //AES final-5 block - load ciphertext - movi v16.8b, #0 //suppress further partial tag feed in - - ins v27.d[0], v8.d[1] //GHASH final-6 block - mid - st1 { v12.16b}, [x2], #16 //AES final-6 block - store result - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-6 block - high - - pmull v26.1q, v8.1d, v23.1d //GHASH final-6 block - low - -.inst 0xce02752c //eor3 v12.16b, v9.16b, v2.16b, v29.16b //AES final-5 block - result - eor v19.16b, v19.16b, v26.16b //GHASH final-6 block - low - eor v27.8b, v27.8b, v8.8b //GHASH final-6 block - mid - - pmull v27.1q, v27.1d, v24.1d //GHASH final-6 block - mid - - eor v18.16b, v18.16b, v27.16b //GHASH final-6 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-6 block - high -.L256_dec_blocks_more_than_5: //blocks left > 5 - - rev64 v8.16b, v9.16b //GHASH final-5 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-5 block - high - ins v27.d[0], v8.d[1] //GHASH final-5 block - mid - - ldr q9, [x0], #16 //AES final-4 block - load ciphertext - - eor v27.8b, v27.8b, v8.8b //GHASH final-5 block - mid - st1 { v12.16b}, [x2], #16 //AES final-5 block - store result - - pmull v26.1q, v8.1d, v22.1d //GHASH final-5 block - low - ins v27.d[1], v27.d[0] //GHASH final-5 block - mid - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-5 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-5 block - high -.inst 0xce03752c //eor3 v12.16b, v9.16b, v3.16b, v29.16b //AES final-4 block - result - eor v19.16b, v19.16b, v26.16b //GHASH final-5 block - low - - eor v18.16b, v18.16b, v27.16b //GHASH final-5 block - mid - movi v16.8b, #0 //suppress further partial tag feed in -.L256_dec_blocks_more_than_4: //blocks left > 4 - - rev64 v8.16b, v9.16b //GHASH final-4 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-4 block - mid - ldr q9, [x0], #16 //AES final-3 block - load ciphertext - - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v26.1q, v8.1d, v20.1d //GHASH final-4 block - low - pmull2 v28.1q, v8.2d, v20.2d //GHASH final-4 block - high - - eor v27.8b, v27.8b, v8.8b //GHASH final-4 block - mid - - eor v17.16b, v17.16b, v28.16b //GHASH final-4 block - high - - pmull v27.1q, v27.1d, v21.1d //GHASH final-4 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-4 block - low - st1 { v12.16b}, [x2], #16 //AES final-4 block - store result - - eor v18.16b, v18.16b, v27.16b //GHASH final-4 block - mid -.inst 0xce04752c //eor3 v12.16b, v9.16b, v4.16b, v29.16b //AES final-3 block - result -.L256_dec_blocks_more_than_3: //blocks left > 3 - - ldr q25, [x3, #112] //load h4l | h4h - ext v25.16b, v25.16b, v25.16b, #8 - rev64 v8.16b, v9.16b //GHASH final-3 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - ldr q9, [x0], #16 //AES final-2 block - load ciphertext - ldr q24, [x3, #96] //load h4k | h3k - - ins v27.d[0], v8.d[1] //GHASH final-3 block - mid - st1 { v12.16b}, [x2], #16 //AES final-3 block - store result - -.inst 0xce05752c //eor3 v12.16b, v9.16b, v5.16b, v29.16b //AES final-2 block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-3 block - mid - - ins v27.d[1], v27.d[0] //GHASH final-3 block - mid - pmull v26.1q, v8.1d, v25.1d //GHASH final-3 block - low - pmull2 v28.1q, v8.2d, v25.2d //GHASH final-3 block - high - - movi v16.8b, #0 //suppress further partial tag feed in - pmull2 v27.1q, v27.2d, v24.2d //GHASH final-3 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-3 block - low - - eor v17.16b, v17.16b, v28.16b //GHASH final-3 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-3 block - mid -.L256_dec_blocks_more_than_2: //blocks left > 2 - - rev64 v8.16b, v9.16b //GHASH final-2 block - - ldr q23, [x3, #80] //load h3l | h3h - ext v23.16b, v23.16b, v23.16b, #8 - ldr q9, [x0], #16 //AES final-1 block - load ciphertext - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-2 block - mid - - pmull v26.1q, v8.1d, v23.1d //GHASH final-2 block - low - st1 { v12.16b}, [x2], #16 //AES final-2 block - store result -.inst 0xce06752c //eor3 v12.16b, v9.16b, v6.16b, v29.16b //AES final-1 block - result - - eor v27.8b, v27.8b, v8.8b //GHASH final-2 block - mid - eor v19.16b, v19.16b, v26.16b //GHASH final-2 block - low - movi v16.8b, #0 //suppress further partial tag feed in - - pmull v27.1q, v27.1d, v24.1d //GHASH final-2 block - mid - pmull2 v28.1q, v8.2d, v23.2d //GHASH final-2 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-2 block - mid - eor v17.16b, v17.16b, v28.16b //GHASH final-2 block - high -.L256_dec_blocks_more_than_1: //blocks left > 1 - - rev64 v8.16b, v9.16b //GHASH final-1 block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v27.d[0], v8.d[1] //GHASH final-1 block - mid - ldr q22, [x3, #64] //load h2l | h2h - ext v22.16b, v22.16b, v22.16b, #8 - - eor v27.8b, v27.8b, v8.8b //GHASH final-1 block - mid - ldr q9, [x0], #16 //AES final block - load ciphertext - st1 { v12.16b}, [x2], #16 //AES final-1 block - store result - - ldr q21, [x3, #48] //load h2k | h1k - pmull v26.1q, v8.1d, v22.1d //GHASH final-1 block - low - - ins v27.d[1], v27.d[0] //GHASH final-1 block - mid - - eor v19.16b, v19.16b, v26.16b //GHASH final-1 block - low - -.inst 0xce07752c //eor3 v12.16b, v9.16b, v7.16b, v29.16b //AES final block - result - pmull2 v28.1q, v8.2d, v22.2d //GHASH final-1 block - high - - pmull2 v27.1q, v27.2d, v21.2d //GHASH final-1 block - mid - - movi v16.8b, #0 //suppress further partial tag feed in - eor v17.16b, v17.16b, v28.16b //GHASH final-1 block - high - - eor v18.16b, v18.16b, v27.16b //GHASH final-1 block - mid -.L256_dec_blocks_less_than_1: //blocks left <= 1 - - ld1 { v26.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored - mvn x6, xzr //temp0_x = 0xffffffffffffffff - and x1, x1, #127 //bit_length %= 128 - - sub x1, x1, #128 //bit_length -= 128 - rev32 v30.16b, v30.16b - str q30, [x16] //store the updated counter - - neg x1, x1 //bit_length = 128 - #bits in input (in range [1,128]) - - and x1, x1, #127 //bit_length %= 128 - - lsr x6, x6, x1 //temp0_x is mask for top 64b of last block - cmp x1, #64 - mvn x7, xzr //temp1_x = 0xffffffffffffffff - - csel x14, x6, xzr, lt - csel x13, x7, x6, lt - - mov v0.d[0], x13 //ctr0b is mask for last block - mov v0.d[1], x14 - - and v9.16b, v9.16b, v0.16b //possibly partial last block has zeroes in highest bits - ldr q20, [x3, #32] //load h1l | h1h - ext v20.16b, v20.16b, v20.16b, #8 - bif v12.16b, v26.16b, v0.16b //insert existing bytes in top end of result before storing - - rev64 v8.16b, v9.16b //GHASH final block - - eor v8.16b, v8.16b, v16.16b //feed in partial tag - - ins v16.d[0], v8.d[1] //GHASH final block - mid - pmull2 v28.1q, v8.2d, v20.2d //GHASH final block - high - - eor v16.8b, v16.8b, v8.8b //GHASH final block - mid - - pmull v26.1q, v8.1d, v20.1d //GHASH final block - low - eor v17.16b, v17.16b, v28.16b //GHASH final block - high - - pmull v16.1q, v16.1d, v21.1d //GHASH final block - mid - - eor v18.16b, v18.16b, v16.16b //GHASH final block - mid - ldr d16, [x10] //MODULO - load modulo constant - eor v19.16b, v19.16b, v26.16b //GHASH final block - low - - pmull v21.1q, v17.1d, v16.1d //MODULO - top 64b align with mid - eor v14.16b, v17.16b, v19.16b //MODULO - karatsuba tidy up - - ext v17.16b, v17.16b, v17.16b, #8 //MODULO - other top alignment - st1 { v12.16b}, [x2] //store all 16B - - eor v18.16b, v18.16b, v14.16b //MODULO - karatsuba tidy up - - eor v21.16b, v17.16b, v21.16b //MODULO - fold into mid - eor v18.16b, v18.16b, v21.16b //MODULO - fold into mid - - pmull v17.1q, v18.1d, v16.1d //MODULO - mid 64b align with low - - ext v18.16b, v18.16b, v18.16b, #8 //MODULO - other mid alignment - eor v19.16b, v19.16b, v17.16b //MODULO - fold into low - - eor v19.16b, v19.16b, v18.16b //MODULO - fold into low - ext v19.16b, v19.16b, v19.16b, #8 - rev64 v19.16b, v19.16b - st1 { v19.16b }, [x3] - mov x0, x9 - - ldp d10, d11, [sp, #16] - ldp d12, d13, [sp, #32] - ldp d14, d15, [sp, #48] - ldp d8, d9, [sp], #80 - ret - -.L256_dec_ret: - mov w0, #0x0 - ret -.size unroll8_eor3_aes_gcm_dec_256_kernel,.-unroll8_eor3_aes_gcm_dec_256_kernel -.byte 65,69,83,32,71,67,77,32,109,111,100,117,108,101,32,102,111,114,32,65,82,77,118,56,44,32,83,80,68,88,32,66,83,68,45,51,45,67,108,97,117,115,101,32,98,121,32,60,120,105,97,111,107,97,110,103,46,113,105,97,110,64,97,114,109,46,99,111,109,62,0 -.align 2 -.align 2 -#endif diff --git a/openssl/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8_64.S b/openssl/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8_64.S index fc4d5161d..3caabfa59 100644 --- a/openssl/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8_64.S +++ b/openssl/src/crypto/modes/gen/linux_arm64/aes-gcm-armv8_64.S @@ -7,7 +7,6 @@ .type aes_gcm_enc_128_kernel,%function .align 4 aes_gcm_enc_128_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, .L128_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -20,36 +19,28 @@ aes_gcm_enc_128_kernel: stp d14, d15, [sp, #96] ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif ldp x13, x14, [x8, #160] //load rk10 -#ifdef __AARCH64EB__ - ror x13, x13, #32 - ror x14, x14, #32 -#endif + ld1 {v11.16b}, [x3] ext v11.16b, v11.16b, v11.16b, #8 rev64 v11.16b, v11.16b lsr x5, x1, #3 //byte_len mov x15, x5 - ld1 {v18.4s}, [x8], #16 //load rk0 + ldr q27, [x8, #144] //load rk9 add x4, x0, x1, lsr #3 //end_input_ptr sub x5, x5, #1 //byte_len - 1 lsr x12, x11, #32 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif + fmov d1, x10 //CTR block 1 rev w12, w12 //rev_ctr32 add w12, w12, #1 //increment rev_ctr32 orr w11, w11, w11 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q18, [x8, #0] //load rk0 rev w9, w12 //CTR block 1 add w12, w12, #1 //CTR block 1 @@ -69,33 +60,30 @@ aes_gcm_enc_128_kernel: rev w9, w12 //CTR block 3 orr x9, x11, x9, lsl #32 //CTR block 3 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q19, [x8, #16] //load rk1 add w12, w12, #1 //CTR block 3 fmov v3.d[1], x9 //CTR block 3 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q20, [x8, #32] //load rk2 aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q26, [x8, #128] //load rk8 aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q21, [x8, #48] //load rk3 aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 @@ -103,11 +91,11 @@ aes_gcm_enc_128_kernel: aese v0.16b, v19.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q24, [x8, #96] //load rk6 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q25, [x8, #112] //load rk7 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 @@ -115,14 +103,12 @@ aes_gcm_enc_128_kernel: aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q23, [x8, #80] //load rk5 aese v1.16b, v20.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 @@ -139,7 +125,7 @@ aes_gcm_enc_128_kernel: aese v2.16b, v21.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q22, [x8, #64] //load rk4 aese v3.16b, v21.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 @@ -222,25 +208,13 @@ aes_gcm_enc_128_kernel: b.ge .L128_enc_tail //handle tail ldp x6, x7, [x0, #0] //AES block 0 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + ldp x21, x22, [x0, #32] //AES block 2 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + ldp x19, x20, [x0, #16] //AES block 1 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + ldp x23, x24, [x0, #48] //AES block 3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + eor x6, x6, x13 //AES block 0 - round 10 low eor x7, x7, x14 //AES block 0 - round 10 high @@ -305,10 +279,6 @@ aes_gcm_enc_128_kernel: .L128_enc_main_loop: //main loop start ldp x23, x24, [x0, #48] //AES block 4k+3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free) rev64 v6.16b, v6.16b //GHASH block 4k+2 (t0, t1, and t2 free) @@ -343,10 +313,7 @@ aes_gcm_enc_128_kernel: pmull2 v28.1q, v5.2d, v14.2d //GHASH block 4k+1 - high eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid ldp x6, x7, [x0, #0] //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + aese v0.16b, v19.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 1 rev w9, w12 //CTR block 4k+8 @@ -428,10 +395,7 @@ aes_gcm_enc_128_kernel: aese v1.16b, v23.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 5 ldp x19, x20, [x0, #16] //AES block 4k+5 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + aese v3.16b, v21.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 3 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid @@ -439,10 +403,7 @@ aes_gcm_enc_128_kernel: aese v0.16b, v23.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 5 ldp x21, x22, [x0, #32] //AES block 4k+6 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + pmull v31.1q, v9.1d, v8.1d //MODULO - top 64b align with mid eor v11.16b, v11.16b, v29.16b //GHASH block 4k+3 - low @@ -751,10 +712,7 @@ aes_gcm_enc_128_kernel: sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldp x6, x7, [x0], #16 //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + cmp x5, #48 ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag @@ -792,10 +750,7 @@ aes_gcm_enc_128_kernel: st1 { v5.16b}, [x2], #16 //AES final-3 block - store result ldp x6, x7, [x0], #16 //AES final-2 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-3 block eor v4.16b, v4.16b, v8.16b //feed in partial tag @@ -824,10 +779,7 @@ aes_gcm_enc_128_kernel: rev64 v4.16b, v5.16b //GHASH final-2 block ldp x6, x7, [x0], #16 //AES final-1 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor v4.16b, v4.16b, v8.16b //feed in partial tag eor x6, x6, x13 //AES final-1 block - round 10 low @@ -861,10 +813,7 @@ aes_gcm_enc_128_kernel: rev64 v4.16b, v5.16b //GHASH final-1 block ldp x6, x7, [x0], #16 //AES final block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor v4.16b, v4.16b, v8.16b //feed in partial tag eor x7, x7, x14 //AES final block - round 10 high @@ -927,11 +876,9 @@ aes_gcm_enc_128_kernel: ld1 { v18.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored eor v8.8b, v8.8b, v4.8b //GHASH final block - mid -#ifndef __AARCH64EB__ + rev w9, w12 -#else - mov w9, w12 -#endif + pmull2 v20.1q, v4.2d, v12.2d //GHASH final block - high pmull v8.1q, v8.1d, v16.1d //GHASH final block - mid @@ -990,7 +937,6 @@ aes_gcm_enc_128_kernel: .type aes_gcm_dec_128_kernel,%function .align 4 aes_gcm_dec_128_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, .L128_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -1005,29 +951,20 @@ aes_gcm_dec_128_kernel: lsr x5, x1, #3 //byte_len mov x15, x5 ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #160] //load rk10 -#ifdef __AARCH64EB__ - ror x14, x14, 32 - ror x13, x13, 32 -#endif + sub x5, x5, #1 //byte_len - 1 - ld1 {v18.4s}, [x8], #16 //load rk0 + ldr q18, [x8, #0] //load rk0 and x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) ld1 { v0.16b}, [x16] //special case vector load initial counter so we can start first AES block as quickly as possible ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif + lsr x12, x11, #32 fmov d2, x10 //CTR block 2 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q19, [x8, #16] //load rk1 orr w11, w11, w11 rev w12, w12 //rev_ctr32 @@ -1039,7 +976,7 @@ aes_gcm_dec_128_kernel: rev w9, w12 //CTR block 1 orr x9, x11, x9, lsl #32 //CTR block 1 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q20, [x8, #32] //load rk2 add w12, w12, #1 //CTR block 1 fmov v1.d[1], x9 //CTR block 1 @@ -1062,19 +999,19 @@ aes_gcm_dec_128_kernel: aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q21, [x8, #48] //load rk3 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q24, [x8, #96] //load rk6 aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q25, [x8, #112] //load rk7 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q22, [x8, #64] //load rk4 aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 @@ -1084,6 +1021,7 @@ aes_gcm_dec_128_kernel: aese v1.16b, v20.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 + ldp x13, x14, [x8, #160] //load rk10 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 @@ -1093,7 +1031,7 @@ aes_gcm_dec_128_kernel: aese v0.16b, v21.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q23, [x8, #80] //load rk5 aese v1.16b, v21.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 @@ -1103,7 +1041,7 @@ aes_gcm_dec_128_kernel: aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q27, [x8, #144] //load rk9 aese v1.16b, v22.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 @@ -1114,12 +1052,11 @@ aes_gcm_dec_128_kernel: aese v2.16b, v21.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v0.16b, v22.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q26, [x8, #128] //load rk8 aese v1.16b, v23.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 @@ -1136,9 +1073,8 @@ aes_gcm_dec_128_kernel: aese v2.16b, v23.16b aesmc v2.16b, v2.16b //AES block 2 - round 5 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v3.16b, v23.16b aesmc v3.16b, v3.16b //AES block 3 - round 5 @@ -1156,9 +1092,7 @@ aes_gcm_dec_128_kernel: trn1 v8.2d, v12.2d, v13.2d //h2h | h1h ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif trn2 v16.2d, v12.2d, v13.2d //h2l | h1l add x5, x5, x0 @@ -1200,10 +1134,12 @@ aes_gcm_dec_128_kernel: eor v17.16b, v17.16b, v9.16b //h4k | h3k b.ge .L128_dec_tail //handle tail - ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0 - load ciphertext; AES block 1 - load ciphertext + ldr q5, [x0, #16] //AES block 1 - load ciphertext + + ldr q4, [x0, #0] //AES block 0 - load ciphertext eor v1.16b, v5.16b, v1.16b //AES block 1 - result - ld1 {v6.16b}, [x0], #16 //AES block 2 - load ciphertext + ldr q6, [x0, #32] //AES block 2 - load ciphertext eor v0.16b, v4.16b, v0.16b //AES block 0 - result rev64 v4.16b, v4.16b //GHASH block 0 @@ -1211,9 +1147,10 @@ aes_gcm_dec_128_kernel: orr x9, x11, x9, lsl #32 //CTR block 4 add w12, w12, #1 //CTR block 4 - ld1 {v7.16b}, [x0], #16 //AES block 3 - load ciphertext + ldr q7, [x0, #48] //AES block 3 - load ciphertext rev64 v5.16b, v5.16b //GHASH block 1 + add x0, x0, #64 //AES input_ptr update mov x19, v1.d[0] //AES block 1 - mov low mov x20, v1.d[1] //AES block 1 - mov high @@ -1228,9 +1165,7 @@ aes_gcm_dec_128_kernel: fmov v0.d[1], x9 //CTR block 4 rev w9, w12 //CTR block 5 eor x19, x19, x13 //AES block 1 - round 10 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif + fmov d1, x10 //CTR block 5 add w12, w12, #1 //CTR block 5 orr x9, x11, x9, lsl #32 //CTR block 5 @@ -1242,19 +1177,10 @@ aes_gcm_dec_128_kernel: orr x9, x11, x9, lsl #32 //CTR block 6 eor x20, x20, x14 //AES block 1 - round 10 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif eor x6, x6, x13 //AES block 0 - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v2.16b, v6.16b, v2.16b //AES block 2 - result eor x7, x7, x14 //AES block 0 - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif stp x6, x7, [x2], #16 //AES block 0 - store result stp x19, x20, [x2], #16 //AES block 1 - store result @@ -1322,14 +1248,9 @@ aes_gcm_dec_128_kernel: aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 1 eor x23, x23, x13 //AES block 4k+3 - round 10 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + pmull v30.1q, v30.1d, v17.1d //GHASH block 4k+1 - mid eor x22, x22, x14 //AES block 4k+2 - round 10 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif mov d31, v6.d[1] //GHASH block 4k+2 - mid aese v0.16b, v19.16b @@ -1367,9 +1288,7 @@ aes_gcm_dec_128_kernel: pmull2 v31.1q, v31.2d, v16.2d //GHASH block 4k+2 - mid eor x24, x24, x14 //AES block 4k+3 - round 10 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif + aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 2 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid @@ -1377,9 +1296,7 @@ aes_gcm_dec_128_kernel: aese v1.16b, v23.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 5 eor x21, x21, x13 //AES block 4k+2 - round 10 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif + aese v0.16b, v23.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 5 movi v8.8b, #0xc2 @@ -1401,7 +1318,7 @@ aes_gcm_dec_128_kernel: pmull v30.1q, v30.1d, v16.1d //GHASH block 4k+3 - mid eor v9.16b, v9.16b, v4.16b //GHASH block 4k+3 - high - ld1 {v4.16b}, [x0], #16 //AES block 4k+3 - load ciphertext + ldr q4, [x0, #0] //AES block 4k+4 - load ciphertext aese v1.16b, v25.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 7 @@ -1428,7 +1345,7 @@ aes_gcm_dec_128_kernel: rev w9, w12 //CTR block 4k+8 pmull v31.1q, v9.1d, v8.1d //MODULO - top 64b align with mid - ld1 {v5.16b}, [x0], #16 //AES block 4k+4 - load ciphertext + ldr q5, [x0, #16] //AES block 4k+5 - load ciphertext ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment aese v0.16b, v27.16b //AES block 4k+4 - round 9 @@ -1446,7 +1363,7 @@ aes_gcm_dec_128_kernel: aese v3.16b, v23.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 5 - ld1 {v6.16b}, [x0], #16 //AES block 4k+5 - load ciphertext + ldr q6, [x0, #32] //AES block 4k+6 - load ciphertext add w12, w12, #1 //CTR block 4k+8 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid @@ -1454,10 +1371,11 @@ aes_gcm_dec_128_kernel: aese v2.16b, v25.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 7 - ld1 {v7.16b}, [x0], #16 //AES block 4k+6 - load ciphertext + ldr q7, [x0, #48] //AES block 4k+3 - load ciphertext aese v3.16b, v24.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 6 + add x0, x0, #64 //AES input_ptr update rev64 v5.16b, v5.16b //GHASH block 4k+5 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid @@ -1482,15 +1400,11 @@ aes_gcm_dec_128_kernel: aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 8 eor x7, x7, x14 //AES block 4k+4 - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor v11.16b, v11.16b, v8.16b //MODULO - fold into low mov x20, v1.d[1] //AES block 4k+5 - mov high eor x6, x6, x13 //AES block 4k+4 - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif + eor v2.16b, v6.16b, v2.16b //AES block 4k+6 - result mov x19, v1.d[0] //AES block 4k+5 - mov low add w12, w12, #1 //CTR block 4k+9 @@ -1507,15 +1421,9 @@ aes_gcm_dec_128_kernel: add w12, w12, #1 //CTR block 4k+10 eor x20, x20, x14 //AES block 4k+5 - round 10 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif stp x6, x7, [x2], #16 //AES block 4k+4 - store result eor x19, x19, x13 //AES block 4k+5 - round 10 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif stp x19, x20, [x2], #16 //AES block 4k+5 - store result orr x9, x11, x9, lsl #32 //CTR block 4k+10 @@ -1620,14 +1528,9 @@ aes_gcm_dec_128_kernel: aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 2 eor x23, x23, x13 //AES block 4k+3 - round 10 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + pmull v30.1q, v30.1d, v16.1d //GHASH block 4k+3 - mid eor x21, x21, x13 //AES block 4k+2 - round 10 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif eor v11.16b, v11.16b, v29.16b //GHASH block 4k+3 - low aese v2.16b, v21.16b @@ -1700,9 +1603,7 @@ aes_gcm_dec_128_kernel: pmull v8.1q, v10.1d, v8.1d //MODULO - mid 64b align with low eor x24, x24, x14 //AES block 4k+3 - round 10 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif + aese v2.16b, v25.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 7 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment @@ -1720,9 +1621,7 @@ aes_gcm_dec_128_kernel: aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 8 eor x22, x22, x14 //AES block 4k+2 - round 10 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif + aese v0.16b, v27.16b //AES block 4k+4 - round 9 stp x21, x22, [x2], #16 //AES block 4k+2 - store result @@ -1746,14 +1645,9 @@ aes_gcm_dec_128_kernel: cmp x5, #48 eor x7, x7, x14 //AES block 4k+4 - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag eor x6, x6, x13 //AES block 4k+4 - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif b.gt .L128_dec_blocks_more_than_3 mov v3.16b, v2.16b @@ -1797,14 +1691,9 @@ aes_gcm_dec_128_kernel: movi v8.8b, #0 //suppress further partial tag feed in eor x7, x7, x14 //AES final-2 block - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + pmull v10.1q, v22.1d, v10.1d //GHASH final-3 block - mid eor x6, x6, x13 //AES final-2 block - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif .L128_dec_blocks_more_than_2: //blocks left > 2 rev64 v4.16b, v5.16b //GHASH final-2 block @@ -1830,18 +1719,12 @@ aes_gcm_dec_128_kernel: pmull v22.1q, v22.1d, v17.1d //GHASH final-2 block - mid eor x6, x6, x13 //AES final-1 block - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v11.16b, v11.16b, v21.16b //GHASH final-2 block - low eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid eor x7, x7, x14 //AES final-1 block - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif .L128_dec_blocks_more_than_1: //blocks left > 1 rev64 v4.16b, v5.16b //GHASH final-1 block @@ -1872,13 +1755,8 @@ aes_gcm_dec_128_kernel: eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high eor x7, x7, x14 //AES final block - round 10 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES final block - round 10 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid .L128_dec_blocks_less_than_1: //blocks left <= 1 @@ -1924,11 +1802,7 @@ aes_gcm_dec_128_kernel: bic x4, x4, x9 //mask out low existing bytes and x6, x6, x9 -#ifndef __AARCH64EB__ rev w9, w12 -#else - mov w9, w12 -#endif eor v10.16b, v10.16b, v8.16b //GHASH final block - mid movi v8.8b, #0xc2 @@ -1983,7 +1857,6 @@ aes_gcm_dec_128_kernel: .type aes_gcm_enc_192_kernel,%function .align 4 aes_gcm_enc_192_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, .L192_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -1996,26 +1869,18 @@ aes_gcm_enc_192_kernel: stp d14, d15, [sp, #96] ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #192] //load rk12 -#ifdef __AARCH64EB__ - ror x13, x13, #32 - ror x14, x14, #32 -#endif - ld1 {v18.4s}, [x8], #16 //load rk0 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q23, [x8, #80] //load rk5 + + ldr q22, [x8, #64] //load rk4 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q26, [x8, #128] //load rk8 lsr x12, x11, #32 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q24, [x8, #96] //load rk6 orr w11, w11, w11 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q25, [x8, #112] //load rk7 rev w12, w12 //rev_ctr32 add w12, w12, #1 //increment rev_ctr32 @@ -2039,13 +1904,15 @@ aes_gcm_enc_192_kernel: rev w9, w12 //CTR block 3 orr x9, x11, x9, lsl #32 //CTR block 3 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q18, [x8, #0] //load rk0 fmov v3.d[1], x9 //CTR block 3 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q21, [x8, #48] //load rk3 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldp x13, x14, [x8, #192] //load rk12 + + ldr q19, [x8, #16] //load rk1 aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 @@ -2055,38 +1922,35 @@ aes_gcm_enc_192_kernel: aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q29, [x8, #176] //load rk11 aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif + aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q20, [x8, #32] //load rk2 aese v0.16b, v19.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 - ld1 {v28.4s}, [x8], #16 //load rk10 + ldr q28, [x8, #160] //load rk10 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 - ld1 {v29.4s}, [x8], #16 //load rk11 + ldr q27, [x8, #144] //load rk9 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 @@ -2143,9 +2007,8 @@ aes_gcm_enc_192_kernel: aese v2.16b, v24.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif + aese v1.16b, v24.16b aesmc v1.16b, v1.16b //AES block 1 - round 6 @@ -2225,26 +2088,13 @@ aes_gcm_enc_192_kernel: rev w9, w12 //CTR block 4 ldp x6, x7, [x0, #0] //AES block 0 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + orr x9, x11, x9, lsl #32 //CTR block 4 ldp x21, x22, [x0, #32] //AES block 2 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + ldp x23, x24, [x0, #48] //AES block 3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + ldp x19, x20, [x0, #16] //AES block 1 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif add x0, x0, #64 //AES input_ptr update cmp x0, x5 //check if we have <= 8 blocks @@ -2316,10 +2166,7 @@ aes_gcm_enc_192_kernel: aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 0 ldp x19, x20, [x0, #16] //AES block 4k+5 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + ext v11.16b, v11.16b, v11.16b, #8 //PRE 0 fmov d3, x10 //CTR block 4k+3 rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free) @@ -2331,17 +2178,11 @@ aes_gcm_enc_192_kernel: pmull2 v30.1q, v5.2d, v14.2d //GHASH block 4k+1 - high rev64 v7.16b, v7.16b //GHASH block 4k+3 (t0, t1, t2 and t3 free) ldp x21, x22, [x0, #32] //AES block 4k+6 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 0 ldp x23, x24, [x0, #48] //AES block 4k+3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + pmull v31.1q, v5.1d, v14.1d //GHASH block 4k+1 - low eor v4.16b, v4.16b, v11.16b //PRE 1 @@ -2434,10 +2275,7 @@ aes_gcm_enc_192_kernel: aese v1.16b, v22.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 4 ldp x6, x7, [x0, #0] //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + aese v0.16b, v24.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 6 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+2 - low @@ -2802,10 +2640,7 @@ aes_gcm_enc_192_kernel: sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldp x6, x7, [x0], #16 //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor x6, x6, x13 //AES block 4k+4 - round 12 low eor x7, x7, x14 //AES block 4k+4 - round 12 high @@ -2842,10 +2677,7 @@ aes_gcm_enc_192_kernel: st1 { v5.16b}, [x2], #16 //AES final-3 block - store result ldp x6, x7, [x0], #16 //AES final-2 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-3 block eor x6, x6, x13 //AES final-2 block - round 12 low @@ -2876,10 +2708,7 @@ aes_gcm_enc_192_kernel: rev64 v4.16b, v5.16b //GHASH final-2 block ldp x6, x7, [x0], #16 //AES final-1 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor v4.16b, v4.16b, v8.16b //feed in partial tag eor x7, x7, x14 //AES final-1 block - round 12 high @@ -2910,10 +2739,7 @@ aes_gcm_enc_192_kernel: st1 { v5.16b}, [x2], #16 //AES final-1 block - store result ldp x6, x7, [x0], #16 //AES final block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-1 block eor x6, x6, x13 //AES final block - round 12 low @@ -2945,11 +2771,7 @@ aes_gcm_enc_192_kernel: .L192_enc_blocks_less_than_1: //blocks left <= 1 ld1 { v18.16b}, [x2] //load existing bytes where the possibly partial last block is to be stored -#ifndef __AARCH64EB__ rev w9, w12 -#else - mov w9, w12 -#endif and x1, x1, #127 //bit_length %= 128 sub x1, x1, #128 //bit_length -= 128 @@ -3041,7 +2863,6 @@ aes_gcm_enc_192_kernel: .type aes_gcm_dec_192_kernel,%function .align 4 aes_gcm_dec_192_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, .L192_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -3055,22 +2876,14 @@ aes_gcm_dec_192_kernel: add x4, x0, x1, lsr #3 //end_input_ptr ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #192] //load rk12 -#ifdef __AARCH64EB__ - ror x13, x13, #32 - ror x14, x14, #32 -#endif + ld1 { v0.16b}, [x16] //special case vector load initial counter so we can start first AES block as quickly as possible - ld1 {v18.4s}, [x8], #16 //load rk0 + ldr q18, [x8, #0] //load rk0 lsr x5, x1, #3 //byte_len mov x15, x5 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q20, [x8, #32] //load rk2 lsr x12, x11, #32 orr w11, w11, w11 @@ -3080,7 +2893,7 @@ aes_gcm_dec_192_kernel: fmov d1, x10 //CTR block 1 add w12, w12, #1 //increment rev_ctr32 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q19, [x8, #16] //load rk1 aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 @@ -3088,7 +2901,7 @@ aes_gcm_dec_192_kernel: add w12, w12, #1 //CTR block 1 orr x9, x11, x9, lsl #32 //CTR block 1 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q21, [x8, #48] //load rk3 fmov v1.d[1], x9 //CTR block 1 rev w9, w12 //CTR block 2 @@ -3106,57 +2919,54 @@ aes_gcm_dec_192_kernel: fmov v3.d[1], x9 //CTR block 3 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q26, [x8, #128] //load rk8 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q29, [x8, #176] //load rk11 aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif + aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 + ldp x13, x14, [x8, #192] //load rk12 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q28, [x8, #160] //load rk10 aese v0.16b, v21.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q27, [x8, #144] //load rk9 aese v1.16b, v20.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q25, [x8, #112] //load rk7 aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q22, [x8, #64] //load rk4 aese v2.16b, v21.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 @@ -3174,7 +2984,7 @@ aes_gcm_dec_192_kernel: aese v0.16b, v22.16b aesmc v0.16b, v0.16b //AES block 0 - round 4 - ld1 {v28.4s}, [x8], #16 //load rk10 + ldr q23, [x8, #80] //load rk5 aese v1.16b, v22.16b aesmc v1.16b, v1.16b //AES block 1 - round 4 @@ -3189,7 +2999,7 @@ aes_gcm_dec_192_kernel: aese v0.16b, v23.16b aesmc v0.16b, v0.16b //AES block 0 - round 5 - ld1 {v29.4s}, [x8], #16 //load rk11 + ldr q24, [x8, #96] //load rk6 aese v1.16b, v23.16b aesmc v1.16b, v1.16b //AES block 1 - round 5 @@ -3276,13 +3086,17 @@ aes_gcm_dec_192_kernel: aese v0.16b, v29.16b //AES block 0 - round 11 b.ge .L192_dec_tail //handle tail - ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0,1 - load ciphertext + ldr q5, [x0, #16] //AES block 1 - load ciphertext + + ldr q4, [x0, #0] //AES block 0 - load ciphertext eor v1.16b, v5.16b, v1.16b //AES block 1 - result eor v0.16b, v4.16b, v0.16b //AES block 0 - result rev w9, w12 //CTR block 4 - ld1 {v6.16b, v7.16b}, [x0], #32 //AES block 2,3 - load ciphertext + ldr q7, [x0, #48] //AES block 3 - load ciphertext + + ldr q6, [x0, #32] //AES block 2 - load ciphertext mov x19, v1.d[0] //AES block 1 - mov low @@ -3294,35 +3108,27 @@ aes_gcm_dec_192_kernel: mov x7, v0.d[1] //AES block 0 - mov high rev64 v4.16b, v4.16b //GHASH block 0 + add x0, x0, #64 //AES input_ptr update fmov d0, x10 //CTR block 4 rev64 v5.16b, v5.16b //GHASH block 1 cmp x0, x5 //check if we have <= 8 blocks eor x19, x19, x13 //AES block 1 - round 12 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif fmov v0.d[1], x9 //CTR block 4 rev w9, w12 //CTR block 5 orr x9, x11, x9, lsl #32 //CTR block 5 fmov d1, x10 //CTR block 5 eor x20, x20, x14 //AES block 1 - round 12 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif + add w12, w12, #1 //CTR block 5 fmov v1.d[1], x9 //CTR block 5 eor x6, x6, x13 //AES block 0 - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif + rev w9, w12 //CTR block 6 eor x7, x7, x14 //AES block 0 - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + stp x6, x7, [x2], #16 //AES block 0 - store result orr x9, x11, x9, lsl #32 //CTR block 6 @@ -3385,9 +3191,7 @@ aes_gcm_dec_192_kernel: aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 2 eor x22, x22, x14 //AES block 4k+2 - round 12 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 1 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid @@ -3404,9 +3208,7 @@ aes_gcm_dec_192_kernel: pmull v4.1q, v4.1d, v17.1d //GHASH block 4k+1 - mid eor v11.16b, v11.16b, v31.16b //GHASH block 4k+1 - low eor x21, x21, x13 //AES block 4k+2 - round 12 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif + aese v1.16b, v22.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 4 @@ -3508,18 +3310,16 @@ aes_gcm_dec_192_kernel: aese v2.16b, v24.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 6 - ld1 {v4.16b}, [x0], #16 //AES block 4k+4 - load ciphertext + ldr q6, [x0, #32] //AES block 4k+6 - load ciphertext aese v3.16b, v24.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 6 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up pmull v31.1q, v9.1d, v8.1d //MODULO - top 64b align with mid - ld1 {v5.16b}, [x0], #16 //AES block 4k+5 - load ciphertext + ldr q7, [x0, #48] //AES block 4k+7 - load ciphertext eor x23, x23, x13 //AES block 4k+3 - round 12 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + aese v2.16b, v25.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 7 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment @@ -3533,10 +3333,10 @@ aes_gcm_dec_192_kernel: aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 8 - ld1 {v6.16b}, [x0], #16 //AES block 4k+6 - load ciphertext + ldr q4, [x0, #0] //AES block 4k+4 - load ciphertext aese v1.16b, v29.16b //AES block 4k+5 - round 11 - ld1 {v7.16b}, [x0], #16 //AES block 4k+7 - load ciphertext + ldr q5, [x0, #16] //AES block 4k+5 - load ciphertext rev w9, w12 //CTR block 4k+8 aese v3.16b, v26.16b @@ -3547,13 +3347,11 @@ aes_gcm_dec_192_kernel: aesmc v2.16b, v2.16b //AES block 4k+6 - round 9 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid + add x0, x0, #64 //AES input_ptr update cmp x0, x5 //.LOOP CONTROL eor v0.16b, v4.16b, v0.16b //AES block 4k+4 - result eor x24, x24, x14 //AES block 4k+3 - round 12 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif eor v1.16b, v5.16b, v1.16b //AES block 4k+5 - result aese v2.16b, v28.16b @@ -3586,28 +3384,18 @@ aes_gcm_dec_192_kernel: rev w9, w12 //CTR block 4k+9 eor x6, x6, x13 //AES block 4k+4 - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif orr x9, x11, x9, lsl #32 //CTR block 4k+9 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low fmov d1, x10 //CTR block 4k+9 add w12, w12, #1 //CTR block 4k+9 eor x19, x19, x13 //AES block 4k+5 - round 12 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif + fmov v1.d[1], x9 //CTR block 4k+9 rev w9, w12 //CTR block 4k+10 eor x20, x20, x14 //AES block 4k+5 - round 12 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif + eor x7, x7, x14 //AES block 4k+4 - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif stp x6, x7, [x2], #16 //AES block 4k+4 - store result eor v11.16b, v11.16b, v10.16b //MODULO - fold into low @@ -3661,29 +3449,18 @@ aes_gcm_dec_192_kernel: pmull v31.1q, v5.1d, v14.1d //GHASH block 4k+1 - low eor x24, x24, x14 //AES block 4k+3 - round 12 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif fmov v3.d[1], x9 //CTR block 4k+7 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 2 eor x21, x21, x13 //AES block 4k+2 - round 12 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif + pmull2 v30.1q, v5.2d, v14.2d //GHASH block 4k+1 - high eor x22, x22, x14 //AES block 4k+2 - round 12 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid pmull v10.1q, v8.1d, v10.1d //GHASH block 4k - mid eor x23, x23, x13 //AES block 4k+3 - round 12 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif stp x21, x22, [x2], #16 //AES block 4k+2 - store result rev64 v7.16b, v7.16b //GHASH block 4k+3 @@ -3873,13 +3650,8 @@ aes_gcm_dec_192_kernel: cmp x5, #48 eor x7, x7, x14 //AES block 4k+4 - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES block 4k+4 - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif b.gt .L192_dec_blocks_more_than_3 movi v11.8b, #0 @@ -3923,16 +3695,10 @@ aes_gcm_dec_192_kernel: pmull2 v9.1q, v4.2d, v15.2d //GHASH final-3 block - high eor x6, x6, x13 //AES final-2 block - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif movi v8.8b, #0 //suppress further partial tag feed in pmull v10.1q, v22.1d, v10.1d //GHASH final-3 block - mid eor x7, x7, x14 //AES final-2 block - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif .L192_dec_blocks_more_than_2: //blocks left > 2 rev64 v4.16b, v5.16b //GHASH final-2 block @@ -3962,13 +3728,8 @@ aes_gcm_dec_192_kernel: eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high eor x7, x7, x14 //AES final-1 block - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES final-1 block - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid .L192_dec_blocks_more_than_1: //blocks left > 1 @@ -3999,13 +3760,9 @@ aes_gcm_dec_192_kernel: movi v8.8b, #0 //suppress further partial tag feed in eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low eor x7, x7, x14 //AES final block - round 12 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES final block - round 12 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif + eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid .L192_dec_blocks_less_than_1: //blocks left <= 1 @@ -4032,11 +3789,8 @@ aes_gcm_dec_192_kernel: orr x6, x6, x4 mov v0.d[1], x10 -#ifndef __AARCH64EB__ + rev w9, w12 -#else - mov w9, w12 -#endif and v5.16b, v5.16b, v0.16b //possibly partial last block has zeroes in highest bits str w9, [x16, #12] //store the updated counter @@ -4109,7 +3863,6 @@ aes_gcm_dec_192_kernel: .type aes_gcm_enc_256_kernel,%function .align 4 aes_gcm_enc_256_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, .L256_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -4125,22 +3878,14 @@ aes_gcm_enc_256_kernel: lsr x5, x1, #3 //byte_len mov x15, x5 ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #224] //load rk14 -#ifdef __AARCH64EB__ - ror x13, x13, #32 - ror x14, x14, #32 -#endif + ld1 { v0.16b}, [x16] //special case vector load initial counter so we can start first AES block as quickly as possible sub x5, x5, #1 //byte_len - 1 - ld1 {v18.4s}, [x8], #16 //load rk0 + ldr q18, [x8, #0] //load rk0 and x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q25, [x8, #112] //load rk7 add x5, x5, x0 lsr x12, x11, #32 @@ -4160,14 +3905,14 @@ aes_gcm_enc_256_kernel: orr x9, x11, x9, lsl #32 //CTR block 1 add w12, w12, #1 //CTR block 1 - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q19, [x8, #16] //load rk1 fmov v1.d[1], x9 //CTR block 1 rev w9, w12 //CTR block 2 add w12, w12, #1 //CTR block 2 orr x9, x11, x9, lsl #32 //CTR block 2 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q20, [x8, #32] //load rk2 fmov v2.d[1], x9 //CTR block 2 rev w9, w12 //CTR block 3 @@ -4180,53 +3925,50 @@ aes_gcm_enc_256_kernel: aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q21, [x8, #48] //load rk3 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q24, [x8, #96] //load rk6 aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q23, [x8, #80] //load rk5 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif + aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q31, [x8, #208] //load rk13 aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q22, [x8, #64] //load rk4 aese v1.16b, v20.16b aesmc v1.16b, v1.16b //AES block 1 - round 2 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif + aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q30, [x8, #192] //load rk12 aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif + aese v1.16b, v21.16b aesmc v1.16b, v1.16b //AES block 1 - round 3 - ld1 {v28.4s}, [x8], #16 //load rk10 + ldr q29, [x8, #176] //load rk11 aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 - ld1 {v29.4s}, [x8], #16 //load rk11 + ldr q26, [x8, #128] //load rk8 aese v2.16b, v21.16b aesmc v2.16b, v2.16b //AES block 2 - round 3 @@ -4234,6 +3976,7 @@ aes_gcm_enc_256_kernel: aese v0.16b, v21.16b aesmc v0.16b, v0.16b //AES block 0 - round 3 + ldp x13, x14, [x8, #224] //load rk14 aese v3.16b, v21.16b aesmc v3.16b, v3.16b //AES block 3 - round 3 @@ -4271,17 +4014,16 @@ aes_gcm_enc_256_kernel: aese v3.16b, v24.16b aesmc v3.16b, v3.16b //AES block 3 - round 6 - ld1 {v30.4s}, [x8], #16 //load rk12 + ldr q27, [x8, #144] //load rk9 aese v0.16b, v24.16b aesmc v0.16b, v0.16b //AES block 0 - round 6 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v2.16b, v24.16b aesmc v2.16b, v2.16b //AES block 2 - round 6 - ld1 {v31.4s}, [x8], #16 //load rk13 + ldr q28, [x8, #160] //load rk10 aese v1.16b, v25.16b aesmc v1.16b, v1.16b //AES block 1 - round 7 @@ -4370,26 +4112,13 @@ aes_gcm_enc_256_kernel: b.ge .L256_enc_tail //handle tail ldp x19, x20, [x0, #16] //AES block 1 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + rev w9, w12 //CTR block 4 ldp x6, x7, [x0, #0] //AES block 0 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + ldp x23, x24, [x0, #48] //AES block 3 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + ldp x21, x22, [x0, #32] //AES block 2 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif add x0, x0, #64 //AES input_ptr update eor x19, x19, x13 //AES block 1 - round 14 low @@ -4472,17 +4201,11 @@ aes_gcm_enc_256_kernel: aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 1 ldp x23, x24, [x0, #48] //AES block 4k+7 - load plaintext -#ifdef __AARCH64EB__ - rev x23, x23 - rev x24, x24 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 1 ldp x21, x22, [x0, #32] //AES block 4k+6 - load plaintext -#ifdef __AARCH64EB__ - rev x21, x21 - rev x22, x22 -#endif + aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 2 eor v4.16b, v4.16b, v11.16b //PRE 1 @@ -4592,10 +4315,7 @@ aes_gcm_enc_256_kernel: aese v3.16b, v24.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 6 ldp x19, x20, [x0, #16] //AES block 4k+5 - load plaintext -#ifdef __AARCH64EB__ - rev x19, x19 - rev x20, x20 -#endif + aese v1.16b, v26.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 8 mov d4, v7.d[1] //GHASH block 4k+3 - mid @@ -4632,10 +4352,7 @@ aes_gcm_enc_256_kernel: aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 8 ldp x6, x7, [x0, #0] //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 10 shl d8, d8, #56 //mod_constant @@ -4997,10 +4714,7 @@ aes_gcm_enc_256_kernel: ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag sub x5, x4, x0 //main_end_input_ptr is number of bytes left to process ldp x6, x7, [x0], #16 //AES block 4k+4 - load plaintext -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor x6, x6, x13 //AES block 4k+4 - round 14 low eor x7, x7, x14 //AES block 4k+4 - round 14 high @@ -5035,10 +4749,7 @@ aes_gcm_enc_256_kernel: st1 { v5.16b}, [x2], #16 //AES final-3 block - store result ldp x6, x7, [x0], #16 //AES final-2 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-3 block eor x6, x6, x13 //AES final-2 block - round 14 low @@ -5067,10 +4778,7 @@ aes_gcm_enc_256_kernel: st1 { v5.16b}, [x2], #16 //AES final-2 block - store result ldp x6, x7, [x0], #16 //AES final-1 block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + rev64 v4.16b, v5.16b //GHASH final-2 block eor x6, x6, x13 //AES final-1 block - round 14 low @@ -5106,10 +4814,7 @@ aes_gcm_enc_256_kernel: rev64 v4.16b, v5.16b //GHASH final-1 block ldp x6, x7, [x0], #16 //AES final block - load input low & high -#ifdef __AARCH64EB__ - rev x6, x6 - rev x7, x7 -#endif + eor v4.16b, v4.16b, v8.16b //feed in partial tag movi v8.8b, #0 //suppress further partial tag feed in @@ -5170,11 +4875,7 @@ aes_gcm_enc_256_kernel: pmull2 v20.1q, v4.2d, v12.2d //GHASH final block - high mov d8, v4.d[1] //GHASH final block - mid -#ifndef __AARCH64EB__ rev w9, w12 -#else - mov w9, w12 -#endif pmull v21.1q, v4.1d, v12.1d //GHASH final block - low @@ -5234,7 +4935,6 @@ aes_gcm_enc_256_kernel: .type aes_gcm_dec_256_kernel,%function .align 4 aes_gcm_dec_256_kernel: - AARCH64_VALID_CALL_TARGET cbz x1, .L256_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -5249,29 +4949,21 @@ aes_gcm_dec_256_kernel: lsr x5, x1, #3 //byte_len mov x15, x5 ldp x10, x11, [x16] //ctr96_b64, ctr96_t32 -#ifdef __AARCH64EB__ - rev x10, x10 - rev x11, x11 -#endif - ldp x13, x14, [x8, #224] //load rk14 -#ifdef __AARCH64EB__ - ror x14, x14, #32 - ror x13, x13, #32 -#endif - ld1 {v18.4s}, [x8], #16 //load rk0 + + ldr q26, [x8, #128] //load rk8 sub x5, x5, #1 //byte_len - 1 - ld1 {v19.4s}, [x8], #16 //load rk1 + ldr q25, [x8, #112] //load rk7 and x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail) add x4, x0, x1, lsr #3 //end_input_ptr - ld1 {v20.4s}, [x8], #16 //load rk2 + ldr q24, [x8, #96] //load rk6 lsr x12, x11, #32 - ld1 {v21.4s}, [x8], #16 //load rk3 + ldr q23, [x8, #80] //load rk5 orr w11, w11, w11 - ld1 {v22.4s}, [x8], #16 //load rk4 + ldr q21, [x8, #48] //load rk3 add x5, x5, x0 rev w12, w12 //rev_ctr32 @@ -5296,44 +4988,39 @@ aes_gcm_dec_256_kernel: rev w9, w12 //CTR block 3 orr x9, x11, x9, lsl #32 //CTR block 3 - ld1 {v23.4s}, [x8], #16 //load rk5 + ldr q18, [x8, #0] //load rk0 fmov v3.d[1], x9 //CTR block 3 add w12, w12, #1 //CTR block 3 - ld1 {v24.4s}, [x8], #16 //load rk6 + ldr q22, [x8, #64] //load rk4 - ld1 {v25.4s}, [x8], #16 //load rk7 + ldr q31, [x8, #208] //load rk13 - ld1 {v26.4s}, [x8], #16 //load rk8 + ldr q19, [x8, #16] //load rk1 aese v0.16b, v18.16b aesmc v0.16b, v0.16b //AES block 0 - round 0 ldr q14, [x3, #80] //load h3l | h3h -#ifndef __AARCH64EB__ ext v14.16b, v14.16b, v14.16b, #8 -#endif aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 3 - round 0 ldr q15, [x3, #112] //load h4l | h4h -#ifndef __AARCH64EB__ ext v15.16b, v15.16b, v15.16b, #8 -#endif aese v1.16b, v18.16b aesmc v1.16b, v1.16b //AES block 1 - round 0 ldr q13, [x3, #64] //load h2l | h2h -#ifndef __AARCH64EB__ ext v13.16b, v13.16b, v13.16b, #8 -#endif aese v2.16b, v18.16b aesmc v2.16b, v2.16b //AES block 2 - round 0 - ld1 {v27.4s}, [x8], #16 //load rk9 + ldr q20, [x8, #32] //load rk2 aese v0.16b, v19.16b aesmc v0.16b, v0.16b //AES block 0 - round 1 + ldp x13, x14, [x8, #224] //load rk14 aese v1.16b, v19.16b aesmc v1.16b, v1.16b //AES block 1 - round 1 @@ -5343,21 +5030,20 @@ aes_gcm_dec_256_kernel: aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 2 - round 1 - ld1 {v28.4s}, [x8], #16 //load rk10 + ldr q27, [x8, #144] //load rk9 aese v3.16b, v19.16b aesmc v3.16b, v3.16b //AES block 3 - round 1 - ld1 {v29.4s}, [x8], #16 //load rk11 + ldr q30, [x8, #192] //load rk12 aese v0.16b, v20.16b aesmc v0.16b, v0.16b //AES block 0 - round 2 ldr q12, [x3, #32] //load h1l | h1h -#ifndef __AARCH64EB__ ext v12.16b, v12.16b, v12.16b, #8 -#endif + aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 2 - round 2 - ld1 {v30.4s}, [x8], #16 //load rk12 + ldr q28, [x8, #160] //load rk10 aese v3.16b, v20.16b aesmc v3.16b, v3.16b //AES block 3 - round 2 @@ -5440,7 +5126,7 @@ aes_gcm_dec_256_kernel: aese v2.16b, v26.16b aesmc v2.16b, v2.16b //AES block 2 - round 8 - ld1 {v31.4s}, [x8], #16 //load rk13 + ldr q29, [x8, #176] //load rk11 aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 1 - round 9 @@ -5505,7 +5191,9 @@ aes_gcm_dec_256_kernel: aese v0.16b, v31.16b //AES block 0 - round 13 b.ge .L256_dec_tail //handle tail - ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0,1 - load ciphertext + ldr q4, [x0, #0] //AES block 0 - load ciphertext + + ldr q5, [x0, #16] //AES block 1 - load ciphertext rev w9, w12 //CTR block 4 @@ -5513,7 +5201,7 @@ aes_gcm_dec_256_kernel: eor v1.16b, v5.16b, v1.16b //AES block 1 - result rev64 v5.16b, v5.16b //GHASH block 1 - ld1 {v6.16b}, [x0], #16 //AES block 2 - load ciphertext + ldr q7, [x0, #48] //AES block 3 - load ciphertext mov x7, v0.d[1] //AES block 0 - mov high @@ -5533,32 +5221,22 @@ aes_gcm_dec_256_kernel: orr x9, x11, x9, lsl #32 //CTR block 5 mov x20, v1.d[1] //AES block 1 - mov high eor x7, x7, x14 //AES block 0 - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + eor x6, x6, x13 //AES block 0 - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif stp x6, x7, [x2], #16 //AES block 0 - store result fmov d1, x10 //CTR block 5 - ld1 {v7.16b}, [x0], #16 //AES block 3 - load ciphertext + ldr q6, [x0, #32] //AES block 2 - load ciphertext + add x0, x0, #64 //AES input_ptr update fmov v1.d[1], x9 //CTR block 5 rev w9, w12 //CTR block 6 add w12, w12, #1 //CTR block 6 eor x19, x19, x13 //AES block 1 - round 14 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif orr x9, x11, x9, lsl #32 //CTR block 6 eor x20, x20, x14 //AES block 1 - round 14 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif stp x19, x20, [x2], #16 //AES block 1 - store result eor v2.16b, v6.16b, v2.16b //AES block 2 - result @@ -5609,9 +5287,7 @@ aes_gcm_dec_256_kernel: aese v0.16b, v21.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 3 eor x22, x22, x14 //AES block 4k+2 - round 14 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif + aese v2.16b, v19.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 1 mov d10, v17.d[1] //GHASH block 4k - mid @@ -5623,9 +5299,7 @@ aes_gcm_dec_256_kernel: aese v3.16b, v18.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 0 eor x21, x21, x13 //AES block 4k+2 - round 14 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif + aese v2.16b, v20.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 2 stp x21, x22, [x2], #16 //AES block 4k+2 - store result @@ -5640,14 +5314,9 @@ aes_gcm_dec_256_kernel: pmull v10.1q, v8.1d, v10.1d //GHASH block 4k - mid eor x23, x23, x13 //AES block 4k+3 - round 14 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + pmull v8.1q, v5.1d, v14.1d //GHASH block 4k+1 - low eor x24, x24, x14 //AES block 4k+3 - round 14 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif eor v9.16b, v9.16b, v4.16b //GHASH block 4k+1 - high aese v2.16b, v22.16b @@ -5768,7 +5437,7 @@ aes_gcm_dec_256_kernel: aese v1.16b, v27.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 9 - ld1 {v4.16b}, [x0], #16 //AES block 4k+4 - load ciphertext + ldr q4, [x0, #0] //AES block 4k+4 - load ciphertext aese v0.16b, v31.16b //AES block 4k+4 - round 13 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment @@ -5779,7 +5448,7 @@ aes_gcm_dec_256_kernel: aese v2.16b, v27.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 9 - ld1 {v5.16b}, [x0], #16 //AES block 4k+5 - load ciphertext + ldr q5, [x0, #16] //AES block 4k+5 - load ciphertext aese v3.16b, v26.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 8 @@ -5795,11 +5464,11 @@ aes_gcm_dec_256_kernel: aese v3.16b, v27.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 9 - ld1 {v6.16b}, [x0], #16 //AES block 4k+6 - load ciphertext + ldr q7, [x0, #48] //AES block 4k+7 - load ciphertext aese v1.16b, v30.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 12 - ld1 {v7.16b}, [x0], #16 //AES block 4k+7 - load ciphertext + ldr q6, [x0, #32] //AES block 4k+6 - load ciphertext aese v2.16b, v29.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 11 @@ -5810,6 +5479,7 @@ aes_gcm_dec_256_kernel: eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid aese v1.16b, v31.16b //AES block 4k+5 - round 13 + add x0, x0, #64 //AES input_ptr update mov x6, v0.d[0] //AES block 4k+4 - mov low aese v2.16b, v30.16b @@ -5831,13 +5501,8 @@ aes_gcm_dec_256_kernel: add w12, w12, #1 //CTR block 4k+9 eor x6, x6, x13 //AES block 4k+4 - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor x7, x7, x14 //AES block 4k+4 - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif + mov x20, v1.d[1] //AES block 4k+5 - mov high eor v2.16b, v6.16b, v2.16b //AES block 4k+6 - result eor v11.16b, v11.16b, v8.16b //MODULO - fold into low @@ -5858,15 +5523,9 @@ aes_gcm_dec_256_kernel: rev64 v5.16b, v5.16b //GHASH block 4k+5 eor x20, x20, x14 //AES block 4k+5 - round 14 high -#ifdef __AARCH64EB__ - rev x20, x20 -#endif stp x6, x7, [x2], #16 //AES block 4k+4 - store result eor x19, x19, x13 //AES block 4k+5 - round 14 low -#ifdef __AARCH64EB__ - rev x19, x19 -#endif stp x19, x20, [x2], #16 //AES block 4k+5 - store result rev64 v4.16b, v4.16b //GHASH block 4k+4 @@ -6073,15 +5732,11 @@ aes_gcm_dec_256_kernel: aese v0.16b, v28.16b aesmc v0.16b, v0.16b //AES block 4k+4 - round 10 eor x22, x22, x14 //AES block 4k+2 - round 14 high -#ifdef __AARCH64EB__ - rev x22, x22 -#endif + aese v1.16b, v28.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 10 eor x23, x23, x13 //AES block 4k+3 - round 14 low -#ifdef __AARCH64EB__ - rev x23, x23 -#endif + aese v2.16b, v29.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 11 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid @@ -6093,18 +5748,12 @@ aes_gcm_dec_256_kernel: aese v1.16b, v29.16b aesmc v1.16b, v1.16b //AES block 4k+5 - round 11 eor x21, x21, x13 //AES block 4k+2 - round 14 low -#ifdef __AARCH64EB__ - rev x21, x21 -#endif aese v2.16b, v30.16b aesmc v2.16b, v2.16b //AES block 4k+6 - round 12 pmull v8.1q, v10.1d, v8.1d //MODULO - mid 64b align with low eor x24, x24, x14 //AES block 4k+3 - round 14 high -#ifdef __AARCH64EB__ - rev x24, x24 -#endif aese v3.16b, v29.16b aesmc v3.16b, v3.16b //AES block 4k+7 - round 11 @@ -6145,14 +5794,8 @@ aes_gcm_dec_256_kernel: cmp x5, #48 eor x6, x6, x13 //AES block 4k+4 - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor x7, x7, x14 //AES block 4k+4 - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif b.gt .L256_dec_blocks_more_than_3 sub w12, w12, #1 @@ -6200,15 +5843,9 @@ aes_gcm_dec_256_kernel: pmull v10.1q, v22.1d, v10.1d //GHASH final-3 block - mid eor x6, x6, x13 //AES final-2 block - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif pmull v11.1q, v4.1d, v15.1d //GHASH final-3 block - low eor x7, x7, x14 //AES final-2 block - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif .L256_dec_blocks_more_than_2: //blocks left > 2 rev64 v4.16b, v5.16b //GHASH final-2 block @@ -6236,15 +5873,9 @@ aes_gcm_dec_256_kernel: eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high eor x6, x6, x13 //AES final-1 block - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid eor x7, x7, x14 //AES final-1 block - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif .L256_dec_blocks_more_than_1: //blocks left > 1 stp x6, x7, [x2], #16 //AES final-1 block - store result @@ -6272,18 +5903,13 @@ aes_gcm_dec_256_kernel: pmull2 v22.1q, v22.2d, v16.2d //GHASH final-1 block - mid eor x6, x6, x13 //AES final block - round 14 low -#ifdef __AARCH64EB__ - rev x6, x6 -#endif + eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid eor x7, x7, x14 //AES final block - round 14 high -#ifdef __AARCH64EB__ - rev x7, x7 -#endif .L256_dec_blocks_less_than_1: //blocks left <= 1 and x1, x1, #127 //bit_length %= 128 @@ -6309,11 +5935,7 @@ aes_gcm_dec_256_kernel: mov v0.d[1], x10 bic x4, x4, x9 //mask out low existing bytes -#ifndef __AARCH64EB__ rev w9, w12 -#else - mov w9, w12 -#endif bic x5, x5, x10 //mask out high existing bytes diff --git a/openssl/src/crypto/modes/gen/linux_arm64/ghashv8-armx.S b/openssl/src/crypto/modes/gen/linux_arm64/ghashv8-armx.S index 0c7a5cc60..acd52eb95 100644 --- a/openssl/src/crypto/modes/gen/linux_arm64/ghashv8-armx.S +++ b/openssl/src/crypto/modes/gen/linux_arm64/ghashv8-armx.S @@ -7,7 +7,6 @@ .type gcm_init_v8,%function .align 4 gcm_init_v8: - AARCH64_VALID_CALL_TARGET ld1 {v17.2d},[x1] //load input H movi v19.16b,#0xe1 shl v19.2d,v19.2d,#57 //0xc2.0 @@ -83,115 +82,26 @@ gcm_init_v8: pmull v5.1q,v5.1d,v19.1d eor v18.16b,v18.16b,v2.16b eor v4.16b,v4.16b,v7.16b - eor v23.16b, v0.16b,v18.16b //H^3 - eor v25.16b,v5.16b,v4.16b //H^4 - - ext v16.16b,v23.16b, v23.16b,#8 //Karatsuba pre-processing - ext v17.16b,v25.16b,v25.16b,#8 - ext v18.16b,v22.16b,v22.16b,#8 - eor v16.16b,v16.16b,v23.16b - eor v17.16b,v17.16b,v25.16b - eor v18.16b,v18.16b,v22.16b - ext v24.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed - st1 {v23.2d,v24.2d,v25.2d},[x0],#48 //store Htable[3..5] - - //calculate H^5 and H^6 - pmull v0.1q,v22.1d, v23.1d - pmull v5.1q,v23.1d,v23.1d - pmull2 v2.1q,v22.2d, v23.2d - pmull2 v7.1q,v23.2d,v23.2d - pmull v1.1q,v16.1d,v18.1d - pmull v6.1q,v16.1d,v16.1d + eor v20.16b, v0.16b,v18.16b //H^3 + eor v22.16b,v5.16b,v4.16b //H^4 - ext v16.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing - ext v17.16b,v5.16b,v7.16b,#8 - eor v18.16b,v0.16b,v2.16b - eor v1.16b,v1.16b,v16.16b - eor v4.16b,v5.16b,v7.16b - eor v6.16b,v6.16b,v17.16b - eor v1.16b,v1.16b,v18.16b - pmull v18.1q,v0.1d,v19.1d //1st phase - eor v6.16b,v6.16b,v4.16b - pmull v4.1q,v5.1d,v19.1d - - ins v2.d[0],v1.d[1] - ins v7.d[0],v6.d[1] - ins v1.d[1],v0.d[0] - ins v6.d[1],v5.d[0] - eor v0.16b,v1.16b,v18.16b - eor v5.16b,v6.16b,v4.16b - - ext v18.16b,v0.16b,v0.16b,#8 //2nd phase - ext v4.16b,v5.16b,v5.16b,#8 - pmull v0.1q,v0.1d,v19.1d - pmull v5.1q,v5.1d,v19.1d - eor v18.16b,v18.16b,v2.16b - eor v4.16b,v4.16b,v7.16b - eor v26.16b,v0.16b,v18.16b //H^5 - eor v28.16b,v5.16b,v4.16b //H^6 - - ext v16.16b,v26.16b, v26.16b,#8 //Karatsuba pre-processing - ext v17.16b,v28.16b,v28.16b,#8 - ext v18.16b,v22.16b,v22.16b,#8 - eor v16.16b,v16.16b,v26.16b - eor v17.16b,v17.16b,v28.16b - eor v18.16b,v18.16b,v22.16b - ext v27.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed - st1 {v26.2d,v27.2d,v28.2d},[x0],#48 //store Htable[6..8] - - //calculate H^7 and H^8 - pmull v0.1q,v22.1d,v26.1d - pmull v5.1q,v22.1d,v28.1d - pmull2 v2.1q,v22.2d,v26.2d - pmull2 v7.1q,v22.2d,v28.2d - pmull v1.1q,v16.1d,v18.1d - pmull v6.1q,v17.1d,v18.1d - - ext v16.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing - ext v17.16b,v5.16b,v7.16b,#8 - eor v18.16b,v0.16b,v2.16b - eor v1.16b,v1.16b,v16.16b - eor v4.16b,v5.16b,v7.16b - eor v6.16b,v6.16b,v17.16b - eor v1.16b,v1.16b,v18.16b - pmull v18.1q,v0.1d,v19.1d //1st phase - eor v6.16b,v6.16b,v4.16b - pmull v4.1q,v5.1d,v19.1d - - ins v2.d[0],v1.d[1] - ins v7.d[0],v6.d[1] - ins v1.d[1],v0.d[0] - ins v6.d[1],v5.d[0] - eor v0.16b,v1.16b,v18.16b - eor v5.16b,v6.16b,v4.16b - - ext v18.16b,v0.16b,v0.16b,#8 //2nd phase - ext v4.16b,v5.16b,v5.16b,#8 - pmull v0.1q,v0.1d,v19.1d - pmull v5.1q,v5.1d,v19.1d - eor v18.16b,v18.16b,v2.16b - eor v4.16b,v4.16b,v7.16b - eor v29.16b,v0.16b,v18.16b //H^7 - eor v31.16b,v5.16b,v4.16b //H^8 - - ext v16.16b,v29.16b,v29.16b,#8 //Karatsuba pre-processing - ext v17.16b,v31.16b,v31.16b,#8 - eor v16.16b,v16.16b,v29.16b - eor v17.16b,v17.16b,v31.16b - ext v30.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed - st1 {v29.2d,v30.2d,v31.2d},[x0] //store Htable[9..11] + ext v16.16b,v20.16b, v20.16b,#8 //Karatsuba pre-processing + ext v17.16b,v22.16b,v22.16b,#8 + eor v16.16b,v16.16b,v20.16b + eor v17.16b,v17.16b,v22.16b + ext v21.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed + st1 {v20.2d,v21.2d,v22.2d},[x0] //store Htable[3..5] ret .size gcm_init_v8,.-gcm_init_v8 .globl gcm_gmult_v8 .type gcm_gmult_v8,%function .align 4 gcm_gmult_v8: - AARCH64_VALID_CALL_TARGET ld1 {v17.2d},[x0] //load Xi movi v19.16b,#0xe1 ld1 {v20.2d,v21.2d},[x1] //load twisted H, ... shl v19.2d,v19.2d,#57 -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v17.16b,v17.16b #endif ext v3.16b,v17.16b,v17.16b,#8 @@ -216,7 +126,7 @@ gcm_gmult_v8: eor v18.16b,v18.16b,v2.16b eor v0.16b,v0.16b,v18.16b -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v0.16b,v0.16b #endif ext v0.16b,v0.16b,v0.16b,#8 @@ -228,7 +138,6 @@ gcm_gmult_v8: .type gcm_ghash_v8,%function .align 4 gcm_ghash_v8: - AARCH64_VALID_CALL_TARGET cmp x3,#64 b.hs .Lgcm_ghash_v8_4x ld1 {v0.2d},[x0] //load [rotated] Xi @@ -254,14 +163,14 @@ gcm_ghash_v8: ext v0.16b,v0.16b,v0.16b,#8 //rotate Xi ld1 {v16.2d},[x2],#16 //load [rotated] I[0] shl v19.2d,v19.2d,#57 //compose 0xc2.0 constant -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v16.16b,v16.16b rev64 v0.16b,v0.16b #endif ext v3.16b,v16.16b,v16.16b,#8 //rotate I[0] b.lo .Lodd_tail_v8 //x3 was less than 32 ld1 {v17.2d},[x2],x12 //load [rotated] I[1] -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v17.16b,v17.16b #endif ext v7.16b,v17.16b,v17.16b,#8 @@ -293,13 +202,13 @@ gcm_ghash_v8: eor v18.16b,v0.16b,v2.16b eor v1.16b,v1.16b,v17.16b ld1 {v17.2d},[x2],x12 //load [rotated] I[i+3] -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v16.16b,v16.16b #endif eor v1.16b,v1.16b,v18.16b pmull v18.1q,v0.1d,v19.1d //1st phase of reduction -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v17.16b,v17.16b #endif ins v2.d[0],v1.d[1] @@ -349,7 +258,7 @@ gcm_ghash_v8: eor v0.16b,v0.16b,v18.16b .Ldone_v8: -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v0.16b,v0.16b #endif ext v0.16b,v0.16b,v0.16b,#8 @@ -368,7 +277,7 @@ gcm_ghash_v8_4x: shl v19.2d,v19.2d,#57 //compose 0xc2.0 constant ld1 {v4.2d,v5.2d,v6.2d,v7.2d},[x2],#64 -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v0.16b,v0.16b rev64 v5.16b,v5.16b rev64 v6.16b,v6.16b @@ -412,7 +321,7 @@ gcm_ghash_v8_4x: eor v16.16b,v4.16b,v0.16b ld1 {v4.2d,v5.2d,v6.2d,v7.2d},[x2],#64 ext v3.16b,v16.16b,v16.16b,#8 -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v5.16b,v5.16b rev64 v6.16b,v6.16b rev64 v7.16b,v7.16b @@ -495,7 +404,7 @@ gcm_ghash_v8_4x: eor v1.16b,v1.16b,v17.16b ld1 {v4.2d,v5.2d,v6.2d},[x2] eor v1.16b,v1.16b,v18.16b -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v5.16b,v5.16b rev64 v6.16b,v6.16b rev64 v4.16b,v4.16b @@ -547,7 +456,7 @@ gcm_ghash_v8_4x: eor v1.16b,v1.16b,v17.16b ld1 {v4.2d,v5.2d},[x2] eor v1.16b,v1.16b,v18.16b -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v5.16b,v5.16b rev64 v4.16b,v4.16b #endif @@ -590,7 +499,7 @@ gcm_ghash_v8_4x: eor v1.16b,v1.16b,v17.16b ld1 {v4.2d},[x2] eor v1.16b,v1.16b,v18.16b -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v4.16b,v4.16b #endif @@ -630,7 +539,7 @@ gcm_ghash_v8_4x: eor v0.16b,v0.16b,v18.16b ext v0.16b,v0.16b,v0.16b,#8 -#ifndef __AARCH64EB__ +#ifndef __ARMEB__ rev64 v0.16b,v0.16b #endif st1 {v0.2d},[x0] //write out Xi diff --git a/openssl/src/crypto/modes/gen/linux_ia32/ghash-x86.S b/openssl/src/crypto/modes/gen/linux_ia32/ghash-x86.S index 9b3361acd..183d7395b 100644 --- a/openssl/src/crypto/modes/gen/linux_ia32/ghash-x86.S +++ b/openssl/src/crypto/modes/gen/linux_ia32/ghash-x86.S @@ -4,11 +4,7 @@ .align 16 gcm_gmult_4bit_x86: .L_gcm_gmult_4bit_x86_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -103,11 +99,7 @@ gcm_gmult_4bit_x86: .align 16 gcm_ghash_4bit_x86: .L_gcm_ghash_4bit_x86_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -217,11 +209,7 @@ gcm_ghash_4bit_x86: .align 16 gcm_gmult_4bit_mmx: .L_gcm_gmult_4bit_mmx_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -321,11 +309,7 @@ gcm_gmult_4bit_mmx: .align 16 gcm_ghash_4bit_mmx: .L_gcm_ghash_4bit_mmx_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -930,11 +914,7 @@ gcm_ghash_4bit_mmx: .align 16 gcm_init_clmul: .L_gcm_init_clmul_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%edx movl 8(%esp),%eax call .L010pic @@ -1004,11 +984,7 @@ gcm_init_clmul: .align 16 gcm_gmult_clmul: .L_gcm_gmult_clmul_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movl 4(%esp),%eax movl 8(%esp),%edx call .L011pic @@ -1062,11 +1038,7 @@ gcm_gmult_clmul: .align 16 gcm_ghash_clmul: .L_gcm_ghash_clmul_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/modes/gen/linux_ppc64/aes-gcm-ppc.s b/openssl/src/crypto/modes/gen/linux_ppc64/aes-gcm-ppc.s deleted file mode 100644 index e933689d6..000000000 --- a/openssl/src/crypto/modes/gen/linux_ppc64/aes-gcm-ppc.s +++ /dev/null @@ -1,1339 +0,0 @@ -.machine "any" -.abiversion 2 -.text - - - - - -.macro .Loop_aes_middle4x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - .long 0x11EF9D08 - .long 0x12109D08 - .long 0x12319D08 - .long 0x12529D08 - - .long 0x11EFA508 - .long 0x1210A508 - .long 0x1231A508 - .long 0x1252A508 - - .long 0x11EFAD08 - .long 0x1210AD08 - .long 0x1231AD08 - .long 0x1252AD08 - - .long 0x11EFB508 - .long 0x1210B508 - .long 0x1231B508 - .long 0x1252B508 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - .long 0x11EF9D08 - .long 0x12109D08 - .long 0x12319D08 - .long 0x12529D08 - - .long 0x11EFA508 - .long 0x1210A508 - .long 0x1231A508 - .long 0x1252A508 - - .long 0x11EFAD08 - .long 0x1210AD08 - .long 0x1231AD08 - .long 0x1252AD08 - - .long 0x11EFB508 - .long 0x1210B508 - .long 0x1231B508 - .long 0x1252B508 - - xxlor 23+32, 9, 9 - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 -.endm - - - - - -.macro .Loop_aes_middle8x - xxlor 23+32, 1, 1 - xxlor 24+32, 2, 2 - xxlor 25+32, 3, 3 - xxlor 26+32, 4, 4 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - .long 0x11EFCD08 - .long 0x1210CD08 - .long 0x1231CD08 - .long 0x1252CD08 - .long 0x1273CD08 - .long 0x1294CD08 - .long 0x12B5CD08 - .long 0x12D6CD08 - - .long 0x11EFD508 - .long 0x1210D508 - .long 0x1231D508 - .long 0x1252D508 - .long 0x1273D508 - .long 0x1294D508 - .long 0x12B5D508 - .long 0x12D6D508 - - xxlor 23+32, 5, 5 - xxlor 24+32, 6, 6 - xxlor 25+32, 7, 7 - xxlor 26+32, 8, 8 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - .long 0x11EFCD08 - .long 0x1210CD08 - .long 0x1231CD08 - .long 0x1252CD08 - .long 0x1273CD08 - .long 0x1294CD08 - .long 0x12B5CD08 - .long 0x12D6CD08 - - .long 0x11EFD508 - .long 0x1210D508 - .long 0x1231D508 - .long 0x1252D508 - .long 0x1273D508 - .long 0x1294D508 - .long 0x12B5D508 - .long 0x12D6D508 - - xxlor 23+32, 9, 9 - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 -.endm - - - - -ppc_aes_gcm_ghash: - vxor 15, 15, 0 - - xxlxor 29, 29, 29 - - .long 0x12EC7CC8 - .long 0x130984C8 - .long 0x13268CC8 - .long 0x134394C8 - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D7CC8 - .long 0x132A84C8 - .long 0x13478CC8 - .long 0x136494C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E7CC8 - .long 0x132B84C8 - .long 0x13488CC8 - .long 0x136594C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 - - blr - - - - - -.macro ppc_aes_gcm_ghash2_4x - - vxor 15, 15, 0 - - xxlxor 29, 29, 29 - - .long 0x12EC7CC8 - .long 0x130984C8 - .long 0x13268CC8 - .long 0x134394C8 - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D7CC8 - .long 0x132A84C8 - .long 0x13478CC8 - .long 0x136494C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - - vxor 24, 24, 27 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E7CC8 - .long 0x132B84C8 - .long 0x13488CC8 - .long 0x136594C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 27, 23, 27 - - - .long 0x1309A4C8 - .long 0x1326ACC8 - .long 0x1343B4C8 - vxor 19, 19, 27 - .long 0x12EC9CC8 - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 - - .long 0x130D9CC8 - .long 0x132AA4C8 - .long 0x1347ACC8 - .long 0x1364B4C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - - - .long 0x139714C8 - - xxlor 29+32, 29, 29 - - vxor 24, 24, 27 - vsldoi 26, 24, 29, 8 - vsldoi 29, 29, 24, 8 - vxor 23, 23, 26 - - vsldoi 23, 23, 23, 8 - vxor 23, 23, 28 - - .long 0x130E9CC8 - .long 0x132BA4C8 - .long 0x1348ACC8 - .long 0x1365B4C8 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - - vsldoi 27, 23, 23, 8 - .long 0x12F714C8 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 - -.endm - - - - -.macro ppc_update_hash_1x - vxor 28, 28, 0 - - vxor 19, 19, 19 - - .long 0x12C3E4C8 - .long 0x12E4E4C8 - .long 0x1305E4C8 - - .long 0x137614C8 - - vsldoi 25, 23, 19, 8 - vsldoi 26, 19, 23, 8 - vxor 22, 22, 25 - vxor 24, 24, 26 - - vsldoi 22, 22, 22, 8 - vxor 22, 22, 27 - - vsldoi 20, 22, 22, 8 - .long 0x12D614C8 - vxor 20, 20, 24 - vxor 22, 22, 20 - - vor 0,22,22 - -.endm - - - - - - - - - - - - - -.global ppc_aes_gcm_encrypt -.align 5 -ppc_aes_gcm_encrypt: -_ppc_aes_gcm_encrypt: - - stdu 1,-512(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - std 0, 528(1) - - - lxvb16x 32, 0, 8 - - - li 10, 32 - lxvd2x 2+32, 10, 8 - li 10, 48 - lxvd2x 3+32, 10, 8 - li 10, 64 - lxvd2x 4+32, 10, 8 - li 10, 80 - lxvd2x 5+32, 10, 8 - - li 10, 96 - lxvd2x 6+32, 10, 8 - li 10, 112 - lxvd2x 7+32, 10, 8 - li 10, 128 - lxvd2x 8+32, 10, 8 - - li 10, 144 - lxvd2x 9+32, 10, 8 - li 10, 160 - lxvd2x 10+32, 10, 8 - li 10, 176 - lxvd2x 11+32, 10, 8 - - li 10, 192 - lxvd2x 12+32, 10, 8 - li 10, 208 - lxvd2x 13+32, 10, 8 - li 10, 224 - lxvd2x 14+32, 10, 8 - - - lxvb16x 30+32, 0, 7 - - mr 12, 5 - li 11, 0 - - - vxor 31, 31, 31 - vspltisb 22,1 - vsldoi 31, 31, 22,1 - - - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - - lwz 9,240(6) - - - - xxlor 32+29, 0, 0 - vxor 15, 30, 29 - - cmpdi 9, 10 - beq .Loop_aes_gcm_8x - - - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) - - cmpdi 9, 12 - beq .Loop_aes_gcm_8x - - - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq .Loop_aes_gcm_8x - - b aes_gcm_out - -.align 5 -.Loop_aes_gcm_8x: - mr 14, 3 - mr 9, 4 - - - li 10, 128 - divdu 10, 5, 10 - cmpdi 10, 0 - beq .Loop_last_block - - .long 0x13DEF8C0 - vxor 16, 30, 29 - .long 0x13DEF8C0 - vxor 17, 30, 29 - .long 0x13DEF8C0 - vxor 18, 30, 29 - .long 0x13DEF8C0 - vxor 19, 30, 29 - .long 0x13DEF8C0 - vxor 20, 30, 29 - .long 0x13DEF8C0 - vxor 21, 30, 29 - .long 0x13DEF8C0 - vxor 22, 30, 29 - - mtctr 10 - - li 15, 16 - li 16, 32 - li 17, 48 - li 18, 64 - li 19, 80 - li 20, 96 - li 21, 112 - - lwz 10, 240(6) - -.Loop_8x_block: - - lxvb16x 15, 0, 14 - lxvb16x 16, 15, 14 - lxvb16x 17, 16, 14 - lxvb16x 18, 17, 14 - lxvb16x 19, 18, 14 - lxvb16x 20, 19, 14 - lxvb16x 21, 20, 14 - lxvb16x 22, 21, 14 - addi 14, 14, 128 - -.Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_ghash - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_ghash - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_ghash - b aes_gcm_out - -Do_next_ghash: - - - - .long 0x11EFBD09 - .long 0x1210BD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 - - .long 0x1231BD09 - .long 0x1252BD09 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 - - .long 0x1273BD09 - .long 0x1294BD09 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 - - .long 0x12B5BD09 - .long 0x12D6BD09 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 - - addi 9, 9, 128 - - - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - .long 0x13DEF8C0 - vor 29,30,30 - vxor 15, 30, 27 - .long 0x13DEF8C0 - vxor 16, 30, 27 - .long 0x13DEF8C0 - vxor 17, 30, 27 - .long 0x13DEF8C0 - vxor 18, 30, 27 - .long 0x13DEF8C0 - vxor 19, 30, 27 - .long 0x13DEF8C0 - vxor 20, 30, 27 - .long 0x13DEF8C0 - vxor 21, 30, 27 - .long 0x13DEF8C0 - vxor 22, 30, 27 - - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz .Loop_8x_block - - vor 30,29,29 - -.Loop_last_block: - cmpdi 12, 0 - beq aes_gcm_out - - - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10, 240(6) - - cmpdi 12, 16 - blt Final_block - -.macro .Loop_aes_middle_1x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - .long 0x11EF9D08 - .long 0x11EFA508 - .long 0x11EFAD08 - .long 0x11EFB508 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - .long 0x11EF9D08 - .long 0x11EFA508 - .long 0x11EFAD08 - .long 0x11EFB508 - - xxlor 19+32, 9, 9 - .long 0x11EF9D08 -.endm - -Next_rem_block: - lxvb16x 15, 0, 14 - -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_1x - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_1x - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x - -Do_next_1x: - .long 0x11EFBD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - addi 14, 14, 16 - addi 9, 9, 16 - - vor 28,15,15 - ppc_update_hash_1x - - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - .long 0x13DEF8C0 - vxor 15, 30, 19 - - bdnz Next_rem_block - - cmpdi 12, 0 - beq aes_gcm_out - -Final_block: -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_final_1x - -Do_final_1x: - .long 0x11EFBD09 - - lxvb16x 15, 0, 14 - xxlxor 47, 47, 15 - - - li 15, 16 - sub 15, 15, 12 - - vspltisb 16,-1 - vspltisb 17,0 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 - - addi 10, 1, 192 - lxvb16x 16, 15, 10 - xxland 47, 47, 16 - - vor 28,15,15 - ppc_update_hash_1x - - - bl Write_partial_block - - b aes_gcm_out - - - - - - - -Write_partial_block: - li 10, 192 - stxvb16x 15+32, 10, 1 - - - addi 10, 9, -1 - addi 16, 1, 191 - - mtctr 12 - li 15, 0 - -Write_last_byte: - lbzu 14, 1(16) - stbu 14, 1(10) - bdnz Write_last_byte - blr - -aes_gcm_out: - - stxvb16x 32, 0, 8 - add 3, 11, 12 - - li 9, 256 - lvx 20, 9, 1 - addi 9, 9, 16 - lvx 21, 9, 1 - addi 9, 9, 16 - lvx 22, 9, 1 - addi 9, 9, 16 - lvx 23, 9, 1 - addi 9, 9, 16 - lvx 24, 9, 1 - addi 9, 9, 16 - lvx 25, 9, 1 - addi 9, 9, 16 - lvx 26, 9, 1 - addi 9, 9, 16 - lvx 27, 9, 1 - addi 9, 9, 16 - lvx 28, 9, 1 - addi 9, 9, 16 - lvx 29, 9, 1 - addi 9, 9, 16 - lvx 30, 9, 1 - addi 9, 9, 16 - lvx 31, 9, 1 - - ld 0, 528(1) - ld 14,112(1) - ld 15,120(1) - ld 16,128(1) - ld 17,136(1) - ld 18,144(1) - ld 19,152(1) - ld 20,160(1) - ld 21,168(1) - - mtlr 0 - addi 1, 1, 512 - blr - - - - -.global ppc_aes_gcm_decrypt -.align 5 -ppc_aes_gcm_decrypt: -_ppc_aes_gcm_decrypt: - - stdu 1,-512(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - std 0, 528(1) - - - lxvb16x 32, 0, 8 - - - li 10, 32 - lxvd2x 2+32, 10, 8 - li 10, 48 - lxvd2x 3+32, 10, 8 - li 10, 64 - lxvd2x 4+32, 10, 8 - li 10, 80 - lxvd2x 5+32, 10, 8 - - li 10, 96 - lxvd2x 6+32, 10, 8 - li 10, 112 - lxvd2x 7+32, 10, 8 - li 10, 128 - lxvd2x 8+32, 10, 8 - - li 10, 144 - lxvd2x 9+32, 10, 8 - li 10, 160 - lxvd2x 10+32, 10, 8 - li 10, 176 - lxvd2x 11+32, 10, 8 - - li 10, 192 - lxvd2x 12+32, 10, 8 - li 10, 208 - lxvd2x 13+32, 10, 8 - li 10, 224 - lxvd2x 14+32, 10, 8 - - - lxvb16x 30+32, 0, 7 - - mr 12, 5 - li 11, 0 - - - vxor 31, 31, 31 - vspltisb 22,1 - vsldoi 31, 31, 22,1 - - - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - - lwz 9,240(6) - - - - xxlor 32+29, 0, 0 - vxor 15, 30, 29 - - cmpdi 9, 10 - beq .Loop_aes_gcm_8x_dec - - - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) - - cmpdi 9, 12 - beq .Loop_aes_gcm_8x_dec - - - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq .Loop_aes_gcm_8x_dec - - b aes_gcm_out - -.align 5 -.Loop_aes_gcm_8x_dec: - mr 14, 3 - mr 9, 4 - - - li 10, 128 - divdu 10, 5, 10 - cmpdi 10, 0 - beq .Loop_last_block_dec - - .long 0x13DEF8C0 - vxor 16, 30, 29 - .long 0x13DEF8C0 - vxor 17, 30, 29 - .long 0x13DEF8C0 - vxor 18, 30, 29 - .long 0x13DEF8C0 - vxor 19, 30, 29 - .long 0x13DEF8C0 - vxor 20, 30, 29 - .long 0x13DEF8C0 - vxor 21, 30, 29 - .long 0x13DEF8C0 - vxor 22, 30, 29 - - mtctr 10 - - li 15, 16 - li 16, 32 - li 17, 48 - li 18, 64 - li 19, 80 - li 20, 96 - li 21, 112 - - lwz 10, 240(6) - -.Loop_8x_block_dec: - - lxvb16x 15, 0, 14 - lxvb16x 16, 15, 14 - lxvb16x 17, 16, 14 - lxvb16x 18, 17, 14 - lxvb16x 19, 18, 14 - lxvb16x 20, 19, 14 - lxvb16x 21, 20, 14 - lxvb16x 22, 21, 14 - addi 14, 14, 128 - -.Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_last_aes_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_last_aes_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x1210BD08 - .long 0x1231BD08 - .long 0x1252BD08 - .long 0x1273BD08 - .long 0x1294BD08 - .long 0x12B5BD08 - .long 0x12D6BD08 - - .long 0x11EFC508 - .long 0x1210C508 - .long 0x1231C508 - .long 0x1252C508 - .long 0x1273C508 - .long 0x1294C508 - .long 0x12B5C508 - .long 0x12D6C508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_last_aes_dec - b aes_gcm_out - -Do_last_aes_dec: - - - - .long 0x11EFBD09 - .long 0x1210BD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 - - .long 0x1231BD09 - .long 0x1252BD09 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 - - .long 0x1273BD09 - .long 0x1294BD09 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 - - .long 0x12B5BD09 - .long 0x12D6BD09 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 - - addi 9, 9, 128 - - xxlor 15+32, 15, 15 - xxlor 16+32, 16, 16 - xxlor 17+32, 17, 17 - xxlor 18+32, 18, 18 - xxlor 19+32, 19, 19 - xxlor 20+32, 20, 20 - xxlor 21+32, 21, 21 - xxlor 22+32, 22, 22 - - - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - .long 0x13DEF8C0 - vor 29,30,30 - vxor 15, 30, 27 - .long 0x13DEF8C0 - vxor 16, 30, 27 - .long 0x13DEF8C0 - vxor 17, 30, 27 - .long 0x13DEF8C0 - vxor 18, 30, 27 - .long 0x13DEF8C0 - vxor 19, 30, 27 - .long 0x13DEF8C0 - vxor 20, 30, 27 - .long 0x13DEF8C0 - vxor 21, 30, 27 - .long 0x13DEF8C0 - vxor 22, 30, 27 - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz .Loop_8x_block_dec - - vor 30,29,29 - -.Loop_last_block_dec: - cmpdi 12, 0 - beq aes_gcm_out - - - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10,240(6) - - cmpdi 12, 16 - blt Final_block_dec - -Next_rem_block_dec: - lxvb16x 15, 0, 14 - -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_1x_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_1x_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x_dec - -Do_next_1x_dec: - .long 0x11EFBD09 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 - addi 14, 14, 16 - addi 9, 9, 16 - - xxlor 28+32, 15, 15 - ppc_update_hash_1x - - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - .long 0x13DEF8C0 - vxor 15, 30, 19 - - bdnz Next_rem_block_dec - - cmpdi 12, 0 - beq aes_gcm_out - -Final_block_dec: -.Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x_dec - - - xxlor 24+32, 11, 11 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x_dec - - - xxlor 24+32, 13, 13 - - .long 0x11EFBD08 - .long 0x11EFC508 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_final_1x_dec - -Do_final_1x_dec: - .long 0x11EFBD09 - - lxvb16x 15, 0, 14 - xxlxor 47, 47, 15 - - - li 15, 16 - sub 15, 15, 12 - - vspltisb 16,-1 - vspltisb 17,0 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 - - addi 10, 1, 192 - lxvb16x 16, 15, 10 - xxland 47, 47, 16 - - xxlor 28+32, 15, 15 - ppc_update_hash_1x - - - bl Write_partial_block - - b aes_gcm_out diff --git a/openssl/src/crypto/modes/gen/linux_riscv64/aes-gcm-riscv64-zvkb-zvkg-zvkned.s b/openssl/src/crypto/modes/gen/linux_riscv64/aes-gcm-riscv64-zvkb-zvkg-zvkned.s deleted file mode 100644 index 6c1c5e1c7..000000000 --- a/openssl/src/crypto/modes/gen/linux_riscv64/aes-gcm-riscv64-zvkb-zvkg-zvkned.s +++ /dev/null @@ -1,1540 +0,0 @@ -.text -.p2align 3 -.globl rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt -.type rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt,@function -rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt: - srli t0, a2, 4 - beqz t0, .Lenc_end - slli t5, t0, 2 - - mv a7, t5 - - # Compute the AES-GCM full-block e32 length for `LMUL=4`. We will handle - # the multiple AES-GCM blocks at the same time within `LMUL=4` register. - # The AES-GCM's SEW is e32 and EGW is 128 bits. - # FULL_BLOCK_LEN32 = (VLEN*LMUL)/(EGW) * (EGW/SEW) = (VLEN*4)/(32*4) * 4 - # = (VLEN*4)/32 - # We could get the block_num using the VL value of `vsetvli with e32, m4`. - .word 220231767 - # If `LEN32 % FULL_BLOCK_LEN32` is not equal to zero, we could fill the - # zero padding data to make sure we could always handle FULL_BLOCK_LEN32 - # blocks for all iterations. - - ## Prepare the H^n multiplier in v16 for GCM multiplier. The `n` is the gcm - ## block number in a LMUL=4 register group. - ## n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) - ## = (VLEN/32) - ## We could use vsetvli with `e32, m1` to compute the `n` number. - .word 218133207 - - # The H is at `gcm128_context.Htable[0]`(addr(Xi)+16*2). - addi t1, a5, 32 - .word 3439489111 - .word 33779591 - - # Compute the H^n - li t1, 1 -1: - .word 2750984183 - slli t1, t1, 1 - bltu t1, t0, 1b - - .word 220754007 - .word 1577072727 - .word 2817763447 - - #### Load plaintext into v24 and handle padding. We also load the init tag - #### data into v20 and prepare the AES ctr input data into v12 and v28. - .word 1577073239 - - ## Prepare the AES ctr input data into v12. - # Setup ctr input mask. - # ctr mask : [000100010001....] - # Note: The actual vl should be `FULL_BLOCK_LEN32/4 * 2`, but we just use - # `FULL_BLOCK_LEN32` here. - .word 201879639 - li t0, 0b10001000 - .word 1577238615 - # Load IV. - .word 3439489111 - .word 34041735 - # Convert the big-endian counter into little-endian. - .word 3305271383 - .word 1240772567 - # Splat the `single block of IV` to v12 - .word 220754007 - .word 1577072215 - .word 2817762935 - # Prepare the ctr counter into v8 - # v8: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...] - .word 1342710871 - # Merge IV and ctr counter into v12. - # v12:[x, x, x, count+0, x, x, x, count+1, ...] - .word 86536279 - .word 12846679 - - li t4, 0 - # Get the SEW32 size in the first round. - # If we have the non-zero value for `LEN32&(FULL_BLOCK_LEN32-1)`, then - # we will have the leading padding zero. - addi t0, a6, -1 - and t0, t0, t5 - beqz t0, 1f - - ## with padding - sub t5, t5, t0 - sub t4, a6, t0 - # padding block size - srli t1, t4, 2 - # padding byte size - slli t2, t4, 2 - - # Adjust the ctr counter to make the counter start from `counter+0` for the - # first non-padding block. - .word 86536279 - .word 147015255 - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 1577455191 - .word 1237626455 - - # Prepare the mask for input loading in the first round. We use - # `VL=FULL_BLOCK_LEN32` with the mask in the first round. - # Adjust input ptr. - sub a0, a0, t2 - # Adjust output ptr. - sub a1, a1, t2 - .word 211316823 - .word 1376297303 - # We don't use the pseudo instruction `vmsgeu` here. Use `vmsgtu` instead. - # The original code is: - # vmsgeu.vx v0, v2, t4 - addi t0, t4, -1 - .word 2049097815 - .word 220754007 - .word 1577073751 - # Load the input for length FULL_BLOCK_LEN32 with mask. - .word 86536279 - .word 355335 - - # Load the init `Xi` data to v20 with preceding zero padding. - # Adjust Xi ptr. - sub t0, a5, t2 - # Load for length `zero-padding-e32-length + 4`. - addi t1, t4, 4 - .word 19099735 - .word 190983 - j 2f - -1: - ## without padding - sub t5, t5, a6 - - .word 220754007 - .word 33909767 - - # Load the init Xi data to v20. - .word 3372380247 - .word 34073095 - - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 86536279 - .word 1577455191 - .word 1237626455 -2: - - - # Load number of rounds - lwu t0, 240(a3) - li t1, 14 - li t2, 12 - li t3, 10 - - beq t0, t1, aes_gcm_enc_blocks_256 - beq t0, t2, aes_gcm_enc_blocks_192 - beq t0, t3, aes_gcm_enc_blocks_128 - -.Lenc_end: - li a0, 0 - ret - -.size rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt,.-rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt -.p2align 3 -aes_gcm_enc_blocks_128: - srli t6, a6, 2 - slli t0, a6, 2 - - # Load all 11 aes round keys to v1-v11 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - # We already have the ciphertext/plaintext and ctr data for the first round. - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2788241015 - .word 2789289591 - .word 2790338167 - .word 2791386743 - .word 2792435319 - .word 2793483895 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr result. - .word 801902167 - - bnez t4, 1f - - ## without padding - # Store ciphertext/plaintext - .word 33943079 - j 2f - - ## with padding -1: - # Store ciphertext/plaintext using mask - .word 388647 - - # Fill zero for the padding blocks - .word 154071127 - .word 1577074263 - - # We have used mask register for `INPUT_PADDING_MASK` before. We need to - # setup the ctr mask back. - # ctr mask : [000100010001....] - .word 201879639 - li t1, 0b10001000 - .word 1577271383 -2: - - - - add a0, a0, t0 - add a1, a1, t0 - - - .word 220754007 - -.Lenc_blocks_128: - # Compute the partial tags. - # The partial tags will multiply with [H^n, H^n, ..., H^n] - # [tag0, tag1, ...] = - # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] - # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. - beqz t5, .Lenc_blocks_128_end - .word 3004050039 - - .word 86536279 - # Increase ctr in v12. - .word 13616727 - sub t5, t5, a6 - # Load plaintext into v24 - .word 220229719 - .word 33909767 - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 1577455191 - add a0, a0, t0 - .word 86011991 - .word 1237626455 - - - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2788241015 - .word 2789289591 - .word 2790338167 - .word 2791386743 - .word 2792435319 - .word 2793483895 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr ciphertext result. - .word 801902167 - - # Store ciphertext - .word 33943079 - add a1, a1, t0 - - j .Lenc_blocks_128 -.Lenc_blocks_128_end: - - # Add ciphertext into partial tag - .word 793643607 - - .word 3441586263 - # Update current ctr value to v12 - .word 13616727 - # Convert ctr to big-endian counter. - .word 1220847191 - .word 484903 - - - # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2). - # Load H to v1 - addi t1, a5, 32 - .word 3439489111 - .word 33775751 - # Multiply H for each partial tag and XOR them together. - # Handle 1st partial tag - .word 1577713751 - .word 2719522935 - # Handle 2nd to N-th partial tags - li t1, 4 -1: - .word 3441586263 - .word 1061372503 - .word 3439489111 - .word 2987532407 - addi t1, t1, 4 - blt t1, a6, 1b - - - # Save the final tag - .word 34070567 - - # return the processed size. - slli a0, a7, 2 - ret -.size aes_gcm_enc_blocks_128,.-aes_gcm_enc_blocks_128 -.p2align 3 -aes_gcm_enc_blocks_192: - srli t6, a6, 2 - slli t0, a6, 2 - - # We run out of 32 vector registers, so we just preserve some round keys - # and load the remaining round keys inside the aes body. - # We keep the round keys for: - # 1, 2, 3, 5, 6, 7, 9, 10, 11 and 12th keys. - # The following keys will be loaded in the aes body: - # 4, 8 and 13th keys. - .word 3439489111 - # key 1 - .word 34005127 - # key 2 - addi t1, a3, 16 - .word 33775879 - # key 3 - addi t1, a3, 32 - .word 33776007 - # key 5 - addi t1, a3, 64 - .word 33776135 - # key 6 - addi t1, a3, 80 - .word 33776263 - # key 7 - addi t1, a3, 96 - .word 33776391 - # key 9 - addi t1, a3, 128 - .word 33776519 - # key 10 - addi t1, a3, 144 - .word 33776647 - # key 11 - addi t1, a3, 160 - .word 33776775 - # key 12 - addi t1, a3, 176 - .word 33776903 - - # We already have the ciphertext/plaintext and ctr data for the first round. - # Load key 4 - .word 3439489111 - addi t1, a3, 48 - .word 33777031 - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2788241015 - .word 2796629623 - # Load key 8 - .word 3439489111 - addi t1, a3, 112 - .word 33777031 - .word 220754007 - .word 2789289591 - .word 2790338167 - .word 2791386743 - .word 2796629623 - # Load key 13 - .word 3439489111 - addi t1, a3, 192 - .word 33777031 - .word 220754007 - .word 2792435319 - .word 2793483895 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr result. - .word 801902167 - - bnez t4, 1f - - ## without padding - # Store ciphertext/plaintext - .word 33943079 - j 2f - - ## with padding -1: - # Store ciphertext/plaintext using mask - .word 388647 - - # Fill zero for the padding blocks - .word 154071127 - .word 1577074263 - - # We have used mask register for `INPUT_PADDING_MASK` before. We need to - # setup the ctr mask back. - # ctr mask : [000100010001....] - .word 201879639 - li t1, 0b10001000 - .word 1577271383 -2: - - - - add a0, a0, t0 - add a1, a1, t0 - - - .word 220754007 - -.Lenc_blocks_192: - # Compute the partial tags. - # The partial tags will multiply with [H^n, H^n, ..., H^n] - # [tag0, tag1, ...] = - # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] - # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. - beqz t5, .Lenc_blocks_192_end - .word 3004050039 - - .word 86536279 - # Increase ctr in v12. - .word 13616727 - sub t5, t5, a6 - # Load plaintext into v24 - .word 220229719 - .word 33909767 - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 1577455191 - add a0, a0, t0 - .word 86011991 - .word 1237626455 - - - # Load key 4 - .word 3439489111 - addi t1, a3, 48 - .word 33777031 - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2788241015 - .word 2796629623 - # Load key 8 - .word 3439489111 - addi t1, a3, 112 - .word 33777031 - .word 220754007 - .word 2789289591 - .word 2790338167 - .word 2791386743 - .word 2796629623 - # Load key 13 - .word 3439489111 - addi t1, a3, 192 - .word 33777031 - .word 220754007 - .word 2792435319 - .word 2793483895 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr ciphertext result. - .word 801902167 - - # Store ciphertext - .word 33943079 - add a1, a1, t0 - - j .Lenc_blocks_192 -.Lenc_blocks_192_end: - - # Add ciphertext into partial tag - .word 793643607 - - .word 3441586263 - # Update current ctr value to v12 - .word 13616727 - # Convert ctr to big-endian counter. - .word 1220847191 - .word 484903 - - - # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2). - # Load H to v1 - addi t1, a5, 32 - .word 3439489111 - .word 33775751 - # Multiply H for each partial tag and XOR them together. - # Handle 1st partial tag - .word 1577713751 - .word 2719522935 - # Handle 2nd to N-th partial tags - li t1, 4 -1: - .word 3441586263 - .word 1061372503 - .word 3439489111 - .word 2987532407 - addi t1, t1, 4 - blt t1, a6, 1b - - - # Save the final tag - .word 34070567 - - # return the processed size. - slli a0, a7, 2 - ret -.size aes_gcm_enc_blocks_192,.-aes_gcm_enc_blocks_192 -.p2align 3 -aes_gcm_enc_blocks_256: - srli t6, a6, 2 - slli t0, a6, 2 - - # We run out of 32 vector registers, so we just preserve some round keys - # and load the remaining round keys inside the aes body. - # We keep the round keys for: - # 1, 2, 4, 5, 7, 8, 10, 11, 13 and 14th keys. - # The following keys will be loaded in the aes body: - # 3, 6, 9, 12 and 15th keys. - .word 3439489111 - # key 1 - .word 34005127 - # key 2 - addi t1, a3, 16 - .word 33775879 - # key 4 - addi t1, a3, 48 - .word 33776007 - # key 5 - addi t1, a3, 64 - .word 33776135 - # key 7 - addi t1, a3, 96 - .word 33776263 - # key 8 - addi t1, a3, 112 - .word 33776391 - # key 10 - addi t1, a3, 144 - .word 33776519 - # key 11 - addi t1, a3, 160 - .word 33776647 - # key 13 - addi t1, a3, 192 - .word 33776775 - # key 14 - addi t1, a3, 208 - .word 33776903 - - # We already have the ciphertext/plaintext and ctr data for the first round. - # Load key 3 - .word 3439489111 - addi t1, a3, 32 - .word 33777031 - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2796629623 - # Load key 6 - .word 3439489111 - addi t1, a3, 80 - .word 33777031 - .word 220754007 - .word 2788241015 - .word 2789289591 - .word 2796629623 - # Load key 9 - .word 3439489111 - addi t1, a3, 128 - .word 33777031 - .word 220754007 - .word 2790338167 - .word 2791386743 - .word 2796629623 - # Load key 12 - .word 3439489111 - addi t1, a3, 176 - .word 33777031 - .word 220754007 - .word 2792435319 - .word 2793483895 - .word 2796629623 - # Load key 15 - .word 3439489111 - addi t1, a3, 224 - .word 33777031 - .word 220754007 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr result. - .word 801902167 - - bnez t4, 1f - - ## without padding - # Store ciphertext/plaintext - .word 33943079 - j 2f - - ## with padding -1: - # Store ciphertext/plaintext using mask - .word 388647 - - # Fill zero for the padding blocks - .word 154071127 - .word 1577074263 - - # We have used mask register for `INPUT_PADDING_MASK` before. We need to - # setup the ctr mask back. - # ctr mask : [000100010001....] - .word 201879639 - li t1, 0b10001000 - .word 1577271383 -2: - - - - add a0, a0, t0 - add a1, a1, t0 - - - .word 220754007 - -.Lenc_blocks_256: - # Compute the partial tags. - # The partial tags will multiply with [H^n, H^n, ..., H^n] - # [tag0, tag1, ...] = - # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] - # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. - beqz t5, .Lenc_blocks_256_end - .word 3004050039 - - .word 86536279 - # Increase ctr in v12. - .word 13616727 - sub t5, t5, a6 - # Load plaintext into v24 - .word 220229719 - .word 33909767 - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 1577455191 - add a0, a0, t0 - .word 86011991 - .word 1237626455 - - - # Load key 3 - .word 3439489111 - addi t1, a3, 32 - .word 33777031 - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2796629623 - # Load key 6 - .word 3439489111 - addi t1, a3, 80 - .word 33777031 - .word 220754007 - .word 2788241015 - .word 2789289591 - .word 2796629623 - # Load key 9 - .word 3439489111 - addi t1, a3, 128 - .word 33777031 - .word 220754007 - .word 2790338167 - .word 2791386743 - .word 2796629623 - # Load key 12 - .word 3439489111 - addi t1, a3, 176 - .word 33777031 - .word 220754007 - .word 2792435319 - .word 2793483895 - .word 2796629623 - # Load key 15 - .word 3439489111 - addi t1, a3, 224 - .word 33777031 - .word 220754007 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr ciphertext result. - .word 801902167 - - # Store ciphertext - .word 33943079 - add a1, a1, t0 - - j .Lenc_blocks_256 -.Lenc_blocks_256_end: - - # Add ciphertext into partial tag - .word 793643607 - - .word 3441586263 - # Update current ctr value to v12 - .word 13616727 - # Convert ctr to big-endian counter. - .word 1220847191 - .word 484903 - - - # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2). - # Load H to v1 - addi t1, a5, 32 - .word 3439489111 - .word 33775751 - # Multiply H for each partial tag and XOR them together. - # Handle 1st partial tag - .word 1577713751 - .word 2719522935 - # Handle 2nd to N-th partial tags - li t1, 4 -1: - .word 3441586263 - .word 1061372503 - .word 3439489111 - .word 2987532407 - addi t1, t1, 4 - blt t1, a6, 1b - - - # Save the final tag - .word 34070567 - - # return the processed size. - slli a0, a7, 2 - ret -.size aes_gcm_enc_blocks_256,.-aes_gcm_enc_blocks_256 -.p2align 3 -.globl rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt -.type rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt,@function -rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt: - srli t0, a2, 4 - beqz t0, .Ldec_end - slli t5, t0, 2 - - mv a7, t5 - - # Compute the AES-GCM full-block e32 length for `LMUL=4`. We will handle - # the multiple AES-GCM blocks at the same time within `LMUL=4` register. - # The AES-GCM's SEW is e32 and EGW is 128 bits. - # FULL_BLOCK_LEN32 = (VLEN*LMUL)/(EGW) * (EGW/SEW) = (VLEN*4)/(32*4) * 4 - # = (VLEN*4)/32 - # We could get the block_num using the VL value of `vsetvli with e32, m4`. - .word 220231767 - # If `LEN32 % FULL_BLOCK_LEN32` is not equal to zero, we could fill the - # zero padding data to make sure we could always handle FULL_BLOCK_LEN32 - # blocks for all iterations. - - ## Prepare the H^n multiplier in v16 for GCM multiplier. The `n` is the gcm - ## block number in a LMUL=4 register group. - ## n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) - ## = (VLEN/32) - ## We could use vsetvli with `e32, m1` to compute the `n` number. - .word 218133207 - - # The H is at `gcm128_context.Htable[0]`(addr(Xi)+16*2). - addi t1, a5, 32 - .word 3439489111 - .word 33779591 - - # Compute the H^n - li t1, 1 -1: - .word 2750984183 - slli t1, t1, 1 - bltu t1, t0, 1b - - .word 220754007 - .word 1577072727 - .word 2817763447 - - #### Load plaintext into v24 and handle padding. We also load the init tag - #### data into v20 and prepare the AES ctr input data into v12 and v28. - .word 1577073239 - - ## Prepare the AES ctr input data into v12. - # Setup ctr input mask. - # ctr mask : [000100010001....] - # Note: The actual vl should be `FULL_BLOCK_LEN32/4 * 2`, but we just use - # `FULL_BLOCK_LEN32` here. - .word 201879639 - li t0, 0b10001000 - .word 1577238615 - # Load IV. - .word 3439489111 - .word 34041735 - # Convert the big-endian counter into little-endian. - .word 3305271383 - .word 1240772567 - # Splat the `single block of IV` to v12 - .word 220754007 - .word 1577072215 - .word 2817762935 - # Prepare the ctr counter into v8 - # v8: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...] - .word 1342710871 - # Merge IV and ctr counter into v12. - # v12:[x, x, x, count+0, x, x, x, count+1, ...] - .word 86536279 - .word 12846679 - - li t4, 0 - # Get the SEW32 size in the first round. - # If we have the non-zero value for `LEN32&(FULL_BLOCK_LEN32-1)`, then - # we will have the leading padding zero. - addi t0, a6, -1 - and t0, t0, t5 - beqz t0, 1f - - ## with padding - sub t5, t5, t0 - sub t4, a6, t0 - # padding block size - srli t1, t4, 2 - # padding byte size - slli t2, t4, 2 - - # Adjust the ctr counter to make the counter start from `counter+0` for the - # first non-padding block. - .word 86536279 - .word 147015255 - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 1577455191 - .word 1237626455 - - # Prepare the mask for input loading in the first round. We use - # `VL=FULL_BLOCK_LEN32` with the mask in the first round. - # Adjust input ptr. - sub a0, a0, t2 - # Adjust output ptr. - sub a1, a1, t2 - .word 211316823 - .word 1376297303 - # We don't use the pseudo instruction `vmsgeu` here. Use `vmsgtu` instead. - # The original code is: - # vmsgeu.vx v0, v2, t4 - addi t0, t4, -1 - .word 2049097815 - .word 220754007 - .word 1577073751 - # Load the input for length FULL_BLOCK_LEN32 with mask. - .word 86536279 - .word 355335 - - # Load the init `Xi` data to v20 with preceding zero padding. - # Adjust Xi ptr. - sub t0, a5, t2 - # Load for length `zero-padding-e32-length + 4`. - addi t1, t4, 4 - .word 19099735 - .word 190983 - j 2f - -1: - ## without padding - sub t5, t5, a6 - - .word 220754007 - .word 33909767 - - # Load the init Xi data to v20. - .word 3372380247 - .word 34073095 - - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 86536279 - .word 1577455191 - .word 1237626455 -2: - - - # Load number of rounds - lwu t0, 240(a3) - li t1, 14 - li t2, 12 - li t3, 10 - - beq t0, t1, aes_gcm_dec_blocks_256 - beq t0, t2, aes_gcm_dec_blocks_192 - beq t0, t3, aes_gcm_dec_blocks_128 - -.Ldec_end: - li a0, 0 - ret -.size rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt,.-rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt -.p2align 3 -aes_gcm_dec_blocks_128: - srli t6, a6, 2 - slli t0, a6, 2 - - # Load all 11 aes round keys to v1-v11 registers. - .word 3439489111 - .word 34005127 - addi a3, a3, 16 - .word 34005255 - addi a3, a3, 16 - .word 34005383 - addi a3, a3, 16 - .word 34005511 - addi a3, a3, 16 - .word 34005639 - addi a3, a3, 16 - .word 34005767 - addi a3, a3, 16 - .word 34005895 - addi a3, a3, 16 - .word 34006023 - addi a3, a3, 16 - .word 34006151 - addi a3, a3, 16 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - - # We already have the ciphertext/plaintext and ctr data for the first round. - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2788241015 - .word 2789289591 - .word 2790338167 - .word 2791386743 - .word 2792435319 - .word 2793483895 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr result. - .word 801902167 - - bnez t4, 1f - - ## without padding - # Store ciphertext/plaintext - .word 33943079 - j 2f - - ## with padding -1: - # Store ciphertext/plaintext using mask - .word 388647 - - # Fill zero for the padding blocks - .word 154071127 - .word 1577074263 - - # We have used mask register for `INPUT_PADDING_MASK` before. We need to - # setup the ctr mask back. - # ctr mask : [000100010001....] - .word 201879639 - li t1, 0b10001000 - .word 1577271383 -2: - - - - add a0, a0, t0 - add a1, a1, t0 - - - .word 220754007 - -.Ldec_blocks_128: - # Compute the partial tags. - # The partial tags will multiply with [H^n, H^n, ..., H^n] - # [tag0, tag1, ...] = - # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] - # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. - beqz t5, .Ldec_blocks_256_end - .word 3003918967 - - .word 86536279 - # Increase ctr in v12. - .word 13616727 - sub t5, t5, a6 - # Load plaintext into v24 - .word 220229719 - .word 33909767 - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 1577455191 - add a0, a0, t0 - .word 86011991 - .word 1237626455 - - - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2788241015 - .word 2789289591 - .word 2790338167 - .word 2791386743 - .word 2792435319 - .word 2793483895 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr plaintext result. - .word 801902167 - - # Store plaintext - .word 33943079 - add a1, a1, t0 - - j .Ldec_blocks_128 -.Ldec_blocks_128_end: - - # Add ciphertext into partial tag - .word 793512535 - - .word 3441586263 - # Update current ctr value to v12 - .word 13616727 - # Convert ctr to big-endian counter. - .word 1220847191 - .word 484903 - - - # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2). - # Load H to v1 - addi t1, a5, 32 - .word 3439489111 - .word 33775751 - # Multiply H for each partial tag and XOR them together. - # Handle 1st partial tag - .word 1577713751 - .word 2719522935 - # Handle 2nd to N-th partial tags - li t1, 4 -1: - .word 3441586263 - .word 1061372503 - .word 3439489111 - .word 2987532407 - addi t1, t1, 4 - blt t1, a6, 1b - - - # Save the final tag - .word 34070567 - - # return the processed size. - slli a0, a7, 2 - ret -.size aes_gcm_dec_blocks_128,.-aes_gcm_dec_blocks_128 -.p2align 3 -aes_gcm_dec_blocks_192: - srli t6, a6, 2 - slli t0, a6, 2 - - # We run out of 32 vector registers, so we just preserve some round keys - # and load the remaining round keys inside the aes body. - # We keep the round keys for: - # 1, 2, 3, 5, 6, 7, 9, 10, 11 and 12th keys. - # The following keys will be loaded in the aes body: - # 4, 8 and 13th keys. - .word 3439489111 - # key 1 - .word 34005127 - # key 2 - addi t1, a3, 16 - .word 33775879 - # key 3 - addi t1, a3, 32 - .word 33776007 - # key 5 - addi t1, a3, 64 - .word 33776135 - # key 6 - addi t1, a3, 80 - .word 33776263 - # key 7 - addi t1, a3, 96 - .word 33776391 - # key 9 - addi t1, a3, 128 - .word 33776519 - # key 10 - addi t1, a3, 144 - .word 33776647 - # key 11 - addi t1, a3, 160 - .word 33776775 - # key 12 - addi t1, a3, 176 - .word 33776903 - - # We already have the ciphertext/plaintext and ctr data for the first round. - # Load key 4 - .word 3439489111 - addi t1, a3, 48 - .word 33777031 - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2788241015 - .word 2796629623 - # Load key 8 - .word 3439489111 - addi t1, a3, 112 - .word 33777031 - .word 220754007 - .word 2789289591 - .word 2790338167 - .word 2791386743 - .word 2796629623 - # Load key 13 - .word 3439489111 - addi t1, a3, 192 - .word 33777031 - .word 220754007 - .word 2792435319 - .word 2793483895 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr result. - .word 801902167 - - bnez t4, 1f - - ## without padding - # Store ciphertext/plaintext - .word 33943079 - j 2f - - ## with padding -1: - # Store ciphertext/plaintext using mask - .word 388647 - - # Fill zero for the padding blocks - .word 154071127 - .word 1577074263 - - # We have used mask register for `INPUT_PADDING_MASK` before. We need to - # setup the ctr mask back. - # ctr mask : [000100010001....] - .word 201879639 - li t1, 0b10001000 - .word 1577271383 -2: - - - - add a0, a0, t0 - add a1, a1, t0 - - - .word 220754007 - -.Ldec_blocks_192: - # Compute the partial tags. - # The partial tags will multiply with [H^n, H^n, ..., H^n] - # [tag0, tag1, ...] = - # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] - # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. - beqz t5, .Ldec_blocks_192_end - .word 3003918967 - - .word 86536279 - # Increase ctr in v12. - .word 13616727 - sub t5, t5, a6 - # Load plaintext into v24 - .word 220229719 - .word 33909767 - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 1577455191 - add a0, a0, t0 - .word 86011991 - .word 1237626455 - - - # Load key 4 - .word 3439489111 - addi t1, a3, 48 - .word 33777031 - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2788241015 - .word 2796629623 - # Load key 8 - .word 3439489111 - addi t1, a3, 112 - .word 33777031 - .word 220754007 - .word 2789289591 - .word 2790338167 - .word 2791386743 - .word 2796629623 - # Load key 13 - .word 3439489111 - addi t1, a3, 192 - .word 33777031 - .word 220754007 - .word 2792435319 - .word 2793483895 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr plaintext result. - .word 801902167 - - # Store plaintext - .word 33943079 - add a1, a1, t0 - - j .Ldec_blocks_192 -.Ldec_blocks_192_end: - - # Add ciphertext into partial tag - .word 793512535 - - .word 3441586263 - # Update current ctr value to v12 - .word 13616727 - # Convert ctr to big-endian counter. - .word 1220847191 - .word 484903 - - - # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2). - # Load H to v1 - addi t1, a5, 32 - .word 3439489111 - .word 33775751 - # Multiply H for each partial tag and XOR them together. - # Handle 1st partial tag - .word 1577713751 - .word 2719522935 - # Handle 2nd to N-th partial tags - li t1, 4 -1: - .word 3441586263 - .word 1061372503 - .word 3439489111 - .word 2987532407 - addi t1, t1, 4 - blt t1, a6, 1b - - - # Save the final tag - .word 34070567 - - # return the processed size. - slli a0, a7, 2 - ret -.size aes_gcm_dec_blocks_192,.-aes_gcm_dec_blocks_192 -.p2align 3 -aes_gcm_dec_blocks_256: - srli t6, a6, 2 - slli t0, a6, 2 - - # We run out of 32 vector registers, so we just preserve some round keys - # and load the remaining round keys inside the aes body. - # We keep the round keys for: - # 1, 2, 4, 5, 7, 8, 10, 11, 13 and 14th keys. - # The following keys will be loaded in the aes body: - # 3, 6, 9, 12 and 15th keys. - .word 3439489111 - # key 1 - .word 34005127 - # key 2 - addi t1, a3, 16 - .word 33775879 - # key 4 - addi t1, a3, 48 - .word 33776007 - # key 5 - addi t1, a3, 64 - .word 33776135 - # key 7 - addi t1, a3, 96 - .word 33776263 - # key 8 - addi t1, a3, 112 - .word 33776391 - # key 10 - addi t1, a3, 144 - .word 33776519 - # key 11 - addi t1, a3, 160 - .word 33776647 - # key 13 - addi t1, a3, 192 - .word 33776775 - # key 14 - addi t1, a3, 208 - .word 33776903 - - # We already have the ciphertext/plaintext and ctr data for the first round. - # Load key 3 - .word 3439489111 - addi t1, a3, 32 - .word 33777031 - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2796629623 - # Load key 6 - .word 3439489111 - addi t1, a3, 80 - .word 33777031 - .word 220754007 - .word 2788241015 - .word 2789289591 - .word 2796629623 - # Load key 9 - .word 3439489111 - addi t1, a3, 128 - .word 33777031 - .word 220754007 - .word 2790338167 - .word 2791386743 - .word 2796629623 - # Load key 12 - .word 3439489111 - addi t1, a3, 176 - .word 33777031 - .word 220754007 - .word 2792435319 - .word 2793483895 - .word 2796629623 - # Load key 15 - .word 3439489111 - addi t1, a3, 224 - .word 33777031 - .word 220754007 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr result. - .word 801902167 - - bnez t4, 1f - - ## without padding - # Store ciphertext/plaintext - .word 33943079 - j 2f - - ## with padding -1: - # Store ciphertext/plaintext using mask - .word 388647 - - # Fill zero for the padding blocks - .word 154071127 - .word 1577074263 - - # We have used mask register for `INPUT_PADDING_MASK` before. We need to - # setup the ctr mask back. - # ctr mask : [000100010001....] - .word 201879639 - li t1, 0b10001000 - .word 1577271383 -2: - - - - add a0, a0, t0 - add a1, a1, t0 - - - .word 220754007 - -.Ldec_blocks_256: - # Compute the partial tags. - # The partial tags will multiply with [H^n, H^n, ..., H^n] - # [tag0, tag1, ...] = - # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] - # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. - beqz t5, .Ldec_blocks_256_end - .word 3003918967 - - .word 86536279 - # Increase ctr in v12. - .word 13616727 - sub t5, t5, a6 - # Load plaintext into v24 - .word 220229719 - .word 33909767 - # Prepare the AES ctr input into v28. - # The ctr data uses big-endian form. - .word 1577455191 - add a0, a0, t0 - .word 86011991 - .word 1237626455 - - - # Load key 3 - .word 3439489111 - addi t1, a3, 32 - .word 33777031 - .word 220754007 - .word 2786307703 - .word 2787192439 - .word 2796629623 - # Load key 6 - .word 3439489111 - addi t1, a3, 80 - .word 33777031 - .word 220754007 - .word 2788241015 - .word 2789289591 - .word 2796629623 - # Load key 9 - .word 3439489111 - addi t1, a3, 128 - .word 33777031 - .word 220754007 - .word 2790338167 - .word 2791386743 - .word 2796629623 - # Load key 12 - .word 3439489111 - addi t1, a3, 176 - .word 33777031 - .word 220754007 - .word 2792435319 - .word 2793483895 - .word 2796629623 - # Load key 15 - .word 3439489111 - addi t1, a3, 224 - .word 33777031 - .word 220754007 - .word 2794532471 - .word 2795581047 - .word 2796662391 - - - # Compute AES ctr plaintext result. - .word 801902167 - - # Store plaintext - .word 33943079 - add a1, a1, t0 - - j .Ldec_blocks_256 -.Ldec_blocks_256_end: - - # Add ciphertext into partial tag - .word 793512535 - - .word 3441586263 - # Update current ctr value to v12 - .word 13616727 - # Convert ctr to big-endian counter. - .word 1220847191 - .word 484903 - - - # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2). - # Load H to v1 - addi t1, a5, 32 - .word 3439489111 - .word 33775751 - # Multiply H for each partial tag and XOR them together. - # Handle 1st partial tag - .word 1577713751 - .word 2719522935 - # Handle 2nd to N-th partial tags - li t1, 4 -1: - .word 3441586263 - .word 1061372503 - .word 3439489111 - .word 2987532407 - addi t1, t1, 4 - blt t1, a6, 1b - - - # Save the final tag - .word 34070567 - - # return the processed size. - slli a0, a7, 2 - ret -.size aes_gcm_dec_blocks_256,.-aes_gcm_dec_blocks_256 diff --git a/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64-zvkb-zvbc.s b/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64-zvkb-zvbc.s deleted file mode 100644 index ca549d473..000000000 --- a/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64-zvkb-zvbc.s +++ /dev/null @@ -1,268 +0,0 @@ -.text -.p2align 3 -.globl gcm_init_rv64i_zvkb_zvbc -.type gcm_init_rv64i_zvkb_zvbc,@function -gcm_init_rv64i_zvkb_zvbc: - # Load/store data in reverse order. - # This is needed as a part of endianness swap. - add a1, a1, 8 - li t0, -8 - li t1, 63 - la t2, Lpolymod - - .word 0xc1817057 # vsetivli x0, 2, e64, m1, tu, mu - - .word 173404295 # vlse64.v v1, (a1), t0 - .word 33812743 # vle64.v v2, (t2) - - # Shift one left and get the carry bits. - .word 2719171031 # vsrl.vx v3, v1, t1 - .word 2517676247 # vsll.vi v1, v1, 1 - - # Use the fact that the polynomial degree is no more than 128, - # i.e. only the LSB of the upper half could be set. - # Thanks to this we don't need to do the full reduction here. - # Instead simply subtract the reduction polynomial. - # This idea was taken from x86 ghash implementation in OpenSSL. - .word 976269911 # vslideup.vi v4, v3, 1 - .word 1043378647 # vslidedown.vi v3, v3, 1 - - .word 1577136215 # vmv.v.i v0, 2 - .word 672268503 # vor.vv v1, v1, v4, v0.t - - # Need to set the mask to 3, if the carry bit is set. - .word 1577156695 # vmv.v.v v0, v3 - .word 1577071063 # vmv.v.i v3, 0 - .word 1546760663 # vmerge.vim v3, v3, 3, v0 - .word 1577156695 # vmv.v.v v0, v3 - - .word 739311831 # vxor.vv v1, v1, v2, v0.t - - .word 33910951 # vse64.v v1, (a0) - ret -.size gcm_init_rv64i_zvkb_zvbc,.-gcm_init_rv64i_zvkb_zvbc -.text -.p2align 3 -.globl gcm_gmult_rv64i_zvkb_zvbc -.type gcm_gmult_rv64i_zvkb_zvbc,@function -gcm_gmult_rv64i_zvkb_zvbc: - ld t0, (a1) - ld t1, 8(a1) - li t2, 63 - la t3, Lpolymod - ld t3, 8(t3) - - # Load/store data in reverse order. - # This is needed as a part of endianness swap. - add a0, a0, 8 - li t4, -8 - - .word 0xc1817057 # vsetivli x0, 2, e64, m1, tu, mu - - .word 198537863 # vlse64.v v5, (a0), t4 - .word 1247060695 # vrev8.v v5, v5 - - # Multiplication - - # Do two 64x64 multiplications in one go to save some time - # and simplify things. - - # A = a1a0 (t1, t0) - # B = b1b0 (v5) - # C = c1c0 (256 bit) - # c1 = a1b1 + (a0b1)h + (a1b0)h - # c0 = a0b0 + (a0b1)l + (a1b0)h - - # v1 = (a0b1)l,(a0b0)l - .word 844292311 # vclmul.vx v1, v5, t0 - # v3 = (a0b1)h,(a0b0)h - .word 911401431 # vclmulh.vx v3, v5, t0 - - # v4 = (a1b1)l,(a1b0)l - .word 844325463 # vclmul.vx v4, v5, t1 - # v2 = (a1b1)h,(a1b0)h - .word 911434071 # vclmulh.vx v2, v5, t1 - - # Is there a better way to do this? - # Would need to swap the order of elements within a vector register. - .word 976270039 # vslideup.vi v5, v3, 1 - .word 977318743 # vslideup.vi v6, v4, 1 - .word 1043378647 # vslidedown.vi v3, v3, 1 - .word 1044427351 # vslidedown.vi v4, v4, 1 - - .word 1577103447 # vmv.v.i v0, 1 - # v2 += (a0b1)h - .word 740393303 # vxor.vv v2, v2, v3, v0.t - # v2 += (a1b1)l - .word 740426071 # vxor.vv v2, v2, v4, v0.t - - .word 1577136215 # vmv.v.i v0, 2 - # v1 += (a0b0)h,0 - .word 739410135 # vxor.vv v1, v1, v5, v0.t - # v1 += (a1b0)l,0 - .word 739442903 # vxor.vv v1, v1, v6, v0.t - - # Now the 256bit product should be stored in (v2,v1) - # v1 = (a0b1)l + (a0b0)h + (a1b0)l, (a0b0)l - # v2 = (a1b1)h, (a1b0)h + (a0b1)h + (a1b1)l - - # Reduction - # Let C := A*B = c3,c2,c1,c0 = v2[1],v2[0],v1[1],v1[0] - # This is a slight variation of the Gueron's Montgomery reduction. - # The difference being the order of some operations has been changed, - # to make a better use of vclmul(h) instructions. - - # First step: - # c1 += (c0 * P)l - # vmv.v.i v0, 2 - .word 940618199 # vslideup.vi v3, v1, 1, v0.t - .word 809394647 # vclmul.vx v3, v3, t3, v0.t - .word 739344599 # vxor.vv v1, v1, v3, v0.t - - # Second step: - # D = d1,d0 is final result - # We want: - # m1 = c1 + (c1 * P)h - # m0 = (c1 * P)l + (c0 * P)h + c0 - # d1 = c3 + m1 - # d0 = c2 + m0 - - #v3 = (c1 * P)l, 0 - .word 807297495 # vclmul.vx v3, v1, t3, v0.t - #v4 = (c1 * P)h, (c0 * P)h - .word 907960919 # vclmulh.vx v4, v1, t3 - - .word 1577103447 # vmv.v.i v0, 1 - .word 1043378647 # vslidedown.vi v3, v3, 1 - - .word 772931799 # vxor.vv v1, v1, v4 - .word 739344599 # vxor.vv v1, v1, v3, v0.t - - # XOR in the upper upper part of the product - .word 773882199 # vxor.vv v2, v2, v1 - - .word 1243914583 # vrev8.v v2, v2 - .word 198537511 # vsse64.v v2, (a0), t4 - ret -.size gcm_gmult_rv64i_zvkb_zvbc,.-gcm_gmult_rv64i_zvkb_zvbc -.p2align 3 -.globl gcm_ghash_rv64i_zvkb_zvbc -.type gcm_ghash_rv64i_zvkb_zvbc,@function -gcm_ghash_rv64i_zvkb_zvbc: - ld t0, (a1) - ld t1, 8(a1) - li t2, 63 - la t3, Lpolymod - ld t3, 8(t3) - - # Load/store data in reverse order. - # This is needed as a part of endianness swap. - add a0, a0, 8 - add a2, a2, 8 - li t4, -8 - - .word 0xc1817057 # vsetivli x0, 2, e64, m1, tu, mu - - .word 198537863 # vlse64.v v5, (a0), t4 - -Lstep: - # Read input data - .word 198603655 # vle64.v v0, (a2) - add a2, a2, 16 - add a3, a3, -16 - # XOR them into Xi - .word 777224919 # vxor.vv v0, v0, v1 - - .word 1247060695 # vrev8.v v5, v5 - - # Multiplication - - # Do two 64x64 multiplications in one go to save some time - # and simplify things. - - # A = a1a0 (t1, t0) - # B = b1b0 (v5) - # C = c1c0 (256 bit) - # c1 = a1b1 + (a0b1)h + (a1b0)h - # c0 = a0b0 + (a0b1)l + (a1b0)h - - # v1 = (a0b1)l,(a0b0)l - .word 844292311 # vclmul.vx v1, v5, t0 - # v3 = (a0b1)h,(a0b0)h - .word 911401431 # vclmulh.vx v3, v5, t0 - - # v4 = (a1b1)l,(a1b0)l - .word 844325463 # vclmul.vx v4, v5, t1 - # v2 = (a1b1)h,(a1b0)h - .word 911434071 # vclmulh.vx v2, v5, t1 - - # Is there a better way to do this? - # Would need to swap the order of elements within a vector register. - .word 976270039 # vslideup.vi v5, v3, 1 - .word 977318743 # vslideup.vi v6, v4, 1 - .word 1043378647 # vslidedown.vi v3, v3, 1 - .word 1044427351 # vslidedown.vi v4, v4, 1 - - .word 1577103447 # vmv.v.i v0, 1 - # v2 += (a0b1)h - .word 740393303 # vxor.vv v2, v2, v3, v0.t - # v2 += (a1b1)l - .word 740426071 # vxor.vv v2, v2, v4, v0.t - - .word 1577136215 # vmv.v.i v0, 2 - # v1 += (a0b0)h,0 - .word 739410135 # vxor.vv v1, v1, v5, v0.t - # v1 += (a1b0)l,0 - .word 739442903 # vxor.vv v1, v1, v6, v0.t - - # Now the 256bit product should be stored in (v2,v1) - # v1 = (a0b1)l + (a0b0)h + (a1b0)l, (a0b0)l - # v2 = (a1b1)h, (a1b0)h + (a0b1)h + (a1b1)l - - # Reduction - # Let C := A*B = c3,c2,c1,c0 = v2[1],v2[0],v1[1],v1[0] - # This is a slight variation of the Gueron's Montgomery reduction. - # The difference being the order of some operations has been changed, - # to make a better use of vclmul(h) instructions. - - # First step: - # c1 += (c0 * P)l - # vmv.v.i v0, 2 - .word 940618199 # vslideup.vi v3, v1, 1, v0.t - .word 809394647 # vclmul.vx v3, v3, t3, v0.t - .word 739344599 # vxor.vv v1, v1, v3, v0.t - - # Second step: - # D = d1,d0 is final result - # We want: - # m1 = c1 + (c1 * P)h - # m0 = (c1 * P)l + (c0 * P)h + c0 - # d1 = c3 + m1 - # d0 = c2 + m0 - - #v3 = (c1 * P)l, 0 - .word 807297495 # vclmul.vx v3, v1, t3, v0.t - #v4 = (c1 * P)h, (c0 * P)h - .word 907960919 # vclmulh.vx v4, v1, t3 - - .word 1577103447 # vmv.v.i v0, 1 - .word 1043378647 # vslidedown.vi v3, v3, 1 - - .word 772931799 # vxor.vv v1, v1, v4 - .word 739344599 # vxor.vv v1, v1, v3, v0.t - - # XOR in the upper upper part of the product - .word 773882199 # vxor.vv v2, v2, v1 - - .word 1243914967 # vrev8.v v2, v2 - - bnez a3, Lstep - - .word 198537895 # vsse64.v v2, (a0), t4 - ret -.size gcm_ghash_rv64i_zvkb_zvbc,.-gcm_ghash_rv64i_zvkb_zvbc -.p2align 4 -Lpolymod: - .dword 0x0000000000000001 - .dword 0xc200000000000000 -.size Lpolymod,.-Lpolymod diff --git a/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64-zvkg.s b/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64-zvkg.s deleted file mode 100644 index 759c7c9c9..000000000 --- a/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64-zvkg.s +++ /dev/null @@ -1,81 +0,0 @@ -.text -.p2align 3 -.globl gcm_init_rv64i_zvkg -.type gcm_init_rv64i_zvkg,@function -gcm_init_rv64i_zvkg: - ld a2, 0(a1) - ld a3, 8(a1) - sb a2, 7(a0) - srli t0, a2, 8 - sb t0, 6(a0) - srli t0, a2, 16 - sb t0, 5(a0) - srli t0, a2, 24 - sb t0, 4(a0) - srli t0, a2, 32 - sb t0, 3(a0) - srli t0, a2, 40 - sb t0, 2(a0) - srli t0, a2, 48 - sb t0, 1(a0) - srli t0, a2, 56 - sb t0, 0(a0) - - sb a3, 15(a0) - srli t0, a3, 8 - sb t0, 14(a0) - srli t0, a3, 16 - sb t0, 13(a0) - srli t0, a3, 24 - sb t0, 12(a0) - srli t0, a3, 32 - sb t0, 11(a0) - srli t0, a3, 40 - sb t0, 10(a0) - srli t0, a3, 48 - sb t0, 9(a0) - srli t0, a3, 56 - sb t0, 8(a0) - - ret -.size gcm_init_rv64i_zvkg,.-gcm_init_rv64i_zvkg -.p2align 3 -.globl gcm_init_rv64i_zvkg_zvkb -.type gcm_init_rv64i_zvkg_zvkb,@function -gcm_init_rv64i_zvkg_zvkb: - .word 0xc1817057 # vsetivli x0, 2, e64, m1, ta, ma - .word 33943559 # vle64.v v0, (a1) - .word 1241817175 # vrev8.v v0, v0 - .word 33910823 # vse64.v v0, (a0) - ret -.size gcm_init_rv64i_zvkg_zvkb,.-gcm_init_rv64i_zvkg_zvkb -.p2align 3 -.globl gcm_gmult_rv64i_zvkg -.type gcm_gmult_rv64i_zvkg,@function -gcm_gmult_rv64i_zvkg: - .word 0xc1027057 - .word 33939719 - .word 33906823 - .word 2720571639 - .word 33906855 - ret -.size gcm_gmult_rv64i_zvkg,.-gcm_gmult_rv64i_zvkg -.p2align 3 -.globl gcm_ghash_rv64i_zvkg -.type gcm_ghash_rv64i_zvkg,@function -gcm_ghash_rv64i_zvkg: - .word 0xc1027057 - .word 33939719 - .word 33906823 - -Lstep: - .word 33972615 - add a2, a2, 16 - add a3, a3, -16 - .word 2988548343 - bnez a3, Lstep - - .word 33906855 - ret - -.size gcm_ghash_rv64i_zvkg,.-gcm_ghash_rv64i_zvkg diff --git a/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64.s b/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64.s deleted file mode 100644 index 337766973..000000000 --- a/openssl/src/crypto/modes/gen/linux_riscv64/ghash-riscv64.s +++ /dev/null @@ -1,613 +0,0 @@ -.text -.p2align 3 -.globl gcm_init_rv64i_zbc -.type gcm_init_rv64i_zbc,@function -gcm_init_rv64i_zbc: - ld a2,0(a1) - ld a3,8(a1) - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a2, 1 - and t1, t1, t0 - and a2, a2, t0 - srli a2, a2, 1 - or a2, t1, a2 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a2, 2 - and t1, t1, t0 - and a2, a2, t0 - srli a2, a2, 2 - or a2, t1, a2 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a2, 4 - and t1, t1, t0 - and a2, a2, t0 - srli a2, a2, 4 - or a2, t1, a2 - - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a3, 1 - and t1, t1, t0 - and a3, a3, t0 - srli a3, a3, 1 - or a3, t1, a3 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a3, 2 - and t1, t1, t0 - and a3, a3, t0 - srli a3, a3, 2 - or a3, t1, a3 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a3, 4 - and t1, t1, t0 - and a3, a3, t0 - srli a3, a3, 4 - or a3, t1, a3 - - sb a2, 7(a0) - srli t0, a2, 8 - sb t0, 6(a0) - srli t0, a2, 16 - sb t0, 5(a0) - srli t0, a2, 24 - sb t0, 4(a0) - srli t0, a2, 32 - sb t0, 3(a0) - srli t0, a2, 40 - sb t0, 2(a0) - srli t0, a2, 48 - sb t0, 1(a0) - srli t0, a2, 56 - sb t0, 0(a0) - - sb a3, 15(a0) - srli t0, a3, 8 - sb t0, 14(a0) - srli t0, a3, 16 - sb t0, 13(a0) - srli t0, a3, 24 - sb t0, 12(a0) - srli t0, a3, 32 - sb t0, 11(a0) - srli t0, a3, 40 - sb t0, 10(a0) - srli t0, a3, 48 - sb t0, 9(a0) - srli t0, a3, 56 - sb t0, 8(a0) - - ret -.size gcm_init_rv64i_zbc,.-gcm_init_rv64i_zbc -.p2align 3 -.globl gcm_init_rv64i_zbc__zbb -.type gcm_init_rv64i_zbc__zbb,@function -gcm_init_rv64i_zbc__zbb: - ld a2,0(a1) - ld a3,8(a1) - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a2, 1 - and t1, t1, t0 - and a2, a2, t0 - srli a2, a2, 1 - or a2, t1, a2 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a2, 2 - and t1, t1, t0 - and a2, a2, t0 - srli a2, a2, 2 - or a2, t1, a2 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a2, 4 - and t1, t1, t0 - and a2, a2, t0 - srli a2, a2, 4 - or a2, t1, a2 - - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a3, 1 - and t1, t1, t0 - and a3, a3, t0 - srli a3, a3, 1 - or a3, t1, a3 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a3, 2 - and t1, t1, t0 - and a3, a3, t0 - srli a3, a3, 2 - or a3, t1, a3 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a3, 4 - and t1, t1, t0 - and a3, a3, t0 - srli a3, a3, 4 - or a3, t1, a3 - - .word 1803965971 - .word 1803998867 - sd a2,0(a0) - sd a3,8(a0) - ret -.size gcm_init_rv64i_zbc__zbb,.-gcm_init_rv64i_zbc__zbb -.p2align 3 -.globl gcm_init_rv64i_zbc__zbkb -.type gcm_init_rv64i_zbc__zbkb,@function -gcm_init_rv64i_zbc__zbkb: - ld t0,0(a1) - ld t1,8(a1) - .word 1752355475 - .word 1752388371 - .word 1803735699 - .word 1803768595 - sd t0,0(a0) - sd t1,8(a0) - ret -.size gcm_init_rv64i_zbc__zbkb,.-gcm_init_rv64i_zbc__zbkb -.p2align 3 -.globl gcm_gmult_rv64i_zbc -.type gcm_gmult_rv64i_zbc,@function -gcm_gmult_rv64i_zbc: - # Load Xi and bit-reverse it - ld a4, 0(a0) - ld a5, 8(a0) - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a4, 1 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 1 - or a4, t1, a4 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a4, 2 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 2 - or a4, t1, a4 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a4, 4 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 4 - or a4, t1, a4 - - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a5, 1 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 1 - or a5, t1, a5 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a5, 2 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 2 - or a5, t1, a5 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a5, 4 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 4 - or a5, t1, a5 - - - # Load the key (already bit-reversed) - ld a6, 0(a1) - ld a7, 8(a1) - - # Load the reduction constant - la t6, Lpolymod - lbu t6, 0(t6) - - # Multiplication (without Karatsuba) - .word 186105395 - .word 186094515 - .word 186072883 - .word 186061619 - xor t2, t2, t5 - .word 185057075 - .word 185048755 - xor t2, t2, t5 - xor t1, t1, t4 - .word 185024307 - .word 185012915 - xor t1, t1, t5 - - # Reduction with clmul - .word 201211699 - .word 201203379 - xor t2, t2, t5 - xor t1, t1, t4 - .word 200523571 - .word 200515251 - xor a5, t1, t5 - xor a4, t0, t4 - - # Bit-reverse Xi back and store it - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a4, 1 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 1 - or a4, t1, a4 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a4, 2 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 2 - or a4, t1, a4 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a4, 4 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 4 - or a4, t1, a4 - - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a5, 1 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 1 - or a5, t1, a5 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a5, 2 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 2 - or a5, t1, a5 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a5, 4 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 4 - or a5, t1, a5 - - sd a4, 0(a0) - sd a5, 8(a0) - ret -.size gcm_gmult_rv64i_zbc,.-gcm_gmult_rv64i_zbc -.p2align 3 -.globl gcm_gmult_rv64i_zbc__zbkb -.type gcm_gmult_rv64i_zbc__zbkb,@function -gcm_gmult_rv64i_zbc__zbkb: - # Load Xi and bit-reverse it - ld a4, 0(a0) - ld a5, 8(a0) - .word 1752651539 - .word 1752684435 - - # Load the key (already bit-reversed) - ld a6, 0(a1) - ld a7, 8(a1) - - # Load the reduction constant - la t6, Lpolymod - lbu t6, 0(t6) - - # Multiplication (without Karatsuba) - .word 186105395 - .word 186094515 - .word 186072883 - .word 186061619 - xor t2, t2, t5 - .word 185057075 - .word 185048755 - xor t2, t2, t5 - xor t1, t1, t4 - .word 185024307 - .word 185012915 - xor t1, t1, t5 - - # Reduction with clmul - .word 201211699 - .word 201203379 - xor t2, t2, t5 - xor t1, t1, t4 - .word 200523571 - .word 200515251 - xor a5, t1, t5 - xor a4, t0, t4 - - # Bit-reverse Xi back and store it - .word 1752651539 - .word 1752684435 - sd a4, 0(a0) - sd a5, 8(a0) - ret -.size gcm_gmult_rv64i_zbc__zbkb,.-gcm_gmult_rv64i_zbc__zbkb -.p2align 3 -.globl gcm_ghash_rv64i_zbc -.type gcm_ghash_rv64i_zbc,@function -gcm_ghash_rv64i_zbc: - # Load Xi and bit-reverse it - ld a4, 0(a0) - ld a5, 8(a0) - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a4, 1 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 1 - or a4, t1, a4 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a4, 2 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 2 - or a4, t1, a4 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a4, 4 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 4 - or a4, t1, a4 - - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a5, 1 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 1 - or a5, t1, a5 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a5, 2 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 2 - or a5, t1, a5 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a5, 4 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 4 - or a5, t1, a5 - - - # Load the key (already bit-reversed) - ld a6, 0(a1) - ld a7, 8(a1) - - # Load the reduction constant - la t6, Lpolymod - lbu t6, 0(t6) - -Lstep: - # Load the input data, bit-reverse them, and XOR them with Xi - ld t4, 0(a2) - ld t5, 8(a2) - add a2, a2, 16 - add a3, a3, -16 - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, t4, 1 - and t1, t1, t0 - and t4, t4, t0 - srli t4, t4, 1 - or t4, t1, t4 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, t4, 2 - and t1, t1, t0 - and t4, t4, t0 - srli t4, t4, 2 - or t4, t1, t4 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, t4, 4 - and t1, t1, t0 - and t4, t4, t0 - srli t4, t4, 4 - or t4, t1, t4 - - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, t5, 1 - and t1, t1, t0 - and t5, t5, t0 - srli t5, t5, 1 - or t5, t1, t5 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, t5, 2 - and t1, t1, t0 - and t5, t5, t0 - srli t5, t5, 2 - or t5, t1, t5 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, t5, 4 - and t1, t1, t0 - and t5, t5, t0 - srli t5, t5, 4 - or t5, t1, t5 - - xor a4, a4, t4 - xor a5, a5, t5 - - # Multiplication (without Karatsuba) - .word 186105395 - .word 186094515 - .word 186072883 - .word 186061619 - xor t2, t2, t5 - .word 185057075 - .word 185048755 - xor t2, t2, t5 - xor t1, t1, t4 - .word 185024307 - .word 185012915 - xor t1, t1, t5 - - # Reduction with clmul - .word 201211699 - .word 201203379 - xor t2, t2, t5 - xor t1, t1, t4 - .word 200523571 - .word 200515251 - xor a5, t1, t5 - xor a4, t0, t4 - - # Iterate over all blocks - bnez a3, Lstep - - # Bit-reverse final Xi back and store it - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a4, 1 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 1 - or a4, t1, a4 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a4, 2 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 2 - or a4, t1, a4 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a4, 4 - and t1, t1, t0 - and a4, a4, t0 - srli a4, a4, 4 - or a4, t1, a4 - - la t2, Lbrev8_const - - ld t0, 0(t2) # 0xAAAAAAAAAAAAAAAA - slli t1, a5, 1 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 1 - or a5, t1, a5 - - ld t0, 8(t2) # 0xCCCCCCCCCCCCCCCC - slli t1, a5, 2 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 2 - or a5, t1, a5 - - ld t0, 16(t2) # 0xF0F0F0F0F0F0F0F0 - slli t1, a5, 4 - and t1, t1, t0 - and a5, a5, t0 - srli a5, a5, 4 - or a5, t1, a5 - - sd a4, 0(a0) - sd a5, 8(a0) - ret -.size gcm_ghash_rv64i_zbc,.-gcm_ghash_rv64i_zbc -.p2align 3 -.globl gcm_ghash_rv64i_zbc__zbkb -.type gcm_ghash_rv64i_zbc__zbkb,@function -gcm_ghash_rv64i_zbc__zbkb: - # Load Xi and bit-reverse it - ld a4, 0(a0) - ld a5, 8(a0) - .word 1752651539 - .word 1752684435 - - # Load the key (already bit-reversed) - ld a6, 0(a1) - ld a7, 8(a1) - - # Load the reduction constant - la t6, Lpolymod - lbu t6, 0(t6) - -Lstep_zkbk: - # Load the input data, bit-reverse them, and XOR them with Xi - ld t4, 0(a2) - ld t5, 8(a2) - add a2, a2, 16 - add a3, a3, -16 - .word 1753144979 - .word 1753177875 - xor a4, a4, t4 - xor a5, a5, t5 - - # Multiplication (without Karatsuba) - .word 186105395 - .word 186094515 - .word 186072883 - .word 186061619 - xor t2, t2, t5 - .word 185057075 - .word 185048755 - xor t2, t2, t5 - xor t1, t1, t4 - .word 185024307 - .word 185012915 - xor t1, t1, t5 - - # Reduction with clmul - .word 201211699 - .word 201203379 - xor t2, t2, t5 - xor t1, t1, t4 - .word 200523571 - .word 200515251 - xor a5, t1, t5 - xor a4, t0, t4 - - # Iterate over all blocks - bnez a3, Lstep_zkbk - - # Bit-reverse final Xi back and store it - .word 1752651539 - .word 1752684435 - sd a4, 0(a0) - sd a5, 8(a0) - ret -.size gcm_ghash_rv64i_zbc__zbkb,.-gcm_ghash_rv64i_zbc__zbkb -.p2align 3 -Lbrev8_const: - .dword 0xAAAAAAAAAAAAAAAA - .dword 0xCCCCCCCCCCCCCCCC - .dword 0xF0F0F0F0F0F0F0F0 -.size Lbrev8_const,.-Lbrev8_const - -Lpolymod: - .byte 0x87 -.size Lpolymod,.-Lpolymod diff --git a/openssl/src/crypto/modes/gen/linux_x64/aes-gcm-avx512.s b/openssl/src/crypto/modes/gen/linux_x64/aes-gcm-avx512.s deleted file mode 100644 index 19ddba6df..000000000 --- a/openssl/src/crypto/modes/gen/linux_x64/aes-gcm-avx512.s +++ /dev/null @@ -1,136131 +0,0 @@ - -.globl ossl_vaes_vpclmulqdq_capable -.type ossl_vaes_vpclmulqdq_capable,@function -.align 32 -ossl_vaes_vpclmulqdq_capable: - movq OPENSSL_ia32cap_P+8(%rip),%rcx - - movq $6600291188736,%rdx - xorl %eax,%eax - andq %rdx,%rcx - cmpq %rdx,%rcx - cmoveq %rcx,%rax - .byte 0xf3,0xc3 -.size ossl_vaes_vpclmulqdq_capable, .-ossl_vaes_vpclmulqdq_capable -.text -.globl ossl_aes_gcm_init_avx512 -.type ossl_aes_gcm_init_avx512,@function -.align 32 -ossl_aes_gcm_init_avx512: -.cfi_startproc -.byte 243,15,30,250 - vpxorq %xmm16,%xmm16,%xmm16 - - - movl 240(%rdi),%eax - cmpl $9,%eax - je .Laes_128_0 - cmpl $11,%eax - je .Laes_192_0 - cmpl $13,%eax - je .Laes_256_0 - jmp .Lexit_aes_0 -.align 32 -.Laes_128_0: - vpxorq 0(%rdi),%xmm16,%xmm16 - - vaesenc 16(%rdi),%xmm16,%xmm16 - - vaesenc 32(%rdi),%xmm16,%xmm16 - - vaesenc 48(%rdi),%xmm16,%xmm16 - - vaesenc 64(%rdi),%xmm16,%xmm16 - - vaesenc 80(%rdi),%xmm16,%xmm16 - - vaesenc 96(%rdi),%xmm16,%xmm16 - - vaesenc 112(%rdi),%xmm16,%xmm16 - - vaesenc 128(%rdi),%xmm16,%xmm16 - - vaesenc 144(%rdi),%xmm16,%xmm16 - - vaesenclast 160(%rdi),%xmm16,%xmm16 - jmp .Lexit_aes_0 -.align 32 -.Laes_192_0: - vpxorq 0(%rdi),%xmm16,%xmm16 - - vaesenc 16(%rdi),%xmm16,%xmm16 - - vaesenc 32(%rdi),%xmm16,%xmm16 - - vaesenc 48(%rdi),%xmm16,%xmm16 - - vaesenc 64(%rdi),%xmm16,%xmm16 - - vaesenc 80(%rdi),%xmm16,%xmm16 - - vaesenc 96(%rdi),%xmm16,%xmm16 - - vaesenc 112(%rdi),%xmm16,%xmm16 - - vaesenc 128(%rdi),%xmm16,%xmm16 - - vaesenc 144(%rdi),%xmm16,%xmm16 - - vaesenc 160(%rdi),%xmm16,%xmm16 - - vaesenc 176(%rdi),%xmm16,%xmm16 - - vaesenclast 192(%rdi),%xmm16,%xmm16 - jmp .Lexit_aes_0 -.align 32 -.Laes_256_0: - vpxorq 0(%rdi),%xmm16,%xmm16 - - vaesenc 16(%rdi),%xmm16,%xmm16 - - vaesenc 32(%rdi),%xmm16,%xmm16 - - vaesenc 48(%rdi),%xmm16,%xmm16 - - vaesenc 64(%rdi),%xmm16,%xmm16 - - vaesenc 80(%rdi),%xmm16,%xmm16 - - vaesenc 96(%rdi),%xmm16,%xmm16 - - vaesenc 112(%rdi),%xmm16,%xmm16 - - vaesenc 128(%rdi),%xmm16,%xmm16 - - vaesenc 144(%rdi),%xmm16,%xmm16 - - vaesenc 160(%rdi),%xmm16,%xmm16 - - vaesenc 176(%rdi),%xmm16,%xmm16 - - vaesenc 192(%rdi),%xmm16,%xmm16 - - vaesenc 208(%rdi),%xmm16,%xmm16 - - vaesenclast 224(%rdi),%xmm16,%xmm16 - jmp .Lexit_aes_0 -.Lexit_aes_0: - - vpshufb SHUF_MASK(%rip),%xmm16,%xmm16 - - vmovdqa64 %xmm16,%xmm2 - vpsllq $1,%xmm16,%xmm16 - vpsrlq $63,%xmm2,%xmm2 - vmovdqa %xmm2,%xmm1 - vpslldq $8,%xmm2,%xmm2 - vpsrldq $8,%xmm1,%xmm1 - vporq %xmm2,%xmm16,%xmm16 - - vpshufd $36,%xmm1,%xmm2 - vpcmpeqd TWOONE(%rip),%xmm2,%xmm2 - vpand POLY(%rip),%xmm2,%xmm2 - vpxorq %xmm2,%xmm16,%xmm16 - - vmovdqu64 %xmm16,336(%rsi) - vshufi32x4 $0x00,%ymm16,%ymm16,%ymm4 - vmovdqa %ymm4,%ymm3 - - vpclmulqdq $0x11,%ymm4,%ymm3,%ymm0 - vpclmulqdq $0x00,%ymm4,%ymm3,%ymm1 - vpclmulqdq $0x01,%ymm4,%ymm3,%ymm2 - vpclmulqdq $0x10,%ymm4,%ymm3,%ymm3 - vpxorq %ymm2,%ymm3,%ymm3 - - vpsrldq $8,%ymm3,%ymm2 - vpslldq $8,%ymm3,%ymm3 - vpxorq %ymm2,%ymm0,%ymm0 - vpxorq %ymm1,%ymm3,%ymm3 - - - - vmovdqu64 POLY2(%rip),%ymm2 - - vpclmulqdq $0x01,%ymm3,%ymm2,%ymm1 - vpslldq $8,%ymm1,%ymm1 - vpxorq %ymm1,%ymm3,%ymm3 - - - - vpclmulqdq $0x00,%ymm3,%ymm2,%ymm1 - vpsrldq $4,%ymm1,%ymm1 - vpclmulqdq $0x10,%ymm3,%ymm2,%ymm3 - vpslldq $4,%ymm3,%ymm3 - - vpternlogq $0x96,%ymm1,%ymm0,%ymm3 - - vmovdqu64 %xmm3,320(%rsi) - vinserti64x2 $1,%xmm16,%ymm3,%ymm4 - vmovdqa64 %ymm4,%ymm5 - - vpclmulqdq $0x11,%ymm3,%ymm4,%ymm0 - vpclmulqdq $0x00,%ymm3,%ymm4,%ymm1 - vpclmulqdq $0x01,%ymm3,%ymm4,%ymm2 - vpclmulqdq $0x10,%ymm3,%ymm4,%ymm4 - vpxorq %ymm2,%ymm4,%ymm4 - - vpsrldq $8,%ymm4,%ymm2 - vpslldq $8,%ymm4,%ymm4 - vpxorq %ymm2,%ymm0,%ymm0 - vpxorq %ymm1,%ymm4,%ymm4 - - - - vmovdqu64 POLY2(%rip),%ymm2 - - vpclmulqdq $0x01,%ymm4,%ymm2,%ymm1 - vpslldq $8,%ymm1,%ymm1 - vpxorq %ymm1,%ymm4,%ymm4 - - - - vpclmulqdq $0x00,%ymm4,%ymm2,%ymm1 - vpsrldq $4,%ymm1,%ymm1 - vpclmulqdq $0x10,%ymm4,%ymm2,%ymm4 - vpslldq $4,%ymm4,%ymm4 - - vpternlogq $0x96,%ymm1,%ymm0,%ymm4 - - vmovdqu64 %ymm4,288(%rsi) - - vinserti64x4 $1,%ymm5,%zmm4,%zmm4 - - - vshufi64x2 $0x00,%zmm4,%zmm4,%zmm3 - vmovdqa64 %zmm4,%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm0 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm1 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm2 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm2,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm2 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm2,%zmm0,%zmm0 - vpxorq %zmm1,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm2 - - vpclmulqdq $0x01,%zmm4,%zmm2,%zmm1 - vpslldq $8,%zmm1,%zmm1 - vpxorq %zmm1,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm2,%zmm1 - vpsrldq $4,%zmm1,%zmm1 - vpclmulqdq $0x10,%zmm4,%zmm2,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm1,%zmm0,%zmm4 - - vmovdqu64 %zmm4,224(%rsi) - vshufi64x2 $0x00,%zmm4,%zmm4,%zmm3 - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm0 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm1 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm2 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm2,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm2 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm2,%zmm0,%zmm0 - vpxorq %zmm1,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm2 - - vpclmulqdq $0x01,%zmm5,%zmm2,%zmm1 - vpslldq $8,%zmm1,%zmm1 - vpxorq %zmm1,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm2,%zmm1 - vpsrldq $4,%zmm1,%zmm1 - vpclmulqdq $0x10,%zmm5,%zmm2,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm1,%zmm0,%zmm5 - - vmovdqu64 %zmm5,160(%rsi) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm0 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm1 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm2 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm2,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm2 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm2,%zmm0,%zmm0 - vpxorq %zmm1,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm2 - - vpclmulqdq $0x01,%zmm4,%zmm2,%zmm1 - vpslldq $8,%zmm1,%zmm1 - vpxorq %zmm1,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm2,%zmm1 - vpsrldq $4,%zmm1,%zmm1 - vpclmulqdq $0x10,%zmm4,%zmm2,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm1,%zmm0,%zmm4 - - vmovdqu64 %zmm4,96(%rsi) - vzeroupper -.Labort_init: - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_aes_gcm_init_avx512, .-ossl_aes_gcm_init_avx512 -.globl ossl_aes_gcm_setiv_avx512 -.type ossl_aes_gcm_setiv_avx512,@function -.align 32 -ossl_aes_gcm_setiv_avx512: -.cfi_startproc -.Lsetiv_seh_begin: -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 -.Lsetiv_seh_push_rbx: - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 -.Lsetiv_seh_push_rbp: - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 -.Lsetiv_seh_push_r12: - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 -.Lsetiv_seh_push_r13: - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 -.Lsetiv_seh_push_r14: - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lsetiv_seh_push_r15: - - - - - - - - - - - leaq 0(%rsp),%rbp -.cfi_def_cfa_register %rbp -.Lsetiv_seh_setfp: - -.Lsetiv_seh_prolog_end: - subq $820,%rsp - andq $(-64),%rsp - cmpq $12,%rcx - je iv_len_12_init_IV - vpxor %xmm2,%xmm2,%xmm2 - movq %rdx,%r10 - movq %rcx,%r11 - orq %r11,%r11 - jz .L_CALC_AAD_done_1 - - xorq %rbx,%rbx - vmovdqa64 SHUF_MASK(%rip),%zmm16 - -.L_get_AAD_loop48x16_1: - cmpq $768,%r11 - jl .L_exit_AAD_loop48x16_1 - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - testq %rbx,%rbx - jnz .L_skip_hkeys_precomputation_2 - - vmovdqu64 288(%rsi),%zmm1 - vmovdqu64 %zmm1,704(%rsp) - - vmovdqu64 224(%rsi),%zmm9 - vmovdqu64 %zmm9,640(%rsp) - - - vshufi64x2 $0x00,%zmm9,%zmm9,%zmm9 - - vmovdqu64 160(%rsi),%zmm10 - vmovdqu64 %zmm10,576(%rsp) - - vmovdqu64 96(%rsi),%zmm12 - vmovdqu64 %zmm12,512(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,448(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,384(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,320(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,256(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,192(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,128(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,64(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,0(%rsp) -.L_skip_hkeys_precomputation_2: - movq $1,%rbx - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 0(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 64(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 128(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 192(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 256(%r10),%zmm11 - vmovdqu64 320(%r10),%zmm3 - vmovdqu64 384(%r10),%zmm4 - vmovdqu64 448(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 256(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 320(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 384(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 448(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 512(%r10),%zmm11 - vmovdqu64 576(%r10),%zmm3 - vmovdqu64 640(%r10),%zmm4 - vmovdqu64 704(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 512(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 576(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 640(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 704(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm6,%xmm9,%xmm2 - - subq $768,%r11 - je .L_CALC_AAD_done_1 - - addq $768,%r10 - jmp .L_get_AAD_loop48x16_1 - -.L_exit_AAD_loop48x16_1: - - cmpq $512,%r11 - jl .L_less_than_32x16_1 - - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - testq %rbx,%rbx - jnz .L_skip_hkeys_precomputation_3 - - vmovdqu64 288(%rsi),%zmm1 - vmovdqu64 %zmm1,704(%rsp) - - vmovdqu64 224(%rsi),%zmm9 - vmovdqu64 %zmm9,640(%rsp) - - - vshufi64x2 $0x00,%zmm9,%zmm9,%zmm9 - - vmovdqu64 160(%rsi),%zmm10 - vmovdqu64 %zmm10,576(%rsp) - - vmovdqu64 96(%rsi),%zmm12 - vmovdqu64 %zmm12,512(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,448(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,384(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,320(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,256(%rsp) -.L_skip_hkeys_precomputation_3: - movq $1,%rbx - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 256(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 320(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 384(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 448(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 256(%r10),%zmm11 - vmovdqu64 320(%r10),%zmm3 - vmovdqu64 384(%r10),%zmm4 - vmovdqu64 448(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 512(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 576(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 640(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 704(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm6,%xmm9,%xmm2 - - subq $512,%r11 - je .L_CALC_AAD_done_1 - - addq $512,%r10 - jmp .L_less_than_16x16_1 - -.L_less_than_32x16_1: - cmpq $256,%r11 - jl .L_less_than_16x16_1 - - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 96(%rsi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 160(%rsi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 224(%rsi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 288(%rsi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm6,%xmm9,%xmm2 - - subq $256,%r11 - je .L_CALC_AAD_done_1 - - addq $256,%r10 - -.L_less_than_16x16_1: - - leaq byte64_len_to_mask_table(%rip),%r12 - leaq (%r12,%r11,8),%r12 - - - addl $15,%r11d - shrl $4,%r11d - cmpl $2,%r11d - jb .L_AAD_blocks_1_1 - je .L_AAD_blocks_2_1 - cmpl $4,%r11d - jb .L_AAD_blocks_3_1 - je .L_AAD_blocks_4_1 - cmpl $6,%r11d - jb .L_AAD_blocks_5_1 - je .L_AAD_blocks_6_1 - cmpl $8,%r11d - jb .L_AAD_blocks_7_1 - je .L_AAD_blocks_8_1 - cmpl $10,%r11d - jb .L_AAD_blocks_9_1 - je .L_AAD_blocks_10_1 - cmpl $12,%r11d - jb .L_AAD_blocks_11_1 - je .L_AAD_blocks_12_1 - cmpl $14,%r11d - jb .L_AAD_blocks_13_1 - je .L_AAD_blocks_14_1 - cmpl $15,%r11d - je .L_AAD_blocks_15_1 -.L_AAD_blocks_16_1: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%zmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 96(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 160(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 224(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm9,%zmm11,%zmm1 - vpternlogq $0x96,%zmm10,%zmm3,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm12,%zmm11,%zmm7 - vpternlogq $0x96,%zmm13,%zmm3,%zmm8 - vmovdqu64 288(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm5,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm5,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm5,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm5,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_15_1: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%zmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 112(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 176(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 240(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 304(%rsi),%ymm15 - vinserti64x2 $2,336(%rsi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm5,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm5,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm5,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm5,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_14_1: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%ymm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %ymm16,%ymm5,%ymm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 128(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 192(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 256(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 320(%rsi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm5,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm5,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm5,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm5,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_13_1: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%xmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %xmm16,%xmm5,%xmm5 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 144(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 208(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 272(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 336(%rsi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm5,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm5,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm5,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm5,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_12_1: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 160(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 224(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 288(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_11_1: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 176(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 240(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 304(%rsi),%ymm15 - vinserti64x2 $2,336(%rsi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_10_1: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%ymm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %ymm16,%ymm4,%ymm4 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 192(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 256(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 320(%rsi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm4,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm4,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm4,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm4,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_9_1: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%xmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %xmm16,%xmm4,%xmm4 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 208(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 272(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 336(%rsi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm4,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm4,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm4,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm4,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_8_1: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 224(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 288(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_7_1: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 240(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 304(%rsi),%ymm15 - vinserti64x2 $2,336(%rsi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_6_1: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%ymm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %ymm16,%ymm3,%ymm3 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 256(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 320(%rsi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm3,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm3,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm3,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm3,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_5_1: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%xmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %xmm16,%xmm3,%xmm3 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 272(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 336(%rsi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm3,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm3,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm3,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm3,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_4_1: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 288(%rsi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_3_1: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 304(%rsi),%ymm15 - vinserti64x2 $2,336(%rsi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_2_1: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%ymm11{%k1}{z} - vpshufb %ymm16,%ymm11,%ymm11 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 320(%rsi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm11,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm11,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm11,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm11,%ymm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - - jmp .L_CALC_AAD_done_1 -.L_AAD_blocks_1_1: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%xmm11{%k1}{z} - vpshufb %xmm16,%xmm11,%xmm11 - vpxorq %zmm2,%zmm11,%zmm11 - vmovdqu64 336(%rsi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm11,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm11,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm11,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm11,%xmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm2 - vpslldq $4,%xmm2,%xmm2 - vpternlogq $0x96,%xmm1,%xmm8,%xmm2 - -.L_CALC_AAD_done_1: - movq %rcx,%r10 - shlq $3,%r10 - vmovq %r10,%xmm3 - - - vpxorq %xmm2,%xmm3,%xmm2 - - vmovdqu64 336(%rsi),%xmm1 - - vpclmulqdq $0x11,%xmm1,%xmm2,%xmm11 - vpclmulqdq $0x00,%xmm1,%xmm2,%xmm3 - vpclmulqdq $0x01,%xmm1,%xmm2,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm2,%xmm2 - vpxorq %xmm4,%xmm2,%xmm2 - - vpsrldq $8,%xmm2,%xmm4 - vpslldq $8,%xmm2,%xmm2 - vpxorq %xmm4,%xmm11,%xmm11 - vpxorq %xmm3,%xmm2,%xmm2 - - - - vmovdqu64 POLY2(%rip),%xmm4 - - vpclmulqdq $0x01,%xmm2,%xmm4,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm2,%xmm2 - - - - vpclmulqdq $0x00,%xmm2,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm2,%xmm4,%xmm2 - vpslldq $4,%xmm2,%xmm2 - - vpternlogq $0x96,%xmm3,%xmm11,%xmm2 - - vpshufb SHUF_MASK(%rip),%xmm2,%xmm2 - jmp skip_iv_len_12_init_IV -iv_len_12_init_IV: - - vmovdqu8 ONEf(%rip),%xmm2 - movq %rdx,%r11 - movl $0x0000000000000fff,%r10d - kmovq %r10,%k1 - vmovdqu8 (%r11),%xmm2{%k1} -skip_iv_len_12_init_IV: - vmovdqu %xmm2,%xmm1 - - - movl 240(%rdi),%r10d - cmpl $9,%r10d - je .Laes_128_4 - cmpl $11,%r10d - je .Laes_192_4 - cmpl $13,%r10d - je .Laes_256_4 - jmp .Lexit_aes_4 -.align 32 -.Laes_128_4: - vpxorq 0(%rdi),%xmm1,%xmm1 - - vaesenc 16(%rdi),%xmm1,%xmm1 - - vaesenc 32(%rdi),%xmm1,%xmm1 - - vaesenc 48(%rdi),%xmm1,%xmm1 - - vaesenc 64(%rdi),%xmm1,%xmm1 - - vaesenc 80(%rdi),%xmm1,%xmm1 - - vaesenc 96(%rdi),%xmm1,%xmm1 - - vaesenc 112(%rdi),%xmm1,%xmm1 - - vaesenc 128(%rdi),%xmm1,%xmm1 - - vaesenc 144(%rdi),%xmm1,%xmm1 - - vaesenclast 160(%rdi),%xmm1,%xmm1 - jmp .Lexit_aes_4 -.align 32 -.Laes_192_4: - vpxorq 0(%rdi),%xmm1,%xmm1 - - vaesenc 16(%rdi),%xmm1,%xmm1 - - vaesenc 32(%rdi),%xmm1,%xmm1 - - vaesenc 48(%rdi),%xmm1,%xmm1 - - vaesenc 64(%rdi),%xmm1,%xmm1 - - vaesenc 80(%rdi),%xmm1,%xmm1 - - vaesenc 96(%rdi),%xmm1,%xmm1 - - vaesenc 112(%rdi),%xmm1,%xmm1 - - vaesenc 128(%rdi),%xmm1,%xmm1 - - vaesenc 144(%rdi),%xmm1,%xmm1 - - vaesenc 160(%rdi),%xmm1,%xmm1 - - vaesenc 176(%rdi),%xmm1,%xmm1 - - vaesenclast 192(%rdi),%xmm1,%xmm1 - jmp .Lexit_aes_4 -.align 32 -.Laes_256_4: - vpxorq 0(%rdi),%xmm1,%xmm1 - - vaesenc 16(%rdi),%xmm1,%xmm1 - - vaesenc 32(%rdi),%xmm1,%xmm1 - - vaesenc 48(%rdi),%xmm1,%xmm1 - - vaesenc 64(%rdi),%xmm1,%xmm1 - - vaesenc 80(%rdi),%xmm1,%xmm1 - - vaesenc 96(%rdi),%xmm1,%xmm1 - - vaesenc 112(%rdi),%xmm1,%xmm1 - - vaesenc 128(%rdi),%xmm1,%xmm1 - - vaesenc 144(%rdi),%xmm1,%xmm1 - - vaesenc 160(%rdi),%xmm1,%xmm1 - - vaesenc 176(%rdi),%xmm1,%xmm1 - - vaesenc 192(%rdi),%xmm1,%xmm1 - - vaesenc 208(%rdi),%xmm1,%xmm1 - - vaesenclast 224(%rdi),%xmm1,%xmm1 - jmp .Lexit_aes_4 -.Lexit_aes_4: - - vmovdqu %xmm1,32(%rsi) - - - vpshufb SHUF_MASK(%rip),%xmm2,%xmm2 - vmovdqu %xmm2,0(%rsi) - cmpq $256,%rcx - jbe .Lskip_hkeys_cleanup_5 - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %zmm0,0(%rsp) - vmovdqa64 %zmm0,64(%rsp) - vmovdqa64 %zmm0,128(%rsp) - vmovdqa64 %zmm0,192(%rsp) - vmovdqa64 %zmm0,256(%rsp) - vmovdqa64 %zmm0,320(%rsp) - vmovdqa64 %zmm0,384(%rsp) - vmovdqa64 %zmm0,448(%rsp) - vmovdqa64 %zmm0,512(%rsp) - vmovdqa64 %zmm0,576(%rsp) - vmovdqa64 %zmm0,640(%rsp) - vmovdqa64 %zmm0,704(%rsp) -.Lskip_hkeys_cleanup_5: - vzeroupper - leaq (%rbp),%rsp -.cfi_def_cfa_register %rsp - popq %r15 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r15 - popq %r14 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r14 - popq %r13 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r13 - popq %r12 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r12 - popq %rbp -.cfi_adjust_cfa_offset -8 -.cfi_restore %rbp - popq %rbx -.cfi_adjust_cfa_offset -8 -.cfi_restore %rbx -.Labort_setiv: - .byte 0xf3,0xc3 -.Lsetiv_seh_end: -.cfi_endproc -.size ossl_aes_gcm_setiv_avx512, .-ossl_aes_gcm_setiv_avx512 -.globl ossl_aes_gcm_update_aad_avx512 -.type ossl_aes_gcm_update_aad_avx512,@function -.align 32 -ossl_aes_gcm_update_aad_avx512: -.cfi_startproc -.Lghash_seh_begin: -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 -.Lghash_seh_push_rbx: - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 -.Lghash_seh_push_rbp: - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 -.Lghash_seh_push_r12: - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 -.Lghash_seh_push_r13: - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 -.Lghash_seh_push_r14: - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lghash_seh_push_r15: - - - - - - - - - - - leaq 0(%rsp),%rbp -.cfi_def_cfa_register %rbp -.Lghash_seh_setfp: - -.Lghash_seh_prolog_end: - subq $820,%rsp - andq $(-64),%rsp - vmovdqu64 64(%rdi),%xmm14 - movq %rsi,%r10 - movq %rdx,%r11 - orq %r11,%r11 - jz .L_CALC_AAD_done_6 - - xorq %rbx,%rbx - vmovdqa64 SHUF_MASK(%rip),%zmm16 - -.L_get_AAD_loop48x16_6: - cmpq $768,%r11 - jl .L_exit_AAD_loop48x16_6 - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - testq %rbx,%rbx - jnz .L_skip_hkeys_precomputation_7 - - vmovdqu64 288(%rdi),%zmm1 - vmovdqu64 %zmm1,704(%rsp) - - vmovdqu64 224(%rdi),%zmm9 - vmovdqu64 %zmm9,640(%rsp) - - - vshufi64x2 $0x00,%zmm9,%zmm9,%zmm9 - - vmovdqu64 160(%rdi),%zmm10 - vmovdqu64 %zmm10,576(%rsp) - - vmovdqu64 96(%rdi),%zmm12 - vmovdqu64 %zmm12,512(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,448(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,384(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,320(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,256(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,192(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,128(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,64(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,0(%rsp) -.L_skip_hkeys_precomputation_7: - movq $1,%rbx - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 0(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 64(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 128(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 192(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 256(%r10),%zmm11 - vmovdqu64 320(%r10),%zmm3 - vmovdqu64 384(%r10),%zmm4 - vmovdqu64 448(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 256(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 320(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 384(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 448(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 512(%r10),%zmm11 - vmovdqu64 576(%r10),%zmm3 - vmovdqu64 640(%r10),%zmm4 - vmovdqu64 704(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 512(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 576(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 640(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 704(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm6,%xmm9,%xmm14 - - subq $768,%r11 - je .L_CALC_AAD_done_6 - - addq $768,%r10 - jmp .L_get_AAD_loop48x16_6 - -.L_exit_AAD_loop48x16_6: - - cmpq $512,%r11 - jl .L_less_than_32x16_6 - - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - testq %rbx,%rbx - jnz .L_skip_hkeys_precomputation_8 - - vmovdqu64 288(%rdi),%zmm1 - vmovdqu64 %zmm1,704(%rsp) - - vmovdqu64 224(%rdi),%zmm9 - vmovdqu64 %zmm9,640(%rsp) - - - vshufi64x2 $0x00,%zmm9,%zmm9,%zmm9 - - vmovdqu64 160(%rdi),%zmm10 - vmovdqu64 %zmm10,576(%rsp) - - vmovdqu64 96(%rdi),%zmm12 - vmovdqu64 %zmm12,512(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,448(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,384(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm10,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm10,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm10,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm10,%zmm10 - vpxorq %zmm17,%zmm10,%zmm10 - - vpsrldq $8,%zmm10,%zmm17 - vpslldq $8,%zmm10,%zmm10 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm10,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm10,%zmm10 - - - - vpclmulqdq $0x00,%zmm10,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm10,%zmm17,%zmm10 - vpslldq $4,%zmm10,%zmm10 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm10 - - vmovdqu64 %zmm10,320(%rsp) - - vpclmulqdq $0x11,%zmm9,%zmm12,%zmm13 - vpclmulqdq $0x00,%zmm9,%zmm12,%zmm15 - vpclmulqdq $0x01,%zmm9,%zmm12,%zmm17 - vpclmulqdq $0x10,%zmm9,%zmm12,%zmm12 - vpxorq %zmm17,%zmm12,%zmm12 - - vpsrldq $8,%zmm12,%zmm17 - vpslldq $8,%zmm12,%zmm12 - vpxorq %zmm17,%zmm13,%zmm13 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vmovdqu64 POLY2(%rip),%zmm17 - - vpclmulqdq $0x01,%zmm12,%zmm17,%zmm15 - vpslldq $8,%zmm15,%zmm15 - vpxorq %zmm15,%zmm12,%zmm12 - - - - vpclmulqdq $0x00,%zmm12,%zmm17,%zmm15 - vpsrldq $4,%zmm15,%zmm15 - vpclmulqdq $0x10,%zmm12,%zmm17,%zmm12 - vpslldq $4,%zmm12,%zmm12 - - vpternlogq $0x96,%zmm15,%zmm13,%zmm12 - - vmovdqu64 %zmm12,256(%rsp) -.L_skip_hkeys_precomputation_8: - movq $1,%rbx - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 256(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 320(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 384(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 448(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 256(%r10),%zmm11 - vmovdqu64 320(%r10),%zmm3 - vmovdqu64 384(%r10),%zmm4 - vmovdqu64 448(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vmovdqu64 512(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 576(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 640(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 704(%rsp),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm6,%xmm9,%xmm14 - - subq $512,%r11 - je .L_CALC_AAD_done_6 - - addq $512,%r10 - jmp .L_less_than_16x16_6 - -.L_less_than_32x16_6: - cmpq $256,%r11 - jl .L_less_than_16x16_6 - - vmovdqu64 0(%r10),%zmm11 - vmovdqu64 64(%r10),%zmm3 - vmovdqu64 128(%r10),%zmm4 - vmovdqu64 192(%r10),%zmm5 - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 96(%rdi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm11,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm11,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm11,%zmm12 - vmovdqu64 160(%rdi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm3,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm3,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm3,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm3,%zmm18 - vpxorq %zmm17,%zmm10,%zmm7 - vpxorq %zmm13,%zmm1,%zmm6 - vpxorq %zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - vmovdqu64 224(%rdi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm19,%zmm4,%zmm9 - vpclmulqdq $0x01,%zmm19,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm19,%zmm4,%zmm12 - vmovdqu64 288(%rdi),%zmm19 - vpclmulqdq $0x11,%zmm19,%zmm5,%zmm13 - vpclmulqdq $0x00,%zmm19,%zmm5,%zmm15 - vpclmulqdq $0x01,%zmm19,%zmm5,%zmm17 - vpclmulqdq $0x10,%zmm19,%zmm5,%zmm18 - - vpternlogq $0x96,%zmm17,%zmm10,%zmm7 - vpternlogq $0x96,%zmm13,%zmm1,%zmm6 - vpternlogq $0x96,%zmm15,%zmm9,%zmm8 - vpternlogq $0x96,%zmm18,%zmm12,%zmm7 - - vpsrldq $8,%zmm7,%zmm1 - vpslldq $8,%zmm7,%zmm9 - vpxorq %zmm1,%zmm6,%zmm6 - vpxorq %zmm9,%zmm8,%zmm8 - vextracti64x4 $1,%zmm6,%ymm1 - vpxorq %ymm1,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm1 - vpxorq %xmm1,%xmm6,%xmm6 - vextracti64x4 $1,%zmm8,%ymm9 - vpxorq %ymm9,%ymm8,%ymm8 - vextracti32x4 $1,%ymm8,%xmm9 - vpxorq %xmm9,%xmm8,%xmm8 - vmovdqa64 POLY2(%rip),%xmm10 - - - vpclmulqdq $0x01,%xmm8,%xmm10,%xmm1 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm1,%xmm8,%xmm1 - - - vpclmulqdq $0x00,%xmm1,%xmm10,%xmm9 - vpsrldq $4,%xmm9,%xmm9 - vpclmulqdq $0x10,%xmm1,%xmm10,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm6,%xmm9,%xmm14 - - subq $256,%r11 - je .L_CALC_AAD_done_6 - - addq $256,%r10 - -.L_less_than_16x16_6: - - leaq byte64_len_to_mask_table(%rip),%r12 - leaq (%r12,%r11,8),%r12 - - - addl $15,%r11d - shrl $4,%r11d - cmpl $2,%r11d - jb .L_AAD_blocks_1_6 - je .L_AAD_blocks_2_6 - cmpl $4,%r11d - jb .L_AAD_blocks_3_6 - je .L_AAD_blocks_4_6 - cmpl $6,%r11d - jb .L_AAD_blocks_5_6 - je .L_AAD_blocks_6_6 - cmpl $8,%r11d - jb .L_AAD_blocks_7_6 - je .L_AAD_blocks_8_6 - cmpl $10,%r11d - jb .L_AAD_blocks_9_6 - je .L_AAD_blocks_10_6 - cmpl $12,%r11d - jb .L_AAD_blocks_11_6 - je .L_AAD_blocks_12_6 - cmpl $14,%r11d - jb .L_AAD_blocks_13_6 - je .L_AAD_blocks_14_6 - cmpl $15,%r11d - je .L_AAD_blocks_15_6 -.L_AAD_blocks_16_6: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%zmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 96(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 160(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 224(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm9,%zmm11,%zmm1 - vpternlogq $0x96,%zmm10,%zmm3,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm12,%zmm11,%zmm7 - vpternlogq $0x96,%zmm13,%zmm3,%zmm8 - vmovdqu64 288(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm5,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm5,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm5,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm5,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_15_6: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%zmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %zmm16,%zmm5,%zmm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 112(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 176(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 240(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 304(%rdi),%ymm15 - vinserti64x2 $2,336(%rdi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm5,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm5,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm5,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm5,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_14_6: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%ymm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %ymm16,%ymm5,%ymm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 128(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 192(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 256(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 320(%rdi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm5,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm5,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm5,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm5,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_13_6: - subq $1536,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4 - vmovdqu8 192(%r10),%xmm5{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpshufb %xmm16,%xmm5,%xmm5 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 144(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 208(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 272(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - vmovdqu64 336(%rdi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm5,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm5,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm5,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm5,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_12_6: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 160(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 224(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vmovdqu64 288(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm1,%zmm11,%zmm9 - vpternlogq $0x96,%zmm6,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm11 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm3 - vpternlogq $0x96,%zmm7,%zmm11,%zmm12 - vpternlogq $0x96,%zmm8,%zmm3,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_11_6: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%zmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %zmm16,%zmm4,%zmm4 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 176(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 240(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 304(%rdi),%ymm15 - vinserti64x2 $2,336(%rdi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm4,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm4,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm4,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm4,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_10_6: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%ymm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %ymm16,%ymm4,%ymm4 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 192(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 256(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 320(%rdi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm4,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm4,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm4,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm4,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_9_6: - subq $1024,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3 - vmovdqu8 128(%r10),%xmm4{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpshufb %xmm16,%xmm4,%xmm4 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 208(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 272(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - vmovdqu64 336(%rdi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm4,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm4,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm4,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm4,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_8_6: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 224(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vmovdqu64 288(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm13 - vpxorq %zmm9,%zmm1,%zmm9 - vpxorq %zmm10,%zmm6,%zmm10 - vpxorq %zmm12,%zmm7,%zmm12 - vpxorq %zmm13,%zmm8,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_7_6: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%zmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %zmm16,%zmm3,%zmm3 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 240(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 304(%rdi),%ymm15 - vinserti64x2 $2,336(%rdi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm3,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm3,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm3,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm3,%zmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_6_6: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%ymm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %ymm16,%ymm3,%ymm3 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 256(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 320(%rdi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm3,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm3,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm3,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm3,%ymm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_5_6: - subq $512,%r12 - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11 - vmovdqu8 64(%r10),%xmm3{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpshufb %xmm16,%xmm3,%xmm3 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 272(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - vmovdqu64 336(%rdi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm3,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm3,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm3,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm3,%xmm6 - - vpxorq %zmm12,%zmm7,%zmm7 - vpxorq %zmm13,%zmm8,%zmm8 - vpxorq %zmm9,%zmm1,%zmm1 - vpxorq %zmm10,%zmm6,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_4_6: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 288(%rdi),%zmm15 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm9 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm10 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm12 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm13 - - vpxorq %zmm13,%zmm12,%zmm12 - vpsrldq $8,%zmm12,%zmm7 - vpslldq $8,%zmm12,%zmm8 - vpxorq %zmm7,%zmm9,%zmm1 - vpxorq %zmm8,%zmm10,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_3_6: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%zmm11{%k1}{z} - vpshufb %zmm16,%zmm11,%zmm11 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 304(%rdi),%ymm15 - vinserti64x2 $2,336(%rdi),%zmm15,%zmm15 - vpclmulqdq $0x01,%zmm15,%zmm11,%zmm7 - vpclmulqdq $0x10,%zmm15,%zmm11,%zmm8 - vpclmulqdq $0x11,%zmm15,%zmm11,%zmm1 - vpclmulqdq $0x00,%zmm15,%zmm11,%zmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_2_6: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%ymm11{%k1}{z} - vpshufb %ymm16,%ymm11,%ymm11 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 320(%rdi),%ymm15 - vpclmulqdq $0x01,%ymm15,%ymm11,%ymm7 - vpclmulqdq $0x10,%ymm15,%ymm11,%ymm8 - vpclmulqdq $0x11,%ymm15,%ymm11,%ymm1 - vpclmulqdq $0x00,%ymm15,%ymm11,%ymm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - - jmp .L_CALC_AAD_done_6 -.L_AAD_blocks_1_6: - kmovq (%r12),%k1 - vmovdqu8 0(%r10),%xmm11{%k1}{z} - vpshufb %xmm16,%xmm11,%xmm11 - vpxorq %zmm14,%zmm11,%zmm11 - vmovdqu64 336(%rdi),%xmm15 - vpclmulqdq $0x01,%xmm15,%xmm11,%xmm7 - vpclmulqdq $0x10,%xmm15,%xmm11,%xmm8 - vpclmulqdq $0x11,%xmm15,%xmm11,%xmm1 - vpclmulqdq $0x00,%xmm15,%xmm11,%xmm6 - - vpxorq %zmm8,%zmm7,%zmm7 - vpsrldq $8,%zmm7,%zmm12 - vpslldq $8,%zmm7,%zmm13 - vpxorq %zmm12,%zmm1,%zmm1 - vpxorq %zmm13,%zmm6,%zmm6 - vextracti64x4 $1,%zmm1,%ymm12 - vpxorq %ymm12,%ymm1,%ymm1 - vextracti32x4 $1,%ymm1,%xmm12 - vpxorq %xmm12,%xmm1,%xmm1 - vextracti64x4 $1,%zmm6,%ymm13 - vpxorq %ymm13,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm13 - vpxorq %xmm13,%xmm6,%xmm6 - vmovdqa64 POLY2(%rip),%xmm15 - - - vpclmulqdq $0x01,%xmm6,%xmm15,%xmm7 - vpslldq $8,%xmm7,%xmm7 - vpxorq %xmm7,%xmm6,%xmm7 - - - vpclmulqdq $0x00,%xmm7,%xmm15,%xmm8 - vpsrldq $4,%xmm8,%xmm8 - vpclmulqdq $0x10,%xmm7,%xmm15,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm1,%xmm8,%xmm14 - -.L_CALC_AAD_done_6: - vmovdqu64 %xmm14,64(%rdi) - cmpq $256,%rdx - jbe .Lskip_hkeys_cleanup_9 - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %zmm0,0(%rsp) - vmovdqa64 %zmm0,64(%rsp) - vmovdqa64 %zmm0,128(%rsp) - vmovdqa64 %zmm0,192(%rsp) - vmovdqa64 %zmm0,256(%rsp) - vmovdqa64 %zmm0,320(%rsp) - vmovdqa64 %zmm0,384(%rsp) - vmovdqa64 %zmm0,448(%rsp) - vmovdqa64 %zmm0,512(%rsp) - vmovdqa64 %zmm0,576(%rsp) - vmovdqa64 %zmm0,640(%rsp) - vmovdqa64 %zmm0,704(%rsp) -.Lskip_hkeys_cleanup_9: - vzeroupper - leaq (%rbp),%rsp -.cfi_def_cfa_register %rsp - popq %r15 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r15 - popq %r14 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r14 - popq %r13 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r13 - popq %r12 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r12 - popq %rbp -.cfi_adjust_cfa_offset -8 -.cfi_restore %rbp - popq %rbx -.cfi_adjust_cfa_offset -8 -.cfi_restore %rbx -.Lexit_update_aad: - .byte 0xf3,0xc3 -.Lghash_seh_end: -.cfi_endproc -.size ossl_aes_gcm_update_aad_avx512, .-ossl_aes_gcm_update_aad_avx512 -.globl ossl_aes_gcm_encrypt_avx512 -.type ossl_aes_gcm_encrypt_avx512,@function -.align 32 -ossl_aes_gcm_encrypt_avx512: -.cfi_startproc -.Lencrypt_seh_begin: -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 -.Lencrypt_seh_push_rbx: - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 -.Lencrypt_seh_push_rbp: - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 -.Lencrypt_seh_push_r12: - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 -.Lencrypt_seh_push_r13: - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 -.Lencrypt_seh_push_r14: - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lencrypt_seh_push_r15: - - - - - - - - - - - leaq 0(%rsp),%rbp -.cfi_def_cfa_register %rbp -.Lencrypt_seh_setfp: - -.Lencrypt_seh_prolog_end: - subq $1588,%rsp - andq $(-64),%rsp - - - movl 240(%rdi),%eax - cmpl $9,%eax - je .Laes_gcm_encrypt_128_avx512 - cmpl $11,%eax - je .Laes_gcm_encrypt_192_avx512 - cmpl $13,%eax - je .Laes_gcm_encrypt_256_avx512 - xorl %eax,%eax - jmp .Lexit_gcm_encrypt -.align 32 -.Laes_gcm_encrypt_128_avx512: - orq %r8,%r8 - je .L_enc_dec_done_10 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je .L_partial_block_done_11 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge .L_no_extra_mask_11 - subq %r13,%r12 -.L_no_extra_mask_11: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm3,%xmm14,%xmm14 - cmpq $0,%r13 - jl .L_partial_incomplete_11 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp .L_enc_dec_done_11 - -.L_partial_incomplete_11: - addq %r8,(%rdx) - movq %r8,%r11 - -.L_enc_dec_done_11: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -.L_partial_block_done_11: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je .L_enc_dec_done_10 - cmpq $256,%r8 - jbe .L_message_below_equal_16_blocks_10 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_12 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_12 -.L_next_16_overflow_12: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_12: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_13 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -.L_skip_hkeys_precomputation_13: - cmpq $512,%r8 - jb .L_message_below_32_blocks_10 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_14 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_14 -.L_next_16_overflow_14: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_14: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_15 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -.L_skip_hkeys_precomputation_15: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb .L_no_more_big_nblocks_10 -.L_encrypt_big_nblocks_10: - cmpb $240,%r15b - jae .L_16_blocks_overflow_16 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_16 -.L_16_blocks_overflow_16: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_16: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_17 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_17 -.L_16_blocks_overflow_17: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_17: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_18 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_18 -.L_16_blocks_overflow_18: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_18: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae .L_encrypt_big_nblocks_10 - -.L_no_more_big_nblocks_10: - - cmpq $512,%r8 - jae .L_encrypt_32_blocks_10 - - cmpq $256,%r8 - jae .L_encrypt_16_blocks_10 -.L_encrypt_0_blocks_ghash_32_10: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_19 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_19 - jb .L_last_num_blocks_is_7_1_19 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_19 - jb .L_last_num_blocks_is_11_9_19 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_19 - ja .L_last_num_blocks_is_16_19 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_19 - jmp .L_last_num_blocks_is_13_19 - -.L_last_num_blocks_is_11_9_19: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_19 - ja .L_last_num_blocks_is_11_19 - jmp .L_last_num_blocks_is_9_19 - -.L_last_num_blocks_is_7_1_19: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_19 - jb .L_last_num_blocks_is_3_1_19 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_19 - je .L_last_num_blocks_is_6_19 - jmp .L_last_num_blocks_is_5_19 - -.L_last_num_blocks_is_3_1_19: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_19 - je .L_last_num_blocks_is_2_19 -.L_last_num_blocks_is_1_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_20 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_20 - -.L_16_blocks_overflow_20: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_20: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_21 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_21 -.L_small_initial_partial_block_21: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_21 -.L_small_initial_compute_done_21: -.L_after_reduction_21: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_2_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_22 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_22 - -.L_16_blocks_overflow_22: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_22: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_23 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_23 -.L_small_initial_partial_block_23: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_23: - - orq %r8,%r8 - je .L_after_reduction_23 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_23: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_3_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_24 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_24 - -.L_16_blocks_overflow_24: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_24: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_25 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_25 -.L_small_initial_partial_block_25: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_25: - - orq %r8,%r8 - je .L_after_reduction_25 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_25: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_4_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_26 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_26 - -.L_16_blocks_overflow_26: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_26: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_27 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_27 -.L_small_initial_partial_block_27: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_27: - - orq %r8,%r8 - je .L_after_reduction_27 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_27: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_5_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_28 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_28 - -.L_16_blocks_overflow_28: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_28: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_29 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_29 -.L_small_initial_partial_block_29: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_29: - - orq %r8,%r8 - je .L_after_reduction_29 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_29: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_6_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_30 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_30 - -.L_16_blocks_overflow_30: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_30: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_31 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_31 -.L_small_initial_partial_block_31: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_31: - - orq %r8,%r8 - je .L_after_reduction_31 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_31: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_7_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_32 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_32 - -.L_16_blocks_overflow_32: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_32: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_33 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_33 -.L_small_initial_partial_block_33: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_33: - - orq %r8,%r8 - je .L_after_reduction_33 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_33: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_8_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_34 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_34 - -.L_16_blocks_overflow_34: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_34: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_35 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_35 -.L_small_initial_partial_block_35: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_35: - - orq %r8,%r8 - je .L_after_reduction_35 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_35: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_9_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_36 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_36 - -.L_16_blocks_overflow_36: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_36: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_37 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_37 -.L_small_initial_partial_block_37: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_37: - - orq %r8,%r8 - je .L_after_reduction_37 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_37: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_10_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_38 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_38 - -.L_16_blocks_overflow_38: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_38: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_39 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_39 -.L_small_initial_partial_block_39: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_39: - - orq %r8,%r8 - je .L_after_reduction_39 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_39: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_11_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_40 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_40 - -.L_16_blocks_overflow_40: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_40: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_41 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_41 -.L_small_initial_partial_block_41: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_41: - - orq %r8,%r8 - je .L_after_reduction_41 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_41: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_12_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_42 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_42 - -.L_16_blocks_overflow_42: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_42: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_43 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_43 -.L_small_initial_partial_block_43: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_43: - - orq %r8,%r8 - je .L_after_reduction_43 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_43: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_13_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_44 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_44 - -.L_16_blocks_overflow_44: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_44: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_45 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_45 -.L_small_initial_partial_block_45: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_45: - - orq %r8,%r8 - je .L_after_reduction_45 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_45: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_14_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_46 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_46 - -.L_16_blocks_overflow_46: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_46: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_47 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_47 -.L_small_initial_partial_block_47: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_47: - - orq %r8,%r8 - je .L_after_reduction_47 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_47: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_15_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_48 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_48 - -.L_16_blocks_overflow_48: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_48: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_49 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_49 -.L_small_initial_partial_block_49: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_49: - - orq %r8,%r8 - je .L_after_reduction_49 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_49: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_16_19: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_50 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_50 - -.L_16_blocks_overflow_50: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_50: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_51: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_51: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_51: - jmp .L_last_blocks_done_19 -.L_last_num_blocks_is_0_19: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_19: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_10 -.L_encrypt_32_blocks_10: - cmpb $240,%r15b - jae .L_16_blocks_overflow_52 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_52 -.L_16_blocks_overflow_52: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_52: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_53 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_53 -.L_16_blocks_overflow_53: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_53: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_54 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_54 - jb .L_last_num_blocks_is_7_1_54 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_54 - jb .L_last_num_blocks_is_11_9_54 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_54 - ja .L_last_num_blocks_is_16_54 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_54 - jmp .L_last_num_blocks_is_13_54 - -.L_last_num_blocks_is_11_9_54: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_54 - ja .L_last_num_blocks_is_11_54 - jmp .L_last_num_blocks_is_9_54 - -.L_last_num_blocks_is_7_1_54: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_54 - jb .L_last_num_blocks_is_3_1_54 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_54 - je .L_last_num_blocks_is_6_54 - jmp .L_last_num_blocks_is_5_54 - -.L_last_num_blocks_is_3_1_54: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_54 - je .L_last_num_blocks_is_2_54 -.L_last_num_blocks_is_1_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_55 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_55 - -.L_16_blocks_overflow_55: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_55: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_56 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_56 -.L_small_initial_partial_block_56: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_56 -.L_small_initial_compute_done_56: -.L_after_reduction_56: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_2_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_57 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_57 - -.L_16_blocks_overflow_57: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_57: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_58 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_58 -.L_small_initial_partial_block_58: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_58: - - orq %r8,%r8 - je .L_after_reduction_58 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_58: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_3_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_59 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_59 - -.L_16_blocks_overflow_59: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_59: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_60 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_60 -.L_small_initial_partial_block_60: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_60: - - orq %r8,%r8 - je .L_after_reduction_60 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_60: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_4_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_61 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_61 - -.L_16_blocks_overflow_61: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_61: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_62 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_62 -.L_small_initial_partial_block_62: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_62: - - orq %r8,%r8 - je .L_after_reduction_62 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_62: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_5_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_63 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_63 - -.L_16_blocks_overflow_63: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_63: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_64 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_64 -.L_small_initial_partial_block_64: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_64: - - orq %r8,%r8 - je .L_after_reduction_64 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_64: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_6_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_65 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_65 - -.L_16_blocks_overflow_65: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_65: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_66 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_66 -.L_small_initial_partial_block_66: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_66: - - orq %r8,%r8 - je .L_after_reduction_66 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_66: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_7_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_67 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_67 - -.L_16_blocks_overflow_67: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_67: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_68 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_68 -.L_small_initial_partial_block_68: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_68: - - orq %r8,%r8 - je .L_after_reduction_68 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_68: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_8_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_69 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_69 - -.L_16_blocks_overflow_69: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_69: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_70 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_70 -.L_small_initial_partial_block_70: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_70: - - orq %r8,%r8 - je .L_after_reduction_70 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_70: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_9_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_71 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_71 - -.L_16_blocks_overflow_71: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_71: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_72 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_72 -.L_small_initial_partial_block_72: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_72: - - orq %r8,%r8 - je .L_after_reduction_72 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_72: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_10_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_73 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_73 - -.L_16_blocks_overflow_73: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_73: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_74 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_74 -.L_small_initial_partial_block_74: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_74: - - orq %r8,%r8 - je .L_after_reduction_74 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_74: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_11_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_75 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_75 - -.L_16_blocks_overflow_75: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_75: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_76 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_76 -.L_small_initial_partial_block_76: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_76: - - orq %r8,%r8 - je .L_after_reduction_76 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_76: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_12_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_77 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_77 - -.L_16_blocks_overflow_77: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_77: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_78 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_78 -.L_small_initial_partial_block_78: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_78: - - orq %r8,%r8 - je .L_after_reduction_78 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_78: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_13_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_79 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_79 - -.L_16_blocks_overflow_79: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_79: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_80 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_80 -.L_small_initial_partial_block_80: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_80: - - orq %r8,%r8 - je .L_after_reduction_80 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_80: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_14_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_81 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_81 - -.L_16_blocks_overflow_81: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_81: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_82 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_82 -.L_small_initial_partial_block_82: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_82: - - orq %r8,%r8 - je .L_after_reduction_82 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_82: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_15_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_83 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_83 - -.L_16_blocks_overflow_83: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_83: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_84 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_84 -.L_small_initial_partial_block_84: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_84: - - orq %r8,%r8 - je .L_after_reduction_84 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_84: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_16_54: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_85 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_85 - -.L_16_blocks_overflow_85: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_85: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_86: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_86: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_86: - jmp .L_last_blocks_done_54 -.L_last_num_blocks_is_0_54: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_54: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_10 -.L_encrypt_16_blocks_10: - cmpb $240,%r15b - jae .L_16_blocks_overflow_87 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_87 -.L_16_blocks_overflow_87: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_87: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_88 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_88 - jb .L_last_num_blocks_is_7_1_88 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_88 - jb .L_last_num_blocks_is_11_9_88 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_88 - ja .L_last_num_blocks_is_16_88 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_88 - jmp .L_last_num_blocks_is_13_88 - -.L_last_num_blocks_is_11_9_88: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_88 - ja .L_last_num_blocks_is_11_88 - jmp .L_last_num_blocks_is_9_88 - -.L_last_num_blocks_is_7_1_88: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_88 - jb .L_last_num_blocks_is_3_1_88 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_88 - je .L_last_num_blocks_is_6_88 - jmp .L_last_num_blocks_is_5_88 - -.L_last_num_blocks_is_3_1_88: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_88 - je .L_last_num_blocks_is_2_88 -.L_last_num_blocks_is_1_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_89 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_89 - -.L_16_blocks_overflow_89: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_89: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_90 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_90 -.L_small_initial_partial_block_90: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_90 -.L_small_initial_compute_done_90: -.L_after_reduction_90: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_2_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_91 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_91 - -.L_16_blocks_overflow_91: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_91: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_92 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_92 -.L_small_initial_partial_block_92: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_92: - - orq %r8,%r8 - je .L_after_reduction_92 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_92: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_3_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_93 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_93 - -.L_16_blocks_overflow_93: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_93: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_94 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_94 -.L_small_initial_partial_block_94: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_94: - - orq %r8,%r8 - je .L_after_reduction_94 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_94: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_4_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_95 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_95 - -.L_16_blocks_overflow_95: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_95: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_96 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_96 -.L_small_initial_partial_block_96: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_96: - - orq %r8,%r8 - je .L_after_reduction_96 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_96: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_5_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_97 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_97 - -.L_16_blocks_overflow_97: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_97: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_98 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_98 -.L_small_initial_partial_block_98: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_98: - - orq %r8,%r8 - je .L_after_reduction_98 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_98: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_6_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_99 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_99 - -.L_16_blocks_overflow_99: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_99: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_100 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_100 -.L_small_initial_partial_block_100: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_100: - - orq %r8,%r8 - je .L_after_reduction_100 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_100: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_7_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_101 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_101 - -.L_16_blocks_overflow_101: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_101: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_102 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_102 -.L_small_initial_partial_block_102: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_102: - - orq %r8,%r8 - je .L_after_reduction_102 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_102: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_8_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_103 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_103 - -.L_16_blocks_overflow_103: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_103: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_104 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_104 -.L_small_initial_partial_block_104: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_104: - - orq %r8,%r8 - je .L_after_reduction_104 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_104: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_9_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_105 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_105 - -.L_16_blocks_overflow_105: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_105: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_106 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_106 -.L_small_initial_partial_block_106: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_106: - - orq %r8,%r8 - je .L_after_reduction_106 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_106: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_10_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_107 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_107 - -.L_16_blocks_overflow_107: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_107: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_108 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_108 -.L_small_initial_partial_block_108: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_108: - - orq %r8,%r8 - je .L_after_reduction_108 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_108: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_11_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_109 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_109 - -.L_16_blocks_overflow_109: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_109: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_110 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_110 -.L_small_initial_partial_block_110: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_110: - - orq %r8,%r8 - je .L_after_reduction_110 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_110: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_12_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_111 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_111 - -.L_16_blocks_overflow_111: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_111: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_112 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_112 -.L_small_initial_partial_block_112: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_112: - - orq %r8,%r8 - je .L_after_reduction_112 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_112: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_13_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_113 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_113 - -.L_16_blocks_overflow_113: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_113: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_114 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_114 -.L_small_initial_partial_block_114: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_114: - - orq %r8,%r8 - je .L_after_reduction_114 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_114: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_14_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_115 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_115 - -.L_16_blocks_overflow_115: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_115: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_116 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_116 -.L_small_initial_partial_block_116: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_116: - - orq %r8,%r8 - je .L_after_reduction_116 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_116: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_15_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_117 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_117 - -.L_16_blocks_overflow_117: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_117: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_118 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_118 -.L_small_initial_partial_block_118: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_118: - - orq %r8,%r8 - je .L_after_reduction_118 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_118: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_16_88: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_119 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_119 - -.L_16_blocks_overflow_119: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_119: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_120: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_120: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_120: - jmp .L_last_blocks_done_88 -.L_last_num_blocks_is_0_88: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_88: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_10 - -.L_message_below_32_blocks_10: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_121 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -.L_skip_hkeys_precomputation_121: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_122 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_122 - jb .L_last_num_blocks_is_7_1_122 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_122 - jb .L_last_num_blocks_is_11_9_122 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_122 - ja .L_last_num_blocks_is_16_122 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_122 - jmp .L_last_num_blocks_is_13_122 - -.L_last_num_blocks_is_11_9_122: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_122 - ja .L_last_num_blocks_is_11_122 - jmp .L_last_num_blocks_is_9_122 - -.L_last_num_blocks_is_7_1_122: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_122 - jb .L_last_num_blocks_is_3_1_122 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_122 - je .L_last_num_blocks_is_6_122 - jmp .L_last_num_blocks_is_5_122 - -.L_last_num_blocks_is_3_1_122: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_122 - je .L_last_num_blocks_is_2_122 -.L_last_num_blocks_is_1_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_123 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_123 - -.L_16_blocks_overflow_123: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_123: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_124 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_124 -.L_small_initial_partial_block_124: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_124 -.L_small_initial_compute_done_124: -.L_after_reduction_124: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_2_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_125 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_125 - -.L_16_blocks_overflow_125: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_125: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_126 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_126 -.L_small_initial_partial_block_126: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_126: - - orq %r8,%r8 - je .L_after_reduction_126 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_126: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_3_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_127 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_127 - -.L_16_blocks_overflow_127: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_127: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_128 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_128 -.L_small_initial_partial_block_128: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_128: - - orq %r8,%r8 - je .L_after_reduction_128 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_128: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_4_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_129 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_129 - -.L_16_blocks_overflow_129: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_129: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_130 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_130 -.L_small_initial_partial_block_130: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_130: - - orq %r8,%r8 - je .L_after_reduction_130 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_130: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_5_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_131 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_131 - -.L_16_blocks_overflow_131: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_131: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_132 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_132 -.L_small_initial_partial_block_132: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_132: - - orq %r8,%r8 - je .L_after_reduction_132 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_132: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_6_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_133 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_133 - -.L_16_blocks_overflow_133: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_133: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_134 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_134 -.L_small_initial_partial_block_134: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_134: - - orq %r8,%r8 - je .L_after_reduction_134 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_134: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_7_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_135 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_135 - -.L_16_blocks_overflow_135: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_135: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_136 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_136 -.L_small_initial_partial_block_136: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_136: - - orq %r8,%r8 - je .L_after_reduction_136 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_136: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_8_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_137 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_137 - -.L_16_blocks_overflow_137: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_137: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_138 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_138 -.L_small_initial_partial_block_138: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_138: - - orq %r8,%r8 - je .L_after_reduction_138 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_138: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_9_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_139 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_139 - -.L_16_blocks_overflow_139: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_139: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_140 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_140 -.L_small_initial_partial_block_140: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_140: - - orq %r8,%r8 - je .L_after_reduction_140 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_140: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_10_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_141 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_141 - -.L_16_blocks_overflow_141: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_141: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_142 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_142 -.L_small_initial_partial_block_142: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_142: - - orq %r8,%r8 - je .L_after_reduction_142 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_142: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_11_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_143 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_143 - -.L_16_blocks_overflow_143: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_143: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_144 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_144 -.L_small_initial_partial_block_144: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_144: - - orq %r8,%r8 - je .L_after_reduction_144 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_144: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_12_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_145 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_145 - -.L_16_blocks_overflow_145: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_145: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_146 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_146 -.L_small_initial_partial_block_146: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_146: - - orq %r8,%r8 - je .L_after_reduction_146 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_146: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_13_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_147 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_147 - -.L_16_blocks_overflow_147: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_147: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_148 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_148 -.L_small_initial_partial_block_148: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_148: - - orq %r8,%r8 - je .L_after_reduction_148 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_148: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_14_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_149 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_149 - -.L_16_blocks_overflow_149: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_149: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_150 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_150 -.L_small_initial_partial_block_150: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_150: - - orq %r8,%r8 - je .L_after_reduction_150 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_150: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_15_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_151 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_151 - -.L_16_blocks_overflow_151: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_151: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_152 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_152 -.L_small_initial_partial_block_152: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_152: - - orq %r8,%r8 - je .L_after_reduction_152 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_152: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_16_122: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_153 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_153 - -.L_16_blocks_overflow_153: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_153: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_154: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_154: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_154: - jmp .L_last_blocks_done_122 -.L_last_num_blocks_is_0_122: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_122: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_10 - -.L_message_below_equal_16_blocks_10: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je .L_small_initial_num_blocks_is_8_155 - jl .L_small_initial_num_blocks_is_7_1_155 - - - cmpq $12,%r12 - je .L_small_initial_num_blocks_is_12_155 - jl .L_small_initial_num_blocks_is_11_9_155 - - - cmpq $16,%r12 - je .L_small_initial_num_blocks_is_16_155 - cmpq $15,%r12 - je .L_small_initial_num_blocks_is_15_155 - cmpq $14,%r12 - je .L_small_initial_num_blocks_is_14_155 - jmp .L_small_initial_num_blocks_is_13_155 - -.L_small_initial_num_blocks_is_11_9_155: - - cmpq $11,%r12 - je .L_small_initial_num_blocks_is_11_155 - cmpq $10,%r12 - je .L_small_initial_num_blocks_is_10_155 - jmp .L_small_initial_num_blocks_is_9_155 - -.L_small_initial_num_blocks_is_7_1_155: - cmpq $4,%r12 - je .L_small_initial_num_blocks_is_4_155 - jl .L_small_initial_num_blocks_is_3_1_155 - - cmpq $7,%r12 - je .L_small_initial_num_blocks_is_7_155 - cmpq $6,%r12 - je .L_small_initial_num_blocks_is_6_155 - jmp .L_small_initial_num_blocks_is_5_155 - -.L_small_initial_num_blocks_is_3_1_155: - - cmpq $3,%r12 - je .L_small_initial_num_blocks_is_3_155 - cmpq $2,%r12 - je .L_small_initial_num_blocks_is_2_155 - - - - - -.L_small_initial_num_blocks_is_1_155: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_156 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_156 -.L_small_initial_partial_block_156: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp .L_after_reduction_156 -.L_small_initial_compute_done_156: -.L_after_reduction_156: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_2_155: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_157 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_157 -.L_small_initial_partial_block_157: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_157: - - orq %r8,%r8 - je .L_after_reduction_157 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_157: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_3_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_158 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_158 -.L_small_initial_partial_block_158: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_158: - - orq %r8,%r8 - je .L_after_reduction_158 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_158: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_4_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_159 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_159 -.L_small_initial_partial_block_159: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_159: - - orq %r8,%r8 - je .L_after_reduction_159 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_159: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_5_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %xmm29,%xmm3,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_160 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_160 -.L_small_initial_partial_block_160: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_160: - - orq %r8,%r8 - je .L_after_reduction_160 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_160: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_6_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %ymm29,%ymm3,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_161 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_161 -.L_small_initial_partial_block_161: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_161: - - orq %r8,%r8 - je .L_after_reduction_161 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_161: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_7_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_162 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_162 -.L_small_initial_partial_block_162: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_162: - - orq %r8,%r8 - je .L_after_reduction_162 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_162: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_8_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_163 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_163 -.L_small_initial_partial_block_163: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_163: - - orq %r8,%r8 - je .L_after_reduction_163 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_163: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_9_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %xmm29,%xmm4,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_164 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_164 -.L_small_initial_partial_block_164: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_164: - - orq %r8,%r8 - je .L_after_reduction_164 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_164: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_10_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %ymm29,%ymm4,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_165 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_165 -.L_small_initial_partial_block_165: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_165: - - orq %r8,%r8 - je .L_after_reduction_165 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_165: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_11_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_166 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_166 -.L_small_initial_partial_block_166: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_166: - - orq %r8,%r8 - je .L_after_reduction_166 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_166: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_12_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_167 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_167 -.L_small_initial_partial_block_167: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_167: - - orq %r8,%r8 - je .L_after_reduction_167 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_167: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_13_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %xmm29,%xmm5,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_168 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_168 -.L_small_initial_partial_block_168: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_168: - - orq %r8,%r8 - je .L_after_reduction_168 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_168: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_14_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %ymm29,%ymm5,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_169 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_169 -.L_small_initial_partial_block_169: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_169: - - orq %r8,%r8 - je .L_after_reduction_169 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_169: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_15_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_170 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_170 -.L_small_initial_partial_block_170: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_170: - - orq %r8,%r8 - je .L_after_reduction_170 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_170: - jmp .L_small_initial_blocks_encrypted_155 -.L_small_initial_num_blocks_is_16_155: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_171: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_171: - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_171: -.L_small_initial_blocks_encrypted_155: -.L_ghash_done_10: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -.L_enc_dec_done_10: - jmp .Lexit_gcm_encrypt -.align 32 -.Laes_gcm_encrypt_192_avx512: - orq %r8,%r8 - je .L_enc_dec_done_172 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je .L_partial_block_done_173 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge .L_no_extra_mask_173 - subq %r13,%r12 -.L_no_extra_mask_173: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm3,%xmm14,%xmm14 - cmpq $0,%r13 - jl .L_partial_incomplete_173 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp .L_enc_dec_done_173 - -.L_partial_incomplete_173: - addq %r8,(%rdx) - movq %r8,%r11 - -.L_enc_dec_done_173: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -.L_partial_block_done_173: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je .L_enc_dec_done_172 - cmpq $256,%r8 - jbe .L_message_below_equal_16_blocks_172 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_174 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_174 -.L_next_16_overflow_174: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_174: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_175 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -.L_skip_hkeys_precomputation_175: - cmpq $512,%r8 - jb .L_message_below_32_blocks_172 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_176 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_176 -.L_next_16_overflow_176: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_176: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_177 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -.L_skip_hkeys_precomputation_177: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb .L_no_more_big_nblocks_172 -.L_encrypt_big_nblocks_172: - cmpb $240,%r15b - jae .L_16_blocks_overflow_178 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_178 -.L_16_blocks_overflow_178: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_178: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_179 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_179 -.L_16_blocks_overflow_179: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_179: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_180 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_180 -.L_16_blocks_overflow_180: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_180: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae .L_encrypt_big_nblocks_172 - -.L_no_more_big_nblocks_172: - - cmpq $512,%r8 - jae .L_encrypt_32_blocks_172 - - cmpq $256,%r8 - jae .L_encrypt_16_blocks_172 -.L_encrypt_0_blocks_ghash_32_172: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_181 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_181 - jb .L_last_num_blocks_is_7_1_181 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_181 - jb .L_last_num_blocks_is_11_9_181 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_181 - ja .L_last_num_blocks_is_16_181 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_181 - jmp .L_last_num_blocks_is_13_181 - -.L_last_num_blocks_is_11_9_181: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_181 - ja .L_last_num_blocks_is_11_181 - jmp .L_last_num_blocks_is_9_181 - -.L_last_num_blocks_is_7_1_181: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_181 - jb .L_last_num_blocks_is_3_1_181 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_181 - je .L_last_num_blocks_is_6_181 - jmp .L_last_num_blocks_is_5_181 - -.L_last_num_blocks_is_3_1_181: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_181 - je .L_last_num_blocks_is_2_181 -.L_last_num_blocks_is_1_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_182 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_182 - -.L_16_blocks_overflow_182: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_182: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_183 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_183 -.L_small_initial_partial_block_183: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_183 -.L_small_initial_compute_done_183: -.L_after_reduction_183: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_2_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_184 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_184 - -.L_16_blocks_overflow_184: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_184: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_185 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_185 -.L_small_initial_partial_block_185: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_185: - - orq %r8,%r8 - je .L_after_reduction_185 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_185: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_3_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_186 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_186 - -.L_16_blocks_overflow_186: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_186: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_187 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_187 -.L_small_initial_partial_block_187: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_187: - - orq %r8,%r8 - je .L_after_reduction_187 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_187: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_4_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_188 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_188 - -.L_16_blocks_overflow_188: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_188: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_189 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_189 -.L_small_initial_partial_block_189: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_189: - - orq %r8,%r8 - je .L_after_reduction_189 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_189: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_5_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_190 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_190 - -.L_16_blocks_overflow_190: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_190: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_191 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_191 -.L_small_initial_partial_block_191: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_191: - - orq %r8,%r8 - je .L_after_reduction_191 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_191: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_6_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_192 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_192 - -.L_16_blocks_overflow_192: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_192: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_193 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_193 -.L_small_initial_partial_block_193: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_193: - - orq %r8,%r8 - je .L_after_reduction_193 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_193: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_7_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_194 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_194 - -.L_16_blocks_overflow_194: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_194: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_195 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_195 -.L_small_initial_partial_block_195: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_195: - - orq %r8,%r8 - je .L_after_reduction_195 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_195: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_8_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_196 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_196 - -.L_16_blocks_overflow_196: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_196: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_197 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_197 -.L_small_initial_partial_block_197: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_197: - - orq %r8,%r8 - je .L_after_reduction_197 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_197: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_9_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_198 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_198 - -.L_16_blocks_overflow_198: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_198: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_199 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_199 -.L_small_initial_partial_block_199: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_199: - - orq %r8,%r8 - je .L_after_reduction_199 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_199: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_10_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_200 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_200 - -.L_16_blocks_overflow_200: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_200: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_201 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_201 -.L_small_initial_partial_block_201: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_201: - - orq %r8,%r8 - je .L_after_reduction_201 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_201: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_11_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_202 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_202 - -.L_16_blocks_overflow_202: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_202: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_203 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_203 -.L_small_initial_partial_block_203: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_203: - - orq %r8,%r8 - je .L_after_reduction_203 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_203: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_12_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_204 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_204 - -.L_16_blocks_overflow_204: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_204: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_205 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_205 -.L_small_initial_partial_block_205: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_205: - - orq %r8,%r8 - je .L_after_reduction_205 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_205: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_13_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_206 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_206 - -.L_16_blocks_overflow_206: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_206: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_207 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_207 -.L_small_initial_partial_block_207: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_207: - - orq %r8,%r8 - je .L_after_reduction_207 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_207: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_14_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_208 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_208 - -.L_16_blocks_overflow_208: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_208: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_209 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_209 -.L_small_initial_partial_block_209: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_209: - - orq %r8,%r8 - je .L_after_reduction_209 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_209: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_15_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_210 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_210 - -.L_16_blocks_overflow_210: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_210: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_211 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_211 -.L_small_initial_partial_block_211: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_211: - - orq %r8,%r8 - je .L_after_reduction_211 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_211: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_16_181: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_212 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_212 - -.L_16_blocks_overflow_212: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_212: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_213: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_213: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_213: - jmp .L_last_blocks_done_181 -.L_last_num_blocks_is_0_181: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_181: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_172 -.L_encrypt_32_blocks_172: - cmpb $240,%r15b - jae .L_16_blocks_overflow_214 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_214 -.L_16_blocks_overflow_214: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_214: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_215 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_215 -.L_16_blocks_overflow_215: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_215: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_216 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_216 - jb .L_last_num_blocks_is_7_1_216 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_216 - jb .L_last_num_blocks_is_11_9_216 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_216 - ja .L_last_num_blocks_is_16_216 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_216 - jmp .L_last_num_blocks_is_13_216 - -.L_last_num_blocks_is_11_9_216: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_216 - ja .L_last_num_blocks_is_11_216 - jmp .L_last_num_blocks_is_9_216 - -.L_last_num_blocks_is_7_1_216: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_216 - jb .L_last_num_blocks_is_3_1_216 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_216 - je .L_last_num_blocks_is_6_216 - jmp .L_last_num_blocks_is_5_216 - -.L_last_num_blocks_is_3_1_216: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_216 - je .L_last_num_blocks_is_2_216 -.L_last_num_blocks_is_1_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_217 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_217 - -.L_16_blocks_overflow_217: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_217: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_218 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_218 -.L_small_initial_partial_block_218: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_218 -.L_small_initial_compute_done_218: -.L_after_reduction_218: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_2_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_219 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_219 - -.L_16_blocks_overflow_219: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_219: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_220 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_220 -.L_small_initial_partial_block_220: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_220: - - orq %r8,%r8 - je .L_after_reduction_220 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_220: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_3_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_221 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_221 - -.L_16_blocks_overflow_221: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_221: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_222 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_222 -.L_small_initial_partial_block_222: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_222: - - orq %r8,%r8 - je .L_after_reduction_222 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_222: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_4_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_223 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_223 - -.L_16_blocks_overflow_223: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_223: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_224 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_224 -.L_small_initial_partial_block_224: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_224: - - orq %r8,%r8 - je .L_after_reduction_224 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_224: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_5_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_225 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_225 - -.L_16_blocks_overflow_225: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_225: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_226 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_226 -.L_small_initial_partial_block_226: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_226: - - orq %r8,%r8 - je .L_after_reduction_226 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_226: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_6_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_227 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_227 - -.L_16_blocks_overflow_227: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_227: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_228 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_228 -.L_small_initial_partial_block_228: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_228: - - orq %r8,%r8 - je .L_after_reduction_228 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_228: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_7_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_229 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_229 - -.L_16_blocks_overflow_229: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_229: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_230 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_230 -.L_small_initial_partial_block_230: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_230: - - orq %r8,%r8 - je .L_after_reduction_230 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_230: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_8_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_231 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_231 - -.L_16_blocks_overflow_231: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_231: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_232 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_232 -.L_small_initial_partial_block_232: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_232: - - orq %r8,%r8 - je .L_after_reduction_232 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_232: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_9_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_233 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_233 - -.L_16_blocks_overflow_233: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_233: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_234 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_234 -.L_small_initial_partial_block_234: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_234: - - orq %r8,%r8 - je .L_after_reduction_234 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_234: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_10_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_235 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_235 - -.L_16_blocks_overflow_235: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_235: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_236 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_236 -.L_small_initial_partial_block_236: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_236: - - orq %r8,%r8 - je .L_after_reduction_236 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_236: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_11_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_237 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_237 - -.L_16_blocks_overflow_237: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_237: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_238 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_238 -.L_small_initial_partial_block_238: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_238: - - orq %r8,%r8 - je .L_after_reduction_238 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_238: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_12_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_239 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_239 - -.L_16_blocks_overflow_239: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_239: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_240 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_240 -.L_small_initial_partial_block_240: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_240: - - orq %r8,%r8 - je .L_after_reduction_240 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_240: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_13_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_241 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_241 - -.L_16_blocks_overflow_241: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_241: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_242 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_242 -.L_small_initial_partial_block_242: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_242: - - orq %r8,%r8 - je .L_after_reduction_242 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_242: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_14_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_243 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_243 - -.L_16_blocks_overflow_243: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_243: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_244 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_244 -.L_small_initial_partial_block_244: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_244: - - orq %r8,%r8 - je .L_after_reduction_244 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_244: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_15_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_245 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_245 - -.L_16_blocks_overflow_245: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_245: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_246 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_246 -.L_small_initial_partial_block_246: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_246: - - orq %r8,%r8 - je .L_after_reduction_246 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_246: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_16_216: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_247 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_247 - -.L_16_blocks_overflow_247: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_247: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_248: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_248: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_248: - jmp .L_last_blocks_done_216 -.L_last_num_blocks_is_0_216: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_216: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_172 -.L_encrypt_16_blocks_172: - cmpb $240,%r15b - jae .L_16_blocks_overflow_249 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_249 -.L_16_blocks_overflow_249: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_249: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_250 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_250 - jb .L_last_num_blocks_is_7_1_250 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_250 - jb .L_last_num_blocks_is_11_9_250 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_250 - ja .L_last_num_blocks_is_16_250 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_250 - jmp .L_last_num_blocks_is_13_250 - -.L_last_num_blocks_is_11_9_250: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_250 - ja .L_last_num_blocks_is_11_250 - jmp .L_last_num_blocks_is_9_250 - -.L_last_num_blocks_is_7_1_250: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_250 - jb .L_last_num_blocks_is_3_1_250 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_250 - je .L_last_num_blocks_is_6_250 - jmp .L_last_num_blocks_is_5_250 - -.L_last_num_blocks_is_3_1_250: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_250 - je .L_last_num_blocks_is_2_250 -.L_last_num_blocks_is_1_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_251 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_251 - -.L_16_blocks_overflow_251: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_251: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_252 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_252 -.L_small_initial_partial_block_252: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_252 -.L_small_initial_compute_done_252: -.L_after_reduction_252: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_2_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_253 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_253 - -.L_16_blocks_overflow_253: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_253: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_254 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_254 -.L_small_initial_partial_block_254: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_254: - - orq %r8,%r8 - je .L_after_reduction_254 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_254: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_3_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_255 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_255 - -.L_16_blocks_overflow_255: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_255: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_256 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_256 -.L_small_initial_partial_block_256: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_256: - - orq %r8,%r8 - je .L_after_reduction_256 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_256: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_4_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_257 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_257 - -.L_16_blocks_overflow_257: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_257: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_258 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_258 -.L_small_initial_partial_block_258: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_258: - - orq %r8,%r8 - je .L_after_reduction_258 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_258: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_5_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_259 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_259 - -.L_16_blocks_overflow_259: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_259: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_260 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_260 -.L_small_initial_partial_block_260: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_260: - - orq %r8,%r8 - je .L_after_reduction_260 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_260: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_6_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_261 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_261 - -.L_16_blocks_overflow_261: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_261: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_262 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_262 -.L_small_initial_partial_block_262: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_262: - - orq %r8,%r8 - je .L_after_reduction_262 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_262: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_7_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_263 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_263 - -.L_16_blocks_overflow_263: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_263: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_264 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_264 -.L_small_initial_partial_block_264: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_264: - - orq %r8,%r8 - je .L_after_reduction_264 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_264: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_8_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_265 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_265 - -.L_16_blocks_overflow_265: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_265: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_266 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_266 -.L_small_initial_partial_block_266: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_266: - - orq %r8,%r8 - je .L_after_reduction_266 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_266: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_9_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_267 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_267 - -.L_16_blocks_overflow_267: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_267: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_268 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_268 -.L_small_initial_partial_block_268: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_268: - - orq %r8,%r8 - je .L_after_reduction_268 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_268: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_10_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_269 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_269 - -.L_16_blocks_overflow_269: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_269: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_270 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_270 -.L_small_initial_partial_block_270: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_270: - - orq %r8,%r8 - je .L_after_reduction_270 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_270: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_11_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_271 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_271 - -.L_16_blocks_overflow_271: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_271: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_272 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_272 -.L_small_initial_partial_block_272: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_272: - - orq %r8,%r8 - je .L_after_reduction_272 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_272: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_12_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_273 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_273 - -.L_16_blocks_overflow_273: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_273: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_274 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_274 -.L_small_initial_partial_block_274: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_274: - - orq %r8,%r8 - je .L_after_reduction_274 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_274: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_13_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_275 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_275 - -.L_16_blocks_overflow_275: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_275: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_276 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_276 -.L_small_initial_partial_block_276: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_276: - - orq %r8,%r8 - je .L_after_reduction_276 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_276: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_14_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_277 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_277 - -.L_16_blocks_overflow_277: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_277: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_278 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_278 -.L_small_initial_partial_block_278: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_278: - - orq %r8,%r8 - je .L_after_reduction_278 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_278: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_15_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_279 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_279 - -.L_16_blocks_overflow_279: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_279: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_280 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_280 -.L_small_initial_partial_block_280: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_280: - - orq %r8,%r8 - je .L_after_reduction_280 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_280: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_16_250: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_281 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_281 - -.L_16_blocks_overflow_281: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_281: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_282: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_282: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_282: - jmp .L_last_blocks_done_250 -.L_last_num_blocks_is_0_250: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_250: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_172 - -.L_message_below_32_blocks_172: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_283 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -.L_skip_hkeys_precomputation_283: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_284 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_284 - jb .L_last_num_blocks_is_7_1_284 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_284 - jb .L_last_num_blocks_is_11_9_284 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_284 - ja .L_last_num_blocks_is_16_284 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_284 - jmp .L_last_num_blocks_is_13_284 - -.L_last_num_blocks_is_11_9_284: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_284 - ja .L_last_num_blocks_is_11_284 - jmp .L_last_num_blocks_is_9_284 - -.L_last_num_blocks_is_7_1_284: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_284 - jb .L_last_num_blocks_is_3_1_284 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_284 - je .L_last_num_blocks_is_6_284 - jmp .L_last_num_blocks_is_5_284 - -.L_last_num_blocks_is_3_1_284: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_284 - je .L_last_num_blocks_is_2_284 -.L_last_num_blocks_is_1_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_285 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_285 - -.L_16_blocks_overflow_285: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_285: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_286 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_286 -.L_small_initial_partial_block_286: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_286 -.L_small_initial_compute_done_286: -.L_after_reduction_286: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_2_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_287 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_287 - -.L_16_blocks_overflow_287: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_287: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_288 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_288 -.L_small_initial_partial_block_288: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_288: - - orq %r8,%r8 - je .L_after_reduction_288 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_288: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_3_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_289 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_289 - -.L_16_blocks_overflow_289: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_289: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_290 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_290 -.L_small_initial_partial_block_290: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_290: - - orq %r8,%r8 - je .L_after_reduction_290 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_290: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_4_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_291 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_291 - -.L_16_blocks_overflow_291: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_291: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_292 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_292 -.L_small_initial_partial_block_292: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_292: - - orq %r8,%r8 - je .L_after_reduction_292 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_292: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_5_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_293 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_293 - -.L_16_blocks_overflow_293: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_293: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_294 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_294 -.L_small_initial_partial_block_294: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_294: - - orq %r8,%r8 - je .L_after_reduction_294 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_294: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_6_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_295 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_295 - -.L_16_blocks_overflow_295: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_295: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_296 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_296 -.L_small_initial_partial_block_296: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_296: - - orq %r8,%r8 - je .L_after_reduction_296 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_296: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_7_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_297 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_297 - -.L_16_blocks_overflow_297: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_297: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_298 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_298 -.L_small_initial_partial_block_298: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_298: - - orq %r8,%r8 - je .L_after_reduction_298 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_298: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_8_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_299 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_299 - -.L_16_blocks_overflow_299: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_299: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_300 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_300 -.L_small_initial_partial_block_300: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_300: - - orq %r8,%r8 - je .L_after_reduction_300 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_300: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_9_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_301 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_301 - -.L_16_blocks_overflow_301: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_301: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_302 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_302 -.L_small_initial_partial_block_302: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_302: - - orq %r8,%r8 - je .L_after_reduction_302 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_302: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_10_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_303 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_303 - -.L_16_blocks_overflow_303: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_303: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_304 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_304 -.L_small_initial_partial_block_304: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_304: - - orq %r8,%r8 - je .L_after_reduction_304 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_304: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_11_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_305 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_305 - -.L_16_blocks_overflow_305: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_305: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_306 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_306 -.L_small_initial_partial_block_306: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_306: - - orq %r8,%r8 - je .L_after_reduction_306 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_306: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_12_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_307 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_307 - -.L_16_blocks_overflow_307: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_307: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_308 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_308 -.L_small_initial_partial_block_308: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_308: - - orq %r8,%r8 - je .L_after_reduction_308 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_308: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_13_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_309 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_309 - -.L_16_blocks_overflow_309: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_309: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_310 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_310 -.L_small_initial_partial_block_310: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_310: - - orq %r8,%r8 - je .L_after_reduction_310 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_310: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_14_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_311 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_311 - -.L_16_blocks_overflow_311: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_311: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_312 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_312 -.L_small_initial_partial_block_312: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_312: - - orq %r8,%r8 - je .L_after_reduction_312 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_312: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_15_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_313 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_313 - -.L_16_blocks_overflow_313: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_313: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_314 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_314 -.L_small_initial_partial_block_314: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_314: - - orq %r8,%r8 - je .L_after_reduction_314 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_314: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_16_284: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_315 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_315 - -.L_16_blocks_overflow_315: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_315: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_316: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_316: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_316: - jmp .L_last_blocks_done_284 -.L_last_num_blocks_is_0_284: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_284: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_172 - -.L_message_below_equal_16_blocks_172: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je .L_small_initial_num_blocks_is_8_317 - jl .L_small_initial_num_blocks_is_7_1_317 - - - cmpq $12,%r12 - je .L_small_initial_num_blocks_is_12_317 - jl .L_small_initial_num_blocks_is_11_9_317 - - - cmpq $16,%r12 - je .L_small_initial_num_blocks_is_16_317 - cmpq $15,%r12 - je .L_small_initial_num_blocks_is_15_317 - cmpq $14,%r12 - je .L_small_initial_num_blocks_is_14_317 - jmp .L_small_initial_num_blocks_is_13_317 - -.L_small_initial_num_blocks_is_11_9_317: - - cmpq $11,%r12 - je .L_small_initial_num_blocks_is_11_317 - cmpq $10,%r12 - je .L_small_initial_num_blocks_is_10_317 - jmp .L_small_initial_num_blocks_is_9_317 - -.L_small_initial_num_blocks_is_7_1_317: - cmpq $4,%r12 - je .L_small_initial_num_blocks_is_4_317 - jl .L_small_initial_num_blocks_is_3_1_317 - - cmpq $7,%r12 - je .L_small_initial_num_blocks_is_7_317 - cmpq $6,%r12 - je .L_small_initial_num_blocks_is_6_317 - jmp .L_small_initial_num_blocks_is_5_317 - -.L_small_initial_num_blocks_is_3_1_317: - - cmpq $3,%r12 - je .L_small_initial_num_blocks_is_3_317 - cmpq $2,%r12 - je .L_small_initial_num_blocks_is_2_317 - - - - - -.L_small_initial_num_blocks_is_1_317: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_318 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_318 -.L_small_initial_partial_block_318: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp .L_after_reduction_318 -.L_small_initial_compute_done_318: -.L_after_reduction_318: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_2_317: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_319 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_319 -.L_small_initial_partial_block_319: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_319: - - orq %r8,%r8 - je .L_after_reduction_319 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_319: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_3_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_320 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_320 -.L_small_initial_partial_block_320: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_320: - - orq %r8,%r8 - je .L_after_reduction_320 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_320: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_4_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_321 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_321 -.L_small_initial_partial_block_321: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_321: - - orq %r8,%r8 - je .L_after_reduction_321 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_321: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_5_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %xmm29,%xmm3,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_322 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_322 -.L_small_initial_partial_block_322: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_322: - - orq %r8,%r8 - je .L_after_reduction_322 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_322: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_6_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %ymm29,%ymm3,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_323 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_323 -.L_small_initial_partial_block_323: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_323: - - orq %r8,%r8 - je .L_after_reduction_323 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_323: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_7_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_324 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_324 -.L_small_initial_partial_block_324: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_324: - - orq %r8,%r8 - je .L_after_reduction_324 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_324: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_8_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_325 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_325 -.L_small_initial_partial_block_325: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_325: - - orq %r8,%r8 - je .L_after_reduction_325 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_325: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_9_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %xmm29,%xmm4,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_326 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_326 -.L_small_initial_partial_block_326: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_326: - - orq %r8,%r8 - je .L_after_reduction_326 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_326: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_10_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %ymm29,%ymm4,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_327 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_327 -.L_small_initial_partial_block_327: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_327: - - orq %r8,%r8 - je .L_after_reduction_327 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_327: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_11_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_328 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_328 -.L_small_initial_partial_block_328: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_328: - - orq %r8,%r8 - je .L_after_reduction_328 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_328: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_12_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_329 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_329 -.L_small_initial_partial_block_329: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_329: - - orq %r8,%r8 - je .L_after_reduction_329 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_329: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_13_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %xmm29,%xmm5,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_330 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_330 -.L_small_initial_partial_block_330: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_330: - - orq %r8,%r8 - je .L_after_reduction_330 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_330: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_14_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %ymm29,%ymm5,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_331 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_331 -.L_small_initial_partial_block_331: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_331: - - orq %r8,%r8 - je .L_after_reduction_331 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_331: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_15_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_332 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_332 -.L_small_initial_partial_block_332: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_332: - - orq %r8,%r8 - je .L_after_reduction_332 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_332: - jmp .L_small_initial_blocks_encrypted_317 -.L_small_initial_num_blocks_is_16_317: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_333: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_333: - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_333: -.L_small_initial_blocks_encrypted_317: -.L_ghash_done_172: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -.L_enc_dec_done_172: - jmp .Lexit_gcm_encrypt -.align 32 -.Laes_gcm_encrypt_256_avx512: - orq %r8,%r8 - je .L_enc_dec_done_334 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je .L_partial_block_done_335 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge .L_no_extra_mask_335 - subq %r13,%r12 -.L_no_extra_mask_335: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - vpxorq %xmm3,%xmm14,%xmm14 - cmpq $0,%r13 - jl .L_partial_incomplete_335 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp .L_enc_dec_done_335 - -.L_partial_incomplete_335: - addq %r8,(%rdx) - movq %r8,%r11 - -.L_enc_dec_done_335: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - - vpshufb SHUF_MASK(%rip),%xmm3,%xmm3 - vpshufb %xmm5,%xmm3,%xmm3 - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -.L_partial_block_done_335: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je .L_enc_dec_done_334 - cmpq $256,%r8 - jbe .L_message_below_equal_16_blocks_334 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_336 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_336 -.L_next_16_overflow_336: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_336: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 208(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 224(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_337 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -.L_skip_hkeys_precomputation_337: - cmpq $512,%r8 - jb .L_message_below_32_blocks_334 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_338 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_338 -.L_next_16_overflow_338: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_338: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 208(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 224(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_339 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -.L_skip_hkeys_precomputation_339: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb .L_no_more_big_nblocks_334 -.L_encrypt_big_nblocks_334: - cmpb $240,%r15b - jae .L_16_blocks_overflow_340 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_340 -.L_16_blocks_overflow_340: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_340: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_341 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_341 -.L_16_blocks_overflow_341: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_341: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_342 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_342 -.L_16_blocks_overflow_342: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_342: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae .L_encrypt_big_nblocks_334 - -.L_no_more_big_nblocks_334: - - cmpq $512,%r8 - jae .L_encrypt_32_blocks_334 - - cmpq $256,%r8 - jae .L_encrypt_16_blocks_334 -.L_encrypt_0_blocks_ghash_32_334: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_343 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_343 - jb .L_last_num_blocks_is_7_1_343 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_343 - jb .L_last_num_blocks_is_11_9_343 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_343 - ja .L_last_num_blocks_is_16_343 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_343 - jmp .L_last_num_blocks_is_13_343 - -.L_last_num_blocks_is_11_9_343: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_343 - ja .L_last_num_blocks_is_11_343 - jmp .L_last_num_blocks_is_9_343 - -.L_last_num_blocks_is_7_1_343: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_343 - jb .L_last_num_blocks_is_3_1_343 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_343 - je .L_last_num_blocks_is_6_343 - jmp .L_last_num_blocks_is_5_343 - -.L_last_num_blocks_is_3_1_343: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_343 - je .L_last_num_blocks_is_2_343 -.L_last_num_blocks_is_1_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_344 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_344 - -.L_16_blocks_overflow_344: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_344: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_345 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_345 -.L_small_initial_partial_block_345: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_345 -.L_small_initial_compute_done_345: -.L_after_reduction_345: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_2_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_346 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_346 - -.L_16_blocks_overflow_346: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_346: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_347 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_347 -.L_small_initial_partial_block_347: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_347: - - orq %r8,%r8 - je .L_after_reduction_347 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_347: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_3_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_348 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_348 - -.L_16_blocks_overflow_348: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_348: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_349 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_349 -.L_small_initial_partial_block_349: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_349: - - orq %r8,%r8 - je .L_after_reduction_349 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_349: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_4_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_350 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_350 - -.L_16_blocks_overflow_350: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_350: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_351 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_351 -.L_small_initial_partial_block_351: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_351: - - orq %r8,%r8 - je .L_after_reduction_351 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_351: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_5_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_352 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_352 - -.L_16_blocks_overflow_352: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_352: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_353 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_353 -.L_small_initial_partial_block_353: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_353: - - orq %r8,%r8 - je .L_after_reduction_353 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_353: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_6_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_354 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_354 - -.L_16_blocks_overflow_354: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_354: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_355 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_355 -.L_small_initial_partial_block_355: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_355: - - orq %r8,%r8 - je .L_after_reduction_355 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_355: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_7_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_356 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_356 - -.L_16_blocks_overflow_356: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_356: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_357 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_357 -.L_small_initial_partial_block_357: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_357: - - orq %r8,%r8 - je .L_after_reduction_357 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_357: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_8_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_358 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_358 - -.L_16_blocks_overflow_358: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_358: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_359 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_359 -.L_small_initial_partial_block_359: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_359: - - orq %r8,%r8 - je .L_after_reduction_359 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_359: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_9_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_360 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_360 - -.L_16_blocks_overflow_360: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_360: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_361 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_361 -.L_small_initial_partial_block_361: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_361: - - orq %r8,%r8 - je .L_after_reduction_361 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_361: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_10_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_362 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_362 - -.L_16_blocks_overflow_362: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_362: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_363 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_363 -.L_small_initial_partial_block_363: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_363: - - orq %r8,%r8 - je .L_after_reduction_363 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_363: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_11_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_364 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_364 - -.L_16_blocks_overflow_364: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_364: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_365 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_365 -.L_small_initial_partial_block_365: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_365: - - orq %r8,%r8 - je .L_after_reduction_365 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_365: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_12_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_366 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_366 - -.L_16_blocks_overflow_366: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_366: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_367 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_367 -.L_small_initial_partial_block_367: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_367: - - orq %r8,%r8 - je .L_after_reduction_367 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_367: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_13_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_368 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_368 - -.L_16_blocks_overflow_368: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_368: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_369 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_369 -.L_small_initial_partial_block_369: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_369: - - orq %r8,%r8 - je .L_after_reduction_369 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_369: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_14_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_370 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_370 - -.L_16_blocks_overflow_370: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_370: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_371 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_371 -.L_small_initial_partial_block_371: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_371: - - orq %r8,%r8 - je .L_after_reduction_371 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_371: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_15_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_372 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_372 - -.L_16_blocks_overflow_372: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_372: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_373 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_373 -.L_small_initial_partial_block_373: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_373: - - orq %r8,%r8 - je .L_after_reduction_373 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_373: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_16_343: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_374 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_374 - -.L_16_blocks_overflow_374: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_374: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_375: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_375: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_375: - jmp .L_last_blocks_done_343 -.L_last_num_blocks_is_0_343: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_343: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_334 -.L_encrypt_32_blocks_334: - cmpb $240,%r15b - jae .L_16_blocks_overflow_376 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_376 -.L_16_blocks_overflow_376: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_376: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_377 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_377 -.L_16_blocks_overflow_377: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_377: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_378 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_378 - jb .L_last_num_blocks_is_7_1_378 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_378 - jb .L_last_num_blocks_is_11_9_378 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_378 - ja .L_last_num_blocks_is_16_378 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_378 - jmp .L_last_num_blocks_is_13_378 - -.L_last_num_blocks_is_11_9_378: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_378 - ja .L_last_num_blocks_is_11_378 - jmp .L_last_num_blocks_is_9_378 - -.L_last_num_blocks_is_7_1_378: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_378 - jb .L_last_num_blocks_is_3_1_378 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_378 - je .L_last_num_blocks_is_6_378 - jmp .L_last_num_blocks_is_5_378 - -.L_last_num_blocks_is_3_1_378: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_378 - je .L_last_num_blocks_is_2_378 -.L_last_num_blocks_is_1_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_379 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_379 - -.L_16_blocks_overflow_379: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_379: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_380 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_380 -.L_small_initial_partial_block_380: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_380 -.L_small_initial_compute_done_380: -.L_after_reduction_380: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_2_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_381 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_381 - -.L_16_blocks_overflow_381: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_381: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_382 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_382 -.L_small_initial_partial_block_382: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_382: - - orq %r8,%r8 - je .L_after_reduction_382 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_382: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_3_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_383 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_383 - -.L_16_blocks_overflow_383: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_383: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_384 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_384 -.L_small_initial_partial_block_384: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_384: - - orq %r8,%r8 - je .L_after_reduction_384 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_384: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_4_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_385 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_385 - -.L_16_blocks_overflow_385: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_385: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_386 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_386 -.L_small_initial_partial_block_386: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_386: - - orq %r8,%r8 - je .L_after_reduction_386 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_386: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_5_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_387 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_387 - -.L_16_blocks_overflow_387: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_387: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_388 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_388 -.L_small_initial_partial_block_388: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_388: - - orq %r8,%r8 - je .L_after_reduction_388 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_388: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_6_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_389 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_389 - -.L_16_blocks_overflow_389: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_389: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_390 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_390 -.L_small_initial_partial_block_390: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_390: - - orq %r8,%r8 - je .L_after_reduction_390 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_390: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_7_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_391 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_391 - -.L_16_blocks_overflow_391: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_391: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_392 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_392 -.L_small_initial_partial_block_392: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_392: - - orq %r8,%r8 - je .L_after_reduction_392 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_392: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_8_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_393 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_393 - -.L_16_blocks_overflow_393: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_393: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_394 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_394 -.L_small_initial_partial_block_394: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_394: - - orq %r8,%r8 - je .L_after_reduction_394 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_394: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_9_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_395 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_395 - -.L_16_blocks_overflow_395: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_395: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_396 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_396 -.L_small_initial_partial_block_396: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_396: - - orq %r8,%r8 - je .L_after_reduction_396 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_396: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_10_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_397 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_397 - -.L_16_blocks_overflow_397: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_397: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_398 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_398 -.L_small_initial_partial_block_398: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_398: - - orq %r8,%r8 - je .L_after_reduction_398 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_398: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_11_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_399 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_399 - -.L_16_blocks_overflow_399: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_399: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_400 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_400 -.L_small_initial_partial_block_400: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_400: - - orq %r8,%r8 - je .L_after_reduction_400 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_400: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_12_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_401 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_401 - -.L_16_blocks_overflow_401: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_401: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_402 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_402 -.L_small_initial_partial_block_402: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_402: - - orq %r8,%r8 - je .L_after_reduction_402 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_402: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_13_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_403 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_403 - -.L_16_blocks_overflow_403: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_403: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_404 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_404 -.L_small_initial_partial_block_404: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_404: - - orq %r8,%r8 - je .L_after_reduction_404 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_404: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_14_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_405 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_405 - -.L_16_blocks_overflow_405: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_405: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_406 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_406 -.L_small_initial_partial_block_406: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_406: - - orq %r8,%r8 - je .L_after_reduction_406 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_406: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_15_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_407 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_407 - -.L_16_blocks_overflow_407: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_407: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_408 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_408 -.L_small_initial_partial_block_408: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_408: - - orq %r8,%r8 - je .L_after_reduction_408 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_408: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_16_378: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_409 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_409 - -.L_16_blocks_overflow_409: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_409: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_410: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_410: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_410: - jmp .L_last_blocks_done_378 -.L_last_num_blocks_is_0_378: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_378: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_334 -.L_encrypt_16_blocks_334: - cmpb $240,%r15b - jae .L_16_blocks_overflow_411 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_411 -.L_16_blocks_overflow_411: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_411: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_412 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_412 - jb .L_last_num_blocks_is_7_1_412 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_412 - jb .L_last_num_blocks_is_11_9_412 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_412 - ja .L_last_num_blocks_is_16_412 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_412 - jmp .L_last_num_blocks_is_13_412 - -.L_last_num_blocks_is_11_9_412: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_412 - ja .L_last_num_blocks_is_11_412 - jmp .L_last_num_blocks_is_9_412 - -.L_last_num_blocks_is_7_1_412: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_412 - jb .L_last_num_blocks_is_3_1_412 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_412 - je .L_last_num_blocks_is_6_412 - jmp .L_last_num_blocks_is_5_412 - -.L_last_num_blocks_is_3_1_412: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_412 - je .L_last_num_blocks_is_2_412 -.L_last_num_blocks_is_1_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_413 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_413 - -.L_16_blocks_overflow_413: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_413: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_414 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_414 -.L_small_initial_partial_block_414: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_414 -.L_small_initial_compute_done_414: -.L_after_reduction_414: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_2_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_415 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_415 - -.L_16_blocks_overflow_415: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_415: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_416 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_416 -.L_small_initial_partial_block_416: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_416: - - orq %r8,%r8 - je .L_after_reduction_416 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_416: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_3_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_417 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_417 - -.L_16_blocks_overflow_417: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_417: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_418 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_418 -.L_small_initial_partial_block_418: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_418: - - orq %r8,%r8 - je .L_after_reduction_418 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_418: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_4_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_419 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_419 - -.L_16_blocks_overflow_419: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_419: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_420 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_420 -.L_small_initial_partial_block_420: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_420: - - orq %r8,%r8 - je .L_after_reduction_420 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_420: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_5_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_421 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_421 - -.L_16_blocks_overflow_421: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_421: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_422 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_422 -.L_small_initial_partial_block_422: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_422: - - orq %r8,%r8 - je .L_after_reduction_422 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_422: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_6_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_423 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_423 - -.L_16_blocks_overflow_423: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_423: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_424 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_424 -.L_small_initial_partial_block_424: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_424: - - orq %r8,%r8 - je .L_after_reduction_424 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_424: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_7_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_425 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_425 - -.L_16_blocks_overflow_425: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_425: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_426 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_426 -.L_small_initial_partial_block_426: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_426: - - orq %r8,%r8 - je .L_after_reduction_426 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_426: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_8_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_427 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_427 - -.L_16_blocks_overflow_427: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_427: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_428 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_428 -.L_small_initial_partial_block_428: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_428: - - orq %r8,%r8 - je .L_after_reduction_428 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_428: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_9_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_429 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_429 - -.L_16_blocks_overflow_429: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_429: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_430 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_430 -.L_small_initial_partial_block_430: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_430: - - orq %r8,%r8 - je .L_after_reduction_430 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_430: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_10_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_431 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_431 - -.L_16_blocks_overflow_431: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_431: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_432 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_432 -.L_small_initial_partial_block_432: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_432: - - orq %r8,%r8 - je .L_after_reduction_432 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_432: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_11_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_433 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_433 - -.L_16_blocks_overflow_433: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_433: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_434 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_434 -.L_small_initial_partial_block_434: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_434: - - orq %r8,%r8 - je .L_after_reduction_434 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_434: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_12_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_435 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_435 - -.L_16_blocks_overflow_435: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_435: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_436 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_436 -.L_small_initial_partial_block_436: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_436: - - orq %r8,%r8 - je .L_after_reduction_436 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_436: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_13_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_437 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_437 - -.L_16_blocks_overflow_437: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_437: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_438 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_438 -.L_small_initial_partial_block_438: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_438: - - orq %r8,%r8 - je .L_after_reduction_438 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_438: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_14_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_439 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_439 - -.L_16_blocks_overflow_439: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_439: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_440 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_440 -.L_small_initial_partial_block_440: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_440: - - orq %r8,%r8 - je .L_after_reduction_440 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_440: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_15_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_441 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_441 - -.L_16_blocks_overflow_441: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_441: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_442 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_442 -.L_small_initial_partial_block_442: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_442: - - orq %r8,%r8 - je .L_after_reduction_442 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_442: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_16_412: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_443 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_443 - -.L_16_blocks_overflow_443: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_443: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_444: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_444: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_444: - jmp .L_last_blocks_done_412 -.L_last_num_blocks_is_0_412: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_412: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_334 - -.L_message_below_32_blocks_334: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_445 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -.L_skip_hkeys_precomputation_445: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_446 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_446 - jb .L_last_num_blocks_is_7_1_446 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_446 - jb .L_last_num_blocks_is_11_9_446 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_446 - ja .L_last_num_blocks_is_16_446 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_446 - jmp .L_last_num_blocks_is_13_446 - -.L_last_num_blocks_is_11_9_446: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_446 - ja .L_last_num_blocks_is_11_446 - jmp .L_last_num_blocks_is_9_446 - -.L_last_num_blocks_is_7_1_446: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_446 - jb .L_last_num_blocks_is_3_1_446 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_446 - je .L_last_num_blocks_is_6_446 - jmp .L_last_num_blocks_is_5_446 - -.L_last_num_blocks_is_3_1_446: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_446 - je .L_last_num_blocks_is_2_446 -.L_last_num_blocks_is_1_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_447 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_447 - -.L_16_blocks_overflow_447: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_447: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_448 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_448 -.L_small_initial_partial_block_448: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_448 -.L_small_initial_compute_done_448: -.L_after_reduction_448: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_2_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_449 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_449 - -.L_16_blocks_overflow_449: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_449: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_450 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_450 -.L_small_initial_partial_block_450: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_450: - - orq %r8,%r8 - je .L_after_reduction_450 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_450: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_3_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_451 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_451 - -.L_16_blocks_overflow_451: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_451: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_452 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_452 -.L_small_initial_partial_block_452: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_452: - - orq %r8,%r8 - je .L_after_reduction_452 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_452: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_4_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_453 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_453 - -.L_16_blocks_overflow_453: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_453: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_454 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_454 -.L_small_initial_partial_block_454: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_454: - - orq %r8,%r8 - je .L_after_reduction_454 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_454: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_5_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_455 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_455 - -.L_16_blocks_overflow_455: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_455: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %xmm29,%xmm3,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_456 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_456 -.L_small_initial_partial_block_456: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_456: - - orq %r8,%r8 - je .L_after_reduction_456 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_456: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_6_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_457 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_457 - -.L_16_blocks_overflow_457: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_457: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %ymm29,%ymm3,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_458 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_458 -.L_small_initial_partial_block_458: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_458: - - orq %r8,%r8 - je .L_after_reduction_458 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_458: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_7_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_459 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_459 - -.L_16_blocks_overflow_459: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_459: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_460 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_460 -.L_small_initial_partial_block_460: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_460: - - orq %r8,%r8 - je .L_after_reduction_460 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_460: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_8_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_461 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_461 - -.L_16_blocks_overflow_461: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_461: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_462 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_462 -.L_small_initial_partial_block_462: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_462: - - orq %r8,%r8 - je .L_after_reduction_462 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_462: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_9_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_463 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_463 - -.L_16_blocks_overflow_463: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_463: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %xmm29,%xmm4,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_464 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_464 -.L_small_initial_partial_block_464: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_464: - - orq %r8,%r8 - je .L_after_reduction_464 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_464: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_10_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_465 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_465 - -.L_16_blocks_overflow_465: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_465: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %ymm29,%ymm4,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_466 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_466 -.L_small_initial_partial_block_466: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_466: - - orq %r8,%r8 - je .L_after_reduction_466 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_466: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_11_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_467 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_467 - -.L_16_blocks_overflow_467: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_467: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_468 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_468 -.L_small_initial_partial_block_468: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_468: - - orq %r8,%r8 - je .L_after_reduction_468 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_468: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_12_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_469 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_469 - -.L_16_blocks_overflow_469: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_469: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_470 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_470 -.L_small_initial_partial_block_470: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_470: - - orq %r8,%r8 - je .L_after_reduction_470 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_470: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_13_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_471 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_471 - -.L_16_blocks_overflow_471: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_471: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %xmm29,%xmm5,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_472 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_472 -.L_small_initial_partial_block_472: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_472: - - orq %r8,%r8 - je .L_after_reduction_472 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_472: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_14_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_473 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_473 - -.L_16_blocks_overflow_473: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_473: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %ymm29,%ymm5,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_474 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_474 -.L_small_initial_partial_block_474: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_474: - - orq %r8,%r8 - je .L_after_reduction_474 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_474: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_15_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_475 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_475 - -.L_16_blocks_overflow_475: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_475: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_476 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_476 -.L_small_initial_partial_block_476: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_476: - - orq %r8,%r8 - je .L_after_reduction_476 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_476: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_16_446: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_477 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_477 - -.L_16_blocks_overflow_477: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_477: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm17 - vpshufb %zmm29,%zmm3,%zmm19 - vpshufb %zmm29,%zmm4,%zmm20 - vpshufb %zmm29,%zmm5,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_478: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_478: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_478: - jmp .L_last_blocks_done_446 -.L_last_num_blocks_is_0_446: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_446: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_334 - -.L_message_below_equal_16_blocks_334: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je .L_small_initial_num_blocks_is_8_479 - jl .L_small_initial_num_blocks_is_7_1_479 - - - cmpq $12,%r12 - je .L_small_initial_num_blocks_is_12_479 - jl .L_small_initial_num_blocks_is_11_9_479 - - - cmpq $16,%r12 - je .L_small_initial_num_blocks_is_16_479 - cmpq $15,%r12 - je .L_small_initial_num_blocks_is_15_479 - cmpq $14,%r12 - je .L_small_initial_num_blocks_is_14_479 - jmp .L_small_initial_num_blocks_is_13_479 - -.L_small_initial_num_blocks_is_11_9_479: - - cmpq $11,%r12 - je .L_small_initial_num_blocks_is_11_479 - cmpq $10,%r12 - je .L_small_initial_num_blocks_is_10_479 - jmp .L_small_initial_num_blocks_is_9_479 - -.L_small_initial_num_blocks_is_7_1_479: - cmpq $4,%r12 - je .L_small_initial_num_blocks_is_4_479 - jl .L_small_initial_num_blocks_is_3_1_479 - - cmpq $7,%r12 - je .L_small_initial_num_blocks_is_7_479 - cmpq $6,%r12 - je .L_small_initial_num_blocks_is_6_479 - jmp .L_small_initial_num_blocks_is_5_479 - -.L_small_initial_num_blocks_is_3_1_479: - - cmpq $3,%r12 - je .L_small_initial_num_blocks_is_3_479 - cmpq $2,%r12 - je .L_small_initial_num_blocks_is_2_479 - - - - - -.L_small_initial_num_blocks_is_1_479: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm0,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_480 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_480 -.L_small_initial_partial_block_480: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp .L_after_reduction_480 -.L_small_initial_compute_done_480: -.L_after_reduction_480: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_2_479: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm0,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_481 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_481 -.L_small_initial_partial_block_481: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_481: - - orq %r8,%r8 - je .L_after_reduction_481 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_481: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_3_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_482 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_482 -.L_small_initial_partial_block_482: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_482: - - orq %r8,%r8 - je .L_after_reduction_482 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_482: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_4_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_483 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_483 -.L_small_initial_partial_block_483: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_483: - - orq %r8,%r8 - je .L_after_reduction_483 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_483: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_5_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %xmm29,%xmm3,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_484 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_484 -.L_small_initial_partial_block_484: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_484: - - orq %r8,%r8 - je .L_after_reduction_484 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_484: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_6_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %ymm29,%ymm3,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_485 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_485 -.L_small_initial_partial_block_485: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_485: - - orq %r8,%r8 - je .L_after_reduction_485 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_485: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_7_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_486 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_486 -.L_small_initial_partial_block_486: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_486: - - orq %r8,%r8 - je .L_after_reduction_486 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_486: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_8_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_487 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_487 -.L_small_initial_partial_block_487: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_487: - - orq %r8,%r8 - je .L_after_reduction_487 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_487: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_9_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %xmm29,%xmm4,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_488 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_488 -.L_small_initial_partial_block_488: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_488: - - orq %r8,%r8 - je .L_after_reduction_488 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_488: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_10_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %ymm29,%ymm4,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_489 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_489 -.L_small_initial_partial_block_489: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_489: - - orq %r8,%r8 - je .L_after_reduction_489 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_489: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_11_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_490 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_490 -.L_small_initial_partial_block_490: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_490: - - orq %r8,%r8 - je .L_after_reduction_490 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_490: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_12_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_491 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_491 -.L_small_initial_partial_block_491: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_491: - - orq %r8,%r8 - je .L_after_reduction_491 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_491: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_13_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %xmm29,%xmm5,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_492 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_492 -.L_small_initial_partial_block_492: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_492: - - orq %r8,%r8 - je .L_after_reduction_492 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_492: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_14_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %ymm29,%ymm5,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_493 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_493 -.L_small_initial_partial_block_493: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_493: - - orq %r8,%r8 - je .L_after_reduction_493 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_493: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_15_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_494 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_494 -.L_small_initial_partial_block_494: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_494: - - orq %r8,%r8 - je .L_after_reduction_494 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_494: - jmp .L_small_initial_blocks_encrypted_479 -.L_small_initial_num_blocks_is_16_479: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm0,%zmm6 - vpshufb %zmm29,%zmm3,%zmm7 - vpshufb %zmm29,%zmm4,%zmm10 - vpshufb %zmm29,%zmm5,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_495: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_495: - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_495: -.L_small_initial_blocks_encrypted_479: -.L_ghash_done_334: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -.L_enc_dec_done_334: - jmp .Lexit_gcm_encrypt -.Lexit_gcm_encrypt: - cmpq $256,%r8 - jbe .Lskip_hkeys_cleanup_496 - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %zmm0,0(%rsp) - vmovdqa64 %zmm0,64(%rsp) - vmovdqa64 %zmm0,128(%rsp) - vmovdqa64 %zmm0,192(%rsp) - vmovdqa64 %zmm0,256(%rsp) - vmovdqa64 %zmm0,320(%rsp) - vmovdqa64 %zmm0,384(%rsp) - vmovdqa64 %zmm0,448(%rsp) - vmovdqa64 %zmm0,512(%rsp) - vmovdqa64 %zmm0,576(%rsp) - vmovdqa64 %zmm0,640(%rsp) - vmovdqa64 %zmm0,704(%rsp) -.Lskip_hkeys_cleanup_496: - vzeroupper - leaq (%rbp),%rsp -.cfi_def_cfa_register %rsp - popq %r15 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r15 - popq %r14 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r14 - popq %r13 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r13 - popq %r12 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r12 - popq %rbp -.cfi_adjust_cfa_offset -8 -.cfi_restore %rbp - popq %rbx -.cfi_adjust_cfa_offset -8 -.cfi_restore %rbx - .byte 0xf3,0xc3 -.Lencrypt_seh_end: -.cfi_endproc -.size ossl_aes_gcm_encrypt_avx512, .-ossl_aes_gcm_encrypt_avx512 -.globl ossl_aes_gcm_decrypt_avx512 -.type ossl_aes_gcm_decrypt_avx512,@function -.align 32 -ossl_aes_gcm_decrypt_avx512: -.cfi_startproc -.Ldecrypt_seh_begin: -.byte 243,15,30,250 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-16 -.Ldecrypt_seh_push_rbx: - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-24 -.Ldecrypt_seh_push_rbp: - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 -.Ldecrypt_seh_push_r12: - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 -.Ldecrypt_seh_push_r13: - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 -.Ldecrypt_seh_push_r14: - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Ldecrypt_seh_push_r15: - - - - - - - - - - - leaq 0(%rsp),%rbp -.cfi_def_cfa_register %rbp -.Ldecrypt_seh_setfp: - -.Ldecrypt_seh_prolog_end: - subq $1588,%rsp - andq $(-64),%rsp - - - movl 240(%rdi),%eax - cmpl $9,%eax - je .Laes_gcm_decrypt_128_avx512 - cmpl $11,%eax - je .Laes_gcm_decrypt_192_avx512 - cmpl $13,%eax - je .Laes_gcm_decrypt_256_avx512 - xorl %eax,%eax - jmp .Lexit_gcm_decrypt -.align 32 -.Laes_gcm_decrypt_128_avx512: - orq %r8,%r8 - je .L_enc_dec_done_497 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je .L_partial_block_done_498 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - - vmovdqa64 %xmm0,%xmm6 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge .L_no_extra_mask_498 - subq %r13,%r12 -.L_no_extra_mask_498: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpand %xmm0,%xmm6,%xmm6 - vpshufb SHUF_MASK(%rip),%xmm6,%xmm6 - vpshufb %xmm5,%xmm6,%xmm6 - vpxorq %xmm6,%xmm14,%xmm14 - cmpq $0,%r13 - jl .L_partial_incomplete_498 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp .L_enc_dec_done_498 - -.L_partial_incomplete_498: - addq %r8,(%rdx) - movq %r8,%r11 - -.L_enc_dec_done_498: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -.L_partial_block_done_498: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je .L_enc_dec_done_497 - cmpq $256,%r8 - jbe .L_message_below_equal_16_blocks_497 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_499 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_499 -.L_next_16_overflow_499: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_499: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_500 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -.L_skip_hkeys_precomputation_500: - cmpq $512,%r8 - jb .L_message_below_32_blocks_497 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_501 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_501 -.L_next_16_overflow_501: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_501: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_502 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -.L_skip_hkeys_precomputation_502: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb .L_no_more_big_nblocks_497 -.L_encrypt_big_nblocks_497: - cmpb $240,%r15b - jae .L_16_blocks_overflow_503 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_503 -.L_16_blocks_overflow_503: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_503: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_504 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_504 -.L_16_blocks_overflow_504: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_504: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_505 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_505 -.L_16_blocks_overflow_505: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_505: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae .L_encrypt_big_nblocks_497 - -.L_no_more_big_nblocks_497: - - cmpq $512,%r8 - jae .L_encrypt_32_blocks_497 - - cmpq $256,%r8 - jae .L_encrypt_16_blocks_497 -.L_encrypt_0_blocks_ghash_32_497: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_506 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_506 - jb .L_last_num_blocks_is_7_1_506 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_506 - jb .L_last_num_blocks_is_11_9_506 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_506 - ja .L_last_num_blocks_is_16_506 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_506 - jmp .L_last_num_blocks_is_13_506 - -.L_last_num_blocks_is_11_9_506: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_506 - ja .L_last_num_blocks_is_11_506 - jmp .L_last_num_blocks_is_9_506 - -.L_last_num_blocks_is_7_1_506: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_506 - jb .L_last_num_blocks_is_3_1_506 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_506 - je .L_last_num_blocks_is_6_506 - jmp .L_last_num_blocks_is_5_506 - -.L_last_num_blocks_is_3_1_506: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_506 - je .L_last_num_blocks_is_2_506 -.L_last_num_blocks_is_1_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_507 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_507 - -.L_16_blocks_overflow_507: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_507: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_508 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_508 -.L_small_initial_partial_block_508: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_508 -.L_small_initial_compute_done_508: -.L_after_reduction_508: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_2_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_509 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_509 - -.L_16_blocks_overflow_509: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_509: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_510 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_510 -.L_small_initial_partial_block_510: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_510: - - orq %r8,%r8 - je .L_after_reduction_510 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_510: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_3_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_511 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_511 - -.L_16_blocks_overflow_511: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_511: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_512 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_512 -.L_small_initial_partial_block_512: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_512: - - orq %r8,%r8 - je .L_after_reduction_512 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_512: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_4_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_513 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_513 - -.L_16_blocks_overflow_513: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_513: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_514 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_514 -.L_small_initial_partial_block_514: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_514: - - orq %r8,%r8 - je .L_after_reduction_514 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_514: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_5_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_515 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_515 - -.L_16_blocks_overflow_515: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_515: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_516 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_516 -.L_small_initial_partial_block_516: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_516: - - orq %r8,%r8 - je .L_after_reduction_516 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_516: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_6_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_517 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_517 - -.L_16_blocks_overflow_517: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_517: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_518 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_518 -.L_small_initial_partial_block_518: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_518: - - orq %r8,%r8 - je .L_after_reduction_518 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_518: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_7_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_519 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_519 - -.L_16_blocks_overflow_519: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_519: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_520 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_520 -.L_small_initial_partial_block_520: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_520: - - orq %r8,%r8 - je .L_after_reduction_520 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_520: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_8_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_521 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_521 - -.L_16_blocks_overflow_521: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_521: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_522 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_522 -.L_small_initial_partial_block_522: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_522: - - orq %r8,%r8 - je .L_after_reduction_522 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_522: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_9_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_523 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_523 - -.L_16_blocks_overflow_523: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_523: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_524 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_524 -.L_small_initial_partial_block_524: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_524: - - orq %r8,%r8 - je .L_after_reduction_524 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_524: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_10_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_525 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_525 - -.L_16_blocks_overflow_525: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_525: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_526 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_526 -.L_small_initial_partial_block_526: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_526: - - orq %r8,%r8 - je .L_after_reduction_526 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_526: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_11_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_527 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_527 - -.L_16_blocks_overflow_527: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_527: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_528 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_528 -.L_small_initial_partial_block_528: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_528: - - orq %r8,%r8 - je .L_after_reduction_528 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_528: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_12_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_529 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_529 - -.L_16_blocks_overflow_529: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_529: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_530 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_530 -.L_small_initial_partial_block_530: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_530: - - orq %r8,%r8 - je .L_after_reduction_530 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_530: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_13_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_531 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_531 - -.L_16_blocks_overflow_531: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_531: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_532 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_532 -.L_small_initial_partial_block_532: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_532: - - orq %r8,%r8 - je .L_after_reduction_532 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_532: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_14_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_533 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_533 - -.L_16_blocks_overflow_533: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_533: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_534 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_534 -.L_small_initial_partial_block_534: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_534: - - orq %r8,%r8 - je .L_after_reduction_534 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_534: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_15_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_535 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_535 - -.L_16_blocks_overflow_535: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_535: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_536 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_536 -.L_small_initial_partial_block_536: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_536: - - orq %r8,%r8 - je .L_after_reduction_536 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_536: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_16_506: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_537 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_537 - -.L_16_blocks_overflow_537: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_537: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_538: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_538: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_538: - jmp .L_last_blocks_done_506 -.L_last_num_blocks_is_0_506: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_506: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_497 -.L_encrypt_32_blocks_497: - cmpb $240,%r15b - jae .L_16_blocks_overflow_539 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_539 -.L_16_blocks_overflow_539: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_539: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_540 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_540 -.L_16_blocks_overflow_540: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_540: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_541 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_541 - jb .L_last_num_blocks_is_7_1_541 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_541 - jb .L_last_num_blocks_is_11_9_541 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_541 - ja .L_last_num_blocks_is_16_541 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_541 - jmp .L_last_num_blocks_is_13_541 - -.L_last_num_blocks_is_11_9_541: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_541 - ja .L_last_num_blocks_is_11_541 - jmp .L_last_num_blocks_is_9_541 - -.L_last_num_blocks_is_7_1_541: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_541 - jb .L_last_num_blocks_is_3_1_541 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_541 - je .L_last_num_blocks_is_6_541 - jmp .L_last_num_blocks_is_5_541 - -.L_last_num_blocks_is_3_1_541: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_541 - je .L_last_num_blocks_is_2_541 -.L_last_num_blocks_is_1_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_542 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_542 - -.L_16_blocks_overflow_542: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_542: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_543 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_543 -.L_small_initial_partial_block_543: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_543 -.L_small_initial_compute_done_543: -.L_after_reduction_543: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_2_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_544 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_544 - -.L_16_blocks_overflow_544: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_544: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_545 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_545 -.L_small_initial_partial_block_545: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_545: - - orq %r8,%r8 - je .L_after_reduction_545 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_545: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_3_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_546 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_546 - -.L_16_blocks_overflow_546: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_546: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_547 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_547 -.L_small_initial_partial_block_547: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_547: - - orq %r8,%r8 - je .L_after_reduction_547 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_547: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_4_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_548 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_548 - -.L_16_blocks_overflow_548: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_548: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_549 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_549 -.L_small_initial_partial_block_549: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_549: - - orq %r8,%r8 - je .L_after_reduction_549 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_549: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_5_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_550 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_550 - -.L_16_blocks_overflow_550: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_550: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_551 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_551 -.L_small_initial_partial_block_551: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_551: - - orq %r8,%r8 - je .L_after_reduction_551 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_551: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_6_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_552 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_552 - -.L_16_blocks_overflow_552: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_552: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_553 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_553 -.L_small_initial_partial_block_553: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_553: - - orq %r8,%r8 - je .L_after_reduction_553 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_553: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_7_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_554 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_554 - -.L_16_blocks_overflow_554: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_554: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_555 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_555 -.L_small_initial_partial_block_555: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_555: - - orq %r8,%r8 - je .L_after_reduction_555 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_555: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_8_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_556 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_556 - -.L_16_blocks_overflow_556: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_556: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_557 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_557 -.L_small_initial_partial_block_557: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_557: - - orq %r8,%r8 - je .L_after_reduction_557 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_557: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_9_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_558 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_558 - -.L_16_blocks_overflow_558: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_558: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_559 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_559 -.L_small_initial_partial_block_559: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_559: - - orq %r8,%r8 - je .L_after_reduction_559 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_559: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_10_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_560 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_560 - -.L_16_blocks_overflow_560: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_560: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_561 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_561 -.L_small_initial_partial_block_561: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_561: - - orq %r8,%r8 - je .L_after_reduction_561 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_561: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_11_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_562 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_562 - -.L_16_blocks_overflow_562: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_562: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_563 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_563 -.L_small_initial_partial_block_563: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_563: - - orq %r8,%r8 - je .L_after_reduction_563 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_563: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_12_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_564 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_564 - -.L_16_blocks_overflow_564: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_564: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_565 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_565 -.L_small_initial_partial_block_565: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_565: - - orq %r8,%r8 - je .L_after_reduction_565 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_565: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_13_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_566 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_566 - -.L_16_blocks_overflow_566: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_566: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_567 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_567 -.L_small_initial_partial_block_567: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_567: - - orq %r8,%r8 - je .L_after_reduction_567 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_567: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_14_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_568 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_568 - -.L_16_blocks_overflow_568: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_568: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_569 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_569 -.L_small_initial_partial_block_569: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_569: - - orq %r8,%r8 - je .L_after_reduction_569 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_569: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_15_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_570 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_570 - -.L_16_blocks_overflow_570: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_570: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_571 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_571 -.L_small_initial_partial_block_571: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_571: - - orq %r8,%r8 - je .L_after_reduction_571 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_571: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_16_541: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_572 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_572 - -.L_16_blocks_overflow_572: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_572: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_573: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_573: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_573: - jmp .L_last_blocks_done_541 -.L_last_num_blocks_is_0_541: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_541: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_497 -.L_encrypt_16_blocks_497: - cmpb $240,%r15b - jae .L_16_blocks_overflow_574 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_574 -.L_16_blocks_overflow_574: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_574: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_575 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_575 - jb .L_last_num_blocks_is_7_1_575 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_575 - jb .L_last_num_blocks_is_11_9_575 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_575 - ja .L_last_num_blocks_is_16_575 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_575 - jmp .L_last_num_blocks_is_13_575 - -.L_last_num_blocks_is_11_9_575: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_575 - ja .L_last_num_blocks_is_11_575 - jmp .L_last_num_blocks_is_9_575 - -.L_last_num_blocks_is_7_1_575: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_575 - jb .L_last_num_blocks_is_3_1_575 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_575 - je .L_last_num_blocks_is_6_575 - jmp .L_last_num_blocks_is_5_575 - -.L_last_num_blocks_is_3_1_575: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_575 - je .L_last_num_blocks_is_2_575 -.L_last_num_blocks_is_1_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_576 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_576 - -.L_16_blocks_overflow_576: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_576: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_577 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_577 -.L_small_initial_partial_block_577: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_577 -.L_small_initial_compute_done_577: -.L_after_reduction_577: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_2_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_578 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_578 - -.L_16_blocks_overflow_578: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_578: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_579 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_579 -.L_small_initial_partial_block_579: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_579: - - orq %r8,%r8 - je .L_after_reduction_579 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_579: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_3_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_580 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_580 - -.L_16_blocks_overflow_580: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_580: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_581 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_581 -.L_small_initial_partial_block_581: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_581: - - orq %r8,%r8 - je .L_after_reduction_581 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_581: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_4_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_582 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_582 - -.L_16_blocks_overflow_582: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_582: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_583 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_583 -.L_small_initial_partial_block_583: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_583: - - orq %r8,%r8 - je .L_after_reduction_583 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_583: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_5_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_584 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_584 - -.L_16_blocks_overflow_584: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_584: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_585 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_585 -.L_small_initial_partial_block_585: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_585: - - orq %r8,%r8 - je .L_after_reduction_585 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_585: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_6_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_586 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_586 - -.L_16_blocks_overflow_586: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_586: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_587 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_587 -.L_small_initial_partial_block_587: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_587: - - orq %r8,%r8 - je .L_after_reduction_587 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_587: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_7_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_588 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_588 - -.L_16_blocks_overflow_588: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_588: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_589 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_589 -.L_small_initial_partial_block_589: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_589: - - orq %r8,%r8 - je .L_after_reduction_589 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_589: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_8_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_590 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_590 - -.L_16_blocks_overflow_590: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_590: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_591 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_591 -.L_small_initial_partial_block_591: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_591: - - orq %r8,%r8 - je .L_after_reduction_591 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_591: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_9_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_592 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_592 - -.L_16_blocks_overflow_592: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_592: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_593 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_593 -.L_small_initial_partial_block_593: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_593: - - orq %r8,%r8 - je .L_after_reduction_593 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_593: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_10_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_594 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_594 - -.L_16_blocks_overflow_594: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_594: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_595 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_595 -.L_small_initial_partial_block_595: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_595: - - orq %r8,%r8 - je .L_after_reduction_595 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_595: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_11_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_596 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_596 - -.L_16_blocks_overflow_596: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_596: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_597 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_597 -.L_small_initial_partial_block_597: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_597: - - orq %r8,%r8 - je .L_after_reduction_597 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_597: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_12_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_598 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_598 - -.L_16_blocks_overflow_598: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_598: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_599 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_599 -.L_small_initial_partial_block_599: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_599: - - orq %r8,%r8 - je .L_after_reduction_599 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_599: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_13_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_600 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_600 - -.L_16_blocks_overflow_600: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_600: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_601 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_601 -.L_small_initial_partial_block_601: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_601: - - orq %r8,%r8 - je .L_after_reduction_601 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_601: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_14_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_602 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_602 - -.L_16_blocks_overflow_602: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_602: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_603 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_603 -.L_small_initial_partial_block_603: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_603: - - orq %r8,%r8 - je .L_after_reduction_603 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_603: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_15_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_604 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_604 - -.L_16_blocks_overflow_604: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_604: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_605 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_605 -.L_small_initial_partial_block_605: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_605: - - orq %r8,%r8 - je .L_after_reduction_605 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_605: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_16_575: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_606 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_606 - -.L_16_blocks_overflow_606: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_606: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_607: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_607: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_607: - jmp .L_last_blocks_done_575 -.L_last_num_blocks_is_0_575: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_575: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_497 - -.L_message_below_32_blocks_497: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_608 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -.L_skip_hkeys_precomputation_608: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_609 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_609 - jb .L_last_num_blocks_is_7_1_609 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_609 - jb .L_last_num_blocks_is_11_9_609 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_609 - ja .L_last_num_blocks_is_16_609 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_609 - jmp .L_last_num_blocks_is_13_609 - -.L_last_num_blocks_is_11_9_609: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_609 - ja .L_last_num_blocks_is_11_609 - jmp .L_last_num_blocks_is_9_609 - -.L_last_num_blocks_is_7_1_609: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_609 - jb .L_last_num_blocks_is_3_1_609 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_609 - je .L_last_num_blocks_is_6_609 - jmp .L_last_num_blocks_is_5_609 - -.L_last_num_blocks_is_3_1_609: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_609 - je .L_last_num_blocks_is_2_609 -.L_last_num_blocks_is_1_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_610 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_610 - -.L_16_blocks_overflow_610: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_610: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_611 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_611 -.L_small_initial_partial_block_611: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_611 -.L_small_initial_compute_done_611: -.L_after_reduction_611: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_2_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_612 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_612 - -.L_16_blocks_overflow_612: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_612: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_613 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_613 -.L_small_initial_partial_block_613: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_613: - - orq %r8,%r8 - je .L_after_reduction_613 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_613: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_3_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_614 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_614 - -.L_16_blocks_overflow_614: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_614: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_615 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_615 -.L_small_initial_partial_block_615: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_615: - - orq %r8,%r8 - je .L_after_reduction_615 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_615: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_4_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_616 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_616 - -.L_16_blocks_overflow_616: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_616: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_617 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_617 -.L_small_initial_partial_block_617: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_617: - - orq %r8,%r8 - je .L_after_reduction_617 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_617: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_5_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_618 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_618 - -.L_16_blocks_overflow_618: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_618: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_619 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_619 -.L_small_initial_partial_block_619: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_619: - - orq %r8,%r8 - je .L_after_reduction_619 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_619: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_6_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_620 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_620 - -.L_16_blocks_overflow_620: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_620: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_621 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_621 -.L_small_initial_partial_block_621: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_621: - - orq %r8,%r8 - je .L_after_reduction_621 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_621: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_7_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_622 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_622 - -.L_16_blocks_overflow_622: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_622: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_623 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_623 -.L_small_initial_partial_block_623: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_623: - - orq %r8,%r8 - je .L_after_reduction_623 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_623: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_8_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_624 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_624 - -.L_16_blocks_overflow_624: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_624: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_625 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_625 -.L_small_initial_partial_block_625: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_625: - - orq %r8,%r8 - je .L_after_reduction_625 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_625: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_9_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_626 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_626 - -.L_16_blocks_overflow_626: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_626: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_627 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_627 -.L_small_initial_partial_block_627: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_627: - - orq %r8,%r8 - je .L_after_reduction_627 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_627: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_10_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_628 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_628 - -.L_16_blocks_overflow_628: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_628: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_629 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_629 -.L_small_initial_partial_block_629: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_629: - - orq %r8,%r8 - je .L_after_reduction_629 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_629: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_11_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_630 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_630 - -.L_16_blocks_overflow_630: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_630: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_631 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_631 -.L_small_initial_partial_block_631: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_631: - - orq %r8,%r8 - je .L_after_reduction_631 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_631: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_12_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_632 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_632 - -.L_16_blocks_overflow_632: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_632: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_633 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_633 -.L_small_initial_partial_block_633: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_633: - - orq %r8,%r8 - je .L_after_reduction_633 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_633: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_13_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_634 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_634 - -.L_16_blocks_overflow_634: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_634: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_635 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_635 -.L_small_initial_partial_block_635: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_635: - - orq %r8,%r8 - je .L_after_reduction_635 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_635: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_14_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_636 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_636 - -.L_16_blocks_overflow_636: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_636: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_637 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_637 -.L_small_initial_partial_block_637: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_637: - - orq %r8,%r8 - je .L_after_reduction_637 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_637: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_15_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_638 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_638 - -.L_16_blocks_overflow_638: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_638: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_639 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_639 -.L_small_initial_partial_block_639: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_639: - - orq %r8,%r8 - je .L_after_reduction_639 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_639: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_16_609: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_640 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_640 - -.L_16_blocks_overflow_640: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_640: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_641: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_641: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_641: - jmp .L_last_blocks_done_609 -.L_last_num_blocks_is_0_609: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_609: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_497 - -.L_message_below_equal_16_blocks_497: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je .L_small_initial_num_blocks_is_8_642 - jl .L_small_initial_num_blocks_is_7_1_642 - - - cmpq $12,%r12 - je .L_small_initial_num_blocks_is_12_642 - jl .L_small_initial_num_blocks_is_11_9_642 - - - cmpq $16,%r12 - je .L_small_initial_num_blocks_is_16_642 - cmpq $15,%r12 - je .L_small_initial_num_blocks_is_15_642 - cmpq $14,%r12 - je .L_small_initial_num_blocks_is_14_642 - jmp .L_small_initial_num_blocks_is_13_642 - -.L_small_initial_num_blocks_is_11_9_642: - - cmpq $11,%r12 - je .L_small_initial_num_blocks_is_11_642 - cmpq $10,%r12 - je .L_small_initial_num_blocks_is_10_642 - jmp .L_small_initial_num_blocks_is_9_642 - -.L_small_initial_num_blocks_is_7_1_642: - cmpq $4,%r12 - je .L_small_initial_num_blocks_is_4_642 - jl .L_small_initial_num_blocks_is_3_1_642 - - cmpq $7,%r12 - je .L_small_initial_num_blocks_is_7_642 - cmpq $6,%r12 - je .L_small_initial_num_blocks_is_6_642 - jmp .L_small_initial_num_blocks_is_5_642 - -.L_small_initial_num_blocks_is_3_1_642: - - cmpq $3,%r12 - je .L_small_initial_num_blocks_is_3_642 - cmpq $2,%r12 - je .L_small_initial_num_blocks_is_2_642 - - - - - -.L_small_initial_num_blocks_is_1_642: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm6,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_643 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_643 -.L_small_initial_partial_block_643: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp .L_after_reduction_643 -.L_small_initial_compute_done_643: -.L_after_reduction_643: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_2_642: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm6,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_644 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_644 -.L_small_initial_partial_block_644: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_644: - - orq %r8,%r8 - je .L_after_reduction_644 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_644: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_3_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_645 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_645 -.L_small_initial_partial_block_645: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_645: - - orq %r8,%r8 - je .L_after_reduction_645 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_645: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_4_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_646 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_646 -.L_small_initial_partial_block_646: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_646: - - orq %r8,%r8 - je .L_after_reduction_646 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_646: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_5_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %xmm29,%xmm7,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_647 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_647 -.L_small_initial_partial_block_647: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_647: - - orq %r8,%r8 - je .L_after_reduction_647 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_647: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_6_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %ymm29,%ymm7,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_648 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_648 -.L_small_initial_partial_block_648: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_648: - - orq %r8,%r8 - je .L_after_reduction_648 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_648: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_7_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_649 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_649 -.L_small_initial_partial_block_649: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_649: - - orq %r8,%r8 - je .L_after_reduction_649 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_649: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_8_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_650 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_650 -.L_small_initial_partial_block_650: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_650: - - orq %r8,%r8 - je .L_after_reduction_650 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_650: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_9_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %xmm29,%xmm10,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_651 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_651 -.L_small_initial_partial_block_651: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_651: - - orq %r8,%r8 - je .L_after_reduction_651 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_651: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_10_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %ymm29,%ymm10,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_652 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_652 -.L_small_initial_partial_block_652: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_652: - - orq %r8,%r8 - je .L_after_reduction_652 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_652: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_11_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_653 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_653 -.L_small_initial_partial_block_653: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_653: - - orq %r8,%r8 - je .L_after_reduction_653 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_653: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_12_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_654 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_654 -.L_small_initial_partial_block_654: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_654: - - orq %r8,%r8 - je .L_after_reduction_654 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_654: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_13_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %xmm29,%xmm11,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_655 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_655 -.L_small_initial_partial_block_655: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_655: - - orq %r8,%r8 - je .L_after_reduction_655 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_655: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_14_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %ymm29,%ymm11,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_656 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_656 -.L_small_initial_partial_block_656: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_656: - - orq %r8,%r8 - je .L_after_reduction_656 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_656: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_15_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_657 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_657 -.L_small_initial_partial_block_657: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_657: - - orq %r8,%r8 - je .L_after_reduction_657 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_657: - jmp .L_small_initial_blocks_encrypted_642 -.L_small_initial_num_blocks_is_16_642: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_658: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_658: - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_658: -.L_small_initial_blocks_encrypted_642: -.L_ghash_done_497: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -.L_enc_dec_done_497: - jmp .Lexit_gcm_decrypt -.align 32 -.Laes_gcm_decrypt_192_avx512: - orq %r8,%r8 - je .L_enc_dec_done_659 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je .L_partial_block_done_660 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - - vmovdqa64 %xmm0,%xmm6 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge .L_no_extra_mask_660 - subq %r13,%r12 -.L_no_extra_mask_660: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpand %xmm0,%xmm6,%xmm6 - vpshufb SHUF_MASK(%rip),%xmm6,%xmm6 - vpshufb %xmm5,%xmm6,%xmm6 - vpxorq %xmm6,%xmm14,%xmm14 - cmpq $0,%r13 - jl .L_partial_incomplete_660 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp .L_enc_dec_done_660 - -.L_partial_incomplete_660: - addq %r8,(%rdx) - movq %r8,%r11 - -.L_enc_dec_done_660: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -.L_partial_block_done_660: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je .L_enc_dec_done_659 - cmpq $256,%r8 - jbe .L_message_below_equal_16_blocks_659 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_661 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_661 -.L_next_16_overflow_661: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_661: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_662 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -.L_skip_hkeys_precomputation_662: - cmpq $512,%r8 - jb .L_message_below_32_blocks_659 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_663 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_663 -.L_next_16_overflow_663: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_663: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_664 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -.L_skip_hkeys_precomputation_664: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb .L_no_more_big_nblocks_659 -.L_encrypt_big_nblocks_659: - cmpb $240,%r15b - jae .L_16_blocks_overflow_665 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_665 -.L_16_blocks_overflow_665: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_665: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_666 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_666 -.L_16_blocks_overflow_666: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_666: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_667 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_667 -.L_16_blocks_overflow_667: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_667: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae .L_encrypt_big_nblocks_659 - -.L_no_more_big_nblocks_659: - - cmpq $512,%r8 - jae .L_encrypt_32_blocks_659 - - cmpq $256,%r8 - jae .L_encrypt_16_blocks_659 -.L_encrypt_0_blocks_ghash_32_659: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_668 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_668 - jb .L_last_num_blocks_is_7_1_668 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_668 - jb .L_last_num_blocks_is_11_9_668 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_668 - ja .L_last_num_blocks_is_16_668 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_668 - jmp .L_last_num_blocks_is_13_668 - -.L_last_num_blocks_is_11_9_668: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_668 - ja .L_last_num_blocks_is_11_668 - jmp .L_last_num_blocks_is_9_668 - -.L_last_num_blocks_is_7_1_668: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_668 - jb .L_last_num_blocks_is_3_1_668 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_668 - je .L_last_num_blocks_is_6_668 - jmp .L_last_num_blocks_is_5_668 - -.L_last_num_blocks_is_3_1_668: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_668 - je .L_last_num_blocks_is_2_668 -.L_last_num_blocks_is_1_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_669 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_669 - -.L_16_blocks_overflow_669: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_669: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_670 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_670 -.L_small_initial_partial_block_670: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_670 -.L_small_initial_compute_done_670: -.L_after_reduction_670: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_2_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_671 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_671 - -.L_16_blocks_overflow_671: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_671: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_672 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_672 -.L_small_initial_partial_block_672: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_672: - - orq %r8,%r8 - je .L_after_reduction_672 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_672: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_3_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_673 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_673 - -.L_16_blocks_overflow_673: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_673: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_674 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_674 -.L_small_initial_partial_block_674: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_674: - - orq %r8,%r8 - je .L_after_reduction_674 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_674: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_4_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_675 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_675 - -.L_16_blocks_overflow_675: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_675: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_676 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_676 -.L_small_initial_partial_block_676: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_676: - - orq %r8,%r8 - je .L_after_reduction_676 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_676: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_5_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_677 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_677 - -.L_16_blocks_overflow_677: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_677: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_678 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_678 -.L_small_initial_partial_block_678: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_678: - - orq %r8,%r8 - je .L_after_reduction_678 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_678: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_6_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_679 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_679 - -.L_16_blocks_overflow_679: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_679: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_680 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_680 -.L_small_initial_partial_block_680: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_680: - - orq %r8,%r8 - je .L_after_reduction_680 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_680: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_7_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_681 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_681 - -.L_16_blocks_overflow_681: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_681: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_682 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_682 -.L_small_initial_partial_block_682: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_682: - - orq %r8,%r8 - je .L_after_reduction_682 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_682: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_8_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_683 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_683 - -.L_16_blocks_overflow_683: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_683: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_684 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_684 -.L_small_initial_partial_block_684: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_684: - - orq %r8,%r8 - je .L_after_reduction_684 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_684: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_9_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_685 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_685 - -.L_16_blocks_overflow_685: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_685: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_686 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_686 -.L_small_initial_partial_block_686: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_686: - - orq %r8,%r8 - je .L_after_reduction_686 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_686: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_10_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_687 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_687 - -.L_16_blocks_overflow_687: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_687: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_688 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_688 -.L_small_initial_partial_block_688: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_688: - - orq %r8,%r8 - je .L_after_reduction_688 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_688: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_11_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_689 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_689 - -.L_16_blocks_overflow_689: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_689: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_690 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_690 -.L_small_initial_partial_block_690: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_690: - - orq %r8,%r8 - je .L_after_reduction_690 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_690: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_12_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_691 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_691 - -.L_16_blocks_overflow_691: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_691: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_692 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_692 -.L_small_initial_partial_block_692: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_692: - - orq %r8,%r8 - je .L_after_reduction_692 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_692: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_13_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_693 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_693 - -.L_16_blocks_overflow_693: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_693: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_694 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_694 -.L_small_initial_partial_block_694: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_694: - - orq %r8,%r8 - je .L_after_reduction_694 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_694: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_14_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_695 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_695 - -.L_16_blocks_overflow_695: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_695: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_696 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_696 -.L_small_initial_partial_block_696: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_696: - - orq %r8,%r8 - je .L_after_reduction_696 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_696: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_15_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_697 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_697 - -.L_16_blocks_overflow_697: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_697: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_698 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_698 -.L_small_initial_partial_block_698: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_698: - - orq %r8,%r8 - je .L_after_reduction_698 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_698: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_16_668: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_699 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_699 - -.L_16_blocks_overflow_699: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_699: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_700: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_700: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_700: - jmp .L_last_blocks_done_668 -.L_last_num_blocks_is_0_668: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_668: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_659 -.L_encrypt_32_blocks_659: - cmpb $240,%r15b - jae .L_16_blocks_overflow_701 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_701 -.L_16_blocks_overflow_701: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_701: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_702 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_702 -.L_16_blocks_overflow_702: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_702: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_703 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_703 - jb .L_last_num_blocks_is_7_1_703 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_703 - jb .L_last_num_blocks_is_11_9_703 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_703 - ja .L_last_num_blocks_is_16_703 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_703 - jmp .L_last_num_blocks_is_13_703 - -.L_last_num_blocks_is_11_9_703: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_703 - ja .L_last_num_blocks_is_11_703 - jmp .L_last_num_blocks_is_9_703 - -.L_last_num_blocks_is_7_1_703: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_703 - jb .L_last_num_blocks_is_3_1_703 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_703 - je .L_last_num_blocks_is_6_703 - jmp .L_last_num_blocks_is_5_703 - -.L_last_num_blocks_is_3_1_703: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_703 - je .L_last_num_blocks_is_2_703 -.L_last_num_blocks_is_1_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_704 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_704 - -.L_16_blocks_overflow_704: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_704: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_705 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_705 -.L_small_initial_partial_block_705: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_705 -.L_small_initial_compute_done_705: -.L_after_reduction_705: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_2_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_706 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_706 - -.L_16_blocks_overflow_706: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_706: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_707 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_707 -.L_small_initial_partial_block_707: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_707: - - orq %r8,%r8 - je .L_after_reduction_707 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_707: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_3_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_708 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_708 - -.L_16_blocks_overflow_708: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_708: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_709 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_709 -.L_small_initial_partial_block_709: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_709: - - orq %r8,%r8 - je .L_after_reduction_709 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_709: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_4_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_710 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_710 - -.L_16_blocks_overflow_710: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_710: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_711 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_711 -.L_small_initial_partial_block_711: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_711: - - orq %r8,%r8 - je .L_after_reduction_711 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_711: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_5_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_712 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_712 - -.L_16_blocks_overflow_712: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_712: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_713 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_713 -.L_small_initial_partial_block_713: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_713: - - orq %r8,%r8 - je .L_after_reduction_713 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_713: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_6_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_714 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_714 - -.L_16_blocks_overflow_714: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_714: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_715 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_715 -.L_small_initial_partial_block_715: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_715: - - orq %r8,%r8 - je .L_after_reduction_715 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_715: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_7_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_716 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_716 - -.L_16_blocks_overflow_716: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_716: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_717 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_717 -.L_small_initial_partial_block_717: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_717: - - orq %r8,%r8 - je .L_after_reduction_717 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_717: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_8_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_718 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_718 - -.L_16_blocks_overflow_718: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_718: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_719 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_719 -.L_small_initial_partial_block_719: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_719: - - orq %r8,%r8 - je .L_after_reduction_719 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_719: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_9_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_720 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_720 - -.L_16_blocks_overflow_720: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_720: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_721 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_721 -.L_small_initial_partial_block_721: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_721: - - orq %r8,%r8 - je .L_after_reduction_721 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_721: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_10_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_722 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_722 - -.L_16_blocks_overflow_722: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_722: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_723 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_723 -.L_small_initial_partial_block_723: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_723: - - orq %r8,%r8 - je .L_after_reduction_723 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_723: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_11_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_724 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_724 - -.L_16_blocks_overflow_724: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_724: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_725 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_725 -.L_small_initial_partial_block_725: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_725: - - orq %r8,%r8 - je .L_after_reduction_725 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_725: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_12_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_726 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_726 - -.L_16_blocks_overflow_726: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_726: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_727 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_727 -.L_small_initial_partial_block_727: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_727: - - orq %r8,%r8 - je .L_after_reduction_727 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_727: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_13_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_728 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_728 - -.L_16_blocks_overflow_728: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_728: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_729 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_729 -.L_small_initial_partial_block_729: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_729: - - orq %r8,%r8 - je .L_after_reduction_729 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_729: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_14_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_730 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_730 - -.L_16_blocks_overflow_730: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_730: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_731 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_731 -.L_small_initial_partial_block_731: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_731: - - orq %r8,%r8 - je .L_after_reduction_731 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_731: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_15_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_732 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_732 - -.L_16_blocks_overflow_732: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_732: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_733 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_733 -.L_small_initial_partial_block_733: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_733: - - orq %r8,%r8 - je .L_after_reduction_733 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_733: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_16_703: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_734 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_734 - -.L_16_blocks_overflow_734: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_734: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_735: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_735: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_735: - jmp .L_last_blocks_done_703 -.L_last_num_blocks_is_0_703: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_703: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_659 -.L_encrypt_16_blocks_659: - cmpb $240,%r15b - jae .L_16_blocks_overflow_736 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_736 -.L_16_blocks_overflow_736: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_736: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_737 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_737 - jb .L_last_num_blocks_is_7_1_737 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_737 - jb .L_last_num_blocks_is_11_9_737 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_737 - ja .L_last_num_blocks_is_16_737 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_737 - jmp .L_last_num_blocks_is_13_737 - -.L_last_num_blocks_is_11_9_737: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_737 - ja .L_last_num_blocks_is_11_737 - jmp .L_last_num_blocks_is_9_737 - -.L_last_num_blocks_is_7_1_737: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_737 - jb .L_last_num_blocks_is_3_1_737 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_737 - je .L_last_num_blocks_is_6_737 - jmp .L_last_num_blocks_is_5_737 - -.L_last_num_blocks_is_3_1_737: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_737 - je .L_last_num_blocks_is_2_737 -.L_last_num_blocks_is_1_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_738 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_738 - -.L_16_blocks_overflow_738: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_738: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_739 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_739 -.L_small_initial_partial_block_739: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_739 -.L_small_initial_compute_done_739: -.L_after_reduction_739: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_2_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_740 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_740 - -.L_16_blocks_overflow_740: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_740: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_741 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_741 -.L_small_initial_partial_block_741: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_741: - - orq %r8,%r8 - je .L_after_reduction_741 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_741: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_3_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_742 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_742 - -.L_16_blocks_overflow_742: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_742: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_743 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_743 -.L_small_initial_partial_block_743: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_743: - - orq %r8,%r8 - je .L_after_reduction_743 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_743: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_4_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_744 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_744 - -.L_16_blocks_overflow_744: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_744: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_745 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_745 -.L_small_initial_partial_block_745: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_745: - - orq %r8,%r8 - je .L_after_reduction_745 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_745: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_5_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_746 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_746 - -.L_16_blocks_overflow_746: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_746: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_747 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_747 -.L_small_initial_partial_block_747: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_747: - - orq %r8,%r8 - je .L_after_reduction_747 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_747: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_6_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_748 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_748 - -.L_16_blocks_overflow_748: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_748: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_749 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_749 -.L_small_initial_partial_block_749: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_749: - - orq %r8,%r8 - je .L_after_reduction_749 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_749: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_7_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_750 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_750 - -.L_16_blocks_overflow_750: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_750: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_751 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_751 -.L_small_initial_partial_block_751: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_751: - - orq %r8,%r8 - je .L_after_reduction_751 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_751: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_8_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_752 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_752 - -.L_16_blocks_overflow_752: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_752: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_753 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_753 -.L_small_initial_partial_block_753: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_753: - - orq %r8,%r8 - je .L_after_reduction_753 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_753: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_9_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_754 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_754 - -.L_16_blocks_overflow_754: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_754: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_755 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_755 -.L_small_initial_partial_block_755: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_755: - - orq %r8,%r8 - je .L_after_reduction_755 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_755: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_10_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_756 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_756 - -.L_16_blocks_overflow_756: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_756: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_757 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_757 -.L_small_initial_partial_block_757: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_757: - - orq %r8,%r8 - je .L_after_reduction_757 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_757: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_11_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_758 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_758 - -.L_16_blocks_overflow_758: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_758: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_759 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_759 -.L_small_initial_partial_block_759: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_759: - - orq %r8,%r8 - je .L_after_reduction_759 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_759: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_12_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_760 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_760 - -.L_16_blocks_overflow_760: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_760: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_761 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_761 -.L_small_initial_partial_block_761: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_761: - - orq %r8,%r8 - je .L_after_reduction_761 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_761: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_13_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_762 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_762 - -.L_16_blocks_overflow_762: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_762: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_763 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_763 -.L_small_initial_partial_block_763: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_763: - - orq %r8,%r8 - je .L_after_reduction_763 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_763: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_14_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_764 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_764 - -.L_16_blocks_overflow_764: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_764: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_765 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_765 -.L_small_initial_partial_block_765: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_765: - - orq %r8,%r8 - je .L_after_reduction_765 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_765: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_15_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_766 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_766 - -.L_16_blocks_overflow_766: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_766: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_767 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_767 -.L_small_initial_partial_block_767: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_767: - - orq %r8,%r8 - je .L_after_reduction_767 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_767: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_16_737: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_768 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_768 - -.L_16_blocks_overflow_768: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_768: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_769: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_769: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_769: - jmp .L_last_blocks_done_737 -.L_last_num_blocks_is_0_737: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_737: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_659 - -.L_message_below_32_blocks_659: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_770 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -.L_skip_hkeys_precomputation_770: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_771 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_771 - jb .L_last_num_blocks_is_7_1_771 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_771 - jb .L_last_num_blocks_is_11_9_771 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_771 - ja .L_last_num_blocks_is_16_771 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_771 - jmp .L_last_num_blocks_is_13_771 - -.L_last_num_blocks_is_11_9_771: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_771 - ja .L_last_num_blocks_is_11_771 - jmp .L_last_num_blocks_is_9_771 - -.L_last_num_blocks_is_7_1_771: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_771 - jb .L_last_num_blocks_is_3_1_771 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_771 - je .L_last_num_blocks_is_6_771 - jmp .L_last_num_blocks_is_5_771 - -.L_last_num_blocks_is_3_1_771: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_771 - je .L_last_num_blocks_is_2_771 -.L_last_num_blocks_is_1_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_772 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_772 - -.L_16_blocks_overflow_772: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_772: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_773 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_773 -.L_small_initial_partial_block_773: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_773 -.L_small_initial_compute_done_773: -.L_after_reduction_773: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_2_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_774 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_774 - -.L_16_blocks_overflow_774: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_774: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_775 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_775 -.L_small_initial_partial_block_775: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_775: - - orq %r8,%r8 - je .L_after_reduction_775 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_775: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_3_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_776 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_776 - -.L_16_blocks_overflow_776: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_776: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_777 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_777 -.L_small_initial_partial_block_777: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_777: - - orq %r8,%r8 - je .L_after_reduction_777 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_777: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_4_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_778 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_778 - -.L_16_blocks_overflow_778: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_778: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_779 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_779 -.L_small_initial_partial_block_779: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_779: - - orq %r8,%r8 - je .L_after_reduction_779 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_779: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_5_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_780 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_780 - -.L_16_blocks_overflow_780: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_780: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_781 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_781 -.L_small_initial_partial_block_781: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_781: - - orq %r8,%r8 - je .L_after_reduction_781 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_781: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_6_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_782 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_782 - -.L_16_blocks_overflow_782: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_782: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_783 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_783 -.L_small_initial_partial_block_783: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_783: - - orq %r8,%r8 - je .L_after_reduction_783 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_783: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_7_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_784 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_784 - -.L_16_blocks_overflow_784: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_784: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_785 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_785 -.L_small_initial_partial_block_785: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_785: - - orq %r8,%r8 - je .L_after_reduction_785 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_785: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_8_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_786 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_786 - -.L_16_blocks_overflow_786: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_786: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_787 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_787 -.L_small_initial_partial_block_787: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_787: - - orq %r8,%r8 - je .L_after_reduction_787 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_787: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_9_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_788 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_788 - -.L_16_blocks_overflow_788: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_788: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_789 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_789 -.L_small_initial_partial_block_789: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_789: - - orq %r8,%r8 - je .L_after_reduction_789 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_789: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_10_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_790 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_790 - -.L_16_blocks_overflow_790: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_790: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_791 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_791 -.L_small_initial_partial_block_791: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_791: - - orq %r8,%r8 - je .L_after_reduction_791 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_791: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_11_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_792 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_792 - -.L_16_blocks_overflow_792: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_792: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_793 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_793 -.L_small_initial_partial_block_793: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_793: - - orq %r8,%r8 - je .L_after_reduction_793 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_793: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_12_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_794 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_794 - -.L_16_blocks_overflow_794: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_794: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_795 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_795 -.L_small_initial_partial_block_795: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_795: - - orq %r8,%r8 - je .L_after_reduction_795 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_795: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_13_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_796 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_796 - -.L_16_blocks_overflow_796: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_796: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_797 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_797 -.L_small_initial_partial_block_797: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_797: - - orq %r8,%r8 - je .L_after_reduction_797 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_797: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_14_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_798 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_798 - -.L_16_blocks_overflow_798: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_798: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_799 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_799 -.L_small_initial_partial_block_799: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_799: - - orq %r8,%r8 - je .L_after_reduction_799 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_799: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_15_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_800 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_800 - -.L_16_blocks_overflow_800: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_800: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_801 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_801 -.L_small_initial_partial_block_801: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_801: - - orq %r8,%r8 - je .L_after_reduction_801 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_801: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_16_771: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_802 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_802 - -.L_16_blocks_overflow_802: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_802: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_803: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_803: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_803: - jmp .L_last_blocks_done_771 -.L_last_num_blocks_is_0_771: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_771: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_659 - -.L_message_below_equal_16_blocks_659: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je .L_small_initial_num_blocks_is_8_804 - jl .L_small_initial_num_blocks_is_7_1_804 - - - cmpq $12,%r12 - je .L_small_initial_num_blocks_is_12_804 - jl .L_small_initial_num_blocks_is_11_9_804 - - - cmpq $16,%r12 - je .L_small_initial_num_blocks_is_16_804 - cmpq $15,%r12 - je .L_small_initial_num_blocks_is_15_804 - cmpq $14,%r12 - je .L_small_initial_num_blocks_is_14_804 - jmp .L_small_initial_num_blocks_is_13_804 - -.L_small_initial_num_blocks_is_11_9_804: - - cmpq $11,%r12 - je .L_small_initial_num_blocks_is_11_804 - cmpq $10,%r12 - je .L_small_initial_num_blocks_is_10_804 - jmp .L_small_initial_num_blocks_is_9_804 - -.L_small_initial_num_blocks_is_7_1_804: - cmpq $4,%r12 - je .L_small_initial_num_blocks_is_4_804 - jl .L_small_initial_num_blocks_is_3_1_804 - - cmpq $7,%r12 - je .L_small_initial_num_blocks_is_7_804 - cmpq $6,%r12 - je .L_small_initial_num_blocks_is_6_804 - jmp .L_small_initial_num_blocks_is_5_804 - -.L_small_initial_num_blocks_is_3_1_804: - - cmpq $3,%r12 - je .L_small_initial_num_blocks_is_3_804 - cmpq $2,%r12 - je .L_small_initial_num_blocks_is_2_804 - - - - - -.L_small_initial_num_blocks_is_1_804: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm6,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_805 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_805 -.L_small_initial_partial_block_805: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp .L_after_reduction_805 -.L_small_initial_compute_done_805: -.L_after_reduction_805: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_2_804: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm6,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_806 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_806 -.L_small_initial_partial_block_806: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_806: - - orq %r8,%r8 - je .L_after_reduction_806 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_806: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_3_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_807 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_807 -.L_small_initial_partial_block_807: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_807: - - orq %r8,%r8 - je .L_after_reduction_807 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_807: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_4_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_808 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_808 -.L_small_initial_partial_block_808: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_808: - - orq %r8,%r8 - je .L_after_reduction_808 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_808: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_5_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %xmm29,%xmm7,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_809 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_809 -.L_small_initial_partial_block_809: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_809: - - orq %r8,%r8 - je .L_after_reduction_809 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_809: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_6_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %ymm29,%ymm7,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_810 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_810 -.L_small_initial_partial_block_810: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_810: - - orq %r8,%r8 - je .L_after_reduction_810 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_810: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_7_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_811 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_811 -.L_small_initial_partial_block_811: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_811: - - orq %r8,%r8 - je .L_after_reduction_811 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_811: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_8_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_812 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_812 -.L_small_initial_partial_block_812: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_812: - - orq %r8,%r8 - je .L_after_reduction_812 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_812: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_9_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %xmm29,%xmm10,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_813 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_813 -.L_small_initial_partial_block_813: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_813: - - orq %r8,%r8 - je .L_after_reduction_813 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_813: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_10_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %ymm29,%ymm10,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_814 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_814 -.L_small_initial_partial_block_814: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_814: - - orq %r8,%r8 - je .L_after_reduction_814 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_814: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_11_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_815 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_815 -.L_small_initial_partial_block_815: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_815: - - orq %r8,%r8 - je .L_after_reduction_815 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_815: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_12_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_816 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_816 -.L_small_initial_partial_block_816: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_816: - - orq %r8,%r8 - je .L_after_reduction_816 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_816: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_13_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %xmm29,%xmm11,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_817 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_817 -.L_small_initial_partial_block_817: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_817: - - orq %r8,%r8 - je .L_after_reduction_817 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_817: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_14_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %ymm29,%ymm11,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_818 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_818 -.L_small_initial_partial_block_818: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_818: - - orq %r8,%r8 - je .L_after_reduction_818 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_818: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_15_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_819 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_819 -.L_small_initial_partial_block_819: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_819: - - orq %r8,%r8 - je .L_after_reduction_819 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_819: - jmp .L_small_initial_blocks_encrypted_804 -.L_small_initial_num_blocks_is_16_804: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_820: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_820: - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_820: -.L_small_initial_blocks_encrypted_804: -.L_ghash_done_659: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -.L_enc_dec_done_659: - jmp .Lexit_gcm_decrypt -.align 32 -.Laes_gcm_decrypt_256_avx512: - orq %r8,%r8 - je .L_enc_dec_done_821 - xorq %r14,%r14 - vmovdqu64 64(%rsi),%xmm14 - - movq (%rdx),%r11 - orq %r11,%r11 - je .L_partial_block_done_822 - movl $16,%r10d - leaq byte_len_to_mask_table(%rip),%r12 - cmpq %r10,%r8 - cmovcq %r8,%r10 - kmovw (%r12,%r10,2),%k1 - vmovdqu8 (%rcx),%xmm0{%k1}{z} - - vmovdqu64 16(%rsi),%xmm3 - vmovdqu64 336(%rsi),%xmm4 - - - - leaq SHIFT_MASK(%rip),%r12 - addq %r11,%r12 - vmovdqu64 (%r12),%xmm5 - vpshufb %xmm5,%xmm3,%xmm3 - - vmovdqa64 %xmm0,%xmm6 - vpxorq %xmm0,%xmm3,%xmm3 - - - leaq (%r8,%r11,1),%r13 - subq $16,%r13 - jge .L_no_extra_mask_822 - subq %r13,%r12 -.L_no_extra_mask_822: - - - - vmovdqu64 16(%r12),%xmm0 - vpand %xmm0,%xmm3,%xmm3 - vpand %xmm0,%xmm6,%xmm6 - vpshufb SHUF_MASK(%rip),%xmm6,%xmm6 - vpshufb %xmm5,%xmm6,%xmm6 - vpxorq %xmm6,%xmm14,%xmm14 - cmpq $0,%r13 - jl .L_partial_incomplete_822 - - vpclmulqdq $0x11,%xmm4,%xmm14,%xmm7 - vpclmulqdq $0x00,%xmm4,%xmm14,%xmm10 - vpclmulqdq $0x01,%xmm4,%xmm14,%xmm11 - vpclmulqdq $0x10,%xmm4,%xmm14,%xmm14 - vpxorq %xmm11,%xmm14,%xmm14 - - vpsrldq $8,%xmm14,%xmm11 - vpslldq $8,%xmm14,%xmm14 - vpxorq %xmm11,%xmm7,%xmm7 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vmovdqu64 POLY2(%rip),%xmm11 - - vpclmulqdq $0x01,%xmm14,%xmm11,%xmm10 - vpslldq $8,%xmm10,%xmm10 - vpxorq %xmm10,%xmm14,%xmm14 - - - - vpclmulqdq $0x00,%xmm14,%xmm11,%xmm10 - vpsrldq $4,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm14,%xmm11,%xmm14 - vpslldq $4,%xmm14,%xmm14 - - vpternlogq $0x96,%xmm10,%xmm7,%xmm14 - - movq $0,(%rdx) - - movq %r11,%r12 - movq $16,%r11 - subq %r12,%r11 - jmp .L_enc_dec_done_822 - -.L_partial_incomplete_822: - addq %r8,(%rdx) - movq %r8,%r11 - -.L_enc_dec_done_822: - - - leaq byte_len_to_mask_table(%rip),%r12 - kmovw (%r12,%r11,2),%k1 - vmovdqu64 %xmm14,64(%rsi) - movq %r9,%r12 - vmovdqu8 %xmm3,(%r12){%k1} -.L_partial_block_done_822: - vmovdqu64 0(%rsi),%xmm2 - subq %r11,%r8 - je .L_enc_dec_done_821 - cmpq $256,%r8 - jbe .L_message_below_equal_16_blocks_821 - - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vmovdqa64 ddq_addbe_4444(%rip),%zmm27 - vmovdqa64 ddq_addbe_1234(%rip),%zmm28 - - - - - - - vmovd %xmm2,%r15d - andl $255,%r15d - - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpshufb %zmm29,%zmm2,%zmm2 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_823 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_823 -.L_next_16_overflow_823: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_823: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 0(%rcx,%r11,1),%zmm0 - vmovdqu8 64(%rcx,%r11,1),%zmm3 - vmovdqu8 128(%rcx,%r11,1),%zmm4 - vmovdqu8 192(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 208(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 224(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,0(%r10,%r11,1) - vmovdqu8 %zmm10,64(%r10,%r11,1) - vmovdqu8 %zmm11,128(%r10,%r11,1) - vmovdqu8 %zmm12,192(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,768(%rsp) - vmovdqa64 %zmm10,832(%rsp) - vmovdqa64 %zmm11,896(%rsp) - vmovdqa64 %zmm12,960(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_824 - - vmovdqu64 288(%rsi),%zmm0 - vmovdqu64 %zmm0,704(%rsp) - - vmovdqu64 224(%rsi),%zmm3 - vmovdqu64 %zmm3,640(%rsp) - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 160(%rsi),%zmm4 - vmovdqu64 %zmm4,576(%rsp) - - vmovdqu64 96(%rsi),%zmm5 - vmovdqu64 %zmm5,512(%rsp) -.L_skip_hkeys_precomputation_824: - cmpq $512,%r8 - jb .L_message_below_32_blocks_821 - - - - cmpb $240,%r15b - jae .L_next_16_overflow_825 - vpaddd %zmm28,%zmm2,%zmm7 - vpaddd %zmm27,%zmm7,%zmm10 - vpaddd %zmm27,%zmm10,%zmm11 - vpaddd %zmm27,%zmm11,%zmm12 - jmp .L_next_16_ok_825 -.L_next_16_overflow_825: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm12 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm7 - vpaddd %zmm12,%zmm7,%zmm10 - vpaddd %zmm12,%zmm10,%zmm11 - vpaddd %zmm12,%zmm11,%zmm12 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vpshufb %zmm29,%zmm12,%zmm12 -.L_next_16_ok_825: - vshufi64x2 $255,%zmm12,%zmm12,%zmm2 - addb $16,%r15b - - vmovdqu8 256(%rcx,%r11,1),%zmm0 - vmovdqu8 320(%rcx,%r11,1),%zmm3 - vmovdqu8 384(%rcx,%r11,1),%zmm4 - vmovdqu8 448(%rcx,%r11,1),%zmm5 - - - vbroadcastf64x2 0(%rdi),%zmm6 - vpxorq %zmm6,%zmm7,%zmm7 - vpxorq %zmm6,%zmm10,%zmm10 - vpxorq %zmm6,%zmm11,%zmm11 - vpxorq %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 16(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 32(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 48(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 64(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 80(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 96(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 112(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 128(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 144(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 160(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 176(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 192(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 208(%rdi),%zmm6 - vaesenc %zmm6,%zmm7,%zmm7 - vaesenc %zmm6,%zmm10,%zmm10 - vaesenc %zmm6,%zmm11,%zmm11 - vaesenc %zmm6,%zmm12,%zmm12 - vbroadcastf64x2 224(%rdi),%zmm6 - vaesenclast %zmm6,%zmm7,%zmm7 - vaesenclast %zmm6,%zmm10,%zmm10 - vaesenclast %zmm6,%zmm11,%zmm11 - vaesenclast %zmm6,%zmm12,%zmm12 - - - vpxorq %zmm0,%zmm7,%zmm7 - vpxorq %zmm3,%zmm10,%zmm10 - vpxorq %zmm4,%zmm11,%zmm11 - vpxorq %zmm5,%zmm12,%zmm12 - - - movq %r9,%r10 - vmovdqu8 %zmm7,256(%r10,%r11,1) - vmovdqu8 %zmm10,320(%r10,%r11,1) - vmovdqu8 %zmm11,384(%r10,%r11,1) - vmovdqu8 %zmm12,448(%r10,%r11,1) - - vpshufb %zmm29,%zmm0,%zmm7 - vpshufb %zmm29,%zmm3,%zmm10 - vpshufb %zmm29,%zmm4,%zmm11 - vpshufb %zmm29,%zmm5,%zmm12 - vmovdqa64 %zmm7,1024(%rsp) - vmovdqa64 %zmm10,1088(%rsp) - vmovdqa64 %zmm11,1152(%rsp) - vmovdqa64 %zmm12,1216(%rsp) - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_826 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,192(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,128(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,64(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,0(%rsp) -.L_skip_hkeys_precomputation_826: - movq $1,%r14 - addq $512,%r11 - subq $512,%r8 - - cmpq $768,%r8 - jb .L_no_more_big_nblocks_821 -.L_encrypt_big_nblocks_821: - cmpb $240,%r15b - jae .L_16_blocks_overflow_827 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_827 -.L_16_blocks_overflow_827: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_827: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_828 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_828 -.L_16_blocks_overflow_828: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_828: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_829 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_829 -.L_16_blocks_overflow_829: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_829: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 512(%rcx,%r11,1),%zmm17 - vmovdqu8 576(%rcx,%r11,1),%zmm19 - vmovdqu8 640(%rcx,%r11,1),%zmm20 - vmovdqu8 704(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - - - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpternlogq $0x96,%zmm15,%zmm12,%zmm6 - vpxorq %zmm24,%zmm6,%zmm6 - vpternlogq $0x96,%zmm10,%zmm13,%zmm7 - vpxorq %zmm25,%zmm7,%zmm7 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vextracti64x4 $1,%zmm6,%ymm12 - vpxorq %ymm12,%ymm6,%ymm6 - vextracti32x4 $1,%ymm6,%xmm12 - vpxorq %xmm12,%xmm6,%xmm6 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm6 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,512(%r10,%r11,1) - vmovdqu8 %zmm3,576(%r10,%r11,1) - vmovdqu8 %zmm4,640(%r10,%r11,1) - vmovdqu8 %zmm5,704(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1024(%rsp) - vmovdqa64 %zmm3,1088(%rsp) - vmovdqa64 %zmm4,1152(%rsp) - vmovdqa64 %zmm5,1216(%rsp) - vmovdqa64 %zmm6,%zmm14 - - addq $768,%r11 - subq $768,%r8 - cmpq $768,%r8 - jae .L_encrypt_big_nblocks_821 - -.L_no_more_big_nblocks_821: - - cmpq $512,%r8 - jae .L_encrypt_32_blocks_821 - - cmpq $256,%r8 - jae .L_encrypt_16_blocks_821 -.L_encrypt_0_blocks_ghash_32_821: - movl %r8d,%r10d - andl $~15,%r10d - movl $256,%ebx - subl %r10d,%ebx - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - addl $256,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_830 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_830 - jb .L_last_num_blocks_is_7_1_830 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_830 - jb .L_last_num_blocks_is_11_9_830 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_830 - ja .L_last_num_blocks_is_16_830 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_830 - jmp .L_last_num_blocks_is_13_830 - -.L_last_num_blocks_is_11_9_830: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_830 - ja .L_last_num_blocks_is_11_830 - jmp .L_last_num_blocks_is_9_830 - -.L_last_num_blocks_is_7_1_830: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_830 - jb .L_last_num_blocks_is_3_1_830 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_830 - je .L_last_num_blocks_is_6_830 - jmp .L_last_num_blocks_is_5_830 - -.L_last_num_blocks_is_3_1_830: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_830 - je .L_last_num_blocks_is_2_830 -.L_last_num_blocks_is_1_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_831 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_831 - -.L_16_blocks_overflow_831: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_831: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_832 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_832 -.L_small_initial_partial_block_832: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_832 -.L_small_initial_compute_done_832: -.L_after_reduction_832: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_2_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_833 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_833 - -.L_16_blocks_overflow_833: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_833: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_834 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_834 -.L_small_initial_partial_block_834: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_834: - - orq %r8,%r8 - je .L_after_reduction_834 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_834: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_3_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_835 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_835 - -.L_16_blocks_overflow_835: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_835: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_836 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_836 -.L_small_initial_partial_block_836: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_836: - - orq %r8,%r8 - je .L_after_reduction_836 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_836: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_4_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_837 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_837 - -.L_16_blocks_overflow_837: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_837: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_838 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_838 -.L_small_initial_partial_block_838: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_838: - - orq %r8,%r8 - je .L_after_reduction_838 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_838: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_5_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_839 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_839 - -.L_16_blocks_overflow_839: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_839: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_840 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_840 -.L_small_initial_partial_block_840: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_840: - - orq %r8,%r8 - je .L_after_reduction_840 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_840: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_6_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_841 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_841 - -.L_16_blocks_overflow_841: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_841: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_842 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_842 -.L_small_initial_partial_block_842: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_842: - - orq %r8,%r8 - je .L_after_reduction_842 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_842: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_7_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_843 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_843 - -.L_16_blocks_overflow_843: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_843: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_844 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_844 -.L_small_initial_partial_block_844: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_844: - - orq %r8,%r8 - je .L_after_reduction_844 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_844: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_8_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_845 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_845 - -.L_16_blocks_overflow_845: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_845: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_846 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_846 -.L_small_initial_partial_block_846: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_846: - - orq %r8,%r8 - je .L_after_reduction_846 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_846: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_9_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_847 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_847 - -.L_16_blocks_overflow_847: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_847: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_848 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_848 -.L_small_initial_partial_block_848: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_848: - - orq %r8,%r8 - je .L_after_reduction_848 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_848: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_10_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_849 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_849 - -.L_16_blocks_overflow_849: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_849: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_850 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_850 -.L_small_initial_partial_block_850: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_850: - - orq %r8,%r8 - je .L_after_reduction_850 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_850: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_11_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_851 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_851 - -.L_16_blocks_overflow_851: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_851: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_852 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_852 -.L_small_initial_partial_block_852: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_852: - - orq %r8,%r8 - je .L_after_reduction_852 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_852: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_12_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_853 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_853 - -.L_16_blocks_overflow_853: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_853: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_854 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_854 -.L_small_initial_partial_block_854: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_854: - - orq %r8,%r8 - je .L_after_reduction_854 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_854: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_13_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_855 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_855 - -.L_16_blocks_overflow_855: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_855: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_856 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_856 -.L_small_initial_partial_block_856: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_856: - - orq %r8,%r8 - je .L_after_reduction_856 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_856: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_14_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_857 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_857 - -.L_16_blocks_overflow_857: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_857: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_858 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_858 -.L_small_initial_partial_block_858: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_858: - - orq %r8,%r8 - je .L_after_reduction_858 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_858: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_15_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_859 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_859 - -.L_16_blocks_overflow_859: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_859: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_860 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_860 -.L_small_initial_partial_block_860: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_860: - - orq %r8,%r8 - je .L_after_reduction_860 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_860: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_16_830: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_861 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_861 - -.L_16_blocks_overflow_861: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_861: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm14,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_862: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_862: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_862: - jmp .L_last_blocks_done_830 -.L_last_num_blocks_is_0_830: - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_830: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_821 -.L_encrypt_32_blocks_821: - cmpb $240,%r15b - jae .L_16_blocks_overflow_863 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_863 -.L_16_blocks_overflow_863: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_863: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - cmpb $240,%r15b - jae .L_16_blocks_overflow_864 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_864 -.L_16_blocks_overflow_864: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_864: - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1024(%rsp),%zmm8 - vmovdqu64 256(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 320(%rsp),%zmm18 - vmovdqa64 1088(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 384(%rsp),%zmm1 - vmovdqa64 1152(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 448(%rsp),%zmm18 - vmovdqa64 1216(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 256(%rcx,%r11,1),%zmm17 - vmovdqu8 320(%rcx,%r11,1),%zmm19 - vmovdqu8 384(%rcx,%r11,1),%zmm20 - vmovdqu8 448(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm15,%zmm10,%zmm26 - vpternlogq $0x96,%zmm12,%zmm6,%zmm24 - vpternlogq $0x96,%zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,256(%r10,%r11,1) - vmovdqu8 %zmm3,320(%r10,%r11,1) - vmovdqu8 %zmm4,384(%r10,%r11,1) - vmovdqu8 %zmm5,448(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,768(%rsp) - vmovdqa64 %zmm3,832(%rsp) - vmovdqa64 %zmm4,896(%rsp) - vmovdqa64 %zmm5,960(%rsp) - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - - subq $512,%r8 - addq $512,%r11 - movl %r8d,%r10d - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_865 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_865 - jb .L_last_num_blocks_is_7_1_865 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_865 - jb .L_last_num_blocks_is_11_9_865 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_865 - ja .L_last_num_blocks_is_16_865 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_865 - jmp .L_last_num_blocks_is_13_865 - -.L_last_num_blocks_is_11_9_865: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_865 - ja .L_last_num_blocks_is_11_865 - jmp .L_last_num_blocks_is_9_865 - -.L_last_num_blocks_is_7_1_865: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_865 - jb .L_last_num_blocks_is_3_1_865 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_865 - je .L_last_num_blocks_is_6_865 - jmp .L_last_num_blocks_is_5_865 - -.L_last_num_blocks_is_3_1_865: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_865 - je .L_last_num_blocks_is_2_865 -.L_last_num_blocks_is_1_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_866 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_866 - -.L_16_blocks_overflow_866: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_866: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_867 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_867 -.L_small_initial_partial_block_867: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_867 -.L_small_initial_compute_done_867: -.L_after_reduction_867: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_2_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_868 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_868 - -.L_16_blocks_overflow_868: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_868: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_869 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_869 -.L_small_initial_partial_block_869: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_869: - - orq %r8,%r8 - je .L_after_reduction_869 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_869: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_3_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_870 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_870 - -.L_16_blocks_overflow_870: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_870: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_871 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_871 -.L_small_initial_partial_block_871: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_871: - - orq %r8,%r8 - je .L_after_reduction_871 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_871: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_4_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_872 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_872 - -.L_16_blocks_overflow_872: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_872: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_873 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_873 -.L_small_initial_partial_block_873: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_873: - - orq %r8,%r8 - je .L_after_reduction_873 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_873: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_5_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_874 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_874 - -.L_16_blocks_overflow_874: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_874: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_875 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_875 -.L_small_initial_partial_block_875: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_875: - - orq %r8,%r8 - je .L_after_reduction_875 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_875: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_6_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_876 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_876 - -.L_16_blocks_overflow_876: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_876: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_877 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_877 -.L_small_initial_partial_block_877: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_877: - - orq %r8,%r8 - je .L_after_reduction_877 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_877: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_7_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_878 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_878 - -.L_16_blocks_overflow_878: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_878: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_879 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_879 -.L_small_initial_partial_block_879: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_879: - - orq %r8,%r8 - je .L_after_reduction_879 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_879: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_8_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_880 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_880 - -.L_16_blocks_overflow_880: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_880: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_881 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_881 -.L_small_initial_partial_block_881: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_881: - - orq %r8,%r8 - je .L_after_reduction_881 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_881: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_9_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_882 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_882 - -.L_16_blocks_overflow_882: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_882: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_883 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_883 -.L_small_initial_partial_block_883: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_883: - - orq %r8,%r8 - je .L_after_reduction_883 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_883: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_10_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_884 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_884 - -.L_16_blocks_overflow_884: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_884: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_885 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_885 -.L_small_initial_partial_block_885: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_885: - - orq %r8,%r8 - je .L_after_reduction_885 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_885: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_11_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_886 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_886 - -.L_16_blocks_overflow_886: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_886: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_887 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_887 -.L_small_initial_partial_block_887: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_887: - - orq %r8,%r8 - je .L_after_reduction_887 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_887: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_12_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_888 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_888 - -.L_16_blocks_overflow_888: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_888: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_889 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_889 -.L_small_initial_partial_block_889: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_889: - - orq %r8,%r8 - je .L_after_reduction_889 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_889: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_13_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_890 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_890 - -.L_16_blocks_overflow_890: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_890: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_891 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_891 -.L_small_initial_partial_block_891: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_891: - - orq %r8,%r8 - je .L_after_reduction_891 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_891: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_14_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_892 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_892 - -.L_16_blocks_overflow_892: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_892: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_893 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_893 -.L_small_initial_partial_block_893: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_893: - - orq %r8,%r8 - je .L_after_reduction_893 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_893: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_15_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_894 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_894 - -.L_16_blocks_overflow_894: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_894: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_895 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_895 -.L_small_initial_partial_block_895: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_895: - - orq %r8,%r8 - je .L_after_reduction_895 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_895: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_16_865: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_896 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_896 - -.L_16_blocks_overflow_896: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_896: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_897: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_897: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_897: - jmp .L_last_blocks_done_865 -.L_last_num_blocks_is_0_865: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_865: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_821 -.L_encrypt_16_blocks_821: - cmpb $240,%r15b - jae .L_16_blocks_overflow_898 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_898 -.L_16_blocks_overflow_898: - vpshufb %zmm29,%zmm2,%zmm2 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_898: - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp),%zmm1 - - - - - vshufi64x2 $255,%zmm5,%zmm5,%zmm2 - addb $16,%r15b - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - - - - - - - - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm6 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm6 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - - - - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21 - - - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm15,%zmm10,%zmm26 - vpxorq %zmm12,%zmm6,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - - - - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - - - - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1) - vpshufb %zmm29,%zmm17,%zmm0 - vpshufb %zmm29,%zmm19,%zmm3 - vpshufb %zmm29,%zmm20,%zmm4 - vpshufb %zmm29,%zmm21,%zmm5 - vmovdqa64 %zmm0,1280(%rsp) - vmovdqa64 %zmm3,1344(%rsp) - vmovdqa64 %zmm4,1408(%rsp) - vmovdqa64 %zmm5,1472(%rsp) - vmovdqa64 1024(%rsp),%zmm13 - vmovdqu64 256(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1088(%rsp),%zmm13 - vmovdqu64 320(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1152(%rsp),%zmm13 - vmovdqu64 384(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1216(%rsp),%zmm13 - vmovdqu64 448(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_899 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_899 - jb .L_last_num_blocks_is_7_1_899 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_899 - jb .L_last_num_blocks_is_11_9_899 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_899 - ja .L_last_num_blocks_is_16_899 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_899 - jmp .L_last_num_blocks_is_13_899 - -.L_last_num_blocks_is_11_9_899: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_899 - ja .L_last_num_blocks_is_11_899 - jmp .L_last_num_blocks_is_9_899 - -.L_last_num_blocks_is_7_1_899: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_899 - jb .L_last_num_blocks_is_3_1_899 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_899 - je .L_last_num_blocks_is_6_899 - jmp .L_last_num_blocks_is_5_899 - -.L_last_num_blocks_is_3_1_899: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_899 - je .L_last_num_blocks_is_2_899 -.L_last_num_blocks_is_1_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_900 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_900 - -.L_16_blocks_overflow_900: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_900: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %xmm31,%xmm0,%xmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_901 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_901 -.L_small_initial_partial_block_901: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_901 -.L_small_initial_compute_done_901: -.L_after_reduction_901: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_2_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_902 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_902 - -.L_16_blocks_overflow_902: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_902: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %ymm31,%ymm0,%ymm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_903 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_903 -.L_small_initial_partial_block_903: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_903: - - orq %r8,%r8 - je .L_after_reduction_903 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_903: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_3_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_904 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_904 - -.L_16_blocks_overflow_904: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_904: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_905 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_905 -.L_small_initial_partial_block_905: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_905: - - orq %r8,%r8 - je .L_after_reduction_905 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_905: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_4_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_906 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_906 - -.L_16_blocks_overflow_906: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_906: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_907 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_907 -.L_small_initial_partial_block_907: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_907: - - orq %r8,%r8 - je .L_after_reduction_907 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_907: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_5_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_908 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_908 - -.L_16_blocks_overflow_908: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_908: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_909 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_909 -.L_small_initial_partial_block_909: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_909: - - orq %r8,%r8 - je .L_after_reduction_909 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_909: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_6_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_910 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_910 - -.L_16_blocks_overflow_910: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_910: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_911 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_911 -.L_small_initial_partial_block_911: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_911: - - orq %r8,%r8 - je .L_after_reduction_911 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_911: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_7_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_912 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_912 - -.L_16_blocks_overflow_912: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_912: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_913 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_913 -.L_small_initial_partial_block_913: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_913: - - orq %r8,%r8 - je .L_after_reduction_913 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_913: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_8_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_914 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_914 - -.L_16_blocks_overflow_914: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_914: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_915 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_915 -.L_small_initial_partial_block_915: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_915: - - orq %r8,%r8 - je .L_after_reduction_915 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_915: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_9_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_916 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_916 - -.L_16_blocks_overflow_916: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_916: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_917 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_917 -.L_small_initial_partial_block_917: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_917: - - orq %r8,%r8 - je .L_after_reduction_917 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_917: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_10_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_918 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_918 - -.L_16_blocks_overflow_918: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_918: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_919 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_919 -.L_small_initial_partial_block_919: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_919: - - orq %r8,%r8 - je .L_after_reduction_919 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_919: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_11_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_920 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_920 - -.L_16_blocks_overflow_920: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_920: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_921 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_921 -.L_small_initial_partial_block_921: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_921: - - orq %r8,%r8 - je .L_after_reduction_921 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_921: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_12_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_922 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_922 - -.L_16_blocks_overflow_922: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_922: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_923 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_923 -.L_small_initial_partial_block_923: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_923: - - orq %r8,%r8 - je .L_after_reduction_923 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_923: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_13_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_924 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_924 - -.L_16_blocks_overflow_924: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_924: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_925 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_925 -.L_small_initial_partial_block_925: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_925: - - orq %r8,%r8 - je .L_after_reduction_925 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_925: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_14_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_926 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_926 - -.L_16_blocks_overflow_926: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_926: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_927 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_927 -.L_small_initial_partial_block_927: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_927: - - orq %r8,%r8 - je .L_after_reduction_927 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_927: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_15_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_928 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_928 - -.L_16_blocks_overflow_928: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_928: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_929 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_929 -.L_small_initial_partial_block_929: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_929: - - orq %r8,%r8 - je .L_after_reduction_929 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_929: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_16_899: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_930 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_930 - -.L_16_blocks_overflow_930: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_930: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vmovdqa64 1280(%rsp),%zmm8 - vmovdqu64 512(%rsp),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 576(%rsp),%zmm18 - vmovdqa64 1344(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 640(%rsp),%zmm1 - vmovdqa64 1408(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 704(%rsp),%zmm18 - vmovdqa64 1472(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpternlogq $0x96,%zmm12,%zmm24,%zmm14 - vpternlogq $0x96,%zmm13,%zmm25,%zmm7 - vpternlogq $0x96,%zmm15,%zmm26,%zmm10 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vpsrldq $8,%zmm10,%zmm15 - vpslldq $8,%zmm10,%zmm10 - - vmovdqa64 POLY2(%rip),%xmm16 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vpxorq %zmm15,%zmm14,%zmm14 - vpxorq %zmm10,%zmm7,%zmm7 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vextracti64x4 $1,%zmm14,%ymm12 - vpxorq %ymm12,%ymm14,%ymm14 - vextracti32x4 $1,%ymm14,%xmm12 - vpxorq %xmm12,%xmm14,%xmm14 - vextracti64x4 $1,%zmm7,%ymm13 - vpxorq %ymm13,%ymm7,%ymm7 - vextracti32x4 $1,%ymm7,%xmm13 - vpxorq %xmm13,%xmm7,%xmm7 - vbroadcastf64x2 176(%rdi),%zmm31 - vpclmulqdq $0x01,%xmm7,%xmm16,%xmm13 - vpslldq $8,%xmm13,%xmm13 - vpxorq %xmm13,%xmm7,%xmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vpclmulqdq $0x00,%xmm13,%xmm16,%xmm12 - vpsrldq $4,%xmm12,%xmm12 - vpclmulqdq $0x10,%xmm13,%xmm16,%xmm15 - vpslldq $4,%xmm15,%xmm15 - - vpternlogq $0x96,%xmm12,%xmm15,%xmm14 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_931: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vpxorq %zmm14,%zmm17,%zmm17 - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm31,%zmm5,%zmm5 - vpxorq %zmm8,%zmm0,%zmm0 - vpxorq %zmm22,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_931: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_931: - jmp .L_last_blocks_done_899 -.L_last_num_blocks_is_0_899: - vmovdqa64 1280(%rsp),%zmm13 - vmovdqu64 512(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1344(%rsp),%zmm13 - vmovdqu64 576(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 1408(%rsp),%zmm13 - vmovdqu64 640(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 1472(%rsp),%zmm13 - vmovdqu64 704(%rsp),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_899: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_821 - -.L_message_below_32_blocks_821: - - - subq $256,%r8 - addq $256,%r11 - movl %r8d,%r10d - testq %r14,%r14 - jnz .L_skip_hkeys_precomputation_932 - vmovdqu64 640(%rsp),%zmm3 - - - vshufi64x2 $0x00,%zmm3,%zmm3,%zmm3 - - vmovdqu64 576(%rsp),%zmm4 - vmovdqu64 512(%rsp),%zmm5 - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,448(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,384(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm4,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm4,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm4,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm4,%zmm4 - vpxorq %zmm10,%zmm4,%zmm4 - - vpsrldq $8,%zmm4,%zmm10 - vpslldq $8,%zmm4,%zmm4 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm4,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm4,%zmm4 - - - - vpclmulqdq $0x00,%zmm4,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm4,%zmm10,%zmm4 - vpslldq $4,%zmm4,%zmm4 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm4 - - vmovdqu64 %zmm4,320(%rsp) - - vpclmulqdq $0x11,%zmm3,%zmm5,%zmm6 - vpclmulqdq $0x00,%zmm3,%zmm5,%zmm7 - vpclmulqdq $0x01,%zmm3,%zmm5,%zmm10 - vpclmulqdq $0x10,%zmm3,%zmm5,%zmm5 - vpxorq %zmm10,%zmm5,%zmm5 - - vpsrldq $8,%zmm5,%zmm10 - vpslldq $8,%zmm5,%zmm5 - vpxorq %zmm10,%zmm6,%zmm6 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vmovdqu64 POLY2(%rip),%zmm10 - - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm7 - vpslldq $8,%zmm7,%zmm7 - vpxorq %zmm7,%zmm5,%zmm5 - - - - vpclmulqdq $0x00,%zmm5,%zmm10,%zmm7 - vpsrldq $4,%zmm7,%zmm7 - vpclmulqdq $0x10,%zmm5,%zmm10,%zmm5 - vpslldq $4,%zmm5,%zmm5 - - vpternlogq $0x96,%zmm7,%zmm6,%zmm5 - - vmovdqu64 %zmm5,256(%rsp) -.L_skip_hkeys_precomputation_932: - movq $1,%r14 - andl $~15,%r10d - movl $512,%ebx - subl %r10d,%ebx - movl %r8d,%r10d - addl $15,%r10d - shrl $4,%r10d - je .L_last_num_blocks_is_0_933 - - cmpl $8,%r10d - je .L_last_num_blocks_is_8_933 - jb .L_last_num_blocks_is_7_1_933 - - - cmpl $12,%r10d - je .L_last_num_blocks_is_12_933 - jb .L_last_num_blocks_is_11_9_933 - - - cmpl $15,%r10d - je .L_last_num_blocks_is_15_933 - ja .L_last_num_blocks_is_16_933 - cmpl $14,%r10d - je .L_last_num_blocks_is_14_933 - jmp .L_last_num_blocks_is_13_933 - -.L_last_num_blocks_is_11_9_933: - - cmpl $10,%r10d - je .L_last_num_blocks_is_10_933 - ja .L_last_num_blocks_is_11_933 - jmp .L_last_num_blocks_is_9_933 - -.L_last_num_blocks_is_7_1_933: - cmpl $4,%r10d - je .L_last_num_blocks_is_4_933 - jb .L_last_num_blocks_is_3_1_933 - - cmpl $6,%r10d - ja .L_last_num_blocks_is_7_933 - je .L_last_num_blocks_is_6_933 - jmp .L_last_num_blocks_is_5_933 - -.L_last_num_blocks_is_3_1_933: - - cmpl $2,%r10d - ja .L_last_num_blocks_is_3_933 - je .L_last_num_blocks_is_2_933 -.L_last_num_blocks_is_1_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $255,%r15d - jae .L_16_blocks_overflow_934 - vpaddd %xmm28,%xmm2,%xmm0 - jmp .L_16_blocks_ok_934 - -.L_16_blocks_overflow_934: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %xmm29,%xmm0,%xmm0 -.L_16_blocks_ok_934: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%xmm17{%k1}{z} - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %xmm30,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %xmm31,%xmm0,%xmm0 - vaesenclast %xmm30,%xmm0,%xmm0 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %xmm29,%xmm17,%xmm17 - vextracti32x4 $0,%zmm17,%xmm7 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_935 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_935 -.L_small_initial_partial_block_935: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm0 - - - vpclmulqdq $0x01,%xmm25,%xmm0,%xmm3 - vpslldq $8,%xmm3,%xmm3 - vpxorq %xmm3,%xmm25,%xmm3 - - - vpclmulqdq $0x00,%xmm3,%xmm0,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm3,%xmm0,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm4,%xmm14 - - - - - - - - - - - - - vpxorq %xmm7,%xmm14,%xmm14 - - jmp .L_after_reduction_935 -.L_small_initial_compute_done_935: -.L_after_reduction_935: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_2_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $254,%r15d - jae .L_16_blocks_overflow_936 - vpaddd %ymm28,%ymm2,%ymm0 - jmp .L_16_blocks_ok_936 - -.L_16_blocks_overflow_936: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %ymm29,%ymm0,%ymm0 -.L_16_blocks_ok_936: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%ymm17{%k1}{z} - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %ymm30,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %ymm31,%ymm0,%ymm0 - vaesenclast %ymm30,%ymm0,%ymm0 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %ymm29,%ymm17,%ymm17 - vextracti32x4 $1,%zmm17,%xmm7 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_937 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_937 -.L_small_initial_partial_block_937: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm17,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm17,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm17,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm17,%xmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_937: - - orq %r8,%r8 - je .L_after_reduction_937 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_937: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_3_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $253,%r15d - jae .L_16_blocks_overflow_938 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_938 - -.L_16_blocks_overflow_938: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_938: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $2,%zmm17,%xmm7 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_939 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_939 -.L_small_initial_partial_block_939: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm17,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm17,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm17,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm17,%ymm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_939: - - orq %r8,%r8 - je .L_after_reduction_939 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_939: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_4_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $252,%r15d - jae .L_16_blocks_overflow_940 - vpaddd %zmm28,%zmm2,%zmm0 - jmp .L_16_blocks_ok_940 - -.L_16_blocks_overflow_940: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpshufb %zmm29,%zmm0,%zmm0 -.L_16_blocks_ok_940: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm0,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm0,%zmm0 - vpxorq %zmm17,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm17,%zmm17{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vextracti32x4 $3,%zmm17,%xmm7 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_941 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_941 -.L_small_initial_partial_block_941: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpxorq %zmm26,%zmm4,%zmm4 - vpxorq %zmm24,%zmm0,%zmm0 - vpxorq %zmm25,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_941: - - orq %r8,%r8 - je .L_after_reduction_941 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_941: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_5_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $251,%r15d - jae .L_16_blocks_overflow_942 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %xmm27,%xmm0,%xmm3 - jmp .L_16_blocks_ok_942 - -.L_16_blocks_overflow_942: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 -.L_16_blocks_ok_942: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%xmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %xmm30,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %xmm31,%xmm3,%xmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %xmm30,%xmm3,%xmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %xmm19,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %xmm29,%xmm19,%xmm19 - vextracti32x4 $0,%zmm19,%xmm7 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_943 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_943 -.L_small_initial_partial_block_943: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_943: - - orq %r8,%r8 - je .L_after_reduction_943 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_943: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_6_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $250,%r15d - jae .L_16_blocks_overflow_944 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %ymm27,%ymm0,%ymm3 - jmp .L_16_blocks_ok_944 - -.L_16_blocks_overflow_944: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 -.L_16_blocks_ok_944: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%ymm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %ymm30,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %ymm31,%ymm3,%ymm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %ymm30,%ymm3,%ymm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %ymm29,%ymm19,%ymm19 - vextracti32x4 $1,%zmm19,%xmm7 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_945 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_945 -.L_small_initial_partial_block_945: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm19,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm19,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm19,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm19,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_945: - - orq %r8,%r8 - je .L_after_reduction_945 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_945: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_7_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $249,%r15d - jae .L_16_blocks_overflow_946 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_946 - -.L_16_blocks_overflow_946: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_946: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $2,%zmm19,%xmm7 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_947 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_947 -.L_small_initial_partial_block_947: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm19,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm19,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm19,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm19,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_947: - - orq %r8,%r8 - je .L_after_reduction_947 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_947: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_8_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $64,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $248,%r15d - jae .L_16_blocks_overflow_948 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - jmp .L_16_blocks_ok_948 - -.L_16_blocks_overflow_948: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 -.L_16_blocks_ok_948: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm3,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm19,%zmm19{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vextracti32x4 $3,%zmm19,%xmm7 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_949 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_949 -.L_small_initial_partial_block_949: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_949: - - orq %r8,%r8 - je .L_after_reduction_949 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_949: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_9_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $247,%r15d - jae .L_16_blocks_overflow_950 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %xmm27,%xmm3,%xmm4 - jmp .L_16_blocks_ok_950 - -.L_16_blocks_overflow_950: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 -.L_16_blocks_ok_950: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%xmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %xmm30,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %xmm31,%xmm4,%xmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %xmm30,%xmm4,%xmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %xmm20,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %xmm29,%xmm20,%xmm20 - vextracti32x4 $0,%zmm20,%xmm7 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_951 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_951 -.L_small_initial_partial_block_951: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_951: - - orq %r8,%r8 - je .L_after_reduction_951 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_951: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_10_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $246,%r15d - jae .L_16_blocks_overflow_952 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %ymm27,%ymm3,%ymm4 - jmp .L_16_blocks_ok_952 - -.L_16_blocks_overflow_952: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 -.L_16_blocks_ok_952: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%ymm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %ymm30,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %ymm31,%ymm4,%ymm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %ymm30,%ymm4,%ymm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %ymm20,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %ymm29,%ymm20,%ymm20 - vextracti32x4 $1,%zmm20,%xmm7 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_953 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_953 -.L_small_initial_partial_block_953: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm20,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm20,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm20,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm20,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_953: - - orq %r8,%r8 - je .L_after_reduction_953 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_953: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_11_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $245,%r15d - jae .L_16_blocks_overflow_954 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_954 - -.L_16_blocks_overflow_954: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_954: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $2,%zmm20,%xmm7 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_955 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_955 -.L_small_initial_partial_block_955: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm20,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm20,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm20,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm20,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_955: - - orq %r8,%r8 - je .L_after_reduction_955 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_955: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_12_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $128,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $244,%r15d - jae .L_16_blocks_overflow_956 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - jmp .L_16_blocks_ok_956 - -.L_16_blocks_overflow_956: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 -.L_16_blocks_ok_956: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm4,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm20,%zmm20{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vextracti32x4 $3,%zmm20,%xmm7 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_957 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_957 -.L_small_initial_partial_block_957: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vpxorq %zmm8,%zmm0,%zmm8 - vpxorq %zmm22,%zmm3,%zmm22 - vpxorq %zmm30,%zmm4,%zmm30 - vpxorq %zmm31,%zmm5,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_957: - - orq %r8,%r8 - je .L_after_reduction_957 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_957: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_13_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $243,%r15d - jae .L_16_blocks_overflow_958 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %xmm27,%xmm4,%xmm5 - jmp .L_16_blocks_ok_958 - -.L_16_blocks_overflow_958: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 -.L_16_blocks_ok_958: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $0,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%xmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %xmm30,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %xmm31,%xmm5,%xmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %xmm30,%xmm5,%xmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %xmm21,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %xmm29,%xmm21,%xmm21 - vextracti32x4 $0,%zmm21,%xmm7 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_959 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_959 -.L_small_initial_partial_block_959: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 160(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 224(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 288(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - - vpxorq %zmm26,%zmm30,%zmm30 - vpxorq %zmm24,%zmm8,%zmm8 - vpxorq %zmm25,%zmm22,%zmm22 - - vpxorq %zmm31,%zmm30,%zmm30 - vpsrldq $8,%zmm30,%zmm4 - vpslldq $8,%zmm30,%zmm5 - vpxorq %zmm4,%zmm8,%zmm0 - vpxorq %zmm5,%zmm22,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_959: - - orq %r8,%r8 - je .L_after_reduction_959 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_959: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_14_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $242,%r15d - jae .L_16_blocks_overflow_960 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %ymm27,%ymm4,%ymm5 - jmp .L_16_blocks_ok_960 - -.L_16_blocks_overflow_960: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 -.L_16_blocks_ok_960: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $1,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%ymm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %ymm30,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %ymm31,%ymm5,%ymm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %ymm30,%ymm5,%ymm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %ymm21,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %ymm29,%ymm21,%ymm21 - vextracti32x4 $1,%zmm21,%xmm7 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_961 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_961 -.L_small_initial_partial_block_961: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 144(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 208(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 272(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 336(%rsi),%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm21,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm21,%xmm5 - vpclmulqdq $0x11,%xmm1,%xmm21,%xmm0 - vpclmulqdq $0x00,%xmm1,%xmm21,%xmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_961: - - orq %r8,%r8 - je .L_after_reduction_961 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_961: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_15_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $241,%r15d - jae .L_16_blocks_overflow_962 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_962 - -.L_16_blocks_overflow_962: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_962: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $2,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $2,%zmm21,%xmm7 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_963 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_963 -.L_small_initial_partial_block_963: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 128(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 192(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 256(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 320(%rsi),%ymm1 - vpclmulqdq $0x01,%ymm1,%ymm21,%ymm4 - vpclmulqdq $0x10,%ymm1,%ymm21,%ymm5 - vpclmulqdq $0x11,%ymm1,%ymm21,%ymm0 - vpclmulqdq $0x00,%ymm1,%ymm21,%ymm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_963: - - orq %r8,%r8 - je .L_after_reduction_963 - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_963: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_16_933: - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%rax - subq $192,%rax - kmovq (%r10,%rax,8),%k1 - cmpl $240,%r15d - jae .L_16_blocks_overflow_964 - vpaddd %zmm28,%zmm2,%zmm0 - vpaddd %zmm27,%zmm0,%zmm3 - vpaddd %zmm27,%zmm3,%zmm4 - vpaddd %zmm27,%zmm4,%zmm5 - jmp .L_16_blocks_ok_964 - -.L_16_blocks_overflow_964: - vpshufb %zmm29,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vmovdqa64 ddq_add_4444(%rip),%zmm5 - vpaddd %zmm5,%zmm0,%zmm3 - vpaddd %zmm5,%zmm3,%zmm4 - vpaddd %zmm5,%zmm4,%zmm5 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 -.L_16_blocks_ok_964: - - - - - vbroadcastf64x2 0(%rdi),%zmm30 - vpxorq 768(%rsp),%zmm14,%zmm8 - vmovdqu64 0(%rsp,%rbx,1),%zmm1 - vextracti32x4 $3,%zmm5,%xmm2 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - - - vbroadcastf64x2 16(%rdi),%zmm31 - vmovdqu64 64(%rsp,%rbx,1),%zmm18 - vmovdqa64 832(%rsp),%zmm22 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm30,%zmm3,%zmm3 - vpxorq %zmm30,%zmm4,%zmm4 - vpxorq %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm30 - - - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm14 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm7 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm10 - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm11 - vmovdqu64 128(%rsp,%rbx,1),%zmm1 - vmovdqa64 896(%rsp),%zmm8 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm18 - vmovdqa64 960(%rsp),%zmm22 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm30 - - - vpclmulqdq $0x10,%zmm1,%zmm8,%zmm20 - vpclmulqdq $0x01,%zmm1,%zmm8,%zmm21 - vpclmulqdq $0x11,%zmm1,%zmm8,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm8,%zmm19 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm31 - - - vpternlogq $0x96,%zmm17,%zmm12,%zmm14 - vpternlogq $0x96,%zmm19,%zmm13,%zmm7 - vpternlogq $0x96,%zmm21,%zmm16,%zmm11 - vpternlogq $0x96,%zmm20,%zmm15,%zmm10 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm30 - vmovdqu8 0(%rcx,%r11,1),%zmm17 - vmovdqu8 64(%rcx,%r11,1),%zmm19 - vmovdqu8 128(%rcx,%r11,1),%zmm20 - vmovdqu8 192(%rcx,%r11,1),%zmm21{%k1}{z} - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm31 - - - vpclmulqdq $0x10,%zmm18,%zmm22,%zmm15 - vpclmulqdq $0x01,%zmm18,%zmm22,%zmm16 - vpclmulqdq $0x11,%zmm18,%zmm22,%zmm12 - vpclmulqdq $0x00,%zmm18,%zmm22,%zmm13 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm30 - vpternlogq $0x96,%zmm16,%zmm11,%zmm10 - vpxorq %zmm12,%zmm14,%zmm24 - vpxorq %zmm13,%zmm7,%zmm25 - vpxorq %zmm15,%zmm10,%zmm26 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm31 - vaesenc %zmm30,%zmm0,%zmm0 - vaesenc %zmm30,%zmm3,%zmm3 - vaesenc %zmm30,%zmm4,%zmm4 - vaesenc %zmm30,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm30 - vaesenc %zmm31,%zmm0,%zmm0 - vaesenc %zmm31,%zmm3,%zmm3 - vaesenc %zmm31,%zmm4,%zmm4 - vaesenc %zmm31,%zmm5,%zmm5 - vaesenclast %zmm30,%zmm0,%zmm0 - vaesenclast %zmm30,%zmm3,%zmm3 - vaesenclast %zmm30,%zmm4,%zmm4 - vaesenclast %zmm30,%zmm5,%zmm5 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vpxorq %zmm20,%zmm4,%zmm4 - vpxorq %zmm21,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm11 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm21,%zmm21{%k1}{z} - vpshufb %zmm29,%zmm17,%zmm17 - vpshufb %zmm29,%zmm19,%zmm19 - vpshufb %zmm29,%zmm20,%zmm20 - vpshufb %zmm29,%zmm21,%zmm21 - vextracti32x4 $3,%zmm21,%xmm7 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_965: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm11,16(%rsi) - vmovdqu64 112(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm17,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm17,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm17,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm17,%zmm5 - vmovdqu64 176(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm19,%zmm8 - vpclmulqdq $0x00,%zmm1,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm19,%zmm30 - vpclmulqdq $0x10,%zmm1,%zmm19,%zmm31 - vmovdqu64 240(%rsi),%zmm1 - vpclmulqdq $0x11,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x00,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm0,%zmm17,%zmm8 - vpternlogq $0x96,%zmm3,%zmm19,%zmm22 - vpclmulqdq $0x01,%zmm1,%zmm20,%zmm17 - vpclmulqdq $0x10,%zmm1,%zmm20,%zmm19 - vpternlogq $0x96,%zmm4,%zmm17,%zmm30 - vpternlogq $0x96,%zmm5,%zmm19,%zmm31 - vmovdqu64 304(%rsi),%ymm1 - vinserti64x2 $2,336(%rsi),%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm1,%zmm21,%zmm4 - vpclmulqdq $0x10,%zmm1,%zmm21,%zmm5 - vpclmulqdq $0x11,%zmm1,%zmm21,%zmm0 - vpclmulqdq $0x00,%zmm1,%zmm21,%zmm3 - - vpxorq %zmm30,%zmm4,%zmm4 - vpternlogq $0x96,%zmm31,%zmm26,%zmm5 - vpternlogq $0x96,%zmm8,%zmm24,%zmm0 - vpternlogq $0x96,%zmm22,%zmm25,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm30 - vpslldq $8,%zmm4,%zmm31 - vpxorq %zmm30,%zmm0,%zmm0 - vpxorq %zmm31,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm30 - vpxorq %ymm30,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm30 - vpxorq %xmm30,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm31 - vpxorq %ymm31,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm31 - vpxorq %xmm31,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm1 - - - vpclmulqdq $0x01,%xmm3,%xmm1,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm1,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm1,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_965: - vpxorq %xmm7,%xmm14,%xmm14 -.L_after_reduction_965: - jmp .L_last_blocks_done_933 -.L_last_num_blocks_is_0_933: - vmovdqa64 768(%rsp),%zmm13 - vpxorq %zmm14,%zmm13,%zmm13 - vmovdqu64 0(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 832(%rsp),%zmm13 - vmovdqu64 64(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - vpxorq %zmm10,%zmm4,%zmm26 - vpxorq %zmm6,%zmm0,%zmm24 - vpxorq %zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - vmovdqa64 896(%rsp),%zmm13 - vmovdqu64 128(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm0 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm3 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm4 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm5 - vmovdqa64 960(%rsp),%zmm13 - vmovdqu64 192(%rsp,%rbx,1),%zmm12 - vpclmulqdq $0x11,%zmm12,%zmm13,%zmm6 - vpclmulqdq $0x00,%zmm12,%zmm13,%zmm7 - vpclmulqdq $0x01,%zmm12,%zmm13,%zmm10 - vpclmulqdq $0x10,%zmm12,%zmm13,%zmm11 - - vpternlogq $0x96,%zmm10,%zmm4,%zmm26 - vpternlogq $0x96,%zmm6,%zmm0,%zmm24 - vpternlogq $0x96,%zmm7,%zmm3,%zmm25 - vpternlogq $0x96,%zmm11,%zmm5,%zmm26 - - vpsrldq $8,%zmm26,%zmm0 - vpslldq $8,%zmm26,%zmm3 - vpxorq %zmm0,%zmm24,%zmm24 - vpxorq %zmm3,%zmm25,%zmm25 - vextracti64x4 $1,%zmm24,%ymm0 - vpxorq %ymm0,%ymm24,%ymm24 - vextracti32x4 $1,%ymm24,%xmm0 - vpxorq %xmm0,%xmm24,%xmm24 - vextracti64x4 $1,%zmm25,%ymm3 - vpxorq %ymm3,%ymm25,%ymm25 - vextracti32x4 $1,%ymm25,%xmm3 - vpxorq %xmm3,%xmm25,%xmm25 - vmovdqa64 POLY2(%rip),%xmm4 - - - vpclmulqdq $0x01,%xmm25,%xmm4,%xmm0 - vpslldq $8,%xmm0,%xmm0 - vpxorq %xmm0,%xmm25,%xmm0 - - - vpclmulqdq $0x00,%xmm0,%xmm4,%xmm3 - vpsrldq $4,%xmm3,%xmm3 - vpclmulqdq $0x10,%xmm0,%xmm4,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm24,%xmm3,%xmm14 - -.L_last_blocks_done_933: - vpshufb %xmm29,%xmm2,%xmm2 - jmp .L_ghash_done_821 - -.L_message_below_equal_16_blocks_821: - - - movl %r8d,%r12d - addl $15,%r12d - shrl $4,%r12d - cmpq $8,%r12 - je .L_small_initial_num_blocks_is_8_966 - jl .L_small_initial_num_blocks_is_7_1_966 - - - cmpq $12,%r12 - je .L_small_initial_num_blocks_is_12_966 - jl .L_small_initial_num_blocks_is_11_9_966 - - - cmpq $16,%r12 - je .L_small_initial_num_blocks_is_16_966 - cmpq $15,%r12 - je .L_small_initial_num_blocks_is_15_966 - cmpq $14,%r12 - je .L_small_initial_num_blocks_is_14_966 - jmp .L_small_initial_num_blocks_is_13_966 - -.L_small_initial_num_blocks_is_11_9_966: - - cmpq $11,%r12 - je .L_small_initial_num_blocks_is_11_966 - cmpq $10,%r12 - je .L_small_initial_num_blocks_is_10_966 - jmp .L_small_initial_num_blocks_is_9_966 - -.L_small_initial_num_blocks_is_7_1_966: - cmpq $4,%r12 - je .L_small_initial_num_blocks_is_4_966 - jl .L_small_initial_num_blocks_is_3_1_966 - - cmpq $7,%r12 - je .L_small_initial_num_blocks_is_7_966 - cmpq $6,%r12 - je .L_small_initial_num_blocks_is_6_966 - jmp .L_small_initial_num_blocks_is_5_966 - -.L_small_initial_num_blocks_is_3_1_966: - - cmpq $3,%r12 - je .L_small_initial_num_blocks_is_3_966 - cmpq $2,%r12 - je .L_small_initial_num_blocks_is_2_966 - - - - - -.L_small_initial_num_blocks_is_1_966: - vmovdqa64 SHUF_MASK(%rip),%xmm29 - vpaddd ONE(%rip),%xmm2,%xmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm0,%xmm2 - vpshufb %xmm29,%xmm0,%xmm0 - vmovdqu8 0(%rcx,%r11,1),%xmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %xmm15,%xmm0,%xmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %xmm15,%xmm0,%xmm0 - vpxorq %xmm6,%xmm0,%xmm0 - vextracti32x4 $0,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %xmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %xmm29,%xmm6,%xmm6 - vextracti32x4 $0,%zmm6,%xmm13 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_967 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_967 -.L_small_initial_partial_block_967: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - - - - - - - - - - - - vpxorq %xmm13,%xmm14,%xmm14 - - jmp .L_after_reduction_967 -.L_small_initial_compute_done_967: -.L_after_reduction_967: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_2_966: - vmovdqa64 SHUF_MASK(%rip),%ymm29 - vshufi64x2 $0,%ymm2,%ymm2,%ymm0 - vpaddd ddq_add_1234(%rip),%ymm0,%ymm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm0,%xmm2 - vpshufb %ymm29,%ymm0,%ymm0 - vmovdqu8 0(%rcx,%r11,1),%ymm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %ymm15,%ymm0,%ymm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %ymm15,%ymm0,%ymm0 - vpxorq %ymm6,%ymm0,%ymm0 - vextracti32x4 $1,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %ymm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %ymm29,%ymm6,%ymm6 - vextracti32x4 $1,%zmm6,%xmm13 - subq $16 * (2 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_968 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_968 -.L_small_initial_partial_block_968: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm6,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm6,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm6,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm6,%xmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_968: - - orq %r8,%r8 - je .L_after_reduction_968 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_968: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_3_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $2,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $2,%zmm6,%xmm13 - subq $16 * (3 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_969 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_969 -.L_small_initial_partial_block_969: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm6,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm6,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm6,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm6,%ymm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_969: - - orq %r8,%r8 - je .L_after_reduction_969 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_969: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_4_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm0,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vmovdqu8 0(%rcx,%r11,1),%zmm6{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vpxorq %zmm6,%zmm0,%zmm0 - vextracti32x4 $3,%zmm0,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1){%k1} - vmovdqu8 %zmm0,%zmm0{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vextracti32x4 $3,%zmm6,%xmm13 - subq $16 * (4 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_970 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_970 -.L_small_initial_partial_block_970: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_970: - - orq %r8,%r8 - je .L_after_reduction_970 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_970: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_5_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %xmm29,%xmm3,%xmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%xmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %xmm15,%xmm3,%xmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %xmm15,%xmm3,%xmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %xmm7,%xmm3,%xmm3 - vextracti32x4 $0,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %xmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %xmm29,%xmm7,%xmm7 - vextracti32x4 $0,%zmm7,%xmm13 - subq $16 * (5 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_971 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_971 -.L_small_initial_partial_block_971: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_971: - - orq %r8,%r8 - je .L_after_reduction_971 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_971: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_6_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %ymm29,%ymm3,%ymm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%ymm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %ymm15,%ymm3,%ymm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %ymm15,%ymm3,%ymm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %ymm7,%ymm3,%ymm3 - vextracti32x4 $1,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %ymm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %ymm29,%ymm7,%ymm7 - vextracti32x4 $1,%zmm7,%xmm13 - subq $16 * (6 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_972 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_972 -.L_small_initial_partial_block_972: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm7,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm7,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm7,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm7,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_972: - - orq %r8,%r8 - je .L_after_reduction_972 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_972: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_7_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $2,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $2,%zmm7,%xmm13 - subq $16 * (7 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_973 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_973 -.L_small_initial_partial_block_973: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm7,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm7,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm7,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm7,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_973: - - orq %r8,%r8 - je .L_after_reduction_973 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_973: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_8_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $64,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm3,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vextracti32x4 $3,%zmm3,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1){%k1} - vmovdqu8 %zmm3,%zmm3{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vextracti32x4 $3,%zmm7,%xmm13 - subq $16 * (8 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_974 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_974 -.L_small_initial_partial_block_974: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_974: - - orq %r8,%r8 - je .L_after_reduction_974 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_974: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_9_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %xmm29,%xmm4,%xmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%xmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %xmm15,%xmm4,%xmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %xmm15,%xmm4,%xmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %xmm10,%xmm4,%xmm4 - vextracti32x4 $0,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %xmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %xmm29,%xmm10,%xmm10 - vextracti32x4 $0,%zmm10,%xmm13 - subq $16 * (9 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_975 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_975 -.L_small_initial_partial_block_975: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_975: - - orq %r8,%r8 - je .L_after_reduction_975 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_975: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_10_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %ymm29,%ymm4,%ymm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%ymm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %ymm15,%ymm4,%ymm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %ymm15,%ymm4,%ymm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %ymm10,%ymm4,%ymm4 - vextracti32x4 $1,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %ymm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %ymm29,%ymm10,%ymm10 - vextracti32x4 $1,%zmm10,%xmm13 - subq $16 * (10 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_976 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_976 -.L_small_initial_partial_block_976: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm10,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm10,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm10,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm10,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_976: - - orq %r8,%r8 - je .L_after_reduction_976 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_976: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_11_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $2,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $2,%zmm10,%xmm13 - subq $16 * (11 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_977 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_977 -.L_small_initial_partial_block_977: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm10,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm10,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm10,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm10,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_977: - - orq %r8,%r8 - je .L_after_reduction_977 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_977: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_12_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $128,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm4,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vextracti32x4 $3,%zmm4,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1){%k1} - vmovdqu8 %zmm4,%zmm4{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vextracti32x4 $3,%zmm10,%xmm13 - subq $16 * (12 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_978 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_978 -.L_small_initial_partial_block_978: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vpxorq %zmm15,%zmm0,%zmm15 - vpxorq %zmm16,%zmm3,%zmm16 - vpxorq %zmm17,%zmm4,%zmm17 - vpxorq %zmm19,%zmm5,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_978: - - orq %r8,%r8 - je .L_after_reduction_978 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_978: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_13_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $0,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %xmm29,%xmm5,%xmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%xmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %xmm15,%xmm5,%xmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %xmm15,%xmm5,%xmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %xmm11,%xmm5,%xmm5 - vextracti32x4 $0,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %xmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %xmm29,%xmm11,%xmm11 - vextracti32x4 $0,%zmm11,%xmm13 - subq $16 * (13 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_979 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_979 -.L_small_initial_partial_block_979: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 160(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 224(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 288(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - - vpxorq %zmm19,%zmm17,%zmm17 - vpsrldq $8,%zmm17,%zmm4 - vpslldq $8,%zmm17,%zmm5 - vpxorq %zmm4,%zmm15,%zmm0 - vpxorq %zmm5,%zmm16,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_979: - - orq %r8,%r8 - je .L_after_reduction_979 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_979: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_14_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $1,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %ymm29,%ymm5,%ymm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%ymm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %ymm15,%ymm5,%ymm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %ymm15,%ymm5,%ymm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %ymm11,%ymm5,%ymm5 - vextracti32x4 $1,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %ymm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %ymm29,%ymm11,%ymm11 - vextracti32x4 $1,%zmm11,%xmm13 - subq $16 * (14 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_980 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_980 -.L_small_initial_partial_block_980: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 144(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 208(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 272(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 336(%rsi),%xmm20 - vpclmulqdq $0x01,%xmm20,%xmm11,%xmm4 - vpclmulqdq $0x10,%xmm20,%xmm11,%xmm5 - vpclmulqdq $0x11,%xmm20,%xmm11,%xmm0 - vpclmulqdq $0x00,%xmm20,%xmm11,%xmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_980: - - orq %r8,%r8 - je .L_after_reduction_980 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_980: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_15_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $2,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $2,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $2,%zmm11,%xmm13 - subq $16 * (15 - 1),%r8 - - - cmpq $16,%r8 - jl .L_small_initial_partial_block_981 - - - - - - subq $16,%r8 - movq $0,(%rdx) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - - jmp .L_small_initial_compute_done_981 -.L_small_initial_partial_block_981: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 128(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 192(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 256(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 320(%rsi),%ymm20 - vpclmulqdq $0x01,%ymm20,%ymm11,%ymm4 - vpclmulqdq $0x10,%ymm20,%ymm11,%ymm5 - vpclmulqdq $0x11,%ymm20,%ymm11,%ymm0 - vpclmulqdq $0x00,%ymm20,%ymm11,%ymm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_981: - - orq %r8,%r8 - je .L_after_reduction_981 - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_981: - jmp .L_small_initial_blocks_encrypted_966 -.L_small_initial_num_blocks_is_16_966: - vmovdqa64 SHUF_MASK(%rip),%zmm29 - vshufi64x2 $0,%zmm2,%zmm2,%zmm2 - vpaddd ddq_add_1234(%rip),%zmm2,%zmm0 - vpaddd ddq_add_5678(%rip),%zmm2,%zmm3 - vpaddd ddq_add_8888(%rip),%zmm0,%zmm4 - vpaddd ddq_add_8888(%rip),%zmm3,%zmm5 - leaq byte64_len_to_mask_table(%rip),%r10 - movq %r8,%r15 - subq $192,%r15 - kmovq (%r10,%r15,8),%k1 - vextracti32x4 $3,%zmm5,%xmm2 - vpshufb %zmm29,%zmm0,%zmm0 - vpshufb %zmm29,%zmm3,%zmm3 - vpshufb %zmm29,%zmm4,%zmm4 - vpshufb %zmm29,%zmm5,%zmm5 - vmovdqu8 0(%rcx,%r11,1),%zmm6 - vmovdqu8 64(%rcx,%r11,1),%zmm7 - vmovdqu8 128(%rcx,%r11,1),%zmm10 - vmovdqu8 192(%rcx,%r11,1),%zmm11{%k1}{z} - vbroadcastf64x2 0(%rdi),%zmm15 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm15,%zmm3,%zmm3 - vpxorq %zmm15,%zmm4,%zmm4 - vpxorq %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 16(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 32(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 48(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 64(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 80(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 96(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 112(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 128(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 144(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 160(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 176(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 192(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 208(%rdi),%zmm15 - vaesenc %zmm15,%zmm0,%zmm0 - vaesenc %zmm15,%zmm3,%zmm3 - vaesenc %zmm15,%zmm4,%zmm4 - vaesenc %zmm15,%zmm5,%zmm5 - vbroadcastf64x2 224(%rdi),%zmm15 - vaesenclast %zmm15,%zmm0,%zmm0 - vaesenclast %zmm15,%zmm3,%zmm3 - vaesenclast %zmm15,%zmm4,%zmm4 - vaesenclast %zmm15,%zmm5,%zmm5 - vpxorq %zmm6,%zmm0,%zmm0 - vpxorq %zmm7,%zmm3,%zmm3 - vpxorq %zmm10,%zmm4,%zmm4 - vpxorq %zmm11,%zmm5,%zmm5 - vextracti32x4 $3,%zmm5,%xmm12 - movq %r9,%r10 - vmovdqu8 %zmm0,0(%r10,%r11,1) - vmovdqu8 %zmm3,64(%r10,%r11,1) - vmovdqu8 %zmm4,128(%r10,%r11,1) - vmovdqu8 %zmm5,192(%r10,%r11,1){%k1} - vmovdqu8 %zmm5,%zmm5{%k1}{z} - vpshufb %zmm29,%zmm6,%zmm6 - vpshufb %zmm29,%zmm7,%zmm7 - vpshufb %zmm29,%zmm10,%zmm10 - vpshufb %zmm29,%zmm11,%zmm11 - vextracti32x4 $3,%zmm11,%xmm13 - subq $16 * (16 - 1),%r8 -.L_small_initial_partial_block_982: - - - - - - - - - movq %r8,(%rdx) - vmovdqu64 %xmm12,16(%rsi) - vpxorq %zmm14,%zmm6,%zmm6 - vmovdqu64 112(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm6,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm6,%zmm3 - vpclmulqdq $0x01,%zmm20,%zmm6,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm6,%zmm5 - vmovdqu64 176(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm7,%zmm15 - vpclmulqdq $0x00,%zmm20,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm7,%zmm17 - vpclmulqdq $0x10,%zmm20,%zmm7,%zmm19 - vmovdqu64 240(%rsi),%zmm20 - vpclmulqdq $0x11,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x00,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm0,%zmm6,%zmm15 - vpternlogq $0x96,%zmm3,%zmm7,%zmm16 - vpclmulqdq $0x01,%zmm20,%zmm10,%zmm6 - vpclmulqdq $0x10,%zmm20,%zmm10,%zmm7 - vpternlogq $0x96,%zmm4,%zmm6,%zmm17 - vpternlogq $0x96,%zmm5,%zmm7,%zmm19 - vmovdqu64 304(%rsi),%ymm20 - vinserti64x2 $2,336(%rsi),%zmm20,%zmm20 - vpclmulqdq $0x01,%zmm20,%zmm11,%zmm4 - vpclmulqdq $0x10,%zmm20,%zmm11,%zmm5 - vpclmulqdq $0x11,%zmm20,%zmm11,%zmm0 - vpclmulqdq $0x00,%zmm20,%zmm11,%zmm3 - - vpxorq %zmm17,%zmm4,%zmm4 - vpxorq %zmm19,%zmm5,%zmm5 - vpxorq %zmm15,%zmm0,%zmm0 - vpxorq %zmm16,%zmm3,%zmm3 - - vpxorq %zmm5,%zmm4,%zmm4 - vpsrldq $8,%zmm4,%zmm17 - vpslldq $8,%zmm4,%zmm19 - vpxorq %zmm17,%zmm0,%zmm0 - vpxorq %zmm19,%zmm3,%zmm3 - vextracti64x4 $1,%zmm0,%ymm17 - vpxorq %ymm17,%ymm0,%ymm0 - vextracti32x4 $1,%ymm0,%xmm17 - vpxorq %xmm17,%xmm0,%xmm0 - vextracti64x4 $1,%zmm3,%ymm19 - vpxorq %ymm19,%ymm3,%ymm3 - vextracti32x4 $1,%ymm3,%xmm19 - vpxorq %xmm19,%xmm3,%xmm3 - vmovdqa64 POLY2(%rip),%xmm20 - - - vpclmulqdq $0x01,%xmm3,%xmm20,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm3,%xmm4 - - - vpclmulqdq $0x00,%xmm4,%xmm20,%xmm5 - vpsrldq $4,%xmm5,%xmm5 - vpclmulqdq $0x10,%xmm4,%xmm20,%xmm14 - vpslldq $4,%xmm14,%xmm14 - vpternlogq $0x96,%xmm0,%xmm5,%xmm14 - -.L_small_initial_compute_done_982: - vpxorq %xmm13,%xmm14,%xmm14 -.L_after_reduction_982: -.L_small_initial_blocks_encrypted_966: -.L_ghash_done_821: - vmovdqu64 %xmm2,0(%rsi) - vmovdqu64 %xmm14,64(%rsi) -.L_enc_dec_done_821: - jmp .Lexit_gcm_decrypt -.Lexit_gcm_decrypt: - cmpq $256,%r8 - jbe .Lskip_hkeys_cleanup_983 - vpxor %xmm0,%xmm0,%xmm0 - vmovdqa64 %zmm0,0(%rsp) - vmovdqa64 %zmm0,64(%rsp) - vmovdqa64 %zmm0,128(%rsp) - vmovdqa64 %zmm0,192(%rsp) - vmovdqa64 %zmm0,256(%rsp) - vmovdqa64 %zmm0,320(%rsp) - vmovdqa64 %zmm0,384(%rsp) - vmovdqa64 %zmm0,448(%rsp) - vmovdqa64 %zmm0,512(%rsp) - vmovdqa64 %zmm0,576(%rsp) - vmovdqa64 %zmm0,640(%rsp) - vmovdqa64 %zmm0,704(%rsp) -.Lskip_hkeys_cleanup_983: - vzeroupper - leaq (%rbp),%rsp -.cfi_def_cfa_register %rsp - popq %r15 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r15 - popq %r14 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r14 - popq %r13 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r13 - popq %r12 -.cfi_adjust_cfa_offset -8 -.cfi_restore %r12 - popq %rbp -.cfi_adjust_cfa_offset -8 -.cfi_restore %rbp - popq %rbx -.cfi_adjust_cfa_offset -8 -.cfi_restore %rbx - .byte 0xf3,0xc3 -.Ldecrypt_seh_end: -.cfi_endproc -.size ossl_aes_gcm_decrypt_avx512, .-ossl_aes_gcm_decrypt_avx512 -.globl ossl_aes_gcm_finalize_avx512 -.type ossl_aes_gcm_finalize_avx512,@function -.align 32 -ossl_aes_gcm_finalize_avx512: -.cfi_startproc -.byte 243,15,30,250 - vmovdqu 336(%rdi),%xmm2 - vmovdqu 32(%rdi),%xmm3 - vmovdqu 64(%rdi),%xmm4 - - - cmpq $0,%rsi - je .L_partial_done_984 - - vpclmulqdq $0x11,%xmm2,%xmm4,%xmm0 - vpclmulqdq $0x00,%xmm2,%xmm4,%xmm16 - vpclmulqdq $0x01,%xmm2,%xmm4,%xmm17 - vpclmulqdq $0x10,%xmm2,%xmm4,%xmm4 - vpxorq %xmm17,%xmm4,%xmm4 - - vpsrldq $8,%xmm4,%xmm17 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm17,%xmm0,%xmm0 - vpxorq %xmm16,%xmm4,%xmm4 - - - - vmovdqu64 POLY2(%rip),%xmm17 - - vpclmulqdq $0x01,%xmm4,%xmm17,%xmm16 - vpslldq $8,%xmm16,%xmm16 - vpxorq %xmm16,%xmm4,%xmm4 - - - - vpclmulqdq $0x00,%xmm4,%xmm17,%xmm16 - vpsrldq $4,%xmm16,%xmm16 - vpclmulqdq $0x10,%xmm4,%xmm17,%xmm4 - vpslldq $4,%xmm4,%xmm4 - - vpternlogq $0x96,%xmm16,%xmm0,%xmm4 - -.L_partial_done_984: - vmovq 56(%rdi),%xmm5 - vpinsrq $1,48(%rdi),%xmm5,%xmm5 - vpsllq $3,%xmm5,%xmm5 - - vpxor %xmm5,%xmm4,%xmm4 - - vpclmulqdq $0x11,%xmm2,%xmm4,%xmm0 - vpclmulqdq $0x00,%xmm2,%xmm4,%xmm16 - vpclmulqdq $0x01,%xmm2,%xmm4,%xmm17 - vpclmulqdq $0x10,%xmm2,%xmm4,%xmm4 - vpxorq %xmm17,%xmm4,%xmm4 - - vpsrldq $8,%xmm4,%xmm17 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm17,%xmm0,%xmm0 - vpxorq %xmm16,%xmm4,%xmm4 - - - - vmovdqu64 POLY2(%rip),%xmm17 - - vpclmulqdq $0x01,%xmm4,%xmm17,%xmm16 - vpslldq $8,%xmm16,%xmm16 - vpxorq %xmm16,%xmm4,%xmm4 - - - - vpclmulqdq $0x00,%xmm4,%xmm17,%xmm16 - vpsrldq $4,%xmm16,%xmm16 - vpclmulqdq $0x10,%xmm4,%xmm17,%xmm4 - vpslldq $4,%xmm4,%xmm4 - - vpternlogq $0x96,%xmm16,%xmm0,%xmm4 - - vpshufb SHUF_MASK(%rip),%xmm4,%xmm4 - vpxor %xmm4,%xmm3,%xmm3 - -.L_return_T_984: - vmovdqu %xmm3,64(%rdi) -.Labort_finalize: - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_aes_gcm_finalize_avx512, .-ossl_aes_gcm_finalize_avx512 -.globl ossl_gcm_gmult_avx512 -.hidden ossl_gcm_gmult_avx512 -.type ossl_gcm_gmult_avx512,@function -.align 32 -ossl_gcm_gmult_avx512: -.cfi_startproc -.byte 243,15,30,250 - vmovdqu64 (%rdi),%xmm1 - vmovdqu64 336(%rsi),%xmm2 - - vpclmulqdq $0x11,%xmm2,%xmm1,%xmm3 - vpclmulqdq $0x00,%xmm2,%xmm1,%xmm4 - vpclmulqdq $0x01,%xmm2,%xmm1,%xmm5 - vpclmulqdq $0x10,%xmm2,%xmm1,%xmm1 - vpxorq %xmm5,%xmm1,%xmm1 - - vpsrldq $8,%xmm1,%xmm5 - vpslldq $8,%xmm1,%xmm1 - vpxorq %xmm5,%xmm3,%xmm3 - vpxorq %xmm4,%xmm1,%xmm1 - - - - vmovdqu64 POLY2(%rip),%xmm5 - - vpclmulqdq $0x01,%xmm1,%xmm5,%xmm4 - vpslldq $8,%xmm4,%xmm4 - vpxorq %xmm4,%xmm1,%xmm1 - - - - vpclmulqdq $0x00,%xmm1,%xmm5,%xmm4 - vpsrldq $4,%xmm4,%xmm4 - vpclmulqdq $0x10,%xmm1,%xmm5,%xmm1 - vpslldq $4,%xmm1,%xmm1 - - vpternlogq $0x96,%xmm4,%xmm3,%xmm1 - - vmovdqu64 %xmm1,(%rdi) - vzeroupper -.Labort_gmult: - .byte 0xf3,0xc3 -.cfi_endproc -.size ossl_gcm_gmult_avx512, .-ossl_gcm_gmult_avx512 -.data -.align 16 -POLY:.quad 0x0000000000000001, 0xC200000000000000 - -.align 64 -POLY2: -.quad 0x00000001C2000000, 0xC200000000000000 -.quad 0x00000001C2000000, 0xC200000000000000 -.quad 0x00000001C2000000, 0xC200000000000000 -.quad 0x00000001C2000000, 0xC200000000000000 - -.align 16 -TWOONE:.quad 0x0000000000000001, 0x0000000100000000 - - - -.align 64 -SHUF_MASK: -.quad 0x08090A0B0C0D0E0F, 0x0001020304050607 -.quad 0x08090A0B0C0D0E0F, 0x0001020304050607 -.quad 0x08090A0B0C0D0E0F, 0x0001020304050607 -.quad 0x08090A0B0C0D0E0F, 0x0001020304050607 - -.align 16 -SHIFT_MASK: -.quad 0x0706050403020100, 0x0f0e0d0c0b0a0908 - -ALL_F: -.quad 0xffffffffffffffff, 0xffffffffffffffff - -ZERO: -.quad 0x0000000000000000, 0x0000000000000000 - -.align 16 -ONE: -.quad 0x0000000000000001, 0x0000000000000000 - -.align 16 -ONEf: -.quad 0x0000000000000000, 0x0100000000000000 - -.align 64 -ddq_add_1234: -.quad 0x0000000000000001, 0x0000000000000000 -.quad 0x0000000000000002, 0x0000000000000000 -.quad 0x0000000000000003, 0x0000000000000000 -.quad 0x0000000000000004, 0x0000000000000000 - -.align 64 -ddq_add_5678: -.quad 0x0000000000000005, 0x0000000000000000 -.quad 0x0000000000000006, 0x0000000000000000 -.quad 0x0000000000000007, 0x0000000000000000 -.quad 0x0000000000000008, 0x0000000000000000 - -.align 64 -ddq_add_4444: -.quad 0x0000000000000004, 0x0000000000000000 -.quad 0x0000000000000004, 0x0000000000000000 -.quad 0x0000000000000004, 0x0000000000000000 -.quad 0x0000000000000004, 0x0000000000000000 - -.align 64 -ddq_add_8888: -.quad 0x0000000000000008, 0x0000000000000000 -.quad 0x0000000000000008, 0x0000000000000000 -.quad 0x0000000000000008, 0x0000000000000000 -.quad 0x0000000000000008, 0x0000000000000000 - -.align 64 -ddq_addbe_1234: -.quad 0x0000000000000000, 0x0100000000000000 -.quad 0x0000000000000000, 0x0200000000000000 -.quad 0x0000000000000000, 0x0300000000000000 -.quad 0x0000000000000000, 0x0400000000000000 - -.align 64 -ddq_addbe_4444: -.quad 0x0000000000000000, 0x0400000000000000 -.quad 0x0000000000000000, 0x0400000000000000 -.quad 0x0000000000000000, 0x0400000000000000 -.quad 0x0000000000000000, 0x0400000000000000 - -.align 64 -byte_len_to_mask_table: -.value 0x0000, 0x0001, 0x0003, 0x0007 -.value 0x000f, 0x001f, 0x003f, 0x007f -.value 0x00ff, 0x01ff, 0x03ff, 0x07ff -.value 0x0fff, 0x1fff, 0x3fff, 0x7fff -.value 0xffff - -.align 64 -byte64_len_to_mask_table: -.quad 0x0000000000000000, 0x0000000000000001 -.quad 0x0000000000000003, 0x0000000000000007 -.quad 0x000000000000000f, 0x000000000000001f -.quad 0x000000000000003f, 0x000000000000007f -.quad 0x00000000000000ff, 0x00000000000001ff -.quad 0x00000000000003ff, 0x00000000000007ff -.quad 0x0000000000000fff, 0x0000000000001fff -.quad 0x0000000000003fff, 0x0000000000007fff -.quad 0x000000000000ffff, 0x000000000001ffff -.quad 0x000000000003ffff, 0x000000000007ffff -.quad 0x00000000000fffff, 0x00000000001fffff -.quad 0x00000000003fffff, 0x00000000007fffff -.quad 0x0000000000ffffff, 0x0000000001ffffff -.quad 0x0000000003ffffff, 0x0000000007ffffff -.quad 0x000000000fffffff, 0x000000001fffffff -.quad 0x000000003fffffff, 0x000000007fffffff -.quad 0x00000000ffffffff, 0x00000001ffffffff -.quad 0x00000003ffffffff, 0x00000007ffffffff -.quad 0x0000000fffffffff, 0x0000001fffffffff -.quad 0x0000003fffffffff, 0x0000007fffffffff -.quad 0x000000ffffffffff, 0x000001ffffffffff -.quad 0x000003ffffffffff, 0x000007ffffffffff -.quad 0x00000fffffffffff, 0x00001fffffffffff -.quad 0x00003fffffffffff, 0x00007fffffffffff -.quad 0x0000ffffffffffff, 0x0001ffffffffffff -.quad 0x0003ffffffffffff, 0x0007ffffffffffff -.quad 0x000fffffffffffff, 0x001fffffffffffff -.quad 0x003fffffffffffff, 0x007fffffffffffff -.quad 0x00ffffffffffffff, 0x01ffffffffffffff -.quad 0x03ffffffffffffff, 0x07ffffffffffffff -.quad 0x0fffffffffffffff, 0x1fffffffffffffff -.quad 0x3fffffffffffffff, 0x7fffffffffffffff -.quad 0xffffffffffffffff - .section ".note.gnu.property", "a" - .p2align 3 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - # "GNU" encoded with .byte, since .asciz isn't supported - # on Solaris. - .byte 0x47 - .byte 0x4e - .byte 0x55 - .byte 0 -1: - .p2align 3 - .long 0xc0000002 - .long 3f - 2f -2: - .long 3 -3: - .p2align 3 -4: diff --git a/openssl/src/crypto/modes/gen/windows_ia32/ghash-x86.asm b/openssl/src/crypto/modes/gen/windows_ia32/ghash-x86.asm index 49bfb41c4..a526bbbc8 100644 --- a/openssl/src/crypto/modes/gen/windows_ia32/ghash-x86.asm +++ b/openssl/src/crypto/modes/gen/windows_ia32/ghash-x86.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/modes/gen/windows_x64/aes-gcm-avx512.asm b/openssl/src/crypto/modes/gen/windows_x64/aes-gcm-avx512.asm deleted file mode 100644 index 95606bc30..000000000 --- a/openssl/src/crypto/modes/gen/windows_x64/aes-gcm-avx512.asm +++ /dev/null @@ -1,136521 +0,0 @@ -default rel -%define XMMWORD -%define YMMWORD -%define ZMMWORD -EXTERN OPENSSL_ia32cap_P -global ossl_vaes_vpclmulqdq_capable - -ALIGN 32 -ossl_vaes_vpclmulqdq_capable: - mov rcx,QWORD[((OPENSSL_ia32cap_P+8))] - - mov rdx,6600291188736 - xor eax,eax - and rcx,rdx - cmp rcx,rdx - cmove rax,rcx - DB 0F3h,0C3h ;repret - -section .text code align=64 - -global ossl_aes_gcm_init_avx512 - -ALIGN 32 -ossl_aes_gcm_init_avx512: - -DB 243,15,30,250 - vpxorq xmm16,xmm16,xmm16 - - - mov eax,DWORD[240+rcx] - cmp eax,9 - je NEAR $L$aes_128_0 - cmp eax,11 - je NEAR $L$aes_192_0 - cmp eax,13 - je NEAR $L$aes_256_0 - jmp NEAR $L$exit_aes_0 -ALIGN 32 -$L$aes_128_0: - vpxorq xmm16,xmm16,XMMWORD[rcx] - - vaesenc xmm16,xmm16,XMMWORD[16+rcx] - - vaesenc xmm16,xmm16,XMMWORD[32+rcx] - - vaesenc xmm16,xmm16,XMMWORD[48+rcx] - - vaesenc xmm16,xmm16,XMMWORD[64+rcx] - - vaesenc xmm16,xmm16,XMMWORD[80+rcx] - - vaesenc xmm16,xmm16,XMMWORD[96+rcx] - - vaesenc xmm16,xmm16,XMMWORD[112+rcx] - - vaesenc xmm16,xmm16,XMMWORD[128+rcx] - - vaesenc xmm16,xmm16,XMMWORD[144+rcx] - - vaesenclast xmm16,xmm16,XMMWORD[160+rcx] - jmp NEAR $L$exit_aes_0 -ALIGN 32 -$L$aes_192_0: - vpxorq xmm16,xmm16,XMMWORD[rcx] - - vaesenc xmm16,xmm16,XMMWORD[16+rcx] - - vaesenc xmm16,xmm16,XMMWORD[32+rcx] - - vaesenc xmm16,xmm16,XMMWORD[48+rcx] - - vaesenc xmm16,xmm16,XMMWORD[64+rcx] - - vaesenc xmm16,xmm16,XMMWORD[80+rcx] - - vaesenc xmm16,xmm16,XMMWORD[96+rcx] - - vaesenc xmm16,xmm16,XMMWORD[112+rcx] - - vaesenc xmm16,xmm16,XMMWORD[128+rcx] - - vaesenc xmm16,xmm16,XMMWORD[144+rcx] - - vaesenc xmm16,xmm16,XMMWORD[160+rcx] - - vaesenc xmm16,xmm16,XMMWORD[176+rcx] - - vaesenclast xmm16,xmm16,XMMWORD[192+rcx] - jmp NEAR $L$exit_aes_0 -ALIGN 32 -$L$aes_256_0: - vpxorq xmm16,xmm16,XMMWORD[rcx] - - vaesenc xmm16,xmm16,XMMWORD[16+rcx] - - vaesenc xmm16,xmm16,XMMWORD[32+rcx] - - vaesenc xmm16,xmm16,XMMWORD[48+rcx] - - vaesenc xmm16,xmm16,XMMWORD[64+rcx] - - vaesenc xmm16,xmm16,XMMWORD[80+rcx] - - vaesenc xmm16,xmm16,XMMWORD[96+rcx] - - vaesenc xmm16,xmm16,XMMWORD[112+rcx] - - vaesenc xmm16,xmm16,XMMWORD[128+rcx] - - vaesenc xmm16,xmm16,XMMWORD[144+rcx] - - vaesenc xmm16,xmm16,XMMWORD[160+rcx] - - vaesenc xmm16,xmm16,XMMWORD[176+rcx] - - vaesenc xmm16,xmm16,XMMWORD[192+rcx] - - vaesenc xmm16,xmm16,XMMWORD[208+rcx] - - vaesenclast xmm16,xmm16,XMMWORD[224+rcx] - jmp NEAR $L$exit_aes_0 -$L$exit_aes_0: - - vpshufb xmm16,xmm16,XMMWORD[SHUF_MASK] - - vmovdqa64 xmm2,xmm16 - vpsllq xmm16,xmm16,1 - vpsrlq xmm2,xmm2,63 - vmovdqa xmm1,xmm2 - vpslldq xmm2,xmm2,8 - vpsrldq xmm1,xmm1,8 - vporq xmm16,xmm16,xmm2 - - vpshufd xmm2,xmm1,36 - vpcmpeqd xmm2,xmm2,XMMWORD[TWOONE] - vpand xmm2,xmm2,XMMWORD[POLY] - vpxorq xmm16,xmm16,xmm2 - - vmovdqu64 XMMWORD[336+rdx],xmm16 - vshufi32x4 ymm4,ymm16,ymm16,0x00 - vmovdqa ymm3,ymm4 - - vpclmulqdq ymm0,ymm3,ymm4,0x11 - vpclmulqdq ymm1,ymm3,ymm4,0x00 - vpclmulqdq ymm2,ymm3,ymm4,0x01 - vpclmulqdq ymm3,ymm3,ymm4,0x10 - vpxorq ymm3,ymm3,ymm2 - - vpsrldq ymm2,ymm3,8 - vpslldq ymm3,ymm3,8 - vpxorq ymm0,ymm0,ymm2 - vpxorq ymm3,ymm3,ymm1 - - - - vmovdqu64 ymm2,YMMWORD[POLY2] - - vpclmulqdq ymm1,ymm2,ymm3,0x01 - vpslldq ymm1,ymm1,8 - vpxorq ymm3,ymm3,ymm1 - - - - vpclmulqdq ymm1,ymm2,ymm3,0x00 - vpsrldq ymm1,ymm1,4 - vpclmulqdq ymm3,ymm2,ymm3,0x10 - vpslldq ymm3,ymm3,4 - - vpternlogq ymm3,ymm0,ymm1,0x96 - - vmovdqu64 XMMWORD[320+rdx],xmm3 - vinserti64x2 ymm4,ymm3,xmm16,1 - vmovdqa64 ymm5,ymm4 - - vpclmulqdq ymm0,ymm4,ymm3,0x11 - vpclmulqdq ymm1,ymm4,ymm3,0x00 - vpclmulqdq ymm2,ymm4,ymm3,0x01 - vpclmulqdq ymm4,ymm4,ymm3,0x10 - vpxorq ymm4,ymm4,ymm2 - - vpsrldq ymm2,ymm4,8 - vpslldq ymm4,ymm4,8 - vpxorq ymm0,ymm0,ymm2 - vpxorq ymm4,ymm4,ymm1 - - - - vmovdqu64 ymm2,YMMWORD[POLY2] - - vpclmulqdq ymm1,ymm2,ymm4,0x01 - vpslldq ymm1,ymm1,8 - vpxorq ymm4,ymm4,ymm1 - - - - vpclmulqdq ymm1,ymm2,ymm4,0x00 - vpsrldq ymm1,ymm1,4 - vpclmulqdq ymm4,ymm2,ymm4,0x10 - vpslldq ymm4,ymm4,4 - - vpternlogq ymm4,ymm0,ymm1,0x96 - - vmovdqu64 YMMWORD[288+rdx],ymm4 - - vinserti64x4 zmm4,zmm4,ymm5,1 - - - vshufi64x2 zmm3,zmm4,zmm4,0x00 - vmovdqa64 zmm5,zmm4 - - vpclmulqdq zmm0,zmm4,zmm3,0x11 - vpclmulqdq zmm1,zmm4,zmm3,0x00 - vpclmulqdq zmm2,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm2 - - vpsrldq zmm2,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm0,zmm0,zmm2 - vpxorq zmm4,zmm4,zmm1 - - - - vmovdqu64 zmm2,ZMMWORD[POLY2] - - vpclmulqdq zmm1,zmm2,zmm4,0x01 - vpslldq zmm1,zmm1,8 - vpxorq zmm4,zmm4,zmm1 - - - - vpclmulqdq zmm1,zmm2,zmm4,0x00 - vpsrldq zmm1,zmm1,4 - vpclmulqdq zmm4,zmm2,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm0,zmm1,0x96 - - vmovdqu64 ZMMWORD[224+rdx],zmm4 - vshufi64x2 zmm3,zmm4,zmm4,0x00 - - vpclmulqdq zmm0,zmm5,zmm3,0x11 - vpclmulqdq zmm1,zmm5,zmm3,0x00 - vpclmulqdq zmm2,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm2 - - vpsrldq zmm2,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm0,zmm0,zmm2 - vpxorq zmm5,zmm5,zmm1 - - - - vmovdqu64 zmm2,ZMMWORD[POLY2] - - vpclmulqdq zmm1,zmm2,zmm5,0x01 - vpslldq zmm1,zmm1,8 - vpxorq zmm5,zmm5,zmm1 - - - - vpclmulqdq zmm1,zmm2,zmm5,0x00 - vpsrldq zmm1,zmm1,4 - vpclmulqdq zmm5,zmm2,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm0,zmm1,0x96 - - vmovdqu64 ZMMWORD[160+rdx],zmm5 - - vpclmulqdq zmm0,zmm4,zmm3,0x11 - vpclmulqdq zmm1,zmm4,zmm3,0x00 - vpclmulqdq zmm2,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm2 - - vpsrldq zmm2,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm0,zmm0,zmm2 - vpxorq zmm4,zmm4,zmm1 - - - - vmovdqu64 zmm2,ZMMWORD[POLY2] - - vpclmulqdq zmm1,zmm2,zmm4,0x01 - vpslldq zmm1,zmm1,8 - vpxorq zmm4,zmm4,zmm1 - - - - vpclmulqdq zmm1,zmm2,zmm4,0x00 - vpsrldq zmm1,zmm1,4 - vpclmulqdq zmm4,zmm2,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm0,zmm1,0x96 - - vmovdqu64 ZMMWORD[96+rdx],zmm4 - vzeroupper -$L$abort_init: - DB 0F3h,0C3h ;repret - - -global ossl_aes_gcm_setiv_avx512 - -ALIGN 32 -ossl_aes_gcm_setiv_avx512: - -$L$setiv_seh_begin: -DB 243,15,30,250 - push rbx - -$L$setiv_seh_push_rbx: - push rbp - -$L$setiv_seh_push_rbp: - push r12 - -$L$setiv_seh_push_r12: - push r13 - -$L$setiv_seh_push_r13: - push r14 - -$L$setiv_seh_push_r14: - push r15 - -$L$setiv_seh_push_r15: - push rdi -$L$setiv_seh_push_rdi: - push rsi -$L$setiv_seh_push_rsi: - - sub rsp,168 -$L$setiv_seh_allocstack_xmm: - - - - - - - - - - - lea rbp,[160+rsp] - -$L$setiv_seh_setfp: - vmovdqu XMMWORD[rsp],xmm6 -$L$setiv_seh_save_xmm6: - vmovdqu XMMWORD[16+rsp],xmm7 -$L$setiv_seh_save_xmm7: - vmovdqu XMMWORD[32+rsp],xmm8 -$L$setiv_seh_save_xmm8: - vmovdqu XMMWORD[48+rsp],xmm9 -$L$setiv_seh_save_xmm9: - vmovdqu XMMWORD[64+rsp],xmm10 -$L$setiv_seh_save_xmm10: - vmovdqu XMMWORD[80+rsp],xmm11 -$L$setiv_seh_save_xmm11: - vmovdqu XMMWORD[96+rsp],xmm12 -$L$setiv_seh_save_xmm12: - vmovdqu XMMWORD[112+rsp],xmm13 -$L$setiv_seh_save_xmm13: - vmovdqu XMMWORD[128+rsp],xmm14 -$L$setiv_seh_save_xmm14: - vmovdqu XMMWORD[144+rsp],xmm15 -$L$setiv_seh_save_xmm15: - -$L$setiv_seh_prolog_end: - sub rsp,816 - and rsp,(-64) - cmp r9,12 - je NEAR iv_len_12_init_IV - vpxor xmm2,xmm2,xmm2 - mov r10,r8 - mov r11,r9 - or r11,r11 - jz NEAR $L$_CALC_AAD_done_1 - - xor rbx,rbx - vmovdqa64 zmm16,ZMMWORD[SHUF_MASK] - -$L$_get_AAD_loop48x16_1: - cmp r11,768 - jl NEAR $L$_exit_AAD_loop48x16_1 - vmovdqu64 zmm11,ZMMWORD[r10] - vmovdqu64 zmm3,ZMMWORD[64+r10] - vmovdqu64 zmm4,ZMMWORD[128+r10] - vmovdqu64 zmm5,ZMMWORD[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - test rbx,rbx - jnz NEAR $L$_skip_hkeys_precomputation_2 - - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vmovdqu64 ZMMWORD[704+rsp],zmm1 - - vmovdqu64 zmm9,ZMMWORD[224+rdx] - vmovdqu64 ZMMWORD[640+rsp],zmm9 - - - vshufi64x2 zmm9,zmm9,zmm9,0x00 - - vmovdqu64 zmm10,ZMMWORD[160+rdx] - vmovdqu64 ZMMWORD[576+rsp],zmm10 - - vmovdqu64 zmm12,ZMMWORD[96+rdx] - vmovdqu64 ZMMWORD[512+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[192+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[128+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[64+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[rsp],zmm12 -$L$_skip_hkeys_precomputation_2: - mov rbx,1 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm19,ZMMWORD[rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[64+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpxorq zmm7,zmm10,zmm17 - vpxorq zmm6,zmm1,zmm13 - vpxorq zmm8,zmm9,zmm15 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[128+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[192+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm11,ZMMWORD[256+r10] - vmovdqu64 zmm3,ZMMWORD[320+r10] - vmovdqu64 zmm4,ZMMWORD[384+r10] - vmovdqu64 zmm5,ZMMWORD[448+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vmovdqu64 zmm19,ZMMWORD[256+rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[320+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[384+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[448+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm11,ZMMWORD[512+r10] - vmovdqu64 zmm3,ZMMWORD[576+r10] - vmovdqu64 zmm4,ZMMWORD[640+r10] - vmovdqu64 zmm5,ZMMWORD[704+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vmovdqu64 zmm19,ZMMWORD[512+rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[576+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[640+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[704+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - - vpsrldq zmm1,zmm7,8 - vpslldq zmm9,zmm7,8 - vpxorq zmm6,zmm6,zmm1 - vpxorq zmm8,zmm8,zmm9 - vextracti64x4 ymm1,zmm6,1 - vpxorq ymm6,ymm6,ymm1 - vextracti32x4 xmm1,ymm6,1 - vpxorq xmm6,xmm6,xmm1 - vextracti64x4 ymm9,zmm8,1 - vpxorq ymm8,ymm8,ymm9 - vextracti32x4 xmm9,ymm8,1 - vpxorq xmm8,xmm8,xmm9 - vmovdqa64 xmm10,XMMWORD[POLY2] - - - vpclmulqdq xmm1,xmm10,xmm8,0x01 - vpslldq xmm1,xmm1,8 - vpxorq xmm1,xmm8,xmm1 - - - vpclmulqdq xmm9,xmm10,xmm1,0x00 - vpsrldq xmm9,xmm9,4 - vpclmulqdq xmm2,xmm10,xmm1,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm9,xmm6,0x96 - - sub r11,768 - je NEAR $L$_CALC_AAD_done_1 - - add r10,768 - jmp NEAR $L$_get_AAD_loop48x16_1 - -$L$_exit_AAD_loop48x16_1: - - cmp r11,512 - jl NEAR $L$_less_than_32x16_1 - - vmovdqu64 zmm11,ZMMWORD[r10] - vmovdqu64 zmm3,ZMMWORD[64+r10] - vmovdqu64 zmm4,ZMMWORD[128+r10] - vmovdqu64 zmm5,ZMMWORD[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - test rbx,rbx - jnz NEAR $L$_skip_hkeys_precomputation_3 - - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vmovdqu64 ZMMWORD[704+rsp],zmm1 - - vmovdqu64 zmm9,ZMMWORD[224+rdx] - vmovdqu64 ZMMWORD[640+rsp],zmm9 - - - vshufi64x2 zmm9,zmm9,zmm9,0x00 - - vmovdqu64 zmm10,ZMMWORD[160+rdx] - vmovdqu64 ZMMWORD[576+rsp],zmm10 - - vmovdqu64 zmm12,ZMMWORD[96+rdx] - vmovdqu64 ZMMWORD[512+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm12 -$L$_skip_hkeys_precomputation_3: - mov rbx,1 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm19,ZMMWORD[256+rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[320+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpxorq zmm7,zmm10,zmm17 - vpxorq zmm6,zmm1,zmm13 - vpxorq zmm8,zmm9,zmm15 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[384+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[448+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm11,ZMMWORD[256+r10] - vmovdqu64 zmm3,ZMMWORD[320+r10] - vmovdqu64 zmm4,ZMMWORD[384+r10] - vmovdqu64 zmm5,ZMMWORD[448+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vmovdqu64 zmm19,ZMMWORD[512+rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[576+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[640+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[704+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - - vpsrldq zmm1,zmm7,8 - vpslldq zmm9,zmm7,8 - vpxorq zmm6,zmm6,zmm1 - vpxorq zmm8,zmm8,zmm9 - vextracti64x4 ymm1,zmm6,1 - vpxorq ymm6,ymm6,ymm1 - vextracti32x4 xmm1,ymm6,1 - vpxorq xmm6,xmm6,xmm1 - vextracti64x4 ymm9,zmm8,1 - vpxorq ymm8,ymm8,ymm9 - vextracti32x4 xmm9,ymm8,1 - vpxorq xmm8,xmm8,xmm9 - vmovdqa64 xmm10,XMMWORD[POLY2] - - - vpclmulqdq xmm1,xmm10,xmm8,0x01 - vpslldq xmm1,xmm1,8 - vpxorq xmm1,xmm8,xmm1 - - - vpclmulqdq xmm9,xmm10,xmm1,0x00 - vpsrldq xmm9,xmm9,4 - vpclmulqdq xmm2,xmm10,xmm1,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm9,xmm6,0x96 - - sub r11,512 - je NEAR $L$_CALC_AAD_done_1 - - add r10,512 - jmp NEAR $L$_less_than_16x16_1 - -$L$_less_than_32x16_1: - cmp r11,256 - jl NEAR $L$_less_than_16x16_1 - - vmovdqu64 zmm11,ZMMWORD[r10] - vmovdqu64 zmm3,ZMMWORD[64+r10] - vmovdqu64 zmm4,ZMMWORD[128+r10] - vmovdqu64 zmm5,ZMMWORD[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm19,ZMMWORD[96+rdx] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[160+rdx] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpxorq zmm7,zmm10,zmm17 - vpxorq zmm6,zmm1,zmm13 - vpxorq zmm8,zmm9,zmm15 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[224+rdx] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[288+rdx] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - - vpsrldq zmm1,zmm7,8 - vpslldq zmm9,zmm7,8 - vpxorq zmm6,zmm6,zmm1 - vpxorq zmm8,zmm8,zmm9 - vextracti64x4 ymm1,zmm6,1 - vpxorq ymm6,ymm6,ymm1 - vextracti32x4 xmm1,ymm6,1 - vpxorq xmm6,xmm6,xmm1 - vextracti64x4 ymm9,zmm8,1 - vpxorq ymm8,ymm8,ymm9 - vextracti32x4 xmm9,ymm8,1 - vpxorq xmm8,xmm8,xmm9 - vmovdqa64 xmm10,XMMWORD[POLY2] - - - vpclmulqdq xmm1,xmm10,xmm8,0x01 - vpslldq xmm1,xmm1,8 - vpxorq xmm1,xmm8,xmm1 - - - vpclmulqdq xmm9,xmm10,xmm1,0x00 - vpsrldq xmm9,xmm9,4 - vpclmulqdq xmm2,xmm10,xmm1,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm9,xmm6,0x96 - - sub r11,256 - je NEAR $L$_CALC_AAD_done_1 - - add r10,256 - -$L$_less_than_16x16_1: - - lea r12,[byte64_len_to_mask_table] - lea r12,[r11*8+r12] - - - add r11d,15 - shr r11d,4 - cmp r11d,2 - jb NEAR $L$_AAD_blocks_1_1 - je NEAR $L$_AAD_blocks_2_1 - cmp r11d,4 - jb NEAR $L$_AAD_blocks_3_1 - je NEAR $L$_AAD_blocks_4_1 - cmp r11d,6 - jb NEAR $L$_AAD_blocks_5_1 - je NEAR $L$_AAD_blocks_6_1 - cmp r11d,8 - jb NEAR $L$_AAD_blocks_7_1 - je NEAR $L$_AAD_blocks_8_1 - cmp r11d,10 - jb NEAR $L$_AAD_blocks_9_1 - je NEAR $L$_AAD_blocks_10_1 - cmp r11d,12 - jb NEAR $L$_AAD_blocks_11_1 - je NEAR $L$_AAD_blocks_12_1 - cmp r11d,14 - jb NEAR $L$_AAD_blocks_13_1 - je NEAR $L$_AAD_blocks_14_1 - cmp r11d,15 - je NEAR $L$_AAD_blocks_15_1 -$L$_AAD_blocks_16_1: - sub r12,1536 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4,ZMMWORD[128+r10] - vmovdqu8 zmm5{k1}{z},[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[96+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[160+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[224+rdx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm1,zmm11,zmm9,0x96 - vpternlogq zmm6,zmm3,zmm10,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm7,zmm11,zmm12,0x96 - vpternlogq zmm8,zmm3,zmm13,0x96 - vmovdqu64 zmm15,ZMMWORD[288+rdx] - vpclmulqdq zmm9,zmm5,zmm15,0x11 - vpclmulqdq zmm10,zmm5,zmm15,0x00 - vpclmulqdq zmm12,zmm5,zmm15,0x01 - vpclmulqdq zmm13,zmm5,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - - vpxorq zmm12,zmm12,zmm13 - vpsrldq zmm7,zmm12,8 - vpslldq zmm8,zmm12,8 - vpxorq zmm1,zmm9,zmm7 - vpxorq zmm6,zmm10,zmm8 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_15_1: - sub r12,1536 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4,ZMMWORD[128+r10] - vmovdqu8 zmm5{k1}{z},[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[112+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[176+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[240+rdx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm9,zmm11,zmm1,0x96 - vpternlogq zmm10,zmm3,zmm6,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm12,zmm11,zmm7,0x96 - vpternlogq zmm13,zmm3,zmm8,0x96 - vmovdqu64 ymm15,YMMWORD[304+rdx] - vinserti64x2 zmm15,zmm15,ZMMWORD[336+rdx],2 - vpclmulqdq zmm7,zmm5,zmm15,0x01 - vpclmulqdq zmm8,zmm5,zmm15,0x10 - vpclmulqdq zmm1,zmm5,zmm15,0x11 - vpclmulqdq zmm6,zmm5,zmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_14_1: - sub r12,1536 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4,ZMMWORD[128+r10] - vmovdqu8 ymm5{k1}{z},[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb ymm5,ymm5,ymm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[128+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[192+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[256+rdx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm9,zmm11,zmm1,0x96 - vpternlogq zmm10,zmm3,zmm6,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm12,zmm11,zmm7,0x96 - vpternlogq zmm13,zmm3,zmm8,0x96 - vmovdqu64 ymm15,YMMWORD[320+rdx] - vpclmulqdq ymm7,ymm5,ymm15,0x01 - vpclmulqdq ymm8,ymm5,ymm15,0x10 - vpclmulqdq ymm1,ymm5,ymm15,0x11 - vpclmulqdq ymm6,ymm5,ymm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_13_1: - sub r12,1536 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4,ZMMWORD[128+r10] - vmovdqu8 xmm5{k1}{z},[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb xmm5,xmm5,xmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[144+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[208+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[272+rdx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm9,zmm11,zmm1,0x96 - vpternlogq zmm10,zmm3,zmm6,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm12,zmm11,zmm7,0x96 - vpternlogq zmm13,zmm3,zmm8,0x96 - vmovdqu64 xmm15,XMMWORD[336+rdx] - vpclmulqdq xmm7,xmm5,xmm15,0x01 - vpclmulqdq xmm8,xmm5,xmm15,0x10 - vpclmulqdq xmm1,xmm5,xmm15,0x11 - vpclmulqdq xmm6,xmm5,xmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_12_1: - sub r12,1024 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4{k1}{z},[128+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[160+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[224+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[288+rdx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm9,zmm11,zmm1,0x96 - vpternlogq zmm10,zmm3,zmm6,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm12,zmm11,zmm7,0x96 - vpternlogq zmm13,zmm3,zmm8,0x96 - - vpxorq zmm12,zmm12,zmm13 - vpsrldq zmm7,zmm12,8 - vpslldq zmm8,zmm12,8 - vpxorq zmm1,zmm9,zmm7 - vpxorq zmm6,zmm10,zmm8 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_11_1: - sub r12,1024 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4{k1}{z},[128+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[176+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[240+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - vmovdqu64 ymm15,YMMWORD[304+rdx] - vinserti64x2 zmm15,zmm15,ZMMWORD[336+rdx],2 - vpclmulqdq zmm7,zmm4,zmm15,0x01 - vpclmulqdq zmm8,zmm4,zmm15,0x10 - vpclmulqdq zmm1,zmm4,zmm15,0x11 - vpclmulqdq zmm6,zmm4,zmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_10_1: - sub r12,1024 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 ymm4{k1}{z},[128+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb ymm4,ymm4,ymm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[192+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[256+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - vmovdqu64 ymm15,YMMWORD[320+rdx] - vpclmulqdq ymm7,ymm4,ymm15,0x01 - vpclmulqdq ymm8,ymm4,ymm15,0x10 - vpclmulqdq ymm1,ymm4,ymm15,0x11 - vpclmulqdq ymm6,ymm4,ymm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_9_1: - sub r12,1024 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 xmm4{k1}{z},[128+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb xmm4,xmm4,xmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[208+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[272+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - vmovdqu64 xmm15,XMMWORD[336+rdx] - vpclmulqdq xmm7,xmm4,xmm15,0x01 - vpclmulqdq xmm8,xmm4,xmm15,0x10 - vpclmulqdq xmm1,xmm4,xmm15,0x11 - vpclmulqdq xmm6,xmm4,xmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_8_1: - sub r12,512 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3{k1}{z},[64+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[224+rdx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[288+rdx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - - vpxorq zmm12,zmm12,zmm13 - vpsrldq zmm7,zmm12,8 - vpslldq zmm8,zmm12,8 - vpxorq zmm1,zmm9,zmm7 - vpxorq zmm6,zmm10,zmm8 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_7_1: - sub r12,512 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3{k1}{z},[64+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[240+rdx] - vpclmulqdq zmm9,zmm11,zmm15,0x11 - vpclmulqdq zmm10,zmm11,zmm15,0x00 - vpclmulqdq zmm12,zmm11,zmm15,0x01 - vpclmulqdq zmm13,zmm11,zmm15,0x10 - vmovdqu64 ymm15,YMMWORD[304+rdx] - vinserti64x2 zmm15,zmm15,ZMMWORD[336+rdx],2 - vpclmulqdq zmm7,zmm3,zmm15,0x01 - vpclmulqdq zmm8,zmm3,zmm15,0x10 - vpclmulqdq zmm1,zmm3,zmm15,0x11 - vpclmulqdq zmm6,zmm3,zmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_6_1: - sub r12,512 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 ymm3{k1}{z},[64+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb ymm3,ymm3,ymm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[256+rdx] - vpclmulqdq zmm9,zmm11,zmm15,0x11 - vpclmulqdq zmm10,zmm11,zmm15,0x00 - vpclmulqdq zmm12,zmm11,zmm15,0x01 - vpclmulqdq zmm13,zmm11,zmm15,0x10 - vmovdqu64 ymm15,YMMWORD[320+rdx] - vpclmulqdq ymm7,ymm3,ymm15,0x01 - vpclmulqdq ymm8,ymm3,ymm15,0x10 - vpclmulqdq ymm1,ymm3,ymm15,0x11 - vpclmulqdq ymm6,ymm3,ymm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_5_1: - sub r12,512 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 xmm3{k1}{z},[64+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb xmm3,xmm3,xmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[272+rdx] - vpclmulqdq zmm9,zmm11,zmm15,0x11 - vpclmulqdq zmm10,zmm11,zmm15,0x00 - vpclmulqdq zmm12,zmm11,zmm15,0x01 - vpclmulqdq zmm13,zmm11,zmm15,0x10 - vmovdqu64 xmm15,XMMWORD[336+rdx] - vpclmulqdq xmm7,xmm3,xmm15,0x01 - vpclmulqdq xmm8,xmm3,xmm15,0x10 - vpclmulqdq xmm1,xmm3,xmm15,0x11 - vpclmulqdq xmm6,xmm3,xmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_4_1: - kmovq k1,[r12] - vmovdqu8 zmm11{k1}{z},[r10] - vpshufb zmm11,zmm11,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 zmm15,ZMMWORD[288+rdx] - vpclmulqdq zmm9,zmm11,zmm15,0x11 - vpclmulqdq zmm10,zmm11,zmm15,0x00 - vpclmulqdq zmm12,zmm11,zmm15,0x01 - vpclmulqdq zmm13,zmm11,zmm15,0x10 - - vpxorq zmm12,zmm12,zmm13 - vpsrldq zmm7,zmm12,8 - vpslldq zmm8,zmm12,8 - vpxorq zmm1,zmm9,zmm7 - vpxorq zmm6,zmm10,zmm8 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_3_1: - kmovq k1,[r12] - vmovdqu8 zmm11{k1}{z},[r10] - vpshufb zmm11,zmm11,zmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 ymm15,YMMWORD[304+rdx] - vinserti64x2 zmm15,zmm15,ZMMWORD[336+rdx],2 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_2_1: - kmovq k1,[r12] - vmovdqu8 ymm11{k1}{z},[r10] - vpshufb ymm11,ymm11,ymm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 ymm15,YMMWORD[320+rdx] - vpclmulqdq ymm7,ymm11,ymm15,0x01 - vpclmulqdq ymm8,ymm11,ymm15,0x10 - vpclmulqdq ymm1,ymm11,ymm15,0x11 - vpclmulqdq ymm6,ymm11,ymm15,0x00 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_1 -$L$_AAD_blocks_1_1: - kmovq k1,[r12] - vmovdqu8 xmm11{k1}{z},[r10] - vpshufb xmm11,xmm11,xmm16 - vpxorq zmm11,zmm11,zmm2 - vmovdqu64 xmm15,XMMWORD[336+rdx] - vpclmulqdq xmm7,xmm11,xmm15,0x01 - vpclmulqdq xmm8,xmm11,xmm15,0x10 - vpclmulqdq xmm1,xmm11,xmm15,0x11 - vpclmulqdq xmm6,xmm11,xmm15,0x00 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm2,xmm15,xmm7,0x10 - vpslldq xmm2,xmm2,4 - vpternlogq xmm2,xmm8,xmm1,0x96 - -$L$_CALC_AAD_done_1: - mov r10,r9 - shl r10,3 - vmovq xmm3,r10 - - - vpxorq xmm2,xmm3,xmm2 - - vmovdqu64 xmm1,XMMWORD[336+rdx] - - vpclmulqdq xmm11,xmm2,xmm1,0x11 - vpclmulqdq xmm3,xmm2,xmm1,0x00 - vpclmulqdq xmm4,xmm2,xmm1,0x01 - vpclmulqdq xmm2,xmm2,xmm1,0x10 - vpxorq xmm2,xmm2,xmm4 - - vpsrldq xmm4,xmm2,8 - vpslldq xmm2,xmm2,8 - vpxorq xmm11,xmm11,xmm4 - vpxorq xmm2,xmm2,xmm3 - - - - vmovdqu64 xmm4,XMMWORD[POLY2] - - vpclmulqdq xmm3,xmm4,xmm2,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm2,xmm2,xmm3 - - - - vpclmulqdq xmm3,xmm4,xmm2,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm2,xmm4,xmm2,0x10 - vpslldq xmm2,xmm2,4 - - vpternlogq xmm2,xmm11,xmm3,0x96 - - vpshufb xmm2,xmm2,XMMWORD[SHUF_MASK] - jmp NEAR skip_iv_len_12_init_IV -iv_len_12_init_IV: - - vmovdqu8 xmm2,XMMWORD[ONEf] - mov r11,r8 - mov r10d,0x0000000000000fff - kmovq k1,r10 - vmovdqu8 xmm2{k1},[r11] -skip_iv_len_12_init_IV: - vmovdqu xmm1,xmm2 - - - mov r10d,DWORD[240+rcx] - cmp r10d,9 - je NEAR $L$aes_128_4 - cmp r10d,11 - je NEAR $L$aes_192_4 - cmp r10d,13 - je NEAR $L$aes_256_4 - jmp NEAR $L$exit_aes_4 -ALIGN 32 -$L$aes_128_4: - vpxorq xmm1,xmm1,XMMWORD[rcx] - - vaesenc xmm1,xmm1,XMMWORD[16+rcx] - - vaesenc xmm1,xmm1,XMMWORD[32+rcx] - - vaesenc xmm1,xmm1,XMMWORD[48+rcx] - - vaesenc xmm1,xmm1,XMMWORD[64+rcx] - - vaesenc xmm1,xmm1,XMMWORD[80+rcx] - - vaesenc xmm1,xmm1,XMMWORD[96+rcx] - - vaesenc xmm1,xmm1,XMMWORD[112+rcx] - - vaesenc xmm1,xmm1,XMMWORD[128+rcx] - - vaesenc xmm1,xmm1,XMMWORD[144+rcx] - - vaesenclast xmm1,xmm1,XMMWORD[160+rcx] - jmp NEAR $L$exit_aes_4 -ALIGN 32 -$L$aes_192_4: - vpxorq xmm1,xmm1,XMMWORD[rcx] - - vaesenc xmm1,xmm1,XMMWORD[16+rcx] - - vaesenc xmm1,xmm1,XMMWORD[32+rcx] - - vaesenc xmm1,xmm1,XMMWORD[48+rcx] - - vaesenc xmm1,xmm1,XMMWORD[64+rcx] - - vaesenc xmm1,xmm1,XMMWORD[80+rcx] - - vaesenc xmm1,xmm1,XMMWORD[96+rcx] - - vaesenc xmm1,xmm1,XMMWORD[112+rcx] - - vaesenc xmm1,xmm1,XMMWORD[128+rcx] - - vaesenc xmm1,xmm1,XMMWORD[144+rcx] - - vaesenc xmm1,xmm1,XMMWORD[160+rcx] - - vaesenc xmm1,xmm1,XMMWORD[176+rcx] - - vaesenclast xmm1,xmm1,XMMWORD[192+rcx] - jmp NEAR $L$exit_aes_4 -ALIGN 32 -$L$aes_256_4: - vpxorq xmm1,xmm1,XMMWORD[rcx] - - vaesenc xmm1,xmm1,XMMWORD[16+rcx] - - vaesenc xmm1,xmm1,XMMWORD[32+rcx] - - vaesenc xmm1,xmm1,XMMWORD[48+rcx] - - vaesenc xmm1,xmm1,XMMWORD[64+rcx] - - vaesenc xmm1,xmm1,XMMWORD[80+rcx] - - vaesenc xmm1,xmm1,XMMWORD[96+rcx] - - vaesenc xmm1,xmm1,XMMWORD[112+rcx] - - vaesenc xmm1,xmm1,XMMWORD[128+rcx] - - vaesenc xmm1,xmm1,XMMWORD[144+rcx] - - vaesenc xmm1,xmm1,XMMWORD[160+rcx] - - vaesenc xmm1,xmm1,XMMWORD[176+rcx] - - vaesenc xmm1,xmm1,XMMWORD[192+rcx] - - vaesenc xmm1,xmm1,XMMWORD[208+rcx] - - vaesenclast xmm1,xmm1,XMMWORD[224+rcx] - jmp NEAR $L$exit_aes_4 -$L$exit_aes_4: - - vmovdqu XMMWORD[32+rdx],xmm1 - - - vpshufb xmm2,xmm2,XMMWORD[SHUF_MASK] - vmovdqu XMMWORD[rdx],xmm2 - cmp r9,256 - jbe NEAR $L$skip_hkeys_cleanup_5 - vpxor xmm0,xmm0,xmm0 - vmovdqa64 ZMMWORD[rsp],zmm0 - vmovdqa64 ZMMWORD[64+rsp],zmm0 - vmovdqa64 ZMMWORD[128+rsp],zmm0 - vmovdqa64 ZMMWORD[192+rsp],zmm0 - vmovdqa64 ZMMWORD[256+rsp],zmm0 - vmovdqa64 ZMMWORD[320+rsp],zmm0 - vmovdqa64 ZMMWORD[384+rsp],zmm0 - vmovdqa64 ZMMWORD[448+rsp],zmm0 - vmovdqa64 ZMMWORD[512+rsp],zmm0 - vmovdqa64 ZMMWORD[576+rsp],zmm0 - vmovdqa64 ZMMWORD[640+rsp],zmm0 - vmovdqa64 ZMMWORD[704+rsp],zmm0 -$L$skip_hkeys_cleanup_5: - vzeroupper - vmovdqu xmm15,XMMWORD[((-16))+rbp] - vmovdqu xmm14,XMMWORD[((-32))+rbp] - vmovdqu xmm13,XMMWORD[((-48))+rbp] - vmovdqu xmm12,XMMWORD[((-64))+rbp] - vmovdqu xmm11,XMMWORD[((-80))+rbp] - vmovdqu xmm10,XMMWORD[((-96))+rbp] - vmovdqu xmm9,XMMWORD[((-112))+rbp] - vmovdqu xmm8,XMMWORD[((-128))+rbp] - vmovdqu xmm7,XMMWORD[((-144))+rbp] - vmovdqu xmm6,XMMWORD[((-160))+rbp] - lea rsp,[8+rbp] - pop rsi - - pop rdi - - pop r15 - - pop r14 - - pop r13 - - pop r12 - - pop rbp - - pop rbx - -$L$abort_setiv: - DB 0F3h,0C3h ;repret -$L$setiv_seh_end: - - -global ossl_aes_gcm_update_aad_avx512 - -ALIGN 32 -ossl_aes_gcm_update_aad_avx512: - -$L$ghash_seh_begin: -DB 243,15,30,250 - push rbx - -$L$ghash_seh_push_rbx: - push rbp - -$L$ghash_seh_push_rbp: - push r12 - -$L$ghash_seh_push_r12: - push r13 - -$L$ghash_seh_push_r13: - push r14 - -$L$ghash_seh_push_r14: - push r15 - -$L$ghash_seh_push_r15: - push rdi -$L$ghash_seh_push_rdi: - push rsi -$L$ghash_seh_push_rsi: - - sub rsp,168 -$L$ghash_seh_allocstack_xmm: - - - - - - - - - - - lea rbp,[160+rsp] - -$L$ghash_seh_setfp: - vmovdqu XMMWORD[rsp],xmm6 -$L$ghash_seh_save_xmm6: - vmovdqu XMMWORD[16+rsp],xmm7 -$L$ghash_seh_save_xmm7: - vmovdqu XMMWORD[32+rsp],xmm8 -$L$ghash_seh_save_xmm8: - vmovdqu XMMWORD[48+rsp],xmm9 -$L$ghash_seh_save_xmm9: - vmovdqu XMMWORD[64+rsp],xmm10 -$L$ghash_seh_save_xmm10: - vmovdqu XMMWORD[80+rsp],xmm11 -$L$ghash_seh_save_xmm11: - vmovdqu XMMWORD[96+rsp],xmm12 -$L$ghash_seh_save_xmm12: - vmovdqu XMMWORD[112+rsp],xmm13 -$L$ghash_seh_save_xmm13: - vmovdqu XMMWORD[128+rsp],xmm14 -$L$ghash_seh_save_xmm14: - vmovdqu XMMWORD[144+rsp],xmm15 -$L$ghash_seh_save_xmm15: - -$L$ghash_seh_prolog_end: - sub rsp,816 - and rsp,(-64) - vmovdqu64 xmm14,XMMWORD[64+rcx] - mov r10,rdx - mov r11,r8 - or r11,r11 - jz NEAR $L$_CALC_AAD_done_6 - - xor rbx,rbx - vmovdqa64 zmm16,ZMMWORD[SHUF_MASK] - -$L$_get_AAD_loop48x16_6: - cmp r11,768 - jl NEAR $L$_exit_AAD_loop48x16_6 - vmovdqu64 zmm11,ZMMWORD[r10] - vmovdqu64 zmm3,ZMMWORD[64+r10] - vmovdqu64 zmm4,ZMMWORD[128+r10] - vmovdqu64 zmm5,ZMMWORD[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - test rbx,rbx - jnz NEAR $L$_skip_hkeys_precomputation_7 - - vmovdqu64 zmm1,ZMMWORD[288+rcx] - vmovdqu64 ZMMWORD[704+rsp],zmm1 - - vmovdqu64 zmm9,ZMMWORD[224+rcx] - vmovdqu64 ZMMWORD[640+rsp],zmm9 - - - vshufi64x2 zmm9,zmm9,zmm9,0x00 - - vmovdqu64 zmm10,ZMMWORD[160+rcx] - vmovdqu64 ZMMWORD[576+rsp],zmm10 - - vmovdqu64 zmm12,ZMMWORD[96+rcx] - vmovdqu64 ZMMWORD[512+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[192+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[128+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[64+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[rsp],zmm12 -$L$_skip_hkeys_precomputation_7: - mov rbx,1 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm19,ZMMWORD[rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[64+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpxorq zmm7,zmm10,zmm17 - vpxorq zmm6,zmm1,zmm13 - vpxorq zmm8,zmm9,zmm15 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[128+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[192+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm11,ZMMWORD[256+r10] - vmovdqu64 zmm3,ZMMWORD[320+r10] - vmovdqu64 zmm4,ZMMWORD[384+r10] - vmovdqu64 zmm5,ZMMWORD[448+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vmovdqu64 zmm19,ZMMWORD[256+rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[320+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[384+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[448+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm11,ZMMWORD[512+r10] - vmovdqu64 zmm3,ZMMWORD[576+r10] - vmovdqu64 zmm4,ZMMWORD[640+r10] - vmovdqu64 zmm5,ZMMWORD[704+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vmovdqu64 zmm19,ZMMWORD[512+rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[576+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[640+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[704+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - - vpsrldq zmm1,zmm7,8 - vpslldq zmm9,zmm7,8 - vpxorq zmm6,zmm6,zmm1 - vpxorq zmm8,zmm8,zmm9 - vextracti64x4 ymm1,zmm6,1 - vpxorq ymm6,ymm6,ymm1 - vextracti32x4 xmm1,ymm6,1 - vpxorq xmm6,xmm6,xmm1 - vextracti64x4 ymm9,zmm8,1 - vpxorq ymm8,ymm8,ymm9 - vextracti32x4 xmm9,ymm8,1 - vpxorq xmm8,xmm8,xmm9 - vmovdqa64 xmm10,XMMWORD[POLY2] - - - vpclmulqdq xmm1,xmm10,xmm8,0x01 - vpslldq xmm1,xmm1,8 - vpxorq xmm1,xmm8,xmm1 - - - vpclmulqdq xmm9,xmm10,xmm1,0x00 - vpsrldq xmm9,xmm9,4 - vpclmulqdq xmm14,xmm10,xmm1,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm9,xmm6,0x96 - - sub r11,768 - je NEAR $L$_CALC_AAD_done_6 - - add r10,768 - jmp NEAR $L$_get_AAD_loop48x16_6 - -$L$_exit_AAD_loop48x16_6: - - cmp r11,512 - jl NEAR $L$_less_than_32x16_6 - - vmovdqu64 zmm11,ZMMWORD[r10] - vmovdqu64 zmm3,ZMMWORD[64+r10] - vmovdqu64 zmm4,ZMMWORD[128+r10] - vmovdqu64 zmm5,ZMMWORD[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - test rbx,rbx - jnz NEAR $L$_skip_hkeys_precomputation_8 - - vmovdqu64 zmm1,ZMMWORD[288+rcx] - vmovdqu64 ZMMWORD[704+rsp],zmm1 - - vmovdqu64 zmm9,ZMMWORD[224+rcx] - vmovdqu64 ZMMWORD[640+rsp],zmm9 - - - vshufi64x2 zmm9,zmm9,zmm9,0x00 - - vmovdqu64 zmm10,ZMMWORD[160+rcx] - vmovdqu64 ZMMWORD[576+rsp],zmm10 - - vmovdqu64 zmm12,ZMMWORD[96+rcx] - vmovdqu64 ZMMWORD[512+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm12 - - vpclmulqdq zmm13,zmm10,zmm9,0x11 - vpclmulqdq zmm15,zmm10,zmm9,0x00 - vpclmulqdq zmm17,zmm10,zmm9,0x01 - vpclmulqdq zmm10,zmm10,zmm9,0x10 - vpxorq zmm10,zmm10,zmm17 - - vpsrldq zmm17,zmm10,8 - vpslldq zmm10,zmm10,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm10,zmm10,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm10,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm10,zmm10,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm10,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm10,zmm17,zmm10,0x10 - vpslldq zmm10,zmm10,4 - - vpternlogq zmm10,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm10 - - vpclmulqdq zmm13,zmm12,zmm9,0x11 - vpclmulqdq zmm15,zmm12,zmm9,0x00 - vpclmulqdq zmm17,zmm12,zmm9,0x01 - vpclmulqdq zmm12,zmm12,zmm9,0x10 - vpxorq zmm12,zmm12,zmm17 - - vpsrldq zmm17,zmm12,8 - vpslldq zmm12,zmm12,8 - vpxorq zmm13,zmm13,zmm17 - vpxorq zmm12,zmm12,zmm15 - - - - vmovdqu64 zmm17,ZMMWORD[POLY2] - - vpclmulqdq zmm15,zmm17,zmm12,0x01 - vpslldq zmm15,zmm15,8 - vpxorq zmm12,zmm12,zmm15 - - - - vpclmulqdq zmm15,zmm17,zmm12,0x00 - vpsrldq zmm15,zmm15,4 - vpclmulqdq zmm12,zmm17,zmm12,0x10 - vpslldq zmm12,zmm12,4 - - vpternlogq zmm12,zmm13,zmm15,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm12 -$L$_skip_hkeys_precomputation_8: - mov rbx,1 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm19,ZMMWORD[256+rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[320+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpxorq zmm7,zmm10,zmm17 - vpxorq zmm6,zmm1,zmm13 - vpxorq zmm8,zmm9,zmm15 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[384+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[448+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm11,ZMMWORD[256+r10] - vmovdqu64 zmm3,ZMMWORD[320+r10] - vmovdqu64 zmm4,ZMMWORD[384+r10] - vmovdqu64 zmm5,ZMMWORD[448+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vmovdqu64 zmm19,ZMMWORD[512+rsp] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[576+rsp] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[640+rsp] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[704+rsp] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - - vpsrldq zmm1,zmm7,8 - vpslldq zmm9,zmm7,8 - vpxorq zmm6,zmm6,zmm1 - vpxorq zmm8,zmm8,zmm9 - vextracti64x4 ymm1,zmm6,1 - vpxorq ymm6,ymm6,ymm1 - vextracti32x4 xmm1,ymm6,1 - vpxorq xmm6,xmm6,xmm1 - vextracti64x4 ymm9,zmm8,1 - vpxorq ymm8,ymm8,ymm9 - vextracti32x4 xmm9,ymm8,1 - vpxorq xmm8,xmm8,xmm9 - vmovdqa64 xmm10,XMMWORD[POLY2] - - - vpclmulqdq xmm1,xmm10,xmm8,0x01 - vpslldq xmm1,xmm1,8 - vpxorq xmm1,xmm8,xmm1 - - - vpclmulqdq xmm9,xmm10,xmm1,0x00 - vpsrldq xmm9,xmm9,4 - vpclmulqdq xmm14,xmm10,xmm1,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm9,xmm6,0x96 - - sub r11,512 - je NEAR $L$_CALC_AAD_done_6 - - add r10,512 - jmp NEAR $L$_less_than_16x16_6 - -$L$_less_than_32x16_6: - cmp r11,256 - jl NEAR $L$_less_than_16x16_6 - - vmovdqu64 zmm11,ZMMWORD[r10] - vmovdqu64 zmm3,ZMMWORD[64+r10] - vmovdqu64 zmm4,ZMMWORD[128+r10] - vmovdqu64 zmm5,ZMMWORD[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm19,ZMMWORD[96+rcx] - vpclmulqdq zmm1,zmm11,zmm19,0x11 - vpclmulqdq zmm9,zmm11,zmm19,0x00 - vpclmulqdq zmm10,zmm11,zmm19,0x01 - vpclmulqdq zmm12,zmm11,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[160+rcx] - vpclmulqdq zmm13,zmm3,zmm19,0x11 - vpclmulqdq zmm15,zmm3,zmm19,0x00 - vpclmulqdq zmm17,zmm3,zmm19,0x01 - vpclmulqdq zmm18,zmm3,zmm19,0x10 - vpxorq zmm7,zmm10,zmm17 - vpxorq zmm6,zmm1,zmm13 - vpxorq zmm8,zmm9,zmm15 - vpternlogq zmm7,zmm12,zmm18,0x96 - vmovdqu64 zmm19,ZMMWORD[224+rcx] - vpclmulqdq zmm1,zmm4,zmm19,0x11 - vpclmulqdq zmm9,zmm4,zmm19,0x00 - vpclmulqdq zmm10,zmm4,zmm19,0x01 - vpclmulqdq zmm12,zmm4,zmm19,0x10 - vmovdqu64 zmm19,ZMMWORD[288+rcx] - vpclmulqdq zmm13,zmm5,zmm19,0x11 - vpclmulqdq zmm15,zmm5,zmm19,0x00 - vpclmulqdq zmm17,zmm5,zmm19,0x01 - vpclmulqdq zmm18,zmm5,zmm19,0x10 - - vpternlogq zmm7,zmm10,zmm17,0x96 - vpternlogq zmm6,zmm1,zmm13,0x96 - vpternlogq zmm8,zmm9,zmm15,0x96 - vpternlogq zmm7,zmm12,zmm18,0x96 - - vpsrldq zmm1,zmm7,8 - vpslldq zmm9,zmm7,8 - vpxorq zmm6,zmm6,zmm1 - vpxorq zmm8,zmm8,zmm9 - vextracti64x4 ymm1,zmm6,1 - vpxorq ymm6,ymm6,ymm1 - vextracti32x4 xmm1,ymm6,1 - vpxorq xmm6,xmm6,xmm1 - vextracti64x4 ymm9,zmm8,1 - vpxorq ymm8,ymm8,ymm9 - vextracti32x4 xmm9,ymm8,1 - vpxorq xmm8,xmm8,xmm9 - vmovdqa64 xmm10,XMMWORD[POLY2] - - - vpclmulqdq xmm1,xmm10,xmm8,0x01 - vpslldq xmm1,xmm1,8 - vpxorq xmm1,xmm8,xmm1 - - - vpclmulqdq xmm9,xmm10,xmm1,0x00 - vpsrldq xmm9,xmm9,4 - vpclmulqdq xmm14,xmm10,xmm1,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm9,xmm6,0x96 - - sub r11,256 - je NEAR $L$_CALC_AAD_done_6 - - add r10,256 - -$L$_less_than_16x16_6: - - lea r12,[byte64_len_to_mask_table] - lea r12,[r11*8+r12] - - - add r11d,15 - shr r11d,4 - cmp r11d,2 - jb NEAR $L$_AAD_blocks_1_6 - je NEAR $L$_AAD_blocks_2_6 - cmp r11d,4 - jb NEAR $L$_AAD_blocks_3_6 - je NEAR $L$_AAD_blocks_4_6 - cmp r11d,6 - jb NEAR $L$_AAD_blocks_5_6 - je NEAR $L$_AAD_blocks_6_6 - cmp r11d,8 - jb NEAR $L$_AAD_blocks_7_6 - je NEAR $L$_AAD_blocks_8_6 - cmp r11d,10 - jb NEAR $L$_AAD_blocks_9_6 - je NEAR $L$_AAD_blocks_10_6 - cmp r11d,12 - jb NEAR $L$_AAD_blocks_11_6 - je NEAR $L$_AAD_blocks_12_6 - cmp r11d,14 - jb NEAR $L$_AAD_blocks_13_6 - je NEAR $L$_AAD_blocks_14_6 - cmp r11d,15 - je NEAR $L$_AAD_blocks_15_6 -$L$_AAD_blocks_16_6: - sub r12,1536 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4,ZMMWORD[128+r10] - vmovdqu8 zmm5{k1}{z},[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[96+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[160+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[224+rcx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm1,zmm11,zmm9,0x96 - vpternlogq zmm6,zmm3,zmm10,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm7,zmm11,zmm12,0x96 - vpternlogq zmm8,zmm3,zmm13,0x96 - vmovdqu64 zmm15,ZMMWORD[288+rcx] - vpclmulqdq zmm9,zmm5,zmm15,0x11 - vpclmulqdq zmm10,zmm5,zmm15,0x00 - vpclmulqdq zmm12,zmm5,zmm15,0x01 - vpclmulqdq zmm13,zmm5,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - - vpxorq zmm12,zmm12,zmm13 - vpsrldq zmm7,zmm12,8 - vpslldq zmm8,zmm12,8 - vpxorq zmm1,zmm9,zmm7 - vpxorq zmm6,zmm10,zmm8 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_15_6: - sub r12,1536 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4,ZMMWORD[128+r10] - vmovdqu8 zmm5{k1}{z},[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb zmm5,zmm5,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[112+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[176+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[240+rcx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm9,zmm11,zmm1,0x96 - vpternlogq zmm10,zmm3,zmm6,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm12,zmm11,zmm7,0x96 - vpternlogq zmm13,zmm3,zmm8,0x96 - vmovdqu64 ymm15,YMMWORD[304+rcx] - vinserti64x2 zmm15,zmm15,ZMMWORD[336+rcx],2 - vpclmulqdq zmm7,zmm5,zmm15,0x01 - vpclmulqdq zmm8,zmm5,zmm15,0x10 - vpclmulqdq zmm1,zmm5,zmm15,0x11 - vpclmulqdq zmm6,zmm5,zmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_14_6: - sub r12,1536 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4,ZMMWORD[128+r10] - vmovdqu8 ymm5{k1}{z},[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb ymm5,ymm5,ymm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[128+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[192+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[256+rcx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm9,zmm11,zmm1,0x96 - vpternlogq zmm10,zmm3,zmm6,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm12,zmm11,zmm7,0x96 - vpternlogq zmm13,zmm3,zmm8,0x96 - vmovdqu64 ymm15,YMMWORD[320+rcx] - vpclmulqdq ymm7,ymm5,ymm15,0x01 - vpclmulqdq ymm8,ymm5,ymm15,0x10 - vpclmulqdq ymm1,ymm5,ymm15,0x11 - vpclmulqdq ymm6,ymm5,ymm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_13_6: - sub r12,1536 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4,ZMMWORD[128+r10] - vmovdqu8 xmm5{k1}{z},[192+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpshufb xmm5,xmm5,xmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[144+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[208+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[272+rcx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm9,zmm11,zmm1,0x96 - vpternlogq zmm10,zmm3,zmm6,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm12,zmm11,zmm7,0x96 - vpternlogq zmm13,zmm3,zmm8,0x96 - vmovdqu64 xmm15,XMMWORD[336+rcx] - vpclmulqdq xmm7,xmm5,xmm15,0x01 - vpclmulqdq xmm8,xmm5,xmm15,0x10 - vpclmulqdq xmm1,xmm5,xmm15,0x11 - vpclmulqdq xmm6,xmm5,xmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_12_6: - sub r12,1024 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4{k1}{z},[128+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[160+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[224+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[288+rcx] - vpclmulqdq zmm11,zmm4,zmm15,0x11 - vpclmulqdq zmm3,zmm4,zmm15,0x00 - vpternlogq zmm9,zmm11,zmm1,0x96 - vpternlogq zmm10,zmm3,zmm6,0x96 - vpclmulqdq zmm11,zmm4,zmm15,0x01 - vpclmulqdq zmm3,zmm4,zmm15,0x10 - vpternlogq zmm12,zmm11,zmm7,0x96 - vpternlogq zmm13,zmm3,zmm8,0x96 - - vpxorq zmm12,zmm12,zmm13 - vpsrldq zmm7,zmm12,8 - vpslldq zmm8,zmm12,8 - vpxorq zmm1,zmm9,zmm7 - vpxorq zmm6,zmm10,zmm8 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_11_6: - sub r12,1024 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 zmm4{k1}{z},[128+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb zmm4,zmm4,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[176+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[240+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - vmovdqu64 ymm15,YMMWORD[304+rcx] - vinserti64x2 zmm15,zmm15,ZMMWORD[336+rcx],2 - vpclmulqdq zmm7,zmm4,zmm15,0x01 - vpclmulqdq zmm8,zmm4,zmm15,0x10 - vpclmulqdq zmm1,zmm4,zmm15,0x11 - vpclmulqdq zmm6,zmm4,zmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_10_6: - sub r12,1024 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 ymm4{k1}{z},[128+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb ymm4,ymm4,ymm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[192+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[256+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - vmovdqu64 ymm15,YMMWORD[320+rcx] - vpclmulqdq ymm7,ymm4,ymm15,0x01 - vpclmulqdq ymm8,ymm4,ymm15,0x10 - vpclmulqdq ymm1,ymm4,ymm15,0x11 - vpclmulqdq ymm6,ymm4,ymm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_9_6: - sub r12,1024 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3,ZMMWORD[64+r10] - vmovdqu8 xmm4{k1}{z},[128+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpshufb xmm4,xmm4,xmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[208+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[272+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - vmovdqu64 xmm15,XMMWORD[336+rcx] - vpclmulqdq xmm7,xmm4,xmm15,0x01 - vpclmulqdq xmm8,xmm4,xmm15,0x10 - vpclmulqdq xmm1,xmm4,xmm15,0x11 - vpclmulqdq xmm6,xmm4,xmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_8_6: - sub r12,512 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3{k1}{z},[64+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[224+rcx] - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vmovdqu64 zmm15,ZMMWORD[288+rcx] - vpclmulqdq zmm9,zmm3,zmm15,0x11 - vpclmulqdq zmm10,zmm3,zmm15,0x00 - vpclmulqdq zmm12,zmm3,zmm15,0x01 - vpclmulqdq zmm13,zmm3,zmm15,0x10 - vpxorq zmm9,zmm1,zmm9 - vpxorq zmm10,zmm6,zmm10 - vpxorq zmm12,zmm7,zmm12 - vpxorq zmm13,zmm8,zmm13 - - vpxorq zmm12,zmm12,zmm13 - vpsrldq zmm7,zmm12,8 - vpslldq zmm8,zmm12,8 - vpxorq zmm1,zmm9,zmm7 - vpxorq zmm6,zmm10,zmm8 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_7_6: - sub r12,512 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 zmm3{k1}{z},[64+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb zmm3,zmm3,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[240+rcx] - vpclmulqdq zmm9,zmm11,zmm15,0x11 - vpclmulqdq zmm10,zmm11,zmm15,0x00 - vpclmulqdq zmm12,zmm11,zmm15,0x01 - vpclmulqdq zmm13,zmm11,zmm15,0x10 - vmovdqu64 ymm15,YMMWORD[304+rcx] - vinserti64x2 zmm15,zmm15,ZMMWORD[336+rcx],2 - vpclmulqdq zmm7,zmm3,zmm15,0x01 - vpclmulqdq zmm8,zmm3,zmm15,0x10 - vpclmulqdq zmm1,zmm3,zmm15,0x11 - vpclmulqdq zmm6,zmm3,zmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_6_6: - sub r12,512 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 ymm3{k1}{z},[64+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb ymm3,ymm3,ymm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[256+rcx] - vpclmulqdq zmm9,zmm11,zmm15,0x11 - vpclmulqdq zmm10,zmm11,zmm15,0x00 - vpclmulqdq zmm12,zmm11,zmm15,0x01 - vpclmulqdq zmm13,zmm11,zmm15,0x10 - vmovdqu64 ymm15,YMMWORD[320+rcx] - vpclmulqdq ymm7,ymm3,ymm15,0x01 - vpclmulqdq ymm8,ymm3,ymm15,0x10 - vpclmulqdq ymm1,ymm3,ymm15,0x11 - vpclmulqdq ymm6,ymm3,ymm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_5_6: - sub r12,512 - kmovq k1,[r12] - vmovdqu8 zmm11,ZMMWORD[r10] - vmovdqu8 xmm3{k1}{z},[64+r10] - vpshufb zmm11,zmm11,zmm16 - vpshufb xmm3,xmm3,xmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[272+rcx] - vpclmulqdq zmm9,zmm11,zmm15,0x11 - vpclmulqdq zmm10,zmm11,zmm15,0x00 - vpclmulqdq zmm12,zmm11,zmm15,0x01 - vpclmulqdq zmm13,zmm11,zmm15,0x10 - vmovdqu64 xmm15,XMMWORD[336+rcx] - vpclmulqdq xmm7,xmm3,xmm15,0x01 - vpclmulqdq xmm8,xmm3,xmm15,0x10 - vpclmulqdq xmm1,xmm3,xmm15,0x11 - vpclmulqdq xmm6,xmm3,xmm15,0x00 - - vpxorq zmm7,zmm7,zmm12 - vpxorq zmm8,zmm8,zmm13 - vpxorq zmm1,zmm1,zmm9 - vpxorq zmm6,zmm6,zmm10 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_4_6: - kmovq k1,[r12] - vmovdqu8 zmm11{k1}{z},[r10] - vpshufb zmm11,zmm11,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 zmm15,ZMMWORD[288+rcx] - vpclmulqdq zmm9,zmm11,zmm15,0x11 - vpclmulqdq zmm10,zmm11,zmm15,0x00 - vpclmulqdq zmm12,zmm11,zmm15,0x01 - vpclmulqdq zmm13,zmm11,zmm15,0x10 - - vpxorq zmm12,zmm12,zmm13 - vpsrldq zmm7,zmm12,8 - vpslldq zmm8,zmm12,8 - vpxorq zmm1,zmm9,zmm7 - vpxorq zmm6,zmm10,zmm8 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_3_6: - kmovq k1,[r12] - vmovdqu8 zmm11{k1}{z},[r10] - vpshufb zmm11,zmm11,zmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 ymm15,YMMWORD[304+rcx] - vinserti64x2 zmm15,zmm15,ZMMWORD[336+rcx],2 - vpclmulqdq zmm7,zmm11,zmm15,0x01 - vpclmulqdq zmm8,zmm11,zmm15,0x10 - vpclmulqdq zmm1,zmm11,zmm15,0x11 - vpclmulqdq zmm6,zmm11,zmm15,0x00 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_2_6: - kmovq k1,[r12] - vmovdqu8 ymm11{k1}{z},[r10] - vpshufb ymm11,ymm11,ymm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 ymm15,YMMWORD[320+rcx] - vpclmulqdq ymm7,ymm11,ymm15,0x01 - vpclmulqdq ymm8,ymm11,ymm15,0x10 - vpclmulqdq ymm1,ymm11,ymm15,0x11 - vpclmulqdq ymm6,ymm11,ymm15,0x00 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - - jmp NEAR $L$_CALC_AAD_done_6 -$L$_AAD_blocks_1_6: - kmovq k1,[r12] - vmovdqu8 xmm11{k1}{z},[r10] - vpshufb xmm11,xmm11,xmm16 - vpxorq zmm11,zmm11,zmm14 - vmovdqu64 xmm15,XMMWORD[336+rcx] - vpclmulqdq xmm7,xmm11,xmm15,0x01 - vpclmulqdq xmm8,xmm11,xmm15,0x10 - vpclmulqdq xmm1,xmm11,xmm15,0x11 - vpclmulqdq xmm6,xmm11,xmm15,0x00 - - vpxorq zmm7,zmm7,zmm8 - vpsrldq zmm12,zmm7,8 - vpslldq zmm13,zmm7,8 - vpxorq zmm1,zmm1,zmm12 - vpxorq zmm6,zmm6,zmm13 - vextracti64x4 ymm12,zmm1,1 - vpxorq ymm1,ymm1,ymm12 - vextracti32x4 xmm12,ymm1,1 - vpxorq xmm1,xmm1,xmm12 - vextracti64x4 ymm13,zmm6,1 - vpxorq ymm6,ymm6,ymm13 - vextracti32x4 xmm13,ymm6,1 - vpxorq xmm6,xmm6,xmm13 - vmovdqa64 xmm15,XMMWORD[POLY2] - - - vpclmulqdq xmm7,xmm15,xmm6,0x01 - vpslldq xmm7,xmm7,8 - vpxorq xmm7,xmm6,xmm7 - - - vpclmulqdq xmm8,xmm15,xmm7,0x00 - vpsrldq xmm8,xmm8,4 - vpclmulqdq xmm14,xmm15,xmm7,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm8,xmm1,0x96 - -$L$_CALC_AAD_done_6: - vmovdqu64 XMMWORD[64+rcx],xmm14 - cmp r8,256 - jbe NEAR $L$skip_hkeys_cleanup_9 - vpxor xmm0,xmm0,xmm0 - vmovdqa64 ZMMWORD[rsp],zmm0 - vmovdqa64 ZMMWORD[64+rsp],zmm0 - vmovdqa64 ZMMWORD[128+rsp],zmm0 - vmovdqa64 ZMMWORD[192+rsp],zmm0 - vmovdqa64 ZMMWORD[256+rsp],zmm0 - vmovdqa64 ZMMWORD[320+rsp],zmm0 - vmovdqa64 ZMMWORD[384+rsp],zmm0 - vmovdqa64 ZMMWORD[448+rsp],zmm0 - vmovdqa64 ZMMWORD[512+rsp],zmm0 - vmovdqa64 ZMMWORD[576+rsp],zmm0 - vmovdqa64 ZMMWORD[640+rsp],zmm0 - vmovdqa64 ZMMWORD[704+rsp],zmm0 -$L$skip_hkeys_cleanup_9: - vzeroupper - vmovdqu xmm15,XMMWORD[((-16))+rbp] - vmovdqu xmm14,XMMWORD[((-32))+rbp] - vmovdqu xmm13,XMMWORD[((-48))+rbp] - vmovdqu xmm12,XMMWORD[((-64))+rbp] - vmovdqu xmm11,XMMWORD[((-80))+rbp] - vmovdqu xmm10,XMMWORD[((-96))+rbp] - vmovdqu xmm9,XMMWORD[((-112))+rbp] - vmovdqu xmm8,XMMWORD[((-128))+rbp] - vmovdqu xmm7,XMMWORD[((-144))+rbp] - vmovdqu xmm6,XMMWORD[((-160))+rbp] - lea rsp,[8+rbp] - pop rsi - - pop rdi - - pop r15 - - pop r14 - - pop r13 - - pop r12 - - pop rbp - - pop rbx - -$L$exit_update_aad: - DB 0F3h,0C3h ;repret -$L$ghash_seh_end: - - -global ossl_aes_gcm_encrypt_avx512 - -ALIGN 32 -ossl_aes_gcm_encrypt_avx512: - -$L$encrypt_seh_begin: -DB 243,15,30,250 - push rbx - -$L$encrypt_seh_push_rbx: - push rbp - -$L$encrypt_seh_push_rbp: - push r12 - -$L$encrypt_seh_push_r12: - push r13 - -$L$encrypt_seh_push_r13: - push r14 - -$L$encrypt_seh_push_r14: - push r15 - -$L$encrypt_seh_push_r15: - push rdi -$L$encrypt_seh_push_rdi: - push rsi -$L$encrypt_seh_push_rsi: - - sub rsp,168 -$L$encrypt_seh_allocstack_xmm: - - - - - - - - - - - lea rbp,[160+rsp] - -$L$encrypt_seh_setfp: - vmovdqu XMMWORD[rsp],xmm6 -$L$encrypt_seh_save_xmm6: - vmovdqu XMMWORD[16+rsp],xmm7 -$L$encrypt_seh_save_xmm7: - vmovdqu XMMWORD[32+rsp],xmm8 -$L$encrypt_seh_save_xmm8: - vmovdqu XMMWORD[48+rsp],xmm9 -$L$encrypt_seh_save_xmm9: - vmovdqu XMMWORD[64+rsp],xmm10 -$L$encrypt_seh_save_xmm10: - vmovdqu XMMWORD[80+rsp],xmm11 -$L$encrypt_seh_save_xmm11: - vmovdqu XMMWORD[96+rsp],xmm12 -$L$encrypt_seh_save_xmm12: - vmovdqu XMMWORD[112+rsp],xmm13 -$L$encrypt_seh_save_xmm13: - vmovdqu XMMWORD[128+rsp],xmm14 -$L$encrypt_seh_save_xmm14: - vmovdqu XMMWORD[144+rsp],xmm15 -$L$encrypt_seh_save_xmm15: - -$L$encrypt_seh_prolog_end: - sub rsp,1584 - and rsp,(-64) - - - mov eax,DWORD[240+rcx] - cmp eax,9 - je NEAR $L$aes_gcm_encrypt_128_avx512 - cmp eax,11 - je NEAR $L$aes_gcm_encrypt_192_avx512 - cmp eax,13 - je NEAR $L$aes_gcm_encrypt_256_avx512 - xor eax,eax - jmp NEAR $L$exit_gcm_encrypt -ALIGN 32 -$L$aes_gcm_encrypt_128_avx512: - cmp QWORD[112+rbp],0 - je NEAR $L$_enc_dec_done_10 - xor r14,r14 - vmovdqu64 xmm14,XMMWORD[64+rdx] - - mov r11,QWORD[r8] - or r11,r11 - je NEAR $L$_partial_block_done_11 - mov r10d,16 - lea r12,[byte_len_to_mask_table] - cmp QWORD[112+rbp],r10 - cmovc r10,QWORD[112+rbp] - add r12,r10 - add r12,r10 - kmovw k1,[r12] - vmovdqu8 xmm0{k1}{z},[r9] - - vmovdqu64 xmm3,XMMWORD[16+rdx] - vmovdqu64 xmm4,XMMWORD[336+rdx] - - - - lea r12,[SHIFT_MASK] - add r12,r11 - vmovdqu64 xmm5,XMMWORD[r12] - vpshufb xmm3,xmm3,xmm5 - vpxorq xmm3,xmm3,xmm0 - - - mov r13,QWORD[112+rbp] - add r13,r11 - sub r13,16 - jge NEAR $L$_no_extra_mask_11 - sub r12,r13 -$L$_no_extra_mask_11: - - - - vmovdqu64 xmm0,XMMWORD[16+r12] - vpand xmm3,xmm3,xmm0 - vpshufb xmm3,xmm3,XMMWORD[SHUF_MASK] - vpshufb xmm3,xmm3,xmm5 - vpxorq xmm14,xmm14,xmm3 - cmp r13,0 - jl NEAR $L$_partial_incomplete_11 - - vpclmulqdq xmm7,xmm14,xmm4,0x11 - vpclmulqdq xmm10,xmm14,xmm4,0x00 - vpclmulqdq xmm11,xmm14,xmm4,0x01 - vpclmulqdq xmm14,xmm14,xmm4,0x10 - vpxorq xmm14,xmm14,xmm11 - - vpsrldq xmm11,xmm14,8 - vpslldq xmm14,xmm14,8 - vpxorq xmm7,xmm7,xmm11 - vpxorq xmm14,xmm14,xmm10 - - - - vmovdqu64 xmm11,XMMWORD[POLY2] - - vpclmulqdq xmm10,xmm11,xmm14,0x01 - vpslldq xmm10,xmm10,8 - vpxorq xmm14,xmm14,xmm10 - - - - vpclmulqdq xmm10,xmm11,xmm14,0x00 - vpsrldq xmm10,xmm10,4 - vpclmulqdq xmm14,xmm11,xmm14,0x10 - vpslldq xmm14,xmm14,4 - - vpternlogq xmm14,xmm7,xmm10,0x96 - - mov QWORD[r8],0 - - mov r12,r11 - mov r11,16 - sub r11,r12 - jmp NEAR $L$_enc_dec_done_11 - -$L$_partial_incomplete_11: - mov r12,QWORD[112+rbp] - add QWORD[r8],r12 - mov r11,QWORD[112+rbp] - -$L$_enc_dec_done_11: - - - lea r12,[byte_len_to_mask_table] - kmovw k1,[r11*2+r12] - vmovdqu64 XMMWORD[64+rdx],xmm14 - - vpshufb xmm3,xmm3,XMMWORD[SHUF_MASK] - vpshufb xmm3,xmm3,xmm5 - mov r12,QWORD[120+rbp] - vmovdqu8 XMMWORD[r12]{k1},xmm3 -$L$_partial_block_done_11: - vmovdqu64 xmm2,XMMWORD[rdx] - mov r13,QWORD[112+rbp] - sub r13,r11 - je NEAR $L$_enc_dec_done_10 - cmp r13,256 - jbe NEAR $L$_message_below_equal_16_blocks_10 - - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vmovdqa64 zmm27,ZMMWORD[ddq_addbe_4444] - vmovdqa64 zmm28,ZMMWORD[ddq_addbe_1234] - - - - - - - vmovd r15d,xmm2 - and r15d,255 - - vshufi64x2 zmm2,zmm2,zmm2,0 - vpshufb zmm2,zmm2,zmm29 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_12 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_12 -$L$_next_16_overflow_12: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_12: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[192+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm12 - - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm7 - vmovdqa64 ZMMWORD[832+rsp],zmm10 - vmovdqa64 ZMMWORD[896+rsp],zmm11 - vmovdqa64 ZMMWORD[960+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_13 - - vmovdqu64 zmm0,ZMMWORD[288+rdx] - vmovdqu64 ZMMWORD[704+rsp],zmm0 - - vmovdqu64 zmm3,ZMMWORD[224+rdx] - vmovdqu64 ZMMWORD[640+rsp],zmm3 - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[160+rdx] - vmovdqu64 ZMMWORD[576+rsp],zmm4 - - vmovdqu64 zmm5,ZMMWORD[96+rdx] - vmovdqu64 ZMMWORD[512+rsp],zmm5 -$L$_skip_hkeys_precomputation_13: - cmp r13,512 - jb NEAR $L$_message_below_32_blocks_10 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_14 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_14 -$L$_next_16_overflow_14: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_14: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[448+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm12 - - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm7 - vmovdqa64 ZMMWORD[1088+rsp],zmm10 - vmovdqa64 ZMMWORD[1152+rsp],zmm11 - vmovdqa64 ZMMWORD[1216+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_15 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[192+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[128+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[64+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[rsp],zmm5 -$L$_skip_hkeys_precomputation_15: - mov r14,1 - add r11,512 - sub r13,512 - - cmp r13,768 - jb NEAR $L$_no_more_big_nblocks_10 -$L$_encrypt_big_nblocks_10: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_16 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_16 -$L$_16_blocks_overflow_16: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_16: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_17 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_17 -$L$_16_blocks_overflow_17: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_17: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_18 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_18 -$L$_16_blocks_overflow_18: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_18: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[512+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[576+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[640+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[704+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - - - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpternlogq zmm6,zmm12,zmm15,0x96 - vpxorq zmm6,zmm6,zmm24 - vpternlogq zmm7,zmm13,zmm10,0x96 - vpxorq zmm7,zmm7,zmm25 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vextracti64x4 ymm12,zmm6,1 - vpxorq ymm6,ymm6,ymm12 - vextracti32x4 xmm12,ymm6,1 - vpxorq xmm6,xmm6,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm6,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[512+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[576+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[640+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[704+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm0 - vmovdqa64 ZMMWORD[1088+rsp],zmm3 - vmovdqa64 ZMMWORD[1152+rsp],zmm4 - vmovdqa64 ZMMWORD[1216+rsp],zmm5 - vmovdqa64 zmm14,zmm6 - - add r11,768 - sub r13,768 - cmp r13,768 - jae NEAR $L$_encrypt_big_nblocks_10 - -$L$_no_more_big_nblocks_10: - - cmp r13,512 - jae NEAR $L$_encrypt_32_blocks_10 - - cmp r13,256 - jae NEAR $L$_encrypt_16_blocks_10 -$L$_encrypt_0_blocks_ghash_32_10: - mov r10d,r13d - and r10d,~15 - mov ebx,256 - sub ebx,r10d - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - add ebx,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_19 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_19 - jb NEAR $L$_last_num_blocks_is_7_1_19 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_19 - jb NEAR $L$_last_num_blocks_is_11_9_19 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_19 - ja NEAR $L$_last_num_blocks_is_16_19 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_19 - jmp NEAR $L$_last_num_blocks_is_13_19 - -$L$_last_num_blocks_is_11_9_19: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_19 - ja NEAR $L$_last_num_blocks_is_11_19 - jmp NEAR $L$_last_num_blocks_is_9_19 - -$L$_last_num_blocks_is_7_1_19: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_19 - jb NEAR $L$_last_num_blocks_is_3_1_19 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_19 - je NEAR $L$_last_num_blocks_is_6_19 - jmp NEAR $L$_last_num_blocks_is_5_19 - -$L$_last_num_blocks_is_3_1_19: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_19 - je NEAR $L$_last_num_blocks_is_2_19 -$L$_last_num_blocks_is_1_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_20 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_20 - -$L$_16_blocks_overflow_20: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_20: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_21 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_21 -$L$_small_initial_partial_block_21: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_21 -$L$_small_initial_compute_done_21: -$L$_after_reduction_21: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_2_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_22 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_22 - -$L$_16_blocks_overflow_22: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_22: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_23 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_23 -$L$_small_initial_partial_block_23: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_23: - - or r13,r13 - je NEAR $L$_after_reduction_23 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_23: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_3_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_24 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_24 - -$L$_16_blocks_overflow_24: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_24: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_25 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_25 -$L$_small_initial_partial_block_25: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_25: - - or r13,r13 - je NEAR $L$_after_reduction_25 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_25: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_4_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_26 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_26 - -$L$_16_blocks_overflow_26: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_26: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_27 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_27 -$L$_small_initial_partial_block_27: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_27: - - or r13,r13 - je NEAR $L$_after_reduction_27 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_27: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_5_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_28 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_28 - -$L$_16_blocks_overflow_28: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_28: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_29 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_29 -$L$_small_initial_partial_block_29: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_29: - - or r13,r13 - je NEAR $L$_after_reduction_29 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_29: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_6_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_30 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_30 - -$L$_16_blocks_overflow_30: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_30: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_31 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_31 -$L$_small_initial_partial_block_31: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_31: - - or r13,r13 - je NEAR $L$_after_reduction_31 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_31: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_7_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_32 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_32 - -$L$_16_blocks_overflow_32: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_32: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_33 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_33 -$L$_small_initial_partial_block_33: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_33: - - or r13,r13 - je NEAR $L$_after_reduction_33 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_33: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_8_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_34 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_34 - -$L$_16_blocks_overflow_34: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_34: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_35 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_35 -$L$_small_initial_partial_block_35: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_35: - - or r13,r13 - je NEAR $L$_after_reduction_35 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_35: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_9_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_36 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_36 - -$L$_16_blocks_overflow_36: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_36: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_37 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_37 -$L$_small_initial_partial_block_37: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_37: - - or r13,r13 - je NEAR $L$_after_reduction_37 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_37: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_10_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_38 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_38 - -$L$_16_blocks_overflow_38: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_38: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_39 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_39 -$L$_small_initial_partial_block_39: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_39: - - or r13,r13 - je NEAR $L$_after_reduction_39 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_39: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_11_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_40 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_40 - -$L$_16_blocks_overflow_40: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_40: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_41 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_41 -$L$_small_initial_partial_block_41: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_41: - - or r13,r13 - je NEAR $L$_after_reduction_41 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_41: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_12_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_42 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_42 - -$L$_16_blocks_overflow_42: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_42: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_43 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_43 -$L$_small_initial_partial_block_43: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_43: - - or r13,r13 - je NEAR $L$_after_reduction_43 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_43: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_13_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_44 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_44 - -$L$_16_blocks_overflow_44: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_44: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_45 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_45 -$L$_small_initial_partial_block_45: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_45: - - or r13,r13 - je NEAR $L$_after_reduction_45 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_45: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_14_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_46 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_46 - -$L$_16_blocks_overflow_46: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_46: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_47 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_47 -$L$_small_initial_partial_block_47: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_47: - - or r13,r13 - je NEAR $L$_after_reduction_47 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_47: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_15_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_48 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_48 - -$L$_16_blocks_overflow_48: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_48: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_49 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_49 -$L$_small_initial_partial_block_49: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_49: - - or r13,r13 - je NEAR $L$_after_reduction_49 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_49: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_16_19: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_50 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_50 - -$L$_16_blocks_overflow_50: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_50: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_51: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_51: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_51: - jmp NEAR $L$_last_blocks_done_19 -$L$_last_num_blocks_is_0_19: - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_19: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_10 -$L$_encrypt_32_blocks_10: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_52 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_52 -$L$_16_blocks_overflow_52: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_52: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_53 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_53 -$L$_16_blocks_overflow_53: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_53: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - - sub r13,512 - add r11,512 - mov r10d,r13d - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_54 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_54 - jb NEAR $L$_last_num_blocks_is_7_1_54 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_54 - jb NEAR $L$_last_num_blocks_is_11_9_54 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_54 - ja NEAR $L$_last_num_blocks_is_16_54 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_54 - jmp NEAR $L$_last_num_blocks_is_13_54 - -$L$_last_num_blocks_is_11_9_54: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_54 - ja NEAR $L$_last_num_blocks_is_11_54 - jmp NEAR $L$_last_num_blocks_is_9_54 - -$L$_last_num_blocks_is_7_1_54: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_54 - jb NEAR $L$_last_num_blocks_is_3_1_54 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_54 - je NEAR $L$_last_num_blocks_is_6_54 - jmp NEAR $L$_last_num_blocks_is_5_54 - -$L$_last_num_blocks_is_3_1_54: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_54 - je NEAR $L$_last_num_blocks_is_2_54 -$L$_last_num_blocks_is_1_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_55 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_55 - -$L$_16_blocks_overflow_55: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_55: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_56 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_56 -$L$_small_initial_partial_block_56: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_56 -$L$_small_initial_compute_done_56: -$L$_after_reduction_56: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_2_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_57 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_57 - -$L$_16_blocks_overflow_57: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_57: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_58 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_58 -$L$_small_initial_partial_block_58: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_58: - - or r13,r13 - je NEAR $L$_after_reduction_58 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_58: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_3_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_59 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_59 - -$L$_16_blocks_overflow_59: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_59: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_60 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_60 -$L$_small_initial_partial_block_60: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_60: - - or r13,r13 - je NEAR $L$_after_reduction_60 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_60: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_4_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_61 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_61 - -$L$_16_blocks_overflow_61: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_61: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_62 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_62 -$L$_small_initial_partial_block_62: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_62: - - or r13,r13 - je NEAR $L$_after_reduction_62 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_62: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_5_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_63 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_63 - -$L$_16_blocks_overflow_63: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_63: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_64 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_64 -$L$_small_initial_partial_block_64: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_64: - - or r13,r13 - je NEAR $L$_after_reduction_64 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_64: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_6_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_65 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_65 - -$L$_16_blocks_overflow_65: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_65: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_66 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_66 -$L$_small_initial_partial_block_66: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_66: - - or r13,r13 - je NEAR $L$_after_reduction_66 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_66: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_7_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_67 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_67 - -$L$_16_blocks_overflow_67: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_67: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_68 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_68 -$L$_small_initial_partial_block_68: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_68: - - or r13,r13 - je NEAR $L$_after_reduction_68 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_68: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_8_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_69 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_69 - -$L$_16_blocks_overflow_69: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_69: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_70 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_70 -$L$_small_initial_partial_block_70: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_70: - - or r13,r13 - je NEAR $L$_after_reduction_70 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_70: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_9_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_71 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_71 - -$L$_16_blocks_overflow_71: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_71: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_72 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_72 -$L$_small_initial_partial_block_72: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_72: - - or r13,r13 - je NEAR $L$_after_reduction_72 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_72: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_10_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_73 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_73 - -$L$_16_blocks_overflow_73: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_73: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_74 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_74 -$L$_small_initial_partial_block_74: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_74: - - or r13,r13 - je NEAR $L$_after_reduction_74 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_74: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_11_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_75 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_75 - -$L$_16_blocks_overflow_75: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_75: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_76 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_76 -$L$_small_initial_partial_block_76: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_76: - - or r13,r13 - je NEAR $L$_after_reduction_76 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_76: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_12_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_77 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_77 - -$L$_16_blocks_overflow_77: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_77: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_78 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_78 -$L$_small_initial_partial_block_78: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_78: - - or r13,r13 - je NEAR $L$_after_reduction_78 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_78: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_13_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_79 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_79 - -$L$_16_blocks_overflow_79: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_79: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_80 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_80 -$L$_small_initial_partial_block_80: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_80: - - or r13,r13 - je NEAR $L$_after_reduction_80 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_80: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_14_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_81 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_81 - -$L$_16_blocks_overflow_81: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_81: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_82 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_82 -$L$_small_initial_partial_block_82: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_82: - - or r13,r13 - je NEAR $L$_after_reduction_82 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_82: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_15_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_83 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_83 - -$L$_16_blocks_overflow_83: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_83: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_84 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_84 -$L$_small_initial_partial_block_84: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_84: - - or r13,r13 - je NEAR $L$_after_reduction_84 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_84: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_16_54: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_85 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_85 - -$L$_16_blocks_overflow_85: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_85: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_86: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_86: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_86: - jmp NEAR $L$_last_blocks_done_54 -$L$_last_num_blocks_is_0_54: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_54: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_10 -$L$_encrypt_16_blocks_10: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_87 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_87 -$L$_16_blocks_overflow_87: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_87: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[256+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[320+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[384+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[448+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - sub r13,256 - add r11,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_88 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_88 - jb NEAR $L$_last_num_blocks_is_7_1_88 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_88 - jb NEAR $L$_last_num_blocks_is_11_9_88 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_88 - ja NEAR $L$_last_num_blocks_is_16_88 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_88 - jmp NEAR $L$_last_num_blocks_is_13_88 - -$L$_last_num_blocks_is_11_9_88: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_88 - ja NEAR $L$_last_num_blocks_is_11_88 - jmp NEAR $L$_last_num_blocks_is_9_88 - -$L$_last_num_blocks_is_7_1_88: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_88 - jb NEAR $L$_last_num_blocks_is_3_1_88 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_88 - je NEAR $L$_last_num_blocks_is_6_88 - jmp NEAR $L$_last_num_blocks_is_5_88 - -$L$_last_num_blocks_is_3_1_88: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_88 - je NEAR $L$_last_num_blocks_is_2_88 -$L$_last_num_blocks_is_1_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_89 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_89 - -$L$_16_blocks_overflow_89: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_89: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc xmm0,xmm0,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_90 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_90 -$L$_small_initial_partial_block_90: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_90 -$L$_small_initial_compute_done_90: -$L$_after_reduction_90: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_2_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_91 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_91 - -$L$_16_blocks_overflow_91: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_91: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc ymm0,ymm0,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_92 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_92 -$L$_small_initial_partial_block_92: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_92: - - or r13,r13 - je NEAR $L$_after_reduction_92 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_92: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_3_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_93 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_93 - -$L$_16_blocks_overflow_93: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_93: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_94 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_94 -$L$_small_initial_partial_block_94: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_94: - - or r13,r13 - je NEAR $L$_after_reduction_94 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_94: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_4_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_95 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_95 - -$L$_16_blocks_overflow_95: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_95: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_96 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_96 -$L$_small_initial_partial_block_96: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_96: - - or r13,r13 - je NEAR $L$_after_reduction_96 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_96: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_5_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_97 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_97 - -$L$_16_blocks_overflow_97: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_97: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_98 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_98 -$L$_small_initial_partial_block_98: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_98: - - or r13,r13 - je NEAR $L$_after_reduction_98 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_98: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_6_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_99 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_99 - -$L$_16_blocks_overflow_99: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_99: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_100 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_100 -$L$_small_initial_partial_block_100: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_100: - - or r13,r13 - je NEAR $L$_after_reduction_100 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_100: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_7_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_101 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_101 - -$L$_16_blocks_overflow_101: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_101: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_102 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_102 -$L$_small_initial_partial_block_102: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_102: - - or r13,r13 - je NEAR $L$_after_reduction_102 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_102: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_8_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_103 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_103 - -$L$_16_blocks_overflow_103: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_103: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_104 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_104 -$L$_small_initial_partial_block_104: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_104: - - or r13,r13 - je NEAR $L$_after_reduction_104 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_104: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_9_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_105 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_105 - -$L$_16_blocks_overflow_105: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_105: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_106 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_106 -$L$_small_initial_partial_block_106: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_106: - - or r13,r13 - je NEAR $L$_after_reduction_106 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_106: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_10_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_107 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_107 - -$L$_16_blocks_overflow_107: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_107: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_108 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_108 -$L$_small_initial_partial_block_108: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_108: - - or r13,r13 - je NEAR $L$_after_reduction_108 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_108: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_11_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_109 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_109 - -$L$_16_blocks_overflow_109: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_109: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_110 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_110 -$L$_small_initial_partial_block_110: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_110: - - or r13,r13 - je NEAR $L$_after_reduction_110 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_110: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_12_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_111 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_111 - -$L$_16_blocks_overflow_111: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_111: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_112 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_112 -$L$_small_initial_partial_block_112: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_112: - - or r13,r13 - je NEAR $L$_after_reduction_112 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_112: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_13_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_113 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_113 - -$L$_16_blocks_overflow_113: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_113: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_114 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_114 -$L$_small_initial_partial_block_114: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_114: - - or r13,r13 - je NEAR $L$_after_reduction_114 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_114: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_14_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_115 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_115 - -$L$_16_blocks_overflow_115: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_115: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_116 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_116 -$L$_small_initial_partial_block_116: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_116: - - or r13,r13 - je NEAR $L$_after_reduction_116 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_116: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_15_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_117 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_117 - -$L$_16_blocks_overflow_117: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_117: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_118 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_118 -$L$_small_initial_partial_block_118: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_118: - - or r13,r13 - je NEAR $L$_after_reduction_118 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_118: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_16_88: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_119 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_119 - -$L$_16_blocks_overflow_119: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_119: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_120: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_120: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_120: - jmp NEAR $L$_last_blocks_done_88 -$L$_last_num_blocks_is_0_88: - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_88: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_10 - -$L$_message_below_32_blocks_10: - - - sub r13,256 - add r11,256 - mov r10d,r13d - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_121 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 -$L$_skip_hkeys_precomputation_121: - mov r14,1 - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_122 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_122 - jb NEAR $L$_last_num_blocks_is_7_1_122 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_122 - jb NEAR $L$_last_num_blocks_is_11_9_122 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_122 - ja NEAR $L$_last_num_blocks_is_16_122 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_122 - jmp NEAR $L$_last_num_blocks_is_13_122 - -$L$_last_num_blocks_is_11_9_122: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_122 - ja NEAR $L$_last_num_blocks_is_11_122 - jmp NEAR $L$_last_num_blocks_is_9_122 - -$L$_last_num_blocks_is_7_1_122: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_122 - jb NEAR $L$_last_num_blocks_is_3_1_122 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_122 - je NEAR $L$_last_num_blocks_is_6_122 - jmp NEAR $L$_last_num_blocks_is_5_122 - -$L$_last_num_blocks_is_3_1_122: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_122 - je NEAR $L$_last_num_blocks_is_2_122 -$L$_last_num_blocks_is_1_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_123 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_123 - -$L$_16_blocks_overflow_123: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_123: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_124 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_124 -$L$_small_initial_partial_block_124: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_124 -$L$_small_initial_compute_done_124: -$L$_after_reduction_124: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_2_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_125 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_125 - -$L$_16_blocks_overflow_125: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_125: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_126 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_126 -$L$_small_initial_partial_block_126: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_126: - - or r13,r13 - je NEAR $L$_after_reduction_126 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_126: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_3_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_127 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_127 - -$L$_16_blocks_overflow_127: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_127: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_128 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_128 -$L$_small_initial_partial_block_128: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_128: - - or r13,r13 - je NEAR $L$_after_reduction_128 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_128: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_4_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_129 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_129 - -$L$_16_blocks_overflow_129: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_129: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_130 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_130 -$L$_small_initial_partial_block_130: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_130: - - or r13,r13 - je NEAR $L$_after_reduction_130 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_130: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_5_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_131 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_131 - -$L$_16_blocks_overflow_131: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_131: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_132 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_132 -$L$_small_initial_partial_block_132: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_132: - - or r13,r13 - je NEAR $L$_after_reduction_132 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_132: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_6_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_133 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_133 - -$L$_16_blocks_overflow_133: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_133: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_134 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_134 -$L$_small_initial_partial_block_134: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_134: - - or r13,r13 - je NEAR $L$_after_reduction_134 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_134: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_7_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_135 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_135 - -$L$_16_blocks_overflow_135: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_135: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_136 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_136 -$L$_small_initial_partial_block_136: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_136: - - or r13,r13 - je NEAR $L$_after_reduction_136 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_136: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_8_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_137 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_137 - -$L$_16_blocks_overflow_137: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_137: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_138 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_138 -$L$_small_initial_partial_block_138: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_138: - - or r13,r13 - je NEAR $L$_after_reduction_138 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_138: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_9_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_139 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_139 - -$L$_16_blocks_overflow_139: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_139: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_140 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_140 -$L$_small_initial_partial_block_140: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_140: - - or r13,r13 - je NEAR $L$_after_reduction_140 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_140: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_10_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_141 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_141 - -$L$_16_blocks_overflow_141: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_141: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_142 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_142 -$L$_small_initial_partial_block_142: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_142: - - or r13,r13 - je NEAR $L$_after_reduction_142 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_142: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_11_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_143 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_143 - -$L$_16_blocks_overflow_143: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_143: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_144 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_144 -$L$_small_initial_partial_block_144: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_144: - - or r13,r13 - je NEAR $L$_after_reduction_144 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_144: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_12_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_145 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_145 - -$L$_16_blocks_overflow_145: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_145: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_146 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_146 -$L$_small_initial_partial_block_146: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_146: - - or r13,r13 - je NEAR $L$_after_reduction_146 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_146: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_13_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_147 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_147 - -$L$_16_blocks_overflow_147: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_147: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_148 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_148 -$L$_small_initial_partial_block_148: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_148: - - or r13,r13 - je NEAR $L$_after_reduction_148 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_148: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_14_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_149 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_149 - -$L$_16_blocks_overflow_149: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_149: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_150 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_150 -$L$_small_initial_partial_block_150: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_150: - - or r13,r13 - je NEAR $L$_after_reduction_150 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_150: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_15_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_151 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_151 - -$L$_16_blocks_overflow_151: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_151: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_152 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_152 -$L$_small_initial_partial_block_152: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_152: - - or r13,r13 - je NEAR $L$_after_reduction_152 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_152: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_16_122: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_153 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_153 - -$L$_16_blocks_overflow_153: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_153: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_154: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_154: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_154: - jmp NEAR $L$_last_blocks_done_122 -$L$_last_num_blocks_is_0_122: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_122: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_10 - -$L$_message_below_equal_16_blocks_10: - - - mov r12d,r13d - add r12d,15 - shr r12d,4 - cmp r12,8 - je NEAR $L$_small_initial_num_blocks_is_8_155 - jl NEAR $L$_small_initial_num_blocks_is_7_1_155 - - - cmp r12,12 - je NEAR $L$_small_initial_num_blocks_is_12_155 - jl NEAR $L$_small_initial_num_blocks_is_11_9_155 - - - cmp r12,16 - je NEAR $L$_small_initial_num_blocks_is_16_155 - cmp r12,15 - je NEAR $L$_small_initial_num_blocks_is_15_155 - cmp r12,14 - je NEAR $L$_small_initial_num_blocks_is_14_155 - jmp NEAR $L$_small_initial_num_blocks_is_13_155 - -$L$_small_initial_num_blocks_is_11_9_155: - - cmp r12,11 - je NEAR $L$_small_initial_num_blocks_is_11_155 - cmp r12,10 - je NEAR $L$_small_initial_num_blocks_is_10_155 - jmp NEAR $L$_small_initial_num_blocks_is_9_155 - -$L$_small_initial_num_blocks_is_7_1_155: - cmp r12,4 - je NEAR $L$_small_initial_num_blocks_is_4_155 - jl NEAR $L$_small_initial_num_blocks_is_3_1_155 - - cmp r12,7 - je NEAR $L$_small_initial_num_blocks_is_7_155 - cmp r12,6 - je NEAR $L$_small_initial_num_blocks_is_6_155 - jmp NEAR $L$_small_initial_num_blocks_is_5_155 - -$L$_small_initial_num_blocks_is_3_1_155: - - cmp r12,3 - je NEAR $L$_small_initial_num_blocks_is_3_155 - cmp r12,2 - je NEAR $L$_small_initial_num_blocks_is_2_155 - - - - - -$L$_small_initial_num_blocks_is_1_155: - vmovdqa64 xmm29,XMMWORD[SHUF_MASK] - vpaddd xmm0,xmm2,XMMWORD[ONE] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,0 - vpshufb xmm0,xmm0,xmm29 - vmovdqu8 xmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast xmm0,xmm0,xmm15 - vpxorq xmm0,xmm0,xmm6 - vextracti32x4 xmm12,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm6,xmm0,xmm29 - vextracti32x4 xmm13,zmm6,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_156 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_156 -$L$_small_initial_partial_block_156: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm13 - - jmp NEAR $L$_after_reduction_156 -$L$_small_initial_compute_done_156: -$L$_after_reduction_156: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_2_155: - vmovdqa64 ymm29,YMMWORD[SHUF_MASK] - vshufi64x2 ymm0,ymm2,ymm2,0 - vpaddd ymm0,ymm0,YMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,1 - vpshufb ymm0,ymm0,ymm29 - vmovdqu8 ymm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast ymm0,ymm0,ymm15 - vpxorq ymm0,ymm0,ymm6 - vextracti32x4 xmm12,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm6,ymm0,ymm29 - vextracti32x4 xmm13,zmm6,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_157 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_157 -$L$_small_initial_partial_block_157: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_157: - - or r13,r13 - je NEAR $L$_after_reduction_157 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_157: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_3_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,2 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm0,zmm29 - vextracti32x4 xmm13,zmm6,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_158 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_158 -$L$_small_initial_partial_block_158: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_158: - - or r13,r13 - je NEAR $L$_after_reduction_158 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_158: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_4_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,3 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm0,zmm29 - vextracti32x4 xmm13,zmm6,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_159 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_159 -$L$_small_initial_partial_block_159: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_159: - - or r13,r13 - je NEAR $L$_after_reduction_159 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_159: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_5_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 xmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast xmm3,xmm3,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq xmm3,xmm3,xmm7 - vextracti32x4 xmm12,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb xmm7,xmm3,xmm29 - vextracti32x4 xmm13,zmm7,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_160 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_160 -$L$_small_initial_partial_block_160: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_160: - - or r13,r13 - je NEAR $L$_after_reduction_160 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_160: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_6_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 ymm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast ymm3,ymm3,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq ymm3,ymm3,ymm7 - vextracti32x4 xmm12,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb ymm7,ymm3,ymm29 - vextracti32x4 xmm13,zmm7,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_161 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_161 -$L$_small_initial_partial_block_161: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_161: - - or r13,r13 - je NEAR $L$_after_reduction_161 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_161: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_7_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vextracti32x4 xmm13,zmm7,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_162 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_162 -$L$_small_initial_partial_block_162: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_162: - - or r13,r13 - je NEAR $L$_after_reduction_162 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_162: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_8_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vextracti32x4 xmm13,zmm7,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_163 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_163 -$L$_small_initial_partial_block_163: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_163: - - or r13,r13 - je NEAR $L$_after_reduction_163 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_163: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_9_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast xmm4,xmm4,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq xmm4,xmm4,xmm10 - vextracti32x4 xmm12,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb xmm10,xmm4,xmm29 - vextracti32x4 xmm13,zmm10,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_164 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_164 -$L$_small_initial_partial_block_164: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_164: - - or r13,r13 - je NEAR $L$_after_reduction_164 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_164: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_10_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast ymm4,ymm4,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq ymm4,ymm4,ymm10 - vextracti32x4 xmm12,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb ymm10,ymm4,ymm29 - vextracti32x4 xmm13,zmm10,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_165 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_165 -$L$_small_initial_partial_block_165: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_165: - - or r13,r13 - je NEAR $L$_after_reduction_165 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_165: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_11_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vextracti32x4 xmm13,zmm10,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_166 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_166 -$L$_small_initial_partial_block_166: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_166: - - or r13,r13 - je NEAR $L$_after_reduction_166 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_166: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_12_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vextracti32x4 xmm13,zmm10,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_167 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_167 -$L$_small_initial_partial_block_167: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_167: - - or r13,r13 - je NEAR $L$_after_reduction_167 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_167: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_13_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast xmm5,xmm5,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq xmm5,xmm5,xmm11 - vextracti32x4 xmm12,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb xmm11,xmm5,xmm29 - vextracti32x4 xmm13,zmm11,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_168 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_168 -$L$_small_initial_partial_block_168: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_168: - - or r13,r13 - je NEAR $L$_after_reduction_168 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_168: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_14_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast ymm5,ymm5,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq ymm5,ymm5,ymm11 - vextracti32x4 xmm12,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb ymm11,ymm5,ymm29 - vextracti32x4 xmm13,zmm11,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_169 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_169 -$L$_small_initial_partial_block_169: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_169: - - or r13,r13 - je NEAR $L$_after_reduction_169 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_169: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_15_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb zmm11,zmm5,zmm29 - vextracti32x4 xmm13,zmm11,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_170 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_170 -$L$_small_initial_partial_block_170: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_170: - - or r13,r13 - je NEAR $L$_after_reduction_170 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_170: - jmp NEAR $L$_small_initial_blocks_encrypted_155 -$L$_small_initial_num_blocks_is_16_155: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb zmm11,zmm5,zmm29 - vextracti32x4 xmm13,zmm11,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_171: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_171: - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_171: -$L$_small_initial_blocks_encrypted_155: -$L$_ghash_done_10: - vmovdqu64 XMMWORD[rdx],xmm2 - vmovdqu64 XMMWORD[64+rdx],xmm14 -$L$_enc_dec_done_10: - jmp NEAR $L$exit_gcm_encrypt -ALIGN 32 -$L$aes_gcm_encrypt_192_avx512: - cmp QWORD[112+rbp],0 - je NEAR $L$_enc_dec_done_172 - xor r14,r14 - vmovdqu64 xmm14,XMMWORD[64+rdx] - - mov r11,QWORD[r8] - or r11,r11 - je NEAR $L$_partial_block_done_173 - mov r10d,16 - lea r12,[byte_len_to_mask_table] - cmp QWORD[112+rbp],r10 - cmovc r10,QWORD[112+rbp] - add r12,r10 - add r12,r10 - kmovw k1,[r12] - vmovdqu8 xmm0{k1}{z},[r9] - - vmovdqu64 xmm3,XMMWORD[16+rdx] - vmovdqu64 xmm4,XMMWORD[336+rdx] - - - - lea r12,[SHIFT_MASK] - add r12,r11 - vmovdqu64 xmm5,XMMWORD[r12] - vpshufb xmm3,xmm3,xmm5 - vpxorq xmm3,xmm3,xmm0 - - - mov r13,QWORD[112+rbp] - add r13,r11 - sub r13,16 - jge NEAR $L$_no_extra_mask_173 - sub r12,r13 -$L$_no_extra_mask_173: - - - - vmovdqu64 xmm0,XMMWORD[16+r12] - vpand xmm3,xmm3,xmm0 - vpshufb xmm3,xmm3,XMMWORD[SHUF_MASK] - vpshufb xmm3,xmm3,xmm5 - vpxorq xmm14,xmm14,xmm3 - cmp r13,0 - jl NEAR $L$_partial_incomplete_173 - - vpclmulqdq xmm7,xmm14,xmm4,0x11 - vpclmulqdq xmm10,xmm14,xmm4,0x00 - vpclmulqdq xmm11,xmm14,xmm4,0x01 - vpclmulqdq xmm14,xmm14,xmm4,0x10 - vpxorq xmm14,xmm14,xmm11 - - vpsrldq xmm11,xmm14,8 - vpslldq xmm14,xmm14,8 - vpxorq xmm7,xmm7,xmm11 - vpxorq xmm14,xmm14,xmm10 - - - - vmovdqu64 xmm11,XMMWORD[POLY2] - - vpclmulqdq xmm10,xmm11,xmm14,0x01 - vpslldq xmm10,xmm10,8 - vpxorq xmm14,xmm14,xmm10 - - - - vpclmulqdq xmm10,xmm11,xmm14,0x00 - vpsrldq xmm10,xmm10,4 - vpclmulqdq xmm14,xmm11,xmm14,0x10 - vpslldq xmm14,xmm14,4 - - vpternlogq xmm14,xmm7,xmm10,0x96 - - mov QWORD[r8],0 - - mov r12,r11 - mov r11,16 - sub r11,r12 - jmp NEAR $L$_enc_dec_done_173 - -$L$_partial_incomplete_173: - mov r12,QWORD[112+rbp] - add QWORD[r8],r12 - mov r11,QWORD[112+rbp] - -$L$_enc_dec_done_173: - - - lea r12,[byte_len_to_mask_table] - kmovw k1,[r11*2+r12] - vmovdqu64 XMMWORD[64+rdx],xmm14 - - vpshufb xmm3,xmm3,XMMWORD[SHUF_MASK] - vpshufb xmm3,xmm3,xmm5 - mov r12,QWORD[120+rbp] - vmovdqu8 XMMWORD[r12]{k1},xmm3 -$L$_partial_block_done_173: - vmovdqu64 xmm2,XMMWORD[rdx] - mov r13,QWORD[112+rbp] - sub r13,r11 - je NEAR $L$_enc_dec_done_172 - cmp r13,256 - jbe NEAR $L$_message_below_equal_16_blocks_172 - - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vmovdqa64 zmm27,ZMMWORD[ddq_addbe_4444] - vmovdqa64 zmm28,ZMMWORD[ddq_addbe_1234] - - - - - - - vmovd r15d,xmm2 - and r15d,255 - - vshufi64x2 zmm2,zmm2,zmm2,0 - vpshufb zmm2,zmm2,zmm29 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_174 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_174 -$L$_next_16_overflow_174: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_174: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[192+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[176+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[192+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm12 - - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm7 - vmovdqa64 ZMMWORD[832+rsp],zmm10 - vmovdqa64 ZMMWORD[896+rsp],zmm11 - vmovdqa64 ZMMWORD[960+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_175 - - vmovdqu64 zmm0,ZMMWORD[288+rdx] - vmovdqu64 ZMMWORD[704+rsp],zmm0 - - vmovdqu64 zmm3,ZMMWORD[224+rdx] - vmovdqu64 ZMMWORD[640+rsp],zmm3 - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[160+rdx] - vmovdqu64 ZMMWORD[576+rsp],zmm4 - - vmovdqu64 zmm5,ZMMWORD[96+rdx] - vmovdqu64 ZMMWORD[512+rsp],zmm5 -$L$_skip_hkeys_precomputation_175: - cmp r13,512 - jb NEAR $L$_message_below_32_blocks_172 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_176 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_176 -$L$_next_16_overflow_176: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_176: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[448+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[176+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[192+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm12 - - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm7 - vmovdqa64 ZMMWORD[1088+rsp],zmm10 - vmovdqa64 ZMMWORD[1152+rsp],zmm11 - vmovdqa64 ZMMWORD[1216+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_177 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[192+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[128+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[64+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[rsp],zmm5 -$L$_skip_hkeys_precomputation_177: - mov r14,1 - add r11,512 - sub r13,512 - - cmp r13,768 - jb NEAR $L$_no_more_big_nblocks_172 -$L$_encrypt_big_nblocks_172: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_178 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_178 -$L$_16_blocks_overflow_178: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_178: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_179 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_179 -$L$_16_blocks_overflow_179: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_179: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_180 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_180 -$L$_16_blocks_overflow_180: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_180: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[512+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[576+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[640+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[704+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - - - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpternlogq zmm6,zmm12,zmm15,0x96 - vpxorq zmm6,zmm6,zmm24 - vpternlogq zmm7,zmm13,zmm10,0x96 - vpxorq zmm7,zmm7,zmm25 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vextracti64x4 ymm12,zmm6,1 - vpxorq ymm6,ymm6,ymm12 - vextracti32x4 xmm12,ymm6,1 - vpxorq xmm6,xmm6,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm6,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[512+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[576+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[640+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[704+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm0 - vmovdqa64 ZMMWORD[1088+rsp],zmm3 - vmovdqa64 ZMMWORD[1152+rsp],zmm4 - vmovdqa64 ZMMWORD[1216+rsp],zmm5 - vmovdqa64 zmm14,zmm6 - - add r11,768 - sub r13,768 - cmp r13,768 - jae NEAR $L$_encrypt_big_nblocks_172 - -$L$_no_more_big_nblocks_172: - - cmp r13,512 - jae NEAR $L$_encrypt_32_blocks_172 - - cmp r13,256 - jae NEAR $L$_encrypt_16_blocks_172 -$L$_encrypt_0_blocks_ghash_32_172: - mov r10d,r13d - and r10d,~15 - mov ebx,256 - sub ebx,r10d - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - add ebx,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_181 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_181 - jb NEAR $L$_last_num_blocks_is_7_1_181 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_181 - jb NEAR $L$_last_num_blocks_is_11_9_181 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_181 - ja NEAR $L$_last_num_blocks_is_16_181 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_181 - jmp NEAR $L$_last_num_blocks_is_13_181 - -$L$_last_num_blocks_is_11_9_181: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_181 - ja NEAR $L$_last_num_blocks_is_11_181 - jmp NEAR $L$_last_num_blocks_is_9_181 - -$L$_last_num_blocks_is_7_1_181: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_181 - jb NEAR $L$_last_num_blocks_is_3_1_181 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_181 - je NEAR $L$_last_num_blocks_is_6_181 - jmp NEAR $L$_last_num_blocks_is_5_181 - -$L$_last_num_blocks_is_3_1_181: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_181 - je NEAR $L$_last_num_blocks_is_2_181 -$L$_last_num_blocks_is_1_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_182 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_182 - -$L$_16_blocks_overflow_182: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_182: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_183 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_183 -$L$_small_initial_partial_block_183: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_183 -$L$_small_initial_compute_done_183: -$L$_after_reduction_183: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_2_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_184 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_184 - -$L$_16_blocks_overflow_184: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_184: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_185 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_185 -$L$_small_initial_partial_block_185: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_185: - - or r13,r13 - je NEAR $L$_after_reduction_185 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_185: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_3_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_186 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_186 - -$L$_16_blocks_overflow_186: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_186: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_187 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_187 -$L$_small_initial_partial_block_187: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_187: - - or r13,r13 - je NEAR $L$_after_reduction_187 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_187: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_4_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_188 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_188 - -$L$_16_blocks_overflow_188: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_188: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_189 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_189 -$L$_small_initial_partial_block_189: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_189: - - or r13,r13 - je NEAR $L$_after_reduction_189 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_189: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_5_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_190 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_190 - -$L$_16_blocks_overflow_190: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_190: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_191 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_191 -$L$_small_initial_partial_block_191: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_191: - - or r13,r13 - je NEAR $L$_after_reduction_191 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_191: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_6_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_192 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_192 - -$L$_16_blocks_overflow_192: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_192: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_193 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_193 -$L$_small_initial_partial_block_193: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_193: - - or r13,r13 - je NEAR $L$_after_reduction_193 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_193: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_7_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_194 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_194 - -$L$_16_blocks_overflow_194: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_194: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_195 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_195 -$L$_small_initial_partial_block_195: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_195: - - or r13,r13 - je NEAR $L$_after_reduction_195 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_195: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_8_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_196 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_196 - -$L$_16_blocks_overflow_196: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_196: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_197 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_197 -$L$_small_initial_partial_block_197: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_197: - - or r13,r13 - je NEAR $L$_after_reduction_197 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_197: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_9_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_198 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_198 - -$L$_16_blocks_overflow_198: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_198: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_199 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_199 -$L$_small_initial_partial_block_199: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_199: - - or r13,r13 - je NEAR $L$_after_reduction_199 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_199: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_10_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_200 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_200 - -$L$_16_blocks_overflow_200: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_200: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_201 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_201 -$L$_small_initial_partial_block_201: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_201: - - or r13,r13 - je NEAR $L$_after_reduction_201 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_201: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_11_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_202 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_202 - -$L$_16_blocks_overflow_202: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_202: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_203 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_203 -$L$_small_initial_partial_block_203: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_203: - - or r13,r13 - je NEAR $L$_after_reduction_203 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_203: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_12_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_204 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_204 - -$L$_16_blocks_overflow_204: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_204: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_205 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_205 -$L$_small_initial_partial_block_205: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_205: - - or r13,r13 - je NEAR $L$_after_reduction_205 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_205: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_13_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_206 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_206 - -$L$_16_blocks_overflow_206: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_206: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_207 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_207 -$L$_small_initial_partial_block_207: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_207: - - or r13,r13 - je NEAR $L$_after_reduction_207 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_207: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_14_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_208 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_208 - -$L$_16_blocks_overflow_208: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_208: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_209 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_209 -$L$_small_initial_partial_block_209: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_209: - - or r13,r13 - je NEAR $L$_after_reduction_209 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_209: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_15_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_210 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_210 - -$L$_16_blocks_overflow_210: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_210: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_211 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_211 -$L$_small_initial_partial_block_211: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_211: - - or r13,r13 - je NEAR $L$_after_reduction_211 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_211: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_16_181: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_212 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_212 - -$L$_16_blocks_overflow_212: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_212: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_213: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_213: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_213: - jmp NEAR $L$_last_blocks_done_181 -$L$_last_num_blocks_is_0_181: - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_181: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_172 -$L$_encrypt_32_blocks_172: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_214 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_214 -$L$_16_blocks_overflow_214: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_214: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_215 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_215 -$L$_16_blocks_overflow_215: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_215: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - - sub r13,512 - add r11,512 - mov r10d,r13d - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_216 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_216 - jb NEAR $L$_last_num_blocks_is_7_1_216 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_216 - jb NEAR $L$_last_num_blocks_is_11_9_216 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_216 - ja NEAR $L$_last_num_blocks_is_16_216 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_216 - jmp NEAR $L$_last_num_blocks_is_13_216 - -$L$_last_num_blocks_is_11_9_216: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_216 - ja NEAR $L$_last_num_blocks_is_11_216 - jmp NEAR $L$_last_num_blocks_is_9_216 - -$L$_last_num_blocks_is_7_1_216: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_216 - jb NEAR $L$_last_num_blocks_is_3_1_216 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_216 - je NEAR $L$_last_num_blocks_is_6_216 - jmp NEAR $L$_last_num_blocks_is_5_216 - -$L$_last_num_blocks_is_3_1_216: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_216 - je NEAR $L$_last_num_blocks_is_2_216 -$L$_last_num_blocks_is_1_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_217 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_217 - -$L$_16_blocks_overflow_217: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_217: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_218 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_218 -$L$_small_initial_partial_block_218: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_218 -$L$_small_initial_compute_done_218: -$L$_after_reduction_218: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_2_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_219 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_219 - -$L$_16_blocks_overflow_219: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_219: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_220 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_220 -$L$_small_initial_partial_block_220: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_220: - - or r13,r13 - je NEAR $L$_after_reduction_220 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_220: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_3_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_221 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_221 - -$L$_16_blocks_overflow_221: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_221: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_222 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_222 -$L$_small_initial_partial_block_222: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_222: - - or r13,r13 - je NEAR $L$_after_reduction_222 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_222: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_4_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_223 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_223 - -$L$_16_blocks_overflow_223: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_223: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_224 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_224 -$L$_small_initial_partial_block_224: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_224: - - or r13,r13 - je NEAR $L$_after_reduction_224 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_224: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_5_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_225 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_225 - -$L$_16_blocks_overflow_225: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_225: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_226 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_226 -$L$_small_initial_partial_block_226: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_226: - - or r13,r13 - je NEAR $L$_after_reduction_226 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_226: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_6_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_227 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_227 - -$L$_16_blocks_overflow_227: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_227: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_228 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_228 -$L$_small_initial_partial_block_228: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_228: - - or r13,r13 - je NEAR $L$_after_reduction_228 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_228: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_7_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_229 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_229 - -$L$_16_blocks_overflow_229: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_229: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_230 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_230 -$L$_small_initial_partial_block_230: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_230: - - or r13,r13 - je NEAR $L$_after_reduction_230 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_230: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_8_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_231 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_231 - -$L$_16_blocks_overflow_231: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_231: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_232 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_232 -$L$_small_initial_partial_block_232: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_232: - - or r13,r13 - je NEAR $L$_after_reduction_232 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_232: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_9_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_233 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_233 - -$L$_16_blocks_overflow_233: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_233: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_234 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_234 -$L$_small_initial_partial_block_234: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_234: - - or r13,r13 - je NEAR $L$_after_reduction_234 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_234: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_10_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_235 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_235 - -$L$_16_blocks_overflow_235: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_235: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_236 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_236 -$L$_small_initial_partial_block_236: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_236: - - or r13,r13 - je NEAR $L$_after_reduction_236 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_236: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_11_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_237 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_237 - -$L$_16_blocks_overflow_237: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_237: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_238 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_238 -$L$_small_initial_partial_block_238: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_238: - - or r13,r13 - je NEAR $L$_after_reduction_238 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_238: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_12_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_239 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_239 - -$L$_16_blocks_overflow_239: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_239: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_240 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_240 -$L$_small_initial_partial_block_240: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_240: - - or r13,r13 - je NEAR $L$_after_reduction_240 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_240: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_13_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_241 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_241 - -$L$_16_blocks_overflow_241: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_241: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_242 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_242 -$L$_small_initial_partial_block_242: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_242: - - or r13,r13 - je NEAR $L$_after_reduction_242 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_242: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_14_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_243 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_243 - -$L$_16_blocks_overflow_243: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_243: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_244 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_244 -$L$_small_initial_partial_block_244: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_244: - - or r13,r13 - je NEAR $L$_after_reduction_244 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_244: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_15_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_245 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_245 - -$L$_16_blocks_overflow_245: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_245: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_246 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_246 -$L$_small_initial_partial_block_246: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_246: - - or r13,r13 - je NEAR $L$_after_reduction_246 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_246: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_16_216: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_247 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_247 - -$L$_16_blocks_overflow_247: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_247: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_248: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_248: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_248: - jmp NEAR $L$_last_blocks_done_216 -$L$_last_num_blocks_is_0_216: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_216: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_172 -$L$_encrypt_16_blocks_172: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_249 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_249 -$L$_16_blocks_overflow_249: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_249: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[256+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[320+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[384+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[448+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - sub r13,256 - add r11,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_250 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_250 - jb NEAR $L$_last_num_blocks_is_7_1_250 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_250 - jb NEAR $L$_last_num_blocks_is_11_9_250 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_250 - ja NEAR $L$_last_num_blocks_is_16_250 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_250 - jmp NEAR $L$_last_num_blocks_is_13_250 - -$L$_last_num_blocks_is_11_9_250: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_250 - ja NEAR $L$_last_num_blocks_is_11_250 - jmp NEAR $L$_last_num_blocks_is_9_250 - -$L$_last_num_blocks_is_7_1_250: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_250 - jb NEAR $L$_last_num_blocks_is_3_1_250 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_250 - je NEAR $L$_last_num_blocks_is_6_250 - jmp NEAR $L$_last_num_blocks_is_5_250 - -$L$_last_num_blocks_is_3_1_250: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_250 - je NEAR $L$_last_num_blocks_is_2_250 -$L$_last_num_blocks_is_1_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_251 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_251 - -$L$_16_blocks_overflow_251: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_251: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc xmm0,xmm0,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_252 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_252 -$L$_small_initial_partial_block_252: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_252 -$L$_small_initial_compute_done_252: -$L$_after_reduction_252: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_2_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_253 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_253 - -$L$_16_blocks_overflow_253: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_253: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc ymm0,ymm0,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_254 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_254 -$L$_small_initial_partial_block_254: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_254: - - or r13,r13 - je NEAR $L$_after_reduction_254 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_254: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_3_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_255 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_255 - -$L$_16_blocks_overflow_255: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_255: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_256 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_256 -$L$_small_initial_partial_block_256: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_256: - - or r13,r13 - je NEAR $L$_after_reduction_256 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_256: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_4_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_257 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_257 - -$L$_16_blocks_overflow_257: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_257: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_258 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_258 -$L$_small_initial_partial_block_258: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_258: - - or r13,r13 - je NEAR $L$_after_reduction_258 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_258: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_5_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_259 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_259 - -$L$_16_blocks_overflow_259: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_259: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_260 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_260 -$L$_small_initial_partial_block_260: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_260: - - or r13,r13 - je NEAR $L$_after_reduction_260 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_260: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_6_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_261 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_261 - -$L$_16_blocks_overflow_261: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_261: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_262 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_262 -$L$_small_initial_partial_block_262: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_262: - - or r13,r13 - je NEAR $L$_after_reduction_262 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_262: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_7_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_263 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_263 - -$L$_16_blocks_overflow_263: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_263: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_264 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_264 -$L$_small_initial_partial_block_264: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_264: - - or r13,r13 - je NEAR $L$_after_reduction_264 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_264: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_8_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_265 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_265 - -$L$_16_blocks_overflow_265: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_265: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_266 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_266 -$L$_small_initial_partial_block_266: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_266: - - or r13,r13 - je NEAR $L$_after_reduction_266 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_266: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_9_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_267 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_267 - -$L$_16_blocks_overflow_267: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_267: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_268 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_268 -$L$_small_initial_partial_block_268: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_268: - - or r13,r13 - je NEAR $L$_after_reduction_268 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_268: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_10_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_269 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_269 - -$L$_16_blocks_overflow_269: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_269: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_270 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_270 -$L$_small_initial_partial_block_270: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_270: - - or r13,r13 - je NEAR $L$_after_reduction_270 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_270: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_11_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_271 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_271 - -$L$_16_blocks_overflow_271: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_271: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_272 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_272 -$L$_small_initial_partial_block_272: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_272: - - or r13,r13 - je NEAR $L$_after_reduction_272 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_272: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_12_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_273 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_273 - -$L$_16_blocks_overflow_273: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_273: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_274 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_274 -$L$_small_initial_partial_block_274: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_274: - - or r13,r13 - je NEAR $L$_after_reduction_274 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_274: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_13_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_275 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_275 - -$L$_16_blocks_overflow_275: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_275: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_276 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_276 -$L$_small_initial_partial_block_276: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_276: - - or r13,r13 - je NEAR $L$_after_reduction_276 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_276: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_14_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_277 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_277 - -$L$_16_blocks_overflow_277: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_277: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_278 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_278 -$L$_small_initial_partial_block_278: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_278: - - or r13,r13 - je NEAR $L$_after_reduction_278 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_278: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_15_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_279 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_279 - -$L$_16_blocks_overflow_279: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_279: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_280 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_280 -$L$_small_initial_partial_block_280: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_280: - - or r13,r13 - je NEAR $L$_after_reduction_280 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_280: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_16_250: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_281 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_281 - -$L$_16_blocks_overflow_281: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_281: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_282: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_282: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_282: - jmp NEAR $L$_last_blocks_done_250 -$L$_last_num_blocks_is_0_250: - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_250: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_172 - -$L$_message_below_32_blocks_172: - - - sub r13,256 - add r11,256 - mov r10d,r13d - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_283 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 -$L$_skip_hkeys_precomputation_283: - mov r14,1 - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_284 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_284 - jb NEAR $L$_last_num_blocks_is_7_1_284 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_284 - jb NEAR $L$_last_num_blocks_is_11_9_284 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_284 - ja NEAR $L$_last_num_blocks_is_16_284 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_284 - jmp NEAR $L$_last_num_blocks_is_13_284 - -$L$_last_num_blocks_is_11_9_284: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_284 - ja NEAR $L$_last_num_blocks_is_11_284 - jmp NEAR $L$_last_num_blocks_is_9_284 - -$L$_last_num_blocks_is_7_1_284: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_284 - jb NEAR $L$_last_num_blocks_is_3_1_284 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_284 - je NEAR $L$_last_num_blocks_is_6_284 - jmp NEAR $L$_last_num_blocks_is_5_284 - -$L$_last_num_blocks_is_3_1_284: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_284 - je NEAR $L$_last_num_blocks_is_2_284 -$L$_last_num_blocks_is_1_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_285 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_285 - -$L$_16_blocks_overflow_285: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_285: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_286 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_286 -$L$_small_initial_partial_block_286: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_286 -$L$_small_initial_compute_done_286: -$L$_after_reduction_286: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_2_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_287 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_287 - -$L$_16_blocks_overflow_287: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_287: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_288 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_288 -$L$_small_initial_partial_block_288: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_288: - - or r13,r13 - je NEAR $L$_after_reduction_288 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_288: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_3_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_289 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_289 - -$L$_16_blocks_overflow_289: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_289: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_290 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_290 -$L$_small_initial_partial_block_290: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_290: - - or r13,r13 - je NEAR $L$_after_reduction_290 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_290: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_4_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_291 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_291 - -$L$_16_blocks_overflow_291: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_291: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_292 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_292 -$L$_small_initial_partial_block_292: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_292: - - or r13,r13 - je NEAR $L$_after_reduction_292 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_292: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_5_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_293 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_293 - -$L$_16_blocks_overflow_293: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_293: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_294 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_294 -$L$_small_initial_partial_block_294: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_294: - - or r13,r13 - je NEAR $L$_after_reduction_294 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_294: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_6_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_295 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_295 - -$L$_16_blocks_overflow_295: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_295: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_296 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_296 -$L$_small_initial_partial_block_296: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_296: - - or r13,r13 - je NEAR $L$_after_reduction_296 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_296: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_7_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_297 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_297 - -$L$_16_blocks_overflow_297: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_297: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_298 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_298 -$L$_small_initial_partial_block_298: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_298: - - or r13,r13 - je NEAR $L$_after_reduction_298 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_298: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_8_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_299 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_299 - -$L$_16_blocks_overflow_299: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_299: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_300 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_300 -$L$_small_initial_partial_block_300: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_300: - - or r13,r13 - je NEAR $L$_after_reduction_300 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_300: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_9_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_301 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_301 - -$L$_16_blocks_overflow_301: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_301: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_302 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_302 -$L$_small_initial_partial_block_302: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_302: - - or r13,r13 - je NEAR $L$_after_reduction_302 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_302: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_10_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_303 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_303 - -$L$_16_blocks_overflow_303: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_303: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_304 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_304 -$L$_small_initial_partial_block_304: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_304: - - or r13,r13 - je NEAR $L$_after_reduction_304 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_304: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_11_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_305 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_305 - -$L$_16_blocks_overflow_305: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_305: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_306 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_306 -$L$_small_initial_partial_block_306: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_306: - - or r13,r13 - je NEAR $L$_after_reduction_306 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_306: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_12_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_307 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_307 - -$L$_16_blocks_overflow_307: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_307: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_308 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_308 -$L$_small_initial_partial_block_308: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_308: - - or r13,r13 - je NEAR $L$_after_reduction_308 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_308: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_13_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_309 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_309 - -$L$_16_blocks_overflow_309: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_309: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_310 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_310 -$L$_small_initial_partial_block_310: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_310: - - or r13,r13 - je NEAR $L$_after_reduction_310 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_310: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_14_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_311 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_311 - -$L$_16_blocks_overflow_311: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_311: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_312 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_312 -$L$_small_initial_partial_block_312: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_312: - - or r13,r13 - je NEAR $L$_after_reduction_312 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_312: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_15_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_313 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_313 - -$L$_16_blocks_overflow_313: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_313: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_314 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_314 -$L$_small_initial_partial_block_314: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_314: - - or r13,r13 - je NEAR $L$_after_reduction_314 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_314: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_16_284: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_315 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_315 - -$L$_16_blocks_overflow_315: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_315: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_316: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_316: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_316: - jmp NEAR $L$_last_blocks_done_284 -$L$_last_num_blocks_is_0_284: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_284: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_172 - -$L$_message_below_equal_16_blocks_172: - - - mov r12d,r13d - add r12d,15 - shr r12d,4 - cmp r12,8 - je NEAR $L$_small_initial_num_blocks_is_8_317 - jl NEAR $L$_small_initial_num_blocks_is_7_1_317 - - - cmp r12,12 - je NEAR $L$_small_initial_num_blocks_is_12_317 - jl NEAR $L$_small_initial_num_blocks_is_11_9_317 - - - cmp r12,16 - je NEAR $L$_small_initial_num_blocks_is_16_317 - cmp r12,15 - je NEAR $L$_small_initial_num_blocks_is_15_317 - cmp r12,14 - je NEAR $L$_small_initial_num_blocks_is_14_317 - jmp NEAR $L$_small_initial_num_blocks_is_13_317 - -$L$_small_initial_num_blocks_is_11_9_317: - - cmp r12,11 - je NEAR $L$_small_initial_num_blocks_is_11_317 - cmp r12,10 - je NEAR $L$_small_initial_num_blocks_is_10_317 - jmp NEAR $L$_small_initial_num_blocks_is_9_317 - -$L$_small_initial_num_blocks_is_7_1_317: - cmp r12,4 - je NEAR $L$_small_initial_num_blocks_is_4_317 - jl NEAR $L$_small_initial_num_blocks_is_3_1_317 - - cmp r12,7 - je NEAR $L$_small_initial_num_blocks_is_7_317 - cmp r12,6 - je NEAR $L$_small_initial_num_blocks_is_6_317 - jmp NEAR $L$_small_initial_num_blocks_is_5_317 - -$L$_small_initial_num_blocks_is_3_1_317: - - cmp r12,3 - je NEAR $L$_small_initial_num_blocks_is_3_317 - cmp r12,2 - je NEAR $L$_small_initial_num_blocks_is_2_317 - - - - - -$L$_small_initial_num_blocks_is_1_317: - vmovdqa64 xmm29,XMMWORD[SHUF_MASK] - vpaddd xmm0,xmm2,XMMWORD[ONE] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,0 - vpshufb xmm0,xmm0,xmm29 - vmovdqu8 xmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast xmm0,xmm0,xmm15 - vpxorq xmm0,xmm0,xmm6 - vextracti32x4 xmm12,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm6,xmm0,xmm29 - vextracti32x4 xmm13,zmm6,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_318 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_318 -$L$_small_initial_partial_block_318: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm13 - - jmp NEAR $L$_after_reduction_318 -$L$_small_initial_compute_done_318: -$L$_after_reduction_318: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_2_317: - vmovdqa64 ymm29,YMMWORD[SHUF_MASK] - vshufi64x2 ymm0,ymm2,ymm2,0 - vpaddd ymm0,ymm0,YMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,1 - vpshufb ymm0,ymm0,ymm29 - vmovdqu8 ymm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast ymm0,ymm0,ymm15 - vpxorq ymm0,ymm0,ymm6 - vextracti32x4 xmm12,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm6,ymm0,ymm29 - vextracti32x4 xmm13,zmm6,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_319 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_319 -$L$_small_initial_partial_block_319: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_319: - - or r13,r13 - je NEAR $L$_after_reduction_319 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_319: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_3_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,2 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm0,zmm29 - vextracti32x4 xmm13,zmm6,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_320 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_320 -$L$_small_initial_partial_block_320: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_320: - - or r13,r13 - je NEAR $L$_after_reduction_320 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_320: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_4_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,3 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm0,zmm29 - vextracti32x4 xmm13,zmm6,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_321 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_321 -$L$_small_initial_partial_block_321: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_321: - - or r13,r13 - je NEAR $L$_after_reduction_321 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_321: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_5_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 xmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast xmm3,xmm3,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq xmm3,xmm3,xmm7 - vextracti32x4 xmm12,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb xmm7,xmm3,xmm29 - vextracti32x4 xmm13,zmm7,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_322 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_322 -$L$_small_initial_partial_block_322: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_322: - - or r13,r13 - je NEAR $L$_after_reduction_322 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_322: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_6_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 ymm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast ymm3,ymm3,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq ymm3,ymm3,ymm7 - vextracti32x4 xmm12,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb ymm7,ymm3,ymm29 - vextracti32x4 xmm13,zmm7,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_323 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_323 -$L$_small_initial_partial_block_323: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_323: - - or r13,r13 - je NEAR $L$_after_reduction_323 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_323: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_7_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vextracti32x4 xmm13,zmm7,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_324 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_324 -$L$_small_initial_partial_block_324: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_324: - - or r13,r13 - je NEAR $L$_after_reduction_324 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_324: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_8_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vextracti32x4 xmm13,zmm7,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_325 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_325 -$L$_small_initial_partial_block_325: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_325: - - or r13,r13 - je NEAR $L$_after_reduction_325 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_325: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_9_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast xmm4,xmm4,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq xmm4,xmm4,xmm10 - vextracti32x4 xmm12,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb xmm10,xmm4,xmm29 - vextracti32x4 xmm13,zmm10,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_326 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_326 -$L$_small_initial_partial_block_326: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_326: - - or r13,r13 - je NEAR $L$_after_reduction_326 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_326: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_10_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast ymm4,ymm4,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq ymm4,ymm4,ymm10 - vextracti32x4 xmm12,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb ymm10,ymm4,ymm29 - vextracti32x4 xmm13,zmm10,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_327 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_327 -$L$_small_initial_partial_block_327: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_327: - - or r13,r13 - je NEAR $L$_after_reduction_327 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_327: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_11_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vextracti32x4 xmm13,zmm10,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_328 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_328 -$L$_small_initial_partial_block_328: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_328: - - or r13,r13 - je NEAR $L$_after_reduction_328 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_328: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_12_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vextracti32x4 xmm13,zmm10,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_329 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_329 -$L$_small_initial_partial_block_329: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_329: - - or r13,r13 - je NEAR $L$_after_reduction_329 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_329: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_13_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast xmm5,xmm5,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq xmm5,xmm5,xmm11 - vextracti32x4 xmm12,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb xmm11,xmm5,xmm29 - vextracti32x4 xmm13,zmm11,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_330 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_330 -$L$_small_initial_partial_block_330: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_330: - - or r13,r13 - je NEAR $L$_after_reduction_330 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_330: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_14_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast ymm5,ymm5,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq ymm5,ymm5,ymm11 - vextracti32x4 xmm12,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb ymm11,ymm5,ymm29 - vextracti32x4 xmm13,zmm11,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_331 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_331 -$L$_small_initial_partial_block_331: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_331: - - or r13,r13 - je NEAR $L$_after_reduction_331 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_331: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_15_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb zmm11,zmm5,zmm29 - vextracti32x4 xmm13,zmm11,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_332 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_332 -$L$_small_initial_partial_block_332: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_332: - - or r13,r13 - je NEAR $L$_after_reduction_332 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_332: - jmp NEAR $L$_small_initial_blocks_encrypted_317 -$L$_small_initial_num_blocks_is_16_317: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb zmm11,zmm5,zmm29 - vextracti32x4 xmm13,zmm11,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_333: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_333: - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_333: -$L$_small_initial_blocks_encrypted_317: -$L$_ghash_done_172: - vmovdqu64 XMMWORD[rdx],xmm2 - vmovdqu64 XMMWORD[64+rdx],xmm14 -$L$_enc_dec_done_172: - jmp NEAR $L$exit_gcm_encrypt -ALIGN 32 -$L$aes_gcm_encrypt_256_avx512: - cmp QWORD[112+rbp],0 - je NEAR $L$_enc_dec_done_334 - xor r14,r14 - vmovdqu64 xmm14,XMMWORD[64+rdx] - - mov r11,QWORD[r8] - or r11,r11 - je NEAR $L$_partial_block_done_335 - mov r10d,16 - lea r12,[byte_len_to_mask_table] - cmp QWORD[112+rbp],r10 - cmovc r10,QWORD[112+rbp] - add r12,r10 - add r12,r10 - kmovw k1,[r12] - vmovdqu8 xmm0{k1}{z},[r9] - - vmovdqu64 xmm3,XMMWORD[16+rdx] - vmovdqu64 xmm4,XMMWORD[336+rdx] - - - - lea r12,[SHIFT_MASK] - add r12,r11 - vmovdqu64 xmm5,XMMWORD[r12] - vpshufb xmm3,xmm3,xmm5 - vpxorq xmm3,xmm3,xmm0 - - - mov r13,QWORD[112+rbp] - add r13,r11 - sub r13,16 - jge NEAR $L$_no_extra_mask_335 - sub r12,r13 -$L$_no_extra_mask_335: - - - - vmovdqu64 xmm0,XMMWORD[16+r12] - vpand xmm3,xmm3,xmm0 - vpshufb xmm3,xmm3,XMMWORD[SHUF_MASK] - vpshufb xmm3,xmm3,xmm5 - vpxorq xmm14,xmm14,xmm3 - cmp r13,0 - jl NEAR $L$_partial_incomplete_335 - - vpclmulqdq xmm7,xmm14,xmm4,0x11 - vpclmulqdq xmm10,xmm14,xmm4,0x00 - vpclmulqdq xmm11,xmm14,xmm4,0x01 - vpclmulqdq xmm14,xmm14,xmm4,0x10 - vpxorq xmm14,xmm14,xmm11 - - vpsrldq xmm11,xmm14,8 - vpslldq xmm14,xmm14,8 - vpxorq xmm7,xmm7,xmm11 - vpxorq xmm14,xmm14,xmm10 - - - - vmovdqu64 xmm11,XMMWORD[POLY2] - - vpclmulqdq xmm10,xmm11,xmm14,0x01 - vpslldq xmm10,xmm10,8 - vpxorq xmm14,xmm14,xmm10 - - - - vpclmulqdq xmm10,xmm11,xmm14,0x00 - vpsrldq xmm10,xmm10,4 - vpclmulqdq xmm14,xmm11,xmm14,0x10 - vpslldq xmm14,xmm14,4 - - vpternlogq xmm14,xmm7,xmm10,0x96 - - mov QWORD[r8],0 - - mov r12,r11 - mov r11,16 - sub r11,r12 - jmp NEAR $L$_enc_dec_done_335 - -$L$_partial_incomplete_335: - mov r12,QWORD[112+rbp] - add QWORD[r8],r12 - mov r11,QWORD[112+rbp] - -$L$_enc_dec_done_335: - - - lea r12,[byte_len_to_mask_table] - kmovw k1,[r11*2+r12] - vmovdqu64 XMMWORD[64+rdx],xmm14 - - vpshufb xmm3,xmm3,XMMWORD[SHUF_MASK] - vpshufb xmm3,xmm3,xmm5 - mov r12,QWORD[120+rbp] - vmovdqu8 XMMWORD[r12]{k1},xmm3 -$L$_partial_block_done_335: - vmovdqu64 xmm2,XMMWORD[rdx] - mov r13,QWORD[112+rbp] - sub r13,r11 - je NEAR $L$_enc_dec_done_334 - cmp r13,256 - jbe NEAR $L$_message_below_equal_16_blocks_334 - - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vmovdqa64 zmm27,ZMMWORD[ddq_addbe_4444] - vmovdqa64 zmm28,ZMMWORD[ddq_addbe_1234] - - - - - - - vmovd r15d,xmm2 - and r15d,255 - - vshufi64x2 zmm2,zmm2,zmm2,0 - vpshufb zmm2,zmm2,zmm29 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_336 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_336 -$L$_next_16_overflow_336: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_336: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[192+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[176+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[192+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[208+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[224+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm12 - - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm7 - vmovdqa64 ZMMWORD[832+rsp],zmm10 - vmovdqa64 ZMMWORD[896+rsp],zmm11 - vmovdqa64 ZMMWORD[960+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_337 - - vmovdqu64 zmm0,ZMMWORD[288+rdx] - vmovdqu64 ZMMWORD[704+rsp],zmm0 - - vmovdqu64 zmm3,ZMMWORD[224+rdx] - vmovdqu64 ZMMWORD[640+rsp],zmm3 - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[160+rdx] - vmovdqu64 ZMMWORD[576+rsp],zmm4 - - vmovdqu64 zmm5,ZMMWORD[96+rdx] - vmovdqu64 ZMMWORD[512+rsp],zmm5 -$L$_skip_hkeys_precomputation_337: - cmp r13,512 - jb NEAR $L$_message_below_32_blocks_334 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_338 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_338 -$L$_next_16_overflow_338: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_338: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[448+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[176+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[192+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[208+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[224+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm12 - - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm7 - vmovdqa64 ZMMWORD[1088+rsp],zmm10 - vmovdqa64 ZMMWORD[1152+rsp],zmm11 - vmovdqa64 ZMMWORD[1216+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_339 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[192+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[128+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[64+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[rsp],zmm5 -$L$_skip_hkeys_precomputation_339: - mov r14,1 - add r11,512 - sub r13,512 - - cmp r13,768 - jb NEAR $L$_no_more_big_nblocks_334 -$L$_encrypt_big_nblocks_334: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_340 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_340 -$L$_16_blocks_overflow_340: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_340: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_341 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_341 -$L$_16_blocks_overflow_341: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_341: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_342 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_342 -$L$_16_blocks_overflow_342: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_342: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[512+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[576+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[640+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[704+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - - - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpternlogq zmm6,zmm12,zmm15,0x96 - vpxorq zmm6,zmm6,zmm24 - vpternlogq zmm7,zmm13,zmm10,0x96 - vpxorq zmm7,zmm7,zmm25 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vextracti64x4 ymm12,zmm6,1 - vpxorq ymm6,ymm6,ymm12 - vextracti32x4 xmm12,ymm6,1 - vpxorq xmm6,xmm6,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm6,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[512+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[576+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[640+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[704+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm0 - vmovdqa64 ZMMWORD[1088+rsp],zmm3 - vmovdqa64 ZMMWORD[1152+rsp],zmm4 - vmovdqa64 ZMMWORD[1216+rsp],zmm5 - vmovdqa64 zmm14,zmm6 - - add r11,768 - sub r13,768 - cmp r13,768 - jae NEAR $L$_encrypt_big_nblocks_334 - -$L$_no_more_big_nblocks_334: - - cmp r13,512 - jae NEAR $L$_encrypt_32_blocks_334 - - cmp r13,256 - jae NEAR $L$_encrypt_16_blocks_334 -$L$_encrypt_0_blocks_ghash_32_334: - mov r10d,r13d - and r10d,~15 - mov ebx,256 - sub ebx,r10d - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - add ebx,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_343 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_343 - jb NEAR $L$_last_num_blocks_is_7_1_343 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_343 - jb NEAR $L$_last_num_blocks_is_11_9_343 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_343 - ja NEAR $L$_last_num_blocks_is_16_343 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_343 - jmp NEAR $L$_last_num_blocks_is_13_343 - -$L$_last_num_blocks_is_11_9_343: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_343 - ja NEAR $L$_last_num_blocks_is_11_343 - jmp NEAR $L$_last_num_blocks_is_9_343 - -$L$_last_num_blocks_is_7_1_343: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_343 - jb NEAR $L$_last_num_blocks_is_3_1_343 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_343 - je NEAR $L$_last_num_blocks_is_6_343 - jmp NEAR $L$_last_num_blocks_is_5_343 - -$L$_last_num_blocks_is_3_1_343: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_343 - je NEAR $L$_last_num_blocks_is_2_343 -$L$_last_num_blocks_is_1_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_344 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_344 - -$L$_16_blocks_overflow_344: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_344: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_345 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_345 -$L$_small_initial_partial_block_345: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_345 -$L$_small_initial_compute_done_345: -$L$_after_reduction_345: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_2_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_346 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_346 - -$L$_16_blocks_overflow_346: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_346: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_347 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_347 -$L$_small_initial_partial_block_347: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_347: - - or r13,r13 - je NEAR $L$_after_reduction_347 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_347: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_3_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_348 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_348 - -$L$_16_blocks_overflow_348: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_348: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_349 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_349 -$L$_small_initial_partial_block_349: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_349: - - or r13,r13 - je NEAR $L$_after_reduction_349 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_349: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_4_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_350 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_350 - -$L$_16_blocks_overflow_350: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_350: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_351 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_351 -$L$_small_initial_partial_block_351: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_351: - - or r13,r13 - je NEAR $L$_after_reduction_351 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_351: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_5_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_352 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_352 - -$L$_16_blocks_overflow_352: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_352: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_353 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_353 -$L$_small_initial_partial_block_353: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_353: - - or r13,r13 - je NEAR $L$_after_reduction_353 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_353: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_6_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_354 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_354 - -$L$_16_blocks_overflow_354: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_354: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_355 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_355 -$L$_small_initial_partial_block_355: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_355: - - or r13,r13 - je NEAR $L$_after_reduction_355 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_355: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_7_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_356 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_356 - -$L$_16_blocks_overflow_356: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_356: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_357 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_357 -$L$_small_initial_partial_block_357: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_357: - - or r13,r13 - je NEAR $L$_after_reduction_357 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_357: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_8_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_358 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_358 - -$L$_16_blocks_overflow_358: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_358: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_359 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_359 -$L$_small_initial_partial_block_359: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_359: - - or r13,r13 - je NEAR $L$_after_reduction_359 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_359: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_9_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_360 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_360 - -$L$_16_blocks_overflow_360: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_360: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_361 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_361 -$L$_small_initial_partial_block_361: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_361: - - or r13,r13 - je NEAR $L$_after_reduction_361 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_361: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_10_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_362 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_362 - -$L$_16_blocks_overflow_362: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_362: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_363 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_363 -$L$_small_initial_partial_block_363: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_363: - - or r13,r13 - je NEAR $L$_after_reduction_363 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_363: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_11_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_364 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_364 - -$L$_16_blocks_overflow_364: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_364: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_365 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_365 -$L$_small_initial_partial_block_365: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_365: - - or r13,r13 - je NEAR $L$_after_reduction_365 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_365: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_12_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_366 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_366 - -$L$_16_blocks_overflow_366: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_366: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_367 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_367 -$L$_small_initial_partial_block_367: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_367: - - or r13,r13 - je NEAR $L$_after_reduction_367 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_367: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_13_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_368 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_368 - -$L$_16_blocks_overflow_368: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_368: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_369 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_369 -$L$_small_initial_partial_block_369: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_369: - - or r13,r13 - je NEAR $L$_after_reduction_369 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_369: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_14_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_370 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_370 - -$L$_16_blocks_overflow_370: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_370: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_371 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_371 -$L$_small_initial_partial_block_371: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_371: - - or r13,r13 - je NEAR $L$_after_reduction_371 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_371: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_15_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_372 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_372 - -$L$_16_blocks_overflow_372: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_372: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_373 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_373 -$L$_small_initial_partial_block_373: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_373: - - or r13,r13 - je NEAR $L$_after_reduction_373 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_373: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_16_343: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_374 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_374 - -$L$_16_blocks_overflow_374: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_374: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_375: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_375: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_375: - jmp NEAR $L$_last_blocks_done_343 -$L$_last_num_blocks_is_0_343: - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_343: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_334 -$L$_encrypt_32_blocks_334: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_376 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_376 -$L$_16_blocks_overflow_376: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_376: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_377 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_377 -$L$_16_blocks_overflow_377: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_377: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - - sub r13,512 - add r11,512 - mov r10d,r13d - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_378 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_378 - jb NEAR $L$_last_num_blocks_is_7_1_378 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_378 - jb NEAR $L$_last_num_blocks_is_11_9_378 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_378 - ja NEAR $L$_last_num_blocks_is_16_378 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_378 - jmp NEAR $L$_last_num_blocks_is_13_378 - -$L$_last_num_blocks_is_11_9_378: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_378 - ja NEAR $L$_last_num_blocks_is_11_378 - jmp NEAR $L$_last_num_blocks_is_9_378 - -$L$_last_num_blocks_is_7_1_378: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_378 - jb NEAR $L$_last_num_blocks_is_3_1_378 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_378 - je NEAR $L$_last_num_blocks_is_6_378 - jmp NEAR $L$_last_num_blocks_is_5_378 - -$L$_last_num_blocks_is_3_1_378: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_378 - je NEAR $L$_last_num_blocks_is_2_378 -$L$_last_num_blocks_is_1_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_379 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_379 - -$L$_16_blocks_overflow_379: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_379: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_380 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_380 -$L$_small_initial_partial_block_380: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_380 -$L$_small_initial_compute_done_380: -$L$_after_reduction_380: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_2_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_381 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_381 - -$L$_16_blocks_overflow_381: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_381: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_382 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_382 -$L$_small_initial_partial_block_382: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_382: - - or r13,r13 - je NEAR $L$_after_reduction_382 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_382: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_3_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_383 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_383 - -$L$_16_blocks_overflow_383: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_383: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_384 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_384 -$L$_small_initial_partial_block_384: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_384: - - or r13,r13 - je NEAR $L$_after_reduction_384 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_384: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_4_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_385 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_385 - -$L$_16_blocks_overflow_385: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_385: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_386 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_386 -$L$_small_initial_partial_block_386: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_386: - - or r13,r13 - je NEAR $L$_after_reduction_386 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_386: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_5_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_387 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_387 - -$L$_16_blocks_overflow_387: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_387: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_388 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_388 -$L$_small_initial_partial_block_388: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_388: - - or r13,r13 - je NEAR $L$_after_reduction_388 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_388: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_6_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_389 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_389 - -$L$_16_blocks_overflow_389: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_389: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_390 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_390 -$L$_small_initial_partial_block_390: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_390: - - or r13,r13 - je NEAR $L$_after_reduction_390 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_390: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_7_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_391 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_391 - -$L$_16_blocks_overflow_391: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_391: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_392 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_392 -$L$_small_initial_partial_block_392: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_392: - - or r13,r13 - je NEAR $L$_after_reduction_392 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_392: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_8_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_393 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_393 - -$L$_16_blocks_overflow_393: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_393: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_394 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_394 -$L$_small_initial_partial_block_394: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_394: - - or r13,r13 - je NEAR $L$_after_reduction_394 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_394: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_9_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_395 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_395 - -$L$_16_blocks_overflow_395: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_395: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_396 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_396 -$L$_small_initial_partial_block_396: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_396: - - or r13,r13 - je NEAR $L$_after_reduction_396 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_396: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_10_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_397 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_397 - -$L$_16_blocks_overflow_397: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_397: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_398 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_398 -$L$_small_initial_partial_block_398: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_398: - - or r13,r13 - je NEAR $L$_after_reduction_398 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_398: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_11_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_399 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_399 - -$L$_16_blocks_overflow_399: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_399: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_400 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_400 -$L$_small_initial_partial_block_400: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_400: - - or r13,r13 - je NEAR $L$_after_reduction_400 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_400: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_12_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_401 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_401 - -$L$_16_blocks_overflow_401: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_401: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_402 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_402 -$L$_small_initial_partial_block_402: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_402: - - or r13,r13 - je NEAR $L$_after_reduction_402 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_402: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_13_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_403 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_403 - -$L$_16_blocks_overflow_403: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_403: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_404 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_404 -$L$_small_initial_partial_block_404: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_404: - - or r13,r13 - je NEAR $L$_after_reduction_404 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_404: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_14_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_405 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_405 - -$L$_16_blocks_overflow_405: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_405: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_406 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_406 -$L$_small_initial_partial_block_406: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_406: - - or r13,r13 - je NEAR $L$_after_reduction_406 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_406: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_15_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_407 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_407 - -$L$_16_blocks_overflow_407: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_407: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_408 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_408 -$L$_small_initial_partial_block_408: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_408: - - or r13,r13 - je NEAR $L$_after_reduction_408 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_408: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_16_378: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_409 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_409 - -$L$_16_blocks_overflow_409: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_409: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_410: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_410: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_410: - jmp NEAR $L$_last_blocks_done_378 -$L$_last_num_blocks_is_0_378: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_378: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_334 -$L$_encrypt_16_blocks_334: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_411 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_411 -$L$_16_blocks_overflow_411: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_411: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[256+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[320+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[384+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[448+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - sub r13,256 - add r11,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_412 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_412 - jb NEAR $L$_last_num_blocks_is_7_1_412 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_412 - jb NEAR $L$_last_num_blocks_is_11_9_412 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_412 - ja NEAR $L$_last_num_blocks_is_16_412 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_412 - jmp NEAR $L$_last_num_blocks_is_13_412 - -$L$_last_num_blocks_is_11_9_412: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_412 - ja NEAR $L$_last_num_blocks_is_11_412 - jmp NEAR $L$_last_num_blocks_is_9_412 - -$L$_last_num_blocks_is_7_1_412: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_412 - jb NEAR $L$_last_num_blocks_is_3_1_412 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_412 - je NEAR $L$_last_num_blocks_is_6_412 - jmp NEAR $L$_last_num_blocks_is_5_412 - -$L$_last_num_blocks_is_3_1_412: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_412 - je NEAR $L$_last_num_blocks_is_2_412 -$L$_last_num_blocks_is_1_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_413 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_413 - -$L$_16_blocks_overflow_413: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_413: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc xmm0,xmm0,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc xmm0,xmm0,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_414 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_414 -$L$_small_initial_partial_block_414: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_414 -$L$_small_initial_compute_done_414: -$L$_after_reduction_414: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_2_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_415 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_415 - -$L$_16_blocks_overflow_415: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_415: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc ymm0,ymm0,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc ymm0,ymm0,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_416 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_416 -$L$_small_initial_partial_block_416: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_416: - - or r13,r13 - je NEAR $L$_after_reduction_416 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_416: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_3_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_417 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_417 - -$L$_16_blocks_overflow_417: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_417: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_418 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_418 -$L$_small_initial_partial_block_418: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_418: - - or r13,r13 - je NEAR $L$_after_reduction_418 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_418: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_4_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_419 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_419 - -$L$_16_blocks_overflow_419: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_419: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_420 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_420 -$L$_small_initial_partial_block_420: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_420: - - or r13,r13 - je NEAR $L$_after_reduction_420 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_420: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_5_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_421 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_421 - -$L$_16_blocks_overflow_421: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_421: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_422 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_422 -$L$_small_initial_partial_block_422: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_422: - - or r13,r13 - je NEAR $L$_after_reduction_422 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_422: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_6_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_423 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_423 - -$L$_16_blocks_overflow_423: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_423: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_424 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_424 -$L$_small_initial_partial_block_424: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_424: - - or r13,r13 - je NEAR $L$_after_reduction_424 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_424: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_7_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_425 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_425 - -$L$_16_blocks_overflow_425: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_425: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_426 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_426 -$L$_small_initial_partial_block_426: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_426: - - or r13,r13 - je NEAR $L$_after_reduction_426 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_426: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_8_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_427 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_427 - -$L$_16_blocks_overflow_427: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_427: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_428 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_428 -$L$_small_initial_partial_block_428: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_428: - - or r13,r13 - je NEAR $L$_after_reduction_428 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_428: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_9_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_429 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_429 - -$L$_16_blocks_overflow_429: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_429: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_430 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_430 -$L$_small_initial_partial_block_430: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_430: - - or r13,r13 - je NEAR $L$_after_reduction_430 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_430: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_10_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_431 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_431 - -$L$_16_blocks_overflow_431: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_431: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_432 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_432 -$L$_small_initial_partial_block_432: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_432: - - or r13,r13 - je NEAR $L$_after_reduction_432 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_432: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_11_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_433 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_433 - -$L$_16_blocks_overflow_433: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_433: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_434 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_434 -$L$_small_initial_partial_block_434: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_434: - - or r13,r13 - je NEAR $L$_after_reduction_434 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_434: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_12_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_435 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_435 - -$L$_16_blocks_overflow_435: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_435: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_436 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_436 -$L$_small_initial_partial_block_436: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_436: - - or r13,r13 - je NEAR $L$_after_reduction_436 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_436: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_13_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_437 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_437 - -$L$_16_blocks_overflow_437: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_437: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_438 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_438 -$L$_small_initial_partial_block_438: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_438: - - or r13,r13 - je NEAR $L$_after_reduction_438 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_438: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_14_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_439 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_439 - -$L$_16_blocks_overflow_439: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_439: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_440 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_440 -$L$_small_initial_partial_block_440: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_440: - - or r13,r13 - je NEAR $L$_after_reduction_440 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_440: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_15_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_441 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_441 - -$L$_16_blocks_overflow_441: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_441: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_442 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_442 -$L$_small_initial_partial_block_442: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_442: - - or r13,r13 - je NEAR $L$_after_reduction_442 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_442: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_16_412: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_443 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_443 - -$L$_16_blocks_overflow_443: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_443: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_444: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_444: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_444: - jmp NEAR $L$_last_blocks_done_412 -$L$_last_num_blocks_is_0_412: - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_412: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_334 - -$L$_message_below_32_blocks_334: - - - sub r13,256 - add r11,256 - mov r10d,r13d - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_445 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 -$L$_skip_hkeys_precomputation_445: - mov r14,1 - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_446 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_446 - jb NEAR $L$_last_num_blocks_is_7_1_446 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_446 - jb NEAR $L$_last_num_blocks_is_11_9_446 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_446 - ja NEAR $L$_last_num_blocks_is_16_446 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_446 - jmp NEAR $L$_last_num_blocks_is_13_446 - -$L$_last_num_blocks_is_11_9_446: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_446 - ja NEAR $L$_last_num_blocks_is_11_446 - jmp NEAR $L$_last_num_blocks_is_9_446 - -$L$_last_num_blocks_is_7_1_446: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_446 - jb NEAR $L$_last_num_blocks_is_3_1_446 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_446 - je NEAR $L$_last_num_blocks_is_6_446 - jmp NEAR $L$_last_num_blocks_is_5_446 - -$L$_last_num_blocks_is_3_1_446: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_446 - je NEAR $L$_last_num_blocks_is_2_446 -$L$_last_num_blocks_is_1_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_447 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_447 - -$L$_16_blocks_overflow_447: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_447: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm17,xmm0,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_448 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_448 -$L$_small_initial_partial_block_448: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_448 -$L$_small_initial_compute_done_448: -$L$_after_reduction_448: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_2_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_449 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_449 - -$L$_16_blocks_overflow_449: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_449: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm17,ymm0,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_450 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_450 -$L$_small_initial_partial_block_450: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_450: - - or r13,r13 - je NEAR $L$_after_reduction_450 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_450: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_3_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_451 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_451 - -$L$_16_blocks_overflow_451: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_451: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_452 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_452 -$L$_small_initial_partial_block_452: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_452: - - or r13,r13 - je NEAR $L$_after_reduction_452 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_452: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_4_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_453 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_453 - -$L$_16_blocks_overflow_453: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_453: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm17,zmm0,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_454 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_454 -$L$_small_initial_partial_block_454: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_454: - - or r13,r13 - je NEAR $L$_after_reduction_454 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_454: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_5_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_455 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_455 - -$L$_16_blocks_overflow_455: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_455: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb xmm19,xmm3,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_456 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_456 -$L$_small_initial_partial_block_456: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_456: - - or r13,r13 - je NEAR $L$_after_reduction_456 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_456: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_6_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_457 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_457 - -$L$_16_blocks_overflow_457: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_457: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb ymm19,ymm3,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_458 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_458 -$L$_small_initial_partial_block_458: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_458: - - or r13,r13 - je NEAR $L$_after_reduction_458 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_458: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_7_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_459 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_459 - -$L$_16_blocks_overflow_459: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_459: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_460 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_460 -$L$_small_initial_partial_block_460: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_460: - - or r13,r13 - je NEAR $L$_after_reduction_460 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_460: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_8_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_461 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_461 - -$L$_16_blocks_overflow_461: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_461: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_462 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_462 -$L$_small_initial_partial_block_462: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_462: - - or r13,r13 - je NEAR $L$_after_reduction_462 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_462: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_9_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_463 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_463 - -$L$_16_blocks_overflow_463: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_463: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb xmm20,xmm4,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_464 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_464 -$L$_small_initial_partial_block_464: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_464: - - or r13,r13 - je NEAR $L$_after_reduction_464 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_464: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_10_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_465 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_465 - -$L$_16_blocks_overflow_465: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_465: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb ymm20,ymm4,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_466 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_466 -$L$_small_initial_partial_block_466: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_466: - - or r13,r13 - je NEAR $L$_after_reduction_466 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_466: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_11_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_467 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_467 - -$L$_16_blocks_overflow_467: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_467: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_468 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_468 -$L$_small_initial_partial_block_468: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_468: - - or r13,r13 - je NEAR $L$_after_reduction_468 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_468: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_12_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_469 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_469 - -$L$_16_blocks_overflow_469: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_469: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_470 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_470 -$L$_small_initial_partial_block_470: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_470: - - or r13,r13 - je NEAR $L$_after_reduction_470 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_470: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_13_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_471 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_471 - -$L$_16_blocks_overflow_471: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_471: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb xmm21,xmm5,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_472 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_472 -$L$_small_initial_partial_block_472: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_472: - - or r13,r13 - je NEAR $L$_after_reduction_472 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_472: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_14_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_473 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_473 - -$L$_16_blocks_overflow_473: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_473: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb ymm21,ymm5,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_474 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_474 -$L$_small_initial_partial_block_474: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_474: - - or r13,r13 - je NEAR $L$_after_reduction_474 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_474: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_15_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_475 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_475 - -$L$_16_blocks_overflow_475: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_475: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_476 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_476 -$L$_small_initial_partial_block_476: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_476: - - or r13,r13 - je NEAR $L$_after_reduction_476 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_476: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_16_446: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_477 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_477 - -$L$_16_blocks_overflow_477: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_477: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm17,zmm0,zmm29 - vpshufb zmm19,zmm3,zmm29 - vpshufb zmm20,zmm4,zmm29 - vpshufb zmm21,zmm5,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_478: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_478: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_478: - jmp NEAR $L$_last_blocks_done_446 -$L$_last_num_blocks_is_0_446: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_446: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_334 - -$L$_message_below_equal_16_blocks_334: - - - mov r12d,r13d - add r12d,15 - shr r12d,4 - cmp r12,8 - je NEAR $L$_small_initial_num_blocks_is_8_479 - jl NEAR $L$_small_initial_num_blocks_is_7_1_479 - - - cmp r12,12 - je NEAR $L$_small_initial_num_blocks_is_12_479 - jl NEAR $L$_small_initial_num_blocks_is_11_9_479 - - - cmp r12,16 - je NEAR $L$_small_initial_num_blocks_is_16_479 - cmp r12,15 - je NEAR $L$_small_initial_num_blocks_is_15_479 - cmp r12,14 - je NEAR $L$_small_initial_num_blocks_is_14_479 - jmp NEAR $L$_small_initial_num_blocks_is_13_479 - -$L$_small_initial_num_blocks_is_11_9_479: - - cmp r12,11 - je NEAR $L$_small_initial_num_blocks_is_11_479 - cmp r12,10 - je NEAR $L$_small_initial_num_blocks_is_10_479 - jmp NEAR $L$_small_initial_num_blocks_is_9_479 - -$L$_small_initial_num_blocks_is_7_1_479: - cmp r12,4 - je NEAR $L$_small_initial_num_blocks_is_4_479 - jl NEAR $L$_small_initial_num_blocks_is_3_1_479 - - cmp r12,7 - je NEAR $L$_small_initial_num_blocks_is_7_479 - cmp r12,6 - je NEAR $L$_small_initial_num_blocks_is_6_479 - jmp NEAR $L$_small_initial_num_blocks_is_5_479 - -$L$_small_initial_num_blocks_is_3_1_479: - - cmp r12,3 - je NEAR $L$_small_initial_num_blocks_is_3_479 - cmp r12,2 - je NEAR $L$_small_initial_num_blocks_is_2_479 - - - - - -$L$_small_initial_num_blocks_is_1_479: - vmovdqa64 xmm29,XMMWORD[SHUF_MASK] - vpaddd xmm0,xmm2,XMMWORD[ONE] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,0 - vpshufb xmm0,xmm0,xmm29 - vmovdqu8 xmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast xmm0,xmm0,xmm15 - vpxorq xmm0,xmm0,xmm6 - vextracti32x4 xmm12,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm6,xmm0,xmm29 - vextracti32x4 xmm13,zmm6,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_480 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_480 -$L$_small_initial_partial_block_480: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm13 - - jmp NEAR $L$_after_reduction_480 -$L$_small_initial_compute_done_480: -$L$_after_reduction_480: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_2_479: - vmovdqa64 ymm29,YMMWORD[SHUF_MASK] - vshufi64x2 ymm0,ymm2,ymm2,0 - vpaddd ymm0,ymm0,YMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,1 - vpshufb ymm0,ymm0,ymm29 - vmovdqu8 ymm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast ymm0,ymm0,ymm15 - vpxorq ymm0,ymm0,ymm6 - vextracti32x4 xmm12,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm6,ymm0,ymm29 - vextracti32x4 xmm13,zmm6,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_481 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_481 -$L$_small_initial_partial_block_481: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_481: - - or r13,r13 - je NEAR $L$_after_reduction_481 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_481: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_3_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,2 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm0,zmm29 - vextracti32x4 xmm13,zmm6,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_482 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_482 -$L$_small_initial_partial_block_482: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_482: - - or r13,r13 - je NEAR $L$_after_reduction_482 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_482: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_4_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,3 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm0,zmm29 - vextracti32x4 xmm13,zmm6,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_483 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_483 -$L$_small_initial_partial_block_483: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_483: - - or r13,r13 - je NEAR $L$_after_reduction_483 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_483: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_5_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 xmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast xmm3,xmm3,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq xmm3,xmm3,xmm7 - vextracti32x4 xmm12,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb xmm7,xmm3,xmm29 - vextracti32x4 xmm13,zmm7,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_484 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_484 -$L$_small_initial_partial_block_484: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_484: - - or r13,r13 - je NEAR $L$_after_reduction_484 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_484: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_6_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 ymm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast ymm3,ymm3,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq ymm3,ymm3,ymm7 - vextracti32x4 xmm12,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb ymm7,ymm3,ymm29 - vextracti32x4 xmm13,zmm7,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_485 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_485 -$L$_small_initial_partial_block_485: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_485: - - or r13,r13 - je NEAR $L$_after_reduction_485 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_485: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_7_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vextracti32x4 xmm13,zmm7,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_486 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_486 -$L$_small_initial_partial_block_486: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_486: - - or r13,r13 - je NEAR $L$_after_reduction_486 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_486: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_8_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vextracti32x4 xmm13,zmm7,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_487 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_487 -$L$_small_initial_partial_block_487: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_487: - - or r13,r13 - je NEAR $L$_after_reduction_487 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_487: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_9_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast xmm4,xmm4,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq xmm4,xmm4,xmm10 - vextracti32x4 xmm12,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb xmm10,xmm4,xmm29 - vextracti32x4 xmm13,zmm10,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_488 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_488 -$L$_small_initial_partial_block_488: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_488: - - or r13,r13 - je NEAR $L$_after_reduction_488 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_488: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_10_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast ymm4,ymm4,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq ymm4,ymm4,ymm10 - vextracti32x4 xmm12,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb ymm10,ymm4,ymm29 - vextracti32x4 xmm13,zmm10,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_489 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_489 -$L$_small_initial_partial_block_489: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_489: - - or r13,r13 - je NEAR $L$_after_reduction_489 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_489: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_11_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vextracti32x4 xmm13,zmm10,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_490 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_490 -$L$_small_initial_partial_block_490: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_490: - - or r13,r13 - je NEAR $L$_after_reduction_490 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_490: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_12_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vextracti32x4 xmm13,zmm10,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_491 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_491 -$L$_small_initial_partial_block_491: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_491: - - or r13,r13 - je NEAR $L$_after_reduction_491 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_491: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_13_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast xmm5,xmm5,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq xmm5,xmm5,xmm11 - vextracti32x4 xmm12,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb xmm11,xmm5,xmm29 - vextracti32x4 xmm13,zmm11,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_492 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_492 -$L$_small_initial_partial_block_492: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_492: - - or r13,r13 - je NEAR $L$_after_reduction_492 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_492: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_14_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast ymm5,ymm5,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq ymm5,ymm5,ymm11 - vextracti32x4 xmm12,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb ymm11,ymm5,ymm29 - vextracti32x4 xmm13,zmm11,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_493 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_493 -$L$_small_initial_partial_block_493: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_493: - - or r13,r13 - je NEAR $L$_after_reduction_493 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_493: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_15_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb zmm11,zmm5,zmm29 - vextracti32x4 xmm13,zmm11,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_494 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_494 -$L$_small_initial_partial_block_494: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_494: - - or r13,r13 - je NEAR $L$_after_reduction_494 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_494: - jmp NEAR $L$_small_initial_blocks_encrypted_479 -$L$_small_initial_num_blocks_is_16_479: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm0,zmm29 - vpshufb zmm7,zmm3,zmm29 - vpshufb zmm10,zmm4,zmm29 - vpshufb zmm11,zmm5,zmm29 - vextracti32x4 xmm13,zmm11,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_495: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_495: - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_495: -$L$_small_initial_blocks_encrypted_479: -$L$_ghash_done_334: - vmovdqu64 XMMWORD[rdx],xmm2 - vmovdqu64 XMMWORD[64+rdx],xmm14 -$L$_enc_dec_done_334: - jmp NEAR $L$exit_gcm_encrypt -$L$exit_gcm_encrypt: - cmp QWORD[112+rbp],256 - jbe NEAR $L$skip_hkeys_cleanup_496 - vpxor xmm0,xmm0,xmm0 - vmovdqa64 ZMMWORD[rsp],zmm0 - vmovdqa64 ZMMWORD[64+rsp],zmm0 - vmovdqa64 ZMMWORD[128+rsp],zmm0 - vmovdqa64 ZMMWORD[192+rsp],zmm0 - vmovdqa64 ZMMWORD[256+rsp],zmm0 - vmovdqa64 ZMMWORD[320+rsp],zmm0 - vmovdqa64 ZMMWORD[384+rsp],zmm0 - vmovdqa64 ZMMWORD[448+rsp],zmm0 - vmovdqa64 ZMMWORD[512+rsp],zmm0 - vmovdqa64 ZMMWORD[576+rsp],zmm0 - vmovdqa64 ZMMWORD[640+rsp],zmm0 - vmovdqa64 ZMMWORD[704+rsp],zmm0 -$L$skip_hkeys_cleanup_496: - vzeroupper - vmovdqu xmm15,XMMWORD[((-16))+rbp] - vmovdqu xmm14,XMMWORD[((-32))+rbp] - vmovdqu xmm13,XMMWORD[((-48))+rbp] - vmovdqu xmm12,XMMWORD[((-64))+rbp] - vmovdqu xmm11,XMMWORD[((-80))+rbp] - vmovdqu xmm10,XMMWORD[((-96))+rbp] - vmovdqu xmm9,XMMWORD[((-112))+rbp] - vmovdqu xmm8,XMMWORD[((-128))+rbp] - vmovdqu xmm7,XMMWORD[((-144))+rbp] - vmovdqu xmm6,XMMWORD[((-160))+rbp] - lea rsp,[8+rbp] - pop rsi - - pop rdi - - pop r15 - - pop r14 - - pop r13 - - pop r12 - - pop rbp - - pop rbx - - DB 0F3h,0C3h ;repret -$L$encrypt_seh_end: - - -global ossl_aes_gcm_decrypt_avx512 - -ALIGN 32 -ossl_aes_gcm_decrypt_avx512: - -$L$decrypt_seh_begin: -DB 243,15,30,250 - push rbx - -$L$decrypt_seh_push_rbx: - push rbp - -$L$decrypt_seh_push_rbp: - push r12 - -$L$decrypt_seh_push_r12: - push r13 - -$L$decrypt_seh_push_r13: - push r14 - -$L$decrypt_seh_push_r14: - push r15 - -$L$decrypt_seh_push_r15: - push rdi -$L$decrypt_seh_push_rdi: - push rsi -$L$decrypt_seh_push_rsi: - - sub rsp,168 -$L$decrypt_seh_allocstack_xmm: - - - - - - - - - - - lea rbp,[160+rsp] - -$L$decrypt_seh_setfp: - vmovdqu XMMWORD[rsp],xmm6 -$L$decrypt_seh_save_xmm6: - vmovdqu XMMWORD[16+rsp],xmm7 -$L$decrypt_seh_save_xmm7: - vmovdqu XMMWORD[32+rsp],xmm8 -$L$decrypt_seh_save_xmm8: - vmovdqu XMMWORD[48+rsp],xmm9 -$L$decrypt_seh_save_xmm9: - vmovdqu XMMWORD[64+rsp],xmm10 -$L$decrypt_seh_save_xmm10: - vmovdqu XMMWORD[80+rsp],xmm11 -$L$decrypt_seh_save_xmm11: - vmovdqu XMMWORD[96+rsp],xmm12 -$L$decrypt_seh_save_xmm12: - vmovdqu XMMWORD[112+rsp],xmm13 -$L$decrypt_seh_save_xmm13: - vmovdqu XMMWORD[128+rsp],xmm14 -$L$decrypt_seh_save_xmm14: - vmovdqu XMMWORD[144+rsp],xmm15 -$L$decrypt_seh_save_xmm15: - -$L$decrypt_seh_prolog_end: - sub rsp,1584 - and rsp,(-64) - - - mov eax,DWORD[240+rcx] - cmp eax,9 - je NEAR $L$aes_gcm_decrypt_128_avx512 - cmp eax,11 - je NEAR $L$aes_gcm_decrypt_192_avx512 - cmp eax,13 - je NEAR $L$aes_gcm_decrypt_256_avx512 - xor eax,eax - jmp NEAR $L$exit_gcm_decrypt -ALIGN 32 -$L$aes_gcm_decrypt_128_avx512: - cmp QWORD[112+rbp],0 - je NEAR $L$_enc_dec_done_497 - xor r14,r14 - vmovdqu64 xmm14,XMMWORD[64+rdx] - - mov r11,QWORD[r8] - or r11,r11 - je NEAR $L$_partial_block_done_498 - mov r10d,16 - lea r12,[byte_len_to_mask_table] - cmp QWORD[112+rbp],r10 - cmovc r10,QWORD[112+rbp] - add r12,r10 - add r12,r10 - kmovw k1,[r12] - vmovdqu8 xmm0{k1}{z},[r9] - - vmovdqu64 xmm3,XMMWORD[16+rdx] - vmovdqu64 xmm4,XMMWORD[336+rdx] - - - - lea r12,[SHIFT_MASK] - add r12,r11 - vmovdqu64 xmm5,XMMWORD[r12] - vpshufb xmm3,xmm3,xmm5 - - vmovdqa64 xmm6,xmm0 - vpxorq xmm3,xmm3,xmm0 - - - mov r13,QWORD[112+rbp] - add r13,r11 - sub r13,16 - jge NEAR $L$_no_extra_mask_498 - sub r12,r13 -$L$_no_extra_mask_498: - - - - vmovdqu64 xmm0,XMMWORD[16+r12] - vpand xmm3,xmm3,xmm0 - vpand xmm6,xmm6,xmm0 - vpshufb xmm6,xmm6,XMMWORD[SHUF_MASK] - vpshufb xmm6,xmm6,xmm5 - vpxorq xmm14,xmm14,xmm6 - cmp r13,0 - jl NEAR $L$_partial_incomplete_498 - - vpclmulqdq xmm7,xmm14,xmm4,0x11 - vpclmulqdq xmm10,xmm14,xmm4,0x00 - vpclmulqdq xmm11,xmm14,xmm4,0x01 - vpclmulqdq xmm14,xmm14,xmm4,0x10 - vpxorq xmm14,xmm14,xmm11 - - vpsrldq xmm11,xmm14,8 - vpslldq xmm14,xmm14,8 - vpxorq xmm7,xmm7,xmm11 - vpxorq xmm14,xmm14,xmm10 - - - - vmovdqu64 xmm11,XMMWORD[POLY2] - - vpclmulqdq xmm10,xmm11,xmm14,0x01 - vpslldq xmm10,xmm10,8 - vpxorq xmm14,xmm14,xmm10 - - - - vpclmulqdq xmm10,xmm11,xmm14,0x00 - vpsrldq xmm10,xmm10,4 - vpclmulqdq xmm14,xmm11,xmm14,0x10 - vpslldq xmm14,xmm14,4 - - vpternlogq xmm14,xmm7,xmm10,0x96 - - mov QWORD[r8],0 - - mov r12,r11 - mov r11,16 - sub r11,r12 - jmp NEAR $L$_enc_dec_done_498 - -$L$_partial_incomplete_498: - mov r12,QWORD[112+rbp] - add QWORD[r8],r12 - mov r11,QWORD[112+rbp] - -$L$_enc_dec_done_498: - - - lea r12,[byte_len_to_mask_table] - kmovw k1,[r11*2+r12] - vmovdqu64 XMMWORD[64+rdx],xmm14 - mov r12,QWORD[120+rbp] - vmovdqu8 XMMWORD[r12]{k1},xmm3 -$L$_partial_block_done_498: - vmovdqu64 xmm2,XMMWORD[rdx] - mov r13,QWORD[112+rbp] - sub r13,r11 - je NEAR $L$_enc_dec_done_497 - cmp r13,256 - jbe NEAR $L$_message_below_equal_16_blocks_497 - - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vmovdqa64 zmm27,ZMMWORD[ddq_addbe_4444] - vmovdqa64 zmm28,ZMMWORD[ddq_addbe_1234] - - - - - - - vmovd r15d,xmm2 - and r15d,255 - - vshufi64x2 zmm2,zmm2,zmm2,0 - vpshufb zmm2,zmm2,zmm29 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_499 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_499 -$L$_next_16_overflow_499: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_499: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[192+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm12 - - vpshufb zmm7,zmm0,zmm29 - vpshufb zmm10,zmm3,zmm29 - vpshufb zmm11,zmm4,zmm29 - vpshufb zmm12,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm7 - vmovdqa64 ZMMWORD[832+rsp],zmm10 - vmovdqa64 ZMMWORD[896+rsp],zmm11 - vmovdqa64 ZMMWORD[960+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_500 - - vmovdqu64 zmm0,ZMMWORD[288+rdx] - vmovdqu64 ZMMWORD[704+rsp],zmm0 - - vmovdqu64 zmm3,ZMMWORD[224+rdx] - vmovdqu64 ZMMWORD[640+rsp],zmm3 - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[160+rdx] - vmovdqu64 ZMMWORD[576+rsp],zmm4 - - vmovdqu64 zmm5,ZMMWORD[96+rdx] - vmovdqu64 ZMMWORD[512+rsp],zmm5 -$L$_skip_hkeys_precomputation_500: - cmp r13,512 - jb NEAR $L$_message_below_32_blocks_497 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_501 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_501 -$L$_next_16_overflow_501: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_501: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[448+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm12 - - vpshufb zmm7,zmm0,zmm29 - vpshufb zmm10,zmm3,zmm29 - vpshufb zmm11,zmm4,zmm29 - vpshufb zmm12,zmm5,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm7 - vmovdqa64 ZMMWORD[1088+rsp],zmm10 - vmovdqa64 ZMMWORD[1152+rsp],zmm11 - vmovdqa64 ZMMWORD[1216+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_502 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[192+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[128+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[64+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[rsp],zmm5 -$L$_skip_hkeys_precomputation_502: - mov r14,1 - add r11,512 - sub r13,512 - - cmp r13,768 - jb NEAR $L$_no_more_big_nblocks_497 -$L$_encrypt_big_nblocks_497: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_503 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_503 -$L$_16_blocks_overflow_503: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_503: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_504 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_504 -$L$_16_blocks_overflow_504: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_504: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_505 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_505 -$L$_16_blocks_overflow_505: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_505: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[512+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[576+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[640+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[704+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - - - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpternlogq zmm6,zmm12,zmm15,0x96 - vpxorq zmm6,zmm6,zmm24 - vpternlogq zmm7,zmm13,zmm10,0x96 - vpxorq zmm7,zmm7,zmm25 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vextracti64x4 ymm12,zmm6,1 - vpxorq ymm6,ymm6,ymm12 - vextracti32x4 xmm12,ymm6,1 - vpxorq xmm6,xmm6,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm6,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[512+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[576+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[640+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[704+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm0 - vmovdqa64 ZMMWORD[1088+rsp],zmm3 - vmovdqa64 ZMMWORD[1152+rsp],zmm4 - vmovdqa64 ZMMWORD[1216+rsp],zmm5 - vmovdqa64 zmm14,zmm6 - - add r11,768 - sub r13,768 - cmp r13,768 - jae NEAR $L$_encrypt_big_nblocks_497 - -$L$_no_more_big_nblocks_497: - - cmp r13,512 - jae NEAR $L$_encrypt_32_blocks_497 - - cmp r13,256 - jae NEAR $L$_encrypt_16_blocks_497 -$L$_encrypt_0_blocks_ghash_32_497: - mov r10d,r13d - and r10d,~15 - mov ebx,256 - sub ebx,r10d - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - add ebx,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_506 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_506 - jb NEAR $L$_last_num_blocks_is_7_1_506 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_506 - jb NEAR $L$_last_num_blocks_is_11_9_506 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_506 - ja NEAR $L$_last_num_blocks_is_16_506 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_506 - jmp NEAR $L$_last_num_blocks_is_13_506 - -$L$_last_num_blocks_is_11_9_506: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_506 - ja NEAR $L$_last_num_blocks_is_11_506 - jmp NEAR $L$_last_num_blocks_is_9_506 - -$L$_last_num_blocks_is_7_1_506: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_506 - jb NEAR $L$_last_num_blocks_is_3_1_506 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_506 - je NEAR $L$_last_num_blocks_is_6_506 - jmp NEAR $L$_last_num_blocks_is_5_506 - -$L$_last_num_blocks_is_3_1_506: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_506 - je NEAR $L$_last_num_blocks_is_2_506 -$L$_last_num_blocks_is_1_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_507 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_507 - -$L$_16_blocks_overflow_507: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_507: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_508 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_508 -$L$_small_initial_partial_block_508: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_508 -$L$_small_initial_compute_done_508: -$L$_after_reduction_508: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_2_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_509 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_509 - -$L$_16_blocks_overflow_509: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_509: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_510 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_510 -$L$_small_initial_partial_block_510: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_510: - - or r13,r13 - je NEAR $L$_after_reduction_510 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_510: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_3_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_511 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_511 - -$L$_16_blocks_overflow_511: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_511: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_512 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_512 -$L$_small_initial_partial_block_512: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_512: - - or r13,r13 - je NEAR $L$_after_reduction_512 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_512: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_4_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_513 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_513 - -$L$_16_blocks_overflow_513: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_513: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_514 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_514 -$L$_small_initial_partial_block_514: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_514: - - or r13,r13 - je NEAR $L$_after_reduction_514 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_514: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_5_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_515 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_515 - -$L$_16_blocks_overflow_515: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_515: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_516 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_516 -$L$_small_initial_partial_block_516: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_516: - - or r13,r13 - je NEAR $L$_after_reduction_516 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_516: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_6_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_517 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_517 - -$L$_16_blocks_overflow_517: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_517: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_518 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_518 -$L$_small_initial_partial_block_518: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_518: - - or r13,r13 - je NEAR $L$_after_reduction_518 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_518: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_7_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_519 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_519 - -$L$_16_blocks_overflow_519: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_519: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_520 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_520 -$L$_small_initial_partial_block_520: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_520: - - or r13,r13 - je NEAR $L$_after_reduction_520 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_520: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_8_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_521 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_521 - -$L$_16_blocks_overflow_521: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_521: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_522 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_522 -$L$_small_initial_partial_block_522: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_522: - - or r13,r13 - je NEAR $L$_after_reduction_522 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_522: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_9_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_523 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_523 - -$L$_16_blocks_overflow_523: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_523: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_524 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_524 -$L$_small_initial_partial_block_524: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_524: - - or r13,r13 - je NEAR $L$_after_reduction_524 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_524: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_10_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_525 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_525 - -$L$_16_blocks_overflow_525: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_525: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_526 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_526 -$L$_small_initial_partial_block_526: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_526: - - or r13,r13 - je NEAR $L$_after_reduction_526 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_526: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_11_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_527 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_527 - -$L$_16_blocks_overflow_527: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_527: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_528 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_528 -$L$_small_initial_partial_block_528: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_528: - - or r13,r13 - je NEAR $L$_after_reduction_528 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_528: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_12_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_529 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_529 - -$L$_16_blocks_overflow_529: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_529: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_530 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_530 -$L$_small_initial_partial_block_530: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_530: - - or r13,r13 - je NEAR $L$_after_reduction_530 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_530: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_13_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_531 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_531 - -$L$_16_blocks_overflow_531: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_531: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_532 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_532 -$L$_small_initial_partial_block_532: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_532: - - or r13,r13 - je NEAR $L$_after_reduction_532 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_532: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_14_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_533 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_533 - -$L$_16_blocks_overflow_533: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_533: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_534 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_534 -$L$_small_initial_partial_block_534: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_534: - - or r13,r13 - je NEAR $L$_after_reduction_534 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_534: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_15_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_535 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_535 - -$L$_16_blocks_overflow_535: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_535: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_536 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_536 -$L$_small_initial_partial_block_536: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_536: - - or r13,r13 - je NEAR $L$_after_reduction_536 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_536: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_16_506: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_537 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_537 - -$L$_16_blocks_overflow_537: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_537: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_538: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_538: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_538: - jmp NEAR $L$_last_blocks_done_506 -$L$_last_num_blocks_is_0_506: - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_506: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_497 -$L$_encrypt_32_blocks_497: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_539 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_539 -$L$_16_blocks_overflow_539: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_539: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_540 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_540 -$L$_16_blocks_overflow_540: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_540: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - - sub r13,512 - add r11,512 - mov r10d,r13d - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_541 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_541 - jb NEAR $L$_last_num_blocks_is_7_1_541 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_541 - jb NEAR $L$_last_num_blocks_is_11_9_541 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_541 - ja NEAR $L$_last_num_blocks_is_16_541 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_541 - jmp NEAR $L$_last_num_blocks_is_13_541 - -$L$_last_num_blocks_is_11_9_541: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_541 - ja NEAR $L$_last_num_blocks_is_11_541 - jmp NEAR $L$_last_num_blocks_is_9_541 - -$L$_last_num_blocks_is_7_1_541: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_541 - jb NEAR $L$_last_num_blocks_is_3_1_541 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_541 - je NEAR $L$_last_num_blocks_is_6_541 - jmp NEAR $L$_last_num_blocks_is_5_541 - -$L$_last_num_blocks_is_3_1_541: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_541 - je NEAR $L$_last_num_blocks_is_2_541 -$L$_last_num_blocks_is_1_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_542 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_542 - -$L$_16_blocks_overflow_542: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_542: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_543 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_543 -$L$_small_initial_partial_block_543: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_543 -$L$_small_initial_compute_done_543: -$L$_after_reduction_543: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_2_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_544 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_544 - -$L$_16_blocks_overflow_544: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_544: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_545 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_545 -$L$_small_initial_partial_block_545: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_545: - - or r13,r13 - je NEAR $L$_after_reduction_545 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_545: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_3_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_546 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_546 - -$L$_16_blocks_overflow_546: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_546: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_547 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_547 -$L$_small_initial_partial_block_547: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_547: - - or r13,r13 - je NEAR $L$_after_reduction_547 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_547: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_4_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_548 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_548 - -$L$_16_blocks_overflow_548: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_548: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_549 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_549 -$L$_small_initial_partial_block_549: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_549: - - or r13,r13 - je NEAR $L$_after_reduction_549 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_549: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_5_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_550 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_550 - -$L$_16_blocks_overflow_550: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_550: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_551 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_551 -$L$_small_initial_partial_block_551: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_551: - - or r13,r13 - je NEAR $L$_after_reduction_551 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_551: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_6_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_552 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_552 - -$L$_16_blocks_overflow_552: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_552: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_553 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_553 -$L$_small_initial_partial_block_553: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_553: - - or r13,r13 - je NEAR $L$_after_reduction_553 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_553: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_7_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_554 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_554 - -$L$_16_blocks_overflow_554: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_554: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_555 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_555 -$L$_small_initial_partial_block_555: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_555: - - or r13,r13 - je NEAR $L$_after_reduction_555 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_555: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_8_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_556 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_556 - -$L$_16_blocks_overflow_556: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_556: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_557 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_557 -$L$_small_initial_partial_block_557: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_557: - - or r13,r13 - je NEAR $L$_after_reduction_557 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_557: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_9_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_558 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_558 - -$L$_16_blocks_overflow_558: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_558: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_559 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_559 -$L$_small_initial_partial_block_559: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_559: - - or r13,r13 - je NEAR $L$_after_reduction_559 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_559: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_10_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_560 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_560 - -$L$_16_blocks_overflow_560: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_560: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_561 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_561 -$L$_small_initial_partial_block_561: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_561: - - or r13,r13 - je NEAR $L$_after_reduction_561 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_561: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_11_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_562 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_562 - -$L$_16_blocks_overflow_562: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_562: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_563 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_563 -$L$_small_initial_partial_block_563: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_563: - - or r13,r13 - je NEAR $L$_after_reduction_563 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_563: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_12_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_564 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_564 - -$L$_16_blocks_overflow_564: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_564: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_565 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_565 -$L$_small_initial_partial_block_565: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_565: - - or r13,r13 - je NEAR $L$_after_reduction_565 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_565: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_13_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_566 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_566 - -$L$_16_blocks_overflow_566: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_566: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_567 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_567 -$L$_small_initial_partial_block_567: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_567: - - or r13,r13 - je NEAR $L$_after_reduction_567 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_567: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_14_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_568 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_568 - -$L$_16_blocks_overflow_568: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_568: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_569 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_569 -$L$_small_initial_partial_block_569: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_569: - - or r13,r13 - je NEAR $L$_after_reduction_569 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_569: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_15_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_570 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_570 - -$L$_16_blocks_overflow_570: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_570: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_571 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_571 -$L$_small_initial_partial_block_571: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_571: - - or r13,r13 - je NEAR $L$_after_reduction_571 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_571: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_16_541: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_572 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_572 - -$L$_16_blocks_overflow_572: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_572: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_573: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_573: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_573: - jmp NEAR $L$_last_blocks_done_541 -$L$_last_num_blocks_is_0_541: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_541: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_497 -$L$_encrypt_16_blocks_497: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_574 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_574 -$L$_16_blocks_overflow_574: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_574: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[256+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[320+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[384+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[448+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - sub r13,256 - add r11,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_575 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_575 - jb NEAR $L$_last_num_blocks_is_7_1_575 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_575 - jb NEAR $L$_last_num_blocks_is_11_9_575 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_575 - ja NEAR $L$_last_num_blocks_is_16_575 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_575 - jmp NEAR $L$_last_num_blocks_is_13_575 - -$L$_last_num_blocks_is_11_9_575: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_575 - ja NEAR $L$_last_num_blocks_is_11_575 - jmp NEAR $L$_last_num_blocks_is_9_575 - -$L$_last_num_blocks_is_7_1_575: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_575 - jb NEAR $L$_last_num_blocks_is_3_1_575 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_575 - je NEAR $L$_last_num_blocks_is_6_575 - jmp NEAR $L$_last_num_blocks_is_5_575 - -$L$_last_num_blocks_is_3_1_575: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_575 - je NEAR $L$_last_num_blocks_is_2_575 -$L$_last_num_blocks_is_1_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_576 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_576 - -$L$_16_blocks_overflow_576: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_576: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc xmm0,xmm0,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_577 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_577 -$L$_small_initial_partial_block_577: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_577 -$L$_small_initial_compute_done_577: -$L$_after_reduction_577: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_2_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_578 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_578 - -$L$_16_blocks_overflow_578: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_578: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc ymm0,ymm0,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_579 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_579 -$L$_small_initial_partial_block_579: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_579: - - or r13,r13 - je NEAR $L$_after_reduction_579 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_579: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_3_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_580 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_580 - -$L$_16_blocks_overflow_580: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_580: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_581 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_581 -$L$_small_initial_partial_block_581: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_581: - - or r13,r13 - je NEAR $L$_after_reduction_581 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_581: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_4_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_582 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_582 - -$L$_16_blocks_overflow_582: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_582: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_583 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_583 -$L$_small_initial_partial_block_583: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_583: - - or r13,r13 - je NEAR $L$_after_reduction_583 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_583: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_5_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_584 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_584 - -$L$_16_blocks_overflow_584: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_584: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_585 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_585 -$L$_small_initial_partial_block_585: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_585: - - or r13,r13 - je NEAR $L$_after_reduction_585 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_585: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_6_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_586 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_586 - -$L$_16_blocks_overflow_586: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_586: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_587 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_587 -$L$_small_initial_partial_block_587: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_587: - - or r13,r13 - je NEAR $L$_after_reduction_587 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_587: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_7_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_588 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_588 - -$L$_16_blocks_overflow_588: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_588: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_589 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_589 -$L$_small_initial_partial_block_589: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_589: - - or r13,r13 - je NEAR $L$_after_reduction_589 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_589: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_8_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_590 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_590 - -$L$_16_blocks_overflow_590: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_590: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_591 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_591 -$L$_small_initial_partial_block_591: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_591: - - or r13,r13 - je NEAR $L$_after_reduction_591 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_591: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_9_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_592 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_592 - -$L$_16_blocks_overflow_592: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_592: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_593 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_593 -$L$_small_initial_partial_block_593: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_593: - - or r13,r13 - je NEAR $L$_after_reduction_593 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_593: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_10_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_594 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_594 - -$L$_16_blocks_overflow_594: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_594: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_595 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_595 -$L$_small_initial_partial_block_595: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_595: - - or r13,r13 - je NEAR $L$_after_reduction_595 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_595: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_11_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_596 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_596 - -$L$_16_blocks_overflow_596: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_596: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_597 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_597 -$L$_small_initial_partial_block_597: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_597: - - or r13,r13 - je NEAR $L$_after_reduction_597 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_597: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_12_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_598 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_598 - -$L$_16_blocks_overflow_598: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_598: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_599 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_599 -$L$_small_initial_partial_block_599: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_599: - - or r13,r13 - je NEAR $L$_after_reduction_599 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_599: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_13_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_600 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_600 - -$L$_16_blocks_overflow_600: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_600: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_601 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_601 -$L$_small_initial_partial_block_601: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_601: - - or r13,r13 - je NEAR $L$_after_reduction_601 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_601: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_14_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_602 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_602 - -$L$_16_blocks_overflow_602: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_602: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_603 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_603 -$L$_small_initial_partial_block_603: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_603: - - or r13,r13 - je NEAR $L$_after_reduction_603 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_603: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_15_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_604 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_604 - -$L$_16_blocks_overflow_604: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_604: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_605 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_605 -$L$_small_initial_partial_block_605: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_605: - - or r13,r13 - je NEAR $L$_after_reduction_605 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_605: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_16_575: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_606 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_606 - -$L$_16_blocks_overflow_606: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_606: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_607: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_607: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_607: - jmp NEAR $L$_last_blocks_done_575 -$L$_last_num_blocks_is_0_575: - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_575: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_497 - -$L$_message_below_32_blocks_497: - - - sub r13,256 - add r11,256 - mov r10d,r13d - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_608 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 -$L$_skip_hkeys_precomputation_608: - mov r14,1 - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_609 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_609 - jb NEAR $L$_last_num_blocks_is_7_1_609 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_609 - jb NEAR $L$_last_num_blocks_is_11_9_609 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_609 - ja NEAR $L$_last_num_blocks_is_16_609 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_609 - jmp NEAR $L$_last_num_blocks_is_13_609 - -$L$_last_num_blocks_is_11_9_609: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_609 - ja NEAR $L$_last_num_blocks_is_11_609 - jmp NEAR $L$_last_num_blocks_is_9_609 - -$L$_last_num_blocks_is_7_1_609: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_609 - jb NEAR $L$_last_num_blocks_is_3_1_609 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_609 - je NEAR $L$_last_num_blocks_is_6_609 - jmp NEAR $L$_last_num_blocks_is_5_609 - -$L$_last_num_blocks_is_3_1_609: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_609 - je NEAR $L$_last_num_blocks_is_2_609 -$L$_last_num_blocks_is_1_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_610 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_610 - -$L$_16_blocks_overflow_610: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_610: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_611 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_611 -$L$_small_initial_partial_block_611: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_611 -$L$_small_initial_compute_done_611: -$L$_after_reduction_611: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_2_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_612 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_612 - -$L$_16_blocks_overflow_612: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_612: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_613 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_613 -$L$_small_initial_partial_block_613: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_613: - - or r13,r13 - je NEAR $L$_after_reduction_613 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_613: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_3_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_614 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_614 - -$L$_16_blocks_overflow_614: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_614: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_615 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_615 -$L$_small_initial_partial_block_615: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_615: - - or r13,r13 - je NEAR $L$_after_reduction_615 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_615: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_4_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_616 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_616 - -$L$_16_blocks_overflow_616: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_616: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_617 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_617 -$L$_small_initial_partial_block_617: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_617: - - or r13,r13 - je NEAR $L$_after_reduction_617 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_617: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_5_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_618 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_618 - -$L$_16_blocks_overflow_618: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_618: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_619 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_619 -$L$_small_initial_partial_block_619: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_619: - - or r13,r13 - je NEAR $L$_after_reduction_619 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_619: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_6_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_620 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_620 - -$L$_16_blocks_overflow_620: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_620: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_621 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_621 -$L$_small_initial_partial_block_621: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_621: - - or r13,r13 - je NEAR $L$_after_reduction_621 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_621: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_7_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_622 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_622 - -$L$_16_blocks_overflow_622: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_622: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_623 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_623 -$L$_small_initial_partial_block_623: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_623: - - or r13,r13 - je NEAR $L$_after_reduction_623 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_623: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_8_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_624 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_624 - -$L$_16_blocks_overflow_624: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_624: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_625 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_625 -$L$_small_initial_partial_block_625: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_625: - - or r13,r13 - je NEAR $L$_after_reduction_625 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_625: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_9_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_626 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_626 - -$L$_16_blocks_overflow_626: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_626: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_627 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_627 -$L$_small_initial_partial_block_627: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_627: - - or r13,r13 - je NEAR $L$_after_reduction_627 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_627: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_10_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_628 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_628 - -$L$_16_blocks_overflow_628: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_628: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_629 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_629 -$L$_small_initial_partial_block_629: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_629: - - or r13,r13 - je NEAR $L$_after_reduction_629 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_629: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_11_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_630 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_630 - -$L$_16_blocks_overflow_630: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_630: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_631 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_631 -$L$_small_initial_partial_block_631: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_631: - - or r13,r13 - je NEAR $L$_after_reduction_631 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_631: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_12_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_632 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_632 - -$L$_16_blocks_overflow_632: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_632: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_633 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_633 -$L$_small_initial_partial_block_633: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_633: - - or r13,r13 - je NEAR $L$_after_reduction_633 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_633: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_13_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_634 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_634 - -$L$_16_blocks_overflow_634: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_634: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_635 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_635 -$L$_small_initial_partial_block_635: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_635: - - or r13,r13 - je NEAR $L$_after_reduction_635 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_635: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_14_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_636 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_636 - -$L$_16_blocks_overflow_636: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_636: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_637 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_637 -$L$_small_initial_partial_block_637: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_637: - - or r13,r13 - je NEAR $L$_after_reduction_637 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_637: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_15_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_638 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_638 - -$L$_16_blocks_overflow_638: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_638: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_639 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_639 -$L$_small_initial_partial_block_639: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_639: - - or r13,r13 - je NEAR $L$_after_reduction_639 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_639: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_16_609: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_640 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_640 - -$L$_16_blocks_overflow_640: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_640: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_641: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_641: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_641: - jmp NEAR $L$_last_blocks_done_609 -$L$_last_num_blocks_is_0_609: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_609: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_497 - -$L$_message_below_equal_16_blocks_497: - - - mov r12d,r13d - add r12d,15 - shr r12d,4 - cmp r12,8 - je NEAR $L$_small_initial_num_blocks_is_8_642 - jl NEAR $L$_small_initial_num_blocks_is_7_1_642 - - - cmp r12,12 - je NEAR $L$_small_initial_num_blocks_is_12_642 - jl NEAR $L$_small_initial_num_blocks_is_11_9_642 - - - cmp r12,16 - je NEAR $L$_small_initial_num_blocks_is_16_642 - cmp r12,15 - je NEAR $L$_small_initial_num_blocks_is_15_642 - cmp r12,14 - je NEAR $L$_small_initial_num_blocks_is_14_642 - jmp NEAR $L$_small_initial_num_blocks_is_13_642 - -$L$_small_initial_num_blocks_is_11_9_642: - - cmp r12,11 - je NEAR $L$_small_initial_num_blocks_is_11_642 - cmp r12,10 - je NEAR $L$_small_initial_num_blocks_is_10_642 - jmp NEAR $L$_small_initial_num_blocks_is_9_642 - -$L$_small_initial_num_blocks_is_7_1_642: - cmp r12,4 - je NEAR $L$_small_initial_num_blocks_is_4_642 - jl NEAR $L$_small_initial_num_blocks_is_3_1_642 - - cmp r12,7 - je NEAR $L$_small_initial_num_blocks_is_7_642 - cmp r12,6 - je NEAR $L$_small_initial_num_blocks_is_6_642 - jmp NEAR $L$_small_initial_num_blocks_is_5_642 - -$L$_small_initial_num_blocks_is_3_1_642: - - cmp r12,3 - je NEAR $L$_small_initial_num_blocks_is_3_642 - cmp r12,2 - je NEAR $L$_small_initial_num_blocks_is_2_642 - - - - - -$L$_small_initial_num_blocks_is_1_642: - vmovdqa64 xmm29,XMMWORD[SHUF_MASK] - vpaddd xmm0,xmm2,XMMWORD[ONE] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,0 - vpshufb xmm0,xmm0,xmm29 - vmovdqu8 xmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast xmm0,xmm0,xmm15 - vpxorq xmm0,xmm0,xmm6 - vextracti32x4 xmm12,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm6,xmm6,xmm29 - vextracti32x4 xmm13,zmm6,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_643 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_643 -$L$_small_initial_partial_block_643: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm13 - - jmp NEAR $L$_after_reduction_643 -$L$_small_initial_compute_done_643: -$L$_after_reduction_643: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_2_642: - vmovdqa64 ymm29,YMMWORD[SHUF_MASK] - vshufi64x2 ymm0,ymm2,ymm2,0 - vpaddd ymm0,ymm0,YMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,1 - vpshufb ymm0,ymm0,ymm29 - vmovdqu8 ymm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast ymm0,ymm0,ymm15 - vpxorq ymm0,ymm0,ymm6 - vextracti32x4 xmm12,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm6,ymm6,ymm29 - vextracti32x4 xmm13,zmm6,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_644 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_644 -$L$_small_initial_partial_block_644: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_644: - - or r13,r13 - je NEAR $L$_after_reduction_644 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_644: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_3_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,2 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm6,zmm29 - vextracti32x4 xmm13,zmm6,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_645 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_645 -$L$_small_initial_partial_block_645: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_645: - - or r13,r13 - je NEAR $L$_after_reduction_645 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_645: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_4_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,3 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm6,zmm29 - vextracti32x4 xmm13,zmm6,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_646 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_646 -$L$_small_initial_partial_block_646: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_646: - - or r13,r13 - je NEAR $L$_after_reduction_646 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_646: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_5_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 xmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast xmm3,xmm3,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq xmm3,xmm3,xmm7 - vextracti32x4 xmm12,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb xmm7,xmm7,xmm29 - vextracti32x4 xmm13,zmm7,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_647 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_647 -$L$_small_initial_partial_block_647: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_647: - - or r13,r13 - je NEAR $L$_after_reduction_647 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_647: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_6_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 ymm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast ymm3,ymm3,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq ymm3,ymm3,ymm7 - vextracti32x4 xmm12,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb ymm7,ymm7,ymm29 - vextracti32x4 xmm13,zmm7,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_648 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_648 -$L$_small_initial_partial_block_648: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_648: - - or r13,r13 - je NEAR $L$_after_reduction_648 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_648: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_7_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vextracti32x4 xmm13,zmm7,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_649 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_649 -$L$_small_initial_partial_block_649: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_649: - - or r13,r13 - je NEAR $L$_after_reduction_649 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_649: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_8_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vextracti32x4 xmm13,zmm7,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_650 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_650 -$L$_small_initial_partial_block_650: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_650: - - or r13,r13 - je NEAR $L$_after_reduction_650 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_650: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_9_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast xmm4,xmm4,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq xmm4,xmm4,xmm10 - vextracti32x4 xmm12,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb xmm10,xmm10,xmm29 - vextracti32x4 xmm13,zmm10,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_651 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_651 -$L$_small_initial_partial_block_651: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_651: - - or r13,r13 - je NEAR $L$_after_reduction_651 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_651: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_10_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast ymm4,ymm4,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq ymm4,ymm4,ymm10 - vextracti32x4 xmm12,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb ymm10,ymm10,ymm29 - vextracti32x4 xmm13,zmm10,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_652 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_652 -$L$_small_initial_partial_block_652: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_652: - - or r13,r13 - je NEAR $L$_after_reduction_652 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_652: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_11_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vextracti32x4 xmm13,zmm10,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_653 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_653 -$L$_small_initial_partial_block_653: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_653: - - or r13,r13 - je NEAR $L$_after_reduction_653 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_653: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_12_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vextracti32x4 xmm13,zmm10,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_654 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_654 -$L$_small_initial_partial_block_654: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_654: - - or r13,r13 - je NEAR $L$_after_reduction_654 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_654: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_13_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast xmm5,xmm5,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq xmm5,xmm5,xmm11 - vextracti32x4 xmm12,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb xmm11,xmm11,xmm29 - vextracti32x4 xmm13,zmm11,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_655 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_655 -$L$_small_initial_partial_block_655: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_655: - - or r13,r13 - je NEAR $L$_after_reduction_655 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_655: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_14_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast ymm5,ymm5,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq ymm5,ymm5,ymm11 - vextracti32x4 xmm12,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb ymm11,ymm11,ymm29 - vextracti32x4 xmm13,zmm11,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_656 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_656 -$L$_small_initial_partial_block_656: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_656: - - or r13,r13 - je NEAR $L$_after_reduction_656 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_656: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_15_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vextracti32x4 xmm13,zmm11,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_657 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_657 -$L$_small_initial_partial_block_657: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_657: - - or r13,r13 - je NEAR $L$_after_reduction_657 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_657: - jmp NEAR $L$_small_initial_blocks_encrypted_642 -$L$_small_initial_num_blocks_is_16_642: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vextracti32x4 xmm13,zmm11,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_658: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_658: - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_658: -$L$_small_initial_blocks_encrypted_642: -$L$_ghash_done_497: - vmovdqu64 XMMWORD[rdx],xmm2 - vmovdqu64 XMMWORD[64+rdx],xmm14 -$L$_enc_dec_done_497: - jmp NEAR $L$exit_gcm_decrypt -ALIGN 32 -$L$aes_gcm_decrypt_192_avx512: - cmp QWORD[112+rbp],0 - je NEAR $L$_enc_dec_done_659 - xor r14,r14 - vmovdqu64 xmm14,XMMWORD[64+rdx] - - mov r11,QWORD[r8] - or r11,r11 - je NEAR $L$_partial_block_done_660 - mov r10d,16 - lea r12,[byte_len_to_mask_table] - cmp QWORD[112+rbp],r10 - cmovc r10,QWORD[112+rbp] - add r12,r10 - add r12,r10 - kmovw k1,[r12] - vmovdqu8 xmm0{k1}{z},[r9] - - vmovdqu64 xmm3,XMMWORD[16+rdx] - vmovdqu64 xmm4,XMMWORD[336+rdx] - - - - lea r12,[SHIFT_MASK] - add r12,r11 - vmovdqu64 xmm5,XMMWORD[r12] - vpshufb xmm3,xmm3,xmm5 - - vmovdqa64 xmm6,xmm0 - vpxorq xmm3,xmm3,xmm0 - - - mov r13,QWORD[112+rbp] - add r13,r11 - sub r13,16 - jge NEAR $L$_no_extra_mask_660 - sub r12,r13 -$L$_no_extra_mask_660: - - - - vmovdqu64 xmm0,XMMWORD[16+r12] - vpand xmm3,xmm3,xmm0 - vpand xmm6,xmm6,xmm0 - vpshufb xmm6,xmm6,XMMWORD[SHUF_MASK] - vpshufb xmm6,xmm6,xmm5 - vpxorq xmm14,xmm14,xmm6 - cmp r13,0 - jl NEAR $L$_partial_incomplete_660 - - vpclmulqdq xmm7,xmm14,xmm4,0x11 - vpclmulqdq xmm10,xmm14,xmm4,0x00 - vpclmulqdq xmm11,xmm14,xmm4,0x01 - vpclmulqdq xmm14,xmm14,xmm4,0x10 - vpxorq xmm14,xmm14,xmm11 - - vpsrldq xmm11,xmm14,8 - vpslldq xmm14,xmm14,8 - vpxorq xmm7,xmm7,xmm11 - vpxorq xmm14,xmm14,xmm10 - - - - vmovdqu64 xmm11,XMMWORD[POLY2] - - vpclmulqdq xmm10,xmm11,xmm14,0x01 - vpslldq xmm10,xmm10,8 - vpxorq xmm14,xmm14,xmm10 - - - - vpclmulqdq xmm10,xmm11,xmm14,0x00 - vpsrldq xmm10,xmm10,4 - vpclmulqdq xmm14,xmm11,xmm14,0x10 - vpslldq xmm14,xmm14,4 - - vpternlogq xmm14,xmm7,xmm10,0x96 - - mov QWORD[r8],0 - - mov r12,r11 - mov r11,16 - sub r11,r12 - jmp NEAR $L$_enc_dec_done_660 - -$L$_partial_incomplete_660: - mov r12,QWORD[112+rbp] - add QWORD[r8],r12 - mov r11,QWORD[112+rbp] - -$L$_enc_dec_done_660: - - - lea r12,[byte_len_to_mask_table] - kmovw k1,[r11*2+r12] - vmovdqu64 XMMWORD[64+rdx],xmm14 - mov r12,QWORD[120+rbp] - vmovdqu8 XMMWORD[r12]{k1},xmm3 -$L$_partial_block_done_660: - vmovdqu64 xmm2,XMMWORD[rdx] - mov r13,QWORD[112+rbp] - sub r13,r11 - je NEAR $L$_enc_dec_done_659 - cmp r13,256 - jbe NEAR $L$_message_below_equal_16_blocks_659 - - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vmovdqa64 zmm27,ZMMWORD[ddq_addbe_4444] - vmovdqa64 zmm28,ZMMWORD[ddq_addbe_1234] - - - - - - - vmovd r15d,xmm2 - and r15d,255 - - vshufi64x2 zmm2,zmm2,zmm2,0 - vpshufb zmm2,zmm2,zmm29 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_661 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_661 -$L$_next_16_overflow_661: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_661: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[192+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[176+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[192+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm12 - - vpshufb zmm7,zmm0,zmm29 - vpshufb zmm10,zmm3,zmm29 - vpshufb zmm11,zmm4,zmm29 - vpshufb zmm12,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm7 - vmovdqa64 ZMMWORD[832+rsp],zmm10 - vmovdqa64 ZMMWORD[896+rsp],zmm11 - vmovdqa64 ZMMWORD[960+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_662 - - vmovdqu64 zmm0,ZMMWORD[288+rdx] - vmovdqu64 ZMMWORD[704+rsp],zmm0 - - vmovdqu64 zmm3,ZMMWORD[224+rdx] - vmovdqu64 ZMMWORD[640+rsp],zmm3 - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[160+rdx] - vmovdqu64 ZMMWORD[576+rsp],zmm4 - - vmovdqu64 zmm5,ZMMWORD[96+rdx] - vmovdqu64 ZMMWORD[512+rsp],zmm5 -$L$_skip_hkeys_precomputation_662: - cmp r13,512 - jb NEAR $L$_message_below_32_blocks_659 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_663 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_663 -$L$_next_16_overflow_663: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_663: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[448+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[176+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[192+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm12 - - vpshufb zmm7,zmm0,zmm29 - vpshufb zmm10,zmm3,zmm29 - vpshufb zmm11,zmm4,zmm29 - vpshufb zmm12,zmm5,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm7 - vmovdqa64 ZMMWORD[1088+rsp],zmm10 - vmovdqa64 ZMMWORD[1152+rsp],zmm11 - vmovdqa64 ZMMWORD[1216+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_664 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[192+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[128+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[64+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[rsp],zmm5 -$L$_skip_hkeys_precomputation_664: - mov r14,1 - add r11,512 - sub r13,512 - - cmp r13,768 - jb NEAR $L$_no_more_big_nblocks_659 -$L$_encrypt_big_nblocks_659: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_665 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_665 -$L$_16_blocks_overflow_665: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_665: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_666 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_666 -$L$_16_blocks_overflow_666: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_666: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_667 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_667 -$L$_16_blocks_overflow_667: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_667: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[512+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[576+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[640+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[704+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - - - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpternlogq zmm6,zmm12,zmm15,0x96 - vpxorq zmm6,zmm6,zmm24 - vpternlogq zmm7,zmm13,zmm10,0x96 - vpxorq zmm7,zmm7,zmm25 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vextracti64x4 ymm12,zmm6,1 - vpxorq ymm6,ymm6,ymm12 - vextracti32x4 xmm12,ymm6,1 - vpxorq xmm6,xmm6,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm6,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[512+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[576+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[640+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[704+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm0 - vmovdqa64 ZMMWORD[1088+rsp],zmm3 - vmovdqa64 ZMMWORD[1152+rsp],zmm4 - vmovdqa64 ZMMWORD[1216+rsp],zmm5 - vmovdqa64 zmm14,zmm6 - - add r11,768 - sub r13,768 - cmp r13,768 - jae NEAR $L$_encrypt_big_nblocks_659 - -$L$_no_more_big_nblocks_659: - - cmp r13,512 - jae NEAR $L$_encrypt_32_blocks_659 - - cmp r13,256 - jae NEAR $L$_encrypt_16_blocks_659 -$L$_encrypt_0_blocks_ghash_32_659: - mov r10d,r13d - and r10d,~15 - mov ebx,256 - sub ebx,r10d - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - add ebx,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_668 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_668 - jb NEAR $L$_last_num_blocks_is_7_1_668 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_668 - jb NEAR $L$_last_num_blocks_is_11_9_668 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_668 - ja NEAR $L$_last_num_blocks_is_16_668 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_668 - jmp NEAR $L$_last_num_blocks_is_13_668 - -$L$_last_num_blocks_is_11_9_668: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_668 - ja NEAR $L$_last_num_blocks_is_11_668 - jmp NEAR $L$_last_num_blocks_is_9_668 - -$L$_last_num_blocks_is_7_1_668: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_668 - jb NEAR $L$_last_num_blocks_is_3_1_668 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_668 - je NEAR $L$_last_num_blocks_is_6_668 - jmp NEAR $L$_last_num_blocks_is_5_668 - -$L$_last_num_blocks_is_3_1_668: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_668 - je NEAR $L$_last_num_blocks_is_2_668 -$L$_last_num_blocks_is_1_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_669 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_669 - -$L$_16_blocks_overflow_669: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_669: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_670 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_670 -$L$_small_initial_partial_block_670: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_670 -$L$_small_initial_compute_done_670: -$L$_after_reduction_670: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_2_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_671 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_671 - -$L$_16_blocks_overflow_671: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_671: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_672 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_672 -$L$_small_initial_partial_block_672: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_672: - - or r13,r13 - je NEAR $L$_after_reduction_672 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_672: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_3_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_673 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_673 - -$L$_16_blocks_overflow_673: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_673: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_674 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_674 -$L$_small_initial_partial_block_674: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_674: - - or r13,r13 - je NEAR $L$_after_reduction_674 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_674: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_4_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_675 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_675 - -$L$_16_blocks_overflow_675: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_675: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_676 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_676 -$L$_small_initial_partial_block_676: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_676: - - or r13,r13 - je NEAR $L$_after_reduction_676 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_676: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_5_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_677 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_677 - -$L$_16_blocks_overflow_677: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_677: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_678 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_678 -$L$_small_initial_partial_block_678: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_678: - - or r13,r13 - je NEAR $L$_after_reduction_678 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_678: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_6_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_679 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_679 - -$L$_16_blocks_overflow_679: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_679: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_680 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_680 -$L$_small_initial_partial_block_680: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_680: - - or r13,r13 - je NEAR $L$_after_reduction_680 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_680: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_7_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_681 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_681 - -$L$_16_blocks_overflow_681: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_681: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_682 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_682 -$L$_small_initial_partial_block_682: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_682: - - or r13,r13 - je NEAR $L$_after_reduction_682 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_682: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_8_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_683 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_683 - -$L$_16_blocks_overflow_683: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_683: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_684 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_684 -$L$_small_initial_partial_block_684: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_684: - - or r13,r13 - je NEAR $L$_after_reduction_684 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_684: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_9_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_685 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_685 - -$L$_16_blocks_overflow_685: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_685: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_686 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_686 -$L$_small_initial_partial_block_686: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_686: - - or r13,r13 - je NEAR $L$_after_reduction_686 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_686: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_10_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_687 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_687 - -$L$_16_blocks_overflow_687: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_687: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_688 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_688 -$L$_small_initial_partial_block_688: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_688: - - or r13,r13 - je NEAR $L$_after_reduction_688 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_688: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_11_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_689 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_689 - -$L$_16_blocks_overflow_689: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_689: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_690 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_690 -$L$_small_initial_partial_block_690: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_690: - - or r13,r13 - je NEAR $L$_after_reduction_690 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_690: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_12_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_691 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_691 - -$L$_16_blocks_overflow_691: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_691: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_692 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_692 -$L$_small_initial_partial_block_692: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_692: - - or r13,r13 - je NEAR $L$_after_reduction_692 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_692: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_13_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_693 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_693 - -$L$_16_blocks_overflow_693: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_693: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_694 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_694 -$L$_small_initial_partial_block_694: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_694: - - or r13,r13 - je NEAR $L$_after_reduction_694 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_694: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_14_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_695 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_695 - -$L$_16_blocks_overflow_695: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_695: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_696 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_696 -$L$_small_initial_partial_block_696: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_696: - - or r13,r13 - je NEAR $L$_after_reduction_696 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_696: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_15_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_697 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_697 - -$L$_16_blocks_overflow_697: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_697: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_698 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_698 -$L$_small_initial_partial_block_698: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_698: - - or r13,r13 - je NEAR $L$_after_reduction_698 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_698: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_16_668: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_699 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_699 - -$L$_16_blocks_overflow_699: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_699: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_700: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_700: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_700: - jmp NEAR $L$_last_blocks_done_668 -$L$_last_num_blocks_is_0_668: - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_668: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_659 -$L$_encrypt_32_blocks_659: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_701 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_701 -$L$_16_blocks_overflow_701: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_701: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_702 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_702 -$L$_16_blocks_overflow_702: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_702: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - - sub r13,512 - add r11,512 - mov r10d,r13d - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_703 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_703 - jb NEAR $L$_last_num_blocks_is_7_1_703 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_703 - jb NEAR $L$_last_num_blocks_is_11_9_703 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_703 - ja NEAR $L$_last_num_blocks_is_16_703 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_703 - jmp NEAR $L$_last_num_blocks_is_13_703 - -$L$_last_num_blocks_is_11_9_703: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_703 - ja NEAR $L$_last_num_blocks_is_11_703 - jmp NEAR $L$_last_num_blocks_is_9_703 - -$L$_last_num_blocks_is_7_1_703: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_703 - jb NEAR $L$_last_num_blocks_is_3_1_703 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_703 - je NEAR $L$_last_num_blocks_is_6_703 - jmp NEAR $L$_last_num_blocks_is_5_703 - -$L$_last_num_blocks_is_3_1_703: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_703 - je NEAR $L$_last_num_blocks_is_2_703 -$L$_last_num_blocks_is_1_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_704 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_704 - -$L$_16_blocks_overflow_704: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_704: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_705 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_705 -$L$_small_initial_partial_block_705: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_705 -$L$_small_initial_compute_done_705: -$L$_after_reduction_705: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_2_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_706 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_706 - -$L$_16_blocks_overflow_706: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_706: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_707 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_707 -$L$_small_initial_partial_block_707: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_707: - - or r13,r13 - je NEAR $L$_after_reduction_707 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_707: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_3_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_708 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_708 - -$L$_16_blocks_overflow_708: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_708: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_709 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_709 -$L$_small_initial_partial_block_709: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_709: - - or r13,r13 - je NEAR $L$_after_reduction_709 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_709: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_4_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_710 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_710 - -$L$_16_blocks_overflow_710: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_710: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_711 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_711 -$L$_small_initial_partial_block_711: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_711: - - or r13,r13 - je NEAR $L$_after_reduction_711 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_711: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_5_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_712 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_712 - -$L$_16_blocks_overflow_712: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_712: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_713 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_713 -$L$_small_initial_partial_block_713: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_713: - - or r13,r13 - je NEAR $L$_after_reduction_713 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_713: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_6_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_714 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_714 - -$L$_16_blocks_overflow_714: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_714: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_715 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_715 -$L$_small_initial_partial_block_715: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_715: - - or r13,r13 - je NEAR $L$_after_reduction_715 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_715: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_7_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_716 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_716 - -$L$_16_blocks_overflow_716: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_716: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_717 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_717 -$L$_small_initial_partial_block_717: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_717: - - or r13,r13 - je NEAR $L$_after_reduction_717 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_717: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_8_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_718 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_718 - -$L$_16_blocks_overflow_718: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_718: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_719 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_719 -$L$_small_initial_partial_block_719: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_719: - - or r13,r13 - je NEAR $L$_after_reduction_719 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_719: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_9_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_720 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_720 - -$L$_16_blocks_overflow_720: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_720: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_721 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_721 -$L$_small_initial_partial_block_721: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_721: - - or r13,r13 - je NEAR $L$_after_reduction_721 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_721: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_10_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_722 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_722 - -$L$_16_blocks_overflow_722: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_722: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_723 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_723 -$L$_small_initial_partial_block_723: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_723: - - or r13,r13 - je NEAR $L$_after_reduction_723 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_723: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_11_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_724 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_724 - -$L$_16_blocks_overflow_724: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_724: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_725 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_725 -$L$_small_initial_partial_block_725: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_725: - - or r13,r13 - je NEAR $L$_after_reduction_725 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_725: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_12_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_726 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_726 - -$L$_16_blocks_overflow_726: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_726: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_727 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_727 -$L$_small_initial_partial_block_727: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_727: - - or r13,r13 - je NEAR $L$_after_reduction_727 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_727: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_13_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_728 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_728 - -$L$_16_blocks_overflow_728: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_728: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_729 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_729 -$L$_small_initial_partial_block_729: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_729: - - or r13,r13 - je NEAR $L$_after_reduction_729 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_729: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_14_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_730 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_730 - -$L$_16_blocks_overflow_730: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_730: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_731 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_731 -$L$_small_initial_partial_block_731: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_731: - - or r13,r13 - je NEAR $L$_after_reduction_731 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_731: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_15_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_732 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_732 - -$L$_16_blocks_overflow_732: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_732: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_733 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_733 -$L$_small_initial_partial_block_733: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_733: - - or r13,r13 - je NEAR $L$_after_reduction_733 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_733: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_16_703: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_734 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_734 - -$L$_16_blocks_overflow_734: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_734: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_735: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_735: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_735: - jmp NEAR $L$_last_blocks_done_703 -$L$_last_num_blocks_is_0_703: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_703: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_659 -$L$_encrypt_16_blocks_659: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_736 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_736 -$L$_16_blocks_overflow_736: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_736: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[256+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[320+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[384+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[448+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - sub r13,256 - add r11,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_737 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_737 - jb NEAR $L$_last_num_blocks_is_7_1_737 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_737 - jb NEAR $L$_last_num_blocks_is_11_9_737 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_737 - ja NEAR $L$_last_num_blocks_is_16_737 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_737 - jmp NEAR $L$_last_num_blocks_is_13_737 - -$L$_last_num_blocks_is_11_9_737: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_737 - ja NEAR $L$_last_num_blocks_is_11_737 - jmp NEAR $L$_last_num_blocks_is_9_737 - -$L$_last_num_blocks_is_7_1_737: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_737 - jb NEAR $L$_last_num_blocks_is_3_1_737 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_737 - je NEAR $L$_last_num_blocks_is_6_737 - jmp NEAR $L$_last_num_blocks_is_5_737 - -$L$_last_num_blocks_is_3_1_737: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_737 - je NEAR $L$_last_num_blocks_is_2_737 -$L$_last_num_blocks_is_1_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_738 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_738 - -$L$_16_blocks_overflow_738: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_738: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc xmm0,xmm0,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_739 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_739 -$L$_small_initial_partial_block_739: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_739 -$L$_small_initial_compute_done_739: -$L$_after_reduction_739: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_2_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_740 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_740 - -$L$_16_blocks_overflow_740: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_740: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc ymm0,ymm0,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_741 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_741 -$L$_small_initial_partial_block_741: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_741: - - or r13,r13 - je NEAR $L$_after_reduction_741 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_741: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_3_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_742 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_742 - -$L$_16_blocks_overflow_742: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_742: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_743 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_743 -$L$_small_initial_partial_block_743: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_743: - - or r13,r13 - je NEAR $L$_after_reduction_743 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_743: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_4_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_744 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_744 - -$L$_16_blocks_overflow_744: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_744: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_745 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_745 -$L$_small_initial_partial_block_745: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_745: - - or r13,r13 - je NEAR $L$_after_reduction_745 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_745: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_5_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_746 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_746 - -$L$_16_blocks_overflow_746: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_746: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_747 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_747 -$L$_small_initial_partial_block_747: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_747: - - or r13,r13 - je NEAR $L$_after_reduction_747 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_747: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_6_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_748 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_748 - -$L$_16_blocks_overflow_748: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_748: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_749 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_749 -$L$_small_initial_partial_block_749: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_749: - - or r13,r13 - je NEAR $L$_after_reduction_749 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_749: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_7_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_750 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_750 - -$L$_16_blocks_overflow_750: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_750: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_751 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_751 -$L$_small_initial_partial_block_751: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_751: - - or r13,r13 - je NEAR $L$_after_reduction_751 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_751: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_8_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_752 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_752 - -$L$_16_blocks_overflow_752: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_752: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_753 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_753 -$L$_small_initial_partial_block_753: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_753: - - or r13,r13 - je NEAR $L$_after_reduction_753 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_753: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_9_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_754 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_754 - -$L$_16_blocks_overflow_754: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_754: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_755 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_755 -$L$_small_initial_partial_block_755: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_755: - - or r13,r13 - je NEAR $L$_after_reduction_755 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_755: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_10_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_756 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_756 - -$L$_16_blocks_overflow_756: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_756: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_757 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_757 -$L$_small_initial_partial_block_757: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_757: - - or r13,r13 - je NEAR $L$_after_reduction_757 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_757: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_11_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_758 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_758 - -$L$_16_blocks_overflow_758: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_758: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_759 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_759 -$L$_small_initial_partial_block_759: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_759: - - or r13,r13 - je NEAR $L$_after_reduction_759 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_759: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_12_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_760 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_760 - -$L$_16_blocks_overflow_760: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_760: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_761 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_761 -$L$_small_initial_partial_block_761: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_761: - - or r13,r13 - je NEAR $L$_after_reduction_761 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_761: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_13_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_762 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_762 - -$L$_16_blocks_overflow_762: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_762: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_763 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_763 -$L$_small_initial_partial_block_763: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_763: - - or r13,r13 - je NEAR $L$_after_reduction_763 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_763: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_14_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_764 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_764 - -$L$_16_blocks_overflow_764: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_764: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_765 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_765 -$L$_small_initial_partial_block_765: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_765: - - or r13,r13 - je NEAR $L$_after_reduction_765 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_765: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_15_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_766 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_766 - -$L$_16_blocks_overflow_766: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_766: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_767 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_767 -$L$_small_initial_partial_block_767: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_767: - - or r13,r13 - je NEAR $L$_after_reduction_767 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_767: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_16_737: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_768 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_768 - -$L$_16_blocks_overflow_768: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_768: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_769: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_769: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_769: - jmp NEAR $L$_last_blocks_done_737 -$L$_last_num_blocks_is_0_737: - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_737: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_659 - -$L$_message_below_32_blocks_659: - - - sub r13,256 - add r11,256 - mov r10d,r13d - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_770 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 -$L$_skip_hkeys_precomputation_770: - mov r14,1 - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_771 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_771 - jb NEAR $L$_last_num_blocks_is_7_1_771 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_771 - jb NEAR $L$_last_num_blocks_is_11_9_771 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_771 - ja NEAR $L$_last_num_blocks_is_16_771 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_771 - jmp NEAR $L$_last_num_blocks_is_13_771 - -$L$_last_num_blocks_is_11_9_771: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_771 - ja NEAR $L$_last_num_blocks_is_11_771 - jmp NEAR $L$_last_num_blocks_is_9_771 - -$L$_last_num_blocks_is_7_1_771: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_771 - jb NEAR $L$_last_num_blocks_is_3_1_771 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_771 - je NEAR $L$_last_num_blocks_is_6_771 - jmp NEAR $L$_last_num_blocks_is_5_771 - -$L$_last_num_blocks_is_3_1_771: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_771 - je NEAR $L$_last_num_blocks_is_2_771 -$L$_last_num_blocks_is_1_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_772 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_772 - -$L$_16_blocks_overflow_772: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_772: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_773 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_773 -$L$_small_initial_partial_block_773: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_773 -$L$_small_initial_compute_done_773: -$L$_after_reduction_773: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_2_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_774 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_774 - -$L$_16_blocks_overflow_774: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_774: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_775 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_775 -$L$_small_initial_partial_block_775: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_775: - - or r13,r13 - je NEAR $L$_after_reduction_775 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_775: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_3_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_776 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_776 - -$L$_16_blocks_overflow_776: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_776: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_777 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_777 -$L$_small_initial_partial_block_777: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_777: - - or r13,r13 - je NEAR $L$_after_reduction_777 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_777: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_4_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_778 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_778 - -$L$_16_blocks_overflow_778: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_778: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_779 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_779 -$L$_small_initial_partial_block_779: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_779: - - or r13,r13 - je NEAR $L$_after_reduction_779 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_779: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_5_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_780 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_780 - -$L$_16_blocks_overflow_780: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_780: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_781 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_781 -$L$_small_initial_partial_block_781: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_781: - - or r13,r13 - je NEAR $L$_after_reduction_781 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_781: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_6_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_782 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_782 - -$L$_16_blocks_overflow_782: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_782: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_783 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_783 -$L$_small_initial_partial_block_783: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_783: - - or r13,r13 - je NEAR $L$_after_reduction_783 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_783: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_7_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_784 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_784 - -$L$_16_blocks_overflow_784: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_784: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_785 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_785 -$L$_small_initial_partial_block_785: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_785: - - or r13,r13 - je NEAR $L$_after_reduction_785 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_785: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_8_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_786 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_786 - -$L$_16_blocks_overflow_786: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_786: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_787 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_787 -$L$_small_initial_partial_block_787: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_787: - - or r13,r13 - je NEAR $L$_after_reduction_787 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_787: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_9_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_788 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_788 - -$L$_16_blocks_overflow_788: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_788: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_789 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_789 -$L$_small_initial_partial_block_789: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_789: - - or r13,r13 - je NEAR $L$_after_reduction_789 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_789: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_10_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_790 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_790 - -$L$_16_blocks_overflow_790: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_790: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_791 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_791 -$L$_small_initial_partial_block_791: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_791: - - or r13,r13 - je NEAR $L$_after_reduction_791 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_791: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_11_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_792 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_792 - -$L$_16_blocks_overflow_792: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_792: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_793 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_793 -$L$_small_initial_partial_block_793: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_793: - - or r13,r13 - je NEAR $L$_after_reduction_793 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_793: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_12_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_794 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_794 - -$L$_16_blocks_overflow_794: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_794: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_795 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_795 -$L$_small_initial_partial_block_795: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_795: - - or r13,r13 - je NEAR $L$_after_reduction_795 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_795: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_13_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_796 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_796 - -$L$_16_blocks_overflow_796: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_796: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_797 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_797 -$L$_small_initial_partial_block_797: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_797: - - or r13,r13 - je NEAR $L$_after_reduction_797 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_797: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_14_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_798 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_798 - -$L$_16_blocks_overflow_798: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_798: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_799 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_799 -$L$_small_initial_partial_block_799: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_799: - - or r13,r13 - je NEAR $L$_after_reduction_799 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_799: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_15_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_800 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_800 - -$L$_16_blocks_overflow_800: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_800: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_801 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_801 -$L$_small_initial_partial_block_801: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_801: - - or r13,r13 - je NEAR $L$_after_reduction_801 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_801: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_16_771: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_802 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_802 - -$L$_16_blocks_overflow_802: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_802: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_803: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_803: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_803: - jmp NEAR $L$_last_blocks_done_771 -$L$_last_num_blocks_is_0_771: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_771: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_659 - -$L$_message_below_equal_16_blocks_659: - - - mov r12d,r13d - add r12d,15 - shr r12d,4 - cmp r12,8 - je NEAR $L$_small_initial_num_blocks_is_8_804 - jl NEAR $L$_small_initial_num_blocks_is_7_1_804 - - - cmp r12,12 - je NEAR $L$_small_initial_num_blocks_is_12_804 - jl NEAR $L$_small_initial_num_blocks_is_11_9_804 - - - cmp r12,16 - je NEAR $L$_small_initial_num_blocks_is_16_804 - cmp r12,15 - je NEAR $L$_small_initial_num_blocks_is_15_804 - cmp r12,14 - je NEAR $L$_small_initial_num_blocks_is_14_804 - jmp NEAR $L$_small_initial_num_blocks_is_13_804 - -$L$_small_initial_num_blocks_is_11_9_804: - - cmp r12,11 - je NEAR $L$_small_initial_num_blocks_is_11_804 - cmp r12,10 - je NEAR $L$_small_initial_num_blocks_is_10_804 - jmp NEAR $L$_small_initial_num_blocks_is_9_804 - -$L$_small_initial_num_blocks_is_7_1_804: - cmp r12,4 - je NEAR $L$_small_initial_num_blocks_is_4_804 - jl NEAR $L$_small_initial_num_blocks_is_3_1_804 - - cmp r12,7 - je NEAR $L$_small_initial_num_blocks_is_7_804 - cmp r12,6 - je NEAR $L$_small_initial_num_blocks_is_6_804 - jmp NEAR $L$_small_initial_num_blocks_is_5_804 - -$L$_small_initial_num_blocks_is_3_1_804: - - cmp r12,3 - je NEAR $L$_small_initial_num_blocks_is_3_804 - cmp r12,2 - je NEAR $L$_small_initial_num_blocks_is_2_804 - - - - - -$L$_small_initial_num_blocks_is_1_804: - vmovdqa64 xmm29,XMMWORD[SHUF_MASK] - vpaddd xmm0,xmm2,XMMWORD[ONE] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,0 - vpshufb xmm0,xmm0,xmm29 - vmovdqu8 xmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast xmm0,xmm0,xmm15 - vpxorq xmm0,xmm0,xmm6 - vextracti32x4 xmm12,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm6,xmm6,xmm29 - vextracti32x4 xmm13,zmm6,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_805 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_805 -$L$_small_initial_partial_block_805: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm13 - - jmp NEAR $L$_after_reduction_805 -$L$_small_initial_compute_done_805: -$L$_after_reduction_805: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_2_804: - vmovdqa64 ymm29,YMMWORD[SHUF_MASK] - vshufi64x2 ymm0,ymm2,ymm2,0 - vpaddd ymm0,ymm0,YMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,1 - vpshufb ymm0,ymm0,ymm29 - vmovdqu8 ymm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast ymm0,ymm0,ymm15 - vpxorq ymm0,ymm0,ymm6 - vextracti32x4 xmm12,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm6,ymm6,ymm29 - vextracti32x4 xmm13,zmm6,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_806 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_806 -$L$_small_initial_partial_block_806: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_806: - - or r13,r13 - je NEAR $L$_after_reduction_806 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_806: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_3_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,2 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm6,zmm29 - vextracti32x4 xmm13,zmm6,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_807 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_807 -$L$_small_initial_partial_block_807: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_807: - - or r13,r13 - je NEAR $L$_after_reduction_807 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_807: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_4_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,3 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm6,zmm29 - vextracti32x4 xmm13,zmm6,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_808 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_808 -$L$_small_initial_partial_block_808: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_808: - - or r13,r13 - je NEAR $L$_after_reduction_808 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_808: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_5_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 xmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast xmm3,xmm3,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq xmm3,xmm3,xmm7 - vextracti32x4 xmm12,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb xmm7,xmm7,xmm29 - vextracti32x4 xmm13,zmm7,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_809 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_809 -$L$_small_initial_partial_block_809: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_809: - - or r13,r13 - je NEAR $L$_after_reduction_809 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_809: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_6_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 ymm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast ymm3,ymm3,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq ymm3,ymm3,ymm7 - vextracti32x4 xmm12,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb ymm7,ymm7,ymm29 - vextracti32x4 xmm13,zmm7,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_810 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_810 -$L$_small_initial_partial_block_810: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_810: - - or r13,r13 - je NEAR $L$_after_reduction_810 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_810: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_7_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vextracti32x4 xmm13,zmm7,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_811 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_811 -$L$_small_initial_partial_block_811: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_811: - - or r13,r13 - je NEAR $L$_after_reduction_811 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_811: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_8_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vextracti32x4 xmm13,zmm7,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_812 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_812 -$L$_small_initial_partial_block_812: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_812: - - or r13,r13 - je NEAR $L$_after_reduction_812 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_812: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_9_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast xmm4,xmm4,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq xmm4,xmm4,xmm10 - vextracti32x4 xmm12,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb xmm10,xmm10,xmm29 - vextracti32x4 xmm13,zmm10,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_813 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_813 -$L$_small_initial_partial_block_813: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_813: - - or r13,r13 - je NEAR $L$_after_reduction_813 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_813: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_10_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast ymm4,ymm4,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq ymm4,ymm4,ymm10 - vextracti32x4 xmm12,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb ymm10,ymm10,ymm29 - vextracti32x4 xmm13,zmm10,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_814 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_814 -$L$_small_initial_partial_block_814: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_814: - - or r13,r13 - je NEAR $L$_after_reduction_814 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_814: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_11_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vextracti32x4 xmm13,zmm10,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_815 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_815 -$L$_small_initial_partial_block_815: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_815: - - or r13,r13 - je NEAR $L$_after_reduction_815 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_815: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_12_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vextracti32x4 xmm13,zmm10,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_816 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_816 -$L$_small_initial_partial_block_816: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_816: - - or r13,r13 - je NEAR $L$_after_reduction_816 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_816: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_13_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast xmm5,xmm5,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq xmm5,xmm5,xmm11 - vextracti32x4 xmm12,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb xmm11,xmm11,xmm29 - vextracti32x4 xmm13,zmm11,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_817 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_817 -$L$_small_initial_partial_block_817: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_817: - - or r13,r13 - je NEAR $L$_after_reduction_817 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_817: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_14_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast ymm5,ymm5,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq ymm5,ymm5,ymm11 - vextracti32x4 xmm12,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb ymm11,ymm11,ymm29 - vextracti32x4 xmm13,zmm11,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_818 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_818 -$L$_small_initial_partial_block_818: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_818: - - or r13,r13 - je NEAR $L$_after_reduction_818 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_818: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_15_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vextracti32x4 xmm13,zmm11,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_819 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_819 -$L$_small_initial_partial_block_819: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_819: - - or r13,r13 - je NEAR $L$_after_reduction_819 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_819: - jmp NEAR $L$_small_initial_blocks_encrypted_804 -$L$_small_initial_num_blocks_is_16_804: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vextracti32x4 xmm13,zmm11,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_820: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_820: - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_820: -$L$_small_initial_blocks_encrypted_804: -$L$_ghash_done_659: - vmovdqu64 XMMWORD[rdx],xmm2 - vmovdqu64 XMMWORD[64+rdx],xmm14 -$L$_enc_dec_done_659: - jmp NEAR $L$exit_gcm_decrypt -ALIGN 32 -$L$aes_gcm_decrypt_256_avx512: - cmp QWORD[112+rbp],0 - je NEAR $L$_enc_dec_done_821 - xor r14,r14 - vmovdqu64 xmm14,XMMWORD[64+rdx] - - mov r11,QWORD[r8] - or r11,r11 - je NEAR $L$_partial_block_done_822 - mov r10d,16 - lea r12,[byte_len_to_mask_table] - cmp QWORD[112+rbp],r10 - cmovc r10,QWORD[112+rbp] - add r12,r10 - add r12,r10 - kmovw k1,[r12] - vmovdqu8 xmm0{k1}{z},[r9] - - vmovdqu64 xmm3,XMMWORD[16+rdx] - vmovdqu64 xmm4,XMMWORD[336+rdx] - - - - lea r12,[SHIFT_MASK] - add r12,r11 - vmovdqu64 xmm5,XMMWORD[r12] - vpshufb xmm3,xmm3,xmm5 - - vmovdqa64 xmm6,xmm0 - vpxorq xmm3,xmm3,xmm0 - - - mov r13,QWORD[112+rbp] - add r13,r11 - sub r13,16 - jge NEAR $L$_no_extra_mask_822 - sub r12,r13 -$L$_no_extra_mask_822: - - - - vmovdqu64 xmm0,XMMWORD[16+r12] - vpand xmm3,xmm3,xmm0 - vpand xmm6,xmm6,xmm0 - vpshufb xmm6,xmm6,XMMWORD[SHUF_MASK] - vpshufb xmm6,xmm6,xmm5 - vpxorq xmm14,xmm14,xmm6 - cmp r13,0 - jl NEAR $L$_partial_incomplete_822 - - vpclmulqdq xmm7,xmm14,xmm4,0x11 - vpclmulqdq xmm10,xmm14,xmm4,0x00 - vpclmulqdq xmm11,xmm14,xmm4,0x01 - vpclmulqdq xmm14,xmm14,xmm4,0x10 - vpxorq xmm14,xmm14,xmm11 - - vpsrldq xmm11,xmm14,8 - vpslldq xmm14,xmm14,8 - vpxorq xmm7,xmm7,xmm11 - vpxorq xmm14,xmm14,xmm10 - - - - vmovdqu64 xmm11,XMMWORD[POLY2] - - vpclmulqdq xmm10,xmm11,xmm14,0x01 - vpslldq xmm10,xmm10,8 - vpxorq xmm14,xmm14,xmm10 - - - - vpclmulqdq xmm10,xmm11,xmm14,0x00 - vpsrldq xmm10,xmm10,4 - vpclmulqdq xmm14,xmm11,xmm14,0x10 - vpslldq xmm14,xmm14,4 - - vpternlogq xmm14,xmm7,xmm10,0x96 - - mov QWORD[r8],0 - - mov r12,r11 - mov r11,16 - sub r11,r12 - jmp NEAR $L$_enc_dec_done_822 - -$L$_partial_incomplete_822: - mov r12,QWORD[112+rbp] - add QWORD[r8],r12 - mov r11,QWORD[112+rbp] - -$L$_enc_dec_done_822: - - - lea r12,[byte_len_to_mask_table] - kmovw k1,[r11*2+r12] - vmovdqu64 XMMWORD[64+rdx],xmm14 - mov r12,QWORD[120+rbp] - vmovdqu8 XMMWORD[r12]{k1},xmm3 -$L$_partial_block_done_822: - vmovdqu64 xmm2,XMMWORD[rdx] - mov r13,QWORD[112+rbp] - sub r13,r11 - je NEAR $L$_enc_dec_done_821 - cmp r13,256 - jbe NEAR $L$_message_below_equal_16_blocks_821 - - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vmovdqa64 zmm27,ZMMWORD[ddq_addbe_4444] - vmovdqa64 zmm28,ZMMWORD[ddq_addbe_1234] - - - - - - - vmovd r15d,xmm2 - and r15d,255 - - vshufi64x2 zmm2,zmm2,zmm2,0 - vpshufb zmm2,zmm2,zmm29 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_823 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_823 -$L$_next_16_overflow_823: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_823: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[192+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[176+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[192+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[208+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[224+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm12 - - vpshufb zmm7,zmm0,zmm29 - vpshufb zmm10,zmm3,zmm29 - vpshufb zmm11,zmm4,zmm29 - vpshufb zmm12,zmm5,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm7 - vmovdqa64 ZMMWORD[832+rsp],zmm10 - vmovdqa64 ZMMWORD[896+rsp],zmm11 - vmovdqa64 ZMMWORD[960+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_824 - - vmovdqu64 zmm0,ZMMWORD[288+rdx] - vmovdqu64 ZMMWORD[704+rsp],zmm0 - - vmovdqu64 zmm3,ZMMWORD[224+rdx] - vmovdqu64 ZMMWORD[640+rsp],zmm3 - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[160+rdx] - vmovdqu64 ZMMWORD[576+rsp],zmm4 - - vmovdqu64 zmm5,ZMMWORD[96+rdx] - vmovdqu64 ZMMWORD[512+rsp],zmm5 -$L$_skip_hkeys_precomputation_824: - cmp r13,512 - jb NEAR $L$_message_below_32_blocks_821 - - - - cmp r15b,240 - jae NEAR $L$_next_16_overflow_825 - vpaddd zmm7,zmm2,zmm28 - vpaddd zmm10,zmm7,zmm27 - vpaddd zmm11,zmm10,zmm27 - vpaddd zmm12,zmm11,zmm27 - jmp NEAR $L$_next_16_ok_825 -$L$_next_16_overflow_825: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm12,ZMMWORD[ddq_add_4444] - vpaddd zmm7,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm10,zmm7,zmm12 - vpaddd zmm11,zmm10,zmm12 - vpaddd zmm12,zmm11,zmm12 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vpshufb zmm12,zmm12,zmm29 -$L$_next_16_ok_825: - vshufi64x2 zmm2,zmm12,zmm12,255 - add r15b,16 - - vmovdqu8 zmm0,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm3,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm4,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm5,ZMMWORD[448+r11*1+r9] - - - vbroadcastf64x2 zmm6,ZMMWORD[rcx] - vpxorq zmm7,zmm7,zmm6 - vpxorq zmm10,zmm10,zmm6 - vpxorq zmm11,zmm11,zmm6 - vpxorq zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[16+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[32+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[48+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[64+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[80+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[96+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[112+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[128+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[144+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[160+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[176+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[192+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[208+rcx] - vaesenc zmm7,zmm7,zmm6 - vaesenc zmm10,zmm10,zmm6 - vaesenc zmm11,zmm11,zmm6 - vaesenc zmm12,zmm12,zmm6 - vbroadcastf64x2 zmm6,ZMMWORD[224+rcx] - vaesenclast zmm7,zmm7,zmm6 - vaesenclast zmm10,zmm10,zmm6 - vaesenclast zmm11,zmm11,zmm6 - vaesenclast zmm12,zmm12,zmm6 - - - vpxorq zmm7,zmm7,zmm0 - vpxorq zmm10,zmm10,zmm3 - vpxorq zmm11,zmm11,zmm4 - vpxorq zmm12,zmm12,zmm5 - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm7 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm10 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm11 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm12 - - vpshufb zmm7,zmm0,zmm29 - vpshufb zmm10,zmm3,zmm29 - vpshufb zmm11,zmm4,zmm29 - vpshufb zmm12,zmm5,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm7 - vmovdqa64 ZMMWORD[1088+rsp],zmm10 - vmovdqa64 ZMMWORD[1152+rsp],zmm11 - vmovdqa64 ZMMWORD[1216+rsp],zmm12 - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_826 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[192+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[128+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[64+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[rsp],zmm5 -$L$_skip_hkeys_precomputation_826: - mov r14,1 - add r11,512 - sub r13,512 - - cmp r13,768 - jb NEAR $L$_no_more_big_nblocks_821 -$L$_encrypt_big_nblocks_821: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_827 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_827 -$L$_16_blocks_overflow_827: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_827: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_828 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_828 -$L$_16_blocks_overflow_828: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_828: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_829 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_829 -$L$_16_blocks_overflow_829: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_829: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[512+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[576+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[640+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[704+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - - - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpternlogq zmm6,zmm12,zmm15,0x96 - vpxorq zmm6,zmm6,zmm24 - vpternlogq zmm7,zmm13,zmm10,0x96 - vpxorq zmm7,zmm7,zmm25 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vextracti64x4 ymm12,zmm6,1 - vpxorq ymm6,ymm6,ymm12 - vextracti32x4 xmm12,ymm6,1 - vpxorq xmm6,xmm6,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm6,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[512+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[576+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[640+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[704+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1024+rsp],zmm0 - vmovdqa64 ZMMWORD[1088+rsp],zmm3 - vmovdqa64 ZMMWORD[1152+rsp],zmm4 - vmovdqa64 ZMMWORD[1216+rsp],zmm5 - vmovdqa64 zmm14,zmm6 - - add r11,768 - sub r13,768 - cmp r13,768 - jae NEAR $L$_encrypt_big_nblocks_821 - -$L$_no_more_big_nblocks_821: - - cmp r13,512 - jae NEAR $L$_encrypt_32_blocks_821 - - cmp r13,256 - jae NEAR $L$_encrypt_16_blocks_821 -$L$_encrypt_0_blocks_ghash_32_821: - mov r10d,r13d - and r10d,~15 - mov ebx,256 - sub ebx,r10d - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - add ebx,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_830 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_830 - jb NEAR $L$_last_num_blocks_is_7_1_830 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_830 - jb NEAR $L$_last_num_blocks_is_11_9_830 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_830 - ja NEAR $L$_last_num_blocks_is_16_830 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_830 - jmp NEAR $L$_last_num_blocks_is_13_830 - -$L$_last_num_blocks_is_11_9_830: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_830 - ja NEAR $L$_last_num_blocks_is_11_830 - jmp NEAR $L$_last_num_blocks_is_9_830 - -$L$_last_num_blocks_is_7_1_830: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_830 - jb NEAR $L$_last_num_blocks_is_3_1_830 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_830 - je NEAR $L$_last_num_blocks_is_6_830 - jmp NEAR $L$_last_num_blocks_is_5_830 - -$L$_last_num_blocks_is_3_1_830: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_830 - je NEAR $L$_last_num_blocks_is_2_830 -$L$_last_num_blocks_is_1_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_831 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_831 - -$L$_16_blocks_overflow_831: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_831: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_832 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_832 -$L$_small_initial_partial_block_832: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_832 -$L$_small_initial_compute_done_832: -$L$_after_reduction_832: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_2_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_833 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_833 - -$L$_16_blocks_overflow_833: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_833: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_834 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_834 -$L$_small_initial_partial_block_834: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_834: - - or r13,r13 - je NEAR $L$_after_reduction_834 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_834: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_3_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_835 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_835 - -$L$_16_blocks_overflow_835: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_835: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_836 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_836 -$L$_small_initial_partial_block_836: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_836: - - or r13,r13 - je NEAR $L$_after_reduction_836 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_836: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_4_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_837 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_837 - -$L$_16_blocks_overflow_837: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_837: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_838 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_838 -$L$_small_initial_partial_block_838: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_838: - - or r13,r13 - je NEAR $L$_after_reduction_838 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_838: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_5_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_839 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_839 - -$L$_16_blocks_overflow_839: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_839: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_840 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_840 -$L$_small_initial_partial_block_840: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_840: - - or r13,r13 - je NEAR $L$_after_reduction_840 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_840: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_6_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_841 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_841 - -$L$_16_blocks_overflow_841: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_841: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_842 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_842 -$L$_small_initial_partial_block_842: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_842: - - or r13,r13 - je NEAR $L$_after_reduction_842 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_842: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_7_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_843 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_843 - -$L$_16_blocks_overflow_843: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_843: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_844 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_844 -$L$_small_initial_partial_block_844: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_844: - - or r13,r13 - je NEAR $L$_after_reduction_844 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_844: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_8_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_845 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_845 - -$L$_16_blocks_overflow_845: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_845: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_846 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_846 -$L$_small_initial_partial_block_846: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_846: - - or r13,r13 - je NEAR $L$_after_reduction_846 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_846: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_9_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_847 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_847 - -$L$_16_blocks_overflow_847: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_847: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_848 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_848 -$L$_small_initial_partial_block_848: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_848: - - or r13,r13 - je NEAR $L$_after_reduction_848 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_848: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_10_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_849 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_849 - -$L$_16_blocks_overflow_849: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_849: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_850 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_850 -$L$_small_initial_partial_block_850: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_850: - - or r13,r13 - je NEAR $L$_after_reduction_850 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_850: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_11_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_851 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_851 - -$L$_16_blocks_overflow_851: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_851: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_852 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_852 -$L$_small_initial_partial_block_852: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_852: - - or r13,r13 - je NEAR $L$_after_reduction_852 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_852: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_12_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_853 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_853 - -$L$_16_blocks_overflow_853: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_853: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_854 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_854 -$L$_small_initial_partial_block_854: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_854: - - or r13,r13 - je NEAR $L$_after_reduction_854 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_854: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_13_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_855 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_855 - -$L$_16_blocks_overflow_855: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_855: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_856 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_856 -$L$_small_initial_partial_block_856: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_856: - - or r13,r13 - je NEAR $L$_after_reduction_856 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_856: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_14_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_857 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_857 - -$L$_16_blocks_overflow_857: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_857: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_858 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_858 -$L$_small_initial_partial_block_858: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_858: - - or r13,r13 - je NEAR $L$_after_reduction_858 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_858: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_15_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_859 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_859 - -$L$_16_blocks_overflow_859: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_859: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_860 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_860 -$L$_small_initial_partial_block_860: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_860: - - or r13,r13 - je NEAR $L$_after_reduction_860 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_860: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_16_830: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_861 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_861 - -$L$_16_blocks_overflow_861: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_861: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm24,zmm14,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_862: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_862: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_862: - jmp NEAR $L$_last_blocks_done_830 -$L$_last_num_blocks_is_0_830: - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_830: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_821 -$L$_encrypt_32_blocks_821: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_863 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_863 -$L$_16_blocks_overflow_863: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_863: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_864 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_864 -$L$_16_blocks_overflow_864: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_864: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1024+rsp] - vmovdqu64 zmm1,ZMMWORD[256+rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[320+rsp] - vmovdqa64 zmm22,ZMMWORD[1088+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[384+rsp] - vmovdqa64 zmm8,ZMMWORD[1152+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[448+rsp] - vmovdqa64 zmm22,ZMMWORD[1216+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[256+r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[320+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[384+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[448+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm26,zmm10,zmm15,0x96 - vpternlogq zmm24,zmm6,zmm12,0x96 - vpternlogq zmm25,zmm7,zmm13,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[256+r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[320+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[384+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[448+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[768+rsp],zmm0 - vmovdqa64 ZMMWORD[832+rsp],zmm3 - vmovdqa64 ZMMWORD[896+rsp],zmm4 - vmovdqa64 ZMMWORD[960+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - - sub r13,512 - add r11,512 - mov r10d,r13d - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_865 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_865 - jb NEAR $L$_last_num_blocks_is_7_1_865 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_865 - jb NEAR $L$_last_num_blocks_is_11_9_865 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_865 - ja NEAR $L$_last_num_blocks_is_16_865 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_865 - jmp NEAR $L$_last_num_blocks_is_13_865 - -$L$_last_num_blocks_is_11_9_865: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_865 - ja NEAR $L$_last_num_blocks_is_11_865 - jmp NEAR $L$_last_num_blocks_is_9_865 - -$L$_last_num_blocks_is_7_1_865: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_865 - jb NEAR $L$_last_num_blocks_is_3_1_865 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_865 - je NEAR $L$_last_num_blocks_is_6_865 - jmp NEAR $L$_last_num_blocks_is_5_865 - -$L$_last_num_blocks_is_3_1_865: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_865 - je NEAR $L$_last_num_blocks_is_2_865 -$L$_last_num_blocks_is_1_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_866 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_866 - -$L$_16_blocks_overflow_866: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_866: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_867 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_867 -$L$_small_initial_partial_block_867: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_867 -$L$_small_initial_compute_done_867: -$L$_after_reduction_867: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_2_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_868 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_868 - -$L$_16_blocks_overflow_868: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_868: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_869 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_869 -$L$_small_initial_partial_block_869: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_869: - - or r13,r13 - je NEAR $L$_after_reduction_869 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_869: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_3_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_870 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_870 - -$L$_16_blocks_overflow_870: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_870: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_871 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_871 -$L$_small_initial_partial_block_871: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_871: - - or r13,r13 - je NEAR $L$_after_reduction_871 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_871: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_4_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_872 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_872 - -$L$_16_blocks_overflow_872: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_872: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_873 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_873 -$L$_small_initial_partial_block_873: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_873: - - or r13,r13 - je NEAR $L$_after_reduction_873 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_873: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_5_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_874 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_874 - -$L$_16_blocks_overflow_874: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_874: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_875 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_875 -$L$_small_initial_partial_block_875: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_875: - - or r13,r13 - je NEAR $L$_after_reduction_875 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_875: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_6_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_876 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_876 - -$L$_16_blocks_overflow_876: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_876: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_877 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_877 -$L$_small_initial_partial_block_877: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_877: - - or r13,r13 - je NEAR $L$_after_reduction_877 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_877: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_7_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_878 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_878 - -$L$_16_blocks_overflow_878: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_878: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_879 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_879 -$L$_small_initial_partial_block_879: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_879: - - or r13,r13 - je NEAR $L$_after_reduction_879 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_879: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_8_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_880 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_880 - -$L$_16_blocks_overflow_880: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_880: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_881 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_881 -$L$_small_initial_partial_block_881: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_881: - - or r13,r13 - je NEAR $L$_after_reduction_881 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_881: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_9_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_882 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_882 - -$L$_16_blocks_overflow_882: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_882: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_883 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_883 -$L$_small_initial_partial_block_883: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_883: - - or r13,r13 - je NEAR $L$_after_reduction_883 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_883: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_10_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_884 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_884 - -$L$_16_blocks_overflow_884: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_884: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_885 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_885 -$L$_small_initial_partial_block_885: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_885: - - or r13,r13 - je NEAR $L$_after_reduction_885 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_885: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_11_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_886 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_886 - -$L$_16_blocks_overflow_886: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_886: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_887 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_887 -$L$_small_initial_partial_block_887: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_887: - - or r13,r13 - je NEAR $L$_after_reduction_887 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_887: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_12_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_888 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_888 - -$L$_16_blocks_overflow_888: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_888: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_889 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_889 -$L$_small_initial_partial_block_889: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_889: - - or r13,r13 - je NEAR $L$_after_reduction_889 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_889: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_13_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_890 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_890 - -$L$_16_blocks_overflow_890: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_890: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_891 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_891 -$L$_small_initial_partial_block_891: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_891: - - or r13,r13 - je NEAR $L$_after_reduction_891 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_891: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_14_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_892 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_892 - -$L$_16_blocks_overflow_892: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_892: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_893 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_893 -$L$_small_initial_partial_block_893: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_893: - - or r13,r13 - je NEAR $L$_after_reduction_893 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_893: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_15_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_894 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_894 - -$L$_16_blocks_overflow_894: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_894: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_895 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_895 -$L$_small_initial_partial_block_895: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_895: - - or r13,r13 - je NEAR $L$_after_reduction_895 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_895: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_16_865: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_896 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_896 - -$L$_16_blocks_overflow_896: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_896: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_897: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_897: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_897: - jmp NEAR $L$_last_blocks_done_865 -$L$_last_num_blocks_is_0_865: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_865: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_821 -$L$_encrypt_16_blocks_821: - cmp r15b,240 - jae NEAR $L$_16_blocks_overflow_898 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_898 -$L$_16_blocks_overflow_898: - vpshufb zmm2,zmm2,zmm29 - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_898: - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rsp] - - - - - vshufi64x2 zmm2,zmm5,zmm5,255 - add r15b,16 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - - - - - - - - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - - vpclmulqdq zmm6,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - - vpternlogq zmm6,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - - - - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21,ZMMWORD[192+r11*1+r9] - - - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm26,zmm10,zmm15 - vpxorq zmm24,zmm6,zmm12 - vpxorq zmm25,zmm7,zmm13 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - - - - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - - - - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10],zmm5 - vpshufb zmm0,zmm17,zmm29 - vpshufb zmm3,zmm19,zmm29 - vpshufb zmm4,zmm20,zmm29 - vpshufb zmm5,zmm21,zmm29 - vmovdqa64 ZMMWORD[1280+rsp],zmm0 - vmovdqa64 ZMMWORD[1344+rsp],zmm3 - vmovdqa64 ZMMWORD[1408+rsp],zmm4 - vmovdqa64 ZMMWORD[1472+rsp],zmm5 - vmovdqa64 zmm13,ZMMWORD[1024+rsp] - vmovdqu64 zmm12,ZMMWORD[256+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1088+rsp] - vmovdqu64 zmm12,ZMMWORD[320+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1152+rsp] - vmovdqu64 zmm12,ZMMWORD[384+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1216+rsp] - vmovdqu64 zmm12,ZMMWORD[448+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - sub r13,256 - add r11,256 - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_899 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_899 - jb NEAR $L$_last_num_blocks_is_7_1_899 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_899 - jb NEAR $L$_last_num_blocks_is_11_9_899 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_899 - ja NEAR $L$_last_num_blocks_is_16_899 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_899 - jmp NEAR $L$_last_num_blocks_is_13_899 - -$L$_last_num_blocks_is_11_9_899: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_899 - ja NEAR $L$_last_num_blocks_is_11_899 - jmp NEAR $L$_last_num_blocks_is_9_899 - -$L$_last_num_blocks_is_7_1_899: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_899 - jb NEAR $L$_last_num_blocks_is_3_1_899 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_899 - je NEAR $L$_last_num_blocks_is_6_899 - jmp NEAR $L$_last_num_blocks_is_5_899 - -$L$_last_num_blocks_is_3_1_899: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_899 - je NEAR $L$_last_num_blocks_is_2_899 -$L$_last_num_blocks_is_1_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_900 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_900 - -$L$_16_blocks_overflow_900: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_900: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc xmm0,xmm0,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc xmm0,xmm0,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_901 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_901 -$L$_small_initial_partial_block_901: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_901 -$L$_small_initial_compute_done_901: -$L$_after_reduction_901: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_2_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_902 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_902 - -$L$_16_blocks_overflow_902: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_902: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc ymm0,ymm0,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc ymm0,ymm0,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_903 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_903 -$L$_small_initial_partial_block_903: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_903: - - or r13,r13 - je NEAR $L$_after_reduction_903 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_903: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_3_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_904 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_904 - -$L$_16_blocks_overflow_904: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_904: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_905 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_905 -$L$_small_initial_partial_block_905: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_905: - - or r13,r13 - je NEAR $L$_after_reduction_905 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_905: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_4_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_906 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_906 - -$L$_16_blocks_overflow_906: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_906: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_907 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_907 -$L$_small_initial_partial_block_907: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_907: - - or r13,r13 - je NEAR $L$_after_reduction_907 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_907: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_5_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_908 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_908 - -$L$_16_blocks_overflow_908: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_908: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_909 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_909 -$L$_small_initial_partial_block_909: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_909: - - or r13,r13 - je NEAR $L$_after_reduction_909 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_909: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_6_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_910 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_910 - -$L$_16_blocks_overflow_910: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_910: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_911 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_911 -$L$_small_initial_partial_block_911: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_911: - - or r13,r13 - je NEAR $L$_after_reduction_911 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_911: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_7_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_912 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_912 - -$L$_16_blocks_overflow_912: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_912: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_913 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_913 -$L$_small_initial_partial_block_913: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_913: - - or r13,r13 - je NEAR $L$_after_reduction_913 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_913: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_8_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_914 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_914 - -$L$_16_blocks_overflow_914: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_914: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_915 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_915 -$L$_small_initial_partial_block_915: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_915: - - or r13,r13 - je NEAR $L$_after_reduction_915 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_915: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_9_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_916 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_916 - -$L$_16_blocks_overflow_916: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_916: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_917 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_917 -$L$_small_initial_partial_block_917: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_917: - - or r13,r13 - je NEAR $L$_after_reduction_917 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_917: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_10_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_918 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_918 - -$L$_16_blocks_overflow_918: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_918: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_919 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_919 -$L$_small_initial_partial_block_919: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_919: - - or r13,r13 - je NEAR $L$_after_reduction_919 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_919: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_11_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_920 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_920 - -$L$_16_blocks_overflow_920: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_920: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_921 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_921 -$L$_small_initial_partial_block_921: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_921: - - or r13,r13 - je NEAR $L$_after_reduction_921 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_921: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_12_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_922 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_922 - -$L$_16_blocks_overflow_922: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_922: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_923 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_923 -$L$_small_initial_partial_block_923: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_923: - - or r13,r13 - je NEAR $L$_after_reduction_923 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_923: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_13_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_924 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_924 - -$L$_16_blocks_overflow_924: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_924: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_925 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_925 -$L$_small_initial_partial_block_925: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_925: - - or r13,r13 - je NEAR $L$_after_reduction_925 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_925: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_14_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_926 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_926 - -$L$_16_blocks_overflow_926: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_926: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_927 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_927 -$L$_small_initial_partial_block_927: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_927: - - or r13,r13 - je NEAR $L$_after_reduction_927 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_927: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_15_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_928 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_928 - -$L$_16_blocks_overflow_928: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_928: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_929 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_929 -$L$_small_initial_partial_block_929: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_929: - - or r13,r13 - je NEAR $L$_after_reduction_929 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_929: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_16_899: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_930 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_930 - -$L$_16_blocks_overflow_930: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_930: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vmovdqa64 zmm8,ZMMWORD[1280+rsp] - vmovdqu64 zmm1,ZMMWORD[512+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[576+rsp] - vmovdqa64 zmm22,ZMMWORD[1344+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[640+rsp] - vmovdqa64 zmm8,ZMMWORD[1408+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[704+rsp] - vmovdqa64 zmm22,ZMMWORD[1472+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpternlogq zmm14,zmm24,zmm12,0x96 - vpternlogq zmm7,zmm25,zmm13,0x96 - vpternlogq zmm10,zmm26,zmm15,0x96 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vpsrldq zmm15,zmm10,8 - vpslldq zmm10,zmm10,8 - - vmovdqa64 xmm16,XMMWORD[POLY2] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vpxorq zmm14,zmm14,zmm15 - vpxorq zmm7,zmm7,zmm10 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vextracti64x4 ymm12,zmm14,1 - vpxorq ymm14,ymm14,ymm12 - vextracti32x4 xmm12,ymm14,1 - vpxorq xmm14,xmm14,xmm12 - vextracti64x4 ymm13,zmm7,1 - vpxorq ymm7,ymm7,ymm13 - vextracti32x4 xmm13,ymm7,1 - vpxorq xmm7,xmm7,xmm13 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vpclmulqdq xmm13,xmm16,xmm7,0x01 - vpslldq xmm13,xmm13,8 - vpxorq xmm13,xmm7,xmm13 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vpclmulqdq xmm12,xmm16,xmm13,0x00 - vpsrldq xmm12,xmm12,4 - vpclmulqdq xmm15,xmm16,xmm13,0x10 - vpslldq xmm15,xmm15,4 - - vpternlogq xmm14,xmm15,xmm12,0x96 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_931: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vpxorq zmm17,zmm17,zmm14 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm31 - vpxorq zmm0,zmm0,zmm8 - vpxorq zmm3,zmm3,zmm22 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_931: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_931: - jmp NEAR $L$_last_blocks_done_899 -$L$_last_num_blocks_is_0_899: - vmovdqa64 zmm13,ZMMWORD[1280+rsp] - vmovdqu64 zmm12,ZMMWORD[512+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1344+rsp] - vmovdqu64 zmm12,ZMMWORD[576+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[1408+rsp] - vmovdqu64 zmm12,ZMMWORD[640+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[1472+rsp] - vmovdqu64 zmm12,ZMMWORD[704+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_899: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_821 - -$L$_message_below_32_blocks_821: - - - sub r13,256 - add r11,256 - mov r10d,r13d - test r14,r14 - jnz NEAR $L$_skip_hkeys_precomputation_932 - vmovdqu64 zmm3,ZMMWORD[640+rsp] - - - vshufi64x2 zmm3,zmm3,zmm3,0x00 - - vmovdqu64 zmm4,ZMMWORD[576+rsp] - vmovdqu64 zmm5,ZMMWORD[512+rsp] - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[448+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[384+rsp],zmm5 - - vpclmulqdq zmm6,zmm4,zmm3,0x11 - vpclmulqdq zmm7,zmm4,zmm3,0x00 - vpclmulqdq zmm10,zmm4,zmm3,0x01 - vpclmulqdq zmm4,zmm4,zmm3,0x10 - vpxorq zmm4,zmm4,zmm10 - - vpsrldq zmm10,zmm4,8 - vpslldq zmm4,zmm4,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm4,zmm4,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm4,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm4,zmm4,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm4,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm4,zmm10,zmm4,0x10 - vpslldq zmm4,zmm4,4 - - vpternlogq zmm4,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[320+rsp],zmm4 - - vpclmulqdq zmm6,zmm5,zmm3,0x11 - vpclmulqdq zmm7,zmm5,zmm3,0x00 - vpclmulqdq zmm10,zmm5,zmm3,0x01 - vpclmulqdq zmm5,zmm5,zmm3,0x10 - vpxorq zmm5,zmm5,zmm10 - - vpsrldq zmm10,zmm5,8 - vpslldq zmm5,zmm5,8 - vpxorq zmm6,zmm6,zmm10 - vpxorq zmm5,zmm5,zmm7 - - - - vmovdqu64 zmm10,ZMMWORD[POLY2] - - vpclmulqdq zmm7,zmm10,zmm5,0x01 - vpslldq zmm7,zmm7,8 - vpxorq zmm5,zmm5,zmm7 - - - - vpclmulqdq zmm7,zmm10,zmm5,0x00 - vpsrldq zmm7,zmm7,4 - vpclmulqdq zmm5,zmm10,zmm5,0x10 - vpslldq zmm5,zmm5,4 - - vpternlogq zmm5,zmm6,zmm7,0x96 - - vmovdqu64 ZMMWORD[256+rsp],zmm5 -$L$_skip_hkeys_precomputation_932: - mov r14,1 - and r10d,~15 - mov ebx,512 - sub ebx,r10d - mov r10d,r13d - add r10d,15 - shr r10d,4 - je NEAR $L$_last_num_blocks_is_0_933 - - cmp r10d,8 - je NEAR $L$_last_num_blocks_is_8_933 - jb NEAR $L$_last_num_blocks_is_7_1_933 - - - cmp r10d,12 - je NEAR $L$_last_num_blocks_is_12_933 - jb NEAR $L$_last_num_blocks_is_11_9_933 - - - cmp r10d,15 - je NEAR $L$_last_num_blocks_is_15_933 - ja NEAR $L$_last_num_blocks_is_16_933 - cmp r10d,14 - je NEAR $L$_last_num_blocks_is_14_933 - jmp NEAR $L$_last_num_blocks_is_13_933 - -$L$_last_num_blocks_is_11_9_933: - - cmp r10d,10 - je NEAR $L$_last_num_blocks_is_10_933 - ja NEAR $L$_last_num_blocks_is_11_933 - jmp NEAR $L$_last_num_blocks_is_9_933 - -$L$_last_num_blocks_is_7_1_933: - cmp r10d,4 - je NEAR $L$_last_num_blocks_is_4_933 - jb NEAR $L$_last_num_blocks_is_3_1_933 - - cmp r10d,6 - ja NEAR $L$_last_num_blocks_is_7_933 - je NEAR $L$_last_num_blocks_is_6_933 - jmp NEAR $L$_last_num_blocks_is_5_933 - -$L$_last_num_blocks_is_3_1_933: - - cmp r10d,2 - ja NEAR $L$_last_num_blocks_is_3_933 - je NEAR $L$_last_num_blocks_is_2_933 -$L$_last_num_blocks_is_1_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,255 - jae NEAR $L$_16_blocks_overflow_934 - vpaddd xmm0,xmm2,xmm28 - jmp NEAR $L$_16_blocks_ok_934 - -$L$_16_blocks_overflow_934: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb xmm0,xmm0,xmm29 -$L$_16_blocks_ok_934: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 xmm17{k1}{z},[r11*1+r9] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc xmm0,xmm0,xmm31 - vaesenclast xmm0,xmm0,xmm30 - vpxorq xmm0,xmm0,xmm17 - vextracti32x4 xmm11,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb xmm17,xmm17,xmm29 - vextracti32x4 xmm7,zmm17,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_935 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_935 -$L$_small_initial_partial_block_935: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm0,XMMWORD[POLY2] - - - vpclmulqdq xmm3,xmm0,xmm25,0x01 - vpslldq xmm3,xmm3,8 - vpxorq xmm3,xmm25,xmm3 - - - vpclmulqdq xmm4,xmm0,xmm3,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm14,xmm0,xmm3,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm4,xmm24,0x96 - - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm7 - - jmp NEAR $L$_after_reduction_935 -$L$_small_initial_compute_done_935: -$L$_after_reduction_935: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_2_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,254 - jae NEAR $L$_16_blocks_overflow_936 - vpaddd ymm0,ymm2,ymm28 - jmp NEAR $L$_16_blocks_ok_936 - -$L$_16_blocks_overflow_936: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb ymm0,ymm0,ymm29 -$L$_16_blocks_ok_936: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 ymm17{k1}{z},[r11*1+r9] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc ymm0,ymm0,ymm31 - vaesenclast ymm0,ymm0,ymm30 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm11,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb ymm17,ymm17,ymm29 - vextracti32x4 xmm7,zmm17,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_937 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_937 -$L$_small_initial_partial_block_937: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm17,xmm1,0x01 - vpclmulqdq xmm5,xmm17,xmm1,0x10 - vpclmulqdq xmm0,xmm17,xmm1,0x11 - vpclmulqdq xmm3,xmm17,xmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_937: - - or r13,r13 - je NEAR $L$_after_reduction_937 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_937: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_3_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,253 - jae NEAR $L$_16_blocks_overflow_938 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_938 - -$L$_16_blocks_overflow_938: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_938: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_939 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_939 -$L$_small_initial_partial_block_939: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm17,ymm1,0x01 - vpclmulqdq ymm5,ymm17,ymm1,0x10 - vpclmulqdq ymm0,ymm17,ymm1,0x11 - vpclmulqdq ymm3,ymm17,ymm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_939: - - or r13,r13 - je NEAR $L$_after_reduction_939 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_939: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_4_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - kmovq k1,[rax*8+r10] - cmp r15d,252 - jae NEAR $L$_16_blocks_overflow_940 - vpaddd zmm0,zmm2,zmm28 - jmp NEAR $L$_16_blocks_ok_940 - -$L$_16_blocks_overflow_940: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpshufb zmm0,zmm0,zmm29 -$L$_16_blocks_ok_940: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm0,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17{k1}{z},[r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vpxorq zmm0,zmm0,zmm17 - vextracti32x4 xmm11,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm17{k1}{z},zmm17 - vpshufb zmm17,zmm17,zmm29 - vextracti32x4 xmm7,zmm17,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_941 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_941 -$L$_small_initial_partial_block_941: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpxorq zmm4,zmm4,zmm26 - vpxorq zmm0,zmm0,zmm24 - vpxorq zmm3,zmm3,zmm25 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_941: - - or r13,r13 - je NEAR $L$_after_reduction_941 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_941: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_5_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,251 - jae NEAR $L$_16_blocks_overflow_942 - vpaddd zmm0,zmm2,zmm28 - vpaddd xmm3,xmm0,xmm27 - jmp NEAR $L$_16_blocks_ok_942 - -$L$_16_blocks_overflow_942: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 -$L$_16_blocks_ok_942: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 xmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc xmm3,xmm3,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc xmm3,xmm3,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast xmm3,xmm3,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq xmm3,xmm3,xmm19 - vextracti32x4 xmm11,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb xmm19,xmm19,xmm29 - vextracti32x4 xmm7,zmm19,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_943 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_943 -$L$_small_initial_partial_block_943: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_943: - - or r13,r13 - je NEAR $L$_after_reduction_943 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_943: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_6_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,250 - jae NEAR $L$_16_blocks_overflow_944 - vpaddd zmm0,zmm2,zmm28 - vpaddd ymm3,ymm0,ymm27 - jmp NEAR $L$_16_blocks_ok_944 - -$L$_16_blocks_overflow_944: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 -$L$_16_blocks_ok_944: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 ymm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc ymm3,ymm3,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc ymm3,ymm3,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast ymm3,ymm3,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm11,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb ymm19,ymm19,ymm29 - vextracti32x4 xmm7,zmm19,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_945 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_945 -$L$_small_initial_partial_block_945: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm19,xmm1,0x01 - vpclmulqdq xmm5,xmm19,xmm1,0x10 - vpclmulqdq xmm0,xmm19,xmm1,0x11 - vpclmulqdq xmm3,xmm19,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_945: - - or r13,r13 - je NEAR $L$_after_reduction_945 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_945: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_7_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,249 - jae NEAR $L$_16_blocks_overflow_946 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_946 - -$L$_16_blocks_overflow_946: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_946: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_947 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_947 -$L$_small_initial_partial_block_947: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm19,ymm1,0x01 - vpclmulqdq ymm5,ymm19,ymm1,0x10 - vpclmulqdq ymm0,ymm19,ymm1,0x11 - vpclmulqdq ymm3,ymm19,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_947: - - or r13,r13 - je NEAR $L$_after_reduction_947 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_947: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_8_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,64 - kmovq k1,[rax*8+r10] - cmp r15d,248 - jae NEAR $L$_16_blocks_overflow_948 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - jmp NEAR $L$_16_blocks_ok_948 - -$L$_16_blocks_overflow_948: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 -$L$_16_blocks_ok_948: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm3,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19{k1}{z},[64+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti32x4 xmm11,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm19{k1}{z},zmm19 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vextracti32x4 xmm7,zmm19,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_949 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_949 -$L$_small_initial_partial_block_949: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm17,zmm1,0x11 - vpclmulqdq zmm22,zmm17,zmm1,0x00 - vpclmulqdq zmm30,zmm17,zmm1,0x01 - vpclmulqdq zmm31,zmm17,zmm1,0x10 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm19,zmm1,0x01 - vpclmulqdq zmm5,zmm19,zmm1,0x10 - vpclmulqdq zmm0,zmm19,zmm1,0x11 - vpclmulqdq zmm3,zmm19,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_949: - - or r13,r13 - je NEAR $L$_after_reduction_949 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_949: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_9_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,247 - jae NEAR $L$_16_blocks_overflow_950 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd xmm4,xmm3,xmm27 - jmp NEAR $L$_16_blocks_ok_950 - -$L$_16_blocks_overflow_950: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 -$L$_16_blocks_ok_950: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc xmm4,xmm4,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc xmm4,xmm4,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast xmm4,xmm4,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq xmm4,xmm4,xmm20 - vextracti32x4 xmm11,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb xmm20,xmm20,xmm29 - vextracti32x4 xmm7,zmm20,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_951 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_951 -$L$_small_initial_partial_block_951: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_951: - - or r13,r13 - je NEAR $L$_after_reduction_951 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_951: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_10_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,246 - jae NEAR $L$_16_blocks_overflow_952 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd ymm4,ymm3,ymm27 - jmp NEAR $L$_16_blocks_ok_952 - -$L$_16_blocks_overflow_952: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 -$L$_16_blocks_ok_952: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc ymm4,ymm4,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc ymm4,ymm4,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast ymm4,ymm4,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq ymm4,ymm4,ymm20 - vextracti32x4 xmm11,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb ymm20,ymm20,ymm29 - vextracti32x4 xmm7,zmm20,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_953 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_953 -$L$_small_initial_partial_block_953: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm20,xmm1,0x01 - vpclmulqdq xmm5,xmm20,xmm1,0x10 - vpclmulqdq xmm0,xmm20,xmm1,0x11 - vpclmulqdq xmm3,xmm20,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_953: - - or r13,r13 - je NEAR $L$_after_reduction_953 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_953: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_11_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,245 - jae NEAR $L$_16_blocks_overflow_954 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_954 - -$L$_16_blocks_overflow_954: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_954: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_955 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_955 -$L$_small_initial_partial_block_955: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm20,ymm1,0x01 - vpclmulqdq ymm5,ymm20,ymm1,0x10 - vpclmulqdq ymm0,ymm20,ymm1,0x11 - vpclmulqdq ymm3,ymm20,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_955: - - or r13,r13 - je NEAR $L$_after_reduction_955 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_955: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_12_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,128 - kmovq k1,[rax*8+r10] - cmp r15d,244 - jae NEAR $L$_16_blocks_overflow_956 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - jmp NEAR $L$_16_blocks_ok_956 - -$L$_16_blocks_overflow_956: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 -$L$_16_blocks_ok_956: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm4,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20{k1}{z},[128+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vextracti32x4 xmm11,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm20{k1}{z},zmm20 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vextracti32x4 xmm7,zmm20,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_957 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_957 -$L$_small_initial_partial_block_957: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vpxorq zmm8,zmm0,zmm8 - vpxorq zmm22,zmm3,zmm22 - vpxorq zmm30,zmm4,zmm30 - vpxorq zmm31,zmm5,zmm31 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm20,zmm1,0x01 - vpclmulqdq zmm5,zmm20,zmm1,0x10 - vpclmulqdq zmm0,zmm20,zmm1,0x11 - vpclmulqdq zmm3,zmm20,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_957: - - or r13,r13 - je NEAR $L$_after_reduction_957 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_957: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_13_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,243 - jae NEAR $L$_16_blocks_overflow_958 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd xmm5,xmm4,xmm27 - jmp NEAR $L$_16_blocks_ok_958 - -$L$_16_blocks_overflow_958: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 -$L$_16_blocks_ok_958: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,0 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc xmm5,xmm5,xmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc xmm5,xmm5,xmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast xmm5,xmm5,xmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq xmm5,xmm5,xmm21 - vextracti32x4 xmm11,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb xmm21,xmm21,xmm29 - vextracti32x4 xmm7,zmm21,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_959 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_959 -$L$_small_initial_partial_block_959: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[224+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[288+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - - vpxorq zmm30,zmm30,zmm26 - vpxorq zmm8,zmm8,zmm24 - vpxorq zmm22,zmm22,zmm25 - - vpxorq zmm30,zmm30,zmm31 - vpsrldq zmm4,zmm30,8 - vpslldq zmm5,zmm30,8 - vpxorq zmm0,zmm8,zmm4 - vpxorq zmm3,zmm22,zmm5 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_959: - - or r13,r13 - je NEAR $L$_after_reduction_959 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_959: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_14_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,242 - jae NEAR $L$_16_blocks_overflow_960 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd ymm5,ymm4,ymm27 - jmp NEAR $L$_16_blocks_ok_960 - -$L$_16_blocks_overflow_960: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 -$L$_16_blocks_ok_960: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,1 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc ymm5,ymm5,ymm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc ymm5,ymm5,ymm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast ymm5,ymm5,ymm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq ymm5,ymm5,ymm21 - vextracti32x4 xmm11,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb ymm21,ymm21,ymm29 - vextracti32x4 xmm7,zmm21,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_961 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_961 -$L$_small_initial_partial_block_961: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[208+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[272+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 xmm1,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm21,xmm1,0x01 - vpclmulqdq xmm5,xmm21,xmm1,0x10 - vpclmulqdq xmm0,xmm21,xmm1,0x11 - vpclmulqdq xmm3,xmm21,xmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_961: - - or r13,r13 - je NEAR $L$_after_reduction_961 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_961: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_15_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,241 - jae NEAR $L$_16_blocks_overflow_962 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_962 - -$L$_16_blocks_overflow_962: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_962: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,2 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_963 - - - - - - sub r13,16 - mov QWORD[r8],0 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_963 -$L$_small_initial_partial_block_963: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[192+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[256+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm21,ymm1,0x01 - vpclmulqdq ymm5,ymm21,ymm1,0x10 - vpclmulqdq ymm0,ymm21,ymm1,0x11 - vpclmulqdq ymm3,ymm21,ymm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_963: - - or r13,r13 - je NEAR $L$_after_reduction_963 - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_963: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_16_933: - lea r10,[byte64_len_to_mask_table] - mov rax,r13 - sub rax,192 - kmovq k1,[rax*8+r10] - cmp r15d,240 - jae NEAR $L$_16_blocks_overflow_964 - vpaddd zmm0,zmm2,zmm28 - vpaddd zmm3,zmm0,zmm27 - vpaddd zmm4,zmm3,zmm27 - vpaddd zmm5,zmm4,zmm27 - jmp NEAR $L$_16_blocks_ok_964 - -$L$_16_blocks_overflow_964: - vpshufb zmm2,zmm2,zmm29 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vmovdqa64 zmm5,ZMMWORD[ddq_add_4444] - vpaddd zmm3,zmm0,zmm5 - vpaddd zmm4,zmm3,zmm5 - vpaddd zmm5,zmm4,zmm5 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 -$L$_16_blocks_ok_964: - - - - - vbroadcastf64x2 zmm30,ZMMWORD[rcx] - vpxorq zmm8,zmm14,ZMMWORD[768+rsp] - vmovdqu64 zmm1,ZMMWORD[rbx*1+rsp] - vextracti32x4 xmm2,zmm5,3 - vshufi64x2 zmm2,zmm2,zmm2,0 - - - vbroadcastf64x2 zmm31,ZMMWORD[16+rcx] - vmovdqu64 zmm18,ZMMWORD[64+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[832+rsp] - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm30 - vpxorq zmm4,zmm4,zmm30 - vpxorq zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[32+rcx] - - - vpclmulqdq zmm14,zmm8,zmm1,0x11 - vpclmulqdq zmm7,zmm8,zmm1,0x00 - vpclmulqdq zmm10,zmm8,zmm1,0x01 - vpclmulqdq zmm11,zmm8,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[128+rbx*1+rsp] - vmovdqa64 zmm8,ZMMWORD[896+rsp] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[48+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vmovdqu64 zmm18,ZMMWORD[192+rbx*1+rsp] - vmovdqa64 zmm22,ZMMWORD[960+rsp] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[64+rcx] - - - vpclmulqdq zmm20,zmm8,zmm1,0x10 - vpclmulqdq zmm21,zmm8,zmm1,0x01 - vpclmulqdq zmm17,zmm8,zmm1,0x11 - vpclmulqdq zmm19,zmm8,zmm1,0x00 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[80+rcx] - - - vpternlogq zmm14,zmm12,zmm17,0x96 - vpternlogq zmm7,zmm13,zmm19,0x96 - vpternlogq zmm11,zmm16,zmm21,0x96 - vpternlogq zmm10,zmm15,zmm20,0x96 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[96+rcx] - vmovdqu8 zmm17,ZMMWORD[r11*1+r9] - vmovdqu8 zmm19,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm20,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm21{k1}{z},[192+r11*1+r9] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[112+rcx] - - - vpclmulqdq zmm15,zmm22,zmm18,0x10 - vpclmulqdq zmm16,zmm22,zmm18,0x01 - vpclmulqdq zmm12,zmm22,zmm18,0x11 - vpclmulqdq zmm13,zmm22,zmm18,0x00 - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[128+rcx] - vpternlogq zmm10,zmm11,zmm16,0x96 - vpxorq zmm24,zmm14,zmm12 - vpxorq zmm25,zmm7,zmm13 - vpxorq zmm26,zmm10,zmm15 - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vbroadcastf64x2 zmm31,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm30 - vaesenc zmm3,zmm3,zmm30 - vaesenc zmm4,zmm4,zmm30 - vaesenc zmm5,zmm5,zmm30 - vbroadcastf64x2 zmm30,ZMMWORD[224+rcx] - vaesenc zmm0,zmm0,zmm31 - vaesenc zmm3,zmm3,zmm31 - vaesenc zmm4,zmm4,zmm31 - vaesenc zmm5,zmm5,zmm31 - vaesenclast zmm0,zmm0,zmm30 - vaesenclast zmm3,zmm3,zmm30 - vaesenclast zmm4,zmm4,zmm30 - vaesenclast zmm5,zmm5,zmm30 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vpxorq zmm4,zmm4,zmm20 - vpxorq zmm5,zmm5,zmm21 - vextracti32x4 xmm11,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm21{k1}{z},zmm21 - vpshufb zmm17,zmm17,zmm29 - vpshufb zmm19,zmm19,zmm29 - vpshufb zmm20,zmm20,zmm29 - vpshufb zmm21,zmm21,zmm29 - vextracti32x4 xmm7,zmm21,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_965: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm11 - vmovdqu64 zmm1,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm17,zmm1,0x11 - vpclmulqdq zmm3,zmm17,zmm1,0x00 - vpclmulqdq zmm4,zmm17,zmm1,0x01 - vpclmulqdq zmm5,zmm17,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[176+rdx] - vpclmulqdq zmm8,zmm19,zmm1,0x11 - vpclmulqdq zmm22,zmm19,zmm1,0x00 - vpclmulqdq zmm30,zmm19,zmm1,0x01 - vpclmulqdq zmm31,zmm19,zmm1,0x10 - vmovdqu64 zmm1,ZMMWORD[240+rdx] - vpclmulqdq zmm17,zmm20,zmm1,0x11 - vpclmulqdq zmm19,zmm20,zmm1,0x00 - vpternlogq zmm8,zmm17,zmm0,0x96 - vpternlogq zmm22,zmm19,zmm3,0x96 - vpclmulqdq zmm17,zmm20,zmm1,0x01 - vpclmulqdq zmm19,zmm20,zmm1,0x10 - vpternlogq zmm30,zmm17,zmm4,0x96 - vpternlogq zmm31,zmm19,zmm5,0x96 - vmovdqu64 ymm1,YMMWORD[304+rdx] - vinserti64x2 zmm1,zmm1,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm21,zmm1,0x01 - vpclmulqdq zmm5,zmm21,zmm1,0x10 - vpclmulqdq zmm0,zmm21,zmm1,0x11 - vpclmulqdq zmm3,zmm21,zmm1,0x00 - - vpxorq zmm4,zmm4,zmm30 - vpternlogq zmm5,zmm26,zmm31,0x96 - vpternlogq zmm0,zmm24,zmm8,0x96 - vpternlogq zmm3,zmm25,zmm22,0x96 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm30,zmm4,8 - vpslldq zmm31,zmm4,8 - vpxorq zmm0,zmm0,zmm30 - vpxorq zmm3,zmm3,zmm31 - vextracti64x4 ymm30,zmm0,1 - vpxorq ymm0,ymm0,ymm30 - vextracti32x4 xmm30,ymm0,1 - vpxorq xmm0,xmm0,xmm30 - vextracti64x4 ymm31,zmm3,1 - vpxorq ymm3,ymm3,ymm31 - vextracti32x4 xmm31,ymm3,1 - vpxorq xmm3,xmm3,xmm31 - vmovdqa64 xmm1,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm1,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm1,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm1,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_965: - vpxorq xmm14,xmm14,xmm7 -$L$_after_reduction_965: - jmp NEAR $L$_last_blocks_done_933 -$L$_last_num_blocks_is_0_933: - vmovdqa64 zmm13,ZMMWORD[768+rsp] - vpxorq zmm13,zmm13,zmm14 - vmovdqu64 zmm12,ZMMWORD[rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[832+rsp] - vmovdqu64 zmm12,ZMMWORD[64+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - vpxorq zmm26,zmm4,zmm10 - vpxorq zmm24,zmm0,zmm6 - vpxorq zmm25,zmm3,zmm7 - vpternlogq zmm26,zmm5,zmm11,0x96 - vmovdqa64 zmm13,ZMMWORD[896+rsp] - vmovdqu64 zmm12,ZMMWORD[128+rbx*1+rsp] - vpclmulqdq zmm0,zmm13,zmm12,0x11 - vpclmulqdq zmm3,zmm13,zmm12,0x00 - vpclmulqdq zmm4,zmm13,zmm12,0x01 - vpclmulqdq zmm5,zmm13,zmm12,0x10 - vmovdqa64 zmm13,ZMMWORD[960+rsp] - vmovdqu64 zmm12,ZMMWORD[192+rbx*1+rsp] - vpclmulqdq zmm6,zmm13,zmm12,0x11 - vpclmulqdq zmm7,zmm13,zmm12,0x00 - vpclmulqdq zmm10,zmm13,zmm12,0x01 - vpclmulqdq zmm11,zmm13,zmm12,0x10 - - vpternlogq zmm26,zmm4,zmm10,0x96 - vpternlogq zmm24,zmm0,zmm6,0x96 - vpternlogq zmm25,zmm3,zmm7,0x96 - vpternlogq zmm26,zmm5,zmm11,0x96 - - vpsrldq zmm0,zmm26,8 - vpslldq zmm3,zmm26,8 - vpxorq zmm24,zmm24,zmm0 - vpxorq zmm25,zmm25,zmm3 - vextracti64x4 ymm0,zmm24,1 - vpxorq ymm24,ymm24,ymm0 - vextracti32x4 xmm0,ymm24,1 - vpxorq xmm24,xmm24,xmm0 - vextracti64x4 ymm3,zmm25,1 - vpxorq ymm25,ymm25,ymm3 - vextracti32x4 xmm3,ymm25,1 - vpxorq xmm25,xmm25,xmm3 - vmovdqa64 xmm4,XMMWORD[POLY2] - - - vpclmulqdq xmm0,xmm4,xmm25,0x01 - vpslldq xmm0,xmm0,8 - vpxorq xmm0,xmm25,xmm0 - - - vpclmulqdq xmm3,xmm4,xmm0,0x00 - vpsrldq xmm3,xmm3,4 - vpclmulqdq xmm14,xmm4,xmm0,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm3,xmm24,0x96 - -$L$_last_blocks_done_933: - vpshufb xmm2,xmm2,xmm29 - jmp NEAR $L$_ghash_done_821 - -$L$_message_below_equal_16_blocks_821: - - - mov r12d,r13d - add r12d,15 - shr r12d,4 - cmp r12,8 - je NEAR $L$_small_initial_num_blocks_is_8_966 - jl NEAR $L$_small_initial_num_blocks_is_7_1_966 - - - cmp r12,12 - je NEAR $L$_small_initial_num_blocks_is_12_966 - jl NEAR $L$_small_initial_num_blocks_is_11_9_966 - - - cmp r12,16 - je NEAR $L$_small_initial_num_blocks_is_16_966 - cmp r12,15 - je NEAR $L$_small_initial_num_blocks_is_15_966 - cmp r12,14 - je NEAR $L$_small_initial_num_blocks_is_14_966 - jmp NEAR $L$_small_initial_num_blocks_is_13_966 - -$L$_small_initial_num_blocks_is_11_9_966: - - cmp r12,11 - je NEAR $L$_small_initial_num_blocks_is_11_966 - cmp r12,10 - je NEAR $L$_small_initial_num_blocks_is_10_966 - jmp NEAR $L$_small_initial_num_blocks_is_9_966 - -$L$_small_initial_num_blocks_is_7_1_966: - cmp r12,4 - je NEAR $L$_small_initial_num_blocks_is_4_966 - jl NEAR $L$_small_initial_num_blocks_is_3_1_966 - - cmp r12,7 - je NEAR $L$_small_initial_num_blocks_is_7_966 - cmp r12,6 - je NEAR $L$_small_initial_num_blocks_is_6_966 - jmp NEAR $L$_small_initial_num_blocks_is_5_966 - -$L$_small_initial_num_blocks_is_3_1_966: - - cmp r12,3 - je NEAR $L$_small_initial_num_blocks_is_3_966 - cmp r12,2 - je NEAR $L$_small_initial_num_blocks_is_2_966 - - - - - -$L$_small_initial_num_blocks_is_1_966: - vmovdqa64 xmm29,XMMWORD[SHUF_MASK] - vpaddd xmm0,xmm2,XMMWORD[ONE] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,0 - vpshufb xmm0,xmm0,xmm29 - vmovdqu8 xmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc xmm0,xmm0,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast xmm0,xmm0,xmm15 - vpxorq xmm0,xmm0,xmm6 - vextracti32x4 xmm12,zmm0,0 - mov r10,QWORD[120+rbp] - vmovdqu8 XMMWORD[r11*1+r10]{k1},xmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb xmm6,xmm6,xmm29 - vextracti32x4 xmm13,zmm6,0 - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_967 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_967 -$L$_small_initial_partial_block_967: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - - - - - - - - - - - - vpxorq xmm14,xmm14,xmm13 - - jmp NEAR $L$_after_reduction_967 -$L$_small_initial_compute_done_967: -$L$_after_reduction_967: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_2_966: - vmovdqa64 ymm29,YMMWORD[SHUF_MASK] - vshufi64x2 ymm0,ymm2,ymm2,0 - vpaddd ymm0,ymm0,YMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,1 - vpshufb ymm0,ymm0,ymm29 - vmovdqu8 ymm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc ymm0,ymm0,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast ymm0,ymm0,ymm15 - vpxorq ymm0,ymm0,ymm6 - vextracti32x4 xmm12,zmm0,1 - mov r10,QWORD[120+rbp] - vmovdqu8 YMMWORD[r11*1+r10]{k1},ymm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb ymm6,ymm6,ymm29 - vextracti32x4 xmm13,zmm6,1 - sub r13,16 * (2 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_968 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_968 -$L$_small_initial_partial_block_968: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm6,xmm20,0x01 - vpclmulqdq xmm5,xmm6,xmm20,0x10 - vpclmulqdq xmm0,xmm6,xmm20,0x11 - vpclmulqdq xmm3,xmm6,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_968: - - or r13,r13 - je NEAR $L$_after_reduction_968 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_968: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_3_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,2 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm6,zmm29 - vextracti32x4 xmm13,zmm6,2 - sub r13,16 * (3 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_969 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_969 -$L$_small_initial_partial_block_969: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm6,ymm20,0x01 - vpclmulqdq ymm5,ymm6,ymm20,0x10 - vpclmulqdq ymm0,ymm6,ymm20,0x11 - vpclmulqdq ymm3,ymm6,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_969: - - or r13,r13 - je NEAR $L$_after_reduction_969 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_969: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_4_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm0,3 - vpshufb zmm0,zmm0,zmm29 - vmovdqu8 zmm6{k1}{z},[r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vpxorq zmm0,zmm0,zmm6 - vextracti32x4 xmm12,zmm0,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10]{k1},zmm0 - vmovdqu8 zmm0{k1}{z},zmm0 - vpshufb zmm6,zmm6,zmm29 - vextracti32x4 xmm13,zmm6,3 - sub r13,16 * (4 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_970 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_970 -$L$_small_initial_partial_block_970: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_970: - - or r13,r13 - je NEAR $L$_after_reduction_970 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_970: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_5_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb xmm3,xmm3,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 xmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc xmm3,xmm3,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast xmm3,xmm3,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq xmm3,xmm3,xmm7 - vextracti32x4 xmm12,zmm3,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 XMMWORD[64+r11*1+r10]{k1},xmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb xmm7,xmm7,xmm29 - vextracti32x4 xmm13,zmm7,0 - sub r13,16 * (5 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_971 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_971 -$L$_small_initial_partial_block_971: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_971: - - or r13,r13 - je NEAR $L$_after_reduction_971 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_971: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_6_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb ymm3,ymm3,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 ymm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc ymm3,ymm3,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast ymm3,ymm3,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq ymm3,ymm3,ymm7 - vextracti32x4 xmm12,zmm3,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 YMMWORD[64+r11*1+r10]{k1},ymm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb ymm7,ymm7,ymm29 - vextracti32x4 xmm13,zmm7,1 - sub r13,16 * (6 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_972 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_972 -$L$_small_initial_partial_block_972: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm7,xmm20,0x01 - vpclmulqdq xmm5,xmm7,xmm20,0x10 - vpclmulqdq xmm0,xmm7,xmm20,0x11 - vpclmulqdq xmm3,xmm7,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_972: - - or r13,r13 - je NEAR $L$_after_reduction_972 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_972: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_7_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vextracti32x4 xmm13,zmm7,2 - sub r13,16 * (7 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_973 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_973 -$L$_small_initial_partial_block_973: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm7,ymm20,0x01 - vpclmulqdq ymm5,ymm7,ymm20,0x10 - vpclmulqdq ymm0,ymm7,ymm20,0x11 - vpclmulqdq ymm3,ymm7,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_973: - - or r13,r13 - je NEAR $L$_after_reduction_973 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_973: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_8_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,64 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm3,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7{k1}{z},[64+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vextracti32x4 xmm12,zmm3,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10]{k1},zmm3 - vmovdqu8 zmm3{k1}{z},zmm3 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vextracti32x4 xmm13,zmm7,3 - sub r13,16 * (8 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_974 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_974 -$L$_small_initial_partial_block_974: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm6,zmm20,0x11 - vpclmulqdq zmm16,zmm6,zmm20,0x00 - vpclmulqdq zmm17,zmm6,zmm20,0x01 - vpclmulqdq zmm19,zmm6,zmm20,0x10 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm7,zmm20,0x01 - vpclmulqdq zmm5,zmm7,zmm20,0x10 - vpclmulqdq zmm0,zmm7,zmm20,0x11 - vpclmulqdq zmm3,zmm7,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_974: - - or r13,r13 - je NEAR $L$_after_reduction_974 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_974: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_9_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb xmm4,xmm4,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 xmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc xmm4,xmm4,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast xmm4,xmm4,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq xmm4,xmm4,xmm10 - vextracti32x4 xmm12,zmm4,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 XMMWORD[128+r11*1+r10]{k1},xmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb xmm10,xmm10,xmm29 - vextracti32x4 xmm13,zmm10,0 - sub r13,16 * (9 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_975 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_975 -$L$_small_initial_partial_block_975: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_975: - - or r13,r13 - je NEAR $L$_after_reduction_975 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_975: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_10_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb ymm4,ymm4,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 ymm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc ymm4,ymm4,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast ymm4,ymm4,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq ymm4,ymm4,ymm10 - vextracti32x4 xmm12,zmm4,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 YMMWORD[128+r11*1+r10]{k1},ymm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb ymm10,ymm10,ymm29 - vextracti32x4 xmm13,zmm10,1 - sub r13,16 * (10 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_976 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_976 -$L$_small_initial_partial_block_976: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm10,xmm20,0x01 - vpclmulqdq xmm5,xmm10,xmm20,0x10 - vpclmulqdq xmm0,xmm10,xmm20,0x11 - vpclmulqdq xmm3,xmm10,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_976: - - or r13,r13 - je NEAR $L$_after_reduction_976 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_976: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_11_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vextracti32x4 xmm13,zmm10,2 - sub r13,16 * (11 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_977 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_977 -$L$_small_initial_partial_block_977: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm10,ymm20,0x01 - vpclmulqdq ymm5,ymm10,ymm20,0x10 - vpclmulqdq ymm0,ymm10,ymm20,0x11 - vpclmulqdq ymm3,ymm10,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_977: - - or r13,r13 - je NEAR $L$_after_reduction_977 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_977: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_12_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,128 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm4,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10{k1}{z},[128+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vextracti32x4 xmm12,zmm4,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10]{k1},zmm4 - vmovdqu8 zmm4{k1}{z},zmm4 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vextracti32x4 xmm13,zmm10,3 - sub r13,16 * (12 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_978 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_978 -$L$_small_initial_partial_block_978: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vpxorq zmm15,zmm0,zmm15 - vpxorq zmm16,zmm3,zmm16 - vpxorq zmm17,zmm4,zmm17 - vpxorq zmm19,zmm5,zmm19 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm10,zmm20,0x01 - vpclmulqdq zmm5,zmm10,zmm20,0x10 - vpclmulqdq zmm0,zmm10,zmm20,0x11 - vpclmulqdq zmm3,zmm10,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_978: - - or r13,r13 - je NEAR $L$_after_reduction_978 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_978: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_13_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,0 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb xmm5,xmm5,xmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 xmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc xmm5,xmm5,xmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast xmm5,xmm5,xmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq xmm5,xmm5,xmm11 - vextracti32x4 xmm12,zmm5,0 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 XMMWORD[192+r11*1+r10]{k1},xmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb xmm11,xmm11,xmm29 - vextracti32x4 xmm13,zmm11,0 - sub r13,16 * (13 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_979 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_979 -$L$_small_initial_partial_block_979: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[160+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[224+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[288+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - - vpxorq zmm17,zmm17,zmm19 - vpsrldq zmm4,zmm17,8 - vpslldq zmm5,zmm17,8 - vpxorq zmm0,zmm15,zmm4 - vpxorq zmm3,zmm16,zmm5 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_979: - - or r13,r13 - je NEAR $L$_after_reduction_979 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_979: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_14_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,1 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb ymm5,ymm5,ymm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 ymm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc ymm5,ymm5,ymm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast ymm5,ymm5,ymm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq ymm5,ymm5,ymm11 - vextracti32x4 xmm12,zmm5,1 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 YMMWORD[192+r11*1+r10]{k1},ymm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb ymm11,ymm11,ymm29 - vextracti32x4 xmm13,zmm11,1 - sub r13,16 * (14 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_980 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_980 -$L$_small_initial_partial_block_980: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[144+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[208+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[272+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 xmm20,XMMWORD[336+rdx] - vpclmulqdq xmm4,xmm11,xmm20,0x01 - vpclmulqdq xmm5,xmm11,xmm20,0x10 - vpclmulqdq xmm0,xmm11,xmm20,0x11 - vpclmulqdq xmm3,xmm11,xmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_980: - - or r13,r13 - je NEAR $L$_after_reduction_980 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_980: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_15_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,2 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,2 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vextracti32x4 xmm13,zmm11,2 - sub r13,16 * (15 - 1) - - - cmp r13,16 - jl NEAR $L$_small_initial_partial_block_981 - - - - - - sub r13,16 - mov QWORD[r8],0 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - - jmp NEAR $L$_small_initial_compute_done_981 -$L$_small_initial_partial_block_981: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[128+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[192+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[256+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[320+rdx] - vpclmulqdq ymm4,ymm11,ymm20,0x01 - vpclmulqdq ymm5,ymm11,ymm20,0x10 - vpclmulqdq ymm0,ymm11,ymm20,0x11 - vpclmulqdq ymm3,ymm11,ymm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_981: - - or r13,r13 - je NEAR $L$_after_reduction_981 - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_981: - jmp NEAR $L$_small_initial_blocks_encrypted_966 -$L$_small_initial_num_blocks_is_16_966: - vmovdqa64 zmm29,ZMMWORD[SHUF_MASK] - vshufi64x2 zmm2,zmm2,zmm2,0 - vpaddd zmm0,zmm2,ZMMWORD[ddq_add_1234] - vpaddd zmm3,zmm2,ZMMWORD[ddq_add_5678] - vpaddd zmm4,zmm0,ZMMWORD[ddq_add_8888] - vpaddd zmm5,zmm3,ZMMWORD[ddq_add_8888] - lea r10,[byte64_len_to_mask_table] - mov r15,r13 - sub r15,192 - kmovq k1,[r15*8+r10] - vextracti32x4 xmm2,zmm5,3 - vpshufb zmm0,zmm0,zmm29 - vpshufb zmm3,zmm3,zmm29 - vpshufb zmm4,zmm4,zmm29 - vpshufb zmm5,zmm5,zmm29 - vmovdqu8 zmm6,ZMMWORD[r11*1+r9] - vmovdqu8 zmm7,ZMMWORD[64+r11*1+r9] - vmovdqu8 zmm10,ZMMWORD[128+r11*1+r9] - vmovdqu8 zmm11{k1}{z},[192+r11*1+r9] - vbroadcastf64x2 zmm15,ZMMWORD[rcx] - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm15 - vpxorq zmm4,zmm4,zmm15 - vpxorq zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[16+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[32+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[48+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[64+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[80+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[96+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[112+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[128+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[144+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[160+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[176+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[192+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[208+rcx] - vaesenc zmm0,zmm0,zmm15 - vaesenc zmm3,zmm3,zmm15 - vaesenc zmm4,zmm4,zmm15 - vaesenc zmm5,zmm5,zmm15 - vbroadcastf64x2 zmm15,ZMMWORD[224+rcx] - vaesenclast zmm0,zmm0,zmm15 - vaesenclast zmm3,zmm3,zmm15 - vaesenclast zmm4,zmm4,zmm15 - vaesenclast zmm5,zmm5,zmm15 - vpxorq zmm0,zmm0,zmm6 - vpxorq zmm3,zmm3,zmm7 - vpxorq zmm4,zmm4,zmm10 - vpxorq zmm5,zmm5,zmm11 - vextracti32x4 xmm12,zmm5,3 - mov r10,QWORD[120+rbp] - vmovdqu8 ZMMWORD[r11*1+r10],zmm0 - vmovdqu8 ZMMWORD[64+r11*1+r10],zmm3 - vmovdqu8 ZMMWORD[128+r11*1+r10],zmm4 - vmovdqu8 ZMMWORD[192+r11*1+r10]{k1},zmm5 - vmovdqu8 zmm5{k1}{z},zmm5 - vpshufb zmm6,zmm6,zmm29 - vpshufb zmm7,zmm7,zmm29 - vpshufb zmm10,zmm10,zmm29 - vpshufb zmm11,zmm11,zmm29 - vextracti32x4 xmm13,zmm11,3 - sub r13,16 * (16 - 1) -$L$_small_initial_partial_block_982: - - - - - - - - - mov QWORD[r8],r13 - vmovdqu64 XMMWORD[16+rdx],xmm12 - vpxorq zmm6,zmm6,zmm14 - vmovdqu64 zmm20,ZMMWORD[112+rdx] - vpclmulqdq zmm0,zmm6,zmm20,0x11 - vpclmulqdq zmm3,zmm6,zmm20,0x00 - vpclmulqdq zmm4,zmm6,zmm20,0x01 - vpclmulqdq zmm5,zmm6,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[176+rdx] - vpclmulqdq zmm15,zmm7,zmm20,0x11 - vpclmulqdq zmm16,zmm7,zmm20,0x00 - vpclmulqdq zmm17,zmm7,zmm20,0x01 - vpclmulqdq zmm19,zmm7,zmm20,0x10 - vmovdqu64 zmm20,ZMMWORD[240+rdx] - vpclmulqdq zmm6,zmm10,zmm20,0x11 - vpclmulqdq zmm7,zmm10,zmm20,0x00 - vpternlogq zmm15,zmm6,zmm0,0x96 - vpternlogq zmm16,zmm7,zmm3,0x96 - vpclmulqdq zmm6,zmm10,zmm20,0x01 - vpclmulqdq zmm7,zmm10,zmm20,0x10 - vpternlogq zmm17,zmm6,zmm4,0x96 - vpternlogq zmm19,zmm7,zmm5,0x96 - vmovdqu64 ymm20,YMMWORD[304+rdx] - vinserti64x2 zmm20,zmm20,ZMMWORD[336+rdx],2 - vpclmulqdq zmm4,zmm11,zmm20,0x01 - vpclmulqdq zmm5,zmm11,zmm20,0x10 - vpclmulqdq zmm0,zmm11,zmm20,0x11 - vpclmulqdq zmm3,zmm11,zmm20,0x00 - - vpxorq zmm4,zmm4,zmm17 - vpxorq zmm5,zmm5,zmm19 - vpxorq zmm0,zmm0,zmm15 - vpxorq zmm3,zmm3,zmm16 - - vpxorq zmm4,zmm4,zmm5 - vpsrldq zmm17,zmm4,8 - vpslldq zmm19,zmm4,8 - vpxorq zmm0,zmm0,zmm17 - vpxorq zmm3,zmm3,zmm19 - vextracti64x4 ymm17,zmm0,1 - vpxorq ymm0,ymm0,ymm17 - vextracti32x4 xmm17,ymm0,1 - vpxorq xmm0,xmm0,xmm17 - vextracti64x4 ymm19,zmm3,1 - vpxorq ymm3,ymm3,ymm19 - vextracti32x4 xmm19,ymm3,1 - vpxorq xmm3,xmm3,xmm19 - vmovdqa64 xmm20,XMMWORD[POLY2] - - - vpclmulqdq xmm4,xmm20,xmm3,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm4,xmm3,xmm4 - - - vpclmulqdq xmm5,xmm20,xmm4,0x00 - vpsrldq xmm5,xmm5,4 - vpclmulqdq xmm14,xmm20,xmm4,0x10 - vpslldq xmm14,xmm14,4 - vpternlogq xmm14,xmm5,xmm0,0x96 - -$L$_small_initial_compute_done_982: - vpxorq xmm14,xmm14,xmm13 -$L$_after_reduction_982: -$L$_small_initial_blocks_encrypted_966: -$L$_ghash_done_821: - vmovdqu64 XMMWORD[rdx],xmm2 - vmovdqu64 XMMWORD[64+rdx],xmm14 -$L$_enc_dec_done_821: - jmp NEAR $L$exit_gcm_decrypt -$L$exit_gcm_decrypt: - cmp QWORD[112+rbp],256 - jbe NEAR $L$skip_hkeys_cleanup_983 - vpxor xmm0,xmm0,xmm0 - vmovdqa64 ZMMWORD[rsp],zmm0 - vmovdqa64 ZMMWORD[64+rsp],zmm0 - vmovdqa64 ZMMWORD[128+rsp],zmm0 - vmovdqa64 ZMMWORD[192+rsp],zmm0 - vmovdqa64 ZMMWORD[256+rsp],zmm0 - vmovdqa64 ZMMWORD[320+rsp],zmm0 - vmovdqa64 ZMMWORD[384+rsp],zmm0 - vmovdqa64 ZMMWORD[448+rsp],zmm0 - vmovdqa64 ZMMWORD[512+rsp],zmm0 - vmovdqa64 ZMMWORD[576+rsp],zmm0 - vmovdqa64 ZMMWORD[640+rsp],zmm0 - vmovdqa64 ZMMWORD[704+rsp],zmm0 -$L$skip_hkeys_cleanup_983: - vzeroupper - vmovdqu xmm15,XMMWORD[((-16))+rbp] - vmovdqu xmm14,XMMWORD[((-32))+rbp] - vmovdqu xmm13,XMMWORD[((-48))+rbp] - vmovdqu xmm12,XMMWORD[((-64))+rbp] - vmovdqu xmm11,XMMWORD[((-80))+rbp] - vmovdqu xmm10,XMMWORD[((-96))+rbp] - vmovdqu xmm9,XMMWORD[((-112))+rbp] - vmovdqu xmm8,XMMWORD[((-128))+rbp] - vmovdqu xmm7,XMMWORD[((-144))+rbp] - vmovdqu xmm6,XMMWORD[((-160))+rbp] - lea rsp,[8+rbp] - pop rsi - - pop rdi - - pop r15 - - pop r14 - - pop r13 - - pop r12 - - pop rbp - - pop rbx - - DB 0F3h,0C3h ;repret -$L$decrypt_seh_end: - - -global ossl_aes_gcm_finalize_avx512 - -ALIGN 32 -ossl_aes_gcm_finalize_avx512: - -DB 243,15,30,250 - vmovdqu xmm2,XMMWORD[336+rcx] - vmovdqu xmm3,XMMWORD[32+rcx] - vmovdqu xmm4,XMMWORD[64+rcx] - - - cmp rdx,0 - je NEAR $L$_partial_done_984 - - vpclmulqdq xmm0,xmm4,xmm2,0x11 - vpclmulqdq xmm16,xmm4,xmm2,0x00 - vpclmulqdq xmm17,xmm4,xmm2,0x01 - vpclmulqdq xmm4,xmm4,xmm2,0x10 - vpxorq xmm4,xmm4,xmm17 - - vpsrldq xmm17,xmm4,8 - vpslldq xmm4,xmm4,8 - vpxorq xmm0,xmm0,xmm17 - vpxorq xmm4,xmm4,xmm16 - - - - vmovdqu64 xmm17,XMMWORD[POLY2] - - vpclmulqdq xmm16,xmm17,xmm4,0x01 - vpslldq xmm16,xmm16,8 - vpxorq xmm4,xmm4,xmm16 - - - - vpclmulqdq xmm16,xmm17,xmm4,0x00 - vpsrldq xmm16,xmm16,4 - vpclmulqdq xmm4,xmm17,xmm4,0x10 - vpslldq xmm4,xmm4,4 - - vpternlogq xmm4,xmm0,xmm16,0x96 - -$L$_partial_done_984: - vmovq xmm5,QWORD[56+rcx] - vpinsrq xmm5,xmm5,QWORD[48+rcx],1 - vpsllq xmm5,xmm5,3 - - vpxor xmm4,xmm4,xmm5 - - vpclmulqdq xmm0,xmm4,xmm2,0x11 - vpclmulqdq xmm16,xmm4,xmm2,0x00 - vpclmulqdq xmm17,xmm4,xmm2,0x01 - vpclmulqdq xmm4,xmm4,xmm2,0x10 - vpxorq xmm4,xmm4,xmm17 - - vpsrldq xmm17,xmm4,8 - vpslldq xmm4,xmm4,8 - vpxorq xmm0,xmm0,xmm17 - vpxorq xmm4,xmm4,xmm16 - - - - vmovdqu64 xmm17,XMMWORD[POLY2] - - vpclmulqdq xmm16,xmm17,xmm4,0x01 - vpslldq xmm16,xmm16,8 - vpxorq xmm4,xmm4,xmm16 - - - - vpclmulqdq xmm16,xmm17,xmm4,0x00 - vpsrldq xmm16,xmm16,4 - vpclmulqdq xmm4,xmm17,xmm4,0x10 - vpslldq xmm4,xmm4,4 - - vpternlogq xmm4,xmm0,xmm16,0x96 - - vpshufb xmm4,xmm4,XMMWORD[SHUF_MASK] - vpxor xmm3,xmm3,xmm4 - -$L$_return_T_984: - vmovdqu XMMWORD[64+rcx],xmm3 -$L$abort_finalize: - DB 0F3h,0C3h ;repret - - -global ossl_gcm_gmult_avx512 - - -ALIGN 32 -ossl_gcm_gmult_avx512: - -DB 243,15,30,250 - vmovdqu64 xmm1,XMMWORD[rcx] - vmovdqu64 xmm2,XMMWORD[336+rdx] - - vpclmulqdq xmm3,xmm1,xmm2,0x11 - vpclmulqdq xmm4,xmm1,xmm2,0x00 - vpclmulqdq xmm5,xmm1,xmm2,0x01 - vpclmulqdq xmm1,xmm1,xmm2,0x10 - vpxorq xmm1,xmm1,xmm5 - - vpsrldq xmm5,xmm1,8 - vpslldq xmm1,xmm1,8 - vpxorq xmm3,xmm3,xmm5 - vpxorq xmm1,xmm1,xmm4 - - - - vmovdqu64 xmm5,XMMWORD[POLY2] - - vpclmulqdq xmm4,xmm5,xmm1,0x01 - vpslldq xmm4,xmm4,8 - vpxorq xmm1,xmm1,xmm4 - - - - vpclmulqdq xmm4,xmm5,xmm1,0x00 - vpsrldq xmm4,xmm4,4 - vpclmulqdq xmm1,xmm5,xmm1,0x10 - vpslldq xmm1,xmm1,4 - - vpternlogq xmm1,xmm3,xmm4,0x96 - - vmovdqu64 XMMWORD[rcx],xmm1 - vzeroupper -$L$abort_gmult: - DB 0F3h,0C3h ;repret - - -section .pdata rdata align=4 -ALIGN 4 - DD $L$setiv_seh_begin wrt ..imagebase - DD $L$setiv_seh_end wrt ..imagebase - DD $L$setiv_seh_info wrt ..imagebase - - DD $L$ghash_seh_begin wrt ..imagebase - DD $L$ghash_seh_end wrt ..imagebase - DD $L$ghash_seh_info wrt ..imagebase - - DD $L$encrypt_seh_begin wrt ..imagebase - DD $L$encrypt_seh_end wrt ..imagebase - DD $L$encrypt_seh_info wrt ..imagebase - - DD $L$decrypt_seh_begin wrt ..imagebase - DD $L$decrypt_seh_end wrt ..imagebase - DD $L$decrypt_seh_info wrt ..imagebase - -section .xdata rdata align=8 -ALIGN 8 -$L$setiv_seh_info: -DB 1 -DB $L$setiv_seh_prolog_end-$L$setiv_seh_begin -DB 31 - -DB 165 -DB $L$setiv_seh_save_xmm15-$L$setiv_seh_begin -DB 248 - DW 9 -DB $L$setiv_seh_save_xmm14-$L$setiv_seh_begin -DB 232 - DW 8 -DB $L$setiv_seh_save_xmm13-$L$setiv_seh_begin -DB 216 - DW 7 -DB $L$setiv_seh_save_xmm12-$L$setiv_seh_begin -DB 200 - DW 6 -DB $L$setiv_seh_save_xmm11-$L$setiv_seh_begin -DB 184 - DW 5 -DB $L$setiv_seh_save_xmm10-$L$setiv_seh_begin -DB 168 - DW 4 -DB $L$setiv_seh_save_xmm9-$L$setiv_seh_begin -DB 152 - DW 3 -DB $L$setiv_seh_save_xmm8-$L$setiv_seh_begin -DB 136 - DW 2 -DB $L$setiv_seh_save_xmm7-$L$setiv_seh_begin -DB 120 - DW 1 -DB $L$setiv_seh_save_xmm6-$L$setiv_seh_begin -DB 104 - DW 0 - -DB $L$setiv_seh_setfp-$L$setiv_seh_begin -DB 3 - - -DB $L$setiv_seh_allocstack_xmm-$L$setiv_seh_begin -DB 1 - DW 21 -DB $L$setiv_seh_push_rsi-$L$setiv_seh_begin -DB 96 -DB $L$setiv_seh_push_rdi-$L$setiv_seh_begin -DB 112 -DB $L$setiv_seh_push_r15-$L$setiv_seh_begin -DB 240 -DB $L$setiv_seh_push_r14-$L$setiv_seh_begin -DB 224 -DB $L$setiv_seh_push_r13-$L$setiv_seh_begin -DB 208 -DB $L$setiv_seh_push_r12-$L$setiv_seh_begin -DB 192 -DB $L$setiv_seh_push_rbp-$L$setiv_seh_begin -DB 80 -DB $L$setiv_seh_push_rbx-$L$setiv_seh_begin -DB 48 -ALIGN 8 -$L$ghash_seh_info: -DB 1 -DB $L$ghash_seh_prolog_end-$L$ghash_seh_begin -DB 31 - -DB 165 -DB $L$ghash_seh_save_xmm15-$L$ghash_seh_begin -DB 248 - DW 9 -DB $L$ghash_seh_save_xmm14-$L$ghash_seh_begin -DB 232 - DW 8 -DB $L$ghash_seh_save_xmm13-$L$ghash_seh_begin -DB 216 - DW 7 -DB $L$ghash_seh_save_xmm12-$L$ghash_seh_begin -DB 200 - DW 6 -DB $L$ghash_seh_save_xmm11-$L$ghash_seh_begin -DB 184 - DW 5 -DB $L$ghash_seh_save_xmm10-$L$ghash_seh_begin -DB 168 - DW 4 -DB $L$ghash_seh_save_xmm9-$L$ghash_seh_begin -DB 152 - DW 3 -DB $L$ghash_seh_save_xmm8-$L$ghash_seh_begin -DB 136 - DW 2 -DB $L$ghash_seh_save_xmm7-$L$ghash_seh_begin -DB 120 - DW 1 -DB $L$ghash_seh_save_xmm6-$L$ghash_seh_begin -DB 104 - DW 0 - -DB $L$ghash_seh_setfp-$L$ghash_seh_begin -DB 3 - - -DB $L$ghash_seh_allocstack_xmm-$L$ghash_seh_begin -DB 1 - DW 21 -DB $L$ghash_seh_push_rsi-$L$ghash_seh_begin -DB 96 -DB $L$ghash_seh_push_rdi-$L$ghash_seh_begin -DB 112 -DB $L$ghash_seh_push_r15-$L$ghash_seh_begin -DB 240 -DB $L$ghash_seh_push_r14-$L$ghash_seh_begin -DB 224 -DB $L$ghash_seh_push_r13-$L$ghash_seh_begin -DB 208 -DB $L$ghash_seh_push_r12-$L$ghash_seh_begin -DB 192 -DB $L$ghash_seh_push_rbp-$L$ghash_seh_begin -DB 80 -DB $L$ghash_seh_push_rbx-$L$ghash_seh_begin -DB 48 -ALIGN 8 -$L$encrypt_seh_info: -DB 1 -DB $L$encrypt_seh_prolog_end-$L$encrypt_seh_begin -DB 31 - -DB 165 -DB $L$encrypt_seh_save_xmm15-$L$encrypt_seh_begin -DB 248 - DW 9 -DB $L$encrypt_seh_save_xmm14-$L$encrypt_seh_begin -DB 232 - DW 8 -DB $L$encrypt_seh_save_xmm13-$L$encrypt_seh_begin -DB 216 - DW 7 -DB $L$encrypt_seh_save_xmm12-$L$encrypt_seh_begin -DB 200 - DW 6 -DB $L$encrypt_seh_save_xmm11-$L$encrypt_seh_begin -DB 184 - DW 5 -DB $L$encrypt_seh_save_xmm10-$L$encrypt_seh_begin -DB 168 - DW 4 -DB $L$encrypt_seh_save_xmm9-$L$encrypt_seh_begin -DB 152 - DW 3 -DB $L$encrypt_seh_save_xmm8-$L$encrypt_seh_begin -DB 136 - DW 2 -DB $L$encrypt_seh_save_xmm7-$L$encrypt_seh_begin -DB 120 - DW 1 -DB $L$encrypt_seh_save_xmm6-$L$encrypt_seh_begin -DB 104 - DW 0 - -DB $L$encrypt_seh_setfp-$L$encrypt_seh_begin -DB 3 - - -DB $L$encrypt_seh_allocstack_xmm-$L$encrypt_seh_begin -DB 1 - DW 21 -DB $L$encrypt_seh_push_rsi-$L$encrypt_seh_begin -DB 96 -DB $L$encrypt_seh_push_rdi-$L$encrypt_seh_begin -DB 112 -DB $L$encrypt_seh_push_r15-$L$encrypt_seh_begin -DB 240 -DB $L$encrypt_seh_push_r14-$L$encrypt_seh_begin -DB 224 -DB $L$encrypt_seh_push_r13-$L$encrypt_seh_begin -DB 208 -DB $L$encrypt_seh_push_r12-$L$encrypt_seh_begin -DB 192 -DB $L$encrypt_seh_push_rbp-$L$encrypt_seh_begin -DB 80 -DB $L$encrypt_seh_push_rbx-$L$encrypt_seh_begin -DB 48 -ALIGN 8 -$L$decrypt_seh_info: -DB 1 -DB $L$decrypt_seh_prolog_end-$L$decrypt_seh_begin -DB 31 - -DB 165 -DB $L$decrypt_seh_save_xmm15-$L$decrypt_seh_begin -DB 248 - DW 9 -DB $L$decrypt_seh_save_xmm14-$L$decrypt_seh_begin -DB 232 - DW 8 -DB $L$decrypt_seh_save_xmm13-$L$decrypt_seh_begin -DB 216 - DW 7 -DB $L$decrypt_seh_save_xmm12-$L$decrypt_seh_begin -DB 200 - DW 6 -DB $L$decrypt_seh_save_xmm11-$L$decrypt_seh_begin -DB 184 - DW 5 -DB $L$decrypt_seh_save_xmm10-$L$decrypt_seh_begin -DB 168 - DW 4 -DB $L$decrypt_seh_save_xmm9-$L$decrypt_seh_begin -DB 152 - DW 3 -DB $L$decrypt_seh_save_xmm8-$L$decrypt_seh_begin -DB 136 - DW 2 -DB $L$decrypt_seh_save_xmm7-$L$decrypt_seh_begin -DB 120 - DW 1 -DB $L$decrypt_seh_save_xmm6-$L$decrypt_seh_begin -DB 104 - DW 0 - -DB $L$decrypt_seh_setfp-$L$decrypt_seh_begin -DB 3 - - -DB $L$decrypt_seh_allocstack_xmm-$L$decrypt_seh_begin -DB 1 - DW 21 -DB $L$decrypt_seh_push_rsi-$L$decrypt_seh_begin -DB 96 -DB $L$decrypt_seh_push_rdi-$L$decrypt_seh_begin -DB 112 -DB $L$decrypt_seh_push_r15-$L$decrypt_seh_begin -DB 240 -DB $L$decrypt_seh_push_r14-$L$decrypt_seh_begin -DB 224 -DB $L$decrypt_seh_push_r13-$L$decrypt_seh_begin -DB 208 -DB $L$decrypt_seh_push_r12-$L$decrypt_seh_begin -DB 192 -DB $L$decrypt_seh_push_rbp-$L$decrypt_seh_begin -DB 80 -DB $L$decrypt_seh_push_rbx-$L$decrypt_seh_begin -DB 48 -section .data data align=8 - -ALIGN 16 -POLY: DQ 0x0000000000000001,0xC200000000000000 - -ALIGN 64 -POLY2: - DQ 0x00000001C2000000,0xC200000000000000 - DQ 0x00000001C2000000,0xC200000000000000 - DQ 0x00000001C2000000,0xC200000000000000 - DQ 0x00000001C2000000,0xC200000000000000 - -ALIGN 16 -TWOONE: DQ 0x0000000000000001,0x0000000100000000 - - - -ALIGN 64 -SHUF_MASK: - DQ 0x08090A0B0C0D0E0F,0x0001020304050607 - DQ 0x08090A0B0C0D0E0F,0x0001020304050607 - DQ 0x08090A0B0C0D0E0F,0x0001020304050607 - DQ 0x08090A0B0C0D0E0F,0x0001020304050607 - -ALIGN 16 -SHIFT_MASK: - DQ 0x0706050403020100,0x0f0e0d0c0b0a0908 - -ALL_F: - DQ 0xffffffffffffffff,0xffffffffffffffff - -ZERO: - DQ 0x0000000000000000,0x0000000000000000 - -ALIGN 16 -ONE: - DQ 0x0000000000000001,0x0000000000000000 - -ALIGN 16 -ONEf: - DQ 0x0000000000000000,0x0100000000000000 - -ALIGN 64 -ddq_add_1234: - DQ 0x0000000000000001,0x0000000000000000 - DQ 0x0000000000000002,0x0000000000000000 - DQ 0x0000000000000003,0x0000000000000000 - DQ 0x0000000000000004,0x0000000000000000 - -ALIGN 64 -ddq_add_5678: - DQ 0x0000000000000005,0x0000000000000000 - DQ 0x0000000000000006,0x0000000000000000 - DQ 0x0000000000000007,0x0000000000000000 - DQ 0x0000000000000008,0x0000000000000000 - -ALIGN 64 -ddq_add_4444: - DQ 0x0000000000000004,0x0000000000000000 - DQ 0x0000000000000004,0x0000000000000000 - DQ 0x0000000000000004,0x0000000000000000 - DQ 0x0000000000000004,0x0000000000000000 - -ALIGN 64 -ddq_add_8888: - DQ 0x0000000000000008,0x0000000000000000 - DQ 0x0000000000000008,0x0000000000000000 - DQ 0x0000000000000008,0x0000000000000000 - DQ 0x0000000000000008,0x0000000000000000 - -ALIGN 64 -ddq_addbe_1234: - DQ 0x0000000000000000,0x0100000000000000 - DQ 0x0000000000000000,0x0200000000000000 - DQ 0x0000000000000000,0x0300000000000000 - DQ 0x0000000000000000,0x0400000000000000 - -ALIGN 64 -ddq_addbe_4444: - DQ 0x0000000000000000,0x0400000000000000 - DQ 0x0000000000000000,0x0400000000000000 - DQ 0x0000000000000000,0x0400000000000000 - DQ 0x0000000000000000,0x0400000000000000 - -ALIGN 64 -byte_len_to_mask_table: - DW 0x0000,0x0001,0x0003,0x0007 - DW 0x000f,0x001f,0x003f,0x007f - DW 0x00ff,0x01ff,0x03ff,0x07ff - DW 0x0fff,0x1fff,0x3fff,0x7fff - DW 0xffff - -ALIGN 64 -byte64_len_to_mask_table: - DQ 0x0000000000000000,0x0000000000000001 - DQ 0x0000000000000003,0x0000000000000007 - DQ 0x000000000000000f,0x000000000000001f - DQ 0x000000000000003f,0x000000000000007f - DQ 0x00000000000000ff,0x00000000000001ff - DQ 0x00000000000003ff,0x00000000000007ff - DQ 0x0000000000000fff,0x0000000000001fff - DQ 0x0000000000003fff,0x0000000000007fff - DQ 0x000000000000ffff,0x000000000001ffff - DQ 0x000000000003ffff,0x000000000007ffff - DQ 0x00000000000fffff,0x00000000001fffff - DQ 0x00000000003fffff,0x00000000007fffff - DQ 0x0000000000ffffff,0x0000000001ffffff - DQ 0x0000000003ffffff,0x0000000007ffffff - DQ 0x000000000fffffff,0x000000001fffffff - DQ 0x000000003fffffff,0x000000007fffffff - DQ 0x00000000ffffffff,0x00000001ffffffff - DQ 0x00000003ffffffff,0x00000007ffffffff - DQ 0x0000000fffffffff,0x0000001fffffffff - DQ 0x0000003fffffffff,0x0000007fffffffff - DQ 0x000000ffffffffff,0x000001ffffffffff - DQ 0x000003ffffffffff,0x000007ffffffffff - DQ 0x00000fffffffffff,0x00001fffffffffff - DQ 0x00003fffffffffff,0x00007fffffffffff - DQ 0x0000ffffffffffff,0x0001ffffffffffff - DQ 0x0003ffffffffffff,0x0007ffffffffffff - DQ 0x000fffffffffffff,0x001fffffffffffff - DQ 0x003fffffffffffff,0x007fffffffffffff - DQ 0x00ffffffffffffff,0x01ffffffffffffff - DQ 0x03ffffffffffffff,0x07ffffffffffffff - DQ 0x0fffffffffffffff,0x1fffffffffffffff - DQ 0x3fffffffffffffff,0x7fffffffffffffff - DQ 0xffffffffffffffff diff --git a/openssl/src/crypto/modes/local.h b/openssl/src/crypto/modes/local.h new file mode 100644 index 000000000..422e3d262 --- /dev/null +++ b/openssl/src/crypto/modes/local.h @@ -0,0 +1,44 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +void gcm_init_clmul(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_clmul(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_clmul(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); +void gcm_init_avx(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_avx(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); +void gcm_gmult_4bit_mmx(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_4bit_mmx(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); + +void gcm_gmult_4bit_x86(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_4bit_x86(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); +void gcm_init_neon(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_neon(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_neon(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); +void gcm_init_v8(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_v8(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_v8(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); +void gcm_init_p8(u128 Htable[16], const u64 Xi[2]); +void gcm_gmult_p8(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_p8(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); + +void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16]); +void gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); diff --git a/openssl/src/crypto/modes/ocb128.c b/openssl/src/crypto/modes/ocb128.c index 1ae807c10..b5202ba5b 100644 --- a/openssl/src/crypto/modes/ocb128.c +++ b/openssl/src/crypto/modes/ocb128.c @@ -155,8 +155,10 @@ int CRYPTO_ocb128_init(OCB128_CONTEXT *ctx, void *keyenc, void *keydec, memset(ctx, 0, sizeof(*ctx)); ctx->l_index = 0; ctx->max_l_index = 5; - if ((ctx->l = OPENSSL_malloc(ctx->max_l_index * 16)) == NULL) + if ((ctx->l = OPENSSL_malloc(ctx->max_l_index * 16)) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } /* * We set both the encryption and decryption key schedules - decryption @@ -200,8 +202,10 @@ int CRYPTO_ocb128_copy_ctx(OCB128_CONTEXT *dest, OCB128_CONTEXT *src, if (keydec) dest->keydec = keydec; if (src->l) { - if ((dest->l = OPENSSL_malloc(src->max_l_index * 16)) == NULL) + if ((dest->l = OPENSSL_malloc(src->max_l_index * 16)) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(dest->l, src->l, (src->l_index + 1) * 16); } return 1; diff --git a/openssl/src/crypto/modes/xts128gb.c b/openssl/src/crypto/modes/xts128gb.c deleted file mode 100644 index 021c0597e..000000000 --- a/openssl/src/crypto/modes/xts128gb.c +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "internal/endian.h" -#include "crypto/modes.h" - -#ifndef STRICT_ALIGNMENT -# ifdef __GNUC__ -typedef u64 u64_a1 __attribute((__aligned__(1))); -# else -typedef u64 u64_a1; -# endif -#endif - -int ossl_crypto_xts128gb_encrypt(const XTS128_CONTEXT *ctx, - const unsigned char iv[16], - const unsigned char *inp, unsigned char *out, - size_t len, int enc) -{ - DECLARE_IS_ENDIAN; - union { - u64 u[2]; - u32 d[4]; - u8 c[16]; - } tweak, scratch; - unsigned int i; - - if (len < 16) - return -1; - - memcpy(tweak.c, iv, 16); - - (*ctx->block2) (tweak.c, tweak.c, ctx->key2); - - if (!enc && (len % 16)) - len -= 16; - - while (len >= 16) { -#if defined(STRICT_ALIGNMENT) - memcpy(scratch.c, inp, 16); - scratch.u[0] ^= tweak.u[0]; - scratch.u[1] ^= tweak.u[1]; -#else - scratch.u[0] = ((u64_a1 *)inp)[0] ^ tweak.u[0]; - scratch.u[1] = ((u64_a1 *)inp)[1] ^ tweak.u[1]; -#endif - (*ctx->block1) (scratch.c, scratch.c, ctx->key1); -#if defined(STRICT_ALIGNMENT) - scratch.u[0] ^= tweak.u[0]; - scratch.u[1] ^= tweak.u[1]; - memcpy(out, scratch.c, 16); -#else - ((u64_a1 *)out)[0] = scratch.u[0] ^= tweak.u[0]; - ((u64_a1 *)out)[1] = scratch.u[1] ^= tweak.u[1]; -#endif - inp += 16; - out += 16; - len -= 16; - - if (len == 0) - return 0; - - if (IS_LITTLE_ENDIAN) { - u8 res; - u64 hi, lo; -#ifdef BSWAP8 - hi = BSWAP8(tweak.u[0]); - lo = BSWAP8(tweak.u[1]); -#else - u8 *p = tweak.c; - - hi = (u64)GETU32(p) << 32 | GETU32(p + 4); - lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12); -#endif - res = (u8)lo & 1; - tweak.u[0] = (lo >> 1) | (hi << 63); - tweak.u[1] = hi >> 1; - if (res) - tweak.c[15] ^= 0xe1; -#ifdef BSWAP8 - hi = BSWAP8(tweak.u[0]); - lo = BSWAP8(tweak.u[1]); -#else - p = tweak.c; - - hi = (u64)GETU32(p) << 32 | GETU32(p + 4); - lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12); -#endif - tweak.u[0] = lo; - tweak.u[1] = hi; - } else { - u8 carry, res; - carry = 0; - for (i = 0; i < 16; ++i) { - res = (tweak.c[i] << 7) & 0x80; - tweak.c[i] = ((tweak.c[i] >> 1) + carry) & 0xff; - carry = res; - } - if (res) - tweak.c[0] ^= 0xe1; - } - } - if (enc) { - for (i = 0; i < len; ++i) { - u8 c = inp[i]; - out[i] = scratch.c[i]; - scratch.c[i] = c; - } - scratch.u[0] ^= tweak.u[0]; - scratch.u[1] ^= tweak.u[1]; - (*ctx->block1) (scratch.c, scratch.c, ctx->key1); - scratch.u[0] ^= tweak.u[0]; - scratch.u[1] ^= tweak.u[1]; - memcpy(out - 16, scratch.c, 16); - } else { - union { - u64 u[2]; - u8 c[16]; - } tweak1; - - if (IS_LITTLE_ENDIAN) { - u8 res; - u64 hi, lo; -#ifdef BSWAP8 - hi = BSWAP8(tweak.u[0]); - lo = BSWAP8(tweak.u[1]); -#else - u8 *p = tweak.c; - - hi = (u64)GETU32(p) << 32 | GETU32(p + 4); - lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12); -#endif - res = (u8)lo & 1; - tweak1.u[0] = (lo >> 1) | (hi << 63); - tweak1.u[1] = hi >> 1; - if (res) - tweak1.c[15] ^= 0xe1; -#ifdef BSWAP8 - hi = BSWAP8(tweak1.u[0]); - lo = BSWAP8(tweak1.u[1]); -#else - p = tweak1.c; - - hi = (u64)GETU32(p) << 32 | GETU32(p + 4); - lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12); -#endif - tweak1.u[0] = lo; - tweak1.u[1] = hi; - } else { - u8 carry, res; - carry = 0; - for (i = 0; i < 16; ++i) { - res = (tweak.c[i] << 7) & 0x80; - tweak1.c[i] = ((tweak.c[i] >> 1) + carry) & 0xff; - carry = res; - } - if (res) - tweak1.c[0] ^= 0xe1; - } -#if defined(STRICT_ALIGNMENT) - memcpy(scratch.c, inp, 16); - scratch.u[0] ^= tweak1.u[0]; - scratch.u[1] ^= tweak1.u[1]; -#else - scratch.u[0] = ((u64_a1 *)inp)[0] ^ tweak1.u[0]; - scratch.u[1] = ((u64_a1 *)inp)[1] ^ tweak1.u[1]; -#endif - (*ctx->block1) (scratch.c, scratch.c, ctx->key1); - scratch.u[0] ^= tweak1.u[0]; - scratch.u[1] ^= tweak1.u[1]; - - for (i = 0; i < len; ++i) { - u8 c = inp[16 + i]; - out[16 + i] = scratch.c[i]; - scratch.c[i] = c; - } - scratch.u[0] ^= tweak.u[0]; - scratch.u[1] ^= tweak.u[1]; - (*ctx->block1) (scratch.c, scratch.c, ctx->key1); -#if defined(STRICT_ALIGNMENT) - scratch.u[0] ^= tweak.u[0]; - scratch.u[1] ^= tweak.u[1]; - memcpy(out, scratch.c, 16); -#else - ((u64_a1 *)out)[0] = scratch.u[0] ^ tweak.u[0]; - ((u64_a1 *)out)[1] = scratch.u[1] ^ tweak.u[1]; -#endif - } - - return 0; -} diff --git a/openssl/src/crypto/o_dir.c b/openssl/src/crypto/o_dir.c index d7f5d64d0..f7e21dad6 100644 --- a/openssl/src/crypto/o_dir.c +++ b/openssl/src/crypto/o_dir.c @@ -7,31 +7,329 @@ * https://www.openssl.org/source/license.html */ +/* + * This file is dual-licensed and is also available under the following + * terms: + * + * Copyright (c) 2004, 2018, Richard Levitte + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + #include "internal/e_os.h" #include +#include "internal/o_dir.h" + +#if defined OPENSSL_SYS_UNIX +/* Original LPdir_unix.c */ +# include +# include +# include +# include +# include +# include +# include /* - * The routines really come from the Levitte Programming, so to make life - * simple, let's just use the raw files and hack the symbols to fit our - * namespace. + * The POSIX macro for the maximum number of characters in a file path is + * NAME_MAX. However, some operating systems use PATH_MAX instead. + * Therefore, it seems natural to first check for PATH_MAX and use that, and + * if it doesn't exist, use NAME_MAX. */ -#define LP_DIR_CTX OPENSSL_DIR_CTX -#define LP_dir_context_st OPENSSL_dir_context_st -#define LP_find_file OPENSSL_DIR_read -#define LP_find_file_end OPENSSL_DIR_end +# if defined(PATH_MAX) +# define LP_ENTRY_SIZE PATH_MAX +# elif defined(NAME_MAX) +# define LP_ENTRY_SIZE NAME_MAX +# endif -#include "internal/o_dir.h" +/* + * Of course, there's the possibility that neither PATH_MAX nor NAME_MAX + * exist. It's also possible that NAME_MAX exists but is define to a very + * small value (HP-UX offers 14), so we need to check if we got a result, and + * if it meets a minimum standard, and create or change it if not. + */ +# if !defined(LP_ENTRY_SIZE) || LP_ENTRY_SIZE<255 +# undef LP_ENTRY_SIZE +# define LP_ENTRY_SIZE 255 +# endif + +struct OPENSSL_dir_context_st { + DIR *dir; + char entry_name[LP_ENTRY_SIZE + 1]; +}; + +const char *OPENSSL_DIR_read(OPENSSL_DIR_CTX **ctx, const char *directory) +{ + struct dirent *direntry = NULL; + + if (ctx == NULL || directory == NULL) { + errno = EINVAL; + return 0; + } + + errno = 0; + if (*ctx == NULL) { + *ctx = malloc(sizeof(**ctx)); + if (*ctx == NULL) { + errno = ENOMEM; + return 0; + } + memset(*ctx, 0, sizeof(**ctx)); + + (*ctx)->dir = opendir(directory); + if ((*ctx)->dir == NULL) { + int save_errno = errno; /* Probably not needed, but I'm paranoid */ + free(*ctx); + *ctx = NULL; + errno = save_errno; + return 0; + } + } + + direntry = readdir((*ctx)->dir); + if (direntry == NULL) { + return 0; + } + + OPENSSL_strlcpy((*ctx)->entry_name, direntry->d_name, + sizeof((*ctx)->entry_name)); + return (*ctx)->entry_name; +} + +int OPENSSL_DIR_end(OPENSSL_DIR_CTX **ctx) +{ + if (ctx != NULL && *ctx != NULL) { + int ret = closedir((*ctx)->dir); + + free(*ctx); + switch (ret) { + case 0: + return 1; + case -1: + return 0; + default: + break; + } + } + errno = EINVAL; + return 0; +} +#elif defined(OPENSSL_SYS_WIN32) || defined(OPENSSL_SYS_WINCE) +/* Original LPdir_win/win32/wince.c */ +# if defined OPENSSL_SYS_WIN32 +# define LP_SYS_WIN32 +# define LP_MULTIBYTE_AVAILABLE +# elif defined OPENSSL_SYS_WINCE +# define LP_SYS_WINCE +# endif +# include +# include +# include "internal/numbers.h" + +/* + * We're most likely overcautious here, but let's reserve for broken WinCE + * headers and explicitly opt for UNICODE call. Keep in mind that our WinCE + * builds are compiled with -DUNICODE [as well as -D_UNICODE]. + */ +# if defined(LP_SYS_WINCE) && !defined(FindFirstFile) +# define FindFirstFile FindFirstFileW +# endif +# if defined(LP_SYS_WINCE) && !defined(FindNextFile) +# define FindNextFile FindNextFileW +# endif -#define LPDIR_H -#if defined OPENSSL_SYS_UNIX || defined DJGPP \ - || (defined __VMS_VER && __VMS_VER >= 70000000) -# include "LPdir_unix.c" -#elif defined OPENSSL_SYS_VMS -# include "LPdir_vms.c" -#elif defined OPENSSL_SYS_WIN32 -# include "LPdir_win32.c" -#elif defined OPENSSL_SYS_WINCE -# include "LPdir_wince.c" +# ifndef NAME_MAX +# define NAME_MAX 255 +# endif + +# ifdef CP_UTF8 +# define CP_DEFAULT CP_UTF8 +# else +# define CP_DEFAULT CP_ACP +# endif + +struct OPENSSL_dir_context_st { + WIN32_FIND_DATA ctx; + HANDLE handle; + char entry_name[NAME_MAX + 1]; +}; + +const char *OPENSSL_DIR_read(OPENSSL_DIR_CTX **ctx, const char *directory) +{ + if (ctx == NULL || directory == NULL) { + errno = EINVAL; + return 0; + } + + errno = 0; + if (*ctx == NULL) { + size_t dirlen = strlen(directory); + + if (dirlen == 0 || dirlen > INT_MAX - 3) { + errno = ENOENT; + return 0; + } + + *ctx = malloc(sizeof(**ctx)); + if (*ctx == NULL) { + errno = ENOMEM; + return 0; + } + memset(*ctx, 0, sizeof(**ctx)); + + if (sizeof(TCHAR) != sizeof(char)) { + TCHAR *wdir = NULL; + /* len_0 denotes string length *with* trailing 0 */ + size_t index = 0, len_0 = dirlen + 1; +# ifdef LP_MULTIBYTE_AVAILABLE + int sz = 0; + UINT cp; + + do { +# ifdef CP_UTF8 + if ((sz = MultiByteToWideChar((cp = CP_UTF8), 0, + directory, len_0, + NULL, 0)) > 0 || + GetLastError() != ERROR_NO_UNICODE_TRANSLATION) + break; +# endif + sz = MultiByteToWideChar((cp = CP_ACP), 0, + directory, len_0, + NULL, 0); + } while (0); + + if (sz > 0) { + /* + * allocate two additional characters in case we need to + * concatenate asterisk, |sz| covers trailing '\0'! + */ + wdir = _alloca((sz + 2) * sizeof(TCHAR)); + if (!MultiByteToWideChar(cp, 0, directory, len_0, + (WCHAR *)wdir, sz)) { + free(*ctx); + *ctx = NULL; + errno = EINVAL; + return 0; + } + } else +# endif + { + sz = len_0; + /* + * allocate two additional characters in case we need to + * concatenate asterisk, |sz| covers trailing '\0'! + */ + wdir = _alloca((sz + 2) * sizeof(TCHAR)); + for (index = 0; index < len_0; index++) + wdir[index] = (TCHAR)directory[index]; + } + + sz--; /* wdir[sz] is trailing '\0' now */ + if (wdir[sz - 1] != TEXT('*')) { + if (wdir[sz - 1] != TEXT('/') && wdir[sz - 1] != TEXT('\\')) + _tcscpy(wdir + sz, TEXT("/*")); + else + _tcscpy(wdir + sz, TEXT("*")); + } + + (*ctx)->handle = FindFirstFile(wdir, &(*ctx)->ctx); + } else { + if (directory[dirlen - 1] != '*') { + char *buf = _alloca(dirlen + 3); + + strcpy(buf, directory); + if (buf[dirlen - 1] != '/' && buf[dirlen - 1] != '\\') + strcpy(buf + dirlen, "/*"); + else + strcpy(buf + dirlen, "*"); + + directory = buf; + } + + (*ctx)->handle = FindFirstFile((TCHAR *)directory, &(*ctx)->ctx); + } + + if ((*ctx)->handle == INVALID_HANDLE_VALUE) { + free(*ctx); + *ctx = NULL; + errno = EINVAL; + return 0; + } + } else { + if (FindNextFile((*ctx)->handle, &(*ctx)->ctx) == FALSE) { + return 0; + } + } + if (sizeof(TCHAR) != sizeof(char)) { + TCHAR *wdir = (*ctx)->ctx.cFileName; + size_t index, len_0 = 0; + + while (wdir[len_0] && len_0 < (sizeof((*ctx)->entry_name) - 1)) + len_0++; + len_0++; + +# ifdef LP_MULTIBYTE_AVAILABLE + if (!WideCharToMultiByte(CP_DEFAULT, 0, (WCHAR *)wdir, len_0, + (*ctx)->entry_name, + sizeof((*ctx)->entry_name), NULL, 0)) +# endif + for (index = 0; index < len_0; index++) + (*ctx)->entry_name[index] = (char)wdir[index]; + } else + strncpy((*ctx)->entry_name, (const char *)(*ctx)->ctx.cFileName, + sizeof((*ctx)->entry_name) - 1); + + (*ctx)->entry_name[sizeof((*ctx)->entry_name) - 1] = '\0'; + + return (*ctx)->entry_name; +} + +int OPENSSL_DIR_end(OPENSSL_DIR_CTX **ctx) +{ + if (ctx != NULL && *ctx != NULL) { + FindClose((*ctx)->handle); + free(*ctx); + *ctx = NULL; + return 1; + } + errno = EINVAL; + return 0; +} #else -# include "LPdir_nyi.c" +/* Original LPdir_nyi.c */ +struct OPENSSL_dir_context_st { + void *dummy; +}; + +const char *OPENSSL_DIR_read(OPENSSL_DIR_CTX **ctx, const char *directory) +{ + errno = EINVAL; + return 0; +} + +int OPENSSL_DIR_end(OPENSSL_DIR_CTX **ctx) +{ + errno = EINVAL; + return 0; +} #endif diff --git a/openssl/src/crypto/o_fopen.c b/openssl/src/crypto/o_fopen.c index 09c28e0bf..abced7595 100644 --- a/openssl/src/crypto/o_fopen.c +++ b/openssl/src/crypto/o_fopen.c @@ -7,11 +7,11 @@ * https://www.openssl.org/source/license.html */ -# if defined(__linux) || defined(__sun) || defined(__hpux) +# if defined(__linux) || defined(__sun) /* * Following definition aliases fopen to fopen64 on above mentioned * platforms. This makes it possible to open and sequentially access files - * larger than 2GB from 32-bit application. It does not allow one to traverse + * larger than 2GB from 32-bit application. It does not allow to traverse * them beyond 2GB with fseek/ftell, but on the other hand *no* 32-bit * platform permits that, not with fseek/ftell. Not to mention that breaking * 2GB limit for seeking would require surgery to *our* API. But sequential @@ -87,8 +87,10 @@ FILE *openssl_fopen(const char *filename, const char *mode) char *iterator; char lastchar; - if ((newname = OPENSSL_malloc(strlen(filename) + 1)) == NULL) + if ((newname = OPENSSL_malloc(strlen(filename) + 1)) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return NULL; + } for (iterator = newname, lastchar = '\0'; *filename; filename++, iterator++) { diff --git a/openssl/src/crypto/o_str.c b/openssl/src/crypto/o_str.c index 065460336..731eb0fa2 100644 --- a/openssl/src/crypto/o_str.c +++ b/openssl/src/crypto/o_str.c @@ -1,5 +1,5 @@ /* - * Copyright 2003-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2003-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,12 +8,10 @@ */ #include "internal/e_os.h" -#include #include #include #include "crypto/ctype.h" #include "internal/cryptlib.h" -#include "internal/thread_once.h" #define DEFAULT_SEPARATOR ':' #define CH_ZERO '\0' @@ -56,8 +54,10 @@ void *CRYPTO_memdup(const void *data, size_t siz, const char* file, int line) return NULL; ret = CRYPTO_malloc(siz, file, line); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return NULL; + } return memcpy(ret, data, siz); } @@ -194,8 +194,10 @@ unsigned char *ossl_hexstr2buf_sep(const char *str, long *buflen, return NULL; } buf_n /= 2; - if ((buf = OPENSSL_malloc(buf_n)) == NULL) + if ((buf = OPENSSL_malloc(buf_n)) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return NULL; + } if (buflen != NULL) *buflen = 0; @@ -247,7 +249,7 @@ static int buf2hexstr_sep(char *str, size_t str_n, size_t *strlength, *q = CH_ZERO; #ifdef CHARSET_EBCDIC - ebcdic2ascii(str, str, q - str); + ebcdic2ascii(str, str, q - str - 1); #endif return 1; } @@ -268,8 +270,10 @@ char *ossl_buf2hexstr_sep(const unsigned char *buf, long buflen, char sep) return OPENSSL_zalloc(1); tmp_n = (sep != CH_ZERO) ? buflen * 3 : 1 + buflen * 2; - if ((tmp = OPENSSL_malloc(tmp_n)) == NULL) + if ((tmp = OPENSSL_malloc(tmp_n)) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return NULL; + } if (buf2hexstr_sep(tmp, tmp_n, NULL, buf, buflen, sep)) return tmp; @@ -279,13 +283,13 @@ char *ossl_buf2hexstr_sep(const unsigned char *buf, long buflen, char sep) /* - * Given a buffer of length 'buflen' return a OPENSSL_malloc'ed string with - * its hex representation @@@ (Contents of buffer are always kept in ASCII, - * also on EBCDIC machines) + * Given a buffer of length 'len' return a OPENSSL_malloc'ed string with its + * hex representation @@@ (Contents of buffer are always kept in ASCII, also + * on EBCDIC machines) */ char *OPENSSL_buf2hexstr(const unsigned char *buf, long buflen) { - return ossl_buf2hexstr_sep(buf, buflen, DEFAULT_SEPARATOR); + return ossl_buf2hexstr_sep(buf, buflen, ':'); } int openssl_strerror_r(int errnum, char *buf, size_t buflen) diff --git a/openssl/src/crypto/o_time.c b/openssl/src/crypto/o_time.c index 23ffe1624..b9b380a7f 100644 --- a/openssl/src/crypto/o_time.c +++ b/openssl/src/crypto/o_time.c @@ -15,29 +15,7 @@ struct tm *OPENSSL_gmtime(const time_t *timer, struct tm *result) { struct tm *ts = NULL; -#if defined(OPENSSL_THREADS) && defined(OPENSSL_SYS_VMS) - { - /* - * On VMS, gmtime_r() takes a 32-bit pointer as second argument. - * Since we can't know that |result| is in a space that can easily - * translate to a 32-bit pointer, we must store temporarily on stack - * and copy the result. The stack is always reachable with 32-bit - * pointers. - */ -#if defined(OPENSSL_SYS_VMS) && __INITIAL_POINTER_SIZE -# pragma pointer_size save -# pragma pointer_size 32 -#endif - struct tm data, *ts2 = &data; -#if defined OPENSSL_SYS_VMS && __INITIAL_POINTER_SIZE -# pragma pointer_size restore -#endif - if (gmtime_r(timer, ts2) == NULL) - return NULL; - memcpy(result, ts2, sizeof(struct tm)); - ts = result; - } -#elif defined(OPENSSL_THREADS) && !defined(OPENSSL_SYS_WIN32) && !defined(OPENSSL_SYS_MACOSX) +#if defined(OPENSSL_THREADS) && !defined(OPENSSL_SYS_WIN32) && !defined(OPENSSL_SYS_MACOSX) if (gmtime_r(timer, result) == NULL) return NULL; ts = result; diff --git a/openssl/src/crypto/objects/o_names.c b/openssl/src/crypto/objects/o_names.c index 5a468bba3..1efa0345f 100644 --- a/openssl/src/crypto/objects/o_names.c +++ b/openssl/src/crypto/objects/o_names.c @@ -89,6 +89,7 @@ int OBJ_NAME_new_index(unsigned long (*hash_func) (const char *), for (i = sk_NAME_FUNCS_num(name_funcs_stack); i < names_type_num; i++) { name_funcs = OPENSSL_zalloc(sizeof(*name_funcs)); if (name_funcs == NULL) { + ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE); ret = 0; goto out; } @@ -97,7 +98,7 @@ int OBJ_NAME_new_index(unsigned long (*hash_func) (const char *), push = sk_NAME_FUNCS_push(name_funcs_stack, name_funcs); if (!push) { - ERR_raise(ERR_LIB_OBJ, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE); OPENSSL_free(name_funcs); ret = 0; goto out; diff --git a/openssl/src/crypto/objects/obj_compat.h b/openssl/src/crypto/objects/obj_compat.h deleted file mode 100644 index 7d890d1c0..000000000 --- a/openssl/src/crypto/objects/obj_compat.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_NO_DEPRECATED_3_0 - -#define SN_id_tc26_cipher_gostr3412_2015_magma_ctracpkm SN_magma_ctr_acpkm -#define NID_id_tc26_cipher_gostr3412_2015_magma_ctracpkm NID_magma_ctr_acpkm -#define OBJ_id_tc26_cipher_gostr3412_2015_magma_ctracpkm OBJ_magma_ctr_acpkm - -#define SN_id_tc26_cipher_gostr3412_2015_magma_ctracpkm_omac SN_magma_ctr_acpkm_omac -#define NID_id_tc26_cipher_gostr3412_2015_magma_ctracpkm_omac NID_magma_ctr_acpkm_omac -#define OBJ_id_tc26_cipher_gostr3412_2015_magma_ctracpkm_omac OBJ_magma_ctr_acpkm_omac - -#define SN_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm SN_kuznyechik_ctr_acpkm -#define NID_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm NID_kuznyechik_ctr_acpkm -#define OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm OBJ_kuznyechik_ctr_acpkm - -#define SN_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm_omac SN_kuznyechik_ctr_acpkm_omac -#define NID_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm_omac NID_kuznyechik_ctr_acpkm_omac -#define OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik_ctracpkm_omac OBJ_kuznyechik_ctr_acpkm_omac - -#define SN_id_tc26_wrap_gostr3412_2015_magma_kexp15 SN_magma_kexp15 -#define NID_id_tc26_wrap_gostr3412_2015_magma_kexp15 NID_magma_kexp15 -#define OBJ_id_tc26_wrap_gostr3412_2015_magma_kexp15 OBJ_magma_kexp15 - -#define SN_id_tc26_wrap_gostr3412_2015_kuznyechik_kexp15 SN_kuznyechik_kexp15 -#define NID_id_tc26_wrap_gostr3412_2015_kuznyechik_kexp15 NID_kuznyechik_kexp15 -#define OBJ_id_tc26_wrap_gostr3412_2015_kuznyechik_kexp15 OBJ_kuznyechik_kexp15 - -#define SN_grasshopper_ecb SN_kuznyechik_ecb -#define NID_grasshopper_ecb NID_kuznyechik_ecb - -#define SN_grasshopper_ctr SN_kuznyechik_ctr -#define NID_grasshopper_ctr NID_kuznyechik_ctr - -#define SN_grasshopper_ofb SN_kuznyechik_ofb -#define NID_grasshopper_ofb NID_kuznyechik_ofb - -#define SN_grasshopper_cbc SN_kuznyechik_cbc -#define NID_grasshopper_cbc NID_kuznyechik_cbc - -#define SN_grasshopper_cfb SN_kuznyechik_cfb -#define NID_grasshopper_cfb NID_kuznyechik_cfb - -#define SN_grasshopper_mac SN_kuznyechik_mac -#define NID_grasshopper_mac NID_kuznyechik_mac - -#endif /* OPENSSL_NO_DEPRECATED_3_0 */ diff --git a/openssl/src/crypto/objects/obj_dat.c b/openssl/src/crypto/objects/obj_dat.c index 493b0e11d..22e0a6334 100644 --- a/openssl/src/crypto/objects/obj_dat.c +++ b/openssl/src/crypto/objects/obj_dat.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,8 +11,6 @@ #include "crypto/ctype.h" #include #include "internal/cryptlib.h" -#include "internal/thread_once.h" -#include "internal/tsan_assist.h" #include #include #include "crypto/objects.h" @@ -20,7 +18,7 @@ #include "crypto/asn1.h" #include "obj_local.h" -/* obj_dat.h is generated from objects.txt and obj_mac.{num,h} by obj_dat.pl */ +/* obj_dat.h is generated from objects.h by obj_dat.pl */ #include "obj_dat.h" DECLARE_OBJ_BSEARCH_CMP_FN(const ASN1_OBJECT *, unsigned int, sn); @@ -37,72 +35,8 @@ struct added_obj_st { ASN1_OBJECT *obj; }; +static int new_nid = NUM_NID; static LHASH_OF(ADDED_OBJ) *added = NULL; -static CRYPTO_RWLOCK *ossl_obj_lock = NULL; -#ifdef TSAN_REQUIRES_LOCKING -static CRYPTO_RWLOCK *ossl_obj_nid_lock = NULL; -#endif - -static CRYPTO_ONCE ossl_obj_lock_init = CRYPTO_ONCE_STATIC_INIT; - -static ossl_inline void objs_free_locks(void) -{ - CRYPTO_THREAD_lock_free(ossl_obj_lock); - ossl_obj_lock = NULL; -#ifdef TSAN_REQUIRES_LOCKING - CRYPTO_THREAD_lock_free(ossl_obj_nid_lock); - ossl_obj_nid_lock = NULL; -#endif -} - -DEFINE_RUN_ONCE_STATIC(obj_lock_initialise) -{ - ossl_obj_lock = CRYPTO_THREAD_lock_new(); - if (ossl_obj_lock == NULL) - return 0; - -#ifdef TSAN_REQUIRES_LOCKING - ossl_obj_nid_lock = CRYPTO_THREAD_lock_new(); - if (ossl_obj_nid_lock == NULL) { - objs_free_locks(); - return 0; - } -#endif - return 1; -} - -static ossl_inline int ossl_init_added_lock(void) -{ -#ifndef OPENSSL_NO_AUTOLOAD_CONFIG - /* Make sure we've loaded config before checking for any "added" objects */ - OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); -#endif - return RUN_ONCE(&ossl_obj_lock_init, obj_lock_initialise); -} - -static ossl_inline int ossl_obj_write_lock(int lock) -{ - if (!lock) - return 1; - if (!ossl_init_added_lock()) - return 0; - return CRYPTO_THREAD_write_lock(ossl_obj_lock); -} - -static ossl_inline int ossl_obj_read_lock(int lock) -{ - if (!lock) - return 1; - if (!ossl_init_added_lock()) - return 0; - return CRYPTO_THREAD_read_lock(ossl_obj_lock); -} - -static ossl_inline void ossl_obj_unlock(int lock) -{ - if (lock) - CRYPTO_THREAD_unlock(ossl_obj_lock); -} static int sn_cmp(const ASN1_OBJECT *const *a, const unsigned int *b) { @@ -128,7 +62,7 @@ static unsigned long added_obj_hash(const ADDED_OBJ *ca) a = ca->obj; switch (ca->type) { case ADDED_DATA: - ret = (unsigned long)a->length << 20UL; + ret = a->length << 20L; p = (unsigned char *)a->data; for (i = 0; i < a->length; i++) ret ^= p[i] << ((i * 3) % 24); @@ -189,6 +123,14 @@ static int added_obj_cmp(const ADDED_OBJ *ca, const ADDED_OBJ *cb) } } +static int init_added(void) +{ + if (added != NULL) + return 1; + added = lh_ADDED_OBJ_new(added_obj_hash, added_obj_cmp); + return added != NULL; +} + static void cleanup1_doall(ADDED_OBJ *a) { a->obj->nid = 0; @@ -210,85 +152,47 @@ static void cleanup3_doall(ADDED_OBJ *a) void ossl_obj_cleanup_int(void) { - if (added != NULL) { - lh_ADDED_OBJ_set_down_load(added, 0); - lh_ADDED_OBJ_doall(added, cleanup1_doall); /* zero counters */ - lh_ADDED_OBJ_doall(added, cleanup2_doall); /* set counters */ - lh_ADDED_OBJ_doall(added, cleanup3_doall); /* free objects */ - lh_ADDED_OBJ_free(added); - added = NULL; - } - objs_free_locks(); + if (added == NULL) + return; + lh_ADDED_OBJ_set_down_load(added, 0); + lh_ADDED_OBJ_doall(added, cleanup1_doall); /* zero counters */ + lh_ADDED_OBJ_doall(added, cleanup2_doall); /* set counters */ + lh_ADDED_OBJ_doall(added, cleanup3_doall); /* free objects */ + lh_ADDED_OBJ_free(added); + added = NULL; } -/* - * Requires that the ossl_obj_lock be held - * if TSAN_REQUIRES_LOCKING defined - */ -static int obj_new_nid_unlocked(int num) +int OBJ_new_nid(int num) { - static TSAN_QUALIFIER int new_nid = NUM_NID; -#ifdef TSAN_REQUIRES_LOCKING int i; i = new_nid; new_nid += num; - - return i; -#else - return tsan_add(&new_nid, num); -#endif -} - -int OBJ_new_nid(int num) -{ -#ifdef TSAN_REQUIRES_LOCKING - int i; - - if (!ossl_obj_write_lock(1)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_WRITE_LOCK); - return NID_undef; - } - - i = obj_new_nid_unlocked(num); - - ossl_obj_unlock(1); - return i; -#else - return obj_new_nid_unlocked(num); -#endif } -static int ossl_obj_add_object(const ASN1_OBJECT *obj, int lock) +int OBJ_add_object(const ASN1_OBJECT *obj) { - ASN1_OBJECT *o = NULL; + ASN1_OBJECT *o; ADDED_OBJ *ao[4] = { NULL, NULL, NULL, NULL }, *aop; int i; + if (added == NULL) + if (!init_added()) + return 0; if ((o = OBJ_dup(obj)) == NULL) - return NID_undef; - if ((ao[ADDED_NID] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL - || (o->length != 0 - && obj->data != NULL - && (ao[ADDED_DATA] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL) - || (o->sn != NULL - && (ao[ADDED_SNAME] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL) - || (o->ln != NULL - && (ao[ADDED_LNAME] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL)) - goto err2; - - if (!ossl_obj_write_lock(lock)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_WRITE_LOCK); + goto err; + if ((ao[ADDED_NID] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL) goto err2; - } - if (added == NULL) { - added = lh_ADDED_OBJ_new(added_obj_hash, added_obj_cmp); - if (added == NULL) { - ERR_raise(ERR_LIB_OBJ, ERR_R_CRYPTO_LIB); - goto err; - } - } + if ((o->length != 0) && (obj->data != NULL)) + if ((ao[ADDED_DATA] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL) + goto err2; + if (o->sn != NULL) + if ((ao[ADDED_SNAME] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL) + goto err2; + if (o->ln != NULL) + if ((ao[ADDED_LNAME] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL) + goto err2; for (i = ADDED_DATA; i <= ADDED_NID; i++) { if (ao[i] != NULL) { @@ -303,12 +207,10 @@ static int ossl_obj_add_object(const ASN1_OBJECT *obj, int lock) ~(ASN1_OBJECT_FLAG_DYNAMIC | ASN1_OBJECT_FLAG_DYNAMIC_STRINGS | ASN1_OBJECT_FLAG_DYNAMIC_DATA); - ossl_obj_unlock(lock); return o->nid; - - err: - ossl_obj_unlock(lock); err2: + ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE); + err: for (i = ADDED_DATA; i <= ADDED_NID; i++) OPENSSL_free(ao[i]); ASN1_OBJECT_free(o); @@ -317,23 +219,27 @@ static int ossl_obj_add_object(const ASN1_OBJECT *obj, int lock) ASN1_OBJECT *OBJ_nid2obj(int n) { - ADDED_OBJ ad, *adp = NULL; + ADDED_OBJ ad, *adp; ASN1_OBJECT ob; - if (n == NID_undef - || (n > 0 && n < NUM_NID && nid_objs[n].nid != NID_undef)) + if ((n >= 0) && (n < NUM_NID)) { + if ((n != NID_undef) && (nid_objs[n].nid == NID_undef)) { + ERR_raise(ERR_LIB_OBJ, OBJ_R_UNKNOWN_NID); + return NULL; + } return (ASN1_OBJECT *)&(nid_objs[n]); + } + + /* Make sure we've loaded config before checking for any "added" objects */ + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); + + if (added == NULL) + return NULL; ad.type = ADDED_NID; ad.obj = &ob; ob.nid = n; - if (!ossl_obj_read_lock(1)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_READ_LOCK); - return NULL; - } - if (added != NULL) - adp = lh_ADDED_OBJ_retrieve(added, &ad); - ossl_obj_unlock(1); + adp = lh_ADDED_OBJ_retrieve(added, &ad); if (adp != NULL) return adp->obj; @@ -343,16 +249,62 @@ ASN1_OBJECT *OBJ_nid2obj(int n) const char *OBJ_nid2sn(int n) { - ASN1_OBJECT *ob = OBJ_nid2obj(n); + ADDED_OBJ ad, *adp; + ASN1_OBJECT ob; + + if ((n >= 0) && (n < NUM_NID)) { + if ((n != NID_undef) && (nid_objs[n].nid == NID_undef)) { + ERR_raise(ERR_LIB_OBJ, OBJ_R_UNKNOWN_NID); + return NULL; + } + return nid_objs[n].sn; + } + + /* Make sure we've loaded config before checking for any "added" objects */ + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); + + if (added == NULL) + return NULL; - return ob == NULL ? NULL : ob->sn; + ad.type = ADDED_NID; + ad.obj = &ob; + ob.nid = n; + adp = lh_ADDED_OBJ_retrieve(added, &ad); + if (adp != NULL) + return adp->obj->sn; + + ERR_raise(ERR_LIB_OBJ, OBJ_R_UNKNOWN_NID); + return NULL; } const char *OBJ_nid2ln(int n) { - ASN1_OBJECT *ob = OBJ_nid2obj(n); + ADDED_OBJ ad, *adp; + ASN1_OBJECT ob; + + if ((n >= 0) && (n < NUM_NID)) { + if ((n != NID_undef) && (nid_objs[n].nid == NID_undef)) { + ERR_raise(ERR_LIB_OBJ, OBJ_R_UNKNOWN_NID); + return NULL; + } + return nid_objs[n].ln; + } + + /* Make sure we've loaded config before checking for any "added" objects */ + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); + + if (added == NULL) + return NULL; - return ob == NULL ? NULL : ob->ln; + ad.type = ADDED_NID; + ad.obj = &ob; + ob.nid = n; + adp = lh_ADDED_OBJ_retrieve(added, &ad); + if (adp != NULL) + return adp->obj->ln; + + ERR_raise(ERR_LIB_OBJ, OBJ_R_UNKNOWN_NID); + return NULL; } static int obj_cmp(const ASN1_OBJECT *const *ap, const unsigned int *bp) @@ -371,35 +323,33 @@ static int obj_cmp(const ASN1_OBJECT *const *ap, const unsigned int *bp) IMPLEMENT_OBJ_BSEARCH_CMP_FN(const ASN1_OBJECT *, unsigned int, obj); -static int ossl_obj_obj2nid(const ASN1_OBJECT *a, const int lock) +int OBJ_obj2nid(const ASN1_OBJECT *a) { - int nid = NID_undef; const unsigned int *op; ADDED_OBJ ad, *adp; if (a == NULL) return NID_undef; - if (a->nid != NID_undef) + if (a->nid != 0) return a->nid; + if (a->length == 0) return NID_undef; - op = OBJ_bsearch_obj(&a, obj_objs, NUM_OBJ); - if (op != NULL) - return nid_objs[*op].nid; - if (!ossl_obj_read_lock(lock)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_READ_LOCK); - return NID_undef; - } + /* Make sure we've loaded config before checking for any "added" objects */ + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); + if (added != NULL) { ad.type = ADDED_DATA; - ad.obj = (ASN1_OBJECT *)a; /* casting away const is harmless here */ + ad.obj = (ASN1_OBJECT *)a; /* XXX: ugly but harmless */ adp = lh_ADDED_OBJ_retrieve(added, &ad); if (adp != NULL) - nid = adp->obj->nid; + return adp->obj->nid; } - ossl_obj_unlock(lock); - return nid; + op = OBJ_bsearch_obj(&a, obj_objs, NUM_OBJ); + if (op == NULL) + return NID_undef; + return nid_objs[*op].nid; } /* @@ -408,20 +358,20 @@ static int ossl_obj_obj2nid(const ASN1_OBJECT *a, const int lock) * into an object: unlike OBJ_txt2nid it can be used with any objects, not * just registered ones. */ + ASN1_OBJECT *OBJ_txt2obj(const char *s, int no_name) { int nid = NID_undef; - ASN1_OBJECT *op = NULL; + ASN1_OBJECT *op; unsigned char *buf; unsigned char *p; const unsigned char *cp; int i, j; if (!no_name) { - if ((nid = OBJ_sn2nid(s)) != NID_undef - || (nid = OBJ_ln2nid(s)) != NID_undef) { + if (((nid = OBJ_sn2nid(s)) != NID_undef) || + ((nid = OBJ_ln2nid(s)) != NID_undef)) return OBJ_nid2obj(nid); - } if (!ossl_isdigit(*s)) { ERR_raise(ERR_LIB_OBJ, OBJ_R_UNKNOWN_OBJECT_NAME); return NULL; @@ -430,16 +380,22 @@ ASN1_OBJECT *OBJ_txt2obj(const char *s, int no_name) /* Work out size of content octets */ i = a2d_ASN1_OBJECT(NULL, 0, s, -1); - if (i <= 0) + if (i <= 0) { + /* Don't clear the error */ + /* + * ERR_clear_error(); + */ return NULL; - + } /* Work out total size */ j = ASN1_object_size(0, i, V_ASN1_OBJECT); if (j < 0) return NULL; - if ((buf = OPENSSL_malloc(j)) == NULL) + if ((buf = OPENSSL_malloc(j)) == NULL) { + ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE); return NULL; + } p = buf; /* Write out tag+length */ @@ -460,23 +416,24 @@ int OBJ_obj2txt(char *buf, int buf_len, const ASN1_OBJECT *a, int no_name) unsigned long l; const unsigned char *p; char tbuf[DECIMAL_SIZE(i) + DECIMAL_SIZE(l) + 2]; - const char *s; /* Ensure that, at every state, |buf| is NUL-terminated. */ - if (buf != NULL && buf_len > 0) + if (buf && buf_len > 0) buf[0] = '\0'; - if (a == NULL || a->data == NULL) + if ((a == NULL) || (a->data == NULL)) return 0; if (!no_name && (nid = OBJ_obj2nid(a)) != NID_undef) { + const char *s; s = OBJ_nid2ln(nid); if (s == NULL) s = OBJ_nid2sn(nid); - if (s != NULL) { - if (buf != NULL) + if (s) { + if (buf) OPENSSL_strlcpy(buf, s, buf_len); - return (int)strlen(s); + n = strlen(s); + return n; } } @@ -510,19 +467,17 @@ int OBJ_obj2txt(char *buf, int buf_len, const ASN1_OBJECT *a, int no_name) use_bn = 0; for (;;) { unsigned char c = *p++; - len--; - if (len == 0 && (c & 0x80) != 0) + if ((len == 0) && (c & 0x80)) goto err; if (use_bn) { if (!BN_add_word(bl, c & 0x7f)) goto err; - } else { + } else l |= c & 0x7f; - } - if ((c & 0x80) == 0) + if (!(c & 0x80)) break; - if (!use_bn && l > (ULONG_MAX >> 7L)) { + if (!use_bn && (l > (ULONG_MAX >> 7L))) { if (bl == NULL && (bl = BN_new()) == NULL) goto err; if (!BN_set_word(bl, l)) @@ -532,9 +487,8 @@ int OBJ_obj2txt(char *buf, int buf_len, const ASN1_OBJECT *a, int no_name) if (use_bn) { if (!BN_lshift(bl, bl, 7)) goto err; - } else { + } else l <<= 7L; - } } if (first) { @@ -544,14 +498,13 @@ int OBJ_obj2txt(char *buf, int buf_len, const ASN1_OBJECT *a, int no_name) if (use_bn) { if (!BN_sub_word(bl, 80)) goto err; - } else { + } else l -= 80; - } } else { i = (int)(l / 40); l -= (long)(i * 40); } - if (buf != NULL && buf_len > 1) { + if (buf && (buf_len > 1)) { *buf++ = i + '0'; *buf = '\0'; buf_len--; @@ -565,7 +518,7 @@ int OBJ_obj2txt(char *buf, int buf_len, const ASN1_OBJECT *a, int no_name) if (!bndec) goto err; i = strlen(bndec); - if (buf != NULL) { + if (buf) { if (buf_len > 1) { *buf++ = '.'; *buf = '\0'; @@ -586,7 +539,7 @@ int OBJ_obj2txt(char *buf, int buf_len, const ASN1_OBJECT *a, int no_name) } else { BIO_snprintf(tbuf, sizeof(tbuf), ".%lu", l); i = strlen(tbuf); - if (buf && buf_len > 0) { + if (buf && (buf_len > 0)) { OPENSSL_strlcpy(buf, tbuf, buf_len); if (i > buf_len) { buf += buf_len; @@ -611,13 +564,11 @@ int OBJ_obj2txt(char *buf, int buf_len, const ASN1_OBJECT *a, int no_name) int OBJ_txt2nid(const char *s) { - ASN1_OBJECT *obj = OBJ_txt2obj(s, 0); - int nid = NID_undef; - - if (obj != NULL) { - nid = OBJ_obj2nid(obj); - ASN1_OBJECT_free(obj); - } + ASN1_OBJECT *obj; + int nid; + obj = OBJ_txt2obj(s, 0); + nid = OBJ_obj2nid(obj); + ASN1_OBJECT_free(obj); return nid; } @@ -627,25 +578,22 @@ int OBJ_ln2nid(const char *s) const ASN1_OBJECT *oo = &o; ADDED_OBJ ad, *adp; const unsigned int *op; - int nid = NID_undef; + + /* Make sure we've loaded config before checking for any "added" objects */ + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); o.ln = s; - op = OBJ_bsearch_ln(&oo, ln_objs, NUM_LN); - if (op != NULL) - return nid_objs[*op].nid; - if (!ossl_obj_read_lock(1)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_READ_LOCK); - return NID_undef; - } if (added != NULL) { ad.type = ADDED_LNAME; ad.obj = &o; adp = lh_ADDED_OBJ_retrieve(added, &ad); if (adp != NULL) - nid = adp->obj->nid; + return adp->obj->nid; } - ossl_obj_unlock(1); - return nid; + op = OBJ_bsearch_ln(&oo, ln_objs, NUM_LN); + if (op == NULL) + return NID_undef; + return nid_objs[*op].nid; } int OBJ_sn2nid(const char *s) @@ -654,25 +602,22 @@ int OBJ_sn2nid(const char *s) const ASN1_OBJECT *oo = &o; ADDED_OBJ ad, *adp; const unsigned int *op; - int nid = NID_undef; + + /* Make sure we've loaded config before checking for any "added" objects */ + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); o.sn = s; - op = OBJ_bsearch_sn(&oo, sn_objs, NUM_SN); - if (op != NULL) - return nid_objs[*op].nid; - if (!ossl_obj_read_lock(1)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_READ_LOCK); - return NID_undef; - } if (added != NULL) { ad.type = ADDED_SNAME; ad.obj = &o; adp = lh_ADDED_OBJ_retrieve(added, &ad); if (adp != NULL) - nid = adp->obj->nid; + return adp->obj->nid; } - ossl_obj_unlock(1); - return nid; + op = OBJ_bsearch_sn(&oo, sn_objs, NUM_SN); + if (op == NULL) + return NID_undef; + return nid_objs[*op].nid; } const void *OBJ_bsearch_(const void *key, const void *base, int num, int size, @@ -697,14 +642,13 @@ const void *OBJ_bsearch_ex_(const void *key, const void *base, int num, if (p == NULL) { const char *base_ = base; int l, h, i = 0, c = 0; - char *p1; for (i = 0; i < num; ++i) { - p1 = &(base_[i * size]); - c = (*cmp) (key, p1); + p = &(base_[i * size]); + c = (*cmp) (key, p); if (c == 0 || (c < 0 && (flags & OBJ_BSEARCH_VALUE_ON_NOMATCH))) - return p1; + return p; } } #endif @@ -769,12 +713,6 @@ int OBJ_create(const char *oid, const char *sn, const char *ln) ASN1_OBJECT *tmpoid = NULL; int ok = 0; - /* With no arguments at all, nothing can be done */ - if (oid == NULL && sn == NULL && ln == NULL) { - ERR_raise(ERR_LIB_OBJ, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - /* Check to see if short or long name already present */ if ((sn != NULL && OBJ_sn2nid(sn) != NID_undef) || (ln != NULL && OBJ_ln2nid(ln) != NID_undef)) { @@ -782,48 +720,27 @@ int OBJ_create(const char *oid, const char *sn, const char *ln) return 0; } - if (oid != NULL) { - /* Convert numerical OID string to an ASN1_OBJECT structure */ - tmpoid = OBJ_txt2obj(oid, 1); - if (tmpoid == NULL) - return 0; - } else { - /* Create a no-OID ASN1_OBJECT */ - tmpoid = ASN1_OBJECT_new(); - if (tmpoid == NULL) { - ERR_raise(ERR_LIB_OBJ, ERR_R_ASN1_LIB); - return 0; - } - } - - if (!ossl_obj_write_lock(1)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_WRITE_LOCK); - ASN1_OBJECT_free(tmpoid); + /* Convert numerical OID string to an ASN1_OBJECT structure */ + tmpoid = OBJ_txt2obj(oid, 1); + if (tmpoid == NULL) return 0; - } /* If NID is not NID_undef then object already exists */ - if (oid != NULL - && ossl_obj_obj2nid(tmpoid, 0) != NID_undef) { + if (OBJ_obj2nid(tmpoid) != NID_undef) { ERR_raise(ERR_LIB_OBJ, OBJ_R_OID_EXISTS); goto err; } - tmpoid->nid = obj_new_nid_unlocked(1); - - if (tmpoid->nid == NID_undef) - goto err; - + tmpoid->nid = OBJ_new_nid(1); tmpoid->sn = (char *)sn; tmpoid->ln = (char *)ln; - ok = ossl_obj_add_object(tmpoid, 0); + ok = OBJ_add_object(tmpoid); tmpoid->sn = NULL; tmpoid->ln = NULL; err: - ossl_obj_unlock(1); ASN1_OBJECT_free(tmpoid); return ok; } @@ -841,13 +758,3 @@ const unsigned char *OBJ_get0_data(const ASN1_OBJECT *obj) return NULL; return obj->data; } - -int OBJ_add_object(const ASN1_OBJECT *obj) -{ - return ossl_obj_add_object(obj, 1); -} - -int OBJ_obj2nid(const ASN1_OBJECT *a) -{ - return ossl_obj_obj2nid(a, 1); -} diff --git a/openssl/src/crypto/objects/obj_dat.h b/openssl/src/crypto/objects/obj_dat.h index eaf098159..16bf5d227 100644 --- a/openssl/src/crypto/objects/obj_dat.h +++ b/openssl/src/crypto/objects/obj_dat.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by crypto/objects/obj_dat.pl * - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at @@ -10,1833 +10,1579 @@ */ /* Serialized OID's */ -static const unsigned char so[8487] = { +static const unsigned char so[6609] = { 0x2A,0x86,0x48,0x86,0xF7,0x0D, /* [ 0] OBJ_rsadsi */ 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01, /* [ 6] OBJ_pkcs */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x02, /* [ 13] OBJ_md2 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x05, /* [ 21] OBJ_md5 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x04, /* [ 29] OBJ_rc4 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x01, /* [ 37] OBJ_rsaEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x02, /* [ 46] OBJ_md2WithRSAEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x04, /* [ 55] OBJ_md5WithRSAEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x01, /* [ 64] OBJ_pbeWithMD2AndDES_CBC */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x03, /* [ 73] OBJ_pbeWithMD5AndDES_CBC */ - 0x55, /* [ 82] OBJ_X500 */ - 0x55,0x04, /* [ 83] OBJ_X509 */ - 0x55,0x04,0x03, /* [ 85] OBJ_commonName */ - 0x55,0x04,0x06, /* [ 88] OBJ_countryName */ - 0x55,0x04,0x07, /* [ 91] OBJ_localityName */ - 0x55,0x04,0x08, /* [ 94] OBJ_stateOrProvinceName */ - 0x55,0x04,0x0A, /* [ 97] OBJ_organizationName */ - 0x55,0x04,0x0B, /* [ 100] OBJ_organizationalUnitName */ - 0x55,0x08,0x01,0x01, /* [ 103] OBJ_rsa */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07, /* [ 107] OBJ_pkcs7 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x01, /* [ 115] OBJ_pkcs7_data */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x02, /* [ 124] OBJ_pkcs7_signed */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x03, /* [ 133] OBJ_pkcs7_enveloped */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x04, /* [ 142] OBJ_pkcs7_signedAndEnveloped */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x05, /* [ 151] OBJ_pkcs7_digest */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x06, /* [ 160] OBJ_pkcs7_encrypted */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x03, /* [ 169] OBJ_pkcs3 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x03,0x01, /* [ 177] OBJ_dhKeyAgreement */ - 0x2B,0x0E,0x03,0x02,0x06, /* [ 186] OBJ_des_ecb */ - 0x2B,0x0E,0x03,0x02,0x09, /* [ 191] OBJ_des_cfb64 */ - 0x2B,0x0E,0x03,0x02,0x07, /* [ 196] OBJ_des_cbc */ - 0x2B,0x0E,0x03,0x02,0x11, /* [ 201] OBJ_des_ede_ecb */ - 0x2B,0x06,0x01,0x04,0x01,0x81,0x3C,0x07,0x01,0x01,0x02, /* [ 206] OBJ_idea_cbc */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x02, /* [ 217] OBJ_rc2_cbc */ - 0x2B,0x0E,0x03,0x02,0x12, /* [ 225] OBJ_sha */ - 0x2B,0x0E,0x03,0x02,0x0F, /* [ 230] OBJ_shaWithRSAEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x07, /* [ 235] OBJ_des_ede3_cbc */ - 0x2B,0x0E,0x03,0x02,0x08, /* [ 243] OBJ_des_ofb64 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09, /* [ 248] OBJ_pkcs9 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x01, /* [ 256] OBJ_pkcs9_emailAddress */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x02, /* [ 265] OBJ_pkcs9_unstructuredName */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x03, /* [ 274] OBJ_pkcs9_contentType */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x04, /* [ 283] OBJ_pkcs9_messageDigest */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x05, /* [ 292] OBJ_pkcs9_signingTime */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x06, /* [ 301] OBJ_pkcs9_countersignature */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x07, /* [ 310] OBJ_pkcs9_challengePassword */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x08, /* [ 319] OBJ_pkcs9_unstructuredAddress */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x09, /* [ 328] OBJ_pkcs9_extCertAttributes */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42, /* [ 337] OBJ_netscape */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01, /* [ 344] OBJ_netscape_cert_extension */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x02, /* [ 352] OBJ_netscape_data_type */ - 0x2B,0x0E,0x03,0x02,0x1A, /* [ 360] OBJ_sha1 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x05, /* [ 365] OBJ_sha1WithRSAEncryption */ - 0x2B,0x0E,0x03,0x02,0x0D, /* [ 374] OBJ_dsaWithSHA */ - 0x2B,0x0E,0x03,0x02,0x0C, /* [ 379] OBJ_dsa_2 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0B, /* [ 384] OBJ_pbeWithSHA1AndRC2_CBC */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0C, /* [ 393] OBJ_id_pbkdf2 */ - 0x2B,0x0E,0x03,0x02,0x1B, /* [ 402] OBJ_dsaWithSHA1_2 */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x01, /* [ 407] OBJ_netscape_cert_type */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x02, /* [ 416] OBJ_netscape_base_url */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x03, /* [ 425] OBJ_netscape_revocation_url */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x04, /* [ 434] OBJ_netscape_ca_revocation_url */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x07, /* [ 443] OBJ_netscape_renewal_url */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x08, /* [ 452] OBJ_netscape_ca_policy_url */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x0C, /* [ 461] OBJ_netscape_ssl_server_name */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x0D, /* [ 470] OBJ_netscape_comment */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x02,0x05, /* [ 479] OBJ_netscape_cert_sequence */ - 0x55,0x1D, /* [ 488] OBJ_id_ce */ - 0x55,0x1D,0x0E, /* [ 490] OBJ_subject_key_identifier */ - 0x55,0x1D,0x0F, /* [ 493] OBJ_key_usage */ - 0x55,0x1D,0x10, /* [ 496] OBJ_private_key_usage_period */ - 0x55,0x1D,0x11, /* [ 499] OBJ_subject_alt_name */ - 0x55,0x1D,0x12, /* [ 502] OBJ_issuer_alt_name */ - 0x55,0x1D,0x13, /* [ 505] OBJ_basic_constraints */ - 0x55,0x1D,0x14, /* [ 508] OBJ_crl_number */ - 0x55,0x1D,0x20, /* [ 511] OBJ_certificate_policies */ - 0x55,0x1D,0x23, /* [ 514] OBJ_authority_key_identifier */ - 0x2B,0x06,0x01,0x04,0x01,0x97,0x55,0x01,0x02, /* [ 517] OBJ_bf_cbc */ - 0x55,0x08,0x03,0x65, /* [ 526] OBJ_mdc2 */ - 0x55,0x08,0x03,0x64, /* [ 530] OBJ_mdc2WithRSA */ - 0x55,0x04,0x2A, /* [ 534] OBJ_givenName */ - 0x55,0x04,0x04, /* [ 537] OBJ_surname */ - 0x55,0x04,0x2B, /* [ 540] OBJ_initials */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2C, /* [ 543] OBJ_uniqueIdentifier */ - 0x55,0x1D,0x1F, /* [ 553] OBJ_crl_distribution_points */ - 0x2B,0x0E,0x03,0x02,0x03, /* [ 556] OBJ_md5WithRSA */ - 0x55,0x04,0x05, /* [ 561] OBJ_serialNumber */ - 0x55,0x04,0x0C, /* [ 564] OBJ_title */ - 0x55,0x04,0x0D, /* [ 567] OBJ_description */ - 0x2A,0x86,0x48,0x86,0xF6,0x7D,0x07,0x42,0x0A, /* [ 570] OBJ_cast5_cbc */ - 0x2A,0x86,0x48,0x86,0xF6,0x7D,0x07,0x42,0x0C, /* [ 579] OBJ_pbeWithMD5AndCast5_CBC */ - 0x2A,0x86,0x48,0xCE,0x38,0x04,0x03, /* [ 588] OBJ_dsaWithSHA1 */ - 0x2B,0x0E,0x03,0x02,0x1D, /* [ 595] OBJ_sha1WithRSA */ - 0x2A,0x86,0x48,0xCE,0x38,0x04,0x01, /* [ 600] OBJ_dsa */ - 0x2B,0x24,0x03,0x02,0x01, /* [ 607] OBJ_ripemd160 */ - 0x2B,0x24,0x03,0x03,0x01,0x02, /* [ 612] OBJ_ripemd160WithRSA */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x08, /* [ 618] OBJ_rc5_cbc */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x08, /* [ 626] OBJ_zlib_compression */ - 0x55,0x1D,0x25, /* [ 637] OBJ_ext_key_usage */ - 0x2B,0x06,0x01,0x05,0x05,0x07, /* [ 640] OBJ_id_pkix */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03, /* [ 646] OBJ_id_kp */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x01, /* [ 653] OBJ_server_auth */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x02, /* [ 661] OBJ_client_auth */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x03, /* [ 669] OBJ_code_sign */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x04, /* [ 677] OBJ_email_protect */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x08, /* [ 685] OBJ_time_stamp */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x02,0x01,0x15, /* [ 693] OBJ_ms_code_ind */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x02,0x01,0x16, /* [ 703] OBJ_ms_code_com */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x0A,0x03,0x01, /* [ 713] OBJ_ms_ctl_sign */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x0A,0x03,0x03, /* [ 723] OBJ_ms_sgc */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x0A,0x03,0x04, /* [ 733] OBJ_ms_efs */ - 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x04,0x01, /* [ 743] OBJ_ns_sgc */ - 0x55,0x1D,0x1B, /* [ 752] OBJ_delta_crl */ - 0x55,0x1D,0x15, /* [ 755] OBJ_crl_reason */ - 0x55,0x1D,0x18, /* [ 758] OBJ_invalidity_date */ - 0x2B,0x65,0x01,0x04,0x01, /* [ 761] OBJ_sxnet */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x01, /* [ 766] OBJ_pbe_WithSHA1And128BitRC4 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x02, /* [ 776] OBJ_pbe_WithSHA1And40BitRC4 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x03, /* [ 786] OBJ_pbe_WithSHA1And3_Key_TripleDES_CBC */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x04, /* [ 796] OBJ_pbe_WithSHA1And2_Key_TripleDES_CBC */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x05, /* [ 806] OBJ_pbe_WithSHA1And128BitRC2_CBC */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x06, /* [ 816] OBJ_pbe_WithSHA1And40BitRC2_CBC */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x01, /* [ 826] OBJ_keyBag */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x02, /* [ 837] OBJ_pkcs8ShroudedKeyBag */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x03, /* [ 848] OBJ_certBag */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x04, /* [ 859] OBJ_crlBag */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x05, /* [ 870] OBJ_secretBag */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x06, /* [ 881] OBJ_safeContentsBag */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x14, /* [ 892] OBJ_friendlyName */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x15, /* [ 901] OBJ_localKeyID */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x16,0x01, /* [ 910] OBJ_x509Certificate */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x16,0x02, /* [ 920] OBJ_sdsiCertificate */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x17,0x01, /* [ 930] OBJ_x509Crl */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0D, /* [ 940] OBJ_pbes2 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0E, /* [ 949] OBJ_pbmac1 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x07, /* [ 958] OBJ_hmacWithSHA1 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x02,0x01, /* [ 966] OBJ_id_qt_cps */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x02,0x02, /* [ 974] OBJ_id_qt_unotice */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x0F, /* [ 982] OBJ_SMIMECapabilities */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x04, /* [ 991] OBJ_pbeWithMD2AndRC2_CBC */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x06, /* [ 1000] OBJ_pbeWithMD5AndRC2_CBC */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0A, /* [ 1009] OBJ_pbeWithSHA1AndDES_CBC */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x02,0x01,0x0E, /* [ 1018] OBJ_ms_ext_req */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x0E, /* [ 1028] OBJ_ext_req */ - 0x55,0x04,0x29, /* [ 1037] OBJ_name */ - 0x55,0x04,0x2E, /* [ 1040] OBJ_dnQualifier */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01, /* [ 1043] OBJ_id_pe */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30, /* [ 1050] OBJ_id_ad */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x01, /* [ 1057] OBJ_info_access */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01, /* [ 1065] OBJ_ad_OCSP */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x02, /* [ 1073] OBJ_ad_ca_issuers */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x09, /* [ 1081] OBJ_OCSP_sign */ - 0x2A, /* [ 1089] OBJ_member_body */ - 0x2A,0x86,0x48, /* [ 1090] OBJ_ISO_US */ - 0x2A,0x86,0x48,0xCE,0x38, /* [ 1093] OBJ_X9_57 */ - 0x2A,0x86,0x48,0xCE,0x38,0x04, /* [ 1098] OBJ_X9cm */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01, /* [ 1104] OBJ_pkcs1 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05, /* [ 1112] OBJ_pkcs5 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10, /* [ 1120] OBJ_SMIME */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00, /* [ 1129] OBJ_id_smime_mod */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01, /* [ 1139] OBJ_id_smime_ct */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02, /* [ 1149] OBJ_id_smime_aa */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03, /* [ 1159] OBJ_id_smime_alg */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x04, /* [ 1169] OBJ_id_smime_cd */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x05, /* [ 1179] OBJ_id_smime_spq */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06, /* [ 1189] OBJ_id_smime_cti */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x01, /* [ 1199] OBJ_id_smime_mod_cms */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x02, /* [ 1210] OBJ_id_smime_mod_ess */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x03, /* [ 1221] OBJ_id_smime_mod_oid */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x04, /* [ 1232] OBJ_id_smime_mod_msg_v3 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x05, /* [ 1243] OBJ_id_smime_mod_ets_eSignature_88 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x06, /* [ 1254] OBJ_id_smime_mod_ets_eSignature_97 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x07, /* [ 1265] OBJ_id_smime_mod_ets_eSigPolicy_88 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x08, /* [ 1276] OBJ_id_smime_mod_ets_eSigPolicy_97 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x01, /* [ 1287] OBJ_id_smime_ct_receipt */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x02, /* [ 1298] OBJ_id_smime_ct_authData */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x03, /* [ 1309] OBJ_id_smime_ct_publishCert */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x04, /* [ 1320] OBJ_id_smime_ct_TSTInfo */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x05, /* [ 1331] OBJ_id_smime_ct_TDTInfo */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x06, /* [ 1342] OBJ_id_smime_ct_contentInfo */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x07, /* [ 1353] OBJ_id_smime_ct_DVCSRequestData */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x08, /* [ 1364] OBJ_id_smime_ct_DVCSResponseData */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x01, /* [ 1375] OBJ_id_smime_aa_receiptRequest */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x02, /* [ 1386] OBJ_id_smime_aa_securityLabel */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x03, /* [ 1397] OBJ_id_smime_aa_mlExpandHistory */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x04, /* [ 1408] OBJ_id_smime_aa_contentHint */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x05, /* [ 1419] OBJ_id_smime_aa_msgSigDigest */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x06, /* [ 1430] OBJ_id_smime_aa_encapContentType */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x07, /* [ 1441] OBJ_id_smime_aa_contentIdentifier */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x08, /* [ 1452] OBJ_id_smime_aa_macValue */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x09, /* [ 1463] OBJ_id_smime_aa_equivalentLabels */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0A, /* [ 1474] OBJ_id_smime_aa_contentReference */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0B, /* [ 1485] OBJ_id_smime_aa_encrypKeyPref */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0C, /* [ 1496] OBJ_id_smime_aa_signingCertificate */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0D, /* [ 1507] OBJ_id_smime_aa_smimeEncryptCerts */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0E, /* [ 1518] OBJ_id_smime_aa_timeStampToken */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0F, /* [ 1529] OBJ_id_smime_aa_ets_sigPolicyId */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x10, /* [ 1540] OBJ_id_smime_aa_ets_commitmentType */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x11, /* [ 1551] OBJ_id_smime_aa_ets_signerLocation */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x12, /* [ 1562] OBJ_id_smime_aa_ets_signerAttr */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x13, /* [ 1573] OBJ_id_smime_aa_ets_otherSigCert */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x14, /* [ 1584] OBJ_id_smime_aa_ets_contentTimestamp */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x15, /* [ 1595] OBJ_id_smime_aa_ets_CertificateRefs */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x16, /* [ 1606] OBJ_id_smime_aa_ets_RevocationRefs */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x17, /* [ 1617] OBJ_id_smime_aa_ets_certValues */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x18, /* [ 1628] OBJ_id_smime_aa_ets_revocationValues */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x19, /* [ 1639] OBJ_id_smime_aa_ets_escTimeStamp */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x1A, /* [ 1650] OBJ_id_smime_aa_ets_certCRLTimestamp */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x1B, /* [ 1661] OBJ_id_smime_aa_ets_archiveTimeStamp */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x1C, /* [ 1672] OBJ_id_smime_aa_signatureType */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x1D, /* [ 1683] OBJ_id_smime_aa_dvcs_dvc */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x01, /* [ 1694] OBJ_id_smime_alg_ESDHwith3DES */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x02, /* [ 1705] OBJ_id_smime_alg_ESDHwithRC2 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x03, /* [ 1716] OBJ_id_smime_alg_3DESwrap */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x04, /* [ 1727] OBJ_id_smime_alg_RC2wrap */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x05, /* [ 1738] OBJ_id_smime_alg_ESDH */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x06, /* [ 1749] OBJ_id_smime_alg_CMS3DESwrap */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x07, /* [ 1760] OBJ_id_smime_alg_CMSRC2wrap */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x04,0x01, /* [ 1771] OBJ_id_smime_cd_ldap */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x05,0x01, /* [ 1782] OBJ_id_smime_spq_ets_sqt_uri */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x05,0x02, /* [ 1793] OBJ_id_smime_spq_ets_sqt_unotice */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x01, /* [ 1804] OBJ_id_smime_cti_ets_proofOfOrigin */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x02, /* [ 1815] OBJ_id_smime_cti_ets_proofOfReceipt */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x03, /* [ 1826] OBJ_id_smime_cti_ets_proofOfDelivery */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x04, /* [ 1837] OBJ_id_smime_cti_ets_proofOfSender */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x05, /* [ 1848] OBJ_id_smime_cti_ets_proofOfApproval */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x06, /* [ 1859] OBJ_id_smime_cti_ets_proofOfCreation */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x04, /* [ 1870] OBJ_md4 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00, /* [ 1878] OBJ_id_pkix_mod */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x02, /* [ 1885] OBJ_id_qt */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04, /* [ 1892] OBJ_id_it */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05, /* [ 1899] OBJ_id_pkip */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x06, /* [ 1906] OBJ_id_alg */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07, /* [ 1913] OBJ_id_cmc */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x08, /* [ 1920] OBJ_id_on */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x09, /* [ 1927] OBJ_id_pda */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A, /* [ 1934] OBJ_id_aca */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0B, /* [ 1941] OBJ_id_qcs */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0C, /* [ 1948] OBJ_id_cct */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x01, /* [ 1955] OBJ_id_pkix1_explicit_88 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x02, /* [ 1963] OBJ_id_pkix1_implicit_88 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x03, /* [ 1971] OBJ_id_pkix1_explicit_93 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x04, /* [ 1979] OBJ_id_pkix1_implicit_93 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x05, /* [ 1987] OBJ_id_mod_crmf */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x06, /* [ 1995] OBJ_id_mod_cmc */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x07, /* [ 2003] OBJ_id_mod_kea_profile_88 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x08, /* [ 2011] OBJ_id_mod_kea_profile_93 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x09, /* [ 2019] OBJ_id_mod_cmp */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0A, /* [ 2027] OBJ_id_mod_qualified_cert_88 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0B, /* [ 2035] OBJ_id_mod_qualified_cert_93 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0C, /* [ 2043] OBJ_id_mod_attribute_cert */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0D, /* [ 2051] OBJ_id_mod_timestamp_protocol */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0E, /* [ 2059] OBJ_id_mod_ocsp */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0F, /* [ 2067] OBJ_id_mod_dvcs */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x10, /* [ 2075] OBJ_id_mod_cmp2000 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x02, /* [ 2083] OBJ_biometricInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x03, /* [ 2091] OBJ_qcStatements */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x04, /* [ 2099] OBJ_ac_auditEntity */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x05, /* [ 2107] OBJ_ac_targeting */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x06, /* [ 2115] OBJ_aaControls */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x07, /* [ 2123] OBJ_sbgp_ipAddrBlock */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x08, /* [ 2131] OBJ_sbgp_autonomousSysNum */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x09, /* [ 2139] OBJ_sbgp_routerIdentifier */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x02,0x03, /* [ 2147] OBJ_textNotice */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x05, /* [ 2155] OBJ_ipsecEndSystem */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x06, /* [ 2163] OBJ_ipsecTunnel */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x07, /* [ 2171] OBJ_ipsecUser */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x0A, /* [ 2179] OBJ_dvcs */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x01, /* [ 2187] OBJ_id_it_caProtEncCert */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x02, /* [ 2195] OBJ_id_it_signKeyPairTypes */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x03, /* [ 2203] OBJ_id_it_encKeyPairTypes */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x04, /* [ 2211] OBJ_id_it_preferredSymmAlg */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x05, /* [ 2219] OBJ_id_it_caKeyUpdateInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x06, /* [ 2227] OBJ_id_it_currentCRL */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x07, /* [ 2235] OBJ_id_it_unsupportedOIDs */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x08, /* [ 2243] OBJ_id_it_subscriptionRequest */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x09, /* [ 2251] OBJ_id_it_subscriptionResponse */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0A, /* [ 2259] OBJ_id_it_keyPairParamReq */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0B, /* [ 2267] OBJ_id_it_keyPairParamRep */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0C, /* [ 2275] OBJ_id_it_revPassphrase */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0D, /* [ 2283] OBJ_id_it_implicitConfirm */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0E, /* [ 2291] OBJ_id_it_confirmWaitTime */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0F, /* [ 2299] OBJ_id_it_origPKIMessage */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01, /* [ 2307] OBJ_id_regCtrl */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x02, /* [ 2315] OBJ_id_regInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x01, /* [ 2323] OBJ_id_regCtrl_regToken */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x02, /* [ 2332] OBJ_id_regCtrl_authenticator */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x03, /* [ 2341] OBJ_id_regCtrl_pkiPublicationInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x04, /* [ 2350] OBJ_id_regCtrl_pkiArchiveOptions */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x05, /* [ 2359] OBJ_id_regCtrl_oldCertID */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x06, /* [ 2368] OBJ_id_regCtrl_protocolEncrKey */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x02,0x01, /* [ 2377] OBJ_id_regInfo_utf8Pairs */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x02,0x02, /* [ 2386] OBJ_id_regInfo_certReq */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x06,0x01, /* [ 2395] OBJ_id_alg_des40 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x06,0x02, /* [ 2403] OBJ_id_alg_noSignature */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x06,0x03, /* [ 2411] OBJ_id_alg_dh_sig_hmac_sha1 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x06,0x04, /* [ 2419] OBJ_id_alg_dh_pop */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x01, /* [ 2427] OBJ_id_cmc_statusInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x02, /* [ 2435] OBJ_id_cmc_identification */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x03, /* [ 2443] OBJ_id_cmc_identityProof */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x04, /* [ 2451] OBJ_id_cmc_dataReturn */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x05, /* [ 2459] OBJ_id_cmc_transactionId */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x06, /* [ 2467] OBJ_id_cmc_senderNonce */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x07, /* [ 2475] OBJ_id_cmc_recipientNonce */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x08, /* [ 2483] OBJ_id_cmc_addExtensions */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x09, /* [ 2491] OBJ_id_cmc_encryptedPOP */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x0A, /* [ 2499] OBJ_id_cmc_decryptedPOP */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x0B, /* [ 2507] OBJ_id_cmc_lraPOPWitness */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x0F, /* [ 2515] OBJ_id_cmc_getCert */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x10, /* [ 2523] OBJ_id_cmc_getCRL */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x11, /* [ 2531] OBJ_id_cmc_revokeRequest */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x12, /* [ 2539] OBJ_id_cmc_regInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x13, /* [ 2547] OBJ_id_cmc_responseInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x15, /* [ 2555] OBJ_id_cmc_queryPending */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x16, /* [ 2563] OBJ_id_cmc_popLinkRandom */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x17, /* [ 2571] OBJ_id_cmc_popLinkWitness */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x18, /* [ 2579] OBJ_id_cmc_confirmCertAcceptance */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x01, /* [ 2587] OBJ_id_on_personalData */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x01, /* [ 2595] OBJ_id_pda_dateOfBirth */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x02, /* [ 2603] OBJ_id_pda_placeOfBirth */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x03, /* [ 2611] OBJ_id_pda_gender */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x04, /* [ 2619] OBJ_id_pda_countryOfCitizenship */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x05, /* [ 2627] OBJ_id_pda_countryOfResidence */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x01, /* [ 2635] OBJ_id_aca_authenticationInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x02, /* [ 2643] OBJ_id_aca_accessIdentity */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x03, /* [ 2651] OBJ_id_aca_chargingIdentity */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x04, /* [ 2659] OBJ_id_aca_group */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x05, /* [ 2667] OBJ_id_aca_role */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0B,0x01, /* [ 2675] OBJ_id_qcs_pkixQCSyntax_v1 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0C,0x01, /* [ 2683] OBJ_id_cct_crs */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0C,0x02, /* [ 2691] OBJ_id_cct_PKIData */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0C,0x03, /* [ 2699] OBJ_id_cct_PKIResponse */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x03, /* [ 2707] OBJ_ad_timeStamping */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x04, /* [ 2715] OBJ_ad_dvcs */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x01, /* [ 2723] OBJ_id_pkix_OCSP_basic */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x02, /* [ 2732] OBJ_id_pkix_OCSP_Nonce */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x03, /* [ 2741] OBJ_id_pkix_OCSP_CrlID */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x04, /* [ 2750] OBJ_id_pkix_OCSP_acceptableResponses */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x05, /* [ 2759] OBJ_id_pkix_OCSP_noCheck */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x06, /* [ 2768] OBJ_id_pkix_OCSP_archiveCutoff */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x07, /* [ 2777] OBJ_id_pkix_OCSP_serviceLocator */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x08, /* [ 2786] OBJ_id_pkix_OCSP_extendedStatus */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x09, /* [ 2795] OBJ_id_pkix_OCSP_valid */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x0A, /* [ 2804] OBJ_id_pkix_OCSP_path */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x0B, /* [ 2813] OBJ_id_pkix_OCSP_trustRoot */ - 0x2B,0x0E,0x03,0x02, /* [ 2822] OBJ_algorithm */ - 0x2B,0x0E,0x03,0x02,0x0B, /* [ 2826] OBJ_rsaSignature */ - 0x55,0x08, /* [ 2831] OBJ_X500algorithms */ - 0x2B, /* [ 2833] OBJ_org */ - 0x2B,0x06, /* [ 2834] OBJ_dod */ - 0x2B,0x06,0x01, /* [ 2836] OBJ_iana */ - 0x2B,0x06,0x01,0x01, /* [ 2839] OBJ_Directory */ - 0x2B,0x06,0x01,0x02, /* [ 2843] OBJ_Management */ - 0x2B,0x06,0x01,0x03, /* [ 2847] OBJ_Experimental */ - 0x2B,0x06,0x01,0x04, /* [ 2851] OBJ_Private */ - 0x2B,0x06,0x01,0x05, /* [ 2855] OBJ_Security */ - 0x2B,0x06,0x01,0x06, /* [ 2859] OBJ_SNMPv2 */ - 0x2B,0x06,0x01,0x07, /* [ 2863] OBJ_Mail */ - 0x2B,0x06,0x01,0x04,0x01, /* [ 2867] OBJ_Enterprises */ - 0x2B,0x06,0x01,0x04,0x01,0x8B,0x3A,0x82,0x58, /* [ 2872] OBJ_dcObject */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x19, /* [ 2881] OBJ_domainComponent */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x0D, /* [ 2891] OBJ_Domain */ - 0x55,0x01,0x05, /* [ 2901] OBJ_selected_attribute_types */ - 0x55,0x01,0x05,0x37, /* [ 2904] OBJ_clearance */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x03, /* [ 2908] OBJ_md4WithRSAEncryption */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x0A, /* [ 2917] OBJ_ac_proxying */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x0B, /* [ 2925] OBJ_sinfo_access */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x06, /* [ 2933] OBJ_id_aca_encAttrs */ - 0x55,0x04,0x48, /* [ 2941] OBJ_role */ - 0x55,0x1D,0x24, /* [ 2944] OBJ_policy_constraints */ - 0x55,0x1D,0x37, /* [ 2947] OBJ_target_information */ - 0x55,0x1D,0x38, /* [ 2950] OBJ_no_rev_avail */ - 0x2A,0x86,0x48,0xCE,0x3D, /* [ 2953] OBJ_ansi_X9_62 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x01, /* [ 2958] OBJ_X9_62_prime_field */ - 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02, /* [ 2965] OBJ_X9_62_characteristic_two_field */ - 0x2A,0x86,0x48,0xCE,0x3D,0x02,0x01, /* [ 2972] OBJ_X9_62_id_ecPublicKey */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x01, /* [ 2979] OBJ_X9_62_prime192v1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x02, /* [ 2987] OBJ_X9_62_prime192v2 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x03, /* [ 2995] OBJ_X9_62_prime192v3 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x04, /* [ 3003] OBJ_X9_62_prime239v1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x05, /* [ 3011] OBJ_X9_62_prime239v2 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x06, /* [ 3019] OBJ_X9_62_prime239v3 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x07, /* [ 3027] OBJ_X9_62_prime256v1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x01, /* [ 3035] OBJ_ecdsa_with_SHA1 */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x11,0x01, /* [ 3042] OBJ_ms_csp_name */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x01, /* [ 3051] OBJ_aes_128_ecb */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x02, /* [ 3060] OBJ_aes_128_cbc */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x03, /* [ 3069] OBJ_aes_128_ofb128 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x04, /* [ 3078] OBJ_aes_128_cfb128 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x15, /* [ 3087] OBJ_aes_192_ecb */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x16, /* [ 3096] OBJ_aes_192_cbc */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x17, /* [ 3105] OBJ_aes_192_ofb128 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x18, /* [ 3114] OBJ_aes_192_cfb128 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x29, /* [ 3123] OBJ_aes_256_ecb */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2A, /* [ 3132] OBJ_aes_256_cbc */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2B, /* [ 3141] OBJ_aes_256_ofb128 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2C, /* [ 3150] OBJ_aes_256_cfb128 */ - 0x55,0x1D,0x17, /* [ 3159] OBJ_hold_instruction_code */ - 0x2A,0x86,0x48,0xCE,0x38,0x02,0x01, /* [ 3162] OBJ_hold_instruction_none */ - 0x2A,0x86,0x48,0xCE,0x38,0x02,0x02, /* [ 3169] OBJ_hold_instruction_call_issuer */ - 0x2A,0x86,0x48,0xCE,0x38,0x02,0x03, /* [ 3176] OBJ_hold_instruction_reject */ - 0x09, /* [ 3183] OBJ_data */ - 0x09,0x92,0x26, /* [ 3184] OBJ_pss */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C, /* [ 3187] OBJ_ucl */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64, /* [ 3194] OBJ_pilot */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01, /* [ 3202] OBJ_pilotAttributeType */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x03, /* [ 3211] OBJ_pilotAttributeSyntax */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04, /* [ 3220] OBJ_pilotObjectClass */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x0A, /* [ 3229] OBJ_pilotGroups */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x03,0x04, /* [ 3238] OBJ_iA5StringSyntax */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x03,0x05, /* [ 3248] OBJ_caseIgnoreIA5StringSyntax */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x03, /* [ 3258] OBJ_pilotObject */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x04, /* [ 3268] OBJ_pilotPerson */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x05, /* [ 3278] OBJ_account */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x06, /* [ 3288] OBJ_document */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x07, /* [ 3298] OBJ_room */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x09, /* [ 3308] OBJ_documentSeries */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x0E, /* [ 3318] OBJ_rFC822localPart */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x0F, /* [ 3328] OBJ_dNSDomain */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x11, /* [ 3338] OBJ_domainRelatedObject */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x12, /* [ 3348] OBJ_friendlyCountry */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x13, /* [ 3358] OBJ_simpleSecurityObject */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x14, /* [ 3368] OBJ_pilotOrganization */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x15, /* [ 3378] OBJ_pilotDSA */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x16, /* [ 3388] OBJ_qualityLabelledData */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x01, /* [ 3398] OBJ_userId */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x02, /* [ 3408] OBJ_textEncodedORAddress */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x03, /* [ 3418] OBJ_rfc822Mailbox */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x04, /* [ 3428] OBJ_info */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x05, /* [ 3438] OBJ_favouriteDrink */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x06, /* [ 3448] OBJ_roomNumber */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x07, /* [ 3458] OBJ_photo */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x08, /* [ 3468] OBJ_userClass */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x09, /* [ 3478] OBJ_host */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0A, /* [ 3488] OBJ_manager */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0B, /* [ 3498] OBJ_documentIdentifier */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0C, /* [ 3508] OBJ_documentTitle */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0D, /* [ 3518] OBJ_documentVersion */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0E, /* [ 3528] OBJ_documentAuthor */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0F, /* [ 3538] OBJ_documentLocation */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x14, /* [ 3548] OBJ_homeTelephoneNumber */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x15, /* [ 3558] OBJ_secretary */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x16, /* [ 3568] OBJ_otherMailbox */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x17, /* [ 3578] OBJ_lastModifiedTime */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x18, /* [ 3588] OBJ_lastModifiedBy */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1A, /* [ 3598] OBJ_aRecord */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1B, /* [ 3608] OBJ_pilotAttributeType27 */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1C, /* [ 3618] OBJ_mXRecord */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1D, /* [ 3628] OBJ_nSRecord */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1E, /* [ 3638] OBJ_sOARecord */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1F, /* [ 3648] OBJ_cNAMERecord */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x25, /* [ 3658] OBJ_associatedDomain */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x26, /* [ 3668] OBJ_associatedName */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x27, /* [ 3678] OBJ_homePostalAddress */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x28, /* [ 3688] OBJ_personalTitle */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x29, /* [ 3698] OBJ_mobileTelephoneNumber */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2A, /* [ 3708] OBJ_pagerTelephoneNumber */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2B, /* [ 3718] OBJ_friendlyCountryName */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2D, /* [ 3728] OBJ_organizationalStatus */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2E, /* [ 3738] OBJ_janetMailbox */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2F, /* [ 3748] OBJ_mailPreferenceOption */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x30, /* [ 3758] OBJ_buildingName */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x31, /* [ 3768] OBJ_dSAQuality */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x32, /* [ 3778] OBJ_singleLevelQuality */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x33, /* [ 3788] OBJ_subtreeMinimumQuality */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x34, /* [ 3798] OBJ_subtreeMaximumQuality */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x35, /* [ 3808] OBJ_personalSignature */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x36, /* [ 3818] OBJ_dITRedirect */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x37, /* [ 3828] OBJ_audio */ - 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x38, /* [ 3838] OBJ_documentPublisher */ - 0x55,0x04,0x2D, /* [ 3848] OBJ_x500UniqueIdentifier */ - 0x2B,0x06,0x01,0x07,0x01, /* [ 3851] OBJ_mime_mhs */ - 0x2B,0x06,0x01,0x07,0x01,0x01, /* [ 3856] OBJ_mime_mhs_headings */ - 0x2B,0x06,0x01,0x07,0x01,0x02, /* [ 3862] OBJ_mime_mhs_bodies */ - 0x2B,0x06,0x01,0x07,0x01,0x01,0x01, /* [ 3868] OBJ_id_hex_partial_message */ - 0x2B,0x06,0x01,0x07,0x01,0x01,0x02, /* [ 3875] OBJ_id_hex_multipart_message */ - 0x55,0x04,0x2C, /* [ 3882] OBJ_generationQualifier */ - 0x55,0x04,0x41, /* [ 3885] OBJ_pseudonym */ - 0x67,0x2A, /* [ 3888] OBJ_id_set */ - 0x67,0x2A,0x00, /* [ 3890] OBJ_set_ctype */ - 0x67,0x2A,0x01, /* [ 3893] OBJ_set_msgExt */ - 0x67,0x2A,0x03, /* [ 3896] OBJ_set_attr */ - 0x67,0x2A,0x05, /* [ 3899] OBJ_set_policy */ - 0x67,0x2A,0x07, /* [ 3902] OBJ_set_certExt */ - 0x67,0x2A,0x08, /* [ 3905] OBJ_set_brand */ - 0x67,0x2A,0x00,0x00, /* [ 3908] OBJ_setct_PANData */ - 0x67,0x2A,0x00,0x01, /* [ 3912] OBJ_setct_PANToken */ - 0x67,0x2A,0x00,0x02, /* [ 3916] OBJ_setct_PANOnly */ - 0x67,0x2A,0x00,0x03, /* [ 3920] OBJ_setct_OIData */ - 0x67,0x2A,0x00,0x04, /* [ 3924] OBJ_setct_PI */ - 0x67,0x2A,0x00,0x05, /* [ 3928] OBJ_setct_PIData */ - 0x67,0x2A,0x00,0x06, /* [ 3932] OBJ_setct_PIDataUnsigned */ - 0x67,0x2A,0x00,0x07, /* [ 3936] OBJ_setct_HODInput */ - 0x67,0x2A,0x00,0x08, /* [ 3940] OBJ_setct_AuthResBaggage */ - 0x67,0x2A,0x00,0x09, /* [ 3944] OBJ_setct_AuthRevReqBaggage */ - 0x67,0x2A,0x00,0x0A, /* [ 3948] OBJ_setct_AuthRevResBaggage */ - 0x67,0x2A,0x00,0x0B, /* [ 3952] OBJ_setct_CapTokenSeq */ - 0x67,0x2A,0x00,0x0C, /* [ 3956] OBJ_setct_PInitResData */ - 0x67,0x2A,0x00,0x0D, /* [ 3960] OBJ_setct_PI_TBS */ - 0x67,0x2A,0x00,0x0E, /* [ 3964] OBJ_setct_PResData */ - 0x67,0x2A,0x00,0x10, /* [ 3968] OBJ_setct_AuthReqTBS */ - 0x67,0x2A,0x00,0x11, /* [ 3972] OBJ_setct_AuthResTBS */ - 0x67,0x2A,0x00,0x12, /* [ 3976] OBJ_setct_AuthResTBSX */ - 0x67,0x2A,0x00,0x13, /* [ 3980] OBJ_setct_AuthTokenTBS */ - 0x67,0x2A,0x00,0x14, /* [ 3984] OBJ_setct_CapTokenData */ - 0x67,0x2A,0x00,0x15, /* [ 3988] OBJ_setct_CapTokenTBS */ - 0x67,0x2A,0x00,0x16, /* [ 3992] OBJ_setct_AcqCardCodeMsg */ - 0x67,0x2A,0x00,0x17, /* [ 3996] OBJ_setct_AuthRevReqTBS */ - 0x67,0x2A,0x00,0x18, /* [ 4000] OBJ_setct_AuthRevResData */ - 0x67,0x2A,0x00,0x19, /* [ 4004] OBJ_setct_AuthRevResTBS */ - 0x67,0x2A,0x00,0x1A, /* [ 4008] OBJ_setct_CapReqTBS */ - 0x67,0x2A,0x00,0x1B, /* [ 4012] OBJ_setct_CapReqTBSX */ - 0x67,0x2A,0x00,0x1C, /* [ 4016] OBJ_setct_CapResData */ - 0x67,0x2A,0x00,0x1D, /* [ 4020] OBJ_setct_CapRevReqTBS */ - 0x67,0x2A,0x00,0x1E, /* [ 4024] OBJ_setct_CapRevReqTBSX */ - 0x67,0x2A,0x00,0x1F, /* [ 4028] OBJ_setct_CapRevResData */ - 0x67,0x2A,0x00,0x20, /* [ 4032] OBJ_setct_CredReqTBS */ - 0x67,0x2A,0x00,0x21, /* [ 4036] OBJ_setct_CredReqTBSX */ - 0x67,0x2A,0x00,0x22, /* [ 4040] OBJ_setct_CredResData */ - 0x67,0x2A,0x00,0x23, /* [ 4044] OBJ_setct_CredRevReqTBS */ - 0x67,0x2A,0x00,0x24, /* [ 4048] OBJ_setct_CredRevReqTBSX */ - 0x67,0x2A,0x00,0x25, /* [ 4052] OBJ_setct_CredRevResData */ - 0x67,0x2A,0x00,0x26, /* [ 4056] OBJ_setct_PCertReqData */ - 0x67,0x2A,0x00,0x27, /* [ 4060] OBJ_setct_PCertResTBS */ - 0x67,0x2A,0x00,0x28, /* [ 4064] OBJ_setct_BatchAdminReqData */ - 0x67,0x2A,0x00,0x29, /* [ 4068] OBJ_setct_BatchAdminResData */ - 0x67,0x2A,0x00,0x2A, /* [ 4072] OBJ_setct_CardCInitResTBS */ - 0x67,0x2A,0x00,0x2B, /* [ 4076] OBJ_setct_MeAqCInitResTBS */ - 0x67,0x2A,0x00,0x2C, /* [ 4080] OBJ_setct_RegFormResTBS */ - 0x67,0x2A,0x00,0x2D, /* [ 4084] OBJ_setct_CertReqData */ - 0x67,0x2A,0x00,0x2E, /* [ 4088] OBJ_setct_CertReqTBS */ - 0x67,0x2A,0x00,0x2F, /* [ 4092] OBJ_setct_CertResData */ - 0x67,0x2A,0x00,0x30, /* [ 4096] OBJ_setct_CertInqReqTBS */ - 0x67,0x2A,0x00,0x31, /* [ 4100] OBJ_setct_ErrorTBS */ - 0x67,0x2A,0x00,0x32, /* [ 4104] OBJ_setct_PIDualSignedTBE */ - 0x67,0x2A,0x00,0x33, /* [ 4108] OBJ_setct_PIUnsignedTBE */ - 0x67,0x2A,0x00,0x34, /* [ 4112] OBJ_setct_AuthReqTBE */ - 0x67,0x2A,0x00,0x35, /* [ 4116] OBJ_setct_AuthResTBE */ - 0x67,0x2A,0x00,0x36, /* [ 4120] OBJ_setct_AuthResTBEX */ - 0x67,0x2A,0x00,0x37, /* [ 4124] OBJ_setct_AuthTokenTBE */ - 0x67,0x2A,0x00,0x38, /* [ 4128] OBJ_setct_CapTokenTBE */ - 0x67,0x2A,0x00,0x39, /* [ 4132] OBJ_setct_CapTokenTBEX */ - 0x67,0x2A,0x00,0x3A, /* [ 4136] OBJ_setct_AcqCardCodeMsgTBE */ - 0x67,0x2A,0x00,0x3B, /* [ 4140] OBJ_setct_AuthRevReqTBE */ - 0x67,0x2A,0x00,0x3C, /* [ 4144] OBJ_setct_AuthRevResTBE */ - 0x67,0x2A,0x00,0x3D, /* [ 4148] OBJ_setct_AuthRevResTBEB */ - 0x67,0x2A,0x00,0x3E, /* [ 4152] OBJ_setct_CapReqTBE */ - 0x67,0x2A,0x00,0x3F, /* [ 4156] OBJ_setct_CapReqTBEX */ - 0x67,0x2A,0x00,0x40, /* [ 4160] OBJ_setct_CapResTBE */ - 0x67,0x2A,0x00,0x41, /* [ 4164] OBJ_setct_CapRevReqTBE */ - 0x67,0x2A,0x00,0x42, /* [ 4168] OBJ_setct_CapRevReqTBEX */ - 0x67,0x2A,0x00,0x43, /* [ 4172] OBJ_setct_CapRevResTBE */ - 0x67,0x2A,0x00,0x44, /* [ 4176] OBJ_setct_CredReqTBE */ - 0x67,0x2A,0x00,0x45, /* [ 4180] OBJ_setct_CredReqTBEX */ - 0x67,0x2A,0x00,0x46, /* [ 4184] OBJ_setct_CredResTBE */ - 0x67,0x2A,0x00,0x47, /* [ 4188] OBJ_setct_CredRevReqTBE */ - 0x67,0x2A,0x00,0x48, /* [ 4192] OBJ_setct_CredRevReqTBEX */ - 0x67,0x2A,0x00,0x49, /* [ 4196] OBJ_setct_CredRevResTBE */ - 0x67,0x2A,0x00,0x4A, /* [ 4200] OBJ_setct_BatchAdminReqTBE */ - 0x67,0x2A,0x00,0x4B, /* [ 4204] OBJ_setct_BatchAdminResTBE */ - 0x67,0x2A,0x00,0x4C, /* [ 4208] OBJ_setct_RegFormReqTBE */ - 0x67,0x2A,0x00,0x4D, /* [ 4212] OBJ_setct_CertReqTBE */ - 0x67,0x2A,0x00,0x4E, /* [ 4216] OBJ_setct_CertReqTBEX */ - 0x67,0x2A,0x00,0x4F, /* [ 4220] OBJ_setct_CertResTBE */ - 0x67,0x2A,0x00,0x50, /* [ 4224] OBJ_setct_CRLNotificationTBS */ - 0x67,0x2A,0x00,0x51, /* [ 4228] OBJ_setct_CRLNotificationResTBS */ - 0x67,0x2A,0x00,0x52, /* [ 4232] OBJ_setct_BCIDistributionTBS */ - 0x67,0x2A,0x01,0x01, /* [ 4236] OBJ_setext_genCrypt */ - 0x67,0x2A,0x01,0x03, /* [ 4240] OBJ_setext_miAuth */ - 0x67,0x2A,0x01,0x04, /* [ 4244] OBJ_setext_pinSecure */ - 0x67,0x2A,0x01,0x05, /* [ 4248] OBJ_setext_pinAny */ - 0x67,0x2A,0x01,0x07, /* [ 4252] OBJ_setext_track2 */ - 0x67,0x2A,0x01,0x08, /* [ 4256] OBJ_setext_cv */ - 0x67,0x2A,0x05,0x00, /* [ 4260] OBJ_set_policy_root */ - 0x67,0x2A,0x07,0x00, /* [ 4264] OBJ_setCext_hashedRoot */ - 0x67,0x2A,0x07,0x01, /* [ 4268] OBJ_setCext_certType */ - 0x67,0x2A,0x07,0x02, /* [ 4272] OBJ_setCext_merchData */ - 0x67,0x2A,0x07,0x03, /* [ 4276] OBJ_setCext_cCertRequired */ - 0x67,0x2A,0x07,0x04, /* [ 4280] OBJ_setCext_tunneling */ - 0x67,0x2A,0x07,0x05, /* [ 4284] OBJ_setCext_setExt */ - 0x67,0x2A,0x07,0x06, /* [ 4288] OBJ_setCext_setQualf */ - 0x67,0x2A,0x07,0x07, /* [ 4292] OBJ_setCext_PGWYcapabilities */ - 0x67,0x2A,0x07,0x08, /* [ 4296] OBJ_setCext_TokenIdentifier */ - 0x67,0x2A,0x07,0x09, /* [ 4300] OBJ_setCext_Track2Data */ - 0x67,0x2A,0x07,0x0A, /* [ 4304] OBJ_setCext_TokenType */ - 0x67,0x2A,0x07,0x0B, /* [ 4308] OBJ_setCext_IssuerCapabilities */ - 0x67,0x2A,0x03,0x00, /* [ 4312] OBJ_setAttr_Cert */ - 0x67,0x2A,0x03,0x01, /* [ 4316] OBJ_setAttr_PGWYcap */ - 0x67,0x2A,0x03,0x02, /* [ 4320] OBJ_setAttr_TokenType */ - 0x67,0x2A,0x03,0x03, /* [ 4324] OBJ_setAttr_IssCap */ - 0x67,0x2A,0x03,0x00,0x00, /* [ 4328] OBJ_set_rootKeyThumb */ - 0x67,0x2A,0x03,0x00,0x01, /* [ 4333] OBJ_set_addPolicy */ - 0x67,0x2A,0x03,0x02,0x01, /* [ 4338] OBJ_setAttr_Token_EMV */ - 0x67,0x2A,0x03,0x02,0x02, /* [ 4343] OBJ_setAttr_Token_B0Prime */ - 0x67,0x2A,0x03,0x03,0x03, /* [ 4348] OBJ_setAttr_IssCap_CVM */ - 0x67,0x2A,0x03,0x03,0x04, /* [ 4353] OBJ_setAttr_IssCap_T2 */ - 0x67,0x2A,0x03,0x03,0x05, /* [ 4358] OBJ_setAttr_IssCap_Sig */ - 0x67,0x2A,0x03,0x03,0x03,0x01, /* [ 4363] OBJ_setAttr_GenCryptgrm */ - 0x67,0x2A,0x03,0x03,0x04,0x01, /* [ 4369] OBJ_setAttr_T2Enc */ - 0x67,0x2A,0x03,0x03,0x04,0x02, /* [ 4375] OBJ_setAttr_T2cleartxt */ - 0x67,0x2A,0x03,0x03,0x05,0x01, /* [ 4381] OBJ_setAttr_TokICCsig */ - 0x67,0x2A,0x03,0x03,0x05,0x02, /* [ 4387] OBJ_setAttr_SecDevSig */ - 0x67,0x2A,0x08,0x01, /* [ 4393] OBJ_set_brand_IATA_ATA */ - 0x67,0x2A,0x08,0x1E, /* [ 4397] OBJ_set_brand_Diners */ - 0x67,0x2A,0x08,0x22, /* [ 4401] OBJ_set_brand_AmericanExpress */ - 0x67,0x2A,0x08,0x23, /* [ 4405] OBJ_set_brand_JCB */ - 0x67,0x2A,0x08,0x04, /* [ 4409] OBJ_set_brand_Visa */ - 0x67,0x2A,0x08,0x05, /* [ 4413] OBJ_set_brand_MasterCard */ - 0x67,0x2A,0x08,0xAE,0x7B, /* [ 4417] OBJ_set_brand_Novus */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x0A, /* [ 4422] OBJ_des_cdmf */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x06, /* [ 4430] OBJ_rsaOAEPEncryptionSET */ - 0x67, /* [ 4439] OBJ_international_organizations */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x14,0x02,0x02, /* [ 4440] OBJ_ms_smartcard_login */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x14,0x02,0x03, /* [ 4450] OBJ_ms_upn */ - 0x55,0x04,0x09, /* [ 4460] OBJ_streetAddress */ - 0x55,0x04,0x11, /* [ 4463] OBJ_postalCode */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x15, /* [ 4466] OBJ_id_ppl */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x0E, /* [ 4473] OBJ_proxyCertInfo */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x15,0x00, /* [ 4481] OBJ_id_ppl_anyLanguage */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x15,0x01, /* [ 4489] OBJ_id_ppl_inheritAll */ - 0x55,0x1D,0x1E, /* [ 4497] OBJ_name_constraints */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x15,0x02, /* [ 4500] OBJ_Independent */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0B, /* [ 4508] OBJ_sha256WithRSAEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0C, /* [ 4517] OBJ_sha384WithRSAEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0D, /* [ 4526] OBJ_sha512WithRSAEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0E, /* [ 4535] OBJ_sha224WithRSAEncryption */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x01, /* [ 4544] OBJ_sha256 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x02, /* [ 4553] OBJ_sha384 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x03, /* [ 4562] OBJ_sha512 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x04, /* [ 4571] OBJ_sha224 */ - 0x2B, /* [ 4580] OBJ_identified_organization */ - 0x2B,0x81,0x04, /* [ 4581] OBJ_certicom_arc */ - 0x67,0x2B, /* [ 4584] OBJ_wap */ - 0x67,0x2B,0x01, /* [ 4586] OBJ_wap_wsg */ - 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02,0x03, /* [ 4589] OBJ_X9_62_id_characteristic_two_basis */ - 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02,0x03,0x01, /* [ 4597] OBJ_X9_62_onBasis */ - 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02,0x03,0x02, /* [ 4606] OBJ_X9_62_tpBasis */ - 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02,0x03,0x03, /* [ 4615] OBJ_X9_62_ppBasis */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x01, /* [ 4624] OBJ_X9_62_c2pnb163v1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x02, /* [ 4632] OBJ_X9_62_c2pnb163v2 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x03, /* [ 4640] OBJ_X9_62_c2pnb163v3 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x04, /* [ 4648] OBJ_X9_62_c2pnb176v1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x05, /* [ 4656] OBJ_X9_62_c2tnb191v1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x06, /* [ 4664] OBJ_X9_62_c2tnb191v2 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x07, /* [ 4672] OBJ_X9_62_c2tnb191v3 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x08, /* [ 4680] OBJ_X9_62_c2onb191v4 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x09, /* [ 4688] OBJ_X9_62_c2onb191v5 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0A, /* [ 4696] OBJ_X9_62_c2pnb208w1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0B, /* [ 4704] OBJ_X9_62_c2tnb239v1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0C, /* [ 4712] OBJ_X9_62_c2tnb239v2 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0D, /* [ 4720] OBJ_X9_62_c2tnb239v3 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0E, /* [ 4728] OBJ_X9_62_c2onb239v4 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0F, /* [ 4736] OBJ_X9_62_c2onb239v5 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x10, /* [ 4744] OBJ_X9_62_c2pnb272w1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x11, /* [ 4752] OBJ_X9_62_c2pnb304w1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x12, /* [ 4760] OBJ_X9_62_c2tnb359v1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x13, /* [ 4768] OBJ_X9_62_c2pnb368w1 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x14, /* [ 4776] OBJ_X9_62_c2tnb431r1 */ - 0x2B,0x81,0x04,0x00,0x06, /* [ 4784] OBJ_secp112r1 */ - 0x2B,0x81,0x04,0x00,0x07, /* [ 4789] OBJ_secp112r2 */ - 0x2B,0x81,0x04,0x00,0x1C, /* [ 4794] OBJ_secp128r1 */ - 0x2B,0x81,0x04,0x00,0x1D, /* [ 4799] OBJ_secp128r2 */ - 0x2B,0x81,0x04,0x00,0x09, /* [ 4804] OBJ_secp160k1 */ - 0x2B,0x81,0x04,0x00,0x08, /* [ 4809] OBJ_secp160r1 */ - 0x2B,0x81,0x04,0x00,0x1E, /* [ 4814] OBJ_secp160r2 */ - 0x2B,0x81,0x04,0x00,0x1F, /* [ 4819] OBJ_secp192k1 */ - 0x2B,0x81,0x04,0x00,0x20, /* [ 4824] OBJ_secp224k1 */ - 0x2B,0x81,0x04,0x00,0x21, /* [ 4829] OBJ_secp224r1 */ - 0x2B,0x81,0x04,0x00,0x0A, /* [ 4834] OBJ_secp256k1 */ - 0x2B,0x81,0x04,0x00,0x22, /* [ 4839] OBJ_secp384r1 */ - 0x2B,0x81,0x04,0x00,0x23, /* [ 4844] OBJ_secp521r1 */ - 0x2B,0x81,0x04,0x00,0x04, /* [ 4849] OBJ_sect113r1 */ - 0x2B,0x81,0x04,0x00,0x05, /* [ 4854] OBJ_sect113r2 */ - 0x2B,0x81,0x04,0x00,0x16, /* [ 4859] OBJ_sect131r1 */ - 0x2B,0x81,0x04,0x00,0x17, /* [ 4864] OBJ_sect131r2 */ - 0x2B,0x81,0x04,0x00,0x01, /* [ 4869] OBJ_sect163k1 */ - 0x2B,0x81,0x04,0x00,0x02, /* [ 4874] OBJ_sect163r1 */ - 0x2B,0x81,0x04,0x00,0x0F, /* [ 4879] OBJ_sect163r2 */ - 0x2B,0x81,0x04,0x00,0x18, /* [ 4884] OBJ_sect193r1 */ - 0x2B,0x81,0x04,0x00,0x19, /* [ 4889] OBJ_sect193r2 */ - 0x2B,0x81,0x04,0x00,0x1A, /* [ 4894] OBJ_sect233k1 */ - 0x2B,0x81,0x04,0x00,0x1B, /* [ 4899] OBJ_sect233r1 */ - 0x2B,0x81,0x04,0x00,0x03, /* [ 4904] OBJ_sect239k1 */ - 0x2B,0x81,0x04,0x00,0x10, /* [ 4909] OBJ_sect283k1 */ - 0x2B,0x81,0x04,0x00,0x11, /* [ 4914] OBJ_sect283r1 */ - 0x2B,0x81,0x04,0x00,0x24, /* [ 4919] OBJ_sect409k1 */ - 0x2B,0x81,0x04,0x00,0x25, /* [ 4924] OBJ_sect409r1 */ - 0x2B,0x81,0x04,0x00,0x26, /* [ 4929] OBJ_sect571k1 */ - 0x2B,0x81,0x04,0x00,0x27, /* [ 4934] OBJ_sect571r1 */ - 0x67,0x2B,0x01,0x04,0x01, /* [ 4939] OBJ_wap_wsg_idm_ecid_wtls1 */ - 0x67,0x2B,0x01,0x04,0x03, /* [ 4944] OBJ_wap_wsg_idm_ecid_wtls3 */ - 0x67,0x2B,0x01,0x04,0x04, /* [ 4949] OBJ_wap_wsg_idm_ecid_wtls4 */ - 0x67,0x2B,0x01,0x04,0x05, /* [ 4954] OBJ_wap_wsg_idm_ecid_wtls5 */ - 0x67,0x2B,0x01,0x04,0x06, /* [ 4959] OBJ_wap_wsg_idm_ecid_wtls6 */ - 0x67,0x2B,0x01,0x04,0x07, /* [ 4964] OBJ_wap_wsg_idm_ecid_wtls7 */ - 0x67,0x2B,0x01,0x04,0x08, /* [ 4969] OBJ_wap_wsg_idm_ecid_wtls8 */ - 0x67,0x2B,0x01,0x04,0x09, /* [ 4974] OBJ_wap_wsg_idm_ecid_wtls9 */ - 0x67,0x2B,0x01,0x04,0x0A, /* [ 4979] OBJ_wap_wsg_idm_ecid_wtls10 */ - 0x67,0x2B,0x01,0x04,0x0B, /* [ 4984] OBJ_wap_wsg_idm_ecid_wtls11 */ - 0x67,0x2B,0x01,0x04,0x0C, /* [ 4989] OBJ_wap_wsg_idm_ecid_wtls12 */ - 0x55,0x1D,0x20,0x00, /* [ 4994] OBJ_any_policy */ - 0x55,0x1D,0x21, /* [ 4998] OBJ_policy_mappings */ - 0x55,0x1D,0x36, /* [ 5001] OBJ_inhibit_any_policy */ - 0x2A,0x83,0x08,0x8C,0x9A,0x4B,0x3D,0x01,0x01,0x01,0x02, /* [ 5004] OBJ_camellia_128_cbc */ - 0x2A,0x83,0x08,0x8C,0x9A,0x4B,0x3D,0x01,0x01,0x01,0x03, /* [ 5015] OBJ_camellia_192_cbc */ - 0x2A,0x83,0x08,0x8C,0x9A,0x4B,0x3D,0x01,0x01,0x01,0x04, /* [ 5026] OBJ_camellia_256_cbc */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x01, /* [ 5037] OBJ_camellia_128_ecb */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x15, /* [ 5045] OBJ_camellia_192_ecb */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x29, /* [ 5053] OBJ_camellia_256_ecb */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x04, /* [ 5061] OBJ_camellia_128_cfb128 */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x18, /* [ 5069] OBJ_camellia_192_cfb128 */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x2C, /* [ 5077] OBJ_camellia_256_cfb128 */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x03, /* [ 5085] OBJ_camellia_128_ofb128 */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x17, /* [ 5093] OBJ_camellia_192_ofb128 */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x2B, /* [ 5101] OBJ_camellia_256_ofb128 */ - 0x55,0x1D,0x09, /* [ 5109] OBJ_subject_directory_attributes */ - 0x55,0x1D,0x1C, /* [ 5112] OBJ_issuing_distribution_point */ - 0x55,0x1D,0x1D, /* [ 5115] OBJ_certificate_issuer */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x44, /* [ 5118] OBJ_kisa */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x44,0x01,0x03, /* [ 5124] OBJ_seed_ecb */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x44,0x01,0x04, /* [ 5132] OBJ_seed_cbc */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x44,0x01,0x06, /* [ 5140] OBJ_seed_ofb128 */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x44,0x01,0x05, /* [ 5148] OBJ_seed_cfb128 */ - 0x2B,0x06,0x01,0x05,0x05,0x08,0x01,0x01, /* [ 5156] OBJ_hmac_md5 */ - 0x2B,0x06,0x01,0x05,0x05,0x08,0x01,0x02, /* [ 5164] OBJ_hmac_sha1 */ - 0x2A,0x86,0x48,0x86,0xF6,0x7D,0x07,0x42,0x0D, /* [ 5172] OBJ_id_PasswordBasedMAC */ - 0x2A,0x86,0x48,0x86,0xF6,0x7D,0x07,0x42,0x1E, /* [ 5181] OBJ_id_DHBasedMac */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x10, /* [ 5190] OBJ_id_it_suppLangTags */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x05, /* [ 5198] OBJ_caRepository */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x09, /* [ 5206] OBJ_id_smime_ct_compressedData */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x1B, /* [ 5217] OBJ_id_ct_asciiTextWithCRLF */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x05, /* [ 5228] OBJ_id_aes128_wrap */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x19, /* [ 5237] OBJ_id_aes192_wrap */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2D, /* [ 5246] OBJ_id_aes256_wrap */ - 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x02, /* [ 5255] OBJ_ecdsa_with_Recommended */ - 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03, /* [ 5262] OBJ_ecdsa_with_Specified */ - 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03,0x01, /* [ 5269] OBJ_ecdsa_with_SHA224 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03,0x02, /* [ 5277] OBJ_ecdsa_with_SHA256 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03,0x03, /* [ 5285] OBJ_ecdsa_with_SHA384 */ - 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03,0x04, /* [ 5293] OBJ_ecdsa_with_SHA512 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x06, /* [ 5301] OBJ_hmacWithMD5 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x08, /* [ 5309] OBJ_hmacWithSHA224 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x09, /* [ 5317] OBJ_hmacWithSHA256 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x0A, /* [ 5325] OBJ_hmacWithSHA384 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x0B, /* [ 5333] OBJ_hmacWithSHA512 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x01, /* [ 5341] OBJ_dsa_with_SHA224 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x02, /* [ 5350] OBJ_dsa_with_SHA256 */ - 0x28,0xCF,0x06,0x03,0x00,0x37, /* [ 5359] OBJ_whirlpool */ - 0x2A,0x85,0x03,0x02,0x02, /* [ 5365] OBJ_cryptopro */ - 0x2A,0x85,0x03,0x02,0x09, /* [ 5370] OBJ_cryptocom */ - 0x2A,0x85,0x03,0x02,0x02,0x03, /* [ 5375] OBJ_id_GostR3411_94_with_GostR3410_2001 */ - 0x2A,0x85,0x03,0x02,0x02,0x04, /* [ 5381] OBJ_id_GostR3411_94_with_GostR3410_94 */ - 0x2A,0x85,0x03,0x02,0x02,0x09, /* [ 5387] OBJ_id_GostR3411_94 */ - 0x2A,0x85,0x03,0x02,0x02,0x0A, /* [ 5393] OBJ_id_HMACGostR3411_94 */ - 0x2A,0x85,0x03,0x02,0x02,0x13, /* [ 5399] OBJ_id_GostR3410_2001 */ - 0x2A,0x85,0x03,0x02,0x02,0x14, /* [ 5405] OBJ_id_GostR3410_94 */ - 0x2A,0x85,0x03,0x02,0x02,0x15, /* [ 5411] OBJ_id_Gost28147_89 */ - 0x2A,0x85,0x03,0x02,0x02,0x16, /* [ 5417] OBJ_id_Gost28147_89_MAC */ - 0x2A,0x85,0x03,0x02,0x02,0x17, /* [ 5423] OBJ_id_GostR3411_94_prf */ - 0x2A,0x85,0x03,0x02,0x02,0x62, /* [ 5429] OBJ_id_GostR3410_2001DH */ - 0x2A,0x85,0x03,0x02,0x02,0x63, /* [ 5435] OBJ_id_GostR3410_94DH */ - 0x2A,0x85,0x03,0x02,0x02,0x0E,0x01, /* [ 5441] OBJ_id_Gost28147_89_CryptoPro_KeyMeshing */ - 0x2A,0x85,0x03,0x02,0x02,0x0E,0x00, /* [ 5448] OBJ_id_Gost28147_89_None_KeyMeshing */ - 0x2A,0x85,0x03,0x02,0x02,0x1E,0x00, /* [ 5455] OBJ_id_GostR3411_94_TestParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1E,0x01, /* [ 5462] OBJ_id_GostR3411_94_CryptoProParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1F,0x00, /* [ 5469] OBJ_id_Gost28147_89_TestParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1F,0x01, /* [ 5476] OBJ_id_Gost28147_89_CryptoPro_A_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1F,0x02, /* [ 5483] OBJ_id_Gost28147_89_CryptoPro_B_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1F,0x03, /* [ 5490] OBJ_id_Gost28147_89_CryptoPro_C_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1F,0x04, /* [ 5497] OBJ_id_Gost28147_89_CryptoPro_D_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1F,0x05, /* [ 5504] OBJ_id_Gost28147_89_CryptoPro_Oscar_1_1_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1F,0x06, /* [ 5511] OBJ_id_Gost28147_89_CryptoPro_Oscar_1_0_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x1F,0x07, /* [ 5518] OBJ_id_Gost28147_89_CryptoPro_RIC_1_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x20,0x00, /* [ 5525] OBJ_id_GostR3410_94_TestParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x20,0x02, /* [ 5532] OBJ_id_GostR3410_94_CryptoPro_A_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x20,0x03, /* [ 5539] OBJ_id_GostR3410_94_CryptoPro_B_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x20,0x04, /* [ 5546] OBJ_id_GostR3410_94_CryptoPro_C_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x20,0x05, /* [ 5553] OBJ_id_GostR3410_94_CryptoPro_D_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x21,0x01, /* [ 5560] OBJ_id_GostR3410_94_CryptoPro_XchA_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x21,0x02, /* [ 5567] OBJ_id_GostR3410_94_CryptoPro_XchB_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x21,0x03, /* [ 5574] OBJ_id_GostR3410_94_CryptoPro_XchC_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x23,0x00, /* [ 5581] OBJ_id_GostR3410_2001_TestParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x23,0x01, /* [ 5588] OBJ_id_GostR3410_2001_CryptoPro_A_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x23,0x02, /* [ 5595] OBJ_id_GostR3410_2001_CryptoPro_B_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x23,0x03, /* [ 5602] OBJ_id_GostR3410_2001_CryptoPro_C_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x24,0x00, /* [ 5609] OBJ_id_GostR3410_2001_CryptoPro_XchA_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x24,0x01, /* [ 5616] OBJ_id_GostR3410_2001_CryptoPro_XchB_ParamSet */ - 0x2A,0x85,0x03,0x02,0x02,0x14,0x01, /* [ 5623] OBJ_id_GostR3410_94_a */ - 0x2A,0x85,0x03,0x02,0x02,0x14,0x02, /* [ 5630] OBJ_id_GostR3410_94_aBis */ - 0x2A,0x85,0x03,0x02,0x02,0x14,0x03, /* [ 5637] OBJ_id_GostR3410_94_b */ - 0x2A,0x85,0x03,0x02,0x02,0x14,0x04, /* [ 5644] OBJ_id_GostR3410_94_bBis */ - 0x2A,0x85,0x03,0x02,0x09,0x01,0x06,0x01, /* [ 5651] OBJ_id_Gost28147_89_cc */ - 0x2A,0x85,0x03,0x02,0x09,0x01,0x05,0x03, /* [ 5659] OBJ_id_GostR3410_94_cc */ - 0x2A,0x85,0x03,0x02,0x09,0x01,0x05,0x04, /* [ 5667] OBJ_id_GostR3410_2001_cc */ - 0x2A,0x85,0x03,0x02,0x09,0x01,0x03,0x03, /* [ 5675] OBJ_id_GostR3411_94_with_GostR3410_94_cc */ - 0x2A,0x85,0x03,0x02,0x09,0x01,0x03,0x04, /* [ 5683] OBJ_id_GostR3411_94_with_GostR3410_2001_cc */ - 0x2A,0x85,0x03,0x02,0x09,0x01,0x08,0x01, /* [ 5691] OBJ_id_GostR3410_2001_ParamSet_cc */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x11,0x02, /* [ 5699] OBJ_LocalKeySet */ - 0x55,0x1D,0x2E, /* [ 5708] OBJ_freshest_crl */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x03, /* [ 5711] OBJ_id_on_permanentIdentifier */ - 0x55,0x04,0x0E, /* [ 5719] OBJ_searchGuide */ - 0x55,0x04,0x0F, /* [ 5722] OBJ_businessCategory */ - 0x55,0x04,0x10, /* [ 5725] OBJ_postalAddress */ - 0x55,0x04,0x12, /* [ 5728] OBJ_postOfficeBox */ - 0x55,0x04,0x13, /* [ 5731] OBJ_physicalDeliveryOfficeName */ - 0x55,0x04,0x14, /* [ 5734] OBJ_telephoneNumber */ - 0x55,0x04,0x15, /* [ 5737] OBJ_telexNumber */ - 0x55,0x04,0x16, /* [ 5740] OBJ_teletexTerminalIdentifier */ - 0x55,0x04,0x17, /* [ 5743] OBJ_facsimileTelephoneNumber */ - 0x55,0x04,0x18, /* [ 5746] OBJ_x121Address */ - 0x55,0x04,0x19, /* [ 5749] OBJ_internationaliSDNNumber */ - 0x55,0x04,0x1A, /* [ 5752] OBJ_registeredAddress */ - 0x55,0x04,0x1B, /* [ 5755] OBJ_destinationIndicator */ - 0x55,0x04,0x1C, /* [ 5758] OBJ_preferredDeliveryMethod */ - 0x55,0x04,0x1D, /* [ 5761] OBJ_presentationAddress */ - 0x55,0x04,0x1E, /* [ 5764] OBJ_supportedApplicationContext */ - 0x55,0x04,0x1F, /* [ 5767] OBJ_member */ - 0x55,0x04,0x20, /* [ 5770] OBJ_owner */ - 0x55,0x04,0x21, /* [ 5773] OBJ_roleOccupant */ - 0x55,0x04,0x22, /* [ 5776] OBJ_seeAlso */ - 0x55,0x04,0x23, /* [ 5779] OBJ_userPassword */ - 0x55,0x04,0x24, /* [ 5782] OBJ_userCertificate */ - 0x55,0x04,0x25, /* [ 5785] OBJ_cACertificate */ - 0x55,0x04,0x26, /* [ 5788] OBJ_authorityRevocationList */ - 0x55,0x04,0x27, /* [ 5791] OBJ_certificateRevocationList */ - 0x55,0x04,0x28, /* [ 5794] OBJ_crossCertificatePair */ - 0x55,0x04,0x2F, /* [ 5797] OBJ_enhancedSearchGuide */ - 0x55,0x04,0x30, /* [ 5800] OBJ_protocolInformation */ - 0x55,0x04,0x31, /* [ 5803] OBJ_distinguishedName */ - 0x55,0x04,0x32, /* [ 5806] OBJ_uniqueMember */ - 0x55,0x04,0x33, /* [ 5809] OBJ_houseIdentifier */ - 0x55,0x04,0x34, /* [ 5812] OBJ_supportedAlgorithms */ - 0x55,0x04,0x35, /* [ 5815] OBJ_deltaRevocationList */ - 0x55,0x04,0x36, /* [ 5818] OBJ_dmdName */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x09, /* [ 5821] OBJ_id_alg_PWRI_KEK */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x06, /* [ 5832] OBJ_aes_128_gcm */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x07, /* [ 5841] OBJ_aes_128_ccm */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x08, /* [ 5850] OBJ_id_aes128_wrap_pad */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x1A, /* [ 5859] OBJ_aes_192_gcm */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x1B, /* [ 5868] OBJ_aes_192_ccm */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x1C, /* [ 5877] OBJ_id_aes192_wrap_pad */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2E, /* [ 5886] OBJ_aes_256_gcm */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2F, /* [ 5895] OBJ_aes_256_ccm */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x30, /* [ 5904] OBJ_id_aes256_wrap_pad */ - 0x2A,0x83,0x08,0x8C,0x9A,0x4B,0x3D,0x01,0x01,0x03,0x02, /* [ 5913] OBJ_id_camellia128_wrap */ - 0x2A,0x83,0x08,0x8C,0x9A,0x4B,0x3D,0x01,0x01,0x03,0x03, /* [ 5924] OBJ_id_camellia192_wrap */ - 0x2A,0x83,0x08,0x8C,0x9A,0x4B,0x3D,0x01,0x01,0x03,0x04, /* [ 5935] OBJ_id_camellia256_wrap */ - 0x55,0x1D,0x25,0x00, /* [ 5946] OBJ_anyExtendedKeyUsage */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x08, /* [ 5950] OBJ_mgf1 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0A, /* [ 5959] OBJ_rsassaPss */ - 0x2B,0x6F,0x02,0x8C,0x53,0x00,0x01,0x01, /* [ 5968] OBJ_aes_128_xts */ - 0x2B,0x6F,0x02,0x8C,0x53,0x00,0x01,0x02, /* [ 5976] OBJ_aes_256_xts */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x07, /* [ 5984] OBJ_rsaesOaep */ - 0x2A,0x86,0x48,0xCE,0x3E,0x02,0x01, /* [ 5993] OBJ_dhpublicnumber */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x01, /* [ 6000] OBJ_brainpoolP160r1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x02, /* [ 6009] OBJ_brainpoolP160t1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x03, /* [ 6018] OBJ_brainpoolP192r1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x04, /* [ 6027] OBJ_brainpoolP192t1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x05, /* [ 6036] OBJ_brainpoolP224r1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x06, /* [ 6045] OBJ_brainpoolP224t1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x07, /* [ 6054] OBJ_brainpoolP256r1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x08, /* [ 6063] OBJ_brainpoolP256t1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x09, /* [ 6072] OBJ_brainpoolP320r1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0A, /* [ 6081] OBJ_brainpoolP320t1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0B, /* [ 6090] OBJ_brainpoolP384r1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0C, /* [ 6099] OBJ_brainpoolP384t1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0D, /* [ 6108] OBJ_brainpoolP512r1 */ - 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0E, /* [ 6117] OBJ_brainpoolP512t1 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x09, /* [ 6126] OBJ_pSpecified */ - 0x2B,0x81,0x05,0x10,0x86,0x48,0x3F,0x00,0x02, /* [ 6135] OBJ_dhSinglePass_stdDH_sha1kdf_scheme */ - 0x2B,0x81,0x04,0x01,0x0B,0x00, /* [ 6144] OBJ_dhSinglePass_stdDH_sha224kdf_scheme */ - 0x2B,0x81,0x04,0x01,0x0B,0x01, /* [ 6150] OBJ_dhSinglePass_stdDH_sha256kdf_scheme */ - 0x2B,0x81,0x04,0x01,0x0B,0x02, /* [ 6156] OBJ_dhSinglePass_stdDH_sha384kdf_scheme */ - 0x2B,0x81,0x04,0x01,0x0B,0x03, /* [ 6162] OBJ_dhSinglePass_stdDH_sha512kdf_scheme */ - 0x2B,0x81,0x05,0x10,0x86,0x48,0x3F,0x00,0x03, /* [ 6168] OBJ_dhSinglePass_cofactorDH_sha1kdf_scheme */ - 0x2B,0x81,0x04,0x01,0x0E,0x00, /* [ 6177] OBJ_dhSinglePass_cofactorDH_sha224kdf_scheme */ - 0x2B,0x81,0x04,0x01,0x0E,0x01, /* [ 6183] OBJ_dhSinglePass_cofactorDH_sha256kdf_scheme */ - 0x2B,0x81,0x04,0x01,0x0E,0x02, /* [ 6189] OBJ_dhSinglePass_cofactorDH_sha384kdf_scheme */ - 0x2B,0x81,0x04,0x01,0x0E,0x03, /* [ 6195] OBJ_dhSinglePass_cofactorDH_sha512kdf_scheme */ - 0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x02, /* [ 6201] OBJ_ct_precert_scts */ - 0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x03, /* [ 6211] OBJ_ct_precert_poison */ - 0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x04, /* [ 6221] OBJ_ct_precert_signer */ - 0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x05, /* [ 6231] OBJ_ct_cert_scts */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x01, /* [ 6241] OBJ_jurisdictionLocalityName */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x02, /* [ 6252] OBJ_jurisdictionStateOrProvinceName */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x03, /* [ 6263] OBJ_jurisdictionCountryName */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x06, /* [ 6274] OBJ_camellia_128_gcm */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x07, /* [ 6282] OBJ_camellia_128_ccm */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x09, /* [ 6290] OBJ_camellia_128_ctr */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x0A, /* [ 6298] OBJ_camellia_128_cmac */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x1A, /* [ 6306] OBJ_camellia_192_gcm */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x1B, /* [ 6314] OBJ_camellia_192_ccm */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x1D, /* [ 6322] OBJ_camellia_192_ctr */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x1E, /* [ 6330] OBJ_camellia_192_cmac */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x2E, /* [ 6338] OBJ_camellia_256_gcm */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x2F, /* [ 6346] OBJ_camellia_256_ccm */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x31, /* [ 6354] OBJ_camellia_256_ctr */ - 0x03,0xA2,0x31,0x05,0x03,0x01,0x09,0x32, /* [ 6362] OBJ_camellia_256_cmac */ - 0x2B,0x06,0x01,0x04,0x01,0xDA,0x47,0x04,0x0B, /* [ 6370] OBJ_id_scrypt */ - 0x2A,0x85,0x03,0x07,0x01, /* [ 6379] OBJ_id_tc26 */ - 0x2A,0x85,0x03,0x07,0x01,0x01, /* [ 6384] OBJ_id_tc26_algorithms */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x01, /* [ 6390] OBJ_id_tc26_sign */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x01,0x01, /* [ 6397] OBJ_id_GostR3410_2012_256 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x01,0x02, /* [ 6405] OBJ_id_GostR3410_2012_512 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x02, /* [ 6413] OBJ_id_tc26_digest */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x02,0x02, /* [ 6420] OBJ_id_GostR3411_2012_256 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x02,0x03, /* [ 6428] OBJ_id_GostR3411_2012_512 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x03, /* [ 6436] OBJ_id_tc26_signwithdigest */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x03,0x02, /* [ 6443] OBJ_id_tc26_signwithdigest_gost3410_2012_256 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x03,0x03, /* [ 6451] OBJ_id_tc26_signwithdigest_gost3410_2012_512 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x04, /* [ 6459] OBJ_id_tc26_mac */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x04,0x01, /* [ 6466] OBJ_id_tc26_hmac_gost_3411_2012_256 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x04,0x02, /* [ 6474] OBJ_id_tc26_hmac_gost_3411_2012_512 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x05, /* [ 6482] OBJ_id_tc26_cipher */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x06, /* [ 6489] OBJ_id_tc26_agreement */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x06,0x01, /* [ 6496] OBJ_id_tc26_agreement_gost_3410_2012_256 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x06,0x02, /* [ 6504] OBJ_id_tc26_agreement_gost_3410_2012_512 */ - 0x2A,0x85,0x03,0x07,0x01,0x02, /* [ 6512] OBJ_id_tc26_constants */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01, /* [ 6518] OBJ_id_tc26_sign_constants */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x02, /* [ 6525] OBJ_id_tc26_gost_3410_2012_512_constants */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x02,0x00, /* [ 6533] OBJ_id_tc26_gost_3410_2012_512_paramSetTest */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x02,0x01, /* [ 6542] OBJ_id_tc26_gost_3410_2012_512_paramSetA */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x02,0x02, /* [ 6551] OBJ_id_tc26_gost_3410_2012_512_paramSetB */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x02, /* [ 6560] OBJ_id_tc26_digest_constants */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x05, /* [ 6567] OBJ_id_tc26_cipher_constants */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x05,0x01, /* [ 6574] OBJ_id_tc26_gost_28147_constants */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x05,0x01,0x01, /* [ 6582] OBJ_id_tc26_gost_28147_param_Z */ - 0x2A,0x85,0x03,0x03,0x81,0x03,0x01,0x01, /* [ 6591] OBJ_INN */ - 0x2A,0x85,0x03,0x64,0x01, /* [ 6599] OBJ_OGRN */ - 0x2A,0x85,0x03,0x64,0x03, /* [ 6604] OBJ_SNILS */ - 0x2A,0x85,0x03,0x64,0x6F, /* [ 6609] OBJ_subjectSignTool */ - 0x2A,0x85,0x03,0x64,0x70, /* [ 6614] OBJ_issuerSignTool */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x18, /* [ 6619] OBJ_tlsfeature */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x11, /* [ 6627] OBJ_ipsec_IKE */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x12, /* [ 6635] OBJ_capwapAC */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x13, /* [ 6643] OBJ_capwapWTP */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x15, /* [ 6651] OBJ_sshClient */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x16, /* [ 6659] OBJ_sshServer */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x17, /* [ 6667] OBJ_sendRouter */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x18, /* [ 6675] OBJ_sendProxiedRouter */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x19, /* [ 6683] OBJ_sendOwner */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1A, /* [ 6691] OBJ_sendProxiedOwner */ - 0x2B,0x06,0x01,0x05,0x02,0x03, /* [ 6699] OBJ_id_pkinit */ - 0x2B,0x06,0x01,0x05,0x02,0x03,0x04, /* [ 6705] OBJ_pkInitClientAuth */ - 0x2B,0x06,0x01,0x05,0x02,0x03,0x05, /* [ 6712] OBJ_pkInitKDC */ - 0x2B,0x65,0x6E, /* [ 6719] OBJ_X25519 */ - 0x2B,0x65,0x6F, /* [ 6722] OBJ_X448 */ - 0x2B,0x06,0x01,0x04,0x01,0x8D,0x3A,0x0C,0x02,0x01,0x10, /* [ 6725] OBJ_blake2b512 */ - 0x2B,0x06,0x01,0x04,0x01,0x8D,0x3A,0x0C,0x02,0x02,0x08, /* [ 6736] OBJ_blake2s256 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x13, /* [ 6747] OBJ_id_smime_ct_contentCollection */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x17, /* [ 6758] OBJ_id_smime_ct_authEnvelopedData */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x1C, /* [ 6769] OBJ_id_ct_xml */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x01, /* [ 6780] OBJ_aria_128_ecb */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x02, /* [ 6789] OBJ_aria_128_cbc */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x03, /* [ 6798] OBJ_aria_128_cfb128 */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x04, /* [ 6807] OBJ_aria_128_ofb128 */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x05, /* [ 6816] OBJ_aria_128_ctr */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x06, /* [ 6825] OBJ_aria_192_ecb */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x07, /* [ 6834] OBJ_aria_192_cbc */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x08, /* [ 6843] OBJ_aria_192_cfb128 */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x09, /* [ 6852] OBJ_aria_192_ofb128 */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x0A, /* [ 6861] OBJ_aria_192_ctr */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x0B, /* [ 6870] OBJ_aria_256_ecb */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x0C, /* [ 6879] OBJ_aria_256_cbc */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x0D, /* [ 6888] OBJ_aria_256_cfb128 */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x0E, /* [ 6897] OBJ_aria_256_ofb128 */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x0F, /* [ 6906] OBJ_aria_256_ctr */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x2F, /* [ 6915] OBJ_id_smime_aa_signingCertificateV2 */ - 0x2B,0x65,0x70, /* [ 6926] OBJ_ED25519 */ - 0x2B,0x65,0x71, /* [ 6929] OBJ_ED448 */ - 0x55,0x04,0x61, /* [ 6932] OBJ_organizationIdentifier */ - 0x55,0x04,0x62, /* [ 6935] OBJ_countryCode3c */ - 0x55,0x04,0x63, /* [ 6938] OBJ_countryCode3n */ - 0x55,0x04,0x64, /* [ 6941] OBJ_dnsName */ - 0x2B,0x24,0x08,0x03,0x03, /* [ 6944] OBJ_x509ExtAdmission */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x05, /* [ 6949] OBJ_sha512_224 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x06, /* [ 6958] OBJ_sha512_256 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x07, /* [ 6967] OBJ_sha3_224 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x08, /* [ 6976] OBJ_sha3_256 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x09, /* [ 6985] OBJ_sha3_384 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0A, /* [ 6994] OBJ_sha3_512 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0B, /* [ 7003] OBJ_shake128 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0C, /* [ 7012] OBJ_shake256 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0D, /* [ 7021] OBJ_hmac_sha3_224 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0E, /* [ 7030] OBJ_hmac_sha3_256 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0F, /* [ 7039] OBJ_hmac_sha3_384 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x10, /* [ 7048] OBJ_hmac_sha3_512 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x03, /* [ 7057] OBJ_dsa_with_SHA384 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x04, /* [ 7066] OBJ_dsa_with_SHA512 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x05, /* [ 7075] OBJ_dsa_with_SHA3_224 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x06, /* [ 7084] OBJ_dsa_with_SHA3_256 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x07, /* [ 7093] OBJ_dsa_with_SHA3_384 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x08, /* [ 7102] OBJ_dsa_with_SHA3_512 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x09, /* [ 7111] OBJ_ecdsa_with_SHA3_224 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0A, /* [ 7120] OBJ_ecdsa_with_SHA3_256 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0B, /* [ 7129] OBJ_ecdsa_with_SHA3_384 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0C, /* [ 7138] OBJ_ecdsa_with_SHA3_512 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0D, /* [ 7147] OBJ_RSA_SHA3_224 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0E, /* [ 7156] OBJ_RSA_SHA3_256 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0F, /* [ 7165] OBJ_RSA_SHA3_384 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x10, /* [ 7174] OBJ_RSA_SHA3_512 */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x25, /* [ 7183] OBJ_aria_128_ccm */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x26, /* [ 7192] OBJ_aria_192_ccm */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x27, /* [ 7201] OBJ_aria_256_ccm */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x22, /* [ 7210] OBJ_aria_128_gcm */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x23, /* [ 7219] OBJ_aria_192_gcm */ - 0x2A,0x83,0x1A,0x8C,0x9A,0x6E,0x01,0x01,0x24, /* [ 7228] OBJ_aria_256_gcm */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1B, /* [ 7237] OBJ_cmcCA */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1C, /* [ 7245] OBJ_cmcRA */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x01, /* [ 7253] OBJ_sm4_ecb */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x02, /* [ 7261] OBJ_sm4_cbc */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x03, /* [ 7269] OBJ_sm4_ofb128 */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x05, /* [ 7277] OBJ_sm4_cfb1 */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x04, /* [ 7285] OBJ_sm4_cfb128 */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x06, /* [ 7293] OBJ_sm4_cfb8 */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x07, /* [ 7301] OBJ_sm4_ctr */ - 0x2A,0x81,0x1C, /* [ 7309] OBJ_ISO_CN */ - 0x2A,0x81,0x1C,0xCF,0x55, /* [ 7312] OBJ_oscca */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01, /* [ 7317] OBJ_sm_scheme */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x83,0x11, /* [ 7323] OBJ_sm3 */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x83,0x78, /* [ 7331] OBJ_sm3WithRSAEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0F, /* [ 7339] OBJ_sha512_224WithRSAEncryption */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x10, /* [ 7348] OBJ_sha512_256WithRSAEncryption */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x01, /* [ 7357] OBJ_id_tc26_gost_3410_2012_256_constants */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x01,0x01, /* [ 7365] OBJ_id_tc26_gost_3410_2012_256_paramSetA */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x02,0x03, /* [ 7374] OBJ_id_tc26_gost_3410_2012_512_paramSetC */ - 0x2A,0x86,0x24, /* [ 7383] OBJ_ISO_UA */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01, /* [ 7386] OBJ_ua_pki */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x01,0x01, /* [ 7393] OBJ_dstu28147 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x02, /* [ 7403] OBJ_dstu28147_ofb */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x03, /* [ 7414] OBJ_dstu28147_cfb */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x05, /* [ 7425] OBJ_dstu28147_wrap */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x01,0x02, /* [ 7436] OBJ_hmacWithDstu34311 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x02,0x01, /* [ 7446] OBJ_dstu34311 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01, /* [ 7456] OBJ_dstu4145le */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x01,0x01, /* [ 7467] OBJ_dstu4145be */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x00, /* [ 7480] OBJ_uacurve0 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x01, /* [ 7493] OBJ_uacurve1 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x02, /* [ 7506] OBJ_uacurve2 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x03, /* [ 7519] OBJ_uacurve3 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x04, /* [ 7532] OBJ_uacurve4 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x05, /* [ 7545] OBJ_uacurve5 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x06, /* [ 7558] OBJ_uacurve6 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x07, /* [ 7571] OBJ_uacurve7 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x08, /* [ 7584] OBJ_uacurve8 */ - 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x09, /* [ 7597] OBJ_uacurve9 */ - 0x2B,0x6F, /* [ 7610] OBJ_ieee */ - 0x2B,0x6F,0x02,0x8C,0x53, /* [ 7612] OBJ_ieee_siswg */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x82,0x2D, /* [ 7617] OBJ_sm2 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x05,0x01, /* [ 7625] OBJ_id_tc26_cipher_gostr3412_2015_magma */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x05,0x01,0x01, /* [ 7633] OBJ_magma_ctr_acpkm */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x05,0x01,0x02, /* [ 7642] OBJ_magma_ctr_acpkm_omac */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x05,0x02, /* [ 7651] OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x05,0x02,0x01, /* [ 7659] OBJ_kuznyechik_ctr_acpkm */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x05,0x02,0x02, /* [ 7668] OBJ_kuznyechik_ctr_acpkm_omac */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x07, /* [ 7677] OBJ_id_tc26_wrap */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x07,0x01, /* [ 7684] OBJ_id_tc26_wrap_gostr3412_2015_magma */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x07,0x01,0x01, /* [ 7692] OBJ_magma_kexp15 */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x07,0x02, /* [ 7701] OBJ_id_tc26_wrap_gostr3412_2015_kuznyechik */ - 0x2A,0x85,0x03,0x07,0x01,0x01,0x07,0x02,0x01, /* [ 7709] OBJ_kuznyechik_kexp15 */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x01,0x02, /* [ 7718] OBJ_id_tc26_gost_3410_2012_256_paramSetB */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x01,0x03, /* [ 7727] OBJ_id_tc26_gost_3410_2012_256_paramSetC */ - 0x2A,0x85,0x03,0x07,0x01,0x02,0x01,0x01,0x04, /* [ 7736] OBJ_id_tc26_gost_3410_2012_256_paramSetD */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x0C, /* [ 7745] OBJ_hmacWithSHA512_224 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x0D, /* [ 7753] OBJ_hmacWithSHA512_256 */ - 0x28,0xCC,0x45,0x03,0x04, /* [ 7761] OBJ_gmac */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x13, /* [ 7766] OBJ_kmac128 */ - 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x14, /* [ 7775] OBJ_kmac256 */ - 0x2B,0x06,0x01,0x04,0x01,0x8D,0x3A,0x0C,0x02,0x01, /* [ 7784] OBJ_blake2bmac */ - 0x2B,0x06,0x01,0x04,0x01,0x8D,0x3A,0x0C,0x02,0x02, /* [ 7794] OBJ_blake2smac */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x83,0x75, /* [ 7804] OBJ_SM2_with_SM3 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x09, /* [ 7812] OBJ_id_on_SmtpUTF8Mailbox */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x05, /* [ 7820] OBJ_XmppAddr */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x07, /* [ 7828] OBJ_SRVName */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x08, /* [ 7836] OBJ_NAIRealm */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1D, /* [ 7844] OBJ_cmcArchive */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1E, /* [ 7852] OBJ_id_kp_bgpsec_router */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1F, /* [ 7860] OBJ_id_kp_BrandIndicatorforMessageIdentification */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x20, /* [ 7868] OBJ_cmKGA */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x11, /* [ 7876] OBJ_id_it_caCerts */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x12, /* [ 7884] OBJ_id_it_rootCaKeyUpdate */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x13, /* [ 7892] OBJ_id_it_certReqTemplate */ - 0x2A,0x85,0x03,0x64,0x05, /* [ 7900] OBJ_OGRNIP */ - 0x2A,0x85,0x03,0x64,0x71, /* [ 7905] OBJ_classSignTool */ - 0x2A,0x85,0x03,0x64,0x71,0x01, /* [ 7910] OBJ_classSignToolKC1 */ - 0x2A,0x85,0x03,0x64,0x71,0x02, /* [ 7916] OBJ_classSignToolKC2 */ - 0x2A,0x85,0x03,0x64,0x71,0x03, /* [ 7922] OBJ_classSignToolKC3 */ - 0x2A,0x85,0x03,0x64,0x71,0x04, /* [ 7928] OBJ_classSignToolKB1 */ - 0x2A,0x85,0x03,0x64,0x71,0x05, /* [ 7934] OBJ_classSignToolKB2 */ - 0x2A,0x85,0x03,0x64,0x71,0x06, /* [ 7940] OBJ_classSignToolKA1 */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x18, /* [ 7946] OBJ_id_ct_routeOriginAuthz */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x1A, /* [ 7957] OBJ_id_ct_rpkiManifest */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x23, /* [ 7968] OBJ_id_ct_rpkiGhostbusters */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x24, /* [ 7979] OBJ_id_ct_resourceTaggedAttest */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0E, /* [ 7990] OBJ_id_cp */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x1C, /* [ 7997] OBJ_sbgp_ipAddrBlockv2 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x1D, /* [ 8005] OBJ_sbgp_autonomousSysNumv2 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0E,0x02, /* [ 8013] OBJ_ipAddr_asNumber */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x0E,0x03, /* [ 8021] OBJ_ipAddr_asNumberv2 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x0A, /* [ 8029] OBJ_rpkiManifest */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x0B, /* [ 8037] OBJ_signedObject */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x0D, /* [ 8045] OBJ_rpkiNotify */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x2F, /* [ 8053] OBJ_id_ct_geofeedCSVwithCRLF */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x30, /* [ 8064] OBJ_id_ct_signedChecklist */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x08, /* [ 8075] OBJ_sm4_gcm */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x09, /* [ 8083] OBJ_sm4_ccm */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x31, /* [ 8091] OBJ_id_ct_ASPA */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x32, /* [ 8102] OBJ_id_mod_cmp2000_02 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x63, /* [ 8110] OBJ_id_mod_cmp2021_88 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x64, /* [ 8118] OBJ_id_mod_cmp2021_02 */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x14, /* [ 8126] OBJ_id_it_rootCaCert */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x15, /* [ 8134] OBJ_id_it_certProfile */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x16, /* [ 8142] OBJ_id_it_crlStatusList */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x17, /* [ 8150] OBJ_id_it_crls */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x07, /* [ 8158] OBJ_id_regCtrl_altCertTemplate */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x0B, /* [ 8167] OBJ_id_regCtrl_algId */ - 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x0C, /* [ 8176] OBJ_id_regCtrl_rsaKeyLen */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x2C, /* [ 8185] OBJ_id_aa_ets_attrCertificateRefs */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x2D, /* [ 8196] OBJ_id_aa_ets_attrRevocationRefs */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x34, /* [ 8207] OBJ_id_aa_CMSAlgorithmProtection */ - 0x04, /* [ 8216] OBJ_itu_t_identified_organization */ - 0x04,0x00, /* [ 8217] OBJ_etsi */ - 0x04,0x00,0x8D,0x45, /* [ 8219] OBJ_electronic_signature_standard */ - 0x04,0x00,0x8D,0x45,0x02, /* [ 8223] OBJ_ess_attributes */ - 0x04,0x00,0x8D,0x45,0x02,0x01, /* [ 8228] OBJ_id_aa_ets_mimeType */ - 0x04,0x00,0x8D,0x45,0x02,0x02, /* [ 8234] OBJ_id_aa_ets_longTermValidation */ - 0x04,0x00,0x8D,0x45,0x02,0x03, /* [ 8240] OBJ_id_aa_ets_SignaturePolicyDocument */ - 0x04,0x00,0x8D,0x45,0x02,0x04, /* [ 8246] OBJ_id_aa_ets_archiveTimestampV3 */ - 0x04,0x00,0x8D,0x45,0x02,0x05, /* [ 8252] OBJ_id_aa_ATSHashIndex */ - 0x04,0x00,0x81,0x95,0x32, /* [ 8258] OBJ_cades */ - 0x04,0x00,0x81,0x95,0x32,0x01, /* [ 8263] OBJ_cades_attributes */ - 0x04,0x00,0x81,0x95,0x32,0x01,0x01, /* [ 8269] OBJ_id_aa_ets_signerAttrV2 */ - 0x04,0x00,0x81,0x95,0x32,0x01,0x03, /* [ 8276] OBJ_id_aa_ets_sigPolicyStore */ - 0x04,0x00,0x81,0x95,0x32,0x01,0x04, /* [ 8283] OBJ_id_aa_ATSHashIndex_v2 */ - 0x04,0x00,0x81,0x95,0x32,0x01,0x05, /* [ 8290] OBJ_id_aa_ATSHashIndex_v3 */ - 0x04,0x00,0x81,0x95,0x32,0x01,0x06, /* [ 8297] OBJ_signedAssertion */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x30, /* [ 8304] OBJ_id_aa_ets_archiveTimestampV2 */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x83,0x11,0x03,0x01, /* [ 8315] OBJ_hmacWithSM3 */ - 0x60,0x86,0x48,0x01,0x86,0xF9,0x66, /* [ 8325] OBJ_oracle */ - 0x60,0x86,0x48,0x01,0x86,0xF9,0x66,0xAD,0xCA,0x7B,0x01,0x01, /* [ 8332] OBJ_oracle_jdk_trustedkeyusage */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x32, /* [ 8344] OBJ_id_ct_signedTAL */ - 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x0A, /* [ 8355] OBJ_sm4_xts */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x19,0x02,0x01, /* [ 8363] OBJ_ms_ntds_obj_sid */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x19,0x02, /* [ 8373] OBJ_ms_ntds_sec_ext */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x15,0x07, /* [ 8382] OBJ_ms_cert_templ */ - 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x15,0x0A, /* [ 8391] OBJ_ms_app_policies */ - 0x55,0x1D,0x26, /* [ 8400] OBJ_authority_attribute_identifier */ - 0x55,0x1D,0x27, /* [ 8403] OBJ_role_spec_cert_identifier */ - 0x55,0x1D,0x29, /* [ 8406] OBJ_basic_att_constraints */ - 0x55,0x1D,0x2A, /* [ 8409] OBJ_delegated_name_constraints */ - 0x55,0x1D,0x2B, /* [ 8412] OBJ_time_specification */ - 0x55,0x1D,0x30, /* [ 8415] OBJ_attribute_descriptor */ - 0x55,0x1D,0x31, /* [ 8418] OBJ_user_notice */ - 0x55,0x1D,0x32, /* [ 8421] OBJ_soa_identifier */ - 0x55,0x1D,0x34, /* [ 8424] OBJ_acceptable_cert_policies */ - 0x55,0x1D,0x39, /* [ 8427] OBJ_acceptable_privilege_policies */ - 0x55,0x1D,0x3D, /* [ 8430] OBJ_indirect_issuer */ - 0x55,0x1D,0x3E, /* [ 8433] OBJ_no_assertion */ - 0x55,0x1D,0x3F, /* [ 8436] OBJ_id_aa_issuing_distribution_point */ - 0x55,0x1D,0x40, /* [ 8439] OBJ_issued_on_behalf_of */ - 0x55,0x1D,0x41, /* [ 8442] OBJ_single_use */ - 0x55,0x1D,0x42, /* [ 8445] OBJ_group_ac */ - 0x55,0x1D,0x43, /* [ 8448] OBJ_allowed_attribute_assignments */ - 0x55,0x1D,0x44, /* [ 8451] OBJ_attribute_mappings */ - 0x55,0x1D,0x45, /* [ 8454] OBJ_holder_name_constraints */ - 0x55,0x1D,0x46, /* [ 8457] OBJ_authorization_validation */ - 0x55,0x1D,0x47, /* [ 8460] OBJ_prot_restrict */ - 0x55,0x1D,0x48, /* [ 8463] OBJ_subject_alt_public_key_info */ - 0x55,0x1D,0x49, /* [ 8466] OBJ_alt_signature_algorithm */ - 0x55,0x1D,0x4A, /* [ 8469] OBJ_alt_signature_value */ - 0x55,0x1D,0x4B, /* [ 8472] OBJ_associated_information */ - 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x33, /* [ 8475] OBJ_id_ct_rpkiSignedPrefixList */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x05, /* [ 13] OBJ_md5 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x04, /* [ 21] OBJ_rc4 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x01, /* [ 29] OBJ_rsaEncryption */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x04, /* [ 38] OBJ_md5WithRSAEncryption */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x03, /* [ 47] OBJ_pbeWithMD5AndDES_CBC */ + 0x55, /* [ 56] OBJ_X500 */ + 0x55,0x04, /* [ 57] OBJ_X509 */ + 0x55,0x04,0x03, /* [ 59] OBJ_commonName */ + 0x55,0x04,0x06, /* [ 62] OBJ_countryName */ + 0x55,0x04,0x07, /* [ 65] OBJ_localityName */ + 0x55,0x04,0x08, /* [ 68] OBJ_stateOrProvinceName */ + 0x55,0x04,0x0A, /* [ 71] OBJ_organizationName */ + 0x55,0x04,0x0B, /* [ 74] OBJ_organizationalUnitName */ + 0x55,0x08,0x01,0x01, /* [ 77] OBJ_rsa */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07, /* [ 81] OBJ_pkcs7 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x01, /* [ 89] OBJ_pkcs7_data */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x02, /* [ 98] OBJ_pkcs7_signed */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x03, /* [ 107] OBJ_pkcs7_enveloped */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x04, /* [ 116] OBJ_pkcs7_signedAndEnveloped */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x05, /* [ 125] OBJ_pkcs7_digest */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x07,0x06, /* [ 134] OBJ_pkcs7_encrypted */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x03, /* [ 143] OBJ_pkcs3 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x03,0x01, /* [ 151] OBJ_dhKeyAgreement */ + 0x2B,0x0E,0x03,0x02,0x06, /* [ 160] OBJ_des_ecb */ + 0x2B,0x0E,0x03,0x02,0x09, /* [ 165] OBJ_des_cfb64 */ + 0x2B,0x0E,0x03,0x02,0x07, /* [ 170] OBJ_des_cbc */ + 0x2B,0x0E,0x03,0x02,0x11, /* [ 175] OBJ_des_ede_ecb */ + 0x2B,0x0E,0x03,0x02,0x12, /* [ 180] OBJ_sha */ + 0x2B,0x0E,0x03,0x02,0x0F, /* [ 185] OBJ_shaWithRSAEncryption */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x07, /* [ 190] OBJ_des_ede3_cbc */ + 0x2B,0x0E,0x03,0x02,0x08, /* [ 198] OBJ_des_ofb64 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09, /* [ 203] OBJ_pkcs9 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x01, /* [ 211] OBJ_pkcs9_emailAddress */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x02, /* [ 220] OBJ_pkcs9_unstructuredName */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x03, /* [ 229] OBJ_pkcs9_contentType */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x04, /* [ 238] OBJ_pkcs9_messageDigest */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x05, /* [ 247] OBJ_pkcs9_signingTime */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x06, /* [ 256] OBJ_pkcs9_countersignature */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x07, /* [ 265] OBJ_pkcs9_challengePassword */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x08, /* [ 274] OBJ_pkcs9_unstructuredAddress */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x09, /* [ 283] OBJ_pkcs9_extCertAttributes */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42, /* [ 292] OBJ_netscape */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01, /* [ 299] OBJ_netscape_cert_extension */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x02, /* [ 307] OBJ_netscape_data_type */ + 0x2B,0x0E,0x03,0x02,0x1A, /* [ 315] OBJ_sha1 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x05, /* [ 320] OBJ_sha1WithRSAEncryption */ + 0x2B,0x0E,0x03,0x02,0x0D, /* [ 329] OBJ_dsaWithSHA */ + 0x2B,0x0E,0x03,0x02,0x0C, /* [ 334] OBJ_dsa_2 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0C, /* [ 339] OBJ_id_pbkdf2 */ + 0x2B,0x0E,0x03,0x02,0x1B, /* [ 348] OBJ_dsaWithSHA1_2 */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x01, /* [ 353] OBJ_netscape_cert_type */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x02, /* [ 362] OBJ_netscape_base_url */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x03, /* [ 371] OBJ_netscape_revocation_url */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x04, /* [ 380] OBJ_netscape_ca_revocation_url */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x07, /* [ 389] OBJ_netscape_renewal_url */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x08, /* [ 398] OBJ_netscape_ca_policy_url */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x0C, /* [ 407] OBJ_netscape_ssl_server_name */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x01,0x0D, /* [ 416] OBJ_netscape_comment */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x02,0x05, /* [ 425] OBJ_netscape_cert_sequence */ + 0x55,0x1D, /* [ 434] OBJ_id_ce */ + 0x55,0x1D,0x0E, /* [ 436] OBJ_subject_key_identifier */ + 0x55,0x1D,0x0F, /* [ 439] OBJ_key_usage */ + 0x55,0x1D,0x10, /* [ 442] OBJ_private_key_usage_period */ + 0x55,0x1D,0x11, /* [ 445] OBJ_subject_alt_name */ + 0x55,0x1D,0x12, /* [ 448] OBJ_issuer_alt_name */ + 0x55,0x1D,0x13, /* [ 451] OBJ_basic_constraints */ + 0x55,0x1D,0x14, /* [ 454] OBJ_crl_number */ + 0x55,0x1D,0x20, /* [ 457] OBJ_certificate_policies */ + 0x55,0x1D,0x23, /* [ 460] OBJ_authority_key_identifier */ + 0x55,0x04,0x2A, /* [ 463] OBJ_givenName */ + 0x55,0x04,0x04, /* [ 466] OBJ_surname */ + 0x55,0x04,0x2B, /* [ 469] OBJ_initials */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2C, /* [ 472] OBJ_uniqueIdentifier */ + 0x55,0x1D,0x1F, /* [ 482] OBJ_crl_distribution_points */ + 0x2B,0x0E,0x03,0x02,0x03, /* [ 485] OBJ_md5WithRSA */ + 0x55,0x04,0x05, /* [ 490] OBJ_serialNumber */ + 0x55,0x04,0x0C, /* [ 493] OBJ_title */ + 0x55,0x04,0x0D, /* [ 496] OBJ_description */ + 0x2A,0x86,0x48,0xCE,0x38,0x04,0x03, /* [ 499] OBJ_dsaWithSHA1 */ + 0x2B,0x0E,0x03,0x02,0x1D, /* [ 506] OBJ_sha1WithRSA */ + 0x2A,0x86,0x48,0xCE,0x38,0x04,0x01, /* [ 511] OBJ_dsa */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x08, /* [ 518] OBJ_rc5_cbc */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x08, /* [ 526] OBJ_zlib_compression */ + 0x55,0x1D,0x25, /* [ 537] OBJ_ext_key_usage */ + 0x2B,0x06,0x01,0x05,0x05,0x07, /* [ 540] OBJ_id_pkix */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03, /* [ 546] OBJ_id_kp */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x01, /* [ 553] OBJ_server_auth */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x02, /* [ 561] OBJ_client_auth */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x03, /* [ 569] OBJ_code_sign */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x04, /* [ 577] OBJ_email_protect */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x08, /* [ 585] OBJ_time_stamp */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x02,0x01,0x15, /* [ 593] OBJ_ms_code_ind */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x02,0x01,0x16, /* [ 603] OBJ_ms_code_com */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x0A,0x03,0x01, /* [ 613] OBJ_ms_ctl_sign */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x0A,0x03,0x03, /* [ 623] OBJ_ms_sgc */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x0A,0x03,0x04, /* [ 633] OBJ_ms_efs */ + 0x60,0x86,0x48,0x01,0x86,0xF8,0x42,0x04,0x01, /* [ 643] OBJ_ns_sgc */ + 0x55,0x1D,0x1B, /* [ 652] OBJ_delta_crl */ + 0x55,0x1D,0x15, /* [ 655] OBJ_crl_reason */ + 0x55,0x1D,0x18, /* [ 658] OBJ_invalidity_date */ + 0x2B,0x65,0x01,0x04,0x01, /* [ 661] OBJ_sxnet */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x01, /* [ 666] OBJ_pbe_WithSHA1And128BitRC4 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x02, /* [ 676] OBJ_pbe_WithSHA1And40BitRC4 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x03, /* [ 686] OBJ_pbe_WithSHA1And3_Key_TripleDES_CBC */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x01,0x04, /* [ 696] OBJ_pbe_WithSHA1And2_Key_TripleDES_CBC */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x01, /* [ 706] OBJ_keyBag */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x02, /* [ 717] OBJ_pkcs8ShroudedKeyBag */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x03, /* [ 728] OBJ_certBag */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x04, /* [ 739] OBJ_crlBag */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x05, /* [ 750] OBJ_secretBag */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x0C,0x0A,0x01,0x06, /* [ 761] OBJ_safeContentsBag */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x14, /* [ 772] OBJ_friendlyName */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x15, /* [ 781] OBJ_localKeyID */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x16,0x01, /* [ 790] OBJ_x509Certificate */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x16,0x02, /* [ 800] OBJ_sdsiCertificate */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x17,0x01, /* [ 810] OBJ_x509Crl */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0D, /* [ 820] OBJ_pbes2 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0E, /* [ 829] OBJ_pbmac1 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x07, /* [ 838] OBJ_hmacWithSHA1 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x02,0x01, /* [ 846] OBJ_id_qt_cps */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x02,0x02, /* [ 854] OBJ_id_qt_unotice */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x0F, /* [ 862] OBJ_SMIMECapabilities */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05,0x0A, /* [ 871] OBJ_pbeWithSHA1AndDES_CBC */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x02,0x01,0x0E, /* [ 880] OBJ_ms_ext_req */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x0E, /* [ 890] OBJ_ext_req */ + 0x55,0x04,0x29, /* [ 899] OBJ_name */ + 0x55,0x04,0x2E, /* [ 902] OBJ_dnQualifier */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01, /* [ 905] OBJ_id_pe */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30, /* [ 912] OBJ_id_ad */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x01, /* [ 919] OBJ_info_access */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01, /* [ 927] OBJ_ad_OCSP */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x02, /* [ 935] OBJ_ad_ca_issuers */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x09, /* [ 943] OBJ_OCSP_sign */ + 0x2A, /* [ 951] OBJ_member_body */ + 0x2A,0x86,0x48, /* [ 952] OBJ_ISO_US */ + 0x2A,0x86,0x48,0xCE,0x38, /* [ 955] OBJ_X9_57 */ + 0x2A,0x86,0x48,0xCE,0x38,0x04, /* [ 960] OBJ_X9cm */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01, /* [ 966] OBJ_pkcs1 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x05, /* [ 974] OBJ_pkcs5 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10, /* [ 982] OBJ_SMIME */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00, /* [ 991] OBJ_id_smime_mod */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01, /* [ 1001] OBJ_id_smime_ct */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02, /* [ 1011] OBJ_id_smime_aa */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03, /* [ 1021] OBJ_id_smime_alg */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x04, /* [ 1031] OBJ_id_smime_cd */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x05, /* [ 1041] OBJ_id_smime_spq */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06, /* [ 1051] OBJ_id_smime_cti */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x01, /* [ 1061] OBJ_id_smime_mod_cms */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x02, /* [ 1072] OBJ_id_smime_mod_ess */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x03, /* [ 1083] OBJ_id_smime_mod_oid */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x04, /* [ 1094] OBJ_id_smime_mod_msg_v3 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x05, /* [ 1105] OBJ_id_smime_mod_ets_eSignature_88 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x06, /* [ 1116] OBJ_id_smime_mod_ets_eSignature_97 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x07, /* [ 1127] OBJ_id_smime_mod_ets_eSigPolicy_88 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x00,0x08, /* [ 1138] OBJ_id_smime_mod_ets_eSigPolicy_97 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x01, /* [ 1149] OBJ_id_smime_ct_receipt */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x02, /* [ 1160] OBJ_id_smime_ct_authData */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x03, /* [ 1171] OBJ_id_smime_ct_publishCert */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x04, /* [ 1182] OBJ_id_smime_ct_TSTInfo */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x05, /* [ 1193] OBJ_id_smime_ct_TDTInfo */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x06, /* [ 1204] OBJ_id_smime_ct_contentInfo */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x07, /* [ 1215] OBJ_id_smime_ct_DVCSRequestData */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x08, /* [ 1226] OBJ_id_smime_ct_DVCSResponseData */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x01, /* [ 1237] OBJ_id_smime_aa_receiptRequest */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x02, /* [ 1248] OBJ_id_smime_aa_securityLabel */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x03, /* [ 1259] OBJ_id_smime_aa_mlExpandHistory */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x04, /* [ 1270] OBJ_id_smime_aa_contentHint */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x05, /* [ 1281] OBJ_id_smime_aa_msgSigDigest */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x06, /* [ 1292] OBJ_id_smime_aa_encapContentType */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x07, /* [ 1303] OBJ_id_smime_aa_contentIdentifier */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x08, /* [ 1314] OBJ_id_smime_aa_macValue */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x09, /* [ 1325] OBJ_id_smime_aa_equivalentLabels */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0A, /* [ 1336] OBJ_id_smime_aa_contentReference */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0B, /* [ 1347] OBJ_id_smime_aa_encrypKeyPref */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0C, /* [ 1358] OBJ_id_smime_aa_signingCertificate */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0D, /* [ 1369] OBJ_id_smime_aa_smimeEncryptCerts */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0E, /* [ 1380] OBJ_id_smime_aa_timeStampToken */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x0F, /* [ 1391] OBJ_id_smime_aa_ets_sigPolicyId */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x10, /* [ 1402] OBJ_id_smime_aa_ets_commitmentType */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x11, /* [ 1413] OBJ_id_smime_aa_ets_signerLocation */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x12, /* [ 1424] OBJ_id_smime_aa_ets_signerAttr */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x13, /* [ 1435] OBJ_id_smime_aa_ets_otherSigCert */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x14, /* [ 1446] OBJ_id_smime_aa_ets_contentTimestamp */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x15, /* [ 1457] OBJ_id_smime_aa_ets_CertificateRefs */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x16, /* [ 1468] OBJ_id_smime_aa_ets_RevocationRefs */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x17, /* [ 1479] OBJ_id_smime_aa_ets_certValues */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x18, /* [ 1490] OBJ_id_smime_aa_ets_revocationValues */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x19, /* [ 1501] OBJ_id_smime_aa_ets_escTimeStamp */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x1A, /* [ 1512] OBJ_id_smime_aa_ets_certCRLTimestamp */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x1B, /* [ 1523] OBJ_id_smime_aa_ets_archiveTimeStamp */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x1C, /* [ 1534] OBJ_id_smime_aa_signatureType */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x1D, /* [ 1545] OBJ_id_smime_aa_dvcs_dvc */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x01, /* [ 1556] OBJ_id_smime_alg_ESDHwith3DES */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x03, /* [ 1567] OBJ_id_smime_alg_3DESwrap */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x05, /* [ 1578] OBJ_id_smime_alg_ESDH */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x06, /* [ 1589] OBJ_id_smime_alg_CMS3DESwrap */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x04,0x01, /* [ 1600] OBJ_id_smime_cd_ldap */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x05,0x01, /* [ 1611] OBJ_id_smime_spq_ets_sqt_uri */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x05,0x02, /* [ 1622] OBJ_id_smime_spq_ets_sqt_unotice */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x01, /* [ 1633] OBJ_id_smime_cti_ets_proofOfOrigin */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x02, /* [ 1644] OBJ_id_smime_cti_ets_proofOfReceipt */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x03, /* [ 1655] OBJ_id_smime_cti_ets_proofOfDelivery */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x04, /* [ 1666] OBJ_id_smime_cti_ets_proofOfSender */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x05, /* [ 1677] OBJ_id_smime_cti_ets_proofOfApproval */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x06,0x06, /* [ 1688] OBJ_id_smime_cti_ets_proofOfCreation */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00, /* [ 1699] OBJ_id_pkix_mod */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x02, /* [ 1706] OBJ_id_qt */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04, /* [ 1713] OBJ_id_it */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05, /* [ 1720] OBJ_id_pkip */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x06, /* [ 1727] OBJ_id_alg */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07, /* [ 1734] OBJ_id_cmc */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x08, /* [ 1741] OBJ_id_on */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x09, /* [ 1748] OBJ_id_pda */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A, /* [ 1755] OBJ_id_aca */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0B, /* [ 1762] OBJ_id_qcs */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0C, /* [ 1769] OBJ_id_cct */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x01, /* [ 1776] OBJ_id_pkix1_explicit_88 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x02, /* [ 1784] OBJ_id_pkix1_implicit_88 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x03, /* [ 1792] OBJ_id_pkix1_explicit_93 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x04, /* [ 1800] OBJ_id_pkix1_implicit_93 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x05, /* [ 1808] OBJ_id_mod_crmf */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x06, /* [ 1816] OBJ_id_mod_cmc */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x07, /* [ 1824] OBJ_id_mod_kea_profile_88 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x08, /* [ 1832] OBJ_id_mod_kea_profile_93 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x09, /* [ 1840] OBJ_id_mod_cmp */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0A, /* [ 1848] OBJ_id_mod_qualified_cert_88 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0B, /* [ 1856] OBJ_id_mod_qualified_cert_93 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0C, /* [ 1864] OBJ_id_mod_attribute_cert */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0D, /* [ 1872] OBJ_id_mod_timestamp_protocol */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0E, /* [ 1880] OBJ_id_mod_ocsp */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x0F, /* [ 1888] OBJ_id_mod_dvcs */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x00,0x10, /* [ 1896] OBJ_id_mod_cmp2000 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x02, /* [ 1904] OBJ_biometricInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x03, /* [ 1912] OBJ_qcStatements */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x04, /* [ 1920] OBJ_ac_auditEntity */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x05, /* [ 1928] OBJ_ac_targeting */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x06, /* [ 1936] OBJ_aaControls */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x07, /* [ 1944] OBJ_sbgp_ipAddrBlock */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x08, /* [ 1952] OBJ_sbgp_autonomousSysNum */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x09, /* [ 1960] OBJ_sbgp_routerIdentifier */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x02,0x03, /* [ 1968] OBJ_textNotice */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x05, /* [ 1976] OBJ_ipsecEndSystem */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x06, /* [ 1984] OBJ_ipsecTunnel */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x07, /* [ 1992] OBJ_ipsecUser */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x0A, /* [ 2000] OBJ_dvcs */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x01, /* [ 2008] OBJ_id_it_caProtEncCert */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x02, /* [ 2016] OBJ_id_it_signKeyPairTypes */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x03, /* [ 2024] OBJ_id_it_encKeyPairTypes */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x04, /* [ 2032] OBJ_id_it_preferredSymmAlg */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x05, /* [ 2040] OBJ_id_it_caKeyUpdateInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x06, /* [ 2048] OBJ_id_it_currentCRL */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x07, /* [ 2056] OBJ_id_it_unsupportedOIDs */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x08, /* [ 2064] OBJ_id_it_subscriptionRequest */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x09, /* [ 2072] OBJ_id_it_subscriptionResponse */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0A, /* [ 2080] OBJ_id_it_keyPairParamReq */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0B, /* [ 2088] OBJ_id_it_keyPairParamRep */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0C, /* [ 2096] OBJ_id_it_revPassphrase */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0D, /* [ 2104] OBJ_id_it_implicitConfirm */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0E, /* [ 2112] OBJ_id_it_confirmWaitTime */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x0F, /* [ 2120] OBJ_id_it_origPKIMessage */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01, /* [ 2128] OBJ_id_regCtrl */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x02, /* [ 2136] OBJ_id_regInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x01, /* [ 2144] OBJ_id_regCtrl_regToken */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x02, /* [ 2153] OBJ_id_regCtrl_authenticator */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x03, /* [ 2162] OBJ_id_regCtrl_pkiPublicationInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x04, /* [ 2171] OBJ_id_regCtrl_pkiArchiveOptions */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x05, /* [ 2180] OBJ_id_regCtrl_oldCertID */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x01,0x06, /* [ 2189] OBJ_id_regCtrl_protocolEncrKey */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x02,0x01, /* [ 2198] OBJ_id_regInfo_utf8Pairs */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x05,0x02,0x02, /* [ 2207] OBJ_id_regInfo_certReq */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x06,0x01, /* [ 2216] OBJ_id_alg_des40 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x06,0x02, /* [ 2224] OBJ_id_alg_noSignature */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x06,0x03, /* [ 2232] OBJ_id_alg_dh_sig_hmac_sha1 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x06,0x04, /* [ 2240] OBJ_id_alg_dh_pop */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x01, /* [ 2248] OBJ_id_cmc_statusInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x02, /* [ 2256] OBJ_id_cmc_identification */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x03, /* [ 2264] OBJ_id_cmc_identityProof */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x04, /* [ 2272] OBJ_id_cmc_dataReturn */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x05, /* [ 2280] OBJ_id_cmc_transactionId */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x06, /* [ 2288] OBJ_id_cmc_senderNonce */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x07, /* [ 2296] OBJ_id_cmc_recipientNonce */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x08, /* [ 2304] OBJ_id_cmc_addExtensions */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x09, /* [ 2312] OBJ_id_cmc_encryptedPOP */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x0A, /* [ 2320] OBJ_id_cmc_decryptedPOP */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x0B, /* [ 2328] OBJ_id_cmc_lraPOPWitness */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x0F, /* [ 2336] OBJ_id_cmc_getCert */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x10, /* [ 2344] OBJ_id_cmc_getCRL */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x11, /* [ 2352] OBJ_id_cmc_revokeRequest */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x12, /* [ 2360] OBJ_id_cmc_regInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x13, /* [ 2368] OBJ_id_cmc_responseInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x15, /* [ 2376] OBJ_id_cmc_queryPending */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x16, /* [ 2384] OBJ_id_cmc_popLinkRandom */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x17, /* [ 2392] OBJ_id_cmc_popLinkWitness */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x07,0x18, /* [ 2400] OBJ_id_cmc_confirmCertAcceptance */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x01, /* [ 2408] OBJ_id_on_personalData */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x01, /* [ 2416] OBJ_id_pda_dateOfBirth */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x02, /* [ 2424] OBJ_id_pda_placeOfBirth */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x03, /* [ 2432] OBJ_id_pda_gender */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x04, /* [ 2440] OBJ_id_pda_countryOfCitizenship */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x09,0x05, /* [ 2448] OBJ_id_pda_countryOfResidence */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x01, /* [ 2456] OBJ_id_aca_authenticationInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x02, /* [ 2464] OBJ_id_aca_accessIdentity */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x03, /* [ 2472] OBJ_id_aca_chargingIdentity */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x04, /* [ 2480] OBJ_id_aca_group */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x05, /* [ 2488] OBJ_id_aca_role */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0B,0x01, /* [ 2496] OBJ_id_qcs_pkixQCSyntax_v1 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0C,0x01, /* [ 2504] OBJ_id_cct_crs */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0C,0x02, /* [ 2512] OBJ_id_cct_PKIData */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0C,0x03, /* [ 2520] OBJ_id_cct_PKIResponse */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x03, /* [ 2528] OBJ_ad_timeStamping */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x04, /* [ 2536] OBJ_ad_dvcs */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x01, /* [ 2544] OBJ_id_pkix_OCSP_basic */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x02, /* [ 2553] OBJ_id_pkix_OCSP_Nonce */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x03, /* [ 2562] OBJ_id_pkix_OCSP_CrlID */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x04, /* [ 2571] OBJ_id_pkix_OCSP_acceptableResponses */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x05, /* [ 2580] OBJ_id_pkix_OCSP_noCheck */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x06, /* [ 2589] OBJ_id_pkix_OCSP_archiveCutoff */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x07, /* [ 2598] OBJ_id_pkix_OCSP_serviceLocator */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x08, /* [ 2607] OBJ_id_pkix_OCSP_extendedStatus */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x09, /* [ 2616] OBJ_id_pkix_OCSP_valid */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x0A, /* [ 2625] OBJ_id_pkix_OCSP_path */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x01,0x0B, /* [ 2634] OBJ_id_pkix_OCSP_trustRoot */ + 0x2B,0x0E,0x03,0x02, /* [ 2643] OBJ_algorithm */ + 0x2B,0x0E,0x03,0x02,0x0B, /* [ 2647] OBJ_rsaSignature */ + 0x55,0x08, /* [ 2652] OBJ_X500algorithms */ + 0x2B, /* [ 2654] OBJ_org */ + 0x2B,0x06, /* [ 2655] OBJ_dod */ + 0x2B,0x06,0x01, /* [ 2657] OBJ_iana */ + 0x2B,0x06,0x01,0x01, /* [ 2660] OBJ_Directory */ + 0x2B,0x06,0x01,0x02, /* [ 2664] OBJ_Management */ + 0x2B,0x06,0x01,0x03, /* [ 2668] OBJ_Experimental */ + 0x2B,0x06,0x01,0x04, /* [ 2672] OBJ_Private */ + 0x2B,0x06,0x01,0x05, /* [ 2676] OBJ_Security */ + 0x2B,0x06,0x01,0x06, /* [ 2680] OBJ_SNMPv2 */ + 0x2B,0x06,0x01,0x07, /* [ 2684] OBJ_Mail */ + 0x2B,0x06,0x01,0x04,0x01, /* [ 2688] OBJ_Enterprises */ + 0x2B,0x06,0x01,0x04,0x01,0x8B,0x3A,0x82,0x58, /* [ 2693] OBJ_dcObject */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x19, /* [ 2702] OBJ_domainComponent */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x0D, /* [ 2712] OBJ_Domain */ + 0x55,0x01,0x05, /* [ 2722] OBJ_selected_attribute_types */ + 0x55,0x01,0x05,0x37, /* [ 2725] OBJ_clearance */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x0A, /* [ 2729] OBJ_ac_proxying */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x0B, /* [ 2737] OBJ_sinfo_access */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0A,0x06, /* [ 2745] OBJ_id_aca_encAttrs */ + 0x55,0x04,0x48, /* [ 2753] OBJ_role */ + 0x55,0x1D,0x24, /* [ 2756] OBJ_policy_constraints */ + 0x55,0x1D,0x37, /* [ 2759] OBJ_target_information */ + 0x55,0x1D,0x38, /* [ 2762] OBJ_no_rev_avail */ + 0x2A,0x86,0x48,0xCE,0x3D, /* [ 2765] OBJ_ansi_X9_62 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x01, /* [ 2770] OBJ_X9_62_prime_field */ + 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02, /* [ 2777] OBJ_X9_62_characteristic_two_field */ + 0x2A,0x86,0x48,0xCE,0x3D,0x02,0x01, /* [ 2784] OBJ_X9_62_id_ecPublicKey */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x01, /* [ 2791] OBJ_X9_62_prime192v1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x02, /* [ 2799] OBJ_X9_62_prime192v2 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x03, /* [ 2807] OBJ_X9_62_prime192v3 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x04, /* [ 2815] OBJ_X9_62_prime239v1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x05, /* [ 2823] OBJ_X9_62_prime239v2 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x06, /* [ 2831] OBJ_X9_62_prime239v3 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x01,0x07, /* [ 2839] OBJ_X9_62_prime256v1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x01, /* [ 2847] OBJ_ecdsa_with_SHA1 */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x11,0x01, /* [ 2854] OBJ_ms_csp_name */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x01, /* [ 2863] OBJ_aes_128_ecb */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x02, /* [ 2872] OBJ_aes_128_cbc */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x03, /* [ 2881] OBJ_aes_128_ofb128 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x04, /* [ 2890] OBJ_aes_128_cfb128 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x15, /* [ 2899] OBJ_aes_192_ecb */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x16, /* [ 2908] OBJ_aes_192_cbc */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x17, /* [ 2917] OBJ_aes_192_ofb128 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x18, /* [ 2926] OBJ_aes_192_cfb128 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x29, /* [ 2935] OBJ_aes_256_ecb */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2A, /* [ 2944] OBJ_aes_256_cbc */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2B, /* [ 2953] OBJ_aes_256_ofb128 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2C, /* [ 2962] OBJ_aes_256_cfb128 */ + 0x55,0x1D,0x17, /* [ 2971] OBJ_hold_instruction_code */ + 0x2A,0x86,0x48,0xCE,0x38,0x02,0x01, /* [ 2974] OBJ_hold_instruction_none */ + 0x2A,0x86,0x48,0xCE,0x38,0x02,0x02, /* [ 2981] OBJ_hold_instruction_call_issuer */ + 0x2A,0x86,0x48,0xCE,0x38,0x02,0x03, /* [ 2988] OBJ_hold_instruction_reject */ + 0x09, /* [ 2995] OBJ_data */ + 0x09,0x92,0x26, /* [ 2996] OBJ_pss */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C, /* [ 2999] OBJ_ucl */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64, /* [ 3006] OBJ_pilot */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01, /* [ 3014] OBJ_pilotAttributeType */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x03, /* [ 3023] OBJ_pilotAttributeSyntax */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04, /* [ 3032] OBJ_pilotObjectClass */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x0A, /* [ 3041] OBJ_pilotGroups */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x03,0x04, /* [ 3050] OBJ_iA5StringSyntax */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x03,0x05, /* [ 3060] OBJ_caseIgnoreIA5StringSyntax */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x03, /* [ 3070] OBJ_pilotObject */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x04, /* [ 3080] OBJ_pilotPerson */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x05, /* [ 3090] OBJ_account */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x06, /* [ 3100] OBJ_document */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x07, /* [ 3110] OBJ_room */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x09, /* [ 3120] OBJ_documentSeries */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x0E, /* [ 3130] OBJ_rFC822localPart */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x0F, /* [ 3140] OBJ_dNSDomain */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x11, /* [ 3150] OBJ_domainRelatedObject */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x12, /* [ 3160] OBJ_friendlyCountry */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x13, /* [ 3170] OBJ_simpleSecurityObject */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x14, /* [ 3180] OBJ_pilotOrganization */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x15, /* [ 3190] OBJ_pilotDSA */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x04,0x16, /* [ 3200] OBJ_qualityLabelledData */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x01, /* [ 3210] OBJ_userId */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x02, /* [ 3220] OBJ_textEncodedORAddress */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x03, /* [ 3230] OBJ_rfc822Mailbox */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x04, /* [ 3240] OBJ_info */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x05, /* [ 3250] OBJ_favouriteDrink */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x06, /* [ 3260] OBJ_roomNumber */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x07, /* [ 3270] OBJ_photo */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x08, /* [ 3280] OBJ_userClass */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x09, /* [ 3290] OBJ_host */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0A, /* [ 3300] OBJ_manager */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0B, /* [ 3310] OBJ_documentIdentifier */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0C, /* [ 3320] OBJ_documentTitle */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0D, /* [ 3330] OBJ_documentVersion */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0E, /* [ 3340] OBJ_documentAuthor */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x0F, /* [ 3350] OBJ_documentLocation */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x14, /* [ 3360] OBJ_homeTelephoneNumber */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x15, /* [ 3370] OBJ_secretary */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x16, /* [ 3380] OBJ_otherMailbox */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x17, /* [ 3390] OBJ_lastModifiedTime */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x18, /* [ 3400] OBJ_lastModifiedBy */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1A, /* [ 3410] OBJ_aRecord */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1B, /* [ 3420] OBJ_pilotAttributeType27 */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1C, /* [ 3430] OBJ_mXRecord */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1D, /* [ 3440] OBJ_nSRecord */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1E, /* [ 3450] OBJ_sOARecord */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x1F, /* [ 3460] OBJ_cNAMERecord */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x25, /* [ 3470] OBJ_associatedDomain */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x26, /* [ 3480] OBJ_associatedName */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x27, /* [ 3490] OBJ_homePostalAddress */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x28, /* [ 3500] OBJ_personalTitle */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x29, /* [ 3510] OBJ_mobileTelephoneNumber */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2A, /* [ 3520] OBJ_pagerTelephoneNumber */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2B, /* [ 3530] OBJ_friendlyCountryName */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2D, /* [ 3540] OBJ_organizationalStatus */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2E, /* [ 3550] OBJ_janetMailbox */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x2F, /* [ 3560] OBJ_mailPreferenceOption */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x30, /* [ 3570] OBJ_buildingName */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x31, /* [ 3580] OBJ_dSAQuality */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x32, /* [ 3590] OBJ_singleLevelQuality */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x33, /* [ 3600] OBJ_subtreeMinimumQuality */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x34, /* [ 3610] OBJ_subtreeMaximumQuality */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x35, /* [ 3620] OBJ_personalSignature */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x36, /* [ 3630] OBJ_dITRedirect */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x37, /* [ 3640] OBJ_audio */ + 0x09,0x92,0x26,0x89,0x93,0xF2,0x2C,0x64,0x01,0x38, /* [ 3650] OBJ_documentPublisher */ + 0x55,0x04,0x2D, /* [ 3660] OBJ_x500UniqueIdentifier */ + 0x2B,0x06,0x01,0x07,0x01, /* [ 3663] OBJ_mime_mhs */ + 0x2B,0x06,0x01,0x07,0x01,0x01, /* [ 3668] OBJ_mime_mhs_headings */ + 0x2B,0x06,0x01,0x07,0x01,0x02, /* [ 3674] OBJ_mime_mhs_bodies */ + 0x2B,0x06,0x01,0x07,0x01,0x01,0x01, /* [ 3680] OBJ_id_hex_partial_message */ + 0x2B,0x06,0x01,0x07,0x01,0x01,0x02, /* [ 3687] OBJ_id_hex_multipart_message */ + 0x55,0x04,0x2C, /* [ 3694] OBJ_generationQualifier */ + 0x55,0x04,0x41, /* [ 3697] OBJ_pseudonym */ + 0x67,0x2A, /* [ 3700] OBJ_id_set */ + 0x67,0x2A,0x00, /* [ 3702] OBJ_set_ctype */ + 0x67,0x2A,0x01, /* [ 3705] OBJ_set_msgExt */ + 0x67,0x2A,0x03, /* [ 3708] OBJ_set_attr */ + 0x67,0x2A,0x05, /* [ 3711] OBJ_set_policy */ + 0x67,0x2A,0x07, /* [ 3714] OBJ_set_certExt */ + 0x67,0x2A,0x08, /* [ 3717] OBJ_set_brand */ + 0x67,0x2A,0x00,0x00, /* [ 3720] OBJ_setct_PANData */ + 0x67,0x2A,0x00,0x01, /* [ 3724] OBJ_setct_PANToken */ + 0x67,0x2A,0x00,0x02, /* [ 3728] OBJ_setct_PANOnly */ + 0x67,0x2A,0x00,0x03, /* [ 3732] OBJ_setct_OIData */ + 0x67,0x2A,0x00,0x04, /* [ 3736] OBJ_setct_PI */ + 0x67,0x2A,0x00,0x05, /* [ 3740] OBJ_setct_PIData */ + 0x67,0x2A,0x00,0x06, /* [ 3744] OBJ_setct_PIDataUnsigned */ + 0x67,0x2A,0x00,0x07, /* [ 3748] OBJ_setct_HODInput */ + 0x67,0x2A,0x00,0x08, /* [ 3752] OBJ_setct_AuthResBaggage */ + 0x67,0x2A,0x00,0x09, /* [ 3756] OBJ_setct_AuthRevReqBaggage */ + 0x67,0x2A,0x00,0x0A, /* [ 3760] OBJ_setct_AuthRevResBaggage */ + 0x67,0x2A,0x00,0x0B, /* [ 3764] OBJ_setct_CapTokenSeq */ + 0x67,0x2A,0x00,0x0C, /* [ 3768] OBJ_setct_PInitResData */ + 0x67,0x2A,0x00,0x0D, /* [ 3772] OBJ_setct_PI_TBS */ + 0x67,0x2A,0x00,0x0E, /* [ 3776] OBJ_setct_PResData */ + 0x67,0x2A,0x00,0x10, /* [ 3780] OBJ_setct_AuthReqTBS */ + 0x67,0x2A,0x00,0x11, /* [ 3784] OBJ_setct_AuthResTBS */ + 0x67,0x2A,0x00,0x12, /* [ 3788] OBJ_setct_AuthResTBSX */ + 0x67,0x2A,0x00,0x13, /* [ 3792] OBJ_setct_AuthTokenTBS */ + 0x67,0x2A,0x00,0x14, /* [ 3796] OBJ_setct_CapTokenData */ + 0x67,0x2A,0x00,0x15, /* [ 3800] OBJ_setct_CapTokenTBS */ + 0x67,0x2A,0x00,0x16, /* [ 3804] OBJ_setct_AcqCardCodeMsg */ + 0x67,0x2A,0x00,0x17, /* [ 3808] OBJ_setct_AuthRevReqTBS */ + 0x67,0x2A,0x00,0x18, /* [ 3812] OBJ_setct_AuthRevResData */ + 0x67,0x2A,0x00,0x19, /* [ 3816] OBJ_setct_AuthRevResTBS */ + 0x67,0x2A,0x00,0x1A, /* [ 3820] OBJ_setct_CapReqTBS */ + 0x67,0x2A,0x00,0x1B, /* [ 3824] OBJ_setct_CapReqTBSX */ + 0x67,0x2A,0x00,0x1C, /* [ 3828] OBJ_setct_CapResData */ + 0x67,0x2A,0x00,0x1D, /* [ 3832] OBJ_setct_CapRevReqTBS */ + 0x67,0x2A,0x00,0x1E, /* [ 3836] OBJ_setct_CapRevReqTBSX */ + 0x67,0x2A,0x00,0x1F, /* [ 3840] OBJ_setct_CapRevResData */ + 0x67,0x2A,0x00,0x20, /* [ 3844] OBJ_setct_CredReqTBS */ + 0x67,0x2A,0x00,0x21, /* [ 3848] OBJ_setct_CredReqTBSX */ + 0x67,0x2A,0x00,0x22, /* [ 3852] OBJ_setct_CredResData */ + 0x67,0x2A,0x00,0x23, /* [ 3856] OBJ_setct_CredRevReqTBS */ + 0x67,0x2A,0x00,0x24, /* [ 3860] OBJ_setct_CredRevReqTBSX */ + 0x67,0x2A,0x00,0x25, /* [ 3864] OBJ_setct_CredRevResData */ + 0x67,0x2A,0x00,0x26, /* [ 3868] OBJ_setct_PCertReqData */ + 0x67,0x2A,0x00,0x27, /* [ 3872] OBJ_setct_PCertResTBS */ + 0x67,0x2A,0x00,0x28, /* [ 3876] OBJ_setct_BatchAdminReqData */ + 0x67,0x2A,0x00,0x29, /* [ 3880] OBJ_setct_BatchAdminResData */ + 0x67,0x2A,0x00,0x2A, /* [ 3884] OBJ_setct_CardCInitResTBS */ + 0x67,0x2A,0x00,0x2B, /* [ 3888] OBJ_setct_MeAqCInitResTBS */ + 0x67,0x2A,0x00,0x2C, /* [ 3892] OBJ_setct_RegFormResTBS */ + 0x67,0x2A,0x00,0x2D, /* [ 3896] OBJ_setct_CertReqData */ + 0x67,0x2A,0x00,0x2E, /* [ 3900] OBJ_setct_CertReqTBS */ + 0x67,0x2A,0x00,0x2F, /* [ 3904] OBJ_setct_CertResData */ + 0x67,0x2A,0x00,0x30, /* [ 3908] OBJ_setct_CertInqReqTBS */ + 0x67,0x2A,0x00,0x31, /* [ 3912] OBJ_setct_ErrorTBS */ + 0x67,0x2A,0x00,0x32, /* [ 3916] OBJ_setct_PIDualSignedTBE */ + 0x67,0x2A,0x00,0x33, /* [ 3920] OBJ_setct_PIUnsignedTBE */ + 0x67,0x2A,0x00,0x34, /* [ 3924] OBJ_setct_AuthReqTBE */ + 0x67,0x2A,0x00,0x35, /* [ 3928] OBJ_setct_AuthResTBE */ + 0x67,0x2A,0x00,0x36, /* [ 3932] OBJ_setct_AuthResTBEX */ + 0x67,0x2A,0x00,0x37, /* [ 3936] OBJ_setct_AuthTokenTBE */ + 0x67,0x2A,0x00,0x38, /* [ 3940] OBJ_setct_CapTokenTBE */ + 0x67,0x2A,0x00,0x39, /* [ 3944] OBJ_setct_CapTokenTBEX */ + 0x67,0x2A,0x00,0x3A, /* [ 3948] OBJ_setct_AcqCardCodeMsgTBE */ + 0x67,0x2A,0x00,0x3B, /* [ 3952] OBJ_setct_AuthRevReqTBE */ + 0x67,0x2A,0x00,0x3C, /* [ 3956] OBJ_setct_AuthRevResTBE */ + 0x67,0x2A,0x00,0x3D, /* [ 3960] OBJ_setct_AuthRevResTBEB */ + 0x67,0x2A,0x00,0x3E, /* [ 3964] OBJ_setct_CapReqTBE */ + 0x67,0x2A,0x00,0x3F, /* [ 3968] OBJ_setct_CapReqTBEX */ + 0x67,0x2A,0x00,0x40, /* [ 3972] OBJ_setct_CapResTBE */ + 0x67,0x2A,0x00,0x41, /* [ 3976] OBJ_setct_CapRevReqTBE */ + 0x67,0x2A,0x00,0x42, /* [ 3980] OBJ_setct_CapRevReqTBEX */ + 0x67,0x2A,0x00,0x43, /* [ 3984] OBJ_setct_CapRevResTBE */ + 0x67,0x2A,0x00,0x44, /* [ 3988] OBJ_setct_CredReqTBE */ + 0x67,0x2A,0x00,0x45, /* [ 3992] OBJ_setct_CredReqTBEX */ + 0x67,0x2A,0x00,0x46, /* [ 3996] OBJ_setct_CredResTBE */ + 0x67,0x2A,0x00,0x47, /* [ 4000] OBJ_setct_CredRevReqTBE */ + 0x67,0x2A,0x00,0x48, /* [ 4004] OBJ_setct_CredRevReqTBEX */ + 0x67,0x2A,0x00,0x49, /* [ 4008] OBJ_setct_CredRevResTBE */ + 0x67,0x2A,0x00,0x4A, /* [ 4012] OBJ_setct_BatchAdminReqTBE */ + 0x67,0x2A,0x00,0x4B, /* [ 4016] OBJ_setct_BatchAdminResTBE */ + 0x67,0x2A,0x00,0x4C, /* [ 4020] OBJ_setct_RegFormReqTBE */ + 0x67,0x2A,0x00,0x4D, /* [ 4024] OBJ_setct_CertReqTBE */ + 0x67,0x2A,0x00,0x4E, /* [ 4028] OBJ_setct_CertReqTBEX */ + 0x67,0x2A,0x00,0x4F, /* [ 4032] OBJ_setct_CertResTBE */ + 0x67,0x2A,0x00,0x50, /* [ 4036] OBJ_setct_CRLNotificationTBS */ + 0x67,0x2A,0x00,0x51, /* [ 4040] OBJ_setct_CRLNotificationResTBS */ + 0x67,0x2A,0x00,0x52, /* [ 4044] OBJ_setct_BCIDistributionTBS */ + 0x67,0x2A,0x01,0x01, /* [ 4048] OBJ_setext_genCrypt */ + 0x67,0x2A,0x01,0x03, /* [ 4052] OBJ_setext_miAuth */ + 0x67,0x2A,0x01,0x04, /* [ 4056] OBJ_setext_pinSecure */ + 0x67,0x2A,0x01,0x05, /* [ 4060] OBJ_setext_pinAny */ + 0x67,0x2A,0x01,0x07, /* [ 4064] OBJ_setext_track2 */ + 0x67,0x2A,0x01,0x08, /* [ 4068] OBJ_setext_cv */ + 0x67,0x2A,0x05,0x00, /* [ 4072] OBJ_set_policy_root */ + 0x67,0x2A,0x07,0x00, /* [ 4076] OBJ_setCext_hashedRoot */ + 0x67,0x2A,0x07,0x01, /* [ 4080] OBJ_setCext_certType */ + 0x67,0x2A,0x07,0x02, /* [ 4084] OBJ_setCext_merchData */ + 0x67,0x2A,0x07,0x03, /* [ 4088] OBJ_setCext_cCertRequired */ + 0x67,0x2A,0x07,0x04, /* [ 4092] OBJ_setCext_tunneling */ + 0x67,0x2A,0x07,0x05, /* [ 4096] OBJ_setCext_setExt */ + 0x67,0x2A,0x07,0x06, /* [ 4100] OBJ_setCext_setQualf */ + 0x67,0x2A,0x07,0x07, /* [ 4104] OBJ_setCext_PGWYcapabilities */ + 0x67,0x2A,0x07,0x08, /* [ 4108] OBJ_setCext_TokenIdentifier */ + 0x67,0x2A,0x07,0x09, /* [ 4112] OBJ_setCext_Track2Data */ + 0x67,0x2A,0x07,0x0A, /* [ 4116] OBJ_setCext_TokenType */ + 0x67,0x2A,0x07,0x0B, /* [ 4120] OBJ_setCext_IssuerCapabilities */ + 0x67,0x2A,0x03,0x00, /* [ 4124] OBJ_setAttr_Cert */ + 0x67,0x2A,0x03,0x01, /* [ 4128] OBJ_setAttr_PGWYcap */ + 0x67,0x2A,0x03,0x02, /* [ 4132] OBJ_setAttr_TokenType */ + 0x67,0x2A,0x03,0x03, /* [ 4136] OBJ_setAttr_IssCap */ + 0x67,0x2A,0x03,0x00,0x00, /* [ 4140] OBJ_set_rootKeyThumb */ + 0x67,0x2A,0x03,0x00,0x01, /* [ 4145] OBJ_set_addPolicy */ + 0x67,0x2A,0x03,0x02,0x01, /* [ 4150] OBJ_setAttr_Token_EMV */ + 0x67,0x2A,0x03,0x02,0x02, /* [ 4155] OBJ_setAttr_Token_B0Prime */ + 0x67,0x2A,0x03,0x03,0x03, /* [ 4160] OBJ_setAttr_IssCap_CVM */ + 0x67,0x2A,0x03,0x03,0x04, /* [ 4165] OBJ_setAttr_IssCap_T2 */ + 0x67,0x2A,0x03,0x03,0x05, /* [ 4170] OBJ_setAttr_IssCap_Sig */ + 0x67,0x2A,0x03,0x03,0x03,0x01, /* [ 4175] OBJ_setAttr_GenCryptgrm */ + 0x67,0x2A,0x03,0x03,0x04,0x01, /* [ 4181] OBJ_setAttr_T2Enc */ + 0x67,0x2A,0x03,0x03,0x04,0x02, /* [ 4187] OBJ_setAttr_T2cleartxt */ + 0x67,0x2A,0x03,0x03,0x05,0x01, /* [ 4193] OBJ_setAttr_TokICCsig */ + 0x67,0x2A,0x03,0x03,0x05,0x02, /* [ 4199] OBJ_setAttr_SecDevSig */ + 0x67,0x2A,0x08,0x01, /* [ 4205] OBJ_set_brand_IATA_ATA */ + 0x67,0x2A,0x08,0x1E, /* [ 4209] OBJ_set_brand_Diners */ + 0x67,0x2A,0x08,0x22, /* [ 4213] OBJ_set_brand_AmericanExpress */ + 0x67,0x2A,0x08,0x23, /* [ 4217] OBJ_set_brand_JCB */ + 0x67,0x2A,0x08,0x04, /* [ 4221] OBJ_set_brand_Visa */ + 0x67,0x2A,0x08,0x05, /* [ 4225] OBJ_set_brand_MasterCard */ + 0x67,0x2A,0x08,0xAE,0x7B, /* [ 4229] OBJ_set_brand_Novus */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x03,0x0A, /* [ 4234] OBJ_des_cdmf */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x06, /* [ 4242] OBJ_rsaOAEPEncryptionSET */ + 0x67, /* [ 4251] OBJ_international_organizations */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x14,0x02,0x02, /* [ 4252] OBJ_ms_smartcard_login */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x14,0x02,0x03, /* [ 4262] OBJ_ms_upn */ + 0x55,0x04,0x09, /* [ 4272] OBJ_streetAddress */ + 0x55,0x04,0x11, /* [ 4275] OBJ_postalCode */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x15, /* [ 4278] OBJ_id_ppl */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x0E, /* [ 4285] OBJ_proxyCertInfo */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x15,0x00, /* [ 4293] OBJ_id_ppl_anyLanguage */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x15,0x01, /* [ 4301] OBJ_id_ppl_inheritAll */ + 0x55,0x1D,0x1E, /* [ 4309] OBJ_name_constraints */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x15,0x02, /* [ 4312] OBJ_Independent */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0B, /* [ 4320] OBJ_sha256WithRSAEncryption */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0C, /* [ 4329] OBJ_sha384WithRSAEncryption */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0D, /* [ 4338] OBJ_sha512WithRSAEncryption */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0E, /* [ 4347] OBJ_sha224WithRSAEncryption */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x01, /* [ 4356] OBJ_sha256 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x02, /* [ 4365] OBJ_sha384 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x03, /* [ 4374] OBJ_sha512 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x04, /* [ 4383] OBJ_sha224 */ + 0x2B, /* [ 4392] OBJ_identified_organization */ + 0x2B,0x81,0x04, /* [ 4393] OBJ_certicom_arc */ + 0x67,0x2B, /* [ 4396] OBJ_wap */ + 0x67,0x2B,0x01, /* [ 4398] OBJ_wap_wsg */ + 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02,0x03, /* [ 4401] OBJ_X9_62_id_characteristic_two_basis */ + 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02,0x03,0x01, /* [ 4409] OBJ_X9_62_onBasis */ + 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02,0x03,0x02, /* [ 4418] OBJ_X9_62_tpBasis */ + 0x2A,0x86,0x48,0xCE,0x3D,0x01,0x02,0x03,0x03, /* [ 4427] OBJ_X9_62_ppBasis */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x01, /* [ 4436] OBJ_X9_62_c2pnb163v1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x02, /* [ 4444] OBJ_X9_62_c2pnb163v2 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x03, /* [ 4452] OBJ_X9_62_c2pnb163v3 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x04, /* [ 4460] OBJ_X9_62_c2pnb176v1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x05, /* [ 4468] OBJ_X9_62_c2tnb191v1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x06, /* [ 4476] OBJ_X9_62_c2tnb191v2 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x07, /* [ 4484] OBJ_X9_62_c2tnb191v3 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x08, /* [ 4492] OBJ_X9_62_c2onb191v4 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x09, /* [ 4500] OBJ_X9_62_c2onb191v5 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0A, /* [ 4508] OBJ_X9_62_c2pnb208w1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0B, /* [ 4516] OBJ_X9_62_c2tnb239v1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0C, /* [ 4524] OBJ_X9_62_c2tnb239v2 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0D, /* [ 4532] OBJ_X9_62_c2tnb239v3 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0E, /* [ 4540] OBJ_X9_62_c2onb239v4 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x0F, /* [ 4548] OBJ_X9_62_c2onb239v5 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x10, /* [ 4556] OBJ_X9_62_c2pnb272w1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x11, /* [ 4564] OBJ_X9_62_c2pnb304w1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x12, /* [ 4572] OBJ_X9_62_c2tnb359v1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x13, /* [ 4580] OBJ_X9_62_c2pnb368w1 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x03,0x00,0x14, /* [ 4588] OBJ_X9_62_c2tnb431r1 */ + 0x2B,0x81,0x04,0x00,0x06, /* [ 4596] OBJ_secp112r1 */ + 0x2B,0x81,0x04,0x00,0x07, /* [ 4601] OBJ_secp112r2 */ + 0x2B,0x81,0x04,0x00,0x1C, /* [ 4606] OBJ_secp128r1 */ + 0x2B,0x81,0x04,0x00,0x1D, /* [ 4611] OBJ_secp128r2 */ + 0x2B,0x81,0x04,0x00,0x09, /* [ 4616] OBJ_secp160k1 */ + 0x2B,0x81,0x04,0x00,0x08, /* [ 4621] OBJ_secp160r1 */ + 0x2B,0x81,0x04,0x00,0x1E, /* [ 4626] OBJ_secp160r2 */ + 0x2B,0x81,0x04,0x00,0x1F, /* [ 4631] OBJ_secp192k1 */ + 0x2B,0x81,0x04,0x00,0x20, /* [ 4636] OBJ_secp224k1 */ + 0x2B,0x81,0x04,0x00,0x21, /* [ 4641] OBJ_secp224r1 */ + 0x2B,0x81,0x04,0x00,0x0A, /* [ 4646] OBJ_secp256k1 */ + 0x2B,0x81,0x04,0x00,0x22, /* [ 4651] OBJ_secp384r1 */ + 0x2B,0x81,0x04,0x00,0x23, /* [ 4656] OBJ_secp521r1 */ + 0x2B,0x81,0x04,0x00,0x04, /* [ 4661] OBJ_sect113r1 */ + 0x2B,0x81,0x04,0x00,0x05, /* [ 4666] OBJ_sect113r2 */ + 0x2B,0x81,0x04,0x00,0x16, /* [ 4671] OBJ_sect131r1 */ + 0x2B,0x81,0x04,0x00,0x17, /* [ 4676] OBJ_sect131r2 */ + 0x2B,0x81,0x04,0x00,0x01, /* [ 4681] OBJ_sect163k1 */ + 0x2B,0x81,0x04,0x00,0x02, /* [ 4686] OBJ_sect163r1 */ + 0x2B,0x81,0x04,0x00,0x0F, /* [ 4691] OBJ_sect163r2 */ + 0x2B,0x81,0x04,0x00,0x18, /* [ 4696] OBJ_sect193r1 */ + 0x2B,0x81,0x04,0x00,0x19, /* [ 4701] OBJ_sect193r2 */ + 0x2B,0x81,0x04,0x00,0x1A, /* [ 4706] OBJ_sect233k1 */ + 0x2B,0x81,0x04,0x00,0x1B, /* [ 4711] OBJ_sect233r1 */ + 0x2B,0x81,0x04,0x00,0x03, /* [ 4716] OBJ_sect239k1 */ + 0x2B,0x81,0x04,0x00,0x10, /* [ 4721] OBJ_sect283k1 */ + 0x2B,0x81,0x04,0x00,0x11, /* [ 4726] OBJ_sect283r1 */ + 0x2B,0x81,0x04,0x00,0x24, /* [ 4731] OBJ_sect409k1 */ + 0x2B,0x81,0x04,0x00,0x25, /* [ 4736] OBJ_sect409r1 */ + 0x2B,0x81,0x04,0x00,0x26, /* [ 4741] OBJ_sect571k1 */ + 0x2B,0x81,0x04,0x00,0x27, /* [ 4746] OBJ_sect571r1 */ + 0x67,0x2B,0x01,0x04,0x01, /* [ 4751] OBJ_wap_wsg_idm_ecid_wtls1 */ + 0x67,0x2B,0x01,0x04,0x03, /* [ 4756] OBJ_wap_wsg_idm_ecid_wtls3 */ + 0x67,0x2B,0x01,0x04,0x04, /* [ 4761] OBJ_wap_wsg_idm_ecid_wtls4 */ + 0x67,0x2B,0x01,0x04,0x05, /* [ 4766] OBJ_wap_wsg_idm_ecid_wtls5 */ + 0x67,0x2B,0x01,0x04,0x06, /* [ 4771] OBJ_wap_wsg_idm_ecid_wtls6 */ + 0x67,0x2B,0x01,0x04,0x07, /* [ 4776] OBJ_wap_wsg_idm_ecid_wtls7 */ + 0x67,0x2B,0x01,0x04,0x08, /* [ 4781] OBJ_wap_wsg_idm_ecid_wtls8 */ + 0x67,0x2B,0x01,0x04,0x09, /* [ 4786] OBJ_wap_wsg_idm_ecid_wtls9 */ + 0x67,0x2B,0x01,0x04,0x0A, /* [ 4791] OBJ_wap_wsg_idm_ecid_wtls10 */ + 0x67,0x2B,0x01,0x04,0x0B, /* [ 4796] OBJ_wap_wsg_idm_ecid_wtls11 */ + 0x67,0x2B,0x01,0x04,0x0C, /* [ 4801] OBJ_wap_wsg_idm_ecid_wtls12 */ + 0x55,0x1D,0x20,0x00, /* [ 4806] OBJ_any_policy */ + 0x55,0x1D,0x21, /* [ 4810] OBJ_policy_mappings */ + 0x55,0x1D,0x36, /* [ 4813] OBJ_inhibit_any_policy */ + 0x55,0x1D,0x09, /* [ 4816] OBJ_subject_directory_attributes */ + 0x55,0x1D,0x1C, /* [ 4819] OBJ_issuing_distribution_point */ + 0x55,0x1D,0x1D, /* [ 4822] OBJ_certificate_issuer */ + 0x2B,0x06,0x01,0x05,0x05,0x08,0x01,0x01, /* [ 4825] OBJ_hmac_md5 */ + 0x2B,0x06,0x01,0x05,0x05,0x08,0x01,0x02, /* [ 4833] OBJ_hmac_sha1 */ + 0x2A,0x86,0x48,0x86,0xF6,0x7D,0x07,0x42,0x0D, /* [ 4841] OBJ_id_PasswordBasedMAC */ + 0x2A,0x86,0x48,0x86,0xF6,0x7D,0x07,0x42,0x1E, /* [ 4850] OBJ_id_DHBasedMac */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x10, /* [ 4859] OBJ_id_it_suppLangTags */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x05, /* [ 4867] OBJ_caRepository */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x09, /* [ 4875] OBJ_id_smime_ct_compressedData */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x1B, /* [ 4886] OBJ_id_ct_asciiTextWithCRLF */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x05, /* [ 4897] OBJ_id_aes128_wrap */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x19, /* [ 4906] OBJ_id_aes192_wrap */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2D, /* [ 4915] OBJ_id_aes256_wrap */ + 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x02, /* [ 4924] OBJ_ecdsa_with_Recommended */ + 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03, /* [ 4931] OBJ_ecdsa_with_Specified */ + 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03,0x01, /* [ 4938] OBJ_ecdsa_with_SHA224 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03,0x02, /* [ 4946] OBJ_ecdsa_with_SHA256 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03,0x03, /* [ 4954] OBJ_ecdsa_with_SHA384 */ + 0x2A,0x86,0x48,0xCE,0x3D,0x04,0x03,0x04, /* [ 4962] OBJ_ecdsa_with_SHA512 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x06, /* [ 4970] OBJ_hmacWithMD5 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x08, /* [ 4978] OBJ_hmacWithSHA224 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x09, /* [ 4986] OBJ_hmacWithSHA256 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x0A, /* [ 4994] OBJ_hmacWithSHA384 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x0B, /* [ 5002] OBJ_hmacWithSHA512 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x01, /* [ 5010] OBJ_dsa_with_SHA224 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x02, /* [ 5019] OBJ_dsa_with_SHA256 */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x11,0x02, /* [ 5028] OBJ_LocalKeySet */ + 0x55,0x1D,0x2E, /* [ 5037] OBJ_freshest_crl */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x03, /* [ 5040] OBJ_id_on_permanentIdentifier */ + 0x55,0x04,0x0E, /* [ 5048] OBJ_searchGuide */ + 0x55,0x04,0x0F, /* [ 5051] OBJ_businessCategory */ + 0x55,0x04,0x10, /* [ 5054] OBJ_postalAddress */ + 0x55,0x04,0x12, /* [ 5057] OBJ_postOfficeBox */ + 0x55,0x04,0x13, /* [ 5060] OBJ_physicalDeliveryOfficeName */ + 0x55,0x04,0x14, /* [ 5063] OBJ_telephoneNumber */ + 0x55,0x04,0x15, /* [ 5066] OBJ_telexNumber */ + 0x55,0x04,0x16, /* [ 5069] OBJ_teletexTerminalIdentifier */ + 0x55,0x04,0x17, /* [ 5072] OBJ_facsimileTelephoneNumber */ + 0x55,0x04,0x18, /* [ 5075] OBJ_x121Address */ + 0x55,0x04,0x19, /* [ 5078] OBJ_internationaliSDNNumber */ + 0x55,0x04,0x1A, /* [ 5081] OBJ_registeredAddress */ + 0x55,0x04,0x1B, /* [ 5084] OBJ_destinationIndicator */ + 0x55,0x04,0x1C, /* [ 5087] OBJ_preferredDeliveryMethod */ + 0x55,0x04,0x1D, /* [ 5090] OBJ_presentationAddress */ + 0x55,0x04,0x1E, /* [ 5093] OBJ_supportedApplicationContext */ + 0x55,0x04,0x1F, /* [ 5096] OBJ_member */ + 0x55,0x04,0x20, /* [ 5099] OBJ_owner */ + 0x55,0x04,0x21, /* [ 5102] OBJ_roleOccupant */ + 0x55,0x04,0x22, /* [ 5105] OBJ_seeAlso */ + 0x55,0x04,0x23, /* [ 5108] OBJ_userPassword */ + 0x55,0x04,0x24, /* [ 5111] OBJ_userCertificate */ + 0x55,0x04,0x25, /* [ 5114] OBJ_cACertificate */ + 0x55,0x04,0x26, /* [ 5117] OBJ_authorityRevocationList */ + 0x55,0x04,0x27, /* [ 5120] OBJ_certificateRevocationList */ + 0x55,0x04,0x28, /* [ 5123] OBJ_crossCertificatePair */ + 0x55,0x04,0x2F, /* [ 5126] OBJ_enhancedSearchGuide */ + 0x55,0x04,0x30, /* [ 5129] OBJ_protocolInformation */ + 0x55,0x04,0x31, /* [ 5132] OBJ_distinguishedName */ + 0x55,0x04,0x32, /* [ 5135] OBJ_uniqueMember */ + 0x55,0x04,0x33, /* [ 5138] OBJ_houseIdentifier */ + 0x55,0x04,0x34, /* [ 5141] OBJ_supportedAlgorithms */ + 0x55,0x04,0x35, /* [ 5144] OBJ_deltaRevocationList */ + 0x55,0x04,0x36, /* [ 5147] OBJ_dmdName */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x03,0x09, /* [ 5150] OBJ_id_alg_PWRI_KEK */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x06, /* [ 5161] OBJ_aes_128_gcm */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x07, /* [ 5170] OBJ_aes_128_ccm */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x08, /* [ 5179] OBJ_id_aes128_wrap_pad */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x1A, /* [ 5188] OBJ_aes_192_gcm */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x1B, /* [ 5197] OBJ_aes_192_ccm */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x1C, /* [ 5206] OBJ_id_aes192_wrap_pad */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2E, /* [ 5215] OBJ_aes_256_gcm */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x2F, /* [ 5224] OBJ_aes_256_ccm */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x01,0x30, /* [ 5233] OBJ_id_aes256_wrap_pad */ + 0x55,0x1D,0x25,0x00, /* [ 5242] OBJ_anyExtendedKeyUsage */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x08, /* [ 5246] OBJ_mgf1 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0A, /* [ 5255] OBJ_rsassaPss */ + 0x2B,0x6F,0x02,0x8C,0x53,0x00,0x01,0x01, /* [ 5264] OBJ_aes_128_xts */ + 0x2B,0x6F,0x02,0x8C,0x53,0x00,0x01,0x02, /* [ 5272] OBJ_aes_256_xts */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x07, /* [ 5280] OBJ_rsaesOaep */ + 0x2A,0x86,0x48,0xCE,0x3E,0x02,0x01, /* [ 5289] OBJ_dhpublicnumber */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x01, /* [ 5296] OBJ_brainpoolP160r1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x02, /* [ 5305] OBJ_brainpoolP160t1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x03, /* [ 5314] OBJ_brainpoolP192r1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x04, /* [ 5323] OBJ_brainpoolP192t1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x05, /* [ 5332] OBJ_brainpoolP224r1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x06, /* [ 5341] OBJ_brainpoolP224t1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x07, /* [ 5350] OBJ_brainpoolP256r1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x08, /* [ 5359] OBJ_brainpoolP256t1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x09, /* [ 5368] OBJ_brainpoolP320r1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0A, /* [ 5377] OBJ_brainpoolP320t1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0B, /* [ 5386] OBJ_brainpoolP384r1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0C, /* [ 5395] OBJ_brainpoolP384t1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0D, /* [ 5404] OBJ_brainpoolP512r1 */ + 0x2B,0x24,0x03,0x03,0x02,0x08,0x01,0x01,0x0E, /* [ 5413] OBJ_brainpoolP512t1 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x09, /* [ 5422] OBJ_pSpecified */ + 0x2B,0x81,0x05,0x10,0x86,0x48,0x3F,0x00,0x02, /* [ 5431] OBJ_dhSinglePass_stdDH_sha1kdf_scheme */ + 0x2B,0x81,0x04,0x01,0x0B,0x00, /* [ 5440] OBJ_dhSinglePass_stdDH_sha224kdf_scheme */ + 0x2B,0x81,0x04,0x01,0x0B,0x01, /* [ 5446] OBJ_dhSinglePass_stdDH_sha256kdf_scheme */ + 0x2B,0x81,0x04,0x01,0x0B,0x02, /* [ 5452] OBJ_dhSinglePass_stdDH_sha384kdf_scheme */ + 0x2B,0x81,0x04,0x01,0x0B,0x03, /* [ 5458] OBJ_dhSinglePass_stdDH_sha512kdf_scheme */ + 0x2B,0x81,0x05,0x10,0x86,0x48,0x3F,0x00,0x03, /* [ 5464] OBJ_dhSinglePass_cofactorDH_sha1kdf_scheme */ + 0x2B,0x81,0x04,0x01,0x0E,0x00, /* [ 5473] OBJ_dhSinglePass_cofactorDH_sha224kdf_scheme */ + 0x2B,0x81,0x04,0x01,0x0E,0x01, /* [ 5479] OBJ_dhSinglePass_cofactorDH_sha256kdf_scheme */ + 0x2B,0x81,0x04,0x01,0x0E,0x02, /* [ 5485] OBJ_dhSinglePass_cofactorDH_sha384kdf_scheme */ + 0x2B,0x81,0x04,0x01,0x0E,0x03, /* [ 5491] OBJ_dhSinglePass_cofactorDH_sha512kdf_scheme */ + 0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x02, /* [ 5497] OBJ_ct_precert_scts */ + 0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x03, /* [ 5507] OBJ_ct_precert_poison */ + 0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x04, /* [ 5517] OBJ_ct_precert_signer */ + 0x2B,0x06,0x01,0x04,0x01,0xD6,0x79,0x02,0x04,0x05, /* [ 5527] OBJ_ct_cert_scts */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x01, /* [ 5537] OBJ_jurisdictionLocalityName */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x02, /* [ 5548] OBJ_jurisdictionStateOrProvinceName */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0x37,0x3C,0x02,0x01,0x03, /* [ 5559] OBJ_jurisdictionCountryName */ + 0x2B,0x06,0x01,0x04,0x01,0xDA,0x47,0x04,0x0B, /* [ 5570] OBJ_id_scrypt */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x18, /* [ 5579] OBJ_tlsfeature */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x11, /* [ 5587] OBJ_ipsec_IKE */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x12, /* [ 5595] OBJ_capwapAC */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x13, /* [ 5603] OBJ_capwapWTP */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x15, /* [ 5611] OBJ_sshClient */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x16, /* [ 5619] OBJ_sshServer */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x17, /* [ 5627] OBJ_sendRouter */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x18, /* [ 5635] OBJ_sendProxiedRouter */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x19, /* [ 5643] OBJ_sendOwner */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1A, /* [ 5651] OBJ_sendProxiedOwner */ + 0x2B,0x06,0x01,0x05,0x02,0x03, /* [ 5659] OBJ_id_pkinit */ + 0x2B,0x06,0x01,0x05,0x02,0x03,0x04, /* [ 5665] OBJ_pkInitClientAuth */ + 0x2B,0x06,0x01,0x05,0x02,0x03,0x05, /* [ 5672] OBJ_pkInitKDC */ + 0x2B,0x65,0x6E, /* [ 5679] OBJ_X25519 */ + 0x2B,0x65,0x6F, /* [ 5682] OBJ_X448 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x13, /* [ 5685] OBJ_id_smime_ct_contentCollection */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x17, /* [ 5696] OBJ_id_smime_ct_authEnvelopedData */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x1C, /* [ 5707] OBJ_id_ct_xml */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x02,0x2F, /* [ 5718] OBJ_id_smime_aa_signingCertificateV2 */ + 0x2B,0x65,0x70, /* [ 5729] OBJ_ED25519 */ + 0x2B,0x65,0x71, /* [ 5732] OBJ_ED448 */ + 0x55,0x04,0x61, /* [ 5735] OBJ_organizationIdentifier */ + 0x55,0x04,0x62, /* [ 5738] OBJ_countryCode3c */ + 0x55,0x04,0x63, /* [ 5741] OBJ_countryCode3n */ + 0x55,0x04,0x64, /* [ 5744] OBJ_dnsName */ + 0x2B,0x24,0x08,0x03,0x03, /* [ 5747] OBJ_x509ExtAdmission */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x05, /* [ 5752] OBJ_sha512_224 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x06, /* [ 5761] OBJ_sha512_256 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x07, /* [ 5770] OBJ_sha3_224 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x08, /* [ 5779] OBJ_sha3_256 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x09, /* [ 5788] OBJ_sha3_384 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0A, /* [ 5797] OBJ_sha3_512 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0B, /* [ 5806] OBJ_shake128 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0C, /* [ 5815] OBJ_shake256 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0D, /* [ 5824] OBJ_hmac_sha3_224 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0E, /* [ 5833] OBJ_hmac_sha3_256 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x0F, /* [ 5842] OBJ_hmac_sha3_384 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x10, /* [ 5851] OBJ_hmac_sha3_512 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x03, /* [ 5860] OBJ_dsa_with_SHA384 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x04, /* [ 5869] OBJ_dsa_with_SHA512 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x05, /* [ 5878] OBJ_dsa_with_SHA3_224 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x06, /* [ 5887] OBJ_dsa_with_SHA3_256 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x07, /* [ 5896] OBJ_dsa_with_SHA3_384 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x08, /* [ 5905] OBJ_dsa_with_SHA3_512 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x09, /* [ 5914] OBJ_ecdsa_with_SHA3_224 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0A, /* [ 5923] OBJ_ecdsa_with_SHA3_256 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0B, /* [ 5932] OBJ_ecdsa_with_SHA3_384 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0C, /* [ 5941] OBJ_ecdsa_with_SHA3_512 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0D, /* [ 5950] OBJ_RSA_SHA3_224 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0E, /* [ 5959] OBJ_RSA_SHA3_256 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x0F, /* [ 5968] OBJ_RSA_SHA3_384 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x03,0x10, /* [ 5977] OBJ_RSA_SHA3_512 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1B, /* [ 5986] OBJ_cmcCA */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1C, /* [ 5994] OBJ_cmcRA */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x01, /* [ 6002] OBJ_sm4_ecb */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x02, /* [ 6010] OBJ_sm4_cbc */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x03, /* [ 6018] OBJ_sm4_ofb128 */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x05, /* [ 6026] OBJ_sm4_cfb1 */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x04, /* [ 6034] OBJ_sm4_cfb128 */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x06, /* [ 6042] OBJ_sm4_cfb8 */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x07, /* [ 6050] OBJ_sm4_ctr */ + 0x2A,0x81,0x1C, /* [ 6058] OBJ_ISO_CN */ + 0x2A,0x81,0x1C,0xCF,0x55, /* [ 6061] OBJ_oscca */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01, /* [ 6066] OBJ_sm_scheme */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x83,0x11, /* [ 6072] OBJ_sm3 */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x83,0x78, /* [ 6080] OBJ_sm3WithRSAEncryption */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x0F, /* [ 6088] OBJ_sha512_224WithRSAEncryption */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x01,0x10, /* [ 6097] OBJ_sha512_256WithRSAEncryption */ + 0x2A,0x86,0x24, /* [ 6106] OBJ_ISO_UA */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01, /* [ 6109] OBJ_ua_pki */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01, /* [ 6116] OBJ_dstu4145le */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x01,0x01, /* [ 6127] OBJ_dstu4145be */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x00, /* [ 6140] OBJ_uacurve0 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x01, /* [ 6153] OBJ_uacurve1 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x02, /* [ 6166] OBJ_uacurve2 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x03, /* [ 6179] OBJ_uacurve3 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x04, /* [ 6192] OBJ_uacurve4 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x05, /* [ 6205] OBJ_uacurve5 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x06, /* [ 6218] OBJ_uacurve6 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x07, /* [ 6231] OBJ_uacurve7 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x08, /* [ 6244] OBJ_uacurve8 */ + 0x2A,0x86,0x24,0x02,0x01,0x01,0x01,0x01,0x03,0x01,0x01,0x02,0x09, /* [ 6257] OBJ_uacurve9 */ + 0x2B,0x6F, /* [ 6270] OBJ_ieee */ + 0x2B,0x6F,0x02,0x8C,0x53, /* [ 6272] OBJ_ieee_siswg */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x82,0x2D, /* [ 6277] OBJ_sm2 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x0C, /* [ 6285] OBJ_hmacWithSHA512_224 */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x02,0x0D, /* [ 6293] OBJ_hmacWithSHA512_256 */ + 0x28,0xCC,0x45,0x03,0x04, /* [ 6301] OBJ_gmac */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x13, /* [ 6306] OBJ_kmac128 */ + 0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x14, /* [ 6315] OBJ_kmac256 */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x83,0x75, /* [ 6324] OBJ_SM2_with_SM3 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x09, /* [ 6332] OBJ_id_on_SmtpUTF8Mailbox */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x05, /* [ 6340] OBJ_XmppAddr */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x07, /* [ 6348] OBJ_SRVName */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x08,0x08, /* [ 6356] OBJ_NAIRealm */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1D, /* [ 6364] OBJ_cmcArchive */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1E, /* [ 6372] OBJ_id_kp_bgpsec_router */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x1F, /* [ 6380] OBJ_id_kp_BrandIndicatorforMessageIdentification */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x03,0x20, /* [ 6388] OBJ_cmKGA */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x11, /* [ 6396] OBJ_id_it_caCerts */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x12, /* [ 6404] OBJ_id_it_rootCaKeyUpdate */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x04,0x13, /* [ 6412] OBJ_id_it_certReqTemplate */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x18, /* [ 6420] OBJ_id_ct_routeOriginAuthz */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x1A, /* [ 6431] OBJ_id_ct_rpkiManifest */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x23, /* [ 6442] OBJ_id_ct_rpkiGhostbusters */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x24, /* [ 6453] OBJ_id_ct_resourceTaggedAttest */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0E, /* [ 6464] OBJ_id_cp */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x1C, /* [ 6471] OBJ_sbgp_ipAddrBlockv2 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x01,0x1D, /* [ 6479] OBJ_sbgp_autonomousSysNumv2 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0E,0x02, /* [ 6487] OBJ_ipAddr_asNumber */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x0E,0x03, /* [ 6495] OBJ_ipAddr_asNumberv2 */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x0A, /* [ 6503] OBJ_rpkiManifest */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x0B, /* [ 6511] OBJ_signedObject */ + 0x2B,0x06,0x01,0x05,0x05,0x07,0x30,0x0D, /* [ 6519] OBJ_rpkiNotify */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x2F, /* [ 6527] OBJ_id_ct_geofeedCSVwithCRLF */ + 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x30, /* [ 6538] OBJ_id_ct_signedChecklist */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x81,0x49, /* [ 6549] OBJ_zuc */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x86,0x21, /* [ 6557] OBJ_zuc_128_eea3 */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x08, /* [ 6565] OBJ_sm4_gcm */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x68,0x09, /* [ 6573] OBJ_sm4_ccm */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x86,0x22, /* [ 6581] OBJ_zuc_128_eia3 */ + 0x2B,0x06,0x01,0x04,0x01,0x82,0xDA,0x4B,0x2C, /* [ 6589] OBJ_delegation_usage */ + 0x2A,0x81,0x1C,0xCF,0x55,0x01,0x83,0x11,0x03,0x01, /* [ 6598] OBJ_hmacWithSM3 */ }; -#define NUM_NID 1321 +#define NUM_NID 1258 static const ASN1_OBJECT nid_objs[NUM_NID] = { {"UNDEF", "undefined", NID_undef}, {"rsadsi", "RSA Data Security, Inc.", NID_rsadsi, 6, &so[0]}, {"pkcs", "RSA Data Security, Inc. PKCS", NID_pkcs, 7, &so[6]}, - {"MD2", "md2", NID_md2, 8, &so[13]}, - {"MD5", "md5", NID_md5, 8, &so[21]}, - {"RC4", "rc4", NID_rc4, 8, &so[29]}, - {"rsaEncryption", "rsaEncryption", NID_rsaEncryption, 9, &so[37]}, - {"RSA-MD2", "md2WithRSAEncryption", NID_md2WithRSAEncryption, 9, &so[46]}, - {"RSA-MD5", "md5WithRSAEncryption", NID_md5WithRSAEncryption, 9, &so[55]}, - {"PBE-MD2-DES", "pbeWithMD2AndDES-CBC", NID_pbeWithMD2AndDES_CBC, 9, &so[64]}, - {"PBE-MD5-DES", "pbeWithMD5AndDES-CBC", NID_pbeWithMD5AndDES_CBC, 9, &so[73]}, - {"X500", "directory services (X.500)", NID_X500, 1, &so[82]}, - {"X509", "X509", NID_X509, 2, &so[83]}, - {"CN", "commonName", NID_commonName, 3, &so[85]}, - {"C", "countryName", NID_countryName, 3, &so[88]}, - {"L", "localityName", NID_localityName, 3, &so[91]}, - {"ST", "stateOrProvinceName", NID_stateOrProvinceName, 3, &so[94]}, - {"O", "organizationName", NID_organizationName, 3, &so[97]}, - {"OU", "organizationalUnitName", NID_organizationalUnitName, 3, &so[100]}, - {"RSA", "rsa", NID_rsa, 4, &so[103]}, - {"pkcs7", "pkcs7", NID_pkcs7, 8, &so[107]}, - {"pkcs7-data", "pkcs7-data", NID_pkcs7_data, 9, &so[115]}, - {"pkcs7-signedData", "pkcs7-signedData", NID_pkcs7_signed, 9, &so[124]}, - {"pkcs7-envelopedData", "pkcs7-envelopedData", NID_pkcs7_enveloped, 9, &so[133]}, - {"pkcs7-signedAndEnvelopedData", "pkcs7-signedAndEnvelopedData", NID_pkcs7_signedAndEnveloped, 9, &so[142]}, - {"pkcs7-digestData", "pkcs7-digestData", NID_pkcs7_digest, 9, &so[151]}, - {"pkcs7-encryptedData", "pkcs7-encryptedData", NID_pkcs7_encrypted, 9, &so[160]}, - {"pkcs3", "pkcs3", NID_pkcs3, 8, &so[169]}, - {"dhKeyAgreement", "dhKeyAgreement", NID_dhKeyAgreement, 9, &so[177]}, - {"DES-ECB", "des-ecb", NID_des_ecb, 5, &so[186]}, - {"DES-CFB", "des-cfb", NID_des_cfb64, 5, &so[191]}, - {"DES-CBC", "des-cbc", NID_des_cbc, 5, &so[196]}, - {"DES-EDE", "des-ede", NID_des_ede_ecb, 5, &so[201]}, + { NULL, NULL, NID_undef }, + {"MD5", "md5", NID_md5, 8, &so[13]}, + {"RC4", "rc4", NID_rc4, 8, &so[21]}, + {"rsaEncryption", "rsaEncryption", NID_rsaEncryption, 9, &so[29]}, + { NULL, NULL, NID_undef }, + {"RSA-MD5", "md5WithRSAEncryption", NID_md5WithRSAEncryption, 9, &so[38]}, + { NULL, NULL, NID_undef }, + {"PBE-MD5-DES", "pbeWithMD5AndDES-CBC", NID_pbeWithMD5AndDES_CBC, 9, &so[47]}, + {"X500", "directory services (X.500)", NID_X500, 1, &so[56]}, + {"X509", "X509", NID_X509, 2, &so[57]}, + {"CN", "commonName", NID_commonName, 3, &so[59]}, + {"C", "countryName", NID_countryName, 3, &so[62]}, + {"L", "localityName", NID_localityName, 3, &so[65]}, + {"ST", "stateOrProvinceName", NID_stateOrProvinceName, 3, &so[68]}, + {"O", "organizationName", NID_organizationName, 3, &so[71]}, + {"OU", "organizationalUnitName", NID_organizationalUnitName, 3, &so[74]}, + {"RSA", "rsa", NID_rsa, 4, &so[77]}, + {"pkcs7", "pkcs7", NID_pkcs7, 8, &so[81]}, + {"pkcs7-data", "pkcs7-data", NID_pkcs7_data, 9, &so[89]}, + {"pkcs7-signedData", "pkcs7-signedData", NID_pkcs7_signed, 9, &so[98]}, + {"pkcs7-envelopedData", "pkcs7-envelopedData", NID_pkcs7_enveloped, 9, &so[107]}, + {"pkcs7-signedAndEnvelopedData", "pkcs7-signedAndEnvelopedData", NID_pkcs7_signedAndEnveloped, 9, &so[116]}, + {"pkcs7-digestData", "pkcs7-digestData", NID_pkcs7_digest, 9, &so[125]}, + {"pkcs7-encryptedData", "pkcs7-encryptedData", NID_pkcs7_encrypted, 9, &so[134]}, + {"pkcs3", "pkcs3", NID_pkcs3, 8, &so[143]}, + {"dhKeyAgreement", "dhKeyAgreement", NID_dhKeyAgreement, 9, &so[151]}, + {"DES-ECB", "des-ecb", NID_des_ecb, 5, &so[160]}, + {"DES-CFB", "des-cfb", NID_des_cfb64, 5, &so[165]}, + {"DES-CBC", "des-cbc", NID_des_cbc, 5, &so[170]}, + {"DES-EDE", "des-ede", NID_des_ede_ecb, 5, &so[175]}, {"DES-EDE3", "des-ede3", NID_des_ede3_ecb}, - {"IDEA-CBC", "idea-cbc", NID_idea_cbc, 11, &so[206]}, - {"IDEA-CFB", "idea-cfb", NID_idea_cfb64}, - {"IDEA-ECB", "idea-ecb", NID_idea_ecb}, - {"RC2-CBC", "rc2-cbc", NID_rc2_cbc, 8, &so[217]}, - {"RC2-ECB", "rc2-ecb", NID_rc2_ecb}, - {"RC2-CFB", "rc2-cfb", NID_rc2_cfb64}, - {"RC2-OFB", "rc2-ofb", NID_rc2_ofb64}, - {"SHA", "sha", NID_sha, 5, &so[225]}, - {"RSA-SHA", "shaWithRSAEncryption", NID_shaWithRSAEncryption, 5, &so[230]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"SHA", "sha", NID_sha, 5, &so[180]}, + {"RSA-SHA", "shaWithRSAEncryption", NID_shaWithRSAEncryption, 5, &so[185]}, {"DES-EDE-CBC", "des-ede-cbc", NID_des_ede_cbc}, - {"DES-EDE3-CBC", "des-ede3-cbc", NID_des_ede3_cbc, 8, &so[235]}, - {"DES-OFB", "des-ofb", NID_des_ofb64, 5, &so[243]}, - {"IDEA-OFB", "idea-ofb", NID_idea_ofb64}, - {"pkcs9", "pkcs9", NID_pkcs9, 8, &so[248]}, - {"emailAddress", "emailAddress", NID_pkcs9_emailAddress, 9, &so[256]}, - {"unstructuredName", "unstructuredName", NID_pkcs9_unstructuredName, 9, &so[265]}, - {"contentType", "contentType", NID_pkcs9_contentType, 9, &so[274]}, - {"messageDigest", "messageDigest", NID_pkcs9_messageDigest, 9, &so[283]}, - {"signingTime", "signingTime", NID_pkcs9_signingTime, 9, &so[292]}, - {"countersignature", "countersignature", NID_pkcs9_countersignature, 9, &so[301]}, - {"challengePassword", "challengePassword", NID_pkcs9_challengePassword, 9, &so[310]}, - {"unstructuredAddress", "unstructuredAddress", NID_pkcs9_unstructuredAddress, 9, &so[319]}, - {"extendedCertificateAttributes", "extendedCertificateAttributes", NID_pkcs9_extCertAttributes, 9, &so[328]}, - {"Netscape", "Netscape Communications Corp.", NID_netscape, 7, &so[337]}, - {"nsCertExt", "Netscape Certificate Extension", NID_netscape_cert_extension, 8, &so[344]}, - {"nsDataType", "Netscape Data Type", NID_netscape_data_type, 8, &so[352]}, + {"DES-EDE3-CBC", "des-ede3-cbc", NID_des_ede3_cbc, 8, &so[190]}, + {"DES-OFB", "des-ofb", NID_des_ofb64, 5, &so[198]}, + { NULL, NULL, NID_undef }, + {"pkcs9", "pkcs9", NID_pkcs9, 8, &so[203]}, + {"emailAddress", "emailAddress", NID_pkcs9_emailAddress, 9, &so[211]}, + {"unstructuredName", "unstructuredName", NID_pkcs9_unstructuredName, 9, &so[220]}, + {"contentType", "contentType", NID_pkcs9_contentType, 9, &so[229]}, + {"messageDigest", "messageDigest", NID_pkcs9_messageDigest, 9, &so[238]}, + {"signingTime", "signingTime", NID_pkcs9_signingTime, 9, &so[247]}, + {"countersignature", "countersignature", NID_pkcs9_countersignature, 9, &so[256]}, + {"challengePassword", "challengePassword", NID_pkcs9_challengePassword, 9, &so[265]}, + {"unstructuredAddress", "unstructuredAddress", NID_pkcs9_unstructuredAddress, 9, &so[274]}, + {"extendedCertificateAttributes", "extendedCertificateAttributes", NID_pkcs9_extCertAttributes, 9, &so[283]}, + {"Netscape", "Netscape Communications Corp.", NID_netscape, 7, &so[292]}, + {"nsCertExt", "Netscape Certificate Extension", NID_netscape_cert_extension, 8, &so[299]}, + {"nsDataType", "Netscape Data Type", NID_netscape_data_type, 8, &so[307]}, {"DES-EDE-CFB", "des-ede-cfb", NID_des_ede_cfb64}, {"DES-EDE3-CFB", "des-ede3-cfb", NID_des_ede3_cfb64}, {"DES-EDE-OFB", "des-ede-ofb", NID_des_ede_ofb64}, {"DES-EDE3-OFB", "des-ede3-ofb", NID_des_ede3_ofb64}, - {"SHA1", "sha1", NID_sha1, 5, &so[360]}, - {"RSA-SHA1", "sha1WithRSAEncryption", NID_sha1WithRSAEncryption, 9, &so[365]}, - {"DSA-SHA", "dsaWithSHA", NID_dsaWithSHA, 5, &so[374]}, - {"DSA-old", "dsaEncryption-old", NID_dsa_2, 5, &so[379]}, - {"PBE-SHA1-RC2-64", "pbeWithSHA1AndRC2-CBC", NID_pbeWithSHA1AndRC2_CBC, 9, &so[384]}, - {"PBKDF2", "PBKDF2", NID_id_pbkdf2, 9, &so[393]}, - {"DSA-SHA1-old", "dsaWithSHA1-old", NID_dsaWithSHA1_2, 5, &so[402]}, - {"nsCertType", "Netscape Cert Type", NID_netscape_cert_type, 9, &so[407]}, - {"nsBaseUrl", "Netscape Base Url", NID_netscape_base_url, 9, &so[416]}, - {"nsRevocationUrl", "Netscape Revocation Url", NID_netscape_revocation_url, 9, &so[425]}, - {"nsCaRevocationUrl", "Netscape CA Revocation Url", NID_netscape_ca_revocation_url, 9, &so[434]}, - {"nsRenewalUrl", "Netscape Renewal Url", NID_netscape_renewal_url, 9, &so[443]}, - {"nsCaPolicyUrl", "Netscape CA Policy Url", NID_netscape_ca_policy_url, 9, &so[452]}, - {"nsSslServerName", "Netscape SSL Server Name", NID_netscape_ssl_server_name, 9, &so[461]}, - {"nsComment", "Netscape Comment", NID_netscape_comment, 9, &so[470]}, - {"nsCertSequence", "Netscape Certificate Sequence", NID_netscape_cert_sequence, 9, &so[479]}, + {"SHA1", "sha1", NID_sha1, 5, &so[315]}, + {"RSA-SHA1", "sha1WithRSAEncryption", NID_sha1WithRSAEncryption, 9, &so[320]}, + {"DSA-SHA", "dsaWithSHA", NID_dsaWithSHA, 5, &so[329]}, + {"DSA-old", "dsaEncryption-old", NID_dsa_2, 5, &so[334]}, + { NULL, NULL, NID_undef }, + {"PBKDF2", "PBKDF2", NID_id_pbkdf2, 9, &so[339]}, + {"DSA-SHA1-old", "dsaWithSHA1-old", NID_dsaWithSHA1_2, 5, &so[348]}, + {"nsCertType", "Netscape Cert Type", NID_netscape_cert_type, 9, &so[353]}, + {"nsBaseUrl", "Netscape Base Url", NID_netscape_base_url, 9, &so[362]}, + {"nsRevocationUrl", "Netscape Revocation Url", NID_netscape_revocation_url, 9, &so[371]}, + {"nsCaRevocationUrl", "Netscape CA Revocation Url", NID_netscape_ca_revocation_url, 9, &so[380]}, + {"nsRenewalUrl", "Netscape Renewal Url", NID_netscape_renewal_url, 9, &so[389]}, + {"nsCaPolicyUrl", "Netscape CA Policy Url", NID_netscape_ca_policy_url, 9, &so[398]}, + {"nsSslServerName", "Netscape SSL Server Name", NID_netscape_ssl_server_name, 9, &so[407]}, + {"nsComment", "Netscape Comment", NID_netscape_comment, 9, &so[416]}, + {"nsCertSequence", "Netscape Certificate Sequence", NID_netscape_cert_sequence, 9, &so[425]}, {"DESX-CBC", "desx-cbc", NID_desx_cbc}, - {"id-ce", "id-ce", NID_id_ce, 2, &so[488]}, - {"subjectKeyIdentifier", "X509v3 Subject Key Identifier", NID_subject_key_identifier, 3, &so[490]}, - {"keyUsage", "X509v3 Key Usage", NID_key_usage, 3, &so[493]}, - {"privateKeyUsagePeriod", "X509v3 Private Key Usage Period", NID_private_key_usage_period, 3, &so[496]}, - {"subjectAltName", "X509v3 Subject Alternative Name", NID_subject_alt_name, 3, &so[499]}, - {"issuerAltName", "X509v3 Issuer Alternative Name", NID_issuer_alt_name, 3, &so[502]}, - {"basicConstraints", "X509v3 Basic Constraints", NID_basic_constraints, 3, &so[505]}, - {"crlNumber", "X509v3 CRL Number", NID_crl_number, 3, &so[508]}, - {"certificatePolicies", "X509v3 Certificate Policies", NID_certificate_policies, 3, &so[511]}, - {"authorityKeyIdentifier", "X509v3 Authority Key Identifier", NID_authority_key_identifier, 3, &so[514]}, - {"BF-CBC", "bf-cbc", NID_bf_cbc, 9, &so[517]}, - {"BF-ECB", "bf-ecb", NID_bf_ecb}, - {"BF-CFB", "bf-cfb", NID_bf_cfb64}, - {"BF-OFB", "bf-ofb", NID_bf_ofb64}, - {"MDC2", "mdc2", NID_mdc2, 4, &so[526]}, - {"RSA-MDC2", "mdc2WithRSA", NID_mdc2WithRSA, 4, &so[530]}, + {"id-ce", "id-ce", NID_id_ce, 2, &so[434]}, + {"subjectKeyIdentifier", "X509v3 Subject Key Identifier", NID_subject_key_identifier, 3, &so[436]}, + {"keyUsage", "X509v3 Key Usage", NID_key_usage, 3, &so[439]}, + {"privateKeyUsagePeriod", "X509v3 Private Key Usage Period", NID_private_key_usage_period, 3, &so[442]}, + {"subjectAltName", "X509v3 Subject Alternative Name", NID_subject_alt_name, 3, &so[445]}, + {"issuerAltName", "X509v3 Issuer Alternative Name", NID_issuer_alt_name, 3, &so[448]}, + {"basicConstraints", "X509v3 Basic Constraints", NID_basic_constraints, 3, &so[451]}, + {"crlNumber", "X509v3 CRL Number", NID_crl_number, 3, &so[454]}, + {"certificatePolicies", "X509v3 Certificate Policies", NID_certificate_policies, 3, &so[457]}, + {"authorityKeyIdentifier", "X509v3 Authority Key Identifier", NID_authority_key_identifier, 3, &so[460]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, {"RC4-40", "rc4-40", NID_rc4_40}, - {"RC2-40-CBC", "rc2-40-cbc", NID_rc2_40_cbc}, - {"GN", "givenName", NID_givenName, 3, &so[534]}, - {"SN", "surname", NID_surname, 3, &so[537]}, - {"initials", "initials", NID_initials, 3, &so[540]}, - {"uid", "uniqueIdentifier", NID_uniqueIdentifier, 10, &so[543]}, - {"crlDistributionPoints", "X509v3 CRL Distribution Points", NID_crl_distribution_points, 3, &so[553]}, - {"RSA-NP-MD5", "md5WithRSA", NID_md5WithRSA, 5, &so[556]}, - {"serialNumber", "serialNumber", NID_serialNumber, 3, &so[561]}, - {"title", "title", NID_title, 3, &so[564]}, - {"description", "description", NID_description, 3, &so[567]}, - {"CAST5-CBC", "cast5-cbc", NID_cast5_cbc, 9, &so[570]}, - {"CAST5-ECB", "cast5-ecb", NID_cast5_ecb}, - {"CAST5-CFB", "cast5-cfb", NID_cast5_cfb64}, - {"CAST5-OFB", "cast5-ofb", NID_cast5_ofb64}, - {"pbeWithMD5AndCast5CBC", "pbeWithMD5AndCast5CBC", NID_pbeWithMD5AndCast5_CBC, 9, &so[579]}, - {"DSA-SHA1", "dsaWithSHA1", NID_dsaWithSHA1, 7, &so[588]}, + { NULL, NULL, NID_undef }, + {"GN", "givenName", NID_givenName, 3, &so[463]}, + {"SN", "surname", NID_surname, 3, &so[466]}, + {"initials", "initials", NID_initials, 3, &so[469]}, + {"uid", "uniqueIdentifier", NID_uniqueIdentifier, 10, &so[472]}, + {"crlDistributionPoints", "X509v3 CRL Distribution Points", NID_crl_distribution_points, 3, &so[482]}, + {"RSA-NP-MD5", "md5WithRSA", NID_md5WithRSA, 5, &so[485]}, + {"serialNumber", "serialNumber", NID_serialNumber, 3, &so[490]}, + {"title", "title", NID_title, 3, &so[493]}, + {"description", "description", NID_description, 3, &so[496]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"DSA-SHA1", "dsaWithSHA1", NID_dsaWithSHA1, 7, &so[499]}, {"MD5-SHA1", "md5-sha1", NID_md5_sha1}, - {"RSA-SHA1-2", "sha1WithRSA", NID_sha1WithRSA, 5, &so[595]}, - {"DSA", "dsaEncryption", NID_dsa, 7, &so[600]}, - {"RIPEMD160", "ripemd160", NID_ripemd160, 5, &so[607]}, + {"RSA-SHA1-2", "sha1WithRSA", NID_sha1WithRSA, 5, &so[506]}, + {"DSA", "dsaEncryption", NID_dsa, 7, &so[511]}, { NULL, NULL, NID_undef }, - {"RSA-RIPEMD160", "ripemd160WithRSA", NID_ripemd160WithRSA, 6, &so[612]}, - {"RC5-CBC", "rc5-cbc", NID_rc5_cbc, 8, &so[618]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"RC5-CBC", "rc5-cbc", NID_rc5_cbc, 8, &so[518]}, {"RC5-ECB", "rc5-ecb", NID_rc5_ecb}, {"RC5-CFB", "rc5-cfb", NID_rc5_cfb64}, {"RC5-OFB", "rc5-ofb", NID_rc5_ofb64}, { NULL, NULL, NID_undef }, - {"ZLIB", "zlib compression", NID_zlib_compression, 11, &so[626]}, - {"extendedKeyUsage", "X509v3 Extended Key Usage", NID_ext_key_usage, 3, &so[637]}, - {"PKIX", "PKIX", NID_id_pkix, 6, &so[640]}, - {"id-kp", "id-kp", NID_id_kp, 7, &so[646]}, - {"serverAuth", "TLS Web Server Authentication", NID_server_auth, 8, &so[653]}, - {"clientAuth", "TLS Web Client Authentication", NID_client_auth, 8, &so[661]}, - {"codeSigning", "Code Signing", NID_code_sign, 8, &so[669]}, - {"emailProtection", "E-mail Protection", NID_email_protect, 8, &so[677]}, - {"timeStamping", "Time Stamping", NID_time_stamp, 8, &so[685]}, - {"msCodeInd", "Microsoft Individual Code Signing", NID_ms_code_ind, 10, &so[693]}, - {"msCodeCom", "Microsoft Commercial Code Signing", NID_ms_code_com, 10, &so[703]}, - {"msCTLSign", "Microsoft Trust List Signing", NID_ms_ctl_sign, 10, &so[713]}, - {"msSGC", "Microsoft Server Gated Crypto", NID_ms_sgc, 10, &so[723]}, - {"msEFS", "Microsoft Encrypted File System", NID_ms_efs, 10, &so[733]}, - {"nsSGC", "Netscape Server Gated Crypto", NID_ns_sgc, 9, &so[743]}, - {"deltaCRL", "X509v3 Delta CRL Indicator", NID_delta_crl, 3, &so[752]}, - {"CRLReason", "X509v3 CRL Reason Code", NID_crl_reason, 3, &so[755]}, - {"invalidityDate", "Invalidity Date", NID_invalidity_date, 3, &so[758]}, - {"SXNetID", "Strong Extranet ID", NID_sxnet, 5, &so[761]}, - {"PBE-SHA1-RC4-128", "pbeWithSHA1And128BitRC4", NID_pbe_WithSHA1And128BitRC4, 10, &so[766]}, - {"PBE-SHA1-RC4-40", "pbeWithSHA1And40BitRC4", NID_pbe_WithSHA1And40BitRC4, 10, &so[776]}, - {"PBE-SHA1-3DES", "pbeWithSHA1And3-KeyTripleDES-CBC", NID_pbe_WithSHA1And3_Key_TripleDES_CBC, 10, &so[786]}, - {"PBE-SHA1-2DES", "pbeWithSHA1And2-KeyTripleDES-CBC", NID_pbe_WithSHA1And2_Key_TripleDES_CBC, 10, &so[796]}, - {"PBE-SHA1-RC2-128", "pbeWithSHA1And128BitRC2-CBC", NID_pbe_WithSHA1And128BitRC2_CBC, 10, &so[806]}, - {"PBE-SHA1-RC2-40", "pbeWithSHA1And40BitRC2-CBC", NID_pbe_WithSHA1And40BitRC2_CBC, 10, &so[816]}, - {"keyBag", "keyBag", NID_keyBag, 11, &so[826]}, - {"pkcs8ShroudedKeyBag", "pkcs8ShroudedKeyBag", NID_pkcs8ShroudedKeyBag, 11, &so[837]}, - {"certBag", "certBag", NID_certBag, 11, &so[848]}, - {"crlBag", "crlBag", NID_crlBag, 11, &so[859]}, - {"secretBag", "secretBag", NID_secretBag, 11, &so[870]}, - {"safeContentsBag", "safeContentsBag", NID_safeContentsBag, 11, &so[881]}, - {"friendlyName", "friendlyName", NID_friendlyName, 9, &so[892]}, - {"localKeyID", "localKeyID", NID_localKeyID, 9, &so[901]}, - {"x509Certificate", "x509Certificate", NID_x509Certificate, 10, &so[910]}, - {"sdsiCertificate", "sdsiCertificate", NID_sdsiCertificate, 10, &so[920]}, - {"x509Crl", "x509Crl", NID_x509Crl, 10, &so[930]}, - {"PBES2", "PBES2", NID_pbes2, 9, &so[940]}, - {"PBMAC1", "PBMAC1", NID_pbmac1, 9, &so[949]}, - {"hmacWithSHA1", "hmacWithSHA1", NID_hmacWithSHA1, 8, &so[958]}, - {"id-qt-cps", "Policy Qualifier CPS", NID_id_qt_cps, 8, &so[966]}, - {"id-qt-unotice", "Policy Qualifier User Notice", NID_id_qt_unotice, 8, &so[974]}, - {"RC2-64-CBC", "rc2-64-cbc", NID_rc2_64_cbc}, - {"SMIME-CAPS", "S/MIME Capabilities", NID_SMIMECapabilities, 9, &so[982]}, - {"PBE-MD2-RC2-64", "pbeWithMD2AndRC2-CBC", NID_pbeWithMD2AndRC2_CBC, 9, &so[991]}, - {"PBE-MD5-RC2-64", "pbeWithMD5AndRC2-CBC", NID_pbeWithMD5AndRC2_CBC, 9, &so[1000]}, - {"PBE-SHA1-DES", "pbeWithSHA1AndDES-CBC", NID_pbeWithSHA1AndDES_CBC, 9, &so[1009]}, - {"msExtReq", "Microsoft Extension Request", NID_ms_ext_req, 10, &so[1018]}, - {"extReq", "Extension Request", NID_ext_req, 9, &so[1028]}, - {"name", "name", NID_name, 3, &so[1037]}, - {"dnQualifier", "dnQualifier", NID_dnQualifier, 3, &so[1040]}, - {"id-pe", "id-pe", NID_id_pe, 7, &so[1043]}, - {"id-ad", "id-ad", NID_id_ad, 7, &so[1050]}, - {"authorityInfoAccess", "Authority Information Access", NID_info_access, 8, &so[1057]}, - {"OCSP", "OCSP", NID_ad_OCSP, 8, &so[1065]}, - {"caIssuers", "CA Issuers", NID_ad_ca_issuers, 8, &so[1073]}, - {"OCSPSigning", "OCSP Signing", NID_OCSP_sign, 8, &so[1081]}, + {"ZLIB", "zlib compression", NID_zlib_compression, 11, &so[526]}, + {"extendedKeyUsage", "X509v3 Extended Key Usage", NID_ext_key_usage, 3, &so[537]}, + {"PKIX", "PKIX", NID_id_pkix, 6, &so[540]}, + {"id-kp", "id-kp", NID_id_kp, 7, &so[546]}, + {"serverAuth", "TLS Web Server Authentication", NID_server_auth, 8, &so[553]}, + {"clientAuth", "TLS Web Client Authentication", NID_client_auth, 8, &so[561]}, + {"codeSigning", "Code Signing", NID_code_sign, 8, &so[569]}, + {"emailProtection", "E-mail Protection", NID_email_protect, 8, &so[577]}, + {"timeStamping", "Time Stamping", NID_time_stamp, 8, &so[585]}, + {"msCodeInd", "Microsoft Individual Code Signing", NID_ms_code_ind, 10, &so[593]}, + {"msCodeCom", "Microsoft Commercial Code Signing", NID_ms_code_com, 10, &so[603]}, + {"msCTLSign", "Microsoft Trust List Signing", NID_ms_ctl_sign, 10, &so[613]}, + {"msSGC", "Microsoft Server Gated Crypto", NID_ms_sgc, 10, &so[623]}, + {"msEFS", "Microsoft Encrypted File System", NID_ms_efs, 10, &so[633]}, + {"nsSGC", "Netscape Server Gated Crypto", NID_ns_sgc, 9, &so[643]}, + {"deltaCRL", "X509v3 Delta CRL Indicator", NID_delta_crl, 3, &so[652]}, + {"CRLReason", "X509v3 CRL Reason Code", NID_crl_reason, 3, &so[655]}, + {"invalidityDate", "Invalidity Date", NID_invalidity_date, 3, &so[658]}, + {"SXNetID", "Strong Extranet ID", NID_sxnet, 5, &so[661]}, + {"PBE-SHA1-RC4-128", "pbeWithSHA1And128BitRC4", NID_pbe_WithSHA1And128BitRC4, 10, &so[666]}, + {"PBE-SHA1-RC4-40", "pbeWithSHA1And40BitRC4", NID_pbe_WithSHA1And40BitRC4, 10, &so[676]}, + {"PBE-SHA1-3DES", "pbeWithSHA1And3-KeyTripleDES-CBC", NID_pbe_WithSHA1And3_Key_TripleDES_CBC, 10, &so[686]}, + {"PBE-SHA1-2DES", "pbeWithSHA1And2-KeyTripleDES-CBC", NID_pbe_WithSHA1And2_Key_TripleDES_CBC, 10, &so[696]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"keyBag", "keyBag", NID_keyBag, 11, &so[706]}, + {"pkcs8ShroudedKeyBag", "pkcs8ShroudedKeyBag", NID_pkcs8ShroudedKeyBag, 11, &so[717]}, + {"certBag", "certBag", NID_certBag, 11, &so[728]}, + {"crlBag", "crlBag", NID_crlBag, 11, &so[739]}, + {"secretBag", "secretBag", NID_secretBag, 11, &so[750]}, + {"safeContentsBag", "safeContentsBag", NID_safeContentsBag, 11, &so[761]}, + {"friendlyName", "friendlyName", NID_friendlyName, 9, &so[772]}, + {"localKeyID", "localKeyID", NID_localKeyID, 9, &so[781]}, + {"x509Certificate", "x509Certificate", NID_x509Certificate, 10, &so[790]}, + {"sdsiCertificate", "sdsiCertificate", NID_sdsiCertificate, 10, &so[800]}, + {"x509Crl", "x509Crl", NID_x509Crl, 10, &so[810]}, + {"PBES2", "PBES2", NID_pbes2, 9, &so[820]}, + {"PBMAC1", "PBMAC1", NID_pbmac1, 9, &so[829]}, + {"hmacWithSHA1", "hmacWithSHA1", NID_hmacWithSHA1, 8, &so[838]}, + {"id-qt-cps", "Policy Qualifier CPS", NID_id_qt_cps, 8, &so[846]}, + {"id-qt-unotice", "Policy Qualifier User Notice", NID_id_qt_unotice, 8, &so[854]}, + { NULL, NULL, NID_undef }, + {"SMIME-CAPS", "S/MIME Capabilities", NID_SMIMECapabilities, 9, &so[862]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"PBE-SHA1-DES", "pbeWithSHA1AndDES-CBC", NID_pbeWithSHA1AndDES_CBC, 9, &so[871]}, + {"msExtReq", "Microsoft Extension Request", NID_ms_ext_req, 10, &so[880]}, + {"extReq", "Extension Request", NID_ext_req, 9, &so[890]}, + {"name", "name", NID_name, 3, &so[899]}, + {"dnQualifier", "dnQualifier", NID_dnQualifier, 3, &so[902]}, + {"id-pe", "id-pe", NID_id_pe, 7, &so[905]}, + {"id-ad", "id-ad", NID_id_ad, 7, &so[912]}, + {"authorityInfoAccess", "Authority Information Access", NID_info_access, 8, &so[919]}, + {"OCSP", "OCSP", NID_ad_OCSP, 8, &so[927]}, + {"caIssuers", "CA Issuers", NID_ad_ca_issuers, 8, &so[935]}, + {"OCSPSigning", "OCSP Signing", NID_OCSP_sign, 8, &so[943]}, {"ISO", "iso", NID_iso}, - {"member-body", "ISO Member Body", NID_member_body, 1, &so[1089]}, - {"ISO-US", "ISO US Member Body", NID_ISO_US, 3, &so[1090]}, - {"X9-57", "X9.57", NID_X9_57, 5, &so[1093]}, - {"X9cm", "X9.57 CM ?", NID_X9cm, 6, &so[1098]}, - {"pkcs1", "pkcs1", NID_pkcs1, 8, &so[1104]}, - {"pkcs5", "pkcs5", NID_pkcs5, 8, &so[1112]}, - {"SMIME", "S/MIME", NID_SMIME, 9, &so[1120]}, - {"id-smime-mod", "id-smime-mod", NID_id_smime_mod, 10, &so[1129]}, - {"id-smime-ct", "id-smime-ct", NID_id_smime_ct, 10, &so[1139]}, - {"id-smime-aa", "id-smime-aa", NID_id_smime_aa, 10, &so[1149]}, - {"id-smime-alg", "id-smime-alg", NID_id_smime_alg, 10, &so[1159]}, - {"id-smime-cd", "id-smime-cd", NID_id_smime_cd, 10, &so[1169]}, - {"id-smime-spq", "id-smime-spq", NID_id_smime_spq, 10, &so[1179]}, - {"id-smime-cti", "id-smime-cti", NID_id_smime_cti, 10, &so[1189]}, - {"id-smime-mod-cms", "id-smime-mod-cms", NID_id_smime_mod_cms, 11, &so[1199]}, - {"id-smime-mod-ess", "id-smime-mod-ess", NID_id_smime_mod_ess, 11, &so[1210]}, - {"id-smime-mod-oid", "id-smime-mod-oid", NID_id_smime_mod_oid, 11, &so[1221]}, - {"id-smime-mod-msg-v3", "id-smime-mod-msg-v3", NID_id_smime_mod_msg_v3, 11, &so[1232]}, - {"id-smime-mod-ets-eSignature-88", "id-smime-mod-ets-eSignature-88", NID_id_smime_mod_ets_eSignature_88, 11, &so[1243]}, - {"id-smime-mod-ets-eSignature-97", "id-smime-mod-ets-eSignature-97", NID_id_smime_mod_ets_eSignature_97, 11, &so[1254]}, - {"id-smime-mod-ets-eSigPolicy-88", "id-smime-mod-ets-eSigPolicy-88", NID_id_smime_mod_ets_eSigPolicy_88, 11, &so[1265]}, - {"id-smime-mod-ets-eSigPolicy-97", "id-smime-mod-ets-eSigPolicy-97", NID_id_smime_mod_ets_eSigPolicy_97, 11, &so[1276]}, - {"id-smime-ct-receipt", "id-smime-ct-receipt", NID_id_smime_ct_receipt, 11, &so[1287]}, - {"id-smime-ct-authData", "id-smime-ct-authData", NID_id_smime_ct_authData, 11, &so[1298]}, - {"id-smime-ct-publishCert", "id-smime-ct-publishCert", NID_id_smime_ct_publishCert, 11, &so[1309]}, - {"id-smime-ct-TSTInfo", "id-smime-ct-TSTInfo", NID_id_smime_ct_TSTInfo, 11, &so[1320]}, - {"id-smime-ct-TDTInfo", "id-smime-ct-TDTInfo", NID_id_smime_ct_TDTInfo, 11, &so[1331]}, - {"id-smime-ct-contentInfo", "id-smime-ct-contentInfo", NID_id_smime_ct_contentInfo, 11, &so[1342]}, - {"id-smime-ct-DVCSRequestData", "id-smime-ct-DVCSRequestData", NID_id_smime_ct_DVCSRequestData, 11, &so[1353]}, - {"id-smime-ct-DVCSResponseData", "id-smime-ct-DVCSResponseData", NID_id_smime_ct_DVCSResponseData, 11, &so[1364]}, - {"id-smime-aa-receiptRequest", "id-smime-aa-receiptRequest", NID_id_smime_aa_receiptRequest, 11, &so[1375]}, - {"id-smime-aa-securityLabel", "id-smime-aa-securityLabel", NID_id_smime_aa_securityLabel, 11, &so[1386]}, - {"id-smime-aa-mlExpandHistory", "id-smime-aa-mlExpandHistory", NID_id_smime_aa_mlExpandHistory, 11, &so[1397]}, - {"id-smime-aa-contentHint", "id-smime-aa-contentHint", NID_id_smime_aa_contentHint, 11, &so[1408]}, - {"id-smime-aa-msgSigDigest", "id-smime-aa-msgSigDigest", NID_id_smime_aa_msgSigDigest, 11, &so[1419]}, - {"id-smime-aa-encapContentType", "id-smime-aa-encapContentType", NID_id_smime_aa_encapContentType, 11, &so[1430]}, - {"id-smime-aa-contentIdentifier", "id-smime-aa-contentIdentifier", NID_id_smime_aa_contentIdentifier, 11, &so[1441]}, - {"id-smime-aa-macValue", "id-smime-aa-macValue", NID_id_smime_aa_macValue, 11, &so[1452]}, - {"id-smime-aa-equivalentLabels", "id-smime-aa-equivalentLabels", NID_id_smime_aa_equivalentLabels, 11, &so[1463]}, - {"id-smime-aa-contentReference", "id-smime-aa-contentReference", NID_id_smime_aa_contentReference, 11, &so[1474]}, - {"id-smime-aa-encrypKeyPref", "id-smime-aa-encrypKeyPref", NID_id_smime_aa_encrypKeyPref, 11, &so[1485]}, - {"id-smime-aa-signingCertificate", "id-smime-aa-signingCertificate", NID_id_smime_aa_signingCertificate, 11, &so[1496]}, - {"id-smime-aa-smimeEncryptCerts", "id-smime-aa-smimeEncryptCerts", NID_id_smime_aa_smimeEncryptCerts, 11, &so[1507]}, - {"id-smime-aa-timeStampToken", "id-smime-aa-timeStampToken", NID_id_smime_aa_timeStampToken, 11, &so[1518]}, - {"id-smime-aa-ets-sigPolicyId", "id-smime-aa-ets-sigPolicyId", NID_id_smime_aa_ets_sigPolicyId, 11, &so[1529]}, - {"id-smime-aa-ets-commitmentType", "id-smime-aa-ets-commitmentType", NID_id_smime_aa_ets_commitmentType, 11, &so[1540]}, - {"id-smime-aa-ets-signerLocation", "id-smime-aa-ets-signerLocation", NID_id_smime_aa_ets_signerLocation, 11, &so[1551]}, - {"id-smime-aa-ets-signerAttr", "id-smime-aa-ets-signerAttr", NID_id_smime_aa_ets_signerAttr, 11, &so[1562]}, - {"id-smime-aa-ets-otherSigCert", "id-smime-aa-ets-otherSigCert", NID_id_smime_aa_ets_otherSigCert, 11, &so[1573]}, - {"id-smime-aa-ets-contentTimestamp", "id-smime-aa-ets-contentTimestamp", NID_id_smime_aa_ets_contentTimestamp, 11, &so[1584]}, - {"id-smime-aa-ets-CertificateRefs", "id-smime-aa-ets-CertificateRefs", NID_id_smime_aa_ets_CertificateRefs, 11, &so[1595]}, - {"id-smime-aa-ets-RevocationRefs", "id-smime-aa-ets-RevocationRefs", NID_id_smime_aa_ets_RevocationRefs, 11, &so[1606]}, - {"id-smime-aa-ets-certValues", "id-smime-aa-ets-certValues", NID_id_smime_aa_ets_certValues, 11, &so[1617]}, - {"id-smime-aa-ets-revocationValues", "id-smime-aa-ets-revocationValues", NID_id_smime_aa_ets_revocationValues, 11, &so[1628]}, - {"id-smime-aa-ets-escTimeStamp", "id-smime-aa-ets-escTimeStamp", NID_id_smime_aa_ets_escTimeStamp, 11, &so[1639]}, - {"id-smime-aa-ets-certCRLTimestamp", "id-smime-aa-ets-certCRLTimestamp", NID_id_smime_aa_ets_certCRLTimestamp, 11, &so[1650]}, - {"id-smime-aa-ets-archiveTimeStamp", "id-smime-aa-ets-archiveTimeStamp", NID_id_smime_aa_ets_archiveTimeStamp, 11, &so[1661]}, - {"id-smime-aa-signatureType", "id-smime-aa-signatureType", NID_id_smime_aa_signatureType, 11, &so[1672]}, - {"id-smime-aa-dvcs-dvc", "id-smime-aa-dvcs-dvc", NID_id_smime_aa_dvcs_dvc, 11, &so[1683]}, - {"id-smime-alg-ESDHwith3DES", "id-smime-alg-ESDHwith3DES", NID_id_smime_alg_ESDHwith3DES, 11, &so[1694]}, - {"id-smime-alg-ESDHwithRC2", "id-smime-alg-ESDHwithRC2", NID_id_smime_alg_ESDHwithRC2, 11, &so[1705]}, - {"id-smime-alg-3DESwrap", "id-smime-alg-3DESwrap", NID_id_smime_alg_3DESwrap, 11, &so[1716]}, - {"id-smime-alg-RC2wrap", "id-smime-alg-RC2wrap", NID_id_smime_alg_RC2wrap, 11, &so[1727]}, - {"id-smime-alg-ESDH", "id-smime-alg-ESDH", NID_id_smime_alg_ESDH, 11, &so[1738]}, - {"id-smime-alg-CMS3DESwrap", "id-smime-alg-CMS3DESwrap", NID_id_smime_alg_CMS3DESwrap, 11, &so[1749]}, - {"id-smime-alg-CMSRC2wrap", "id-smime-alg-CMSRC2wrap", NID_id_smime_alg_CMSRC2wrap, 11, &so[1760]}, - {"id-smime-cd-ldap", "id-smime-cd-ldap", NID_id_smime_cd_ldap, 11, &so[1771]}, - {"id-smime-spq-ets-sqt-uri", "id-smime-spq-ets-sqt-uri", NID_id_smime_spq_ets_sqt_uri, 11, &so[1782]}, - {"id-smime-spq-ets-sqt-unotice", "id-smime-spq-ets-sqt-unotice", NID_id_smime_spq_ets_sqt_unotice, 11, &so[1793]}, - {"id-smime-cti-ets-proofOfOrigin", "id-smime-cti-ets-proofOfOrigin", NID_id_smime_cti_ets_proofOfOrigin, 11, &so[1804]}, - {"id-smime-cti-ets-proofOfReceipt", "id-smime-cti-ets-proofOfReceipt", NID_id_smime_cti_ets_proofOfReceipt, 11, &so[1815]}, - {"id-smime-cti-ets-proofOfDelivery", "id-smime-cti-ets-proofOfDelivery", NID_id_smime_cti_ets_proofOfDelivery, 11, &so[1826]}, - {"id-smime-cti-ets-proofOfSender", "id-smime-cti-ets-proofOfSender", NID_id_smime_cti_ets_proofOfSender, 11, &so[1837]}, - {"id-smime-cti-ets-proofOfApproval", "id-smime-cti-ets-proofOfApproval", NID_id_smime_cti_ets_proofOfApproval, 11, &so[1848]}, - {"id-smime-cti-ets-proofOfCreation", "id-smime-cti-ets-proofOfCreation", NID_id_smime_cti_ets_proofOfCreation, 11, &so[1859]}, - {"MD4", "md4", NID_md4, 8, &so[1870]}, - {"id-pkix-mod", "id-pkix-mod", NID_id_pkix_mod, 7, &so[1878]}, - {"id-qt", "id-qt", NID_id_qt, 7, &so[1885]}, - {"id-it", "id-it", NID_id_it, 7, &so[1892]}, - {"id-pkip", "id-pkip", NID_id_pkip, 7, &so[1899]}, - {"id-alg", "id-alg", NID_id_alg, 7, &so[1906]}, - {"id-cmc", "id-cmc", NID_id_cmc, 7, &so[1913]}, - {"id-on", "id-on", NID_id_on, 7, &so[1920]}, - {"id-pda", "id-pda", NID_id_pda, 7, &so[1927]}, - {"id-aca", "id-aca", NID_id_aca, 7, &so[1934]}, - {"id-qcs", "id-qcs", NID_id_qcs, 7, &so[1941]}, - {"id-cct", "id-cct", NID_id_cct, 7, &so[1948]}, - {"id-pkix1-explicit-88", "id-pkix1-explicit-88", NID_id_pkix1_explicit_88, 8, &so[1955]}, - {"id-pkix1-implicit-88", "id-pkix1-implicit-88", NID_id_pkix1_implicit_88, 8, &so[1963]}, - {"id-pkix1-explicit-93", "id-pkix1-explicit-93", NID_id_pkix1_explicit_93, 8, &so[1971]}, - {"id-pkix1-implicit-93", "id-pkix1-implicit-93", NID_id_pkix1_implicit_93, 8, &so[1979]}, - {"id-mod-crmf", "id-mod-crmf", NID_id_mod_crmf, 8, &so[1987]}, - {"id-mod-cmc", "id-mod-cmc", NID_id_mod_cmc, 8, &so[1995]}, - {"id-mod-kea-profile-88", "id-mod-kea-profile-88", NID_id_mod_kea_profile_88, 8, &so[2003]}, - {"id-mod-kea-profile-93", "id-mod-kea-profile-93", NID_id_mod_kea_profile_93, 8, &so[2011]}, - {"id-mod-cmp", "id-mod-cmp", NID_id_mod_cmp, 8, &so[2019]}, - {"id-mod-qualified-cert-88", "id-mod-qualified-cert-88", NID_id_mod_qualified_cert_88, 8, &so[2027]}, - {"id-mod-qualified-cert-93", "id-mod-qualified-cert-93", NID_id_mod_qualified_cert_93, 8, &so[2035]}, - {"id-mod-attribute-cert", "id-mod-attribute-cert", NID_id_mod_attribute_cert, 8, &so[2043]}, - {"id-mod-timestamp-protocol", "id-mod-timestamp-protocol", NID_id_mod_timestamp_protocol, 8, &so[2051]}, - {"id-mod-ocsp", "id-mod-ocsp", NID_id_mod_ocsp, 8, &so[2059]}, - {"id-mod-dvcs", "id-mod-dvcs", NID_id_mod_dvcs, 8, &so[2067]}, - {"id-mod-cmp2000", "id-mod-cmp2000", NID_id_mod_cmp2000, 8, &so[2075]}, - {"biometricInfo", "Biometric Info", NID_biometricInfo, 8, &so[2083]}, - {"qcStatements", "qcStatements", NID_qcStatements, 8, &so[2091]}, - {"ac-auditEntity", "ac-auditEntity", NID_ac_auditEntity, 8, &so[2099]}, - {"ac-targeting", "ac-targeting", NID_ac_targeting, 8, &so[2107]}, - {"aaControls", "aaControls", NID_aaControls, 8, &so[2115]}, - {"sbgp-ipAddrBlock", "sbgp-ipAddrBlock", NID_sbgp_ipAddrBlock, 8, &so[2123]}, - {"sbgp-autonomousSysNum", "sbgp-autonomousSysNum", NID_sbgp_autonomousSysNum, 8, &so[2131]}, - {"sbgp-routerIdentifier", "sbgp-routerIdentifier", NID_sbgp_routerIdentifier, 8, &so[2139]}, - {"textNotice", "textNotice", NID_textNotice, 8, &so[2147]}, - {"ipsecEndSystem", "IPSec End System", NID_ipsecEndSystem, 8, &so[2155]}, - {"ipsecTunnel", "IPSec Tunnel", NID_ipsecTunnel, 8, &so[2163]}, - {"ipsecUser", "IPSec User", NID_ipsecUser, 8, &so[2171]}, - {"DVCS", "dvcs", NID_dvcs, 8, &so[2179]}, - {"id-it-caProtEncCert", "id-it-caProtEncCert", NID_id_it_caProtEncCert, 8, &so[2187]}, - {"id-it-signKeyPairTypes", "id-it-signKeyPairTypes", NID_id_it_signKeyPairTypes, 8, &so[2195]}, - {"id-it-encKeyPairTypes", "id-it-encKeyPairTypes", NID_id_it_encKeyPairTypes, 8, &so[2203]}, - {"id-it-preferredSymmAlg", "id-it-preferredSymmAlg", NID_id_it_preferredSymmAlg, 8, &so[2211]}, - {"id-it-caKeyUpdateInfo", "id-it-caKeyUpdateInfo", NID_id_it_caKeyUpdateInfo, 8, &so[2219]}, - {"id-it-currentCRL", "id-it-currentCRL", NID_id_it_currentCRL, 8, &so[2227]}, - {"id-it-unsupportedOIDs", "id-it-unsupportedOIDs", NID_id_it_unsupportedOIDs, 8, &so[2235]}, - {"id-it-subscriptionRequest", "id-it-subscriptionRequest", NID_id_it_subscriptionRequest, 8, &so[2243]}, - {"id-it-subscriptionResponse", "id-it-subscriptionResponse", NID_id_it_subscriptionResponse, 8, &so[2251]}, - {"id-it-keyPairParamReq", "id-it-keyPairParamReq", NID_id_it_keyPairParamReq, 8, &so[2259]}, - {"id-it-keyPairParamRep", "id-it-keyPairParamRep", NID_id_it_keyPairParamRep, 8, &so[2267]}, - {"id-it-revPassphrase", "id-it-revPassphrase", NID_id_it_revPassphrase, 8, &so[2275]}, - {"id-it-implicitConfirm", "id-it-implicitConfirm", NID_id_it_implicitConfirm, 8, &so[2283]}, - {"id-it-confirmWaitTime", "id-it-confirmWaitTime", NID_id_it_confirmWaitTime, 8, &so[2291]}, - {"id-it-origPKIMessage", "id-it-origPKIMessage", NID_id_it_origPKIMessage, 8, &so[2299]}, - {"id-regCtrl", "id-regCtrl", NID_id_regCtrl, 8, &so[2307]}, - {"id-regInfo", "id-regInfo", NID_id_regInfo, 8, &so[2315]}, - {"id-regCtrl-regToken", "id-regCtrl-regToken", NID_id_regCtrl_regToken, 9, &so[2323]}, - {"id-regCtrl-authenticator", "id-regCtrl-authenticator", NID_id_regCtrl_authenticator, 9, &so[2332]}, - {"id-regCtrl-pkiPublicationInfo", "id-regCtrl-pkiPublicationInfo", NID_id_regCtrl_pkiPublicationInfo, 9, &so[2341]}, - {"id-regCtrl-pkiArchiveOptions", "id-regCtrl-pkiArchiveOptions", NID_id_regCtrl_pkiArchiveOptions, 9, &so[2350]}, - {"id-regCtrl-oldCertID", "id-regCtrl-oldCertID", NID_id_regCtrl_oldCertID, 9, &so[2359]}, - {"id-regCtrl-protocolEncrKey", "id-regCtrl-protocolEncrKey", NID_id_regCtrl_protocolEncrKey, 9, &so[2368]}, - {"id-regInfo-utf8Pairs", "id-regInfo-utf8Pairs", NID_id_regInfo_utf8Pairs, 9, &so[2377]}, - {"id-regInfo-certReq", "id-regInfo-certReq", NID_id_regInfo_certReq, 9, &so[2386]}, - {"id-alg-des40", "id-alg-des40", NID_id_alg_des40, 8, &so[2395]}, - {"id-alg-noSignature", "id-alg-noSignature", NID_id_alg_noSignature, 8, &so[2403]}, - {"id-alg-dh-sig-hmac-sha1", "id-alg-dh-sig-hmac-sha1", NID_id_alg_dh_sig_hmac_sha1, 8, &so[2411]}, - {"id-alg-dh-pop", "id-alg-dh-pop", NID_id_alg_dh_pop, 8, &so[2419]}, - {"id-cmc-statusInfo", "id-cmc-statusInfo", NID_id_cmc_statusInfo, 8, &so[2427]}, - {"id-cmc-identification", "id-cmc-identification", NID_id_cmc_identification, 8, &so[2435]}, - {"id-cmc-identityProof", "id-cmc-identityProof", NID_id_cmc_identityProof, 8, &so[2443]}, - {"id-cmc-dataReturn", "id-cmc-dataReturn", NID_id_cmc_dataReturn, 8, &so[2451]}, - {"id-cmc-transactionId", "id-cmc-transactionId", NID_id_cmc_transactionId, 8, &so[2459]}, - {"id-cmc-senderNonce", "id-cmc-senderNonce", NID_id_cmc_senderNonce, 8, &so[2467]}, - {"id-cmc-recipientNonce", "id-cmc-recipientNonce", NID_id_cmc_recipientNonce, 8, &so[2475]}, - {"id-cmc-addExtensions", "id-cmc-addExtensions", NID_id_cmc_addExtensions, 8, &so[2483]}, - {"id-cmc-encryptedPOP", "id-cmc-encryptedPOP", NID_id_cmc_encryptedPOP, 8, &so[2491]}, - {"id-cmc-decryptedPOP", "id-cmc-decryptedPOP", NID_id_cmc_decryptedPOP, 8, &so[2499]}, - {"id-cmc-lraPOPWitness", "id-cmc-lraPOPWitness", NID_id_cmc_lraPOPWitness, 8, &so[2507]}, - {"id-cmc-getCert", "id-cmc-getCert", NID_id_cmc_getCert, 8, &so[2515]}, - {"id-cmc-getCRL", "id-cmc-getCRL", NID_id_cmc_getCRL, 8, &so[2523]}, - {"id-cmc-revokeRequest", "id-cmc-revokeRequest", NID_id_cmc_revokeRequest, 8, &so[2531]}, - {"id-cmc-regInfo", "id-cmc-regInfo", NID_id_cmc_regInfo, 8, &so[2539]}, - {"id-cmc-responseInfo", "id-cmc-responseInfo", NID_id_cmc_responseInfo, 8, &so[2547]}, - {"id-cmc-queryPending", "id-cmc-queryPending", NID_id_cmc_queryPending, 8, &so[2555]}, - {"id-cmc-popLinkRandom", "id-cmc-popLinkRandom", NID_id_cmc_popLinkRandom, 8, &so[2563]}, - {"id-cmc-popLinkWitness", "id-cmc-popLinkWitness", NID_id_cmc_popLinkWitness, 8, &so[2571]}, - {"id-cmc-confirmCertAcceptance", "id-cmc-confirmCertAcceptance", NID_id_cmc_confirmCertAcceptance, 8, &so[2579]}, - {"id-on-personalData", "id-on-personalData", NID_id_on_personalData, 8, &so[2587]}, - {"id-pda-dateOfBirth", "id-pda-dateOfBirth", NID_id_pda_dateOfBirth, 8, &so[2595]}, - {"id-pda-placeOfBirth", "id-pda-placeOfBirth", NID_id_pda_placeOfBirth, 8, &so[2603]}, - { NULL, NULL, NID_undef }, - {"id-pda-gender", "id-pda-gender", NID_id_pda_gender, 8, &so[2611]}, - {"id-pda-countryOfCitizenship", "id-pda-countryOfCitizenship", NID_id_pda_countryOfCitizenship, 8, &so[2619]}, - {"id-pda-countryOfResidence", "id-pda-countryOfResidence", NID_id_pda_countryOfResidence, 8, &so[2627]}, - {"id-aca-authenticationInfo", "id-aca-authenticationInfo", NID_id_aca_authenticationInfo, 8, &so[2635]}, - {"id-aca-accessIdentity", "id-aca-accessIdentity", NID_id_aca_accessIdentity, 8, &so[2643]}, - {"id-aca-chargingIdentity", "id-aca-chargingIdentity", NID_id_aca_chargingIdentity, 8, &so[2651]}, - {"id-aca-group", "id-aca-group", NID_id_aca_group, 8, &so[2659]}, - {"id-aca-role", "id-aca-role", NID_id_aca_role, 8, &so[2667]}, - {"id-qcs-pkixQCSyntax-v1", "id-qcs-pkixQCSyntax-v1", NID_id_qcs_pkixQCSyntax_v1, 8, &so[2675]}, - {"id-cct-crs", "id-cct-crs", NID_id_cct_crs, 8, &so[2683]}, - {"id-cct-PKIData", "id-cct-PKIData", NID_id_cct_PKIData, 8, &so[2691]}, - {"id-cct-PKIResponse", "id-cct-PKIResponse", NID_id_cct_PKIResponse, 8, &so[2699]}, - {"ad_timestamping", "AD Time Stamping", NID_ad_timeStamping, 8, &so[2707]}, - {"AD_DVCS", "ad dvcs", NID_ad_dvcs, 8, &so[2715]}, - {"basicOCSPResponse", "Basic OCSP Response", NID_id_pkix_OCSP_basic, 9, &so[2723]}, - {"Nonce", "OCSP Nonce", NID_id_pkix_OCSP_Nonce, 9, &so[2732]}, - {"CrlID", "OCSP CRL ID", NID_id_pkix_OCSP_CrlID, 9, &so[2741]}, - {"acceptableResponses", "Acceptable OCSP Responses", NID_id_pkix_OCSP_acceptableResponses, 9, &so[2750]}, - {"noCheck", "OCSP No Check", NID_id_pkix_OCSP_noCheck, 9, &so[2759]}, - {"archiveCutoff", "OCSP Archive Cutoff", NID_id_pkix_OCSP_archiveCutoff, 9, &so[2768]}, - {"serviceLocator", "OCSP Service Locator", NID_id_pkix_OCSP_serviceLocator, 9, &so[2777]}, - {"extendedStatus", "Extended OCSP Status", NID_id_pkix_OCSP_extendedStatus, 9, &so[2786]}, - {"valid", "valid", NID_id_pkix_OCSP_valid, 9, &so[2795]}, - {"path", "path", NID_id_pkix_OCSP_path, 9, &so[2804]}, - {"trustRoot", "Trust Root", NID_id_pkix_OCSP_trustRoot, 9, &so[2813]}, - {"algorithm", "algorithm", NID_algorithm, 4, &so[2822]}, - {"rsaSignature", "rsaSignature", NID_rsaSignature, 5, &so[2826]}, - {"X500algorithms", "directory services - algorithms", NID_X500algorithms, 2, &so[2831]}, - {"ORG", "org", NID_org, 1, &so[2833]}, - {"DOD", "dod", NID_dod, 2, &so[2834]}, - {"IANA", "iana", NID_iana, 3, &so[2836]}, - {"directory", "Directory", NID_Directory, 4, &so[2839]}, - {"mgmt", "Management", NID_Management, 4, &so[2843]}, - {"experimental", "Experimental", NID_Experimental, 4, &so[2847]}, - {"private", "Private", NID_Private, 4, &so[2851]}, - {"security", "Security", NID_Security, 4, &so[2855]}, - {"snmpv2", "SNMPv2", NID_SNMPv2, 4, &so[2859]}, - {"Mail", "Mail", NID_Mail, 4, &so[2863]}, - {"enterprises", "Enterprises", NID_Enterprises, 5, &so[2867]}, - {"dcobject", "dcObject", NID_dcObject, 9, &so[2872]}, - {"DC", "domainComponent", NID_domainComponent, 10, &so[2881]}, - {"domain", "Domain", NID_Domain, 10, &so[2891]}, + {"member-body", "ISO Member Body", NID_member_body, 1, &so[951]}, + {"ISO-US", "ISO US Member Body", NID_ISO_US, 3, &so[952]}, + {"X9-57", "X9.57", NID_X9_57, 5, &so[955]}, + {"X9cm", "X9.57 CM ?", NID_X9cm, 6, &so[960]}, + {"pkcs1", "pkcs1", NID_pkcs1, 8, &so[966]}, + {"pkcs5", "pkcs5", NID_pkcs5, 8, &so[974]}, + {"SMIME", "S/MIME", NID_SMIME, 9, &so[982]}, + {"id-smime-mod", "id-smime-mod", NID_id_smime_mod, 10, &so[991]}, + {"id-smime-ct", "id-smime-ct", NID_id_smime_ct, 10, &so[1001]}, + {"id-smime-aa", "id-smime-aa", NID_id_smime_aa, 10, &so[1011]}, + {"id-smime-alg", "id-smime-alg", NID_id_smime_alg, 10, &so[1021]}, + {"id-smime-cd", "id-smime-cd", NID_id_smime_cd, 10, &so[1031]}, + {"id-smime-spq", "id-smime-spq", NID_id_smime_spq, 10, &so[1041]}, + {"id-smime-cti", "id-smime-cti", NID_id_smime_cti, 10, &so[1051]}, + {"id-smime-mod-cms", "id-smime-mod-cms", NID_id_smime_mod_cms, 11, &so[1061]}, + {"id-smime-mod-ess", "id-smime-mod-ess", NID_id_smime_mod_ess, 11, &so[1072]}, + {"id-smime-mod-oid", "id-smime-mod-oid", NID_id_smime_mod_oid, 11, &so[1083]}, + {"id-smime-mod-msg-v3", "id-smime-mod-msg-v3", NID_id_smime_mod_msg_v3, 11, &so[1094]}, + {"id-smime-mod-ets-eSignature-88", "id-smime-mod-ets-eSignature-88", NID_id_smime_mod_ets_eSignature_88, 11, &so[1105]}, + {"id-smime-mod-ets-eSignature-97", "id-smime-mod-ets-eSignature-97", NID_id_smime_mod_ets_eSignature_97, 11, &so[1116]}, + {"id-smime-mod-ets-eSigPolicy-88", "id-smime-mod-ets-eSigPolicy-88", NID_id_smime_mod_ets_eSigPolicy_88, 11, &so[1127]}, + {"id-smime-mod-ets-eSigPolicy-97", "id-smime-mod-ets-eSigPolicy-97", NID_id_smime_mod_ets_eSigPolicy_97, 11, &so[1138]}, + {"id-smime-ct-receipt", "id-smime-ct-receipt", NID_id_smime_ct_receipt, 11, &so[1149]}, + {"id-smime-ct-authData", "id-smime-ct-authData", NID_id_smime_ct_authData, 11, &so[1160]}, + {"id-smime-ct-publishCert", "id-smime-ct-publishCert", NID_id_smime_ct_publishCert, 11, &so[1171]}, + {"id-smime-ct-TSTInfo", "id-smime-ct-TSTInfo", NID_id_smime_ct_TSTInfo, 11, &so[1182]}, + {"id-smime-ct-TDTInfo", "id-smime-ct-TDTInfo", NID_id_smime_ct_TDTInfo, 11, &so[1193]}, + {"id-smime-ct-contentInfo", "id-smime-ct-contentInfo", NID_id_smime_ct_contentInfo, 11, &so[1204]}, + {"id-smime-ct-DVCSRequestData", "id-smime-ct-DVCSRequestData", NID_id_smime_ct_DVCSRequestData, 11, &so[1215]}, + {"id-smime-ct-DVCSResponseData", "id-smime-ct-DVCSResponseData", NID_id_smime_ct_DVCSResponseData, 11, &so[1226]}, + {"id-smime-aa-receiptRequest", "id-smime-aa-receiptRequest", NID_id_smime_aa_receiptRequest, 11, &so[1237]}, + {"id-smime-aa-securityLabel", "id-smime-aa-securityLabel", NID_id_smime_aa_securityLabel, 11, &so[1248]}, + {"id-smime-aa-mlExpandHistory", "id-smime-aa-mlExpandHistory", NID_id_smime_aa_mlExpandHistory, 11, &so[1259]}, + {"id-smime-aa-contentHint", "id-smime-aa-contentHint", NID_id_smime_aa_contentHint, 11, &so[1270]}, + {"id-smime-aa-msgSigDigest", "id-smime-aa-msgSigDigest", NID_id_smime_aa_msgSigDigest, 11, &so[1281]}, + {"id-smime-aa-encapContentType", "id-smime-aa-encapContentType", NID_id_smime_aa_encapContentType, 11, &so[1292]}, + {"id-smime-aa-contentIdentifier", "id-smime-aa-contentIdentifier", NID_id_smime_aa_contentIdentifier, 11, &so[1303]}, + {"id-smime-aa-macValue", "id-smime-aa-macValue", NID_id_smime_aa_macValue, 11, &so[1314]}, + {"id-smime-aa-equivalentLabels", "id-smime-aa-equivalentLabels", NID_id_smime_aa_equivalentLabels, 11, &so[1325]}, + {"id-smime-aa-contentReference", "id-smime-aa-contentReference", NID_id_smime_aa_contentReference, 11, &so[1336]}, + {"id-smime-aa-encrypKeyPref", "id-smime-aa-encrypKeyPref", NID_id_smime_aa_encrypKeyPref, 11, &so[1347]}, + {"id-smime-aa-signingCertificate", "id-smime-aa-signingCertificate", NID_id_smime_aa_signingCertificate, 11, &so[1358]}, + {"id-smime-aa-smimeEncryptCerts", "id-smime-aa-smimeEncryptCerts", NID_id_smime_aa_smimeEncryptCerts, 11, &so[1369]}, + {"id-smime-aa-timeStampToken", "id-smime-aa-timeStampToken", NID_id_smime_aa_timeStampToken, 11, &so[1380]}, + {"id-smime-aa-ets-sigPolicyId", "id-smime-aa-ets-sigPolicyId", NID_id_smime_aa_ets_sigPolicyId, 11, &so[1391]}, + {"id-smime-aa-ets-commitmentType", "id-smime-aa-ets-commitmentType", NID_id_smime_aa_ets_commitmentType, 11, &so[1402]}, + {"id-smime-aa-ets-signerLocation", "id-smime-aa-ets-signerLocation", NID_id_smime_aa_ets_signerLocation, 11, &so[1413]}, + {"id-smime-aa-ets-signerAttr", "id-smime-aa-ets-signerAttr", NID_id_smime_aa_ets_signerAttr, 11, &so[1424]}, + {"id-smime-aa-ets-otherSigCert", "id-smime-aa-ets-otherSigCert", NID_id_smime_aa_ets_otherSigCert, 11, &so[1435]}, + {"id-smime-aa-ets-contentTimestamp", "id-smime-aa-ets-contentTimestamp", NID_id_smime_aa_ets_contentTimestamp, 11, &so[1446]}, + {"id-smime-aa-ets-CertificateRefs", "id-smime-aa-ets-CertificateRefs", NID_id_smime_aa_ets_CertificateRefs, 11, &so[1457]}, + {"id-smime-aa-ets-RevocationRefs", "id-smime-aa-ets-RevocationRefs", NID_id_smime_aa_ets_RevocationRefs, 11, &so[1468]}, + {"id-smime-aa-ets-certValues", "id-smime-aa-ets-certValues", NID_id_smime_aa_ets_certValues, 11, &so[1479]}, + {"id-smime-aa-ets-revocationValues", "id-smime-aa-ets-revocationValues", NID_id_smime_aa_ets_revocationValues, 11, &so[1490]}, + {"id-smime-aa-ets-escTimeStamp", "id-smime-aa-ets-escTimeStamp", NID_id_smime_aa_ets_escTimeStamp, 11, &so[1501]}, + {"id-smime-aa-ets-certCRLTimestamp", "id-smime-aa-ets-certCRLTimestamp", NID_id_smime_aa_ets_certCRLTimestamp, 11, &so[1512]}, + {"id-smime-aa-ets-archiveTimeStamp", "id-smime-aa-ets-archiveTimeStamp", NID_id_smime_aa_ets_archiveTimeStamp, 11, &so[1523]}, + {"id-smime-aa-signatureType", "id-smime-aa-signatureType", NID_id_smime_aa_signatureType, 11, &so[1534]}, + {"id-smime-aa-dvcs-dvc", "id-smime-aa-dvcs-dvc", NID_id_smime_aa_dvcs_dvc, 11, &so[1545]}, + {"id-smime-alg-ESDHwith3DES", "id-smime-alg-ESDHwith3DES", NID_id_smime_alg_ESDHwith3DES, 11, &so[1556]}, + { NULL, NULL, NID_undef }, + {"id-smime-alg-3DESwrap", "id-smime-alg-3DESwrap", NID_id_smime_alg_3DESwrap, 11, &so[1567]}, + { NULL, NULL, NID_undef }, + {"id-smime-alg-ESDH", "id-smime-alg-ESDH", NID_id_smime_alg_ESDH, 11, &so[1578]}, + {"id-smime-alg-CMS3DESwrap", "id-smime-alg-CMS3DESwrap", NID_id_smime_alg_CMS3DESwrap, 11, &so[1589]}, + { NULL, NULL, NID_undef }, + {"id-smime-cd-ldap", "id-smime-cd-ldap", NID_id_smime_cd_ldap, 11, &so[1600]}, + {"id-smime-spq-ets-sqt-uri", "id-smime-spq-ets-sqt-uri", NID_id_smime_spq_ets_sqt_uri, 11, &so[1611]}, + {"id-smime-spq-ets-sqt-unotice", "id-smime-spq-ets-sqt-unotice", NID_id_smime_spq_ets_sqt_unotice, 11, &so[1622]}, + {"id-smime-cti-ets-proofOfOrigin", "id-smime-cti-ets-proofOfOrigin", NID_id_smime_cti_ets_proofOfOrigin, 11, &so[1633]}, + {"id-smime-cti-ets-proofOfReceipt", "id-smime-cti-ets-proofOfReceipt", NID_id_smime_cti_ets_proofOfReceipt, 11, &so[1644]}, + {"id-smime-cti-ets-proofOfDelivery", "id-smime-cti-ets-proofOfDelivery", NID_id_smime_cti_ets_proofOfDelivery, 11, &so[1655]}, + {"id-smime-cti-ets-proofOfSender", "id-smime-cti-ets-proofOfSender", NID_id_smime_cti_ets_proofOfSender, 11, &so[1666]}, + {"id-smime-cti-ets-proofOfApproval", "id-smime-cti-ets-proofOfApproval", NID_id_smime_cti_ets_proofOfApproval, 11, &so[1677]}, + {"id-smime-cti-ets-proofOfCreation", "id-smime-cti-ets-proofOfCreation", NID_id_smime_cti_ets_proofOfCreation, 11, &so[1688]}, + { NULL, NULL, NID_undef }, + {"id-pkix-mod", "id-pkix-mod", NID_id_pkix_mod, 7, &so[1699]}, + {"id-qt", "id-qt", NID_id_qt, 7, &so[1706]}, + {"id-it", "id-it", NID_id_it, 7, &so[1713]}, + {"id-pkip", "id-pkip", NID_id_pkip, 7, &so[1720]}, + {"id-alg", "id-alg", NID_id_alg, 7, &so[1727]}, + {"id-cmc", "id-cmc", NID_id_cmc, 7, &so[1734]}, + {"id-on", "id-on", NID_id_on, 7, &so[1741]}, + {"id-pda", "id-pda", NID_id_pda, 7, &so[1748]}, + {"id-aca", "id-aca", NID_id_aca, 7, &so[1755]}, + {"id-qcs", "id-qcs", NID_id_qcs, 7, &so[1762]}, + {"id-cct", "id-cct", NID_id_cct, 7, &so[1769]}, + {"id-pkix1-explicit-88", "id-pkix1-explicit-88", NID_id_pkix1_explicit_88, 8, &so[1776]}, + {"id-pkix1-implicit-88", "id-pkix1-implicit-88", NID_id_pkix1_implicit_88, 8, &so[1784]}, + {"id-pkix1-explicit-93", "id-pkix1-explicit-93", NID_id_pkix1_explicit_93, 8, &so[1792]}, + {"id-pkix1-implicit-93", "id-pkix1-implicit-93", NID_id_pkix1_implicit_93, 8, &so[1800]}, + {"id-mod-crmf", "id-mod-crmf", NID_id_mod_crmf, 8, &so[1808]}, + {"id-mod-cmc", "id-mod-cmc", NID_id_mod_cmc, 8, &so[1816]}, + {"id-mod-kea-profile-88", "id-mod-kea-profile-88", NID_id_mod_kea_profile_88, 8, &so[1824]}, + {"id-mod-kea-profile-93", "id-mod-kea-profile-93", NID_id_mod_kea_profile_93, 8, &so[1832]}, + {"id-mod-cmp", "id-mod-cmp", NID_id_mod_cmp, 8, &so[1840]}, + {"id-mod-qualified-cert-88", "id-mod-qualified-cert-88", NID_id_mod_qualified_cert_88, 8, &so[1848]}, + {"id-mod-qualified-cert-93", "id-mod-qualified-cert-93", NID_id_mod_qualified_cert_93, 8, &so[1856]}, + {"id-mod-attribute-cert", "id-mod-attribute-cert", NID_id_mod_attribute_cert, 8, &so[1864]}, + {"id-mod-timestamp-protocol", "id-mod-timestamp-protocol", NID_id_mod_timestamp_protocol, 8, &so[1872]}, + {"id-mod-ocsp", "id-mod-ocsp", NID_id_mod_ocsp, 8, &so[1880]}, + {"id-mod-dvcs", "id-mod-dvcs", NID_id_mod_dvcs, 8, &so[1888]}, + {"id-mod-cmp2000", "id-mod-cmp2000", NID_id_mod_cmp2000, 8, &so[1896]}, + {"biometricInfo", "Biometric Info", NID_biometricInfo, 8, &so[1904]}, + {"qcStatements", "qcStatements", NID_qcStatements, 8, &so[1912]}, + {"ac-auditEntity", "ac-auditEntity", NID_ac_auditEntity, 8, &so[1920]}, + {"ac-targeting", "ac-targeting", NID_ac_targeting, 8, &so[1928]}, + {"aaControls", "aaControls", NID_aaControls, 8, &so[1936]}, + {"sbgp-ipAddrBlock", "sbgp-ipAddrBlock", NID_sbgp_ipAddrBlock, 8, &so[1944]}, + {"sbgp-autonomousSysNum", "sbgp-autonomousSysNum", NID_sbgp_autonomousSysNum, 8, &so[1952]}, + {"sbgp-routerIdentifier", "sbgp-routerIdentifier", NID_sbgp_routerIdentifier, 8, &so[1960]}, + {"textNotice", "textNotice", NID_textNotice, 8, &so[1968]}, + {"ipsecEndSystem", "IPSec End System", NID_ipsecEndSystem, 8, &so[1976]}, + {"ipsecTunnel", "IPSec Tunnel", NID_ipsecTunnel, 8, &so[1984]}, + {"ipsecUser", "IPSec User", NID_ipsecUser, 8, &so[1992]}, + {"DVCS", "dvcs", NID_dvcs, 8, &so[2000]}, + {"id-it-caProtEncCert", "id-it-caProtEncCert", NID_id_it_caProtEncCert, 8, &so[2008]}, + {"id-it-signKeyPairTypes", "id-it-signKeyPairTypes", NID_id_it_signKeyPairTypes, 8, &so[2016]}, + {"id-it-encKeyPairTypes", "id-it-encKeyPairTypes", NID_id_it_encKeyPairTypes, 8, &so[2024]}, + {"id-it-preferredSymmAlg", "id-it-preferredSymmAlg", NID_id_it_preferredSymmAlg, 8, &so[2032]}, + {"id-it-caKeyUpdateInfo", "id-it-caKeyUpdateInfo", NID_id_it_caKeyUpdateInfo, 8, &so[2040]}, + {"id-it-currentCRL", "id-it-currentCRL", NID_id_it_currentCRL, 8, &so[2048]}, + {"id-it-unsupportedOIDs", "id-it-unsupportedOIDs", NID_id_it_unsupportedOIDs, 8, &so[2056]}, + {"id-it-subscriptionRequest", "id-it-subscriptionRequest", NID_id_it_subscriptionRequest, 8, &so[2064]}, + {"id-it-subscriptionResponse", "id-it-subscriptionResponse", NID_id_it_subscriptionResponse, 8, &so[2072]}, + {"id-it-keyPairParamReq", "id-it-keyPairParamReq", NID_id_it_keyPairParamReq, 8, &so[2080]}, + {"id-it-keyPairParamRep", "id-it-keyPairParamRep", NID_id_it_keyPairParamRep, 8, &so[2088]}, + {"id-it-revPassphrase", "id-it-revPassphrase", NID_id_it_revPassphrase, 8, &so[2096]}, + {"id-it-implicitConfirm", "id-it-implicitConfirm", NID_id_it_implicitConfirm, 8, &so[2104]}, + {"id-it-confirmWaitTime", "id-it-confirmWaitTime", NID_id_it_confirmWaitTime, 8, &so[2112]}, + {"id-it-origPKIMessage", "id-it-origPKIMessage", NID_id_it_origPKIMessage, 8, &so[2120]}, + {"id-regCtrl", "id-regCtrl", NID_id_regCtrl, 8, &so[2128]}, + {"id-regInfo", "id-regInfo", NID_id_regInfo, 8, &so[2136]}, + {"id-regCtrl-regToken", "id-regCtrl-regToken", NID_id_regCtrl_regToken, 9, &so[2144]}, + {"id-regCtrl-authenticator", "id-regCtrl-authenticator", NID_id_regCtrl_authenticator, 9, &so[2153]}, + {"id-regCtrl-pkiPublicationInfo", "id-regCtrl-pkiPublicationInfo", NID_id_regCtrl_pkiPublicationInfo, 9, &so[2162]}, + {"id-regCtrl-pkiArchiveOptions", "id-regCtrl-pkiArchiveOptions", NID_id_regCtrl_pkiArchiveOptions, 9, &so[2171]}, + {"id-regCtrl-oldCertID", "id-regCtrl-oldCertID", NID_id_regCtrl_oldCertID, 9, &so[2180]}, + {"id-regCtrl-protocolEncrKey", "id-regCtrl-protocolEncrKey", NID_id_regCtrl_protocolEncrKey, 9, &so[2189]}, + {"id-regInfo-utf8Pairs", "id-regInfo-utf8Pairs", NID_id_regInfo_utf8Pairs, 9, &so[2198]}, + {"id-regInfo-certReq", "id-regInfo-certReq", NID_id_regInfo_certReq, 9, &so[2207]}, + {"id-alg-des40", "id-alg-des40", NID_id_alg_des40, 8, &so[2216]}, + {"id-alg-noSignature", "id-alg-noSignature", NID_id_alg_noSignature, 8, &so[2224]}, + {"id-alg-dh-sig-hmac-sha1", "id-alg-dh-sig-hmac-sha1", NID_id_alg_dh_sig_hmac_sha1, 8, &so[2232]}, + {"id-alg-dh-pop", "id-alg-dh-pop", NID_id_alg_dh_pop, 8, &so[2240]}, + {"id-cmc-statusInfo", "id-cmc-statusInfo", NID_id_cmc_statusInfo, 8, &so[2248]}, + {"id-cmc-identification", "id-cmc-identification", NID_id_cmc_identification, 8, &so[2256]}, + {"id-cmc-identityProof", "id-cmc-identityProof", NID_id_cmc_identityProof, 8, &so[2264]}, + {"id-cmc-dataReturn", "id-cmc-dataReturn", NID_id_cmc_dataReturn, 8, &so[2272]}, + {"id-cmc-transactionId", "id-cmc-transactionId", NID_id_cmc_transactionId, 8, &so[2280]}, + {"id-cmc-senderNonce", "id-cmc-senderNonce", NID_id_cmc_senderNonce, 8, &so[2288]}, + {"id-cmc-recipientNonce", "id-cmc-recipientNonce", NID_id_cmc_recipientNonce, 8, &so[2296]}, + {"id-cmc-addExtensions", "id-cmc-addExtensions", NID_id_cmc_addExtensions, 8, &so[2304]}, + {"id-cmc-encryptedPOP", "id-cmc-encryptedPOP", NID_id_cmc_encryptedPOP, 8, &so[2312]}, + {"id-cmc-decryptedPOP", "id-cmc-decryptedPOP", NID_id_cmc_decryptedPOP, 8, &so[2320]}, + {"id-cmc-lraPOPWitness", "id-cmc-lraPOPWitness", NID_id_cmc_lraPOPWitness, 8, &so[2328]}, + {"id-cmc-getCert", "id-cmc-getCert", NID_id_cmc_getCert, 8, &so[2336]}, + {"id-cmc-getCRL", "id-cmc-getCRL", NID_id_cmc_getCRL, 8, &so[2344]}, + {"id-cmc-revokeRequest", "id-cmc-revokeRequest", NID_id_cmc_revokeRequest, 8, &so[2352]}, + {"id-cmc-regInfo", "id-cmc-regInfo", NID_id_cmc_regInfo, 8, &so[2360]}, + {"id-cmc-responseInfo", "id-cmc-responseInfo", NID_id_cmc_responseInfo, 8, &so[2368]}, + {"id-cmc-queryPending", "id-cmc-queryPending", NID_id_cmc_queryPending, 8, &so[2376]}, + {"id-cmc-popLinkRandom", "id-cmc-popLinkRandom", NID_id_cmc_popLinkRandom, 8, &so[2384]}, + {"id-cmc-popLinkWitness", "id-cmc-popLinkWitness", NID_id_cmc_popLinkWitness, 8, &so[2392]}, + {"id-cmc-confirmCertAcceptance", "id-cmc-confirmCertAcceptance", NID_id_cmc_confirmCertAcceptance, 8, &so[2400]}, + {"id-on-personalData", "id-on-personalData", NID_id_on_personalData, 8, &so[2408]}, + {"id-pda-dateOfBirth", "id-pda-dateOfBirth", NID_id_pda_dateOfBirth, 8, &so[2416]}, + {"id-pda-placeOfBirth", "id-pda-placeOfBirth", NID_id_pda_placeOfBirth, 8, &so[2424]}, + { NULL, NULL, NID_undef }, + {"id-pda-gender", "id-pda-gender", NID_id_pda_gender, 8, &so[2432]}, + {"id-pda-countryOfCitizenship", "id-pda-countryOfCitizenship", NID_id_pda_countryOfCitizenship, 8, &so[2440]}, + {"id-pda-countryOfResidence", "id-pda-countryOfResidence", NID_id_pda_countryOfResidence, 8, &so[2448]}, + {"id-aca-authenticationInfo", "id-aca-authenticationInfo", NID_id_aca_authenticationInfo, 8, &so[2456]}, + {"id-aca-accessIdentity", "id-aca-accessIdentity", NID_id_aca_accessIdentity, 8, &so[2464]}, + {"id-aca-chargingIdentity", "id-aca-chargingIdentity", NID_id_aca_chargingIdentity, 8, &so[2472]}, + {"id-aca-group", "id-aca-group", NID_id_aca_group, 8, &so[2480]}, + {"id-aca-role", "id-aca-role", NID_id_aca_role, 8, &so[2488]}, + {"id-qcs-pkixQCSyntax-v1", "id-qcs-pkixQCSyntax-v1", NID_id_qcs_pkixQCSyntax_v1, 8, &so[2496]}, + {"id-cct-crs", "id-cct-crs", NID_id_cct_crs, 8, &so[2504]}, + {"id-cct-PKIData", "id-cct-PKIData", NID_id_cct_PKIData, 8, &so[2512]}, + {"id-cct-PKIResponse", "id-cct-PKIResponse", NID_id_cct_PKIResponse, 8, &so[2520]}, + {"ad_timestamping", "AD Time Stamping", NID_ad_timeStamping, 8, &so[2528]}, + {"AD_DVCS", "ad dvcs", NID_ad_dvcs, 8, &so[2536]}, + {"basicOCSPResponse", "Basic OCSP Response", NID_id_pkix_OCSP_basic, 9, &so[2544]}, + {"Nonce", "OCSP Nonce", NID_id_pkix_OCSP_Nonce, 9, &so[2553]}, + {"CrlID", "OCSP CRL ID", NID_id_pkix_OCSP_CrlID, 9, &so[2562]}, + {"acceptableResponses", "Acceptable OCSP Responses", NID_id_pkix_OCSP_acceptableResponses, 9, &so[2571]}, + {"noCheck", "OCSP No Check", NID_id_pkix_OCSP_noCheck, 9, &so[2580]}, + {"archiveCutoff", "OCSP Archive Cutoff", NID_id_pkix_OCSP_archiveCutoff, 9, &so[2589]}, + {"serviceLocator", "OCSP Service Locator", NID_id_pkix_OCSP_serviceLocator, 9, &so[2598]}, + {"extendedStatus", "Extended OCSP Status", NID_id_pkix_OCSP_extendedStatus, 9, &so[2607]}, + {"valid", "valid", NID_id_pkix_OCSP_valid, 9, &so[2616]}, + {"path", "path", NID_id_pkix_OCSP_path, 9, &so[2625]}, + {"trustRoot", "Trust Root", NID_id_pkix_OCSP_trustRoot, 9, &so[2634]}, + {"algorithm", "algorithm", NID_algorithm, 4, &so[2643]}, + {"rsaSignature", "rsaSignature", NID_rsaSignature, 5, &so[2647]}, + {"X500algorithms", "directory services - algorithms", NID_X500algorithms, 2, &so[2652]}, + {"ORG", "org", NID_org, 1, &so[2654]}, + {"DOD", "dod", NID_dod, 2, &so[2655]}, + {"IANA", "iana", NID_iana, 3, &so[2657]}, + {"directory", "Directory", NID_Directory, 4, &so[2660]}, + {"mgmt", "Management", NID_Management, 4, &so[2664]}, + {"experimental", "Experimental", NID_Experimental, 4, &so[2668]}, + {"private", "Private", NID_Private, 4, &so[2672]}, + {"security", "Security", NID_Security, 4, &so[2676]}, + {"snmpv2", "SNMPv2", NID_SNMPv2, 4, &so[2680]}, + {"Mail", "Mail", NID_Mail, 4, &so[2684]}, + {"enterprises", "Enterprises", NID_Enterprises, 5, &so[2688]}, + {"dcobject", "dcObject", NID_dcObject, 9, &so[2693]}, + {"DC", "domainComponent", NID_domainComponent, 10, &so[2702]}, + {"domain", "Domain", NID_Domain, 10, &so[2712]}, {"NULL", "NULL", NID_joint_iso_ccitt}, - {"selected-attribute-types", "Selected Attribute Types", NID_selected_attribute_types, 3, &so[2901]}, - {"clearance", "clearance", NID_clearance, 4, &so[2904]}, - {"RSA-MD4", "md4WithRSAEncryption", NID_md4WithRSAEncryption, 9, &so[2908]}, - {"ac-proxying", "ac-proxying", NID_ac_proxying, 8, &so[2917]}, - {"subjectInfoAccess", "Subject Information Access", NID_sinfo_access, 8, &so[2925]}, - {"id-aca-encAttrs", "id-aca-encAttrs", NID_id_aca_encAttrs, 8, &so[2933]}, - {"role", "role", NID_role, 3, &so[2941]}, - {"policyConstraints", "X509v3 Policy Constraints", NID_policy_constraints, 3, &so[2944]}, - {"targetInformation", "X509v3 AC Targeting", NID_target_information, 3, &so[2947]}, - {"noRevAvail", "X509v3 No Revocation Available", NID_no_rev_avail, 3, &so[2950]}, + {"selected-attribute-types", "Selected Attribute Types", NID_selected_attribute_types, 3, &so[2722]}, + {"clearance", "clearance", NID_clearance, 4, &so[2725]}, + { NULL, NULL, NID_undef }, + {"ac-proxying", "ac-proxying", NID_ac_proxying, 8, &so[2729]}, + {"subjectInfoAccess", "Subject Information Access", NID_sinfo_access, 8, &so[2737]}, + {"id-aca-encAttrs", "id-aca-encAttrs", NID_id_aca_encAttrs, 8, &so[2745]}, + {"role", "role", NID_role, 3, &so[2753]}, + {"policyConstraints", "X509v3 Policy Constraints", NID_policy_constraints, 3, &so[2756]}, + {"targetInformation", "X509v3 AC Targeting", NID_target_information, 3, &so[2759]}, + {"noRevAvail", "X509v3 No Revocation Available", NID_no_rev_avail, 3, &so[2762]}, {"NULL", "NULL", NID_ccitt}, - {"ansi-X9-62", "ANSI X9.62", NID_ansi_X9_62, 5, &so[2953]}, - {"prime-field", "prime-field", NID_X9_62_prime_field, 7, &so[2958]}, - {"characteristic-two-field", "characteristic-two-field", NID_X9_62_characteristic_two_field, 7, &so[2965]}, - {"id-ecPublicKey", "id-ecPublicKey", NID_X9_62_id_ecPublicKey, 7, &so[2972]}, - {"prime192v1", "prime192v1", NID_X9_62_prime192v1, 8, &so[2979]}, - {"prime192v2", "prime192v2", NID_X9_62_prime192v2, 8, &so[2987]}, - {"prime192v3", "prime192v3", NID_X9_62_prime192v3, 8, &so[2995]}, - {"prime239v1", "prime239v1", NID_X9_62_prime239v1, 8, &so[3003]}, - {"prime239v2", "prime239v2", NID_X9_62_prime239v2, 8, &so[3011]}, - {"prime239v3", "prime239v3", NID_X9_62_prime239v3, 8, &so[3019]}, - {"prime256v1", "prime256v1", NID_X9_62_prime256v1, 8, &so[3027]}, - {"ecdsa-with-SHA1", "ecdsa-with-SHA1", NID_ecdsa_with_SHA1, 7, &so[3035]}, - {"CSPName", "Microsoft CSP Name", NID_ms_csp_name, 9, &so[3042]}, - {"AES-128-ECB", "aes-128-ecb", NID_aes_128_ecb, 9, &so[3051]}, - {"AES-128-CBC", "aes-128-cbc", NID_aes_128_cbc, 9, &so[3060]}, - {"AES-128-OFB", "aes-128-ofb", NID_aes_128_ofb128, 9, &so[3069]}, - {"AES-128-CFB", "aes-128-cfb", NID_aes_128_cfb128, 9, &so[3078]}, - {"AES-192-ECB", "aes-192-ecb", NID_aes_192_ecb, 9, &so[3087]}, - {"AES-192-CBC", "aes-192-cbc", NID_aes_192_cbc, 9, &so[3096]}, - {"AES-192-OFB", "aes-192-ofb", NID_aes_192_ofb128, 9, &so[3105]}, - {"AES-192-CFB", "aes-192-cfb", NID_aes_192_cfb128, 9, &so[3114]}, - {"AES-256-ECB", "aes-256-ecb", NID_aes_256_ecb, 9, &so[3123]}, - {"AES-256-CBC", "aes-256-cbc", NID_aes_256_cbc, 9, &so[3132]}, - {"AES-256-OFB", "aes-256-ofb", NID_aes_256_ofb128, 9, &so[3141]}, - {"AES-256-CFB", "aes-256-cfb", NID_aes_256_cfb128, 9, &so[3150]}, - {"holdInstructionCode", "Hold Instruction Code", NID_hold_instruction_code, 3, &so[3159]}, - {"holdInstructionNone", "Hold Instruction None", NID_hold_instruction_none, 7, &so[3162]}, - {"holdInstructionCallIssuer", "Hold Instruction Call Issuer", NID_hold_instruction_call_issuer, 7, &so[3169]}, - {"holdInstructionReject", "Hold Instruction Reject", NID_hold_instruction_reject, 7, &so[3176]}, - {"data", "data", NID_data, 1, &so[3183]}, - {"pss", "pss", NID_pss, 3, &so[3184]}, - {"ucl", "ucl", NID_ucl, 7, &so[3187]}, - {"pilot", "pilot", NID_pilot, 8, &so[3194]}, - {"pilotAttributeType", "pilotAttributeType", NID_pilotAttributeType, 9, &so[3202]}, - {"pilotAttributeSyntax", "pilotAttributeSyntax", NID_pilotAttributeSyntax, 9, &so[3211]}, - {"pilotObjectClass", "pilotObjectClass", NID_pilotObjectClass, 9, &so[3220]}, - {"pilotGroups", "pilotGroups", NID_pilotGroups, 9, &so[3229]}, - {"iA5StringSyntax", "iA5StringSyntax", NID_iA5StringSyntax, 10, &so[3238]}, - {"caseIgnoreIA5StringSyntax", "caseIgnoreIA5StringSyntax", NID_caseIgnoreIA5StringSyntax, 10, &so[3248]}, - {"pilotObject", "pilotObject", NID_pilotObject, 10, &so[3258]}, - {"pilotPerson", "pilotPerson", NID_pilotPerson, 10, &so[3268]}, - {"account", "account", NID_account, 10, &so[3278]}, - {"document", "document", NID_document, 10, &so[3288]}, - {"room", "room", NID_room, 10, &so[3298]}, - {"documentSeries", "documentSeries", NID_documentSeries, 10, &so[3308]}, - {"rFC822localPart", "rFC822localPart", NID_rFC822localPart, 10, &so[3318]}, - {"dNSDomain", "dNSDomain", NID_dNSDomain, 10, &so[3328]}, - {"domainRelatedObject", "domainRelatedObject", NID_domainRelatedObject, 10, &so[3338]}, - {"friendlyCountry", "friendlyCountry", NID_friendlyCountry, 10, &so[3348]}, - {"simpleSecurityObject", "simpleSecurityObject", NID_simpleSecurityObject, 10, &so[3358]}, - {"pilotOrganization", "pilotOrganization", NID_pilotOrganization, 10, &so[3368]}, - {"pilotDSA", "pilotDSA", NID_pilotDSA, 10, &so[3378]}, - {"qualityLabelledData", "qualityLabelledData", NID_qualityLabelledData, 10, &so[3388]}, - {"UID", "userId", NID_userId, 10, &so[3398]}, - {"textEncodedORAddress", "textEncodedORAddress", NID_textEncodedORAddress, 10, &so[3408]}, - {"mail", "rfc822Mailbox", NID_rfc822Mailbox, 10, &so[3418]}, - {"info", "info", NID_info, 10, &so[3428]}, - {"favouriteDrink", "favouriteDrink", NID_favouriteDrink, 10, &so[3438]}, - {"roomNumber", "roomNumber", NID_roomNumber, 10, &so[3448]}, - {"photo", "photo", NID_photo, 10, &so[3458]}, - {"userClass", "userClass", NID_userClass, 10, &so[3468]}, - {"host", "host", NID_host, 10, &so[3478]}, - {"manager", "manager", NID_manager, 10, &so[3488]}, - {"documentIdentifier", "documentIdentifier", NID_documentIdentifier, 10, &so[3498]}, - {"documentTitle", "documentTitle", NID_documentTitle, 10, &so[3508]}, - {"documentVersion", "documentVersion", NID_documentVersion, 10, &so[3518]}, - {"documentAuthor", "documentAuthor", NID_documentAuthor, 10, &so[3528]}, - {"documentLocation", "documentLocation", NID_documentLocation, 10, &so[3538]}, - {"homeTelephoneNumber", "homeTelephoneNumber", NID_homeTelephoneNumber, 10, &so[3548]}, - {"secretary", "secretary", NID_secretary, 10, &so[3558]}, - {"otherMailbox", "otherMailbox", NID_otherMailbox, 10, &so[3568]}, - {"lastModifiedTime", "lastModifiedTime", NID_lastModifiedTime, 10, &so[3578]}, - {"lastModifiedBy", "lastModifiedBy", NID_lastModifiedBy, 10, &so[3588]}, - {"aRecord", "aRecord", NID_aRecord, 10, &so[3598]}, - {"pilotAttributeType27", "pilotAttributeType27", NID_pilotAttributeType27, 10, &so[3608]}, - {"mXRecord", "mXRecord", NID_mXRecord, 10, &so[3618]}, - {"nSRecord", "nSRecord", NID_nSRecord, 10, &so[3628]}, - {"sOARecord", "sOARecord", NID_sOARecord, 10, &so[3638]}, - {"cNAMERecord", "cNAMERecord", NID_cNAMERecord, 10, &so[3648]}, - {"associatedDomain", "associatedDomain", NID_associatedDomain, 10, &so[3658]}, - {"associatedName", "associatedName", NID_associatedName, 10, &so[3668]}, - {"homePostalAddress", "homePostalAddress", NID_homePostalAddress, 10, &so[3678]}, - {"personalTitle", "personalTitle", NID_personalTitle, 10, &so[3688]}, - {"mobileTelephoneNumber", "mobileTelephoneNumber", NID_mobileTelephoneNumber, 10, &so[3698]}, - {"pagerTelephoneNumber", "pagerTelephoneNumber", NID_pagerTelephoneNumber, 10, &so[3708]}, - {"friendlyCountryName", "friendlyCountryName", NID_friendlyCountryName, 10, &so[3718]}, - {"organizationalStatus", "organizationalStatus", NID_organizationalStatus, 10, &so[3728]}, - {"janetMailbox", "janetMailbox", NID_janetMailbox, 10, &so[3738]}, - {"mailPreferenceOption", "mailPreferenceOption", NID_mailPreferenceOption, 10, &so[3748]}, - {"buildingName", "buildingName", NID_buildingName, 10, &so[3758]}, - {"dSAQuality", "dSAQuality", NID_dSAQuality, 10, &so[3768]}, - {"singleLevelQuality", "singleLevelQuality", NID_singleLevelQuality, 10, &so[3778]}, - {"subtreeMinimumQuality", "subtreeMinimumQuality", NID_subtreeMinimumQuality, 10, &so[3788]}, - {"subtreeMaximumQuality", "subtreeMaximumQuality", NID_subtreeMaximumQuality, 10, &so[3798]}, - {"personalSignature", "personalSignature", NID_personalSignature, 10, &so[3808]}, - {"dITRedirect", "dITRedirect", NID_dITRedirect, 10, &so[3818]}, - {"audio", "audio", NID_audio, 10, &so[3828]}, - {"documentPublisher", "documentPublisher", NID_documentPublisher, 10, &so[3838]}, - {"x500UniqueIdentifier", "x500UniqueIdentifier", NID_x500UniqueIdentifier, 3, &so[3848]}, - {"mime-mhs", "MIME MHS", NID_mime_mhs, 5, &so[3851]}, - {"mime-mhs-headings", "mime-mhs-headings", NID_mime_mhs_headings, 6, &so[3856]}, - {"mime-mhs-bodies", "mime-mhs-bodies", NID_mime_mhs_bodies, 6, &so[3862]}, - {"id-hex-partial-message", "id-hex-partial-message", NID_id_hex_partial_message, 7, &so[3868]}, - {"id-hex-multipart-message", "id-hex-multipart-message", NID_id_hex_multipart_message, 7, &so[3875]}, - {"generationQualifier", "generationQualifier", NID_generationQualifier, 3, &so[3882]}, - {"pseudonym", "pseudonym", NID_pseudonym, 3, &so[3885]}, - { NULL, NULL, NID_undef }, - {"id-set", "Secure Electronic Transactions", NID_id_set, 2, &so[3888]}, - {"set-ctype", "content types", NID_set_ctype, 3, &so[3890]}, - {"set-msgExt", "message extensions", NID_set_msgExt, 3, &so[3893]}, - {"set-attr", "set-attr", NID_set_attr, 3, &so[3896]}, - {"set-policy", "set-policy", NID_set_policy, 3, &so[3899]}, - {"set-certExt", "certificate extensions", NID_set_certExt, 3, &so[3902]}, - {"set-brand", "set-brand", NID_set_brand, 3, &so[3905]}, - {"setct-PANData", "setct-PANData", NID_setct_PANData, 4, &so[3908]}, - {"setct-PANToken", "setct-PANToken", NID_setct_PANToken, 4, &so[3912]}, - {"setct-PANOnly", "setct-PANOnly", NID_setct_PANOnly, 4, &so[3916]}, - {"setct-OIData", "setct-OIData", NID_setct_OIData, 4, &so[3920]}, - {"setct-PI", "setct-PI", NID_setct_PI, 4, &so[3924]}, - {"setct-PIData", "setct-PIData", NID_setct_PIData, 4, &so[3928]}, - {"setct-PIDataUnsigned", "setct-PIDataUnsigned", NID_setct_PIDataUnsigned, 4, &so[3932]}, - {"setct-HODInput", "setct-HODInput", NID_setct_HODInput, 4, &so[3936]}, - {"setct-AuthResBaggage", "setct-AuthResBaggage", NID_setct_AuthResBaggage, 4, &so[3940]}, - {"setct-AuthRevReqBaggage", "setct-AuthRevReqBaggage", NID_setct_AuthRevReqBaggage, 4, &so[3944]}, - {"setct-AuthRevResBaggage", "setct-AuthRevResBaggage", NID_setct_AuthRevResBaggage, 4, &so[3948]}, - {"setct-CapTokenSeq", "setct-CapTokenSeq", NID_setct_CapTokenSeq, 4, &so[3952]}, - {"setct-PInitResData", "setct-PInitResData", NID_setct_PInitResData, 4, &so[3956]}, - {"setct-PI-TBS", "setct-PI-TBS", NID_setct_PI_TBS, 4, &so[3960]}, - {"setct-PResData", "setct-PResData", NID_setct_PResData, 4, &so[3964]}, - {"setct-AuthReqTBS", "setct-AuthReqTBS", NID_setct_AuthReqTBS, 4, &so[3968]}, - {"setct-AuthResTBS", "setct-AuthResTBS", NID_setct_AuthResTBS, 4, &so[3972]}, - {"setct-AuthResTBSX", "setct-AuthResTBSX", NID_setct_AuthResTBSX, 4, &so[3976]}, - {"setct-AuthTokenTBS", "setct-AuthTokenTBS", NID_setct_AuthTokenTBS, 4, &so[3980]}, - {"setct-CapTokenData", "setct-CapTokenData", NID_setct_CapTokenData, 4, &so[3984]}, - {"setct-CapTokenTBS", "setct-CapTokenTBS", NID_setct_CapTokenTBS, 4, &so[3988]}, - {"setct-AcqCardCodeMsg", "setct-AcqCardCodeMsg", NID_setct_AcqCardCodeMsg, 4, &so[3992]}, - {"setct-AuthRevReqTBS", "setct-AuthRevReqTBS", NID_setct_AuthRevReqTBS, 4, &so[3996]}, - {"setct-AuthRevResData", "setct-AuthRevResData", NID_setct_AuthRevResData, 4, &so[4000]}, - {"setct-AuthRevResTBS", "setct-AuthRevResTBS", NID_setct_AuthRevResTBS, 4, &so[4004]}, - {"setct-CapReqTBS", "setct-CapReqTBS", NID_setct_CapReqTBS, 4, &so[4008]}, - {"setct-CapReqTBSX", "setct-CapReqTBSX", NID_setct_CapReqTBSX, 4, &so[4012]}, - {"setct-CapResData", "setct-CapResData", NID_setct_CapResData, 4, &so[4016]}, - {"setct-CapRevReqTBS", "setct-CapRevReqTBS", NID_setct_CapRevReqTBS, 4, &so[4020]}, - {"setct-CapRevReqTBSX", "setct-CapRevReqTBSX", NID_setct_CapRevReqTBSX, 4, &so[4024]}, - {"setct-CapRevResData", "setct-CapRevResData", NID_setct_CapRevResData, 4, &so[4028]}, - {"setct-CredReqTBS", "setct-CredReqTBS", NID_setct_CredReqTBS, 4, &so[4032]}, - {"setct-CredReqTBSX", "setct-CredReqTBSX", NID_setct_CredReqTBSX, 4, &so[4036]}, - {"setct-CredResData", "setct-CredResData", NID_setct_CredResData, 4, &so[4040]}, - {"setct-CredRevReqTBS", "setct-CredRevReqTBS", NID_setct_CredRevReqTBS, 4, &so[4044]}, - {"setct-CredRevReqTBSX", "setct-CredRevReqTBSX", NID_setct_CredRevReqTBSX, 4, &so[4048]}, - {"setct-CredRevResData", "setct-CredRevResData", NID_setct_CredRevResData, 4, &so[4052]}, - {"setct-PCertReqData", "setct-PCertReqData", NID_setct_PCertReqData, 4, &so[4056]}, - {"setct-PCertResTBS", "setct-PCertResTBS", NID_setct_PCertResTBS, 4, &so[4060]}, - {"setct-BatchAdminReqData", "setct-BatchAdminReqData", NID_setct_BatchAdminReqData, 4, &so[4064]}, - {"setct-BatchAdminResData", "setct-BatchAdminResData", NID_setct_BatchAdminResData, 4, &so[4068]}, - {"setct-CardCInitResTBS", "setct-CardCInitResTBS", NID_setct_CardCInitResTBS, 4, &so[4072]}, - {"setct-MeAqCInitResTBS", "setct-MeAqCInitResTBS", NID_setct_MeAqCInitResTBS, 4, &so[4076]}, - {"setct-RegFormResTBS", "setct-RegFormResTBS", NID_setct_RegFormResTBS, 4, &so[4080]}, - {"setct-CertReqData", "setct-CertReqData", NID_setct_CertReqData, 4, &so[4084]}, - {"setct-CertReqTBS", "setct-CertReqTBS", NID_setct_CertReqTBS, 4, &so[4088]}, - {"setct-CertResData", "setct-CertResData", NID_setct_CertResData, 4, &so[4092]}, - {"setct-CertInqReqTBS", "setct-CertInqReqTBS", NID_setct_CertInqReqTBS, 4, &so[4096]}, - {"setct-ErrorTBS", "setct-ErrorTBS", NID_setct_ErrorTBS, 4, &so[4100]}, - {"setct-PIDualSignedTBE", "setct-PIDualSignedTBE", NID_setct_PIDualSignedTBE, 4, &so[4104]}, - {"setct-PIUnsignedTBE", "setct-PIUnsignedTBE", NID_setct_PIUnsignedTBE, 4, &so[4108]}, - {"setct-AuthReqTBE", "setct-AuthReqTBE", NID_setct_AuthReqTBE, 4, &so[4112]}, - {"setct-AuthResTBE", "setct-AuthResTBE", NID_setct_AuthResTBE, 4, &so[4116]}, - {"setct-AuthResTBEX", "setct-AuthResTBEX", NID_setct_AuthResTBEX, 4, &so[4120]}, - {"setct-AuthTokenTBE", "setct-AuthTokenTBE", NID_setct_AuthTokenTBE, 4, &so[4124]}, - {"setct-CapTokenTBE", "setct-CapTokenTBE", NID_setct_CapTokenTBE, 4, &so[4128]}, - {"setct-CapTokenTBEX", "setct-CapTokenTBEX", NID_setct_CapTokenTBEX, 4, &so[4132]}, - {"setct-AcqCardCodeMsgTBE", "setct-AcqCardCodeMsgTBE", NID_setct_AcqCardCodeMsgTBE, 4, &so[4136]}, - {"setct-AuthRevReqTBE", "setct-AuthRevReqTBE", NID_setct_AuthRevReqTBE, 4, &so[4140]}, - {"setct-AuthRevResTBE", "setct-AuthRevResTBE", NID_setct_AuthRevResTBE, 4, &so[4144]}, - {"setct-AuthRevResTBEB", "setct-AuthRevResTBEB", NID_setct_AuthRevResTBEB, 4, &so[4148]}, - {"setct-CapReqTBE", "setct-CapReqTBE", NID_setct_CapReqTBE, 4, &so[4152]}, - {"setct-CapReqTBEX", "setct-CapReqTBEX", NID_setct_CapReqTBEX, 4, &so[4156]}, - {"setct-CapResTBE", "setct-CapResTBE", NID_setct_CapResTBE, 4, &so[4160]}, - {"setct-CapRevReqTBE", "setct-CapRevReqTBE", NID_setct_CapRevReqTBE, 4, &so[4164]}, - {"setct-CapRevReqTBEX", "setct-CapRevReqTBEX", NID_setct_CapRevReqTBEX, 4, &so[4168]}, - {"setct-CapRevResTBE", "setct-CapRevResTBE", NID_setct_CapRevResTBE, 4, &so[4172]}, - {"setct-CredReqTBE", "setct-CredReqTBE", NID_setct_CredReqTBE, 4, &so[4176]}, - {"setct-CredReqTBEX", "setct-CredReqTBEX", NID_setct_CredReqTBEX, 4, &so[4180]}, - {"setct-CredResTBE", "setct-CredResTBE", NID_setct_CredResTBE, 4, &so[4184]}, - {"setct-CredRevReqTBE", "setct-CredRevReqTBE", NID_setct_CredRevReqTBE, 4, &so[4188]}, - {"setct-CredRevReqTBEX", "setct-CredRevReqTBEX", NID_setct_CredRevReqTBEX, 4, &so[4192]}, - {"setct-CredRevResTBE", "setct-CredRevResTBE", NID_setct_CredRevResTBE, 4, &so[4196]}, - {"setct-BatchAdminReqTBE", "setct-BatchAdminReqTBE", NID_setct_BatchAdminReqTBE, 4, &so[4200]}, - {"setct-BatchAdminResTBE", "setct-BatchAdminResTBE", NID_setct_BatchAdminResTBE, 4, &so[4204]}, - {"setct-RegFormReqTBE", "setct-RegFormReqTBE", NID_setct_RegFormReqTBE, 4, &so[4208]}, - {"setct-CertReqTBE", "setct-CertReqTBE", NID_setct_CertReqTBE, 4, &so[4212]}, - {"setct-CertReqTBEX", "setct-CertReqTBEX", NID_setct_CertReqTBEX, 4, &so[4216]}, - {"setct-CertResTBE", "setct-CertResTBE", NID_setct_CertResTBE, 4, &so[4220]}, - {"setct-CRLNotificationTBS", "setct-CRLNotificationTBS", NID_setct_CRLNotificationTBS, 4, &so[4224]}, - {"setct-CRLNotificationResTBS", "setct-CRLNotificationResTBS", NID_setct_CRLNotificationResTBS, 4, &so[4228]}, - {"setct-BCIDistributionTBS", "setct-BCIDistributionTBS", NID_setct_BCIDistributionTBS, 4, &so[4232]}, - {"setext-genCrypt", "generic cryptogram", NID_setext_genCrypt, 4, &so[4236]}, - {"setext-miAuth", "merchant initiated auth", NID_setext_miAuth, 4, &so[4240]}, - {"setext-pinSecure", "setext-pinSecure", NID_setext_pinSecure, 4, &so[4244]}, - {"setext-pinAny", "setext-pinAny", NID_setext_pinAny, 4, &so[4248]}, - {"setext-track2", "setext-track2", NID_setext_track2, 4, &so[4252]}, - {"setext-cv", "additional verification", NID_setext_cv, 4, &so[4256]}, - {"set-policy-root", "set-policy-root", NID_set_policy_root, 4, &so[4260]}, - {"setCext-hashedRoot", "setCext-hashedRoot", NID_setCext_hashedRoot, 4, &so[4264]}, - {"setCext-certType", "setCext-certType", NID_setCext_certType, 4, &so[4268]}, - {"setCext-merchData", "setCext-merchData", NID_setCext_merchData, 4, &so[4272]}, - {"setCext-cCertRequired", "setCext-cCertRequired", NID_setCext_cCertRequired, 4, &so[4276]}, - {"setCext-tunneling", "setCext-tunneling", NID_setCext_tunneling, 4, &so[4280]}, - {"setCext-setExt", "setCext-setExt", NID_setCext_setExt, 4, &so[4284]}, - {"setCext-setQualf", "setCext-setQualf", NID_setCext_setQualf, 4, &so[4288]}, - {"setCext-PGWYcapabilities", "setCext-PGWYcapabilities", NID_setCext_PGWYcapabilities, 4, &so[4292]}, - {"setCext-TokenIdentifier", "setCext-TokenIdentifier", NID_setCext_TokenIdentifier, 4, &so[4296]}, - {"setCext-Track2Data", "setCext-Track2Data", NID_setCext_Track2Data, 4, &so[4300]}, - {"setCext-TokenType", "setCext-TokenType", NID_setCext_TokenType, 4, &so[4304]}, - {"setCext-IssuerCapabilities", "setCext-IssuerCapabilities", NID_setCext_IssuerCapabilities, 4, &so[4308]}, - {"setAttr-Cert", "setAttr-Cert", NID_setAttr_Cert, 4, &so[4312]}, - {"setAttr-PGWYcap", "payment gateway capabilities", NID_setAttr_PGWYcap, 4, &so[4316]}, - {"setAttr-TokenType", "setAttr-TokenType", NID_setAttr_TokenType, 4, &so[4320]}, - {"setAttr-IssCap", "issuer capabilities", NID_setAttr_IssCap, 4, &so[4324]}, - {"set-rootKeyThumb", "set-rootKeyThumb", NID_set_rootKeyThumb, 5, &so[4328]}, - {"set-addPolicy", "set-addPolicy", NID_set_addPolicy, 5, &so[4333]}, - {"setAttr-Token-EMV", "setAttr-Token-EMV", NID_setAttr_Token_EMV, 5, &so[4338]}, - {"setAttr-Token-B0Prime", "setAttr-Token-B0Prime", NID_setAttr_Token_B0Prime, 5, &so[4343]}, - {"setAttr-IssCap-CVM", "setAttr-IssCap-CVM", NID_setAttr_IssCap_CVM, 5, &so[4348]}, - {"setAttr-IssCap-T2", "setAttr-IssCap-T2", NID_setAttr_IssCap_T2, 5, &so[4353]}, - {"setAttr-IssCap-Sig", "setAttr-IssCap-Sig", NID_setAttr_IssCap_Sig, 5, &so[4358]}, - {"setAttr-GenCryptgrm", "generate cryptogram", NID_setAttr_GenCryptgrm, 6, &so[4363]}, - {"setAttr-T2Enc", "encrypted track 2", NID_setAttr_T2Enc, 6, &so[4369]}, - {"setAttr-T2cleartxt", "cleartext track 2", NID_setAttr_T2cleartxt, 6, &so[4375]}, - {"setAttr-TokICCsig", "ICC or token signature", NID_setAttr_TokICCsig, 6, &so[4381]}, - {"setAttr-SecDevSig", "secure device signature", NID_setAttr_SecDevSig, 6, &so[4387]}, - {"set-brand-IATA-ATA", "set-brand-IATA-ATA", NID_set_brand_IATA_ATA, 4, &so[4393]}, - {"set-brand-Diners", "set-brand-Diners", NID_set_brand_Diners, 4, &so[4397]}, - {"set-brand-AmericanExpress", "set-brand-AmericanExpress", NID_set_brand_AmericanExpress, 4, &so[4401]}, - {"set-brand-JCB", "set-brand-JCB", NID_set_brand_JCB, 4, &so[4405]}, - {"set-brand-Visa", "set-brand-Visa", NID_set_brand_Visa, 4, &so[4409]}, - {"set-brand-MasterCard", "set-brand-MasterCard", NID_set_brand_MasterCard, 4, &so[4413]}, - {"set-brand-Novus", "set-brand-Novus", NID_set_brand_Novus, 5, &so[4417]}, - {"DES-CDMF", "des-cdmf", NID_des_cdmf, 8, &so[4422]}, - {"rsaOAEPEncryptionSET", "rsaOAEPEncryptionSET", NID_rsaOAEPEncryptionSET, 9, &so[4430]}, + {"ansi-X9-62", "ANSI X9.62", NID_ansi_X9_62, 5, &so[2765]}, + {"prime-field", "prime-field", NID_X9_62_prime_field, 7, &so[2770]}, + {"characteristic-two-field", "characteristic-two-field", NID_X9_62_characteristic_two_field, 7, &so[2777]}, + {"id-ecPublicKey", "id-ecPublicKey", NID_X9_62_id_ecPublicKey, 7, &so[2784]}, + {"prime192v1", "prime192v1", NID_X9_62_prime192v1, 8, &so[2791]}, + {"prime192v2", "prime192v2", NID_X9_62_prime192v2, 8, &so[2799]}, + {"prime192v3", "prime192v3", NID_X9_62_prime192v3, 8, &so[2807]}, + {"prime239v1", "prime239v1", NID_X9_62_prime239v1, 8, &so[2815]}, + {"prime239v2", "prime239v2", NID_X9_62_prime239v2, 8, &so[2823]}, + {"prime239v3", "prime239v3", NID_X9_62_prime239v3, 8, &so[2831]}, + {"prime256v1", "prime256v1", NID_X9_62_prime256v1, 8, &so[2839]}, + {"ecdsa-with-SHA1", "ecdsa-with-SHA1", NID_ecdsa_with_SHA1, 7, &so[2847]}, + {"CSPName", "Microsoft CSP Name", NID_ms_csp_name, 9, &so[2854]}, + {"AES-128-ECB", "aes-128-ecb", NID_aes_128_ecb, 9, &so[2863]}, + {"AES-128-CBC", "aes-128-cbc", NID_aes_128_cbc, 9, &so[2872]}, + {"AES-128-OFB", "aes-128-ofb", NID_aes_128_ofb128, 9, &so[2881]}, + {"AES-128-CFB", "aes-128-cfb", NID_aes_128_cfb128, 9, &so[2890]}, + {"AES-192-ECB", "aes-192-ecb", NID_aes_192_ecb, 9, &so[2899]}, + {"AES-192-CBC", "aes-192-cbc", NID_aes_192_cbc, 9, &so[2908]}, + {"AES-192-OFB", "aes-192-ofb", NID_aes_192_ofb128, 9, &so[2917]}, + {"AES-192-CFB", "aes-192-cfb", NID_aes_192_cfb128, 9, &so[2926]}, + {"AES-256-ECB", "aes-256-ecb", NID_aes_256_ecb, 9, &so[2935]}, + {"AES-256-CBC", "aes-256-cbc", NID_aes_256_cbc, 9, &so[2944]}, + {"AES-256-OFB", "aes-256-ofb", NID_aes_256_ofb128, 9, &so[2953]}, + {"AES-256-CFB", "aes-256-cfb", NID_aes_256_cfb128, 9, &so[2962]}, + {"holdInstructionCode", "Hold Instruction Code", NID_hold_instruction_code, 3, &so[2971]}, + {"holdInstructionNone", "Hold Instruction None", NID_hold_instruction_none, 7, &so[2974]}, + {"holdInstructionCallIssuer", "Hold Instruction Call Issuer", NID_hold_instruction_call_issuer, 7, &so[2981]}, + {"holdInstructionReject", "Hold Instruction Reject", NID_hold_instruction_reject, 7, &so[2988]}, + {"data", "data", NID_data, 1, &so[2995]}, + {"pss", "pss", NID_pss, 3, &so[2996]}, + {"ucl", "ucl", NID_ucl, 7, &so[2999]}, + {"pilot", "pilot", NID_pilot, 8, &so[3006]}, + {"pilotAttributeType", "pilotAttributeType", NID_pilotAttributeType, 9, &so[3014]}, + {"pilotAttributeSyntax", "pilotAttributeSyntax", NID_pilotAttributeSyntax, 9, &so[3023]}, + {"pilotObjectClass", "pilotObjectClass", NID_pilotObjectClass, 9, &so[3032]}, + {"pilotGroups", "pilotGroups", NID_pilotGroups, 9, &so[3041]}, + {"iA5StringSyntax", "iA5StringSyntax", NID_iA5StringSyntax, 10, &so[3050]}, + {"caseIgnoreIA5StringSyntax", "caseIgnoreIA5StringSyntax", NID_caseIgnoreIA5StringSyntax, 10, &so[3060]}, + {"pilotObject", "pilotObject", NID_pilotObject, 10, &so[3070]}, + {"pilotPerson", "pilotPerson", NID_pilotPerson, 10, &so[3080]}, + {"account", "account", NID_account, 10, &so[3090]}, + {"document", "document", NID_document, 10, &so[3100]}, + {"room", "room", NID_room, 10, &so[3110]}, + {"documentSeries", "documentSeries", NID_documentSeries, 10, &so[3120]}, + {"rFC822localPart", "rFC822localPart", NID_rFC822localPart, 10, &so[3130]}, + {"dNSDomain", "dNSDomain", NID_dNSDomain, 10, &so[3140]}, + {"domainRelatedObject", "domainRelatedObject", NID_domainRelatedObject, 10, &so[3150]}, + {"friendlyCountry", "friendlyCountry", NID_friendlyCountry, 10, &so[3160]}, + {"simpleSecurityObject", "simpleSecurityObject", NID_simpleSecurityObject, 10, &so[3170]}, + {"pilotOrganization", "pilotOrganization", NID_pilotOrganization, 10, &so[3180]}, + {"pilotDSA", "pilotDSA", NID_pilotDSA, 10, &so[3190]}, + {"qualityLabelledData", "qualityLabelledData", NID_qualityLabelledData, 10, &so[3200]}, + {"UID", "userId", NID_userId, 10, &so[3210]}, + {"textEncodedORAddress", "textEncodedORAddress", NID_textEncodedORAddress, 10, &so[3220]}, + {"mail", "rfc822Mailbox", NID_rfc822Mailbox, 10, &so[3230]}, + {"info", "info", NID_info, 10, &so[3240]}, + {"favouriteDrink", "favouriteDrink", NID_favouriteDrink, 10, &so[3250]}, + {"roomNumber", "roomNumber", NID_roomNumber, 10, &so[3260]}, + {"photo", "photo", NID_photo, 10, &so[3270]}, + {"userClass", "userClass", NID_userClass, 10, &so[3280]}, + {"host", "host", NID_host, 10, &so[3290]}, + {"manager", "manager", NID_manager, 10, &so[3300]}, + {"documentIdentifier", "documentIdentifier", NID_documentIdentifier, 10, &so[3310]}, + {"documentTitle", "documentTitle", NID_documentTitle, 10, &so[3320]}, + {"documentVersion", "documentVersion", NID_documentVersion, 10, &so[3330]}, + {"documentAuthor", "documentAuthor", NID_documentAuthor, 10, &so[3340]}, + {"documentLocation", "documentLocation", NID_documentLocation, 10, &so[3350]}, + {"homeTelephoneNumber", "homeTelephoneNumber", NID_homeTelephoneNumber, 10, &so[3360]}, + {"secretary", "secretary", NID_secretary, 10, &so[3370]}, + {"otherMailbox", "otherMailbox", NID_otherMailbox, 10, &so[3380]}, + {"lastModifiedTime", "lastModifiedTime", NID_lastModifiedTime, 10, &so[3390]}, + {"lastModifiedBy", "lastModifiedBy", NID_lastModifiedBy, 10, &so[3400]}, + {"aRecord", "aRecord", NID_aRecord, 10, &so[3410]}, + {"pilotAttributeType27", "pilotAttributeType27", NID_pilotAttributeType27, 10, &so[3420]}, + {"mXRecord", "mXRecord", NID_mXRecord, 10, &so[3430]}, + {"nSRecord", "nSRecord", NID_nSRecord, 10, &so[3440]}, + {"sOARecord", "sOARecord", NID_sOARecord, 10, &so[3450]}, + {"cNAMERecord", "cNAMERecord", NID_cNAMERecord, 10, &so[3460]}, + {"associatedDomain", "associatedDomain", NID_associatedDomain, 10, &so[3470]}, + {"associatedName", "associatedName", NID_associatedName, 10, &so[3480]}, + {"homePostalAddress", "homePostalAddress", NID_homePostalAddress, 10, &so[3490]}, + {"personalTitle", "personalTitle", NID_personalTitle, 10, &so[3500]}, + {"mobileTelephoneNumber", "mobileTelephoneNumber", NID_mobileTelephoneNumber, 10, &so[3510]}, + {"pagerTelephoneNumber", "pagerTelephoneNumber", NID_pagerTelephoneNumber, 10, &so[3520]}, + {"friendlyCountryName", "friendlyCountryName", NID_friendlyCountryName, 10, &so[3530]}, + {"organizationalStatus", "organizationalStatus", NID_organizationalStatus, 10, &so[3540]}, + {"janetMailbox", "janetMailbox", NID_janetMailbox, 10, &so[3550]}, + {"mailPreferenceOption", "mailPreferenceOption", NID_mailPreferenceOption, 10, &so[3560]}, + {"buildingName", "buildingName", NID_buildingName, 10, &so[3570]}, + {"dSAQuality", "dSAQuality", NID_dSAQuality, 10, &so[3580]}, + {"singleLevelQuality", "singleLevelQuality", NID_singleLevelQuality, 10, &so[3590]}, + {"subtreeMinimumQuality", "subtreeMinimumQuality", NID_subtreeMinimumQuality, 10, &so[3600]}, + {"subtreeMaximumQuality", "subtreeMaximumQuality", NID_subtreeMaximumQuality, 10, &so[3610]}, + {"personalSignature", "personalSignature", NID_personalSignature, 10, &so[3620]}, + {"dITRedirect", "dITRedirect", NID_dITRedirect, 10, &so[3630]}, + {"audio", "audio", NID_audio, 10, &so[3640]}, + {"documentPublisher", "documentPublisher", NID_documentPublisher, 10, &so[3650]}, + {"x500UniqueIdentifier", "x500UniqueIdentifier", NID_x500UniqueIdentifier, 3, &so[3660]}, + {"mime-mhs", "MIME MHS", NID_mime_mhs, 5, &so[3663]}, + {"mime-mhs-headings", "mime-mhs-headings", NID_mime_mhs_headings, 6, &so[3668]}, + {"mime-mhs-bodies", "mime-mhs-bodies", NID_mime_mhs_bodies, 6, &so[3674]}, + {"id-hex-partial-message", "id-hex-partial-message", NID_id_hex_partial_message, 7, &so[3680]}, + {"id-hex-multipart-message", "id-hex-multipart-message", NID_id_hex_multipart_message, 7, &so[3687]}, + {"generationQualifier", "generationQualifier", NID_generationQualifier, 3, &so[3694]}, + {"pseudonym", "pseudonym", NID_pseudonym, 3, &so[3697]}, + { NULL, NULL, NID_undef }, + {"id-set", "Secure Electronic Transactions", NID_id_set, 2, &so[3700]}, + {"set-ctype", "content types", NID_set_ctype, 3, &so[3702]}, + {"set-msgExt", "message extensions", NID_set_msgExt, 3, &so[3705]}, + {"set-attr", "set-attr", NID_set_attr, 3, &so[3708]}, + {"set-policy", "set-policy", NID_set_policy, 3, &so[3711]}, + {"set-certExt", "certificate extensions", NID_set_certExt, 3, &so[3714]}, + {"set-brand", "set-brand", NID_set_brand, 3, &so[3717]}, + {"setct-PANData", "setct-PANData", NID_setct_PANData, 4, &so[3720]}, + {"setct-PANToken", "setct-PANToken", NID_setct_PANToken, 4, &so[3724]}, + {"setct-PANOnly", "setct-PANOnly", NID_setct_PANOnly, 4, &so[3728]}, + {"setct-OIData", "setct-OIData", NID_setct_OIData, 4, &so[3732]}, + {"setct-PI", "setct-PI", NID_setct_PI, 4, &so[3736]}, + {"setct-PIData", "setct-PIData", NID_setct_PIData, 4, &so[3740]}, + {"setct-PIDataUnsigned", "setct-PIDataUnsigned", NID_setct_PIDataUnsigned, 4, &so[3744]}, + {"setct-HODInput", "setct-HODInput", NID_setct_HODInput, 4, &so[3748]}, + {"setct-AuthResBaggage", "setct-AuthResBaggage", NID_setct_AuthResBaggage, 4, &so[3752]}, + {"setct-AuthRevReqBaggage", "setct-AuthRevReqBaggage", NID_setct_AuthRevReqBaggage, 4, &so[3756]}, + {"setct-AuthRevResBaggage", "setct-AuthRevResBaggage", NID_setct_AuthRevResBaggage, 4, &so[3760]}, + {"setct-CapTokenSeq", "setct-CapTokenSeq", NID_setct_CapTokenSeq, 4, &so[3764]}, + {"setct-PInitResData", "setct-PInitResData", NID_setct_PInitResData, 4, &so[3768]}, + {"setct-PI-TBS", "setct-PI-TBS", NID_setct_PI_TBS, 4, &so[3772]}, + {"setct-PResData", "setct-PResData", NID_setct_PResData, 4, &so[3776]}, + {"setct-AuthReqTBS", "setct-AuthReqTBS", NID_setct_AuthReqTBS, 4, &so[3780]}, + {"setct-AuthResTBS", "setct-AuthResTBS", NID_setct_AuthResTBS, 4, &so[3784]}, + {"setct-AuthResTBSX", "setct-AuthResTBSX", NID_setct_AuthResTBSX, 4, &so[3788]}, + {"setct-AuthTokenTBS", "setct-AuthTokenTBS", NID_setct_AuthTokenTBS, 4, &so[3792]}, + {"setct-CapTokenData", "setct-CapTokenData", NID_setct_CapTokenData, 4, &so[3796]}, + {"setct-CapTokenTBS", "setct-CapTokenTBS", NID_setct_CapTokenTBS, 4, &so[3800]}, + {"setct-AcqCardCodeMsg", "setct-AcqCardCodeMsg", NID_setct_AcqCardCodeMsg, 4, &so[3804]}, + {"setct-AuthRevReqTBS", "setct-AuthRevReqTBS", NID_setct_AuthRevReqTBS, 4, &so[3808]}, + {"setct-AuthRevResData", "setct-AuthRevResData", NID_setct_AuthRevResData, 4, &so[3812]}, + {"setct-AuthRevResTBS", "setct-AuthRevResTBS", NID_setct_AuthRevResTBS, 4, &so[3816]}, + {"setct-CapReqTBS", "setct-CapReqTBS", NID_setct_CapReqTBS, 4, &so[3820]}, + {"setct-CapReqTBSX", "setct-CapReqTBSX", NID_setct_CapReqTBSX, 4, &so[3824]}, + {"setct-CapResData", "setct-CapResData", NID_setct_CapResData, 4, &so[3828]}, + {"setct-CapRevReqTBS", "setct-CapRevReqTBS", NID_setct_CapRevReqTBS, 4, &so[3832]}, + {"setct-CapRevReqTBSX", "setct-CapRevReqTBSX", NID_setct_CapRevReqTBSX, 4, &so[3836]}, + {"setct-CapRevResData", "setct-CapRevResData", NID_setct_CapRevResData, 4, &so[3840]}, + {"setct-CredReqTBS", "setct-CredReqTBS", NID_setct_CredReqTBS, 4, &so[3844]}, + {"setct-CredReqTBSX", "setct-CredReqTBSX", NID_setct_CredReqTBSX, 4, &so[3848]}, + {"setct-CredResData", "setct-CredResData", NID_setct_CredResData, 4, &so[3852]}, + {"setct-CredRevReqTBS", "setct-CredRevReqTBS", NID_setct_CredRevReqTBS, 4, &so[3856]}, + {"setct-CredRevReqTBSX", "setct-CredRevReqTBSX", NID_setct_CredRevReqTBSX, 4, &so[3860]}, + {"setct-CredRevResData", "setct-CredRevResData", NID_setct_CredRevResData, 4, &so[3864]}, + {"setct-PCertReqData", "setct-PCertReqData", NID_setct_PCertReqData, 4, &so[3868]}, + {"setct-PCertResTBS", "setct-PCertResTBS", NID_setct_PCertResTBS, 4, &so[3872]}, + {"setct-BatchAdminReqData", "setct-BatchAdminReqData", NID_setct_BatchAdminReqData, 4, &so[3876]}, + {"setct-BatchAdminResData", "setct-BatchAdminResData", NID_setct_BatchAdminResData, 4, &so[3880]}, + {"setct-CardCInitResTBS", "setct-CardCInitResTBS", NID_setct_CardCInitResTBS, 4, &so[3884]}, + {"setct-MeAqCInitResTBS", "setct-MeAqCInitResTBS", NID_setct_MeAqCInitResTBS, 4, &so[3888]}, + {"setct-RegFormResTBS", "setct-RegFormResTBS", NID_setct_RegFormResTBS, 4, &so[3892]}, + {"setct-CertReqData", "setct-CertReqData", NID_setct_CertReqData, 4, &so[3896]}, + {"setct-CertReqTBS", "setct-CertReqTBS", NID_setct_CertReqTBS, 4, &so[3900]}, + {"setct-CertResData", "setct-CertResData", NID_setct_CertResData, 4, &so[3904]}, + {"setct-CertInqReqTBS", "setct-CertInqReqTBS", NID_setct_CertInqReqTBS, 4, &so[3908]}, + {"setct-ErrorTBS", "setct-ErrorTBS", NID_setct_ErrorTBS, 4, &so[3912]}, + {"setct-PIDualSignedTBE", "setct-PIDualSignedTBE", NID_setct_PIDualSignedTBE, 4, &so[3916]}, + {"setct-PIUnsignedTBE", "setct-PIUnsignedTBE", NID_setct_PIUnsignedTBE, 4, &so[3920]}, + {"setct-AuthReqTBE", "setct-AuthReqTBE", NID_setct_AuthReqTBE, 4, &so[3924]}, + {"setct-AuthResTBE", "setct-AuthResTBE", NID_setct_AuthResTBE, 4, &so[3928]}, + {"setct-AuthResTBEX", "setct-AuthResTBEX", NID_setct_AuthResTBEX, 4, &so[3932]}, + {"setct-AuthTokenTBE", "setct-AuthTokenTBE", NID_setct_AuthTokenTBE, 4, &so[3936]}, + {"setct-CapTokenTBE", "setct-CapTokenTBE", NID_setct_CapTokenTBE, 4, &so[3940]}, + {"setct-CapTokenTBEX", "setct-CapTokenTBEX", NID_setct_CapTokenTBEX, 4, &so[3944]}, + {"setct-AcqCardCodeMsgTBE", "setct-AcqCardCodeMsgTBE", NID_setct_AcqCardCodeMsgTBE, 4, &so[3948]}, + {"setct-AuthRevReqTBE", "setct-AuthRevReqTBE", NID_setct_AuthRevReqTBE, 4, &so[3952]}, + {"setct-AuthRevResTBE", "setct-AuthRevResTBE", NID_setct_AuthRevResTBE, 4, &so[3956]}, + {"setct-AuthRevResTBEB", "setct-AuthRevResTBEB", NID_setct_AuthRevResTBEB, 4, &so[3960]}, + {"setct-CapReqTBE", "setct-CapReqTBE", NID_setct_CapReqTBE, 4, &so[3964]}, + {"setct-CapReqTBEX", "setct-CapReqTBEX", NID_setct_CapReqTBEX, 4, &so[3968]}, + {"setct-CapResTBE", "setct-CapResTBE", NID_setct_CapResTBE, 4, &so[3972]}, + {"setct-CapRevReqTBE", "setct-CapRevReqTBE", NID_setct_CapRevReqTBE, 4, &so[3976]}, + {"setct-CapRevReqTBEX", "setct-CapRevReqTBEX", NID_setct_CapRevReqTBEX, 4, &so[3980]}, + {"setct-CapRevResTBE", "setct-CapRevResTBE", NID_setct_CapRevResTBE, 4, &so[3984]}, + {"setct-CredReqTBE", "setct-CredReqTBE", NID_setct_CredReqTBE, 4, &so[3988]}, + {"setct-CredReqTBEX", "setct-CredReqTBEX", NID_setct_CredReqTBEX, 4, &so[3992]}, + {"setct-CredResTBE", "setct-CredResTBE", NID_setct_CredResTBE, 4, &so[3996]}, + {"setct-CredRevReqTBE", "setct-CredRevReqTBE", NID_setct_CredRevReqTBE, 4, &so[4000]}, + {"setct-CredRevReqTBEX", "setct-CredRevReqTBEX", NID_setct_CredRevReqTBEX, 4, &so[4004]}, + {"setct-CredRevResTBE", "setct-CredRevResTBE", NID_setct_CredRevResTBE, 4, &so[4008]}, + {"setct-BatchAdminReqTBE", "setct-BatchAdminReqTBE", NID_setct_BatchAdminReqTBE, 4, &so[4012]}, + {"setct-BatchAdminResTBE", "setct-BatchAdminResTBE", NID_setct_BatchAdminResTBE, 4, &so[4016]}, + {"setct-RegFormReqTBE", "setct-RegFormReqTBE", NID_setct_RegFormReqTBE, 4, &so[4020]}, + {"setct-CertReqTBE", "setct-CertReqTBE", NID_setct_CertReqTBE, 4, &so[4024]}, + {"setct-CertReqTBEX", "setct-CertReqTBEX", NID_setct_CertReqTBEX, 4, &so[4028]}, + {"setct-CertResTBE", "setct-CertResTBE", NID_setct_CertResTBE, 4, &so[4032]}, + {"setct-CRLNotificationTBS", "setct-CRLNotificationTBS", NID_setct_CRLNotificationTBS, 4, &so[4036]}, + {"setct-CRLNotificationResTBS", "setct-CRLNotificationResTBS", NID_setct_CRLNotificationResTBS, 4, &so[4040]}, + {"setct-BCIDistributionTBS", "setct-BCIDistributionTBS", NID_setct_BCIDistributionTBS, 4, &so[4044]}, + {"setext-genCrypt", "generic cryptogram", NID_setext_genCrypt, 4, &so[4048]}, + {"setext-miAuth", "merchant initiated auth", NID_setext_miAuth, 4, &so[4052]}, + {"setext-pinSecure", "setext-pinSecure", NID_setext_pinSecure, 4, &so[4056]}, + {"setext-pinAny", "setext-pinAny", NID_setext_pinAny, 4, &so[4060]}, + {"setext-track2", "setext-track2", NID_setext_track2, 4, &so[4064]}, + {"setext-cv", "additional verification", NID_setext_cv, 4, &so[4068]}, + {"set-policy-root", "set-policy-root", NID_set_policy_root, 4, &so[4072]}, + {"setCext-hashedRoot", "setCext-hashedRoot", NID_setCext_hashedRoot, 4, &so[4076]}, + {"setCext-certType", "setCext-certType", NID_setCext_certType, 4, &so[4080]}, + {"setCext-merchData", "setCext-merchData", NID_setCext_merchData, 4, &so[4084]}, + {"setCext-cCertRequired", "setCext-cCertRequired", NID_setCext_cCertRequired, 4, &so[4088]}, + {"setCext-tunneling", "setCext-tunneling", NID_setCext_tunneling, 4, &so[4092]}, + {"setCext-setExt", "setCext-setExt", NID_setCext_setExt, 4, &so[4096]}, + {"setCext-setQualf", "setCext-setQualf", NID_setCext_setQualf, 4, &so[4100]}, + {"setCext-PGWYcapabilities", "setCext-PGWYcapabilities", NID_setCext_PGWYcapabilities, 4, &so[4104]}, + {"setCext-TokenIdentifier", "setCext-TokenIdentifier", NID_setCext_TokenIdentifier, 4, &so[4108]}, + {"setCext-Track2Data", "setCext-Track2Data", NID_setCext_Track2Data, 4, &so[4112]}, + {"setCext-TokenType", "setCext-TokenType", NID_setCext_TokenType, 4, &so[4116]}, + {"setCext-IssuerCapabilities", "setCext-IssuerCapabilities", NID_setCext_IssuerCapabilities, 4, &so[4120]}, + {"setAttr-Cert", "setAttr-Cert", NID_setAttr_Cert, 4, &so[4124]}, + {"setAttr-PGWYcap", "payment gateway capabilities", NID_setAttr_PGWYcap, 4, &so[4128]}, + {"setAttr-TokenType", "setAttr-TokenType", NID_setAttr_TokenType, 4, &so[4132]}, + {"setAttr-IssCap", "issuer capabilities", NID_setAttr_IssCap, 4, &so[4136]}, + {"set-rootKeyThumb", "set-rootKeyThumb", NID_set_rootKeyThumb, 5, &so[4140]}, + {"set-addPolicy", "set-addPolicy", NID_set_addPolicy, 5, &so[4145]}, + {"setAttr-Token-EMV", "setAttr-Token-EMV", NID_setAttr_Token_EMV, 5, &so[4150]}, + {"setAttr-Token-B0Prime", "setAttr-Token-B0Prime", NID_setAttr_Token_B0Prime, 5, &so[4155]}, + {"setAttr-IssCap-CVM", "setAttr-IssCap-CVM", NID_setAttr_IssCap_CVM, 5, &so[4160]}, + {"setAttr-IssCap-T2", "setAttr-IssCap-T2", NID_setAttr_IssCap_T2, 5, &so[4165]}, + {"setAttr-IssCap-Sig", "setAttr-IssCap-Sig", NID_setAttr_IssCap_Sig, 5, &so[4170]}, + {"setAttr-GenCryptgrm", "generate cryptogram", NID_setAttr_GenCryptgrm, 6, &so[4175]}, + {"setAttr-T2Enc", "encrypted track 2", NID_setAttr_T2Enc, 6, &so[4181]}, + {"setAttr-T2cleartxt", "cleartext track 2", NID_setAttr_T2cleartxt, 6, &so[4187]}, + {"setAttr-TokICCsig", "ICC or token signature", NID_setAttr_TokICCsig, 6, &so[4193]}, + {"setAttr-SecDevSig", "secure device signature", NID_setAttr_SecDevSig, 6, &so[4199]}, + {"set-brand-IATA-ATA", "set-brand-IATA-ATA", NID_set_brand_IATA_ATA, 4, &so[4205]}, + {"set-brand-Diners", "set-brand-Diners", NID_set_brand_Diners, 4, &so[4209]}, + {"set-brand-AmericanExpress", "set-brand-AmericanExpress", NID_set_brand_AmericanExpress, 4, &so[4213]}, + {"set-brand-JCB", "set-brand-JCB", NID_set_brand_JCB, 4, &so[4217]}, + {"set-brand-Visa", "set-brand-Visa", NID_set_brand_Visa, 4, &so[4221]}, + {"set-brand-MasterCard", "set-brand-MasterCard", NID_set_brand_MasterCard, 4, &so[4225]}, + {"set-brand-Novus", "set-brand-Novus", NID_set_brand_Novus, 5, &so[4229]}, + {"DES-CDMF", "des-cdmf", NID_des_cdmf, 8, &so[4234]}, + {"rsaOAEPEncryptionSET", "rsaOAEPEncryptionSET", NID_rsaOAEPEncryptionSET, 9, &so[4242]}, {"ITU-T", "itu-t", NID_itu_t}, {"JOINT-ISO-ITU-T", "joint-iso-itu-t", NID_joint_iso_itu_t}, - {"international-organizations", "International Organizations", NID_international_organizations, 1, &so[4439]}, - {"msSmartcardLogin", "Microsoft Smartcard Login", NID_ms_smartcard_login, 10, &so[4440]}, - {"msUPN", "Microsoft User Principal Name", NID_ms_upn, 10, &so[4450]}, + {"international-organizations", "International Organizations", NID_international_organizations, 1, &so[4251]}, + {"msSmartcardLogin", "Microsoft Smartcard Login", NID_ms_smartcard_login, 10, &so[4252]}, + {"msUPN", "Microsoft User Principal Name", NID_ms_upn, 10, &so[4262]}, {"AES-128-CFB1", "aes-128-cfb1", NID_aes_128_cfb1}, {"AES-192-CFB1", "aes-192-cfb1", NID_aes_192_cfb1}, {"AES-256-CFB1", "aes-256-cfb1", NID_aes_256_cfb1}, @@ -1847,382 +1593,382 @@ static const ASN1_OBJECT nid_objs[NUM_NID] = { {"DES-CFB8", "des-cfb8", NID_des_cfb8}, {"DES-EDE3-CFB1", "des-ede3-cfb1", NID_des_ede3_cfb1}, {"DES-EDE3-CFB8", "des-ede3-cfb8", NID_des_ede3_cfb8}, - {"street", "streetAddress", NID_streetAddress, 3, &so[4460]}, - {"postalCode", "postalCode", NID_postalCode, 3, &so[4463]}, - {"id-ppl", "id-ppl", NID_id_ppl, 7, &so[4466]}, - {"proxyCertInfo", "Proxy Certificate Information", NID_proxyCertInfo, 8, &so[4473]}, - {"id-ppl-anyLanguage", "Any language", NID_id_ppl_anyLanguage, 8, &so[4481]}, - {"id-ppl-inheritAll", "Inherit all", NID_id_ppl_inheritAll, 8, &so[4489]}, - {"nameConstraints", "X509v3 Name Constraints", NID_name_constraints, 3, &so[4497]}, - {"id-ppl-independent", "Independent", NID_Independent, 8, &so[4500]}, - {"RSA-SHA256", "sha256WithRSAEncryption", NID_sha256WithRSAEncryption, 9, &so[4508]}, - {"RSA-SHA384", "sha384WithRSAEncryption", NID_sha384WithRSAEncryption, 9, &so[4517]}, - {"RSA-SHA512", "sha512WithRSAEncryption", NID_sha512WithRSAEncryption, 9, &so[4526]}, - {"RSA-SHA224", "sha224WithRSAEncryption", NID_sha224WithRSAEncryption, 9, &so[4535]}, - {"SHA256", "sha256", NID_sha256, 9, &so[4544]}, - {"SHA384", "sha384", NID_sha384, 9, &so[4553]}, - {"SHA512", "sha512", NID_sha512, 9, &so[4562]}, - {"SHA224", "sha224", NID_sha224, 9, &so[4571]}, - {"identified-organization", "identified-organization", NID_identified_organization, 1, &so[4580]}, - {"certicom-arc", "certicom-arc", NID_certicom_arc, 3, &so[4581]}, - {"wap", "wap", NID_wap, 2, &so[4584]}, - {"wap-wsg", "wap-wsg", NID_wap_wsg, 3, &so[4586]}, - {"id-characteristic-two-basis", "id-characteristic-two-basis", NID_X9_62_id_characteristic_two_basis, 8, &so[4589]}, - {"onBasis", "onBasis", NID_X9_62_onBasis, 9, &so[4597]}, - {"tpBasis", "tpBasis", NID_X9_62_tpBasis, 9, &so[4606]}, - {"ppBasis", "ppBasis", NID_X9_62_ppBasis, 9, &so[4615]}, - {"c2pnb163v1", "c2pnb163v1", NID_X9_62_c2pnb163v1, 8, &so[4624]}, - {"c2pnb163v2", "c2pnb163v2", NID_X9_62_c2pnb163v2, 8, &so[4632]}, - {"c2pnb163v3", "c2pnb163v3", NID_X9_62_c2pnb163v3, 8, &so[4640]}, - {"c2pnb176v1", "c2pnb176v1", NID_X9_62_c2pnb176v1, 8, &so[4648]}, - {"c2tnb191v1", "c2tnb191v1", NID_X9_62_c2tnb191v1, 8, &so[4656]}, - {"c2tnb191v2", "c2tnb191v2", NID_X9_62_c2tnb191v2, 8, &so[4664]}, - {"c2tnb191v3", "c2tnb191v3", NID_X9_62_c2tnb191v3, 8, &so[4672]}, - {"c2onb191v4", "c2onb191v4", NID_X9_62_c2onb191v4, 8, &so[4680]}, - {"c2onb191v5", "c2onb191v5", NID_X9_62_c2onb191v5, 8, &so[4688]}, - {"c2pnb208w1", "c2pnb208w1", NID_X9_62_c2pnb208w1, 8, &so[4696]}, - {"c2tnb239v1", "c2tnb239v1", NID_X9_62_c2tnb239v1, 8, &so[4704]}, - {"c2tnb239v2", "c2tnb239v2", NID_X9_62_c2tnb239v2, 8, &so[4712]}, - {"c2tnb239v3", "c2tnb239v3", NID_X9_62_c2tnb239v3, 8, &so[4720]}, - {"c2onb239v4", "c2onb239v4", NID_X9_62_c2onb239v4, 8, &so[4728]}, - {"c2onb239v5", "c2onb239v5", NID_X9_62_c2onb239v5, 8, &so[4736]}, - {"c2pnb272w1", "c2pnb272w1", NID_X9_62_c2pnb272w1, 8, &so[4744]}, - {"c2pnb304w1", "c2pnb304w1", NID_X9_62_c2pnb304w1, 8, &so[4752]}, - {"c2tnb359v1", "c2tnb359v1", NID_X9_62_c2tnb359v1, 8, &so[4760]}, - {"c2pnb368w1", "c2pnb368w1", NID_X9_62_c2pnb368w1, 8, &so[4768]}, - {"c2tnb431r1", "c2tnb431r1", NID_X9_62_c2tnb431r1, 8, &so[4776]}, - {"secp112r1", "secp112r1", NID_secp112r1, 5, &so[4784]}, - {"secp112r2", "secp112r2", NID_secp112r2, 5, &so[4789]}, - {"secp128r1", "secp128r1", NID_secp128r1, 5, &so[4794]}, - {"secp128r2", "secp128r2", NID_secp128r2, 5, &so[4799]}, - {"secp160k1", "secp160k1", NID_secp160k1, 5, &so[4804]}, - {"secp160r1", "secp160r1", NID_secp160r1, 5, &so[4809]}, - {"secp160r2", "secp160r2", NID_secp160r2, 5, &so[4814]}, - {"secp192k1", "secp192k1", NID_secp192k1, 5, &so[4819]}, - {"secp224k1", "secp224k1", NID_secp224k1, 5, &so[4824]}, - {"secp224r1", "secp224r1", NID_secp224r1, 5, &so[4829]}, - {"secp256k1", "secp256k1", NID_secp256k1, 5, &so[4834]}, - {"secp384r1", "secp384r1", NID_secp384r1, 5, &so[4839]}, - {"secp521r1", "secp521r1", NID_secp521r1, 5, &so[4844]}, - {"sect113r1", "sect113r1", NID_sect113r1, 5, &so[4849]}, - {"sect113r2", "sect113r2", NID_sect113r2, 5, &so[4854]}, - {"sect131r1", "sect131r1", NID_sect131r1, 5, &so[4859]}, - {"sect131r2", "sect131r2", NID_sect131r2, 5, &so[4864]}, - {"sect163k1", "sect163k1", NID_sect163k1, 5, &so[4869]}, - {"sect163r1", "sect163r1", NID_sect163r1, 5, &so[4874]}, - {"sect163r2", "sect163r2", NID_sect163r2, 5, &so[4879]}, - {"sect193r1", "sect193r1", NID_sect193r1, 5, &so[4884]}, - {"sect193r2", "sect193r2", NID_sect193r2, 5, &so[4889]}, - {"sect233k1", "sect233k1", NID_sect233k1, 5, &so[4894]}, - {"sect233r1", "sect233r1", NID_sect233r1, 5, &so[4899]}, - {"sect239k1", "sect239k1", NID_sect239k1, 5, &so[4904]}, - {"sect283k1", "sect283k1", NID_sect283k1, 5, &so[4909]}, - {"sect283r1", "sect283r1", NID_sect283r1, 5, &so[4914]}, - {"sect409k1", "sect409k1", NID_sect409k1, 5, &so[4919]}, - {"sect409r1", "sect409r1", NID_sect409r1, 5, &so[4924]}, - {"sect571k1", "sect571k1", NID_sect571k1, 5, &so[4929]}, - {"sect571r1", "sect571r1", NID_sect571r1, 5, &so[4934]}, - {"wap-wsg-idm-ecid-wtls1", "wap-wsg-idm-ecid-wtls1", NID_wap_wsg_idm_ecid_wtls1, 5, &so[4939]}, - {"wap-wsg-idm-ecid-wtls3", "wap-wsg-idm-ecid-wtls3", NID_wap_wsg_idm_ecid_wtls3, 5, &so[4944]}, - {"wap-wsg-idm-ecid-wtls4", "wap-wsg-idm-ecid-wtls4", NID_wap_wsg_idm_ecid_wtls4, 5, &so[4949]}, - {"wap-wsg-idm-ecid-wtls5", "wap-wsg-idm-ecid-wtls5", NID_wap_wsg_idm_ecid_wtls5, 5, &so[4954]}, - {"wap-wsg-idm-ecid-wtls6", "wap-wsg-idm-ecid-wtls6", NID_wap_wsg_idm_ecid_wtls6, 5, &so[4959]}, - {"wap-wsg-idm-ecid-wtls7", "wap-wsg-idm-ecid-wtls7", NID_wap_wsg_idm_ecid_wtls7, 5, &so[4964]}, - {"wap-wsg-idm-ecid-wtls8", "wap-wsg-idm-ecid-wtls8", NID_wap_wsg_idm_ecid_wtls8, 5, &so[4969]}, - {"wap-wsg-idm-ecid-wtls9", "wap-wsg-idm-ecid-wtls9", NID_wap_wsg_idm_ecid_wtls9, 5, &so[4974]}, - {"wap-wsg-idm-ecid-wtls10", "wap-wsg-idm-ecid-wtls10", NID_wap_wsg_idm_ecid_wtls10, 5, &so[4979]}, - {"wap-wsg-idm-ecid-wtls11", "wap-wsg-idm-ecid-wtls11", NID_wap_wsg_idm_ecid_wtls11, 5, &so[4984]}, - {"wap-wsg-idm-ecid-wtls12", "wap-wsg-idm-ecid-wtls12", NID_wap_wsg_idm_ecid_wtls12, 5, &so[4989]}, - {"anyPolicy", "X509v3 Any Policy", NID_any_policy, 4, &so[4994]}, - {"policyMappings", "X509v3 Policy Mappings", NID_policy_mappings, 3, &so[4998]}, - {"inhibitAnyPolicy", "X509v3 Inhibit Any Policy", NID_inhibit_any_policy, 3, &so[5001]}, + {"street", "streetAddress", NID_streetAddress, 3, &so[4272]}, + {"postalCode", "postalCode", NID_postalCode, 3, &so[4275]}, + {"id-ppl", "id-ppl", NID_id_ppl, 7, &so[4278]}, + {"proxyCertInfo", "Proxy Certificate Information", NID_proxyCertInfo, 8, &so[4285]}, + {"id-ppl-anyLanguage", "Any language", NID_id_ppl_anyLanguage, 8, &so[4293]}, + {"id-ppl-inheritAll", "Inherit all", NID_id_ppl_inheritAll, 8, &so[4301]}, + {"nameConstraints", "X509v3 Name Constraints", NID_name_constraints, 3, &so[4309]}, + {"id-ppl-independent", "Independent", NID_Independent, 8, &so[4312]}, + {"RSA-SHA256", "sha256WithRSAEncryption", NID_sha256WithRSAEncryption, 9, &so[4320]}, + {"RSA-SHA384", "sha384WithRSAEncryption", NID_sha384WithRSAEncryption, 9, &so[4329]}, + {"RSA-SHA512", "sha512WithRSAEncryption", NID_sha512WithRSAEncryption, 9, &so[4338]}, + {"RSA-SHA224", "sha224WithRSAEncryption", NID_sha224WithRSAEncryption, 9, &so[4347]}, + {"SHA256", "sha256", NID_sha256, 9, &so[4356]}, + {"SHA384", "sha384", NID_sha384, 9, &so[4365]}, + {"SHA512", "sha512", NID_sha512, 9, &so[4374]}, + {"SHA224", "sha224", NID_sha224, 9, &so[4383]}, + {"identified-organization", "identified-organization", NID_identified_organization, 1, &so[4392]}, + {"certicom-arc", "certicom-arc", NID_certicom_arc, 3, &so[4393]}, + {"wap", "wap", NID_wap, 2, &so[4396]}, + {"wap-wsg", "wap-wsg", NID_wap_wsg, 3, &so[4398]}, + {"id-characteristic-two-basis", "id-characteristic-two-basis", NID_X9_62_id_characteristic_two_basis, 8, &so[4401]}, + {"onBasis", "onBasis", NID_X9_62_onBasis, 9, &so[4409]}, + {"tpBasis", "tpBasis", NID_X9_62_tpBasis, 9, &so[4418]}, + {"ppBasis", "ppBasis", NID_X9_62_ppBasis, 9, &so[4427]}, + {"c2pnb163v1", "c2pnb163v1", NID_X9_62_c2pnb163v1, 8, &so[4436]}, + {"c2pnb163v2", "c2pnb163v2", NID_X9_62_c2pnb163v2, 8, &so[4444]}, + {"c2pnb163v3", "c2pnb163v3", NID_X9_62_c2pnb163v3, 8, &so[4452]}, + {"c2pnb176v1", "c2pnb176v1", NID_X9_62_c2pnb176v1, 8, &so[4460]}, + {"c2tnb191v1", "c2tnb191v1", NID_X9_62_c2tnb191v1, 8, &so[4468]}, + {"c2tnb191v2", "c2tnb191v2", NID_X9_62_c2tnb191v2, 8, &so[4476]}, + {"c2tnb191v3", "c2tnb191v3", NID_X9_62_c2tnb191v3, 8, &so[4484]}, + {"c2onb191v4", "c2onb191v4", NID_X9_62_c2onb191v4, 8, &so[4492]}, + {"c2onb191v5", "c2onb191v5", NID_X9_62_c2onb191v5, 8, &so[4500]}, + {"c2pnb208w1", "c2pnb208w1", NID_X9_62_c2pnb208w1, 8, &so[4508]}, + {"c2tnb239v1", "c2tnb239v1", NID_X9_62_c2tnb239v1, 8, &so[4516]}, + {"c2tnb239v2", "c2tnb239v2", NID_X9_62_c2tnb239v2, 8, &so[4524]}, + {"c2tnb239v3", "c2tnb239v3", NID_X9_62_c2tnb239v3, 8, &so[4532]}, + {"c2onb239v4", "c2onb239v4", NID_X9_62_c2onb239v4, 8, &so[4540]}, + {"c2onb239v5", "c2onb239v5", NID_X9_62_c2onb239v5, 8, &so[4548]}, + {"c2pnb272w1", "c2pnb272w1", NID_X9_62_c2pnb272w1, 8, &so[4556]}, + {"c2pnb304w1", "c2pnb304w1", NID_X9_62_c2pnb304w1, 8, &so[4564]}, + {"c2tnb359v1", "c2tnb359v1", NID_X9_62_c2tnb359v1, 8, &so[4572]}, + {"c2pnb368w1", "c2pnb368w1", NID_X9_62_c2pnb368w1, 8, &so[4580]}, + {"c2tnb431r1", "c2tnb431r1", NID_X9_62_c2tnb431r1, 8, &so[4588]}, + {"secp112r1", "secp112r1", NID_secp112r1, 5, &so[4596]}, + {"secp112r2", "secp112r2", NID_secp112r2, 5, &so[4601]}, + {"secp128r1", "secp128r1", NID_secp128r1, 5, &so[4606]}, + {"secp128r2", "secp128r2", NID_secp128r2, 5, &so[4611]}, + {"secp160k1", "secp160k1", NID_secp160k1, 5, &so[4616]}, + {"secp160r1", "secp160r1", NID_secp160r1, 5, &so[4621]}, + {"secp160r2", "secp160r2", NID_secp160r2, 5, &so[4626]}, + {"secp192k1", "secp192k1", NID_secp192k1, 5, &so[4631]}, + {"secp224k1", "secp224k1", NID_secp224k1, 5, &so[4636]}, + {"secp224r1", "secp224r1", NID_secp224r1, 5, &so[4641]}, + {"secp256k1", "secp256k1", NID_secp256k1, 5, &so[4646]}, + {"secp384r1", "secp384r1", NID_secp384r1, 5, &so[4651]}, + {"secp521r1", "secp521r1", NID_secp521r1, 5, &so[4656]}, + {"sect113r1", "sect113r1", NID_sect113r1, 5, &so[4661]}, + {"sect113r2", "sect113r2", NID_sect113r2, 5, &so[4666]}, + {"sect131r1", "sect131r1", NID_sect131r1, 5, &so[4671]}, + {"sect131r2", "sect131r2", NID_sect131r2, 5, &so[4676]}, + {"sect163k1", "sect163k1", NID_sect163k1, 5, &so[4681]}, + {"sect163r1", "sect163r1", NID_sect163r1, 5, &so[4686]}, + {"sect163r2", "sect163r2", NID_sect163r2, 5, &so[4691]}, + {"sect193r1", "sect193r1", NID_sect193r1, 5, &so[4696]}, + {"sect193r2", "sect193r2", NID_sect193r2, 5, &so[4701]}, + {"sect233k1", "sect233k1", NID_sect233k1, 5, &so[4706]}, + {"sect233r1", "sect233r1", NID_sect233r1, 5, &so[4711]}, + {"sect239k1", "sect239k1", NID_sect239k1, 5, &so[4716]}, + {"sect283k1", "sect283k1", NID_sect283k1, 5, &so[4721]}, + {"sect283r1", "sect283r1", NID_sect283r1, 5, &so[4726]}, + {"sect409k1", "sect409k1", NID_sect409k1, 5, &so[4731]}, + {"sect409r1", "sect409r1", NID_sect409r1, 5, &so[4736]}, + {"sect571k1", "sect571k1", NID_sect571k1, 5, &so[4741]}, + {"sect571r1", "sect571r1", NID_sect571r1, 5, &so[4746]}, + {"wap-wsg-idm-ecid-wtls1", "wap-wsg-idm-ecid-wtls1", NID_wap_wsg_idm_ecid_wtls1, 5, &so[4751]}, + {"wap-wsg-idm-ecid-wtls3", "wap-wsg-idm-ecid-wtls3", NID_wap_wsg_idm_ecid_wtls3, 5, &so[4756]}, + {"wap-wsg-idm-ecid-wtls4", "wap-wsg-idm-ecid-wtls4", NID_wap_wsg_idm_ecid_wtls4, 5, &so[4761]}, + {"wap-wsg-idm-ecid-wtls5", "wap-wsg-idm-ecid-wtls5", NID_wap_wsg_idm_ecid_wtls5, 5, &so[4766]}, + {"wap-wsg-idm-ecid-wtls6", "wap-wsg-idm-ecid-wtls6", NID_wap_wsg_idm_ecid_wtls6, 5, &so[4771]}, + {"wap-wsg-idm-ecid-wtls7", "wap-wsg-idm-ecid-wtls7", NID_wap_wsg_idm_ecid_wtls7, 5, &so[4776]}, + {"wap-wsg-idm-ecid-wtls8", "wap-wsg-idm-ecid-wtls8", NID_wap_wsg_idm_ecid_wtls8, 5, &so[4781]}, + {"wap-wsg-idm-ecid-wtls9", "wap-wsg-idm-ecid-wtls9", NID_wap_wsg_idm_ecid_wtls9, 5, &so[4786]}, + {"wap-wsg-idm-ecid-wtls10", "wap-wsg-idm-ecid-wtls10", NID_wap_wsg_idm_ecid_wtls10, 5, &so[4791]}, + {"wap-wsg-idm-ecid-wtls11", "wap-wsg-idm-ecid-wtls11", NID_wap_wsg_idm_ecid_wtls11, 5, &so[4796]}, + {"wap-wsg-idm-ecid-wtls12", "wap-wsg-idm-ecid-wtls12", NID_wap_wsg_idm_ecid_wtls12, 5, &so[4801]}, + {"anyPolicy", "X509v3 Any Policy", NID_any_policy, 4, &so[4806]}, + {"policyMappings", "X509v3 Policy Mappings", NID_policy_mappings, 3, &so[4810]}, + {"inhibitAnyPolicy", "X509v3 Inhibit Any Policy", NID_inhibit_any_policy, 3, &so[4813]}, {"Oakley-EC2N-3", "ipsec3", NID_ipsec3}, {"Oakley-EC2N-4", "ipsec4", NID_ipsec4}, - {"CAMELLIA-128-CBC", "camellia-128-cbc", NID_camellia_128_cbc, 11, &so[5004]}, - {"CAMELLIA-192-CBC", "camellia-192-cbc", NID_camellia_192_cbc, 11, &so[5015]}, - {"CAMELLIA-256-CBC", "camellia-256-cbc", NID_camellia_256_cbc, 11, &so[5026]}, - {"CAMELLIA-128-ECB", "camellia-128-ecb", NID_camellia_128_ecb, 8, &so[5037]}, - {"CAMELLIA-192-ECB", "camellia-192-ecb", NID_camellia_192_ecb, 8, &so[5045]}, - {"CAMELLIA-256-ECB", "camellia-256-ecb", NID_camellia_256_ecb, 8, &so[5053]}, - {"CAMELLIA-128-CFB", "camellia-128-cfb", NID_camellia_128_cfb128, 8, &so[5061]}, - {"CAMELLIA-192-CFB", "camellia-192-cfb", NID_camellia_192_cfb128, 8, &so[5069]}, - {"CAMELLIA-256-CFB", "camellia-256-cfb", NID_camellia_256_cfb128, 8, &so[5077]}, - {"CAMELLIA-128-CFB1", "camellia-128-cfb1", NID_camellia_128_cfb1}, - {"CAMELLIA-192-CFB1", "camellia-192-cfb1", NID_camellia_192_cfb1}, - {"CAMELLIA-256-CFB1", "camellia-256-cfb1", NID_camellia_256_cfb1}, - {"CAMELLIA-128-CFB8", "camellia-128-cfb8", NID_camellia_128_cfb8}, - {"CAMELLIA-192-CFB8", "camellia-192-cfb8", NID_camellia_192_cfb8}, - {"CAMELLIA-256-CFB8", "camellia-256-cfb8", NID_camellia_256_cfb8}, - {"CAMELLIA-128-OFB", "camellia-128-ofb", NID_camellia_128_ofb128, 8, &so[5085]}, - {"CAMELLIA-192-OFB", "camellia-192-ofb", NID_camellia_192_ofb128, 8, &so[5093]}, - {"CAMELLIA-256-OFB", "camellia-256-ofb", NID_camellia_256_ofb128, 8, &so[5101]}, - {"subjectDirectoryAttributes", "X509v3 Subject Directory Attributes", NID_subject_directory_attributes, 3, &so[5109]}, - {"issuingDistributionPoint", "X509v3 Issuing Distribution Point", NID_issuing_distribution_point, 3, &so[5112]}, - {"certificateIssuer", "X509v3 Certificate Issuer", NID_certificate_issuer, 3, &so[5115]}, - { NULL, NULL, NID_undef }, - {"KISA", "kisa", NID_kisa, 6, &so[5118]}, - { NULL, NULL, NID_undef }, - { NULL, NULL, NID_undef }, - {"SEED-ECB", "seed-ecb", NID_seed_ecb, 8, &so[5124]}, - {"SEED-CBC", "seed-cbc", NID_seed_cbc, 8, &so[5132]}, - {"SEED-OFB", "seed-ofb", NID_seed_ofb128, 8, &so[5140]}, - {"SEED-CFB", "seed-cfb", NID_seed_cfb128, 8, &so[5148]}, - {"HMAC-MD5", "hmac-md5", NID_hmac_md5, 8, &so[5156]}, - {"HMAC-SHA1", "hmac-sha1", NID_hmac_sha1, 8, &so[5164]}, - {"id-PasswordBasedMAC", "password based MAC", NID_id_PasswordBasedMAC, 9, &so[5172]}, - {"id-DHBasedMac", "Diffie-Hellman based MAC", NID_id_DHBasedMac, 9, &so[5181]}, - {"id-it-suppLangTags", "id-it-suppLangTags", NID_id_it_suppLangTags, 8, &so[5190]}, - {"caRepository", "CA Repository", NID_caRepository, 8, &so[5198]}, - {"id-smime-ct-compressedData", "id-smime-ct-compressedData", NID_id_smime_ct_compressedData, 11, &so[5206]}, - {"id-ct-asciiTextWithCRLF", "id-ct-asciiTextWithCRLF", NID_id_ct_asciiTextWithCRLF, 11, &so[5217]}, - {"id-aes128-wrap", "id-aes128-wrap", NID_id_aes128_wrap, 9, &so[5228]}, - {"id-aes192-wrap", "id-aes192-wrap", NID_id_aes192_wrap, 9, &so[5237]}, - {"id-aes256-wrap", "id-aes256-wrap", NID_id_aes256_wrap, 9, &so[5246]}, - {"ecdsa-with-Recommended", "ecdsa-with-Recommended", NID_ecdsa_with_Recommended, 7, &so[5255]}, - {"ecdsa-with-Specified", "ecdsa-with-Specified", NID_ecdsa_with_Specified, 7, &so[5262]}, - {"ecdsa-with-SHA224", "ecdsa-with-SHA224", NID_ecdsa_with_SHA224, 8, &so[5269]}, - {"ecdsa-with-SHA256", "ecdsa-with-SHA256", NID_ecdsa_with_SHA256, 8, &so[5277]}, - {"ecdsa-with-SHA384", "ecdsa-with-SHA384", NID_ecdsa_with_SHA384, 8, &so[5285]}, - {"ecdsa-with-SHA512", "ecdsa-with-SHA512", NID_ecdsa_with_SHA512, 8, &so[5293]}, - {"hmacWithMD5", "hmacWithMD5", NID_hmacWithMD5, 8, &so[5301]}, - {"hmacWithSHA224", "hmacWithSHA224", NID_hmacWithSHA224, 8, &so[5309]}, - {"hmacWithSHA256", "hmacWithSHA256", NID_hmacWithSHA256, 8, &so[5317]}, - {"hmacWithSHA384", "hmacWithSHA384", NID_hmacWithSHA384, 8, &so[5325]}, - {"hmacWithSHA512", "hmacWithSHA512", NID_hmacWithSHA512, 8, &so[5333]}, - {"dsa_with_SHA224", "dsa_with_SHA224", NID_dsa_with_SHA224, 9, &so[5341]}, - {"dsa_with_SHA256", "dsa_with_SHA256", NID_dsa_with_SHA256, 9, &so[5350]}, - {"whirlpool", "whirlpool", NID_whirlpool, 6, &so[5359]}, - {"cryptopro", "cryptopro", NID_cryptopro, 5, &so[5365]}, - {"cryptocom", "cryptocom", NID_cryptocom, 5, &so[5370]}, - {"id-GostR3411-94-with-GostR3410-2001", "GOST R 34.11-94 with GOST R 34.10-2001", NID_id_GostR3411_94_with_GostR3410_2001, 6, &so[5375]}, - {"id-GostR3411-94-with-GostR3410-94", "GOST R 34.11-94 with GOST R 34.10-94", NID_id_GostR3411_94_with_GostR3410_94, 6, &so[5381]}, - {"md_gost94", "GOST R 34.11-94", NID_id_GostR3411_94, 6, &so[5387]}, - {"id-HMACGostR3411-94", "HMAC GOST 34.11-94", NID_id_HMACGostR3411_94, 6, &so[5393]}, - {"gost2001", "GOST R 34.10-2001", NID_id_GostR3410_2001, 6, &so[5399]}, - {"gost94", "GOST R 34.10-94", NID_id_GostR3410_94, 6, &so[5405]}, - {"gost89", "GOST 28147-89", NID_id_Gost28147_89, 6, &so[5411]}, - {"gost89-cnt", "gost89-cnt", NID_gost89_cnt}, - {"gost-mac", "GOST 28147-89 MAC", NID_id_Gost28147_89_MAC, 6, &so[5417]}, - {"prf-gostr3411-94", "GOST R 34.11-94 PRF", NID_id_GostR3411_94_prf, 6, &so[5423]}, - {"id-GostR3410-2001DH", "GOST R 34.10-2001 DH", NID_id_GostR3410_2001DH, 6, &so[5429]}, - {"id-GostR3410-94DH", "GOST R 34.10-94 DH", NID_id_GostR3410_94DH, 6, &so[5435]}, - {"id-Gost28147-89-CryptoPro-KeyMeshing", "id-Gost28147-89-CryptoPro-KeyMeshing", NID_id_Gost28147_89_CryptoPro_KeyMeshing, 7, &so[5441]}, - {"id-Gost28147-89-None-KeyMeshing", "id-Gost28147-89-None-KeyMeshing", NID_id_Gost28147_89_None_KeyMeshing, 7, &so[5448]}, - {"id-GostR3411-94-TestParamSet", "id-GostR3411-94-TestParamSet", NID_id_GostR3411_94_TestParamSet, 7, &so[5455]}, - {"id-GostR3411-94-CryptoProParamSet", "id-GostR3411-94-CryptoProParamSet", NID_id_GostR3411_94_CryptoProParamSet, 7, &so[5462]}, - {"id-Gost28147-89-TestParamSet", "id-Gost28147-89-TestParamSet", NID_id_Gost28147_89_TestParamSet, 7, &so[5469]}, - {"id-Gost28147-89-CryptoPro-A-ParamSet", "id-Gost28147-89-CryptoPro-A-ParamSet", NID_id_Gost28147_89_CryptoPro_A_ParamSet, 7, &so[5476]}, - {"id-Gost28147-89-CryptoPro-B-ParamSet", "id-Gost28147-89-CryptoPro-B-ParamSet", NID_id_Gost28147_89_CryptoPro_B_ParamSet, 7, &so[5483]}, - {"id-Gost28147-89-CryptoPro-C-ParamSet", "id-Gost28147-89-CryptoPro-C-ParamSet", NID_id_Gost28147_89_CryptoPro_C_ParamSet, 7, &so[5490]}, - {"id-Gost28147-89-CryptoPro-D-ParamSet", "id-Gost28147-89-CryptoPro-D-ParamSet", NID_id_Gost28147_89_CryptoPro_D_ParamSet, 7, &so[5497]}, - {"id-Gost28147-89-CryptoPro-Oscar-1-1-ParamSet", "id-Gost28147-89-CryptoPro-Oscar-1-1-ParamSet", NID_id_Gost28147_89_CryptoPro_Oscar_1_1_ParamSet, 7, &so[5504]}, - {"id-Gost28147-89-CryptoPro-Oscar-1-0-ParamSet", "id-Gost28147-89-CryptoPro-Oscar-1-0-ParamSet", NID_id_Gost28147_89_CryptoPro_Oscar_1_0_ParamSet, 7, &so[5511]}, - {"id-Gost28147-89-CryptoPro-RIC-1-ParamSet", "id-Gost28147-89-CryptoPro-RIC-1-ParamSet", NID_id_Gost28147_89_CryptoPro_RIC_1_ParamSet, 7, &so[5518]}, - {"id-GostR3410-94-TestParamSet", "id-GostR3410-94-TestParamSet", NID_id_GostR3410_94_TestParamSet, 7, &so[5525]}, - {"id-GostR3410-94-CryptoPro-A-ParamSet", "id-GostR3410-94-CryptoPro-A-ParamSet", NID_id_GostR3410_94_CryptoPro_A_ParamSet, 7, &so[5532]}, - {"id-GostR3410-94-CryptoPro-B-ParamSet", "id-GostR3410-94-CryptoPro-B-ParamSet", NID_id_GostR3410_94_CryptoPro_B_ParamSet, 7, &so[5539]}, - {"id-GostR3410-94-CryptoPro-C-ParamSet", "id-GostR3410-94-CryptoPro-C-ParamSet", NID_id_GostR3410_94_CryptoPro_C_ParamSet, 7, &so[5546]}, - {"id-GostR3410-94-CryptoPro-D-ParamSet", "id-GostR3410-94-CryptoPro-D-ParamSet", NID_id_GostR3410_94_CryptoPro_D_ParamSet, 7, &so[5553]}, - {"id-GostR3410-94-CryptoPro-XchA-ParamSet", "id-GostR3410-94-CryptoPro-XchA-ParamSet", NID_id_GostR3410_94_CryptoPro_XchA_ParamSet, 7, &so[5560]}, - {"id-GostR3410-94-CryptoPro-XchB-ParamSet", "id-GostR3410-94-CryptoPro-XchB-ParamSet", NID_id_GostR3410_94_CryptoPro_XchB_ParamSet, 7, &so[5567]}, - {"id-GostR3410-94-CryptoPro-XchC-ParamSet", "id-GostR3410-94-CryptoPro-XchC-ParamSet", NID_id_GostR3410_94_CryptoPro_XchC_ParamSet, 7, &so[5574]}, - {"id-GostR3410-2001-TestParamSet", "id-GostR3410-2001-TestParamSet", NID_id_GostR3410_2001_TestParamSet, 7, &so[5581]}, - {"id-GostR3410-2001-CryptoPro-A-ParamSet", "id-GostR3410-2001-CryptoPro-A-ParamSet", NID_id_GostR3410_2001_CryptoPro_A_ParamSet, 7, &so[5588]}, - {"id-GostR3410-2001-CryptoPro-B-ParamSet", "id-GostR3410-2001-CryptoPro-B-ParamSet", NID_id_GostR3410_2001_CryptoPro_B_ParamSet, 7, &so[5595]}, - {"id-GostR3410-2001-CryptoPro-C-ParamSet", "id-GostR3410-2001-CryptoPro-C-ParamSet", NID_id_GostR3410_2001_CryptoPro_C_ParamSet, 7, &so[5602]}, - {"id-GostR3410-2001-CryptoPro-XchA-ParamSet", "id-GostR3410-2001-CryptoPro-XchA-ParamSet", NID_id_GostR3410_2001_CryptoPro_XchA_ParamSet, 7, &so[5609]}, - {"id-GostR3410-2001-CryptoPro-XchB-ParamSet", "id-GostR3410-2001-CryptoPro-XchB-ParamSet", NID_id_GostR3410_2001_CryptoPro_XchB_ParamSet, 7, &so[5616]}, - {"id-GostR3410-94-a", "id-GostR3410-94-a", NID_id_GostR3410_94_a, 7, &so[5623]}, - {"id-GostR3410-94-aBis", "id-GostR3410-94-aBis", NID_id_GostR3410_94_aBis, 7, &so[5630]}, - {"id-GostR3410-94-b", "id-GostR3410-94-b", NID_id_GostR3410_94_b, 7, &so[5637]}, - {"id-GostR3410-94-bBis", "id-GostR3410-94-bBis", NID_id_GostR3410_94_bBis, 7, &so[5644]}, - {"id-Gost28147-89-cc", "GOST 28147-89 Cryptocom ParamSet", NID_id_Gost28147_89_cc, 8, &so[5651]}, - {"gost94cc", "GOST 34.10-94 Cryptocom", NID_id_GostR3410_94_cc, 8, &so[5659]}, - {"gost2001cc", "GOST 34.10-2001 Cryptocom", NID_id_GostR3410_2001_cc, 8, &so[5667]}, - {"id-GostR3411-94-with-GostR3410-94-cc", "GOST R 34.11-94 with GOST R 34.10-94 Cryptocom", NID_id_GostR3411_94_with_GostR3410_94_cc, 8, &so[5675]}, - {"id-GostR3411-94-with-GostR3410-2001-cc", "GOST R 34.11-94 with GOST R 34.10-2001 Cryptocom", NID_id_GostR3411_94_with_GostR3410_2001_cc, 8, &so[5683]}, - {"id-GostR3410-2001-ParamSet-cc", "GOST R 3410-2001 Parameter Set Cryptocom", NID_id_GostR3410_2001_ParamSet_cc, 8, &so[5691]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"subjectDirectoryAttributes", "X509v3 Subject Directory Attributes", NID_subject_directory_attributes, 3, &so[4816]}, + {"issuingDistributionPoint", "X509v3 Issuing Distribution Point", NID_issuing_distribution_point, 3, &so[4819]}, + {"certificateIssuer", "X509v3 Certificate Issuer", NID_certificate_issuer, 3, &so[4822]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"HMAC-MD5", "hmac-md5", NID_hmac_md5, 8, &so[4825]}, + {"HMAC-SHA1", "hmac-sha1", NID_hmac_sha1, 8, &so[4833]}, + {"id-PasswordBasedMAC", "password based MAC", NID_id_PasswordBasedMAC, 9, &so[4841]}, + {"id-DHBasedMac", "Diffie-Hellman based MAC", NID_id_DHBasedMac, 9, &so[4850]}, + {"id-it-suppLangTags", "id-it-suppLangTags", NID_id_it_suppLangTags, 8, &so[4859]}, + {"caRepository", "CA Repository", NID_caRepository, 8, &so[4867]}, + {"id-smime-ct-compressedData", "id-smime-ct-compressedData", NID_id_smime_ct_compressedData, 11, &so[4875]}, + {"id-ct-asciiTextWithCRLF", "id-ct-asciiTextWithCRLF", NID_id_ct_asciiTextWithCRLF, 11, &so[4886]}, + {"id-aes128-wrap", "id-aes128-wrap", NID_id_aes128_wrap, 9, &so[4897]}, + {"id-aes192-wrap", "id-aes192-wrap", NID_id_aes192_wrap, 9, &so[4906]}, + {"id-aes256-wrap", "id-aes256-wrap", NID_id_aes256_wrap, 9, &so[4915]}, + {"ecdsa-with-Recommended", "ecdsa-with-Recommended", NID_ecdsa_with_Recommended, 7, &so[4924]}, + {"ecdsa-with-Specified", "ecdsa-with-Specified", NID_ecdsa_with_Specified, 7, &so[4931]}, + {"ecdsa-with-SHA224", "ecdsa-with-SHA224", NID_ecdsa_with_SHA224, 8, &so[4938]}, + {"ecdsa-with-SHA256", "ecdsa-with-SHA256", NID_ecdsa_with_SHA256, 8, &so[4946]}, + {"ecdsa-with-SHA384", "ecdsa-with-SHA384", NID_ecdsa_with_SHA384, 8, &so[4954]}, + {"ecdsa-with-SHA512", "ecdsa-with-SHA512", NID_ecdsa_with_SHA512, 8, &so[4962]}, + {"hmacWithMD5", "hmacWithMD5", NID_hmacWithMD5, 8, &so[4970]}, + {"hmacWithSHA224", "hmacWithSHA224", NID_hmacWithSHA224, 8, &so[4978]}, + {"hmacWithSHA256", "hmacWithSHA256", NID_hmacWithSHA256, 8, &so[4986]}, + {"hmacWithSHA384", "hmacWithSHA384", NID_hmacWithSHA384, 8, &so[4994]}, + {"hmacWithSHA512", "hmacWithSHA512", NID_hmacWithSHA512, 8, &so[5002]}, + {"dsa_with_SHA224", "dsa_with_SHA224", NID_dsa_with_SHA224, 9, &so[5010]}, + {"dsa_with_SHA256", "dsa_with_SHA256", NID_dsa_with_SHA256, 9, &so[5019]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, {"HMAC", "hmac", NID_hmac}, - {"LocalKeySet", "Microsoft Local Key set", NID_LocalKeySet, 9, &so[5699]}, - {"freshestCRL", "X509v3 Freshest CRL", NID_freshest_crl, 3, &so[5708]}, - {"id-on-permanentIdentifier", "Permanent Identifier", NID_id_on_permanentIdentifier, 8, &so[5711]}, - {"searchGuide", "searchGuide", NID_searchGuide, 3, &so[5719]}, - {"businessCategory", "businessCategory", NID_businessCategory, 3, &so[5722]}, - {"postalAddress", "postalAddress", NID_postalAddress, 3, &so[5725]}, - {"postOfficeBox", "postOfficeBox", NID_postOfficeBox, 3, &so[5728]}, - {"physicalDeliveryOfficeName", "physicalDeliveryOfficeName", NID_physicalDeliveryOfficeName, 3, &so[5731]}, - {"telephoneNumber", "telephoneNumber", NID_telephoneNumber, 3, &so[5734]}, - {"telexNumber", "telexNumber", NID_telexNumber, 3, &so[5737]}, - {"teletexTerminalIdentifier", "teletexTerminalIdentifier", NID_teletexTerminalIdentifier, 3, &so[5740]}, - {"facsimileTelephoneNumber", "facsimileTelephoneNumber", NID_facsimileTelephoneNumber, 3, &so[5743]}, - {"x121Address", "x121Address", NID_x121Address, 3, &so[5746]}, - {"internationaliSDNNumber", "internationaliSDNNumber", NID_internationaliSDNNumber, 3, &so[5749]}, - {"registeredAddress", "registeredAddress", NID_registeredAddress, 3, &so[5752]}, - {"destinationIndicator", "destinationIndicator", NID_destinationIndicator, 3, &so[5755]}, - {"preferredDeliveryMethod", "preferredDeliveryMethod", NID_preferredDeliveryMethod, 3, &so[5758]}, - {"presentationAddress", "presentationAddress", NID_presentationAddress, 3, &so[5761]}, - {"supportedApplicationContext", "supportedApplicationContext", NID_supportedApplicationContext, 3, &so[5764]}, - {"member", "member", NID_member, 3, &so[5767]}, - {"owner", "owner", NID_owner, 3, &so[5770]}, - {"roleOccupant", "roleOccupant", NID_roleOccupant, 3, &so[5773]}, - {"seeAlso", "seeAlso", NID_seeAlso, 3, &so[5776]}, - {"userPassword", "userPassword", NID_userPassword, 3, &so[5779]}, - {"userCertificate", "userCertificate", NID_userCertificate, 3, &so[5782]}, - {"cACertificate", "cACertificate", NID_cACertificate, 3, &so[5785]}, - {"authorityRevocationList", "authorityRevocationList", NID_authorityRevocationList, 3, &so[5788]}, - {"certificateRevocationList", "certificateRevocationList", NID_certificateRevocationList, 3, &so[5791]}, - {"crossCertificatePair", "crossCertificatePair", NID_crossCertificatePair, 3, &so[5794]}, - {"enhancedSearchGuide", "enhancedSearchGuide", NID_enhancedSearchGuide, 3, &so[5797]}, - {"protocolInformation", "protocolInformation", NID_protocolInformation, 3, &so[5800]}, - {"distinguishedName", "distinguishedName", NID_distinguishedName, 3, &so[5803]}, - {"uniqueMember", "uniqueMember", NID_uniqueMember, 3, &so[5806]}, - {"houseIdentifier", "houseIdentifier", NID_houseIdentifier, 3, &so[5809]}, - {"supportedAlgorithms", "supportedAlgorithms", NID_supportedAlgorithms, 3, &so[5812]}, - {"deltaRevocationList", "deltaRevocationList", NID_deltaRevocationList, 3, &so[5815]}, - {"dmdName", "dmdName", NID_dmdName, 3, &so[5818]}, - {"id-alg-PWRI-KEK", "id-alg-PWRI-KEK", NID_id_alg_PWRI_KEK, 11, &so[5821]}, + {"LocalKeySet", "Microsoft Local Key set", NID_LocalKeySet, 9, &so[5028]}, + {"freshestCRL", "X509v3 Freshest CRL", NID_freshest_crl, 3, &so[5037]}, + {"id-on-permanentIdentifier", "Permanent Identifier", NID_id_on_permanentIdentifier, 8, &so[5040]}, + {"searchGuide", "searchGuide", NID_searchGuide, 3, &so[5048]}, + {"businessCategory", "businessCategory", NID_businessCategory, 3, &so[5051]}, + {"postalAddress", "postalAddress", NID_postalAddress, 3, &so[5054]}, + {"postOfficeBox", "postOfficeBox", NID_postOfficeBox, 3, &so[5057]}, + {"physicalDeliveryOfficeName", "physicalDeliveryOfficeName", NID_physicalDeliveryOfficeName, 3, &so[5060]}, + {"telephoneNumber", "telephoneNumber", NID_telephoneNumber, 3, &so[5063]}, + {"telexNumber", "telexNumber", NID_telexNumber, 3, &so[5066]}, + {"teletexTerminalIdentifier", "teletexTerminalIdentifier", NID_teletexTerminalIdentifier, 3, &so[5069]}, + {"facsimileTelephoneNumber", "facsimileTelephoneNumber", NID_facsimileTelephoneNumber, 3, &so[5072]}, + {"x121Address", "x121Address", NID_x121Address, 3, &so[5075]}, + {"internationaliSDNNumber", "internationaliSDNNumber", NID_internationaliSDNNumber, 3, &so[5078]}, + {"registeredAddress", "registeredAddress", NID_registeredAddress, 3, &so[5081]}, + {"destinationIndicator", "destinationIndicator", NID_destinationIndicator, 3, &so[5084]}, + {"preferredDeliveryMethod", "preferredDeliveryMethod", NID_preferredDeliveryMethod, 3, &so[5087]}, + {"presentationAddress", "presentationAddress", NID_presentationAddress, 3, &so[5090]}, + {"supportedApplicationContext", "supportedApplicationContext", NID_supportedApplicationContext, 3, &so[5093]}, + {"member", "member", NID_member, 3, &so[5096]}, + {"owner", "owner", NID_owner, 3, &so[5099]}, + {"roleOccupant", "roleOccupant", NID_roleOccupant, 3, &so[5102]}, + {"seeAlso", "seeAlso", NID_seeAlso, 3, &so[5105]}, + {"userPassword", "userPassword", NID_userPassword, 3, &so[5108]}, + {"userCertificate", "userCertificate", NID_userCertificate, 3, &so[5111]}, + {"cACertificate", "cACertificate", NID_cACertificate, 3, &so[5114]}, + {"authorityRevocationList", "authorityRevocationList", NID_authorityRevocationList, 3, &so[5117]}, + {"certificateRevocationList", "certificateRevocationList", NID_certificateRevocationList, 3, &so[5120]}, + {"crossCertificatePair", "crossCertificatePair", NID_crossCertificatePair, 3, &so[5123]}, + {"enhancedSearchGuide", "enhancedSearchGuide", NID_enhancedSearchGuide, 3, &so[5126]}, + {"protocolInformation", "protocolInformation", NID_protocolInformation, 3, &so[5129]}, + {"distinguishedName", "distinguishedName", NID_distinguishedName, 3, &so[5132]}, + {"uniqueMember", "uniqueMember", NID_uniqueMember, 3, &so[5135]}, + {"houseIdentifier", "houseIdentifier", NID_houseIdentifier, 3, &so[5138]}, + {"supportedAlgorithms", "supportedAlgorithms", NID_supportedAlgorithms, 3, &so[5141]}, + {"deltaRevocationList", "deltaRevocationList", NID_deltaRevocationList, 3, &so[5144]}, + {"dmdName", "dmdName", NID_dmdName, 3, &so[5147]}, + {"id-alg-PWRI-KEK", "id-alg-PWRI-KEK", NID_id_alg_PWRI_KEK, 11, &so[5150]}, {"CMAC", "cmac", NID_cmac}, - {"id-aes128-GCM", "aes-128-gcm", NID_aes_128_gcm, 9, &so[5832]}, - {"id-aes128-CCM", "aes-128-ccm", NID_aes_128_ccm, 9, &so[5841]}, - {"id-aes128-wrap-pad", "id-aes128-wrap-pad", NID_id_aes128_wrap_pad, 9, &so[5850]}, - {"id-aes192-GCM", "aes-192-gcm", NID_aes_192_gcm, 9, &so[5859]}, - {"id-aes192-CCM", "aes-192-ccm", NID_aes_192_ccm, 9, &so[5868]}, - {"id-aes192-wrap-pad", "id-aes192-wrap-pad", NID_id_aes192_wrap_pad, 9, &so[5877]}, - {"id-aes256-GCM", "aes-256-gcm", NID_aes_256_gcm, 9, &so[5886]}, - {"id-aes256-CCM", "aes-256-ccm", NID_aes_256_ccm, 9, &so[5895]}, - {"id-aes256-wrap-pad", "id-aes256-wrap-pad", NID_id_aes256_wrap_pad, 9, &so[5904]}, + {"id-aes128-GCM", "aes-128-gcm", NID_aes_128_gcm, 9, &so[5161]}, + {"id-aes128-CCM", "aes-128-ccm", NID_aes_128_ccm, 9, &so[5170]}, + {"id-aes128-wrap-pad", "id-aes128-wrap-pad", NID_id_aes128_wrap_pad, 9, &so[5179]}, + {"id-aes192-GCM", "aes-192-gcm", NID_aes_192_gcm, 9, &so[5188]}, + {"id-aes192-CCM", "aes-192-ccm", NID_aes_192_ccm, 9, &so[5197]}, + {"id-aes192-wrap-pad", "id-aes192-wrap-pad", NID_id_aes192_wrap_pad, 9, &so[5206]}, + {"id-aes256-GCM", "aes-256-gcm", NID_aes_256_gcm, 9, &so[5215]}, + {"id-aes256-CCM", "aes-256-ccm", NID_aes_256_ccm, 9, &so[5224]}, + {"id-aes256-wrap-pad", "id-aes256-wrap-pad", NID_id_aes256_wrap_pad, 9, &so[5233]}, {"AES-128-CTR", "aes-128-ctr", NID_aes_128_ctr}, {"AES-192-CTR", "aes-192-ctr", NID_aes_192_ctr}, {"AES-256-CTR", "aes-256-ctr", NID_aes_256_ctr}, - {"id-camellia128-wrap", "id-camellia128-wrap", NID_id_camellia128_wrap, 11, &so[5913]}, - {"id-camellia192-wrap", "id-camellia192-wrap", NID_id_camellia192_wrap, 11, &so[5924]}, - {"id-camellia256-wrap", "id-camellia256-wrap", NID_id_camellia256_wrap, 11, &so[5935]}, - {"anyExtendedKeyUsage", "Any Extended Key Usage", NID_anyExtendedKeyUsage, 4, &so[5946]}, - {"MGF1", "mgf1", NID_mgf1, 9, &so[5950]}, - {"RSASSA-PSS", "rsassaPss", NID_rsassaPss, 9, &so[5959]}, - {"AES-128-XTS", "aes-128-xts", NID_aes_128_xts, 8, &so[5968]}, - {"AES-256-XTS", "aes-256-xts", NID_aes_256_xts, 8, &so[5976]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"anyExtendedKeyUsage", "Any Extended Key Usage", NID_anyExtendedKeyUsage, 4, &so[5242]}, + {"MGF1", "mgf1", NID_mgf1, 9, &so[5246]}, + {"RSASSA-PSS", "rsassaPss", NID_rsassaPss, 9, &so[5255]}, + {"AES-128-XTS", "aes-128-xts", NID_aes_128_xts, 8, &so[5264]}, + {"AES-256-XTS", "aes-256-xts", NID_aes_256_xts, 8, &so[5272]}, {"RC4-HMAC-MD5", "rc4-hmac-md5", NID_rc4_hmac_md5}, {"AES-128-CBC-HMAC-SHA1", "aes-128-cbc-hmac-sha1", NID_aes_128_cbc_hmac_sha1}, {"AES-192-CBC-HMAC-SHA1", "aes-192-cbc-hmac-sha1", NID_aes_192_cbc_hmac_sha1}, {"AES-256-CBC-HMAC-SHA1", "aes-256-cbc-hmac-sha1", NID_aes_256_cbc_hmac_sha1}, - {"RSAES-OAEP", "rsaesOaep", NID_rsaesOaep, 9, &so[5984]}, - {"dhpublicnumber", "X9.42 DH", NID_dhpublicnumber, 7, &so[5993]}, - {"brainpoolP160r1", "brainpoolP160r1", NID_brainpoolP160r1, 9, &so[6000]}, - {"brainpoolP160t1", "brainpoolP160t1", NID_brainpoolP160t1, 9, &so[6009]}, - {"brainpoolP192r1", "brainpoolP192r1", NID_brainpoolP192r1, 9, &so[6018]}, - {"brainpoolP192t1", "brainpoolP192t1", NID_brainpoolP192t1, 9, &so[6027]}, - {"brainpoolP224r1", "brainpoolP224r1", NID_brainpoolP224r1, 9, &so[6036]}, - {"brainpoolP224t1", "brainpoolP224t1", NID_brainpoolP224t1, 9, &so[6045]}, - {"brainpoolP256r1", "brainpoolP256r1", NID_brainpoolP256r1, 9, &so[6054]}, - {"brainpoolP256t1", "brainpoolP256t1", NID_brainpoolP256t1, 9, &so[6063]}, - {"brainpoolP320r1", "brainpoolP320r1", NID_brainpoolP320r1, 9, &so[6072]}, - {"brainpoolP320t1", "brainpoolP320t1", NID_brainpoolP320t1, 9, &so[6081]}, - {"brainpoolP384r1", "brainpoolP384r1", NID_brainpoolP384r1, 9, &so[6090]}, - {"brainpoolP384t1", "brainpoolP384t1", NID_brainpoolP384t1, 9, &so[6099]}, - {"brainpoolP512r1", "brainpoolP512r1", NID_brainpoolP512r1, 9, &so[6108]}, - {"brainpoolP512t1", "brainpoolP512t1", NID_brainpoolP512t1, 9, &so[6117]}, - {"PSPECIFIED", "pSpecified", NID_pSpecified, 9, &so[6126]}, - {"dhSinglePass-stdDH-sha1kdf-scheme", "dhSinglePass-stdDH-sha1kdf-scheme", NID_dhSinglePass_stdDH_sha1kdf_scheme, 9, &so[6135]}, - {"dhSinglePass-stdDH-sha224kdf-scheme", "dhSinglePass-stdDH-sha224kdf-scheme", NID_dhSinglePass_stdDH_sha224kdf_scheme, 6, &so[6144]}, - {"dhSinglePass-stdDH-sha256kdf-scheme", "dhSinglePass-stdDH-sha256kdf-scheme", NID_dhSinglePass_stdDH_sha256kdf_scheme, 6, &so[6150]}, - {"dhSinglePass-stdDH-sha384kdf-scheme", "dhSinglePass-stdDH-sha384kdf-scheme", NID_dhSinglePass_stdDH_sha384kdf_scheme, 6, &so[6156]}, - {"dhSinglePass-stdDH-sha512kdf-scheme", "dhSinglePass-stdDH-sha512kdf-scheme", NID_dhSinglePass_stdDH_sha512kdf_scheme, 6, &so[6162]}, - {"dhSinglePass-cofactorDH-sha1kdf-scheme", "dhSinglePass-cofactorDH-sha1kdf-scheme", NID_dhSinglePass_cofactorDH_sha1kdf_scheme, 9, &so[6168]}, - {"dhSinglePass-cofactorDH-sha224kdf-scheme", "dhSinglePass-cofactorDH-sha224kdf-scheme", NID_dhSinglePass_cofactorDH_sha224kdf_scheme, 6, &so[6177]}, - {"dhSinglePass-cofactorDH-sha256kdf-scheme", "dhSinglePass-cofactorDH-sha256kdf-scheme", NID_dhSinglePass_cofactorDH_sha256kdf_scheme, 6, &so[6183]}, - {"dhSinglePass-cofactorDH-sha384kdf-scheme", "dhSinglePass-cofactorDH-sha384kdf-scheme", NID_dhSinglePass_cofactorDH_sha384kdf_scheme, 6, &so[6189]}, - {"dhSinglePass-cofactorDH-sha512kdf-scheme", "dhSinglePass-cofactorDH-sha512kdf-scheme", NID_dhSinglePass_cofactorDH_sha512kdf_scheme, 6, &so[6195]}, + {"RSAES-OAEP", "rsaesOaep", NID_rsaesOaep, 9, &so[5280]}, + {"dhpublicnumber", "X9.42 DH", NID_dhpublicnumber, 7, &so[5289]}, + {"brainpoolP160r1", "brainpoolP160r1", NID_brainpoolP160r1, 9, &so[5296]}, + {"brainpoolP160t1", "brainpoolP160t1", NID_brainpoolP160t1, 9, &so[5305]}, + {"brainpoolP192r1", "brainpoolP192r1", NID_brainpoolP192r1, 9, &so[5314]}, + {"brainpoolP192t1", "brainpoolP192t1", NID_brainpoolP192t1, 9, &so[5323]}, + {"brainpoolP224r1", "brainpoolP224r1", NID_brainpoolP224r1, 9, &so[5332]}, + {"brainpoolP224t1", "brainpoolP224t1", NID_brainpoolP224t1, 9, &so[5341]}, + {"brainpoolP256r1", "brainpoolP256r1", NID_brainpoolP256r1, 9, &so[5350]}, + {"brainpoolP256t1", "brainpoolP256t1", NID_brainpoolP256t1, 9, &so[5359]}, + {"brainpoolP320r1", "brainpoolP320r1", NID_brainpoolP320r1, 9, &so[5368]}, + {"brainpoolP320t1", "brainpoolP320t1", NID_brainpoolP320t1, 9, &so[5377]}, + {"brainpoolP384r1", "brainpoolP384r1", NID_brainpoolP384r1, 9, &so[5386]}, + {"brainpoolP384t1", "brainpoolP384t1", NID_brainpoolP384t1, 9, &so[5395]}, + {"brainpoolP512r1", "brainpoolP512r1", NID_brainpoolP512r1, 9, &so[5404]}, + {"brainpoolP512t1", "brainpoolP512t1", NID_brainpoolP512t1, 9, &so[5413]}, + {"PSPECIFIED", "pSpecified", NID_pSpecified, 9, &so[5422]}, + {"dhSinglePass-stdDH-sha1kdf-scheme", "dhSinglePass-stdDH-sha1kdf-scheme", NID_dhSinglePass_stdDH_sha1kdf_scheme, 9, &so[5431]}, + {"dhSinglePass-stdDH-sha224kdf-scheme", "dhSinglePass-stdDH-sha224kdf-scheme", NID_dhSinglePass_stdDH_sha224kdf_scheme, 6, &so[5440]}, + {"dhSinglePass-stdDH-sha256kdf-scheme", "dhSinglePass-stdDH-sha256kdf-scheme", NID_dhSinglePass_stdDH_sha256kdf_scheme, 6, &so[5446]}, + {"dhSinglePass-stdDH-sha384kdf-scheme", "dhSinglePass-stdDH-sha384kdf-scheme", NID_dhSinglePass_stdDH_sha384kdf_scheme, 6, &so[5452]}, + {"dhSinglePass-stdDH-sha512kdf-scheme", "dhSinglePass-stdDH-sha512kdf-scheme", NID_dhSinglePass_stdDH_sha512kdf_scheme, 6, &so[5458]}, + {"dhSinglePass-cofactorDH-sha1kdf-scheme", "dhSinglePass-cofactorDH-sha1kdf-scheme", NID_dhSinglePass_cofactorDH_sha1kdf_scheme, 9, &so[5464]}, + {"dhSinglePass-cofactorDH-sha224kdf-scheme", "dhSinglePass-cofactorDH-sha224kdf-scheme", NID_dhSinglePass_cofactorDH_sha224kdf_scheme, 6, &so[5473]}, + {"dhSinglePass-cofactorDH-sha256kdf-scheme", "dhSinglePass-cofactorDH-sha256kdf-scheme", NID_dhSinglePass_cofactorDH_sha256kdf_scheme, 6, &so[5479]}, + {"dhSinglePass-cofactorDH-sha384kdf-scheme", "dhSinglePass-cofactorDH-sha384kdf-scheme", NID_dhSinglePass_cofactorDH_sha384kdf_scheme, 6, &so[5485]}, + {"dhSinglePass-cofactorDH-sha512kdf-scheme", "dhSinglePass-cofactorDH-sha512kdf-scheme", NID_dhSinglePass_cofactorDH_sha512kdf_scheme, 6, &so[5491]}, {"dh-std-kdf", "dh-std-kdf", NID_dh_std_kdf}, {"dh-cofactor-kdf", "dh-cofactor-kdf", NID_dh_cofactor_kdf}, {"AES-128-CBC-HMAC-SHA256", "aes-128-cbc-hmac-sha256", NID_aes_128_cbc_hmac_sha256}, {"AES-192-CBC-HMAC-SHA256", "aes-192-cbc-hmac-sha256", NID_aes_192_cbc_hmac_sha256}, {"AES-256-CBC-HMAC-SHA256", "aes-256-cbc-hmac-sha256", NID_aes_256_cbc_hmac_sha256}, - {"ct_precert_scts", "CT Precertificate SCTs", NID_ct_precert_scts, 10, &so[6201]}, - {"ct_precert_poison", "CT Precertificate Poison", NID_ct_precert_poison, 10, &so[6211]}, - {"ct_precert_signer", "CT Precertificate Signer", NID_ct_precert_signer, 10, &so[6221]}, - {"ct_cert_scts", "CT Certificate SCTs", NID_ct_cert_scts, 10, &so[6231]}, - {"jurisdictionL", "jurisdictionLocalityName", NID_jurisdictionLocalityName, 11, &so[6241]}, - {"jurisdictionST", "jurisdictionStateOrProvinceName", NID_jurisdictionStateOrProvinceName, 11, &so[6252]}, - {"jurisdictionC", "jurisdictionCountryName", NID_jurisdictionCountryName, 11, &so[6263]}, + {"ct_precert_scts", "CT Precertificate SCTs", NID_ct_precert_scts, 10, &so[5497]}, + {"ct_precert_poison", "CT Precertificate Poison", NID_ct_precert_poison, 10, &so[5507]}, + {"ct_precert_signer", "CT Precertificate Signer", NID_ct_precert_signer, 10, &so[5517]}, + {"ct_cert_scts", "CT Certificate SCTs", NID_ct_cert_scts, 10, &so[5527]}, + {"jurisdictionL", "jurisdictionLocalityName", NID_jurisdictionLocalityName, 11, &so[5537]}, + {"jurisdictionST", "jurisdictionStateOrProvinceName", NID_jurisdictionStateOrProvinceName, 11, &so[5548]}, + {"jurisdictionC", "jurisdictionCountryName", NID_jurisdictionCountryName, 11, &so[5559]}, {"AES-128-OCB", "aes-128-ocb", NID_aes_128_ocb}, {"AES-192-OCB", "aes-192-ocb", NID_aes_192_ocb}, {"AES-256-OCB", "aes-256-ocb", NID_aes_256_ocb}, - {"CAMELLIA-128-GCM", "camellia-128-gcm", NID_camellia_128_gcm, 8, &so[6274]}, - {"CAMELLIA-128-CCM", "camellia-128-ccm", NID_camellia_128_ccm, 8, &so[6282]}, - {"CAMELLIA-128-CTR", "camellia-128-ctr", NID_camellia_128_ctr, 8, &so[6290]}, - {"CAMELLIA-128-CMAC", "camellia-128-cmac", NID_camellia_128_cmac, 8, &so[6298]}, - {"CAMELLIA-192-GCM", "camellia-192-gcm", NID_camellia_192_gcm, 8, &so[6306]}, - {"CAMELLIA-192-CCM", "camellia-192-ccm", NID_camellia_192_ccm, 8, &so[6314]}, - {"CAMELLIA-192-CTR", "camellia-192-ctr", NID_camellia_192_ctr, 8, &so[6322]}, - {"CAMELLIA-192-CMAC", "camellia-192-cmac", NID_camellia_192_cmac, 8, &so[6330]}, - {"CAMELLIA-256-GCM", "camellia-256-gcm", NID_camellia_256_gcm, 8, &so[6338]}, - {"CAMELLIA-256-CCM", "camellia-256-ccm", NID_camellia_256_ccm, 8, &so[6346]}, - {"CAMELLIA-256-CTR", "camellia-256-ctr", NID_camellia_256_ctr, 8, &so[6354]}, - {"CAMELLIA-256-CMAC", "camellia-256-cmac", NID_camellia_256_cmac, 8, &so[6362]}, - {"id-scrypt", "scrypt", NID_id_scrypt, 9, &so[6370]}, - {"id-tc26", "id-tc26", NID_id_tc26, 5, &so[6379]}, - {"gost89-cnt-12", "gost89-cnt-12", NID_gost89_cnt_12}, - {"gost-mac-12", "gost-mac-12", NID_gost_mac_12}, - {"id-tc26-algorithms", "id-tc26-algorithms", NID_id_tc26_algorithms, 6, &so[6384]}, - {"id-tc26-sign", "id-tc26-sign", NID_id_tc26_sign, 7, &so[6390]}, - {"gost2012_256", "GOST R 34.10-2012 with 256 bit modulus", NID_id_GostR3410_2012_256, 8, &so[6397]}, - {"gost2012_512", "GOST R 34.10-2012 with 512 bit modulus", NID_id_GostR3410_2012_512, 8, &so[6405]}, - {"id-tc26-digest", "id-tc26-digest", NID_id_tc26_digest, 7, &so[6413]}, - {"md_gost12_256", "GOST R 34.11-2012 with 256 bit hash", NID_id_GostR3411_2012_256, 8, &so[6420]}, - {"md_gost12_512", "GOST R 34.11-2012 with 512 bit hash", NID_id_GostR3411_2012_512, 8, &so[6428]}, - {"id-tc26-signwithdigest", "id-tc26-signwithdigest", NID_id_tc26_signwithdigest, 7, &so[6436]}, - {"id-tc26-signwithdigest-gost3410-2012-256", "GOST R 34.10-2012 with GOST R 34.11-2012 (256 bit)", NID_id_tc26_signwithdigest_gost3410_2012_256, 8, &so[6443]}, - {"id-tc26-signwithdigest-gost3410-2012-512", "GOST R 34.10-2012 with GOST R 34.11-2012 (512 bit)", NID_id_tc26_signwithdigest_gost3410_2012_512, 8, &so[6451]}, - {"id-tc26-mac", "id-tc26-mac", NID_id_tc26_mac, 7, &so[6459]}, - {"id-tc26-hmac-gost-3411-2012-256", "HMAC GOST 34.11-2012 256 bit", NID_id_tc26_hmac_gost_3411_2012_256, 8, &so[6466]}, - {"id-tc26-hmac-gost-3411-2012-512", "HMAC GOST 34.11-2012 512 bit", NID_id_tc26_hmac_gost_3411_2012_512, 8, &so[6474]}, - {"id-tc26-cipher", "id-tc26-cipher", NID_id_tc26_cipher, 7, &so[6482]}, - {"id-tc26-agreement", "id-tc26-agreement", NID_id_tc26_agreement, 7, &so[6489]}, - {"id-tc26-agreement-gost-3410-2012-256", "id-tc26-agreement-gost-3410-2012-256", NID_id_tc26_agreement_gost_3410_2012_256, 8, &so[6496]}, - {"id-tc26-agreement-gost-3410-2012-512", "id-tc26-agreement-gost-3410-2012-512", NID_id_tc26_agreement_gost_3410_2012_512, 8, &so[6504]}, - {"id-tc26-constants", "id-tc26-constants", NID_id_tc26_constants, 6, &so[6512]}, - {"id-tc26-sign-constants", "id-tc26-sign-constants", NID_id_tc26_sign_constants, 7, &so[6518]}, - {"id-tc26-gost-3410-2012-512-constants", "id-tc26-gost-3410-2012-512-constants", NID_id_tc26_gost_3410_2012_512_constants, 8, &so[6525]}, - {"id-tc26-gost-3410-2012-512-paramSetTest", "GOST R 34.10-2012 (512 bit) testing parameter set", NID_id_tc26_gost_3410_2012_512_paramSetTest, 9, &so[6533]}, - {"id-tc26-gost-3410-2012-512-paramSetA", "GOST R 34.10-2012 (512 bit) ParamSet A", NID_id_tc26_gost_3410_2012_512_paramSetA, 9, &so[6542]}, - {"id-tc26-gost-3410-2012-512-paramSetB", "GOST R 34.10-2012 (512 bit) ParamSet B", NID_id_tc26_gost_3410_2012_512_paramSetB, 9, &so[6551]}, - {"id-tc26-digest-constants", "id-tc26-digest-constants", NID_id_tc26_digest_constants, 7, &so[6560]}, - {"id-tc26-cipher-constants", "id-tc26-cipher-constants", NID_id_tc26_cipher_constants, 7, &so[6567]}, - {"id-tc26-gost-28147-constants", "id-tc26-gost-28147-constants", NID_id_tc26_gost_28147_constants, 8, &so[6574]}, - {"id-tc26-gost-28147-param-Z", "GOST 28147-89 TC26 parameter set", NID_id_tc26_gost_28147_param_Z, 9, &so[6582]}, - {"INN", "INN", NID_INN, 8, &so[6591]}, - {"OGRN", "OGRN", NID_OGRN, 5, &so[6599]}, - {"SNILS", "SNILS", NID_SNILS, 5, &so[6604]}, - {"subjectSignTool", "Signing Tool of Subject", NID_subjectSignTool, 5, &so[6609]}, - {"issuerSignTool", "Signing Tool of Issuer", NID_issuerSignTool, 5, &so[6614]}, - {"gost89-cbc", "gost89-cbc", NID_gost89_cbc}, - {"gost89-ecb", "gost89-ecb", NID_gost89_ecb}, - {"gost89-ctr", "gost89-ctr", NID_gost89_ctr}, - {"kuznyechik-ecb", "kuznyechik-ecb", NID_kuznyechik_ecb}, - {"kuznyechik-ctr", "kuznyechik-ctr", NID_kuznyechik_ctr}, - {"kuznyechik-ofb", "kuznyechik-ofb", NID_kuznyechik_ofb}, - {"kuznyechik-cbc", "kuznyechik-cbc", NID_kuznyechik_cbc}, - {"kuznyechik-cfb", "kuznyechik-cfb", NID_kuznyechik_cfb}, - {"kuznyechik-mac", "kuznyechik-mac", NID_kuznyechik_mac}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"id-scrypt", "scrypt", NID_id_scrypt, 9, &so[5570]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, {"ChaCha20-Poly1305", "chacha20-poly1305", NID_chacha20_poly1305}, {"ChaCha20", "chacha20", NID_chacha20}, - {"tlsfeature", "TLS Feature", NID_tlsfeature, 8, &so[6619]}, + {"tlsfeature", "TLS Feature", NID_tlsfeature, 8, &so[5579]}, {"TLS1-PRF", "tls1-prf", NID_tls1_prf}, - {"ipsecIKE", "ipsec Internet Key Exchange", NID_ipsec_IKE, 8, &so[6627]}, - {"capwapAC", "Ctrl/provision WAP Access", NID_capwapAC, 8, &so[6635]}, - {"capwapWTP", "Ctrl/Provision WAP Termination", NID_capwapWTP, 8, &so[6643]}, - {"secureShellClient", "SSH Client", NID_sshClient, 8, &so[6651]}, - {"secureShellServer", "SSH Server", NID_sshServer, 8, &so[6659]}, - {"sendRouter", "Send Router", NID_sendRouter, 8, &so[6667]}, - {"sendProxiedRouter", "Send Proxied Router", NID_sendProxiedRouter, 8, &so[6675]}, - {"sendOwner", "Send Owner", NID_sendOwner, 8, &so[6683]}, - {"sendProxiedOwner", "Send Proxied Owner", NID_sendProxiedOwner, 8, &so[6691]}, - {"id-pkinit", "id-pkinit", NID_id_pkinit, 6, &so[6699]}, - {"pkInitClientAuth", "PKINIT Client Auth", NID_pkInitClientAuth, 7, &so[6705]}, - {"pkInitKDC", "Signing KDC Response", NID_pkInitKDC, 7, &so[6712]}, - {"X25519", "X25519", NID_X25519, 3, &so[6719]}, - {"X448", "X448", NID_X448, 3, &so[6722]}, + {"ipsecIKE", "ipsec Internet Key Exchange", NID_ipsec_IKE, 8, &so[5587]}, + {"capwapAC", "Ctrl/provision WAP Access", NID_capwapAC, 8, &so[5595]}, + {"capwapWTP", "Ctrl/Provision WAP Termination", NID_capwapWTP, 8, &so[5603]}, + {"secureShellClient", "SSH Client", NID_sshClient, 8, &so[5611]}, + {"secureShellServer", "SSH Server", NID_sshServer, 8, &so[5619]}, + {"sendRouter", "Send Router", NID_sendRouter, 8, &so[5627]}, + {"sendProxiedRouter", "Send Proxied Router", NID_sendProxiedRouter, 8, &so[5635]}, + {"sendOwner", "Send Owner", NID_sendOwner, 8, &so[5643]}, + {"sendProxiedOwner", "Send Proxied Owner", NID_sendProxiedOwner, 8, &so[5651]}, + {"id-pkinit", "id-pkinit", NID_id_pkinit, 6, &so[5659]}, + {"pkInitClientAuth", "PKINIT Client Auth", NID_pkInitClientAuth, 7, &so[5665]}, + {"pkInitKDC", "Signing KDC Response", NID_pkInitKDC, 7, &so[5672]}, + {"X25519", "X25519", NID_X25519, 3, &so[5679]}, + {"X448", "X448", NID_X448, 3, &so[5682]}, {"HKDF", "hkdf", NID_hkdf}, {"KxRSA", "kx-rsa", NID_kx_rsa}, {"KxECDHE", "kx-ecdhe", NID_kx_ecdhe}, @@ -2232,285 +1978,222 @@ static const ASN1_OBJECT nid_objs[NUM_NID] = { {"KxRSA_PSK", "kx-rsa-psk", NID_kx_rsa_psk}, {"KxPSK", "kx-psk", NID_kx_psk}, {"KxSRP", "kx-srp", NID_kx_srp}, - {"KxGOST", "kx-gost", NID_kx_gost}, + { NULL, NULL, NID_undef }, {"AuthRSA", "auth-rsa", NID_auth_rsa}, {"AuthECDSA", "auth-ecdsa", NID_auth_ecdsa}, {"AuthPSK", "auth-psk", NID_auth_psk}, {"AuthDSS", "auth-dss", NID_auth_dss}, - {"AuthGOST01", "auth-gost01", NID_auth_gost01}, - {"AuthGOST12", "auth-gost12", NID_auth_gost12}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, {"AuthSRP", "auth-srp", NID_auth_srp}, {"AuthNULL", "auth-null", NID_auth_null}, { NULL, NULL, NID_undef }, { NULL, NULL, NID_undef }, - {"BLAKE2b512", "blake2b512", NID_blake2b512, 11, &so[6725]}, - {"BLAKE2s256", "blake2s256", NID_blake2s256, 11, &so[6736]}, - {"id-smime-ct-contentCollection", "id-smime-ct-contentCollection", NID_id_smime_ct_contentCollection, 11, &so[6747]}, - {"id-smime-ct-authEnvelopedData", "id-smime-ct-authEnvelopedData", NID_id_smime_ct_authEnvelopedData, 11, &so[6758]}, - {"id-ct-xml", "id-ct-xml", NID_id_ct_xml, 11, &so[6769]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"id-smime-ct-contentCollection", "id-smime-ct-contentCollection", NID_id_smime_ct_contentCollection, 11, &so[5685]}, + {"id-smime-ct-authEnvelopedData", "id-smime-ct-authEnvelopedData", NID_id_smime_ct_authEnvelopedData, 11, &so[5696]}, + {"id-ct-xml", "id-ct-xml", NID_id_ct_xml, 11, &so[5707]}, {"Poly1305", "poly1305", NID_poly1305}, {"SipHash", "siphash", NID_siphash}, {"KxANY", "kx-any", NID_kx_any}, {"AuthANY", "auth-any", NID_auth_any}, - {"ARIA-128-ECB", "aria-128-ecb", NID_aria_128_ecb, 9, &so[6780]}, - {"ARIA-128-CBC", "aria-128-cbc", NID_aria_128_cbc, 9, &so[6789]}, - {"ARIA-128-CFB", "aria-128-cfb", NID_aria_128_cfb128, 9, &so[6798]}, - {"ARIA-128-OFB", "aria-128-ofb", NID_aria_128_ofb128, 9, &so[6807]}, - {"ARIA-128-CTR", "aria-128-ctr", NID_aria_128_ctr, 9, &so[6816]}, - {"ARIA-192-ECB", "aria-192-ecb", NID_aria_192_ecb, 9, &so[6825]}, - {"ARIA-192-CBC", "aria-192-cbc", NID_aria_192_cbc, 9, &so[6834]}, - {"ARIA-192-CFB", "aria-192-cfb", NID_aria_192_cfb128, 9, &so[6843]}, - {"ARIA-192-OFB", "aria-192-ofb", NID_aria_192_ofb128, 9, &so[6852]}, - {"ARIA-192-CTR", "aria-192-ctr", NID_aria_192_ctr, 9, &so[6861]}, - {"ARIA-256-ECB", "aria-256-ecb", NID_aria_256_ecb, 9, &so[6870]}, - {"ARIA-256-CBC", "aria-256-cbc", NID_aria_256_cbc, 9, &so[6879]}, - {"ARIA-256-CFB", "aria-256-cfb", NID_aria_256_cfb128, 9, &so[6888]}, - {"ARIA-256-OFB", "aria-256-ofb", NID_aria_256_ofb128, 9, &so[6897]}, - {"ARIA-256-CTR", "aria-256-ctr", NID_aria_256_ctr, 9, &so[6906]}, - {"ARIA-128-CFB1", "aria-128-cfb1", NID_aria_128_cfb1}, - {"ARIA-192-CFB1", "aria-192-cfb1", NID_aria_192_cfb1}, - {"ARIA-256-CFB1", "aria-256-cfb1", NID_aria_256_cfb1}, - {"ARIA-128-CFB8", "aria-128-cfb8", NID_aria_128_cfb8}, - {"ARIA-192-CFB8", "aria-192-cfb8", NID_aria_192_cfb8}, - {"ARIA-256-CFB8", "aria-256-cfb8", NID_aria_256_cfb8}, - {"id-smime-aa-signingCertificateV2", "id-smime-aa-signingCertificateV2", NID_id_smime_aa_signingCertificateV2, 11, &so[6915]}, - {"ED25519", "ED25519", NID_ED25519, 3, &so[6926]}, - {"ED448", "ED448", NID_ED448, 3, &so[6929]}, - {"organizationIdentifier", "organizationIdentifier", NID_organizationIdentifier, 3, &so[6932]}, - {"c3", "countryCode3c", NID_countryCode3c, 3, &so[6935]}, - {"n3", "countryCode3n", NID_countryCode3n, 3, &so[6938]}, - {"dnsName", "dnsName", NID_dnsName, 3, &so[6941]}, - {"x509ExtAdmission", "Professional Information or basis for Admission", NID_x509ExtAdmission, 5, &so[6944]}, - {"SHA512-224", "sha512-224", NID_sha512_224, 9, &so[6949]}, - {"SHA512-256", "sha512-256", NID_sha512_256, 9, &so[6958]}, - {"SHA3-224", "sha3-224", NID_sha3_224, 9, &so[6967]}, - {"SHA3-256", "sha3-256", NID_sha3_256, 9, &so[6976]}, - {"SHA3-384", "sha3-384", NID_sha3_384, 9, &so[6985]}, - {"SHA3-512", "sha3-512", NID_sha3_512, 9, &so[6994]}, - {"SHAKE128", "shake128", NID_shake128, 9, &so[7003]}, - {"SHAKE256", "shake256", NID_shake256, 9, &so[7012]}, - {"id-hmacWithSHA3-224", "hmac-sha3-224", NID_hmac_sha3_224, 9, &so[7021]}, - {"id-hmacWithSHA3-256", "hmac-sha3-256", NID_hmac_sha3_256, 9, &so[7030]}, - {"id-hmacWithSHA3-384", "hmac-sha3-384", NID_hmac_sha3_384, 9, &so[7039]}, - {"id-hmacWithSHA3-512", "hmac-sha3-512", NID_hmac_sha3_512, 9, &so[7048]}, - {"id-dsa-with-sha384", "dsa_with_SHA384", NID_dsa_with_SHA384, 9, &so[7057]}, - {"id-dsa-with-sha512", "dsa_with_SHA512", NID_dsa_with_SHA512, 9, &so[7066]}, - {"id-dsa-with-sha3-224", "dsa_with_SHA3-224", NID_dsa_with_SHA3_224, 9, &so[7075]}, - {"id-dsa-with-sha3-256", "dsa_with_SHA3-256", NID_dsa_with_SHA3_256, 9, &so[7084]}, - {"id-dsa-with-sha3-384", "dsa_with_SHA3-384", NID_dsa_with_SHA3_384, 9, &so[7093]}, - {"id-dsa-with-sha3-512", "dsa_with_SHA3-512", NID_dsa_with_SHA3_512, 9, &so[7102]}, - {"id-ecdsa-with-sha3-224", "ecdsa_with_SHA3-224", NID_ecdsa_with_SHA3_224, 9, &so[7111]}, - {"id-ecdsa-with-sha3-256", "ecdsa_with_SHA3-256", NID_ecdsa_with_SHA3_256, 9, &so[7120]}, - {"id-ecdsa-with-sha3-384", "ecdsa_with_SHA3-384", NID_ecdsa_with_SHA3_384, 9, &so[7129]}, - {"id-ecdsa-with-sha3-512", "ecdsa_with_SHA3-512", NID_ecdsa_with_SHA3_512, 9, &so[7138]}, - {"id-rsassa-pkcs1-v1_5-with-sha3-224", "RSA-SHA3-224", NID_RSA_SHA3_224, 9, &so[7147]}, - {"id-rsassa-pkcs1-v1_5-with-sha3-256", "RSA-SHA3-256", NID_RSA_SHA3_256, 9, &so[7156]}, - {"id-rsassa-pkcs1-v1_5-with-sha3-384", "RSA-SHA3-384", NID_RSA_SHA3_384, 9, &so[7165]}, - {"id-rsassa-pkcs1-v1_5-with-sha3-512", "RSA-SHA3-512", NID_RSA_SHA3_512, 9, &so[7174]}, - {"ARIA-128-CCM", "aria-128-ccm", NID_aria_128_ccm, 9, &so[7183]}, - {"ARIA-192-CCM", "aria-192-ccm", NID_aria_192_ccm, 9, &so[7192]}, - {"ARIA-256-CCM", "aria-256-ccm", NID_aria_256_ccm, 9, &so[7201]}, - {"ARIA-128-GCM", "aria-128-gcm", NID_aria_128_gcm, 9, &so[7210]}, - {"ARIA-192-GCM", "aria-192-gcm", NID_aria_192_gcm, 9, &so[7219]}, - {"ARIA-256-GCM", "aria-256-gcm", NID_aria_256_gcm, 9, &so[7228]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"id-smime-aa-signingCertificateV2", "id-smime-aa-signingCertificateV2", NID_id_smime_aa_signingCertificateV2, 11, &so[5718]}, + {"ED25519", "ED25519", NID_ED25519, 3, &so[5729]}, + {"ED448", "ED448", NID_ED448, 3, &so[5732]}, + {"organizationIdentifier", "organizationIdentifier", NID_organizationIdentifier, 3, &so[5735]}, + {"c3", "countryCode3c", NID_countryCode3c, 3, &so[5738]}, + {"n3", "countryCode3n", NID_countryCode3n, 3, &so[5741]}, + {"dnsName", "dnsName", NID_dnsName, 3, &so[5744]}, + {"x509ExtAdmission", "Professional Information or basis for Admission", NID_x509ExtAdmission, 5, &so[5747]}, + {"SHA512-224", "sha512-224", NID_sha512_224, 9, &so[5752]}, + {"SHA512-256", "sha512-256", NID_sha512_256, 9, &so[5761]}, + {"SHA3-224", "sha3-224", NID_sha3_224, 9, &so[5770]}, + {"SHA3-256", "sha3-256", NID_sha3_256, 9, &so[5779]}, + {"SHA3-384", "sha3-384", NID_sha3_384, 9, &so[5788]}, + {"SHA3-512", "sha3-512", NID_sha3_512, 9, &so[5797]}, + {"SHAKE128", "shake128", NID_shake128, 9, &so[5806]}, + {"SHAKE256", "shake256", NID_shake256, 9, &so[5815]}, + {"id-hmacWithSHA3-224", "hmac-sha3-224", NID_hmac_sha3_224, 9, &so[5824]}, + {"id-hmacWithSHA3-256", "hmac-sha3-256", NID_hmac_sha3_256, 9, &so[5833]}, + {"id-hmacWithSHA3-384", "hmac-sha3-384", NID_hmac_sha3_384, 9, &so[5842]}, + {"id-hmacWithSHA3-512", "hmac-sha3-512", NID_hmac_sha3_512, 9, &so[5851]}, + {"id-dsa-with-sha384", "dsa_with_SHA384", NID_dsa_with_SHA384, 9, &so[5860]}, + {"id-dsa-with-sha512", "dsa_with_SHA512", NID_dsa_with_SHA512, 9, &so[5869]}, + {"id-dsa-with-sha3-224", "dsa_with_SHA3-224", NID_dsa_with_SHA3_224, 9, &so[5878]}, + {"id-dsa-with-sha3-256", "dsa_with_SHA3-256", NID_dsa_with_SHA3_256, 9, &so[5887]}, + {"id-dsa-with-sha3-384", "dsa_with_SHA3-384", NID_dsa_with_SHA3_384, 9, &so[5896]}, + {"id-dsa-with-sha3-512", "dsa_with_SHA3-512", NID_dsa_with_SHA3_512, 9, &so[5905]}, + {"id-ecdsa-with-sha3-224", "ecdsa_with_SHA3-224", NID_ecdsa_with_SHA3_224, 9, &so[5914]}, + {"id-ecdsa-with-sha3-256", "ecdsa_with_SHA3-256", NID_ecdsa_with_SHA3_256, 9, &so[5923]}, + {"id-ecdsa-with-sha3-384", "ecdsa_with_SHA3-384", NID_ecdsa_with_SHA3_384, 9, &so[5932]}, + {"id-ecdsa-with-sha3-512", "ecdsa_with_SHA3-512", NID_ecdsa_with_SHA3_512, 9, &so[5941]}, + {"id-rsassa-pkcs1-v1_5-with-sha3-224", "RSA-SHA3-224", NID_RSA_SHA3_224, 9, &so[5950]}, + {"id-rsassa-pkcs1-v1_5-with-sha3-256", "RSA-SHA3-256", NID_RSA_SHA3_256, 9, &so[5959]}, + {"id-rsassa-pkcs1-v1_5-with-sha3-384", "RSA-SHA3-384", NID_RSA_SHA3_384, 9, &so[5968]}, + {"id-rsassa-pkcs1-v1_5-with-sha3-512", "RSA-SHA3-512", NID_RSA_SHA3_512, 9, &so[5977]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, {"ffdhe2048", "ffdhe2048", NID_ffdhe2048}, {"ffdhe3072", "ffdhe3072", NID_ffdhe3072}, {"ffdhe4096", "ffdhe4096", NID_ffdhe4096}, {"ffdhe6144", "ffdhe6144", NID_ffdhe6144}, {"ffdhe8192", "ffdhe8192", NID_ffdhe8192}, - {"cmcCA", "CMC Certificate Authority", NID_cmcCA, 8, &so[7237]}, - {"cmcRA", "CMC Registration Authority", NID_cmcRA, 8, &so[7245]}, - {"SM4-ECB", "sm4-ecb", NID_sm4_ecb, 8, &so[7253]}, - {"SM4-CBC", "sm4-cbc", NID_sm4_cbc, 8, &so[7261]}, - {"SM4-OFB", "sm4-ofb", NID_sm4_ofb128, 8, &so[7269]}, - {"SM4-CFB1", "sm4-cfb1", NID_sm4_cfb1, 8, &so[7277]}, - {"SM4-CFB", "sm4-cfb", NID_sm4_cfb128, 8, &so[7285]}, - {"SM4-CFB8", "sm4-cfb8", NID_sm4_cfb8, 8, &so[7293]}, - {"SM4-CTR", "sm4-ctr", NID_sm4_ctr, 8, &so[7301]}, - {"ISO-CN", "ISO CN Member Body", NID_ISO_CN, 3, &so[7309]}, - {"oscca", "oscca", NID_oscca, 5, &so[7312]}, - {"sm-scheme", "sm-scheme", NID_sm_scheme, 6, &so[7317]}, - {"SM3", "sm3", NID_sm3, 8, &so[7323]}, - {"RSA-SM3", "sm3WithRSAEncryption", NID_sm3WithRSAEncryption, 8, &so[7331]}, - {"RSA-SHA512/224", "sha512-224WithRSAEncryption", NID_sha512_224WithRSAEncryption, 9, &so[7339]}, - {"RSA-SHA512/256", "sha512-256WithRSAEncryption", NID_sha512_256WithRSAEncryption, 9, &so[7348]}, - {"id-tc26-gost-3410-2012-256-constants", "id-tc26-gost-3410-2012-256-constants", NID_id_tc26_gost_3410_2012_256_constants, 8, &so[7357]}, - {"id-tc26-gost-3410-2012-256-paramSetA", "GOST R 34.10-2012 (256 bit) ParamSet A", NID_id_tc26_gost_3410_2012_256_paramSetA, 9, &so[7365]}, - {"id-tc26-gost-3410-2012-512-paramSetC", "GOST R 34.10-2012 (512 bit) ParamSet C", NID_id_tc26_gost_3410_2012_512_paramSetC, 9, &so[7374]}, - {"ISO-UA", "ISO-UA", NID_ISO_UA, 3, &so[7383]}, - {"ua-pki", "ua-pki", NID_ua_pki, 7, &so[7386]}, - {"dstu28147", "DSTU Gost 28147-2009", NID_dstu28147, 10, &so[7393]}, - {"dstu28147-ofb", "DSTU Gost 28147-2009 OFB mode", NID_dstu28147_ofb, 11, &so[7403]}, - {"dstu28147-cfb", "DSTU Gost 28147-2009 CFB mode", NID_dstu28147_cfb, 11, &so[7414]}, - {"dstu28147-wrap", "DSTU Gost 28147-2009 key wrap", NID_dstu28147_wrap, 11, &so[7425]}, - {"hmacWithDstu34311", "HMAC DSTU Gost 34311-95", NID_hmacWithDstu34311, 10, &so[7436]}, - {"dstu34311", "DSTU Gost 34311-95", NID_dstu34311, 10, &so[7446]}, - {"dstu4145le", "DSTU 4145-2002 little endian", NID_dstu4145le, 11, &so[7456]}, - {"dstu4145be", "DSTU 4145-2002 big endian", NID_dstu4145be, 13, &so[7467]}, - {"uacurve0", "DSTU curve 0", NID_uacurve0, 13, &so[7480]}, - {"uacurve1", "DSTU curve 1", NID_uacurve1, 13, &so[7493]}, - {"uacurve2", "DSTU curve 2", NID_uacurve2, 13, &so[7506]}, - {"uacurve3", "DSTU curve 3", NID_uacurve3, 13, &so[7519]}, - {"uacurve4", "DSTU curve 4", NID_uacurve4, 13, &so[7532]}, - {"uacurve5", "DSTU curve 5", NID_uacurve5, 13, &so[7545]}, - {"uacurve6", "DSTU curve 6", NID_uacurve6, 13, &so[7558]}, - {"uacurve7", "DSTU curve 7", NID_uacurve7, 13, &so[7571]}, - {"uacurve8", "DSTU curve 8", NID_uacurve8, 13, &so[7584]}, - {"uacurve9", "DSTU curve 9", NID_uacurve9, 13, &so[7597]}, - {"ieee", "ieee", NID_ieee, 2, &so[7610]}, - {"ieee-siswg", "IEEE Security in Storage Working Group", NID_ieee_siswg, 5, &so[7612]}, - {"SM2", "sm2", NID_sm2, 8, &so[7617]}, - {"id-tc26-cipher-gostr3412-2015-magma", "id-tc26-cipher-gostr3412-2015-magma", NID_id_tc26_cipher_gostr3412_2015_magma, 8, &so[7625]}, - {"magma-ctr-acpkm", "magma-ctr-acpkm", NID_magma_ctr_acpkm, 9, &so[7633]}, - {"magma-ctr-acpkm-omac", "magma-ctr-acpkm-omac", NID_magma_ctr_acpkm_omac, 9, &so[7642]}, - {"id-tc26-cipher-gostr3412-2015-kuznyechik", "id-tc26-cipher-gostr3412-2015-kuznyechik", NID_id_tc26_cipher_gostr3412_2015_kuznyechik, 8, &so[7651]}, - {"kuznyechik-ctr-acpkm", "kuznyechik-ctr-acpkm", NID_kuznyechik_ctr_acpkm, 9, &so[7659]}, - {"kuznyechik-ctr-acpkm-omac", "kuznyechik-ctr-acpkm-omac", NID_kuznyechik_ctr_acpkm_omac, 9, &so[7668]}, - {"id-tc26-wrap", "id-tc26-wrap", NID_id_tc26_wrap, 7, &so[7677]}, - {"id-tc26-wrap-gostr3412-2015-magma", "id-tc26-wrap-gostr3412-2015-magma", NID_id_tc26_wrap_gostr3412_2015_magma, 8, &so[7684]}, - {"magma-kexp15", "magma-kexp15", NID_magma_kexp15, 9, &so[7692]}, - {"id-tc26-wrap-gostr3412-2015-kuznyechik", "id-tc26-wrap-gostr3412-2015-kuznyechik", NID_id_tc26_wrap_gostr3412_2015_kuznyechik, 8, &so[7701]}, - {"kuznyechik-kexp15", "kuznyechik-kexp15", NID_kuznyechik_kexp15, 9, &so[7709]}, - {"id-tc26-gost-3410-2012-256-paramSetB", "GOST R 34.10-2012 (256 bit) ParamSet B", NID_id_tc26_gost_3410_2012_256_paramSetB, 9, &so[7718]}, - {"id-tc26-gost-3410-2012-256-paramSetC", "GOST R 34.10-2012 (256 bit) ParamSet C", NID_id_tc26_gost_3410_2012_256_paramSetC, 9, &so[7727]}, - {"id-tc26-gost-3410-2012-256-paramSetD", "GOST R 34.10-2012 (256 bit) ParamSet D", NID_id_tc26_gost_3410_2012_256_paramSetD, 9, &so[7736]}, - {"magma-ecb", "magma-ecb", NID_magma_ecb}, - {"magma-ctr", "magma-ctr", NID_magma_ctr}, - {"magma-ofb", "magma-ofb", NID_magma_ofb}, - {"magma-cbc", "magma-cbc", NID_magma_cbc}, - {"magma-cfb", "magma-cfb", NID_magma_cfb}, - {"magma-mac", "magma-mac", NID_magma_mac}, - {"hmacWithSHA512-224", "hmacWithSHA512-224", NID_hmacWithSHA512_224, 8, &so[7745]}, - {"hmacWithSHA512-256", "hmacWithSHA512-256", NID_hmacWithSHA512_256, 8, &so[7753]}, - {"GMAC", "gmac", NID_gmac, 5, &so[7761]}, - {"KMAC128", "kmac128", NID_kmac128, 9, &so[7766]}, - {"KMAC256", "kmac256", NID_kmac256, 9, &so[7775]}, + {"cmcCA", "CMC Certificate Authority", NID_cmcCA, 8, &so[5986]}, + {"cmcRA", "CMC Registration Authority", NID_cmcRA, 8, &so[5994]}, + {"SM4-ECB", "sm4-ecb", NID_sm4_ecb, 8, &so[6002]}, + {"SM4-CBC", "sm4-cbc", NID_sm4_cbc, 8, &so[6010]}, + {"SM4-OFB", "sm4-ofb", NID_sm4_ofb128, 8, &so[6018]}, + {"SM4-CFB1", "sm4-cfb1", NID_sm4_cfb1, 8, &so[6026]}, + {"SM4-CFB", "sm4-cfb", NID_sm4_cfb128, 8, &so[6034]}, + {"SM4-CFB8", "sm4-cfb8", NID_sm4_cfb8, 8, &so[6042]}, + {"SM4-CTR", "sm4-ctr", NID_sm4_ctr, 8, &so[6050]}, + {"ISO-CN", "ISO CN Member Body", NID_ISO_CN, 3, &so[6058]}, + {"oscca", "oscca", NID_oscca, 5, &so[6061]}, + {"sm-scheme", "sm-scheme", NID_sm_scheme, 6, &so[6066]}, + {"SM3", "sm3", NID_sm3, 8, &so[6072]}, + {"RSA-SM3", "sm3WithRSAEncryption", NID_sm3WithRSAEncryption, 8, &so[6080]}, + {"RSA-SHA512/224", "sha512-224WithRSAEncryption", NID_sha512_224WithRSAEncryption, 9, &so[6088]}, + {"RSA-SHA512/256", "sha512-256WithRSAEncryption", NID_sha512_256WithRSAEncryption, 9, &so[6097]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"ISO-UA", "ISO-UA", NID_ISO_UA, 3, &so[6106]}, + {"ua-pki", "ua-pki", NID_ua_pki, 7, &so[6109]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"dstu4145le", "DSTU 4145-2002 little endian", NID_dstu4145le, 11, &so[6116]}, + {"dstu4145be", "DSTU 4145-2002 big endian", NID_dstu4145be, 13, &so[6127]}, + {"uacurve0", "DSTU curve 0", NID_uacurve0, 13, &so[6140]}, + {"uacurve1", "DSTU curve 1", NID_uacurve1, 13, &so[6153]}, + {"uacurve2", "DSTU curve 2", NID_uacurve2, 13, &so[6166]}, + {"uacurve3", "DSTU curve 3", NID_uacurve3, 13, &so[6179]}, + {"uacurve4", "DSTU curve 4", NID_uacurve4, 13, &so[6192]}, + {"uacurve5", "DSTU curve 5", NID_uacurve5, 13, &so[6205]}, + {"uacurve6", "DSTU curve 6", NID_uacurve6, 13, &so[6218]}, + {"uacurve7", "DSTU curve 7", NID_uacurve7, 13, &so[6231]}, + {"uacurve8", "DSTU curve 8", NID_uacurve8, 13, &so[6244]}, + {"uacurve9", "DSTU curve 9", NID_uacurve9, 13, &so[6257]}, + {"ieee", "ieee", NID_ieee, 2, &so[6270]}, + {"ieee-siswg", "IEEE Security in Storage Working Group", NID_ieee_siswg, 5, &so[6272]}, + {"SM2", "sm2", NID_sm2, 8, &so[6277]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"hmacWithSHA512-224", "hmacWithSHA512-224", NID_hmacWithSHA512_224, 8, &so[6285]}, + {"hmacWithSHA512-256", "hmacWithSHA512-256", NID_hmacWithSHA512_256, 8, &so[6293]}, + {"GMAC", "gmac", NID_gmac, 5, &so[6301]}, + {"KMAC128", "kmac128", NID_kmac128, 9, &so[6306]}, + {"KMAC256", "kmac256", NID_kmac256, 9, &so[6315]}, {"AES-128-SIV", "aes-128-siv", NID_aes_128_siv}, {"AES-192-SIV", "aes-192-siv", NID_aes_192_siv}, {"AES-256-SIV", "aes-256-siv", NID_aes_256_siv}, - {"BLAKE2BMAC", "blake2bmac", NID_blake2bmac, 10, &so[7784]}, - {"BLAKE2SMAC", "blake2smac", NID_blake2smac, 10, &so[7794]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, {"SSHKDF", "sshkdf", NID_sshkdf}, - {"SM2-SM3", "SM2-with-SM3", NID_SM2_with_SM3, 8, &so[7804]}, + {"SM2-SM3", "SM2-with-SM3", NID_SM2_with_SM3, 8, &so[6324]}, {"SSKDF", "sskdf", NID_sskdf}, {"X963KDF", "x963kdf", NID_x963kdf}, {"X942KDF", "x942kdf", NID_x942kdf}, - {"id-on-SmtpUTF8Mailbox", "Smtp UTF8 Mailbox", NID_id_on_SmtpUTF8Mailbox, 8, &so[7812]}, - {"id-on-xmppAddr", "XmppAddr", NID_XmppAddr, 8, &so[7820]}, - {"id-on-dnsSRV", "SRVName", NID_SRVName, 8, &so[7828]}, - {"id-on-NAIRealm", "NAIRealm", NID_NAIRealm, 8, &so[7836]}, + {"id-on-SmtpUTF8Mailbox", "Smtp UTF8 Mailbox", NID_id_on_SmtpUTF8Mailbox, 8, &so[6332]}, + {"id-on-xmppAddr", "XmppAddr", NID_XmppAddr, 8, &so[6340]}, + {"id-on-dnsSRV", "SRVName", NID_SRVName, 8, &so[6348]}, + {"id-on-NAIRealm", "NAIRealm", NID_NAIRealm, 8, &so[6356]}, {"modp_1536", "modp_1536", NID_modp_1536}, {"modp_2048", "modp_2048", NID_modp_2048}, {"modp_3072", "modp_3072", NID_modp_3072}, {"modp_4096", "modp_4096", NID_modp_4096}, {"modp_6144", "modp_6144", NID_modp_6144}, {"modp_8192", "modp_8192", NID_modp_8192}, - {"KxGOST18", "kx-gost18", NID_kx_gost18}, - {"cmcArchive", "CMC Archive Server", NID_cmcArchive, 8, &so[7844]}, - {"id-kp-bgpsec-router", "BGPsec Router", NID_id_kp_bgpsec_router, 8, &so[7852]}, - {"id-kp-BrandIndicatorforMessageIdentification", "Brand Indicator for Message Identification", NID_id_kp_BrandIndicatorforMessageIdentification, 8, &so[7860]}, - {"cmKGA", "Certificate Management Key Generation Authority", NID_cmKGA, 8, &so[7868]}, - {"id-it-caCerts", "id-it-caCerts", NID_id_it_caCerts, 8, &so[7876]}, - {"id-it-rootCaKeyUpdate", "id-it-rootCaKeyUpdate", NID_id_it_rootCaKeyUpdate, 8, &so[7884]}, - {"id-it-certReqTemplate", "id-it-certReqTemplate", NID_id_it_certReqTemplate, 8, &so[7892]}, - {"OGRNIP", "OGRNIP", NID_OGRNIP, 5, &so[7900]}, - {"classSignTool", "Class of Signing Tool", NID_classSignTool, 5, &so[7905]}, - {"classSignToolKC1", "Class of Signing Tool KC1", NID_classSignToolKC1, 6, &so[7910]}, - {"classSignToolKC2", "Class of Signing Tool KC2", NID_classSignToolKC2, 6, &so[7916]}, - {"classSignToolKC3", "Class of Signing Tool KC3", NID_classSignToolKC3, 6, &so[7922]}, - {"classSignToolKB1", "Class of Signing Tool KB1", NID_classSignToolKB1, 6, &so[7928]}, - {"classSignToolKB2", "Class of Signing Tool KB2", NID_classSignToolKB2, 6, &so[7934]}, - {"classSignToolKA1", "Class of Signing Tool KA1", NID_classSignToolKA1, 6, &so[7940]}, - {"id-ct-routeOriginAuthz", "id-ct-routeOriginAuthz", NID_id_ct_routeOriginAuthz, 11, &so[7946]}, - {"id-ct-rpkiManifest", "id-ct-rpkiManifest", NID_id_ct_rpkiManifest, 11, &so[7957]}, - {"id-ct-rpkiGhostbusters", "id-ct-rpkiGhostbusters", NID_id_ct_rpkiGhostbusters, 11, &so[7968]}, - {"id-ct-resourceTaggedAttest", "id-ct-resourceTaggedAttest", NID_id_ct_resourceTaggedAttest, 11, &so[7979]}, - {"id-cp", "id-cp", NID_id_cp, 7, &so[7990]}, - {"sbgp-ipAddrBlockv2", "sbgp-ipAddrBlockv2", NID_sbgp_ipAddrBlockv2, 8, &so[7997]}, - {"sbgp-autonomousSysNumv2", "sbgp-autonomousSysNumv2", NID_sbgp_autonomousSysNumv2, 8, &so[8005]}, - {"ipAddr-asNumber", "ipAddr-asNumber", NID_ipAddr_asNumber, 8, &so[8013]}, - {"ipAddr-asNumberv2", "ipAddr-asNumberv2", NID_ipAddr_asNumberv2, 8, &so[8021]}, - {"rpkiManifest", "RPKI Manifest", NID_rpkiManifest, 8, &so[8029]}, - {"signedObject", "Signed Object", NID_signedObject, 8, &so[8037]}, - {"rpkiNotify", "RPKI Notify", NID_rpkiNotify, 8, &so[8045]}, - {"id-ct-geofeedCSVwithCRLF", "id-ct-geofeedCSVwithCRLF", NID_id_ct_geofeedCSVwithCRLF, 11, &so[8053]}, - {"id-ct-signedChecklist", "id-ct-signedChecklist", NID_id_ct_signedChecklist, 11, &so[8064]}, - {"SM4-GCM", "sm4-gcm", NID_sm4_gcm, 8, &so[8075]}, - {"SM4-CCM", "sm4-ccm", NID_sm4_ccm, 8, &so[8083]}, - {"id-ct-ASPA", "id-ct-ASPA", NID_id_ct_ASPA, 11, &so[8091]}, - {"id-mod-cmp2000-02", "id-mod-cmp2000-02", NID_id_mod_cmp2000_02, 8, &so[8102]}, - {"id-mod-cmp2021-88", "id-mod-cmp2021-88", NID_id_mod_cmp2021_88, 8, &so[8110]}, - {"id-mod-cmp2021-02", "id-mod-cmp2021-02", NID_id_mod_cmp2021_02, 8, &so[8118]}, - {"id-it-rootCaCert", "id-it-rootCaCert", NID_id_it_rootCaCert, 8, &so[8126]}, - {"id-it-certProfile", "id-it-certProfile", NID_id_it_certProfile, 8, &so[8134]}, - {"id-it-crlStatusList", "id-it-crlStatusList", NID_id_it_crlStatusList, 8, &so[8142]}, - {"id-it-crls", "id-it-crls", NID_id_it_crls, 8, &so[8150]}, - {"id-regCtrl-altCertTemplate", "id-regCtrl-altCertTemplate", NID_id_regCtrl_altCertTemplate, 9, &so[8158]}, - {"id-regCtrl-algId", "id-regCtrl-algId", NID_id_regCtrl_algId, 9, &so[8167]}, - {"id-regCtrl-rsaKeyLen", "id-regCtrl-rsaKeyLen", NID_id_regCtrl_rsaKeyLen, 9, &so[8176]}, - {"id-aa-ets-attrCertificateRefs", "id-aa-ets-attrCertificateRefs", NID_id_aa_ets_attrCertificateRefs, 11, &so[8185]}, - {"id-aa-ets-attrRevocationRefs", "id-aa-ets-attrRevocationRefs", NID_id_aa_ets_attrRevocationRefs, 11, &so[8196]}, - {"id-aa-CMSAlgorithmProtection", "id-aa-CMSAlgorithmProtection", NID_id_aa_CMSAlgorithmProtection, 9, &so[8207]}, - {"itu-t-identified-organization", "itu-t-identified-organization", NID_itu_t_identified_organization, 1, &so[8216]}, - {"etsi", "etsi", NID_etsi, 2, &so[8217]}, - {"electronic-signature-standard", "electronic-signature-standard", NID_electronic_signature_standard, 4, &so[8219]}, - {"ess-attributes", "ess-attributes", NID_ess_attributes, 5, &so[8223]}, - {"id-aa-ets-mimeType", "id-aa-ets-mimeType", NID_id_aa_ets_mimeType, 6, &so[8228]}, - {"id-aa-ets-longTermValidation", "id-aa-ets-longTermValidation", NID_id_aa_ets_longTermValidation, 6, &so[8234]}, - {"id-aa-ets-SignaturePolicyDocument", "id-aa-ets-SignaturePolicyDocument", NID_id_aa_ets_SignaturePolicyDocument, 6, &so[8240]}, - {"id-aa-ets-archiveTimestampV3", "id-aa-ets-archiveTimestampV3", NID_id_aa_ets_archiveTimestampV3, 6, &so[8246]}, - {"id-aa-ATSHashIndex", "id-aa-ATSHashIndex", NID_id_aa_ATSHashIndex, 6, &so[8252]}, - {"cades", "cades", NID_cades, 5, &so[8258]}, - {"cades-attributes", "cades-attributes", NID_cades_attributes, 6, &so[8263]}, - {"id-aa-ets-signerAttrV2", "id-aa-ets-signerAttrV2", NID_id_aa_ets_signerAttrV2, 7, &so[8269]}, - {"id-aa-ets-sigPolicyStore", "id-aa-ets-sigPolicyStore", NID_id_aa_ets_sigPolicyStore, 7, &so[8276]}, - {"id-aa-ATSHashIndex-v2", "id-aa-ATSHashIndex-v2", NID_id_aa_ATSHashIndex_v2, 7, &so[8283]}, - {"id-aa-ATSHashIndex-v3", "id-aa-ATSHashIndex-v3", NID_id_aa_ATSHashIndex_v3, 7, &so[8290]}, - {"signedAssertion", "signedAssertion", NID_signedAssertion, 7, &so[8297]}, - {"id-aa-ets-archiveTimestampV2", "id-aa-ets-archiveTimestampV2", NID_id_aa_ets_archiveTimestampV2, 11, &so[8304]}, - {"hmacWithSM3", "hmacWithSM3", NID_hmacWithSM3, 10, &so[8315]}, - {"oracle-organization", "Oracle organization", NID_oracle, 7, &so[8325]}, - {"oracle-jdk-trustedkeyusage", "Trusted key usage (Oracle)", NID_oracle_jdk_trustedkeyusage, 12, &so[8332]}, - {"id-ct-signedTAL", "id-ct-signedTAL", NID_id_ct_signedTAL, 11, &so[8344]}, - {"brainpoolP256r1tls13", "brainpoolP256r1tls13", NID_brainpoolP256r1tls13}, - {"brainpoolP384r1tls13", "brainpoolP384r1tls13", NID_brainpoolP384r1tls13}, - {"brainpoolP512r1tls13", "brainpoolP512r1tls13", NID_brainpoolP512r1tls13}, - {"brotli", "Brotli compression", NID_brotli}, - {"zstd", "Zstandard compression", NID_zstd}, - {"SM4-XTS", "sm4-xts", NID_sm4_xts, 8, &so[8355]}, - {"ms-ntds-obj-sid", "Microsoft NTDS AD objectSid", NID_ms_ntds_obj_sid, 10, &so[8363]}, - {"ms-ntds-sec-ext", "Microsoft NTDS CA Extension", NID_ms_ntds_sec_ext, 9, &so[8373]}, - {"ms-cert-templ", "Microsoft certificate template", NID_ms_cert_templ, 9, &so[8382]}, - {"ms-app-policies", "Microsoft Application Policies Extension", NID_ms_app_policies, 9, &so[8391]}, - {"authorityAttributeIdentifier", "X509v3 Authority Attribute Identifier", NID_authority_attribute_identifier, 3, &so[8400]}, - {"roleSpecCertIdentifier", "X509v3 Role Specification Certificate Identifier", NID_role_spec_cert_identifier, 3, &so[8403]}, - {"basicAttConstraints", "X509v3 Basic Attribute Certificate Constraints", NID_basic_att_constraints, 3, &so[8406]}, - {"delegatedNameConstraints", "X509v3 Delegated Name Constraints", NID_delegated_name_constraints, 3, &so[8409]}, - {"timeSpecification", "X509v3 Time Specification", NID_time_specification, 3, &so[8412]}, - {"attributeDescriptor", "X509v3 Attribute Descriptor", NID_attribute_descriptor, 3, &so[8415]}, - {"userNotice", "X509v3 User Notice", NID_user_notice, 3, &so[8418]}, - {"sOAIdentifier", "X509v3 Source of Authority Identifier", NID_soa_identifier, 3, &so[8421]}, - {"acceptableCertPolicies", "X509v3 Acceptable Certification Policies", NID_acceptable_cert_policies, 3, &so[8424]}, - {"acceptablePrivPolicies", "X509v3 Acceptable Privilege Policies", NID_acceptable_privilege_policies, 3, &so[8427]}, - {"indirectIssuer", "X509v3 Indirect Issuer", NID_indirect_issuer, 3, &so[8430]}, - {"noAssertion", "X509v3 No Assertion", NID_no_assertion, 3, &so[8433]}, - {"aAissuingDistributionPoint", "X509v3 Attribute Authority Issuing Distribution Point", NID_id_aa_issuing_distribution_point, 3, &so[8436]}, - {"issuedOnBehalfOf", "X509v3 Issued On Behalf Of", NID_issued_on_behalf_of, 3, &so[8439]}, - {"singleUse", "X509v3 Single Use", NID_single_use, 3, &so[8442]}, - {"groupAC", "X509v3 Group Attribute Certificate", NID_group_ac, 3, &so[8445]}, - {"allowedAttributeAssignments", "X509v3 Allowed Attribute Assignments", NID_allowed_attribute_assignments, 3, &so[8448]}, - {"attributeMappings", "X509v3 Attribute Mappings", NID_attribute_mappings, 3, &so[8451]}, - {"holderNameConstraints", "X509v3 Holder Name Constraints", NID_holder_name_constraints, 3, &so[8454]}, - {"authorizationValidation", "X509v3 Authorization Validation", NID_authorization_validation, 3, &so[8457]}, - {"protRestrict", "X509v3 Protocol Restriction", NID_prot_restrict, 3, &so[8460]}, - {"subjectAltPublicKeyInfo", "X509v3 Subject Alternative Public Key Info", NID_subject_alt_public_key_info, 3, &so[8463]}, - {"altSignatureAlgorithm", "X509v3 Alternative Signature Algorithm", NID_alt_signature_algorithm, 3, &so[8466]}, - {"altSignatureValue", "X509v3 Alternative Signature Value", NID_alt_signature_value, 3, &so[8469]}, - {"associatedInformation", "X509v3 Associated Information", NID_associated_information, 3, &so[8472]}, - {"id-ct-rpkiSignedPrefixList", "id-ct-rpkiSignedPrefixList", NID_id_ct_rpkiSignedPrefixList, 11, &so[8475]}, + { NULL, NULL, NID_undef }, + {"cmcArchive", "CMC Archive Server", NID_cmcArchive, 8, &so[6364]}, + {"id-kp-bgpsec-router", "BGPsec Router", NID_id_kp_bgpsec_router, 8, &so[6372]}, + {"id-kp-BrandIndicatorforMessageIdentification", "Brand Indicator for Message Identification", NID_id_kp_BrandIndicatorforMessageIdentification, 8, &so[6380]}, + {"cmKGA", "Certificate Management Key Generation Authority", NID_cmKGA, 8, &so[6388]}, + {"id-it-caCerts", "id-it-caCerts", NID_id_it_caCerts, 8, &so[6396]}, + {"id-it-rootCaKeyUpdate", "id-it-rootCaKeyUpdate", NID_id_it_rootCaKeyUpdate, 8, &so[6404]}, + {"id-it-certReqTemplate", "id-it-certReqTemplate", NID_id_it_certReqTemplate, 8, &so[6412]}, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + { NULL, NULL, NID_undef }, + {"id-ct-routeOriginAuthz", "id-ct-routeOriginAuthz", NID_id_ct_routeOriginAuthz, 11, &so[6420]}, + {"id-ct-rpkiManifest", "id-ct-rpkiManifest", NID_id_ct_rpkiManifest, 11, &so[6431]}, + {"id-ct-rpkiGhostbusters", "id-ct-rpkiGhostbusters", NID_id_ct_rpkiGhostbusters, 11, &so[6442]}, + {"id-ct-resourceTaggedAttest", "id-ct-resourceTaggedAttest", NID_id_ct_resourceTaggedAttest, 11, &so[6453]}, + {"id-cp", "id-cp", NID_id_cp, 7, &so[6464]}, + {"sbgp-ipAddrBlockv2", "sbgp-ipAddrBlockv2", NID_sbgp_ipAddrBlockv2, 8, &so[6471]}, + {"sbgp-autonomousSysNumv2", "sbgp-autonomousSysNumv2", NID_sbgp_autonomousSysNumv2, 8, &so[6479]}, + {"ipAddr-asNumber", "ipAddr-asNumber", NID_ipAddr_asNumber, 8, &so[6487]}, + {"ipAddr-asNumberv2", "ipAddr-asNumberv2", NID_ipAddr_asNumberv2, 8, &so[6495]}, + {"rpkiManifest", "RPKI Manifest", NID_rpkiManifest, 8, &so[6503]}, + {"signedObject", "Signed Object", NID_signedObject, 8, &so[6511]}, + {"rpkiNotify", "RPKI Notify", NID_rpkiNotify, 8, &so[6519]}, + {"id-ct-geofeedCSVwithCRLF", "id-ct-geofeedCSVwithCRLF", NID_id_ct_geofeedCSVwithCRLF, 11, &so[6527]}, + {"id-ct-signedChecklist", "id-ct-signedChecklist", NID_id_ct_signedChecklist, 11, &so[6538]}, + {"ZUC", "zuc", NID_zuc, 8, &so[6549]}, + {"ZUC-128-EEA3", "zuc-128-eea3", NID_zuc_128_eea3, 8, &so[6557]}, + {"SM4-GCM", "sm4-gcm", NID_sm4_gcm, 8, &so[6565]}, + {"SM4-CCM", "sm4-ccm", NID_sm4_ccm, 8, &so[6573]}, + {"KxSM2", "kx-sm2", NID_kx_sm2}, + {"KxSM2DHE", "kx-sm2dhe", NID_kx_sm2dhe}, + {"AuthSM2", "auth-sm2", NID_auth_sm2}, + {"ZUC-128-EIA3", "zuc-128-eia3", NID_zuc_128_eia3, 8, &so[6581]}, + {"delegationUsage", "X509v3 Delegation Usage", NID_delegation_usage, 9, &so[6589]}, + {"hmacWithSM3", "hmacWithSM3", NID_hmacWithSM3, 10, &so[6598]}, }; -#define NUM_SN 1312 +#define NUM_SN 1008 static const unsigned int sn_objs[NUM_SN] = { 364, /* "AD_DVCS" */ 419, /* "AES-128-CBC" */ @@ -2548,85 +2231,15 @@ static const unsigned int sn_objs[NUM_SN] = { 428, /* "AES-256-OFB" */ 1200, /* "AES-256-SIV" */ 914, /* "AES-256-XTS" */ - 1066, /* "ARIA-128-CBC" */ - 1120, /* "ARIA-128-CCM" */ - 1067, /* "ARIA-128-CFB" */ - 1080, /* "ARIA-128-CFB1" */ - 1083, /* "ARIA-128-CFB8" */ - 1069, /* "ARIA-128-CTR" */ - 1065, /* "ARIA-128-ECB" */ - 1123, /* "ARIA-128-GCM" */ - 1068, /* "ARIA-128-OFB" */ - 1071, /* "ARIA-192-CBC" */ - 1121, /* "ARIA-192-CCM" */ - 1072, /* "ARIA-192-CFB" */ - 1081, /* "ARIA-192-CFB1" */ - 1084, /* "ARIA-192-CFB8" */ - 1074, /* "ARIA-192-CTR" */ - 1070, /* "ARIA-192-ECB" */ - 1124, /* "ARIA-192-GCM" */ - 1073, /* "ARIA-192-OFB" */ - 1076, /* "ARIA-256-CBC" */ - 1122, /* "ARIA-256-CCM" */ - 1077, /* "ARIA-256-CFB" */ - 1082, /* "ARIA-256-CFB1" */ - 1085, /* "ARIA-256-CFB8" */ - 1079, /* "ARIA-256-CTR" */ - 1075, /* "ARIA-256-ECB" */ - 1125, /* "ARIA-256-GCM" */ - 1078, /* "ARIA-256-OFB" */ 1064, /* "AuthANY" */ 1049, /* "AuthDSS" */ 1047, /* "AuthECDSA" */ - 1050, /* "AuthGOST01" */ - 1051, /* "AuthGOST12" */ 1053, /* "AuthNULL" */ 1048, /* "AuthPSK" */ 1046, /* "AuthRSA" */ + 1254, /* "AuthSM2" */ 1052, /* "AuthSRP" */ - 91, /* "BF-CBC" */ - 93, /* "BF-CFB" */ - 92, /* "BF-ECB" */ - 94, /* "BF-OFB" */ - 1201, /* "BLAKE2BMAC" */ - 1202, /* "BLAKE2SMAC" */ - 1056, /* "BLAKE2b512" */ - 1057, /* "BLAKE2s256" */ 14, /* "C" */ - 751, /* "CAMELLIA-128-CBC" */ - 962, /* "CAMELLIA-128-CCM" */ - 757, /* "CAMELLIA-128-CFB" */ - 760, /* "CAMELLIA-128-CFB1" */ - 763, /* "CAMELLIA-128-CFB8" */ - 964, /* "CAMELLIA-128-CMAC" */ - 963, /* "CAMELLIA-128-CTR" */ - 754, /* "CAMELLIA-128-ECB" */ - 961, /* "CAMELLIA-128-GCM" */ - 766, /* "CAMELLIA-128-OFB" */ - 752, /* "CAMELLIA-192-CBC" */ - 966, /* "CAMELLIA-192-CCM" */ - 758, /* "CAMELLIA-192-CFB" */ - 761, /* "CAMELLIA-192-CFB1" */ - 764, /* "CAMELLIA-192-CFB8" */ - 968, /* "CAMELLIA-192-CMAC" */ - 967, /* "CAMELLIA-192-CTR" */ - 755, /* "CAMELLIA-192-ECB" */ - 965, /* "CAMELLIA-192-GCM" */ - 767, /* "CAMELLIA-192-OFB" */ - 753, /* "CAMELLIA-256-CBC" */ - 970, /* "CAMELLIA-256-CCM" */ - 759, /* "CAMELLIA-256-CFB" */ - 762, /* "CAMELLIA-256-CFB1" */ - 765, /* "CAMELLIA-256-CFB8" */ - 972, /* "CAMELLIA-256-CMAC" */ - 971, /* "CAMELLIA-256-CTR" */ - 756, /* "CAMELLIA-256-ECB" */ - 969, /* "CAMELLIA-256-GCM" */ - 768, /* "CAMELLIA-256-OFB" */ - 108, /* "CAST5-CBC" */ - 110, /* "CAST5-CFB" */ - 109, /* "CAST5-ECB" */ - 111, /* "CAST5-OFB" */ 894, /* "CMAC" */ 13, /* "CN" */ 141, /* "CRLReason" */ @@ -2669,18 +2282,12 @@ static const unsigned int sn_objs[NUM_SN] = { 780, /* "HMAC-MD5" */ 781, /* "HMAC-SHA1" */ 381, /* "IANA" */ - 34, /* "IDEA-CBC" */ - 35, /* "IDEA-CFB" */ - 36, /* "IDEA-ECB" */ - 46, /* "IDEA-OFB" */ - 1004, /* "INN" */ 181, /* "ISO" */ 1140, /* "ISO-CN" */ 1150, /* "ISO-UA" */ 183, /* "ISO-US" */ 645, /* "ITU-T" */ 646, /* "JOINT-ISO-ITU-T" */ - 773, /* "KISA" */ 1196, /* "KMAC128" */ 1197, /* "KMAC256" */ 1063, /* "KxANY" */ @@ -2688,19 +2295,16 @@ static const unsigned int sn_objs[NUM_SN] = { 1041, /* "KxDHE-PSK" */ 1038, /* "KxECDHE" */ 1040, /* "KxECDHE-PSK" */ - 1045, /* "KxGOST" */ - 1218, /* "KxGOST18" */ 1043, /* "KxPSK" */ 1037, /* "KxRSA" */ 1042, /* "KxRSA_PSK" */ + 1252, /* "KxSM2" */ + 1253, /* "KxSM2DHE" */ 1044, /* "KxSRP" */ 15, /* "L" */ 856, /* "LocalKeySet" */ - 3, /* "MD2" */ - 257, /* "MD4" */ 4, /* "MD5" */ 114, /* "MD5-SHA1" */ - 95, /* "MDC2" */ 911, /* "MGF1" */ 388, /* "Mail" */ 393, /* "NULL" */ @@ -2710,22 +2314,14 @@ static const unsigned int sn_objs[NUM_SN] = { 17, /* "O" */ 178, /* "OCSP" */ 180, /* "OCSPSigning" */ - 1005, /* "OGRN" */ - 1226, /* "OGRNIP" */ 379, /* "ORG" */ 18, /* "OU" */ 749, /* "Oakley-EC2N-3" */ 750, /* "Oakley-EC2N-4" */ - 9, /* "PBE-MD2-DES" */ - 168, /* "PBE-MD2-RC2-64" */ 10, /* "PBE-MD5-DES" */ - 169, /* "PBE-MD5-RC2-64" */ 147, /* "PBE-SHA1-2DES" */ 146, /* "PBE-SHA1-3DES" */ 170, /* "PBE-SHA1-DES" */ - 148, /* "PBE-SHA1-RC2-128" */ - 149, /* "PBE-SHA1-RC2-40" */ - 68, /* "PBE-SHA1-RC2-64" */ 144, /* "PBE-SHA1-RC4-128" */ 145, /* "PBE-SHA1-RC4-40" */ 161, /* "PBES2" */ @@ -2734,12 +2330,6 @@ static const unsigned int sn_objs[NUM_SN] = { 127, /* "PKIX" */ 935, /* "PSPECIFIED" */ 1061, /* "Poly1305" */ - 98, /* "RC2-40-CBC" */ - 166, /* "RC2-64-CBC" */ - 37, /* "RC2-CBC" */ - 39, /* "RC2-CFB" */ - 38, /* "RC2-ECB" */ - 40, /* "RC2-OFB" */ 5, /* "RC4" */ 97, /* "RC4-40" */ 915, /* "RC4-HMAC-MD5" */ @@ -2747,14 +2337,9 @@ static const unsigned int sn_objs[NUM_SN] = { 122, /* "RC5-CFB" */ 121, /* "RC5-ECB" */ 123, /* "RC5-OFB" */ - 117, /* "RIPEMD160" */ 19, /* "RSA" */ - 7, /* "RSA-MD2" */ - 396, /* "RSA-MD4" */ 8, /* "RSA-MD5" */ - 96, /* "RSA-MDC2" */ 104, /* "RSA-NP-MD5" */ - 119, /* "RSA-RIPEMD160" */ 42, /* "RSA-SHA" */ 65, /* "RSA-SHA1" */ 115, /* "RSA-SHA1-2" */ @@ -2767,10 +2352,6 @@ static const unsigned int sn_objs[NUM_SN] = { 1144, /* "RSA-SM3" */ 919, /* "RSAES-OAEP" */ 912, /* "RSASSA-PSS" */ - 777, /* "SEED-CBC" */ - 779, /* "SEED-CFB" */ - 776, /* "SEED-ECB" */ - 778, /* "SEED-OFB" */ 41, /* "SHA" */ 64, /* "SHA1" */ 675, /* "SHA224" */ @@ -2789,19 +2370,17 @@ static const unsigned int sn_objs[NUM_SN] = { 1204, /* "SM2-SM3" */ 1143, /* "SM3" */ 1134, /* "SM4-CBC" */ - 1249, /* "SM4-CCM" */ + 1251, /* "SM4-CCM" */ 1137, /* "SM4-CFB" */ 1136, /* "SM4-CFB1" */ 1138, /* "SM4-CFB8" */ 1139, /* "SM4-CTR" */ 1133, /* "SM4-ECB" */ - 1248, /* "SM4-GCM" */ + 1250, /* "SM4-GCM" */ 1135, /* "SM4-OFB" */ - 1290, /* "SM4-XTS" */ 188, /* "SMIME" */ 167, /* "SMIME-CAPS" */ 100, /* "SN" */ - 1006, /* "SNILS" */ 1203, /* "SSHKDF" */ 1205, /* "SSKDF" */ 16, /* "ST" */ @@ -2820,37 +2399,28 @@ static const unsigned int sn_objs[NUM_SN] = { 1206, /* "X963KDF" */ 185, /* "X9cm" */ 125, /* "ZLIB" */ - 1307, /* "aAissuingDistributionPoint" */ + 1248, /* "ZUC" */ + 1249, /* "ZUC-128-EEA3" */ + 1255, /* "ZUC-128-EIA3" */ 478, /* "aRecord" */ 289, /* "aaControls" */ 287, /* "ac-auditEntity" */ 397, /* "ac-proxying" */ 288, /* "ac-targeting" */ - 1303, /* "acceptableCertPolicies" */ - 1304, /* "acceptablePrivPolicies" */ 368, /* "acceptableResponses" */ 446, /* "account" */ 363, /* "ad_timestamping" */ 376, /* "algorithm" */ - 1311, /* "allowedAttributeAssignments" */ - 1317, /* "altSignatureAlgorithm" */ - 1318, /* "altSignatureValue" */ 405, /* "ansi-X9-62" */ 910, /* "anyExtendedKeyUsage" */ 746, /* "anyPolicy" */ 370, /* "archiveCutoff" */ 484, /* "associatedDomain" */ - 1319, /* "associatedInformation" */ 485, /* "associatedName" */ - 1300, /* "attributeDescriptor" */ - 1312, /* "attributeMappings" */ 501, /* "audio" */ - 1295, /* "authorityAttributeIdentifier" */ 177, /* "authorityInfoAccess" */ 90, /* "authorityKeyIdentifier" */ 882, /* "authorityRevocationList" */ - 1314, /* "authorizationValidation" */ - 1297, /* "basicAttConstraints" */ 87, /* "basicConstraints" */ 365, /* "basicOCSPResponse" */ 285, /* "biometricInfo" */ @@ -2861,17 +2431,13 @@ static const unsigned int sn_objs[NUM_SN] = { 925, /* "brainpoolP224r1" */ 926, /* "brainpoolP224t1" */ 927, /* "brainpoolP256r1" */ - 1285, /* "brainpoolP256r1tls13" */ 928, /* "brainpoolP256t1" */ 929, /* "brainpoolP320r1" */ 930, /* "brainpoolP320t1" */ 931, /* "brainpoolP384r1" */ - 1286, /* "brainpoolP384r1tls13" */ 932, /* "brainpoolP384t1" */ 933, /* "brainpoolP512r1" */ - 1287, /* "brainpoolP512r1tls13" */ 934, /* "brainpoolP512t1" */ - 1288, /* "brotli" */ 494, /* "buildingName" */ 860, /* "businessCategory" */ 691, /* "c2onb191v4" */ @@ -2899,8 +2465,6 @@ static const unsigned int sn_objs[NUM_SN] = { 483, /* "cNAMERecord" */ 179, /* "caIssuers" */ 785, /* "caRepository" */ - 1273, /* "cades" */ - 1274, /* "cades-attributes" */ 1023, /* "capwapAC" */ 1024, /* "capwapWTP" */ 443, /* "caseIgnoreIA5StringSyntax" */ @@ -2911,13 +2475,6 @@ static const unsigned int sn_objs[NUM_SN] = { 883, /* "certificateRevocationList" */ 54, /* "challengePassword" */ 407, /* "characteristic-two-field" */ - 1227, /* "classSignTool" */ - 1233, /* "classSignToolKA1" */ - 1231, /* "classSignToolKB1" */ - 1232, /* "classSignToolKB2" */ - 1228, /* "classSignToolKC1" */ - 1229, /* "classSignToolKC2" */ - 1230, /* "classSignToolKC3" */ 395, /* "clearance" */ 130, /* "clientAuth" */ 1222, /* "cmKGA" */ @@ -2931,8 +2488,6 @@ static const unsigned int sn_objs[NUM_SN] = { 103, /* "crlDistributionPoints" */ 88, /* "crlNumber" */ 884, /* "crossCertificatePair" */ - 806, /* "cryptocom" */ - 805, /* "cryptopro" */ 954, /* "ct_cert_scts" */ 952, /* "ct_precert_poison" */ 951, /* "ct_precert_scts" */ @@ -2942,7 +2497,7 @@ static const unsigned int sn_objs[NUM_SN] = { 495, /* "dSAQuality" */ 434, /* "data" */ 390, /* "dcobject" */ - 1298, /* "delegatedNameConstraints" */ + 1256, /* "delegationUsage" */ 140, /* "deltaCRL" */ 891, /* "deltaRevocationList" */ 107, /* "description" */ @@ -2978,11 +2533,6 @@ static const unsigned int sn_objs[NUM_SN] = { 452, /* "domainRelatedObject" */ 802, /* "dsa_with_SHA224" */ 803, /* "dsa_with_SHA256" */ - 1152, /* "dstu28147" */ - 1154, /* "dstu28147-cfb" */ - 1153, /* "dstu28147-ofb" */ - 1155, /* "dstu28147-wrap" */ - 1157, /* "dstu34311" */ 1159, /* "dstu4145be" */ 1158, /* "dstu4145le" */ 791, /* "ecdsa-with-Recommended" */ @@ -2992,13 +2542,10 @@ static const unsigned int sn_objs[NUM_SN] = { 795, /* "ecdsa-with-SHA384" */ 796, /* "ecdsa-with-SHA512" */ 792, /* "ecdsa-with-Specified" */ - 1266, /* "electronic-signature-standard" */ 48, /* "emailAddress" */ 132, /* "emailProtection" */ 885, /* "enhancedSearchGuide" */ 389, /* "enterprises" */ - 1267, /* "ess-attributes" */ - 1265, /* "etsi" */ 384, /* "experimental" */ 172, /* "extReq" */ 56, /* "extendedCertificateAttributes" */ @@ -3016,22 +2563,6 @@ static const unsigned int sn_objs[NUM_SN] = { 490, /* "friendlyCountryName" */ 156, /* "friendlyName" */ 509, /* "generationQualifier" */ - 815, /* "gost-mac" */ - 976, /* "gost-mac-12" */ - 811, /* "gost2001" */ - 851, /* "gost2001cc" */ - 979, /* "gost2012_256" */ - 980, /* "gost2012_512" */ - 813, /* "gost89" */ - 1009, /* "gost89-cbc" */ - 814, /* "gost89-cnt" */ - 975, /* "gost89-cnt-12" */ - 1011, /* "gost89-ctr" */ - 1010, /* "gost89-ecb" */ - 812, /* "gost94" */ - 850, /* "gost94cc" */ - 1310, /* "groupAC" */ - 1156, /* "hmacWithDstu34311" */ 797, /* "hmacWithMD5" */ 163, /* "hmacWithSHA1" */ 798, /* "hmacWithSHA224" */ @@ -3040,71 +2571,18 @@ static const unsigned int sn_objs[NUM_SN] = { 801, /* "hmacWithSHA512" */ 1193, /* "hmacWithSHA512-224" */ 1194, /* "hmacWithSHA512-256" */ - 1281, /* "hmacWithSM3" */ + 1257, /* "hmacWithSM3" */ 432, /* "holdInstructionCallIssuer" */ 430, /* "holdInstructionCode" */ 431, /* "holdInstructionNone" */ 433, /* "holdInstructionReject" */ - 1313, /* "holderNameConstraints" */ 486, /* "homePostalAddress" */ 473, /* "homeTelephoneNumber" */ 466, /* "host" */ 889, /* "houseIdentifier" */ 442, /* "iA5StringSyntax" */ 783, /* "id-DHBasedMac" */ - 824, /* "id-Gost28147-89-CryptoPro-A-ParamSet" */ - 825, /* "id-Gost28147-89-CryptoPro-B-ParamSet" */ - 826, /* "id-Gost28147-89-CryptoPro-C-ParamSet" */ - 827, /* "id-Gost28147-89-CryptoPro-D-ParamSet" */ - 819, /* "id-Gost28147-89-CryptoPro-KeyMeshing" */ - 829, /* "id-Gost28147-89-CryptoPro-Oscar-1-0-ParamSet" */ - 828, /* "id-Gost28147-89-CryptoPro-Oscar-1-1-ParamSet" */ - 830, /* "id-Gost28147-89-CryptoPro-RIC-1-ParamSet" */ - 820, /* "id-Gost28147-89-None-KeyMeshing" */ - 823, /* "id-Gost28147-89-TestParamSet" */ - 849, /* "id-Gost28147-89-cc" */ - 840, /* "id-GostR3410-2001-CryptoPro-A-ParamSet" */ - 841, /* "id-GostR3410-2001-CryptoPro-B-ParamSet" */ - 842, /* "id-GostR3410-2001-CryptoPro-C-ParamSet" */ - 843, /* "id-GostR3410-2001-CryptoPro-XchA-ParamSet" */ - 844, /* "id-GostR3410-2001-CryptoPro-XchB-ParamSet" */ - 854, /* "id-GostR3410-2001-ParamSet-cc" */ - 839, /* "id-GostR3410-2001-TestParamSet" */ - 817, /* "id-GostR3410-2001DH" */ - 832, /* "id-GostR3410-94-CryptoPro-A-ParamSet" */ - 833, /* "id-GostR3410-94-CryptoPro-B-ParamSet" */ - 834, /* "id-GostR3410-94-CryptoPro-C-ParamSet" */ - 835, /* "id-GostR3410-94-CryptoPro-D-ParamSet" */ - 836, /* "id-GostR3410-94-CryptoPro-XchA-ParamSet" */ - 837, /* "id-GostR3410-94-CryptoPro-XchB-ParamSet" */ - 838, /* "id-GostR3410-94-CryptoPro-XchC-ParamSet" */ - 831, /* "id-GostR3410-94-TestParamSet" */ - 845, /* "id-GostR3410-94-a" */ - 846, /* "id-GostR3410-94-aBis" */ - 847, /* "id-GostR3410-94-b" */ - 848, /* "id-GostR3410-94-bBis" */ - 818, /* "id-GostR3410-94DH" */ - 822, /* "id-GostR3411-94-CryptoProParamSet" */ - 821, /* "id-GostR3411-94-TestParamSet" */ - 807, /* "id-GostR3411-94-with-GostR3410-2001" */ - 853, /* "id-GostR3411-94-with-GostR3410-2001-cc" */ - 808, /* "id-GostR3411-94-with-GostR3410-94" */ - 852, /* "id-GostR3411-94-with-GostR3410-94-cc" */ - 810, /* "id-HMACGostR3411-94" */ 782, /* "id-PasswordBasedMAC" */ - 1272, /* "id-aa-ATSHashIndex" */ - 1277, /* "id-aa-ATSHashIndex-v2" */ - 1278, /* "id-aa-ATSHashIndex-v3" */ - 1263, /* "id-aa-CMSAlgorithmProtection" */ - 1270, /* "id-aa-ets-SignaturePolicyDocument" */ - 1280, /* "id-aa-ets-archiveTimestampV2" */ - 1271, /* "id-aa-ets-archiveTimestampV3" */ - 1261, /* "id-aa-ets-attrCertificateRefs" */ - 1262, /* "id-aa-ets-attrRevocationRefs" */ - 1269, /* "id-aa-ets-longTermValidation" */ - 1268, /* "id-aa-ets-mimeType" */ - 1276, /* "id-aa-ets-sigPolicyStore" */ - 1275, /* "id-aa-ets-signerAttrV2" */ 266, /* "id-aca" */ 355, /* "id-aca-accessIdentity" */ 354, /* "id-aca-authenticationInfo" */ @@ -3131,9 +2609,6 @@ static const unsigned int sn_objs[NUM_SN] = { 326, /* "id-alg-dh-pop" */ 325, /* "id-alg-dh-sig-hmac-sha1" */ 324, /* "id-alg-noSignature" */ - 907, /* "id-camellia128-wrap" */ - 908, /* "id-camellia192-wrap" */ - 909, /* "id-camellia256-wrap" */ 268, /* "id-cct" */ 361, /* "id-cct-PKIData" */ 362, /* "id-cct-PKIResponse" */ @@ -3162,16 +2637,13 @@ static const unsigned int sn_objs[NUM_SN] = { 327, /* "id-cmc-statusInfo" */ 331, /* "id-cmc-transactionId" */ 1238, /* "id-cp" */ - 1250, /* "id-ct-ASPA" */ 787, /* "id-ct-asciiTextWithCRLF" */ 1246, /* "id-ct-geofeedCSVwithCRLF" */ 1237, /* "id-ct-resourceTaggedAttest" */ 1234, /* "id-ct-routeOriginAuthz" */ 1236, /* "id-ct-rpkiGhostbusters" */ 1235, /* "id-ct-rpkiManifest" */ - 1320, /* "id-ct-rpkiSignedPrefixList" */ 1247, /* "id-ct-signedChecklist" */ - 1284, /* "id-ct-signedTAL" */ 1060, /* "id-ct-xml" */ 1108, /* "id-dsa-with-sha3-224" */ 1109, /* "id-dsa-with-sha3-256" */ @@ -3194,11 +2666,8 @@ static const unsigned int sn_objs[NUM_SN] = { 1223, /* "id-it-caCerts" */ 302, /* "id-it-caKeyUpdateInfo" */ 298, /* "id-it-caProtEncCert" */ - 1255, /* "id-it-certProfile" */ 1225, /* "id-it-certReqTemplate" */ 311, /* "id-it-confirmWaitTime" */ - 1256, /* "id-it-crlStatusList" */ - 1257, /* "id-it-crls" */ 303, /* "id-it-currentCRL" */ 300, /* "id-it-encKeyPairTypes" */ 310, /* "id-it-implicitConfirm" */ @@ -3207,7 +2676,6 @@ static const unsigned int sn_objs[NUM_SN] = { 312, /* "id-it-origPKIMessage" */ 301, /* "id-it-preferredSymmAlg" */ 309, /* "id-it-revPassphrase" */ - 1254, /* "id-it-rootCaCert" */ 1224, /* "id-it-rootCaKeyUpdate" */ 299, /* "id-it-signKeyPairTypes" */ 305, /* "id-it-subscriptionRequest" */ @@ -3221,9 +2689,6 @@ static const unsigned int sn_objs[NUM_SN] = { 274, /* "id-mod-cmc" */ 277, /* "id-mod-cmp" */ 284, /* "id-mod-cmp2000" */ - 1251, /* "id-mod-cmp2000-02" */ - 1253, /* "id-mod-cmp2021-02" */ - 1252, /* "id-mod-cmp2021-88" */ 273, /* "id-mod-crmf" */ 283, /* "id-mod-dvcs" */ 275, /* "id-mod-kea-profile-88" */ @@ -3263,15 +2728,12 @@ static const unsigned int sn_objs[NUM_SN] = { 164, /* "id-qt-cps" */ 165, /* "id-qt-unotice" */ 313, /* "id-regCtrl" */ - 1259, /* "id-regCtrl-algId" */ - 1258, /* "id-regCtrl-altCertTemplate" */ 316, /* "id-regCtrl-authenticator" */ 319, /* "id-regCtrl-oldCertID" */ 318, /* "id-regCtrl-pkiArchiveOptions" */ 317, /* "id-regCtrl-pkiPublicationInfo" */ 320, /* "id-regCtrl-protocolEncrKey" */ 315, /* "id-regCtrl-regToken" */ - 1260, /* "id-regCtrl-rsaKeyLen" */ 314, /* "id-regInfo" */ 322, /* "id-regInfo-certReq" */ 321, /* "id-regInfo-utf8Pairs" */ @@ -3315,11 +2777,8 @@ static const unsigned int sn_objs[NUM_SN] = { 192, /* "id-smime-alg" */ 243, /* "id-smime-alg-3DESwrap" */ 246, /* "id-smime-alg-CMS3DESwrap" */ - 247, /* "id-smime-alg-CMSRC2wrap" */ 245, /* "id-smime-alg-ESDH" */ 241, /* "id-smime-alg-ESDHwith3DES" */ - 242, /* "id-smime-alg-ESDHwithRC2" */ - 244, /* "id-smime-alg-RC2wrap" */ 193, /* "id-smime-cd" */ 248, /* "id-smime-cd-ldap" */ 190, /* "id-smime-ct" */ @@ -3353,45 +2812,9 @@ static const unsigned int sn_objs[NUM_SN] = { 194, /* "id-smime-spq" */ 250, /* "id-smime-spq-ets-sqt-unotice" */ 249, /* "id-smime-spq-ets-sqt-uri" */ - 974, /* "id-tc26" */ - 991, /* "id-tc26-agreement" */ - 992, /* "id-tc26-agreement-gost-3410-2012-256" */ - 993, /* "id-tc26-agreement-gost-3410-2012-512" */ - 977, /* "id-tc26-algorithms" */ - 990, /* "id-tc26-cipher" */ - 1001, /* "id-tc26-cipher-constants" */ - 1176, /* "id-tc26-cipher-gostr3412-2015-kuznyechik" */ - 1173, /* "id-tc26-cipher-gostr3412-2015-magma" */ - 994, /* "id-tc26-constants" */ - 981, /* "id-tc26-digest" */ - 1000, /* "id-tc26-digest-constants" */ - 1002, /* "id-tc26-gost-28147-constants" */ - 1003, /* "id-tc26-gost-28147-param-Z" */ - 1147, /* "id-tc26-gost-3410-2012-256-constants" */ - 1148, /* "id-tc26-gost-3410-2012-256-paramSetA" */ - 1184, /* "id-tc26-gost-3410-2012-256-paramSetB" */ - 1185, /* "id-tc26-gost-3410-2012-256-paramSetC" */ - 1186, /* "id-tc26-gost-3410-2012-256-paramSetD" */ - 996, /* "id-tc26-gost-3410-2012-512-constants" */ - 998, /* "id-tc26-gost-3410-2012-512-paramSetA" */ - 999, /* "id-tc26-gost-3410-2012-512-paramSetB" */ - 1149, /* "id-tc26-gost-3410-2012-512-paramSetC" */ - 997, /* "id-tc26-gost-3410-2012-512-paramSetTest" */ - 988, /* "id-tc26-hmac-gost-3411-2012-256" */ - 989, /* "id-tc26-hmac-gost-3411-2012-512" */ - 987, /* "id-tc26-mac" */ - 978, /* "id-tc26-sign" */ - 995, /* "id-tc26-sign-constants" */ - 984, /* "id-tc26-signwithdigest" */ - 985, /* "id-tc26-signwithdigest-gost3410-2012-256" */ - 986, /* "id-tc26-signwithdigest-gost3410-2012-512" */ - 1179, /* "id-tc26-wrap" */ - 1182, /* "id-tc26-wrap-gostr3412-2015-kuznyechik" */ - 1180, /* "id-tc26-wrap-gostr3412-2015-magma" */ 676, /* "identified-organization" */ 1170, /* "ieee" */ 1171, /* "ieee-siswg" */ - 1305, /* "indirectIssuer" */ 461, /* "info" */ 748, /* "inhibitAnyPolicy" */ 101, /* "initials" */ @@ -3404,45 +2827,21 @@ static const unsigned int sn_objs[NUM_SN] = { 1022, /* "ipsecIKE" */ 295, /* "ipsecTunnel" */ 296, /* "ipsecUser" */ - 1308, /* "issuedOnBehalfOf" */ 86, /* "issuerAltName" */ - 1008, /* "issuerSignTool" */ 770, /* "issuingDistributionPoint" */ - 1264, /* "itu-t-identified-organization" */ 492, /* "janetMailbox" */ 957, /* "jurisdictionC" */ 955, /* "jurisdictionL" */ 956, /* "jurisdictionST" */ 150, /* "keyBag" */ 83, /* "keyUsage" */ - 1015, /* "kuznyechik-cbc" */ - 1016, /* "kuznyechik-cfb" */ - 1013, /* "kuznyechik-ctr" */ - 1177, /* "kuznyechik-ctr-acpkm" */ - 1178, /* "kuznyechik-ctr-acpkm-omac" */ - 1012, /* "kuznyechik-ecb" */ - 1183, /* "kuznyechik-kexp15" */ - 1017, /* "kuznyechik-mac" */ - 1014, /* "kuznyechik-ofb" */ 477, /* "lastModifiedBy" */ 476, /* "lastModifiedTime" */ 157, /* "localKeyID" */ 480, /* "mXRecord" */ - 1190, /* "magma-cbc" */ - 1191, /* "magma-cfb" */ - 1188, /* "magma-ctr" */ - 1174, /* "magma-ctr-acpkm" */ - 1175, /* "magma-ctr-acpkm-omac" */ - 1187, /* "magma-ecb" */ - 1181, /* "magma-kexp15" */ - 1192, /* "magma-mac" */ - 1189, /* "magma-ofb" */ 460, /* "mail" */ 493, /* "mailPreferenceOption" */ 467, /* "manager" */ - 982, /* "md_gost12_256" */ - 983, /* "md_gost12_512" */ - 809, /* "md_gost94" */ 875, /* "member" */ 182, /* "member-body" */ 51, /* "messageDigest" */ @@ -3457,10 +2856,6 @@ static const unsigned int sn_objs[NUM_SN] = { 1215, /* "modp_4096" */ 1216, /* "modp_6144" */ 1217, /* "modp_8192" */ - 1294, /* "ms-app-policies" */ - 1293, /* "ms-cert-templ" */ - 1291, /* "ms-ntds-obj-sid" */ - 1292, /* "ms-ntds-sec-ext" */ 136, /* "msCTLSign" */ 135, /* "msCodeCom" */ 134, /* "msCodeInd" */ @@ -3473,7 +2868,6 @@ static const unsigned int sn_objs[NUM_SN] = { 481, /* "nSRecord" */ 173, /* "name" */ 666, /* "nameConstraints" */ - 1306, /* "noAssertion" */ 369, /* "noCheck" */ 403, /* "noRevAvail" */ 72, /* "nsBaseUrl" */ @@ -3489,8 +2883,6 @@ static const unsigned int sn_objs[NUM_SN] = { 139, /* "nsSGC" */ 77, /* "nsSslServerName" */ 681, /* "onBasis" */ - 1283, /* "oracle-jdk-trustedkeyusage" */ - 1282, /* "oracle-organization" */ 1089, /* "organizationIdentifier" */ 491, /* "organizationalStatus" */ 1141, /* "oscca" */ @@ -3498,7 +2890,6 @@ static const unsigned int sn_objs[NUM_SN] = { 876, /* "owner" */ 489, /* "pagerTelephoneNumber" */ 374, /* "path" */ - 112, /* "pbeWithMD5AndCast5CBC" */ 499, /* "personalSignature" */ 487, /* "personalTitle" */ 464, /* "photo" */ @@ -3536,7 +2927,6 @@ static const unsigned int sn_objs[NUM_SN] = { 683, /* "ppBasis" */ 872, /* "preferredDeliveryMethod" */ 873, /* "presentationAddress" */ - 816, /* "prf-gostr3411-94" */ 406, /* "prime-field" */ 409, /* "prime192v1" */ 410, /* "prime192v2" */ @@ -3547,7 +2937,6 @@ static const unsigned int sn_objs[NUM_SN] = { 415, /* "prime256v1" */ 385, /* "private" */ 84, /* "privateKeyUsagePeriod" */ - 1315, /* "protRestrict" */ 886, /* "protocolInformation" */ 663, /* "proxyCertInfo" */ 510, /* "pseudonym" */ @@ -3558,7 +2947,6 @@ static const unsigned int sn_objs[NUM_SN] = { 870, /* "registeredAddress" */ 400, /* "role" */ 877, /* "roleOccupant" */ - 1296, /* "roleSpecCertIdentifier" */ 448, /* "room" */ 463, /* "roomNumber" */ 1243, /* "rpkiManifest" */ @@ -3567,7 +2955,6 @@ static const unsigned int sn_objs[NUM_SN] = { 644, /* "rsaOAEPEncryptionSET" */ 377, /* "rsaSignature" */ 1, /* "rsadsi" */ - 1302, /* "sOAIdentifier" */ 482, /* "sOARecord" */ 155, /* "safeContentsBag" */ 291, /* "sbgp-autonomousSysNum" */ @@ -3752,21 +3139,17 @@ static const unsigned int sn_objs[NUM_SN] = { 604, /* "setext-pinAny" */ 603, /* "setext-pinSecure" */ 605, /* "setext-track2" */ - 1279, /* "signedAssertion" */ 1244, /* "signedObject" */ 52, /* "signingTime" */ 454, /* "simpleSecurityObject" */ 496, /* "singleLevelQuality" */ - 1309, /* "singleUse" */ 1142, /* "sm-scheme" */ 387, /* "snmpv2" */ 660, /* "street" */ 85, /* "subjectAltName" */ - 1316, /* "subjectAltPublicKeyInfo" */ 769, /* "subjectDirectoryAttributes" */ 398, /* "subjectInfoAccess" */ 82, /* "subjectKeyIdentifier" */ - 1007, /* "subjectSignTool" */ 498, /* "subtreeMaximumQuality" */ 497, /* "subtreeMinimumQuality" */ 890, /* "supportedAlgorithms" */ @@ -3777,7 +3160,6 @@ static const unsigned int sn_objs[NUM_SN] = { 865, /* "telexNumber" */ 459, /* "textEncodedORAddress" */ 293, /* "textNotice" */ - 1299, /* "timeSpecification" */ 133, /* "timeStamping" */ 106, /* "title" */ 1020, /* "tlsfeature" */ @@ -3801,7 +3183,6 @@ static const unsigned int sn_objs[NUM_SN] = { 49, /* "unstructuredName" */ 880, /* "userCertificate" */ 465, /* "userClass" */ - 1301, /* "userNotice" */ 879, /* "userPassword" */ 373, /* "valid" */ 678, /* "wap" */ @@ -3817,16 +3198,14 @@ static const unsigned int sn_objs[NUM_SN] = { 740, /* "wap-wsg-idm-ecid-wtls7" */ 741, /* "wap-wsg-idm-ecid-wtls8" */ 742, /* "wap-wsg-idm-ecid-wtls9" */ - 804, /* "whirlpool" */ 868, /* "x121Address" */ 503, /* "x500UniqueIdentifier" */ 158, /* "x509Certificate" */ 160, /* "x509Crl" */ 1093, /* "x509ExtAdmission" */ - 1289, /* "zstd" */ }; -#define NUM_LN 1312 +#define NUM_LN 1008 static const unsigned int ln_objs[NUM_LN] = { 363, /* "AD Time Stamping" */ 405, /* "ANSI X9.62" */ @@ -3838,7 +3217,6 @@ static const unsigned int ln_objs[NUM_LN] = { 365, /* "Basic OCSP Response" */ 285, /* "Biometric Info" */ 1221, /* "Brand Indicator for Message Identification" */ - 1288, /* "Brotli compression" */ 179, /* "CA Issuers" */ 785, /* "CA Repository" */ 1219, /* "CMC Archive Server" */ @@ -3849,23 +3227,11 @@ static const unsigned int ln_objs[NUM_LN] = { 951, /* "CT Precertificate SCTs" */ 953, /* "CT Precertificate Signer" */ 1222, /* "Certificate Management Key Generation Authority" */ - 1227, /* "Class of Signing Tool" */ - 1233, /* "Class of Signing Tool KA1" */ - 1231, /* "Class of Signing Tool KB1" */ - 1232, /* "Class of Signing Tool KB2" */ - 1228, /* "Class of Signing Tool KC1" */ - 1229, /* "Class of Signing Tool KC2" */ - 1230, /* "Class of Signing Tool KC3" */ 131, /* "Code Signing" */ 1024, /* "Ctrl/Provision WAP Termination" */ 1023, /* "Ctrl/provision WAP Access" */ 1159, /* "DSTU 4145-2002 big endian" */ 1158, /* "DSTU 4145-2002 little endian" */ - 1152, /* "DSTU Gost 28147-2009" */ - 1154, /* "DSTU Gost 28147-2009 CFB mode" */ - 1153, /* "DSTU Gost 28147-2009 OFB mode" */ - 1155, /* "DSTU Gost 28147-2009 key wrap" */ - 1157, /* "DSTU Gost 34311-95" */ 1160, /* "DSTU curve 0" */ 1161, /* "DSTU curve 1" */ 1162, /* "DSTU curve 2" */ @@ -3886,48 +3252,12 @@ static const unsigned int ln_objs[NUM_LN] = { 384, /* "Experimental" */ 372, /* "Extended OCSP Status" */ 172, /* "Extension Request" */ - 813, /* "GOST 28147-89" */ - 849, /* "GOST 28147-89 Cryptocom ParamSet" */ - 815, /* "GOST 28147-89 MAC" */ - 1003, /* "GOST 28147-89 TC26 parameter set" */ - 851, /* "GOST 34.10-2001 Cryptocom" */ - 850, /* "GOST 34.10-94 Cryptocom" */ - 811, /* "GOST R 34.10-2001" */ - 817, /* "GOST R 34.10-2001 DH" */ - 1148, /* "GOST R 34.10-2012 (256 bit) ParamSet A" */ - 1184, /* "GOST R 34.10-2012 (256 bit) ParamSet B" */ - 1185, /* "GOST R 34.10-2012 (256 bit) ParamSet C" */ - 1186, /* "GOST R 34.10-2012 (256 bit) ParamSet D" */ - 998, /* "GOST R 34.10-2012 (512 bit) ParamSet A" */ - 999, /* "GOST R 34.10-2012 (512 bit) ParamSet B" */ - 1149, /* "GOST R 34.10-2012 (512 bit) ParamSet C" */ - 997, /* "GOST R 34.10-2012 (512 bit) testing parameter set" */ - 979, /* "GOST R 34.10-2012 with 256 bit modulus" */ - 980, /* "GOST R 34.10-2012 with 512 bit modulus" */ - 985, /* "GOST R 34.10-2012 with GOST R 34.11-2012 (256 bit)" */ - 986, /* "GOST R 34.10-2012 with GOST R 34.11-2012 (512 bit)" */ - 812, /* "GOST R 34.10-94" */ - 818, /* "GOST R 34.10-94 DH" */ - 982, /* "GOST R 34.11-2012 with 256 bit hash" */ - 983, /* "GOST R 34.11-2012 with 512 bit hash" */ - 809, /* "GOST R 34.11-94" */ - 816, /* "GOST R 34.11-94 PRF" */ - 807, /* "GOST R 34.11-94 with GOST R 34.10-2001" */ - 853, /* "GOST R 34.11-94 with GOST R 34.10-2001 Cryptocom" */ - 808, /* "GOST R 34.11-94 with GOST R 34.10-94" */ - 852, /* "GOST R 34.11-94 with GOST R 34.10-94 Cryptocom" */ - 854, /* "GOST R 3410-2001 Parameter Set Cryptocom" */ - 1156, /* "HMAC DSTU Gost 34311-95" */ - 988, /* "HMAC GOST 34.11-2012 256 bit" */ - 989, /* "HMAC GOST 34.11-2012 512 bit" */ - 810, /* "HMAC GOST 34.11-94" */ 432, /* "Hold Instruction Call Issuer" */ 430, /* "Hold Instruction Code" */ 431, /* "Hold Instruction None" */ 433, /* "Hold Instruction Reject" */ 634, /* "ICC or token signature" */ 1171, /* "IEEE Security in Storage Working Group" */ - 1004, /* "INN" */ 294, /* "IPSec End System" */ 295, /* "IPSec Tunnel" */ 296, /* "IPSec User" */ @@ -3942,20 +3272,16 @@ static const unsigned int ln_objs[NUM_LN] = { 504, /* "MIME MHS" */ 388, /* "Mail" */ 383, /* "Management" */ - 1294, /* "Microsoft Application Policies Extension" */ 417, /* "Microsoft CSP Name" */ 135, /* "Microsoft Commercial Code Signing" */ 138, /* "Microsoft Encrypted File System" */ 171, /* "Microsoft Extension Request" */ 134, /* "Microsoft Individual Code Signing" */ 856, /* "Microsoft Local Key set" */ - 1291, /* "Microsoft NTDS AD objectSid" */ - 1292, /* "Microsoft NTDS CA Extension" */ 137, /* "Microsoft Server Gated Crypto" */ 648, /* "Microsoft Smartcard Login" */ 136, /* "Microsoft Trust List Signing" */ 649, /* "Microsoft User Principal Name" */ - 1293, /* "Microsoft certificate template" */ 1211, /* "NAIRealm" */ 393, /* "NULL" */ 404, /* "NULL" */ @@ -3979,9 +3305,6 @@ static const unsigned int ln_objs[NUM_LN] = { 366, /* "OCSP Nonce" */ 371, /* "OCSP Service Locator" */ 180, /* "OCSP Signing" */ - 1005, /* "OGRN" */ - 1226, /* "OGRNIP" */ - 1282, /* "Oracle organization" */ 161, /* "PBES2" */ 69, /* "PBKDF2" */ 162, /* "PBMAC1" */ @@ -4004,7 +3327,6 @@ static const unsigned int ln_objs[NUM_LN] = { 188, /* "S/MIME" */ 167, /* "S/MIME Capabilities" */ 1204, /* "SM2-with-SM3" */ - 1006, /* "SNILS" */ 387, /* "SNMPv2" */ 1210, /* "SRVName" */ 1025, /* "SSH Client" */ @@ -4018,8 +3340,6 @@ static const unsigned int ln_objs[NUM_LN] = { 1027, /* "Send Router" */ 1244, /* "Signed Object" */ 1033, /* "Signing KDC Response" */ - 1008, /* "Signing Tool of Issuer" */ - 1007, /* "Signing Tool of Subject" */ 1208, /* "Smtp UTF8 Mailbox" */ 143, /* "Strong Extranet ID" */ 398, /* "Subject Information Access" */ @@ -4028,64 +3348,38 @@ static const unsigned int ln_objs[NUM_LN] = { 129, /* "TLS Web Server Authentication" */ 133, /* "Time Stamping" */ 375, /* "Trust Root" */ - 1283, /* "Trusted key usage (Oracle)" */ 1034, /* "X25519" */ 1035, /* "X448" */ 12, /* "X509" */ 402, /* "X509v3 AC Targeting" */ - 1303, /* "X509v3 Acceptable Certification Policies" */ - 1304, /* "X509v3 Acceptable Privilege Policies" */ - 1311, /* "X509v3 Allowed Attribute Assignments" */ - 1317, /* "X509v3 Alternative Signature Algorithm" */ - 1318, /* "X509v3 Alternative Signature Value" */ 746, /* "X509v3 Any Policy" */ - 1319, /* "X509v3 Associated Information" */ - 1307, /* "X509v3 Attribute Authority Issuing Distribution Point" */ - 1300, /* "X509v3 Attribute Descriptor" */ - 1312, /* "X509v3 Attribute Mappings" */ - 1295, /* "X509v3 Authority Attribute Identifier" */ 90, /* "X509v3 Authority Key Identifier" */ - 1314, /* "X509v3 Authorization Validation" */ - 1297, /* "X509v3 Basic Attribute Certificate Constraints" */ 87, /* "X509v3 Basic Constraints" */ 103, /* "X509v3 CRL Distribution Points" */ 88, /* "X509v3 CRL Number" */ 141, /* "X509v3 CRL Reason Code" */ 771, /* "X509v3 Certificate Issuer" */ 89, /* "X509v3 Certificate Policies" */ - 1298, /* "X509v3 Delegated Name Constraints" */ + 1256, /* "X509v3 Delegation Usage" */ 140, /* "X509v3 Delta CRL Indicator" */ 126, /* "X509v3 Extended Key Usage" */ 857, /* "X509v3 Freshest CRL" */ - 1310, /* "X509v3 Group Attribute Certificate" */ - 1313, /* "X509v3 Holder Name Constraints" */ - 1305, /* "X509v3 Indirect Issuer" */ 748, /* "X509v3 Inhibit Any Policy" */ - 1308, /* "X509v3 Issued On Behalf Of" */ 86, /* "X509v3 Issuer Alternative Name" */ 770, /* "X509v3 Issuing Distribution Point" */ 83, /* "X509v3 Key Usage" */ 666, /* "X509v3 Name Constraints" */ - 1306, /* "X509v3 No Assertion" */ 403, /* "X509v3 No Revocation Available" */ 401, /* "X509v3 Policy Constraints" */ 747, /* "X509v3 Policy Mappings" */ 84, /* "X509v3 Private Key Usage Period" */ - 1315, /* "X509v3 Protocol Restriction" */ - 1296, /* "X509v3 Role Specification Certificate Identifier" */ - 1309, /* "X509v3 Single Use" */ - 1302, /* "X509v3 Source of Authority Identifier" */ 85, /* "X509v3 Subject Alternative Name" */ - 1316, /* "X509v3 Subject Alternative Public Key Info" */ 769, /* "X509v3 Subject Directory Attributes" */ 82, /* "X509v3 Subject Key Identifier" */ - 1299, /* "X509v3 Time Specification" */ - 1301, /* "X509v3 User Notice" */ 920, /* "X9.42 DH" */ 184, /* "X9.57" */ 185, /* "X9.57 CM ?" */ 1209, /* "XmppAddr" */ - 1289, /* "Zstandard compression" */ 478, /* "aRecord" */ 289, /* "aaControls" */ 287, /* "ac-auditEntity" */ @@ -4136,54 +3430,18 @@ static const unsigned int ln_objs[NUM_LN] = { 1200, /* "aes-256-siv" */ 914, /* "aes-256-xts" */ 376, /* "algorithm" */ - 1066, /* "aria-128-cbc" */ - 1120, /* "aria-128-ccm" */ - 1067, /* "aria-128-cfb" */ - 1080, /* "aria-128-cfb1" */ - 1083, /* "aria-128-cfb8" */ - 1069, /* "aria-128-ctr" */ - 1065, /* "aria-128-ecb" */ - 1123, /* "aria-128-gcm" */ - 1068, /* "aria-128-ofb" */ - 1071, /* "aria-192-cbc" */ - 1121, /* "aria-192-ccm" */ - 1072, /* "aria-192-cfb" */ - 1081, /* "aria-192-cfb1" */ - 1084, /* "aria-192-cfb8" */ - 1074, /* "aria-192-ctr" */ - 1070, /* "aria-192-ecb" */ - 1124, /* "aria-192-gcm" */ - 1073, /* "aria-192-ofb" */ - 1076, /* "aria-256-cbc" */ - 1122, /* "aria-256-ccm" */ - 1077, /* "aria-256-cfb" */ - 1082, /* "aria-256-cfb1" */ - 1085, /* "aria-256-cfb8" */ - 1079, /* "aria-256-ctr" */ - 1075, /* "aria-256-ecb" */ - 1125, /* "aria-256-gcm" */ - 1078, /* "aria-256-ofb" */ 484, /* "associatedDomain" */ 485, /* "associatedName" */ 501, /* "audio" */ 1064, /* "auth-any" */ 1049, /* "auth-dss" */ 1047, /* "auth-ecdsa" */ - 1050, /* "auth-gost01" */ - 1051, /* "auth-gost12" */ 1053, /* "auth-null" */ 1048, /* "auth-psk" */ 1046, /* "auth-rsa" */ + 1254, /* "auth-sm2" */ 1052, /* "auth-srp" */ 882, /* "authorityRevocationList" */ - 91, /* "bf-cbc" */ - 93, /* "bf-cfb" */ - 92, /* "bf-ecb" */ - 94, /* "bf-ofb" */ - 1056, /* "blake2b512" */ - 1201, /* "blake2bmac" */ - 1057, /* "blake2s256" */ - 1202, /* "blake2smac" */ 921, /* "brainpoolP160r1" */ 922, /* "brainpoolP160t1" */ 923, /* "brainpoolP192r1" */ @@ -4191,15 +3449,12 @@ static const unsigned int ln_objs[NUM_LN] = { 925, /* "brainpoolP224r1" */ 926, /* "brainpoolP224t1" */ 927, /* "brainpoolP256r1" */ - 1285, /* "brainpoolP256r1tls13" */ 928, /* "brainpoolP256t1" */ 929, /* "brainpoolP320r1" */ 930, /* "brainpoolP320t1" */ 931, /* "brainpoolP384r1" */ - 1286, /* "brainpoolP384r1tls13" */ 932, /* "brainpoolP384t1" */ 933, /* "brainpoolP512r1" */ - 1287, /* "brainpoolP512r1tls13" */ 934, /* "brainpoolP512t1" */ 494, /* "buildingName" */ 860, /* "businessCategory" */ @@ -4225,43 +3480,7 @@ static const unsigned int ln_objs[NUM_LN] = { 703, /* "c2tnb431r1" */ 881, /* "cACertificate" */ 483, /* "cNAMERecord" */ - 1273, /* "cades" */ - 1274, /* "cades-attributes" */ - 751, /* "camellia-128-cbc" */ - 962, /* "camellia-128-ccm" */ - 757, /* "camellia-128-cfb" */ - 760, /* "camellia-128-cfb1" */ - 763, /* "camellia-128-cfb8" */ - 964, /* "camellia-128-cmac" */ - 963, /* "camellia-128-ctr" */ - 754, /* "camellia-128-ecb" */ - 961, /* "camellia-128-gcm" */ - 766, /* "camellia-128-ofb" */ - 752, /* "camellia-192-cbc" */ - 966, /* "camellia-192-ccm" */ - 758, /* "camellia-192-cfb" */ - 761, /* "camellia-192-cfb1" */ - 764, /* "camellia-192-cfb8" */ - 968, /* "camellia-192-cmac" */ - 967, /* "camellia-192-ctr" */ - 755, /* "camellia-192-ecb" */ - 965, /* "camellia-192-gcm" */ - 767, /* "camellia-192-ofb" */ - 753, /* "camellia-256-cbc" */ - 970, /* "camellia-256-ccm" */ - 759, /* "camellia-256-cfb" */ - 762, /* "camellia-256-cfb1" */ - 765, /* "camellia-256-cfb8" */ - 972, /* "camellia-256-cmac" */ - 971, /* "camellia-256-ctr" */ - 756, /* "camellia-256-ecb" */ - 969, /* "camellia-256-gcm" */ - 768, /* "camellia-256-ofb" */ 443, /* "caseIgnoreIA5StringSyntax" */ - 108, /* "cast5-cbc" */ - 110, /* "cast5-cfb" */ - 109, /* "cast5-ecb" */ - 111, /* "cast5-ofb" */ 152, /* "certBag" */ 677, /* "certicom-arc" */ 517, /* "certificate extensions" */ @@ -4282,8 +3501,6 @@ static const unsigned int ln_objs[NUM_LN] = { 14, /* "countryName" */ 153, /* "crlBag" */ 884, /* "crossCertificatePair" */ - 806, /* "cryptocom" */ - 805, /* "cryptopro" */ 500, /* "dITRedirect" */ 451, /* "dNSDomain" */ 495, /* "dSAQuality" */ @@ -4365,12 +3582,9 @@ static const unsigned int ln_objs[NUM_LN] = { 1113, /* "ecdsa_with_SHA3-256" */ 1114, /* "ecdsa_with_SHA3-384" */ 1115, /* "ecdsa_with_SHA3-512" */ - 1266, /* "electronic-signature-standard" */ 48, /* "emailAddress" */ 632, /* "encrypted track 2" */ 885, /* "enhancedSearchGuide" */ - 1267, /* "ess-attributes" */ - 1265, /* "etsi" */ 56, /* "extendedCertificateAttributes" */ 867, /* "facsimileTelephoneNumber" */ 462, /* "favouriteDrink" */ @@ -4387,12 +3601,6 @@ static const unsigned int ln_objs[NUM_LN] = { 601, /* "generic cryptogram" */ 99, /* "givenName" */ 1195, /* "gmac" */ - 976, /* "gost-mac-12" */ - 1009, /* "gost89-cbc" */ - 814, /* "gost89-cnt" */ - 975, /* "gost89-cnt-12" */ - 1011, /* "gost89-ctr" */ - 1010, /* "gost89-ecb" */ 1036, /* "hkdf" */ 855, /* "hmac" */ 780, /* "hmac-md5" */ @@ -4409,56 +3617,13 @@ static const unsigned int ln_objs[NUM_LN] = { 801, /* "hmacWithSHA512" */ 1193, /* "hmacWithSHA512-224" */ 1194, /* "hmacWithSHA512-256" */ - 1281, /* "hmacWithSM3" */ + 1257, /* "hmacWithSM3" */ 486, /* "homePostalAddress" */ 473, /* "homeTelephoneNumber" */ 466, /* "host" */ 889, /* "houseIdentifier" */ 442, /* "iA5StringSyntax" */ 381, /* "iana" */ - 824, /* "id-Gost28147-89-CryptoPro-A-ParamSet" */ - 825, /* "id-Gost28147-89-CryptoPro-B-ParamSet" */ - 826, /* "id-Gost28147-89-CryptoPro-C-ParamSet" */ - 827, /* "id-Gost28147-89-CryptoPro-D-ParamSet" */ - 819, /* "id-Gost28147-89-CryptoPro-KeyMeshing" */ - 829, /* "id-Gost28147-89-CryptoPro-Oscar-1-0-ParamSet" */ - 828, /* "id-Gost28147-89-CryptoPro-Oscar-1-1-ParamSet" */ - 830, /* "id-Gost28147-89-CryptoPro-RIC-1-ParamSet" */ - 820, /* "id-Gost28147-89-None-KeyMeshing" */ - 823, /* "id-Gost28147-89-TestParamSet" */ - 840, /* "id-GostR3410-2001-CryptoPro-A-ParamSet" */ - 841, /* "id-GostR3410-2001-CryptoPro-B-ParamSet" */ - 842, /* "id-GostR3410-2001-CryptoPro-C-ParamSet" */ - 843, /* "id-GostR3410-2001-CryptoPro-XchA-ParamSet" */ - 844, /* "id-GostR3410-2001-CryptoPro-XchB-ParamSet" */ - 839, /* "id-GostR3410-2001-TestParamSet" */ - 832, /* "id-GostR3410-94-CryptoPro-A-ParamSet" */ - 833, /* "id-GostR3410-94-CryptoPro-B-ParamSet" */ - 834, /* "id-GostR3410-94-CryptoPro-C-ParamSet" */ - 835, /* "id-GostR3410-94-CryptoPro-D-ParamSet" */ - 836, /* "id-GostR3410-94-CryptoPro-XchA-ParamSet" */ - 837, /* "id-GostR3410-94-CryptoPro-XchB-ParamSet" */ - 838, /* "id-GostR3410-94-CryptoPro-XchC-ParamSet" */ - 831, /* "id-GostR3410-94-TestParamSet" */ - 845, /* "id-GostR3410-94-a" */ - 846, /* "id-GostR3410-94-aBis" */ - 847, /* "id-GostR3410-94-b" */ - 848, /* "id-GostR3410-94-bBis" */ - 822, /* "id-GostR3411-94-CryptoProParamSet" */ - 821, /* "id-GostR3411-94-TestParamSet" */ - 1272, /* "id-aa-ATSHashIndex" */ - 1277, /* "id-aa-ATSHashIndex-v2" */ - 1278, /* "id-aa-ATSHashIndex-v3" */ - 1263, /* "id-aa-CMSAlgorithmProtection" */ - 1270, /* "id-aa-ets-SignaturePolicyDocument" */ - 1280, /* "id-aa-ets-archiveTimestampV2" */ - 1271, /* "id-aa-ets-archiveTimestampV3" */ - 1261, /* "id-aa-ets-attrCertificateRefs" */ - 1262, /* "id-aa-ets-attrRevocationRefs" */ - 1269, /* "id-aa-ets-longTermValidation" */ - 1268, /* "id-aa-ets-mimeType" */ - 1276, /* "id-aa-ets-sigPolicyStore" */ - 1275, /* "id-aa-ets-signerAttrV2" */ 266, /* "id-aca" */ 355, /* "id-aca-accessIdentity" */ 354, /* "id-aca-authenticationInfo" */ @@ -4479,9 +3644,6 @@ static const unsigned int ln_objs[NUM_LN] = { 326, /* "id-alg-dh-pop" */ 325, /* "id-alg-dh-sig-hmac-sha1" */ 324, /* "id-alg-noSignature" */ - 907, /* "id-camellia128-wrap" */ - 908, /* "id-camellia192-wrap" */ - 909, /* "id-camellia256-wrap" */ 268, /* "id-cct" */ 361, /* "id-cct-PKIData" */ 362, /* "id-cct-PKIResponse" */ @@ -4510,16 +3672,13 @@ static const unsigned int ln_objs[NUM_LN] = { 327, /* "id-cmc-statusInfo" */ 331, /* "id-cmc-transactionId" */ 1238, /* "id-cp" */ - 1250, /* "id-ct-ASPA" */ 787, /* "id-ct-asciiTextWithCRLF" */ 1246, /* "id-ct-geofeedCSVwithCRLF" */ 1237, /* "id-ct-resourceTaggedAttest" */ 1234, /* "id-ct-routeOriginAuthz" */ 1236, /* "id-ct-rpkiGhostbusters" */ 1235, /* "id-ct-rpkiManifest" */ - 1320, /* "id-ct-rpkiSignedPrefixList" */ 1247, /* "id-ct-signedChecklist" */ - 1284, /* "id-ct-signedTAL" */ 1060, /* "id-ct-xml" */ 408, /* "id-ecPublicKey" */ 508, /* "id-hex-multipart-message" */ @@ -4528,11 +3687,8 @@ static const unsigned int ln_objs[NUM_LN] = { 1223, /* "id-it-caCerts" */ 302, /* "id-it-caKeyUpdateInfo" */ 298, /* "id-it-caProtEncCert" */ - 1255, /* "id-it-certProfile" */ 1225, /* "id-it-certReqTemplate" */ 311, /* "id-it-confirmWaitTime" */ - 1256, /* "id-it-crlStatusList" */ - 1257, /* "id-it-crls" */ 303, /* "id-it-currentCRL" */ 300, /* "id-it-encKeyPairTypes" */ 310, /* "id-it-implicitConfirm" */ @@ -4541,7 +3697,6 @@ static const unsigned int ln_objs[NUM_LN] = { 312, /* "id-it-origPKIMessage" */ 301, /* "id-it-preferredSymmAlg" */ 309, /* "id-it-revPassphrase" */ - 1254, /* "id-it-rootCaCert" */ 1224, /* "id-it-rootCaKeyUpdate" */ 299, /* "id-it-signKeyPairTypes" */ 305, /* "id-it-subscriptionRequest" */ @@ -4553,9 +3708,6 @@ static const unsigned int ln_objs[NUM_LN] = { 274, /* "id-mod-cmc" */ 277, /* "id-mod-cmp" */ 284, /* "id-mod-cmp2000" */ - 1251, /* "id-mod-cmp2000-02" */ - 1253, /* "id-mod-cmp2021-02" */ - 1252, /* "id-mod-cmp2021-88" */ 273, /* "id-mod-crmf" */ 283, /* "id-mod-dvcs" */ 275, /* "id-mod-kea-profile-88" */ @@ -4585,15 +3737,12 @@ static const unsigned int ln_objs[NUM_LN] = { 359, /* "id-qcs-pkixQCSyntax-v1" */ 259, /* "id-qt" */ 313, /* "id-regCtrl" */ - 1259, /* "id-regCtrl-algId" */ - 1258, /* "id-regCtrl-altCertTemplate" */ 316, /* "id-regCtrl-authenticator" */ 319, /* "id-regCtrl-oldCertID" */ 318, /* "id-regCtrl-pkiArchiveOptions" */ 317, /* "id-regCtrl-pkiPublicationInfo" */ 320, /* "id-regCtrl-protocolEncrKey" */ 315, /* "id-regCtrl-regToken" */ - 1260, /* "id-regCtrl-rsaKeyLen" */ 314, /* "id-regInfo" */ 322, /* "id-regInfo-certReq" */ 321, /* "id-regInfo-utf8Pairs" */ @@ -4631,11 +3780,8 @@ static const unsigned int ln_objs[NUM_LN] = { 192, /* "id-smime-alg" */ 243, /* "id-smime-alg-3DESwrap" */ 246, /* "id-smime-alg-CMS3DESwrap" */ - 247, /* "id-smime-alg-CMSRC2wrap" */ 245, /* "id-smime-alg-ESDH" */ 241, /* "id-smime-alg-ESDHwith3DES" */ - 242, /* "id-smime-alg-ESDHwithRC2" */ - 244, /* "id-smime-alg-RC2wrap" */ 193, /* "id-smime-cd" */ 248, /* "id-smime-cd-ldap" */ 190, /* "id-smime-ct" */ @@ -4669,32 +3815,6 @@ static const unsigned int ln_objs[NUM_LN] = { 194, /* "id-smime-spq" */ 250, /* "id-smime-spq-ets-sqt-unotice" */ 249, /* "id-smime-spq-ets-sqt-uri" */ - 974, /* "id-tc26" */ - 991, /* "id-tc26-agreement" */ - 992, /* "id-tc26-agreement-gost-3410-2012-256" */ - 993, /* "id-tc26-agreement-gost-3410-2012-512" */ - 977, /* "id-tc26-algorithms" */ - 990, /* "id-tc26-cipher" */ - 1001, /* "id-tc26-cipher-constants" */ - 1176, /* "id-tc26-cipher-gostr3412-2015-kuznyechik" */ - 1173, /* "id-tc26-cipher-gostr3412-2015-magma" */ - 994, /* "id-tc26-constants" */ - 981, /* "id-tc26-digest" */ - 1000, /* "id-tc26-digest-constants" */ - 1002, /* "id-tc26-gost-28147-constants" */ - 1147, /* "id-tc26-gost-3410-2012-256-constants" */ - 996, /* "id-tc26-gost-3410-2012-512-constants" */ - 987, /* "id-tc26-mac" */ - 978, /* "id-tc26-sign" */ - 995, /* "id-tc26-sign-constants" */ - 984, /* "id-tc26-signwithdigest" */ - 1179, /* "id-tc26-wrap" */ - 1182, /* "id-tc26-wrap-gostr3412-2015-kuznyechik" */ - 1180, /* "id-tc26-wrap-gostr3412-2015-magma" */ - 34, /* "idea-cbc" */ - 35, /* "idea-cfb" */ - 36, /* "idea-ecb" */ - 46, /* "idea-ofb" */ 676, /* "identified-organization" */ 1170, /* "ieee" */ 461, /* "info" */ @@ -4708,62 +3828,36 @@ static const unsigned int ln_objs[NUM_LN] = { 181, /* "iso" */ 623, /* "issuer capabilities" */ 645, /* "itu-t" */ - 1264, /* "itu-t-identified-organization" */ 492, /* "janetMailbox" */ 646, /* "joint-iso-itu-t" */ 957, /* "jurisdictionCountryName" */ 955, /* "jurisdictionLocalityName" */ 956, /* "jurisdictionStateOrProvinceName" */ 150, /* "keyBag" */ - 773, /* "kisa" */ 1196, /* "kmac128" */ 1197, /* "kmac256" */ - 1015, /* "kuznyechik-cbc" */ - 1016, /* "kuznyechik-cfb" */ - 1013, /* "kuznyechik-ctr" */ - 1177, /* "kuznyechik-ctr-acpkm" */ - 1178, /* "kuznyechik-ctr-acpkm-omac" */ - 1012, /* "kuznyechik-ecb" */ - 1183, /* "kuznyechik-kexp15" */ - 1017, /* "kuznyechik-mac" */ - 1014, /* "kuznyechik-ofb" */ 1063, /* "kx-any" */ 1039, /* "kx-dhe" */ 1041, /* "kx-dhe-psk" */ 1038, /* "kx-ecdhe" */ 1040, /* "kx-ecdhe-psk" */ - 1045, /* "kx-gost" */ - 1218, /* "kx-gost18" */ 1043, /* "kx-psk" */ 1037, /* "kx-rsa" */ 1042, /* "kx-rsa-psk" */ + 1252, /* "kx-sm2" */ + 1253, /* "kx-sm2dhe" */ 1044, /* "kx-srp" */ 477, /* "lastModifiedBy" */ 476, /* "lastModifiedTime" */ 157, /* "localKeyID" */ 15, /* "localityName" */ 480, /* "mXRecord" */ - 1190, /* "magma-cbc" */ - 1191, /* "magma-cfb" */ - 1188, /* "magma-ctr" */ - 1174, /* "magma-ctr-acpkm" */ - 1175, /* "magma-ctr-acpkm-omac" */ - 1187, /* "magma-ecb" */ - 1181, /* "magma-kexp15" */ - 1192, /* "magma-mac" */ - 1189, /* "magma-ofb" */ 493, /* "mailPreferenceOption" */ 467, /* "manager" */ - 3, /* "md2" */ - 7, /* "md2WithRSAEncryption" */ - 257, /* "md4" */ - 396, /* "md4WithRSAEncryption" */ 4, /* "md5" */ 114, /* "md5-sha1" */ 104, /* "md5WithRSA" */ 8, /* "md5WithRSAEncryption" */ - 95, /* "mdc2" */ - 96, /* "mdc2WithRSA" */ 875, /* "member" */ 602, /* "merchant initiated auth" */ 514, /* "message extensions" */ @@ -4794,19 +3888,12 @@ static const unsigned int ln_objs[NUM_LN] = { 782, /* "password based MAC" */ 374, /* "path" */ 621, /* "payment gateway capabilities" */ - 9, /* "pbeWithMD2AndDES-CBC" */ - 168, /* "pbeWithMD2AndRC2-CBC" */ - 112, /* "pbeWithMD5AndCast5CBC" */ 10, /* "pbeWithMD5AndDES-CBC" */ - 169, /* "pbeWithMD5AndRC2-CBC" */ - 148, /* "pbeWithSHA1And128BitRC2-CBC" */ 144, /* "pbeWithSHA1And128BitRC4" */ 147, /* "pbeWithSHA1And2-KeyTripleDES-CBC" */ 146, /* "pbeWithSHA1And3-KeyTripleDES-CBC" */ - 149, /* "pbeWithSHA1And40BitRC2-CBC" */ 145, /* "pbeWithSHA1And40BitRC4" */ 170, /* "pbeWithSHA1AndDES-CBC" */ - 68, /* "pbeWithSHA1AndRC2-CBC" */ 499, /* "personalSignature" */ 487, /* "personalTitle" */ 464, /* "photo" */ @@ -4854,12 +3941,6 @@ static const unsigned int ln_objs[NUM_LN] = { 286, /* "qcStatements" */ 457, /* "qualityLabelledData" */ 450, /* "rFC822localPart" */ - 98, /* "rc2-40-cbc" */ - 166, /* "rc2-64-cbc" */ - 37, /* "rc2-cbc" */ - 39, /* "rc2-cfb" */ - 38, /* "rc2-ecb" */ - 40, /* "rc2-ofb" */ 5, /* "rc4" */ 97, /* "rc4-40" */ 915, /* "rc4-hmac-md5" */ @@ -4869,8 +3950,6 @@ static const unsigned int ln_objs[NUM_LN] = { 123, /* "rc5-ofb" */ 870, /* "registeredAddress" */ 460, /* "rfc822Mailbox" */ - 117, /* "ripemd160" */ - 119, /* "ripemd160WithRSA" */ 400, /* "role" */ 877, /* "roleOccupant" */ 448, /* "room" */ @@ -4926,10 +4005,6 @@ static const unsigned int ln_objs[NUM_LN] = { 734, /* "sect571r1" */ 635, /* "secure device signature" */ 878, /* "seeAlso" */ - 777, /* "seed-cbc" */ - 779, /* "seed-cfb" */ - 776, /* "seed-ecb" */ - 778, /* "seed-ofb" */ 105, /* "serialNumber" */ 625, /* "set-addPolicy" */ 515, /* "set-attr" */ @@ -5071,7 +4146,6 @@ static const unsigned int ln_objs[NUM_LN] = { 42, /* "shaWithRSAEncryption" */ 1100, /* "shake128" */ 1101, /* "shake256" */ - 1279, /* "signedAssertion" */ 52, /* "signingTime" */ 454, /* "simpleSecurityObject" */ 496, /* "singleLevelQuality" */ @@ -5081,15 +4155,14 @@ static const unsigned int ln_objs[NUM_LN] = { 1143, /* "sm3" */ 1144, /* "sm3WithRSAEncryption" */ 1134, /* "sm4-cbc" */ - 1249, /* "sm4-ccm" */ + 1251, /* "sm4-ccm" */ 1137, /* "sm4-cfb" */ 1136, /* "sm4-cfb1" */ 1138, /* "sm4-cfb8" */ 1139, /* "sm4-ctr" */ 1133, /* "sm4-ecb" */ - 1248, /* "sm4-gcm" */ + 1250, /* "sm4-gcm" */ 1135, /* "sm4-ofb" */ - 1290, /* "sm4-xts" */ 1203, /* "sshkdf" */ 1205, /* "sskdf" */ 16, /* "stateOrProvinceName" */ @@ -5132,7 +4205,6 @@ static const unsigned int ln_objs[NUM_LN] = { 740, /* "wap-wsg-idm-ecid-wtls7" */ 741, /* "wap-wsg-idm-ecid-wtls8" */ 742, /* "wap-wsg-idm-ecid-wtls9" */ - 804, /* "whirlpool" */ 868, /* "x121Address" */ 503, /* "x500UniqueIdentifier" */ 158, /* "x509Certificate" */ @@ -5140,9 +4212,12 @@ static const unsigned int ln_objs[NUM_LN] = { 1207, /* "x942kdf" */ 1206, /* "x963kdf" */ 125, /* "zlib compression" */ + 1248, /* "zuc" */ + 1249, /* "zuc-128-eea3" */ + 1255, /* "zuc-128-eia3" */ }; -#define NUM_OBJ 1178 +#define NUM_OBJ 924 static const unsigned int obj_objs[NUM_OBJ] = { 0, /* OBJ_undef 0 */ 181, /* OBJ_iso 1 */ @@ -5150,14 +4225,12 @@ static const unsigned int obj_objs[NUM_OBJ] = { 404, /* OBJ_ccitt OBJ_itu_t */ 645, /* OBJ_itu_t 0 */ 646, /* OBJ_joint_iso_itu_t 2 */ - 1264, /* OBJ_itu_t_identified_organization 0 4 */ 434, /* OBJ_data 0 9 */ 182, /* OBJ_member_body 1 2 */ 379, /* OBJ_org 1 3 */ 676, /* OBJ_identified_organization 1 3 */ 11, /* OBJ_X500 2 5 */ 647, /* OBJ_international_organizations 2 23 */ - 1265, /* OBJ_etsi 0 4 0 */ 380, /* OBJ_dod 1 3 6 */ 1170, /* OBJ_ieee 1 3 111 */ 12, /* OBJ_X509 2 5 4 */ @@ -5255,35 +4328,10 @@ static const unsigned int obj_objs[NUM_OBJ] = { 90, /* OBJ_authority_key_identifier 2 5 29 35 */ 401, /* OBJ_policy_constraints 2 5 29 36 */ 126, /* OBJ_ext_key_usage 2 5 29 37 */ - 1295, /* OBJ_authority_attribute_identifier 2 5 29 38 */ - 1296, /* OBJ_role_spec_cert_identifier 2 5 29 39 */ - 1297, /* OBJ_basic_att_constraints 2 5 29 41 */ - 1298, /* OBJ_delegated_name_constraints 2 5 29 42 */ - 1299, /* OBJ_time_specification 2 5 29 43 */ 857, /* OBJ_freshest_crl 2 5 29 46 */ - 1300, /* OBJ_attribute_descriptor 2 5 29 48 */ - 1301, /* OBJ_user_notice 2 5 29 49 */ - 1302, /* OBJ_soa_identifier 2 5 29 50 */ - 1303, /* OBJ_acceptable_cert_policies 2 5 29 52 */ 748, /* OBJ_inhibit_any_policy 2 5 29 54 */ 402, /* OBJ_target_information 2 5 29 55 */ 403, /* OBJ_no_rev_avail 2 5 29 56 */ - 1304, /* OBJ_acceptable_privilege_policies 2 5 29 57 */ - 1305, /* OBJ_indirect_issuer 2 5 29 61 */ - 1306, /* OBJ_no_assertion 2 5 29 62 */ - 1307, /* OBJ_id_aa_issuing_distribution_point 2 5 29 63 */ - 1308, /* OBJ_issued_on_behalf_of 2 5 29 64 */ - 1309, /* OBJ_single_use 2 5 29 65 */ - 1310, /* OBJ_group_ac 2 5 29 66 */ - 1311, /* OBJ_allowed_attribute_assignments 2 5 29 67 */ - 1312, /* OBJ_attribute_mappings 2 5 29 68 */ - 1313, /* OBJ_holder_name_constraints 2 5 29 69 */ - 1314, /* OBJ_authorization_validation 2 5 29 70 */ - 1315, /* OBJ_prot_restrict 2 5 29 71 */ - 1316, /* OBJ_subject_alt_public_key_info 2 5 29 72 */ - 1317, /* OBJ_alt_signature_algorithm 2 5 29 73 */ - 1318, /* OBJ_alt_signature_value 2 5 29 74 */ - 1319, /* OBJ_associated_information 2 5 29 75 */ 513, /* OBJ_set_ctype 2 23 42 0 */ 514, /* OBJ_set_msgExt 2 23 42 1 */ 515, /* OBJ_set_attr 2 23 42 3 */ @@ -5291,7 +4339,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 517, /* OBJ_set_certExt 2 23 42 7 */ 518, /* OBJ_set_brand 2 23 42 8 */ 679, /* OBJ_wap_wsg 2 23 43 1 */ - 1266, /* OBJ_electronic_signature_standard 0 4 0 1733 */ 382, /* OBJ_Directory 1 3 6 1 1 */ 383, /* OBJ_Management 1 3 6 1 2 */ 384, /* OBJ_Experimental 1 3 6 1 3 */ @@ -5302,8 +4349,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 376, /* OBJ_algorithm 1 3 14 3 2 */ 395, /* OBJ_clearance 2 5 1 5 55 */ 19, /* OBJ_rsa 2 5 8 1 1 */ - 96, /* OBJ_mdc2WithRSA 2 5 8 3 100 */ - 95, /* OBJ_mdc2 2 5 8 3 101 */ 746, /* OBJ_any_policy 2 5 29 32 0 */ 910, /* OBJ_anyExtendedKeyUsage 2 5 29 37 0 */ 519, /* OBJ_setct_PANData 2 23 42 0 0 */ @@ -5417,19 +4462,8 @@ static const unsigned int obj_objs[NUM_OBJ] = { 637, /* OBJ_set_brand_Diners 2 23 42 8 30 */ 638, /* OBJ_set_brand_AmericanExpress 2 23 42 8 34 */ 639, /* OBJ_set_brand_JCB 2 23 42 8 35 */ - 1273, /* OBJ_cades 0 4 0 19122 */ - 1267, /* OBJ_ess_attributes 0 4 0 1733 2 */ 1195, /* OBJ_gmac 1 0 9797 3 4 */ 1141, /* OBJ_oscca 1 2 156 10197 */ - 805, /* OBJ_cryptopro 1 2 643 2 2 */ - 806, /* OBJ_cryptocom 1 2 643 2 9 */ - 974, /* OBJ_id_tc26 1 2 643 7 1 */ - 1005, /* OBJ_OGRN 1 2 643 100 1 */ - 1006, /* OBJ_SNILS 1 2 643 100 3 */ - 1226, /* OBJ_OGRNIP 1 2 643 100 5 */ - 1007, /* OBJ_subjectSignTool 1 2 643 100 111 */ - 1008, /* OBJ_issuerSignTool 1 2 643 100 112 */ - 1227, /* OBJ_classSignTool 1 2 643 100 113 */ 184, /* OBJ_X9_57 1 2 840 10040 */ 405, /* OBJ_ansi_X9_62 1 2 840 10045 */ 389, /* OBJ_Enterprises 1 3 6 1 4 1 */ @@ -5448,7 +4482,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 64, /* OBJ_sha1 1 3 14 3 2 26 */ 70, /* OBJ_dsaWithSHA1_2 1 3 14 3 2 27 */ 115, /* OBJ_sha1WithRSA 1 3 14 3 2 29 */ - 117, /* OBJ_ripemd160 1 3 36 3 2 1 */ 1093, /* OBJ_x509ExtAdmission 1 3 36 8 3 3 */ 143, /* OBJ_sxnet 1 3 101 1 4 1 */ 1171, /* OBJ_ieee_siswg 1 3 111 2 1619 */ @@ -5502,41 +4535,13 @@ static const unsigned int obj_objs[NUM_OBJ] = { 743, /* OBJ_wap_wsg_idm_ecid_wtls10 2 23 43 1 4 10 */ 744, /* OBJ_wap_wsg_idm_ecid_wtls11 2 23 43 1 4 11 */ 745, /* OBJ_wap_wsg_idm_ecid_wtls12 2 23 43 1 4 12 */ - 1274, /* OBJ_cades_attributes 0 4 0 19122 1 */ - 1268, /* OBJ_id_aa_ets_mimeType 0 4 0 1733 2 1 */ - 1269, /* OBJ_id_aa_ets_longTermValidation 0 4 0 1733 2 2 */ - 1270, /* OBJ_id_aa_ets_SignaturePolicyDocument 0 4 0 1733 2 3 */ - 1271, /* OBJ_id_aa_ets_archiveTimestampV3 0 4 0 1733 2 4 */ - 1272, /* OBJ_id_aa_ATSHashIndex 0 4 0 1733 2 5 */ - 804, /* OBJ_whirlpool 1 0 10118 3 0 55 */ 1142, /* OBJ_sm_scheme 1 2 156 10197 1 */ - 773, /* OBJ_kisa 1 2 410 200004 */ - 807, /* OBJ_id_GostR3411_94_with_GostR3410_2001 1 2 643 2 2 3 */ - 808, /* OBJ_id_GostR3411_94_with_GostR3410_94 1 2 643 2 2 4 */ - 809, /* OBJ_id_GostR3411_94 1 2 643 2 2 9 */ - 810, /* OBJ_id_HMACGostR3411_94 1 2 643 2 2 10 */ - 811, /* OBJ_id_GostR3410_2001 1 2 643 2 2 19 */ - 812, /* OBJ_id_GostR3410_94 1 2 643 2 2 20 */ - 813, /* OBJ_id_Gost28147_89 1 2 643 2 2 21 */ - 815, /* OBJ_id_Gost28147_89_MAC 1 2 643 2 2 22 */ - 816, /* OBJ_id_GostR3411_94_prf 1 2 643 2 2 23 */ - 817, /* OBJ_id_GostR3410_2001DH 1 2 643 2 2 98 */ - 818, /* OBJ_id_GostR3410_94DH 1 2 643 2 2 99 */ - 977, /* OBJ_id_tc26_algorithms 1 2 643 7 1 1 */ - 994, /* OBJ_id_tc26_constants 1 2 643 7 1 2 */ - 1228, /* OBJ_classSignToolKC1 1 2 643 100 113 1 */ - 1229, /* OBJ_classSignToolKC2 1 2 643 100 113 2 */ - 1230, /* OBJ_classSignToolKC3 1 2 643 100 113 3 */ - 1231, /* OBJ_classSignToolKB1 1 2 643 100 113 4 */ - 1232, /* OBJ_classSignToolKB2 1 2 643 100 113 5 */ - 1233, /* OBJ_classSignToolKA1 1 2 643 100 113 6 */ 1, /* OBJ_rsadsi 1 2 840 113549 */ 185, /* OBJ_X9cm 1 2 840 10040 4 */ 1031, /* OBJ_id_pkinit 1 3 6 1 5 2 3 */ 127, /* OBJ_id_pkix 1 3 6 1 5 5 7 */ 505, /* OBJ_mime_mhs_headings 1 3 6 1 7 1 1 */ 506, /* OBJ_mime_mhs_bodies 1 3 6 1 7 1 2 */ - 119, /* OBJ_ripemd160WithRSA 1 3 36 3 3 1 2 */ 937, /* OBJ_dhSinglePass_stdDH_sha224kdf_scheme 1 3 132 1 11 0 */ 938, /* OBJ_dhSinglePass_stdDH_sha256kdf_scheme 1 3 132 1 11 1 */ 939, /* OBJ_dhSinglePass_stdDH_sha384kdf_scheme 1 3 132 1 11 2 */ @@ -5550,52 +4555,7 @@ static const unsigned int obj_objs[NUM_OBJ] = { 633, /* OBJ_setAttr_T2cleartxt 2 23 42 3 3 4 2 */ 634, /* OBJ_setAttr_TokICCsig 2 23 42 3 3 5 1 */ 635, /* OBJ_setAttr_SecDevSig 2 23 42 3 3 5 2 */ - 1275, /* OBJ_id_aa_ets_signerAttrV2 0 4 0 19122 1 1 */ - 1276, /* OBJ_id_aa_ets_sigPolicyStore 0 4 0 19122 1 3 */ - 1277, /* OBJ_id_aa_ATSHashIndex_v2 0 4 0 19122 1 4 */ - 1278, /* OBJ_id_aa_ATSHashIndex_v3 0 4 0 19122 1 5 */ - 1279, /* OBJ_signedAssertion 0 4 0 19122 1 6 */ 436, /* OBJ_ucl 0 9 2342 19200300 */ - 820, /* OBJ_id_Gost28147_89_None_KeyMeshing 1 2 643 2 2 14 0 */ - 819, /* OBJ_id_Gost28147_89_CryptoPro_KeyMeshing 1 2 643 2 2 14 1 */ - 845, /* OBJ_id_GostR3410_94_a 1 2 643 2 2 20 1 */ - 846, /* OBJ_id_GostR3410_94_aBis 1 2 643 2 2 20 2 */ - 847, /* OBJ_id_GostR3410_94_b 1 2 643 2 2 20 3 */ - 848, /* OBJ_id_GostR3410_94_bBis 1 2 643 2 2 20 4 */ - 821, /* OBJ_id_GostR3411_94_TestParamSet 1 2 643 2 2 30 0 */ - 822, /* OBJ_id_GostR3411_94_CryptoProParamSet 1 2 643 2 2 30 1 */ - 823, /* OBJ_id_Gost28147_89_TestParamSet 1 2 643 2 2 31 0 */ - 824, /* OBJ_id_Gost28147_89_CryptoPro_A_ParamSet 1 2 643 2 2 31 1 */ - 825, /* OBJ_id_Gost28147_89_CryptoPro_B_ParamSet 1 2 643 2 2 31 2 */ - 826, /* OBJ_id_Gost28147_89_CryptoPro_C_ParamSet 1 2 643 2 2 31 3 */ - 827, /* OBJ_id_Gost28147_89_CryptoPro_D_ParamSet 1 2 643 2 2 31 4 */ - 828, /* OBJ_id_Gost28147_89_CryptoPro_Oscar_1_1_ParamSet 1 2 643 2 2 31 5 */ - 829, /* OBJ_id_Gost28147_89_CryptoPro_Oscar_1_0_ParamSet 1 2 643 2 2 31 6 */ - 830, /* OBJ_id_Gost28147_89_CryptoPro_RIC_1_ParamSet 1 2 643 2 2 31 7 */ - 831, /* OBJ_id_GostR3410_94_TestParamSet 1 2 643 2 2 32 0 */ - 832, /* OBJ_id_GostR3410_94_CryptoPro_A_ParamSet 1 2 643 2 2 32 2 */ - 833, /* OBJ_id_GostR3410_94_CryptoPro_B_ParamSet 1 2 643 2 2 32 3 */ - 834, /* OBJ_id_GostR3410_94_CryptoPro_C_ParamSet 1 2 643 2 2 32 4 */ - 835, /* OBJ_id_GostR3410_94_CryptoPro_D_ParamSet 1 2 643 2 2 32 5 */ - 836, /* OBJ_id_GostR3410_94_CryptoPro_XchA_ParamSet 1 2 643 2 2 33 1 */ - 837, /* OBJ_id_GostR3410_94_CryptoPro_XchB_ParamSet 1 2 643 2 2 33 2 */ - 838, /* OBJ_id_GostR3410_94_CryptoPro_XchC_ParamSet 1 2 643 2 2 33 3 */ - 839, /* OBJ_id_GostR3410_2001_TestParamSet 1 2 643 2 2 35 0 */ - 840, /* OBJ_id_GostR3410_2001_CryptoPro_A_ParamSet 1 2 643 2 2 35 1 */ - 841, /* OBJ_id_GostR3410_2001_CryptoPro_B_ParamSet 1 2 643 2 2 35 2 */ - 842, /* OBJ_id_GostR3410_2001_CryptoPro_C_ParamSet 1 2 643 2 2 35 3 */ - 843, /* OBJ_id_GostR3410_2001_CryptoPro_XchA_ParamSet 1 2 643 2 2 36 0 */ - 844, /* OBJ_id_GostR3410_2001_CryptoPro_XchB_ParamSet 1 2 643 2 2 36 1 */ - 978, /* OBJ_id_tc26_sign 1 2 643 7 1 1 1 */ - 981, /* OBJ_id_tc26_digest 1 2 643 7 1 1 2 */ - 984, /* OBJ_id_tc26_signwithdigest 1 2 643 7 1 1 3 */ - 987, /* OBJ_id_tc26_mac 1 2 643 7 1 1 4 */ - 990, /* OBJ_id_tc26_cipher 1 2 643 7 1 1 5 */ - 991, /* OBJ_id_tc26_agreement 1 2 643 7 1 1 6 */ - 1179, /* OBJ_id_tc26_wrap 1 2 643 7 1 1 7 */ - 995, /* OBJ_id_tc26_sign_constants 1 2 643 7 1 2 1 */ - 1000, /* OBJ_id_tc26_digest_constants 1 2 643 7 1 2 2 */ - 1001, /* OBJ_id_tc26_cipher_constants 1 2 643 7 1 2 5 */ 1151, /* OBJ_ua_pki 1 2 804 2 1 1 1 */ 2, /* OBJ_pkcs 1 2 840 113549 1 */ 431, /* OBJ_hold_instruction_none 1 2 840 10040 2 1 */ @@ -5631,28 +4591,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 507, /* OBJ_id_hex_partial_message 1 3 6 1 7 1 1 1 */ 508, /* OBJ_id_hex_multipart_message 1 3 6 1 7 1 1 2 */ 57, /* OBJ_netscape 2 16 840 1 113730 */ - 1282, /* OBJ_oracle 2 16 840 1 113894 */ - 754, /* OBJ_camellia_128_ecb 0 3 4401 5 3 1 9 1 */ - 766, /* OBJ_camellia_128_ofb128 0 3 4401 5 3 1 9 3 */ - 757, /* OBJ_camellia_128_cfb128 0 3 4401 5 3 1 9 4 */ - 961, /* OBJ_camellia_128_gcm 0 3 4401 5 3 1 9 6 */ - 962, /* OBJ_camellia_128_ccm 0 3 4401 5 3 1 9 7 */ - 963, /* OBJ_camellia_128_ctr 0 3 4401 5 3 1 9 9 */ - 964, /* OBJ_camellia_128_cmac 0 3 4401 5 3 1 9 10 */ - 755, /* OBJ_camellia_192_ecb 0 3 4401 5 3 1 9 21 */ - 767, /* OBJ_camellia_192_ofb128 0 3 4401 5 3 1 9 23 */ - 758, /* OBJ_camellia_192_cfb128 0 3 4401 5 3 1 9 24 */ - 965, /* OBJ_camellia_192_gcm 0 3 4401 5 3 1 9 26 */ - 966, /* OBJ_camellia_192_ccm 0 3 4401 5 3 1 9 27 */ - 967, /* OBJ_camellia_192_ctr 0 3 4401 5 3 1 9 29 */ - 968, /* OBJ_camellia_192_cmac 0 3 4401 5 3 1 9 30 */ - 756, /* OBJ_camellia_256_ecb 0 3 4401 5 3 1 9 41 */ - 768, /* OBJ_camellia_256_ofb128 0 3 4401 5 3 1 9 43 */ - 759, /* OBJ_camellia_256_cfb128 0 3 4401 5 3 1 9 44 */ - 969, /* OBJ_camellia_256_gcm 0 3 4401 5 3 1 9 46 */ - 970, /* OBJ_camellia_256_ccm 0 3 4401 5 3 1 9 47 */ - 971, /* OBJ_camellia_256_ctr 0 3 4401 5 3 1 9 49 */ - 972, /* OBJ_camellia_256_cmac 0 3 4401 5 3 1 9 50 */ 437, /* OBJ_pilot 0 9 2342 19200300 100 */ 1133, /* OBJ_sm4_ecb 1 2 156 10197 1 104 1 */ 1134, /* OBJ_sm4_cbc 1 2 156 10197 1 104 2 */ @@ -5661,48 +4599,20 @@ static const unsigned int obj_objs[NUM_OBJ] = { 1136, /* OBJ_sm4_cfb1 1 2 156 10197 1 104 5 */ 1138, /* OBJ_sm4_cfb8 1 2 156 10197 1 104 6 */ 1139, /* OBJ_sm4_ctr 1 2 156 10197 1 104 7 */ - 1248, /* OBJ_sm4_gcm 1 2 156 10197 1 104 8 */ - 1249, /* OBJ_sm4_ccm 1 2 156 10197 1 104 9 */ - 1290, /* OBJ_sm4_xts 1 2 156 10197 1 104 10 */ + 1250, /* OBJ_sm4_gcm 1 2 156 10197 1 104 8 */ + 1251, /* OBJ_sm4_ccm 1 2 156 10197 1 104 9 */ + 1248, /* OBJ_zuc 1 2 156 10197 1 201 */ 1172, /* OBJ_sm2 1 2 156 10197 1 301 */ 1143, /* OBJ_sm3 1 2 156 10197 1 401 */ 1204, /* OBJ_SM2_with_SM3 1 2 156 10197 1 501 */ 1144, /* OBJ_sm3WithRSAEncryption 1 2 156 10197 1 504 */ - 776, /* OBJ_seed_ecb 1 2 410 200004 1 3 */ - 777, /* OBJ_seed_cbc 1 2 410 200004 1 4 */ - 779, /* OBJ_seed_cfb128 1 2 410 200004 1 5 */ - 778, /* OBJ_seed_ofb128 1 2 410 200004 1 6 */ - 852, /* OBJ_id_GostR3411_94_with_GostR3410_94_cc 1 2 643 2 9 1 3 3 */ - 853, /* OBJ_id_GostR3411_94_with_GostR3410_2001_cc 1 2 643 2 9 1 3 4 */ - 850, /* OBJ_id_GostR3410_94_cc 1 2 643 2 9 1 5 3 */ - 851, /* OBJ_id_GostR3410_2001_cc 1 2 643 2 9 1 5 4 */ - 849, /* OBJ_id_Gost28147_89_cc 1 2 643 2 9 1 6 1 */ - 854, /* OBJ_id_GostR3410_2001_ParamSet_cc 1 2 643 2 9 1 8 1 */ - 1004, /* OBJ_INN 1 2 643 3 131 1 1 */ - 979, /* OBJ_id_GostR3410_2012_256 1 2 643 7 1 1 1 1 */ - 980, /* OBJ_id_GostR3410_2012_512 1 2 643 7 1 1 1 2 */ - 982, /* OBJ_id_GostR3411_2012_256 1 2 643 7 1 1 2 2 */ - 983, /* OBJ_id_GostR3411_2012_512 1 2 643 7 1 1 2 3 */ - 985, /* OBJ_id_tc26_signwithdigest_gost3410_2012_256 1 2 643 7 1 1 3 2 */ - 986, /* OBJ_id_tc26_signwithdigest_gost3410_2012_512 1 2 643 7 1 1 3 3 */ - 988, /* OBJ_id_tc26_hmac_gost_3411_2012_256 1 2 643 7 1 1 4 1 */ - 989, /* OBJ_id_tc26_hmac_gost_3411_2012_512 1 2 643 7 1 1 4 2 */ - 1173, /* OBJ_id_tc26_cipher_gostr3412_2015_magma 1 2 643 7 1 1 5 1 */ - 1176, /* OBJ_id_tc26_cipher_gostr3412_2015_kuznyechik 1 2 643 7 1 1 5 2 */ - 992, /* OBJ_id_tc26_agreement_gost_3410_2012_256 1 2 643 7 1 1 6 1 */ - 993, /* OBJ_id_tc26_agreement_gost_3410_2012_512 1 2 643 7 1 1 6 2 */ - 1180, /* OBJ_id_tc26_wrap_gostr3412_2015_magma 1 2 643 7 1 1 7 1 */ - 1182, /* OBJ_id_tc26_wrap_gostr3412_2015_kuznyechik 1 2 643 7 1 1 7 2 */ - 1147, /* OBJ_id_tc26_gost_3410_2012_256_constants 1 2 643 7 1 2 1 1 */ - 996, /* OBJ_id_tc26_gost_3410_2012_512_constants 1 2 643 7 1 2 1 2 */ - 1002, /* OBJ_id_tc26_gost_28147_constants 1 2 643 7 1 2 5 1 */ + 1249, /* OBJ_zuc_128_eea3 1 2 156 10197 1 801 */ + 1255, /* OBJ_zuc_128_eia3 1 2 156 10197 1 802 */ 186, /* OBJ_pkcs1 1 2 840 113549 1 1 */ 27, /* OBJ_pkcs3 1 2 840 113549 1 3 */ 187, /* OBJ_pkcs5 1 2 840 113549 1 5 */ 20, /* OBJ_pkcs7 1 2 840 113549 1 7 */ 47, /* OBJ_pkcs9 1 2 840 113549 1 9 */ - 3, /* OBJ_md2 1 2 840 113549 2 2 */ - 257, /* OBJ_md4 1 2 840 113549 2 4 */ 4, /* OBJ_md5 1 2 840 113549 2 5 */ 797, /* OBJ_hmacWithMD5 1 2 840 113549 2 6 */ 163, /* OBJ_hmacWithSHA1 1 2 840 113549 2 7 */ @@ -5712,7 +4622,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 801, /* OBJ_hmacWithSHA512 1 2 840 113549 2 11 */ 1193, /* OBJ_hmacWithSHA512_224 1 2 840 113549 2 12 */ 1194, /* OBJ_hmacWithSHA512_256 1 2 840 113549 2 13 */ - 37, /* OBJ_rc2_cbc 1 2 840 113549 3 2 */ 5, /* OBJ_rc4 1 2 840 113549 3 4 */ 44, /* OBJ_des_ede3_cbc 1 2 840 113549 3 7 */ 120, /* OBJ_rc5_cbc 1 2 840 113549 3 8 */ @@ -5765,9 +4674,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 282, /* OBJ_id_mod_ocsp 1 3 6 1 5 5 7 0 14 */ 283, /* OBJ_id_mod_dvcs 1 3 6 1 5 5 7 0 15 */ 284, /* OBJ_id_mod_cmp2000 1 3 6 1 5 5 7 0 16 */ - 1251, /* OBJ_id_mod_cmp2000_02 1 3 6 1 5 5 7 0 50 */ - 1252, /* OBJ_id_mod_cmp2021_88 1 3 6 1 5 5 7 0 99 */ - 1253, /* OBJ_id_mod_cmp2021_02 1 3 6 1 5 5 7 0 100 */ 177, /* OBJ_info_access 1 3 6 1 5 5 7 1 1 */ 285, /* OBJ_biometricInfo 1 3 6 1 5 5 7 1 2 */ 286, /* OBJ_qcStatements 1 3 6 1 5 5 7 1 3 */ @@ -5830,10 +4736,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 1223, /* OBJ_id_it_caCerts 1 3 6 1 5 5 7 4 17 */ 1224, /* OBJ_id_it_rootCaKeyUpdate 1 3 6 1 5 5 7 4 18 */ 1225, /* OBJ_id_it_certReqTemplate 1 3 6 1 5 5 7 4 19 */ - 1254, /* OBJ_id_it_rootCaCert 1 3 6 1 5 5 7 4 20 */ - 1255, /* OBJ_id_it_certProfile 1 3 6 1 5 5 7 4 21 */ - 1256, /* OBJ_id_it_crlStatusList 1 3 6 1 5 5 7 4 22 */ - 1257, /* OBJ_id_it_crls 1 3 6 1 5 5 7 4 23 */ 313, /* OBJ_id_regCtrl 1 3 6 1 5 5 7 5 1 */ 314, /* OBJ_id_regInfo 1 3 6 1 5 5 7 5 2 */ 323, /* OBJ_id_alg_des40 1 3 6 1 5 5 7 6 1 */ @@ -5904,49 +4806,9 @@ static const unsigned int obj_objs[NUM_OBJ] = { 439, /* OBJ_pilotAttributeSyntax 0 9 2342 19200300 100 3 */ 440, /* OBJ_pilotObjectClass 0 9 2342 19200300 100 4 */ 441, /* OBJ_pilotGroups 0 9 2342 19200300 100 10 */ - 1065, /* OBJ_aria_128_ecb 1 2 410 200046 1 1 1 */ - 1066, /* OBJ_aria_128_cbc 1 2 410 200046 1 1 2 */ - 1067, /* OBJ_aria_128_cfb128 1 2 410 200046 1 1 3 */ - 1068, /* OBJ_aria_128_ofb128 1 2 410 200046 1 1 4 */ - 1069, /* OBJ_aria_128_ctr 1 2 410 200046 1 1 5 */ - 1070, /* OBJ_aria_192_ecb 1 2 410 200046 1 1 6 */ - 1071, /* OBJ_aria_192_cbc 1 2 410 200046 1 1 7 */ - 1072, /* OBJ_aria_192_cfb128 1 2 410 200046 1 1 8 */ - 1073, /* OBJ_aria_192_ofb128 1 2 410 200046 1 1 9 */ - 1074, /* OBJ_aria_192_ctr 1 2 410 200046 1 1 10 */ - 1075, /* OBJ_aria_256_ecb 1 2 410 200046 1 1 11 */ - 1076, /* OBJ_aria_256_cbc 1 2 410 200046 1 1 12 */ - 1077, /* OBJ_aria_256_cfb128 1 2 410 200046 1 1 13 */ - 1078, /* OBJ_aria_256_ofb128 1 2 410 200046 1 1 14 */ - 1079, /* OBJ_aria_256_ctr 1 2 410 200046 1 1 15 */ - 1123, /* OBJ_aria_128_gcm 1 2 410 200046 1 1 34 */ - 1124, /* OBJ_aria_192_gcm 1 2 410 200046 1 1 35 */ - 1125, /* OBJ_aria_256_gcm 1 2 410 200046 1 1 36 */ - 1120, /* OBJ_aria_128_ccm 1 2 410 200046 1 1 37 */ - 1121, /* OBJ_aria_192_ccm 1 2 410 200046 1 1 38 */ - 1122, /* OBJ_aria_256_ccm 1 2 410 200046 1 1 39 */ - 1174, /* OBJ_magma_ctr_acpkm 1 2 643 7 1 1 5 1 1 */ - 1175, /* OBJ_magma_ctr_acpkm_omac 1 2 643 7 1 1 5 1 2 */ - 1177, /* OBJ_kuznyechik_ctr_acpkm 1 2 643 7 1 1 5 2 1 */ - 1178, /* OBJ_kuznyechik_ctr_acpkm_omac 1 2 643 7 1 1 5 2 2 */ - 1181, /* OBJ_magma_kexp15 1 2 643 7 1 1 7 1 1 */ - 1183, /* OBJ_kuznyechik_kexp15 1 2 643 7 1 1 7 2 1 */ - 1148, /* OBJ_id_tc26_gost_3410_2012_256_paramSetA 1 2 643 7 1 2 1 1 1 */ - 1184, /* OBJ_id_tc26_gost_3410_2012_256_paramSetB 1 2 643 7 1 2 1 1 2 */ - 1185, /* OBJ_id_tc26_gost_3410_2012_256_paramSetC 1 2 643 7 1 2 1 1 3 */ - 1186, /* OBJ_id_tc26_gost_3410_2012_256_paramSetD 1 2 643 7 1 2 1 1 4 */ - 997, /* OBJ_id_tc26_gost_3410_2012_512_paramSetTest 1 2 643 7 1 2 1 2 0 */ - 998, /* OBJ_id_tc26_gost_3410_2012_512_paramSetA 1 2 643 7 1 2 1 2 1 */ - 999, /* OBJ_id_tc26_gost_3410_2012_512_paramSetB 1 2 643 7 1 2 1 2 2 */ - 1149, /* OBJ_id_tc26_gost_3410_2012_512_paramSetC 1 2 643 7 1 2 1 2 3 */ - 1003, /* OBJ_id_tc26_gost_28147_param_Z 1 2 643 7 1 2 5 1 1 */ - 108, /* OBJ_cast5_cbc 1 2 840 113533 7 66 10 */ - 112, /* OBJ_pbeWithMD5AndCast5_CBC 1 2 840 113533 7 66 12 */ 782, /* OBJ_id_PasswordBasedMAC 1 2 840 113533 7 66 13 */ 783, /* OBJ_id_DHBasedMac 1 2 840 113533 7 66 30 */ 6, /* OBJ_rsaEncryption 1 2 840 113549 1 1 1 */ - 7, /* OBJ_md2WithRSAEncryption 1 2 840 113549 1 1 2 */ - 396, /* OBJ_md4WithRSAEncryption 1 2 840 113549 1 1 3 */ 8, /* OBJ_md5WithRSAEncryption 1 2 840 113549 1 1 4 */ 65, /* OBJ_sha1WithRSAEncryption 1 2 840 113549 1 1 5 */ 644, /* OBJ_rsaOAEPEncryptionSET 1 2 840 113549 1 1 6 */ @@ -5961,12 +4823,8 @@ static const unsigned int obj_objs[NUM_OBJ] = { 1145, /* OBJ_sha512_224WithRSAEncryption 1 2 840 113549 1 1 15 */ 1146, /* OBJ_sha512_256WithRSAEncryption 1 2 840 113549 1 1 16 */ 28, /* OBJ_dhKeyAgreement 1 2 840 113549 1 3 1 */ - 9, /* OBJ_pbeWithMD2AndDES_CBC 1 2 840 113549 1 5 1 */ 10, /* OBJ_pbeWithMD5AndDES_CBC 1 2 840 113549 1 5 3 */ - 168, /* OBJ_pbeWithMD2AndRC2_CBC 1 2 840 113549 1 5 4 */ - 169, /* OBJ_pbeWithMD5AndRC2_CBC 1 2 840 113549 1 5 6 */ 170, /* OBJ_pbeWithSHA1AndDES_CBC 1 2 840 113549 1 5 10 */ - 68, /* OBJ_pbeWithSHA1AndRC2_CBC 1 2 840 113549 1 5 11 */ 69, /* OBJ_id_pbkdf2 1 2 840 113549 1 5 12 */ 161, /* OBJ_pbes2 1 2 840 113549 1 5 13 */ 162, /* OBJ_pbmac1 1 2 840 113549 1 5 14 */ @@ -5990,17 +4848,13 @@ static const unsigned int obj_objs[NUM_OBJ] = { 188, /* OBJ_SMIME 1 2 840 113549 1 9 16 */ 156, /* OBJ_friendlyName 1 2 840 113549 1 9 20 */ 157, /* OBJ_localKeyID 1 2 840 113549 1 9 21 */ - 1263, /* OBJ_id_aa_CMSAlgorithmProtection 1 2 840 113549 1 9 52 */ 681, /* OBJ_X9_62_onBasis 1 2 840 10045 1 2 3 1 */ 682, /* OBJ_X9_62_tpBasis 1 2 840 10045 1 2 3 2 */ 683, /* OBJ_X9_62_ppBasis 1 2 840 10045 1 2 3 3 */ 417, /* OBJ_ms_csp_name 1 3 6 1 4 1 311 17 1 */ 856, /* OBJ_LocalKeySet 1 3 6 1 4 1 311 17 2 */ - 1293, /* OBJ_ms_cert_templ 1 3 6 1 4 1 311 21 7 */ - 1294, /* OBJ_ms_app_policies 1 3 6 1 4 1 311 21 10 */ - 1292, /* OBJ_ms_ntds_sec_ext 1 3 6 1 4 1 311 25 2 */ + 1256, /* OBJ_delegation_usage 1 3 6 1 4 1 44363 44 */ 390, /* OBJ_dcObject 1 3 6 1 4 1 1466 344 */ - 91, /* OBJ_bf_cbc 1 3 6 1 4 1 3029 1 2 */ 973, /* OBJ_id_scrypt 1 3 6 1 4 1 11591 4 11 */ 315, /* OBJ_id_regCtrl_regToken 1 3 6 1 5 5 7 5 1 1 */ 316, /* OBJ_id_regCtrl_authenticator 1 3 6 1 5 5 7 5 1 2 */ @@ -6008,9 +4862,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 318, /* OBJ_id_regCtrl_pkiArchiveOptions 1 3 6 1 5 5 7 5 1 4 */ 319, /* OBJ_id_regCtrl_oldCertID 1 3 6 1 5 5 7 5 1 5 */ 320, /* OBJ_id_regCtrl_protocolEncrKey 1 3 6 1 5 5 7 5 1 6 */ - 1258, /* OBJ_id_regCtrl_altCertTemplate 1 3 6 1 5 5 7 5 1 7 */ - 1259, /* OBJ_id_regCtrl_algId 1 3 6 1 5 5 7 5 1 11 */ - 1260, /* OBJ_id_regCtrl_rsaKeyLen 1 3 6 1 5 5 7 5 1 12 */ 321, /* OBJ_id_regInfo_utf8Pairs 1 3 6 1 5 5 7 5 2 1 */ 322, /* OBJ_id_regInfo_certReq 1 3 6 1 5 5 7 5 2 2 */ 365, /* OBJ_id_pkix_OCSP_basic 1 3 6 1 5 5 7 48 1 1 */ @@ -6172,10 +5023,7 @@ static const unsigned int obj_objs[NUM_OBJ] = { 455, /* OBJ_pilotOrganization 0 9 2342 19200300 100 4 20 */ 456, /* OBJ_pilotDSA 0 9 2342 19200300 100 4 21 */ 457, /* OBJ_qualityLabelledData 0 9 2342 19200300 100 4 22 */ - 1281, /* OBJ_hmacWithSM3 1 2 156 10197 1 401 3 1 */ - 1152, /* OBJ_dstu28147 1 2 804 2 1 1 1 1 1 1 */ - 1156, /* OBJ_hmacWithDstu34311 1 2 804 2 1 1 1 1 1 2 */ - 1157, /* OBJ_dstu34311 1 2 804 2 1 1 1 1 2 1 */ + 1257, /* OBJ_hmacWithSM3 1 2 156 10197 1 401 3 1 */ 189, /* OBJ_id_smime_mod 1 2 840 113549 1 9 16 0 */ 190, /* OBJ_id_smime_ct 1 2 840 113549 1 9 16 1 */ 191, /* OBJ_id_smime_aa 1 2 840 113549 1 9 16 2 */ @@ -6190,8 +5038,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 145, /* OBJ_pbe_WithSHA1And40BitRC4 1 2 840 113549 1 12 1 2 */ 146, /* OBJ_pbe_WithSHA1And3_Key_TripleDES_CBC 1 2 840 113549 1 12 1 3 */ 147, /* OBJ_pbe_WithSHA1And2_Key_TripleDES_CBC 1 2 840 113549 1 12 1 4 */ - 148, /* OBJ_pbe_WithSHA1And128BitRC2_CBC 1 2 840 113549 1 12 1 5 */ - 149, /* OBJ_pbe_WithSHA1And40BitRC2_CBC 1 2 840 113549 1 12 1 6 */ 171, /* OBJ_ms_ext_req 1 3 6 1 4 1 311 2 1 14 */ 134, /* OBJ_ms_code_ind 1 3 6 1 4 1 311 2 1 21 */ 135, /* OBJ_ms_code_com 1 3 6 1 4 1 311 2 1 22 */ @@ -6200,22 +5046,10 @@ static const unsigned int obj_objs[NUM_OBJ] = { 138, /* OBJ_ms_efs 1 3 6 1 4 1 311 10 3 4 */ 648, /* OBJ_ms_smartcard_login 1 3 6 1 4 1 311 20 2 2 */ 649, /* OBJ_ms_upn 1 3 6 1 4 1 311 20 2 3 */ - 1291, /* OBJ_ms_ntds_obj_sid 1 3 6 1 4 1 311 25 2 1 */ - 1201, /* OBJ_blake2bmac 1 3 6 1 4 1 1722 12 2 1 */ - 1202, /* OBJ_blake2smac 1 3 6 1 4 1 1722 12 2 2 */ 951, /* OBJ_ct_precert_scts 1 3 6 1 4 1 11129 2 4 2 */ 952, /* OBJ_ct_precert_poison 1 3 6 1 4 1 11129 2 4 3 */ 953, /* OBJ_ct_precert_signer 1 3 6 1 4 1 11129 2 4 4 */ 954, /* OBJ_ct_cert_scts 1 3 6 1 4 1 11129 2 4 5 */ - 751, /* OBJ_camellia_128_cbc 1 2 392 200011 61 1 1 1 2 */ - 752, /* OBJ_camellia_192_cbc 1 2 392 200011 61 1 1 1 3 */ - 753, /* OBJ_camellia_256_cbc 1 2 392 200011 61 1 1 1 4 */ - 907, /* OBJ_id_camellia128_wrap 1 2 392 200011 61 1 1 3 2 */ - 908, /* OBJ_id_camellia192_wrap 1 2 392 200011 61 1 1 3 3 */ - 909, /* OBJ_id_camellia256_wrap 1 2 392 200011 61 1 1 3 4 */ - 1153, /* OBJ_dstu28147_ofb 1 2 804 2 1 1 1 1 1 1 2 */ - 1154, /* OBJ_dstu28147_cfb 1 2 804 2 1 1 1 1 1 1 3 */ - 1155, /* OBJ_dstu28147_wrap 1 2 804 2 1 1 1 1 1 1 5 */ 1158, /* OBJ_dstu4145le 1 2 804 2 1 1 1 1 3 1 1 */ 196, /* OBJ_id_smime_mod_cms 1 2 840 113549 1 9 16 0 1 */ 197, /* OBJ_id_smime_mod_ess 1 2 840 113549 1 9 16 0 2 */ @@ -6244,9 +5078,6 @@ static const unsigned int obj_objs[NUM_OBJ] = { 1237, /* OBJ_id_ct_resourceTaggedAttest 1 2 840 113549 1 9 16 1 36 */ 1246, /* OBJ_id_ct_geofeedCSVwithCRLF 1 2 840 113549 1 9 16 1 47 */ 1247, /* OBJ_id_ct_signedChecklist 1 2 840 113549 1 9 16 1 48 */ - 1250, /* OBJ_id_ct_ASPA 1 2 840 113549 1 9 16 1 49 */ - 1284, /* OBJ_id_ct_signedTAL 1 2 840 113549 1 9 16 1 50 */ - 1320, /* OBJ_id_ct_rpkiSignedPrefixList 1 2 840 113549 1 9 16 1 51 */ 212, /* OBJ_id_smime_aa_receiptRequest 1 2 840 113549 1 9 16 2 1 */ 213, /* OBJ_id_smime_aa_securityLabel 1 2 840 113549 1 9 16 2 2 */ 214, /* OBJ_id_smime_aa_mlExpandHistory 1 2 840 113549 1 9 16 2 3 */ @@ -6276,17 +5107,11 @@ static const unsigned int obj_objs[NUM_OBJ] = { 238, /* OBJ_id_smime_aa_ets_archiveTimeStamp 1 2 840 113549 1 9 16 2 27 */ 239, /* OBJ_id_smime_aa_signatureType 1 2 840 113549 1 9 16 2 28 */ 240, /* OBJ_id_smime_aa_dvcs_dvc 1 2 840 113549 1 9 16 2 29 */ - 1261, /* OBJ_id_aa_ets_attrCertificateRefs 1 2 840 113549 1 9 16 2 44 */ - 1262, /* OBJ_id_aa_ets_attrRevocationRefs 1 2 840 113549 1 9 16 2 45 */ 1086, /* OBJ_id_smime_aa_signingCertificateV2 1 2 840 113549 1 9 16 2 47 */ - 1280, /* OBJ_id_aa_ets_archiveTimestampV2 1 2 840 113549 1 9 16 2 48 */ 241, /* OBJ_id_smime_alg_ESDHwith3DES 1 2 840 113549 1 9 16 3 1 */ - 242, /* OBJ_id_smime_alg_ESDHwithRC2 1 2 840 113549 1 9 16 3 2 */ 243, /* OBJ_id_smime_alg_3DESwrap 1 2 840 113549 1 9 16 3 3 */ - 244, /* OBJ_id_smime_alg_RC2wrap 1 2 840 113549 1 9 16 3 4 */ 245, /* OBJ_id_smime_alg_ESDH 1 2 840 113549 1 9 16 3 5 */ 246, /* OBJ_id_smime_alg_CMS3DESwrap 1 2 840 113549 1 9 16 3 6 */ - 247, /* OBJ_id_smime_alg_CMSRC2wrap 1 2 840 113549 1 9 16 3 7 */ 125, /* OBJ_zlib_compression 1 2 840 113549 1 9 16 3 8 */ 893, /* OBJ_id_alg_PWRI_KEK 1 2 840 113549 1 9 16 3 9 */ 248, /* OBJ_id_smime_cd_ldap 1 2 840 113549 1 9 16 4 1 */ @@ -6304,13 +5129,9 @@ static const unsigned int obj_objs[NUM_OBJ] = { 153, /* OBJ_crlBag 1 2 840 113549 1 12 10 1 4 */ 154, /* OBJ_secretBag 1 2 840 113549 1 12 10 1 5 */ 155, /* OBJ_safeContentsBag 1 2 840 113549 1 12 10 1 6 */ - 34, /* OBJ_idea_cbc 1 3 6 1 4 1 188 7 1 1 2 */ 955, /* OBJ_jurisdictionLocalityName 1 3 6 1 4 1 311 60 2 1 1 */ 956, /* OBJ_jurisdictionStateOrProvinceName 1 3 6 1 4 1 311 60 2 1 2 */ 957, /* OBJ_jurisdictionCountryName 1 3 6 1 4 1 311 60 2 1 3 */ - 1056, /* OBJ_blake2b512 1 3 6 1 4 1 1722 12 2 1 16 */ - 1057, /* OBJ_blake2s256 1 3 6 1 4 1 1722 12 2 2 8 */ - 1283, /* OBJ_oracle_jdk_trustedkeyusage 2 16 840 1 113894 746875 1 1 */ 1159, /* OBJ_dstu4145be 1 2 804 2 1 1 1 1 3 1 1 1 1 */ 1160, /* OBJ_uacurve0 1 2 804 2 1 1 1 1 3 1 1 2 0 */ 1161, /* OBJ_uacurve1 1 2 804 2 1 1 1 1 3 1 1 2 1 */ diff --git a/openssl/src/crypto/objects/obj_lib.c b/openssl/src/crypto/objects/obj_lib.c index 6cdc1d7e7..72c0c2c81 100644 --- a/openssl/src/crypto/objects/obj_lib.c +++ b/openssl/src/crypto/objects/obj_lib.c @@ -50,6 +50,7 @@ ASN1_OBJECT *OBJ_dup(const ASN1_OBJECT *o) return r; err: ASN1_OBJECT_free(r); + ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/objects/obj_local.h b/openssl/src/crypto/objects/obj_local.h index 73848a6fb..4436b799f 100644 --- a/openssl/src/crypto/objects/obj_local.h +++ b/openssl/src/crypto/objects/obj_local.h @@ -9,6 +9,6 @@ typedef struct name_funcs_st NAME_FUNCS; DEFINE_STACK_OF(NAME_FUNCS) -DEFINE_LHASH_OF_EX(OBJ_NAME); +DEFINE_LHASH_OF(OBJ_NAME); typedef struct added_obj_st ADDED_OBJ; -DEFINE_LHASH_OF_EX(ADDED_OBJ); +DEFINE_LHASH_OF(ADDED_OBJ); diff --git a/openssl/src/crypto/objects/obj_xref.c b/openssl/src/crypto/objects/obj_xref.c index 2eb757cb7..da1035112 100644 --- a/openssl/src/crypto/objects/obj_xref.c +++ b/openssl/src/crypto/objects/obj_xref.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,11 +10,9 @@ #include #include "obj_xref.h" #include "internal/nelem.h" -#include "internal/thread_once.h" #include static STACK_OF(nid_triple) *sig_app, *sigx_app; -static CRYPTO_RWLOCK *sig_lock; static int sig_cmp(const nid_triple *a, const nid_triple *b) { @@ -34,112 +32,62 @@ DECLARE_OBJ_BSEARCH_CMP_FN(const nid_triple *, const nid_triple *, sigx); static int sigx_cmp(const nid_triple *const *a, const nid_triple *const *b) { int ret; - ret = (*a)->hash_id - (*b)->hash_id; - /* The "b" side of the comparison carries the algorithms already - * registered. A NID_undef for 'hash_id' there means that the - * signature algorithm doesn't need a digest to operate OK. In - * such case, any hash_id/digest algorithm on the test side (a), - * incl. NID_undef, is acceptable. signature algorithm NID - * (pkey_id) must match in any case. - */ - if ((ret != 0) && ((*b)->hash_id != NID_undef)) + if (ret) return ret; return (*a)->pkey_id - (*b)->pkey_id; } IMPLEMENT_OBJ_BSEARCH_CMP_FN(const nid_triple *, const nid_triple *, sigx); -static CRYPTO_ONCE sig_init = CRYPTO_ONCE_STATIC_INIT; - -DEFINE_RUN_ONCE_STATIC(o_sig_init) -{ - sig_lock = CRYPTO_THREAD_lock_new(); - return sig_lock != NULL; -} - -static ossl_inline int obj_sig_init(void) -{ - return RUN_ONCE(&sig_init, o_sig_init); -} - -static int ossl_obj_find_sigid_algs(int signid, int *pdig_nid, int *ppkey_nid, - int lock) +int OBJ_find_sigid_algs(int signid, int *pdig_nid, int *ppkey_nid) { nid_triple tmp; - const nid_triple *rv; - int idx; - - if (signid == NID_undef) - return 0; - + const nid_triple *rv = NULL; tmp.sign_id = signid; - rv = OBJ_bsearch_sig(&tmp, sigoid_srt, OSSL_NELEM(sigoid_srt)); + + if (sig_app != NULL) { + int idx = sk_nid_triple_find(sig_app, &tmp); + rv = sk_nid_triple_value(sig_app, idx); + } +#ifndef OBJ_XREF_TEST2 if (rv == NULL) { - if (!obj_sig_init()) - return 0; - if (lock && !CRYPTO_THREAD_read_lock(sig_lock)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_READ_LOCK); - return 0; - } - if (sig_app != NULL) { - idx = sk_nid_triple_find(sig_app, &tmp); - if (idx >= 0) - rv = sk_nid_triple_value(sig_app, idx); - } - if (lock) - CRYPTO_THREAD_unlock(sig_lock); - if (rv == NULL) - return 0; + rv = OBJ_bsearch_sig(&tmp, sigoid_srt, OSSL_NELEM(sigoid_srt)); } - - if (pdig_nid != NULL) +#endif + if (rv == NULL) + return 0; + if (pdig_nid) *pdig_nid = rv->hash_id; - if (ppkey_nid != NULL) + if (ppkey_nid) *ppkey_nid = rv->pkey_id; return 1; } -int OBJ_find_sigid_algs(int signid, int *pdig_nid, int *ppkey_nid) -{ - return ossl_obj_find_sigid_algs(signid, pdig_nid, ppkey_nid, 1); -} - int OBJ_find_sigid_by_algs(int *psignid, int dig_nid, int pkey_nid) { nid_triple tmp; const nid_triple *t = &tmp; - const nid_triple **rv; - int idx; - - /* permitting searches for sig algs without digest: */ - if (pkey_nid == NID_undef) - return 0; + const nid_triple **rv = NULL; tmp.hash_id = dig_nid; tmp.pkey_id = pkey_nid; - rv = OBJ_bsearch_sigx(&t, sigoid_srt_xref, OSSL_NELEM(sigoid_srt_xref)); - if (rv == NULL) { - if (!obj_sig_init()) - return 0; - if (!CRYPTO_THREAD_read_lock(sig_lock)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_READ_LOCK); - return 0; - } - if (sigx_app != NULL) { - idx = sk_nid_triple_find(sigx_app, &tmp); - if (idx >= 0) { - t = sk_nid_triple_value(sigx_app, idx); - rv = &t; - } + if (sigx_app) { + int idx = sk_nid_triple_find(sigx_app, &tmp); + if (idx >= 0) { + t = sk_nid_triple_value(sigx_app, idx); + rv = &t; } - CRYPTO_THREAD_unlock(sig_lock); - if (rv == NULL) - return 0; } - - if (psignid != NULL) +#ifndef OBJ_XREF_TEST2 + if (rv == NULL) { + rv = OBJ_bsearch_sigx(&t, sigoid_srt_xref, OSSL_NELEM(sigoid_srt_xref)); + } +#endif + if (rv == NULL) + return 0; + if (psignid) *psignid = (*rv)->sign_id; return 1; } @@ -147,63 +95,34 @@ int OBJ_find_sigid_by_algs(int *psignid, int dig_nid, int pkey_nid) int OBJ_add_sigid(int signid, int dig_id, int pkey_id) { nid_triple *ntr; - int dnid = NID_undef, pnid = NID_undef, ret = 0; - - if (signid == NID_undef || pkey_id == NID_undef) + if (sig_app == NULL) + sig_app = sk_nid_triple_new(sig_sk_cmp); + if (sig_app == NULL) return 0; - - if (!obj_sig_init()) + if (sigx_app == NULL) + sigx_app = sk_nid_triple_new(sigx_cmp); + if (sigx_app == NULL) return 0; - - if ((ntr = OPENSSL_malloc(sizeof(*ntr))) == NULL) + if ((ntr = OPENSSL_malloc(sizeof(*ntr))) == NULL) { + ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE); return 0; + } ntr->sign_id = signid; ntr->hash_id = dig_id; ntr->pkey_id = pkey_id; - if (!CRYPTO_THREAD_write_lock(sig_lock)) { - ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_WRITE_LOCK); + if (!sk_nid_triple_push(sig_app, ntr)) { OPENSSL_free(ntr); return 0; } - /* Check that the entry doesn't exist or exists as desired */ - if (ossl_obj_find_sigid_algs(signid, &dnid, &pnid, 0)) { - ret = dnid == dig_id && pnid == pkey_id; - goto err; - } - - if (sig_app == NULL) { - sig_app = sk_nid_triple_new(sig_sk_cmp); - if (sig_app == NULL) - goto err; - } - if (sigx_app == NULL) { - sigx_app = sk_nid_triple_new(sigx_cmp); - if (sigx_app == NULL) - goto err; - } - - /* - * Better might be to find where to insert the element and insert it there. - * This would avoid the sorting steps below. - */ - if (!sk_nid_triple_push(sig_app, ntr)) - goto err; - if (!sk_nid_triple_push(sigx_app, ntr)) { - ntr = NULL; /* This is referenced by sig_app still */ - goto err; - } + if (!sk_nid_triple_push(sigx_app, ntr)) + return 0; sk_nid_triple_sort(sig_app); sk_nid_triple_sort(sigx_app); - ntr = NULL; - ret = 1; - err: - OPENSSL_free(ntr); - CRYPTO_THREAD_unlock(sig_lock); - return ret; + return 1; } static void sid_free(nid_triple *tt) @@ -214,9 +133,7 @@ static void sid_free(nid_triple *tt) void OBJ_sigid_free(void) { sk_nid_triple_pop_free(sig_app, sid_free); - sk_nid_triple_free(sigx_app); - CRYPTO_THREAD_lock_free(sig_lock); sig_app = NULL; + sk_nid_triple_free(sigx_app); sigx_app = NULL; - sig_lock = NULL; } diff --git a/openssl/src/crypto/objects/obj_xref.h b/openssl/src/crypto/objects/obj_xref.h index 01339d9da..cb6c67986 100644 --- a/openssl/src/crypto/objects/obj_xref.h +++ b/openssl/src/crypto/objects/obj_xref.h @@ -2,7 +2,7 @@ * WARNING: do not edit! * Generated by objxref.pl * - * Copyright 1998-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1998-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,18 +20,14 @@ typedef struct { DEFINE_STACK_OF(nid_triple) static const nid_triple sigoid_srt[] = { - {NID_md2WithRSAEncryption, NID_md2, NID_rsaEncryption}, {NID_md5WithRSAEncryption, NID_md5, NID_rsaEncryption}, {NID_shaWithRSAEncryption, NID_sha, NID_rsaEncryption}, {NID_sha1WithRSAEncryption, NID_sha1, NID_rsaEncryption}, {NID_dsaWithSHA, NID_sha, NID_dsa}, {NID_dsaWithSHA1_2, NID_sha1, NID_dsa_2}, - {NID_mdc2WithRSA, NID_mdc2, NID_rsaEncryption}, {NID_md5WithRSA, NID_md5, NID_rsa}, {NID_dsaWithSHA1, NID_sha1, NID_dsa}, {NID_sha1WithRSA, NID_sha1, NID_rsa}, - {NID_ripemd160WithRSA, NID_ripemd160, NID_rsaEncryption}, - {NID_md4WithRSAEncryption, NID_md4, NID_rsaEncryption}, {NID_ecdsa_with_SHA1, NID_sha1, NID_X9_62_id_ecPublicKey}, {NID_sha256WithRSAEncryption, NID_sha256, NID_rsaEncryption}, {NID_sha384WithRSAEncryption, NID_sha384, NID_rsaEncryption}, @@ -45,14 +41,6 @@ static const nid_triple sigoid_srt[] = { {NID_ecdsa_with_SHA512, NID_sha512, NID_X9_62_id_ecPublicKey}, {NID_dsa_with_SHA224, NID_sha224, NID_dsa}, {NID_dsa_with_SHA256, NID_sha256, NID_dsa}, - {NID_id_GostR3411_94_with_GostR3410_2001, NID_id_GostR3411_94, - NID_id_GostR3410_2001}, - {NID_id_GostR3411_94_with_GostR3410_94, NID_id_GostR3411_94, - NID_id_GostR3410_94}, - {NID_id_GostR3411_94_with_GostR3410_94_cc, NID_id_GostR3411_94, - NID_id_GostR3410_94_cc}, - {NID_id_GostR3411_94_with_GostR3410_2001_cc, NID_id_GostR3411_94, - NID_id_GostR3410_2001_cc}, {NID_rsassaPss, NID_undef, NID_rsassaPss}, {NID_dhSinglePass_stdDH_sha1kdf_scheme, NID_sha1, NID_dh_std_kdf}, {NID_dhSinglePass_stdDH_sha224kdf_scheme, NID_sha224, NID_dh_std_kdf}, @@ -69,16 +57,8 @@ static const nid_triple sigoid_srt[] = { NID_dh_cofactor_kdf}, {NID_dhSinglePass_cofactorDH_sha512kdf_scheme, NID_sha512, NID_dh_cofactor_kdf}, - {NID_id_tc26_signwithdigest_gost3410_2012_256, NID_id_GostR3411_2012_256, - NID_id_GostR3410_2012_256}, - {NID_id_tc26_signwithdigest_gost3410_2012_512, NID_id_GostR3411_2012_512, - NID_id_GostR3410_2012_512}, {NID_ED25519, NID_undef, NID_ED25519}, {NID_ED448, NID_undef, NID_ED448}, - {NID_ecdsa_with_SHA3_224, NID_sha3_224, NID_X9_62_id_ecPublicKey}, - {NID_ecdsa_with_SHA3_256, NID_sha3_256, NID_X9_62_id_ecPublicKey}, - {NID_ecdsa_with_SHA3_384, NID_sha3_384, NID_X9_62_id_ecPublicKey}, - {NID_ecdsa_with_SHA3_512, NID_sha3_512, NID_X9_62_id_ecPublicKey}, {NID_RSA_SHA3_224, NID_sha3_224, NID_rsaEncryption}, {NID_RSA_SHA3_256, NID_sha3_256, NID_rsaEncryption}, {NID_RSA_SHA3_384, NID_sha3_384, NID_rsaEncryption}, @@ -88,51 +68,37 @@ static const nid_triple sigoid_srt[] = { static const nid_triple *const sigoid_srt_xref[] = { &sigoid_srt[0], + &sigoid_srt[5], &sigoid_srt[1], - &sigoid_srt[7], + &sigoid_srt[3], &sigoid_srt[2], + &sigoid_srt[7], &sigoid_srt[4], - &sigoid_srt[3], - &sigoid_srt[9], - &sigoid_srt[5], - &sigoid_srt[8], - &sigoid_srt[12], - &sigoid_srt[30], - &sigoid_srt[35], &sigoid_srt[6], - &sigoid_srt[10], - &sigoid_srt[11], - &sigoid_srt[13], - &sigoid_srt[24], - &sigoid_srt[20], - &sigoid_srt[32], - &sigoid_srt[37], - &sigoid_srt[14], - &sigoid_srt[21], - &sigoid_srt[33], - &sigoid_srt[38], - &sigoid_srt[15], + &sigoid_srt[8], &sigoid_srt[22], - &sigoid_srt[34], - &sigoid_srt[39], + &sigoid_srt[27], + &sigoid_srt[9], + &sigoid_srt[20], &sigoid_srt[16], - &sigoid_srt[23], - &sigoid_srt[19], - &sigoid_srt[31], - &sigoid_srt[36], + &sigoid_srt[24], + &sigoid_srt[29], + &sigoid_srt[10], + &sigoid_srt[17], &sigoid_srt[25], + &sigoid_srt[30], + &sigoid_srt[11], + &sigoid_srt[18], &sigoid_srt[26], - &sigoid_srt[27], + &sigoid_srt[31], + &sigoid_srt[12], + &sigoid_srt[19], + &sigoid_srt[15], + &sigoid_srt[23], &sigoid_srt[28], - &sigoid_srt[40], - &sigoid_srt[41], - &sigoid_srt[48], - &sigoid_srt[44], - &sigoid_srt[49], - &sigoid_srt[45], - &sigoid_srt[50], - &sigoid_srt[46], - &sigoid_srt[51], - &sigoid_srt[47], - &sigoid_srt[52], + &sigoid_srt[34], + &sigoid_srt[35], + &sigoid_srt[36], + &sigoid_srt[37], + &sigoid_srt[38], }; diff --git a/openssl/src/crypto/ocsp/ocsp_ext.c b/openssl/src/crypto/ocsp/ocsp_ext.c index 9707ccb94..c2b61bd4f 100644 --- a/openssl/src/crypto/ocsp/ocsp_ext.c +++ b/openssl/src/crypto/ocsp/ocsp_ext.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -268,8 +268,8 @@ static int ocsp_add1_nonce(STACK_OF(X509_EXTENSION) **exts, memcpy(tmpval, val, len); else if (RAND_bytes(tmpval, len) <= 0) goto err; - if (X509V3_add1_i2d(exts, NID_id_pkix_OCSP_Nonce, - &os, 0, X509V3_ADD_REPLACE) <= 0) + if (!X509V3_add1_i2d(exts, NID_id_pkix_OCSP_Nonce, + &os, 0, X509V3_ADD_REPLACE)) goto err; ret = 1; err: diff --git a/openssl/src/crypto/ocsp/ocsp_prn.c b/openssl/src/crypto/ocsp/ocsp_prn.c index 6fe65b6c0..654ddbc7f 100644 --- a/openssl/src/crypto/ocsp/ocsp_prn.c +++ b/openssl/src/crypto/ocsp/ocsp_prn.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -79,9 +79,7 @@ const char *OCSP_crl_reason_str(long s) {OCSP_REVOKED_STATUS_SUPERSEDED, "superseded"}, {OCSP_REVOKED_STATUS_CESSATIONOFOPERATION, "cessationOfOperation"}, {OCSP_REVOKED_STATUS_CERTIFICATEHOLD, "certificateHold"}, - {OCSP_REVOKED_STATUS_REMOVEFROMCRL, "removeFromCRL"}, - {OCSP_REVOKED_STATUS_PRIVILEGEWITHDRAWN, "privilegeWithdrawn"}, - {OCSP_REVOKED_STATUS_AACOMPROMISE, "aACompromise"} + {OCSP_REVOKED_STATUS_REMOVEFROMCRL, "removeFromCRL"} }; return table2string(s, reason_tbl); } diff --git a/openssl/src/crypto/ocsp/ocsp_vfy.c b/openssl/src/crypto/ocsp/ocsp_vfy.c index b0827e9a2..fa07539d7 100644 --- a/openssl/src/crypto/ocsp/ocsp_vfy.c +++ b/openssl/src/crypto/ocsp/ocsp_vfy.c @@ -36,7 +36,7 @@ static int ocsp_verify_signer(X509 *signer, int response, int ret = -1; if (ctx == NULL) { - ERR_raise(ERR_LIB_OCSP, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_OCSP, ERR_R_MALLOC_FAILURE); goto end; } if (!X509_STORE_CTX_init(ctx, st, signer, untrusted)) { @@ -154,7 +154,7 @@ int OCSP_basic_verify(OCSP_BASICRESP *bs, STACK_OF(X509) *certs, } end: - OSSL_STACK_OF_X509_free(chain); + sk_X509_pop_free(chain, X509_free); sk_X509_free(untrusted); return ret; } diff --git a/openssl/src/crypto/ocsp/v3_ocsp.c b/openssl/src/crypto/ocsp/v3_ocsp.c index 4f54b7cea..2250208a1 100644 --- a/openssl/src/crypto/ocsp/v3_ocsp.c +++ b/openssl/src/crypto/ocsp/v3_ocsp.c @@ -203,7 +203,7 @@ static void *d2i_ocsp_nonce(void *a, const unsigned char **pp, long length) err: if ((pos == NULL) || (*pos != os)) ASN1_OCTET_STRING_free(os); - ERR_raise(ERR_LIB_OCSP, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_OCSP, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/packet.c b/openssl/src/crypto/packet.c index 6a43b3565..09f6a9cea 100644 --- a/openssl/src/crypto/packet.c +++ b/openssl/src/crypto/packet.c @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,9 +9,6 @@ #include "internal/cryptlib.h" #include "internal/packet.h" -#if !defined OPENSSL_NO_QUIC && !defined FIPS_MODULE -# include "internal/packet_quic.h" -#endif #include #define DEFAULT_BUF_SIZE 256 @@ -107,8 +104,10 @@ static int wpacket_intern_init_len(WPACKET *pkt, size_t lenbytes) pkt->curr = 0; pkt->written = 0; - if ((pkt->subs = OPENSSL_zalloc(sizeof(*pkt->subs))) == NULL) + if ((pkt->subs = OPENSSL_zalloc(sizeof(*pkt->subs))) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } if (lenbytes == 0) return 1; @@ -208,7 +207,7 @@ int WPACKET_set_flags(WPACKET *pkt, unsigned int flags) } /* Store the |value| of length |len| at location |data| */ -static int put_value(unsigned char *data, uint64_t value, size_t len) +static int put_value(unsigned char *data, size_t value, size_t len) { if (data == NULL) return 1; @@ -226,20 +225,6 @@ static int put_value(unsigned char *data, uint64_t value, size_t len) return 1; } -#if !defined OPENSSL_NO_QUIC && !defined FIPS_MODULE -static int put_quic_value(unsigned char *data, size_t value, size_t len) -{ - if (data == NULL) - return 1; - - /* Value too large for field. */ - if (ossl_quic_vlint_encode_len(value) > len) - return 0; - - ossl_quic_vlint_encode_n(data, value, len); - return 1; -} -#endif /* * Internal helper function used by WPACKET_close(), WPACKET_finish() and @@ -276,20 +261,10 @@ static int wpacket_intern_close(WPACKET *pkt, WPACKET_SUB *sub, int doclose) if (sub->lenbytes > 0) { unsigned char *buf = GETBUF(pkt); - if (buf != NULL) { -#if !defined OPENSSL_NO_QUIC && !defined FIPS_MODULE - if ((sub->flags & WPACKET_FLAGS_QUIC_VLINT) == 0) { - if (!put_value(&buf[sub->packet_len], packlen, sub->lenbytes)) - return 0; - } else { - if (!put_quic_value(&buf[sub->packet_len], packlen, sub->lenbytes)) - return 0; - } -#else - if (!put_value(&buf[sub->packet_len], packlen, sub->lenbytes)) - return 0; -#endif - } + if (buf != NULL + && !put_value(&buf[sub->packet_len], packlen, + sub->lenbytes)) + return 0; } else if (pkt->endfirst && sub->parent != NULL && (packlen != 0 || (sub->flags @@ -376,8 +351,10 @@ int WPACKET_start_sub_packet_len__(WPACKET *pkt, size_t lenbytes) if (lenbytes > 0 && pkt->endfirst) return 0; - if ((sub = OPENSSL_zalloc(sizeof(*sub))) == NULL) + if ((sub = OPENSSL_zalloc(sizeof(*sub))) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } sub->parent = pkt->subs; pkt->subs = sub; @@ -402,12 +379,12 @@ int WPACKET_start_sub_packet(WPACKET *pkt) return WPACKET_start_sub_packet_len__(pkt, 0); } -int WPACKET_put_bytes__(WPACKET *pkt, uint64_t val, size_t size) +int WPACKET_put_bytes__(WPACKET *pkt, unsigned int val, size_t size) { unsigned char *data; /* Internal API, so should not fail */ - if (!ossl_assert(size <= sizeof(uint64_t)) + if (!ossl_assert(size <= sizeof(unsigned int)) || !WPACKET_allocate_bytes(pkt, size, &data) || !put_value(data, val, size)) return 0; @@ -533,58 +510,3 @@ void WPACKET_cleanup(WPACKET *pkt) } pkt->subs = NULL; } - -#if !defined OPENSSL_NO_QUIC && !defined FIPS_MODULE - -int WPACKET_start_quic_sub_packet_bound(WPACKET *pkt, size_t max_len) -{ - size_t enclen = ossl_quic_vlint_encode_len(max_len); - - if (enclen == 0) - return 0; - - if (WPACKET_start_sub_packet_len__(pkt, enclen) == 0) - return 0; - - pkt->subs->flags |= WPACKET_FLAGS_QUIC_VLINT; - return 1; -} - -int WPACKET_start_quic_sub_packet(WPACKET *pkt) -{ - /* - * Assume no (sub)packet will exceed 4GiB, thus the 8-byte encoding need not - * be used. - */ - return WPACKET_start_quic_sub_packet_bound(pkt, OSSL_QUIC_VLINT_4B_MIN); -} - -int WPACKET_quic_sub_allocate_bytes(WPACKET *pkt, size_t len, unsigned char **allocbytes) -{ - if (!WPACKET_start_quic_sub_packet_bound(pkt, len) - || !WPACKET_allocate_bytes(pkt, len, allocbytes) - || !WPACKET_close(pkt)) - return 0; - - return 1; -} - -/* - * Write a QUIC variable-length integer to the packet. - */ -int WPACKET_quic_write_vlint(WPACKET *pkt, uint64_t v) -{ - unsigned char *b = NULL; - size_t enclen = ossl_quic_vlint_encode_len(v); - - if (enclen == 0) - return 0; - - if (WPACKET_allocate_bytes(pkt, enclen, &b) == 0) - return 0; - - ossl_quic_vlint_encode(b, v); - return 1; -} - -#endif diff --git a/openssl/src/crypto/paillier/paillier_asn1.c b/openssl/src/crypto/paillier/paillier_asn1.c new file mode 100644 index 000000000..b07bedbdc --- /dev/null +++ b/openssl/src/crypto/paillier/paillier_asn1.c @@ -0,0 +1,71 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include "internal/deprecated.h" +#include +#include +#include +#include +#include "paillier_local.h" + +static int paillier_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, + void *exarg) +{ + PAILLIER_KEY *key; + BN_CTX *bn_ctx = NULL; + + if (operation == ASN1_OP_NEW_PRE) { + *pval = (ASN1_VALUE *)PAILLIER_KEY_new(); + if (*pval != NULL) + return 2; + return 0; + } else if (operation == ASN1_OP_FREE_PRE) { + PAILLIER_KEY_free((PAILLIER_KEY *)*pval); + *pval = NULL; + return 2; + } else if (operation == ASN1_OP_D2I_POST) { + key = (PAILLIER_KEY *)*pval; + if (key->version != PAILLIER_ASN1_VERSION_MULTI) { + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + return 0; + + if (!BN_sqr(key->n_square, key->n, bn_ctx)) { + BN_CTX_free(bn_ctx); + return 0; + } + + BN_CTX_free(bn_ctx); + return 1; + } + return (ossl_paillier_multip_calc_product((PAILLIER_KEY *)*pval) == 1) ? 2 : 0; + } + return 1; +} + +ASN1_SEQUENCE_cb(PAILLIER_PrivateKey, paillier_cb) = { + ASN1_EMBED(PAILLIER_KEY, version, INT32), + ASN1_SIMPLE(PAILLIER_KEY, n, BIGNUM), + ASN1_SIMPLE(PAILLIER_KEY, p, CBIGNUM), + ASN1_SIMPLE(PAILLIER_KEY, q, CBIGNUM), + ASN1_SIMPLE(PAILLIER_KEY, g, CBIGNUM), + ASN1_SIMPLE(PAILLIER_KEY, lambda, CBIGNUM), + ASN1_SIMPLE(PAILLIER_KEY, u, CBIGNUM), +} static_ASN1_SEQUENCE_END_cb(PAILLIER_KEY, PAILLIER_PrivateKey) + +ASN1_SEQUENCE_cb(PAILLIER_PublicKey, paillier_cb) = { + ASN1_SIMPLE(PAILLIER_KEY, n, BIGNUM), + ASN1_SIMPLE(PAILLIER_KEY, g, CBIGNUM), +} static_ASN1_SEQUENCE_END_cb(PAILLIER_KEY, PAILLIER_PublicKey) + +IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(PAILLIER_KEY, PAILLIER_PrivateKey, PAILLIER_PrivateKey) +IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(PAILLIER_KEY, PAILLIER_PublicKey, PAILLIER_PublicKey) + +IMPLEMENT_PEM_rw(PAILLIER_PrivateKey, PAILLIER_KEY, PEM_STRING_PAILLIER_PRIVATE_KEY, PAILLIER_PrivateKey) +IMPLEMENT_PEM_rw(PAILLIER_PublicKey, PAILLIER_KEY, PEM_STRING_PAILLIER_PUBLIC_KEY, PAILLIER_PublicKey) diff --git a/openssl/src/crypto/paillier/paillier_crypt.c b/openssl/src/crypto/paillier/paillier_crypt.c new file mode 100644 index 000000000..797fea74e --- /dev/null +++ b/openssl/src/crypto/paillier/paillier_crypt.c @@ -0,0 +1,343 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include "paillier_local.h" + +/** Encrypts an Integer with additadive homomorphic Paillier + * \param ctx PAILLIER_CTX object. + * \param r PAILLIER_CIPHERTEXT object that stores the result of + * the encryption + * \param m The plaintext integer to be encrypted + * \return 1 on success and 0 otherwise + */ +int PAILLIER_encrypt(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *out, int32_t m) +{ + int ret = 0; + PAILLIER_KEY *key; + BN_CTX *bn_ctx = NULL; + BIGNUM *r, *r_exp_n, *g_exp_m, *bn_plain; + + if (ctx == NULL || out == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + + BN_CTX_start(bn_ctx); + +#if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_BN_METHOD) + if (ctx->engine != NULL && !BN_CTX_set_engine(bn_ctx, ctx->engine)) + goto err; +#endif + + key = ctx->key; + + bn_plain = BN_CTX_get(bn_ctx); + r = BN_CTX_get(bn_ctx); + r_exp_n = BN_CTX_get(bn_ctx); + g_exp_m = BN_CTX_get(bn_ctx); + if (g_exp_m == NULL) + goto err; + + BN_set_word(bn_plain, (BN_ULONG)(m > 0 ? m : -m)); + BN_set_negative(bn_plain, m < 0 ? 1 : 0); + + if (!BN_rand_range(r, key->n)) + goto err; + + if (!BN_mod_exp(r_exp_n, r, key->n, key->n_square, bn_ctx)) + goto err; + + if (key->flag & PAILLIER_FLAG_G_OPTIMIZE) { + if (!BN_mul(g_exp_m, bn_plain, key->n, bn_ctx)) + goto err; + + if (!BN_add_word(g_exp_m, (BN_ULONG)1)) + goto err; + + if (!BN_mod(g_exp_m, g_exp_m, key->n_square, bn_ctx)) + goto err; + } else { + if (!BN_mod_exp(g_exp_m, key->g, bn_plain, key->n_square, bn_ctx)) + goto err; + + if (m < 0 && !BN_mod_inverse(g_exp_m, g_exp_m, key->n_square, bn_ctx)) + goto err; + } + + if (!BN_mod_mul(out->data, g_exp_m, r_exp_n, key->n_square, bn_ctx)) + goto err; + + ret = 1; + +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/** Decrypts the ciphertext + * \param ctx PAILLIER_CTX object + * \param r The resulting plaintext integer + * \param c PAILLIER_CIPHERTEXT object to be decrypted + * \return 1 on success and 0 otherwise + */ +int PAILLIER_decrypt(PAILLIER_CTX *ctx, int32_t *out, PAILLIER_CIPHERTEXT *c) +{ + int ret = 0; + int32_t result; + char *p = NULL; + PAILLIER_KEY *key; + BN_CTX *bn_ctx = NULL; + BIGNUM *c_exp_lambda, *l_ret, *bn_out; + + if (ctx == NULL || out == NULL || c == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + + BN_CTX_start(bn_ctx); + +#if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_BN_METHOD) + if (ctx->engine != NULL && !BN_CTX_set_engine(bn_ctx, ctx->engine)) + goto err; +#endif + + key = ctx->key; + + bn_out = BN_CTX_get(bn_ctx); + c_exp_lambda = BN_CTX_get(bn_ctx); + l_ret = BN_CTX_get(bn_ctx); + if (l_ret == NULL) + goto err; + + if (!BN_mod_exp(c_exp_lambda, c->data, key->lambda, key->n_square, bn_ctx)) + goto err; + + if (!paillier_l_func(l_ret, c_exp_lambda, key->n, bn_ctx)) + goto err; + + if (!BN_mod_mul(bn_out, l_ret, key->u, key->n, bn_ctx)) + goto err; + + if (BN_cmp(bn_out, ctx->threshold) == 1) { + if (!BN_sub(bn_out, bn_out, key->n)) + goto err; + } + + p = BN_bn2dec(bn_out); + if (p == NULL) + goto err; + + result = atoi(p); + if (result == 0 && *p != '0') + goto err; + + *out = result; + ret = 1; + +err: + OPENSSL_free(p); + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/** Adds two paillier ciphertext and stores it in r: + * E(r) = E(c1 + c2) = E(c1) * E(c2) + * \param ctx PAILLIER_CTX object + * \param r The PAILLIER_CIPHERTEXT object that stores the addition + * result + * \param c1 PAILLIER_CIPHERTEXT object + * \param c2 PAILLIER_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int PAILLIER_add(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + PAILLIER_CIPHERTEXT *c1, PAILLIER_CIPHERTEXT *c2) +{ + int ret = 0; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL || r == NULL || c1 == NULL || c2 == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + return 0; + +#if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_BN_METHOD) + if (ctx->engine != NULL && !BN_CTX_set_engine(bn_ctx, ctx->engine)) { + BN_CTX_free(bn_ctx); + return ret; + } +#endif + + ret = BN_mod_mul(r->data, c1->data, c2->data, ctx->key->n_square, bn_ctx); + + BN_CTX_free(bn_ctx); + return ret; +} + +/** Add a paillier ciphertext to a plaintext, and stores it in r: + * E(r) = E(c1 + m) = E(c1) * g^m + * \param ctx PAILLIER_CTX object + * \param r The PAILLIER_CIPHERTEXT object that stores the addition + * result + * \param c1 PAILLIER_CIPHERTEXT object + * \param m The plaintext integer to be added + * \return 1 on success and 0 otherwise + */ +int PAILLIER_add_plain(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + PAILLIER_CIPHERTEXT *c, int32_t m) +{ + int ret = 0; + BN_CTX *bn_ctx = NULL; + BIGNUM *g_exp_p, *bn_plain; + + if (ctx == NULL || r == NULL || c == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + + BN_CTX_start(bn_ctx); + +#if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_BN_METHOD) + if (ctx->engine != NULL && !BN_CTX_set_engine(bn_ctx, ctx->engine)) + goto err; +#endif + + bn_plain = BN_CTX_get(bn_ctx); + g_exp_p = BN_CTX_get(bn_ctx); + if (g_exp_p == NULL) + goto err; + + BN_set_word(bn_plain, (BN_ULONG)(m > 0 ? m : -m)); + //BN_set_negative(bn_plain, m < 0 ? 1 : 0); + + if (!BN_mod_exp(g_exp_p, ctx->key->g, bn_plain, ctx->key->n_square, bn_ctx)) + goto err; + + if (m < 0 && !BN_mod_inverse(g_exp_p, g_exp_p, ctx->key->n_square, bn_ctx)) + goto err; + + ret = BN_mod_mul(r->data, c->data, g_exp_p, ctx->key->n_square, bn_ctx); + +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/** Substracts two paillier ciphertext and stores it in r: + * E(r) = E(c1 - c2) = E(c1) * E(-c2) = E(c1) / E(c2) + * \param ctx PAILLIER_CTX object + * \param r The PAILLIER_CIPHERTEXT object that stores the + * subtraction result + * \param c1 PAILLIER_CIPHERTEXT object + * \param c2 PAILLIER_CIPHERTEXT object + * \return 1 on success and 0 otherwise + */ +int PAILLIER_sub(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + PAILLIER_CIPHERTEXT *c1, PAILLIER_CIPHERTEXT *c2) +{ + int ret = 0; + BN_CTX *bn_ctx = NULL; + BIGNUM *inv; + + if (ctx == NULL || r == NULL || c1 == NULL || c2 == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + return 0; + + BN_CTX_start(bn_ctx); + +#if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_BN_METHOD) + if (ctx->engine != NULL && !BN_CTX_set_engine(bn_ctx, ctx->engine)) + goto err; +#endif + + inv = BN_CTX_get(bn_ctx); + if (inv == NULL) + goto err; + + if (!BN_mod_inverse(inv, c2->data, ctx->key->n_square, bn_ctx)) + goto err; + + ret = BN_mod_mul(r->data, c1->data, inv, ctx->key->n_square, bn_ctx); + +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/** Ciphertext multiplication, computes E(r) = E(c * m) = E(c) ^ m + * \param ctx PAILLIER_CTX object + * \param r The PAILLIER_CIPHERTEXT object that stores the + * multiplication result + * \param c1 PAILLIER_CIPHERTEXT object + * \param m The plaintext integer to be multiplied + * \return 1 on success and 0 otherwise + */ +int PAILLIER_mul(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + PAILLIER_CIPHERTEXT *c, int32_t m) +{ + int ret = 0; + BN_CTX *bn_ctx = NULL; + BIGNUM *bn_plain; + + if (ctx == NULL || r == NULL || c == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + + BN_CTX_start(bn_ctx); + +#if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_BN_METHOD) + if (ctx->engine != NULL && !BN_CTX_set_engine(bn_ctx, ctx->engine)) + goto err; +#endif + + bn_plain = BN_CTX_get(bn_ctx); + if (bn_plain == NULL) + goto err; + + BN_set_word(bn_plain, (BN_ULONG)(m > 0 ? m : -m)); + + ret = BN_mod_exp(r->data, c->data, bn_plain, ctx->key->n_square, bn_ctx); + if (m < 0) + ret = BN_mod_inverse(r->data, r->data, ctx->key->n_square, bn_ctx) != NULL; + +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} diff --git a/openssl/src/crypto/paillier/paillier_ctx.c b/openssl/src/crypto/paillier/paillier_ctx.c new file mode 100644 index 000000000..418f05675 --- /dev/null +++ b/openssl/src/crypto/paillier/paillier_ctx.c @@ -0,0 +1,146 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include "paillier_local.h" + +/** Creates a new PAILLIER object + * \param key PAILLIER_KEY to use + * \param threshold The threshold should be greater than the maximum integer + * that will be encrypted. + * \return newly created PAILLIER_CTX object or NULL in case of an error + */ +PAILLIER_CTX *PAILLIER_CTX_new(PAILLIER_KEY *key, int64_t threshold) +{ + char tmp[20]; + PAILLIER_CTX *ctx = NULL; + + if (key == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!PAILLIER_KEY_up_ref(key)) + goto err; + + ctx->key = key; + ctx->threshold = BN_new(); + if (ctx->threshold == NULL) + goto err; + + memset(tmp, 0, sizeof(tmp)); + BIO_snprintf(tmp, sizeof(tmp), "%lld", (long long int)threshold); + + if (!BN_dec2bn(&ctx->threshold, (char *)tmp)) + goto err; + + return ctx; + +err: + OPENSSL_free(ctx); + return NULL; +} + +/** Frees a PAILLIER_CTX object + * \param ctx PAILLIER_CTX object to be freed + */ +void PAILLIER_CTX_free(PAILLIER_CTX *ctx) +{ + if (ctx == NULL) + return; + +# ifndef OPENSSL_NO_BN_METHOD + ENGINE_free(ctx->engine); +# endif + + PAILLIER_KEY_free(ctx->key); + BN_free(ctx->threshold); + OPENSSL_clear_free((void *)ctx, sizeof(PAILLIER_CTX)); +} + +/** Copies a PAILLIER_KEY object. + * \param dst destination PAILLIER_KEY object + * \param src src PAILLIER_KEY object + * \return dst or NULL if an error occurred. + */ +PAILLIER_CTX *PAILLIER_CTX_copy(PAILLIER_CTX *dest, PAILLIER_CTX *src) +{ + if (dest == NULL || src == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (dest == src) + return dest; + + if (!PAILLIER_KEY_copy(dest->key, src->key)) + return NULL; + + return dest; +} + +/** Creates a new PAILLIER_KEY object and copies the content from src to it. + * \param src the source PAILLIER_KEY object + * \return newly created PAILLIER_KEY object or NULL if an error occurred. + */ +PAILLIER_CTX *PAILLIER_CTX_dup(PAILLIER_CTX *src) +{ + PAILLIER_CTX *ret = NULL; + + if (src == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ret = OPENSSL_zalloc(sizeof(*ret)); + if (ret == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ret->key = PAILLIER_KEY_dup(src->key); + if (ret->key == NULL) + goto err; + + return ret; +err: + OPENSSL_free(ret); + return NULL; +} + +#ifndef OPENSSL_NO_ENGINE +/** set ENGINE pointer to the PAILLIER object + * \param ctx PAILLIER_CTX object. + * \param engine ENGINE object to use + * \return 1 on success and 0 otherwise + */ +int PAILLIER_CTX_set_engine(PAILLIER_CTX *ctx, ENGINE *engine) +{ +# ifndef OPENSSL_NO_BN_METHOD + if (ctx == NULL || engine == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!ENGINE_up_ref(engine)) + return 0; + + ctx->engine = engine; + return 1; +# else + return 0; +# endif +} +#endif diff --git a/openssl/src/crypto/paillier/paillier_encode.c b/openssl/src/crypto/paillier/paillier_encode.c new file mode 100644 index 000000000..a86a9200b --- /dev/null +++ b/openssl/src/crypto/paillier/paillier_encode.c @@ -0,0 +1,108 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include "paillier_local.h" + +/** Creates a new PAILLIER_CIPHERTEXT object for paillier oparations + * \param ctx PAILLIER_CTX object + * \return newly created PAILLIER_CIPHERTEXT object or NULL in case of an error + */ +PAILLIER_CIPHERTEXT *PAILLIER_CIPHERTEXT_new(PAILLIER_CTX *ctx) +{ + PAILLIER_CIPHERTEXT *ciphertext = NULL; + + ciphertext = OPENSSL_zalloc(sizeof(*ciphertext)); + if (ciphertext == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ciphertext->data = BN_new(); + if (ciphertext->data == NULL) + goto err; + + return ciphertext; +err: + OPENSSL_free(ciphertext); + return NULL; +} + +/** Frees a PAILLIER_CIPHERTEXT object + * \param ciphertext PAILLIER_CIPHERTEXT object to be freed + */ +void PAILLIER_CIPHERTEXT_free(PAILLIER_CIPHERTEXT *ciphertext) +{ + if (ciphertext == NULL) + return; + + BN_free(ciphertext->data); + OPENSSL_clear_free((void *)ciphertext, sizeof(PAILLIER_CIPHERTEXT)); +} + +/** Encodes PAILLIER_CIPHERTEXT to binary + * \param ctx PAILLIER_CTX object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param ciphertext PAILLIER_CIPHERTEXT object + * \param compressed Whether to compress the encoding (either 0 or 1) + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t PAILLIER_CIPHERTEXT_encode(PAILLIER_CTX *ctx, unsigned char *out, + size_t size, + const PAILLIER_CIPHERTEXT *ciphertext, + int flag) +{ + size_t ret = 0, len; + + if (ctx == NULL || ctx->key == NULL + || ciphertext == NULL || ciphertext->data == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + len = BN_num_bytes(ciphertext->data); + + if (out == NULL) + return len; + + if (size < len) + goto end; + + if (!BN_bn2bin(ciphertext->data, out)) + goto end; + + ret = len; + +end: + return ret; +} + +/** Decodes binary to PAILLIER_CIPHERTEXT + * \param ctx PAILLIER_CTX object + * \param r the resulting ciphertext + * \param in Memory buffer with the encoded PAILLIER_CIPHERTEXT + * object + * \param size The memory size of the in pointer object + * \return 1 on success and 0 otherwise + */ +int PAILLIER_CIPHERTEXT_decode(PAILLIER_CTX *ctx, PAILLIER_CIPHERTEXT *r, + unsigned char *in, size_t size) +{ + if (ctx == NULL || ctx->key == NULL || r == NULL || r->data == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!BN_bin2bn(in, (int)size, r->data)) + return 0; + + return 1; +} diff --git a/openssl/src/crypto/paillier/paillier_key.c b/openssl/src/crypto/paillier/paillier_key.c new file mode 100644 index 000000000..ca7323126 --- /dev/null +++ b/openssl/src/crypto/paillier/paillier_key.c @@ -0,0 +1,429 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include "paillier_local.h" + +/** + * Creates a new PAILLIER_KEY object. + * \return PAILLIER_KEY object or NULL if an error occurred. + */ +PAILLIER_KEY *PAILLIER_KEY_new(void) +{ + PAILLIER_KEY *key = NULL; + + key = OPENSSL_zalloc(sizeof(*key)); + if (key == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if ((key->n = BN_new()) == NULL + || (key->n_square = BN_new()) == NULL + || (key->g = BN_new()) == NULL + || (key->lambda = BN_new()) == NULL + || (key->u = BN_new()) == NULL) + goto err; + + key->references = 1; + if ((key->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_MALLOC_FAILURE); + goto err; + } + + return key; +err: + PAILLIER_KEY_free(key); + return NULL; +} + +/** Frees a PAILLIER_KEY object. + * \param key PAILLIER_KEY object to be freed. + */ +void PAILLIER_KEY_free(PAILLIER_KEY *key) +{ + int i; + + if (key == NULL) + return; + + CRYPTO_DOWN_REF(&key->references, &i, key->lock); + REF_PRINT_COUNT("PAILLIER_KEY", key); + if (i > 0) + return; + REF_ASSERT_ISNT(i < 0); + + BN_free(key->p); + BN_free(key->q); + BN_free(key->n); + BN_free(key->n_square); + BN_free(key->g); + BN_free(key->lambda); + BN_free(key->u); + + CRYPTO_THREAD_lock_free(key->lock); + + OPENSSL_clear_free((void *)key, sizeof(PAILLIER_KEY)); +} + +/** Copies a PAILLIER_KEY object. + * \param dst destination PAILLIER_KEY object + * \param src src PAILLIER_KEY object + * \return dst or NULL if an error occurred. + */ +PAILLIER_KEY *PAILLIER_KEY_copy(PAILLIER_KEY *dest, PAILLIER_KEY *src) +{ + if (dest == NULL || src == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (dest == src) + return dest; + + dest->version = src->version; + + dest->p = (dest->p ? BN_copy(dest->p, src->p) : BN_dup(src->p)); + dest->q = (dest->q ? BN_copy(dest->q, src->q) : BN_dup(src->q)); + + if (!dest->p || !dest->q + || !BN_copy(dest->n, src->n) + || !BN_copy(dest->n_square, src->n_square) + || !BN_copy(dest->g, src->g) + || !BN_copy(dest->lambda, src->lambda) + || !BN_copy(dest->u, src->u)) + return NULL; + + return dest; +} + +/** Creates a new PAILLIER_KEY object and copies the content from src to it. + * \param src the source PAILLIER_KEY object + * \return newly created PAILLIER_KEY object or NULL if an error occurred. + */ +PAILLIER_KEY *PAILLIER_KEY_dup(PAILLIER_KEY *key) +{ + PAILLIER_KEY *ret = NULL; + + if (key == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ret = OPENSSL_zalloc(sizeof(*ret)); + if (ret == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if ((ret->p = BN_dup(key->p)) == NULL + || (ret->q = BN_dup(key->q)) == NULL + || (ret->n = BN_dup(key->n)) == NULL + || (ret->n_square = BN_dup(key->n_square)) == NULL + || (ret->g = BN_dup(key->g)) == NULL + || (ret->lambda = BN_dup(key->lambda)) == NULL + || (ret->u = BN_dup(key->u)) == NULL) { + OPENSSL_free(ret); + return NULL; + } + + return ret; +} + +/** Increases the internal reference count of a PAILLIER_KEY object. + * \param key PAILLIER_KEY object + * \return 1 on success and 0 if an error occurred. + */ +int PAILLIER_KEY_up_ref(PAILLIER_KEY *key) +{ + int i; + + if (CRYPTO_UP_REF(&key->references, &i, key->lock) <= 0) + return 0; + + REF_PRINT_COUNT("PAILLIER_KEY", key); + REF_ASSERT_ISNT(i < 2); + return ((i > 1) ? 1 : 0); +} + +/** Creates a new paillier private (and optional a new public) key. + * \param key PAILLIER_KEY object + * \param bits use BN_generate_prime_ex() to generate a pseudo-random prime number + * of bit length + * \return 1 on success and 0 if an error occurred. + */ +int PAILLIER_KEY_generate_key(PAILLIER_KEY *key, int bits) +{ + int ret = 0; + BIGNUM *p, *q, *g_exp_lambda; + BN_CTX *bn_ctx = NULL; + + if (key == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + if (bits <= 0 || bits > OPENSSL_PAILLIER_MAX_MODULUS_BITS) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_INVALID_ARGUMENT); + return ret; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + + BN_CTX_start(bn_ctx); + p = BN_CTX_get(bn_ctx); + q = BN_CTX_get(bn_ctx); + g_exp_lambda = BN_CTX_get(bn_ctx); + if (g_exp_lambda == NULL) + goto err; + + if (!BN_generate_prime_ex(p, bits, 1, NULL, NULL, NULL) + || !BN_generate_prime_ex(q, bits, 1, NULL, NULL, NULL) + || !BN_mul(key->n, p, q, bn_ctx) + || !BN_sqr(key->n_square, key->n, bn_ctx) + || !BN_add(key->g, key->n, BN_value_one()) + || !paillier_g_check(key->g, key->n_square, bn_ctx) + || !paillier_lambda_calc(key->lambda, p, q, bn_ctx) + || !BN_mod_exp(g_exp_lambda, key->g, key->lambda, key->n_square, bn_ctx) + || !paillier_l_func(key->u, g_exp_lambda, key->n, bn_ctx) + || !BN_mod_inverse(key->u, key->u, key->n, bn_ctx) + || (key->p = BN_dup(p)) == NULL + || (key->q = BN_dup(q)) == NULL) + goto err; + + key->version = PAILLIER_ASN1_VERSION_DEFAULT; + key->flag |= PAILLIER_FLAG_G_OPTIMIZE; + ret = 1; +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/** Returns the type of the PAILLIER_KEY. + * \param key PAILLIER_KEY object + * \return PAILLIER_KEY_TYPE_PRIVATE or PAILLIER_KEY_TYPE_PUBLIC. + */ +int PAILLIER_KEY_type(PAILLIER_KEY *key) +{ + if (key != NULL && key->p != NULL && key->q != NULL + && key->lambda != NULL && key->u != NULL) + return PAILLIER_KEY_TYPE_PRIVATE; + + return PAILLIER_KEY_TYPE_PUBLIC; +} + +int paillier_g_check(BIGNUM *g, BIGNUM *n_square, BN_CTX *ctx) +{ + int ret = 0; + BIGNUM *gcd; + BN_CTX *bn_ctx = NULL; + + if (g == NULL || n_square == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + if (ctx == NULL) { + bn_ctx = ctx = BN_CTX_new(); + if (ctx == NULL) + goto err; + } + + BN_CTX_start(ctx); + gcd = BN_CTX_get(ctx); + if (gcd == NULL) + goto err; + + if (!BN_gcd(gcd, g, n_square, ctx)) + goto err; + + ret = BN_is_one(gcd); + +err: + BN_CTX_end(ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/* + * lambda calc + */ +int paillier_lambda_calc(BIGNUM *out, BIGNUM *p, BIGNUM *q, BN_CTX *ctx) +{ + int ret = 0; + BIGNUM *gcd, *p_1, *q_1, *pq, *lambda; + BN_CTX *bn_ctx = NULL; + + if (p == NULL || q == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + if (ctx == NULL) { + bn_ctx = ctx = BN_CTX_new(); + if (ctx == NULL) + goto err; + } + + BN_CTX_start(ctx); + lambda = out ? out : BN_CTX_get(ctx); + gcd = BN_CTX_get(ctx); + p_1 = BN_CTX_get(ctx); + q_1 = BN_CTX_get(ctx); + pq = BN_CTX_get(ctx); + if (pq == NULL) + goto err; + + /* p_1 = p - 1 */ + if (!BN_sub(p_1, p, BN_value_one())) + goto err; + + /* q_1 = q - 1 */ + if (!BN_sub(q_1, q, BN_value_one())) + goto err; + + /* gcd = gcd(p - 1, q - 1) */ + if (!BN_gcd(gcd, p_1, q_1, ctx)) + goto err; + + /* pq = (p - 1) * (q - 1) */ + if (!BN_mul(pq, p_1, q_1, ctx)) + goto err; + + /* lambda = (p - 1) * (q - 1) / gcd */ + if (!BN_div(lambda, NULL, pq, gcd, ctx)) + goto err; + + if (!paillier_lambda_check(lambda, pq, ctx)) + goto err; + + ret = 1; + +err: + BN_CTX_end(ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/* + * lambda check + */ +int paillier_lambda_check(BIGNUM *lambda, BIGNUM *n, BN_CTX *ctx) +{ + int ret = 0; + BIGNUM *n_square, *n_lambda, *r, *r_exp_lambda, *r_exp_n_lambda; + BN_CTX *bn_ctx = NULL; + + if (lambda == NULL || n == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + if (ctx == NULL) { + bn_ctx = ctx = BN_CTX_new(); + if (ctx == NULL) + goto err; + } + + BN_CTX_start(ctx); + n_square = BN_CTX_get(ctx); + n_lambda = BN_CTX_get(ctx); + r = BN_CTX_get(ctx); + r_exp_lambda = BN_CTX_get(ctx); + r_exp_n_lambda = BN_CTX_get(ctx); + if (r_exp_n_lambda == NULL) + goto err; + + /* n_square = n ^ 2 */ + if (!BN_sqr(n_square, n, ctx)) + goto err; + + /* n_lambda = n * lambda */ + if (!BN_mul(n_lambda, n, lambda, ctx)) + goto err; + + if (!BN_rand_range(r, n)) + goto err; + + /* r = r * n */ + if (!BN_mul(r, r, n, ctx)) + goto err; + + /* r = r - 1 */ + if (!BN_sub_word(r, 1)) + goto err; + + /* r_exp_lambda = (r ^ lambda) mod n */ + if (!BN_mod_exp(r_exp_lambda, r, lambda, n, ctx)) + goto err; + + /* r_exp_n_lambda = (r ^ n_lambda) mod n_square */ + if (!BN_mod_exp(r_exp_n_lambda, r, n_lambda, n_square, ctx)) + goto err; + + ret = BN_cmp(r_exp_lambda, r_exp_n_lambda) == 0; + +err: + BN_CTX_end(ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/* + * The L function is used in decryption process and generate key. + * L(x) = (x-1)/n + * where x is an element of {x < n^2 | x = 1 mod n} + * n is our modulus (p*q) + */ +int paillier_l_func(BIGNUM *out, BIGNUM *x, BIGNUM *n, BN_CTX *ctx) +{ + int ret = 0; + BIGNUM *x_1; + BN_CTX *bn_ctx = NULL; + + if (out == NULL || x == NULL || n == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + if (ctx == NULL) { + bn_ctx = ctx = BN_CTX_new(); + if (ctx == NULL) + goto err; + } + + BN_CTX_start(ctx); + x_1 = BN_CTX_get(ctx); + if (x_1 == NULL) + goto err; + + /* x_1 = x - 1 */ + if (!BN_sub(x_1, x, BN_value_one())) + goto err; + + /* out = (x - 1) / n */ + if (!BN_div(out, NULL, x_1, n, ctx)) + goto err; + + ret = 1; + +err: + BN_CTX_end(ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +int ossl_paillier_multip_calc_product(PAILLIER_KEY *pail) +{ + /* TODO */ + return 0; +} diff --git a/openssl/src/crypto/paillier/paillier_local.h b/openssl/src/crypto/paillier/paillier_local.h new file mode 100644 index 000000000..d37307d82 --- /dev/null +++ b/openssl/src/crypto/paillier/paillier_local.h @@ -0,0 +1,63 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_PAILLIER_LOCAL_H +# define HEADER_PAILLIER_LOCAL_H + +# include + +# ifndef OPENSSL_NO_PAILLIER_LOCAL +# ifdef __cplusplus +extern "C" { +# endif + +# include +# if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_BN_METHOD) +# include +# endif +# include "internal/refcount.h" + +struct paillier_key_st { + int32_t version; + BIGNUM *p; + BIGNUM *q; + BIGNUM *n; + BIGNUM *n_square; + BIGNUM *g; + BIGNUM *lambda; + BIGNUM *u; + int32_t flag; + CRYPTO_REF_COUNT references; + CRYPTO_RWLOCK *lock; +}; + +struct paillier_ciphertext_st { + BIGNUM *data; +}; + +struct paillier_ctx_st { + PAILLIER_KEY *key; + BIGNUM *threshold; +# ifndef OPENSSL_NO_ENGINE + ENGINE *engine; +# endif +}; + +int paillier_g_check(BIGNUM *g, BIGNUM *n_square, BN_CTX *ctx); +int paillier_lambda_calc(BIGNUM *out, BIGNUM *p, BIGNUM *q, BN_CTX *ctx); +int paillier_lambda_check(BIGNUM *lambda, BIGNUM *n, BN_CTX *ctx); +int paillier_l_func(BIGNUM *out, BIGNUM *x, BIGNUM *n, BN_CTX *ctx); +int ossl_paillier_multip_calc_product(PAILLIER_KEY *pail); + +# ifdef __cplusplus +} +# endif +# endif + +#endif diff --git a/openssl/src/crypto/paillier/paillier_prn.c b/openssl/src/crypto/paillier/paillier_prn.c new file mode 100644 index 000000000..98bc67a20 --- /dev/null +++ b/openssl/src/crypto/paillier/paillier_prn.c @@ -0,0 +1,69 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include "internal/cryptlib.h" +#include +#include "paillier_local.h" + +#ifndef OPENSSL_NO_STDIO +int PAILLIER_KEY_print_fp(FILE *fp, const PAILLIER_KEY *key, int indent) +{ + BIO *b; + int ret; + + if ((b = BIO_new(BIO_s_file())) == NULL) { + ERR_raise(ERR_LIB_PAILLIER, ERR_R_BUF_LIB); + return 0; + } + BIO_set_fp(b, fp, BIO_NOCLOSE); + ret = PAILLIER_KEY_print(b, key, indent); + BIO_free(b); + return ret; +} +#endif + +int PAILLIER_KEY_print(BIO *bp, const PAILLIER_KEY *key, int indent) +{ + int ret = 0, is_pub = 1; + + if (key == NULL) + return 0; + + if (key->p && key->q && key->lambda && key->u) { + is_pub = 0; + BIO_printf(bp, "Paillier Private Key: \n"); + } else { + BIO_printf(bp, "Paillier Public Key: \n"); + } + + if (!ASN1_bn_print(bp, "n:", key->n, NULL, indent)) + goto end; + + if (!ASN1_bn_print(bp, "g:", key->g, NULL, indent)) + goto end; + + if (!is_pub) { + if (!ASN1_bn_print(bp, "p:", key->p, NULL, indent)) + goto end; + + if (!ASN1_bn_print(bp, "q:", key->q, NULL, indent)) + goto end; + + if (!ASN1_bn_print(bp, "lambda:", key->lambda, NULL, indent)) + goto end; + + if (!ASN1_bn_print(bp, "u:", key->u, NULL, indent)) + goto end; + } + + ret = 1; +end: + return ret; +} diff --git a/openssl/src/crypto/param_build.c b/openssl/src/crypto/param_build.c index 329404515..eaece0026 100644 --- a/openssl/src/crypto/param_build.c +++ b/openssl/src/crypto/param_build.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -32,7 +32,7 @@ typedef struct { union { /* * These fields are never directly addressed, but their sizes are - * important so that all native types can be copied here without overrun. + * imporant so that all native types can be copied here without overrun. */ ossl_intmax_t i; ossl_uintmax_t u; @@ -49,13 +49,15 @@ struct ossl_param_bld_st { }; static OSSL_PARAM_BLD_DEF *param_push(OSSL_PARAM_BLD *bld, const char *key, - size_t size, size_t alloc, int type, + int size, size_t alloc, int type, int secure) { OSSL_PARAM_BLD_DEF *pd = OPENSSL_zalloc(sizeof(*pd)); - if (pd == NULL) + if (pd == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return NULL; + } pd->key = key; pd->type = type; pd->size = size; @@ -76,10 +78,8 @@ static int param_push_num(OSSL_PARAM_BLD *bld, const char *key, { OSSL_PARAM_BLD_DEF *pd = param_push(bld, key, size, size, type, 0); - if (pd == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); + if (pd == NULL) return 0; - } if (size > sizeof(pd->num)) { ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_TOO_MANY_BYTES); return 0; @@ -190,20 +190,23 @@ int OSSL_PARAM_BLD_push_double(OSSL_PARAM_BLD *bld, const char *key, return param_push_num(bld, key, &num, sizeof(num), OSSL_PARAM_REAL); } -static int push_BN(OSSL_PARAM_BLD *bld, const char *key, - const BIGNUM *bn, size_t sz, int type) +int OSSL_PARAM_BLD_push_BN(OSSL_PARAM_BLD *bld, const char *key, + const BIGNUM *bn) +{ + return OSSL_PARAM_BLD_push_BN_pad(bld, key, bn, + bn == NULL ? 0 : BN_num_bytes(bn)); +} + +int OSSL_PARAM_BLD_push_BN_pad(OSSL_PARAM_BLD *bld, const char *key, + const BIGNUM *bn, size_t sz) { int n, secure = 0; OSSL_PARAM_BLD_DEF *pd; - if (!ossl_assert(type == OSSL_PARAM_UNSIGNED_INTEGER - || type == OSSL_PARAM_INTEGER)) - return 0; - if (bn != NULL) { - if (type == OSSL_PARAM_UNSIGNED_INTEGER && BN_is_negative(bn)) { + if (BN_is_negative(bn)) { ERR_raise_data(ERR_LIB_CRYPTO, ERR_R_UNSUPPORTED, - "Negative big numbers are unsupported for OSSL_PARAM_UNSIGNED_INTEGER"); + "Negative big numbers are unsupported for OSSL_PARAM"); return 0; } @@ -218,45 +221,26 @@ static int push_BN(OSSL_PARAM_BLD *bld, const char *key, } if (BN_get_flags(bn, BN_FLG_SECURE) == BN_FLG_SECURE) secure = 1; - - /* The BIGNUM is zero, we must transfer at least one byte */ - if (sz == 0) - sz++; } - pd = param_push(bld, key, sz, sz, type, secure); + pd = param_push(bld, key, sz, sz, OSSL_PARAM_UNSIGNED_INTEGER, secure); if (pd == NULL) return 0; pd->bn = bn; return 1; } -int OSSL_PARAM_BLD_push_BN(OSSL_PARAM_BLD *bld, const char *key, - const BIGNUM *bn) -{ - if (bn != NULL && BN_is_negative(bn)) - return push_BN(bld, key, bn, BN_num_bytes(bn) + 1, - OSSL_PARAM_INTEGER); - return push_BN(bld, key, bn, bn == NULL ? 0 : BN_num_bytes(bn), - OSSL_PARAM_UNSIGNED_INTEGER); -} - -int OSSL_PARAM_BLD_push_BN_pad(OSSL_PARAM_BLD *bld, const char *key, - const BIGNUM *bn, size_t sz) -{ - if (bn != NULL && BN_is_negative(bn)) - return push_BN(bld, key, bn, BN_num_bytes(bn), - OSSL_PARAM_INTEGER); - return push_BN(bld, key, bn, sz, OSSL_PARAM_UNSIGNED_INTEGER); -} - int OSSL_PARAM_BLD_push_utf8_string(OSSL_PARAM_BLD *bld, const char *key, const char *buf, size_t bsize) { OSSL_PARAM_BLD_DEF *pd; int secure; - if (bsize == 0) + if (bsize == 0) { bsize = strlen(buf); + } else if (bsize > INT_MAX) { + ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_STRING_TOO_LONG); + return 0; + } secure = CRYPTO_secure_allocated(buf); pd = param_push(bld, key, bsize, bsize + 1, OSSL_PARAM_UTF8_STRING, secure); if (pd == NULL) @@ -270,8 +254,12 @@ int OSSL_PARAM_BLD_push_utf8_ptr(OSSL_PARAM_BLD *bld, const char *key, { OSSL_PARAM_BLD_DEF *pd; - if (bsize == 0) + if (bsize == 0) { bsize = strlen(buf); + } else if (bsize > INT_MAX) { + ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_STRING_TOO_LONG); + return 0; + } pd = param_push(bld, key, bsize, sizeof(buf), OSSL_PARAM_UTF8_PTR, 0); if (pd == NULL) return 0; @@ -285,6 +273,10 @@ int OSSL_PARAM_BLD_push_octet_string(OSSL_PARAM_BLD *bld, const char *key, OSSL_PARAM_BLD_DEF *pd; int secure; + if (bsize > INT_MAX) { + ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_STRING_TOO_LONG); + return 0; + } secure = CRYPTO_secure_allocated(buf); pd = param_push(bld, key, bsize, bsize, OSSL_PARAM_OCTET_STRING, secure); if (pd == NULL) @@ -298,6 +290,10 @@ int OSSL_PARAM_BLD_push_octet_ptr(OSSL_PARAM_BLD *bld, const char *key, { OSSL_PARAM_BLD_DEF *pd; + if (bsize > INT_MAX) { + ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_STRING_TOO_LONG); + return 0; + } pd = param_push(bld, key, bsize, sizeof(buf), OSSL_PARAM_OCTET_PTR, 0); if (pd == NULL) return 0; @@ -330,10 +326,7 @@ static OSSL_PARAM *param_bld_convert(OSSL_PARAM_BLD *bld, OSSL_PARAM *param, param[i].data = p; if (pd->bn != NULL) { /* BIGNUM */ - if (pd->type == OSSL_PARAM_UNSIGNED_INTEGER) - BN_bn2nativepad(pd->bn, (unsigned char *)p, pd->size); - else - BN_signed_bn2native(pd->bn, (unsigned char *)p, pd->size); + BN_bn2nativepad(pd->bn, (unsigned char *)p, pd->size); } else if (pd->type == OSSL_PARAM_OCTET_PTR || pd->type == OSSL_PARAM_UTF8_PTR) { /* PTR */ @@ -376,6 +369,7 @@ OSSL_PARAM *OSSL_PARAM_BLD_to_param(OSSL_PARAM_BLD *bld) } params = OPENSSL_malloc(total); if (params == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); OPENSSL_secure_free(s); return NULL; } diff --git a/openssl/src/crypto/param_build_set.c b/openssl/src/crypto/param_build_set.c index f205d1019..8b570ded9 100644 --- a/openssl/src/crypto/param_build_set.c +++ b/openssl/src/crypto/param_build_set.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -73,10 +73,8 @@ int ossl_param_build_set_bn_pad(OSSL_PARAM_BLD *bld, OSSL_PARAM *p, return OSSL_PARAM_BLD_push_BN_pad(bld, key, bn, sz); p = OSSL_PARAM_locate(p, key); if (p != NULL) { - if (sz > p->data_size) { - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_TOO_SMALL_BUFFER); + if (sz > p->data_size) return 0; - } p->data_size = sz; return OSSL_PARAM_set_BN(p, bn); } @@ -101,22 +99,21 @@ int ossl_param_build_set_multi_key_bn(OSSL_PARAM_BLD *bld, OSSL_PARAM *params, { int i, sz = sk_BIGNUM_const_num(stk); OSSL_PARAM *p; - const BIGNUM *bn; + if (bld != NULL) { for (i = 0; i < sz && names[i] != NULL; ++i) { - bn = sk_BIGNUM_const_value(stk, i); - if (bn != NULL && !OSSL_PARAM_BLD_push_BN(bld, names[i], bn)) + if (!OSSL_PARAM_BLD_push_BN(bld, names[i], + sk_BIGNUM_const_value(stk, i))) return 0; } return 1; } for (i = 0; i < sz && names[i] != NULL; ++i) { - bn = sk_BIGNUM_const_value(stk, i); p = OSSL_PARAM_locate(params, names[i]); - if (p != NULL && bn != NULL) { - if (!OSSL_PARAM_set_BN(p, bn)) + if (p != NULL) { + if (!OSSL_PARAM_set_BN(p, sk_BIGNUM_const_value(stk, i))) return 0; } } diff --git a/openssl/src/crypto/params.c b/openssl/src/crypto/params.c index c109cabd4..9049041e3 100644 --- a/openssl/src/crypto/params.c +++ b/openssl/src/crypto/params.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -10,35 +10,10 @@ #include #include -#include #include "internal/thread_once.h" #include "internal/numbers.h" #include "internal/endian.h" -#include "internal/params.h" -#include "internal/packet.h" - -/* Shortcuts for raising errors that are widely used */ -#define err_unsigned_negative \ - ERR_raise(ERR_LIB_CRYPTO, \ - CRYPTO_R_PARAM_UNSIGNED_INTEGER_NEGATIVE_VALUE_UNSUPPORTED) -#define err_out_of_range \ - ERR_raise(ERR_LIB_CRYPTO, \ - CRYPTO_R_PARAM_VALUE_TOO_LARGE_FOR_DESTINATION) -#define err_inexact \ - ERR_raise(ERR_LIB_CRYPTO, \ - CRYPTO_R_PARAM_CANNOT_BE_REPRESENTED_EXACTLY) -#define err_not_integer \ - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_PARAM_NOT_INTEGER_TYPE) -#define err_too_small \ - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_TOO_SMALL_BUFFER) -#define err_bad_type \ - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_PARAM_OF_INCOMPATIBLE_TYPE) -#define err_null_argument \ - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER) -#define err_unsupported_real \ - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_PARAM_UNSUPPORTED_FLOATING_POINT_FORMAT) - -#ifndef OPENSSL_SYS_UEFI + /* * Return the number of bits in the mantissa of a double. This is used to * shift a larger integral value to determine if it will exactly fit into a @@ -48,7 +23,6 @@ static unsigned int real_shift(void) { return sizeof(double) == 4 ? 24 : 53; } -#endif OSSL_PARAM *OSSL_PARAM_locate(OSSL_PARAM *p, const char *key) { @@ -133,10 +107,8 @@ static int copy_integer(unsigned char *dest, size_t dest_len, * Shortening a signed value must retain the correct sign. * Avoiding this kind of thing: -253 = 0xff03 -> 0x03 = 3 */ - || (signed_int && ((pad ^ src[n]) & 0x80) != 0)) { - err_out_of_range; + || (signed_int && ((pad ^ src[n]) & 0x80) != 0)) return 0; - } memcpy(dest, src + n, dest_len); } } else /* IS_LITTLE_ENDIAN */ { @@ -151,10 +123,8 @@ static int copy_integer(unsigned char *dest, size_t dest_len, * Shortening a signed value must retain the correct sign. * Avoiding this kind of thing: 130 = 0x0082 -> 0x82 = -126 */ - || (signed_int && ((pad ^ src[dest_len - 1]) & 0x80) != 0)) { - err_out_of_range; + || (signed_int && ((pad ^ src[dest_len - 1]) & 0x80) != 0)) return 0; - } memcpy(dest, src, dest_len); } } @@ -180,10 +150,8 @@ static int signed_from_unsigned(void *dest, size_t dest_len, static int unsigned_from_signed(void *dest, size_t dest_len, const void *src, size_t src_len) { - if (is_negative(src, src_len)) { - err_unsigned_negative; + if (is_negative(src, src_len)) return 0; - } return copy_integer(dest, dest_len, src, src_len, 0, 0); } @@ -197,15 +165,10 @@ static int unsigned_from_unsigned(void *dest, size_t dest_len, /* General purpose get integer parameter call that handles odd sizes */ static int general_get_int(const OSSL_PARAM *p, void *val, size_t val_size) { - if (p->data == NULL) { - err_null_argument; - return 0; - } if (p->data_type == OSSL_PARAM_INTEGER) return signed_from_signed(val, val_size, p->data, p->data_size); if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) return signed_from_unsigned(val, val_size, p->data, p->data_size); - err_not_integer; return 0; } @@ -221,8 +184,6 @@ static int general_set_int(OSSL_PARAM *p, void *val, size_t val_size) r = signed_from_signed(p->data, p->data_size, val, val_size); else if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) r = unsigned_from_signed(p->data, p->data_size, val, val_size); - else - err_not_integer; p->return_size = r ? p->data_size : val_size; return r; } @@ -230,16 +191,10 @@ static int general_set_int(OSSL_PARAM *p, void *val, size_t val_size) /* General purpose get unsigned integer parameter call that handles odd sizes */ static int general_get_uint(const OSSL_PARAM *p, void *val, size_t val_size) { - - if (p->data == NULL) { - err_null_argument; - return 0; - } if (p->data_type == OSSL_PARAM_INTEGER) return unsigned_from_signed(val, val_size, p->data, p->data_size); if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) return unsigned_from_unsigned(val, val_size, p->data, p->data_size); - err_not_integer; return 0; } @@ -255,8 +210,6 @@ static int general_set_uint(OSSL_PARAM *p, void *val, size_t val_size) r = signed_from_unsigned(p->data, p->data_size, val, val_size); else if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) r = unsigned_from_unsigned(p->data, p->data_size, val, val_size); - else - err_not_integer; p->return_size = r ? p->data_size : val_size; return r; } @@ -389,15 +342,10 @@ OSSL_PARAM OSSL_PARAM_construct_ulong(const char *key, unsigned long int *buf) int OSSL_PARAM_get_int32(const OSSL_PARAM *p, int32_t *val) { - if (val == NULL || p == NULL) { - err_null_argument; - return 0; - } + double d; - if (p->data == NULL) { - err_null_argument; + if (val == NULL || p == NULL ) return 0; - } if (p->data_type == OSSL_PARAM_INTEGER) { #ifndef OPENSSL_SMALL_FOOTPRINT @@ -413,7 +361,6 @@ int OSSL_PARAM_get_int32(const OSSL_PARAM *p, int32_t *val) *val = (int32_t)i64; return 1; } - err_out_of_range; return 0; } #endif @@ -431,7 +378,6 @@ int OSSL_PARAM_get_int32(const OSSL_PARAM *p, int32_t *val) *val = (int32_t)u32; return 1; } - err_out_of_range; return 0; case sizeof(uint64_t): u64 = *(const uint64_t *)p->data; @@ -439,16 +385,12 @@ int OSSL_PARAM_get_int32(const OSSL_PARAM *p, int32_t *val) *val = (int32_t)u64; return 1; } - err_out_of_range; return 0; } #endif return general_get_int(p, val, sizeof(*val)); } else if (p->data_type == OSSL_PARAM_REAL) { -#ifndef OPENSSL_SYS_UEFI - double d; - switch (p->data_size) { case sizeof(double): d = *(const double *)p->data; @@ -456,23 +398,16 @@ int OSSL_PARAM_get_int32(const OSSL_PARAM *p, int32_t *val) *val = (int32_t)d; return 1; } - err_out_of_range; - return 0; + break; } - err_unsupported_real; - return 0; -#endif } - err_bad_type; return 0; } int OSSL_PARAM_set_int32(OSSL_PARAM *p, int32_t val) { - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; if (p->data_type == OSSL_PARAM_INTEGER) { #ifndef OPENSSL_SMALL_FOOTPRINT @@ -507,31 +442,15 @@ int OSSL_PARAM_set_int32(OSSL_PARAM *p, int32_t val) #endif return general_set_int(p, &val, sizeof(val)); } else if (p->data_type == OSSL_PARAM_REAL) { -#ifndef OPENSSL_SYS_UEFI - uint32_t u32; - unsigned int shift; - p->return_size = sizeof(double); if (p->data == NULL) return 1; switch (p->data_size) { case sizeof(double): - shift = real_shift(); - if (shift < 8 * sizeof(val) - 1) { - u32 = val < 0 ? -val : val; - if ((u32 >> shift) != 0) { - err_inexact; - return 0; - } - } *(double *)p->data = (double)val; return 1; } - err_unsupported_real; - return 0; -#endif } - err_bad_type; return 0; } @@ -543,15 +462,10 @@ OSSL_PARAM OSSL_PARAM_construct_int32(const char *key, int32_t *buf) int OSSL_PARAM_get_uint32(const OSSL_PARAM *p, uint32_t *val) { - if (val == NULL || p == NULL) { - err_null_argument; - return 0; - } + double d; - if (p->data == NULL) { - err_null_argument; + if (val == NULL || p == NULL) return 0; - } if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) { #ifndef OPENSSL_SMALL_FOOTPRINT @@ -567,7 +481,6 @@ int OSSL_PARAM_get_uint32(const OSSL_PARAM *p, uint32_t *val) *val = (uint32_t)u64; return 1; } - err_out_of_range; return 0; } #endif @@ -584,7 +497,6 @@ int OSSL_PARAM_get_uint32(const OSSL_PARAM *p, uint32_t *val) *val = i32; return 1; } - err_unsigned_negative; return 0; case sizeof(int64_t): i64 = *(const int64_t *)p->data; @@ -592,18 +504,11 @@ int OSSL_PARAM_get_uint32(const OSSL_PARAM *p, uint32_t *val) *val = (uint32_t)i64; return 1; } - if (i64 < 0) - err_unsigned_negative; - else - err_out_of_range; return 0; } #endif return general_get_uint(p, val, sizeof(*val)); } else if (p->data_type == OSSL_PARAM_REAL) { -#ifndef OPENSSL_SYS_UEFI - double d; - switch (p->data_size) { case sizeof(double): d = *(const double *)p->data; @@ -611,23 +516,16 @@ int OSSL_PARAM_get_uint32(const OSSL_PARAM *p, uint32_t *val) *val = (uint32_t)d; return 1; } - err_inexact; - return 0; + break; } - err_unsupported_real; - return 0; -#endif } - err_bad_type; return 0; } int OSSL_PARAM_set_uint32(OSSL_PARAM *p, uint32_t val) { - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) { @@ -657,7 +555,6 @@ int OSSL_PARAM_set_uint32(OSSL_PARAM *p, uint32_t val) *(int32_t *)p->data = (int32_t)val; return 1; } - err_out_of_range; return 0; case sizeof(int64_t): p->return_size = sizeof(int64_t); @@ -667,27 +564,15 @@ int OSSL_PARAM_set_uint32(OSSL_PARAM *p, uint32_t val) #endif return general_set_uint(p, &val, sizeof(val)); } else if (p->data_type == OSSL_PARAM_REAL) { -#ifndef OPENSSL_SYS_UEFI - unsigned int shift; - p->return_size = sizeof(double); if (p->data == NULL) return 1; switch (p->data_size) { case sizeof(double): - shift = real_shift(); - if (shift < 8 * sizeof(val) && (val >> shift) != 0) { - err_inexact; - return 0; - } *(double *)p->data = (double)val; return 1; } - err_unsupported_real; - return 0; -#endif } - err_bad_type; return 0; } @@ -699,15 +584,10 @@ OSSL_PARAM OSSL_PARAM_construct_uint32(const char *key, uint32_t *buf) int OSSL_PARAM_get_int64(const OSSL_PARAM *p, int64_t *val) { - if (val == NULL || p == NULL) { - err_null_argument; - return 0; - } + double d; - if (p->data == NULL) { - err_null_argument; + if (val == NULL || p == NULL ) return 0; - } if (p->data_type == OSSL_PARAM_INTEGER) { #ifndef OPENSSL_SMALL_FOOTPRINT @@ -735,15 +615,11 @@ int OSSL_PARAM_get_int64(const OSSL_PARAM *p, int64_t *val) *val = (int64_t)u64; return 1; } - err_out_of_range; return 0; } #endif return general_get_int(p, val, sizeof(*val)); } else if (p->data_type == OSSL_PARAM_REAL) { -#ifndef OPENSSL_SYS_UEFI - double d; - switch (p->data_size) { case sizeof(double): d = *(const double *)p->data; @@ -758,23 +634,18 @@ int OSSL_PARAM_get_int64(const OSSL_PARAM *p, int64_t *val) *val = (int64_t)d; return 1; } - err_inexact; - return 0; + break; } - err_unsupported_real; - return 0; -#endif } - err_bad_type; return 0; } int OSSL_PARAM_set_int64(OSSL_PARAM *p, int64_t val) { - if (p == NULL) { - err_null_argument; + uint64_t u64; + + if (p == NULL) return 0; - } p->return_size = 0; if (p->data_type == OSSL_PARAM_INTEGER) { #ifndef OPENSSL_SMALL_FOOTPRINT @@ -788,7 +659,6 @@ int OSSL_PARAM_set_int64(OSSL_PARAM *p, int64_t val) *(int32_t *)p->data = (int32_t)val; return 1; } - err_out_of_range; return 0; case sizeof(int64_t): *(int64_t *)p->data = val; @@ -808,7 +678,6 @@ int OSSL_PARAM_set_int64(OSSL_PARAM *p, int64_t val) *(uint32_t *)p->data = (uint32_t)val; return 1; } - err_out_of_range; return 0; case sizeof(uint64_t): *(uint64_t *)p->data = (uint64_t)val; @@ -817,9 +686,6 @@ int OSSL_PARAM_set_int64(OSSL_PARAM *p, int64_t val) #endif return general_set_int(p, &val, sizeof(val)); } else if (p->data_type == OSSL_PARAM_REAL) { -#ifndef OPENSSL_SYS_UEFI - uint64_t u64; - p->return_size = sizeof(double); if (p->data == NULL) return 1; @@ -830,14 +696,9 @@ int OSSL_PARAM_set_int64(OSSL_PARAM *p, int64_t val) *(double *)p->data = (double)val; return 1; } - err_inexact; - return 0; + break; } - err_unsupported_real; - return 0; -#endif } - err_bad_type; return 0; } @@ -848,15 +709,10 @@ OSSL_PARAM OSSL_PARAM_construct_int64(const char *key, int64_t *buf) int OSSL_PARAM_get_uint64(const OSSL_PARAM *p, uint64_t *val) { - if (val == NULL || p == NULL) { - err_null_argument; - return 0; - } + double d; - if (p->data == NULL) { - err_null_argument; + if (val == NULL || p == NULL) return 0; - } if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) { #ifndef OPENSSL_SMALL_FOOTPRINT @@ -882,7 +738,6 @@ int OSSL_PARAM_get_uint64(const OSSL_PARAM *p, uint64_t *val) *val = (uint64_t)i32; return 1; } - err_unsigned_negative; return 0; case sizeof(int64_t): i64 = *(const int64_t *)p->data; @@ -890,15 +745,11 @@ int OSSL_PARAM_get_uint64(const OSSL_PARAM *p, uint64_t *val) *val = (uint64_t)i64; return 1; } - err_unsigned_negative; return 0; } #endif return general_get_uint(p, val, sizeof(*val)); } else if (p->data_type == OSSL_PARAM_REAL) { -#ifndef OPENSSL_SYS_UEFI - double d; - switch (p->data_size) { case sizeof(double): d = *(const double *)p->data; @@ -913,23 +764,16 @@ int OSSL_PARAM_get_uint64(const OSSL_PARAM *p, uint64_t *val) *val = (uint64_t)d; return 1; } - err_inexact; - return 0; + break; } - err_unsupported_real; - return 0; -#endif } - err_bad_type; return 0; } int OSSL_PARAM_set_uint64(OSSL_PARAM *p, uint64_t val) { - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) { @@ -944,7 +788,6 @@ int OSSL_PARAM_set_uint64(OSSL_PARAM *p, uint64_t val) *(uint32_t *)p->data = (uint32_t)val; return 1; } - err_out_of_range; return 0; case sizeof(uint64_t): *(uint64_t *)p->data = val; @@ -964,20 +807,17 @@ int OSSL_PARAM_set_uint64(OSSL_PARAM *p, uint64_t val) *(int32_t *)p->data = (int32_t)val; return 1; } - err_out_of_range; return 0; case sizeof(int64_t): if (val <= INT64_MAX) { *(int64_t *)p->data = (int64_t)val; return 1; } - err_out_of_range; return 0; } #endif return general_set_uint(p, &val, sizeof(val)); } else if (p->data_type == OSSL_PARAM_REAL) { -#ifndef OPENSSL_SYS_UEFI p->return_size = sizeof(double); switch (p->data_size) { case sizeof(double): @@ -985,14 +825,9 @@ int OSSL_PARAM_set_uint64(OSSL_PARAM *p, uint64_t val) *(double *)p->data = (double)val; return 1; } - err_inexact; - return 0; + break; } - err_unsupported_real; - return 0; -#endif } - err_bad_type; return 0; } @@ -1067,84 +902,43 @@ OSSL_PARAM OSSL_PARAM_construct_time_t(const char *key, time_t *buf) int OSSL_PARAM_get_BN(const OSSL_PARAM *p, BIGNUM **val) { - BIGNUM *b = NULL; + BIGNUM *b; - if (val == NULL || p == NULL || p->data == NULL) { - err_null_argument; + if (val == NULL + || p == NULL + || p->data_type != OSSL_PARAM_UNSIGNED_INTEGER) return 0; - } - switch (p->data_type) { - case OSSL_PARAM_UNSIGNED_INTEGER: - b = BN_native2bn(p->data, (int)p->data_size, *val); - break; - case OSSL_PARAM_INTEGER: - b = BN_signed_native2bn(p->data, (int)p->data_size, *val); - break; - default: - err_bad_type; - break; - } - - if (b == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_BN_LIB); - return 0; + b = BN_native2bn(p->data, (int)p->data_size, *val); + if (b != NULL) { + *val = b; + return 1; } - - *val = b; - return 1; + return 0; } int OSSL_PARAM_set_BN(OSSL_PARAM *p, const BIGNUM *val) { size_t bytes; - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; - if (val == NULL) { - err_null_argument; + if (val == NULL || p->data_type != OSSL_PARAM_UNSIGNED_INTEGER) return 0; - } - if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER && BN_is_negative(val)) { - err_bad_type; + + /* For the moment, only positive values are permitted */ + if (BN_is_negative(val)) return 0; - } bytes = (size_t)BN_num_bytes(val); - /* We add 1 byte for signed numbers, to make space for a sign extension */ - if (p->data_type == OSSL_PARAM_INTEGER) - bytes++; - /* We make sure that at least one byte is used, so zero is properly set */ - if (bytes == 0) - bytes++; - p->return_size = bytes; if (p->data == NULL) return 1; if (p->data_size >= bytes) { p->return_size = p->data_size; - - switch (p->data_type) { - case OSSL_PARAM_UNSIGNED_INTEGER: - if (BN_bn2nativepad(val, p->data, p->data_size) >= 0) - return 1; - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_INTEGER_OVERFLOW); - break; - case OSSL_PARAM_INTEGER: - if (BN_signed_bn2native(val, p->data, p->data_size) >= 0) - return 1; - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_INTEGER_OVERFLOW); - break; - default: - err_bad_type; - break; - } - return 0; + return BN_bn2nativepad(val, p->data, p->data_size) >= 0; } - err_too_small; return 0; } @@ -1155,16 +949,13 @@ OSSL_PARAM OSSL_PARAM_construct_BN(const char *key, unsigned char *buf, buf, bsize); } -#ifndef OPENSSL_SYS_UEFI int OSSL_PARAM_get_double(const OSSL_PARAM *p, double *val) { int64_t i64; uint64_t u64; - if (val == NULL || p == NULL || p->data == NULL) { - err_null_argument; + if (val == NULL || p == NULL) return 0; - } if (p->data_type == OSSL_PARAM_REAL) { switch (p->data_size) { @@ -1172,8 +963,6 @@ int OSSL_PARAM_get_double(const OSSL_PARAM *p, double *val) *val = *(const double *)p->data; return 1; } - err_unsupported_real; - return 0; } else if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) { switch (p->data_size) { case sizeof(uint32_t): @@ -1185,8 +974,7 @@ int OSSL_PARAM_get_double(const OSSL_PARAM *p, double *val) *val = (double)u64; return 1; } - err_inexact; - return 0; + break; } } else if (p->data_type == OSSL_PARAM_INTEGER) { switch (p->data_size) { @@ -1200,20 +988,16 @@ int OSSL_PARAM_get_double(const OSSL_PARAM *p, double *val) *val = 0.0 + i64; return 1; } - err_inexact; - return 0; + break; } } - err_bad_type; return 0; } int OSSL_PARAM_set_double(OSSL_PARAM *p, double val) { - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; if (p->data_type == OSSL_PARAM_REAL) { @@ -1225,16 +1009,11 @@ int OSSL_PARAM_set_double(OSSL_PARAM *p, double val) *(double *)p->data = val; return 1; } - err_unsupported_real; - return 0; - } else if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER) { + } else if (p->data_type == OSSL_PARAM_UNSIGNED_INTEGER + && val == (uint64_t)val) { p->return_size = sizeof(double); if (p->data == NULL) return 1; - if (val != (uint64_t)val) { - err_inexact; - return 0; - } switch (p->data_size) { case sizeof(uint32_t): if (val >= 0 && val <= UINT32_MAX) { @@ -1242,8 +1021,7 @@ int OSSL_PARAM_set_double(OSSL_PARAM *p, double val) *(uint32_t *)p->data = (uint32_t)val; return 1; } - err_out_of_range; - return 0; + break; case sizeof(uint64_t): if (val >= 0 /* @@ -1256,17 +1034,11 @@ int OSSL_PARAM_set_double(OSSL_PARAM *p, double val) *(uint64_t *)p->data = (uint64_t)val; return 1; } - err_out_of_range; - return 0; - } - } else if (p->data_type == OSSL_PARAM_INTEGER) { + break; } + } else if (p->data_type == OSSL_PARAM_INTEGER && val == (int64_t)val) { p->return_size = sizeof(double); if (p->data == NULL) return 1; - if (val != (int64_t)val) { - err_inexact; - return 0; - } switch (p->data_size) { case sizeof(int32_t): if (val >= INT32_MIN && val <= INT32_MAX) { @@ -1274,8 +1046,7 @@ int OSSL_PARAM_set_double(OSSL_PARAM *p, double val) *(int32_t *)p->data = (int32_t)val; return 1; } - err_out_of_range; - return 0; + break; case sizeof(int64_t): if (val >= INT64_MIN /* @@ -1288,11 +1059,9 @@ int OSSL_PARAM_set_double(OSSL_PARAM *p, double val) *(int64_t *)p->data = (int64_t)val; return 1; } - err_out_of_range; - return 0; + break; } } - err_bad_type; return 0; } @@ -1300,7 +1069,6 @@ OSSL_PARAM OSSL_PARAM_construct_double(const char *key, double *buf) { return ossl_param_construct(key, OSSL_PARAM_REAL, buf, sizeof(double)); } -#endif static int get_string_internal(const OSSL_PARAM *p, void **val, size_t *max_len, size_t *used_len, @@ -1308,14 +1076,8 @@ static int get_string_internal(const OSSL_PARAM *p, void **val, { size_t sz, alloc_sz; - if ((val == NULL && used_len == NULL) || p == NULL) { - err_null_argument; - return 0; - } - if (p->data_type != type) { - err_bad_type; + if ((val == NULL && used_len == NULL) || p == NULL || p->data_type != type) return 0; - } sz = p->data_size; /* @@ -1327,10 +1089,8 @@ static int get_string_internal(const OSSL_PARAM *p, void **val, if (used_len != NULL) *used_len = sz; - if (p->data == NULL) { - err_null_argument; + if (p->data == NULL) return 0; - } if (val == NULL) return 1; @@ -1344,10 +1104,8 @@ static int get_string_internal(const OSSL_PARAM *p, void **val, *max_len = alloc_sz; } - if (*max_len < sz) { - err_too_small; + if (*max_len < sz) return 0; - } memcpy(*val, p->data, sz); return 1; } @@ -1374,10 +1132,8 @@ int OSSL_PARAM_get_utf8_string(const OSSL_PARAM *p, char **val, size_t max_len) return 0; if (data_length >= max_len) data_length = OPENSSL_strnlen(p->data, data_length); - if (data_length >= max_len) { - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_NO_SPACE_FOR_TERMINATING_NULL); + if (data_length >= max_len) return 0; /* No space for a terminating NUL byte */ - } (*val)[data_length] = '\0'; return ret; @@ -1396,14 +1152,8 @@ static int set_string_internal(OSSL_PARAM *p, const void *val, size_t len, p->return_size = len; if (p->data == NULL) return 1; - if (p->data_type != type) { - err_bad_type; - return 0; - } - if (p->data_size < len) { - err_too_small; + if (p->data_type != type || p->data_size < len) return 0; - } memcpy(p->data, val, len); /* If possible within the size of p->data, add a NUL terminator byte */ @@ -1414,32 +1164,24 @@ static int set_string_internal(OSSL_PARAM *p, const void *val, size_t len, int OSSL_PARAM_set_utf8_string(OSSL_PARAM *p, const char *val) { - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; - if (val == NULL) { - err_null_argument; + if (val == NULL) return 0; - } return set_string_internal(p, val, strlen(val), OSSL_PARAM_UTF8_STRING); } int OSSL_PARAM_set_octet_string(OSSL_PARAM *p, const void *val, size_t len) { - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; - if (val == NULL) { - err_null_argument; + if (val == NULL) return 0; - } return set_string_internal(p, val, len, OSSL_PARAM_OCTET_STRING); } @@ -1460,14 +1202,8 @@ OSSL_PARAM OSSL_PARAM_construct_octet_string(const char *key, void *buf, static int get_ptr_internal(const OSSL_PARAM *p, const void **val, size_t *used_len, unsigned int type) { - if (val == NULL || p == NULL) { - err_null_argument; + if (val == NULL || p == NULL || p->data_type != type) return 0; - } - if (p->data_type != type) { - err_bad_type; - return 0; - } if (used_len != NULL) *used_len = p->data_size; *val = *(const void **)p->data; @@ -1489,10 +1225,8 @@ static int set_ptr_internal(OSSL_PARAM *p, const void *val, unsigned int type, size_t len) { p->return_size = len; - if (p->data_type != type) { - err_bad_type; + if (p->data_type != type) return 0; - } if (p->data != NULL) *(const void **)p->data = val; return 1; @@ -1500,10 +1234,8 @@ static int set_ptr_internal(OSSL_PARAM *p, const void *val, int OSSL_PARAM_set_utf8_ptr(OSSL_PARAM *p, const char *val) { - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; return set_ptr_internal(p, val, OSSL_PARAM_UTF8_PTR, val == NULL ? 0 : strlen(val)); @@ -1512,10 +1244,8 @@ int OSSL_PARAM_set_utf8_ptr(OSSL_PARAM *p, const char *val) int OSSL_PARAM_set_octet_ptr(OSSL_PARAM *p, const void *val, size_t used_len) { - if (p == NULL) { - err_null_argument; + if (p == NULL) return 0; - } p->return_size = 0; return set_ptr_internal(p, val, OSSL_PARAM_OCTET_PTR, used_len); } @@ -1532,111 +1262,6 @@ OSSL_PARAM OSSL_PARAM_construct_octet_ptr(const char *key, void **buf, return ossl_param_construct(key, OSSL_PARAM_OCTET_PTR, buf, bsize); } -/* - * Extract the parameter into an allocated buffer. - * Any existing allocation in *out is cleared and freed. - * - * Returns 1 on success, 0 on failure and -1 if there are no matching params. - * - * *out and *out_len are guaranteed to be untouched if this function - * doesn't return success. - */ -int ossl_param_get1_octet_string(const OSSL_PARAM *params, const char *name, - unsigned char **out, size_t *out_len) -{ - const OSSL_PARAM *p = OSSL_PARAM_locate_const(params, name); - void *buf = NULL; - size_t len = 0; - - if (p == NULL) - return -1; - - if (p->data != NULL - && p->data_size > 0 - && !OSSL_PARAM_get_octet_string(p, &buf, 0, &len)) - return 0; - - OPENSSL_clear_free(*out, *out_len); - *out = buf; - *out_len = len; - return 1; -} - -static int setbuf_fromparams(const OSSL_PARAM *p, const char *name, - unsigned char *out, size_t *outlen) -{ - int ret = 0; - WPACKET pkt; - - if (out == NULL) { - if (!WPACKET_init_null(&pkt, 0)) - return 0; - } else { - if (!WPACKET_init_static_len(&pkt, out, *outlen, 0)) - return 0; - } - - for (; p != NULL; p = OSSL_PARAM_locate_const(p + 1, name)) { - if (p->data_type != OSSL_PARAM_OCTET_STRING) - goto err; - if (p->data != NULL - && p->data_size != 0 - && !WPACKET_memcpy(&pkt, p->data, p->data_size)) - goto err; - } - if (!WPACKET_get_total_written(&pkt, outlen) - || !WPACKET_finish(&pkt)) - goto err; - ret = 1; -err: - WPACKET_cleanup(&pkt); - return ret; -} - -int ossl_param_get1_concat_octet_string(const OSSL_PARAM *params, const char *name, - unsigned char **out, - size_t *out_len, size_t maxsize) -{ - const OSSL_PARAM *p = OSSL_PARAM_locate_const(params, name); - unsigned char *res; - size_t sz = 0; - - if (p == NULL) - return -1; - - /* Calculate the total size */ - if (!setbuf_fromparams(p, name, NULL, &sz)) - return 0; - - /* Check that it's not oversized */ - if (maxsize > 0 && sz > maxsize) - return 0; - - /* Special case zero length */ - if (sz == 0) { - if ((res = OPENSSL_zalloc(1)) == NULL) - return 0; - goto fin; - } - - /* Allocate the buffer */ - res = OPENSSL_malloc(sz); - if (res == NULL) - return 0; - - /* Concat one or more OSSL_KDF_PARAM_INFO fields */ - if (!setbuf_fromparams(p, name, res, &sz)) { - OPENSSL_clear_free(res, sz); - return 0; - } - - fin: - OPENSSL_clear_free(*out, *out_len); - *out = res; - *out_len = sz; - return 1; -} - OSSL_PARAM OSSL_PARAM_construct_end(void) { OSSL_PARAM end = OSSL_PARAM_END; @@ -1647,14 +1272,8 @@ OSSL_PARAM OSSL_PARAM_construct_end(void) static int get_string_ptr_internal(const OSSL_PARAM *p, const void **val, size_t *used_len, unsigned int type) { - if (val == NULL || p == NULL) { - err_null_argument; - return 0; - } - if (p->data_type != type) { - err_bad_type; + if (val == NULL || p == NULL || p->data_type != type) return 0; - } if (used_len != NULL) *used_len = p->data_size; *val = p->data; @@ -1663,25 +1282,14 @@ static int get_string_ptr_internal(const OSSL_PARAM *p, const void **val, int OSSL_PARAM_get_utf8_string_ptr(const OSSL_PARAM *p, const char **val) { - int rv; - - ERR_set_mark(); - rv = OSSL_PARAM_get_utf8_ptr(p, val); - ERR_pop_to_mark(); - - return rv || get_string_ptr_internal(p, (const void **)val, NULL, - OSSL_PARAM_UTF8_STRING); + return OSSL_PARAM_get_utf8_ptr(p, val) + || get_string_ptr_internal(p, (const void **)val, NULL, + OSSL_PARAM_UTF8_STRING); } int OSSL_PARAM_get_octet_string_ptr(const OSSL_PARAM *p, const void **val, size_t *used_len) { - int rv; - - ERR_set_mark(); - rv = OSSL_PARAM_get_octet_ptr(p, val, used_len); - ERR_pop_to_mark(); - - return rv || get_string_ptr_internal(p, val, used_len, - OSSL_PARAM_OCTET_STRING); + return OSSL_PARAM_get_octet_ptr(p, val, used_len) + || get_string_ptr_internal(p, val, used_len, OSSL_PARAM_OCTET_STRING); } diff --git a/openssl/src/crypto/params_dup.c b/openssl/src/crypto/params_dup.c index 769629bbf..bc1546fc5 100644 --- a/openssl/src/crypto/params_dup.c +++ b/openssl/src/crypto/params_dup.c @@ -37,8 +37,11 @@ static int ossl_param_buf_alloc(OSSL_PARAM_BUF *out, size_t extra_blocks, size_t sz = OSSL_PARAM_ALIGN_SIZE * (extra_blocks + out->blocks); out->alloc = is_secure ? OPENSSL_secure_zalloc(sz) : OPENSSL_zalloc(sz); - if (out->alloc == NULL) + if (out->alloc == NULL) { + ERR_raise(ERR_LIB_CRYPTO, is_secure ? CRYPTO_R_SECURE_MALLOC_FAILURE + : ERR_R_MALLOC_FAILURE); return 0; + } out->alloc_sz = sz; out->cur = out->alloc + extra_blocks; return 1; @@ -102,10 +105,8 @@ OSSL_PARAM *OSSL_PARAM_dup(const OSSL_PARAM *src) OSSL_PARAM *last, *dst; int param_count = 1; /* Include terminator in the count */ - if (src == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); + if (src == NULL) return NULL; - } memset(buf, 0, sizeof(buf)); @@ -153,10 +154,8 @@ OSSL_PARAM *OSSL_PARAM_merge(const OSSL_PARAM *p1, const OSSL_PARAM *p2) size_t list1_sz = 0, list2_sz = 0; int diff; - if (p1 == NULL && p2 == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); + if (p1 == NULL && p2 == NULL) return NULL; - } /* Copy p1 to list1 */ if (p1 != NULL) { @@ -171,10 +170,8 @@ OSSL_PARAM *OSSL_PARAM_merge(const OSSL_PARAM *p1, const OSSL_PARAM *p2) list2[list2_sz++] = p; } list2[list2_sz] = NULL; - if (list1_sz == 0 && list2_sz == 0) { - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_NO_PARAMS_TO_MERGE); + if (list1_sz == 0 && list2_sz == 0) return NULL; - } /* Sort the 2 lists */ qsort(list1, list1_sz, sizeof(OSSL_PARAM *), compare_params); @@ -182,8 +179,10 @@ OSSL_PARAM *OSSL_PARAM_merge(const OSSL_PARAM *p1, const OSSL_PARAM *p2) /* Allocate enough space to store the merged parameters */ params = OPENSSL_zalloc((list1_sz + list2_sz + 1) * sizeof(*p1)); - if (params == NULL) + if (params == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return NULL; + } dst = params; p1cur = list1; p2cur = list2; diff --git a/openssl/src/crypto/params_from_text.c b/openssl/src/crypto/params_from_text.c index e603dfec6..360f8933e 100644 --- a/openssl/src/crypto/params_from_text.c +++ b/openssl/src/crypto/params_from_text.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -8,7 +8,7 @@ * https://www.openssl.org/source/license.html */ -#include "internal/common.h" /* for HAS_PREFIX */ +#include #include #include #include @@ -35,7 +35,10 @@ static int prepare_from_text(const OSSL_PARAM *paramdefs, const char *key, * ishex is used to translate legacy style string controls in hex format * to octet string parameters. */ - *ishex = CHECK_AND_SKIP_PREFIX(key, "hex"); + *ishex = strncmp(key, "hex", 3) == 0; + + if (*ishex) + key += 3; p = *paramdef = OSSL_PARAM_locate_const(paramdefs, key); if (found != NULL) @@ -115,13 +118,7 @@ static int prepare_from_text(const OSSL_PARAM *paramdefs, const char *key, break; case OSSL_PARAM_OCTET_STRING: if (*ishex) { - size_t hexdigits = strlen(value); - if ((hexdigits % 2) != 0) { - /* We don't accept an odd number of hex digits */ - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_ODD_NUMBER_OF_DIGITS); - return 0; - } - *buf_n = hexdigits >> 1; + *buf_n = strlen(value) >> 1; } else { *buf_n = value_n; } @@ -216,8 +213,10 @@ int OSSL_PARAM_allocate_from_text(OSSL_PARAM *to, ¶mdef, &ishex, &buf_n, &tmpbn, found)) goto err; - if ((buf = OPENSSL_zalloc(buf_n > 0 ? buf_n : 1)) == NULL) + if ((buf = OPENSSL_zalloc(buf_n > 0 ? buf_n : 1)) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto err; + } ok = construct_from_text(to, paramdef, value, value_n, ishex, buf, buf_n, tmpbn); diff --git a/openssl/src/crypto/passphrase.c b/openssl/src/crypto/passphrase.c index 563c5acd1..fcc40f6da 100644 --- a/openssl/src/crypto/passphrase.c +++ b/openssl/src/crypto/passphrase.c @@ -43,8 +43,10 @@ int ossl_pw_set_passphrase(struct ossl_passphrase_data_st *data, data->_.expl_passphrase.passphrase_copy = passphrase_len != 0 ? OPENSSL_memdup(passphrase, passphrase_len) : OPENSSL_malloc(1); - if (data->_.expl_passphrase.passphrase_copy == NULL) + if (data->_.expl_passphrase.passphrase_copy == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } data->_.expl_passphrase.passphrase_len = passphrase_len; return 1; } @@ -128,7 +130,7 @@ static int do_ui_passphrase(char *pass, size_t pass_size, size_t *pass_len, } if ((ui = UI_new()) == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_UI_LIB); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; } @@ -141,14 +143,16 @@ static int do_ui_passphrase(char *pass, size_t pass_size, size_t *pass_len, /* Get an application constructed prompt */ prompt = UI_construct_prompt(ui, "pass phrase", prompt_info); if (prompt == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_UI_LIB); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto end; } /* Get a buffer for verification prompt */ ipass = OPENSSL_zalloc(pass_size + 1); - if (ipass == NULL) + if (ipass == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto end; + } prompt_idx = UI_add_input_string(ui, prompt, UI_INPUT_FLAG_DEFAULT_PWD, @@ -161,8 +165,10 @@ static int do_ui_passphrase(char *pass, size_t pass_size, size_t *pass_len, if (verify) { /* Get a buffer for verification prompt */ vpass = OPENSSL_zalloc(pass_size + 1); - if (vpass == NULL) + if (vpass == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto end; + } verify_idx = UI_add_verify_string(ui, prompt, UI_INPUT_FLAG_DEFAULT_PWD, vpass, 0, pass_size, @@ -263,7 +269,7 @@ int ossl_pw_get_passphrase(char *pass, size_t pass_size, size_t *pass_len, ui_data = data->_.pem_password.password_cbarg; if (ui_method == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_UI_LIB); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; } } else if (data->type == is_ui_method) { @@ -293,6 +299,7 @@ int ossl_pw_get_passphrase(char *pass, size_t pass_size, size_t *pass_len, if (new_cache == NULL) { OPENSSL_cleanse(pass, *pass_len); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; } data->cached_passphrase = new_cache; diff --git a/openssl/src/crypto/pem/local.h b/openssl/src/crypto/pem/local.h new file mode 100644 index 000000000..51a42fb99 --- /dev/null +++ b/openssl/src/crypto/pem/local.h @@ -0,0 +1,15 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +int pem_check_suffix(const char *pem_str, const char *suffix); +int ossl_pem_check_suffix(const char *pem_str, const char *suffix); diff --git a/openssl/src/crypto/pem/pem_info.c b/openssl/src/crypto/pem/pem_info.c index f8dc4416e..061c9b9f6 100644 --- a/openssl/src/crypto/pem/pem_info.c +++ b/openssl/src/crypto/pem/pem_info.c @@ -67,7 +67,7 @@ STACK_OF(X509_INFO) *PEM_X509_INFO_read_bio_ex(BIO *bp, STACK_OF(X509_INFO) *sk, if (sk == NULL) { if ((ret = sk_X509_INFO_new_null()) == NULL) { - ERR_raise(ERR_LIB_PEM, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; } } else diff --git a/openssl/src/crypto/pem/pem_lib.c b/openssl/src/crypto/pem/pem_lib.c index 9d8ad35ad..b81ef2185 100644 --- a/openssl/src/crypto/pem/pem_lib.c +++ b/openssl/src/crypto/pem/pem_lib.c @@ -218,25 +218,18 @@ static int check_pem(const char *nm, const char *name) return 0; } -#define PEM_FREE(p, flags, num) \ - pem_free((p), (flags), (num), OPENSSL_FILE, OPENSSL_LINE) -static void pem_free(void *p, unsigned int flags, size_t num, - const char *file, int line) +static void pem_free(void *p, unsigned int flags, size_t num) { if (flags & PEM_FLAG_SECURE) - CRYPTO_secure_clear_free(p, num, file, line); + OPENSSL_secure_clear_free(p, num); else - CRYPTO_free(p, file, line); + OPENSSL_free(p); } -#define PEM_MALLOC(num, flags) \ - pem_malloc((num), (flags), OPENSSL_FILE, OPENSSL_LINE) -static void *pem_malloc(int num, unsigned int flags, - const char *file, int line) +static void *pem_malloc(int num, unsigned int flags) { - return (flags & PEM_FLAG_SECURE) ? CRYPTO_secure_malloc(num, file, line) - : CRYPTO_malloc(num, file, line); - + return (flags & PEM_FLAG_SECURE) ? OPENSSL_secure_malloc(num) + : OPENSSL_malloc(num); } static int pem_bytes_read_bio_flags(unsigned char **pdata, long *plen, @@ -251,9 +244,9 @@ static int pem_bytes_read_bio_flags(unsigned char **pdata, long *plen, int ret = 0; do { - PEM_FREE(nm, flags, 0); - PEM_FREE(header, flags, 0); - PEM_FREE(data, flags, len); + pem_free(nm, flags, 0); + pem_free(header, flags, 0); + pem_free(data, flags, len); if (!PEM_read_bio_ex(bp, &nm, &header, &data, &len, flags)) { if (ERR_GET_REASON(ERR_peek_error()) == PEM_R_NO_START_LINE) ERR_add_error_data(2, "Expecting: ", name); @@ -275,10 +268,10 @@ static int pem_bytes_read_bio_flags(unsigned char **pdata, long *plen, err: if (!ret || pnm == NULL) - PEM_FREE(nm, flags, 0); - PEM_FREE(header, flags, 0); + pem_free(nm, flags, 0); + pem_free(header, flags, 0); if (!ret) - PEM_FREE(data, flags, len); + pem_free(data, flags, len); return ret; } @@ -352,8 +345,10 @@ int PEM_ASN1_write_bio(i2d_of_void *i2d, const char *name, BIO *bp, /* dsize + 8 bytes are needed */ /* actually it needs the cipher block size extra... */ data = OPENSSL_malloc((unsigned int)dsize + 20); - if (data == NULL) + if (data == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; + } p = data; i = i2d(x, &p); @@ -489,11 +484,11 @@ int PEM_do_header(EVP_CIPHER_INFO *cipher, unsigned char *data, long *plen, * presumably we also parse rfc822-style headers for S/MIME, so a common * abstraction might well be more generally useful. */ -#define PROC_TYPE "Proc-Type:" -#define ENCRYPTED "ENCRYPTED" -#define DEK_INFO "DEK-Info:" int PEM_get_EVP_CIPHER_INFO(char *header, EVP_CIPHER_INFO *cipher) { + static const char ProcType[] = "Proc-Type:"; + static const char ENCRYPTED[] = "ENCRYPTED"; + static const char DEKInfo[] = "DEK-Info:"; const EVP_CIPHER *enc = NULL; int ivlen; char *dekinfostart, c; @@ -503,10 +498,11 @@ int PEM_get_EVP_CIPHER_INFO(char *header, EVP_CIPHER_INFO *cipher) if ((header == NULL) || (*header == '\0') || (*header == '\n')) return 1; - if (!CHECK_AND_SKIP_PREFIX(header, PROC_TYPE)) { + if (strncmp(header, ProcType, sizeof(ProcType)-1) != 0) { ERR_raise(ERR_LIB_PEM, PEM_R_NOT_PROC_TYPE); return 0; } + header += sizeof(ProcType)-1; header += strspn(header, " \t"); if (*header++ != '4' || *header++ != ',') @@ -514,11 +510,12 @@ int PEM_get_EVP_CIPHER_INFO(char *header, EVP_CIPHER_INFO *cipher) header += strspn(header, " \t"); /* We expect "ENCRYPTED" followed by optional white-space + line break */ - if (!CHECK_AND_SKIP_PREFIX(header, ENCRYPTED) || - strspn(header, " \t\r\n") == 0) { + if (strncmp(header, ENCRYPTED, sizeof(ENCRYPTED)-1) != 0 || + strspn(header+sizeof(ENCRYPTED)-1, " \t\r\n") == 0) { ERR_raise(ERR_LIB_PEM, PEM_R_NOT_ENCRYPTED); return 0; } + header += sizeof(ENCRYPTED)-1; header += strspn(header, " \t\r"); if (*header++ != '\n') { ERR_raise(ERR_LIB_PEM, PEM_R_SHORT_HEADER); @@ -529,10 +526,11 @@ int PEM_get_EVP_CIPHER_INFO(char *header, EVP_CIPHER_INFO *cipher) * https://tools.ietf.org/html/rfc1421#section-4.6.1.3 * We expect "DEK-Info: algo[,hex-parameters]" */ - if (!CHECK_AND_SKIP_PREFIX(header, DEK_INFO)) { + if (strncmp(header, DEKInfo, sizeof(DEKInfo)-1) != 0) { ERR_raise(ERR_LIB_PEM, PEM_R_NOT_DEK_INFO); return 0; } + header += sizeof(DEKInfo)-1; header += strspn(header, " \t"); /* @@ -613,11 +611,11 @@ int PEM_write_bio(BIO *bp, const char *name, const char *header, int nlen, n, i, j, outl; unsigned char *buf = NULL; EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new(); - int reason = 0; + int reason = ERR_R_BUF_LIB; int retval = 0; if (ctx == NULL) { - reason = ERR_R_EVP_LIB; + reason = ERR_R_MALLOC_FAILURE; goto err; } @@ -626,53 +624,43 @@ int PEM_write_bio(BIO *bp, const char *name, const char *header, if ((BIO_write(bp, "-----BEGIN ", 11) != 11) || (BIO_write(bp, name, nlen) != nlen) || - (BIO_write(bp, "-----\n", 6) != 6)) { - reason = ERR_R_BIO_LIB; + (BIO_write(bp, "-----\n", 6) != 6)) goto err; - } - i = header != NULL ? strlen(header) : 0; + i = strlen(header); if (i > 0) { - if ((BIO_write(bp, header, i) != i) || (BIO_write(bp, "\n", 1) != 1)) { - reason = ERR_R_BIO_LIB; + if ((BIO_write(bp, header, i) != i) || (BIO_write(bp, "\n", 1) != 1)) goto err; - } } buf = OPENSSL_malloc(PEM_BUFSIZE * 8); - if (buf == NULL) + if (buf == NULL) { + reason = ERR_R_MALLOC_FAILURE; goto err; + } i = j = 0; while (len > 0) { n = (int)((len > (PEM_BUFSIZE * 5)) ? (PEM_BUFSIZE * 5) : len); - if (!EVP_EncodeUpdate(ctx, buf, &outl, &(data[j]), n)) { - reason = ERR_R_EVP_LIB; + if (!EVP_EncodeUpdate(ctx, buf, &outl, &(data[j]), n)) goto err; - } - if ((outl) && (BIO_write(bp, (char *)buf, outl) != outl)) { - reason = ERR_R_BIO_LIB; + if ((outl) && (BIO_write(bp, (char *)buf, outl) != outl)) goto err; - } i += outl; len -= n; j += n; } EVP_EncodeFinal(ctx, buf, &outl); - if ((outl > 0) && (BIO_write(bp, (char *)buf, outl) != outl)) { - reason = ERR_R_BIO_LIB; + if ((outl > 0) && (BIO_write(bp, (char *)buf, outl) != outl)) goto err; - } if ((BIO_write(bp, "-----END ", 9) != 9) || (BIO_write(bp, name, nlen) != nlen) || - (BIO_write(bp, "-----\n", 6) != 6)) { - reason = ERR_R_BIO_LIB; + (BIO_write(bp, "-----\n", 6) != 6)) goto err; - } retval = i + outl; err: - if (retval == 0 && reason != 0) + if (retval == 0) ERR_raise(ERR_LIB_PEM, reason); EVP_ENCODE_CTX_free(ctx); OPENSSL_clear_free(buf, PEM_BUFSIZE * 8); @@ -745,12 +733,12 @@ static int sanitize_line(char *linebuf, int len, unsigned int flags, int first_c #define LINESIZE 255 /* Note trailing spaces for begin and end. */ -#define BEGINSTR "-----BEGIN " -#define ENDSTR "-----END " -#define TAILSTR "-----\n" -#define BEGINLEN ((int)(sizeof(BEGINSTR) - 1)) -#define ENDLEN ((int)(sizeof(ENDSTR) - 1)) -#define TAILLEN ((int)(sizeof(TAILSTR) - 1)) +static const char beginstr[] = "-----BEGIN "; +static const char endstr[] = "-----END "; +static const char tailstr[] = "-----\n"; +#define BEGINLEN ((int)(sizeof(beginstr) - 1)) +#define ENDLEN ((int)(sizeof(endstr) - 1)) +#define TAILLEN ((int)(sizeof(tailstr) - 1)) static int get_name(BIO *bp, char **name, unsigned int flags) { char *linebuf; @@ -762,9 +750,11 @@ static int get_name(BIO *bp, char **name, unsigned int flags) * Need to hold trailing NUL (accounted for by BIO_gets() and the newline * that will be added by sanitize_line() (the extra '1'). */ - linebuf = PEM_MALLOC(LINESIZE + 1, flags); - if (linebuf == NULL) + linebuf = pem_malloc(LINESIZE + 1, flags); + if (linebuf == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); return 0; + } do { len = BIO_gets(bp, linebuf, LINESIZE); @@ -779,19 +769,21 @@ static int get_name(BIO *bp, char **name, unsigned int flags) first_call = 0; /* Allow leading empty or non-matching lines. */ - } while (!HAS_PREFIX(linebuf, BEGINSTR) + } while (strncmp(linebuf, beginstr, BEGINLEN) != 0 || len < TAILLEN - || !HAS_PREFIX(linebuf + len - TAILLEN, TAILSTR)); + || strncmp(linebuf + len - TAILLEN, tailstr, TAILLEN) != 0); linebuf[len - TAILLEN] = '\0'; len = len - BEGINLEN - TAILLEN + 1; - *name = PEM_MALLOC(len, flags); - if (*name == NULL) + *name = pem_malloc(len, flags); + if (*name == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; + } memcpy(*name, linebuf + BEGINLEN, len); ret = 1; err: - PEM_FREE(linebuf, flags, LINESIZE + 1); + pem_free(linebuf, flags, LINESIZE + 1); return ret; } @@ -826,9 +818,11 @@ static int get_header_and_data(BIO *bp, BIO **header, BIO **data, char *name, /* Need to hold trailing NUL (accounted for by BIO_gets() and the newline * that will be added by sanitize_line() (the extra '1'). */ - linebuf = PEM_MALLOC(LINESIZE + 1, flags); - if (linebuf == NULL) + linebuf = pem_malloc(LINESIZE + 1, flags); + if (linebuf == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); return 0; + } while(1) { flags_mask = ~0u; @@ -850,7 +844,7 @@ static int get_header_and_data(BIO *bp, BIO **header, BIO **data, char *name, if (memchr(linebuf, ':', len) != NULL) got_header = IN_HEADER; } - if (HAS_PREFIX(linebuf, ENDSTR) || got_header == IN_HEADER) + if (!strncmp(linebuf, endstr, ENDLEN) || got_header == IN_HEADER) flags_mask &= ~PEM_FLAG_ONLY_B64; len = sanitize_line(linebuf, len, flags & flags_mask, 0); @@ -873,11 +867,11 @@ static int get_header_and_data(BIO *bp, BIO **header, BIO **data, char *name, } /* Check for end of stream (which means there is no header). */ - p = linebuf; - if (CHECK_AND_SKIP_PREFIX(p, ENDSTR)) { + if (strncmp(linebuf, endstr, ENDLEN) == 0) { + p = linebuf + ENDLEN; namelen = strlen(name); if (strncmp(p, name, namelen) != 0 || - !HAS_PREFIX(p + namelen, TAILSTR)) { + strncmp(p + namelen, tailstr, TAILLEN) != 0) { ERR_raise(ERR_LIB_PEM, PEM_R_BAD_END_LINE); goto err; } @@ -911,7 +905,7 @@ static int get_header_and_data(BIO *bp, BIO **header, BIO **data, char *name, ret = 1; err: - PEM_FREE(linebuf, flags, LINESIZE + 1); + pem_free(linebuf, flags, LINESIZE + 1); return ret; } @@ -929,7 +923,7 @@ int PEM_read_bio_ex(BIO *bp, char **name_out, char **header, BIO *headerB = NULL, *dataB = NULL; char *name = NULL; int len, taillen, headerlen, ret = 0; - BUF_MEM *buf_mem; + BUF_MEM * buf_mem; *len_out = 0; *name_out = *header = NULL; @@ -944,7 +938,7 @@ int PEM_read_bio_ex(BIO *bp, char **name_out, char **header, headerB = BIO_new(bmeth); dataB = BIO_new(bmeth); if (headerB == NULL || dataB == NULL) { - ERR_raise(ERR_LIB_PEM, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto end; } @@ -962,7 +956,7 @@ int PEM_read_bio_ex(BIO *bp, char **name_out, char **header, ctx = EVP_ENCODE_CTX_new(); if (ctx == NULL) { - ERR_raise(ERR_LIB_PEM, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto end; } @@ -978,8 +972,8 @@ int PEM_read_bio_ex(BIO *bp, char **name_out, char **header, buf_mem->length = len; headerlen = BIO_get_mem_data(headerB, NULL); - *header = PEM_MALLOC(headerlen + 1, flags); - *data = PEM_MALLOC(len, flags); + *header = pem_malloc(headerlen + 1, flags); + *data = pem_malloc(len, flags); if (*header == NULL || *data == NULL) goto out_free; if (headerlen != 0 && BIO_read(headerB, *header, headerlen) != headerlen) @@ -994,13 +988,13 @@ int PEM_read_bio_ex(BIO *bp, char **name_out, char **header, goto end; out_free: - PEM_FREE(*header, flags, 0); + pem_free(*header, flags, 0); *header = NULL; - PEM_FREE(*data, flags, 0); + pem_free(*data, flags, 0); *data = NULL; end: EVP_ENCODE_CTX_free(ctx); - PEM_FREE(name, flags, 0); + pem_free(name, flags, 0); BIO_free(headerB); BIO_free(dataB); return ret; diff --git a/openssl/src/crypto/pem/pem_pkey.c b/openssl/src/crypto/pem/pem_pkey.c index 4deee46ce..f9346486d 100644 --- a/openssl/src/crypto/pem/pem_pkey.c +++ b/openssl/src/crypto/pem/pem_pkey.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -75,10 +75,6 @@ static EVP_PKEY *pem_read_bio_key_decoder(BIO *bp, EVP_PKEY **x, } ERR_pop_to_mark(); - /* if we were asked for private key, the public key is optional */ - if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) - selection = selection & ~OSSL_KEYMGMT_SELECT_PUBLIC_KEY; - if (!evp_keymgmt_util_has(pkey, selection)) { EVP_PKEY_free(pkey); pkey = NULL; @@ -110,7 +106,7 @@ static EVP_PKEY *pem_read_bio_key_legacy(BIO *bp, EVP_PKEY **x, EVP_PKEY *ret = NULL; ERR_set_mark(); /* not interested in PEM read errors */ - if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) { + if (selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) { if (!PEM_bytes_read_bio_secmem(&data, &len, &nm, PEM_STRING_EVP_PKEY, bp, cb, u)) { @@ -120,7 +116,7 @@ static EVP_PKEY *pem_read_bio_key_legacy(BIO *bp, EVP_PKEY **x, } else { const char *pem_string = PEM_STRING_PARAMETERS; - if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) + if (selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) pem_string = PEM_STRING_PUBLIC; if (!PEM_bytes_read_bio(&data, &len, &nm, pem_string, @@ -178,13 +174,9 @@ static EVP_PKEY *pem_read_bio_key_legacy(BIO *bp, EVP_PKEY **x, goto p8err; ret = ossl_d2i_PrivateKey_legacy(ameth->pkey_id, x, &p, len, libctx, propq); - } else if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) == 0 - && (selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) { - /* Trying legacy PUBKEY decoding only if we do not want private key. */ + } else if (selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) { ret = ossl_d2i_PUBKEY_legacy(x, &p, len); - } else if ((selection & EVP_PKEY_KEYPAIR) == 0 - && (slen = ossl_pem_check_suffix(nm, "PARAMETERS")) > 0) { - /* Trying legacy params decoding only if we do not want a key. */ + } else if ((slen = ossl_pem_check_suffix(nm, "PARAMETERS")) > 0) { ret = EVP_PKEY_new(); if (ret == NULL) goto err; @@ -302,7 +294,6 @@ EVP_PKEY *PEM_read_bio_PrivateKey_ex(BIO *bp, EVP_PKEY **x, OSSL_LIB_CTX *libctx, const char *propq) { return pem_read_bio_key(bp, x, cb, u, libctx, propq, - /* we also want the public key, if available */ EVP_PKEY_KEYPAIR); } @@ -320,7 +311,7 @@ PEM_write_cb_ex_fnsig(PrivateKey, EVP_PKEY, BIO, write_bio) IMPLEMENT_PEM_provided_write_body_main(pkey, bio); legacy: - if (x != NULL && (x->ameth == NULL || x->ameth->priv_encode != NULL)) + if (x->ameth == NULL || x->ameth->priv_encode != NULL) return PEM_write_bio_PKCS8PrivateKey(out, x, enc, (const char *)kstr, klen, cb, u); return PEM_write_bio_PrivateKey_traditional(out, x, enc, kstr, klen, cb, u); @@ -345,9 +336,6 @@ int PEM_write_bio_PrivateKey_traditional(BIO *bp, const EVP_PKEY *x, EVP_PKEY *copy = NULL; int ret; - if (x == NULL) - return 0; - if (evp_pkey_is_assigned(x) && evp_pkey_is_provided(x) && evp_pkey_copy_downgraded(©, x)) @@ -355,7 +343,6 @@ int PEM_write_bio_PrivateKey_traditional(BIO *bp, const EVP_PKEY *x, if (x->ameth == NULL || x->ameth->old_priv_encode == NULL) { ERR_raise(ERR_LIB_PEM, PEM_R_UNSUPPORTED_PUBLIC_KEY_TYPE); - EVP_PKEY_free(copy); return 0; } BIO_snprintf(pem_str, 80, "%s PRIVATE KEY", x->ameth->pem_str); @@ -366,19 +353,10 @@ int PEM_write_bio_PrivateKey_traditional(BIO *bp, const EVP_PKEY *x, return ret; } -static int no_password_cb(char *buf, int num, int rwflag, void *userdata) -{ - return -1; -} - EVP_PKEY *PEM_read_bio_Parameters_ex(BIO *bp, EVP_PKEY **x, OSSL_LIB_CTX *libctx, const char *propq) { - /* - * PEM_read_bio_Parameters(_ex) should never ask for a password. Any attempt - * to get a password just fails. - */ - return pem_read_bio_key(bp, x, no_password_cb, NULL, libctx, propq, + return pem_read_bio_key(bp, x, NULL, NULL, libctx, propq, EVP_PKEY_KEY_PARAMETERS); } diff --git a/openssl/src/crypto/pem/pem_sign.c b/openssl/src/crypto/pem/pem_sign.c index f6b0ff4dd..6ad8e4303 100644 --- a/openssl/src/crypto/pem/pem_sign.c +++ b/openssl/src/crypto/pem/pem_sign.c @@ -33,8 +33,10 @@ int PEM_SignFinal(EVP_MD_CTX *ctx, unsigned char *sigret, unsigned int m_len; m = OPENSSL_malloc(EVP_PKEY_get_size(pkey)); - if (m == NULL) + if (m == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; + } if (EVP_SignFinal(ctx, m, &m_len, pkey) <= 0) goto err; diff --git a/openssl/src/crypto/pem/pvkfmt.c b/openssl/src/crypto/pem/pvkfmt.c index ed7905661..21b16f592 100644 --- a/openssl/src/crypto/pem/pvkfmt.c +++ b/openssl/src/crypto/pem/pvkfmt.c @@ -1,5 +1,5 @@ /* - * Copyright 2005-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,8 +23,6 @@ #include #include #include -#include -#include #include "internal/cryptlib.h" #include "crypto/pem.h" #include "crypto/evp.h" @@ -90,7 +88,6 @@ static EVP_PKEY *evp_pkey_new0_key(void *key, int evp_type) case EVP_PKEY_RSA: if (EVP_PKEY_set1_RSA(pkey, key)) break; - ERR_raise(ERR_LIB_PEM, ERR_R_EVP_LIB); EVP_PKEY_free(pkey); pkey = NULL; break; @@ -98,14 +95,11 @@ static EVP_PKEY *evp_pkey_new0_key(void *key, int evp_type) case EVP_PKEY_DSA: if (EVP_PKEY_set1_DSA(pkey, key)) break; - ERR_raise(ERR_LIB_PEM, ERR_R_EVP_LIB); EVP_PKEY_free(pkey); pkey = NULL; break; #endif } - } else { - ERR_raise(ERR_LIB_PEM, ERR_R_EVP_LIB); } switch (evp_type) { @@ -119,6 +113,8 @@ static EVP_PKEY *evp_pkey_new0_key(void *key, int evp_type) #endif } + if (pkey == NULL) + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); return pkey; } @@ -150,7 +146,7 @@ static EVP_PKEY *evp_pkey_new0_key(void *key, int evp_type) * Read the MSBLOB header and get relevant data from it. * * |pisdss| and |pispub| have a double role, as they can be used for - * discovery as well as to check the blob meets expectations. + * discovery as well as to check the the blob meets expectations. * |*pisdss| is the indicator for whether the key is a DSA key or not. * |*pispub| is the indicator for whether the key is public or not. * In both cases, the following input values apply: @@ -345,8 +341,10 @@ EVP_PKEY *ossl_b2i_bio(BIO *in, int *ispub) return NULL; } buf = OPENSSL_malloc(length); - if (buf == NULL) + if (buf == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; + } p = buf; if (BIO_read(in, buf, length) != (int)length) { ERR_raise(ERR_LIB_PEM, PEM_R_KEYBLOB_TOO_SHORT); @@ -384,22 +382,22 @@ DSA *ossl_b2i_DSA_after_header(const unsigned char **in, unsigned int bitlen, dsa = DSA_new(); if (dsa == NULL) - goto dsaerr; + goto memerr; if (!read_lebn(&p, nbyte, &pbn)) - goto bnerr; + goto memerr; if (!read_lebn(&p, 20, &qbn)) - goto bnerr; + goto memerr; if (!read_lebn(&p, nbyte, &gbn)) - goto bnerr; + goto memerr; if (ispub) { if (!read_lebn(&p, nbyte, &pub_key)) - goto bnerr; + goto memerr; } else { if (!read_lebn(&p, 20, &priv_key)) - goto bnerr; + goto memerr; /* Set constant time flag before public key calculation */ BN_set_flags(priv_key, BN_FLG_CONSTTIME); @@ -407,33 +405,28 @@ DSA *ossl_b2i_DSA_after_header(const unsigned char **in, unsigned int bitlen, /* Calculate public key */ pub_key = BN_new(); if (pub_key == NULL) - goto bnerr; + goto memerr; if ((ctx = BN_CTX_new()) == NULL) - goto bnerr; + goto memerr; if (!BN_mod_exp(pub_key, gbn, priv_key, pbn, ctx)) - goto bnerr; + goto memerr; BN_CTX_free(ctx); ctx = NULL; } if (!DSA_set0_pqg(dsa, pbn, qbn, gbn)) - goto dsaerr; + goto memerr; pbn = qbn = gbn = NULL; if (!DSA_set0_key(dsa, pub_key, priv_key)) - goto dsaerr; + goto memerr; pub_key = priv_key = NULL; *in = p; return dsa; - dsaerr: - ERR_raise(ERR_LIB_PEM, ERR_R_DSA_LIB); - goto err; - bnerr: - ERR_raise(ERR_LIB_PEM, ERR_R_BN_LIB); - - err: + memerr: + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); DSA_free(dsa); BN_free(pbn); BN_free(qbn); @@ -457,48 +450,42 @@ RSA *ossl_b2i_RSA_after_header(const unsigned char **in, unsigned int bitlen, rsa = RSA_new(); if (rsa == NULL) - goto rsaerr; + goto memerr; e = BN_new(); if (e == NULL) - goto bnerr; + goto memerr; if (!BN_set_word(e, read_ledword(&pin))) - goto bnerr; + goto memerr; if (!read_lebn(&pin, nbyte, &n)) - goto bnerr; + goto memerr; if (!ispub) { if (!read_lebn(&pin, hnbyte, &p)) - goto bnerr; + goto memerr; if (!read_lebn(&pin, hnbyte, &q)) - goto bnerr; + goto memerr; if (!read_lebn(&pin, hnbyte, &dmp1)) - goto bnerr; + goto memerr; if (!read_lebn(&pin, hnbyte, &dmq1)) - goto bnerr; + goto memerr; if (!read_lebn(&pin, hnbyte, &iqmp)) - goto bnerr; + goto memerr; if (!read_lebn(&pin, nbyte, &d)) - goto bnerr; + goto memerr; if (!RSA_set0_factors(rsa, p, q)) - goto rsaerr; + goto memerr; p = q = NULL; if (!RSA_set0_crt_params(rsa, dmp1, dmq1, iqmp)) - goto rsaerr; + goto memerr; dmp1 = dmq1 = iqmp = NULL; } if (!RSA_set0_key(rsa, n, e, d)) - goto rsaerr; + goto memerr; n = e = d = NULL; *in = pin; return rsa; - - rsaerr: - ERR_raise(ERR_LIB_PEM, ERR_R_RSA_LIB); - goto err; - bnerr: - ERR_raise(ERR_LIB_PEM, ERR_R_BN_LIB); - - err: + memerr: + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); BN_free(e); BN_free(n); BN_free(p); @@ -590,6 +577,7 @@ static int do_i2b(unsigned char **out, const EVP_PKEY *pk, int ispub) p = *out; else { if ((p = OPENSSL_malloc(outlen)) == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); outlen = -1; goto end; } @@ -802,34 +790,29 @@ int ossl_do_PVK_header(const unsigned char **in, unsigned int length, } #ifndef OPENSSL_NO_RC4 -static int derive_pvk_key(unsigned char *key, size_t keylen, +static int derive_pvk_key(unsigned char *key, const unsigned char *salt, unsigned int saltlen, const unsigned char *pass, int passlen, OSSL_LIB_CTX *libctx, const char *propq) { - EVP_KDF *kdf; - EVP_KDF_CTX *ctx; - OSSL_PARAM params[5], *p = params; - int rv; + EVP_MD_CTX *mctx = EVP_MD_CTX_new(); + int rv = 0; + EVP_MD *sha1 = NULL; - if ((kdf = EVP_KDF_fetch(libctx, "PVKKDF", propq)) == NULL) - return 0; - ctx = EVP_KDF_CTX_new(kdf); - EVP_KDF_free(kdf); - if (ctx == NULL) - return 0; + if ((sha1 = EVP_MD_fetch(libctx, SN_sha1, propq)) == NULL) + goto err; + + if (mctx == NULL + || !EVP_DigestInit_ex(mctx, sha1, NULL) + || !EVP_DigestUpdate(mctx, salt, saltlen) + || !EVP_DigestUpdate(mctx, pass, passlen) + || !EVP_DigestFinal_ex(mctx, key, NULL)) + goto err; - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_SALT, - (void *)salt, saltlen); - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_PASSWORD, - (void *)pass, passlen); - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST, SN_sha1, 0); - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_PROPERTIES, - (char *)propq, 0); - *p = OSSL_PARAM_construct_end(); - - rv = EVP_KDF_derive(ctx, key, keylen, params); - EVP_KDF_CTX_free(ctx); + rv = 1; +err: + EVP_MD_CTX_free(mctx); + EVP_MD_free(sha1); return rv; } #endif @@ -850,7 +833,7 @@ static void *do_PVK_body_key(const unsigned char **in, EVP_CIPHER_CTX *cctx = EVP_CIPHER_CTX_new(); if (cctx == NULL) { - ERR_raise(ERR_LIB_PEM, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; } @@ -870,9 +853,11 @@ static void *do_PVK_body_key(const unsigned char **in, goto err; } enctmp = OPENSSL_malloc(keylen + 8); - if (enctmp == NULL) + if (enctmp == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; - if (!derive_pvk_key(keybuf, sizeof(keybuf), p, saltlen, + } + if (!derive_pvk_key(keybuf, p, saltlen, (unsigned char *)psbuf, inlen, libctx, propq)) goto err; p += saltlen; @@ -949,8 +934,10 @@ static void *do_PVK_key_bio(BIO *in, pem_password_cb *cb, void *u, return 0; buflen = (int)keylen + saltlen; buf = OPENSSL_malloc(buflen); - if (buf == NULL) + if (buf == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); return 0; + } p = buf; if (BIO_read(in, buf, buflen) != buflen) { ERR_raise(ERR_LIB_PEM, PEM_R_PVK_DATA_TOO_SHORT); @@ -1033,8 +1020,10 @@ static int i2b_PVK(unsigned char **out, const EVP_PKEY *pk, int enclevel, p = *out; } else { start = p = OPENSSL_malloc(outlen); - if (p == NULL) + if (p == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); return -1; + } } cctx = EVP_CIPHER_CTX_new(); @@ -1074,7 +1063,7 @@ static int i2b_PVK(unsigned char **out, const EVP_PKEY *pk, int enclevel, ERR_raise(ERR_LIB_PEM, PEM_R_BAD_PASSWORD_READ); goto error; } - if (!derive_pvk_key(keybuf, sizeof(keybuf), salt, PVK_SALTLEN, + if (!derive_pvk_key(keybuf, salt, PVK_SALTLEN, (unsigned char *)psbuf, inlen, libctx, propq)) goto error; if ((rc4 = EVP_CIPHER_fetch(libctx, "RC4", propq)) == NULL) diff --git a/openssl/src/crypto/pkcs12/p12_add.c b/openssl/src/crypto/pkcs12/p12_add.c index b2635ff1f..6fd4184af 100644 --- a/openssl/src/crypto/pkcs12/p12_add.c +++ b/openssl/src/crypto/pkcs12/p12_add.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,16 +24,16 @@ PKCS12_SAFEBAG *PKCS12_item_pack_safebag(void *obj, const ASN1_ITEM *it, PKCS12_SAFEBAG *safebag; if ((bag = PKCS12_BAGS_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; } bag->type = OBJ_nid2obj(nid1); if (!ASN1_item_pack(obj, it, &bag->value.octet)) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } if ((safebag = PKCS12_SAFEBAG_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } safebag->value.bag = bag; @@ -51,12 +51,12 @@ PKCS7 *PKCS12_pack_p7data(STACK_OF(PKCS12_SAFEBAG) *sk) PKCS7 *p7; if ((p7 = PKCS7_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; } p7->type = OBJ_nid2obj(NID_pkcs7_data); if ((p7->d.data = ASN1_OCTET_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } @@ -78,15 +78,7 @@ STACK_OF(PKCS12_SAFEBAG) *PKCS12_unpack_p7data(PKCS7 *p7) ERR_raise(ERR_LIB_PKCS12, PKCS12_R_CONTENT_TYPE_NOT_DATA); return NULL; } - - if (p7->d.data == NULL) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_DECODE_ERROR); - return NULL; - } - - return ASN1_item_unpack_ex(p7->d.data, ASN1_ITEM_rptr(PKCS12_SAFEBAGS), - ossl_pkcs7_ctx_get0_libctx(&p7->ctx), - ossl_pkcs7_ctx_get0_propq(&p7->ctx)); + return ASN1_item_unpack(p7->d.data, ASN1_ITEM_rptr(PKCS12_SAFEBAGS)); } /* Turn a stack of SAFEBAGS into a PKCS#7 encrypted data ContentInfo */ @@ -102,7 +94,7 @@ PKCS7 *PKCS12_pack_p7encdata_ex(int pbe_nid, const char *pass, int passlen, EVP_CIPHER *pbe_ciph_fetch = NULL; if ((p7 = PKCS7_new_ex(ctx, propq)) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; } if (!PKCS7_set_type(p7, NID_pkcs7_encrypted)) { @@ -123,7 +115,7 @@ PKCS7 *PKCS12_pack_p7encdata_ex(int pbe_nid, const char *pass, int passlen, } if (pbe == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } X509_ALGOR_free(p7->d.encrypted->enc_data->algorithm); @@ -158,12 +150,6 @@ STACK_OF(PKCS12_SAFEBAG) *PKCS12_unpack_p7encdata(PKCS7 *p7, const char *pass, { if (!PKCS7_type_is_encrypted(p7)) return NULL; - - if (p7->d.encrypted == NULL) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_DECODE_ERROR); - return NULL; - } - return PKCS12_item_decrypt_d2i_ex(p7->d.encrypted->enc_data->algorithm, ASN1_ITEM_rptr(PKCS12_SAFEBAGS), pass, passlen, @@ -195,7 +181,6 @@ int PKCS12_pack_authsafes(PKCS12 *p12, STACK_OF(PKCS7) *safes) STACK_OF(PKCS7) *PKCS12_unpack_authsafes(const PKCS12 *p12) { STACK_OF(PKCS7) *p7s; - PKCS7_CTX *p7ctx; PKCS7 *p7; int i; @@ -203,17 +188,8 @@ STACK_OF(PKCS7) *PKCS12_unpack_authsafes(const PKCS12 *p12) ERR_raise(ERR_LIB_PKCS12, PKCS12_R_CONTENT_TYPE_NOT_DATA); return NULL; } - - if (p12->authsafes->d.data == NULL) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_DECODE_ERROR); - return NULL; - } - - p7ctx = &p12->authsafes->ctx; - p7s = ASN1_item_unpack_ex(p12->authsafes->d.data, - ASN1_ITEM_rptr(PKCS12_AUTHSAFES), - ossl_pkcs7_ctx_get0_libctx(p7ctx), - ossl_pkcs7_ctx_get0_propq(p7ctx)); + p7s = ASN1_item_unpack(p12->authsafes->d.data, + ASN1_ITEM_rptr(PKCS12_AUTHSAFES)); if (p7s != NULL) { for (i = 0; i < sk_PKCS7_num(p7s); i++) { p7 = sk_PKCS7_value(p7s, i); diff --git a/openssl/src/crypto/pkcs12/p12_asn.c b/openssl/src/crypto/pkcs12/p12_asn.c index e4247b27f..aabbd38ee 100644 --- a/openssl/src/crypto/pkcs12/p12_asn.c +++ b/openssl/src/crypto/pkcs12/p12_asn.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2018 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,6 @@ #include #include #include "p12_local.h" -#include "crypto/pkcs7.h" /* PKCS#12 ASN1 module */ @@ -22,21 +21,7 @@ ASN1_SEQUENCE(PKCS12) = { ASN1_OPT(PKCS12, mac, PKCS12_MAC_DATA) } ASN1_SEQUENCE_END(PKCS12) -IMPLEMENT_ASN1_ENCODE_FUNCTIONS_fname(PKCS12, PKCS12, PKCS12) - -PKCS12 *PKCS12_new(void) -{ - return (PKCS12 *)ASN1_item_new(ASN1_ITEM_rptr(PKCS12)); -} - -void PKCS12_free(PKCS12 *p12) -{ - if (p12 != NULL && p12->authsafes != NULL) { - OPENSSL_free(p12->authsafes->ctx.propq); - p12->authsafes->ctx.propq = NULL; - } - ASN1_item_free((ASN1_VALUE *)p12, ASN1_ITEM_rptr(PKCS12)); -} +IMPLEMENT_ASN1_FUNCTIONS(PKCS12) ASN1_SEQUENCE(PKCS12_MAC_DATA) = { ASN1_SIMPLE(PKCS12_MAC_DATA, dinfo, X509_SIG), diff --git a/openssl/src/crypto/pkcs12/p12_attr.c b/openssl/src/crypto/pkcs12/p12_attr.c index ea0d027c1..da228336e 100644 --- a/openssl/src/crypto/pkcs12/p12_attr.c +++ b/openssl/src/crypto/pkcs12/p12_attr.c @@ -95,11 +95,11 @@ int PKCS12_add1_attr_by_txt(PKCS12_SAFEBAG *bag, const char *attrname, int type, ASN1_TYPE *PKCS12_get_attr_gen(const STACK_OF(X509_ATTRIBUTE) *attrs, int attr_nid) { - int i = X509at_get_attr_by_NID(attrs, attr_nid, -1); - - if (i < 0) - return NULL; - return X509_ATTRIBUTE_get0_type(X509at_get_attr(attrs, i), 0); + X509_ATTRIBUTE *attrib; + int i; + i = X509at_get_attr_by_NID(attrs, attr_nid, -1); + attrib = X509at_get_attr(attrs, i); + return X509_ATTRIBUTE_get0_type(attrib, 0); } char *PKCS12_get_friendlyname(PKCS12_SAFEBAG *bag) @@ -119,11 +119,3 @@ PKCS12_SAFEBAG_get0_attrs(const PKCS12_SAFEBAG *bag) { return bag->attrib; } - -void PKCS12_SAFEBAG_set0_attrs(PKCS12_SAFEBAG *bag, STACK_OF(X509_ATTRIBUTE) *attrs) -{ - if (bag->attrib != attrs) - sk_X509_ATTRIBUTE_free(bag->attrib); - - bag->attrib = attrs; -} diff --git a/openssl/src/crypto/pkcs12/p12_crt.c b/openssl/src/crypto/pkcs12/p12_crt.c index 2e40dd93c..f0190761e 100644 --- a/openssl/src/crypto/pkcs12/p12_crt.c +++ b/openssl/src/crypto/pkcs12/p12_crt.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,40 +14,32 @@ static int pkcs12_add_bag(STACK_OF(PKCS12_SAFEBAG) **pbags, PKCS12_SAFEBAG *bag); -static int pkcs12_remove_bag(STACK_OF(PKCS12_SAFEBAG) **pbags, - PKCS12_SAFEBAG *bag); -static PKCS12_SAFEBAG *pkcs12_add_cert_bag(STACK_OF(PKCS12_SAFEBAG) **pbags, - X509 *cert, - const char *name, - int namelen, - unsigned char *keyid, - int keyidlen); static int copy_bag_attr(PKCS12_SAFEBAG *bag, EVP_PKEY *pkey, int nid) { - int idx = EVP_PKEY_get_attr_by_NID(pkey, nid, -1); - + int idx; + X509_ATTRIBUTE *attr; + idx = EVP_PKEY_get_attr_by_NID(pkey, nid, -1); if (idx < 0) return 1; - return X509at_add1_attr(&bag->attrib, EVP_PKEY_get_attr(pkey, idx)) != NULL; + attr = EVP_PKEY_get_attr(pkey, idx); + if (!X509at_add1_attr(&bag->attrib, attr)) + return 0; + return 1; } -PKCS12 *PKCS12_create_ex2(const char *pass, const char *name, EVP_PKEY *pkey, - X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, - int iter, int mac_iter, int keytype, - OSSL_LIB_CTX *ctx, const char *propq, - PKCS12_create_cb *cb, void *cbarg) +PKCS12 *PKCS12_create_ex(const char *pass, const char *name, EVP_PKEY *pkey, + X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, + int iter, int mac_iter, int keytype, + OSSL_LIB_CTX *ctx, const char *propq) { PKCS12 *p12 = NULL; STACK_OF(PKCS7) *safes = NULL; STACK_OF(PKCS12_SAFEBAG) *bags = NULL; PKCS12_SAFEBAG *bag = NULL; - int i, cbret; + int i; unsigned char keyid[EVP_MAX_MD_SIZE]; unsigned int keyidlen = 0; - int namelen = -1; - unsigned char *pkeyid = NULL; - int pkeyidlen = -1; /* Set defaults */ if (nid_cert == NID_undef) @@ -72,40 +64,17 @@ PKCS12 *PKCS12_create_ex2(const char *pass, const char *name, EVP_PKEY *pkey, } if (cert) { - if (name == NULL) - name = (char *)X509_alias_get0(cert, &namelen); - if (keyidlen > 0) { - pkeyid = keyid; - pkeyidlen = keyidlen; - } else { - pkeyid = X509_keyid_get0(cert, &pkeyidlen); - } - - bag = pkcs12_add_cert_bag(&bags, cert, name, namelen, pkeyid, pkeyidlen); - if (cb != NULL) { - cbret = cb(bag, cbarg); - if (cbret == -1) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_CALLBACK_FAILED); - goto err; - } else if (cbret == 0) { - pkcs12_remove_bag(&bags, bag); - } - } + bag = PKCS12_add_cert(&bags, cert); + if (name && !PKCS12_add_friendlyname(bag, name, -1)) + goto err; + if (keyidlen && !PKCS12_add_localkeyid(bag, keyid, keyidlen)) + goto err; } /* Add all other certificates */ for (i = 0; i < sk_X509_num(ca); i++) { - if ((bag = PKCS12_add_cert(&bags, sk_X509_value(ca, i))) == NULL) + if (!PKCS12_add_cert(&bags, sk_X509_value(ca, i))) goto err; - if (cb != NULL) { - cbret = cb(bag, cbarg); - if (cbret == -1) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_CALLBACK_FAILED); - goto err; - } else if (cbret == 0) { - pkcs12_remove_bag(&bags, bag); - } - } } if (bags && !PKCS12_add_safe_ex(&safes, bags, nid_cert, iter, pass, @@ -131,15 +100,6 @@ PKCS12 *PKCS12_create_ex2(const char *pass, const char *name, EVP_PKEY *pkey, goto err; if (keyidlen && !PKCS12_add_localkeyid(bag, keyid, keyidlen)) goto err; - if (cb != NULL) { - cbret = cb(bag, cbarg); - if (cbret == -1) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_CALLBACK_FAILED); - goto err; - } else if (cbret == 0) { - pkcs12_remove_bag(&bags, bag); - } - } } if (bags && !PKCS12_add_safe(&safes, bags, -1, 0, NULL)) @@ -171,16 +131,6 @@ PKCS12 *PKCS12_create_ex2(const char *pass, const char *name, EVP_PKEY *pkey, } -PKCS12 *PKCS12_create_ex(const char *pass, const char *name, EVP_PKEY *pkey, X509 *cert, - STACK_OF(X509) *ca, int nid_key, int nid_cert, int iter, - int mac_iter, int keytype, - OSSL_LIB_CTX *ctx, const char *propq) -{ - return PKCS12_create_ex2(pass, name, pkey, cert, ca, nid_key, nid_cert, - iter, mac_iter, keytype, ctx, propq, - NULL, NULL); -} - PKCS12 *PKCS12_create(const char *pass, const char *name, EVP_PKEY *pkey, X509 *cert, STACK_OF(X509) *ca, int nid_key, int nid_cert, int iter, int mac_iter, int keytype) @@ -189,23 +139,30 @@ PKCS12 *PKCS12_create(const char *pass, const char *name, EVP_PKEY *pkey, X509 * iter, mac_iter, keytype, NULL, NULL); } -static PKCS12_SAFEBAG *pkcs12_add_cert_bag(STACK_OF(PKCS12_SAFEBAG) **pbags, - X509 *cert, - const char *name, - int namelen, - unsigned char *keyid, - int keyidlen) +PKCS12_SAFEBAG *PKCS12_add_cert(STACK_OF(PKCS12_SAFEBAG) **pbags, X509 *cert) { PKCS12_SAFEBAG *bag = NULL; + char *name; + int namelen = -1; + unsigned char *keyid; + int keyidlen = -1; /* Add user certificate */ if ((bag = PKCS12_SAFEBAG_create_cert(cert)) == NULL) goto err; - if (name != NULL && !PKCS12_add_friendlyname(bag, name, namelen)) + /* + * Use friendlyName and localKeyID in certificate. (if present) + */ + + name = (char *)X509_alias_get0(cert, &namelen); + + if (name && !PKCS12_add_friendlyname(bag, name, namelen)) goto err; - if (keyid != NULL && !PKCS12_add_localkeyid(bag, keyid, keyidlen)) + keyid = X509_keyid_get0(cert, &keyidlen); + + if (keyid && !PKCS12_add_localkeyid(bag, keyid, keyidlen)) goto err; if (!pkcs12_add_bag(pbags, bag)) @@ -216,22 +173,7 @@ static PKCS12_SAFEBAG *pkcs12_add_cert_bag(STACK_OF(PKCS12_SAFEBAG) **pbags, err: PKCS12_SAFEBAG_free(bag); return NULL; -} -PKCS12_SAFEBAG *PKCS12_add_cert(STACK_OF(PKCS12_SAFEBAG) **pbags, X509 *cert) -{ - char *name = NULL; - int namelen = -1; - unsigned char *keyid = NULL; - int keyidlen = -1; - - /* - * Use friendlyName and localKeyID in certificate. (if present) - */ - name = (char *)X509_alias_get0(cert, &namelen); - keyid = X509_keyid_get0(cert, &keyidlen); - - return pkcs12_add_cert_bag(pbags, cert, name, namelen, keyid, keyidlen); } PKCS12_SAFEBAG *PKCS12_add_key_ex(STACK_OF(PKCS12_SAFEBAG) **pbags, @@ -310,11 +252,7 @@ int PKCS12_add_safe_ex(STACK_OF(PKCS7) **psafes, STACK_OF(PKCS12_SAFEBAG) *bags, } if (nid_safe == 0) -#ifdef OPENSSL_NO_RC2 nid_safe = NID_pbe_WithSHA1And3_Key_TripleDES_CBC; -#else - nid_safe = NID_pbe_WithSHA1And40BitRC2_CBC; -#endif if (nid_safe == -1) p7 = PKCS12_pack_p7data(bags); @@ -343,22 +281,6 @@ int PKCS12_add_safe(STACK_OF(PKCS7) **psafes, STACK_OF(PKCS12_SAFEBAG) *bags, return PKCS12_add_safe_ex(psafes, bags, nid_safe, iter, pass, NULL, NULL); } - -static int pkcs12_remove_bag(STACK_OF(PKCS12_SAFEBAG) **pbags, - PKCS12_SAFEBAG *bag) -{ - PKCS12_SAFEBAG *tmp; - - if (pbags == NULL || bag == NULL) - return 1; - - if ((tmp = sk_PKCS12_SAFEBAG_delete_ptr(*pbags, bag)) == NULL) - return 0; - - PKCS12_SAFEBAG_free(tmp); - return 1; -} - static int pkcs12_add_bag(STACK_OF(PKCS12_SAFEBAG) **pbags, PKCS12_SAFEBAG *bag) { diff --git a/openssl/src/crypto/pkcs12/p12_decr.c b/openssl/src/crypto/pkcs12/p12_decr.c index 3fa9c9c8c..0e3e825d6 100644 --- a/openssl/src/crypto/pkcs12/p12_decr.c +++ b/openssl/src/crypto/pkcs12/p12_decr.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,10 +26,9 @@ unsigned char *PKCS12_pbe_crypt_ex(const X509_ALGOR *algor, int outlen, i; EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); int max_out_len, mac_len = 0; - int block_size; if (ctx == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } @@ -38,20 +37,7 @@ unsigned char *PKCS12_pbe_crypt_ex(const X509_ALGOR *algor, algor->parameter, ctx, en_de, libctx, propq)) goto err; - /* - * GOST algorithm specifics: - * OMAC algorithm calculate and encrypt MAC of the encrypted objects - * It's appended to encrypted text on encrypting - * MAC should be processed on decrypting separately from plain text - */ - block_size = EVP_CIPHER_CTX_get_block_size(ctx); - - if (block_size == 0) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_PASSED_NULL_PARAMETER); - goto err; - } - - max_out_len = inlen + block_size; + max_out_len = inlen + EVP_CIPHER_CTX_get_block_size(ctx); if ((EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(ctx)) & EVP_CIPH_FLAG_CIPHER_WITH_MAC) != 0) { if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_TLS1_AAD, 0, &mac_len) < 0) { @@ -75,8 +61,10 @@ unsigned char *PKCS12_pbe_crypt_ex(const X509_ALGOR *algor, } } - if ((out = OPENSSL_malloc(max_out_len)) == NULL) + if ((out = OPENSSL_malloc(max_out_len)) == NULL) { + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; + } if (!EVP_CipherUpdate(ctx, out, &i, in, inlen)) { OPENSSL_free(out); @@ -100,8 +88,6 @@ unsigned char *PKCS12_pbe_crypt_ex(const X509_ALGOR *algor, if (EVP_CIPHER_CTX_is_encrypting(ctx)) { if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, (int)mac_len, out+outlen) < 0) { - OPENSSL_free(out); - out = NULL; ERR_raise(ERR_LIB_PKCS12, ERR_R_INTERNAL_ERROR); goto err; } @@ -186,7 +172,7 @@ ASN1_OCTET_STRING *PKCS12_item_i2d_encrypt_ex(X509_ALGOR *algor, int inlen; if ((oct = ASN1_OCTET_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } inlen = ASN1_item_i2d(obj, &in, it); diff --git a/openssl/src/crypto/pkcs12/p12_init.c b/openssl/src/crypto/pkcs12/p12_init.c index 537a1e316..45aa2f915 100644 --- a/openssl/src/crypto/pkcs12/p12_init.c +++ b/openssl/src/crypto/pkcs12/p12_init.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,7 +20,7 @@ PKCS12 *PKCS12_init_ex(int mode, OSSL_LIB_CTX *ctx, const char *propq) PKCS12 *pkcs12; if ((pkcs12 = PKCS12_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; } if (!ASN1_INTEGER_set(pkcs12->version, 3)) @@ -29,14 +29,14 @@ PKCS12 *PKCS12_init_ex(int mode, OSSL_LIB_CTX *ctx, const char *propq) ossl_pkcs7_set0_libctx(pkcs12->authsafes, ctx); if (!ossl_pkcs7_set1_propq(pkcs12->authsafes, propq)) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_PKCS7_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } switch (mode) { case NID_pkcs7_data: if ((pkcs12->authsafes->d.data = ASN1_OCTET_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } break; @@ -56,9 +56,3 @@ PKCS12 *PKCS12_init(int mode) return PKCS12_init_ex(mode, NULL, NULL); } -const PKCS7_CTX *ossl_pkcs12_get0_pkcs7ctx(const PKCS12 *p12) -{ - if (p12 == NULL || p12->authsafes == NULL) - return NULL; - return &p12->authsafes->ctx; -} diff --git a/openssl/src/crypto/pkcs12/p12_key.c b/openssl/src/crypto/pkcs12/p12_key.c index 9f7012a2c..41a2d7293 100644 --- a/openssl/src/crypto/pkcs12/p12_key.c +++ b/openssl/src/crypto/pkcs12/p12_key.c @@ -29,7 +29,7 @@ int PKCS12_key_gen_asc_ex(const char *pass, int passlen, unsigned char *salt, unipass = NULL; uniplen = 0; } else if (!OPENSSL_asc2uni(pass, passlen, &unipass, &uniplen)) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_PKCS12_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return 0; } ret = PKCS12_key_gen_uni_ex(unipass, uniplen, salt, saltlen, id, iter, @@ -59,7 +59,7 @@ int PKCS12_key_gen_utf8_ex(const char *pass, int passlen, unsigned char *salt, unipass = NULL; uniplen = 0; } else if (!OPENSSL_utf82uni(pass, passlen, &unipass, &uniplen)) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_PKCS12_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return 0; } ret = PKCS12_key_gen_uni_ex(unipass, uniplen, salt, saltlen, id, iter, diff --git a/openssl/src/crypto/pkcs12/p12_kiss.c b/openssl/src/crypto/pkcs12/p12_kiss.c index 0901dc940..229b34cf6 100644 --- a/openssl/src/crypto/pkcs12/p12_kiss.c +++ b/openssl/src/crypto/pkcs12/p12_kiss.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,12 +18,10 @@ static int parse_pk12(PKCS12 *p12, const char *pass, int passlen, EVP_PKEY **pkey, STACK_OF(X509) *ocerts); static int parse_bags(const STACK_OF(PKCS12_SAFEBAG) *bags, const char *pass, - int passlen, EVP_PKEY **pkey, STACK_OF(X509) *ocerts, - OSSL_LIB_CTX *libctx, const char *propq); + int passlen, EVP_PKEY **pkey, STACK_OF(X509) *ocerts); static int parse_bag(PKCS12_SAFEBAG *bag, const char *pass, int passlen, - EVP_PKEY **pkey, STACK_OF(X509) *ocerts, - OSSL_LIB_CTX *libctx, const char *propq); + EVP_PKEY **pkey, STACK_OF(X509) *ocerts); /* * Parse and decrypt a PKCS#12 structure returning user key, user cert and @@ -51,34 +49,33 @@ int PKCS12_parse(PKCS12 *p12, const char *pass, EVP_PKEY **pkey, X509 **cert, } /* Check the mac */ - if (PKCS12_mac_present(p12)) { - /* - * If password is zero length or NULL then try verifying both cases to - * determine which password is correct. The reason for this is that under - * PKCS#12 password based encryption no password and a zero length - * password are two different things... - */ - if (pass == NULL || *pass == '\0') { - if (PKCS12_verify_mac(p12, NULL, 0)) - pass = NULL; - else if (PKCS12_verify_mac(p12, "", 0)) - pass = ""; - else { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_MAC_VERIFY_FAILURE); - goto err; - } - } else if (!PKCS12_verify_mac(p12, pass, -1)) { + + /* + * If password is zero length or NULL then try verifying both cases to + * determine which password is correct. The reason for this is that under + * PKCS#12 password based encryption no password and a zero length + * password are two different things... + */ + + if (pass == NULL || *pass == '\0') { + if (!PKCS12_mac_present(p12) + || PKCS12_verify_mac(p12, NULL, 0)) + pass = NULL; + else if (PKCS12_verify_mac(p12, "", 0)) + pass = ""; + else { ERR_raise(ERR_LIB_PKCS12, PKCS12_R_MAC_VERIFY_FAILURE); goto err; } - } else if (pass == NULL || *pass == '\0') { - pass = NULL; + } else if (!PKCS12_verify_mac(p12, pass, -1)) { + ERR_raise(ERR_LIB_PKCS12, PKCS12_R_MAC_VERIFY_FAILURE); + goto err; } /* If needed, allocate stack for other certificates */ if ((cert != NULL || ca != NULL) && (ocerts = sk_X509_new_null()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } @@ -128,7 +125,7 @@ int PKCS12_parse(PKCS12 *p12, const char *pass, EVP_PKEY **pkey, X509 **cert, *cert = NULL; } X509_free(x); - OSSL_STACK_OF_X509_free(ocerts); + sk_X509_pop_free(ocerts, X509_free); return 0; } @@ -159,8 +156,7 @@ static int parse_pk12(PKCS12 *p12, const char *pass, int passlen, sk_PKCS7_pop_free(asafes, PKCS7_free); return 0; } - if (!parse_bags(bags, pass, passlen, pkey, ocerts, - p7->ctx.libctx, p7->ctx.propq)) { + if (!parse_bags(bags, pass, passlen, pkey, ocerts)) { sk_PKCS12_SAFEBAG_pop_free(bags, PKCS12_SAFEBAG_free); sk_PKCS7_pop_free(asafes, PKCS7_free); return 0; @@ -173,14 +169,12 @@ static int parse_pk12(PKCS12 *p12, const char *pass, int passlen, /* pkey and/or ocerts may be NULL */ static int parse_bags(const STACK_OF(PKCS12_SAFEBAG) *bags, const char *pass, - int passlen, EVP_PKEY **pkey, STACK_OF(X509) *ocerts, - OSSL_LIB_CTX *libctx, const char *propq) + int passlen, EVP_PKEY **pkey, STACK_OF(X509) *ocerts) { int i; for (i = 0; i < sk_PKCS12_SAFEBAG_num(bags); i++) { if (!parse_bag(sk_PKCS12_SAFEBAG_value(bags, i), - pass, passlen, pkey, ocerts, - libctx, propq)) + pass, passlen, pkey, ocerts)) return 0; } return 1; @@ -188,8 +182,7 @@ static int parse_bags(const STACK_OF(PKCS12_SAFEBAG) *bags, const char *pass, /* pkey and/or ocerts may be NULL */ static int parse_bag(PKCS12_SAFEBAG *bag, const char *pass, int passlen, - EVP_PKEY **pkey, STACK_OF(X509) *ocerts, - OSSL_LIB_CTX *libctx, const char *propq) + EVP_PKEY **pkey, STACK_OF(X509) *ocerts) { PKCS8_PRIV_KEY_INFO *p8; X509 *x509; @@ -207,8 +200,7 @@ static int parse_bag(PKCS12_SAFEBAG *bag, const char *pass, int passlen, case NID_keyBag: if (pkey == NULL || *pkey != NULL) return 1; - *pkey = EVP_PKCS82PKEY_ex(PKCS12_SAFEBAG_get0_p8inf(bag), - libctx, propq); + *pkey = EVP_PKCS82PKEY(PKCS12_SAFEBAG_get0_p8inf(bag)); if (*pkey == NULL) return 0; break; @@ -216,10 +208,9 @@ static int parse_bag(PKCS12_SAFEBAG *bag, const char *pass, int passlen, case NID_pkcs8ShroudedKeyBag: if (pkey == NULL || *pkey != NULL) return 1; - if ((p8 = PKCS12_decrypt_skey_ex(bag, pass, passlen, - libctx, propq)) == NULL) + if ((p8 = PKCS12_decrypt_skey(bag, pass, passlen)) == NULL) return 0; - *pkey = EVP_PKCS82PKEY_ex(p8, libctx, propq); + *pkey = EVP_PKCS82PKEY(p8); PKCS8_PRIV_KEY_INFO_free(p8); if (!(*pkey)) return 0; @@ -229,7 +220,7 @@ static int parse_bag(PKCS12_SAFEBAG *bag, const char *pass, int passlen, if (ocerts == NULL || PKCS12_SAFEBAG_get_bag_nid(bag) != NID_x509Certificate) return 1; - if ((x509 = PKCS12_SAFEBAG_get1_cert_ex(bag, libctx, propq)) == NULL) + if ((x509 = PKCS12_SAFEBAG_get1_cert(bag)) == NULL) return 0; if (lkid && !X509_keyid_set1(x509, lkid->data, lkid->length)) { X509_free(x509); @@ -259,7 +250,7 @@ static int parse_bag(PKCS12_SAFEBAG *bag, const char *pass, int passlen, case NID_safeContentsBag: return parse_bags(PKCS12_SAFEBAG_get0_safes(bag), pass, passlen, pkey, - ocerts, libctx, propq); + ocerts); default: return 1; diff --git a/openssl/src/crypto/pkcs12/p12_local.h b/openssl/src/crypto/pkcs12/p12_local.h index 7f02874a9..acaa27b19 100644 --- a/openssl/src/crypto/pkcs12/p12_local.h +++ b/openssl/src/crypto/pkcs12/p12_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -41,5 +41,3 @@ struct pkcs12_bag_st { ASN1_TYPE *other; /* Secret or other bag */ } value; }; - -const PKCS7_CTX *ossl_pkcs12_get0_pkcs7ctx(const PKCS12 *p12); diff --git a/openssl/src/crypto/pkcs12/p12_mutl.c b/openssl/src/crypto/pkcs12/p12_mutl.c index 4091e61d9..118d598ea 100644 --- a/openssl/src/crypto/pkcs12/p12_mutl.c +++ b/openssl/src/crypto/pkcs12/p12_mutl.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -50,28 +50,6 @@ void PKCS12_get0_mac(const ASN1_OCTET_STRING **pmac, } } -#define TK26_MAC_KEY_LEN 32 - -static int pkcs12_gen_gost_mac_key(const char *pass, int passlen, - const unsigned char *salt, int saltlen, - int iter, int keylen, unsigned char *key, - const EVP_MD *digest) -{ - unsigned char out[96]; - - if (keylen != TK26_MAC_KEY_LEN) { - return 0; - } - - if (!PKCS5_PBKDF2_HMAC(pass, passlen, salt, saltlen, iter, - digest, sizeof(out), out)) { - return 0; - } - memcpy(key, out + sizeof(out) - TK26_MAC_KEY_LEN, TK26_MAC_KEY_LEN); - OPENSSL_cleanse(out, sizeof(out)); - return 1; -} - /* Generate a MAC */ static int pkcs12_gen_mac(PKCS12 *p12, const char *pass, int passlen, unsigned char *mac, unsigned int *maclen, @@ -89,7 +67,6 @@ static int pkcs12_gen_mac(PKCS12 *p12, const char *pass, int passlen, int saltlen, iter; char md_name[80]; int md_size = 0; - int md_nid; const X509_ALGOR *macalg; const ASN1_OBJECT *macoid; @@ -98,11 +75,6 @@ static int pkcs12_gen_mac(PKCS12 *p12, const char *pass, int passlen, return 0; } - if (p12->authsafes->d.data == NULL) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_DECODE_ERROR); - return 0; - } - salt = p12->mac->salt->data; saltlen = p12->mac->salt->length; if (p12->mac->iter == NULL) @@ -113,50 +85,32 @@ static int pkcs12_gen_mac(PKCS12 *p12, const char *pass, int passlen, X509_ALGOR_get0(&macoid, NULL, NULL, macalg); if (OBJ_obj2txt(md_name, sizeof(md_name), macoid, 0) < 0) return 0; - - (void)ERR_set_mark(); md = md_fetch = EVP_MD_fetch(p12->authsafes->ctx.libctx, md_name, p12->authsafes->ctx.propq); if (md == NULL) md = EVP_get_digestbynid(OBJ_obj2nid(macoid)); if (md == NULL) { - (void)ERR_clear_last_mark(); ERR_raise(ERR_LIB_PKCS12, PKCS12_R_UNKNOWN_DIGEST_ALGORITHM); return 0; } - (void)ERR_pop_to_mark(); - md_size = EVP_MD_get_size(md); - md_nid = EVP_MD_get_type(md); if (md_size < 0) goto err; - if ((md_nid == NID_id_GostR3411_94 - || md_nid == NID_id_GostR3411_2012_256 - || md_nid == NID_id_GostR3411_2012_512) - && ossl_safe_getenv("LEGACY_GOST_PKCS12") == NULL) { - md_size = TK26_MAC_KEY_LEN; - if (!pkcs12_gen_gost_mac_key(pass, passlen, salt, saltlen, iter, - md_size, key, md)) { + if (pkcs12_key_gen != NULL) { + if (!(*pkcs12_key_gen)(pass, passlen, salt, saltlen, PKCS12_MAC_ID, + iter, md_size, key, md)) { ERR_raise(ERR_LIB_PKCS12, PKCS12_R_KEY_GEN_ERROR); goto err; } } else { - if (pkcs12_key_gen != NULL) { - if (!(*pkcs12_key_gen)(pass, passlen, salt, saltlen, PKCS12_MAC_ID, - iter, md_size, key, md)) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_KEY_GEN_ERROR); - goto err; - } - } else { - /* Default to UTF-8 password */ - if (!PKCS12_key_gen_utf8_ex(pass, passlen, salt, saltlen, PKCS12_MAC_ID, - iter, md_size, key, md, - p12->authsafes->ctx.libctx, - p12->authsafes->ctx.propq)) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_KEY_GEN_ERROR); - goto err; - } + /* Default to UTF-8 password */ + if (!PKCS12_key_gen_utf8_ex(pass, passlen, salt, saltlen, PKCS12_MAC_ID, + iter, md_size, key, md, + p12->authsafes->ctx.libctx, + p12->authsafes->ctx.propq)) { + ERR_raise(ERR_LIB_PKCS12, PKCS12_R_KEY_GEN_ERROR); + goto err; } } if ((hmac = HMAC_CTX_new()) == NULL @@ -251,11 +205,11 @@ int PKCS12_setup_mac(PKCS12 *p12, int iter, unsigned char *salt, int saltlen, return PKCS12_ERROR; if (iter > 1) { if ((p12->mac->iter = ASN1_INTEGER_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return 0; } if (!ASN1_INTEGER_set(p12->mac->iter, iter)) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return 0; } } @@ -263,8 +217,10 @@ int PKCS12_setup_mac(PKCS12 *p12, int iter, unsigned char *salt, int saltlen, saltlen = PKCS12_SALT_LEN; else if (saltlen < 0) return 0; - if ((p12->mac->salt->data = OPENSSL_malloc(saltlen)) == NULL) + if ((p12->mac->salt->data = OPENSSL_malloc(saltlen)) == NULL) { + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return 0; + } p12->mac->salt->length = saltlen; if (salt == NULL) { if (RAND_bytes_ex(p12->authsafes->ctx.libctx, p12->mac->salt->data, @@ -276,7 +232,7 @@ int PKCS12_setup_mac(PKCS12 *p12, int iter, unsigned char *salt, int saltlen, X509_SIG_getm(p12->mac->dinfo, &macalg, NULL); if (!X509_ALGOR_set0(macalg, OBJ_nid2obj(EVP_MD_get_type(md_type)), V_ASN1_NULL, NULL)) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return 0; } diff --git a/openssl/src/crypto/pkcs12/p12_npas.c b/openssl/src/crypto/pkcs12/p12_npas.c index 78be2b5a8..62230bc61 100644 --- a/openssl/src/crypto/pkcs12/p12_npas.c +++ b/openssl/src/crypto/pkcs12/p12_npas.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,13 +19,11 @@ static int newpass_p12(PKCS12 *p12, const char *oldpass, const char *newpass); static int newpass_bags(STACK_OF(PKCS12_SAFEBAG) *bags, const char *oldpass, - const char *newpass, - OSSL_LIB_CTX *libctx, const char *propq); + const char *newpass); static int newpass_bag(PKCS12_SAFEBAG *bag, const char *oldpass, - const char *newpass, - OSSL_LIB_CTX *libctx, const char *propq); + const char *newpass); static int alg_get(const X509_ALGOR *alg, int *pnid, int *piter, - int *psaltlen, int *cipherid); + int *psaltlen); /* * Change the password on a PKCS#12 structure. @@ -41,12 +39,12 @@ int PKCS12_newpass(PKCS12 *p12, const char *oldpass, const char *newpass) } /* Check the mac */ - if (p12->mac != NULL) { - if (!PKCS12_verify_mac(p12, oldpass, -1)) { - ERR_raise(ERR_LIB_PKCS12, PKCS12_R_MAC_VERIFY_FAILURE); - return 0; - } + + if (!PKCS12_verify_mac(p12, oldpass, -1)) { + ERR_raise(ERR_LIB_PKCS12, PKCS12_R_MAC_VERIFY_FAILURE); + return 0; } + if (!newpass_p12(p12, oldpass, newpass)) { ERR_raise(ERR_LIB_PKCS12, PKCS12_R_PARSE_ERROR); return 0; @@ -61,7 +59,7 @@ static int newpass_p12(PKCS12 *p12, const char *oldpass, const char *newpass) { STACK_OF(PKCS7) *asafes = NULL, *newsafes = NULL; STACK_OF(PKCS12_SAFEBAG) *bags = NULL; - int i, bagnid, pbe_nid = 0, pbe_iter = 0, pbe_saltlen = 0, cipherid = NID_undef; + int i, bagnid, pbe_nid = 0, pbe_iter = 0, pbe_saltlen = 0; PKCS7 *p7, *p7new; ASN1_OCTET_STRING *p12_data_tmp = NULL, *macoct = NULL; unsigned char mac[EVP_MAX_MD_SIZE]; @@ -74,31 +72,27 @@ static int newpass_p12(PKCS12 *p12, const char *oldpass, const char *newpass) goto err; for (i = 0; i < sk_PKCS7_num(asafes); i++) { p7 = sk_PKCS7_value(asafes, i); - bagnid = OBJ_obj2nid(p7->type); if (bagnid == NID_pkcs7_data) { bags = PKCS12_unpack_p7data(p7); } else if (bagnid == NID_pkcs7_encrypted) { bags = PKCS12_unpack_p7encdata(p7, oldpass, -1); - if (p7->d.encrypted == NULL - || !alg_get(p7->d.encrypted->enc_data->algorithm, - &pbe_nid, &pbe_iter, &pbe_saltlen, &cipherid)) + if (!alg_get(p7->d.encrypted->enc_data->algorithm, + &pbe_nid, &pbe_iter, &pbe_saltlen)) goto err; } else { continue; } if (bags == NULL) goto err; - if (!newpass_bags(bags, oldpass, newpass, - p7->ctx.libctx, p7->ctx.propq)) + if (!newpass_bags(bags, oldpass, newpass)) goto err; /* Repack bag in same form with new password */ if (bagnid == NID_pkcs7_data) p7new = PKCS12_pack_p7data(bags); else - p7new = PKCS12_pack_p7encdata_ex(pbe_nid, newpass, -1, NULL, - pbe_saltlen, pbe_iter, bags, - p7->ctx.libctx, p7->ctx.propq); + p7new = PKCS12_pack_p7encdata(pbe_nid, newpass, -1, NULL, + pbe_saltlen, pbe_iter, bags); if (p7new == NULL || !sk_PKCS7_push(newsafes, p7new)) goto err; sk_PKCS12_SAFEBAG_pop_free(bags, PKCS12_SAFEBAG_free); @@ -113,13 +107,11 @@ static int newpass_p12(PKCS12 *p12, const char *oldpass, const char *newpass) if (!PKCS12_pack_authsafes(p12, newsafes)) goto err; - if (p12->mac != NULL) { - if (!PKCS12_gen_mac(p12, newpass, -1, mac, &maclen)) - goto err; - X509_SIG_getm(p12->mac->dinfo, NULL, &macoct); - if (!ASN1_OCTET_STRING_set(macoct, mac, maclen)) - goto err; - } + if (!PKCS12_gen_mac(p12, newpass, -1, mac, &maclen)) + goto err; + X509_SIG_getm(p12->mac->dinfo, NULL, &macoct); + if (!ASN1_OCTET_STRING_set(macoct, mac, maclen)) + goto err; rv = 1; @@ -138,13 +130,11 @@ static int newpass_p12(PKCS12 *p12, const char *oldpass, const char *newpass) } static int newpass_bags(STACK_OF(PKCS12_SAFEBAG) *bags, const char *oldpass, - const char *newpass, - OSSL_LIB_CTX *libctx, const char *propq) + const char *newpass) { int i; for (i = 0; i < sk_PKCS12_SAFEBAG_num(bags); i++) { - if (!newpass_bag(sk_PKCS12_SAFEBAG_value(bags, i), oldpass, newpass, - libctx, propq)) + if (!newpass_bag(sk_PKCS12_SAFEBAG_value(bags, i), oldpass, newpass)) return 0; } return 1; @@ -153,37 +143,26 @@ static int newpass_bags(STACK_OF(PKCS12_SAFEBAG) *bags, const char *oldpass, /* Change password of safebag: only needs handle shrouded keybags */ static int newpass_bag(PKCS12_SAFEBAG *bag, const char *oldpass, - const char *newpass, - OSSL_LIB_CTX *libctx, const char *propq) + const char *newpass) { - EVP_CIPHER *cipher = NULL; PKCS8_PRIV_KEY_INFO *p8; X509_SIG *p8new; - int p8_nid, p8_saltlen, p8_iter, cipherid = 0; + int p8_nid, p8_saltlen, p8_iter; const X509_ALGOR *shalg; if (PKCS12_SAFEBAG_get_nid(bag) != NID_pkcs8ShroudedKeyBag) return 1; - if ((p8 = PKCS8_decrypt_ex(bag->value.shkeybag, oldpass, -1, - libctx, propq)) == NULL) + if ((p8 = PKCS8_decrypt(bag->value.shkeybag, oldpass, -1)) == NULL) return 0; X509_SIG_get0(bag->value.shkeybag, &shalg, NULL); - if (!alg_get(shalg, &p8_nid, &p8_iter, &p8_saltlen, &cipherid)) { + if (!alg_get(shalg, &p8_nid, &p8_iter, &p8_saltlen)) { PKCS8_PRIV_KEY_INFO_free(p8); return 0; } - if (cipherid != NID_undef) { - cipher = EVP_CIPHER_fetch(libctx, OBJ_nid2sn(cipherid), propq); - if (cipher == NULL) { - PKCS8_PRIV_KEY_INFO_free(p8); - return 0; - } - } - p8new = PKCS8_encrypt_ex(p8_nid, cipher, newpass, -1, NULL, p8_saltlen, - p8_iter, p8, libctx, propq); + p8new = PKCS8_encrypt(p8_nid, NULL, newpass, -1, NULL, p8_saltlen, + p8_iter, p8); PKCS8_PRIV_KEY_INFO_free(p8); - EVP_CIPHER_free(cipher); if (p8new == NULL) return 0; X509_SIG_free(bag->value.shkeybag); @@ -192,69 +171,16 @@ static int newpass_bag(PKCS12_SAFEBAG *bag, const char *oldpass, } static int alg_get(const X509_ALGOR *alg, int *pnid, int *piter, - int *psaltlen, int *cipherid) + int *psaltlen) { - int ret = 0, pbenid, aparamtype; - int encnid, prfnid; - const ASN1_OBJECT *aoid; - const void *aparam; - PBEPARAM *pbe = NULL; - PBE2PARAM *pbe2 = NULL; - PBKDF2PARAM *kdf = NULL; - - X509_ALGOR_get0(&aoid, &aparamtype, &aparam, alg); - pbenid = OBJ_obj2nid(aoid); - - switch (pbenid) { - case NID_pbes2: - if (aparamtype == V_ASN1_SEQUENCE) - pbe2 = ASN1_item_unpack(aparam, ASN1_ITEM_rptr(PBE2PARAM)); - if (pbe2 == NULL) - goto done; - - X509_ALGOR_get0(&aoid, &aparamtype, &aparam, pbe2->keyfunc); - pbenid = OBJ_obj2nid(aoid); - X509_ALGOR_get0(&aoid, NULL, NULL, pbe2->encryption); - encnid = OBJ_obj2nid(aoid); - - if (aparamtype == V_ASN1_SEQUENCE) - kdf = ASN1_item_unpack(aparam, ASN1_ITEM_rptr(PBKDF2PARAM)); - if (kdf == NULL) - goto done; + PBEPARAM *pbe; - /* Only OCTET_STRING is supported */ - if (kdf->salt->type != V_ASN1_OCTET_STRING) - goto done; - - if (kdf->prf == NULL) { - prfnid = NID_hmacWithSHA1; - } else { - X509_ALGOR_get0(&aoid, NULL, NULL, kdf->prf); - prfnid = OBJ_obj2nid(aoid); - } - *psaltlen = kdf->salt->value.octet_string->length; - *piter = ASN1_INTEGER_get(kdf->iter); - *pnid = prfnid; - *cipherid = encnid; - break; - default: - pbe = ASN1_TYPE_unpack_sequence(ASN1_ITEM_rptr(PBEPARAM), alg->parameter); - if (pbe == NULL) - goto done; - *pnid = OBJ_obj2nid(alg->algorithm); - *piter = ASN1_INTEGER_get(pbe->iter); - *psaltlen = pbe->salt->length; - *cipherid = NID_undef; - ret = 1; - break; - } - ret = 1; -done: - if (kdf != NULL) - PBKDF2PARAM_free(kdf); - if (pbe2 != NULL) - PBE2PARAM_free(pbe2); - if (pbe != NULL) - PBEPARAM_free(pbe); - return ret; + pbe = ASN1_TYPE_unpack_sequence(ASN1_ITEM_rptr(PBEPARAM), alg->parameter); + if (pbe == NULL) + return 0; + *pnid = OBJ_obj2nid(alg->algorithm); + *piter = ASN1_INTEGER_get(pbe->iter); + *psaltlen = pbe->salt->length; + PBEPARAM_free(pbe); + return 1; } diff --git a/openssl/src/crypto/pkcs12/p12_p8e.c b/openssl/src/crypto/pkcs12/p12_p8e.c index 1230c8c88..9c2753401 100644 --- a/openssl/src/crypto/pkcs12/p12_p8e.c +++ b/openssl/src/crypto/pkcs12/p12_p8e.c @@ -84,6 +84,7 @@ X509_SIG *PKCS8_set0_pbe_ex(const char *pass, int passlen, p8 = OPENSSL_zalloc(sizeof(*p8)); if (p8 == NULL) { + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); ASN1_OCTET_STRING_free(enckey); return NULL; } diff --git a/openssl/src/crypto/pkcs12/p12_sbag.c b/openssl/src/crypto/pkcs12/p12_sbag.c index 04ef0b74e..7574c5412 100644 --- a/openssl/src/crypto/pkcs12/p12_sbag.c +++ b/openssl/src/crypto/pkcs12/p12_sbag.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,7 +11,6 @@ #include "internal/cryptlib.h" #include #include "p12_local.h" -#include "crypto/x509.h" #ifndef OPENSSL_NO_DEPRECATED_1_1_0 ASN1_TYPE *PKCS12_get_attr(const PKCS12_SAFEBAG *bag, int attr_nid) @@ -102,42 +101,6 @@ X509_CRL *PKCS12_SAFEBAG_get1_crl(const PKCS12_SAFEBAG *bag) ASN1_ITEM_rptr(X509_CRL)); } -X509 *PKCS12_SAFEBAG_get1_cert_ex(const PKCS12_SAFEBAG *bag, - OSSL_LIB_CTX *libctx, const char *propq) -{ - X509 *ret = NULL; - - if (PKCS12_SAFEBAG_get_nid(bag) != NID_certBag) - return NULL; - if (OBJ_obj2nid(bag->value.bag->type) != NID_x509Certificate) - return NULL; - ret = ASN1_item_unpack_ex(bag->value.bag->value.octet, - ASN1_ITEM_rptr(X509), libctx, propq); - if (!ossl_x509_set0_libctx(ret, libctx, propq)) { - X509_free(ret); - return NULL; - } - return ret; -} - -X509_CRL *PKCS12_SAFEBAG_get1_crl_ex(const PKCS12_SAFEBAG *bag, - OSSL_LIB_CTX *libctx, const char *propq) -{ - X509_CRL *ret = NULL; - - if (PKCS12_SAFEBAG_get_nid(bag) != NID_crlBag) - return NULL; - if (OBJ_obj2nid(bag->value.bag->type) != NID_x509Crl) - return NULL; - ret = ASN1_item_unpack_ex(bag->value.bag->value.octet, - ASN1_ITEM_rptr(X509_CRL), libctx, propq); - if (!ossl_x509_crl_set0_libctx(ret, libctx, propq)) { - X509_CRL_free(ret); - return NULL; - } - return ret; -} - PKCS12_SAFEBAG *PKCS12_SAFEBAG_create_cert(X509 *x509) { return PKCS12_item_pack_safebag(x509, ASN1_ITEM_rptr(X509), @@ -156,18 +119,18 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create_secret(int type, int vtype, const unsigned PKCS12_SAFEBAG *safebag; if ((bag = PKCS12_BAGS_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; } bag->type = OBJ_nid2obj(type); - switch (vtype) { + switch(vtype) { case V_ASN1_OCTET_STRING: { ASN1_OCTET_STRING *strtmp = ASN1_OCTET_STRING_new(); if (strtmp == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } /* Pack data into an octet string */ @@ -179,7 +142,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create_secret(int type, int vtype, const unsigned bag->value.other = ASN1_TYPE_new(); if (bag->value.other == NULL) { ASN1_OCTET_STRING_free(strtmp); - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } ASN1_TYPE_set(bag->value.other, vtype, strtmp); @@ -192,13 +155,13 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create_secret(int type, int vtype, const unsigned } if ((safebag = PKCS12_SAFEBAG_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); goto err; } safebag->value.bag = bag; safebag->type = OBJ_nid2obj(NID_secretBag); return safebag; - + err: PKCS12_BAGS_free(bag); return NULL; @@ -211,7 +174,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create0_p8inf(PKCS8_PRIV_KEY_INFO *p8) PKCS12_SAFEBAG *bag = PKCS12_SAFEBAG_new(); if (bag == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; } bag->type = OBJ_nid2obj(NID_keyBag); @@ -227,7 +190,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create0_pkcs8(X509_SIG *p8) /* Set up the safe bag */ if (bag == NULL) { - ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; } bag->type = OBJ_nid2obj(NID_pkcs8ShroudedKeyBag); diff --git a/openssl/src/crypto/pkcs12/p12_utl.c b/openssl/src/crypto/pkcs12/p12_utl.c index a96623f19..3afc8b2f1 100644 --- a/openssl/src/crypto/pkcs12/p12_utl.c +++ b/openssl/src/crypto/pkcs12/p12_utl.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,8 +10,6 @@ #include #include "internal/cryptlib.h" #include -#include "p12_local.h" -#include "crypto/pkcs7/pk7_local.h" /* Cheap and nasty Unicode stuff */ @@ -26,8 +24,10 @@ unsigned char *OPENSSL_asc2uni(const char *asc, int asclen, if (asclen < 0) return NULL; ulen = asclen * 2 + 2; - if ((unitmp = OPENSSL_malloc(ulen)) == NULL) + if ((unitmp = OPENSSL_malloc(ulen)) == NULL) { + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; + } for (i = 0; i < ulen - 2; i += 2) { unitmp[i] = 0; unitmp[i + 1] = asc[i >> 1]; @@ -57,8 +57,10 @@ char *OPENSSL_uni2asc(const unsigned char *uni, int unilen) if (!unilen || uni[unilen - 1]) asclen++; uni++; - if ((asctmp = OPENSSL_malloc(asclen)) == NULL) + if ((asctmp = OPENSSL_malloc(asclen)) == NULL) { + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; + } for (i = 0; i < unilen; i += 2) asctmp[i >> 1] = uni[i]; asctmp[asclen - 1] = 0; @@ -117,8 +119,10 @@ unsigned char *OPENSSL_utf82uni(const char *asc, int asclen, ulen += 2; /* for trailing UTF16 zero */ - if ((ret = OPENSSL_malloc(ulen)) == NULL) + if ((ret = OPENSSL_malloc(ulen)) == NULL) { + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; + } /* re-run the loop writing down UTF-16 characters in big-endian order */ for (unitmp = ret, i = 0; i < asclen; i += j) { j = UTF8_getc((const unsigned char *)asc+i, asclen-i, &utf32chr); @@ -200,8 +204,10 @@ char *OPENSSL_uni2utf8(const unsigned char *uni, int unilen) if (!unilen || (uni[unilen-2]||uni[unilen - 1])) asclen++; - if ((asctmp = OPENSSL_malloc(asclen)) == NULL) + if ((asctmp = OPENSSL_malloc(asclen)) == NULL) { + ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE); return NULL; + } /* re-run the loop emitting UTF-8 string */ for (asclen = 0, i = 0; i < unilen; ) { @@ -232,34 +238,12 @@ int i2d_PKCS12_fp(FILE *fp, const PKCS12 *p12) PKCS12 *d2i_PKCS12_bio(BIO *bp, PKCS12 **p12) { - OSSL_LIB_CTX *libctx = NULL; - const char *propq = NULL; - const PKCS7_CTX *p7ctx = NULL; - - if (p12 != NULL) { - p7ctx = ossl_pkcs12_get0_pkcs7ctx(*p12); - if (p7ctx != NULL) { - libctx = ossl_pkcs7_ctx_get0_libctx(p7ctx); - propq = ossl_pkcs7_ctx_get0_propq(p7ctx); - } - } - return ASN1_item_d2i_bio_ex(ASN1_ITEM_rptr(PKCS12), bp, p12, libctx, propq); + return ASN1_item_d2i_bio(ASN1_ITEM_rptr(PKCS12), bp, p12); } #ifndef OPENSSL_NO_STDIO PKCS12 *d2i_PKCS12_fp(FILE *fp, PKCS12 **p12) { - OSSL_LIB_CTX *libctx = NULL; - const char *propq = NULL; - const PKCS7_CTX *p7ctx = NULL; - - if (p12 != NULL) { - p7ctx = ossl_pkcs12_get0_pkcs7ctx(*p12); - if (p7ctx != NULL) { - libctx = ossl_pkcs7_ctx_get0_libctx(p7ctx); - propq = ossl_pkcs7_ctx_get0_propq(p7ctx); - } - } - return ASN1_item_d2i_fp_ex(ASN1_ITEM_rptr(PKCS12), fp, p12, libctx, propq); + return ASN1_item_d2i_fp(ASN1_ITEM_rptr(PKCS12), fp, p12); } #endif diff --git a/openssl/src/crypto/pkcs12/pk12err.c b/openssl/src/crypto/pkcs12/pk12err.c index e9bcaf4b6..6e3ec78cd 100644 --- a/openssl/src/crypto/pkcs12/pk12err.c +++ b/openssl/src/crypto/pkcs12/pk12err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,9 +15,8 @@ #ifndef OPENSSL_NO_ERR static const ERR_STRING_DATA PKCS12_str_reasons[] = { - {ERR_PACK(ERR_LIB_PKCS12, 0, PKCS12_R_CALLBACK_FAILED), "callback failed"}, {ERR_PACK(ERR_LIB_PKCS12, 0, PKCS12_R_CANT_PACK_STRUCTURE), - "can't pack structure"}, + "cant pack structure"}, {ERR_PACK(ERR_LIB_PKCS12, 0, PKCS12_R_CONTENT_TYPE_NOT_DATA), "content type not data"}, {ERR_PACK(ERR_LIB_PKCS12, 0, PKCS12_R_DECODE_ERROR), "decode error"}, diff --git a/openssl/src/crypto/pkcs7/bio_pk7.c b/openssl/src/crypto/pkcs7/bio_pk7.c index 36f4d6196..414f0da1c 100644 --- a/openssl/src/crypto/pkcs7/bio_pk7.c +++ b/openssl/src/crypto/pkcs7/bio_pk7.c @@ -11,6 +11,11 @@ #include #include +#if !defined(OPENSSL_SYS_VXWORKS) +# include +#endif +#include + /* Streaming encode support for PKCS#7 */ BIO *BIO_new_PKCS7(BIO *out, PKCS7 *p7) diff --git a/openssl/src/crypto/pkcs7/pk7_asn1.c b/openssl/src/crypto/pkcs7/pk7_asn1.c index 3abcc3dc8..1cd867721 100644 --- a/openssl/src/crypto/pkcs7/pk7_asn1.c +++ b/openssl/src/crypto/pkcs7/pk7_asn1.c @@ -41,7 +41,7 @@ static int pk7_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, case ASN1_OP_STREAM_PRE: if (PKCS7_stream(&sarg->boundary, *pp7) <= 0) return 0; - /* fall through */ + /* fall thru */ case ASN1_OP_DETACHED_PRE: sarg->ndef_bio = PKCS7_dataInit(*pp7, sarg->out); if (!sarg->ndef_bio) @@ -104,6 +104,7 @@ PKCS7 *PKCS7_new_ex(OSSL_LIB_CTX *libctx, const char *propq) if (pkcs7->ctx.propq == NULL) { PKCS7_free(pkcs7); pkcs7 = NULL; + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); } } } diff --git a/openssl/src/crypto/pkcs7/pk7_attr.c b/openssl/src/crypto/pkcs7/pk7_attr.c index a12d65bb8..e9904c595 100644 --- a/openssl/src/crypto/pkcs7/pk7_attr.c +++ b/openssl/src/crypto/pkcs7/pk7_attr.c @@ -23,17 +23,13 @@ int PKCS7_add_attrib_smimecap(PKCS7_SIGNER_INFO *si, ASN1_STRING *seq; if ((seq = ASN1_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } seq->length = ASN1_item_i2d((ASN1_VALUE *)cap, &seq->data, ASN1_ITEM_rptr(X509_ALGORS)); - if (!PKCS7_add_signed_attribute(si, NID_SMIMECapabilities, - V_ASN1_SEQUENCE, seq)) { - ASN1_STRING_free(seq); - return 0; - } - return 1; + return PKCS7_add_signed_attribute(si, NID_SMIMECapabilities, + V_ASN1_SEQUENCE, seq); } STACK_OF(X509_ALGOR) *PKCS7_get_smimecap(PKCS7_SIGNER_INFO *si) @@ -57,22 +53,19 @@ int PKCS7_simple_smimecap(STACK_OF(X509_ALGOR) *sk, int nid, int arg) X509_ALGOR *alg; if ((alg = X509_ALGOR_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } ASN1_OBJECT_free(alg->algorithm); alg->algorithm = OBJ_nid2obj(nid); if (arg > 0) { if ((alg->parameter = ASN1_TYPE_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); goto err; } if ((nbit = ASN1_INTEGER_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); goto err; } if (!ASN1_INTEGER_set(nbit, arg)) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); goto err; } alg->parameter->value.integer = nbit; @@ -80,11 +73,11 @@ int PKCS7_simple_smimecap(STACK_OF(X509_ALGOR) *sk, int nid, int arg) nbit = NULL; } if (!sk_X509_ALGOR_push(sk, alg)) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_CRYPTO_LIB); goto err; } return 1; err: + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); ASN1_INTEGER_free(nbit); X509_ALGOR_free(alg); return 0; @@ -102,18 +95,12 @@ int PKCS7_add_attrib_content_type(PKCS7_SIGNER_INFO *si, ASN1_OBJECT *coid) int PKCS7_add0_attrib_signing_time(PKCS7_SIGNER_INFO *si, ASN1_TIME *t) { - ASN1_TIME *tmp = NULL; - - if (t == NULL && (tmp = t = X509_gmtime_adj(NULL, 0)) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_X509_LIB); + if (t == NULL && (t = X509_gmtime_adj(NULL, 0)) == NULL) { + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } - if (!PKCS7_add_signed_attribute(si, NID_pkcs9_signingTime, - V_ASN1_UTCTIME, t)) { - ASN1_TIME_free(tmp); - return 0; - } - return 1; + return PKCS7_add_signed_attribute(si, NID_pkcs9_signingTime, + V_ASN1_UTCTIME, t); } int PKCS7_add1_attrib_digest(PKCS7_SIGNER_INFO *si, diff --git a/openssl/src/crypto/pkcs7/pk7_doit.c b/openssl/src/crypto/pkcs7/pk7_doit.c index c753a0880..5836e50f2 100644 --- a/openssl/src/crypto/pkcs7/pk7_doit.c +++ b/openssl/src/crypto/pkcs7/pk7_doit.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,7 +15,6 @@ #include #include "internal/cryptlib.h" #include "internal/sizes.h" -#include "crypto/evp.h" #include "pk7_local.h" static int add_attribute(STACK_OF(X509_ATTRIBUTE) **sk, int nid, int atrtype, @@ -132,8 +131,11 @@ static int pkcs7_encode_rinfo(PKCS7_RECIP_INFO *ri, goto err; ek = OPENSSL_malloc(eklen); - if (ek == NULL) + + if (ek == NULL) { + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; + } if (EVP_PKEY_encrypt(pctx, ek, &eklen, key, keylen) <= 0) goto err; @@ -168,17 +170,25 @@ static int pkcs7_decrypt_rinfo(unsigned char **pek, int *peklen, if (EVP_PKEY_decrypt_init(pctx) <= 0) goto err; - if (EVP_PKEY_is_a(pkey, "RSA")) - /* upper layer pkcs7 code incorrectly assumes that a successful RSA - * decryption means that the key matches ciphertext (which never - * was the case, implicit rejection or not), so to make it work - * disable implicit rejection for RSA keys */ - EVP_PKEY_CTX_ctrl_str(pctx, "rsa_pkcs1_implicit_rejection", "0"); + if (EVP_PKEY_decrypt(pctx, NULL, &eklen, + ri->enc_key->data, ri->enc_key->length) <= 0) + goto err; + + ek = OPENSSL_malloc(eklen); - ret = evp_pkey_decrypt_alloc(pctx, &ek, &eklen, fixlen, - ri->enc_key->data, ri->enc_key->length); - if (ret <= 0) + if (ek == NULL) { + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; + } + + if (EVP_PKEY_decrypt(pctx, ek, &eklen, + ri->enc_key->data, ri->enc_key->length) <= 0 + || eklen == 0 + || (fixlen != 0 && eklen != fixlen)) { + ret = 0; + ERR_raise(ERR_LIB_PKCS7, ERR_R_EVP_LIB); + goto err; + } ret = 1; @@ -324,7 +334,7 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio) if (xalg->parameter == NULL) goto err; } - if (EVP_CIPHER_param_to_asn1(ctx, xalg->parameter) <= 0) + if (EVP_CIPHER_param_to_asn1(ctx, xalg->parameter) < 0) goto err; } @@ -586,7 +596,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) BIO_get_cipher_ctx(etmp, &evp_ctx); if (EVP_CipherInit_ex(evp_ctx, cipher, NULL, NULL, NULL, 0) <= 0) goto err; - if (EVP_CIPHER_asn1_to_param(evp_ctx, enc_alg->parameter) <= 0) + if (EVP_CIPHER_asn1_to_param(evp_ctx, enc_alg->parameter) < 0) goto err; /* Generate random key as MMA defence */ len = EVP_CIPHER_CTX_get_key_length(evp_ctx); @@ -610,7 +620,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) * length. The key length is determined by the size of the * decrypted RSA key. */ - if (EVP_CIPHER_CTX_set_key_length(evp_ctx, eklen) <= 0) { + if (!EVP_CIPHER_CTX_set_key_length(evp_ctx, eklen)) { /* Use random key as MMA defence */ OPENSSL_clear_free(ek, eklen); ek = tkey; @@ -692,7 +702,7 @@ static int do_pkcs7_signed_attrib(PKCS7_SIGNER_INFO *si, EVP_MD_CTX *mctx) /* Add signing time if not already present */ if (!PKCS7_get_signed_attribute(si, NID_pkcs9_signingTime)) { if (!PKCS7_add0_attrib_signing_time(si, NULL)) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } } @@ -703,7 +713,7 @@ static int do_pkcs7_signed_attrib(PKCS7_SIGNER_INFO *si, EVP_MD_CTX *mctx) return 0; } if (!PKCS7_add1_attrib_digest(si, md_data, md_len)) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } @@ -740,7 +750,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio) ctx_tmp = EVP_MD_CTX_new(); if (ctx_tmp == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } @@ -758,7 +768,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio) if (os == NULL) { os = ASN1_OCTET_STRING_new(); if (os == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; } p7->d.signed_and_enveloped->enc_data->enc_data = os; @@ -770,7 +780,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio) if (os == NULL) { os = ASN1_OCTET_STRING_new(); if (os == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; } p7->d.enveloped->enc_data->enc_data = os; @@ -834,9 +844,10 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio) goto err; } else { unsigned char *abuf = NULL; - unsigned int abuflen = EVP_PKEY_get_size(si->pkey); - - if (abuflen == 0 || (abuf = OPENSSL_malloc(abuflen)) == NULL) + unsigned int abuflen; + abuflen = EVP_PKEY_get_size(si->pkey); + abuf = OPENSSL_malloc(abuflen); + if (abuf == NULL) goto err; if (!EVP_SignFinal_ex(ctx_tmp, abuf, &abuflen, si->pkey, @@ -908,7 +919,7 @@ int PKCS7_SIGNER_INFO_sign(PKCS7_SIGNER_INFO *si) mctx = EVP_MD_CTX_new(); if (mctx == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; } @@ -946,15 +957,13 @@ int PKCS7_SIGNER_INFO_sign(PKCS7_SIGNER_INFO *si) return 0; } -/* This partly overlaps with PKCS7_verify(). It does not support flags. */ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si) { PKCS7_ISSUER_AND_SERIAL *ias; int ret = 0, i; - STACK_OF(X509) *untrusted; - STACK_OF(X509_CRL) *crls; - X509 *signer; + STACK_OF(X509) *cert; + X509 *x509; if (p7 == NULL) { ERR_raise(ERR_LIB_PKCS7, PKCS7_R_INVALID_NULL_POINTER); @@ -967,30 +976,26 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, } if (PKCS7_type_is_signed(p7)) { - untrusted = p7->d.sign->cert; - crls = p7->d.sign->crl; + cert = p7->d.sign->cert; } else if (PKCS7_type_is_signedAndEnveloped(p7)) { - untrusted = p7->d.signed_and_enveloped->cert; - crls = p7->d.signed_and_enveloped->crl; + cert = p7->d.signed_and_enveloped->cert; } else { ERR_raise(ERR_LIB_PKCS7, PKCS7_R_WRONG_PKCS7_TYPE); goto err; } - X509_STORE_CTX_set0_crls(ctx, crls); - /* XXXXXXXXXXXXXXXXXXXXXXX */ ias = si->issuer_and_serial; - signer = X509_find_by_issuer_and_serial(untrusted, ias->issuer, ias->serial); + x509 = X509_find_by_issuer_and_serial(cert, ias->issuer, ias->serial); - /* Were we able to find the signer certificate in passed to us? */ - if (signer == NULL) { + /* were we able to find the cert in passed to us */ + if (x509 == NULL) { ERR_raise(ERR_LIB_PKCS7, PKCS7_R_UNABLE_TO_FIND_CERTIFICATE); goto err; } /* Lets verify */ - if (!X509_STORE_CTX_init(ctx, cert_store, signer, untrusted)) { + if (!X509_STORE_CTX_init(ctx, cert_store, x509, cert)) { ERR_raise(ERR_LIB_PKCS7, ERR_R_X509_LIB); goto err; } @@ -1001,13 +1006,13 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, goto err; } - return PKCS7_signatureVerify(bio, p7, si, signer); + return PKCS7_signatureVerify(bio, p7, si, x509); err: return ret; } int PKCS7_signatureVerify(BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si, - X509 *signer) + X509 *x509) { ASN1_OCTET_STRING *os; EVP_MD_CTX *mdc_tmp, *mdc; @@ -1024,7 +1029,7 @@ int PKCS7_signatureVerify(BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si, mdc_tmp = EVP_MD_CTX_new(); if (mdc_tmp == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; } @@ -1114,7 +1119,7 @@ int PKCS7_signatureVerify(BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si, } os = si->enc_digest; - pkey = X509_get0_pubkey(signer); + pkey = X509_get0_pubkey(x509); if (pkey == NULL) { ret = -1; goto err; @@ -1165,11 +1170,11 @@ ASN1_TYPE *PKCS7_get_attribute(const PKCS7_SIGNER_INFO *si, int nid) static ASN1_TYPE *get_attribute(const STACK_OF(X509_ATTRIBUTE) *sk, int nid) { - int idx = X509at_get_attr_by_NID(sk, nid, -1); - - if (idx < 0) - return NULL; - return X509_ATTRIBUTE_get0_type(X509at_get_attr(sk, idx), 0); + int idx; + X509_ATTRIBUTE *xa; + idx = X509at_get_attr_by_NID(sk, nid, -1); + xa = X509at_get_attr(sk, idx); + return X509_ATTRIBUTE_get0_type(xa, 0); } ASN1_OCTET_STRING *PKCS7_digest_from_attributes(STACK_OF(X509_ATTRIBUTE) *sk) diff --git a/openssl/src/crypto/pkcs7/pk7_lib.c b/openssl/src/crypto/pkcs7/pk7_lib.c index 7be292854..abfab9291 100644 --- a/openssl/src/crypto/pkcs7/pk7_lib.c +++ b/openssl/src/crypto/pkcs7/pk7_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -224,7 +224,7 @@ int PKCS7_add_signer(PKCS7 *p7, PKCS7_SIGNER_INFO *psi) if ((alg = X509_ALGOR_new()) == NULL || (alg->parameter = ASN1_TYPE_new()) == NULL) { X509_ALGOR_free(alg); - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } /* @@ -290,7 +290,7 @@ int PKCS7_add_crl(PKCS7 *p7, X509_CRL *crl) if (*sk == NULL) *sk = sk_X509_CRL_new_null(); if (*sk == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } @@ -305,7 +305,7 @@ int PKCS7_add_crl(PKCS7 *p7, X509_CRL *crl) static int pkcs7_ecdsa_or_dsa_sign_verify_setup(PKCS7_SIGNER_INFO *si, int verify) { - if (!verify) { + if (verify == 0) { int snid, hnid; X509_ALGOR *alg1, *alg2; EVP_PKEY *pkey = si->pkey; @@ -318,20 +318,19 @@ static int pkcs7_ecdsa_or_dsa_sign_verify_setup(PKCS7_SIGNER_INFO *si, return -1; if (!OBJ_find_sigid_by_algs(&snid, hnid, EVP_PKEY_get_id(pkey))) return -1; - return X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, NULL); + X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0); } return 1; } static int pkcs7_rsa_sign_verify_setup(PKCS7_SIGNER_INFO *si, int verify) { - if (!verify) { + if (verify == 0) { X509_ALGOR *alg = NULL; PKCS7_SIGNER_INFO_get0_algs(si, NULL, NULL, &alg); if (alg != NULL) - return X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), - V_ASN1_NULL, NULL); + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0); } return 1; } @@ -343,10 +342,10 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey, /* We now need to add another PKCS7_SIGNER_INFO entry */ if (!ASN1_INTEGER_set(p7i->version, 1)) - return 0; + goto err; if (!X509_NAME_set(&p7i->issuer_and_serial->issuer, X509_get_issuer_name(x509))) - return 0; + goto err; /* * because ASN1_INTEGER_set is used to set a 'long' we will do things the @@ -355,7 +354,7 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey, ASN1_INTEGER_free(p7i->issuer_and_serial->serial); if (!(p7i->issuer_and_serial->serial = ASN1_INTEGER_dup(X509_get0_serialNumber(x509)))) - return 0; + goto err; /* lets keep the pkey around for a while */ EVP_PKEY_up_ref(pkey); @@ -363,9 +362,8 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey, /* Set the algorithms */ - if (!X509_ALGOR_set0(p7i->digest_alg, OBJ_nid2obj(EVP_MD_get_type(dgst)), - V_ASN1_NULL, NULL)) - return 0; + X509_ALGOR_set0(p7i->digest_alg, OBJ_nid2obj(EVP_MD_get_type(dgst)), + V_ASN1_NULL, NULL); if (EVP_PKEY_is_a(pkey, "EC") || EVP_PKEY_is_a(pkey, "DSA")) return pkcs7_ecdsa_or_dsa_sign_verify_setup(p7i, 0); @@ -382,6 +380,7 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey, } } ERR_raise(ERR_LIB_PKCS7, PKCS7_R_SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE); + err: return 0; } @@ -403,7 +402,7 @@ PKCS7_SIGNER_INFO *PKCS7_add_signature(PKCS7 *p7, X509 *x509, EVP_PKEY *pkey, if ((si = PKCS7_SIGNER_INFO_new()) == NULL) goto err; - if (PKCS7_SIGNER_INFO_set(si, x509, pkey, dgst) <= 0) + if (!PKCS7_SIGNER_INFO_set(si, x509, pkey, dgst)) goto err; if (!PKCS7_add_signer(p7, si)) goto err; @@ -491,8 +490,10 @@ int ossl_pkcs7_set1_propq(PKCS7 *p7, const char *propq) } if (propq != NULL) { p7->ctx.propq = OPENSSL_strdup(propq); - if (p7->ctx.propq == NULL) + if (p7->ctx.propq == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } } return 1; } @@ -520,7 +521,7 @@ int PKCS7_set_digest(PKCS7 *p7, const EVP_MD *md) { if (PKCS7_type_is_digest(p7)) { if ((p7->d.digest->md->parameter = ASN1_TYPE_new()) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } p7->d.digest->md->parameter->type = V_ASN1_NULL; @@ -567,7 +568,7 @@ PKCS7_RECIP_INFO *PKCS7_add_recipient(PKCS7 *p7, X509 *x509) if ((ri = PKCS7_RECIP_INFO_new()) == NULL) goto err; - if (PKCS7_RECIP_INFO_set(ri, x509) <= 0) + if (!PKCS7_RECIP_INFO_set(ri, x509)) goto err; if (!PKCS7_add_recipient_info(p7, ri)) goto err; @@ -605,11 +606,10 @@ static int pkcs7_rsa_encrypt_decrypt_setup(PKCS7_RECIP_INFO *ri, int decrypt) { X509_ALGOR *alg = NULL; - if (!decrypt) { + if (decrypt == 0) { PKCS7_RECIP_INFO_get0_alg(ri, &alg); if (alg != NULL) - return X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), - V_ASN1_NULL, NULL); + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0); } return 1; } diff --git a/openssl/src/crypto/pkcs7/pk7_mime.c b/openssl/src/crypto/pkcs7/pk7_mime.c index d23f7a869..49a0da5f8 100644 --- a/openssl/src/crypto/pkcs7/pk7_mime.c +++ b/openssl/src/crypto/pkcs7/pk7_mime.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,13 +33,10 @@ int SMIME_write_PKCS7(BIO *bio, PKCS7 *p7, BIO *data, int flags) int ctype_nid = OBJ_obj2nid(p7->type); const PKCS7_CTX *ctx = ossl_pkcs7_get0_ctx(p7); - if (ctype_nid == NID_pkcs7_signed) { - if (p7->d.sign == NULL) - return 0; + if (ctype_nid == NID_pkcs7_signed) mdalgs = p7->d.sign->md_algs; - } else { + else mdalgs = NULL; - } flags ^= SMIME_OLDMIME; diff --git a/openssl/src/crypto/pkcs7/pk7_smime.c b/openssl/src/crypto/pkcs7/pk7_smime.c index 747c41771..df0ed90f9 100644 --- a/openssl/src/crypto/pkcs7/pk7_smime.c +++ b/openssl/src/crypto/pkcs7/pk7_smime.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -28,7 +28,7 @@ PKCS7 *PKCS7_sign_ex(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs, int i; if ((p7 = PKCS7_new_ex(libctx, propq)) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return NULL; } @@ -77,12 +77,11 @@ int PKCS7_final(PKCS7 *p7, BIO *data, int flags) int ret = 0; if ((p7bio = PKCS7_dataInit(p7, NULL)) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return 0; } - if (!SMIME_crlf_copy(data, p7bio, flags)) - goto err; + SMIME_crlf_copy(data, p7bio, flags); (void)BIO_flush(p7bio); @@ -107,13 +106,6 @@ static int add_cipher_smcap(STACK_OF(X509_ALGOR) *sk, int nid, int arg) return 1; } -static int add_digest_smcap(STACK_OF(X509_ALGOR) *sk, int nid, int arg) -{ - if (EVP_get_digestbynid(nid)) - return PKCS7_simple_smimecap(sk, nid, arg); - return 1; -} - PKCS7_SIGNER_INFO *PKCS7_sign_add_signer(PKCS7 *p7, X509 *signcert, EVP_PKEY *pkey, const EVP_MD *md, int flags) @@ -144,21 +136,14 @@ PKCS7_SIGNER_INFO *PKCS7_sign_add_signer(PKCS7 *p7, X509 *signcert, /* Add SMIMECapabilities */ if (!(flags & PKCS7_NOSMIMECAP)) { if ((smcap = sk_X509_ALGOR_new_null()) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; } if (!add_cipher_smcap(smcap, NID_aes_256_cbc, -1) - || !add_digest_smcap(smcap, NID_id_GostR3411_2012_256, -1) - || !add_digest_smcap(smcap, NID_id_GostR3411_2012_512, -1) - || !add_digest_smcap(smcap, NID_id_GostR3411_94, -1) - || !add_cipher_smcap(smcap, NID_id_Gost28147_89, -1) || !add_cipher_smcap(smcap, NID_aes_192_cbc, -1) || !add_cipher_smcap(smcap, NID_aes_128_cbc, -1) || !add_cipher_smcap(smcap, NID_des_ede3_cbc, -1) - || !add_cipher_smcap(smcap, NID_rc2_cbc, 128) - || !add_cipher_smcap(smcap, NID_rc2_cbc, 64) || !add_cipher_smcap(smcap, NID_des_cbc, -1) - || !add_cipher_smcap(smcap, NID_rc2_cbc, 40) || !PKCS7_add_attrib_smimecap(si, smcap)) goto err; sk_X509_ALGOR_pop_free(smcap, X509_ALGOR_free); @@ -210,7 +195,6 @@ static int pkcs7_copy_existing_digest(PKCS7 *p7, PKCS7_SIGNER_INFO *si) return 0; } -/* This strongly overlaps with CMS_verify(), partly with PKCS7_dataVerify() */ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, BIO *indata, BIO *out, int flags) { @@ -222,7 +206,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, char *buf = NULL; int i, j = 0, k, ret = 0; BIO *p7bio = NULL; - BIO *tmpout = NULL; + BIO *tmpin = NULL, *tmpout = NULL; const PKCS7_CTX *p7_ctx; if (p7 == NULL) { @@ -236,7 +220,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, } /* Check for no data and no content: no data to verify signature */ - if (PKCS7_get_detached(p7) && indata == NULL) { + if (PKCS7_get_detached(p7) && !indata) { ERR_raise(ERR_LIB_PKCS7, PKCS7_R_NO_CONTENT); return 0; } @@ -249,7 +233,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, * tools like osslsigncode need it. In Authenticode the verification * process is different, but the existing PKCs7 verification works. */ - if (!PKCS7_get_detached(p7) && indata != NULL) { + if (!PKCS7_get_detached(p7) && indata) { ERR_raise(ERR_LIB_PKCS7, PKCS7_R_CONTENT_AND_DATA_PRESENT); return 0; } @@ -281,8 +265,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, ERR_raise(ERR_LIB_PKCS7, ERR_R_X509_LIB); goto err; } - if (!X509_STORE_CTX_set_default(cert_ctx, "smime_sign")) - goto err; + X509_STORE_CTX_set_default(cert_ctx, "smime_sign"); } else if (!X509_STORE_CTX_init(cert_ctx, store, signer, NULL)) { ERR_raise(ERR_LIB_PKCS7, ERR_R_X509_LIB); goto err; @@ -290,8 +273,9 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, if (!(flags & PKCS7_NOCRL)) X509_STORE_CTX_set0_crls(cert_ctx, p7->d.sign->crl); i = X509_verify_cert(cert_ctx); - if (i <= 0) { + if (i <= 0) j = X509_STORE_CTX_get_error(cert_ctx); + if (i <= 0) { ERR_raise_data(ERR_LIB_PKCS7, PKCS7_R_CERTIFICATE_VERIFY_ERROR, "Verify error: %s", X509_verify_cert_error_string(j)); @@ -300,12 +284,31 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, /* Check for revocation status here */ } - if ((p7bio = PKCS7_dataInit(p7, indata)) == NULL) + /* + * Performance optimization: if the content is a memory BIO then store + * its contents in a temporary read only memory BIO. This avoids + * potentially large numbers of slow copies of data which will occur when + * reading from a read write memory BIO when signatures are calculated. + */ + + if (indata && (BIO_method_type(indata) == BIO_TYPE_MEM)) { + char *ptr; + long len; + len = BIO_get_mem_data(indata, &ptr); + tmpin = (len == 0) ? indata : BIO_new_mem_buf(ptr, len); + if (tmpin == NULL) { + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); + goto err; + } + } else + tmpin = indata; + + if ((p7bio = PKCS7_dataInit(p7, tmpin)) == NULL) goto err; if (flags & PKCS7_TEXT) { if ((tmpout = BIO_new(BIO_s_mem())) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; } BIO_set_mem_eof_return(tmpout, 0); @@ -313,8 +316,10 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, tmpout = out; /* We now have to 'read' from p7bio to calculate digests etc. */ - if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) + if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) { + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; + } for (;;) { i = BIO_read(p7bio, buf, BUFFERSIZE); if (i <= 0) @@ -349,8 +354,10 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, err: X509_STORE_CTX_free(cert_ctx); OPENSSL_free(buf); - if (indata != NULL) - BIO_pop(p7bio); + if (tmpin == indata) { + if (indata) + BIO_pop(p7bio); + } BIO_free_all(p7bio); sk_X509_free(signers); return ret; @@ -386,7 +393,7 @@ STACK_OF(X509) *PKCS7_get0_signers(PKCS7 *p7, STACK_OF(X509) *certs, } if ((signers = sk_X509_new_null()) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return NULL; } @@ -395,15 +402,15 @@ STACK_OF(X509) *PKCS7_get0_signers(PKCS7 *p7, STACK_OF(X509) *certs, ias = si->issuer_and_serial; signer = NULL; /* If any certificates passed they take priority */ - if (certs != NULL) + if (certs) signer = X509_find_by_issuer_and_serial(certs, ias->issuer, ias->serial); - if (signer == NULL && !(flags & PKCS7_NOINTERN) + if (!signer && !(flags & PKCS7_NOINTERN) && p7->d.sign->cert) signer = X509_find_by_issuer_and_serial(p7->d.sign->cert, ias->issuer, ias->serial); - if (signer == NULL) { + if (!signer) { ERR_raise(ERR_LIB_PKCS7, PKCS7_R_SIGNER_CERTIFICATE_NOT_FOUND); sk_X509_free(signers); return 0; @@ -429,7 +436,7 @@ PKCS7 *PKCS7_encrypt_ex(STACK_OF(X509) *certs, BIO *in, X509 *x509; if ((p7 = PKCS7_new_ex(libctx, propq)) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); return NULL; } @@ -480,8 +487,7 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags) return 0; } - if (!PKCS7_type_is_enveloped(p7) - && !PKCS7_type_is_signedAndEnveloped(p7)) { + if (!PKCS7_type_is_enveloped(p7)) { ERR_raise(ERR_LIB_PKCS7, PKCS7_R_WRONG_CONTENT_TYPE); return 0; } @@ -501,32 +507,34 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags) BIO *tmpbuf, *bread; /* Encrypt BIOs can't do BIO_gets() so add a buffer BIO */ if ((tmpbuf = BIO_new(BIO_f_buffer())) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); BIO_free_all(tmpmem); return 0; } if ((bread = BIO_push(tmpbuf, tmpmem)) == NULL) { - ERR_raise(ERR_LIB_PKCS7, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); BIO_free_all(tmpbuf); BIO_free_all(tmpmem); return 0; } ret = SMIME_text(bread, data); if (ret > 0 && BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) { - if (BIO_get_cipher_status(tmpmem) <= 0) + if (!BIO_get_cipher_status(tmpmem)) ret = 0; } BIO_free_all(bread); return ret; } - if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) + if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) { + ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE); goto err; + } for (;;) { i = BIO_read(tmpmem, buf, BUFFERSIZE); if (i <= 0) { ret = 1; if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) { - if (BIO_get_cipher_status(tmpmem) <= 0) + if (!BIO_get_cipher_status(tmpmem)) ret = 0; } diff --git a/openssl/src/crypto/poly1305/gen/darwin_arm64/poly1305-armv8.S b/openssl/src/crypto/poly1305/gen/darwin_arm64/poly1305-armv8.S index 85db7d816..90b6d357a 100644 --- a/openssl/src/crypto/poly1305/gen/darwin_arm64/poly1305-armv8.S +++ b/openssl/src/crypto/poly1305/gen/darwin_arm64/poly1305-armv8.S @@ -15,7 +15,6 @@ .align 5 _poly1305_init: - AARCH64_VALID_CALL_TARGET cmp x1,xzr stp xzr,xzr,[x0] // zero hash value stp xzr,xzr,[x0,#16] // [along with is_base2_26] @@ -29,7 +28,7 @@ _poly1305_init: ldp x7,x8,[x1] // load key mov x9,#0xfffffffc0fffffff movk x9,#0x0fff,lsl#48 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x7,x7 // flip bytes rev x8,x8 #endif @@ -63,9 +62,6 @@ Lno_key: .align 5 _poly1305_blocks: Lpoly1305_blocks: - // The symbol .Lpoly1305_blocks is not a .globl symbol - // but a pointer to it is returned by poly1305_init - AARCH64_VALID_CALL_TARGET ands x2,x2,#-16 b.eq Lno_data @@ -79,7 +75,7 @@ Lpoly1305_blocks: Loop: ldp x10,x11,[x1],#16 // load input sub x2,x2,#16 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x10,x10 rev x11,x11 #endif @@ -131,9 +127,6 @@ Lno_data: .align 5 _poly1305_emit: Lpoly1305_emit: - // The symbol .poly1305_emit is not a .globl symbol - // but a pointer to it is returned by poly1305_init - AARCH64_VALID_CALL_TARGET ldp x4,x5,[x0] // load hash base 2^64 ldr x6,[x0,#16] ldp x10,x11,[x2] // load nonce @@ -147,13 +140,13 @@ Lpoly1305_emit: csel x4,x4,x12,eq csel x5,x5,x13,eq -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ ror x10,x10,#32 // flip nonce words ror x11,x11,#32 #endif adds x4,x4,x10 // accumulate nonce adc x5,x5,x11 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x4,x4 // flip output bytes rev x5,x5 #endif @@ -229,16 +222,13 @@ poly1305_splat: .align 5 poly1305_blocks_neon: Lpoly1305_blocks_neon: - // The symbol .Lpoly1305_blocks_neon is not a .globl symbol - // but a pointer to it is returned by poly1305_init - AARCH64_VALID_CALL_TARGET ldr x17,[x0,#24] cmp x2,#128 b.hs Lblocks_neon cbz x17,Lpoly1305_blocks Lblocks_neon: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 @@ -276,7 +266,7 @@ Lblocks_neon: adcs x5,x5,xzr adc x6,x6,xzr -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x12,x12 rev x13,x13 #endif @@ -322,7 +312,7 @@ Lbase2_64_neon: ldp x12,x13,[x1],#16 // load input sub x2,x2,#16 add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x12,x12 rev x13,x13 #endif @@ -407,7 +397,7 @@ Ldo_neon: lsl x3,x3,#24 add x15,x0,#48 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x8,x8 rev x12,x12 rev x9,x9 @@ -443,7 +433,7 @@ Ldo_neon: ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x15],#64 ld1 {v8.4s},[x15] -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x8,x8 rev x12,x12 rev x9,x9 @@ -504,7 +494,7 @@ Loop_neon: umull v20.2d,v14.2s,v1.s[2] ldp x9,x13,[x16],#48 umull v19.2d,v14.2s,v0.s[2] -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x8,x8 rev x12,x12 rev x9,x9 @@ -569,7 +559,7 @@ Loop_neon: umlal v23.2d,v11.2s,v3.s[0] umlal v20.2d,v11.2s,v8.s[0] umlal v21.2d,v11.2s,v0.s[0] -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x8,x8 rev x12,x12 rev x9,x9 @@ -808,7 +798,7 @@ Lshort_tail: Lno_data_neon: ldr x29,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -816,9 +806,6 @@ Lno_data_neon: .align 5 poly1305_emit_neon: Lpoly1305_emit_neon: - // The symbol .Lpoly1305_emit_neon is not a .globl symbol - // but a pointer to it is returned by poly1305_init - AARCH64_VALID_CALL_TARGET ldr x17,[x0,#24] cbz x17,_poly1305_emit @@ -853,13 +840,13 @@ Lpoly1305_emit_neon: csel x4,x4,x12,eq csel x5,x5,x13,eq -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ ror x10,x10,#32 // flip nonce words ror x11,x11,#32 #endif adds x4,x4,x10 // accumulate nonce adc x5,x5,x11 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x4,x4 // flip output bytes rev x5,x5 #endif diff --git a/openssl/src/crypto/poly1305/gen/linux_arm/poly1305-armv4.S b/openssl/src/crypto/poly1305/gen/linux_arm/poly1305-armv4.S index 091599ad8..14f80a85c 100644 --- a/openssl/src/crypto/poly1305/gen/linux_arm/poly1305-armv4.S +++ b/openssl/src/crypto/poly1305/gen/linux_arm/poly1305-armv4.S @@ -1164,5 +1164,5 @@ poly1305_emit_neon: .align 2 .align 2 #if __ARM_MAX_ARCH__>=7 - +.comm OPENSSL_armcap_P,4,4 #endif diff --git a/openssl/src/crypto/poly1305/gen/linux_arm64/poly1305-armv8.S b/openssl/src/crypto/poly1305/gen/linux_arm64/poly1305-armv8.S index eec87276c..ea8f39022 100644 --- a/openssl/src/crypto/poly1305/gen/linux_arm64/poly1305-armv8.S +++ b/openssl/src/crypto/poly1305/gen/linux_arm64/poly1305-armv8.S @@ -15,7 +15,6 @@ .type poly1305_init,%function .align 5 poly1305_init: - AARCH64_VALID_CALL_TARGET cmp x1,xzr stp xzr,xzr,[x0] // zero hash value stp xzr,xzr,[x0,#16] // [along with is_base2_26] @@ -29,7 +28,7 @@ poly1305_init: ldp x7,x8,[x1] // load key mov x9,#0xfffffffc0fffffff movk x9,#0x0fff,lsl#48 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x7,x7 // flip bytes rev x8,x8 #endif @@ -63,9 +62,6 @@ poly1305_init: .align 5 poly1305_blocks: .Lpoly1305_blocks: - // The symbol .Lpoly1305_blocks is not a .globl symbol - // but a pointer to it is returned by poly1305_init - AARCH64_VALID_CALL_TARGET ands x2,x2,#-16 b.eq .Lno_data @@ -79,7 +75,7 @@ poly1305_blocks: .Loop: ldp x10,x11,[x1],#16 // load input sub x2,x2,#16 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x10,x10 rev x11,x11 #endif @@ -131,9 +127,6 @@ poly1305_blocks: .align 5 poly1305_emit: .Lpoly1305_emit: - // The symbol .poly1305_emit is not a .globl symbol - // but a pointer to it is returned by poly1305_init - AARCH64_VALID_CALL_TARGET ldp x4,x5,[x0] // load hash base 2^64 ldr x6,[x0,#16] ldp x10,x11,[x2] // load nonce @@ -147,13 +140,13 @@ poly1305_emit: csel x4,x4,x12,eq csel x5,x5,x13,eq -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ ror x10,x10,#32 // flip nonce words ror x11,x11,#32 #endif adds x4,x4,x10 // accumulate nonce adc x5,x5,x11 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x4,x4 // flip output bytes rev x5,x5 #endif @@ -229,16 +222,13 @@ poly1305_splat: .align 5 poly1305_blocks_neon: .Lpoly1305_blocks_neon: - // The symbol .Lpoly1305_blocks_neon is not a .globl symbol - // but a pointer to it is returned by poly1305_init - AARCH64_VALID_CALL_TARGET ldr x17,[x0,#24] cmp x2,#128 b.hs .Lblocks_neon cbz x17,.Lpoly1305_blocks .Lblocks_neon: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 @@ -276,7 +266,7 @@ poly1305_blocks_neon: adcs x5,x5,xzr adc x6,x6,xzr -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x12,x12 rev x13,x13 #endif @@ -322,7 +312,7 @@ poly1305_blocks_neon: ldp x12,x13,[x1],#16 // load input sub x2,x2,#16 add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x12,x12 rev x13,x13 #endif @@ -407,7 +397,7 @@ poly1305_blocks_neon: lsl x3,x3,#24 add x15,x0,#48 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x8,x8 rev x12,x12 rev x9,x9 @@ -443,7 +433,7 @@ poly1305_blocks_neon: ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x15],#64 ld1 {v8.4s},[x15] -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x8,x8 rev x12,x12 rev x9,x9 @@ -504,7 +494,7 @@ poly1305_blocks_neon: umull v20.2d,v14.2s,v1.s[2] ldp x9,x13,[x16],#48 umull v19.2d,v14.2s,v0.s[2] -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x8,x8 rev x12,x12 rev x9,x9 @@ -569,7 +559,7 @@ poly1305_blocks_neon: umlal v23.2d,v11.2s,v3.s[0] umlal v20.2d,v11.2s,v8.s[0] umlal v21.2d,v11.2s,v0.s[0] -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x8,x8 rev x12,x12 rev x9,x9 @@ -808,7 +798,7 @@ poly1305_blocks_neon: .Lno_data_neon: ldr x29,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size poly1305_blocks_neon,.-poly1305_blocks_neon @@ -816,9 +806,6 @@ poly1305_blocks_neon: .align 5 poly1305_emit_neon: .Lpoly1305_emit_neon: - // The symbol .Lpoly1305_emit_neon is not a .globl symbol - // but a pointer to it is returned by poly1305_init - AARCH64_VALID_CALL_TARGET ldr x17,[x0,#24] cbz x17,poly1305_emit @@ -853,13 +840,13 @@ poly1305_emit_neon: csel x4,x4,x12,eq csel x5,x5,x13,eq -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ ror x10,x10,#32 // flip nonce words ror x11,x11,#32 #endif adds x4,x4,x10 // accumulate nonce adc x5,x5,x11 -#ifdef __AARCH64EB__ +#ifdef __ARMEB__ rev x4,x4 // flip output bytes rev x5,x5 #endif diff --git a/openssl/src/crypto/poly1305/gen/linux_ia32/poly1305-x86.S b/openssl/src/crypto/poly1305/gen/linux_ia32/poly1305-x86.S index e76632cd1..62fd11e14 100644 --- a/openssl/src/crypto/poly1305/gen/linux_ia32/poly1305-x86.S +++ b/openssl/src/crypto/poly1305/gen/linux_ia32/poly1305-x86.S @@ -5,11 +5,7 @@ .align 16 poly1305_init: .L_poly1305_init_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -71,11 +67,7 @@ poly1305_init: .align 16 poly1305_blocks: .L_poly1305_blocks_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -244,11 +236,7 @@ poly1305_blocks: .align 16 poly1305_emit: .L_poly1305_emit_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -308,11 +296,7 @@ poly1305_emit: .type _poly1305_init_sse2,@function .align 16 _poly1305_init_sse2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - movdqu 24(%edi),%xmm4 leal 48(%edi),%edi movl %esp,%ebp @@ -515,11 +499,7 @@ _poly1305_init_sse2: .type _poly1305_blocks_sse2,@function .align 16 _poly1305_blocks_sse2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1281,11 +1261,7 @@ _poly1305_blocks_sse2: .type _poly1305_emit_sse2,@function .align 16 _poly1305_emit_sse2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1379,11 +1355,7 @@ _poly1305_emit_sse2: .type _poly1305_init_avx2,@function .align 16 _poly1305_init_avx2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - vmovdqu 24(%edi),%xmm4 leal 48(%edi),%edi movl %esp,%ebp @@ -1555,11 +1527,7 @@ _poly1305_init_avx2: .type _poly1305_blocks_avx2,@function .align 16 _poly1305_blocks_avx2: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/poly1305/gen/linux_ppc64/poly1305-ppc.s b/openssl/src/crypto/poly1305/gen/linux_ppc64/poly1305-ppc.s index b6ffc97d2..fc3b2251a 100644 --- a/openssl/src/crypto/poly1305/gen/linux_ppc64/poly1305-ppc.s +++ b/openssl/src/crypto/poly1305/gen/linux_ppc64/poly1305-ppc.s @@ -365,7 +365,7 @@ __poly1305_splat: .align 5 __poly1305_blocks_vsx: - stdu 1,-416(1) + stdu 1,-432(1) mflr 0 li 10,191 li 11,207 @@ -376,12 +376,12 @@ __poly1305_blocks_vsx: addi 11,11,32 stvx 22,10,1 addi 10,10,32 - stvx 23,11,1 - addi 11,11,32 - stvx 24,10,1 + stvx 23,10,1 addi 10,10,32 - stvx 25,11,1 + stvx 24,11,1 addi 11,11,32 + stvx 25,10,1 + addi 10,10,32 stvx 26,10,1 addi 10,10,32 stvx 27,11,1 @@ -392,15 +392,15 @@ __poly1305_blocks_vsx: addi 11,11,32 stvx 30,10,1 stvx 31,11,1 - stw 12,372(1) + stw 12,388(1) li 12,-1 or 12,12,12 - std 27,376(1) - std 28,384(1) - std 29,392(1) - std 30,400(1) - std 31,408(1) - std 0,432(1) + std 27,392(1) + std 28,400(1) + std 29,408(1) + std 30,416(1) + std 31,424(1) + std 0,448(1) bl .LPICmeup @@ -1035,7 +1035,7 @@ __poly1305_blocks_vsx: .align 4 .Ldone_vsx: - ld 0,432(1) + ld 0,448(1) li 27,4 li 28,8 li 29,12 @@ -1046,39 +1046,39 @@ __poly1305_blocks_vsx: .long 0x7C7D1919 .long 0x7C9E1919 - lwz 12,372(1) + lwz 12,388(1) mtlr 0 li 10,191 li 11,207 or 12,12,12 lvx 20,10,1 addi 10,10,32 - lvx 21,11,1 - addi 11,11,32 - lvx 22,10,1 + lvx 21,10,1 addi 10,10,32 - lvx 23,11,1 + lvx 22,11,1 addi 11,11,32 - lvx 24,10,1 + lvx 23,10,1 addi 10,10,32 - lvx 25,11,1 + lvx 24,11,1 addi 11,11,32 - lvx 26,10,1 + lvx 25,10,1 addi 10,10,32 - lvx 27,11,1 + lvx 26,11,1 addi 11,11,32 - lvx 28,10,1 + lvx 27,10,1 addi 10,10,32 - lvx 29,11,1 + lvx 28,11,1 addi 11,11,32 - lvx 30,10,1 - lvx 31,11,1 - ld 27,376(1) - ld 28,384(1) - ld 29,392(1) - ld 30,400(1) - ld 31,408(1) - addi 1,1,416 + lvx 29,10,1 + addi 10,10,32 + lvx 30,11,1 + lvx 31,10,1 + ld 27,392(1) + ld 28,400(1) + ld 29,408(1) + ld 30,416(1) + ld 31,424(1) + addi 1,1,432 blr .long 0 .byte 0,12,0x04,1,0x80,5,4,0 diff --git a/openssl/src/crypto/poly1305/gen/windows_ia32/poly1305-x86.asm b/openssl/src/crypto/poly1305/gen/windows_ia32/poly1305-x86.asm index 4878c94d6..ba241b799 100644 --- a/openssl/src/crypto/poly1305/gen/windows_ia32/poly1305-x86.asm +++ b/openssl/src/crypto/poly1305/gen/windows_ia32/poly1305-x86.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/poly1305/local.h b/openssl/src/crypto/poly1305/local.h new file mode 100644 index 000000000..20027cd6e --- /dev/null +++ b/openssl/src/crypto/poly1305/local.h @@ -0,0 +1,20 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +#ifdef POLY1305_ASM +int poly1305_init(void *ctx, const unsigned char key[16], void *func); +void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, + unsigned int padbit); +void poly1305_emit(void *ctx, unsigned char mac[16], + const unsigned int nonce[4]); +#endif diff --git a/openssl/src/crypto/poly1305/poly1305.c b/openssl/src/crypto/poly1305/poly1305.c index b3bf2b117..1cbe73243 100644 --- a/openssl/src/crypto/poly1305/poly1305.c +++ b/openssl/src/crypto/poly1305/poly1305.c @@ -461,6 +461,12 @@ void Poly1305_Init(POLY1305 *ctx, const unsigned char key[32]) * This "eclipses" poly1305_blocks and poly1305_emit, but it's * conscious choice imposed by -Wshadow compiler warnings. */ +# ifdef poly1305_blocks +# undef poly1305_blocks +# endif +# ifdef poly1305_emit +# undef poly1305_emit +# endif # define poly1305_blocks (*poly1305_blocks_p) # define poly1305_emit (*poly1305_emit_p) #endif diff --git a/openssl/src/crypto/poly1305/poly1305_ieee754.c b/openssl/src/crypto/poly1305/poly1305_ieee754.c index 57a08aa24..5104274d5 100644 --- a/openssl/src/crypto/poly1305/poly1305_ieee754.c +++ b/openssl/src/crypto/poly1305/poly1305_ieee754.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,7 +10,7 @@ /* * This module is meant to be used as template for non-x87 floating- * point assembly modules. The template itself is x86_64-specific - * though, as it was debugged on x86_64. So that implementer would + * though, as it was debugged on x86_64. So that implementor would * have to recognize platform-specific parts, UxTOy and inline asm, * and act accordingly. * @@ -41,9 +41,6 @@ * * z10 11.2 * z196+ 7.30 - * - * UltraSPARC III 16.0 - * SPARC T4 16.1 */ #if !(defined(__GNUC__) && __GNUC__>=2) @@ -69,7 +66,7 @@ typedef union { double d; u64 u; } elem64; #if defined(__x86_64__) || (defined(__PPC__) && defined(__LITTLE_ENDIAN__)) # define U8TOU32(p) (*(const u32 *)(p)) # define U32TO8(p,v) (*(u32 *)(p) = (v)) -#elif defined(__PPC__) || defined(__POWERPC__) +#elif defined(__PPC__) # define U8TOU32(p) ({u32 ret; asm ("lwbrx %0,0,%1":"=r"(ret):"b"(p)); ret; }) # define U32TO8(p,v) asm ("stwbrx %0,0,%1"::"r"(v),"b"(p):"memory") #elif defined(__s390x__) @@ -95,12 +92,10 @@ typedef struct { /* "round toward zero (truncate), mask all exceptions" */ #if defined(__x86_64__) static const u32 mxcsr = 0x7f80; -#elif defined(__PPC__) || defined(__POWERPC__) +#elif defined(__PPC__) static const u64 one = 1; #elif defined(__s390x__) static const u32 fpc = 1; -#elif defined(__sparc__) -static const u64 fsr = 1ULL<<30; #elif defined(__mips__) static const u32 fcsr = 1; #else @@ -134,7 +129,7 @@ int poly1305_init(void *ctx, const unsigned char key[16]) asm volatile ("stmxcsr %0":"=m"(mxcsr_orig)); asm volatile ("ldmxcsr %0"::"m"(mxcsr)); -#elif defined(__PPC__) || defined(__POWERPC__) +#elif defined(__PPC__) double fpscr_orig, fpscr = *(double *)&one; asm volatile ("mffs %0":"=f"(fpscr_orig)); @@ -144,11 +139,6 @@ int poly1305_init(void *ctx, const unsigned char key[16]) asm volatile ("stfpc %0":"=m"(fpc_orig)); asm volatile ("lfpc %0"::"m"(fpc)); -#elif defined(__sparc__) - u64 fsr_orig; - - asm volatile ("stx %%fsr,%0":"=m"(fsr_orig)); - asm volatile ("ldx %0,%%fsr"::"m"(fsr)); #elif defined(__mips__) u32 fcsr_orig; @@ -207,12 +197,10 @@ int poly1305_init(void *ctx, const unsigned char key[16]) */ #if defined(__x86_64__) asm volatile ("ldmxcsr %0"::"m"(mxcsr_orig)); -#elif defined(__PPC__) || defined(__POWERPC__) +#elif defined(__PPC__) asm volatile ("mtfsf 255,%0"::"f"(fpscr_orig)); #elif defined(__s390x__) asm volatile ("lfpc %0"::"m"(fpc_orig)); -#elif defined(__sparc__) - asm volatile ("ldx %0,%%fsr"::"m"(fsr_orig)); #elif defined(__mips__) asm volatile ("ctc1 %0,$31"::"r"(fcsr_orig)); #endif @@ -256,7 +244,7 @@ void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, asm volatile ("stmxcsr %0":"=m"(mxcsr_orig)); asm volatile ("ldmxcsr %0"::"m"(mxcsr)); -#elif defined(__PPC__) || defined(__POWERPC__) +#elif defined(__PPC__) double fpscr_orig, fpscr = *(double *)&one; asm volatile ("mffs %0":"=f"(fpscr_orig)); @@ -266,11 +254,6 @@ void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, asm volatile ("stfpc %0":"=m"(fpc_orig)); asm volatile ("lfpc %0"::"m"(fpc)); -#elif defined(__sparc__) - u64 fsr_orig; - - asm volatile ("stx %%fsr,%0":"=m"(fsr_orig)); - asm volatile ("ldx %0,%%fsr"::"m"(fsr)); #elif defined(__mips__) u32 fcsr_orig; @@ -416,12 +399,10 @@ void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, */ #if defined(__x86_64__) asm volatile ("ldmxcsr %0"::"m"(mxcsr_orig)); -#elif defined(__PPC__) || defined(__POWERPC__) +#elif defined(__PPC__) asm volatile ("mtfsf 255,%0"::"f"(fpscr_orig)); #elif defined(__s390x__) asm volatile ("lfpc %0"::"m"(fpc_orig)); -#elif defined(__sparc__) - asm volatile ("ldx %0,%%fsr"::"m"(fsr_orig)); #elif defined(__mips__) asm volatile ("ctc1 %0,$31"::"r"(fcsr_orig)); #endif diff --git a/openssl/src/crypto/ppccap.c b/openssl/src/crypto/ppccap.c index a38c819d2..8bcfed25e 100644 --- a/openssl/src/crypto/ppccap.c +++ b/openssl/src/crypto/ppccap.c @@ -1,5 +1,5 @@ /* - * Copyright 2009-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2009-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -45,7 +45,6 @@ void OPENSSL_ppc64_probe(void); void OPENSSL_altivec_probe(void); void OPENSSL_crypto207_probe(void); void OPENSSL_madd300_probe(void); -void OPENSSL_brd31_probe(void); long OPENSSL_rdtsc_mftb(void); long OPENSSL_rdtsc_mfspr268(void); @@ -118,21 +117,16 @@ static unsigned long getauxval(unsigned long key) #endif /* I wish was universally available */ -#ifndef AT_HWCAP -# define AT_HWCAP 16 /* AT_HWCAP */ -#endif +#define HWCAP 16 /* AT_HWCAP */ #define HWCAP_PPC64 (1U << 30) #define HWCAP_ALTIVEC (1U << 28) #define HWCAP_FPU (1U << 27) #define HWCAP_POWER6_EXT (1U << 9) #define HWCAP_VSX (1U << 7) -#ifndef AT_HWCAP2 -# define AT_HWCAP2 26 /* AT_HWCAP2 */ -#endif +#define HWCAP2 26 /* AT_HWCAP2 */ #define HWCAP_VEC_CRYPTO (1U << 25) #define HWCAP_ARCH_3_00 (1U << 23) -#define HWCAP_ARCH_3_1 (1U << 18) # if defined(__GNUC__) && __GNUC__>=2 __attribute__ ((constructor)) @@ -193,9 +187,6 @@ void OPENSSL_cpuid_setup(void) if (__power_set(0xffffffffU<<17)) /* POWER9 and later */ OPENSSL_ppccap_P |= PPC_MADD300; - if (__power_set(0xffffffffU<<18)) /* POWER10 and later */ - OPENSSL_ppccap_P |= PPC_BRD31; - return; # endif #endif @@ -224,8 +215,8 @@ void OPENSSL_cpuid_setup(void) #ifdef OSSL_IMPLEMENT_GETAUXVAL { - unsigned long hwcap = getauxval(AT_HWCAP); - unsigned long hwcap2 = getauxval(AT_HWCAP2); + unsigned long hwcap = getauxval(HWCAP); + unsigned long hwcap2 = getauxval(HWCAP2); if (hwcap & HWCAP_FPU) { OPENSSL_ppccap_P |= PPC_FPU; @@ -251,10 +242,6 @@ void OPENSSL_cpuid_setup(void) if (hwcap2 & HWCAP_ARCH_3_00) { OPENSSL_ppccap_P |= PPC_MADD300; } - - if (hwcap2 & HWCAP_ARCH_3_1) { - OPENSSL_ppccap_P |= PPC_BRD31; - } } #endif @@ -276,7 +263,7 @@ void OPENSSL_cpuid_setup(void) sigaction(SIGILL, &ill_act, &ill_oact); #ifndef OSSL_IMPLEMENT_GETAUXVAL - if (sigsetjmp(ill_jmp, 1) == 0) { + if (sigsetjmp(ill_jmp,1) == 0) { OPENSSL_fpu_probe(); OPENSSL_ppccap_P |= PPC_FPU; diff --git a/openssl/src/crypto/property/defn_cache.c b/openssl/src/crypto/property/defn_cache.c index eb68a55aa..800759952 100644 --- a/openssl/src/crypto/property/defn_cache.c +++ b/openssl/src/crypto/property/defn_cache.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -15,7 +15,6 @@ #include "internal/property.h" #include "internal/core.h" #include "property_local.h" -#include "crypto/context.h" /* * Implement a property definition cache. @@ -29,7 +28,7 @@ typedef struct { char body[1]; } PROPERTY_DEFN_ELEM; -DEFINE_LHASH_OF_EX(PROPERTY_DEFN_ELEM); +DEFINE_LHASH_OF(PROPERTY_DEFN_ELEM); static unsigned long property_defn_hash(const PROPERTY_DEFN_ELEM *a) { @@ -48,7 +47,7 @@ static void property_defn_free(PROPERTY_DEFN_ELEM *elem) OPENSSL_free(elem); } -void ossl_property_defns_free(void *vproperty_defns) +static void property_defns_free(void *vproperty_defns) { LHASH_OF(PROPERTY_DEFN_ELEM) *property_defns = vproperty_defns; @@ -59,35 +58,35 @@ void ossl_property_defns_free(void *vproperty_defns) } } -void *ossl_property_defns_new(OSSL_LIB_CTX *ctx) { +static void *property_defns_new(OSSL_LIB_CTX *ctx) { return lh_PROPERTY_DEFN_ELEM_new(&property_defn_hash, &property_defn_cmp); } +static const OSSL_LIB_CTX_METHOD property_defns_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + property_defns_new, + property_defns_free, +}; + OSSL_PROPERTY_LIST *ossl_prop_defn_get(OSSL_LIB_CTX *ctx, const char *prop) { PROPERTY_DEFN_ELEM elem, *r; LHASH_OF(PROPERTY_DEFN_ELEM) *property_defns; property_defns = ossl_lib_ctx_get_data(ctx, - OSSL_LIB_CTX_PROPERTY_DEFN_INDEX); - if (!ossl_assert(property_defns != NULL) || !ossl_lib_ctx_read_lock(ctx)) + OSSL_LIB_CTX_PROPERTY_DEFN_INDEX, + &property_defns_method); + if (property_defns == NULL || !ossl_lib_ctx_read_lock(ctx)) return NULL; elem.prop = prop; r = lh_PROPERTY_DEFN_ELEM_retrieve(property_defns, &elem); ossl_lib_ctx_unlock(ctx); - if (r == NULL || !ossl_assert(r->defn != NULL)) - return NULL; - return r->defn; + return r != NULL ? r->defn : NULL; } -/* - * Cache the property list for a given property string *pl. - * If an entry already exists in the cache *pl is freed and - * overwritten with the existing entry from the cache. - */ int ossl_prop_defn_set(OSSL_LIB_CTX *ctx, const char *prop, - OSSL_PROPERTY_LIST **pl) + OSSL_PROPERTY_LIST *pl) { PROPERTY_DEFN_ELEM elem, *old, *p = NULL; size_t len; @@ -95,7 +94,8 @@ int ossl_prop_defn_set(OSSL_LIB_CTX *ctx, const char *prop, int res = 1; property_defns = ossl_lib_ctx_get_data(ctx, - OSSL_LIB_CTX_PROPERTY_DEFN_INDEX); + OSSL_LIB_CTX_PROPERTY_DEFN_INDEX, + &property_defns_method); if (property_defns == NULL) return 0; @@ -104,27 +104,22 @@ int ossl_prop_defn_set(OSSL_LIB_CTX *ctx, const char *prop, if (!ossl_lib_ctx_write_lock(ctx)) return 0; - elem.prop = prop; if (pl == NULL) { + elem.prop = prop; lh_PROPERTY_DEFN_ELEM_delete(property_defns, &elem); goto end; } - /* check if property definition is in the cache already */ - if ((p = lh_PROPERTY_DEFN_ELEM_retrieve(property_defns, &elem)) != NULL) { - ossl_property_free(*pl); - *pl = p->defn; - goto end; - } len = strlen(prop); p = OPENSSL_malloc(sizeof(*p) + len); if (p != NULL) { p->prop = p->body; - p->defn = *pl; + p->defn = pl; memcpy(p->body, prop, len + 1); old = lh_PROPERTY_DEFN_ELEM_insert(property_defns, p); - if (!ossl_assert(old == NULL)) - /* This should not happen. An existing entry is handled above. */ + if (old != NULL) { + property_defn_free(old); goto end; + } if (!lh_PROPERTY_DEFN_ELEM_error(property_defns)) goto end; } diff --git a/openssl/src/crypto/property/property.c b/openssl/src/crypto/property/property.c index c551c825b..509191a89 100644 --- a/openssl/src/crypto/property/property.c +++ b/openssl/src/crypto/property/property.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -15,7 +15,6 @@ #include "internal/core.h" #include "internal/property.h" #include "internal/provider.h" -#include "internal/tsan_assist.h" #include "crypto/ctype.h" #include #include @@ -23,7 +22,6 @@ #include "crypto/lhash.h" #include "crypto/sparse_array.h" #include "property_local.h" -#include "crypto/context.h" /* * The number of elements in the query cache before we initiate a flush. @@ -53,7 +51,7 @@ typedef struct { char body[1]; } QUERY; -DEFINE_LHASH_OF_EX(QUERY); +DEFINE_LHASH_OF(QUERY); typedef struct { int nid; @@ -63,35 +61,16 @@ typedef struct { struct ossl_method_store_st { OSSL_LIB_CTX *ctx; + size_t nelem; SPARSE_ARRAY_OF(ALGORITHM) *algs; - /* - * Lock to protect the |algs| array from concurrent writing, when - * individual implementations or queries are inserted. This is used - * by the appropriate functions here. - */ + int need_flush; CRYPTO_RWLOCK *lock; - /* - * Lock to reserve the whole store. This is used when fetching a set - * of algorithms, via these functions, found in crypto/core_fetch.c: - * ossl_method_construct_reserve_store() - * ossl_method_construct_unreserve_store() - */ - CRYPTO_RWLOCK *biglock; - - /* query cache specific values */ - - /* Count of the query cache entries for all algs */ - size_t cache_nelem; - - /* Flag: 1 if query cache entries for all algs need flushing */ - int cache_need_flush; }; typedef struct { LHASH_OF(QUERY) *cache; size_t nelem; uint32_t seed; - unsigned char using_global_seed; } IMPL_CACHE_FLUSH; DEFINE_SPARSE_ARRAY_OF(ALGORITHM); @@ -103,12 +82,10 @@ typedef struct ossl_global_properties_st { #endif } OSSL_GLOBAL_PROPERTIES; -static void ossl_method_cache_flush_alg(OSSL_METHOD_STORE *store, - ALGORITHM *alg); static void ossl_method_cache_flush(OSSL_METHOD_STORE *store, int nid); /* Global properties are stored per library context */ -void ossl_ctx_global_properties_free(void *vglobp) +static void ossl_ctx_global_properties_free(void *vglobp) { OSSL_GLOBAL_PROPERTIES *globp = vglobp; @@ -118,21 +95,28 @@ void ossl_ctx_global_properties_free(void *vglobp) } } -void *ossl_ctx_global_properties_new(OSSL_LIB_CTX *ctx) +static void *ossl_ctx_global_properties_new(OSSL_LIB_CTX *ctx) { return OPENSSL_zalloc(sizeof(OSSL_GLOBAL_PROPERTIES)); } +static const OSSL_LIB_CTX_METHOD ossl_ctx_global_properties_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + ossl_ctx_global_properties_new, + ossl_ctx_global_properties_free, +}; + OSSL_PROPERTY_LIST **ossl_ctx_global_properties(OSSL_LIB_CTX *libctx, - ossl_unused int loadconfig) + int loadconfig) { OSSL_GLOBAL_PROPERTIES *globp; -#if !defined(FIPS_MODULE) && !defined(OPENSSL_NO_AUTOLOAD_CONFIG) +#ifndef FIPS_MODULE if (loadconfig && !OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL)) return NULL; #endif - globp = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_GLOBAL_PROPERTIES); + globp = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_GLOBAL_PROPERTIES, + &ossl_ctx_global_properties_method); return globp != NULL ? &globp->list : NULL; } @@ -141,7 +125,8 @@ OSSL_PROPERTY_LIST **ossl_ctx_global_properties(OSSL_LIB_CTX *libctx, int ossl_global_properties_no_mirrored(OSSL_LIB_CTX *libctx) { OSSL_GLOBAL_PROPERTIES *globp - = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_GLOBAL_PROPERTIES); + = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_GLOBAL_PROPERTIES, + &ossl_ctx_global_properties_method); return globp != NULL && globp->no_mirrored ? 1 : 0; } @@ -149,7 +134,8 @@ int ossl_global_properties_no_mirrored(OSSL_LIB_CTX *libctx) void ossl_global_properties_stop_mirroring(OSSL_LIB_CTX *libctx) { OSSL_GLOBAL_PROPERTIES *globp - = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_GLOBAL_PROPERTIES); + = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_GLOBAL_PROPERTIES, + &ossl_ctx_global_properties_method); if (globp != NULL) globp->no_mirrored = 1; @@ -213,24 +199,14 @@ static void impl_cache_free(QUERY *elem) } } -static void impl_cache_flush_alg(ossl_uintmax_t idx, ALGORITHM *alg) -{ - lh_QUERY_doall(alg->cache, &impl_cache_free); - lh_QUERY_flush(alg->cache); -} - -static void alg_cleanup(ossl_uintmax_t idx, ALGORITHM *a, void *arg) +static void alg_cleanup(ossl_uintmax_t idx, ALGORITHM *a) { - OSSL_METHOD_STORE *store = arg; - if (a != NULL) { sk_IMPLEMENTATION_pop_free(a->impls, &impl_free); lh_QUERY_doall(a->cache, &impl_cache_free); lh_QUERY_free(a->cache); OPENSSL_free(a); } - if (store != NULL) - ossl_sa_ALGORITHM_set(store->algs, idx, NULL); } /* @@ -244,10 +220,13 @@ OSSL_METHOD_STORE *ossl_method_store_new(OSSL_LIB_CTX *ctx) res = OPENSSL_zalloc(sizeof(*res)); if (res != NULL) { res->ctx = ctx; - if ((res->algs = ossl_sa_ALGORITHM_new()) == NULL - || (res->lock = CRYPTO_THREAD_lock_new()) == NULL - || (res->biglock = CRYPTO_THREAD_lock_new()) == NULL) { - ossl_method_store_free(res); + if ((res->algs = ossl_sa_ALGORITHM_new()) == NULL) { + OPENSSL_free(res); + return NULL; + } + if ((res->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ossl_sa_ALGORITHM_free(res->algs); + OPENSSL_free(res); return NULL; } } @@ -257,25 +236,13 @@ OSSL_METHOD_STORE *ossl_method_store_new(OSSL_LIB_CTX *ctx) void ossl_method_store_free(OSSL_METHOD_STORE *store) { if (store != NULL) { - if (store->algs != NULL) - ossl_sa_ALGORITHM_doall_arg(store->algs, &alg_cleanup, store); + ossl_sa_ALGORITHM_doall(store->algs, &alg_cleanup); ossl_sa_ALGORITHM_free(store->algs); CRYPTO_THREAD_lock_free(store->lock); - CRYPTO_THREAD_lock_free(store->biglock); OPENSSL_free(store); } } -int ossl_method_lock_store(OSSL_METHOD_STORE *store) -{ - return store != NULL ? CRYPTO_THREAD_write_lock(store->biglock) : 0; -} - -int ossl_method_unlock_store(OSSL_METHOD_STORE *store) -{ - return store != NULL ? CRYPTO_THREAD_unlock(store->biglock) : 0; -} - static ALGORITHM *ossl_method_store_retrieve(OSSL_METHOD_STORE *store, int nid) { return ossl_sa_ALGORITHM_get(store->algs, nid); @@ -283,7 +250,7 @@ static ALGORITHM *ossl_method_store_retrieve(OSSL_METHOD_STORE *store, int nid) static int ossl_method_store_insert(OSSL_METHOD_STORE *store, ALGORITHM *alg) { - return ossl_sa_ALGORITHM_set(store->algs, alg->nid, alg); + return ossl_sa_ALGORITHM_set(store->algs, alg->nid, alg); } int ossl_method_store_add(OSSL_METHOD_STORE *store, const OSSL_PROVIDER *prov, @@ -327,11 +294,7 @@ int ossl_method_store_add(OSSL_METHOD_STORE *store, const OSSL_PROVIDER *prov, impl->properties = ossl_parse_property(store->ctx, properties); if (impl->properties == NULL) goto err; - if (!ossl_prop_defn_set(store->ctx, properties, &impl->properties)) { - ossl_property_free(impl->properties); - impl->properties = NULL; - goto err; - } + ossl_prop_defn_set(store->ctx, properties, impl->properties); } alg = ossl_method_store_retrieve(store, nid); @@ -363,7 +326,7 @@ int ossl_method_store_add(OSSL_METHOD_STORE *store, const OSSL_PROVIDER *prov, err: ossl_property_unlock(store); - alg_cleanup(0, alg, NULL); + alg_cleanup(0, alg); impl_free(impl); return 0; } @@ -405,55 +368,6 @@ int ossl_method_store_remove(OSSL_METHOD_STORE *store, int nid, return 0; } -struct alg_cleanup_by_provider_data_st { - OSSL_METHOD_STORE *store; - const OSSL_PROVIDER *prov; -}; - -static void -alg_cleanup_by_provider(ossl_uintmax_t idx, ALGORITHM *alg, void *arg) -{ - struct alg_cleanup_by_provider_data_st *data = arg; - int i, count; - - /* - * We walk the stack backwards, to avoid having to deal with stack shifts - * caused by deletion - */ - for (count = 0, i = sk_IMPLEMENTATION_num(alg->impls); i-- > 0;) { - IMPLEMENTATION *impl = sk_IMPLEMENTATION_value(alg->impls, i); - - if (impl->provider == data->prov) { - impl_free(impl); - (void)sk_IMPLEMENTATION_delete(alg->impls, i); - count++; - } - } - - /* - * If we removed any implementation, we also clear the whole associated - * cache, 'cause that's the sensible thing to do. - * There's no point flushing the cache entries where we didn't remove - * any implementation, though. - */ - if (count > 0) - ossl_method_cache_flush_alg(data->store, alg); -} - -int ossl_method_store_remove_all_provided(OSSL_METHOD_STORE *store, - const OSSL_PROVIDER *prov) -{ - struct alg_cleanup_by_provider_data_st data; - - if (!ossl_property_write_lock(store)) - return 0; - data.prov = prov; - data.store = store; - ossl_sa_ALGORITHM_doall_arg(store->algs, &alg_cleanup_by_provider, &data); - ossl_property_unlock(store); - return 1; -} - static void alg_do_one(ALGORITHM *alg, IMPLEMENTATION *impl, void (*fn)(int id, void *method, void *fnarg), void *fnarg) @@ -502,14 +416,13 @@ int ossl_method_store_fetch(OSSL_METHOD_STORE *store, int ret = 0; int j, best = -1, score, optional; - if (nid <= 0 || method == NULL || store == NULL) +#ifndef FIPS_MODULE + if (!OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL)) return 0; +#endif -#if !defined(FIPS_MODULE) && !defined(OPENSSL_NO_AUTOLOAD_CONFIG) - if (ossl_lib_ctx_is_default(store->ctx) - && !OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL)) + if (nid <= 0 || method == NULL || store == NULL) return 0; -#endif /* This only needs to be a read lock, because the query won't create anything */ if (!ossl_property_read_lock(store)) @@ -573,27 +486,41 @@ int ossl_method_store_fetch(OSSL_METHOD_STORE *store, return ret; } -static void ossl_method_cache_flush_alg(OSSL_METHOD_STORE *store, - ALGORITHM *alg) +static void impl_cache_flush_alg(ossl_uintmax_t idx, ALGORITHM *alg, void *arg) { - store->cache_nelem -= lh_QUERY_num_items(alg->cache); - impl_cache_flush_alg(0, alg); + SPARSE_ARRAY_OF(ALGORITHM) *algs = arg; + + lh_QUERY_doall(alg->cache, &impl_cache_free); + if (algs != NULL) { + sk_IMPLEMENTATION_pop_free(alg->impls, &impl_free); + lh_QUERY_free(alg->cache); + OPENSSL_free(alg); + ossl_sa_ALGORITHM_set(algs, idx, NULL); + } else { + lh_QUERY_flush(alg->cache); + } } static void ossl_method_cache_flush(OSSL_METHOD_STORE *store, int nid) { ALGORITHM *alg = ossl_method_store_retrieve(store, nid); - if (alg != NULL) - ossl_method_cache_flush_alg(store, alg); + if (alg != NULL) { + ossl_provider_clear_all_operation_bits(store->ctx); + store->nelem -= lh_QUERY_num_items(alg->cache); + impl_cache_flush_alg(0, alg, NULL); + } } -int ossl_method_store_cache_flush_all(OSSL_METHOD_STORE *store) +int ossl_method_store_flush_cache(OSSL_METHOD_STORE *store, int all) { + void *arg = (all != 0 ? store->algs : NULL); + if (!ossl_property_write_lock(store)) return 0; - ossl_sa_ALGORITHM_doall(store->algs, &impl_cache_flush_alg); - store->cache_nelem = 0; + ossl_provider_clear_all_operation_bits(store->ctx); + ossl_sa_ALGORITHM_doall_arg(store->algs, &impl_cache_flush_alg, arg); + store->nelem = 0; ossl_property_unlock(store); return 1; } @@ -652,21 +579,14 @@ static void impl_cache_flush_one_alg(ossl_uintmax_t idx, ALGORITHM *alg, static void ossl_method_cache_flush_some(OSSL_METHOD_STORE *store) { IMPL_CACHE_FLUSH state; - static TSAN_QUALIFIER uint32_t global_seed = 1; state.nelem = 0; - state.using_global_seed = 0; - if ((state.seed = OPENSSL_rdtsc()) == 0) { - /* If there is no timer available, seed another way */ - state.using_global_seed = 1; - state.seed = tsan_load(&global_seed); - } - store->cache_need_flush = 0; + if ((state.seed = OPENSSL_rdtsc()) == 0) + state.seed = 1; + ossl_provider_clear_all_operation_bits(store->ctx); + store->need_flush = 0; ossl_sa_ALGORITHM_doall_arg(store->algs, &impl_cache_flush_one_alg, &state); - store->cache_nelem = state.nelem; - /* Without a timer, update the global seed */ - if (state.using_global_seed) - tsan_add(&global_seed, state.seed); + store->nelem = state.nelem; } int ossl_method_store_cache_get(OSSL_METHOD_STORE *store, OSSL_PROVIDER *prov, @@ -717,7 +637,7 @@ int ossl_method_store_cache_set(OSSL_METHOD_STORE *store, OSSL_PROVIDER *prov, if (!ossl_property_write_lock(store)) return 0; - if (store->cache_need_flush) + if (store->need_flush) ossl_method_cache_flush_some(store); alg = ossl_method_store_retrieve(store, nid); if (alg == NULL) @@ -728,7 +648,7 @@ int ossl_method_store_cache_set(OSSL_METHOD_STORE *store, OSSL_PROVIDER *prov, elem.provider = prov; if ((old = lh_QUERY_delete(alg->cache, &elem)) != NULL) { impl_cache_free(old); - store->cache_nelem--; + store->nelem--; } goto end; } @@ -747,8 +667,8 @@ int ossl_method_store_cache_set(OSSL_METHOD_STORE *store, OSSL_PROVIDER *prov, goto end; } if (!lh_QUERY_error(alg->cache)) { - if (++store->cache_nelem >= IMPL_CACHE_FLUSH_THRESHOLD) - store->cache_need_flush = 1; + if (++store->nelem >= IMPL_CACHE_FLUSH_THRESHOLD) + store->need_flush = 1; goto end; } ossl_method_free(&p->method); diff --git a/openssl/src/crypto/property/property_local.h b/openssl/src/crypto/property/property_local.h index 797fb3bf5..6b85ce158 100644 --- a/openssl/src/crypto/property/property_local.h +++ b/openssl/src/crypto/property/property_local.h @@ -52,4 +52,4 @@ int ossl_property_has_optional(const OSSL_PROPERTY_LIST *query); /* Property definition cache functions */ OSSL_PROPERTY_LIST *ossl_prop_defn_get(OSSL_LIB_CTX *ctx, const char *prop); int ossl_prop_defn_set(OSSL_LIB_CTX *ctx, const char *prop, - OSSL_PROPERTY_LIST **pl); + OSSL_PROPERTY_LIST *pl); diff --git a/openssl/src/crypto/property/property_parse.c b/openssl/src/crypto/property/property_parse.c index dbe766d39..cea859793 100644 --- a/openssl/src/crypto/property/property_parse.c +++ b/openssl/src/crypto/property/property_parse.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -14,7 +14,6 @@ #include #include "internal/propertyerr.h" #include "internal/property.h" -#include "internal/numbers.h" #include "crypto/ctype.h" #include "internal/nelem.h" #include "property_local.h" @@ -98,18 +97,9 @@ static int parse_number(const char *t[], OSSL_PROPERTY_DEFINITION *res) const char *s = *t; int64_t v = 0; + if (!ossl_isdigit(*s)) + return 0; do { - if (!ossl_isdigit(*s)) { - ERR_raise_data(ERR_LIB_PROP, PROP_R_NOT_A_DECIMAL_DIGIT, - "HERE-->%s", *t); - return 0; - } - /* overflow check */ - if (v > ((INT64_MAX - (*s - '0')) / 10)) { - ERR_raise_data(ERR_LIB_PROP, PROP_R_PARSE_FAILED, - "Property %s overflows", *t); - return 0; - } v = v * 10 + (*s++ - '0'); } while (ossl_isdigit(*s)); if (!ossl_isspace(*s) && *s != '\0' && *s != ',') { @@ -127,27 +117,15 @@ static int parse_hex(const char *t[], OSSL_PROPERTY_DEFINITION *res) { const char *s = *t; int64_t v = 0; - int sval; + if (!ossl_isxdigit(*s)) + return 0; do { - if (ossl_isdigit(*s)) { - sval = *s - '0'; - } else if (ossl_isxdigit(*s)) { - sval = ossl_tolower(*s) - 'a' + 10; - } else { - ERR_raise_data(ERR_LIB_PROP, PROP_R_NOT_AN_HEXADECIMAL_DIGIT, - "%s", *t); - return 0; - } - - if (v > ((INT64_MAX - sval) / 16)) { - ERR_raise_data(ERR_LIB_PROP, PROP_R_PARSE_FAILED, - "Property %s overflows", *t); - return 0; - } - v <<= 4; - v += sval; + if (ossl_isdigit(*s)) + v += *s - '0'; + else + v += ossl_tolower(*s) - 'a'; } while (ossl_isxdigit(*++s)); if (!ossl_isspace(*s) && *s != '\0' && *s != ',') { ERR_raise_data(ERR_LIB_PROP, PROP_R_NOT_AN_HEXADECIMAL_DIGIT, @@ -165,18 +143,9 @@ static int parse_oct(const char *t[], OSSL_PROPERTY_DEFINITION *res) const char *s = *t; int64_t v = 0; + if (*s == '9' || *s == '8' || !ossl_isdigit(*s)) + return 0; do { - if (*s == '9' || *s == '8' || !ossl_isdigit(*s)) { - ERR_raise_data(ERR_LIB_PROP, PROP_R_NOT_AN_OCTAL_DIGIT, - "HERE-->%s", *t); - return 0; - } - if (v > ((INT64_MAX - (*s - '0')) / 8)) { - ERR_raise_data(ERR_LIB_PROP, PROP_R_PARSE_FAILED, - "Property %s overflows", *t); - return 0; - } - v = (v << 3) + (*s - '0'); } while (ossl_isdigit(*++s) && *s != '9' && *s != '8'); if (!ossl_isspace(*s) && *s != '\0' && *s != ',') { @@ -244,10 +213,11 @@ static int parse_unquoted(OSSL_LIB_CTX *ctx, const char *t[], return 0; } v[i] = 0; - if (err) + if (err) { ERR_raise_data(ERR_LIB_PROP, PROP_R_STRING_TOO_LONG, "HERE-->%s", *t); - else if ((res->v.str_val = ossl_property_value(ctx, v, create)) == 0) - err = 1; + } else { + res->v.str_val = ossl_property_value(ctx, v, create); + } *t = skip_space(s); res->type = OSSL_PROPERTY_TYPE_STRING; return !err; @@ -608,7 +578,7 @@ static void put_char(char ch, char **buf, size_t *remain, size_t *needed) ++*needed; return; } - if (*remain == 1) + if(*remain == 1) **buf = '\0'; else **buf = ch; @@ -619,50 +589,24 @@ static void put_char(char ch, char **buf, size_t *remain, size_t *needed) static void put_str(const char *str, char **buf, size_t *remain, size_t *needed) { - size_t olen, len, i; - char quote = '\0'; - int quotes; + size_t olen, len; len = olen = strlen(str); *needed += len; - /* - * Check to see if we need quotes or not. - * Characters that are legal in a PropertyName don't need quoting. - * We simply assume all others require quotes. - */ - for (i = 0; i < len; i++) - if (!ossl_isalnum(str[i]) && str[i] != '.' && str[i] != '_') { - /* Default to single quotes ... */ - if (quote == '\0') - quote = '\''; - /* ... but use double quotes if a single is present */ - if (str[i] == '\'') - quote = '"'; - } - - quotes = quote != '\0'; - if (*remain == 0) { - *needed += 2 * quotes; + if (*remain == 0) return; - } - if (quotes) - put_char(quote, buf, remain, needed); - - if (*remain < len + 1 + quotes) + if(*remain < len + 1) len = *remain - 1; - if (len > 0) { + if(len > 0) { memcpy(*buf, str, len); *buf += len; *remain -= len; } - if (quotes) - put_char(quote, buf, remain, needed); - - if (len < olen && *remain == 1) { + if(len < olen && *remain == 1) { **buf = '\0'; ++*buf; --*remain; diff --git a/openssl/src/crypto/property/property_string.c b/openssl/src/crypto/property/property_string.c index e06f47a6b..ef87a6a78 100644 --- a/openssl/src/crypto/property/property_string.c +++ b/openssl/src/crypto/property/property_string.c @@ -13,7 +13,6 @@ #include #include "crypto/lhash.h" #include "property_local.h" -#include "crypto/context.h" /* * Property strings are a consolidation of all strings seen by the property @@ -32,7 +31,7 @@ typedef struct { char body[1]; } PROPERTY_STRING; -DEFINE_LHASH_OF_EX(PROPERTY_STRING); +DEFINE_LHASH_OF(PROPERTY_STRING); typedef LHASH_OF(PROPERTY_STRING) PROP_TABLE; typedef struct { @@ -41,10 +40,6 @@ typedef struct { PROP_TABLE *prop_values; OSSL_PROPERTY_IDX prop_name_idx; OSSL_PROPERTY_IDX prop_value_idx; -#ifndef OPENSSL_SMALL_FOOTPRINT - STACK_OF(OPENSSL_CSTRING) *prop_namelist; - STACK_OF(OPENSSL_CSTRING) *prop_valuelist; -#endif } PROPERTY_STRING_DATA; static unsigned long property_hash(const PROPERTY_STRING *a) @@ -73,7 +68,7 @@ static void property_table_free(PROP_TABLE **pt) } } -void ossl_property_string_data_free(void *vpropdata) +static void property_string_data_free(void *vpropdata) { PROPERTY_STRING_DATA *propdata = vpropdata; @@ -83,44 +78,44 @@ void ossl_property_string_data_free(void *vpropdata) CRYPTO_THREAD_lock_free(propdata->lock); property_table_free(&propdata->prop_names); property_table_free(&propdata->prop_values); -#ifndef OPENSSL_SMALL_FOOTPRINT - sk_OPENSSL_CSTRING_free(propdata->prop_namelist); - sk_OPENSSL_CSTRING_free(propdata->prop_valuelist); - propdata->prop_namelist = propdata->prop_valuelist = NULL; -#endif propdata->prop_name_idx = propdata->prop_value_idx = 0; OPENSSL_free(propdata); } -void *ossl_property_string_data_new(OSSL_LIB_CTX *ctx) { +static void *property_string_data_new(OSSL_LIB_CTX *ctx) { PROPERTY_STRING_DATA *propdata = OPENSSL_zalloc(sizeof(*propdata)); if (propdata == NULL) return NULL; propdata->lock = CRYPTO_THREAD_lock_new(); + if (propdata->lock == NULL) + goto err; + propdata->prop_names = lh_PROPERTY_STRING_new(&property_hash, &property_cmp); + if (propdata->prop_names == NULL) + goto err; + propdata->prop_values = lh_PROPERTY_STRING_new(&property_hash, &property_cmp); -#ifndef OPENSSL_SMALL_FOOTPRINT - propdata->prop_namelist = sk_OPENSSL_CSTRING_new_null(); - propdata->prop_valuelist = sk_OPENSSL_CSTRING_new_null(); -#endif - if (propdata->lock == NULL -#ifndef OPENSSL_SMALL_FOOTPRINT - || propdata->prop_namelist == NULL - || propdata->prop_valuelist == NULL -#endif - || propdata->prop_names == NULL - || propdata->prop_values == NULL) { - ossl_property_string_data_free(propdata); - return NULL; - } + if (propdata->prop_values == NULL) + goto err; + return propdata; + +err: + property_string_data_free(propdata); + return NULL; } +static const OSSL_LIB_CTX_METHOD property_string_data_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + property_string_data_new, + property_string_data_free, +}; + static PROPERTY_STRING *new_property_string(const char *s, OSSL_PROPERTY_IDX *pidx) { @@ -139,66 +134,40 @@ static PROPERTY_STRING *new_property_string(const char *s, return ps; } -static OSSL_PROPERTY_IDX ossl_property_string(OSSL_LIB_CTX *ctx, int name, - int create, const char *s) +static OSSL_PROPERTY_IDX ossl_property_string(CRYPTO_RWLOCK *lock, + PROP_TABLE *t, + OSSL_PROPERTY_IDX *pidx, + const char *s) { PROPERTY_STRING p, *ps, *ps_new; - PROP_TABLE *t; - OSSL_PROPERTY_IDX *pidx; - PROPERTY_STRING_DATA *propdata - = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_PROPERTY_STRING_INDEX); - - if (propdata == NULL) - return 0; - t = name ? propdata->prop_names : propdata->prop_values; p.s = s; - if (!CRYPTO_THREAD_read_lock(propdata->lock)) { + if (!CRYPTO_THREAD_read_lock(lock)) { ERR_raise(ERR_LIB_CRYPTO, ERR_R_UNABLE_TO_GET_READ_LOCK); return 0; } ps = lh_PROPERTY_STRING_retrieve(t, &p); - if (ps == NULL && create) { - CRYPTO_THREAD_unlock(propdata->lock); - if (!CRYPTO_THREAD_write_lock(propdata->lock)) { + if (ps == NULL && pidx != NULL) { + CRYPTO_THREAD_unlock(lock); + if (!CRYPTO_THREAD_write_lock(lock)) { ERR_raise(ERR_LIB_CRYPTO, ERR_R_UNABLE_TO_GET_WRITE_LOCK); return 0; } - pidx = name ? &propdata->prop_name_idx : &propdata->prop_value_idx; ps = lh_PROPERTY_STRING_retrieve(t, &p); if (ps == NULL && (ps_new = new_property_string(s, pidx)) != NULL) { -#ifndef OPENSSL_SMALL_FOOTPRINT - STACK_OF(OPENSSL_CSTRING) *slist; - - slist = name ? propdata->prop_namelist : propdata->prop_valuelist; - if (sk_OPENSSL_CSTRING_push(slist, ps_new->s) <= 0) { - property_free(ps_new); - CRYPTO_THREAD_unlock(propdata->lock); - return 0; - } -#endif lh_PROPERTY_STRING_insert(t, ps_new); if (lh_PROPERTY_STRING_error(t)) { - /*- - * Undo the previous push which means also decrementing the - * index and freeing the allocated storage. - */ -#ifndef OPENSSL_SMALL_FOOTPRINT - sk_OPENSSL_CSTRING_pop(slist); -#endif property_free(ps_new); - --*pidx; - CRYPTO_THREAD_unlock(propdata->lock); + CRYPTO_THREAD_unlock(lock); return 0; } ps = ps_new; } } - CRYPTO_THREAD_unlock(propdata->lock); + CRYPTO_THREAD_unlock(lock); return ps != NULL ? ps->idx : 0; } -#ifdef OPENSSL_SMALL_FOOTPRINT struct find_str_st { const char *str; OSSL_PROPERTY_IDX idx; @@ -211,47 +180,45 @@ static void find_str_fn(PROPERTY_STRING *prop, void *vfindstr) if (prop->idx == findstr->idx) findstr->str = prop->s; } -#endif static const char *ossl_property_str(int name, OSSL_LIB_CTX *ctx, OSSL_PROPERTY_IDX idx) { - const char *r; + struct find_str_st findstr; PROPERTY_STRING_DATA *propdata - = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_PROPERTY_STRING_INDEX); + = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_PROPERTY_STRING_INDEX, + &property_string_data_method); if (propdata == NULL) return NULL; + findstr.str = NULL; + findstr.idx = idx; + if (!CRYPTO_THREAD_read_lock(propdata->lock)) { ERR_raise(ERR_LIB_CRYPTO, ERR_R_UNABLE_TO_GET_READ_LOCK); return NULL; } -#ifdef OPENSSL_SMALL_FOOTPRINT - { - struct find_str_st findstr; - - findstr.str = NULL; - findstr.idx = idx; - - lh_PROPERTY_STRING_doall_arg(name ? propdata->prop_names - : propdata->prop_values, - find_str_fn, &findstr); - r = findstr.str; - } -#else - r = sk_OPENSSL_CSTRING_value(name ? propdata->prop_namelist - : propdata->prop_valuelist, idx - 1); -#endif + lh_PROPERTY_STRING_doall_arg(name ? propdata->prop_names + : propdata->prop_values, + find_str_fn, &findstr); CRYPTO_THREAD_unlock(propdata->lock); - return r; + return findstr.str; } OSSL_PROPERTY_IDX ossl_property_name(OSSL_LIB_CTX *ctx, const char *s, int create) { - return ossl_property_string(ctx, 1, create, s); + PROPERTY_STRING_DATA *propdata + = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_PROPERTY_STRING_INDEX, + &property_string_data_method); + + if (propdata == NULL) + return 0; + return ossl_property_string(propdata->lock, propdata->prop_names, + create ? &propdata->prop_name_idx : NULL, + s); } const char *ossl_property_name_str(OSSL_LIB_CTX *ctx, OSSL_PROPERTY_IDX idx) @@ -262,7 +229,15 @@ const char *ossl_property_name_str(OSSL_LIB_CTX *ctx, OSSL_PROPERTY_IDX idx) OSSL_PROPERTY_IDX ossl_property_value(OSSL_LIB_CTX *ctx, const char *s, int create) { - return ossl_property_string(ctx, 0, create, s); + PROPERTY_STRING_DATA *propdata + = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_PROPERTY_STRING_INDEX, + &property_string_data_method); + + if (propdata == NULL) + return 0; + return ossl_property_string(propdata->lock, propdata->prop_values, + create ? &propdata->prop_value_idx : NULL, + s); } const char *ossl_property_value_str(OSSL_LIB_CTX *ctx, OSSL_PROPERTY_IDX idx) diff --git a/openssl/src/crypto/provider.c b/openssl/src/crypto/provider.c index b55561abf..114b42692 100644 --- a/openssl/src/crypto/provider.c +++ b/openssl/src/crypto/provider.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,15 +15,15 @@ #include "internal/provider.h" #include "provider_local.h" -OSSL_PROVIDER *OSSL_PROVIDER_try_load_ex(OSSL_LIB_CTX *libctx, const char *name, - OSSL_PARAM *params, int retain_fallbacks) +OSSL_PROVIDER *OSSL_PROVIDER_try_load(OSSL_LIB_CTX *libctx, const char *name, + int retain_fallbacks) { OSSL_PROVIDER *prov = NULL, *actual; int isnew = 0; /* Find it or create it */ if ((prov = ossl_provider_find(libctx, name, 0)) == NULL) { - if ((prov = ossl_provider_new(libctx, name, NULL, params, 0)) == NULL) + if ((prov = ossl_provider_new(libctx, name, NULL, 0)) == NULL) return NULL; isnew = 1; } @@ -49,25 +49,14 @@ OSSL_PROVIDER *OSSL_PROVIDER_try_load_ex(OSSL_LIB_CTX *libctx, const char *name, return actual; } -OSSL_PROVIDER *OSSL_PROVIDER_try_load(OSSL_LIB_CTX *libctx, const char *name, - int retain_fallbacks) -{ - return OSSL_PROVIDER_try_load_ex(libctx, name, NULL, retain_fallbacks); -} - -OSSL_PROVIDER *OSSL_PROVIDER_load_ex(OSSL_LIB_CTX *libctx, const char *name, OSSL_PARAM *params) +OSSL_PROVIDER *OSSL_PROVIDER_load(OSSL_LIB_CTX *libctx, const char *name) { /* Any attempt to load a provider disables auto-loading of defaults */ if (ossl_provider_disable_fallback_loading(libctx)) - return OSSL_PROVIDER_try_load_ex(libctx, name, params, 0); + return OSSL_PROVIDER_try_load(libctx, name, 0); return NULL; } -OSSL_PROVIDER *OSSL_PROVIDER_load(OSSL_LIB_CTX *libctx, const char *name) -{ - return OSSL_PROVIDER_load_ex(libctx, name, NULL); -} - int OSSL_PROVIDER_unload(OSSL_PROVIDER *prov) { if (!ossl_provider_deactivate(prov, 1)) @@ -134,8 +123,10 @@ int OSSL_PROVIDER_add_builtin(OSSL_LIB_CTX *libctx, const char *name, } memset(&entry, 0, sizeof(entry)); entry.name = OPENSSL_strdup(name); - if (entry.name == NULL) + if (entry.name == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } entry.init = init_fn; if (!ossl_provider_info_add_to_store(libctx, &entry)) { ossl_provider_info_clear(&entry); diff --git a/openssl/src/crypto/provider_child.c b/openssl/src/crypto/provider_child.c index 52e9cb405..977ea4db3 100644 --- a/openssl/src/crypto/provider_child.c +++ b/openssl/src/crypto/provider_child.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -16,7 +16,6 @@ #include "internal/provider.h" #include "internal/cryptlib.h" #include "crypto/evp.h" -#include "crypto/context.h" DEFINE_STACK_OF(OSSL_PROVIDER) @@ -34,12 +33,12 @@ struct child_prov_globals { OSSL_FUNC_provider_free_fn *c_prov_free; }; -void *ossl_child_prov_ctx_new(OSSL_LIB_CTX *libctx) +static void *child_prov_ossl_ctx_new(OSSL_LIB_CTX *libctx) { return OPENSSL_zalloc(sizeof(struct child_prov_globals)); } -void ossl_child_prov_ctx_free(void *vgbl) +static void child_prov_ossl_ctx_free(void *vgbl) { struct child_prov_globals *gbl = vgbl; @@ -47,6 +46,12 @@ void ossl_child_prov_ctx_free(void *vgbl) OPENSSL_free(gbl); } +static const OSSL_LIB_CTX_METHOD child_prov_ossl_ctx_method = { + OSSL_LIB_CTX_METHOD_LOW_PRIORITY, + child_prov_ossl_ctx_new, + child_prov_ossl_ctx_free, +}; + static OSSL_provider_init_fn ossl_child_provider_init; static int ossl_child_provider_init(const OSSL_CORE_HANDLE *handle, @@ -79,7 +84,8 @@ static int ossl_child_provider_init(const OSSL_CORE_HANDLE *handle, */ ctx = (OSSL_LIB_CTX *)c_get_libctx(handle); - gbl = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX); + gbl = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX, + &child_prov_ossl_ctx_method); if (gbl == NULL) return 0; @@ -97,7 +103,8 @@ static int provider_create_child_cb(const OSSL_CORE_HANDLE *prov, void *cbdata) OSSL_PROVIDER *cprov; int ret = 0; - gbl = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX); + gbl = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX, + &child_prov_ossl_ctx_method); if (gbl == NULL) return 0; @@ -132,13 +139,11 @@ static int provider_create_child_cb(const OSSL_CORE_HANDLE *prov, void *cbdata) * init children */ if ((cprov = ossl_provider_new(ctx, provname, ossl_child_provider_init, - NULL, 1)) == NULL) + 1)) == NULL) goto err; - if (!ossl_provider_activate(cprov, 0, 0)) { - ossl_provider_free(cprov); + if (!ossl_provider_activate(cprov, 0, 0)) goto err; - } if (!ossl_provider_set_child(cprov, prov) || !ossl_provider_add_to_store(cprov, NULL, 0)) { @@ -161,7 +166,8 @@ static int provider_remove_child_cb(const OSSL_CORE_HANDLE *prov, void *cbdata) const char *provname; OSSL_PROVIDER *cprov; - gbl = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX); + gbl = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX, + &child_prov_ossl_ctx_method); if (gbl == NULL) return 0; @@ -197,7 +203,8 @@ int ossl_provider_init_as_child(OSSL_LIB_CTX *ctx, if (ctx == NULL) return 0; - gbl = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX); + gbl = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX, + &child_prov_ossl_ctx_method); if (gbl == NULL) return 0; @@ -264,53 +271,36 @@ int ossl_provider_init_as_child(OSSL_LIB_CTX *ctx, void ossl_provider_deinit_child(OSSL_LIB_CTX *ctx) { struct child_prov_globals *gbl - = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX); + = ossl_lib_ctx_get_data(ctx, OSSL_LIB_CTX_CHILD_PROVIDER_INDEX, + &child_prov_ossl_ctx_method); if (gbl == NULL) return; gbl->c_provider_deregister_child_cb(gbl->handle); } -/* - * ossl_provider_up_ref_parent() and ossl_provider_free_parent() do - * nothing in "self-referencing" child providers, i.e. when the parent - * of the child provider is the same as the provider where this child - * provider was created. - * This allows the teardown function in the parent provider to be called - * at the correct moment. - * For child providers in other providers, the reference count is done to - * ensure that cross referencing is recorded. These should be cleared up - * through that providers teardown, as part of freeing its child libctx. - */ - int ossl_provider_up_ref_parent(OSSL_PROVIDER *prov, int activate) { struct child_prov_globals *gbl; - const OSSL_CORE_HANDLE *parent_handle; gbl = ossl_lib_ctx_get_data(ossl_provider_libctx(prov), - OSSL_LIB_CTX_CHILD_PROVIDER_INDEX); + OSSL_LIB_CTX_CHILD_PROVIDER_INDEX, + &child_prov_ossl_ctx_method); if (gbl == NULL) return 0; - parent_handle = ossl_provider_get_parent(prov); - if (parent_handle == gbl->handle) - return 1; - return gbl->c_prov_up_ref(parent_handle, activate); + return gbl->c_prov_up_ref(ossl_provider_get_parent(prov), activate); } int ossl_provider_free_parent(OSSL_PROVIDER *prov, int deactivate) { struct child_prov_globals *gbl; - const OSSL_CORE_HANDLE *parent_handle; gbl = ossl_lib_ctx_get_data(ossl_provider_libctx(prov), - OSSL_LIB_CTX_CHILD_PROVIDER_INDEX); + OSSL_LIB_CTX_CHILD_PROVIDER_INDEX, + &child_prov_ossl_ctx_method); if (gbl == NULL) return 0; - parent_handle = ossl_provider_get_parent(prov); - if (parent_handle == gbl->handle) - return 1; return gbl->c_prov_free(ossl_provider_get_parent(prov), deactivate); } diff --git a/openssl/src/crypto/provider_conf.c b/openssl/src/crypto/provider_conf.c index 6a8b88e2e..c13c887c3 100644 --- a/openssl/src/crypto/provider_conf.c +++ b/openssl/src/crypto/provider_conf.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -16,7 +16,6 @@ #include "internal/provider.h" #include "internal/cryptlib.h" #include "provider_local.h" -#include "crypto/context.h" DEFINE_STACK_OF(OSSL_PROVIDER) @@ -27,7 +26,7 @@ typedef struct { STACK_OF(OSSL_PROVIDER) *activated_providers; } PROVIDER_CONF_GLOBAL; -void *ossl_prov_conf_ctx_new(OSSL_LIB_CTX *libctx) +static void *prov_conf_ossl_ctx_new(OSSL_LIB_CTX *libctx) { PROVIDER_CONF_GLOBAL *pcgbl = OPENSSL_zalloc(sizeof(*pcgbl)); @@ -43,7 +42,7 @@ void *ossl_prov_conf_ctx_new(OSSL_LIB_CTX *libctx) return pcgbl; } -void ossl_prov_conf_ctx_free(void *vpcgbl) +static void prov_conf_ossl_ctx_free(void *vpcgbl) { PROVIDER_CONF_GLOBAL *pcgbl = vpcgbl; @@ -55,6 +54,13 @@ void ossl_prov_conf_ctx_free(void *vpcgbl) OPENSSL_free(pcgbl); } +static const OSSL_LIB_CTX_METHOD provider_conf_ossl_ctx_method = { + /* Must be freed before the provider store is freed */ + OSSL_LIB_CTX_METHOD_PRIORITY_2, + prov_conf_ossl_ctx_new, + prov_conf_ossl_ctx_free, +}; + static const char *skip_dot(const char *name) { const char *p = strchr(name, '.'); @@ -64,22 +70,13 @@ static const char *skip_dot(const char *name) return name; } -/* - * Parse the provider params section - * Returns: - * 1 for success - * 0 for non-fatal errors - * < 0 for fatal errors - */ -static int provider_conf_params_internal(OSSL_PROVIDER *prov, - OSSL_PROVIDER_INFO *provinfo, - const char *name, const char *value, - const CONF *cnf, - STACK_OF(OPENSSL_CSTRING) *visited) +static int provider_conf_params(OSSL_PROVIDER *prov, + OSSL_PROVIDER_INFO *provinfo, + const char *name, const char *value, + const CONF *cnf) { STACK_OF(CONF_VALUE) *sect; int ok = 1; - int rc = 0; sect = NCONF_get_section(cnf, value); if (sect != NULL) { @@ -89,25 +86,6 @@ static int provider_conf_params_internal(OSSL_PROVIDER *prov, OSSL_TRACE1(CONF, "Provider params: start section %s\n", value); - /* - * Check to see if the provided section value has already - * been visited. If it has, then we have a recursive lookup - * in the configuration which isn't valid. As such we should error - * out - */ - for (i = 0; i < sk_OPENSSL_CSTRING_num(visited); i++) { - if (sk_OPENSSL_CSTRING_value(visited, i) == value) { - ERR_raise(ERR_LIB_CONF, CONF_R_RECURSIVE_SECTION_REFERENCE); - return -1; - } - } - - /* - * We've not visited this node yet, so record it on the stack - */ - if (!sk_OPENSSL_CSTRING_push(visited, value)) - return -1; - if (name != NULL) { OPENSSL_strlcpy(buffer, name, sizeof(buffer)); OPENSSL_strlcat(buffer, ".", sizeof(buffer)); @@ -117,20 +95,14 @@ static int provider_conf_params_internal(OSSL_PROVIDER *prov, for (i = 0; i < sk_CONF_VALUE_num(sect); i++) { CONF_VALUE *sectconf = sk_CONF_VALUE_value(sect, i); - if (buffer_len + strlen(sectconf->name) >= sizeof(buffer)) { - sk_OPENSSL_CSTRING_pop(visited); - return -1; - } + if (buffer_len + strlen(sectconf->name) >= sizeof(buffer)) + return 0; buffer[buffer_len] = '\0'; OPENSSL_strlcat(buffer, sectconf->name, sizeof(buffer)); - rc = provider_conf_params_internal(prov, provinfo, buffer, - sectconf->value, cnf, visited); - if (rc < 0) { - sk_OPENSSL_CSTRING_pop(visited); - return rc; - } + if (!provider_conf_params(prov, provinfo, buffer, sectconf->value, + cnf)) + return 0; } - sk_OPENSSL_CSTRING_pop(visited); OSSL_TRACE1(CONF, "Provider params: finish section %s\n", value); } else { @@ -144,33 +116,6 @@ static int provider_conf_params_internal(OSSL_PROVIDER *prov, return ok; } -/* - * recursively parse the provider configuration section - * of the config file. - * Returns - * 1 on success - * 0 on non-fatal error - * < 0 on fatal errors - */ -static int provider_conf_params(OSSL_PROVIDER *prov, - OSSL_PROVIDER_INFO *provinfo, - const char *name, const char *value, - const CONF *cnf) -{ - int rc; - STACK_OF(OPENSSL_CSTRING) *visited = sk_OPENSSL_CSTRING_new_null(); - - if (visited == NULL) - return -1; - - rc = provider_conf_params_internal(prov, provinfo, name, - value, cnf, visited); - - sk_OPENSSL_CSTRING_free(visited); - - return rc; -} - static int prov_already_activated(const char *name, STACK_OF(OSSL_PROVIDER) *activated) { @@ -191,133 +136,16 @@ static int prov_already_activated(const char *name, return 0; } -/* - * Attempt to activate a provider - * Returns: - * 1 on successful activation - * 0 on failed activation for non-fatal error - * < 0 on failed activation for fatal errors - */ -static int provider_conf_activate(OSSL_LIB_CTX *libctx, const char *name, - const char *value, const char *path, - int soft, const CONF *cnf) -{ - PROVIDER_CONF_GLOBAL *pcgbl - = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_PROVIDER_CONF_INDEX); - OSSL_PROVIDER *prov = NULL, *actual = NULL; - int ok = 0; - - if (pcgbl == NULL || !CRYPTO_THREAD_write_lock(pcgbl->lock)) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return -1; - } - if (!prov_already_activated(name, pcgbl->activated_providers)) { - /* - * There is an attempt to activate a provider, so we should disable - * loading of fallbacks. Otherwise a misconfiguration could mean the - * intended provider does not get loaded. Subsequent fetches could - * then fallback to the default provider - which may be the wrong - * thing. - */ - if (!ossl_provider_disable_fallback_loading(libctx)) { - CRYPTO_THREAD_unlock(pcgbl->lock); - ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); - return -1; - } - prov = ossl_provider_find(libctx, name, 1); - if (prov == NULL) - prov = ossl_provider_new(libctx, name, NULL, NULL, 1); - if (prov == NULL) { - CRYPTO_THREAD_unlock(pcgbl->lock); - if (soft) - ERR_clear_error(); - return (soft == 0) ? -1 : 0; - } - - if (path != NULL) - ossl_provider_set_module_path(prov, path); - - ok = provider_conf_params(prov, NULL, NULL, value, cnf); - - if (ok == 1) { - if (!ossl_provider_activate(prov, 1, 0)) { - ok = 0; - } else if (!ossl_provider_add_to_store(prov, &actual, 0)) { - ossl_provider_deactivate(prov, 1); - ok = 0; - } else if (actual != prov - && !ossl_provider_activate(actual, 1, 0)) { - ossl_provider_free(actual); - ok = 0; - } else { - if (pcgbl->activated_providers == NULL) - pcgbl->activated_providers = sk_OSSL_PROVIDER_new_null(); - if (pcgbl->activated_providers == NULL - || !sk_OSSL_PROVIDER_push(pcgbl->activated_providers, - actual)) { - ossl_provider_deactivate(actual, 1); - ossl_provider_free(actual); - ok = 0; - } else { - ok = 1; - } - } - } - - if (ok <= 0) - ossl_provider_free(prov); - } - CRYPTO_THREAD_unlock(pcgbl->lock); - - return ok; -} - -static int provider_conf_parse_bool_setting(const char *confname, - const char *confvalue, int *val) -{ - - if (confvalue == NULL) { - ERR_raise_data(ERR_LIB_CRYPTO, CRYPTO_R_PROVIDER_SECTION_ERROR, - "directive %s set to unrecognized value", - confname); - return 0; - } - if ((strcmp(confvalue, "1") == 0) - || (strcmp(confvalue, "yes") == 0) - || (strcmp(confvalue, "YES") == 0) - || (strcmp(confvalue, "true") == 0) - || (strcmp(confvalue, "TRUE") == 0) - || (strcmp(confvalue, "on") == 0) - || (strcmp(confvalue, "ON") == 0)) { - *val = 1; - } else if ((strcmp(confvalue, "0") == 0) - || (strcmp(confvalue, "no") == 0) - || (strcmp(confvalue, "NO") == 0) - || (strcmp(confvalue, "false") == 0) - || (strcmp(confvalue, "FALSE") == 0) - || (strcmp(confvalue, "off") == 0) - || (strcmp(confvalue, "OFF") == 0)) { - *val = 0; - } else { - ERR_raise_data(ERR_LIB_CRYPTO, CRYPTO_R_PROVIDER_SECTION_ERROR, - "directive %s set to unrecognized value", - confname); - return 0; - } - - return 1; -} - static int provider_conf_load(OSSL_LIB_CTX *libctx, const char *name, const char *value, const CONF *cnf) { int i; STACK_OF(CONF_VALUE) *ecmds; int soft = 0; + OSSL_PROVIDER *prov = NULL, *actual = NULL; const char *path = NULL; - int activate = 0; + long activate = 0; int ok = 0; - int added = 0; name = skip_dot(name); OSSL_TRACE1(CONF, "Configuring provider %s\n", name); @@ -342,24 +170,82 @@ static int provider_conf_load(OSSL_LIB_CTX *libctx, const char *name, /* First handle some special pseudo confs */ /* Override provider name to use */ - if (strcmp(confname, "identity") == 0) { + if (strcmp(confname, "identity") == 0) name = confvalue; - } else if (strcmp(confname, "soft_load") == 0) { - if (!provider_conf_parse_bool_setting(confname, - confvalue, &soft)) - return 0; + else if (strcmp(confname, "soft_load") == 0) + soft = 1; /* Load a dynamic PROVIDER */ - } else if (strcmp(confname, "module") == 0) { + else if (strcmp(confname, "module") == 0) path = confvalue; - } else if (strcmp(confname, "activate") == 0) { - if (!provider_conf_parse_bool_setting(confname, - confvalue, &activate)) - return 0; - } + else if (strcmp(confname, "activate") == 0) + activate = 1; } if (activate) { - ok = provider_conf_activate(libctx, name, value, path, soft, cnf); + PROVIDER_CONF_GLOBAL *pcgbl + = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_PROVIDER_CONF_INDEX, + &provider_conf_ossl_ctx_method); + + if (pcgbl == NULL || !CRYPTO_THREAD_write_lock(pcgbl->lock)) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); + return 0; + } + if (!prov_already_activated(name, pcgbl->activated_providers)) { + /* + * There is an attempt to activate a provider, so we should disable + * loading of fallbacks. Otherwise a misconfiguration could mean the + * intended provider does not get loaded. Subsequent fetches could + * then fallback to the default provider - which may be the wrong + * thing. + */ + if (!ossl_provider_disable_fallback_loading(libctx)) { + CRYPTO_THREAD_unlock(pcgbl->lock); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); + return 0; + } + prov = ossl_provider_find(libctx, name, 1); + if (prov == NULL) + prov = ossl_provider_new(libctx, name, NULL, 1); + if (prov == NULL) { + CRYPTO_THREAD_unlock(pcgbl->lock); + if (soft) + ERR_clear_error(); + return 0; + } + + if (path != NULL) + ossl_provider_set_module_path(prov, path); + + ok = provider_conf_params(prov, NULL, NULL, value, cnf); + + if (ok) { + if (!ossl_provider_activate(prov, 1, 0)) { + ok = 0; + } else if (!ossl_provider_add_to_store(prov, &actual, 0)) { + ossl_provider_deactivate(prov, 1); + ok = 0; + } else if (actual != prov + && !ossl_provider_activate(actual, 1, 0)) { + ossl_provider_free(actual); + ok = 0; + } else { + if (pcgbl->activated_providers == NULL) + pcgbl->activated_providers = sk_OSSL_PROVIDER_new_null(); + if (pcgbl->activated_providers == NULL + || !sk_OSSL_PROVIDER_push(pcgbl->activated_providers, + actual)) { + ossl_provider_deactivate(actual, 1); + ossl_provider_free(actual); + ok = 0; + } else { + ok = 1; + } + } + } + if (!ok) + ossl_provider_free(prov); + } + CRYPTO_THREAD_unlock(pcgbl->lock); } else { OSSL_PROVIDER_INFO entry; @@ -367,33 +253,33 @@ static int provider_conf_load(OSSL_LIB_CTX *libctx, const char *name, ok = 1; if (name != NULL) { entry.name = OPENSSL_strdup(name); - if (entry.name == NULL) + if (entry.name == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); ok = 0; + } } if (ok && path != NULL) { entry.path = OPENSSL_strdup(path); - if (entry.path == NULL) + if (entry.path == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); ok = 0; + } } if (ok) ok = provider_conf_params(NULL, &entry, NULL, value, cnf); - if (ok >= 1 && (entry.path != NULL || entry.parameters != NULL)) { + if (ok && (entry.path != NULL || entry.parameters != NULL)) ok = ossl_provider_info_add_to_store(libctx, &entry); - added = 1; - } - if (added == 0) + if (!ok || (entry.path == NULL && entry.parameters == NULL)) { ossl_provider_info_clear(&entry); + } + } /* - * Provider activation returns a tristate: - * 1 for successful activation - * 0 for non-fatal activation failure - * < 0 for fatal activation failure - * We return success (1) for activation, (1) for non-fatal activation - * failure, and (0) for fatal activation failure + * Even if ok is 0, we still return success. Failure to load a provider is + * not fatal. We want to continue to load the rest of the config file. */ - return ok >= 0; + return 1; } static int provider_conf_init(CONF_IMODULE *md, const CONF *cnf) @@ -416,7 +302,7 @@ static int provider_conf_init(CONF_IMODULE *md, const CONF *cnf) for (i = 0; i < sk_CONF_VALUE_num(elist); i++) { cval = sk_CONF_VALUE_value(elist, i); if (!provider_conf_load(NCONF_get0_libctx((CONF *)cnf), - cval->name, cval->value, cnf)) + cval->name, cval->value, cnf)) return 0; } diff --git a/openssl/src/crypto/provider_core.c b/openssl/src/crypto/provider_core.c index 297b281a3..177358f05 100644 --- a/openssl/src/crypto/provider_core.c +++ b/openssl/src/crypto/provider_core.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,12 +15,7 @@ #include #include #include "crypto/cryptlib.h" -#ifndef FIPS_MODULE -#include "crypto/decoder.h" /* ossl_decoder_store_cache_flush */ -#include "crypto/encoder.h" /* ossl_encoder_store_cache_flush */ -#include "crypto/store.h" /* ossl_store_loader_store_cache_flush */ -#endif -#include "crypto/evp.h" /* evp_method_store_cache_flush */ +#include "crypto/evp.h" /* evp_method_store_flush */ #include "crypto/rand.h" #include "internal/nelem.h" #include "internal/thread_once.h" @@ -29,7 +24,6 @@ #include "internal/bio.h" #include "internal/core.h" #include "provider_local.h" -#include "crypto/context.h" #ifndef FIPS_MODULE # include #endif @@ -73,10 +67,11 @@ * The locks available are: * * The provider flag_lock: Used to control updates to the various provider - * "flags" (flag_initialized and flag_activated). + * "flags" (flag_initialized, flag_activated, flag_fallback) and associated + * "counts" (activatecnt). * - * The provider activatecnt_lock: Used to control updates to the provider - * activatecnt value. + * The provider refcnt_lock: Only ever used to control updates to the provider + * refcnt value. * * The provider optbits_lock: Used to control access to the provider's * operation_bits and operation_bits_sz fields. @@ -99,12 +94,12 @@ * introducing the possibility of deadlock. The following rules MUST be adhered * to in order to avoid that: * - Holding multiple locks at the same time is only allowed for the - * provider store lock, the provider activatecnt_lock and the provider flag_lock. + * provider store lock, the provider flag_lock and the provider refcnt_lock. * - When holding multiple locks they must be acquired in the following order of * precedence: * 1) provider store lock * 2) provider flag_lock - * 3) provider activatecnt_lock + * 3) provider refcnt_lock * - When releasing locks they must be released in the reverse order to which * they were acquired * - No locks may be held when making an upcall. NOTE: Some common functions @@ -142,13 +137,14 @@ struct ossl_provider_st { /* Flag bits */ unsigned int flag_initialized:1; unsigned int flag_activated:1; + unsigned int flag_fallback:1; /* Can be used as fallback */ /* Getting and setting the flags require synchronization */ CRYPTO_RWLOCK *flag_lock; /* OpenSSL library side data */ CRYPTO_REF_COUNT refcnt; - CRYPTO_RWLOCK *activatecnt_lock; /* For the activatecnt counter */ + CRYPTO_RWLOCK *refcnt_lock; /* For the ref counter */ int activatecnt; char *name; char *path; @@ -282,7 +278,7 @@ void ossl_provider_info_clear(OSSL_PROVIDER_INFO *info) sk_INFOPAIR_pop_free(info->parameters, infopair_free); } -void ossl_provider_store_free(void *vstore) +static void provider_store_free(void *vstore) { struct provider_store_st *store = vstore; size_t i; @@ -304,7 +300,7 @@ void ossl_provider_store_free(void *vstore) OPENSSL_free(store); } -void *ossl_provider_store_new(OSSL_LIB_CTX *ctx) +static void *provider_store_new(OSSL_LIB_CTX *ctx) { struct provider_store_st *store = OPENSSL_zalloc(sizeof(*store)); @@ -315,7 +311,7 @@ void *ossl_provider_store_new(OSSL_LIB_CTX *ctx) || (store->child_cbs = sk_OSSL_PROVIDER_CHILD_CB_new_null()) == NULL #endif || (store->lock = CRYPTO_THREAD_lock_new()) == NULL) { - ossl_provider_store_free(store); + provider_store_free(store); return NULL; } store->libctx = ctx; @@ -324,11 +320,19 @@ void *ossl_provider_store_new(OSSL_LIB_CTX *ctx) return store; } +static const OSSL_LIB_CTX_METHOD provider_store_method = { + /* Needs to be freed before the child provider data is freed */ + OSSL_LIB_CTX_METHOD_PRIORITY_1, + provider_store_new, + provider_store_free, +}; + static struct provider_store_st *get_provider_store(OSSL_LIB_CTX *libctx) { struct provider_store_st *store = NULL; - store = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_PROVIDER_STORE_INDEX); + store = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_PROVIDER_STORE_INDEX, + &provider_store_method); if (store == NULL) ERR_raise(ERR_LIB_CRYPTO, ERR_R_INTERNAL_ERROR); return store; @@ -371,8 +375,10 @@ int ossl_provider_info_add_to_store(OSSL_LIB_CTX *libctx, if (store->provinfosz == 0) { store->provinfo = OPENSSL_zalloc(sizeof(*store->provinfo) * BUILTINS_BLOCK_SIZE); - if (store->provinfo == NULL) + if (store->provinfo == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto err; + } store->provinfosz = BUILTINS_BLOCK_SIZE; } else if (store->numprovinfo == store->provinfosz) { OSSL_PROVIDER_INFO *tmpbuiltins; @@ -380,8 +386,10 @@ int ossl_provider_info_add_to_store(OSSL_LIB_CTX *libctx, tmpbuiltins = OPENSSL_realloc(store->provinfo, sizeof(*store->provinfo) * newsz); - if (tmpbuiltins == NULL) + if (tmpbuiltins == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto err; + } store->provinfo = tmpbuiltins; store->provinfosz = newsz; } @@ -395,7 +403,7 @@ int ossl_provider_info_add_to_store(OSSL_LIB_CTX *libctx, } OSSL_PROVIDER *ossl_provider_find(OSSL_LIB_CTX *libctx, const char *name, - ossl_unused int noconfig) + int noconfig) { struct provider_store_st *store = NULL; OSSL_PROVIDER *prov = NULL; @@ -404,7 +412,7 @@ OSSL_PROVIDER *ossl_provider_find(OSSL_LIB_CTX *libctx, const char *name, OSSL_PROVIDER tmpl = { 0, }; int i; -#if !defined(FIPS_MODULE) && !defined(OPENSSL_NO_AUTOLOAD_CONFIG) +#ifndef FIPS_MODULE /* * Make sure any providers are loaded from config before we try to find * them. @@ -416,9 +424,12 @@ OSSL_PROVIDER *ossl_provider_find(OSSL_LIB_CTX *libctx, const char *name, #endif tmpl.name = (char *)name; + /* + * A "find" operation can sort the stack, and therefore a write lock is + * required. + */ if (!CRYPTO_THREAD_write_lock(store->lock)) return NULL; - sk_OSSL_PROVIDER_sort(store->providers); if ((i = sk_OSSL_PROVIDER_find(store->providers, &tmpl)) != -1) prov = sk_OSSL_PROVIDER_value(store->providers, i); CRYPTO_THREAD_unlock(store->lock); @@ -440,32 +451,22 @@ static OSSL_PROVIDER *provider_new(const char *name, { OSSL_PROVIDER *prov = NULL; - if ((prov = OPENSSL_zalloc(sizeof(*prov))) == NULL) - return NULL; - if (!CRYPTO_NEW_REF(&prov->refcnt, 1)) { - OPENSSL_free(prov); - return NULL; - } - if ((prov->activatecnt_lock = CRYPTO_THREAD_lock_new()) == NULL) { - ossl_provider_free(prov); - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); - return NULL; - } - - if ((prov->opbits_lock = CRYPTO_THREAD_lock_new()) == NULL + if ((prov = OPENSSL_zalloc(sizeof(*prov))) == NULL +#ifndef HAVE_ATOMICS + || (prov->refcnt_lock = CRYPTO_THREAD_lock_new()) == NULL +#endif + || (prov->opbits_lock = CRYPTO_THREAD_lock_new()) == NULL || (prov->flag_lock = CRYPTO_THREAD_lock_new()) == NULL + || (prov->name = OPENSSL_strdup(name)) == NULL || (prov->parameters = sk_INFOPAIR_deep_copy(parameters, infopair_copy, infopair_free)) == NULL) { ossl_provider_free(prov); - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); - return NULL; - } - if ((prov->name = OPENSSL_strdup(name)) == NULL) { - ossl_provider_free(prov); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return NULL; } + prov->refcnt = 1; /* 1 One reference to be returned */ prov->init_function = init_function; return prov; @@ -475,7 +476,7 @@ int ossl_provider_up_ref(OSSL_PROVIDER *prov) { int ref = 0; - if (CRYPTO_UP_REF(&prov->refcnt, &ref) <= 0) + if (CRYPTO_UP_REF(&prov->refcnt, &ref, prov->refcnt_lock) <= 0) return 0; #ifndef FIPS_MODULE @@ -516,7 +517,7 @@ static int provider_free_intern(OSSL_PROVIDER *prov, int deactivate) */ OSSL_PROVIDER *ossl_provider_new(OSSL_LIB_CTX *libctx, const char *name, OSSL_provider_init_fn *init_function, - OSSL_PARAM *params, int noconfig) + int noconfig) { struct provider_store_st *store = NULL; OSSL_PROVIDER_INFO template; @@ -538,7 +539,7 @@ OSSL_PROVIDER *ossl_provider_new(OSSL_LIB_CTX *libctx, const char *name, } } if (p->name == NULL) { - /* Check if this is a user added provider */ + /* Check if this is a user added builtin provider */ if (!CRYPTO_THREAD_read_lock(store->lock)) return NULL; for (i = 0, p = store->provinfo; i < store->numprovinfo; p++, i++) { @@ -553,38 +554,10 @@ OSSL_PROVIDER *ossl_provider_new(OSSL_LIB_CTX *libctx, const char *name, template.init = init_function; } - if (params != NULL) { - int i; - - template.parameters = sk_INFOPAIR_new_null(); - if (template.parameters == NULL) - return NULL; - - for (i = 0; params[i].key != NULL; i++) { - if (params[i].data_type != OSSL_PARAM_UTF8_STRING) - continue; - if (ossl_provider_info_add_parameter(&template, params[i].key, - (char *)params[i].data) <= 0) { - sk_INFOPAIR_pop_free(template.parameters, infopair_free); - return NULL; - } - } - } - /* provider_new() generates an error, so no need here */ - prov = provider_new(name, template.init, template.parameters); - - if (params != NULL) /* We copied the parameters, let's free them */ - sk_INFOPAIR_pop_free(template.parameters, infopair_free); - - if (prov == NULL) + if ((prov = provider_new(name, template.init, template.parameters)) == NULL) return NULL; - if (!ossl_provider_set_module_path(prov, template.path)) { - ossl_provider_free(prov); - return NULL; - } - prov->libctx = libctx; #ifndef FIPS_MODULE prov->error_lib = ERR_get_next_error_library(); @@ -662,9 +635,9 @@ int ossl_provider_add_to_store(OSSL_PROVIDER *prov, OSSL_PROVIDER **actualprov, if (actualprov != NULL) { if (!ossl_provider_up_ref(actualtmp)) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); actualtmp = NULL; - return 0; + goto err; } *actualprov = actualtmp; } @@ -683,20 +656,13 @@ int ossl_provider_add_to_store(OSSL_PROVIDER *prov, OSSL_PROVIDER **actualprov, ossl_provider_deactivate(prov, 0); ossl_provider_free(prov); } -#ifndef FIPS_MODULE - else { - /* - * This can be done outside the lock. We tolerate other threads getting - * the wrong result briefly when creating OSSL_DECODER_CTXs. - */ - ossl_decoder_cache_flush(prov->libctx); - } -#endif return 1; err: CRYPTO_THREAD_unlock(store->lock); + if (actualprov != NULL) + ossl_provider_free(*actualprov); return 0; } @@ -705,7 +671,7 @@ void ossl_provider_free(OSSL_PROVIDER *prov) if (prov != NULL) { int ref = 0; - CRYPTO_DOWN_REF(&prov->refcnt, &ref); + CRYPTO_DOWN_REF(&prov->refcnt, &ref, prov->refcnt_lock); /* * When the refcount drops to zero, we clean up the provider. @@ -747,8 +713,9 @@ void ossl_provider_free(OSSL_PROVIDER *prov) sk_INFOPAIR_pop_free(prov->parameters, infopair_free); CRYPTO_THREAD_lock_free(prov->opbits_lock); CRYPTO_THREAD_lock_free(prov->flag_lock); - CRYPTO_THREAD_lock_free(prov->activatecnt_lock); - CRYPTO_FREE_REF(&prov->refcnt); +#ifndef HAVE_ATOMICS + CRYPTO_THREAD_lock_free(prov->refcnt_lock); +#endif OPENSSL_free(prov); } #ifndef FIPS_MODULE @@ -768,6 +735,7 @@ int ossl_provider_set_module_path(OSSL_PROVIDER *prov, const char *module_path) return 1; if ((prov->path = OPENSSL_strdup(module_path)) != NULL) return 1; + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; } @@ -776,26 +744,20 @@ static int infopair_add(STACK_OF(INFOPAIR) **infopairsk, const char *name, { INFOPAIR *pair = NULL; - if ((pair = OPENSSL_zalloc(sizeof(*pair))) == NULL - || (pair->name = OPENSSL_strdup(name)) == NULL - || (pair->value = OPENSSL_strdup(value)) == NULL) - goto err; - - if ((*infopairsk == NULL - && (*infopairsk = sk_INFOPAIR_new_null()) == NULL) - || sk_INFOPAIR_push(*infopairsk, pair) <= 0) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB); - goto err; - } - - return 1; + if ((pair = OPENSSL_zalloc(sizeof(*pair))) != NULL + && (*infopairsk != NULL + || (*infopairsk = sk_INFOPAIR_new_null()) != NULL) + && (pair->name = OPENSSL_strdup(name)) != NULL + && (pair->value = OPENSSL_strdup(value)) != NULL + && sk_INFOPAIR_push(*infopairsk, pair) > 0) + return 1; - err: if (pair != NULL) { OPENSSL_free(pair->name); OPENSSL_free(pair->value); OPENSSL_free(pair); } + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; } @@ -834,8 +796,10 @@ int OSSL_PROVIDER_set_default_search_path(OSSL_LIB_CTX *libctx, if (path != NULL) { p = OPENSSL_strdup(path); - if (p == NULL) + if (p == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } } if ((store = get_provider_store(libctx)) != NULL && CRYPTO_THREAD_write_lock(store->default_path_lock)) { @@ -848,19 +812,6 @@ int OSSL_PROVIDER_set_default_search_path(OSSL_LIB_CTX *libctx, return 0; } -const char *OSSL_PROVIDER_get0_default_search_path(OSSL_LIB_CTX *libctx) -{ - struct provider_store_st *store; - char *path = NULL; - - if ((store = get_provider_store(libctx)) != NULL - && CRYPTO_THREAD_read_lock(store->default_path_lock)) { - path = store->default_path; - CRYPTO_THREAD_unlock(store->default_path_lock); - } - return path; -} - /* * Internal version that doesn't affect the store flags, and thereby avoid * locking. Direct callers must remember to set the store flags when @@ -910,8 +861,10 @@ static int provider_init(OSSL_PROVIDER *prov) if (store->default_path != NULL) { allocated_load_dir = OPENSSL_strdup(store->default_path); CRYPTO_THREAD_unlock(store->default_path_lock); - if (allocated_load_dir == NULL) + if (allocated_load_dir == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); goto end; + } load_dir = allocated_load_dir; } else { CRYPTO_THREAD_unlock(store->default_path_lock); @@ -944,28 +897,16 @@ static int provider_init(OSSL_PROVIDER *prov) OPENSSL_free(allocated_load_dir); } - if (prov->module == NULL) { - /* DSO has already recorded errors, this is just a tracepoint */ - ERR_raise_data(ERR_LIB_CRYPTO, ERR_R_DSO_LIB, - "name=%s", prov->name); - goto end; - } - - prov->init_function = (OSSL_provider_init_fn *) - DSO_bind_func(prov->module, "OSSL_provider_init"); + if (prov->module != NULL) + prov->init_function = (OSSL_provider_init_fn *) + DSO_bind_func(prov->module, "OSSL_provider_init"); #endif } - /* Check for and call the initialise function for the provider. */ - if (prov->init_function == NULL) { - ERR_raise_data(ERR_LIB_CRYPTO, ERR_R_UNSUPPORTED, - "name=%s, provider has no provider init function", - prov->name); - goto end; - } - - if (!prov->init_function((OSSL_CORE_HANDLE *)prov, core_dispatch, - &provider_dispatch, &tmp_provctx)) { + /* Call the initialise function for the provider. */ + if (prov->init_function == NULL + || !prov->init_function((OSSL_CORE_HANDLE *)prov, core_dispatch, + &provider_dispatch, &tmp_provctx)) { ERR_raise_data(ERR_LIB_CRYPTO, ERR_R_INIT_FAIL, "name=%s", prov->name); goto end; @@ -973,46 +914,44 @@ static int provider_init(OSSL_PROVIDER *prov) prov->provctx = tmp_provctx; prov->dispatch = provider_dispatch; - if (provider_dispatch != NULL) { - for (; provider_dispatch->function_id != 0; provider_dispatch++) { - switch (provider_dispatch->function_id) { - case OSSL_FUNC_PROVIDER_TEARDOWN: - prov->teardown = - OSSL_FUNC_provider_teardown(provider_dispatch); - break; - case OSSL_FUNC_PROVIDER_GETTABLE_PARAMS: - prov->gettable_params = - OSSL_FUNC_provider_gettable_params(provider_dispatch); - break; - case OSSL_FUNC_PROVIDER_GET_PARAMS: - prov->get_params = - OSSL_FUNC_provider_get_params(provider_dispatch); - break; - case OSSL_FUNC_PROVIDER_SELF_TEST: - prov->self_test = - OSSL_FUNC_provider_self_test(provider_dispatch); - break; - case OSSL_FUNC_PROVIDER_GET_CAPABILITIES: - prov->get_capabilities = - OSSL_FUNC_provider_get_capabilities(provider_dispatch); - break; - case OSSL_FUNC_PROVIDER_QUERY_OPERATION: - prov->query_operation = - OSSL_FUNC_provider_query_operation(provider_dispatch); - break; - case OSSL_FUNC_PROVIDER_UNQUERY_OPERATION: - prov->unquery_operation = - OSSL_FUNC_provider_unquery_operation(provider_dispatch); - break; + for (; provider_dispatch->function_id != 0; provider_dispatch++) { + switch (provider_dispatch->function_id) { + case OSSL_FUNC_PROVIDER_TEARDOWN: + prov->teardown = + OSSL_FUNC_provider_teardown(provider_dispatch); + break; + case OSSL_FUNC_PROVIDER_GETTABLE_PARAMS: + prov->gettable_params = + OSSL_FUNC_provider_gettable_params(provider_dispatch); + break; + case OSSL_FUNC_PROVIDER_GET_PARAMS: + prov->get_params = + OSSL_FUNC_provider_get_params(provider_dispatch); + break; + case OSSL_FUNC_PROVIDER_SELF_TEST: + prov->self_test = + OSSL_FUNC_provider_self_test(provider_dispatch); + break; + case OSSL_FUNC_PROVIDER_GET_CAPABILITIES: + prov->get_capabilities = + OSSL_FUNC_provider_get_capabilities(provider_dispatch); + break; + case OSSL_FUNC_PROVIDER_QUERY_OPERATION: + prov->query_operation = + OSSL_FUNC_provider_query_operation(provider_dispatch); + break; + case OSSL_FUNC_PROVIDER_UNQUERY_OPERATION: + prov->unquery_operation = + OSSL_FUNC_provider_unquery_operation(provider_dispatch); + break; #ifndef OPENSSL_NO_ERR # ifndef FIPS_MODULE - case OSSL_FUNC_PROVIDER_GET_REASON_STRINGS: - p_get_reason_strings = - OSSL_FUNC_provider_get_reason_strings(provider_dispatch); - break; + case OSSL_FUNC_PROVIDER_GET_REASON_STRINGS: + p_get_reason_strings = + OSSL_FUNC_provider_get_reason_strings(provider_dispatch); + break; # endif #endif - } } } @@ -1104,9 +1043,8 @@ static int provider_deactivate(OSSL_PROVIDER *prov, int upcalls, return -1; } - CRYPTO_atomic_add(&prov->activatecnt, -1, &count, prov->activatecnt_lock); #ifndef FIPS_MODULE - if (count >= 1 && prov->ischild && upcalls) { + if (prov->activatecnt >= 2 && prov->ischild && upcalls) { /* * We have had a direct activation in this child libctx so we need to * now down the ref count in the parent provider. We do the actual down @@ -1117,7 +1055,7 @@ static int provider_deactivate(OSSL_PROVIDER *prov, int upcalls, } #endif - if (count < 1) + if ((count = --prov->activatecnt) < 1) prov->flag_activated = 0; #ifndef FIPS_MODULE else @@ -1138,14 +1076,6 @@ static int provider_deactivate(OSSL_PROVIDER *prov, int upcalls, if (lock) { CRYPTO_THREAD_unlock(prov->flag_lock); CRYPTO_THREAD_unlock(store->lock); - /* - * This can be done outside the lock. We tolerate other threads getting - * the wrong result briefly when creating OSSL_DECODER_CTXs. - */ -#ifndef FIPS_MODULE - if (count < 1) - ossl_decoder_cache_flush(prov->libctx); -#endif } #ifndef FIPS_MODULE if (freeparent) @@ -1198,24 +1128,16 @@ static int provider_activate(OSSL_PROVIDER *prov, int lock, int upcalls) #endif return -1; } - if (CRYPTO_atomic_add(&prov->activatecnt, 1, &count, prov->activatecnt_lock)) { - prov->flag_activated = 1; - if (count == 1 && store != NULL) { - ret = create_provider_children(prov); - } + count = ++prov->activatecnt; + prov->flag_activated = 1; + + if (prov->activatecnt == 1 && store != NULL) { + ret = create_provider_children(prov); } if (lock) { CRYPTO_THREAD_unlock(prov->flag_lock); CRYPTO_THREAD_unlock(store->lock); - /* - * This can be done outside the lock. We tolerate other threads getting - * the wrong result briefly when creating OSSL_DECODER_CTXs. - */ -#ifndef FIPS_MODULE - if (count == 1) - ossl_decoder_cache_flush(prov->libctx); -#endif } if (!ret) @@ -1237,62 +1159,8 @@ static int provider_flush_store_cache(const OSSL_PROVIDER *prov) freeing = store->freeing; CRYPTO_THREAD_unlock(store->lock); - if (!freeing) { - int acc - = evp_method_store_cache_flush(prov->libctx) -#ifndef FIPS_MODULE - + ossl_encoder_store_cache_flush(prov->libctx) - + ossl_decoder_store_cache_flush(prov->libctx) - + ossl_store_loader_store_cache_flush(prov->libctx) -#endif - ; - -#ifndef FIPS_MODULE - return acc == 4; -#else - return acc == 1; -#endif - } - return 1; -} - -static int provider_remove_store_methods(OSSL_PROVIDER *prov) -{ - struct provider_store_st *store; - int freeing; - - if ((store = get_provider_store(prov->libctx)) == NULL) - return 0; - - if (!CRYPTO_THREAD_read_lock(store->lock)) - return 0; - freeing = store->freeing; - CRYPTO_THREAD_unlock(store->lock); - - if (!freeing) { - int acc; - - if (!CRYPTO_THREAD_write_lock(prov->opbits_lock)) - return 0; - OPENSSL_free(prov->operation_bits); - prov->operation_bits = NULL; - prov->operation_bits_sz = 0; - CRYPTO_THREAD_unlock(prov->opbits_lock); - - acc = evp_method_store_remove_all_provided(prov) -#ifndef FIPS_MODULE - + ossl_encoder_store_remove_all_provided(prov) - + ossl_decoder_store_remove_all_provided(prov) - + ossl_store_loader_store_remove_all_provided(prov) -#endif - ; - -#ifndef FIPS_MODULE - return acc == 4; -#else - return acc == 1; -#endif - } + if (!freeing) + return evp_method_store_flush(prov->libctx); return 1; } @@ -1323,12 +1191,12 @@ int ossl_provider_deactivate(OSSL_PROVIDER *prov, int removechildren) if (prov == NULL || (count = provider_deactivate(prov, 1, removechildren)) < 0) return 0; - return count == 0 ? provider_remove_store_methods(prov) : 1; + return count == 0 ? provider_flush_store_cache(prov) : 1; } void *ossl_provider_ctx(const OSSL_PROVIDER *prov) { - return prov != NULL ? prov->provctx : NULL; + return prov->provctx; } /* @@ -1412,7 +1280,7 @@ int ossl_provider_doall_activated(OSSL_LIB_CTX *ctx, struct provider_store_st *store = get_provider_store(ctx); STACK_OF(OSSL_PROVIDER) *provs = NULL; -#if !defined(FIPS_MODULE) && !defined(OPENSSL_NO_AUTOLOAD_CONFIG) +#ifndef FIPS_MODULE /* * Make sure any providers are loaded from config before we try to use * them. @@ -1445,7 +1313,7 @@ int ossl_provider_doall_activated(OSSL_LIB_CTX *ctx, for (curr = max - 1; curr >= 0; curr--) { OSSL_PROVIDER *prov = sk_OSSL_PROVIDER_value(provs, curr); - if (!CRYPTO_THREAD_read_lock(prov->flag_lock)) + if (!CRYPTO_THREAD_write_lock(prov->flag_lock)) goto err_unlock; if (prov->flag_activated) { /* @@ -1453,19 +1321,20 @@ int ossl_provider_doall_activated(OSSL_LIB_CTX *ctx, * to avoid upping the ref count on the parent provider, which we * must not do while holding locks. */ - if (CRYPTO_UP_REF(&prov->refcnt, &ref) <= 0) { + if (CRYPTO_UP_REF(&prov->refcnt, &ref, prov->refcnt_lock) <= 0) { CRYPTO_THREAD_unlock(prov->flag_lock); goto err_unlock; } /* * It's already activated, but we up the activated count to ensure * it remains activated until after we've called the user callback. - * In theory this could mean the parent provider goes inactive, - * whilst still activated in the child for a short period. That's ok. + * We do this with no locking (because we already hold the locks) + * and no upcalls (which must not be called when locks are held). In + * theory this could mean the parent provider goes inactive, whilst + * still activated in the child for a short period. That's ok. */ - if (!CRYPTO_atomic_add(&prov->activatecnt, 1, &ref, - prov->activatecnt_lock)) { - CRYPTO_DOWN_REF(&prov->refcnt, &ref); + if (provider_activate(prov, 0, 0) < 0) { + CRYPTO_DOWN_REF(&prov->refcnt, &ref, prov->refcnt_lock); CRYPTO_THREAD_unlock(prov->flag_lock); goto err_unlock; } @@ -1483,10 +1352,8 @@ int ossl_provider_doall_activated(OSSL_LIB_CTX *ctx, for (curr = 0; curr < max; curr++) { OSSL_PROVIDER *prov = sk_OSSL_PROVIDER_value(provs, curr); - if (!cb(prov, cbdata)) { - curr = -1; + if (!cb(prov, cbdata)) goto finish; - } } curr = -1; @@ -1504,32 +1371,13 @@ int ossl_provider_doall_activated(OSSL_LIB_CTX *ctx, for (curr++; curr < max; curr++) { OSSL_PROVIDER *prov = sk_OSSL_PROVIDER_value(provs, curr); - if (!CRYPTO_atomic_add(&prov->activatecnt, -1, &ref, - prov->activatecnt_lock)) { - ret = 0; - continue; - } - if (ref < 1) { - /* - * Looks like we need to deactivate properly. We could just have - * done this originally, but it involves taking a write lock so - * we avoid it. We up the count again and do a full deactivation - */ - if (CRYPTO_atomic_add(&prov->activatecnt, 1, &ref, - prov->activatecnt_lock)) - provider_deactivate(prov, 0, 1); - else - ret = 0; - } + provider_deactivate(prov, 0, 1); /* * As above where we did the up-ref, we don't call ossl_provider_free * to avoid making upcalls. There should always be at least one ref * to the provider in the store, so this should never drop to 0. */ - if (!CRYPTO_DOWN_REF(&prov->refcnt, &ref)) { - ret = 0; - continue; - } + CRYPTO_DOWN_REF(&prov->refcnt, &ref, prov->refcnt_lock); /* * Not much we can do if this assert ever fails. So we don't use * ossl_assert here. @@ -1560,6 +1408,16 @@ int OSSL_PROVIDER_available(OSSL_LIB_CTX *libctx, const char *name) return available; } +/* Setters of Provider Object data */ +int ossl_provider_set_fallback(OSSL_PROVIDER *prov) +{ + if (prov == NULL) + return 0; + + prov->flag_fallback = 1; + return 1; +} + /* Getters of Provider Object data */ const char *ossl_provider_name(const OSSL_PROVIDER *prov) { @@ -1642,7 +1500,7 @@ int ossl_provider_self_test(const OSSL_PROVIDER *prov) return 1; ret = prov->self_test(prov->provctx); if (ret == 0) - (void)provider_remove_store_methods((OSSL_PROVIDER *)prov); + (void)provider_flush_store_cache(prov); return ret; } @@ -1680,6 +1538,33 @@ void ossl_provider_unquery_operation(const OSSL_PROVIDER *prov, prov->unquery_operation(prov->provctx, operation_id, algs); } +int ossl_provider_clear_all_operation_bits(OSSL_LIB_CTX *libctx) +{ + struct provider_store_st *store; + OSSL_PROVIDER *provider; + int i, num, res = 1; + + if ((store = get_provider_store(libctx)) != NULL) { + if (!CRYPTO_THREAD_read_lock(store->lock)) + return 0; + num = sk_OSSL_PROVIDER_num(store->providers); + for (i = 0; i < num; i++) { + provider = sk_OSSL_PROVIDER_value(store->providers, i); + if (!CRYPTO_THREAD_write_lock(provider->opbits_lock)) { + res = 0; + continue; + } + if (provider->operation_bits != NULL) + memset(provider->operation_bits, 0, + provider->operation_bits_sz); + CRYPTO_THREAD_unlock(provider->opbits_lock); + } + CRYPTO_THREAD_unlock(store->lock); + return res; + } + return 0; +} + int ossl_provider_set_operation_bit(OSSL_PROVIDER *provider, size_t bitnum) { size_t byte = bitnum / 8; @@ -1693,6 +1578,7 @@ int ossl_provider_set_operation_bit(OSSL_PROVIDER *provider, size_t bitnum) if (tmp == NULL) { CRYPTO_THREAD_unlock(provider->opbits_lock); + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; } provider->operation_bits = tmp; @@ -1915,8 +1801,8 @@ static const OSSL_PARAM param_types[] = { */ static OSSL_FUNC_core_gettable_params_fn core_gettable_params; static OSSL_FUNC_core_get_params_fn core_get_params; -static OSSL_FUNC_core_get_libctx_fn core_get_libctx; static OSSL_FUNC_core_thread_start_fn core_thread_start; +static OSSL_FUNC_core_get_libctx_fn core_get_libctx; #ifndef FIPS_MODULE static OSSL_FUNC_core_new_error_fn core_new_error; static OSSL_FUNC_core_set_error_debug_fn core_set_error_debug; @@ -1924,46 +1810,6 @@ static OSSL_FUNC_core_vset_error_fn core_vset_error; static OSSL_FUNC_core_set_error_mark_fn core_set_error_mark; static OSSL_FUNC_core_clear_last_error_mark_fn core_clear_last_error_mark; static OSSL_FUNC_core_pop_error_to_mark_fn core_pop_error_to_mark; -OSSL_FUNC_BIO_new_file_fn ossl_core_bio_new_file; -OSSL_FUNC_BIO_new_membuf_fn ossl_core_bio_new_mem_buf; -OSSL_FUNC_BIO_read_ex_fn ossl_core_bio_read_ex; -OSSL_FUNC_BIO_write_ex_fn ossl_core_bio_write_ex; -OSSL_FUNC_BIO_gets_fn ossl_core_bio_gets; -OSSL_FUNC_BIO_puts_fn ossl_core_bio_puts; -OSSL_FUNC_BIO_up_ref_fn ossl_core_bio_up_ref; -OSSL_FUNC_BIO_free_fn ossl_core_bio_free; -OSSL_FUNC_BIO_vprintf_fn ossl_core_bio_vprintf; -OSSL_FUNC_BIO_vsnprintf_fn BIO_vsnprintf; -static OSSL_FUNC_self_test_cb_fn core_self_test_get_callback; -static OSSL_FUNC_get_entropy_fn rand_get_entropy; -static OSSL_FUNC_get_user_entropy_fn rand_get_user_entropy; -static OSSL_FUNC_cleanup_entropy_fn rand_cleanup_entropy; -static OSSL_FUNC_cleanup_user_entropy_fn rand_cleanup_user_entropy; -static OSSL_FUNC_get_nonce_fn rand_get_nonce; -static OSSL_FUNC_get_user_nonce_fn rand_get_user_nonce; -static OSSL_FUNC_cleanup_nonce_fn rand_cleanup_nonce; -static OSSL_FUNC_cleanup_user_nonce_fn rand_cleanup_user_nonce; -#endif -OSSL_FUNC_CRYPTO_malloc_fn CRYPTO_malloc; -OSSL_FUNC_CRYPTO_zalloc_fn CRYPTO_zalloc; -OSSL_FUNC_CRYPTO_free_fn CRYPTO_free; -OSSL_FUNC_CRYPTO_clear_free_fn CRYPTO_clear_free; -OSSL_FUNC_CRYPTO_realloc_fn CRYPTO_realloc; -OSSL_FUNC_CRYPTO_clear_realloc_fn CRYPTO_clear_realloc; -OSSL_FUNC_CRYPTO_secure_malloc_fn CRYPTO_secure_malloc; -OSSL_FUNC_CRYPTO_secure_zalloc_fn CRYPTO_secure_zalloc; -OSSL_FUNC_CRYPTO_secure_free_fn CRYPTO_secure_free; -OSSL_FUNC_CRYPTO_secure_clear_free_fn CRYPTO_secure_clear_free; -OSSL_FUNC_CRYPTO_secure_allocated_fn CRYPTO_secure_allocated; -OSSL_FUNC_OPENSSL_cleanse_fn OPENSSL_cleanse; -#ifndef FIPS_MODULE -OSSL_FUNC_provider_register_child_cb_fn ossl_provider_register_child_cb; -OSSL_FUNC_provider_deregister_child_cb_fn ossl_provider_deregister_child_cb; -static OSSL_FUNC_provider_name_fn core_provider_get0_name; -static OSSL_FUNC_provider_get0_provider_ctx_fn core_provider_get0_provider_ctx; -static OSSL_FUNC_provider_get0_dispatch_fn core_provider_get0_dispatch; -static OSSL_FUNC_provider_up_ref_fn core_provider_up_ref_intern; -static OSSL_FUNC_provider_free_fn core_provider_free_intern; static OSSL_FUNC_core_obj_add_sigid_fn core_obj_add_sigid; static OSSL_FUNC_core_obj_create_fn core_obj_create; #endif @@ -2097,102 +1943,6 @@ static int core_pop_error_to_mark(const OSSL_CORE_HANDLE *handle) return ERR_pop_to_mark(); } -static void core_self_test_get_callback(OPENSSL_CORE_CTX *libctx, - OSSL_CALLBACK **cb, void **cbarg) -{ - OSSL_SELF_TEST_get_callback((OSSL_LIB_CTX *)libctx, cb, cbarg); -} - -static size_t rand_get_entropy(const OSSL_CORE_HANDLE *handle, - unsigned char **pout, int entropy, - size_t min_len, size_t max_len) -{ - return ossl_rand_get_entropy((OSSL_LIB_CTX *)core_get_libctx(handle), - pout, entropy, min_len, max_len); -} - -static size_t rand_get_user_entropy(const OSSL_CORE_HANDLE *handle, - unsigned char **pout, int entropy, - size_t min_len, size_t max_len) -{ - return ossl_rand_get_user_entropy((OSSL_LIB_CTX *)core_get_libctx(handle), - pout, entropy, min_len, max_len); -} - -static void rand_cleanup_entropy(const OSSL_CORE_HANDLE *handle, - unsigned char *buf, size_t len) -{ - ossl_rand_cleanup_entropy((OSSL_LIB_CTX *)core_get_libctx(handle), - buf, len); -} - -static void rand_cleanup_user_entropy(const OSSL_CORE_HANDLE *handle, - unsigned char *buf, size_t len) -{ - ossl_rand_cleanup_user_entropy((OSSL_LIB_CTX *)core_get_libctx(handle), - buf, len); -} - -static size_t rand_get_nonce(const OSSL_CORE_HANDLE *handle, - unsigned char **pout, - size_t min_len, size_t max_len, - const void *salt, size_t salt_len) -{ - return ossl_rand_get_nonce((OSSL_LIB_CTX *)core_get_libctx(handle), - pout, min_len, max_len, salt, salt_len); -} - -static size_t rand_get_user_nonce(const OSSL_CORE_HANDLE *handle, - unsigned char **pout, - size_t min_len, size_t max_len, - const void *salt, size_t salt_len) -{ - return ossl_rand_get_user_nonce((OSSL_LIB_CTX *)core_get_libctx(handle), - pout, min_len, max_len, salt, salt_len); -} - -static void rand_cleanup_nonce(const OSSL_CORE_HANDLE *handle, - unsigned char *buf, size_t len) -{ - ossl_rand_cleanup_nonce((OSSL_LIB_CTX *)core_get_libctx(handle), - buf, len); -} - -static void rand_cleanup_user_nonce(const OSSL_CORE_HANDLE *handle, - unsigned char *buf, size_t len) -{ - ossl_rand_cleanup_user_nonce((OSSL_LIB_CTX *)core_get_libctx(handle), - buf, len); -} - -static const char *core_provider_get0_name(const OSSL_CORE_HANDLE *prov) -{ - return OSSL_PROVIDER_get0_name((const OSSL_PROVIDER *)prov); -} - -static void *core_provider_get0_provider_ctx(const OSSL_CORE_HANDLE *prov) -{ - return OSSL_PROVIDER_get0_provider_ctx((const OSSL_PROVIDER *)prov); -} - -static const OSSL_DISPATCH * -core_provider_get0_dispatch(const OSSL_CORE_HANDLE *prov) -{ - return OSSL_PROVIDER_get0_dispatch((const OSSL_PROVIDER *)prov); -} - -static int core_provider_up_ref_intern(const OSSL_CORE_HANDLE *prov, - int activate) -{ - return provider_up_ref_intern((OSSL_PROVIDER *)prov, activate); -} - -static int core_provider_free_intern(const OSSL_CORE_HANDLE *prov, - int deactivate) -{ - return provider_free_intern((OSSL_PROVIDER *)prov, deactivate); -} - static int core_obj_add_sigid(const OSSL_CORE_HANDLE *prov, const char *sign_name, const char *digest_name, const char *pkey_name) @@ -2257,15 +2007,11 @@ static const OSSL_DISPATCH core_dispatch_[] = { { OSSL_FUNC_BIO_FREE, (void (*)(void))ossl_core_bio_free }, { OSSL_FUNC_BIO_VPRINTF, (void (*)(void))ossl_core_bio_vprintf }, { OSSL_FUNC_BIO_VSNPRINTF, (void (*)(void))BIO_vsnprintf }, - { OSSL_FUNC_SELF_TEST_CB, (void (*)(void))core_self_test_get_callback }, - { OSSL_FUNC_GET_ENTROPY, (void (*)(void))rand_get_entropy }, - { OSSL_FUNC_GET_USER_ENTROPY, (void (*)(void))rand_get_user_entropy }, - { OSSL_FUNC_CLEANUP_ENTROPY, (void (*)(void))rand_cleanup_entropy }, - { OSSL_FUNC_CLEANUP_USER_ENTROPY, (void (*)(void))rand_cleanup_user_entropy }, - { OSSL_FUNC_GET_NONCE, (void (*)(void))rand_get_nonce }, - { OSSL_FUNC_GET_USER_NONCE, (void (*)(void))rand_get_user_nonce }, - { OSSL_FUNC_CLEANUP_NONCE, (void (*)(void))rand_cleanup_nonce }, - { OSSL_FUNC_CLEANUP_USER_NONCE, (void (*)(void))rand_cleanup_user_nonce }, + { OSSL_FUNC_SELF_TEST_CB, (void (*)(void))OSSL_SELF_TEST_get_callback }, + { OSSL_FUNC_GET_ENTROPY, (void (*)(void))ossl_rand_get_entropy }, + { OSSL_FUNC_CLEANUP_ENTROPY, (void (*)(void))ossl_rand_cleanup_entropy }, + { OSSL_FUNC_GET_NONCE, (void (*)(void))ossl_rand_get_nonce }, + { OSSL_FUNC_CLEANUP_NONCE, (void (*)(void))ossl_rand_cleanup_nonce }, #endif { OSSL_FUNC_CRYPTO_MALLOC, (void (*)(void))CRYPTO_malloc }, { OSSL_FUNC_CRYPTO_ZALLOC, (void (*)(void))CRYPTO_zalloc }, @@ -2287,18 +2033,18 @@ static const OSSL_DISPATCH core_dispatch_[] = { { OSSL_FUNC_PROVIDER_DEREGISTER_CHILD_CB, (void (*)(void))ossl_provider_deregister_child_cb }, { OSSL_FUNC_PROVIDER_NAME, - (void (*)(void))core_provider_get0_name }, + (void (*)(void))OSSL_PROVIDER_get0_name }, { OSSL_FUNC_PROVIDER_GET0_PROVIDER_CTX, - (void (*)(void))core_provider_get0_provider_ctx }, + (void (*)(void))OSSL_PROVIDER_get0_provider_ctx }, { OSSL_FUNC_PROVIDER_GET0_DISPATCH, - (void (*)(void))core_provider_get0_dispatch }, + (void (*)(void))OSSL_PROVIDER_get0_dispatch }, { OSSL_FUNC_PROVIDER_UP_REF, - (void (*)(void))core_provider_up_ref_intern }, + (void (*)(void))provider_up_ref_intern }, { OSSL_FUNC_PROVIDER_FREE, - (void (*)(void))core_provider_free_intern }, + (void (*)(void))provider_free_intern }, { OSSL_FUNC_CORE_OBJ_ADD_SIGID, (void (*)(void))core_obj_add_sigid }, { OSSL_FUNC_CORE_OBJ_CREATE, (void (*)(void))core_obj_create }, #endif - OSSL_DISPATCH_END + { 0, NULL } }; static const OSSL_DISPATCH *core_dispatch = core_dispatch_; diff --git a/openssl/src/crypto/punycode.c b/openssl/src/crypto/punycode.c index 68fc586e6..b9b4e3d78 100644 --- a/openssl/src/crypto/punycode.c +++ b/openssl/src/crypto/punycode.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,11 +8,10 @@ */ #include +#include #include #include #include "crypto/punycode.h" -#include "internal/common.h" /* for HAS_PREFIX */ -#include "internal/packet.h" /* for WPACKET */ static const unsigned int base = 36; static const unsigned int tmin = 1; @@ -240,12 +239,12 @@ static int codepoint2utf8(unsigned char *out, unsigned long utf) /*- * Return values: - * 1 - ok - * 0 - ok but buf was too short - * -1 - bad string passed or other error + * 1 - ok, *outlen contains valid buf length + * 0 - ok but buf was too short, *outlen contains valid buf length + * -1 - bad string passed */ -int ossl_a2ulabel(const char *in, char *out, size_t outlen) +int ossl_a2ulabel(const char *in, char *out, size_t *outlen) { /*- * Domain name has some parts consisting of ASCII chars joined with dot. @@ -253,60 +252,81 @@ int ossl_a2ulabel(const char *in, char *out, size_t outlen) * If it does not start with xn--, it becomes U-label as is. * Otherwise we try to decode it. */ + char *outptr = out; const char *inptr = in; + size_t size = 0, maxsize; int result = 1; - unsigned int i; + unsigned int i, j; unsigned int buf[LABEL_BUF_SIZE]; /* It's a hostname */ - WPACKET pkt; - /* Internal API, so should not fail */ - if (!ossl_assert(out != NULL)) - return -1; + if (out == NULL) { + result = 0; + maxsize = 0; + } else { + maxsize = *outlen; + } - if (!WPACKET_init_static_len(&pkt, (unsigned char *)out, outlen, 0)) - return -1; +#define PUSHC(c) \ + do \ + if (size++ < maxsize) \ + *outptr++ = c; \ + else \ + result = 0; \ + while (0) while (1) { char *tmpptr = strchr(inptr, '.'); size_t delta = tmpptr != NULL ? (size_t)(tmpptr - inptr) : strlen(inptr); - if (!HAS_PREFIX(inptr, "xn--")) { - if (!WPACKET_memcpy(&pkt, inptr, delta)) - result = 0; + if (strncmp(inptr, "xn--", 4) != 0) { + for (i = 0; i < delta + 1; i++) + PUSHC(inptr[i]); } else { unsigned int bufsize = LABEL_BUF_SIZE; - if (ossl_punycode_decode(inptr + 4, delta - 4, buf, &bufsize) <= 0) { - result = -1; - goto end; - } + if (ossl_punycode_decode(inptr + 4, delta - 4, buf, &bufsize) <= 0) + return -1; for (i = 0; i < bufsize; i++) { unsigned char seed[6]; size_t utfsize = codepoint2utf8(seed, buf[i]); - if (utfsize == 0) { - result = -1; - goto end; - } + if (utfsize == 0) + return -1; - if (!WPACKET_memcpy(&pkt, seed, utfsize)) - result = 0; + for (j = 0; j < utfsize; j++) + PUSHC(seed[j]); } + + PUSHC(tmpptr != NULL ? '.' : '\0'); } if (tmpptr == NULL) break; - if (!WPACKET_put_bytes_u8(&pkt, '.')) - result = 0; - inptr = tmpptr + 1; } +#undef PUSHC - if (!WPACKET_put_bytes_u8(&pkt, '\0')) - result = 0; - end: - WPACKET_cleanup(&pkt); + *outlen = size; return result; } + +/*- + * a MUST be A-label + * u MUST be U-label + * Returns 0 if compared values are equal + * 1 if not + * -1 in case of errors + */ + +int ossl_a2ucompare(const char *a, const char *u) +{ + char a_ulabel[LABEL_BUF_SIZE + 1]; + size_t a_size = sizeof(a_ulabel); + + if (ossl_a2ulabel(a, a_ulabel, &a_size) <= 0) + return -1; + + return strcmp(a_ulabel, u) != 0; +} diff --git a/openssl/src/crypto/quic_vlint.c b/openssl/src/crypto/quic_vlint.c deleted file mode 100644 index 023898596..000000000 --- a/openssl/src/crypto/quic_vlint.c +++ /dev/null @@ -1,81 +0,0 @@ -#include "internal/quic_vlint.h" -#include "internal/e_os.h" - -#ifndef OPENSSL_NO_QUIC - -void ossl_quic_vlint_encode_n(uint8_t *buf, uint64_t v, int n) -{ - if (n == 1) { - buf[0] = (uint8_t)v; - } else if (n == 2) { - buf[0] = (uint8_t)(0x40 | ((v >> 8) & 0x3F)); - buf[1] = (uint8_t)v; - } else if (n == 4) { - buf[0] = (uint8_t)(0x80 | ((v >> 24) & 0x3F)); - buf[1] = (uint8_t)(v >> 16); - buf[2] = (uint8_t)(v >> 8); - buf[3] = (uint8_t)v; - } else { - buf[0] = (uint8_t)(0xC0 | ((v >> 56) & 0x3F)); - buf[1] = (uint8_t)(v >> 48); - buf[2] = (uint8_t)(v >> 40); - buf[3] = (uint8_t)(v >> 32); - buf[4] = (uint8_t)(v >> 24); - buf[5] = (uint8_t)(v >> 16); - buf[6] = (uint8_t)(v >> 8); - buf[7] = (uint8_t)v; - } -} - -void ossl_quic_vlint_encode(uint8_t *buf, uint64_t v) -{ - ossl_quic_vlint_encode_n(buf, v, ossl_quic_vlint_encode_len(v)); -} - -uint64_t ossl_quic_vlint_decode_unchecked(const unsigned char *buf) -{ - uint8_t first_byte = buf[0]; - size_t sz = ossl_quic_vlint_decode_len(first_byte); - - if (sz == 1) - return first_byte & 0x3F; - - if (sz == 2) - return ((uint64_t)(first_byte & 0x3F) << 8) - | buf[1]; - - if (sz == 4) - return ((uint64_t)(first_byte & 0x3F) << 24) - | ((uint64_t)buf[1] << 16) - | ((uint64_t)buf[2] << 8) - | buf[3]; - - return ((uint64_t)(first_byte & 0x3F) << 56) - | ((uint64_t)buf[1] << 48) - | ((uint64_t)buf[2] << 40) - | ((uint64_t)buf[3] << 32) - | ((uint64_t)buf[4] << 24) - | ((uint64_t)buf[5] << 16) - | ((uint64_t)buf[6] << 8) - | buf[7]; -} - -int ossl_quic_vlint_decode(const unsigned char *buf, size_t buf_len, uint64_t *v) -{ - size_t dec_len; - uint64_t x; - - if (buf_len < 1) - return 0; - - dec_len = ossl_quic_vlint_decode_len(buf[0]); - if (buf_len < dec_len) - return 0; - - x = ossl_quic_vlint_decode_unchecked(buf); - - *v = x; - return dec_len; -} - -#endif diff --git a/openssl/src/crypto/rand/local.h b/openssl/src/crypto/rand/local.h new file mode 100644 index 000000000..0e9cd60bf --- /dev/null +++ b/openssl/src/crypto/rand/local.h @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +size_t OPENSSL_ia32_rdseed_bytes(unsigned char *buf, size_t len); +size_t OPENSSL_ia32_rdrand_bytes(unsigned char *buf, size_t len); + +extern unsigned int OPENSSL_ia32cap_P[]; diff --git a/openssl/src/crypto/rand/prov_seed.c b/openssl/src/crypto/rand/prov_seed.c index 2985c7f2d..afa85ab76 100644 --- a/openssl/src/crypto/rand/prov_seed.c +++ b/openssl/src/crypto/rand/prov_seed.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,15 +7,12 @@ * https://www.openssl.org/source/license.html */ -#include "rand_local.h" -#include "crypto/evp.h" #include "crypto/rand.h" #include "crypto/rand_pool.h" -#include "internal/core.h" #include #include -size_t ossl_rand_get_entropy(ossl_unused OSSL_LIB_CTX *ctx, +size_t ossl_rand_get_entropy(ossl_unused OSSL_CORE_HANDLE *handle, unsigned char **pout, int entropy, size_t min_len, size_t max_len) { @@ -25,7 +22,7 @@ size_t ossl_rand_get_entropy(ossl_unused OSSL_LIB_CTX *ctx, pool = ossl_rand_pool_new(entropy, 1, min_len, max_len); if (pool == NULL) { - ERR_raise(ERR_LIB_RAND, ERR_R_RAND_LIB); + ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE); return 0; } @@ -41,39 +38,14 @@ size_t ossl_rand_get_entropy(ossl_unused OSSL_LIB_CTX *ctx, return ret; } -size_t ossl_rand_get_user_entropy(OSSL_LIB_CTX *ctx, - unsigned char **pout, int entropy, - size_t min_len, size_t max_len) -{ - EVP_RAND_CTX *rng = ossl_rand_get0_seed_noncreating(ctx); - - if (rng != NULL && evp_rand_can_seed(rng)) - return evp_rand_get_seed(rng, pout, entropy, min_len, max_len, - 0, NULL, 0); - else - return ossl_rand_get_entropy(ctx, pout, entropy, min_len, max_len); -} - -void ossl_rand_cleanup_entropy(ossl_unused OSSL_LIB_CTX *ctx, +void ossl_rand_cleanup_entropy(ossl_unused OSSL_CORE_HANDLE *handle, unsigned char *buf, size_t len) { OPENSSL_secure_clear_free(buf, len); } -void ossl_rand_cleanup_user_entropy(OSSL_LIB_CTX *ctx, - unsigned char *buf, size_t len) -{ - EVP_RAND_CTX *rng = ossl_rand_get0_seed_noncreating(ctx); - - if (rng != NULL && evp_rand_can_seed(rng)) - evp_rand_clear_seed(rng, buf, len); - else - OPENSSL_secure_clear_free(buf, len); -} - -size_t ossl_rand_get_nonce(ossl_unused OSSL_LIB_CTX *ctx, - unsigned char **pout, - size_t min_len, ossl_unused size_t max_len, +size_t ossl_rand_get_nonce(ossl_unused OSSL_CORE_HANDLE *handle, + unsigned char **pout, size_t min_len, size_t max_len, const void *salt, size_t salt_len) { size_t ret = 0; @@ -81,7 +53,7 @@ size_t ossl_rand_get_nonce(ossl_unused OSSL_LIB_CTX *ctx, pool = ossl_rand_pool_new(0, 0, min_len, max_len); if (pool == NULL) { - ERR_raise(ERR_LIB_RAND, ERR_R_RAND_LIB); + ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE); return 0; } @@ -97,36 +69,8 @@ size_t ossl_rand_get_nonce(ossl_unused OSSL_LIB_CTX *ctx, return ret; } -size_t ossl_rand_get_user_nonce(OSSL_LIB_CTX *ctx, - unsigned char **pout, - size_t min_len, size_t max_len, - const void *salt, size_t salt_len) -{ - unsigned char *buf; - EVP_RAND_CTX *rng = ossl_rand_get0_seed_noncreating(ctx); - - if (rng == NULL) - return ossl_rand_get_nonce(ctx, pout, min_len, max_len, salt, salt_len); - - if ((buf = OPENSSL_malloc(min_len)) == NULL) - return 0; - - if (!EVP_RAND_generate(rng, buf, min_len, 0, 0, salt, salt_len)) { - OPENSSL_free(buf); - return 0; - } - *pout = buf; - return min_len; -} - -void ossl_rand_cleanup_nonce(ossl_unused OSSL_LIB_CTX *ctx, +void ossl_rand_cleanup_nonce(ossl_unused OSSL_CORE_HANDLE *handle, unsigned char *buf, size_t len) { OPENSSL_clear_free(buf, len); } - -void ossl_rand_cleanup_user_nonce(ossl_unused OSSL_LIB_CTX *ctx, - unsigned char *buf, size_t len) -{ - OPENSSL_clear_free(buf, len); -} diff --git a/openssl/src/crypto/rand/rand_egd.c b/openssl/src/crypto/rand/rand_egd.c index f44b38d1a..27d0f2496 100644 --- a/openssl/src/crypto/rand/rand_egd.c +++ b/openssl/src/crypto/rand/rand_egd.c @@ -1,5 +1,5 @@ /* - * Copyright 2000-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2000-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,7 +17,7 @@ * Query an EGD */ -#if defined(OPENSSL_SYS_WIN32) || defined(OPENSSL_SYS_VMS) || defined(OPENSSL_SYS_MSDOS) || defined(OPENSSL_SYS_VXWORKS) || defined(OPENSSL_SYS_VOS) || defined(OPENSSL_SYS_UEFI) +#if defined(OPENSSL_SYS_WIN32) || defined(OPENSSL_SYS_MSDOS) || defined(OPENSSL_SYS_VXWORKS) || defined(OPENSSL_SYS_VOS) || defined(OPENSSL_SYS_UEFI) int RAND_query_egd_bytes(const char *path, unsigned char *buf, int bytes) { return -1; @@ -40,7 +40,11 @@ int RAND_egd_bytes(const char *path, int bytes) # include # include # ifndef NO_SYS_UN_H -# include +# ifdef OPENSSL_SYS_VXWORKS +# include +# else +# include +# endif # else struct sockaddr_un { short sun_family; /* AF_UNIX */ @@ -54,17 +58,15 @@ struct sockaddr_un { /* * HPNS: * - * This code forces the use of compatibility mode if required on HPE NonStop - * when coreutils PRNGD is used and then restores the previous mode - * after establishing the socket. This is not required on x86 where hardware - * randomization should be used instead of EGD available as of OpenSSL 3.0. - * Use --with-rand-seed=rdcpu when configuring x86 with 3.0 and above. + * Our current MQ 5.3 EGD requies compatability-mode sockets + * This code forces the mode to compatibility if required + * and then restores the mode. * * Needs review: * * The better long-term solution is to either run two EGD's each in one of * the two modes or revise the EGD code to listen on two different sockets - * (each in one of the two modes) or use the hardware randomizer. + * (each in one of the two modes). */ _variable int hpns_socket(int family, @@ -83,14 +85,14 @@ int hpns_socket(int family, socket_transport_name_get(AF_UNIX, current_transport, 20); - if (strcmp(current_transport, transport) == 0) + if (strcmp(current_transport,transport) == 0) return socket(family, type, protocol); /* set the requested socket transport */ if (socket_transport_name_set(AF_UNIX, transport)) return -1; - socket_rc = socket(family, type, protocol); + socket_rc = socket(family,type,protocol); /* set mode back to what it was */ if (socket_transport_name_set(AF_UNIX, current_transport)) @@ -133,7 +135,7 @@ int RAND_query_egd_bytes(const char *path, unsigned char *buf, int bytes) setbuf(fp, NULL); /* Try to connect */ - for (;;) { + for ( ; ; ) { if (connect(fd, (struct sockaddr *)&addr, i) == 0) break; # ifdef EISCONN diff --git a/openssl/src/crypto/rand/rand_err.c b/openssl/src/crypto/rand/rand_err.c index 41a4c9cde..b9c2bf176 100644 --- a/openssl/src/crypto/rand/rand_err.c +++ b/openssl/src/crypto/rand/rand_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -51,8 +51,6 @@ static const ERR_STRING_DATA RAND_str_reasons[] = { {ERR_PACK(ERR_LIB_RAND, 0, RAND_R_INSUFFICIENT_DRBG_STRENGTH), "insufficient drbg strength"}, {ERR_PACK(ERR_LIB_RAND, 0, RAND_R_INTERNAL_ERROR), "internal error"}, - {ERR_PACK(ERR_LIB_RAND, 0, RAND_R_INVALID_PROPERTY_QUERY), - "invalid property query"}, {ERR_PACK(ERR_LIB_RAND, 0, RAND_R_IN_ERROR_STATE), "in error state"}, {ERR_PACK(ERR_LIB_RAND, 0, RAND_R_NOT_A_REGULAR_FILE), "Not a regular file"}, diff --git a/openssl/src/crypto/rand/rand_lib.c b/openssl/src/crypto/rand/rand_lib.c index 14999540a..a00514e52 100644 --- a/openssl/src/crypto/rand/rand_lib.c +++ b/openssl/src/crypto/rand/rand_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,7 +18,6 @@ #include "crypto/rand.h" #include "crypto/cryptlib.h" #include "rand_local.h" -#include "crypto/context.h" #ifndef FIPS_MODULE # include @@ -30,7 +29,6 @@ # include "crypto/rand_pool.h" # include "prov/seeding.h" # include "internal/e_os.h" -# include "internal/property.h" # ifndef OPENSSL_NO_ENGINE /* non-NULL if default_RAND_meth is ENGINE-provided */ @@ -98,7 +96,6 @@ void ossl_rand_cleanup_int(void) CRYPTO_THREAD_lock_free(rand_meth_lock); rand_meth_lock = NULL; # endif - ossl_release_default_drbg_ctx(); rand_inited = 0; } @@ -122,8 +119,6 @@ void RAND_keep_random_devices_open(int keep) */ int RAND_poll(void) { - static const char salt[] = "polling"; - # ifndef OPENSSL_NO_DEPRECATED_3_0 const RAND_METHOD *meth = RAND_get_rand_method(); int ret = meth == RAND_OpenSSL(); @@ -152,12 +147,14 @@ int RAND_poll(void) ret = 1; err: ossl_rand_pool_free(pool); - return ret; } -# endif + return ret; +# else + static const char salt[] = "polling"; RAND_seed(salt, sizeof(salt)); return 1; +# endif } # ifndef OPENSSL_NO_DEPRECATED_3_0 @@ -190,13 +187,6 @@ const RAND_METHOD *RAND_get_rand_method(void) if (!RUN_ONCE(&rand_init, do_rand_init)) return NULL; - if (!CRYPTO_THREAD_read_lock(rand_meth_lock)) - return NULL; - tmp_meth = default_RAND_meth; - CRYPTO_THREAD_unlock(rand_meth_lock); - if (tmp_meth != NULL) - return tmp_meth; - if (!CRYPTO_THREAD_write_lock(rand_meth_lock)) return NULL; if (default_RAND_meth == NULL) { @@ -281,13 +271,7 @@ void RAND_add(const void *buf, int num, double randomness) # endif drbg = RAND_get0_primary(NULL); if (drbg != NULL && num > 0) -# ifdef OPENSSL_RAND_SEED_NONE - /* Without an entropy source, we have to rely on the user */ - EVP_RAND_reseed(drbg, 0, buf, num, NULL, 0); -# else - /* With an entropy source, we downgrade this to additional input */ EVP_RAND_reseed(drbg, 0, NULL, 0, buf, num); -# endif } # if !defined(OPENSSL_NO_DEPRECATED_1_1_0) @@ -450,7 +434,7 @@ typedef struct rand_global_st { * Initialize the OSSL_LIB_CTX global DRBGs on first use. * Returns the allocated global data on success or NULL on failure. */ -void *ossl_rand_ctx_new(OSSL_LIB_CTX *libctx) +static void *rand_ossl_ctx_new(OSSL_LIB_CTX *libctx) { RAND_GLOBAL *dgbl = OPENSSL_zalloc(sizeof(*dgbl)); @@ -485,7 +469,7 @@ void *ossl_rand_ctx_new(OSSL_LIB_CTX *libctx) return NULL; } -void ossl_rand_ctx_free(void *vdgbl) +static void rand_ossl_ctx_free(void *vdgbl) { RAND_GLOBAL *dgbl = vdgbl; @@ -507,9 +491,16 @@ void ossl_rand_ctx_free(void *vdgbl) OPENSSL_free(dgbl); } +static const OSSL_LIB_CTX_METHOD rand_drbg_ossl_ctx_method = { + OSSL_LIB_CTX_METHOD_PRIORITY_2, + rand_ossl_ctx_new, + rand_ossl_ctx_free, +}; + static RAND_GLOBAL *rand_get_global(OSSL_LIB_CTX *libctx) { - return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DRBG_INDEX); + return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DRBG_INDEX, + &rand_drbg_ossl_ctx_method); } static void rand_delete_thread_state(void *arg) @@ -535,116 +526,40 @@ static EVP_RAND_CTX *rand_new_seed(OSSL_LIB_CTX *libctx) { EVP_RAND *rand; RAND_GLOBAL *dgbl = rand_get_global(libctx); - EVP_RAND_CTX *ctx = NULL; - const char *propq; - char *name, *props = NULL; - size_t props_len; - OSSL_PROPERTY_LIST *pl1, *pl2, *pl3 = NULL; + EVP_RAND_CTX *ctx; + char *name; if (dgbl == NULL) return NULL; - propq = dgbl->seed_propq; - if (dgbl->seed_name != NULL) { - name = dgbl->seed_name; - } else { - /* - * Default to our internal seed source. This isn't part of the FIPS - * provider so we need to override any FIPS properties. - */ - if (propq == NULL || *propq == '\0') { - propq = "-fips"; - } else { - pl1 = ossl_parse_query(libctx, propq, 1); - if (pl1 == NULL) { - ERR_raise(ERR_LIB_RAND, RAND_R_INVALID_PROPERTY_QUERY); - return NULL; - } - pl2 = ossl_parse_query(libctx, "-fips", 1); - if (pl2 == NULL) { - ossl_property_free(pl1); - ERR_raise(ERR_LIB_RAND, ERR_R_INTERNAL_ERROR); - return NULL; - } - pl3 = ossl_property_merge(pl2, pl1); - ossl_property_free(pl1); - ossl_property_free(pl2); - if (pl3 == NULL) { - ERR_raise(ERR_LIB_RAND, ERR_R_INTERNAL_ERROR); - return NULL; - } - props_len = ossl_property_list_to_string(libctx, pl3, NULL, 0); - if (props_len == 0) { - /* Shouldn't happen since we added a query element */ - ERR_raise(ERR_LIB_RAND, ERR_R_INTERNAL_ERROR); - goto err; - } else { - props = OPENSSL_malloc(props_len); - if (props == NULL) { - ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE); - goto err; - } - if (ossl_property_list_to_string(libctx, pl3, - props, props_len) == 0) { - ERR_raise(ERR_LIB_RAND, ERR_R_INTERNAL_ERROR); - goto err; - } - ossl_property_free(pl3); - pl3 = NULL; - propq = props; - } - } - name = "SEED-SRC"; - } - - rand = EVP_RAND_fetch(libctx, name, propq); + name = dgbl->seed_name != NULL ? dgbl->seed_name : "SEED-SRC"; + rand = EVP_RAND_fetch(libctx, name, dgbl->seed_propq); if (rand == NULL) { ERR_raise(ERR_LIB_RAND, RAND_R_UNABLE_TO_FETCH_DRBG); - goto err; + return NULL; } ctx = EVP_RAND_CTX_new(rand, NULL); EVP_RAND_free(rand); if (ctx == NULL) { ERR_raise(ERR_LIB_RAND, RAND_R_UNABLE_TO_CREATE_DRBG); - goto err; + return NULL; } if (!EVP_RAND_instantiate(ctx, 0, 0, NULL, 0, NULL)) { ERR_raise(ERR_LIB_RAND, RAND_R_ERROR_INSTANTIATING_DRBG); - goto err; + EVP_RAND_CTX_free(ctx); + return NULL; } - OPENSSL_free(props); return ctx; - err: - EVP_RAND_CTX_free(ctx); - ossl_property_free(pl3); - OPENSSL_free(props); - return NULL; -} - -EVP_RAND_CTX *ossl_rand_get0_seed_noncreating(OSSL_LIB_CTX *ctx) -{ - RAND_GLOBAL *dgbl = rand_get_global(ctx); - EVP_RAND_CTX *ret; - - if (dgbl == NULL) - return NULL; - - if (!CRYPTO_THREAD_read_lock(dgbl->lock)) - return NULL; - ret = dgbl->seed; - CRYPTO_THREAD_unlock(dgbl->lock); - return ret; } #endif static EVP_RAND_CTX *rand_new_drbg(OSSL_LIB_CTX *libctx, EVP_RAND_CTX *parent, unsigned int reseed_interval, - time_t reseed_time_interval, int use_df) + time_t reseed_time_interval) { EVP_RAND *rand; RAND_GLOBAL *dgbl = rand_get_global(libctx); EVP_RAND_CTX *ctx; - OSSL_PARAM params[8], *p = params; - const OSSL_PARAM *settables; + OSSL_PARAM params[7], *p = params; char *name, *cipher; if (dgbl == NULL) @@ -662,23 +577,20 @@ static EVP_RAND_CTX *rand_new_drbg(OSSL_LIB_CTX *libctx, EVP_RAND_CTX *parent, return NULL; } - settables = EVP_RAND_CTX_settable_params(ctx); - if (OSSL_PARAM_locate_const(settables, OSSL_DRBG_PARAM_CIPHER)) { - cipher = dgbl->rng_cipher != NULL ? dgbl->rng_cipher : "AES-256-CTR"; - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_DRBG_PARAM_CIPHER, - cipher, 0); - } - if (dgbl->rng_digest != NULL - && OSSL_PARAM_locate_const(settables, OSSL_DRBG_PARAM_DIGEST)) + /* + * Rather than trying to decode the DRBG settings, just pass them through + * and rely on the other end to ignore those it doesn't care about. + */ + cipher = dgbl->rng_cipher != NULL ? dgbl->rng_cipher : "AES-256-CTR"; + *p++ = OSSL_PARAM_construct_utf8_string(OSSL_DRBG_PARAM_CIPHER, + cipher, 0); + if (dgbl->rng_digest != NULL) *p++ = OSSL_PARAM_construct_utf8_string(OSSL_DRBG_PARAM_DIGEST, dgbl->rng_digest, 0); if (dgbl->rng_propq != NULL) *p++ = OSSL_PARAM_construct_utf8_string(OSSL_DRBG_PARAM_PROPERTIES, dgbl->rng_propq, 0); - if (OSSL_PARAM_locate_const(settables, OSSL_ALG_PARAM_MAC)) - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_ALG_PARAM_MAC, "HMAC", 0); - if (OSSL_PARAM_locate_const(settables, OSSL_DRBG_PARAM_USE_DF)) - *p++ = OSSL_PARAM_construct_int(OSSL_DRBG_PARAM_USE_DF, &use_df); + *p++ = OSSL_PARAM_construct_utf8_string(OSSL_ALG_PARAM_MAC, "HMAC", 0); *p++ = OSSL_PARAM_construct_uint(OSSL_DRBG_PARAM_RESEED_REQUESTS, &reseed_interval); *p++ = OSSL_PARAM_construct_time_t(OSSL_DRBG_PARAM_RESEED_TIME_INTERVAL, @@ -733,7 +645,7 @@ EVP_RAND_CTX *RAND_get0_primary(OSSL_LIB_CTX *ctx) ret = dgbl->primary = rand_new_drbg(ctx, dgbl->seed, PRIMARY_RESEED_INTERVAL, - PRIMARY_RESEED_TIME_INTERVAL, 1); + PRIMARY_RESEED_TIME_INTERVAL); /* * The primary DRBG may be shared between multiple threads so we must * enable locking. @@ -775,7 +687,7 @@ EVP_RAND_CTX *RAND_get0_public(OSSL_LIB_CTX *ctx) && !ossl_init_thread_start(NULL, ctx, rand_delete_thread_state)) return NULL; rand = rand_new_drbg(ctx, primary, SECONDARY_RESEED_INTERVAL, - SECONDARY_RESEED_TIME_INTERVAL, 0); + SECONDARY_RESEED_TIME_INTERVAL); CRYPTO_THREAD_set_local(&dgbl->public, rand); } return rand; @@ -808,52 +720,12 @@ EVP_RAND_CTX *RAND_get0_private(OSSL_LIB_CTX *ctx) && !ossl_init_thread_start(NULL, ctx, rand_delete_thread_state)) return NULL; rand = rand_new_drbg(ctx, primary, SECONDARY_RESEED_INTERVAL, - SECONDARY_RESEED_TIME_INTERVAL, 0); + SECONDARY_RESEED_TIME_INTERVAL); CRYPTO_THREAD_set_local(&dgbl->private, rand); } return rand; } -#ifdef FIPS_MODULE -EVP_RAND_CTX *ossl_rand_get0_private_noncreating(OSSL_LIB_CTX *ctx) -{ - RAND_GLOBAL *dgbl = rand_get_global(ctx); - - if (dgbl == NULL) - return NULL; - - return CRYPTO_THREAD_get_local(&dgbl->private); -} -#endif - -int RAND_set0_public(OSSL_LIB_CTX *ctx, EVP_RAND_CTX *rand) -{ - RAND_GLOBAL *dgbl = rand_get_global(ctx); - EVP_RAND_CTX *old; - int r; - - if (dgbl == NULL) - return 0; - old = CRYPTO_THREAD_get_local(&dgbl->public); - if ((r = CRYPTO_THREAD_set_local(&dgbl->public, rand)) > 0) - EVP_RAND_CTX_free(old); - return r; -} - -int RAND_set0_private(OSSL_LIB_CTX *ctx, EVP_RAND_CTX *rand) -{ - RAND_GLOBAL *dgbl = rand_get_global(ctx); - EVP_RAND_CTX *old; - int r; - - if (dgbl == NULL) - return 0; - old = CRYPTO_THREAD_get_local(&dgbl->private); - if ((r = CRYPTO_THREAD_set_local(&dgbl->private, rand)) > 0) - EVP_RAND_CTX_free(old); - return r; -} - #ifndef FIPS_MODULE static int random_set_string(char **p, const char *s) { @@ -861,8 +733,10 @@ static int random_set_string(char **p, const char *s) if (s != NULL) { d = OPENSSL_strdup(s); - if (d == NULL) + if (d == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } } OPENSSL_free(*p); *p = d; @@ -958,7 +832,7 @@ int RAND_set_seed_source_type(OSSL_LIB_CTX *ctx, const char *seed, if (dgbl == NULL) return 0; - if (dgbl->seed != NULL) { + if (dgbl->primary != NULL) { ERR_raise(ERR_LIB_CRYPTO, RAND_R_ALREADY_INSTANTIATED); return 0; } diff --git a/openssl/src/crypto/rand/rand_pool.c b/openssl/src/crypto/rand/rand_pool.c index 8d77b77fd..55f14be60 100644 --- a/openssl/src/crypto/rand/rand_pool.c +++ b/openssl/src/crypto/rand/rand_pool.c @@ -25,8 +25,10 @@ RAND_POOL *ossl_rand_pool_new(int entropy_requested, int secure, RAND_POOL *pool = OPENSSL_zalloc(sizeof(*pool)); size_t min_alloc_size = RAND_POOL_MIN_ALLOCATION(secure); - if (pool == NULL) + if (pool == NULL) { + ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE); return NULL; + } pool->min_len = min_len; pool->max_len = (max_len > RAND_POOL_MAX_LENGTH) ? @@ -40,8 +42,10 @@ RAND_POOL *ossl_rand_pool_new(int entropy_requested, int secure, else pool->buffer = OPENSSL_zalloc(pool->alloc_len); - if (pool->buffer == NULL) + if (pool->buffer == NULL) { + ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE); goto err; + } pool->entropy_requested = entropy_requested; pool->secure = secure; @@ -63,8 +67,10 @@ RAND_POOL *ossl_rand_pool_attach(const unsigned char *buffer, size_t len, { RAND_POOL *pool = OPENSSL_zalloc(sizeof(*pool)); - if (pool == NULL) + if (pool == NULL) { + ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE); return NULL; + } /* * The const needs to be cast away, but attached buffers will not be @@ -216,8 +222,10 @@ static int rand_pool_grow(RAND_POOL *pool, size_t len) p = OPENSSL_secure_zalloc(newlen); else p = OPENSSL_zalloc(newlen); - if (p == NULL) + if (p == NULL) { + ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(p, pool->buffer, pool->len); if (pool->secure) OPENSSL_secure_clear_free(pool->buffer, pool->alloc_len); @@ -249,11 +257,7 @@ size_t ossl_rand_pool_bytes_needed(RAND_POOL *pool, unsigned int entropy_factor) if (bytes_needed > pool->max_len - pool->len) { /* not enough space left */ - ERR_raise_data(ERR_LIB_RAND, RAND_R_RANDOM_POOL_OVERFLOW, - "entropy_factor=%u, entropy_needed=%zu, bytes_needed=%zu," - "pool->max_len=%zu, pool->len=%zu", - entropy_factor, entropy_needed, bytes_needed, - pool->max_len, pool->len); + ERR_raise(ERR_LIB_RAND, RAND_R_RANDOM_POOL_OVERFLOW); return 0; } diff --git a/openssl/src/crypto/rand/rand_uniform.c b/openssl/src/crypto/rand/rand_uniform.c deleted file mode 100644 index f0b199b95..000000000 --- a/openssl/src/crypto/rand/rand_uniform.c +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "crypto/rand.h" -#include "internal/common.h" - -/* - * Implementation an optimal random integer in a range function. - * - * Essentially it boils down to incrementally generating a fixed point - * number on the interval [0, 1) and multiplying this number by the upper - * range limit. Once it is certain what the fractional part contributes to - * the integral part of the product, the algorithm has produced a definitive - * result. - * - * Refer: https://github.com/apple/swift/pull/39143 for a fuller description - * of the algorithm. - */ -uint32_t ossl_rand_uniform_uint32(OSSL_LIB_CTX *ctx, uint32_t upper, int *err) -{ - uint32_t i, f; /* integer and fractional parts */ - uint32_t f2, rand; /* extra fractional part and random material */ - uint64_t prod; /* temporary holding double width product */ - const int max_followup_iterations = 10; - int j; - - if (!ossl_assert(upper > 0)) { - *err = 0; - return 0; - } - if (ossl_unlikely(upper == 1)) - return 0; - - /* Get 32 bits of entropy */ - if (RAND_bytes_ex(ctx, (unsigned char *)&rand, sizeof(rand), 0) <= 0) { - *err = 1; - return 0; - } - - /* - * We are generating a fixed point number on the interval [0, 1). - * Multiplying this by the range gives us a number on [0, upper). - * The high word of the multiplication result represents the integral - * part we want. The lower word is the fractional part. We can early exit if - * if the fractional part is small enough that no carry from the next lower - * word can cause an overflow and carry into the integer part. This - * happens when the fractional part is bounded by 2^32 - upper which - * can be simplified to just -upper (as an unsigned integer). - */ - prod = (uint64_t)upper * rand; - i = prod >> 32; - f = prod & 0xffffffff; - if (ossl_likely(f <= 1 + ~upper)) /* 1+~upper == -upper but compilers whine */ - return i; - - /* - * We're in the position where the carry from the next word *might* cause - * a carry to the integral part. The process here is to generate the next - * word, multiply it by the range and add that to the current word. If - * it overflows, the carry propagates to the integer part (return i+1). - * If it can no longer overflow regardless of further lower order bits, - * we are done (return i). If there is still a chance of overflow, we - * repeat the process with the next lower word. - * - * Each *bit* of randomness has a probability of one half of terminating - * this process, so each each word beyond the first has a probability - * of 2^-32 of not terminating the process. That is, we're extremely - * likely to stop very rapidly. - */ - for (j = 0; j < max_followup_iterations; j++) { - if (RAND_bytes_ex(ctx, (unsigned char *)&rand, sizeof(rand), 0) <= 0) { - *err = 1; - return 0; - } - prod = (uint64_t)upper * rand; - f2 = prod >> 32; - f += f2; - /* On overflow, add the carry to our result */ - if (f < f2) - return i + 1; - /* For not all 1 bits, there is no carry so return the result */ - if (ossl_likely(f != 0xffffffff)) - return i; - /* setup for the next word of randomness */ - f = prod & 0xffffffff; - } - /* - * If we get here, we've consumed 32 * max_followup_iterations + 32 bits - * with no firm decision, this gives a bias with probability < 2^-(32*n), - * which is likely acceptable. - */ - return i; -} - -uint32_t ossl_rand_range_uint32(OSSL_LIB_CTX *ctx, uint32_t lower, uint32_t upper, - int *err) -{ - if (!ossl_assert(lower < upper)) { - *err = 1; - return 0; - } - return lower + ossl_rand_uniform_uint32(ctx, upper - lower, err); -} diff --git a/openssl/src/crypto/rand/randfile.c b/openssl/src/crypto/rand/randfile.c index 31edd4f56..adbd88f38 100644 --- a/openssl/src/crypto/rand/randfile.c +++ b/openssl/src/crypto/rand/randfile.c @@ -27,9 +27,6 @@ #include #include -#ifdef OPENSSL_SYS_VMS -# include -#endif #include #ifndef OPENSSL_NO_POSIX_IO # include @@ -60,22 +57,6 @@ #define RAND_BUF_SIZE 1024 #define RFILE ".rnd" -#ifdef OPENSSL_SYS_VMS -/* - * __FILE_ptr32 is a type provided by DEC C headers (types.h specifically) - * to make sure the FILE* is a 32-bit pointer no matter what. We know that - * stdio functions return this type (a study of stdio.h proves it). - * - * This declaration is a nasty hack to get around vms' extension to fopen for - * passing in sharing options being disabled by /STANDARD=ANSI89 - */ -static __FILE_ptr32 (*const vms_fopen)(const char *, const char *, ...) = - (__FILE_ptr32 (*)(const char *, const char *, ...))fopen; -# define VMS_OPEN_ATTRS \ - "shr=get,put,upd,del","ctx=bin,stm","rfm=stm","rat=none","mrs=0" -# define openssl_fopen(fname, mode) vms_fopen((fname), (mode), VMS_OPEN_ATTRS) -#endif - /* * Note that these functions are intended for seed files only. Entropy * devices and EGD sockets are handled in rand_unix.c If |bytes| is @@ -122,17 +103,6 @@ int RAND_load_file(const char *file, long bytes) else bytes = RAND_DRBG_STRENGTH; } -#endif - /* - * On VMS, setbuf() will only take 32-bit pointers, and a compilation - * with /POINTER_SIZE=64 will give off a MAYLOSEDATA2 warning here. - * However, we trust that the C RTL will never give us a FILE pointer - * above the first 4 GB of memory, so we simply turn off the warning - * temporarily. - */ -#if defined(OPENSSL_SYS_VMS) && defined(__DECC) -# pragma environment save -# pragma message disable maylosedata2 #endif /* * Don't buffer, because even if |file| is regular file, we have @@ -140,18 +110,15 @@ int RAND_load_file(const char *file, long bytes) * contents lying around? */ setbuf(in, NULL); -#if defined(OPENSSL_SYS_VMS) && defined(__DECC) -# pragma environment restore -#endif - for (;;) { + for ( ; ; ) { if (bytes > 0) n = (bytes <= RAND_LOAD_BUF_SIZE) ? (int)bytes : RAND_BUF_SIZE; else n = RAND_LOAD_BUF_SIZE; i = fread(buf, 1, n, in); #ifdef EINTR - if (ferror(in) && errno == EINTR) { + if (ferror(in) && errno == EINTR){ clearerr(in); if (i == 0) continue; @@ -198,7 +165,7 @@ int RAND_write_file(const char *file) return -1; #if defined(O_CREAT) && !defined(OPENSSL_NO_POSIX_IO) && \ - !defined(OPENSSL_SYS_VMS) && !defined(OPENSSL_SYS_WINDOWS) + !defined(OPENSSL_SYS_WINDOWS) { # ifndef O_BINARY # define O_BINARY 0 @@ -213,27 +180,6 @@ int RAND_write_file(const char *file) } #endif -#ifdef OPENSSL_SYS_VMS - /* - * VMS NOTE: Prior versions of this routine created a _new_ version of - * the rand file for each call into this routine, then deleted all - * existing versions named ;-1, and finally renamed the current version - * as ';1'. Under concurrent usage, this resulted in an RMS race - * condition in rename() which could orphan files (see vms message help - * for RMS$_REENT). With the fopen() calls below, openssl/VMS now shares - * the top-level version of the rand file. Note that there may still be - * conditions where the top-level rand file is locked. If so, this code - * will then create a new version of the rand file. Without the delete - * and rename code, this can result in ascending file versions that stop - * at version 32767, and this routine will then return an error. The - * remedy for this is to recode the calling application to avoid - * concurrent use of the rand file, or synchronize usage at the - * application level. Also consider whether or not you NEED a persistent - * rand file in a concurrent use situation. - */ - out = openssl_fopen(file, "rb+"); -#endif - if (out == NULL) out = openssl_fopen(file, "wb"); if (out == NULL) { @@ -311,9 +257,7 @@ const char *RAND_file_name(char *buf, size_t size) if (len + 1 + strlen(RFILE) + 1 >= size) return NULL; strcpy(buf, s); -#ifndef OPENSSL_SYS_VMS strcat(buf, "/"); -#endif strcat(buf, RFILE); } diff --git a/openssl/src/crypto/rc2/rc2_cbc.c b/openssl/src/crypto/rc2/rc2_cbc.c deleted file mode 100644 index d37093fcc..000000000 --- a/openssl/src/crypto/rc2/rc2_cbc.c +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RC2 low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "rc2_local.h" - -void RC2_cbc_encrypt(const unsigned char *in, unsigned char *out, long length, - RC2_KEY *ks, unsigned char *iv, int encrypt) -{ - register unsigned long tin0, tin1; - register unsigned long tout0, tout1, xor0, xor1; - register long l = length; - unsigned long tin[2]; - - if (encrypt) { - c2l(iv, tout0); - c2l(iv, tout1); - iv -= 8; - for (l -= 8; l >= 0; l -= 8) { - c2l(in, tin0); - c2l(in, tin1); - tin0 ^= tout0; - tin1 ^= tout1; - tin[0] = tin0; - tin[1] = tin1; - RC2_encrypt(tin, ks); - tout0 = tin[0]; - l2c(tout0, out); - tout1 = tin[1]; - l2c(tout1, out); - } - if (l != -8) { - c2ln(in, tin0, tin1, l + 8); - tin0 ^= tout0; - tin1 ^= tout1; - tin[0] = tin0; - tin[1] = tin1; - RC2_encrypt(tin, ks); - tout0 = tin[0]; - l2c(tout0, out); - tout1 = tin[1]; - l2c(tout1, out); - } - l2c(tout0, iv); - l2c(tout1, iv); - } else { - c2l(iv, xor0); - c2l(iv, xor1); - iv -= 8; - for (l -= 8; l >= 0; l -= 8) { - c2l(in, tin0); - tin[0] = tin0; - c2l(in, tin1); - tin[1] = tin1; - RC2_decrypt(tin, ks); - tout0 = tin[0] ^ xor0; - tout1 = tin[1] ^ xor1; - l2c(tout0, out); - l2c(tout1, out); - xor0 = tin0; - xor1 = tin1; - } - if (l != -8) { - c2l(in, tin0); - tin[0] = tin0; - c2l(in, tin1); - tin[1] = tin1; - RC2_decrypt(tin, ks); - tout0 = tin[0] ^ xor0; - tout1 = tin[1] ^ xor1; - l2cn(tout0, tout1, out, l + 8); - xor0 = tin0; - xor1 = tin1; - } - l2c(xor0, iv); - l2c(xor1, iv); - } - tin0 = tin1 = tout0 = tout1 = xor0 = xor1 = 0; - tin[0] = tin[1] = 0; -} - -void RC2_encrypt(unsigned long *d, RC2_KEY *key) -{ - int i, n; - register RC2_INT *p0, *p1; - register RC2_INT x0, x1, x2, x3, t; - unsigned long l; - - l = d[0]; - x0 = (RC2_INT) l & 0xffff; - x1 = (RC2_INT) (l >> 16L); - l = d[1]; - x2 = (RC2_INT) l & 0xffff; - x3 = (RC2_INT) (l >> 16L); - - n = 3; - i = 5; - - p0 = p1 = &(key->data[0]); - for (;;) { - t = (x0 + (x1 & ~x3) + (x2 & x3) + *(p0++)) & 0xffff; - x0 = (t << 1) | (t >> 15); - t = (x1 + (x2 & ~x0) + (x3 & x0) + *(p0++)) & 0xffff; - x1 = (t << 2) | (t >> 14); - t = (x2 + (x3 & ~x1) + (x0 & x1) + *(p0++)) & 0xffff; - x2 = (t << 3) | (t >> 13); - t = (x3 + (x0 & ~x2) + (x1 & x2) + *(p0++)) & 0xffff; - x3 = (t << 5) | (t >> 11); - - if (--i == 0) { - if (--n == 0) - break; - i = (n == 2) ? 6 : 5; - - x0 += p1[x3 & 0x3f]; - x1 += p1[x0 & 0x3f]; - x2 += p1[x1 & 0x3f]; - x3 += p1[x2 & 0x3f]; - } - } - - d[0] = - (unsigned long)(x0 & 0xffff) | ((unsigned long)(x1 & 0xffff) << 16L); - d[1] = - (unsigned long)(x2 & 0xffff) | ((unsigned long)(x3 & 0xffff) << 16L); -} - -void RC2_decrypt(unsigned long *d, RC2_KEY *key) -{ - int i, n; - register RC2_INT *p0, *p1; - register RC2_INT x0, x1, x2, x3, t; - unsigned long l; - - l = d[0]; - x0 = (RC2_INT) l & 0xffff; - x1 = (RC2_INT) (l >> 16L); - l = d[1]; - x2 = (RC2_INT) l & 0xffff; - x3 = (RC2_INT) (l >> 16L); - - n = 3; - i = 5; - - p0 = &(key->data[63]); - p1 = &(key->data[0]); - for (;;) { - t = ((x3 << 11) | (x3 >> 5)) & 0xffff; - x3 = (t - (x0 & ~x2) - (x1 & x2) - *(p0--)) & 0xffff; - t = ((x2 << 13) | (x2 >> 3)) & 0xffff; - x2 = (t - (x3 & ~x1) - (x0 & x1) - *(p0--)) & 0xffff; - t = ((x1 << 14) | (x1 >> 2)) & 0xffff; - x1 = (t - (x2 & ~x0) - (x3 & x0) - *(p0--)) & 0xffff; - t = ((x0 << 15) | (x0 >> 1)) & 0xffff; - x0 = (t - (x1 & ~x3) - (x2 & x3) - *(p0--)) & 0xffff; - - if (--i == 0) { - if (--n == 0) - break; - i = (n == 2) ? 6 : 5; - - x3 = (x3 - p1[x2 & 0x3f]) & 0xffff; - x2 = (x2 - p1[x1 & 0x3f]) & 0xffff; - x1 = (x1 - p1[x0 & 0x3f]) & 0xffff; - x0 = (x0 - p1[x3 & 0x3f]) & 0xffff; - } - } - - d[0] = - (unsigned long)(x0 & 0xffff) | ((unsigned long)(x1 & 0xffff) << 16L); - d[1] = - (unsigned long)(x2 & 0xffff) | ((unsigned long)(x3 & 0xffff) << 16L); -} diff --git a/openssl/src/crypto/rc2/rc2_ecb.c b/openssl/src/crypto/rc2/rc2_ecb.c deleted file mode 100644 index 8861d0567..000000000 --- a/openssl/src/crypto/rc2/rc2_ecb.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RC2 low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "rc2_local.h" -#include - -/*- - * RC2 as implemented frm a posting from - * Newsgroups: sci.crypt - * Subject: Specification for Ron Rivests Cipher No.2 - * Message-ID: <4fk39f$f70@net.auckland.ac.nz> - * Date: 11 Feb 1996 06:45:03 GMT - */ - -void RC2_ecb_encrypt(const unsigned char *in, unsigned char *out, RC2_KEY *ks, - int encrypt) -{ - unsigned long l, d[2]; - - c2l(in, l); - d[0] = l; - c2l(in, l); - d[1] = l; - if (encrypt) - RC2_encrypt(d, ks); - else - RC2_decrypt(d, ks); - l = d[0]; - l2c(l, out); - l = d[1]; - l2c(l, out); - l = d[0] = d[1] = 0; -} diff --git a/openssl/src/crypto/rc2/rc2_local.h b/openssl/src/crypto/rc2/rc2_local.h deleted file mode 100644 index f9ca888a6..000000000 --- a/openssl/src/crypto/rc2/rc2_local.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#undef c2l -#define c2l(c,l) (l =((unsigned long)(*((c)++))) , \ - l|=((unsigned long)(*((c)++)))<< 8L, \ - l|=((unsigned long)(*((c)++)))<<16L, \ - l|=((unsigned long)(*((c)++)))<<24L) - -/* NOTE - c is not incremented as per c2l */ -#undef c2ln -#define c2ln(c,l1,l2,n) { \ - c+=n; \ - l1=l2=0; \ - switch (n) { \ - case 8: l2 =((unsigned long)(*(--(c))))<<24L; \ - /* fall through */ \ - case 7: l2|=((unsigned long)(*(--(c))))<<16L; \ - /* fall through */ \ - case 6: l2|=((unsigned long)(*(--(c))))<< 8L; \ - /* fall through */ \ - case 5: l2|=((unsigned long)(*(--(c)))); \ - /* fall through */ \ - case 4: l1 =((unsigned long)(*(--(c))))<<24L; \ - /* fall through */ \ - case 3: l1|=((unsigned long)(*(--(c))))<<16L; \ - /* fall through */ \ - case 2: l1|=((unsigned long)(*(--(c))))<< 8L; \ - /* fall through */ \ - case 1: l1|=((unsigned long)(*(--(c)))); \ - } \ - } - -#undef l2c -#define l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8L)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16L)&0xff), \ - *((c)++)=(unsigned char)(((l)>>24L)&0xff)) - -/* NOTE - c is not incremented as per l2c */ -#undef l2cn -#define l2cn(l1,l2,c,n) { \ - c+=n; \ - switch (n) { \ - case 8: *(--(c))=(unsigned char)(((l2)>>24L)&0xff); \ - /* fall through */ \ - case 7: *(--(c))=(unsigned char)(((l2)>>16L)&0xff); \ - /* fall through */ \ - case 6: *(--(c))=(unsigned char)(((l2)>> 8L)&0xff); \ - /* fall through */ \ - case 5: *(--(c))=(unsigned char)(((l2) )&0xff); \ - /* fall through */ \ - case 4: *(--(c))=(unsigned char)(((l1)>>24L)&0xff); \ - /* fall through */ \ - case 3: *(--(c))=(unsigned char)(((l1)>>16L)&0xff); \ - /* fall through */ \ - case 2: *(--(c))=(unsigned char)(((l1)>> 8L)&0xff); \ - /* fall through */ \ - case 1: *(--(c))=(unsigned char)(((l1) )&0xff); \ - } \ - } - diff --git a/openssl/src/crypto/rc2/rc2_skey.c b/openssl/src/crypto/rc2/rc2_skey.c deleted file mode 100644 index e43b84af1..000000000 --- a/openssl/src/crypto/rc2/rc2_skey.c +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RC2 low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "rc2_local.h" - -static const unsigned char key_table[256] = { - 0xd9, 0x78, 0xf9, 0xc4, 0x19, 0xdd, 0xb5, 0xed, 0x28, 0xe9, 0xfd, 0x79, - 0x4a, 0xa0, 0xd8, 0x9d, 0xc6, 0x7e, 0x37, 0x83, 0x2b, 0x76, 0x53, 0x8e, - 0x62, 0x4c, 0x64, 0x88, 0x44, 0x8b, 0xfb, 0xa2, 0x17, 0x9a, 0x59, 0xf5, - 0x87, 0xb3, 0x4f, 0x13, 0x61, 0x45, 0x6d, 0x8d, 0x09, 0x81, 0x7d, 0x32, - 0xbd, 0x8f, 0x40, 0xeb, 0x86, 0xb7, 0x7b, 0x0b, 0xf0, 0x95, 0x21, 0x22, - 0x5c, 0x6b, 0x4e, 0x82, 0x54, 0xd6, 0x65, 0x93, 0xce, 0x60, 0xb2, 0x1c, - 0x73, 0x56, 0xc0, 0x14, 0xa7, 0x8c, 0xf1, 0xdc, 0x12, 0x75, 0xca, 0x1f, - 0x3b, 0xbe, 0xe4, 0xd1, 0x42, 0x3d, 0xd4, 0x30, 0xa3, 0x3c, 0xb6, 0x26, - 0x6f, 0xbf, 0x0e, 0xda, 0x46, 0x69, 0x07, 0x57, 0x27, 0xf2, 0x1d, 0x9b, - 0xbc, 0x94, 0x43, 0x03, 0xf8, 0x11, 0xc7, 0xf6, 0x90, 0xef, 0x3e, 0xe7, - 0x06, 0xc3, 0xd5, 0x2f, 0xc8, 0x66, 0x1e, 0xd7, 0x08, 0xe8, 0xea, 0xde, - 0x80, 0x52, 0xee, 0xf7, 0x84, 0xaa, 0x72, 0xac, 0x35, 0x4d, 0x6a, 0x2a, - 0x96, 0x1a, 0xd2, 0x71, 0x5a, 0x15, 0x49, 0x74, 0x4b, 0x9f, 0xd0, 0x5e, - 0x04, 0x18, 0xa4, 0xec, 0xc2, 0xe0, 0x41, 0x6e, 0x0f, 0x51, 0xcb, 0xcc, - 0x24, 0x91, 0xaf, 0x50, 0xa1, 0xf4, 0x70, 0x39, 0x99, 0x7c, 0x3a, 0x85, - 0x23, 0xb8, 0xb4, 0x7a, 0xfc, 0x02, 0x36, 0x5b, 0x25, 0x55, 0x97, 0x31, - 0x2d, 0x5d, 0xfa, 0x98, 0xe3, 0x8a, 0x92, 0xae, 0x05, 0xdf, 0x29, 0x10, - 0x67, 0x6c, 0xba, 0xc9, 0xd3, 0x00, 0xe6, 0xcf, 0xe1, 0x9e, 0xa8, 0x2c, - 0x63, 0x16, 0x01, 0x3f, 0x58, 0xe2, 0x89, 0xa9, 0x0d, 0x38, 0x34, 0x1b, - 0xab, 0x33, 0xff, 0xb0, 0xbb, 0x48, 0x0c, 0x5f, 0xb9, 0xb1, 0xcd, 0x2e, - 0xc5, 0xf3, 0xdb, 0x47, 0xe5, 0xa5, 0x9c, 0x77, 0x0a, 0xa6, 0x20, 0x68, - 0xfe, 0x7f, 0xc1, 0xad, -}; - -#if defined(_MSC_VER) && defined(_ARM_) -# pragma optimize("g",off) -#endif - -/* - * It has come to my attention that there are 2 versions of the RC2 key - * schedule. One which is normal, and another which has a hook to use a - * reduced key length. BSAFE uses the latter version. What I previously - * shipped is the same as specifying 1024 for the 'bits' parameter. Bsafe - * uses a version where the bits parameter is the same as len*8 - */ -void RC2_set_key(RC2_KEY *key, int len, const unsigned char *data, int bits) -{ - int i, j; - unsigned char *k; - RC2_INT *ki; - unsigned int c, d; - - k = (unsigned char *)&(key->data[0]); - *k = 0; /* for if there is a zero length key */ - - if (len > 128) - len = 128; - if (bits <= 0) - bits = 1024; - if (bits > 1024) - bits = 1024; - - for (i = 0; i < len; i++) - k[i] = data[i]; - - /* expand table */ - d = k[len - 1]; - j = 0; - for (i = len; i < 128; i++, j++) { - d = key_table[(k[j] + d) & 0xff]; - k[i] = d; - } - - /* hmm.... key reduction to 'bits' bits */ - - j = (bits + 7) >> 3; - i = 128 - j; - c = (0xff >> (-bits & 0x07)); - - d = key_table[k[i] & c]; - k[i] = d; - while (i--) { - d = key_table[k[i + j] ^ d]; - k[i] = d; - } - - /* copy from bytes into RC2_INT's */ - ki = &(key->data[63]); - for (i = 127; i >= 0; i -= 2) - *(ki--) = ((k[i] << 8) | k[i - 1]) & 0xffff; -} - -#if defined(_MSC_VER) -# pragma optimize("",on) -#endif diff --git a/openssl/src/crypto/rc2/rc2cfb64.c b/openssl/src/crypto/rc2/rc2cfb64.c deleted file mode 100644 index 19612d933..000000000 --- a/openssl/src/crypto/rc2/rc2cfb64.c +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RC2 low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "rc2_local.h" - -/* - * The input and output encrypted as though 64bit cfb mode is being used. - * The extra state information to record how much of the 64bit block we have - * used is contained in *num; - */ - -void RC2_cfb64_encrypt(const unsigned char *in, unsigned char *out, - long length, RC2_KEY *schedule, unsigned char *ivec, - int *num, int encrypt) -{ - register unsigned long v0, v1, t; - register int n = *num; - register long l = length; - unsigned long ti[2]; - unsigned char *iv, c, cc; - - iv = (unsigned char *)ivec; - if (encrypt) { - while (l--) { - if (n == 0) { - c2l(iv, v0); - ti[0] = v0; - c2l(iv, v1); - ti[1] = v1; - RC2_encrypt((unsigned long *)ti, schedule); - iv = (unsigned char *)ivec; - t = ti[0]; - l2c(t, iv); - t = ti[1]; - l2c(t, iv); - iv = (unsigned char *)ivec; - } - c = *(in++) ^ iv[n]; - *(out++) = c; - iv[n] = c; - n = (n + 1) & 0x07; - } - } else { - while (l--) { - if (n == 0) { - c2l(iv, v0); - ti[0] = v0; - c2l(iv, v1); - ti[1] = v1; - RC2_encrypt((unsigned long *)ti, schedule); - iv = (unsigned char *)ivec; - t = ti[0]; - l2c(t, iv); - t = ti[1]; - l2c(t, iv); - iv = (unsigned char *)ivec; - } - cc = *(in++); - c = iv[n]; - iv[n] = cc; - *(out++) = c ^ cc; - n = (n + 1) & 0x07; - } - } - v0 = v1 = ti[0] = ti[1] = t = c = cc = 0; - *num = n; -} diff --git a/openssl/src/crypto/rc2/rc2ofb64.c b/openssl/src/crypto/rc2/rc2ofb64.c deleted file mode 100644 index 82c34b3aa..000000000 --- a/openssl/src/crypto/rc2/rc2ofb64.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RC2 low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "rc2_local.h" - -/* - * The input and output encrypted as though 64bit ofb mode is being used. - * The extra state information to record how much of the 64bit block we have - * used is contained in *num; - */ -void RC2_ofb64_encrypt(const unsigned char *in, unsigned char *out, - long length, RC2_KEY *schedule, unsigned char *ivec, - int *num) -{ - register unsigned long v0, v1, t; - register int n = *num; - register long l = length; - unsigned char d[8]; - register char *dp; - unsigned long ti[2]; - unsigned char *iv; - int save = 0; - - iv = (unsigned char *)ivec; - c2l(iv, v0); - c2l(iv, v1); - ti[0] = v0; - ti[1] = v1; - dp = (char *)d; - l2c(v0, dp); - l2c(v1, dp); - while (l--) { - if (n == 0) { - RC2_encrypt((unsigned long *)ti, schedule); - dp = (char *)d; - t = ti[0]; - l2c(t, dp); - t = ti[1]; - l2c(t, dp); - save++; - } - *(out++) = *(in++) ^ d[n]; - n = (n + 1) & 0x07; - } - if (save) { - v0 = ti[0]; - v1 = ti[1]; - iv = (unsigned char *)ivec; - l2c(v0, iv); - l2c(v1, iv); - } - t = v0 = v1 = ti[0] = ti[1] = 0; - *num = n; -} diff --git a/openssl/src/crypto/rc4/gen/linux_ia32/rc4-586.S b/openssl/src/crypto/rc4/gen/linux_ia32/rc4-586.S index a10b047a7..cf425a53d 100644 --- a/openssl/src/crypto/rc4/gen/linux_ia32/rc4-586.S +++ b/openssl/src/crypto/rc4/gen/linux_ia32/rc4-586.S @@ -4,11 +4,7 @@ .align 16 RC4: .L_RC4_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -275,11 +271,7 @@ RC4: .align 16 RC4_set_key: .L_RC4_set_key_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -358,11 +350,7 @@ RC4_set_key: .align 16 RC4_options: .L_RC4_options_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - call .L018pic_point .L018pic_point: popl %eax diff --git a/openssl/src/crypto/rc4/gen/windows_ia32/rc4-586.asm b/openssl/src/crypto/rc4/gen/windows_ia32/rc4-586.asm index e9b7e1007..c2758bfe8 100644 --- a/openssl/src/crypto/rc4/gen/windows_ia32/rc4-586.asm +++ b/openssl/src/crypto/rc4/gen/windows_ia32/rc4-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/rc5/rc5_local.h b/openssl/src/crypto/rc5/rc5_local.h index 4ba8745ca..df7df608d 100644 --- a/openssl/src/crypto/rc5/rc5_local.h +++ b/openssl/src/crypto/rc5/rc5_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,19 +22,19 @@ l1=l2=0; \ switch (n) { \ case 8: l2 =((unsigned long)(*(--(c))))<<24L; \ - /* fall through */ \ + /* fall thru */ \ case 7: l2|=((unsigned long)(*(--(c))))<<16L; \ - /* fall through */ \ + /* fall thru */ \ case 6: l2|=((unsigned long)(*(--(c))))<< 8L; \ - /* fall through */ \ + /* fall thru */ \ case 5: l2|=((unsigned long)(*(--(c)))); \ - /* fall through */ \ + /* fall thru */ \ case 4: l1 =((unsigned long)(*(--(c))))<<24L; \ - /* fall through */ \ + /* fall thru */ \ case 3: l1|=((unsigned long)(*(--(c))))<<16L; \ - /* fall through */ \ + /* fall thru */ \ case 2: l1|=((unsigned long)(*(--(c))))<< 8L; \ - /* fall through */ \ + /* fall thru */ \ case 1: l1|=((unsigned long)(*(--(c)))); \ } \ } @@ -51,23 +51,80 @@ c+=n; \ switch (n) { \ case 8: *(--(c))=(unsigned char)(((l2)>>24L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 7: *(--(c))=(unsigned char)(((l2)>>16L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 6: *(--(c))=(unsigned char)(((l2)>> 8L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 5: *(--(c))=(unsigned char)(((l2) )&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 4: *(--(c))=(unsigned char)(((l1)>>24L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 3: *(--(c))=(unsigned char)(((l1)>>16L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 2: *(--(c))=(unsigned char)(((l1)>> 8L)&0xff); \ - /* fall through */ \ + /* fall thru */ \ case 1: *(--(c))=(unsigned char)(((l1) )&0xff); \ } \ } +/* NOTE - c is not incremented as per n2l */ +#define n2ln(c,l1,l2,n) { \ + c+=n; \ + l1=l2=0; \ + switch (n) { \ + case 8: l2 =((unsigned long)(*(--(c)))) ; \ + /* fall thru */ \ + case 7: l2|=((unsigned long)(*(--(c))))<< 8; \ + /* fall thru */ \ + case 6: l2|=((unsigned long)(*(--(c))))<<16; \ + /* fall thru */ \ + case 5: l2|=((unsigned long)(*(--(c))))<<24; \ + /* fall thru */ \ + case 4: l1 =((unsigned long)(*(--(c)))) ; \ + /* fall thru */ \ + case 3: l1|=((unsigned long)(*(--(c))))<< 8; \ + /* fall thru */ \ + case 2: l1|=((unsigned long)(*(--(c))))<<16; \ + /* fall thru */ \ + case 1: l1|=((unsigned long)(*(--(c))))<<24; \ + } \ + } + +/* NOTE - c is not incremented as per l2n */ +#define l2nn(l1,l2,c,n) { \ + c+=n; \ + switch (n) { \ + case 8: *(--(c))=(unsigned char)(((l2) )&0xff); \ + /* fall thru */ \ + case 7: *(--(c))=(unsigned char)(((l2)>> 8)&0xff); \ + /* fall thru */ \ + case 6: *(--(c))=(unsigned char)(((l2)>>16)&0xff); \ + /* fall thru */ \ + case 5: *(--(c))=(unsigned char)(((l2)>>24)&0xff); \ + /* fall thru */ \ + case 4: *(--(c))=(unsigned char)(((l1) )&0xff); \ + /* fall thru */ \ + case 3: *(--(c))=(unsigned char)(((l1)>> 8)&0xff); \ + /* fall thru */ \ + case 2: *(--(c))=(unsigned char)(((l1)>>16)&0xff); \ + /* fall thru */ \ + case 1: *(--(c))=(unsigned char)(((l1)>>24)&0xff); \ + } \ + } + +#undef n2l +#define n2l(c,l) (l =((unsigned long)(*((c)++)))<<24L, \ + l|=((unsigned long)(*((c)++)))<<16L, \ + l|=((unsigned long)(*((c)++)))<< 8L, \ + l|=((unsigned long)(*((c)++)))) + +#undef l2n +#define l2n(l,c) (*((c)++)=(unsigned char)(((l)>>24L)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16L)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8L)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff)) + #if (defined(OPENSSL_SYS_WIN32) && defined(_MSC_VER)) # define ROTATE_l32(a,n) _lrotl(a,n) # define ROTATE_r32(a,n) _lrotr(a,n) @@ -101,8 +158,12 @@ #define RC5_32_MASK 0xffffffffL +#define RC5_16_P 0xB7E1 +#define RC5_16_Q 0x9E37 #define RC5_32_P 0xB7E15163L #define RC5_32_Q 0x9E3779B9L +#define RC5_64_P 0xB7E151628AED2A6BLL +#define RC5_64_Q 0x9E3779B97F4A7C15LL #define E_RC5_32(a,b,s,n) \ a^=b; \ diff --git a/openssl/src/crypto/rcu_internal.h b/openssl/src/crypto/rcu_internal.h deleted file mode 100644 index fb718580e..000000000 --- a/openssl/src/crypto/rcu_internal.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OPENSSL_RCU_INTERNAL_H -# define OPENSSL_RCU_INTERNAL_H -# pragma once - -struct rcu_qp; - -struct rcu_cb_item { - rcu_cb_fn fn; - void *data; - struct rcu_cb_item *next; -}; - -#endif diff --git a/openssl/src/crypto/ripemd/gen/linux_ia32/rmd-586.S b/openssl/src/crypto/ripemd/gen/linux_ia32/rmd-586.S deleted file mode 100644 index 4f3ea459c..000000000 --- a/openssl/src/crypto/ripemd/gen/linux_ia32/rmd-586.S +++ /dev/null @@ -1,1986 +0,0 @@ -.text -.globl ripemd160_block_asm_data_order -.type ripemd160_block_asm_data_order,@function -.align 16 -ripemd160_block_asm_data_order: -.L_ripemd160_block_asm_data_order_begin: - #ifdef __CET__ - -.byte 243,15,30,251 - #endif - - movl 4(%esp),%edx - movl 8(%esp),%eax - pushl %esi - movl (%edx),%ecx - pushl %edi - movl 4(%edx),%esi - pushl %ebp - movl 8(%edx),%edi - pushl %ebx - subl $108,%esp -.L000start: - - movl (%eax),%ebx - movl 4(%eax),%ebp - movl %ebx,(%esp) - movl %ebp,4(%esp) - movl 8(%eax),%ebx - movl 12(%eax),%ebp - movl %ebx,8(%esp) - movl %ebp,12(%esp) - movl 16(%eax),%ebx - movl 20(%eax),%ebp - movl %ebx,16(%esp) - movl %ebp,20(%esp) - movl 24(%eax),%ebx - movl 28(%eax),%ebp - movl %ebx,24(%esp) - movl %ebp,28(%esp) - movl 32(%eax),%ebx - movl 36(%eax),%ebp - movl %ebx,32(%esp) - movl %ebp,36(%esp) - movl 40(%eax),%ebx - movl 44(%eax),%ebp - movl %ebx,40(%esp) - movl %ebp,44(%esp) - movl 48(%eax),%ebx - movl 52(%eax),%ebp - movl %ebx,48(%esp) - movl %ebp,52(%esp) - movl 56(%eax),%ebx - movl 60(%eax),%ebp - movl %ebx,56(%esp) - movl %ebp,60(%esp) - movl %edi,%eax - movl 12(%edx),%ebx - movl 16(%edx),%ebp - - xorl %ebx,%eax - movl (%esp),%edx - xorl %esi,%eax - addl %edx,%ecx - roll $10,%edi - addl %eax,%ecx - movl %esi,%eax - roll $11,%ecx - addl %ebp,%ecx - - xorl %edi,%eax - movl 4(%esp),%edx - xorl %ecx,%eax - addl %eax,%ebp - movl %ecx,%eax - roll $10,%esi - addl %edx,%ebp - xorl %esi,%eax - roll $14,%ebp - addl %ebx,%ebp - - movl 8(%esp),%edx - xorl %ebp,%eax - addl %edx,%ebx - roll $10,%ecx - addl %eax,%ebx - movl %ebp,%eax - roll $15,%ebx - addl %edi,%ebx - - xorl %ecx,%eax - movl 12(%esp),%edx - xorl %ebx,%eax - addl %eax,%edi - movl %ebx,%eax - roll $10,%ebp - addl %edx,%edi - xorl %ebp,%eax - roll $12,%edi - addl %esi,%edi - - movl 16(%esp),%edx - xorl %edi,%eax - addl %edx,%esi - roll $10,%ebx - addl %eax,%esi - movl %edi,%eax - roll $5,%esi - addl %ecx,%esi - - xorl %ebx,%eax - movl 20(%esp),%edx - xorl %esi,%eax - addl %eax,%ecx - movl %esi,%eax - roll $10,%edi - addl %edx,%ecx - xorl %edi,%eax - roll $8,%ecx - addl %ebp,%ecx - - movl 24(%esp),%edx - xorl %ecx,%eax - addl %edx,%ebp - roll $10,%esi - addl %eax,%ebp - movl %ecx,%eax - roll $7,%ebp - addl %ebx,%ebp - - xorl %esi,%eax - movl 28(%esp),%edx - xorl %ebp,%eax - addl %eax,%ebx - movl %ebp,%eax - roll $10,%ecx - addl %edx,%ebx - xorl %ecx,%eax - roll $9,%ebx - addl %edi,%ebx - - movl 32(%esp),%edx - xorl %ebx,%eax - addl %edx,%edi - roll $10,%ebp - addl %eax,%edi - movl %ebx,%eax - roll $11,%edi - addl %esi,%edi - - xorl %ebp,%eax - movl 36(%esp),%edx - xorl %edi,%eax - addl %eax,%esi - movl %edi,%eax - roll $10,%ebx - addl %edx,%esi - xorl %ebx,%eax - roll $13,%esi - addl %ecx,%esi - - movl 40(%esp),%edx - xorl %esi,%eax - addl %edx,%ecx - roll $10,%edi - addl %eax,%ecx - movl %esi,%eax - roll $14,%ecx - addl %ebp,%ecx - - xorl %edi,%eax - movl 44(%esp),%edx - xorl %ecx,%eax - addl %eax,%ebp - movl %ecx,%eax - roll $10,%esi - addl %edx,%ebp - xorl %esi,%eax - roll $15,%ebp - addl %ebx,%ebp - - movl 48(%esp),%edx - xorl %ebp,%eax - addl %edx,%ebx - roll $10,%ecx - addl %eax,%ebx - movl %ebp,%eax - roll $6,%ebx - addl %edi,%ebx - - xorl %ecx,%eax - movl 52(%esp),%edx - xorl %ebx,%eax - addl %eax,%edi - movl %ebx,%eax - roll $10,%ebp - addl %edx,%edi - xorl %ebp,%eax - roll $7,%edi - addl %esi,%edi - - movl 56(%esp),%edx - xorl %edi,%eax - addl %edx,%esi - roll $10,%ebx - addl %eax,%esi - movl %edi,%eax - roll $9,%esi - addl %ecx,%esi - - xorl %ebx,%eax - movl 60(%esp),%edx - xorl %esi,%eax - addl %eax,%ecx - movl $-1,%eax - roll $10,%edi - addl %edx,%ecx - movl 28(%esp),%edx - roll $8,%ecx - addl %ebp,%ecx - - addl %edx,%ebp - movl %esi,%edx - subl %ecx,%eax - andl %ecx,%edx - andl %edi,%eax - orl %eax,%edx - movl 16(%esp),%eax - roll $10,%esi - leal 1518500249(%ebp,%edx,1),%ebp - movl $-1,%edx - roll $7,%ebp - addl %ebx,%ebp - - addl %eax,%ebx - movl %ecx,%eax - subl %ebp,%edx - andl %ebp,%eax - andl %esi,%edx - orl %edx,%eax - movl 52(%esp),%edx - roll $10,%ecx - leal 1518500249(%ebx,%eax,1),%ebx - movl $-1,%eax - roll $6,%ebx - addl %edi,%ebx - - addl %edx,%edi - movl %ebp,%edx - subl %ebx,%eax - andl %ebx,%edx - andl %ecx,%eax - orl %eax,%edx - movl 4(%esp),%eax - roll $10,%ebp - leal 1518500249(%edi,%edx,1),%edi - movl $-1,%edx - roll $8,%edi - addl %esi,%edi - - addl %eax,%esi - movl %ebx,%eax - subl %edi,%edx - andl %edi,%eax - andl %ebp,%edx - orl %edx,%eax - movl 40(%esp),%edx - roll $10,%ebx - leal 1518500249(%esi,%eax,1),%esi - movl $-1,%eax - roll $13,%esi - addl %ecx,%esi - - addl %edx,%ecx - movl %edi,%edx - subl %esi,%eax - andl %esi,%edx - andl %ebx,%eax - orl %eax,%edx - movl 24(%esp),%eax - roll $10,%edi - leal 1518500249(%ecx,%edx,1),%ecx - movl $-1,%edx - roll $11,%ecx - addl %ebp,%ecx - - addl %eax,%ebp - movl %esi,%eax - subl %ecx,%edx - andl %ecx,%eax - andl %edi,%edx - orl %edx,%eax - movl 60(%esp),%edx - roll $10,%esi - leal 1518500249(%ebp,%eax,1),%ebp - movl $-1,%eax - roll $9,%ebp - addl %ebx,%ebp - - addl %edx,%ebx - movl %ecx,%edx - subl %ebp,%eax - andl %ebp,%edx - andl %esi,%eax - orl %eax,%edx - movl 12(%esp),%eax - roll $10,%ecx - leal 1518500249(%ebx,%edx,1),%ebx - movl $-1,%edx - roll $7,%ebx - addl %edi,%ebx - - addl %eax,%edi - movl %ebp,%eax - subl %ebx,%edx - andl %ebx,%eax - andl %ecx,%edx - orl %edx,%eax - movl 48(%esp),%edx - roll $10,%ebp - leal 1518500249(%edi,%eax,1),%edi - movl $-1,%eax - roll $15,%edi - addl %esi,%edi - - addl %edx,%esi - movl %ebx,%edx - subl %edi,%eax - andl %edi,%edx - andl %ebp,%eax - orl %eax,%edx - movl (%esp),%eax - roll $10,%ebx - leal 1518500249(%esi,%edx,1),%esi - movl $-1,%edx - roll $7,%esi - addl %ecx,%esi - - addl %eax,%ecx - movl %edi,%eax - subl %esi,%edx - andl %esi,%eax - andl %ebx,%edx - orl %edx,%eax - movl 36(%esp),%edx - roll $10,%edi - leal 1518500249(%ecx,%eax,1),%ecx - movl $-1,%eax - roll $12,%ecx - addl %ebp,%ecx - - addl %edx,%ebp - movl %esi,%edx - subl %ecx,%eax - andl %ecx,%edx - andl %edi,%eax - orl %eax,%edx - movl 20(%esp),%eax - roll $10,%esi - leal 1518500249(%ebp,%edx,1),%ebp - movl $-1,%edx - roll $15,%ebp - addl %ebx,%ebp - - addl %eax,%ebx - movl %ecx,%eax - subl %ebp,%edx - andl %ebp,%eax - andl %esi,%edx - orl %edx,%eax - movl 8(%esp),%edx - roll $10,%ecx - leal 1518500249(%ebx,%eax,1),%ebx - movl $-1,%eax - roll $9,%ebx - addl %edi,%ebx - - addl %edx,%edi - movl %ebp,%edx - subl %ebx,%eax - andl %ebx,%edx - andl %ecx,%eax - orl %eax,%edx - movl 56(%esp),%eax - roll $10,%ebp - leal 1518500249(%edi,%edx,1),%edi - movl $-1,%edx - roll $11,%edi - addl %esi,%edi - - addl %eax,%esi - movl %ebx,%eax - subl %edi,%edx - andl %edi,%eax - andl %ebp,%edx - orl %edx,%eax - movl 44(%esp),%edx - roll $10,%ebx - leal 1518500249(%esi,%eax,1),%esi - movl $-1,%eax - roll $7,%esi - addl %ecx,%esi - - addl %edx,%ecx - movl %edi,%edx - subl %esi,%eax - andl %esi,%edx - andl %ebx,%eax - orl %eax,%edx - movl 32(%esp),%eax - roll $10,%edi - leal 1518500249(%ecx,%edx,1),%ecx - movl $-1,%edx - roll $13,%ecx - addl %ebp,%ecx - - addl %eax,%ebp - movl %esi,%eax - subl %ecx,%edx - andl %ecx,%eax - andl %edi,%edx - orl %edx,%eax - movl $-1,%edx - roll $10,%esi - leal 1518500249(%ebp,%eax,1),%ebp - subl %ecx,%edx - roll $12,%ebp - addl %ebx,%ebp - - movl 12(%esp),%eax - orl %ebp,%edx - addl %eax,%ebx - xorl %esi,%edx - movl $-1,%eax - roll $10,%ecx - leal 1859775393(%ebx,%edx,1),%ebx - subl %ebp,%eax - roll $11,%ebx - addl %edi,%ebx - - movl 40(%esp),%edx - orl %ebx,%eax - addl %edx,%edi - xorl %ecx,%eax - movl $-1,%edx - roll $10,%ebp - leal 1859775393(%edi,%eax,1),%edi - subl %ebx,%edx - roll $13,%edi - addl %esi,%edi - - movl 56(%esp),%eax - orl %edi,%edx - addl %eax,%esi - xorl %ebp,%edx - movl $-1,%eax - roll $10,%ebx - leal 1859775393(%esi,%edx,1),%esi - subl %edi,%eax - roll $6,%esi - addl %ecx,%esi - - movl 16(%esp),%edx - orl %esi,%eax - addl %edx,%ecx - xorl %ebx,%eax - movl $-1,%edx - roll $10,%edi - leal 1859775393(%ecx,%eax,1),%ecx - subl %esi,%edx - roll $7,%ecx - addl %ebp,%ecx - - movl 36(%esp),%eax - orl %ecx,%edx - addl %eax,%ebp - xorl %edi,%edx - movl $-1,%eax - roll $10,%esi - leal 1859775393(%ebp,%edx,1),%ebp - subl %ecx,%eax - roll $14,%ebp - addl %ebx,%ebp - - movl 60(%esp),%edx - orl %ebp,%eax - addl %edx,%ebx - xorl %esi,%eax - movl $-1,%edx - roll $10,%ecx - leal 1859775393(%ebx,%eax,1),%ebx - subl %ebp,%edx - roll $9,%ebx - addl %edi,%ebx - - movl 32(%esp),%eax - orl %ebx,%edx - addl %eax,%edi - xorl %ecx,%edx - movl $-1,%eax - roll $10,%ebp - leal 1859775393(%edi,%edx,1),%edi - subl %ebx,%eax - roll $13,%edi - addl %esi,%edi - - movl 4(%esp),%edx - orl %edi,%eax - addl %edx,%esi - xorl %ebp,%eax - movl $-1,%edx - roll $10,%ebx - leal 1859775393(%esi,%eax,1),%esi - subl %edi,%edx - roll $15,%esi - addl %ecx,%esi - - movl 8(%esp),%eax - orl %esi,%edx - addl %eax,%ecx - xorl %ebx,%edx - movl $-1,%eax - roll $10,%edi - leal 1859775393(%ecx,%edx,1),%ecx - subl %esi,%eax - roll $14,%ecx - addl %ebp,%ecx - - movl 28(%esp),%edx - orl %ecx,%eax - addl %edx,%ebp - xorl %edi,%eax - movl $-1,%edx - roll $10,%esi - leal 1859775393(%ebp,%eax,1),%ebp - subl %ecx,%edx - roll $8,%ebp - addl %ebx,%ebp - - movl (%esp),%eax - orl %ebp,%edx - addl %eax,%ebx - xorl %esi,%edx - movl $-1,%eax - roll $10,%ecx - leal 1859775393(%ebx,%edx,1),%ebx - subl %ebp,%eax - roll $13,%ebx - addl %edi,%ebx - - movl 24(%esp),%edx - orl %ebx,%eax - addl %edx,%edi - xorl %ecx,%eax - movl $-1,%edx - roll $10,%ebp - leal 1859775393(%edi,%eax,1),%edi - subl %ebx,%edx - roll $6,%edi - addl %esi,%edi - - movl 52(%esp),%eax - orl %edi,%edx - addl %eax,%esi - xorl %ebp,%edx - movl $-1,%eax - roll $10,%ebx - leal 1859775393(%esi,%edx,1),%esi - subl %edi,%eax - roll $5,%esi - addl %ecx,%esi - - movl 44(%esp),%edx - orl %esi,%eax - addl %edx,%ecx - xorl %ebx,%eax - movl $-1,%edx - roll $10,%edi - leal 1859775393(%ecx,%eax,1),%ecx - subl %esi,%edx - roll $12,%ecx - addl %ebp,%ecx - - movl 20(%esp),%eax - orl %ecx,%edx - addl %eax,%ebp - xorl %edi,%edx - movl $-1,%eax - roll $10,%esi - leal 1859775393(%ebp,%edx,1),%ebp - subl %ecx,%eax - roll $7,%ebp - addl %ebx,%ebp - - movl 48(%esp),%edx - orl %ebp,%eax - addl %edx,%ebx - xorl %esi,%eax - movl $-1,%edx - roll $10,%ecx - leal 1859775393(%ebx,%eax,1),%ebx - movl %ecx,%eax - roll $5,%ebx - addl %edi,%ebx - - subl %ecx,%edx - andl %ebx,%eax - andl %ebp,%edx - orl %eax,%edx - movl 4(%esp),%eax - roll $10,%ebp - leal 2400959708(%edi,%edx,1),%edi - movl $-1,%edx - addl %eax,%edi - movl %ebp,%eax - roll $11,%edi - addl %esi,%edi - - subl %ebp,%edx - andl %edi,%eax - andl %ebx,%edx - orl %eax,%edx - movl 36(%esp),%eax - roll $10,%ebx - leal 2400959708(%esi,%edx,1),%esi - movl $-1,%edx - addl %eax,%esi - movl %ebx,%eax - roll $12,%esi - addl %ecx,%esi - - subl %ebx,%edx - andl %esi,%eax - andl %edi,%edx - orl %eax,%edx - movl 44(%esp),%eax - roll $10,%edi - leal 2400959708(%ecx,%edx,1),%ecx - movl $-1,%edx - addl %eax,%ecx - movl %edi,%eax - roll $14,%ecx - addl %ebp,%ecx - - subl %edi,%edx - andl %ecx,%eax - andl %esi,%edx - orl %eax,%edx - movl 40(%esp),%eax - roll $10,%esi - leal 2400959708(%ebp,%edx,1),%ebp - movl $-1,%edx - addl %eax,%ebp - movl %esi,%eax - roll $15,%ebp - addl %ebx,%ebp - - subl %esi,%edx - andl %ebp,%eax - andl %ecx,%edx - orl %eax,%edx - movl (%esp),%eax - roll $10,%ecx - leal 2400959708(%ebx,%edx,1),%ebx - movl $-1,%edx - addl %eax,%ebx - movl %ecx,%eax - roll $14,%ebx - addl %edi,%ebx - - subl %ecx,%edx - andl %ebx,%eax - andl %ebp,%edx - orl %eax,%edx - movl 32(%esp),%eax - roll $10,%ebp - leal 2400959708(%edi,%edx,1),%edi - movl $-1,%edx - addl %eax,%edi - movl %ebp,%eax - roll $15,%edi - addl %esi,%edi - - subl %ebp,%edx - andl %edi,%eax - andl %ebx,%edx - orl %eax,%edx - movl 48(%esp),%eax - roll $10,%ebx - leal 2400959708(%esi,%edx,1),%esi - movl $-1,%edx - addl %eax,%esi - movl %ebx,%eax - roll $9,%esi - addl %ecx,%esi - - subl %ebx,%edx - andl %esi,%eax - andl %edi,%edx - orl %eax,%edx - movl 16(%esp),%eax - roll $10,%edi - leal 2400959708(%ecx,%edx,1),%ecx - movl $-1,%edx - addl %eax,%ecx - movl %edi,%eax - roll $8,%ecx - addl %ebp,%ecx - - subl %edi,%edx - andl %ecx,%eax - andl %esi,%edx - orl %eax,%edx - movl 52(%esp),%eax - roll $10,%esi - leal 2400959708(%ebp,%edx,1),%ebp - movl $-1,%edx - addl %eax,%ebp - movl %esi,%eax - roll $9,%ebp - addl %ebx,%ebp - - subl %esi,%edx - andl %ebp,%eax - andl %ecx,%edx - orl %eax,%edx - movl 12(%esp),%eax - roll $10,%ecx - leal 2400959708(%ebx,%edx,1),%ebx - movl $-1,%edx - addl %eax,%ebx - movl %ecx,%eax - roll $14,%ebx - addl %edi,%ebx - - subl %ecx,%edx - andl %ebx,%eax - andl %ebp,%edx - orl %eax,%edx - movl 28(%esp),%eax - roll $10,%ebp - leal 2400959708(%edi,%edx,1),%edi - movl $-1,%edx - addl %eax,%edi - movl %ebp,%eax - roll $5,%edi - addl %esi,%edi - - subl %ebp,%edx - andl %edi,%eax - andl %ebx,%edx - orl %eax,%edx - movl 60(%esp),%eax - roll $10,%ebx - leal 2400959708(%esi,%edx,1),%esi - movl $-1,%edx - addl %eax,%esi - movl %ebx,%eax - roll $6,%esi - addl %ecx,%esi - - subl %ebx,%edx - andl %esi,%eax - andl %edi,%edx - orl %eax,%edx - movl 56(%esp),%eax - roll $10,%edi - leal 2400959708(%ecx,%edx,1),%ecx - movl $-1,%edx - addl %eax,%ecx - movl %edi,%eax - roll $8,%ecx - addl %ebp,%ecx - - subl %edi,%edx - andl %ecx,%eax - andl %esi,%edx - orl %eax,%edx - movl 20(%esp),%eax - roll $10,%esi - leal 2400959708(%ebp,%edx,1),%ebp - movl $-1,%edx - addl %eax,%ebp - movl %esi,%eax - roll $6,%ebp - addl %ebx,%ebp - - subl %esi,%edx - andl %ebp,%eax - andl %ecx,%edx - orl %eax,%edx - movl 24(%esp),%eax - roll $10,%ecx - leal 2400959708(%ebx,%edx,1),%ebx - movl $-1,%edx - addl %eax,%ebx - movl %ecx,%eax - roll $5,%ebx - addl %edi,%ebx - - subl %ecx,%edx - andl %ebx,%eax - andl %ebp,%edx - orl %eax,%edx - movl 8(%esp),%eax - roll $10,%ebp - leal 2400959708(%edi,%edx,1),%edi - movl $-1,%edx - addl %eax,%edi - subl %ebp,%edx - roll $12,%edi - addl %esi,%edi - - movl 16(%esp),%eax - orl %ebx,%edx - addl %eax,%esi - xorl %edi,%edx - movl $-1,%eax - roll $10,%ebx - leal 2840853838(%esi,%edx,1),%esi - subl %ebx,%eax - roll $9,%esi - addl %ecx,%esi - - movl (%esp),%edx - orl %edi,%eax - addl %edx,%ecx - xorl %esi,%eax - movl $-1,%edx - roll $10,%edi - leal 2840853838(%ecx,%eax,1),%ecx - subl %edi,%edx - roll $15,%ecx - addl %ebp,%ecx - - movl 20(%esp),%eax - orl %esi,%edx - addl %eax,%ebp - xorl %ecx,%edx - movl $-1,%eax - roll $10,%esi - leal 2840853838(%ebp,%edx,1),%ebp - subl %esi,%eax - roll $5,%ebp - addl %ebx,%ebp - - movl 36(%esp),%edx - orl %ecx,%eax - addl %edx,%ebx - xorl %ebp,%eax - movl $-1,%edx - roll $10,%ecx - leal 2840853838(%ebx,%eax,1),%ebx - subl %ecx,%edx - roll $11,%ebx - addl %edi,%ebx - - movl 28(%esp),%eax - orl %ebp,%edx - addl %eax,%edi - xorl %ebx,%edx - movl $-1,%eax - roll $10,%ebp - leal 2840853838(%edi,%edx,1),%edi - subl %ebp,%eax - roll $6,%edi - addl %esi,%edi - - movl 48(%esp),%edx - orl %ebx,%eax - addl %edx,%esi - xorl %edi,%eax - movl $-1,%edx - roll $10,%ebx - leal 2840853838(%esi,%eax,1),%esi - subl %ebx,%edx - roll $8,%esi - addl %ecx,%esi - - movl 8(%esp),%eax - orl %edi,%edx - addl %eax,%ecx - xorl %esi,%edx - movl $-1,%eax - roll $10,%edi - leal 2840853838(%ecx,%edx,1),%ecx - subl %edi,%eax - roll $13,%ecx - addl %ebp,%ecx - - movl 40(%esp),%edx - orl %esi,%eax - addl %edx,%ebp - xorl %ecx,%eax - movl $-1,%edx - roll $10,%esi - leal 2840853838(%ebp,%eax,1),%ebp - subl %esi,%edx - roll $12,%ebp - addl %ebx,%ebp - - movl 56(%esp),%eax - orl %ecx,%edx - addl %eax,%ebx - xorl %ebp,%edx - movl $-1,%eax - roll $10,%ecx - leal 2840853838(%ebx,%edx,1),%ebx - subl %ecx,%eax - roll $5,%ebx - addl %edi,%ebx - - movl 4(%esp),%edx - orl %ebp,%eax - addl %edx,%edi - xorl %ebx,%eax - movl $-1,%edx - roll $10,%ebp - leal 2840853838(%edi,%eax,1),%edi - subl %ebp,%edx - roll $12,%edi - addl %esi,%edi - - movl 12(%esp),%eax - orl %ebx,%edx - addl %eax,%esi - xorl %edi,%edx - movl $-1,%eax - roll $10,%ebx - leal 2840853838(%esi,%edx,1),%esi - subl %ebx,%eax - roll $13,%esi - addl %ecx,%esi - - movl 32(%esp),%edx - orl %edi,%eax - addl %edx,%ecx - xorl %esi,%eax - movl $-1,%edx - roll $10,%edi - leal 2840853838(%ecx,%eax,1),%ecx - subl %edi,%edx - roll $14,%ecx - addl %ebp,%ecx - - movl 44(%esp),%eax - orl %esi,%edx - addl %eax,%ebp - xorl %ecx,%edx - movl $-1,%eax - roll $10,%esi - leal 2840853838(%ebp,%edx,1),%ebp - subl %esi,%eax - roll $11,%ebp - addl %ebx,%ebp - - movl 24(%esp),%edx - orl %ecx,%eax - addl %edx,%ebx - xorl %ebp,%eax - movl $-1,%edx - roll $10,%ecx - leal 2840853838(%ebx,%eax,1),%ebx - subl %ecx,%edx - roll $8,%ebx - addl %edi,%ebx - - movl 60(%esp),%eax - orl %ebp,%edx - addl %eax,%edi - xorl %ebx,%edx - movl $-1,%eax - roll $10,%ebp - leal 2840853838(%edi,%edx,1),%edi - subl %ebp,%eax - roll $5,%edi - addl %esi,%edi - - movl 52(%esp),%edx - orl %ebx,%eax - addl %edx,%esi - xorl %edi,%eax - movl 128(%esp),%edx - roll $10,%ebx - leal 2840853838(%esi,%eax,1),%esi - movl %ecx,64(%esp) - roll $6,%esi - addl %ecx,%esi - movl (%edx),%ecx - movl %esi,68(%esp) - movl %edi,72(%esp) - movl 4(%edx),%esi - movl %ebx,76(%esp) - movl 8(%edx),%edi - movl %ebp,80(%esp) - movl 12(%edx),%ebx - movl 16(%edx),%ebp - - movl $-1,%edx - subl %ebx,%edx - movl 20(%esp),%eax - orl %edi,%edx - addl %eax,%ecx - xorl %esi,%edx - movl $-1,%eax - roll $10,%edi - leal 1352829926(%ecx,%edx,1),%ecx - subl %edi,%eax - roll $8,%ecx - addl %ebp,%ecx - - movl 56(%esp),%edx - orl %esi,%eax - addl %edx,%ebp - xorl %ecx,%eax - movl $-1,%edx - roll $10,%esi - leal 1352829926(%ebp,%eax,1),%ebp - subl %esi,%edx - roll $9,%ebp - addl %ebx,%ebp - - movl 28(%esp),%eax - orl %ecx,%edx - addl %eax,%ebx - xorl %ebp,%edx - movl $-1,%eax - roll $10,%ecx - leal 1352829926(%ebx,%edx,1),%ebx - subl %ecx,%eax - roll $9,%ebx - addl %edi,%ebx - - movl (%esp),%edx - orl %ebp,%eax - addl %edx,%edi - xorl %ebx,%eax - movl $-1,%edx - roll $10,%ebp - leal 1352829926(%edi,%eax,1),%edi - subl %ebp,%edx - roll $11,%edi - addl %esi,%edi - - movl 36(%esp),%eax - orl %ebx,%edx - addl %eax,%esi - xorl %edi,%edx - movl $-1,%eax - roll $10,%ebx - leal 1352829926(%esi,%edx,1),%esi - subl %ebx,%eax - roll $13,%esi - addl %ecx,%esi - - movl 8(%esp),%edx - orl %edi,%eax - addl %edx,%ecx - xorl %esi,%eax - movl $-1,%edx - roll $10,%edi - leal 1352829926(%ecx,%eax,1),%ecx - subl %edi,%edx - roll $15,%ecx - addl %ebp,%ecx - - movl 44(%esp),%eax - orl %esi,%edx - addl %eax,%ebp - xorl %ecx,%edx - movl $-1,%eax - roll $10,%esi - leal 1352829926(%ebp,%edx,1),%ebp - subl %esi,%eax - roll $15,%ebp - addl %ebx,%ebp - - movl 16(%esp),%edx - orl %ecx,%eax - addl %edx,%ebx - xorl %ebp,%eax - movl $-1,%edx - roll $10,%ecx - leal 1352829926(%ebx,%eax,1),%ebx - subl %ecx,%edx - roll $5,%ebx - addl %edi,%ebx - - movl 52(%esp),%eax - orl %ebp,%edx - addl %eax,%edi - xorl %ebx,%edx - movl $-1,%eax - roll $10,%ebp - leal 1352829926(%edi,%edx,1),%edi - subl %ebp,%eax - roll $7,%edi - addl %esi,%edi - - movl 24(%esp),%edx - orl %ebx,%eax - addl %edx,%esi - xorl %edi,%eax - movl $-1,%edx - roll $10,%ebx - leal 1352829926(%esi,%eax,1),%esi - subl %ebx,%edx - roll $7,%esi - addl %ecx,%esi - - movl 60(%esp),%eax - orl %edi,%edx - addl %eax,%ecx - xorl %esi,%edx - movl $-1,%eax - roll $10,%edi - leal 1352829926(%ecx,%edx,1),%ecx - subl %edi,%eax - roll $8,%ecx - addl %ebp,%ecx - - movl 32(%esp),%edx - orl %esi,%eax - addl %edx,%ebp - xorl %ecx,%eax - movl $-1,%edx - roll $10,%esi - leal 1352829926(%ebp,%eax,1),%ebp - subl %esi,%edx - roll $11,%ebp - addl %ebx,%ebp - - movl 4(%esp),%eax - orl %ecx,%edx - addl %eax,%ebx - xorl %ebp,%edx - movl $-1,%eax - roll $10,%ecx - leal 1352829926(%ebx,%edx,1),%ebx - subl %ecx,%eax - roll $14,%ebx - addl %edi,%ebx - - movl 40(%esp),%edx - orl %ebp,%eax - addl %edx,%edi - xorl %ebx,%eax - movl $-1,%edx - roll $10,%ebp - leal 1352829926(%edi,%eax,1),%edi - subl %ebp,%edx - roll $14,%edi - addl %esi,%edi - - movl 12(%esp),%eax - orl %ebx,%edx - addl %eax,%esi - xorl %edi,%edx - movl $-1,%eax - roll $10,%ebx - leal 1352829926(%esi,%edx,1),%esi - subl %ebx,%eax - roll $12,%esi - addl %ecx,%esi - - movl 48(%esp),%edx - orl %edi,%eax - addl %edx,%ecx - xorl %esi,%eax - movl $-1,%edx - roll $10,%edi - leal 1352829926(%ecx,%eax,1),%ecx - movl %edi,%eax - roll $6,%ecx - addl %ebp,%ecx - - subl %edi,%edx - andl %ecx,%eax - andl %esi,%edx - orl %eax,%edx - movl 24(%esp),%eax - roll $10,%esi - leal 1548603684(%ebp,%edx,1),%ebp - movl $-1,%edx - addl %eax,%ebp - movl %esi,%eax - roll $9,%ebp - addl %ebx,%ebp - - subl %esi,%edx - andl %ebp,%eax - andl %ecx,%edx - orl %eax,%edx - movl 44(%esp),%eax - roll $10,%ecx - leal 1548603684(%ebx,%edx,1),%ebx - movl $-1,%edx - addl %eax,%ebx - movl %ecx,%eax - roll $13,%ebx - addl %edi,%ebx - - subl %ecx,%edx - andl %ebx,%eax - andl %ebp,%edx - orl %eax,%edx - movl 12(%esp),%eax - roll $10,%ebp - leal 1548603684(%edi,%edx,1),%edi - movl $-1,%edx - addl %eax,%edi - movl %ebp,%eax - roll $15,%edi - addl %esi,%edi - - subl %ebp,%edx - andl %edi,%eax - andl %ebx,%edx - orl %eax,%edx - movl 28(%esp),%eax - roll $10,%ebx - leal 1548603684(%esi,%edx,1),%esi - movl $-1,%edx - addl %eax,%esi - movl %ebx,%eax - roll $7,%esi - addl %ecx,%esi - - subl %ebx,%edx - andl %esi,%eax - andl %edi,%edx - orl %eax,%edx - movl (%esp),%eax - roll $10,%edi - leal 1548603684(%ecx,%edx,1),%ecx - movl $-1,%edx - addl %eax,%ecx - movl %edi,%eax - roll $12,%ecx - addl %ebp,%ecx - - subl %edi,%edx - andl %ecx,%eax - andl %esi,%edx - orl %eax,%edx - movl 52(%esp),%eax - roll $10,%esi - leal 1548603684(%ebp,%edx,1),%ebp - movl $-1,%edx - addl %eax,%ebp - movl %esi,%eax - roll $8,%ebp - addl %ebx,%ebp - - subl %esi,%edx - andl %ebp,%eax - andl %ecx,%edx - orl %eax,%edx - movl 20(%esp),%eax - roll $10,%ecx - leal 1548603684(%ebx,%edx,1),%ebx - movl $-1,%edx - addl %eax,%ebx - movl %ecx,%eax - roll $9,%ebx - addl %edi,%ebx - - subl %ecx,%edx - andl %ebx,%eax - andl %ebp,%edx - orl %eax,%edx - movl 40(%esp),%eax - roll $10,%ebp - leal 1548603684(%edi,%edx,1),%edi - movl $-1,%edx - addl %eax,%edi - movl %ebp,%eax - roll $11,%edi - addl %esi,%edi - - subl %ebp,%edx - andl %edi,%eax - andl %ebx,%edx - orl %eax,%edx - movl 56(%esp),%eax - roll $10,%ebx - leal 1548603684(%esi,%edx,1),%esi - movl $-1,%edx - addl %eax,%esi - movl %ebx,%eax - roll $7,%esi - addl %ecx,%esi - - subl %ebx,%edx - andl %esi,%eax - andl %edi,%edx - orl %eax,%edx - movl 60(%esp),%eax - roll $10,%edi - leal 1548603684(%ecx,%edx,1),%ecx - movl $-1,%edx - addl %eax,%ecx - movl %edi,%eax - roll $7,%ecx - addl %ebp,%ecx - - subl %edi,%edx - andl %ecx,%eax - andl %esi,%edx - orl %eax,%edx - movl 32(%esp),%eax - roll $10,%esi - leal 1548603684(%ebp,%edx,1),%ebp - movl $-1,%edx - addl %eax,%ebp - movl %esi,%eax - roll $12,%ebp - addl %ebx,%ebp - - subl %esi,%edx - andl %ebp,%eax - andl %ecx,%edx - orl %eax,%edx - movl 48(%esp),%eax - roll $10,%ecx - leal 1548603684(%ebx,%edx,1),%ebx - movl $-1,%edx - addl %eax,%ebx - movl %ecx,%eax - roll $7,%ebx - addl %edi,%ebx - - subl %ecx,%edx - andl %ebx,%eax - andl %ebp,%edx - orl %eax,%edx - movl 16(%esp),%eax - roll $10,%ebp - leal 1548603684(%edi,%edx,1),%edi - movl $-1,%edx - addl %eax,%edi - movl %ebp,%eax - roll $6,%edi - addl %esi,%edi - - subl %ebp,%edx - andl %edi,%eax - andl %ebx,%edx - orl %eax,%edx - movl 36(%esp),%eax - roll $10,%ebx - leal 1548603684(%esi,%edx,1),%esi - movl $-1,%edx - addl %eax,%esi - movl %ebx,%eax - roll $15,%esi - addl %ecx,%esi - - subl %ebx,%edx - andl %esi,%eax - andl %edi,%edx - orl %eax,%edx - movl 4(%esp),%eax - roll $10,%edi - leal 1548603684(%ecx,%edx,1),%ecx - movl $-1,%edx - addl %eax,%ecx - movl %edi,%eax - roll $13,%ecx - addl %ebp,%ecx - - subl %edi,%edx - andl %ecx,%eax - andl %esi,%edx - orl %eax,%edx - movl 8(%esp),%eax - roll $10,%esi - leal 1548603684(%ebp,%edx,1),%ebp - movl $-1,%edx - addl %eax,%ebp - subl %ecx,%edx - roll $11,%ebp - addl %ebx,%ebp - - movl 60(%esp),%eax - orl %ebp,%edx - addl %eax,%ebx - xorl %esi,%edx - movl $-1,%eax - roll $10,%ecx - leal 1836072691(%ebx,%edx,1),%ebx - subl %ebp,%eax - roll $9,%ebx - addl %edi,%ebx - - movl 20(%esp),%edx - orl %ebx,%eax - addl %edx,%edi - xorl %ecx,%eax - movl $-1,%edx - roll $10,%ebp - leal 1836072691(%edi,%eax,1),%edi - subl %ebx,%edx - roll $7,%edi - addl %esi,%edi - - movl 4(%esp),%eax - orl %edi,%edx - addl %eax,%esi - xorl %ebp,%edx - movl $-1,%eax - roll $10,%ebx - leal 1836072691(%esi,%edx,1),%esi - subl %edi,%eax - roll $15,%esi - addl %ecx,%esi - - movl 12(%esp),%edx - orl %esi,%eax - addl %edx,%ecx - xorl %ebx,%eax - movl $-1,%edx - roll $10,%edi - leal 1836072691(%ecx,%eax,1),%ecx - subl %esi,%edx - roll $11,%ecx - addl %ebp,%ecx - - movl 28(%esp),%eax - orl %ecx,%edx - addl %eax,%ebp - xorl %edi,%edx - movl $-1,%eax - roll $10,%esi - leal 1836072691(%ebp,%edx,1),%ebp - subl %ecx,%eax - roll $8,%ebp - addl %ebx,%ebp - - movl 56(%esp),%edx - orl %ebp,%eax - addl %edx,%ebx - xorl %esi,%eax - movl $-1,%edx - roll $10,%ecx - leal 1836072691(%ebx,%eax,1),%ebx - subl %ebp,%edx - roll $6,%ebx - addl %edi,%ebx - - movl 24(%esp),%eax - orl %ebx,%edx - addl %eax,%edi - xorl %ecx,%edx - movl $-1,%eax - roll $10,%ebp - leal 1836072691(%edi,%edx,1),%edi - subl %ebx,%eax - roll $6,%edi - addl %esi,%edi - - movl 36(%esp),%edx - orl %edi,%eax - addl %edx,%esi - xorl %ebp,%eax - movl $-1,%edx - roll $10,%ebx - leal 1836072691(%esi,%eax,1),%esi - subl %edi,%edx - roll $14,%esi - addl %ecx,%esi - - movl 44(%esp),%eax - orl %esi,%edx - addl %eax,%ecx - xorl %ebx,%edx - movl $-1,%eax - roll $10,%edi - leal 1836072691(%ecx,%edx,1),%ecx - subl %esi,%eax - roll $12,%ecx - addl %ebp,%ecx - - movl 32(%esp),%edx - orl %ecx,%eax - addl %edx,%ebp - xorl %edi,%eax - movl $-1,%edx - roll $10,%esi - leal 1836072691(%ebp,%eax,1),%ebp - subl %ecx,%edx - roll $13,%ebp - addl %ebx,%ebp - - movl 48(%esp),%eax - orl %ebp,%edx - addl %eax,%ebx - xorl %esi,%edx - movl $-1,%eax - roll $10,%ecx - leal 1836072691(%ebx,%edx,1),%ebx - subl %ebp,%eax - roll $5,%ebx - addl %edi,%ebx - - movl 8(%esp),%edx - orl %ebx,%eax - addl %edx,%edi - xorl %ecx,%eax - movl $-1,%edx - roll $10,%ebp - leal 1836072691(%edi,%eax,1),%edi - subl %ebx,%edx - roll $14,%edi - addl %esi,%edi - - movl 40(%esp),%eax - orl %edi,%edx - addl %eax,%esi - xorl %ebp,%edx - movl $-1,%eax - roll $10,%ebx - leal 1836072691(%esi,%edx,1),%esi - subl %edi,%eax - roll $13,%esi - addl %ecx,%esi - - movl (%esp),%edx - orl %esi,%eax - addl %edx,%ecx - xorl %ebx,%eax - movl $-1,%edx - roll $10,%edi - leal 1836072691(%ecx,%eax,1),%ecx - subl %esi,%edx - roll $13,%ecx - addl %ebp,%ecx - - movl 16(%esp),%eax - orl %ecx,%edx - addl %eax,%ebp - xorl %edi,%edx - movl $-1,%eax - roll $10,%esi - leal 1836072691(%ebp,%edx,1),%ebp - subl %ecx,%eax - roll $7,%ebp - addl %ebx,%ebp - - movl 52(%esp),%edx - orl %ebp,%eax - addl %edx,%ebx - xorl %esi,%eax - movl 32(%esp),%edx - roll $10,%ecx - leal 1836072691(%ebx,%eax,1),%ebx - movl $-1,%eax - roll $5,%ebx - addl %edi,%ebx - - addl %edx,%edi - movl %ebp,%edx - subl %ebx,%eax - andl %ebx,%edx - andl %ecx,%eax - orl %eax,%edx - movl 24(%esp),%eax - roll $10,%ebp - leal 2053994217(%edi,%edx,1),%edi - movl $-1,%edx - roll $15,%edi - addl %esi,%edi - - addl %eax,%esi - movl %ebx,%eax - subl %edi,%edx - andl %edi,%eax - andl %ebp,%edx - orl %edx,%eax - movl 16(%esp),%edx - roll $10,%ebx - leal 2053994217(%esi,%eax,1),%esi - movl $-1,%eax - roll $5,%esi - addl %ecx,%esi - - addl %edx,%ecx - movl %edi,%edx - subl %esi,%eax - andl %esi,%edx - andl %ebx,%eax - orl %eax,%edx - movl 4(%esp),%eax - roll $10,%edi - leal 2053994217(%ecx,%edx,1),%ecx - movl $-1,%edx - roll $8,%ecx - addl %ebp,%ecx - - addl %eax,%ebp - movl %esi,%eax - subl %ecx,%edx - andl %ecx,%eax - andl %edi,%edx - orl %edx,%eax - movl 12(%esp),%edx - roll $10,%esi - leal 2053994217(%ebp,%eax,1),%ebp - movl $-1,%eax - roll $11,%ebp - addl %ebx,%ebp - - addl %edx,%ebx - movl %ecx,%edx - subl %ebp,%eax - andl %ebp,%edx - andl %esi,%eax - orl %eax,%edx - movl 44(%esp),%eax - roll $10,%ecx - leal 2053994217(%ebx,%edx,1),%ebx - movl $-1,%edx - roll $14,%ebx - addl %edi,%ebx - - addl %eax,%edi - movl %ebp,%eax - subl %ebx,%edx - andl %ebx,%eax - andl %ecx,%edx - orl %edx,%eax - movl 60(%esp),%edx - roll $10,%ebp - leal 2053994217(%edi,%eax,1),%edi - movl $-1,%eax - roll $14,%edi - addl %esi,%edi - - addl %edx,%esi - movl %ebx,%edx - subl %edi,%eax - andl %edi,%edx - andl %ebp,%eax - orl %eax,%edx - movl (%esp),%eax - roll $10,%ebx - leal 2053994217(%esi,%edx,1),%esi - movl $-1,%edx - roll $6,%esi - addl %ecx,%esi - - addl %eax,%ecx - movl %edi,%eax - subl %esi,%edx - andl %esi,%eax - andl %ebx,%edx - orl %edx,%eax - movl 20(%esp),%edx - roll $10,%edi - leal 2053994217(%ecx,%eax,1),%ecx - movl $-1,%eax - roll $14,%ecx - addl %ebp,%ecx - - addl %edx,%ebp - movl %esi,%edx - subl %ecx,%eax - andl %ecx,%edx - andl %edi,%eax - orl %eax,%edx - movl 48(%esp),%eax - roll $10,%esi - leal 2053994217(%ebp,%edx,1),%ebp - movl $-1,%edx - roll $6,%ebp - addl %ebx,%ebp - - addl %eax,%ebx - movl %ecx,%eax - subl %ebp,%edx - andl %ebp,%eax - andl %esi,%edx - orl %edx,%eax - movl 8(%esp),%edx - roll $10,%ecx - leal 2053994217(%ebx,%eax,1),%ebx - movl $-1,%eax - roll $9,%ebx - addl %edi,%ebx - - addl %edx,%edi - movl %ebp,%edx - subl %ebx,%eax - andl %ebx,%edx - andl %ecx,%eax - orl %eax,%edx - movl 52(%esp),%eax - roll $10,%ebp - leal 2053994217(%edi,%edx,1),%edi - movl $-1,%edx - roll $12,%edi - addl %esi,%edi - - addl %eax,%esi - movl %ebx,%eax - subl %edi,%edx - andl %edi,%eax - andl %ebp,%edx - orl %edx,%eax - movl 36(%esp),%edx - roll $10,%ebx - leal 2053994217(%esi,%eax,1),%esi - movl $-1,%eax - roll $9,%esi - addl %ecx,%esi - - addl %edx,%ecx - movl %edi,%edx - subl %esi,%eax - andl %esi,%edx - andl %ebx,%eax - orl %eax,%edx - movl 28(%esp),%eax - roll $10,%edi - leal 2053994217(%ecx,%edx,1),%ecx - movl $-1,%edx - roll $12,%ecx - addl %ebp,%ecx - - addl %eax,%ebp - movl %esi,%eax - subl %ecx,%edx - andl %ecx,%eax - andl %edi,%edx - orl %edx,%eax - movl 40(%esp),%edx - roll $10,%esi - leal 2053994217(%ebp,%eax,1),%ebp - movl $-1,%eax - roll $5,%ebp - addl %ebx,%ebp - - addl %edx,%ebx - movl %ecx,%edx - subl %ebp,%eax - andl %ebp,%edx - andl %esi,%eax - orl %eax,%edx - movl 56(%esp),%eax - roll $10,%ecx - leal 2053994217(%ebx,%edx,1),%ebx - movl $-1,%edx - roll $15,%ebx - addl %edi,%ebx - - addl %eax,%edi - movl %ebp,%eax - subl %ebx,%edx - andl %ebx,%eax - andl %ecx,%edx - orl %eax,%edx - movl %ebx,%eax - roll $10,%ebp - leal 2053994217(%edi,%edx,1),%edi - xorl %ebp,%eax - roll $8,%edi - addl %esi,%edi - - movl 48(%esp),%edx - xorl %edi,%eax - addl %edx,%esi - roll $10,%ebx - addl %eax,%esi - movl %edi,%eax - roll $8,%esi - addl %ecx,%esi - - xorl %ebx,%eax - movl 60(%esp),%edx - xorl %esi,%eax - addl %eax,%ecx - movl %esi,%eax - roll $10,%edi - addl %edx,%ecx - xorl %edi,%eax - roll $5,%ecx - addl %ebp,%ecx - - movl 40(%esp),%edx - xorl %ecx,%eax - addl %edx,%ebp - roll $10,%esi - addl %eax,%ebp - movl %ecx,%eax - roll $12,%ebp - addl %ebx,%ebp - - xorl %esi,%eax - movl 16(%esp),%edx - xorl %ebp,%eax - addl %eax,%ebx - movl %ebp,%eax - roll $10,%ecx - addl %edx,%ebx - xorl %ecx,%eax - roll $9,%ebx - addl %edi,%ebx - - movl 4(%esp),%edx - xorl %ebx,%eax - addl %edx,%edi - roll $10,%ebp - addl %eax,%edi - movl %ebx,%eax - roll $12,%edi - addl %esi,%edi - - xorl %ebp,%eax - movl 20(%esp),%edx - xorl %edi,%eax - addl %eax,%esi - movl %edi,%eax - roll $10,%ebx - addl %edx,%esi - xorl %ebx,%eax - roll $5,%esi - addl %ecx,%esi - - movl 32(%esp),%edx - xorl %esi,%eax - addl %edx,%ecx - roll $10,%edi - addl %eax,%ecx - movl %esi,%eax - roll $14,%ecx - addl %ebp,%ecx - - xorl %edi,%eax - movl 28(%esp),%edx - xorl %ecx,%eax - addl %eax,%ebp - movl %ecx,%eax - roll $10,%esi - addl %edx,%ebp - xorl %esi,%eax - roll $6,%ebp - addl %ebx,%ebp - - movl 24(%esp),%edx - xorl %ebp,%eax - addl %edx,%ebx - roll $10,%ecx - addl %eax,%ebx - movl %ebp,%eax - roll $8,%ebx - addl %edi,%ebx - - xorl %ecx,%eax - movl 8(%esp),%edx - xorl %ebx,%eax - addl %eax,%edi - movl %ebx,%eax - roll $10,%ebp - addl %edx,%edi - xorl %ebp,%eax - roll $13,%edi - addl %esi,%edi - - movl 52(%esp),%edx - xorl %edi,%eax - addl %edx,%esi - roll $10,%ebx - addl %eax,%esi - movl %edi,%eax - roll $6,%esi - addl %ecx,%esi - - xorl %ebx,%eax - movl 56(%esp),%edx - xorl %esi,%eax - addl %eax,%ecx - movl %esi,%eax - roll $10,%edi - addl %edx,%ecx - xorl %edi,%eax - roll $5,%ecx - addl %ebp,%ecx - - movl (%esp),%edx - xorl %ecx,%eax - addl %edx,%ebp - roll $10,%esi - addl %eax,%ebp - movl %ecx,%eax - roll $15,%ebp - addl %ebx,%ebp - - xorl %esi,%eax - movl 12(%esp),%edx - xorl %ebp,%eax - addl %eax,%ebx - movl %ebp,%eax - roll $10,%ecx - addl %edx,%ebx - xorl %ecx,%eax - roll $13,%ebx - addl %edi,%ebx - - movl 36(%esp),%edx - xorl %ebx,%eax - addl %edx,%edi - roll $10,%ebp - addl %eax,%edi - movl %ebx,%eax - roll $11,%edi - addl %esi,%edi - - xorl %ebp,%eax - movl 44(%esp),%edx - xorl %edi,%eax - addl %eax,%esi - roll $10,%ebx - addl %edx,%esi - movl 128(%esp),%edx - roll $11,%esi - addl %ecx,%esi - movl 4(%edx),%eax - addl %eax,%ebx - movl 72(%esp),%eax - addl %eax,%ebx - movl 8(%edx),%eax - addl %eax,%ebp - movl 76(%esp),%eax - addl %eax,%ebp - movl 12(%edx),%eax - addl %eax,%ecx - movl 80(%esp),%eax - addl %eax,%ecx - movl 16(%edx),%eax - addl %eax,%esi - movl 64(%esp),%eax - addl %eax,%esi - movl (%edx),%eax - addl %eax,%edi - movl 68(%esp),%eax - addl %eax,%edi - movl 136(%esp),%eax - movl %ebx,(%edx) - movl %ebp,4(%edx) - movl %ecx,8(%edx) - subl $1,%eax - movl %esi,12(%edx) - movl %edi,16(%edx) - jle .L001get_out - movl %eax,136(%esp) - movl %ecx,%edi - movl 132(%esp),%eax - movl %ebx,%ecx - addl $64,%eax - movl %ebp,%esi - movl %eax,132(%esp) - jmp .L000start -.L001get_out: - addl $108,%esp - popl %ebx - popl %ebp - popl %edi - popl %esi - ret -.size ripemd160_block_asm_data_order,.-.L_ripemd160_block_asm_data_order_begin - - .section ".note.gnu.property", "a" - .p2align 2 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - .asciz "GNU" -1: - .p2align 2 - .long 0xc0000002 - .long 3f - 2f -2: - .long 3 -3: - .p2align 2 -4: diff --git a/openssl/src/crypto/ripemd/gen/windows_ia32/rmd-586.asm b/openssl/src/crypto/ripemd/gen/windows_ia32/rmd-586.asm deleted file mode 100644 index 579ac0bd6..000000000 --- a/openssl/src/crypto/ripemd/gen/windows_ia32/rmd-586.asm +++ /dev/null @@ -1,1970 +0,0 @@ - -%ifidn __OUTPUT_FORMAT__,obj -section code use32 class=code align=64 -%elifidn __OUTPUT_FORMAT__,win32 -$@feat.00 equ 1 -section .text code align=64 -%else -section .text code -%endif -global _ripemd160_block_asm_data_order -align 16 -_ripemd160_block_asm_data_order: -L$_ripemd160_block_asm_data_order_begin: - mov edx,DWORD [4+esp] - mov eax,DWORD [8+esp] - push esi - mov ecx,DWORD [edx] - push edi - mov esi,DWORD [4+edx] - push ebp - mov edi,DWORD [8+edx] - push ebx - sub esp,108 -L$000start: - ; - mov ebx,DWORD [eax] - mov ebp,DWORD [4+eax] - mov DWORD [esp],ebx - mov DWORD [4+esp],ebp - mov ebx,DWORD [8+eax] - mov ebp,DWORD [12+eax] - mov DWORD [8+esp],ebx - mov DWORD [12+esp],ebp - mov ebx,DWORD [16+eax] - mov ebp,DWORD [20+eax] - mov DWORD [16+esp],ebx - mov DWORD [20+esp],ebp - mov ebx,DWORD [24+eax] - mov ebp,DWORD [28+eax] - mov DWORD [24+esp],ebx - mov DWORD [28+esp],ebp - mov ebx,DWORD [32+eax] - mov ebp,DWORD [36+eax] - mov DWORD [32+esp],ebx - mov DWORD [36+esp],ebp - mov ebx,DWORD [40+eax] - mov ebp,DWORD [44+eax] - mov DWORD [40+esp],ebx - mov DWORD [44+esp],ebp - mov ebx,DWORD [48+eax] - mov ebp,DWORD [52+eax] - mov DWORD [48+esp],ebx - mov DWORD [52+esp],ebp - mov ebx,DWORD [56+eax] - mov ebp,DWORD [60+eax] - mov DWORD [56+esp],ebx - mov DWORD [60+esp],ebp - mov eax,edi - mov ebx,DWORD [12+edx] - mov ebp,DWORD [16+edx] - ; 0 - xor eax,ebx - mov edx,DWORD [esp] - xor eax,esi - add ecx,edx - rol edi,10 - add ecx,eax - mov eax,esi - rol ecx,11 - add ecx,ebp - ; 1 - xor eax,edi - mov edx,DWORD [4+esp] - xor eax,ecx - add ebp,eax - mov eax,ecx - rol esi,10 - add ebp,edx - xor eax,esi - rol ebp,14 - add ebp,ebx - ; 2 - mov edx,DWORD [8+esp] - xor eax,ebp - add ebx,edx - rol ecx,10 - add ebx,eax - mov eax,ebp - rol ebx,15 - add ebx,edi - ; 3 - xor eax,ecx - mov edx,DWORD [12+esp] - xor eax,ebx - add edi,eax - mov eax,ebx - rol ebp,10 - add edi,edx - xor eax,ebp - rol edi,12 - add edi,esi - ; 4 - mov edx,DWORD [16+esp] - xor eax,edi - add esi,edx - rol ebx,10 - add esi,eax - mov eax,edi - rol esi,5 - add esi,ecx - ; 5 - xor eax,ebx - mov edx,DWORD [20+esp] - xor eax,esi - add ecx,eax - mov eax,esi - rol edi,10 - add ecx,edx - xor eax,edi - rol ecx,8 - add ecx,ebp - ; 6 - mov edx,DWORD [24+esp] - xor eax,ecx - add ebp,edx - rol esi,10 - add ebp,eax - mov eax,ecx - rol ebp,7 - add ebp,ebx - ; 7 - xor eax,esi - mov edx,DWORD [28+esp] - xor eax,ebp - add ebx,eax - mov eax,ebp - rol ecx,10 - add ebx,edx - xor eax,ecx - rol ebx,9 - add ebx,edi - ; 8 - mov edx,DWORD [32+esp] - xor eax,ebx - add edi,edx - rol ebp,10 - add edi,eax - mov eax,ebx - rol edi,11 - add edi,esi - ; 9 - xor eax,ebp - mov edx,DWORD [36+esp] - xor eax,edi - add esi,eax - mov eax,edi - rol ebx,10 - add esi,edx - xor eax,ebx - rol esi,13 - add esi,ecx - ; 10 - mov edx,DWORD [40+esp] - xor eax,esi - add ecx,edx - rol edi,10 - add ecx,eax - mov eax,esi - rol ecx,14 - add ecx,ebp - ; 11 - xor eax,edi - mov edx,DWORD [44+esp] - xor eax,ecx - add ebp,eax - mov eax,ecx - rol esi,10 - add ebp,edx - xor eax,esi - rol ebp,15 - add ebp,ebx - ; 12 - mov edx,DWORD [48+esp] - xor eax,ebp - add ebx,edx - rol ecx,10 - add ebx,eax - mov eax,ebp - rol ebx,6 - add ebx,edi - ; 13 - xor eax,ecx - mov edx,DWORD [52+esp] - xor eax,ebx - add edi,eax - mov eax,ebx - rol ebp,10 - add edi,edx - xor eax,ebp - rol edi,7 - add edi,esi - ; 14 - mov edx,DWORD [56+esp] - xor eax,edi - add esi,edx - rol ebx,10 - add esi,eax - mov eax,edi - rol esi,9 - add esi,ecx - ; 15 - xor eax,ebx - mov edx,DWORD [60+esp] - xor eax,esi - add ecx,eax - mov eax,-1 - rol edi,10 - add ecx,edx - mov edx,DWORD [28+esp] - rol ecx,8 - add ecx,ebp - ; 16 - add ebp,edx - mov edx,esi - sub eax,ecx - and edx,ecx - and eax,edi - or edx,eax - mov eax,DWORD [16+esp] - rol esi,10 - lea ebp,[1518500249+edx*1+ebp] - mov edx,-1 - rol ebp,7 - add ebp,ebx - ; 17 - add ebx,eax - mov eax,ecx - sub edx,ebp - and eax,ebp - and edx,esi - or eax,edx - mov edx,DWORD [52+esp] - rol ecx,10 - lea ebx,[1518500249+eax*1+ebx] - mov eax,-1 - rol ebx,6 - add ebx,edi - ; 18 - add edi,edx - mov edx,ebp - sub eax,ebx - and edx,ebx - and eax,ecx - or edx,eax - mov eax,DWORD [4+esp] - rol ebp,10 - lea edi,[1518500249+edx*1+edi] - mov edx,-1 - rol edi,8 - add edi,esi - ; 19 - add esi,eax - mov eax,ebx - sub edx,edi - and eax,edi - and edx,ebp - or eax,edx - mov edx,DWORD [40+esp] - rol ebx,10 - lea esi,[1518500249+eax*1+esi] - mov eax,-1 - rol esi,13 - add esi,ecx - ; 20 - add ecx,edx - mov edx,edi - sub eax,esi - and edx,esi - and eax,ebx - or edx,eax - mov eax,DWORD [24+esp] - rol edi,10 - lea ecx,[1518500249+edx*1+ecx] - mov edx,-1 - rol ecx,11 - add ecx,ebp - ; 21 - add ebp,eax - mov eax,esi - sub edx,ecx - and eax,ecx - and edx,edi - or eax,edx - mov edx,DWORD [60+esp] - rol esi,10 - lea ebp,[1518500249+eax*1+ebp] - mov eax,-1 - rol ebp,9 - add ebp,ebx - ; 22 - add ebx,edx - mov edx,ecx - sub eax,ebp - and edx,ebp - and eax,esi - or edx,eax - mov eax,DWORD [12+esp] - rol ecx,10 - lea ebx,[1518500249+edx*1+ebx] - mov edx,-1 - rol ebx,7 - add ebx,edi - ; 23 - add edi,eax - mov eax,ebp - sub edx,ebx - and eax,ebx - and edx,ecx - or eax,edx - mov edx,DWORD [48+esp] - rol ebp,10 - lea edi,[1518500249+eax*1+edi] - mov eax,-1 - rol edi,15 - add edi,esi - ; 24 - add esi,edx - mov edx,ebx - sub eax,edi - and edx,edi - and eax,ebp - or edx,eax - mov eax,DWORD [esp] - rol ebx,10 - lea esi,[1518500249+edx*1+esi] - mov edx,-1 - rol esi,7 - add esi,ecx - ; 25 - add ecx,eax - mov eax,edi - sub edx,esi - and eax,esi - and edx,ebx - or eax,edx - mov edx,DWORD [36+esp] - rol edi,10 - lea ecx,[1518500249+eax*1+ecx] - mov eax,-1 - rol ecx,12 - add ecx,ebp - ; 26 - add ebp,edx - mov edx,esi - sub eax,ecx - and edx,ecx - and eax,edi - or edx,eax - mov eax,DWORD [20+esp] - rol esi,10 - lea ebp,[1518500249+edx*1+ebp] - mov edx,-1 - rol ebp,15 - add ebp,ebx - ; 27 - add ebx,eax - mov eax,ecx - sub edx,ebp - and eax,ebp - and edx,esi - or eax,edx - mov edx,DWORD [8+esp] - rol ecx,10 - lea ebx,[1518500249+eax*1+ebx] - mov eax,-1 - rol ebx,9 - add ebx,edi - ; 28 - add edi,edx - mov edx,ebp - sub eax,ebx - and edx,ebx - and eax,ecx - or edx,eax - mov eax,DWORD [56+esp] - rol ebp,10 - lea edi,[1518500249+edx*1+edi] - mov edx,-1 - rol edi,11 - add edi,esi - ; 29 - add esi,eax - mov eax,ebx - sub edx,edi - and eax,edi - and edx,ebp - or eax,edx - mov edx,DWORD [44+esp] - rol ebx,10 - lea esi,[1518500249+eax*1+esi] - mov eax,-1 - rol esi,7 - add esi,ecx - ; 30 - add ecx,edx - mov edx,edi - sub eax,esi - and edx,esi - and eax,ebx - or edx,eax - mov eax,DWORD [32+esp] - rol edi,10 - lea ecx,[1518500249+edx*1+ecx] - mov edx,-1 - rol ecx,13 - add ecx,ebp - ; 31 - add ebp,eax - mov eax,esi - sub edx,ecx - and eax,ecx - and edx,edi - or eax,edx - mov edx,-1 - rol esi,10 - lea ebp,[1518500249+eax*1+ebp] - sub edx,ecx - rol ebp,12 - add ebp,ebx - ; 32 - mov eax,DWORD [12+esp] - or edx,ebp - add ebx,eax - xor edx,esi - mov eax,-1 - rol ecx,10 - lea ebx,[1859775393+edx*1+ebx] - sub eax,ebp - rol ebx,11 - add ebx,edi - ; 33 - mov edx,DWORD [40+esp] - or eax,ebx - add edi,edx - xor eax,ecx - mov edx,-1 - rol ebp,10 - lea edi,[1859775393+eax*1+edi] - sub edx,ebx - rol edi,13 - add edi,esi - ; 34 - mov eax,DWORD [56+esp] - or edx,edi - add esi,eax - xor edx,ebp - mov eax,-1 - rol ebx,10 - lea esi,[1859775393+edx*1+esi] - sub eax,edi - rol esi,6 - add esi,ecx - ; 35 - mov edx,DWORD [16+esp] - or eax,esi - add ecx,edx - xor eax,ebx - mov edx,-1 - rol edi,10 - lea ecx,[1859775393+eax*1+ecx] - sub edx,esi - rol ecx,7 - add ecx,ebp - ; 36 - mov eax,DWORD [36+esp] - or edx,ecx - add ebp,eax - xor edx,edi - mov eax,-1 - rol esi,10 - lea ebp,[1859775393+edx*1+ebp] - sub eax,ecx - rol ebp,14 - add ebp,ebx - ; 37 - mov edx,DWORD [60+esp] - or eax,ebp - add ebx,edx - xor eax,esi - mov edx,-1 - rol ecx,10 - lea ebx,[1859775393+eax*1+ebx] - sub edx,ebp - rol ebx,9 - add ebx,edi - ; 38 - mov eax,DWORD [32+esp] - or edx,ebx - add edi,eax - xor edx,ecx - mov eax,-1 - rol ebp,10 - lea edi,[1859775393+edx*1+edi] - sub eax,ebx - rol edi,13 - add edi,esi - ; 39 - mov edx,DWORD [4+esp] - or eax,edi - add esi,edx - xor eax,ebp - mov edx,-1 - rol ebx,10 - lea esi,[1859775393+eax*1+esi] - sub edx,edi - rol esi,15 - add esi,ecx - ; 40 - mov eax,DWORD [8+esp] - or edx,esi - add ecx,eax - xor edx,ebx - mov eax,-1 - rol edi,10 - lea ecx,[1859775393+edx*1+ecx] - sub eax,esi - rol ecx,14 - add ecx,ebp - ; 41 - mov edx,DWORD [28+esp] - or eax,ecx - add ebp,edx - xor eax,edi - mov edx,-1 - rol esi,10 - lea ebp,[1859775393+eax*1+ebp] - sub edx,ecx - rol ebp,8 - add ebp,ebx - ; 42 - mov eax,DWORD [esp] - or edx,ebp - add ebx,eax - xor edx,esi - mov eax,-1 - rol ecx,10 - lea ebx,[1859775393+edx*1+ebx] - sub eax,ebp - rol ebx,13 - add ebx,edi - ; 43 - mov edx,DWORD [24+esp] - or eax,ebx - add edi,edx - xor eax,ecx - mov edx,-1 - rol ebp,10 - lea edi,[1859775393+eax*1+edi] - sub edx,ebx - rol edi,6 - add edi,esi - ; 44 - mov eax,DWORD [52+esp] - or edx,edi - add esi,eax - xor edx,ebp - mov eax,-1 - rol ebx,10 - lea esi,[1859775393+edx*1+esi] - sub eax,edi - rol esi,5 - add esi,ecx - ; 45 - mov edx,DWORD [44+esp] - or eax,esi - add ecx,edx - xor eax,ebx - mov edx,-1 - rol edi,10 - lea ecx,[1859775393+eax*1+ecx] - sub edx,esi - rol ecx,12 - add ecx,ebp - ; 46 - mov eax,DWORD [20+esp] - or edx,ecx - add ebp,eax - xor edx,edi - mov eax,-1 - rol esi,10 - lea ebp,[1859775393+edx*1+ebp] - sub eax,ecx - rol ebp,7 - add ebp,ebx - ; 47 - mov edx,DWORD [48+esp] - or eax,ebp - add ebx,edx - xor eax,esi - mov edx,-1 - rol ecx,10 - lea ebx,[1859775393+eax*1+ebx] - mov eax,ecx - rol ebx,5 - add ebx,edi - ; 48 - sub edx,ecx - and eax,ebx - and edx,ebp - or edx,eax - mov eax,DWORD [4+esp] - rol ebp,10 - lea edi,[2400959708+edx*1+edi] - mov edx,-1 - add edi,eax - mov eax,ebp - rol edi,11 - add edi,esi - ; 49 - sub edx,ebp - and eax,edi - and edx,ebx - or edx,eax - mov eax,DWORD [36+esp] - rol ebx,10 - lea esi,[2400959708+edx*1+esi] - mov edx,-1 - add esi,eax - mov eax,ebx - rol esi,12 - add esi,ecx - ; 50 - sub edx,ebx - and eax,esi - and edx,edi - or edx,eax - mov eax,DWORD [44+esp] - rol edi,10 - lea ecx,[2400959708+edx*1+ecx] - mov edx,-1 - add ecx,eax - mov eax,edi - rol ecx,14 - add ecx,ebp - ; 51 - sub edx,edi - and eax,ecx - and edx,esi - or edx,eax - mov eax,DWORD [40+esp] - rol esi,10 - lea ebp,[2400959708+edx*1+ebp] - mov edx,-1 - add ebp,eax - mov eax,esi - rol ebp,15 - add ebp,ebx - ; 52 - sub edx,esi - and eax,ebp - and edx,ecx - or edx,eax - mov eax,DWORD [esp] - rol ecx,10 - lea ebx,[2400959708+edx*1+ebx] - mov edx,-1 - add ebx,eax - mov eax,ecx - rol ebx,14 - add ebx,edi - ; 53 - sub edx,ecx - and eax,ebx - and edx,ebp - or edx,eax - mov eax,DWORD [32+esp] - rol ebp,10 - lea edi,[2400959708+edx*1+edi] - mov edx,-1 - add edi,eax - mov eax,ebp - rol edi,15 - add edi,esi - ; 54 - sub edx,ebp - and eax,edi - and edx,ebx - or edx,eax - mov eax,DWORD [48+esp] - rol ebx,10 - lea esi,[2400959708+edx*1+esi] - mov edx,-1 - add esi,eax - mov eax,ebx - rol esi,9 - add esi,ecx - ; 55 - sub edx,ebx - and eax,esi - and edx,edi - or edx,eax - mov eax,DWORD [16+esp] - rol edi,10 - lea ecx,[2400959708+edx*1+ecx] - mov edx,-1 - add ecx,eax - mov eax,edi - rol ecx,8 - add ecx,ebp - ; 56 - sub edx,edi - and eax,ecx - and edx,esi - or edx,eax - mov eax,DWORD [52+esp] - rol esi,10 - lea ebp,[2400959708+edx*1+ebp] - mov edx,-1 - add ebp,eax - mov eax,esi - rol ebp,9 - add ebp,ebx - ; 57 - sub edx,esi - and eax,ebp - and edx,ecx - or edx,eax - mov eax,DWORD [12+esp] - rol ecx,10 - lea ebx,[2400959708+edx*1+ebx] - mov edx,-1 - add ebx,eax - mov eax,ecx - rol ebx,14 - add ebx,edi - ; 58 - sub edx,ecx - and eax,ebx - and edx,ebp - or edx,eax - mov eax,DWORD [28+esp] - rol ebp,10 - lea edi,[2400959708+edx*1+edi] - mov edx,-1 - add edi,eax - mov eax,ebp - rol edi,5 - add edi,esi - ; 59 - sub edx,ebp - and eax,edi - and edx,ebx - or edx,eax - mov eax,DWORD [60+esp] - rol ebx,10 - lea esi,[2400959708+edx*1+esi] - mov edx,-1 - add esi,eax - mov eax,ebx - rol esi,6 - add esi,ecx - ; 60 - sub edx,ebx - and eax,esi - and edx,edi - or edx,eax - mov eax,DWORD [56+esp] - rol edi,10 - lea ecx,[2400959708+edx*1+ecx] - mov edx,-1 - add ecx,eax - mov eax,edi - rol ecx,8 - add ecx,ebp - ; 61 - sub edx,edi - and eax,ecx - and edx,esi - or edx,eax - mov eax,DWORD [20+esp] - rol esi,10 - lea ebp,[2400959708+edx*1+ebp] - mov edx,-1 - add ebp,eax - mov eax,esi - rol ebp,6 - add ebp,ebx - ; 62 - sub edx,esi - and eax,ebp - and edx,ecx - or edx,eax - mov eax,DWORD [24+esp] - rol ecx,10 - lea ebx,[2400959708+edx*1+ebx] - mov edx,-1 - add ebx,eax - mov eax,ecx - rol ebx,5 - add ebx,edi - ; 63 - sub edx,ecx - and eax,ebx - and edx,ebp - or edx,eax - mov eax,DWORD [8+esp] - rol ebp,10 - lea edi,[2400959708+edx*1+edi] - mov edx,-1 - add edi,eax - sub edx,ebp - rol edi,12 - add edi,esi - ; 64 - mov eax,DWORD [16+esp] - or edx,ebx - add esi,eax - xor edx,edi - mov eax,-1 - rol ebx,10 - lea esi,[2840853838+edx*1+esi] - sub eax,ebx - rol esi,9 - add esi,ecx - ; 65 - mov edx,DWORD [esp] - or eax,edi - add ecx,edx - xor eax,esi - mov edx,-1 - rol edi,10 - lea ecx,[2840853838+eax*1+ecx] - sub edx,edi - rol ecx,15 - add ecx,ebp - ; 66 - mov eax,DWORD [20+esp] - or edx,esi - add ebp,eax - xor edx,ecx - mov eax,-1 - rol esi,10 - lea ebp,[2840853838+edx*1+ebp] - sub eax,esi - rol ebp,5 - add ebp,ebx - ; 67 - mov edx,DWORD [36+esp] - or eax,ecx - add ebx,edx - xor eax,ebp - mov edx,-1 - rol ecx,10 - lea ebx,[2840853838+eax*1+ebx] - sub edx,ecx - rol ebx,11 - add ebx,edi - ; 68 - mov eax,DWORD [28+esp] - or edx,ebp - add edi,eax - xor edx,ebx - mov eax,-1 - rol ebp,10 - lea edi,[2840853838+edx*1+edi] - sub eax,ebp - rol edi,6 - add edi,esi - ; 69 - mov edx,DWORD [48+esp] - or eax,ebx - add esi,edx - xor eax,edi - mov edx,-1 - rol ebx,10 - lea esi,[2840853838+eax*1+esi] - sub edx,ebx - rol esi,8 - add esi,ecx - ; 70 - mov eax,DWORD [8+esp] - or edx,edi - add ecx,eax - xor edx,esi - mov eax,-1 - rol edi,10 - lea ecx,[2840853838+edx*1+ecx] - sub eax,edi - rol ecx,13 - add ecx,ebp - ; 71 - mov edx,DWORD [40+esp] - or eax,esi - add ebp,edx - xor eax,ecx - mov edx,-1 - rol esi,10 - lea ebp,[2840853838+eax*1+ebp] - sub edx,esi - rol ebp,12 - add ebp,ebx - ; 72 - mov eax,DWORD [56+esp] - or edx,ecx - add ebx,eax - xor edx,ebp - mov eax,-1 - rol ecx,10 - lea ebx,[2840853838+edx*1+ebx] - sub eax,ecx - rol ebx,5 - add ebx,edi - ; 73 - mov edx,DWORD [4+esp] - or eax,ebp - add edi,edx - xor eax,ebx - mov edx,-1 - rol ebp,10 - lea edi,[2840853838+eax*1+edi] - sub edx,ebp - rol edi,12 - add edi,esi - ; 74 - mov eax,DWORD [12+esp] - or edx,ebx - add esi,eax - xor edx,edi - mov eax,-1 - rol ebx,10 - lea esi,[2840853838+edx*1+esi] - sub eax,ebx - rol esi,13 - add esi,ecx - ; 75 - mov edx,DWORD [32+esp] - or eax,edi - add ecx,edx - xor eax,esi - mov edx,-1 - rol edi,10 - lea ecx,[2840853838+eax*1+ecx] - sub edx,edi - rol ecx,14 - add ecx,ebp - ; 76 - mov eax,DWORD [44+esp] - or edx,esi - add ebp,eax - xor edx,ecx - mov eax,-1 - rol esi,10 - lea ebp,[2840853838+edx*1+ebp] - sub eax,esi - rol ebp,11 - add ebp,ebx - ; 77 - mov edx,DWORD [24+esp] - or eax,ecx - add ebx,edx - xor eax,ebp - mov edx,-1 - rol ecx,10 - lea ebx,[2840853838+eax*1+ebx] - sub edx,ecx - rol ebx,8 - add ebx,edi - ; 78 - mov eax,DWORD [60+esp] - or edx,ebp - add edi,eax - xor edx,ebx - mov eax,-1 - rol ebp,10 - lea edi,[2840853838+edx*1+edi] - sub eax,ebp - rol edi,5 - add edi,esi - ; 79 - mov edx,DWORD [52+esp] - or eax,ebx - add esi,edx - xor eax,edi - mov edx,DWORD [128+esp] - rol ebx,10 - lea esi,[2840853838+eax*1+esi] - mov DWORD [64+esp],ecx - rol esi,6 - add esi,ecx - mov ecx,DWORD [edx] - mov DWORD [68+esp],esi - mov DWORD [72+esp],edi - mov esi,DWORD [4+edx] - mov DWORD [76+esp],ebx - mov edi,DWORD [8+edx] - mov DWORD [80+esp],ebp - mov ebx,DWORD [12+edx] - mov ebp,DWORD [16+edx] - ; 80 - mov edx,-1 - sub edx,ebx - mov eax,DWORD [20+esp] - or edx,edi - add ecx,eax - xor edx,esi - mov eax,-1 - rol edi,10 - lea ecx,[1352829926+edx*1+ecx] - sub eax,edi - rol ecx,8 - add ecx,ebp - ; 81 - mov edx,DWORD [56+esp] - or eax,esi - add ebp,edx - xor eax,ecx - mov edx,-1 - rol esi,10 - lea ebp,[1352829926+eax*1+ebp] - sub edx,esi - rol ebp,9 - add ebp,ebx - ; 82 - mov eax,DWORD [28+esp] - or edx,ecx - add ebx,eax - xor edx,ebp - mov eax,-1 - rol ecx,10 - lea ebx,[1352829926+edx*1+ebx] - sub eax,ecx - rol ebx,9 - add ebx,edi - ; 83 - mov edx,DWORD [esp] - or eax,ebp - add edi,edx - xor eax,ebx - mov edx,-1 - rol ebp,10 - lea edi,[1352829926+eax*1+edi] - sub edx,ebp - rol edi,11 - add edi,esi - ; 84 - mov eax,DWORD [36+esp] - or edx,ebx - add esi,eax - xor edx,edi - mov eax,-1 - rol ebx,10 - lea esi,[1352829926+edx*1+esi] - sub eax,ebx - rol esi,13 - add esi,ecx - ; 85 - mov edx,DWORD [8+esp] - or eax,edi - add ecx,edx - xor eax,esi - mov edx,-1 - rol edi,10 - lea ecx,[1352829926+eax*1+ecx] - sub edx,edi - rol ecx,15 - add ecx,ebp - ; 86 - mov eax,DWORD [44+esp] - or edx,esi - add ebp,eax - xor edx,ecx - mov eax,-1 - rol esi,10 - lea ebp,[1352829926+edx*1+ebp] - sub eax,esi - rol ebp,15 - add ebp,ebx - ; 87 - mov edx,DWORD [16+esp] - or eax,ecx - add ebx,edx - xor eax,ebp - mov edx,-1 - rol ecx,10 - lea ebx,[1352829926+eax*1+ebx] - sub edx,ecx - rol ebx,5 - add ebx,edi - ; 88 - mov eax,DWORD [52+esp] - or edx,ebp - add edi,eax - xor edx,ebx - mov eax,-1 - rol ebp,10 - lea edi,[1352829926+edx*1+edi] - sub eax,ebp - rol edi,7 - add edi,esi - ; 89 - mov edx,DWORD [24+esp] - or eax,ebx - add esi,edx - xor eax,edi - mov edx,-1 - rol ebx,10 - lea esi,[1352829926+eax*1+esi] - sub edx,ebx - rol esi,7 - add esi,ecx - ; 90 - mov eax,DWORD [60+esp] - or edx,edi - add ecx,eax - xor edx,esi - mov eax,-1 - rol edi,10 - lea ecx,[1352829926+edx*1+ecx] - sub eax,edi - rol ecx,8 - add ecx,ebp - ; 91 - mov edx,DWORD [32+esp] - or eax,esi - add ebp,edx - xor eax,ecx - mov edx,-1 - rol esi,10 - lea ebp,[1352829926+eax*1+ebp] - sub edx,esi - rol ebp,11 - add ebp,ebx - ; 92 - mov eax,DWORD [4+esp] - or edx,ecx - add ebx,eax - xor edx,ebp - mov eax,-1 - rol ecx,10 - lea ebx,[1352829926+edx*1+ebx] - sub eax,ecx - rol ebx,14 - add ebx,edi - ; 93 - mov edx,DWORD [40+esp] - or eax,ebp - add edi,edx - xor eax,ebx - mov edx,-1 - rol ebp,10 - lea edi,[1352829926+eax*1+edi] - sub edx,ebp - rol edi,14 - add edi,esi - ; 94 - mov eax,DWORD [12+esp] - or edx,ebx - add esi,eax - xor edx,edi - mov eax,-1 - rol ebx,10 - lea esi,[1352829926+edx*1+esi] - sub eax,ebx - rol esi,12 - add esi,ecx - ; 95 - mov edx,DWORD [48+esp] - or eax,edi - add ecx,edx - xor eax,esi - mov edx,-1 - rol edi,10 - lea ecx,[1352829926+eax*1+ecx] - mov eax,edi - rol ecx,6 - add ecx,ebp - ; 96 - sub edx,edi - and eax,ecx - and edx,esi - or edx,eax - mov eax,DWORD [24+esp] - rol esi,10 - lea ebp,[1548603684+edx*1+ebp] - mov edx,-1 - add ebp,eax - mov eax,esi - rol ebp,9 - add ebp,ebx - ; 97 - sub edx,esi - and eax,ebp - and edx,ecx - or edx,eax - mov eax,DWORD [44+esp] - rol ecx,10 - lea ebx,[1548603684+edx*1+ebx] - mov edx,-1 - add ebx,eax - mov eax,ecx - rol ebx,13 - add ebx,edi - ; 98 - sub edx,ecx - and eax,ebx - and edx,ebp - or edx,eax - mov eax,DWORD [12+esp] - rol ebp,10 - lea edi,[1548603684+edx*1+edi] - mov edx,-1 - add edi,eax - mov eax,ebp - rol edi,15 - add edi,esi - ; 99 - sub edx,ebp - and eax,edi - and edx,ebx - or edx,eax - mov eax,DWORD [28+esp] - rol ebx,10 - lea esi,[1548603684+edx*1+esi] - mov edx,-1 - add esi,eax - mov eax,ebx - rol esi,7 - add esi,ecx - ; 100 - sub edx,ebx - and eax,esi - and edx,edi - or edx,eax - mov eax,DWORD [esp] - rol edi,10 - lea ecx,[1548603684+edx*1+ecx] - mov edx,-1 - add ecx,eax - mov eax,edi - rol ecx,12 - add ecx,ebp - ; 101 - sub edx,edi - and eax,ecx - and edx,esi - or edx,eax - mov eax,DWORD [52+esp] - rol esi,10 - lea ebp,[1548603684+edx*1+ebp] - mov edx,-1 - add ebp,eax - mov eax,esi - rol ebp,8 - add ebp,ebx - ; 102 - sub edx,esi - and eax,ebp - and edx,ecx - or edx,eax - mov eax,DWORD [20+esp] - rol ecx,10 - lea ebx,[1548603684+edx*1+ebx] - mov edx,-1 - add ebx,eax - mov eax,ecx - rol ebx,9 - add ebx,edi - ; 103 - sub edx,ecx - and eax,ebx - and edx,ebp - or edx,eax - mov eax,DWORD [40+esp] - rol ebp,10 - lea edi,[1548603684+edx*1+edi] - mov edx,-1 - add edi,eax - mov eax,ebp - rol edi,11 - add edi,esi - ; 104 - sub edx,ebp - and eax,edi - and edx,ebx - or edx,eax - mov eax,DWORD [56+esp] - rol ebx,10 - lea esi,[1548603684+edx*1+esi] - mov edx,-1 - add esi,eax - mov eax,ebx - rol esi,7 - add esi,ecx - ; 105 - sub edx,ebx - and eax,esi - and edx,edi - or edx,eax - mov eax,DWORD [60+esp] - rol edi,10 - lea ecx,[1548603684+edx*1+ecx] - mov edx,-1 - add ecx,eax - mov eax,edi - rol ecx,7 - add ecx,ebp - ; 106 - sub edx,edi - and eax,ecx - and edx,esi - or edx,eax - mov eax,DWORD [32+esp] - rol esi,10 - lea ebp,[1548603684+edx*1+ebp] - mov edx,-1 - add ebp,eax - mov eax,esi - rol ebp,12 - add ebp,ebx - ; 107 - sub edx,esi - and eax,ebp - and edx,ecx - or edx,eax - mov eax,DWORD [48+esp] - rol ecx,10 - lea ebx,[1548603684+edx*1+ebx] - mov edx,-1 - add ebx,eax - mov eax,ecx - rol ebx,7 - add ebx,edi - ; 108 - sub edx,ecx - and eax,ebx - and edx,ebp - or edx,eax - mov eax,DWORD [16+esp] - rol ebp,10 - lea edi,[1548603684+edx*1+edi] - mov edx,-1 - add edi,eax - mov eax,ebp - rol edi,6 - add edi,esi - ; 109 - sub edx,ebp - and eax,edi - and edx,ebx - or edx,eax - mov eax,DWORD [36+esp] - rol ebx,10 - lea esi,[1548603684+edx*1+esi] - mov edx,-1 - add esi,eax - mov eax,ebx - rol esi,15 - add esi,ecx - ; 110 - sub edx,ebx - and eax,esi - and edx,edi - or edx,eax - mov eax,DWORD [4+esp] - rol edi,10 - lea ecx,[1548603684+edx*1+ecx] - mov edx,-1 - add ecx,eax - mov eax,edi - rol ecx,13 - add ecx,ebp - ; 111 - sub edx,edi - and eax,ecx - and edx,esi - or edx,eax - mov eax,DWORD [8+esp] - rol esi,10 - lea ebp,[1548603684+edx*1+ebp] - mov edx,-1 - add ebp,eax - sub edx,ecx - rol ebp,11 - add ebp,ebx - ; 112 - mov eax,DWORD [60+esp] - or edx,ebp - add ebx,eax - xor edx,esi - mov eax,-1 - rol ecx,10 - lea ebx,[1836072691+edx*1+ebx] - sub eax,ebp - rol ebx,9 - add ebx,edi - ; 113 - mov edx,DWORD [20+esp] - or eax,ebx - add edi,edx - xor eax,ecx - mov edx,-1 - rol ebp,10 - lea edi,[1836072691+eax*1+edi] - sub edx,ebx - rol edi,7 - add edi,esi - ; 114 - mov eax,DWORD [4+esp] - or edx,edi - add esi,eax - xor edx,ebp - mov eax,-1 - rol ebx,10 - lea esi,[1836072691+edx*1+esi] - sub eax,edi - rol esi,15 - add esi,ecx - ; 115 - mov edx,DWORD [12+esp] - or eax,esi - add ecx,edx - xor eax,ebx - mov edx,-1 - rol edi,10 - lea ecx,[1836072691+eax*1+ecx] - sub edx,esi - rol ecx,11 - add ecx,ebp - ; 116 - mov eax,DWORD [28+esp] - or edx,ecx - add ebp,eax - xor edx,edi - mov eax,-1 - rol esi,10 - lea ebp,[1836072691+edx*1+ebp] - sub eax,ecx - rol ebp,8 - add ebp,ebx - ; 117 - mov edx,DWORD [56+esp] - or eax,ebp - add ebx,edx - xor eax,esi - mov edx,-1 - rol ecx,10 - lea ebx,[1836072691+eax*1+ebx] - sub edx,ebp - rol ebx,6 - add ebx,edi - ; 118 - mov eax,DWORD [24+esp] - or edx,ebx - add edi,eax - xor edx,ecx - mov eax,-1 - rol ebp,10 - lea edi,[1836072691+edx*1+edi] - sub eax,ebx - rol edi,6 - add edi,esi - ; 119 - mov edx,DWORD [36+esp] - or eax,edi - add esi,edx - xor eax,ebp - mov edx,-1 - rol ebx,10 - lea esi,[1836072691+eax*1+esi] - sub edx,edi - rol esi,14 - add esi,ecx - ; 120 - mov eax,DWORD [44+esp] - or edx,esi - add ecx,eax - xor edx,ebx - mov eax,-1 - rol edi,10 - lea ecx,[1836072691+edx*1+ecx] - sub eax,esi - rol ecx,12 - add ecx,ebp - ; 121 - mov edx,DWORD [32+esp] - or eax,ecx - add ebp,edx - xor eax,edi - mov edx,-1 - rol esi,10 - lea ebp,[1836072691+eax*1+ebp] - sub edx,ecx - rol ebp,13 - add ebp,ebx - ; 122 - mov eax,DWORD [48+esp] - or edx,ebp - add ebx,eax - xor edx,esi - mov eax,-1 - rol ecx,10 - lea ebx,[1836072691+edx*1+ebx] - sub eax,ebp - rol ebx,5 - add ebx,edi - ; 123 - mov edx,DWORD [8+esp] - or eax,ebx - add edi,edx - xor eax,ecx - mov edx,-1 - rol ebp,10 - lea edi,[1836072691+eax*1+edi] - sub edx,ebx - rol edi,14 - add edi,esi - ; 124 - mov eax,DWORD [40+esp] - or edx,edi - add esi,eax - xor edx,ebp - mov eax,-1 - rol ebx,10 - lea esi,[1836072691+edx*1+esi] - sub eax,edi - rol esi,13 - add esi,ecx - ; 125 - mov edx,DWORD [esp] - or eax,esi - add ecx,edx - xor eax,ebx - mov edx,-1 - rol edi,10 - lea ecx,[1836072691+eax*1+ecx] - sub edx,esi - rol ecx,13 - add ecx,ebp - ; 126 - mov eax,DWORD [16+esp] - or edx,ecx - add ebp,eax - xor edx,edi - mov eax,-1 - rol esi,10 - lea ebp,[1836072691+edx*1+ebp] - sub eax,ecx - rol ebp,7 - add ebp,ebx - ; 127 - mov edx,DWORD [52+esp] - or eax,ebp - add ebx,edx - xor eax,esi - mov edx,DWORD [32+esp] - rol ecx,10 - lea ebx,[1836072691+eax*1+ebx] - mov eax,-1 - rol ebx,5 - add ebx,edi - ; 128 - add edi,edx - mov edx,ebp - sub eax,ebx - and edx,ebx - and eax,ecx - or edx,eax - mov eax,DWORD [24+esp] - rol ebp,10 - lea edi,[2053994217+edx*1+edi] - mov edx,-1 - rol edi,15 - add edi,esi - ; 129 - add esi,eax - mov eax,ebx - sub edx,edi - and eax,edi - and edx,ebp - or eax,edx - mov edx,DWORD [16+esp] - rol ebx,10 - lea esi,[2053994217+eax*1+esi] - mov eax,-1 - rol esi,5 - add esi,ecx - ; 130 - add ecx,edx - mov edx,edi - sub eax,esi - and edx,esi - and eax,ebx - or edx,eax - mov eax,DWORD [4+esp] - rol edi,10 - lea ecx,[2053994217+edx*1+ecx] - mov edx,-1 - rol ecx,8 - add ecx,ebp - ; 131 - add ebp,eax - mov eax,esi - sub edx,ecx - and eax,ecx - and edx,edi - or eax,edx - mov edx,DWORD [12+esp] - rol esi,10 - lea ebp,[2053994217+eax*1+ebp] - mov eax,-1 - rol ebp,11 - add ebp,ebx - ; 132 - add ebx,edx - mov edx,ecx - sub eax,ebp - and edx,ebp - and eax,esi - or edx,eax - mov eax,DWORD [44+esp] - rol ecx,10 - lea ebx,[2053994217+edx*1+ebx] - mov edx,-1 - rol ebx,14 - add ebx,edi - ; 133 - add edi,eax - mov eax,ebp - sub edx,ebx - and eax,ebx - and edx,ecx - or eax,edx - mov edx,DWORD [60+esp] - rol ebp,10 - lea edi,[2053994217+eax*1+edi] - mov eax,-1 - rol edi,14 - add edi,esi - ; 134 - add esi,edx - mov edx,ebx - sub eax,edi - and edx,edi - and eax,ebp - or edx,eax - mov eax,DWORD [esp] - rol ebx,10 - lea esi,[2053994217+edx*1+esi] - mov edx,-1 - rol esi,6 - add esi,ecx - ; 135 - add ecx,eax - mov eax,edi - sub edx,esi - and eax,esi - and edx,ebx - or eax,edx - mov edx,DWORD [20+esp] - rol edi,10 - lea ecx,[2053994217+eax*1+ecx] - mov eax,-1 - rol ecx,14 - add ecx,ebp - ; 136 - add ebp,edx - mov edx,esi - sub eax,ecx - and edx,ecx - and eax,edi - or edx,eax - mov eax,DWORD [48+esp] - rol esi,10 - lea ebp,[2053994217+edx*1+ebp] - mov edx,-1 - rol ebp,6 - add ebp,ebx - ; 137 - add ebx,eax - mov eax,ecx - sub edx,ebp - and eax,ebp - and edx,esi - or eax,edx - mov edx,DWORD [8+esp] - rol ecx,10 - lea ebx,[2053994217+eax*1+ebx] - mov eax,-1 - rol ebx,9 - add ebx,edi - ; 138 - add edi,edx - mov edx,ebp - sub eax,ebx - and edx,ebx - and eax,ecx - or edx,eax - mov eax,DWORD [52+esp] - rol ebp,10 - lea edi,[2053994217+edx*1+edi] - mov edx,-1 - rol edi,12 - add edi,esi - ; 139 - add esi,eax - mov eax,ebx - sub edx,edi - and eax,edi - and edx,ebp - or eax,edx - mov edx,DWORD [36+esp] - rol ebx,10 - lea esi,[2053994217+eax*1+esi] - mov eax,-1 - rol esi,9 - add esi,ecx - ; 140 - add ecx,edx - mov edx,edi - sub eax,esi - and edx,esi - and eax,ebx - or edx,eax - mov eax,DWORD [28+esp] - rol edi,10 - lea ecx,[2053994217+edx*1+ecx] - mov edx,-1 - rol ecx,12 - add ecx,ebp - ; 141 - add ebp,eax - mov eax,esi - sub edx,ecx - and eax,ecx - and edx,edi - or eax,edx - mov edx,DWORD [40+esp] - rol esi,10 - lea ebp,[2053994217+eax*1+ebp] - mov eax,-1 - rol ebp,5 - add ebp,ebx - ; 142 - add ebx,edx - mov edx,ecx - sub eax,ebp - and edx,ebp - and eax,esi - or edx,eax - mov eax,DWORD [56+esp] - rol ecx,10 - lea ebx,[2053994217+edx*1+ebx] - mov edx,-1 - rol ebx,15 - add ebx,edi - ; 143 - add edi,eax - mov eax,ebp - sub edx,ebx - and eax,ebx - and edx,ecx - or edx,eax - mov eax,ebx - rol ebp,10 - lea edi,[2053994217+edx*1+edi] - xor eax,ebp - rol edi,8 - add edi,esi - ; 144 - mov edx,DWORD [48+esp] - xor eax,edi - add esi,edx - rol ebx,10 - add esi,eax - mov eax,edi - rol esi,8 - add esi,ecx - ; 145 - xor eax,ebx - mov edx,DWORD [60+esp] - xor eax,esi - add ecx,eax - mov eax,esi - rol edi,10 - add ecx,edx - xor eax,edi - rol ecx,5 - add ecx,ebp - ; 146 - mov edx,DWORD [40+esp] - xor eax,ecx - add ebp,edx - rol esi,10 - add ebp,eax - mov eax,ecx - rol ebp,12 - add ebp,ebx - ; 147 - xor eax,esi - mov edx,DWORD [16+esp] - xor eax,ebp - add ebx,eax - mov eax,ebp - rol ecx,10 - add ebx,edx - xor eax,ecx - rol ebx,9 - add ebx,edi - ; 148 - mov edx,DWORD [4+esp] - xor eax,ebx - add edi,edx - rol ebp,10 - add edi,eax - mov eax,ebx - rol edi,12 - add edi,esi - ; 149 - xor eax,ebp - mov edx,DWORD [20+esp] - xor eax,edi - add esi,eax - mov eax,edi - rol ebx,10 - add esi,edx - xor eax,ebx - rol esi,5 - add esi,ecx - ; 150 - mov edx,DWORD [32+esp] - xor eax,esi - add ecx,edx - rol edi,10 - add ecx,eax - mov eax,esi - rol ecx,14 - add ecx,ebp - ; 151 - xor eax,edi - mov edx,DWORD [28+esp] - xor eax,ecx - add ebp,eax - mov eax,ecx - rol esi,10 - add ebp,edx - xor eax,esi - rol ebp,6 - add ebp,ebx - ; 152 - mov edx,DWORD [24+esp] - xor eax,ebp - add ebx,edx - rol ecx,10 - add ebx,eax - mov eax,ebp - rol ebx,8 - add ebx,edi - ; 153 - xor eax,ecx - mov edx,DWORD [8+esp] - xor eax,ebx - add edi,eax - mov eax,ebx - rol ebp,10 - add edi,edx - xor eax,ebp - rol edi,13 - add edi,esi - ; 154 - mov edx,DWORD [52+esp] - xor eax,edi - add esi,edx - rol ebx,10 - add esi,eax - mov eax,edi - rol esi,6 - add esi,ecx - ; 155 - xor eax,ebx - mov edx,DWORD [56+esp] - xor eax,esi - add ecx,eax - mov eax,esi - rol edi,10 - add ecx,edx - xor eax,edi - rol ecx,5 - add ecx,ebp - ; 156 - mov edx,DWORD [esp] - xor eax,ecx - add ebp,edx - rol esi,10 - add ebp,eax - mov eax,ecx - rol ebp,15 - add ebp,ebx - ; 157 - xor eax,esi - mov edx,DWORD [12+esp] - xor eax,ebp - add ebx,eax - mov eax,ebp - rol ecx,10 - add ebx,edx - xor eax,ecx - rol ebx,13 - add ebx,edi - ; 158 - mov edx,DWORD [36+esp] - xor eax,ebx - add edi,edx - rol ebp,10 - add edi,eax - mov eax,ebx - rol edi,11 - add edi,esi - ; 159 - xor eax,ebp - mov edx,DWORD [44+esp] - xor eax,edi - add esi,eax - rol ebx,10 - add esi,edx - mov edx,DWORD [128+esp] - rol esi,11 - add esi,ecx - mov eax,DWORD [4+edx] - add ebx,eax - mov eax,DWORD [72+esp] - add ebx,eax - mov eax,DWORD [8+edx] - add ebp,eax - mov eax,DWORD [76+esp] - add ebp,eax - mov eax,DWORD [12+edx] - add ecx,eax - mov eax,DWORD [80+esp] - add ecx,eax - mov eax,DWORD [16+edx] - add esi,eax - mov eax,DWORD [64+esp] - add esi,eax - mov eax,DWORD [edx] - add edi,eax - mov eax,DWORD [68+esp] - add edi,eax - mov eax,DWORD [136+esp] - mov DWORD [edx],ebx - mov DWORD [4+edx],ebp - mov DWORD [8+edx],ecx - sub eax,1 - mov DWORD [12+edx],esi - mov DWORD [16+edx],edi - jle NEAR L$001get_out - mov DWORD [136+esp],eax - mov edi,ecx - mov eax,DWORD [132+esp] - mov ecx,ebx - add eax,64 - mov esi,ebp - mov DWORD [132+esp],eax - jmp NEAR L$000start -L$001get_out: - add esp,108 - pop ebx - pop ebp - pop edi - pop esi - ret diff --git a/openssl/src/crypto/ripemd/rmd_dgst.c b/openssl/src/crypto/ripemd/rmd_dgst.c deleted file mode 100644 index 77b9d32d2..000000000 --- a/openssl/src/crypto/ripemd/rmd_dgst.c +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RIPEMD160 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "rmd_local.h" -#include - -#ifdef RMD160_ASM -void ripemd160_block_x86(RIPEMD160_CTX *c, unsigned long *p, size_t num); -# define ripemd160_block ripemd160_block_x86 -#else -void ripemd160_block(RIPEMD160_CTX *c, unsigned long *p, size_t num); -#endif - -int RIPEMD160_Init(RIPEMD160_CTX *c) -{ - memset(c, 0, sizeof(*c)); - c->A = RIPEMD160_A; - c->B = RIPEMD160_B; - c->C = RIPEMD160_C; - c->D = RIPEMD160_D; - c->E = RIPEMD160_E; - return 1; -} - -#ifndef ripemd160_block_data_order -# ifdef X -# undef X -# endif -void ripemd160_block_data_order(RIPEMD160_CTX *ctx, const void *p, size_t num) -{ - const unsigned char *data = p; - register unsigned MD32_REG_T A, B, C, D, E; - unsigned MD32_REG_T a, b, c, d, e, l; -# ifndef MD32_XARRAY - /* See comment in crypto/sha/sha_local.h for details. */ - unsigned MD32_REG_T XX0, XX1, XX2, XX3, XX4, XX5, XX6, XX7, - XX8, XX9, XX10, XX11, XX12, XX13, XX14, XX15; -# define X(i) XX##i -# else - RIPEMD160_LONG XX[16]; -# define X(i) XX[i] -# endif - - for (; num--;) { - - A = ctx->A; - B = ctx->B; - C = ctx->C; - D = ctx->D; - E = ctx->E; - - (void)HOST_c2l(data, l); - X(0) = l; - (void)HOST_c2l(data, l); - X(1) = l; - RIP1(A, B, C, D, E, WL00, SL00); - (void)HOST_c2l(data, l); - X(2) = l; - RIP1(E, A, B, C, D, WL01, SL01); - (void)HOST_c2l(data, l); - X(3) = l; - RIP1(D, E, A, B, C, WL02, SL02); - (void)HOST_c2l(data, l); - X(4) = l; - RIP1(C, D, E, A, B, WL03, SL03); - (void)HOST_c2l(data, l); - X(5) = l; - RIP1(B, C, D, E, A, WL04, SL04); - (void)HOST_c2l(data, l); - X(6) = l; - RIP1(A, B, C, D, E, WL05, SL05); - (void)HOST_c2l(data, l); - X(7) = l; - RIP1(E, A, B, C, D, WL06, SL06); - (void)HOST_c2l(data, l); - X(8) = l; - RIP1(D, E, A, B, C, WL07, SL07); - (void)HOST_c2l(data, l); - X(9) = l; - RIP1(C, D, E, A, B, WL08, SL08); - (void)HOST_c2l(data, l); - X(10) = l; - RIP1(B, C, D, E, A, WL09, SL09); - (void)HOST_c2l(data, l); - X(11) = l; - RIP1(A, B, C, D, E, WL10, SL10); - (void)HOST_c2l(data, l); - X(12) = l; - RIP1(E, A, B, C, D, WL11, SL11); - (void)HOST_c2l(data, l); - X(13) = l; - RIP1(D, E, A, B, C, WL12, SL12); - (void)HOST_c2l(data, l); - X(14) = l; - RIP1(C, D, E, A, B, WL13, SL13); - (void)HOST_c2l(data, l); - X(15) = l; - RIP1(B, C, D, E, A, WL14, SL14); - RIP1(A, B, C, D, E, WL15, SL15); - - RIP2(E, A, B, C, D, WL16, SL16, KL1); - RIP2(D, E, A, B, C, WL17, SL17, KL1); - RIP2(C, D, E, A, B, WL18, SL18, KL1); - RIP2(B, C, D, E, A, WL19, SL19, KL1); - RIP2(A, B, C, D, E, WL20, SL20, KL1); - RIP2(E, A, B, C, D, WL21, SL21, KL1); - RIP2(D, E, A, B, C, WL22, SL22, KL1); - RIP2(C, D, E, A, B, WL23, SL23, KL1); - RIP2(B, C, D, E, A, WL24, SL24, KL1); - RIP2(A, B, C, D, E, WL25, SL25, KL1); - RIP2(E, A, B, C, D, WL26, SL26, KL1); - RIP2(D, E, A, B, C, WL27, SL27, KL1); - RIP2(C, D, E, A, B, WL28, SL28, KL1); - RIP2(B, C, D, E, A, WL29, SL29, KL1); - RIP2(A, B, C, D, E, WL30, SL30, KL1); - RIP2(E, A, B, C, D, WL31, SL31, KL1); - - RIP3(D, E, A, B, C, WL32, SL32, KL2); - RIP3(C, D, E, A, B, WL33, SL33, KL2); - RIP3(B, C, D, E, A, WL34, SL34, KL2); - RIP3(A, B, C, D, E, WL35, SL35, KL2); - RIP3(E, A, B, C, D, WL36, SL36, KL2); - RIP3(D, E, A, B, C, WL37, SL37, KL2); - RIP3(C, D, E, A, B, WL38, SL38, KL2); - RIP3(B, C, D, E, A, WL39, SL39, KL2); - RIP3(A, B, C, D, E, WL40, SL40, KL2); - RIP3(E, A, B, C, D, WL41, SL41, KL2); - RIP3(D, E, A, B, C, WL42, SL42, KL2); - RIP3(C, D, E, A, B, WL43, SL43, KL2); - RIP3(B, C, D, E, A, WL44, SL44, KL2); - RIP3(A, B, C, D, E, WL45, SL45, KL2); - RIP3(E, A, B, C, D, WL46, SL46, KL2); - RIP3(D, E, A, B, C, WL47, SL47, KL2); - - RIP4(C, D, E, A, B, WL48, SL48, KL3); - RIP4(B, C, D, E, A, WL49, SL49, KL3); - RIP4(A, B, C, D, E, WL50, SL50, KL3); - RIP4(E, A, B, C, D, WL51, SL51, KL3); - RIP4(D, E, A, B, C, WL52, SL52, KL3); - RIP4(C, D, E, A, B, WL53, SL53, KL3); - RIP4(B, C, D, E, A, WL54, SL54, KL3); - RIP4(A, B, C, D, E, WL55, SL55, KL3); - RIP4(E, A, B, C, D, WL56, SL56, KL3); - RIP4(D, E, A, B, C, WL57, SL57, KL3); - RIP4(C, D, E, A, B, WL58, SL58, KL3); - RIP4(B, C, D, E, A, WL59, SL59, KL3); - RIP4(A, B, C, D, E, WL60, SL60, KL3); - RIP4(E, A, B, C, D, WL61, SL61, KL3); - RIP4(D, E, A, B, C, WL62, SL62, KL3); - RIP4(C, D, E, A, B, WL63, SL63, KL3); - - RIP5(B, C, D, E, A, WL64, SL64, KL4); - RIP5(A, B, C, D, E, WL65, SL65, KL4); - RIP5(E, A, B, C, D, WL66, SL66, KL4); - RIP5(D, E, A, B, C, WL67, SL67, KL4); - RIP5(C, D, E, A, B, WL68, SL68, KL4); - RIP5(B, C, D, E, A, WL69, SL69, KL4); - RIP5(A, B, C, D, E, WL70, SL70, KL4); - RIP5(E, A, B, C, D, WL71, SL71, KL4); - RIP5(D, E, A, B, C, WL72, SL72, KL4); - RIP5(C, D, E, A, B, WL73, SL73, KL4); - RIP5(B, C, D, E, A, WL74, SL74, KL4); - RIP5(A, B, C, D, E, WL75, SL75, KL4); - RIP5(E, A, B, C, D, WL76, SL76, KL4); - RIP5(D, E, A, B, C, WL77, SL77, KL4); - RIP5(C, D, E, A, B, WL78, SL78, KL4); - RIP5(B, C, D, E, A, WL79, SL79, KL4); - - a = A; - b = B; - c = C; - d = D; - e = E; - /* Do other half */ - A = ctx->A; - B = ctx->B; - C = ctx->C; - D = ctx->D; - E = ctx->E; - - RIP5(A, B, C, D, E, WR00, SR00, KR0); - RIP5(E, A, B, C, D, WR01, SR01, KR0); - RIP5(D, E, A, B, C, WR02, SR02, KR0); - RIP5(C, D, E, A, B, WR03, SR03, KR0); - RIP5(B, C, D, E, A, WR04, SR04, KR0); - RIP5(A, B, C, D, E, WR05, SR05, KR0); - RIP5(E, A, B, C, D, WR06, SR06, KR0); - RIP5(D, E, A, B, C, WR07, SR07, KR0); - RIP5(C, D, E, A, B, WR08, SR08, KR0); - RIP5(B, C, D, E, A, WR09, SR09, KR0); - RIP5(A, B, C, D, E, WR10, SR10, KR0); - RIP5(E, A, B, C, D, WR11, SR11, KR0); - RIP5(D, E, A, B, C, WR12, SR12, KR0); - RIP5(C, D, E, A, B, WR13, SR13, KR0); - RIP5(B, C, D, E, A, WR14, SR14, KR0); - RIP5(A, B, C, D, E, WR15, SR15, KR0); - - RIP4(E, A, B, C, D, WR16, SR16, KR1); - RIP4(D, E, A, B, C, WR17, SR17, KR1); - RIP4(C, D, E, A, B, WR18, SR18, KR1); - RIP4(B, C, D, E, A, WR19, SR19, KR1); - RIP4(A, B, C, D, E, WR20, SR20, KR1); - RIP4(E, A, B, C, D, WR21, SR21, KR1); - RIP4(D, E, A, B, C, WR22, SR22, KR1); - RIP4(C, D, E, A, B, WR23, SR23, KR1); - RIP4(B, C, D, E, A, WR24, SR24, KR1); - RIP4(A, B, C, D, E, WR25, SR25, KR1); - RIP4(E, A, B, C, D, WR26, SR26, KR1); - RIP4(D, E, A, B, C, WR27, SR27, KR1); - RIP4(C, D, E, A, B, WR28, SR28, KR1); - RIP4(B, C, D, E, A, WR29, SR29, KR1); - RIP4(A, B, C, D, E, WR30, SR30, KR1); - RIP4(E, A, B, C, D, WR31, SR31, KR1); - - RIP3(D, E, A, B, C, WR32, SR32, KR2); - RIP3(C, D, E, A, B, WR33, SR33, KR2); - RIP3(B, C, D, E, A, WR34, SR34, KR2); - RIP3(A, B, C, D, E, WR35, SR35, KR2); - RIP3(E, A, B, C, D, WR36, SR36, KR2); - RIP3(D, E, A, B, C, WR37, SR37, KR2); - RIP3(C, D, E, A, B, WR38, SR38, KR2); - RIP3(B, C, D, E, A, WR39, SR39, KR2); - RIP3(A, B, C, D, E, WR40, SR40, KR2); - RIP3(E, A, B, C, D, WR41, SR41, KR2); - RIP3(D, E, A, B, C, WR42, SR42, KR2); - RIP3(C, D, E, A, B, WR43, SR43, KR2); - RIP3(B, C, D, E, A, WR44, SR44, KR2); - RIP3(A, B, C, D, E, WR45, SR45, KR2); - RIP3(E, A, B, C, D, WR46, SR46, KR2); - RIP3(D, E, A, B, C, WR47, SR47, KR2); - - RIP2(C, D, E, A, B, WR48, SR48, KR3); - RIP2(B, C, D, E, A, WR49, SR49, KR3); - RIP2(A, B, C, D, E, WR50, SR50, KR3); - RIP2(E, A, B, C, D, WR51, SR51, KR3); - RIP2(D, E, A, B, C, WR52, SR52, KR3); - RIP2(C, D, E, A, B, WR53, SR53, KR3); - RIP2(B, C, D, E, A, WR54, SR54, KR3); - RIP2(A, B, C, D, E, WR55, SR55, KR3); - RIP2(E, A, B, C, D, WR56, SR56, KR3); - RIP2(D, E, A, B, C, WR57, SR57, KR3); - RIP2(C, D, E, A, B, WR58, SR58, KR3); - RIP2(B, C, D, E, A, WR59, SR59, KR3); - RIP2(A, B, C, D, E, WR60, SR60, KR3); - RIP2(E, A, B, C, D, WR61, SR61, KR3); - RIP2(D, E, A, B, C, WR62, SR62, KR3); - RIP2(C, D, E, A, B, WR63, SR63, KR3); - - RIP1(B, C, D, E, A, WR64, SR64); - RIP1(A, B, C, D, E, WR65, SR65); - RIP1(E, A, B, C, D, WR66, SR66); - RIP1(D, E, A, B, C, WR67, SR67); - RIP1(C, D, E, A, B, WR68, SR68); - RIP1(B, C, D, E, A, WR69, SR69); - RIP1(A, B, C, D, E, WR70, SR70); - RIP1(E, A, B, C, D, WR71, SR71); - RIP1(D, E, A, B, C, WR72, SR72); - RIP1(C, D, E, A, B, WR73, SR73); - RIP1(B, C, D, E, A, WR74, SR74); - RIP1(A, B, C, D, E, WR75, SR75); - RIP1(E, A, B, C, D, WR76, SR76); - RIP1(D, E, A, B, C, WR77, SR77); - RIP1(C, D, E, A, B, WR78, SR78); - RIP1(B, C, D, E, A, WR79, SR79); - - D = ctx->B + c + D; - ctx->B = ctx->C + d + E; - ctx->C = ctx->D + e + A; - ctx->D = ctx->E + a + B; - ctx->E = ctx->A + b + C; - ctx->A = D; - - } -} -#endif diff --git a/openssl/src/crypto/ripemd/rmd_local.h b/openssl/src/crypto/ripemd/rmd_local.h deleted file mode 100644 index 325cb98cc..000000000 --- a/openssl/src/crypto/ripemd/rmd_local.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include - -/* - * DO EXAMINE COMMENTS IN crypto/md5/md5_local.h & crypto/md5/md5_dgst.c - * FOR EXPLANATIONS ON FOLLOWING "CODE." - */ -#ifdef RMD160_ASM -# if defined(__i386) || defined(__i386__) || defined(_M_IX86) -# define ripemd160_block_data_order ripemd160_block_asm_data_order -# endif -#endif - -void ripemd160_block_data_order(RIPEMD160_CTX *c, const void *p, size_t num); - -#define DATA_ORDER_IS_LITTLE_ENDIAN - -#define HASH_LONG RIPEMD160_LONG -#define HASH_CTX RIPEMD160_CTX -#define HASH_CBLOCK RIPEMD160_CBLOCK -#define HASH_UPDATE RIPEMD160_Update -#define HASH_TRANSFORM RIPEMD160_Transform -#define HASH_FINAL RIPEMD160_Final -#define HASH_MAKE_STRING(c,s) do { \ - unsigned long ll; \ - ll=(c)->A; (void)HOST_l2c(ll,(s)); \ - ll=(c)->B; (void)HOST_l2c(ll,(s)); \ - ll=(c)->C; (void)HOST_l2c(ll,(s)); \ - ll=(c)->D; (void)HOST_l2c(ll,(s)); \ - ll=(c)->E; (void)HOST_l2c(ll,(s)); \ - } while (0) -#define HASH_BLOCK_DATA_ORDER ripemd160_block_data_order - -#include "crypto/md32_common.h" - -/* - * Transformed F2 and F4 are courtesy of Wei Dai - */ -#define F1(x,y,z) ((x) ^ (y) ^ (z)) -#define F2(x,y,z) ((((y) ^ (z)) & (x)) ^ (z)) -#define F3(x,y,z) (((~(y)) | (x)) ^ (z)) -#define F4(x,y,z) ((((x) ^ (y)) & (z)) ^ (y)) -#define F5(x,y,z) (((~(z)) | (y)) ^ (x)) - -#define RIPEMD160_A 0x67452301L -#define RIPEMD160_B 0xEFCDAB89L -#define RIPEMD160_C 0x98BADCFEL -#define RIPEMD160_D 0x10325476L -#define RIPEMD160_E 0xC3D2E1F0L - -#include "rmdconst.h" - -#define RIP1(a,b,c,d,e,w,s) { \ - a+=F1(b,c,d)+X(w); \ - a=ROTATE(a,s)+e; \ - c=ROTATE(c,10); } - -#define RIP2(a,b,c,d,e,w,s,K) { \ - a+=F2(b,c,d)+X(w)+K; \ - a=ROTATE(a,s)+e; \ - c=ROTATE(c,10); } - -#define RIP3(a,b,c,d,e,w,s,K) { \ - a+=F3(b,c,d)+X(w)+K; \ - a=ROTATE(a,s)+e; \ - c=ROTATE(c,10); } - -#define RIP4(a,b,c,d,e,w,s,K) { \ - a+=F4(b,c,d)+X(w)+K; \ - a=ROTATE(a,s)+e; \ - c=ROTATE(c,10); } - -#define RIP5(a,b,c,d,e,w,s,K) { \ - a+=F5(b,c,d)+X(w)+K; \ - a=ROTATE(a,s)+e; \ - c=ROTATE(c,10); } diff --git a/openssl/src/crypto/ripemd/rmd_one.c b/openssl/src/crypto/ripemd/rmd_one.c deleted file mode 100644 index dcd7bae1d..000000000 --- a/openssl/src/crypto/ripemd/rmd_one.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RIPEMD160 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include - -unsigned char *RIPEMD160(const unsigned char *d, size_t n, unsigned char *md) -{ - RIPEMD160_CTX c; - static unsigned char m[RIPEMD160_DIGEST_LENGTH]; - - if (md == NULL) - md = m; - if (!RIPEMD160_Init(&c)) - return NULL; - RIPEMD160_Update(&c, d, n); - RIPEMD160_Final(md, &c); - OPENSSL_cleanse(&c, sizeof(c)); /* security consideration */ - return md; -} diff --git a/openssl/src/crypto/ripemd/rmdconst.h b/openssl/src/crypto/ripemd/rmdconst.h deleted file mode 100644 index 0e19410ad..000000000 --- a/openssl/src/crypto/ripemd/rmdconst.h +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#define KL0 0x00000000L -#define KL1 0x5A827999L -#define KL2 0x6ED9EBA1L -#define KL3 0x8F1BBCDCL -#define KL4 0xA953FD4EL - -#define KR0 0x50A28BE6L -#define KR1 0x5C4DD124L -#define KR2 0x6D703EF3L -#define KR3 0x7A6D76E9L -#define KR4 0x00000000L - -#define WL00 0 -#define SL00 11 -#define WL01 1 -#define SL01 14 -#define WL02 2 -#define SL02 15 -#define WL03 3 -#define SL03 12 -#define WL04 4 -#define SL04 5 -#define WL05 5 -#define SL05 8 -#define WL06 6 -#define SL06 7 -#define WL07 7 -#define SL07 9 -#define WL08 8 -#define SL08 11 -#define WL09 9 -#define SL09 13 -#define WL10 10 -#define SL10 14 -#define WL11 11 -#define SL11 15 -#define WL12 12 -#define SL12 6 -#define WL13 13 -#define SL13 7 -#define WL14 14 -#define SL14 9 -#define WL15 15 -#define SL15 8 - -#define WL16 7 -#define SL16 7 -#define WL17 4 -#define SL17 6 -#define WL18 13 -#define SL18 8 -#define WL19 1 -#define SL19 13 -#define WL20 10 -#define SL20 11 -#define WL21 6 -#define SL21 9 -#define WL22 15 -#define SL22 7 -#define WL23 3 -#define SL23 15 -#define WL24 12 -#define SL24 7 -#define WL25 0 -#define SL25 12 -#define WL26 9 -#define SL26 15 -#define WL27 5 -#define SL27 9 -#define WL28 2 -#define SL28 11 -#define WL29 14 -#define SL29 7 -#define WL30 11 -#define SL30 13 -#define WL31 8 -#define SL31 12 - -#define WL32 3 -#define SL32 11 -#define WL33 10 -#define SL33 13 -#define WL34 14 -#define SL34 6 -#define WL35 4 -#define SL35 7 -#define WL36 9 -#define SL36 14 -#define WL37 15 -#define SL37 9 -#define WL38 8 -#define SL38 13 -#define WL39 1 -#define SL39 15 -#define WL40 2 -#define SL40 14 -#define WL41 7 -#define SL41 8 -#define WL42 0 -#define SL42 13 -#define WL43 6 -#define SL43 6 -#define WL44 13 -#define SL44 5 -#define WL45 11 -#define SL45 12 -#define WL46 5 -#define SL46 7 -#define WL47 12 -#define SL47 5 - -#define WL48 1 -#define SL48 11 -#define WL49 9 -#define SL49 12 -#define WL50 11 -#define SL50 14 -#define WL51 10 -#define SL51 15 -#define WL52 0 -#define SL52 14 -#define WL53 8 -#define SL53 15 -#define WL54 12 -#define SL54 9 -#define WL55 4 -#define SL55 8 -#define WL56 13 -#define SL56 9 -#define WL57 3 -#define SL57 14 -#define WL58 7 -#define SL58 5 -#define WL59 15 -#define SL59 6 -#define WL60 14 -#define SL60 8 -#define WL61 5 -#define SL61 6 -#define WL62 6 -#define SL62 5 -#define WL63 2 -#define SL63 12 - -#define WL64 4 -#define SL64 9 -#define WL65 0 -#define SL65 15 -#define WL66 5 -#define SL66 5 -#define WL67 9 -#define SL67 11 -#define WL68 7 -#define SL68 6 -#define WL69 12 -#define SL69 8 -#define WL70 2 -#define SL70 13 -#define WL71 10 -#define SL71 12 -#define WL72 14 -#define SL72 5 -#define WL73 1 -#define SL73 12 -#define WL74 3 -#define SL74 13 -#define WL75 8 -#define SL75 14 -#define WL76 11 -#define SL76 11 -#define WL77 6 -#define SL77 8 -#define WL78 15 -#define SL78 5 -#define WL79 13 -#define SL79 6 - -#define WR00 5 -#define SR00 8 -#define WR01 14 -#define SR01 9 -#define WR02 7 -#define SR02 9 -#define WR03 0 -#define SR03 11 -#define WR04 9 -#define SR04 13 -#define WR05 2 -#define SR05 15 -#define WR06 11 -#define SR06 15 -#define WR07 4 -#define SR07 5 -#define WR08 13 -#define SR08 7 -#define WR09 6 -#define SR09 7 -#define WR10 15 -#define SR10 8 -#define WR11 8 -#define SR11 11 -#define WR12 1 -#define SR12 14 -#define WR13 10 -#define SR13 14 -#define WR14 3 -#define SR14 12 -#define WR15 12 -#define SR15 6 - -#define WR16 6 -#define SR16 9 -#define WR17 11 -#define SR17 13 -#define WR18 3 -#define SR18 15 -#define WR19 7 -#define SR19 7 -#define WR20 0 -#define SR20 12 -#define WR21 13 -#define SR21 8 -#define WR22 5 -#define SR22 9 -#define WR23 10 -#define SR23 11 -#define WR24 14 -#define SR24 7 -#define WR25 15 -#define SR25 7 -#define WR26 8 -#define SR26 12 -#define WR27 12 -#define SR27 7 -#define WR28 4 -#define SR28 6 -#define WR29 9 -#define SR29 15 -#define WR30 1 -#define SR30 13 -#define WR31 2 -#define SR31 11 - -#define WR32 15 -#define SR32 9 -#define WR33 5 -#define SR33 7 -#define WR34 1 -#define SR34 15 -#define WR35 3 -#define SR35 11 -#define WR36 7 -#define SR36 8 -#define WR37 14 -#define SR37 6 -#define WR38 6 -#define SR38 6 -#define WR39 9 -#define SR39 14 -#define WR40 11 -#define SR40 12 -#define WR41 8 -#define SR41 13 -#define WR42 12 -#define SR42 5 -#define WR43 2 -#define SR43 14 -#define WR44 10 -#define SR44 13 -#define WR45 0 -#define SR45 13 -#define WR46 4 -#define SR46 7 -#define WR47 13 -#define SR47 5 - -#define WR48 8 -#define SR48 15 -#define WR49 6 -#define SR49 5 -#define WR50 4 -#define SR50 8 -#define WR51 1 -#define SR51 11 -#define WR52 3 -#define SR52 14 -#define WR53 11 -#define SR53 14 -#define WR54 15 -#define SR54 6 -#define WR55 0 -#define SR55 14 -#define WR56 5 -#define SR56 6 -#define WR57 12 -#define SR57 9 -#define WR58 2 -#define SR58 12 -#define WR59 13 -#define SR59 9 -#define WR60 9 -#define SR60 12 -#define WR61 7 -#define SR61 5 -#define WR62 10 -#define SR62 15 -#define WR63 14 -#define SR63 8 - -#define WR64 12 -#define SR64 8 -#define WR65 15 -#define SR65 5 -#define WR66 10 -#define SR66 12 -#define WR67 4 -#define SR67 9 -#define WR68 1 -#define SR68 12 -#define WR69 5 -#define SR69 5 -#define WR70 8 -#define SR70 14 -#define WR71 7 -#define SR71 6 -#define WR72 6 -#define SR72 8 -#define WR73 2 -#define SR73 13 -#define WR74 13 -#define SR74 6 -#define WR75 14 -#define SR75 5 -#define WR76 0 -#define SR76 15 -#define WR77 3 -#define SR77 13 -#define WR78 9 -#define SR78 11 -#define WR79 11 -#define SR79 11 diff --git a/openssl/src/crypto/riscvcap.c b/openssl/src/crypto/riscvcap.c deleted file mode 100644 index db75c21b2..000000000 --- a/openssl/src/crypto/riscvcap.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include -#include "internal/cryptlib.h" - -#define OPENSSL_RISCVCAP_IMPL -#include "crypto/riscv_arch.h" - -extern size_t riscv_vlen_asm(void); - -static void parse_env(const char *envstr); -static void strtoupper(char *str); - -static size_t vlen = 0; - -uint32_t OPENSSL_rdtsc(void) -{ - return 0; -} - -size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt) -{ - return 0; -} - -size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max) -{ - return 0; -} - -static void strtoupper(char *str) -{ - for (char *x = str; *x; ++x) - *x = toupper(*x); -} - -/* parse_env() parses a RISC-V architecture string. An example of such a string - * is "rv64gc_zba_zbb_zbc_zbs". Currently, the rv64gc part is ignored - * and we simply search for "_[extension]" in the arch string to see if we - * should enable a given extension. - */ -#define BUFLEN 256 -static void parse_env(const char *envstr) -{ - char envstrupper[BUFLEN]; - char buf[BUFLEN]; - - /* Convert env str to all uppercase */ - OPENSSL_strlcpy(envstrupper, envstr, sizeof(envstrupper)); - strtoupper(envstrupper); - - for (size_t i = 0; i < kRISCVNumCaps; ++i) { - /* Prefix capability with underscore in preparation for search */ - BIO_snprintf(buf, BUFLEN, "_%s", RISCV_capabilities[i].name); - if (strstr(envstrupper, buf) != NULL) { - /* Match, set relevant bit in OPENSSL_riscvcap_P[] */ - OPENSSL_riscvcap_P[RISCV_capabilities[i].index] |= - (1 << RISCV_capabilities[i].bit_offset); - } - } -} - -size_t riscv_vlen(void) -{ - return vlen; -} - -# if defined(__GNUC__) && __GNUC__>=2 -__attribute__ ((constructor)) -# endif -void OPENSSL_cpuid_setup(void) -{ - char *e; - static int trigger = 0; - - if (trigger != 0) - return; - trigger = 1; - - if ((e = getenv("OPENSSL_riscvcap"))) { - parse_env(e); - } - - if (RISCV_HAS_V()) { - vlen = riscv_vlen_asm(); - } -} diff --git a/openssl/src/crypto/rsa/rsa_ameth.c b/openssl/src/crypto/rsa/rsa_ameth.c index 148d0bbbd..796485bff 100644 --- a/openssl/src/crypto/rsa/rsa_ameth.c +++ b/openssl/src/crypto/rsa/rsa_ameth.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -60,16 +60,13 @@ static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) if (!rsa_param_encode(pkey, &str, &strtype)) return 0; penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc); - if (penclen <= 0) { - ASN1_STRING_free(str); + if (penclen <= 0) return 0; - } if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(pkey->ameth->pkey_id), strtype, str, penc, penclen)) return 1; OPENSSL_free(penc); - ASN1_STRING_free(str); return 0; } @@ -133,6 +130,17 @@ static int old_rsa_priv_decode(EVP_PKEY *pkey, if ((rsa = d2i_RSAPrivateKey(NULL, pder, derlen)) == NULL) return 0; +# ifndef OPENSSL_NO_RSA_MULTI_PRIME_KEY_COMPAT + if (rsa->version == RSA_ASN1_VERSION_DEFAULT + && sk_RSA_PRIME_INFO_num(rsa->prime_infos) > 0) { + rsa->version = RSA_ASN1_VERSION_MULTI; + if (!ossl_rsa_multip_calc_product(rsa)) { + RSAerr(RSA_F_OLD_RSA_PRIV_DECODE, ERR_R_RSA_LIB); + return 0; + } + } +# endif + EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa); return 1; } @@ -154,16 +162,15 @@ static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk); if (rklen <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); ASN1_STRING_free(str); return 0; } if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(pkey->ameth->pkey_id), 0, strtype, str, rk, rklen)) { - ERR_raise(ERR_LIB_RSA, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); ASN1_STRING_free(str); - OPENSSL_clear_free(rk, rklen); return 0; } @@ -220,7 +227,7 @@ static int rsa_pss_param_print(BIO *bp, int pss_key, RSA_PSS_PARAMS *pss, return 0; } } else if (pss == NULL) { - if (BIO_puts(bp, "(INVALID PSS PARAMETERS)\n") <= 0) + if (BIO_puts(bp,"(INVALID PSS PARAMETERS)\n") <= 0) return 0; return 1; } @@ -453,35 +460,21 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx) const EVP_MD *sigmd, *mgf1md; EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx); int saltlen; - int saltlenMax = -1; if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0) return NULL; if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0) return NULL; - if (EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen) <= 0) + if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen)) return NULL; - if (saltlen == RSA_PSS_SALTLEN_DIGEST) { + if (saltlen == -1) { saltlen = EVP_MD_get_size(sigmd); - } else if (saltlen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) { - /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", - * subsection 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in - * bytes) of the salt (sLen) shall satisfy 0 <= sLen <= hLen, where - * hLen is the length of the hash function output block (in bytes)." - * - * Provide a way to use at most the digest length, so that the default - * does not violate FIPS 186-4. */ - saltlen = RSA_PSS_SALTLEN_MAX; - saltlenMax = EVP_MD_get_size(sigmd); - } - if (saltlen == RSA_PSS_SALTLEN_MAX || saltlen == RSA_PSS_SALTLEN_AUTO) { + } else if (saltlen == -2 || saltlen == -3) { saltlen = EVP_PKEY_get_size(pk) - EVP_MD_get_size(sigmd) - 2; if ((EVP_PKEY_get_bits(pk) & 0x7) == 1) saltlen--; if (saltlen < 0) return NULL; - if (saltlenMax >= 0 && saltlen > saltlenMax) - saltlen = saltlenMax; } return ossl_rsa_pss_params_create(sigmd, mgf1md, saltlen); @@ -654,62 +647,22 @@ static int rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, const void *asn, if (pad_mode == RSA_PKCS1_PADDING) return 2; if (pad_mode == RSA_PKCS1_PSS_PADDING) { - unsigned char aid[128]; - size_t aid_len = 0; - OSSL_PARAM params[2]; - - if (evp_pkey_ctx_is_legacy(pkctx)) { - /* No provider -> we cannot query it for algorithm ID. */ - ASN1_STRING *os1 = NULL; - - os1 = ossl_rsa_ctx_to_pss_string(pkctx); - if (os1 == NULL) - return 0; - /* Duplicate parameters if we have to */ - if (alg2 != NULL) { - ASN1_STRING *os2 = ASN1_STRING_dup(os1); - - if (os2 == NULL) { - ASN1_STRING_free(os1); - return 0; - } - if (!X509_ALGOR_set0(alg2, OBJ_nid2obj(EVP_PKEY_RSA_PSS), - V_ASN1_SEQUENCE, os2)) { - ASN1_STRING_free(os1); - ASN1_STRING_free(os2); - return 0; - } - } - if (!X509_ALGOR_set0(alg1, OBJ_nid2obj(EVP_PKEY_RSA_PSS), - V_ASN1_SEQUENCE, os1)) { - ASN1_STRING_free(os1); - return 0; - } - return 3; - } - - params[0] = OSSL_PARAM_construct_octet_string( - OSSL_SIGNATURE_PARAM_ALGORITHM_ID, aid, sizeof(aid)); - params[1] = OSSL_PARAM_construct_end(); - - if (EVP_PKEY_CTX_get_params(pkctx, params) <= 0) - return 0; - if ((aid_len = params[0].return_size) == 0) + ASN1_STRING *os1 = NULL; + os1 = ossl_rsa_ctx_to_pss_string(pkctx); + if (!os1) return 0; - - if (alg1 != NULL) { - const unsigned char *pp = aid; - - if (d2i_X509_ALGOR(&alg1, &pp, aid_len) == NULL) - return 0; - } - if (alg2 != NULL) { - const unsigned char *pp = aid; - - if (d2i_X509_ALGOR(&alg2, &pp, aid_len) == NULL) + /* Duplicate parameters if we have to */ + if (alg2) { + ASN1_STRING *os2 = ASN1_STRING_dup(os1); + if (!os2) { + ASN1_STRING_free(os1); return 0; + } + X509_ALGOR_set0(alg2, OBJ_nid2obj(EVP_PKEY_RSA_PSS), + V_ASN1_SEQUENCE, os2); } - + X509_ALGOR_set0(alg1, OBJ_nid2obj(EVP_PKEY_RSA_PSS), + V_ASN1_SEQUENCE, os1); return 3; } return 2; @@ -850,7 +803,7 @@ static int rsa_int_import_from(const OSSL_PARAM params[], void *vpctx, int ok = 0; if (rsa == NULL) { - ERR_raise(ERR_LIB_DH, ERR_R_RSA_LIB); + ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE); return 0; } diff --git a/openssl/src/crypto/rsa/rsa_backend.c b/openssl/src/crypto/rsa/rsa_backend.c index 36ee28337..254ebdb24 100644 --- a/openssl/src/crypto/rsa/rsa_backend.c +++ b/openssl/src/crypto/rsa/rsa_backend.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -44,17 +44,14 @@ static int collect_numbers(STACK_OF(BIGNUM) *numbers, if (numbers == NULL) return 0; - for (i = 0; names[i] != NULL; i++) { + for (i = 0; names[i] != NULL; i++){ p = OSSL_PARAM_locate_const(params, names[i]); if (p != NULL) { BIGNUM *tmp = NULL; - if (!OSSL_PARAM_get_BN(p, &tmp)) + if (!OSSL_PARAM_get_BN(p, &tmp) + || sk_BIGNUM_push(numbers, tmp) == 0) return 0; - if (sk_BIGNUM_push(numbers, tmp) == 0) { - BN_clear_free(tmp); - return 0; - } } } @@ -64,56 +61,22 @@ static int collect_numbers(STACK_OF(BIGNUM) *numbers, int ossl_rsa_fromdata(RSA *rsa, const OSSL_PARAM params[], int include_private) { const OSSL_PARAM *param_n, *param_e, *param_d = NULL; - const OSSL_PARAM *param_p, *param_q = NULL; - const OSSL_PARAM *param_derive = NULL; - BIGNUM *p = NULL, *q = NULL, *n = NULL, *e = NULL, *d = NULL; + BIGNUM *n = NULL, *e = NULL, *d = NULL; STACK_OF(BIGNUM) *factors = NULL, *exps = NULL, *coeffs = NULL; int is_private = 0; - int derive_from_pq = 0; - BN_CTX *ctx = NULL; if (rsa == NULL) return 0; param_n = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_N); param_e = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_E); - - if ((param_n == NULL || !OSSL_PARAM_get_BN(param_n, &n)) - || (param_e == NULL || !OSSL_PARAM_get_BN(param_e, &e))) { - ERR_raise(ERR_LIB_RSA, ERR_R_PASSED_NULL_PARAMETER); - goto err; - } - - if (include_private) { - - param_derive = OSSL_PARAM_locate_const(params, - OSSL_PKEY_PARAM_RSA_DERIVE_FROM_PQ); - if ((param_derive != NULL) - && !OSSL_PARAM_get_int(param_derive, &derive_from_pq)) - goto err; - + if (include_private) param_d = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_D); - if (param_d != NULL && !OSSL_PARAM_get_BN(param_d, &d)) { - ERR_raise(ERR_LIB_RSA, ERR_R_PASSED_NULL_PARAMETER); - goto err; - } - if (derive_from_pq) { - ctx = BN_CTX_new_ex(rsa->libctx); - if (ctx == NULL) - goto err; - - /* we need at minimum p, q */ - param_p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_FACTOR1); - param_q = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_FACTOR2); - if ((param_p == NULL || !OSSL_PARAM_get_BN(param_p, &p)) - || (param_q == NULL || !OSSL_PARAM_get_BN(param_q, &q))) { - ERR_raise(ERR_LIB_RSA, ERR_R_PASSED_NULL_PARAMETER); - goto err; - } - - } - } + if ((param_n != NULL && !OSSL_PARAM_get_BN(param_n, &n)) + || (param_e != NULL && !OSSL_PARAM_get_BN(param_e, &e)) + || (param_d != NULL && !OSSL_PARAM_get_BN(param_d, &d))) + goto err; is_private = (d != NULL); @@ -130,121 +93,25 @@ int ossl_rsa_fromdata(RSA *rsa, const OSSL_PARAM params[], int include_private) ossl_rsa_mp_coeff_names)) goto err; - if (derive_from_pq && sk_BIGNUM_num(exps) == 0 - && sk_BIGNUM_num(coeffs) == 0) { - /* - * If we want to use crt to derive our exponents/coefficients, we - * need to have at least 2 factors - */ - if (sk_BIGNUM_num(factors) < 2) { - ERR_raise(ERR_LIB_RSA, ERR_R_PASSED_NULL_PARAMETER); - goto err; - } - - /* - * if we have more than two factors, n and d must also have - * been provided - */ - if (sk_BIGNUM_num(factors) > 2 - && (param_n == NULL || param_d == NULL)) { - ERR_raise(ERR_LIB_RSA, ERR_R_PASSED_NULL_PARAMETER); - goto err; - } - - /* build our exponents and coefficients here */ - if (sk_BIGNUM_num(factors) == 2) { - /* for 2 factors we can use the sp800 functions to do this */ - if (!RSA_set0_factors(rsa, sk_BIGNUM_value(factors, 0), - sk_BIGNUM_value(factors, 1))) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - /* - * once consumed by RSA_set0_factors, pop those off the stack - * so we don't free them below - */ - sk_BIGNUM_pop(factors); - sk_BIGNUM_pop(factors); - - /* - * Note: Because we only have 2 factors here, there will be no - * additional pinfo fields to hold additional factors, and - * since we set our key and 2 factors above we can skip - * the call to ossl_rsa_set0_all_params - */ - if (!ossl_rsa_sp800_56b_derive_params_from_pq(rsa, - RSA_bits(rsa), - NULL, ctx)) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - } else { -#ifndef FIPS_MODULE - /* - * in the multiprime case we have to generate exps/coeffs here - * for each additional prime - */ - if (!ossl_rsa_multiprime_derive(rsa, RSA_bits(rsa), - sk_BIGNUM_num(factors), - rsa->e, factors, exps, - coeffs)) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* - * Now we should have all our factors, exponents and - * coefficients - */ - if (!ossl_rsa_set0_all_params(rsa, factors, exps, coeffs)) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - -#else - /* multiprime case is disallowed in FIPS mode, raise an error */ - ERR_raise(ERR_LIB_RSA, ERR_R_UNSUPPORTED); - goto err; -#endif - } - - } else { - /* - * It's ok if this private key just has n, e and d - * but only if we're not using derive_from_pq - */ - if (sk_BIGNUM_num(factors) != 0 - && !ossl_rsa_set0_all_params(rsa, factors, exps, coeffs)) - goto err; - } - /* sanity check to ensure we used everything in our stacks */ + /* It's ok if this private key just has n, e and d */ if (sk_BIGNUM_num(factors) != 0 - || sk_BIGNUM_num(exps) != 0 - || sk_BIGNUM_num(coeffs) != 0) { - ERR_raise_data(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR, - "There are %d, %d, %d elements left on our factors, exps, coeffs stacks\n", - sk_BIGNUM_num(factors), sk_BIGNUM_num(exps), - sk_BIGNUM_num(coeffs)); + && !ossl_rsa_set0_all_params(rsa, factors, exps, coeffs)) goto err; - } } - BN_clear_free(p); - BN_clear_free(q); + sk_BIGNUM_free(factors); sk_BIGNUM_free(exps); sk_BIGNUM_free(coeffs); - BN_CTX_free(ctx); return 1; err: BN_free(n); BN_free(e); BN_free(d); - sk_BIGNUM_pop_free(factors, BN_clear_free); - sk_BIGNUM_pop_free(exps, BN_clear_free); - sk_BIGNUM_pop_free(coeffs, BN_clear_free); - BN_CTX_free(ctx); + sk_BIGNUM_pop_free(factors, BN_free); + sk_BIGNUM_pop_free(exps, BN_free); + sk_BIGNUM_pop_free(coeffs, BN_free); return 0; } @@ -271,6 +138,18 @@ int ossl_rsa_todata(RSA *rsa, OSSL_PARAM_BLD *bld, OSSL_PARAM params[], /* Check private key data integrity */ if (include_private && rsa_d != NULL) { + int numprimes = sk_BIGNUM_const_num(factors); + int numexps = sk_BIGNUM_const_num(exps); + int numcoeffs = sk_BIGNUM_const_num(coeffs); + + /* + * It's permissible to have zero primes, i.e. no CRT params. + * Otherwise, there must be at least two, as many exponents, + * and one coefficient less. + */ + if (numprimes != 0 + && (numprimes < 2 || numexps < 2 || numcoeffs < 1)) + goto err; if (!ossl_param_build_set_bn(bld, params, OSSL_PKEY_PARAM_RSA_D, rsa_d) @@ -282,7 +161,7 @@ int ossl_rsa_todata(RSA *rsa, OSSL_PARAM_BLD *bld, OSSL_PARAM params[], || !ossl_param_build_set_multi_key_bn(bld, params, ossl_rsa_mp_coeff_names, coeffs)) - goto err; + goto err; } #if defined(FIPS_MODULE) && !defined(OPENSSL_NO_ACVP_TESTS) @@ -519,8 +398,10 @@ RSA *ossl_rsa_dup(const RSA *rsa, int selection) const RSA_PRIME_INFO *pinfo = NULL; RSA_PRIME_INFO *duppinfo = NULL; - if ((duppinfo = OPENSSL_zalloc(sizeof(*duppinfo))) == NULL) + if ((duppinfo = OPENSSL_zalloc(sizeof(*duppinfo))) == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } /* push first so cleanup in error case works */ (void)sk_RSA_PRIME_INFO_push(dupkey->prime_infos, duppinfo); @@ -642,7 +523,7 @@ int ossl_rsa_pss_get_param_unverified(const RSA_PSS_PARAMS *pss, if (pss->trailerField) *ptrailerField = ASN1_INTEGER_get(pss->trailerField); else - *ptrailerField = ossl_rsa_pss_params_30_trailerfield(&pss_params); + *ptrailerField = ossl_rsa_pss_params_30_trailerfield(&pss_params);; return 1; } diff --git a/openssl/src/crypto/rsa/rsa_chk.c b/openssl/src/crypto/rsa/rsa_chk.c index 0df254676..01fe9ead6 100644 --- a/openssl/src/crypto/rsa/rsa_chk.c +++ b/openssl/src/crypto/rsa/rsa_chk.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -51,7 +51,7 @@ static int rsa_validate_keypair_multiprime(const RSA *key, BN_GENCB *cb) if (i == NULL || j == NULL || k == NULL || l == NULL || m == NULL || ctx == NULL) { ret = -1; - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; } @@ -124,17 +124,13 @@ static int rsa_validate_keypair_multiprime(const RSA *key, BN_GENCB *cb) ret = -1; goto err; } - if (!BN_div(m, NULL, l, m, ctx)) { /* remainder is 0 */ - ret = -1; - goto err; - } for (idx = 0; idx < ex_primes; idx++) { pinfo = sk_RSA_PRIME_INFO_value(key->prime_infos, idx); if (!BN_sub(k, pinfo->r, BN_value_one())) { ret = -1; goto err; } - if (!BN_mul(l, m, k, ctx)) { + if (!BN_mul(l, l, k, ctx)) { ret = -1; goto err; } @@ -142,12 +138,12 @@ static int rsa_validate_keypair_multiprime(const RSA *key, BN_GENCB *cb) ret = -1; goto err; } - if (!BN_div(m, NULL, l, m, ctx)) { /* remainder is 0 */ - ret = -1; - goto err; - } } - if (!BN_mod_mul(i, key->d, key->e, m, ctx)) { + if (!BN_div(k, NULL, l, m, ctx)) { /* remainder is 0 */ + ret = -1; + goto err; + } + if (!BN_mod_mul(i, key->d, key->e, k, ctx)) { ret = -1; goto err; } @@ -249,7 +245,7 @@ int ossl_rsa_validate_pairwise(const RSA *key) #ifdef FIPS_MODULE return ossl_rsa_sp800_56b_check_keypair(key, NULL, -1, RSA_bits(key)); #else - return rsa_validate_keypair_multiprime(key, NULL) > 0; + return rsa_validate_keypair_multiprime(key, NULL); #endif } diff --git a/openssl/src/crypto/rsa/rsa_crpt.c b/openssl/src/crypto/rsa/rsa_crpt.c index 21c922e60..6bc6aafcc 100644 --- a/openssl/src/crypto/rsa/rsa_crpt.c +++ b/openssl/src/crypto/rsa/rsa_crpt.c @@ -129,7 +129,7 @@ BN_BLINDING *RSA_setup_blinding(RSA *rsa, BN_CTX *in_ctx) BN_CTX_start(ctx); e = BN_CTX_get(ctx); if (e == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; } @@ -147,7 +147,7 @@ BN_BLINDING *RSA_setup_blinding(RSA *rsa, BN_CTX *in_ctx) BIGNUM *n = BN_new(); if (n == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; } BN_with_flags(n, rsa->n, BN_FLG_CONSTTIME); diff --git a/openssl/src/crypto/rsa/rsa_gen.c b/openssl/src/crypto/rsa/rsa_gen.c index 75347d800..ac64483e6 100644 --- a/openssl/src/crypto/rsa/rsa_gen.c +++ b/openssl/src/crypto/rsa/rsa_gen.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -71,236 +71,37 @@ int RSA_generate_multi_prime_key(RSA *rsa, int bits, int primes, return rsa_keygen(rsa->libctx, rsa, bits, primes, e_value, cb, 0); } -DEFINE_STACK_OF(BIGNUM) - -/* - * Given input values, q, p, n, d and e, derive the exponents - * and coefficients for each prime in this key, placing the result - * on their respective exps and coeffs stacks - */ #ifndef FIPS_MODULE -int ossl_rsa_multiprime_derive(RSA *rsa, int bits, int primes, - BIGNUM *e_value, - STACK_OF(BIGNUM) *factors, - STACK_OF(BIGNUM) *exps, - STACK_OF(BIGNUM) *coeffs) -{ - STACK_OF(BIGNUM) *pplist = NULL, *pdlist = NULL; - BIGNUM *factor = NULL, *newpp = NULL, *newpd = NULL; - BIGNUM *dval = NULL, *newexp = NULL, *newcoeff = NULL; - BIGNUM *p = NULL, *q = NULL; - BIGNUM *dmp1 = NULL, *dmq1 = NULL, *iqmp = NULL; - BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL; - BN_CTX *ctx = NULL; - BIGNUM *tmp = NULL; - int i; - int ret = 0; - - ctx = BN_CTX_new_ex(rsa->libctx); - if (ctx == NULL) - goto err; - - BN_CTX_start(ctx); - - pplist = sk_BIGNUM_new_null(); - if (pplist == NULL) - goto err; - - pdlist = sk_BIGNUM_new_null(); - if (pdlist == NULL) - goto err; - - r0 = BN_CTX_get(ctx); - r1 = BN_CTX_get(ctx); - r2 = BN_CTX_get(ctx); - - if (r2 == NULL) - goto err; - - BN_set_flags(r0, BN_FLG_CONSTTIME); - BN_set_flags(r1, BN_FLG_CONSTTIME); - BN_set_flags(r2, BN_FLG_CONSTTIME); - - if (BN_copy(r1, rsa->n) == NULL) - goto err; - - p = sk_BIGNUM_value(factors, 0); - q = sk_BIGNUM_value(factors, 1); - - /* Build list of partial products of primes */ - for (i = 0; i < sk_BIGNUM_num(factors); i++) { - switch (i) { - case 0: - /* our first prime, p */ - if (!BN_sub(r2, p, BN_value_one())) - goto err; - BN_set_flags(r2, BN_FLG_CONSTTIME); - if (BN_mod_inverse(r1, r2, rsa->e, ctx) == NULL) - goto err; - break; - case 1: - /* second prime q */ - if (!BN_mul(r1, p, q, ctx)) - goto err; - tmp = BN_dup(r1); - if (tmp == NULL) - goto err; - if (!sk_BIGNUM_insert(pplist, tmp, sk_BIGNUM_num(pplist))) - goto err; - break; - default: - factor = sk_BIGNUM_value(factors, i); - /* all other primes */ - if (!BN_mul(r1, r1, factor, ctx)) - goto err; - tmp = BN_dup(r1); - if (tmp == NULL) - goto err; - if (!sk_BIGNUM_insert(pplist, tmp, sk_BIGNUM_num(pplist))) - goto err; - break; - } - } - - /* build list of relative d values */ - /* p -1 */ - if (!BN_sub(r1, p, BN_value_one())) - goto err; - if (!BN_sub(r2, q, BN_value_one())) - goto err; - if (!BN_mul(r0, r1, r2, ctx)) - goto err; - for (i = 2; i < sk_BIGNUM_num(factors); i++) { - factor = sk_BIGNUM_value(factors, i); - dval = BN_new(); - if (dval == NULL) - goto err; - BN_set_flags(dval, BN_FLG_CONSTTIME); - if (!BN_sub(dval, factor, BN_value_one())) - goto err; - if (!BN_mul(r0, r0, dval, ctx)) - goto err; - if (!sk_BIGNUM_insert(pdlist, dval, sk_BIGNUM_num(pdlist))) - goto err; - } - - /* Calculate dmp1, dmq1 and additional exponents */ - dmp1 = BN_secure_new(); - if (dmp1 == NULL) - goto err; - dmq1 = BN_secure_new(); - if (dmq1 == NULL) - goto err; - - if (!BN_mod(dmp1, rsa->d, r1, ctx)) - goto err; - if (!sk_BIGNUM_insert(exps, dmp1, sk_BIGNUM_num(exps))) - goto err; - dmp1 = NULL; - - if (!BN_mod(dmq1, rsa->d, r2, ctx)) - goto err; - if (!sk_BIGNUM_insert(exps, dmq1, sk_BIGNUM_num(exps))) - goto err; - dmq1 = NULL; - - for (i = 2; i < sk_BIGNUM_num(factors); i++) { - newpd = sk_BIGNUM_value(pdlist, i - 2); - newexp = BN_new(); - if (newexp == NULL) - goto err; - if (!BN_mod(newexp, rsa->d, newpd, ctx)) { - BN_free(newexp); - goto err; - } - if (!sk_BIGNUM_insert(exps, newexp, sk_BIGNUM_num(exps))) - goto err; - } - - /* Calculate iqmp and additional coefficients */ - iqmp = BN_new(); - if (iqmp == NULL) - goto err; - - if (BN_mod_inverse(iqmp, sk_BIGNUM_value(factors, 1), - sk_BIGNUM_value(factors, 0), ctx) == NULL) - goto err; - if (!sk_BIGNUM_insert(coeffs, iqmp, sk_BIGNUM_num(coeffs))) - goto err; - iqmp = NULL; - - for (i = 2; i < sk_BIGNUM_num(factors); i++) { - newpp = sk_BIGNUM_value(pplist, i - 2); - newcoeff = BN_new(); - if (newcoeff == NULL) - goto err; - if (BN_mod_inverse(newcoeff, newpp, sk_BIGNUM_value(factors, i), - ctx) == NULL) { - BN_free(newcoeff); - goto err; - } - if (!sk_BIGNUM_insert(coeffs, newcoeff, sk_BIGNUM_num(coeffs))) - goto err; - } - - ret = 1; - err: - sk_BIGNUM_pop_free(pplist, BN_free); - sk_BIGNUM_pop_free(pdlist, BN_free); - BN_CTX_end(ctx); - BN_CTX_free(ctx); - BN_clear_free(dmp1); - BN_clear_free(dmq1); - BN_clear_free(iqmp); - return ret; -} - static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes, BIGNUM *e_value, BN_GENCB *cb) { - BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *tmp, *tmp2, *prime; + BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *tmp, *prime; int n = 0, bitsr[RSA_MAX_PRIME_NUM], bitse = 0; int i = 0, quo = 0, rmd = 0, adj = 0, retries = 0; RSA_PRIME_INFO *pinfo = NULL; STACK_OF(RSA_PRIME_INFO) *prime_infos = NULL; - STACK_OF(BIGNUM) *factors = NULL; - STACK_OF(BIGNUM) *exps = NULL; - STACK_OF(BIGNUM) *coeffs = NULL; BN_CTX *ctx = NULL; BN_ULONG bitst = 0; unsigned long error = 0; int ok = -1; if (bits < RSA_MIN_MODULUS_BITS) { + ok = 0; /* we set our own err */ ERR_raise(ERR_LIB_RSA, RSA_R_KEY_SIZE_TOO_SMALL); - return 0; - } - if (e_value == NULL) { - ERR_raise(ERR_LIB_RSA, RSA_R_BAD_E_VALUE); - return 0; + goto err; } + /* A bad value for e can cause infinite loops */ - if (!ossl_rsa_check_public_exponent(e_value)) { + if (e_value != NULL && !ossl_rsa_check_public_exponent(e_value)) { ERR_raise(ERR_LIB_RSA, RSA_R_PUB_EXPONENT_OUT_OF_RANGE); return 0; } if (primes < RSA_DEFAULT_PRIME_NUM || primes > ossl_rsa_multip_cap(bits)) { + ok = 0; /* we set our own err */ ERR_raise(ERR_LIB_RSA, RSA_R_KEY_PRIME_NUM_INVALID); - return 0; - } - - factors = sk_BIGNUM_new_null(); - if (factors == NULL) - return 0; - - exps = sk_BIGNUM_new_null(); - if (exps == NULL) - goto err; - - coeffs = sk_BIGNUM_new_null(); - if (coeffs == NULL) goto err; + } ctx = BN_CTX_new_ex(rsa->libctx); if (ctx == NULL) @@ -335,6 +136,15 @@ static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes, if (!rsa->q && ((rsa->q = BN_secure_new()) == NULL)) goto err; BN_set_flags(rsa->q, BN_FLG_CONSTTIME); + if (!rsa->dmp1 && ((rsa->dmp1 = BN_secure_new()) == NULL)) + goto err; + BN_set_flags(rsa->dmp1, BN_FLG_CONSTTIME); + if (!rsa->dmq1 && ((rsa->dmq1 = BN_secure_new()) == NULL)) + goto err; + BN_set_flags(rsa->dmq1, BN_FLG_CONSTTIME); + if (!rsa->iqmp && ((rsa->iqmp = BN_secure_new()) == NULL)) + goto err; + BN_set_flags(rsa->iqmp, BN_FLG_CONSTTIME); /* initialize multi-prime components */ if (primes > RSA_DEFAULT_PRIME_NUM) { @@ -409,7 +219,7 @@ static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes, ERR_set_mark(); BN_set_flags(r2, BN_FLG_CONSTTIME); if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) { - /* GCD == 1 since inverse exists */ + /* GCD == 1 since inverse exists */ break; } error = ERR_peek_last_error(); @@ -439,14 +249,8 @@ static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes, /* i == 0, do nothing */ if (!BN_GENCB_call(cb, 3, i)) goto err; - tmp = BN_dup(prime); - if (tmp == NULL) - goto err; - if (!sk_BIGNUM_insert(factors, tmp, sk_BIGNUM_num(factors))) - goto err; continue; } - /* * if |r1|, product of factors so far, is not as long as expected * (by checking the first 4 bits are less than 0x9 or greater than @@ -493,10 +297,6 @@ static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes, */ i = -1; bitse = 0; - sk_BIGNUM_pop_free(factors, BN_clear_free); - factors = sk_BIGNUM_new_null(); - if (factors == NULL) - goto err; continue; } retries++; @@ -509,20 +309,12 @@ static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes, goto err; if (!BN_GENCB_call(cb, 3, i)) goto err; - tmp = BN_dup(prime); - if (tmp == NULL) - goto err; - if (!sk_BIGNUM_insert(factors, tmp, sk_BIGNUM_num(factors))) - goto err; } if (BN_cmp(rsa->p, rsa->q) < 0) { tmp = rsa->p; rsa->p = rsa->q; rsa->q = tmp; - /* mirror this in our factor stack */ - if (!sk_BIGNUM_insert(factors, sk_BIGNUM_delete(factors, 0), 1)) - goto err; } /* calculate d */ @@ -546,51 +338,79 @@ static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes, goto err; } + { + BIGNUM *pr0 = BN_new(); + + if (pr0 == NULL) + goto err; - BN_set_flags(r0, BN_FLG_CONSTTIME); - if (BN_mod_inverse(rsa->d, rsa->e, r0, ctx) == NULL) { - goto err; /* d */ + BN_with_flags(pr0, r0, BN_FLG_CONSTTIME); + if (!BN_mod_inverse(rsa->d, rsa->e, pr0, ctx)) { + BN_free(pr0); + goto err; /* d */ + } + /* We MUST free pr0 before any further use of r0 */ + BN_free(pr0); } - /* derive any missing exponents and coefficients */ - if (!ossl_rsa_multiprime_derive(rsa, bits, primes, e_value, - factors, exps, coeffs)) - goto err; + { + BIGNUM *d = BN_new(); - /* - * first 2 factors/exps are already tracked in p/q/dmq1/dmp1 - * and the first coeff is in iqmp, so pop those off the stack - * Note, the first 2 factors/exponents are already tracked by p and q - * assign dmp1/dmq1 and iqmp - * the remaining pinfo values are separately allocated, so copy and delete - * those - */ - BN_clear_free(sk_BIGNUM_delete(factors, 0)); - BN_clear_free(sk_BIGNUM_delete(factors, 0)); - rsa->dmp1 = sk_BIGNUM_delete(exps, 0); - rsa->dmq1 = sk_BIGNUM_delete(exps, 0); - rsa->iqmp = sk_BIGNUM_delete(coeffs, 0); - for (i = 2; i < primes; i++) { - pinfo = sk_RSA_PRIME_INFO_value(prime_infos, i - 2); - tmp = sk_BIGNUM_delete(factors, 0); - BN_copy(pinfo->r, tmp); - BN_clear_free(tmp); - tmp = sk_BIGNUM_delete(exps, 0); - tmp2 = BN_copy(pinfo->d, tmp); - BN_clear_free(tmp); - if (tmp2 == NULL) + if (d == NULL) goto err; - tmp = sk_BIGNUM_delete(coeffs, 0); - tmp2 = BN_copy(pinfo->t, tmp); - BN_clear_free(tmp); - if (tmp2 == NULL) + + BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME); + + /* calculate d mod (p-1) and d mod (q - 1) */ + if (!BN_mod(rsa->dmp1, d, r1, ctx) + || !BN_mod(rsa->dmq1, d, r2, ctx)) { + BN_free(d); goto err; + } + + /* calculate CRT exponents */ + for (i = 2; i < primes; i++) { + pinfo = sk_RSA_PRIME_INFO_value(prime_infos, i - 2); + /* pinfo->d == r_i - 1 */ + if (!BN_mod(pinfo->d, d, pinfo->d, ctx)) { + BN_free(d); + goto err; + } + } + + /* We MUST free d before any further use of rsa->d */ + BN_free(d); } + + { + BIGNUM *p = BN_new(); + + if (p == NULL) + goto err; + BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME); + + /* calculate inverse of q mod p */ + if (!BN_mod_inverse(rsa->iqmp, rsa->q, p, ctx)) { + BN_free(p); + goto err; + } + + /* calculate CRT coefficient for other primes */ + for (i = 2; i < primes; i++) { + pinfo = sk_RSA_PRIME_INFO_value(prime_infos, i - 2); + BN_with_flags(p, pinfo->r, BN_FLG_CONSTTIME); + if (!BN_mod_inverse(pinfo->t, pinfo->pp, p, ctx)) { + BN_free(p); + goto err; + } + } + + /* We MUST free p before any further use of rsa->p */ + BN_free(p); + } + ok = 1; err: - sk_BIGNUM_free(factors); - sk_BIGNUM_free(exps); - sk_BIGNUM_free(coeffs); if (ok == -1) { ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); ok = 0; @@ -606,22 +426,20 @@ static int rsa_keygen(OSSL_LIB_CTX *libctx, RSA *rsa, int bits, int primes, { int ok = 0; -#ifdef FIPS_MODULE - ok = ossl_rsa_sp800_56b_generate_key(rsa, bits, e_value, cb); - pairwise_test = 1; /* FIPS MODE needs to always run the pairwise test */ -#else /* - * Only multi-prime keys or insecure keys with a small key length or a - * public exponent <= 2^16 will use the older rsa_multiprime_keygen(). + * Only multi-prime keys or insecure keys with a small key length will use + * the older rsa_multiprime_keygen(). */ - if (primes == 2 - && bits >= 2048 - && (e_value == NULL || BN_num_bits(e_value) > 16)) + if (primes == 2 && bits >= 2048) ok = ossl_rsa_sp800_56b_generate_key(rsa, bits, e_value, cb); +#ifndef FIPS_MODULE else ok = rsa_multiprime_keygen(rsa, bits, primes, e_value, cb); #endif /* FIPS_MODULE */ +#ifdef FIPS_MODULE + pairwise_test = 1; /* FIPS MODE needs to always run the pairwise test */ +#endif if (pairwise_test && ok > 0) { OSSL_CALLBACK *stcb = NULL; void *stcbarg = NULL; diff --git a/openssl/src/crypto/rsa/rsa_lib.c b/openssl/src/crypto/rsa/rsa_lib.c index 5350a4e65..a8a6d6c75 100644 --- a/openssl/src/crypto/rsa/rsa_lib.c +++ b/openssl/src/crypto/rsa/rsa_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -76,18 +76,15 @@ static RSA *rsa_new_intern(ENGINE *engine, OSSL_LIB_CTX *libctx) { RSA *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return NULL; + } + ret->references = 1; ret->lock = CRYPTO_THREAD_lock_new(); if (ret->lock == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_CRYPTO_LIB); - OPENSSL_free(ret); - return NULL; - } - - if (!CRYPTO_NEW_REF(&ret->references, 1)) { - CRYPTO_THREAD_lock_free(ret->lock); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -140,7 +137,7 @@ void RSA_free(RSA *r) if (r == NULL) return; - CRYPTO_DOWN_REF(&r->references, &i); + CRYPTO_DOWN_REF(&r->references, &i, r->lock); REF_PRINT_COUNT("RSA", r); if (i > 0) return; @@ -157,7 +154,6 @@ void RSA_free(RSA *r) #endif CRYPTO_THREAD_lock_free(r->lock); - CRYPTO_FREE_REF(&r->references); BN_free(r->n); BN_free(r->e); @@ -185,7 +181,7 @@ int RSA_up_ref(RSA *r) { int i; - if (CRYPTO_UP_REF(&r->references, &i) <= 0) + if (CRYPTO_UP_REF(&r->references, &i, r->lock) <= 0) return 0; REF_PRINT_COUNT("RSA", r); @@ -744,13 +740,9 @@ int RSA_pkey_ctx_ctrl(EVP_PKEY_CTX *ctx, int optype, int cmd, int p1, void *p2) DEFINE_STACK_OF(BIGNUM) -/* - * Note: This function deletes values from the parameter - * stack values as they are consumed and set in the RSA key. - */ -int ossl_rsa_set0_all_params(RSA *r, STACK_OF(BIGNUM) *primes, - STACK_OF(BIGNUM) *exps, - STACK_OF(BIGNUM) *coeffs) +int ossl_rsa_set0_all_params(RSA *r, const STACK_OF(BIGNUM) *primes, + const STACK_OF(BIGNUM) *exps, + const STACK_OF(BIGNUM) *coeffs) { #ifndef FIPS_MODULE STACK_OF(RSA_PRIME_INFO) *prime_infos, *old_infos = NULL; @@ -761,40 +753,18 @@ int ossl_rsa_set0_all_params(RSA *r, STACK_OF(BIGNUM) *primes, return 0; pnum = sk_BIGNUM_num(primes); - - /* we need at least 2 primes */ - if (pnum < 2) + if (pnum < 2 + || pnum != sk_BIGNUM_num(exps) + || pnum != sk_BIGNUM_num(coeffs) + 1) return 0; if (!RSA_set0_factors(r, sk_BIGNUM_value(primes, 0), - sk_BIGNUM_value(primes, 1))) - return 0; - - /* - * if we managed to set everything above, remove those elements from the - * stack - * Note, we do this after the above all to ensure that we have taken - * ownership of all the elements in the RSA key to avoid memory leaks - * we also use delete 0 here as we are grabbing items from the end of the - * stack rather than the start, otherwise we could use pop - */ - sk_BIGNUM_delete(primes, 0); - sk_BIGNUM_delete(primes, 0); - - if (pnum == sk_BIGNUM_num(exps) - && pnum == sk_BIGNUM_num(coeffs) + 1) { - - if (!RSA_set0_crt_params(r, sk_BIGNUM_value(exps, 0), - sk_BIGNUM_value(exps, 1), - sk_BIGNUM_value(coeffs, 0))) + sk_BIGNUM_value(primes, 1)) + || !RSA_set0_crt_params(r, sk_BIGNUM_value(exps, 0), + sk_BIGNUM_value(exps, 1), + sk_BIGNUM_value(coeffs, 0))) return 0; - /* as above, once we consume the above params, delete them from the list */ - sk_BIGNUM_delete(exps, 0); - sk_BIGNUM_delete(exps, 0); - sk_BIGNUM_delete(coeffs, 0); - } - #ifndef FIPS_MODULE old_infos = r->prime_infos; #endif @@ -808,17 +778,19 @@ int ossl_rsa_set0_all_params(RSA *r, STACK_OF(BIGNUM) *primes, return 0; for (i = 2; i < pnum; i++) { - BIGNUM *prime = sk_BIGNUM_pop(primes); - BIGNUM *exp = sk_BIGNUM_pop(exps); - BIGNUM *coeff = sk_BIGNUM_pop(coeffs); + BIGNUM *prime = sk_BIGNUM_value(primes, i); + BIGNUM *exp = sk_BIGNUM_value(exps, i); + BIGNUM *coeff = sk_BIGNUM_value(coeffs, i - 1); RSA_PRIME_INFO *pinfo = NULL; if (!ossl_assert(prime != NULL && exp != NULL && coeff != NULL)) goto err; /* Using ossl_rsa_multip_info_new() is wasteful, so allocate directly */ - if ((pinfo = OPENSSL_zalloc(sizeof(*pinfo))) == NULL) + if ((pinfo = OPENSSL_zalloc(sizeof(*pinfo))) == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } pinfo->r = prime; pinfo->d = exp; @@ -1023,10 +995,6 @@ int EVP_PKEY_CTX_set_rsa_pss_keygen_md_name(EVP_PKEY_CTX *ctx, */ int EVP_PKEY_CTX_set_rsa_oaep_md(EVP_PKEY_CTX *ctx, const EVP_MD *md) { - /* If key type not RSA return error */ - if (!EVP_PKEY_CTX_is_a(ctx, "RSA")) - return -1; - return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, EVP_PKEY_CTRL_RSA_OAEP_MD, 0, (void *)(md)); } @@ -1054,10 +1022,6 @@ int EVP_PKEY_CTX_get_rsa_oaep_md_name(EVP_PKEY_CTX *ctx, char *name, */ int EVP_PKEY_CTX_get_rsa_oaep_md(EVP_PKEY_CTX *ctx, const EVP_MD **md) { - /* If key type not RSA return error */ - if (!EVP_PKEY_CTX_is_a(ctx, "RSA")) - return -1; - return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, EVP_PKEY_CTRL_GET_RSA_OAEP_MD, 0, (void *)md); } @@ -1120,13 +1084,6 @@ int EVP_PKEY_CTX_get_rsa_mgf1_md(EVP_PKEY_CTX *ctx, const EVP_MD **md) int EVP_PKEY_CTX_set0_rsa_oaep_label(EVP_PKEY_CTX *ctx, void *label, int llen) { OSSL_PARAM rsa_params[2], *p = rsa_params; - const char *empty = ""; - /* - * Needed as we swap label with empty if it is NULL, and label is - * freed at the end of this function. - */ - void *plabel = label; - int ret; if (ctx == NULL || !EVP_PKEY_CTX_IS_ASYM_CIPHER_OP(ctx)) { ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED); @@ -1138,20 +1095,15 @@ int EVP_PKEY_CTX_set0_rsa_oaep_label(EVP_PKEY_CTX *ctx, void *label, int llen) if (!EVP_PKEY_CTX_is_a(ctx, "RSA")) return -1; - /* Accept NULL for backward compatibility */ - if (label == NULL && llen == 0) - plabel = (void *)empty; - /* Cast away the const. This is read only so should be safe */ *p++ = OSSL_PARAM_construct_octet_string(OSSL_ASYM_CIPHER_PARAM_OAEP_LABEL, - (void *)plabel, (size_t)llen); + (void *)label, (size_t)llen); *p++ = OSSL_PARAM_construct_end(); - ret = evp_pkey_ctx_set_params_strict(ctx, rsa_params); - if (ret <= 0) - return ret; + if (!evp_pkey_ctx_set_params_strict(ctx, rsa_params)) + return 0; - /* Ownership is supposed to be transferred to the callee. */ + /* Ownership is supposed to be transfered to the callee. */ OPENSSL_free(label); return 1; } diff --git a/openssl/src/crypto/rsa/rsa_local.h b/openssl/src/crypto/rsa/rsa_local.h index db9eb2a1d..0fd2a2729 100644 --- a/openssl/src/crypto/rsa/rsa_local.h +++ b/openssl/src/crypto/rsa/rsa_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2006-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,7 +13,11 @@ #include "internal/refcount.h" #include "crypto/rsa.h" -#define RSA_MAX_PRIME_NUM 5 +# ifndef OPENSSL_NO_RSA_MULTI_PRIME_KEY_COMPAT +# define RSA_MAX_PRIME_NUM 512 +# else +# define RSA_MAX_PRIME_NUM 5 +# endif typedef struct rsa_prime_info_st { BIGNUM *r; @@ -150,10 +154,6 @@ struct rsa_meth_st { /* Macros to test if a pkey or ctx is for a PSS key */ #define pkey_is_pss(pkey) (pkey->ameth->pkey_id == EVP_PKEY_RSA_PSS) #define pkey_ctx_is_pss(ctx) (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS) -int ossl_rsa_multiprime_derive(RSA *rsa, int bits, int primes, - BIGNUM *e_value, - STACK_OF(BIGNUM) *factors, STACK_OF(BIGNUM) *exps, - STACK_OF(BIGNUM) *coeffs); RSA_PSS_PARAMS *ossl_rsa_pss_params_create(const EVP_MD *sigmd, const EVP_MD *mgf1md, int saltlen); diff --git a/openssl/src/crypto/rsa/rsa_meth.c b/openssl/src/crypto/rsa/rsa_meth.c index f04098bd0..82f13bb35 100644 --- a/openssl/src/crypto/rsa/rsa_meth.c +++ b/openssl/src/crypto/rsa/rsa_meth.c @@ -31,6 +31,7 @@ RSA_METHOD *RSA_meth_new(const char *name, int flags) OPENSSL_free(meth); } + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return NULL; } @@ -56,6 +57,7 @@ RSA_METHOD *RSA_meth_dup(const RSA_METHOD *meth) OPENSSL_free(ret); } + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return NULL; } @@ -68,8 +70,10 @@ int RSA_meth_set1_name(RSA_METHOD *meth, const char *name) { char *tmpname = OPENSSL_strdup(name); - if (tmpname == NULL) + if (tmpname == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return 0; + } OPENSSL_free(meth->name); meth->name = tmpname; diff --git a/openssl/src/crypto/rsa/rsa_mp.c b/openssl/src/crypto/rsa/rsa_mp.c index cb2fb7d1e..415b085a6 100644 --- a/openssl/src/crypto/rsa/rsa_mp.c +++ b/openssl/src/crypto/rsa/rsa_mp.c @@ -21,7 +21,7 @@ void ossl_rsa_multip_info_free_ex(RSA_PRIME_INFO *pinfo) void ossl_rsa_multip_info_free(RSA_PRIME_INFO *pinfo) { - /* free an RSA_PRIME_INFO structure */ + /* free a RSA_PRIME_INFO structure */ BN_clear_free(pinfo->r); BN_clear_free(pinfo->d); BN_clear_free(pinfo->t); @@ -32,9 +32,11 @@ RSA_PRIME_INFO *ossl_rsa_multip_info_new(void) { RSA_PRIME_INFO *pinfo; - /* create an RSA_PRIME_INFO structure */ - if ((pinfo = OPENSSL_zalloc(sizeof(RSA_PRIME_INFO))) == NULL) + /* create a RSA_PRIME_INFO structure */ + if ((pinfo = OPENSSL_zalloc(sizeof(RSA_PRIME_INFO))) == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return NULL; + } if ((pinfo->r = BN_secure_new()) == NULL) goto err; if ((pinfo->d = BN_secure_new()) == NULL) @@ -97,6 +99,9 @@ int ossl_rsa_multip_calc_product(RSA *rsa) int ossl_rsa_multip_cap(int bits) { +# ifndef OPENSSL_NO_RSA_MULTI_PRIME_KEY_COMPAT + return RSA_MAX_PRIME_NUM; +# else int cap = 5; if (bits < 1024) @@ -110,4 +115,5 @@ int ossl_rsa_multip_cap(int bits) cap = RSA_MAX_PRIME_NUM; return cap; +# endif } diff --git a/openssl/src/crypto/rsa/rsa_oaep.c b/openssl/src/crypto/rsa/rsa_oaep.c index b9030440c..d9be1a4f9 100644 --- a/openssl/src/crypto/rsa/rsa_oaep.c +++ b/openssl/src/crypto/rsa/rsa_oaep.c @@ -112,8 +112,10 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(OSSL_LIB_CTX *libctx, dbmask_len = emlen - mdlen; dbmask = OPENSSL_malloc(dbmask_len); - if (dbmask == NULL) + if (dbmask == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } /* step 3e: dbMask = MGF(mgfSeed, nLen - HLen - 1) */ if (PKCS1_MGF1(dbmask, dbmask_len, seed, mdlen, mgf1md) < 0) @@ -201,12 +203,16 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen, dblen = num - mdlen - 1; db = OPENSSL_malloc(dblen); - if (db == NULL) + if (db == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto cleanup; + } em = OPENSSL_malloc(num); - if (em == NULL) + if (em == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto cleanup; + } /* * Caller is encouraged to pass zero-padded message created with diff --git a/openssl/src/crypto/rsa/rsa_ossl.c b/openssl/src/crypto/rsa/rsa_ossl.c index 14dfd457f..8dea31b61 100644 --- a/openssl/src/crypto/rsa/rsa_ossl.c +++ b/openssl/src/crypto/rsa/rsa_ossl.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,13 +13,14 @@ */ #include "internal/deprecated.h" +#include +#include +#include + #include "internal/cryptlib.h" #include "crypto/bn.h" #include "rsa_local.h" #include "internal/constant_time.h" -#include -#include -#include static int rsa_ossl_public_encrypt(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding); @@ -33,27 +34,6 @@ static int rsa_ossl_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa, BN_CTX *ctx); static int rsa_ossl_init(RSA *rsa); static int rsa_ossl_finish(RSA *rsa); -#ifdef S390X_MOD_EXP -static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa, - BN_CTX *ctx); -static RSA_METHOD rsa_pkcs1_ossl_meth = { - "OpenSSL PKCS#1 RSA", - rsa_ossl_public_encrypt, - rsa_ossl_public_decrypt, /* signature verification */ - rsa_ossl_private_encrypt, /* signing */ - rsa_ossl_private_decrypt, - rsa_ossl_s390x_mod_exp, - s390x_mod_exp, - rsa_ossl_init, - rsa_ossl_finish, - RSA_FLAG_FIPS_METHOD, /* flags */ - NULL, - 0, /* rsa_sign */ - 0, /* rsa_verify */ - NULL, /* rsa_keygen */ - NULL /* rsa_multi_prime_keygen */ -}; -#else static RSA_METHOD rsa_pkcs1_ossl_meth = { "OpenSSL PKCS#1 RSA", rsa_ossl_public_encrypt, @@ -72,7 +52,6 @@ static RSA_METHOD rsa_pkcs1_ossl_meth = { NULL, /* rsa_keygen */ NULL /* rsa_multi_prime_keygen */ }; -#endif static const RSA_METHOD *default_RSA_meth = &rsa_pkcs1_ossl_meth; @@ -129,8 +108,10 @@ static int rsa_ossl_public_encrypt(int flen, const unsigned char *from, ret = BN_CTX_get(ctx); num = BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); - if (ret == NULL || buf == NULL) + if (ret == NULL || buf == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } switch (padding) { case RSA_PKCS1_PADDING: @@ -186,21 +167,11 @@ static BN_BLINDING *rsa_get_blinding(RSA *rsa, int *local, BN_CTX *ctx) { BN_BLINDING *ret; - if (!CRYPTO_THREAD_read_lock(rsa->lock)) + if (!CRYPTO_THREAD_write_lock(rsa->lock)) return NULL; if (rsa->blinding == NULL) { - /* - * This dance with upgrading the lock from read to write will be - * slower in cases of a single use RSA object, but should be - * significantly better in multi-thread cases (e.g. servers). It's - * probably worth it. - */ - CRYPTO_THREAD_unlock(rsa->lock); - if (!CRYPTO_THREAD_write_lock(rsa->lock)) - return NULL; - if (rsa->blinding == NULL) - rsa->blinding = RSA_setup_blinding(rsa, ctx); + rsa->blinding = RSA_setup_blinding(rsa, ctx); } ret = rsa->blinding; @@ -222,11 +193,7 @@ static BN_BLINDING *rsa_get_blinding(RSA *rsa, int *local, BN_CTX *ctx) *local = 0; if (rsa->mt_blinding == NULL) { - CRYPTO_THREAD_unlock(rsa->lock); - if (!CRYPTO_THREAD_write_lock(rsa->lock)) - return NULL; - if (rsa->mt_blinding == NULL) - rsa->mt_blinding = RSA_setup_blinding(rsa, ctx); + rsa->mt_blinding = RSA_setup_blinding(rsa, ctx); } ret = rsa->mt_blinding; } @@ -271,7 +238,6 @@ static int rsa_blinding_invert(BN_BLINDING *b, BIGNUM *f, BIGNUM *unblind, * will only read the modulus from BN_BLINDING. In both cases it's safe * to access the blinding without a lock. */ - BN_set_flags(f, BN_FLG_CONSTTIME); return BN_BLINDING_invert_ex(f, unblind, b, ctx); } @@ -299,8 +265,10 @@ static int rsa_ossl_private_encrypt(int flen, const unsigned char *from, ret = BN_CTX_get(ctx); num = BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); - if (ret == NULL || buf == NULL) + if (ret == NULL || buf == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } switch (padding) { case RSA_PKCS1_PADDING: @@ -343,7 +311,7 @@ static int rsa_ossl_private_encrypt(int flen, const unsigned char *from, if (blinding != NULL) { if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) { - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; } if (!rsa_blinding_convert(blinding, f, unblind, ctx)) @@ -360,7 +328,7 @@ static int rsa_ossl_private_encrypt(int flen, const unsigned char *from, } else { BIGNUM *d = BN_new(); if (d == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; } if (rsa->d == NULL) { @@ -426,23 +394,20 @@ static int derive_kdk(int flen, const unsigned char *from, RSA *rsa, } if (rsa->d == NULL) { ERR_raise(ERR_LIB_RSA, RSA_R_MISSING_PRIVATE_KEY); - BN_free(d); goto err; } BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME); if (BN_bn2binpad(d, buf, num) < 0) { ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - BN_free(d); goto err; } - BN_free(d); /* * we use hardcoded hash so that migrating between versions that use * different hash doesn't provide a Bleichenbacher oracle: * if the attacker can see that different versions return different * messages for the same ciphertext, they'll know that the message is - * synthetically generated, which means that the padding check failed + * syntethically generated, which means that the padding check failed */ md = EVP_MD_fetch(rsa->libctx, "sha256", NULL); if (md == NULL) { @@ -486,6 +451,7 @@ static int derive_kdk(int flen, const unsigned char *from, RSA *rsa, ret = 1; err: + BN_free(d); HMAC_CTX_free(hmac); EVP_MD_free(md); return ret; @@ -508,25 +474,17 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, BIGNUM *unblind = NULL; BN_BLINDING *blinding = NULL; - /* - * we need the value of the private exponent to perform implicit rejection - */ - if ((rsa->flags & RSA_FLAG_EXT_PKEY) && (padding == RSA_PKCS1_PADDING)) - padding = RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING; - if ((ctx = BN_CTX_new_ex(rsa->libctx)) == NULL) goto err; BN_CTX_start(ctx); f = BN_CTX_get(ctx); ret = BN_CTX_get(ctx); - if (ret == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); - goto err; - } num = BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); - if (buf == NULL) + if (ret == NULL || buf == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } /* * This check was for equality but PGP does evil things and chops off the @@ -537,11 +495,6 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, goto err; } - if (flen < 1) { - ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_SMALL); - goto err; - } - /* make data into a big number */ if (BN_bin2bn(from, (int)flen, f) == NULL) goto err; @@ -551,11 +504,6 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, goto err; } - if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) - if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, rsa->lock, - rsa->n, ctx)) - goto err; - if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) { blinding = rsa_get_blinding(rsa, &local_blinding, ctx); if (blinding == NULL) { @@ -566,7 +514,7 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, if (blinding != NULL) { if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) { - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; } if (!rsa_blinding_convert(blinding, f, unblind, ctx)) @@ -584,7 +532,7 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, } else { BIGNUM *d = BN_new(); if (d == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; } if (rsa->d == NULL) { @@ -593,6 +541,13 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, goto err; } BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME); + + if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) + if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, rsa->lock, + rsa->n, ctx)) { + BN_free(d); + goto err; + } if (!rsa->meth->bn_mod_exp(ret, f, d, rsa->n, ctx, rsa->_method_mod_n)) { BN_free(d); @@ -602,10 +557,6 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, BN_free(d); } - if (blinding) - if (!rsa_blinding_invert(blinding, ret, unblind, ctx)) - goto err; - /* * derive the Key Derivation Key from private exponent and public * ciphertext @@ -615,16 +566,24 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, goto err; } - j = BN_bn2binpad(ret, buf, num); - if (j < 0) - goto err; + if (blinding) { + /* + * ossl_bn_rsa_do_unblind() combines blinding inversion and + * 0-padded BN BE serialization + */ + j = ossl_bn_rsa_do_unblind(ret, blinding, unblind, rsa->n, ctx, + buf, num); + if (j == 0) + goto err; + } else { + j = BN_bn2binpad(ret, buf, num); + if (j < 0) + goto err; + } switch (padding) { - case RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING: - r = RSA_padding_check_PKCS1_type_2(to, num, buf, j, num); - break; case RSA_PKCS1_PADDING: - r = ossl_rsa_padding_check_PKCS1_type_2(rsa->libctx, to, num, buf, j, num, kdk); + r = RSA_padding_check_PKCS1_type_2(to, num, buf, j, num); break; case RSA_PKCS1_OAEP_PADDING: r = RSA_padding_check_PKCS1_OAEP(to, num, buf, j, num, NULL, 0); @@ -685,14 +644,12 @@ static int rsa_ossl_public_decrypt(int flen, const unsigned char *from, BN_CTX_start(ctx); f = BN_CTX_get(ctx); ret = BN_CTX_get(ctx); - if (ret == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_BN_LIB); - goto err; - } num = BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); - if (buf == NULL) + if (ret == NULL || buf == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } /* * This check was for equality but PGP does evil things and chops off the @@ -1146,16 +1103,3 @@ static int rsa_ossl_finish(RSA *rsa) BN_MONT_CTX_free(rsa->_method_mod_q); return 1; } - -#ifdef S390X_MOD_EXP -static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa, - BN_CTX *ctx) -{ - if (rsa->version != RSA_ASN1_VERSION_MULTI) { - if (s390x_crt(r0, i, rsa->p, rsa->q, rsa->dmp1, rsa->dmq1, rsa->iqmp) == 1) - return 1; - } - return rsa_ossl_mod_exp(r0, i, rsa, ctx); -} - -#endif diff --git a/openssl/src/crypto/rsa/rsa_pk1.c b/openssl/src/crypto/rsa/rsa_pk1.c index 7655ef9a9..51507fc03 100644 --- a/openssl/src/crypto/rsa/rsa_pk1.c +++ b/openssl/src/crypto/rsa/rsa_pk1.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,14 +21,10 @@ #include /* Just for the SSL_MAX_MASTER_KEY_LENGTH value */ #include -#include -#include -#include #include "internal/cryptlib.h" #include "crypto/rsa.h" #include "rsa_local.h" - int RSA_padding_add_PKCS1_type_1(unsigned char *to, int tlen, const unsigned char *from, int flen) { @@ -192,8 +188,10 @@ int RSA_padding_check_PKCS1_type_2(unsigned char *to, int tlen, } em = OPENSSL_malloc(num); - if (em == NULL) + if (em == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return -1; + } /* * Caller is encouraged to pass zero-padded message created with * BN_bn2binpad. Trouble is that since we can't read out of |from|'s @@ -275,254 +273,6 @@ int RSA_padding_check_PKCS1_type_2(unsigned char *to, int tlen, return constant_time_select_int(good, mlen, -1); } - -static int ossl_rsa_prf(OSSL_LIB_CTX *ctx, - unsigned char *to, int tlen, - const char *label, int llen, - const unsigned char *kdk, - uint16_t bitlen) -{ - int pos; - int ret = -1; - uint16_t iter = 0; - unsigned char be_iter[sizeof(iter)]; - unsigned char be_bitlen[sizeof(bitlen)]; - HMAC_CTX *hmac = NULL; - EVP_MD *md = NULL; - unsigned char hmac_out[SHA256_DIGEST_LENGTH]; - unsigned int md_len; - - if (tlen * 8 != bitlen) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - return ret; - } - - be_bitlen[0] = (bitlen >> 8) & 0xff; - be_bitlen[1] = bitlen & 0xff; - - hmac = HMAC_CTX_new(); - if (hmac == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* - * we use hardcoded hash so that migrating between versions that use - * different hash doesn't provide a Bleichenbacher oracle: - * if the attacker can see that different versions return different - * messages for the same ciphertext, they'll know that the message is - * synthetically generated, which means that the padding check failed - */ - md = EVP_MD_fetch(ctx, "sha256", NULL); - if (md == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - - if (HMAC_Init_ex(hmac, kdk, SHA256_DIGEST_LENGTH, md, NULL) <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - - for (pos = 0; pos < tlen; pos += SHA256_DIGEST_LENGTH, iter++) { - if (HMAC_Init_ex(hmac, NULL, 0, NULL, NULL) <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - - be_iter[0] = (iter >> 8) & 0xff; - be_iter[1] = iter & 0xff; - - if (HMAC_Update(hmac, be_iter, sizeof(be_iter)) <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - if (HMAC_Update(hmac, (unsigned char *)label, llen) <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - if (HMAC_Update(hmac, be_bitlen, sizeof(be_bitlen)) <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* - * HMAC_Final requires the output buffer to fit the whole MAC - * value, so we need to use the intermediate buffer for the last - * unaligned block - */ - md_len = SHA256_DIGEST_LENGTH; - if (pos + SHA256_DIGEST_LENGTH > tlen) { - if (HMAC_Final(hmac, hmac_out, &md_len) <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - memcpy(to + pos, hmac_out, tlen - pos); - } else { - if (HMAC_Final(hmac, to + pos, &md_len) <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - goto err; - } - } - } - - ret = 0; - -err: - HMAC_CTX_free(hmac); - EVP_MD_free(md); - return ret; -} - -/* - * ossl_rsa_padding_check_PKCS1_type_2() checks and removes the PKCS#1 type 2 - * padding from a decrypted RSA message. Unlike the - * RSA_padding_check_PKCS1_type_2() it will not return an error in case it - * detects a padding error, rather it will return a deterministically generated - * random message. In other words it will perform an implicit rejection - * of an invalid padding. This means that the returned value does not indicate - * if the padding of the encrypted message was correct or not, making - * side channel attacks like the ones described by Bleichenbacher impossible - * without access to the full decrypted value and a brute-force search of - * remaining padding bytes - */ -int ossl_rsa_padding_check_PKCS1_type_2(OSSL_LIB_CTX *ctx, - unsigned char *to, int tlen, - const unsigned char *from, int flen, - int num, unsigned char *kdk) -{ -/* - * We need to generate a random length for the synthetic message, to avoid - * bias towards zero and avoid non-constant timeness of DIV, we prepare - * 128 values to check if they are not too large for the used key size, - * and use 0 in case none of them are small enough, as 2^-128 is a good enough - * safety margin - */ -#define MAX_LEN_GEN_TRIES 128 - unsigned char *synthetic = NULL; - int synthetic_length; - uint16_t len_candidate; - unsigned char candidate_lengths[MAX_LEN_GEN_TRIES * sizeof(len_candidate)]; - uint16_t len_mask; - uint16_t max_sep_offset; - int synth_msg_index = 0; - int ret = -1; - int i, j; - unsigned int good, found_zero_byte; - int zero_index = 0, msg_index; - - /* - * If these checks fail then either the message in publicly invalid, or - * we've been called incorrectly. We can fail immediately. - * Since this code is called only internally by openssl, those are just - * sanity checks - */ - if (num != flen || tlen <= 0 || flen <= 0) { - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - return -1; - } - - /* Generate a random message to return in case the padding checks fail */ - synthetic = OPENSSL_malloc(flen); - if (synthetic == NULL) { - ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); - return -1; - } - - if (ossl_rsa_prf(ctx, synthetic, flen, "message", 7, kdk, flen * 8) < 0) - goto err; - - /* decide how long the random message should be */ - if (ossl_rsa_prf(ctx, candidate_lengths, sizeof(candidate_lengths), - "length", 6, kdk, - MAX_LEN_GEN_TRIES * sizeof(len_candidate) * 8) < 0) - goto err; - - /* - * max message size is the size of the modulus size less 2 bytes for - * version and padding type and a minimum of 8 bytes padding - */ - len_mask = max_sep_offset = flen - 2 - 8; - /* - * we want a mask so lets propagate the high bit to all positions less - * significant than it - */ - len_mask |= len_mask >> 1; - len_mask |= len_mask >> 2; - len_mask |= len_mask >> 4; - len_mask |= len_mask >> 8; - - synthetic_length = 0; - for (i = 0; i < MAX_LEN_GEN_TRIES * (int)sizeof(len_candidate); - i += sizeof(len_candidate)) { - len_candidate = (candidate_lengths[i] << 8) | candidate_lengths[i + 1]; - len_candidate &= len_mask; - - synthetic_length = constant_time_select_int( - constant_time_lt(len_candidate, max_sep_offset), - len_candidate, synthetic_length); - } - - synth_msg_index = flen - synthetic_length; - - /* we have alternative message ready, check the real one */ - good = constant_time_is_zero(from[0]); - good &= constant_time_eq(from[1], 2); - - /* then look for the padding|message separator (the first zero byte) */ - found_zero_byte = 0; - for (i = 2; i < flen; i++) { - unsigned int equals0 = constant_time_is_zero(from[i]); - zero_index = constant_time_select_int(~found_zero_byte & equals0, - i, zero_index); - found_zero_byte |= equals0; - } - - /* - * padding must be at least 8 bytes long, and it starts two bytes into - * |from|. If we never found a 0-byte, then |zero_index| is 0 and the check - * also fails. - */ - good &= constant_time_ge(zero_index, 2 + 8); - - /* - * Skip the zero byte. This is incorrect if we never found a zero-byte - * but in this case we also do not copy the message out. - */ - msg_index = zero_index + 1; - - /* - * old code returned an error in case the decrypted message wouldn't fit - * into the |to|, since that would leak information, return the synthetic - * message instead - */ - good &= constant_time_ge(tlen, num - msg_index); - - msg_index = constant_time_select_int(good, msg_index, synth_msg_index); - - /* - * since at this point the |msg_index| does not provide the signal - * indicating if the padding check failed or not, we don't have to worry - * about leaking the length of returned message, we still need to ensure - * that we read contents of both buffers so that cache accesses don't leak - * the value of |good| - */ - for (i = msg_index, j = 0; i < flen && j < tlen; i++, j++) - to[j] = constant_time_select_8(good, from[i], synthetic[i]); - ret = j; - -err: - /* - * the only time ret < 0 is when the ciphertext is publicly invalid - * or we were called with invalid parameters, so we don't have to perform - * a side-channel secure raising of the error - */ - if (ret < 0) - ERR_raise(ERR_LIB_RSA, ERR_R_INTERNAL_ERROR); - OPENSSL_free(synthetic); - return ret; -} - /* * ossl_rsa_padding_check_PKCS1_type_2_TLS() checks and removes the PKCS1 type 2 * padding from a decrypted RSA message in a TLS signature. The result is stored diff --git a/openssl/src/crypto/rsa/rsa_pmeth.c b/openssl/src/crypto/rsa/rsa_pmeth.c index fc3391ead..e6568444c 100644 --- a/openssl/src/crypto/rsa/rsa_pmeth.c +++ b/openssl/src/crypto/rsa/rsa_pmeth.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -52,8 +52,6 @@ typedef struct { /* OAEP label */ unsigned char *oaep_label; size_t oaep_labellen; - /* if to use implicit rejection in PKCS#1 v1.5 decryption */ - int implicit_rejection; } RSA_PKEY_CTX; /* True if PSS parameters are restricted */ @@ -74,7 +72,6 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx) /* Maximum for sign, auto for verify */ rctx->saltlen = RSA_PSS_SALTLEN_AUTO; rctx->min_saltlen = -1; - rctx->implicit_rejection = 1; ctx->data = rctx; ctx->keygen_info = rctx->gentmp; ctx->keygen_info_count = 2; @@ -100,7 +97,6 @@ static int pkey_rsa_copy(EVP_PKEY_CTX *dst, const EVP_PKEY_CTX *src) dctx->md = sctx->md; dctx->mgf1md = sctx->mgf1md; dctx->saltlen = sctx->saltlen; - dctx->implicit_rejection = sctx->implicit_rejection; if (sctx->oaep_label) { OPENSSL_free(dctx->oaep_label); dctx->oaep_label = OPENSSL_memdup(sctx->oaep_label, sctx->oaep_labellen); @@ -116,8 +112,10 @@ static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk) if (ctx->tbuf != NULL) return 1; if ((ctx->tbuf = - OPENSSL_malloc(RSA_size(EVP_PKEY_get0_RSA(pk->pkey)))) == NULL) + OPENSSL_malloc(RSA_size(EVP_PKEY_get0_RSA(pk->pkey)))) == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return 0; + } return 1; } @@ -151,22 +149,13 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, return -1; } - if (EVP_MD_get_type(rctx->md) == NID_mdc2) { - unsigned int sltmp; - if (rctx->pad_mode != RSA_PKCS1_PADDING) - return -1; - ret = RSA_sign_ASN1_OCTET_STRING(0, tbs, tbslen, sig, &sltmp, rsa); - - if (ret <= 0) - return ret; - ret = sltmp; - } else if (rctx->pad_mode == RSA_X931_PADDING) { + if (rctx->pad_mode == RSA_X931_PADDING) { if ((size_t)RSA_size(rsa) < tbslen + 1) { ERR_raise(ERR_LIB_RSA, RSA_R_KEY_SIZE_TOO_SMALL); return -1; } if (!setup_tbuf(rctx, ctx)) { - ERR_raise(ERR_LIB_RSA, ERR_R_RSA_LIB); + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return -1; } memcpy(rctx->tbuf, tbs, tbslen); @@ -349,7 +338,6 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, const unsigned char *in, size_t inlen) { int ret; - int pad_mode; RSA_PKEY_CTX *rctx = ctx->data; /* * Discard const. Its marked as const because this may be a cached copy of @@ -370,12 +358,7 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, rctx->oaep_labellen, rctx->md, rctx->mgf1md); } else { - if (rctx->pad_mode == RSA_PKCS1_PADDING && - rctx->implicit_rejection == 0) - pad_mode = RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING; - else - pad_mode = rctx->pad_mode; - ret = RSA_private_decrypt(inlen, in, out, rsa, pad_mode); + ret = RSA_private_decrypt(inlen, in, out, rsa, rctx->pad_mode); } *outlen = constant_time_select_s(constant_time_msb_s(ret), *outlen, ret); ret = constant_time_select_int(constant_time_msb(ret), ret, 1); @@ -402,7 +385,7 @@ static int check_padding_md(const EVP_MD *md, int padding) return 0; } } else { - switch (mdnid) { + switch(mdnid) { /* List of all supported RSA digests */ case NID_sha1: case NID_sha224: @@ -413,10 +396,6 @@ static int check_padding_md(const EVP_MD *md, int padding) case NID_sha512_256: case NID_md5: case NID_md5_sha1: - case NID_md2: - case NID_md4: - case NID_mdc2: - case NID_ripemd160: case NID_sha3_224: case NID_sha3_256: case NID_sha3_384: @@ -592,21 +571,9 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE); return -2; } - if (p2 == NULL) { - ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } *(unsigned char **)p2 = rctx->oaep_label; return rctx->oaep_labellen; - case EVP_PKEY_CTRL_RSA_IMPLICIT_REJECTION: - if (rctx->pad_mode != RSA_PKCS1_PADDING) { - ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_PADDING_MODE); - return -2; - } - rctx->implicit_rejection = p1; - return 1; - case EVP_PKEY_CTRL_DIGESTINIT: case EVP_PKEY_CTRL_PKCS7_SIGN: #ifndef OPENSSL_NO_CMS diff --git a/openssl/src/crypto/rsa/rsa_pss.c b/openssl/src/crypto/rsa/rsa_pss.c index 089730bba..33874bfef 100644 --- a/openssl/src/crypto/rsa/rsa_pss.c +++ b/openssl/src/crypto/rsa/rsa_pss.c @@ -1,5 +1,5 @@ /* - * Copyright 2005-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -61,12 +61,11 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash, * -1 sLen == hLen * -2 salt length is autorecovered from signature * -3 salt length is maximized - * -4 salt length is autorecovered from signature * -N reserved */ if (sLen == RSA_PSS_SALTLEN_DIGEST) { sLen = hLen; - } else if (sLen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) { + } else if (sLen < RSA_PSS_SALTLEN_MAX) { ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED); goto err; } @@ -98,8 +97,10 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash, maskedDBLen = emLen - hLen - 1; H = EM + maskedDBLen; DB = OPENSSL_malloc(maskedDBLen); - if (DB == NULL) + if (DB == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } if (PKCS1_MGF1(DB, maskedDBLen, H, hLen, mgf1Hash) < 0) goto err; for (i = 0; i < maskedDBLen; i++) @@ -111,9 +112,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash, ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_RECOVERY_FAILED); goto err; } - if (sLen != RSA_PSS_SALTLEN_AUTO - && sLen != RSA_PSS_SALTLEN_AUTO_DIGEST_MAX - && (maskedDBLen - i) != sLen) { + if (sLen != RSA_PSS_SALTLEN_AUTO && (maskedDBLen - i) != sLen) { ERR_raise_data(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED, "expected: %d retrieved: %d", sLen, maskedDBLen - i); @@ -161,7 +160,6 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM, int hLen, maskedDBLen, MSBits, emLen; unsigned char *H, *salt = NULL, *p; EVP_MD_CTX *ctx = NULL; - int sLenMax = -1; if (mgf1Hash == NULL) mgf1Hash = Hash; @@ -174,25 +172,13 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM, * -1 sLen == hLen * -2 salt length is maximized * -3 same as above (on signing) - * -4 salt length is min(hLen, maximum salt length) * -N reserved */ - /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection - * 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the - * salt (sLen) shall satisfy 0 <= sLen <= hLen, where hLen is the length of - * the hash function output block (in bytes)." - * - * Provide a way to use at most the digest length, so that the default does - * not violate FIPS 186-4. */ if (sLen == RSA_PSS_SALTLEN_DIGEST) { sLen = hLen; - } else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN - || sLen == RSA_PSS_SALTLEN_AUTO) { - sLen = RSA_PSS_SALTLEN_MAX; - } else if (sLen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) { + } else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN) { sLen = RSA_PSS_SALTLEN_MAX; - sLenMax = hLen; - } else if (sLen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) { + } else if (sLen < RSA_PSS_SALTLEN_MAX) { ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED); goto err; } @@ -209,16 +195,16 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM, } if (sLen == RSA_PSS_SALTLEN_MAX) { sLen = emLen - hLen - 2; - if (sLenMax >= 0 && sLen > sLenMax) - sLen = sLenMax; } else if (sLen > emLen - hLen - 2) { ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE); goto err; } if (sLen > 0) { salt = OPENSSL_malloc(sLen); - if (salt == NULL) + if (salt == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } if (RAND_bytes_ex(rsa->libctx, salt, sLen, 0) <= 0) goto err; } @@ -334,6 +320,15 @@ int ossl_rsa_pss_params_30_set_hashalg(RSA_PSS_PARAMS_30 *rsa_pss_params, return 1; } +int ossl_rsa_pss_params_30_set_maskgenalg(RSA_PSS_PARAMS_30 *rsa_pss_params, + int maskgenalg_nid) +{ + if (rsa_pss_params == NULL) + return 0; + rsa_pss_params->mask_gen.algorithm_nid = maskgenalg_nid; + return 1; +} + int ossl_rsa_pss_params_30_set_maskgenhashalg(RSA_PSS_PARAMS_30 *rsa_pss_params, int maskgenhashalg_nid) { diff --git a/openssl/src/crypto/rsa/rsa_saos.c b/openssl/src/crypto/rsa/rsa_saos.c index dc96b6dad..58fa50785 100644 --- a/openssl/src/crypto/rsa/rsa_saos.c +++ b/openssl/src/crypto/rsa/rsa_saos.c @@ -40,8 +40,10 @@ int RSA_sign_ASN1_OCTET_STRING(int type, return 0; } s = OPENSSL_malloc((unsigned int)j + 1); - if (s == NULL) + if (s == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return 0; + } p = s; i2d_ASN1_OCTET_STRING(&sig, &p); i = RSA_private_encrypt(i, s, sigret, rsa, RSA_PKCS1_PADDING); @@ -70,8 +72,10 @@ int RSA_verify_ASN1_OCTET_STRING(int dtype, } s = OPENSSL_malloc((unsigned int)siglen); - if (s == NULL) + if (s == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } i = RSA_public_decrypt((int)siglen, sigbuf, s, rsa, RSA_PKCS1_PADDING); if (i <= 0) diff --git a/openssl/src/crypto/rsa/rsa_sign.c b/openssl/src/crypto/rsa/rsa_sign.c index b14b13408..dc046562c 100644 --- a/openssl/src/crypto/rsa/rsa_sign.c +++ b/openssl/src/crypto/rsa/rsa_sign.c @@ -19,21 +19,9 @@ #include #include #ifndef FIPS_MODULE -# ifndef OPENSSL_NO_MD2 -# include /* uses MD2_DIGEST_LENGTH */ -# endif -# ifndef OPENSSL_NO_MD4 -# include /* uses MD4_DIGEST_LENGTH */ -# endif # ifndef OPENSSL_NO_MD5 # include /* uses MD5_DIGEST_LENGTH */ # endif -# ifndef OPENSSL_NO_MDC2 -# include /* uses MDC2_DIGEST_LENGTH */ -# endif -# ifndef OPENSSL_NO_RMD160 -# include /* uses RIPEMD160_DIGEST_LENGTH */ -# endif #endif #include /* uses SHA???_DIGEST_LENGTH */ #include "crypto/rsa.h" @@ -83,7 +71,7 @@ static const unsigned char digestinfo_##name##_der[] = { \ ASN1_OCTET_STRING, sz \ }; -/* MD2, MD4 and MD5 OIDs are of the form: (1 2 840 113549 2 |n|) */ +/* MD5 OID is of the form: (1 2 840 113549 2 |n|) */ #define ENCODE_DIGESTINFO_MD(name, n, sz) \ static const unsigned char digestinfo_##name##_der[] = { \ ASN1_SEQUENCE, 0x10 + sz, \ @@ -94,35 +82,9 @@ static const unsigned char digestinfo_##name##_der[] = { \ }; #ifndef FIPS_MODULE -# ifndef OPENSSL_NO_MD2 -ENCODE_DIGESTINFO_MD(md2, 0x02, MD2_DIGEST_LENGTH) -# endif -# ifndef OPENSSL_NO_MD4 -ENCODE_DIGESTINFO_MD(md4, 0x03, MD4_DIGEST_LENGTH) -# endif # ifndef OPENSSL_NO_MD5 ENCODE_DIGESTINFO_MD(md5, 0x05, MD5_DIGEST_LENGTH) # endif -# ifndef OPENSSL_NO_MDC2 -/* MDC-2 (2 5 8 3 101) */ -static const unsigned char digestinfo_mdc2_der[] = { - ASN1_SEQUENCE, 0x0c + MDC2_DIGEST_LENGTH, - ASN1_SEQUENCE, 0x08, - ASN1_OID, 0x04, 2 * 40 + 5, 8, 3, 101, - ASN1_NULL, 0x00, - ASN1_OCTET_STRING, MDC2_DIGEST_LENGTH -}; -# endif -# ifndef OPENSSL_NO_RMD160 -/* RIPEMD160 (1 3 36 3 2 1) */ -static const unsigned char digestinfo_ripemd160_der[] = { - ASN1_SEQUENCE, 0x0d + RIPEMD160_DIGEST_LENGTH, - ASN1_SEQUENCE, 0x09, - ASN1_OID, 0x05, 1 * 40 + 3, 36, 3, 2, 1, - ASN1_NULL, 0x00, - ASN1_OCTET_STRING, RIPEMD160_DIGEST_LENGTH -}; -# endif #endif /* FIPS_MODULE */ /* SHA-1 (1 3 14 3 2 26) */ @@ -154,21 +116,9 @@ const unsigned char *ossl_rsa_digestinfo_encoding(int md_nid, size_t *len) { switch (md_nid) { #ifndef FIPS_MODULE -# ifndef OPENSSL_NO_MDC2 - MD_CASE(mdc2) -# endif -# ifndef OPENSSL_NO_MD2 - MD_CASE(md2) -# endif -# ifndef OPENSSL_NO_MD4 - MD_CASE(md4) -# endif # ifndef OPENSSL_NO_MD5 MD_CASE(md5) # endif -# ifndef OPENSSL_NO_RMD160 - MD_CASE(ripemd160) -# endif #endif /* FIPS_MODULE */ MD_CASE(sha1) MD_CASE(sha224) @@ -194,21 +144,9 @@ static int digest_sz_from_nid(int nid) { switch (nid) { #ifndef FIPS_MODULE -# ifndef OPENSSL_NO_MDC2 - MD_NID_CASE(mdc2, MDC2_DIGEST_LENGTH) -# endif -# ifndef OPENSSL_NO_MD2 - MD_NID_CASE(md2, MD2_DIGEST_LENGTH) -# endif -# ifndef OPENSSL_NO_MD4 - MD_NID_CASE(md4, MD4_DIGEST_LENGTH) -# endif # ifndef OPENSSL_NO_MD5 MD_NID_CASE(md5, MD5_DIGEST_LENGTH) # endif -# ifndef OPENSSL_NO_RMD160 - MD_NID_CASE(ripemd160, RIPEMD160_DIGEST_LENGTH) -# endif #endif /* FIPS_MODULE */ MD_NID_CASE(sha1, SHA_DIGEST_LENGTH) MD_NID_CASE(sha224, SHA224_DIGEST_LENGTH) @@ -258,8 +196,10 @@ static int encode_pkcs1(unsigned char **out, size_t *out_len, int type, } dig_info_len = di_prefix_len + m_len; dig_info = OPENSSL_malloc(dig_info_len); - if (dig_info == NULL) + if (dig_info == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(dig_info, di_prefix, di_prefix_len); memcpy(dig_info + di_prefix_len, m, m_len); @@ -278,7 +218,7 @@ int RSA_sign(int type, const unsigned char *m, unsigned int m_len, #ifndef FIPS_MODULE if (rsa->meth->rsa_sign != NULL) - return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa) > 0; + return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa); #endif /* FIPS_MODULE */ /* Compute the encoded digest. */ @@ -341,8 +281,10 @@ int ossl_rsa_verify(int type, const unsigned char *m, unsigned int m_len, /* Recover the encoded digest. */ decrypt_buf = OPENSSL_malloc(siglen); - if (decrypt_buf == NULL) + if (decrypt_buf == NULL) { + ERR_raise(ERR_LIB_RSA, ERR_R_MALLOC_FAILURE); goto err; + } len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa, RSA_PKCS1_PADDING); @@ -376,26 +318,6 @@ int ossl_rsa_verify(int type, const unsigned char *m, unsigned int m_len, goto err; } } - } else if (type == NID_mdc2 && decrypt_len == 2 + 16 - && decrypt_buf[0] == 0x04 && decrypt_buf[1] == 0x10) { - /* - * Oddball MDC2 case: signature can be OCTET STRING. check for correct - * tag and length octets. - */ - if (rm != NULL) { - memcpy(rm, decrypt_buf + 2, 16); - *prm_len = 16; - } else { - if (m_len != 16) { - ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_MESSAGE_LENGTH); - goto err; - } - - if (memcmp(m, decrypt_buf + 2, 16) != 0) { - ERR_raise(ERR_LIB_RSA, RSA_R_BAD_SIGNATURE); - goto err; - } - } } else #endif /* FIPS_MODULE */ { diff --git a/openssl/src/crypto/rsa/rsa_sp800_56b_check.c b/openssl/src/crypto/rsa/rsa_sp800_56b_check.c index b9aafdfe6..fc8f19b48 100644 --- a/openssl/src/crypto/rsa/rsa_sp800_56b_check.c +++ b/openssl/src/crypto/rsa/rsa_sp800_56b_check.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2018-2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -289,11 +289,6 @@ int ossl_rsa_sp800_56b_check_public(const RSA *rsa) return 0; nbits = BN_num_bits(rsa->n); - if (nbits > OPENSSL_RSA_MAX_MODULUS_BITS) { - ERR_raise(ERR_LIB_RSA, RSA_R_MODULUS_TOO_LARGE); - return 0; - } - #ifdef FIPS_MODULE /* * (Step a): modulus must be 2048 or 3072 (caveat from SP800-56Br1) @@ -329,8 +324,7 @@ int ossl_rsa_sp800_56b_check_public(const RSA *rsa) goto err; } - /* Highest number of MR rounds from FIPS 186-5 Section B.3 Table B.1 */ - ret = ossl_bn_miller_rabin_is_prime(rsa->n, 5, ctx, NULL, 1, &status); + ret = ossl_bn_miller_rabin_is_prime(rsa->n, 0, ctx, NULL, 1, &status); #ifdef FIPS_MODULE if (ret != 1 || status != BN_PRIMETEST_COMPOSITE_NOT_POWER_OF_PRIME) { #else @@ -409,11 +403,6 @@ int ossl_rsa_sp800_56b_check_keypair(const RSA *rsa, const BIGNUM *efixed, ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_KEYPAIR); return 0; } - /* (Step 3.c): check that the modulus length is a positive even integer */ - if (nbits <= 0 || (nbits & 0x1)) { - ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_KEYPAIR); - return 0; - } ctx = BN_CTX_new_ex(rsa->libctx); if (ctx == NULL) diff --git a/openssl/src/crypto/rsa/rsa_sp800_56b_gen.c b/openssl/src/crypto/rsa/rsa_sp800_56b_gen.c index b0d9104b7..df2240555 100644 --- a/openssl/src/crypto/rsa/rsa_sp800_56b_gen.c +++ b/openssl/src/crypto/rsa/rsa_sp800_56b_gen.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2018-2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -30,6 +30,7 @@ * test Object used for CAVS testing only.that contains.. * p1, p2 The returned auxiliary primes for p. * If NULL they are not returned. + * Xpout An optionally returned random number used during generation of p. * Xp An optional passed in value (that is random number used during * generation of p). * Xp1, Xp2 Optionally passed in randomly generated numbers from which @@ -37,6 +38,7 @@ * are generated internally. * q1, q2 The returned auxiliary primes for q. * If NULL they are not returned. + * Xqout An optionally returned random number used during generation of q. * Xq An optional passed in value (that is random number used during * generation of q). * Xq1, Xq2 Optionally passed in randomly generated numbers from which @@ -48,7 +50,7 @@ * cb An optional BIGNUM callback. * Returns: 1 if successful, or 0 otherwise. * Notes: - * p1, p2, q1, q2 are returned if they are not NULL. + * p1, p2, q1, q2, Xpout, Xqout are returned if they are not NULL. * Xp, Xp1, Xp2, Xq, Xq1, Xq2 are optionally passed in. * (Required for CAVS testing). */ @@ -63,6 +65,7 @@ int ossl_rsa_fips186_4_gen_prob_primes(RSA *rsa, RSA_ACVP_TEST *test, BIGNUM *p1 = NULL, *p2 = NULL; BIGNUM *q1 = NULL, *q2 = NULL; /* Intermediate BIGNUMS that can be input for testing */ + BIGNUM *Xpout = NULL, *Xqout = NULL; BIGNUM *Xp = NULL, *Xp1 = NULL, *Xp2 = NULL; BIGNUM *Xq = NULL, *Xq1 = NULL, *Xq2 = NULL; @@ -102,8 +105,8 @@ int ossl_rsa_fips186_4_gen_prob_primes(RSA *rsa, RSA_ACVP_TEST *test, BN_CTX_start(ctx); tmp = BN_CTX_get(ctx); - Xpo = BN_CTX_get(ctx); - Xqo = BN_CTX_get(ctx); + Xpo = (Xpout != NULL) ? Xpout : BN_CTX_get(ctx); + Xqo = (Xqout != NULL) ? Xqout : BN_CTX_get(ctx); if (tmp == NULL || Xpo == NULL || Xqo == NULL) goto err; BN_set_flags(Xpo, BN_FLG_CONSTTIME); @@ -122,7 +125,7 @@ int ossl_rsa_fips186_4_gen_prob_primes(RSA *rsa, RSA_ACVP_TEST *test, if (!ossl_bn_rsa_fips186_4_gen_prob_primes(rsa->p, Xpo, p1, p2, Xp, Xp1, Xp2, nbits, e, ctx, cb)) goto err; - for (;;) { + for(;;) { /* (Step 5) Generate q, Xq*/ if (!ossl_bn_rsa_fips186_4_gen_prob_primes(rsa->q, Xqo, q1, q2, Xq, Xq1, Xq2, nbits, e, ctx, cb)) @@ -147,9 +150,9 @@ int ossl_rsa_fips186_4_gen_prob_primes(RSA *rsa, RSA_ACVP_TEST *test, ret = 1; err: /* Zeroize any internally generated values that are not returned */ - if (Xpo != NULL) + if (Xpo != Xpout) BN_clear(Xpo); - if (Xqo != NULL) + if (Xqo != Xqout) BN_clear(Xqo); BN_clear(tmp); @@ -225,16 +228,13 @@ static int rsa_validate_rng_strength(EVP_RAND_CTX *rng, int nbits) * Returns: -1 = error, * 0 = d is too small, * 1 = success. - * - * SP800-56b key generation always passes a non NULL value for e. - * For other purposes, if e is NULL then it is assumed that e, n and d are - * already set in the RSA key and do not need to be recalculated. */ int ossl_rsa_sp800_56b_derive_params_from_pq(RSA *rsa, int nbits, const BIGNUM *e, BN_CTX *ctx) { int ret = -1; BIGNUM *p1, *q1, *lcm, *p1q1, *gcd; + BN_CTX_start(ctx); p1 = BN_CTX_get(ctx); q1 = BN_CTX_get(ctx); @@ -254,38 +254,33 @@ int ossl_rsa_sp800_56b_derive_params_from_pq(RSA *rsa, int nbits, if (ossl_rsa_get_lcm(ctx, rsa->p, rsa->q, lcm, gcd, p1, q1, p1q1) != 1) goto err; - /* - * if e is provided as a parameter, don't recompute e, d or n - */ - if (e != NULL) { - /* copy e */ - BN_free(rsa->e); - rsa->e = BN_dup(e); - if (rsa->e == NULL) - goto err; - - BN_clear_free(rsa->d); - /* (Step 3) d = (e^-1) mod (LCM(p-1, q-1)) */ - rsa->d = BN_secure_new(); - if (rsa->d == NULL) - goto err; - BN_set_flags(rsa->d, BN_FLG_CONSTTIME); - if (BN_mod_inverse(rsa->d, e, lcm, ctx) == NULL) - goto err; + /* copy e */ + BN_free(rsa->e); + rsa->e = BN_dup(e); + if (rsa->e == NULL) + goto err; - /* (Step 3) return an error if d is too small */ - if (BN_num_bits(rsa->d) <= (nbits >> 1)) { - ret = 0; - goto err; - } + BN_clear_free(rsa->d); + /* (Step 3) d = (e^-1) mod (LCM(p-1, q-1)) */ + rsa->d = BN_secure_new(); + if (rsa->d == NULL) + goto err; + BN_set_flags(rsa->d, BN_FLG_CONSTTIME); + if (BN_mod_inverse(rsa->d, e, lcm, ctx) == NULL) + goto err; - /* (Step 4) n = pq */ - if (rsa->n == NULL) - rsa->n = BN_new(); - if (rsa->n == NULL || !BN_mul(rsa->n, rsa->p, rsa->q, ctx)) - goto err; + /* (Step 3) return an error if d is too small */ + if (BN_num_bits(rsa->d) <= (nbits >> 1)) { + ret = 0; + goto err; } + /* (Step 4) n = pq */ + if (rsa->n == NULL) + rsa->n = BN_new(); + if (rsa->n == NULL || !BN_mul(rsa->n, rsa->p, rsa->q, ctx)) + goto err; + /* (Step 5a) dP = d mod (p-1) */ if (rsa->dmp1 == NULL) rsa->dmp1 = BN_secure_new(); @@ -366,7 +361,6 @@ int ossl_rsa_sp800_56b_generate_key(RSA *rsa, int nbits, const BIGNUM *efixed, BN_CTX *ctx = NULL; BIGNUM *e = NULL; RSA_ACVP_TEST *info = NULL; - BIGNUM *tmp; #if defined(FIPS_MODULE) && !defined(OPENSSL_NO_ACVP_TESTS) info = rsa->acvp_test; @@ -398,14 +392,6 @@ int ossl_rsa_sp800_56b_generate_key(RSA *rsa, int nbits, const BIGNUM *efixed, /* (Step 2) Generate prime factors */ if (!ossl_rsa_fips186_4_gen_prob_primes(rsa, info, nbits, e, ctx, cb)) goto err; - - /* p>q check and skipping in case of acvp test */ - if (info == NULL && BN_cmp(rsa->p, rsa->q) < 0) { - tmp = rsa->p; - rsa->p = rsa->q; - rsa->q = tmp; - } - /* (Steps 3-5) Compute params d, n, dP, dQ, qInv */ ok = ossl_rsa_sp800_56b_derive_params_from_pq(rsa, nbits, e, ctx); if (ok < 0) @@ -428,7 +414,7 @@ int ossl_rsa_sp800_56b_generate_key(RSA *rsa, int nbits, const BIGNUM *efixed, * See SP800-56Br1 6.3.1.3 (Step 6) Perform a pair-wise consistency test by * verifying that: k = (k^e)^d mod n for some integer k where 1 < k < n-1. * - * Returns 1 if the RSA key passes the pairwise test or 0 if it fails. + * Returns 1 if the RSA key passes the pairwise test or 0 it it fails. */ int ossl_rsa_sp800_56b_pairwise_test(RSA *rsa, BN_CTX *ctx) { diff --git a/openssl/src/crypto/rsa/rsa_x931g.c b/openssl/src/crypto/rsa/rsa_x931g.c index 290e95b46..5a309a98c 100644 --- a/openssl/src/crypto/rsa/rsa_x931g.c +++ b/openssl/src/crypto/rsa/rsa_x931g.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -31,10 +31,10 @@ int RSA_X931_derive_ex(RSA *rsa, BIGNUM *p1, BIGNUM *p2, BIGNUM *q1, BN_CTX *ctx = NULL, *ctx2 = NULL; int ret = 0; - if (rsa == NULL) + if (!rsa) goto err; - ctx = BN_CTX_new_ex(rsa->libctx); + ctx = BN_CTX_new(); if (ctx == NULL) goto err; BN_CTX_start(ctx); @@ -145,6 +145,7 @@ int RSA_X931_derive_ex(RSA *rsa, BIGNUM *p1, BIGNUM *p2, BIGNUM *q1, BN_CTX_free(ctx2); return ret; + } int RSA_X931_generate_key_ex(RSA *rsa, int bits, const BIGNUM *e, @@ -154,7 +155,7 @@ int RSA_X931_generate_key_ex(RSA *rsa, int bits, const BIGNUM *e, BIGNUM *Xp = NULL, *Xq = NULL; BN_CTX *ctx = NULL; - ctx = BN_CTX_new_ex(rsa->libctx); + ctx = BN_CTX_new(); if (ctx == NULL) goto error; diff --git a/openssl/src/crypto/s390x_arch.h b/openssl/src/crypto/s390x_arch.h index fdc682af0..a7bde67d9 100644 --- a/openssl/src/crypto/s390x_arch.h +++ b/openssl/src/crypto/s390x_arch.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,8 +12,6 @@ # ifndef __ASSEMBLER__ -#include "crypto/bn.h" - void s390x_kimd(const unsigned char *in, size_t len, unsigned int fc, void *param); void s390x_klmd(const unsigned char *in, size_t inlen, unsigned char *out, @@ -79,13 +77,6 @@ __attribute__ ((visibility("hidden"))) #endif extern struct OPENSSL_s390xcap_st OPENSSL_s390xcap_P; -#ifdef S390X_MOD_EXP -# if defined(__GNUC__) && defined(__linux) -__attribute__ ((visibility("hidden"))) -# endif -extern int OPENSSL_s390xcex; -#endif - /* Max number of 64-bit words currently returned by STFLE */ # define S390X_STFLE_MAX 3 @@ -134,10 +125,6 @@ extern int OPENSSL_s390xcex; # define S390X_SHA3_256 33 # define S390X_SHA3_384 34 # define S390X_SHA3_512 35 -# define S390X_KECCAK_224 32 -# define S390X_KECCAK_256 33 -# define S390X_KECCAK_384 34 -# define S390X_KECCAK_512 35 # define S390X_SHAKE_128 36 # define S390X_SHAKE_256 37 # define S390X_GHASH 65 @@ -182,6 +169,5 @@ extern int OPENSSL_s390xcex; # define S390X_KMA_LAAD 0x200 # define S390X_KMA_HS 0x400 # define S390X_KDSA_D 0x80 -# define S390X_KLMD_PS 0x100 #endif diff --git a/openssl/src/crypto/s390xcap.c b/openssl/src/crypto/s390xcap.c index 7721b5c80..ea38ff8f0 100644 --- a/openssl/src/crypto/s390xcap.c +++ b/openssl/src/crypto/s390xcap.c @@ -1,5 +1,5 @@ /* - * Copyright 2010-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -16,15 +16,6 @@ #include "crypto/ctype.h" #include "s390x_arch.h" -#if defined(OPENSSL_SYS_LINUX) && !defined(FIPS_MODULE) -# include -# include -# include -# include -# include -# include -#endif - #if defined(__GLIBC__) && defined(__GLIBC_PREREQ) # if __GLIBC_PREREQ(2, 16) # include @@ -57,16 +48,14 @@ cap->NAME[1] = ~cap->NAME[1]; \ } -#define TOK_CPU_ALIAS(NAME, STRUCT_NAME) \ +#define TOK_CPU(NAME) \ (sscanf(tok_begin, \ " %" STR(LEN) "s %" STR(LEN) "s ", \ tok[0], tok[1]) == 1 \ && !strcmp(tok[0], #NAME)) { \ - memcpy(cap, &STRUCT_NAME, sizeof(*cap)); \ + memcpy(cap, &NAME, sizeof(*cap)); \ } -#define TOK_CPU(NAME) TOK_CPU_ALIAS(NAME, NAME) - #ifndef OSSL_IMPLEMENT_GETAUXVAL static sigjmp_buf ill_jmp; static void ill_handler(int sig) @@ -78,41 +67,19 @@ void OPENSSL_vx_probe(void); #endif static const char *env; -static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex); +static int parse_env(struct OPENSSL_s390xcap_st *cap); void OPENSSL_s390x_facilities(void); void OPENSSL_s390x_functions(void); struct OPENSSL_s390xcap_st OPENSSL_s390xcap_P; -#ifdef S390X_MOD_EXP -static int probe_cex(void); -int OPENSSL_s390xcex; - -#if defined(__GNUC__) -__attribute__ ((visibility("hidden"))) -#endif -void OPENSSL_s390x_cleanup(void); - -#if defined(__GNUC__) -__attribute__ ((visibility("hidden"))) -#endif -void OPENSSL_s390x_cleanup(void) -{ - if (OPENSSL_s390xcex != -1) { - (void)close(OPENSSL_s390xcex); - OPENSSL_s390xcex = -1; - } -} -#endif - #if defined(__GNUC__) && defined(__linux) __attribute__ ((visibility("hidden"))) #endif void OPENSSL_cpuid_setup(void) { struct OPENSSL_s390xcap_st cap; - int cex = 1; if (OPENSSL_s390xcap_P.stfle[0]) return; @@ -173,7 +140,7 @@ void OPENSSL_cpuid_setup(void) env = getenv("OPENSSL_s390xcap"); if (env != NULL) { - if (!parse_env(&cap, &cex)) + if (!parse_env(&cap)) env = NULL; } @@ -211,52 +178,9 @@ void OPENSSL_cpuid_setup(void) OPENSSL_s390xcap_P.kdsa[0] &= cap.kdsa[0]; OPENSSL_s390xcap_P.kdsa[1] &= cap.kdsa[1]; } - -#ifdef S390X_MOD_EXP - if (cex == 0) { - OPENSSL_s390xcex = -1; - } else { - OPENSSL_s390xcex = open("/dev/z90crypt", O_RDWR | O_CLOEXEC); - if (probe_cex() == 1) - OPENSSL_atexit(OPENSSL_s390x_cleanup); - } -#endif } -#ifdef S390X_MOD_EXP -static int probe_cex(void) -{ - struct ica_rsa_modexpo me; - const unsigned char inval[16] = { - 0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,2 - }; - const unsigned char modulus[16] = { - 0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,3 - }; - unsigned char res[16]; - int olderrno; - int rc = 1; - - me.inputdata = (unsigned char *)inval; - me.inputdatalength = sizeof(inval); - me.outputdata = (unsigned char *)res; - me.outputdatalength = sizeof(res); - me.b_key = (unsigned char *)inval; - me.n_modulus = (unsigned char *)modulus; - olderrno = errno; - if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) == -1) { - (void)close(OPENSSL_s390xcex); - OPENSSL_s390xcex = -1; - rc = 0; - } - errno = olderrno; - return rc; -} -#endif - -static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex) +static int parse_env(struct OPENSSL_s390xcap_st *cap) { /*- * CPU model data @@ -746,11 +670,6 @@ static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex) 0ULL}, }; - /*- - * z16 (2022) - z/Architecture POP - * Implements MSA and MSA1-9 (same as z15, no need to repeat). - */ - char *tok_begin, *tok_end, *buff, tok[S390X_STFLE_MAX][LEN + 1]; int rc, off, i, n; @@ -805,14 +724,6 @@ static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex) else if TOK_CPU(z13) else if TOK_CPU(z14) else if TOK_CPU(z15) - else if TOK_CPU_ALIAS(z16, z15) - - /* nocex to deactivate cex support */ - else if (sscanf(tok_begin, " %" STR(LEN) "s %" STR(LEN) "s ", - tok[0], tok[1]) == 1 - && !strcmp(tok[0], "nocex")) { - *cex = 0; - } /* whitespace(ignored) or invalid tokens */ else { diff --git a/openssl/src/crypto/seed/seed.c b/openssl/src/crypto/seed/seed.c deleted file mode 100644 index de3b3958e..000000000 --- a/openssl/src/crypto/seed/seed.c +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Copyright (c) 2007 KISA(Korea Information Security Agency). All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Neither the name of author nor the names of its contributors may - * be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ -#ifndef OPENSSL_NO_SEED - -/* - * SEED low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -# include -# include -# include -# ifdef _WIN32 -# include -# endif - -# include -# include "seed_local.h" - -# ifdef SS /* can get defined on Solaris by inclusion of - * */ -# undef SS -# endif - -# if !defined(OPENSSL_SMALL_FOOTPRINT) - -# define G_FUNC(v) \ - SS[0][(unsigned char) (v) & 0xff] ^ \ - SS[1][(unsigned char) ((v)>>8) & 0xff] ^ \ - SS[2][(unsigned char)((v)>>16) & 0xff] ^ \ - SS[3][(unsigned char)((v)>>24) & 0xff] - -static const seed_word SS[4][256] = { - { 0x2989a1a8, 0x05858184, 0x16c6d2d4, 0x13c3d3d0, - 0x14445054, 0x1d0d111c, 0x2c8ca0ac, 0x25052124, - 0x1d4d515c, 0x03434340, 0x18081018, 0x1e0e121c, - 0x11415150, 0x3cccf0fc, 0x0acac2c8, 0x23436360, - 0x28082028, 0x04444044, 0x20002020, 0x1d8d919c, - 0x20c0e0e0, 0x22c2e2e0, 0x08c8c0c8, 0x17071314, - 0x2585a1a4, 0x0f8f838c, 0x03030300, 0x3b4b7378, - 0x3b8bb3b8, 0x13031310, 0x12c2d2d0, 0x2ecee2ec, - 0x30407070, 0x0c8c808c, 0x3f0f333c, 0x2888a0a8, - 0x32023230, 0x1dcdd1dc, 0x36c6f2f4, 0x34447074, - 0x2ccce0ec, 0x15859194, 0x0b0b0308, 0x17475354, - 0x1c4c505c, 0x1b4b5358, 0x3d8db1bc, 0x01010100, - 0x24042024, 0x1c0c101c, 0x33437370, 0x18889098, - 0x10001010, 0x0cccc0cc, 0x32c2f2f0, 0x19c9d1d8, - 0x2c0c202c, 0x27c7e3e4, 0x32427270, 0x03838380, - 0x1b8b9398, 0x11c1d1d0, 0x06868284, 0x09c9c1c8, - 0x20406060, 0x10405050, 0x2383a3a0, 0x2bcbe3e8, - 0x0d0d010c, 0x3686b2b4, 0x1e8e929c, 0x0f4f434c, - 0x3787b3b4, 0x1a4a5258, 0x06c6c2c4, 0x38487078, - 0x2686a2a4, 0x12021210, 0x2f8fa3ac, 0x15c5d1d4, - 0x21416160, 0x03c3c3c0, 0x3484b0b4, 0x01414140, - 0x12425250, 0x3d4d717c, 0x0d8d818c, 0x08080008, - 0x1f0f131c, 0x19899198, 0x00000000, 0x19091118, - 0x04040004, 0x13435350, 0x37c7f3f4, 0x21c1e1e0, - 0x3dcdf1fc, 0x36467274, 0x2f0f232c, 0x27072324, - 0x3080b0b0, 0x0b8b8388, 0x0e0e020c, 0x2b8ba3a8, - 0x2282a2a0, 0x2e4e626c, 0x13839390, 0x0d4d414c, - 0x29496168, 0x3c4c707c, 0x09090108, 0x0a0a0208, - 0x3f8fb3bc, 0x2fcfe3ec, 0x33c3f3f0, 0x05c5c1c4, - 0x07878384, 0x14041014, 0x3ecef2fc, 0x24446064, - 0x1eced2dc, 0x2e0e222c, 0x0b4b4348, 0x1a0a1218, - 0x06060204, 0x21012120, 0x2b4b6368, 0x26466264, - 0x02020200, 0x35c5f1f4, 0x12829290, 0x0a8a8288, - 0x0c0c000c, 0x3383b3b0, 0x3e4e727c, 0x10c0d0d0, - 0x3a4a7278, 0x07474344, 0x16869294, 0x25c5e1e4, - 0x26062224, 0x00808080, 0x2d8da1ac, 0x1fcfd3dc, - 0x2181a1a0, 0x30003030, 0x37073334, 0x2e8ea2ac, - 0x36063234, 0x15051114, 0x22022220, 0x38083038, - 0x34c4f0f4, 0x2787a3a4, 0x05454144, 0x0c4c404c, - 0x01818180, 0x29c9e1e8, 0x04848084, 0x17879394, - 0x35053134, 0x0bcbc3c8, 0x0ecec2cc, 0x3c0c303c, - 0x31417170, 0x11011110, 0x07c7c3c4, 0x09898188, - 0x35457174, 0x3bcbf3f8, 0x1acad2d8, 0x38c8f0f8, - 0x14849094, 0x19495158, 0x02828280, 0x04c4c0c4, - 0x3fcff3fc, 0x09494148, 0x39093138, 0x27476364, - 0x00c0c0c0, 0x0fcfc3cc, 0x17c7d3d4, 0x3888b0b8, - 0x0f0f030c, 0x0e8e828c, 0x02424240, 0x23032320, - 0x11819190, 0x2c4c606c, 0x1bcbd3d8, 0x2484a0a4, - 0x34043034, 0x31c1f1f0, 0x08484048, 0x02c2c2c0, - 0x2f4f636c, 0x3d0d313c, 0x2d0d212c, 0x00404040, - 0x3e8eb2bc, 0x3e0e323c, 0x3c8cb0bc, 0x01c1c1c0, - 0x2a8aa2a8, 0x3a8ab2b8, 0x0e4e424c, 0x15455154, - 0x3b0b3338, 0x1cccd0dc, 0x28486068, 0x3f4f737c, - 0x1c8c909c, 0x18c8d0d8, 0x0a4a4248, 0x16465254, - 0x37477374, 0x2080a0a0, 0x2dcde1ec, 0x06464244, - 0x3585b1b4, 0x2b0b2328, 0x25456164, 0x3acaf2f8, - 0x23c3e3e0, 0x3989b1b8, 0x3181b1b0, 0x1f8f939c, - 0x1e4e525c, 0x39c9f1f8, 0x26c6e2e4, 0x3282b2b0, - 0x31013130, 0x2acae2e8, 0x2d4d616c, 0x1f4f535c, - 0x24c4e0e4, 0x30c0f0f0, 0x0dcdc1cc, 0x08888088, - 0x16061214, 0x3a0a3238, 0x18485058, 0x14c4d0d4, - 0x22426260, 0x29092128, 0x07070304, 0x33033330, - 0x28c8e0e8, 0x1b0b1318, 0x05050104, 0x39497178, - 0x10809090, 0x2a4a6268, 0x2a0a2228, 0x1a8a9298 - }, - { 0x38380830, 0xe828c8e0, 0x2c2d0d21, 0xa42686a2, - 0xcc0fcfc3, 0xdc1eced2, 0xb03383b3, 0xb83888b0, - 0xac2f8fa3, 0x60204060, 0x54154551, 0xc407c7c3, - 0x44044440, 0x6c2f4f63, 0x682b4b63, 0x581b4b53, - 0xc003c3c3, 0x60224262, 0x30330333, 0xb43585b1, - 0x28290921, 0xa02080a0, 0xe022c2e2, 0xa42787a3, - 0xd013c3d3, 0x90118191, 0x10110111, 0x04060602, - 0x1c1c0c10, 0xbc3c8cb0, 0x34360632, 0x480b4b43, - 0xec2fcfe3, 0x88088880, 0x6c2c4c60, 0xa82888a0, - 0x14170713, 0xc404c4c0, 0x14160612, 0xf434c4f0, - 0xc002c2c2, 0x44054541, 0xe021c1e1, 0xd416c6d2, - 0x3c3f0f33, 0x3c3d0d31, 0x8c0e8e82, 0x98188890, - 0x28280820, 0x4c0e4e42, 0xf436c6f2, 0x3c3e0e32, - 0xa42585a1, 0xf839c9f1, 0x0c0d0d01, 0xdc1fcfd3, - 0xd818c8d0, 0x282b0b23, 0x64264662, 0x783a4a72, - 0x24270723, 0x2c2f0f23, 0xf031c1f1, 0x70324272, - 0x40024242, 0xd414c4d0, 0x40014141, 0xc000c0c0, - 0x70334373, 0x64274763, 0xac2c8ca0, 0x880b8b83, - 0xf437c7f3, 0xac2d8da1, 0x80008080, 0x1c1f0f13, - 0xc80acac2, 0x2c2c0c20, 0xa82a8aa2, 0x34340430, - 0xd012c2d2, 0x080b0b03, 0xec2ecee2, 0xe829c9e1, - 0x5c1d4d51, 0x94148490, 0x18180810, 0xf838c8f0, - 0x54174753, 0xac2e8ea2, 0x08080800, 0xc405c5c1, - 0x10130313, 0xcc0dcdc1, 0x84068682, 0xb83989b1, - 0xfc3fcff3, 0x7c3d4d71, 0xc001c1c1, 0x30310131, - 0xf435c5f1, 0x880a8a82, 0x682a4a62, 0xb03181b1, - 0xd011c1d1, 0x20200020, 0xd417c7d3, 0x00020202, - 0x20220222, 0x04040400, 0x68284860, 0x70314171, - 0x04070703, 0xd81bcbd3, 0x9c1d8d91, 0x98198991, - 0x60214161, 0xbc3e8eb2, 0xe426c6e2, 0x58194951, - 0xdc1dcdd1, 0x50114151, 0x90108090, 0xdc1cccd0, - 0x981a8a92, 0xa02383a3, 0xa82b8ba3, 0xd010c0d0, - 0x80018181, 0x0c0f0f03, 0x44074743, 0x181a0a12, - 0xe023c3e3, 0xec2ccce0, 0x8c0d8d81, 0xbc3f8fb3, - 0x94168692, 0x783b4b73, 0x5c1c4c50, 0xa02282a2, - 0xa02181a1, 0x60234363, 0x20230323, 0x4c0d4d41, - 0xc808c8c0, 0x9c1e8e92, 0x9c1c8c90, 0x383a0a32, - 0x0c0c0c00, 0x2c2e0e22, 0xb83a8ab2, 0x6c2e4e62, - 0x9c1f8f93, 0x581a4a52, 0xf032c2f2, 0x90128292, - 0xf033c3f3, 0x48094941, 0x78384870, 0xcc0cccc0, - 0x14150511, 0xf83bcbf3, 0x70304070, 0x74354571, - 0x7c3f4f73, 0x34350531, 0x10100010, 0x00030303, - 0x64244460, 0x6c2d4d61, 0xc406c6c2, 0x74344470, - 0xd415c5d1, 0xb43484b0, 0xe82acae2, 0x08090901, - 0x74364672, 0x18190911, 0xfc3ecef2, 0x40004040, - 0x10120212, 0xe020c0e0, 0xbc3d8db1, 0x04050501, - 0xf83acaf2, 0x00010101, 0xf030c0f0, 0x282a0a22, - 0x5c1e4e52, 0xa82989a1, 0x54164652, 0x40034343, - 0x84058581, 0x14140410, 0x88098981, 0x981b8b93, - 0xb03080b0, 0xe425c5e1, 0x48084840, 0x78394971, - 0x94178793, 0xfc3cccf0, 0x1c1e0e12, 0x80028282, - 0x20210121, 0x8c0c8c80, 0x181b0b13, 0x5c1f4f53, - 0x74374773, 0x54144450, 0xb03282b2, 0x1c1d0d11, - 0x24250521, 0x4c0f4f43, 0x00000000, 0x44064642, - 0xec2dcde1, 0x58184850, 0x50124252, 0xe82bcbe3, - 0x7c3e4e72, 0xd81acad2, 0xc809c9c1, 0xfc3dcdf1, - 0x30300030, 0x94158591, 0x64254561, 0x3c3c0c30, - 0xb43686b2, 0xe424c4e0, 0xb83b8bb3, 0x7c3c4c70, - 0x0c0e0e02, 0x50104050, 0x38390931, 0x24260622, - 0x30320232, 0x84048480, 0x68294961, 0x90138393, - 0x34370733, 0xe427c7e3, 0x24240420, 0xa42484a0, - 0xc80bcbc3, 0x50134353, 0x080a0a02, 0x84078783, - 0xd819c9d1, 0x4c0c4c40, 0x80038383, 0x8c0f8f83, - 0xcc0ecec2, 0x383b0b33, 0x480a4a42, 0xb43787b3 - }, - { 0xa1a82989, 0x81840585, 0xd2d416c6, 0xd3d013c3, - 0x50541444, 0x111c1d0d, 0xa0ac2c8c, 0x21242505, - 0x515c1d4d, 0x43400343, 0x10181808, 0x121c1e0e, - 0x51501141, 0xf0fc3ccc, 0xc2c80aca, 0x63602343, - 0x20282808, 0x40440444, 0x20202000, 0x919c1d8d, - 0xe0e020c0, 0xe2e022c2, 0xc0c808c8, 0x13141707, - 0xa1a42585, 0x838c0f8f, 0x03000303, 0x73783b4b, - 0xb3b83b8b, 0x13101303, 0xd2d012c2, 0xe2ec2ece, - 0x70703040, 0x808c0c8c, 0x333c3f0f, 0xa0a82888, - 0x32303202, 0xd1dc1dcd, 0xf2f436c6, 0x70743444, - 0xe0ec2ccc, 0x91941585, 0x03080b0b, 0x53541747, - 0x505c1c4c, 0x53581b4b, 0xb1bc3d8d, 0x01000101, - 0x20242404, 0x101c1c0c, 0x73703343, 0x90981888, - 0x10101000, 0xc0cc0ccc, 0xf2f032c2, 0xd1d819c9, - 0x202c2c0c, 0xe3e427c7, 0x72703242, 0x83800383, - 0x93981b8b, 0xd1d011c1, 0x82840686, 0xc1c809c9, - 0x60602040, 0x50501040, 0xa3a02383, 0xe3e82bcb, - 0x010c0d0d, 0xb2b43686, 0x929c1e8e, 0x434c0f4f, - 0xb3b43787, 0x52581a4a, 0xc2c406c6, 0x70783848, - 0xa2a42686, 0x12101202, 0xa3ac2f8f, 0xd1d415c5, - 0x61602141, 0xc3c003c3, 0xb0b43484, 0x41400141, - 0x52501242, 0x717c3d4d, 0x818c0d8d, 0x00080808, - 0x131c1f0f, 0x91981989, 0x00000000, 0x11181909, - 0x00040404, 0x53501343, 0xf3f437c7, 0xe1e021c1, - 0xf1fc3dcd, 0x72743646, 0x232c2f0f, 0x23242707, - 0xb0b03080, 0x83880b8b, 0x020c0e0e, 0xa3a82b8b, - 0xa2a02282, 0x626c2e4e, 0x93901383, 0x414c0d4d, - 0x61682949, 0x707c3c4c, 0x01080909, 0x02080a0a, - 0xb3bc3f8f, 0xe3ec2fcf, 0xf3f033c3, 0xc1c405c5, - 0x83840787, 0x10141404, 0xf2fc3ece, 0x60642444, - 0xd2dc1ece, 0x222c2e0e, 0x43480b4b, 0x12181a0a, - 0x02040606, 0x21202101, 0x63682b4b, 0x62642646, - 0x02000202, 0xf1f435c5, 0x92901282, 0x82880a8a, - 0x000c0c0c, 0xb3b03383, 0x727c3e4e, 0xd0d010c0, - 0x72783a4a, 0x43440747, 0x92941686, 0xe1e425c5, - 0x22242606, 0x80800080, 0xa1ac2d8d, 0xd3dc1fcf, - 0xa1a02181, 0x30303000, 0x33343707, 0xa2ac2e8e, - 0x32343606, 0x11141505, 0x22202202, 0x30383808, - 0xf0f434c4, 0xa3a42787, 0x41440545, 0x404c0c4c, - 0x81800181, 0xe1e829c9, 0x80840484, 0x93941787, - 0x31343505, 0xc3c80bcb, 0xc2cc0ece, 0x303c3c0c, - 0x71703141, 0x11101101, 0xc3c407c7, 0x81880989, - 0x71743545, 0xf3f83bcb, 0xd2d81aca, 0xf0f838c8, - 0x90941484, 0x51581949, 0x82800282, 0xc0c404c4, - 0xf3fc3fcf, 0x41480949, 0x31383909, 0x63642747, - 0xc0c000c0, 0xc3cc0fcf, 0xd3d417c7, 0xb0b83888, - 0x030c0f0f, 0x828c0e8e, 0x42400242, 0x23202303, - 0x91901181, 0x606c2c4c, 0xd3d81bcb, 0xa0a42484, - 0x30343404, 0xf1f031c1, 0x40480848, 0xc2c002c2, - 0x636c2f4f, 0x313c3d0d, 0x212c2d0d, 0x40400040, - 0xb2bc3e8e, 0x323c3e0e, 0xb0bc3c8c, 0xc1c001c1, - 0xa2a82a8a, 0xb2b83a8a, 0x424c0e4e, 0x51541545, - 0x33383b0b, 0xd0dc1ccc, 0x60682848, 0x737c3f4f, - 0x909c1c8c, 0xd0d818c8, 0x42480a4a, 0x52541646, - 0x73743747, 0xa0a02080, 0xe1ec2dcd, 0x42440646, - 0xb1b43585, 0x23282b0b, 0x61642545, 0xf2f83aca, - 0xe3e023c3, 0xb1b83989, 0xb1b03181, 0x939c1f8f, - 0x525c1e4e, 0xf1f839c9, 0xe2e426c6, 0xb2b03282, - 0x31303101, 0xe2e82aca, 0x616c2d4d, 0x535c1f4f, - 0xe0e424c4, 0xf0f030c0, 0xc1cc0dcd, 0x80880888, - 0x12141606, 0x32383a0a, 0x50581848, 0xd0d414c4, - 0x62602242, 0x21282909, 0x03040707, 0x33303303, - 0xe0e828c8, 0x13181b0b, 0x01040505, 0x71783949, - 0x90901080, 0x62682a4a, 0x22282a0a, 0x92981a8a - }, - { 0x08303838, 0xc8e0e828, 0x0d212c2d, 0x86a2a426, - 0xcfc3cc0f, 0xced2dc1e, 0x83b3b033, 0x88b0b838, - 0x8fa3ac2f, 0x40606020, 0x45515415, 0xc7c3c407, - 0x44404404, 0x4f636c2f, 0x4b63682b, 0x4b53581b, - 0xc3c3c003, 0x42626022, 0x03333033, 0x85b1b435, - 0x09212829, 0x80a0a020, 0xc2e2e022, 0x87a3a427, - 0xc3d3d013, 0x81919011, 0x01111011, 0x06020406, - 0x0c101c1c, 0x8cb0bc3c, 0x06323436, 0x4b43480b, - 0xcfe3ec2f, 0x88808808, 0x4c606c2c, 0x88a0a828, - 0x07131417, 0xc4c0c404, 0x06121416, 0xc4f0f434, - 0xc2c2c002, 0x45414405, 0xc1e1e021, 0xc6d2d416, - 0x0f333c3f, 0x0d313c3d, 0x8e828c0e, 0x88909818, - 0x08202828, 0x4e424c0e, 0xc6f2f436, 0x0e323c3e, - 0x85a1a425, 0xc9f1f839, 0x0d010c0d, 0xcfd3dc1f, - 0xc8d0d818, 0x0b23282b, 0x46626426, 0x4a72783a, - 0x07232427, 0x0f232c2f, 0xc1f1f031, 0x42727032, - 0x42424002, 0xc4d0d414, 0x41414001, 0xc0c0c000, - 0x43737033, 0x47636427, 0x8ca0ac2c, 0x8b83880b, - 0xc7f3f437, 0x8da1ac2d, 0x80808000, 0x0f131c1f, - 0xcac2c80a, 0x0c202c2c, 0x8aa2a82a, 0x04303434, - 0xc2d2d012, 0x0b03080b, 0xcee2ec2e, 0xc9e1e829, - 0x4d515c1d, 0x84909414, 0x08101818, 0xc8f0f838, - 0x47535417, 0x8ea2ac2e, 0x08000808, 0xc5c1c405, - 0x03131013, 0xcdc1cc0d, 0x86828406, 0x89b1b839, - 0xcff3fc3f, 0x4d717c3d, 0xc1c1c001, 0x01313031, - 0xc5f1f435, 0x8a82880a, 0x4a62682a, 0x81b1b031, - 0xc1d1d011, 0x00202020, 0xc7d3d417, 0x02020002, - 0x02222022, 0x04000404, 0x48606828, 0x41717031, - 0x07030407, 0xcbd3d81b, 0x8d919c1d, 0x89919819, - 0x41616021, 0x8eb2bc3e, 0xc6e2e426, 0x49515819, - 0xcdd1dc1d, 0x41515011, 0x80909010, 0xccd0dc1c, - 0x8a92981a, 0x83a3a023, 0x8ba3a82b, 0xc0d0d010, - 0x81818001, 0x0f030c0f, 0x47434407, 0x0a12181a, - 0xc3e3e023, 0xcce0ec2c, 0x8d818c0d, 0x8fb3bc3f, - 0x86929416, 0x4b73783b, 0x4c505c1c, 0x82a2a022, - 0x81a1a021, 0x43636023, 0x03232023, 0x4d414c0d, - 0xc8c0c808, 0x8e929c1e, 0x8c909c1c, 0x0a32383a, - 0x0c000c0c, 0x0e222c2e, 0x8ab2b83a, 0x4e626c2e, - 0x8f939c1f, 0x4a52581a, 0xc2f2f032, 0x82929012, - 0xc3f3f033, 0x49414809, 0x48707838, 0xccc0cc0c, - 0x05111415, 0xcbf3f83b, 0x40707030, 0x45717435, - 0x4f737c3f, 0x05313435, 0x00101010, 0x03030003, - 0x44606424, 0x4d616c2d, 0xc6c2c406, 0x44707434, - 0xc5d1d415, 0x84b0b434, 0xcae2e82a, 0x09010809, - 0x46727436, 0x09111819, 0xcef2fc3e, 0x40404000, - 0x02121012, 0xc0e0e020, 0x8db1bc3d, 0x05010405, - 0xcaf2f83a, 0x01010001, 0xc0f0f030, 0x0a22282a, - 0x4e525c1e, 0x89a1a829, 0x46525416, 0x43434003, - 0x85818405, 0x04101414, 0x89818809, 0x8b93981b, - 0x80b0b030, 0xc5e1e425, 0x48404808, 0x49717839, - 0x87939417, 0xccf0fc3c, 0x0e121c1e, 0x82828002, - 0x01212021, 0x8c808c0c, 0x0b13181b, 0x4f535c1f, - 0x47737437, 0x44505414, 0x82b2b032, 0x0d111c1d, - 0x05212425, 0x4f434c0f, 0x00000000, 0x46424406, - 0xcde1ec2d, 0x48505818, 0x42525012, 0xcbe3e82b, - 0x4e727c3e, 0xcad2d81a, 0xc9c1c809, 0xcdf1fc3d, - 0x00303030, 0x85919415, 0x45616425, 0x0c303c3c, - 0x86b2b436, 0xc4e0e424, 0x8bb3b83b, 0x4c707c3c, - 0x0e020c0e, 0x40505010, 0x09313839, 0x06222426, - 0x02323032, 0x84808404, 0x49616829, 0x83939013, - 0x07333437, 0xc7e3e427, 0x04202424, 0x84a0a424, - 0xcbc3c80b, 0x43535013, 0x0a02080a, 0x87838407, - 0xc9d1d819, 0x4c404c0c, 0x83838003, 0x8f838c0f, - 0xcec2cc0e, 0x0b33383b, 0x4a42480a, 0x87b3b437 - } -}; - -#else - -/* on x86_64 >5x size reduction at 40% performance penalty */ -static const unsigned char SEED_Sbox[2][256] = { -{ - 0xA9, 0x85, 0xD6, 0xD3, 0x54, 0x1D, 0xAC, 0x25, - 0x5D, 0x43, 0x18, 0x1E, 0x51, 0xFC, 0xCA, 0x63, - 0x28, 0x44, 0x20, 0x9D, 0xE0, 0xE2, 0xC8, 0x17, - 0xA5, 0x8F, 0x03, 0x7B, 0xBB, 0x13, 0xD2, 0xEE, - 0x70, 0x8C, 0x3F, 0xA8, 0x32, 0xDD, 0xF6, 0x74, - 0xEC, 0x95, 0x0B, 0x57, 0x5C, 0x5B, 0xBD, 0x01, - 0x24, 0x1C, 0x73, 0x98, 0x10, 0xCC, 0xF2, 0xD9, - 0x2C, 0xE7, 0x72, 0x83, 0x9B, 0xD1, 0x86, 0xC9, - 0x60, 0x50, 0xA3, 0xEB, 0x0D, 0xB6, 0x9E, 0x4F, - 0xB7, 0x5A, 0xC6, 0x78, 0xA6, 0x12, 0xAF, 0xD5, - 0x61, 0xC3, 0xB4, 0x41, 0x52, 0x7D, 0x8D, 0x08, - 0x1F, 0x99, 0x00, 0x19, 0x04, 0x53, 0xF7, 0xE1, - 0xFD, 0x76, 0x2F, 0x27, 0xB0, 0x8B, 0x0E, 0xAB, - 0xA2, 0x6E, 0x93, 0x4D, 0x69, 0x7C, 0x09, 0x0A, - 0xBF, 0xEF, 0xF3, 0xC5, 0x87, 0x14, 0xFE, 0x64, - 0xDE, 0x2E, 0x4B, 0x1A, 0x06, 0x21, 0x6B, 0x66, - 0x02, 0xF5, 0x92, 0x8A, 0x0C, 0xB3, 0x7E, 0xD0, - 0x7A, 0x47, 0x96, 0xE5, 0x26, 0x80, 0xAD, 0xDF, - 0xA1, 0x30, 0x37, 0xAE, 0x36, 0x15, 0x22, 0x38, - 0xF4, 0xA7, 0x45, 0x4C, 0x81, 0xE9, 0x84, 0x97, - 0x35, 0xCB, 0xCE, 0x3C, 0x71, 0x11, 0xC7, 0x89, - 0x75, 0xFB, 0xDA, 0xF8, 0x94, 0x59, 0x82, 0xC4, - 0xFF, 0x49, 0x39, 0x67, 0xC0, 0xCF, 0xD7, 0xB8, - 0x0F, 0x8E, 0x42, 0x23, 0x91, 0x6C, 0xDB, 0xA4, - 0x34, 0xF1, 0x48, 0xC2, 0x6F, 0x3D, 0x2D, 0x40, - 0xBE, 0x3E, 0xBC, 0xC1, 0xAA, 0xBA, 0x4E, 0x55, - 0x3B, 0xDC, 0x68, 0x7F, 0x9C, 0xD8, 0x4A, 0x56, - 0x77, 0xA0, 0xED, 0x46, 0xB5, 0x2B, 0x65, 0xFA, - 0xE3, 0xB9, 0xB1, 0x9F, 0x5E, 0xF9, 0xE6, 0xB2, - 0x31, 0xEA, 0x6D, 0x5F, 0xE4, 0xF0, 0xCD, 0x88, - 0x16, 0x3A, 0x58, 0xD4, 0x62, 0x29, 0x07, 0x33, - 0xE8, 0x1B, 0x05, 0x79, 0x90, 0x6A, 0x2A, 0x9A - }, - { - 0x38, 0xE8, 0x2D, 0xA6, 0xCF, 0xDE, 0xB3, 0xB8, - 0xAF, 0x60, 0x55, 0xC7, 0x44, 0x6F, 0x6B, 0x5B, - 0xC3, 0x62, 0x33, 0xB5, 0x29, 0xA0, 0xE2, 0xA7, - 0xD3, 0x91, 0x11, 0x06, 0x1C, 0xBC, 0x36, 0x4B, - 0xEF, 0x88, 0x6C, 0xA8, 0x17, 0xC4, 0x16, 0xF4, - 0xC2, 0x45, 0xE1, 0xD6, 0x3F, 0x3D, 0x8E, 0x98, - 0x28, 0x4E, 0xF6, 0x3E, 0xA5, 0xF9, 0x0D, 0xDF, - 0xD8, 0x2B, 0x66, 0x7A, 0x27, 0x2F, 0xF1, 0x72, - 0x42, 0xD4, 0x41, 0xC0, 0x73, 0x67, 0xAC, 0x8B, - 0xF7, 0xAD, 0x80, 0x1F, 0xCA, 0x2C, 0xAA, 0x34, - 0xD2, 0x0B, 0xEE, 0xE9, 0x5D, 0x94, 0x18, 0xF8, - 0x57, 0xAE, 0x08, 0xC5, 0x13, 0xCD, 0x86, 0xB9, - 0xFF, 0x7D, 0xC1, 0x31, 0xF5, 0x8A, 0x6A, 0xB1, - 0xD1, 0x20, 0xD7, 0x02, 0x22, 0x04, 0x68, 0x71, - 0x07, 0xDB, 0x9D, 0x99, 0x61, 0xBE, 0xE6, 0x59, - 0xDD, 0x51, 0x90, 0xDC, 0x9A, 0xA3, 0xAB, 0xD0, - 0x81, 0x0F, 0x47, 0x1A, 0xE3, 0xEC, 0x8D, 0xBF, - 0x96, 0x7B, 0x5C, 0xA2, 0xA1, 0x63, 0x23, 0x4D, - 0xC8, 0x9E, 0x9C, 0x3A, 0x0C, 0x2E, 0xBA, 0x6E, - 0x9F, 0x5A, 0xF2, 0x92, 0xF3, 0x49, 0x78, 0xCC, - 0x15, 0xFB, 0x70, 0x75, 0x7F, 0x35, 0x10, 0x03, - 0x64, 0x6D, 0xC6, 0x74, 0xD5, 0xB4, 0xEA, 0x09, - 0x76, 0x19, 0xFE, 0x40, 0x12, 0xE0, 0xBD, 0x05, - 0xFA, 0x01, 0xF0, 0x2A, 0x5E, 0xA9, 0x56, 0x43, - 0x85, 0x14, 0x89, 0x9B, 0xB0, 0xE5, 0x48, 0x79, - 0x97, 0xFC, 0x1E, 0x82, 0x21, 0x8C, 0x1B, 0x5F, - 0x77, 0x54, 0xB2, 0x1D, 0x25, 0x4F, 0x00, 0x46, - 0xED, 0x58, 0x52, 0xEB, 0x7E, 0xDA, 0xC9, 0xFD, - 0x30, 0x95, 0x65, 0x3C, 0xB6, 0xE4, 0xBB, 0x7C, - 0x0E, 0x50, 0x39, 0x26, 0x32, 0x84, 0x69, 0x93, - 0x37, 0xE7, 0x24, 0xA4, 0xCB, 0x53, 0x0A, 0x87, - 0xD9, 0x4C, 0x83, 0x8F, 0xCE, 0x3B, 0x4A, 0xB7 - } -}; - -static unsigned int G_FUNC(unsigned int v) -{ - unsigned int s0, s1, s2, s3, ret; - - s0 = SEED_Sbox[0][(unsigned char) (v) & 0xff]; - s1 = SEED_Sbox[1][(unsigned char)((v)>> 8) & 0xff]; - s2 = SEED_Sbox[0][(unsigned char)((v)>>16) & 0xff]; - s3 = SEED_Sbox[1][(unsigned char)((v)>>24) & 0xff]; - - ret = ((s0 & 0xFC) ^ (s1 & 0xF3) ^ (s2 & 0xCF) ^ (s3 & 0x3F)); - ret |= ((s0 & 0xF3) ^ (s1 & 0xCF) ^ (s2 & 0x3F) ^ (s3 & 0xFC)) << 8; - ret |= ((s0 & 0xCF) ^ (s1 & 0x3F) ^ (s2 & 0xFC) ^ (s3 & 0xF3)) << 16; - ret |= ((s0 & 0x3F) ^ (s1 & 0xFC) ^ (s2 & 0xF3) ^ (s3 & 0xCF)) << 24; - - return ret; -} -# endif - -/* key schedule constants - golden ratio */ -# define KC0 0x9e3779b9 -# define KC1 0x3c6ef373 -# define KC2 0x78dde6e6 -# define KC3 0xf1bbcdcc -# define KC4 0xe3779b99 -# define KC5 0xc6ef3733 -# define KC6 0x8dde6e67 -# define KC7 0x1bbcdccf -# define KC8 0x3779b99e -# define KC9 0x6ef3733c -# define KC10 0xdde6e678 -# define KC11 0xbbcdccf1 -# define KC12 0x779b99e3 -# define KC13 0xef3733c6 -# define KC14 0xde6e678d -# define KC15 0xbcdccf1b - -# if defined(OPENSSL_SMALL_FOOTPRINT) -static const seed_word KC[] = { - KC0, KC1, KC2, KC3, KC4, KC5, KC6, KC7, - KC8, KC9, KC10, KC11, KC12, KC13, KC14, KC15 -}; -# endif - -void SEED_set_key(const unsigned char rawkey[SEED_KEY_LENGTH], - SEED_KEY_SCHEDULE *ks) -{ - seed_word x1, x2, x3, x4; - seed_word t0, t1; - - char2word(rawkey, x1); - char2word(rawkey + 4, x2); - char2word(rawkey + 8, x3); - char2word(rawkey + 12, x4); - - t0 = (x1 + x3 - KC0) & 0xffffffff; - t1 = (x2 - x4 + KC0) & 0xffffffff; - KEYUPDATE_TEMP(t0, t1, &ks->data[0]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC1); - KEYUPDATE_TEMP(t0, t1, &ks->data[2]); - -# if !defined(OPENSSL_SMALL_FOOTPRINT) - KEYSCHEDULE_UPDATE0(t0, t1, x1, x2, x3, x4, KC2); - KEYUPDATE_TEMP(t0, t1, &ks->data[4]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC3); - KEYUPDATE_TEMP(t0, t1, &ks->data[6]); - KEYSCHEDULE_UPDATE0(t0, t1, x1, x2, x3, x4, KC4); - KEYUPDATE_TEMP(t0, t1, &ks->data[8]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC5); - KEYUPDATE_TEMP(t0, t1, &ks->data[10]); - KEYSCHEDULE_UPDATE0(t0, t1, x1, x2, x3, x4, KC6); - KEYUPDATE_TEMP(t0, t1, &ks->data[12]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC7); - KEYUPDATE_TEMP(t0, t1, &ks->data[14]); - KEYSCHEDULE_UPDATE0(t0, t1, x1, x2, x3, x4, KC8); - KEYUPDATE_TEMP(t0, t1, &ks->data[16]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC9); - KEYUPDATE_TEMP(t0, t1, &ks->data[18]); - KEYSCHEDULE_UPDATE0(t0, t1, x1, x2, x3, x4, KC10); - KEYUPDATE_TEMP(t0, t1, &ks->data[20]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC11); - KEYUPDATE_TEMP(t0, t1, &ks->data[22]); - KEYSCHEDULE_UPDATE0(t0, t1, x1, x2, x3, x4, KC12); - KEYUPDATE_TEMP(t0, t1, &ks->data[24]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC13); - KEYUPDATE_TEMP(t0, t1, &ks->data[26]); - KEYSCHEDULE_UPDATE0(t0, t1, x1, x2, x3, x4, KC14); - KEYUPDATE_TEMP(t0, t1, &ks->data[28]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC15); - KEYUPDATE_TEMP(t0, t1, &ks->data[30]); -# else - { - int i; - for (i = 2; i < 16; i += 2) { - KEYSCHEDULE_UPDATE0(t0, t1, x1, x2, x3, x4, KC[i]); - KEYUPDATE_TEMP(t0, t1, &ks->data[i * 2]); - KEYSCHEDULE_UPDATE1(t0, t1, x1, x2, x3, x4, KC[i + 1]); - KEYUPDATE_TEMP(t0, t1, &ks->data[i * 2 + 2]); - } - } -# endif -} - -void SEED_encrypt(const unsigned char s[SEED_BLOCK_SIZE], - unsigned char d[SEED_BLOCK_SIZE], - const SEED_KEY_SCHEDULE *ks) -{ - seed_word x1, x2, x3, x4; - seed_word t0, t1; - - char2word(s, x1); - char2word(s + 4, x2); - char2word(s + 8, x3); - char2word(s + 12, x4); - -# if !defined(OPENSSL_SMALL_FOOTPRINT) - E_SEED(t0, t1, x1, x2, x3, x4, 0); - E_SEED(t0, t1, x3, x4, x1, x2, 2); - E_SEED(t0, t1, x1, x2, x3, x4, 4); - E_SEED(t0, t1, x3, x4, x1, x2, 6); - E_SEED(t0, t1, x1, x2, x3, x4, 8); - E_SEED(t0, t1, x3, x4, x1, x2, 10); - E_SEED(t0, t1, x1, x2, x3, x4, 12); - E_SEED(t0, t1, x3, x4, x1, x2, 14); - E_SEED(t0, t1, x1, x2, x3, x4, 16); - E_SEED(t0, t1, x3, x4, x1, x2, 18); - E_SEED(t0, t1, x1, x2, x3, x4, 20); - E_SEED(t0, t1, x3, x4, x1, x2, 22); - E_SEED(t0, t1, x1, x2, x3, x4, 24); - E_SEED(t0, t1, x3, x4, x1, x2, 26); - E_SEED(t0, t1, x1, x2, x3, x4, 28); - E_SEED(t0, t1, x3, x4, x1, x2, 30); -# else - { - int i; - for (i = 0; i < 30; i += 4) { - E_SEED(t0, t1, x1, x2, x3, x4, i); - E_SEED(t0, t1, x3, x4, x1, x2, i + 2); - } - } -# endif - - word2char(x3, d); - word2char(x4, d + 4); - word2char(x1, d + 8); - word2char(x2, d + 12); -} - -void SEED_decrypt(const unsigned char s[SEED_BLOCK_SIZE], - unsigned char d[SEED_BLOCK_SIZE], - const SEED_KEY_SCHEDULE *ks) -{ - seed_word x1, x2, x3, x4; - seed_word t0, t1; - - char2word(s, x1); - char2word(s + 4, x2); - char2word(s + 8, x3); - char2word(s + 12, x4); - -# if !defined(OPENSSL_SMALL_FOOTPRINT) - E_SEED(t0, t1, x1, x2, x3, x4, 30); - E_SEED(t0, t1, x3, x4, x1, x2, 28); - E_SEED(t0, t1, x1, x2, x3, x4, 26); - E_SEED(t0, t1, x3, x4, x1, x2, 24); - E_SEED(t0, t1, x1, x2, x3, x4, 22); - E_SEED(t0, t1, x3, x4, x1, x2, 20); - E_SEED(t0, t1, x1, x2, x3, x4, 18); - E_SEED(t0, t1, x3, x4, x1, x2, 16); - E_SEED(t0, t1, x1, x2, x3, x4, 14); - E_SEED(t0, t1, x3, x4, x1, x2, 12); - E_SEED(t0, t1, x1, x2, x3, x4, 10); - E_SEED(t0, t1, x3, x4, x1, x2, 8); - E_SEED(t0, t1, x1, x2, x3, x4, 6); - E_SEED(t0, t1, x3, x4, x1, x2, 4); - E_SEED(t0, t1, x1, x2, x3, x4, 2); - E_SEED(t0, t1, x3, x4, x1, x2, 0); -# else - { - int i; - for (i = 30; i > 0; i -= 4) { - E_SEED(t0, t1, x1, x2, x3, x4, i); - E_SEED(t0, t1, x3, x4, x1, x2, i - 2); - - } - } -# endif - - word2char(x3, d); - word2char(x4, d + 4); - word2char(x1, d + 8); - word2char(x2, d + 12); -} - -#endif /* OPENSSL_NO_SEED */ diff --git a/openssl/src/crypto/seed/seed_cbc.c b/openssl/src/crypto/seed/seed_cbc.c deleted file mode 100644 index d5a324fd4..000000000 --- a/openssl/src/crypto/seed/seed_cbc.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * SEED low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include - -void SEED_cbc_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const SEED_KEY_SCHEDULE *ks, - unsigned char ivec[SEED_BLOCK_SIZE], int enc) -{ - if (enc) - CRYPTO_cbc128_encrypt(in, out, len, ks, ivec, - (block128_f) SEED_encrypt); - else - CRYPTO_cbc128_decrypt(in, out, len, ks, ivec, - (block128_f) SEED_decrypt); -} diff --git a/openssl/src/crypto/seed/seed_cfb.c b/openssl/src/crypto/seed/seed_cfb.c deleted file mode 100644 index 24fbfbf74..000000000 --- a/openssl/src/crypto/seed/seed_cfb.c +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * SEED low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include - -void SEED_cfb128_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const SEED_KEY_SCHEDULE *ks, - unsigned char ivec[SEED_BLOCK_SIZE], int *num, - int enc) -{ - CRYPTO_cfb128_encrypt(in, out, len, ks, ivec, num, enc, - (block128_f) SEED_encrypt); -} diff --git a/openssl/src/crypto/seed/seed_ecb.c b/openssl/src/crypto/seed/seed_ecb.c deleted file mode 100644 index 9f357511c..000000000 --- a/openssl/src/crypto/seed/seed_ecb.c +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * SEED low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include - -void SEED_ecb_encrypt(const unsigned char *in, unsigned char *out, - const SEED_KEY_SCHEDULE *ks, int enc) -{ - if (enc) - SEED_encrypt(in, out, ks); - else - SEED_decrypt(in, out, ks); -} diff --git a/openssl/src/crypto/seed/seed_local.h b/openssl/src/crypto/seed/seed_local.h deleted file mode 100644 index dd40ee69d..000000000 --- a/openssl/src/crypto/seed/seed_local.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Copyright (c) 2007 KISA(Korea Information Security Agency). All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Neither the name of author nor the names of its contributors may - * be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ -#ifndef OSSL_CRYPTO_SEED_LOCAL_H -# define OSSL_CRYPTO_SEED_LOCAL_H - -# include -# include - -# ifdef SEED_LONG /* need 32-bit type */ -typedef unsigned long seed_word; -# else -typedef unsigned int seed_word; -# endif - - -# define char2word(c, i) \ - (i) = ((((seed_word)(c)[0]) << 24) | (((seed_word)(c)[1]) << 16) | (((seed_word)(c)[2]) << 8) | ((seed_word)(c)[3])) - -# define word2char(l, c) \ - *((c)+0) = (unsigned char)((l)>>24) & 0xff; \ - *((c)+1) = (unsigned char)((l)>>16) & 0xff; \ - *((c)+2) = (unsigned char)((l)>> 8) & 0xff; \ - *((c)+3) = (unsigned char)((l)) & 0xff - -# define KEYSCHEDULE_UPDATE0(T0, T1, X1, X2, X3, X4, KC) \ - (T0) = (X3); \ - (X3) = (((X3)<<8) ^ ((X4)>>24)) & 0xffffffff; \ - (X4) = (((X4)<<8) ^ ((T0)>>24)) & 0xffffffff; \ - (T0) = ((X1) + (X3) - (KC)) & 0xffffffff; \ - (T1) = ((X2) + (KC) - (X4)) & 0xffffffff - -# define KEYSCHEDULE_UPDATE1(T0, T1, X1, X2, X3, X4, KC) \ - (T0) = (X1); \ - (X1) = (((X1)>>8) ^ ((X2)<<24)) & 0xffffffff; \ - (X2) = (((X2)>>8) ^ ((T0)<<24)) & 0xffffffff; \ - (T0) = ((X1) + (X3) - (KC)) & 0xffffffff; \ - (T1) = ((X2) + (KC) - (X4)) & 0xffffffff - -# define KEYUPDATE_TEMP(T0, T1, K) \ - (K)[0] = G_FUNC((T0)); \ - (K)[1] = G_FUNC((T1)) - -# define XOR_SEEDBLOCK(DST, SRC) \ - ((DST))[0] ^= ((SRC))[0]; \ - ((DST))[1] ^= ((SRC))[1]; \ - ((DST))[2] ^= ((SRC))[2]; \ - ((DST))[3] ^= ((SRC))[3] - -# define MOV_SEEDBLOCK(DST, SRC) \ - ((DST))[0] = ((SRC))[0]; \ - ((DST))[1] = ((SRC))[1]; \ - ((DST))[2] = ((SRC))[2]; \ - ((DST))[3] = ((SRC))[3] - -# define CHAR2WORD(C, I) \ - char2word((C), (I)[0]); \ - char2word((C+4), (I)[1]); \ - char2word((C+8), (I)[2]); \ - char2word((C+12), (I)[3]) - -# define WORD2CHAR(I, C) \ - word2char((I)[0], (C)); \ - word2char((I)[1], (C+4)); \ - word2char((I)[2], (C+8)); \ - word2char((I)[3], (C+12)) - -# define E_SEED(T0, T1, X1, X2, X3, X4, rbase) \ - (T0) = (X3) ^ (ks->data)[(rbase)]; \ - (T1) = (X4) ^ (ks->data)[(rbase)+1]; \ - (T1) ^= (T0); \ - (T1) = G_FUNC((T1)); \ - (T0) = ((T0) + (T1)) & 0xffffffff; \ - (T0) = G_FUNC((T0)); \ - (T1) = ((T1) + (T0)) & 0xffffffff; \ - (T1) = G_FUNC((T1)); \ - (T0) = ((T0) + (T1)) & 0xffffffff; \ - (X1) ^= (T0); \ - (X2) ^= (T1) - -#endif /* OSSL_CRYPTO_SEED_LOCAL_H */ diff --git a/openssl/src/crypto/seed/seed_ofb.c b/openssl/src/crypto/seed/seed_ofb.c deleted file mode 100644 index b2e905331..000000000 --- a/openssl/src/crypto/seed/seed_ofb.c +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * SEED low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include - -void SEED_ofb128_encrypt(const unsigned char *in, unsigned char *out, - size_t len, const SEED_KEY_SCHEDULE *ks, - unsigned char ivec[SEED_BLOCK_SIZE], int *num) -{ - CRYPTO_ofb128_encrypt(in, out, len, ks, ivec, num, - (block128_f) SEED_encrypt); -} diff --git a/openssl/src/crypto/self_test_core.c b/openssl/src/crypto/self_test_core.c index f31fce57c..dad4be208 100644 --- a/openssl/src/crypto/self_test_core.c +++ b/openssl/src/crypto/self_test_core.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,7 +11,6 @@ #include #include #include "internal/cryptlib.h" -#include "crypto/context.h" typedef struct self_test_cb_st { @@ -33,7 +32,7 @@ struct ossl_self_test_st }; #ifndef FIPS_MODULE -void *ossl_self_test_set_callback_new(OSSL_LIB_CTX *ctx) +static void *self_test_set_callback_new(OSSL_LIB_CTX *ctx) { SELF_TEST_CB *stcb; @@ -41,14 +40,21 @@ void *ossl_self_test_set_callback_new(OSSL_LIB_CTX *ctx) return stcb; } -void ossl_self_test_set_callback_free(void *stcb) +static void self_test_set_callback_free(void *stcb) { OPENSSL_free(stcb); } +static const OSSL_LIB_CTX_METHOD self_test_set_callback_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + self_test_set_callback_new, + self_test_set_callback_free, +}; + static SELF_TEST_CB *get_self_test_callback(OSSL_LIB_CTX *libctx) { - return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_SELF_TEST_CB_INDEX); + return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_SELF_TEST_CB_INDEX, + &self_test_set_callback_method); } void OSSL_SELF_TEST_set_callback(OSSL_LIB_CTX *libctx, OSSL_CALLBACK *cb, diff --git a/openssl/src/crypto/sha/gen/darwin_arm64/keccak1600-armv8.S b/openssl/src/crypto/sha/gen/darwin_arm64/keccak1600-armv8.S index 139c41bd5..fcebbfeb9 100644 --- a/openssl/src/crypto/sha/gen/darwin_arm64/keccak1600-armv8.S +++ b/openssl/src/crypto/sha/gen/darwin_arm64/keccak1600-armv8.S @@ -1,5 +1,3 @@ -#include "arm_arch.h" - .text .align 8 // strategic alignment and padding that allows to use @@ -35,8 +33,8 @@ iotas: .align 5 KeccakF1600_int: - AARCH64_SIGN_LINK_REGISTER adr x28,iotas +.long 0xd503233f // paciasp stp x28,x30,[sp,#16] // 32 bytes on top are mine b Loop .align 4 @@ -200,14 +198,14 @@ Loop: bne Loop ldr x30,[sp,#24] - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .align 5 KeccakF1600: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -257,7 +255,7 @@ KeccakF1600: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -265,7 +263,7 @@ KeccakF1600: .align 5 _SHA3_absorb: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -499,14 +497,14 @@ Labsorbed: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .globl _SHA3_squeeze .align 5 _SHA3_squeeze: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-48]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -516,8 +514,6 @@ _SHA3_squeeze: mov x20,x1 mov x21,x2 mov x22,x3 - cmp w4, #0 // w4 = 'next' argument - bne Lnext_block Loop_squeeze: ldr x4,[x0],#8 @@ -532,7 +528,7 @@ Loop_squeeze: subs x3,x3,#8 bhi Loop_squeeze -Lnext_block: + mov x0,x19 bl KeccakF1600 mov x0,x19 @@ -571,7 +567,7 @@ Lsqueeze_done: ldp x19,x20,[sp,#16] ldp x21,x22,[sp,#32] ldp x29,x30,[sp],#48 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -679,7 +675,7 @@ Loop_ce: .align 5 KeccakF1600_cext: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 stp d8,d9,[sp,#16] // per ABI requirement @@ -720,14 +716,14 @@ KeccakF1600_cext: ldp d12,d13,[sp,#48] ldp d14,d15,[sp,#64] ldr x29,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .globl _SHA3_absorb_cext .align 5 _SHA3_absorb_cext: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 stp d8,d9,[sp,#16] // per ABI requirement @@ -943,14 +939,14 @@ Labsorbed_ce: ldp d12,d13,[sp,#48] ldp d14,d15,[sp,#64] ldp x29,x30,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .globl _SHA3_squeeze_cext .align 5 _SHA3_squeeze_cext: - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 mov x9,x0 @@ -1006,7 +1002,7 @@ Lsqueeze_tail_ce: Lsqueeze_done_ce: ldr x29,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret .byte 75,101,99,99,97,107,45,49,54,48,48,32,97,98,115,111,114,98,32,97,110,100,32,115,113,117,101,101,122,101,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 diff --git a/openssl/src/crypto/sha/gen/darwin_arm64/sha1-armv8.S b/openssl/src/crypto/sha/gen/darwin_arm64/sha1-armv8.S index c811458c0..1d453b608 100644 --- a/openssl/src/crypto/sha/gen/darwin_arm64/sha1-armv8.S +++ b/openssl/src/crypto/sha/gen/darwin_arm64/sha1-armv8.S @@ -1,5 +1,5 @@ -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .private_extern _OPENSSL_armcap_P #endif @@ -10,13 +10,11 @@ .align 6 _sha1_block_data_order: - AARCH64_VALID_CALL_TARGET adrp x16,_OPENSSL_armcap_P@PAGE ldr w16,[x16,_OPENSSL_armcap_P@PAGEOFF] tst w16,#ARMV8_SHA1 b.ne Lv8_entry - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1076,7 +1074,6 @@ Loop: .align 6 sha1_block_armv8: Lv8_entry: - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/openssl/src/crypto/sha/gen/darwin_arm64/sha256-armv8.S b/openssl/src/crypto/sha/gen/darwin_arm64/sha256-armv8.S index 6443ac268..908ea9870 100644 --- a/openssl/src/crypto/sha/gen/darwin_arm64/sha256-armv8.S +++ b/openssl/src/crypto/sha/gen/darwin_arm64/sha256-armv8.S @@ -55,8 +55,8 @@ // $output is the last argument if it looks like a file (it has an extension) // $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .private_extern _OPENSSL_armcap_P #endif @@ -67,7 +67,6 @@ .align 6 _sha256_block_data_order: - AARCH64_VALID_CALL_TARGET #ifndef __KERNEL__ adrp x16,_OPENSSL_armcap_P@PAGE ldr w16,[x16,_OPENSSL_armcap_P@PAGEOFF] @@ -76,7 +75,7 @@ _sha256_block_data_order: tst w16,#ARMV7_NEON b.ne Lneon_entry #endif - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 @@ -1036,7 +1035,7 @@ Loop_16_xx: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -1069,7 +1068,6 @@ LK256: .align 6 sha256_block_armv8: Lv8_entry: - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1211,9 +1209,7 @@ Loop_hw: .align 4 _sha256_block_neon: - AARCH64_VALID_CALL_TARGET Lneon_entry: - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later stp x29, x30, [sp, #-16]! mov x29, sp sub sp,sp,#16*4 diff --git a/openssl/src/crypto/sha/gen/darwin_arm64/sha512-armv8.S b/openssl/src/crypto/sha/gen/darwin_arm64/sha512-armv8.S index 513d662bf..c36c40bb0 100644 --- a/openssl/src/crypto/sha/gen/darwin_arm64/sha512-armv8.S +++ b/openssl/src/crypto/sha/gen/darwin_arm64/sha512-armv8.S @@ -55,8 +55,8 @@ // $output is the last argument if it looks like a file (it has an extension) // $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .private_extern _OPENSSL_armcap_P #endif @@ -67,14 +67,13 @@ .align 6 _sha512_block_data_order: - AARCH64_VALID_CALL_TARGET #ifndef __KERNEL__ adrp x16,_OPENSSL_armcap_P@PAGE ldr w16,[x16,_OPENSSL_armcap_P@PAGEOFF] tst w16,#ARMV8_SHA512 b.ne Lv8_entry #endif - AARCH64_SIGN_LINK_REGISTER +.long 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 @@ -1034,7 +1033,7 @@ Loop_16_xx: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - AARCH64_VALIDATE_LINK_REGISTER +.long 0xd50323bf // autiasp ret @@ -1091,7 +1090,6 @@ LK512: .align 6 sha512_block_armv8: Lv8_entry: - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/openssl/src/crypto/sha/gen/darwin_x64/keccak1600-x86_64.s b/openssl/src/crypto/sha/gen/darwin_x64/keccak1600-x86_64.s index 4f7936ece..5105d9818 100644 --- a/openssl/src/crypto/sha/gen/darwin_x64/keccak1600-x86_64.s +++ b/openssl/src/crypto/sha/gen/darwin_x64/keccak1600-x86_64.s @@ -420,12 +420,10 @@ _SHA3_squeeze: shrq $3,%rcx - movq %rdi,%r9 + movq %rdi,%r8 movq %rsi,%r12 movq %rdx,%r13 movq %rcx,%r14 - btl $0,%r8d - jc L$next_block jmp L$oop_squeeze .p2align 5 @@ -433,8 +431,8 @@ L$oop_squeeze: cmpq $8,%r13 jb L$tail_squeeze - movq (%r9),%rax - leaq 8(%r9),%r9 + movq (%r8),%rax + leaq 8(%r8),%r8 movq %rax,(%r12) leaq 8(%r12),%r12 subq $8,%r13 @@ -442,14 +440,14 @@ L$oop_squeeze: subq $1,%rcx jnz L$oop_squeeze -L$next_block: + call KeccakF1600 - movq %rdi,%r9 + movq %rdi,%r8 movq %r14,%rcx jmp L$oop_squeeze L$tail_squeeze: - movq %r9,%rsi + movq %r8,%rsi movq %r12,%rdi movq %r13,%rcx .byte 0xf3,0xa4 diff --git a/openssl/src/crypto/sha/gen/linux_arm/keccak1600-armv4.S b/openssl/src/crypto/sha/gen/linux_arm/keccak1600-armv4.S index 74d16c417..e601fb1a2 100644 --- a/openssl/src/crypto/sha/gen/linux_arm/keccak1600-armv4.S +++ b/openssl/src/crypto/sha/gen/linux_arm/keccak1600-armv4.S @@ -2043,7 +2043,6 @@ SHA3_squeeze: mov r4,r1 mov r5,r2 mov r12,r3 - ldr r0, [sp, #40] @ next is after the 10 pushed registers (10*4) #ifdef __thumb2__ mov r9,#0x00ff00ff @@ -2065,8 +2064,6 @@ SHA3_squeeze: stmdb sp!,{r6,r7,r8,r9} mov r14,r10 - cmp r0, #1 - beq .Lnext_block b .Loop_squeeze .align 4 @@ -2138,7 +2135,7 @@ SHA3_squeeze: subs r12,r12,#8 @ bsz -= 8 bhi .Loop_squeeze -.Lnext_block: + mov r0,r14 @ original r10 bl KeccakF1600 diff --git a/openssl/src/crypto/sha/gen/linux_arm/sha1-armv4-large.S b/openssl/src/crypto/sha/gen/linux_arm/sha1-armv4-large.S index e2013f608..472912265 100644 --- a/openssl/src/crypto/sha/gen/linux_arm/sha1-armv4-large.S +++ b/openssl/src/crypto/sha/gen/linux_arm/sha1-armv4-large.S @@ -1494,5 +1494,5 @@ sha1_block_data_order_armv8: .size sha1_block_data_order_armv8,.-sha1_block_data_order_armv8 #endif #if __ARM_MAX_ARCH__>=7 - +.comm OPENSSL_armcap_P,4,4 #endif diff --git a/openssl/src/crypto/sha/gen/linux_arm/sha256-armv4.S b/openssl/src/crypto/sha/gen/linux_arm/sha256-armv4.S index c3587fb06..4348a68a9 100644 --- a/openssl/src/crypto/sha/gen/linux_arm/sha256-armv4.S +++ b/openssl/src/crypto/sha/gen/linux_arm/sha256-armv4.S @@ -1,4 +1,4 @@ -@ Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. +@ Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. @ @ Licensed under the Apache License 2.0 (the "License"). You may not use @ this file except in compliance with the License. You can obtain a copy @@ -2818,5 +2818,5 @@ sha256_block_data_order_armv8: .align 2 .align 2 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) - +.comm OPENSSL_armcap_P,4,4 #endif diff --git a/openssl/src/crypto/sha/gen/linux_arm/sha512-armv4.S b/openssl/src/crypto/sha/gen/linux_arm/sha512-armv4.S index 60d0ddae8..9ee892276 100644 --- a/openssl/src/crypto/sha/gen/linux_arm/sha512-armv4.S +++ b/openssl/src/crypto/sha/gen/linux_arm/sha512-armv4.S @@ -1,4 +1,4 @@ -@ Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. +@ Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. @ @ Licensed under the Apache License 2.0 (the "License"). You may not use @ this file except in compliance with the License. You can obtain a copy @@ -1872,5 +1872,5 @@ sha512_block_data_order_neon: .align 2 .align 2 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) - +.comm OPENSSL_armcap_P,4,4 #endif diff --git a/openssl/src/crypto/sha/gen/linux_arm64/keccak1600-armv8.S b/openssl/src/crypto/sha/gen/linux_arm64/keccak1600-armv8.S index 35865529f..3a3385e28 100644 --- a/openssl/src/crypto/sha/gen/linux_arm64/keccak1600-armv8.S +++ b/openssl/src/crypto/sha/gen/linux_arm64/keccak1600-armv8.S @@ -1,5 +1,3 @@ -#include "arm_arch.h" - .text .align 8 // strategic alignment and padding that allows to use @@ -35,8 +33,8 @@ iotas: .type KeccakF1600_int,%function .align 5 KeccakF1600_int: - AARCH64_SIGN_LINK_REGISTER adr x28,iotas +.inst 0xd503233f // paciasp stp x28,x30,[sp,#16] // 32 bytes on top are mine b .Loop .align 4 @@ -200,14 +198,14 @@ KeccakF1600_int: bne .Loop ldr x30,[sp,#24] - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size KeccakF1600_int,.-KeccakF1600_int .type KeccakF1600,%function .align 5 KeccakF1600: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -257,7 +255,7 @@ KeccakF1600: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size KeccakF1600,.-KeccakF1600 @@ -265,7 +263,7 @@ KeccakF1600: .type SHA3_absorb,%function .align 5 SHA3_absorb: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -499,14 +497,14 @@ SHA3_absorb: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size SHA3_absorb,.-SHA3_absorb .globl SHA3_squeeze .type SHA3_squeeze,%function .align 5 SHA3_squeeze: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-48]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -516,8 +514,6 @@ SHA3_squeeze: mov x20,x1 mov x21,x2 mov x22,x3 - cmp w4, #0 // w4 = 'next' argument - bne .Lnext_block .Loop_squeeze: ldr x4,[x0],#8 @@ -532,7 +528,7 @@ SHA3_squeeze: subs x3,x3,#8 bhi .Loop_squeeze -.Lnext_block: + mov x0,x19 bl KeccakF1600 mov x0,x19 @@ -571,7 +567,7 @@ SHA3_squeeze: ldp x19,x20,[sp,#16] ldp x21,x22,[sp,#32] ldp x29,x30,[sp],#48 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size SHA3_squeeze,.-SHA3_squeeze .type KeccakF1600_ce,%function @@ -679,7 +675,7 @@ KeccakF1600_ce: .type KeccakF1600_cext,%function .align 5 KeccakF1600_cext: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 stp d8,d9,[sp,#16] // per ABI requirement @@ -720,14 +716,14 @@ KeccakF1600_cext: ldp d12,d13,[sp,#48] ldp d14,d15,[sp,#64] ldr x29,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size KeccakF1600_cext,.-KeccakF1600_cext .globl SHA3_absorb_cext .type SHA3_absorb_cext,%function .align 5 SHA3_absorb_cext: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 stp d8,d9,[sp,#16] // per ABI requirement @@ -943,14 +939,14 @@ SHA3_absorb_cext: ldp d12,d13,[sp,#48] ldp d14,d15,[sp,#64] ldp x29,x30,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size SHA3_absorb_cext,.-SHA3_absorb_cext .globl SHA3_squeeze_cext .type SHA3_squeeze_cext,%function .align 5 SHA3_squeeze_cext: - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 mov x9,x0 @@ -1006,7 +1002,7 @@ SHA3_squeeze_cext: .Lsqueeze_done_ce: ldr x29,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size SHA3_squeeze_cext,.-SHA3_squeeze_cext .byte 75,101,99,99,97,107,45,49,54,48,48,32,97,98,115,111,114,98,32,97,110,100,32,115,113,117,101,101,122,101,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 diff --git a/openssl/src/crypto/sha/gen/linux_arm64/sha1-armv8.S b/openssl/src/crypto/sha/gen/linux_arm64/sha1-armv8.S index 329ab5670..a3cb53020 100644 --- a/openssl/src/crypto/sha/gen/linux_arm64/sha1-armv8.S +++ b/openssl/src/crypto/sha/gen/linux_arm64/sha1-armv8.S @@ -1,5 +1,5 @@ -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .hidden OPENSSL_armcap_P #endif @@ -10,13 +10,11 @@ .type sha1_block_data_order,%function .align 6 sha1_block_data_order: - AARCH64_VALID_CALL_TARGET adrp x16,OPENSSL_armcap_P ldr w16,[x16,#:lo12:OPENSSL_armcap_P] tst w16,#ARMV8_SHA1 b.ne .Lv8_entry - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1076,7 +1074,6 @@ sha1_block_data_order: .align 6 sha1_block_armv8: .Lv8_entry: - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/openssl/src/crypto/sha/gen/linux_arm64/sha256-armv8.S b/openssl/src/crypto/sha/gen/linux_arm64/sha256-armv8.S index 6d1cb3180..e0965b668 100644 --- a/openssl/src/crypto/sha/gen/linux_arm64/sha256-armv8.S +++ b/openssl/src/crypto/sha/gen/linux_arm64/sha256-armv8.S @@ -55,8 +55,8 @@ // $output is the last argument if it looks like a file (it has an extension) // $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .hidden OPENSSL_armcap_P #endif @@ -67,7 +67,6 @@ .type sha256_block_data_order,%function .align 6 sha256_block_data_order: - AARCH64_VALID_CALL_TARGET #ifndef __KERNEL__ adrp x16,OPENSSL_armcap_P ldr w16,[x16,#:lo12:OPENSSL_armcap_P] @@ -76,7 +75,7 @@ sha256_block_data_order: tst w16,#ARMV7_NEON b.ne .Lneon_entry #endif - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 @@ -1036,7 +1035,7 @@ sha256_block_data_order: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size sha256_block_data_order,.-sha256_block_data_order @@ -1069,7 +1068,6 @@ sha256_block_data_order: .align 6 sha256_block_armv8: .Lv8_entry: - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1211,9 +1209,7 @@ sha256_block_armv8: .type sha256_block_neon,%function .align 4 sha256_block_neon: - AARCH64_VALID_CALL_TARGET .Lneon_entry: - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later stp x29, x30, [sp, #-16]! mov x29, sp sub sp,sp,#16*4 diff --git a/openssl/src/crypto/sha/gen/linux_arm64/sha512-armv8.S b/openssl/src/crypto/sha/gen/linux_arm64/sha512-armv8.S index 37091dcc7..799144426 100644 --- a/openssl/src/crypto/sha/gen/linux_arm64/sha512-armv8.S +++ b/openssl/src/crypto/sha/gen/linux_arm64/sha512-armv8.S @@ -55,8 +55,8 @@ // $output is the last argument if it looks like a file (it has an extension) // $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" #ifndef __KERNEL__ +# include "arm_arch.h" .hidden OPENSSL_armcap_P #endif @@ -67,14 +67,13 @@ .type sha512_block_data_order,%function .align 6 sha512_block_data_order: - AARCH64_VALID_CALL_TARGET #ifndef __KERNEL__ adrp x16,OPENSSL_armcap_P ldr w16,[x16,#:lo12:OPENSSL_armcap_P] tst w16,#ARMV8_SHA512 b.ne .Lv8_entry #endif - AARCH64_SIGN_LINK_REGISTER +.inst 0xd503233f // paciasp stp x29,x30,[sp,#-128]! add x29,sp,#0 @@ -1034,7 +1033,7 @@ sha512_block_data_order: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - AARCH64_VALIDATE_LINK_REGISTER +.inst 0xd50323bf // autiasp ret .size sha512_block_data_order,.-sha512_block_data_order @@ -1091,7 +1090,6 @@ sha512_block_data_order: .align 6 sha512_block_armv8: .Lv8_entry: - // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/openssl/src/crypto/sha/gen/linux_ia32/sha1-586.S b/openssl/src/crypto/sha/gen/linux_ia32/sha1-586.S index 4b60a71ee..7819ccc53 100644 --- a/openssl/src/crypto/sha/gen/linux_ia32/sha1-586.S +++ b/openssl/src/crypto/sha/gen/linux_ia32/sha1-586.S @@ -4,11 +4,7 @@ .align 16 sha1_block_data_order: .L_sha1_block_data_order_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1403,11 +1399,7 @@ sha1_block_data_order: .type _sha1_block_data_order_shaext,@function .align 16 _sha1_block_data_order_shaext: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -1578,11 +1570,7 @@ _sha1_block_data_order_shaext: .type _sha1_block_data_order_ssse3,@function .align 16 _sha1_block_data_order_ssse3: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi @@ -2802,11 +2790,7 @@ _sha1_block_data_order_ssse3: .type _sha1_block_data_order_avx,@function .align 16 _sha1_block_data_order_avx: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/sha/gen/linux_ia32/sha256-586.S b/openssl/src/crypto/sha/gen/linux_ia32/sha256-586.S index c8599bd58..09acf55bb 100644 --- a/openssl/src/crypto/sha/gen/linux_ia32/sha256-586.S +++ b/openssl/src/crypto/sha/gen/linux_ia32/sha256-586.S @@ -4,11 +4,7 @@ .align 16 sha256_block_data_order: .L_sha256_block_data_order_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/sha/gen/linux_ia32/sha512-586.S b/openssl/src/crypto/sha/gen/linux_ia32/sha512-586.S index 99d198dd9..34bc402ed 100644 --- a/openssl/src/crypto/sha/gen/linux_ia32/sha512-586.S +++ b/openssl/src/crypto/sha/gen/linux_ia32/sha512-586.S @@ -4,11 +4,7 @@ .align 16 sha512_block_data_order: .L_sha512_block_data_order_begin: - #ifdef __CET__ - .byte 243,15,30,251 - #endif - pushl %ebp pushl %ebx pushl %esi diff --git a/openssl/src/crypto/sha/gen/linux_ppc64/keccak1600-ppc64.s b/openssl/src/crypto/sha/gen/linux_ppc64/keccak1600-ppc64.s index 2e7dd468f..98e654cb3 100644 --- a/openssl/src/crypto/sha/gen/linux_ppc64/keccak1600-ppc64.s +++ b/openssl/src/crypto/sha/gen/linux_ppc64/keccak1600-ppc64.s @@ -297,6 +297,33 @@ KeccakF1600: .byte 0,12,4,1,0x80,18,1,0 .long 0 .size KeccakF1600,.-KeccakF1600 + +.type dword_le_load,@function +.align 5 +dword_le_load: +.localentry dword_le_load,0 + + lbz 0,1(3) + lbz 4,2(3) + lbz 5,3(3) + insrdi 0,4,8,48 + lbz 4,4(3) + insrdi 0,5,8,40 + lbz 5,5(3) + insrdi 0,4,8,32 + lbz 4,6(3) + insrdi 0,5,8,24 + lbz 5,7(3) + insrdi 0,4,8,16 + lbzu 4,8(3) + insrdi 0,5,8,8 + insrdi 0,4,8,0 + blr +.long 0 +.byte 0,12,0x14,0,0,0,1,0 +.long 0 +.size dword_le_load,.-dword_le_load + .globl SHA3_absorb .type SHA3_absorb,@function .type SHA3_absorb,@function @@ -327,7 +354,7 @@ SHA3_absorb: std 0,288(1) bl PICmeup - subi 4,4,8 + subi 4,4,1 subi 12,12,8 std 3,48(1) @@ -378,79 +405,79 @@ SHA3_absorb: srwi 5,5,3 std 4,64(1) mtctr 5 - ldu 0,8(3) + bl dword_le_load xor 7,7,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 8,8,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 9,9,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 10,10,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 11,11,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 12,12,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 6,6,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 14,14,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 15,15,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 16,16,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 17,17,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 18,18,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 19,19,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 20,20,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 21,21,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 22,22,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 23,23,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 24,24,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 25,25,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 26,26,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 27,27,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 28,28,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 29,29,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 30,30,0 bdz .Lprocess_block - ldu 0,8(3) + bl dword_le_load xor 31,31,0 .Lprocess_block: @@ -543,8 +570,6 @@ SHA3_squeeze: subi 29,4,1 mr 30,5 mr 31,6 - cmplwi 7,0 - bne .Lnext_block b .Loop_squeeze .align 4 @@ -575,7 +600,6 @@ SHA3_squeeze: subic. 6,6,8 bgt .Loop_squeeze -.Lnext_block: mr 3,28 bl KeccakF1600 subi 3,28,8 diff --git a/openssl/src/crypto/sha/gen/linux_riscv64/sha256-riscv64-zvkb-zvknha_or_zvknhb.S b/openssl/src/crypto/sha/gen/linux_riscv64/sha256-riscv64-zvkb-zvknha_or_zvknhb.S deleted file mode 100644 index 92ea27ec9..000000000 --- a/openssl/src/crypto/sha/gen/linux_riscv64/sha256-riscv64-zvkb-zvknha_or_zvknhb.S +++ /dev/null @@ -1,229 +0,0 @@ -.text -.p2align 2 -.globl sha256_block_data_order_zvkb_zvknha_or_zvknhb -.type sha256_block_data_order_zvkb_zvknha_or_zvknhb,@function -sha256_block_data_order_zvkb_zvknha_or_zvknhb: - .word 3439489111 - - la a3, K256 # Load round constants K256 - .word 34006279 - addi a3, a3, 16 - .word 34006407 - addi a3, a3, 16 - .word 34006535 - addi a3, a3, 16 - .word 34006663 - addi a3, a3, 16 - .word 34006791 - addi a3, a3, 16 - .word 34006919 - addi a3, a3, 16 - .word 34007047 - addi a3, a3, 16 - .word 34007175 - addi a3, a3, 16 - .word 34007303 - addi a3, a3, 16 - .word 34007431 - addi a3, a3, 16 - .word 34007559 - addi a3, a3, 16 - .word 34007687 - addi a3, a3, 16 - .word 34007815 - addi a3, a3, 16 - .word 34007943 - addi a3, a3, 16 - .word 34008071 - addi a3, a3, 16 - .word 34008199 - - - # H is stored as {a,b,c,d},{e,f,g,h}, but we need {f,e,b,a},{h,g,d,c} - # The dst vtype is e32m1 and the index vtype is e8mf4. - # We use index-load with the following index pattern at v26. - # i8 index: - # 20, 16, 4, 0 - # Instead of setting the i8 index, we could use a single 32bit - # little-endian value to cover the 4xi8 index. - # i32 value: - # 0x 00 04 10 14 - li t4, 0x00041014 - .word 3439390807 - .word 1578028375 - - addi t3, a0, 8 - - # Use index-load to get {f,e,b,a},{h,g,d,c} - .word 3439489111 - .word 128254727 - .word 128844679 - - # Setup v0 mask for the vmerge to replace the first word (idx==0) in key-scheduling. - # The AVL is 4 in SHA, so we could use a single e8(8 element masking) for masking. - .word 3422613591 - .word 1577103447 - - .word 3439489111 - -L_round_loop: - # Decrement length by 1 - add a2, a2, -1 - - # Keep the current state as we need it later: H' = H+{a',b',c',...,h'}. - .word 1577258839 - .word 1577291735 - - # Load the 512-bits of the message block in v1-v4 and perform - # an endian swap on each 4 bytes element. - .word 33939591 - .word 1242865879 - add a1, a1, 16 - .word 33939719 - .word 1243914583 - add a1, a1, 16 - .word 33939847 - .word 1244963287 - add a1, a1, 16 - .word 33939975 - .word 1246011991 - add a1, a1, 16 - - # Quad-round 0 (+0, Wt from oldest to newest in v1->v2->v3->v4) - .word 44073687 - .word 3194135543 - .word 3128075127 - .word 1546715863 - .word 3058835703 # Generate W[19:16] - - # Quad-round 1 (+1, v2->v3->v4->v1) - .word 45155031 - .word 3194135543 - .word 3128075127 - .word 1547797207 - .word 3058737527 # Generate W[23:20] - - # Quad-round 2 (+2, v3->v4->v1->v2) - .word 46236375 - .word 3194135543 - .word 3128075127 - .word 1544684247 - .word 3058770423 # Generate W[27:24] - - # Quad-round 3 (+3, v4->v1->v2->v3) - .word 47317719 - .word 3194135543 - .word 3128075127 - .word 1545634519 - .word 3058803319 # Generate W[31:28] - - # Quad-round 4 (+0, v1->v2->v3->v4) - .word 48267991 - .word 3194135543 - .word 3128075127 - .word 1546715863 - .word 3058835703 # Generate W[35:32] - - # Quad-round 5 (+1, v2->v3->v4->v1) - .word 49349335 - .word 3194135543 - .word 3128075127 - .word 1547797207 - .word 3058737527 # Generate W[39:36] - - # Quad-round 6 (+2, v3->v4->v1->v2) - .word 50430679 - .word 3194135543 - .word 3128075127 - .word 1544684247 - .word 3058770423 # Generate W[43:40] - - # Quad-round 7 (+3, v4->v1->v2->v3) - .word 51512023 - .word 3194135543 - .word 3128075127 - .word 1545634519 - .word 3058803319 # Generate W[47:44] - - # Quad-round 8 (+0, v1->v2->v3->v4) - .word 52462295 - .word 3194135543 - .word 3128075127 - .word 1546715863 - .word 3058835703 # Generate W[51:48] - - # Quad-round 9 (+1, v2->v3->v4->v1) - .word 53543639 - .word 3194135543 - .word 3128075127 - .word 1547797207 - .word 3058737527 # Generate W[55:52] - - # Quad-round 10 (+2, v3->v4->v1->v2) - .word 54624983 - .word 3194135543 - .word 3128075127 - .word 1544684247 - .word 3058770423 # Generate W[59:56] - - # Quad-round 11 (+3, v4->v1->v2->v3) - .word 55706327 - .word 3194135543 - .word 3128075127 - .word 1545634519 - .word 3058803319 # Generate W[63:60] - - # Quad-round 12 (+0, v1->v2->v3->v4) - # Note that we stop generating new message schedule words (Wt, v1-13) - # as we already generated all the words we end up consuming (i.e., W[63:60]). - .word 56656599 - .word 3194135543 - .word 3128075127 - - # Quad-round 13 (+1, v2->v3->v4->v1) - .word 57737943 - .word 3194135543 - .word 3128075127 - - # Quad-round 14 (+2, v3->v4->v1->v2) - .word 58819287 - .word 3194135543 - .word 3128075127 - - # Quad-round 15 (+3, v4->v1->v2->v3) - .word 59900631 - .word 3194135543 - .word 3128075127 - - # H' = H+{a',b',c',...,h'} - .word 65209175 - .word 66290647 - bnez a2, L_round_loop - - # Store {f,e,b,a},{h,g,d,c} back to {a,b,c,d},{e,f,g,h}. - .word 128254759 - .word 128844711 - - ret -.size sha256_block_data_order_zvkb_zvknha_or_zvknhb,.-sha256_block_data_order_zvkb_zvknha_or_zvknhb - -.p2align 2 -.type K256,@object -K256: - .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 - .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 - .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 - .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 - .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc - .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da - .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 - .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 - .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 - .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 - .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 - .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 - .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 - .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 - .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 - .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 -.size K256,.-K256 diff --git a/openssl/src/crypto/sha/gen/linux_riscv64/sha512-riscv64-zvkb-zvknhb.S b/openssl/src/crypto/sha/gen/linux_riscv64/sha512-riscv64-zvkb-zvknhb.S deleted file mode 100644 index da1817a64..000000000 --- a/openssl/src/crypto/sha/gen/linux_riscv64/sha512-riscv64-zvkb-zvknhb.S +++ /dev/null @@ -1,184 +0,0 @@ -.text -.p2align 2 -.globl sha512_block_data_order_zvkb_zvknhb -.type sha512_block_data_order_zvkb_zvknhb,@function -sha512_block_data_order_zvkb_zvknhb: - .word 3448926295 - - # H is stored as {a,b,c,d},{e,f,g,h}, but we need {f,e,b,a},{h,g,d,c} - # The dst vtype is e64m2 and the index vtype is e8mf4. - # We use index-load with the following index pattern at v1. - # i8 index: - # 40, 32, 8, 0 - # Instead of setting the i8 index, we could use a single 32bit - # little-endian value to cover the 4xi8 index. - # i32 value: - # 0x 00 08 20 28 - li t4, 0x00082028 - .word 3439390807 - .word 1578025175 - - addi t3, a0, 16 - - # Use index-load to get {f,e,b,a},{h,g,d,c} - .word 3448926295 - .word 102042375 - .word 102632455 - - # Setup v0 mask for the vmerge to replace the first word (idx==0) in key-scheduling. - # The AVL is 4 in SHA, so we could use a single e8(8 element masking) for masking. - .word 3422613591 - .word 1577103447 - - .word 3448926295 - -L_round_loop: - # Load round constants K512 - la a3, K512 - - # Decrement length by 1 - addi a2, a2, -1 - - # Keep the current state as we need it later: H' = H+{a',b',c',...,h'}. - .word 1577782615 - .word 1577848407 - - # Load the 1024-bits of the message block in v10-v16 and perform the endian - # swap. - .word 33944839 - .word 1252304215 - addi a1, a1, 32 - .word 33945095 - .word 1254401623 - addi a1, a1, 32 - .word 33945351 - .word 1256499031 - addi a1, a1, 32 - .word 33945607 - .word 1258596439 - addi a1, a1, 32 - - .rept 4 - # Quad-round 0 (+0, v10->v12->v14->v16) - .word 34011655 - addi a3, a3, 32 - .word 54856023 - .word 3211340919 - .word 3146328951 - .word 1558579543 - .word 3072861559 - - # Quad-round 1 (+1, v12->v14->v16->v10) - .word 34011655 - addi a3, a3, 32 - .word 54921559 - .word 3211340919 - .word 3146328951 - .word 1560742231 - .word 3072665207 - - # Quad-round 2 (+2, v14->v16->v10->v12) - .word 34011655 - addi a3, a3, 32 - .word 54987095 - .word 3211340919 - .word 3146328951 - .word 1554516311 - .word 3072730999 - - # Quad-round 3 (+3, v16->v10->v12->v14) - .word 34011655 - addi a3, a3, 32 - .word 55052631 - .word 3211340919 - .word 3146328951 - .word 1556416855 - .word 3072796791 - .endr - - # Quad-round 16 (+0, v10->v12->v14->v16) - # Note that we stop generating new message schedule words (Wt, v10-16) - # as we already generated all the words we end up consuming (i.e., W[79:76]). - .word 34011655 - addi a3, a3, 32 - .word 54856023 - .word 3211340919 - .word 3146328951 - - # Quad-round 17 (+1, v12->v14->v16->v10) - .word 34011655 - addi a3, a3, 32 - .word 54921559 - .word 3211340919 - .word 3146328951 - - # Quad-round 18 (+2, v14->v16->v10->v12) - .word 34011655 - addi a3, a3, 32 - .word 54987095 - .word 3211340919 - .word 3146328951 - - # Quad-round 19 (+3, v16->v10->v12->v14) - .word 34011655 - # No t1 increment needed. - .word 55052631 - .word 3211340919 - .word 3146328951 - - # H' = H+{a',b',c',...,h'} - .word 61541207 - .word 63704151 - bnez a2, L_round_loop - - # Store {f,e,b,a},{h,g,d,c} back to {a,b,c,d},{e,f,g,h}. - .word 102042407 - .word 102632487 - - ret -.size sha512_block_data_order_zvkb_zvknhb,.-sha512_block_data_order_zvkb_zvknhb - -.p2align 3 -.type K512,@object -K512: - .dword 0x428a2f98d728ae22, 0x7137449123ef65cd - .dword 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc - .dword 0x3956c25bf348b538, 0x59f111f1b605d019 - .dword 0x923f82a4af194f9b, 0xab1c5ed5da6d8118 - .dword 0xd807aa98a3030242, 0x12835b0145706fbe - .dword 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2 - .dword 0x72be5d74f27b896f, 0x80deb1fe3b1696b1 - .dword 0x9bdc06a725c71235, 0xc19bf174cf692694 - .dword 0xe49b69c19ef14ad2, 0xefbe4786384f25e3 - .dword 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65 - .dword 0x2de92c6f592b0275, 0x4a7484aa6ea6e483 - .dword 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5 - .dword 0x983e5152ee66dfab, 0xa831c66d2db43210 - .dword 0xb00327c898fb213f, 0xbf597fc7beef0ee4 - .dword 0xc6e00bf33da88fc2, 0xd5a79147930aa725 - .dword 0x06ca6351e003826f, 0x142929670a0e6e70 - .dword 0x27b70a8546d22ffc, 0x2e1b21385c26c926 - .dword 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df - .dword 0x650a73548baf63de, 0x766a0abb3c77b2a8 - .dword 0x81c2c92e47edaee6, 0x92722c851482353b - .dword 0xa2bfe8a14cf10364, 0xa81a664bbc423001 - .dword 0xc24b8b70d0f89791, 0xc76c51a30654be30 - .dword 0xd192e819d6ef5218, 0xd69906245565a910 - .dword 0xf40e35855771202a, 0x106aa07032bbd1b8 - .dword 0x19a4c116b8d2d0c8, 0x1e376c085141ab53 - .dword 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8 - .dword 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb - .dword 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3 - .dword 0x748f82ee5defb2fc, 0x78a5636f43172f60 - .dword 0x84c87814a1f0ab72, 0x8cc702081a6439ec - .dword 0x90befffa23631e28, 0xa4506cebde82bde9 - .dword 0xbef9a3f7b2c67915, 0xc67178f2e372532b - .dword 0xca273eceea26619c, 0xd186b8c721c0c207 - .dword 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178 - .dword 0x06f067aa72176fba, 0x0a637dc5a2c898a6 - .dword 0x113f9804bef90dae, 0x1b710b35131c471b - .dword 0x28db77f523047d84, 0x32caab7b40c72493 - .dword 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c - .dword 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a - .dword 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 -.size K512,.-K512 diff --git a/openssl/src/crypto/sha/gen/linux_x64/keccak1600-x86_64.s b/openssl/src/crypto/sha/gen/linux_x64/keccak1600-x86_64.s index c8854544e..bb5222b73 100644 --- a/openssl/src/crypto/sha/gen/linux_x64/keccak1600-x86_64.s +++ b/openssl/src/crypto/sha/gen/linux_x64/keccak1600-x86_64.s @@ -447,12 +447,10 @@ SHA3_squeeze: .cfi_offset %r14,-32 shrq $3,%rcx - movq %rdi,%r9 + movq %rdi,%r8 movq %rsi,%r12 movq %rdx,%r13 movq %rcx,%r14 - btl $0,%r8d - jc .Lnext_block jmp .Loop_squeeze .align 32 @@ -460,8 +458,8 @@ SHA3_squeeze: cmpq $8,%r13 jb .Ltail_squeeze - movq (%r9),%rax - leaq 8(%r9),%r9 + movq (%r8),%rax + leaq 8(%r8),%r8 movq %rax,(%r12) leaq 8(%r12),%r12 subq $8,%r13 @@ -469,14 +467,14 @@ SHA3_squeeze: subq $1,%rcx jnz .Loop_squeeze -.Lnext_block: + call KeccakF1600 - movq %rdi,%r9 + movq %rdi,%r8 movq %r14,%rcx jmp .Loop_squeeze .Ltail_squeeze: - movq %r9,%rsi + movq %r8,%rsi movq %r12,%rdi movq %r13,%rcx .byte 0xf3,0xa4 diff --git a/openssl/src/crypto/sha/gen/windows_ia32/sha1-586.asm b/openssl/src/crypto/sha/gen/windows_ia32/sha1-586.asm index 6948eaf6d..1112de399 100644 --- a/openssl/src/crypto/sha/gen/windows_ia32/sha1-586.asm +++ b/openssl/src/crypto/sha/gen/windows_ia32/sha1-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/sha/gen/windows_ia32/sha256-586.asm b/openssl/src/crypto/sha/gen/windows_ia32/sha256-586.asm index 81ae1a07e..1108147ae 100644 --- a/openssl/src/crypto/sha/gen/windows_ia32/sha256-586.asm +++ b/openssl/src/crypto/sha/gen/windows_ia32/sha256-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/sha/gen/windows_ia32/sha512-586.asm b/openssl/src/crypto/sha/gen/windows_ia32/sha512-586.asm index e037e7d92..2202228ce 100644 --- a/openssl/src/crypto/sha/gen/windows_ia32/sha512-586.asm +++ b/openssl/src/crypto/sha/gen/windows_ia32/sha512-586.asm @@ -1,4 +1,3 @@ - %ifidn __OUTPUT_FORMAT__,obj section code use32 class=code align=64 %elifidn __OUTPUT_FORMAT__,win32 diff --git a/openssl/src/crypto/sha/gen/windows_x64/keccak1600-x86_64.asm b/openssl/src/crypto/sha/gen/windows_x64/keccak1600-x86_64.asm index 1c4ed5557..fdab35d95 100644 --- a/openssl/src/crypto/sha/gen/windows_x64/keccak1600-x86_64.asm +++ b/openssl/src/crypto/sha/gen/windows_x64/keccak1600-x86_64.asm @@ -436,7 +436,6 @@ $L$SEH_begin_SHA3_squeeze: mov rsi,rdx mov rdx,r8 mov rcx,r9 - mov r8,QWORD[40+rsp] @@ -448,12 +447,10 @@ $L$SEH_begin_SHA3_squeeze: shr rcx,3 - mov r9,rdi + mov r8,rdi mov r12,rsi mov r13,rdx mov r14,rcx - bt r8d,0 - jc NEAR $L$next_block jmp NEAR $L$oop_squeeze ALIGN 32 @@ -461,8 +458,8 @@ $L$oop_squeeze: cmp r13,8 jb NEAR $L$tail_squeeze - mov rax,QWORD[r9] - lea r9,[8+r9] + mov rax,QWORD[r8] + lea r8,[8+r8] mov QWORD[r12],rax lea r12,[8+r12] sub r13,8 @@ -470,14 +467,14 @@ $L$oop_squeeze: sub rcx,1 jnz NEAR $L$oop_squeeze -$L$next_block: + call KeccakF1600 - mov r9,rdi + mov r8,rdi mov rcx,r14 jmp NEAR $L$oop_squeeze $L$tail_squeeze: - mov rsi,r9 + mov rsi,r8 mov rdi,r12 mov rcx,r13 DB 0xf3,0xa4 diff --git a/openssl/src/crypto/sha/keccak1600.c b/openssl/src/crypto/sha/keccak1600.c index 6682367be..e3dd6f168 100644 --- a/openssl/src/crypto/sha/keccak1600.c +++ b/openssl/src/crypto/sha/keccak1600.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,7 +13,7 @@ size_t SHA3_absorb(uint64_t A[5][5], const unsigned char *inp, size_t len, size_t r); -void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r, int next); +void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r); #if !defined(KECCAK1600_ASM) || !defined(SELFTEST) @@ -37,8 +37,7 @@ void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r, in #endif #if defined(__x86_64__) || defined(__aarch64__) || \ - defined(__mips64) || defined(__ia64) || \ - (defined(__VMS) && !defined(__vax)) + defined(__mips64) || defined(__ia64) /* * These are available even in ILP32 flavours, but even then they are * capable of performing 64-bit operations as efficiently as in *P64. @@ -1090,16 +1089,10 @@ size_t SHA3_absorb(uint64_t A[5][5], const unsigned char *inp, size_t len, } /* - * SHA3_squeeze may be called after SHA3_absorb to generate |out| hash value of - * |len| bytes. - * If multiple SHA3_squeeze calls are required the output length |len| must be a - * multiple of the blocksize, with |next| being 0 on the first call and 1 on - * subsequent calls. It is the callers responsibility to buffer the results. - * When only a single call to SHA3_squeeze is required, |len| can be any size - * and |next| must be 0. + * sha3_squeeze is called once at the end to generate |out| hash value + * of |len| bytes. */ -void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r, - int next) +void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r) { uint64_t *A_flat = (uint64_t *)A; size_t i, w = r / 8; @@ -1107,9 +1100,6 @@ void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r, assert(r < (25 * sizeof(A[0][0])) && (r % 8) == 0); while (len != 0) { - if (next) - KeccakF1600(A); - next = 1; for (i = 0; i < w && len != 0; i++) { uint64_t Ai = BitDeinterleave(A_flat[i]); @@ -1132,6 +1122,8 @@ void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r, out += 8; len -= 8; } + if (len) + KeccakF1600(A); } } #endif @@ -1160,7 +1152,7 @@ void SHA3_sponge(const unsigned char *inp, size_t len, # include -int main(void) +int main() { /* * This is 5-bit SHAKE128 test from http://csrc.nist.gov/groups/ST/toolkit/examples.html#aHashing @@ -1249,11 +1241,11 @@ int main(void) printf(++i % 16 && i != sizeof(out) ? " " : "\n"); } - if (memcmp(out, result, sizeof(out))) { - fprintf(stderr, "failure\n"); + if (memcmp(out,result,sizeof(out))) { + fprintf(stderr,"failure\n"); return 1; } else { - fprintf(stderr, "success\n"); + fprintf(stderr,"success\n"); return 0; } } diff --git a/openssl/src/crypto/sha/sha1dgst.c b/openssl/src/crypto/sha/sha1dgst.c index e8f1ef524..65d7e62e5 100644 --- a/openssl/src/crypto/sha/sha1dgst.c +++ b/openssl/src/crypto/sha/sha1dgst.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,7 +20,7 @@ #include #include -/* The implementation is in crypto/md32_common.h */ +/* The implementation is in ../md32_common.h */ #include "sha_local.h" #include "crypto/sha.h" diff --git a/openssl/src/crypto/sha/sha256.c b/openssl/src/crypto/sha/sha256.c index 6ef218e86..5845c3893 100644 --- a/openssl/src/crypto/sha/sha256.c +++ b/openssl/src/crypto/sha/sha256.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2004-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,7 +22,6 @@ #include #include #include "internal/endian.h" -#include "crypto/sha.h" int SHA224_Init(SHA256_CTX *c) { @@ -54,13 +53,6 @@ int SHA256_Init(SHA256_CTX *c) return 1; } -int ossl_sha256_192_init(SHA256_CTX *c) -{ - SHA256_Init(c); - c->md_len = SHA256_192_DIGEST_LENGTH; - return 1; -} - int SHA224_Update(SHA256_CTX *c, const void *data, size_t len) { return SHA256_Update(c, data, len); @@ -89,11 +81,7 @@ int SHA224_Final(unsigned char *md, SHA256_CTX *c) unsigned long ll; \ unsigned int nn; \ switch ((c)->md_len) \ - { case SHA256_192_DIGEST_LENGTH: \ - for (nn=0;nnh[nn]; (void)HOST_l2c(ll,(s)); } \ - break; \ - case SHA224_DIGEST_LENGTH: \ + { case SHA224_DIGEST_LENGTH: \ for (nn=0;nnh[nn]; (void)HOST_l2c(ll,(s)); } \ break; \ @@ -116,16 +104,12 @@ int SHA224_Final(unsigned char *md, SHA256_CTX *c) #define HASH_BLOCK_DATA_ORDER sha256_block_data_order #ifndef SHA256_ASM static -#else -# ifdef INCLUDE_C_SHA256 -void sha256_block_data_order_c(SHA256_CTX *ctx, const void *in, size_t num); -# endif /* INCLUDE_C_SHA256 */ -#endif /* SHA256_ASM */ +#endif void sha256_block_data_order(SHA256_CTX *ctx, const void *in, size_t num); #include "crypto/md32_common.h" -#if !defined(SHA256_ASM) || defined(INCLUDE_C_SHA256) +#ifndef SHA256_ASM static const SHA_LONG K256[64] = { 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, @@ -145,63 +129,18 @@ static const SHA_LONG K256[64] = { 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL }; -# ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && \ - !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if defined(__riscv_zknh) -# define Sigma0(x) ({ MD32_REG_T ret; \ - asm ("sha256sum0 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# define Sigma1(x) ({ MD32_REG_T ret; \ - asm ("sha256sum1 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# define sigma0(x) ({ MD32_REG_T ret; \ - asm ("sha256sig0 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# define sigma1(x) ({ MD32_REG_T ret; \ - asm ("sha256sig1 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# endif -# if defined(__riscv_zbt) || defined(__riscv_zpn) -# define Ch(x,y,z) ({ MD32_REG_T ret; \ - asm (".insn r4 0x33, 1, 0x3, %0, %2, %1, %3"\ - : "=r"(ret) \ - : "r"(x), "r"(y), "r"(z)); ret; }) -# define Maj(x,y,z) ({ MD32_REG_T ret; \ - asm (".insn r4 0x33, 1, 0x3, %0, %2, %1, %3"\ - : "=r"(ret) \ - : "r"(x^z), "r"(y), "r"(x)); ret; }) -# endif -# endif -# endif - /* * FIPS specification refers to right rotations, while our ROTATE macro * is left one. This is why you might notice that rotation coefficients * differ from those observed in FIPS document by 32-N... */ -# ifndef Sigma0 -# define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10)) -# endif -# ifndef Sigma1 -# define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7)) -# endif -# ifndef sigma0 -# define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3)) -# endif -# ifndef sigma1 -# define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10)) -# endif -# ifndef Ch -# define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) -# endif -# ifndef Maj -# define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) -# endif +# define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10)) +# define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7)) +# define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3)) +# define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10)) + +# define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) +# define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) # ifdef OPENSSL_SMALL_FOOTPRINT @@ -283,12 +222,8 @@ static void sha256_block_data_order(SHA256_CTX *ctx, const void *in, T1 = X[(i)&0x0f] += s0 + s1 + X[(i+9)&0x0f]; \ ROUND_00_15(i,a,b,c,d,e,f,g,h); } while (0) -#ifdef INCLUDE_C_SHA256 -void sha256_block_data_order_c(SHA256_CTX *ctx, const void *in, size_t num) -#else static void sha256_block_data_order(SHA256_CTX *ctx, const void *in, size_t num) -#endif { unsigned MD32_REG_T a, b, c, d, e, f, g, h, s0, s1, T1; SHA_LONG X[16]; diff --git a/openssl/src/crypto/sha/sha3.c b/openssl/src/crypto/sha/sha3.c index 2411b3f1f..633bc2e12 100644 --- a/openssl/src/crypto/sha/sha3.c +++ b/openssl/src/crypto/sha/sha3.c @@ -10,13 +10,12 @@ #include #include "internal/sha3.h" -void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r, int next); +void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r); void ossl_sha3_reset(KECCAK1600_CTX *ctx) { memset(ctx->A, 0, sizeof(ctx->A)); ctx->bufsz = 0; - ctx->xof_state = XOF_STATE_INIT; } int ossl_sha3_init(KECCAK1600_CTX *ctx, unsigned char pad, size_t bitlen) @@ -52,10 +51,6 @@ int ossl_sha3_update(KECCAK1600_CTX *ctx, const void *_inp, size_t len) if (len == 0) return 1; - if (ctx->xof_state == XOF_STATE_SQUEEZE - || ctx->xof_state == XOF_STATE_FINAL) - return 0; - if ((num = ctx->bufsz) != 0) { /* process intermediate buffer? */ rem = bsz - num; @@ -89,21 +84,13 @@ int ossl_sha3_update(KECCAK1600_CTX *ctx, const void *_inp, size_t len) return 1; } -/* - * ossl_sha3_final()is a single shot method - * (Use ossl_sha3_squeeze for multiple calls). - * outlen is the variable size output. - */ -int ossl_sha3_final(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen) +int ossl_sha3_final(unsigned char *md, KECCAK1600_CTX *ctx) { size_t bsz = ctx->block_size; size_t num = ctx->bufsz; - if (outlen == 0) + if (ctx->md_size == 0) return 1; - if (ctx->xof_state == XOF_STATE_SQUEEZE - || ctx->xof_state == XOF_STATE_FINAL) - return 0; /* * Pad the data with 10*1. Note that |num| can be |bsz - 1| @@ -116,86 +103,7 @@ int ossl_sha3_final(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen) (void)SHA3_absorb(ctx->A, ctx->buf, bsz, bsz); - ctx->xof_state = XOF_STATE_FINAL; - SHA3_squeeze(ctx->A, out, outlen, bsz, 0); - return 1; -} - -/* - * This method can be called multiple times. - * Rather than heavily modifying assembler for SHA3_squeeze(), - * we instead just use the limitations of the existing function. - * i.e. Only request multiples of the ctx->block_size when calling - * SHA3_squeeze(). For output length requests smaller than the - * ctx->block_size just request a single ctx->block_size bytes and - * buffer the results. The next request will use the buffer first - * to grab output bytes. - */ -int ossl_sha3_squeeze(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen) -{ - size_t bsz = ctx->block_size; - size_t num = ctx->bufsz; - size_t len; - int next = 1; - - if (outlen == 0) - return 1; - - if (ctx->xof_state == XOF_STATE_FINAL) - return 0; - - /* - * On the first squeeze call, finish the absorb process, - * by adding the trailing padding and then doing - * a final absorb. - */ - if (ctx->xof_state != XOF_STATE_SQUEEZE) { - /* - * Pad the data with 10*1. Note that |num| can be |bsz - 1| - * in which case both byte operations below are performed on - * same byte... - */ - memset(ctx->buf + num, 0, bsz - num); - ctx->buf[num] = ctx->pad; - ctx->buf[bsz - 1] |= 0x80; - (void)SHA3_absorb(ctx->A, ctx->buf, bsz, bsz); - ctx->xof_state = XOF_STATE_SQUEEZE; - num = ctx->bufsz = 0; - next = 0; - } - - /* - * Step 1. Consume any bytes left over from a previous squeeze - * (See Step 4 below). - */ - if (num != 0) { - if (outlen > ctx->bufsz) - len = ctx->bufsz; - else - len = outlen; - memcpy(out, ctx->buf + bsz - ctx->bufsz, len); - out += len; - outlen -= len; - ctx->bufsz -= len; - } - if (outlen == 0) - return 1; - - /* Step 2. Copy full sized squeezed blocks to the output buffer directly */ - if (outlen >= bsz) { - len = bsz * (outlen / bsz); - SHA3_squeeze(ctx->A, out, len, bsz, next); - next = 1; - out += len; - outlen -= len; - } - if (outlen > 0) { - /* Step 3. Squeeze one more block into a buffer */ - SHA3_squeeze(ctx->A, ctx->buf, bsz, bsz, next); - memcpy(out, ctx->buf, outlen); - /* Step 4. Remember the leftover part of the squeezed block */ - ctx->bufsz = bsz - outlen; - } + SHA3_squeeze(ctx->A, md, ctx->md_size, bsz); return 1; } diff --git a/openssl/src/crypto/sha/sha512.c b/openssl/src/crypto/sha/sha512.c index bc547d7cd..ff035c469 100644 --- a/openssl/src/crypto/sha/sha512.c +++ b/openssl/src/crypto/sha/sha512.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2004-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,7 +25,7 @@ * on [aligned] data in host byte order and one - on data in input * stream byte order; * - share common byte-order neutral collector and padding function - * implementations, crypto/md32_common.h; + * implementations, ../md32_common.h; * * Neither of the above applies to this SHA-512 implementations. Reasons * [in reverse order] are: @@ -149,10 +149,6 @@ int SHA512_Init(SHA512_CTX *c) #ifndef SHA512_ASM static -#else -# ifdef INCLUDE_C_SHA512 -void sha512_block_data_order_c(SHA512_CTX *ctx, const void *in, size_t num); -# endif #endif void sha512_block_data_order(SHA512_CTX *ctx, const void *in, size_t num); @@ -342,7 +338,7 @@ void SHA512_Transform(SHA512_CTX *c, const unsigned char *data) sha512_block_data_order(c, data, 1); } -#if !defined(SHA512_ASM) || defined(INCLUDE_C_SHA512) +#ifndef SHA512_ASM static const SHA_LONG64 K512[80] = { U64(0x428a2f98d728ae22), U64(0x7137449123ef65cd), U64(0xb5c0fbcfec4d3b2f), U64(0xe9b5dba58189dbbc), @@ -436,103 +432,6 @@ static const SHA_LONG64 K512[80] = { : "=r"(ret) \ : "r"(*((const SHA_LONG64 *)(&(x))))); ret; }) # endif -# elif (defined(__riscv_zbkb) || defined(__riscv_zbb)) && __riscv_xlen == 32 -# define PULL64(x) ({ SHA_LONG64 ret; \ - unsigned int *r = (unsigned int *)(&(ret)); \ - const unsigned int *p = (const unsigned int *)(&(x)); \ - asm ("rev8 %0, %1" \ - : "=r"(r[0]) \ - : "r" (p[1])); \ - asm ("rev8 %0, %1" \ - : "=r"(r[1]) \ - : "r" (p[0])); ret; }) -# elif (defined(__riscv_zbkb) || defined(__riscv_zbb)) && __riscv_xlen == 64 -# define PULL64(x) ({ SHA_LONG64 ret; \ - asm ("rev8 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# endif -# if defined(__riscv_zknh) && __riscv_xlen == 32 -# define Sigma0(x) ({ SHA_LONG64 ret; unsigned int *r = (unsigned int *)(&(ret)); \ - const unsigned int *p = (const unsigned int *)(&(x)); \ - asm ("sha512sum0r %0, %1, %2" \ - : "=r"(r[0]) \ - : "r" (p[0]), "r" (p[1])); \ - asm ("sha512sum0r %0, %2, %1" \ - : "=r"(r[1]) \ - : "r" (p[0]), "r" (p[1])); ret; }) -# define Sigma1(x) ({ SHA_LONG64 ret; unsigned int *r = (unsigned int *)(&(ret)); \ - const unsigned int *p = (const unsigned int *)(&(x)); \ - asm ("sha512sum1r %0, %1, %2" \ - : "=r"(r[0]) \ - : "r" (p[0]), "r" (p[1])); \ - asm ("sha512sum1r %0, %2, %1" \ - : "=r"(r[1]) \ - : "r" (p[0]), "r" (p[1])); ret; }) -# define sigma0(x) ({ SHA_LONG64 ret; unsigned int *r = (unsigned int *)(&(ret)); \ - const unsigned int *p = (const unsigned int *)(&(x)); \ - asm ("sha512sig0l %0, %1, %2" \ - : "=r"(r[0]) \ - : "r" (p[0]), "r" (p[1])); \ - asm ("sha512sig0h %0, %2, %1" \ - : "=r"(r[1]) \ - : "r" (p[0]), "r" (p[1])); ret; }) -# define sigma1(x) ({ SHA_LONG64 ret; unsigned int *r = (unsigned int *)(&(ret)); \ - const unsigned int *p = (const unsigned int *)(&(x)); \ - asm ("sha512sig1l %0, %1, %2" \ - : "=r"(r[0]) \ - : "r" (p[0]), "r" (p[1])); \ - asm ("sha512sig1h %0, %2, %1" \ - : "=r"(r[1]) \ - : "r" (p[0]), "r" (p[1])); ret; }) -# elif defined(__riscv_zknh) && __riscv_xlen == 64 -# define Sigma0(x) ({ SHA_LONG64 ret; \ - asm ("sha512sum0 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# define Sigma1(x) ({ SHA_LONG64 ret; \ - asm ("sha512sum1 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# define sigma0(x) ({ SHA_LONG64 ret; \ - asm ("sha512sig0 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# define sigma1(x) ({ SHA_LONG64 ret; \ - asm ("sha512sig1 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# endif -# if (defined(__riscv_zbt) || defined(__riscv_zpn)) && __riscv_xlen == 32 -# define Ch(x,y,z) ({ SHA_LONG64 ret; unsigned int *r = (unsigned int *)(&(ret)); \ - const unsigned int *xp = (const unsigned int *)(&(x)); \ - const unsigned int *yp = (const unsigned int *)(&(y)); \ - const unsigned int *zp = (const unsigned int *)(&(z)); \ - asm (".insn r4 0x33, 1, 0x3, %0, %2, %1, %3\n\t" \ - : "=r"(r[0]) \ - : "r"(xp[0]), "r"(yp[0]), "r"(zp[0])); \ - asm (".insn r4 0x33, 1, 0x3, %0, %2, %1, %3\n\t" \ - : "=r"(r[1]) \ - : "r"(xp[1]), "r"(yp[1]), "r"(zp[1])); ret; }) -# define Maj(x,y,z) ({ SHA_LONG64 ret; unsigned int *r = (unsigned int *)(&(ret)); \ - const unsigned int *xp = (const unsigned int *)(&(x)); \ - const unsigned int *yp = (const unsigned int *)(&(y)); \ - const unsigned int *zp = (const unsigned int *)(&(z)); \ - asm (".insn r4 0x33, 1, 0x3, %0, %2, %1, %3\n\t" \ - : "=r"(r[0]) \ - : "r"(xp[0]^zp[0]), "r"(yp[0]), "r"(zp[0])); \ - asm (".insn r4 0x33, 1, 0x3, %0, %2, %1, %3\n\t" \ - : "=r"(r[1]) \ - : "r"(xp[1]^zp[1]), "r"(yp[1]), "r"(zp[1])); ret; }) -# elif (defined(__riscv_zbt) || defined(__riscv_zpn)) && __riscv_xlen == 64 -# define Ch(x,y,z) ({ SHA_LONG64 ret; \ - asm (".insn r4 0x33, 1, 0x3, %0, %2, %1, %3"\ - : "=r"(ret) \ - : "r"(x), "r"(y), "r"(z)); ret; }) -# define Maj(x,y,z) ({ SHA_LONG64 ret; \ - asm (".insn r4 0x33, 1, 0x3, %0, %2, %1, %3"\ - : "=r"(ret) \ - : "r"(x^z), "r"(y), "r"(x)); ret; }) # endif # elif defined(_MSC_VER) # if defined(_WIN64) /* applies to both IA-64 and AMD64 */ @@ -573,24 +472,12 @@ static SHA_LONG64 __fastcall __pull64be(const void *x) # ifndef ROTR # define ROTR(x,s) (((x)>>s) | (x)<<(64-s)) # endif -# ifndef Sigma0 -# define Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39)) -# endif -# ifndef Sigma1 -# define Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41)) -# endif -# ifndef sigma0 -# define sigma0(x) (ROTR((x),1) ^ ROTR((x),8) ^ ((x)>>7)) -# endif -# ifndef sigma1 -# define sigma1(x) (ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6)) -# endif -# ifndef Ch -# define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) -# endif -# ifndef Maj -# define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) -# endif +# define Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39)) +# define Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41)) +# define sigma0(x) (ROTR((x),1) ^ ROTR((x),8) ^ ((x)>>7)) +# define sigma1(x) (ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6)) +# define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) +# define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) # if defined(__i386) || defined(__i386__) || defined(_M_IX86) /* @@ -741,12 +628,8 @@ static void sha512_block_data_order(SHA512_CTX *ctx, const void *in, T1 = X[(j)&0x0f] += s0 + s1 + X[(j+9)&0x0f]; \ ROUND_00_15(i+j,a,b,c,d,e,f,g,h); } while (0) -#ifdef INCLUDE_C_SHA512 -void sha512_block_data_order_c(SHA512_CTX *ctx, const void *in, size_t num) -#else static void sha512_block_data_order(SHA512_CTX *ctx, const void *in, size_t num) -#endif { const SHA_LONG64 *W = in; SHA_LONG64 a, b, c, d, e, f, g, h, s0, s1, T1; diff --git a/openssl/src/crypto/sha/sha_riscv.c b/openssl/src/crypto/sha/sha_riscv.c deleted file mode 100644 index c4a77a3d0..000000000 --- a/openssl/src/crypto/sha/sha_riscv.c +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include - -#include -#include -#include "crypto/riscv_arch.h" - -void sha256_block_data_order_zvkb_zvknha_or_zvknhb(void *ctx, const void *in, - size_t num); -void sha256_block_data_order_c(void *ctx, const void *in, size_t num); -void sha256_block_data_order(SHA256_CTX *ctx, const void *in, size_t num); - -void sha256_block_data_order(SHA256_CTX *ctx, const void *in, size_t num) -{ - if (RISCV_HAS_ZVKB() && (RISCV_HAS_ZVKNHA() || RISCV_HAS_ZVKNHB()) && - riscv_vlen() >= 128) { - sha256_block_data_order_zvkb_zvknha_or_zvknhb(ctx, in, num); - } else { - sha256_block_data_order_c(ctx, in, num); - } -} - -void sha512_block_data_order_zvkb_zvknhb(void *ctx, const void *in, size_t num); -void sha512_block_data_order_c(void *ctx, const void *in, size_t num); -void sha512_block_data_order(SHA512_CTX *ctx, const void *in, size_t num); - -void sha512_block_data_order(SHA512_CTX *ctx, const void *in, size_t num) -{ - if (RISCV_HAS_ZVKB_AND_ZVKNHB() && riscv_vlen() >= 128) { - sha512_block_data_order_zvkb_zvknhb(ctx, in, num); - } else { - sha512_block_data_order_c(ctx, in, num); - } -} diff --git a/openssl/src/crypto/siphash/siphash.c b/openssl/src/crypto/siphash/siphash.c index e2be3ca59..57f61c1db 100644 --- a/openssl/src/crypto/siphash/siphash.c +++ b/openssl/src/crypto/siphash/siphash.c @@ -210,22 +210,22 @@ int SipHash_Final(SIPHASH *ctx, unsigned char *out, size_t outlen) switch (ctx->len) { case 7: b |= ((uint64_t)ctx->leavings[6]) << 48; - /* fall through */ + /* fall thru */ case 6: b |= ((uint64_t)ctx->leavings[5]) << 40; - /* fall through */ + /* fall thru */ case 5: b |= ((uint64_t)ctx->leavings[4]) << 32; - /* fall through */ + /* fall thru */ case 4: b |= ((uint64_t)ctx->leavings[3]) << 24; - /* fall through */ + /* fall thru */ case 3: b |= ((uint64_t)ctx->leavings[2]) << 16; - /* fall through */ + /* fall thru */ case 2: b |= ((uint64_t)ctx->leavings[1]) << 8; - /* fall through */ + /* fall thru */ case 1: b |= ((uint64_t)ctx->leavings[0]); case 0: diff --git a/openssl/src/crypto/sleep.c b/openssl/src/crypto/sleep.c deleted file mode 100644 index 73467fb85..000000000 --- a/openssl/src/crypto/sleep.c +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/e_os.h" - -/* system-specific variants defining OSSL_sleep() */ -#if defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__) -#include - -void OSSL_sleep(uint64_t millis) -{ -# ifdef OPENSSL_SYS_VXWORKS - struct timespec ts; - - ts.tv_sec = (long int) (millis / 1000); - ts.tv_nsec = (long int) (millis % 1000) * 1000000ul; - nanosleep(&ts, NULL); -# elif defined(__TANDEM) && !defined(_REENTRANT) -# include - - /* HPNS does not support usleep for non threaded apps */ - PROCESS_DELAY_(millis * 1000); -# else - unsigned int s = (unsigned int)(millis / 1000); - unsigned int us = (unsigned int)((millis % 1000) * 1000); - - if (s > 0) - sleep(s); - usleep(us); -# endif -} -#elif defined(_WIN32) && !defined(OPENSSL_SYS_UEFI) -# include - -void OSSL_sleep(uint64_t millis) -{ - /* - * Windows' Sleep() takes a DWORD argument, which is smaller than - * a uint64_t, so we need to limit it to 49 days, which should be enough. - */ - DWORD limited_millis = (DWORD)-1; - - if (millis < limited_millis) - limited_millis = (DWORD)millis; - Sleep(limited_millis); -} - -#else -/* Fallback to a busy wait */ -# include "internal/time.h" - -static void ossl_sleep_secs(uint64_t secs) -{ - /* - * sleep() takes an unsigned int argument, which is smaller than - * a uint64_t, so it needs to be limited to 136 years which - * should be enough even for Sleeping Beauty. - */ - unsigned int limited_secs = UINT_MAX; - - if (secs < limited_secs) - limited_secs = (unsigned int)secs; - sleep(limited_secs); -} - -static void ossl_sleep_millis(uint64_t millis) -{ - const OSSL_TIME finish - = ossl_time_add(ossl_time_now(), ossl_ms2time(millis)); - - while (ossl_time_compare(ossl_time_now(), finish) < 0) - /* busy wait */ ; -} - -void OSSL_sleep(uint64_t millis) -{ - ossl_sleep_secs(millis / 1000); - ossl_sleep_millis(millis % 1000); -} -#endif /* defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__) */ diff --git a/openssl/src/crypto/sm2/sm2_crypt.c b/openssl/src/crypto/sm2/sm2_crypt.c index b7303af52..dc2a86ab5 100644 --- a/openssl/src/crypto/sm2/sm2_crypt.c +++ b/openssl/src/crypto/sm2/sm2_crypt.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2017 Ribose Inc. All Rights Reserved. * Ported from Ribose contributions from Botan. * @@ -26,7 +26,7 @@ #include typedef struct SM2_Ciphertext_st SM2_Ciphertext; -DECLARE_ASN1_FUNCTIONS(SM2_Ciphertext) +DECLARE_STATIC_ASN1_FUNCTIONS(SM2_Ciphertext) struct SM2_Ciphertext_st { BIGNUM *C1x; @@ -40,30 +40,31 @@ ASN1_SEQUENCE(SM2_Ciphertext) = { ASN1_SIMPLE(SM2_Ciphertext, C1y, BIGNUM), ASN1_SIMPLE(SM2_Ciphertext, C3, ASN1_OCTET_STRING), ASN1_SIMPLE(SM2_Ciphertext, C2, ASN1_OCTET_STRING), -} ASN1_SEQUENCE_END(SM2_Ciphertext) +} static_ASN1_SEQUENCE_END(SM2_Ciphertext) -IMPLEMENT_ASN1_FUNCTIONS(SM2_Ciphertext) +IMPLEMENT_STATIC_ASN1_FUNCTIONS(SM2_Ciphertext) static size_t ec_field_size(const EC_GROUP *group) { - const BIGNUM *p = EC_GROUP_get0_field(group); + /* Is there some simpler way to do this? */ + BIGNUM *p = BN_new(); + BIGNUM *a = BN_new(); + BIGNUM *b = BN_new(); + size_t field_size = 0; - if (p == NULL) - return 0; - - return BN_num_bytes(p); -} + if (p == NULL || a == NULL || b == NULL) + goto done; -static int is_all_zeros(const unsigned char *msg, size_t msglen) -{ - unsigned char re = 0; - size_t i; + if (!EC_GROUP_get_curve(group, p, a, b, NULL)) + goto done; + field_size = (BN_num_bits(p) + 7) / 8; - for (i = 0; i < msglen; i++) { - re |= msg[i]; - } + done: + BN_free(p); + BN_free(a); + BN_free(b); - return re == 0 ? 1 : 0; + return field_size; } int ossl_sm2_plaintext_size(const unsigned char *ct, size_t ct_size, @@ -150,13 +151,9 @@ int ossl_sm2_encrypt(const EC_KEY *key, kG = EC_POINT_new(group); kP = EC_POINT_new(group); - if (kG == NULL || kP == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); - goto done; - } ctx = BN_CTX_new_ex(libctx); - if (ctx == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + if (kG == NULL || kP == NULL || ctx == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -175,16 +172,13 @@ int ossl_sm2_encrypt(const EC_KEY *key, x2y2 = OPENSSL_zalloc(2 * field_size); C3 = OPENSSL_zalloc(C3_size); - if (x2y2 == NULL || C3 == NULL) + if (x2y2 == NULL || C3 == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; + } memset(ciphertext_buf, 0, *ciphertext_len); - msg_mask = OPENSSL_zalloc(msg_len); - if (msg_mask == NULL) - goto done; - -again: if (!BN_priv_rand_range_ex(k, order, 0, ctx)) { ERR_raise(ERR_LIB_SM2, ERR_R_INTERNAL_ERROR); goto done; @@ -204,6 +198,12 @@ int ossl_sm2_encrypt(const EC_KEY *key, goto done; } + msg_mask = OPENSSL_zalloc(msg_len); + if (msg_mask == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); + goto done; + } + /* X9.63 with no salt happens to match the KDF used in SM2 */ if (!ossl_ecdh_kdf_X9_63(msg_mask, msg_len, x2y2, 2 * field_size, NULL, 0, digest, libctx, propq)) { @@ -211,11 +211,6 @@ int ossl_sm2_encrypt(const EC_KEY *key, goto done; } - if (is_all_zeros(msg_mask, msg_len)) { - memset(x2y2, 0, 2 * field_size); - goto again; - } - for (i = 0; i != msg_len; ++i) msg_mask[i] ^= msg[i]; @@ -239,7 +234,7 @@ int ossl_sm2_encrypt(const EC_KEY *key, ctext_struct.C2 = ASN1_OCTET_STRING_new(); if (ctext_struct.C3 == NULL || ctext_struct.C2 == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } if (!ASN1_OCTET_STRING_set(ctext_struct.C3, C3, C3_size) @@ -324,7 +319,7 @@ int ossl_sm2_decrypt(const EC_KEY *key, ctx = BN_CTX_new_ex(libctx); if (ctx == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -341,12 +336,14 @@ int ossl_sm2_decrypt(const EC_KEY *key, x2y2 = OPENSSL_zalloc(2 * field_size); computed_C3 = OPENSSL_zalloc(hash_size); - if (msg_mask == NULL || x2y2 == NULL || computed_C3 == NULL) + if (msg_mask == NULL || x2y2 == NULL || computed_C3 == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; + } C1 = EC_POINT_new(group); if (C1 == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -367,17 +364,12 @@ int ossl_sm2_decrypt(const EC_KEY *key, goto done; } - if (is_all_zeros(msg_mask, msg_len)) { - ERR_raise(ERR_LIB_SM2, SM2_R_INVALID_ENCODING); - goto done; - } - for (i = 0; i != msg_len; ++i) ptext_buf[i] = C2[i] ^ msg_mask[i]; hash = EVP_MD_CTX_new(); if (hash == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } diff --git a/openssl/src/crypto/sm2/sm2_err.c b/openssl/src/crypto/sm2/sm2_err.c index d420d4e59..946dd62e0 100644 --- a/openssl/src/crypto/sm2/sm2_err.c +++ b/openssl/src/crypto/sm2/sm2_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -31,6 +31,8 @@ static const ERR_STRING_DATA SM2_str_reasons[] = { {ERR_PACK(ERR_LIB_SM2, 0, SM2_R_INVALID_PRIVATE_KEY), "invalid private key"}, {ERR_PACK(ERR_LIB_SM2, 0, SM2_R_NO_PARAMETERS_SET), "no parameters set"}, + {ERR_PACK(ERR_LIB_SM2, 0, SM2_R_POINT_ARITHMETIC_FAILURE), + "point arithmetic failure"}, {ERR_PACK(ERR_LIB_SM2, 0, SM2_R_USER_ID_TOO_LARGE), "user id too large"}, {0, NULL} }; diff --git a/openssl/src/crypto/sm2/sm2_key.c b/openssl/src/crypto/sm2/sm2_key.c index e3a10d38e..9d0b9208f 100644 --- a/openssl/src/crypto/sm2/sm2_key.c +++ b/openssl/src/crypto/sm2/sm2_key.c @@ -29,7 +29,7 @@ int ossl_sm2_key_private_check(const EC_KEY *eckey) if (eckey == NULL || (group = EC_KEY_get0_group(eckey)) == NULL || (priv_key = EC_KEY_get0_private_key(eckey)) == NULL - || (order = EC_GROUP_get0_order(group)) == NULL) { + || (order = EC_GROUP_get0_order(group)) == NULL ) { ERR_raise(ERR_LIB_SM2, ERR_R_PASSED_NULL_PARAMETER); return 0; } diff --git a/openssl/src/crypto/sm2/sm2_kmeth.c b/openssl/src/crypto/sm2/sm2_kmeth.c new file mode 100644 index 000000000..640d55d1e --- /dev/null +++ b/openssl/src/crypto/sm2/sm2_kmeth.c @@ -0,0 +1,255 @@ +/* + * Copyright 2022-2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "internal/deprecated.h" + +#include "internal/cryptlib.h" +#include +#include +#include +#include "crypto/sm2.h" +#include "crypto/ec.h" /* ecdh_KDF_X9_63() */ +#include "crypto/sm2err.h" + + +int SM2_compute_key(void *out, size_t outlen, int initiator, + const uint8_t *peer_id, size_t peer_id_len, + const uint8_t *self_id, size_t self_id_len, + const EC_KEY *peer_ecdhe_key, const EC_KEY *self_ecdhe_key, + const EC_KEY *peer_pub_key, const EC_KEY *self_eckey, + const EVP_MD *md, OSSL_LIB_CTX *libctx, + const char *propq) +{ + BN_CTX *ctx = NULL; + EC_POINT *UorV = NULL; + const EC_POINT *Rs, *Rp; + BIGNUM *Xuv = NULL, *Yuv = NULL, *Xs = NULL, *Xp = NULL; + BIGNUM *h = NULL, *t = NULL, *two_power_w = NULL, *order = NULL; + const BIGNUM *priv_key, *r; + const EC_GROUP *group; + int w; + int ret = 0; + size_t buflen = 0, md_len; + unsigned char *buf = NULL; + size_t field_len, idx = 0; + + if (peer_id == NULL || self_id == NULL || peer_ecdhe_key == NULL + || self_ecdhe_key == NULL || peer_pub_key == NULL + || self_eckey == NULL || md == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (outlen > INT_MAX) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); + return 0; + } + + priv_key = EC_KEY_get0_private_key(self_eckey); + if (priv_key == NULL) { + ERR_raise(ERR_LIB_SM2, SM2_R_INVALID_PRIVATE_KEY); + return 0; + } + + Rs = EC_KEY_get0_public_key(self_ecdhe_key); + Rp = EC_KEY_get0_public_key(peer_ecdhe_key); + r = EC_KEY_get0_private_key(self_ecdhe_key); + + if (Rs == NULL || Rp == NULL || r == NULL) { + ERR_raise(ERR_LIB_SM2, SM2_R_INVALID_PRIVATE_KEY); + return 0; + } + + ctx = BN_CTX_new_ex(libctx); + if (ctx == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); + goto err; + } + + BN_CTX_start(ctx); + Xuv = BN_CTX_get(ctx); + Yuv = BN_CTX_get(ctx); + Xs = BN_CTX_get(ctx); + Xp = BN_CTX_get(ctx); + h = BN_CTX_get(ctx); + t = BN_CTX_get(ctx); + two_power_w = BN_CTX_get(ctx); + order = BN_CTX_get(ctx); + + if (order == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); + goto err; + } + + group = EC_KEY_get0_group(self_eckey); + + if (!EC_GROUP_get_order(group, order, ctx) + || !EC_GROUP_get_cofactor(group, h, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); + goto err; + } + + w = (BN_num_bits(order) + 1) / 2 - 1; + if (!BN_lshift(two_power_w, BN_value_one(), w)) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + /*Third: Caculate -- X = 2 ^ w + (x & (2 ^ w - 1)) = 2 ^ w + (x mod 2 ^ w)*/ + UorV = EC_POINT_new(group); + if (UorV == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); + goto err; + } + + /* Test peer public key On curve */ + if (!EC_POINT_is_on_curve(group, Rp, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); + goto err; + } + + /* Get x */ + if (!EC_POINT_get_affine_coordinates(group, Rs, Xs, NULL, ctx) + || !EC_POINT_get_affine_coordinates(group, Rp, Xp, NULL, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); + goto err; + } + + /*x mod 2 ^ w*/ + /*Caculate Self x*/ + if (!BN_nnmod(Xs, Xs, two_power_w, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + if (!BN_add(Xs, Xs, two_power_w)) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + /*Caculate Peer x*/ + if (!BN_nnmod(Xp, Xp, two_power_w, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + if (!BN_add(Xp, Xp, two_power_w)) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + /*Forth: Caculate t*/ + if (!BN_mod_mul(t, Xs, r, order, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + if (!BN_mod_add(t, t, priv_key, order, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + /*Fifth: Caculate V or U*/ + if (!BN_mul(t, t, h, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + /* [x]R */ + if (!EC_POINT_mul(group, UorV, NULL, Rp, Xp, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); + goto err; + } + + /* P + [x]R */ + if (!EC_POINT_add(group, UorV, UorV, + EC_KEY_get0_public_key(peer_pub_key), ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); + goto err; + } + + if (!EC_POINT_mul(group, UorV, NULL, UorV, t, ctx)) { + ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); + goto err; + } + + if (EC_POINT_is_at_infinity(group, UorV)) { + ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); + goto err; + } + + /*Sixth: Caculate Key -- Need Xuorv, Yuorv, Z_A, Z_B, klen*/ + if (!EC_POINT_get_affine_coordinates(group, UorV, Xuv, Yuv, ctx)) { + ERR_raise(ERR_LIB_SM2, SM2_R_POINT_ARITHMETIC_FAILURE); + goto err; + } + + field_len = ((size_t)EC_GROUP_get_degree(group) + 7) / 8; + md_len = EVP_MD_size(md); + + /* Xuorv || Yuorv || Z_A || Z_B */ + buflen = field_len * 2 + md_len * 2 ; + + buf = OPENSSL_secure_malloc(buflen); + if (buf == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); + goto err; + } + + /*1 : Get public key for UorV, Notice: the first byte is a tag, not a valid char*/ + if (BN_bn2binpad(Xuv, buf, field_len) < 0 + || BN_bn2binpad(Yuv, buf + field_len, field_len) < 0) { + ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + goto err; + } + + idx += field_len * 2; + + if (initiator) { + if (!ossl_sm2_compute_z_digest((uint8_t *)(buf + idx), md, + self_id, self_id_len, + self_eckey)) + goto err; + + idx += md_len; + } + + if (!ossl_sm2_compute_z_digest((uint8_t *)(buf + idx), md, + peer_id, peer_id_len, + peer_pub_key)) + goto err; + + idx += md_len; + + if (!initiator) { + if (!ossl_sm2_compute_z_digest((uint8_t *)(buf + idx), md, + self_id, self_id_len, + self_eckey)) + goto err; + + idx += md_len; + } + + if (!ossl_ecdh_kdf_X9_63(out, outlen, buf, idx, NULL, 0, md, libctx, + propq)) { + ERR_raise(ERR_LIB_SM2, ERR_R_INTERNAL_ERROR); + goto err; + } + + ret = outlen; + + err: + EC_POINT_free(UorV); + OPENSSL_secure_clear_free(buf, buflen); + if (ctx != NULL) + BN_CTX_end(ctx); + BN_CTX_free(ctx); + + return ret; +} diff --git a/openssl/src/crypto/sm2/sm2_sign.c b/openssl/src/crypto/sm2/sm2_sign.c index 9ddf889ed..104096766 100644 --- a/openssl/src/crypto/sm2/sm2_sign.c +++ b/openssl/src/crypto/sm2/sm2_sign.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2017 Ribose Inc. All Rights Reserved. * Ported from Ribose contributions from Botan. * @@ -17,18 +17,18 @@ #include "internal/numbers.h" #include #include +#include #include #include int ossl_sm2_compute_z_digest(uint8_t *out, const EVP_MD *digest, const uint8_t *id, - const size_t id_len, + size_t id_len, const EC_KEY *key) { int rc = 0; const EC_GROUP *group = EC_KEY_get0_group(key); - const EC_POINT *pubkey = EC_KEY_get0_public_key(key); BN_CTX *ctx = NULL; EVP_MD_CTX *hash = NULL; BIGNUM *p = NULL; @@ -43,20 +43,10 @@ int ossl_sm2_compute_z_digest(uint8_t *out, uint16_t entl = 0; uint8_t e_byte = 0; - /* SM2 Signatures require a public key, check for it */ - if (pubkey == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_PASSED_NULL_PARAMETER); - goto done; - } - hash = EVP_MD_CTX_new(); - if (hash == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_EVP_LIB); - goto done; - } ctx = BN_CTX_new_ex(ossl_ec_key_get_libctx(key)); - if (ctx == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + if (hash == NULL || ctx == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -69,7 +59,7 @@ int ossl_sm2_compute_z_digest(uint8_t *out, yA = BN_CTX_get(ctx); if (yA == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -80,6 +70,11 @@ int ossl_sm2_compute_z_digest(uint8_t *out, /* Z = h(ENTL || ID || a || b || xG || yG || xA || yA) */ + if (id == NULL) { + id = (const uint8_t *)SM2_DEFAULT_USERID; + id_len = strlen(SM2_DEFAULT_USERID); + } + if (id_len >= (UINT16_MAX / 8)) { /* too large */ ERR_raise(ERR_LIB_SM2, SM2_R_ID_TOO_LARGE); @@ -111,8 +106,10 @@ int ossl_sm2_compute_z_digest(uint8_t *out, p_bytes = BN_num_bytes(p); buf = OPENSSL_zalloc(p_bytes); - if (buf == NULL) + if (buf == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; + } if (BN_bn2binpad(a, buf, p_bytes) < 0 || !EVP_DigestUpdate(hash, buf, p_bytes) @@ -126,7 +123,7 @@ int ossl_sm2_compute_z_digest(uint8_t *out, || BN_bn2binpad(yG, buf, p_bytes) < 0 || !EVP_DigestUpdate(hash, buf, p_bytes) || !EC_POINT_get_affine_coordinates(group, - pubkey, + EC_KEY_get0_public_key(key), xA, yA, ctx) || BN_bn2binpad(xA, buf, p_bytes) < 0 || !EVP_DigestUpdate(hash, buf, p_bytes) @@ -164,14 +161,12 @@ static BIGNUM *sm2_compute_msg_hash(const EVP_MD *digest, ERR_raise(ERR_LIB_SM2, SM2_R_INVALID_DIGEST); goto done; } - if (hash == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_EVP_LIB); - goto done; - } z = OPENSSL_zalloc(md_size); - if (z == NULL) + if (hash == NULL || z == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; + } fetched_digest = EVP_MD_fetch(libctx, EVP_MD_get0_name(digest), propq); if (fetched_digest == NULL) { @@ -221,13 +216,9 @@ static ECDSA_SIG *sm2_sig_gen(const EC_KEY *key, const BIGNUM *e) OSSL_LIB_CTX *libctx = ossl_ec_key_get_libctx(key); kG = EC_POINT_new(group); - if (kG == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); - goto done; - } ctx = BN_CTX_new_ex(libctx); - if (ctx == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + if (kG == NULL || ctx == NULL) { + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -237,7 +228,7 @@ static ECDSA_SIG *sm2_sig_gen(const EC_KEY *key, const BIGNUM *e) x1 = BN_CTX_get(ctx); tmp = BN_CTX_get(ctx); if (tmp == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -249,7 +240,7 @@ static ECDSA_SIG *sm2_sig_gen(const EC_KEY *key, const BIGNUM *e) s = BN_new(); if (r == NULL || s == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -303,7 +294,7 @@ static ECDSA_SIG *sm2_sig_gen(const EC_KEY *key, const BIGNUM *e) sig = ECDSA_SIG_new(); if (sig == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_ECDSA_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -340,7 +331,7 @@ static int sm2_sig_verify(const EC_KEY *key, const ECDSA_SIG *sig, ctx = BN_CTX_new_ex(libctx); pt = EC_POINT_new(group); if (ctx == NULL || pt == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -348,7 +339,7 @@ static int sm2_sig_verify(const EC_KEY *key, const ECDSA_SIG *sig, t = BN_CTX_get(ctx); x1 = BN_CTX_get(ctx); if (x1 == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } @@ -397,7 +388,6 @@ static int sm2_sig_verify(const EC_KEY *key, const ECDSA_SIG *sig, ret = 1; done: - BN_CTX_end(ctx); EC_POINT_free(pt); BN_CTX_free(ctx); return ret; @@ -457,11 +447,6 @@ int ossl_sm2_internal_sign(const unsigned char *dgst, int dgstlen, int sigleni; int ret = -1; - if (sig == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_PASSED_NULL_PARAMETER); - goto done; - } - e = BN_bin2bn(dgst, dgstlen, NULL); if (e == NULL) { ERR_raise(ERR_LIB_SM2, ERR_R_BN_LIB); @@ -502,7 +487,7 @@ int ossl_sm2_internal_verify(const unsigned char *dgst, int dgstlen, s = ECDSA_SIG_new(); if (s == NULL) { - ERR_raise(ERR_LIB_SM2, ERR_R_ECDSA_LIB); + ERR_raise(ERR_LIB_SM2, ERR_R_MALLOC_FAILURE); goto done; } if (d2i_ECDSA_SIG(&s, &p, sig_len) == NULL) { diff --git a/openssl/src/crypto/sm3/gen/darwin_arm64/sm3-armv8.S b/openssl/src/crypto/sm3/gen/darwin_arm64/sm3-armv8.S index ac7f0b8a0..10401eedd 100644 --- a/openssl/src/crypto/sm3/gen/darwin_arm64/sm3-armv8.S +++ b/openssl/src/crypto/sm3/gen/darwin_arm64/sm3-armv8.S @@ -1,4 +1,4 @@ -// Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. +// Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. // // Licensed under the Apache License 2.0 (the "License"). You may not use // this file except in compliance with the License. You can obtain a copy @@ -10,12 +10,12 @@ // $output is the last argument if it looks like a file (it has an extension) // $flavour is the first argument if it doesn't look like a file #include "arm_arch.h" + .text .globl _ossl_hwsm3_block_data_order .align 5 _ossl_hwsm3_block_data_order: - AARCH64_VALID_CALL_TARGET // load state ld1 {v5.4s,v6.4s}, [x0] rev64 v5.4s, v5.4s diff --git a/openssl/src/crypto/sm3/gen/linux_arm64/sm3-armv8.S b/openssl/src/crypto/sm3/gen/linux_arm64/sm3-armv8.S index 537ac82d3..51cdd7d89 100644 --- a/openssl/src/crypto/sm3/gen/linux_arm64/sm3-armv8.S +++ b/openssl/src/crypto/sm3/gen/linux_arm64/sm3-armv8.S @@ -1,4 +1,4 @@ -// Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. +// Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. // // Licensed under the Apache License 2.0 (the "License"). You may not use // this file except in compliance with the License. You can obtain a copy @@ -10,12 +10,12 @@ // $output is the last argument if it looks like a file (it has an extension) // $flavour is the first argument if it doesn't look like a file #include "arm_arch.h" +.arch armv8.2-a .text .globl ossl_hwsm3_block_data_order .type ossl_hwsm3_block_data_order,%function .align 5 ossl_hwsm3_block_data_order: - AARCH64_VALID_CALL_TARGET // load state ld1 {v5.4s,v6.4s}, [x0] rev64 v5.4s, v5.4s diff --git a/openssl/src/crypto/sm3/gen/linux_riscv64/sm3-riscv64-zvksh.S b/openssl/src/crypto/sm3/gen/linux_riscv64/sm3-riscv64-zvksh.S deleted file mode 100644 index a5457b974..000000000 --- a/openssl/src/crypto/sm3/gen/linux_riscv64/sm3-riscv64-zvksh.S +++ /dev/null @@ -1,150 +0,0 @@ -.text -.text -.p2align 3 -.globl ossl_hwsm3_block_data_order_zvksh -.type ossl_hwsm3_block_data_order_zvksh,@function -ossl_hwsm3_block_data_order_zvksh: - .word 3440668759 - - # Load initial state of hash context (c->A-H). - .word 33906695 - .word 1241817175 - -L_sm3_loop: - # Copy the previous state to v2. - # It will be XOR'ed with the current state at the end of the round. - .word 1577058647 - - # Load the 64B block in 2x32B chunks. - .word 33940231 # v6 := {w7, ..., w0} - addi a1, a1, 32 - - .word 33940487 # v8 := {w15, ..., w8} - addi a1, a1, 32 - - addi a2, a2, -1 - - # As vsm3c consumes only w0, w1, w4, w5 we need to slide the input - # 2 elements down so we process elements w2, w3, w6, w7 - # This will be repeated for each odd round. - .word 1046557271 # v4 := {X, X, w7, ..., w2} - - .word 2925535351 - .word 2923470967 - - # Prepare a vector with {w11, ..., w4} - .word 1044460119 # v4 := {X, X, X, X, w7, ..., w4} - .word 981611095 # v4 := {w11, w10, w9, w8, w7, w6, w5, w4} - - .word 2923503735 - .word 1044460119 # v4 := {X, X, w11, w10, w9, w8, w7, w6} - .word 2923536503 - - .word 2927763575 - .word 1048654423 # v4 := {X, X, w15, w14, w13, w12, w11, w10} - .word 2923602039 - - .word 2189632375 # v6 := {w23, w22, w21, w20, w19, w18, w17, w16} - - # Prepare a register with {w19, w18, w17, w16, w15, w14, w13, w12} - .word 1044460119 # v4 := {X, X, X, X, w15, w14, w13, w12} - .word 979513943 # v4 := {w19, w18, w17, w16, w15, w14, w13, w12} - - .word 2923634807 - .word 1044460119 # v4 := {X, X, w19, w18, w17, w16, w15, w14} - .word 2923667575 - - .word 2925797495 - .word 1046557271 # v4 := {X, X, w23, w22, w21, w20, w19, w18} - .word 2923733111 - - .word 2187601015 # v8 := {w31, w30, w29, w28, w27, w26, w25, w24} - - # Prepare a register with {w27, w26, w25, w24, w23, w22, w21, w20} - .word 1044460119 # v4 := {X, X, X, X, w23, w22, w21, w20} - .word 981611095 # v4 := {w27, w26, w25, w24, w23, w22, w21, w20} - - .word 2923765879 - .word 1044460119 # v4 := {X, X, w27, w26, w25, w24, w23, w22} - .word 2923798647 - - .word 2928025719 - .word 1048654423 # v4 := {x, X, w31, w30, w29, w28, w27, w26} - .word 2923864183 - - .word 2189632375 # v6 := {w32, w33, w34, w35, w36, w37, w38, w39} - - # Prepare a register with {w35, w34, w33, w32, w31, w30, w29, w28} - .word 1044460119 # v4 := {X, X, X, X, w31, w30, w29, w28} - .word 979513943 # v4 := {w35, w34, w33, w32, w31, w30, w29, w28} - - .word 2923896951 - .word 1044460119 # v4 := {X, X, w35, w34, w33, w32, w31, w30} - .word 2923929719 - - .word 2926059639 - .word 1046557271 # v4 := {X, X, w39, w38, w37, w36, w35, w34} - .word 2923995255 - - .word 2187601015 # v8 := {w47, w46, w45, w44, w43, w42, w41, w40} - - # Prepare a register with {w43, w42, w41, w40, w39, w38, w37, w36} - .word 1044460119 # v4 := {X, X, X, X, w39, w38, w37, w36} - .word 981611095 # v4 := {w43, w42, w41, w40, w39, w38, w37, w36} - - .word 2924028023 - .word 1044460119 # v4 := {X, X, w43, w42, w41, w40, w39, w38} - .word 2924060791 - - .word 2928287863 - .word 1048654423 # v4 := {X, X, w47, w46, w45, w44, w43, w42} - .word 2924126327 - - .word 2189632375 # v6 := {w55, w54, w53, w52, w51, w50, w49, w48} - - # Prepare a register with {w51, w50, w49, w48, w47, w46, w45, w44} - .word 1044460119 # v4 := {X, X, X, X, w47, w46, w45, w44} - .word 979513943 # v4 := {w51, w50, w49, w48, w47, w46, w45, w44} - - .word 2924159095 - .word 1044460119 # v4 := {X, X, w51, w50, w49, w48, w47, w46} - .word 2924191863 - - .word 2926321783 - .word 1046557271 # v4 := {X, X, w55, w54, w53, w52, w51, w50} - .word 2924257399 - - .word 2187601015 # v8 := {w63, w62, w61, w60, w59, w58, w57, w56} - - # Prepare a register with {w59, w58, w57, w56, w55, w54, w53, w52} - .word 1044460119 # v4 := {X, X, X, X, w55, w54, w53, w52} - .word 981611095 # v4 := {w59, w58, w57, w56, w55, w54, w53, w52} - - .word 2924290167 - .word 1044460119 # v4 := {X, X, w59, w58, w57, w56, w55, w54} - .word 2924322935 - - .word 2928550007 - .word 1048654423 # v4 := {X, X, w63, w62, w61, w60, w59, w58} - .word 2924388471 - - .word 2189632375 # v6 := {w71, w70, w69, w68, w67, w66, w65, w64} - - # Prepare a register with {w67, w66, w65, w64, w63, w62, w61, w60} - .word 1044460119 # v4 := {X, X, X, X, w63, w62, w61, w60} - .word 979513943 # v4 := {w67, w66, w65, w64, w63, w62, w61, w60} - - .word 2924421239 - .word 1044460119 # v4 := {X, X, w67, w66, w65, w64, w63, w62} - .word 2924454007 - - # XOR in the previous state. - .word 771817559 - - bnez a2, L_sm3_loop # Check if there are any more block to process -L_sm3_end: - .word 1241817175 - .word 33906727 - ret - -.size ossl_hwsm3_block_data_order_zvksh,.-ossl_hwsm3_block_data_order_zvksh diff --git a/openssl/src/crypto/sm3/legacy_sm3.c b/openssl/src/crypto/sm3/legacy_sm3.c index c81f3b4ab..033d2095d 100644 --- a/openssl/src/crypto/sm3/legacy_sm3.c +++ b/openssl/src/crypto/sm3/legacy_sm3.c @@ -8,12 +8,13 @@ * https://www.openssl.org/source/license.html */ +#include "internal/deprecated.h" #include "crypto/evp.h" #include "../evp/legacy_meth.h" -#include "internal/sm3.h" +#include -IMPLEMENT_LEGACY_EVP_MD_METH_LC(sm3_int, ossl_sm3) +IMPLEMENT_LEGACY_EVP_MD_METH(sm3_int, SM3) static const EVP_MD sm3_md = { NID_sm3, diff --git a/openssl/src/crypto/sm3/sm3.c b/openssl/src/crypto/sm3/sm3.c index ff78fb763..619f6b6e9 100644 --- a/openssl/src/crypto/sm3/sm3.c +++ b/openssl/src/crypto/sm3/sm3.c @@ -8,11 +8,12 @@ * in the file LICENSE in the source distribution or at * https://www.openssl.org/source/license.html */ +#include "internal/deprecated.h" #include #include "sm3_local.h" -int ossl_sm3_init(SM3_CTX *c) +int SM3_Init(SM3_CTX *c) { memset(c, 0, sizeof(*c)); c->A = SM3_A; diff --git a/openssl/src/crypto/sm3/sm3_local.h b/openssl/src/crypto/sm3/sm3_local.h index d2845f967..da53037c3 100644 --- a/openssl/src/crypto/sm3/sm3_local.h +++ b/openssl/src/crypto/sm3/sm3_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2017 Ribose Inc. All Rights Reserved. * Ported from Ribose contributions from Botan. * @@ -10,16 +10,16 @@ */ #include -#include "internal/sm3.h" +#include #define DATA_ORDER_IS_BIG_ENDIAN #define HASH_LONG SM3_WORD #define HASH_CTX SM3_CTX #define HASH_CBLOCK SM3_CBLOCK -#define HASH_UPDATE ossl_sm3_update -#define HASH_TRANSFORM ossl_sm3_transform -#define HASH_FINAL ossl_sm3_final +#define HASH_UPDATE SM3_Update +#define HASH_TRANSFORM SM3_Transform +#define HASH_FINAL SM3_Final #define HASH_MAKE_STRING(c, s) \ do { \ unsigned long ll; \ @@ -34,16 +34,11 @@ } while (0) #if defined(OPENSSL_SM3_ASM) -# if defined(__aarch64__) || defined(_M_ARM64) +# if defined(__aarch64__) # include "crypto/arm_arch.h" # define HWSM3_CAPABLE (OPENSSL_armcap_P & ARMV8_SM3) void ossl_hwsm3_block_data_order(SM3_CTX *c, const void *p, size_t num); # endif -# if defined(__riscv) && __riscv_xlen == 64 -# include "crypto/riscv_arch.h" -# define HWSM3_CAPABLE 1 -void ossl_hwsm3_block_data_order(SM3_CTX *c, const void *p, size_t num); -# endif #endif #if defined(HWSM3_CAPABLE) @@ -54,32 +49,11 @@ void ossl_hwsm3_block_data_order(SM3_CTX *c, const void *p, size_t num); #endif void ossl_sm3_block_data_order(SM3_CTX *c, const void *p, size_t num); -void ossl_sm3_transform(SM3_CTX *c, const unsigned char *data); #include "crypto/md32_common.h" -#ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && \ - !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if defined(__riscv_zksh) -# define P0(x) ({ MD32_REG_T ret; \ - asm ("sm3p0 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# define P1(x) ({ MD32_REG_T ret; \ - asm ("sm3p1 %0, %1" \ - : "=r"(ret) \ - : "r"(x)); ret; }) -# endif -# endif -#endif - -#ifndef P0 -# define P0(X) (X ^ ROTATE(X, 9) ^ ROTATE(X, 17)) -#endif -#ifndef P1 -# define P1(X) (X ^ ROTATE(X, 15) ^ ROTATE(X, 23)) -#endif +#define P0(X) (X ^ ROTATE(X, 9) ^ ROTATE(X, 17)) +#define P1(X) (X ^ ROTATE(X, 15) ^ ROTATE(X, 23)) #define FF0(X,Y,Z) (X ^ Y ^ Z) #define GG0(X,Y,Z) (X ^ Y ^ Z) diff --git a/openssl/src/crypto/sm3/sm3_riscv.c b/openssl/src/crypto/sm3/sm3_riscv.c deleted file mode 100644 index 21ee3772b..000000000 --- a/openssl/src/crypto/sm3/sm3_riscv.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include - -#include -#include "internal/sm3.h" -#include "crypto/riscv_arch.h" -#include - -void ossl_hwsm3_block_data_order_zvksh(SM3_CTX *c, const void *p, size_t num); -void ossl_sm3_block_data_order(SM3_CTX *c, const void *p, size_t num); -void ossl_hwsm3_block_data_order(SM3_CTX *c, const void *p, size_t num); - -void ossl_hwsm3_block_data_order(SM3_CTX *c, const void *p, size_t num) -{ - if (RISCV_HAS_ZVKB_AND_ZVKSH() && riscv_vlen() >= 128) { - ossl_hwsm3_block_data_order_zvksh(c, p, num); - } else { - ossl_sm3_block_data_order(c, p, num); - } -} diff --git a/openssl/src/crypto/sm4/gen/darwin_arm64/sm4-armv8.S b/openssl/src/crypto/sm4/gen/darwin_arm64/sm4-armv8.S index 82fc40103..b8cadc3af 100644 --- a/openssl/src/crypto/sm4/gen/darwin_arm64/sm4-armv8.S +++ b/openssl/src/crypto/sm4/gen/darwin_arm64/sm4-armv8.S @@ -31,7 +31,6 @@ Lfk: .align 5 _sm4_v8_set_encrypt_key: - AARCH64_VALID_CALL_TARGET ld1 {v0.4s},[x0] adr x2,Lfk ld1 {v24.4s},[x2] @@ -58,7 +57,6 @@ _sm4_v8_set_encrypt_key: .align 5 _sm4_v8_set_decrypt_key: - AARCH64_VALID_CALL_TARGET ld1 {v7.4s},[x0] adr x2,Lfk ld1 {v24.4s},[x2] @@ -101,7 +99,6 @@ _sm4_v8_set_decrypt_key: .align 5 _sm4_v8_encrypt: - AARCH64_VALID_CALL_TARGET ld1 {v16.4s},[x0] ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x2],64 ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x2] @@ -128,7 +125,6 @@ _sm4_v8_encrypt: .align 5 _sm4_v8_decrypt: - AARCH64_VALID_CALL_TARGET ld1 {v16.4s},[x0] ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x2],64 ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x2] @@ -155,7 +151,6 @@ _sm4_v8_decrypt: .align 5 _sm4_v8_ecb_encrypt: - AARCH64_VALID_CALL_TARGET ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x3],#64 ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x3] 1: @@ -418,7 +413,6 @@ _sm4_v8_ecb_encrypt: .align 5 _sm4_v8_cbc_encrypt: - AARCH64_VALID_CALL_TARGET stp d8,d9,[sp, #-16]! ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x3],#64 @@ -809,7 +803,6 @@ Ldec: .align 5 _sm4_v8_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET stp d8,d9,[sp, #-16]! ld1 {v8.4s},[x4] diff --git a/openssl/src/crypto/sm4/gen/darwin_arm64/vpsm4-armv8.S b/openssl/src/crypto/sm4/gen/darwin_arm64/vpsm4-armv8.S deleted file mode 100644 index c25b3c1ec..000000000 --- a/openssl/src/crypto/sm4/gen/darwin_arm64/vpsm4-armv8.S +++ /dev/null @@ -1,4999 +0,0 @@ -// Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html - -// -// This module implements SM4 with ASIMD on aarch64 -// -// Feb 2022 -// - -// $output is the last argument if it looks like a file (it has an extension) -// $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" - -.text - - -.align 7 -_vpsm4_consts: -Lsbox: -.byte 0xD6,0x90,0xE9,0xFE,0xCC,0xE1,0x3D,0xB7,0x16,0xB6,0x14,0xC2,0x28,0xFB,0x2C,0x05 -.byte 0x2B,0x67,0x9A,0x76,0x2A,0xBE,0x04,0xC3,0xAA,0x44,0x13,0x26,0x49,0x86,0x06,0x99 -.byte 0x9C,0x42,0x50,0xF4,0x91,0xEF,0x98,0x7A,0x33,0x54,0x0B,0x43,0xED,0xCF,0xAC,0x62 -.byte 0xE4,0xB3,0x1C,0xA9,0xC9,0x08,0xE8,0x95,0x80,0xDF,0x94,0xFA,0x75,0x8F,0x3F,0xA6 -.byte 0x47,0x07,0xA7,0xFC,0xF3,0x73,0x17,0xBA,0x83,0x59,0x3C,0x19,0xE6,0x85,0x4F,0xA8 -.byte 0x68,0x6B,0x81,0xB2,0x71,0x64,0xDA,0x8B,0xF8,0xEB,0x0F,0x4B,0x70,0x56,0x9D,0x35 -.byte 0x1E,0x24,0x0E,0x5E,0x63,0x58,0xD1,0xA2,0x25,0x22,0x7C,0x3B,0x01,0x21,0x78,0x87 -.byte 0xD4,0x00,0x46,0x57,0x9F,0xD3,0x27,0x52,0x4C,0x36,0x02,0xE7,0xA0,0xC4,0xC8,0x9E -.byte 0xEA,0xBF,0x8A,0xD2,0x40,0xC7,0x38,0xB5,0xA3,0xF7,0xF2,0xCE,0xF9,0x61,0x15,0xA1 -.byte 0xE0,0xAE,0x5D,0xA4,0x9B,0x34,0x1A,0x55,0xAD,0x93,0x32,0x30,0xF5,0x8C,0xB1,0xE3 -.byte 0x1D,0xF6,0xE2,0x2E,0x82,0x66,0xCA,0x60,0xC0,0x29,0x23,0xAB,0x0D,0x53,0x4E,0x6F -.byte 0xD5,0xDB,0x37,0x45,0xDE,0xFD,0x8E,0x2F,0x03,0xFF,0x6A,0x72,0x6D,0x6C,0x5B,0x51 -.byte 0x8D,0x1B,0xAF,0x92,0xBB,0xDD,0xBC,0x7F,0x11,0xD9,0x5C,0x41,0x1F,0x10,0x5A,0xD8 -.byte 0x0A,0xC1,0x31,0x88,0xA5,0xCD,0x7B,0xBD,0x2D,0x74,0xD0,0x12,0xB8,0xE5,0xB4,0xB0 -.byte 0x89,0x69,0x97,0x4A,0x0C,0x96,0x77,0x7E,0x65,0xB9,0xF1,0x09,0xC5,0x6E,0xC6,0x84 -.byte 0x18,0xF0,0x7D,0xEC,0x3A,0xDC,0x4D,0x20,0x79,0xEE,0x5F,0x3E,0xD7,0xCB,0x39,0x48 -Lck: -.long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 -.long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 -.long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 -.long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 -.long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 -.long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 -.long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 -.long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 -Lfk: -.quad 0x56aa3350a3b1bac6,0xb27022dc677d9197 -Lshuffles: -.quad 0x0B0A090807060504,0x030201000F0E0D0C -Lxts_magic: -.quad 0x0101010101010187,0x0101010101010101 - - - -.align 4 -_vpsm4_set_key: - AARCH64_VALID_CALL_TARGET - ld1 {v5.4s},[x0] - adr x10,Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - adr x5,Lshuffles - ld1 {v7.2d},[x5] - adr x5,Lfk - ld1 {v6.2d},[x5] - eor v5.16b,v5.16b,v6.16b - mov x6,#32 - adr x5,Lck - movi v0.16b,#64 - cbnz w2,1f - add x1,x1,124 -1: - mov w7,v5.s[1] - ldr w8,[x5],#4 - eor w8,w8,w7 - mov w7,v5.s[2] - eor w8,w8,w7 - mov w7,v5.s[3] - eor w8,w8,w7 - // sbox lookup - mov v4.s[0],w8 - tbl v1.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v4.16b - sub v4.16b,v4.16b,v0.16b - tbx v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v4.16b - sub v4.16b,v4.16b,v0.16b - tbx v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v4.16b - sub v4.16b,v4.16b,v0.16b - tbx v1.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v4.16b - mov w7,v1.s[0] - eor w8,w7,w7,ror #19 - eor w8,w8,w7,ror #9 - mov w7,v5.s[0] - eor w8,w8,w7 - mov v5.s[0],w8 - cbz w2,2f - str w8,[x1],#4 - b 3f -2: - str w8,[x1],#-4 -3: - tbl v5.16b,{v5.16b},v7.16b - subs x6,x6,#1 - b.ne 1b - ret - - -.align 4 -_vpsm4_enc_4blks: - AARCH64_VALID_CALL_TARGET - mov x10,x3 - mov w11,#8 -10: - ldp w7,w8,[x10],8 - dup v12.4s,w7 - dup v13.4s,w8 - - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor v14.16b,v6.16b,v7.16b - eor v12.16b,v5.16b,v12.16b - eor v12.16b,v14.16b,v12.16b - movi v0.16b,#64 - movi v1.16b,#128 - movi v2.16b,#192 - sub v0.16b,v12.16b,v0.16b - sub v1.16b,v12.16b,v1.16b - sub v2.16b,v12.16b,v2.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v0.2d,v0.2d,v1.2d - add v2.2d,v2.2d,v12.2d - add v12.2d,v0.2d,v2.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v12.4s,32-10 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v12.4s,10 - eor v1.16b,v2.16b,v1.16b - ushr v0.4s,v12.4s,32-18 - sli v0.4s,v12.4s,18 - ushr v2.4s,v12.4s,32-24 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v12.4s,24 - eor v12.16b,v2.16b,v1.16b - eor v4.16b,v4.16b,v12.16b - - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor v14.16b,v14.16b,v4.16b - eor v13.16b,v14.16b,v13.16b - movi v0.16b,#64 - movi v1.16b,#128 - movi v2.16b,#192 - sub v0.16b,v13.16b,v0.16b - sub v1.16b,v13.16b,v1.16b - sub v2.16b,v13.16b,v2.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v0.2d,v0.2d,v1.2d - add v2.2d,v2.2d,v13.2d - add v13.2d,v0.2d,v2.2d - - ushr v0.4s,v13.4s,32-2 - sli v0.4s,v13.4s,2 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v13.16b - sli v2.4s,v13.4s,10 - eor v1.16b,v2.16b,v1.16b - ushr v0.4s,v13.4s,32-18 - sli v0.4s,v13.4s,18 - ushr v2.4s,v13.4s,32-24 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v1.16b - ldp w7,w8,[x10],8 - eor v5.16b,v5.16b,v13.16b - - dup v12.4s,w7 - dup v13.4s,w8 - - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor v14.16b,v4.16b,v5.16b - eor v12.16b,v7.16b,v12.16b - eor v12.16b,v14.16b,v12.16b - movi v0.16b,#64 - movi v1.16b,#128 - movi v2.16b,#192 - sub v0.16b,v12.16b,v0.16b - sub v1.16b,v12.16b,v1.16b - sub v2.16b,v12.16b,v2.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v0.2d,v0.2d,v1.2d - add v2.2d,v2.2d,v12.2d - add v12.2d,v0.2d,v2.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v12.4s,32-10 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v12.4s,10 - eor v1.16b,v2.16b,v1.16b - ushr v0.4s,v12.4s,32-18 - sli v0.4s,v12.4s,18 - ushr v2.4s,v12.4s,32-24 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v12.4s,24 - eor v12.16b,v2.16b,v1.16b - eor v6.16b,v6.16b,v12.16b - - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor v14.16b,v14.16b,v6.16b - eor v13.16b,v14.16b,v13.16b - movi v0.16b,#64 - movi v1.16b,#128 - movi v2.16b,#192 - sub v0.16b,v13.16b,v0.16b - sub v1.16b,v13.16b,v1.16b - sub v2.16b,v13.16b,v2.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v0.2d,v0.2d,v1.2d - add v2.2d,v2.2d,v13.2d - add v13.2d,v0.2d,v2.2d - - ushr v0.4s,v13.4s,32-2 - sli v0.4s,v13.4s,2 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v13.16b - sli v2.4s,v13.4s,10 - eor v1.16b,v2.16b,v1.16b - ushr v0.4s,v13.4s,32-18 - sli v0.4s,v13.4s,18 - ushr v2.4s,v13.4s,32-24 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v1.16b - eor v7.16b,v7.16b,v13.16b - subs w11,w11,#1 - b.ne 10b -#ifndef __AARCH64EB__ - rev32 v3.16b,v4.16b -#else - mov v3.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v2.16b,v5.16b -#else - mov v2.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v1.16b,v6.16b -#else - mov v1.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v0.16b,v7.16b -#else - mov v0.16b,v7.16b -#endif - ret - - -.align 4 -_vpsm4_enc_8blks: - AARCH64_VALID_CALL_TARGET - mov x10,x3 - mov w11,#8 -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - dup v12.4s,w7 - eor v14.16b,v6.16b,v7.16b - eor v15.16b,v10.16b,v11.16b - eor v0.16b,v5.16b,v12.16b - eor v1.16b,v9.16b,v12.16b - eor v12.16b,v14.16b,v0.16b - eor v13.16b,v15.16b,v1.16b - movi v3.16b,#64 - sub v0.16b,v12.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v12.2d,v2.2d,v12.2d - add v12.2d,v1.2d,v12.2d - - sub v0.16b,v13.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v13.2d,v2.2d,v13.2d - add v13.2d,v1.2d,v13.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v13.4s,32-2 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v13.4s,2 - - ushr v0.4s,v12.4s,32-10 - eor v3.16b,v2.16b,v13.16b - sli v0.4s,v12.4s,10 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,10 - - ushr v0.4s,v12.4s,32-18 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,18 - ushr v2.4s,v13.4s,32-18 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,18 - - ushr v0.4s,v12.4s,32-24 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,24 - ushr v2.4s,v13.4s,32-24 - eor v12.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v3.16b - eor v4.16b,v4.16b,v12.16b - eor v8.16b,v8.16b,v13.16b - - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - dup v13.4s,w8 - eor v14.16b,v14.16b,v4.16b - eor v15.16b,v15.16b,v8.16b - eor v12.16b,v14.16b,v13.16b - eor v13.16b,v15.16b,v13.16b - movi v3.16b,#64 - sub v0.16b,v12.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v12.2d,v2.2d,v12.2d - add v12.2d,v1.2d,v12.2d - - sub v0.16b,v13.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v13.2d,v2.2d,v13.2d - add v13.2d,v1.2d,v13.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v13.4s,32-2 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v13.4s,2 - - ushr v0.4s,v12.4s,32-10 - eor v3.16b,v2.16b,v13.16b - sli v0.4s,v12.4s,10 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,10 - - ushr v0.4s,v12.4s,32-18 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,18 - ushr v2.4s,v13.4s,32-18 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,18 - - ushr v0.4s,v12.4s,32-24 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,24 - ushr v2.4s,v13.4s,32-24 - eor v12.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v3.16b - ldp w7,w8,[x10],8 - eor v5.16b,v5.16b,v12.16b - eor v9.16b,v9.16b,v13.16b - - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - dup v12.4s,w7 - eor v14.16b,v4.16b,v5.16b - eor v15.16b,v8.16b,v9.16b - eor v0.16b,v7.16b,v12.16b - eor v1.16b,v11.16b,v12.16b - eor v12.16b,v14.16b,v0.16b - eor v13.16b,v15.16b,v1.16b - movi v3.16b,#64 - sub v0.16b,v12.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v12.2d,v2.2d,v12.2d - add v12.2d,v1.2d,v12.2d - - sub v0.16b,v13.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v13.2d,v2.2d,v13.2d - add v13.2d,v1.2d,v13.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v13.4s,32-2 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v13.4s,2 - - ushr v0.4s,v12.4s,32-10 - eor v3.16b,v2.16b,v13.16b - sli v0.4s,v12.4s,10 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,10 - - ushr v0.4s,v12.4s,32-18 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,18 - ushr v2.4s,v13.4s,32-18 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,18 - - ushr v0.4s,v12.4s,32-24 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,24 - ushr v2.4s,v13.4s,32-24 - eor v12.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v3.16b - eor v6.16b,v6.16b,v12.16b - eor v10.16b,v10.16b,v13.16b - - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - dup v13.4s,w8 - eor v14.16b,v14.16b,v6.16b - eor v15.16b,v15.16b,v10.16b - eor v12.16b,v14.16b,v13.16b - eor v13.16b,v15.16b,v13.16b - movi v3.16b,#64 - sub v0.16b,v12.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v12.2d,v2.2d,v12.2d - add v12.2d,v1.2d,v12.2d - - sub v0.16b,v13.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v13.2d,v2.2d,v13.2d - add v13.2d,v1.2d,v13.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v13.4s,32-2 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v13.4s,2 - - ushr v0.4s,v12.4s,32-10 - eor v3.16b,v2.16b,v13.16b - sli v0.4s,v12.4s,10 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,10 - - ushr v0.4s,v12.4s,32-18 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,18 - ushr v2.4s,v13.4s,32-18 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,18 - - ushr v0.4s,v12.4s,32-24 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,24 - ushr v2.4s,v13.4s,32-24 - eor v12.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v3.16b - eor v7.16b,v7.16b,v12.16b - eor v11.16b,v11.16b,v13.16b - subs w11,w11,#1 - b.ne 10b -#ifndef __AARCH64EB__ - rev32 v3.16b,v4.16b -#else - mov v3.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v2.16b,v5.16b -#else - mov v2.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v1.16b,v6.16b -#else - mov v1.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v0.16b,v7.16b -#else - mov v0.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v8.16b -#else - mov v7.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v9.16b -#else - mov v6.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v10.16b -#else - mov v5.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v4.16b,v11.16b -#else - mov v4.16b,v11.16b -#endif - ret - -.globl _vpsm4_set_encrypt_key - -.align 5 -_vpsm4_set_encrypt_key: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - mov w2,1 - bl _vpsm4_set_key - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_set_decrypt_key - -.align 5 -_vpsm4_set_decrypt_key: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - mov w2,0 - bl _vpsm4_set_key - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_encrypt - -.align 5 -_vpsm4_encrypt: - AARCH64_VALID_CALL_TARGET - ld1 {v4.4s},[x0] - adr x10,Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x3,x2 - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - ret - -.globl _vpsm4_decrypt - -.align 5 -_vpsm4_decrypt: - AARCH64_VALID_CALL_TARGET - ld1 {v4.4s},[x0] - adr x10,Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x3,x2 - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - ret - -.globl _vpsm4_ecb_encrypt - -.align 5 -_vpsm4_ecb_encrypt: - AARCH64_SIGN_LINK_REGISTER - // convert length into blocks - lsr x2,x2,4 - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] - adr x10,Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -Lecb_8_blocks_process: - cmp w2,#8 - b.lt Lecb_4_blocks_process - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - bl _vpsm4_enc_8blks - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st4 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.gt Lecb_8_blocks_process - b 100f -Lecb_4_blocks_process: - cmp w2,#4 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub w2,w2,#4 -1: - // process last block - cmp w2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - b 100f -1: // process last 2 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[0],[x0],#16 - ld4 {v4.s,v5.s,v6.s,v7.s}[1],[x0],#16 - cmp w2,#2 - b.gt 1f -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1] - b 100f -1: // process last 3 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[2],[x0],#16 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[2],[x1] -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_cbc_encrypt - -.align 5 -_vpsm4_cbc_encrypt: - AARCH64_VALID_CALL_TARGET - lsr x2,x2,4 - adr x10,Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] - cbz w5,Ldec - ld1 {v3.4s},[x4] -Lcbc_4_blocks_enc: - cmp w2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b,v4.16b,v3.16b -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 - eor v5.16b,v5.16b,v4.16b - mov x10,x3 - mov w11,#8 - mov w12,v5.s[0] - mov w13,v5.s[1] - mov w14,v5.s[2] - mov w15,v5.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v5.s[0],w15 - mov v5.s[1],w14 - mov v5.s[2],w13 - mov v5.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v6.16b,v6.16b,v5.16b - mov x10,x3 - mov w11,#8 - mov w12,v6.s[0] - mov w13,v6.s[1] - mov w14,v6.s[2] - mov w15,v6.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v6.s[0],w15 - mov v6.s[1],w14 - mov v6.s[2],w13 - mov v6.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - eor v7.16b,v7.16b,v6.16b - mov x10,x3 - mov w11,#8 - mov w12,v7.s[0] - mov w13,v7.s[1] - mov w14,v7.s[2] - mov w15,v7.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v7.s[0],w15 - mov v7.s[1],w14 - mov v7.s[2],w13 - mov v7.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - orr v3.16b,v7.16b,v7.16b - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#4 - b.ne Lcbc_4_blocks_enc - b 2f -1: - subs w2,w2,#1 - b.lt 2f - ld1 {v4.4s},[x0],#16 - eor v3.16b,v3.16b,v4.16b -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - st1 {v3.4s},[x1],#16 - b 1b -2: - // save back IV - st1 {v3.4s},[x4] - ret - -Ldec: - // decryption mode starts - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] -Lcbc_8_blocks_dec: - cmp w2,#8 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0] - add x10,x0,#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - bl _vpsm4_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - ld1 {v15.4s},[x4] - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - // note ivec1 and vtmpx[3] are reusing the same register - // care needs to be taken to avoid conflict - eor v0.16b,v0.16b,v15.16b - ld1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - eor v1.16b,v1.16b,v8.16b - eor v2.16b,v2.16b,v9.16b - eor v3.16b,v3.16b,v10.16b - // save back IV - st1 {v15.4s}, [x4] - eor v4.16b,v4.16b,v11.16b - eor v5.16b,v5.16b,v12.16b - eor v6.16b,v6.16b,v13.16b - eor v7.16b,v7.16b,v14.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.gt Lcbc_8_blocks_dec - b.eq 100f -1: - ld1 {v15.4s},[x4] -Lcbc_4_blocks_dec: - cmp w2,#4 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - orr v15.16b,v7.16b,v7.16b - eor v2.16b,v2.16b,v5.16b - eor v3.16b,v3.16b,v6.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - subs w2,w2,#4 - b.gt Lcbc_4_blocks_dec - // save back IV - st1 {v7.4s}, [x4] - b 100f -1: // last block - subs w2,w2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - // save back IV - st1 {v4.4s}, [x4] -#ifndef __AARCH64EB__ - rev32 v8.16b,v4.16b -#else - mov v8.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v8.s[0] - mov w13,v8.s[1] - mov w14,v8.s[2] - mov w15,v8.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v8.s[0],w15 - mov v8.s[1],w14 - mov v8.s[2],w13 - mov v8.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - eor v8.16b,v8.16b,v15.16b - st1 {v8.4s},[x1],#16 - b 100f -1: // last two blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[0],[x0] - add x10,x0,#16 - ld4 {v4.s,v5.s,v6.s,v7.s}[1],[x10],#16 - subs w2,w2,1 - b.gt 1f -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - ld1 {v4.4s,v5.4s},[x0],#32 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save back IV - st1 {v5.4s}, [x4] - b 100f -1: // last 3 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[2],[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - eor v2.16b,v2.16b,v5.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save back IV - st1 {v6.4s}, [x4] -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_ctr32_encrypt_blocks - -.align 5 -_vpsm4_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET - ld1 {v3.4s},[x4] -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - adr x10,Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] - cmp w2,#1 - b.ne 1f - // fast processing for one single block without - // context saving overhead - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ld1 {v4.4s},[x0] - eor v4.16b,v4.16b,v3.16b - st1 {v4.4s},[x1] - ret -1: - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w5,v3.s[3] -Lctr32_4_blocks_process: - cmp w2,#4 - b.lt 1f - dup v4.4s,w12 - dup v5.4s,w13 - dup v6.4s,w14 - mov v7.s[0],w5 - add w5,w5,#1 - mov v7.s[1],w5 - add w5,w5,#1 - mov v7.s[2],w5 - add w5,w5,#1 - mov v7.s[3],w5 - add w5,w5,#1 - cmp w2,#8 - b.ge Lctr32_8_blocks_process - bl _vpsm4_enc_4blks - ld4 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - subs w2,w2,#4 - b.ne Lctr32_4_blocks_process - b 100f -Lctr32_8_blocks_process: - dup v8.4s,w12 - dup v9.4s,w13 - dup v10.4s,w14 - mov v11.s[0],w5 - add w5,w5,#1 - mov v11.s[1],w5 - add w5,w5,#1 - mov v11.s[2],w5 - add w5,w5,#1 - mov v11.s[3],w5 - add w5,w5,#1 - bl _vpsm4_enc_8blks - ld4 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - eor v4.16b,v4.16b,v8.16b - eor v5.16b,v5.16b,v9.16b - eor v6.16b,v6.16b,v10.16b - eor v7.16b,v7.16b,v11.16b - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st4 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.ne Lctr32_4_blocks_process - b 100f -1: // last block processing - subs w2,w2,#1 - b.lt 100f - b.gt 1f - mov v3.s[0],w12 - mov v3.s[1],w13 - mov v3.s[2],w14 - mov v3.s[3],w5 - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ld1 {v4.4s},[x0] - eor v4.16b,v4.16b,v3.16b - st1 {v4.4s},[x1] - b 100f -1: // last 2 blocks processing - dup v4.4s,w12 - dup v5.4s,w13 - dup v6.4s,w14 - mov v7.s[0],w5 - add w5,w5,#1 - mov v7.s[1],w5 - subs w2,w2,#1 - b.ne 1f - bl _vpsm4_enc_4blks - ld4 {v12.s,v13.s,v14.s,v15.s}[0],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[1],[x0],#16 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - b 100f -1: // last 3 blocks processing - add w5,w5,#1 - mov v7.s[2],w5 - bl _vpsm4_enc_4blks - ld4 {v12.s,v13.s,v14.s,v15.s}[0],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[1],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[2],[x0],#16 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[2],[x1],#16 -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_xts_encrypt_gb - -.align 5 -_vpsm4_xts_encrypt_gb: - AARCH64_SIGN_LINK_REGISTER - stp x15, x16, [sp, #-0x10]! - stp x17, x18, [sp, #-0x10]! - stp x19, x20, [sp, #-0x10]! - stp x21, x22, [sp, #-0x10]! - stp x23, x24, [sp, #-0x10]! - stp x25, x26, [sp, #-0x10]! - stp x27, x28, [sp, #-0x10]! - stp x29, x30, [sp, #-0x10]! - stp d8, d9, [sp, #-0x10]! - stp d10, d11, [sp, #-0x10]! - stp d12, d13, [sp, #-0x10]! - stp d14, d15, [sp, #-0x10]! - mov x26,x3 - mov x27,x4 - mov w28,w6 - ld1 {v8.4s}, [x5] - mov x3,x27 - adr x10,Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v8.s[0] - mov w13,v8.s[1] - mov w14,v8.s[2] - mov w15,v8.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v8.s[0],w15 - mov v8.s[1],w14 - mov v8.s[2],w13 - mov v8.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x3,x26 - and x29,x2,#0x0F - // convert length into blocks - lsr x2,x2,4 - cmp x2,#1 - b.lt .return_gb - - cmp x29,0 - // If the encryption/decryption Length is N times of 16, - // the all blocks are encrypted/decrypted in .xts_encrypt_blocks_gb - b.eq .xts_encrypt_blocks_gb - - // If the encryption/decryption length is not N times of 16, - // the last two blocks are encrypted/decrypted in .last_2blks_tweak_gb or .only_2blks_tweak_gb - // the other blocks are encrypted/decrypted in .xts_encrypt_blocks_gb - subs x2,x2,#1 - b.eq .only_2blks_tweak_gb -.xts_encrypt_blocks_gb: - rbit v8.16b,v8.16b -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x12,v8.d[0] - mov x13,v8.d[1] - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 -Lxts_8_blocks_process_gb: - cmp x2,#8 - b.lt Lxts_4_blocks_process_gb - mov v0.d[0],x12 - mov v0.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v0.16b,v0.16b -#endif - mov v1.d[0],x14 - mov v1.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v1.16b,v1.16b -#endif - mov v2.d[0],x16 - mov v2.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v2.16b,v2.16b -#endif - mov v3.d[0],x18 - mov v3.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - mov v12.d[0],x20 - mov v12.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v12.16b,v12.16b -#endif - mov v13.d[0],x22 - mov v13.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v13.16b,v13.16b -#endif - mov v14.d[0],x24 - mov v14.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v14.16b,v14.16b -#endif - mov v15.d[0],x26 - mov v15.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v15.16b,v15.16b -#endif - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - rbit v0.16b,v0.16b - rbit v1.16b,v1.16b - rbit v2.16b,v2.16b - rbit v3.16b,v3.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - rbit v12.16b,v12.16b - rbit v13.16b,v13.16b - rbit v14.16b,v14.16b - rbit v15.16b,v15.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - zip1 v0.4s,v8.4s,v9.4s - zip2 v1.4s,v8.4s,v9.4s - zip1 v2.4s,v10.4s,v11.4s - zip2 v3.4s,v10.4s,v11.4s - zip1 v8.2d,v0.2d,v2.2d - zip2 v9.2d,v0.2d,v2.2d - zip1 v10.2d,v1.2d,v3.2d - zip2 v11.2d,v1.2d,v3.2d - bl _vpsm4_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - mov v12.d[0],x12 - mov v12.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v12.16b,v12.16b -#endif - mov w7,0x87 - extr x9,x27,x27,#32 - extr x13,x27,x26,#63 - and w8,w7,w9,asr#31 - eor x12,x8,x26,lsl#1 - mov v13.d[0],x14 - mov v13.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v13.16b,v13.16b -#endif - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov v14.d[0],x16 - mov v14.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v14.16b,v14.16b -#endif - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov v15.d[0],x18 - mov v15.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v15.16b,v15.16b -#endif - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov v8.d[0],x20 - mov v8.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov v9.d[0],x22 - mov v9.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov v10.d[0],x24 - mov v10.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov v11.d[0],x26 - mov v11.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - - // save the last tweak - st1 {v11.4s},[x5] - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs x2,x2,#8 - b.gt Lxts_8_blocks_process_gb - b 100f -Lxts_4_blocks_process_gb: - mov v8.d[0],x12 - mov v8.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v9.d[0],x14 - mov v9.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v10.d[0],x16 - mov v10.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - mov v11.d[0],x18 - mov v11.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - cmp x2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - rbit v8.16b,v8.16b - rbit v9.16b,v9.16b - rbit v10.16b,v10.16b - rbit v11.16b,v11.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub x2,x2,#4 - mov v8.d[0],x20 - mov v8.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v9.d[0],x22 - mov v9.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v10.d[0],x24 - mov v10.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - // save the last tweak - st1 {v11.4s},[x5] -1: - // process last block - cmp x2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - rbit v8.16b,v8.16b - eor v4.16b, v4.16b, v8.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v8.16b - st1 {v4.4s},[x1],#16 - // save the last tweak - st1 {v8.4s},[x5] - b 100f -1: // process last 2 blocks - cmp x2,#2 - b.gt 1f - ld1 {v4.4s,v5.4s},[x0],#32 - rbit v8.16b,v8.16b - rbit v9.16b,v9.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save the last tweak - st1 {v9.4s},[x5] - b 100f -1: // process last 3 blocks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - rbit v8.16b,v8.16b - rbit v9.16b,v9.16b - rbit v10.16b,v10.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save the last tweak - st1 {v10.4s},[x5] -100: - cmp x29,0 - b.eq .return_gb - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is larger than 32 -.last_2blks_tweak_gb: - ld1 {v8.4s},[x5] -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - rbit v2.16b,v8.16b - ldr q0, Lxts_magic - shl v9.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - rbit v9.16b,v9.16b - rbit v2.16b,v9.16b - ldr q0, Lxts_magic - shl v10.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v10.16b, v10.16b, v1.16b - rbit v10.16b,v10.16b - b .check_dec_gb - - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is equal to 32, who only need two tweaks -.only_2blks_tweak_gb: - mov v9.16b,v8.16b -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - rbit v2.16b,v9.16b - ldr q0, Lxts_magic - shl v10.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v10.16b, v10.16b, v1.16b - rbit v10.16b,v10.16b - b .check_dec_gb - - -// Determine whether encryption or decryption is required. -// The last two tweaks need to be swapped for decryption. -.check_dec_gb: - // encryption:1 decryption:0 - cmp w28,1 - b.eq .process_last_2blks_gb - mov v0.16B,v9.16b - mov v9.16B,v10.16b - mov v10.16B,v0.16b - -.process_last_2blks_gb: -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v9.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v9.16b - st1 {v4.4s},[x1],#16 - - sub x26,x1,16 -.loop_gb: - subs x29,x29,1 - ldrb w7,[x26,x29] - ldrb w8,[x0,x29] - strb w8,[x26,x29] - strb w7,[x1,x29] - b.gt .loop_gb - ld1 {v4.4s}, [x26] - eor v4.16b, v4.16b, v10.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v10.16b - st1 {v4.4s}, [x26] -.return_gb: - ldp d14, d15, [sp], #0x10 - ldp d12, d13, [sp], #0x10 - ldp d10, d11, [sp], #0x10 - ldp d8, d9, [sp], #0x10 - ldp x29, x30, [sp], #0x10 - ldp x27, x28, [sp], #0x10 - ldp x25, x26, [sp], #0x10 - ldp x23, x24, [sp], #0x10 - ldp x21, x22, [sp], #0x10 - ldp x19, x20, [sp], #0x10 - ldp x17, x18, [sp], #0x10 - ldp x15, x16, [sp], #0x10 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_xts_encrypt - -.align 5 -_vpsm4_xts_encrypt: - AARCH64_SIGN_LINK_REGISTER - stp x15, x16, [sp, #-0x10]! - stp x17, x18, [sp, #-0x10]! - stp x19, x20, [sp, #-0x10]! - stp x21, x22, [sp, #-0x10]! - stp x23, x24, [sp, #-0x10]! - stp x25, x26, [sp, #-0x10]! - stp x27, x28, [sp, #-0x10]! - stp x29, x30, [sp, #-0x10]! - stp d8, d9, [sp, #-0x10]! - stp d10, d11, [sp, #-0x10]! - stp d12, d13, [sp, #-0x10]! - stp d14, d15, [sp, #-0x10]! - mov x26,x3 - mov x27,x4 - mov w28,w6 - ld1 {v8.4s}, [x5] - mov x3,x27 - adr x10,Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v8.s[0] - mov w13,v8.s[1] - mov w14,v8.s[2] - mov w15,v8.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v8.s[0],w15 - mov v8.s[1],w14 - mov v8.s[2],w13 - mov v8.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x3,x26 - and x29,x2,#0x0F - // convert length into blocks - lsr x2,x2,4 - cmp x2,#1 - b.lt .return - - cmp x29,0 - // If the encryption/decryption Length is N times of 16, - // the all blocks are encrypted/decrypted in .xts_encrypt_blocks - b.eq .xts_encrypt_blocks - - // If the encryption/decryption length is not N times of 16, - // the last two blocks are encrypted/decrypted in .last_2blks_tweak or .only_2blks_tweak - // the other blocks are encrypted/decrypted in .xts_encrypt_blocks - subs x2,x2,#1 - b.eq .only_2blks_tweak -.xts_encrypt_blocks: -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x12,v8.d[0] - mov x13,v8.d[1] - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 -Lxts_8_blocks_process: - cmp x2,#8 - b.lt Lxts_4_blocks_process - mov v0.d[0],x12 - mov v0.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v0.16b,v0.16b -#endif - mov v1.d[0],x14 - mov v1.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v1.16b,v1.16b -#endif - mov v2.d[0],x16 - mov v2.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v2.16b,v2.16b -#endif - mov v3.d[0],x18 - mov v3.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - mov v12.d[0],x20 - mov v12.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v12.16b,v12.16b -#endif - mov v13.d[0],x22 - mov v13.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v13.16b,v13.16b -#endif - mov v14.d[0],x24 - mov v14.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v14.16b,v14.16b -#endif - mov v15.d[0],x26 - mov v15.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v15.16b,v15.16b -#endif - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - zip1 v0.4s,v8.4s,v9.4s - zip2 v1.4s,v8.4s,v9.4s - zip1 v2.4s,v10.4s,v11.4s - zip2 v3.4s,v10.4s,v11.4s - zip1 v8.2d,v0.2d,v2.2d - zip2 v9.2d,v0.2d,v2.2d - zip1 v10.2d,v1.2d,v3.2d - zip2 v11.2d,v1.2d,v3.2d - bl _vpsm4_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - mov v12.d[0],x12 - mov v12.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v12.16b,v12.16b -#endif - mov w7,0x87 - extr x9,x27,x27,#32 - extr x13,x27,x26,#63 - and w8,w7,w9,asr#31 - eor x12,x8,x26,lsl#1 - mov v13.d[0],x14 - mov v13.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v13.16b,v13.16b -#endif - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov v14.d[0],x16 - mov v14.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v14.16b,v14.16b -#endif - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov v15.d[0],x18 - mov v15.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v15.16b,v15.16b -#endif - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov v8.d[0],x20 - mov v8.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov v9.d[0],x22 - mov v9.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov v10.d[0],x24 - mov v10.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov v11.d[0],x26 - mov v11.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - - // save the last tweak - st1 {v11.4s},[x5] - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs x2,x2,#8 - b.gt Lxts_8_blocks_process - b 100f -Lxts_4_blocks_process: - mov v8.d[0],x12 - mov v8.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v9.d[0],x14 - mov v9.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v10.d[0],x16 - mov v10.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - mov v11.d[0],x18 - mov v11.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - cmp x2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub x2,x2,#4 - mov v8.d[0],x20 - mov v8.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v9.d[0],x22 - mov v9.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v10.d[0],x24 - mov v10.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - // save the last tweak - st1 {v11.4s},[x5] -1: - // process last block - cmp x2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v8.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v8.16b - st1 {v4.4s},[x1],#16 - // save the last tweak - st1 {v8.4s},[x5] - b 100f -1: // process last 2 blocks - cmp x2,#2 - b.gt 1f - ld1 {v4.4s,v5.4s},[x0],#32 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save the last tweak - st1 {v9.4s},[x5] - b 100f -1: // process last 3 blocks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save the last tweak - st1 {v10.4s},[x5] -100: - cmp x29,0 - b.eq .return - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is larger than 32 -.last_2blks_tweak: - ld1 {v8.4s},[x5] -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v2.16b,v8.16b - ldr q0, Lxts_magic - shl v9.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - mov v2.16b,v9.16b - ldr q0, Lxts_magic - shl v10.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v10.16b, v10.16b, v1.16b - b .check_dec - - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is equal to 32, who only need two tweaks -.only_2blks_tweak: - mov v9.16b,v8.16b -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v2.16b,v9.16b - ldr q0, Lxts_magic - shl v10.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v10.16b, v10.16b, v1.16b - b .check_dec - - -// Determine whether encryption or decryption is required. -// The last two tweaks need to be swapped for decryption. -.check_dec: - // encryption:1 decryption:0 - cmp w28,1 - b.eq .process_last_2blks - mov v0.16B,v9.16b - mov v9.16B,v10.16b - mov v10.16B,v0.16b - -.process_last_2blks: -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v9.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v9.16b - st1 {v4.4s},[x1],#16 - - sub x26,x1,16 -.loop: - subs x29,x29,1 - ldrb w7,[x26,x29] - ldrb w8,[x0,x29] - strb w8,[x26,x29] - strb w7,[x1,x29] - b.gt .loop - ld1 {v4.4s}, [x26] - eor v4.16b, v4.16b, v10.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v10.16b - st1 {v4.4s}, [x26] -.return: - ldp d14, d15, [sp], #0x10 - ldp d12, d13, [sp], #0x10 - ldp d10, d11, [sp], #0x10 - ldp d8, d9, [sp], #0x10 - ldp x29, x30, [sp], #0x10 - ldp x27, x28, [sp], #0x10 - ldp x25, x26, [sp], #0x10 - ldp x23, x24, [sp], #0x10 - ldp x21, x22, [sp], #0x10 - ldp x19, x20, [sp], #0x10 - ldp x17, x18, [sp], #0x10 - ldp x15, x16, [sp], #0x10 - AARCH64_VALIDATE_LINK_REGISTER - ret - diff --git a/openssl/src/crypto/sm4/gen/darwin_arm64/vpsm4_ex-armv8.S b/openssl/src/crypto/sm4/gen/darwin_arm64/vpsm4_ex-armv8.S deleted file mode 100644 index 423ed0521..000000000 --- a/openssl/src/crypto/sm4/gen/darwin_arm64/vpsm4_ex-armv8.S +++ /dev/null @@ -1,4505 +0,0 @@ -// Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html - -// -// This module implements SM4 with ASIMD and AESE on AARCH64 -// -// Dec 2022 -// - -// $output is the last argument if it looks like a file (it has an extension) -// $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" - -.text - - -.align 7 -_vpsm4_ex_consts: -Lck: -.long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 -.long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 -.long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 -.long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 -.long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 -.long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 -.long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 -.long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 -Lfk: -.quad 0x56aa3350a3b1bac6,0xb27022dc677d9197 -Lshuffles: -.quad 0x0B0A090807060504,0x030201000F0E0D0C -Lxts_magic: -.quad 0x0101010101010187,0x0101010101010101 -Lsbox_magic: -.quad 0x0b0e0104070a0d00,0x0306090c0f020508 -.quad 0x62185a2042387a00,0x22581a6002783a40 -.quad 0x15df62a89e54e923,0xc10bb67c4a803df7 -.quad 0xb9aa6b78c1d21300,0x1407c6d56c7fbead -.quad 0x6404462679195b3b,0xe383c1a1fe9edcbc -.quad 0x0f0f0f0f0f0f0f0f,0x0f0f0f0f0f0f0f0f - - - -.align 4 -_vpsm4_ex_set_key: - AARCH64_VALID_CALL_TARGET - ld1 {v5.4s},[x0] - ldr q26, Lsbox_magic - ldr q27, Lsbox_magic+16 - ldr q28, Lsbox_magic+32 - ldr q29, Lsbox_magic+48 - ldr q30, Lsbox_magic+64 - ldr q31, Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - adr x5,Lshuffles - ld1 {v7.2d},[x5] - adr x5,Lfk - ld1 {v6.2d},[x5] - eor v5.16b,v5.16b,v6.16b - mov x6,#32 - adr x5,Lck - movi v0.16b,#64 - cbnz w2,1f - add x1,x1,124 -1: - mov w7,v5.s[1] - ldr w8,[x5],#4 - eor w8,w8,w7 - mov w7,v5.s[2] - eor w8,w8,w7 - mov w7,v5.s[3] - eor w8,w8,w7 - // optimize sbox using AESE instruction - mov v4.s[0],w8 - tbl v0.16b, {v4.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - mov w7,v0.s[0] - eor w8,w7,w7,ror #19 - eor w8,w8,w7,ror #9 - mov w7,v5.s[0] - eor w8,w8,w7 - mov v5.s[0],w8 - cbz w2,2f - str w8,[x1],#4 - b 3f -2: - str w8,[x1],#-4 -3: - tbl v5.16b,{v5.16b},v7.16b - subs x6,x6,#1 - b.ne 1b - ret - - -.align 4 -_vpsm4_ex_enc_4blks: - AARCH64_VALID_CALL_TARGET - mov x10,x3 - mov w11,#8 -10: - ldp w7,w8,[x10],8 - dup v12.4s,w7 - dup v13.4s,w8 - - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor v14.16b,v6.16b,v7.16b - eor v12.16b,v5.16b,v12.16b - eor v12.16b,v14.16b,v12.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - mov v12.16b,v0.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - eor v4.16b,v4.16b,v12.16b - - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor v14.16b,v14.16b,v4.16b - eor v13.16b,v14.16b,v13.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - mov v13.16b,v0.16b - - // linear transformation - ushr v0.4s,v13.4s,32-2 - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v0.4s,v13.4s,2 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v0.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - ldp w7,w8,[x10],8 - eor v5.16b,v5.16b,v13.16b - - dup v12.4s,w7 - dup v13.4s,w8 - - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor v14.16b,v4.16b,v5.16b - eor v12.16b,v7.16b,v12.16b - eor v12.16b,v14.16b,v12.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - mov v12.16b,v0.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - eor v6.16b,v6.16b,v12.16b - - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor v14.16b,v14.16b,v6.16b - eor v13.16b,v14.16b,v13.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - mov v13.16b,v0.16b - - // linear transformation - ushr v0.4s,v13.4s,32-2 - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v0.4s,v13.4s,2 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v0.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - eor v7.16b,v7.16b,v13.16b - subs w11,w11,#1 - b.ne 10b -#ifndef __AARCH64EB__ - rev32 v3.16b,v4.16b -#else - mov v3.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v2.16b,v5.16b -#else - mov v2.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v1.16b,v6.16b -#else - mov v1.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v0.16b,v7.16b -#else - mov v0.16b,v7.16b -#endif - ret - - -.align 4 -_vpsm4_ex_enc_8blks: - AARCH64_VALID_CALL_TARGET - mov x10,x3 - mov w11,#8 -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - dup v12.4s,w7 - eor v14.16b,v6.16b,v7.16b - eor v15.16b,v10.16b,v11.16b - eor v0.16b,v5.16b,v12.16b - eor v1.16b,v9.16b,v12.16b - eor v12.16b,v14.16b,v0.16b - eor v13.16b,v15.16b,v1.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - tbl v1.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v28.16b}, v1.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - aese v0.16b,v25.16b - aese v1.16b,v25.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v30.16b}, v1.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - mov v12.16b,v0.16b - mov v13.16b,v1.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v25.4s,v13.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v25.4s,v13.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v25.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - eor v4.16b,v4.16b,v12.16b - eor v8.16b,v8.16b,v13.16b - - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - dup v13.4s,w8 - eor v14.16b,v14.16b,v4.16b - eor v15.16b,v15.16b,v8.16b - eor v12.16b,v14.16b,v13.16b - eor v13.16b,v15.16b,v13.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - tbl v1.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v28.16b}, v1.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - aese v0.16b,v25.16b - aese v1.16b,v25.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v30.16b}, v1.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - mov v12.16b,v0.16b - mov v13.16b,v1.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v25.4s,v13.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v25.4s,v13.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v25.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - ldp w7,w8,[x10],8 - eor v5.16b,v5.16b,v12.16b - eor v9.16b,v9.16b,v13.16b - - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - dup v12.4s,w7 - eor v14.16b,v4.16b,v5.16b - eor v15.16b,v8.16b,v9.16b - eor v0.16b,v7.16b,v12.16b - eor v1.16b,v11.16b,v12.16b - eor v12.16b,v14.16b,v0.16b - eor v13.16b,v15.16b,v1.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - tbl v1.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v28.16b}, v1.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - aese v0.16b,v25.16b - aese v1.16b,v25.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v30.16b}, v1.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - mov v12.16b,v0.16b - mov v13.16b,v1.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v25.4s,v13.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v25.4s,v13.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v25.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - eor v6.16b,v6.16b,v12.16b - eor v10.16b,v10.16b,v13.16b - - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - dup v13.4s,w8 - eor v14.16b,v14.16b,v6.16b - eor v15.16b,v15.16b,v10.16b - eor v12.16b,v14.16b,v13.16b - eor v13.16b,v15.16b,v13.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - tbl v1.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v28.16b}, v1.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - aese v0.16b,v25.16b - aese v1.16b,v25.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v30.16b}, v1.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - mov v12.16b,v0.16b - mov v13.16b,v1.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v25.4s,v13.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v25.4s,v13.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v25.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - eor v7.16b,v7.16b,v12.16b - eor v11.16b,v11.16b,v13.16b - subs w11,w11,#1 - b.ne 10b -#ifndef __AARCH64EB__ - rev32 v3.16b,v4.16b -#else - mov v3.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v2.16b,v5.16b -#else - mov v2.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v1.16b,v6.16b -#else - mov v1.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v0.16b,v7.16b -#else - mov v0.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v8.16b -#else - mov v7.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v9.16b -#else - mov v6.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v10.16b -#else - mov v5.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v4.16b,v11.16b -#else - mov v4.16b,v11.16b -#endif - ret - -.globl _vpsm4_ex_set_encrypt_key - -.align 5 -_vpsm4_ex_set_encrypt_key: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - mov w2,1 - bl _vpsm4_ex_set_key - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_ex_set_decrypt_key - -.align 5 -_vpsm4_ex_set_decrypt_key: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - mov w2,0 - bl _vpsm4_ex_set_key - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_ex_encrypt - -.align 5 -_vpsm4_ex_encrypt: - AARCH64_VALID_CALL_TARGET - ld1 {v4.4s},[x0] - ldr q26, Lsbox_magic - ldr q27, Lsbox_magic+16 - ldr q28, Lsbox_magic+32 - ldr q29, Lsbox_magic+48 - ldr q30, Lsbox_magic+64 - ldr q31, Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x3,x2 - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - ret - -.globl _vpsm4_ex_decrypt - -.align 5 -_vpsm4_ex_decrypt: - AARCH64_VALID_CALL_TARGET - ld1 {v4.4s},[x0] - ldr q26, Lsbox_magic - ldr q27, Lsbox_magic+16 - ldr q28, Lsbox_magic+32 - ldr q29, Lsbox_magic+48 - ldr q30, Lsbox_magic+64 - ldr q31, Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x3,x2 - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - ret - -.globl _vpsm4_ex_ecb_encrypt - -.align 5 -_vpsm4_ex_ecb_encrypt: - AARCH64_SIGN_LINK_REGISTER - // convert length into blocks - lsr x2,x2,4 - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] - ldr q26, Lsbox_magic - ldr q27, Lsbox_magic+16 - ldr q28, Lsbox_magic+32 - ldr q29, Lsbox_magic+48 - ldr q30, Lsbox_magic+64 - ldr q31, Lsbox_magic+80 -Lecb_8_blocks_process: - cmp w2,#8 - b.lt Lecb_4_blocks_process - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - bl _vpsm4_ex_enc_8blks - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st4 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.gt Lecb_8_blocks_process - b 100f -Lecb_4_blocks_process: - cmp w2,#4 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub w2,w2,#4 -1: - // process last block - cmp w2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - b 100f -1: // process last 2 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[0],[x0],#16 - ld4 {v4.s,v5.s,v6.s,v7.s}[1],[x0],#16 - cmp w2,#2 - b.gt 1f -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1] - b 100f -1: // process last 3 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[2],[x0],#16 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[2],[x1] -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_ex_cbc_encrypt - -.align 5 -_vpsm4_ex_cbc_encrypt: - AARCH64_VALID_CALL_TARGET - lsr x2,x2,4 - ldr q26, Lsbox_magic - ldr q27, Lsbox_magic+16 - ldr q28, Lsbox_magic+32 - ldr q29, Lsbox_magic+48 - ldr q30, Lsbox_magic+64 - ldr q31, Lsbox_magic+80 - cbz w5,Ldec - ld1 {v3.4s},[x4] -Lcbc_4_blocks_enc: - cmp w2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b,v4.16b,v3.16b -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 - eor v5.16b,v5.16b,v4.16b - mov x10,x3 - mov w11,#8 - mov w12,v5.s[0] - mov w13,v5.s[1] - mov w14,v5.s[2] - mov w15,v5.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v5.s[0],w15 - mov v5.s[1],w14 - mov v5.s[2],w13 - mov v5.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v6.16b,v6.16b,v5.16b - mov x10,x3 - mov w11,#8 - mov w12,v6.s[0] - mov w13,v6.s[1] - mov w14,v6.s[2] - mov w15,v6.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v6.s[0],w15 - mov v6.s[1],w14 - mov v6.s[2],w13 - mov v6.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - eor v7.16b,v7.16b,v6.16b - mov x10,x3 - mov w11,#8 - mov w12,v7.s[0] - mov w13,v7.s[1] - mov w14,v7.s[2] - mov w15,v7.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v7.s[0],w15 - mov v7.s[1],w14 - mov v7.s[2],w13 - mov v7.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - orr v3.16b,v7.16b,v7.16b - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#4 - b.ne Lcbc_4_blocks_enc - b 2f -1: - subs w2,w2,#1 - b.lt 2f - ld1 {v4.4s},[x0],#16 - eor v3.16b,v3.16b,v4.16b -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - st1 {v3.4s},[x1],#16 - b 1b -2: - // save back IV - st1 {v3.4s},[x4] - ret - -Ldec: - // decryption mode starts - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] -Lcbc_8_blocks_dec: - cmp w2,#8 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0] - add x10,x0,#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - bl _vpsm4_ex_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - ld1 {v15.4s},[x4] - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - // note ivec1 and vtmpx[3] are reusing the same register - // care needs to be taken to avoid conflict - eor v0.16b,v0.16b,v15.16b - ld1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - eor v1.16b,v1.16b,v8.16b - eor v2.16b,v2.16b,v9.16b - eor v3.16b,v3.16b,v10.16b - // save back IV - st1 {v15.4s}, [x4] - eor v4.16b,v4.16b,v11.16b - eor v5.16b,v5.16b,v12.16b - eor v6.16b,v6.16b,v13.16b - eor v7.16b,v7.16b,v14.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.gt Lcbc_8_blocks_dec - b.eq 100f -1: - ld1 {v15.4s},[x4] -Lcbc_4_blocks_dec: - cmp w2,#4 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - orr v15.16b,v7.16b,v7.16b - eor v2.16b,v2.16b,v5.16b - eor v3.16b,v3.16b,v6.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - subs w2,w2,#4 - b.gt Lcbc_4_blocks_dec - // save back IV - st1 {v7.4s}, [x4] - b 100f -1: // last block - subs w2,w2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - // save back IV - st1 {v4.4s}, [x4] -#ifndef __AARCH64EB__ - rev32 v8.16b,v4.16b -#else - mov v8.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v8.s[0] - mov w13,v8.s[1] - mov w14,v8.s[2] - mov w15,v8.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v8.s[0],w15 - mov v8.s[1],w14 - mov v8.s[2],w13 - mov v8.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - eor v8.16b,v8.16b,v15.16b - st1 {v8.4s},[x1],#16 - b 100f -1: // last two blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[0],[x0] - add x10,x0,#16 - ld4 {v4.s,v5.s,v6.s,v7.s}[1],[x10],#16 - subs w2,w2,1 - b.gt 1f -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - ld1 {v4.4s,v5.4s},[x0],#32 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save back IV - st1 {v5.4s}, [x4] - b 100f -1: // last 3 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[2],[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - eor v2.16b,v2.16b,v5.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save back IV - st1 {v6.4s}, [x4] -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_ex_ctr32_encrypt_blocks - -.align 5 -_vpsm4_ex_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET - ld1 {v3.4s},[x4] -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ldr q26, Lsbox_magic - ldr q27, Lsbox_magic+16 - ldr q28, Lsbox_magic+32 - ldr q29, Lsbox_magic+48 - ldr q30, Lsbox_magic+64 - ldr q31, Lsbox_magic+80 - cmp w2,#1 - b.ne 1f - // fast processing for one single block without - // context saving overhead - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ld1 {v4.4s},[x0] - eor v4.16b,v4.16b,v3.16b - st1 {v4.4s},[x1] - ret -1: - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w5,v3.s[3] -Lctr32_4_blocks_process: - cmp w2,#4 - b.lt 1f - dup v4.4s,w12 - dup v5.4s,w13 - dup v6.4s,w14 - mov v7.s[0],w5 - add w5,w5,#1 - mov v7.s[1],w5 - add w5,w5,#1 - mov v7.s[2],w5 - add w5,w5,#1 - mov v7.s[3],w5 - add w5,w5,#1 - cmp w2,#8 - b.ge Lctr32_8_blocks_process - bl _vpsm4_ex_enc_4blks - ld4 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - subs w2,w2,#4 - b.ne Lctr32_4_blocks_process - b 100f -Lctr32_8_blocks_process: - dup v8.4s,w12 - dup v9.4s,w13 - dup v10.4s,w14 - mov v11.s[0],w5 - add w5,w5,#1 - mov v11.s[1],w5 - add w5,w5,#1 - mov v11.s[2],w5 - add w5,w5,#1 - mov v11.s[3],w5 - add w5,w5,#1 - bl _vpsm4_ex_enc_8blks - ld4 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - eor v4.16b,v4.16b,v8.16b - eor v5.16b,v5.16b,v9.16b - eor v6.16b,v6.16b,v10.16b - eor v7.16b,v7.16b,v11.16b - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st4 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.ne Lctr32_4_blocks_process - b 100f -1: // last block processing - subs w2,w2,#1 - b.lt 100f - b.gt 1f - mov v3.s[0],w12 - mov v3.s[1],w13 - mov v3.s[2],w14 - mov v3.s[3],w5 - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ld1 {v4.4s},[x0] - eor v4.16b,v4.16b,v3.16b - st1 {v4.4s},[x1] - b 100f -1: // last 2 blocks processing - dup v4.4s,w12 - dup v5.4s,w13 - dup v6.4s,w14 - mov v7.s[0],w5 - add w5,w5,#1 - mov v7.s[1],w5 - subs w2,w2,#1 - b.ne 1f - bl _vpsm4_ex_enc_4blks - ld4 {v12.s,v13.s,v14.s,v15.s}[0],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[1],[x0],#16 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - b 100f -1: // last 3 blocks processing - add w5,w5,#1 - mov v7.s[2],w5 - bl _vpsm4_ex_enc_4blks - ld4 {v12.s,v13.s,v14.s,v15.s}[0],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[1],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[2],[x0],#16 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[2],[x1],#16 -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_ex_xts_encrypt_gb - -.align 5 -_vpsm4_ex_xts_encrypt_gb: - AARCH64_SIGN_LINK_REGISTER - stp x15, x16, [sp, #-0x10]! - stp x17, x18, [sp, #-0x10]! - stp x19, x20, [sp, #-0x10]! - stp x21, x22, [sp, #-0x10]! - stp x23, x24, [sp, #-0x10]! - stp x25, x26, [sp, #-0x10]! - stp x27, x28, [sp, #-0x10]! - stp x29, x30, [sp, #-0x10]! - stp d8, d9, [sp, #-0x10]! - stp d10, d11, [sp, #-0x10]! - stp d12, d13, [sp, #-0x10]! - stp d14, d15, [sp, #-0x10]! - mov x26,x3 - mov x27,x4 - mov w28,w6 - ld1 {v16.4s}, [x5] - mov x3,x27 - ldr q26, Lsbox_magic - ldr q27, Lsbox_magic+16 - ldr q28, Lsbox_magic+32 - ldr q29, Lsbox_magic+48 - ldr q30, Lsbox_magic+64 - ldr q31, Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v16.s[0] - mov w13,v16.s[1] - mov w14,v16.s[2] - mov w15,v16.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v16.s[0],w15 - mov v16.s[1],w14 - mov v16.s[2],w13 - mov v16.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x3,x26 - and x29,x2,#0x0F - // convert length into blocks - lsr x2,x2,4 - cmp x2,#1 - b.lt .return_gb - - cmp x29,0 - // If the encryption/decryption Length is N times of 16, - // the all blocks are encrypted/decrypted in .xts_encrypt_blocks_gb - b.eq .xts_encrypt_blocks_gb - - // If the encryption/decryption length is not N times of 16, - // the last two blocks are encrypted/decrypted in .last_2blks_tweak_gb or .only_2blks_tweak_gb - // the other blocks are encrypted/decrypted in .xts_encrypt_blocks_gb - subs x2,x2,#1 - b.eq .only_2blks_tweak_gb -.xts_encrypt_blocks_gb: - rbit v16.16b,v16.16b -#ifdef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x12,v16.d[0] - mov x13,v16.d[1] - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 -Lxts_8_blocks_process_gb: - cmp x2,#8 - mov v16.d[0],x12 - mov v16.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov w7,0x87 - extr x9,x27,x27,#32 - extr x13,x27,x26,#63 - and w8,w7,w9,asr#31 - eor x12,x8,x26,lsl#1 - mov v17.d[0],x14 - mov v17.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov v18.d[0],x16 - mov v18.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v18.16b,v18.16b -#endif - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov v19.d[0],x18 - mov v19.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v19.16b,v19.16b -#endif - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov v20.d[0],x20 - mov v20.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v20.16b,v20.16b -#endif - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov v21.d[0],x22 - mov v21.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v21.16b,v21.16b -#endif - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov v22.d[0],x24 - mov v22.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v22.16b,v22.16b -#endif - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov v23.d[0],x26 - mov v23.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v23.16b,v23.16b -#endif - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 - b.lt Lxts_4_blocks_process_gb - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - rbit v16.16b,v16.16b - rbit v17.16b,v17.16b - rbit v18.16b,v18.16b - rbit v19.16b,v19.16b - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b - eor v7.16b, v7.16b, v19.16b - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - rbit v20.16b,v20.16b - rbit v21.16b,v21.16b - rbit v22.16b,v22.16b - rbit v23.16b,v23.16b - eor v8.16b, v8.16b, v20.16b - eor v9.16b, v9.16b, v21.16b - eor v10.16b, v10.16b, v22.16b - eor v11.16b, v11.16b, v23.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - zip1 v0.4s,v8.4s,v9.4s - zip2 v1.4s,v8.4s,v9.4s - zip1 v2.4s,v10.4s,v11.4s - zip2 v3.4s,v10.4s,v11.4s - zip1 v8.2d,v0.2d,v2.2d - zip2 v9.2d,v0.2d,v2.2d - zip1 v10.2d,v1.2d,v3.2d - zip2 v11.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - eor v3.16b, v3.16b, v19.16b - eor v4.16b, v4.16b, v20.16b - eor v5.16b, v5.16b, v21.16b - eor v6.16b, v6.16b, v22.16b - eor v7.16b, v7.16b, v23.16b - - // save the last tweak - mov v25.16b,v23.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs x2,x2,#8 - b.gt Lxts_8_blocks_process_gb - b 100f -Lxts_4_blocks_process_gb: - cmp x2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - rbit v16.16b,v16.16b - rbit v17.16b,v17.16b - rbit v18.16b,v18.16b - rbit v19.16b,v19.16b - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b - eor v7.16b, v7.16b, v19.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - eor v3.16b, v3.16b, v19.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub x2,x2,#4 - mov v16.16b,v20.16b - mov v17.16b,v21.16b - mov v18.16b,v22.16b - // save the last tweak - mov v25.16b,v19.16b -1: - // process last block - cmp x2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - rbit v16.16b,v16.16b - eor v4.16b, v4.16b, v16.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v16.16b - st1 {v4.4s},[x1],#16 - // save the last tweak - mov v25.16b,v16.16b - b 100f -1: // process last 2 blocks - cmp x2,#2 - b.gt 1f - ld1 {v4.4s,v5.4s},[x0],#32 - rbit v16.16b,v16.16b - rbit v17.16b,v17.16b - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save the last tweak - mov v25.16b,v17.16b - b 100f -1: // process last 3 blocks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - rbit v16.16b,v16.16b - rbit v17.16b,v17.16b - rbit v18.16b,v18.16b - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save the last tweak - mov v25.16b,v18.16b -100: - cmp x29,0 - b.eq .return_gb - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is larger than 32 -.last_2blks_tweak_gb: -#ifdef __AARCH64EB__ - rev32 v25.16b,v25.16b -#endif - rbit v2.16b,v25.16b - ldr q0, Lxts_magic - shl v17.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v17.16b, v17.16b, v1.16b - rbit v17.16b,v17.16b - rbit v2.16b,v17.16b - ldr q0, Lxts_magic - shl v18.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v18.16b, v18.16b, v1.16b - rbit v18.16b,v18.16b - b .check_dec_gb - - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is equal to 32, who only need two tweaks -.only_2blks_tweak_gb: - mov v17.16b,v16.16b -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif - rbit v2.16b,v17.16b - ldr q0, Lxts_magic - shl v18.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v18.16b, v18.16b, v1.16b - rbit v18.16b,v18.16b - b .check_dec_gb - - -// Determine whether encryption or decryption is required. -// The last two tweaks need to be swapped for decryption. -.check_dec_gb: - // encryption:1 decryption:0 - cmp w28,1 - b.eq .process_last_2blks_gb - mov v0.16B,v17.16b - mov v17.16B,v18.16b - mov v18.16B,v0.16b - -.process_last_2blks_gb: -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif -#ifdef __AARCH64EB__ - rev32 v18.16b,v18.16b -#endif - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v17.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v17.16b - st1 {v4.4s},[x1],#16 - - sub x26,x1,16 -.loop_gb: - subs x29,x29,1 - ldrb w7,[x26,x29] - ldrb w8,[x0,x29] - strb w8,[x26,x29] - strb w7,[x1,x29] - b.gt .loop_gb - ld1 {v4.4s}, [x26] - eor v4.16b, v4.16b, v18.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v18.16b - st1 {v4.4s}, [x26] -.return_gb: - ldp d14, d15, [sp], #0x10 - ldp d12, d13, [sp], #0x10 - ldp d10, d11, [sp], #0x10 - ldp d8, d9, [sp], #0x10 - ldp x29, x30, [sp], #0x10 - ldp x27, x28, [sp], #0x10 - ldp x25, x26, [sp], #0x10 - ldp x23, x24, [sp], #0x10 - ldp x21, x22, [sp], #0x10 - ldp x19, x20, [sp], #0x10 - ldp x17, x18, [sp], #0x10 - ldp x15, x16, [sp], #0x10 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _vpsm4_ex_xts_encrypt - -.align 5 -_vpsm4_ex_xts_encrypt: - AARCH64_SIGN_LINK_REGISTER - stp x15, x16, [sp, #-0x10]! - stp x17, x18, [sp, #-0x10]! - stp x19, x20, [sp, #-0x10]! - stp x21, x22, [sp, #-0x10]! - stp x23, x24, [sp, #-0x10]! - stp x25, x26, [sp, #-0x10]! - stp x27, x28, [sp, #-0x10]! - stp x29, x30, [sp, #-0x10]! - stp d8, d9, [sp, #-0x10]! - stp d10, d11, [sp, #-0x10]! - stp d12, d13, [sp, #-0x10]! - stp d14, d15, [sp, #-0x10]! - mov x26,x3 - mov x27,x4 - mov w28,w6 - ld1 {v16.4s}, [x5] - mov x3,x27 - ldr q26, Lsbox_magic - ldr q27, Lsbox_magic+16 - ldr q28, Lsbox_magic+32 - ldr q29, Lsbox_magic+48 - ldr q30, Lsbox_magic+64 - ldr q31, Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v16.s[0] - mov w13,v16.s[1] - mov w14,v16.s[2] - mov w15,v16.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v16.s[0],w15 - mov v16.s[1],w14 - mov v16.s[2],w13 - mov v16.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x3,x26 - and x29,x2,#0x0F - // convert length into blocks - lsr x2,x2,4 - cmp x2,#1 - b.lt .return - - cmp x29,0 - // If the encryption/decryption Length is N times of 16, - // the all blocks are encrypted/decrypted in .xts_encrypt_blocks - b.eq .xts_encrypt_blocks - - // If the encryption/decryption length is not N times of 16, - // the last two blocks are encrypted/decrypted in .last_2blks_tweak or .only_2blks_tweak - // the other blocks are encrypted/decrypted in .xts_encrypt_blocks - subs x2,x2,#1 - b.eq .only_2blks_tweak -.xts_encrypt_blocks: -#ifdef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x12,v16.d[0] - mov x13,v16.d[1] - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 -Lxts_8_blocks_process: - cmp x2,#8 - mov v16.d[0],x12 - mov v16.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov w7,0x87 - extr x9,x27,x27,#32 - extr x13,x27,x26,#63 - and w8,w7,w9,asr#31 - eor x12,x8,x26,lsl#1 - mov v17.d[0],x14 - mov v17.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov v18.d[0],x16 - mov v18.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v18.16b,v18.16b -#endif - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov v19.d[0],x18 - mov v19.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v19.16b,v19.16b -#endif - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov v20.d[0],x20 - mov v20.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v20.16b,v20.16b -#endif - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov v21.d[0],x22 - mov v21.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v21.16b,v21.16b -#endif - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov v22.d[0],x24 - mov v22.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v22.16b,v22.16b -#endif - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov v23.d[0],x26 - mov v23.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v23.16b,v23.16b -#endif - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 - b.lt Lxts_4_blocks_process - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b - eor v7.16b, v7.16b, v19.16b - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - eor v8.16b, v8.16b, v20.16b - eor v9.16b, v9.16b, v21.16b - eor v10.16b, v10.16b, v22.16b - eor v11.16b, v11.16b, v23.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - zip1 v0.4s,v8.4s,v9.4s - zip2 v1.4s,v8.4s,v9.4s - zip1 v2.4s,v10.4s,v11.4s - zip2 v3.4s,v10.4s,v11.4s - zip1 v8.2d,v0.2d,v2.2d - zip2 v9.2d,v0.2d,v2.2d - zip1 v10.2d,v1.2d,v3.2d - zip2 v11.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - eor v3.16b, v3.16b, v19.16b - eor v4.16b, v4.16b, v20.16b - eor v5.16b, v5.16b, v21.16b - eor v6.16b, v6.16b, v22.16b - eor v7.16b, v7.16b, v23.16b - - // save the last tweak - mov v25.16b,v23.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs x2,x2,#8 - b.gt Lxts_8_blocks_process - b 100f -Lxts_4_blocks_process: - cmp x2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b - eor v7.16b, v7.16b, v19.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - eor v3.16b, v3.16b, v19.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub x2,x2,#4 - mov v16.16b,v20.16b - mov v17.16b,v21.16b - mov v18.16b,v22.16b - // save the last tweak - mov v25.16b,v19.16b -1: - // process last block - cmp x2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v16.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v16.16b - st1 {v4.4s},[x1],#16 - // save the last tweak - mov v25.16b,v16.16b - b 100f -1: // process last 2 blocks - cmp x2,#2 - b.gt 1f - ld1 {v4.4s,v5.4s},[x0],#32 - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save the last tweak - mov v25.16b,v17.16b - b 100f -1: // process last 3 blocks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save the last tweak - mov v25.16b,v18.16b -100: - cmp x29,0 - b.eq .return - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is larger than 32 -.last_2blks_tweak: -#ifdef __AARCH64EB__ - rev32 v25.16b,v25.16b -#endif - mov v2.16b,v25.16b - ldr q0, Lxts_magic - shl v17.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v17.16b, v17.16b, v1.16b - mov v2.16b,v17.16b - ldr q0, Lxts_magic - shl v18.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v18.16b, v18.16b, v1.16b - b .check_dec - - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is equal to 32, who only need two tweaks -.only_2blks_tweak: - mov v17.16b,v16.16b -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif - mov v2.16b,v17.16b - ldr q0, Lxts_magic - shl v18.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v18.16b, v18.16b, v1.16b - b .check_dec - - -// Determine whether encryption or decryption is required. -// The last two tweaks need to be swapped for decryption. -.check_dec: - // encryption:1 decryption:0 - cmp w28,1 - b.eq .process_last_2blks - mov v0.16B,v17.16b - mov v17.16B,v18.16b - mov v18.16B,v0.16b - -.process_last_2blks: -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif -#ifdef __AARCH64EB__ - rev32 v18.16b,v18.16b -#endif - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v17.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v17.16b - st1 {v4.4s},[x1],#16 - - sub x26,x1,16 -.loop: - subs x29,x29,1 - ldrb w7,[x26,x29] - ldrb w8,[x0,x29] - strb w8,[x26,x29] - strb w7,[x1,x29] - b.gt .loop - ld1 {v4.4s}, [x26] - eor v4.16b, v4.16b, v18.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v18.16b - st1 {v4.4s}, [x26] -.return: - ldp d14, d15, [sp], #0x10 - ldp d12, d13, [sp], #0x10 - ldp d10, d11, [sp], #0x10 - ldp d8, d9, [sp], #0x10 - ldp x29, x30, [sp], #0x10 - ldp x27, x28, [sp], #0x10 - ldp x25, x26, [sp], #0x10 - ldp x23, x24, [sp], #0x10 - ldp x21, x22, [sp], #0x10 - ldp x19, x20, [sp], #0x10 - ldp x17, x18, [sp], #0x10 - ldp x15, x16, [sp], #0x10 - AARCH64_VALIDATE_LINK_REGISTER - ret - diff --git a/openssl/src/crypto/sm4/gen/linux_arm64/sm4-armv8.S b/openssl/src/crypto/sm4/gen/linux_arm64/sm4-armv8.S index a0328d7ca..7d0b445f1 100644 --- a/openssl/src/crypto/sm4/gen/linux_arm64/sm4-armv8.S +++ b/openssl/src/crypto/sm4/gen/linux_arm64/sm4-armv8.S @@ -31,7 +31,6 @@ .type sm4_v8_set_encrypt_key,%function .align 5 sm4_v8_set_encrypt_key: - AARCH64_VALID_CALL_TARGET ld1 {v0.4s},[x0] adr x2,.Lfk ld1 {v24.4s},[x2] @@ -58,7 +57,6 @@ sm4_v8_set_encrypt_key: .type sm4_v8_set_decrypt_key,%function .align 5 sm4_v8_set_decrypt_key: - AARCH64_VALID_CALL_TARGET ld1 {v7.4s},[x0] adr x2,.Lfk ld1 {v24.4s},[x2] @@ -101,7 +99,6 @@ sm4_v8_set_decrypt_key: .type sm4_v8_encrypt,%function .align 5 sm4_v8_encrypt: - AARCH64_VALID_CALL_TARGET ld1 {v16.4s},[x0] ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x2],64 ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x2] @@ -128,7 +125,6 @@ sm4_v8_encrypt: .type sm4_v8_decrypt,%function .align 5 sm4_v8_decrypt: - AARCH64_VALID_CALL_TARGET ld1 {v16.4s},[x0] ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x2],64 ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x2] @@ -155,7 +151,6 @@ sm4_v8_decrypt: .type sm4_v8_ecb_encrypt,%function .align 5 sm4_v8_ecb_encrypt: - AARCH64_VALID_CALL_TARGET ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x3],#64 ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x3] 1: @@ -418,7 +413,6 @@ sm4_v8_ecb_encrypt: .type sm4_v8_cbc_encrypt,%function .align 5 sm4_v8_cbc_encrypt: - AARCH64_VALID_CALL_TARGET stp d8,d9,[sp, #-16]! ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x3],#64 @@ -809,7 +803,6 @@ sm4_v8_cbc_encrypt: .type sm4_v8_ctr32_encrypt_blocks,%function .align 5 sm4_v8_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET stp d8,d9,[sp, #-16]! ld1 {v8.4s},[x4] diff --git a/openssl/src/crypto/sm4/gen/linux_arm64/vpsm4-armv8.S b/openssl/src/crypto/sm4/gen/linux_arm64/vpsm4-armv8.S deleted file mode 100644 index 7aef996c6..000000000 --- a/openssl/src/crypto/sm4/gen/linux_arm64/vpsm4-armv8.S +++ /dev/null @@ -1,4999 +0,0 @@ -// Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html - -// -// This module implements SM4 with ASIMD on aarch64 -// -// Feb 2022 -// - -// $output is the last argument if it looks like a file (it has an extension) -// $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" -.arch armv8-a -.text - -.type _vpsm4_consts,%object -.align 7 -_vpsm4_consts: -.Lsbox: -.byte 0xD6,0x90,0xE9,0xFE,0xCC,0xE1,0x3D,0xB7,0x16,0xB6,0x14,0xC2,0x28,0xFB,0x2C,0x05 -.byte 0x2B,0x67,0x9A,0x76,0x2A,0xBE,0x04,0xC3,0xAA,0x44,0x13,0x26,0x49,0x86,0x06,0x99 -.byte 0x9C,0x42,0x50,0xF4,0x91,0xEF,0x98,0x7A,0x33,0x54,0x0B,0x43,0xED,0xCF,0xAC,0x62 -.byte 0xE4,0xB3,0x1C,0xA9,0xC9,0x08,0xE8,0x95,0x80,0xDF,0x94,0xFA,0x75,0x8F,0x3F,0xA6 -.byte 0x47,0x07,0xA7,0xFC,0xF3,0x73,0x17,0xBA,0x83,0x59,0x3C,0x19,0xE6,0x85,0x4F,0xA8 -.byte 0x68,0x6B,0x81,0xB2,0x71,0x64,0xDA,0x8B,0xF8,0xEB,0x0F,0x4B,0x70,0x56,0x9D,0x35 -.byte 0x1E,0x24,0x0E,0x5E,0x63,0x58,0xD1,0xA2,0x25,0x22,0x7C,0x3B,0x01,0x21,0x78,0x87 -.byte 0xD4,0x00,0x46,0x57,0x9F,0xD3,0x27,0x52,0x4C,0x36,0x02,0xE7,0xA0,0xC4,0xC8,0x9E -.byte 0xEA,0xBF,0x8A,0xD2,0x40,0xC7,0x38,0xB5,0xA3,0xF7,0xF2,0xCE,0xF9,0x61,0x15,0xA1 -.byte 0xE0,0xAE,0x5D,0xA4,0x9B,0x34,0x1A,0x55,0xAD,0x93,0x32,0x30,0xF5,0x8C,0xB1,0xE3 -.byte 0x1D,0xF6,0xE2,0x2E,0x82,0x66,0xCA,0x60,0xC0,0x29,0x23,0xAB,0x0D,0x53,0x4E,0x6F -.byte 0xD5,0xDB,0x37,0x45,0xDE,0xFD,0x8E,0x2F,0x03,0xFF,0x6A,0x72,0x6D,0x6C,0x5B,0x51 -.byte 0x8D,0x1B,0xAF,0x92,0xBB,0xDD,0xBC,0x7F,0x11,0xD9,0x5C,0x41,0x1F,0x10,0x5A,0xD8 -.byte 0x0A,0xC1,0x31,0x88,0xA5,0xCD,0x7B,0xBD,0x2D,0x74,0xD0,0x12,0xB8,0xE5,0xB4,0xB0 -.byte 0x89,0x69,0x97,0x4A,0x0C,0x96,0x77,0x7E,0x65,0xB9,0xF1,0x09,0xC5,0x6E,0xC6,0x84 -.byte 0x18,0xF0,0x7D,0xEC,0x3A,0xDC,0x4D,0x20,0x79,0xEE,0x5F,0x3E,0xD7,0xCB,0x39,0x48 -.Lck: -.long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 -.long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 -.long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 -.long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 -.long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 -.long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 -.long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 -.long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 -.Lfk: -.quad 0x56aa3350a3b1bac6,0xb27022dc677d9197 -.Lshuffles: -.quad 0x0B0A090807060504,0x030201000F0E0D0C -.Lxts_magic: -.quad 0x0101010101010187,0x0101010101010101 - -.size _vpsm4_consts,.-_vpsm4_consts -.type _vpsm4_set_key,%function -.align 4 -_vpsm4_set_key: - AARCH64_VALID_CALL_TARGET - ld1 {v5.4s},[x0] - adr x10,.Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - adr x5,.Lshuffles - ld1 {v7.2d},[x5] - adr x5,.Lfk - ld1 {v6.2d},[x5] - eor v5.16b,v5.16b,v6.16b - mov x6,#32 - adr x5,.Lck - movi v0.16b,#64 - cbnz w2,1f - add x1,x1,124 -1: - mov w7,v5.s[1] - ldr w8,[x5],#4 - eor w8,w8,w7 - mov w7,v5.s[2] - eor w8,w8,w7 - mov w7,v5.s[3] - eor w8,w8,w7 - // sbox lookup - mov v4.s[0],w8 - tbl v1.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v4.16b - sub v4.16b,v4.16b,v0.16b - tbx v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v4.16b - sub v4.16b,v4.16b,v0.16b - tbx v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v4.16b - sub v4.16b,v4.16b,v0.16b - tbx v1.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v4.16b - mov w7,v1.s[0] - eor w8,w7,w7,ror #19 - eor w8,w8,w7,ror #9 - mov w7,v5.s[0] - eor w8,w8,w7 - mov v5.s[0],w8 - cbz w2,2f - str w8,[x1],#4 - b 3f -2: - str w8,[x1],#-4 -3: - tbl v5.16b,{v5.16b},v7.16b - subs x6,x6,#1 - b.ne 1b - ret -.size _vpsm4_set_key,.-_vpsm4_set_key -.type _vpsm4_enc_4blks,%function -.align 4 -_vpsm4_enc_4blks: - AARCH64_VALID_CALL_TARGET - mov x10,x3 - mov w11,#8 -10: - ldp w7,w8,[x10],8 - dup v12.4s,w7 - dup v13.4s,w8 - - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor v14.16b,v6.16b,v7.16b - eor v12.16b,v5.16b,v12.16b - eor v12.16b,v14.16b,v12.16b - movi v0.16b,#64 - movi v1.16b,#128 - movi v2.16b,#192 - sub v0.16b,v12.16b,v0.16b - sub v1.16b,v12.16b,v1.16b - sub v2.16b,v12.16b,v2.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v0.2d,v0.2d,v1.2d - add v2.2d,v2.2d,v12.2d - add v12.2d,v0.2d,v2.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v12.4s,32-10 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v12.4s,10 - eor v1.16b,v2.16b,v1.16b - ushr v0.4s,v12.4s,32-18 - sli v0.4s,v12.4s,18 - ushr v2.4s,v12.4s,32-24 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v12.4s,24 - eor v12.16b,v2.16b,v1.16b - eor v4.16b,v4.16b,v12.16b - - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor v14.16b,v14.16b,v4.16b - eor v13.16b,v14.16b,v13.16b - movi v0.16b,#64 - movi v1.16b,#128 - movi v2.16b,#192 - sub v0.16b,v13.16b,v0.16b - sub v1.16b,v13.16b,v1.16b - sub v2.16b,v13.16b,v2.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v0.2d,v0.2d,v1.2d - add v2.2d,v2.2d,v13.2d - add v13.2d,v0.2d,v2.2d - - ushr v0.4s,v13.4s,32-2 - sli v0.4s,v13.4s,2 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v13.16b - sli v2.4s,v13.4s,10 - eor v1.16b,v2.16b,v1.16b - ushr v0.4s,v13.4s,32-18 - sli v0.4s,v13.4s,18 - ushr v2.4s,v13.4s,32-24 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v1.16b - ldp w7,w8,[x10],8 - eor v5.16b,v5.16b,v13.16b - - dup v12.4s,w7 - dup v13.4s,w8 - - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor v14.16b,v4.16b,v5.16b - eor v12.16b,v7.16b,v12.16b - eor v12.16b,v14.16b,v12.16b - movi v0.16b,#64 - movi v1.16b,#128 - movi v2.16b,#192 - sub v0.16b,v12.16b,v0.16b - sub v1.16b,v12.16b,v1.16b - sub v2.16b,v12.16b,v2.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v0.2d,v0.2d,v1.2d - add v2.2d,v2.2d,v12.2d - add v12.2d,v0.2d,v2.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v12.4s,32-10 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v12.4s,10 - eor v1.16b,v2.16b,v1.16b - ushr v0.4s,v12.4s,32-18 - sli v0.4s,v12.4s,18 - ushr v2.4s,v12.4s,32-24 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v12.4s,24 - eor v12.16b,v2.16b,v1.16b - eor v6.16b,v6.16b,v12.16b - - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor v14.16b,v14.16b,v6.16b - eor v13.16b,v14.16b,v13.16b - movi v0.16b,#64 - movi v1.16b,#128 - movi v2.16b,#192 - sub v0.16b,v13.16b,v0.16b - sub v1.16b,v13.16b,v1.16b - sub v2.16b,v13.16b,v2.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v0.2d,v0.2d,v1.2d - add v2.2d,v2.2d,v13.2d - add v13.2d,v0.2d,v2.2d - - ushr v0.4s,v13.4s,32-2 - sli v0.4s,v13.4s,2 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v13.16b - sli v2.4s,v13.4s,10 - eor v1.16b,v2.16b,v1.16b - ushr v0.4s,v13.4s,32-18 - sli v0.4s,v13.4s,18 - ushr v2.4s,v13.4s,32-24 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v1.16b - eor v7.16b,v7.16b,v13.16b - subs w11,w11,#1 - b.ne 10b -#ifndef __AARCH64EB__ - rev32 v3.16b,v4.16b -#else - mov v3.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v2.16b,v5.16b -#else - mov v2.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v1.16b,v6.16b -#else - mov v1.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v0.16b,v7.16b -#else - mov v0.16b,v7.16b -#endif - ret -.size _vpsm4_enc_4blks,.-_vpsm4_enc_4blks -.type _vpsm4_enc_8blks,%function -.align 4 -_vpsm4_enc_8blks: - AARCH64_VALID_CALL_TARGET - mov x10,x3 - mov w11,#8 -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - dup v12.4s,w7 - eor v14.16b,v6.16b,v7.16b - eor v15.16b,v10.16b,v11.16b - eor v0.16b,v5.16b,v12.16b - eor v1.16b,v9.16b,v12.16b - eor v12.16b,v14.16b,v0.16b - eor v13.16b,v15.16b,v1.16b - movi v3.16b,#64 - sub v0.16b,v12.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v12.2d,v2.2d,v12.2d - add v12.2d,v1.2d,v12.2d - - sub v0.16b,v13.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v13.2d,v2.2d,v13.2d - add v13.2d,v1.2d,v13.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v13.4s,32-2 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v13.4s,2 - - ushr v0.4s,v12.4s,32-10 - eor v3.16b,v2.16b,v13.16b - sli v0.4s,v12.4s,10 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,10 - - ushr v0.4s,v12.4s,32-18 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,18 - ushr v2.4s,v13.4s,32-18 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,18 - - ushr v0.4s,v12.4s,32-24 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,24 - ushr v2.4s,v13.4s,32-24 - eor v12.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v3.16b - eor v4.16b,v4.16b,v12.16b - eor v8.16b,v8.16b,v13.16b - - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - dup v13.4s,w8 - eor v14.16b,v14.16b,v4.16b - eor v15.16b,v15.16b,v8.16b - eor v12.16b,v14.16b,v13.16b - eor v13.16b,v15.16b,v13.16b - movi v3.16b,#64 - sub v0.16b,v12.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v12.2d,v2.2d,v12.2d - add v12.2d,v1.2d,v12.2d - - sub v0.16b,v13.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v13.2d,v2.2d,v13.2d - add v13.2d,v1.2d,v13.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v13.4s,32-2 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v13.4s,2 - - ushr v0.4s,v12.4s,32-10 - eor v3.16b,v2.16b,v13.16b - sli v0.4s,v12.4s,10 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,10 - - ushr v0.4s,v12.4s,32-18 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,18 - ushr v2.4s,v13.4s,32-18 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,18 - - ushr v0.4s,v12.4s,32-24 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,24 - ushr v2.4s,v13.4s,32-24 - eor v12.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v3.16b - ldp w7,w8,[x10],8 - eor v5.16b,v5.16b,v12.16b - eor v9.16b,v9.16b,v13.16b - - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - dup v12.4s,w7 - eor v14.16b,v4.16b,v5.16b - eor v15.16b,v8.16b,v9.16b - eor v0.16b,v7.16b,v12.16b - eor v1.16b,v11.16b,v12.16b - eor v12.16b,v14.16b,v0.16b - eor v13.16b,v15.16b,v1.16b - movi v3.16b,#64 - sub v0.16b,v12.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v12.2d,v2.2d,v12.2d - add v12.2d,v1.2d,v12.2d - - sub v0.16b,v13.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v13.2d,v2.2d,v13.2d - add v13.2d,v1.2d,v13.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v13.4s,32-2 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v13.4s,2 - - ushr v0.4s,v12.4s,32-10 - eor v3.16b,v2.16b,v13.16b - sli v0.4s,v12.4s,10 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,10 - - ushr v0.4s,v12.4s,32-18 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,18 - ushr v2.4s,v13.4s,32-18 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,18 - - ushr v0.4s,v12.4s,32-24 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,24 - ushr v2.4s,v13.4s,32-24 - eor v12.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v3.16b - eor v6.16b,v6.16b,v12.16b - eor v10.16b,v10.16b,v13.16b - - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - dup v13.4s,w8 - eor v14.16b,v14.16b,v6.16b - eor v15.16b,v15.16b,v10.16b - eor v12.16b,v14.16b,v13.16b - eor v13.16b,v15.16b,v13.16b - movi v3.16b,#64 - sub v0.16b,v12.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v12.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v12.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v12.2d,v2.2d,v12.2d - add v12.2d,v1.2d,v12.2d - - sub v0.16b,v13.16b,v3.16b - sub v1.16b,v0.16b,v3.16b - sub v2.16b,v1.16b,v3.16b - tbl v13.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v13.16b - tbl v0.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v0.16b - tbl v1.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v1.16b - tbl v2.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v2.16b - add v1.2d,v0.2d,v1.2d - add v13.2d,v2.2d,v13.2d - add v13.2d,v1.2d,v13.2d - - ushr v0.4s,v12.4s,32-2 - sli v0.4s,v12.4s,2 - ushr v2.4s,v13.4s,32-2 - eor v1.16b,v0.16b,v12.16b - sli v2.4s,v13.4s,2 - - ushr v0.4s,v12.4s,32-10 - eor v3.16b,v2.16b,v13.16b - sli v0.4s,v12.4s,10 - ushr v2.4s,v13.4s,32-10 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,10 - - ushr v0.4s,v12.4s,32-18 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,18 - ushr v2.4s,v13.4s,32-18 - eor v1.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,18 - - ushr v0.4s,v12.4s,32-24 - eor v3.16b,v2.16b,v3.16b - sli v0.4s,v12.4s,24 - ushr v2.4s,v13.4s,32-24 - eor v12.16b,v0.16b,v1.16b - sli v2.4s,v13.4s,24 - eor v13.16b,v2.16b,v3.16b - eor v7.16b,v7.16b,v12.16b - eor v11.16b,v11.16b,v13.16b - subs w11,w11,#1 - b.ne 10b -#ifndef __AARCH64EB__ - rev32 v3.16b,v4.16b -#else - mov v3.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v2.16b,v5.16b -#else - mov v2.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v1.16b,v6.16b -#else - mov v1.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v0.16b,v7.16b -#else - mov v0.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v8.16b -#else - mov v7.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v9.16b -#else - mov v6.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v10.16b -#else - mov v5.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v4.16b,v11.16b -#else - mov v4.16b,v11.16b -#endif - ret -.size _vpsm4_enc_8blks,.-_vpsm4_enc_8blks -.globl vpsm4_set_encrypt_key -.type vpsm4_set_encrypt_key,%function -.align 5 -vpsm4_set_encrypt_key: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - mov w2,1 - bl _vpsm4_set_key - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_set_encrypt_key,.-vpsm4_set_encrypt_key -.globl vpsm4_set_decrypt_key -.type vpsm4_set_decrypt_key,%function -.align 5 -vpsm4_set_decrypt_key: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - mov w2,0 - bl _vpsm4_set_key - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_set_decrypt_key,.-vpsm4_set_decrypt_key -.globl vpsm4_encrypt -.type vpsm4_encrypt,%function -.align 5 -vpsm4_encrypt: - AARCH64_VALID_CALL_TARGET - ld1 {v4.4s},[x0] - adr x10,.Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x3,x2 - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - ret -.size vpsm4_encrypt,.-vpsm4_encrypt -.globl vpsm4_decrypt -.type vpsm4_decrypt,%function -.align 5 -vpsm4_decrypt: - AARCH64_VALID_CALL_TARGET - ld1 {v4.4s},[x0] - adr x10,.Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x3,x2 - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - ret -.size vpsm4_decrypt,.-vpsm4_decrypt -.globl vpsm4_ecb_encrypt -.type vpsm4_ecb_encrypt,%function -.align 5 -vpsm4_ecb_encrypt: - AARCH64_SIGN_LINK_REGISTER - // convert length into blocks - lsr x2,x2,4 - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] - adr x10,.Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -.Lecb_8_blocks_process: - cmp w2,#8 - b.lt .Lecb_4_blocks_process - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - bl _vpsm4_enc_8blks - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st4 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.gt .Lecb_8_blocks_process - b 100f -.Lecb_4_blocks_process: - cmp w2,#4 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub w2,w2,#4 -1: - // process last block - cmp w2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - b 100f -1: // process last 2 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[0],[x0],#16 - ld4 {v4.s,v5.s,v6.s,v7.s}[1],[x0],#16 - cmp w2,#2 - b.gt 1f -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1] - b 100f -1: // process last 3 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[2],[x0],#16 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[2],[x1] -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ecb_encrypt,.-vpsm4_ecb_encrypt -.globl vpsm4_cbc_encrypt -.type vpsm4_cbc_encrypt,%function -.align 5 -vpsm4_cbc_encrypt: - AARCH64_VALID_CALL_TARGET - lsr x2,x2,4 - adr x10,.Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] - cbz w5,.Ldec - ld1 {v3.4s},[x4] -.Lcbc_4_blocks_enc: - cmp w2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b,v4.16b,v3.16b -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 - eor v5.16b,v5.16b,v4.16b - mov x10,x3 - mov w11,#8 - mov w12,v5.s[0] - mov w13,v5.s[1] - mov w14,v5.s[2] - mov w15,v5.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v5.s[0],w15 - mov v5.s[1],w14 - mov v5.s[2],w13 - mov v5.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v6.16b,v6.16b,v5.16b - mov x10,x3 - mov w11,#8 - mov w12,v6.s[0] - mov w13,v6.s[1] - mov w14,v6.s[2] - mov w15,v6.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v6.s[0],w15 - mov v6.s[1],w14 - mov v6.s[2],w13 - mov v6.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - eor v7.16b,v7.16b,v6.16b - mov x10,x3 - mov w11,#8 - mov w12,v7.s[0] - mov w13,v7.s[1] - mov w14,v7.s[2] - mov w15,v7.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v7.s[0],w15 - mov v7.s[1],w14 - mov v7.s[2],w13 - mov v7.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - orr v3.16b,v7.16b,v7.16b - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#4 - b.ne .Lcbc_4_blocks_enc - b 2f -1: - subs w2,w2,#1 - b.lt 2f - ld1 {v4.4s},[x0],#16 - eor v3.16b,v3.16b,v4.16b -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - st1 {v3.4s},[x1],#16 - b 1b -2: - // save back IV - st1 {v3.4s},[x4] - ret - -.Ldec: - // decryption mode starts - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] -.Lcbc_8_blocks_dec: - cmp w2,#8 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0] - add x10,x0,#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - bl _vpsm4_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - ld1 {v15.4s},[x4] - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - // note ivec1 and vtmpx[3] are reusing the same register - // care needs to be taken to avoid conflict - eor v0.16b,v0.16b,v15.16b - ld1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - eor v1.16b,v1.16b,v8.16b - eor v2.16b,v2.16b,v9.16b - eor v3.16b,v3.16b,v10.16b - // save back IV - st1 {v15.4s}, [x4] - eor v4.16b,v4.16b,v11.16b - eor v5.16b,v5.16b,v12.16b - eor v6.16b,v6.16b,v13.16b - eor v7.16b,v7.16b,v14.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.gt .Lcbc_8_blocks_dec - b.eq 100f -1: - ld1 {v15.4s},[x4] -.Lcbc_4_blocks_dec: - cmp w2,#4 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - orr v15.16b,v7.16b,v7.16b - eor v2.16b,v2.16b,v5.16b - eor v3.16b,v3.16b,v6.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - subs w2,w2,#4 - b.gt .Lcbc_4_blocks_dec - // save back IV - st1 {v7.4s}, [x4] - b 100f -1: // last block - subs w2,w2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - // save back IV - st1 {v4.4s}, [x4] -#ifndef __AARCH64EB__ - rev32 v8.16b,v4.16b -#else - mov v8.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v8.s[0] - mov w13,v8.s[1] - mov w14,v8.s[2] - mov w15,v8.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v8.s[0],w15 - mov v8.s[1],w14 - mov v8.s[2],w13 - mov v8.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - eor v8.16b,v8.16b,v15.16b - st1 {v8.4s},[x1],#16 - b 100f -1: // last two blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[0],[x0] - add x10,x0,#16 - ld4 {v4.s,v5.s,v6.s,v7.s}[1],[x10],#16 - subs w2,w2,1 - b.gt 1f -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - ld1 {v4.4s,v5.4s},[x0],#32 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save back IV - st1 {v5.4s}, [x4] - b 100f -1: // last 3 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[2],[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_enc_4blks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - eor v2.16b,v2.16b,v5.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save back IV - st1 {v6.4s}, [x4] -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_cbc_encrypt,.-vpsm4_cbc_encrypt -.globl vpsm4_ctr32_encrypt_blocks -.type vpsm4_ctr32_encrypt_blocks,%function -.align 5 -vpsm4_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET - ld1 {v3.4s},[x4] -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - adr x10,.Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] - cmp w2,#1 - b.ne 1f - // fast processing for one single block without - // context saving overhead - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ld1 {v4.4s},[x0] - eor v4.16b,v4.16b,v3.16b - st1 {v4.4s},[x1] - ret -1: - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w5,v3.s[3] -.Lctr32_4_blocks_process: - cmp w2,#4 - b.lt 1f - dup v4.4s,w12 - dup v5.4s,w13 - dup v6.4s,w14 - mov v7.s[0],w5 - add w5,w5,#1 - mov v7.s[1],w5 - add w5,w5,#1 - mov v7.s[2],w5 - add w5,w5,#1 - mov v7.s[3],w5 - add w5,w5,#1 - cmp w2,#8 - b.ge .Lctr32_8_blocks_process - bl _vpsm4_enc_4blks - ld4 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - subs w2,w2,#4 - b.ne .Lctr32_4_blocks_process - b 100f -.Lctr32_8_blocks_process: - dup v8.4s,w12 - dup v9.4s,w13 - dup v10.4s,w14 - mov v11.s[0],w5 - add w5,w5,#1 - mov v11.s[1],w5 - add w5,w5,#1 - mov v11.s[2],w5 - add w5,w5,#1 - mov v11.s[3],w5 - add w5,w5,#1 - bl _vpsm4_enc_8blks - ld4 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - eor v4.16b,v4.16b,v8.16b - eor v5.16b,v5.16b,v9.16b - eor v6.16b,v6.16b,v10.16b - eor v7.16b,v7.16b,v11.16b - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st4 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.ne .Lctr32_4_blocks_process - b 100f -1: // last block processing - subs w2,w2,#1 - b.lt 100f - b.gt 1f - mov v3.s[0],w12 - mov v3.s[1],w13 - mov v3.s[2],w14 - mov v3.s[3],w5 - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ld1 {v4.4s},[x0] - eor v4.16b,v4.16b,v3.16b - st1 {v4.4s},[x1] - b 100f -1: // last 2 blocks processing - dup v4.4s,w12 - dup v5.4s,w13 - dup v6.4s,w14 - mov v7.s[0],w5 - add w5,w5,#1 - mov v7.s[1],w5 - subs w2,w2,#1 - b.ne 1f - bl _vpsm4_enc_4blks - ld4 {v12.s,v13.s,v14.s,v15.s}[0],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[1],[x0],#16 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - b 100f -1: // last 3 blocks processing - add w5,w5,#1 - mov v7.s[2],w5 - bl _vpsm4_enc_4blks - ld4 {v12.s,v13.s,v14.s,v15.s}[0],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[1],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[2],[x0],#16 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[2],[x1],#16 -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ctr32_encrypt_blocks,.-vpsm4_ctr32_encrypt_blocks -.globl vpsm4_xts_encrypt_gb -.type vpsm4_xts_encrypt_gb,%function -.align 5 -vpsm4_xts_encrypt_gb: - AARCH64_SIGN_LINK_REGISTER - stp x15, x16, [sp, #-0x10]! - stp x17, x18, [sp, #-0x10]! - stp x19, x20, [sp, #-0x10]! - stp x21, x22, [sp, #-0x10]! - stp x23, x24, [sp, #-0x10]! - stp x25, x26, [sp, #-0x10]! - stp x27, x28, [sp, #-0x10]! - stp x29, x30, [sp, #-0x10]! - stp d8, d9, [sp, #-0x10]! - stp d10, d11, [sp, #-0x10]! - stp d12, d13, [sp, #-0x10]! - stp d14, d15, [sp, #-0x10]! - mov x26,x3 - mov x27,x4 - mov w28,w6 - ld1 {v8.4s}, [x5] - mov x3,x27 - adr x10,.Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v8.s[0] - mov w13,v8.s[1] - mov w14,v8.s[2] - mov w15,v8.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v8.s[0],w15 - mov v8.s[1],w14 - mov v8.s[2],w13 - mov v8.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x3,x26 - and x29,x2,#0x0F - // convert length into blocks - lsr x2,x2,4 - cmp x2,#1 - b.lt .return_gb - - cmp x29,0 - // If the encryption/decryption Length is N times of 16, - // the all blocks are encrypted/decrypted in .xts_encrypt_blocks_gb - b.eq .xts_encrypt_blocks_gb - - // If the encryption/decryption length is not N times of 16, - // the last two blocks are encrypted/decrypted in .last_2blks_tweak_gb or .only_2blks_tweak_gb - // the other blocks are encrypted/decrypted in .xts_encrypt_blocks_gb - subs x2,x2,#1 - b.eq .only_2blks_tweak_gb -.xts_encrypt_blocks_gb: - rbit v8.16b,v8.16b -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x12,v8.d[0] - mov x13,v8.d[1] - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 -.Lxts_8_blocks_process_gb: - cmp x2,#8 - b.lt .Lxts_4_blocks_process_gb - mov v0.d[0],x12 - mov v0.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v0.16b,v0.16b -#endif - mov v1.d[0],x14 - mov v1.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v1.16b,v1.16b -#endif - mov v2.d[0],x16 - mov v2.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v2.16b,v2.16b -#endif - mov v3.d[0],x18 - mov v3.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - mov v12.d[0],x20 - mov v12.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v12.16b,v12.16b -#endif - mov v13.d[0],x22 - mov v13.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v13.16b,v13.16b -#endif - mov v14.d[0],x24 - mov v14.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v14.16b,v14.16b -#endif - mov v15.d[0],x26 - mov v15.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v15.16b,v15.16b -#endif - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - rbit v0.16b,v0.16b - rbit v1.16b,v1.16b - rbit v2.16b,v2.16b - rbit v3.16b,v3.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - rbit v12.16b,v12.16b - rbit v13.16b,v13.16b - rbit v14.16b,v14.16b - rbit v15.16b,v15.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - zip1 v0.4s,v8.4s,v9.4s - zip2 v1.4s,v8.4s,v9.4s - zip1 v2.4s,v10.4s,v11.4s - zip2 v3.4s,v10.4s,v11.4s - zip1 v8.2d,v0.2d,v2.2d - zip2 v9.2d,v0.2d,v2.2d - zip1 v10.2d,v1.2d,v3.2d - zip2 v11.2d,v1.2d,v3.2d - bl _vpsm4_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - mov v12.d[0],x12 - mov v12.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v12.16b,v12.16b -#endif - mov w7,0x87 - extr x9,x27,x27,#32 - extr x13,x27,x26,#63 - and w8,w7,w9,asr#31 - eor x12,x8,x26,lsl#1 - mov v13.d[0],x14 - mov v13.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v13.16b,v13.16b -#endif - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov v14.d[0],x16 - mov v14.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v14.16b,v14.16b -#endif - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov v15.d[0],x18 - mov v15.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v15.16b,v15.16b -#endif - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov v8.d[0],x20 - mov v8.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov v9.d[0],x22 - mov v9.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov v10.d[0],x24 - mov v10.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov v11.d[0],x26 - mov v11.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - - // save the last tweak - st1 {v11.4s},[x5] - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs x2,x2,#8 - b.gt .Lxts_8_blocks_process_gb - b 100f -.Lxts_4_blocks_process_gb: - mov v8.d[0],x12 - mov v8.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v9.d[0],x14 - mov v9.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v10.d[0],x16 - mov v10.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - mov v11.d[0],x18 - mov v11.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - cmp x2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - rbit v8.16b,v8.16b - rbit v9.16b,v9.16b - rbit v10.16b,v10.16b - rbit v11.16b,v11.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub x2,x2,#4 - mov v8.d[0],x20 - mov v8.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v9.d[0],x22 - mov v9.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v10.d[0],x24 - mov v10.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - // save the last tweak - st1 {v11.4s},[x5] -1: - // process last block - cmp x2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - rbit v8.16b,v8.16b - eor v4.16b, v4.16b, v8.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v8.16b - st1 {v4.4s},[x1],#16 - // save the last tweak - st1 {v8.4s},[x5] - b 100f -1: // process last 2 blocks - cmp x2,#2 - b.gt 1f - ld1 {v4.4s,v5.4s},[x0],#32 - rbit v8.16b,v8.16b - rbit v9.16b,v9.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save the last tweak - st1 {v9.4s},[x5] - b 100f -1: // process last 3 blocks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - rbit v8.16b,v8.16b - rbit v9.16b,v9.16b - rbit v10.16b,v10.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save the last tweak - st1 {v10.4s},[x5] -100: - cmp x29,0 - b.eq .return_gb - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is larger than 32 -.last_2blks_tweak_gb: - ld1 {v8.4s},[x5] -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - rbit v2.16b,v8.16b - ldr q0, .Lxts_magic - shl v9.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - rbit v9.16b,v9.16b - rbit v2.16b,v9.16b - ldr q0, .Lxts_magic - shl v10.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v10.16b, v10.16b, v1.16b - rbit v10.16b,v10.16b - b .check_dec_gb - - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is equal to 32, who only need two tweaks -.only_2blks_tweak_gb: - mov v9.16b,v8.16b -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - rbit v2.16b,v9.16b - ldr q0, .Lxts_magic - shl v10.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v10.16b, v10.16b, v1.16b - rbit v10.16b,v10.16b - b .check_dec_gb - - -// Determine whether encryption or decryption is required. -// The last two tweaks need to be swapped for decryption. -.check_dec_gb: - // encryption:1 decryption:0 - cmp w28,1 - b.eq .process_last_2blks_gb - mov v0.16B,v9.16b - mov v9.16B,v10.16b - mov v10.16B,v0.16b - -.process_last_2blks_gb: -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v9.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v9.16b - st1 {v4.4s},[x1],#16 - - sub x26,x1,16 -.loop_gb: - subs x29,x29,1 - ldrb w7,[x26,x29] - ldrb w8,[x0,x29] - strb w8,[x26,x29] - strb w7,[x1,x29] - b.gt .loop_gb - ld1 {v4.4s}, [x26] - eor v4.16b, v4.16b, v10.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v10.16b - st1 {v4.4s}, [x26] -.return_gb: - ldp d14, d15, [sp], #0x10 - ldp d12, d13, [sp], #0x10 - ldp d10, d11, [sp], #0x10 - ldp d8, d9, [sp], #0x10 - ldp x29, x30, [sp], #0x10 - ldp x27, x28, [sp], #0x10 - ldp x25, x26, [sp], #0x10 - ldp x23, x24, [sp], #0x10 - ldp x21, x22, [sp], #0x10 - ldp x19, x20, [sp], #0x10 - ldp x17, x18, [sp], #0x10 - ldp x15, x16, [sp], #0x10 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_xts_encrypt_gb,.-vpsm4_xts_encrypt_gb -.globl vpsm4_xts_encrypt -.type vpsm4_xts_encrypt,%function -.align 5 -vpsm4_xts_encrypt: - AARCH64_SIGN_LINK_REGISTER - stp x15, x16, [sp, #-0x10]! - stp x17, x18, [sp, #-0x10]! - stp x19, x20, [sp, #-0x10]! - stp x21, x22, [sp, #-0x10]! - stp x23, x24, [sp, #-0x10]! - stp x25, x26, [sp, #-0x10]! - stp x27, x28, [sp, #-0x10]! - stp x29, x30, [sp, #-0x10]! - stp d8, d9, [sp, #-0x10]! - stp d10, d11, [sp, #-0x10]! - stp d12, d13, [sp, #-0x10]! - stp d14, d15, [sp, #-0x10]! - mov x26,x3 - mov x27,x4 - mov w28,w6 - ld1 {v8.4s}, [x5] - mov x3,x27 - adr x10,.Lsbox - ld1 {v16.16b,v17.16b,v18.16b,v19.16b},[x10],#64 - ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x10],#64 - ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x10],#64 - ld1 {v28.16b,v29.16b,v30.16b,v31.16b},[x10] -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v8.s[0] - mov w13,v8.s[1] - mov w14,v8.s[2] - mov w15,v8.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v8.s[0],w15 - mov v8.s[1],w14 - mov v8.s[2],w13 - mov v8.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x3,x26 - and x29,x2,#0x0F - // convert length into blocks - lsr x2,x2,4 - cmp x2,#1 - b.lt .return - - cmp x29,0 - // If the encryption/decryption Length is N times of 16, - // the all blocks are encrypted/decrypted in .xts_encrypt_blocks - b.eq .xts_encrypt_blocks - - // If the encryption/decryption length is not N times of 16, - // the last two blocks are encrypted/decrypted in .last_2blks_tweak or .only_2blks_tweak - // the other blocks are encrypted/decrypted in .xts_encrypt_blocks - subs x2,x2,#1 - b.eq .only_2blks_tweak -.xts_encrypt_blocks: -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov x12,v8.d[0] - mov x13,v8.d[1] - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 -.Lxts_8_blocks_process: - cmp x2,#8 - b.lt .Lxts_4_blocks_process - mov v0.d[0],x12 - mov v0.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v0.16b,v0.16b -#endif - mov v1.d[0],x14 - mov v1.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v1.16b,v1.16b -#endif - mov v2.d[0],x16 - mov v2.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v2.16b,v2.16b -#endif - mov v3.d[0],x18 - mov v3.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - mov v12.d[0],x20 - mov v12.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v12.16b,v12.16b -#endif - mov v13.d[0],x22 - mov v13.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v13.16b,v13.16b -#endif - mov v14.d[0],x24 - mov v14.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v14.16b,v14.16b -#endif - mov v15.d[0],x26 - mov v15.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v15.16b,v15.16b -#endif - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - zip1 v0.4s,v8.4s,v9.4s - zip2 v1.4s,v8.4s,v9.4s - zip1 v2.4s,v10.4s,v11.4s - zip2 v3.4s,v10.4s,v11.4s - zip1 v8.2d,v0.2d,v2.2d - zip2 v9.2d,v0.2d,v2.2d - zip1 v10.2d,v1.2d,v3.2d - zip2 v11.2d,v1.2d,v3.2d - bl _vpsm4_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - mov v12.d[0],x12 - mov v12.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v12.16b,v12.16b -#endif - mov w7,0x87 - extr x9,x27,x27,#32 - extr x13,x27,x26,#63 - and w8,w7,w9,asr#31 - eor x12,x8,x26,lsl#1 - mov v13.d[0],x14 - mov v13.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v13.16b,v13.16b -#endif - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov v14.d[0],x16 - mov v14.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v14.16b,v14.16b -#endif - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov v15.d[0],x18 - mov v15.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v15.16b,v15.16b -#endif - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov v8.d[0],x20 - mov v8.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov v9.d[0],x22 - mov v9.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov v10.d[0],x24 - mov v10.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov v11.d[0],x26 - mov v11.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - - // save the last tweak - st1 {v11.4s},[x5] - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs x2,x2,#8 - b.gt .Lxts_8_blocks_process - b 100f -.Lxts_4_blocks_process: - mov v8.d[0],x12 - mov v8.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v9.d[0],x14 - mov v9.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v10.d[0],x16 - mov v10.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - mov v11.d[0],x18 - mov v11.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - cmp x2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub x2,x2,#4 - mov v8.d[0],x20 - mov v8.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v9.d[0],x22 - mov v9.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v10.d[0],x24 - mov v10.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - // save the last tweak - st1 {v11.4s},[x5] -1: - // process last block - cmp x2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v8.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v8.16b - st1 {v4.4s},[x1],#16 - // save the last tweak - st1 {v8.4s},[x5] - b 100f -1: // process last 2 blocks - cmp x2,#2 - b.gt 1f - ld1 {v4.4s,v5.4s},[x0],#32 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save the last tweak - st1 {v9.4s},[x5] - b 100f -1: // process last 3 blocks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save the last tweak - st1 {v10.4s},[x5] -100: - cmp x29,0 - b.eq .return - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is larger than 32 -.last_2blks_tweak: - ld1 {v8.4s},[x5] -#ifdef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - mov v2.16b,v8.16b - ldr q0, .Lxts_magic - shl v9.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - mov v2.16b,v9.16b - ldr q0, .Lxts_magic - shl v10.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v10.16b, v10.16b, v1.16b - b .check_dec - - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is equal to 32, who only need two tweaks -.only_2blks_tweak: - mov v9.16b,v8.16b -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif - mov v2.16b,v9.16b - ldr q0, .Lxts_magic - shl v10.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v10.16b, v10.16b, v1.16b - b .check_dec - - -// Determine whether encryption or decryption is required. -// The last two tweaks need to be swapped for decryption. -.check_dec: - // encryption:1 decryption:0 - cmp w28,1 - b.eq .process_last_2blks - mov v0.16B,v9.16b - mov v9.16B,v10.16b - mov v10.16B,v0.16b - -.process_last_2blks: -#ifdef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifdef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v9.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v9.16b - st1 {v4.4s},[x1],#16 - - sub x26,x1,16 -.loop: - subs x29,x29,1 - ldrb w7,[x26,x29] - ldrb w8,[x0,x29] - strb w8,[x26,x29] - strb w7,[x1,x29] - b.gt .loop - ld1 {v4.4s}, [x26] - eor v4.16b, v4.16b, v10.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - movi v1.16b,#64 - movi v2.16b,#128 - movi v3.16b,#192 - mov v0.s[0],w6 - - sub v1.16b,v0.16b,v1.16b - sub v2.16b,v0.16b,v2.16b - sub v3.16b,v0.16b,v3.16b - - tbl v0.16b,{v16.16b,v17.16b,v18.16b,v19.16b},v0.16b - tbl v1.16b,{v20.16b,v21.16b,v22.16b,v23.16b},v1.16b - tbl v2.16b,{v24.16b,v25.16b,v26.16b,v27.16b},v2.16b - tbl v3.16b,{v28.16b,v29.16b,v30.16b,v31.16b},v3.16b - - mov w6,v0.s[0] - mov w7,v1.s[0] - mov w9,v2.s[0] - add w7,w6,w7 - mov w6,v3.s[0] - add w7,w7,w9 - add w7,w7,w6 - - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v10.16b - st1 {v4.4s}, [x26] -.return: - ldp d14, d15, [sp], #0x10 - ldp d12, d13, [sp], #0x10 - ldp d10, d11, [sp], #0x10 - ldp d8, d9, [sp], #0x10 - ldp x29, x30, [sp], #0x10 - ldp x27, x28, [sp], #0x10 - ldp x25, x26, [sp], #0x10 - ldp x23, x24, [sp], #0x10 - ldp x21, x22, [sp], #0x10 - ldp x19, x20, [sp], #0x10 - ldp x17, x18, [sp], #0x10 - ldp x15, x16, [sp], #0x10 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_xts_encrypt,.-vpsm4_xts_encrypt diff --git a/openssl/src/crypto/sm4/gen/linux_arm64/vpsm4_ex-armv8.S b/openssl/src/crypto/sm4/gen/linux_arm64/vpsm4_ex-armv8.S deleted file mode 100644 index 1d6a03d14..000000000 --- a/openssl/src/crypto/sm4/gen/linux_arm64/vpsm4_ex-armv8.S +++ /dev/null @@ -1,4505 +0,0 @@ -// Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html - -// -// This module implements SM4 with ASIMD and AESE on AARCH64 -// -// Dec 2022 -// - -// $output is the last argument if it looks like a file (it has an extension) -// $flavour is the first argument if it doesn't look like a file -#include "arm_arch.h" -.arch armv8-a+crypto -.text - -.type _vpsm4_ex_consts,%object -.align 7 -_vpsm4_ex_consts: -.Lck: -.long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 -.long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 -.long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 -.long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 -.long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 -.long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 -.long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 -.long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 -.Lfk: -.quad 0x56aa3350a3b1bac6,0xb27022dc677d9197 -.Lshuffles: -.quad 0x0B0A090807060504,0x030201000F0E0D0C -.Lxts_magic: -.quad 0x0101010101010187,0x0101010101010101 -.Lsbox_magic: -.quad 0x0b0e0104070a0d00,0x0306090c0f020508 -.quad 0x62185a2042387a00,0x22581a6002783a40 -.quad 0x15df62a89e54e923,0xc10bb67c4a803df7 -.quad 0xb9aa6b78c1d21300,0x1407c6d56c7fbead -.quad 0x6404462679195b3b,0xe383c1a1fe9edcbc -.quad 0x0f0f0f0f0f0f0f0f,0x0f0f0f0f0f0f0f0f - -.size _vpsm4_ex_consts,.-_vpsm4_ex_consts -.type _vpsm4_ex_set_key,%function -.align 4 -_vpsm4_ex_set_key: - AARCH64_VALID_CALL_TARGET - ld1 {v5.4s},[x0] - ldr q26, .Lsbox_magic - ldr q27, .Lsbox_magic+16 - ldr q28, .Lsbox_magic+32 - ldr q29, .Lsbox_magic+48 - ldr q30, .Lsbox_magic+64 - ldr q31, .Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - adr x5,.Lshuffles - ld1 {v7.2d},[x5] - adr x5,.Lfk - ld1 {v6.2d},[x5] - eor v5.16b,v5.16b,v6.16b - mov x6,#32 - adr x5,.Lck - movi v0.16b,#64 - cbnz w2,1f - add x1,x1,124 -1: - mov w7,v5.s[1] - ldr w8,[x5],#4 - eor w8,w8,w7 - mov w7,v5.s[2] - eor w8,w8,w7 - mov w7,v5.s[3] - eor w8,w8,w7 - // optimize sbox using AESE instruction - mov v4.s[0],w8 - tbl v0.16b, {v4.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - mov w7,v0.s[0] - eor w8,w7,w7,ror #19 - eor w8,w8,w7,ror #9 - mov w7,v5.s[0] - eor w8,w8,w7 - mov v5.s[0],w8 - cbz w2,2f - str w8,[x1],#4 - b 3f -2: - str w8,[x1],#-4 -3: - tbl v5.16b,{v5.16b},v7.16b - subs x6,x6,#1 - b.ne 1b - ret -.size _vpsm4_ex_set_key,.-_vpsm4_ex_set_key -.type _vpsm4_ex_enc_4blks,%function -.align 4 -_vpsm4_ex_enc_4blks: - AARCH64_VALID_CALL_TARGET - mov x10,x3 - mov w11,#8 -10: - ldp w7,w8,[x10],8 - dup v12.4s,w7 - dup v13.4s,w8 - - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor v14.16b,v6.16b,v7.16b - eor v12.16b,v5.16b,v12.16b - eor v12.16b,v14.16b,v12.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - mov v12.16b,v0.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - eor v4.16b,v4.16b,v12.16b - - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor v14.16b,v14.16b,v4.16b - eor v13.16b,v14.16b,v13.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - mov v13.16b,v0.16b - - // linear transformation - ushr v0.4s,v13.4s,32-2 - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v0.4s,v13.4s,2 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v0.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - ldp w7,w8,[x10],8 - eor v5.16b,v5.16b,v13.16b - - dup v12.4s,w7 - dup v13.4s,w8 - - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor v14.16b,v4.16b,v5.16b - eor v12.16b,v7.16b,v12.16b - eor v12.16b,v14.16b,v12.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - mov v12.16b,v0.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - eor v6.16b,v6.16b,v12.16b - - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor v14.16b,v14.16b,v6.16b - eor v13.16b,v14.16b,v13.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - mov v13.16b,v0.16b - - // linear transformation - ushr v0.4s,v13.4s,32-2 - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v0.4s,v13.4s,2 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v0.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - eor v7.16b,v7.16b,v13.16b - subs w11,w11,#1 - b.ne 10b -#ifndef __AARCH64EB__ - rev32 v3.16b,v4.16b -#else - mov v3.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v2.16b,v5.16b -#else - mov v2.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v1.16b,v6.16b -#else - mov v1.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v0.16b,v7.16b -#else - mov v0.16b,v7.16b -#endif - ret -.size _vpsm4_ex_enc_4blks,.-_vpsm4_ex_enc_4blks -.type _vpsm4_ex_enc_8blks,%function -.align 4 -_vpsm4_ex_enc_8blks: - AARCH64_VALID_CALL_TARGET - mov x10,x3 - mov w11,#8 -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - dup v12.4s,w7 - eor v14.16b,v6.16b,v7.16b - eor v15.16b,v10.16b,v11.16b - eor v0.16b,v5.16b,v12.16b - eor v1.16b,v9.16b,v12.16b - eor v12.16b,v14.16b,v0.16b - eor v13.16b,v15.16b,v1.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - tbl v1.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v28.16b}, v1.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - aese v0.16b,v25.16b - aese v1.16b,v25.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v30.16b}, v1.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - mov v12.16b,v0.16b - mov v13.16b,v1.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v25.4s,v13.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v25.4s,v13.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v25.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - eor v4.16b,v4.16b,v12.16b - eor v8.16b,v8.16b,v13.16b - - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - dup v13.4s,w8 - eor v14.16b,v14.16b,v4.16b - eor v15.16b,v15.16b,v8.16b - eor v12.16b,v14.16b,v13.16b - eor v13.16b,v15.16b,v13.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - tbl v1.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v28.16b}, v1.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - aese v0.16b,v25.16b - aese v1.16b,v25.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v30.16b}, v1.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - mov v12.16b,v0.16b - mov v13.16b,v1.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v25.4s,v13.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v25.4s,v13.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v25.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - ldp w7,w8,[x10],8 - eor v5.16b,v5.16b,v12.16b - eor v9.16b,v9.16b,v13.16b - - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - dup v12.4s,w7 - eor v14.16b,v4.16b,v5.16b - eor v15.16b,v8.16b,v9.16b - eor v0.16b,v7.16b,v12.16b - eor v1.16b,v11.16b,v12.16b - eor v12.16b,v14.16b,v0.16b - eor v13.16b,v15.16b,v1.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - tbl v1.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v28.16b}, v1.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - aese v0.16b,v25.16b - aese v1.16b,v25.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v30.16b}, v1.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - mov v12.16b,v0.16b - mov v13.16b,v1.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v25.4s,v13.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v25.4s,v13.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v25.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - eor v6.16b,v6.16b,v12.16b - eor v10.16b,v10.16b,v13.16b - - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - dup v13.4s,w8 - eor v14.16b,v14.16b,v6.16b - eor v15.16b,v15.16b,v10.16b - eor v12.16b,v14.16b,v13.16b - eor v13.16b,v15.16b,v13.16b - // optimize sbox using AESE instruction - tbl v0.16b, {v12.16b}, v26.16b - tbl v1.16b, {v13.16b}, v26.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v28.16b}, v1.16b - tbl v24.16b, {v27.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - eor v25.16b, v25.16b, v25.16b - aese v0.16b,v25.16b - aese v1.16b,v25.16b - ushr v24.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v0.16b, v0.16b, v24.16b - ushr v24.16b, v1.16b, 4 - and v1.16b, v1.16b, v31.16b - tbl v1.16b, {v30.16b}, v1.16b - tbl v24.16b, {v29.16b}, v24.16b - eor v1.16b, v1.16b, v24.16b - mov v12.16b,v0.16b - mov v13.16b,v1.16b - - // linear transformation - ushr v0.4s,v12.4s,32-2 - ushr v25.4s,v13.4s,32-2 - ushr v1.4s,v12.4s,32-10 - ushr v2.4s,v12.4s,32-18 - ushr v3.4s,v12.4s,32-24 - sli v0.4s,v12.4s,2 - sli v25.4s,v13.4s,2 - sli v1.4s,v12.4s,10 - sli v2.4s,v12.4s,18 - sli v3.4s,v12.4s,24 - eor v24.16b,v0.16b,v12.16b - eor v24.16b,v24.16b,v1.16b - eor v12.16b,v2.16b,v3.16b - eor v12.16b,v12.16b,v24.16b - ushr v1.4s,v13.4s,32-10 - ushr v2.4s,v13.4s,32-18 - ushr v3.4s,v13.4s,32-24 - sli v1.4s,v13.4s,10 - sli v2.4s,v13.4s,18 - sli v3.4s,v13.4s,24 - eor v24.16b,v25.16b,v13.16b - eor v24.16b,v24.16b,v1.16b - eor v13.16b,v2.16b,v3.16b - eor v13.16b,v13.16b,v24.16b - eor v7.16b,v7.16b,v12.16b - eor v11.16b,v11.16b,v13.16b - subs w11,w11,#1 - b.ne 10b -#ifndef __AARCH64EB__ - rev32 v3.16b,v4.16b -#else - mov v3.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v2.16b,v5.16b -#else - mov v2.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v1.16b,v6.16b -#else - mov v1.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v0.16b,v7.16b -#else - mov v0.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v8.16b -#else - mov v7.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v9.16b -#else - mov v6.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v10.16b -#else - mov v5.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v4.16b,v11.16b -#else - mov v4.16b,v11.16b -#endif - ret -.size _vpsm4_ex_enc_8blks,.-_vpsm4_ex_enc_8blks -.globl vpsm4_ex_set_encrypt_key -.type vpsm4_ex_set_encrypt_key,%function -.align 5 -vpsm4_ex_set_encrypt_key: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - mov w2,1 - bl _vpsm4_ex_set_key - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ex_set_encrypt_key,.-vpsm4_ex_set_encrypt_key -.globl vpsm4_ex_set_decrypt_key -.type vpsm4_ex_set_decrypt_key,%function -.align 5 -vpsm4_ex_set_decrypt_key: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - mov w2,0 - bl _vpsm4_ex_set_key - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ex_set_decrypt_key,.-vpsm4_ex_set_decrypt_key -.globl vpsm4_ex_encrypt -.type vpsm4_ex_encrypt,%function -.align 5 -vpsm4_ex_encrypt: - AARCH64_VALID_CALL_TARGET - ld1 {v4.4s},[x0] - ldr q26, .Lsbox_magic - ldr q27, .Lsbox_magic+16 - ldr q28, .Lsbox_magic+32 - ldr q29, .Lsbox_magic+48 - ldr q30, .Lsbox_magic+64 - ldr q31, .Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x3,x2 - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - ret -.size vpsm4_ex_encrypt,.-vpsm4_ex_encrypt -.globl vpsm4_ex_decrypt -.type vpsm4_ex_decrypt,%function -.align 5 -vpsm4_ex_decrypt: - AARCH64_VALID_CALL_TARGET - ld1 {v4.4s},[x0] - ldr q26, .Lsbox_magic - ldr q27, .Lsbox_magic+16 - ldr q28, .Lsbox_magic+32 - ldr q29, .Lsbox_magic+48 - ldr q30, .Lsbox_magic+64 - ldr q31, .Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x3,x2 - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - ret -.size vpsm4_ex_decrypt,.-vpsm4_ex_decrypt -.globl vpsm4_ex_ecb_encrypt -.type vpsm4_ex_ecb_encrypt,%function -.align 5 -vpsm4_ex_ecb_encrypt: - AARCH64_SIGN_LINK_REGISTER - // convert length into blocks - lsr x2,x2,4 - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] - ldr q26, .Lsbox_magic - ldr q27, .Lsbox_magic+16 - ldr q28, .Lsbox_magic+32 - ldr q29, .Lsbox_magic+48 - ldr q30, .Lsbox_magic+64 - ldr q31, .Lsbox_magic+80 -.Lecb_8_blocks_process: - cmp w2,#8 - b.lt .Lecb_4_blocks_process - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - bl _vpsm4_ex_enc_8blks - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st4 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.gt .Lecb_8_blocks_process - b 100f -.Lecb_4_blocks_process: - cmp w2,#4 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub w2,w2,#4 -1: - // process last block - cmp w2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - st1 {v4.4s},[x1] - b 100f -1: // process last 2 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[0],[x0],#16 - ld4 {v4.s,v5.s,v6.s,v7.s}[1],[x0],#16 - cmp w2,#2 - b.gt 1f -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1] - b 100f -1: // process last 3 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[2],[x0],#16 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[2],[x1] -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ex_ecb_encrypt,.-vpsm4_ex_ecb_encrypt -.globl vpsm4_ex_cbc_encrypt -.type vpsm4_ex_cbc_encrypt,%function -.align 5 -vpsm4_ex_cbc_encrypt: - AARCH64_VALID_CALL_TARGET - lsr x2,x2,4 - ldr q26, .Lsbox_magic - ldr q27, .Lsbox_magic+16 - ldr q28, .Lsbox_magic+32 - ldr q29, .Lsbox_magic+48 - ldr q30, .Lsbox_magic+64 - ldr q31, .Lsbox_magic+80 - cbz w5,.Ldec - ld1 {v3.4s},[x4] -.Lcbc_4_blocks_enc: - cmp w2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b,v4.16b,v3.16b -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 - eor v5.16b,v5.16b,v4.16b - mov x10,x3 - mov w11,#8 - mov w12,v5.s[0] - mov w13,v5.s[1] - mov w14,v5.s[2] - mov w15,v5.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v5.s[0],w15 - mov v5.s[1],w14 - mov v5.s[2],w13 - mov v5.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v6.16b,v6.16b,v5.16b - mov x10,x3 - mov w11,#8 - mov w12,v6.s[0] - mov w13,v6.s[1] - mov w14,v6.s[2] - mov w15,v6.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v6.s[0],w15 - mov v6.s[1],w14 - mov v6.s[2],w13 - mov v6.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - eor v7.16b,v7.16b,v6.16b - mov x10,x3 - mov w11,#8 - mov w12,v7.s[0] - mov w13,v7.s[1] - mov w14,v7.s[2] - mov w15,v7.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v7.s[0],w15 - mov v7.s[1],w14 - mov v7.s[2],w13 - mov v7.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - orr v3.16b,v7.16b,v7.16b - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#4 - b.ne .Lcbc_4_blocks_enc - b 2f -1: - subs w2,w2,#1 - b.lt 2f - ld1 {v4.4s},[x0],#16 - eor v3.16b,v3.16b,v4.16b -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - st1 {v3.4s},[x1],#16 - b 1b -2: - // save back IV - st1 {v3.4s},[x4] - ret - -.Ldec: - // decryption mode starts - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] -.Lcbc_8_blocks_dec: - cmp w2,#8 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0] - add x10,x0,#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - bl _vpsm4_ex_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - ld1 {v15.4s},[x4] - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - // note ivec1 and vtmpx[3] are reusing the same register - // care needs to be taken to avoid conflict - eor v0.16b,v0.16b,v15.16b - ld1 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - eor v1.16b,v1.16b,v8.16b - eor v2.16b,v2.16b,v9.16b - eor v3.16b,v3.16b,v10.16b - // save back IV - st1 {v15.4s}, [x4] - eor v4.16b,v4.16b,v11.16b - eor v5.16b,v5.16b,v12.16b - eor v6.16b,v6.16b,v13.16b - eor v7.16b,v7.16b,v14.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.gt .Lcbc_8_blocks_dec - b.eq 100f -1: - ld1 {v15.4s},[x4] -.Lcbc_4_blocks_dec: - cmp w2,#4 - b.lt 1f - ld4 {v4.4s,v5.4s,v6.4s,v7.4s},[x0] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - orr v15.16b,v7.16b,v7.16b - eor v2.16b,v2.16b,v5.16b - eor v3.16b,v3.16b,v6.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - subs w2,w2,#4 - b.gt .Lcbc_4_blocks_dec - // save back IV - st1 {v7.4s}, [x4] - b 100f -1: // last block - subs w2,w2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - // save back IV - st1 {v4.4s}, [x4] -#ifndef __AARCH64EB__ - rev32 v8.16b,v4.16b -#else - mov v8.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v8.s[0] - mov w13,v8.s[1] - mov w14,v8.s[2] - mov w15,v8.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v8.s[0],w15 - mov v8.s[1],w14 - mov v8.s[2],w13 - mov v8.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif - eor v8.16b,v8.16b,v15.16b - st1 {v8.4s},[x1],#16 - b 100f -1: // last two blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[0],[x0] - add x10,x0,#16 - ld4 {v4.s,v5.s,v6.s,v7.s}[1],[x10],#16 - subs w2,w2,1 - b.gt 1f -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - ld1 {v4.4s,v5.4s},[x0],#32 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save back IV - st1 {v5.4s}, [x4] - b 100f -1: // last 3 blocks - ld4 {v4.s,v5.s,v6.s,v7.s}[2],[x10] -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - bl _vpsm4_ex_enc_4blks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - eor v0.16b,v0.16b,v15.16b - eor v1.16b,v1.16b,v4.16b - eor v2.16b,v2.16b,v5.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save back IV - st1 {v6.4s}, [x4] -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ex_cbc_encrypt,.-vpsm4_ex_cbc_encrypt -.globl vpsm4_ex_ctr32_encrypt_blocks -.type vpsm4_ex_ctr32_encrypt_blocks,%function -.align 5 -vpsm4_ex_ctr32_encrypt_blocks: - AARCH64_VALID_CALL_TARGET - ld1 {v3.4s},[x4] -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ldr q26, .Lsbox_magic - ldr q27, .Lsbox_magic+16 - ldr q28, .Lsbox_magic+32 - ldr q29, .Lsbox_magic+48 - ldr q30, .Lsbox_magic+64 - ldr q31, .Lsbox_magic+80 - cmp w2,#1 - b.ne 1f - // fast processing for one single block without - // context saving overhead - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ld1 {v4.4s},[x0] - eor v4.16b,v4.16b,v3.16b - st1 {v4.4s},[x1] - ret -1: - AARCH64_SIGN_LINK_REGISTER - stp d8,d9,[sp,#-80]! - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] - stp x29,x30,[sp,#64] - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w5,v3.s[3] -.Lctr32_4_blocks_process: - cmp w2,#4 - b.lt 1f - dup v4.4s,w12 - dup v5.4s,w13 - dup v6.4s,w14 - mov v7.s[0],w5 - add w5,w5,#1 - mov v7.s[1],w5 - add w5,w5,#1 - mov v7.s[2],w5 - add w5,w5,#1 - mov v7.s[3],w5 - add w5,w5,#1 - cmp w2,#8 - b.ge .Lctr32_8_blocks_process - bl _vpsm4_ex_enc_4blks - ld4 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - subs w2,w2,#4 - b.ne .Lctr32_4_blocks_process - b 100f -.Lctr32_8_blocks_process: - dup v8.4s,w12 - dup v9.4s,w13 - dup v10.4s,w14 - mov v11.s[0],w5 - add w5,w5,#1 - mov v11.s[1],w5 - add w5,w5,#1 - mov v11.s[2],w5 - add w5,w5,#1 - mov v11.s[3],w5 - add w5,w5,#1 - bl _vpsm4_ex_enc_8blks - ld4 {v12.4s,v13.4s,v14.4s,v15.4s},[x0],#64 - ld4 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - eor v4.16b,v4.16b,v8.16b - eor v5.16b,v5.16b,v9.16b - eor v6.16b,v6.16b,v10.16b - eor v7.16b,v7.16b,v11.16b - st4 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st4 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs w2,w2,#8 - b.ne .Lctr32_4_blocks_process - b 100f -1: // last block processing - subs w2,w2,#1 - b.lt 100f - b.gt 1f - mov v3.s[0],w12 - mov v3.s[1],w13 - mov v3.s[2],w14 - mov v3.s[3],w5 - mov x10,x3 - mov w11,#8 - mov w12,v3.s[0] - mov w13,v3.s[1] - mov w14,v3.s[2] - mov w15,v3.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v3.s[0],w15 - mov v3.s[1],w14 - mov v3.s[2],w13 - mov v3.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v3.16b,v3.16b -#endif - ld1 {v4.4s},[x0] - eor v4.16b,v4.16b,v3.16b - st1 {v4.4s},[x1] - b 100f -1: // last 2 blocks processing - dup v4.4s,w12 - dup v5.4s,w13 - dup v6.4s,w14 - mov v7.s[0],w5 - add w5,w5,#1 - mov v7.s[1],w5 - subs w2,w2,#1 - b.ne 1f - bl _vpsm4_ex_enc_4blks - ld4 {v12.s,v13.s,v14.s,v15.s}[0],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[1],[x0],#16 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - b 100f -1: // last 3 blocks processing - add w5,w5,#1 - mov v7.s[2],w5 - bl _vpsm4_ex_enc_4blks - ld4 {v12.s,v13.s,v14.s,v15.s}[0],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[1],[x0],#16 - ld4 {v12.s,v13.s,v14.s,v15.s}[2],[x0],#16 - eor v0.16b,v0.16b,v12.16b - eor v1.16b,v1.16b,v13.16b - eor v2.16b,v2.16b,v14.16b - eor v3.16b,v3.16b,v15.16b - st4 {v0.s,v1.s,v2.s,v3.s}[0],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[1],[x1],#16 - st4 {v0.s,v1.s,v2.s,v3.s}[2],[x1],#16 -100: - ldp d10,d11,[sp,#16] - ldp d12,d13,[sp,#32] - ldp d14,d15,[sp,#48] - ldp x29,x30,[sp,#64] - ldp d8,d9,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ex_ctr32_encrypt_blocks,.-vpsm4_ex_ctr32_encrypt_blocks -.globl vpsm4_ex_xts_encrypt_gb -.type vpsm4_ex_xts_encrypt_gb,%function -.align 5 -vpsm4_ex_xts_encrypt_gb: - AARCH64_SIGN_LINK_REGISTER - stp x15, x16, [sp, #-0x10]! - stp x17, x18, [sp, #-0x10]! - stp x19, x20, [sp, #-0x10]! - stp x21, x22, [sp, #-0x10]! - stp x23, x24, [sp, #-0x10]! - stp x25, x26, [sp, #-0x10]! - stp x27, x28, [sp, #-0x10]! - stp x29, x30, [sp, #-0x10]! - stp d8, d9, [sp, #-0x10]! - stp d10, d11, [sp, #-0x10]! - stp d12, d13, [sp, #-0x10]! - stp d14, d15, [sp, #-0x10]! - mov x26,x3 - mov x27,x4 - mov w28,w6 - ld1 {v16.4s}, [x5] - mov x3,x27 - ldr q26, .Lsbox_magic - ldr q27, .Lsbox_magic+16 - ldr q28, .Lsbox_magic+32 - ldr q29, .Lsbox_magic+48 - ldr q30, .Lsbox_magic+64 - ldr q31, .Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v16.s[0] - mov w13,v16.s[1] - mov w14,v16.s[2] - mov w15,v16.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v16.s[0],w15 - mov v16.s[1],w14 - mov v16.s[2],w13 - mov v16.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x3,x26 - and x29,x2,#0x0F - // convert length into blocks - lsr x2,x2,4 - cmp x2,#1 - b.lt .return_gb - - cmp x29,0 - // If the encryption/decryption Length is N times of 16, - // the all blocks are encrypted/decrypted in .xts_encrypt_blocks_gb - b.eq .xts_encrypt_blocks_gb - - // If the encryption/decryption length is not N times of 16, - // the last two blocks are encrypted/decrypted in .last_2blks_tweak_gb or .only_2blks_tweak_gb - // the other blocks are encrypted/decrypted in .xts_encrypt_blocks_gb - subs x2,x2,#1 - b.eq .only_2blks_tweak_gb -.xts_encrypt_blocks_gb: - rbit v16.16b,v16.16b -#ifdef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x12,v16.d[0] - mov x13,v16.d[1] - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 -.Lxts_8_blocks_process_gb: - cmp x2,#8 - mov v16.d[0],x12 - mov v16.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov w7,0x87 - extr x9,x27,x27,#32 - extr x13,x27,x26,#63 - and w8,w7,w9,asr#31 - eor x12,x8,x26,lsl#1 - mov v17.d[0],x14 - mov v17.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov v18.d[0],x16 - mov v18.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v18.16b,v18.16b -#endif - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov v19.d[0],x18 - mov v19.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v19.16b,v19.16b -#endif - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov v20.d[0],x20 - mov v20.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v20.16b,v20.16b -#endif - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov v21.d[0],x22 - mov v21.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v21.16b,v21.16b -#endif - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov v22.d[0],x24 - mov v22.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v22.16b,v22.16b -#endif - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov v23.d[0],x26 - mov v23.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v23.16b,v23.16b -#endif - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 - b.lt .Lxts_4_blocks_process_gb - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - rbit v16.16b,v16.16b - rbit v17.16b,v17.16b - rbit v18.16b,v18.16b - rbit v19.16b,v19.16b - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b - eor v7.16b, v7.16b, v19.16b - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - rbit v20.16b,v20.16b - rbit v21.16b,v21.16b - rbit v22.16b,v22.16b - rbit v23.16b,v23.16b - eor v8.16b, v8.16b, v20.16b - eor v9.16b, v9.16b, v21.16b - eor v10.16b, v10.16b, v22.16b - eor v11.16b, v11.16b, v23.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - zip1 v0.4s,v8.4s,v9.4s - zip2 v1.4s,v8.4s,v9.4s - zip1 v2.4s,v10.4s,v11.4s - zip2 v3.4s,v10.4s,v11.4s - zip1 v8.2d,v0.2d,v2.2d - zip2 v9.2d,v0.2d,v2.2d - zip1 v10.2d,v1.2d,v3.2d - zip2 v11.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - eor v3.16b, v3.16b, v19.16b - eor v4.16b, v4.16b, v20.16b - eor v5.16b, v5.16b, v21.16b - eor v6.16b, v6.16b, v22.16b - eor v7.16b, v7.16b, v23.16b - - // save the last tweak - mov v25.16b,v23.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs x2,x2,#8 - b.gt .Lxts_8_blocks_process_gb - b 100f -.Lxts_4_blocks_process_gb: - cmp x2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - rbit v16.16b,v16.16b - rbit v17.16b,v17.16b - rbit v18.16b,v18.16b - rbit v19.16b,v19.16b - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b - eor v7.16b, v7.16b, v19.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - eor v3.16b, v3.16b, v19.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub x2,x2,#4 - mov v16.16b,v20.16b - mov v17.16b,v21.16b - mov v18.16b,v22.16b - // save the last tweak - mov v25.16b,v19.16b -1: - // process last block - cmp x2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - rbit v16.16b,v16.16b - eor v4.16b, v4.16b, v16.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v16.16b - st1 {v4.4s},[x1],#16 - // save the last tweak - mov v25.16b,v16.16b - b 100f -1: // process last 2 blocks - cmp x2,#2 - b.gt 1f - ld1 {v4.4s,v5.4s},[x0],#32 - rbit v16.16b,v16.16b - rbit v17.16b,v17.16b - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save the last tweak - mov v25.16b,v17.16b - b 100f -1: // process last 3 blocks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - rbit v16.16b,v16.16b - rbit v17.16b,v17.16b - rbit v18.16b,v18.16b - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save the last tweak - mov v25.16b,v18.16b -100: - cmp x29,0 - b.eq .return_gb - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is larger than 32 -.last_2blks_tweak_gb: -#ifdef __AARCH64EB__ - rev32 v25.16b,v25.16b -#endif - rbit v2.16b,v25.16b - ldr q0, .Lxts_magic - shl v17.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v17.16b, v17.16b, v1.16b - rbit v17.16b,v17.16b - rbit v2.16b,v17.16b - ldr q0, .Lxts_magic - shl v18.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v18.16b, v18.16b, v1.16b - rbit v18.16b,v18.16b - b .check_dec_gb - - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is equal to 32, who only need two tweaks -.only_2blks_tweak_gb: - mov v17.16b,v16.16b -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif - rbit v2.16b,v17.16b - ldr q0, .Lxts_magic - shl v18.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v18.16b, v18.16b, v1.16b - rbit v18.16b,v18.16b - b .check_dec_gb - - -// Determine whether encryption or decryption is required. -// The last two tweaks need to be swapped for decryption. -.check_dec_gb: - // encryption:1 decryption:0 - cmp w28,1 - b.eq .process_last_2blks_gb - mov v0.16B,v17.16b - mov v17.16B,v18.16b - mov v18.16B,v0.16b - -.process_last_2blks_gb: -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif -#ifdef __AARCH64EB__ - rev32 v18.16b,v18.16b -#endif - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v17.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v17.16b - st1 {v4.4s},[x1],#16 - - sub x26,x1,16 -.loop_gb: - subs x29,x29,1 - ldrb w7,[x26,x29] - ldrb w8,[x0,x29] - strb w8,[x26,x29] - strb w7,[x1,x29] - b.gt .loop_gb - ld1 {v4.4s}, [x26] - eor v4.16b, v4.16b, v18.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v18.16b - st1 {v4.4s}, [x26] -.return_gb: - ldp d14, d15, [sp], #0x10 - ldp d12, d13, [sp], #0x10 - ldp d10, d11, [sp], #0x10 - ldp d8, d9, [sp], #0x10 - ldp x29, x30, [sp], #0x10 - ldp x27, x28, [sp], #0x10 - ldp x25, x26, [sp], #0x10 - ldp x23, x24, [sp], #0x10 - ldp x21, x22, [sp], #0x10 - ldp x19, x20, [sp], #0x10 - ldp x17, x18, [sp], #0x10 - ldp x15, x16, [sp], #0x10 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ex_xts_encrypt_gb,.-vpsm4_ex_xts_encrypt_gb -.globl vpsm4_ex_xts_encrypt -.type vpsm4_ex_xts_encrypt,%function -.align 5 -vpsm4_ex_xts_encrypt: - AARCH64_SIGN_LINK_REGISTER - stp x15, x16, [sp, #-0x10]! - stp x17, x18, [sp, #-0x10]! - stp x19, x20, [sp, #-0x10]! - stp x21, x22, [sp, #-0x10]! - stp x23, x24, [sp, #-0x10]! - stp x25, x26, [sp, #-0x10]! - stp x27, x28, [sp, #-0x10]! - stp x29, x30, [sp, #-0x10]! - stp d8, d9, [sp, #-0x10]! - stp d10, d11, [sp, #-0x10]! - stp d12, d13, [sp, #-0x10]! - stp d14, d15, [sp, #-0x10]! - mov x26,x3 - mov x27,x4 - mov w28,w6 - ld1 {v16.4s}, [x5] - mov x3,x27 - ldr q26, .Lsbox_magic - ldr q27, .Lsbox_magic+16 - ldr q28, .Lsbox_magic+32 - ldr q29, .Lsbox_magic+48 - ldr q30, .Lsbox_magic+64 - ldr q31, .Lsbox_magic+80 -#ifndef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v16.s[0] - mov w13,v16.s[1] - mov w14,v16.s[2] - mov w15,v16.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v16.s[0],w15 - mov v16.s[1],w14 - mov v16.s[2],w13 - mov v16.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x3,x26 - and x29,x2,#0x0F - // convert length into blocks - lsr x2,x2,4 - cmp x2,#1 - b.lt .return - - cmp x29,0 - // If the encryption/decryption Length is N times of 16, - // the all blocks are encrypted/decrypted in .xts_encrypt_blocks - b.eq .xts_encrypt_blocks - - // If the encryption/decryption length is not N times of 16, - // the last two blocks are encrypted/decrypted in .last_2blks_tweak or .only_2blks_tweak - // the other blocks are encrypted/decrypted in .xts_encrypt_blocks - subs x2,x2,#1 - b.eq .only_2blks_tweak -.xts_encrypt_blocks: -#ifdef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov x12,v16.d[0] - mov x13,v16.d[1] - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 -.Lxts_8_blocks_process: - cmp x2,#8 - mov v16.d[0],x12 - mov v16.d[1],x13 -#ifdef __AARCH64EB__ - rev32 v16.16b,v16.16b -#endif - mov w7,0x87 - extr x9,x27,x27,#32 - extr x13,x27,x26,#63 - and w8,w7,w9,asr#31 - eor x12,x8,x26,lsl#1 - mov v17.d[0],x14 - mov v17.d[1],x15 -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif - mov w7,0x87 - extr x9,x13,x13,#32 - extr x15,x13,x12,#63 - and w8,w7,w9,asr#31 - eor x14,x8,x12,lsl#1 - mov v18.d[0],x16 - mov v18.d[1],x17 -#ifdef __AARCH64EB__ - rev32 v18.16b,v18.16b -#endif - mov w7,0x87 - extr x9,x15,x15,#32 - extr x17,x15,x14,#63 - and w8,w7,w9,asr#31 - eor x16,x8,x14,lsl#1 - mov v19.d[0],x18 - mov v19.d[1],x19 -#ifdef __AARCH64EB__ - rev32 v19.16b,v19.16b -#endif - mov w7,0x87 - extr x9,x17,x17,#32 - extr x19,x17,x16,#63 - and w8,w7,w9,asr#31 - eor x18,x8,x16,lsl#1 - mov v20.d[0],x20 - mov v20.d[1],x21 -#ifdef __AARCH64EB__ - rev32 v20.16b,v20.16b -#endif - mov w7,0x87 - extr x9,x19,x19,#32 - extr x21,x19,x18,#63 - and w8,w7,w9,asr#31 - eor x20,x8,x18,lsl#1 - mov v21.d[0],x22 - mov v21.d[1],x23 -#ifdef __AARCH64EB__ - rev32 v21.16b,v21.16b -#endif - mov w7,0x87 - extr x9,x21,x21,#32 - extr x23,x21,x20,#63 - and w8,w7,w9,asr#31 - eor x22,x8,x20,lsl#1 - mov v22.d[0],x24 - mov v22.d[1],x25 -#ifdef __AARCH64EB__ - rev32 v22.16b,v22.16b -#endif - mov w7,0x87 - extr x9,x23,x23,#32 - extr x25,x23,x22,#63 - and w8,w7,w9,asr#31 - eor x24,x8,x22,lsl#1 - mov v23.d[0],x26 - mov v23.d[1],x27 -#ifdef __AARCH64EB__ - rev32 v23.16b,v23.16b -#endif - mov w7,0x87 - extr x9,x25,x25,#32 - extr x27,x25,x24,#63 - and w8,w7,w9,asr#31 - eor x26,x8,x24,lsl#1 - b.lt .Lxts_4_blocks_process - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b - eor v7.16b, v7.16b, v19.16b - ld1 {v8.4s,v9.4s,v10.4s,v11.4s},[x0],#64 - eor v8.16b, v8.16b, v20.16b - eor v9.16b, v9.16b, v21.16b - eor v10.16b, v10.16b, v22.16b - eor v11.16b, v11.16b, v23.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif -#ifndef __AARCH64EB__ - rev32 v8.16b,v8.16b -#endif -#ifndef __AARCH64EB__ - rev32 v9.16b,v9.16b -#endif -#ifndef __AARCH64EB__ - rev32 v10.16b,v10.16b -#endif -#ifndef __AARCH64EB__ - rev32 v11.16b,v11.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - zip1 v0.4s,v8.4s,v9.4s - zip2 v1.4s,v8.4s,v9.4s - zip1 v2.4s,v10.4s,v11.4s - zip2 v3.4s,v10.4s,v11.4s - zip1 v8.2d,v0.2d,v2.2d - zip2 v9.2d,v0.2d,v2.2d - zip1 v10.2d,v1.2d,v3.2d - zip2 v11.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_8blks - zip1 v8.4s,v0.4s,v1.4s - zip2 v9.4s,v0.4s,v1.4s - zip1 v10.4s,v2.4s,v3.4s - zip2 v11.4s,v2.4s,v3.4s - zip1 v0.2d,v8.2d,v10.2d - zip2 v1.2d,v8.2d,v10.2d - zip1 v2.2d,v9.2d,v11.2d - zip2 v3.2d,v9.2d,v11.2d - zip1 v8.4s,v4.4s,v5.4s - zip2 v9.4s,v4.4s,v5.4s - zip1 v10.4s,v6.4s,v7.4s - zip2 v11.4s,v6.4s,v7.4s - zip1 v4.2d,v8.2d,v10.2d - zip2 v5.2d,v8.2d,v10.2d - zip1 v6.2d,v9.2d,v11.2d - zip2 v7.2d,v9.2d,v11.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - eor v3.16b, v3.16b, v19.16b - eor v4.16b, v4.16b, v20.16b - eor v5.16b, v5.16b, v21.16b - eor v6.16b, v6.16b, v22.16b - eor v7.16b, v7.16b, v23.16b - - // save the last tweak - mov v25.16b,v23.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - st1 {v4.4s,v5.4s,v6.4s,v7.4s},[x1],#64 - subs x2,x2,#8 - b.gt .Lxts_8_blocks_process - b 100f -.Lxts_4_blocks_process: - cmp x2,#4 - b.lt 1f - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x0],#64 - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b - eor v7.16b, v7.16b, v19.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif -#ifndef __AARCH64EB__ - rev32 v7.16b,v7.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - eor v3.16b, v3.16b, v19.16b - st1 {v0.4s,v1.4s,v2.4s,v3.4s},[x1],#64 - sub x2,x2,#4 - mov v16.16b,v20.16b - mov v17.16b,v21.16b - mov v18.16b,v22.16b - // save the last tweak - mov v25.16b,v19.16b -1: - // process last block - cmp x2,#1 - b.lt 100f - b.gt 1f - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v16.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v16.16b - st1 {v4.4s},[x1],#16 - // save the last tweak - mov v25.16b,v16.16b - b 100f -1: // process last 2 blocks - cmp x2,#2 - b.gt 1f - ld1 {v4.4s,v5.4s},[x0],#32 - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - st1 {v0.4s,v1.4s},[x1],#32 - // save the last tweak - mov v25.16b,v17.16b - b 100f -1: // process last 3 blocks - ld1 {v4.4s,v5.4s,v6.4s},[x0],#48 - eor v4.16b, v4.16b, v16.16b - eor v5.16b, v5.16b, v17.16b - eor v6.16b, v6.16b, v18.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif -#ifndef __AARCH64EB__ - rev32 v5.16b,v5.16b -#endif -#ifndef __AARCH64EB__ - rev32 v6.16b,v6.16b -#endif - zip1 v0.4s,v4.4s,v5.4s - zip2 v1.4s,v4.4s,v5.4s - zip1 v2.4s,v6.4s,v7.4s - zip2 v3.4s,v6.4s,v7.4s - zip1 v4.2d,v0.2d,v2.2d - zip2 v5.2d,v0.2d,v2.2d - zip1 v6.2d,v1.2d,v3.2d - zip2 v7.2d,v1.2d,v3.2d - bl _vpsm4_ex_enc_4blks - zip1 v4.4s,v0.4s,v1.4s - zip2 v5.4s,v0.4s,v1.4s - zip1 v6.4s,v2.4s,v3.4s - zip2 v7.4s,v2.4s,v3.4s - zip1 v0.2d,v4.2d,v6.2d - zip2 v1.2d,v4.2d,v6.2d - zip1 v2.2d,v5.2d,v7.2d - zip2 v3.2d,v5.2d,v7.2d - eor v0.16b, v0.16b, v16.16b - eor v1.16b, v1.16b, v17.16b - eor v2.16b, v2.16b, v18.16b - st1 {v0.4s,v1.4s,v2.4s},[x1],#48 - // save the last tweak - mov v25.16b,v18.16b -100: - cmp x29,0 - b.eq .return - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is larger than 32 -.last_2blks_tweak: -#ifdef __AARCH64EB__ - rev32 v25.16b,v25.16b -#endif - mov v2.16b,v25.16b - ldr q0, .Lxts_magic - shl v17.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v17.16b, v17.16b, v1.16b - mov v2.16b,v17.16b - ldr q0, .Lxts_magic - shl v18.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v18.16b, v18.16b, v1.16b - b .check_dec - - -// This branch calculates the last two tweaks, -// while the encryption/decryption length is equal to 32, who only need two tweaks -.only_2blks_tweak: - mov v17.16b,v16.16b -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif - mov v2.16b,v17.16b - ldr q0, .Lxts_magic - shl v18.16b, v2.16b, #1 - ext v1.16b, v2.16b, v2.16b,#15 - ushr v1.16b, v1.16b, #7 - mul v1.16b, v1.16b, v0.16b - eor v18.16b, v18.16b, v1.16b - b .check_dec - - -// Determine whether encryption or decryption is required. -// The last two tweaks need to be swapped for decryption. -.check_dec: - // encryption:1 decryption:0 - cmp w28,1 - b.eq .process_last_2blks - mov v0.16B,v17.16b - mov v17.16B,v18.16b - mov v18.16B,v0.16b - -.process_last_2blks: -#ifdef __AARCH64EB__ - rev32 v17.16b,v17.16b -#endif -#ifdef __AARCH64EB__ - rev32 v18.16b,v18.16b -#endif - ld1 {v4.4s},[x0],#16 - eor v4.16b, v4.16b, v17.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v17.16b - st1 {v4.4s},[x1],#16 - - sub x26,x1,16 -.loop: - subs x29,x29,1 - ldrb w7,[x26,x29] - ldrb w8,[x0,x29] - strb w8,[x26,x29] - strb w7,[x1,x29] - b.gt .loop - ld1 {v4.4s}, [x26] - eor v4.16b, v4.16b, v18.16b -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - mov x10,x3 - mov w11,#8 - mov w12,v4.s[0] - mov w13,v4.s[1] - mov w14,v4.s[2] - mov w15,v4.s[3] -10: - ldp w7,w8,[x10],8 - // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) - eor w6,w14,w15 - eor w9,w7,w13 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w12,w12,w6 - // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) - eor w6,w14,w15 - eor w9,w12,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - ldp w7,w8,[x10],8 - eor w13,w13,w6 - // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) - eor w6,w12,w13 - eor w9,w7,w15 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w14,w14,w6 - // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) - eor w6,w12,w13 - eor w9,w14,w8 - eor w6,w6,w9 - mov v3.s[0],w6 - // optimize sbox using AESE instruction - tbl v0.16b, {v3.16b}, v26.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v28.16b}, v0.16b - tbl v2.16b, {v27.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v1.16b - aese v0.16b,v1.16b - ushr v2.16b, v0.16b, 4 - and v0.16b, v0.16b, v31.16b - tbl v0.16b, {v30.16b}, v0.16b - tbl v2.16b, {v29.16b}, v2.16b - eor v0.16b, v0.16b, v2.16b - - mov w7,v0.s[0] - eor w6,w7,w7,ror #32-2 - eor w6,w6,w7,ror #32-10 - eor w6,w6,w7,ror #32-18 - eor w6,w6,w7,ror #32-24 - eor w15,w15,w6 - subs w11,w11,#1 - b.ne 10b - mov v4.s[0],w15 - mov v4.s[1],w14 - mov v4.s[2],w13 - mov v4.s[3],w12 -#ifndef __AARCH64EB__ - rev32 v4.16b,v4.16b -#endif - eor v4.16b, v4.16b, v18.16b - st1 {v4.4s}, [x26] -.return: - ldp d14, d15, [sp], #0x10 - ldp d12, d13, [sp], #0x10 - ldp d10, d11, [sp], #0x10 - ldp d8, d9, [sp], #0x10 - ldp x29, x30, [sp], #0x10 - ldp x27, x28, [sp], #0x10 - ldp x25, x26, [sp], #0x10 - ldp x23, x24, [sp], #0x10 - ldp x21, x22, [sp], #0x10 - ldp x19, x20, [sp], #0x10 - ldp x17, x18, [sp], #0x10 - ldp x15, x16, [sp], #0x10 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size vpsm4_ex_xts_encrypt,.-vpsm4_ex_xts_encrypt diff --git a/openssl/src/crypto/sm4/gen/linux_riscv64/sm4-riscv64-zvksed.s b/openssl/src/crypto/sm4/gen/linux_riscv64/sm4-riscv64-zvksed.s deleted file mode 100644 index c353c27e1..000000000 --- a/openssl/src/crypto/sm4/gen/linux_riscv64/sm4-riscv64-zvksed.s +++ /dev/null @@ -1,188 +0,0 @@ -.text -.p2align 3 -.globl rv64i_zvksed_sm4_set_encrypt_key -.type rv64i_zvksed_sm4_set_encrypt_key,@function -rv64i_zvksed_sm4_set_encrypt_key: - .word 0xc1027057 - - # Load the user key - .word 33906823 - .word 1242865879 - - # Load the FK. - la t0, FK - .word 33743111 - - # Generate round keys. - .word 772866263 - .word 2249204215 # rk[0:3] - .word 2251334263 # rk[4:7] - .word 2252415735 # rk[8:11] - .word 2253497207 # rk[12:15] - .word 2254578679 # rk[16:19] - .word 2255660151 # rk[20:23] - .word 2256741623 # rk[24:27] - .word 2257823095 # rk[28:31] - - # Store round keys - .word 33939879 # rk[0:3] - addi a1, a1, 16 - .word 33940007 # rk[4:7] - addi a1, a1, 16 - .word 33940135 # rk[8:11] - addi a1, a1, 16 - .word 33940263 # rk[12:15] - addi a1, a1, 16 - .word 33940391 # rk[16:19] - addi a1, a1, 16 - .word 33940519 # rk[20:23] - addi a1, a1, 16 - .word 33940647 # rk[24:27] - addi a1, a1, 16 - .word 33940775 # rk[28:31] - - li a0, 1 - ret -.size rv64i_zvksed_sm4_set_encrypt_key,.-rv64i_zvksed_sm4_set_encrypt_key -.p2align 3 -.globl rv64i_zvksed_sm4_set_decrypt_key -.type rv64i_zvksed_sm4_set_decrypt_key,@function -rv64i_zvksed_sm4_set_decrypt_key: - .word 0xc1027057 - - # Load the user key - .word 33906823 - .word 1242865879 - - # Load the FK. - la t0, FK - .word 33743111 - - # Generate round keys. - .word 772866263 - .word 2249204215 # rk[0:3] - .word 2251334263 # rk[4:7] - .word 2252415735 # rk[8:11] - .word 2253497207 # rk[12:15] - .word 2254578679 # rk[16:19] - .word 2255660151 # rk[20:23] - .word 2256741623 # rk[24:27] - .word 2257823095 # rk[28:31] - - # Store round keys in reverse order - addi a1, a1, 12 - li t1, -4 - .word 174449959 # rk[31:28] - addi a1, a1, 16 - .word 174449831 # rk[27:24] - addi a1, a1, 16 - .word 174449703 # rk[23:20] - addi a1, a1, 16 - .word 174449575 # rk[19:16] - addi a1, a1, 16 - .word 174449447 # rk[15:12] - addi a1, a1, 16 - .word 174449319 # rk[11:8] - addi a1, a1, 16 - .word 174449191 # rk[7:4] - addi a1, a1, 16 - .word 174449063 # rk[3:0] - - li a0, 1 - ret -.size rv64i_zvksed_sm4_set_decrypt_key,.-rv64i_zvksed_sm4_set_decrypt_key -.p2align 3 -.globl rv64i_zvksed_sm4_encrypt -.type rv64i_zvksed_sm4_encrypt,@function -rv64i_zvksed_sm4_encrypt: - .word 0xc1027057 - - # Order of elements was adjusted in set_encrypt_key() - .word 33972487 # rk[0:3] - addi a2, a2, 16 - .word 33972615 # rk[4:7] - addi a2, a2, 16 - .word 33972743 # rk[8:11] - addi a2, a2, 16 - .word 33972871 # rk[12:15] - addi a2, a2, 16 - .word 33972999 # rk[16:19] - addi a2, a2, 16 - .word 33973127 # rk[20:23] - addi a2, a2, 16 - .word 33973255 # rk[24:27] - addi a2, a2, 16 - .word 33973383 # rk[28:31] - - # Load input data - .word 33906823 - .word 1242865879 - - # Encrypt with all keys - .word 2787647735 - .word 2788696311 - .word 2789744887 - .word 2790793463 - .word 2791842039 - .word 2792890615 - .word 2793939191 - .word 2794987767 - - # Save the ciphertext (in reverse element order) - .word 1242865879 - li t0, -4 - addi a1, a1, 12 - .word 173400231 - - ret -.size rv64i_zvksed_sm4_encrypt,.-rv64i_zvksed_sm4_encrypt -.p2align 3 -.globl rv64i_zvksed_sm4_decrypt -.type rv64i_zvksed_sm4_decrypt,@function -rv64i_zvksed_sm4_decrypt: - .word 0xc1027057 - - # Order of elements was adjusted in set_decrypt_key() - .word 33973383 # rk[31:28] - addi a2, a2, 16 - .word 33973255 # rk[27:24] - addi a2, a2, 16 - .word 33973127 # rk[23:20] - addi a2, a2, 16 - .word 33972999 # rk[19:16] - addi a2, a2, 16 - .word 33972871 # rk[15:11] - addi a2, a2, 16 - .word 33972743 # rk[11:8] - addi a2, a2, 16 - .word 33972615 # rk[7:4] - addi a2, a2, 16 - .word 33972487 # rk[3:0] - - # Load input data - .word 33906823 - .word 1242865879 - - # Encrypt with all keys - .word 2794987767 - .word 2793939191 - .word 2792890615 - .word 2791842039 - .word 2790793463 - .word 2789744887 - .word 2788696311 - .word 2787647735 - - # Save the ciphertext (in reverse element order) - .word 1242865879 - li t0, -4 - addi a1, a1, 12 - .word 173400231 - - ret -.size rv64i_zvksed_sm4_decrypt,.-rv64i_zvksed_sm4_decrypt -# Family Key (little-endian 32-bit chunks) -.p2align 3 -FK: - .word 0xA3B1BAC6, 0x56AA3350, 0x677D9197, 0xB27022DC -.size FK,.-FK diff --git a/openssl/src/crypto/sm4/sm4.c b/openssl/src/crypto/sm4/sm4.c index 4c58c25fa..f9d659f06 100644 --- a/openssl/src/crypto/sm4/sm4.c +++ b/openssl/src/crypto/sm4/sm4.c @@ -241,7 +241,7 @@ static ossl_inline void store_u32_be(uint32_t v, uint8_t *b) b[3] = (uint8_t)(v); } -static ossl_inline uint32_t SM4_T_non_lin_sub(uint32_t X) +static ossl_inline uint32_t SM4_T_slow(uint32_t X) { uint32_t t = 0; @@ -250,13 +250,6 @@ static ossl_inline uint32_t SM4_T_non_lin_sub(uint32_t X) t |= ((uint32_t)SM4_S[(uint8_t)(X >> 8)]) << 8; t |= SM4_S[(uint8_t)X]; - return t; -} - -static ossl_inline uint32_t SM4_T_slow(uint32_t X) -{ - uint32_t t = SM4_T_non_lin_sub(X); - /* * L linear transform */ @@ -271,13 +264,6 @@ static ossl_inline uint32_t SM4_T(uint32_t X) SM4_SBOX_T3[(uint8_t)X]; } -static ossl_inline uint32_t SM4_key_sub(uint32_t X) -{ - uint32_t t = SM4_T_non_lin_sub(X); - - return t ^ rotl(t, 13) ^ rotl(t, 23); -} - int ossl_sm4_set_key(const uint8_t *key, SM4_KEY *ks) { /* @@ -308,15 +294,18 @@ int ossl_sm4_set_key(const uint8_t *key, SM4_KEY *ks) K[2] = load_u32_be(key, 2) ^ FK[2]; K[3] = load_u32_be(key, 3) ^ FK[3]; - for (i = 0; i < SM4_KEY_SCHEDULE; i = i + 4) { - K[0] ^= SM4_key_sub(K[1] ^ K[2] ^ K[3] ^ CK[i]); - K[1] ^= SM4_key_sub(K[2] ^ K[3] ^ K[0] ^ CK[i + 1]); - K[2] ^= SM4_key_sub(K[3] ^ K[0] ^ K[1] ^ CK[i + 2]); - K[3] ^= SM4_key_sub(K[0] ^ K[1] ^ K[2] ^ CK[i + 3]); - ks->rk[i ] = K[0]; - ks->rk[i + 1] = K[1]; - ks->rk[i + 2] = K[2]; - ks->rk[i + 3] = K[3]; + for (i = 0; i != SM4_KEY_SCHEDULE; ++i) { + uint32_t X = K[(i + 1) % 4] ^ K[(i + 2) % 4] ^ K[(i + 3) % 4] ^ CK[i]; + uint32_t t = 0; + + t |= ((uint32_t)SM4_S[(uint8_t)(X >> 24)]) << 24; + t |= ((uint32_t)SM4_S[(uint8_t)(X >> 16)]) << 16; + t |= ((uint32_t)SM4_S[(uint8_t)(X >> 8)]) << 8; + t |= SM4_S[(uint8_t)X]; + + t = t ^ rotl(t, 13) ^ rotl(t, 23); + K[i % 4] ^= t; + ks->rk[i] = K[i % 4]; } return 1; diff --git a/openssl/src/crypto/sparcv9cap.c b/openssl/src/crypto/sparcv9cap.c deleted file mode 100644 index 53c0c3e04..000000000 --- a/openssl/src/crypto/sparcv9cap.c +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "internal/cryptlib.h" -#include "crypto/sparc_arch.h" - -#if defined(__GNUC__) && defined(__linux) -__attribute__ ((visibility("hidden"))) -#endif -unsigned int OPENSSL_sparcv9cap_P[2] = { SPARCV9_TICK_PRIVILEGED, 0 }; - -unsigned long _sparcv9_rdtick(void); -void _sparcv9_vis1_probe(void); -unsigned long _sparcv9_vis1_instrument(void); -void _sparcv9_vis2_probe(void); -void _sparcv9_fmadd_probe(void); -unsigned long _sparcv9_rdcfr(void); -void _sparcv9_vis3_probe(void); -void _sparcv9_fjaesx_probe(void); -unsigned long _sparcv9_random(void); -size_t _sparcv9_vis1_instrument_bus(unsigned int *, size_t); -size_t _sparcv9_vis1_instrument_bus2(unsigned int *, size_t, size_t); - -uint32_t OPENSSL_rdtsc(void) -{ - if (OPENSSL_sparcv9cap_P[0] & SPARCV9_TICK_PRIVILEGED) -#if defined(__sun) && defined(__SVR4) - return gethrtime(); -#else - return 0; -#endif - else - return _sparcv9_rdtick(); -} - -size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt) -{ - if ((OPENSSL_sparcv9cap_P[0] & (SPARCV9_TICK_PRIVILEGED | SPARCV9_BLK)) == - SPARCV9_BLK) - return _sparcv9_vis1_instrument_bus(out, cnt); - else - return 0; -} - -size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max) -{ - if ((OPENSSL_sparcv9cap_P[0] & (SPARCV9_TICK_PRIVILEGED | SPARCV9_BLK)) == - SPARCV9_BLK) - return _sparcv9_vis1_instrument_bus2(out, cnt, max); - else - return 0; -} - -static sigjmp_buf common_jmp; -static void common_handler(int sig) -{ - siglongjmp(common_jmp, sig); -} - -#if defined(__sun) && defined(__SVR4) -# if defined(__GNUC__) && __GNUC__>=2 -extern unsigned int getisax(unsigned int vec[], unsigned int sz) __attribute__ ((weak)); -# elif defined(__SUNPRO_C) -#pragma weak getisax -extern unsigned int getisax(unsigned int vec[], unsigned int sz); -# else -static unsigned int (*getisax) (unsigned int vec[], unsigned int sz) = NULL; -# endif -#endif - -void OPENSSL_cpuid_setup(void) -{ - char *e; - struct sigaction common_act, ill_oact, bus_oact; - sigset_t all_masked, oset; - static int trigger = 0; - - if (trigger) - return; - trigger = 1; - - if ((e = getenv("OPENSSL_sparcv9cap"))) { - OPENSSL_sparcv9cap_P[0] = strtoul(e, NULL, 0); - if ((e = strchr(e, ':'))) - OPENSSL_sparcv9cap_P[1] = strtoul(e + 1, NULL, 0); - return; - } - -#if defined(__sun) && defined(__SVR4) - if (getisax != NULL) { - unsigned int vec[2] = { 0, 0 }; - - if (getisax (vec,2)) { - if (vec[0]&0x00020) OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS1; - if (vec[0]&0x00040) OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS2; - if (vec[0]&0x00080) OPENSSL_sparcv9cap_P[0] |= SPARCV9_BLK; - if (vec[0]&0x00100) OPENSSL_sparcv9cap_P[0] |= SPARCV9_FMADD; - if (vec[0]&0x00400) OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS3; - if (vec[0]&0x01000) OPENSSL_sparcv9cap_P[0] |= SPARCV9_FJHPCACE; - if (vec[0]&0x02000) OPENSSL_sparcv9cap_P[0] |= SPARCV9_FJDESX; - if (vec[0]&0x08000) OPENSSL_sparcv9cap_P[0] |= SPARCV9_IMA; - if (vec[0]&0x10000) OPENSSL_sparcv9cap_P[0] |= SPARCV9_FJAESX; - if (vec[1]&0x00008) OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS4; - - /* reconstruct %cfr copy */ - OPENSSL_sparcv9cap_P[1] = (vec[0]>>17)&0x3ff; - OPENSSL_sparcv9cap_P[1] |= (OPENSSL_sparcv9cap_P[1]&CFR_MONTMUL)<<1; - if (vec[0]&0x20000000) OPENSSL_sparcv9cap_P[1] |= CFR_CRC32C; - if (vec[1]&0x00000020) OPENSSL_sparcv9cap_P[1] |= CFR_XMPMUL; - if (vec[1]&0x00000040) - OPENSSL_sparcv9cap_P[1] |= CFR_XMONTMUL|CFR_XMONTSQR; - - /* Some heuristics */ - /* all known VIS2-capable CPUs have unprivileged tick counter */ - if (OPENSSL_sparcv9cap_P[0]&SPARCV9_VIS2) - OPENSSL_sparcv9cap_P[0] &= ~SPARCV9_TICK_PRIVILEGED; - - OPENSSL_sparcv9cap_P[0] |= SPARCV9_PREFER_FPU; - - /* detect UltraSPARC-Tx, see sparccpud.S for details... */ - if ((OPENSSL_sparcv9cap_P[0]&SPARCV9_VIS1) && - _sparcv9_vis1_instrument() >= 12) - OPENSSL_sparcv9cap_P[0] &= ~(SPARCV9_VIS1 | SPARCV9_PREFER_FPU); - } - - if (sizeof(size_t) == 8) - OPENSSL_sparcv9cap_P[0] |= SPARCV9_64BIT_STACK; - - return; - } -#endif - - /* Initial value, fits UltraSPARC-I&II... */ - OPENSSL_sparcv9cap_P[0] = SPARCV9_PREFER_FPU | SPARCV9_TICK_PRIVILEGED; - - sigfillset(&all_masked); - sigdelset(&all_masked, SIGILL); - sigdelset(&all_masked, SIGTRAP); -# ifdef SIGEMT - sigdelset(&all_masked, SIGEMT); -# endif - sigdelset(&all_masked, SIGFPE); - sigdelset(&all_masked, SIGBUS); - sigdelset(&all_masked, SIGSEGV); - sigprocmask(SIG_SETMASK, &all_masked, &oset); - - memset(&common_act, 0, sizeof(common_act)); - common_act.sa_handler = common_handler; - common_act.sa_mask = all_masked; - - sigaction(SIGILL, &common_act, &ill_oact); - sigaction(SIGBUS, &common_act, &bus_oact); /* T1 fails 16-bit ldda [on - * Linux] */ - - if (sigsetjmp(common_jmp, 1) == 0) { - _sparcv9_rdtick(); - OPENSSL_sparcv9cap_P[0] &= ~SPARCV9_TICK_PRIVILEGED; - } - - if (sigsetjmp(common_jmp, 1) == 0) { - _sparcv9_vis1_probe(); - OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS1 | SPARCV9_BLK; - /* detect UltraSPARC-Tx, see sparccpud.S for details... */ - if (_sparcv9_vis1_instrument() >= 12) - OPENSSL_sparcv9cap_P[0] &= ~(SPARCV9_VIS1 | SPARCV9_PREFER_FPU); - else { - _sparcv9_vis2_probe(); - OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS2; - } - } - - if (sigsetjmp(common_jmp, 1) == 0) { - _sparcv9_fmadd_probe(); - OPENSSL_sparcv9cap_P[0] |= SPARCV9_FMADD; - } - - /* - * VIS3 flag is tested independently from VIS1, unlike VIS2 that is, - * because VIS3 defines even integer instructions. - */ - if (sigsetjmp(common_jmp, 1) == 0) { - _sparcv9_vis3_probe(); - OPENSSL_sparcv9cap_P[0] |= SPARCV9_VIS3; - } - - if (sigsetjmp(common_jmp, 1) == 0) { - _sparcv9_fjaesx_probe(); - OPENSSL_sparcv9cap_P[0] |= SPARCV9_FJAESX; - } - - /* - * In wait for better solution _sparcv9_rdcfr is masked by - * VIS3 flag, because it goes to uninterruptible endless - * loop on UltraSPARC II running Solaris. Things might be - * different on Linux... - */ - if ((OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) && - sigsetjmp(common_jmp, 1) == 0) { - OPENSSL_sparcv9cap_P[1] = (unsigned int)_sparcv9_rdcfr(); - } - - sigaction(SIGBUS, &bus_oact, NULL); - sigaction(SIGILL, &ill_oact, NULL); - - sigprocmask(SIG_SETMASK, &oset, NULL); - - if (sizeof(size_t) == 8) - OPENSSL_sparcv9cap_P[0] |= SPARCV9_64BIT_STACK; -# ifdef __linux - else { - int ret = syscall(340); - - if (ret >= 0 && ret & 1) - OPENSSL_sparcv9cap_P[0] |= SPARCV9_64BIT_STACK; - } -# endif -} diff --git a/openssl/src/crypto/sparse_array.c b/openssl/src/crypto/sparse_array.c index bbbc9cdb3..53e6e7d46 100644 --- a/openssl/src/crypto/sparse_array.c +++ b/openssl/src/crypto/sparse_array.c @@ -109,10 +109,8 @@ static void sa_free_leaf(ossl_uintmax_t n, void *p, void *arg) void ossl_sa_free(OPENSSL_SA *sa) { - if (sa != NULL) { - sa_doall(sa, &sa_free_node, NULL, NULL); - OPENSSL_free(sa); - } + sa_doall(sa, &sa_free_node, NULL, NULL); + OPENSSL_free(sa); } void ossl_sa_free_leaves(OPENSSL_SA *sa) diff --git a/openssl/src/crypto/srp/srp_vfy.c b/openssl/src/crypto/srp/srp_vfy.c index e89f58b20..e8beb60d2 100644 --- a/openssl/src/crypto/srp/srp_vfy.c +++ b/openssl/src/crypto/srp/srp_vfy.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2004-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2004, EdelKey Project. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -191,8 +191,10 @@ SRP_user_pwd *SRP_user_pwd_new(void) { SRP_user_pwd *ret; - if ((ret = OPENSSL_malloc(sizeof(*ret))) == NULL) + if ((ret = OPENSSL_malloc(sizeof(*ret))) == NULL) { + /* ERR_raise(ERR_LIB_SRP, ERR_R_MALLOC_FAILURE); */ /*ckerr_ignore*/ return NULL; + } ret->N = NULL; ret->g = NULL; ret->s = NULL; @@ -281,7 +283,6 @@ SRP_VBASE *SRP_VBASE_new(char *seed_key) return NULL; if ((vb->users_pwd = sk_SRP_user_pwd_new_null()) == NULL || (vb->gN_cache = sk_SRP_gN_cache_new_null()) == NULL) { - sk_SRP_user_pwd_free(vb->users_pwd); OPENSSL_free(vb); return NULL; } @@ -392,7 +393,7 @@ static BIGNUM *SRP_gN_place_bn(STACK_OF(SRP_gN_cache) *gN_cache, char *ch) int SRP_VBASE_init(SRP_VBASE *vb, char *verifier_file) { - int error_code = SRP_ERR_MEMORY; + int error_code; STACK_OF(SRP_gN) *SRP_gN_tab = sk_SRP_gN_new_null(); char *last_index = NULL; int i; @@ -404,9 +405,6 @@ int SRP_VBASE_init(SRP_VBASE *vb, char *verifier_file) TXT_DB *tmpdb = NULL; BIO *in = BIO_new(BIO_s_file()); - if (SRP_gN_tab == NULL) - goto err; - error_code = SRP_ERR_OPEN_FILE; if (in == NULL || BIO_read_filename(in, verifier_file) <= 0) @@ -630,7 +628,7 @@ char *SRP_create_verifier_ex(const char *user, const char *pass, char **salt, if (N_bn_alloc == NULL) goto err; N_bn = N_bn_alloc; - if ((len = t_fromb64(tmp, sizeof(tmp), g)) <= 0) + if ((len = t_fromb64(tmp, sizeof(tmp) ,g)) <= 0) goto err; g_bn_alloc = BN_bin2bn(tmp, len, NULL); if (g_bn_alloc == NULL) diff --git a/openssl/src/crypto/stack/stack.c b/openssl/src/crypto/stack/stack.c index e81398962..3d8e4746c 100644 --- a/openssl/src/crypto/stack/stack.c +++ b/openssl/src/crypto/stack/stack.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,19 +10,17 @@ #include #include "internal/cryptlib.h" #include "internal/numbers.h" -#include "internal/safe_math.h" #include #include #include /* For ossl_inline */ -OSSL_SAFE_MATH_SIGNED(int, int) - /* * The initial number of nodes in the array. */ static const int min_nodes = 4; static const int max_nodes = SIZE_MAX / sizeof(void *) < INT_MAX - ? (int)(SIZE_MAX / sizeof(void *)) : INT_MAX; + ? (int)(SIZE_MAX / sizeof(void *)) + : INT_MAX; struct stack_st { int num; @@ -32,8 +30,7 @@ struct stack_st { OPENSSL_sk_compfunc comp; }; -OPENSSL_sk_compfunc OPENSSL_sk_set_cmp_func(OPENSSL_STACK *sk, - OPENSSL_sk_compfunc c) +OPENSSL_sk_compfunc OPENSSL_sk_set_cmp_func(OPENSSL_STACK *sk, OPENSSL_sk_compfunc c) { OPENSSL_sk_compfunc old = sk->comp; @@ -68,20 +65,20 @@ OPENSSL_STACK *OPENSSL_sk_dup(const OPENSSL_STACK *sk) } /* duplicate |sk->data| content */ - ret->data = OPENSSL_malloc(sizeof(*ret->data) * sk->num_alloc); - if (ret->data == NULL) + if ((ret->data = OPENSSL_malloc(sizeof(*ret->data) * sk->num_alloc)) == NULL) goto err; memcpy(ret->data, sk->data, sizeof(void *) * sk->num); return ret; err: + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); OPENSSL_sk_free(ret); return NULL; } OPENSSL_STACK *OPENSSL_sk_deep_copy(const OPENSSL_STACK *sk, - OPENSSL_sk_copyfunc copy_func, - OPENSSL_sk_freefunc free_func) + OPENSSL_sk_copyfunc copy_func, + OPENSSL_sk_freefunc free_func) { OPENSSL_STACK *ret; int i; @@ -123,6 +120,7 @@ OPENSSL_STACK *OPENSSL_sk_deep_copy(const OPENSSL_STACK *sk, return ret; err: + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); OPENSSL_sk_free(ret); return NULL; } @@ -140,35 +138,32 @@ OPENSSL_STACK *OPENSSL_sk_new(OPENSSL_sk_compfunc c) /* * Calculate the array growth based on the target size. * - * The growth factor is a rational number and is defined by a numerator + * The growth fraction is a rational number and is defined by a numerator * and a denominator. According to Andrew Koenig in his paper "Why Are * Vectors Efficient?" from JOOP 11(5) 1998, this factor should be less * than the golden ratio (1.618...). * - * Considering only the Fibonacci ratios less than the golden ratio, the - * number of steps from the minimum allocation to integer overflow is: - * factor decimal growths - * 3/2 1.5 51 - * 8/5 1.6 45 - * 21/13 1.615... 44 + * We use 3/2 = 1.5 for simplicity of calculation and overflow checking. + * Another option 8/5 = 1.6 allows for slightly faster growth, although safe + * computation is more difficult. * - * All larger factors have the same number of growths. + * The limit to avoid overflow is spot on. The modulo three correction term + * ensures that the limit is the largest number than can be expanded by the + * growth factor without exceeding the hard limit. * - * 3/2 and 8/5 have nice power of two shifts, so seem like a good choice. + * Do not call it with |current| lower than 2, or it will infinitely loop. */ static ossl_inline int compute_growth(int target, int current) { - int err = 0; + const int limit = (max_nodes / 3) * 2 + (max_nodes % 3 ? 1 : 0); while (current < target) { + /* Check to see if we're at the hard limit */ if (current >= max_nodes) return 0; - current = safe_muldiv_int(current, 8, 5, &err); - if (err != 0) - return 0; - if (current >= max_nodes) - current = max_nodes; + /* Expand the size by a factor of 3/2 if it is within range */ + current = current < limit ? current + current / 2 : max_nodes; } return current; } @@ -180,10 +175,8 @@ static int sk_reserve(OPENSSL_STACK *st, int n, int exact) int num_alloc; /* Check to see the reservation isn't exceeding the hard limit */ - if (n > max_nodes - st->num) { - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_TOO_MANY_RECORDS); + if (n > max_nodes - st->num) return 0; - } /* Figure out the new size */ num_alloc = st->num + n; @@ -196,8 +189,10 @@ static int sk_reserve(OPENSSL_STACK *st, int n, int exact) * At this point, |st->num_alloc| and |st->num| are 0; * so |num_alloc| value is |n| or |min_nodes| if greater than |n|. */ - if ((st->data = OPENSSL_zalloc(sizeof(void *) * num_alloc)) == NULL) + if ((st->data = OPENSSL_zalloc(sizeof(void *) * num_alloc)) == NULL) { + ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE); return 0; + } st->num_alloc = num_alloc; return 1; } @@ -206,10 +201,8 @@ static int sk_reserve(OPENSSL_STACK *st, int n, int exact) if (num_alloc <= st->num_alloc) return 1; num_alloc = compute_growth(num_alloc, st->num_alloc); - if (num_alloc == 0) { - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_TOO_MANY_RECORDS); + if (num_alloc == 0) return 0; - } } else if (num_alloc == st->num_alloc) { return 1; } @@ -245,10 +238,8 @@ OPENSSL_STACK *OPENSSL_sk_new_reserve(OPENSSL_sk_compfunc c, int n) int OPENSSL_sk_reserve(OPENSSL_STACK *st, int n) { - if (st == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); + if (st == NULL) return 0; - } if (n < 0) return 1; @@ -257,14 +248,8 @@ int OPENSSL_sk_reserve(OPENSSL_STACK *st, int n) int OPENSSL_sk_insert(OPENSSL_STACK *st, const void *data, int loc) { - if (st == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (st->num == max_nodes) { - ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_TOO_MANY_RECORDS); + if (st == NULL || st->num == max_nodes) return 0; - } if (!sk_reserve(st, 1, 0)) return 0; @@ -286,8 +271,8 @@ static ossl_inline void *internal_delete(OPENSSL_STACK *st, int loc) const void *ret = st->data[loc]; if (loc != st->num - 1) - memmove(&st->data[loc], &st->data[loc + 1], - sizeof(st->data[0]) * (st->num - loc - 1)); + memmove(&st->data[loc], &st->data[loc + 1], + sizeof(st->data[0]) * (st->num - loc - 1)); st->num--; return (void *)ret; @@ -297,9 +282,6 @@ void *OPENSSL_sk_delete_ptr(OPENSSL_STACK *st, const void *p) { int i; - if (st == NULL) - return NULL; - for (i = 0; i < st->num; i++) if (st->data[i] == p) return internal_delete(st, i); @@ -315,54 +297,39 @@ void *OPENSSL_sk_delete(OPENSSL_STACK *st, int loc) } static int internal_find(OPENSSL_STACK *st, const void *data, - int ret_val_options, int *pnum_matched) + int ret_val_options, int *pnum) { const void *r; - int i, count = 0; - int *pnum = pnum_matched; + int i; if (st == NULL || st->num == 0) return -1; - if (pnum == NULL) - pnum = &count; - if (st->comp == NULL) { for (i = 0; i < st->num; i++) if (st->data[i] == data) { - *pnum = 1; + if (pnum != NULL) + *pnum = 1; return i; } - *pnum = 0; + if (pnum != NULL) + *pnum = 0; return -1; } - if (data == NULL) - return -1; - if (!st->sorted) { - int res = -1; - - for (i = 0; i < st->num; i++) - if (st->comp(&data, st->data + i) == 0) { - if (res == -1) - res = i; - ++*pnum; - /* Check if only one result is wanted and exit if so */ - if (pnum_matched == NULL) - return i; - } - if (res == -1) - *pnum = 0; - return res; + if (st->num > 1) + qsort(st->data, st->num, sizeof(void *), st->comp); + st->sorted = 1; /* empty or single-element stack is considered sorted */ } - - if (pnum_matched != NULL) + if (data == NULL) + return -1; + if (pnum != NULL) ret_val_options |= OSSL_BSEARCH_FIRST_VALUE_ON_MATCH; r = ossl_bsearch(&data, st->data, st->num, sizeof(void *), st->comp, ret_val_options); - if (pnum_matched != NULL) { + if (pnum != NULL) { *pnum = 0; if (r != NULL) { const void **p = (const void **)r; @@ -397,7 +364,7 @@ int OPENSSL_sk_find_all(OPENSSL_STACK *st, const void *data, int *pnum) int OPENSSL_sk_push(OPENSSL_STACK *st, const void *data) { if (st == NULL) - return 0; + return -1; return OPENSSL_sk_insert(st, data, st->num); } @@ -462,15 +429,8 @@ void *OPENSSL_sk_value(const OPENSSL_STACK *st, int i) void *OPENSSL_sk_set(OPENSSL_STACK *st, int i, const void *data) { - if (st == NULL) { - ERR_raise(ERR_LIB_CRYPTO, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - if (i < 0 || i >= st->num) { - ERR_raise_data(ERR_LIB_CRYPTO, ERR_R_PASSED_INVALID_ARGUMENT, - "i=%d", i); + if (st == NULL || i < 0 || i >= st->num) return NULL; - } st->data[i] = data; st->sorted = 0; return (void *)st->data[i]; diff --git a/openssl/src/crypto/store/store_lib.c b/openssl/src/crypto/store/store_lib.c index 0b55123d8..563d65204 100644 --- a/openssl/src/crypto/store/store_lib.c +++ b/openssl/src/crypto/store/store_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -66,7 +66,6 @@ OSSL_STORE_open_ex(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, OSSL_STORE_post_process_info_fn post_process, void *post_process_data) { - struct ossl_passphrase_data_st pwdata = { 0 }; const OSSL_STORE_LOADER *loader = NULL; OSSL_STORE_LOADER *fetched_loader = NULL; OSSL_STORE_LOADER_CTX *loader_ctx = NULL; @@ -95,7 +94,7 @@ OSSL_STORE_open_ex(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, if ((p = strchr(scheme_copy, ':')) != NULL) { *p++ = '\0'; if (OPENSSL_strcasecmp(scheme_copy, "file") != 0) { - if (HAS_PREFIX(p, "//")) + if (strncmp(p, "//", 2) == 0) schemes_n--; /* Invalidate the file scheme */ schemes[schemes_n++] = scheme_copy; } @@ -103,13 +102,6 @@ OSSL_STORE_open_ex(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, ERR_set_mark(); - if (ui_method != NULL - && (!ossl_pw_set_ui_method(&pwdata, ui_method, ui_data) - || !ossl_pw_enable_passphrase_caching(&pwdata))) { - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_CRYPTO_LIB); - goto err; - } - /* * Try each scheme until we find one that could open the URI. * @@ -122,17 +114,13 @@ OSSL_STORE_open_ex(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, scheme = schemes[i]; OSSL_TRACE1(STORE, "Looking up scheme %s\n", scheme); #ifndef OPENSSL_NO_DEPRECATED_3_0 - ERR_set_mark(); if ((loader = ossl_store_get0_loader_int(scheme)) != NULL) { - ERR_clear_last_mark(); no_loader_found = 0; if (loader->open_ex != NULL) loader_ctx = loader->open_ex(loader, uri, libctx, propq, ui_method, ui_data); else loader_ctx = loader->open(loader, uri, ui_method, ui_data); - } else { - ERR_pop_to_mark(); } #endif if (loader == NULL @@ -143,28 +131,17 @@ OSSL_STORE_open_ex(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, void *provctx = OSSL_PROVIDER_get0_provider_ctx(provider); no_loader_found = 0; - if (fetched_loader->p_open_ex != NULL) { - loader_ctx = - fetched_loader->p_open_ex(provctx, uri, params, - ossl_pw_passphrase_callback_dec, - &pwdata); - } else { - if (fetched_loader->p_open != NULL && - (loader_ctx = fetched_loader->p_open(provctx, uri)) != NULL && - !loader_set_params(fetched_loader, loader_ctx, - params, propq)) { - (void)fetched_loader->p_close(loader_ctx); - loader_ctx = NULL; - } - } + loader_ctx = fetched_loader->p_open(provctx, uri); if (loader_ctx == NULL) { OSSL_STORE_LOADER_free(fetched_loader); fetched_loader = NULL; + } else if(!loader_set_params(fetched_loader, loader_ctx, + params, propq)) { + (void)fetched_loader->p_close(loader_ctx); + OSSL_STORE_LOADER_free(fetched_loader); + fetched_loader = NULL; } loader = fetched_loader; - - /* Clear any internally cached passphrase */ - (void)ossl_pw_clear_passphrase_cache(&pwdata); } } @@ -187,16 +164,23 @@ OSSL_STORE_open_ex(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, OSSL_TRACE2(STORE, "Opened %s => %p\n", uri, (void *)loader_ctx); if ((propq != NULL && (propq_copy = OPENSSL_strdup(propq)) == NULL) - || (ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + || (ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) { + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); goto err; + } + if (ui_method != NULL + && (!ossl_pw_set_ui_method(&ctx->pwdata, ui_method, ui_data) + || !ossl_pw_enable_passphrase_caching(&ctx->pwdata))) { + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_CRYPTO_LIB); + goto err; + } ctx->properties = propq_copy; ctx->fetched_loader = fetched_loader; ctx->loader = loader; ctx->loader_ctx = loader_ctx; ctx->post_process = post_process; ctx->post_process_data = post_process_data; - ctx->pwdata = pwdata; /* * If the attempt to open with the 'file' scheme loader failed and the @@ -227,8 +211,6 @@ OSSL_STORE_open_ex(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, */ (void)ossl_store_close_it(&tmpctx); } - /* Coverity false positive, the reference counting is confusing it */ - /* coverity[pass_freed_arg] */ OSSL_STORE_LOADER_free(fetched_loader); OPENSSL_free(propq_copy); OPENSSL_free(ctx); @@ -350,7 +332,7 @@ int OSSL_STORE_find(OSSL_STORE_CTX *ctx, const OSSL_STORE_SEARCH *search) } if ((bld = OSSL_PARAM_BLD_new()) == NULL) { - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return 0; } @@ -442,14 +424,14 @@ OSSL_STORE_INFO *OSSL_STORE_load(OSSL_STORE_CTX *ctx) load_data.v = NULL; load_data.ctx = ctx; - ctx->error_flag = 0; if (!ctx->fetched_loader->p_load(ctx->loader_ctx, ossl_store_handle_load_result, &load_data, ossl_pw_passphrase_callback_dec, &ctx->pwdata)) { - ctx->error_flag = 1; + if (!OSSL_STORE_eof(ctx)) + ctx->error_flag = 1; return NULL; } v = load_data.v; @@ -494,53 +476,6 @@ OSSL_STORE_INFO *OSSL_STORE_load(OSSL_STORE_CTX *ctx) return v; } -int OSSL_STORE_delete(const char *uri, OSSL_LIB_CTX *libctx, const char *propq, - const UI_METHOD *ui_method, void *ui_data, - const OSSL_PARAM params[]) -{ - OSSL_STORE_LOADER *fetched_loader = NULL; - char scheme[256], *p; - int res = 0; - struct ossl_passphrase_data_st pwdata = {0}; - - OPENSSL_strlcpy(scheme, uri, sizeof(scheme)); - if ((p = strchr(scheme, ':')) != NULL) - *p++ = '\0'; - else /* We don't work without explicit scheme */ - return 0; - - if (ui_method != NULL - && (!ossl_pw_set_ui_method(&pwdata, ui_method, ui_data) - || !ossl_pw_enable_passphrase_caching(&pwdata))) { - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_CRYPTO_LIB); - return 0; - } - - OSSL_TRACE1(STORE, "Looking up scheme %s\n", scheme); - fetched_loader = OSSL_STORE_LOADER_fetch(libctx, scheme, propq); - - if (fetched_loader != NULL && fetched_loader->p_delete != NULL) { - const OSSL_PROVIDER *provider = - OSSL_STORE_LOADER_get0_provider(fetched_loader); - void *provctx = OSSL_PROVIDER_get0_provider_ctx(provider); - - /* - * It's assumed that the loader's delete() method reports its own - * errors - */ - OSSL_TRACE1(STORE, "Performing URI delete %s\n", uri); - res = fetched_loader->p_delete(provctx, uri, params, - ossl_pw_passphrase_callback_dec, - &pwdata); - } - /* Clear any internally cached passphrase */ - (void)ossl_pw_clear_passphrase_cache(&pwdata); - - OSSL_STORE_LOADER_free(fetched_loader); - - return res; -} - int OSSL_STORE_error(OSSL_STORE_CTX *ctx) { int ret = 1; @@ -621,7 +556,7 @@ OSSL_STORE_INFO *OSSL_STORE_INFO_new_NAME(char *name) OSSL_STORE_INFO *info = OSSL_STORE_INFO_new(OSSL_STORE_INFO_NAME, NULL); if (info == NULL) { - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_OSSL_STORE_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; } @@ -647,7 +582,7 @@ OSSL_STORE_INFO *OSSL_STORE_INFO_new_PARAMS(EVP_PKEY *params) OSSL_STORE_INFO *info = OSSL_STORE_INFO_new(OSSL_STORE_INFO_PARAMS, params); if (info == NULL) - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_OSSL_STORE_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return info; } @@ -656,7 +591,7 @@ OSSL_STORE_INFO *OSSL_STORE_INFO_new_PUBKEY(EVP_PKEY *pkey) OSSL_STORE_INFO *info = OSSL_STORE_INFO_new(OSSL_STORE_INFO_PUBKEY, pkey); if (info == NULL) - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_OSSL_STORE_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return info; } @@ -665,7 +600,7 @@ OSSL_STORE_INFO *OSSL_STORE_INFO_new_PKEY(EVP_PKEY *pkey) OSSL_STORE_INFO *info = OSSL_STORE_INFO_new(OSSL_STORE_INFO_PKEY, pkey); if (info == NULL) - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_OSSL_STORE_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return info; } @@ -674,7 +609,7 @@ OSSL_STORE_INFO *OSSL_STORE_INFO_new_CERT(X509 *x509) OSSL_STORE_INFO *info = OSSL_STORE_INFO_new(OSSL_STORE_INFO_CERT, x509); if (info == NULL) - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_OSSL_STORE_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return info; } @@ -683,12 +618,12 @@ OSSL_STORE_INFO *OSSL_STORE_INFO_new_CRL(X509_CRL *crl) OSSL_STORE_INFO *info = OSSL_STORE_INFO_new(OSSL_STORE_INFO_CRL, crl); if (info == NULL) - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_OSSL_STORE_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return info; } /* - * Functions to try to extract data from an OSSL_STORE_INFO. + * Functions to try to extract data from a OSSL_STORE_INFO. */ int OSSL_STORE_INFO_get_type(const OSSL_STORE_INFO *info) { @@ -711,8 +646,13 @@ const char *OSSL_STORE_INFO_get0_NAME(const OSSL_STORE_INFO *info) char *OSSL_STORE_INFO_get1_NAME(const OSSL_STORE_INFO *info) { - if (info->type == OSSL_STORE_INFO_NAME) - return OPENSSL_strdup(info->_.name.name); + if (info->type == OSSL_STORE_INFO_NAME) { + char *ret = OPENSSL_strdup(info->_.name.name); + + if (ret == NULL) + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); + return ret; + } ERR_raise(ERR_LIB_OSSL_STORE, OSSL_STORE_R_NOT_A_NAME); return NULL; } @@ -726,8 +666,14 @@ const char *OSSL_STORE_INFO_get0_NAME_description(const OSSL_STORE_INFO *info) char *OSSL_STORE_INFO_get1_NAME_description(const OSSL_STORE_INFO *info) { - if (info->type == OSSL_STORE_INFO_NAME) - return OPENSSL_strdup(info->_.name.desc ? info->_.name.desc : ""); + if (info->type == OSSL_STORE_INFO_NAME) { + char *ret = OPENSSL_strdup(info->_.name.desc + ? info->_.name.desc : ""); + + if (ret == NULL) + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); + return ret; + } ERR_raise(ERR_LIB_OSSL_STORE, OSSL_STORE_R_NOT_A_NAME); return NULL; } @@ -906,8 +852,10 @@ OSSL_STORE_SEARCH *OSSL_STORE_SEARCH_by_name(X509_NAME *name) { OSSL_STORE_SEARCH *search = OPENSSL_zalloc(sizeof(*search)); - if (search == NULL) + if (search == NULL) { + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; + } search->search_type = OSSL_STORE_SEARCH_BY_NAME; search->name = name; @@ -919,8 +867,10 @@ OSSL_STORE_SEARCH *OSSL_STORE_SEARCH_by_issuer_serial(X509_NAME *name, { OSSL_STORE_SEARCH *search = OPENSSL_zalloc(sizeof(*search)); - if (search == NULL) + if (search == NULL) { + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; + } search->search_type = OSSL_STORE_SEARCH_BY_ISSUER_SERIAL; search->name = name; @@ -934,8 +884,10 @@ OSSL_STORE_SEARCH *OSSL_STORE_SEARCH_by_key_fingerprint(const EVP_MD *digest, { OSSL_STORE_SEARCH *search = OPENSSL_zalloc(sizeof(*search)); - if (search == NULL) + if (search == NULL) { + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; + } if (digest != NULL && len != (size_t)EVP_MD_get_size(digest)) { ERR_raise_data(ERR_LIB_OSSL_STORE, @@ -957,8 +909,10 @@ OSSL_STORE_SEARCH *OSSL_STORE_SEARCH_by_alias(const char *alias) { OSSL_STORE_SEARCH *search = OPENSSL_zalloc(sizeof(*search)); - if (search == NULL) + if (search == NULL) { + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; + } search->search_type = OSSL_STORE_SEARCH_BY_ALIAS; search->string = (const unsigned char *)alias; @@ -1037,7 +991,6 @@ OSSL_STORE_CTX *OSSL_STORE_attach(BIO *bp, const char *scheme, OSSL_CORE_BIO *cbio = ossl_core_bio_new_from_bio(bp); if (cbio == NULL - || fetched_loader->p_attach == NULL || (loader_ctx = fetched_loader->p_attach(provctx, cbio)) == NULL) { OSSL_STORE_LOADER_free(fetched_loader); fetched_loader = NULL; @@ -1058,6 +1011,7 @@ OSSL_STORE_CTX *OSSL_STORE_attach(BIO *bp, const char *scheme, if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) { ERR_clear_last_mark(); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; } @@ -1075,7 +1029,7 @@ OSSL_STORE_CTX *OSSL_STORE_attach(BIO *bp, const char *scheme, ctx->post_process_data = post_process_data; /* - * ossl_store_get0_loader_int will raise an error if the loader for + * ossl_store_get0_loader_int will raise an error if the loader for the * the scheme cannot be retrieved. But if a loader was successfully * fetched then we remove this error from the error stack. */ diff --git a/openssl/src/crypto/store/store_local.h b/openssl/src/crypto/store/store_local.h index 6ad79180a..8f817fd51 100644 --- a/openssl/src/crypto/store/store_local.h +++ b/openssl/src/crypto/store/store_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -103,6 +103,7 @@ struct ossl_store_loader_st { const char *description; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; OSSL_FUNC_store_open_fn *p_open; OSSL_FUNC_store_attach_fn *p_attach; @@ -112,10 +113,8 @@ struct ossl_store_loader_st { OSSL_FUNC_store_eof_fn *p_eof; OSSL_FUNC_store_close_fn *p_close; OSSL_FUNC_store_export_object_fn *p_export_object; - OSSL_FUNC_store_delete_fn *p_delete; - OSSL_FUNC_store_open_ex_fn *p_open_ex; }; -DEFINE_LHASH_OF_EX(OSSL_STORE_LOADER); +DEFINE_LHASH_OF(OSSL_STORE_LOADER); const OSSL_STORE_LOADER *ossl_store_get0_loader_int(const char *scheme); void ossl_store_destroy_loaders_int(void); @@ -169,6 +168,9 @@ int ossl_store_file_detach_pem_bio_int(OSSL_STORE_LOADER_CTX *ctx); OSSL_STORE_LOADER *ossl_store_loader_fetch(OSSL_LIB_CTX *libctx, const char *scheme, const char *properties); +OSSL_STORE_LOADER *ossl_store_loader_fetch_by_number(OSSL_LIB_CTX *libctx, + int scheme_id, + const char *properties); /* Standard function to handle the result from OSSL_FUNC_store_load() */ struct ossl_load_result_data_st { diff --git a/openssl/src/crypto/store/store_meth.c b/openssl/src/crypto/store/store_meth.c index 6ac8fd5f9..db13f62c6 100644 --- a/openssl/src/crypto/store/store_meth.c +++ b/openssl/src/crypto/store/store_meth.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,14 +14,13 @@ #include "internal/property.h" #include "internal/provider.h" #include "store_local.h" -#include "crypto/context.h" int OSSL_STORE_LOADER_up_ref(OSSL_STORE_LOADER *loader) { int ref = 0; if (loader->prov != NULL) - CRYPTO_UP_REF(&loader->refcnt, &ref); + CRYPTO_UP_REF(&loader->refcnt, &ref, loader->lock); return 1; } @@ -30,11 +29,11 @@ void OSSL_STORE_LOADER_free(OSSL_STORE_LOADER *loader) if (loader != NULL && loader->prov != NULL) { int i; - CRYPTO_DOWN_REF(&loader->refcnt, &i); + CRYPTO_DOWN_REF(&loader->refcnt, &i, loader->lock); if (i > 0) return; ossl_provider_free(loader->prov); - CRYPTO_FREE_REF(&loader->refcnt); + CRYPTO_THREAD_lock_free(loader->lock); } OPENSSL_free(loader); } @@ -48,12 +47,13 @@ static OSSL_STORE_LOADER *new_loader(OSSL_PROVIDER *prov) OSSL_STORE_LOADER *loader; if ((loader = OPENSSL_zalloc(sizeof(*loader))) == NULL - || !CRYPTO_NEW_REF(&loader->refcnt, 1)) { + || (loader->lock = CRYPTO_THREAD_lock_new()) == NULL) { OPENSSL_free(loader); return NULL; } loader->prov = prov; ossl_provider_up_ref(prov); + loader->refcnt = 1; return loader; } @@ -68,6 +68,25 @@ static void free_loader(void *method) OSSL_STORE_LOADER_free(method); } +/* Permanent loader method store, constructor and destructor */ +static void loader_store_free(void *vstore) +{ + ossl_method_store_free(vstore); +} + +static void *loader_store_new(OSSL_LIB_CTX *ctx) +{ + return ossl_method_store_new(ctx); +} + + +static const OSSL_LIB_CTX_METHOD loader_store_method = { + /* We want loader_store to be cleaned up before the provider store */ + OSSL_LIB_CTX_METHOD_PRIORITY_2, + loader_store_new, + loader_store_free, +}; + /* Data to be passed through ossl_method_construct() */ struct loader_data_st { OSSL_LIB_CTX *libctx; @@ -104,29 +123,8 @@ static void *get_tmp_loader_store(void *data) /* Get the permanent loader store */ static OSSL_METHOD_STORE *get_loader_store(OSSL_LIB_CTX *libctx) { - return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_STORE_LOADER_STORE_INDEX); -} - -static int reserve_loader_store(void *store, void *data) -{ - struct loader_data_st *methdata = data; - - if (store == NULL - && (store = get_loader_store(methdata->libctx)) == NULL) - return 0; - - return ossl_method_lock_store(store); -} - -static int unreserve_loader_store(void *store, void *data) -{ - struct loader_data_st *methdata = data; - - if (store == NULL - && (store = get_loader_store(methdata->libctx)) == NULL) - return 0; - - return ossl_method_unlock_store(store); + return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_STORE_LOADER_STORE_INDEX, + &loader_store_method); } /* Get loader methods from a store, or put one in */ @@ -219,14 +217,6 @@ static void *loader_from_algorithm(int scheme_id, const OSSL_ALGORITHM *algodef, if (loader->p_export_object == NULL) loader->p_export_object = OSSL_FUNC_store_export_object(fns); break; - case OSSL_FUNC_STORE_DELETE: - if (loader->p_delete == NULL) - loader->p_delete = OSSL_FUNC_store_delete(fns); - break; - case OSSL_FUNC_STORE_OPEN_EX: - if (loader->p_open_ex == NULL) - loader->p_open_ex = OSSL_FUNC_store_open_ex(fns); - break; } } @@ -234,7 +224,7 @@ static void *loader_from_algorithm(int scheme_id, const OSSL_ALGORITHM *algodef, || loader->p_load == NULL || loader->p_eof == NULL || loader->p_close == NULL) { - /* Only set_ctx_params is optional */ + /* Only set_ctx_params is optionaal */ OSSL_STORE_LOADER_free(loader); ERR_raise(ERR_LIB_OSSL_STORE, OSSL_STORE_R_LOADER_INCOMPLETE); return NULL; @@ -285,35 +275,44 @@ static void destruct_loader(void *method, void *data) /* Fetching support. Can fetch by numeric identity or by scheme */ static OSSL_STORE_LOADER * -inner_loader_fetch(struct loader_data_st *methdata, +inner_loader_fetch(struct loader_data_st *methdata, int id, const char *scheme, const char *properties) { OSSL_METHOD_STORE *store = get_loader_store(methdata->libctx); OSSL_NAMEMAP *namemap = ossl_namemap_stored(methdata->libctx); const char *const propq = properties != NULL ? properties : ""; void *method = NULL; - int unsupported, id; + int unsupported = 0; if (store == NULL || namemap == NULL) { ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_PASSED_INVALID_ARGUMENT); return NULL; } + /* + * If we have been passed both an id and a scheme, we have an + * internal programming error. + */ + if (!ossl_assert(id == 0 || scheme == NULL)) { + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_INTERNAL_ERROR); + return NULL; + } + /* If we haven't received a name id yet, try to get one for the name */ - id = scheme != NULL ? ossl_namemap_name2num(namemap, scheme) : 0; + if (id == 0 && scheme != NULL) + id = ossl_namemap_name2num(namemap, scheme); /* * If we haven't found the name yet, chances are that the algorithm to * be fetched is unsupported. */ - unsupported = id == 0; + if (id == 0) + unsupported = 1; if (id == 0 || !ossl_method_store_cache_get(store, NULL, id, propq, &method)) { OSSL_METHOD_CONSTRUCT_METHOD mcm = { get_tmp_loader_store, - reserve_loader_store, - unreserve_loader_store, get_loader_from_store, put_loader_in_store, construct_loader, @@ -361,7 +360,7 @@ inner_loader_fetch(struct loader_data_st *methdata, "%s%s, Scheme (%s : %d), Properties (%s)", helpful_msg, ossl_lib_ctx_get_descriptor(methdata->libctx), - scheme == NULL ? "" : scheme, id, + scheme = NULL ? "" : scheme, id, properties == NULL ? "" : properties); } @@ -377,28 +376,23 @@ OSSL_STORE_LOADER *OSSL_STORE_LOADER_fetch(OSSL_LIB_CTX *libctx, methdata.libctx = libctx; methdata.tmp_store = NULL; - method = inner_loader_fetch(&methdata, scheme, properties); + method = inner_loader_fetch(&methdata, 0, scheme, properties); dealloc_tmp_loader_store(methdata.tmp_store); return method; } -int ossl_store_loader_store_cache_flush(OSSL_LIB_CTX *libctx) -{ - OSSL_METHOD_STORE *store = get_loader_store(libctx); - - if (store != NULL) - return ossl_method_store_cache_flush_all(store); - return 1; -} - -int ossl_store_loader_store_remove_all_provided(const OSSL_PROVIDER *prov) +OSSL_STORE_LOADER *ossl_store_loader_fetch_by_number(OSSL_LIB_CTX *libctx, + int scheme_id, + const char *properties) { - OSSL_LIB_CTX *libctx = ossl_provider_libctx(prov); - OSSL_METHOD_STORE *store = get_loader_store(libctx); + struct loader_data_st methdata; + void *method; - if (store != NULL) - return ossl_method_store_remove_all_provided(store, prov); - return 1; + methdata.libctx = libctx; + methdata.tmp_store = NULL; + method = inner_loader_fetch(&methdata, scheme_id, NULL, properties); + dealloc_tmp_loader_store(methdata.tmp_store); + return method; } /* @@ -473,7 +467,7 @@ void OSSL_STORE_LOADER_do_all_provided(OSSL_LIB_CTX *libctx, methdata.libctx = libctx; methdata.tmp_store = NULL; - (void)inner_loader_fetch(&methdata, NULL, NULL /* properties */); + (void)inner_loader_fetch(&methdata, 0, NULL, NULL /* properties */); data.user_fn = user_fn; data.user_arg = user_arg; diff --git a/openssl/src/crypto/store/store_register.c b/openssl/src/crypto/store/store_register.c index 6f73e1972..6fa7352cc 100644 --- a/openssl/src/crypto/store/store_register.c +++ b/openssl/src/crypto/store/store_register.c @@ -43,8 +43,10 @@ OSSL_STORE_LOADER *OSSL_STORE_LOADER_new(ENGINE *e, const char *scheme) return NULL; } - if ((res = OPENSSL_zalloc(sizeof(*res))) == NULL) + if ((res = OPENSSL_zalloc(sizeof(*res))) == NULL) { + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; + } res->engine = e; res->scheme = scheme; @@ -189,8 +191,7 @@ int ossl_store_register_loader_int(OSSL_STORE_LOADER *loader) } if (!RUN_ONCE(®istry_init, do_registry_init)) { - /* Should this error be raised in do_registry_init()? */ - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return 0; } if (!CRYPTO_THREAD_write_lock(registry_lock)) @@ -223,8 +224,7 @@ const OSSL_STORE_LOADER *ossl_store_get0_loader_int(const char *scheme) template.open_ex = NULL; if (!RUN_ONCE(®istry_init, do_registry_init)) { - /* Should this error be raised in do_registry_init()? */ - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; } if (!CRYPTO_THREAD_write_lock(registry_lock)) @@ -254,8 +254,7 @@ OSSL_STORE_LOADER *ossl_store_unregister_loader_int(const char *scheme) template.closefn = NULL; if (!RUN_ONCE(®istry_init, do_registry_init)) { - /* Should this error be raised in do_registry_init()? */ - ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_OSSL_STORE, ERR_R_MALLOC_FAILURE); return NULL; } if (!CRYPTO_THREAD_write_lock(registry_lock)) diff --git a/openssl/src/crypto/store/store_result.c b/openssl/src/crypto/store/store_result.c index 27323ad2b..8176b5379 100644 --- a/openssl/src/crypto/store/store_result.c +++ b/openssl/src/crypto/store/store_result.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -191,15 +191,13 @@ static EVP_PKEY *try_key_ref(struct extracted_param_data_st *data, EVP_PKEY *pk = NULL; EVP_KEYMGMT *keymgmt = NULL; void *keydata = NULL; - int try_fallback = 2; /* If we have an object reference, we must have a data type */ if (data->data_type == NULL) return 0; keymgmt = EVP_KEYMGMT_fetch(libctx, data->data_type, propq); - ERR_set_mark(); - while (keymgmt != NULL && keydata == NULL && try_fallback-- > 0) { + if (keymgmt != NULL) { /* * There are two possible cases * @@ -209,8 +207,6 @@ static EVP_PKEY *try_key_ref(struct extracted_param_data_st *data, * do the export/import dance. */ if (EVP_KEYMGMT_get0_provider(keymgmt) == provider) { - /* no point trying fallback here */ - try_fallback = 0; keydata = evp_keymgmt_load(keymgmt, data->ref, data->ref_size); } else { struct evp_keymgmt_util_try_import_data_st import_data; @@ -234,23 +230,9 @@ static EVP_PKEY *try_key_ref(struct extracted_param_data_st *data, keydata = import_data.keydata; } - - if (keydata == NULL && try_fallback > 0) { - EVP_KEYMGMT_free(keymgmt); - keymgmt = evp_keymgmt_fetch_from_prov((OSSL_PROVIDER *)provider, - data->data_type, propq); - if (keymgmt != NULL) { - ERR_pop_to_mark(); - ERR_set_mark(); - } - } } - if (keydata != NULL) { - ERR_pop_to_mark(); + if (keydata != NULL) pk = evp_keymgmt_util_make_pkey(keymgmt, keydata); - } else { - ERR_clear_last_mark(); - } EVP_KEYMGMT_free(keymgmt); return pk; @@ -553,10 +535,8 @@ static int try_pkcs12(struct extracted_param_data_st *data, OSSL_STORE_INFO **v, ok = 0; /* Assume decryption or parse error */ - if (!PKCS12_mac_present(p12) + if (PKCS12_verify_mac(p12, "", 0) || PKCS12_verify_mac(p12, NULL, 0)) { - pass = NULL; - } else if (PKCS12_verify_mac(p12, "", 0)) { pass = ""; } else { static char prompt_info[] = "PKCS12 import pass phrase"; @@ -629,7 +609,7 @@ static int try_pkcs12(struct extracted_param_data_st *data, OSSL_STORE_INFO **v, } EVP_PKEY_free(pkey); X509_free(cert); - OSSL_STACK_OF_X509_free(chain); + sk_X509_pop_free(chain, X509_free); OSSL_STORE_INFO_free(osi_pkey); OSSL_STORE_INFO_free(osi_cert); OSSL_STORE_INFO_free(osi_ca); diff --git a/openssl/src/crypto/thread/api.c b/openssl/src/crypto/thread/api.c deleted file mode 100644 index e025d24ce..000000000 --- a/openssl/src/crypto/thread/api.c +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include - -uint32_t OSSL_get_thread_support_flags(void) -{ - int support = 0; - -#if !defined(OPENSSL_NO_THREAD_POOL) - support |= OSSL_THREAD_SUPPORT_FLAG_THREAD_POOL; -#endif -#if !defined(OPENSSL_NO_DEFAULT_THREAD_POOL) - support |= OSSL_THREAD_SUPPORT_FLAG_DEFAULT_SPAWN; -#endif - - return support; -} - -#if defined(OPENSSL_NO_THREAD_POOL) || defined(OPENSSL_NO_DEFAULT_THREAD_POOL) - -int OSSL_set_max_threads(OSSL_LIB_CTX *ctx, uint64_t max_threads) -{ - return 0; -} - -uint64_t OSSL_get_max_threads(OSSL_LIB_CTX *ctx) -{ - return 0; -} - -#else - -uint64_t OSSL_get_max_threads(OSSL_LIB_CTX *ctx) -{ - uint64_t ret = 0; - OSSL_LIB_CTX_THREADS *tdata = OSSL_LIB_CTX_GET_THREADS(ctx); - - if (tdata == NULL) - goto fail; - - ossl_crypto_mutex_lock(tdata->lock); - ret = tdata->max_threads; - ossl_crypto_mutex_unlock(tdata->lock); - -fail: - return ret; -} - -int OSSL_set_max_threads(OSSL_LIB_CTX *ctx, uint64_t max_threads) -{ - OSSL_LIB_CTX_THREADS *tdata; - - tdata = OSSL_LIB_CTX_GET_THREADS(ctx); - if (tdata == NULL) - return 0; - - ossl_crypto_mutex_lock(tdata->lock); - tdata->max_threads = max_threads; - ossl_crypto_mutex_unlock(tdata->lock); - - return 1; -} - -#endif diff --git a/openssl/src/crypto/thread/arch.c b/openssl/src/crypto/thread/arch.c deleted file mode 100644 index 7c139a6a6..000000000 --- a/openssl/src/crypto/thread/arch.c +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include - -CRYPTO_THREAD *ossl_crypto_thread_native_start(CRYPTO_THREAD_ROUTINE routine, - void *data, int joinable) -{ - CRYPTO_THREAD *handle; - - if (routine == NULL) - return NULL; - - handle = OPENSSL_zalloc(sizeof(*handle)); - if (handle == NULL) - return NULL; - - if ((handle->lock = ossl_crypto_mutex_new()) == NULL) - goto fail; - if ((handle->statelock = ossl_crypto_mutex_new()) == NULL) - goto fail; - if ((handle->condvar = ossl_crypto_condvar_new()) == NULL) - goto fail; - - handle->data = data; - handle->routine = routine; - handle->joinable = joinable; - - if (ossl_crypto_thread_native_spawn(handle) == 1) - return handle; - -fail: - ossl_crypto_condvar_free(&handle->condvar); - ossl_crypto_mutex_free(&handle->statelock); - ossl_crypto_mutex_free(&handle->lock); - OPENSSL_free(handle); - return NULL; -} - -int ossl_crypto_thread_native_join(CRYPTO_THREAD *thread, CRYPTO_THREAD_RETVAL *retval) -{ - uint64_t req_state_mask; - - if (thread == NULL) - return 0; - - ossl_crypto_mutex_lock(thread->statelock); - req_state_mask = CRYPTO_THREAD_FINISHED | CRYPTO_THREAD_JOINED; - while (!CRYPTO_THREAD_GET_STATE(thread, req_state_mask)) - ossl_crypto_condvar_wait(thread->condvar, thread->statelock); - - if (CRYPTO_THREAD_GET_STATE(thread, CRYPTO_THREAD_JOINED)) - goto pass; - - /* Await concurrent join completion, if any. */ - while (CRYPTO_THREAD_GET_STATE(thread, CRYPTO_THREAD_JOIN_AWAIT)) { - if (!CRYPTO_THREAD_GET_STATE(thread, CRYPTO_THREAD_JOINED)) - ossl_crypto_condvar_wait(thread->condvar, thread->statelock); - if (CRYPTO_THREAD_GET_STATE(thread, CRYPTO_THREAD_JOINED)) - goto pass; - } - CRYPTO_THREAD_SET_STATE(thread, CRYPTO_THREAD_JOIN_AWAIT); - ossl_crypto_mutex_unlock(thread->statelock); - - if (ossl_crypto_thread_native_perform_join(thread, retval) == 0) - goto fail; - - ossl_crypto_mutex_lock(thread->statelock); -pass: - CRYPTO_THREAD_UNSET_ERROR(thread, CRYPTO_THREAD_JOINED); - CRYPTO_THREAD_SET_STATE(thread, CRYPTO_THREAD_JOINED); - - /* - * Signal join completion. It is important to signal even if we haven't - * performed an actual join. Multiple threads could be awaiting the - * CRYPTO_THREAD_JOIN_AWAIT -> CRYPTO_THREAD_JOINED transition, but signal - * on actual join would wake only one. Signalling here will always wake one. - */ - ossl_crypto_condvar_signal(thread->condvar); - ossl_crypto_mutex_unlock(thread->statelock); - - if (retval != NULL) - *retval = thread->retval; - return 1; - -fail: - ossl_crypto_mutex_lock(thread->statelock); - CRYPTO_THREAD_SET_ERROR(thread, CRYPTO_THREAD_JOINED); - - /* Have another thread that's awaiting join retry to avoid that - * thread deadlock. */ - CRYPTO_THREAD_UNSET_STATE(thread, CRYPTO_THREAD_JOIN_AWAIT); - ossl_crypto_condvar_signal(thread->condvar); - - ossl_crypto_mutex_unlock(thread->statelock); - return 0; -} - -int ossl_crypto_thread_native_clean(CRYPTO_THREAD *handle) -{ - uint64_t req_state_mask; - - if (handle == NULL) - return 0; - - req_state_mask = 0; - req_state_mask |= CRYPTO_THREAD_FINISHED; - req_state_mask |= CRYPTO_THREAD_JOINED; - - ossl_crypto_mutex_lock(handle->statelock); - if (CRYPTO_THREAD_GET_STATE(handle, req_state_mask) == 0) { - ossl_crypto_mutex_unlock(handle->statelock); - return 0; - } - ossl_crypto_mutex_unlock(handle->statelock); - - ossl_crypto_mutex_free(&handle->lock); - ossl_crypto_mutex_free(&handle->statelock); - ossl_crypto_condvar_free(&handle->condvar); - - OPENSSL_free(handle->handle); - OPENSSL_free(handle); - - return 1; -} diff --git a/openssl/src/crypto/thread/arch/thread_none.c b/openssl/src/crypto/thread/arch/thread_none.c deleted file mode 100644 index 10a804f1b..000000000 --- a/openssl/src/crypto/thread/arch/thread_none.c +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include - -#if defined(OPENSSL_THREADS_NONE) - -int ossl_crypto_thread_native_spawn(CRYPTO_THREAD *thread) -{ - return 0; -} - -int ossl_crypto_thread_native_perform_join(CRYPTO_THREAD *thread, CRYPTO_THREAD_RETVAL *retval) -{ - return 0; -} - -int ossl_crypto_thread_native_exit(void) -{ - return 0; -} - -int ossl_crypto_thread_native_is_self(CRYPTO_THREAD *thread) -{ - return 0; -} - -CRYPTO_MUTEX *ossl_crypto_mutex_new(void) -{ - return NULL; -} - -void ossl_crypto_mutex_lock(CRYPTO_MUTEX *mutex) -{ -} - -int ossl_crypto_mutex_try_lock(CRYPTO_MUTEX *mutex) -{ - return 0; -} - -void ossl_crypto_mutex_unlock(CRYPTO_MUTEX *mutex) -{ -} - -void ossl_crypto_mutex_free(CRYPTO_MUTEX **mutex) -{ -} - -CRYPTO_CONDVAR *ossl_crypto_condvar_new(void) -{ - return NULL; -} - -void ossl_crypto_condvar_wait(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *mutex) -{ -} - -void ossl_crypto_condvar_wait_timeout(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *mutex, - OSSL_TIME deadline) -{ -} - -void ossl_crypto_condvar_broadcast(CRYPTO_CONDVAR *cv) -{ -} - -void ossl_crypto_condvar_signal(CRYPTO_CONDVAR *cv) -{ -} - -void ossl_crypto_condvar_free(CRYPTO_CONDVAR **cv) -{ -} - -#endif diff --git a/openssl/src/crypto/thread/arch/thread_posix.c b/openssl/src/crypto/thread/arch/thread_posix.c deleted file mode 100644 index 7650ddc85..000000000 --- a/openssl/src/crypto/thread/arch/thread_posix.c +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include - -#if defined(OPENSSL_THREADS_POSIX) -# define _GNU_SOURCE -# include -# include -# include - -static void *thread_start_thunk(void *vthread) -{ - CRYPTO_THREAD *thread; - CRYPTO_THREAD_RETVAL ret; - - thread = (CRYPTO_THREAD *)vthread; - - ret = thread->routine(thread->data); - ossl_crypto_mutex_lock(thread->statelock); - CRYPTO_THREAD_SET_STATE(thread, CRYPTO_THREAD_FINISHED); - thread->retval = ret; - ossl_crypto_condvar_broadcast(thread->condvar); - ossl_crypto_mutex_unlock(thread->statelock); - - return NULL; -} - -int ossl_crypto_thread_native_spawn(CRYPTO_THREAD *thread) -{ - int ret; - pthread_attr_t attr; - pthread_t *handle; - - handle = OPENSSL_zalloc(sizeof(*handle)); - if (handle == NULL) - goto fail; - - pthread_attr_init(&attr); - if (!thread->joinable) - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - ret = pthread_create(handle, &attr, thread_start_thunk, thread); - pthread_attr_destroy(&attr); - - if (ret != 0) - goto fail; - - thread->handle = handle; - return 1; - -fail: - thread->handle = NULL; - OPENSSL_free(handle); - return 0; -} - -int ossl_crypto_thread_native_perform_join(CRYPTO_THREAD *thread, CRYPTO_THREAD_RETVAL *retval) -{ - void *thread_retval; - pthread_t *handle; - - if (thread == NULL || thread->handle == NULL) - return 0; - - handle = (pthread_t *) thread->handle; - if (pthread_join(*handle, &thread_retval) != 0) - return 0; - - /* - * Join return value may be non-NULL when the thread has been cancelled, - * as indicated by thread_retval set to PTHREAD_CANCELLED. - */ - if (thread_retval != NULL) - return 0; - - return 1; -} - -int ossl_crypto_thread_native_exit(void) -{ - pthread_exit(NULL); - return 1; -} - -int ossl_crypto_thread_native_is_self(CRYPTO_THREAD *thread) -{ - return pthread_equal(*(pthread_t *)thread->handle, pthread_self()); -} - -CRYPTO_MUTEX *ossl_crypto_mutex_new(void) -{ - pthread_mutex_t *mutex; - - if ((mutex = OPENSSL_zalloc(sizeof(*mutex))) == NULL) - return NULL; - if (pthread_mutex_init(mutex, NULL) != 0) { - OPENSSL_free(mutex); - return NULL; - } - return (CRYPTO_MUTEX *)mutex; -} - -int ossl_crypto_mutex_try_lock(CRYPTO_MUTEX *mutex) -{ - pthread_mutex_t *mutex_p; - - mutex_p = (pthread_mutex_t *)mutex; - - if (pthread_mutex_trylock(mutex_p) == EBUSY) - return 0; - - return 1; -} - -void ossl_crypto_mutex_lock(CRYPTO_MUTEX *mutex) -{ - int rc; - pthread_mutex_t *mutex_p; - - mutex_p = (pthread_mutex_t *)mutex; - rc = pthread_mutex_lock(mutex_p); - OPENSSL_assert(rc == 0); -} - -void ossl_crypto_mutex_unlock(CRYPTO_MUTEX *mutex) -{ - int rc; - pthread_mutex_t *mutex_p; - - mutex_p = (pthread_mutex_t *)mutex; - rc = pthread_mutex_unlock(mutex_p); - OPENSSL_assert(rc == 0); -} - -void ossl_crypto_mutex_free(CRYPTO_MUTEX **mutex) -{ - pthread_mutex_t **mutex_p; - - if (mutex == NULL) - return; - - mutex_p = (pthread_mutex_t **)mutex; - if (*mutex_p != NULL) - pthread_mutex_destroy(*mutex_p); - OPENSSL_free(*mutex_p); - *mutex = NULL; -} - -CRYPTO_CONDVAR *ossl_crypto_condvar_new(void) -{ - pthread_cond_t *cv_p; - - if ((cv_p = OPENSSL_zalloc(sizeof(*cv_p))) == NULL) - return NULL; - if (pthread_cond_init(cv_p, NULL) != 0) { - OPENSSL_free(cv_p); - return NULL; - } - return (CRYPTO_CONDVAR *) cv_p; -} - -void ossl_crypto_condvar_wait(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *mutex) -{ - pthread_cond_t *cv_p; - pthread_mutex_t *mutex_p; - - cv_p = (pthread_cond_t *)cv; - mutex_p = (pthread_mutex_t *)mutex; - pthread_cond_wait(cv_p, mutex_p); -} - -void ossl_crypto_condvar_wait_timeout(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *mutex, - OSSL_TIME deadline) -{ - pthread_cond_t *cv_p = (pthread_cond_t *)cv; - pthread_mutex_t *mutex_p = (pthread_mutex_t *)mutex; - - if (ossl_time_is_infinite(deadline)) { - /* - * No deadline. Some pthread implementations allow - * pthread_cond_timedwait to work the same as pthread_cond_wait when - * abstime is NULL, but it is unclear whether this is POSIXly correct. - */ - pthread_cond_wait(cv_p, mutex_p); - } else { - struct timespec deadline_ts; - - deadline_ts.tv_sec - = ossl_time2seconds(deadline); - deadline_ts.tv_nsec - = (ossl_time2ticks(deadline) % OSSL_TIME_SECOND) / OSSL_TIME_NS; - - pthread_cond_timedwait(cv_p, mutex_p, &deadline_ts); - } -} - -void ossl_crypto_condvar_broadcast(CRYPTO_CONDVAR *cv) -{ - pthread_cond_t *cv_p; - - cv_p = (pthread_cond_t *)cv; - pthread_cond_broadcast(cv_p); -} - -void ossl_crypto_condvar_signal(CRYPTO_CONDVAR *cv) -{ - pthread_cond_t *cv_p; - - cv_p = (pthread_cond_t *)cv; - pthread_cond_signal(cv_p); -} - -void ossl_crypto_condvar_free(CRYPTO_CONDVAR **cv) -{ - pthread_cond_t **cv_p; - - if (cv == NULL) - return; - - cv_p = (pthread_cond_t **)cv; - if (*cv_p != NULL) - pthread_cond_destroy(*cv_p); - OPENSSL_free(*cv_p); - *cv_p = NULL; -} - -#endif diff --git a/openssl/src/crypto/thread/arch/thread_win.c b/openssl/src/crypto/thread/arch/thread_win.c deleted file mode 100644 index fc0c21477..000000000 --- a/openssl/src/crypto/thread/arch/thread_win.c +++ /dev/null @@ -1,599 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include - -#if defined(OPENSSL_THREADS_WINNT) -# include -# include - -static unsigned __stdcall thread_start_thunk(LPVOID vthread) -{ - CRYPTO_THREAD *thread; - CRYPTO_THREAD_RETVAL ret; - - thread = (CRYPTO_THREAD *)vthread; - - thread->thread_id = GetCurrentThreadId(); - - ret = thread->routine(thread->data); - ossl_crypto_mutex_lock(thread->statelock); - CRYPTO_THREAD_SET_STATE(thread, CRYPTO_THREAD_FINISHED); - thread->retval = ret; - ossl_crypto_condvar_signal(thread->condvar); - ossl_crypto_mutex_unlock(thread->statelock); - - return 0; -} - -int ossl_crypto_thread_native_spawn(CRYPTO_THREAD *thread) -{ - HANDLE *handle; - - handle = OPENSSL_zalloc(sizeof(*handle)); - if (handle == NULL) - goto fail; - - *handle = (HANDLE)_beginthreadex(NULL, 0, &thread_start_thunk, thread, 0, NULL); - if (*handle == NULL) - goto fail; - - thread->handle = handle; - return 1; - -fail: - thread->handle = NULL; - OPENSSL_free(handle); - return 0; -} - -int ossl_crypto_thread_native_perform_join(CRYPTO_THREAD *thread, CRYPTO_THREAD_RETVAL *retval) -{ - DWORD thread_retval; - HANDLE *handle; - - if (thread == NULL || thread->handle == NULL) - return 0; - - handle = (HANDLE *) thread->handle; - if (WaitForSingleObject(*handle, INFINITE) != WAIT_OBJECT_0) - return 0; - - if (GetExitCodeThread(*handle, &thread_retval) == 0) - return 0; - - /* - * GetExitCodeThread call followed by this check is to make sure that - * the thread exited properly. In particular, thread_retval may be - * non-zero when exited via explicit ExitThread/TerminateThread or - * if the thread is still active (returns STILL_ACTIVE (259)). - */ - if (thread_retval != 0) - return 0; - - if (CloseHandle(*handle) == 0) - return 0; - - return 1; -} - -int ossl_crypto_thread_native_exit(void) -{ - _endthreadex(0); - return 1; -} - -int ossl_crypto_thread_native_is_self(CRYPTO_THREAD *thread) -{ - return thread->thread_id == GetCurrentThreadId(); -} - -CRYPTO_MUTEX *ossl_crypto_mutex_new(void) -{ - CRITICAL_SECTION *mutex; - - if ((mutex = OPENSSL_zalloc(sizeof(*mutex))) == NULL) - return NULL; - InitializeCriticalSection(mutex); - return (CRYPTO_MUTEX *)mutex; -} - -void ossl_crypto_mutex_lock(CRYPTO_MUTEX *mutex) -{ - CRITICAL_SECTION *mutex_p; - - mutex_p = (CRITICAL_SECTION *)mutex; - EnterCriticalSection(mutex_p); -} - -int ossl_crypto_mutex_try_lock(CRYPTO_MUTEX *mutex) -{ - CRITICAL_SECTION *mutex_p; - - mutex_p = (CRITICAL_SECTION *)mutex; - if (TryEnterCriticalSection(mutex_p)) - return 1; - - return 0; -} - -void ossl_crypto_mutex_unlock(CRYPTO_MUTEX *mutex) -{ - CRITICAL_SECTION *mutex_p; - - mutex_p = (CRITICAL_SECTION *)mutex; - LeaveCriticalSection(mutex_p); -} - -void ossl_crypto_mutex_free(CRYPTO_MUTEX **mutex) -{ - CRITICAL_SECTION **mutex_p; - - mutex_p = (CRITICAL_SECTION **)mutex; - if (*mutex_p != NULL) - DeleteCriticalSection(*mutex_p); - OPENSSL_free(*mutex_p); - *mutex = NULL; -} - -static int determine_timeout(OSSL_TIME deadline, DWORD *w_timeout_p) -{ - OSSL_TIME now, delta; - uint64_t ms; - - if (ossl_time_is_infinite(deadline)) { - *w_timeout_p = INFINITE; - return 1; - } - - now = ossl_time_now(); - delta = ossl_time_subtract(deadline, now); - - if (ossl_time_is_zero(delta)) - return 0; - - ms = ossl_time2ms(delta); - - /* - * Amount of time we want to wait is too long for the 32-bit argument to - * the Win32 API, so just wait as long as possible. - */ - if (ms > (uint64_t)(INFINITE - 1)) - *w_timeout_p = INFINITE - 1; - else - *w_timeout_p = (DWORD)ms; - - return 1; -} - -# if defined(OPENSSL_THREADS_WINNT_LEGACY) -# include - -/* - * Win32, before Vista, did not have an OS-provided condition variable - * construct. This leads to the need to construct our own condition variable - * construct in order to support Windows XP. - * - * It is difficult to construct a condition variable construct using the - * OS-provided primitives in a way that is both correct (avoiding race - * conditions where broadcasts get lost) and fair. - * - * CORRECTNESS: - * A blocked thread is a thread which is calling wait(), between the - * precise instants at which the external mutex passed to wait() is - * unlocked and the instant at which it is relocked. - * - * a) - * - If broadcast() is called, ALL blocked threads MUST be unblocked. - * - If signal() is called, at least one blocked thread MUST be unblocked. - * - * (i.e.: a signal or broadcast must never get 'lost') - * - * b) - * - If broadcast() or signal() is called, this must not cause a thread - * which is not blocked to return immediately from a subsequent - * call to wait(). - * - * FAIRNESS: - * If broadcast() is called at time T1, all blocked threads must be unblocked - * before any thread which subsequently calls wait() at time T2 > T1 is - * unblocked. - * - * An example of an implementation which lacks fairness is as follows: - * - * t1 enters wait() - * t2 enters wait() - * - * tZ calls broadcast() - * - * t1 exits wait() - * t1 enters wait() - * - * tZ calls broadcast() - * - * t1 exits wait() - * - * IMPLEMENTATION: - * - * The most suitable primitives available to us in Windows XP are semaphores, - * auto-reset events and manual-reset events. A solution based on semaphores - * is chosen. - * - * PROBLEM. Designing a solution based on semaphores is non-trivial because, - * while it is easy to track the number of waiters in an interlocked data - * structure and then add that number to the semaphore, this does not - * guarantee fairness or correctness. Consider the following situation: - * - * - t1 enters wait(), adding 1 to the wait counter & blocks on the semaphore - * - t2 enters wait(), adding 1 to the wait counter & blocks on the semaphore - * - tZ calls broadcast(), finds the wait counter is 2, adds 2 to the semaphore - * - * - t1 exits wait() - * - t1 immediately reenters wait() and blocks on the semaphore - * - The semaphore is still positive due to also having been signalled - * for t2, therefore it is decremented - * - t1 exits wait() immediately; t2 is never woken - * - * GENERATION COUNTERS. One naive solution to this is to use a generation - * counter. Each broadcast() invocation increments a generation counter. If - * the generation counter has not changed during a semaphore wait operation - * inside wait(), this indicates that no broadcast() call has been made in - * the meantime; therefore, the successful semaphore decrement must have - * 'stolen' a wakeup from another thread which was waiting to wakeup from the - * prior broadcast() call but which had not yet had a chance to do so. The - * semaphore can then be reincremented and the wait() operation repeated. - * - * However, this suffers from the obvious problem that without OS guarantees - * as to how semaphore readiness events are distributed amongst threads, - * there is no particular guarantee that the semaphore readiness event will - * not be immediately redistributed back to the same thread t1. - * - * SOLUTION. A solution is chosen as follows. In its initial state, a - * condition variable can accept waiters, who wait for the semaphore - * normally. However, once broadcast() is called, the condition - * variable becomes 'closed'. Any existing blocked threads are unblocked, - * but any new calls to wait() will instead enter a blocking pre-wait stage. - * Pre-wait threads are not considered to be waiting (and the external - * mutex remains held). A call to wait() in pre-wait cannot progress - * to waiting until all threads due to be unblocked by the prior broadcast() - * call have returned and had a chance to execute. - * - * This pre-wait does not affect a thread if it does not call wait() - * again until after all threads have had a chance to execute. - * - * RESOURCE USAGE. Aside from an allocation for the condition variable - * structure, this solution uses two Win32 semaphores. - * - * FUTURE OPTIMISATIONS: - * - * An optimised multi-generation implementation is possible at the cost of - * higher Win32 resource usage. Multiple 'buckets' could be defined, with - * usage rotating between buckets internally as buckets become closed. - * This would avoid the need for the prewait in more cases, depending - * on intensity of usage. - * - */ -typedef struct legacy_condvar_st { - CRYPTO_MUTEX *int_m; /* internal mutex */ - HANDLE sema; /* main wait semaphore */ - HANDLE prewait_sema; /* prewait semaphore */ - /* - * All of the following fields are protected by int_m. - * - * num_wake only ever increases by virtue of a corresponding decrease in - * num_wait. num_wait can decrease for other reasons (for example due to a - * wait operation timing out). - */ - size_t num_wait; /* Num. threads currently blocked */ - size_t num_wake; /* Num. threads due to wake up */ - size_t num_prewait; /* Num. threads in prewait */ - size_t gen; /* Prewait generation */ - int closed; /* Is closed? */ -} LEGACY_CONDVAR; - -CRYPTO_CONDVAR *ossl_crypto_condvar_new(void) -{ - LEGACY_CONDVAR *cv; - - if ((cv = OPENSSL_malloc(sizeof(LEGACY_CONDVAR))) == NULL) - return NULL; - - if ((cv->int_m = ossl_crypto_mutex_new()) == NULL) { - OPENSSL_free(cv); - return NULL; - } - - if ((cv->sema = CreateSemaphoreA(NULL, 0, LONG_MAX, NULL)) == NULL) { - ossl_crypto_mutex_free(&cv->int_m); - OPENSSL_free(cv); - return NULL; - } - - if ((cv->prewait_sema = CreateSemaphoreA(NULL, 0, LONG_MAX, NULL)) == NULL) { - CloseHandle(cv->sema); - ossl_crypto_mutex_free(&cv->int_m); - OPENSSL_free(cv); - return NULL; - } - - cv->num_wait = 0; - cv->num_wake = 0; - cv->num_prewait = 0; - cv->closed = 0; - - return (CRYPTO_CONDVAR *)cv; -} - -void ossl_crypto_condvar_free(CRYPTO_CONDVAR **cv_p) -{ - if (*cv_p != NULL) { - LEGACY_CONDVAR *cv = *(LEGACY_CONDVAR **)cv_p; - - CloseHandle(cv->sema); - CloseHandle(cv->prewait_sema); - ossl_crypto_mutex_free(&cv->int_m); - OPENSSL_free(cv); - } - - *cv_p = NULL; -} - -static uint32_t obj_wait(HANDLE h, OSSL_TIME deadline) -{ - DWORD timeout; - - if (!determine_timeout(deadline, &timeout)) - timeout = 1; - - return WaitForSingleObject(h, timeout); -} - -void ossl_crypto_condvar_wait_timeout(CRYPTO_CONDVAR *cv_, CRYPTO_MUTEX *ext_m, - OSSL_TIME deadline) -{ - LEGACY_CONDVAR *cv = (LEGACY_CONDVAR *)cv_; - int closed, set_prewait = 0, have_orig_gen = 0; - uint32_t rc; - size_t orig_gen; - - /* Admission control - prewait until we can enter our actual wait phase. */ - do { - ossl_crypto_mutex_lock(cv->int_m); - - closed = cv->closed; - - /* - * Once prewait is over the prewait semaphore is signalled and - * num_prewait is set to 0. Use a generation counter to track if we need - * to remove a value we added to num_prewait when exiting (e.g. due to - * timeout or failure of WaitForSingleObject). - */ - if (!have_orig_gen) { - orig_gen = cv->gen; - have_orig_gen = 1; - } else if (cv->gen != orig_gen) { - set_prewait = 0; - orig_gen = cv->gen; - } - - if (!closed) { - /* We can now be admitted. */ - ++cv->num_wait; - if (set_prewait) { - --cv->num_prewait; - set_prewait = 0; - } - } else if (!set_prewait) { - ++cv->num_prewait; - set_prewait = 1; - } - - ossl_crypto_mutex_unlock(cv->int_m); - - if (closed) - if (obj_wait(cv->prewait_sema, deadline) != WAIT_OBJECT_0) { - /* - * If we got WAIT_OBJECT_0 we are safe - num_prewait has been - * set to 0 and the semaphore has been consumed. On the other - * hand if we timed out, there may be a residual posting that - * was made just after we timed out. However in the worst case - * this will just cause an internal spurious wakeup here in the - * future, so we do not care too much about this. We treat - * failure and timeout cases as the same, and simply exit in - * this case. - */ - ossl_crypto_mutex_lock(cv->int_m); - if (set_prewait && cv->gen == orig_gen) - --cv->num_prewait; - ossl_crypto_mutex_unlock(cv->int_m); - return; - } - } while (closed); - - /* - * Unlock external mutex. Do not do this until we have been admitted, as we - * must guarantee we wake if broadcast is called at any time after ext_m is - * unlocked. - */ - ossl_crypto_mutex_unlock(ext_m); - - for (;;) { - /* Wait. */ - rc = obj_wait(cv->sema, deadline); - - /* Reacquire internal mutex and probe state. */ - ossl_crypto_mutex_lock(cv->int_m); - - if (cv->num_wake > 0) { - /* - * A wake token is available, so we can wake up. Consume the token - * and get out of here. We don't care what WaitForSingleObject - * returned here (e.g. if it timed out coincidentally). In the - * latter case a signal might be left in the semaphore which causes - * a future WaitForSingleObject call to return immediately, but in - * this case we will just loop again. - */ - --cv->num_wake; - if (cv->num_wake == 0 && cv->closed) { - /* - * We consumed the last wake token, so we can now open the - * condition variable for new admissions. - */ - cv->closed = 0; - if (cv->num_prewait > 0) { - ReleaseSemaphore(cv->prewait_sema, (LONG)cv->num_prewait, NULL); - cv->num_prewait = 0; - ++cv->gen; - } - } - } else if (rc == WAIT_OBJECT_0) { - /* - * We got a wakeup from the semaphore but we did not have any wake - * tokens. This ideally does not happen, but might if during a - * previous wait() call the semaphore is posted just after - * WaitForSingleObject returns due to a timeout (such that the - * num_wake > 0 case is taken above). Just spin again. (It is worth - * noting that repeated WaitForSingleObject calls is the only method - * documented for decrementing a Win32 semaphore, so this is - * basically the best possible strategy.) - */ - ossl_crypto_mutex_unlock(cv->int_m); - continue; - } else { - /* - * Assume we timed out. The WaitForSingleObject call may also have - * failed for some other reason, which we treat as a timeout. - */ - assert(cv->num_wait > 0); - --cv->num_wait; - } - - break; - } - - ossl_crypto_mutex_unlock(cv->int_m); - ossl_crypto_mutex_lock(ext_m); -} - -void ossl_crypto_condvar_wait(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *ext_m) -{ - ossl_crypto_condvar_wait_timeout(cv, ext_m, ossl_time_infinite()); -} - -void ossl_crypto_condvar_broadcast(CRYPTO_CONDVAR *cv_) -{ - LEGACY_CONDVAR *cv = (LEGACY_CONDVAR *)cv_; - size_t num_wake; - - ossl_crypto_mutex_lock(cv->int_m); - - num_wake = cv->num_wait; - if (num_wake == 0) { - ossl_crypto_mutex_unlock(cv->int_m); - return; - } - - cv->num_wake += num_wake; - cv->num_wait -= num_wake; - cv->closed = 1; - - ossl_crypto_mutex_unlock(cv->int_m); - ReleaseSemaphore(cv->sema, num_wake, NULL); -} - -void ossl_crypto_condvar_signal(CRYPTO_CONDVAR *cv_) -{ - LEGACY_CONDVAR *cv = (LEGACY_CONDVAR *)cv_; - - ossl_crypto_mutex_lock(cv->int_m); - - if (cv->num_wait == 0) { - ossl_crypto_mutex_unlock(cv->int_m); - return; - } - - /* - * We do not close the condition variable when merely signalling, as there - * are no guaranteed fairness semantics here, unlike for a broadcast. - */ - --cv->num_wait; - ++cv->num_wake; - - ossl_crypto_mutex_unlock(cv->int_m); - ReleaseSemaphore(cv->sema, 1, NULL); -} - -# else - -CRYPTO_CONDVAR *ossl_crypto_condvar_new(void) -{ - CONDITION_VARIABLE *cv_p; - - if ((cv_p = OPENSSL_zalloc(sizeof(*cv_p))) == NULL) - return NULL; - InitializeConditionVariable(cv_p); - return (CRYPTO_CONDVAR *)cv_p; -} - -void ossl_crypto_condvar_wait(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *mutex) -{ - CONDITION_VARIABLE *cv_p; - CRITICAL_SECTION *mutex_p; - - cv_p = (CONDITION_VARIABLE *)cv; - mutex_p = (CRITICAL_SECTION *)mutex; - SleepConditionVariableCS(cv_p, mutex_p, INFINITE); -} - -void ossl_crypto_condvar_wait_timeout(CRYPTO_CONDVAR *cv, CRYPTO_MUTEX *mutex, - OSSL_TIME deadline) -{ - DWORD timeout; - CONDITION_VARIABLE *cv_p = (CONDITION_VARIABLE *)cv; - CRITICAL_SECTION *mutex_p = (CRITICAL_SECTION *)mutex; - - if (!determine_timeout(deadline, &timeout)) - timeout = 1; - - SleepConditionVariableCS(cv_p, mutex_p, timeout); -} - -void ossl_crypto_condvar_broadcast(CRYPTO_CONDVAR *cv) -{ - CONDITION_VARIABLE *cv_p; - - cv_p = (CONDITION_VARIABLE *)cv; - WakeAllConditionVariable(cv_p); -} - -void ossl_crypto_condvar_signal(CRYPTO_CONDVAR *cv) -{ - CONDITION_VARIABLE *cv_p; - - cv_p = (CONDITION_VARIABLE *)cv; - WakeConditionVariable(cv_p); -} - -void ossl_crypto_condvar_free(CRYPTO_CONDVAR **cv) -{ - CONDITION_VARIABLE **cv_p; - - cv_p = (CONDITION_VARIABLE **)cv; - OPENSSL_free(*cv_p); - *cv_p = NULL; -} - -# endif - -void ossl_crypto_mem_barrier(void) -{ - MemoryBarrier(); -} - -#endif diff --git a/openssl/src/crypto/thread/internal.c b/openssl/src/crypto/thread/internal.c deleted file mode 100644 index 61486c8d4..000000000 --- a/openssl/src/crypto/thread/internal.c +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include -#include - -#if !defined(OPENSSL_NO_DEFAULT_THREAD_POOL) - -static ossl_inline uint64_t _ossl_get_avail_threads(OSSL_LIB_CTX_THREADS *tdata) -{ - /* assumes that tdata->lock is taken */ - return tdata->max_threads - tdata->active_threads; -} - -uint64_t ossl_get_avail_threads(OSSL_LIB_CTX *ctx) -{ - uint64_t retval = 0; - OSSL_LIB_CTX_THREADS *tdata = OSSL_LIB_CTX_GET_THREADS(ctx); - - if (tdata == NULL) - return retval; - - ossl_crypto_mutex_lock(tdata->lock); - retval = _ossl_get_avail_threads(tdata); - ossl_crypto_mutex_unlock(tdata->lock); - - return retval; -} - -void *ossl_crypto_thread_start(OSSL_LIB_CTX *ctx, CRYPTO_THREAD_ROUTINE start, - void *data) -{ - CRYPTO_THREAD *thread; - OSSL_LIB_CTX_THREADS *tdata = OSSL_LIB_CTX_GET_THREADS(ctx); - - if (tdata == NULL) - return NULL; - - ossl_crypto_mutex_lock(tdata->lock); - if (tdata == NULL || tdata->max_threads == 0) { - ossl_crypto_mutex_unlock(tdata->lock); - return NULL; - } - - while (_ossl_get_avail_threads(tdata) == 0) - ossl_crypto_condvar_wait(tdata->cond_finished, tdata->lock); - tdata->active_threads++; - ossl_crypto_mutex_unlock(tdata->lock); - - thread = ossl_crypto_thread_native_start(start, data, 1); - if (thread == NULL) { - ossl_crypto_mutex_lock(tdata->lock); - tdata->active_threads--; - ossl_crypto_mutex_unlock(tdata->lock); - goto fail; - } - thread->ctx = ctx; - -fail: - return (void *) thread; -} - -int ossl_crypto_thread_join(void *vhandle, CRYPTO_THREAD_RETVAL *retval) -{ - CRYPTO_THREAD *handle = vhandle; - OSSL_LIB_CTX_THREADS *tdata; - - if (vhandle == NULL) - return 0; - - tdata = OSSL_LIB_CTX_GET_THREADS(handle->ctx); - if (tdata == NULL) - return 0; - - if (ossl_crypto_thread_native_join(handle, retval) == 0) - return 0; - - ossl_crypto_mutex_lock(tdata->lock); - tdata->active_threads--; - ossl_crypto_condvar_signal(tdata->cond_finished); - ossl_crypto_mutex_unlock(tdata->lock); - return 1; -} - -int ossl_crypto_thread_clean(void *vhandle) -{ - CRYPTO_THREAD *handle = vhandle; - - return ossl_crypto_thread_native_clean(handle); -} - -#else - -ossl_inline uint64_t ossl_get_avail_threads(OSSL_LIB_CTX *ctx) -{ - return 0; -} - -void *ossl_crypto_thread_start(OSSL_LIB_CTX *ctx, CRYPTO_THREAD_ROUTINE start, - void *data) -{ - return NULL; -} - -int ossl_crypto_thread_join(void *vhandle, CRYPTO_THREAD_RETVAL *retval) -{ - return 0; -} - -int ossl_crypto_thread_clean(void *vhandle) -{ - return 0; -} - -#endif - -void *ossl_threads_ctx_new(OSSL_LIB_CTX *ctx) -{ - struct openssl_threads_st *t = OPENSSL_zalloc(sizeof(*t)); - - if (t == NULL) - return NULL; - - t->lock = ossl_crypto_mutex_new(); - t->cond_finished = ossl_crypto_condvar_new(); - - if (t->lock == NULL || t->cond_finished == NULL) - goto fail; - - return t; - -fail: - ossl_threads_ctx_free((void *)t); - return NULL; -} - -void ossl_threads_ctx_free(void *vdata) -{ - OSSL_LIB_CTX_THREADS *t = (OSSL_LIB_CTX_THREADS *) vdata; - - if (t == NULL) - return; - - ossl_crypto_mutex_free(&t->lock); - ossl_crypto_condvar_free(&t->cond_finished); - OPENSSL_free(t); -} diff --git a/openssl/src/crypto/threads_lib.c b/openssl/src/crypto/threads_lib.c index 240ef726e..0c7162392 100644 --- a/openssl/src/crypto/threads_lib.c +++ b/openssl/src/crypto/threads_lib.c @@ -8,8 +8,7 @@ */ #include -#ifdef OPENSSL_SYS_UNIX -# ifndef OPENSSL_NO_DEPRECATED_3_0 +#ifndef OPENSSL_NO_DEPRECATED_3_0 void OPENSSL_fork_prepare(void) { @@ -23,5 +22,4 @@ void OPENSSL_fork_child(void) { } -# endif #endif diff --git a/openssl/src/crypto/threads_none.c b/openssl/src/crypto/threads_none.c index c57c59bde..2570efde2 100644 --- a/openssl/src/crypto/threads_none.c +++ b/openssl/src/crypto/threads_none.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,8 +9,6 @@ #include #include "internal/cryptlib.h" -#include "internal/rcu.h" -#include "rcu_internal.h" #if !defined(OPENSSL_THREADS) || defined(CRYPTO_TDEBUG) @@ -19,89 +17,14 @@ # include # endif -struct rcu_lock_st { - struct rcu_cb_item *cb_items; -}; - -CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers) -{ - struct rcu_lock_st *lock; - - lock = OPENSSL_zalloc(sizeof(*lock)); - return lock; -} - -void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock) -{ - OPENSSL_free(lock); -} - -void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock) -{ - return; -} - -void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock) -{ - return; -} - -void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock) -{ - return; -} - -void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock) -{ - return; -} - -void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock) -{ - struct rcu_cb_item *items = lock->cb_items; - struct rcu_cb_item *tmp; - - lock->cb_items = NULL; - - while (items != NULL) { - tmp = items->next; - items->fn(items->data); - OPENSSL_free(items); - items = tmp; - } -} - -int ossl_rcu_call(CRYPTO_RCU_LOCK *lock, rcu_cb_fn cb, void *data) -{ - struct rcu_cb_item *new = OPENSSL_zalloc(sizeof(*new)); - - if (new == NULL) - return 0; - - new->fn = cb; - new->data = data; - new->next = lock->cb_items; - lock->cb_items = new; - return 1; -} - -void *ossl_rcu_uptr_deref(void **p) -{ - return (void *)*p; -} - -void ossl_rcu_assign_uptr(void **p, void **v) -{ - *(void **)p = *(void **)v; -} - CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) { CRYPTO_RWLOCK *lock; - if ((lock = CRYPTO_zalloc(sizeof(unsigned int), NULL, 0)) == NULL) + if ((lock = OPENSSL_zalloc(sizeof(unsigned int))) == NULL) { /* Don't set error, to avoid recursion blowup. */ return NULL; + } *(unsigned int *)lock = 1; @@ -226,13 +149,6 @@ int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock) return 1; } -int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock) -{ - *ret = *val; - - return 1; -} - int openssl_init_fork_handlers(void) { return 0; diff --git a/openssl/src/crypto/threads_pthread.c b/openssl/src/crypto/threads_pthread.c index 92346e168..bfc05a4e8 100644 --- a/openssl/src/crypto/threads_pthread.c +++ b/openssl/src/crypto/threads_pthread.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,10 +11,7 @@ #define OPENSSL_SUPPRESS_DEPRECATED #include -#include #include "internal/cryptlib.h" -#include "internal/rcu.h" -#include "rcu_internal.h" #if defined(__sun) # include @@ -29,7 +26,7 @@ * * See: https://github.com/llvm/llvm-project/commit/a4c2602b714e6c6edb98164550a5ae829b2de760 */ -# define BROKEN_CLANG_ATOMICS +#define BROKEN_CLANG_ATOMICS #endif #if defined(OPENSSL_THREADS) && !defined(CRYPTO_TDEBUG) && !defined(OPENSSL_SYS_WINDOWS) @@ -37,7 +34,7 @@ # if defined(OPENSSL_SYS_UNIX) # include # include -# endif +#endif # include @@ -45,650 +42,15 @@ # define USE_RWLOCK # endif -/* - * For all GNU/clang atomic builtins, we also need fallbacks, to cover all - * other compilers. - - * Unfortunately, we can't do that with some "generic type", because there's no - * guarantee that the chosen generic type is large enough to cover all cases. - * Therefore, we implement fallbacks for each applicable type, with composed - * names that include the type they handle. - * - * (an anecdote: we previously tried to use |void *| as the generic type, with - * the thought that the pointer itself is the largest type. However, this is - * not true on 32-bit pointer platforms, as a |uint64_t| is twice as large) - * - * All applicable ATOMIC_ macros take the intended type as first parameter, so - * they can map to the correct fallback function. In the GNU/clang case, that - * parameter is simply ignored. - */ - -/* - * Internal types used with the ATOMIC_ macros, to make it possible to compose - * fallback function names. - */ -typedef void *pvoid; -typedef struct rcu_cb_item *prcu_cb_item; - -# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS) \ - && !defined(USE_ATOMIC_FALLBACKS) -# if defined(__APPLE__) && defined(__clang__) && defined(__aarch64__) -/* - * For pointers, Apple M1 virtualized cpu seems to have some problem using the - * ldapr instruction (see https://github.com/openssl/openssl/pull/23974) - * When using the native apple clang compiler, this instruction is emitted for - * atomic loads, which is bad. So, if - * 1) We are building on a target that defines __APPLE__ AND - * 2) We are building on a target using clang (__clang__) AND - * 3) We are building for an M1 processor (__aarch64__) - * Then we shold not use __atomic_load_n and instead implement our own - * function to issue the ldar instruction instead, which procuces the proper - * sequencing guarantees - */ -static inline void *apple_atomic_load_n_pvoid(void **p, - ossl_unused int memorder) -{ - void *ret; - - __asm volatile("ldar %0, [%1]" : "=r" (ret): "r" (p):); - - return ret; -} - -/* For uint64_t, we should be fine, though */ -# define apple_atomic_load_n_uint64_t(p, o) __atomic_load_n(p, o) - -# define ATOMIC_LOAD_N(t, p, o) apple_atomic_load_n_##t(p, o) -# else -# define ATOMIC_LOAD_N(t, p, o) __atomic_load_n(p, o) -# endif -# define ATOMIC_STORE_N(t, p, v, o) __atomic_store_n(p, v, o) -# define ATOMIC_STORE(t, p, v, o) __atomic_store(p, v, o) -# define ATOMIC_EXCHANGE_N(t, p, v, o) __atomic_exchange_n(p, v, o) -# define ATOMIC_ADD_FETCH(p, v, o) __atomic_add_fetch(p, v, o) -# define ATOMIC_FETCH_ADD(p, v, o) __atomic_fetch_add(p, v, o) -# define ATOMIC_SUB_FETCH(p, v, o) __atomic_sub_fetch(p, v, o) -# define ATOMIC_AND_FETCH(p, m, o) __atomic_and_fetch(p, m, o) -# define ATOMIC_OR_FETCH(p, m, o) __atomic_or_fetch(p, m, o) -# else -static pthread_mutex_t atomic_sim_lock = PTHREAD_MUTEX_INITIALIZER; - -# define IMPL_fallback_atomic_load_n(t) \ - static ossl_inline t fallback_atomic_load_n_##t(t *p) \ - { \ - t ret; \ - \ - pthread_mutex_lock(&atomic_sim_lock); \ - ret = *p; \ - pthread_mutex_unlock(&atomic_sim_lock); \ - return ret; \ - } -IMPL_fallback_atomic_load_n(uint64_t) -IMPL_fallback_atomic_load_n(pvoid) - -# define ATOMIC_LOAD_N(t, p, o) fallback_atomic_load_n_##t(p) - -# define IMPL_fallback_atomic_store_n(t) \ - static ossl_inline t fallback_atomic_store_n_##t(t *p, t v) \ - { \ - t ret; \ - \ - pthread_mutex_lock(&atomic_sim_lock); \ - ret = *p; \ - *p = v; \ - pthread_mutex_unlock(&atomic_sim_lock); \ - return ret; \ - } -IMPL_fallback_atomic_store_n(uint64_t) - -# define ATOMIC_STORE_N(t, p, v, o) fallback_atomic_store_n_##t(p, v) - -# define IMPL_fallback_atomic_store(t) \ - static ossl_inline void fallback_atomic_store_##t(t *p, t *v) \ - { \ - pthread_mutex_lock(&atomic_sim_lock); \ - *p = *v; \ - pthread_mutex_unlock(&atomic_sim_lock); \ - } -IMPL_fallback_atomic_store(uint64_t) -IMPL_fallback_atomic_store(pvoid) - -# define ATOMIC_STORE(t, p, v, o) fallback_atomic_store_##t(p, v) - -# define IMPL_fallback_atomic_exchange_n(t) \ - static ossl_inline t fallback_atomic_exchange_n_##t(t *p, t v) \ - { \ - t ret; \ - \ - pthread_mutex_lock(&atomic_sim_lock); \ - ret = *p; \ - *p = v; \ - pthread_mutex_unlock(&atomic_sim_lock); \ - return ret; \ - } -IMPL_fallback_atomic_exchange_n(uint64_t) -IMPL_fallback_atomic_exchange_n(prcu_cb_item) - -# define ATOMIC_EXCHANGE_N(t, p, v, o) fallback_atomic_exchange_n_##t(p, v) - -/* - * The fallbacks that follow don't need any per type implementation, as - * they are designed for uint64_t only. If there comes a time when multiple - * types need to be covered, it's relatively easy to refactor them the same - * way as the fallbacks above. - */ - -static ossl_inline uint64_t fallback_atomic_add_fetch(uint64_t *p, uint64_t v) -{ - uint64_t ret; - - pthread_mutex_lock(&atomic_sim_lock); - *p += v; - ret = *p; - pthread_mutex_unlock(&atomic_sim_lock); - return ret; -} - -# define ATOMIC_ADD_FETCH(p, v, o) fallback_atomic_add_fetch(p, v) - -static ossl_inline uint64_t fallback_atomic_fetch_add(uint64_t *p, uint64_t v) -{ - uint64_t ret; - - pthread_mutex_lock(&atomic_sim_lock); - ret = *p; - *p += v; - pthread_mutex_unlock(&atomic_sim_lock); - return ret; -} - -# define ATOMIC_FETCH_ADD(p, v, o) fallback_atomic_fetch_add(p, v) - -static ossl_inline uint64_t fallback_atomic_sub_fetch(uint64_t *p, uint64_t v) -{ - uint64_t ret; - - pthread_mutex_lock(&atomic_sim_lock); - *p -= v; - ret = *p; - pthread_mutex_unlock(&atomic_sim_lock); - return ret; -} - -# define ATOMIC_SUB_FETCH(p, v, o) fallback_atomic_sub_fetch(p, v) - -static ossl_inline uint64_t fallback_atomic_and_fetch(uint64_t *p, uint64_t m) -{ - uint64_t ret; - - pthread_mutex_lock(&atomic_sim_lock); - *p &= m; - ret = *p; - pthread_mutex_unlock(&atomic_sim_lock); - return ret; -} - -# define ATOMIC_AND_FETCH(p, v, o) fallback_atomic_and_fetch(p, v) - -static ossl_inline uint64_t fallback_atomic_or_fetch(uint64_t *p, uint64_t m) -{ - uint64_t ret; - - pthread_mutex_lock(&atomic_sim_lock); - *p |= m; - ret = *p; - pthread_mutex_unlock(&atomic_sim_lock); - return ret; -} - -# define ATOMIC_OR_FETCH(p, v, o) fallback_atomic_or_fetch(p, v) -# endif - -static CRYPTO_THREAD_LOCAL rcu_thr_key; - -/* - * users is broken up into 2 parts - * bits 0-15 current readers - * bit 32-63 - ID - */ -# define READER_SHIFT 0 -# define ID_SHIFT 32 -# define READER_SIZE 16 -# define ID_SIZE 32 - -# define READER_MASK (((uint64_t)1 << READER_SIZE) - 1) -# define ID_MASK (((uint64_t)1 << ID_SIZE) - 1) -# define READER_COUNT(x) (((uint64_t)(x) >> READER_SHIFT) & READER_MASK) -# define ID_VAL(x) (((uint64_t)(x) >> ID_SHIFT) & ID_MASK) -# define VAL_READER ((uint64_t)1 << READER_SHIFT) -# define VAL_ID(x) ((uint64_t)x << ID_SHIFT) - -/* - * This is the core of an rcu lock. It tracks the readers and writers for the - * current quiescence point for a given lock. Users is the 64 bit value that - * stores the READERS/ID as defined above - * - */ -struct rcu_qp { - uint64_t users; -}; - -struct thread_qp { - struct rcu_qp *qp; - unsigned int depth; - CRYPTO_RCU_LOCK *lock; -}; - -# define MAX_QPS 10 -/* - * This is the per thread tracking data - * that is assigned to each thread participating - * in an rcu qp - * - * qp points to the qp that it last acquired - * - */ -struct rcu_thr_data { - struct thread_qp thread_qps[MAX_QPS]; -}; - -/* - * This is the internal version of a CRYPTO_RCU_LOCK - * it is cast from CRYPTO_RCU_LOCK - */ -struct rcu_lock_st { - /* Callbacks to call for next ossl_synchronize_rcu */ - struct rcu_cb_item *cb_items; - - /* rcu generation counter for in-order retirement */ - uint32_t id_ctr; - - /* Array of quiescent points for synchronization */ - struct rcu_qp *qp_group; - - /* Number of elements in qp_group array */ - size_t group_count; - - /* Index of the current qp in the qp_group array */ - uint64_t reader_idx; - - /* value of the next id_ctr value to be retired */ - uint32_t next_to_retire; - - /* index of the next free rcu_qp in the qp_group */ - uint64_t current_alloc_idx; - - /* number of qp's in qp_group array currently being retired */ - uint32_t writers_alloced; - - /* lock protecting write side operations */ - pthread_mutex_t write_lock; - - /* lock protecting updates to writers_alloced/current_alloc_idx */ - pthread_mutex_t alloc_lock; - - /* signal to wake threads waiting on alloc_lock */ - pthread_cond_t alloc_signal; - - /* lock to enforce in-order retirement */ - pthread_mutex_t prior_lock; - - /* signal to wake threads waiting on prior_lock */ - pthread_cond_t prior_signal; -}; - -/* - * Called on thread exit to free the pthread key - * associated with this thread, if any - */ -static void free_rcu_thr_data(void *ptr) -{ - struct rcu_thr_data *data = - (struct rcu_thr_data *)CRYPTO_THREAD_get_local(&rcu_thr_key); - - OPENSSL_free(data); - CRYPTO_THREAD_set_local(&rcu_thr_key, NULL); -} - -static void ossl_rcu_init(void) -{ - CRYPTO_THREAD_init_local(&rcu_thr_key, NULL); -} - -/* Read side acquisition of the current qp */ -static struct rcu_qp *get_hold_current_qp(struct rcu_lock_st *lock) -{ - uint64_t qp_idx; - - /* get the current qp index */ - for (;;) { - /* - * Notes on use of __ATOMIC_ACQUIRE - * We need to ensure the following: - * 1) That subsequent operations aren't optimized by hoisting them above - * this operation. Specifically, we don't want the below re-load of - * qp_idx to get optimized away - * 2) We want to ensure that any updating of reader_idx on the write side - * of the lock is flushed from a local cpu cache so that we see any - * updates prior to the load. This is a non-issue on cache coherent - * systems like x86, but is relevant on other arches - * Note: This applies to the reload below as well - */ - qp_idx = ATOMIC_LOAD_N(uint64_t, &lock->reader_idx, __ATOMIC_ACQUIRE); - - /* - * Notes of use of __ATOMIC_RELEASE - * This counter is only read by the write side of the lock, and so we - * specify __ATOMIC_RELEASE here to ensure that the write side of the - * lock see this during the spin loop read of users, as it waits for the - * reader count to approach zero - */ - ATOMIC_ADD_FETCH(&lock->qp_group[qp_idx].users, VAL_READER, - __ATOMIC_RELEASE); - - /* if the idx hasn't changed, we're good, else try again */ - if (qp_idx == ATOMIC_LOAD_N(uint64_t, &lock->reader_idx, __ATOMIC_ACQUIRE)) - break; - - /* - * Notes on use of __ATOMIC_RELEASE - * As with the add above, we want to ensure that this decrement is - * seen by the write side of the lock as soon as it happens to prevent - * undue spinning waiting for write side completion - */ - ATOMIC_SUB_FETCH(&lock->qp_group[qp_idx].users, VAL_READER, - __ATOMIC_RELEASE); - } - - return &lock->qp_group[qp_idx]; -} - -void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock) -{ - struct rcu_thr_data *data; - int i, available_qp = -1; - - /* - * we're going to access current_qp here so ask the - * processor to fetch it - */ - data = CRYPTO_THREAD_get_local(&rcu_thr_key); - - if (data == NULL) { - data = OPENSSL_zalloc(sizeof(*data)); - OPENSSL_assert(data != NULL); - CRYPTO_THREAD_set_local(&rcu_thr_key, data); - ossl_init_thread_start(NULL, NULL, free_rcu_thr_data); - } - - for (i = 0; i < MAX_QPS; i++) { - if (data->thread_qps[i].qp == NULL && available_qp == -1) - available_qp = i; - /* If we have a hold on this lock already, we're good */ - if (data->thread_qps[i].lock == lock) { - data->thread_qps[i].depth++; - return; - } - } - - /* - * if we get here, then we don't have a hold on this lock yet - */ - assert(available_qp != -1); - - data->thread_qps[available_qp].qp = get_hold_current_qp(lock); - data->thread_qps[available_qp].depth = 1; - data->thread_qps[available_qp].lock = lock; -} - -void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock) -{ - int i; - struct rcu_thr_data *data = CRYPTO_THREAD_get_local(&rcu_thr_key); - uint64_t ret; - - assert(data != NULL); - - for (i = 0; i < MAX_QPS; i++) { - if (data->thread_qps[i].lock == lock) { - /* - * As with read side acquisition, we use __ATOMIC_RELEASE here - * to ensure that the decrement is published immediately - * to any write side waiters - */ - data->thread_qps[i].depth--; - if (data->thread_qps[i].depth == 0) { - ret = ATOMIC_SUB_FETCH(&data->thread_qps[i].qp->users, VAL_READER, - __ATOMIC_RELEASE); - OPENSSL_assert(ret != UINT64_MAX); - data->thread_qps[i].qp = NULL; - data->thread_qps[i].lock = NULL; - } - return; - } - } - /* - * If we get here, we're trying to unlock a lock that we never acquired - - * that's fatal. - */ - assert(0); -} - -/* - * Write side allocation routine to get the current qp - * and replace it with a new one - */ -static struct rcu_qp *update_qp(CRYPTO_RCU_LOCK *lock) -{ - uint64_t new_id; - uint64_t current_idx; - - pthread_mutex_lock(&lock->alloc_lock); - - /* - * we need at least one qp to be available with one - * left over, so that readers can start working on - * one that isn't yet being waited on - */ - while (lock->group_count - lock->writers_alloced < 2) - /* we have to wait for one to be free */ - pthread_cond_wait(&lock->alloc_signal, &lock->alloc_lock); - - current_idx = lock->current_alloc_idx; - - /* Allocate the qp */ - lock->writers_alloced++; - - /* increment the allocation index */ - lock->current_alloc_idx = - (lock->current_alloc_idx + 1) % lock->group_count; - - /* get and insert a new id */ - new_id = lock->id_ctr; - lock->id_ctr++; - - new_id = VAL_ID(new_id); - /* - * Even though we are under a write side lock here - * We need to use atomic instructions to ensure that the results - * of this update are published to the read side prior to updating the - * reader idx below - */ - ATOMIC_AND_FETCH(&lock->qp_group[current_idx].users, ID_MASK, - __ATOMIC_RELEASE); - ATOMIC_OR_FETCH(&lock->qp_group[current_idx].users, new_id, - __ATOMIC_RELEASE); - - /* - * Update the reader index to be the prior qp. - * Note the use of __ATOMIC_RELEASE here is based on the corresponding use - * of __ATOMIC_ACQUIRE in get_hold_current_qp, as we want any publication - * of this value to be seen on the read side immediately after it happens - */ - ATOMIC_STORE_N(uint64_t, &lock->reader_idx, lock->current_alloc_idx, - __ATOMIC_RELEASE); - - /* wake up any waiters */ - pthread_cond_signal(&lock->alloc_signal); - pthread_mutex_unlock(&lock->alloc_lock); - return &lock->qp_group[current_idx]; -} - -static void retire_qp(CRYPTO_RCU_LOCK *lock, struct rcu_qp *qp) -{ - pthread_mutex_lock(&lock->alloc_lock); - lock->writers_alloced--; - pthread_cond_signal(&lock->alloc_signal); - pthread_mutex_unlock(&lock->alloc_lock); -} - -static struct rcu_qp *allocate_new_qp_group(CRYPTO_RCU_LOCK *lock, - int count) -{ - struct rcu_qp *new = - OPENSSL_zalloc(sizeof(*new) * count); - - lock->group_count = count; - return new; -} - -void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock) -{ - pthread_mutex_lock(&lock->write_lock); -} - -void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock) -{ - pthread_mutex_unlock(&lock->write_lock); -} - -void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock) -{ - struct rcu_qp *qp; - uint64_t count; - struct rcu_cb_item *cb_items, *tmpcb; - - /* - * __ATOMIC_ACQ_REL is used here to ensure that we get any prior published - * writes before we read, and publish our write immediately - */ - cb_items = ATOMIC_EXCHANGE_N(prcu_cb_item, &lock->cb_items, NULL, - __ATOMIC_ACQ_REL); - - qp = update_qp(lock); - - /* - * wait for the reader count to reach zero - * Note the use of __ATOMIC_ACQUIRE here to ensure that any - * prior __ATOMIC_RELEASE write operation in get_hold_current_qp - * is visible prior to our read - */ - do { - count = ATOMIC_LOAD_N(uint64_t, &qp->users, __ATOMIC_ACQUIRE); - } while (READER_COUNT(count) != 0); - - /* retire in order */ - pthread_mutex_lock(&lock->prior_lock); - while (lock->next_to_retire != ID_VAL(count)) - pthread_cond_wait(&lock->prior_signal, &lock->prior_lock); - lock->next_to_retire++; - pthread_cond_broadcast(&lock->prior_signal); - pthread_mutex_unlock(&lock->prior_lock); - - retire_qp(lock, qp); - - /* handle any callbacks that we have */ - while (cb_items != NULL) { - tmpcb = cb_items; - cb_items = cb_items->next; - tmpcb->fn(tmpcb->data); - OPENSSL_free(tmpcb); - } -} - -int ossl_rcu_call(CRYPTO_RCU_LOCK *lock, rcu_cb_fn cb, void *data) -{ - struct rcu_cb_item *new = - OPENSSL_zalloc(sizeof(*new)); - - if (new == NULL) - return 0; - - new->data = data; - new->fn = cb; - /* - * Use __ATOMIC_ACQ_REL here to indicate that any prior writes to this - * list are visible to us prior to reading, and publish the new value - * immediately - */ - new->next = ATOMIC_EXCHANGE_N(prcu_cb_item, &lock->cb_items, new, - __ATOMIC_ACQ_REL); - - return 1; -} - -void *ossl_rcu_uptr_deref(void **p) -{ - return ATOMIC_LOAD_N(pvoid, p, __ATOMIC_ACQUIRE); -} - -void ossl_rcu_assign_uptr(void **p, void **v) -{ - ATOMIC_STORE(pvoid, p, v, __ATOMIC_RELEASE); -} - -static CRYPTO_ONCE rcu_init_once = CRYPTO_ONCE_STATIC_INIT; - -CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers) -{ - struct rcu_lock_st *new; - - if (!CRYPTO_THREAD_run_once(&rcu_init_once, ossl_rcu_init)) - return NULL; - - if (num_writers < 1) - num_writers = 1; - - new = OPENSSL_zalloc(sizeof(*new)); - if (new == NULL) - return NULL; - - pthread_mutex_init(&new->write_lock, NULL); - pthread_mutex_init(&new->prior_lock, NULL); - pthread_mutex_init(&new->alloc_lock, NULL); - pthread_cond_init(&new->prior_signal, NULL); - pthread_cond_init(&new->alloc_signal, NULL); - new->qp_group = allocate_new_qp_group(new, num_writers + 1); - if (new->qp_group == NULL) { - OPENSSL_free(new); - new = NULL; - } - return new; -} - -void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock) -{ - struct rcu_lock_st *rlock = (struct rcu_lock_st *)lock; - - if (lock == NULL) - return; - - /* make sure we're synchronized */ - ossl_synchronize_rcu(rlock); - - OPENSSL_free(rlock->qp_group); - /* There should only be a single qp left now */ - OPENSSL_free(rlock); -} - CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) { # ifdef USE_RWLOCK CRYPTO_RWLOCK *lock; - if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL) + if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL) { /* Don't set error, to avoid recursion blowup. */ return NULL; + } if (pthread_rwlock_init(lock, NULL) != 0) { OPENSSL_free(lock); @@ -698,9 +60,10 @@ CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) pthread_mutexattr_t attr; CRYPTO_RWLOCK *lock; - if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL) + if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL) { /* Don't set error, to avoid recursion blowup. */ return NULL; + } /* * We don't use recursive mutexes, but try to catch errors if we do. @@ -709,6 +72,8 @@ CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) # if !defined (__TANDEM) && !defined (_SPT_MODEL_) # if !defined(NDEBUG) && !defined(OPENSSL_NO_MUTEX_ERRORCHECK) pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); +# else + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); # endif # else /* The SPT Thread Library does not define MUTEX attributes. */ @@ -907,30 +272,6 @@ int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock) return 1; } - -int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock) -{ -# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS) - if (__atomic_is_lock_free(sizeof(*val), val)) { - __atomic_load(val, ret, __ATOMIC_ACQUIRE); - return 1; - } -# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) - /* This will work for all future Solaris versions. */ - if (ret != NULL) { - *ret = (int *)atomic_or_uint_nv((unsigned int *)val, 0); - return 1; - } -# endif - if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) - return 0; - *ret = *val; - if (!CRYPTO_THREAD_unlock(lock)) - return 0; - - return 1; -} - # ifndef FIPS_MODULE int openssl_init_fork_handlers(void) { diff --git a/openssl/src/crypto/threads_win.c b/openssl/src/crypto/threads_win.c index 64354dc42..d65b3826d 100644 --- a/openssl/src/crypto/threads_win.c +++ b/openssl/src/crypto/threads_win.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,26 +13,8 @@ # define USE_RWLOCK # endif #endif -#include - -/* - * VC++ 2008 or earlier x86 compilers do not have an inline implementation - * of InterlockedOr64 for 32bit and will fail to run on Windows XP 32bit. - * https://docs.microsoft.com/en-us/cpp/intrinsics/interlockedor-intrinsic-functions#requirements - * To work around this problem, we implement a manual locking mechanism for - * only VC++ 2008 or earlier x86 compilers. - */ - -#if (defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER <= 1600) -# define NO_INTERLOCKEDOR64 -#endif #include -#include -#include "internal/common.h" -#include "internal/thread_arch.h" -#include "internal/rcu.h" -#include "rcu_internal.h" #if defined(OPENSSL_THREADS) && !defined(CRYPTO_TDEBUG) && defined(OPENSSL_SYS_WINDOWS) @@ -43,355 +25,6 @@ typedef struct { } CRYPTO_win_rwlock; # endif -static CRYPTO_THREAD_LOCAL rcu_thr_key; - -# define READER_SHIFT 0 -# define ID_SHIFT 32 -# define READER_SIZE 32 -# define ID_SIZE 32 - -# define READER_MASK (((LONG64)1 << READER_SIZE)-1) -# define ID_MASK (((LONG64)1 << ID_SIZE)-1) -# define READER_COUNT(x) (((LONG64)(x) >> READER_SHIFT) & READER_MASK) -# define ID_VAL(x) (((LONG64)(x) >> ID_SHIFT) & ID_MASK) -# define VAL_READER ((LONG64)1 << READER_SHIFT) -# define VAL_ID(x) ((LONG64)x << ID_SHIFT) - -/* - * This defines a quescent point (qp) - * This is the barrier beyond which a writer - * must wait before freeing data that was - * atomically updated - */ -struct rcu_qp { - volatile LONG64 users; -}; - -struct thread_qp { - struct rcu_qp *qp; - unsigned int depth; - CRYPTO_RCU_LOCK *lock; -}; - -#define MAX_QPS 10 -/* - * This is the per thread tracking data - * that is assigned to each thread participating - * in an rcu qp - * - * qp points to the qp that it last acquired - * - */ -struct rcu_thr_data { - struct thread_qp thread_qps[MAX_QPS]; -}; - -/* - * This is the internal version of a CRYPTO_RCU_LOCK - * it is cast from CRYPTO_RCU_LOCK - */ -struct rcu_lock_st { - struct rcu_cb_item *cb_items; - uint32_t id_ctr; - struct rcu_qp *qp_group; - size_t group_count; - uint32_t next_to_retire; - volatile long int reader_idx; - uint32_t current_alloc_idx; - uint32_t writers_alloced; - CRYPTO_MUTEX *write_lock; - CRYPTO_MUTEX *alloc_lock; - CRYPTO_CONDVAR *alloc_signal; - CRYPTO_MUTEX *prior_lock; - CRYPTO_CONDVAR *prior_signal; -}; - -/* - * Called on thread exit to free the pthread key - * associated with this thread, if any - */ -static void free_rcu_thr_data(void *ptr) -{ - struct rcu_thr_data *data = - (struct rcu_thr_data *)CRYPTO_THREAD_get_local(&rcu_thr_key); - - OPENSSL_free(data); - CRYPTO_THREAD_set_local(&rcu_thr_key, NULL); -} - - -static void ossl_rcu_init(void) -{ - CRYPTO_THREAD_init_local(&rcu_thr_key, NULL); - ossl_init_thread_start(NULL, NULL, free_rcu_thr_data); -} - -static struct rcu_qp *allocate_new_qp_group(struct rcu_lock_st *lock, - int count) -{ - struct rcu_qp *new = - OPENSSL_zalloc(sizeof(*new) * count); - - lock->group_count = count; - return new; -} - -static CRYPTO_ONCE rcu_init_once = CRYPTO_ONCE_STATIC_INIT; - -CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers) -{ - struct rcu_lock_st *new; - - if (!CRYPTO_THREAD_run_once(&rcu_init_once, ossl_rcu_init)) - return NULL; - - if (num_writers < 1) - num_writers = 1; - - new = OPENSSL_zalloc(sizeof(*new)); - - if (new == NULL) - return NULL; - - new->write_lock = ossl_crypto_mutex_new(); - new->alloc_signal = ossl_crypto_condvar_new(); - new->prior_signal = ossl_crypto_condvar_new(); - new->alloc_lock = ossl_crypto_mutex_new(); - new->prior_lock = ossl_crypto_mutex_new(); - new->qp_group = allocate_new_qp_group(new, num_writers + 1); - if (new->qp_group == NULL - || new->alloc_signal == NULL - || new->prior_signal == NULL - || new->write_lock == NULL - || new->alloc_lock == NULL - || new->prior_lock == NULL) { - OPENSSL_free(new->qp_group); - ossl_crypto_condvar_free(&new->alloc_signal); - ossl_crypto_condvar_free(&new->prior_signal); - ossl_crypto_mutex_free(&new->alloc_lock); - ossl_crypto_mutex_free(&new->prior_lock); - ossl_crypto_mutex_free(&new->write_lock); - OPENSSL_free(new); - new = NULL; - } - return new; - -} - -void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock) -{ - OPENSSL_free(lock->qp_group); - ossl_crypto_condvar_free(&lock->alloc_signal); - ossl_crypto_condvar_free(&lock->prior_signal); - ossl_crypto_mutex_free(&lock->alloc_lock); - ossl_crypto_mutex_free(&lock->prior_lock); - ossl_crypto_mutex_free(&lock->write_lock); - OPENSSL_free(lock); -} - -static ossl_inline struct rcu_qp *get_hold_current_qp(CRYPTO_RCU_LOCK *lock) -{ - uint32_t qp_idx; - - /* get the current qp index */ - for (;;) { - qp_idx = InterlockedOr(&lock->reader_idx, 0); - InterlockedAdd64(&lock->qp_group[qp_idx].users, VAL_READER); - if (qp_idx == InterlockedOr(&lock->reader_idx, 0)) - break; - InterlockedAdd64(&lock->qp_group[qp_idx].users, -VAL_READER); - } - - return &lock->qp_group[qp_idx]; -} - -void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock) -{ - struct rcu_thr_data *data; - int i; - int available_qp = -1; - - /* - * we're going to access current_qp here so ask the - * processor to fetch it - */ - data = CRYPTO_THREAD_get_local(&rcu_thr_key); - - if (data == NULL) { - data = OPENSSL_zalloc(sizeof(*data)); - OPENSSL_assert(data != NULL); - CRYPTO_THREAD_set_local(&rcu_thr_key, data); - } - - for (i = 0; i < MAX_QPS; i++) { - if (data->thread_qps[i].qp == NULL && available_qp == -1) - available_qp = i; - /* If we have a hold on this lock already, we're good */ - if (data->thread_qps[i].lock == lock) - return; - } - - /* - * if we get here, then we don't have a hold on this lock yet - */ - assert(available_qp != -1); - - data->thread_qps[available_qp].qp = get_hold_current_qp(lock); - data->thread_qps[available_qp].depth = 1; - data->thread_qps[available_qp].lock = lock; -} - -void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock) -{ - ossl_crypto_mutex_lock(lock->write_lock); -} - -void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock) -{ - ossl_crypto_mutex_unlock(lock->write_lock); -} - -void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock) -{ - struct rcu_thr_data *data = CRYPTO_THREAD_get_local(&rcu_thr_key); - int i; - LONG64 ret; - - assert(data != NULL); - - for (i = 0; i < MAX_QPS; i++) { - if (data->thread_qps[i].lock == lock) { - data->thread_qps[i].depth--; - if (data->thread_qps[i].depth == 0) { - ret = InterlockedAdd64(&data->thread_qps[i].qp->users, -VAL_READER); - OPENSSL_assert(ret >= 0); - data->thread_qps[i].qp = NULL; - data->thread_qps[i].lock = NULL; - } - return; - } - } -} - -static struct rcu_qp *update_qp(CRYPTO_RCU_LOCK *lock) -{ - uint64_t new_id; - uint32_t current_idx; - uint32_t tmp; - - ossl_crypto_mutex_lock(lock->alloc_lock); - /* - * we need at least one qp to be available with one - * left over, so that readers can start working on - * one that isn't yet being waited on - */ - while (lock->group_count - lock->writers_alloced < 2) - ossl_crypto_condvar_wait(lock->alloc_signal, lock->alloc_lock); - - current_idx = lock->current_alloc_idx; - /* Allocate the qp */ - lock->writers_alloced++; - - /* increment the allocation index */ - lock->current_alloc_idx = - (lock->current_alloc_idx + 1) % lock->group_count; - - /* get and insert a new id */ - new_id = lock->id_ctr; - lock->id_ctr++; - - new_id = VAL_ID(new_id); - InterlockedAnd64(&lock->qp_group[current_idx].users, ID_MASK); - InterlockedAdd64(&lock->qp_group[current_idx].users, new_id); - - /* update the reader index to be the prior qp */ - tmp = lock->current_alloc_idx; - InterlockedExchange(&lock->reader_idx, tmp); - - /* wake up any waiters */ - ossl_crypto_condvar_broadcast(lock->alloc_signal); - ossl_crypto_mutex_unlock(lock->alloc_lock); - return &lock->qp_group[current_idx]; -} - -static void retire_qp(CRYPTO_RCU_LOCK *lock, - struct rcu_qp *qp) -{ - ossl_crypto_mutex_lock(lock->alloc_lock); - lock->writers_alloced--; - ossl_crypto_condvar_broadcast(lock->alloc_signal); - ossl_crypto_mutex_unlock(lock->alloc_lock); -} - - -void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock) -{ - struct rcu_qp *qp; - uint64_t count; - struct rcu_cb_item *cb_items, *tmpcb; - - /* before we do anything else, lets grab the cb list */ - cb_items = InterlockedExchangePointer((void * volatile *)&lock->cb_items, NULL); - - qp = update_qp(lock); - - /* wait for the reader count to reach zero */ - do { - count = InterlockedOr64(&qp->users, 0); - } while (READER_COUNT(count) != 0); - - /* retire in order */ - ossl_crypto_mutex_lock(lock->prior_lock); - while (lock->next_to_retire != ID_VAL(count)) - ossl_crypto_condvar_wait(lock->prior_signal, lock->prior_lock); - - lock->next_to_retire++; - ossl_crypto_condvar_broadcast(lock->prior_signal); - ossl_crypto_mutex_unlock(lock->prior_lock); - - retire_qp(lock, qp); - - /* handle any callbacks that we have */ - while (cb_items != NULL) { - tmpcb = cb_items; - cb_items = cb_items->next; - tmpcb->fn(tmpcb->data); - OPENSSL_free(tmpcb); - } - - /* and we're done */ - return; - -} - -int ossl_rcu_call(CRYPTO_RCU_LOCK *lock, rcu_cb_fn cb, void *data) -{ - struct rcu_cb_item *new; - struct rcu_cb_item *prev; - - new = OPENSSL_zalloc(sizeof(struct rcu_cb_item)); - if (new == NULL) - return 0; - prev = new; - new->data = data; - new->fn = cb; - - InterlockedExchangePointer((void * volatile *)&lock->cb_items, prev); - new->next = prev; - return 1; -} - -void *ossl_rcu_uptr_deref(void **p) -{ - return (void *)*p; -} - -void ossl_rcu_assign_uptr(void **p, void **v) -{ - InterlockedExchangePointer((void * volatile *)p, (void *)*v); -} - - CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) { CRYPTO_RWLOCK *lock; @@ -399,15 +32,15 @@ CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) CRYPTO_win_rwlock *rwlock; if ((lock = OPENSSL_zalloc(sizeof(CRYPTO_win_rwlock))) == NULL) - /* Don't set error, to avoid recursion blowup. */ return NULL; rwlock = lock; InitializeSRWLock(&rwlock->lock); # else - if ((lock = OPENSSL_zalloc(sizeof(CRITICAL_SECTION))) == NULL) + if ((lock = OPENSSL_zalloc(sizeof(CRITICAL_SECTION))) == NULL) { /* Don't set error, to avoid recursion blowup. */ return NULL; + } # if !defined(_WIN32_WCE) /* 0x400 is the spin count value suggested in the documentation */ @@ -574,53 +207,14 @@ int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock) int CRYPTO_atomic_or(uint64_t *val, uint64_t op, uint64_t *ret, CRYPTO_RWLOCK *lock) { -#if (defined(NO_INTERLOCKEDOR64)) - if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) - return 0; - *val |= op; - *ret = *val; - - if (!CRYPTO_THREAD_unlock(lock)) - return 0; - - return 1; -#else *ret = (uint64_t)InterlockedOr64((LONG64 volatile *)val, (LONG64)op) | op; return 1; -#endif } int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock) { -#if (defined(NO_INTERLOCKEDOR64)) - if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) - return 0; - *ret = *val; - if (!CRYPTO_THREAD_unlock(lock)) - return 0; - - return 1; -#else *ret = (uint64_t)InterlockedOr64((LONG64 volatile *)val, 0); return 1; -#endif -} - -int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock) -{ -#if (defined(NO_INTERLOCKEDOR64)) - if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) - return 0; - *ret = *val; - if (!CRYPTO_THREAD_unlock(lock)) - return 0; - - return 1; -#else - /* On Windows, LONG is always the same size as int. */ - *ret = (int)InterlockedOr((LONG volatile *)val, 0); - return 1; -#endif } int openssl_init_fork_handlers(void) diff --git a/openssl/src/crypto/time.c b/openssl/src/crypto/time.c deleted file mode 100644 index b0593a238..000000000 --- a/openssl/src/crypto/time.c +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "internal/time.h" - -OSSL_TIME ossl_time_now(void) -{ - OSSL_TIME r; - -#if defined(_WIN32) && !defined(OPENSSL_SYS_UEFI) - SYSTEMTIME st; - union { - unsigned __int64 ul; - FILETIME ft; - } now; - - GetSystemTime(&st); - SystemTimeToFileTime(&st, &now.ft); - /* re-bias to 1/1/1970 */ -# ifdef __MINGW32__ - now.ul -= 116444736000000000ULL; -# else - now.ul -= 116444736000000000UI64; -# endif - r.t = ((uint64_t)now.ul) * (OSSL_TIME_SECOND / 10000000); -#else /* defined(_WIN32) */ - struct timeval t; - - if (gettimeofday(&t, NULL) < 0) { - ERR_raise_data(ERR_LIB_SYS, get_last_sys_error(), - "calling gettimeofday()"); - return ossl_time_zero(); - } - if (t.tv_sec <= 0) - r.t = t.tv_usec <= 0 ? 0 : t.tv_usec * OSSL_TIME_US; - else - r.t = ((uint64_t)t.tv_sec * 1000000 + t.tv_usec) * OSSL_TIME_US; -#endif /* defined(_WIN32) */ - return r; -} diff --git a/openssl/src/crypto/trace.c b/openssl/src/crypto/trace.c index 51387641d..d790409a2 100644 --- a/openssl/src/crypto/trace.c +++ b/openssl/src/crypto/trace.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,7 +18,6 @@ #include "internal/nelem.h" #include "internal/refcount.h" #include "crypto/cryptlib.h" -#include "crypto/ctype.h" #ifndef OPENSSL_NO_TRACE @@ -119,16 +118,17 @@ struct trace_category_st { }; #define TRACE_CATEGORY_(name) { #name, OSSL_TRACE_CATEGORY_##name } -static const struct trace_category_st - trace_categories[OSSL_TRACE_CATEGORY_NUM] = { +static const struct trace_category_st trace_categories[] = { TRACE_CATEGORY_(ALL), TRACE_CATEGORY_(TRACE), TRACE_CATEGORY_(INIT), TRACE_CATEGORY_(TLS), TRACE_CATEGORY_(TLS_CIPHER), TRACE_CATEGORY_(CONF), +#ifndef OPENSSL_NO_ENGINE TRACE_CATEGORY_(ENGINE_TABLE), TRACE_CATEGORY_(ENGINE_REF_COUNT), +#endif TRACE_CATEGORY_(PKCS5V2), TRACE_CATEGORY_(PKCS12_KEYGEN), TRACE_CATEGORY_(PKCS12_DECRYPT), @@ -138,35 +138,26 @@ static const struct trace_category_st TRACE_CATEGORY_(STORE), TRACE_CATEGORY_(DECODER), TRACE_CATEGORY_(ENCODER), - TRACE_CATEGORY_(REF_COUNT), - TRACE_CATEGORY_(HTTP), -}; /* KEEP THIS LIST IN SYNC with #define OSSL_TRACE_CATEGORY_... in trace.h */ + TRACE_CATEGORY_(REF_COUNT) +}; const char *OSSL_trace_get_category_name(int num) { - if (num < 0 || (size_t)num >= OSSL_NELEM(trace_categories)) - return NULL; - /* - * Partial check that OSSL_TRACE_CATEGORY_... macros - * are synced with trace_categories array - */ - if (!ossl_assert(trace_categories[num].name != NULL) - || !ossl_assert(trace_categories[num].num == num)) - return NULL; - return trace_categories[num].name; + size_t i; + + for (i = 0; i < OSSL_NELEM(trace_categories); i++) + if (trace_categories[i].num == num) + return trace_categories[i].name; + return NULL; /* not found */ } int OSSL_trace_get_category_num(const char *name) { size_t i; - if (name == NULL) - return -1; - for (i = 0; i < OSSL_NELEM(trace_categories); i++) if (OPENSSL_strcasecmp(name, trace_categories[i].name) == 0) return trace_categories[i].num; - return -1; /* not found */ } @@ -289,6 +280,11 @@ static int set_trace_data(int category, int type, BIO **channel, } /* Before running callbacks are done, set new data where appropriate */ + if (channel != NULL && *channel != NULL) { + trace_channels[category].type = type; + trace_channels[category].bio = *channel; + } + if (prefix != NULL && *prefix != NULL) { if ((curr_prefix = OPENSSL_strdup(*prefix)) == NULL) return 0; @@ -301,15 +297,6 @@ static int set_trace_data(int category, int type, BIO **channel, trace_channels[category].suffix = curr_suffix; } - if (channel != NULL && *channel != NULL) { - trace_channels[category].type = type; - trace_channels[category].bio = *channel; - /* - * This must not be done before setting prefix/suffix, - * as those may fail, and then the caller is mislead to free *channel. - */ - } - /* Finally, run the attach callback on the new data */ if (channel != NULL && *channel != NULL) { attach_cb(category, CHANNEL, *channel); @@ -502,7 +489,7 @@ BIO *OSSL_trace_begin(int category) return channel; } -void OSSL_trace_end(int category, BIO *channel) +void OSSL_trace_end(int category, BIO * channel) { #ifndef OPENSSL_NO_TRACE char *suffix = NULL; @@ -531,27 +518,3 @@ void OSSL_trace_end(int category, BIO *channel) } #endif } - -int OSSL_trace_string(BIO *out, int text, int full, - const unsigned char *data, size_t size) -{ - unsigned char buf[OSSL_TRACE_STRING_MAX + 1]; - int len, i; - - if (!full && size > OSSL_TRACE_STRING_MAX) { - BIO_printf(out, "[len %zu limited to %d]: ", - size, OSSL_TRACE_STRING_MAX); - len = OSSL_TRACE_STRING_MAX; - } else { - len = (int)size; - } - if (!text) { /* mask control characters while preserving newlines */ - for (i = 0; i < len; i++, data++) - buf[i] = (char)*data != '\n' && ossl_iscntrl((int)*data) - ? ' ' : *data; - if (len == 0 || data[-1] != '\n') - buf[len++] = '\n'; - data = buf; - } - return BIO_printf(out, "%.*s", len, data); -} diff --git a/openssl/src/crypto/ts/ts_conf.c b/openssl/src/crypto/ts/ts_conf.c index 158e1c424..fd2ad9075 100644 --- a/openssl/src/crypto/ts/ts_conf.c +++ b/openssl/src/crypto/ts/ts_conf.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -78,7 +78,7 @@ STACK_OF(X509) *TS_CONF_load_certs(const char *file) if (xi->x509 != NULL) { if (!X509_add_cert(othercerts, xi->x509, X509_ADD_FLAG_DEFAULT)) { - OSSL_STACK_OF_X509_free(othercerts); + sk_X509_pop_free(othercerts, X509_free); othercerts = NULL; goto end; } @@ -233,7 +233,7 @@ int TS_CONF_set_certs(CONF *conf, const char *section, const char *certs, end: ret = 1; err: - OSSL_STACK_OF_X509_free(certs_obj); + sk_X509_pop_free(certs_obj, X509_free); return ret; } @@ -481,7 +481,7 @@ int TS_CONF_set_ess_cert_id_digest(CONF *conf, const char *section, const char *md = NCONF_get_string(conf, section, ENV_ESS_CERT_ID_ALG); if (md == NULL) - md = "sha256"; + md = "sha1"; cert_md = EVP_get_digestbyname(md); if (cert_md == NULL) { diff --git a/openssl/src/crypto/ts/ts_local.h b/openssl/src/crypto/ts/ts_local.h index b0dbd5fdd..4dcb7af96 100644 --- a/openssl/src/crypto/ts/ts_local.h +++ b/openssl/src/crypto/ts/ts_local.h @@ -110,7 +110,7 @@ struct TS_resp_ctx { ASN1_INTEGER *seconds; /* accuracy, 0 means not specified. */ ASN1_INTEGER *millis; /* accuracy, 0 means not specified. */ ASN1_INTEGER *micros; /* accuracy, 0 means not specified. */ - unsigned clock_precision_digits; /* fraction of seconds in timestamp + unsigned clock_precision_digits; /* fraction of seconds in time stamp * token. */ unsigned flags; /* Optional info, see values above. */ /* Callback functions. */ diff --git a/openssl/src/crypto/ts/ts_req_utils.c b/openssl/src/crypto/ts/ts_req_utils.c index 89e1bda45..b560fc7b3 100644 --- a/openssl/src/crypto/ts/ts_req_utils.c +++ b/openssl/src/crypto/ts/ts_req_utils.c @@ -32,7 +32,7 @@ int TS_REQ_set_msg_imprint(TS_REQ *a, TS_MSG_IMPRINT *msg_imprint) return 1; new_msg_imprint = TS_MSG_IMPRINT_dup(msg_imprint); if (new_msg_imprint == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_TS_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } TS_MSG_IMPRINT_free(a->msg_imprint); @@ -53,7 +53,7 @@ int TS_MSG_IMPRINT_set_algo(TS_MSG_IMPRINT *a, X509_ALGOR *alg) return 1; new_alg = X509_ALGOR_dup(alg); if (new_alg == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } X509_ALGOR_free(a->hash_algo); @@ -84,7 +84,7 @@ int TS_REQ_set_policy_id(TS_REQ *a, const ASN1_OBJECT *policy) return 1; new_policy = OBJ_dup(policy); if (new_policy == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_OBJ_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } ASN1_OBJECT_free(a->policy_id); @@ -105,7 +105,7 @@ int TS_REQ_set_nonce(TS_REQ *a, const ASN1_INTEGER *nonce) return 1; new_nonce = ASN1_INTEGER_dup(nonce); if (new_nonce == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } ASN1_INTEGER_free(a->nonce); diff --git a/openssl/src/crypto/ts/ts_rsp_sign.c b/openssl/src/crypto/ts/ts_rsp_sign.c index 79d3e6783..46c0af1bc 100644 --- a/openssl/src/crypto/ts/ts_rsp_sign.c +++ b/openssl/src/crypto/ts/ts_rsp_sign.c @@ -15,7 +15,6 @@ #include #include "internal/cryptlib.h" #include "internal/sizes.h" -#include "internal/time.h" #include "crypto/ess.h" #include "ts_local.h" @@ -52,34 +51,53 @@ static ASN1_INTEGER *def_serial_cb(struct TS_resp_ctx *ctx, void *data) return serial; err: - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); TS_RESP_CTX_set_status_info(ctx, TS_STATUS_REJECTION, "Error during serial number generation."); ASN1_INTEGER_free(serial); return NULL; } +#if defined(OPENSSL_SYS_UNIX) + static int def_time_cb(struct TS_resp_ctx *ctx, void *data, long *sec, long *usec) { - OSSL_TIME t; struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) { + ERR_raise(ERR_LIB_TS, TS_R_TIME_SYSCALL_ERROR); + TS_RESP_CTX_set_status_info(ctx, TS_STATUS_REJECTION, + "Time is not available."); + TS_RESP_CTX_add_failure_info(ctx, TS_INFO_TIME_NOT_AVAILABLE); + return 0; + } + *sec = tv.tv_sec; + *usec = tv.tv_usec; - t = ossl_time_now(); - if (ossl_time_is_zero(t)) { + return 1; +} + +#else + +static int def_time_cb(struct TS_resp_ctx *ctx, void *data, + long *sec, long *usec) +{ + time_t t; + if (time(&t) == (time_t)-1) { ERR_raise(ERR_LIB_TS, TS_R_TIME_SYSCALL_ERROR); TS_RESP_CTX_set_status_info(ctx, TS_STATUS_REJECTION, "Time is not available."); TS_RESP_CTX_add_failure_info(ctx, TS_INFO_TIME_NOT_AVAILABLE); return 0; } - tv = ossl_time_to_timeval(t); - *sec = (long int)tv.tv_sec; - *usec = (long int)tv.tv_usec; + *sec = (long)t; + *usec = 0; return 1; } +#endif + static int def_extension_cb(struct TS_resp_ctx *ctx, X509_EXTENSION *ext, void *data) { @@ -95,13 +113,16 @@ TS_RESP_CTX *TS_RESP_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq) { TS_RESP_CTX *ctx; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) { + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return NULL; + } if (propq != NULL) { ctx->propq = OPENSSL_strdup(propq); if (ctx->propq == NULL) { OPENSSL_free(ctx); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return NULL; } } @@ -126,7 +147,7 @@ void TS_RESP_CTX_free(TS_RESP_CTX *ctx) OPENSSL_free(ctx->propq); X509_free(ctx->signer_cert); EVP_PKEY_free(ctx->signer_key); - OSSL_STACK_OF_X509_free(ctx->certs); + sk_X509_pop_free(ctx->certs, X509_free); sk_ASN1_OBJECT_pop_free(ctx->policies, ASN1_OBJECT_free); ASN1_OBJECT_free(ctx->default_policy); sk_EVP_MD_free(ctx->mds); /* No EVP_MD_free method exists. */ @@ -170,13 +191,13 @@ int TS_RESP_CTX_set_def_policy(TS_RESP_CTX *ctx, const ASN1_OBJECT *def_policy) goto err; return 1; err: - ERR_raise(ERR_LIB_TS, ERR_R_OBJ_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } int TS_RESP_CTX_set_certs(TS_RESP_CTX *ctx, STACK_OF(X509) *certs) { - OSSL_STACK_OF_X509_free(ctx->certs); + sk_X509_pop_free(ctx->certs, X509_free); ctx->certs = NULL; return certs == NULL || (ctx->certs = X509_chain_up_ref(certs)) != NULL; @@ -187,21 +208,16 @@ int TS_RESP_CTX_add_policy(TS_RESP_CTX *ctx, const ASN1_OBJECT *policy) ASN1_OBJECT *copy = NULL; if (ctx->policies == NULL - && (ctx->policies = sk_ASN1_OBJECT_new_null()) == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_CRYPTO_LIB); + && (ctx->policies = sk_ASN1_OBJECT_new_null()) == NULL) goto err; - } - if ((copy = OBJ_dup(policy)) == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_OBJ_LIB); + if ((copy = OBJ_dup(policy)) == NULL) goto err; - } - if (!sk_ASN1_OBJECT_push(ctx->policies, copy)) { - ERR_raise(ERR_LIB_TS, ERR_R_CRYPTO_LIB); + if (!sk_ASN1_OBJECT_push(ctx->policies, copy)) goto err; - } return 1; err: + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); ASN1_OBJECT_free(copy); return 0; } @@ -216,7 +232,7 @@ int TS_RESP_CTX_add_md(TS_RESP_CTX *ctx, const EVP_MD *md) return 1; err: - ERR_raise(ERR_LIB_TS, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } @@ -249,7 +265,7 @@ int TS_RESP_CTX_set_accuracy(TS_RESP_CTX *ctx, return 1; err: TS_RESP_CTX_accuracy_free(ctx); - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } @@ -284,37 +300,27 @@ int TS_RESP_CTX_set_status_info(TS_RESP_CTX *ctx, ASN1_UTF8STRING *utf8_text = NULL; int ret = 0; - if ((si = TS_STATUS_INFO_new()) == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_TS_LIB); + if ((si = TS_STATUS_INFO_new()) == NULL) goto err; - } - if (!ASN1_INTEGER_set(si->status, status)) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + if (!ASN1_INTEGER_set(si->status, status)) goto err; - } if (text) { if ((utf8_text = ASN1_UTF8STRING_new()) == NULL - || !ASN1_STRING_set(utf8_text, text, strlen(text))) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + || !ASN1_STRING_set(utf8_text, text, strlen(text))) goto err; - } if (si->text == NULL - && (si->text = sk_ASN1_UTF8STRING_new_null()) == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_CRYPTO_LIB); + && (si->text = sk_ASN1_UTF8STRING_new_null()) == NULL) goto err; - } - if (!sk_ASN1_UTF8STRING_push(si->text, utf8_text)) { - ERR_raise(ERR_LIB_TS, ERR_R_CRYPTO_LIB); + if (!sk_ASN1_UTF8STRING_push(si->text, utf8_text)) goto err; - } utf8_text = NULL; /* Ownership is lost. */ } - if (!TS_RESP_set_status_info(ctx->response, si)) { - ERR_raise(ERR_LIB_TS, ERR_R_TS_LIB); + if (!TS_RESP_set_status_info(ctx->response, si)) goto err; - } ret = 1; err: + if (!ret) + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); TS_STATUS_INFO_free(si); ASN1_UTF8STRING_free(utf8_text); return ret; @@ -342,7 +348,7 @@ int TS_RESP_CTX_add_failure_info(TS_RESP_CTX *ctx, int failure) goto err; return 1; err: - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } @@ -375,7 +381,7 @@ TS_RESP *TS_RESP_create_response(TS_RESP_CTX *ctx, BIO *req_bio) ts_RESP_CTX_init(ctx); if ((ctx->response = TS_RESP_new()) == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_TS_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); goto end; } if ((ctx->request = d2i_TS_REQ_bio(req_bio, NULL)) == NULL) { @@ -685,7 +691,7 @@ static int ts_RESP_sign(TS_RESP_CTX *ctx) } if ((p7 = PKCS7_new_ex(ctx->libctx, ctx->propq)) == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); goto err; } if (!PKCS7_set_type(p7, NID_pkcs7_signed)) @@ -750,7 +756,7 @@ static int ts_RESP_sign(TS_RESP_CTX *ctx) if (!ts_TST_INFO_content_new(p7)) goto err; if ((p7bio = PKCS7_dataInit(p7, NULL)) == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_PKCS7_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); goto err; } if (!i2d_TS_TST_INFO_bio(p7bio, ctx->tst_info)) { diff --git a/openssl/src/crypto/ts/ts_rsp_utils.c b/openssl/src/crypto/ts/ts_rsp_utils.c index 2352c7adb..cae076f21 100644 --- a/openssl/src/crypto/ts/ts_rsp_utils.c +++ b/openssl/src/crypto/ts/ts_rsp_utils.c @@ -22,7 +22,7 @@ int TS_RESP_set_status_info(TS_RESP *a, TS_STATUS_INFO *status_info) return 1; new_status_info = TS_STATUS_INFO_dup(status_info); if (new_status_info == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_TS_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } TS_STATUS_INFO_free(a->status_info); @@ -73,7 +73,7 @@ int TS_TST_INFO_set_policy_id(TS_TST_INFO *a, ASN1_OBJECT *policy) return 1; new_policy = OBJ_dup(policy); if (new_policy == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_OBJ_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } ASN1_OBJECT_free(a->policy_id); @@ -94,7 +94,7 @@ int TS_TST_INFO_set_msg_imprint(TS_TST_INFO *a, TS_MSG_IMPRINT *msg_imprint) return 1; new_msg_imprint = TS_MSG_IMPRINT_dup(msg_imprint); if (new_msg_imprint == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_TS_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } TS_MSG_IMPRINT_free(a->msg_imprint); @@ -115,7 +115,7 @@ int TS_TST_INFO_set_serial(TS_TST_INFO *a, const ASN1_INTEGER *serial) return 1; new_serial = ASN1_INTEGER_dup(serial); if (new_serial == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } ASN1_INTEGER_free(a->serial); @@ -136,7 +136,7 @@ int TS_TST_INFO_set_time(TS_TST_INFO *a, const ASN1_GENERALIZEDTIME *gtime) return 1; new_time = ASN1_STRING_dup(gtime); if (new_time == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } ASN1_GENERALIZEDTIME_free(a->time); @@ -157,7 +157,7 @@ int TS_TST_INFO_set_accuracy(TS_TST_INFO *a, TS_ACCURACY *accuracy) return 1; new_accuracy = TS_ACCURACY_dup(accuracy); if (new_accuracy == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_TS_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } TS_ACCURACY_free(a->accuracy); @@ -178,7 +178,7 @@ int TS_ACCURACY_set_seconds(TS_ACCURACY *a, const ASN1_INTEGER *seconds) return 1; new_seconds = ASN1_INTEGER_dup(seconds); if (new_seconds == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } ASN1_INTEGER_free(a->seconds); @@ -200,7 +200,7 @@ int TS_ACCURACY_set_millis(TS_ACCURACY *a, const ASN1_INTEGER *millis) if (millis != NULL) { new_millis = ASN1_INTEGER_dup(millis); if (new_millis == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } } @@ -223,7 +223,7 @@ int TS_ACCURACY_set_micros(TS_ACCURACY *a, const ASN1_INTEGER *micros) if (micros != NULL) { new_micros = ASN1_INTEGER_dup(micros); if (new_micros == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } } @@ -256,7 +256,7 @@ int TS_TST_INFO_set_nonce(TS_TST_INFO *a, const ASN1_INTEGER *nonce) return 1; new_nonce = ASN1_INTEGER_dup(nonce); if (new_nonce == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } ASN1_INTEGER_free(a->nonce); @@ -277,7 +277,7 @@ int TS_TST_INFO_set_tsa(TS_TST_INFO *a, GENERAL_NAME *tsa) return 1; new_tsa = GENERAL_NAME_dup(tsa); if (new_tsa == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return 0; } GENERAL_NAME_free(a->tsa); diff --git a/openssl/src/crypto/ts/ts_rsp_verify.c b/openssl/src/crypto/ts/ts_rsp_verify.c index 2dae352d0..792a27ce5 100644 --- a/openssl/src/crypto/ts/ts_rsp_verify.c +++ b/openssl/src/crypto/ts/ts_rsp_verify.c @@ -158,7 +158,7 @@ int TS_RESP_verify_signature(PKCS7 *token, STACK_OF(X509) *certs, err: BIO_free_all(p7bio); sk_X509_free(untrusted); - OSSL_STACK_OF_X509_free(chain); + sk_X509_pop_free(chain, X509_free); sk_X509_free(signers); return ret; @@ -178,7 +178,7 @@ static int ts_verify_cert(X509_STORE *store, STACK_OF(X509) *untrusted, *chain = NULL; cert_ctx = X509_STORE_CTX_new(); if (cert_ctx == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); goto err; } if (!X509_STORE_CTX_init(cert_ctx, store, signer, untrusted)) @@ -278,7 +278,7 @@ int TS_RESP_verify_token(TS_VERIFY_CTX *ctx, PKCS7 *token) } /*- - * Verifies whether the 'token' contains a valid timestamp token + * Verifies whether the 'token' contains a valid time stamp token * with regards to the settings of the context. Only those checks are * carried out that are specified in the context: * - Verifies the signature of the TS_TST_INFO. @@ -451,12 +451,14 @@ static int ts_compute_imprint(BIO *data, TS_TST_INFO *tst_info, if (length < 0) goto err; *imprint_len = length; - if ((*imprint = OPENSSL_malloc(*imprint_len)) == NULL) + if ((*imprint = OPENSSL_malloc(*imprint_len)) == NULL) { + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); goto err; + } md_ctx = EVP_MD_CTX_new(); if (md_ctx == NULL) { - ERR_raise(ERR_LIB_TS, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); goto err; } if (!EVP_DigestInit(md_ctx, md)) diff --git a/openssl/src/crypto/ts/ts_verify_ctx.c b/openssl/src/crypto/ts/ts_verify_ctx.c index 6dbba3df5..2f6f00c0c 100644 --- a/openssl/src/crypto/ts/ts_verify_ctx.c +++ b/openssl/src/crypto/ts/ts_verify_ctx.c @@ -16,6 +16,8 @@ TS_VERIFY_CTX *TS_VERIFY_CTX_new(void) { TS_VERIFY_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) + ERR_raise(ERR_LIB_TS, ERR_R_MALLOC_FAILURE); return ctx; } @@ -80,7 +82,7 @@ void TS_VERIFY_CTX_cleanup(TS_VERIFY_CTX *ctx) return; X509_STORE_free(ctx->store); - OSSL_STACK_OF_X509_free(ctx->certs); + sk_X509_pop_free(ctx->certs, X509_free); ASN1_OBJECT_free(ctx->policy); diff --git a/openssl/src/crypto/txt_db/txt_db.c b/openssl/src/crypto/txt_db/txt_db.c index 25fe9bf64..437b1b9c1 100644 --- a/openssl/src/crypto/txt_db/txt_db.c +++ b/openssl/src/crypto/txt_db/txt_db.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2018 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/ui/ui_lib.c b/openssl/src/crypto/ui/ui_lib.c index a8756af1c..1ff8c6fa3 100644 --- a/openssl/src/crypto/ui/ui_lib.c +++ b/openssl/src/crypto/ui/ui_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,12 +24,14 @@ UI *UI_new_method(const UI_METHOD *method) { UI *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); return NULL; + } ret->lock = CRYPTO_THREAD_lock_new(); if (ret->lock == NULL) { - ERR_raise(ERR_LIB_UI, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -208,8 +210,10 @@ int UI_dup_input_string(UI *ui, const char *prompt, int flags, if (prompt != NULL) { prompt_copy = OPENSSL_strdup(prompt); - if (prompt_copy == NULL) + if (prompt_copy == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); return 0; + } } return general_allocate_string(ui, prompt_copy, 1, @@ -234,8 +238,10 @@ int UI_dup_verify_string(UI *ui, const char *prompt, int flags, if (prompt != NULL) { prompt_copy = OPENSSL_strdup(prompt); - if (prompt_copy == NULL) + if (prompt_copy == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); return -1; + } } return general_allocate_string(ui, prompt_copy, 1, @@ -263,26 +269,34 @@ int UI_dup_input_boolean(UI *ui, const char *prompt, const char *action_desc, if (prompt != NULL) { prompt_copy = OPENSSL_strdup(prompt); - if (prompt_copy == NULL) + if (prompt_copy == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); goto err; + } } if (action_desc != NULL) { action_desc_copy = OPENSSL_strdup(action_desc); - if (action_desc_copy == NULL) + if (action_desc_copy == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); goto err; + } } if (ok_chars != NULL) { ok_chars_copy = OPENSSL_strdup(ok_chars); - if (ok_chars_copy == NULL) + if (ok_chars_copy == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); goto err; + } } if (cancel_chars != NULL) { cancel_chars_copy = OPENSSL_strdup(cancel_chars); - if (cancel_chars_copy == NULL) + if (cancel_chars_copy == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); goto err; + } } return general_allocate_boolean(ui, prompt_copy, action_desc_copy, @@ -308,8 +322,10 @@ int UI_dup_info_string(UI *ui, const char *text) if (text != NULL) { text_copy = OPENSSL_strdup(text); - if (text_copy == NULL) + if (text_copy == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); return -1; + } } return general_allocate_string(ui, text_copy, 1, UIT_INFO, 0, NULL, @@ -328,8 +344,10 @@ int UI_dup_error_string(UI *ui, const char *text) if (text != NULL) { text_copy = OPENSSL_strdup(text); - if (text_copy == NULL) + if (text_copy == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); return -1; + } } return general_allocate_string(ui, text_copy, 1, UIT_ERROR, 0, NULL, 0, 0, NULL); @@ -355,8 +373,10 @@ char *UI_construct_prompt(UI *ui, const char *phrase_desc, len += sizeof(prompt2) - 1 + strlen(object_name); len += sizeof(prompt3) - 1; - if ((prompt = OPENSSL_malloc(len + 1)) == NULL) + if ((prompt = OPENSSL_malloc(len + 1)) == NULL) { + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); return NULL; + } OPENSSL_strlcpy(prompt, prompt1, len + 1); OPENSSL_strlcat(prompt, phrase_desc, len + 1); if (object_name != NULL) { @@ -393,7 +413,7 @@ int UI_dup_user_data(UI *ui, void *user_data) duplicate = ui->meth->ui_duplicate_data(ui, user_data); if (duplicate == NULL) { - ERR_raise(ERR_LIB_UI, ERR_R_UI_LIB); + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); return -1; } @@ -508,10 +528,6 @@ int UI_process(UI *ui) ok = 0; break; } - } else { - ui->flags &= ~UI_FLAG_REDOABLE; - ok = -2; - goto err; } } @@ -583,17 +599,10 @@ UI_METHOD *UI_create_method(const char *name) || (ui_method->name = OPENSSL_strdup(name)) == NULL || !CRYPTO_new_ex_data(CRYPTO_EX_INDEX_UI_METHOD, ui_method, &ui_method->ex_data)) { - - if (ui_method != NULL) { - if (ui_method->name != NULL) - /* - * These conditions indicate that the CRYPTO_new_ex_data() - * call failed. - */ - ERR_raise(ERR_LIB_UI, ERR_R_CRYPTO_LIB); + if (ui_method) OPENSSL_free(ui_method->name); - } OPENSSL_free(ui_method); + ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE); return NULL; } return ui_method; diff --git a/openssl/src/crypto/ui/ui_openssl.c b/openssl/src/crypto/ui/ui_openssl.c index 544415e5b..355974247 100644 --- a/openssl/src/crypto/ui/ui_openssl.c +++ b/openssl/src/crypto/ui/ui_openssl.c @@ -23,42 +23,28 @@ # include # endif -# if !defined(_POSIX_C_SOURCE) && defined(OPENSSL_SYS_VMS) -# ifndef _POSIX_C_SOURCE -# define _POSIX_C_SOURCE 2 -# endif -# endif # include # include # include # include -# if !defined(OPENSSL_SYS_MSDOS) && !defined(OPENSSL_SYS_VMS) +# if !defined(OPENSSL_SYS_MSDOS) # include /* * If unistd.h defines _POSIX_VERSION, we conclude that we are on a POSIX * system and have sigaction and termios. */ # if defined(_POSIX_VERSION) && _POSIX_VERSION>=199309L - # define SIGACTION # if !defined(TERMIOS) && !defined(TERMIO) && !defined(SGTTY) # define TERMIOS # endif - # endif # endif # include "ui_local.h" # include "internal/cryptlib.h" -# ifdef OPENSSL_SYS_VMS /* prototypes for sys$whatever */ -# include -# ifdef __DECC -# pragma message disable DOLLARID -# endif -# endif - # ifdef WIN_CONSOLE_BUG # include # ifndef OPENSSL_SYS_WINCE @@ -67,7 +53,7 @@ # endif /* - * There are 6 types of terminal interface supported, TERMIO, TERMIOS, VMS, + * There are 5 types of terminal interface supported, TERMIO, TERMIOS, * MSDOS, WIN32 Console and SGTTY. * * If someone defines one of the macros TERMIO, TERMIOS or SGTTY, it will @@ -85,10 +71,9 @@ # define TERMIO # undef SGTTY /* - * We know that VMS, MSDOS, VXWORKS, use entirely other mechanisms. + * We know that MSDOS, VXWORKS, use entirely other mechanisms. */ -# elif !defined(OPENSSL_SYS_VMS) \ - && !defined(OPENSSL_SYS_MSDOS) \ +# elif !defined(OPENSSL_SYS_MSDOS) \ && !defined(OPENSSL_SYS_VXWORKS) # define TERMIOS # undef TERMIO @@ -127,7 +112,7 @@ # define TTY_set(tty,data) ioctl(tty,TIOCSETP,data) # endif -# if !defined(_LIBC) && !defined(OPENSSL_SYS_MSDOS) && !defined(OPENSSL_SYS_VMS) && ! (defined(OPENSSL_SYS_TANDEM) && defined(_SPT_MODEL_)) +# if !defined(_LIBC) && !defined(OPENSSL_SYS_MSDOS) && !(defined(OPENSSL_SYS_TANDEM) && defined(_SPT_MODEL_)) # include # endif @@ -135,18 +120,6 @@ # include # endif -# ifdef OPENSSL_SYS_VMS -# include -# include -# include -# include -struct IOSB { - short iosb$w_value; - short iosb$w_count; - long iosb$l_info; -}; -# endif - # ifndef NX509_SIG # define NX509_SIG 32 # endif @@ -158,20 +131,10 @@ static struct sigaction savsig[NX509_SIG]; static void (*savsig[NX509_SIG]) (int); # endif -# ifdef OPENSSL_SYS_VMS -static struct IOSB iosb; -static $DESCRIPTOR(terminal, "TT"); -static long tty_orig[3], tty_new[3]; /* XXX Is there any guarantee that this - * will always suffice for the actual - * structures? */ -static long status; -static unsigned short channel = 0; -# elif defined(_WIN32) && !defined(_WIN32_WCE) +# if defined(_WIN32) && !defined(_WIN32_WCE) static DWORD tty_orig, tty_new; # else -# if !defined(OPENSSL_SYS_MSDOS) || defined(__DJGPP__) static TTY_STRUCT tty_orig, tty_new; -# endif # endif static FILE *tty_in, *tty_out; static int is_a_tty; @@ -402,7 +365,7 @@ static int open_console(UI *ui) tty_out = stderr; # endif -# if defined(TTY_get) && !defined(OPENSSL_SYS_VMS) +# if defined(TTY_get) if (TTY_get(fileno(tty_in), &tty_orig) == -1) { # ifdef ENOTTY if (errno == ENOTTY) @@ -461,23 +424,6 @@ static int open_console(UI *ui) return 0; } } -# endif -# ifdef OPENSSL_SYS_VMS - status = sys$assign(&terminal, &channel, 0, 0); - - /* if there isn't a TT device, something is very wrong */ - if (status != SS$_NORMAL) { - ERR_raise_data(ERR_LIB_UI, UI_R_SYSASSIGN_ERROR, - "status=%%X%08X", status); - return 0; - } - - status = sys$qiow(0, channel, IO$_SENSEMODE, &iosb, 0, 0, tty_orig, 12, - 0, 0, 0, 0); - - /* If IO$_SENSEMODE doesn't work, this is not a terminal device */ - if ((status != SS$_NORMAL) || (iosb.iosb$w_value != SS$_NORMAL)) - is_a_tty = 0; # endif return 1; } @@ -489,25 +435,10 @@ static int noecho_console(UI *ui) tty_new.TTY_FLAGS &= ~ECHO; # endif -# if defined(TTY_set) && !defined(OPENSSL_SYS_VMS) +# if defined(TTY_set) if (is_a_tty && (TTY_set(fileno(tty_in), &tty_new) == -1)) return 0; # endif -# ifdef OPENSSL_SYS_VMS - if (is_a_tty) { - tty_new[0] = tty_orig[0]; - tty_new[1] = tty_orig[1] | TT$M_NOECHO; - tty_new[2] = tty_orig[2]; - status = sys$qiow(0, channel, IO$_SETMODE, &iosb, 0, 0, tty_new, 12, - 0, 0, 0, 0); - if ((status != SS$_NORMAL) || (iosb.iosb$w_value != SS$_NORMAL)) { - ERR_raise_data(ERR_LIB_UI, UI_R_SYSQIOW_ERROR, - "status=%%X%08X, iosb.iosb$w_value=%%X%08X", - status, iosb.iosb$w_value); - return 0; - } - } -# endif # if defined(_WIN32) && !defined(_WIN32_WCE) if (is_a_tty) { tty_new = tty_orig; @@ -520,26 +451,11 @@ static int noecho_console(UI *ui) static int echo_console(UI *ui) { -# if defined(TTY_set) && !defined(OPENSSL_SYS_VMS) +# if defined(TTY_set) memcpy(&(tty_new), &(tty_orig), sizeof(tty_orig)); if (is_a_tty && (TTY_set(fileno(tty_in), &tty_new) == -1)) return 0; # endif -# ifdef OPENSSL_SYS_VMS - if (is_a_tty) { - tty_new[0] = tty_orig[0]; - tty_new[1] = tty_orig[1]; - tty_new[2] = tty_orig[2]; - status = sys$qiow(0, channel, IO$_SETMODE, &iosb, 0, 0, tty_new, 12, - 0, 0, 0, 0); - if ((status != SS$_NORMAL) || (iosb.iosb$w_value != SS$_NORMAL)) { - ERR_raise_data(ERR_LIB_UI, UI_R_SYSQIOW_ERROR, - "status=%%X%08X, iosb.iosb$w_value=%%X%08X", - status, iosb.iosb$w_value); - return 0; - } - } -# endif # if defined(_WIN32) && !defined(_WIN32_WCE) if (is_a_tty) { tty_new = tty_orig; @@ -557,14 +473,6 @@ static int close_console(UI *ui) fclose(tty_in); if (tty_out != stderr) fclose(tty_out); -# ifdef OPENSSL_SYS_VMS - status = sys$dassgn(channel); - if (status != SS$_NORMAL) { - ERR_raise_data(ERR_LIB_UI, UI_R_SYSDASSGN_ERROR, - "status=%%X%08X", status); - ret = 0; - } -# endif CRYPTO_THREAD_unlock(ui->lock); return ret; diff --git a/openssl/src/crypto/ui/ui_util.c b/openssl/src/crypto/ui/ui_util.c index 59b00b225..54fa43f0c 100644 --- a/openssl/src/crypto/ui/ui_util.c +++ b/openssl/src/crypto/ui/ui_util.c @@ -1,5 +1,5 @@ /* - * Copyright 2002-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2002-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -32,7 +32,7 @@ int UI_UTIL_read_pw_string(char *buf, int length, const char *prompt, int UI_UTIL_read_pw(char *buf, char *buff, int size, const char *prompt, int verify) { - int ok = -2; + int ok = 0; UI *ui; if (size < 1) @@ -47,6 +47,8 @@ int UI_UTIL_read_pw(char *buf, char *buff, int size, const char *prompt, ok = UI_process(ui); UI_free(ui); } + if (ok > 0) + ok = 0; return ok; } @@ -151,7 +153,7 @@ UI_METHOD *UI_UTIL_wrap_read_pem_callback(pem_password_cb *cb, int rwflag) || UI_method_set_writer(ui_method, ui_write) < 0 || UI_method_set_closer(ui_method, ui_close) < 0 || !RUN_ONCE(&get_index_once, ui_method_data_index_init) - || !UI_method_set_ex_data(ui_method, ui_method_data_index, data)) { + || UI_method_set_ex_data(ui_method, ui_method_data_index, data) < 0) { UI_destroy_method(ui_method); OPENSSL_free(data); return NULL; diff --git a/openssl/src/crypto/uid.c b/openssl/src/crypto/uid.c index 45b63a431..698127779 100644 --- a/openssl/src/crypto/uid.c +++ b/openssl/src/crypto/uid.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,7 +10,7 @@ #include #include -#if defined(OPENSSL_SYS_WIN32) || defined(OPENSSL_SYS_VXWORKS) || defined(OPENSSL_SYS_UEFI) || defined(__wasi__) +#if defined(OPENSSL_SYS_WIN32) || defined(OPENSSL_SYS_VXWORKS) || defined(OPENSSL_SYS_UEFI) int OPENSSL_issetugid(void) { diff --git a/openssl/src/crypto/vms_rms.h b/openssl/src/crypto/vms_rms.h deleted file mode 100644 index ae74ba682..000000000 --- a/openssl/src/crypto/vms_rms.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2011-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifdef NAML$C_MAXRSS - -# define CC_RMS_NAMX cc$rms_naml -# define FAB_NAMX fab$l_naml -# define FAB_OR_NAML( fab, naml) naml -# define FAB_OR_NAML_DNA naml$l_long_defname -# define FAB_OR_NAML_DNS naml$l_long_defname_size -# define FAB_OR_NAML_FNA naml$l_long_filename -# define FAB_OR_NAML_FNS naml$l_long_filename_size -# define NAMX_ESA naml$l_long_expand -# define NAMX_ESL naml$l_long_expand_size -# define NAMX_ESS naml$l_long_expand_alloc -# define NAMX_NOP naml$b_nop -# define SET_NAMX_NO_SHORT_UPCASE( nam) nam.naml$v_no_short_upcase = 1 - -# if __INITIAL_POINTER_SIZE == 64 -# define NAMX_DNA_FNA_SET(fab) fab.fab$l_dna = (__char_ptr32) -1; \ - fab.fab$l_fna = (__char_ptr32) -1; -# else /* __INITIAL_POINTER_SIZE == 64 */ -# define NAMX_DNA_FNA_SET(fab) fab.fab$l_dna = (char *) -1; \ - fab.fab$l_fna = (char *) -1; -# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ - -# define NAMX_MAXRSS NAML$C_MAXRSS -# define NAMX_STRUCT NAML - -#else /* def NAML$C_MAXRSS */ - -# define CC_RMS_NAMX cc$rms_nam -# define FAB_NAMX fab$l_nam -# define FAB_OR_NAML( fab, naml) fab -# define FAB_OR_NAML_DNA fab$l_dna -# define FAB_OR_NAML_DNS fab$b_dns -# define FAB_OR_NAML_FNA fab$l_fna -# define FAB_OR_NAML_FNS fab$b_fns -# define NAMX_ESA nam$l_esa -# define NAMX_ESL nam$b_esl -# define NAMX_ESS nam$b_ess -# define NAMX_NOP nam$b_nop -# define NAMX_DNA_FNA_SET(fab) -# define NAMX_MAXRSS NAM$C_MAXRSS -# define NAMX_STRUCT NAM -# ifdef NAM$M_NO_SHORT_UPCASE -# define SET_NAMX_NO_SHORT_UPCASE( nam) naml.naml$v_no_short_upcase = 1 -# else /* def NAM$M_NO_SHORT_UPCASE */ -# define SET_NAMX_NO_SHORT_UPCASE( nam) -# endif /* def NAM$M_NO_SHORT_UPCASE [else] */ - -#endif /* def NAML$C_MAXRSS [else] */ diff --git a/openssl/src/crypto/whrlpool/wp_block.c b/openssl/src/crypto/whrlpool/wp_block.c deleted file mode 100644 index bcf7a199e..000000000 --- a/openssl/src/crypto/whrlpool/wp_block.c +++ /dev/null @@ -1,805 +0,0 @@ -/* - * Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/** - * The Whirlpool hashing function. - * - * See - * P.S.L.M. Barreto, V. Rijmen, - * ``The Whirlpool hashing function,'' - * NESSIE submission, 2000 (tweaked version, 2001), - * - * - * Based on "@version 3.0 (2003.03.12)" by Paulo S.L.M. Barreto and - * Vincent Rijmen. Lookup "reference implementations" on - * - * - * ============================================================================= - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS - * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE - * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* - * Whirlpool low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include "internal/cryptlib.h" -#include "wp_local.h" -#include - -typedef unsigned char u8; -#if (defined(_WIN32) || defined(_WIN64)) && !defined(__MINGW32) -typedef unsigned __int64 u64; -#elif defined(__arch64__) -typedef unsigned long u64; -#else -typedef unsigned long long u64; -#endif - -#define ROUNDS 10 - -#define STRICT_ALIGNMENT -#if !defined(PEDANTIC) && (defined(__i386) || defined(__i386__) || \ - defined(__x86_64) || defined(__x86_64__) || \ - defined(_M_IX86) || defined(_M_AMD64) || \ - defined(_M_X64)) -/* - * Well, formally there're couple of other architectures, which permit - * unaligned loads, specifically those not crossing cache lines, IA-64 and - * PowerPC... - */ -# undef STRICT_ALIGNMENT -#endif - -#ifndef STRICT_ALIGNMENT -# ifdef __GNUC__ -typedef u64 u64_a1 __attribute((__aligned__(1))); -# else -typedef u64 u64_a1; -# endif -#endif - -#if defined(__GNUC__) && !defined(STRICT_ALIGNMENT) -typedef u64 u64_aX __attribute((__aligned__(1))); -#else -typedef u64 u64_aX; -#endif - -#undef SMALL_REGISTER_BANK -#if defined(__i386) || defined(__i386__) || defined(_M_IX86) -# define SMALL_REGISTER_BANK -# if defined(WHIRLPOOL_ASM) -# ifndef OPENSSL_SMALL_FOOTPRINT -/* - * it appears that for elder non-MMX - * CPUs this is actually faster! - */ -# define OPENSSL_SMALL_FOOTPRINT -# endif -# define GO_FOR_MMX(ctx,inp,num) do { \ - void whirlpool_block_mmx(void *,const void *,size_t); \ - if (!(OPENSSL_ia32cap_P[0] & (1<<23))) break; \ - whirlpool_block_mmx(ctx->H.c,inp,num); return; \ - } while (0) -# endif -#endif - -#undef ROTATE -#ifndef PEDANTIC -# if defined(_MSC_VER) -# if defined(_WIN64) /* applies to both IA-64 and AMD64 */ -# include -# pragma intrinsic(_rotl64) -# define ROTATE(a,n) _rotl64((a),n) -# endif -# elif defined(__GNUC__) && __GNUC__>=2 -# if defined(__x86_64) || defined(__x86_64__) -# if defined(L_ENDIAN) -# define ROTATE(a,n) ({ u64 ret; asm ("rolq %1,%0" \ - : "=r"(ret) : "J"(n),"0"(a) : "cc"); ret; }) -# elif defined(B_ENDIAN) - /* - * Most will argue that x86_64 is always little-endian. Well, yes, but - * then we have stratus.com who has modified gcc to "emulate" - * big-endian on x86. Is there evidence that they [or somebody else] - * won't do same for x86_64? Naturally no. And this line is waiting - * ready for that brave soul:-) - */ -# define ROTATE(a,n) ({ u64 ret; asm ("rorq %1,%0" \ - : "=r"(ret) : "J"(n),"0"(a) : "cc"); ret; }) -# endif -# elif defined(__ia64) || defined(__ia64__) -# if defined(L_ENDIAN) -# define ROTATE(a,n) ({ u64 ret; asm ("shrp %0=%1,%1,%2" \ - : "=r"(ret) : "r"(a),"M"(64-(n))); ret; }) -# elif defined(B_ENDIAN) -# define ROTATE(a,n) ({ u64 ret; asm ("shrp %0=%1,%1,%2" \ - : "=r"(ret) : "r"(a),"M"(n)); ret; }) -# endif -# endif -# endif -#endif - -#if defined(OPENSSL_SMALL_FOOTPRINT) -# if !defined(ROTATE) -# if defined(L_ENDIAN) /* little-endians have to rotate left */ -# define ROTATE(i,n) ((i)<<(n) ^ (i)>>(64-n)) -# elif defined(B_ENDIAN) /* big-endians have to rotate right */ -# define ROTATE(i,n) ((i)>>(n) ^ (i)<<(64-n)) -# endif -# endif -# if defined(ROTATE) && !defined(STRICT_ALIGNMENT) -# define STRICT_ALIGNMENT /* ensure smallest table size */ -# endif -#endif - -/* - * Table size depends on STRICT_ALIGNMENT and whether or not endian- - * specific ROTATE macro is defined. If STRICT_ALIGNMENT is not - * defined, which is normally the case on x86[_64] CPUs, the table is - * 4KB large unconditionally. Otherwise if ROTATE is defined, the - * table is 2KB large, and otherwise - 16KB. 2KB table requires a - * whole bunch of additional rotations, but I'm willing to "trade," - * because 16KB table certainly trashes L1 cache. I wish all CPUs - * could handle unaligned load as 4KB table doesn't trash the cache, - * nor does it require additional rotations. - */ -/* - * Note that every Cn macro expands as two loads: one byte load and - * one quadword load. One can argue that many single-byte loads - * is too excessive, as one could load a quadword and "milk" it for - * eight 8-bit values instead. Well, yes, but in order to do so *and* - * avoid excessive loads you have to accommodate a handful of 64-bit - * values in the register bank and issue a bunch of shifts and mask. - * It's a tradeoff: loads vs. shift and mask in big register bank[!]. - * On most CPUs eight single-byte loads are faster and I let other - * ones to depend on smart compiler to fold byte loads if beneficial. - * Hand-coded assembler would be another alternative:-) - */ -#ifdef STRICT_ALIGNMENT -# if defined(ROTATE) -# define N 1 -# define LL(c0,c1,c2,c3,c4,c5,c6,c7) c0,c1,c2,c3,c4,c5,c6,c7 -# define C0(K,i) (Cx.q[K.c[(i)*8+0]]) -# define C1(K,i) ROTATE(Cx.q[K.c[(i)*8+1]],8) -# define C2(K,i) ROTATE(Cx.q[K.c[(i)*8+2]],16) -# define C3(K,i) ROTATE(Cx.q[K.c[(i)*8+3]],24) -# define C4(K,i) ROTATE(Cx.q[K.c[(i)*8+4]],32) -# define C5(K,i) ROTATE(Cx.q[K.c[(i)*8+5]],40) -# define C6(K,i) ROTATE(Cx.q[K.c[(i)*8+6]],48) -# define C7(K,i) ROTATE(Cx.q[K.c[(i)*8+7]],56) -# else -# define N 8 -# define LL(c0,c1,c2,c3,c4,c5,c6,c7) c0,c1,c2,c3,c4,c5,c6,c7, \ - c7,c0,c1,c2,c3,c4,c5,c6, \ - c6,c7,c0,c1,c2,c3,c4,c5, \ - c5,c6,c7,c0,c1,c2,c3,c4, \ - c4,c5,c6,c7,c0,c1,c2,c3, \ - c3,c4,c5,c6,c7,c0,c1,c2, \ - c2,c3,c4,c5,c6,c7,c0,c1, \ - c1,c2,c3,c4,c5,c6,c7,c0 -# define C0(K,i) (Cx.q[0+8*K.c[(i)*8+0]]) -# define C1(K,i) (Cx.q[1+8*K.c[(i)*8+1]]) -# define C2(K,i) (Cx.q[2+8*K.c[(i)*8+2]]) -# define C3(K,i) (Cx.q[3+8*K.c[(i)*8+3]]) -# define C4(K,i) (Cx.q[4+8*K.c[(i)*8+4]]) -# define C5(K,i) (Cx.q[5+8*K.c[(i)*8+5]]) -# define C6(K,i) (Cx.q[6+8*K.c[(i)*8+6]]) -# define C7(K,i) (Cx.q[7+8*K.c[(i)*8+7]]) -# endif -#else -# define N 2 -# define LL(c0,c1,c2,c3,c4,c5,c6,c7) c0,c1,c2,c3,c4,c5,c6,c7, \ - c0,c1,c2,c3,c4,c5,c6,c7 -# define C0(K,i) (((u64*)(Cx.c+0))[2*K.c[(i)*8+0]]) -# define C1(K,i) (((u64_a1*)(Cx.c+7))[2*K.c[(i)*8+1]]) -# define C2(K,i) (((u64_a1*)(Cx.c+6))[2*K.c[(i)*8+2]]) -# define C3(K,i) (((u64_a1*)(Cx.c+5))[2*K.c[(i)*8+3]]) -# define C4(K,i) (((u64_a1*)(Cx.c+4))[2*K.c[(i)*8+4]]) -# define C5(K,i) (((u64_a1*)(Cx.c+3))[2*K.c[(i)*8+5]]) -# define C6(K,i) (((u64_a1*)(Cx.c+2))[2*K.c[(i)*8+6]]) -# define C7(K,i) (((u64_a1*)(Cx.c+1))[2*K.c[(i)*8+7]]) -#endif - -static const - union { - u8 c[(256 * N + ROUNDS) * sizeof(u64)]; - u64 q[(256 * N + ROUNDS)]; -} Cx = { - { - /* Note endian-neutral representation:-) */ - LL(0x18, 0x18, 0x60, 0x18, 0xc0, 0x78, 0x30, 0xd8), - LL(0x23, 0x23, 0x8c, 0x23, 0x05, 0xaf, 0x46, 0x26), - LL(0xc6, 0xc6, 0x3f, 0xc6, 0x7e, 0xf9, 0x91, 0xb8), - LL(0xe8, 0xe8, 0x87, 0xe8, 0x13, 0x6f, 0xcd, 0xfb), - LL(0x87, 0x87, 0x26, 0x87, 0x4c, 0xa1, 0x13, 0xcb), - LL(0xb8, 0xb8, 0xda, 0xb8, 0xa9, 0x62, 0x6d, 0x11), - LL(0x01, 0x01, 0x04, 0x01, 0x08, 0x05, 0x02, 0x09), - LL(0x4f, 0x4f, 0x21, 0x4f, 0x42, 0x6e, 0x9e, 0x0d), - LL(0x36, 0x36, 0xd8, 0x36, 0xad, 0xee, 0x6c, 0x9b), - LL(0xa6, 0xa6, 0xa2, 0xa6, 0x59, 0x04, 0x51, 0xff), - LL(0xd2, 0xd2, 0x6f, 0xd2, 0xde, 0xbd, 0xb9, 0x0c), - LL(0xf5, 0xf5, 0xf3, 0xf5, 0xfb, 0x06, 0xf7, 0x0e), - LL(0x79, 0x79, 0xf9, 0x79, 0xef, 0x80, 0xf2, 0x96), - LL(0x6f, 0x6f, 0xa1, 0x6f, 0x5f, 0xce, 0xde, 0x30), - LL(0x91, 0x91, 0x7e, 0x91, 0xfc, 0xef, 0x3f, 0x6d), - LL(0x52, 0x52, 0x55, 0x52, 0xaa, 0x07, 0xa4, 0xf8), - LL(0x60, 0x60, 0x9d, 0x60, 0x27, 0xfd, 0xc0, 0x47), - LL(0xbc, 0xbc, 0xca, 0xbc, 0x89, 0x76, 0x65, 0x35), - LL(0x9b, 0x9b, 0x56, 0x9b, 0xac, 0xcd, 0x2b, 0x37), - LL(0x8e, 0x8e, 0x02, 0x8e, 0x04, 0x8c, 0x01, 0x8a), - LL(0xa3, 0xa3, 0xb6, 0xa3, 0x71, 0x15, 0x5b, 0xd2), - LL(0x0c, 0x0c, 0x30, 0x0c, 0x60, 0x3c, 0x18, 0x6c), - LL(0x7b, 0x7b, 0xf1, 0x7b, 0xff, 0x8a, 0xf6, 0x84), - LL(0x35, 0x35, 0xd4, 0x35, 0xb5, 0xe1, 0x6a, 0x80), - LL(0x1d, 0x1d, 0x74, 0x1d, 0xe8, 0x69, 0x3a, 0xf5), - LL(0xe0, 0xe0, 0xa7, 0xe0, 0x53, 0x47, 0xdd, 0xb3), - LL(0xd7, 0xd7, 0x7b, 0xd7, 0xf6, 0xac, 0xb3, 0x21), - LL(0xc2, 0xc2, 0x2f, 0xc2, 0x5e, 0xed, 0x99, 0x9c), - LL(0x2e, 0x2e, 0xb8, 0x2e, 0x6d, 0x96, 0x5c, 0x43), - LL(0x4b, 0x4b, 0x31, 0x4b, 0x62, 0x7a, 0x96, 0x29), - LL(0xfe, 0xfe, 0xdf, 0xfe, 0xa3, 0x21, 0xe1, 0x5d), - LL(0x57, 0x57, 0x41, 0x57, 0x82, 0x16, 0xae, 0xd5), - LL(0x15, 0x15, 0x54, 0x15, 0xa8, 0x41, 0x2a, 0xbd), - LL(0x77, 0x77, 0xc1, 0x77, 0x9f, 0xb6, 0xee, 0xe8), - LL(0x37, 0x37, 0xdc, 0x37, 0xa5, 0xeb, 0x6e, 0x92), - LL(0xe5, 0xe5, 0xb3, 0xe5, 0x7b, 0x56, 0xd7, 0x9e), - LL(0x9f, 0x9f, 0x46, 0x9f, 0x8c, 0xd9, 0x23, 0x13), - LL(0xf0, 0xf0, 0xe7, 0xf0, 0xd3, 0x17, 0xfd, 0x23), - LL(0x4a, 0x4a, 0x35, 0x4a, 0x6a, 0x7f, 0x94, 0x20), - LL(0xda, 0xda, 0x4f, 0xda, 0x9e, 0x95, 0xa9, 0x44), - LL(0x58, 0x58, 0x7d, 0x58, 0xfa, 0x25, 0xb0, 0xa2), - LL(0xc9, 0xc9, 0x03, 0xc9, 0x06, 0xca, 0x8f, 0xcf), - LL(0x29, 0x29, 0xa4, 0x29, 0x55, 0x8d, 0x52, 0x7c), - LL(0x0a, 0x0a, 0x28, 0x0a, 0x50, 0x22, 0x14, 0x5a), - LL(0xb1, 0xb1, 0xfe, 0xb1, 0xe1, 0x4f, 0x7f, 0x50), - LL(0xa0, 0xa0, 0xba, 0xa0, 0x69, 0x1a, 0x5d, 0xc9), - LL(0x6b, 0x6b, 0xb1, 0x6b, 0x7f, 0xda, 0xd6, 0x14), - LL(0x85, 0x85, 0x2e, 0x85, 0x5c, 0xab, 0x17, 0xd9), - LL(0xbd, 0xbd, 0xce, 0xbd, 0x81, 0x73, 0x67, 0x3c), - LL(0x5d, 0x5d, 0x69, 0x5d, 0xd2, 0x34, 0xba, 0x8f), - LL(0x10, 0x10, 0x40, 0x10, 0x80, 0x50, 0x20, 0x90), - LL(0xf4, 0xf4, 0xf7, 0xf4, 0xf3, 0x03, 0xf5, 0x07), - LL(0xcb, 0xcb, 0x0b, 0xcb, 0x16, 0xc0, 0x8b, 0xdd), - LL(0x3e, 0x3e, 0xf8, 0x3e, 0xed, 0xc6, 0x7c, 0xd3), - LL(0x05, 0x05, 0x14, 0x05, 0x28, 0x11, 0x0a, 0x2d), - LL(0x67, 0x67, 0x81, 0x67, 0x1f, 0xe6, 0xce, 0x78), - LL(0xe4, 0xe4, 0xb7, 0xe4, 0x73, 0x53, 0xd5, 0x97), - LL(0x27, 0x27, 0x9c, 0x27, 0x25, 0xbb, 0x4e, 0x02), - LL(0x41, 0x41, 0x19, 0x41, 0x32, 0x58, 0x82, 0x73), - LL(0x8b, 0x8b, 0x16, 0x8b, 0x2c, 0x9d, 0x0b, 0xa7), - LL(0xa7, 0xa7, 0xa6, 0xa7, 0x51, 0x01, 0x53, 0xf6), - LL(0x7d, 0x7d, 0xe9, 0x7d, 0xcf, 0x94, 0xfa, 0xb2), - LL(0x95, 0x95, 0x6e, 0x95, 0xdc, 0xfb, 0x37, 0x49), - LL(0xd8, 0xd8, 0x47, 0xd8, 0x8e, 0x9f, 0xad, 0x56), - LL(0xfb, 0xfb, 0xcb, 0xfb, 0x8b, 0x30, 0xeb, 0x70), - LL(0xee, 0xee, 0x9f, 0xee, 0x23, 0x71, 0xc1, 0xcd), - LL(0x7c, 0x7c, 0xed, 0x7c, 0xc7, 0x91, 0xf8, 0xbb), - LL(0x66, 0x66, 0x85, 0x66, 0x17, 0xe3, 0xcc, 0x71), - LL(0xdd, 0xdd, 0x53, 0xdd, 0xa6, 0x8e, 0xa7, 0x7b), - LL(0x17, 0x17, 0x5c, 0x17, 0xb8, 0x4b, 0x2e, 0xaf), - LL(0x47, 0x47, 0x01, 0x47, 0x02, 0x46, 0x8e, 0x45), - LL(0x9e, 0x9e, 0x42, 0x9e, 0x84, 0xdc, 0x21, 0x1a), - LL(0xca, 0xca, 0x0f, 0xca, 0x1e, 0xc5, 0x89, 0xd4), - LL(0x2d, 0x2d, 0xb4, 0x2d, 0x75, 0x99, 0x5a, 0x58), - LL(0xbf, 0xbf, 0xc6, 0xbf, 0x91, 0x79, 0x63, 0x2e), - LL(0x07, 0x07, 0x1c, 0x07, 0x38, 0x1b, 0x0e, 0x3f), - LL(0xad, 0xad, 0x8e, 0xad, 0x01, 0x23, 0x47, 0xac), - LL(0x5a, 0x5a, 0x75, 0x5a, 0xea, 0x2f, 0xb4, 0xb0), - LL(0x83, 0x83, 0x36, 0x83, 0x6c, 0xb5, 0x1b, 0xef), - LL(0x33, 0x33, 0xcc, 0x33, 0x85, 0xff, 0x66, 0xb6), - LL(0x63, 0x63, 0x91, 0x63, 0x3f, 0xf2, 0xc6, 0x5c), - LL(0x02, 0x02, 0x08, 0x02, 0x10, 0x0a, 0x04, 0x12), - LL(0xaa, 0xaa, 0x92, 0xaa, 0x39, 0x38, 0x49, 0x93), - LL(0x71, 0x71, 0xd9, 0x71, 0xaf, 0xa8, 0xe2, 0xde), - LL(0xc8, 0xc8, 0x07, 0xc8, 0x0e, 0xcf, 0x8d, 0xc6), - LL(0x19, 0x19, 0x64, 0x19, 0xc8, 0x7d, 0x32, 0xd1), - LL(0x49, 0x49, 0x39, 0x49, 0x72, 0x70, 0x92, 0x3b), - LL(0xd9, 0xd9, 0x43, 0xd9, 0x86, 0x9a, 0xaf, 0x5f), - LL(0xf2, 0xf2, 0xef, 0xf2, 0xc3, 0x1d, 0xf9, 0x31), - LL(0xe3, 0xe3, 0xab, 0xe3, 0x4b, 0x48, 0xdb, 0xa8), - LL(0x5b, 0x5b, 0x71, 0x5b, 0xe2, 0x2a, 0xb6, 0xb9), - LL(0x88, 0x88, 0x1a, 0x88, 0x34, 0x92, 0x0d, 0xbc), - LL(0x9a, 0x9a, 0x52, 0x9a, 0xa4, 0xc8, 0x29, 0x3e), - LL(0x26, 0x26, 0x98, 0x26, 0x2d, 0xbe, 0x4c, 0x0b), - LL(0x32, 0x32, 0xc8, 0x32, 0x8d, 0xfa, 0x64, 0xbf), - LL(0xb0, 0xb0, 0xfa, 0xb0, 0xe9, 0x4a, 0x7d, 0x59), - LL(0xe9, 0xe9, 0x83, 0xe9, 0x1b, 0x6a, 0xcf, 0xf2), - LL(0x0f, 0x0f, 0x3c, 0x0f, 0x78, 0x33, 0x1e, 0x77), - LL(0xd5, 0xd5, 0x73, 0xd5, 0xe6, 0xa6, 0xb7, 0x33), - LL(0x80, 0x80, 0x3a, 0x80, 0x74, 0xba, 0x1d, 0xf4), - LL(0xbe, 0xbe, 0xc2, 0xbe, 0x99, 0x7c, 0x61, 0x27), - LL(0xcd, 0xcd, 0x13, 0xcd, 0x26, 0xde, 0x87, 0xeb), - LL(0x34, 0x34, 0xd0, 0x34, 0xbd, 0xe4, 0x68, 0x89), - LL(0x48, 0x48, 0x3d, 0x48, 0x7a, 0x75, 0x90, 0x32), - LL(0xff, 0xff, 0xdb, 0xff, 0xab, 0x24, 0xe3, 0x54), - LL(0x7a, 0x7a, 0xf5, 0x7a, 0xf7, 0x8f, 0xf4, 0x8d), - LL(0x90, 0x90, 0x7a, 0x90, 0xf4, 0xea, 0x3d, 0x64), - LL(0x5f, 0x5f, 0x61, 0x5f, 0xc2, 0x3e, 0xbe, 0x9d), - LL(0x20, 0x20, 0x80, 0x20, 0x1d, 0xa0, 0x40, 0x3d), - LL(0x68, 0x68, 0xbd, 0x68, 0x67, 0xd5, 0xd0, 0x0f), - LL(0x1a, 0x1a, 0x68, 0x1a, 0xd0, 0x72, 0x34, 0xca), - LL(0xae, 0xae, 0x82, 0xae, 0x19, 0x2c, 0x41, 0xb7), - LL(0xb4, 0xb4, 0xea, 0xb4, 0xc9, 0x5e, 0x75, 0x7d), - LL(0x54, 0x54, 0x4d, 0x54, 0x9a, 0x19, 0xa8, 0xce), - LL(0x93, 0x93, 0x76, 0x93, 0xec, 0xe5, 0x3b, 0x7f), - LL(0x22, 0x22, 0x88, 0x22, 0x0d, 0xaa, 0x44, 0x2f), - LL(0x64, 0x64, 0x8d, 0x64, 0x07, 0xe9, 0xc8, 0x63), - LL(0xf1, 0xf1, 0xe3, 0xf1, 0xdb, 0x12, 0xff, 0x2a), - LL(0x73, 0x73, 0xd1, 0x73, 0xbf, 0xa2, 0xe6, 0xcc), - LL(0x12, 0x12, 0x48, 0x12, 0x90, 0x5a, 0x24, 0x82), - LL(0x40, 0x40, 0x1d, 0x40, 0x3a, 0x5d, 0x80, 0x7a), - LL(0x08, 0x08, 0x20, 0x08, 0x40, 0x28, 0x10, 0x48), - LL(0xc3, 0xc3, 0x2b, 0xc3, 0x56, 0xe8, 0x9b, 0x95), - LL(0xec, 0xec, 0x97, 0xec, 0x33, 0x7b, 0xc5, 0xdf), - LL(0xdb, 0xdb, 0x4b, 0xdb, 0x96, 0x90, 0xab, 0x4d), - LL(0xa1, 0xa1, 0xbe, 0xa1, 0x61, 0x1f, 0x5f, 0xc0), - LL(0x8d, 0x8d, 0x0e, 0x8d, 0x1c, 0x83, 0x07, 0x91), - LL(0x3d, 0x3d, 0xf4, 0x3d, 0xf5, 0xc9, 0x7a, 0xc8), - LL(0x97, 0x97, 0x66, 0x97, 0xcc, 0xf1, 0x33, 0x5b), - LL(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00), - LL(0xcf, 0xcf, 0x1b, 0xcf, 0x36, 0xd4, 0x83, 0xf9), - LL(0x2b, 0x2b, 0xac, 0x2b, 0x45, 0x87, 0x56, 0x6e), - LL(0x76, 0x76, 0xc5, 0x76, 0x97, 0xb3, 0xec, 0xe1), - LL(0x82, 0x82, 0x32, 0x82, 0x64, 0xb0, 0x19, 0xe6), - LL(0xd6, 0xd6, 0x7f, 0xd6, 0xfe, 0xa9, 0xb1, 0x28), - LL(0x1b, 0x1b, 0x6c, 0x1b, 0xd8, 0x77, 0x36, 0xc3), - LL(0xb5, 0xb5, 0xee, 0xb5, 0xc1, 0x5b, 0x77, 0x74), - LL(0xaf, 0xaf, 0x86, 0xaf, 0x11, 0x29, 0x43, 0xbe), - LL(0x6a, 0x6a, 0xb5, 0x6a, 0x77, 0xdf, 0xd4, 0x1d), - LL(0x50, 0x50, 0x5d, 0x50, 0xba, 0x0d, 0xa0, 0xea), - LL(0x45, 0x45, 0x09, 0x45, 0x12, 0x4c, 0x8a, 0x57), - LL(0xf3, 0xf3, 0xeb, 0xf3, 0xcb, 0x18, 0xfb, 0x38), - LL(0x30, 0x30, 0xc0, 0x30, 0x9d, 0xf0, 0x60, 0xad), - LL(0xef, 0xef, 0x9b, 0xef, 0x2b, 0x74, 0xc3, 0xc4), - LL(0x3f, 0x3f, 0xfc, 0x3f, 0xe5, 0xc3, 0x7e, 0xda), - LL(0x55, 0x55, 0x49, 0x55, 0x92, 0x1c, 0xaa, 0xc7), - LL(0xa2, 0xa2, 0xb2, 0xa2, 0x79, 0x10, 0x59, 0xdb), - LL(0xea, 0xea, 0x8f, 0xea, 0x03, 0x65, 0xc9, 0xe9), - LL(0x65, 0x65, 0x89, 0x65, 0x0f, 0xec, 0xca, 0x6a), - LL(0xba, 0xba, 0xd2, 0xba, 0xb9, 0x68, 0x69, 0x03), - LL(0x2f, 0x2f, 0xbc, 0x2f, 0x65, 0x93, 0x5e, 0x4a), - LL(0xc0, 0xc0, 0x27, 0xc0, 0x4e, 0xe7, 0x9d, 0x8e), - LL(0xde, 0xde, 0x5f, 0xde, 0xbe, 0x81, 0xa1, 0x60), - LL(0x1c, 0x1c, 0x70, 0x1c, 0xe0, 0x6c, 0x38, 0xfc), - LL(0xfd, 0xfd, 0xd3, 0xfd, 0xbb, 0x2e, 0xe7, 0x46), - LL(0x4d, 0x4d, 0x29, 0x4d, 0x52, 0x64, 0x9a, 0x1f), - LL(0x92, 0x92, 0x72, 0x92, 0xe4, 0xe0, 0x39, 0x76), - LL(0x75, 0x75, 0xc9, 0x75, 0x8f, 0xbc, 0xea, 0xfa), - LL(0x06, 0x06, 0x18, 0x06, 0x30, 0x1e, 0x0c, 0x36), - LL(0x8a, 0x8a, 0x12, 0x8a, 0x24, 0x98, 0x09, 0xae), - LL(0xb2, 0xb2, 0xf2, 0xb2, 0xf9, 0x40, 0x79, 0x4b), - LL(0xe6, 0xe6, 0xbf, 0xe6, 0x63, 0x59, 0xd1, 0x85), - LL(0x0e, 0x0e, 0x38, 0x0e, 0x70, 0x36, 0x1c, 0x7e), - LL(0x1f, 0x1f, 0x7c, 0x1f, 0xf8, 0x63, 0x3e, 0xe7), - LL(0x62, 0x62, 0x95, 0x62, 0x37, 0xf7, 0xc4, 0x55), - LL(0xd4, 0xd4, 0x77, 0xd4, 0xee, 0xa3, 0xb5, 0x3a), - LL(0xa8, 0xa8, 0x9a, 0xa8, 0x29, 0x32, 0x4d, 0x81), - LL(0x96, 0x96, 0x62, 0x96, 0xc4, 0xf4, 0x31, 0x52), - LL(0xf9, 0xf9, 0xc3, 0xf9, 0x9b, 0x3a, 0xef, 0x62), - LL(0xc5, 0xc5, 0x33, 0xc5, 0x66, 0xf6, 0x97, 0xa3), - LL(0x25, 0x25, 0x94, 0x25, 0x35, 0xb1, 0x4a, 0x10), - LL(0x59, 0x59, 0x79, 0x59, 0xf2, 0x20, 0xb2, 0xab), - LL(0x84, 0x84, 0x2a, 0x84, 0x54, 0xae, 0x15, 0xd0), - LL(0x72, 0x72, 0xd5, 0x72, 0xb7, 0xa7, 0xe4, 0xc5), - LL(0x39, 0x39, 0xe4, 0x39, 0xd5, 0xdd, 0x72, 0xec), - LL(0x4c, 0x4c, 0x2d, 0x4c, 0x5a, 0x61, 0x98, 0x16), - LL(0x5e, 0x5e, 0x65, 0x5e, 0xca, 0x3b, 0xbc, 0x94), - LL(0x78, 0x78, 0xfd, 0x78, 0xe7, 0x85, 0xf0, 0x9f), - LL(0x38, 0x38, 0xe0, 0x38, 0xdd, 0xd8, 0x70, 0xe5), - LL(0x8c, 0x8c, 0x0a, 0x8c, 0x14, 0x86, 0x05, 0x98), - LL(0xd1, 0xd1, 0x63, 0xd1, 0xc6, 0xb2, 0xbf, 0x17), - LL(0xa5, 0xa5, 0xae, 0xa5, 0x41, 0x0b, 0x57, 0xe4), - LL(0xe2, 0xe2, 0xaf, 0xe2, 0x43, 0x4d, 0xd9, 0xa1), - LL(0x61, 0x61, 0x99, 0x61, 0x2f, 0xf8, 0xc2, 0x4e), - LL(0xb3, 0xb3, 0xf6, 0xb3, 0xf1, 0x45, 0x7b, 0x42), - LL(0x21, 0x21, 0x84, 0x21, 0x15, 0xa5, 0x42, 0x34), - LL(0x9c, 0x9c, 0x4a, 0x9c, 0x94, 0xd6, 0x25, 0x08), - LL(0x1e, 0x1e, 0x78, 0x1e, 0xf0, 0x66, 0x3c, 0xee), - LL(0x43, 0x43, 0x11, 0x43, 0x22, 0x52, 0x86, 0x61), - LL(0xc7, 0xc7, 0x3b, 0xc7, 0x76, 0xfc, 0x93, 0xb1), - LL(0xfc, 0xfc, 0xd7, 0xfc, 0xb3, 0x2b, 0xe5, 0x4f), - LL(0x04, 0x04, 0x10, 0x04, 0x20, 0x14, 0x08, 0x24), - LL(0x51, 0x51, 0x59, 0x51, 0xb2, 0x08, 0xa2, 0xe3), - LL(0x99, 0x99, 0x5e, 0x99, 0xbc, 0xc7, 0x2f, 0x25), - LL(0x6d, 0x6d, 0xa9, 0x6d, 0x4f, 0xc4, 0xda, 0x22), - LL(0x0d, 0x0d, 0x34, 0x0d, 0x68, 0x39, 0x1a, 0x65), - LL(0xfa, 0xfa, 0xcf, 0xfa, 0x83, 0x35, 0xe9, 0x79), - LL(0xdf, 0xdf, 0x5b, 0xdf, 0xb6, 0x84, 0xa3, 0x69), - LL(0x7e, 0x7e, 0xe5, 0x7e, 0xd7, 0x9b, 0xfc, 0xa9), - LL(0x24, 0x24, 0x90, 0x24, 0x3d, 0xb4, 0x48, 0x19), - LL(0x3b, 0x3b, 0xec, 0x3b, 0xc5, 0xd7, 0x76, 0xfe), - LL(0xab, 0xab, 0x96, 0xab, 0x31, 0x3d, 0x4b, 0x9a), - LL(0xce, 0xce, 0x1f, 0xce, 0x3e, 0xd1, 0x81, 0xf0), - LL(0x11, 0x11, 0x44, 0x11, 0x88, 0x55, 0x22, 0x99), - LL(0x8f, 0x8f, 0x06, 0x8f, 0x0c, 0x89, 0x03, 0x83), - LL(0x4e, 0x4e, 0x25, 0x4e, 0x4a, 0x6b, 0x9c, 0x04), - LL(0xb7, 0xb7, 0xe6, 0xb7, 0xd1, 0x51, 0x73, 0x66), - LL(0xeb, 0xeb, 0x8b, 0xeb, 0x0b, 0x60, 0xcb, 0xe0), - LL(0x3c, 0x3c, 0xf0, 0x3c, 0xfd, 0xcc, 0x78, 0xc1), - LL(0x81, 0x81, 0x3e, 0x81, 0x7c, 0xbf, 0x1f, 0xfd), - LL(0x94, 0x94, 0x6a, 0x94, 0xd4, 0xfe, 0x35, 0x40), - LL(0xf7, 0xf7, 0xfb, 0xf7, 0xeb, 0x0c, 0xf3, 0x1c), - LL(0xb9, 0xb9, 0xde, 0xb9, 0xa1, 0x67, 0x6f, 0x18), - LL(0x13, 0x13, 0x4c, 0x13, 0x98, 0x5f, 0x26, 0x8b), - LL(0x2c, 0x2c, 0xb0, 0x2c, 0x7d, 0x9c, 0x58, 0x51), - LL(0xd3, 0xd3, 0x6b, 0xd3, 0xd6, 0xb8, 0xbb, 0x05), - LL(0xe7, 0xe7, 0xbb, 0xe7, 0x6b, 0x5c, 0xd3, 0x8c), - LL(0x6e, 0x6e, 0xa5, 0x6e, 0x57, 0xcb, 0xdc, 0x39), - LL(0xc4, 0xc4, 0x37, 0xc4, 0x6e, 0xf3, 0x95, 0xaa), - LL(0x03, 0x03, 0x0c, 0x03, 0x18, 0x0f, 0x06, 0x1b), - LL(0x56, 0x56, 0x45, 0x56, 0x8a, 0x13, 0xac, 0xdc), - LL(0x44, 0x44, 0x0d, 0x44, 0x1a, 0x49, 0x88, 0x5e), - LL(0x7f, 0x7f, 0xe1, 0x7f, 0xdf, 0x9e, 0xfe, 0xa0), - LL(0xa9, 0xa9, 0x9e, 0xa9, 0x21, 0x37, 0x4f, 0x88), - LL(0x2a, 0x2a, 0xa8, 0x2a, 0x4d, 0x82, 0x54, 0x67), - LL(0xbb, 0xbb, 0xd6, 0xbb, 0xb1, 0x6d, 0x6b, 0x0a), - LL(0xc1, 0xc1, 0x23, 0xc1, 0x46, 0xe2, 0x9f, 0x87), - LL(0x53, 0x53, 0x51, 0x53, 0xa2, 0x02, 0xa6, 0xf1), - LL(0xdc, 0xdc, 0x57, 0xdc, 0xae, 0x8b, 0xa5, 0x72), - LL(0x0b, 0x0b, 0x2c, 0x0b, 0x58, 0x27, 0x16, 0x53), - LL(0x9d, 0x9d, 0x4e, 0x9d, 0x9c, 0xd3, 0x27, 0x01), - LL(0x6c, 0x6c, 0xad, 0x6c, 0x47, 0xc1, 0xd8, 0x2b), - LL(0x31, 0x31, 0xc4, 0x31, 0x95, 0xf5, 0x62, 0xa4), - LL(0x74, 0x74, 0xcd, 0x74, 0x87, 0xb9, 0xe8, 0xf3), - LL(0xf6, 0xf6, 0xff, 0xf6, 0xe3, 0x09, 0xf1, 0x15), - LL(0x46, 0x46, 0x05, 0x46, 0x0a, 0x43, 0x8c, 0x4c), - LL(0xac, 0xac, 0x8a, 0xac, 0x09, 0x26, 0x45, 0xa5), - LL(0x89, 0x89, 0x1e, 0x89, 0x3c, 0x97, 0x0f, 0xb5), - LL(0x14, 0x14, 0x50, 0x14, 0xa0, 0x44, 0x28, 0xb4), - LL(0xe1, 0xe1, 0xa3, 0xe1, 0x5b, 0x42, 0xdf, 0xba), - LL(0x16, 0x16, 0x58, 0x16, 0xb0, 0x4e, 0x2c, 0xa6), - LL(0x3a, 0x3a, 0xe8, 0x3a, 0xcd, 0xd2, 0x74, 0xf7), - LL(0x69, 0x69, 0xb9, 0x69, 0x6f, 0xd0, 0xd2, 0x06), - LL(0x09, 0x09, 0x24, 0x09, 0x48, 0x2d, 0x12, 0x41), - LL(0x70, 0x70, 0xdd, 0x70, 0xa7, 0xad, 0xe0, 0xd7), - LL(0xb6, 0xb6, 0xe2, 0xb6, 0xd9, 0x54, 0x71, 0x6f), - LL(0xd0, 0xd0, 0x67, 0xd0, 0xce, 0xb7, 0xbd, 0x1e), - LL(0xed, 0xed, 0x93, 0xed, 0x3b, 0x7e, 0xc7, 0xd6), - LL(0xcc, 0xcc, 0x17, 0xcc, 0x2e, 0xdb, 0x85, 0xe2), - LL(0x42, 0x42, 0x15, 0x42, 0x2a, 0x57, 0x84, 0x68), - LL(0x98, 0x98, 0x5a, 0x98, 0xb4, 0xc2, 0x2d, 0x2c), - LL(0xa4, 0xa4, 0xaa, 0xa4, 0x49, 0x0e, 0x55, 0xed), - LL(0x28, 0x28, 0xa0, 0x28, 0x5d, 0x88, 0x50, 0x75), - LL(0x5c, 0x5c, 0x6d, 0x5c, 0xda, 0x31, 0xb8, 0x86), - LL(0xf8, 0xf8, 0xc7, 0xf8, 0x93, 0x3f, 0xed, 0x6b), - LL(0x86, 0x86, 0x22, 0x86, 0x44, 0xa4, 0x11, 0xc2), -#define RC (&(Cx.q[256*N])) - 0x18, 0x23, 0xc6, 0xe8, 0x87, 0xb8, 0x01, 0x4f, - /* rc[ROUNDS] */ - 0x36, 0xa6, 0xd2, 0xf5, 0x79, 0x6f, 0x91, 0x52, 0x60, 0xbc, 0x9b, - 0x8e, 0xa3, 0x0c, 0x7b, 0x35, 0x1d, 0xe0, 0xd7, 0xc2, 0x2e, 0x4b, - 0xfe, 0x57, 0x15, 0x77, 0x37, 0xe5, 0x9f, 0xf0, 0x4a, 0xda, 0x58, - 0xc9, 0x29, 0x0a, 0xb1, 0xa0, 0x6b, 0x85, 0xbd, 0x5d, 0x10, 0xf4, - 0xcb, 0x3e, 0x05, 0x67, 0xe4, 0x27, 0x41, 0x8b, 0xa7, 0x7d, 0x95, - 0xd8, 0xfb, 0xee, 0x7c, 0x66, 0xdd, 0x17, 0x47, 0x9e, 0xca, 0x2d, - 0xbf, 0x07, 0xad, 0x5a, 0x83, 0x33 - } - }; - -void whirlpool_block(WHIRLPOOL_CTX *ctx, const void *inp, size_t n) -{ - int r; - const u8 *p = inp; - union { - u64 q[8]; - u8 c[64]; - } S, K, *H = (void *)ctx->H.q; - -#ifdef GO_FOR_MMX - GO_FOR_MMX(ctx, inp, n); -#endif - do { -#ifdef OPENSSL_SMALL_FOOTPRINT - u64 L[8]; - int i; - - for (i = 0; i < 64; i++) - S.c[i] = (K.c[i] = H->c[i]) ^ p[i]; - for (r = 0; r < ROUNDS; r++) { - for (i = 0; i < 8; i++) { - L[i] = i ? 0 : RC[r]; - L[i] ^= C0(K, i) ^ C1(K, (i - 1) & 7) ^ - C2(K, (i - 2) & 7) ^ C3(K, (i - 3) & 7) ^ - C4(K, (i - 4) & 7) ^ C5(K, (i - 5) & 7) ^ - C6(K, (i - 6) & 7) ^ C7(K, (i - 7) & 7); - } - memcpy(K.q, L, 64); - for (i = 0; i < 8; i++) { - L[i] ^= C0(S, i) ^ C1(S, (i - 1) & 7) ^ - C2(S, (i - 2) & 7) ^ C3(S, (i - 3) & 7) ^ - C4(S, (i - 4) & 7) ^ C5(S, (i - 5) & 7) ^ - C6(S, (i - 6) & 7) ^ C7(S, (i - 7) & 7); - } - memcpy(S.q, L, 64); - } - for (i = 0; i < 64; i++) - H->c[i] ^= S.c[i] ^ p[i]; -#else - u64 L0, L1, L2, L3, L4, L5, L6, L7; - -# ifdef STRICT_ALIGNMENT - if ((size_t)p & 7) { - memcpy(S.c, p, 64); - S.q[0] ^= (K.q[0] = H->q[0]); - S.q[1] ^= (K.q[1] = H->q[1]); - S.q[2] ^= (K.q[2] = H->q[2]); - S.q[3] ^= (K.q[3] = H->q[3]); - S.q[4] ^= (K.q[4] = H->q[4]); - S.q[5] ^= (K.q[5] = H->q[5]); - S.q[6] ^= (K.q[6] = H->q[6]); - S.q[7] ^= (K.q[7] = H->q[7]); - } else -# endif - { - const u64_aX *pa = (const u64_aX *)p; - S.q[0] = (K.q[0] = H->q[0]) ^ pa[0]; - S.q[1] = (K.q[1] = H->q[1]) ^ pa[1]; - S.q[2] = (K.q[2] = H->q[2]) ^ pa[2]; - S.q[3] = (K.q[3] = H->q[3]) ^ pa[3]; - S.q[4] = (K.q[4] = H->q[4]) ^ pa[4]; - S.q[5] = (K.q[5] = H->q[5]) ^ pa[5]; - S.q[6] = (K.q[6] = H->q[6]) ^ pa[6]; - S.q[7] = (K.q[7] = H->q[7]) ^ pa[7]; - } - - for (r = 0; r < ROUNDS; r++) { -# ifdef SMALL_REGISTER_BANK - L0 = C0(K, 0) ^ C1(K, 7) ^ C2(K, 6) ^ C3(K, 5) ^ - C4(K, 4) ^ C5(K, 3) ^ C6(K, 2) ^ C7(K, 1) ^ RC[r]; - L1 = C0(K, 1) ^ C1(K, 0) ^ C2(K, 7) ^ C3(K, 6) ^ - C4(K, 5) ^ C5(K, 4) ^ C6(K, 3) ^ C7(K, 2); - L2 = C0(K, 2) ^ C1(K, 1) ^ C2(K, 0) ^ C3(K, 7) ^ - C4(K, 6) ^ C5(K, 5) ^ C6(K, 4) ^ C7(K, 3); - L3 = C0(K, 3) ^ C1(K, 2) ^ C2(K, 1) ^ C3(K, 0) ^ - C4(K, 7) ^ C5(K, 6) ^ C6(K, 5) ^ C7(K, 4); - L4 = C0(K, 4) ^ C1(K, 3) ^ C2(K, 2) ^ C3(K, 1) ^ - C4(K, 0) ^ C5(K, 7) ^ C6(K, 6) ^ C7(K, 5); - L5 = C0(K, 5) ^ C1(K, 4) ^ C2(K, 3) ^ C3(K, 2) ^ - C4(K, 1) ^ C5(K, 0) ^ C6(K, 7) ^ C7(K, 6); - L6 = C0(K, 6) ^ C1(K, 5) ^ C2(K, 4) ^ C3(K, 3) ^ - C4(K, 2) ^ C5(K, 1) ^ C6(K, 0) ^ C7(K, 7); - L7 = C0(K, 7) ^ C1(K, 6) ^ C2(K, 5) ^ C3(K, 4) ^ - C4(K, 3) ^ C5(K, 2) ^ C6(K, 1) ^ C7(K, 0); - - K.q[0] = L0; - K.q[1] = L1; - K.q[2] = L2; - K.q[3] = L3; - K.q[4] = L4; - K.q[5] = L5; - K.q[6] = L6; - K.q[7] = L7; - - L0 ^= C0(S, 0) ^ C1(S, 7) ^ C2(S, 6) ^ C3(S, 5) ^ - C4(S, 4) ^ C5(S, 3) ^ C6(S, 2) ^ C7(S, 1); - L1 ^= C0(S, 1) ^ C1(S, 0) ^ C2(S, 7) ^ C3(S, 6) ^ - C4(S, 5) ^ C5(S, 4) ^ C6(S, 3) ^ C7(S, 2); - L2 ^= C0(S, 2) ^ C1(S, 1) ^ C2(S, 0) ^ C3(S, 7) ^ - C4(S, 6) ^ C5(S, 5) ^ C6(S, 4) ^ C7(S, 3); - L3 ^= C0(S, 3) ^ C1(S, 2) ^ C2(S, 1) ^ C3(S, 0) ^ - C4(S, 7) ^ C5(S, 6) ^ C6(S, 5) ^ C7(S, 4); - L4 ^= C0(S, 4) ^ C1(S, 3) ^ C2(S, 2) ^ C3(S, 1) ^ - C4(S, 0) ^ C5(S, 7) ^ C6(S, 6) ^ C7(S, 5); - L5 ^= C0(S, 5) ^ C1(S, 4) ^ C2(S, 3) ^ C3(S, 2) ^ - C4(S, 1) ^ C5(S, 0) ^ C6(S, 7) ^ C7(S, 6); - L6 ^= C0(S, 6) ^ C1(S, 5) ^ C2(S, 4) ^ C3(S, 3) ^ - C4(S, 2) ^ C5(S, 1) ^ C6(S, 0) ^ C7(S, 7); - L7 ^= C0(S, 7) ^ C1(S, 6) ^ C2(S, 5) ^ C3(S, 4) ^ - C4(S, 3) ^ C5(S, 2) ^ C6(S, 1) ^ C7(S, 0); - - S.q[0] = L0; - S.q[1] = L1; - S.q[2] = L2; - S.q[3] = L3; - S.q[4] = L4; - S.q[5] = L5; - S.q[6] = L6; - S.q[7] = L7; -# else - L0 = C0(K, 0); - L1 = C1(K, 0); - L2 = C2(K, 0); - L3 = C3(K, 0); - L4 = C4(K, 0); - L5 = C5(K, 0); - L6 = C6(K, 0); - L7 = C7(K, 0); - L0 ^= RC[r]; - - L1 ^= C0(K, 1); - L2 ^= C1(K, 1); - L3 ^= C2(K, 1); - L4 ^= C3(K, 1); - L5 ^= C4(K, 1); - L6 ^= C5(K, 1); - L7 ^= C6(K, 1); - L0 ^= C7(K, 1); - - L2 ^= C0(K, 2); - L3 ^= C1(K, 2); - L4 ^= C2(K, 2); - L5 ^= C3(K, 2); - L6 ^= C4(K, 2); - L7 ^= C5(K, 2); - L0 ^= C6(K, 2); - L1 ^= C7(K, 2); - - L3 ^= C0(K, 3); - L4 ^= C1(K, 3); - L5 ^= C2(K, 3); - L6 ^= C3(K, 3); - L7 ^= C4(K, 3); - L0 ^= C5(K, 3); - L1 ^= C6(K, 3); - L2 ^= C7(K, 3); - - L4 ^= C0(K, 4); - L5 ^= C1(K, 4); - L6 ^= C2(K, 4); - L7 ^= C3(K, 4); - L0 ^= C4(K, 4); - L1 ^= C5(K, 4); - L2 ^= C6(K, 4); - L3 ^= C7(K, 4); - - L5 ^= C0(K, 5); - L6 ^= C1(K, 5); - L7 ^= C2(K, 5); - L0 ^= C3(K, 5); - L1 ^= C4(K, 5); - L2 ^= C5(K, 5); - L3 ^= C6(K, 5); - L4 ^= C7(K, 5); - - L6 ^= C0(K, 6); - L7 ^= C1(K, 6); - L0 ^= C2(K, 6); - L1 ^= C3(K, 6); - L2 ^= C4(K, 6); - L3 ^= C5(K, 6); - L4 ^= C6(K, 6); - L5 ^= C7(K, 6); - - L7 ^= C0(K, 7); - L0 ^= C1(K, 7); - L1 ^= C2(K, 7); - L2 ^= C3(K, 7); - L3 ^= C4(K, 7); - L4 ^= C5(K, 7); - L5 ^= C6(K, 7); - L6 ^= C7(K, 7); - - K.q[0] = L0; - K.q[1] = L1; - K.q[2] = L2; - K.q[3] = L3; - K.q[4] = L4; - K.q[5] = L5; - K.q[6] = L6; - K.q[7] = L7; - - L0 ^= C0(S, 0); - L1 ^= C1(S, 0); - L2 ^= C2(S, 0); - L3 ^= C3(S, 0); - L4 ^= C4(S, 0); - L5 ^= C5(S, 0); - L6 ^= C6(S, 0); - L7 ^= C7(S, 0); - - L1 ^= C0(S, 1); - L2 ^= C1(S, 1); - L3 ^= C2(S, 1); - L4 ^= C3(S, 1); - L5 ^= C4(S, 1); - L6 ^= C5(S, 1); - L7 ^= C6(S, 1); - L0 ^= C7(S, 1); - - L2 ^= C0(S, 2); - L3 ^= C1(S, 2); - L4 ^= C2(S, 2); - L5 ^= C3(S, 2); - L6 ^= C4(S, 2); - L7 ^= C5(S, 2); - L0 ^= C6(S, 2); - L1 ^= C7(S, 2); - - L3 ^= C0(S, 3); - L4 ^= C1(S, 3); - L5 ^= C2(S, 3); - L6 ^= C3(S, 3); - L7 ^= C4(S, 3); - L0 ^= C5(S, 3); - L1 ^= C6(S, 3); - L2 ^= C7(S, 3); - - L4 ^= C0(S, 4); - L5 ^= C1(S, 4); - L6 ^= C2(S, 4); - L7 ^= C3(S, 4); - L0 ^= C4(S, 4); - L1 ^= C5(S, 4); - L2 ^= C6(S, 4); - L3 ^= C7(S, 4); - - L5 ^= C0(S, 5); - L6 ^= C1(S, 5); - L7 ^= C2(S, 5); - L0 ^= C3(S, 5); - L1 ^= C4(S, 5); - L2 ^= C5(S, 5); - L3 ^= C6(S, 5); - L4 ^= C7(S, 5); - - L6 ^= C0(S, 6); - L7 ^= C1(S, 6); - L0 ^= C2(S, 6); - L1 ^= C3(S, 6); - L2 ^= C4(S, 6); - L3 ^= C5(S, 6); - L4 ^= C6(S, 6); - L5 ^= C7(S, 6); - - L7 ^= C0(S, 7); - L0 ^= C1(S, 7); - L1 ^= C2(S, 7); - L2 ^= C3(S, 7); - L3 ^= C4(S, 7); - L4 ^= C5(S, 7); - L5 ^= C6(S, 7); - L6 ^= C7(S, 7); - - S.q[0] = L0; - S.q[1] = L1; - S.q[2] = L2; - S.q[3] = L3; - S.q[4] = L4; - S.q[5] = L5; - S.q[6] = L6; - S.q[7] = L7; -# endif - } - -# ifdef STRICT_ALIGNMENT - if ((size_t)p & 7) { - int i; - for (i = 0; i < 64; i++) - H->c[i] ^= S.c[i] ^ p[i]; - } else -# endif - { - const u64_aX *pa = (const u64_aX *)p; - H->q[0] ^= S.q[0] ^ pa[0]; - H->q[1] ^= S.q[1] ^ pa[1]; - H->q[2] ^= S.q[2] ^ pa[2]; - H->q[3] ^= S.q[3] ^ pa[3]; - H->q[4] ^= S.q[4] ^ pa[4]; - H->q[5] ^= S.q[5] ^ pa[5]; - H->q[6] ^= S.q[6] ^ pa[6]; - H->q[7] ^= S.q[7] ^ pa[7]; - } -#endif - p += 64; - } while (--n); -} diff --git a/openssl/src/crypto/whrlpool/wp_dgst.c b/openssl/src/crypto/whrlpool/wp_dgst.c deleted file mode 100644 index 3f970deb9..000000000 --- a/openssl/src/crypto/whrlpool/wp_dgst.c +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright 2005-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/** - * The Whirlpool hashing function. - * - * See - * P.S.L.M. Barreto, V. Rijmen, - * ``The Whirlpool hashing function,'' - * NESSIE submission, 2000 (tweaked version, 2001), - * - * - * Based on "@version 3.0 (2003.03.12)" by Paulo S.L.M. Barreto and - * Vincent Rijmen. Lookup "reference implementations" on - * - * - * ============================================================================= - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS - * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE - * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* - * OpenSSL-specific implementation notes. - * - * WHIRLPOOL_Update as well as one-stroke WHIRLPOOL both expect - * number of *bytes* as input length argument. Bit-oriented routine - * as specified by authors is called WHIRLPOOL_BitUpdate[!] and - * does not have one-stroke counterpart. - * - * WHIRLPOOL_BitUpdate implements byte-oriented loop, essentially - * to serve WHIRLPOOL_Update. This is done for performance. - * - * Unlike authors' reference implementation, block processing - * routine whirlpool_block is designed to operate on multi-block - * input. This is done for performance. - */ - -/* - * Whirlpool low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include "wp_local.h" -#include - -int WHIRLPOOL_Init(WHIRLPOOL_CTX *c) -{ - memset(c, 0, sizeof(*c)); - return 1; -} - -int WHIRLPOOL_Update(WHIRLPOOL_CTX *c, const void *_inp, size_t bytes) -{ - /* - * Well, largest suitable chunk size actually is - * (1<<(sizeof(size_t)*8-3))-64, but below number is large enough for not - * to care about excessive calls to WHIRLPOOL_BitUpdate... - */ - size_t chunk = ((size_t)1) << (sizeof(size_t) * 8 - 4); - const unsigned char *inp = _inp; - - while (bytes >= chunk) { - WHIRLPOOL_BitUpdate(c, inp, chunk * 8); - bytes -= chunk; - inp += chunk; - } - if (bytes) - WHIRLPOOL_BitUpdate(c, inp, bytes * 8); - - return 1; -} - -void WHIRLPOOL_BitUpdate(WHIRLPOOL_CTX *c, const void *_inp, size_t bits) -{ - size_t n; - unsigned int bitoff = c->bitoff, - bitrem = bitoff % 8, inpgap = (8 - (unsigned int)bits % 8) & 7; - const unsigned char *inp = _inp; - - /* - * This 256-bit increment procedure relies on the size_t being natural - * size of CPU register, so that we don't have to mask the value in order - * to detect overflows. - */ - c->bitlen[0] += bits; - if (c->bitlen[0] < bits) { /* overflow */ - n = 1; - do { - c->bitlen[n]++; - } while (c->bitlen[n] == 0 - && ++n < (WHIRLPOOL_COUNTER / sizeof(size_t))); - } -#ifndef OPENSSL_SMALL_FOOTPRINT - reconsider: - if (inpgap == 0 && bitrem == 0) { /* byte-oriented loop */ - while (bits) { - if (bitoff == 0 && (n = bits / WHIRLPOOL_BBLOCK)) { - whirlpool_block(c, inp, n); - inp += n * WHIRLPOOL_BBLOCK / 8; - bits %= WHIRLPOOL_BBLOCK; - } else { - unsigned int byteoff = bitoff / 8; - - bitrem = WHIRLPOOL_BBLOCK - bitoff; /* reuse bitrem */ - if (bits >= bitrem) { - bits -= bitrem; - bitrem /= 8; - memcpy(c->data + byteoff, inp, bitrem); - inp += bitrem; - whirlpool_block(c, c->data, 1); - bitoff = 0; - } else { - memcpy(c->data + byteoff, inp, bits / 8); - bitoff += (unsigned int)bits; - bits = 0; - } - c->bitoff = bitoff; - } - } - } else /* bit-oriented loop */ -#endif - { - /*- - inp - | - +-------+-------+------- - ||||||||||||||||||||| - +-------+-------+------- - +-------+-------+-------+-------+------- - |||||||||||||| c->data - +-------+-------+-------+-------+------- - | - c->bitoff/8 - */ - while (bits) { - unsigned int byteoff = bitoff / 8; - unsigned char b; - -#ifndef OPENSSL_SMALL_FOOTPRINT - if (bitrem == inpgap) { - c->data[byteoff++] |= inp[0] & (0xff >> inpgap); - inpgap = 8 - inpgap; - bitoff += inpgap; - bitrem = 0; /* bitoff%8 */ - bits -= inpgap; - inpgap = 0; /* bits%8 */ - inp++; - if (bitoff == WHIRLPOOL_BBLOCK) { - whirlpool_block(c, c->data, 1); - bitoff = 0; - } - c->bitoff = bitoff; - goto reconsider; - } else -#endif - if (bits > 8) { - b = ((inp[0] << inpgap) | (inp[1] >> (8 - inpgap))); - b &= 0xff; - if (bitrem) - c->data[byteoff++] |= b >> bitrem; - else - c->data[byteoff++] = b; - bitoff += 8; - bits -= 8; - inp++; - if (bitoff >= WHIRLPOOL_BBLOCK) { - whirlpool_block(c, c->data, 1); - byteoff = 0; - bitoff %= WHIRLPOOL_BBLOCK; - } - if (bitrem) - c->data[byteoff] = b << (8 - bitrem); - } else { /* remaining less than or equal to 8 bits */ - - b = (inp[0] << inpgap) & 0xff; - if (bitrem) - c->data[byteoff++] |= b >> bitrem; - else - c->data[byteoff++] = b; - bitoff += (unsigned int)bits; - if (bitoff == WHIRLPOOL_BBLOCK) { - whirlpool_block(c, c->data, 1); - byteoff = 0; - bitoff %= WHIRLPOOL_BBLOCK; - } - if (bitrem) - c->data[byteoff] = b << (8 - bitrem); - bits = 0; - } - c->bitoff = bitoff; - } - } -} - -int WHIRLPOOL_Final(unsigned char *md, WHIRLPOOL_CTX *c) -{ - unsigned int bitoff = c->bitoff, byteoff = bitoff / 8; - size_t i, j, v; - unsigned char *p; - - bitoff %= 8; - if (bitoff) - c->data[byteoff] |= 0x80 >> bitoff; - else - c->data[byteoff] = 0x80; - byteoff++; - - /* pad with zeros */ - if (byteoff > (WHIRLPOOL_BBLOCK / 8 - WHIRLPOOL_COUNTER)) { - if (byteoff < WHIRLPOOL_BBLOCK / 8) - memset(&c->data[byteoff], 0, WHIRLPOOL_BBLOCK / 8 - byteoff); - whirlpool_block(c, c->data, 1); - byteoff = 0; - } - if (byteoff < (WHIRLPOOL_BBLOCK / 8 - WHIRLPOOL_COUNTER)) - memset(&c->data[byteoff], 0, - (WHIRLPOOL_BBLOCK / 8 - WHIRLPOOL_COUNTER) - byteoff); - /* smash 256-bit c->bitlen in big-endian order */ - p = &c->data[WHIRLPOOL_BBLOCK / 8 - 1]; /* last byte in c->data */ - for (i = 0; i < WHIRLPOOL_COUNTER / sizeof(size_t); i++) - for (v = c->bitlen[i], j = 0; j < sizeof(size_t); j++, v >>= 8) - *p-- = (unsigned char)(v & 0xff); - - whirlpool_block(c, c->data, 1); - - if (md) { - memcpy(md, c->H.c, WHIRLPOOL_DIGEST_LENGTH); - OPENSSL_cleanse(c, sizeof(*c)); - return 1; - } - return 0; -} - -unsigned char *WHIRLPOOL(const void *inp, size_t bytes, unsigned char *md) -{ - WHIRLPOOL_CTX ctx; - static unsigned char m[WHIRLPOOL_DIGEST_LENGTH]; - - if (md == NULL) - md = m; - WHIRLPOOL_Init(&ctx); - WHIRLPOOL_Update(&ctx, inp, bytes); - WHIRLPOOL_Final(md, &ctx); - return md; -} diff --git a/openssl/src/crypto/whrlpool/wp_local.h b/openssl/src/crypto/whrlpool/wp_local.h deleted file mode 100644 index 73dc2a003..000000000 --- a/openssl/src/crypto/whrlpool/wp_local.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright 2005-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include - -void whirlpool_block(WHIRLPOOL_CTX *, const void *, size_t); diff --git a/openssl/src/crypto/x509/by_dir.c b/openssl/src/crypto/x509/by_dir.c index bdcdc4555..0e2407d3f 100644 --- a/openssl/src/crypto/x509/by_dir.c +++ b/openssl/src/crypto/x509/by_dir.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -109,18 +109,20 @@ static int new_dir(X509_LOOKUP *lu) { BY_DIR *a = OPENSSL_malloc(sizeof(*a)); - if (a == NULL) + if (a == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; + } if ((a->buffer = BUF_MEM_new()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } a->dirs = NULL; a->lock = CRYPTO_THREAD_lock_new(); if (a->lock == NULL) { BUF_MEM_free(a->buffer); - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } lu->method_data = a; @@ -195,13 +197,15 @@ static int add_cert_dir(BY_DIR *ctx, const char *dir, int type) if (ctx->dirs == NULL) { ctx->dirs = sk_BY_DIR_ENTRY_new_null(); if (!ctx->dirs) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; } } ent = OPENSSL_malloc(sizeof(*ent)); - if (ent == NULL) + if (ent == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; + } ent->dir_type = type; ent->hashes = sk_BY_DIR_HASH_new(by_dir_hash_cmp); ent->dir = OPENSSL_strndup(ss, len); @@ -211,7 +215,7 @@ static int add_cert_dir(BY_DIR *ctx, const char *dir, int type) } if (!sk_BY_DIR_ENTRY_push(ctx->dirs, ent)) { by_dir_entry_free(ent); - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; } } @@ -268,7 +272,7 @@ static int get_cert_by_subject_ex(X509_LOOKUP *xl, X509_LOOKUP_TYPE type, ent = sk_BY_DIR_ENTRY_value(ctx->dirs, i); j = strlen(ent->dir) + 1 + 8 + 6 + 1 + 1; if (!BUF_MEM_grow(b, j)) { - ERR_raise(ERR_LIB_X509, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto finish; } if (type == X509_LU_CRL && ent->hashes) { @@ -291,33 +295,8 @@ static int get_cert_by_subject_ex(X509_LOOKUP *xl, X509_LOOKUP_TYPE type, for (;;) { char c = '/'; -#ifdef OPENSSL_SYS_VMS - c = ent->dir[strlen(ent->dir) - 1]; - if (c != ':' && c != '>' && c != ']') { - /* - * If no separator is present, we assume the directory - * specifier is a logical name, and add a colon. We really - * should use better VMS routines for merging things like - * this, but this will do for now... -- Richard Levitte - */ - c = ':'; - } else { - c = '\0'; - } - - if (c == '\0') { - /* - * This is special. When c == '\0', no directory separator - * should be added. - */ - BIO_snprintf(b->data, b->max, - "%s%08lx.%s%d", ent->dir, h, postfix, k); - } else -#endif - { - BIO_snprintf(b->data, b->max, - "%s%c%08lx.%s%d", ent->dir, c, h, postfix, k); - } + BIO_snprintf(b->data, b->max, + "%s%c%08lx.%s%d", ent->dir, c, h, postfix, k); #ifndef OPENSSL_NO_POSIX_IO # ifdef _WIN32 # define stat _stat @@ -343,26 +322,15 @@ static int get_cert_by_subject_ex(X509_LOOKUP *xl, X509_LOOKUP_TYPE type, /* * we have added it to the cache so now pull it out again - * - * Note: quadratic time find here since the objects won't generally be - * sorted and sorting the would result in O(n^2 log n) complexity. */ - if (k > 0) { - if (!X509_STORE_lock(xl->store_ctx)) - goto finish; - j = sk_X509_OBJECT_find(xl->store_ctx->objs, &stmp); - tmp = sk_X509_OBJECT_value(xl->store_ctx->objs, j); - X509_STORE_unlock(xl->store_ctx); - } else { - tmp = NULL; - } - /* - * If a CRL, update the last file suffix added for this. - * We don't need to add an entry if k is 0 as this is the initial value. - * This avoids the need for a write lock and sort operation in the - * simple case where no CRL is present for a hash. - */ - if (type == X509_LU_CRL && k > 0) { + X509_STORE_lock(xl->store_ctx); + j = sk_X509_OBJECT_find(xl->store_ctx->objs, &stmp); + tmp = sk_X509_OBJECT_value(xl->store_ctx->objs, j); + X509_STORE_unlock(xl->store_ctx); + + /* If a CRL, update the last file suffix added for this */ + + if (type == X509_LU_CRL) { if (!CRYPTO_THREAD_write_lock(ctx->lock)) goto finish; /* @@ -378,6 +346,7 @@ static int get_cert_by_subject_ex(X509_LOOKUP *xl, X509_LOOKUP_TYPE type, hent = OPENSSL_malloc(sizeof(*hent)); if (hent == NULL) { CRYPTO_THREAD_unlock(ctx->lock); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); ok = 0; goto finish; } @@ -386,16 +355,10 @@ static int get_cert_by_subject_ex(X509_LOOKUP *xl, X509_LOOKUP_TYPE type, if (!sk_BY_DIR_HASH_push(ent->hashes, hent)) { CRYPTO_THREAD_unlock(ctx->lock); OPENSSL_free(hent); - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); ok = 0; goto finish; } - - /* - * Ensure stack is sorted so that subsequent sk_BY_DIR_HASH_find - * will not mutate the stack and therefore require a write lock. - */ - sk_BY_DIR_HASH_sort(ent->hashes); } else if (hent->suffix < k) { hent->suffix = k; } @@ -419,14 +382,6 @@ static int get_cert_by_subject_ex(X509_LOOKUP *xl, X509_LOOKUP_TYPE type, } } finish: - /* If we changed anything, resort the objects for faster lookup */ - if (X509_STORE_lock(xl->store_ctx)) { - if (!sk_X509_OBJECT_is_sorted(xl->store_ctx->objs)) { - sk_X509_OBJECT_sort(xl->store_ctx->objs); - } - X509_STORE_unlock(xl->store_ctx); - } - BUF_MEM_free(b); return ok; } diff --git a/openssl/src/crypto/x509/by_file.c b/openssl/src/crypto/x509/by_file.c index cd5b75d3a..37d73ca84 100644 --- a/openssl/src/crypto/x509/by_file.c +++ b/openssl/src/crypto/x509/by_file.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -58,13 +58,15 @@ static int by_file_ctrl_ex(X509_LOOKUP *ctx, int cmd, const char *argp, if (file) ok = (X509_load_cert_crl_file_ex(ctx, file, X509_FILETYPE_PEM, libctx, propq) != 0); + else ok = (X509_load_cert_crl_file_ex( ctx, X509_get_default_cert_file(), X509_FILETYPE_PEM, libctx, propq) != 0); - if (!ok) + if (!ok) { ERR_raise(ERR_LIB_X509, X509_R_LOADING_DEFAULTS); + } } else { if (argl == X509_FILETYPE_PEM) ok = (X509_load_cert_crl_file_ex(ctx, argp, X509_FILETYPE_PEM, @@ -87,20 +89,25 @@ static int by_file_ctrl(X509_LOOKUP *ctx, int cmd, int X509_load_cert_file_ex(X509_LOOKUP *ctx, const char *file, int type, OSSL_LIB_CTX *libctx, const char *propq) { + int ret = 0; BIO *in = NULL; - int count = 0; + int i, count = 0; X509 *x = NULL; in = BIO_new(BIO_s_file()); if ((in == NULL) || (BIO_read_filename(in, file) <= 0)) { - ERR_raise(ERR_LIB_X509, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_SYS_LIB); goto err; } + if (type != X509_FILETYPE_PEM && type != X509_FILETYPE_ASN1) { + ERR_raise(ERR_LIB_X509, X509_R_BAD_X509_FILETYPE); + goto err; + } x = X509_new_ex(libctx, propq); if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } @@ -114,47 +121,34 @@ int X509_load_cert_file_ex(X509_LOOKUP *ctx, const char *file, int type, break; } else { ERR_clear_last_mark(); - if (count == 0) { - ERR_raise(ERR_LIB_X509, X509_R_NO_CERTIFICATE_FOUND); - } else { - ERR_raise(ERR_LIB_X509, ERR_R_PEM_LIB); - count = 0; - } goto err; } } ERR_clear_last_mark(); - if (!X509_STORE_add_cert(ctx->store_ctx, x)) { - count = 0; + i = X509_STORE_add_cert(ctx->store_ctx, x); + if (!i) goto err; - } - /* - * X509_STORE_add_cert() added a reference rather than a copy, - * so we need a fresh X509 object. - */ - X509_free(x); - x = X509_new_ex(libctx, propq); - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); - count = 0; - goto err; - } count++; + X509_free(x); + x = NULL; } + ret = count; } else if (type == X509_FILETYPE_ASN1) { if (d2i_X509_bio(in, &x) == NULL) { - ERR_raise(ERR_LIB_X509, X509_R_NO_CERTIFICATE_FOUND); + ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); goto err; } - count = X509_STORE_add_cert(ctx->store_ctx, x); - } else { - ERR_raise(ERR_LIB_X509, X509_R_BAD_X509_FILETYPE); - goto err; + i = X509_STORE_add_cert(ctx->store_ctx, x); + if (!i) + goto err; + ret = i; } + if (ret == 0) + ERR_raise(ERR_LIB_X509, X509_R_NO_CERTIFICATE_FOUND); err: X509_free(x); BIO_free(in); - return count; + return ret; } int X509_load_cert_file(X509_LOOKUP *ctx, const char *file, int type) @@ -164,14 +158,15 @@ int X509_load_cert_file(X509_LOOKUP *ctx, const char *file, int type) int X509_load_crl_file(X509_LOOKUP *ctx, const char *file, int type) { + int ret = 0; BIO *in = NULL; - int count = 0; + int i, count = 0; X509_CRL *x = NULL; in = BIO_new(BIO_s_file()); if ((in == NULL) || (BIO_read_filename(in, file) <= 0)) { - ERR_raise(ERR_LIB_X509, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_SYS_LIB); goto err; } @@ -184,75 +179,71 @@ int X509_load_crl_file(X509_LOOKUP *ctx, const char *file, int type) ERR_clear_error(); break; } else { - if (count == 0) { - ERR_raise(ERR_LIB_X509, X509_R_NO_CRL_FOUND); - } else { - ERR_raise(ERR_LIB_X509, ERR_R_PEM_LIB); - count = 0; - } + ERR_raise(ERR_LIB_X509, ERR_R_PEM_LIB); goto err; } } - if (!X509_STORE_add_crl(ctx->store_ctx, x)) { - count = 0; + i = X509_STORE_add_crl(ctx->store_ctx, x); + if (!i) goto err; - } count++; X509_CRL_free(x); x = NULL; } + ret = count; } else if (type == X509_FILETYPE_ASN1) { x = d2i_X509_CRL_bio(in, NULL); if (x == NULL) { - ERR_raise(ERR_LIB_X509, X509_R_NO_CRL_FOUND); + ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); goto err; } - count = X509_STORE_add_crl(ctx->store_ctx, x); + i = X509_STORE_add_crl(ctx->store_ctx, x); + if (!i) + goto err; + ret = i; } else { ERR_raise(ERR_LIB_X509, X509_R_BAD_X509_FILETYPE); goto err; } + if (ret == 0) + ERR_raise(ERR_LIB_X509, X509_R_NO_CRL_FOUND); err: X509_CRL_free(x); BIO_free(in); - return count; + return ret; } int X509_load_cert_crl_file_ex(X509_LOOKUP *ctx, const char *file, int type, OSSL_LIB_CTX *libctx, const char *propq) { - STACK_OF(X509_INFO) *inf = NULL; - X509_INFO *itmp = NULL; - BIO *in = NULL; + STACK_OF(X509_INFO) *inf; + X509_INFO *itmp; + BIO *in; int i, count = 0; if (type != X509_FILETYPE_PEM) return X509_load_cert_file_ex(ctx, file, type, libctx, propq); in = BIO_new_file(file, "r"); - if (in == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_BIO_LIB); + if (!in) { + ERR_raise(ERR_LIB_X509, ERR_R_SYS_LIB); return 0; } inf = PEM_X509_INFO_read_bio_ex(in, NULL, NULL, "", libctx, propq); BIO_free(in); - if (inf == NULL) { + if (!inf) { ERR_raise(ERR_LIB_X509, ERR_R_PEM_LIB); return 0; } for (i = 0; i < sk_X509_INFO_num(inf); i++) { itmp = sk_X509_INFO_value(inf, i); if (itmp->x509) { - if (!X509_STORE_add_cert(ctx->store_ctx, itmp->x509)) { - count = 0; + if (!X509_STORE_add_cert(ctx->store_ctx, itmp->x509)) goto err; - } count++; } if (itmp->crl) { - if (!X509_STORE_add_crl(ctx->store_ctx, itmp->crl)) { - count = 0; + if (!X509_STORE_add_crl(ctx->store_ctx, itmp->crl)) goto err; - } count++; } } diff --git a/openssl/src/crypto/x509/by_store.c b/openssl/src/crypto/x509/by_store.c index ee92f4b16..b9feb038b 100644 --- a/openssl/src/crypto/x509/by_store.c +++ b/openssl/src/crypto/x509/by_store.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -27,7 +27,7 @@ static int cache_objects(X509_LOOKUP *lctx, const char *uri, /* * We try to set the criterion, but don't care if it was valid or not. - * For an OSSL_STORE, it merely serves as an optimization, the expectation + * For a OSSL_STORE, it merely serves as an optimization, the expectation * being that if the criterion couldn't be used, we will get *everything* * from the container that the URI represents rather than the subset that * the criterion indicates, so the biggest harm is that we cache more @@ -114,22 +114,17 @@ static int by_store_ctrl_ex(X509_LOOKUP *ctx, int cmd, const char *argp, /* If no URI is given, use the default cert dir as default URI */ if (argp == NULL) argp = ossl_safe_getenv(X509_get_default_cert_dir_env()); - if (argp == NULL) argp = X509_get_default_cert_dir(); { STACK_OF(OPENSSL_STRING) *uris = X509_LOOKUP_get_method_data(ctx); - char *data = OPENSSL_strdup(argp); - if (data == NULL) { - return 0; - } if (uris == NULL) { uris = sk_OPENSSL_STRING_new_null(); X509_LOOKUP_set_method_data(ctx, uris); } - return sk_OPENSSL_STRING_push(uris, data) > 0; + return sk_OPENSSL_STRING_push(uris, OPENSSL_strdup(argp)) > 0; } case X509_L_LOAD_STORE: /* This is a shortcut for quick loading of specific containers */ diff --git a/openssl/src/crypto/x509/ext_dat.h b/openssl/src/crypto/x509/ext_dat.h index 1ffc816e5..07b5562c2 100644 --- a/openssl/src/crypto/x509/ext_dat.h +++ b/openssl/src/crypto/x509/ext_dat.h @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,11 +23,6 @@ extern const X509V3_EXT_METHOD ossl_v3_addr, ossl_v3_asid; extern const X509V3_EXT_METHOD ossl_v3_ct_scts[3]; extern const X509V3_EXT_METHOD ossl_v3_tls_feature; extern const X509V3_EXT_METHOD ossl_v3_ext_admission; -extern const X509V3_EXT_METHOD ossl_v3_utf8_list[1]; -extern const X509V3_EXT_METHOD ossl_v3_issuer_sign_tool; -extern const X509V3_EXT_METHOD ossl_v3_group_ac; -extern const X509V3_EXT_METHOD ossl_v3_soa_identifier; -extern const X509V3_EXT_METHOD ossl_v3_no_assertion; -extern const X509V3_EXT_METHOD ossl_v3_no_rev_avail; -extern const X509V3_EXT_METHOD ossl_v3_single_use; -extern const X509V3_EXT_METHOD ossl_v3_indirect_issuer; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +extern const X509V3_EXT_METHOD ossl_v3_dc_usage; +#endif diff --git a/openssl/src/crypto/x509/pcy_cache.c b/openssl/src/crypto/x509/pcy_cache.c index 2d1d4cd36..1339f994a 100644 --- a/openssl/src/crypto/x509/pcy_cache.c +++ b/openssl/src/crypto/x509/pcy_cache.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2004-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -35,14 +35,14 @@ static int policy_cache_create(X509 *x, goto bad_policy; cache->data = sk_X509_POLICY_DATA_new(policy_data_cmp); if (cache->data == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto just_cleanup; } for (i = 0; i < num; i++) { policy = sk_POLICYINFO_value(policies, i); data = ossl_policy_data_new(policy, NULL, crit); if (data == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto just_cleanup; } /* @@ -54,17 +54,15 @@ static int policy_cache_create(X509 *x, goto bad_policy; } cache->anyPolicy = data; - } else if (sk_X509_POLICY_DATA_find(cache->data, data) >=0) { + } else if (sk_X509_POLICY_DATA_find(cache->data, data) >=0 ) { ret = -1; goto bad_policy; } else if (!sk_X509_POLICY_DATA_push(cache->data, data)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto bad_policy; } data = NULL; } - /* Sort so we can find more quickly */ - sk_X509_POLICY_DATA_sort(cache->data); ret = 1; bad_policy: @@ -92,8 +90,10 @@ static int policy_cache_new(X509 *x) if (x->policy_cache != NULL) return 1; cache = OPENSSL_malloc(sizeof(*cache)); - if (cache == NULL) + if (cache == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return 0; + } cache->anyPolicy = NULL; cache->data = NULL; cache->any_skip = -1; diff --git a/openssl/src/crypto/x509/pcy_data.c b/openssl/src/crypto/x509/pcy_data.c index 8e8b91a78..6fb8f14ba 100644 --- a/openssl/src/crypto/x509/pcy_data.c +++ b/openssl/src/crypto/x509/pcy_data.c @@ -52,13 +52,14 @@ X509_POLICY_DATA *ossl_policy_data_new(POLICYINFO *policy, ret = OPENSSL_zalloc(sizeof(*ret)); if (ret == NULL) { ASN1_OBJECT_free(id); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } ret->expected_policy_set = sk_ASN1_OBJECT_new_null(); if (ret->expected_policy_set == NULL) { OPENSSL_free(ret); ASN1_OBJECT_free(id); - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/x509/pcy_local.h b/openssl/src/crypto/x509/pcy_local.h index 523f3e35f..cba107ca0 100644 --- a/openssl/src/crypto/x509/pcy_local.h +++ b/openssl/src/crypto/x509/pcy_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2004-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2004-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/x509/pcy_node.c b/openssl/src/crypto/x509/pcy_node.c index c6e7af5ab..450f95a65 100644 --- a/openssl/src/crypto/x509/pcy_node.c +++ b/openssl/src/crypto/x509/pcy_node.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2004-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -69,8 +69,10 @@ X509_POLICY_NODE *ossl_policy_level_add_node(X509_POLICY_LEVEL *level, return NULL; node = OPENSSL_zalloc(sizeof(*node)); - if (node == NULL) + if (node == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; + } node->data = data; node->parent = parent; if (level != NULL) { @@ -83,11 +85,11 @@ X509_POLICY_NODE *ossl_policy_level_add_node(X509_POLICY_LEVEL *level, if (level->nodes == NULL) level->nodes = ossl_policy_node_cmp_new(); if (level->nodes == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto node_error; } if (!sk_X509_POLICY_NODE_push(level->nodes, node)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto node_error; } } @@ -96,13 +98,13 @@ X509_POLICY_NODE *ossl_policy_level_add_node(X509_POLICY_LEVEL *level, if (extra_data) { if (tree->extra_data == NULL) tree->extra_data = sk_X509_POLICY_DATA_new_null(); - if (tree->extra_data == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); - goto extra_data_error; + if (tree->extra_data == NULL){ + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); + goto node_error; } if (!sk_X509_POLICY_DATA_push(tree->extra_data, data)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); - goto extra_data_error; + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); + goto node_error; } } @@ -112,14 +114,6 @@ X509_POLICY_NODE *ossl_policy_level_add_node(X509_POLICY_LEVEL *level, return node; - extra_data_error: - if (level != NULL) { - if (level->anyPolicy == node) - level->anyPolicy = NULL; - else - (void) sk_X509_POLICY_NODE_pop(level->nodes); - } - node_error: ossl_policy_node_free(node); return NULL; diff --git a/openssl/src/crypto/x509/pcy_tree.c b/openssl/src/crypto/x509/pcy_tree.c index d7307b12d..f953a05a4 100644 --- a/openssl/src/crypto/x509/pcy_tree.c +++ b/openssl/src/crypto/x509/pcy_tree.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2004-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,8 +25,6 @@ # define OPENSSL_POLICY_TREE_NODES_MAX 1000 #endif -static void exnode_free(X509_POLICY_NODE *node); - static void expected_print(BIO *channel, X509_POLICY_LEVEL *lev, X509_POLICY_NODE *node, int indent) @@ -171,8 +169,10 @@ static int tree_init(X509_POLICY_TREE **ptree, STACK_OF(X509) *certs, return ret; /* If we get this far initialize the tree */ - if ((tree = OPENSSL_zalloc(sizeof(*tree))) == NULL) + if ((tree = OPENSSL_zalloc(sizeof(*tree))) == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return X509_PCY_TREE_INTERNAL; + } /* Limit the growth of the tree to mitigate CVE-2023-0464 */ tree->node_maximum = OPENSSL_POLICY_TREE_NODES_MAX; @@ -186,6 +186,7 @@ static int tree_init(X509_POLICY_TREE **ptree, STACK_OF(X509) *certs, */ if ((tree->levels = OPENSSL_zalloc(sizeof(*tree->levels)*(n+1))) == NULL) { OPENSSL_free(tree); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return X509_PCY_TREE_INTERNAL; } tree->nlevel = n+1; @@ -569,22 +570,14 @@ static int tree_calculate_user_set(X509_POLICY_TREE *tree, | POLICY_DATA_FLAG_EXTRA_NODE; node = ossl_policy_level_add_node(NULL, extra, anyPolicy->parent, tree, 1); - if (node == NULL) { - ossl_policy_data_free(extra); - return 0; - } } if (!tree->user_policies) { tree->user_policies = sk_X509_POLICY_NODE_new_null(); - if (!tree->user_policies) { - exnode_free(node); - return 0; - } + if (!tree->user_policies) + return 1; } - if (!sk_X509_POLICY_NODE_push(tree->user_policies, node)) { - exnode_free(node); + if (!sk_X509_POLICY_NODE_push(tree->user_policies, node)) return 0; - } } return 1; } @@ -699,7 +692,6 @@ int X509_policy_check(X509_POLICY_TREE **ptree, int *pexplicit_policy, if ((calc_ret = tree_calculate_authority_set(tree, &auth_nodes)) == 0) goto error; - sk_X509_POLICY_NODE_sort(auth_nodes); ret = tree_calculate_user_set(tree, policy_oids, auth_nodes); if (calc_ret == TREE_CALC_OK_DOFREE) sk_X509_POLICY_NODE_free(auth_nodes); diff --git a/openssl/src/crypto/x509/standard_exts.h b/openssl/src/crypto/x509/standard_exts.h index 87a564b23..bad82c1ed 100644 --- a/openssl/src/crypto/x509/standard_exts.h +++ b/openssl/src/crypto/x509/standard_exts.h @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -53,7 +53,6 @@ static const X509V3_EXT_METHOD *standard_exts[] = { #endif &ossl_v3_sinfo, &ossl_v3_policy_constraints, - &ossl_v3_no_rev_avail, #ifndef OPENSSL_NO_OCSP &ossl_v3_crl_hold, #endif @@ -69,15 +68,11 @@ static const X509V3_EXT_METHOD *standard_exts[] = { &ossl_v3_ct_scts[1], &ossl_v3_ct_scts[2], #endif - &ossl_v3_utf8_list[0], - &ossl_v3_issuer_sign_tool, &ossl_v3_tls_feature, &ossl_v3_ext_admission, - &ossl_v3_soa_identifier, - &ossl_v3_indirect_issuer, - &ossl_v3_no_assertion, - &ossl_v3_single_use, - &ossl_v3_group_ac +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + &ossl_v3_dc_usage, +#endif }; /* Number of standard extensions */ diff --git a/openssl/src/crypto/x509/t_req.c b/openssl/src/crypto/x509/t_req.c index 63626c0d9..095c16510 100644 --- a/openssl/src/crypto/x509/t_req.c +++ b/openssl/src/crypto/x509/t_req.c @@ -42,15 +42,15 @@ int X509_REQ_print_ex(BIO *bp, X509_REQ *x, unsigned long nmflags, EVP_PKEY *pkey; STACK_OF(X509_EXTENSION) *exts; char mlch = ' '; - int nmindent = 0, printok = 0; + int nmindent = 0; if ((nmflags & XN_FLAG_SEP_MASK) == XN_FLAG_SEP_MULTILINE) { mlch = '\n'; nmindent = 12; } - if (nmflags == XN_FLAG_COMPAT) - printok = 1; + if (nmflags == X509_FLAG_COMPAT) + nmindent = 16; if (!(cflag & X509_FLAG_NO_HEADER)) { if (BIO_write(bp, "Certificate Request:\n", 21) <= 0) @@ -72,7 +72,7 @@ int X509_REQ_print_ex(BIO *bp, X509_REQ *x, unsigned long nmflags, if (BIO_printf(bp, " Subject:%c", mlch) <= 0) goto err; if (X509_NAME_print_ex(bp, X509_REQ_get_subject_name(x), - nmindent, nmflags) < printok) + nmindent, nmflags) < 0) goto err; if (BIO_write(bp, "\n", 1) <= 0) goto err; diff --git a/openssl/src/crypto/x509/t_x509.c b/openssl/src/crypto/x509/t_x509.c index 192998d45..95ee5f519 100644 --- a/openssl/src/crypto/x509/t_x509.c +++ b/openssl/src/crypto/x509/t_x509.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,11 +17,6 @@ #include "crypto/asn1.h" #include "crypto/x509.h" -void OSSL_STACK_OF_X509_free(STACK_OF(X509) *certs) -{ - sk_X509_pop_free(certs, X509_free); -} - #ifndef OPENSSL_NO_STDIO int X509_print_fp(FILE *fp, X509 *x) { @@ -55,7 +50,7 @@ int X509_print_ex(BIO *bp, X509 *x, unsigned long nmflags, { long l; int ret = 0, i; - char mlch = ' '; + char *m = NULL, mlch = ' '; int nmindent = 0, printok = 0; EVP_PKEY *pkey = NULL; const char *neg; @@ -65,8 +60,10 @@ int X509_print_ex(BIO *bp, X509 *x, unsigned long nmflags, nmindent = 12; } - if (nmflags == XN_FLAG_COMPAT) + if (nmflags == X509_FLAG_COMPAT) { + nmindent = 16; printok = 1; + } if (!(cflag & X509_FLAG_NO_HEADER)) { if (BIO_write(bp, "Certificate:\n", 13) <= 0) @@ -220,6 +217,7 @@ int X509_print_ex(BIO *bp, X509 *x, unsigned long nmflags, } ret = 1; err: + OPENSSL_free(m); return ret; } @@ -458,7 +456,7 @@ static int print_store_certs(BIO *bio, X509_STORE *store) STACK_OF(X509) *certs = X509_STORE_get1_all_certs(store); int ret = print_certs(bio, certs); - OSSL_STACK_OF_X509_free(certs); + sk_X509_pop_free(certs, X509_free); return ret; } else { return BIO_printf(bio, " (no trusted store)\n") >= 0; @@ -472,8 +470,6 @@ int X509_STORE_CTX_print_verify_cb(int ok, X509_STORE_CTX *ctx) int cert_error = X509_STORE_CTX_get_error(ctx); BIO *bio = BIO_new(BIO_s_mem()); /* may be NULL */ - if (bio == NULL) - return 0; BIO_printf(bio, "%s at depth = %d error = %d (%s)\n", X509_STORE_CTX_get0_parent_ctx(ctx) != NULL ? "CRL path validation" diff --git a/openssl/src/crypto/x509/v3_addr.c b/openssl/src/crypto/x509/v3_addr.c index d0e5f9efe..8bb35bd8a 100644 --- a/openssl/src/crypto/x509/v3_addr.c +++ b/openssl/src/crypto/x509/v3_addr.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,16 +13,13 @@ #include #include -#include -#include +#include "internal/cryptlib.h" #include #include #include #include #include -#include "internal/cryptlib.h" -#include "crypto/asn1.h" #include "crypto/x509.h" #include "ext_dat.h" #include "x509_local.h" @@ -34,28 +31,28 @@ */ ASN1_SEQUENCE(IPAddressRange) = { - ASN1_SIMPLE(IPAddressRange, min, ASN1_BIT_STRING), - ASN1_SIMPLE(IPAddressRange, max, ASN1_BIT_STRING) + ASN1_SIMPLE(IPAddressRange, min, ASN1_BIT_STRING), + ASN1_SIMPLE(IPAddressRange, max, ASN1_BIT_STRING) } ASN1_SEQUENCE_END(IPAddressRange) ASN1_CHOICE(IPAddressOrRange) = { - ASN1_SIMPLE(IPAddressOrRange, u.addressPrefix, ASN1_BIT_STRING), - ASN1_SIMPLE(IPAddressOrRange, u.addressRange, IPAddressRange) + ASN1_SIMPLE(IPAddressOrRange, u.addressPrefix, ASN1_BIT_STRING), + ASN1_SIMPLE(IPAddressOrRange, u.addressRange, IPAddressRange) } ASN1_CHOICE_END(IPAddressOrRange) ASN1_CHOICE(IPAddressChoice) = { - ASN1_SIMPLE(IPAddressChoice, u.inherit, ASN1_NULL), - ASN1_SEQUENCE_OF(IPAddressChoice, u.addressesOrRanges, IPAddressOrRange) + ASN1_SIMPLE(IPAddressChoice, u.inherit, ASN1_NULL), + ASN1_SEQUENCE_OF(IPAddressChoice, u.addressesOrRanges, IPAddressOrRange) } ASN1_CHOICE_END(IPAddressChoice) ASN1_SEQUENCE(IPAddressFamily) = { - ASN1_SIMPLE(IPAddressFamily, addressFamily, ASN1_OCTET_STRING), - ASN1_SIMPLE(IPAddressFamily, ipAddressChoice, IPAddressChoice) + ASN1_SIMPLE(IPAddressFamily, addressFamily, ASN1_OCTET_STRING), + ASN1_SIMPLE(IPAddressFamily, ipAddressChoice, IPAddressChoice) } ASN1_SEQUENCE_END(IPAddressFamily) ASN1_ITEM_TEMPLATE(IPAddrBlocks) = - ASN1_EX_TEMPLATE_TYPE(ASN1_TFLG_SEQUENCE_OF, 0, - IPAddrBlocks, IPAddressFamily) + ASN1_EX_TEMPLATE_TYPE(ASN1_TFLG_SEQUENCE_OF, 0, + IPAddrBlocks, IPAddressFamily) static_ASN1_ITEM_TEMPLATE_END(IPAddrBlocks) IMPLEMENT_ASN1_FUNCTIONS(IPAddressRange) @@ -66,7 +63,7 @@ IMPLEMENT_ASN1_FUNCTIONS(IPAddressFamily) /* * How much buffer space do we need for a raw address? */ -# define ADDR_RAW_BUF_LEN 16 +#define ADDR_RAW_BUF_LEN 16 /* * What's the address length associated with this AFI? @@ -110,7 +107,6 @@ static int addr_expand(unsigned char *addr, memcpy(addr, bs->data, bs->length); if ((bs->flags & 7) != 0) { unsigned char mask = 0xFF >> (8 - (bs->flags & 7)); - if (fill == 0) addr[bs->length - 1] &= ~mask; else @@ -124,7 +120,7 @@ static int addr_expand(unsigned char *addr, /* * Extract the prefix length from a bitstring. */ -# define addr_prefixlen(bs) ((int)((bs)->length * 8 - ((bs)->flags & 7))) +#define addr_prefixlen(bs) ((int) ((bs)->length * 8 - ((bs)->flags & 7))) /* * i2r handler for one address bitstring. @@ -175,10 +171,8 @@ static int i2r_IPAddressOrRanges(BIO *out, const unsigned afi) { int i; - for (i = 0; i < sk_IPAddressOrRange_num(aors); i++) { const IPAddressOrRange *aor = sk_IPAddressOrRange_value(aors, i); - BIO_printf(out, "%*s", indent, ""); switch (aor->type) { case IPAddressOrRange_addressPrefix: @@ -207,11 +201,9 @@ static int i2r_IPAddrBlocks(const X509V3_EXT_METHOD *method, { const IPAddrBlocks *addr = ext; int i; - for (i = 0; i < sk_IPAddressFamily_num(addr); i++) { IPAddressFamily *f = sk_IPAddressFamily_value(addr, i); const unsigned int afi = X509v3_addr_get_afi(f); - switch (afi) { case IANA_AFI_IPV4: BIO_printf(out, "%*sIPv4", indent, ""); @@ -300,8 +292,6 @@ static int IPAddressOrRange_cmp(const IPAddressOrRange *a, return -1; prefixlen_a = length * 8; break; - default: - return -1; } switch (b->type) { @@ -315,8 +305,6 @@ static int IPAddressOrRange_cmp(const IPAddressOrRange *a, return -1; prefixlen_b = length * 8; break; - default: - return -1; } if ((r = memcmp(addr_a, addr_b, length)) != 0) @@ -355,13 +343,8 @@ static int range_should_be_prefix(const unsigned char *min, unsigned char mask; int i, j; - /* - * It is the responsibility of the caller to confirm min <= max. We don't - * use ossl_assert() here since we have no way of signalling an error from - * this function - so we just use a plain assert instead. - */ - assert(memcmp(min, max, length) <= 0); - + if (memcmp(min, max, length) <= 0) + return -1; for (i = 0; i < length && min[i] == max[i]; i++) ; for (j = length - 1; j >= 0 && min[j] == 0x00 && max[j] == 0xFF; j--) ; if (i < j) @@ -403,15 +386,13 @@ static int range_should_be_prefix(const unsigned char *min, /* * Construct a prefix. */ -static int make_addressPrefix(IPAddressOrRange **result, unsigned char *addr, - const int prefixlen, const int afilen) +static int make_addressPrefix(IPAddressOrRange **result, + unsigned char *addr, const int prefixlen) { int bytelen = (prefixlen + 7) / 8, bitlen = prefixlen % 8; - IPAddressOrRange *aor; + IPAddressOrRange *aor = IPAddressOrRange_new(); - if (prefixlen < 0 || prefixlen > (afilen * 8)) - return 0; - if ((aor = IPAddressOrRange_new()) == NULL) + if (aor == NULL) return 0; aor->type = IPAddressOrRange_addressPrefix; if (aor->u.addressPrefix == NULL && @@ -419,9 +400,12 @@ static int make_addressPrefix(IPAddressOrRange **result, unsigned char *addr, goto err; if (!ASN1_BIT_STRING_set(aor->u.addressPrefix, addr, bytelen)) goto err; - if (bitlen > 0) + aor->u.addressPrefix->flags &= ~7; + aor->u.addressPrefix->flags |= ASN1_STRING_FLAG_BITS_LEFT; + if (bitlen > 0) { aor->u.addressPrefix->data[bytelen - 1] &= ~(0xFF >> bitlen); - ossl_asn1_string_set_bits_left(aor->u.addressPrefix, 8 - bitlen); + aor->u.addressPrefix->flags |= 8 - bitlen; + } *result = aor; return 1; @@ -443,11 +427,8 @@ static int make_addressRange(IPAddressOrRange **result, IPAddressOrRange *aor; int i, prefixlen; - if (memcmp(min, max, length) > 0) - return 0; - if ((prefixlen = range_should_be_prefix(min, max, length)) >= 0) - return make_addressPrefix(result, min, prefixlen, length); + return make_addressPrefix(result, min, prefixlen); if ((aor = IPAddressOrRange_new()) == NULL) return 0; @@ -464,11 +445,11 @@ static int make_addressRange(IPAddressOrRange **result, for (i = length; i > 0 && min[i - 1] == 0x00; --i) ; if (!ASN1_BIT_STRING_set(aor->u.addressRange->min, min, i)) goto err; - ossl_asn1_string_set_bits_left(aor->u.addressRange->min, 0); + aor->u.addressRange->min->flags &= ~7; + aor->u.addressRange->min->flags |= ASN1_STRING_FLAG_BITS_LEFT; if (i > 0) { unsigned char b = min[i - 1]; int j = 1; - while ((b & (0xFFU >> j)) != 0) ++j; aor->u.addressRange->min->flags |= 8 - j; @@ -477,11 +458,11 @@ static int make_addressRange(IPAddressOrRange **result, for (i = length; i > 0 && max[i - 1] == 0xFF; --i) ; if (!ASN1_BIT_STRING_set(aor->u.addressRange->max, max, i)) goto err; - ossl_asn1_string_set_bits_left(aor->u.addressRange->max, 0); + aor->u.addressRange->max->flags &= ~7; + aor->u.addressRange->max->flags |= ASN1_STRING_FLAG_BITS_LEFT; if (i > 0) { unsigned char b = max[i - 1]; int j = 1; - while ((b & (0xFFU >> j)) != (0xFFU >> j)) ++j; aor->u.addressRange->max->flags |= 8 - j; @@ -550,7 +531,6 @@ int X509v3_addr_add_inherit(IPAddrBlocks *addr, const unsigned afi, const unsigned *safi) { IPAddressFamily *f = make_IPAddressFamily(addr, afi, safi); - if (f == NULL || f->ipAddressChoice == NULL || (f->ipAddressChoice->type == IPAddressChoice_addressesOrRanges && @@ -610,9 +590,7 @@ int X509v3_addr_add_prefix(IPAddrBlocks *addr, { IPAddressOrRanges *aors = make_prefix_or_range(addr, afi, safi); IPAddressOrRange *aor; - - if (aors == NULL - || !make_addressPrefix(&aor, a, prefixlen, length_from_afi(afi))) + if (aors == NULL || !make_addressPrefix(&aor, a, prefixlen)) return 0; if (sk_IPAddressOrRange_push(aors, aor)) return 1; @@ -631,7 +609,6 @@ int X509v3_addr_add_range(IPAddrBlocks *addr, IPAddressOrRanges *aors = make_prefix_or_range(addr, afi, safi); IPAddressOrRange *aor; int length = length_from_afi(afi); - if (aors == NULL) return 0; if (!make_addressRange(&aor, min, max, length)) @@ -670,7 +647,6 @@ int X509v3_addr_get_range(IPAddressOrRange *aor, unsigned char *max, const int length) { int afi_length = length_from_afi(afi); - if (aor == NULL || min == NULL || max == NULL || afi_length == 0 || length < afi_length || (aor->type != IPAddressOrRange_addressPrefix && @@ -698,18 +674,9 @@ static int IPAddressFamily_cmp(const IPAddressFamily *const *a_, const ASN1_OCTET_STRING *b = (*b_)->addressFamily; int len = ((a->length <= b->length) ? a->length : b->length); int cmp = memcmp(a->data, b->data, len); - return cmp ? cmp : a->length - b->length; } -static int IPAddressFamily_check_len(const IPAddressFamily *f) -{ - if (f->addressFamily->length < 2 || f->addressFamily->length > 3) - return 0; - else - return 1; -} - /* * Check whether an IPAddrBLocks is in canonical form. */ @@ -732,10 +699,6 @@ int X509v3_addr_is_canonical(IPAddrBlocks *addr) for (i = 0; i < sk_IPAddressFamily_num(addr) - 1; i++) { const IPAddressFamily *a = sk_IPAddressFamily_value(addr, i); const IPAddressFamily *b = sk_IPAddressFamily_value(addr, i + 1); - - if (!IPAddressFamily_check_len(a) || !IPAddressFamily_check_len(b)) - return 0; - if (IPAddressFamily_cmp(&a, &b) >= 0) return 0; } @@ -762,9 +725,6 @@ int X509v3_addr_is_canonical(IPAddrBlocks *addr) return 0; } - if (!IPAddressFamily_check_len(f)) - return 0; - /* * It's an IPAddressOrRanges sequence, check it. */ @@ -810,7 +770,6 @@ int X509v3_addr_is_canonical(IPAddrBlocks *addr) j = sk_IPAddressOrRange_num(aors) - 1; { IPAddressOrRange *a = sk_IPAddressOrRange_value(aors, j); - if (a != NULL && a->type == IPAddressOrRange_addressRange) { if (!extract_min_max(a, a_min, a_max, length)) return 0; @@ -873,7 +832,6 @@ static int IPAddressOrRanges_canonize(IPAddressOrRanges *aors, for (j = length - 1; j >= 0 && b_min[j]-- == 0x00; j--) ; if (memcmp(a_max, b_min, length) == 0) { IPAddressOrRange *merged; - if (!make_addressRange(&merged, a_min, b_max, length)) return 0; (void)sk_IPAddressOrRange_set(aors, i, merged); @@ -891,10 +849,8 @@ static int IPAddressOrRanges_canonize(IPAddressOrRanges *aors, j = sk_IPAddressOrRange_num(aors) - 1; { IPAddressOrRange *a = sk_IPAddressOrRange_value(aors, j); - if (a != NULL && a->type == IPAddressOrRange_addressRange) { unsigned char a_min[ADDR_RAW_BUF_LEN], a_max[ADDR_RAW_BUF_LEN]; - if (!extract_min_max(a, a_min, a_max, length)) return 0; if (memcmp(a_min, a_max, length) > 0) @@ -911,13 +867,8 @@ static int IPAddressOrRanges_canonize(IPAddressOrRanges *aors, int X509v3_addr_canonize(IPAddrBlocks *addr) { int i; - for (i = 0; i < sk_IPAddressFamily_num(addr); i++) { IPAddressFamily *f = sk_IPAddressFamily_value(addr, i); - - if (!IPAddressFamily_check_len(f)) - return 0; - if (f->ipAddressChoice->type == IPAddressChoice_addressesOrRanges && !IPAddressOrRanges_canonize(f->ipAddressChoice-> u.addressesOrRanges, @@ -945,7 +896,7 @@ static void *v2i_IPAddrBlocks(const struct v3_ext_method *method, int i; if ((addr = sk_IPAddressFamily_new(IPAddressFamily_cmp)) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } @@ -988,10 +939,6 @@ static void *v2i_IPAddrBlocks(const struct v3_ext_method *method, * the other input values. */ if (safi != NULL) { - if (val->value == NULL) { - ERR_raise(ERR_LIB_X509V3, X509V3_R_MISSING_VALUE); - goto err; - } *safi = strtoul(val->value, &t, 0); t += strspn(t, " \t"); if (*safi > 0xFF || *t++ != ':') { @@ -1004,8 +951,10 @@ static void *v2i_IPAddrBlocks(const struct v3_ext_method *method, } else { s = OPENSSL_strdup(val->value); } - if (s == NULL) + if (s == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; + } /* * Check for inheritance. Not worth additional complexity to @@ -1036,16 +985,13 @@ static void *v2i_IPAddrBlocks(const struct v3_ext_method *method, switch (delim) { case '/': prefixlen = (int)strtoul(s + i2, &t, 10); - if (t == s + i2 - || *t != '\0' - || prefixlen > (length * 8) - || prefixlen < 0) { + if (t == s + i2 || *t != '\0') { ERR_raise(ERR_LIB_X509V3, X509V3_R_EXTENSION_VALUE_ERROR); X509V3_conf_add_error_name_value(val); goto err; } if (!X509v3_addr_add_prefix(addr, afi, safi, min, prefixlen)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } break; @@ -1068,13 +1014,13 @@ static void *v2i_IPAddrBlocks(const struct v3_ext_method *method, goto err; } if (!X509v3_addr_add_range(addr, afi, safi, min, max)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } break; case '\0': if (!X509v3_addr_add_prefix(addr, afi, safi, min, length * 8)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } break; @@ -1124,12 +1070,10 @@ const X509V3_EXT_METHOD ossl_v3_addr = { int X509v3_addr_inherits(IPAddrBlocks *addr) { int i; - if (addr == NULL) return 0; for (i = 0; i < sk_IPAddressFamily_num(addr); i++) { IPAddressFamily *f = sk_IPAddressFamily_value(addr, i); - if (f->ipAddressChoice->type == IPAddressChoice_inherit) return 1; } @@ -1155,7 +1099,7 @@ static int addr_contains(IPAddressOrRanges *parent, for (c = 0; c < sk_IPAddressOrRange_num(child); c++) { if (!extract_min_max(sk_IPAddressOrRange_value(child, c), c_min, c_max, length)) - return 0; + return -1; for (;; p++) { if (p >= sk_IPAddressOrRange_num(parent)) return 0; @@ -1179,23 +1123,18 @@ static int addr_contains(IPAddressOrRanges *parent, int X509v3_addr_subset(IPAddrBlocks *a, IPAddrBlocks *b) { int i; - if (a == NULL || a == b) return 1; if (b == NULL || X509v3_addr_inherits(a) || X509v3_addr_inherits(b)) return 0; (void)sk_IPAddressFamily_set_cmp_func(b, IPAddressFamily_cmp); - sk_IPAddressFamily_sort(b); - /* Could sort a here too and get O(|a|) running time instead of O(|a| ln |b|) */ for (i = 0; i < sk_IPAddressFamily_num(a); i++) { IPAddressFamily *fa = sk_IPAddressFamily_value(a, i); int j = sk_IPAddressFamily_find(b, fa); - IPAddressFamily *fb = sk_IPAddressFamily_value(b, j); - + IPAddressFamily *fb; + fb = sk_IPAddressFamily_value(b, j); if (fb == NULL) return 0; - if (!IPAddressFamily_check_len(fa) || !IPAddressFamily_check_len(fb)) - return 0; if (!addr_contains(fb->ipAddressChoice->u.addressesOrRanges, fa->ipAddressChoice->u.addressesOrRanges, length_from_afi(X509v3_addr_get_afi(fb)))) @@ -1207,19 +1146,19 @@ int X509v3_addr_subset(IPAddrBlocks *a, IPAddrBlocks *b) /* * Validation error handling via callback. */ -# define validation_err(_err_) \ - do { \ - if (ctx != NULL) { \ - ctx->error = _err_; \ - ctx->error_depth = i; \ - ctx->current_cert = x; \ - rv = ctx->verify_cb(0, ctx); \ - } else { \ - rv = 0; \ - } \ - if (rv == 0) \ - goto done; \ - } while (0) +#define validation_err(_err_) \ + do { \ + if (ctx != NULL) { \ + ctx->error = _err_; \ + ctx->error_depth = i; \ + ctx->current_cert = x; \ + ret = ctx->verify_cb(0, ctx); \ + } else { \ + ret = 0; \ + } \ + if (!ret) \ + goto done; \ + } while (0) /* * Core code for RFC 3779 2.3 path validation. @@ -1234,7 +1173,7 @@ static int addr_validate_path_internal(X509_STORE_CTX *ctx, IPAddrBlocks *ext) { IPAddrBlocks *child = NULL; - int i, j, ret = 0, rv; + int i, j, ret = 1; X509 *x; if (!ossl_assert(chain != NULL && sk_X509_num(chain) > 0) @@ -1257,18 +1196,18 @@ static int addr_validate_path_internal(X509_STORE_CTX *ctx, i = 0; x = sk_X509_value(chain, i); if ((ext = x->rfc3779_addr) == NULL) - return 1; /* Return success */ + goto done; } if (!X509v3_addr_is_canonical(ext)) validation_err(X509_V_ERR_INVALID_EXTENSION); (void)sk_IPAddressFamily_set_cmp_func(ext, IPAddressFamily_cmp); if ((child = sk_IPAddressFamily_dup(ext)) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); if (ctx != NULL) ctx->error = X509_V_ERR_OUT_OF_MEM; + ret = 0; goto done; } - sk_IPAddressFamily_sort(child); /* * Now walk up the chain. No cert may list resources that its @@ -1281,10 +1220,6 @@ static int addr_validate_path_internal(X509_STORE_CTX *ctx, if (x->rfc3779_addr == NULL) { for (j = 0; j < sk_IPAddressFamily_num(child); j++) { IPAddressFamily *fc = sk_IPAddressFamily_value(child, j); - - if (!IPAddressFamily_check_len(fc)) - goto done; - if (fc->ipAddressChoice->type != IPAddressChoice_inherit) { validation_err(X509_V_ERR_UNNESTED_RESOURCE); break; @@ -1294,13 +1229,11 @@ static int addr_validate_path_internal(X509_STORE_CTX *ctx, } (void)sk_IPAddressFamily_set_cmp_func(x->rfc3779_addr, IPAddressFamily_cmp); - sk_IPAddressFamily_sort(x->rfc3779_addr); for (j = 0; j < sk_IPAddressFamily_num(child); j++) { IPAddressFamily *fc = sk_IPAddressFamily_value(child, j); int k = sk_IPAddressFamily_find(x->rfc3779_addr, fc); IPAddressFamily *fp = sk_IPAddressFamily_value(x->rfc3779_addr, k); - if (fp == NULL) { if (fc->ipAddressChoice->type == IPAddressChoice_addressesOrRanges) { @@ -1309,10 +1242,6 @@ static int addr_validate_path_internal(X509_STORE_CTX *ctx, } continue; } - - if (!IPAddressFamily_check_len(fc) || !IPAddressFamily_check_len(fp)) - goto done; - if (fp->ipAddressChoice->type == IPAddressChoice_addressesOrRanges) { if (fc->ipAddressChoice->type == IPAddressChoice_inherit @@ -1331,23 +1260,20 @@ static int addr_validate_path_internal(X509_STORE_CTX *ctx, */ if (x->rfc3779_addr != NULL) { for (j = 0; j < sk_IPAddressFamily_num(x->rfc3779_addr); j++) { - IPAddressFamily *fp = sk_IPAddressFamily_value(x->rfc3779_addr, j); - - if (!IPAddressFamily_check_len(fp)) - goto done; - + IPAddressFamily *fp = + sk_IPAddressFamily_value(x->rfc3779_addr, j); if (fp->ipAddressChoice->type == IPAddressChoice_inherit && sk_IPAddressFamily_find(child, fp) >= 0) validation_err(X509_V_ERR_UNNESTED_RESOURCE); } } - ret = 1; + done: sk_IPAddressFamily_free(child); return ret; } -# undef validation_err +#undef validation_err /* * RFC 3779 2.3 path validation -- called from X509_verify_cert(). @@ -1368,7 +1294,7 @@ int X509v3_addr_validate_path(X509_STORE_CTX *ctx) * Test whether chain covers extension. */ int X509v3_addr_validate_resource_set(STACK_OF(X509) *chain, - IPAddrBlocks *ext, int allow_inheritance) + IPAddrBlocks *ext, int allow_inheritance) { if (ext == NULL) return 1; @@ -1379,4 +1305,4 @@ int X509v3_addr_validate_resource_set(STACK_OF(X509) *chain, return addr_validate_path_internal(NULL, chain, ext); } -#endif /* OPENSSL_NO_RFC3779 */ +#endif /* OPENSSL_NO_RFC3779 */ diff --git a/openssl/src/crypto/x509/v3_admis.c b/openssl/src/crypto/x509/v3_admis.c index c3182a71d..bee9cc993 100644 --- a/openssl/src/crypto/x509/v3_admis.c +++ b/openssl/src/crypto/x509/v3_admis.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -71,7 +71,7 @@ const X509V3_EXT_METHOD ossl_v3_ext_admission = { static int i2r_NAMING_AUTHORITY(const struct v3_ext_method *method, void *in, BIO *bp, int ind) { - NAMING_AUTHORITY *namingAuthority = (NAMING_AUTHORITY*) in; + NAMING_AUTHORITY * namingAuthority = (NAMING_AUTHORITY*) in; if (namingAuthority == NULL) return 0; @@ -103,7 +103,7 @@ static int i2r_NAMING_AUTHORITY(const struct v3_ext_method *method, void *in, || BIO_printf(bp, "\n") <= 0) goto err; } - if (namingAuthority->namingAuthorityUrl != NULL) { + if (namingAuthority->namingAuthorityUrl != NULL ) { if (BIO_printf(bp, "%*s namingAuthorityUrl: ", ind, "") <= 0 || ASN1_STRING_print(bp, namingAuthority->namingAuthorityUrl) <= 0 || BIO_printf(bp, "\n") <= 0) @@ -118,7 +118,7 @@ static int i2r_NAMING_AUTHORITY(const struct v3_ext_method *method, void *in, static int i2r_ADMISSION_SYNTAX(const struct v3_ext_method *method, void *in, BIO *bp, int ind) { - ADMISSION_SYNTAX *admission = (ADMISSION_SYNTAX *)in; + ADMISSION_SYNTAX * admission = (ADMISSION_SYNTAX *)in; int i, j, k; if (admission->admissionAuthority != NULL) { @@ -199,7 +199,7 @@ static int i2r_ADMISSION_SYNTAX(const struct v3_ext_method *method, void *in, return 1; err: - return 0; + return -1; } const ASN1_OBJECT *NAMING_AUTHORITY_get0_authorityId(const NAMING_AUTHORITY *n) diff --git a/openssl/src/crypto/x509/v3_akid.c b/openssl/src/crypto/x509/v3_akid.c index de93dae70..43b515f50 100644 --- a/openssl/src/crypto/x509/v3_akid.c +++ b/openssl/src/crypto/x509/v3_akid.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -44,9 +44,9 @@ static STACK_OF(CONF_VALUE) *i2v_AUTHORITY_KEYID(X509V3_EXT_METHOD *method, STACK_OF(CONF_VALUE) *origextlist = extlist, *tmpextlist; if (akeyid->keyid) { - tmp = i2s_ASN1_OCTET_STRING(NULL, akeyid->keyid); + tmp = OPENSSL_buf2hexstr(akeyid->keyid->data, akeyid->keyid->length); if (tmp == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } if (!X509V3_add_value((akeyid->issuer || akeyid->serial) ? "keyid" : NULL, @@ -66,9 +66,9 @@ static STACK_OF(CONF_VALUE) *i2v_AUTHORITY_KEYID(X509V3_EXT_METHOD *method, extlist = tmpextlist; } if (akeyid->serial) { - tmp = i2s_ASN1_OCTET_STRING(NULL, akeyid->serial); + tmp = OPENSSL_buf2hexstr(akeyid->serial->data, akeyid->serial->length); if (tmp == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } if (!X509V3_add_value("serial", tmp, &extlist)) { @@ -85,14 +85,14 @@ static STACK_OF(CONF_VALUE) *i2v_AUTHORITY_KEYID(X509V3_EXT_METHOD *method, } /*- - * Three explicit tags may be given, where 'keyid' and 'issuer' may be combined: - * 'none': do not add any authority key identifier. - * 'keyid': use the issuer's subject keyid; the option 'always' means its is - * an error if the issuer certificate doesn't have a subject key id. - * 'issuer': use the issuer's cert issuer and serial number. The default is - * to only use this if 'keyid' is not present. With the option 'always' + * Currently two options: + * keyid: use the issuers subject keyid, the value 'always' means its is + * an error if the issuer certificate doesn't have a key id. + * issuer: use the issuers cert issuer and serial number. The default is + * to only use this if keyid is not present. With the option 'always' * this is always included. */ + static AUTHORITY_KEYID *v2i_AUTHORITY_KEYID(X509V3_EXT_METHOD *method, X509V3_CTX *ctx, STACK_OF(CONF_VALUE) *values) @@ -119,27 +119,16 @@ static AUTHORITY_KEYID *v2i_AUTHORITY_KEYID(X509V3_EXT_METHOD *method, for (i = 0; i < n; i++) { cnf = sk_CONF_VALUE_value(values, i); - if (cnf->value != NULL && strcmp(cnf->value, "always") != 0) { - ERR_raise_data(ERR_LIB_X509V3, X509V3_R_UNKNOWN_OPTION, - "name=%s option=%s", cnf->name, cnf->value); - goto err; - } - if (strcmp(cnf->name, "keyid") == 0 && keyid == 0) { + if (strcmp(cnf->name, "keyid") == 0) { keyid = 1; - if (cnf->value != NULL) + if (cnf->value && strcmp(cnf->value, "always") == 0) keyid = 2; - } else if (strcmp(cnf->name, "issuer") == 0 && issuer == 0) { + } else if (strcmp(cnf->name, "issuer") == 0) { issuer = 1; - if (cnf->value != NULL) + if (cnf->value && strcmp(cnf->value, "always") == 0) issuer = 2; - } else if (strcmp(cnf->name, "none") == 0 - || strcmp(cnf->name, "keyid") == 0 - || strcmp(cnf->name, "issuer") == 0) { - ERR_raise_data(ERR_LIB_X509V3, X509V3_R_BAD_VALUE, - "name=%s", cnf->name); - goto err; } else { - ERR_raise_data(ERR_LIB_X509V3, X509V3_R_UNKNOWN_VALUE, + ERR_raise_data(ERR_LIB_X509V3, X509V3_R_UNKNOWN_OPTION, "name=%s", cnf->name); goto err; } @@ -172,13 +161,8 @@ static AUTHORITY_KEYID *v2i_AUTHORITY_KEYID(X509V3_EXT_METHOD *method, */ i = X509_get_ext_by_NID(issuer_cert, NID_subject_key_identifier, -1); if (i >= 0 && (ext = X509_get_ext(issuer_cert, i)) != NULL - && !(same_issuer && !ss)) { + && !(same_issuer && !ss)) ikeyid = X509V3_EXT_d2i(ext); - if (ASN1_STRING_length(ikeyid) == 0) /* indicating "none" */ { - ASN1_OCTET_STRING_free(ikeyid); - ikeyid = NULL; - } - } if (ikeyid == NULL && same_issuer && ctx->issuer_pkey != NULL) { /* generate fallback AKID, emulating s2i_skey_id(..., "hash") */ X509_PUBKEY *pubkey = NULL; @@ -187,13 +171,15 @@ static AUTHORITY_KEYID *v2i_AUTHORITY_KEYID(X509V3_EXT_METHOD *method, ikeyid = ossl_x509_pubkey_hash(pubkey); X509_PUBKEY_free(pubkey); } - if (keyid == 2 && ikeyid == NULL) { + if ((keyid == 2 || issuer == 0) + && (ikeyid == NULL + || ASN1_STRING_length(ikeyid) <= 2) /* indicating "none" */) { ERR_raise(ERR_LIB_X509V3, X509V3_R_UNABLE_TO_GET_ISSUER_KEYID); goto err; } } - if (issuer == 2 || (issuer == 1 && !ss && ikeyid == NULL)) { + if (issuer == 2 || (issuer == 1 && ikeyid == NULL)) { isname = X509_NAME_dup(X509_get_issuer_name(issuer_cert)); serial = ASN1_INTEGER_dup(X509_get0_serialNumber(issuer_cert)); if (isname == NULL || serial == NULL) { @@ -204,12 +190,9 @@ static AUTHORITY_KEYID *v2i_AUTHORITY_KEYID(X509V3_EXT_METHOD *method, if (isname != NULL) { if ((gens = sk_GENERAL_NAME_new_null()) == NULL - || (gen = GENERAL_NAME_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } - if (!sk_GENERAL_NAME_push(gens, gen)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + || (gen = GENERAL_NAME_new()) == NULL + || !sk_GENERAL_NAME_push(gens, gen)) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } gen->type = GEN_DIRNAME; diff --git a/openssl/src/crypto/x509/v3_asid.c b/openssl/src/crypto/x509/v3_asid.c index e08e11d58..faa672610 100644 --- a/openssl/src/crypto/x509/v3_asid.c +++ b/openssl/src/crypto/x509/v3_asid.c @@ -1,5 +1,5 @@ /* - * Copyright 2006-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2006-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -169,11 +169,8 @@ int X509v3_asid_add_inherit(ASIdentifiers *asid, int which) if (*choice == NULL) { if ((*choice = ASIdentifierChoice_new()) == NULL) return 0; - if (((*choice)->u.inherit = ASN1_NULL_new()) == NULL) { - ASIdentifierChoice_free(*choice); - *choice = NULL; + if (((*choice)->u.inherit = ASN1_NULL_new()) == NULL) return 0; - } (*choice)->type = ASIdentifierChoice_inherit; } return (*choice)->type == ASIdentifierChoice_inherit; @@ -199,23 +196,18 @@ int X509v3_asid_add_id_or_range(ASIdentifiers *asid, default: return 0; } - if (*choice != NULL && (*choice)->type != ASIdentifierChoice_asIdsOrRanges) + if (*choice != NULL && (*choice)->type == ASIdentifierChoice_inherit) return 0; if (*choice == NULL) { if ((*choice = ASIdentifierChoice_new()) == NULL) return 0; (*choice)->u.asIdsOrRanges = sk_ASIdOrRange_new(ASIdOrRange_cmp); - if ((*choice)->u.asIdsOrRanges == NULL) { - ASIdentifierChoice_free(*choice); - *choice = NULL; + if ((*choice)->u.asIdsOrRanges == NULL) return 0; - } (*choice)->type = ASIdentifierChoice_asIdsOrRanges; } if ((aor = ASIdOrRange_new()) == NULL) return 0; - if (!sk_ASIdOrRange_reserve((*choice)->u.asIdsOrRanges, 1)) - goto err; if (max == NULL) { aor->type = ASIdOrRange_id; aor->u.id = min; @@ -228,8 +220,7 @@ int X509v3_asid_add_id_or_range(ASIdentifiers *asid, ASN1_INTEGER_free(aor->u.range->max); aor->u.range->max = max; } - /* Cannot fail due to the reservation above */ - if (!ossl_assert(sk_ASIdOrRange_push((*choice)->u.asIdsOrRanges, aor))) + if (!(sk_ASIdOrRange_push((*choice)->u.asIdsOrRanges, aor))) goto err; return 1; @@ -310,14 +301,14 @@ static int ASIdentifierChoice_is_canonical(ASIdentifierChoice *choice) if ((bn == NULL && (bn = BN_new()) == NULL) || ASN1_INTEGER_to_BN(a_max, bn) == NULL || !BN_add_word(bn, 1)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto done; } if ((a_max_plus_one = BN_to_ASN1_INTEGER(bn, orig = a_max_plus_one)) == NULL) { a_max_plus_one = orig; - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto done; } @@ -431,14 +422,14 @@ static int ASIdentifierChoice_canonize(ASIdentifierChoice *choice) if ((bn == NULL && (bn = BN_new()) == NULL) || ASN1_INTEGER_to_BN(a_max, bn) == NULL || !BN_add_word(bn, 1)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto done; } if ((a_max_plus_one = BN_to_ASN1_INTEGER(bn, orig = a_max_plus_one)) == NULL) { a_max_plus_one = orig; - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto done; } @@ -449,8 +440,10 @@ static int ASIdentifierChoice_canonize(ASIdentifierChoice *choice) ASRange *r; switch (a->type) { case ASIdOrRange_id: - if ((r = OPENSSL_malloc(sizeof(*r))) == NULL) + if ((r = OPENSSL_malloc(sizeof(*r))) == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto done; + } r->min = a_min; r->max = b_max; a->type = ASIdOrRange_range; @@ -524,7 +517,7 @@ static void *v2i_ASIdentifiers(const struct v3_ext_method *method, int i; if ((asid = ASIdentifiers_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } @@ -545,11 +538,6 @@ static void *v2i_ASIdentifiers(const struct v3_ext_method *method, goto err; } - if (val->value == NULL) { - ERR_raise(ERR_LIB_X509V3, X509V3_R_EXTENSION_VALUE_ERROR); - goto err; - } - /* * Handle inheritance. */ @@ -590,19 +578,21 @@ static void *v2i_ASIdentifiers(const struct v3_ext_method *method, */ if (!is_range) { if (!X509V3_get_value_int(val, &min)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } } else { char *s = OPENSSL_strdup(val->value); - if (s == NULL) + if (s == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; + } s[i1] = '\0'; min = s2i_ASN1_INTEGER(NULL, s); max = s2i_ASN1_INTEGER(NULL, s + i2); OPENSSL_free(s); if (min == NULL || max == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } if (ASN1_INTEGER_cmp(min, max) > 0) { @@ -611,7 +601,7 @@ static void *v2i_ASIdentifiers(const struct v3_ext_method *method, } } if (!X509v3_asid_add_id_or_range(asid, which, min, max)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } min = max = NULL; @@ -699,28 +689,15 @@ static int asid_contains(ASIdOrRanges *parent, ASIdOrRanges *child) */ int X509v3_asid_subset(ASIdentifiers *a, ASIdentifiers *b) { - int subset; - - if (a == NULL || a == b) - return 1; - - if (b == NULL) - return 0; - - if (X509v3_asid_inherits(a) || X509v3_asid_inherits(b)) - return 0; - - subset = a->asnum == NULL - || (b->asnum != NULL - && asid_contains(b->asnum->u.asIdsOrRanges, - a->asnum->u.asIdsOrRanges)); - if (!subset) - return 0; - - return a->rdi == NULL - || (b->rdi != NULL - && asid_contains(b->rdi->u.asIdsOrRanges, - a->rdi->u.asIdsOrRanges)); + return (a == NULL || + a == b || + (b != NULL && + !X509v3_asid_inherits(a) && + !X509v3_asid_inherits(b) && + asid_contains(b->asnum->u.asIdsOrRanges, + a->asnum->u.asIdsOrRanges) && + asid_contains(b->rdi->u.asIdsOrRanges, + a->rdi->u.asIdsOrRanges))); } /* diff --git a/openssl/src/crypto/x509/v3_bcons.c b/openssl/src/crypto/x509/v3_bcons.c index 17962ed43..6e7a165f2 100644 --- a/openssl/src/crypto/x509/v3_bcons.c +++ b/openssl/src/crypto/x509/v3_bcons.c @@ -61,7 +61,7 @@ static BASIC_CONSTRAINTS *v2i_BASIC_CONSTRAINTS(X509V3_EXT_METHOD *method, int i; if ((bcons = BASIC_CONSTRAINTS_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } for (i = 0; i < sk_CONF_VALUE_num(values); i++) { diff --git a/openssl/src/crypto/x509/v3_bitst.c b/openssl/src/crypto/x509/v3_bitst.c index d41c95b51..1a3198c70 100644 --- a/openssl/src/crypto/x509/v3_bitst.c +++ b/openssl/src/crypto/x509/v3_bitst.c @@ -38,10 +38,22 @@ static BIT_STRING_BITNAME key_usage_type_table[] = { {-1, NULL, NULL} }; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +static BIT_STRING_BITNAME dc_usage_type_table[] = { + {0, "Server Delegation", "serverDelegation"}, + {1, "Client Delegation", "clientDelegation"}, + {-1, NULL, NULL} +}; +#endif + const X509V3_EXT_METHOD ossl_v3_nscert = EXT_BITSTRING(NID_netscape_cert_type, ns_cert_type_table); const X509V3_EXT_METHOD ossl_v3_key_usage = EXT_BITSTRING(NID_key_usage, key_usage_type_table); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +const X509V3_EXT_METHOD ossl_v3_dc_usage = +EXT_BITSTRING(NID_delegation_usage, dc_usage_type_table); +#endif STACK_OF(CONF_VALUE) *i2v_ASN1_BIT_STRING(X509V3_EXT_METHOD *method, ASN1_BIT_STRING *bits, @@ -64,7 +76,7 @@ ASN1_BIT_STRING *v2i_ASN1_BIT_STRING(X509V3_EXT_METHOD *method, int i; BIT_STRING_BITNAME *bnam; if ((bs = ASN1_BIT_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } for (i = 0; i < sk_CONF_VALUE_num(nval); i++) { @@ -73,7 +85,7 @@ ASN1_BIT_STRING *v2i_ASN1_BIT_STRING(X509V3_EXT_METHOD *method, if (strcmp(bnam->sname, val->name) == 0 || strcmp(bnam->lname, val->name) == 0) { if (!ASN1_BIT_STRING_set_bit(bs, bnam->bitnum, 1)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); ASN1_BIT_STRING_free(bs); return NULL; } diff --git a/openssl/src/crypto/x509/v3_conf.c b/openssl/src/crypto/x509/v3_conf.c index c575a4345..1c11d671b 100644 --- a/openssl/src/crypto/x509/v3_conf.c +++ b/openssl/src/crypto/x509/v3_conf.c @@ -148,41 +148,34 @@ static X509_EXTENSION *do_ext_i2d(const X509V3_EXT_METHOD *method, ext_der = NULL; ext_len = ASN1_item_i2d(ext_struc, &ext_der, ASN1_ITEM_ptr(method->it)); - if (ext_len < 0) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if (ext_len < 0) + goto merr; } else { unsigned char *p; ext_len = method->i2d(ext_struc, NULL); - if (ext_len <= 0) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if (ext_len <= 0) + goto merr; if ((ext_der = OPENSSL_malloc(ext_len)) == NULL) - goto err; + goto merr; p = ext_der; method->i2d(ext_struc, &p); } - if ((ext_oct = ASN1_OCTET_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((ext_oct = ASN1_OCTET_STRING_new()) == NULL) + goto merr; ext_oct->data = ext_der; ext_der = NULL; ext_oct->length = ext_len; ext = X509_EXTENSION_create_by_NID(NULL, ext_nid, crit, ext_oct); - if (!ext) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); - goto err; - } + if (!ext) + goto merr; ASN1_OCTET_STRING_free(ext_oct); return ext; - err: + merr: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); OPENSSL_free(ext_der); ASN1_OCTET_STRING_free(ext_oct); return NULL; @@ -207,8 +200,9 @@ static int v3_check_critical(const char **value) { const char *p = *value; - if (!CHECK_AND_SKIP_PREFIX(p, "critical,")) + if ((strlen(p) < 9) || strncmp(p, "critical,", 9)) return 0; + p += 9; while (ossl_isspace(*p)) p++; *value = p; @@ -221,9 +215,11 @@ static int v3_check_generic(const char **value) int gen_type = 0; const char *p = *value; - if (CHECK_AND_SKIP_PREFIX(p, "DER:")) { + if ((strlen(p) >= 4) && strncmp(p, "DER:", 4) == 0) { + p += 4; gen_type = 1; - } else if (CHECK_AND_SKIP_PREFIX(p, "ASN1:")) { + } else if ((strlen(p) >= 5) && strncmp(p, "ASN1:", 5) == 0) { + p += 5; gen_type = 2; } else return 0; @@ -263,7 +259,7 @@ static X509_EXTENSION *v3_generic_extension(const char *ext, const char *value, } if ((oct = ASN1_OCTET_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } @@ -315,27 +311,13 @@ int X509V3_EXT_add_nconf_sk(CONF *conf, X509V3_CTX *ctx, const char *section, { X509_EXTENSION *ext; STACK_OF(CONF_VALUE) *nval; - const CONF_VALUE *val; - int i, akid = -1, skid = -1; + CONF_VALUE *val; + int i; if ((nval = NCONF_get_section(conf, section)) == NULL) return 0; for (i = 0; i < sk_CONF_VALUE_num(nval); i++) { val = sk_CONF_VALUE_value(nval, i); - if (strcmp(val->name, "authorityKeyIdentifier") == 0) - akid = i; - else if (strcmp(val->name, "subjectKeyIdentifier") == 0) - skid = i; - } - for (i = 0; i < sk_CONF_VALUE_num(nval); i++) { - val = sk_CONF_VALUE_value(nval, i); - if (skid > akid && akid >= 0) { - /* make sure SKID is handled before AKID */ - if (i == akid) - val = sk_CONF_VALUE_value(nval, skid); - else if (i == skid) - val = sk_CONF_VALUE_value(nval, akid); - } if ((ext = X509V3_EXT_nconf_int(conf, ctx, val->section, val->name, val->value)) == NULL) return 0; diff --git a/openssl/src/crypto/x509/v3_cpols.c b/openssl/src/crypto/x509/v3_cpols.c index ae602ea2c..5353a6916 100644 --- a/openssl/src/crypto/x509/v3_cpols.c +++ b/openssl/src/crypto/x509/v3_cpols.c @@ -105,7 +105,7 @@ static STACK_OF(POLICYINFO) *r2i_certpol(X509V3_EXT_METHOD *method, pols = sk_POLICYINFO_new_reserve(NULL, num); if (pols == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } @@ -144,14 +144,14 @@ static STACK_OF(POLICYINFO) *r2i_certpol(X509V3_EXT_METHOD *method, pol = POLICYINFO_new(); if (pol == NULL) { ASN1_OBJECT_free(pobj); - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } pol->policyid = pobj; } if (!sk_POLICYINFO_push(pols, pol)) { POLICYINFO_free(pol); - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } } @@ -171,10 +171,8 @@ static POLICYINFO *policy_section(X509V3_CTX *ctx, POLICYINFO *pol; POLICYQUALINFO *qual; - if ((pol = POLICYINFO_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((pol = POLICYINFO_new()) == NULL) + goto merr; for (i = 0; i < sk_CONF_VALUE_num(polstrs); i++) { cnf = sk_CONF_VALUE_value(polstrs, i); if (strcmp(cnf->name, "policyIdentifier") == 0) { @@ -190,27 +188,19 @@ static POLICYINFO *policy_section(X509V3_CTX *ctx, } else if (!ossl_v3_name_cmp(cnf->name, "CPS")) { if (pol->qualifiers == NULL) pol->qualifiers = sk_POLICYQUALINFO_new_null(); - if ((qual = POLICYQUALINFO_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } - if (!sk_POLICYQUALINFO_push(pol->qualifiers, qual)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); - goto err; - } + if ((qual = POLICYQUALINFO_new()) == NULL) + goto merr; + if (!sk_POLICYQUALINFO_push(pol->qualifiers, qual)) + goto merr; if ((qual->pqualid = OBJ_nid2obj(NID_id_qt_cps)) == NULL) { ERR_raise(ERR_LIB_X509V3, ERR_R_INTERNAL_ERROR); goto err; } - if ((qual->d.cpsuri = ASN1_IA5STRING_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((qual->d.cpsuri = ASN1_IA5STRING_new()) == NULL) + goto merr; if (!ASN1_STRING_set(qual->d.cpsuri, cnf->value, - strlen(cnf->value))) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + strlen(cnf->value))) + goto merr; } else if (!ossl_v3_name_cmp(cnf->name, "userNotice")) { STACK_OF(CONF_VALUE) *unot; if (*cnf->value != '@') { @@ -231,10 +221,8 @@ static POLICYINFO *policy_section(X509V3_CTX *ctx, goto err; if (pol->qualifiers == NULL) pol->qualifiers = sk_POLICYQUALINFO_new_null(); - if (!sk_POLICYQUALINFO_push(pol->qualifiers, qual)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); - goto err; - } + if (!sk_POLICYQUALINFO_push(pol->qualifiers, qual)) + goto merr; } else { ERR_raise(ERR_LIB_X509V3, X509V3_R_INVALID_OPTION); X509V3_conf_err(cnf); @@ -248,6 +236,9 @@ static POLICYINFO *policy_section(X509V3_CTX *ctx, return pol; + merr: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); + err: POLICYINFO_free(pol); return NULL; @@ -270,17 +261,17 @@ static int displaytext_str2tag(const char *tagstr, unsigned int *tag_len) if (len == -1) return V_ASN1_VISIBLESTRING; *tag_len = len; - if (len == sizeof("UTF8") - 1 && HAS_PREFIX(tagstr, "UTF8")) + if (len == sizeof("UTF8") - 1 && strncmp(tagstr, "UTF8", len) == 0) return V_ASN1_UTF8STRING; - if (len == sizeof("UTF8String") - 1 && HAS_PREFIX(tagstr, "UTF8String")) + if (len == sizeof("UTF8String") - 1 && strncmp(tagstr, "UTF8String", len) == 0) return V_ASN1_UTF8STRING; - if (len == sizeof("BMP") - 1 && HAS_PREFIX(tagstr, "BMP")) + if (len == sizeof("BMP") - 1 && strncmp(tagstr, "BMP", len) == 0) return V_ASN1_BMPSTRING; - if (len == sizeof("BMPSTRING") - 1 && HAS_PREFIX(tagstr, "BMPSTRING")) + if (len == sizeof("BMPSTRING") - 1 && strncmp(tagstr, "BMPSTRING", len) == 0) return V_ASN1_BMPSTRING; - if (len == sizeof("VISIBLE") - 1 && HAS_PREFIX(tagstr, "VISIBLE")) + if (len == sizeof("VISIBLE") - 1 && strncmp(tagstr, "VISIBLE", len) == 0) return V_ASN1_VISIBLESTRING; - if (len == sizeof("VISIBLESTRING") - 1 && HAS_PREFIX(tagstr, "VISIBLESTRING")) + if (len == sizeof("VISIBLESTRING") - 1 && strncmp(tagstr, "VISIBLESTRING", len) == 0) return V_ASN1_VISIBLESTRING; *tag_len = 0; return V_ASN1_VISIBLESTRING; @@ -296,18 +287,14 @@ static POLICYQUALINFO *notice_section(X509V3_CTX *ctx, POLICYQUALINFO *qual; char *value = NULL; - if ((qual = POLICYQUALINFO_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((qual = POLICYQUALINFO_new()) == NULL) + goto merr; if ((qual->pqualid = OBJ_nid2obj(NID_id_qt_unotice)) == NULL) { ERR_raise(ERR_LIB_X509V3, ERR_R_INTERNAL_ERROR); goto err; } - if ((not = USERNOTICE_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((not = USERNOTICE_new()) == NULL) + goto merr; qual->d.usernotice = not; for (i = 0; i < sk_CONF_VALUE_num(unot); i++) { cnf = sk_CONF_VALUE_value(unot, i); @@ -315,25 +302,19 @@ static POLICYQUALINFO *notice_section(X509V3_CTX *ctx, value = cnf->value; if (strcmp(cnf->name, "explicitText") == 0) { tag = displaytext_str2tag(value, &tag_len); - if ((not->exptext = ASN1_STRING_type_new(tag)) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((not->exptext = ASN1_STRING_type_new(tag)) == NULL) + goto merr; if (tag_len != 0) value += tag_len + 1; len = strlen(value); - if (!ASN1_STRING_set(not->exptext, value, len)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if (!ASN1_STRING_set(not->exptext, value, len)) + goto merr; } else if (strcmp(cnf->name, "organization") == 0) { NOTICEREF *nref; if (!not->noticeref) { - if ((nref = NOTICEREF_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((nref = NOTICEREF_new()) == NULL) + goto merr; not->noticeref = nref; } else nref = not->noticeref; @@ -342,19 +323,15 @@ static POLICYQUALINFO *notice_section(X509V3_CTX *ctx, else nref->organization->type = V_ASN1_VISIBLESTRING; if (!ASN1_STRING_set(nref->organization, cnf->value, - strlen(cnf->value))) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + strlen(cnf->value))) + goto merr; } else if (strcmp(cnf->name, "noticeNumbers") == 0) { NOTICEREF *nref; STACK_OF(CONF_VALUE) *nos; if (!not->noticeref) { - if ((nref = NOTICEREF_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((nref = NOTICEREF_new()) == NULL) + goto merr; not->noticeref = nref; } else nref = not->noticeref; @@ -384,6 +361,9 @@ static POLICYQUALINFO *notice_section(X509V3_CTX *ctx, return qual; + merr: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); + err: POLICYQUALINFO_free(qual); return NULL; @@ -400,15 +380,19 @@ static int nref_nos(STACK_OF(ASN1_INTEGER) *nnums, STACK_OF(CONF_VALUE) *nos) cnf = sk_CONF_VALUE_value(nos, i); if ((aint = s2i_ASN1_INTEGER(NULL, cnf->name)) == NULL) { ERR_raise(ERR_LIB_X509V3, X509V3_R_INVALID_NUMBER); - return 0; - } - if (!sk_ASN1_INTEGER_push(nnums, aint)) { - ASN1_INTEGER_free(aint); - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); - return 0; + goto err; } + if (!sk_ASN1_INTEGER_push(nnums, aint)) + goto merr; } return 1; + + merr: + ASN1_INTEGER_free(aint); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); + + err: + return 0; } static int i2r_certpol(X509V3_EXT_METHOD *method, STACK_OF(POLICYINFO) *pol, diff --git a/openssl/src/crypto/x509/v3_crld.c b/openssl/src/crypto/x509/v3_crld.c index 8f560e171..0289df4de 100644 --- a/openssl/src/crypto/x509/v3_crld.c +++ b/openssl/src/crypto/x509/v3_crld.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -70,12 +70,7 @@ static int set_dist_point_name(DIST_POINT_NAME **pdp, X509V3_CTX *ctx, STACK_OF(GENERAL_NAME) *fnm = NULL; STACK_OF(X509_NAME_ENTRY) *rnm = NULL; - if (cnf->value == NULL) { - ERR_raise(ERR_LIB_X509V3, X509V3_R_MISSING_VALUE); - goto err; - } - - if (HAS_PREFIX(cnf->name, "fullname")) { + if (strncmp(cnf->name, "fullname", 9) == 0) { fnm = gnames_from_sectname(ctx, cnf->value); if (!fnm) goto err; @@ -249,10 +244,8 @@ static void *v2i_crld(const X509V3_EXT_METHOD *method, int i; crld = sk_DIST_POINT_new_reserve(NULL, num); - if (crld == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); - goto err; - } + if (crld == NULL) + goto merr; for (i = 0; i < num; i++) { DIST_POINT *point; @@ -270,24 +263,16 @@ static void *v2i_crld(const X509V3_EXT_METHOD *method, } else { if ((gen = v2i_GENERAL_NAME(method, ctx, cnf)) == NULL) goto err; - if ((gens = GENERAL_NAMES_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } - if (!sk_GENERAL_NAME_push(gens, gen)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); - goto err; - } + if ((gens = GENERAL_NAMES_new()) == NULL) + goto merr; + if (!sk_GENERAL_NAME_push(gens, gen)) + goto merr; gen = NULL; - if ((point = DIST_POINT_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((point = DIST_POINT_new()) == NULL) + goto merr; sk_DIST_POINT_push(crld, point); /* no failure as it was reserved */ - if ((point->distpoint = DIST_POINT_NAME_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if ((point->distpoint = DIST_POINT_NAME_new()) == NULL) + goto merr; point->distpoint->name.fullname = gens; point->distpoint->type = 0; gens = NULL; @@ -295,6 +280,8 @@ static void *v2i_crld(const X509V3_EXT_METHOD *method, } return crld; + merr: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); err: GENERAL_NAME_free(gen); GENERAL_NAMES_free(gens); @@ -377,10 +364,8 @@ static void *v2i_idp(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, char *name, *val; int i, ret; idp = ISSUING_DIST_POINT_new(); - if (idp == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if (idp == NULL) + goto merr; for (i = 0; i < sk_CONF_VALUE_num(nval); i++) { cnf = sk_CONF_VALUE_value(nval, i); name = cnf->name; @@ -413,6 +398,8 @@ static void *v2i_idp(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, } return idp; + merr: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); err: ISSUING_DIST_POINT_free(idp); return NULL; diff --git a/openssl/src/crypto/x509/v3_extku.c b/openssl/src/crypto/x509/v3_extku.c index 22c951e25..4f2a86bdc 100644 --- a/openssl/src/crypto/x509/v3_extku.c +++ b/openssl/src/crypto/x509/v3_extku.c @@ -79,7 +79,7 @@ static void *v2i_EXTENDED_KEY_USAGE(const X509V3_EXT_METHOD *method, extku = sk_ASN1_OBJECT_new_reserve(NULL, num); if (extku == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); sk_ASN1_OBJECT_free(extku); return NULL; } diff --git a/openssl/src/crypto/x509/v3_genn.c b/openssl/src/crypto/x509/v3_genn.c index 1f67bf2f6..1741c2d2f 100644 --- a/openssl/src/crypto/x509/v3_genn.c +++ b/openssl/src/crypto/x509/v3_genn.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/x509/v3_group_ac.c b/openssl/src/crypto/x509/v3_group_ac.c deleted file mode 100644 index 9a2b69dec..000000000 --- a/openssl/src/crypto/x509/v3_group_ac.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/cryptlib.h" -#include -#include -#include -#include "ext_dat.h" - -static int i2r_GROUP_AC(X509V3_EXT_METHOD *method, - void *su, BIO *out, - int indent) -{ - return 1; -} - -static void *r2i_GROUP_AC(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *value) -{ - return ASN1_NULL_new(); -} - -static char *i2s_GROUP_AC(const X509V3_EXT_METHOD *method, void *val) -{ - return OPENSSL_strdup("NULL"); -} - -static void *s2i_GROUP_AC(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, const char *str) -{ - return ASN1_NULL_new(); -} - -/* - * The groupAC X.509v3 extension is defined in ITU Recommendation X.509 - * (2019), Section 17.1.2.6. See: https://www.itu.int/rec/T-REC-X.509-201910-I/en. - */ -const X509V3_EXT_METHOD ossl_v3_group_ac = { - NID_group_ac, 0, ASN1_ITEM_ref(ASN1_NULL), - 0, 0, 0, 0, - (X509V3_EXT_I2S)i2s_GROUP_AC, - (X509V3_EXT_S2I)s2i_GROUP_AC, - 0, 0, - (X509V3_EXT_I2R)i2r_GROUP_AC, - (X509V3_EXT_R2I)r2i_GROUP_AC, - NULL -}; diff --git a/openssl/src/crypto/x509/v3_ia5.c b/openssl/src/crypto/x509/v3_ia5.c index 7b7993587..6722b6c01 100644 --- a/openssl/src/crypto/x509/v3_ia5.c +++ b/openssl/src/crypto/x509/v3_ia5.c @@ -31,8 +31,10 @@ char *i2s_ASN1_IA5STRING(X509V3_EXT_METHOD *method, ASN1_IA5STRING *ia5) if (ia5 == NULL || ia5->length <= 0) return NULL; - if ((tmp = OPENSSL_malloc(ia5->length + 1)) == NULL) + if ((tmp = OPENSSL_malloc(ia5->length + 1)) == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; + } memcpy(tmp, ia5->data, ia5->length); tmp[ia5->length] = 0; return tmp; @@ -46,10 +48,8 @@ ASN1_IA5STRING *s2i_ASN1_IA5STRING(X509V3_EXT_METHOD *method, ERR_raise(ERR_LIB_X509V3, X509V3_R_INVALID_NULL_ARGUMENT); return NULL; } - if ((ia5 = ASN1_IA5STRING_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - return NULL; - } + if ((ia5 = ASN1_IA5STRING_new()) == NULL) + goto err; if (!ASN1_STRING_set((ASN1_STRING *)ia5, str, strlen(str))) { ASN1_IA5STRING_free(ia5); return NULL; @@ -58,4 +58,7 @@ ASN1_IA5STRING *s2i_ASN1_IA5STRING(X509V3_EXT_METHOD *method, ebcdic2ascii(ia5->data, ia5->data, ia5->length); #endif /* CHARSET_EBCDIC */ return ia5; + err: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); + return NULL; } diff --git a/openssl/src/crypto/x509/v3_ind_iss.c b/openssl/src/crypto/x509/v3_ind_iss.c deleted file mode 100644 index ff41ba7be..000000000 --- a/openssl/src/crypto/x509/v3_ind_iss.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/cryptlib.h" -#include -#include -#include -#include "ext_dat.h" - -static int i2r_INDIRECT_ISSUER(X509V3_EXT_METHOD *method, - void *su, BIO *out, - int indent) -{ - return 1; -} - -static void *r2i_INDIRECT_ISSUER(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *value) -{ - return ASN1_NULL_new(); -} - -static char *i2s_INDIRECT_ISSUER(const X509V3_EXT_METHOD *method, void *val) -{ - return OPENSSL_strdup("NULL"); -} - -static void *s2i_INDIRECT_ISSUER(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, const char *str) -{ - return ASN1_NULL_new(); -} - -/* - * The indirectIssuer X.509v3 extension is defined in ITU Recommendation X.509 - * (2019), Section 17.5.2.5. See: https://www.itu.int/rec/T-REC-X.509-201910-I/en. - */ -const X509V3_EXT_METHOD ossl_v3_indirect_issuer = { - NID_indirect_issuer, 0, ASN1_ITEM_ref(ASN1_NULL), - 0, 0, 0, 0, - (X509V3_EXT_I2S)i2s_INDIRECT_ISSUER, - (X509V3_EXT_S2I)s2i_INDIRECT_ISSUER, - 0, 0, - (X509V3_EXT_I2R)i2r_INDIRECT_ISSUER, - (X509V3_EXT_R2I)r2i_INDIRECT_ISSUER, - NULL -}; diff --git a/openssl/src/crypto/x509/v3_info.c b/openssl/src/crypto/x509/v3_info.c index 7e4d9313d..5f21ce11e 100644 --- a/openssl/src/crypto/x509/v3_info.c +++ b/openssl/src/crypto/x509/v3_info.c @@ -73,10 +73,8 @@ static STACK_OF(CONF_VALUE) *i2v_AUTHORITY_INFO_ACCESS( desc = sk_ACCESS_DESCRIPTION_value(ainfo, i); tmp = i2v_GENERAL_NAME(method, desc->location, tret); - if (tmp == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + if (tmp == NULL) goto err; - } tret = tmp; vtmp = sk_CONF_VALUE_value(tret, i); i2t_ASN1_OBJECT(objtmp, sizeof(objtmp), desc->method); @@ -93,6 +91,7 @@ static STACK_OF(CONF_VALUE) *i2v_AUTHORITY_INFO_ACCESS( return tret; err: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); if (ret == NULL && tret != NULL) sk_CONF_VALUE_pop_free(tret, X509V3_conf_free); return NULL; @@ -112,13 +111,13 @@ static AUTHORITY_INFO_ACCESS *v2i_AUTHORITY_INFO_ACCESS(X509V3_EXT_METHOD char *objtmp, *ptmp; if ((ainfo = sk_ACCESS_DESCRIPTION_new_reserve(NULL, num)) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } for (i = 0; i < num; i++) { cnf = sk_CONF_VALUE_value(nval, i); if ((acc = ACCESS_DESCRIPTION_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } sk_ACCESS_DESCRIPTION_push(ainfo, acc); /* Cannot fail due to reserve */ @@ -131,8 +130,10 @@ static AUTHORITY_INFO_ACCESS *v2i_AUTHORITY_INFO_ACCESS(X509V3_EXT_METHOD ctmp.value = cnf->value; if (!v2i_GENERAL_NAME_ex(acc->location, method, ctx, &ctmp, 0)) goto err; - if ((objtmp = OPENSSL_strndup(cnf->name, ptmp - cnf->name)) == NULL) + if ((objtmp = OPENSSL_strndup(cnf->name, ptmp - cnf->name)) == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; + } acc->method = OBJ_txt2obj(objtmp, 0); if (!acc->method) { ERR_raise_data(ERR_LIB_X509V3, X509V3_R_BAD_OBJECT, diff --git a/openssl/src/crypto/x509/v3_ist.c b/openssl/src/crypto/x509/v3_ist.c deleted file mode 100644 index b7ce4bb91..000000000 --- a/openssl/src/crypto/x509/v3_ist.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/cryptlib.h" -#include -#include -#include -#include -#include "ext_dat.h" - -/* - * Issuer Sign Tool (1.2.643.100.112) The name of the tool used to signs the subject (ASN1_SEQUENCE) - * This extension is required to obtain the status of a qualified certificate at Russian Federation. - * RFC-style description is available here: https://tools.ietf.org/html/draft-deremin-rfc4491-bis-04#section-5 - * Russian Federal Law 63 "Digital Sign" is available here: http://www.consultant.ru/document/cons_doc_LAW_112701/ - */ - -ASN1_SEQUENCE(ISSUER_SIGN_TOOL) = { - ASN1_SIMPLE(ISSUER_SIGN_TOOL, signTool, ASN1_UTF8STRING), - ASN1_SIMPLE(ISSUER_SIGN_TOOL, cATool, ASN1_UTF8STRING), - ASN1_SIMPLE(ISSUER_SIGN_TOOL, signToolCert, ASN1_UTF8STRING), - ASN1_SIMPLE(ISSUER_SIGN_TOOL, cAToolCert, ASN1_UTF8STRING) -} ASN1_SEQUENCE_END(ISSUER_SIGN_TOOL) - -IMPLEMENT_ASN1_FUNCTIONS(ISSUER_SIGN_TOOL) - - -static ISSUER_SIGN_TOOL *v2i_issuer_sign_tool(X509V3_EXT_METHOD *method, X509V3_CTX *ctx, - STACK_OF(CONF_VALUE) *nval) -{ - ISSUER_SIGN_TOOL *ist = ISSUER_SIGN_TOOL_new(); - int i; - - if (ist == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - return NULL; - } - for (i = 0; i < sk_CONF_VALUE_num(nval); ++i) { - CONF_VALUE *cnf = sk_CONF_VALUE_value(nval, i); - - if (cnf == NULL) { - continue; - } - if (strcmp(cnf->name, "signTool") == 0) { - ist->signTool = ASN1_UTF8STRING_new(); - if (ist->signTool == NULL - || cnf->value == NULL - || !ASN1_STRING_set(ist->signTool, cnf->value, strlen(cnf->value))) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } - } else if (strcmp(cnf->name, "cATool") == 0) { - ist->cATool = ASN1_UTF8STRING_new(); - if (ist->cATool == NULL - || cnf->value == NULL - || !ASN1_STRING_set(ist->cATool, cnf->value, strlen(cnf->value))) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } - } else if (strcmp(cnf->name, "signToolCert") == 0) { - ist->signToolCert = ASN1_UTF8STRING_new(); - if (ist->signToolCert == NULL - || cnf->value == NULL - || !ASN1_STRING_set(ist->signToolCert, cnf->value, strlen(cnf->value))) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } - } else if (strcmp(cnf->name, "cAToolCert") == 0) { - ist->cAToolCert = ASN1_UTF8STRING_new(); - if (ist->cAToolCert == NULL - || cnf->value == NULL - || !ASN1_STRING_set(ist->cAToolCert, cnf->value, strlen(cnf->value))) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } - } else { - ERR_raise(ERR_LIB_X509V3, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - } - return ist; - -err: - ISSUER_SIGN_TOOL_free(ist); - return NULL; -} - -static int i2r_issuer_sign_tool(X509V3_EXT_METHOD *method, - ISSUER_SIGN_TOOL *ist, BIO *out, - int indent) -{ - int new_line = 0; - - if (ist == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - if (ist->signTool != NULL) { - BIO_printf(out, "%*ssignTool : ", indent, ""); - BIO_write(out, ist->signTool->data, ist->signTool->length); - new_line = 1; - } - if (ist->cATool != NULL) { - if (new_line == 1) { - BIO_write(out, "\n", 1); - } - BIO_printf(out, "%*scATool : ", indent, ""); - BIO_write(out, ist->cATool->data, ist->cATool->length); - new_line = 1; - } - if (ist->signToolCert != NULL) { - if (new_line == 1) { - BIO_write(out, "\n", 1); - } - BIO_printf(out, "%*ssignToolCert: ", indent, ""); - BIO_write(out, ist->signToolCert->data, ist->signToolCert->length); - new_line = 1; - } - if (ist->cAToolCert != NULL) { - if (new_line == 1) { - BIO_write(out, "\n", 1); - } - BIO_printf(out, "%*scAToolCert : ", indent, ""); - BIO_write(out, ist->cAToolCert->data, ist->cAToolCert->length); - new_line = 1; - } - return 1; -} - -const X509V3_EXT_METHOD ossl_v3_issuer_sign_tool = { - NID_issuerSignTool, /* nid */ - X509V3_EXT_MULTILINE, /* flags */ - ASN1_ITEM_ref(ISSUER_SIGN_TOOL), /* template */ - 0, 0, 0, 0, /* old functions, ignored */ - 0, /* i2s */ - 0, /* s2i */ - 0, /* i2v */ - (X509V3_EXT_V2I)v2i_issuer_sign_tool, /* v2i */ - (X509V3_EXT_I2R)i2r_issuer_sign_tool, /* i2r */ - 0, /* r2i */ - NULL /* extension-specific data */ -}; diff --git a/openssl/src/crypto/x509/v3_lib.c b/openssl/src/crypto/x509/v3_lib.c index 077b22c86..42b6ff152 100644 --- a/openssl/src/crypto/x509/v3_lib.c +++ b/openssl/src/crypto/x509/v3_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,11 +26,11 @@ int X509V3_EXT_add(X509V3_EXT_METHOD *ext) { if (ext_list == NULL && (ext_list = sk_X509V3_EXT_METHOD_new(ext_cmp)) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return 0; } if (!sk_X509V3_EXT_METHOD_push(ext_list, ext)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return 0; } return 1; @@ -63,10 +63,7 @@ const X509V3_EXT_METHOD *X509V3_EXT_get_nid(int nid) return *ret; if (!ext_list) return NULL; - /* Ideally, this would be done under a lock */ - sk_X509V3_EXT_METHOD_sort(ext_list); idx = sk_X509V3_EXT_METHOD_find(ext_list, &tmp); - /* A failure to locate the item is handled by the value method */ return sk_X509V3_EXT_METHOD_value(ext_list, idx); } @@ -95,8 +92,10 @@ int X509V3_EXT_add_alias(int nid_to, int nid_from) ERR_raise(ERR_LIB_X509V3, X509V3_R_EXTENSION_NOT_FOUND); return 0; } - if ((tmpext = OPENSSL_malloc(sizeof(*tmpext))) == NULL) + if ((tmpext = OPENSSL_malloc(sizeof(*tmpext))) == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return 0; + } *tmpext = *ext; tmpext->ext_nid = nid_to; tmpext->ext_flags |= X509V3_EXT_DYNAMIC; @@ -243,10 +242,8 @@ int X509V3_add1_i2d(STACK_OF(X509_EXTENSION) **x, int nid, void *value, } /* If delete, just delete it */ if (ext_op == X509V3_ADD_DELETE) { - extmp = sk_X509_EXTENSION_delete(*x, extidx); - if (extmp == NULL) + if (!sk_X509_EXTENSION_delete(*x, extidx)) return -1; - X509_EXTENSION_free(extmp); return 1; } } else { @@ -292,7 +289,7 @@ int X509V3_add1_i2d(STACK_OF(X509_EXTENSION) **x, int nid, void *value, return 1; m_fail: - /* ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); */ + /* ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); */ if (ret != *x) sk_X509_EXTENSION_free(ret); X509_EXTENSION_free(ext); diff --git a/openssl/src/crypto/x509/v3_ncons.c b/openssl/src/crypto/x509/v3_ncons.c index a6817b9e1..510159851 100644 --- a/openssl/src/crypto/x509/v3_ncons.c +++ b/openssl/src/crypto/x509/v3_ncons.c @@ -1,5 +1,5 @@ /* - * Copyright 2003-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2003-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,7 +9,6 @@ #include "internal/cryptlib.h" #include "internal/numbers.h" -#include "internal/safe_math.h" #include #include "crypto/asn1.h" #include @@ -21,8 +20,6 @@ #include "crypto/punycode.h" #include "ext_dat.h" -OSSL_SAFE_MATH_SIGNED(int, int) - static void *v2i_NAME_CONSTRAINTS(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, STACK_OF(CONF_VALUE) *nval); @@ -135,16 +132,14 @@ static void *v2i_NAME_CONSTRAINTS(const X509V3_EXT_METHOD *method, GENERAL_SUBTREE *sub = NULL; ncons = NAME_CONSTRAINTS_new(); - if (ncons == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - goto err; - } + if (ncons == NULL) + goto memerr; for (i = 0; i < sk_CONF_VALUE_num(nval); i++) { val = sk_CONF_VALUE_value(nval, i); - if (HAS_PREFIX(val->name, "permitted") && val->name[9]) { + if (strncmp(val->name, "permitted", 9) == 0 && val->name[9]) { ptree = &ncons->permittedSubtrees; tval.name = val->name + 10; - } else if (HAS_PREFIX(val->name, "excluded") && val->name[8]) { + } else if (strncmp(val->name, "excluded", 8) == 0 && val->name[8]) { ptree = &ncons->excludedSubtrees; tval.name = val->name + 9; } else { @@ -153,25 +148,21 @@ static void *v2i_NAME_CONSTRAINTS(const X509V3_EXT_METHOD *method, } tval.value = val->value; sub = GENERAL_SUBTREE_new(); - if (sub == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + if (sub == NULL) + goto memerr; + if (!v2i_GENERAL_NAME_ex(sub->base, method, ctx, &tval, 1)) goto err; - } - if (!v2i_GENERAL_NAME_ex(sub->base, method, ctx, &tval, 1)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); - goto err; - } if (*ptree == NULL) *ptree = sk_GENERAL_SUBTREE_new_null(); - if (*ptree == NULL || !sk_GENERAL_SUBTREE_push(*ptree, sub)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); - goto err; - } + if (*ptree == NULL || !sk_GENERAL_SUBTREE_push(*ptree, sub)) + goto memerr; sub = NULL; } return ncons; + memerr: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); err: NAME_CONSTRAINTS_free(ncons); GENERAL_SUBTREE_free(sub); @@ -232,16 +223,16 @@ static int print_nc_ipadd(BIO *bp, ASN1_OCTET_STRING *ip) static int add_lengths(int *out, int a, int b) { - int err = 0; - /* sk_FOO_num(NULL) returns -1 but is effectively 0 when iterating. */ if (a < 0) a = 0; if (b < 0) b = 0; - *out = safe_add_int(a, b, &err); - return !err; + if (a > INT_MAX - b) + return 0; + *out = a + b; + return 1; } /*- @@ -441,7 +432,7 @@ int NAME_CONSTRAINTS_check_CN(X509 *x, NAME_CONSTRAINTS *nc) ne = X509_NAME_get_entry(nm, i); cn = X509_NAME_ENTRY_get_data(ne); - /* Only process attributes that look like hostnames */ + /* Only process attributes that look like host names */ if ((r = cn2dnsid(cn, &idval, &idlen)) != X509_V_OK) return r; if (idlen == 0) @@ -649,7 +640,7 @@ static int nc_email_eai(ASN1_TYPE *emltype, ASN1_IA5STRING *base) const char *emlptr; const char *emlat; char ulabel[256]; - size_t size = sizeof(ulabel); + size_t size = sizeof(ulabel) - 1; int ret = X509_V_OK; size_t emlhostlen; @@ -676,16 +667,18 @@ static int nc_email_eai(ASN1_TYPE *emltype, ASN1_IA5STRING *base) goto end; } + memset(ulabel, 0, sizeof(ulabel)); /* Special case: initial '.' is RHS match */ if (*baseptr == '.') { ulabel[0] = '.'; - if (ossl_a2ulabel(baseptr, ulabel + 1, size - 1) <= 0) { + size -= 1; + if (ossl_a2ulabel(baseptr, ulabel + 1, &size) <= 0) { ret = X509_V_ERR_UNSPECIFIED; goto end; } if ((size_t)eml->length > strlen(ulabel)) { - emlptr += eml->length - strlen(ulabel); + emlptr += eml->length - (strlen(ulabel)); /* X509_V_OK */ if (ia5ncasecmp(ulabel, emlptr, strlen(ulabel)) == 0) goto end; @@ -694,7 +687,7 @@ static int nc_email_eai(ASN1_TYPE *emltype, ASN1_IA5STRING *base) goto end; } - if (ossl_a2ulabel(baseptr, ulabel, size) <= 0) { + if (ossl_a2ulabel(baseptr, ulabel, &size) <= 0) { ret = X509_V_ERR_UNSPECIFIED; goto end; } diff --git a/openssl/src/crypto/x509/v3_no_ass.c b/openssl/src/crypto/x509/v3_no_ass.c deleted file mode 100644 index 873e20a75..000000000 --- a/openssl/src/crypto/x509/v3_no_ass.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/cryptlib.h" -#include -#include -#include -#include "ext_dat.h" - -static int i2r_NO_ASSERTION(X509V3_EXT_METHOD *method, - void *su, BIO *out, - int indent) -{ - return 1; -} - -static void *r2i_NO_ASSERTION(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *value) -{ - return ASN1_NULL_new(); -} - -static char *i2s_NO_ASSERTION(const X509V3_EXT_METHOD *method, void *val) -{ - return OPENSSL_strdup("NULL"); -} - -static void *s2i_NO_ASSERTION(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, const char *str) -{ - return ASN1_NULL_new(); -} - -/* - * The noAssertion X.509v3 extension is defined in ITU Recommendation X.509 - * (2019), Section 17.5.2.7. See: https://www.itu.int/rec/T-REC-X.509-201910-I/en. - */ -const X509V3_EXT_METHOD ossl_v3_no_assertion = { - NID_no_assertion, 0, ASN1_ITEM_ref(ASN1_NULL), - 0, 0, 0, 0, - (X509V3_EXT_I2S)i2s_NO_ASSERTION, - (X509V3_EXT_S2I)s2i_NO_ASSERTION, - 0, 0, - (X509V3_EXT_I2R)i2r_NO_ASSERTION, - (X509V3_EXT_R2I)r2i_NO_ASSERTION, - NULL -}; diff --git a/openssl/src/crypto/x509/v3_no_rev_avail.c b/openssl/src/crypto/x509/v3_no_rev_avail.c deleted file mode 100644 index 849feda5e..000000000 --- a/openssl/src/crypto/x509/v3_no_rev_avail.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/cryptlib.h" -#include -#include -#include -#include "ext_dat.h" - -static int i2r_NO_REV_AVAIL(X509V3_EXT_METHOD *method, - void *su, BIO *out, - int indent) -{ - return 1; -} - -static void *r2i_NO_REV_AVAIL(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *value) -{ - return ASN1_NULL_new(); -} - -static char *i2s_NO_REV_AVAIL(const X509V3_EXT_METHOD *method, void *val) -{ - return OPENSSL_strdup("NULL"); -} - -static void *s2i_NO_REV_AVAIL(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, const char *str) -{ - return ASN1_NULL_new(); -} - -/* - * The noRevAvail X.509v3 extension is defined in ITU Recommendation X.509 - * (2019), Section 17.2.2.7. See: https://www.itu.int/rec/T-REC-X.509-201910-I/en. - */ -const X509V3_EXT_METHOD ossl_v3_no_rev_avail = { - NID_no_rev_avail, 0, ASN1_ITEM_ref(ASN1_NULL), - 0, 0, 0, 0, - (X509V3_EXT_I2S)i2s_NO_REV_AVAIL, - (X509V3_EXT_S2I)s2i_NO_REV_AVAIL, - 0, 0, - (X509V3_EXT_I2R)i2r_NO_REV_AVAIL, - (X509V3_EXT_R2I)r2i_NO_REV_AVAIL, - NULL -}; diff --git a/openssl/src/crypto/x509/v3_pci.c b/openssl/src/crypto/x509/v3_pci.c index 8b8b6e3ab..a931e01a9 100644 --- a/openssl/src/crypto/x509/v3_pci.c +++ b/openssl/src/crypto/x509/v3_pci.c @@ -112,22 +112,21 @@ static int process_pci_value(CONF_VALUE *val, return 0; } } else if (strcmp(val->name, "policy") == 0) { - char *valp = val->value; unsigned char *tmp_data = NULL; long val_len; if (*policy == NULL) { *policy = ASN1_OCTET_STRING_new(); if (*policy == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); X509V3_conf_err(val); return 0; } free_policy = 1; } - if (CHECK_AND_SKIP_PREFIX(valp, "hex:")) { + if (strncmp(val->value, "hex:", 4) == 0) { unsigned char *tmp_data2 = - OPENSSL_hexstr2buf(valp, &val_len); + OPENSSL_hexstr2buf(val->value + 4, &val_len); if (!tmp_data2) { X509V3_conf_err(val); @@ -151,14 +150,15 @@ static int process_pci_value(CONF_VALUE *val, OPENSSL_free((*policy)->data); (*policy)->data = NULL; (*policy)->length = 0; + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); X509V3_conf_err(val); goto err; } OPENSSL_free(tmp_data2); - } else if (CHECK_AND_SKIP_PREFIX(valp, "file:")) { + } else if (strncmp(val->value, "file:", 5) == 0) { unsigned char buf[2048]; int n; - BIO *b = BIO_new_file(valp, "r"); + BIO *b = BIO_new_file(val->value + 5, "r"); if (!b) { ERR_raise(ERR_LIB_X509V3, ERR_R_BIO_LIB); X509V3_conf_err(val); @@ -176,6 +176,7 @@ static int process_pci_value(CONF_VALUE *val, OPENSSL_free((*policy)->data); (*policy)->data = NULL; (*policy)->length = 0; + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); X509V3_conf_err(val); BIO_free_all(b); goto err; @@ -193,8 +194,8 @@ static int process_pci_value(CONF_VALUE *val, X509V3_conf_err(val); goto err; } - } else if (CHECK_AND_SKIP_PREFIX(valp, "text:")) { - val_len = strlen(valp); + } else if (strncmp(val->value, "text:", 5) == 0) { + val_len = strlen(val->value + 5); tmp_data = OPENSSL_realloc((*policy)->data, (*policy)->length + val_len + 1); if (tmp_data) { @@ -211,6 +212,7 @@ static int process_pci_value(CONF_VALUE *val, OPENSSL_free((*policy)->data); (*policy)->data = NULL; (*policy)->length = 0; + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); X509V3_conf_err(val); goto err; } @@ -220,6 +222,7 @@ static int process_pci_value(CONF_VALUE *val, goto err; } if (!tmp_data) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); X509V3_conf_err(val); goto err; } @@ -293,7 +296,7 @@ static PROXY_CERT_INFO_EXTENSION *r2i_pci(X509V3_EXT_METHOD *method, pci = PROXY_CERT_INFO_EXTENSION_new(); if (pci == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } diff --git a/openssl/src/crypto/x509/v3_pcia.c b/openssl/src/crypto/x509/v3_pcia.c index 478de610a..7f5985f5e 100644 --- a/openssl/src/crypto/x509/v3_pcia.c +++ b/openssl/src/crypto/x509/v3_pcia.c @@ -47,16 +47,18 @@ #include #include -ASN1_SEQUENCE(PROXY_POLICY) = { - ASN1_SIMPLE(PROXY_POLICY, policyLanguage, ASN1_OBJECT), - ASN1_OPT(PROXY_POLICY, policy, ASN1_OCTET_STRING) +ASN1_SEQUENCE(PROXY_POLICY) = + { + ASN1_SIMPLE(PROXY_POLICY,policyLanguage,ASN1_OBJECT), + ASN1_OPT(PROXY_POLICY,policy,ASN1_OCTET_STRING) } ASN1_SEQUENCE_END(PROXY_POLICY) IMPLEMENT_ASN1_FUNCTIONS(PROXY_POLICY) -ASN1_SEQUENCE(PROXY_CERT_INFO_EXTENSION) = { - ASN1_OPT(PROXY_CERT_INFO_EXTENSION, pcPathLengthConstraint, ASN1_INTEGER), - ASN1_SIMPLE(PROXY_CERT_INFO_EXTENSION, proxyPolicy, PROXY_POLICY) +ASN1_SEQUENCE(PROXY_CERT_INFO_EXTENSION) = + { + ASN1_OPT(PROXY_CERT_INFO_EXTENSION,pcPathLengthConstraint,ASN1_INTEGER), + ASN1_SIMPLE(PROXY_CERT_INFO_EXTENSION,proxyPolicy,PROXY_POLICY) } ASN1_SEQUENCE_END(PROXY_CERT_INFO_EXTENSION) IMPLEMENT_ASN1_FUNCTIONS(PROXY_CERT_INFO_EXTENSION) diff --git a/openssl/src/crypto/x509/v3_pcons.c b/openssl/src/crypto/x509/v3_pcons.c index 72c2364b0..128365f57 100644 --- a/openssl/src/crypto/x509/v3_pcons.c +++ b/openssl/src/crypto/x509/v3_pcons.c @@ -61,7 +61,7 @@ static void *v2i_POLICY_CONSTRAINTS(const X509V3_EXT_METHOD *method, int i; if ((pcons = POLICY_CONSTRAINTS_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } for (i = 0; i < sk_CONF_VALUE_num(values); i++) { diff --git a/openssl/src/crypto/x509/v3_pmaps.c b/openssl/src/crypto/x509/v3_pmaps.c index e5d7dddc0..2094e9671 100644 --- a/openssl/src/crypto/x509/v3_pmaps.c +++ b/openssl/src/crypto/x509/v3_pmaps.c @@ -73,7 +73,7 @@ static void *v2i_POLICY_MAPPINGS(const X509V3_EXT_METHOD *method, int i; if ((pmaps = sk_POLICY_MAPPING_new_reserve(NULL, num)) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } @@ -93,7 +93,7 @@ static void *v2i_POLICY_MAPPINGS(const X509V3_EXT_METHOD *method, } pmap = POLICY_MAPPING_new(); if (pmap == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } pmap->issuerDomainPolicy = obj1; diff --git a/openssl/src/crypto/x509/v3_purp.c b/openssl/src/crypto/x509/v3_purp.c index e917c455d..a6ebbd5f9 100644 --- a/openssl/src/crypto/x509/v3_purp.c +++ b/openssl/src/crypto/x509/v3_purp.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,26 +18,24 @@ static int check_ssl_ca(const X509 *x); static int check_purpose_ssl_client(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); + int require_ca); static int check_purpose_ssl_server(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); + int require_ca); static int check_purpose_ns_ssl_server(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); -static int purpose_smime(const X509 *x, int non_leaf); + int require_ca); +static int purpose_smime(const X509 *x, int require_ca); static int check_purpose_smime_sign(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); + int require_ca); static int check_purpose_smime_encrypt(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); + int require_ca); static int check_purpose_crl_sign(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); + int require_ca); static int check_purpose_timestamp_sign(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); -static int check_purpose_code_sign(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); + int require_ca); static int no_check_purpose(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); + int require_ca); static int check_purpose_ocsp_helper(const X509_PURPOSE *xp, const X509 *x, - int non_leaf); + int require_ca); static int xp_cmp(const X509_PURPOSE *const *a, const X509_PURPOSE *const *b); static void xptable_free(X509_PURPOSE *p); @@ -63,9 +61,6 @@ static X509_PURPOSE xstandard[] = { {X509_PURPOSE_TIMESTAMP_SIGN, X509_TRUST_TSA, 0, check_purpose_timestamp_sign, "Time Stamp signing", "timestampsign", NULL}, - {X509_PURPOSE_CODE_SIGN, X509_TRUST_OBJECT_SIGN, 0, - check_purpose_code_sign, "Code signing", "codesign", - NULL}, }; #define X509_PURPOSE_COUNT OSSL_NELEM(xstandard) @@ -83,7 +78,7 @@ static int xp_cmp(const X509_PURPOSE *const *a, const X509_PURPOSE *const *b) * If id == -1 it just calls x509v3_cache_extensions() for its side-effect. * Returns 1 on success, 0 if x does not allow purpose, -1 on (internal) error. */ -int X509_check_purpose(X509 *x, int id, int non_leaf) +int X509_check_purpose(X509 *x, int id, int require_ca) { int idx; const X509_PURPOSE *pt; @@ -97,7 +92,7 @@ int X509_check_purpose(X509 *x, int id, int non_leaf) if (idx == -1) return -1; pt = X509_PURPOSE_get0(idx); - return pt->check_purpose(pt, x, non_leaf); + return pt->check_purpose(pt, x, require_ca); } int X509_PURPOSE_set(int *p, int purpose) @@ -130,7 +125,6 @@ int X509_PURPOSE_get_by_sname(const char *sname) { int i; X509_PURPOSE *xptmp; - for (i = 0; i < X509_PURPOSE_get_count(); i++) { xptmp = X509_PURPOSE_get0(i); if (strcmp(xptmp->sname, sname) == 0) @@ -171,23 +165,26 @@ int X509_PURPOSE_add(int id, int trust, int flags, idx = X509_PURPOSE_get_by_id(id); /* Need a new entry */ if (idx == -1) { - if ((ptmp = OPENSSL_malloc(sizeof(*ptmp))) == NULL) + if ((ptmp = OPENSSL_malloc(sizeof(*ptmp))) == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return 0; + } ptmp->flags = X509_PURPOSE_DYNAMIC; - } else { + } else ptmp = X509_PURPOSE_get0(idx); - } /* OPENSSL_free existing name if dynamic */ - if ((ptmp->flags & X509_PURPOSE_DYNAMIC_NAME) != 0) { + if (ptmp->flags & X509_PURPOSE_DYNAMIC_NAME) { OPENSSL_free(ptmp->name); OPENSSL_free(ptmp->sname); } /* Dup supplied name */ ptmp->name = OPENSSL_strdup(name); ptmp->sname = OPENSSL_strdup(sname); - if (ptmp->name == NULL || ptmp->sname == NULL) + if (ptmp->name == NULL|| ptmp->sname == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; + } /* Keep the dynamic flag of existing entry */ ptmp->flags &= X509_PURPOSE_DYNAMIC; /* Set all other flags */ @@ -202,11 +199,11 @@ int X509_PURPOSE_add(int id, int trust, int flags, if (idx == -1) { if (xptable == NULL && (xptable = sk_X509_PURPOSE_new(xp_cmp)) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } if (!sk_X509_PURPOSE_push(xptable, ptmp)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } } @@ -224,8 +221,8 @@ static void xptable_free(X509_PURPOSE *p) { if (p == NULL) return; - if ((p->flags & X509_PURPOSE_DYNAMIC) != 0) { - if ((p->flags & X509_PURPOSE_DYNAMIC_NAME) != 0) { + if (p->flags & X509_PURPOSE_DYNAMIC) { + if (p->flags & X509_PURPOSE_DYNAMIC_NAME) { OPENSSL_free(p->name); OPENSSL_free(p->sname); } @@ -375,14 +372,14 @@ static int check_sig_alg_match(const EVP_PKEY *issuer_key, const X509 *subject) return X509_V_ERR_NO_ISSUER_PUBLIC_KEY; if (OBJ_find_sigid_algs(OBJ_obj2nid(subject->cert_info.signature.algorithm), NULL, &subj_sig_nid) == 0) - return X509_V_ERR_UNSUPPORTED_SIGNATURE_ALGORITHM; + return X509_V_ERR_UNSUPPORTED_SIGNATURE_ALGORITHM; if (EVP_PKEY_is_a(issuer_key, OBJ_nid2sn(subj_sig_nid)) || (EVP_PKEY_is_a(issuer_key, "RSA") && subj_sig_nid == NID_rsassaPss)) return X509_V_OK; return X509_V_ERR_SIGNATURE_ALGORITHM_MISMATCH; } -#define V1_ROOT (EXFLAG_V1 | EXFLAG_SS) +#define V1_ROOT (EXFLAG_V1|EXFLAG_SS) #define ku_reject(x, usage) \ (((x)->ex_flags & EXFLAG_KUSAGE) != 0 && ((x)->ex_kusage & (usage)) == 0) #define xku_reject(x, usage) \ @@ -415,17 +412,17 @@ int ossl_x509v3_cache_extensions(X509 *x) if (!CRYPTO_THREAD_write_lock(x->lock)) return 0; - if ((x->ex_flags & EXFLAG_SET) != 0) { /* Cert has already been processed */ + if (x->ex_flags & EXFLAG_SET) { /* Cert has already been processed */ CRYPTO_THREAD_unlock(x->lock); return (x->ex_flags & EXFLAG_INVALID) == 0; } - ERR_set_mark(); - /* Cache the SHA1 digest of the cert */ if (!X509_digest(x, EVP_sha1(), x->sha1_hash, NULL)) x->ex_flags |= EXFLAG_NO_FINGERPRINT; + ERR_set_mark(); + /* V1 should mean no extensions ... */ if (X509_get_version(x) == X509_VERSION_1) x->ex_flags |= EXFLAG_V1; @@ -441,7 +438,7 @@ int ossl_x509v3_cache_extensions(X509 *x) * in case ctx->param->flags & X509_V_FLAG_X509_STRICT */ if (bs->pathlen->type == V_ASN1_NEG_INTEGER) { - ERR_raise(ERR_LIB_X509V3, X509V3_R_NEGATIVE_PATHLEN); + ERR_raise(ERR_LIB_X509, X509V3_R_NEGATIVE_PATHLEN); x->ex_flags |= EXFLAG_INVALID; } else { x->ex_pathlen = ASN1_INTEGER_get(bs->pathlen); @@ -455,7 +452,7 @@ int ossl_x509v3_cache_extensions(X509 *x) /* Handle proxy certificates */ if ((pci = X509_get_ext_d2i(x, NID_proxyCertInfo, &i, NULL)) != NULL) { - if ((x->ex_flags & EXFLAG_CA) != 0 + if (x->ex_flags & EXFLAG_CA || X509_get_ext_by_NID(x, NID_subject_alt_name, -1) >= 0 || X509_get_ext_by_NID(x, NID_issuer_alt_name, -1) >= 0) { x->ex_flags |= EXFLAG_INVALID; @@ -482,7 +479,7 @@ int ossl_x509v3_cache_extensions(X509 *x) ASN1_BIT_STRING_free(usage); /* Check for empty key usage according to RFC 5280 section 4.2.1.3 */ if (x->ex_kusage == 0) { - ERR_raise(ERR_LIB_X509V3, X509V3_R_EMPTY_KEY_USAGE); + ERR_raise(ERR_LIB_X509, X509V3_R_EMPTY_KEY_USAGE); x->ex_flags |= EXFLAG_INVALID; } } else if (i != -1) { @@ -576,6 +573,8 @@ int ossl_x509v3_cache_extensions(X509 *x) res = setup_crldp(x); if (res == 0) x->ex_flags |= EXFLAG_INVALID; + else if (res < 0) + goto err; #ifndef OPENSSL_NO_RFC3779 x->rfc3779_addr = X509_get_ext_d2i(x, NID_sbgp_ipAddrBlock, &i, NULL); @@ -628,13 +627,17 @@ int ossl_x509v3_cache_extensions(X509 *x) */ #endif ERR_pop_to_mark(); - - if ((x->ex_flags & EXFLAG_INVALID) == 0) { + if ((x->ex_flags & (EXFLAG_INVALID | EXFLAG_NO_FINGERPRINT)) == 0) { CRYPTO_THREAD_unlock(x->lock); return 1; } + if ((x->ex_flags & EXFLAG_INVALID) != 0) + ERR_raise(ERR_LIB_X509, X509V3_R_INVALID_CERTIFICATE); + /* If computing sha1_hash failed the error queue already reflects this. */ + + err: + x->ex_flags |= EXFLAG_SET; /* indicate that cert has been processed */ CRYPTO_THREAD_unlock(x->lock); - ERR_raise(ERR_LIB_X509V3, X509V3_R_INVALID_CERTIFICATE); return 0; } @@ -665,11 +668,10 @@ static int check_ca(const X509 *x) /* * If key usage present it must have certSign so tolerate it */ - else if ((x->ex_flags & EXFLAG_KUSAGE) != 0) + else if (x->ex_flags & EXFLAG_KUSAGE) return 4; /* Older certificates could have Netscape-specific CA types */ - else if ((x->ex_flags & EXFLAG_NSCERT) != 0 - && (x->ex_nscert & NS_ANY_CA) != 0) + else if (x->ex_flags & EXFLAG_NSCERT && x->ex_nscert & NS_ANY_CA) return 5; /* Can this still be regarded a CA certificate? I doubt it. */ return 0; @@ -710,11 +712,11 @@ static int check_ssl_ca(const X509 *x) } static int check_purpose_ssl_client(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { if (xku_reject(x, XKU_SSL_CLIENT)) return 0; - if (non_leaf) + if (require_ca) return check_ssl_ca(x); /* We need to do digital signatures or key agreement */ if (ku_reject(x, KU_DIGITAL_SIGNATURE | KU_KEY_AGREEMENT)) @@ -731,14 +733,14 @@ static int check_purpose_ssl_client(const X509_PURPOSE *xp, const X509 *x, * key types. */ #define KU_TLS \ - KU_DIGITAL_SIGNATURE | KU_KEY_ENCIPHERMENT | KU_KEY_AGREEMENT + KU_DIGITAL_SIGNATURE|KU_KEY_ENCIPHERMENT|KU_KEY_AGREEMENT static int check_purpose_ssl_server(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { if (xku_reject(x, XKU_SSL_SERVER | XKU_SGC)) return 0; - if (non_leaf) + if (require_ca) return check_ssl_ca(x); if (ns_reject(x, NS_SSL_SERVER)) @@ -751,70 +753,82 @@ static int check_purpose_ssl_server(const X509_PURPOSE *xp, const X509 *x, } static int check_purpose_ns_ssl_server(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { - int ret = check_purpose_ssl_server(xp, x, non_leaf); - - if (!ret || non_leaf) + int ret; + ret = check_purpose_ssl_server(xp, x, require_ca); + if (!ret || require_ca) return ret; /* We need to encipher or Netscape complains */ - return ku_reject(x, KU_KEY_ENCIPHERMENT) ? 0 : ret; + if (ku_reject(x, KU_KEY_ENCIPHERMENT)) + return 0; + return ret; } /* common S/MIME checks */ -static int purpose_smime(const X509 *x, int non_leaf) +static int purpose_smime(const X509 *x, int require_ca) { if (xku_reject(x, XKU_SMIME)) return 0; - if (non_leaf) { - int ca_ret = check_ca(x); - + if (require_ca) { + int ca_ret; + ca_ret = check_ca(x); if (ca_ret == 0) return 0; /* Check nsCertType if present */ - if (ca_ret != 5 || (x->ex_nscert & NS_SMIME_CA) != 0) + if (ca_ret != 5 || x->ex_nscert & NS_SMIME_CA) return ca_ret; else return 0; } - if ((x->ex_flags & EXFLAG_NSCERT) != 0) { - if ((x->ex_nscert & NS_SMIME) != 0) + if (x->ex_flags & EXFLAG_NSCERT) { + if (x->ex_nscert & NS_SMIME) return 1; /* Workaround for some buggy certificates */ - return (x->ex_nscert & NS_SSL_CLIENT) != 0 ? 2 : 0; + if (x->ex_nscert & NS_SSL_CLIENT) + return 2; + return 0; } return 1; } static int check_purpose_smime_sign(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { - int ret = purpose_smime(x, non_leaf); - - if (!ret || non_leaf) + int ret; + ret = purpose_smime(x, require_ca); + if (!ret || require_ca) return ret; - return ku_reject(x, KU_DIGITAL_SIGNATURE | KU_NON_REPUDIATION) ? 0 : ret; + if (ku_reject(x, KU_DIGITAL_SIGNATURE | KU_NON_REPUDIATION)) + return 0; + return ret; } static int check_purpose_smime_encrypt(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { - int ret = purpose_smime(x, non_leaf); - - if (!ret || non_leaf) + int ret; + ret = purpose_smime(x, require_ca); + if (!ret || require_ca) return ret; - return ku_reject(x, KU_KEY_ENCIPHERMENT) ? 0 : ret; + if (ku_reject(x, KU_KEY_ENCIPHERMENT)) + return 0; + return ret; } static int check_purpose_crl_sign(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { - if (non_leaf) { - int ca_ret = check_ca(x); - - return ca_ret == 2 ? 0 : ca_ret; + if (require_ca) { + int ca_ret; + if ((ca_ret = check_ca(x)) != 2) + return ca_ret; + else + return 0; } - return !ku_reject(x, KU_CRL_SIGN); + if (ku_reject(x, KU_CRL_SIGN)) + return 0; + return 1; } /* @@ -822,117 +836,55 @@ static int check_purpose_crl_sign(const X509_PURPOSE *xp, const X509 *x, * is valid. Additional checks must be made on the chain. */ static int check_purpose_ocsp_helper(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { /* * Must be a valid CA. Should we really support the "I don't know" value * (2)? */ - if (non_leaf) + if (require_ca) return check_ca(x); /* Leaf certificate is checked in OCSP_verify() */ return 1; } static int check_purpose_timestamp_sign(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { int i_ext; - /* - * If non_leaf is true we must check if this is a valid CA certificate. - * The extra requirements by the CA/Browser Forum are not checked. - */ - if (non_leaf) + /* If ca is true we must return if this is a valid CA certificate. */ + if (require_ca) return check_ca(x); - /* - * Key Usage is checked according to RFC 5280 and - * Extended Key Usage attributes is checked according to RFC 3161. - * The extra (and somewhat conflicting) CA/Browser Forum - * Baseline Requirements for the Issuance and Management of - * Publicly‐Trusted Code Signing Certificates, Version 3.0.0, - * Section 7.1.2.3: Code signing and Timestamp Certificate are not checked. - */ /* * Check the optional key usage field: * if Key Usage is present, it must be one of digitalSignature * and/or nonRepudiation (other values are not consistent and shall * be rejected). */ - if ((x->ex_flags & EXFLAG_KUSAGE) != 0 + if ((x->ex_flags & EXFLAG_KUSAGE) && ((x->ex_kusage & ~(KU_NON_REPUDIATION | KU_DIGITAL_SIGNATURE)) || !(x->ex_kusage & (KU_NON_REPUDIATION | KU_DIGITAL_SIGNATURE)))) return 0; - /* Only timestamp key usage is permitted and it's required. */ - if ((x->ex_flags & EXFLAG_XKUSAGE) == 0 || x->ex_xkusage != XKU_TIMESTAMP) + /* Only time stamp key usage is permitted and it's required. */ + if (!(x->ex_flags & EXFLAG_XKUSAGE) || x->ex_xkusage != XKU_TIMESTAMP) return 0; /* Extended Key Usage MUST be critical */ i_ext = X509_get_ext_by_NID(x, NID_ext_key_usage, -1); - if (i_ext >= 0 - && !X509_EXTENSION_get_critical(X509_get_ext((X509 *)x, i_ext))) - return 0; - return 1; -} - -static int check_purpose_code_sign(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) -{ - int i_ext; - - /* - * If non_leaf is true we must check if this is a valid CA certificate. - * The extra requirements by the CA/Browser Forum are not checked. - */ - if (non_leaf) - return check_ca(x); - - /* - * Check the key usage and extended key usage fields: - * - * Reference: CA/Browser Forum, - * Baseline Requirements for the Issuance and Management of - * Publicly‐Trusted Code Signing Certificates, Version 3.0.0, - * Section 7.1.2.3: Code signing and Timestamp Certificate - * - * Checking covers Key Usage and Extended Key Usage attributes. - * The certificatePolicies, cRLDistributionPoints (CDP), and - * authorityInformationAccess (AIA) extensions are so far not checked. - */ - /* Key Usage */ - if ((x->ex_flags & EXFLAG_KUSAGE) == 0) - return 0; - if ((x->ex_kusage & KU_DIGITAL_SIGNATURE) == 0) - return 0; - if ((x->ex_kusage & (KU_KEY_CERT_SIGN | KU_CRL_SIGN)) != 0) - return 0; - - /* Key Usage MUST be critical */ - i_ext = X509_get_ext_by_NID(x, NID_key_usage, -1); - if (i_ext < 0) - return 0; if (i_ext >= 0) { X509_EXTENSION *ext = X509_get_ext((X509 *)x, i_ext); if (!X509_EXTENSION_get_critical(ext)) return 0; } - /* Extended Key Usage */ - if ((x->ex_flags & EXFLAG_XKUSAGE) == 0) - return 0; - if ((x->ex_xkusage & XKU_CODE_SIGN) == 0) - return 0; - if ((x->ex_xkusage & (XKU_ANYEKU | XKU_SSL_SERVER)) != 0) - return 0; - return 1; - } static int no_check_purpose(const X509_PURPOSE *xp, const X509 *x, - int non_leaf) + int require_ca) { return 1; } @@ -990,12 +942,11 @@ int ossl_x509_likely_issued(X509 *issuer, X509 *subject) */ int ossl_x509_signing_allowed(const X509 *issuer, const X509 *subject) { - if ((subject->ex_flags & EXFLAG_PROXY) != 0) { + if (subject->ex_flags & EXFLAG_PROXY) { if (ku_reject(issuer, KU_DIGITAL_SIGNATURE)) return X509_V_ERR_KEYUSAGE_NO_DIGITAL_SIGNATURE; - } else if (ku_reject(issuer, KU_KEY_CERT_SIGN)) { + } else if (ku_reject(issuer, KU_KEY_CERT_SIGN)) return X509_V_ERR_KEYUSAGE_NO_CERTSIGN; - } return X509_V_OK; } @@ -1019,11 +970,11 @@ int X509_check_akid(const X509 *issuer, const AUTHORITY_KEYID *akid) * GeneralName. So look for a DirName. There may be more than one but * we only take any notice of the first. */ - GENERAL_NAMES *gens = akid->issuer; + GENERAL_NAMES *gens; GENERAL_NAME *gen; X509_NAME *nm = NULL; int i; - + gens = akid->issuer; for (i = 0; i < sk_GENERAL_NAME_num(gens); i++) { gen = sk_GENERAL_NAME_value(gens, i); if (gen->type == GEN_DIRNAME) { @@ -1049,7 +1000,9 @@ uint32_t X509_get_key_usage(X509 *x) /* Call for side-effect of computing hash and caching extensions */ if (X509_check_purpose(x, -1, 0) != 1) return 0; - return (x->ex_flags & EXFLAG_KUSAGE) != 0 ? x->ex_kusage : UINT32_MAX; + if (x->ex_flags & EXFLAG_KUSAGE) + return x->ex_kusage; + return UINT32_MAX; } uint32_t X509_get_extended_key_usage(X509 *x) @@ -1057,7 +1010,9 @@ uint32_t X509_get_extended_key_usage(X509 *x) /* Call for side-effect of computing hash and caching extensions */ if (X509_check_purpose(x, -1, 0) != 1) return 0; - return (x->ex_flags & EXFLAG_XKUSAGE) != 0 ? x->ex_xkusage : UINT32_MAX; + if (x->ex_flags & EXFLAG_XKUSAGE) + return x->ex_xkusage; + return UINT32_MAX; } const ASN1_OCTET_STRING *X509_get0_subject_key_id(X509 *x) diff --git a/openssl/src/crypto/x509/v3_san.c b/openssl/src/crypto/x509/v3_san.c index 9adf49470..c081f02e1 100644 --- a/openssl/src/crypto/x509/v3_san.c +++ b/openssl/src/crypto/x509/v3_san.c @@ -307,7 +307,7 @@ static GENERAL_NAMES *v2i_issuer_alt(X509V3_EXT_METHOD *method, int i; if (gens == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); sk_GENERAL_NAME_free(gens); return NULL; } @@ -358,7 +358,7 @@ static int copy_issuer(X509V3_CTX *ctx, GENERAL_NAMES *gens) num = sk_GENERAL_NAME_num(ialt); if (!sk_GENERAL_NAME_reserve(gens, num)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } @@ -386,7 +386,7 @@ static GENERAL_NAMES *v2i_subject_alt(X509V3_EXT_METHOD *method, gens = sk_GENERAL_NAME_new_reserve(NULL, num); if (gens == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); sk_GENERAL_NAME_free(gens); return NULL; } @@ -449,14 +449,14 @@ static int copy_email(X509V3_CTX *ctx, GENERAL_NAMES *gens, int move_p) i--; } if (email == NULL || (gen = GENERAL_NAME_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } gen->d.ia5 = email; email = NULL; gen->type = GEN_EMAIL; if (!sk_GENERAL_NAME_push(gens, gen)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } gen = NULL; @@ -482,7 +482,7 @@ GENERAL_NAMES *v2i_GENERAL_NAMES(const X509V3_EXT_METHOD *method, gens = sk_GENERAL_NAME_new_reserve(NULL, num); if (gens == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); sk_GENERAL_NAME_free(gens); return NULL; } @@ -523,7 +523,7 @@ GENERAL_NAME *a2i_GENERAL_NAME(GENERAL_NAME *out, else { gen = GENERAL_NAME_new(); if (gen == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } } @@ -581,9 +581,7 @@ GENERAL_NAME *a2i_GENERAL_NAME(GENERAL_NAME *out, if ((gen->d.ia5 = ASN1_IA5STRING_new()) == NULL || !ASN1_STRING_set(gen->d.ia5, (unsigned char *)value, strlen(value))) { - ASN1_IA5STRING_free(gen->d.ia5); - gen->d.ia5 = NULL; - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } } @@ -653,21 +651,16 @@ static int do_othername(GENERAL_NAME *gen, const char *value, X509V3_CTX *ctx) */ ASN1_TYPE_free(gen->d.otherName->value); if ((gen->d.otherName->value = ASN1_generate_v3(p + 1, ctx)) == NULL) - goto err; + return 0; objlen = p - value; objtmp = OPENSSL_strndup(value, objlen); if (objtmp == NULL) - goto err; + return 0; gen->d.otherName->type_id = OBJ_txt2obj(objtmp, 0); OPENSSL_free(objtmp); if (!gen->d.otherName->type_id) - goto err; + return 0; return 1; - - err: - OTHERNAME_free(gen->d.otherName); - gen->d.otherName = NULL; - return 0; } static int do_dirname(GENERAL_NAME *gen, const char *value, X509V3_CTX *ctx) diff --git a/openssl/src/crypto/x509/v3_single_use.c b/openssl/src/crypto/x509/v3_single_use.c deleted file mode 100644 index 50da6e177..000000000 --- a/openssl/src/crypto/x509/v3_single_use.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/cryptlib.h" -#include -#include -#include -#include "ext_dat.h" - -static int i2r_SINGLE_USE(X509V3_EXT_METHOD *method, - void *su, BIO *out, - int indent) -{ - return 1; -} - -static void *r2i_SINGLE_USE(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *value) -{ - return ASN1_NULL_new(); -} - -static char *i2s_SINGLE_USE(const X509V3_EXT_METHOD *method, void *val) -{ - return OPENSSL_strdup("NULL"); -} - -static void *s2i_SINGLE_USE(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, const char *str) -{ - return ASN1_NULL_new(); -} - -/* - * The singleUse X.509v3 extension is defined in ITU Recommendation X.509 - * (2019), Section 17.1.2.5. See: https://www.itu.int/rec/T-REC-X.509-201910-I/en. - */ -const X509V3_EXT_METHOD ossl_v3_single_use = { - NID_single_use, 0, ASN1_ITEM_ref(ASN1_NULL), - 0, 0, 0, 0, - (X509V3_EXT_I2S)i2s_SINGLE_USE, - (X509V3_EXT_S2I)s2i_SINGLE_USE, - 0, 0, - (X509V3_EXT_I2R)i2r_SINGLE_USE, - (X509V3_EXT_R2I)r2i_SINGLE_USE, - NULL -}; diff --git a/openssl/src/crypto/x509/v3_skid.c b/openssl/src/crypto/x509/v3_skid.c index 8657f4cdf..18223f2ef 100644 --- a/openssl/src/crypto/x509/v3_skid.c +++ b/openssl/src/crypto/x509/v3_skid.c @@ -37,7 +37,7 @@ ASN1_OCTET_STRING *s2i_ASN1_OCTET_STRING(X509V3_EXT_METHOD *method, long length; if ((oct = ASN1_OCTET_STRING_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/x509/v3_soa_id.c b/openssl/src/crypto/x509/v3_soa_id.c deleted file mode 100644 index 8be5bd8aa..000000000 --- a/openssl/src/crypto/x509/v3_soa_id.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/cryptlib.h" -#include -#include -#include -#include "ext_dat.h" - -static int i2r_SOA_IDENTIFIER(X509V3_EXT_METHOD *method, - void *su, BIO *out, - int indent) -{ - return 1; -} - -static void *r2i_SOA_IDENTIFIER(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *value) -{ - return ASN1_NULL_new(); -} - -static char *i2s_SOA_IDENTIFIER(const X509V3_EXT_METHOD *method, void *val) -{ - return OPENSSL_strdup("NULL"); -} - -static void *s2i_SOA_IDENTIFIER(const X509V3_EXT_METHOD *method, X509V3_CTX *ctx, const char *str) -{ - return ASN1_NULL_new(); -} - -/* - * The sOAIdentifier X.509v3 extension is defined in ITU Recommendation X.509 - * (2019), Section 17.3.2.1.1. See: https://www.itu.int/rec/T-REC-X.509-201910-I/en. - */ -const X509V3_EXT_METHOD ossl_v3_soa_identifier = { - NID_soa_identifier, 0, ASN1_ITEM_ref(ASN1_NULL), - 0, 0, 0, 0, - (X509V3_EXT_I2S)i2s_SOA_IDENTIFIER, - (X509V3_EXT_S2I)s2i_SOA_IDENTIFIER, - 0, 0, - (X509V3_EXT_I2R)i2r_SOA_IDENTIFIER, - (X509V3_EXT_R2I)r2i_SOA_IDENTIFIER, - NULL -}; diff --git a/openssl/src/crypto/x509/v3_sxnet.c b/openssl/src/crypto/x509/v3_sxnet.c index 507945f82..ad62a6d16 100644 --- a/openssl/src/crypto/x509/v3_sxnet.c +++ b/openssl/src/crypto/x509/v3_sxnet.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -78,8 +78,6 @@ static int sxnet_i2r(X509V3_EXT_METHOD *method, SXNET *sx, BIO *out, for (i = 0; i < sk_SXNETID_num(sx->ids); i++) { id = sk_SXNETID_value(sx->ids, i); tmp = i2s_ASN1_INTEGER(NULL, id->zone); - if (tmp == NULL) - return 0; BIO_printf(out, "\n%*sZone: %s, User: ", indent, "", tmp); OPENSSL_free(tmp); ASN1_STRING_print(out, id->user); @@ -103,10 +101,8 @@ static SXNET *sxnet_v2i(X509V3_EXT_METHOD *method, X509V3_CTX *ctx, int i; for (i = 0; i < sk_CONF_VALUE_num(nval); i++) { cnf = sk_CONF_VALUE_value(nval, i); - if (!SXNET_add_id_asc(&sx, cnf->name, cnf->value, -1)) { - SXNET_free(sx); + if (!SXNET_add_id_asc(&sx, cnf->name, cnf->value, -1)) return NULL; - } } return sx; } @@ -125,11 +121,7 @@ int SXNET_add_id_asc(SXNET **psx, const char *zone, const char *user, int userle ERR_raise(ERR_LIB_X509V3, X509V3_R_ERROR_CONVERTING_ZONE); return 0; } - if (!SXNET_add_id_INTEGER(psx, izone, user, userlen)) { - ASN1_INTEGER_free(izone); - return 0; - } - return 1; + return SXNET_add_id_INTEGER(psx, izone, user, userlen); } /* Add an id given the zone as an unsigned long */ @@ -141,15 +133,12 @@ int SXNET_add_id_ulong(SXNET **psx, unsigned long lzone, const char *user, if ((izone = ASN1_INTEGER_new()) == NULL || !ASN1_INTEGER_set(izone, lzone)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - ASN1_INTEGER_free(izone); - return 0; - } - if (!SXNET_add_id_INTEGER(psx, izone, user, userlen)) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); ASN1_INTEGER_free(izone); return 0; } - return 1; + return SXNET_add_id_INTEGER(psx, izone, user, userlen); + } /* @@ -174,14 +163,10 @@ int SXNET_add_id_INTEGER(SXNET **psx, ASN1_INTEGER *zone, const char *user, return 0; } if (*psx == NULL) { - if ((sx = SXNET_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + if ((sx = SXNET_new()) == NULL) goto err; - } - if (!ASN1_INTEGER_set(sx->version, 0)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + if (!ASN1_INTEGER_set(sx->version, 0)) goto err; - } } else sx = *psx; if (SXNET_get_id_INTEGER(sx, zone)) { @@ -191,25 +176,21 @@ int SXNET_add_id_INTEGER(SXNET **psx, ASN1_INTEGER *zone, const char *user, return 0; } - if ((id = SXNETID_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + if ((id = SXNETID_new()) == NULL) goto err; - } + if (userlen == -1) + userlen = strlen(user); - if (!ASN1_OCTET_STRING_set(id->user, (const unsigned char *)user, userlen)){ - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + if (!ASN1_OCTET_STRING_set(id->user, (const unsigned char *)user, userlen)) goto err; - } - if (!sk_SXNETID_push(sx->ids, id)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + if (!sk_SXNETID_push(sx->ids, id)) goto err; - } - ASN1_INTEGER_free(id->zone); id->zone = zone; *psx = sx; return 1; err: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); SXNETID_free(id); if (*psx == NULL) SXNET_free(sx); @@ -237,7 +218,7 @@ ASN1_OCTET_STRING *SXNET_get_id_ulong(SXNET *sx, unsigned long lzone) if ((izone = ASN1_INTEGER_new()) == NULL || !ASN1_INTEGER_set(izone, lzone)) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); ASN1_INTEGER_free(izone); return NULL; } diff --git a/openssl/src/crypto/x509/v3_tlsf.c b/openssl/src/crypto/x509/v3_tlsf.c index 85dea65f3..a1446bc07 100644 --- a/openssl/src/crypto/x509/v3_tlsf.c +++ b/openssl/src/crypto/x509/v3_tlsf.c @@ -96,7 +96,7 @@ static TLS_FEATURE *v2i_TLS_FEATURE(const X509V3_EXT_METHOD *method, long tlsextid; if ((tlsf = sk_ASN1_INTEGER_new_null()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } @@ -125,7 +125,7 @@ static TLS_FEATURE *v2i_TLS_FEATURE(const X509V3_EXT_METHOD *method, if ((ai = ASN1_INTEGER_new()) == NULL || !ASN1_INTEGER_set(ai, tlsextid) || sk_ASN1_INTEGER_push(tlsf, ai) <= 0) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; } /* So it doesn't get purged if an error occurs next time around */ diff --git a/openssl/src/crypto/x509/v3_utf8.c b/openssl/src/crypto/x509/v3_utf8.c deleted file mode 100644 index 22345c3a6..000000000 --- a/openssl/src/crypto/x509/v3_utf8.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/cryptlib.h" -#include -#include -#include -#include "ext_dat.h" - -/* - * Subject Sign Tool (1.2.643.100.111) The name of the tool used to signs the subject (UTF8String) - * This extension is required to obtain the status of a qualified certificate at Russian Federation. - * RFC-style description is available here: https://tools.ietf.org/html/draft-deremin-rfc4491-bis-04#section-5 - * Russian Federal Law 63 "Digital Sign" is available here: http://www.consultant.ru/document/cons_doc_LAW_112701/ - */ - - -const X509V3_EXT_METHOD ossl_v3_utf8_list[1] = { - EXT_UTF8STRING(NID_subjectSignTool), -}; - -char *i2s_ASN1_UTF8STRING(X509V3_EXT_METHOD *method, - ASN1_UTF8STRING *utf8) -{ - char *tmp; - - if (utf8 == NULL || utf8->length == 0) { - ERR_raise(ERR_LIB_X509V3, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - if ((tmp = OPENSSL_malloc(utf8->length + 1)) == NULL) - return NULL; - memcpy(tmp, utf8->data, utf8->length); - tmp[utf8->length] = 0; - return tmp; -} - -ASN1_UTF8STRING *s2i_ASN1_UTF8STRING(X509V3_EXT_METHOD *method, - X509V3_CTX *ctx, const char *str) -{ - ASN1_UTF8STRING *utf8; - if (str == NULL) { - ERR_raise(ERR_LIB_X509V3, X509V3_R_INVALID_NULL_ARGUMENT); - return NULL; - } - if ((utf8 = ASN1_UTF8STRING_new()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - return NULL; - } - if (!ASN1_STRING_set((ASN1_STRING *)utf8, str, strlen(str))) { - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - ASN1_UTF8STRING_free(utf8); - return NULL; - } -#ifdef CHARSET_EBCDIC - ebcdic2ascii(utf8->data, utf8->data, utf8->length); -#endif /* CHARSET_EBCDIC */ - return utf8; -} diff --git a/openssl/src/crypto/x509/v3_utl.c b/openssl/src/crypto/x509/v3_utl.c index 1a1817499..8c2e164db 100644 --- a/openssl/src/crypto/x509/v3_utl.c +++ b/openssl/src/crypto/x509/v3_utl.c @@ -47,7 +47,7 @@ static int x509v3_add_len_value(const char *name, const char *value, if (name != NULL && (tname = OPENSSL_strdup(name)) == NULL) goto err; if (value != NULL) { - /* We don't allow embedded NUL characters */ + /* We don't allow embeded NUL characters */ if (memchr(value, 0, vallen) != NULL) goto err; tvalue = OPENSSL_strndup(value, vallen); @@ -56,10 +56,8 @@ static int x509v3_add_len_value(const char *name, const char *value, } if ((vtmp = OPENSSL_malloc(sizeof(*vtmp))) == NULL) goto err; - if (sk_allocated && (*extlist = sk_CONF_VALUE_new_null()) == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_CRYPTO_LIB); + if (sk_allocated && (*extlist = sk_CONF_VALUE_new_null()) == NULL) goto err; - } vtmp->section = NULL; vtmp->name = tname; vtmp->value = tvalue; @@ -67,6 +65,7 @@ static int x509v3_add_len_value(const char *name, const char *value, goto err; return 1; err: + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); if (sk_allocated) { sk_CONF_VALUE_free(*extlist); *extlist = NULL; @@ -147,6 +146,7 @@ static char *bignum_to_string(const BIGNUM *bn) len = strlen(tmp) + 3; ret = OPENSSL_malloc(len); if (ret == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); OPENSSL_free(tmp); return NULL; } @@ -170,10 +170,9 @@ char *i2s_ASN1_ENUMERATED(X509V3_EXT_METHOD *method, const ASN1_ENUMERATED *a) if (!a) return NULL; - if ((bntmp = ASN1_ENUMERATED_to_BN(a, NULL)) == NULL) - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - else if ((strtmp = bignum_to_string(bntmp)) == NULL) - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + if ((bntmp = ASN1_ENUMERATED_to_BN(a, NULL)) == NULL + || (strtmp = bignum_to_string(bntmp)) == NULL) + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); BN_free(bntmp); return strtmp; } @@ -185,10 +184,9 @@ char *i2s_ASN1_INTEGER(X509V3_EXT_METHOD *method, const ASN1_INTEGER *a) if (!a) return NULL; - if ((bntmp = ASN1_INTEGER_to_BN(a, NULL)) == NULL) - ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB); - else if ((strtmp = bignum_to_string(bntmp)) == NULL) - ERR_raise(ERR_LIB_X509V3, ERR_R_X509V3_LIB); + if ((bntmp = ASN1_INTEGER_to_BN(a, NULL)) == NULL + || (strtmp = bignum_to_string(bntmp)) == NULL) + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); BN_free(bntmp); return strtmp; } @@ -206,7 +204,7 @@ ASN1_INTEGER *s2i_ASN1_INTEGER(X509V3_EXT_METHOD *method, const char *value) } bn = BN_new(); if (bn == NULL) { - ERR_raise(ERR_LIB_X509V3, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); return NULL; } if (value[0] == '-') { @@ -322,8 +320,10 @@ STACK_OF(CONF_VALUE) *X509V3_parse_list(const char *line) /* We are going to modify the line so copy it first */ linebuf = OPENSSL_strdup(line); - if (linebuf == NULL) + if (linebuf == NULL) { + ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE); goto err; + } state = HDR_NAME; ntmp = NULL; /* Go through all characters */ @@ -715,7 +715,7 @@ static int wildcard_match(const unsigned char *prefix, size_t prefix_len, } /* IDNA labels cannot match partial wildcards */ if (!allow_idna && - subject_len >= 4 && HAS_CASE_PREFIX((const char *)subject, "xn--")) + subject_len >= 4 && OPENSSL_strncasecmp((char *)subject, "xn--", 4) == 0) return 0; /* The wildcard may match a literal '*' */ if (wildcard_end == wildcard_start + 1 && *wildcard_start == '*') @@ -775,7 +775,7 @@ static const unsigned char *valid_star(const unsigned char *p, size_t len, || ('A' <= p[i] && p[i] <= 'Z') || ('0' <= p[i] && p[i] <= '9')) { if ((state & LABEL_START) != 0 - && len - i >= 4 && HAS_CASE_PREFIX((const char *)&p[i], "xn--")) + && len - i >= 4 && OPENSSL_strncasecmp((char *)&p[i], "xn--", 4) == 0) state |= LABEL_IDNA; state &= ~(LABEL_HYPHEN | LABEL_START); } else if (p[i] == '.') { diff --git a/openssl/src/crypto/x509/v3err.c b/openssl/src/crypto/x509/v3err.c index 6a75af416..6f38034c1 100644 --- a/openssl/src/crypto/x509/v3err.c +++ b/openssl/src/crypto/x509/v3err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,8 +17,6 @@ static const ERR_STRING_DATA X509V3_str_reasons[] = { {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_BAD_IP_ADDRESS), "bad ip address"}, {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_BAD_OBJECT), "bad object"}, - {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_BAD_OPTION), "bad option"}, - {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_BAD_VALUE), "bad value"}, {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_BN_DEC2BN_ERROR), "bn dec2bn error"}, {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_BN_TO_ASN1_INTEGER_ERROR), "bn to asn1 integer error"}, @@ -129,7 +127,6 @@ static const ERR_STRING_DATA X509V3_str_reasons[] = { {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_UNKNOWN_EXTENSION_NAME), "unknown extension name"}, {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_UNKNOWN_OPTION), "unknown option"}, - {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_UNKNOWN_VALUE), "unknown value"}, {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_UNSUPPORTED_OPTION), "unsupported option"}, {ERR_PACK(ERR_LIB_X509V3, 0, X509V3_R_UNSUPPORTED_TYPE), diff --git a/openssl/src/crypto/x509/x509_att.c b/openssl/src/crypto/x509/x509_att.c index 5b002832c..73ac59454 100644 --- a/openssl/src/crypto/x509/x509_att.c +++ b/openssl/src/crypto/x509/x509_att.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -55,128 +55,68 @@ int X509at_get_attr_by_OBJ(const STACK_OF(X509_ATTRIBUTE) *sk, X509_ATTRIBUTE *X509at_get_attr(const STACK_OF(X509_ATTRIBUTE) *x, int loc) { - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - if (sk_X509_ATTRIBUTE_num(x) <= loc || loc < 0) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_INVALID_ARGUMENT); + if (x == NULL || sk_X509_ATTRIBUTE_num(x) <= loc || loc < 0) return NULL; - } + return sk_X509_ATTRIBUTE_value(x, loc); } X509_ATTRIBUTE *X509at_delete_attr(STACK_OF(X509_ATTRIBUTE) *x, int loc) { - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - if (sk_X509_ATTRIBUTE_num(x) <= loc || loc < 0) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_INVALID_ARGUMENT); + X509_ATTRIBUTE *ret; + + if (x == NULL || sk_X509_ATTRIBUTE_num(x) <= loc || loc < 0) return NULL; - } - return sk_X509_ATTRIBUTE_delete(x, loc); + ret = sk_X509_ATTRIBUTE_delete(x, loc); + return ret; } -STACK_OF(X509_ATTRIBUTE) *ossl_x509at_add1_attr(STACK_OF(X509_ATTRIBUTE) **x, - X509_ATTRIBUTE *attr) +STACK_OF(X509_ATTRIBUTE) *X509at_add1_attr(STACK_OF(X509_ATTRIBUTE) **x, + X509_ATTRIBUTE *attr) { X509_ATTRIBUTE *new_attr = NULL; STACK_OF(X509_ATTRIBUTE) *sk = NULL; - if (x == NULL || attr == NULL) { + if (x == NULL) { ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); return NULL; } if (*x == NULL) { - if ((sk = sk_X509_ATTRIBUTE_new_null()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + if ((sk = sk_X509_ATTRIBUTE_new_null()) == NULL) goto err; - } } else { sk = *x; } if ((new_attr = X509_ATTRIBUTE_dup(attr)) == NULL) + goto err2; + if (!sk_X509_ATTRIBUTE_push(sk, new_attr)) goto err; - if (!sk_X509_ATTRIBUTE_push(sk, new_attr)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); - goto err; - } if (*x == NULL) *x = sk; return sk; err: + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); + err2: X509_ATTRIBUTE_free(new_attr); if (*x == NULL) sk_X509_ATTRIBUTE_free(sk); return NULL; } -STACK_OF(X509_ATTRIBUTE) *X509at_add1_attr(STACK_OF(X509_ATTRIBUTE) **x, - X509_ATTRIBUTE *attr) -{ - if (x == NULL || attr == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - if (*x != NULL && X509at_get_attr_by_OBJ(*x, attr->object, -1) != -1) { - ERR_raise(ERR_LIB_X509, X509_R_DUPLICATE_ATTRIBUTE); - return NULL; - } - - return ossl_x509at_add1_attr(x, attr); -} - -STACK_OF(X509_ATTRIBUTE) *ossl_x509at_add1_attr_by_OBJ(STACK_OF(X509_ATTRIBUTE) **x, - const ASN1_OBJECT *obj, - int type, - const unsigned char *bytes, - int len) -{ - X509_ATTRIBUTE *attr; - STACK_OF(X509_ATTRIBUTE) *ret; - - attr = X509_ATTRIBUTE_create_by_OBJ(NULL, obj, type, bytes, len); - if (attr == NULL) - return 0; - ret = ossl_x509at_add1_attr(x, attr); - X509_ATTRIBUTE_free(attr); - return ret; -} - STACK_OF(X509_ATTRIBUTE) *X509at_add1_attr_by_OBJ(STACK_OF(X509_ATTRIBUTE) **x, const ASN1_OBJECT *obj, int type, const unsigned char *bytes, int len) -{ - if (x == NULL || obj == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - if (*x != NULL && X509at_get_attr_by_OBJ(*x, obj, -1) != -1) { - ERR_raise(ERR_LIB_X509, X509_R_DUPLICATE_ATTRIBUTE); - return NULL; - } - - return ossl_x509at_add1_attr_by_OBJ(x, obj, type, bytes, len); -} - -STACK_OF(X509_ATTRIBUTE) *ossl_x509at_add1_attr_by_NID(STACK_OF(X509_ATTRIBUTE) **x, - int nid, int type, - const unsigned char *bytes, - int len) { X509_ATTRIBUTE *attr; STACK_OF(X509_ATTRIBUTE) *ret; - - attr = X509_ATTRIBUTE_create_by_NID(NULL, nid, type, bytes, len); - if (attr == NULL) + attr = X509_ATTRIBUTE_create_by_OBJ(NULL, obj, type, bytes, len); + if (!attr) return 0; - ret = ossl_x509at_add1_attr(x, attr); + ret = X509at_add1_attr(x, attr); X509_ATTRIBUTE_free(attr); return ret; } @@ -185,32 +125,13 @@ STACK_OF(X509_ATTRIBUTE) *X509at_add1_attr_by_NID(STACK_OF(X509_ATTRIBUTE) **x, int nid, int type, const unsigned char *bytes, int len) -{ - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - if (*x != NULL && X509at_get_attr_by_NID(*x, nid, -1) != -1) { - ERR_raise(ERR_LIB_X509, X509_R_DUPLICATE_ATTRIBUTE); - return NULL; - } - - return ossl_x509at_add1_attr_by_NID(x, nid, type, bytes, len); -} - -STACK_OF(X509_ATTRIBUTE) *ossl_x509at_add1_attr_by_txt(STACK_OF(X509_ATTRIBUTE) **x, - const char *attrname, - int type, - const unsigned char *bytes, - int len) { X509_ATTRIBUTE *attr; STACK_OF(X509_ATTRIBUTE) *ret; - - attr = X509_ATTRIBUTE_create_by_txt(NULL, attrname, type, bytes, len); - if (attr == NULL) + attr = X509_ATTRIBUTE_create_by_NID(NULL, nid, type, bytes, len); + if (!attr) return 0; - ret = ossl_x509at_add1_attr(x, attr); + ret = X509at_add1_attr(x, attr); X509_ATTRIBUTE_free(attr); return ret; } @@ -223,9 +144,8 @@ STACK_OF(X509_ATTRIBUTE) *X509at_add1_attr_by_txt(STACK_OF(X509_ATTRIBUTE) { X509_ATTRIBUTE *attr; STACK_OF(X509_ATTRIBUTE) *ret; - attr = X509_ATTRIBUTE_create_by_txt(NULL, attrname, type, bytes, len); - if (attr == NULL) + if (!attr) return 0; ret = X509at_add1_attr(x, attr); X509_ATTRIBUTE_free(attr); @@ -235,26 +155,29 @@ STACK_OF(X509_ATTRIBUTE) *X509at_add1_attr_by_txt(STACK_OF(X509_ATTRIBUTE) void *X509at_get0_data_by_OBJ(const STACK_OF(X509_ATTRIBUTE) *x, const ASN1_OBJECT *obj, int lastpos, int type) { - int i = X509at_get_attr_by_OBJ(x, obj, lastpos); + int i; X509_ATTRIBUTE *at; - + i = X509at_get_attr_by_OBJ(x, obj, lastpos); if (i == -1) return NULL; - if (lastpos <= -2 && X509at_get_attr_by_OBJ(x, obj, i) != -1) + if ((lastpos <= -2) && (X509at_get_attr_by_OBJ(x, obj, i) != -1)) return NULL; at = X509at_get_attr(x, i); - if (lastpos <= -3 && X509_ATTRIBUTE_count(at) != 1) + if (lastpos <= -3 && (X509_ATTRIBUTE_count(at) != 1)) return NULL; return X509_ATTRIBUTE_get0_data(at, 0, type, NULL); } STACK_OF(X509_ATTRIBUTE) *ossl_x509at_dup(const STACK_OF(X509_ATTRIBUTE) *x) { - int i, n = sk_X509_ATTRIBUTE_num(x); + int i, n; STACK_OF(X509_ATTRIBUTE) *sk = NULL; + n = sk_X509_ATTRIBUTE_num(x); for (i = 0; i < n; ++i) { - if (X509at_add1_attr(&sk, sk_X509_ATTRIBUTE_value(x, i)) == NULL) { + X509_ATTRIBUTE *attr = sk_X509_ATTRIBUTE_value(x, i); + + if (X509at_add1_attr(&sk, attr) == NULL) { sk_X509_ATTRIBUTE_pop_free(sk, X509_ATTRIBUTE_free); return NULL; } @@ -266,9 +189,10 @@ X509_ATTRIBUTE *X509_ATTRIBUTE_create_by_NID(X509_ATTRIBUTE **attr, int nid, int atrtype, const void *data, int len) { - ASN1_OBJECT *obj = OBJ_nid2obj(nid); + ASN1_OBJECT *obj; X509_ATTRIBUTE *ret; + obj = OBJ_nid2obj(nid); if (obj == NULL) { ERR_raise(ERR_LIB_X509, X509_R_UNKNOWN_NID); return NULL; @@ -286,25 +210,24 @@ X509_ATTRIBUTE *X509_ATTRIBUTE_create_by_OBJ(X509_ATTRIBUTE **attr, { X509_ATTRIBUTE *ret; - if (attr == NULL || *attr == NULL) { + if ((attr == NULL) || (*attr == NULL)) { if ((ret = X509_ATTRIBUTE_new()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; } - } else { + } else ret = *attr; - } if (!X509_ATTRIBUTE_set1_object(ret, obj)) goto err; if (!X509_ATTRIBUTE_set1_data(ret, atrtype, data, len)) goto err; - if (attr != NULL && *attr == NULL) + if ((attr != NULL) && (*attr == NULL)) *attr = ret; return ret; err: - if (attr == NULL || ret != *attr) + if ((attr == NULL) || (ret != *attr)) X509_ATTRIBUTE_free(ret); return NULL; } @@ -314,9 +237,10 @@ X509_ATTRIBUTE *X509_ATTRIBUTE_create_by_txt(X509_ATTRIBUTE **attr, const unsigned char *bytes, int len) { - ASN1_OBJECT *obj = OBJ_txt2obj(atrname, 0); + ASN1_OBJECT *obj; X509_ATTRIBUTE *nattr; + obj = OBJ_txt2obj(atrname, 0); if (obj == NULL) { ERR_raise_data(ERR_LIB_X509, X509_R_INVALID_FIELD_NAME, "name=%s", atrname); @@ -329,10 +253,8 @@ X509_ATTRIBUTE *X509_ATTRIBUTE_create_by_txt(X509_ATTRIBUTE **attr, int X509_ATTRIBUTE_set1_object(X509_ATTRIBUTE *attr, const ASN1_OBJECT *obj) { - if (attr == NULL || obj == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); + if ((attr == NULL) || (obj == NULL)) return 0; - } ASN1_OBJECT_free(attr->object); attr->object = OBJ_dup(obj); return attr->object != NULL; @@ -344,25 +266,21 @@ int X509_ATTRIBUTE_set1_data(X509_ATTRIBUTE *attr, int attrtype, ASN1_TYPE *ttmp = NULL; ASN1_STRING *stmp = NULL; int atype = 0; - - if (attr == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); + if (!attr) return 0; - } - if ((attrtype & MBSTRING_FLAG) != 0) { + if (attrtype & MBSTRING_FLAG) { stmp = ASN1_STRING_set_by_NID(NULL, data, len, attrtype, OBJ_obj2nid(attr->object)); - if (stmp == NULL) { + if (!stmp) { ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); return 0; } atype = stmp->type; } else if (len != -1) { - if ((stmp = ASN1_STRING_type_new(attrtype)) == NULL - || !ASN1_STRING_set(stmp, data, len)) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + if ((stmp = ASN1_STRING_type_new(attrtype)) == NULL) + goto err; + if (!ASN1_STRING_set(stmp, data, len)) goto err; - } atype = attrtype; } /* @@ -374,25 +292,20 @@ int X509_ATTRIBUTE_set1_data(X509_ATTRIBUTE *attr, int attrtype, ASN1_STRING_free(stmp); return 1; } - if ((ttmp = ASN1_TYPE_new()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + if ((ttmp = ASN1_TYPE_new()) == NULL) goto err; - } - if (len == -1 && (attrtype & MBSTRING_FLAG) == 0) { - if (!ASN1_TYPE_set1(ttmp, attrtype, data)) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + if ((len == -1) && !(attrtype & MBSTRING_FLAG)) { + if (!ASN1_TYPE_set1(ttmp, attrtype, data)) goto err; - } } else { ASN1_TYPE_set(ttmp, atype, stmp); stmp = NULL; } - if (!sk_ASN1_TYPE_push(attr->set, ttmp)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + if (!sk_ASN1_TYPE_push(attr->set, ttmp)) goto err; - } return 1; err: + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); ASN1_TYPE_free(ttmp); ASN1_STRING_free(stmp); return 0; @@ -407,19 +320,17 @@ int X509_ATTRIBUTE_count(const X509_ATTRIBUTE *attr) ASN1_OBJECT *X509_ATTRIBUTE_get0_object(X509_ATTRIBUTE *attr) { - if (attr == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); + if (attr == NULL) return NULL; - } return attr->object; } void *X509_ATTRIBUTE_get0_data(X509_ATTRIBUTE *attr, int idx, int atrtype, void *data) { - ASN1_TYPE *ttmp = X509_ATTRIBUTE_get0_type(attr, idx); - - if (ttmp == NULL) + ASN1_TYPE *ttmp; + ttmp = X509_ATTRIBUTE_get0_type(attr, idx); + if (!ttmp) return NULL; if (atrtype == V_ASN1_BOOLEAN || atrtype == V_ASN1_NULL @@ -432,9 +343,7 @@ void *X509_ATTRIBUTE_get0_data(X509_ATTRIBUTE *attr, int idx, ASN1_TYPE *X509_ATTRIBUTE_get0_type(X509_ATTRIBUTE *attr, int idx) { - if (attr == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); + if (attr == NULL) return NULL; - } return sk_ASN1_TYPE_value(attr->set, idx); } diff --git a/openssl/src/crypto/x509/x509_cmp.c b/openssl/src/crypto/x509/x509_cmp.c index 7094280d4..ff12d237d 100644 --- a/openssl/src/crypto/x509/x509_cmp.c +++ b/openssl/src/crypto/x509/x509_cmp.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -39,7 +39,7 @@ unsigned long X509_issuer_and_serial_hash(X509 *a) unsigned long ret = 0; EVP_MD_CTX *ctx = EVP_MD_CTX_new(); unsigned char md[16]; - char *f = NULL; + char *f; EVP_MD *digest = NULL; if (ctx == NULL) @@ -55,6 +55,7 @@ unsigned long X509_issuer_and_serial_hash(X509 *a) goto err; if (!EVP_DigestUpdate(ctx, (unsigned char *)f, strlen(f))) goto err; + OPENSSL_free(f); if (!EVP_DigestUpdate (ctx, (unsigned char *)a->cert_info.serialNumber.data, (unsigned long)a->cert_info.serialNumber.length)) @@ -65,7 +66,6 @@ unsigned long X509_issuer_and_serial_hash(X509 *a) ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L) ) & 0xffffffffL; err: - OPENSSL_free(f); EVP_MD_free(digest); EVP_MD_CTX_free(ctx); return ret; @@ -184,7 +184,7 @@ int X509_cmp(const X509 *a, const X509 *b) int ossl_x509_add_cert_new(STACK_OF(X509) **p_sk, X509 *cert, int flags) { if (*p_sk == NULL && (*p_sk = sk_X509_new_null()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; } return X509_add_cert(*p_sk, cert, flags); @@ -216,7 +216,7 @@ int X509_add_cert(STACK_OF(X509) *sk, X509 *cert, int flags) } if (!sk_X509_insert(sk, cert, (flags & X509_ADD_FLAG_PREPEND) != 0 ? 0 : -1)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; } if ((flags & X509_ADD_FLAG_UP_REF) != 0) @@ -277,11 +277,11 @@ int X509_NAME_cmp(const X509_NAME *a, const X509_NAME *b) if (ret == 0 && a->canon_enclen == 0) return 0; - if (ret == 0) { - if (a->canon_enc == NULL || b->canon_enc == NULL) - return -2; + if (a->canon_enc == NULL || b->canon_enc == NULL) + return -2; + + if (ret == 0) ret = memcmp(a->canon_enc, b->canon_enc, a->canon_enclen); - } return ret < 0 ? -1 : ret > 0; } @@ -292,13 +292,12 @@ unsigned long X509_NAME_hash_ex(const X509_NAME *x, OSSL_LIB_CTX *libctx, unsigned long ret = 0; unsigned char md[SHA_DIGEST_LENGTH]; EVP_MD *sha1 = EVP_MD_fetch(libctx, "SHA1", propq); - int i2d_ret; /* Make sure X509_NAME structure contains valid cached encoding */ - i2d_ret = i2d_X509_NAME(x, NULL); + i2d_X509_NAME(x, NULL); if (ok != NULL) *ok = 0; - if (i2d_ret >= 0 && sha1 != NULL + if (sha1 != NULL && EVP_Digest(x->canon_enc, x->canon_enclen, md, NULL, sha1, NULL)) { ret = (((unsigned long)md[0]) | ((unsigned long)md[1] << 8L) | ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L) @@ -326,9 +325,7 @@ unsigned long X509_NAME_hash_old(const X509_NAME *x) goto end; /* Make sure X509_NAME structure contains valid cached encoding */ - if (i2d_X509_NAME(x, NULL) < 0) - goto end; - + i2d_X509_NAME(x, NULL); if (EVP_DigestInit_ex(md_ctx, md5, NULL) && EVP_DigestUpdate(md_ctx, x->bytes->data, x->bytes->length) && EVP_DigestFinal_ex(md_ctx, md, NULL)) @@ -392,38 +389,30 @@ EVP_PKEY *X509_get_pubkey(X509 *x) return X509_PUBKEY_get(x->cert_info.key); } -int X509_check_private_key(const X509 *cert, const EVP_PKEY *pkey) +int X509_check_private_key(const X509 *x, const EVP_PKEY *k) { - const EVP_PKEY *xk = X509_get0_pubkey(cert); + const EVP_PKEY *xk; + int ret; + xk = X509_get0_pubkey(x); if (xk == NULL) { ERR_raise(ERR_LIB_X509, X509_R_UNABLE_TO_GET_CERTS_PUBLIC_KEY); return 0; } - return ossl_x509_check_private_key(xk, pkey); -} -int ossl_x509_check_private_key(const EVP_PKEY *x, const EVP_PKEY *pkey) -{ - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - switch (EVP_PKEY_eq(x, pkey)) { - case 1: - return 1; + switch (ret = EVP_PKEY_eq(xk, k)) { case 0: ERR_raise(ERR_LIB_X509, X509_R_KEY_VALUES_MISMATCH); - return 0; + break; case -1: ERR_raise(ERR_LIB_X509, X509_R_KEY_TYPE_MISMATCH); - return 0; + break; case -2: ERR_raise(ERR_LIB_X509, X509_R_UNKNOWN_KEY_TYPE); - /* fall thru */ - default: - return 0; + break; } + + return ret > 0; } /* @@ -594,3 +583,251 @@ STACK_OF(X509) *X509_chain_up_ref(STACK_OF(X509) *chain) sk_X509_free(ret); return NULL; } + +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +int DC_check_private_key(DELEGATED_CREDENTIAL *dc, EVP_PKEY *pkey) +{ + EVP_PKEY *pub_key; + int ret; + + pub_key = dc->pkey; + + if (pub_key) + ret = EVP_PKEY_eq(pub_key, pkey); + else + ret = -2; + + switch (ret) { + case 1: + break; + case 0: + X509err(X509_F_DC_CHECK_PRIVATE_KEY, X509_R_KEY_VALUES_MISMATCH); + break; + case -1: + X509err(X509_F_DC_CHECK_PRIVATE_KEY, X509_R_KEY_TYPE_MISMATCH); + break; + case -2: + X509err(X509_F_DC_CHECK_PRIVATE_KEY, X509_R_UNKNOWN_KEY_TYPE); + } + if (ret > 0) + return 1; + return 0; +} + +int DC_check_valid(X509 *parent_cert, DELEGATED_CREDENTIAL *dc) +{ + /* + * check if dc time expire + */ + if (!DC_check_time_valid(parent_cert, dc)) + return 0; + /* + * check dc parent_cert has DelegationUsage extension. + * check dc parent_cert has the digitalSignature KeyUsage + * see https://tools.ietf.org/html/draft-ietf-tls-subcerts-07#section-4.2 + */ + if (!DC_check_parent_cert_valid(parent_cert)) + return 0; + + return 1; +} + +int DC_check_time_valid(X509 *parent_cert, DELEGATED_CREDENTIAL *dc) +{ + ASN1_TIME *time; + struct tm tm; + int ret = 0; + + time = ASN1_STRING_dup(X509_get0_notBefore(parent_cert)); + if (time == NULL) + goto err; + if (!ASN1_TIME_to_tm(time, &tm)) + goto err; + if (ASN1_TIME_adj(time, mktime(&tm), 0, DC_get_valid_time(dc)) == NULL) + goto err; + if (X509_cmp_time(time, NULL) <= 0) + goto err; + + ret = 1; +err: + ASN1_STRING_clear_free(time); + return ret; +} + +int DC_check_parent_cert_valid(X509 *parent_cert) +{ + const STACK_OF(X509_EXTENSION) *exts; + int i; + + if ((X509_get_key_usage(parent_cert) & X509v3_KU_DIGITAL_SIGNATURE) == 0) + return 0; + + exts = X509_get0_extensions(parent_cert); + for (i = 0; i < sk_X509_EXTENSION_num(exts); i++) { + ASN1_OBJECT *obj; + X509_EXTENSION *ex; + + ex = sk_X509_EXTENSION_value(exts, i); + obj = X509_EXTENSION_get_object(ex); + + if (OBJ_obj2nid(obj) == NID_delegation_usage) + return X509_EXTENSION_get_critical(ex) == 1 ? 0 : 1; + } + + return 0; +} + +unsigned long DC_get_valid_time(DELEGATED_CREDENTIAL *dc) +{ + return dc->valid_time; +} + +unsigned int DC_get_expected_cert_verify_algorithm(DELEGATED_CREDENTIAL *dc) +{ + return dc->expected_cert_verify_algorithm; +} + +size_t DC_get_dc_publickey_raw_len(DELEGATED_CREDENTIAL *dc) +{ + return dc->dc_publickey_raw_len; +} + +unsigned char *DC_get0_dc_publickey_raw(DELEGATED_CREDENTIAL *dc) +{ + return dc->dc_publickey_raw; +} + +unsigned int DC_get_signature_sign_algorithm(DELEGATED_CREDENTIAL *dc) +{ + return dc->signature_sign_algorithm; +} + +size_t DC_get_dc_signature_len(DELEGATED_CREDENTIAL *dc) +{ + return dc->dc_signature_len; +} + +unsigned char *DC_get0_dc_signature(DELEGATED_CREDENTIAL *dc) +{ + return dc->dc_signature; +} + +EVP_PKEY *DC_get0_publickey(DELEGATED_CREDENTIAL *dc) +{ + return dc->pkey; +} + +unsigned char *DC_get0_raw_byte(DELEGATED_CREDENTIAL *dc) +{ + return dc->raw_byte; +} + +size_t DC_get_raw_byte_len(DELEGATED_CREDENTIAL *dc) +{ + return dc->raw_byte_len; +} + +int DC_set_valid_time(DELEGATED_CREDENTIAL *dc, unsigned long valid_time) +{ + if (dc == NULL) + return 0; + dc->valid_time = valid_time; + return 1; +} + +int DC_set_expected_cert_verify_algorithm(DELEGATED_CREDENTIAL *dc, unsigned int alg) +{ + if (dc == NULL) + return 0; + dc->expected_cert_verify_algorithm = alg; + return 1; +} + +int DC_set_dc_publickey_len(DELEGATED_CREDENTIAL *dc, size_t len) +{ + if (dc == NULL) + return 0; + dc->dc_publickey_raw_len = len; + return 1; +} + +int DC_set0_dc_publickey(DELEGATED_CREDENTIAL *dc, unsigned char *pub_key) +{ + if (dc == NULL) + return 0; + dc->dc_publickey_raw = pub_key; + return 1; +} + +int DC_set_signature_sign_algorithm(DELEGATED_CREDENTIAL *dc, unsigned int alg) +{ + if (dc == NULL) + return 0; + dc->signature_sign_algorithm = alg; + return 1; +} + +int DC_set_dc_signature_len(DELEGATED_CREDENTIAL *dc, size_t len) +{ + if (dc == NULL) + return 0; + dc->dc_signature_len = len; + return 1; +} + +int DC_set0_dc_signature(DELEGATED_CREDENTIAL *dc, unsigned char *sig) +{ + if (dc == NULL) + return 0; + dc->dc_signature = sig; + return 1; +} + +int DC_set0_publickey(DELEGATED_CREDENTIAL *dc, EVP_PKEY *pkey) +{ + if (dc == NULL) + return 0; + dc->pkey = pkey; + return 1; +} + +int DC_set0_raw_byte(DELEGATED_CREDENTIAL *dc, unsigned char *byte, size_t len) +{ + if (dc == NULL) + return 0; + + if (dc->raw_byte && dc->raw_byte != byte) + OPENSSL_free(dc->raw_byte); + + dc->raw_byte = byte; + dc->raw_byte_len = len; + + return 1; +} + +int DC_set1_raw_byte(DELEGATED_CREDENTIAL *dc, const unsigned char *byte, + size_t len) +{ + unsigned char *raw_byte = NULL; + + if (dc == NULL || byte == NULL || len <= 0) + return 0; + + if (dc->raw_byte == byte) { + dc->raw_byte_len = len; + return 1; + } + + raw_byte = OPENSSL_malloc(len); + if (raw_byte == NULL) + return 0; + + if (dc->raw_byte) + OPENSSL_free(dc->raw_byte); + + memcpy(raw_byte, byte, len); + dc->raw_byte = raw_byte; + dc->raw_byte_len = len; + return 1; +} +#endif diff --git a/openssl/src/crypto/x509/x509_d2.c b/openssl/src/crypto/x509/x509_d2.c index 7838b703d..4c2bc4def 100644 --- a/openssl/src/crypto/x509/x509_d2.c +++ b/openssl/src/crypto/x509/x509_d2.c @@ -50,7 +50,7 @@ int X509_STORE_load_file_ex(X509_STORE *ctx, const char *file, if (file == NULL || (lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_file())) == NULL || X509_LOOKUP_load_file_ex(lookup, file, X509_FILETYPE_PEM, libctx, - propq) <= 0) + propq) == 0) return 0; return 1; @@ -67,7 +67,7 @@ int X509_STORE_load_path(X509_STORE *ctx, const char *path) if (path == NULL || (lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_hash_dir())) == NULL - || X509_LOOKUP_add_dir(lookup, path, X509_FILETYPE_PEM) <= 0) + || X509_LOOKUP_add_dir(lookup, path, X509_FILETYPE_PEM) == 0) return 0; return 1; diff --git a/openssl/src/crypto/x509/x509_def.c b/openssl/src/crypto/x509/x509_def.c index 2851fbcd9..b8bdcb484 100644 --- a/openssl/src/crypto/x509/x509_def.c +++ b/openssl/src/crypto/x509/x509_def.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/crypto/x509/x509_err.c b/openssl/src/crypto/x509/x509_err.c index 226e45a73..a933aeef3 100644 --- a/openssl/src/crypto/x509/x509_err.c +++ b/openssl/src/crypto/x509/x509_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,7 +20,7 @@ static const ERR_STRING_DATA X509_str_reasons[] = { {ERR_PACK(ERR_LIB_X509, 0, X509_R_BAD_X509_FILETYPE), "bad x509 filetype"}, {ERR_PACK(ERR_LIB_X509, 0, X509_R_BASE64_DECODE_ERROR), "base64 decode error"}, - {ERR_PACK(ERR_LIB_X509, 0, X509_R_CANT_CHECK_DH_KEY), "can't check dh key"}, + {ERR_PACK(ERR_LIB_X509, 0, X509_R_CANT_CHECK_DH_KEY), "cant check dh key"}, {ERR_PACK(ERR_LIB_X509, 0, X509_R_CERTIFICATE_VERIFICATION_FAILED), "certificate verification failed"}, {ERR_PACK(ERR_LIB_X509, 0, X509_R_CERT_ALREADY_IN_HASH_TABLE), @@ -28,8 +28,6 @@ static const ERR_STRING_DATA X509_str_reasons[] = { {ERR_PACK(ERR_LIB_X509, 0, X509_R_CRL_ALREADY_DELTA), "crl already delta"}, {ERR_PACK(ERR_LIB_X509, 0, X509_R_CRL_VERIFY_FAILURE), "crl verify failure"}, - {ERR_PACK(ERR_LIB_X509, 0, X509_R_DUPLICATE_ATTRIBUTE), - "duplicate attribute"}, {ERR_PACK(ERR_LIB_X509, 0, X509_R_ERROR_GETTING_MD_BY_NID), "error getting md by nid"}, {ERR_PACK(ERR_LIB_X509, 0, X509_R_ERROR_USING_SIGINF_SET), diff --git a/openssl/src/crypto/x509/x509_lu.c b/openssl/src/crypto/x509/x509_lu.c index e7fdf3d6a..c5ace03cf 100644 --- a/openssl/src/crypto/x509/x509_lu.c +++ b/openssl/src/crypto/x509/x509_lu.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,8 +19,10 @@ X509_LOOKUP *X509_LOOKUP_new(X509_LOOKUP_METHOD *method) { X509_LOOKUP *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; + } ret->method = method; if (method->new_item != NULL && method->new_item(ret) == 0) { @@ -39,19 +41,14 @@ void X509_LOOKUP_free(X509_LOOKUP *ctx) OPENSSL_free(ctx); } -int X509_STORE_lock(X509_STORE *xs) -{ - return CRYPTO_THREAD_write_lock(xs->lock); -} - -static int x509_store_read_lock(X509_STORE *xs) +int X509_STORE_lock(X509_STORE *s) { - return CRYPTO_THREAD_read_lock(xs->lock); + return CRYPTO_THREAD_write_lock(s->lock); } -int X509_STORE_unlock(X509_STORE *xs) +int X509_STORE_unlock(X509_STORE *s) { - return CRYPTO_THREAD_unlock(xs->lock); + return CRYPTO_THREAD_unlock(s->lock); } int X509_LOOKUP_init(X509_LOOKUP *ctx) @@ -157,6 +154,7 @@ X509_STORE *X509_LOOKUP_get_store(const X509_LOOKUP *ctx) return ctx->store_ctx; } + static int x509_object_cmp(const X509_OBJECT *const *a, const X509_OBJECT *const *b) { @@ -183,95 +181,147 @@ X509_STORE *X509_STORE_new(void) { X509_STORE *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; + } if ((ret->objs = sk_X509_OBJECT_new(x509_object_cmp)) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } ret->cache = 1; if ((ret->get_cert_methods = sk_X509_LOOKUP_new_null()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } if ((ret->param = X509_VERIFY_PARAM_new()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_X509_STORE, ret, &ret->ex_data)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } ret->lock = CRYPTO_THREAD_lock_new(); if (ret->lock == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } - - if (!CRYPTO_NEW_REF(&ret->references, 1)) - goto err; + ret->references = 1; return ret; err: X509_VERIFY_PARAM_free(ret->param); sk_X509_OBJECT_free(ret->objs); sk_X509_LOOKUP_free(ret->get_cert_methods); - CRYPTO_THREAD_lock_free(ret->lock); OPENSSL_free(ret); return NULL; } -void X509_STORE_free(X509_STORE *xs) +void X509_STORE_free(X509_STORE *vfy) { int i; STACK_OF(X509_LOOKUP) *sk; X509_LOOKUP *lu; - if (xs == NULL) + if (vfy == NULL) return; - CRYPTO_DOWN_REF(&xs->references, &i); - REF_PRINT_COUNT("X509_STORE", xs); + CRYPTO_DOWN_REF(&vfy->references, &i, vfy->lock); + REF_PRINT_COUNT("X509_STORE", vfy); if (i > 0) return; REF_ASSERT_ISNT(i < 0); - sk = xs->get_cert_methods; + sk = vfy->get_cert_methods; for (i = 0; i < sk_X509_LOOKUP_num(sk); i++) { lu = sk_X509_LOOKUP_value(sk, i); X509_LOOKUP_shutdown(lu); X509_LOOKUP_free(lu); } sk_X509_LOOKUP_free(sk); - sk_X509_OBJECT_pop_free(xs->objs, X509_OBJECT_free); + sk_X509_OBJECT_pop_free(vfy->objs, X509_OBJECT_free); - CRYPTO_free_ex_data(CRYPTO_EX_INDEX_X509_STORE, xs, &xs->ex_data); - X509_VERIFY_PARAM_free(xs->param); - CRYPTO_THREAD_lock_free(xs->lock); - CRYPTO_FREE_REF(&xs->references); - OPENSSL_free(xs); + CRYPTO_free_ex_data(CRYPTO_EX_INDEX_X509_STORE, vfy, &vfy->ex_data); + X509_VERIFY_PARAM_free(vfy->param); + CRYPTO_THREAD_lock_free(vfy->lock); + OPENSSL_free(vfy); } -int X509_STORE_up_ref(X509_STORE *xs) +int X509_STORE_up_ref(X509_STORE *vfy) { int i; - if (CRYPTO_UP_REF(&xs->references, &i) <= 0) + if (CRYPTO_UP_REF(&vfy->references, &i, vfy->lock) <= 0) return 0; - REF_PRINT_COUNT("X509_STORE", xs); + REF_PRINT_COUNT("X509_STORE", vfy); REF_ASSERT_ISNT(i < 2); - return i > 1 ? 1 : 0; + return ((i > 1) ? 1 : 0); +} + +int X509_STORE_copy(X509_STORE *dest, const X509_STORE *src) +{ + X509_OBJECT *obj; + X509_LOOKUP *lu; + int i, num; + + if (dest == NULL || src == NULL || dest == src) + return 0; + + if (src->get_cert_methods) { + num = sk_X509_LOOKUP_num(src->get_cert_methods); + for (i = 0; i < num; i++) { + lu = sk_X509_LOOKUP_value(src->get_cert_methods, i); + if (!X509_STORE_add_lookup(dest, lu->method)) + return 0; + } + } + + if (src->objs) { + num = sk_X509_OBJECT_num(src->objs); + for (i = 0; i < num; i++) { + obj = sk_X509_OBJECT_value(src->objs, i); + if (obj->type == X509_LU_X509) { + X509_STORE_add_cert(dest, obj->data.x509); + } else if (obj->type == X509_LU_CRL) { + X509_STORE_add_crl(dest, obj->data.crl); + } else { + /* abort(); */ + } + } + } + + if (src->param && !X509_VERIFY_PARAM_copy(dest->param, src->param)) + return 0; + + dest->verify = src->verify; + dest->verify_cb = src->verify_cb; + dest->get_issuer = src->get_issuer; + dest->check_issued = src->check_issued; + dest->check_revocation = src->check_revocation; + dest->get_crl = src->get_crl; + dest->check_crl = src->check_crl; + dest->cert_crl = src->cert_crl; + dest->check_policy = src->check_policy; + dest->lookup_certs = src->lookup_certs; + dest->lookup_crls = src->lookup_crls; + dest->cleanup = src->cleanup; + + if (!CRYPTO_dup_ex_data(CRYPTO_EX_INDEX_X509_STORE, &dest->ex_data, &src->ex_data)) + return 0; + + return 1; } -X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *xs, X509_LOOKUP_METHOD *m) +X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *v, X509_LOOKUP_METHOD *m) { int i; STACK_OF(X509_LOOKUP) *sk; X509_LOOKUP *lu; - sk = xs->get_cert_methods; + sk = v->get_cert_methods; for (i = 0; i < sk_X509_LOOKUP_num(sk); i++) { lu = sk_X509_LOOKUP_value(sk, i); if (m == lu->method) { @@ -281,21 +331,20 @@ X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *xs, X509_LOOKUP_METHOD *m) /* a new one */ lu = X509_LOOKUP_new(m); if (lu == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; } - lu->store_ctx = xs; - if (sk_X509_LOOKUP_push(xs->get_cert_methods, lu)) + lu->store_ctx = v; + if (sk_X509_LOOKUP_push(v->get_cert_methods, lu)) return lu; - /* sk_X509_LOOKUP_push() failed */ - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + /* malloc failed */ + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); X509_LOOKUP_free(lu); return NULL; } -/* Also fill the cache (ctx->store->objs) with all matching certificates. */ -X509_OBJECT *X509_STORE_CTX_get_obj_by_subject(X509_STORE_CTX *ctx, +X509_OBJECT *X509_STORE_CTX_get_obj_by_subject(X509_STORE_CTX *vs, X509_LOOKUP_TYPE type, const X509_NAME *name) { @@ -303,26 +352,19 @@ X509_OBJECT *X509_STORE_CTX_get_obj_by_subject(X509_STORE_CTX *ctx, if (ret == NULL) return NULL; - if (!X509_STORE_CTX_get_by_subject(ctx, type, name, ret)) { + if (!X509_STORE_CTX_get_by_subject(vs, type, name, ret)) { X509_OBJECT_free(ret); return NULL; } return ret; } -/* - * May be called with |ret| == NULL just for the side effect of - * caching all certs matching the given subject DN in |ctx->store->objs|. - * Returns 1 if successful, - * 0 if not found or X509_LOOKUP_by_subject_ex() returns an error, - * -1 on failure - */ -static int ossl_x509_store_ctx_get_by_subject(const X509_STORE_CTX *ctx, - X509_LOOKUP_TYPE type, - const X509_NAME *name, - X509_OBJECT *ret) +/* Also fill the cache with all matching certificates */ +int X509_STORE_CTX_get_by_subject(const X509_STORE_CTX *vs, + X509_LOOKUP_TYPE type, + const X509_NAME *name, X509_OBJECT *ret) { - X509_STORE *store = ctx->store; + X509_STORE *store = vs->store; X509_LOOKUP *lu; X509_OBJECT stmp, *tmp; int i, j; @@ -333,33 +375,18 @@ static int ossl_x509_store_ctx_get_by_subject(const X509_STORE_CTX *ctx, stmp.type = X509_LU_NONE; stmp.data.ptr = NULL; - if (!x509_store_read_lock(store)) + if (!X509_STORE_lock(store)) return 0; - /* Should already be sorted...but just in case */ - if (!sk_X509_OBJECT_is_sorted(store->objs)) { - X509_STORE_unlock(store); - /* Take a write lock instead of a read lock */ - if (!X509_STORE_lock(store)) - return 0; - /* - * Another thread might have sorted it in the meantime. But if so, - * sk_X509_OBJECT_sort() exits early. - */ - sk_X509_OBJECT_sort(store->objs); - } + tmp = X509_OBJECT_retrieve_by_subject(store->objs, type, name); X509_STORE_unlock(store); if (tmp == NULL || type == X509_LU_CRL) { for (i = 0; i < sk_X509_LOOKUP_num(store->get_cert_methods); i++) { lu = sk_X509_LOOKUP_value(store->get_cert_methods, i); - if (lu->skip) - continue; - if (lu->method == NULL) - return -1; - j = X509_LOOKUP_by_subject_ex(lu, type, name, &stmp, - ctx->libctx, ctx->propq); - if (j != 0) { /* non-zero value is considered success here */ + j = X509_LOOKUP_by_subject_ex(lu, type, name, &stmp, vs->libctx, + vs->propq); + if (j) { tmp = &stmp; break; } @@ -367,24 +394,17 @@ static int ossl_x509_store_ctx_get_by_subject(const X509_STORE_CTX *ctx, if (tmp == NULL) return 0; } + if (!X509_OBJECT_up_ref_count(tmp)) - return -1; + return 0; ret->type = tmp->type; ret->data.ptr = tmp->data.ptr; - return 1; -} -/* Also fill the cache |ctx->store->objs| with all matching certificates. */ -int X509_STORE_CTX_get_by_subject(const X509_STORE_CTX *ctx, - X509_LOOKUP_TYPE type, - const X509_NAME *name, X509_OBJECT *ret) -{ - return ossl_x509_store_ctx_get_by_subject(ctx, type, name, ret) > 0; + return 1; } -static int x509_store_add(X509_STORE *store, void *x, int crl) -{ +static int x509_store_add(X509_STORE *store, void *x, int crl) { X509_OBJECT *obj; int ret = 0, added = 0; @@ -427,19 +447,19 @@ static int x509_store_add(X509_STORE *store, void *x, int crl) return ret; } -int X509_STORE_add_cert(X509_STORE *xs, X509 *x) +int X509_STORE_add_cert(X509_STORE *ctx, X509 *x) { - if (!x509_store_add(xs, x, 0)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); + if (!x509_store_add(ctx, x, 0)) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; } return 1; } -int X509_STORE_add_crl(X509_STORE *xs, X509_CRL *x) +int X509_STORE_add_crl(X509_STORE *ctx, X509_CRL *x) { - if (!x509_store_add(xs, x, 1)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); + if (!x509_store_add(ctx, x, 1)) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; } return 1; @@ -481,8 +501,10 @@ X509_OBJECT *X509_OBJECT_new(void) { X509_OBJECT *ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; + } ret->type = X509_LU_NONE; return ret; } @@ -531,13 +553,13 @@ void X509_OBJECT_free(X509_OBJECT *a) OPENSSL_free(a); } -/* Returns -1 if not found, but also on error */ static int x509_object_idx_cnt(STACK_OF(X509_OBJECT) *h, X509_LOOKUP_TYPE type, const X509_NAME *name, int *pnmatch) { X509_OBJECT stmp; X509 x509_s; X509_CRL crl_s; + int idx; stmp.type = type; switch (type) { @@ -550,67 +572,34 @@ static int x509_object_idx_cnt(STACK_OF(X509_OBJECT) *h, X509_LOOKUP_TYPE type, crl_s.crl.issuer = (X509_NAME *)name; /* won't modify it */ break; case X509_LU_NONE: - default: /* abort(); */ return -1; } - /* Assumes h is locked for read if applicable */ - return sk_X509_OBJECT_find_all(h, &stmp, pnmatch); + idx = sk_X509_OBJECT_find_all(h, &stmp, pnmatch); + return idx; } -/* Assumes h is locked for read if applicable */ int X509_OBJECT_idx_by_subject(STACK_OF(X509_OBJECT) *h, X509_LOOKUP_TYPE type, const X509_NAME *name) { return x509_object_idx_cnt(h, type, name, NULL); } -/* Assumes h is locked for read if applicable */ X509_OBJECT *X509_OBJECT_retrieve_by_subject(STACK_OF(X509_OBJECT) *h, X509_LOOKUP_TYPE type, const X509_NAME *name) { - int idx = X509_OBJECT_idx_by_subject(h, type, name); - + int idx; + idx = X509_OBJECT_idx_by_subject(h, type, name); if (idx == -1) return NULL; return sk_X509_OBJECT_value(h, idx); } -STACK_OF(X509_OBJECT) *X509_STORE_get0_objects(const X509_STORE *xs) +STACK_OF(X509_OBJECT) *X509_STORE_get0_objects(const X509_STORE *v) { - return xs->objs; -} - -static X509_OBJECT *x509_object_dup(const X509_OBJECT *obj) -{ - X509_OBJECT *ret = X509_OBJECT_new(); - if (ret == NULL) - return NULL; - - ret->type = obj->type; - ret->data = obj->data; - X509_OBJECT_up_ref_count(ret); - return ret; -} - -STACK_OF(X509_OBJECT) *X509_STORE_get1_objects(X509_STORE *store) -{ - STACK_OF(X509_OBJECT) *objs; - - if (store == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - - if (!x509_store_read_lock(store)) - return NULL; - - objs = sk_X509_OBJECT_deep_copy(store->objs, x509_object_dup, - X509_OBJECT_free); - X509_STORE_unlock(store); - return objs; + return v->objs; } STACK_OF(X509) *X509_STORE_get1_all_certs(X509_STORE *store) @@ -628,7 +617,6 @@ STACK_OF(X509) *X509_STORE_get1_all_certs(X509_STORE *store) if (!X509_STORE_lock(store)) goto out_free; - sk_X509_OBJECT_sort(store->objs); objs = X509_STORE_get0_objects(store); for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { X509 *cert = X509_OBJECT_get0_X509(sk_X509_OBJECT_value(objs, i)); @@ -643,11 +631,10 @@ STACK_OF(X509) *X509_STORE_get1_all_certs(X509_STORE *store) err: X509_STORE_unlock(store); out_free: - OSSL_STACK_OF_X509_free(sk); + sk_X509_pop_free(sk, X509_free); return NULL; } -/* Returns NULL on internal/fatal error, empty stack if not found */ STACK_OF(X509) *X509_STORE_CTX_get1_certs(X509_STORE_CTX *ctx, const X509_NAME *nm) { @@ -658,12 +645,11 @@ STACK_OF(X509) *X509_STORE_CTX_get1_certs(X509_STORE_CTX *ctx, X509_STORE *store = ctx->store; if (store == NULL) - return sk_X509_new_null(); + return NULL; if (!X509_STORE_lock(store)) return NULL; - sk_X509_OBJECT_sort(store->objs); idx = x509_object_idx_cnt(store->objs, X509_LU_X509, nm, &cnt); if (idx < 0) { /* @@ -673,46 +659,41 @@ STACK_OF(X509) *X509_STORE_CTX_get1_certs(X509_STORE_CTX *ctx, X509_OBJECT *xobj = X509_OBJECT_new(); X509_STORE_unlock(store); + if (xobj == NULL) return NULL; - i = ossl_x509_store_ctx_get_by_subject(ctx, X509_LU_X509, nm, xobj); - if (i <= 0) { + if (!X509_STORE_CTX_get_by_subject(ctx, X509_LU_X509, nm, xobj)) { X509_OBJECT_free(xobj); - return i < 0 ? NULL : sk_X509_new_null(); + return NULL; } X509_OBJECT_free(xobj); if (!X509_STORE_lock(store)) return NULL; - sk_X509_OBJECT_sort(store->objs); idx = x509_object_idx_cnt(store->objs, X509_LU_X509, nm, &cnt); if (idx < 0) { - sk = sk_X509_new_null(); - goto end; + X509_STORE_unlock(store); + return NULL; } } sk = sk_X509_new_null(); - if (sk == NULL) - goto end; for (i = 0; i < cnt; i++, idx++) { obj = sk_X509_OBJECT_value(store->objs, idx); x = obj->data.x509; if (!X509_add_cert(sk, x, X509_ADD_FLAG_UP_REF)) { X509_STORE_unlock(store); - OSSL_STACK_OF_X509_free(sk); + sk_X509_pop_free(sk, X509_free); return NULL; } } - end: X509_STORE_unlock(store); return sk; } -/* Returns NULL on internal/fatal error, empty stack if not found */ STACK_OF(X509_CRL) *X509_STORE_CTX_get1_crls(const X509_STORE_CTX *ctx, const X509_NAME *nm) { - int i = 1, idx, cnt; + int i, idx, cnt; STACK_OF(X509_CRL) *sk = sk_X509_CRL_new_null(); X509_CRL *x; X509_OBJECT *obj, *xobj = X509_OBJECT_new(); @@ -720,25 +701,23 @@ STACK_OF(X509_CRL) *X509_STORE_CTX_get1_crls(const X509_STORE_CTX *ctx, /* Always do lookup to possibly add new CRLs to cache */ if (sk == NULL - || xobj == NULL - || (i = ossl_x509_store_ctx_get_by_subject(ctx, X509_LU_CRL, - nm, xobj)) < 0) { + || xobj == NULL + || store == NULL + || !X509_STORE_CTX_get_by_subject(ctx, X509_LU_CRL, nm, xobj)) { X509_OBJECT_free(xobj); sk_X509_CRL_free(sk); return NULL; } X509_OBJECT_free(xobj); - if (i == 0) - return sk; if (!X509_STORE_lock(store)) { sk_X509_CRL_free(sk); return NULL; } - sk_X509_OBJECT_sort(store->objs); idx = x509_object_idx_cnt(store->objs, X509_LU_CRL, nm, &cnt); if (idx < 0) { X509_STORE_unlock(store); - return sk; + sk_X509_CRL_free(sk); + return NULL; } for (i = 0; i < cnt; i++, idx++) { @@ -782,9 +761,8 @@ X509_OBJECT *X509_OBJECT_retrieve_match(STACK_OF(X509_OBJECT) *h, } else if (x->type == X509_LU_CRL) { if (X509_CRL_match(obj->data.crl, x->data.crl) == 0) return obj; - } else { + } else return obj; - } } return NULL; } @@ -809,10 +787,10 @@ int X509_STORE_CTX_get1_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *x) return -1; *issuer = NULL; xn = X509_get_issuer_name(x); - ok = ossl_x509_store_ctx_get_by_subject(ctx, X509_LU_X509, xn, obj); + ok = X509_STORE_CTX_get_by_subject(ctx, X509_LU_X509, xn, obj); if (ok != 1) { X509_OBJECT_free(obj); - return ok; + return 0; } /* If certificate matches and is currently valid all OK */ if (ctx->check_issued(ctx, x, obj->data.x509)) { @@ -839,7 +817,6 @@ int X509_STORE_CTX_get1_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *x) if (!X509_STORE_lock(store)) return 0; - sk_X509_OBJECT_sort(store->objs); idx = x509_object_idx_cnt(store->objs, X509_LU_X509, xn, &nmatch); if (idx != -1) { /* should be true as we've had at least one match */ /* Look through all matching certs for suitable issuer */ @@ -874,176 +851,176 @@ int X509_STORE_CTX_get1_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *x) return ret; } -int X509_STORE_set_flags(X509_STORE *xs, unsigned long flags) +int X509_STORE_set_flags(X509_STORE *ctx, unsigned long flags) { - return X509_VERIFY_PARAM_set_flags(xs->param, flags); + return X509_VERIFY_PARAM_set_flags(ctx->param, flags); } -int X509_STORE_set_depth(X509_STORE *xs, int depth) +int X509_STORE_set_depth(X509_STORE *ctx, int depth) { - X509_VERIFY_PARAM_set_depth(xs->param, depth); + X509_VERIFY_PARAM_set_depth(ctx->param, depth); return 1; } -int X509_STORE_set_purpose(X509_STORE *xs, int purpose) +int X509_STORE_set_purpose(X509_STORE *ctx, int purpose) { - return X509_VERIFY_PARAM_set_purpose(xs->param, purpose); + return X509_VERIFY_PARAM_set_purpose(ctx->param, purpose); } -int X509_STORE_set_trust(X509_STORE *xs, int trust) +int X509_STORE_set_trust(X509_STORE *ctx, int trust) { - return X509_VERIFY_PARAM_set_trust(xs->param, trust); + return X509_VERIFY_PARAM_set_trust(ctx->param, trust); } -int X509_STORE_set1_param(X509_STORE *xs, const X509_VERIFY_PARAM *param) +int X509_STORE_set1_param(X509_STORE *ctx, const X509_VERIFY_PARAM *param) { - return X509_VERIFY_PARAM_set1(xs->param, param); + return X509_VERIFY_PARAM_set1(ctx->param, param); } -X509_VERIFY_PARAM *X509_STORE_get0_param(const X509_STORE *xs) +X509_VERIFY_PARAM *X509_STORE_get0_param(const X509_STORE *ctx) { - return xs->param; + return ctx->param; } -void X509_STORE_set_verify(X509_STORE *xs, X509_STORE_CTX_verify_fn verify) +void X509_STORE_set_verify(X509_STORE *ctx, X509_STORE_CTX_verify_fn verify) { - xs->verify = verify; + ctx->verify = verify; } -X509_STORE_CTX_verify_fn X509_STORE_get_verify(const X509_STORE *xs) +X509_STORE_CTX_verify_fn X509_STORE_get_verify(const X509_STORE *ctx) { - return xs->verify; + return ctx->verify; } -void X509_STORE_set_verify_cb(X509_STORE *xs, +void X509_STORE_set_verify_cb(X509_STORE *ctx, X509_STORE_CTX_verify_cb verify_cb) { - xs->verify_cb = verify_cb; + ctx->verify_cb = verify_cb; } -X509_STORE_CTX_verify_cb X509_STORE_get_verify_cb(const X509_STORE *xs) +X509_STORE_CTX_verify_cb X509_STORE_get_verify_cb(const X509_STORE *ctx) { - return xs->verify_cb; + return ctx->verify_cb; } -void X509_STORE_set_get_issuer(X509_STORE *xs, +void X509_STORE_set_get_issuer(X509_STORE *ctx, X509_STORE_CTX_get_issuer_fn get_issuer) { - xs->get_issuer = get_issuer; + ctx->get_issuer = get_issuer; } -X509_STORE_CTX_get_issuer_fn X509_STORE_get_get_issuer(const X509_STORE *xs) +X509_STORE_CTX_get_issuer_fn X509_STORE_get_get_issuer(const X509_STORE *ctx) { - return xs->get_issuer; + return ctx->get_issuer; } -void X509_STORE_set_check_issued(X509_STORE *xs, +void X509_STORE_set_check_issued(X509_STORE *ctx, X509_STORE_CTX_check_issued_fn check_issued) { - xs->check_issued = check_issued; + ctx->check_issued = check_issued; } -X509_STORE_CTX_check_issued_fn X509_STORE_get_check_issued(const X509_STORE *xs) +X509_STORE_CTX_check_issued_fn X509_STORE_get_check_issued(const X509_STORE *ctx) { - return xs->check_issued; + return ctx->check_issued; } -void X509_STORE_set_check_revocation(X509_STORE *xs, - X509_STORE_CTX_check_revocation_fn cb) +void X509_STORE_set_check_revocation(X509_STORE *ctx, + X509_STORE_CTX_check_revocation_fn check_revocation) { - xs->check_revocation = cb; + ctx->check_revocation = check_revocation; } -X509_STORE_CTX_check_revocation_fn X509_STORE_get_check_revocation(const X509_STORE *xs) +X509_STORE_CTX_check_revocation_fn X509_STORE_get_check_revocation(const X509_STORE *ctx) { - return xs->check_revocation; + return ctx->check_revocation; } -void X509_STORE_set_get_crl(X509_STORE *xs, +void X509_STORE_set_get_crl(X509_STORE *ctx, X509_STORE_CTX_get_crl_fn get_crl) { - xs->get_crl = get_crl; + ctx->get_crl = get_crl; } -X509_STORE_CTX_get_crl_fn X509_STORE_get_get_crl(const X509_STORE *xs) +X509_STORE_CTX_get_crl_fn X509_STORE_get_get_crl(const X509_STORE *ctx) { - return xs->get_crl; + return ctx->get_crl; } -void X509_STORE_set_check_crl(X509_STORE *xs, +void X509_STORE_set_check_crl(X509_STORE *ctx, X509_STORE_CTX_check_crl_fn check_crl) { - xs->check_crl = check_crl; + ctx->check_crl = check_crl; } -X509_STORE_CTX_check_crl_fn X509_STORE_get_check_crl(const X509_STORE *xs) +X509_STORE_CTX_check_crl_fn X509_STORE_get_check_crl(const X509_STORE *ctx) { - return xs->check_crl; + return ctx->check_crl; } -void X509_STORE_set_cert_crl(X509_STORE *xs, +void X509_STORE_set_cert_crl(X509_STORE *ctx, X509_STORE_CTX_cert_crl_fn cert_crl) { - xs->cert_crl = cert_crl; + ctx->cert_crl = cert_crl; } -X509_STORE_CTX_cert_crl_fn X509_STORE_get_cert_crl(const X509_STORE *xs) +X509_STORE_CTX_cert_crl_fn X509_STORE_get_cert_crl(const X509_STORE *ctx) { - return xs->cert_crl; + return ctx->cert_crl; } -void X509_STORE_set_check_policy(X509_STORE *xs, +void X509_STORE_set_check_policy(X509_STORE *ctx, X509_STORE_CTX_check_policy_fn check_policy) { - xs->check_policy = check_policy; + ctx->check_policy = check_policy; } -X509_STORE_CTX_check_policy_fn X509_STORE_get_check_policy(const X509_STORE *xs) +X509_STORE_CTX_check_policy_fn X509_STORE_get_check_policy(const X509_STORE *ctx) { - return xs->check_policy; + return ctx->check_policy; } -void X509_STORE_set_lookup_certs(X509_STORE *xs, +void X509_STORE_set_lookup_certs(X509_STORE *ctx, X509_STORE_CTX_lookup_certs_fn lookup_certs) { - xs->lookup_certs = lookup_certs; + ctx->lookup_certs = lookup_certs; } -X509_STORE_CTX_lookup_certs_fn X509_STORE_get_lookup_certs(const X509_STORE *xs) +X509_STORE_CTX_lookup_certs_fn X509_STORE_get_lookup_certs(const X509_STORE *ctx) { - return xs->lookup_certs; + return ctx->lookup_certs; } -void X509_STORE_set_lookup_crls(X509_STORE *xs, +void X509_STORE_set_lookup_crls(X509_STORE *ctx, X509_STORE_CTX_lookup_crls_fn lookup_crls) { - xs->lookup_crls = lookup_crls; + ctx->lookup_crls = lookup_crls; } -X509_STORE_CTX_lookup_crls_fn X509_STORE_get_lookup_crls(const X509_STORE *xs) +X509_STORE_CTX_lookup_crls_fn X509_STORE_get_lookup_crls(const X509_STORE *ctx) { - return xs->lookup_crls; + return ctx->lookup_crls; } -void X509_STORE_set_cleanup(X509_STORE *xs, - X509_STORE_CTX_cleanup_fn cleanup) +void X509_STORE_set_cleanup(X509_STORE *ctx, + X509_STORE_CTX_cleanup_fn ctx_cleanup) { - xs->cleanup = cleanup; + ctx->cleanup = ctx_cleanup; } -X509_STORE_CTX_cleanup_fn X509_STORE_get_cleanup(const X509_STORE *xs) +X509_STORE_CTX_cleanup_fn X509_STORE_get_cleanup(const X509_STORE *ctx) { - return xs->cleanup; + return ctx->cleanup; } -int X509_STORE_set_ex_data(X509_STORE *xs, int idx, void *data) +int X509_STORE_set_ex_data(X509_STORE *ctx, int idx, void *data) { - return CRYPTO_set_ex_data(&xs->ex_data, idx, data); + return CRYPTO_set_ex_data(&ctx->ex_data, idx, data); } -void *X509_STORE_get_ex_data(const X509_STORE *xs, int idx) +void *X509_STORE_get_ex_data(const X509_STORE *ctx, int idx) { - return CRYPTO_get_ex_data(&xs->ex_data, idx); + return CRYPTO_get_ex_data(&ctx->ex_data, idx); } X509_STORE *X509_STORE_CTX_get0_store(const X509_STORE_CTX *ctx) diff --git a/openssl/src/crypto/x509/x509_meth.c b/openssl/src/crypto/x509/x509_meth.c index 305fe4c6d..a8eedd9b5 100644 --- a/openssl/src/crypto/x509/x509_meth.c +++ b/openssl/src/crypto/x509/x509_meth.c @@ -23,8 +23,10 @@ X509_LOOKUP_METHOD *X509_LOOKUP_meth_new(const char *name) if (method != NULL) { method->name = OPENSSL_strdup(name); - if (method->name == NULL) + if (method->name == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; + } } return method; diff --git a/openssl/src/crypto/x509/x509_obj.c b/openssl/src/crypto/x509/x509_obj.c index 2af7203b0..12c6d6f78 100644 --- a/openssl/src/crypto/x509/x509_obj.c +++ b/openssl/src/crypto/x509/x509_obj.c @@ -41,9 +41,9 @@ char *X509_NAME_oneline(const X509_NAME *a, char *buf, int len) if (buf == NULL) { if ((b = BUF_MEM_new()) == NULL) - goto buferr; + goto err; if (!BUF_MEM_grow(b, 200)) - goto buferr; + goto err; b->data[0] = '\0'; len = 200; } else if (len == 0) { @@ -124,7 +124,7 @@ char *X509_NAME_oneline(const X509_NAME *a, char *buf, int len) } if (b != NULL) { if (!BUF_MEM_grow(b, l + 1)) - goto buferr; + goto err; p = &(b->data[lold]); } else if (l > len) { break; @@ -179,8 +179,8 @@ char *X509_NAME_oneline(const X509_NAME *a, char *buf, int len) if (i == 0) *p = '\0'; return p; - buferr: - ERR_raise(ERR_LIB_X509, ERR_R_BUF_LIB); + err: + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); end: BUF_MEM_free(b); return NULL; diff --git a/openssl/src/crypto/x509/x509_r2x.c b/openssl/src/crypto/x509/x509_r2x.c index a6ea8e36a..c7f6181c4 100644 --- a/openssl/src/crypto/x509/x509_r2x.c +++ b/openssl/src/crypto/x509/x509_r2x.c @@ -25,7 +25,7 @@ X509 *X509_REQ_to_X509(X509_REQ *r, int days, EVP_PKEY *pkey) EVP_PKEY *pubkey = NULL; if ((ret = X509_new()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; } diff --git a/openssl/src/crypto/x509/x509_req.c b/openssl/src/crypto/x509/x509_req.c index 74d1d2993..e3f5c2add 100644 --- a/openssl/src/crypto/x509/x509_req.c +++ b/openssl/src/crypto/x509/x509_req.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -28,7 +28,7 @@ X509_REQ *X509_to_X509_REQ(X509 *x, EVP_PKEY *pkey, const EVP_MD *md) ret = X509_REQ_new_ex(x->libctx, x->propq); if (ret == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } @@ -67,7 +67,7 @@ EVP_PKEY *X509_REQ_get_pubkey(X509_REQ *req) return X509_PUBKEY_get(req->req_info.pubkey); } -EVP_PKEY *X509_REQ_get0_pubkey(const X509_REQ *req) +EVP_PKEY *X509_REQ_get0_pubkey(X509_REQ *req) { if (req == NULL) return NULL; @@ -79,9 +79,28 @@ X509_PUBKEY *X509_REQ_get_X509_PUBKEY(X509_REQ *req) return req->req_info.pubkey; } -int X509_REQ_check_private_key(const X509_REQ *req, EVP_PKEY *pkey) +int X509_REQ_check_private_key(X509_REQ *x, EVP_PKEY *k) { - return ossl_x509_check_private_key(X509_REQ_get0_pubkey(req), pkey); + EVP_PKEY *xk = NULL; + int ok = 0; + + xk = X509_REQ_get_pubkey(x); + switch (EVP_PKEY_eq(xk, k)) { + case 1: + ok = 1; + break; + case 0: + ERR_raise(ERR_LIB_X509, X509_R_KEY_VALUES_MISMATCH); + break; + case -1: + ERR_raise(ERR_LIB_X509, X509_R_KEY_TYPE_MISMATCH); + break; + case -2: + ERR_raise(ERR_LIB_X509, X509_R_UNKNOWN_KEY_TYPE); + } + + EVP_PKEY_free(xk); + return ok; } /* @@ -97,7 +116,6 @@ static int *ext_nids = ext_nid_list; int X509_REQ_extension_nid(int req_nid) { int i, nid; - for (i = 0;; i++) { nid = ext_nids[i]; if (nid == NID_undef) @@ -124,11 +142,11 @@ STACK_OF(X509_EXTENSION) *X509_REQ_get_extensions(X509_REQ *req) int idx, *pnid; const unsigned char *p; - if (req == NULL || !ext_nids) + if ((req == NULL) || !ext_nids) return NULL; for (pnid = ext_nids; *pnid != NID_undef; pnid++) { idx = X509_REQ_get_attr_by_NID(req, *pnid, -1); - if (idx < 0) + if (idx == -1) continue; attr = X509_REQ_get_attr(req, idx); ext = X509_ATTRIBUTE_get0_type(attr, 0); @@ -136,10 +154,8 @@ STACK_OF(X509_EXTENSION) *X509_REQ_get_extensions(X509_REQ *req) } if (ext == NULL) /* no extensions is not an error */ return sk_X509_EXTENSION_new_null(); - if (ext->type != V_ASN1_SEQUENCE) { - ERR_raise(ERR_LIB_X509, X509_R_WRONG_TYPE); + if (ext->type != V_ASN1_SEQUENCE) return NULL; - } p = ext->value.sequence->data; return (STACK_OF(X509_EXTENSION) *) ASN1_item_d2i(NULL, &p, ext->value.sequence->length, @@ -198,73 +214,44 @@ X509_ATTRIBUTE *X509_REQ_get_attr(const X509_REQ *req, int loc) X509_ATTRIBUTE *X509_REQ_delete_attr(X509_REQ *req, int loc) { - X509_ATTRIBUTE *attr; - - if (req == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - attr = X509at_delete_attr(req->req_info.attributes, loc); - if (attr != NULL) - req->req_info.enc.modified = 1; - return attr; + return X509at_delete_attr(req->req_info.attributes, loc); } int X509_REQ_add1_attr(X509_REQ *req, X509_ATTRIBUTE *attr) { - if (req == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (!X509at_add1_attr(&req->req_info.attributes, attr)) - return 0; - req->req_info.enc.modified = 1; - return 1; + if (X509at_add1_attr(&req->req_info.attributes, attr)) + return 1; + return 0; } int X509_REQ_add1_attr_by_OBJ(X509_REQ *req, const ASN1_OBJECT *obj, int type, const unsigned char *bytes, int len) { - if (req == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (!X509at_add1_attr_by_OBJ(&req->req_info.attributes, obj, - type, bytes, len)) - return 0; - req->req_info.enc.modified = 1; - return 1; + if (X509at_add1_attr_by_OBJ(&req->req_info.attributes, obj, + type, bytes, len)) + return 1; + return 0; } int X509_REQ_add1_attr_by_NID(X509_REQ *req, int nid, int type, const unsigned char *bytes, int len) { - if (req == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (!X509at_add1_attr_by_NID(&req->req_info.attributes, nid, - type, bytes, len)) - return 0; - req->req_info.enc.modified = 1; - return 1; + if (X509at_add1_attr_by_NID(&req->req_info.attributes, nid, + type, bytes, len)) + return 1; + return 0; } int X509_REQ_add1_attr_by_txt(X509_REQ *req, const char *attrname, int type, const unsigned char *bytes, int len) { - if (req == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (!X509at_add1_attr_by_txt(&req->req_info.attributes, attrname, - type, bytes, len)) - return 0; - req->req_info.enc.modified = 1; - return 1; + if (X509at_add1_attr_by_txt(&req->req_info.attributes, attrname, + type, bytes, len)) + return 1; + return 0; } long X509_REQ_get_version(const X509_REQ *req) @@ -289,7 +276,7 @@ void X509_REQ_get0_signature(const X509_REQ *req, const ASN1_BIT_STRING **psig, void X509_REQ_set0_signature(X509_REQ *req, ASN1_BIT_STRING *psig) { if (req->signature) - ASN1_BIT_STRING_free(req->signature); + ASN1_BIT_STRING_free(req->signature); req->signature = psig; } @@ -305,10 +292,6 @@ int X509_REQ_get_signature_nid(const X509_REQ *req) int i2d_re_X509_REQ_tbs(X509_REQ *req, unsigned char **pp) { - if (req == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } req->req_info.enc.modified = 1; return i2d_X509_REQ_INFO(&req->req_info, pp); } diff --git a/openssl/src/crypto/x509/x509_set.c b/openssl/src/crypto/x509/x509_set.c index 0881be729..dbd0f3e32 100644 --- a/openssl/src/crypto/x509/x509_set.c +++ b/openssl/src/crypto/x509/x509_set.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,22 +23,16 @@ int X509_set_version(X509 *x, long version) { if (x == NULL) return 0; - if (version == X509_get_version(x)) - return 1; /* avoid needless modification even re-allocation */ - if (version == X509_VERSION_1) { + if (version == 0) { ASN1_INTEGER_free(x->cert_info.version); x->cert_info.version = NULL; - x->cert_info.enc.modified = 1; return 1; } if (x->cert_info.version == NULL) { if ((x->cert_info.version = ASN1_INTEGER_new()) == NULL) return 0; } - if (!ASN1_INTEGER_set(x->cert_info.version, version)) - return 0; - x->cert_info.enc.modified = 1; - return 1; + return ASN1_INTEGER_set(x->cert_info.version, version); } int X509_set_serialNumber(X509 *x, ASN1_INTEGER *serial) @@ -50,78 +44,68 @@ int X509_set_serialNumber(X509 *x, ASN1_INTEGER *serial) in = &x->cert_info.serialNumber; if (in != serial) return ASN1_STRING_copy(in, serial); - x->cert_info.enc.modified = 1; return 1; } int X509_set_issuer_name(X509 *x, const X509_NAME *name) { - if (x == NULL || !X509_NAME_set(&x->cert_info.issuer, name)) + if (x == NULL) return 0; - x->cert_info.enc.modified = 1; - return 1; + return X509_NAME_set(&x->cert_info.issuer, name); } int X509_set_subject_name(X509 *x, const X509_NAME *name) { - if (x == NULL || !X509_NAME_set(&x->cert_info.subject, name)) + if (x == NULL) return 0; - x->cert_info.enc.modified = 1; - return 1; + return X509_NAME_set(&x->cert_info.subject, name); } -int ossl_x509_set1_time(int *modified, ASN1_TIME **ptm, const ASN1_TIME *tm) +int ossl_x509_set1_time(ASN1_TIME **ptm, const ASN1_TIME *tm) { - ASN1_TIME *new; - - if (*ptm == tm) - return 1; - new = ASN1_STRING_dup(tm); - if (tm != NULL && new == NULL) - return 0; - ASN1_TIME_free(*ptm); - *ptm = new; - if (modified != NULL) - *modified = 1; - return 1; + ASN1_TIME *in; + in = *ptm; + if (in != tm) { + in = ASN1_STRING_dup(tm); + if (in != NULL) { + ASN1_TIME_free(*ptm); + *ptm = in; + } + } + return (in != NULL); } int X509_set1_notBefore(X509 *x, const ASN1_TIME *tm) { - if (x == NULL || tm == NULL) + if (x == NULL) return 0; - return ossl_x509_set1_time(&x->cert_info.enc.modified, - &x->cert_info.validity.notBefore, tm); + return ossl_x509_set1_time(&x->cert_info.validity.notBefore, tm); } int X509_set1_notAfter(X509 *x, const ASN1_TIME *tm) { - if (x == NULL || tm == NULL) + if (x == NULL) return 0; - return ossl_x509_set1_time(&x->cert_info.enc.modified, - &x->cert_info.validity.notAfter, tm); + return ossl_x509_set1_time(&x->cert_info.validity.notAfter, tm); } int X509_set_pubkey(X509 *x, EVP_PKEY *pkey) { if (x == NULL) return 0; - if (!X509_PUBKEY_set(&(x->cert_info.key), pkey)) - return 0; - x->cert_info.enc.modified = 1; - return 1; + return X509_PUBKEY_set(&(x->cert_info.key), pkey); } int X509_up_ref(X509 *x) { int i; - if (CRYPTO_UP_REF(&x->references, &i) <= 0) + if (CRYPTO_UP_REF(&x->references, &i, x->lock) <= 0) return 0; REF_PRINT_COUNT("X509", x); REF_ASSERT_ISNT(i < 2); - return i > 1; + return ((i > 1) ? 1 : 0); } long X509_get_version(const X509 *x) @@ -210,7 +194,7 @@ int X509_get_signature_info(X509 *x, int *mdnid, int *pknid, int *secbits, /* Modify *siginf according to alg and sig. Return 1 on success, else 0. */ static int x509_sig_info_init(X509_SIG_INFO *siginf, const X509_ALGOR *alg, - const ASN1_STRING *sig, const EVP_PKEY *pubkey) + const ASN1_STRING *sig) { int pknid, mdnid; const EVP_MD *md; @@ -232,20 +216,12 @@ static int x509_sig_info_init(X509_SIG_INFO *siginf, const X509_ALGOR *alg, case NID_undef: /* If we have one, use a custom handler for this algorithm */ ameth = EVP_PKEY_asn1_find(NULL, pknid); - if (ameth != NULL && ameth->siginf_set != NULL - && ameth->siginf_set(siginf, alg, sig)) - break; - if (pubkey != NULL) { - int secbits; - - secbits = EVP_PKEY_get_security_bits(pubkey); - if (secbits != 0) { - siginf->secbits = secbits; - break; - } + if (ameth == NULL || ameth->siginf_set == NULL + || !ameth->siginf_set(siginf, alg, sig)) { + ERR_raise(ERR_LIB_X509, X509_R_ERROR_USING_SIGINF_SET); + return 0; } - ERR_raise(ERR_LIB_X509, X509_R_ERROR_USING_SIGINF_SET); - return 0; + break; /* * SHA1 and MD5 are known to be broken. Reduce security bits so that * they're no longer accepted at security level 1. @@ -266,13 +242,6 @@ static int x509_sig_info_init(X509_SIG_INFO *siginf, const X509_ALGOR *alg, */ siginf->secbits = 39; break; - case NID_id_GostR3411_94: - /* - * There is a collision attack on GOST R 34.11-94 at 2^105, see - * https://link.springer.com/chapter/10.1007%2F978-3-540-85174-5_10 - */ - siginf->secbits = 105; - break; default: /* Security bits: half number of bits in digest */ if ((md = EVP_get_digestbynid(mdnid)) == NULL) { @@ -296,6 +265,5 @@ static int x509_sig_info_init(X509_SIG_INFO *siginf, const X509_ALGOR *alg, /* Returns 1 on success, 0 on failure */ int ossl_x509_init_sig_info(X509 *x) { - return x509_sig_info_init(&x->siginf, &x->sig_alg, &x->signature, - X509_PUBKEY_get0(x->cert_info.key)); + return x509_sig_info_init(&x->siginf, &x->sig_alg, &x->signature); } diff --git a/openssl/src/crypto/x509/x509_trust.c b/openssl/src/crypto/x509/x509_trust.c index 1a4345f2f..fd77b0c6f 100644 --- a/openssl/src/crypto/x509/x509_trust.c +++ b/openssl/src/crypto/x509/x509_trust.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -62,7 +62,6 @@ int (*X509_TRUST_set_default(int (*trust) (int, X509 *, int))) (int, X509 *, return oldtrust; } -/* Returns X509_TRUST_TRUSTED, X509_TRUST_REJECTED, or X509_TRUST_UNTRUSTED */ int X509_check_trust(X509 *x, int id, int flags) { X509_TRUST *pt; @@ -105,8 +104,6 @@ int X509_TRUST_get_by_id(int id) if (trtable == NULL) return -1; tmp.trust = id; - /* Ideally, this would be done under lock */ - sk_X509_TRUST_sort(trtable); idx = sk_X509_TRUST_find(trtable, &tmp); if (idx < 0) return -1; @@ -138,8 +135,10 @@ int X509_TRUST_add(int id, int flags, int (*ck) (X509_TRUST *, X509 *, int), idx = X509_TRUST_get_by_id(id); /* Need a new entry */ if (idx < 0) { - if ((trtmp = OPENSSL_malloc(sizeof(*trtmp))) == NULL) + if ((trtmp = OPENSSL_malloc(sizeof(*trtmp))) == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; + } trtmp->flags = X509_TRUST_DYNAMIC; } else trtmp = X509_TRUST_get0(idx); @@ -148,8 +147,10 @@ int X509_TRUST_add(int id, int flags, int (*ck) (X509_TRUST *, X509 *, int), if (trtmp->flags & X509_TRUST_DYNAMIC_NAME) OPENSSL_free(trtmp->name); /* dup supplied name */ - if ((trtmp->name = OPENSSL_strdup(name)) == NULL) + if ((trtmp->name = OPENSSL_strdup(name)) == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; + } /* Keep the dynamic flag of existing entry */ trtmp->flags &= X509_TRUST_DYNAMIC; /* Set all other flags */ @@ -164,11 +165,11 @@ int X509_TRUST_add(int id, int flags, int (*ck) (X509_TRUST *, X509 *, int), if (idx < 0) { if (trtable == NULL && (trtable = sk_X509_TRUST_new(tr_cmp)) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); - goto err; + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); + goto err;; } if (!sk_X509_TRUST_push(trtable, trtmp)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } } @@ -252,7 +253,7 @@ static int obj_trust(int id, X509 *x, int flags) X509_CERT_AUX *ax = x->aux; int i; - if (ax != NULL && ax->reject != NULL) { + if (ax && ax->reject) { for (i = 0; i < sk_ASN1_OBJECT_num(ax->reject); i++) { ASN1_OBJECT *obj = sk_ASN1_OBJECT_value(ax->reject, i); int nid = OBJ_obj2nid(obj); @@ -263,7 +264,7 @@ static int obj_trust(int id, X509 *x, int flags) } } - if (ax != NULL && ax->trust != NULL) { + if (ax && ax->trust) { for (i = 0; i < sk_ASN1_OBJECT_num(ax->trust); i++) { ASN1_OBJECT *obj = sk_ASN1_OBJECT_value(ax->trust, i); int nid = OBJ_obj2nid(obj); @@ -275,7 +276,7 @@ static int obj_trust(int id, X509 *x, int flags) /* * Reject when explicit trust EKU are set and none match. * - * Returning untrusted is enough for full chains that end in + * Returning untrusted is enough for for full chains that end in * self-signed roots, because when explicit trust is specified it * suppresses the default blanket trust of self-signed objects. * diff --git a/openssl/src/crypto/x509/x509_txt.c b/openssl/src/crypto/x509/x509_txt.c index e825ce2db..0c7ae1ed7 100644 --- a/openssl/src/crypto/x509/x509_txt.c +++ b/openssl/src/crypto/x509/x509_txt.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -74,7 +74,7 @@ const char *X509_verify_cert_error_string(long n) case X509_V_ERR_PATH_LENGTH_EXCEEDED: return "path length constraint exceeded"; case X509_V_ERR_INVALID_PURPOSE: - return "unsuitable certificate purpose"; + return "unsupported certificate purpose"; case X509_V_ERR_CERT_UNTRUSTED: return "certificate not trusted"; case X509_V_ERR_CERT_REJECTED: @@ -179,7 +179,7 @@ const char *X509_verify_cert_error_string(long n) case X509_V_ERR_SIGNATURE_ALGORITHM_MISMATCH: return "subject signature algorithm and issuer public key algorithm mismatch"; case X509_V_ERR_SIGNATURE_ALGORITHM_INCONSISTENCY: - return "cert info signature and signature algorithm mismatch"; + return "cert info siganature and signature algorithm mismatch"; case X509_V_ERR_INVALID_CA: return "invalid CA certificate"; case X509_V_ERR_PATHLEN_INVALID_FOR_NON_CA: @@ -212,13 +212,6 @@ const char *X509_verify_cert_error_string(long n) return "Using cert extension requires at least X509v3"; case X509_V_ERR_EC_KEY_EXPLICIT_PARAMS: return "Certificate public key has explicit ECC parameters"; - case X509_V_ERR_RPK_UNTRUSTED: - return "Raw public key untrusted, no trusted keys configured"; - - /* - * Entries must be kept consistent with include/openssl/x509_vfy.h.in - * and with doc/man3/X509_STORE_CTX_get_error.pod - */ default: /* Printing an error number into a static buffer is not thread-safe */ diff --git a/openssl/src/crypto/x509/x509_v3.c b/openssl/src/crypto/x509/x509_v3.c index c29856e5b..262061a20 100644 --- a/openssl/src/crypto/x509/x509_v3.c +++ b/openssl/src/crypto/x509/x509_v3.c @@ -19,12 +19,9 @@ int X509v3_get_ext_count(const STACK_OF(X509_EXTENSION) *x) { - int ret; - if (x == NULL) return 0; - ret = sk_X509_EXTENSION_num(x); - return ret > 0 ? ret : 0; + return sk_X509_EXTENSION_num(x); } int X509v3_get_ext_by_NID(const STACK_OF(X509_EXTENSION) *x, int nid, @@ -105,14 +102,12 @@ STACK_OF(X509_EXTENSION) *X509v3_add_ext(STACK_OF(X509_EXTENSION) **x, if (x == NULL) { ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - goto err; + goto err2; } if (*x == NULL) { - if ((sk = sk_X509_EXTENSION_new_null()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + if ((sk = sk_X509_EXTENSION_new_null()) == NULL) goto err; - } } else sk = *x; @@ -122,18 +117,16 @@ STACK_OF(X509_EXTENSION) *X509v3_add_ext(STACK_OF(X509_EXTENSION) **x, else if (loc < 0) loc = n; - if ((new_ex = X509_EXTENSION_dup(ex)) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + if ((new_ex = X509_EXTENSION_dup(ex)) == NULL) + goto err2; + if (!sk_X509_EXTENSION_insert(sk, new_ex, loc)) goto err; - } - if (!sk_X509_EXTENSION_insert(sk, new_ex, loc)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); - goto err; - } if (*x == NULL) *x = sk; return sk; err: + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); + err2: X509_EXTENSION_free(new_ex); if (x != NULL && *x == NULL) sk_X509_EXTENSION_free(sk); @@ -166,7 +159,7 @@ X509_EXTENSION *X509_EXTENSION_create_by_OBJ(X509_EXTENSION **ex, if ((ex == NULL) || (*ex == NULL)) { if ((ret = X509_EXTENSION_new()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; } } else diff --git a/openssl/src/crypto/x509/x509_vfy.c b/openssl/src/crypto/x509/x509_vfy.c index 1794c14e9..4b18ba86c 100644 --- a/openssl/src/crypto/x509/x509_vfy.c +++ b/openssl/src/crypto/x509/x509_vfy.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -41,13 +41,9 @@ #define CRL_SCORE_AKID 0x004 /* CRL issuer matches CRL AKID */ #define CRL_SCORE_TIME_DELTA 0x002 /* Have a delta CRL with valid times */ -static int x509_verify_x509(X509_STORE_CTX *ctx); -static int x509_verify_rpk(X509_STORE_CTX *ctx); static int build_chain(X509_STORE_CTX *ctx); static int verify_chain(X509_STORE_CTX *ctx); -static int verify_rpk(X509_STORE_CTX *ctx); static int dane_verify(X509_STORE_CTX *ctx); -static int dane_verify_rpk(X509_STORE_CTX *ctx); static int null_callback(int ok, X509_STORE_CTX *e); static int check_issued(X509_STORE_CTX *ctx, X509 *x, X509 *issuer); static X509 *find_issuer(X509_STORE_CTX *ctx, STACK_OF(X509) *sk, X509 *x); @@ -60,8 +56,7 @@ static int check_cert(X509_STORE_CTX *ctx); static int check_policy(X509_STORE_CTX *ctx); static int get_issuer_sk(X509 **issuer, X509_STORE_CTX *ctx, X509 *x); static int check_dane_issuer(X509_STORE_CTX *ctx, int depth); -static int check_cert_key_level(X509_STORE_CTX *ctx, X509 *cert); -static int check_key_level(X509_STORE_CTX *ctx, EVP_PKEY *pkey); +static int check_key_level(X509_STORE_CTX *ctx, X509 *cert); static int check_sig_level(X509_STORE_CTX *ctx, X509 *cert); static int check_curve(X509 *cert); @@ -129,7 +124,6 @@ static int lookup_cert_match(X509 **result, X509_STORE_CTX *ctx, X509 *x) ERR_pop_to_mark(); if (certs == NULL) return -1; - /* Look for exact match */ for (i = 0; i < sk_X509_num(certs); i++) { xtmp = sk_X509_value(certs, i); @@ -144,7 +138,7 @@ static int lookup_cert_match(X509 **result, X509_STORE_CTX *ctx, X509 *x) else *result = xtmp; } - OSSL_STACK_OF_X509_free(certs); + sk_X509_pop_free(certs, X509_free); return ret; } @@ -153,7 +147,7 @@ static int lookup_cert_match(X509 **result, X509_STORE_CTX *ctx, X509 *x) * The error code is set to |err| if |err| is not X509_V_OK, else * |ctx->error| is left unchanged (under the assumption it is set elsewhere). * The error depth is |depth| if >= 0, else it defaults to |ctx->error_depth|. - * The error cert is |x| if not NULL, else the cert in |ctx->chain| at |depth|. + * The error cert is |x| if not NULL, else defaults to the chain cert at depth. * * Returns 0 to abort verification with an error, non-zero to continue. */ @@ -163,7 +157,7 @@ static int verify_cb_cert(X509_STORE_CTX *ctx, X509 *x, int depth, int err) depth = ctx->error_depth; else ctx->error_depth = depth; - ctx->current_cert = x != NULL ? x : sk_X509_value(ctx->chain, depth); + ctx->current_cert = (x != NULL) ? x : sk_X509_value(ctx->chain, depth); if (err != X509_V_OK) ctx->error = err; return ctx->verify_cb(0, ctx); @@ -186,7 +180,6 @@ static int verify_cb_crl(X509_STORE_CTX *ctx, int err) return ctx->verify_cb(0, ctx); } -/* Sadly, returns 0 also on internal error in ctx->verify_cb(). */ static int check_auth_level(X509_STORE_CTX *ctx) { int i; @@ -202,7 +195,7 @@ static int check_auth_level(X509_STORE_CTX *ctx) * We've already checked the security of the leaf key, so here we only * check the security of issuer keys. */ - CB_FAIL_IF(i > 0 && !check_cert_key_level(ctx, cert), + CB_FAIL_IF(i > 0 && !check_key_level(ctx, cert), ctx, cert, i, X509_V_ERR_CA_KEY_TOO_SMALL); /* * We also check the signature algorithm security of all certificates @@ -214,24 +207,7 @@ static int check_auth_level(X509_STORE_CTX *ctx) return 1; } -/*- - * Returns -1 on internal error. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). - */ -static int verify_rpk(X509_STORE_CTX *ctx) -{ - /* Not much to verify on a RPK */ - if (ctx->verify != NULL) - return ctx->verify(ctx); - - return !!ctx->verify_cb(ctx->error == X509_V_OK, ctx); -} - - -/*- - * Returns -1 on internal error. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). - */ +/* Returns -1 on internal error */ static int verify_chain(X509_STORE_CTX *ctx) { int err; @@ -277,58 +253,19 @@ int X509_STORE_CTX_verify(X509_STORE_CTX *ctx) ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); return -1; } - if (ctx->rpk != NULL) - return x509_verify_rpk(ctx); if (ctx->cert == NULL && sk_X509_num(ctx->untrusted) >= 1) ctx->cert = sk_X509_value(ctx->untrusted, 0); - return x509_verify_x509(ctx); + return X509_verify_cert(ctx); } int X509_verify_cert(X509_STORE_CTX *ctx) { + int ret; + if (ctx == NULL) { ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); return -1; } - return (ctx->rpk != NULL) ? x509_verify_rpk(ctx) : x509_verify_x509(ctx); -} - -/*- - * Returns -1 on internal error. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). - */ -static int x509_verify_rpk(X509_STORE_CTX *ctx) -{ - int ret; - - /* If the peer's public key is too weak, we can stop early. */ - if (!check_key_level(ctx, ctx->rpk) - && verify_cb_cert(ctx, NULL, 0, X509_V_ERR_EE_KEY_TOO_SMALL) == 0) - return 0; - - /* Barring any data to verify the RPK, simply report it as untrusted */ - ctx->error = X509_V_ERR_RPK_UNTRUSTED; - - ret = DANETLS_ENABLED(ctx->dane) ? dane_verify_rpk(ctx) : verify_rpk(ctx); - - /* - * Safety-net. If we are returning an error, we must also set ctx->error, - * so that the chain is not considered verified should the error be ignored - * (e.g. TLS with SSL_VERIFY_NONE). - */ - if (ret <= 0 && ctx->error == X509_V_OK) - ctx->error = X509_V_ERR_UNSPECIFIED; - return ret; -} - -/*- - * Returns -1 on internal error. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). - */ -static int x509_verify_x509(X509_STORE_CTX *ctx) -{ - int ret; - if (ctx->cert == NULL) { ERR_raise(ERR_LIB_X509, X509_R_NO_CERT_SET_FOR_US_TO_VERIFY); ctx->error = X509_V_ERR_INVALID_CALL; @@ -352,7 +289,7 @@ static int x509_verify_x509(X509_STORE_CTX *ctx) ctx->num_untrusted = 1; /* If the peer's public key is too weak, we can stop early. */ - CB_FAIL_IF(!check_cert_key_level(ctx, ctx->cert), + CB_FAIL_IF(!check_key_level(ctx, ctx->cert), ctx, ctx->cert, 0, X509_V_ERR_EE_KEY_TOO_SMALL); ret = DANETLS_ENABLED(ctx->dane) ? dane_verify(ctx) : verify_chain(ctx); @@ -403,7 +340,7 @@ static X509 *find_issuer(X509_STORE_CTX *ctx, STACK_OF(X509) *sk, X509 *x) return rv; } -/* Check that the given certificate |x| is issued by the certificate |issuer| */ +/* Check that the given certificate 'x' is issued by the certificate 'issuer' */ static int check_issued(ossl_unused X509_STORE_CTX *ctx, X509 *x, X509 *issuer) { int err = ossl_x509_likely_issued(issuer, x); @@ -414,6 +351,8 @@ static int check_issued(ossl_unused X509_STORE_CTX *ctx, X509 *x, X509 *issuer) * SUBJECT_ISSUER_MISMATCH just means 'x' is clearly not issued by 'issuer'. * Every other error code likely indicates a real error. */ + if (err != X509_V_ERR_SUBJECT_ISSUER_MISMATCH) + ctx->error = err; return 0; } @@ -424,16 +363,17 @@ static int check_issued(ossl_unused X509_STORE_CTX *ctx, X509 *x, X509 *issuer) static int get_issuer_sk(X509 **issuer, X509_STORE_CTX *ctx, X509 *x) { *issuer = find_issuer(ctx, ctx->other_ctx, x); - if (*issuer == NULL) - return 0; - return X509_up_ref(*issuer) ? 1 : -1; + if (*issuer != NULL) + return X509_up_ref(*issuer) ? 1 : -1; + return 0; } /*- * Alternative lookup method: look from a STACK stored in other_ctx. - * Returns NULL on internal/fatal error, empty stack if not found. + * Returns NULL on internal error (such as out of memory). */ -static STACK_OF(X509) *lookup_certs_sk(X509_STORE_CTX *ctx, const X509_NAME *nm) +static STACK_OF(X509) *lookup_certs_sk(X509_STORE_CTX *ctx, + const X509_NAME *nm) { STACK_OF(X509) *sk = sk_X509_new_null(); X509 *x; @@ -445,7 +385,7 @@ static STACK_OF(X509) *lookup_certs_sk(X509_STORE_CTX *ctx, const X509_NAME *nm) x = sk_X509_value(ctx->other_ctx, i); if (X509_NAME_cmp(nm, X509_get_subject_name(x)) == 0) { if (!X509_add_cert(sk, x, X509_ADD_FLAG_UP_REF)) { - OSSL_STACK_OF_X509_free(sk); + sk_X509_pop_free(sk, X509_free); ctx->error = X509_V_ERR_OUT_OF_MEM; return NULL; } @@ -457,7 +397,7 @@ static STACK_OF(X509) *lookup_certs_sk(X509_STORE_CTX *ctx, const X509_NAME *nm) /* * Check EE or CA certificate purpose. For trusted certificates explicit local * auxiliary trust can be used to override EKU-restrictions. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). + * Sadly, returns 0 also on internal error. */ static int check_purpose(X509_STORE_CTX *ctx, X509 *x, int purpose, int depth, int must_be_ca) @@ -490,7 +430,7 @@ static int check_purpose(X509_STORE_CTX *ctx, X509 *x, int purpose, int depth, return 1; case X509_TRUST_REJECTED: break; - default: /* can only be X509_TRUST_UNTRUSTED */ + default: switch (X509_check_purpose(x, purpose, must_be_ca > 0)) { case 1: return 1; @@ -506,9 +446,9 @@ static int check_purpose(X509_STORE_CTX *ctx, X509 *x, int purpose, int depth, return verify_cb_cert(ctx, x, depth, X509_V_ERR_INVALID_PURPOSE); } -/*- +/* * Check extensions of a cert chain for consistency with the supplied purpose. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). + * Sadly, returns 0 also on internal error. */ static int check_extensions(X509_STORE_CTX *ctx) { @@ -704,10 +644,7 @@ static int has_san_id(X509 *x, int gtype) return ret; } -/*- - * Returns -1 on internal error. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). - */ +/* Returns -1 on internal error */ static int check_name_constraints(X509_STORE_CTX *ctx) { int i; @@ -768,7 +705,7 @@ static int check_name_constraints(X509_STORE_CTX *ctx) */ tmpsubject = X509_NAME_dup(tmpsubject); if (tmpsubject == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); ctx->error = X509_V_ERR_OUT_OF_MEM; return -1; } @@ -928,7 +865,7 @@ static int check_trust(X509_STORE_CTX *ctx, int num_untrusted) res = lookup_cert_match(&mx, ctx, x); if (res < 0) return res; - if (res == 0) + if (mx == NULL) return X509_TRUST_UNTRUSTED; /* @@ -980,7 +917,7 @@ static int check_revocation(X509_STORE_CTX *ctx) last = sk_X509_num(ctx->chain) - 1; } else { /* If checking CRL paths this isn't the EE certificate */ - if (ctx->parent != NULL) + if (ctx->parent) return 1; last = 0; } @@ -1072,14 +1009,14 @@ static int check_crl_time(X509_STORE_CTX *ctx, X509_CRL *crl, int notify) time_t *ptime; int i; + if (notify) + ctx->current_crl = crl; if ((ctx->param->flags & X509_V_FLAG_USE_CHECK_TIME) != 0) ptime = &ctx->param->check_time; else if ((ctx->param->flags & X509_V_FLAG_NO_CHECK_TIME) != 0) return 1; else ptime = NULL; - if (notify) - ctx->current_crl = crl; i = X509_cmp_time(X509_CRL_get0_lastUpdate(crl), ptime); if (i == 0) { @@ -1691,7 +1628,6 @@ static int cert_crl(X509_STORE_CTX *ctx, X509_CRL *crl, X509 *x) return 1; } -/* Sadly, returns 0 also on internal error in ctx->verify_cb(). */ static int check_policy(X509_STORE_CTX *ctx) { int ret; @@ -1709,19 +1645,15 @@ static int check_policy(X509_STORE_CTX *ctx) * was verified via a bare public key, and pop it off right after the * X509_policy_check() call. */ - if (ctx->bare_ta_signed && !sk_X509_push(ctx->chain, NULL)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + if (ctx->bare_ta_signed && !sk_X509_push(ctx->chain, NULL)) goto memerr; - } ret = X509_policy_check(&ctx->tree, &ctx->explicit_policy, ctx->chain, ctx->param->policies, ctx->param->flags); if (ctx->bare_ta_signed) (void)sk_X509_pop(ctx->chain); - if (ret == X509_PCY_TREE_INTERNAL) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); + if (ret == X509_PCY_TREE_INTERNAL) goto memerr; - } /* Invalid or inconsistent extensions */ if (ret == X509_PCY_TREE_INVALID) { int i, cbcalled = 0; @@ -1768,6 +1700,7 @@ static int check_policy(X509_STORE_CTX *ctx) return 1; memerr: + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); ctx->error = X509_V_ERR_OUT_OF_MEM; return -1; } @@ -1778,7 +1711,6 @@ static int check_policy(X509_STORE_CTX *ctx) * the validation status. * * Return 1 on success, 0 otherwise. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). */ int ossl_x509_check_cert_time(X509_STORE_CTX *ctx, X509 *x, int depth) { @@ -1806,25 +1738,36 @@ int ossl_x509_check_cert_time(X509_STORE_CTX *ctx, X509 *x, int depth) return 1; } +static int pkey_ctrl_string(EVP_PKEY_CTX *ctx, const char *value) +{ + int rv = 0; + char *stmp, *vtmp = NULL; + + stmp = OPENSSL_strdup(value); + if (stmp == NULL) + return -1; + vtmp = strchr(stmp, ':'); + if (vtmp == NULL) + goto err; + + *vtmp = 0; + vtmp++; + rv = EVP_PKEY_CTX_ctrl_str(ctx, stmp, vtmp); + + err: + OPENSSL_free(stmp); + return rv; +} + /* * Verify the issuer signatures and cert times of ctx->chain. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). + * Sadly, returns 0 also on internal error. */ static int internal_verify(X509_STORE_CTX *ctx) { - int n; - X509 *xi; - X509 *xs; - - /* For RPK: just do the verify callback */ - if (ctx->rpk != NULL) { - if (!ctx->verify_cb(ctx->error == X509_V_OK, ctx)) - return 0; - return 1; - } - n = sk_X509_num(ctx->chain) - 1; - xi = sk_X509_value(ctx->chain, n); - xs = xi; + int n = sk_X509_num(ctx->chain) - 1; + X509 *xi = sk_X509_value(ctx->chain, n); + X509 *xs = xi; ctx->error_depth = n; if (ctx->bare_ta_signed) { @@ -1900,7 +1843,41 @@ static int internal_verify(X509_STORE_CTX *ctx) CB_FAIL_IF(1, ctx, xi, issuer_depth, X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY); } else { - CB_FAIL_IF(X509_verify(xs, pkey) <= 0, + EVP_MD_CTX *mctx = EVP_MD_CTX_new(); + EVP_PKEY_CTX *pkctx = NULL; + const char *mdname = NULL; + int mdnid, pknid; + int i; + + if (mctx == NULL) + return 0; + + if (OBJ_find_sigid_algs(X509_get_signature_nid(xs), + &mdnid, &pknid) + && mdnid != NID_undef) + mdname = OBJ_nid2sn(mdnid); + + if (!EVP_DigestVerifyInit_ex(mctx, &pkctx, mdname, xs->libctx, + xs->propq, pkey, NULL)) { + EVP_MD_CTX_free(mctx); + return 0; + } + + if (ctx->vfyopts) { + for (i = 0; i < sk_OPENSSL_STRING_num(ctx->vfyopts); i++) { + char *opt = sk_OPENSSL_STRING_value(ctx->vfyopts, i); + + if (pkey_ctrl_string(pkctx, opt) <= 0) { + EVP_MD_CTX_free(mctx); + return 0; + } + } + } + + ret = X509_verify_ctx(xs, mctx); + + EVP_MD_CTX_free(mctx); + CB_FAIL_IF(ret <= 0, ctx, xs, n, X509_V_ERR_CERT_SIGNATURE_FAILURE); } } @@ -1933,7 +1910,6 @@ int X509_cmp_current_time(const ASN1_TIME *ctm) return X509_cmp_time(ctm, NULL); } -/* returns 0 on error, otherwise 1 if ctm > cmp_time, else -1 */ int X509_cmp_time(const ASN1_TIME *ctm, time_t *cmp_time) { static const size_t utctime_length = sizeof("YYMMDDHHMMSSZ") - 1; @@ -2103,8 +2079,8 @@ X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer, { X509_CRL *crl = NULL; int i; - STACK_OF(X509_REVOKED) *revs = NULL; + STACK_OF(X509_REVOKED) *revs = NULL; /* CRLs can't be delta already */ if (base->base_crl_number != NULL || newer->base_crl_number != NULL) { ERR_raise(ERR_LIB_X509, X509_R_CRL_ALREADY_DELTA); @@ -2143,30 +2119,20 @@ X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer, } /* Create new CRL */ crl = X509_CRL_new_ex(base->libctx, base->propq); - if (crl == NULL || !X509_CRL_set_version(crl, X509_CRL_VERSION_2)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - goto err; - } + if (crl == NULL || !X509_CRL_set_version(crl, X509_CRL_VERSION_2)) + goto memerr; /* Set issuer name */ - if (!X509_CRL_set_issuer_name(crl, X509_CRL_get_issuer(newer))) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - goto err; - } + if (!X509_CRL_set_issuer_name(crl, X509_CRL_get_issuer(newer))) + goto memerr; - if (!X509_CRL_set1_lastUpdate(crl, X509_CRL_get0_lastUpdate(newer))) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - goto err; - } - if (!X509_CRL_set1_nextUpdate(crl, X509_CRL_get0_nextUpdate(newer))) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - goto err; - } + if (!X509_CRL_set1_lastUpdate(crl, X509_CRL_get0_lastUpdate(newer))) + goto memerr; + if (!X509_CRL_set1_nextUpdate(crl, X509_CRL_get0_nextUpdate(newer))) + goto memerr; /* Set base CRL number: must be critical */ - if (!X509_CRL_add1_ext_i2d(crl, NID_delta_crl, base->crl_number, 1, 0)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - goto err; - } + if (!X509_CRL_add1_ext_i2d(crl, NID_delta_crl, base->crl_number, 1, 0)) + goto memerr; /* * Copy extensions across from newest CRL to delta: this will set CRL @@ -2175,10 +2141,8 @@ X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer, for (i = 0; i < X509_CRL_get_ext_count(newer); i++) { X509_EXTENSION *ext = X509_CRL_get_ext(newer, i); - if (!X509_CRL_add_ext(crl, ext, -1)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - goto err; - } + if (!X509_CRL_add_ext(crl, ext, -1)) + goto memerr; } /* Go through revoked entries, copying as needed */ @@ -2195,26 +2159,22 @@ X509_CRL *X509_CRL_diff(X509_CRL *base, X509_CRL *newer, */ if (!X509_CRL_get0_by_serial(base, &rvtmp, &rvn->serialNumber)) { rvtmp = X509_REVOKED_dup(rvn); - if (rvtmp == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); - goto err; - } + if (rvtmp == NULL) + goto memerr; if (!X509_CRL_add0_revoked(crl, rvtmp)) { X509_REVOKED_free(rvtmp); - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - goto err; + goto memerr; } } } - if (skey != NULL && md != NULL && !X509_CRL_sign(crl, skey, md)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - goto err; - } + if (skey != NULL && md != NULL && !X509_CRL_sign(crl, skey, md)) + goto memerr; return crl; - err: + memerr: + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); X509_CRL_free(crl); return NULL; } @@ -2291,14 +2251,15 @@ void X509_STORE_CTX_set_cert(X509_STORE_CTX *ctx, X509 *x) ctx->cert = x; } -void X509_STORE_CTX_set0_rpk(X509_STORE_CTX *ctx, EVP_PKEY *rpk) +void X509_STORE_CTX_set0_crls(X509_STORE_CTX *ctx, STACK_OF(X509_CRL) *sk) { - ctx->rpk = rpk; + ctx->crls = sk; } -void X509_STORE_CTX_set0_crls(X509_STORE_CTX *ctx, STACK_OF(X509_CRL) *sk) +void X509_STORE_CTX_set0_vfyopts(X509_STORE_CTX *ctx, + STACK_OF(OPENSSL_STRING) *vfyopts) { - ctx->crls = sk; + ctx->vfyopts = vfyopts; } int X509_STORE_CTX_set_purpose(X509_STORE_CTX *ctx, int purpose) @@ -2385,14 +2346,17 @@ X509_STORE_CTX *X509_STORE_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq) { X509_STORE_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->libctx = libctx; if (propq != NULL) { ctx->propq = OPENSSL_strdup(propq); if (ctx->propq == NULL) { OPENSSL_free(ctx); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; } } @@ -2417,18 +2381,11 @@ void X509_STORE_CTX_free(X509_STORE_CTX *ctx) OPENSSL_free(ctx); } - -int X509_STORE_CTX_init_rpk(X509_STORE_CTX *ctx, X509_STORE *store, EVP_PKEY *rpk) -{ - if (!X509_STORE_CTX_init(ctx, store, NULL, NULL)) - return 0; - ctx->rpk = rpk; - return 1; -} - int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *store, X509 *x509, STACK_OF(X509) *chain) { + int ret = 1; + if (ctx == NULL) { ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); return 0; @@ -2455,7 +2412,7 @@ int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *store, X509 *x509, ctx->parent = NULL; ctx->dane = NULL; ctx->bare_ta_signed = 0; - ctx->rpk = NULL; + ctx->vfyopts = NULL; /* Zero ex_data to make sure we're cleanup-safe */ memset(&ctx->ex_data, 0, sizeof(ctx->ex_data)); @@ -2522,18 +2479,24 @@ int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *store, X509 *x509, ctx->param = X509_VERIFY_PARAM_new(); if (ctx->param == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } /* Inherit callbacks and flags from X509_STORE if not set use defaults. */ - if (store == NULL) + if (store != NULL) + ret = X509_VERIFY_PARAM_inherit(ctx->param, store->param); + else ctx->param->inh_flags |= X509_VP_FLAG_DEFAULT | X509_VP_FLAG_ONCE; - else if (X509_VERIFY_PARAM_inherit(ctx->param, store->param) == 0) - goto err; - if (!X509_STORE_CTX_set_default(ctx, "default")) + if (ret) + ret = X509_VERIFY_PARAM_inherit(ctx->param, + X509_VERIFY_PARAM_lookup("default")); + + if (ret == 0) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; + } /* * XXX: For now, continue to inherit trust from VPM, but infer from the @@ -2550,7 +2513,7 @@ int X509_STORE_CTX_init(X509_STORE_CTX *ctx, X509_STORE *store, X509 *x509, if (CRYPTO_new_ex_data(CRYPTO_EX_INDEX_X509_STORE_CTX, ctx, &ctx->ex_data)) return 1; - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); err: /* @@ -2592,7 +2555,7 @@ void X509_STORE_CTX_cleanup(X509_STORE_CTX *ctx) } X509_policy_tree_free(ctx->tree); ctx->tree = NULL; - OSSL_STACK_OF_X509_free(ctx->chain); + sk_X509_pop_free(ctx->chain, X509_free); ctx->chain = NULL; CRYPTO_free_ex_data(CRYPTO_EX_INDEX_X509_STORE_CTX, ctx, &(ctx->ex_data)); memset(&ctx->ex_data, 0, sizeof(ctx->ex_data)); @@ -2614,22 +2577,11 @@ void X509_STORE_CTX_set_time(X509_STORE_CTX *ctx, unsigned long flags, X509_VERIFY_PARAM_set_time(ctx->param, t); } -void X509_STORE_CTX_set_current_reasons(X509_STORE_CTX *ctx, - unsigned int current_reasons) -{ - ctx->current_reasons = current_reasons; -} - X509 *X509_STORE_CTX_get0_cert(const X509_STORE_CTX *ctx) { return ctx->cert; } -EVP_PKEY *X509_STORE_CTX_get0_rpk(const X509_STORE_CTX *ctx) -{ - return ctx->rpk; -} - STACK_OF(X509) *X509_STORE_CTX_get0_untrusted(const X509_STORE_CTX *ctx) { return ctx->untrusted; @@ -2642,7 +2594,7 @@ void X509_STORE_CTX_set0_untrusted(X509_STORE_CTX *ctx, STACK_OF(X509) *sk) void X509_STORE_CTX_set0_verified_chain(X509_STORE_CTX *ctx, STACK_OF(X509) *sk) { - OSSL_STACK_OF_X509_free(ctx->chain); + sk_X509_pop_free(ctx->chain, X509_free); ctx->chain = sk; } @@ -2691,12 +2643,6 @@ X509_STORE_CTX_get_crl_fn X509_STORE_CTX_get_get_crl(const X509_STORE_CTX *ctx) return ctx->get_crl; } -void X509_STORE_CTX_set_get_crl(X509_STORE_CTX *ctx, - X509_STORE_CTX_get_crl_fn get_crl) -{ - ctx->get_crl = get_crl; -} - X509_STORE_CTX_check_crl_fn X509_STORE_CTX_get_check_crl(const X509_STORE_CTX *ctx) { @@ -2752,10 +2698,8 @@ int X509_STORE_CTX_set_default(X509_STORE_CTX *ctx, const char *name) const X509_VERIFY_PARAM *param; param = X509_VERIFY_PARAM_lookup(name); - if (param == NULL) { - ERR_raise_data(ERR_LIB_X509, X509_R_UNKNOWN_PURPOSE_ID, "name=%s", name); + if (param == NULL) return 0; - } return X509_VERIFY_PARAM_inherit(ctx->param, param); } @@ -2797,7 +2741,7 @@ static unsigned char *dane_i2d(X509 *cert, uint8_t selector, } if (len < 0 || buf == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; } @@ -2808,7 +2752,7 @@ static unsigned char *dane_i2d(X509 *cert, uint8_t selector, #define DANETLS_NONE 256 /* impossible uint8_t */ /* Returns -1 on internal error */ -static int dane_match_cert(X509_STORE_CTX *ctx, X509 *cert, int depth) +static int dane_match(X509_STORE_CTX *ctx, X509 *cert, int depth) { SSL_DANE *dane = ctx->dane; unsigned usage = DANETLS_NONE; @@ -2966,7 +2910,7 @@ static int check_dane_issuer(X509_STORE_CTX *ctx, int depth) * for an exact match for the leaf certificate). */ cert = sk_X509_value(ctx->chain, depth); - if (cert != NULL && (matched = dane_match_cert(ctx, cert, depth)) < 0) + if (cert != NULL && (matched = dane_match(ctx, cert, depth)) < 0) return matched; if (matched > 0) { ctx->num_untrusted = depth - 1; @@ -3013,62 +2957,6 @@ static int check_dane_pkeys(X509_STORE_CTX *ctx) return X509_TRUST_UNTRUSTED; } -/* - * Only DANE-EE and SPKI are supported - * Returns -1 on internal error - */ -static int dane_match_rpk(X509_STORE_CTX *ctx, EVP_PKEY *rpk) -{ - SSL_DANE *dane = ctx->dane; - danetls_record *t = NULL; - int mtype = DANETLS_MATCHING_FULL; - unsigned char *i2dbuf = NULL; - unsigned int i2dlen = 0; - unsigned char mdbuf[EVP_MAX_MD_SIZE]; - unsigned char *cmpbuf; - unsigned int cmplen = 0; - int len; - int recnum = sk_danetls_record_num(dane->trecs); - int i; - int matched = 0; - - /* Calculate ASN.1 DER of RPK */ - if ((len = i2d_PUBKEY(rpk, &i2dbuf)) <= 0) - return -1; - cmplen = i2dlen = (unsigned int)len; - cmpbuf = i2dbuf; - - for (i = 0; i < recnum; i++) { - t = sk_danetls_record_value(dane->trecs, i); - if (t->usage != DANETLS_USAGE_DANE_EE || t->selector != DANETLS_SELECTOR_SPKI) - continue; - - /* Calculate hash - keep only one around */ - if (t->mtype != mtype) { - const EVP_MD *md = dane->dctx->mdevp[mtype = t->mtype]; - - cmpbuf = i2dbuf; - cmplen = i2dlen; - - if (md != NULL) { - cmpbuf = mdbuf; - if (!EVP_Digest(i2dbuf, i2dlen, cmpbuf, &cmplen, md, 0)) { - matched = -1; - break; - } - } - } - if (cmplen == t->dlen && memcmp(cmpbuf, t->data, cmplen) == 0) { - matched = 1; - dane->mdpth = 0; - dane->mtlsa = t; - break; - } - } - OPENSSL_free(i2dbuf); - return matched; -} - static void dane_reset(SSL_DANE *dane) { /* Reset state to verify another chain, or clear after failure. */ @@ -3079,7 +2967,6 @@ static void dane_reset(SSL_DANE *dane) dane->pdpth = -1; } -/* Sadly, returns 0 also on internal error in ctx->verify_cb(). */ static int check_leaf_suiteb(X509_STORE_CTX *ctx, X509 *cert) { int err = X509_chain_check_suiteb(NULL, cert, NULL, ctx->param->flags); @@ -3088,36 +2975,6 @@ static int check_leaf_suiteb(X509_STORE_CTX *ctx, X509 *cert) return 1; } -/* Returns -1 on internal error */ -static int dane_verify_rpk(X509_STORE_CTX *ctx) -{ - SSL_DANE *dane = ctx->dane; - int matched; - - dane_reset(dane); - - /* - * Look for a DANE record for RPK - * If error, return -1 - * If found, call ctx->verify_cb(1, ctx) - * If not found call ctx->verify_cb(0, ctx) - */ - matched = dane_match_rpk(ctx, ctx->rpk); - ctx->error_depth = 0; - - if (matched < 0) { - ctx->error = X509_V_ERR_UNSPECIFIED; - return -1; - } - - if (matched > 0) - ctx->error = X509_V_OK; - else - ctx->error = X509_V_ERR_DANE_NO_MATCH; - - return verify_rpk(ctx); -} - /* Returns -1 on internal error */ static int dane_verify(X509_STORE_CTX *ctx) { @@ -3140,7 +2997,7 @@ static int dane_verify(X509_STORE_CTX *ctx) * + matched == 0, mdepth < 0 (no PKIX-EE match) and there are no * DANE-TA(2) or PKIX-TA(0) to test. */ - matched = dane_match_cert(ctx, ctx->cert, 0); + matched = dane_match(ctx, ctx->cert, 0); done = matched != 0 || (!DANETLS_HAS_TA(dane) && dane->mdpth < 0); if (done && !X509_get_pubkey_parameters(NULL, ctx->chain)) @@ -3197,10 +3054,7 @@ static int get1_trusted_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *cert) return ok; } -/*- - * Returns -1 on internal error. - * Sadly, returns 0 also on internal error in ctx->verify_cb(). - */ +/* Returns -1 on internal error */ static int build_chain(X509_STORE_CTX *ctx) { SSL_DANE *dane = ctx->dane; @@ -3213,6 +3067,7 @@ static int build_chain(X509_STORE_CTX *ctx) int alt_untrusted = 0; int max_depth; int ok = 0; + int prev_error = ctx->error; int i; /* Our chain starts with a single untrusted element. */ @@ -3240,30 +3095,24 @@ static int build_chain(X509_STORE_CTX *ctx) } /* Initialize empty untrusted stack. */ - if ((sk_untrusted = sk_X509_new_null()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + if ((sk_untrusted = sk_X509_new_null()) == NULL) goto memerr; - } /* * If we got any "Cert(0) Full(0)" trust anchors from DNS, *prepend* them * to our working copy of the untrusted certificate stack. */ if (DANETLS_ENABLED(dane) && dane->certs != NULL - && !X509_add_certs(sk_untrusted, dane->certs, X509_ADD_FLAG_DEFAULT)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); + && !X509_add_certs(sk_untrusted, dane->certs, X509_ADD_FLAG_DEFAULT)) goto memerr; - } /* * Shallow-copy the stack of untrusted certificates (with TLS, this is * typically the content of the peer's certificate message) so we can make * multiple passes over it, while free to remove elements as we go. */ - if (!X509_add_certs(sk_untrusted, ctx->untrusted, X509_ADD_FLAG_DEFAULT)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); + if (!X509_add_certs(sk_untrusted, ctx->untrusted, X509_ADD_FLAG_DEFAULT)) goto memerr; - } /* * Still absurdly large, but arithmetically safe, a lower hard upper bound @@ -3371,18 +3220,20 @@ static int build_chain(X509_STORE_CTX *ctx) dane->pdpth = -1; } - if (!self_signed) { /* untrusted not self-signed certificate */ - /* Grow the chain by trusted issuer */ + /* + * Self-signed untrusted certificates get replaced by their + * trusted matching issuer. Otherwise, grow the chain. + */ + if (!self_signed) { if (!sk_X509_push(ctx->chain, issuer)) { X509_free(issuer); - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); goto memerr; } if ((self_signed = X509_self_signed(issuer, 0)) < 0) goto int_err; } else { /* - * We have a self-signed untrusted cert that has the same + * We have a self-signed certificate that has the same * subject name (and perhaps keyid and/or serial number) as * a trust anchor. We must have an exact match to avoid * possible impersonation via key substitution etc. @@ -3392,10 +3243,6 @@ static int build_chain(X509_STORE_CTX *ctx) X509_free(issuer); ok = 0; } else { /* curr "==" issuer */ - /* - * Replace self-signed untrusted certificate - * by its trusted matching issuer. - */ X509_free(curr); ctx->num_untrusted = --num; (void)sk_X509_set(ctx->chain, num, issuer); @@ -3448,7 +3295,7 @@ static int build_chain(X509_STORE_CTX *ctx) } /* - * Try to extend chain with peer-provided untrusted certificate + * Extend chain with peer-provided untrusted certificates */ if ((search & S_DOUNTRUSTED) != 0) { num = sk_X509_num(ctx->chain); @@ -3472,7 +3319,6 @@ static int build_chain(X509_STORE_CTX *ctx) /* Drop this issuer from future consideration */ (void)sk_X509_delete_ptr(sk_untrusted, issuer); - /* Grow the chain by untrusted issuer */ if (!X509_add_cert(ctx->chain, issuer, X509_ADD_FLAG_UP_REF)) goto int_err; @@ -3503,18 +3349,20 @@ static int build_chain(X509_STORE_CTX *ctx) switch (trust) { case X509_TRUST_TRUSTED: + /* Must restore any previous error value for backward compatibility */ + ctx->error = prev_error; return 1; case X509_TRUST_REJECTED: /* Callback already issued */ return 0; case X509_TRUST_UNTRUSTED: default: - switch (ctx->error) { + switch(ctx->error) { case X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD: case X509_V_ERR_CERT_NOT_YET_VALID: case X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD: case X509_V_ERR_CERT_HAS_EXPIRED: - return 0; /* Callback already done by ossl_x509_check_cert_time() */ + return 0; /* Callback already issued by ossl_x509_check_cert_time() */ default: /* A preliminary error has become final */ return verify_cb_cert(ctx, NULL, num - 1, ctx->error); case X509_V_OK: @@ -3543,6 +3391,7 @@ static int build_chain(X509_STORE_CTX *ctx) return -1; memerr: + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); ctx->error = X509_V_ERR_OUT_OF_MEM; sk_X509_free(sk_untrusted); return -1; @@ -3598,11 +3447,12 @@ static const int minbits_table[] = { 80, 112, 128, 192, 256 }; static const int NUM_AUTH_LEVELS = OSSL_NELEM(minbits_table); /*- - * Check whether the given public key meets the security level of `ctx`. + * Check whether the public key of `cert` meets the security level of `ctx`. * Returns 1 on success, 0 otherwise. */ -static int check_key_level(X509_STORE_CTX *ctx, EVP_PKEY *pkey) +static int check_key_level(X509_STORE_CTX *ctx, X509 *cert) { + EVP_PKEY *pkey = X509_get0_pubkey(cert); int level = ctx->param->auth_level; /* @@ -3624,15 +3474,6 @@ static int check_key_level(X509_STORE_CTX *ctx, EVP_PKEY *pkey) return EVP_PKEY_get_security_bits(pkey) >= minbits_table[level - 1]; } -/*- - * Check whether the public key of `cert` meets the security level of `ctx`. - * Returns 1 on success, 0 otherwise. - */ -static int check_cert_key_level(X509_STORE_CTX *ctx, X509 *cert) -{ - return check_key_level(ctx, X509_get0_pubkey(cert)); -} - /*- * Check whether the public key of ``cert`` does not use explicit params * for an elliptic curve. @@ -3642,19 +3483,21 @@ static int check_cert_key_level(X509_STORE_CTX *ctx, X509 *cert) static int check_curve(X509 *cert) { EVP_PKEY *pkey = X509_get0_pubkey(cert); - int ret, val; /* Unsupported or malformed key */ if (pkey == NULL) return -1; - if (EVP_PKEY_get_id(pkey) != EVP_PKEY_EC) - return 1; - ret = - EVP_PKEY_get_int_param(pkey, - OSSL_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS, - &val); - return ret == 1 ? !val : -1; + if (EVP_PKEY_get_id(pkey) == EVP_PKEY_EC) { + int ret, val; + + ret = EVP_PKEY_get_int_param(pkey, + OSSL_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS, + &val); + return ret < 0 ? ret : !val; + } + + return 1; } /*- diff --git a/openssl/src/crypto/x509/x509_vpm.c b/openssl/src/crypto/x509/x509_vpm.c index 023a38a1c..7fb9dca31 100644 --- a/openssl/src/crypto/x509/x509_vpm.c +++ b/openssl/src/crypto/x509/x509_vpm.c @@ -1,5 +1,5 @@ /* - * Copyright 2004-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2004-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,6 +23,9 @@ #define SET_HOST 0 #define ADD_HOST 1 +static int int_x509_param_set1(char **pdest, size_t *pdestlen, + const char *src, size_t srclen); + static char *str_copy(const char *s) { return OPENSSL_strdup(s); @@ -44,8 +47,7 @@ static int int_x509_param_set_hosts(X509_VERIFY_PARAM *vpm, int mode, */ if (namelen == 0 || name == NULL) namelen = name ? strlen(name) : 0; - else if (name != NULL - && memchr(name, '\0', namelen > 1 ? namelen - 1 : namelen) != NULL) + else if (name && memchr(name, '\0', namelen > 1 ? namelen - 1 : namelen)) return 0; if (namelen > 0 && name[namelen - 1] == '\0') --namelen; @@ -79,13 +81,16 @@ static int int_x509_param_set_hosts(X509_VERIFY_PARAM *vpm, int mode, return 1; } + X509_VERIFY_PARAM *X509_VERIFY_PARAM_new(void) { X509_VERIFY_PARAM *param; param = OPENSSL_zalloc(sizeof(*param)); - if (param == NULL) + if (param == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; + } param->trust = X509_TRUST_DEFAULT; /* param->inh_flags = X509_VP_FLAG_DEFAULT; */ param->depth = -1; @@ -105,6 +110,48 @@ void X509_VERIFY_PARAM_free(X509_VERIFY_PARAM *param) OPENSSL_free(param); } +int X509_VERIFY_PARAM_copy(X509_VERIFY_PARAM *dest, const X509_VERIFY_PARAM *src) +{ + if (dest == NULL || src == NULL || dest == src) + return 0; + + if (src->name && !X509_VERIFY_PARAM_set1_name(dest, src->name)) + return 0; + + dest->check_time = src->check_time; + dest->inh_flags = src->inh_flags; + dest->flags = src->flags; + dest->purpose = src->purpose; + dest->trust = src->trust; + dest->depth = src->depth; + dest->auth_level = src->auth_level; + + if (src->policies && !X509_VERIFY_PARAM_set1_policies(dest, src->policies)) + return 0; + + if (src->hosts) { + sk_OPENSSL_STRING_pop_free(dest->hosts, str_free); + dest->hosts = sk_OPENSSL_STRING_deep_copy(src->hosts, str_copy, str_free); + if (dest->hosts == NULL) + return 0; + dest->hostflags = src->hostflags; + } + + if (src->peername && !int_x509_param_set1(&dest->peername, NULL, + src->peername, + strlen(src->peername))) + return 0; + + if (src->email && !X509_VERIFY_PARAM_set1_email(dest, src->email, + src->emaillen)) + return 0; + + if (src->ip && !X509_VERIFY_PARAM_set1_ip(dest, src->ip, src->iplen)) + return 0; + + return 1; +} + /*- * This function determines how parameters are "inherited" from one structure * to another. There are several different ways this can happen. @@ -140,7 +187,8 @@ void X509_VERIFY_PARAM_free(X509_VERIFY_PARAM *param) /* Macro to test if a field should be copied from src to dest */ #define test_x509_verify_param_copy(field, def) \ - (to_overwrite || (src->field != def && (to_default || dest->field == def))) + (to_overwrite \ + || ((src->field != def) && (to_default || (dest->field == def)))) /* Macro to test and copy a field if necessary */ @@ -153,19 +201,25 @@ int X509_VERIFY_PARAM_inherit(X509_VERIFY_PARAM *dest, { unsigned long inh_flags; int to_default, to_overwrite; - - if (src == NULL) + if (!src) return 1; inh_flags = dest->inh_flags | src->inh_flags; - if ((inh_flags & X509_VP_FLAG_ONCE) != 0) + if (inh_flags & X509_VP_FLAG_ONCE) dest->inh_flags = 0; - if ((inh_flags & X509_VP_FLAG_LOCKED) != 0) + if (inh_flags & X509_VP_FLAG_LOCKED) return 1; - to_default = (inh_flags & X509_VP_FLAG_DEFAULT) != 0; - to_overwrite = (inh_flags & X509_VP_FLAG_OVERWRITE) != 0; + if (inh_flags & X509_VP_FLAG_DEFAULT) + to_default = 1; + else + to_default = 0; + + if (inh_flags & X509_VP_FLAG_OVERWRITE) + to_overwrite = 1; + else + to_overwrite = 0; x509_verify_param_copy(purpose, 0); x509_verify_param_copy(trust, X509_TRUST_DEFAULT); @@ -174,13 +228,13 @@ int X509_VERIFY_PARAM_inherit(X509_VERIFY_PARAM *dest, /* If overwrite or check time not set, copy across */ - if (to_overwrite || (dest->flags & X509_V_FLAG_USE_CHECK_TIME) == 0) { + if (to_overwrite || !(dest->flags & X509_V_FLAG_USE_CHECK_TIME)) { dest->check_time = src->check_time; dest->flags &= ~X509_V_FLAG_USE_CHECK_TIME; /* Don't need to copy flag: that is done below */ } - if ((inh_flags & X509_VP_FLAG_RESET_FLAGS) != 0) + if (inh_flags & X509_VP_FLAG_RESET_FLAGS) dest->flags = 0; dest->flags |= src->flags; @@ -195,7 +249,7 @@ int X509_VERIFY_PARAM_inherit(X509_VERIFY_PARAM *dest, if (test_x509_verify_param_copy(hosts, NULL)) { sk_OPENSSL_STRING_pop_free(dest->hosts, str_free); dest->hosts = NULL; - if (src->hosts != NULL) { + if (src->hosts) { dest->hosts = sk_OPENSSL_STRING_deep_copy(src->hosts, str_copy, str_free); if (dest->hosts == NULL) @@ -219,14 +273,8 @@ int X509_VERIFY_PARAM_inherit(X509_VERIFY_PARAM *dest, int X509_VERIFY_PARAM_set1(X509_VERIFY_PARAM *to, const X509_VERIFY_PARAM *from) { - unsigned long save_flags; + unsigned long save_flags = to->inh_flags; int ret; - - if (to == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - save_flags = to->inh_flags; to->inh_flags |= X509_VP_FLAG_DEFAULT; ret = X509_VERIFY_PARAM_inherit(to, from); to->inh_flags = save_flags; @@ -237,8 +285,7 @@ static int int_x509_param_set1(char **pdest, size_t *pdestlen, const char *src, size_t srclen) { char *tmp; - - if (src != NULL) { + if (src) { if (srclen == 0) srclen = strlen(src); @@ -262,13 +309,15 @@ int X509_VERIFY_PARAM_set1_name(X509_VERIFY_PARAM *param, const char *name) { OPENSSL_free(param->name); param->name = OPENSSL_strdup(name); - return param->name != NULL; + if (param->name) + return 1; + return 0; } int X509_VERIFY_PARAM_set_flags(X509_VERIFY_PARAM *param, unsigned long flags) { param->flags |= flags; - if ((flags & X509_V_FLAG_POLICY_MASK) != 0) + if (flags & X509_V_FLAG_POLICY_MASK) param->flags |= X509_V_FLAG_POLICY_CHECK; return 1; } @@ -335,8 +384,7 @@ int X509_VERIFY_PARAM_add0_policy(X509_VERIFY_PARAM *param, if (param->policies == NULL) return 0; } - - if (sk_ASN1_OBJECT_push(param->policies, policy) <= 0) + if (!sk_ASN1_OBJECT_push(param->policies, policy)) return 0; return 1; } @@ -347,10 +395,8 @@ int X509_VERIFY_PARAM_set1_policies(X509_VERIFY_PARAM *param, int i; ASN1_OBJECT *oid, *doid; - if (param == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); + if (param == NULL) return 0; - } sk_ASN1_OBJECT_pop_free(param->policies, ASN1_OBJECT_free); if (policies == NULL) { @@ -365,7 +411,7 @@ int X509_VERIFY_PARAM_set1_policies(X509_VERIFY_PARAM *param, for (i = 0; i < sk_ASN1_OBJECT_num(policies); i++) { oid = sk_ASN1_OBJECT_value(policies, i); doid = OBJ_dup(oid); - if (doid == NULL) + if (!doid) return 0; if (!sk_ASN1_OBJECT_push(param->policies, doid)) { ASN1_OBJECT_free(doid); @@ -423,7 +469,7 @@ void X509_VERIFY_PARAM_move_peername(X509_VERIFY_PARAM *to, OPENSSL_free(to->peername); to->peername = peername; } - if (from != NULL) + if (from) from->peername = NULL; } @@ -442,10 +488,8 @@ int X509_VERIFY_PARAM_set1_email(X509_VERIFY_PARAM *param, static unsigned char *int_X509_VERIFY_PARAM_get0_ip(X509_VERIFY_PARAM *param, size_t *plen) { - if (param == NULL || param->ip == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); + if (param == NULL || param->ip == NULL) return NULL; - } if (plen != NULL) *plen = param->iplen; return param->ip; @@ -456,16 +500,14 @@ char *X509_VERIFY_PARAM_get1_ip_asc(X509_VERIFY_PARAM *param) size_t iplen; unsigned char *ip = int_X509_VERIFY_PARAM_get0_ip(param, &iplen); - return ip == NULL ? NULL : ossl_ipaddr_to_asc(ip, iplen); + return ip == NULL ? NULL : ossl_ipaddr_to_asc(ip, iplen); } int X509_VERIFY_PARAM_set1_ip(X509_VERIFY_PARAM *param, const unsigned char *ip, size_t iplen) { - if (iplen != 0 && iplen != 4 && iplen != 16) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_INVALID_ARGUMENT); + if (iplen != 0 && iplen != 4 && iplen != 16) return 0; - } return int_x509_param_set1((char **)¶m->ip, ¶m->iplen, (char *)ip, iplen); } @@ -473,8 +515,9 @@ int X509_VERIFY_PARAM_set1_ip(X509_VERIFY_PARAM *param, int X509_VERIFY_PARAM_set1_ip_asc(X509_VERIFY_PARAM *param, const char *ipasc) { unsigned char ipout[16]; - size_t iplen = (size_t)ossl_a2i_ipadd(ipout, ipasc); + size_t iplen; + iplen = (size_t)ossl_a2i_ipadd(ipout, ipasc); if (iplen == 0) return 0; return X509_VERIFY_PARAM_set1_ip(param, ipout, iplen); @@ -504,18 +547,6 @@ const char *X509_VERIFY_PARAM_get0_name(const X509_VERIFY_PARAM *param) */ static const X509_VERIFY_PARAM default_table[] = { - { - "code_sign", /* Code sign parameters */ - 0, /* check time to use */ - 0, /* inheritance flags */ - 0, /* flags */ - X509_PURPOSE_CODE_SIGN, /* purpose */ - X509_TRUST_OBJECT_SIGN, /* trust */ - -1, /* depth */ - -1, /* auth_level */ - NULL, /* policies */ - vpm_empty_id - }, { "default", /* X509 default parameters */ 0, /* check time to use */ @@ -526,8 +557,7 @@ static const X509_VERIFY_PARAM default_table[] = { 100, /* depth */ -1, /* auth_level */ NULL, /* policies */ - vpm_empty_id - }, + vpm_empty_id}, { "pkcs7", /* S/MIME sign parameters */ 0, /* check time to use */ @@ -538,8 +568,7 @@ static const X509_VERIFY_PARAM default_table[] = { -1, /* depth */ -1, /* auth_level */ NULL, /* policies */ - vpm_empty_id - }, + vpm_empty_id}, { "smime_sign", /* S/MIME sign parameters */ 0, /* check time to use */ @@ -550,8 +579,7 @@ static const X509_VERIFY_PARAM default_table[] = { -1, /* depth */ -1, /* auth_level */ NULL, /* policies */ - vpm_empty_id - }, + vpm_empty_id}, { "ssl_client", /* SSL/TLS client parameters */ 0, /* check time to use */ @@ -562,8 +590,7 @@ static const X509_VERIFY_PARAM default_table[] = { -1, /* depth */ -1, /* auth_level */ NULL, /* policies */ - vpm_empty_id - }, + vpm_empty_id}, { "ssl_server", /* SSL/TLS server parameters */ 0, /* check time to use */ @@ -574,8 +601,7 @@ static const X509_VERIFY_PARAM default_table[] = { -1, /* depth */ -1, /* auth_level */ NULL, /* policies */ - vpm_empty_id - } + vpm_empty_id} }; static STACK_OF(X509_VERIFY_PARAM) *param_table = NULL; @@ -598,7 +624,6 @@ int X509_VERIFY_PARAM_add0_table(X509_VERIFY_PARAM *param) { int idx; X509_VERIFY_PARAM *ptmp; - if (param_table == NULL) { param_table = sk_X509_VERIFY_PARAM_new(param_cmp); if (param_table == NULL) @@ -610,8 +635,7 @@ int X509_VERIFY_PARAM_add0_table(X509_VERIFY_PARAM *param) X509_VERIFY_PARAM_free(ptmp); } } - - if (sk_X509_VERIFY_PARAM_push(param_table, param) <= 0) + if (!sk_X509_VERIFY_PARAM_push(param_table, param)) return 0; return 1; } @@ -619,8 +643,7 @@ int X509_VERIFY_PARAM_add0_table(X509_VERIFY_PARAM *param) int X509_VERIFY_PARAM_get_count(void) { int num = OSSL_NELEM(default_table); - - if (param_table != NULL) + if (param_table) num += sk_X509_VERIFY_PARAM_num(param_table); return num; } @@ -628,7 +651,6 @@ int X509_VERIFY_PARAM_get_count(void) const X509_VERIFY_PARAM *X509_VERIFY_PARAM_get0(int id) { int num = OSSL_NELEM(default_table); - if (id < num) return default_table + id; return sk_X509_VERIFY_PARAM_value(param_table, id - num); @@ -641,8 +663,6 @@ const X509_VERIFY_PARAM *X509_VERIFY_PARAM_lookup(const char *name) pm.name = (char *)name; if (param_table != NULL) { - /* Ideally, this would be done under a lock */ - sk_X509_VERIFY_PARAM_sort(param_table); idx = sk_X509_VERIFY_PARAM_find(param_table, &pm); if (idx >= 0) return sk_X509_VERIFY_PARAM_value(param_table, idx); diff --git a/openssl/src/crypto/x509/x509cset.c b/openssl/src/crypto/x509/x509cset.c index 205fe3d6e..2746b9892 100644 --- a/openssl/src/crypto/x509/x509cset.c +++ b/openssl/src/crypto/x509/x509cset.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,41 +24,34 @@ int X509_CRL_set_version(X509_CRL *x, long version) if ((x->crl.version = ASN1_INTEGER_new()) == NULL) return 0; } - if (!ASN1_INTEGER_set(x->crl.version, version)) - return 0; - x->crl.enc.modified = 1; - return 1; + return ASN1_INTEGER_set(x->crl.version, version); } int X509_CRL_set_issuer_name(X509_CRL *x, const X509_NAME *name) { if (x == NULL) return 0; - if (!X509_NAME_set(&x->crl.issuer, name)) - return 0; - x->crl.enc.modified = 1; - return 1; + return X509_NAME_set(&x->crl.issuer, name); } int X509_CRL_set1_lastUpdate(X509_CRL *x, const ASN1_TIME *tm) { - if (x == NULL || tm == NULL) + if (x == NULL) return 0; - return ossl_x509_set1_time(&x->crl.enc.modified, &x->crl.lastUpdate, tm); + return ossl_x509_set1_time(&x->crl.lastUpdate, tm); } int X509_CRL_set1_nextUpdate(X509_CRL *x, const ASN1_TIME *tm) { if (x == NULL) return 0; - return ossl_x509_set1_time(&x->crl.enc.modified, &x->crl.nextUpdate, tm); + return ossl_x509_set1_time(&x->crl.nextUpdate, tm); } int X509_CRL_sort(X509_CRL *c) { int i; X509_REVOKED *r; - /* * sort the data so it will be written in serial number order */ @@ -75,12 +68,12 @@ int X509_CRL_up_ref(X509_CRL *crl) { int i; - if (CRYPTO_UP_REF(&crl->references, &i) <= 0) + if (CRYPTO_UP_REF(&crl->references, &i, crl->lock) <= 0) return 0; REF_PRINT_COUNT("X509_CRL", crl); REF_ASSERT_ISNT(i < 2); - return i > 1; + return ((i > 1) ? 1 : 0); } long X509_CRL_get_version(const X509_CRL *crl) @@ -146,9 +139,19 @@ const ASN1_TIME *X509_REVOKED_get0_revocationDate(const X509_REVOKED *x) int X509_REVOKED_set_revocationDate(X509_REVOKED *x, ASN1_TIME *tm) { - if (x == NULL || tm == NULL) + ASN1_TIME *in; + + if (x == NULL) return 0; - return ossl_x509_set1_time(NULL, &x->revocationDate, tm); + in = x->revocationDate; + if (in != tm) { + in = ASN1_STRING_dup(tm); + if (in != NULL) { + ASN1_TIME_free(x->revocationDate); + x->revocationDate = in; + } + } + return (in != NULL); } const ASN1_INTEGER *X509_REVOKED_get0_serialNumber(const X509_REVOKED *x) @@ -168,8 +171,7 @@ int X509_REVOKED_set_serialNumber(X509_REVOKED *x, ASN1_INTEGER *serial) return 1; } -const STACK_OF(X509_EXTENSION) *X509_REVOKED_get0_extensions(const - X509_REVOKED *r) +const STACK_OF(X509_EXTENSION) *X509_REVOKED_get0_extensions(const X509_REVOKED *r) { return r->extensions; } diff --git a/openssl/src/crypto/x509/x509name.c b/openssl/src/crypto/x509/x509name.c index 75ff07d89..690e2799f 100644 --- a/openssl/src/crypto/x509/x509name.c +++ b/openssl/src/crypto/x509/x509name.c @@ -49,12 +49,9 @@ int X509_NAME_get_text_by_OBJ(const X509_NAME *name, const ASN1_OBJECT *obj, int X509_NAME_entry_count(const X509_NAME *name) { - int ret; - if (name == NULL) return 0; - ret = sk_X509_NAME_ENTRY_num(name->entries); - return ret > 0 ? ret : 0; + return sk_X509_NAME_ENTRY_num(name->entries); } int X509_NAME_get_index_by_NID(const X509_NAME *name, int nid, int lastpos) @@ -225,7 +222,7 @@ int X509_NAME_add_entry(X509_NAME *name, const X509_NAME_ENTRY *ne, int loc, goto err; new_name->set = set; if (!sk_X509_NAME_ENTRY_insert(sk, new_name, loc)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } if (inc) { diff --git a/openssl/src/crypto/x509/x509spki.c b/openssl/src/crypto/x509/x509spki.c index 142eeb79b..1d66697db 100644 --- a/openssl/src/crypto/x509/x509spki.c +++ b/openssl/src/crypto/x509/x509spki.c @@ -35,8 +35,10 @@ NETSCAPE_SPKI *NETSCAPE_SPKI_b64_decode(const char *str, int len) NETSCAPE_SPKI *spki; if (len <= 0) len = strlen(str); - if ((spki_der = OPENSSL_malloc(len + 1)) == NULL) + if ((spki_der = OPENSSL_malloc(len + 1)) == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; + } spki_len = EVP_DecodeBlock(spki_der, (const unsigned char *)str, len); if (spki_len < 0) { ERR_raise(ERR_LIB_X509, X509_R_BASE64_DECODE_ERROR); @@ -63,6 +65,7 @@ char *NETSCAPE_SPKI_b64_encode(NETSCAPE_SPKI *spki) der_spki = OPENSSL_malloc(der_len); b64_str = OPENSSL_malloc(der_len * 2); if (der_spki == NULL || b64_str == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); OPENSSL_free(der_spki); OPENSSL_free(b64_str); return NULL; diff --git a/openssl/src/crypto/x509/x509type.c b/openssl/src/crypto/x509/x509type.c index 79fd5e7db..699b6d5fb 100644 --- a/openssl/src/crypto/x509/x509type.c +++ b/openssl/src/crypto/x509/x509type.c @@ -51,11 +51,6 @@ int X509_certificate_type(const X509 *x, const EVP_PKEY *pkey) case EVP_PKEY_DH: ret = EVP_PK_DH | EVP_PKT_EXCH; break; - case NID_id_GostR3410_2001: - case NID_id_GostR3410_2012_256: - case NID_id_GostR3410_2012_512: - ret = EVP_PKT_EXCH | EVP_PKT_SIGN; - break; default: break; } diff --git a/openssl/src/crypto/x509/x_all.c b/openssl/src/crypto/x509/x_all.c index 3e4c852b7..8874d7c14 100644 --- a/openssl/src/crypto/x509/x_all.c +++ b/openssl/src/crypto/x509/x_all.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -30,7 +30,7 @@ int X509_verify(X509 *a, EVP_PKEY *r) { - if (X509_ALGOR_cmp(&a->sig_alg, &a->cert_info.signature) != 0) + if (X509_ALGOR_cmp(&a->sig_alg, &a->cert_info.signature)) return 0; return ASN1_item_verify_ex(ASN1_ITEM_rptr(X509_CINF), &a->sig_alg, @@ -38,6 +38,15 @@ int X509_verify(X509 *a, EVP_PKEY *r) a->distinguishing_id, r, a->libctx, a->propq); } +int X509_verify_ctx(X509 *a, EVP_MD_CTX *ctx) +{ + if (X509_ALGOR_cmp(&a->sig_alg, &a->cert_info.signature) != 0) + return 0; + + return ASN1_item_verify_ctx(ASN1_ITEM_rptr(X509_CINF), &a->sig_alg, + &a->signature, &a->cert_info, ctx); +} + int X509_REQ_verify_ex(X509_REQ *a, EVP_PKEY *r, OSSL_LIB_CTX *libctx, const char *propq) { @@ -51,6 +60,12 @@ int X509_REQ_verify(X509_REQ *a, EVP_PKEY *r) return X509_REQ_verify_ex(a, r, NULL, NULL); } +int X509_REQ_verify_ctx(X509_REQ *a, EVP_MD_CTX *ctx) +{ + return ASN1_item_verify_ctx(ASN1_ITEM_rptr(X509_REQ_INFO), &a->sig_alg, + a->signature, &a->req_info, ctx); +} + int NETSCAPE_SPKI_verify(NETSCAPE_SPKI *a, EVP_PKEY *r) { return ASN1_item_verify(ASN1_ITEM_rptr(NETSCAPE_SPKAC), @@ -59,21 +74,6 @@ int NETSCAPE_SPKI_verify(NETSCAPE_SPKI *a, EVP_PKEY *r) int X509_sign(X509 *x, EVP_PKEY *pkey, const EVP_MD *md) { - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (sk_X509_EXTENSION_num(X509_get0_extensions(x)) > 0 - && !X509_set_version(x, X509_VERSION_3)) - return 0; - - /* - * Setting the modified flag before signing it. This makes the cached - * encoding to be ignored, so even if the certificate fields have changed, - * they are signed correctly. - * The X509_sign_ctx, X509_REQ_sign{,_ctx}, X509_CRL_sign{,_ctx} functions - * which exist below are the same. - */ x->cert_info.enc.modified = 1; return ASN1_item_sign_ex(ASN1_ITEM_rptr(X509_CINF), &x->cert_info.signature, &x->sig_alg, &x->signature, &x->cert_info, NULL, @@ -82,13 +82,6 @@ int X509_sign(X509 *x, EVP_PKEY *pkey, const EVP_MD *md) int X509_sign_ctx(X509 *x, EVP_MD_CTX *ctx) { - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - if (sk_X509_EXTENSION_num(X509_get0_extensions(x)) > 0 - && !X509_set_version(x, X509_VERSION_3)) - return 0; x->cert_info.enc.modified = 1; return ASN1_item_sign_ctx(ASN1_ITEM_rptr(X509_CINF), &x->cert_info.signature, @@ -98,9 +91,8 @@ int X509_sign_ctx(X509 *x, EVP_MD_CTX *ctx) static ASN1_VALUE *simple_get_asn1(const char *url, BIO *bio, BIO *rbio, int timeout, const ASN1_ITEM *it) { -#ifndef OPENSSL_NO_HTTP BIO *mem = OSSL_HTTP_get(url, NULL /* proxy */, NULL /* no_proxy */, - bio, rbio, NULL /* cb */, NULL /* arg */, + bio, rbio, NULL /* cb */ , NULL /* arg */, 1024 /* buf_size */, NULL /* headers */, NULL /* expected_ct */, 1 /* expect_asn1 */, OSSL_HTTP_DEFAULT_MAX_RESP_LEN, timeout); @@ -108,9 +100,6 @@ static ASN1_VALUE *simple_get_asn1(const char *url, BIO *bio, BIO *rbio, BIO_free(mem); return res; -#else - return 0; -#endif } X509 *X509_load_http(const char *url, BIO *bio, BIO *rbio, int timeout) @@ -121,11 +110,6 @@ X509 *X509_load_http(const char *url, BIO *bio, BIO *rbio, int timeout) int X509_REQ_sign(X509_REQ *x, EVP_PKEY *pkey, const EVP_MD *md) { - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - x->req_info.enc.modified = 1; return ASN1_item_sign_ex(ASN1_ITEM_rptr(X509_REQ_INFO), &x->sig_alg, NULL, x->signature, &x->req_info, NULL, pkey, md, x->libctx, x->propq); @@ -133,11 +117,6 @@ int X509_REQ_sign(X509_REQ *x, EVP_PKEY *pkey, const EVP_MD *md) int X509_REQ_sign_ctx(X509_REQ *x, EVP_MD_CTX *ctx) { - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - x->req_info.enc.modified = 1; return ASN1_item_sign_ctx(ASN1_ITEM_rptr(X509_REQ_INFO), &x->sig_alg, NULL, x->signature, &x->req_info, ctx); @@ -145,10 +124,6 @@ int X509_REQ_sign_ctx(X509_REQ *x, EVP_MD_CTX *ctx) int X509_CRL_sign(X509_CRL *x, EVP_PKEY *pkey, const EVP_MD *md) { - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } x->crl.enc.modified = 1; return ASN1_item_sign_ex(ASN1_ITEM_rptr(X509_CRL_INFO), &x->crl.sig_alg, &x->sig_alg, &x->signature, &x->crl, NULL, @@ -157,10 +132,6 @@ int X509_CRL_sign(X509_CRL *x, EVP_PKEY *pkey, const EVP_MD *md) int X509_CRL_sign_ctx(X509_CRL *x, EVP_MD_CTX *ctx) { - if (x == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } x->crl.enc.modified = 1; return ASN1_item_sign_ctx(ASN1_ITEM_rptr(X509_CRL_INFO), &x->crl.sig_alg, &x->sig_alg, &x->signature, @@ -175,8 +146,7 @@ X509_CRL *X509_CRL_load_http(const char *url, BIO *bio, BIO *rbio, int timeout) int NETSCAPE_SPKI_sign(NETSCAPE_SPKI *x, EVP_PKEY *pkey, const EVP_MD *md) { - return - ASN1_item_sign_ex(ASN1_ITEM_rptr(NETSCAPE_SPKAC), &x->sig_algor, NULL, + return ASN1_item_sign_ex(ASN1_ITEM_rptr(NETSCAPE_SPKAC), &x->sig_algor, NULL, x->signature, x->spkac, NULL, pkey, md, NULL, NULL); } @@ -259,6 +229,7 @@ PKCS7 *d2i_PKCS7_bio(BIO *bp, PKCS7 **p7) propq = (*p7)->ctx.propq; } + ret = ASN1_item_d2i_bio_ex(ASN1_ITEM_rptr(PKCS7), bp, p7, libctx, propq); if (ret != NULL) ossl_pkcs7_resolve_libctx(ret); @@ -292,8 +263,7 @@ X509_REQ *d2i_X509_REQ_bio(BIO *bp, X509_REQ **req) propq = (*req)->propq; } - return - ASN1_item_d2i_bio_ex(ASN1_ITEM_rptr(X509_REQ), bp, req, libctx, propq); + return ASN1_item_d2i_bio_ex(ASN1_ITEM_rptr(X509_REQ), bp, req, libctx, propq); } int i2d_X509_REQ_bio(BIO *bp, const X509_REQ *req) @@ -456,9 +426,9 @@ int i2d_ECPrivateKey_bio(BIO *bp, const EC_KEY *eckey) int X509_pubkey_digest(const X509 *data, const EVP_MD *type, unsigned char *md, unsigned int *len) { - ASN1_BIT_STRING *key = X509_get0_pubkey_bitstr(data); - - if (key == NULL) + ASN1_BIT_STRING *key; + key = X509_get0_pubkey_bitstr(data); + if (!key) return 0; return EVP_Digest(key->data, key->length, md, len, type, NULL); } @@ -514,7 +484,7 @@ ASN1_OCTET_STRING *X509_digest_sig(const X509 *cert, || !ossl_rsa_pss_get_param_unverified(pss, &mmd, &mgf1md, &saltlen, &trailerfield) - || mmd == NULL) { + || mmd == NULL) { RSA_PSS_PARAMS_free(pss); ERR_raise(ERR_LIB_X509, X509_R_UNSUPPORTED_ALGORITHM); return NULL; @@ -557,7 +527,7 @@ ASN1_OCTET_STRING *X509_digest_sig(const X509 *cert, if (!X509_digest(cert, md, hash, &len) || (new = ASN1_OCTET_STRING_new()) == NULL) goto err; - if (ASN1_OCTET_STRING_set(new, hash, len)) { + if ((ASN1_OCTET_STRING_set(new, hash, len))) { if (md_used != NULL) *md_used = md; else @@ -586,17 +556,15 @@ int X509_CRL_digest(const X509_CRL *data, const EVP_MD *type, memcpy(md, data->sha1_hash, sizeof(data->sha1_hash)); return 1; } - return - ossl_asn1_item_digest_ex(ASN1_ITEM_rptr(X509_CRL), type, (char *)data, - md, len, data->libctx, data->propq); + return ossl_asn1_item_digest_ex(ASN1_ITEM_rptr(X509_CRL), type, (char *)data, + md, len, data->libctx, data->propq); } int X509_REQ_digest(const X509_REQ *data, const EVP_MD *type, unsigned char *md, unsigned int *len) { - return - ossl_asn1_item_digest_ex(ASN1_ITEM_rptr(X509_REQ), type, (char *)data, - md, len, data->libctx, data->propq); + return ossl_asn1_item_digest_ex(ASN1_ITEM_rptr(X509_REQ), type, (char *)data, + md, len, data->libctx, data->propq); } int X509_NAME_digest(const X509_NAME *data, const EVP_MD *type, @@ -718,22 +686,6 @@ int i2d_PUBKEY_fp(FILE *fp, const EVP_PKEY *pkey) return ASN1_i2d_fp_of(EVP_PKEY, i2d_PUBKEY, fp, pkey); } -EVP_PKEY *d2i_PUBKEY_ex_fp(FILE *fp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, - const char *propq) -{ - BIO *b; - void *ret; - - if ((b = BIO_new(BIO_s_file())) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_BUF_LIB); - return NULL; - } - BIO_set_fp(b, fp, BIO_NOCLOSE); - ret = d2i_PUBKEY_ex_bio(b, a, libctx, propq); - BIO_free(b); - return ret; -} - EVP_PKEY *d2i_PUBKEY_fp(FILE *fp, EVP_PKEY **a) { return ASN1_d2i_fp_of(EVP_PKEY, EVP_PKEY_new, d2i_PUBKEY, fp, a); @@ -801,25 +753,6 @@ int i2d_PUBKEY_bio(BIO *bp, const EVP_PKEY *pkey) return ASN1_i2d_bio_of(EVP_PKEY, i2d_PUBKEY, bp, pkey); } -EVP_PKEY *d2i_PUBKEY_ex_bio(BIO *bp, EVP_PKEY **a, OSSL_LIB_CTX *libctx, - const char *propq) -{ - BUF_MEM *b = NULL; - const unsigned char *p; - void *ret = NULL; - int len; - - len = asn1_d2i_read_bio(bp, &b); - if (len < 0) - goto err; - - p = (unsigned char *)b->data; - ret = d2i_PUBKEY_ex(a, &p, len, libctx, propq); - err: - BUF_MEM_free(b); - return ret; -} - EVP_PKEY *d2i_PUBKEY_bio(BIO *bp, EVP_PKEY **a) { return ASN1_d2i_bio_of(EVP_PKEY, EVP_PKEY_new, d2i_PUBKEY, bp, a); diff --git a/openssl/src/crypto/x509/x_crl.c b/openssl/src/crypto/x509/x_crl.c index 2601a019f..d77746a2b 100644 --- a/openssl/src/crypto/x509/x_crl.c +++ b/openssl/src/crypto/x509/x_crl.c @@ -20,9 +20,9 @@ static int X509_REVOKED_cmp(const X509_REVOKED *const *a, static int setup_idp(X509_CRL *crl, ISSUING_DIST_POINT *idp); ASN1_SEQUENCE(X509_REVOKED) = { - ASN1_EMBED(X509_REVOKED, serialNumber, ASN1_INTEGER), - ASN1_SIMPLE(X509_REVOKED, revocationDate, ASN1_TIME), - ASN1_SEQUENCE_OF_OPT(X509_REVOKED, extensions, X509_EXTENSION) + ASN1_EMBED(X509_REVOKED,serialNumber, ASN1_INTEGER), + ASN1_SIMPLE(X509_REVOKED,revocationDate, ASN1_TIME), + ASN1_SEQUENCE_OF_OPT(X509_REVOKED,extensions, X509_EXTENSION) } ASN1_SEQUENCE_END(X509_REVOKED) static int def_crl_verify(X509_CRL *crl, EVP_PKEY *r); @@ -94,37 +94,32 @@ static int crl_set_issuers(X509_CRL *crl) STACK_OF(X509_EXTENSION) *exts; ASN1_ENUMERATED *reason; X509_EXTENSION *ext; - gtmp = X509_REVOKED_get_ext_d2i(rev, NID_certificate_issuer, &j, NULL); - if (gtmp == NULL && j != -1) { + if (!gtmp && (j != -1)) { crl->flags |= EXFLAG_INVALID; return 1; } - if (gtmp != NULL) { - if (crl->issuers == NULL) { + if (gtmp) { + gens = gtmp; + if (!crl->issuers) { crl->issuers = sk_GENERAL_NAMES_new_null(); - if (crl->issuers == NULL) { - GENERAL_NAMES_free(gtmp); + if (!crl->issuers) return 0; - } } - if (!sk_GENERAL_NAMES_push(crl->issuers, gtmp)) { - GENERAL_NAMES_free(gtmp); + if (!sk_GENERAL_NAMES_push(crl->issuers, gtmp)) return 0; - } - gens = gtmp; } rev->issuer = gens; reason = X509_REVOKED_get_ext_d2i(rev, NID_crl_reason, &j, NULL); - if (reason == NULL && j != -1) { + if (!reason && (j != -1)) { crl->flags |= EXFLAG_INVALID; return 1; } - if (reason != NULL) { + if (reason) { rev->reason = ASN1_ENUMERATED_get(reason); ASN1_ENUMERATED_free(reason); } else @@ -137,8 +132,7 @@ static int crl_set_issuers(X509_CRL *crl) for (j = 0; j < sk_X509_EXTENSION_num(exts); j++) { ext = sk_X509_EXTENSION_value(exts, j); if (X509_EXTENSION_get_critical(ext)) { - if (OBJ_obj2nid(X509_EXTENSION_get_object(ext)) - == NID_certificate_issuer) + if (OBJ_obj2nid(X509_EXTENSION_get_object(ext)) == NID_certificate_issuer) continue; crl->flags |= EXFLAG_CRITICAL; break; @@ -174,7 +168,7 @@ static int crl_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, ASN1_INTEGER_free(crl->crl_number); ASN1_INTEGER_free(crl->base_crl_number); sk_GENERAL_NAMES_pop_free(crl->issuers, GENERAL_NAMES_free); - /* fall through */ + /* fall thru */ case ASN1_OP_NEW_POST: crl->idp = NULL; @@ -261,7 +255,7 @@ static int crl_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, break; case ASN1_OP_FREE_POST: - if (crl->meth != NULL && crl->meth->crl_free != NULL) { + if (crl->meth->crl_free) { if (!crl->meth->crl_free(crl)) return 0; } @@ -366,7 +360,7 @@ int X509_CRL_add0_revoked(X509_CRL *crl, X509_REVOKED *rev) if (inf->revoked == NULL) inf->revoked = sk_X509_REVOKED_new(X509_REVOKED_cmp); if (inf->revoked == NULL || !sk_X509_REVOKED_push(inf->revoked, rev)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } inf->enc.modified = 1; @@ -490,8 +484,10 @@ X509_CRL_METHOD *X509_CRL_METHOD_new(int (*crl_init) (X509_CRL *crl), { X509_CRL_METHOD *m = OPENSSL_malloc(sizeof(*m)); - if (m == NULL) + if (m == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; + } m->crl_init = crl_init; m->crl_free = crl_free; m->crl_lookup = crl_lookup; diff --git a/openssl/src/crypto/x509/x_name.c b/openssl/src/crypto/x509/x_name.c index 4568833f8..1e62f4130 100644 --- a/openssl/src/crypto/x509/x_name.c +++ b/openssl/src/crypto/x509/x_name.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -92,20 +92,17 @@ static int x509_name_ex_new(ASN1_VALUE **val, const ASN1_ITEM *it) X509_NAME *ret = OPENSSL_zalloc(sizeof(*ret)); if (ret == NULL) - return 0; - if ((ret->entries = sk_X509_NAME_ENTRY_new_null()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_CRYPTO_LIB); - goto err; - } - if ((ret->bytes = BUF_MEM_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB); - goto err; - } + goto memerr; + if ((ret->entries = sk_X509_NAME_ENTRY_new_null()) == NULL) + goto memerr; + if ((ret->bytes = BUF_MEM_new()) == NULL) + goto memerr; ret->modified = 1; *val = (ASN1_VALUE *)ret; return 1; - err: + memerr: + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); if (ret) { sk_X509_NAME_ENTRY_free(ret->entries); OPENSSL_free(ret); @@ -249,28 +246,26 @@ static int x509_name_encode(X509_NAME *a) intname.s = sk_STACK_OF_X509_NAME_ENTRY_new_null(); if (!intname.s) - goto cerr; + goto memerr; for (i = 0; i < sk_X509_NAME_ENTRY_num(a->entries); i++) { entry = sk_X509_NAME_ENTRY_value(a->entries, i); if (entry->set != set) { entries = sk_X509_NAME_ENTRY_new_null(); if (!entries) - goto cerr; + goto memerr; if (!sk_STACK_OF_X509_NAME_ENTRY_push(intname.s, entries)) { sk_X509_NAME_ENTRY_free(entries); - goto cerr; + goto memerr; } set = entry->set; } if (!sk_X509_NAME_ENTRY_push(entries, entry)) - goto cerr; + goto memerr; } len = ASN1_item_ex_i2d(&intname.a, NULL, ASN1_ITEM_rptr(X509_NAME_INTERNAL), -1, -1); - if (!BUF_MEM_grow(a->bytes, len)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_BUF_LIB); - goto err; - } + if (!BUF_MEM_grow(a->bytes, len)) + goto memerr; p = (unsigned char *)a->bytes->data; ASN1_item_ex_i2d(&intname.a, &p, ASN1_ITEM_rptr(X509_NAME_INTERNAL), -1, -1); @@ -278,11 +273,10 @@ static int x509_name_encode(X509_NAME *a) local_sk_X509_NAME_ENTRY_free); a->modified = 0; return len; - cerr: - ERR_raise(ERR_LIB_ASN1, ERR_R_CRYPTO_LIB); - err: + memerr: sk_STACK_OF_X509_NAME_ENTRY_pop_free(intname.s, local_sk_X509_NAME_ENTRY_free); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } @@ -324,7 +318,7 @@ static int x509_name_canon(X509_NAME *a) } intname = sk_STACK_OF_X509_NAME_ENTRY_new_null(); if (intname == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } for (i = 0; i < sk_X509_NAME_ENTRY_num(a->entries); i++) { @@ -335,25 +329,25 @@ static int x509_name_canon(X509_NAME *a) goto err; if (!sk_STACK_OF_X509_NAME_ENTRY_push(intname, entries)) { sk_X509_NAME_ENTRY_free(entries); - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } set = entry->set; } tmpentry = X509_NAME_ENTRY_new(); if (tmpentry == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } tmpentry->object = OBJ_dup(entry->object); if (tmpentry->object == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_OBJ_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } if (!asn1_string_canon(tmpentry->value, entry->value)) goto err; if (!sk_X509_NAME_ENTRY_push(entries, tmpentry)) { - ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; } tmpentry = NULL; @@ -366,8 +360,10 @@ static int x509_name_canon(X509_NAME *a) a->canon_enclen = len; p = OPENSSL_malloc(a->canon_enclen); - if (p == NULL) + if (p == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto err; + } a->canon_enc = p; diff --git a/openssl/src/crypto/x509/x_pubkey.c b/openssl/src/crypto/x509/x_pubkey.c index 004c7bdfe..bc90ddd89 100644 --- a/openssl/src/crypto/x509/x_pubkey.c +++ b/openssl/src/crypto/x509/x_pubkey.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -66,8 +66,7 @@ ASN1_SEQUENCE(X509_PUBKEY_INTERNAL) = { } static_ASN1_SEQUENCE_END_name(X509_PUBKEY, X509_PUBKEY_INTERNAL) X509_PUBKEY *ossl_d2i_X509_PUBKEY_INTERNAL(const unsigned char **pp, - long len, OSSL_LIB_CTX *libctx, - const char *propq) + long len, OSSL_LIB_CTX *libctx) { X509_PUBKEY *xpub = OPENSSL_zalloc(sizeof(*xpub)); @@ -75,7 +74,7 @@ X509_PUBKEY *ossl_d2i_X509_PUBKEY_INTERNAL(const unsigned char **pp, return NULL; return (X509_PUBKEY *)ASN1_item_d2i_ex((ASN1_VALUE **)&xpub, pp, len, ASN1_ITEM_rptr(X509_PUBKEY_INTERNAL), - libctx, propq); + libctx, NULL); } void ossl_X509_PUBKEY_INTERNAL_free(X509_PUBKEY *xpub) @@ -113,13 +112,12 @@ static int x509_pubkey_ex_new_ex(ASN1_VALUE **pval, const ASN1_ITEM *it, { X509_PUBKEY *ret; - if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) - return 0; - if (!x509_pubkey_ex_populate((ASN1_VALUE **)&ret, NULL) + if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL + || !x509_pubkey_ex_populate((ASN1_VALUE **)&ret, NULL) || !x509_pubkey_set0_libctx(ret, libctx, propq)) { x509_pubkey_ex_free((ASN1_VALUE **)&ret, NULL); ret = NULL; - ERR_raise(ERR_LIB_ASN1, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); } else { *pval = (ASN1_VALUE *)ret; } @@ -143,7 +141,7 @@ static int x509_pubkey_ex_d2i_ex(ASN1_VALUE **pval, if (*pval == NULL && !x509_pubkey_ex_new_ex(pval, it, libctx, propq)) return 0; if (!x509_pubkey_ex_populate(pval, NULL)) { - ERR_raise(ERR_LIB_ASN1, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; } @@ -172,7 +170,7 @@ static int x509_pubkey_ex_d2i_ex(ASN1_VALUE **pval, /* * Try to decode with legacy method first. This ensures that engines - * aren't overridden by providers. + * aren't overriden by providers. */ if ((ret = x509_pubkey_decode(&pubkey->pkey, pubkey)) == -1) { /* -1 indicates a fatal error, like malloc failure */ @@ -192,8 +190,10 @@ static int x509_pubkey_ex_d2i_ex(ASN1_VALUE **pval, */ if (aclass != V_ASN1_UNIVERSAL) { tmpbuf = OPENSSL_memdup(in_saved, publen); - if (tmpbuf == NULL) + if (tmpbuf == NULL) { + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return 0; + } in_saved = tmpbuf; *tmpbuf = V_ASN1_CONSTRUCTED | V_ASN1_SEQUENCE; } @@ -284,22 +284,16 @@ X509_PUBKEY *X509_PUBKEY_dup(const X509_PUBKEY *a) { X509_PUBKEY *pubkey = OPENSSL_zalloc(sizeof(*pubkey)); - if (pubkey == NULL) - return NULL; - if (!x509_pubkey_set0_libctx(pubkey, a->libctx, a->propq)) { - ERR_raise(ERR_LIB_X509, ERR_R_X509_LIB); - x509_pubkey_ex_free((ASN1_VALUE **)&pubkey, - ASN1_ITEM_rptr(X509_PUBKEY_INTERNAL)); - return NULL; - } - if ((pubkey->algor = X509_ALGOR_dup(a->algor)) == NULL - || (pubkey->public_key = ASN1_BIT_STRING_new()) == NULL - || !ASN1_BIT_STRING_set(pubkey->public_key, - a->public_key->data, - a->public_key->length)) { + if (pubkey == NULL + || !x509_pubkey_set0_libctx(pubkey, a->libctx, a->propq) + || (pubkey->algor = X509_ALGOR_dup(a->algor)) == NULL + || (pubkey->public_key = ASN1_BIT_STRING_new()) == NULL + || !ASN1_BIT_STRING_set(pubkey->public_key, + a->public_key->data, + a->public_key->length)) { x509_pubkey_ex_free((ASN1_VALUE **)&pubkey, ASN1_ITEM_rptr(X509_PUBKEY_INTERNAL)); - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; } @@ -331,7 +325,7 @@ int X509_PUBKEY_set(X509_PUBKEY **x, EVP_PKEY *pkey) if (pkey->ameth != NULL) { if ((pk = X509_PUBKEY_new()) == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); goto error; } if (pkey->ameth->pub_encode != NULL) { @@ -422,7 +416,7 @@ static int x509_pubkey_decode(EVP_PKEY **ppkey, const X509_PUBKEY *key) pkey = EVP_PKEY_new(); if (pkey == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return -1; } @@ -505,8 +499,10 @@ static EVP_PKEY *d2i_PUBKEY_int(EVP_PKEY **a, */ if (libctx != NULL || propq != NULL || force_legacy) { xpk2 = OPENSSL_zalloc(sizeof(*xpk2)); - if (xpk2 == NULL) + if (xpk2 == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return NULL; + } if (!x509_pubkey_set0_libctx(xpk2, libctx, propq)) goto end; xpk2->flag_force_legacy = !!force_legacy; @@ -632,7 +628,7 @@ int i2d_RSA_PUBKEY(const RSA *a, unsigned char **pp) return 0; pktmp = EVP_PKEY_new(); if (pktmp == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign_RSA(pktmp, (RSA *)a); @@ -674,7 +670,7 @@ int ossl_i2d_DH_PUBKEY(const DH *a, unsigned char **pp) return 0; pktmp = EVP_PKEY_new(); if (pktmp == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign_DH(pktmp, (DH *)a); @@ -715,7 +711,7 @@ int ossl_i2d_DHx_PUBKEY(const DH *a, unsigned char **pp) return 0; pktmp = EVP_PKEY_new(); if (pktmp == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign(pktmp, EVP_PKEY_DHX, (DH *)a); @@ -749,30 +745,6 @@ DSA *d2i_DSA_PUBKEY(DSA **a, const unsigned char **pp, long length) return key; } -/* Called from decoders; disallows provided DSA keys without parameters. */ -DSA *ossl_d2i_DSA_PUBKEY(DSA **a, const unsigned char **pp, long length) -{ - DSA *key = NULL; - const unsigned char *data; - const BIGNUM *p, *q, *g; - - data = *pp; - key = d2i_DSA_PUBKEY(NULL, &data, length); - if (key == NULL) - return NULL; - DSA_get0_pqg(key, &p, &q, &g); - if (p == NULL || q == NULL || g == NULL) { - DSA_free(key); - return NULL; - } - *pp = data; - if (a != NULL) { - DSA_free(*a); - *a = key; - } - return key; -} - int i2d_DSA_PUBKEY(const DSA *a, unsigned char **pp) { EVP_PKEY *pktmp; @@ -781,7 +753,7 @@ int i2d_DSA_PUBKEY(const DSA *a, unsigned char **pp) return 0; pktmp = EVP_PKEY_new(); if (pktmp == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign_DSA(pktmp, (DSA *)a); @@ -826,7 +798,7 @@ int i2d_EC_PUBKEY(const EC_KEY *a, unsigned char **pp) if (a == NULL) return 0; if ((pktmp = EVP_PKEY_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign_EC_KEY(pktmp, (EC_KEY *)a); @@ -836,7 +808,6 @@ int i2d_EC_PUBKEY(const EC_KEY *a, unsigned char **pp) return ret; } -# ifndef OPENSSL_NO_ECX ECX_KEY *ossl_d2i_ED25519_PUBKEY(ECX_KEY **a, const unsigned char **pp, long length) { @@ -868,7 +839,7 @@ int ossl_i2d_ED25519_PUBKEY(const ECX_KEY *a, unsigned char **pp) if (a == NULL) return 0; if ((pktmp = EVP_PKEY_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign(pktmp, EVP_PKEY_ED25519, (ECX_KEY *)a); @@ -910,7 +881,7 @@ int ossl_i2d_ED448_PUBKEY(const ECX_KEY *a, unsigned char **pp) if (a == NULL) return 0; if ((pktmp = EVP_PKEY_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign(pktmp, EVP_PKEY_ED448, (ECX_KEY *)a); @@ -952,7 +923,7 @@ int ossl_i2d_X25519_PUBKEY(const ECX_KEY *a, unsigned char **pp) if (a == NULL) return 0; if ((pktmp = EVP_PKEY_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign(pktmp, EVP_PKEY_X25519, (ECX_KEY *)a); @@ -994,7 +965,7 @@ int ossl_i2d_X448_PUBKEY(const ECX_KEY *a, unsigned char **pp) if (a == NULL) return 0; if ((pktmp = EVP_PKEY_new()) == NULL) { - ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE); return -1; } (void)EVP_PKEY_assign(pktmp, EVP_PKEY_X448, (ECX_KEY *)a); @@ -1004,24 +975,22 @@ int ossl_i2d_X448_PUBKEY(const ECX_KEY *a, unsigned char **pp) return ret; } -# endif /* OPENSSL_NO_ECX */ #endif -void X509_PUBKEY_set0_public_key(X509_PUBKEY *pub, - unsigned char *penc, int penclen) -{ - ASN1_STRING_set0(pub->public_key, penc, penclen); - ossl_asn1_string_set_bits_left(pub->public_key, 0); -} - int X509_PUBKEY_set0_param(X509_PUBKEY *pub, ASN1_OBJECT *aobj, int ptype, void *pval, unsigned char *penc, int penclen) { if (!X509_ALGOR_set0(pub->algor, aobj, ptype, pval)) return 0; - if (penc != NULL) - X509_PUBKEY_set0_public_key(pub, penc, penclen); + if (penc) { + OPENSSL_free(pub->public_key->data); + pub->public_key->data = penc; + pub->public_key->length = penclen; + /* Set number of unused bits to zero */ + pub->public_key->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07); + pub->public_key->flags |= ASN1_STRING_FLAG_BITS_LEFT; + } return 1; } diff --git a/openssl/src/crypto/x509/x_req.c b/openssl/src/crypto/x509/x_req.c index ca712386b..293d4be71 100644 --- a/openssl/src/crypto/x509/x_req.c +++ b/openssl/src/crypto/x509/x_req.c @@ -53,7 +53,7 @@ static int req_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, switch (operation) { case ASN1_OP_D2I_PRE: ASN1_OCTET_STRING_free(ret->distinguishing_id); - /* fall through */ + /* fall thru */ case ASN1_OP_NEW_POST: ret->distinguishing_id = NULL; break; @@ -74,7 +74,7 @@ static int req_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, if (pkey != NULL) { pkey = EVP_PKEY_dup(pkey); if (pkey == NULL) { - ERR_raise(ERR_LIB_X509, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return 0; } if (!X509_PUBKEY_set(&ret->req_info.pubkey, pkey)) { diff --git a/openssl/src/crypto/x509/x_x509.c b/openssl/src/crypto/x509/x_x509.c index 75c5c9223..010578b19 100644 --- a/openssl/src/crypto/x509/x_x509.c +++ b/openssl/src/crypto/x509/x_x509.c @@ -55,7 +55,7 @@ static int x509_cb(int operation, ASN1_VALUE **pval, const ASN1_ITEM *it, #endif ASN1_OCTET_STRING_free(ret->distinguishing_id); - /* fall through */ + /* fall thru */ case ASN1_OP_NEW_POST: ret->ex_cached = 0; @@ -272,8 +272,10 @@ int i2d_X509_AUX(const X509 *a, unsigned char **pp) /* Allocate requisite combined storage */ *pp = tmp = OPENSSL_malloc(length); - if (tmp == NULL) + if (tmp == NULL) { + ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE); return -1; + } /* Encode, but keep *pp at the originally malloced pointer */ length = i2d_x509_aux_internal(a, &tmp); diff --git a/openssl/src/crypto/zkp/bulletproofs/bp_debug.c b/openssl/src/crypto/zkp/bulletproofs/bp_debug.c new file mode 100644 index 000000000..7d9e18636 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/bp_debug.c @@ -0,0 +1,172 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include +#include "bp_debug.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) +DEFINE_STACK_OF(BP_VARIABLE) + +void BP_WITNESS_debug_print(BP_WITNESS *witness, const char *note) +{ + BIO *bio = NULL; + + if (!(bio = BIO_new(BIO_s_file()))) + goto err; + + BIO_set_fp(bio, stderr, BIO_NOCLOSE); + + BIO_printf(bio, "%s: \n", note); + BIO_printf(bio, "witness->n: %d\n", sk_BP_VARIABLE_num(witness->sk_V)); + + bp_stack_of_variable_debug_print(bio, witness->sk_V, "witness->sk_V"); + zkp_stack_of_bignum_debug_print(bio, witness->sk_r, "witness->sk_r"); + zkp_stack_of_bignum_debug_print(bio, witness->sk_v, "witness->sk_v"); + +err: + BIO_free(bio); +} + +void BP_RANGE_PROOF_debug_print(BP_RANGE_PROOF *proof, const EC_GROUP *group, const char *note) +{ + BIO *bio = NULL; + BN_CTX *bn_ctx = NULL; + + if (!(bio = BIO_new(BIO_s_file()))) + goto err; + + BIO_set_fp(bio, stderr, BIO_NOCLOSE); + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + + BIO_printf(bio, "%s: \n", note); + + EC_POINT_debug_print_affine(bio, group, proof->A, "proof->A", bn_ctx); + EC_POINT_debug_print_affine(bio, group, proof->S, "proof->S", bn_ctx); + EC_POINT_debug_print_affine(bio, group, proof->T1, "proof->T1", bn_ctx); + EC_POINT_debug_print_affine(bio, group, proof->T2, "proof->T2", bn_ctx); + BN_debug_print(bio, proof->taux, "proof->taux"); + BN_debug_print(bio, proof->mu, "proof->mu"); + BN_debug_print(bio, proof->tx, "proof->tx"); + bp_inner_product_proof_debug_print(proof->ip_proof, group, "ip_proof"); + +err: + BN_CTX_free(bn_ctx); + BIO_free(bio); +} + +void bp_inner_product_pub_param_debug_print(bp_inner_product_pub_param_t *pp, + const char *note) +{ + BIO *bio = NULL; + int curve_id; + + if (!(bio = BIO_new(BIO_s_file()))) + goto err; + + BIO_set_fp(bio, stderr, BIO_NOCLOSE); + + curve_id = EC_GROUP_get_curve_name(pp->group); + + BIO_printf(bio, "%s: \n", note); + BIO_printf(bio, "ip_pp->curve_id: %zu\n", curve_id); + BIO_printf(bio, "ip_pp->n: %zu\n", sk_EC_POINT_num(pp->sk_G)); + + zkp_stack_of_point_debug_print(bio, pp->sk_G, "ip_pp->sk_G"); + zkp_stack_of_point_debug_print(bio, pp->sk_H, "ip_pp->sk_H"); + +err: + BIO_free(bio); +} + +void bp_inner_product_witness_debug_print(bp_inner_product_witness_t *witness, + const char *note) +{ + BIO *bio = NULL; + + if (!(bio = BIO_new(BIO_s_file()))) + goto err; + + BIO_set_fp(bio, stderr, BIO_NOCLOSE); + + BIO_printf(bio, "%s: \n", note); + BIO_printf(bio, "ip_witness->n: %zu\n", sk_BIGNUM_num(witness->sk_a)); + + zkp_stack_of_bignum_debug_print(bio, witness->sk_a, "ip_witness->sk_a"); + zkp_stack_of_bignum_debug_print(bio, witness->sk_b, "ip_witness->sk_b"); + +err: + BIO_free(bio); +} + +void bp_inner_product_proof_debug_print(bp_inner_product_proof_t *proof, + const EC_GROUP *group, const char *note) +{ + BIO *bio = NULL; + + if (!(bio = BIO_new(BIO_s_file()))) + goto err; + + BIO_set_fp(bio, stderr, BIO_NOCLOSE); + + BIO_printf(bio, "%s: \n", note); + BIO_printf(bio, "ip_proof->n: %zu\n", sk_EC_POINT_num(proof->sk_L)); + + zkp_stack_of_point_debug_print(bio, proof->sk_L, "ip_proof->sk_L"); + zkp_stack_of_point_debug_print(bio, proof->sk_R, "ip_proof->sk_R"); + + BN_debug_print(bio, proof->a, "ip_proof->a"); + BN_debug_print(bio, proof->b, "ip_proof->b"); + +err: + BIO_free(bio); +} + +void bp_stack_of_variable_debug_print(BIO *bio, STACK_OF(BP_VARIABLE) *sk, const char *name) +{ + BIO *b = NULL; + int i, n; + EC_POINT *V; + BP_VARIABLE *var; + + if (sk == NULL) + return; + + if (bio == NULL) { + b = bio = BIO_new(BIO_s_file()); + BIO_set_fp(b, stderr, BIO_NOCLOSE); + } + + n = sk_BP_VARIABLE_num(sk); + for (i = 0; i < n; i++) { + var = sk_BP_VARIABLE_value(sk, i); + if (var == NULL) + goto err; + + V = var->point; + + BIO_printf(b, "%s[%d], name: %s, X: ", name, var->name, i); + BN_print(b, V->X); + BIO_printf(b, ", Y: "); + BN_print(b, V->Y); + BIO_printf(b, ", Z: "); + BN_print(b, V->Z); + BIO_printf(b, "\n"); + } + +err: + BIO_free(b); +} diff --git a/openssl/src/crypto/zkp/bulletproofs/bp_debug.h b/openssl/src/crypto/zkp/bulletproofs/bp_debug.h new file mode 100644 index 000000000..dd9494e17 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/bp_debug.h @@ -0,0 +1,46 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_BP_DEBUG_LOCAL_H +# define HEADER_BP_DEBUG_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include "internal/refcount.h" +# include "bulletproofs.h" +# include "range_proof.h" +# include "inner_product.h" + +STACK_OF(EC_POINT); +STACK_OF(BP_VARIABLE); + +void BP_PUB_PARAM_debug_print(BP_PUB_PARAM *pp, const char *note); +void BP_WITNESS_debug_print(BP_WITNESS *witness, const char *note); +void BP_RANGE_PROOF_debug_print(BP_RANGE_PROOF *proof, const EC_GROUP *group, const char *note); + +void bp_inner_product_pub_param_debug_print(bp_inner_product_pub_param_t *pp, + const char *note); +void bp_inner_product_witness_debug_print(bp_inner_product_witness_t *witness, + const char *note); +void bp_inner_product_proof_debug_print(bp_inner_product_proof_t *proof, + const EC_GROUP *group, const char *note); +void bp_stack_of_variable_debug_print(BIO *bio, STACK_OF(BP_VARIABLE) *sk, const char *name); + +# ifdef __cplusplus +} +# endif + +#endif + diff --git a/openssl/src/crypto/zkp/bulletproofs/bp_err.c b/openssl/src/crypto/zkp/bulletproofs/bp_err.c new file mode 100644 index 000000000..9bef19160 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/bp_err.c @@ -0,0 +1,59 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include +#include "crypto/zkpbperr.h" + +#ifndef OPENSSL_NO_ERR + +static const ERR_STRING_DATA ZKP_BP_str_reasons[] = { + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_EXCEEDS_GENS_CAPACITY), + "exceeds gens capacity"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_EXCEEDS_MAX_AGG_NUM), + "exceeds max agg num"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_EXCEEDS_MAX_BITS), + "exceeds max bits"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_EXCEEDS_PARTY_CAPACITY), + "exceeds party capacity"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_EXCEEDS_PP_CAPACITY), + "exceeds pp capacity"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_FORMAT_ERROR), + "r1cs constraint expression format error"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_NO_VAR), + "r1cs constraint expression no var"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_PROCESS_ERROR), + "r1cs constraint expression process error"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_NOT_FOUND), + "r1cs constraint expression var not found"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_TOO_LONG), + "r1cs constraint expression var too long"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_RANGE_LEN_MUST_BE_POWER_OF_TWO), + "range len must be power of two"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_TRANSCRIPT_INIT_FAILED), + "transcript init failed"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_VARIABLE_DUPLICATED), + "variable duplicated"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_VARIABLE_NAME_TOO_LONG), + "variable name too long"}, + {ERR_PACK(ERR_LIB_ZKP_BP, 0, ZKP_BP_R_WITNESS_INVALID), "witness invalid"}, + {0, NULL} +}; + +#endif + +int ossl_err_load_ZKP_BP_strings(void) +{ +#ifndef OPENSSL_NO_ERR + if (ERR_reason_error_string(ZKP_BP_str_reasons[0].error) == NULL) + ERR_load_strings_const(ZKP_BP_str_reasons); +#endif + return 1; +} diff --git a/openssl/src/crypto/zkp/bulletproofs/bulletproofs.c b/openssl/src/crypto/zkp/bulletproofs/bulletproofs.c new file mode 100644 index 000000000..9c1a1e424 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/bulletproofs.c @@ -0,0 +1,520 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "bulletproofs.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) +DEFINE_STACK_OF(BP_VARIABLE) + +/** Creates a new BP_PUB_PARAM object + * \param group underlying EC_GROUP object + * \param gens_capacity the number of generators to precompute for each party. + * For range_proof, it is the maximum bitsize of the + * range_proof, maximum value is 64. For r1cs_proof, + * the capacity must be greater than the number of + * multipliers, rounded up to the next power of two. + * \param party_capacity the maximum number of parties that can produce on + * aggregated proof. For r1cs_proof, set to 1. + * \return newly created BP_PUB_PARAM object or NULL in case of an error + */ +BP_PUB_PARAM *BP_PUB_PARAM_new(const EC_GROUP *group, int gens_capacity, + int party_capacity) +{ + int i, n; + size_t plen; + unsigned char *pstr = NULL; + BN_CTX *bn_ctx = NULL; + const EC_POINT *G = NULL; + EC_POINT *P = NULL; + BP_PUB_PARAM *pp = NULL; + point_conversion_form_t format = POINT_CONVERSION_COMPRESSED; + + if (group == NULL || gens_capacity <= 0 || party_capacity <= 0) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (gens_capacity > BULLET_PROOF_MAX_GENS_CAPACITY) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_EXCEEDS_GENS_CAPACITY); + return NULL; + } + + if (party_capacity > BULLET_PROOF_MAX_PARTY_CAPACITY) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_EXCEEDS_PARTY_CAPACITY); + return NULL; + } + + pp = OPENSSL_zalloc(sizeof(*pp)); + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!(pp->group = EC_GROUP_dup(group))) + goto err; + + G = EC_GROUP_get0_generator(group); + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + pp->H = EC_POINT_new(group); + if (pp->H == NULL) + goto err; + + plen = EC_POINT_point2oct(group, G, format, NULL, 0, bn_ctx); + if (plen <= 0) + goto err; + + pstr = OPENSSL_zalloc(plen); + if (pstr == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (EC_POINT_point2oct(group, G, format, pstr, plen, bn_ctx) <= 0) + goto err; + + if (!zkp_str2point(group, pstr, plen, pp->H, bn_ctx)) + goto err; + + if (!(pp->U = zkp_random_ec_point_new(group, bn_ctx))) + goto err; + + pp->gens_capacity = gens_capacity; + pp->party_capacity = party_capacity; + n = gens_capacity * party_capacity; + + if (!(pp->sk_G = sk_EC_POINT_new_reserve(NULL, n)) + || !(pp->sk_H = sk_EC_POINT_new_reserve(NULL, n))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + for (i = 0; i < n; i++) { + P = zkp_random_ec_point_new(group, bn_ctx); + if (P == NULL) + goto err; + + if (sk_EC_POINT_push(pp->sk_G, P) <= 0) + goto err; + + P = zkp_random_ec_point_new(group, bn_ctx); + if (P == NULL) + goto err; + + if (sk_EC_POINT_push(pp->sk_H, P) <= 0) + goto err; + } + + pp->references = 1; + if ((pp->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + OPENSSL_free(pstr); + BN_CTX_free(bn_ctx); + return pp; + +err: + EC_POINT_free(P); + OPENSSL_free(pstr); + BN_CTX_free(bn_ctx); + BP_PUB_PARAM_free(pp); + return NULL; +} + +/** Creates a new BP_PUB_PARAM object by curve name + * \param curve_name the elliptic curve name + * \param gens_capacity the number of generators to precompute for each party. + * For range_proof, it is the maximum bitsize of the + * range_proof, maximum value is 64. For r1cs_proof, + * the capacity must be greater than the number of + * multipliers, rounded up to the next power of two. + * \param party_capacity the maximum number of parties that can produce on + * aggregated proof. For r1cs_proof, set to 1. + * \return newly created BP_PUB_PARAM object or NULL in case of an error + */ +BP_PUB_PARAM *BP_PUB_PARAM_new_by_curve_name(const char *curve_name, + int gens_capacity, + int party_capacity) +{ + int curve_id = ossl_ec_curve_name2nid(curve_name); + + if (curve_id == NID_undef) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + return BP_PUB_PARAM_new_by_curve_id(curve_id, gens_capacity, party_capacity); +} + +/** Creates a new BP_PUB_PARAM object by curve id + * \param curve_id the elliptic curve id + * \param gens_capacity the number of generators to precompute for each party. + * For range_proof, it is the maximum bitsize of the + * range_proof, maximum value is 64. For r1cs_proof, + * the capacity must be greater than the number of + * multipliers, rounded up to the next power of two. + * \param party_capacity the maximum number of parties that can produce on + * aggregated proof. For r1cs_proof, set to 1. + * \return newly created BP_PUB_PARAM object or NULL in case of an error + */ +BP_PUB_PARAM *BP_PUB_PARAM_new_by_curve_id(int curve_id, + int gens_capacity, + int party_capacity) +{ + BP_PUB_PARAM *ret; + EC_GROUP *group = NULL; + + if (!(group = EC_GROUP_new_by_curve_name(curve_id))) + return NULL; + + ret = BP_PUB_PARAM_new(group, gens_capacity, party_capacity); + + EC_GROUP_free(group); + + return ret; +} + +/** Frees a BP_PUB_PARAM object + * \param pp BP_PUB_PARAM object to be freed + */ +void BP_PUB_PARAM_free(BP_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return; + + CRYPTO_DOWN_REF(&pp->references, &ref, pp->lock); + REF_PRINT_COUNT("BP_PUB_PARAM", pp); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + sk_EC_POINT_pop_free(pp->sk_G, EC_POINT_free); + sk_EC_POINT_pop_free(pp->sk_H, EC_POINT_free); + EC_POINT_free(pp->U); + EC_POINT_free(pp->H); + EC_GROUP_free(pp->group); + CRYPTO_THREAD_lock_free(pp->lock); + OPENSSL_clear_free((void *)pp, sizeof(*pp)); +} + +/** Increases the internal reference count of a BP_PUB_PARAM object. + * \param pp BP_PUB_PARAM object + * \return 1 on success and 0 if an error occurred. + */ +int BP_PUB_PARAM_up_ref(BP_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return 0; + + if (CRYPTO_UP_REF(&pp->references, &ref, pp->lock) <= 0) + return 0; + + REF_PRINT_COUNT("BP_PUB_PARAM", pp); + REF_ASSERT_ISNT(ref < 2); + return ((ref > 1) ? 1 : 0); +} + +/** Decreases the internal reference count of a BP_PUB_PARAM object. + * \param pp BP_PUB_PARAM object + * \return 1 on success and 0 if an error occurred. + */ +int BP_PUB_PARAM_down_ref(BP_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return 0; + + if (CRYPTO_DOWN_REF(&pp->references, &ref, pp->lock) <= 0) + return 0; + + REF_PRINT_COUNT("BP_PUB_PARAM", pp); + REF_ASSERT_ISNT(ref < 0); + return ((ref > 0) ? 1 : 0); +} + +/** Creates a new BP_VARIABLE object + * \param name the bulletproofs variable name, used for indexing. + * \param point EC_POINT object + * \param group EC_GROUP object + * \return newly created BP_WITNESS object or NULL in case of an error + */ +BP_VARIABLE *BP_VARIABLE_new(const char *name, const EC_POINT *point, + const EC_GROUP *group) +{ + BP_VARIABLE *ret = NULL; + + if (!(ret = OPENSSL_zalloc(sizeof(*ret)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ret->point = EC_POINT_dup(point, group); + + if (name != NULL) + ret->name = OPENSSL_strdup(name); + + if (ret->point == NULL) + goto err; + + return ret; +err: + BP_VARIABLE_free(ret); + return NULL; +} + +/** Frees a BP_VARIABLE object + * \param var BP_VARIABLE object to be freed + */ +void BP_VARIABLE_free(BP_VARIABLE *var) +{ + if (var == NULL) + return; + + EC_POINT_free(var->point); + OPENSSL_free(var->name); + OPENSSL_free(var); +} + +/** Creates a new BP_WITNESS object + * \param pp underlying BP_PUB_PARAM object + * \return newly created BP_WITNESS object or NULL in case of an error + */ +BP_WITNESS *BP_WITNESS_new(const BP_PUB_PARAM *pp) +{ + BP_WITNESS *witness = NULL; + + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(witness = OPENSSL_zalloc(sizeof(*witness)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!(witness->sk_r = sk_BIGNUM_new_null()) + || !(witness->sk_v = sk_BIGNUM_new_null()) + || !(witness->sk_V = sk_BP_VARIABLE_new_null())) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!(witness->group = EC_GROUP_dup(pp->group)) + || !(witness->H = EC_POINT_dup(pp->H, pp->group))) + goto err; + + witness->references = 1; + if ((witness->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + return witness; +err: + BP_WITNESS_free(witness); + return NULL; +} + +/** Frees a BP_WITNESS object + * \param witness BP_WITNESS object to be freed + */ +void BP_WITNESS_free(BP_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return; + + CRYPTO_DOWN_REF(&witness->references, &ref, witness->lock); + REF_PRINT_COUNT("BP_WITNESS", witness); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + sk_BIGNUM_pop_free(witness->sk_r, BN_free); + sk_BIGNUM_pop_free(witness->sk_v, BN_free); + sk_BP_VARIABLE_pop_free(witness->sk_V, BP_VARIABLE_free); + EC_POINT_free(witness->H); + EC_GROUP_free(witness->group); + CRYPTO_THREAD_lock_free(witness->lock); + OPENSSL_free(witness); +} + +/** Increases the internal reference count of a BP_WITNESS object. + * \param witness BP_WITNESS object + * \return 1 on success and 0 if an error occurred. + */ +int BP_WITNESS_up_ref(BP_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return 0; + + if (CRYPTO_UP_REF(&witness->references, &ref, witness->lock) <= 0) + return 0; + + REF_PRINT_COUNT("BP_WITNESS", witness); + REF_ASSERT_ISNT(ref < 2); + return ((ref > 1) ? 1 : 0); +} + +/** Decreases the internal reference count of a BP_WITNESS object. + * \param witness BP_WITNESS object + * \return 1 on success and 0 if an error occurred. + */ +int BP_WITNESS_down_ref(BP_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return 0; + + if (CRYPTO_DOWN_REF(&witness->references, &ref, witness->lock) <= 0) + return 0; + + REF_PRINT_COUNT("BP_WITNESS", witness); + REF_ASSERT_ISNT(ref < 0); + return ((ref > 0) ? 1 : 0); +} + +/** Commit v to the witness and calculate V=G^r*H^v + * \param witness BP_WITNESS object + * \param name the name used to index the BP_VARIABLE object + * \param v plaintext BIGNUM object + * \return 1 on success and 0 otherwise + */ +int BP_WITNESS_commit(BP_WITNESS *witness, const char *name, const BIGNUM *v) +{ + const BIGNUM *order; + BIGNUM *r = NULL, *val = NULL; + EC_POINT *V = NULL; + BP_VARIABLE *var = NULL; + + if (witness == NULL || v == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (name != NULL && strlen(name) > BP_VARIABLE_NAME_MAX_LEN) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_VARIABLE_NAME_TOO_LONG); + return 0; + } + + if (name != NULL && BP_WITNESS_get_variable_index(witness, name) >= 0) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_VARIABLE_DUPLICATED); + return 0; + } + + order = EC_GROUP_get0_order(witness->group); + + r = BN_new(); + val = BN_dup(v); + V = EC_POINT_new(witness->group); + if (r == NULL || val == NULL || V == NULL) + goto err; + + if (!zkp_rand_range(r, order)) + goto err; + + /* (69) */ + if (!EC_POINT_mul(witness->group, V, r, witness->H, v, NULL)) + goto err; + + if (!(var = BP_VARIABLE_new(name, V, witness->group))) + goto err; + + if (sk_BIGNUM_push(witness->sk_r, r) <= 0) + goto err; + + r = NULL; + + if (sk_BIGNUM_push(witness->sk_v, val) <= 0) + goto err; + + val = NULL; + + if (sk_BP_VARIABLE_push(witness->sk_V, var) <= 0) + goto err; + + EC_POINT_free(V); + return 1; +err: + BN_free(r); + BN_free(val); + EC_POINT_free(V); + BP_VARIABLE_free(var); + return 0; +} + +/** Get the BP_VARIABLE with the variable name from the witness. + * \param witness BP_WITNESS object + * \param name the name of the BP_VARIABLE object + * \return the BP_VARIABLE object when found by name, otherwise return NULL. + */ +BP_VARIABLE *BP_WITNESS_get_variable(BP_WITNESS *witness, const char *name) +{ + int i; + + if (witness == NULL || name == NULL) { + return NULL; + } + + i = BP_WITNESS_get_variable_index(witness, name); + if (i < 0) { + return NULL; + } + + return sk_BP_VARIABLE_value(witness->sk_V, i); +} + +/** Get the index of the BP_VARIABLE in the stack that corresponds to the variable + * name from the witness. + * \param witness BP_WITNESS object + * \param name the name of the BP_VARIABLE object + * \return the index of the BP_VARIABLE object when found by name, + * otherwise return -1. + */ +int BP_WITNESS_get_variable_index(BP_WITNESS *witness, const char *name) +{ + int i, num; + BP_VARIABLE *V; + + if (witness == NULL || name == NULL) { + return -1; + } + + num = sk_BP_VARIABLE_num(witness->sk_V); + for (i = 0; i < num; i++) { + V = sk_BP_VARIABLE_value(witness->sk_V, i); + if (V == NULL || V->name == NULL) + return -1; + + if (OPENSSL_strcasecmp(V->name, name) == 0) + return i; + } + + return -1; +} diff --git a/openssl/src/crypto/zkp/bulletproofs/bulletproofs.h b/openssl/src/crypto/zkp/bulletproofs/bulletproofs.h new file mode 100644 index 000000000..cb8fca1c1 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/bulletproofs.h @@ -0,0 +1,70 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_BULLETPROOFS_LOCAL_H +# define HEADER_BULLETPROOFS_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include "internal/refcount.h" + +# define BP_VARIABLE_NAME_MAX_LEN 16 + +STACK_OF(BIGNUM); +STACK_OF(EC_POINT); +STACK_OF(BP_VARIABLE); + +struct bp_pub_param_st { + EC_GROUP *group; + /* `gens_capacity` is the number of generators to precompute for each party. + * For range_proof, it is the maximum bitsize of the range_proof, + * maximum value is 64. For r1cs_proof, the capacity must be greater + * than the number of multipliers, rounded up to the next power of two. + */ + int gens_capacity; + /* `party_capacity` is the maximum number of parties that can produce an + * aggregated range proof. For r1cs_proof, set to 1. + */ + int party_capacity; + STACK_OF(EC_POINT) *sk_G; + STACK_OF(EC_POINT) *sk_H; + EC_POINT *H; + EC_POINT *U; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +struct bp_variable_st { + EC_POINT *point; + char *name; +}; + +struct bp_witness_st { + EC_GROUP *group; + EC_POINT *H; + STACK_OF(BIGNUM) *sk_r; + STACK_OF(BIGNUM) *sk_v; + STACK_OF(BP_VARIABLE) *sk_V; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +# ifdef __cplusplus +} +# endif + +#endif + diff --git a/openssl/src/crypto/zkp/bulletproofs/bulletproofs_asn1.c b/openssl/src/crypto/zkp/bulletproofs/bulletproofs_asn1.c new file mode 100644 index 000000000..0d6d67953 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/bulletproofs_asn1.c @@ -0,0 +1,112 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include "internal/deprecated.h" +#include +#include +#include +#include "range_proof.h" + +#define IMPLEMENT_d2i_i2d(type) \ + type *d2i_##type(type **out, const unsigned char **in, long len) \ + { \ + type *d = NULL; \ + const unsigned char *p = *in; \ + if (p == NULL) \ + return NULL; \ + if ((d = type##_decode(p, len)) == NULL) \ + return NULL; \ + if (out) { \ + type##_free(*out); \ + *out = d; \ + } \ + *in = p; \ + return d; \ + } \ + int i2d_##type(const type *in, unsigned char **out) \ + { \ + size_t size; \ + if ((size = type##_encode(in, NULL, 0)) <= 0) \ + return 0; \ + if (out == NULL) \ + return (int)size; \ + \ + if (type##_encode(in, *out, size) <= 0) \ + return 0; \ + return (int)size; \ + } \ + +static BP_WITNESS *d2i_BP_WITNESS(BP_WITNESS **witness, const unsigned char **in, + long len, int flag) +{ + BP_WITNESS *ret = NULL; + const unsigned char *p = *in; + + if (p == NULL) + return NULL; + + if ((ret = BP_WITNESS_decode(p, len, flag)) == NULL) + return NULL; + + if (witness) { + BP_WITNESS_free(*witness); + *witness = ret; + } + + *in = p; + return ret; +} + +static int i2d_BP_WITNESS(const BP_WITNESS *witness, unsigned char **out, int flag) +{ + size_t size; + + if ((size = BP_WITNESS_encode(witness, NULL, 0, flag)) <= 0) + return 0; + + if (out == NULL) + return (int)size; + + if (BP_WITNESS_encode(witness, *out, size, flag) <= 0) + return 0; + + return (int)size; +} + +BP_WITNESS *d2i_long_BP_WITNESS(BP_WITNESS **witness, const unsigned char **in, + long len) +{ + return d2i_BP_WITNESS(witness, in, len, 1); +} + +int i2d_long_BP_WITNESS(const BP_WITNESS *witness, unsigned char **out) +{ + return i2d_BP_WITNESS(witness, out, 1); +} + +BP_WITNESS *d2i_short_BP_WITNESS(BP_WITNESS **witness, const unsigned char **in, + long len) +{ + return d2i_BP_WITNESS(witness, in, len, 0); +} + +int i2d_short_BP_WITNESS(const BP_WITNESS *witness, unsigned char **out) +{ + return i2d_BP_WITNESS(witness, out, 0); +} + +IMPLEMENT_d2i_i2d(BP_PUB_PARAM) +IMPLEMENT_d2i_i2d(BP_RANGE_PROOF) +IMPLEMENT_d2i_i2d(BP_R1CS_PROOF) + +IMPLEMENT_PEM_rw(BULLETPROOFS_PublicParam, BP_PUB_PARAM, PEM_STRING_BULLETPROOFS_PUB_PARAM, BP_PUB_PARAM) +IMPLEMENT_PEM_rw(BULLETPROOFS_LongWitness, BP_WITNESS, PEM_STRING_BULLETPROOFS_WITNESS, long_BP_WITNESS) +IMPLEMENT_PEM_rw(BULLETPROOFS_ShortWitness, BP_WITNESS, PEM_STRING_BULLETPROOFS_WITNESS, short_BP_WITNESS) +IMPLEMENT_PEM_rw(BULLETPROOFS_RangeProof, BP_RANGE_PROOF, PEM_STRING_BULLETPROOFS_RANGE_PROOF, BP_RANGE_PROOF) +IMPLEMENT_PEM_rw(BULLETPROOFS_R1CSProof, BP_R1CS_PROOF, PEM_STRING_BULLETPROOFS_R1CS_PROOF, BP_R1CS_PROOF) diff --git a/openssl/src/crypto/zkp/bulletproofs/bulletproofs_encode.c b/openssl/src/crypto/zkp/bulletproofs/bulletproofs_encode.c new file mode 100644 index 000000000..ef03cedd6 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/bulletproofs_encode.c @@ -0,0 +1,1137 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include "internal/endian.h" +#include "bulletproofs.h" +#include "range_proof.h" +#include "r1cs.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) +DEFINE_STACK_OF(BP_VARIABLE) + +static point_conversion_form_t form = POINT_CONVERSION_COMPRESSED; + +static int bp_stack_of_variable_encode(STACK_OF(BP_VARIABLE) *sk, unsigned char *out, + const EC_GROUP *group, BN_CTX *bn_ctx) +{ + int i, n, *q, size; + size_t point_len; + unsigned char *p; + BP_VARIABLE *V; + + if (sk == NULL || group == NULL) + return 0; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + n = sk_BP_VARIABLE_num(sk); + if (out == NULL) { + size = sizeof(n) + n * point_len; + for (i = 0; i < n; i++) { + V = sk_BP_VARIABLE_value(sk, i); + if (V == NULL) + break; + + if (V->name != NULL) { + size += strlen(V->name); + } + + size += 1; + } + + return size; + } + + q = (int *)out; + *q++ = zkp_l2n((int)n); + p = (unsigned char *)q; + + for (i = 0; i < n; i++) { + V = sk_BP_VARIABLE_value(sk, i); + if (V == NULL) + goto end; + + if (EC_POINT_point2oct(group, V->point, form, p, point_len, bn_ctx) == 0) + goto end; + + p += point_len; + + if (V->name == NULL) { + *p++ = '\0'; + continue; + } + + stpcpy((char *)p, V->name); + p += strlen(V->name) + 1; + } + +end: + return p - out; +} + +static STACK_OF(BP_VARIABLE) *bp_stack_of_variable_decode(const unsigned char *in, + int *len, + const EC_GROUP *group, + BN_CTX *bn_ctx) +{ + char *name; + unsigned char *p; + int *q = (int *)in, n, i; + size_t point_len; + EC_POINT *V = NULL; + BP_VARIABLE *var = NULL; + STACK_OF(BP_VARIABLE) *ret = NULL; + + if (in == NULL || group == NULL) + return 0; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + n = (int)zkp_n2l(*q); + q++; + p = (unsigned char *)q; + + if (n < 0) { + return NULL; + } + + if (!(ret = sk_BP_VARIABLE_new_reserve(NULL, n))) + return NULL; + + for (i = 0; i < n; i++) { + if (!(V = EC_POINT_new(group))) + goto err; + + if (!EC_POINT_oct2point(group, V, p, point_len, bn_ctx)) + goto err; + + p += point_len; + name = (char *)p; + if (*name == '\0') { + name = NULL; + } else { + p += strlen(name); + } + + p += 1; + + if (!(var = BP_VARIABLE_new(name, V, group))) + goto err; + + if (sk_BP_VARIABLE_push(ret, var) <= 0) + goto err; + + EC_POINT_free(V); + } + + if (len != NULL) + *len = p - in; + + return ret; +err: + EC_POINT_free(V); + BP_VARIABLE_free(var); + sk_BP_VARIABLE_pop_free(ret, BP_VARIABLE_free); + return NULL; +} + +static int bp_inner_product_proof_encode(bp_inner_product_proof_t *ip_proof, + unsigned char *out, const EC_GROUP *group, + BN_CTX *bn_ctx) +{ + int bn_len, sk_len, len; + unsigned char *p = out; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (ip_proof == NULL || group == NULL || bn_ctx == NULL) + return 0; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + sk_bn = sk_BIGNUM_new_reserve(NULL, 2); + if (sk_bn == NULL) + goto end; + + if (sk_BIGNUM_push(sk_bn, ip_proof->a) <= 0 + || sk_BIGNUM_push(sk_bn, ip_proof->b) <= 0) + goto end; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, NULL, bn_len); + if (sk_len == 0) + goto end; + + len = sk_len; + + sk_len = zkp_stack_of_point_encode(ip_proof->sk_L, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + + len += sk_len; + + sk_len = zkp_stack_of_point_encode(ip_proof->sk_R, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + + len += sk_len; + + if (out == NULL) + return len; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, p, bn_len); + if (sk_len == 0) + goto end; + + p += sk_len; + + sk_len = zkp_stack_of_point_encode(ip_proof->sk_L, p, group, bn_ctx); + if (sk_len == 0) + goto end; + + p += sk_len; + + sk_len = zkp_stack_of_point_encode(ip_proof->sk_R, p, group, bn_ctx); + if (sk_len == 0) + goto end; + + p += sk_len; + +end: + sk_BIGNUM_free(sk_bn); + return p != NULL ? p - out : 0; +} + +static bp_inner_product_proof_t *bp_inner_product_proof_decode(const unsigned char *in, + int *len, + const EC_GROUP *group, + BN_CTX *bn_ctx) +{ + int bn_len, sk_len; + unsigned char *p = (unsigned char *)in; + STACK_OF(BIGNUM) *sk_bn = NULL; + bp_inner_product_proof_t *ip_proof = NULL; + + if (in == NULL || group == NULL || bn_ctx == NULL) + return NULL; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + if (!(ip_proof = bp_inner_product_proof_alloc(1))) + goto err; + + sk_EC_POINT_free(ip_proof->sk_L); + sk_EC_POINT_free(ip_proof->sk_R); + ip_proof->sk_L = NULL; + ip_proof->sk_R = NULL; + + if (!(sk_bn = zkp_stack_of_bignum_decode(p, &sk_len, bn_len))) + goto err; + + if (sk_BIGNUM_num(sk_bn) != 2) + goto err; + + if (!BN_copy(ip_proof->a, sk_BIGNUM_value(sk_bn, 0)) + || !BN_copy(ip_proof->b, sk_BIGNUM_value(sk_bn, 1))) + goto err; + + p += sk_len; + + if (!(ip_proof->sk_L = zkp_stack_of_point_decode(p, &sk_len, group, bn_ctx))) + goto err; + + p += sk_len; + + if (!(ip_proof->sk_R = zkp_stack_of_point_decode(p, &sk_len, group, bn_ctx))) + goto err; + + p += sk_len; + + if (len != NULL) + *len = p - in; + + sk_BIGNUM_free(sk_bn); + return ip_proof; + +err: + sk_BIGNUM_pop_free(sk_bn, BN_free); + bp_inner_product_proof_free(ip_proof); + return NULL; +} + +/** Encodes BP_PUB_PARAM to binary + * \param pp BP_PUB_PARAM object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t BP_PUB_PARAM_encode(const BP_PUB_PARAM *pp, unsigned char *out, size_t size) +{ + int *q, sk_len, curve_id; + size_t point_len, ret = 0, len; + unsigned char *p; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + group = pp->group; + + curve_id = EC_GROUP_get_curve_name(group); + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto end; + } + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + + sk_len = zkp_stack_of_point_encode(pp->sk_G, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + + len = sizeof(int) * 3 + point_len * 2 + sk_len * 2; + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + q = (int *)out; + *q++ = zkp_l2n((int)curve_id); + *q++ = zkp_l2n((int)pp->gens_capacity); + *q++ = zkp_l2n((int)pp->party_capacity); + p = (unsigned char *)q; + + if (EC_POINT_point2oct(group, pp->H, form, p, point_len, bn_ctx) == 0) + goto end; + + p += point_len; + + if (EC_POINT_point2oct(group, pp->U, form, p, point_len, bn_ctx) == 0) + goto end; + + p += point_len; + + sk_len = zkp_stack_of_point_encode(pp->sk_G, p, group, bn_ctx); + if (sk_len == 0) + goto end; + + p += sk_len; + + sk_len = zkp_stack_of_point_encode(pp->sk_H, p, group, bn_ctx); + if (sk_len == 0) + goto end; + + p += sk_len; + + ret = len; + +end: + BN_CTX_free(bn_ctx); + return ret; +} + +/** Decodes binary to BP_PUB_PARAM + * \param in Memory buffer with the encoded BP_PUB_PARAM + * object + * \param size The memory size of the in pointer object + * \return BP_PUB_PARAM object pointer on success and NULL otherwise + */ +BP_PUB_PARAM *BP_PUB_PARAM_decode(const unsigned char *in, size_t size) +{ + unsigned char *p; + int curve_id, *q = (int *)in, sk_len; + size_t point_len, gens_capacity, party_capacity, n; + BP_PUB_PARAM *pp = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + + if (in == NULL || size <= 12) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + curve_id = zkp_n2l(*q); + q++; + gens_capacity = (size_t)zkp_n2l(*q); + q++; + party_capacity = (size_t)zkp_n2l(*q); + q++; + p = (unsigned char *)q; + n = gens_capacity * party_capacity; + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + + if (size < (sizeof(int) * 3 + point_len * (n * 2 + 2))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + pp = BP_PUB_PARAM_new(group, gens_capacity, party_capacity); + if (pp == NULL) + goto err; + + sk_EC_POINT_pop_free(pp->sk_G, EC_POINT_free); + sk_EC_POINT_pop_free(pp->sk_H, EC_POINT_free); + pp->sk_G = NULL; + pp->sk_H = NULL; + + if (!EC_POINT_oct2point(group, pp->H, p, point_len, bn_ctx)) + goto err; + + p += point_len; + + if (!EC_POINT_oct2point(group, pp->U, p, point_len, bn_ctx)) + goto err; + + p += point_len; + + if (!(pp->sk_G = zkp_stack_of_point_decode(p, &sk_len, group, bn_ctx))) + goto err; + + p += sk_len; + + if (!(pp->sk_H = zkp_stack_of_point_decode(p, &sk_len, group, bn_ctx))) + goto err; + + p += sk_len; + + EC_GROUP_free(group); + BN_CTX_free(bn_ctx); + return pp; + +err: + EC_GROUP_free(group); + BP_PUB_PARAM_free(pp); + BN_CTX_free(bn_ctx); + return NULL; +} + +/** Encodes BP_WITNESS to binary + * \param pp BP_WITNESS object + * \param out The buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param flag The flag is an indicator for encoding random number 'r' + * and plaintext 'v', with 1 indicating encoding and 0 + * indicating no encoding. + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t BP_WITNESS_encode(const BP_WITNESS *witness, unsigned char *out, + size_t size, int flag) +{ + int *q, curve_id, bn_len, sk_len; + size_t ret = 0, len, n, point_len; + unsigned char *p; + BP_VARIABLE *V; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + + if (witness == NULL || witness->sk_V == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + n = sk_BP_VARIABLE_num(witness->sk_V); + if (n == 0) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return ret; + } + + V = sk_BP_VARIABLE_value(witness->sk_V, 0); + if ((curve_id = EC_POINT_get_curve_name(V->point)) == NID_undef) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto end; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto end; + } + + if (!(sk_len = bp_stack_of_variable_encode(witness->sk_V, NULL, group, bn_ctx))) + goto end; + + len = 4 + point_len + sk_len; + + if (!(sk_len = zkp_stack_of_bignum_encode(witness->sk_r, NULL, bn_len))) + goto end; + + if (flag == 1) + len += sk_len * 2; + + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + q = (int *)out; + *q++ = zkp_l2n((int)curve_id); + p = (unsigned char *)q; + + if (EC_POINT_point2oct(group, witness->H, form, p, point_len, bn_ctx) == 0) + goto end; + + p += point_len; + + if (!(sk_len = bp_stack_of_variable_encode(witness->sk_V, p, group, bn_ctx))) + goto end; + + p += sk_len; + + if (flag == 1) { + if (!(sk_len = zkp_stack_of_bignum_encode(witness->sk_r, p, bn_len))) + goto end; + + p += sk_len; + + if (!(sk_len = zkp_stack_of_bignum_encode(witness->sk_v, p, bn_len))) + goto end; + + p += sk_len; + } + + ret = len; + +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return ret; +} + +/** Decodes binary to BP_WITNESS + * \param in Memory buffer with the encoded BP_WITNESS + * object + * \param size The memory size of the in pointer object + * \param flag The flag is an indicator for decoding random number 'r' + * and plaintext 'v', with 1 indicating decoding and 0 + * indicating no decoding. + * \return BP_WITNESS object pointer on success and NULL otherwise + */ +BP_WITNESS *BP_WITNESS_decode(const unsigned char *in, size_t size, int flag) +{ + unsigned char *p; + int curve_id, *q = (int *)in, bn_len, sk_len; + size_t point_len; + BP_WITNESS *witness = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + + if (in == NULL || size <= 12) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + curve_id = zkp_n2l(*q); + q++; + p = (unsigned char *)q; + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + + if (!(witness = OPENSSL_zalloc(sizeof(*witness)))) { + goto err; + } + + if (!(witness->H = EC_POINT_new(group))) + goto err; + + if (!EC_POINT_oct2point(group, witness->H, p, point_len, bn_ctx)) + goto err; + + p += point_len; + + if (!(witness->sk_V = bp_stack_of_variable_decode(p, &sk_len, group, bn_ctx))) + goto err; + + p += sk_len; + + if (flag == 1) { + if (!(witness->sk_r = zkp_stack_of_bignum_decode(p, &sk_len, bn_len))) + goto err; + + p += sk_len; + + if (!(witness->sk_v = zkp_stack_of_bignum_decode(p, &sk_len, bn_len))) + goto err; + + p += sk_len; + } + + witness->group = group; + + witness->references = 1; + if ((witness->lock = CRYPTO_THREAD_lock_new()) == NULL) + goto err; + + BN_CTX_free(bn_ctx); + return witness; + +err: + EC_GROUP_free(group); + BP_WITNESS_free(witness); + BN_CTX_free(bn_ctx); + return NULL; +} + +/** Encodes BP_RANGE_PROOF to binary + * \param proof BP_RANGE_PROOF object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t BP_RANGE_PROOF_encode(const BP_RANGE_PROOF *proof, unsigned char *out, + size_t size) +{ + int *q, curve_id, bn_len, ret = 0, sk_len; + size_t len; + unsigned char *p = NULL; + bp_inner_product_proof_t *ip_proof; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + sk_point = sk_EC_POINT_new_reserve(NULL, 4); + sk_bn = sk_BIGNUM_new_reserve(NULL, 3); + if (sk_point == NULL || sk_bn == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return 0; + } + + ip_proof = proof->ip_proof; + + if ((curve_id = EC_POINT_get_curve_name(proof->A)) == NID_undef + || ip_proof == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto end; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto end; + } + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + len = sizeof(int); + + if (sk_EC_POINT_push(sk_point, proof->A) <= 0 + || sk_EC_POINT_push(sk_point, proof->S) <= 0 + || sk_EC_POINT_push(sk_point, proof->T1) <= 0 + || sk_EC_POINT_push(sk_point, proof->T2) <= 0) + goto end; + + sk_len = zkp_stack_of_point_encode(sk_point, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + len += sk_len; + + if (sk_BIGNUM_push(sk_bn, proof->taux) <= 0 + || sk_BIGNUM_push(sk_bn, proof->mu) <= 0 + || sk_BIGNUM_push(sk_bn, proof->tx) <= 0) + goto end; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, NULL, bn_len); + if (sk_len == 0) + goto end; + len += sk_len; + + sk_len = bp_inner_product_proof_encode(ip_proof, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + len += sk_len; + + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + /* encoding proof */ + q = (int *)out; + *q++ = zkp_l2n(curve_id); + p = (unsigned char *)q; + + sk_len = zkp_stack_of_point_encode(sk_point, p, group, bn_ctx); + if (sk_len == 0) + goto end; + p += sk_len; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, p, bn_len); + if (sk_len == 0) + goto end; + p += sk_len; + + /* encoding ip_proof */ + len = bp_inner_product_proof_encode(ip_proof, p, group, bn_ctx); + if (len == 0) + goto end; + p += len; + + ret = p - out; +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + return ret; +} + +/** Decodes binary to BP_RANGE_PROOF + * \param in Memory buffer with the encoded BP_RANGE_PROOF object + * \param size The memory size of the in pointer object + * \return BP_RANGE_PROOF object pointer on success and NULL otherwise + */ +BP_RANGE_PROOF *BP_RANGE_PROOF_decode(const unsigned char *in, size_t size) +{ + unsigned char *p; + int *q = (int *)in, curve_id, len; + size_t point_len, bn_len, proof_len; + BP_RANGE_PROOF *proof = NULL; + bp_inner_product_proof_t *ip_proof = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (in == NULL || size <= 8) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + curve_id = zkp_n2l(*q); + q++; + + if (curve_id <= 0) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + p = (unsigned char *)q; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + + /* len(curve_id) + len(A+S+T1+T2) + len(taux+mu+tx) + len(a+b) */ + proof_len = 4 + point_len * 4 + bn_len * 3 + bn_len * 2; + if (size < proof_len) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) + goto err; + + proof->references = 1; + if ((proof->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + sk_point = zkp_stack_of_point_decode(p, &len, group, bn_ctx); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_EC_POINT_num(sk_point) < 4) + goto err; + + proof->A = sk_EC_POINT_value(sk_point, 0); + proof->S = sk_EC_POINT_value(sk_point, 1); + proof->T1 = sk_EC_POINT_value(sk_point, 2); + proof->T2 = sk_EC_POINT_value(sk_point, 3); + + sk_bn = zkp_stack_of_bignum_decode(p, &len, bn_len); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_BIGNUM_num(sk_bn) < 3) + goto err; + + proof->taux = sk_BIGNUM_value(sk_bn, 0); + proof->mu = sk_BIGNUM_value(sk_bn, 1); + proof->tx = sk_BIGNUM_value(sk_bn, 2); + + ip_proof = bp_inner_product_proof_decode(p, &len, group, bn_ctx); + if (ip_proof == NULL) + goto err; + p += len; + + proof->ip_proof = ip_proof; + + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return proof; + +err: + sk_BIGNUM_pop_free(sk_bn, BN_free); + sk_EC_POINT_pop_free(sk_point, EC_POINT_free); + bp_inner_product_proof_free(ip_proof); + OPENSSL_free(proof); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return NULL; +} + +/** Encodes BP_R1CS_PROOF to binary + * \param proof BP_R1CS_PROOF object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t BP_R1CS_PROOF_encode(const BP_R1CS_PROOF *proof, unsigned char *out, + size_t size) +{ + int *q, curve_id, bn_len, ret = 0, sk_len; + size_t len; + unsigned char *p = NULL; + bp_inner_product_proof_t *ip_proof; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + sk_point = sk_EC_POINT_new_reserve(NULL, 11); + sk_bn = sk_BIGNUM_new_reserve(NULL, 3); + if (sk_point == NULL || sk_bn == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return 0; + } + + ip_proof = proof->ip_proof; + + if ((curve_id = EC_POINT_get_curve_name(proof->AI1)) == NID_undef + || ip_proof == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto end; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto end; + } + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + len = sizeof(int); + + if (sk_EC_POINT_push(sk_point, proof->AI1) <= 0 + || sk_EC_POINT_push(sk_point, proof->AO1) <= 0 + || sk_EC_POINT_push(sk_point, proof->S1) <= 0 + || sk_EC_POINT_push(sk_point, proof->T1) <= 0 + || sk_EC_POINT_push(sk_point, proof->T3) <= 0 + || sk_EC_POINT_push(sk_point, proof->T4) <= 0 + || sk_EC_POINT_push(sk_point, proof->T5) <= 0 + || sk_EC_POINT_push(sk_point, proof->T6) <= 0 +#if 0 + || sk_EC_POINT_push(sk_point, proof->AI2) <= 0 + || sk_EC_POINT_push(sk_point, proof->AO2) <= 0 + || sk_EC_POINT_push(sk_point, proof->S2) <= 0 +#endif + ) + goto end; + + sk_len = zkp_stack_of_point_encode(sk_point, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + len += sk_len; + + if (sk_BIGNUM_push(sk_bn, proof->taux) <= 0 + || sk_BIGNUM_push(sk_bn, proof->mu) <= 0 + || sk_BIGNUM_push(sk_bn, proof->tx) <= 0) + goto end; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, NULL, bn_len); + if (sk_len == 0) + goto end; + len += sk_len; + + sk_len = bp_inner_product_proof_encode(ip_proof, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + len += sk_len; + + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + /* encoding proof */ + q = (int *)out; + *q++ = zkp_l2n(curve_id); + p = (unsigned char *)q; + + sk_len = zkp_stack_of_point_encode(sk_point, p, group, bn_ctx); + if (sk_len == 0) + goto end; + p += sk_len; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, p, bn_len); + if (sk_len == 0) + goto end; + p += sk_len; + + /* encoding ip_proof */ + sk_len = bp_inner_product_proof_encode(ip_proof, p, group, bn_ctx); + if (sk_len == 0) + goto end; + p += sk_len; + + ret = p - out; +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + return ret; +} + +/** Decodes binary to BP_R1CS_PROOF + * \param in Memory buffer with the encoded BP_R1CS_PROOF object + * \param size The memory size of the in pointer object + * \return BP_R1CS_PROOF object pointer on success and NULL otherwise + */ +BP_R1CS_PROOF *BP_R1CS_PROOF_decode(const unsigned char *in, size_t size) +{ + unsigned char *p; + int *q = (int *)in, curve_id, len; + size_t point_len, bn_len, proof_len; + BP_R1CS_PROOF *proof = NULL; + bp_inner_product_proof_t *ip_proof = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + + if (in == NULL || size <= 8) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + curve_id = zkp_n2l(*q); + q++; + + if (curve_id <= 0) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + p = (unsigned char *)q; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + +#if 1 + proof_len = 4 + point_len * 8 + bn_len * 3 + bn_len * 2; +#else + proof_len = 4 + point_len * 11 + bn_len * 3 + bn_len * 2; +#endif + if (size < proof_len) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) + goto err; + + proof->references = 1; + if ((proof->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + sk_point = zkp_stack_of_point_decode(p, &len, group, bn_ctx); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_EC_POINT_num(sk_point) < 8) + goto err; + + proof->AI1 = sk_EC_POINT_value(sk_point, 0); + proof->AO1 = sk_EC_POINT_value(sk_point, 1); + proof->S1 = sk_EC_POINT_value(sk_point, 2); + proof->T1 = sk_EC_POINT_value(sk_point, 3); + proof->T3 = sk_EC_POINT_value(sk_point, 4); + proof->T4 = sk_EC_POINT_value(sk_point, 5); + proof->T5 = sk_EC_POINT_value(sk_point, 6); + proof->T6 = sk_EC_POINT_value(sk_point, 7); +#if 1 + proof->AI2 = EC_POINT_new(group); + proof->AO2 = EC_POINT_new(group); + proof->S2 = EC_POINT_new(group); + if (proof->AI2 == NULL || proof->AO2 == NULL || proof->S2 == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + EC_POINT_set_to_infinity(group, proof->AI2); + EC_POINT_set_to_infinity(group, proof->AO2); + EC_POINT_set_to_infinity(group, proof->S2); +#else + proof->AI2 = sk_EC_POINT_value(sk_point, 8); + proof->AO2 = sk_EC_POINT_value(sk_point, 9); + proof->S2 = sk_EC_POINT_value(sk_point, 10); +#endif + + sk_bn = zkp_stack_of_bignum_decode(p, &len, bn_len); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_BIGNUM_num(sk_bn) < 3) + goto err; + + proof->taux = sk_BIGNUM_value(sk_bn, 0); + proof->mu = sk_BIGNUM_value(sk_bn, 1); + proof->tx = sk_BIGNUM_value(sk_bn, 2); + + ip_proof = bp_inner_product_proof_decode(p, &len, group, bn_ctx); + if (ip_proof == NULL) + goto err; + p += len; + + proof->ip_proof = ip_proof; + + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return proof; + +err: + sk_BIGNUM_pop_free(sk_bn, BN_free); + sk_EC_POINT_pop_free(sk_point, EC_POINT_free); + bp_inner_product_proof_free(ip_proof); + BP_R1CS_PROOF_free(proof); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return NULL; +} diff --git a/openssl/src/crypto/zkp/bulletproofs/bulletproofs_prn.c b/openssl/src/crypto/zkp/bulletproofs/bulletproofs_prn.c new file mode 100644 index 000000000..fd413ddfc --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/bulletproofs_prn.c @@ -0,0 +1,505 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include "internal/cryptlib.h" +#include +#include +#include +#include +#include +#include +#include +#include "range_proof.h" +#include "r1cs.h" + +/* Number of octets per line */ +#define ASN1_BUF_PRINT_WIDTH 127 +/* Maximum indent */ +#define ASN1_PRINT_MAX_INDENT 128 + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) +DEFINE_STACK_OF(BP_VARIABLE) + +static int bp_bio_printf(BIO *bio, int indent, const char *format, ...) +{ + va_list args; + int ret; + + if (!BIO_indent(bio, indent, ASN1_PRINT_MAX_INDENT)) + return 0; + + va_start(args, format); + + ret = BIO_vprintf(bio, format, args); + + va_end(args); + return ret; +} + +static int bp_buf_print(BIO *bp, const unsigned char *buf, size_t buflen, + int indent) +{ + size_t i; + + for (i = 0; i < buflen; i++) { + if ((i % ASN1_BUF_PRINT_WIDTH) == 0) { + if (i > 0 && BIO_puts(bp, "\n") <= 0) + return 0; + if (!BIO_indent(bp, indent, ASN1_PRINT_MAX_INDENT)) + return 0; + } + /* + * Use colon separators for each octet for compatibility as + * this function is used to print out key components. + */ + if (BIO_printf(bp, "%02x%s", buf[i], + (i == buflen - 1) ? "" : ":") <= 0) + return 0; + } + if (BIO_write(bp, "\n", 1) <= 0) + return 0; + return 1; +} + +static int bp_point_print(BIO *bp, const EC_GROUP *group, const EC_POINT *point, + const char *name, int indent, BN_CTX *bn_ctx) +{ + int ret = 0; + size_t point_len; + unsigned char *p = NULL; + + if (bp == NULL || group == NULL || point == NULL || bn_ctx == NULL) + return ret; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + POINT_CONVERSION_COMPRESSED, NULL, 0, bn_ctx); + p = OPENSSL_zalloc(point_len); + if (p == NULL) + goto end; + + if (!BIO_indent(bp, indent, ASN1_PRINT_MAX_INDENT)) + goto end; + + if (name != NULL) + BIO_printf(bp, "%s", name); + + if (EC_POINT_point2oct(group, point, POINT_CONVERSION_COMPRESSED, + p, point_len, bn_ctx) == 0) + goto end; + + if (!bp_buf_print(bp, p, point_len, 0)) + goto end; + + ret = 1; +end: + OPENSSL_free(p); + return ret; +} + +static int bp_bn_print(BIO *bp, const char *name, const BIGNUM *num, + unsigned char *ign, int indent) +{ + int n, rv = 0; + const char *neg; + unsigned char *buf = NULL, *tmp = NULL; + int buflen; + + if (num == NULL) + return 1; + neg = BN_is_negative(num) ? "-" : ""; + if (!BIO_indent(bp, indent, ASN1_PRINT_MAX_INDENT)) + return 0; + if (BN_is_zero(num)) { + if (name != NULL) + BIO_printf(bp, "%s: ", name); + + if (BIO_printf(bp, "0\n") <= 0) + return 0; + return 1; + } + + if (BN_num_bytes(num) <= BN_BYTES) { + if (name != NULL) + BIO_printf(bp, "%s: ", name); + + if (BIO_printf(bp, "%s%lu (%s0x%lx)\n", neg, + (unsigned long)bn_get_words(num)[0], neg, + (unsigned long)bn_get_words(num)[0]) <= 0) + return 0; + return 1; + } + + buflen = BN_num_bytes(num) + 1; + buf = tmp = OPENSSL_malloc(buflen); + if (buf == NULL) + goto err; + buf[0] = 0; + + if (name != NULL) + BIO_printf(bp, "%s: ", name); + + BIO_printf(bp, "%s", neg); + + n = BN_bn2bin(num, buf + 1); + + if (buf[1] & 0x80) + n++; + else + tmp++; + + if (bp_buf_print(bp, tmp, n, 0) == 0) + goto err; + rv = 1; + err: + OPENSSL_clear_free(buf, buflen); + return rv; +} + +static int bp_inner_product_proof_print(BIO *bp, + const bp_inner_product_proof_t *ip_proof, + const EC_GROUP *group, BN_CTX *bn_ctx, + int indent) +{ + int ret = 0, i, n; + EC_POINT *L, *R; + + if (bp == NULL || ip_proof == NULL || group == NULL || bn_ctx == NULL) + return ret; + + bp_bio_printf(bp, indent, "inner proof:\n"); + indent += 4; + n = sk_EC_POINT_num(ip_proof->sk_L); + bp_bio_printf(bp, indent, "n: %zu\n", n); + + bp_bio_printf(bp, indent, "L[n]:\n"); + for (i = 0; i < n; i++) { + L = sk_EC_POINT_value(ip_proof->sk_L, i); + if (L == NULL) + goto end; + + bp_bio_printf(bp, indent + 4, "[%zu]: ", i); + if (!bp_point_print(bp, group, L, NULL, 0, bn_ctx)) + goto end; + } + + bp_bio_printf(bp, indent, "R[n]:\n"); + for (i = 0; i < n; i++) { + R = sk_EC_POINT_value(ip_proof->sk_R, i); + if (R == NULL) + goto end; + + bp_bio_printf(bp, indent + 4, "[%zu]: ", i); + if (!bp_point_print(bp, group, R, NULL, 0, bn_ctx)) + goto end; + } + + if (!bp_bn_print(bp, "a", ip_proof->a, NULL, indent) + || !bp_bn_print(bp, "b", ip_proof->b, NULL, indent)) + goto end; + + ret = 1; +end: + return ret; +} + +#ifndef OPENSSL_NO_STDIO +int BP_PUB_PARAM_print_fp(FILE *fp, const BP_PUB_PARAM *pp, int indent) +{ + BIO *b; + int ret; + + if ((b = BIO_new(BIO_s_file())) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_BUF_LIB); + return 0; + } + BIO_set_fp(b, fp, BIO_NOCLOSE); + ret = BP_PUB_PARAM_print(b, pp, indent); + BIO_free(b); + return ret; +} + +int BP_WITNESS_print_fp(FILE *fp, const BP_WITNESS *witness, int indent, int flag) +{ + BIO *b; + int ret; + + if ((b = BIO_new(BIO_s_file())) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_BUF_LIB); + return 0; + } + BIO_set_fp(b, fp, BIO_NOCLOSE); + ret = BP_WITNESS_print(b, witness, indent, flag); + BIO_free(b); + return ret; +} + +int BP_RANGE_PROOF_print_fp(FILE *fp, const BP_RANGE_PROOF *proof, int indent) +{ + BIO *b; + int ret; + + if ((b = BIO_new(BIO_s_file())) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_BUF_LIB); + return 0; + } + BIO_set_fp(b, fp, BIO_NOCLOSE); + ret = BP_RANGE_PROOF_print(b, proof, indent); + BIO_free(b); + return ret; +} + +int BP_R1CS_PROOF_print_fp(FILE *fp, const BP_R1CS_PROOF *proof, int indent) +{ + BIO *b; + int ret; + + if ((b = BIO_new(BIO_s_file())) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_BUF_LIB); + return 0; + } + BIO_set_fp(b, fp, BIO_NOCLOSE); + ret = BP_R1CS_PROOF_print(b, proof, indent); + BIO_free(b); + return ret; +} +#endif + +int BP_PUB_PARAM_print(BIO *bp, const BP_PUB_PARAM *pp, int indent) +{ + int ret = 0, i, n, curve_id; + BN_CTX *bn_ctx = NULL; + EC_POINT *G, *H; + EC_GROUP *group = NULL; + + if (pp == NULL) + return 0; + + curve_id = EC_GROUP_get_curve_name(pp->group); + + bp_bio_printf(bp, indent, "Bulletproofs Public Parameter: \n"); + bp_bio_printf(bp, indent, "curve: %s (%d)\n", OSSL_EC_curve_nid2name(curve_id), + curve_id); + bp_bio_printf(bp, indent, "gens_capacity: %zu\n", pp->gens_capacity); + bp_bio_printf(bp, indent, "party_capacity: %zu\n", pp->party_capacity); + + group = pp->group; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto end; + + bp_bio_printf(bp, indent, "G[n]:\n"); + n = pp->gens_capacity * pp->party_capacity; + for (i = 0; i < n; i++) { + G = sk_EC_POINT_value(pp->sk_G, i); + if (G == NULL) + goto end; + + bp_bio_printf(bp, indent + 4, "[%zu]: ", i); + if (!bp_point_print(bp, group, G, NULL, 0, bn_ctx)) + goto end; + } + + bp_bio_printf(bp, indent, "H[n]:\n"); + for (i = 0; i < n; i++) { + H = sk_EC_POINT_value(pp->sk_H, i); + if (H == NULL) + goto end; + + bp_bio_printf(bp, indent + 4, "[%zu]: ", i); + if (!bp_point_print(bp, group, H, NULL, 0, bn_ctx)) + goto end; + } + + if (!bp_point_print(bp, group, pp->U, "U: ", indent, bn_ctx) + || !bp_point_print(bp, group, pp->H, "H: ", indent, bn_ctx)) + goto end; + + ret = 1; +end: + BN_CTX_free(bn_ctx); + return ret; +} + +int BP_WITNESS_print(BIO *bp, const BP_WITNESS *witness, int indent, int flag) +{ + int ret = 0, i, n, curve_id; + BN_CTX *bn_ctx = NULL; + BP_VARIABLE *var; + BIGNUM *v, *r; + EC_GROUP *group = NULL; + + if (witness == NULL) + return 0; + + group = witness->group; + curve_id = EC_GROUP_get_curve_name(group); + + bp_bio_printf(bp, indent, "Witness: \n"); + bp_bio_printf(bp, indent, "curve: %s (%d)\n", OSSL_EC_curve_nid2name(curve_id), + curve_id); + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto end; + + bp_bio_printf(bp, indent, "H: "); + if (!bp_point_print(bp, group, witness->H, NULL, 0, bn_ctx)) + goto end; + + bp_bio_printf(bp, indent, "V[n]:\n"); + n = sk_BP_VARIABLE_num(witness->sk_V); + for (i = 0; i < n; i++) { + var = sk_BP_VARIABLE_value(witness->sk_V, i); + if (var == NULL) + goto end; + + if (var->name != NULL) + bp_bio_printf(bp, indent + 4, "[%s]: ", var->name); + else + bp_bio_printf(bp, indent + 4, "[%zu]: ", i); + + if (!bp_point_print(bp, group, var->point, NULL, 0, bn_ctx)) + goto end; + } + + n = sk_BIGNUM_num(witness->sk_v); + if (n != 0 && flag == 1) { + bp_bio_printf(bp, indent, "v[n]:\n"); + for (i = 0; i < n; i++) { + var = sk_BP_VARIABLE_value(witness->sk_V, i); + v = sk_BIGNUM_value(witness->sk_v, i); + if (v == NULL) + goto end; + + if (var->name != NULL) + bp_bio_printf(bp, indent + 4, "[%s]: ", var->name); + else + bp_bio_printf(bp, indent + 4, "[%zu]: ", i); + + if (!bp_bn_print(bp, NULL, v, NULL, 0)) + goto end; + } + + bp_bio_printf(bp, indent, "r[n]:\n"); + for (i = 0; i < n; i++) { + r = sk_BIGNUM_value(witness->sk_r, i); + if (r == NULL) + goto end; + + bp_bio_printf(bp, indent + 4, "[%zu]: ", i); + if (!bp_bn_print(bp, NULL, r, NULL, 0)) + goto end; + } + } + + ret = 1; +end: + BN_CTX_free(bn_ctx); + return ret; +} + +int BP_RANGE_PROOF_print(BIO *bp, const BP_RANGE_PROOF *proof, int indent) +{ + int ret = 0, curve_id; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + + if (proof == NULL) + return 0; + + bp_bio_printf(bp, indent, "Range Proof: \n"); + + curve_id = EC_POINT_get_curve_name(proof->A); + if (curve_id <= 0) + goto end; + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto end; + + if (!bp_point_print(bp, group, proof->A, "A: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->S, "S: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->T1, "T1: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->T2, "T2: ", indent, bn_ctx) + || !bp_bn_print(bp, "taux", proof->taux, NULL, indent) + || !bp_bn_print(bp, "mu", proof->mu, NULL, indent) + || !bp_bn_print(bp, "tx", proof->tx, NULL, indent)) + goto end; + + if (proof->ip_proof != NULL) { + ret = bp_inner_product_proof_print(bp, proof->ip_proof, group, bn_ctx, indent); + } else { + bp_bio_printf(bp, indent, "inner proof: not found\n"); + } + + ret = 1; +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return ret; +} + +int BP_R1CS_PROOF_print(BIO *bp, const BP_R1CS_PROOF *proof, int indent) +{ + int ret = 0, curve_id; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + + if (proof == NULL) + return 0; + + bp_bio_printf(bp, indent, "R1CS Proof: \n"); + + curve_id = EC_POINT_get_curve_name(proof->AI1); + if (curve_id <= 0) + goto end; + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto end; + + if (!bp_point_print(bp, group, proof->AI1, "AI1: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->AO1, "AO1: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->S1, "S1: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->AI2, "AI2: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->AO2, "AO2: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->S2, "S2: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->T1, "T1: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->T3, "T3: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->T4, "T4: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->T5, "T5: ", indent, bn_ctx) + || !bp_point_print(bp, group, proof->T6, "T6: ", indent, bn_ctx) + || !bp_bn_print(bp, "taux", proof->taux, NULL, indent) + || !bp_bn_print(bp, "mu", proof->mu, NULL, indent) + || !bp_bn_print(bp, "tx", proof->tx, NULL, indent)) + goto end; + + if (proof->ip_proof != NULL) { + ret = bp_inner_product_proof_print(bp, proof->ip_proof, group, bn_ctx, indent); + } else { + bp_bio_printf(bp, indent, "inner proof: not found\n"); + } + + ret = 1; +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return ret; +} diff --git a/openssl/src/crypto/zkp/bulletproofs/inner_product.c b/openssl/src/crypto/zkp/bulletproofs/inner_product.c new file mode 100644 index 000000000..fabbb44ff --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/inner_product.c @@ -0,0 +1,638 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include +#include "inner_product.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) + +bp_inner_product_pub_param_t *bp_inner_product_pub_param_new(const EC_GROUP *group, + STACK_OF(EC_POINT) *sk_G, + STACK_OF(EC_POINT) *sk_H) +{ + bp_inner_product_pub_param_t *pp = NULL; + + if (sk_EC_POINT_num(sk_G) != sk_EC_POINT_num(sk_H)) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(pp = OPENSSL_zalloc(sizeof(*pp)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + pp->group = group; + pp->sk_G = sk_G; + pp->sk_H = sk_H; + + return pp; +} + +void bp_inner_product_pub_param_free(bp_inner_product_pub_param_t *pp) +{ + if (!pp) + return; + + OPENSSL_clear_free((void *)pp, sizeof(*pp)); +} + +bp_inner_product_ctx_t *bp_inner_product_ctx_new(bp_inner_product_pub_param_t *pp, + ZKP_TRANSCRIPT *transcript, + EC_POINT *U, EC_POINT *P, + STACK_OF(BIGNUM) *sk_G_factors, + STACK_OF(BIGNUM) *sk_H_factors) +{ + bp_inner_product_ctx_t *ctx = NULL; + + if (pp == NULL || U == NULL || transcript == NULL + || sk_G_factors == NULL || sk_H_factors == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (sk_BIGNUM_num(sk_G_factors) != sk_EC_POINT_num(pp->sk_G) + || sk_BIGNUM_num(sk_H_factors) != sk_EC_POINT_num(pp->sk_H)) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(ctx = OPENSSL_zalloc(sizeof(*ctx)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ctx->pp = pp; + ctx->transcript = transcript; + ctx->sk_G_factors = sk_G_factors; + ctx->sk_H_factors = sk_H_factors; + + if (!(ctx->U = EC_POINT_dup(U, pp->group))) + goto err; + + if (P != NULL && !(ctx->P = EC_POINT_dup(P, pp->group))) + goto err; + + return ctx; + +err: + bp_inner_product_ctx_free(ctx); + return NULL; +} + +void bp_inner_product_ctx_free(bp_inner_product_ctx_t *ctx) +{ + if (!ctx) + return; + + EC_POINT_free(ctx->U); + EC_POINT_free(ctx->P); + OPENSSL_clear_free((void *)ctx, sizeof(*ctx)); +} + +bp_inner_product_witness_t *bp_inner_product_witness_new(STACK_OF(BIGNUM) *sk_a, + STACK_OF(BIGNUM) *sk_b) +{ + bp_inner_product_witness_t *witness = NULL; + + if (!sk_a || !sk_b) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (sk_BIGNUM_num(sk_a) != sk_BIGNUM_num(sk_b)) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(witness = OPENSSL_zalloc(sizeof(*witness)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + witness->sk_a = sk_a; + witness->sk_b = sk_b; + + return witness; +} + +void bp_inner_product_witness_free(bp_inner_product_witness_t *witness) +{ + if (!witness) + return; + + OPENSSL_free(witness); +} + +bp_inner_product_proof_t *bp_inner_product_proof_alloc(int n) +{ + bp_inner_product_proof_t *proof = NULL; + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (!proof) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + proof->sk_L = sk_EC_POINT_new_reserve(NULL, n); + if (!proof->sk_L) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + proof->sk_R = sk_EC_POINT_new_reserve(NULL, n); + if (!proof->sk_R) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!(proof->a = BN_new()) || !(proof->b = BN_new())) + goto err; + + return proof; +err: + bp_inner_product_proof_free(proof); + return NULL; +} + +bp_inner_product_proof_t *bp_inner_product_proof_new(bp_inner_product_ctx_t *ctx) +{ + int n; + + if (ctx == NULL || ctx->pp == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + } + + n = sk_EC_POINT_num(ctx->pp->sk_G); + return bp_inner_product_proof_alloc(n); +} + +void bp_inner_product_proof_free(bp_inner_product_proof_t *proof) +{ + if (!proof) + return; + + BN_free(proof->a); + BN_free(proof->b); + + sk_EC_POINT_pop_free(proof->sk_L, EC_POINT_free); + sk_EC_POINT_pop_free(proof->sk_R, EC_POINT_free); + + OPENSSL_free(proof); +} + +bp_inner_product_proof_t *bp_inner_product_proof_prove(bp_inner_product_ctx_t *ctx, + bp_inner_product_witness_t *witness) +{ + int i, j, m, n, pp_num, poly_num; + ZKP_TRANSCRIPT *transcript; + BN_CTX *bn_ctx = NULL; + BIGNUM *x, *x_inv, *t, *cL, *cR, *a, *b, *u, *u_inv; + BIGNUM *G_factors_L, *G_factors_R, *H_factors_L, *H_factors_R; + BIGNUM *a_L, *a_R, *b_L, *b_R, *aL, *aR, *bL, *bR, *sk_a_L, *sk_b_L; + EC_POINT *L = NULL, *R = NULL, *P = NULL; + EC_POINT *G_L, *G_R, *H_L, *H_R, *sk_G_L, *sk_H_L; + STACK_OF(EC_POINT) *sk_G = NULL, *sk_H = NULL, *p_sk_G, *p_sk_H; + STACK_OF(BIGNUM) *sk_a = NULL, *sk_b = NULL, *p_sk_a, *p_sk_b; + zkp_poly_points_t *poly_l = NULL, *poly_r = NULL, *poly_g = NULL, *poly_h = NULL; + const BIGNUM *order; + const EC_GROUP *group; + bp_inner_product_pub_param_t *pp; + bp_inner_product_proof_t *proof = NULL, *ret = NULL; + + if (!ctx || !witness || !ctx->pp) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + transcript = ctx->transcript; + pp = ctx->pp; + group = pp->group; + order = EC_GROUP_get0_order(group); + pp_num = sk_EC_POINT_num(pp->sk_G); + poly_num = pp_num + 1; + + if (pp_num != sk_BIGNUM_num(witness->sk_a)) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(proof = bp_inner_product_proof_new(ctx))) + goto end; + + if (!(poly_l = zkp_poly_points_new(poly_num)) || !(poly_r = zkp_poly_points_new(poly_num)) + || !(poly_g = zkp_poly_points_new(2)) || !(poly_h = zkp_poly_points_new(2))) + goto end; + + if (!(sk_G = sk_EC_POINT_new_reserve(NULL, pp_num)) + || !(sk_H = sk_EC_POINT_new_reserve(NULL, pp_num)) + || !(sk_a = sk_BIGNUM_new_reserve(NULL, pp_num)) + || !(sk_b = sk_BIGNUM_new_reserve(NULL, pp_num))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto end; + } + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto end; + + BN_CTX_start(bn_ctx); + x = BN_CTX_get(bn_ctx); + x_inv = BN_CTX_get(bn_ctx); + cL = BN_CTX_get(bn_ctx); + cR = BN_CTX_get(bn_ctx); + t = BN_CTX_get(bn_ctx); + if (t == NULL) + goto end; + + p_sk_G = pp->sk_G; + p_sk_H = pp->sk_H; + p_sk_a = witness->sk_a; + p_sk_b = witness->sk_b; + + for (i = 0; i < pp_num; i++) { + if (!(P = EC_POINT_new(group)) || sk_EC_POINT_push(sk_G, P) <= 0) + goto end; + + P = NULL; + + if (!(P = EC_POINT_new(group)) || sk_EC_POINT_push(sk_H, P) <= 0) + goto end; + + P = NULL; + + if (!(a = BN_CTX_get(bn_ctx)) || sk_BIGNUM_push(sk_a, a) <= 0 + || !(b = BN_CTX_get(bn_ctx)) || sk_BIGNUM_push(sk_b, b) <= 0) + goto end; + } + + for (n = pp_num, j = 0; n > 1; n = m, j++) { + m = n / 2; + + BN_zero(cL); + BN_zero(cR); + + if (!(L = EC_POINT_new(group)) || !(R = EC_POINT_new(group))) + goto end; + + if (!EC_POINT_set_to_infinity(group, L) + || !EC_POINT_set_to_infinity(group, R)) + goto end; + + zkp_poly_points_reset(poly_l); + zkp_poly_points_reset(poly_r); + + for (i = 0; i < m; i++) { + /* (21) */ + a_L = sk_BIGNUM_value(p_sk_a, i); + b_R = sk_BIGNUM_value(p_sk_b, i + m); + if (!BN_mul(t, a_L, b_R, bn_ctx) + || !BN_mod_add(cL, cL, t, order, bn_ctx)) + goto end; + + /* (22) */ + a_R = sk_BIGNUM_value(p_sk_a, i + m); + b_L = sk_BIGNUM_value(p_sk_b, i); + if (!BN_mul(t, a_R, b_L, bn_ctx) + || !BN_mod_add(cR, cR, t, order, bn_ctx)) + goto end; + + aL = BN_CTX_get(bn_ctx); + aR = BN_CTX_get(bn_ctx); + bL = BN_CTX_get(bn_ctx); + bR = BN_CTX_get(bn_ctx); + if (bR == NULL) + goto end; + + if (p_sk_G == pp->sk_G) { + G_factors_L = sk_BIGNUM_value(ctx->sk_G_factors, i); + G_factors_R = sk_BIGNUM_value(ctx->sk_G_factors, i + m); + + if (!BN_mod_mul(aL, a_L, G_factors_R, order, bn_ctx)) + goto end; + + if (!BN_mod_mul(aR, a_R, G_factors_L, order, bn_ctx)) + goto end; + + H_factors_L = sk_BIGNUM_value(ctx->sk_H_factors, i); + H_factors_R = sk_BIGNUM_value(ctx->sk_H_factors, i + m); + + if (!BN_mod_mul(bL, b_L, H_factors_R, order, bn_ctx)) + goto end; + + if (!BN_mod_mul(bR, b_R, H_factors_L, order, bn_ctx)) + goto end; + } else { + if (!BN_copy(aL, a_L) || !BN_copy(aR, a_R) + || !BN_copy(bL, b_L) || !BN_copy(bR, b_R)) + goto end; + + } + + G_L = sk_EC_POINT_value(p_sk_G, i); + G_R = sk_EC_POINT_value(p_sk_G, i + m); + H_L = sk_EC_POINT_value(p_sk_H, i); + H_R = sk_EC_POINT_value(p_sk_H, i + m); + + if (!zkp_poly_points_append(poly_l, G_R, aL) + || !zkp_poly_points_append(poly_l, H_L, bR) + || !zkp_poly_points_append(poly_r, G_L, aR) + || !zkp_poly_points_append(poly_r, H_R, bL)) + goto end; + } + + /* (23, 24) */ + if (!zkp_poly_points_append(poly_l, ctx->U, cL) + || !zkp_poly_points_append(poly_r, ctx->U, cR)) + goto end; + + if (!zkp_poly_points_mul(poly_l, L, NULL, group, bn_ctx) + || !zkp_poly_points_mul(poly_r, R, NULL, group, bn_ctx)) + goto end; + + /* compute the challenge */ + if (!ZKP_TRANSCRIPT_append_point(transcript, "L", L, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "R", R, group)) + goto end; + + if (!ZKP_TRANSCRIPT_challange(transcript, "x", x)) + goto end; + + /* (26, 27) */ + if (!BN_mod_inverse(x_inv, x, order, bn_ctx)) + goto end; + + for (i = 0; i < m; i++) { + u = BN_CTX_get(bn_ctx); + u_inv = BN_CTX_get(bn_ctx); + if (u_inv == NULL) + goto end; + + if (n == pp_num) { + G_factors_L = sk_BIGNUM_value(ctx->sk_G_factors, i); + G_factors_R = sk_BIGNUM_value(ctx->sk_G_factors, i + m); + + if (!BN_mod_mul(u_inv, x_inv, G_factors_L, order, bn_ctx) + || !BN_mod_mul(u, x, G_factors_R, order, bn_ctx)) + goto end; + } else { + if (!BN_copy(u, x) || !BN_copy(u_inv, x_inv)) + goto end; + } + + zkp_poly_points_reset(poly_g); + + G_L = sk_EC_POINT_value(p_sk_G, i); + G_R = sk_EC_POINT_value(p_sk_G, i + m); + sk_G_L = sk_EC_POINT_value(sk_G, i); + + if (!zkp_poly_points_append(poly_g, G_L, u_inv) + || !zkp_poly_points_append(poly_g, G_R, u)) + goto end; + + /* (29) */ + if (!zkp_poly_points_mul(poly_g, sk_G_L, NULL, group, bn_ctx)) + goto end; + + u = BN_CTX_get(bn_ctx); + u_inv = BN_CTX_get(bn_ctx); + if (u_inv == NULL) + goto end; + + if (n == pp_num) { + H_factors_L = sk_BIGNUM_value(ctx->sk_H_factors, i); + H_factors_R = sk_BIGNUM_value(ctx->sk_H_factors, i + m); + + if (!BN_mod_mul(u, x, H_factors_L, order, bn_ctx) + || !BN_mod_mul(u_inv, x_inv, H_factors_R, order, bn_ctx)) + goto end; + } else { + if (!BN_copy(u, x) || !BN_copy(u_inv, x_inv)) + goto end; + } + + zkp_poly_points_reset(poly_h); + + H_L = sk_EC_POINT_value(p_sk_H, i); + H_R = sk_EC_POINT_value(p_sk_H, i + m); + sk_H_L = sk_EC_POINT_value(sk_H, i); + + if (!zkp_poly_points_append(poly_h, H_L, u) + || !zkp_poly_points_append(poly_h, H_R, u_inv)) + goto end; + + /* (30) */ + if (!zkp_poly_points_mul(poly_h, sk_H_L, NULL, group, bn_ctx)) + goto end; + + sk_a_L = sk_BIGNUM_value(sk_a, i); + sk_b_L = sk_BIGNUM_value(sk_b, i); + a_L = sk_BIGNUM_value(p_sk_a, i); + a_R = sk_BIGNUM_value(p_sk_a, i + m); + b_L = sk_BIGNUM_value(p_sk_b, i); + b_R = sk_BIGNUM_value(p_sk_b, i + m); + + /* (33) */ + if (!BN_mod_mul(sk_a_L, a_L, x, order, bn_ctx) + || !BN_mod_mul(t, a_R, x_inv, order, bn_ctx) + || !BN_mod_add(sk_a_L, sk_a_L, t, order, bn_ctx)) + goto end; + + /* (34) */ + if (!BN_mod_mul(sk_b_L, b_L, x_inv, order, bn_ctx) + || !BN_mod_mul(t, b_R, x, order, bn_ctx) + || !BN_mod_add(sk_b_L, sk_b_L, t, order, bn_ctx)) + goto end; + } + + if (sk_EC_POINT_push(proof->sk_L, L) <= 0 + || sk_EC_POINT_push(proof->sk_R, R) <= 0) + goto end; + + L = R = NULL; + p_sk_G = sk_G; + p_sk_H = sk_H; + p_sk_a = sk_a; + p_sk_b = sk_b; + } + + if (!BN_copy(proof->a, sk_BIGNUM_value(p_sk_a, 0)) + || !BN_copy(proof->b, sk_BIGNUM_value(p_sk_b, 0))) + goto end; + + /* + BN_debug_print(NULL, proof->a, "ip_proof->a"); + BN_debug_print(NULL, proof->b, "ip_proof->b"); + */ + + ret = proof; + proof = NULL; + +end: + sk_BIGNUM_free(sk_a); + sk_BIGNUM_free(sk_b); + sk_EC_POINT_pop_free(sk_G, EC_POINT_free); + sk_EC_POINT_pop_free(sk_H, EC_POINT_free); + + EC_POINT_free(L); + EC_POINT_free(R); + EC_POINT_free(P); + + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + + zkp_poly_points_free(poly_l); + zkp_poly_points_free(poly_r); + zkp_poly_points_free(poly_g); + zkp_poly_points_free(poly_h); + + bp_inner_product_proof_free(proof); + + return ret; +} + + +int bp_inner_product_proof_verify(bp_inner_product_ctx_t *ctx, + bp_inner_product_proof_t *proof) +{ + int ret = 0; + int i, j, m, n, proof_num, pp_num; + EC_POINT *P = NULL, *L, *R, *G, *H; + ZKP_TRANSCRIPT *transcript; + BN_CTX *bn_ctx = NULL; + BIGNUM **vec_x = NULL, **vec_x_inv = NULL, *G_factors, *H_factors; + BIGNUM *s, *s_inv, *u, *u_inv, *x2, *x2_inv; + zkp_poly_points_t *poly = NULL; + const BIGNUM *order; + const EC_GROUP *group; + bp_inner_product_pub_param_t *pp; + + if (!ctx || !proof) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + transcript = ctx->transcript; + pp = ctx->pp; + group = pp->group; + order = EC_GROUP_get0_order(group); + pp_num = sk_EC_POINT_num(pp->sk_G); + proof_num = sk_EC_POINT_num(proof->sk_L); + n = 2 * proof_num + 2 * pp_num + 1; + + if (!(vec_x = OPENSSL_zalloc(proof_num * sizeof(*vec_x))) + || !(vec_x_inv = OPENSSL_zalloc(proof_num * sizeof(*vec_x_inv)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto end; + } + + if (!(poly = zkp_poly_points_new(n))) + goto end; + + if (!(P = EC_POINT_new(group))) + goto end; + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (!bn_ctx) + goto end; + + BN_CTX_start(bn_ctx); + s = BN_CTX_get(bn_ctx); + s_inv = BN_CTX_get(bn_ctx); + if (s_inv == NULL) + goto end; + + for (i = 0; i < proof_num; i++) { + vec_x[i] = BN_CTX_get(bn_ctx); + vec_x_inv[i] = BN_CTX_get(bn_ctx); + x2 = BN_CTX_get(bn_ctx); + x2_inv = BN_CTX_get(bn_ctx); + if (x2_inv == NULL) + goto end; + + L = sk_EC_POINT_value(proof->sk_L, i); + R = sk_EC_POINT_value(proof->sk_R, i); + + /* compute hash */ + if (!ZKP_TRANSCRIPT_append_point(transcript, "L", L, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "R", R, group)) + goto end; + + if (!ZKP_TRANSCRIPT_challange(transcript, "x", vec_x[i])) + goto end; + + if (!BN_mod_inverse(vec_x_inv[i], vec_x[i], order, bn_ctx) + || !BN_mod_sqr(x2, vec_x[i], order, bn_ctx) + || !BN_mod_inverse(x2_inv, x2, order, bn_ctx)) + goto end; + + BN_set_negative(x2, !BN_is_negative(x2)); + BN_set_negative(x2_inv, !BN_is_negative(x2_inv)); + + if (!zkp_poly_points_append(poly, L, x2) || !zkp_poly_points_append(poly, R, x2_inv)) + goto end; + } + + for (i = 0; i < pp_num; i++) { + G = sk_EC_POINT_value(pp->sk_G, i); + H = sk_EC_POINT_value(pp->sk_H, i); + G_factors = sk_BIGNUM_value(ctx->sk_G_factors, i); + H_factors = sk_BIGNUM_value(ctx->sk_H_factors, i); + + u = BN_CTX_get(bn_ctx); + u_inv = BN_CTX_get(bn_ctx); + if (u == NULL) + goto end; + + BN_one(s); + for (j = 0; j < proof_num; j++) { + m = i & (1 << (proof_num - j - 1)); + if (!BN_mod_mul(s, s, m ? vec_x[j] : vec_x_inv[j], order, bn_ctx)) + goto end; + } + + if (!BN_mod_inverse(s_inv, s, order, bn_ctx)) + goto end; + + if (!BN_mod_mul(s, s, proof->a, order, bn_ctx) + || !BN_mod_mul(u, s, G_factors, order, bn_ctx) + || !BN_mod_mul(s_inv, s_inv, proof->b, order, bn_ctx) + || !BN_mod_mul(u_inv, s_inv, H_factors, order, bn_ctx)) + goto end; + + if (!zkp_poly_points_append(poly, G, u) || !zkp_poly_points_append(poly, H, u_inv)) + goto end; + } + + if (!BN_mod_mul(s, proof->a, proof->b, order, bn_ctx)) + goto end; + + if (!zkp_poly_points_append(poly, ctx->U, s)) + goto end; + + if (!zkp_poly_points_mul(poly, P, NULL, group, bn_ctx)) + goto end; + + ret = EC_POINT_cmp(group, P, ctx->P, bn_ctx) == 0; + +end: + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + + zkp_poly_points_free(poly); + EC_POINT_free(P); + OPENSSL_free(vec_x); + OPENSSL_free(vec_x_inv); + return ret; +} diff --git a/openssl/src/crypto/zkp/bulletproofs/inner_product.h b/openssl/src/crypto/zkp/bulletproofs/inner_product.h new file mode 100644 index 000000000..fa59b0c78 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/inner_product.h @@ -0,0 +1,76 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_BP_INNER_PRODUCT_LOCAL_H +# define HEADER_BP_INNER_PRODUCT_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include "internal/refcount.h" + +typedef struct bp_inner_product_pub_param_st { + const EC_GROUP *group; + STACK_OF(EC_POINT) *sk_G; + STACK_OF(EC_POINT) *sk_H; +} bp_inner_product_pub_param_t; + +typedef struct bp_inner_product_ctx_st { + ZKP_TRANSCRIPT *transcript; + EC_POINT *P; + EC_POINT *U; + STACK_OF(BIGNUM) *sk_G_factors; + STACK_OF(BIGNUM) *sk_H_factors; + bp_inner_product_pub_param_t *pp; +} bp_inner_product_ctx_t; + +typedef struct bp_inner_product_witness_st { + STACK_OF(BIGNUM) *sk_a; + STACK_OF(BIGNUM) *sk_b; +} bp_inner_product_witness_t; + +typedef struct bp_inner_product_proof_st { + STACK_OF(EC_POINT) *sk_L; + STACK_OF(EC_POINT) *sk_R; + BIGNUM *a; + BIGNUM *b; +} bp_inner_product_proof_t; + +bp_inner_product_pub_param_t *bp_inner_product_pub_param_new(const EC_GROUP *group, + STACK_OF(EC_POINT) *sk_G, + STACK_OF(EC_POINT) *sk_H); +void bp_inner_product_pub_param_free(bp_inner_product_pub_param_t *pp); +bp_inner_product_ctx_t *bp_inner_product_ctx_new(bp_inner_product_pub_param_t *pp, + ZKP_TRANSCRIPT *transcript, + EC_POINT *U, EC_POINT *P, + STACK_OF(BIGNUM) *sk_G_factors, + STACK_OF(BIGNUM) *sk_H_factors); +void bp_inner_product_ctx_free(bp_inner_product_ctx_t *ctx); +bp_inner_product_witness_t *bp_inner_product_witness_new(STACK_OF(BIGNUM) *sk_a, + STACK_OF(BIGNUM) *sk_b); +void bp_inner_product_witness_free(bp_inner_product_witness_t *witness); +bp_inner_product_proof_t *bp_inner_product_proof_alloc(int n); +bp_inner_product_proof_t *bp_inner_product_proof_new(bp_inner_product_ctx_t *ctx); +void bp_inner_product_proof_free(bp_inner_product_proof_t *proof); +bp_inner_product_proof_t *bp_inner_product_proof_prove(bp_inner_product_ctx_t *ctx, + bp_inner_product_witness_t *witness); +int bp_inner_product_proof_verify(bp_inner_product_ctx_t *ctx, + bp_inner_product_proof_t *proof); + +# ifdef __cplusplus +} +# endif + +#endif + diff --git a/openssl/src/crypto/zkp/bulletproofs/r1cs.c b/openssl/src/crypto/zkp/bulletproofs/r1cs.c new file mode 100644 index 000000000..c577e72ce --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/r1cs.c @@ -0,0 +1,1228 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include +#include "r1cs.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) +DEFINE_STACK_OF(BP_VARIABLE) +DEFINE_STACK_OF(BP_R1CS_LINEAR_COMBINATION) +DEFINE_STACK_OF(BP_R1CS_LINEAR_COMBINATION_ITEM) + +BP_R1CS_CTX *BP_R1CS_CTX_new(BP_PUB_PARAM *pp, BP_WITNESS *witness, + ZKP_TRANSCRIPT *transcript) +{ + BP_R1CS_CTX *ctx = NULL; + + if (pp == NULL || witness == NULL || transcript == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ctx->transcript = transcript; + + if (!BP_PUB_PARAM_up_ref(pp)) + goto err; + + ctx->pp = pp; + + if (!BP_WITNESS_up_ref(witness)) + goto err; + + ctx->witness = witness; + + if ((ctx->constraints = sk_BP_R1CS_LINEAR_COMBINATION_new_null()) == NULL + || (ctx->aL = sk_BIGNUM_new_null()) == NULL + || (ctx->aR = sk_BIGNUM_new_null()) == NULL + || (ctx->aO = sk_BIGNUM_new_null()) == NULL) + goto err; + + return ctx; + +err: + BP_R1CS_CTX_free(ctx); + return NULL; +} + +void BP_R1CS_CTX_free(BP_R1CS_CTX *ctx) +{ + if (ctx == NULL) + return; + + BP_PUB_PARAM_down_ref(ctx->pp); + BP_WITNESS_down_ref(ctx->witness); + + sk_BP_R1CS_LINEAR_COMBINATION_pop_free(ctx->constraints, + BP_R1CS_LINEAR_COMBINATION_free); + sk_BIGNUM_pop_free(ctx->aL, BN_free); + sk_BIGNUM_pop_free(ctx->aR, BN_free); + sk_BIGNUM_pop_free(ctx->aO, BN_free); + + OPENSSL_clear_free((void *)ctx, sizeof(*ctx)); +} + +BP_R1CS_PROOF *BP_R1CS_PROOF_new(BP_R1CS_CTX *ctx) +{ + BP_R1CS_PROOF *proof = NULL; + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if ((proof->AI1 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->AO1 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->S1 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->AI2 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->AO2 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->S2 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->T1 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->T3 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->T4 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->T5 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->T6 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->taux = BN_new()) == NULL + || (proof->mu = BN_new()) == NULL + || (proof->tx = BN_new()) == NULL) + goto err; + + EC_POINT_set_to_infinity(ctx->pp->group, proof->AI1); + EC_POINT_set_to_infinity(ctx->pp->group, proof->AO1); + EC_POINT_set_to_infinity(ctx->pp->group, proof->S1); + EC_POINT_set_to_infinity(ctx->pp->group, proof->AI2); + EC_POINT_set_to_infinity(ctx->pp->group, proof->AO2); + EC_POINT_set_to_infinity(ctx->pp->group, proof->S2); + EC_POINT_set_to_infinity(ctx->pp->group, proof->T1); + EC_POINT_set_to_infinity(ctx->pp->group, proof->T3); + EC_POINT_set_to_infinity(ctx->pp->group, proof->T4); + EC_POINT_set_to_infinity(ctx->pp->group, proof->T5); + EC_POINT_set_to_infinity(ctx->pp->group, proof->T6); + + BN_zero(proof->taux); + BN_zero(proof->mu); + BN_zero(proof->tx); + + proof->references = 1; + + if ((proof->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + return proof; +err: + BP_R1CS_PROOF_free(proof); + return NULL; +} + +void BP_R1CS_PROOF_free(BP_R1CS_PROOF *proof) +{ + int ref; + + if (proof == NULL) + return; + + CRYPTO_DOWN_REF(&proof->references, &ref, proof->lock); + REF_PRINT_COUNT("BP_R1CS_PROOF", proof); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + EC_POINT_free(proof->AI1); + EC_POINT_free(proof->AO1); + EC_POINT_free(proof->S1); + EC_POINT_free(proof->AI2); + EC_POINT_free(proof->AO2); + EC_POINT_free(proof->S2); + EC_POINT_free(proof->T1); + EC_POINT_free(proof->T3); + EC_POINT_free(proof->T4); + EC_POINT_free(proof->T5); + EC_POINT_free(proof->T6); + BN_free(proof->taux); + BN_free(proof->mu); + BN_free(proof->tx); + CRYPTO_THREAD_lock_free(proof->lock); + OPENSSL_clear_free((void *)proof, sizeof(*proof)); +} + +int BP_WITNESS_r1cs_commit(BP_WITNESS *witness, const char *name, BIGNUM *v) +{ + const BIGNUM *order; + BIGNUM *r = NULL, *val = NULL; + EC_POINT *V = NULL; + BP_VARIABLE *var = NULL; + + if (witness == NULL || name == NULL || v == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (strlen(name) > BP_VARIABLE_NAME_MAX_LEN) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_VARIABLE_NAME_TOO_LONG); + return 0; + } + + order = EC_GROUP_get0_order(witness->group); + + if (BP_WITNESS_get_variable_index(witness, name) >= 0) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_VARIABLE_DUPLICATED); + return 0; + } + + r = BN_new(); + val = BN_dup(v); + V = EC_POINT_new(witness->group); + if (r == NULL || val == NULL || V == NULL) + goto err; + + if (!zkp_rand_range(r, order)) + goto err; + + /* (69) */ + if (!EC_POINT_mul(witness->group, V, v, witness->H, r, NULL)) + goto err; + + if (!(var = BP_VARIABLE_new(name, V, witness->group))) + goto err; + + if (sk_BIGNUM_push(witness->sk_r, r) <= 0) + goto err; + + r = NULL; + + if (sk_BIGNUM_push(witness->sk_v, val) <= 0) + goto err; + + val = NULL; + + if (sk_BP_VARIABLE_push(witness->sk_V, var) <= 0) + goto err; + + return 1; +err: + BN_free(r); + BN_free(val); + EC_POINT_free(V); + BP_VARIABLE_free(var); + return 0; +} + +BP_R1CS_LINEAR_COMBINATION *BP_WITNESS_r1cs_linear_combination_commit(BP_WITNESS *witness, + const char *name, + BIGNUM *v) +{ + int num; + BP_R1CS_VARIABLE *r1cs_var = NULL; + BP_R1CS_LINEAR_COMBINATION *lc = NULL; + + if (!BP_WITNESS_r1cs_commit(witness, name, v)) + return 0; + + num = sk_BP_VARIABLE_num(witness->sk_V) - 1; + + if ((r1cs_var = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_COMMITTED, num)) == NULL) + goto err; + + if (!(lc = BP_R1CS_LINEAR_COMBINATION_new_from_param(r1cs_var, NULL))) + goto err; + + lc->type = BP_R1CS_LC_TYPE_PROVE; + + return lc; +err: + BP_R1CS_VARIABLE_free(r1cs_var); + BP_R1CS_LINEAR_COMBINATION_free(lc); + return NULL; +} + +BP_R1CS_LINEAR_COMBINATION *BP_WITNESS_r1cs_linear_combination_get(BP_WITNESS *witness, + const char *name) +{ + int i; + BP_R1CS_VARIABLE *var = NULL; + BP_R1CS_LINEAR_COMBINATION *ret = NULL; + + if (witness == NULL || name == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + i = BP_WITNESS_get_variable_index(witness, name); + if (i < 0) + return NULL; + + if ((var = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_COMMITTED, i)) == NULL) + goto err; + + if (!(ret = BP_R1CS_LINEAR_COMBINATION_new_from_param(var, NULL))) + goto err; + + ret->type = BP_R1CS_LC_TYPE_VERIFY; + + return ret; +err: + BP_R1CS_VARIABLE_free(var); + BP_R1CS_LINEAR_COMBINATION_free(ret); + return NULL; +} + +BP_R1CS_PROOF *BP_R1CS_PROOF_prove(BP_R1CS_CTX *ctx) +{ + EC_GROUP *group; + const BIGNUM *order; + BN_CTX *bn_ctx = NULL; + int i, j, k, m, n, nn, n1, seed_buf_len, size, padded_n, pp_capacity; + unsigned char *seed_buf = NULL, *buf = NULL; + BIGNUM *alpha, *beta, *rho, *r = NULL, *product; + BIGNUM **sL = NULL, **sR = NULL; + BIGNUM *x, *u, *w, *z, *z2, *pow_z, *pw = NULL; + BIGNUM *y, *y_inv, *pow_y_inv, *pow_y; + BIGNUM **wL = NULL, **wR = NULL, **wO = NULL, **wV = NULL; + BIGNUM *tau1, *tau2, *tau3, *tau4, *tau5, *tau6; + BIGNUM *g_scalar, *h_scalar, *padded_l, *padded_r; + STACK_OF(EC_POINT) *sk_G = NULL, *sk_H = NULL; + STACK_OF(BIGNUM) *sk_G_scalars = NULL, *sk_H_scalars = NULL; + STACK_OF(BIGNUM) *sk_l = NULL, *sk_r = NULL; + EC_POINT *U = NULL, *G, *H; + zkp_poly3_t *poly_l = NULL, *poly_r = NULL; + zkp_poly6_t *poly_t = NULL, *poly_tau = NULL; + zkp_poly_points_t *poly_ai1 = NULL, *poly_ao1 = NULL, *poly_s1 = NULL; + bp_inner_product_ctx_t *ip_ctx = NULL; + bp_inner_product_witness_t *ip_witness = NULL; + bp_inner_product_pub_param_t *ip_pp = NULL; + ZKP_TRANSCRIPT *transcript; + BP_PUB_PARAM *pp; + BP_WITNESS *witness; + BP_VARIABLE *var; + BP_R1CS_VARIABLE *r1cs_var; + BP_R1CS_LINEAR_COMBINATION *lc; + BP_R1CS_LINEAR_COMBINATION_ITEM *item; + BP_R1CS_PROOF *proof = NULL, *ret = NULL; + + if (ctx == NULL || ctx->constraints == NULL || ctx->witness == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + transcript = ctx->transcript; + witness = ctx->witness; + pp = ctx->pp; + group = pp->group; + order = EC_GROUP_get0_order(group); + + n1 = sk_BIGNUM_num(ctx->aL); + nn = n1 + 1; + padded_n = zkp_next_power_of_two(n1); + pp_capacity = pp->gens_capacity * pp->party_capacity; + if (pp_capacity < padded_n) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_EXCEEDS_PP_CAPACITY); + goto err; + } + + if (!(proof = BP_R1CS_PROOF_new(ctx)) || !(U = EC_POINT_new(group))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!(sk_G = sk_EC_POINT_new_reserve(NULL, padded_n)) + || !(sk_H = sk_EC_POINT_new_reserve(NULL, padded_n)) + || !(sk_G_scalars = sk_BIGNUM_new_reserve(NULL, padded_n)) + || !(sk_H_scalars = sk_BIGNUM_new_reserve(NULL, padded_n))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + return NULL; + + BN_CTX_start(bn_ctx); + + alpha = BN_CTX_get(bn_ctx); + beta = BN_CTX_get(bn_ctx); + rho = BN_CTX_get(bn_ctx); + product = BN_CTX_get(bn_ctx); + x = BN_CTX_get(bn_ctx); + y = BN_CTX_get(bn_ctx); + y_inv = BN_CTX_get(bn_ctx); + pow_y = BN_CTX_get(bn_ctx); + pow_y_inv = BN_CTX_get(bn_ctx); + z = BN_CTX_get(bn_ctx); + pow_z = BN_CTX_get(bn_ctx); + z2 = BN_CTX_get(bn_ctx); + u = BN_CTX_get(bn_ctx); + w = BN_CTX_get(bn_ctx); + if (w == NULL) + goto err; + + m = sk_BP_VARIABLE_num(witness->sk_V); + for (i = 0; i < m; i++) { + var = sk_BP_VARIABLE_value(witness->sk_V, i); + if (var == NULL) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "V", var->point, group)) + goto err; + } + + m = sk_BIGNUM_num(witness->sk_v); + if (!ZKP_TRANSCRIPT_append_int64(transcript, "m", m)) + goto err; + + m = sk_BIGNUM_num(witness->sk_r); + if (m > 0) { + r = sk_BIGNUM_value(witness->sk_r, 0); + if (r == NULL) + goto err; + + seed_buf_len = BN_num_bytes(r) * m * 2; + if ((seed_buf = OPENSSL_zalloc(seed_buf_len)) == NULL) + goto err; + } + + buf = seed_buf; + + for (i = 0; i < m; i++) { + r = sk_BIGNUM_value(witness->sk_r, i); + if (r == NULL) + goto err; + + size = BN_num_bytes(r); + if (!BN_bn2bin(r, buf)) + goto err; + + buf += size; + } + + RAND_seed(seed_buf, buf - seed_buf); + + n = n1 * 2 + 1; + + if (!(poly_ai1 = zkp_poly_points_new(n)) + || !(poly_ao1 = zkp_poly_points_new(n)) + || !(poly_s1 = zkp_poly_points_new(n))) + goto err; + + if (!zkp_rand_range(alpha, order) + || !zkp_rand_range(beta, order) + || !zkp_rand_range(rho, order)) + goto err; + + if (!(sL = OPENSSL_zalloc(sizeof(*sL) * nn)) + || !(sR = OPENSSL_zalloc(sizeof(*sR) * nn))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!zkp_random_bn_gen(group, sL, n1, bn_ctx) + || !zkp_random_bn_gen(group, sR, n1, bn_ctx)) + goto err; + + if (!zkp_poly_points_append(poly_ai1, pp->H, alpha) + || !zkp_poly_points_append(poly_ao1, pp->H, beta) + || !zkp_poly_points_append(poly_s1, pp->H, rho)) + goto err; + + for (i = 0; i < n1; i++) { + G = sk_EC_POINT_value(pp->sk_G, i); + H = sk_EC_POINT_value(pp->sk_H, i); + if (!zkp_poly_points_append(poly_ai1, G, sk_BIGNUM_value(ctx->aL, i)) + || !zkp_poly_points_append(poly_ai1, H, sk_BIGNUM_value(ctx->aR, i)) + || !zkp_poly_points_append(poly_ao1, G, sk_BIGNUM_value(ctx->aO, i)) + || !zkp_poly_points_append(poly_s1, G, sL[i]) + || !zkp_poly_points_append(poly_s1, H, sR[i])) + goto err; + } + + if (!zkp_poly_points_mul(poly_ai1, proof->AI1, NULL, group, bn_ctx) + || !zkp_poly_points_mul(poly_ao1, proof->AO1, NULL, group, bn_ctx) + || !zkp_poly_points_mul(poly_s1, proof->S1, NULL, group, bn_ctx)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "A_I1", proof->AI1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A_O1", proof->AO1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "S1", proof->S1, group)) + goto err; + + /* + * TODO + * Process the remaining constraints. + */ + + n = sk_BIGNUM_num(ctx->aL); + nn = n + 1; + padded_n = zkp_next_power_of_two(n); + if (pp_capacity < padded_n) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_EXCEEDS_PP_CAPACITY); + goto err; + } + + if (!ZKP_TRANSCRIPT_append_point(transcript, "A_I2", proof->AI2, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A_O2", proof->AO2, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "S2", proof->S2, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "y", y) + || !ZKP_TRANSCRIPT_challange(transcript, "z", z)) + goto err; + + if (!BN_mod_sqr(z2, z, order, bn_ctx) || !BN_copy(pow_z, z) + || !BN_mod_inverse(y_inv, y, order, bn_ctx)) + goto err; + + /* + * flatten the constraints + */ + m = sk_BIGNUM_num(witness->sk_v); + if ((wL = OPENSSL_zalloc(sizeof(*wL) * nn)) == NULL + || (wR = OPENSSL_zalloc(sizeof(*wR) * nn)) == NULL + || (wO = OPENSSL_zalloc(sizeof(*wO) * nn)) == NULL + || (wV = OPENSSL_zalloc(sizeof(*wV) * m)) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + for (i = 0; i < n; i++) { + wL[i] = BN_CTX_get(bn_ctx); + wR[i] = BN_CTX_get(bn_ctx); + wO[i] = BN_CTX_get(bn_ctx); + if (wO[i] == NULL) + goto err; + + BN_zero(wL[i]); + BN_zero(wR[i]); + BN_zero(wO[i]); + } + + for (i = 0; i < m; i++) { + wV[i] = BN_CTX_get(bn_ctx); + if (wV[i] == NULL) + goto err; + + BN_zero(wV[i]); + } + + k = sk_BP_R1CS_LINEAR_COMBINATION_num(ctx->constraints); + for (i = 0; i < k; i++) { + lc = sk_BP_R1CS_LINEAR_COMBINATION_value(ctx->constraints, i); + if (lc == NULL) + goto err; + + if (lc->type != BP_R1CS_LC_TYPE_PROVE) + continue; + + m = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_num(lc->items); + for (j = 0; j < m; j++) { + item = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_value(lc->items, j); + if (item == NULL) + goto err; + + r1cs_var = item->variable; + + switch (r1cs_var->type) { + case BP_R1CS_VARIABLE_COMMITTED: + pw = wV[r1cs_var->value]; + break; + case BP_R1CS_VARIABLE_MULTIPLIER_LEFT: + pw = wL[r1cs_var->value]; + break; + case BP_R1CS_VARIABLE_MULTIPLIER_RIGHT: + pw = wR[r1cs_var->value]; + break; + case BP_R1CS_VARIABLE_MULTIPLIER_OUTPUT: + pw = wO[r1cs_var->value]; + break; + default: + break; + } + + if (pw == NULL) + continue; + + if (!BN_mod_mul(product, pow_z, item->scalar, order, bn_ctx)) + goto err; + + if (r1cs_var->type == BP_R1CS_VARIABLE_COMMITTED) { + if (!BN_mod_sub(pw, pw, product, order, bn_ctx)) + goto err; + } else { + if (!BN_mod_add(pw, pw, product, order, bn_ctx)) + goto err; + } + + pw = NULL; + } + + if (!BN_mod_mul(pow_z, pow_z, z, order, bn_ctx)) + goto err; + } + + BN_one(pow_y); + BN_one(pow_y_inv); + + if (!(poly_l = zkp_poly3_new(n, order)) || !(poly_r = zkp_poly3_new(n, order))) + goto err; + + for (i = 0; i < n; i++) { + g_scalar = BN_CTX_get(bn_ctx); + h_scalar = BN_CTX_get(bn_ctx); + if (h_scalar == NULL) + goto err; + + if (!BN_mod_mul(poly_l->x1[i], pow_y_inv, wR[i], order, bn_ctx) + || !BN_mod_add(poly_l->x1[i], poly_l->x1[i], sk_BIGNUM_value(ctx->aL, i), + order, bn_ctx) + || !BN_copy(poly_l->x2[i], sk_BIGNUM_value(ctx->aO, i)) + || !BN_copy(poly_l->x3[i], sL[i])) + goto err; + + if (!BN_mod_sub(poly_r->x0[i], wO[i], pow_y, order, bn_ctx) + || !BN_mod_mul(poly_r->x1[i], pow_y, sk_BIGNUM_value(ctx->aR, i), + order, bn_ctx) + || !BN_mod_add(poly_r->x1[i], poly_r->x1[i], wL[i], order, bn_ctx) + || !BN_mod_mul(poly_r->x3[i], pow_y, sR[i], order, bn_ctx)) + goto err; + + BN_one(g_scalar); + + if (!BN_copy(h_scalar, pow_y_inv)) + goto err; + + G = sk_EC_POINT_value(pp->sk_G, i); + H = sk_EC_POINT_value(pp->sk_H, i); + + if (sk_EC_POINT_push(sk_G, G) <= 0 + || sk_EC_POINT_push(sk_H, H) <= 0 + || sk_BIGNUM_push(sk_G_scalars, g_scalar) <= 0 + || sk_BIGNUM_push(sk_H_scalars, h_scalar) <= 0) + goto err; + + if (!BN_mod_mul(pow_y, pow_y, y, order, bn_ctx) + || !BN_mod_mul(pow_y_inv, pow_y_inv, y_inv, order, bn_ctx)) + goto err; + } + + if (!(poly_t = zkp_poly6_new(order))) + goto err; + + if (!zkp_poly3_special_inner_product(poly_t, poly_l, poly_r)) + goto err; + + tau1 = BN_CTX_get(bn_ctx); + tau2 = BN_CTX_get(bn_ctx); + tau3 = BN_CTX_get(bn_ctx); + tau4 = BN_CTX_get(bn_ctx); + tau5 = BN_CTX_get(bn_ctx); + if (!(tau6 = BN_CTX_get(bn_ctx))) + goto err; + + if (!zkp_rand_range(tau1, order) + || !zkp_rand_range(tau3, order) + || !zkp_rand_range(tau4, order) + || !zkp_rand_range(tau5, order) + || !zkp_rand_range(tau6, order)) + goto err; + + if (!EC_POINT_mul(group, proof->T1, poly_t->t1, pp->H, tau1, bn_ctx) + || !EC_POINT_mul(group, proof->T3, poly_t->t3, pp->H, tau3, bn_ctx) + || !EC_POINT_mul(group, proof->T4, poly_t->t4, pp->H, tau4, bn_ctx) + || !EC_POINT_mul(group, proof->T5, poly_t->t5, pp->H, tau5, bn_ctx) + || !EC_POINT_mul(group, proof->T6, poly_t->t6, pp->H, tau6, bn_ctx)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "T_1", proof->T1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T_3", proof->T3, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T_4", proof->T4, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T_5", proof->T5, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T_6", proof->T6, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "u", u) + || !ZKP_TRANSCRIPT_challange(transcript, "x", x)) + goto err; + + BN_zero(tau2); + m = sk_BIGNUM_num(witness->sk_r); + for (i = 0; i < m; i++) { + r = sk_BIGNUM_value(witness->sk_r, i); + if (r == NULL || !BN_mod_mul(product, wV[i], r, order, bn_ctx)) + goto err; + + if (!BN_mod_add(tau2, tau2, product, order, bn_ctx)) + goto err; + } + + if (!(poly_tau = zkp_poly6_new(order))) + goto err; + + poly_tau->t1 = tau1; + poly_tau->t2 = tau2; + poly_tau->t3 = tau3; + poly_tau->t4 = tau4; + poly_tau->t5 = tau5; + poly_tau->t6 = tau6; + + if (!zkp_poly6_eval(poly_t, x, proof->tx) || !zkp_poly6_eval(poly_tau, x, proof->taux)) + goto err; + + if (!(sk_l = zkp_poly3_eval(poly_l, x)) || !(sk_r = zkp_poly3_eval(poly_r, x))) + goto err; + + /* TODO: 2nd phase commitments */ + + if (!BN_mod_mul(proof->mu, x, rho, order, bn_ctx) + || !BN_mod_add(proof->mu, proof->mu, beta, order, bn_ctx) + || !BN_mod_mul(proof->mu, proof->mu, x, order, bn_ctx) + || !BN_mod_add(proof->mu, proof->mu, alpha, order, bn_ctx) + || !BN_mod_mul(proof->mu, proof->mu, x, order, bn_ctx)) + goto err; + + if (!ZKP_TRANSCRIPT_append_bn(transcript, "t_x", proof->tx) + || !ZKP_TRANSCRIPT_append_bn(transcript, "t_x_blinding", proof->taux) + || !ZKP_TRANSCRIPT_append_bn(transcript, "e_blinding", proof->mu)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "w", w)) + goto err; + + if (!EC_POINT_mul(group, U, w, NULL, NULL, bn_ctx)) + goto err; + + for (i = n; i < padded_n; i++) { + g_scalar = BN_CTX_get(bn_ctx); + h_scalar = BN_CTX_get(bn_ctx); + padded_l = BN_CTX_get(bn_ctx); + padded_r = BN_CTX_get(bn_ctx); + if (padded_r == NULL) + goto err; + + if (!BN_copy(g_scalar, u) + || !BN_mod_mul(h_scalar, pow_y_inv, u, order, bn_ctx)) + goto err; + + if (!BN_copy(padded_r, pow_y)) + goto err; + + BN_set_negative(padded_r, 1); + BN_zero(padded_l); + + G = sk_EC_POINT_value(pp->sk_G, i); + H = sk_EC_POINT_value(pp->sk_H, i); + + if (sk_EC_POINT_push(sk_G, G) <= 0 + || sk_EC_POINT_push(sk_H, H) <= 0 + || sk_BIGNUM_push(sk_G_scalars, g_scalar) <= 0 + || sk_BIGNUM_push(sk_H_scalars, h_scalar) <= 0 + || sk_BIGNUM_push(sk_l, padded_l) <= 0 + || sk_BIGNUM_push(sk_r, padded_r) <= 0) + goto err; + + if (!BN_mod_mul(pow_y, pow_y, y, order, bn_ctx) + || !BN_mod_mul(pow_y_inv, pow_y_inv, y_inv, order, bn_ctx)) + goto err; + } + + if (!(ip_pp = bp_inner_product_pub_param_new(group, sk_G, sk_H)) + || !(ip_ctx = bp_inner_product_ctx_new(ip_pp, transcript, U, NULL, + sk_G_scalars, sk_H_scalars)) + || !(ip_witness = bp_inner_product_witness_new(sk_l, sk_r))) + goto err; + + if (!(proof->ip_proof = bp_inner_product_proof_prove(ip_ctx, ip_witness))) + goto err; + + ret = proof; + proof = NULL; + +err: + ZKP_TRANSCRIPT_reset(transcript); + + bp_inner_product_ctx_free(ip_ctx); + bp_inner_product_pub_param_free(ip_pp); + + zkp_poly3_free(poly_l); + zkp_poly3_free(poly_r); + zkp_poly6_free(poly_t); + + zkp_poly_points_free(poly_ai1); + zkp_poly_points_free(poly_ao1); + zkp_poly_points_free(poly_s1); + + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + + OPENSSL_free(sL); + OPENSSL_free(sR); + sk_EC_POINT_free(sk_G); + sk_EC_POINT_free(sk_H); + sk_BIGNUM_free(sk_G_scalars); + sk_BIGNUM_free(sk_H_scalars); + sk_BIGNUM_free(sk_l); + sk_BIGNUM_free(sk_r); + EC_POINT_free(U); + OPENSSL_free(seed_buf); + BP_R1CS_PROOF_free(proof); + return ret; +} + +int BP_R1CS_PROOF_verify(BP_R1CS_CTX *ctx, BP_R1CS_PROOF *proof) +{ + BN_CTX *bn_ctx = NULL; + EC_GROUP *group; + const BIGNUM *order; + int i, j, m, n, nn, padded_n, v_n, lg_i, lg_n, pp_capacity, ret = 0; + BIGNUM *delta, *bn1; + BIGNUM **vec_s = NULL, **vec_ip_x2 = NULL; + BIGNUM *x, *x2, *x3, *ip_x, *ip_x_inv, *ip_x2, *ip_x2_inv; + BIGNUM *y, *y_inv, *pow_y_inv, *wR_pow_y_inv; + BIGNUM *z, *u, *w, *wc, *pw = NULL, *pow_z, *tmp, *product; + BIGNUM *ux, *ux2, *ux3, *r, *rx, *rx2, *rx3, *rx4, *rx5, *rx6; + BIGNUM *scalar, *g_scalar, *h_scalar, *b_scalar, *v_scalar, *s_a, *s_b; + BIGNUM **wL = NULL, **wR = NULL, **wO = NULL, **wV = NULL; + EC_POINT *P = NULL, *L, *R, *G, *H; + zkp_poly_points_t *poly_p = NULL; + ZKP_TRANSCRIPT *transcript; + BP_PUB_PARAM *pp; + BP_WITNESS *witness; + BP_VARIABLE *var; + BP_R1CS_VARIABLE *r1cs_var; + BP_R1CS_LINEAR_COMBINATION *lc; + BP_R1CS_LINEAR_COMBINATION_ITEM *item; + bp_inner_product_proof_t *ip_proof = NULL; + + if (ctx == NULL || ctx->constraints == NULL || ctx->witness == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + pp = ctx->pp; + transcript = ctx->transcript; + witness = ctx->witness; + group = pp->group; + order = EC_GROUP_get0_order(group); + ip_proof = proof->ip_proof; + pp_capacity = pp->gens_capacity * pp->party_capacity; + + nn = ctx->vars_num + 1; + padded_n = zkp_next_power_of_two(ctx->vars_num); + if (pp_capacity < padded_n) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_EXCEEDS_PP_CAPACITY); + goto err; + } + + v_n = sk_BP_VARIABLE_num(witness->sk_V); + lg_n = sk_EC_POINT_num(ip_proof->sk_L); + if (padded_n != 1 << lg_n) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + if (!(poly_p = zkp_poly_points_new(12 + padded_n * 2 + lg_n * 2 + v_n))) + goto err; + + if (!(P = EC_POINT_new(group)) + || !(vec_ip_x2 = OPENSSL_zalloc((lg_n + 1) * sizeof(*vec_ip_x2))) + || !(vec_s = OPENSSL_zalloc(padded_n * sizeof(*vec_s)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + + BN_CTX_start(bn_ctx); + + bn1 = BN_CTX_get(bn_ctx); + product = BN_CTX_get(bn_ctx); + delta = BN_CTX_get(bn_ctx); + x = BN_CTX_get(bn_ctx); + y = BN_CTX_get(bn_ctx); + y_inv = BN_CTX_get(bn_ctx); + pow_y_inv = BN_CTX_get(bn_ctx); + wR_pow_y_inv = BN_CTX_get(bn_ctx); + z = BN_CTX_get(bn_ctx); + pow_z = BN_CTX_get(bn_ctx); + u = BN_CTX_get(bn_ctx); + w = BN_CTX_get(bn_ctx); + wc = BN_CTX_get(bn_ctx); + s_a = BN_CTX_get(bn_ctx); + s_b = BN_CTX_get(bn_ctx); + x2 = BN_CTX_get(bn_ctx); + x3 = BN_CTX_get(bn_ctx); + ux = BN_CTX_get(bn_ctx); + ux2 = BN_CTX_get(bn_ctx); + ux3 = BN_CTX_get(bn_ctx); + r = BN_CTX_get(bn_ctx); + rx = BN_CTX_get(bn_ctx); + rx2 = BN_CTX_get(bn_ctx); + rx3 = BN_CTX_get(bn_ctx); + rx4 = BN_CTX_get(bn_ctx); + rx5 = BN_CTX_get(bn_ctx); + rx6 = BN_CTX_get(bn_ctx); + ip_x = BN_CTX_get(bn_ctx); + ip_x_inv = BN_CTX_get(bn_ctx); + ip_x2 = BN_CTX_get(bn_ctx); + ip_x2_inv = BN_CTX_get(bn_ctx); + vec_s[0] = BN_CTX_get(bn_ctx); + tmp = BN_CTX_get(bn_ctx); + if (tmp == NULL) + goto err; + + BN_zero(delta); + BN_one(bn1); + BN_one(pow_y_inv); + + if (!zkp_rand_range(r, order)) + goto err; + + //START + for (i = 0; i < v_n; i++) { + var = sk_BP_VARIABLE_value(witness->sk_V, i); + if (var == NULL) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "V", var->point, group)) + goto err; + } + + if (!ZKP_TRANSCRIPT_append_int64(transcript, "m", v_n)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "A_I1", proof->AI1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A_O1", proof->AO1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "S1", proof->S1, group)) + goto err; + + /* + * TODO + * Process the remaining constraints. + */ + + if (!ZKP_TRANSCRIPT_append_point(transcript, "A_I2", proof->AI2, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A_O2", proof->AO2, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "S2", proof->S2, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "y", y) + || !ZKP_TRANSCRIPT_challange(transcript, "z", z)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "T_1", proof->T1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T_3", proof->T3, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T_4", proof->T4, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T_5", proof->T5, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T_6", proof->T6, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "u", u) + || !ZKP_TRANSCRIPT_challange(transcript, "x", x)) + goto err; + + if (!ZKP_TRANSCRIPT_append_bn(transcript, "t_x", proof->tx) + || !ZKP_TRANSCRIPT_append_bn(transcript, "t_x_blinding", proof->taux) + || !ZKP_TRANSCRIPT_append_bn(transcript, "e_blinding", proof->mu)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "w", w)) + goto err; + + if (!BN_mod_inverse(y_inv, y, order, bn_ctx) + || !BN_mod_sqr(x2, x, order, bn_ctx) + || !BN_mod_mul(x3, x2, x, order, bn_ctx) + || !BN_mod_mul(ux, u, x, order, bn_ctx) + || !BN_mod_mul(ux2, ux, x, order, bn_ctx) + || !BN_mod_mul(ux3, ux2, x, order, bn_ctx) + || !BN_mod_mul(rx, r, x, order, bn_ctx) + || !BN_mod_mul(rx2, rx, x, order, bn_ctx) + || !BN_mod_mul(rx3, rx2, x, order, bn_ctx) + || !BN_mod_mul(rx4, rx3, x, order, bn_ctx) + || !BN_mod_mul(rx5, rx4, x, order, bn_ctx) + || !BN_mod_mul(rx6, rx5, x, order, bn_ctx) + || !BN_copy(pow_z, z)) + goto err; + + /* + * flatten the constraints + */ + if ((wL = OPENSSL_zalloc(sizeof(*wL) * nn)) == NULL + || (wR = OPENSSL_zalloc(sizeof(*wR) * nn)) == NULL + || (wO = OPENSSL_zalloc(sizeof(*wO) * nn)) == NULL + || (wV = OPENSSL_zalloc(sizeof(*wV) * v_n)) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + for (i = 0; i < ctx->vars_num; i++) { + wL[i] = BN_CTX_get(bn_ctx); + wR[i] = BN_CTX_get(bn_ctx); + wO[i] = BN_CTX_get(bn_ctx); + if (wO[i] == NULL) + goto err; + + BN_zero(wL[i]); + BN_zero(wR[i]); + BN_zero(wO[i]); + } + + for (i = 0; i < v_n; i++) { + wV[i] = BN_CTX_get(bn_ctx); + if (wV[i] == NULL) + goto err; + + BN_zero(wV[i]); + } + + BN_zero(wc); + + m = sk_BP_R1CS_LINEAR_COMBINATION_num(ctx->constraints); + for (i = 0; i < m; i++) { + lc = sk_BP_R1CS_LINEAR_COMBINATION_value(ctx->constraints, i); + if (lc == NULL) + goto err; + + if (lc->type != BP_R1CS_LC_TYPE_VERIFY) + continue; + + n = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_num(lc->items); + for (j = 0; j < n; j++) { + item = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_value(lc->items, j); + if (item == NULL) + goto err; + + r1cs_var = item->variable; + + switch (r1cs_var->type) { + case BP_R1CS_VARIABLE_COMMITTED: + pw = wV[r1cs_var->value]; + break; + case BP_R1CS_VARIABLE_MULTIPLIER_LEFT: + pw = wL[r1cs_var->value]; + break; + case BP_R1CS_VARIABLE_MULTIPLIER_RIGHT: + pw = wR[r1cs_var->value]; + break; + case BP_R1CS_VARIABLE_MULTIPLIER_OUTPUT: + pw = wO[r1cs_var->value]; + break; + case BP_R1CS_VARIABLE_ONE: + pw = wc; + break; + default: + break; + } + + if (w == NULL) + continue; + + if (!BN_mod_mul(product, pow_z, item->scalar, order, bn_ctx)) + goto err; + + if (r1cs_var->type == BP_R1CS_VARIABLE_COMMITTED + || r1cs_var->type == BP_R1CS_VARIABLE_ONE) { + if (!BN_mod_sub(pw, pw, product, order, bn_ctx)) + goto err; + } else { + if (!BN_mod_add(pw, pw, product, order, bn_ctx)) + goto err; + } + + pw = NULL; + } + + if (!BN_mod_mul(pow_z, pow_z, z, order, bn_ctx)) + goto err; + } + + if (!zkp_poly_points_append(poly_p, proof->AI1, x) + || !zkp_poly_points_append(poly_p, proof->AO1, x2) + || !zkp_poly_points_append(poly_p, proof->S1, x3) + || !zkp_poly_points_append(poly_p, proof->AI2, ux) + || !zkp_poly_points_append(poly_p, proof->AO2, ux2) + || !zkp_poly_points_append(poly_p, proof->S2, ux3) + || !zkp_poly_points_append(poly_p, proof->T1, rx) + || !zkp_poly_points_append(poly_p, proof->T3, rx3) + || !zkp_poly_points_append(poly_p, proof->T4, rx4) + || !zkp_poly_points_append(poly_p, proof->T5, rx5) + || !zkp_poly_points_append(poly_p, proof->T6, rx6)) + goto err; + + BN_one(vec_s[0]); + + for (i = 0; i < lg_n; i++) { + scalar = BN_CTX_get(bn_ctx); + vec_ip_x2[i] = BN_CTX_get(bn_ctx); + if (vec_ip_x2[i] == NULL) + goto err; + + L = sk_EC_POINT_value(ip_proof->sk_L, i); + R = sk_EC_POINT_value(ip_proof->sk_R, i); + if (L == NULL || R == NULL) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "L", L, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "R", R, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "x", ip_x)) + goto err; + + if (!BN_mod_sqr(ip_x2, ip_x, order, bn_ctx) + || !BN_copy(vec_ip_x2[i], ip_x2) + || !BN_mod_inverse(ip_x_inv, ip_x, order, bn_ctx) + || !BN_mod_inverse(ip_x2_inv, ip_x2, order, bn_ctx) + || !BN_mod_mul(vec_s[0], vec_s[0], ip_x_inv, order, bn_ctx)) + goto err; + + if (!BN_copy(scalar, ip_x2) || !zkp_poly_points_append(poly_p, L, scalar)) + goto err; + + scalar = BN_CTX_get(bn_ctx); + if (scalar == NULL || !BN_copy(scalar, ip_x2_inv) || !zkp_poly_points_append(poly_p, R, scalar)) + goto err; + } + + for (i = 1; i < padded_n; i++) { + lg_i = zkp_floor_log2(i); + + vec_s[i] = BN_CTX_get(bn_ctx); + if (vec_s[i] == NULL) + goto err; + + if (!BN_mod_mul(vec_s[i], vec_s[i - (1 << lg_i)], + vec_ip_x2[lg_n - 1 - lg_i], order, bn_ctx)) + goto err; + } + + for (i = 0; i < padded_n; i++) { + g_scalar = BN_CTX_get(bn_ctx); + h_scalar = BN_CTX_get(bn_ctx); + if (h_scalar == NULL) + goto err; + + if (!BN_mod_mul(s_a, ip_proof->a, vec_s[i], order, bn_ctx) + || !BN_mod_mul(s_b, ip_proof->b, vec_s[padded_n - i - 1], order, bn_ctx)) + goto err; + + BN_set_negative(s_a, 1); + BN_set_negative(s_b, 1); + + if (i < ctx->vars_num) { + if (!BN_mod_mul(wR_pow_y_inv, wR[i], pow_y_inv, order, bn_ctx) + || !BN_mod_mul(tmp, wR_pow_y_inv, wL[i], order, bn_ctx) + || !BN_mod_add(delta, delta, tmp, order, bn_ctx)) + goto err; + + if (!BN_mod_mul(g_scalar, x, wR_pow_y_inv, order, bn_ctx) + || !BN_mod_add(g_scalar, g_scalar, s_a, order, bn_ctx)) + goto err; + + if (!BN_mod_mul(h_scalar, x, wL[i], order, bn_ctx) + || !BN_mod_add(h_scalar, h_scalar, wO[i], order, bn_ctx) + || !BN_mod_add(h_scalar, h_scalar, s_b, order, bn_ctx) + || !BN_mod_mul(h_scalar, h_scalar, pow_y_inv, order, bn_ctx) + || !BN_mod_sub(h_scalar, h_scalar, bn1, order, bn_ctx)) + goto err; + } else { + if (!BN_mod_mul(g_scalar, u, s_a, order, bn_ctx)) + goto err; + + if (!BN_mod_mul(h_scalar, pow_y_inv, s_b, order, bn_ctx) + || !BN_mod_sub(h_scalar, h_scalar, bn1, order, bn_ctx) + || !BN_mod_mul(h_scalar, h_scalar, u, order, bn_ctx)) + goto err; + } + + G = sk_EC_POINT_value(pp->sk_G, i); + H = sk_EC_POINT_value(pp->sk_H, i); + if (G == NULL || H == NULL) + goto err; + + if (!zkp_poly_points_append(poly_p, G, g_scalar) + || !zkp_poly_points_append(poly_p, H, h_scalar)) + goto err; + + if (!BN_mod_mul(pow_y_inv, pow_y_inv, y_inv, order, bn_ctx)) + goto err; + } + + for (i = 0; i < v_n; i++) { + var = sk_BP_VARIABLE_value(witness->sk_V, i); + if (var == NULL || var->point == NULL) + goto err; + + v_scalar = BN_CTX_get(bn_ctx); + if (v_scalar == NULL) + goto err; + + if (!BN_mod_mul(v_scalar, wV[i], rx2, order, bn_ctx)) + goto err; + + if (!zkp_poly_points_append(poly_p, var->point, v_scalar)) + goto err; + } + + b_scalar = BN_CTX_get(bn_ctx); + h_scalar = BN_CTX_get(bn_ctx); + if (h_scalar == NULL) + goto err; + + if (!BN_mod_add(wc, wc, delta, order, bn_ctx) + || !BN_mod_mul(wc, wc, x2, order, bn_ctx) + || !BN_mod_sub(wc, wc, proof->tx, order, bn_ctx) + || !BN_mod_mul(wc, wc, r, order, bn_ctx) + || !BN_mod_mul(b_scalar, ip_proof->a, ip_proof->b, order, bn_ctx) + || !BN_mod_sub(b_scalar, proof->tx, b_scalar, order, bn_ctx) + || !BN_mod_mul(b_scalar, b_scalar, w, order, bn_ctx) + || !BN_mod_add(b_scalar, b_scalar, wc, order, bn_ctx) + || !BN_mod_mul(h_scalar, r, proof->taux, order, bn_ctx) + || !BN_mod_add(h_scalar, h_scalar, proof->mu, order, bn_ctx)) + goto err; + + BN_set_negative(h_scalar, 1); + + if (!zkp_poly_points_append(poly_p, pp->H, h_scalar) + || !zkp_poly_points_mul(poly_p, P, b_scalar, group, bn_ctx)) + goto err; + + ret = EC_POINT_is_at_infinity(group, P); + +err: + ZKP_TRANSCRIPT_reset(transcript); + + OPENSSL_free(wV); + OPENSSL_free(wO); + OPENSSL_free(wR); + OPENSSL_free(wL); + + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + + OPENSSL_free(vec_s); + OPENSSL_free(vec_ip_x2); + + EC_POINT_free(P); + + zkp_poly_points_free(poly_p); + + return ret; +} diff --git a/openssl/src/crypto/zkp/bulletproofs/r1cs.h b/openssl/src/crypto/zkp/bulletproofs/r1cs.h new file mode 100644 index 000000000..d3534cf35 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/r1cs.h @@ -0,0 +1,110 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_BULLET_PROOF_R1CS_LOCAL_H +# define HEADER_BULLET_PROOF_R1CS_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif +# include +# include +# include +# include +# include +# include "internal/refcount.h" +# include "bulletproofs.h" +# include "inner_product.h" + +STACK_OF(BP_R1CS_VARIABLE); +STACK_OF(BP_R1CS_LINEAR_COMBINATION_ITEM); +STACK_OF(BP_R1CS_LINEAR_COMBINATION); + +typedef enum bp_r1cs_variable_type { + BP_R1CS_VARIABLE_COMMITTED, + BP_R1CS_VARIABLE_MULTIPLIER_LEFT, + BP_R1CS_VARIABLE_MULTIPLIER_RIGHT, + BP_R1CS_VARIABLE_MULTIPLIER_OUTPUT, + BP_R1CS_VARIABLE_ONE, +} BP_R1CS_VARIABLE_TYPE; + +typedef enum bp_r1cs_lc_type { + BP_R1CS_LC_TYPE_UNKOWN, + BP_R1CS_LC_TYPE_PROVE, + BP_R1CS_LC_TYPE_VERIFY, +} BP_R1CS_LC_TYPE; + +struct bp_r1cs_variable_st { + BP_R1CS_VARIABLE_TYPE type; + uint64_t value; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +struct bp_r1cs_linear_combination_item_st { + BP_R1CS_VARIABLE *variable; + BIGNUM *scalar; +}; + +struct bp_r1cs_linear_combination_st { + BP_R1CS_LC_TYPE type; + STACK_OF(BP_R1CS_LINEAR_COMBINATION_ITEM) *items; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +struct bp_r1cs_ctx_st { + ZKP_TRANSCRIPT *transcript; + BP_PUB_PARAM *pp; + BP_WITNESS *witness; + STACK_OF(BP_R1CS_LINEAR_COMBINATION) *constraints; + STACK_OF(BIGNUM) *aL; + STACK_OF(BIGNUM) *aR; + STACK_OF(BIGNUM) *aO; + int vars_num; +}; + +struct bp_r1cs_proof_st { + EC_POINT *AI1; + EC_POINT *AO1; + EC_POINT *S1; + EC_POINT *AI2; + EC_POINT *AO2; + EC_POINT *S2; + EC_POINT *T1; + EC_POINT *T3; + EC_POINT *T4; + EC_POINT *T5; + EC_POINT *T6; + BIGNUM *taux; + BIGNUM *mu; + BIGNUM *tx; + bp_inner_product_proof_t *ip_proof; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +BP_R1CS_VARIABLE *BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_TYPE type, uint64_t value); +BP_R1CS_VARIABLE *BP_R1CS_VARIABLE_dup(const BP_R1CS_VARIABLE *var); +void BP_R1CS_VARIABLE_free(BP_R1CS_VARIABLE *var); +BP_R1CS_LC_ITEM *BP_R1CS_LC_ITEM_new(BP_R1CS_VARIABLE *var, const BIGNUM *scalar); +BP_R1CS_LC_ITEM *BP_R1CS_LC_ITEM_dup(BP_R1CS_LC_ITEM *item); +void BP_R1CS_LC_ITEM_free(BP_R1CS_LC_ITEM *item); + +BP_R1CS_LINEAR_COMBINATION *BP_R1CS_LINEAR_COMBINATION_new_from_param(BP_R1CS_VARIABLE *var, + const BIGNUM *scalar); + +# ifdef __cplusplus +} +# endif + +#endif + diff --git a/openssl/src/crypto/zkp/bulletproofs/r1cs_constraint_expression.c b/openssl/src/crypto/zkp/bulletproofs/r1cs_constraint_expression.c new file mode 100644 index 000000000..0da23eff0 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/r1cs_constraint_expression.c @@ -0,0 +1,341 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include "crypto/ctype.h" +#include "internal/cryptlib.h" +#include +#include "r1cs.h" + +typedef struct bp_r1cs_expression_st { + char *expression; + int len; + int pos; + int is_prove; + int var_found; +} bp_r1cs_expression_t; + +DEFINE_STACK_OF(BP_R1CS_LINEAR_COMBINATION_ITEM) + +static bp_r1cs_expression_t *bp_r1cs_expression_new(const char *exp_str, int is_prove); +static void bp_r1cs_expression_free(bp_r1cs_expression_t *e); +static int bp_r1cs_expression_evaluate_expression(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx); +static int bp_r1cs_expression_evaluate_term(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx); +static int bp_r1cs_expression_evaluate_factor(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx); +static int bp_r1cs_expression_evaluate_number(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx); +static int bp_r1cs_expression_evaluate_variable(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx); +static void bp_r1cs_expression_skip_whitespace(bp_r1cs_expression_t *e); + +static bp_r1cs_expression_t *bp_r1cs_expression_new(const char *exp_str, int is_prove) +{ + bp_r1cs_expression_t *e; + + if (exp_str == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + e = OPENSSL_malloc(sizeof(*e)); + if (e == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + e->expression = OPENSSL_strdup(exp_str); + if (e->expression == NULL) { + OPENSSL_free(e); + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + e->len = strlen(e->expression); + e->pos = 0; + e->var_found = 0; + e->is_prove = is_prove; + + return e; +} + +static void bp_r1cs_expression_free(bp_r1cs_expression_t *e) +{ + if (e == NULL) + return; + + OPENSSL_free(e->expression); + OPENSSL_free(e); +} + +static int bp_r1cs_expression_evaluate_expression(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx) +{ + int ret = 0; + BP_R1CS_LINEAR_COMBINATION *right = NULL; + + if (e == NULL) + return 0; + + if (!bp_r1cs_expression_evaluate_term(e, lc, ctx)) + return 0; + + right = BP_R1CS_LINEAR_COMBINATION_new(); + if (right == NULL) + return 0; + + while (e->expression[e->pos] == '+' || e->expression[e->pos] == '-') { + char operator = e->expression[e->pos++]; + bp_r1cs_expression_skip_whitespace(e); + + if (!bp_r1cs_expression_evaluate_term(e, right, ctx)) + goto err; + + if (operator == '+') { + if (!BP_R1CS_LINEAR_COMBINATION_add(lc, right)) + goto err; + } else { + if (!BP_R1CS_LINEAR_COMBINATION_sub(lc, right)) + goto err; + } + + if (!BP_R1CS_LINEAR_COMBINATION_clean(right)) + goto err; + } + + bp_r1cs_expression_skip_whitespace(e); + ret = 1; + +err: + BP_R1CS_LINEAR_COMBINATION_free(right); + return ret; +} + +static int bp_r1cs_expression_evaluate_term(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx) +{ + int ret = 0; + BP_R1CS_LINEAR_COMBINATION *right = NULL; + + if (e == NULL) + return 0; + + if (!bp_r1cs_expression_evaluate_factor(e, lc, ctx)) + return 0; + + right = BP_R1CS_LINEAR_COMBINATION_new(); + if (right == NULL) + return 0; + + while (e->expression[e->pos] == '*') { + e->pos++; + bp_r1cs_expression_skip_whitespace(e); + + if (!bp_r1cs_expression_evaluate_factor(e, right, ctx)) + goto err; + + if (!BP_R1CS_LINEAR_COMBINATION_mul(lc, right, ctx)) + goto err; + + if (!BP_R1CS_LINEAR_COMBINATION_clean(right)) + goto err; + } + + ret = 1; + +err: + BP_R1CS_LINEAR_COMBINATION_free(right); + return ret; +} + +static int bp_r1cs_expression_evaluate_factor(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx) +{ + if (e == NULL) + return 0; + + if (e->expression[e->pos] == '(') { + e->pos++; + bp_r1cs_expression_skip_whitespace(e); + if (!bp_r1cs_expression_evaluate_expression(e, lc, ctx)) + return 0; + + if (e->expression[e->pos] != ')') { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_FORMAT_ERROR); + return 0; + } + + e->pos++; + return 1; + } else if (ossl_isalpha(e->expression[e->pos])) { + return bp_r1cs_expression_evaluate_variable(e, lc, ctx); + } else if (ossl_isdigit(e->expression[e->pos])) { + return bp_r1cs_expression_evaluate_number(e, lc, ctx); + } + + return 0; +} + +static int bp_r1cs_expression_evaluate_number(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx) +{ + int number = 0; + BIGNUM *bn = NULL; + BP_R1CS_LC_ITEM *item = NULL; + + if (e == NULL) + return 0; + + if (!ossl_isdigit(e->expression[e->pos])) + return 0; + + while (ossl_isdigit(e->expression[e->pos])) { + number = 10 * number + (e->expression[e->pos] - '0'); + e->pos++; + } + + bp_r1cs_expression_skip_whitespace(e); + + bn = BN_new(); + if (bn == NULL) + return 0; + + BN_set_word(bn, number); + + if ((item = BP_R1CS_LC_ITEM_new(NULL, bn)) == NULL + || sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(lc->items, item) <= 0) + goto err; + + lc->type = e->is_prove ? BP_R1CS_LC_TYPE_PROVE : BP_R1CS_LC_TYPE_VERIFY; + + BN_free(bn); + return 1; +err: + BN_free(bn); + BP_R1CS_LC_ITEM_free(item); + return 0; +} + +static int bp_r1cs_expression_evaluate_variable(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx) +{ + int i = 0; + char var[BP_VARIABLE_NAME_MAX_LEN + 1]; + BP_R1CS_VARIABLE *r1cs_var = NULL; + BP_R1CS_LC_ITEM *item = NULL; + + if (e == NULL) + return 0; + + if (!ossl_isalpha(e->expression[e->pos])) + return 0; + + memset(var, 0, sizeof(var)); + + while (ossl_isalpha(e->expression[e->pos]) || ossl_isdigit(e->expression[e->pos]) + || e->expression[e->pos] == '_') { + if (i > BP_VARIABLE_NAME_MAX_LEN) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_TOO_LONG); + return 0; + } + + var[i++] = e->expression[e->pos++]; + } + + bp_r1cs_expression_skip_whitespace(e); + + i = BP_WITNESS_get_variable_index(ctx->witness, var); + if (i < 0) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_NOT_FOUND); + return 0; + } + + if ((r1cs_var = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_COMMITTED, i)) == NULL) + goto err; + + if ((item = BP_R1CS_LC_ITEM_new(r1cs_var, NULL)) == NULL + || sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(lc->items, item) <= 0) + goto err; + + lc->type = e->is_prove ? BP_R1CS_LC_TYPE_PROVE : BP_R1CS_LC_TYPE_VERIFY; + e->var_found = 1; + + BP_R1CS_VARIABLE_free(r1cs_var); + return 1; +err: + BP_R1CS_VARIABLE_free(r1cs_var); + BP_R1CS_LC_ITEM_free(item); + return 0; +} + +static void bp_r1cs_expression_skip_whitespace(bp_r1cs_expression_t *e) +{ + if (e == NULL) + return; + + while (ossl_isspace(e->expression[e->pos])) + e->pos++; +} + +static int bp_r1cs_expression_process(bp_r1cs_expression_t *e, + BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx) +{ + if (!bp_r1cs_expression_evaluate_expression(e, lc, ctx)) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_PROCESS_ERROR); + return 0; + } + + if (e->pos != e->len) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_FORMAT_ERROR); + return 0; + } + + if (!e->var_found) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_NO_VAR); + return 0; + } + + return 1; +} + +int BP_R1CS_constraint_expression(BP_R1CS_CTX *ctx, const char *constraint, int is_prove) +{ + int ret = 0; + bp_r1cs_expression_t *e = NULL; + BP_R1CS_LINEAR_COMBINATION *lc = NULL; + + e = bp_r1cs_expression_new(constraint, is_prove); + lc = BP_R1CS_LINEAR_COMBINATION_new(); + if (e == NULL || lc == NULL) + goto err; + + if (!bp_r1cs_expression_process(e, lc, ctx)) + goto err; + + ret = BP_R1CS_LINEAR_COMBINATION_constrain(lc, ctx); + +err: + BP_R1CS_LINEAR_COMBINATION_free(lc); + bp_r1cs_expression_free(e); + return ret; +} diff --git a/openssl/src/crypto/zkp/bulletproofs/r1cs_linear_combination.c b/openssl/src/crypto/zkp/bulletproofs/r1cs_linear_combination.c new file mode 100644 index 000000000..f43490293 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/r1cs_linear_combination.c @@ -0,0 +1,767 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include "r1cs.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(BP_R1CS_VARIABLE) +DEFINE_STACK_OF(BP_R1CS_LINEAR_COMBINATION_ITEM) +DEFINE_STACK_OF(BP_R1CS_LINEAR_COMBINATION) + +BP_R1CS_VARIABLE *BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_TYPE type, uint64_t value) +{ + BP_R1CS_VARIABLE *var = NULL; + + var = OPENSSL_zalloc(sizeof(*var)); + if (var == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + var->references = 1; + if ((var->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + var->type = type; + var->value = value; + + return var; +err: + OPENSSL_free(var); + return NULL; +} + +BP_R1CS_VARIABLE *BP_R1CS_VARIABLE_dup(const BP_R1CS_VARIABLE *var) +{ + BP_R1CS_VARIABLE *ret; + + if (var == NULL) + return NULL; + + if ((ret = BP_R1CS_VARIABLE_new(var->type, var->value)) == NULL) + return NULL; + + return ret; +} + +void BP_R1CS_VARIABLE_free(BP_R1CS_VARIABLE *var) +{ + int ref; + + if (var == NULL) + return; + + CRYPTO_DOWN_REF(&var->references, &ref, var->lock); + REF_PRINT_COUNT("BP_R1CS_VARIABLE", var); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + CRYPTO_THREAD_lock_free(var->lock); + OPENSSL_clear_free((void *)var, sizeof(*var)); +} + +BP_R1CS_LC_ITEM *BP_R1CS_LC_ITEM_new(BP_R1CS_VARIABLE *var, const BIGNUM *scalar) +{ + int ref; + BP_R1CS_LC_ITEM *item = NULL; + BP_R1CS_VARIABLE *v = NULL; + BIGNUM *s = NULL; + + item = OPENSSL_zalloc(sizeof(*item)); + if (item == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (var == NULL) { + if (!(v = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_ONE, 1))) { + goto err; + } + var = v; + } else { + if (CRYPTO_UP_REF(&var->references, &ref, var->lock) <= 0) + goto err; + } + + if (scalar == NULL) { + if ((s = BN_new()) == NULL) + goto err; + + BN_one(s); + } else { + if ((s = BN_dup(scalar)) == NULL) + goto err; + } + + item->variable = var; + item->scalar = s; + + return item; +err: + BN_free(s); + BP_R1CS_VARIABLE_free(v); + BP_R1CS_LC_ITEM_free(item); + return NULL; +} + +BP_R1CS_LC_ITEM *BP_R1CS_LC_ITEM_dup(BP_R1CS_LC_ITEM *item) +{ + if (item == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + return BP_R1CS_LC_ITEM_new(item->variable, item->scalar); +} + +void BP_R1CS_LC_ITEM_free(BP_R1CS_LC_ITEM *item) +{ + if (item == NULL) + return; + + BP_R1CS_VARIABLE_free(item->variable); + BN_free(item->scalar); + OPENSSL_clear_free((void *)item, sizeof(*item)); +} + +BP_R1CS_LINEAR_COMBINATION *BP_R1CS_LINEAR_COMBINATION_new(void) +{ + BP_R1CS_LINEAR_COMBINATION *lc = NULL; + + lc = OPENSSL_zalloc(sizeof(*lc)); + if (lc == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if ((lc->items = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_new_null()) == NULL) + goto err; + + lc->references = 1; + if ((lc->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + lc->type = BP_R1CS_LC_TYPE_UNKOWN; + + return lc; +err: + BP_R1CS_LINEAR_COMBINATION_free(lc); + return NULL; +} + +BP_R1CS_LINEAR_COMBINATION *BP_R1CS_LINEAR_COMBINATION_new_from_param(BP_R1CS_VARIABLE *var, + const BIGNUM *scalar) +{ + BP_R1CS_LINEAR_COMBINATION *lc = NULL; + BP_R1CS_LC_ITEM *item = NULL; + + lc = BP_R1CS_LINEAR_COMBINATION_new(); + if (lc == NULL) { + return NULL; + } + + if ((item = BP_R1CS_LC_ITEM_new(var, scalar)) == NULL + || sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(lc->items, item) <= 0) + goto err; + + return lc; +err: + BP_R1CS_LINEAR_COMBINATION_free(lc); + return NULL; +} + +BP_R1CS_LINEAR_COMBINATION *BP_R1CS_LINEAR_COMBINATION_dup(const BP_R1CS_LINEAR_COMBINATION *lc) +{ + int i, num; + BP_R1CS_LINEAR_COMBINATION *ret = NULL; + BP_R1CS_LC_ITEM *item, *item_dup = NULL; + + if (lc == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ret = BP_R1CS_LINEAR_COMBINATION_new(); + if (ret == NULL) { + return NULL; + } + + num = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_num(lc->items); + for (i = 0; i < num; i++) { + item = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_value(lc->items, i); + if (item == NULL) + goto err; + + item_dup = BP_R1CS_LC_ITEM_dup(item); + if (item_dup == NULL) + goto err; + + sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(ret->items, item_dup); + } + + ret->type = lc->type; + + return ret; + +err: + BP_R1CS_LINEAR_COMBINATION_free(ret); + return NULL; +} + +void BP_R1CS_LINEAR_COMBINATION_free(BP_R1CS_LINEAR_COMBINATION *lc) +{ + int ref; + + if (lc == NULL) + return; + + CRYPTO_DOWN_REF(&lc->references, &ref, lc->lock); + REF_PRINT_COUNT("BP_R1CS_LINEAR_COMBINATION", lc); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + CRYPTO_THREAD_lock_free(lc->lock); + sk_BP_R1CS_LINEAR_COMBINATION_ITEM_pop_free(lc->items, BP_R1CS_LC_ITEM_free); + OPENSSL_clear_free((void *)lc, sizeof(*lc)); +} + +int BP_R1CS_LINEAR_COMBINATION_clean(BP_R1CS_LINEAR_COMBINATION *lc) +{ + if (lc == NULL) + return 0; + + sk_BP_R1CS_LINEAR_COMBINATION_ITEM_pop_free(lc->items, BP_R1CS_LC_ITEM_free); + if ((lc->items = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_new_null()) == NULL) + return 0; + + lc->type = BP_R1CS_LC_TYPE_UNKOWN; + return 1; +} + +static int BP_R1CS_LINEAR_COMBINATION_eval(BP_R1CS_CTX *ctx, + const BP_R1CS_LINEAR_COMBINATION *lc, + BIGNUM *r, BN_CTX *bn_ctx) +{ + int i, num, ret = 0; + BN_CTX *bctx = NULL; + BIGNUM *a, *product, *sum, *one; + BP_WITNESS *witness; + BP_R1CS_VARIABLE *var; + BP_R1CS_LINEAR_COMBINATION_ITEM *item; + + if (ctx == NULL || ctx->witness == NULL || lc == NULL || r == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + witness = ctx->witness; + + if (bn_ctx == NULL) { + bctx = bn_ctx = BN_CTX_new(); + if (bctx == NULL) + return 0; + } + + BN_CTX_start(bn_ctx); + + sum = BN_CTX_get(bn_ctx); + product = BN_CTX_get(bn_ctx); + one = BN_CTX_get(bn_ctx); + if (one == NULL) + goto err; + + BN_zero(sum); + BN_one(one); + + num = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_num(lc->items); + for (i = 0; i < num; i++) { + item = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_value(lc->items, i); + if (item == NULL) + goto err; + + var = item->variable; + + switch (var->type) { + case BP_R1CS_VARIABLE_COMMITTED: + a = sk_BIGNUM_value(witness->sk_v, var->value); + break; + case BP_R1CS_VARIABLE_MULTIPLIER_LEFT: + a = sk_BIGNUM_value(ctx->aL, var->value); + break; + case BP_R1CS_VARIABLE_MULTIPLIER_RIGHT: + a = sk_BIGNUM_value(ctx->aR, var->value); + break; + case BP_R1CS_VARIABLE_MULTIPLIER_OUTPUT: + a = sk_BIGNUM_value(ctx->aO, var->value); + break; + case BP_R1CS_VARIABLE_ONE: + default: + a = one; + } + + if (!BN_mul(product, a, item->scalar, bn_ctx) + || !BN_add(sum, sum, product)) + goto err; + } + + BN_copy(r, sum); + ret = 1; + +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bctx); + return ret; +} + +/* + * left = lc(l) + * right = lc(r) + * output = left * right + */ +int BP_R1CS_LINEAR_COMBINATION_raw_mul(BP_R1CS_LINEAR_COMBINATION **output, + BP_R1CS_LINEAR_COMBINATION **left, + BP_R1CS_LINEAR_COMBINATION **right, + const BIGNUM *l, const BIGNUM *r, + BP_R1CS_CTX *ctx) +{ + int ln, rn, on, ret = 0; + BN_CTX *bn_ctx = NULL; + BIGNUM *lb = NULL, *rb = NULL, *ob = NULL; + BP_R1CS_VARIABLE *lv = NULL, *rv = NULL, *ov = NULL; + BP_R1CS_LINEAR_COMBINATION *llc = NULL, *rlc = NULL, *olc = NULL; + + if (output == NULL || ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (l != NULL && r != NULL) { + ln = sk_BIGNUM_num(ctx->aL); + rn = sk_BIGNUM_num(ctx->aR); + on = sk_BIGNUM_num(ctx->aO); + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + + lb = BN_dup(l); + rb = BN_dup(r); + ob = BN_new(); + if (lb == NULL || rb == NULL || ob == NULL) + goto err; + + if (!BN_mul(ob, lb, rb, bn_ctx)) + goto err; + + if (sk_BIGNUM_push(ctx->aL, lb) <= 0) + goto err; + lb = NULL; + + if (sk_BIGNUM_push(ctx->aR, rb) <= 0) + goto err; + rb = NULL; + + if (sk_BIGNUM_push(ctx->aO, ob) <= 0) + goto err; + ob = NULL; + } else { + ln = rn = on = ctx->vars_num; + ctx->vars_num += 1; + } + + ov = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_MULTIPLIER_OUTPUT, on); + if (ov == NULL) + goto err; + + olc = BP_R1CS_LINEAR_COMBINATION_new_from_param(ov, NULL); + if (olc == NULL) { + goto err; + } + + *output = olc; + + if (left != NULL) { + lv = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_MULTIPLIER_LEFT, ln); + if (lv == NULL) + goto err; + + llc = BP_R1CS_LINEAR_COMBINATION_new_from_param(lv, NULL); + if (llc == NULL) { + goto err; + } + + *left = llc; + } + + if (right != NULL) { + rv = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_MULTIPLIER_RIGHT, rn); + if (rv == NULL) + goto err; + + rlc = BP_R1CS_LINEAR_COMBINATION_new_from_param(rv, NULL); + if (rlc == NULL) { + goto err; + } + + *right = rlc; + } + + BP_R1CS_VARIABLE_free(lv); + BP_R1CS_VARIABLE_free(rv); + BP_R1CS_VARIABLE_free(ov); + BN_CTX_free(bn_ctx); + return 1; +err: + if (output == NULL) + output = NULL; + if (left == NULL) + left = NULL; + if (right == NULL) + right = NULL; + + BP_R1CS_LINEAR_COMBINATION_free(llc); + BP_R1CS_LINEAR_COMBINATION_free(rlc); + BP_R1CS_LINEAR_COMBINATION_free(olc); + BP_R1CS_VARIABLE_free(lv); + BP_R1CS_VARIABLE_free(rv); + BP_R1CS_VARIABLE_free(ov); + BN_free(lb); + BN_free(rb); + BN_free(ob); + BN_CTX_free(bn_ctx); + return ret; +} + +/* lc *= other */ +int BP_R1CS_LINEAR_COMBINATION_mul(BP_R1CS_LINEAR_COMBINATION *lc, + const BP_R1CS_LINEAR_COMBINATION *other, + BP_R1CS_CTX *ctx) +{ + int ln, rn, on, ret = 0; + BN_CTX *bn_ctx = NULL; + BIGNUM *l = NULL, *r = NULL, *o = NULL, *bn_1; + BP_R1CS_VARIABLE *lv = NULL, *rv = NULL, *ov = NULL; + BP_R1CS_LINEAR_COMBINATION_ITEM *li = NULL, *ri = NULL, *oi = NULL; + BP_R1CS_LINEAR_COMBINATION *llc = NULL, *rlc = NULL; + STACK_OF(BP_R1CS_LINEAR_COMBINATION_ITEM) *lc_items = NULL; + + if (lc == NULL || other == NULL || ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (lc->type != BP_R1CS_LC_TYPE_UNKOWN && lc->type != other->type) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return 0; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + return 0; + + BN_CTX_start(bn_ctx); + + bn_1 = BN_CTX_get(bn_ctx); + if (bn_1 == NULL) + goto err; + + BN_one(bn_1); + BN_set_negative(bn_1, 1); + + if (lc->type == BP_R1CS_LC_TYPE_PROVE) { + l = BN_new(); + r = BN_new(); + o = BN_new(); + if (l == NULL || r == NULL || o == NULL) + goto err; + + if (!BP_R1CS_LINEAR_COMBINATION_eval(ctx, lc, l, bn_ctx) + || !BP_R1CS_LINEAR_COMBINATION_eval(ctx, other, r, bn_ctx)) + goto err; + + if (!BN_mul(o, l, r, bn_ctx)) + goto err; + + ln = sk_BIGNUM_num(ctx->aL); + rn = sk_BIGNUM_num(ctx->aR); + on = sk_BIGNUM_num(ctx->aO); + + if (sk_BIGNUM_push(ctx->aL, l) <= 0) + goto err; + l = NULL; + + if (sk_BIGNUM_push(ctx->aR, r) <= 0) + goto err; + r = NULL; + + if (sk_BIGNUM_push(ctx->aO, o) <= 0) + goto err; + o = NULL; + } else { + ln = rn = on = ctx->vars_num; + ctx->vars_num += 1; + } + + llc = BP_R1CS_LINEAR_COMBINATION_dup(lc); + rlc = BP_R1CS_LINEAR_COMBINATION_dup(other); + if (llc == NULL || rlc == NULL) + goto err; + + lv = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_MULTIPLIER_LEFT, ln); + rv = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_MULTIPLIER_RIGHT, rn); + ov = BP_R1CS_VARIABLE_new(BP_R1CS_VARIABLE_MULTIPLIER_OUTPUT, on); + if (lv == NULL || rv == NULL || ov == NULL) + goto err; + + if ((li = BP_R1CS_LC_ITEM_new(lv, bn_1)) == NULL + || sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(llc->items, li) <= 0) + goto err; + li = NULL; + + if ((ri = BP_R1CS_LC_ITEM_new(rv, bn_1)) == NULL + || sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(rlc->items, ri) <= 0) + goto err; + ri = NULL; + + if (!BP_R1CS_LINEAR_COMBINATION_constrain(llc, ctx) + || !BP_R1CS_LINEAR_COMBINATION_constrain(rlc, ctx)) + goto err; + + if (!(lc_items = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_new_reserve(NULL, 1))) + goto err; + + if ((oi = BP_R1CS_LC_ITEM_new(ov, NULL)) == NULL + || sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(lc_items, oi) <= 0) + goto err; + + sk_BP_R1CS_LINEAR_COMBINATION_ITEM_pop_free(lc->items, BP_R1CS_LC_ITEM_free); + lc->items = lc_items; + lc_items = NULL; + oi = NULL; + llc = rlc = NULL; + + ret = 1; + +err: + sk_BP_R1CS_LINEAR_COMBINATION_ITEM_free(lc_items); + BP_R1CS_LINEAR_COMBINATION_free(llc); + BP_R1CS_LINEAR_COMBINATION_free(rlc); + BP_R1CS_LC_ITEM_free(li); + BP_R1CS_LC_ITEM_free(ri); + BP_R1CS_LC_ITEM_free(oi); + BP_R1CS_VARIABLE_free(lv); + BP_R1CS_VARIABLE_free(rv); + BP_R1CS_VARIABLE_free(ov); + BN_free(l); + BN_free(r); + BN_free(o); + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/* lc += other */ +int BP_R1CS_LINEAR_COMBINATION_add(BP_R1CS_LINEAR_COMBINATION *lc, + const BP_R1CS_LINEAR_COMBINATION *other) +{ + int i, num; + BP_R1CS_LINEAR_COMBINATION_ITEM *item = NULL, *p; + + if (lc == NULL || other == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + num = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_num(other->items); + for (i = 0; i < num; i++) { + p = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_value(other->items, i); + if (p == NULL) + goto err; + + if ((item = BP_R1CS_LC_ITEM_dup(p)) == NULL) + goto err; + + if (sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(lc->items, item) <= 0) + goto err; + } + + return 1; +err: + BP_R1CS_LC_ITEM_free(item); + return 0; +} + +/* lc -= other */ +int BP_R1CS_LINEAR_COMBINATION_sub(BP_R1CS_LINEAR_COMBINATION *lc, + const BP_R1CS_LINEAR_COMBINATION *other) +{ + int i, num; + BP_R1CS_LINEAR_COMBINATION_ITEM *item = NULL, *p; + + if (lc == NULL || other == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + num = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_num(other->items); + for (i = 0; i < num; i++) { + p = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_value(other->items, i); + if (p == NULL) + goto err; + + if ((item = BP_R1CS_LC_ITEM_dup(p)) == NULL) + goto err; + + BN_set_negative(item->scalar, 1); + + if (sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(lc->items, item) <= 0) + goto err; + } + + return 1; +err: + BP_R1CS_LC_ITEM_free(item); + return 0; +} + +/* lc = -lc */ +int BP_R1CS_LINEAR_COMBINATION_neg(BP_R1CS_LINEAR_COMBINATION *lc) +{ + int i, num, ret = 0; + BP_R1CS_LINEAR_COMBINATION_ITEM *item; + + if (lc == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + num = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_num(lc->items); + for (i = 0; i < num; i++) { + item = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_value(lc->items, i); + if (item == NULL || item->scalar == NULL) + goto err; + + BN_set_negative(item->scalar, 1); + } + + ret = 1; + +err: + return ret; +} + +/* lc = lc * value */ +int BP_R1CS_LINEAR_COMBINATION_mul_bn(BP_R1CS_LINEAR_COMBINATION *lc, + const BIGNUM *value) +{ + int i, num, ret = 0; + BN_CTX *bn_ctx = NULL; + BP_R1CS_LINEAR_COMBINATION_ITEM *item; + + if (lc == NULL || value == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!(bn_ctx = BN_CTX_new())) + goto err; + + num = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_num(lc->items); + for (i = 0; i < num; i++) { + item = sk_BP_R1CS_LINEAR_COMBINATION_ITEM_value(lc->items, i); + if (item == NULL || item->scalar == NULL) + goto err; + + if (!BN_mul(item->scalar, item->scalar, value, bn_ctx)) + goto err; + } + + ret = 1; + +err: + BN_CTX_free(bn_ctx); + return ret; +} + +/* lc = lc + value */ +int BP_R1CS_LINEAR_COMBINATION_add_bn(BP_R1CS_LINEAR_COMBINATION *lc, + const BIGNUM *value) +{ + BP_R1CS_LINEAR_COMBINATION_ITEM *item; + + if (lc == NULL || value == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if ((item = BP_R1CS_LC_ITEM_new(NULL, value)) == NULL) + return 0; + + if (sk_BP_R1CS_LINEAR_COMBINATION_ITEM_push(lc->items, item) <= 0) { + BP_R1CS_LC_ITEM_free(item); + return 0; + } + + return 1; +} + +/* lc = lc - value */ +int BP_R1CS_LINEAR_COMBINATION_sub_bn(BP_R1CS_LINEAR_COMBINATION *lc, + const BIGNUM *value) +{ + int ret = 0; + BIGNUM *scalar = NULL; + + if (lc == NULL || value == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!(scalar = BN_dup(value))) + return 0; + + BN_set_negative(scalar, 1); + + ret = BP_R1CS_LINEAR_COMBINATION_add_bn(lc, scalar); + + BN_free(scalar); + return ret; +} + +int BP_R1CS_LINEAR_COMBINATION_constrain(BP_R1CS_LINEAR_COMBINATION *lc, + BP_R1CS_CTX *ctx) +{ + int ref; + + if (ctx == NULL || lc == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (CRYPTO_UP_REF(&lc->references, &ref, lc->lock) <= 0) + return 0; + + if (sk_BP_R1CS_LINEAR_COMBINATION_push(ctx->constraints, lc) <= 0) + goto err; + + return 1; +err: + CRYPTO_DOWN_REF(&lc->references, &ref, lc->lock); + return 0; +} + diff --git a/openssl/src/crypto/zkp/bulletproofs/range_proof.c b/openssl/src/crypto/zkp/bulletproofs/range_proof.c new file mode 100644 index 000000000..d8237a9c0 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/range_proof.c @@ -0,0 +1,893 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "range_proof.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) +DEFINE_STACK_OF(BP_VARIABLE) + +static void bp_range_proof_cleanup(BP_RANGE_PROOF *proof); + +BP_RANGE_PROOF *bp_range_proof_alloc(const EC_GROUP *group) +{ + BP_RANGE_PROOF *proof = NULL; + + if (group == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!(proof->A = EC_POINT_new(group)) + || !(proof->S = EC_POINT_new(group)) + || !(proof->T1 = EC_POINT_new(group)) + || !(proof->T2 = EC_POINT_new(group)) + || !(proof->taux = BN_new()) + || !(proof->mu = BN_new()) + || !(proof->tx = BN_new())) + goto err; + + proof->references = 1; + if ((proof->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + return proof; +err: + BP_RANGE_PROOF_free(proof); + return NULL; +} + +/** Creates a new BP_RANGE_CTX object + * \param pp BP_PUB_PARAM object + * \param witness BP_WITNESS object + * \param transcript ZKP_TRANSCRIPT object + * \return newly created BP_RANGE_CTX object or NULL in case of an error + */ +BP_RANGE_CTX *BP_RANGE_CTX_new(BP_PUB_PARAM *pp, BP_WITNESS *witness, + ZKP_TRANSCRIPT *transcript) +{ + BP_RANGE_CTX *ctx = NULL; + + if (pp == NULL || transcript == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!BP_PUB_PARAM_up_ref(pp)) + goto err; + + ctx->pp = pp; + + if (!BP_WITNESS_up_ref(witness)) + goto err; + + ctx->witness = witness; + + ctx->transcript = transcript; + + return ctx; + +err: + BP_RANGE_CTX_free(ctx); + return NULL; +} + +/** Frees a BP_RANGE_CTX object + * \param ctx BP_RANGE_CTX object to be freed + */ +void BP_RANGE_CTX_free(BP_RANGE_CTX *ctx) +{ + if (ctx == NULL) + return; + + BP_PUB_PARAM_down_ref(ctx->pp); + BP_WITNESS_down_ref(ctx->witness); + OPENSSL_clear_free((void *)ctx, sizeof(*ctx)); +} + +/** Creates a new BP_RANGE_PROOF object + * \param pp BP_PUB_PARAM object + * \return newly created BP_RANGE_PROOF object or NULL in case of an error + */ +BP_RANGE_PROOF *BP_RANGE_PROOF_new(const BP_PUB_PARAM *pp) +{ + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + return bp_range_proof_alloc(pp->group); +} + +/** Frees a BP_RANGE_PROOF object + * \param proof BP_RANGE_PROOF object to be freed + */ +void BP_RANGE_PROOF_free(BP_RANGE_PROOF *proof) +{ + int ref; + + if (proof == NULL) + return; + + CRYPTO_DOWN_REF(&proof->references, &ref, proof->lock); + REF_PRINT_COUNT("BP_RANGE_PROOF", proof); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + EC_POINT_free(proof->A); + EC_POINT_free(proof->S); + EC_POINT_free(proof->T1); + EC_POINT_free(proof->T2); + BN_free(proof->taux); + BN_free(proof->mu); + BN_free(proof->tx); + bp_inner_product_proof_free(proof->ip_proof); + CRYPTO_THREAD_lock_free(proof->lock); + OPENSSL_free(proof); +} + +static void bp_range_proof_cleanup(BP_RANGE_PROOF *proof) +{ + if (proof == NULL) + return; + + bp_inner_product_proof_free(proof->ip_proof); + proof->ip_proof = NULL; +} + +/** Increases the internal reference count of a BP_RANGE_PROOF object. + * \param proof BP_RANGE_PROOF object + * \return 1 on success and 0 if an error occurred. + */ +int BP_RANGE_PROOF_up_ref(BP_RANGE_PROOF *proof) +{ + int ref; + + if (CRYPTO_UP_REF(&proof->references, &ref, proof->lock) <= 0) + return 0; + + REF_PRINT_COUNT("BP_RANGE_PROOF", proof); + REF_ASSERT_ISNT(ref < 2); + return ((ref > 1) ? 1 : 0); +} + +/** Decreases the internal reference count of a BP_RANGE_PROOF object. + * \param proof BP_RANGE_PROOF object + * \return 1 on success and 0 if an error occurred. + */ +int BP_RANGE_PROOF_down_ref(BP_RANGE_PROOF *proof) +{ + int ref; + + if (CRYPTO_DOWN_REF(&proof->references, &ref, proof->lock) <= 0) + return 0; + + REF_PRINT_COUNT("BP_RANGE_PROOF", proof); + REF_ASSERT_ISNT(ref > 0); + return ((ref > 0) ? 1 : 0); +} + +/** Prove computes the ZK rangeproof. + * \param ctx BP_RANGE_CTX object + * \param proof BP_RANGE_PROOF object + * \return 1 on success and 0 if an error occurred. + */ +int BP_RANGE_PROOF_prove(BP_RANGE_CTX *ctx, BP_RANGE_PROOF *proof) +{ + int i, j, m = 0, n, ret = 0; + int bits, poly_num, witness_n, witness_r_n, witness_v_n, witness_padded_n; + int *aL = NULL, *aR = NULL; + ZKP_TRANSCRIPT *transcript; + BP_PUB_PARAM *pp; + BP_WITNESS *witness; + BIGNUM *witness_r, *witness_v; + BIGNUM *alpha, *rho, *tau1, *tau2, *bn0, *bn1, *bn2, *bn_1, *tmp; + BIGNUM *x, *y, *y_inv, *pow_y_inv, *z, *z2, *pow_zn, **pow_y = NULL; + BIGNUM *pow_2, *dv, *t, *t1, *t2, *r0, *r1, **sL = NULL, **sR = NULL; + BIGNUM **ll0 = NULL, **rr1 = NULL, **rr2 = NULL; + BIGNUM *g_scalar, *h_scalar, *l, *r; + STACK_OF(BIGNUM) *sk_G_scalars = NULL, *sk_H_scalars = NULL; + STACK_OF(BIGNUM) *sk_l = NULL, *sk_r = NULL; + STACK_OF(EC_POINT) *sk_G = NULL, *sk_H = NULL; + EC_POINT *P = NULL, *T = NULL, *U = NULL, *G, *H; + zkp_poly_points_t *poly_a = NULL, *poly_s = NULL, *poly_p = NULL; + const BIGNUM *order; + EC_GROUP *group; + BN_CTX *bn_ctx = NULL; + bp_inner_product_ctx_t *ip_ctx = NULL; + bp_inner_product_pub_param_t *ip_pp = NULL; + bp_inner_product_witness_t *ip_witness = NULL; + + if (ctx == NULL || ctx->pp == NULL || ctx->witness == NULL || proof == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + pp = ctx->pp; + witness = ctx->witness; + transcript = ctx->transcript; + group = pp->group; + order = EC_GROUP_get0_order(group); + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + BN_CTX_start(bn_ctx); + alpha = BN_CTX_get(bn_ctx); + rho = BN_CTX_get(bn_ctx); + tau1 = BN_CTX_get(bn_ctx); + tau2 = BN_CTX_get(bn_ctx); + bn0 = BN_CTX_get(bn_ctx); + bn1 = BN_CTX_get(bn_ctx); + bn2 = BN_CTX_get(bn_ctx); + bn_1 = BN_CTX_get(bn_ctx); + x = BN_CTX_get(bn_ctx); + y = BN_CTX_get(bn_ctx); + y_inv = BN_CTX_get(bn_ctx); + pow_y_inv = BN_CTX_get(bn_ctx); + z = BN_CTX_get(bn_ctx); + z2 = BN_CTX_get(bn_ctx); + pow_zn = BN_CTX_get(bn_ctx); + pow_2 = BN_CTX_get(bn_ctx); + t1 = BN_CTX_get(bn_ctx); + t2 = BN_CTX_get(bn_ctx); + t = BN_CTX_get(bn_ctx); + r0 = BN_CTX_get(bn_ctx); + r1 = BN_CTX_get(bn_ctx); + dv = BN_CTX_get(bn_ctx); + if (dv == NULL) + goto err; + + BN_zero(t1); + BN_zero(t2); + BN_zero(bn0); + BN_one(bn1); + BN_one(bn_1); + BN_set_negative(bn_1, 1); + BN_set_word(bn2, 2); + BN_one(pow_y_inv); + + witness_n = sk_BP_VARIABLE_num(witness->sk_V); + witness_padded_n = zkp_next_power_of_two(witness_n); + if (witness_padded_n > ctx->pp->party_capacity) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_EXCEEDS_PARTY_CAPACITY); + goto err; + } + + for (i = witness_n; i < witness_padded_n; i++) { + if (!BP_WITNESS_commit(witness, NULL, bn0)) + goto err; + } + + witness_r_n = sk_BIGNUM_num(witness->sk_r); + witness_v_n = sk_BIGNUM_num(witness->sk_v); + witness_n = sk_BP_VARIABLE_num(witness->sk_V); + witness_padded_n = zkp_next_power_of_two(witness_n); + + if (witness_r_n != witness_v_n || witness_v_n != witness_n) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_WITNESS_INVALID); + goto err; + } + + n = pp->gens_capacity * witness_padded_n; + poly_num = n * 2 + 1; + bits = pp->gens_capacity; + + if (!zkp_is_power_of_two(n)) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_RANGE_LEN_MUST_BE_POWER_OF_TWO); + goto err; + } + + if (proof->ip_proof != NULL) + bp_range_proof_cleanup(proof); + + if (!(P = EC_POINT_new(group)) + || !(T = EC_POINT_new(group)) + || !(U = EC_POINT_new(group))) + goto err; + + if (!(aL = OPENSSL_zalloc(sizeof(*aL) * n)) + || !(aR = OPENSSL_zalloc(sizeof(*aL) * n)) + || !(sL = OPENSSL_zalloc(sizeof(*sL) * n)) + || !(sR = OPENSSL_zalloc(sizeof(*sR) * n)) + || !(sk_G = sk_EC_POINT_new_reserve(NULL, n)) + || !(sk_H = sk_EC_POINT_new_reserve(NULL, n)) + || !(sk_G_scalars = sk_BIGNUM_new_reserve(NULL, n)) + || !(sk_H_scalars = sk_BIGNUM_new_reserve(NULL, n)) + || !(sk_l = sk_BIGNUM_new_reserve(NULL, n)) + || !(sk_r = sk_BIGNUM_new_reserve(NULL, n)) + || !(pow_y = OPENSSL_zalloc(sizeof(*pow_y) * n)) + || !(ll0 = OPENSSL_zalloc(sizeof(*ll0) * n)) + || !(rr1 = OPENSSL_zalloc(sizeof(*rr1) * n)) + || !(rr2 = OPENSSL_zalloc(sizeof(*rr2) * n))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!(poly_a = zkp_poly_points_new(poly_num)) + || !(poly_s = zkp_poly_points_new(poly_num)) + || !(poly_p = zkp_poly_points_new(poly_num))) + goto err; + + if (!zkp_rand_range(alpha, order) + || !zkp_rand_range(rho, order) + || !zkp_rand_range(tau1, order) + || !zkp_rand_range(tau2, order)) + goto err; + + /* (45) */ + if (!zkp_random_bn_gen(group, sL, n, bn_ctx) + || !zkp_random_bn_gen(group, sR, n, bn_ctx)) + goto err; + + for (i = 0; i < witness_n; i++) { + witness_v = sk_BIGNUM_value(witness->sk_v, i); + for (j = 0; j < bits; j++) { + if (!BN_div(dv, t, witness_v, bn2, bn_ctx)) + goto err; + + witness_v = dv; + m = i * pp->gens_capacity + j; + aL[m] = BN_is_one(t); + aR[m] = aL[m] - 1; + + G = sk_EC_POINT_value(pp->sk_G, m); + H = sk_EC_POINT_value(pp->sk_H, m); + if (G == NULL || H == NULL) + goto err; + + if (!zkp_poly_points_append(poly_a, G, aL[m] == 1 ? bn1 : bn0) + || !zkp_poly_points_append(poly_a, H, aR[m] == -1 ? bn_1 : bn0) + || !zkp_poly_points_append(poly_s, G, sL[m]) + || !zkp_poly_points_append(poly_s, H, sR[m])) + goto err; + } + } + + if (!zkp_poly_points_append(poly_a, pp->H, alpha) + || !zkp_poly_points_append(poly_s, pp->H, rho)) + goto err; + + /* (44, 47) */ + if (!zkp_poly_points_mul(poly_a, proof->A, NULL, group, bn_ctx) + || !zkp_poly_points_mul(poly_s, proof->S, NULL, group, bn_ctx)) + goto err; + + /* compute hash */ + if (!ZKP_TRANSCRIPT_append_point(transcript, "A", proof->A, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "S", proof->S, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "y", y) + || !ZKP_TRANSCRIPT_challange(transcript, "z", z)) + goto err; + + if (!BN_mod_sqr(z2, z, order, bn_ctx) || !BN_copy(pow_zn, z2) + || !BN_mod_inverse(y_inv, y, order, bn_ctx)) + goto err; + + pow_y[0] = bn1; + BN_zero(proof->taux); + + /* + * ll0 = aL - z * 1^n + * rr1 = aR + z * 1^n + * rr2 = z^(n+1) * 2^n + * r0 = y^n * (aR + z * 1^n) + z^(n+1) * 2^n = y^n * rr1 + rr2 + * r1 = y^n * sR + * l = ll0 + sL * x + * r = y^n * (aR + z * 1^n + sR * x) + z^(n+1) * 2^n = y^n * (rr1 + sR * x) + rr2 + * t1 = + * t2 = = + */ + for (i = 0; i < witness_n; i++) { + witness_r = sk_BIGNUM_value(witness->sk_r, i); + BN_one(pow_2); + + for (j = 0; j < bits; j++) { + m = i * bits + j; + if (m > 0) { + if ((pow_y[m] = BN_CTX_get(bn_ctx)) == NULL) + goto err; + + if (!BN_mod_mul(pow_y[m], pow_y[m-1], y, order, bn_ctx)) + goto err; + } + + if ((ll0[m] = BN_CTX_get(bn_ctx)) == NULL + || (rr1[m] = BN_CTX_get(bn_ctx)) == NULL + || (rr2[m] = BN_CTX_get(bn_ctx)) == NULL) + goto err; + + if (!BN_mod_sub(ll0[m], aL[m] == 1 ? bn1 : bn0, z, order, bn_ctx) + || !BN_mod_mul(r1, pow_y[m], sR[m], order, bn_ctx) + || !BN_mod_mul(t, ll0[m], r1, order, bn_ctx) + || !BN_mod_add(t1, t1, t, order, bn_ctx) + || !BN_mod_add(rr1[m], aR[m] == 0 ? bn0 : bn_1, z, order, bn_ctx) + || !BN_mod_mul(t, pow_y[m], rr1[m], order, bn_ctx)) + goto err; + + if (!BN_mod_mul(rr2[m], pow_zn, pow_2, order, bn_ctx) + || !BN_mod_add(r0, t, rr2[m], order, bn_ctx) + || !BN_mod_mul(t, r0, sL[m], order, bn_ctx) + || !BN_mod_add(t1, t1, t, order, bn_ctx) + || !BN_mod_mul(t, r1, sL[m], order, bn_ctx) + || !BN_mod_add(t2, t2, t, order, bn_ctx)) + goto err; + + if (!BN_mod_mul(pow_2, pow_2, bn2, order, bn_ctx)) + goto err; + } + + if (!BN_mul(t, pow_zn, witness_r, bn_ctx) + || !BN_mod_add(proof->taux, proof->taux, t, order, bn_ctx)) + goto err; + + if (!BN_mod_mul(pow_zn, pow_zn, z, order, bn_ctx)) + goto err; + } + + /* (53, 54) */ + if (!EC_POINT_mul(group, proof->T1, tau1, pp->H, t1, bn_ctx) + || !EC_POINT_mul(group, proof->T2, tau2, pp->H, t2, bn_ctx)) + goto err; + + /* (55, 56) */ + if (!ZKP_TRANSCRIPT_append_point(transcript, "T1", proof->T1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T2", proof->T2, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "x", x)) + goto err; + + BN_zero(proof->tx); + + for (i = 0; i < witness_n; i++) { + for (j = 0; j < bits; j++) { + m = i * bits + j; + tmp = BN_CTX_get(bn_ctx); + l = BN_CTX_get(bn_ctx); + r = BN_CTX_get(bn_ctx); + g_scalar = BN_CTX_get(bn_ctx); + h_scalar = BN_CTX_get(bn_ctx); + if (h_scalar == NULL) + goto err; + + G = sk_EC_POINT_value(pp->sk_G, m); + H = sk_EC_POINT_value(pp->sk_H, m); + if (G == NULL || H == NULL) + goto err; + + BN_one(g_scalar); + + /* (58, 59, 60) */ + if (!BN_mod_mul(t, sL[m], x, order, bn_ctx) + || !BN_mod_add(l, ll0[m], t, order, bn_ctx) + || !BN_mod_mul(t, sR[m], x, order, bn_ctx) + || !BN_mod_add(rr1[m], rr1[m], t, order, bn_ctx) + || !BN_mod_mul(dv, pow_y[m], rr1[m], order, bn_ctx) + || !BN_mod_add(r, dv, rr2[m], order, bn_ctx) + || !BN_mod_mul(t, l, r, order, bn_ctx) + || !BN_mod_add(proof->tx, proof->tx, t, order, bn_ctx)) + goto err; + + if (!BN_copy(h_scalar, pow_y_inv)) + goto err; + + if (sk_EC_POINT_push(sk_G, sk_EC_POINT_value(pp->sk_G, m)) <= 0 + || sk_EC_POINT_push(sk_H, sk_EC_POINT_value(pp->sk_H, m)) <= 0 + || sk_BIGNUM_push(sk_G_scalars, g_scalar) <= 0 + || sk_BIGNUM_push(sk_H_scalars, h_scalar) <= 0 + || sk_BIGNUM_push(sk_l, l) <= 0 + || sk_BIGNUM_push(sk_r, r) <= 0) + goto err; + + if (!BN_mod_mul(tmp, r, pow_y_inv, order, bn_ctx)) + goto err; + + if (!BN_mod_mul(pow_y_inv, pow_y_inv, y_inv, order, bn_ctx)) + goto err; + + if (!zkp_poly_points_append(poly_p, G, l) + || !zkp_poly_points_append(poly_p, H, tmp)) + goto err; + } + } + + /* (61) */ + if (!BN_mod_sqr(t, x, order, bn_ctx) + || !BN_mod_mul(t, t, tau2, order, bn_ctx) + || !BN_mod_add(proof->taux, proof->taux, t, order, bn_ctx) + || !BN_mod_mul(t, x, tau1, order, bn_ctx) + || !BN_mod_add(proof->taux, proof->taux, t, order, bn_ctx)) + goto err; + + /* (62) */ + if (!BN_mul(proof->mu, rho, x, bn_ctx) + || !BN_mod_add(proof->mu, proof->mu, alpha, order, bn_ctx)) + goto err; + + /* (67) */ + if (!EC_POINT_mul(group, U, NULL, pp->U, x, bn_ctx) + || !zkp_poly_points_append(poly_p, U, proof->tx) + || !zkp_poly_points_mul(poly_p, P, NULL, group, bn_ctx)) + goto err; + + if (!(ip_pp = bp_inner_product_pub_param_new(group, sk_G, sk_H)) + || !(ip_ctx = bp_inner_product_ctx_new(ip_pp, transcript, U, P, + sk_G_scalars, sk_H_scalars)) + || !(ip_witness = bp_inner_product_witness_new(sk_l, sk_r)) + || !(proof->ip_proof = bp_inner_product_proof_prove(ip_ctx, ip_witness))) + goto err; + + ret = 1; + +err: + ZKP_TRANSCRIPT_reset(transcript); + + bp_inner_product_witness_free(ip_witness); + bp_inner_product_pub_param_free(ip_pp); + bp_inner_product_ctx_free(ip_ctx); + + zkp_poly_points_free(poly_a); + zkp_poly_points_free(poly_s); + zkp_poly_points_free(poly_p); + + sk_EC_POINT_free(sk_G); + sk_EC_POINT_free(sk_H); + sk_BIGNUM_free(sk_G_scalars); + sk_BIGNUM_free(sk_H_scalars); + sk_BIGNUM_free(sk_l); + sk_BIGNUM_free(sk_r); + + OPENSSL_free(pow_y); + OPENSSL_free(ll0); + OPENSSL_free(rr1); + OPENSSL_free(rr2); + OPENSSL_free(sL); + OPENSSL_free(sR); + OPENSSL_free(aL); + OPENSSL_free(aR); + EC_POINT_free(P); + EC_POINT_free(T); + EC_POINT_free(U); + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + return ret; +} + +/** Prove computes the ZK rangeproof and new a proof object. + * \param ctx BP_RANGE_CTX object + * \return BP_RANGE_PROOF object on success or NULL in case of an error + */ +BP_RANGE_PROOF *BP_RANGE_PROOF_new_prove(BP_RANGE_CTX *ctx) +{ + BP_RANGE_PROOF *proof = NULL; + + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(proof = BP_RANGE_PROOF_new(ctx->pp))) + return NULL; + + if (!BP_RANGE_PROOF_prove(ctx, proof)) + goto err; + + return proof; +err: + BP_RANGE_PROOF_free(proof); + return NULL; +} + +/** Verifies that the supplied proof is a valid proof + * for the supplied secret values using the supplied public parameters. + * \param ctx BP_RANGE_CTX object + * \param proof BP_RANGE_PROOF object + * \return 1 if the proof is valid, 0 if the proof is invalid and -1 on error + */ +int BP_RANGE_PROOF_verify(BP_RANGE_CTX *ctx, const BP_RANGE_PROOF *proof) +{ + int ret = 0, i = 0, j, m, n, bits, poly_p_num, poly_r_num, witness_n, witness_padded_n; + ZKP_TRANSCRIPT *transcript; + BP_PUB_PARAM *pp; + BP_WITNESS *witness; + BP_VARIABLE *V; + BIGNUM *bn0, *bn1, *bn2, *delta; + BIGNUM *x, *x2, *y, *y_inv, *z, *z2, *nz, *t, *tmp, *z_pow_y; + BIGNUM *pow_y, *pow_y_inv, *pow_z, *pow_2, *sum_pow_y, *sum_pow_z, *sum_pow_2; + BIGNUM *g_scalar, *h_scalar; + STACK_OF(BIGNUM) *sk_G_scalars = NULL, *sk_H_scalars = NULL; + STACK_OF(EC_POINT) *sk_G = NULL, *sk_H = NULL; + EC_POINT *O = NULL, *P = NULL, *U = NULL, *L = NULL, *R = NULL, *G, *H; + BN_CTX *bn_ctx = NULL; + zkp_poly_points_t *poly_p = NULL, *poly_r = NULL; + EC_GROUP *group; + const BIGNUM *order; + bp_inner_product_ctx_t *ip_ctx = NULL; + bp_inner_product_pub_param_t *ip_pp = NULL; + + if (ctx == NULL || ctx->pp == NULL || ctx->witness == NULL || proof == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + transcript = ctx->transcript; + pp = ctx->pp; + witness = ctx->witness; + bits = pp->gens_capacity; + witness_n = sk_BP_VARIABLE_num(witness->sk_V); + witness_padded_n = zkp_next_power_of_two(witness_n); + n = bits * witness_padded_n; + poly_p_num = bits * witness_padded_n * 2 + 4; + poly_r_num = bits * witness_padded_n + 3; + group = pp->group; + order = EC_GROUP_get0_order(group); + + if (!zkp_is_power_of_two(n)) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_RANGE_LEN_MUST_BE_POWER_OF_TWO); + return 0; + } + + if (witness_n != witness_padded_n) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_WITNESS_INVALID); + return 0; + } + + if (witness_padded_n > ctx->pp->party_capacity) { + ERR_raise(ERR_LIB_ZKP_BP, ZKP_BP_R_EXCEEDS_PARTY_CAPACITY); + return 0; + } + + if (EC_GROUP_get_curve_name(pp->group) != EC_POINT_get_curve_name(proof->A)) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return 0; + } + + if (!(sk_G = sk_EC_POINT_new_reserve(NULL, n)) + || !(sk_H = sk_EC_POINT_new_reserve(NULL, n)) + || !(sk_G_scalars = sk_BIGNUM_new_reserve(NULL, n)) + || !(sk_H_scalars = sk_BIGNUM_new_reserve(NULL, n))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!(poly_p = zkp_poly_points_new(poly_p_num)) || !(poly_r = zkp_poly_points_new(poly_r_num))) + goto err; + + if (!(O = EC_POINT_new(group)) + || !(P = EC_POINT_new(group)) + || !(U = EC_POINT_new(group)) + || !(L = EC_POINT_new(group)) + || !(R = EC_POINT_new(group))) + goto err; + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + BN_CTX_start(bn_ctx); + bn0 = BN_CTX_get(bn_ctx); + bn1 = BN_CTX_get(bn_ctx); + bn2 = BN_CTX_get(bn_ctx); + x = BN_CTX_get(bn_ctx); + x2 = BN_CTX_get(bn_ctx); + y = BN_CTX_get(bn_ctx); + y_inv = BN_CTX_get(bn_ctx); + z = BN_CTX_get(bn_ctx); + z2 = BN_CTX_get(bn_ctx); + z_pow_y = BN_CTX_get(bn_ctx); + nz = BN_CTX_get(bn_ctx); + sum_pow_y = BN_CTX_get(bn_ctx); + sum_pow_z = BN_CTX_get(bn_ctx); + sum_pow_2 = BN_CTX_get(bn_ctx); + t = BN_CTX_get(bn_ctx); + pow_y = BN_CTX_get(bn_ctx); + pow_y_inv = BN_CTX_get(bn_ctx); + pow_z = BN_CTX_get(bn_ctx); + pow_2 = BN_CTX_get(bn_ctx); + delta = BN_CTX_get(bn_ctx); + if (delta == NULL) + goto err; + + BN_zero(sum_pow_y); + BN_zero(sum_pow_z); + BN_zero(sum_pow_2); + BN_one(pow_y); + BN_one(pow_y_inv); + BN_one(bn0); + BN_one(bn1); + BN_set_word(bn2, 2); + + EC_POINT_set_to_infinity(group, O); + + if (!ZKP_TRANSCRIPT_append_point(transcript, "A", proof->A, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "S", proof->S, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "y", y) + || !ZKP_TRANSCRIPT_challange(transcript, "z", z)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "T1", proof->T1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "T2", proof->T2, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "x", x)) + goto err; + + if (!BN_mod_inverse(y_inv, y, order, bn_ctx) + || !BN_mod_sqr(x2, x, order, bn_ctx) + || !BN_mod_sqr(z2, z, order, bn_ctx) + || !BN_sub(nz, order, z) + || !BN_copy(pow_z, z)) + goto err; + + for (i = 0; i < witness_n; i++) { + V = sk_BP_VARIABLE_value(witness->sk_V, i); + if (V == NULL) + goto err; + + BN_one(pow_2); + + if (!BN_mod_mul(pow_z, pow_z, z, order, bn_ctx) + || !BN_mod_add(sum_pow_z, sum_pow_z, pow_z, order, bn_ctx)) + goto err; + + for (j = 0; j < bits; j++) { + m = i * bits + j; + if (i == 0) { + if (!BN_mod_add(sum_pow_2, sum_pow_2, pow_2, order, bn_ctx)) + goto err; + } + + G = sk_EC_POINT_value(pp->sk_G, m); + H = sk_EC_POINT_value(pp->sk_H, m); + if (G == NULL || H == NULL) + goto err; + + tmp = BN_CTX_get(bn_ctx); + g_scalar = BN_CTX_get(bn_ctx); + h_scalar = BN_CTX_get(bn_ctx); + if (h_scalar == NULL) + goto err; + + BN_one(g_scalar); + + if (!BN_copy(h_scalar, pow_y_inv)) + goto err; + + if (sk_EC_POINT_push(sk_G, sk_EC_POINT_value(pp->sk_G, m)) <= 0 + || sk_EC_POINT_push(sk_H, sk_EC_POINT_value(pp->sk_H, m)) <= 0 + || sk_BIGNUM_push(sk_G_scalars, g_scalar) <= 0 + || sk_BIGNUM_push(sk_H_scalars, h_scalar) <= 0) + goto err; + + if (!BN_mod_add(sum_pow_y, sum_pow_y, pow_y, order, bn_ctx) + || !BN_mod_mul(z_pow_y, z, pow_y, order, bn_ctx) + || !BN_mod_mul(t, pow_z, pow_2, order, bn_ctx) + || !BN_mod_add(t, t, z_pow_y, order, bn_ctx)) + goto err; + + if (!BN_copy(tmp, t) || !BN_mod_mul(tmp, tmp, pow_y_inv, order, bn_ctx)) + goto err; + + if (!zkp_poly_points_append(poly_p, G, nz) + || !zkp_poly_points_append(poly_p, H, tmp)) + goto err; + + if (!BN_mod_mul(pow_y, pow_y, y, order, bn_ctx) + || !BN_mod_mul(pow_y_inv, pow_y_inv, y_inv, order, bn_ctx) + || !BN_mod_mul(pow_2, pow_2, bn2, order, bn_ctx)) + goto err; + } + + tmp = BN_CTX_get(bn_ctx); + if (tmp == NULL || !BN_copy(tmp, pow_z)) + goto err; + + if (!zkp_poly_points_append(poly_r, V->point, tmp)) + goto err; + } + + if (!BN_mod_mul(sum_pow_z, sum_pow_z, z, order, bn_ctx)) + goto err; + + /* (39) also see page 21 */ + if (!BN_mod_sub(delta, z, z2, order, bn_ctx) + || !BN_mod_mul(delta, delta, sum_pow_y, order, bn_ctx) + || !BN_mod_mul(t, sum_pow_z, sum_pow_2, order, bn_ctx) + || !BN_mod_sub(delta, delta, t, order, bn_ctx)) + goto err; + + /* (72) */ + if (!zkp_poly_points_append(poly_r, pp->H, delta) + || !zkp_poly_points_append(poly_r, proof->T1, x) + || !zkp_poly_points_append(poly_r, proof->T2, x2) + || !zkp_poly_points_mul(poly_r, R, NULL, group, bn_ctx)) + goto err; + + /* (65) */ + if (!EC_POINT_mul(group, L, proof->taux, pp->H, proof->tx, bn_ctx) + || !EC_POINT_invert(group, L, bn_ctx) + || !EC_POINT_add(group, R, R, L, bn_ctx) + || !EC_POINT_is_at_infinity(group, R)) + goto err; + + if (!EC_POINT_mul(group, U, NULL, pp->U, x, bn_ctx)) + goto err; + + tmp = BN_CTX_get(bn_ctx); + if (tmp == NULL) + goto err; + + if (!BN_copy(tmp, proof->mu)) + goto err; + + BN_set_negative(tmp, !BN_is_negative(tmp)); + + if (!zkp_poly_points_append(poly_p, proof->S, x) + || !zkp_poly_points_append(poly_p, proof->A, bn1) + || !zkp_poly_points_append(poly_p, pp->H, tmp) + || !zkp_poly_points_append(poly_p, U, proof->tx) + || !zkp_poly_points_mul(poly_p, P, NULL, group, bn_ctx)) + goto err; + + if (!(ip_pp = bp_inner_product_pub_param_new(group, sk_G, sk_H)) + || !(ip_ctx = bp_inner_product_ctx_new(ip_pp, transcript, U, P, + sk_G_scalars, sk_H_scalars))) + goto err; + + ret = bp_inner_product_proof_verify(ip_ctx, proof->ip_proof); + +err: + ZKP_TRANSCRIPT_reset(transcript); + + bp_inner_product_ctx_free(ip_ctx); + bp_inner_product_pub_param_free(ip_pp); + + zkp_poly_points_free(poly_p); + zkp_poly_points_free(poly_r); + + BN_CTX_end(bn_ctx); + BN_CTX_free(bn_ctx); + + EC_POINT_free(L); + EC_POINT_free(R); + EC_POINT_free(U); + EC_POINT_free(P); + EC_POINT_free(O); + + sk_EC_POINT_free(sk_G); + sk_EC_POINT_free(sk_H); + sk_BIGNUM_free(sk_G_scalars); + sk_BIGNUM_free(sk_H_scalars); + return ret; +} diff --git a/openssl/src/crypto/zkp/bulletproofs/range_proof.h b/openssl/src/crypto/zkp/bulletproofs/range_proof.h new file mode 100644 index 000000000..fa1ce4728 --- /dev/null +++ b/openssl/src/crypto/zkp/bulletproofs/range_proof.h @@ -0,0 +1,53 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_BULLET_PROOF_LOCAL_H +# define HEADER_BULLET_PROOF_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include +# include "internal/refcount.h" +# include "bulletproofs.h" +# include "inner_product.h" + +struct bp_range_ctx_st { + ZKP_TRANSCRIPT *transcript; + BP_PUB_PARAM *pp; + BP_WITNESS *witness; +}; + +struct bp_range_proof_st { + EC_POINT *A; + EC_POINT *S; + EC_POINT *T1; + EC_POINT *T2; + BIGNUM *taux; + BIGNUM *mu; + BIGNUM *tx; + bp_inner_product_proof_t *ip_proof; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +BP_RANGE_PROOF *bp_range_proof_alloc(const EC_GROUP *group); + +# ifdef __cplusplus +} +# endif + +#endif + diff --git a/openssl/src/crypto/zkp/common/zkp_debug.c b/openssl/src/crypto/zkp/common/zkp_debug.c new file mode 100644 index 000000000..ca50a1d2d --- /dev/null +++ b/openssl/src/crypto/zkp/common/zkp_debug.c @@ -0,0 +1,221 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include "zkp_debug.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) + +int zkp_rand_range_debug_one(BIGNUM *rnd, const BIGNUM *range) +{ + BN_set_word(rnd, 1); + return 1; +} + +int zkp_buf2hexstr_print(BIO *bio, const unsigned char *buf, size_t size, + char *field, int text) +{ + unsigned char *out = NULL; + size_t out_n; + BIO *b = NULL; + + if (bio == NULL) { + bio = b = BIO_new(BIO_s_file()); + BIO_set_fp(b, stderr, BIO_NOCLOSE); + } + + BIO_printf(bio, "%s: ", field); + + if (text) { + BIO_puts(bio, "\n"); + BIO_indent(bio, 4, 4); + BIO_hex_string(bio, 4, 16, buf, size); + } else { + out_n = size * 2 + 1; + if (!(out = (unsigned char *)OPENSSL_zalloc(out_n)) + || !OPENSSL_buf2hexstr_ex((char *)out, out_n, NULL, buf, size, '\0')) { + OPENSSL_free(out); + return 0; + } + BIO_printf(bio, "%s", out); + OPENSSL_free(out); + } + + BIO_puts(bio, "\n"); + BIO_free(b); + return 1; +} + + +void BN_debug_print(BIO *b, const BIGNUM *n, const char *name) +{ + BIO *bi = NULL; + + if (b == NULL) { + b = bi = BIO_new(BIO_s_file()); + BIO_set_fp(b, stderr, BIO_NOCLOSE); + } + + BIO_printf(b, "%s: ", name); + BN_print(b, n); + BIO_printf(b, "\n"); + + BIO_free(bi); +} + +void EC_POINT_debug_print(BIO *b, const EC_POINT *p, const char *name) +{ + BIO *bi = NULL; + + if (b == NULL) { + b = bi = BIO_new(BIO_s_file()); + BIO_set_fp(b, stderr, BIO_NOCLOSE); + } + + BIO_printf(b, "%s->X: ", name); + BN_print(b, p->X); + BIO_printf(b, ", %s->Y: ", name); + BN_print(b, p->Y); + BIO_printf(b, ", %s->Z: ", name); + BN_print(b, p->Z); + BIO_printf(b, "\n"); + + BIO_free(bi); +} + +void EC_POINT_debug_print_affine(BIO *b, const EC_GROUP *group, const EC_POINT *p, + const char *name, BN_CTX *ctx) +{ + BIO *bi = NULL; + BIGNUM *x, *y; + BN_CTX *bn_ctx = NULL; + + if (ctx == NULL) + return; + + if (b == NULL) { + b = bi = BIO_new(BIO_s_file()); + BIO_set_fp(b, stderr, BIO_NOCLOSE); + } + + if (ctx == NULL) { + bn_ctx = ctx = BN_CTX_new(); + if (bn_ctx == NULL) + goto err; + } + + BN_CTX_start(ctx); + x = BN_CTX_get(ctx); + y = BN_CTX_get(ctx); + if (y == NULL) + goto err; + + EC_POINT_get_affine_coordinates(group, p, x, y, ctx); + + BIO_printf(b, "%s->x: ", name); + BN_print(b, x); + BIO_printf(b, ", %s->y: ", name); + BN_print(b, y); + BIO_printf(b, "\n"); + +err: + BN_CTX_end(ctx); + BN_CTX_free(bn_ctx); + BIO_free(bi); +} + +void zkp_bn_vector_debug_print(BIO *bio, BIGNUM **bv, int n, const char *note) +{ + int i; + + if (bv == NULL) + return; + + for (i = 0; i < n; i++) { + BN_debug_print(bio, bv[i], note); + } +} + +void zkp_point_vector_debug_print(BIO *bio, const EC_GROUP *group, EC_POINT **pv, + int n, const char *note, BN_CTX *bn_ctx) +{ + int i; + + if (group == NULL || pv == NULL) + return; + + for (i = 0; i < n; i++) { + EC_POINT_debug_print_affine(bio, group, pv[i], note, bn_ctx); + } +} + +void zkp_stack_of_bignum_debug_print(BIO *bio, STACK_OF(BIGNUM) *sk, const char *name) +{ + BIO *b = NULL; + int i, n; + BIGNUM *bn; + + if (sk == NULL) + return; + + if (bio == NULL) { + b = bio = BIO_new(BIO_s_file()); + BIO_set_fp(b, stderr, BIO_NOCLOSE); + } + + n = sk_BIGNUM_num(sk); + for (i = 0; i < n; i++) { + bn = sk_BIGNUM_value(sk, i); + if (bn == NULL) + goto err; + + BIO_printf(bio, "%s[%d]: ", name, i); + BN_print(bio, bn); + BIO_printf(bio, "\n"); + } + +err: + BIO_free(b); +} + +void zkp_stack_of_point_debug_print(BIO *bio, STACK_OF(EC_POINT) *sk, const char *name) +{ + BIO *b = NULL; + int i, n; + EC_POINT *p; + + if (sk == NULL) + return; + + if (bio == NULL) { + b = bio = BIO_new(BIO_s_file()); + BIO_set_fp(b, stderr, BIO_NOCLOSE); + } + + n = sk_EC_POINT_num(sk); + for (i = 0; i < n; i++) { + p = sk_EC_POINT_value(sk, i); + if (p == NULL) + goto err; + + BIO_printf(b, "%s[%d]->X: ", name, i); + BN_print(b, p->X); + BIO_printf(b, ", %s[%d]->Y: ", name, i); + BN_print(b, p->Y); + BIO_printf(b, ", %s[%d]->Z: ", name, i); + BN_print(b, p->Z); + BIO_printf(b, "\n"); + } + +err: + BIO_free(b); +} diff --git a/openssl/src/crypto/zkp/common/zkp_debug.h b/openssl/src/crypto/zkp/common/zkp_debug.h new file mode 100644 index 000000000..84f297ede --- /dev/null +++ b/openssl/src/crypto/zkp/common/zkp_debug.h @@ -0,0 +1,47 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_ZKP_DEBUG_LOCAL_H +# define HEADER_ZKP_DEBUG_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include "internal/refcount.h" + +STACK_OF(BIGNUM); +STACK_OF(EC_POINT); + +int zkp_rand_range_debug_one(BIGNUM *rnd, const BIGNUM *range); + +int zkp_buf2hexstr_print(BIO *bio, const unsigned char *buf, size_t size, + char *field, int text); + +void BN_debug_print(BIO *b, const BIGNUM *n, const char *name); +void EC_POINT_debug_print(BIO *b, const EC_POINT *p, const char *name); +void EC_POINT_debug_print_affine(BIO *b, const EC_GROUP *group, const EC_POINT *p, + const char *name, BN_CTX *ctx); + +void zkp_bn_vector_debug_print(BIO *bio, BIGNUM **bv, int n, const char *note); +void zkp_point_vector_debug_print(BIO *bio, const EC_GROUP *group, EC_POINT **pv, + int n, const char *note, BN_CTX *bn_ctx); +void zkp_stack_of_bignum_debug_print(BIO *bio, STACK_OF(BIGNUM) *sk, const char *name); +void zkp_stack_of_point_debug_print(BIO *bio, STACK_OF(EC_POINT) *sk, const char *nam); + +# ifdef __cplusplus +} +# endif + +#endif + diff --git a/openssl/src/crypto/zkp/common/zkp_err.c b/openssl/src/crypto/zkp/common/zkp_err.c new file mode 100644 index 000000000..be427241c --- /dev/null +++ b/openssl/src/crypto/zkp/common/zkp_err.c @@ -0,0 +1,71 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include +#include "crypto/zkperr.h" + +#ifndef OPENSSL_NO_ERR + +static const ERR_STRING_DATA ZKP_str_reasons[] = { + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_EXCEEDS_GENS_CAPACITY), + "exceeds gens capacity"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_EXCEEDS_MAX_AGG_NUM), + "exceeds max agg num"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_EXCEEDS_MAX_BITS), "exceeds max bits"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_EXCEEDS_PARTY_CAPACITY), + "exceeds party capacity"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_EXCEEDS_PP_CAPACITY), + "exceeds pp capacity"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_FORMAT_ERROR), + "r1cs constraint expression format error"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_NO_VAR), + "r1cs constraint expression no var"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_PROCESS_ERROR), + "r1cs constraint expression process error"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_NOT_FOUND), + "r1cs constraint expression var not found"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_R1CS_CONSTRAINT_EXPRESSION_VAR_TOO_LONG), + "r1cs constraint expression var too long"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_TRANSCRIPT_INIT_FAILED), + "transcript init failed"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_VARIABLE_DUPLICATED), + "variable duplicated"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_VARIABLE_NAME_TOO_LONG), + "variable name too long"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_BP_R_WITNESS_INVALID), "witness invalid"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_NIZK_R_TRANSCRIPT_INIT_FAILED), + "transcript init failed"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_R_BULLETPROOFS_RANGE_PROVE_FAILED), + "bulletproofs range prove failed"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_R_BULLETPROOFS_RANGE_VERIFY_FAILED), + "bulletproofs range verify failed"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_R_NIZK_PLAINTEXT_KNOWLEDGE_PROVE_FAILED), + "nizk plaintext knowledge prove failed"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_R_NIZK_PLAINTEXT_KNOWLEDGE_VERIFY_FAILED), + "nizk plaintext knowledge verify failed"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_R_RANGE_PROVE_FAILED), "range prove failed"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_R_RANGE_VERIFY_FAILED), + "range verify failed"}, + {ERR_PACK(ERR_LIB_ZKP, 0, ZKP_R_TRANSCRIPT_INIT_FAILED), + "transcript init failed"}, + {0, NULL} +}; + +#endif + +int ossl_err_load_ZKP_strings(void) +{ +#ifndef OPENSSL_NO_ERR + if (ERR_reason_error_string(ZKP_str_reasons[0].error) == NULL) + ERR_load_strings_const(ZKP_str_reasons); +#endif + return 1; +} diff --git a/openssl/src/crypto/zkp/common/zkp_transcript.c b/openssl/src/crypto/zkp/common/zkp_transcript.c new file mode 100644 index 000000000..84e866dbf --- /dev/null +++ b/openssl/src/crypto/zkp/common/zkp_transcript.c @@ -0,0 +1,115 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include "zkp_transcript.h" + +ZKP_TRANSCRIPT *ZKP_TRANSCRIPT_new(const ZKP_TRANSCRIPT_METHOD *method, + const char *label) +{ + ZKP_TRANSCRIPT *transcript = NULL; + + if (method == NULL || label == NULL) { + return NULL; + } + + transcript = OPENSSL_zalloc(sizeof(*transcript)); + if (transcript == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + transcript->method = method; + transcript->label = OPENSSL_strdup(label); + if (transcript->label == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!transcript->method->init(transcript)) { + ERR_raise(ERR_LIB_ZKP, ZKP_R_TRANSCRIPT_INIT_FAILED); + goto err; + } + + return transcript; +err: + ZKP_TRANSCRIPT_free(transcript); + return NULL; +} + +ZKP_TRANSCRIPT *ZKP_TRANSCRIPT_dup(const ZKP_TRANSCRIPT *src) +{ + return ZKP_TRANSCRIPT_new(src->method, src->label); +} + +void ZKP_TRANSCRIPT_free(ZKP_TRANSCRIPT *transcript) +{ + if (transcript == NULL) + return; + + if (transcript->method) + transcript->method->cleanup(transcript); + + OPENSSL_free(transcript->label); + OPENSSL_free(transcript); +} + +int ZKP_TRANSCRIPT_append_int64(ZKP_TRANSCRIPT *transcript, const char *label, + int64_t i64) +{ + if (transcript == NULL || transcript->method == NULL) + return 0; + + return transcript->method->append_int64(transcript, label, i64); +} + +int ZKP_TRANSCRIPT_append_str(ZKP_TRANSCRIPT *transcript, const char *label, + const char *str, int len) +{ + if (transcript == NULL || transcript->method == NULL) + return 0; + + return transcript->method->append_str(transcript, label, str, len); +} + +int ZKP_TRANSCRIPT_append_point(ZKP_TRANSCRIPT *transcript, const char *label, + const EC_POINT *point, const EC_GROUP *group) +{ + if (transcript == NULL || transcript->method == NULL) + return 0; + + return transcript->method->append_point(transcript, label, point, group); +} + +int ZKP_TRANSCRIPT_append_bn(ZKP_TRANSCRIPT *transcript, const char *label, + const BIGNUM *bn) +{ + if (transcript == NULL || transcript->method == NULL) + return 0; + + return transcript->method->append_bn(transcript, label, bn); +} + +int ZKP_TRANSCRIPT_challange(ZKP_TRANSCRIPT *transcript, const char *label, + BIGNUM *out) +{ + if (transcript == NULL || transcript->method == NULL) + return 0; + + return transcript->method->challange(transcript, label, out); +} + +int ZKP_TRANSCRIPT_reset(ZKP_TRANSCRIPT *transcript) +{ + if (transcript == NULL || transcript->method == NULL) + return 0; + + return transcript->method->reset(transcript); +} diff --git a/openssl/src/crypto/zkp/common/zkp_transcript.h b/openssl/src/crypto/zkp/common/zkp_transcript.h new file mode 100644 index 000000000..54847083e --- /dev/null +++ b/openssl/src/crypto/zkp/common/zkp_transcript.h @@ -0,0 +1,45 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_ZKP_TRANSCRIPT_LOCAL_H +# define HEADER_ZKP_TRANSCRIPT_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include + +struct zkp_transcript_method_st { + int (*init)(ZKP_TRANSCRIPT *transcript); + int (*reset)(ZKP_TRANSCRIPT *transcript); + int (*cleanup)(ZKP_TRANSCRIPT *transcript); + int (*append_int64)(ZKP_TRANSCRIPT *transcript, const char *label, int64_t i64); + int (*append_str)(ZKP_TRANSCRIPT *transcript, const char *label, + const char *str, int len); + int (*append_point)(ZKP_TRANSCRIPT *transcript, const char *label, + const EC_POINT *point, const EC_GROUP *group); + int (*append_bn)(ZKP_TRANSCRIPT *transcript, const char *label, const BIGNUM *bn); + int (*challange)(ZKP_TRANSCRIPT *transcript, const char *label, BIGNUM *out); +}; + +struct zkp_transcript_st { + char *label; + void *data; + const ZKP_TRANSCRIPT_METHOD *method; +}; + +# ifdef __cplusplus +} +# endif + +#endif + diff --git a/openssl/src/crypto/zkp/common/zkp_transcript_sha256.c b/openssl/src/crypto/zkp/common/zkp_transcript_sha256.c new file mode 100644 index 000000000..f268f7a37 --- /dev/null +++ b/openssl/src/crypto/zkp/common/zkp_transcript_sha256.c @@ -0,0 +1,268 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "zkp_transcript.h" + +#ifndef __bswap_constant_64 +# define __bswap_constant_64(x) \ + ((((x) & 0xff00000000000000ull) >> 56) \ + | (((x) & 0x00ff000000000000ull) >> 40) \ + | (((x) & 0x0000ff0000000000ull) >> 24) \ + | (((x) & 0x000000ff00000000ull) >> 8) \ + | (((x) & 0x00000000ff000000ull) << 8) \ + | (((x) & 0x0000000000ff0000ull) << 24) \ + | (((x) & 0x000000000000ff00ull) << 40) \ + | (((x) & 0x00000000000000ffull) << 56)) +#endif + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define int64_n2l(x) (x) +# define int64_l2n(x) (x) +#else +# define int64_n2l(x) __bswap_constant_64(x) +# define int64_l2n(x) __bswap_constant_64(x) +#endif + +typedef struct zkp_transcript_sha256_ctx_st { + EVP_MD *sha256; + EVP_MD_CTX *md_ctx; +} zkp_transcript_sha256_ctx; + +static int zkp_transcript_sha256_init(ZKP_TRANSCRIPT *transcript) +{ + size_t len; + zkp_transcript_sha256_ctx *ctx = NULL; + + if (transcript == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!(ctx = OPENSSL_zalloc(sizeof(*ctx)))) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + goto err; + } + + ctx->md_ctx = EVP_MD_CTX_new(); + if (ctx->md_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!(ctx->sha256 = EVP_MD_fetch(NULL, "sha256", NULL)) + || !EVP_DigestInit(ctx->md_ctx, ctx->sha256)) + goto err; + + if (transcript->label != NULL) { + len = strlen(transcript->label); + if (!EVP_DigestUpdate(ctx->md_ctx, transcript->label, len)) + goto err; + } + + transcript->data = ctx; + + return 1; +err: + EVP_MD_CTX_free(ctx->md_ctx); + OPENSSL_free(ctx); + return 0; +} + +static int zkp_transcript_sha256_reset(ZKP_TRANSCRIPT *transcript) +{ + size_t len; + zkp_transcript_sha256_ctx *ctx = NULL; + + if (transcript == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_INVALID_ARGUMENT); + return 0; + } + + ctx = transcript->data; + + if (!EVP_DigestInit(ctx->md_ctx, ctx->sha256)) + return 0; + + if (transcript->label != NULL) { + len = strlen(transcript->label); + if (!EVP_DigestUpdate(ctx->md_ctx, transcript->label, len)) + return 0; + } + + return 1; +} + +static int zkp_transcript_sha256_cleanup(ZKP_TRANSCRIPT *transcript) +{ + zkp_transcript_sha256_ctx *ctx = NULL; + + if (transcript == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_INVALID_ARGUMENT); + return 0; + } + + ctx = transcript->data; + + EVP_MD_CTX_free(ctx->md_ctx); + + return 1; +} + +static int zkp_transcript_sha256_append_int64(ZKP_TRANSCRIPT *transcript, + const char *label, const int64_t i64) +{ + int64_t num; + zkp_transcript_sha256_ctx *ctx = NULL; + + if (transcript == NULL || label == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_INVALID_ARGUMENT); + return 0; + } + + ctx = transcript->data; + num = int64_l2n(i64); + + return EVP_DigestUpdate(ctx->md_ctx, label, strlen(label)) + && EVP_DigestUpdate(ctx->md_ctx, (char *)&num, sizeof(num)); +} + +static int zkp_transcript_sha256_append_str(ZKP_TRANSCRIPT *transcript, + const char *label, + const char *str, int len) +{ + zkp_transcript_sha256_ctx *ctx = NULL; + + if (transcript == NULL || str == NULL || len <= 0) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_INVALID_ARGUMENT); + return 0; + } + + ctx = transcript->data; + + return EVP_DigestUpdate(ctx->md_ctx, label, strlen(label)) + && EVP_DigestUpdate(ctx->md_ctx, str, len); +} + +static int zkp_transcript_sha256_append_point(ZKP_TRANSCRIPT *transcript, + const char *label, + const EC_POINT *point, + const EC_GROUP *group) +{ + int ret = 0; + size_t len; + unsigned char buf[128], *str = NULL; + point_conversion_form_t format = POINT_CONVERSION_COMPRESSED; + zkp_transcript_sha256_ctx *ctx = NULL; + + if (transcript == NULL || point == NULL || group == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + ctx = transcript->data; + + len = EC_POINT_point2oct(group, point, format, NULL, 0, NULL); + if (len > sizeof(buf)) { + if (!(str = OPENSSL_zalloc(len))) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + return 0; + } + } else { + str = &buf[0]; + } + + ret = EVP_DigestUpdate(ctx->md_ctx, label, strlen(label)) + && EC_POINT_point2oct(group, point, format, str, len, NULL) > 0 + && EVP_DigestUpdate(ctx->md_ctx, str, len); + + if (len > sizeof(buf)) + OPENSSL_free(str); + + return ret; +} + +static int zkp_transcript_sha256_append_bn(ZKP_TRANSCRIPT *transcript, + const char *label, const BIGNUM *bn) +{ + int ret = 0; + size_t len; + unsigned char buf[256] = {0}, *str = NULL; + zkp_transcript_sha256_ctx *ctx = NULL; + + if (transcript == NULL || bn == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + ctx = transcript->data; + + len = BN_is_zero(bn) ? 1 : BN_num_bytes(bn); + if (len > sizeof(buf)) { + if (!(str = OPENSSL_zalloc(len))) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + return 0; + } + } else { + str = &buf[0]; + } + + ret = EVP_DigestUpdate(ctx->md_ctx, label, strlen(label)) + && BN_bn2binpad(bn, str, len) && EVP_DigestUpdate(ctx->md_ctx, str, len); + + if (len > sizeof(buf)) + OPENSSL_free(str); + + return ret; +} + +static int zkp_transcript_sha256_challange(ZKP_TRANSCRIPT *transcript, + const char *label, BIGNUM *out) +{ + unsigned char hash_res[SHA256_DIGEST_LENGTH]; + zkp_transcript_sha256_ctx *ctx = NULL; + + if (transcript == NULL || out == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + ctx = transcript->data; + + if (!EVP_DigestUpdate(ctx->md_ctx, label, strlen(label)) + || !EVP_DigestFinal(ctx->md_ctx, hash_res, NULL)) + return 0; + + if (!BN_bin2bn(hash_res, SHA256_DIGEST_LENGTH, out)) + return 0; + + if (!EVP_DigestInit(ctx->md_ctx, ctx->sha256)) + return 0; + + return EVP_DigestUpdate(ctx->md_ctx, hash_res, SHA256_DIGEST_LENGTH); +} + +const ZKP_TRANSCRIPT_METHOD *ZKP_TRANSCRIPT_METHOD_sha256(void) +{ + static const ZKP_TRANSCRIPT_METHOD ret = { + zkp_transcript_sha256_init, + zkp_transcript_sha256_reset, + zkp_transcript_sha256_cleanup, + zkp_transcript_sha256_append_int64, + zkp_transcript_sha256_append_str, + zkp_transcript_sha256_append_point, + zkp_transcript_sha256_append_bn, + zkp_transcript_sha256_challange, + }; + + return &ret; +} diff --git a/openssl/src/crypto/zkp/common/zkp_util.c b/openssl/src/crypto/zkp/common/zkp_util.c new file mode 100644 index 000000000..36f6a5968 --- /dev/null +++ b/openssl/src/crypto/zkp/common/zkp_util.c @@ -0,0 +1,805 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "zkp_util.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) + +static point_conversion_form_t form = POINT_CONVERSION_COMPRESSED; + +EC_POINT *zkp_random_ec_point_new(const EC_GROUP *group, BN_CTX *bn_ctx) +{ + BIGNUM *r = NULL; + BN_CTX *bctx = NULL; + EC_POINT *P = NULL; + const BIGNUM *order; + + if (group == NULL) + return NULL; + + if (bn_ctx == NULL) { + bctx = bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + } + + order = EC_GROUP_get0_order(group); + + BN_CTX_start(bn_ctx); + r = BN_CTX_get(bn_ctx); + if (r == NULL) + goto err; + + zkp_rand_range(r, order); + + if (!(P = EC_POINT_new(group)) || !EC_POINT_mul(group, P, r, NULL, NULL, bn_ctx)) + goto err; + + BN_CTX_end(bn_ctx); + BN_CTX_free(bctx); + return P; +err: + BN_CTX_end(bn_ctx); + BN_CTX_free(bctx); + zkp_random_ec_point_free(P); + return NULL; +} + +void zkp_random_ec_point_free(EC_POINT *P) +{ + if (P == NULL) + return; + + EC_POINT_free(P); +} + +int zkp_random_bn_gen(const EC_GROUP *group, BIGNUM **r, size_t n, BN_CTX *bn_ctx) +{ + size_t i; + const BIGNUM *order; + + if (group == NULL || r == NULL || bn_ctx == NULL) + return 0; + + order = EC_GROUP_get0_order(group); + + for (i = 0; i < n; i++) { + if (!(r[i] = BN_CTX_get(bn_ctx)) || !zkp_rand_range(r[i], order)) + return 0; + } + + return 1; +} + +int zkp_str2point(const EC_GROUP *group, const unsigned char *str, size_t len, + EC_POINT *r, BN_CTX *bn_ctx) +{ + int ret = 0, i = 0; + unsigned char hash_res[SHA256_DIGEST_LENGTH]; + unsigned char *p = (unsigned char *)str; + BN_CTX *ctx = NULL; + BIGNUM *x; + + memset(hash_res, 0, sizeof(hash_res)); + + if (bn_ctx == NULL) { + if ((ctx = bn_ctx = BN_CTX_new_ex(group->libctx)) == NULL) + goto end; + } + + BN_CTX_start(bn_ctx); + if ((x = BN_CTX_get(bn_ctx)) == NULL) + goto end; + + do { + if (!SHA256(p, len, hash_res)) + goto end; + + BN_bin2bn(hash_res, SHA256_DIGEST_LENGTH, x); + + p = &hash_res[0]; + len = sizeof(hash_res); + + if(EC_POINT_set_compressed_coordinates(group, r, x, 0, bn_ctx) == 1) { + ret = 1; + break; + } + + ERR_clear_error(); + } while (i++ < 10); + +end: + BN_CTX_end(bn_ctx); + BN_CTX_free(ctx); + return ret; +} + +size_t zkp_point2oct(const EC_GROUP *group, const EC_POINT *P, + unsigned char *buf, BN_CTX *bn_ctx) +{ + size_t plen; + point_conversion_form_t format = POINT_CONVERSION_COMPRESSED; + + if (group == NULL || P == NULL || bn_ctx == NULL) + return -1; + + plen = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + format, NULL, 0, bn_ctx); + if (plen <= 0 || buf == NULL) + return plen; + + if (EC_POINT_point2oct(group, P, format, buf, plen, bn_ctx) <= 0) + return -1; + + return plen; +} + +int zkp_point2point(const EC_GROUP *group, const EC_POINT *P, EC_POINT *H, BN_CTX *bn_ctx) +{ + int ret = 0; + size_t len; + unsigned char *buf = NULL; + BN_CTX *bctx = NULL; + + if (group == NULL || P == NULL || H == NULL) + return -1; + + if (bn_ctx == NULL) { + bctx = bn_ctx = BN_CTX_new(); + } + + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_EC, ERR_R_MALLOC_FAILURE); + goto err; + } + + len = EC_POINT_point2oct(group, P, POINT_CONVERSION_COMPRESSED, NULL, 0, bn_ctx); + if (len <= 0) + goto err; + + buf = OPENSSL_zalloc(len); + if (buf == NULL) + goto err; + + if (!EC_POINT_point2oct(group, P, POINT_CONVERSION_COMPRESSED, buf, len, bn_ctx)) + goto err; + + if (!EC_POINT_from_string(group, H, buf, len)) + goto err; + + ret = 1; +err: + OPENSSL_free(buf); + BN_CTX_free(bctx); + return ret; +} + +int zkp_bin_hash2bn(const unsigned char *data, size_t len, BIGNUM *r) +{ + int ret = 0; + unsigned char hash_res[SHA256_DIGEST_LENGTH]; + + if (data == NULL || len <= 0 || r == NULL) + return ret; + + if (!SHA256(data, len, hash_res)) + goto end; + + if (!BN_bin2bn(hash_res, SHA256_DIGEST_LENGTH, r)) + goto end; + + ret = 1; +end: + return ret; +} + +int zkp_next_power_of_two(int num) +{ + int next_power_of_2 = 1; + + while(next_power_of_2 < num) { + next_power_of_2 <<= 1; + } + + return next_power_of_2; +} + +int zkp_is_power_of_two(int num) +{ + return (num != 0) && ((num & (num - 1)) == 0); +} + +int zkp_floor_log2(int x) +{ + int result = 0; + + while (x > 1) { + x >>= 1; + result++; + } + + return result; +} + +int zkp_inner_product(BIGNUM *r, int num, const BIGNUM *a[], const BIGNUM *b[], + const BIGNUM *order, BN_CTX *bn_ctx) +{ + int ret = 0, i; + BN_CTX *ctx = NULL; + BIGNUM *v, *t; + const BIGNUM *p; + + if (r == NULL || num <= 0 || (a == NULL && b == NULL)) + return 0; + + if (bn_ctx == NULL) { + if ((ctx = bn_ctx = BN_CTX_new()) == NULL) + goto end; + } + + BN_CTX_start(bn_ctx); + v = BN_CTX_get(bn_ctx); + if ((t = BN_CTX_get(bn_ctx)) == NULL) + goto end; + + BN_zero(v); + + for (i = 0; i < num; i++) { + if (a == NULL) { + p = b[i]; + } else if (b == NULL) { + p = a[i]; + } else { + if (!BN_mod_mul(t, a[i], b[i], order, bn_ctx)) + goto end; + p = t; + } + + if (!BN_mod_add(v, v, p, order, bn_ctx)) + goto end; + } + + if (!BN_copy(r, v)) + goto end; + + ret = 1; + +end: + BN_CTX_end(bn_ctx); + BN_CTX_free(ctx); + return ret; +} + +zkp_poly3_t *zkp_poly3_new(int n, const BIGNUM *order) +{ + int i; + zkp_poly3_t *ret = NULL; + + if (n < 0 || order == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(ret = OPENSSL_zalloc(sizeof(*ret)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ret->order = order; + + if (!(ret->bn_ctx = BN_CTX_new())) + goto err; + + if (n == 0) { + ret->n = 0; + return ret; + } + + if (!(ret->x0 = OPENSSL_zalloc(sizeof(*ret->x0) * n)) + || !(ret->x1 = OPENSSL_zalloc(sizeof(*ret->x1) * n)) + || !(ret->x2 = OPENSSL_zalloc(sizeof(*ret->x2) * n)) + || !(ret->x3 = OPENSSL_zalloc(sizeof(*ret->x3) * n))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + ret->n = n; + + for (i = 0; i < n; i++) { + ret->x0[i] = BN_CTX_get(ret->bn_ctx); + ret->x1[i] = BN_CTX_get(ret->bn_ctx); + ret->x2[i] = BN_CTX_get(ret->bn_ctx); + if (!(ret->x3[i] = BN_CTX_get(ret->bn_ctx))) + goto err; + + BN_zero(ret->x0[i]); + BN_zero(ret->x1[i]); + BN_zero(ret->x2[i]); + BN_zero(ret->x3[i]); + } + + return ret; +err: + zkp_poly3_free(ret); + return NULL; +} + +void zkp_poly3_free(zkp_poly3_t *poly3) +{ + if (poly3 == NULL) + return; + + BN_CTX_free(poly3->bn_ctx); + OPENSSL_free(poly3->x0); + OPENSSL_free(poly3->x1); + OPENSSL_free(poly3->x2); + OPENSSL_free(poly3->x3); + OPENSSL_free(poly3); +} + +STACK_OF(BIGNUM) *zkp_poly3_eval(zkp_poly3_t *poly3, const BIGNUM *x) +{ + int i; + BIGNUM *eval = NULL; + STACK_OF(BIGNUM) *ret = NULL; + + if (poly3 == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!(ret = sk_BIGNUM_new_reserve(NULL, poly3->n))) + return 0; + + for (i = 0; i < poly3->n; i++) { + if (!(eval = BN_CTX_get(poly3->bn_ctx))) + goto err; + if (!BN_mod_mul(eval, x, poly3->x3[i], poly3->order, poly3->bn_ctx) + || !BN_mod_add(eval, eval, poly3->x2[i], poly3->order, poly3->bn_ctx) + || !BN_mod_mul(eval, eval, x, poly3->order, poly3->bn_ctx) + || !BN_mod_add(eval, eval, poly3->x1[i], poly3->order, poly3->bn_ctx) + || !BN_mod_mul(eval, eval, x, poly3->order, poly3->bn_ctx) + || !BN_mod_add(eval, eval, poly3->x0[i], poly3->order, poly3->bn_ctx)) + goto err; + + if (sk_BIGNUM_push(ret, eval) <= 0) + goto err; + } + + return ret; +err: + sk_BIGNUM_free(ret); + return NULL; +} + +int zkp_poly3_special_inner_product(zkp_poly6_t *r, zkp_poly3_t *lhs, zkp_poly3_t *rhs) +{ + int ret = 0; + BIGNUM *t; + + if (r == NULL || lhs == NULL || rhs == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (lhs->n != rhs->n) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return 0; + } + + if (lhs->n == 0) { + BN_zero(r->t1); + BN_zero(r->t2); + BN_zero(r->t3); + BN_zero(r->t4); + BN_zero(r->t5); + BN_zero(r->t6); + return 1; + } + + BN_CTX_start(r->bn_ctx); + + if (!(t = BN_CTX_get(r->bn_ctx))) + goto err; + + if (!zkp_inner_product(r->t1, lhs->n, (const BIGNUM **)lhs->x1, + (const BIGNUM **)rhs->x0, r->order, r->bn_ctx) + || !zkp_inner_product(r->t2, lhs->n, (const BIGNUM **)lhs->x1, + (const BIGNUM **)rhs->x1, r->order, r->bn_ctx) + || !zkp_inner_product(t, lhs->n, (const BIGNUM **)lhs->x2, + (const BIGNUM **)rhs->x0, r->order, r->bn_ctx) + || !BN_mod_add(r->t2, r->t2, t, r->order, r->bn_ctx) + || !zkp_inner_product(r->t3, lhs->n, (const BIGNUM **)lhs->x2, + (const BIGNUM **)rhs->x1, r->order, r->bn_ctx) + || !zkp_inner_product(t, lhs->n, (const BIGNUM **)lhs->x3, + (const BIGNUM **)rhs->x0, r->order, r->bn_ctx) + || !BN_mod_add(r->t3, r->t3, t, r->order, r->bn_ctx) + || !zkp_inner_product(r->t4, lhs->n, (const BIGNUM **)lhs->x1, + (const BIGNUM **)rhs->x3, r->order, r->bn_ctx) + || !zkp_inner_product(t, lhs->n, (const BIGNUM **)lhs->x3, + (const BIGNUM **)rhs->x1, r->order, r->bn_ctx) + || !BN_mod_add(r->t4, r->t4, t, r->order, r->bn_ctx) + || !zkp_inner_product(r->t5, lhs->n, (const BIGNUM **)lhs->x2, + (const BIGNUM **)rhs->x3, r->order, r->bn_ctx) + || !zkp_inner_product(r->t6, lhs->n, (const BIGNUM **)lhs->x3, + (const BIGNUM **)rhs->x3, r->order, r->bn_ctx)) + goto err; + + ret = 1; + +err: + BN_CTX_end(r->bn_ctx); + return ret; +} + +zkp_poly6_t *zkp_poly6_new(const BIGNUM *order) +{ + zkp_poly6_t *ret = NULL; + + if (order == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(ret = OPENSSL_zalloc(sizeof(zkp_poly6_t)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!(ret->bn_ctx = BN_CTX_new())) + goto err; + + ret->t1 = BN_CTX_get(ret->bn_ctx); + ret->t2 = BN_CTX_get(ret->bn_ctx); + ret->t3 = BN_CTX_get(ret->bn_ctx); + ret->t4 = BN_CTX_get(ret->bn_ctx); + ret->t5 = BN_CTX_get(ret->bn_ctx); + ret->t6 = BN_CTX_get(ret->bn_ctx); + if (ret->t6 == NULL) + goto err; + + ret->order = order; + return ret; +err: + zkp_poly6_free(ret); + return NULL; +} + +void zkp_poly6_free(zkp_poly6_t *poly6) +{ + if (poly6 == NULL) + return; + + BN_CTX_free(poly6->bn_ctx); + OPENSSL_free(poly6); +} + +int zkp_poly6_eval(zkp_poly6_t *poly6, const BIGNUM *x, BIGNUM *r) +{ + int ret = 0; + + if (poly6 == NULL || r == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!BN_mod_mul(r, x, poly6->t6, poly6->order, poly6->bn_ctx) + || !BN_mod_add(r, r, poly6->t5, poly6->order, poly6->bn_ctx) + || !BN_mod_mul(r, r, x, poly6->order, poly6->bn_ctx) + || !BN_mod_add(r, r, poly6->t4, poly6->order, poly6->bn_ctx) + || !BN_mod_mul(r, r, x, poly6->order, poly6->bn_ctx) + || !BN_mod_add(r, r, poly6->t3, poly6->order, poly6->bn_ctx) + || !BN_mod_mul(r, r, x, poly6->order, poly6->bn_ctx) + || !BN_mod_add(r, r, poly6->t2, poly6->order, poly6->bn_ctx) + || !BN_mod_mul(r, r, x, poly6->order, poly6->bn_ctx) + || !BN_mod_add(r, r, poly6->t1, poly6->order, poly6->bn_ctx) + || !BN_mod_mul(r, r, x, poly6->order, poly6->bn_ctx)) + goto err; + + ret = 1; +err: + return ret; +} + +zkp_poly_points_t *zkp_poly_points_new(int capacity) +{ + zkp_poly_points_t *ret = NULL; + + if (capacity <= 0) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(ret = OPENSSL_zalloc(sizeof(*ret)))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!(ret->points = OPENSSL_zalloc(sizeof(*ret->points) * capacity)) + || !(ret->scalars = OPENSSL_zalloc(sizeof(*ret->scalars) * capacity))) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + + ret->num = 0; + ret->capacity = capacity; + + return ret; +err: + zkp_poly_points_free(ret); + return NULL; +} + +void zkp_poly_points_free(zkp_poly_points_t *ps) +{ + if (ps == NULL) + return; + + OPENSSL_free(ps->points); + OPENSSL_free(ps->scalars); + OPENSSL_free(ps); +} + +void zkp_poly_points_reset(zkp_poly_points_t *ps) +{ + if (ps == NULL || ps->num == 0) + return; + + memset(ps->points, 0, sizeof(*ps->points) * ps->num); + memset(ps->scalars, 0, sizeof(*ps->scalars) * ps->num); + ps->num = 0; +} + +int zkp_poly_points_append(zkp_poly_points_t *ps, EC_POINT *point, BIGNUM *scalar) +{ + if (ps == NULL || point == NULL || scalar == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (ps->num >= ps->capacity) + return 0; + + ps->points[ps->num] = point; + ps->scalars[ps->num] = scalar; + ps->num++; + + return 1; +} + +int zkp_poly_points_mul(zkp_poly_points_t *ps, EC_POINT *r, BIGNUM *scalar, + const EC_GROUP *group, BN_CTX *bn_ctx) +{ + if (ps == NULL || r == NULL || group == NULL || bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + return EC_POINTs_mul(group, r, scalar, ps->num, (const EC_POINT **)ps->points, + (const BIGNUM **)ps->scalars, bn_ctx); +} + +int zkp_bignum_encode(BIGNUM *bn, unsigned char *out, int bn_len) +{ + unsigned char *p = out; + + if (bn == NULL) + return 0; + + *p++ = BN_is_negative(bn) ? '-' : '+'; + + if (!BN_bn2binpad(bn, p, bn_len)) + goto end; + + p += bn_len; + +end: + return p - out; +} + +BIGNUM *zkp_bignum_decode(const unsigned char *in, int *len, int bn_len) +{ + int neg; + unsigned char *p = (unsigned char *)in; + BIGNUM *b = NULL; + + if (in == NULL) + return NULL; + + b = BN_new(); + if (b == NULL) + return NULL; + + neg = *p++ == '-' ? 1 : 0; + + if (!BN_bin2bn(p, bn_len, b)) + goto err; + + BN_set_negative(b, neg); + + p += bn_len; + + if (len != NULL) + *len = p - in; + + return b; +err: + BN_free(b); + return NULL; +} + +int zkp_stack_of_bignum_encode(STACK_OF(BIGNUM) *sk, unsigned char *out, + int bn_len) +{ + int i, n, *q; + unsigned char *p; + BIGNUM *b; + + n = sk ? sk_BIGNUM_num(sk) : 0; + if (out == NULL) + return sizeof(n) + n * (bn_len + 1); + + q = (int *)out; + *q++ = zkp_l2n((int)n); + p = (unsigned char *)q; + + for (i = 0; i < n; i++) { + b = sk_BIGNUM_value(sk, i); + if (b == NULL) + goto end; + + *p++ = BN_is_negative(b) ? '-' : '+'; + + if (!BN_bn2binpad(b, p, bn_len)) + goto end; + + p += bn_len; + } + +end: + return p - out; +} + +STACK_OF(BIGNUM) *zkp_stack_of_bignum_decode(const unsigned char *in, + int *len, int bn_len) +{ + unsigned char *p; + int *q = (int *)in, n, i, neg; + BIGNUM *b = NULL; + STACK_OF(BIGNUM) *ret; + + n = (int)zkp_n2l(*q); + q++; + p = (unsigned char *)q; + + if (n < 0) { + return NULL; + } + + if (!(ret = sk_BIGNUM_new_reserve(NULL, n))) + return NULL; + + for (i = 0; i < n; i++) { + b = BN_new(); + if (b == NULL) + goto err; + + neg = *p++ == '-' ? 1 : 0; + + if (!BN_bin2bn(p, (int)bn_len, b)) + goto err; + + BN_set_negative(b, neg); + + if (sk_BIGNUM_push(ret, b) <= 0) + goto err; + + p += bn_len; + } + + if (len != NULL) + *len = p - in; + + return ret; +err: + BN_free(b); + sk_BIGNUM_pop_free(ret, BN_free); + return NULL; +} + +int zkp_stack_of_point_encode(STACK_OF(EC_POINT) *sk, unsigned char *out, + const EC_GROUP *group, BN_CTX *bn_ctx) +{ + int i, n, *q; + size_t point_len; + unsigned char *p; + EC_POINT *P; + + if (sk == NULL || group == NULL) + return 0; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + n = sk_EC_POINT_num(sk); + if (out == NULL) + return sizeof(n) + n * point_len; + + q = (int *)out; + *q++ = zkp_l2n((int)n); + p = (unsigned char *)q; + + for (i = 0; i < n; i++) { + P = sk_EC_POINT_value(sk, i); + if (P == NULL) + goto end; + + if (EC_POINT_point2oct(group, P, form, p, point_len, bn_ctx) == 0) + goto end; + + p += point_len; + } + +end: + return p - out; +} + +STACK_OF(EC_POINT) *zkp_stack_of_point_decode(const unsigned char *in, int *len, + const EC_GROUP *group, + BN_CTX *bn_ctx) +{ + unsigned char *p; + int *q = (int *)in, n, i; + size_t point_len; + EC_POINT *P = NULL; + STACK_OF(EC_POINT) *ret = NULL; + + if (in == NULL || group == NULL) + return 0; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + n = (int)zkp_n2l(*q); + q++; + p = (unsigned char *)q; + + if (n < 0) { + return NULL; + } + + if (!(ret = sk_EC_POINT_new_reserve(NULL, n))) + return NULL; + + for (i = 0; i < n; i++) { + if (!(P = EC_POINT_new(group))) + goto err; + + if (!EC_POINT_oct2point(group, P, p, point_len, bn_ctx)) + goto err; + + if (sk_EC_POINT_push(ret, P) <= 0) + goto err; + + p += point_len; + } + + if (len != NULL) + *len = p - in; + + return ret; +err: + EC_POINT_free(P); + sk_EC_POINT_pop_free(ret, EC_POINT_free); + return NULL; +} + diff --git a/openssl/src/crypto/zkp/common/zkp_util.h b/openssl/src/crypto/zkp/common/zkp_util.h new file mode 100644 index 000000000..f6663e259 --- /dev/null +++ b/openssl/src/crypto/zkp/common/zkp_util.h @@ -0,0 +1,119 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_ZKP_UTIL_LOCAL_H +# define HEADER_ZKP_UTIL_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include "internal/refcount.h" + +#ifdef __bswap_constant_32 +# undef __bswap_constant_32 +#endif +#define __bswap_constant_32(x) \ + ((((uint32_t)(x) & 0xff000000u) >> 24) | \ + (((uint32_t)(x) & 0x00ff0000u) >> 8) | \ + (((uint32_t)(x) & 0x0000ff00u) << 8) | \ + (((uint32_t)(x) & 0x000000ffu) << 24)) + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define zkp_n2l(x) (x) +# define zkp_l2n(x) (x) +#else +# define zkp_n2l(x) __bswap_constant_32(x) +# define zkp_l2n(x) __bswap_constant_32(x) +#endif + +# define zkp_rand_range BN_rand_range + +STACK_OF(EC_POINT); + +typedef struct zkp_poly3_st { + int n; + const BIGNUM *order; + BN_CTX *bn_ctx; + BIGNUM **x0; + BIGNUM **x1; + BIGNUM **x2; + BIGNUM **x3; +} zkp_poly3_t; + +typedef struct zkp_poly6_st { + const BIGNUM *order; + BN_CTX *bn_ctx; + BIGNUM *t1; + BIGNUM *t2; + BIGNUM *t3; + BIGNUM *t4; + BIGNUM *t5; + BIGNUM *t6; +} zkp_poly6_t; + +typedef struct zkp_poly_points_st { + int capacity; + int num; + EC_POINT **points; + BIGNUM **scalars; +} zkp_poly_points_t; + +EC_POINT *zkp_random_ec_point_new(const EC_GROUP *group, BN_CTX *bn_ctx); +void zkp_random_ec_point_free(EC_POINT *P); +int zkp_random_bn_gen(const EC_GROUP *group, BIGNUM **r, size_t n, BN_CTX *bn_ctx); +int zkp_str2point(const EC_GROUP *group, const unsigned char *str, size_t len, + EC_POINT *r, BN_CTX *bn_ctx); +size_t zkp_point2oct(const EC_GROUP *group, const EC_POINT *P, + unsigned char *buf, BN_CTX *bn_ctx); +int zkp_point2point(const EC_GROUP *group, const EC_POINT *P, EC_POINT *H, BN_CTX *bn_ctx); +int zkp_bin_hash2bn(const unsigned char *data, size_t len, BIGNUM *r); +int zkp_next_power_of_two(int num); +int zkp_is_power_of_two(int num); +int zkp_floor_log2(int x); +int zkp_inner_product(BIGNUM *r, int num, const BIGNUM *a[], const BIGNUM *b[], + const BIGNUM *order, BN_CTX *bn_ctx); + +zkp_poly3_t *zkp_poly3_new(int n, const BIGNUM *order); +void zkp_poly3_free(zkp_poly3_t *poly3); +STACK_OF(BIGNUM) *zkp_poly3_eval(zkp_poly3_t *poly3, const BIGNUM *x); +int zkp_poly3_special_inner_product(zkp_poly6_t *r, zkp_poly3_t *lhs, zkp_poly3_t *rhs); +zkp_poly6_t *zkp_poly6_new(const BIGNUM *order); +void zkp_poly6_free(zkp_poly6_t *poly6); +int zkp_poly6_eval(zkp_poly6_t *poly6, const BIGNUM *x, BIGNUM *r); + +zkp_poly_points_t *zkp_poly_points_new(int capacity); +void zkp_poly_points_free(zkp_poly_points_t *ps); +void zkp_poly_points_reset(zkp_poly_points_t *ps); +int zkp_poly_points_append(zkp_poly_points_t *ps, EC_POINT *point, BIGNUM *scalar); +int zkp_poly_points_mul(zkp_poly_points_t *ps, EC_POINT *r, BIGNUM *scalar, + const EC_GROUP *group, BN_CTX *bn_ctx); + +int zkp_bignum_encode(BIGNUM *bn, unsigned char *out, int bn_len); +BIGNUM *zkp_bignum_decode(const unsigned char *in, int *len, int bn_len); +int zkp_stack_of_bignum_encode(STACK_OF(BIGNUM) *sk, unsigned char *out, + int bn_len); +STACK_OF(BIGNUM) *zkp_stack_of_bignum_decode(const unsigned char *in, + int *len, int bn_len); +int zkp_stack_of_point_encode(STACK_OF(EC_POINT) *sk, unsigned char *out, + const EC_GROUP *group, BN_CTX *bn_ctx); +STACK_OF(EC_POINT) *zkp_stack_of_point_decode(const unsigned char *in, int *len, + const EC_GROUP *group, + BN_CTX *bn_ctx); + +# ifdef __cplusplus +} +# endif + +#endif diff --git a/openssl/src/crypto/zkp/gadget/zkp_range_proof.c b/openssl/src/crypto/zkp/gadget/zkp_range_proof.c new file mode 100644 index 000000000..5a3235086 --- /dev/null +++ b/openssl/src/crypto/zkp/gadget/zkp_range_proof.c @@ -0,0 +1,623 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "zkp_range_proof.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(BP_VARIABLE) + +static int zkp_bp_range_witness_adjust(BP_WITNESS *witness, int left_bound_bits, + int right_bound_bits, int range_bits, + int is_prove, const EC_GROUP *group) +{ + int ret = 0; + BIGNUM *bn1, *bn2, *bn_left_bound, *bn_right_bound, *bn_range, *bn_delta, *v0, *v1; + BN_CTX *bn_ctx = NULL; + BP_VARIABLE *var0, *var1; + EC_POINT *P = NULL; + zkp_poly_points_t *poly = NULL; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + return 0; + + bn1 = BN_CTX_get(bn_ctx); + bn2 = BN_CTX_get(bn_ctx); + bn_left_bound = BN_CTX_get(bn_ctx); + bn_right_bound = BN_CTX_get(bn_ctx); + bn_range = BN_CTX_get(bn_ctx); + bn_delta = BN_CTX_get(bn_ctx); + if (bn_delta == NULL) + goto err; + + BN_one(bn1); + BN_set_word(bn2, 2); + BN_set_word(bn_left_bound, left_bound_bits); + BN_set_word(bn_right_bound, right_bound_bits); + BN_set_word(bn_range, range_bits); + + if (!BN_exp(bn_left_bound, bn2, bn_left_bound, bn_ctx) + || !BN_exp(bn_right_bound, bn2, bn_right_bound, bn_ctx) + || !BN_exp(bn_range, bn2, bn_range, bn_ctx) + || !BN_sub(bn_delta, bn_range, bn_right_bound)) + goto err; + + if (is_prove == 1) { + v0 = sk_BIGNUM_value(witness->sk_v, 0); + v1 = sk_BIGNUM_value(witness->sk_v, 1); + + if (!BN_sub(v0, v0, bn_left_bound) || !BN_add(v1, v1, bn_delta)) + goto err; + } + + P = EC_POINT_new(group); + if (P == NULL) + goto err; + + var0 = sk_BP_VARIABLE_value(witness->sk_V, 0); + var1 = sk_BP_VARIABLE_value(witness->sk_V, 1); + + BN_set_negative(bn_left_bound, 1); + + if (!(poly = zkp_poly_points_new(2))) + goto err; + + if (!zkp_poly_points_append(poly, var0->point, bn1) + || !zkp_poly_points_append(poly, witness->H, bn_left_bound)) + goto err; + + if (!zkp_poly_points_mul(poly, var0->point, NULL, group, bn_ctx)) + goto err; + + zkp_poly_points_reset(poly); + + if (!zkp_poly_points_append(poly, var1->point, bn1) + || !zkp_poly_points_append(poly, witness->H, bn_delta)) + goto err; + + if (!zkp_poly_points_mul(poly, var1->point, NULL, group, bn_ctx)) + goto err; + + ret = 1; +err: + zkp_poly_points_free(poly); + EC_POINT_free(P); + BN_CTX_free(bn_ctx); + return ret; +} + +ZKP_RANGE_PUB_PARAM *ZKP_RANGE_PUB_PARAM_raw_new(BP_PUB_PARAM *bp_pp) +{ + ZKP_RANGE_PUB_PARAM *pp = NULL; + + if (bp_pp == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + pp = OPENSSL_zalloc(sizeof(*pp)); + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!BP_PUB_PARAM_up_ref(bp_pp)) + goto err; + + pp->bp_pp = bp_pp; + + pp->references = 1; + if ((pp->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + return pp; +err: + ZKP_RANGE_PUB_PARAM_free(pp); + return NULL; +} + +ZKP_RANGE_PUB_PARAM *ZKP_RANGE_PUB_PARAM_new(const EC_GROUP *group, int max_bits) +{ + BP_PUB_PARAM *bp_pp = NULL; + ZKP_RANGE_PUB_PARAM *pp = NULL; + + if (group == NULL || max_bits > 64) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(bp_pp = BP_PUB_PARAM_new(group, max_bits, 2))) + return NULL; + + pp = ZKP_RANGE_PUB_PARAM_raw_new(bp_pp); + BP_PUB_PARAM_free(bp_pp); + return pp; +} + +void ZKP_RANGE_PUB_PARAM_free(ZKP_RANGE_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return; + + CRYPTO_DOWN_REF(&pp->references, &ref, pp->lock); + REF_PRINT_COUNT("ZKP_RANGE_PUB_PARAM", pp); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + BP_PUB_PARAM_down_ref(pp->bp_pp); + OPENSSL_clear_free((void *)pp, sizeof(*pp)); +} + +int ZKP_RANGE_PUB_PARAM_up_ref(ZKP_RANGE_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return 0; + + if (CRYPTO_UP_REF(&pp->references, &ref, pp->lock) <= 0) + return 0; + + REF_PRINT_COUNT("ZKP_RANGE_PUB_PARAM", pp); + REF_ASSERT_ISNT(ref < 2); + return ((ref > 1) ? 1 : 0); +} + +int ZKP_RANGE_PUB_PARAM_down_ref(ZKP_RANGE_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return 0; + + if (CRYPTO_DOWN_REF(&pp->references, &ref, pp->lock) <= 0) + return 0; + + REF_PRINT_COUNT("ZKP_RANGE_PUB_PARAM", pp); + REF_ASSERT_ISNT(ref < 0); + return ((ref > 0) ? 1 : 0); +} + +ZKP_RANGE_WITNESS *ZKP_RANGE_WITNESS_new(const ZKP_RANGE_PUB_PARAM *pp, + const BIGNUM *r, const BIGNUM *v) + +{ + ZKP_RANGE_WITNESS *witness = NULL; + + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(witness = OPENSSL_zalloc(sizeof(*witness)))) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!(witness->order = BN_dup(EC_GROUP_get0_order(pp->bp_pp->group))) + || !(witness->r = BN_new()) + || !(witness->v = BN_new())) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (r != NULL) { + if (!BN_copy(witness->r, r)) + goto err; + } else { + zkp_rand_range(witness->r, witness->order); + } + + if (v != NULL && !BN_copy(witness->v, v)) + goto err; + + witness->references = 1; + if ((witness->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + goto err; + } + + return witness; +err: + ZKP_RANGE_WITNESS_free(witness); + return NULL; +} + +void ZKP_RANGE_WITNESS_free(ZKP_RANGE_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return; + + CRYPTO_DOWN_REF(&witness->references, &ref, witness->lock); + REF_PRINT_COUNT("ZKP_RANGE_WITNESS", witness); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + BN_free(witness->order); + BN_free(witness->r); + BN_free(witness->v); + CRYPTO_THREAD_lock_free(witness->lock); + OPENSSL_free(witness); +} + +int ZKP_RANGE_WITNESS_up_ref(ZKP_RANGE_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return 0; + + if (CRYPTO_UP_REF(&witness->references, &ref, witness->lock) <= 0) + return 0; + + REF_PRINT_COUNT("ZKP_RANGE_WITNESS", witness); + REF_ASSERT_ISNT(ref < 2); + return ((ref > 1) ? 1 : 0); +} + +int ZKP_RANGE_WITNESS_down_ref(ZKP_RANGE_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return 0; + + if (CRYPTO_DOWN_REF(&witness->references, &ref, witness->lock) <= 0) + return 0; + + REF_PRINT_COUNT("ZKP_RANGE_WITNESS", witness); + REF_ASSERT_ISNT(ref < 0); + return ((ref > 0) ? 1 : 0); +} + +ZKP_RANGE_CTX *ZKP_RANGE_CTX_raw_new(ZKP_TRANSCRIPT *transcript, + ZKP_RANGE_PUB_PARAM *pp, + ZKP_RANGE_WITNESS *witness, + const EC_POINT *pk, + EC_ELGAMAL_CTX *enc_ctx, + EC_ELGAMAL_CIPHERTEXT *enc_ct) +{ + ZKP_RANGE_CTX *ctx = NULL; + + if (transcript == NULL || pp == NULL || witness == NULL || pk == NULL + || enc_ct == NULL || enc_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (enc_ctx->flag != EC_ELGAMAL_FLAG_TWISTED) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ctx->transcript = transcript; + + if (!ZKP_RANGE_PUB_PARAM_up_ref(pp)) + goto err; + + ctx->pp = pp; + + if (!ZKP_RANGE_WITNESS_up_ref(witness)) + goto err; + + ctx->witness = witness; + + ctx->PK = EC_POINT_dup(pk, pp->bp_pp->group); + if (ctx->PK == NULL) + goto err; + + ctx->enc_ctx = EC_ELGAMAL_CTX_dup(enc_ctx); + ctx->enc_ct = EC_ELGAMAL_CIPHERTEXT_dup(enc_ct, pp->bp_pp->group); + if (ctx->enc_ctx == NULL || ctx->enc_ct == NULL) + goto err; + + return ctx; + +err: + ZKP_RANGE_CTX_free(ctx); + return NULL; +} + +ZKP_RANGE_CTX *ZKP_RANGE_CTX_new(ZKP_TRANSCRIPT *transcript, + ZKP_RANGE_PUB_PARAM *pp, + ZKP_RANGE_WITNESS *witness, + EC_KEY *key) +{ + ZKP_RANGE_CTX *ret = NULL; + EC_ELGAMAL_CTX *enc_ctx = NULL; + EC_ELGAMAL_CIPHERTEXT *enc_ct = NULL; + + if (transcript == NULL || pp == NULL || witness == NULL || key == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(enc_ctx = EC_ELGAMAL_CTX_new(key, pp->bp_pp->H, EC_ELGAMAL_FLAG_TWISTED))) + goto err; + + if (!(enc_ct = EC_ELGAMAL_CIPHERTEXT_new(enc_ctx))) + goto err; + + if (!EC_ELGAMAL_bn_encrypt(enc_ctx, enc_ct, witness->v, witness->r)) + goto err; + + ret = ZKP_RANGE_CTX_raw_new(transcript, pp, witness, key->pub_key, enc_ctx, enc_ct); +err: + EC_ELGAMAL_CIPHERTEXT_free(enc_ct); + EC_ELGAMAL_CTX_free(enc_ctx); + return ret; +} + +void ZKP_RANGE_CTX_free(ZKP_RANGE_CTX *ctx) +{ + if (ctx == NULL) + return; + + ZKP_RANGE_PUB_PARAM_down_ref(ctx->pp); + ZKP_RANGE_WITNESS_down_ref(ctx->witness); + EC_ELGAMAL_CIPHERTEXT_free(ctx->enc_ct); + EC_ELGAMAL_CTX_free(ctx->enc_ctx); + OPENSSL_clear_free((void *)ctx, sizeof(*ctx)); +} + +ZKP_RANGE_PROOF *ZKP_RANGE_PROOF_new(void) +{ + ZKP_RANGE_PROOF *proof = NULL; + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_MALLOC_FAILURE); + return NULL; + } + + return proof; +} + +void ZKP_RANGE_PROOF_free(ZKP_RANGE_PROOF *proof) +{ + if (proof == NULL) + return; + + NIZK_PLAINTEXT_KNOWLEDGE_PROOF_free(proof->ptke_proof); + BP_RANGE_PROOF_free(proof->bp_proof); +} + +ZKP_RANGE_PROOF *ZKP_RANGE_PROOF_prove(ZKP_RANGE_CTX *ctx, int left_bound_bits, + int right_bound_bits) +{ + EC_GROUP *group; + const EC_POINT *G, *H; + ZKP_RANGE_PUB_PARAM *pp; + ZKP_RANGE_WITNESS *witness; + ZKP_RANGE_PROOF *proof = NULL, *ret = NULL; + NIZK_PUB_PARAM *nizk_pp = NULL; + NIZK_WITNESS *nizk_witness = NULL; + NIZK_PLAINTEXT_KNOWLEDGE_CTX *ptke_ctx = NULL; + BIGNUM *v1 = NULL, *v2 = NULL; + BP_WITNESS *bp_witness = NULL; + BP_VARIABLE *bp_var1 = NULL, *bp_var2 = NULL; + BP_RANGE_CTX *bp_ctx = NULL; + + if (ctx == NULL || ctx->pp == NULL || ctx->pp->bp_pp == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + pp = ctx->pp; + witness = ctx->witness; + group = pp->bp_pp->group; + G = EC_GROUP_get0_generator(group); + H = pp->bp_pp->H; + + if (left_bound_bits < 0 + || right_bound_bits < 0 + || left_bound_bits > right_bound_bits + || right_bound_bits > pp->bp_pp->gens_capacity) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + proof = ZKP_RANGE_PROOF_new(); + if (proof == NULL) + return NULL; + + nizk_pp = NIZK_PUB_PARAM_new(group, G, H); + if (nizk_pp == NULL) + goto err; + + nizk_witness = NIZK_WITNESS_new(nizk_pp, witness->r, witness->v); + if (nizk_witness == NULL) + goto err; + + ptke_ctx = NIZK_PLAINTEXT_KNOWLEDGE_CTX_new(ctx->transcript, nizk_pp, + nizk_witness, ctx->PK, ctx->enc_ct); + if (ptke_ctx == NULL) + goto err; + + proof->ptke_proof = NIZK_PLAINTEXT_KNOWLEDGE_PROOF_prove(ptke_ctx); + if (proof->ptke_proof == NULL) { + ERR_raise(ERR_LIB_ZKP, ZKP_R_RANGE_PROVE_FAILED); + goto err; + } + + bp_witness = BP_WITNESS_new(pp->bp_pp); + if (bp_witness == NULL) + goto err; + + bp_var1 = BP_VARIABLE_new(NULL, ctx->enc_ct->C2, group); + bp_var2 = BP_VARIABLE_new(NULL, ctx->enc_ct->C2, group); + if (bp_var1 == NULL || bp_var2 == NULL) + goto err; + + v1 = BN_dup(ctx->witness->v); + v2 = BN_dup(ctx->witness->v); + if (v1 == NULL || v2 == NULL) + goto err; + + if (sk_BIGNUM_push(bp_witness->sk_r, ctx->witness->r) <= 0 + || sk_BIGNUM_push(bp_witness->sk_r, ctx->witness->r) <= 1 + || sk_BIGNUM_push(bp_witness->sk_v, v1) <= 0 + || sk_BIGNUM_push(bp_witness->sk_v, v2) <= 1 + || sk_BP_VARIABLE_push(bp_witness->sk_V, bp_var1) <= 0 + || sk_BP_VARIABLE_push(bp_witness->sk_V, bp_var2) <= 1) + goto err; + + if (!zkp_bp_range_witness_adjust(bp_witness, left_bound_bits, right_bound_bits, + pp->bp_pp->gens_capacity, 1, group)) + goto err; + + bp_ctx = BP_RANGE_CTX_new(pp->bp_pp, bp_witness, ctx->transcript); + if (bp_ctx == NULL) + goto err; + + proof->bp_proof = BP_RANGE_PROOF_new_prove(bp_ctx); + if (proof->bp_proof == NULL) { + ERR_raise(ERR_LIB_ZKP, ZKP_R_RANGE_PROVE_FAILED); + goto err; + } + + ret = proof; + proof = NULL; + +err: + BP_RANGE_CTX_free(bp_ctx); + BN_free(v1); + BN_free(v2); + BP_VARIABLE_free(bp_var1); + BP_VARIABLE_free(bp_var2); + if (bp_witness != NULL) { + sk_BIGNUM_zero(bp_witness->sk_r); + sk_BIGNUM_zero(bp_witness->sk_v); + sk_BP_VARIABLE_zero(bp_witness->sk_V); + BP_WITNESS_free(bp_witness); + } + NIZK_PLAINTEXT_KNOWLEDGE_CTX_free(ptke_ctx); + NIZK_WITNESS_free(nizk_witness); + NIZK_PUB_PARAM_free(nizk_pp); + ZKP_RANGE_PROOF_free(proof); + return ret; +} + +int ZKP_RANGE_PROOF_verify(ZKP_RANGE_CTX *ctx, ZKP_RANGE_PROOF *proof, + int left_bound_bits, int right_bound_bits) +{ + int ret = 0; + EC_GROUP *group; + const EC_POINT *G, *H; + ZKP_RANGE_PUB_PARAM *pp; + NIZK_PUB_PARAM *nizk_pp = NULL; + NIZK_WITNESS *nizk_witness = NULL; + NIZK_PLAINTEXT_KNOWLEDGE_CTX *ptke_ctx = NULL; + BP_WITNESS *bp_witness = NULL; + BP_VARIABLE *bp_var1 = NULL, *bp_var2 = NULL; + BP_RANGE_CTX *bp_ctx = NULL; + + if (ctx == NULL || ctx->pp == NULL || ctx->pp->bp_pp == NULL + || proof->ptke_proof == NULL || proof->bp_proof == NULL) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + pp = ctx->pp; + group = pp->bp_pp->group; + G = EC_GROUP_get0_generator(group); + H = pp->bp_pp->H; + + if (left_bound_bits < 0 + || right_bound_bits < 0 + || left_bound_bits > right_bound_bits + || right_bound_bits > pp->bp_pp->gens_capacity) { + ERR_raise(ERR_LIB_ZKP, ERR_R_PASSED_INVALID_ARGUMENT); + return ret; + } + + nizk_pp = NIZK_PUB_PARAM_new(group, G, H); + if (nizk_pp == NULL) + goto err; + + ptke_ctx = NIZK_PLAINTEXT_KNOWLEDGE_CTX_new(ctx->transcript, nizk_pp, NULL, + ctx->PK, ctx->enc_ct); + if (ptke_ctx == NULL) + goto err; + + if (!NIZK_PLAINTEXT_KNOWLEDGE_PROOF_verify(ptke_ctx, proof->ptke_proof)) { + ERR_raise(ERR_LIB_ZKP, ZKP_R_RANGE_VERIFY_FAILED); + goto err; + } + + bp_witness = BP_WITNESS_new(pp->bp_pp); + if (bp_witness == NULL) + goto err; + + bp_var1 = BP_VARIABLE_new(NULL, ctx->enc_ct->C2, group); + if (bp_var1 == NULL) + goto err; + + bp_var2 = BP_VARIABLE_new(NULL, ctx->enc_ct->C2, group); + if (bp_var2 == NULL) + goto err; + + if (sk_BP_VARIABLE_push(bp_witness->sk_V, bp_var1) <= 0 + || sk_BP_VARIABLE_push(bp_witness->sk_V, bp_var2) <= 1) + goto err; + + if (!zkp_bp_range_witness_adjust(bp_witness, left_bound_bits, right_bound_bits, + pp->bp_pp->gens_capacity, 0, group)) + goto err; + + bp_ctx = BP_RANGE_CTX_new(pp->bp_pp, bp_witness, ctx->transcript); + if (bp_ctx == NULL) + goto err; + + if (!BP_RANGE_PROOF_verify(bp_ctx, proof->bp_proof)) { + ERR_raise(ERR_LIB_ZKP, ZKP_R_RANGE_VERIFY_FAILED); + goto err; + } + + ret = 1; + +err: + BP_RANGE_CTX_free(bp_ctx); + BP_VARIABLE_free(bp_var1); + BP_VARIABLE_free(bp_var2); + + if (bp_witness != NULL) { + sk_BP_VARIABLE_zero(bp_witness->sk_V); + BP_WITNESS_free(bp_witness); + } + + NIZK_PLAINTEXT_KNOWLEDGE_CTX_free(ptke_ctx); + NIZK_WITNESS_free(nizk_witness); + NIZK_PUB_PARAM_free(nizk_pp); + return ret; +} diff --git a/openssl/src/crypto/zkp/gadget/zkp_range_proof.h b/openssl/src/crypto/zkp/gadget/zkp_range_proof.h new file mode 100644 index 000000000..478c734ec --- /dev/null +++ b/openssl/src/crypto/zkp/gadget/zkp_range_proof.h @@ -0,0 +1,58 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_ZKP_RANGE_PROOF_LOCAL_H +# define HEADER_ZKP_RANGE_PROOF_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include +# include +# include "internal/refcount.h" + +struct zkp_range_pub_param_st { + BP_PUB_PARAM *bp_pp; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +struct zkp_range_witness_st { + BIGNUM *order; + BIGNUM *r; + BIGNUM *v; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +struct zkp_range_ctx_st { + ZKP_TRANSCRIPT *transcript; + ZKP_RANGE_PUB_PARAM *pp; + ZKP_RANGE_WITNESS *witness; + EC_POINT *PK; + EC_ELGAMAL_CTX *enc_ctx; + EC_ELGAMAL_CIPHERTEXT *enc_ct; +}; + +struct zkp_range_proof_st { + NIZK_PLAINTEXT_KNOWLEDGE_PROOF *ptke_proof; + BP_RANGE_PROOF *bp_proof; +}; + +# ifdef __cplusplus +} +# endif + +#endif diff --git a/openssl/src/crypto/zkp/nizk/nizk.c b/openssl/src/crypto/zkp/nizk/nizk.c new file mode 100644 index 000000000..e725af7c0 --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk.c @@ -0,0 +1,226 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include +#include "nizk.h" + +NIZK_PUB_PARAM *NIZK_PUB_PARAM_new(const EC_GROUP *group, const EC_POINT *G, + const EC_POINT *H) +{ + NIZK_PUB_PARAM *pp = NULL; + + if (group == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + pp = OPENSSL_zalloc(sizeof(*pp)); + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + pp->group = EC_GROUP_dup(group); + pp->G = EC_POINT_dup(G ? G : EC_GROUP_get0_generator(group), group); + + if (H != NULL) { + pp->H = EC_POINT_dup(H, group); + } else { + pp->H = EC_POINT_new(group); + } + + if (pp->group == NULL || pp->G == NULL || pp->H == NULL) + goto err; + + if (H == NULL) { + if (!zkp_point2point(group, pp->G, pp->H, NULL)) + goto err; + } + + pp->references = 1; + if ((pp->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_BP, ERR_R_MALLOC_FAILURE); + goto err; + } + return pp; +err: + NIZK_PUB_PARAM_free(pp); + return NULL; +} + +void NIZK_PUB_PARAM_free(NIZK_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return; + + CRYPTO_DOWN_REF(&pp->references, &ref, pp->lock); + REF_PRINT_COUNT("NIZK_PUB_PARAM", pp); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + EC_POINT_free(pp->G); + EC_POINT_free(pp->H); + EC_GROUP_free(pp->group); + CRYPTO_THREAD_lock_free(pp->lock); + OPENSSL_clear_free((void *)pp, sizeof(*pp)); +} + +/** Increases the internal reference count of a NIZK_PUB_PARAM object. + * \param pp NIZK_PUB_PARAM object + * \return 1 on success and 0 if an error occurred. + */ +int NIZK_PUB_PARAM_up_ref(NIZK_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return 0; + + if (CRYPTO_UP_REF(&pp->references, &ref, pp->lock) <= 0) + return 0; + + REF_PRINT_COUNT("NIZK_PUB_PARAM", pp); + REF_ASSERT_ISNT(ref < 2); + return ((ref > 1) ? 1 : 0); +} + +/** Decreases the internal reference count of a NIZK_PUB_PARAM object. + * \param pp NIZK_PUB_PARAM object + * \return 1 on success and 0 if an error occurred. + */ +int NIZK_PUB_PARAM_down_ref(NIZK_PUB_PARAM *pp) +{ + int ref; + + if (pp == NULL) + return 0; + + if (CRYPTO_DOWN_REF(&pp->references, &ref, pp->lock) <= 0) + return 0; + + REF_PRINT_COUNT("NIZK_PUB_PARAM", pp); + REF_ASSERT_ISNT(ref < 0); + return ((ref > 0) ? 1 : 0); +} + +/** Creates a new NIZK_WITNESS object + * \param pp underlying NIZK_PUB_PARAM object + * \return newly created NIZK_WITNESS object or NULL in case of an error + */ +NIZK_WITNESS *NIZK_WITNESS_new(const NIZK_PUB_PARAM *pp, const BIGNUM *r, + const BIGNUM *v) +{ + NIZK_WITNESS *witness = NULL; + + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(witness = OPENSSL_zalloc(sizeof(*witness)))) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if (!(witness->order = BN_dup(EC_GROUP_get0_order(pp->group))) + || !(witness->r = BN_new()) + || !(witness->v = BN_new())) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (r != NULL) { + if (!BN_copy(witness->r, r)) + goto err; + } else { + zkp_rand_range(witness->r, witness->order); + } + + if (v != NULL && !BN_copy(witness->v, v)) + goto err; + + witness->references = 1; + if ((witness->lock = CRYPTO_THREAD_lock_new()) == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto err; + } + + return witness; +err: + NIZK_WITNESS_free(witness); + return NULL; +} + +/** Frees a NIZK_WITNESS object + * \param witness NIZK_WITNESS object to be freed + */ +void NIZK_WITNESS_free(NIZK_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return; + + CRYPTO_DOWN_REF(&witness->references, &ref, witness->lock); + REF_PRINT_COUNT("NIZK_WITNESS", witness); + if (ref > 0) + return; + REF_ASSERT_ISNT(ref < 0); + + BN_free(witness->order); + BN_free(witness->r); + BN_free(witness->v); + CRYPTO_THREAD_lock_free(witness->lock); + OPENSSL_free(witness); +} + +/** Increases the internal reference count of a NIZK_WITNESS object. + * \param witness NIZK_WITNESS object + * \return 1 on success and 0 if an error occurred. + */ +int NIZK_WITNESS_up_ref(NIZK_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return 0; + + if (CRYPTO_UP_REF(&witness->references, &ref, witness->lock) <= 0) + return 0; + + REF_PRINT_COUNT("NIZK_WITNESS", witness); + REF_ASSERT_ISNT(ref < 2); + return ((ref > 1) ? 1 : 0); +} + +/** Decreases the internal reference count of a NIZK_WITNESS object. + * \param witness NIZK_WITNESS object + * \return 1 on success and 0 if an error occurred. + */ +int NIZK_WITNESS_down_ref(NIZK_WITNESS *witness) +{ + int ref; + + if (witness == NULL) + return 0; + + if (CRYPTO_DOWN_REF(&witness->references, &ref, witness->lock) <= 0) + return 0; + + REF_PRINT_COUNT("NIZK_WITNESS", witness); + REF_ASSERT_ISNT(ref < 0); + return ((ref > 0) ? 1 : 0); +} diff --git a/openssl/src/crypto/zkp/nizk/nizk.h b/openssl/src/crypto/zkp/nizk/nizk.h new file mode 100644 index 000000000..ab1f3d220 --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk.h @@ -0,0 +1,44 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_NIZK_LOCAL_H +# define HEADER_NIZK_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include "internal/refcount.h" + +struct nizk_pub_param_st { + EC_GROUP *group; + EC_POINT *G; + EC_POINT *H; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +struct nizk_witness_st { + BIGNUM *order; + BIGNUM *r; + BIGNUM *v; + CRYPTO_RWLOCK *lock; + CRYPTO_REF_COUNT references; +}; + +# ifdef __cplusplus +} +# endif + +#endif diff --git a/openssl/src/crypto/zkp/nizk/nizk_dlog_equality.c b/openssl/src/crypto/zkp/nizk/nizk_dlog_equality.c new file mode 100644 index 000000000..7169298ca --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_dlog_equality.c @@ -0,0 +1,261 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "nizk_dlog_equality.h" + +NIZK_DLOG_EQUALITY_CTX *NIZK_DLOG_EQUALITY_CTX_new(ZKP_TRANSCRIPT *transcript, + NIZK_PUB_PARAM *pp, + NIZK_WITNESS *witness, + const EC_POINT *G, + const EC_POINT *H) +{ + NIZK_DLOG_EQUALITY_CTX *ctx = NULL; + + if (pp == NULL || transcript == NULL || G == NULL || H == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ctx->transcript = transcript; + + if (!NIZK_PUB_PARAM_up_ref(pp)) + goto err; + + ctx->pp = pp; + + if (witness != NULL) { + if (!NIZK_WITNESS_up_ref(witness)) + goto err; + + ctx->witness = witness; + } + + if (!(ctx->G = EC_POINT_dup(G, pp->group)) + || !(ctx->H = EC_POINT_dup(H, pp->group))) + goto err; + + return ctx; + +err: + NIZK_DLOG_EQUALITY_CTX_free(ctx); + return NULL; +} + +void NIZK_DLOG_EQUALITY_CTX_free(NIZK_DLOG_EQUALITY_CTX *ctx) +{ + if (ctx == NULL) + return; + + NIZK_PUB_PARAM_down_ref(ctx->pp); + NIZK_WITNESS_down_ref(ctx->witness); + + EC_POINT_free(ctx->G); + EC_POINT_free(ctx->H); + + OPENSSL_clear_free((void *)ctx, sizeof(*ctx)); +} + +NIZK_DLOG_EQUALITY_PROOF *NIZK_DLOG_EQUALITY_PROOF_new(NIZK_DLOG_EQUALITY_CTX *ctx) +{ + NIZK_DLOG_EQUALITY_PROOF *proof = NULL; + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if ((proof->A1 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->A2 = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->z = BN_new()) == NULL) + goto err; + + EC_POINT_set_to_infinity(ctx->pp->group, proof->A1); + EC_POINT_set_to_infinity(ctx->pp->group, proof->A2); + + BN_zero(proof->z); + + return proof; +err: + NIZK_DLOG_EQUALITY_PROOF_free(proof); + return NULL; +} + +void NIZK_DLOG_EQUALITY_PROOF_free(NIZK_DLOG_EQUALITY_PROOF *proof) +{ + if (proof == NULL) + return; + + EC_POINT_free(proof->A1); + EC_POINT_free(proof->A2); + BN_free(proof->z); + OPENSSL_clear_free((void *)proof, sizeof(*proof)); +} + +NIZK_DLOG_EQUALITY_PROOF *NIZK_DLOG_EQUALITY_PROOF_prove(NIZK_DLOG_EQUALITY_CTX *ctx) +{ + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + NIZK_WITNESS *witness; + NIZK_DLOG_EQUALITY_PROOF *proof = NULL, *ret = NULL; + const BIGNUM *order; + EC_GROUP *group; + BN_CTX *bn_ctx = NULL; + BIGNUM *a, *e, *t; + + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(proof = NIZK_DLOG_EQUALITY_PROOF_new(ctx))) + return NULL; + + pp = ctx->pp; + witness = ctx->witness; + transcript = ctx->transcript; + group = pp->group; + order = EC_GROUP_get0_order(group); + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + a = BN_CTX_get(bn_ctx); + e = BN_CTX_get(bn_ctx); + t = BN_CTX_get(bn_ctx); + if (t == NULL) + goto err; + + if (!zkp_rand_range(a, order)) + goto err; + + if (!EC_POINT_mul(group, proof->A1, NULL, pp->G, a, bn_ctx) + || !EC_POINT_mul(group, proof->A2, NULL, ctx->G, a, bn_ctx)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "G1", pp->G, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "G2", ctx->G, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "H1", pp->H, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "H2", ctx->H, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A1", proof->A1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A2", proof->A2, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "e", e)) + goto err; + + if (!BN_mul(t, e, witness->v, bn_ctx) + || !BN_mod_add(proof->z, a, t, order, bn_ctx)) + goto err; + + ret = proof; + proof = NULL; +err: + BN_CTX_free(bn_ctx); + NIZK_DLOG_EQUALITY_PROOF_free(proof); + ZKP_TRANSCRIPT_reset(transcript); + return ret; +} + +int NIZK_DLOG_EQUALITY_PROOF_verify(NIZK_DLOG_EQUALITY_CTX *ctx, NIZK_DLOG_EQUALITY_PROOF *proof) +{ + int ret = 0; + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + EC_GROUP *group; + BN_CTX *bn_ctx = NULL; + BIGNUM *e, *bn1; + EC_POINT *L = NULL, *R = NULL; + zkp_poly_points_t *poly = NULL; + + if (ctx == NULL || proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + pp = ctx->pp; + transcript = ctx->transcript; + group = pp->group; + + if (!(L = EC_POINT_new(group)) || !(R = EC_POINT_new(group))) + goto err; + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + e = BN_CTX_get(bn_ctx); + bn1 = BN_CTX_get(bn_ctx); + if (bn1 == NULL) + goto err; + + BN_one(bn1); + + if (!ZKP_TRANSCRIPT_append_point(transcript, "G1", pp->G, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "G2", ctx->G, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "H1", pp->H, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "H2", ctx->H, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A1", proof->A1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A2", proof->A2, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "e", e)) + goto err; + + if (!EC_POINT_mul(group, L, NULL, pp->G, proof->z, bn_ctx)) + goto err; + + if (!(poly = zkp_poly_points_new(2))) + goto err; + + if (!zkp_poly_points_append(poly, proof->A1, bn1) + || !zkp_poly_points_append(poly, pp->H, e)) + goto err; + + if (!zkp_poly_points_mul(poly, R, NULL, group, bn_ctx)) + goto err; + + if (EC_POINT_cmp(group, L, R, bn_ctx) != 0) + goto err; + + if (!EC_POINT_mul(group, L, NULL, ctx->G, proof->z, bn_ctx)) + goto err; + + zkp_poly_points_reset(poly); + + if (!zkp_poly_points_append(poly, proof->A2, bn1) + || !zkp_poly_points_append(poly, ctx->H, e)) + goto err; + + if (!zkp_poly_points_mul(poly, R, NULL, group, bn_ctx)) + goto err; + + if (EC_POINT_cmp(group, L, R, bn_ctx) != 0) + goto err; + + ret = 1; +err: + EC_POINT_free(L); + EC_POINT_free(R); + zkp_poly_points_free(poly); + ZKP_TRANSCRIPT_reset(transcript); + return ret; +} + diff --git a/openssl/src/crypto/zkp/nizk/nizk_dlog_equality.h b/openssl/src/crypto/zkp/nizk/nizk_dlog_equality.h new file mode 100644 index 000000000..df9cbe72d --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_dlog_equality.h @@ -0,0 +1,45 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_NIZK_DLOG_EQUALITY_LOCAL_H +# define HEADER_NIZK_DLOG_EQUALITY_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include "internal/refcount.h" +# include "nizk.h" + +struct nizk_dlog_equality_ctx_st { + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + NIZK_WITNESS *witness; + EC_POINT *G; + EC_POINT *H; +}; + +struct nizk_dlog_equality_proof_st { + EC_POINT *A1; + EC_POINT *A2; + BIGNUM *z; +}; + +# ifdef __cplusplus +} +# endif + +#endif + + diff --git a/openssl/src/crypto/zkp/nizk/nizk_dlog_knowledge.c b/openssl/src/crypto/zkp/nizk/nizk_dlog_knowledge.c new file mode 100644 index 000000000..ec92e81c0 --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_dlog_knowledge.c @@ -0,0 +1,233 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "nizk_dlog_knowledge.h" + +NIZK_DLOG_KNOWLEDGE_CTX *NIZK_DLOG_KNOWLEDGE_CTX_new(ZKP_TRANSCRIPT *transcript, + NIZK_PUB_PARAM *pp, + NIZK_WITNESS *witness) +{ + NIZK_DLOG_KNOWLEDGE_CTX *ctx = NULL; + + if (pp == NULL || transcript == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ctx->transcript = transcript; + + if (!NIZK_PUB_PARAM_up_ref(pp)) + goto err; + + ctx->pp = pp; + + if (witness != NULL) { + if (!NIZK_WITNESS_up_ref(witness)) + goto err; + + ctx->witness = witness; + } + + return ctx; + +err: + NIZK_DLOG_KNOWLEDGE_CTX_free(ctx); + return NULL; +} + +void NIZK_DLOG_KNOWLEDGE_CTX_free(NIZK_DLOG_KNOWLEDGE_CTX *ctx) +{ + if (ctx == NULL) + return; + + NIZK_PUB_PARAM_down_ref(ctx->pp); + NIZK_WITNESS_down_ref(ctx->witness); + + OPENSSL_clear_free((void *)ctx, sizeof(*ctx)); +} + +NIZK_DLOG_KNOWLEDGE_PROOF *NIZK_DLOG_KNOWLEDGE_PROOF_new(NIZK_DLOG_KNOWLEDGE_CTX *ctx) +{ + NIZK_DLOG_KNOWLEDGE_PROOF *proof = NULL; + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if ((proof->A = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->z = BN_new()) == NULL) + goto err; + + EC_POINT_set_to_infinity(ctx->pp->group, proof->A); + + BN_zero(proof->z); + + return proof; +err: + NIZK_DLOG_KNOWLEDGE_PROOF_free(proof); + return NULL; +} + +void NIZK_DLOG_KNOWLEDGE_PROOF_free(NIZK_DLOG_KNOWLEDGE_PROOF *proof) +{ + if (proof == NULL) + return; + + EC_POINT_free(proof->A); + BN_free(proof->z); + OPENSSL_clear_free((void *)proof, sizeof(*proof)); +} + +NIZK_DLOG_KNOWLEDGE_PROOF *NIZK_DLOG_KNOWLEDGE_PROOF_prove(NIZK_DLOG_KNOWLEDGE_CTX *ctx) +{ + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + NIZK_WITNESS *witness; + NIZK_DLOG_KNOWLEDGE_PROOF *proof = NULL, *ret = NULL; + const BIGNUM *order; + EC_GROUP *group; + BN_CTX *bn_ctx = NULL; + BIGNUM *a, *e, *t; + + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(proof = NIZK_DLOG_KNOWLEDGE_PROOF_new(ctx))) + return NULL; + + pp = ctx->pp; + witness = ctx->witness; + transcript = ctx->transcript; + group = pp->group; + order = EC_GROUP_get0_order(group); + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + a = BN_CTX_get(bn_ctx); + e = BN_CTX_get(bn_ctx); + t = BN_CTX_get(bn_ctx); + if (t == NULL) + goto err; + + if (!zkp_rand_range(a, order)) + goto err; + + if (!EC_POINT_mul(group, proof->A, NULL, pp->G, a, bn_ctx)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "G", pp->G, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "H", pp->H, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A", proof->A, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "e", e)) + goto err; + + if (!BN_mul(t, e, witness->v, bn_ctx) + || !BN_mod_add(proof->z, a, t, order, bn_ctx)) + goto err; + + ret = proof; + proof = NULL; +err: + BN_CTX_free(bn_ctx); + NIZK_DLOG_KNOWLEDGE_PROOF_free(proof); + ZKP_TRANSCRIPT_reset(transcript); + return ret; +} + +int NIZK_DLOG_KNOWLEDGE_PROOF_verify(NIZK_DLOG_KNOWLEDGE_CTX *ctx, + NIZK_DLOG_KNOWLEDGE_PROOF *proof) +{ + int ret = 0; + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + EC_GROUP *group; + BN_CTX *bn_ctx = NULL; + BIGNUM *e, *bn1, *bn_1; + EC_POINT *L = NULL, *R = NULL; + zkp_poly_points_t *poly = NULL; + + if (ctx == NULL || proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + pp = ctx->pp; + transcript = ctx->transcript; + group = pp->group; + + if (!(L = EC_POINT_new(group)) || !(R = EC_POINT_new(group))) + goto err; + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + e = BN_CTX_get(bn_ctx); + bn1 = BN_CTX_get(bn_ctx); + bn_1 = BN_CTX_get(bn_ctx); + if (bn_1 == NULL) + goto err; + + BN_one(bn1); + BN_one(bn_1); + BN_set_negative(bn_1, 1); + + if (!ZKP_TRANSCRIPT_append_point(transcript, "G", pp->G, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "H", pp->H, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A", proof->A, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "e", e)) + goto err; + + if (!EC_POINT_mul(group, L, NULL, pp->G, proof->z, bn_ctx)) + goto err; + + if (!(poly = zkp_poly_points_new(3))) + goto err; + + if (!zkp_poly_points_append(poly, proof->A, bn1) + || !zkp_poly_points_append(poly, pp->H, e) + || !zkp_poly_points_append(poly, L, bn_1)) + goto err; + + if (!zkp_poly_points_mul(poly, R, NULL, group, bn_ctx)) + goto err; + + if (!EC_POINT_is_at_infinity(group, R)) + goto err; + + ret = 1; +err: + EC_POINT_free(L); + EC_POINT_free(R); + zkp_poly_points_free(poly); + ZKP_TRANSCRIPT_reset(transcript); + return ret; +} + + diff --git a/openssl/src/crypto/zkp/nizk/nizk_dlog_knowledge.h b/openssl/src/crypto/zkp/nizk/nizk_dlog_knowledge.h new file mode 100644 index 000000000..e2942db6e --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_dlog_knowledge.h @@ -0,0 +1,42 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_NIZK_DLOG_KNOWLEDGE_LOCAL_H +# define HEADER_NIZK_DLOG_KNOWLEDGE_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include "internal/refcount.h" +# include "nizk.h" + +struct nizk_dlog_knowledge_ctx_st { + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + NIZK_WITNESS *witness; +}; + +struct nizk_dlog_knowledge_proof_st { + EC_POINT *A; + BIGNUM *z; +}; + +# ifdef __cplusplus +} +# endif + +#endif + + diff --git a/openssl/src/crypto/zkp/nizk/nizk_encode.c b/openssl/src/crypto/zkp/nizk/nizk_encode.c new file mode 100644 index 000000000..6d139f538 --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_encode.c @@ -0,0 +1,1097 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include "internal/endian.h" +#include +#include "nizk.h" +#include "nizk_plaintext_knowledge.h" +#include "nizk_plaintext_equality.h" +#include "nizk_dlog_knowledge.h" +#include "nizk_dlog_equality.h" + +DEFINE_STACK_OF(BIGNUM) +DEFINE_STACK_OF(EC_POINT) + +static point_conversion_form_t form = POINT_CONVERSION_COMPRESSED; + +/** Encodes NIZK_PUB_PARAM to binary + * \param pp NIZK_PUB_PARAM object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t NIZK_PUB_PARAM_encode(const NIZK_PUB_PARAM *pp, unsigned char *out, size_t size) +{ + int *q, curve_id; + size_t point_len, ret = 0, len; + unsigned char *p; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + + if (pp == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + group = pp->group; + + curve_id = EC_GROUP_get_curve_name(group); + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto end; + } + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + + len = sizeof(int) + point_len * 2; + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + q = (int *)out; + *q++ = zkp_l2n((int)curve_id); + p = (unsigned char *)q; + + if (EC_POINT_point2oct(group, pp->G, form, p, point_len, bn_ctx) == 0) + goto end; + + p += point_len; + + if (EC_POINT_point2oct(group, pp->H, form, p, point_len, bn_ctx) == 0) + goto end; + + p += point_len; + + ret = p - out; + +end: + BN_CTX_free(bn_ctx); + return ret; +} + +/** Decodes binary to NIZK_PUB_PARAM + * \param in Memory buffer with the encoded NIZK_PUB_PARAM + * object + * \param size The memory size of the in pointer object + * \return NIZK_PUB_PARAM object pointer on success and NULL otherwise + */ +NIZK_PUB_PARAM *NIZK_PUB_PARAM_decode(const unsigned char *in, size_t size) +{ + unsigned char *p; + int curve_id, *q = (int *)in; + size_t point_len; + NIZK_PUB_PARAM *pp = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + EC_POINT *G = NULL, *H = NULL; + + if (in == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto err; + } + + curve_id = zkp_n2l(*q); + q++; + p = (unsigned char *)q; + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + + if (size < (sizeof(int) + point_len * 2)) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + G = EC_POINT_new(group); + H = EC_POINT_new(group); + if (G == NULL || H == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!EC_POINT_oct2point(group, G, p, point_len, bn_ctx)) + goto err; + + p += point_len; + + if (!EC_POINT_oct2point(group, H, p, point_len, bn_ctx)) + goto err; + + p += point_len; + + pp = NIZK_PUB_PARAM_new(group, G, H); + if (pp == NULL) + goto err; + + EC_POINT_free(G); + EC_POINT_free(H); + EC_GROUP_free(group); + BN_CTX_free(bn_ctx); + return pp; + +err: + EC_POINT_free(G); + EC_POINT_free(H); + EC_GROUP_free(group); + NIZK_PUB_PARAM_free(pp); + BN_CTX_free(bn_ctx); + return NULL; +} + +/** Encodes NIZK_WITNESS to binary + * \param pp NIZK_WITNESS object + * \param out The buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \param flag The flag is an indicator for encoding random number 'r' + * and plaintext 'v', with 1 indicating encoding and 0 + * indicating no encoding. + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t NIZK_WITNESS_encode(const NIZK_WITNESS *witness, unsigned char *out, + size_t size, int flag) +{ + int *q, bn_len; + size_t ret = 0, len; + unsigned char *p; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + + if (witness == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return ret; + } + + bn_len = BN_num_bytes(witness->order); + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto end; + } + + len = sizeof(int) + bn_len * 2; + + if (flag == 1) + len += bn_len; + + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + q = (int *)out; + *q++ = zkp_l2n((int)bn_len); + p = (unsigned char *)q; + + len = zkp_bignum_encode(witness->order, p, bn_len); + if (len <= 0) + goto end; + + p += len; + + len = zkp_bignum_encode(witness->r, p, bn_len); + if (len <= 0) + goto end; + + p += len; + + if (flag == 1) { + len = zkp_bignum_encode(witness->v, p, bn_len); + if (len <= 0) + goto end; + + p += len; + } + + ret = p - out; + +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return ret; +} + +/** Decodes binary to NIZK_WITNESS + * \param in Memory buffer with the encoded NIZK_WITNESS + * object + * \param size The memory size of the in pointer object + * \param flag The flag is an indicator for decoding random number 'r' + * and plaintext 'v', with 1 indicating decoding and 0 + * indicating no decoding. + * \return NIZK_WITNESS object pointer on success and NULL otherwise + */ +NIZK_WITNESS *NIZK_WITNESS_decode(const unsigned char *in, size_t size, int flag) +{ + unsigned char *p; + int *q = (int *)in, bn_len; + NIZK_WITNESS *witness = NULL; + + if (in == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + bn_len = zkp_n2l(*q); + q++; + p = (unsigned char *)q; + + if (size < (sizeof(int) + bn_len * 2)) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(witness = OPENSSL_zalloc(sizeof(*witness)))) { + goto err; + } + + witness->order = zkp_bignum_decode(p, NULL, bn_len); + if (witness->order == NULL) { + goto err; + } + + p += bn_len; + + witness->r = zkp_bignum_decode(p, NULL, bn_len); + if (witness->r == NULL) { + goto err; + } + + p += bn_len; + + if (flag == 1) { + if (size < (sizeof(int) + bn_len * 3)) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + witness->v = zkp_bignum_decode(p, NULL, bn_len); + if (witness->v == NULL) { + goto err; + } + } + + witness->references = 1; + if ((witness->lock = CRYPTO_THREAD_lock_new()) == NULL) + goto err; + + return witness; + +err: + NIZK_WITNESS_free(witness); + return NULL; +} + +/** Encodes NIZK_PLAINTEXT_KNOWLEDGE_PROOF to binary + * \param proof NIZK_PLAINTEXT_KNOWLEDGE_PROOF object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t NIZK_PLAINTEXT_KNOWLEDGE_PROOF_encode(const NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof, + unsigned char *out, size_t size) +{ + int *q, curve_id, bn_len, ret = 0, sk_len; + size_t len; + unsigned char *p = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + sk_point = sk_EC_POINT_new_reserve(NULL, 2); + sk_bn = sk_BIGNUM_new_reserve(NULL, 2); + if (sk_point == NULL || sk_bn == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return 0; + } + + if ((curve_id = EC_POINT_get_curve_name(proof->A)) == NID_undef) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto end; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto end; + } + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + len = sizeof(int); + + if (sk_EC_POINT_push(sk_point, proof->A) <= 0 + || sk_EC_POINT_push(sk_point, proof->B) <= 0) + goto end; + + sk_len = zkp_stack_of_point_encode(sk_point, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + len += sk_len; + + if (sk_BIGNUM_push(sk_bn, proof->z1) <= 0 + || sk_BIGNUM_push(sk_bn, proof->z2) <= 0) + goto end; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, NULL, bn_len); + if (sk_len == 0) + goto end; + len += sk_len; + + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + /* encoding proof */ + q = (int *)out; + *q++ = zkp_l2n(curve_id); + p = (unsigned char *)q; + + sk_len = zkp_stack_of_point_encode(sk_point, p, group, bn_ctx); + if (sk_len == 0) + goto end; + p += sk_len; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, p, bn_len); + if (sk_len == 0) + goto end; + p += sk_len; + + ret = p - out; +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + return ret; +} + +/** Decodes binary to NIZK_PLAINTEXT_KNOWLEDGE_PROOF + * \param in Memory buffer with the encoded NIZK_PLAINTEXT_KNOWLEDGE_PROOF object + * \param size The memory size of the in pointer object + * \return NIZK_RANGE_PROOF object pointer on success and NULL otherwise + */ +NIZK_PLAINTEXT_KNOWLEDGE_PROOF *NIZK_PLAINTEXT_KNOWLEDGE_PROOF_decode(const unsigned char *in, + size_t size) +{ + unsigned char *p; + int *q = (int *)in, curve_id, len; + size_t point_len, bn_len, proof_len; + NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (in == NULL || size <= 4) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + curve_id = zkp_n2l(*q); + q++; + + if (curve_id <= 0) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto err; + } + + p = (unsigned char *)q; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + + /* len(curve_id) + len(A+B) + len(z1+z2) */ + proof_len = 4 + point_len * 2 + bn_len * 2; + if (size < proof_len) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) + goto err; + + sk_point = zkp_stack_of_point_decode(p, &len, group, bn_ctx); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_EC_POINT_num(sk_point) < 2) + goto err; + + proof->A = sk_EC_POINT_value(sk_point, 0); + proof->B = sk_EC_POINT_value(sk_point, 1); + + sk_bn = zkp_stack_of_bignum_decode(p, &len, bn_len); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_BIGNUM_num(sk_bn) < 2) + goto err; + + proof->z1 = sk_BIGNUM_value(sk_bn, 0); + proof->z2 = sk_BIGNUM_value(sk_bn, 1); + + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return proof; + +err: + sk_BIGNUM_pop_free(sk_bn, BN_free); + sk_EC_POINT_pop_free(sk_point, EC_POINT_free); + OPENSSL_free(proof); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return NULL; +} + +/** Encodes NIZK_PLAINTEXT_EQUALITY_PROOF to binary + * \param proof NIZK_PLAINTEXT_EQUALITY_PROOF object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t NIZK_PLAINTEXT_EQUALITY_PROOF_encode(const NIZK_PLAINTEXT_EQUALITY_PROOF *proof, + unsigned char *out, size_t size) +{ + int *q, curve_id, bn_len, ret = 0, sk_len; + size_t len; + unsigned char *p = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + sk_point = sk_EC_POINT_dup(proof->sk_A); + sk_bn = sk_BIGNUM_new_reserve(NULL, 2); + if (sk_point == NULL || sk_bn == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return 0; + } + + if ((curve_id = EC_POINT_get_curve_name(proof->B)) == NID_undef) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto end; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto end; + } + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + len = sizeof(int); + + if (sk_EC_POINT_push(sk_point, proof->B) <= 0) + goto end; + + sk_len = zkp_stack_of_point_encode(sk_point, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + len += sk_len; + + if (sk_BIGNUM_push(sk_bn, proof->z) <= 0 + || sk_BIGNUM_push(sk_bn, proof->t) <= 0) + goto end; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, NULL, bn_len); + if (sk_len == 0) + goto end; + len += sk_len; + + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + /* encoding proof */ + q = (int *)out; + *q++ = zkp_l2n(curve_id); + p = (unsigned char *)q; + + sk_len = zkp_stack_of_point_encode(sk_point, p, group, bn_ctx); + if (sk_len == 0) + goto end; + p += sk_len; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, p, bn_len); + if (sk_len == 0) + goto end; + p += sk_len; + + ret = p - out; +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + return ret; +} + +/** Decodes binary to NIZK_PLAINTEXT_EQUALITY_PROOF + * \param in Memory buffer with the encoded NIZK_PLAINTEXT_EQUALITY_PROOF object + * \param size The memory size of the in pointer object + * \return NIZK_RANGE_PROOF object pointer on success and NULL otherwise + */ +NIZK_PLAINTEXT_EQUALITY_PROOF *NIZK_PLAINTEXT_EQUALITY_PROOF_decode(const unsigned char *in, + size_t size) +{ + unsigned char *p; + int *q = (int *)in, curve_id, len; + size_t point_len, bn_len, proof_len; + NIZK_PLAINTEXT_EQUALITY_PROOF *proof = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (in == NULL || size <= 4) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + curve_id = zkp_n2l(*q); + q++; + + if (curve_id <= 0) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto err; + } + + p = (unsigned char *)q; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + + /* len(curve_id) + len(sk_A+B) + len(z1+z2) */ + proof_len = 4 + point_len + bn_len * 2; + if (size < proof_len) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) + goto err; + + sk_point = zkp_stack_of_point_decode(p, &len, group, bn_ctx); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_EC_POINT_num(sk_point) < 1) + goto err; + + proof->sk_A = sk_point; + proof->B = sk_EC_POINT_pop(sk_point); + + sk_bn = zkp_stack_of_bignum_decode(p, &len, bn_len); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_BIGNUM_num(sk_bn) < 2) + goto err; + + proof->z = sk_BIGNUM_value(sk_bn, 0); + proof->t = sk_BIGNUM_value(sk_bn, 1); + + sk_BIGNUM_free(sk_bn); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return proof; + +err: + sk_BIGNUM_pop_free(sk_bn, BN_free); + sk_EC_POINT_pop_free(sk_point, EC_POINT_free); + OPENSSL_free(proof); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return NULL; +} + +/** Encodes NIZK_DLOG_KNOWLEDGE_PROOF to binary + * \param proof NIZK_DLOG_KNOWLEDGE_PROOF object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t NIZK_DLOG_KNOWLEDGE_PROOF_encode(const NIZK_DLOG_KNOWLEDGE_PROOF *proof, + unsigned char *out, size_t size) +{ + int *q, curve_id, bn_len, ret = 0, sk_len; + size_t len; + unsigned char *p = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + sk_point = sk_EC_POINT_new_reserve(NULL, 1); + sk_bn = sk_BIGNUM_new_reserve(NULL, 1); + if (sk_point == NULL || sk_bn == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return 0; + } + + if ((curve_id = EC_POINT_get_curve_name(proof->A)) == NID_undef) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto end; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto end; + } + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + len = sizeof(int); + + if (sk_EC_POINT_push(sk_point, proof->A) <= 0) + goto end; + + sk_len = zkp_stack_of_point_encode(sk_point, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + len += sk_len; + + if (sk_BIGNUM_push(sk_bn, proof->z) <= 0) + goto end; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, NULL, bn_len); + if (sk_len == 0) + goto end; + len += sk_len; + + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + /* encoding proof */ + q = (int *)out; + *q++ = zkp_l2n(curve_id); + p = (unsigned char *)q; + + sk_len = zkp_stack_of_point_encode(sk_point, p, group, bn_ctx); + if (sk_len == 0) + goto end; + p += sk_len; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, p, bn_len); + if (sk_len == 0) + goto end; + p += sk_len; + + ret = p - out; +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + return ret; +} + +/** Decodes binary to NIZK_DLOG_KNOWLEDGE_PROOF + * \param in Memory buffer with the encoded NIZK_DLOG_KNOWLEDGE_PROOF object + * \param size The memory size of the in pointer object + * \return NIZK_RANGE_PROOF object pointer on success and NULL otherwise + */ +NIZK_DLOG_KNOWLEDGE_PROOF *NIZK_DLOG_KNOWLEDGE_PROOF_decode(const unsigned char *in, + size_t size) +{ + unsigned char *p; + int *q = (int *)in, curve_id, len; + size_t point_len, bn_len, proof_len; + NIZK_DLOG_KNOWLEDGE_PROOF *proof = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (in == NULL || size <= 4) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + curve_id = zkp_n2l(*q); + q++; + + if (curve_id <= 0) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto err; + } + + p = (unsigned char *)q; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + + /* len(curve_id) + len(A) + len(z) */ + proof_len = 4 + point_len + bn_len; + if (size < proof_len) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) + goto err; + + sk_point = zkp_stack_of_point_decode(p, &len, group, bn_ctx); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_EC_POINT_num(sk_point) < 1) + goto err; + + proof->A = sk_EC_POINT_value(sk_point, 0); + + sk_bn = zkp_stack_of_bignum_decode(p, &len, bn_len); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_BIGNUM_num(sk_bn) < 1) + goto err; + + proof->z = sk_BIGNUM_value(sk_bn, 0); + + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return proof; + +err: + sk_BIGNUM_pop_free(sk_bn, BN_free); + sk_EC_POINT_pop_free(sk_point, EC_POINT_free); + OPENSSL_free(proof); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return NULL; +} + +/** Encodes NIZK_DLOG_EQUALITY_PROOF to binary + * \param proof NIZK_DLOG_EQUALITY_PROOF object + * \param out the buffer for the result (if NULL the function returns + * number of bytes needed). + * \param size The memory size of the out pointer object + * \return the length of the encoded octet string or 0 if an error occurred + */ +size_t NIZK_DLOG_EQUALITY_PROOF_encode(const NIZK_DLOG_EQUALITY_PROOF *proof, + unsigned char *out, size_t size) +{ + int *q, curve_id, bn_len, ret = 0, sk_len; + size_t len; + unsigned char *p = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + sk_point = sk_EC_POINT_new_reserve(NULL, 2); + sk_bn = sk_BIGNUM_new_reserve(NULL, 1); + if (sk_point == NULL || sk_bn == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return 0; + } + + if ((curve_id = EC_POINT_get_curve_name(proof->A1)) == NID_undef) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto end; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto end; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto end; + } + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + len = sizeof(int); + + if (sk_EC_POINT_push(sk_point, proof->A1) <= 0 + || sk_EC_POINT_push(sk_point, proof->A2) <= 0) + goto end; + + sk_len = zkp_stack_of_point_encode(sk_point, NULL, group, bn_ctx); + if (sk_len == 0) + goto end; + len += sk_len; + + if (sk_BIGNUM_push(sk_bn, proof->z) <= 0) + goto end; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, NULL, bn_len); + if (sk_len == 0) + goto end; + len += sk_len; + + if (out == NULL) { + ret = len; + goto end; + } + + if (size < len) + goto end; + + memset(out, 0, size); + + /* encoding proof */ + q = (int *)out; + *q++ = zkp_l2n(curve_id); + p = (unsigned char *)q; + + sk_len = zkp_stack_of_point_encode(sk_point, p, group, bn_ctx); + if (sk_len == 0) + goto end; + p += sk_len; + + sk_len = zkp_stack_of_bignum_encode(sk_bn, p, bn_len); + if (sk_len == 0) + goto end; + p += sk_len; + + ret = p - out; +end: + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + return ret; +} + +/** Decodes binary to NIZK_DLOG_EQUALITY_PROOF + * \param in Memory buffer with the encoded NIZK_DLOG_EQUALITY_PROOF object + * \param size The memory size of the in pointer object + * \return NIZK_RANGE_PROOF object pointer on success and NULL otherwise + */ +NIZK_DLOG_EQUALITY_PROOF *NIZK_DLOG_EQUALITY_PROOF_decode(const unsigned char *in, + size_t size) +{ + unsigned char *p; + int *q = (int *)in, curve_id, len; + size_t point_len, bn_len, proof_len; + NIZK_DLOG_EQUALITY_PROOF *proof = NULL; + BN_CTX *bn_ctx = NULL; + EC_GROUP *group = NULL; + STACK_OF(EC_POINT) *sk_point = NULL; + STACK_OF(BIGNUM) *sk_bn = NULL; + + if (in == NULL || size <= 4) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + curve_id = zkp_n2l(*q); + q++; + + if (curve_id <= 0) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + group = EC_GROUP_new_by_curve_name_ex(NULL, NULL, curve_id); + if (group == NULL) + goto err; + + bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + goto err; + } + + p = (unsigned char *)q; + + bn_len = BN_num_bytes(EC_GROUP_get0_order(group)); + + point_len = EC_POINT_point2oct(group, EC_GROUP_get0_generator(group), + form, NULL, 0, bn_ctx); + if (point_len <= 0) + goto err; + + /* len(curve_id) + len(A1+A2) + len(z) */ + proof_len = 4 + point_len*2 + bn_len; + if (size < proof_len) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + goto err; + } + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) + goto err; + + sk_point = zkp_stack_of_point_decode(p, &len, group, bn_ctx); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_EC_POINT_num(sk_point) < 2) + goto err; + + proof->A1 = sk_EC_POINT_value(sk_point, 0); + proof->A2 = sk_EC_POINT_value(sk_point, 1); + + sk_bn = zkp_stack_of_bignum_decode(p, &len, bn_len); + if (sk_point == NULL) + goto err; + p += len; + + if (sk_BIGNUM_num(sk_bn) < 1) + goto err; + + proof->z = sk_BIGNUM_value(sk_bn, 0); + + sk_BIGNUM_free(sk_bn); + sk_EC_POINT_free(sk_point); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return proof; + +err: + sk_BIGNUM_pop_free(sk_bn, BN_free); + sk_EC_POINT_pop_free(sk_point, EC_POINT_free); + OPENSSL_free(proof); + BN_CTX_free(bn_ctx); + EC_GROUP_free(group); + return NULL; +} diff --git a/openssl/src/crypto/zkp/nizk/nizk_err.c b/openssl/src/crypto/zkp/nizk/nizk_err.c new file mode 100644 index 000000000..8a1a2595f --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_err.c @@ -0,0 +1,32 @@ +/* + * Generated by util/mkerr.pl DO NOT EDIT + * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include +#include "crypto/zkpnizkerr.h" + +#ifndef OPENSSL_NO_ERR + +static const ERR_STRING_DATA ZKP_NIZK_str_reasons[] = { + {ERR_PACK(ERR_LIB_ZKP_NIZK, 0, ZKP_NIZK_R_TRANSCRIPT_INIT_FAILED), + "transcript init failed"}, + {0, NULL} +}; + +#endif + +int ossl_err_load_ZKP_NIZK_strings(void) +{ +#ifndef OPENSSL_NO_ERR + if (ERR_reason_error_string(ZKP_NIZK_str_reasons[0].error) == NULL) + ERR_load_strings_const(ZKP_NIZK_str_reasons); +#endif + return 1; +} diff --git a/openssl/src/crypto/zkp/nizk/nizk_plaintext_equality.c b/openssl/src/crypto/zkp/nizk/nizk_plaintext_equality.c new file mode 100644 index 000000000..bbd124e60 --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_plaintext_equality.c @@ -0,0 +1,356 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "nizk_plaintext_equality.h" + +DEFINE_STACK_OF(EC_POINT) + +NIZK_PLAINTEXT_EQUALITY_CTX *NIZK_PLAINTEXT_EQUALITY_CTX_new(ZKP_TRANSCRIPT *transcript, + NIZK_PUB_PARAM *pp, + NIZK_WITNESS *witness, + STACK_OF(EC_POINT) *pk, + EC_ELGAMAL_MR_CIPHERTEXT *ct) +{ + int i; + EC_POINT *PK, *P = NULL; + NIZK_PLAINTEXT_EQUALITY_CTX *ctx = NULL; + + if (pp == NULL || transcript == NULL || pk == NULL || ct == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (sk_EC_POINT_num(pk) == 0 || + sk_EC_POINT_num(pk) != sk_EC_POINT_num(ct->sk_C1)) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ctx->transcript = transcript; + + if (!NIZK_PUB_PARAM_up_ref(pp)) + goto err; + + ctx->pp = pp; + + if (witness != NULL) { + if (!NIZK_WITNESS_up_ref(witness)) + goto err; + + ctx->witness = witness; + } + + if (!(ctx->sk_PK = sk_EC_POINT_new_null())) + goto err; + + for (i = 0; i < sk_EC_POINT_num(pk); i++) { + PK = sk_EC_POINT_value(pk, i); + if (!(P = EC_POINT_dup(PK, pp->group))) + goto err; + + if (sk_EC_POINT_push(ctx->sk_PK, P) <= 0) + goto err; + + P = NULL; + } + + if (!(ctx->ct = EC_ELGAMAL_MR_CIPHERTEXT_dup(ct, pp->group))) + goto err; + + return ctx; +err: + EC_POINT_free(P); + NIZK_PLAINTEXT_EQUALITY_CTX_free(ctx); + return NULL; +} + +void NIZK_PLAINTEXT_EQUALITY_CTX_free(NIZK_PLAINTEXT_EQUALITY_CTX *ctx) +{ + if (ctx == NULL) + return; + + NIZK_PUB_PARAM_down_ref(ctx->pp); + NIZK_WITNESS_down_ref(ctx->witness); + + sk_EC_POINT_pop_free(ctx->sk_PK, EC_POINT_free); + EC_ELGAMAL_MR_CIPHERTEXT_free(ctx->ct); + + OPENSSL_clear_free((void *)ctx, sizeof(*ctx)); +} + +NIZK_PLAINTEXT_EQUALITY_PROOF *NIZK_PLAINTEXT_EQUALITY_PROOF_new(NIZK_PLAINTEXT_EQUALITY_CTX *ctx) +{ + NIZK_PLAINTEXT_EQUALITY_PROOF *proof = NULL; + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if ((proof->sk_A = sk_EC_POINT_new_null()) == NULL + || (proof->B = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->z = BN_new()) == NULL + || (proof->t = BN_new()) == NULL) + goto err; + + EC_POINT_set_to_infinity(ctx->pp->group, proof->B); + + BN_zero(proof->z); + BN_zero(proof->t); + + return proof; +err: + NIZK_PLAINTEXT_EQUALITY_PROOF_free(proof); + return NULL; +} + +void NIZK_PLAINTEXT_EQUALITY_PROOF_free(NIZK_PLAINTEXT_EQUALITY_PROOF *proof) +{ + if (proof == NULL) + return; + + sk_EC_POINT_pop_free(proof->sk_A, EC_POINT_free); + EC_POINT_free(proof->B); + BN_free(proof->z); + BN_free(proof->t); + OPENSSL_clear_free((void *)proof, sizeof(*proof)); +} + +NIZK_PLAINTEXT_EQUALITY_PROOF *NIZK_PLAINTEXT_EQUALITY_PROOF_prove(NIZK_PLAINTEXT_EQUALITY_CTX *ctx) +{ + int i; + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + NIZK_WITNESS *witness; + NIZK_PLAINTEXT_EQUALITY_PROOF *proof = NULL, *ret = NULL; + const BIGNUM *order; + EC_GROUP *group; + EC_POINT *A = NULL, *P; + BN_CTX *bn_ctx = NULL; + BIGNUM *a, *b, *t, *e; + zkp_poly_points_t *poly = NULL; + + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (ctx->pp == NULL || ctx->sk_PK == NULL || ctx->ct == NULL || + sk_EC_POINT_num(ctx->sk_PK) != sk_EC_POINT_num(ctx->ct->sk_C1)) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } + + if (!(proof = NIZK_PLAINTEXT_EQUALITY_PROOF_new(ctx))) + return NULL; + + pp = ctx->pp; + witness = ctx->witness; + transcript = ctx->transcript; + group = pp->group; + order = EC_GROUP_get0_order(group); + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + a = BN_CTX_get(bn_ctx); + b = BN_CTX_get(bn_ctx); + e = BN_CTX_get(bn_ctx); + t = BN_CTX_get(bn_ctx); + if (t == NULL) + goto err; + + if (!zkp_rand_range(a, order) || !zkp_rand_range(b, order)) + goto err; + + for (i = 0; i < sk_EC_POINT_num(ctx->sk_PK); i++) { + P = sk_EC_POINT_value(ctx->sk_PK, i); + if (!ZKP_TRANSCRIPT_append_point(transcript, "PK", P, group)) + goto err; + + A = EC_POINT_new(group); + if (A == NULL) + goto err; + + if (!EC_POINT_mul(group, A, NULL, P, a, bn_ctx)) + goto err; + + if (sk_EC_POINT_push(proof->sk_A, A) <= 0) + goto err; + + A = NULL; + + P = sk_EC_POINT_value(ctx->ct->sk_C1, i); + if (!ZKP_TRANSCRIPT_append_point(transcript, "C1", P, group)) + goto err; + + P = sk_EC_POINT_value(proof->sk_A, i); + if (!ZKP_TRANSCRIPT_append_point(transcript, "A", P, group)) + goto err; + } + + if (!(poly = zkp_poly_points_new(2))) + goto err; + + if (!zkp_poly_points_append(poly, pp->G, a) + || !zkp_poly_points_append(poly, pp->H, b)) + goto err; + + if (!zkp_poly_points_mul(poly, proof->B, NULL, group, bn_ctx)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "C2", ctx->ct->C2, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "B", proof->B, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "e", e)) + goto err; + + if (!BN_mul(t, e, witness->r, bn_ctx) + || !BN_mod_add(proof->z, a, t, order, bn_ctx) + || !BN_mul(t, e, witness->v, bn_ctx) + || !BN_mod_add(proof->t, b, t, order, bn_ctx)) + goto err; + + ret = proof; + proof = NULL; +err: + EC_POINT_free(A); + BN_CTX_free(bn_ctx); + zkp_poly_points_free(poly); + NIZK_PLAINTEXT_EQUALITY_PROOF_free(proof); + ZKP_TRANSCRIPT_reset(transcript); + return ret; +} + +int NIZK_PLAINTEXT_EQUALITY_PROOF_verify(NIZK_PLAINTEXT_EQUALITY_CTX *ctx, + NIZK_PLAINTEXT_EQUALITY_PROOF *proof) +{ + int ret = 0, i; + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + EC_GROUP *group; + BN_CTX *bn_ctx = NULL; + BIGNUM *e, *bn1; + EC_POINT *A, *P, *L = NULL, *R = NULL; + zkp_poly_points_t *poly = NULL; + + if (ctx == NULL || proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (ctx->pp == NULL || ctx->sk_PK == NULL || ctx->ct == NULL || + sk_EC_POINT_num(ctx->sk_PK) != sk_EC_POINT_num(ctx->ct->sk_C1)) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_INVALID_ARGUMENT); + return 0; + } + + pp = ctx->pp; + transcript = ctx->transcript; + group = pp->group; + + if (!(L = EC_POINT_new(group)) || !(R = EC_POINT_new(group))) + goto err; + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + e = BN_CTX_get(bn_ctx); + bn1 = BN_CTX_get(bn_ctx); + if (bn1 == NULL) + goto err; + + BN_one(bn1); + + for (i = 0; i < sk_EC_POINT_num(ctx->sk_PK); i++) { + P = sk_EC_POINT_value(ctx->sk_PK, i); + if (!ZKP_TRANSCRIPT_append_point(transcript, "PK", P, group)) + goto err; + + P = sk_EC_POINT_value(ctx->ct->sk_C1, i); + if (!ZKP_TRANSCRIPT_append_point(transcript, "C1", P, group)) + goto err; + + P = sk_EC_POINT_value(proof->sk_A, i); + if (!ZKP_TRANSCRIPT_append_point(transcript, "A", P, group)) + goto err; + } + + if (!ZKP_TRANSCRIPT_append_point(transcript, "C2", ctx->ct->C2, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "B", proof->B, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "e", e)) + goto err; + + if (!(poly = zkp_poly_points_new(2))) + goto err; + + for (i = 0; i < sk_EC_POINT_num(ctx->sk_PK); i++) { + P = sk_EC_POINT_value(ctx->sk_PK, i); + if (!EC_POINT_mul(group, L, NULL, P, proof->z, bn_ctx)) + goto err; + + A = sk_EC_POINT_value(proof->sk_A, i); + P = sk_EC_POINT_value(ctx->ct->sk_C1, i); + + if (!zkp_poly_points_append(poly, A, bn1) + || !zkp_poly_points_append(poly, P, e)) + goto err; + + if (!zkp_poly_points_mul(poly, R, NULL, group, bn_ctx)) + goto err; + + if (EC_POINT_cmp(group, L, R, bn_ctx) != 0) + goto err; + + zkp_poly_points_reset(poly); + } + + if (!zkp_poly_points_append(poly, pp->G, proof->z) + || !zkp_poly_points_append(poly, pp->H, proof->t)) + goto err; + + if (!zkp_poly_points_mul(poly, L, NULL, group, bn_ctx)) + goto err; + + zkp_poly_points_reset(poly); + + if (!zkp_poly_points_append(poly, proof->B, bn1) + || !zkp_poly_points_append(poly, ctx->ct->C2, e)) + goto err; + + if (!zkp_poly_points_mul(poly, R, NULL, group, bn_ctx)) + goto err; + + if (EC_POINT_cmp(group, L, R, bn_ctx) != 0) + goto err; + + ret = 1; +err: + EC_POINT_free(L); + EC_POINT_free(R); + zkp_poly_points_free(poly); + ZKP_TRANSCRIPT_reset(transcript); + return ret; +} diff --git a/openssl/src/crypto/zkp/nizk/nizk_plaintext_equality.h b/openssl/src/crypto/zkp/nizk/nizk_plaintext_equality.h new file mode 100644 index 000000000..35faebcb3 --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_plaintext_equality.h @@ -0,0 +1,46 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_NIZK_PLAINTEXT_EQUALITY_LOCAL_H +# define HEADER_NIZK_PLAINTEXT_EQUALITY_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include +# include "internal/refcount.h" +# include "nizk.h" + +struct nizk_plaintext_equality_ctx_st { + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + NIZK_WITNESS *witness; + STACK_OF(EC_POINT) *sk_PK; + EC_ELGAMAL_MR_CIPHERTEXT *ct; +}; + +struct nizk_plaintext_equality_proof_st { + STACK_OF(EC_POINT) *sk_A; + EC_POINT *B; + BIGNUM *z; + BIGNUM *t; +}; + +# ifdef __cplusplus +} +# endif + +#endif + diff --git a/openssl/src/crypto/zkp/nizk/nizk_plaintext_knowledge.c b/openssl/src/crypto/zkp/nizk/nizk_plaintext_knowledge.c new file mode 100644 index 000000000..a86de3b8b --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_plaintext_knowledge.c @@ -0,0 +1,288 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include "nizk_plaintext_knowledge.h" + +NIZK_PLAINTEXT_KNOWLEDGE_CTX *NIZK_PLAINTEXT_KNOWLEDGE_CTX_new(ZKP_TRANSCRIPT *transcript, + NIZK_PUB_PARAM *pp, + NIZK_WITNESS *witness, + EC_POINT *pk, + EC_ELGAMAL_CIPHERTEXT *ct) +{ + NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx = NULL; + + if (pp == NULL || transcript == NULL || pk == NULL || ct == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + ctx->transcript = transcript; + + if (!NIZK_PUB_PARAM_up_ref(pp)) + goto err; + + ctx->pp = pp; + + if (witness != NULL) { + if (!NIZK_WITNESS_up_ref(witness)) + goto err; + + ctx->witness = witness; + } + + ctx->PK = EC_POINT_dup(pk, pp->group); + if (ctx->PK == NULL) + goto err; + + ctx->ct = EC_ELGAMAL_CIPHERTEXT_dup(ct, pp->group); + if (ctx->ct == NULL) + goto err; + + return ctx; + +err: + NIZK_PLAINTEXT_KNOWLEDGE_CTX_free(ctx); + return NULL; +} + +void NIZK_PLAINTEXT_KNOWLEDGE_CTX_free(NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx) +{ + if (ctx == NULL) + return; + + NIZK_PUB_PARAM_down_ref(ctx->pp); + NIZK_WITNESS_down_ref(ctx->witness); + + OPENSSL_clear_free((void *)ctx, sizeof(*ctx)); +} + +NIZK_PLAINTEXT_KNOWLEDGE_PROOF *NIZK_PLAINTEXT_KNOWLEDGE_PROOF_new(NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx) +{ + NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof = NULL; + + proof = OPENSSL_zalloc(sizeof(*proof)); + if (proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_MALLOC_FAILURE); + return NULL; + } + + if ((proof->A = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->B = EC_POINT_new(ctx->pp->group)) == NULL + || (proof->z1 = BN_new()) == NULL + || (proof->z2 = BN_new()) == NULL) + goto err; + + EC_POINT_set_to_infinity(ctx->pp->group, proof->A); + EC_POINT_set_to_infinity(ctx->pp->group, proof->B); + + BN_zero(proof->z1); + BN_zero(proof->z2); + + return proof; +err: + NIZK_PLAINTEXT_KNOWLEDGE_PROOF_free(proof); + return NULL; +} + +void NIZK_PLAINTEXT_KNOWLEDGE_PROOF_free(NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof) +{ + if (proof == NULL) + return; + + EC_POINT_free(proof->A); + EC_POINT_free(proof->B); + BN_free(proof->z1); + BN_free(proof->z2); + OPENSSL_clear_free((void *)proof, sizeof(*proof)); +} + +NIZK_PLAINTEXT_KNOWLEDGE_PROOF *NIZK_PLAINTEXT_KNOWLEDGE_PROOF_prove(NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx) +{ + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + NIZK_WITNESS *witness; + NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof = NULL, *ret = NULL; + const BIGNUM *order; + EC_GROUP *group; + BN_CTX *bn_ctx = NULL; + BIGNUM *a, *b, *e, *t; + zkp_poly_points_t *poly = NULL; + + if (ctx == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return NULL; + } + + if (!(proof = NIZK_PLAINTEXT_KNOWLEDGE_PROOF_new(ctx))) + return NULL; + + pp = ctx->pp; + witness = ctx->witness; + transcript = ctx->transcript; + group = pp->group; + order = EC_GROUP_get0_order(group); + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + a = BN_CTX_get(bn_ctx); + b = BN_CTX_get(bn_ctx); + e = BN_CTX_get(bn_ctx); + t = BN_CTX_get(bn_ctx); + if (t == NULL) + goto err; + + if (!zkp_rand_range(a, order) || !zkp_rand_range(b, order)) + goto err; + + if (!EC_POINT_mul(group, proof->A, NULL, ctx->PK, a, bn_ctx)) + goto err; + + if (!(poly = zkp_poly_points_new(2))) + goto err; + + if (!zkp_poly_points_append(poly, pp->G, a) + || !zkp_poly_points_append(poly, pp->H, b)) + goto err; + + if (!zkp_poly_points_mul(poly, proof->B, NULL, group, bn_ctx)) + goto err; + + if (!ZKP_TRANSCRIPT_append_point(transcript, "PK", ctx->PK, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "C1", ctx->ct->C1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "C2", ctx->ct->C2, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A", proof->A, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "B", proof->B, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "e", e)) + goto err; + + if (!BN_mul(t, e, witness->r, bn_ctx) + || !BN_mod_add(proof->z1, a, t, order, bn_ctx) + || !BN_mul(t, e, witness->v, bn_ctx) + || !BN_mod_add(proof->z2, b, t, order, bn_ctx)) + goto err; + + ret = proof; + proof = NULL; +err: + BN_CTX_free(bn_ctx); + zkp_poly_points_free(poly); + NIZK_PLAINTEXT_KNOWLEDGE_PROOF_free(proof); + ZKP_TRANSCRIPT_reset(transcript); + return ret; +} + +int NIZK_PLAINTEXT_KNOWLEDGE_PROOF_verify(NIZK_PLAINTEXT_KNOWLEDGE_CTX *ctx, + NIZK_PLAINTEXT_KNOWLEDGE_PROOF *proof) +{ + int ret = 0; + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + EC_GROUP *group; + BN_CTX *bn_ctx = NULL; + BIGNUM *e, *bn1, *bn_1; + EC_POINT *L = NULL, *R = NULL; + zkp_poly_points_t *poly = NULL; + + if (ctx == NULL || proof == NULL) { + ERR_raise(ERR_LIB_ZKP_NIZK, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + pp = ctx->pp; + transcript = ctx->transcript; + group = pp->group; + + if (!(L = EC_POINT_new(group)) || !(R = EC_POINT_new(group))) + goto err; + + bn_ctx = BN_CTX_new_ex(group->libctx); + if (bn_ctx == NULL) + goto err; + + e = BN_CTX_get(bn_ctx); + bn_1 = BN_CTX_get(bn_ctx); + bn1 = BN_CTX_get(bn_ctx); + if (bn1 == NULL) + goto err; + + BN_one(bn1); + BN_one(bn_1); + BN_set_negative(bn_1, 1); + + if (!ZKP_TRANSCRIPT_append_point(transcript, "PK", ctx->PK, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "C1", ctx->ct->C1, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "C2", ctx->ct->C2, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "A", proof->A, group) + || !ZKP_TRANSCRIPT_append_point(transcript, "B", proof->B, group)) + goto err; + + if (!ZKP_TRANSCRIPT_challange(transcript, "e", e)) + goto err; + + if (!EC_POINT_mul(group, L, NULL, ctx->PK, proof->z1, bn_ctx)) + goto err; + + if (!(poly = zkp_poly_points_new(3))) + goto err; + + if (!zkp_poly_points_append(poly, proof->A, bn1) + || !zkp_poly_points_append(poly, ctx->ct->C1, e) + || !zkp_poly_points_append(poly, L, bn_1)) + goto err; + + if (!zkp_poly_points_mul(poly, R, NULL, group, bn_ctx)) + goto err; + + if (!EC_POINT_is_at_infinity(group, R)) + goto err; + + zkp_poly_points_reset(poly); + + if (!zkp_poly_points_append(poly, pp->G, proof->z1) + || !zkp_poly_points_append(poly, pp->H, proof->z2)) + goto err; + + if (!zkp_poly_points_mul(poly, L, NULL, group, bn_ctx)) + goto err; + + zkp_poly_points_reset(poly); + + if (!zkp_poly_points_append(poly, proof->B, bn1) + || !zkp_poly_points_append(poly, ctx->ct->C2, e) + || !zkp_poly_points_append(poly, L, bn_1)) + goto err; + + if (!zkp_poly_points_mul(poly, R, NULL, group, bn_ctx)) + goto err; + + if (!EC_POINT_is_at_infinity(group, R)) + goto err; + + ret = 1; +err: + EC_POINT_free(L); + EC_POINT_free(R); + zkp_poly_points_free(poly); + ZKP_TRANSCRIPT_reset(transcript); + return ret; +} diff --git a/openssl/src/crypto/zkp/nizk/nizk_plaintext_knowledge.h b/openssl/src/crypto/zkp/nizk/nizk_plaintext_knowledge.h new file mode 100644 index 000000000..839ae244c --- /dev/null +++ b/openssl/src/crypto/zkp/nizk/nizk_plaintext_knowledge.h @@ -0,0 +1,44 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef HEADER_NIZK_PLAINTEXT_KNOWLEDGE_LOCAL_H +# define HEADER_NIZK_PLAINTEXT_KNOWLEDGE_LOCAL_H + +# include + +# ifdef __cplusplus +extern "C" { +# endif + +# include +# include +# include +# include "internal/refcount.h" +# include "nizk.h" + +struct nizk_plaintext_knowledge_ctx_st { + ZKP_TRANSCRIPT *transcript; + NIZK_PUB_PARAM *pp; + NIZK_WITNESS *witness; + EC_POINT *PK; + EC_ELGAMAL_CIPHERTEXT *ct; +}; + +struct nizk_plaintext_knowledge_proof_st { + EC_POINT *A; + EC_POINT *B; + BIGNUM *z1; + BIGNUM *z2; +}; + +# ifdef __cplusplus +} +# endif + +#endif diff --git a/openssl/src/crypto/zuc/zuc.c b/openssl/src/crypto/zuc/zuc.c new file mode 100644 index 000000000..a6fe16049 --- /dev/null +++ b/openssl/src/crypto/zuc/zuc.c @@ -0,0 +1,335 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include "crypto/zuc.h" + +/* + * The ZUC is a stream cipher defined originally in Chinese cipher standard + * GM/T 0001.1-2012, which is Chinese only. + * + * ZUC is also standardized by 3GPP in the 'release 11' spec as an cryptographic + * method for LTE. So two new names were given in the 3GPP specs: 128-EEA3 and + * 128-EIA3, indicating the ZUC encryption method and ZUC MAC method. + * + * This part focuses on the keystream generation. The following implamentation + * utilze the sample code in 3GPP's specification as a reference. + * + */ + +/* S-Box S0 and S1 */ +static unsigned char S0[256] = { + 0x3E, 0x72, 0x5B, 0x47, 0xCA, 0xE0, 0x00, 0x33, 0x04, 0xD1, 0x54, 0x98, 0x09, 0xB9, 0x6D, 0xCB, + 0x7B, 0x1B, 0xF9, 0x32, 0xAF, 0x9D, 0x6A, 0xA5, 0xB8, 0x2D, 0xFC, 0x1D, 0x08, 0x53, 0x03, 0x90, + 0x4D, 0x4E, 0x84, 0x99, 0xE4, 0xCE, 0xD9, 0x91, 0xDD, 0xB6, 0x85, 0x48, 0x8B, 0x29, 0x6E, 0xAC, + 0xCD, 0xC1, 0xF8, 0x1E, 0x73, 0x43, 0x69, 0xC6, 0xB5, 0xBD, 0xFD, 0x39, 0x63, 0x20, 0xD4, 0x38, + 0x76, 0x7D, 0xB2, 0xA7, 0xCF, 0xED, 0x57, 0xC5, 0xF3, 0x2C, 0xBB, 0x14, 0x21, 0x06, 0x55, 0x9B, + 0xE3, 0xEF, 0x5E, 0x31, 0x4F, 0x7F, 0x5A, 0xA4, 0x0D, 0x82, 0x51, 0x49, 0x5F, 0xBA, 0x58, 0x1C, + 0x4A, 0x16, 0xD5, 0x17, 0xA8, 0x92, 0x24, 0x1F, 0x8C, 0xFF, 0xD8, 0xAE, 0x2E, 0x01, 0xD3, 0xAD, + 0x3B, 0x4B, 0xDA, 0x46, 0xEB, 0xC9, 0xDE, 0x9A, 0x8F, 0x87, 0xD7, 0x3A, 0x80, 0x6F, 0x2F, 0xC8, + 0xB1, 0xB4, 0x37, 0xF7, 0x0A, 0x22, 0x13, 0x28, 0x7C, 0xCC, 0x3C, 0x89, 0xC7, 0xC3, 0x96, 0x56, + 0x07, 0xBF, 0x7E, 0xF0, 0x0B, 0x2B, 0x97, 0x52, 0x35, 0x41, 0x79, 0x61, 0xA6, 0x4C, 0x10, 0xFE, + 0xBC, 0x26, 0x95, 0x88, 0x8A, 0xB0, 0xA3, 0xFB, 0xC0, 0x18, 0x94, 0xF2, 0xE1, 0xE5, 0xE9, 0x5D, + 0xD0, 0xDC, 0x11, 0x66, 0x64, 0x5C, 0xEC, 0x59, 0x42, 0x75, 0x12, 0xF5, 0x74, 0x9C, 0xAA, 0x23, + 0x0E, 0x86, 0xAB, 0xBE, 0x2A, 0x02, 0xE7, 0x67, 0xE6, 0x44, 0xA2, 0x6C, 0xC2, 0x93, 0x9F, 0xF1, + 0xF6, 0xFA, 0x36, 0xD2, 0x50, 0x68, 0x9E, 0x62, 0x71, 0x15, 0x3D, 0xD6, 0x40, 0xC4, 0xE2, 0x0F, + 0x8E, 0x83, 0x77, 0x6B, 0x25, 0x05, 0x3F, 0x0C, 0x30, 0xEA, 0x70, 0xB7, 0xA1, 0xE8, 0xA9, 0x65, + 0x8D, 0x27, 0x1A, 0xDB, 0x81, 0xB3, 0xA0, 0xF4, 0x45, 0x7A, 0x19, 0xDF, 0xEE, 0x78, 0x34, 0x60, +}; + +static unsigned char S1[256] = { + 0x55, 0xC2, 0x63, 0x71, 0x3B, 0xC8, 0x47, 0x86, 0x9F, 0x3C, 0xDA, 0x5B, 0x29, 0xAA, 0xFD, 0x77, + 0x8C, 0xC5, 0x94, 0x0C, 0xA6, 0x1A, 0x13, 0x00, 0xE3, 0xA8, 0x16, 0x72, 0x40, 0xF9, 0xF8, 0x42, + 0x44, 0x26, 0x68, 0x96, 0x81, 0xD9, 0x45, 0x3E, 0x10, 0x76, 0xC6, 0xA7, 0x8B, 0x39, 0x43, 0xE1, + 0x3A, 0xB5, 0x56, 0x2A, 0xC0, 0x6D, 0xB3, 0x05, 0x22, 0x66, 0xBF, 0xDC, 0x0B, 0xFA, 0x62, 0x48, + 0xDD, 0x20, 0x11, 0x06, 0x36, 0xC9, 0xC1, 0xCF, 0xF6, 0x27, 0x52, 0xBB, 0x69, 0xF5, 0xD4, 0x87, + 0x7F, 0x84, 0x4C, 0xD2, 0x9C, 0x57, 0xA4, 0xBC, 0x4F, 0x9A, 0xDF, 0xFE, 0xD6, 0x8D, 0x7A, 0xEB, + 0x2B, 0x53, 0xD8, 0x5C, 0xA1, 0x14, 0x17, 0xFB, 0x23, 0xD5, 0x7D, 0x30, 0x67, 0x73, 0x08, 0x09, + 0xEE, 0xB7, 0x70, 0x3F, 0x61, 0xB2, 0x19, 0x8E, 0x4E, 0xE5, 0x4B, 0x93, 0x8F, 0x5D, 0xDB, 0xA9, + 0xAD, 0xF1, 0xAE, 0x2E, 0xCB, 0x0D, 0xFC, 0xF4, 0x2D, 0x46, 0x6E, 0x1D, 0x97, 0xE8, 0xD1, 0xE9, + 0x4D, 0x37, 0xA5, 0x75, 0x5E, 0x83, 0x9E, 0xAB, 0x82, 0x9D, 0xB9, 0x1C, 0xE0, 0xCD, 0x49, 0x89, + 0x01, 0xB6, 0xBD, 0x58, 0x24, 0xA2, 0x5F, 0x38, 0x78, 0x99, 0x15, 0x90, 0x50, 0xB8, 0x95, 0xE4, + 0xD0, 0x91, 0xC7, 0xCE, 0xED, 0x0F, 0xB4, 0x6F, 0xA0, 0xCC, 0xF0, 0x02, 0x4A, 0x79, 0xC3, 0xDE, + 0xA3, 0xEF, 0xEA, 0x51, 0xE6, 0x6B, 0x18, 0xEC, 0x1B, 0x2C, 0x80, 0xF7, 0x74, 0xE7, 0xFF, 0x21, + 0x5A, 0x6A, 0x54, 0x1E, 0x41, 0x31, 0x92, 0x35, 0xC4, 0x33, 0x07, 0x0A, 0xBA, 0x7E, 0x0E, 0x34, + 0x88, 0xB1, 0x98, 0x7C, 0xF3, 0x3D, 0x60, 0x6C, 0x7B, 0xCA, 0xD3, 0x1F, 0x32, 0x65, 0x04, 0x28, + 0x64, 0xBE, 0x85, 0x9B, 0x2F, 0x59, 0x8A, 0xD7, 0xB0, 0x25, 0xAC, 0xAF, 0x12, 0x03, 0xE2, 0xF2, +}; + +/* D */ +static uint32_t D[16] = { + 0x44D7, 0x26BC, 0x626B, 0x135E, 0x5789, 0x35E2, 0x7135, 0x09AF, + 0x4D78, 0x2F13, 0x6BC4, 0x1AF1, 0x5E26, 0x3C4D, 0x789A, 0x47AC, +}; + +/* + * This is a method to calculate a + b mod (2 ^ 31 -1), + * described in ZUC specification. + */ +static ossl_inline uint32_t modular_add(uint32_t a, uint32_t b) +{ + uint32_t c = a + b; + + return (c & 0x7FFFFFFF) + (c >> 31); +} + +static ossl_inline uint32_t mulp2(uint32_t a, uint32_t b) +{ + return ((a << b) | (a >> (31 - b))) & 0x7FFFFFFF; +} + +/* LFSR with initialization mode */ +static void zuc_lfsr_init_mode(ZUC_KEY *zk, uint32_t u) +{ + uint32_t tmp, v, s16; + + v = mulp2(zk->s15, 15); + tmp = mulp2(zk->s13, 17); + v = modular_add(tmp, v); + tmp = mulp2(zk->s10, 21); + v = modular_add(tmp, v); + tmp = mulp2(zk->s4, 20); + v = modular_add(tmp, v); + tmp = mulp2(zk->s0, 8); + v = modular_add(tmp, v); + v = modular_add(zk->s0, v); + + /* s16... */ + s16 = modular_add(v, u); + + zk->s0 = zk->s1; + zk->s1 = zk->s2; + zk->s2 = zk->s3; + zk->s3 = zk->s4; + zk->s4 = zk->s5; + zk->s5 = zk->s6; + zk->s6 = zk->s7; + zk->s7 = zk->s8; + zk->s8 = zk->s9; + zk->s9 = zk->s10; + zk->s10 = zk->s11; + zk->s11 = zk->s12; + zk->s12 = zk->s13; + zk->s13 = zk->s14; + zk->s14 = zk->s15; + zk->s15 = s16; +} + +/* LFSR with work mode */ +static void zuc_lfsr_work_mode(ZUC_KEY *zk) +{ + uint32_t tmp, s16; + + s16 = zk->s0; + tmp = mulp2(zk->s0, 8); + s16 = modular_add(s16, tmp); + tmp = mulp2(zk->s4, 20); + s16 = modular_add(s16, tmp); + tmp = mulp2(zk->s10, 21); + s16 = modular_add(s16, tmp); + tmp = mulp2(zk->s13, 17); + s16 = modular_add(s16, tmp); + tmp = mulp2(zk->s15, 15); + s16 = modular_add(s16, tmp); + + zk->s0 = zk->s1; + zk->s1 = zk->s2; + zk->s2 = zk->s3; + zk->s3 = zk->s4; + zk->s4 = zk->s5; + zk->s5 = zk->s6; + zk->s6 = zk->s7; + zk->s7 = zk->s8; + zk->s8 = zk->s9; + zk->s9 = zk->s10; + zk->s10 = zk->s11; + zk->s11 = zk->s12; + zk->s12 = zk->s13; + zk->s13 = zk->s14; + zk->s14 = zk->s15; + zk->s15 = s16; +} + +/* bit reorganization */ +static ossl_inline void zuc_br(ZUC_KEY *zk) +{ + zk->X0 = ((zk->s15 & 0x7FFF8000) << 1) | (zk->s14 & 0xFFFF); + zk->X1 = ((zk->s11 & 0xFFFF) << 16) | (zk->s9 >> 15); + zk->X2 = ((zk->s7 & 0xFFFF) << 16) | (zk->s5 >> 15); + zk->X3 = ((zk->s2 & 0xFFFF) << 16) | (zk->s0 >> 15); +} + +#define ROT(a, k) (((a) << k) | ((a) >> (32 - k))) + +/* L1 */ +static ossl_inline uint32_t L1(uint32_t X) +{ + return (X ^ ROT(X, 2) ^ ROT(X, 10) ^ ROT(X, 18) ^ ROT(X, 24)); +} + +/* L2 */ +static ossl_inline uint32_t L2(uint32_t X) +{ + return (X ^ ROT(X, 8) ^ ROT(X, 14) ^ ROT(X, 22) ^ ROT(X, 30)); +} + +#define MAKEU32(a, b, c, d) \ + (((uint32_t)(a) << 24) | ((uint32_t)(b) << 16) | ((uint32_t)(c) << 8) | ((uint32_t)(d))) + +static ossl_inline uint32_t zuc_f_function(ZUC_KEY *zk) +{ + uint32_t W, W1, W2, u, v; + + W = (zk->X0 ^ zk->R1) + zk->R2; + W1 = zk->R1 + zk->X1; + W2 = zk->R2 ^ zk->X2; + u = L1((W1 << 16) | (W2 >> 16)); + v = L2((W2 << 16) | (W1 >> 16)); + /* S-Box... */ + zk->R1 = MAKEU32(S0[u >> 24], S1[(u >> 16) & 0xFF], + S0[(u >> 8) & 0xFF], S1[u & 0xFF]); + zk->R2 = MAKEU32(S0[v >> 24], S1[(v >> 16) & 0xFF], + S0[(v >> 8) & 0xFF], S1[v & 0xFF]); + return W; +} + +#define MAKEU31(a, b, c) (((uint32_t)(a) << 23) | ((uint32_t)(b) << 8) | (uint32_t)(c)) + +/* initialize */ +void ZUC_init(ZUC_KEY *zk) +{ + uint32_t w, count = 32; + + if (zk->inited) + return; + + /* expand key */ + zk->s0 = MAKEU31(zk->k[0], D[0], zk->iv[0]); + zk->s1 = MAKEU31(zk->k[1], D[1], zk->iv[1]); + zk->s2 = MAKEU31(zk->k[2], D[2], zk->iv[2]); + zk->s3 = MAKEU31(zk->k[3], D[3], zk->iv[3]); + zk->s4 = MAKEU31(zk->k[4], D[4], zk->iv[4]); + zk->s5 = MAKEU31(zk->k[5], D[5], zk->iv[5]); + zk->s6 = MAKEU31(zk->k[6], D[6], zk->iv[6]); + zk->s7 = MAKEU31(zk->k[7], D[7], zk->iv[7]); + zk->s8 = MAKEU31(zk->k[8], D[8], zk->iv[8]); + zk->s9 = MAKEU31(zk->k[9], D[9], zk->iv[9]); + zk->s10 = MAKEU31(zk->k[10], D[10], zk->iv[10]); + zk->s11 = MAKEU31(zk->k[11], D[11], zk->iv[11]); + zk->s12 = MAKEU31(zk->k[12], D[12], zk->iv[12]); + zk->s13 = MAKEU31(zk->k[13], D[13], zk->iv[13]); + zk->s14 = MAKEU31(zk->k[14], D[14], zk->iv[14]); + zk->s15 = MAKEU31(zk->k[15], D[15], zk->iv[15]); + + zk->R1 = 0; + zk->R2 = 0; + + while (count > 0) { + zuc_br(zk); + w = zuc_f_function(zk); + zuc_lfsr_init_mode(zk, w >> 1); + count--; + } + + /* this part is arranged in the working stage in the ZUC spec */ + zuc_br(zk); + zuc_f_function(zk); + zuc_lfsr_work_mode(zk); + + zk->inited = 1; + + return; +} + +int ZUC_generate_keystream(ZUC_KEY *zk) +{ + int i, len; + uint32_t keystream; + uint32_t pos = 0; + + if (!zk->inited) + return 0; + + zk->L = (sizeof(zk->keystream) * 8 + 31) / 32; + len = zk->L * sizeof(uint32_t); + + zk->keystream_tail[0] = zk->keystream[len - 4]; + zk->keystream_tail[1] = zk->keystream[len - 3]; + zk->keystream_tail[2] = zk->keystream[len - 2]; + zk->keystream_tail[3] = zk->keystream[len - 1]; + + for (i = 0; i < zk->L; i++) { + zuc_br(zk); + keystream = zuc_f_function(zk) ^ zk->X3; + zuc_lfsr_work_mode(zk); + + /* break 4-byte 'keystream' into key bytes */ + zk->keystream[pos] = (keystream >> 24) & 0xFF; + zk->keystream[pos + 1] = (keystream >> 16) & 0xFF; + zk->keystream[pos + 2] = (keystream >> 8) & 0xFF; + zk->keystream[pos + 3] = keystream & 0xFF; + + pos += 4; + } + + zk->keystream_tail[4] = zk->keystream[0]; + zk->keystream_tail[5] = zk->keystream[1]; + zk->keystream_tail[6] = zk->keystream[2]; + zk->keystream_tail[7] = zk->keystream[3]; + + zk->keystream_len += len; + + return 1; +} + +void ZUC_destroy_keystream(ZUC_KEY *zk) +{ + return; +} + +int ZUC_keystream_get_word(ZUC_KEY *zk, int i) +{ + uint32_t word = 0, ti, j = i / 8, k, len; + uint8_t *data; + + if (zk == NULL) + return 0; + + len = zk->L * sizeof(uint32_t); + data = zk->keystream; + k = j % len; + + if ((k + 4) >= len) { + data = zk->keystream_tail; + j = k + 4 - len; + } else { + j = k; + } + + ti = i % 8; + if (ti == 0) { + word = (uint32_t)data[j] << 24; + word |= ((uint32_t)data[j + 1] << 16); + word |= ((uint32_t)data[j + 2] << 8); + word |= data[j + 3]; + } else { + word = (uint32_t)((uint8_t)(data[j] << ti) | (uint8_t)(data[j + 1] >> (8 - ti))) << 24; + word |= (uint32_t)((uint8_t)(data[j + 1] << ti) | (uint8_t)(data[j + 2] >> (8 - ti))) << 16; + word |= (uint32_t)((uint8_t)(data[j + 2] << ti) | (uint8_t)(data[j + 3] >> (8 - ti))) << 8; + word |= (data[j + 3] << ti) | (data[j + 4] >> (8 - ti)); + } + + return word; +} + +int ZUC_keystream_get_byte(ZUC_KEY *zk, int i) +{ + return zk->keystream[i % (zk->L * sizeof(uint32_t))]; +} diff --git a/openssl/src/providers/baseprov.c b/openssl/src/providers/baseprov.c index 6b8de7cb3..44c6e8b7e 100644 --- a/openssl/src/providers/baseprov.c +++ b/openssl/src/providers/baseprov.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,7 +19,7 @@ #include "prov/providercommon.h" #include "prov/implementations.h" #include "prov/provider_util.h" -#include "prov/names.h" +#include "internal/nelem.h" /* * Forward declarations to ensure that interface functions are correctly @@ -91,11 +91,6 @@ static const OSSL_ALGORITHM base_store[] = { #undef STORE }; -static const OSSL_ALGORITHM base_rands[] = { - { PROV_NAMES_SEED_SRC, "provider=base", ossl_seed_src_functions }, - { NULL, NULL, NULL } -}; - static const OSSL_ALGORITHM *base_query(void *provctx, int operation_id, int *no_cache) { @@ -107,8 +102,6 @@ static const OSSL_ALGORITHM *base_query(void *provctx, int operation_id, return base_decoder; case OSSL_OP_STORE: return base_store; - case OSSL_OP_RAND: - return base_rands; } return NULL; } @@ -126,7 +119,7 @@ static const OSSL_DISPATCH base_dispatch_table[] = { (void (*)(void))base_gettable_params }, { OSSL_FUNC_PROVIDER_GET_PARAMS, (void (*)(void))base_get_params }, { OSSL_FUNC_PROVIDER_QUERY_OPERATION, (void (*)(void))base_query }, - OSSL_DISPATCH_END + { 0, NULL } }; OSSL_provider_init_fn ossl_base_provider_init; diff --git a/openssl/src/providers/common/capabilities.c b/openssl/src/providers/common/capabilities.c index f7234615e..769ce4d8f 100644 --- a/openssl/src/providers/common/capabilities.c +++ b/openssl/src/providers/common/capabilities.c @@ -30,7 +30,7 @@ typedef struct tls_group_constants_st { int maxdtls; /* Maximum DTLS version (or 0 for undefined) */ } TLS_GROUP_CONSTANTS; -static const TLS_GROUP_CONSTANTS group_list[] = { +static const TLS_GROUP_CONSTANTS group_list[36] = { { OSSL_TLS_GROUP_ID_sect163k1, 80, TLS1_VERSION, TLS1_2_VERSION, DTLS1_VERSION, DTLS1_2_VERSION }, { OSSL_TLS_GROUP_ID_sect163r1, 80, TLS1_VERSION, TLS1_2_VERSION, @@ -86,15 +86,15 @@ static const TLS_GROUP_CONSTANTS group_list[] = { DTLS1_VERSION, DTLS1_2_VERSION }, { OSSL_TLS_GROUP_ID_x25519, 128, TLS1_VERSION, 0, DTLS1_VERSION, 0 }, { OSSL_TLS_GROUP_ID_x448, 224, TLS1_VERSION, 0, DTLS1_VERSION, 0 }, - { OSSL_TLS_GROUP_ID_brainpoolP256r1_tls13, 128, TLS1_3_VERSION, 0, -1, -1 }, - { OSSL_TLS_GROUP_ID_brainpoolP384r1_tls13, 192, TLS1_3_VERSION, 0, -1, -1 }, - { OSSL_TLS_GROUP_ID_brainpoolP512r1_tls13, 256, TLS1_3_VERSION, 0, -1, -1 }, /* Security bit values as given by BN_security_bits() */ { OSSL_TLS_GROUP_ID_ffdhe2048, 112, TLS1_3_VERSION, 0, -1, -1 }, { OSSL_TLS_GROUP_ID_ffdhe3072, 128, TLS1_3_VERSION, 0, -1, -1 }, { OSSL_TLS_GROUP_ID_ffdhe4096, 128, TLS1_3_VERSION, 0, -1, -1 }, { OSSL_TLS_GROUP_ID_ffdhe6144, 128, TLS1_3_VERSION, 0, -1, -1 }, { OSSL_TLS_GROUP_ID_ffdhe8192, 192, TLS1_3_VERSION, 0, -1, -1 }, +#if !defined(OPENSSL_NO_SM2) && !defined(FIPS_MODULE) + { OSSL_TLS_GROUP_ID_sm2, 128, NTLS1_1_VERSION, 0, -1, -1 }, +#endif }; #define TLS_GROUP_ENTRY(tlsname, realname, algorithm, idx) \ @@ -192,19 +192,17 @@ static const OSSL_PARAM param_group_list[][10] = { # endif TLS_GROUP_ENTRY("x25519", "X25519", "X25519", 28), TLS_GROUP_ENTRY("x448", "X448", "X448", 29), -# ifndef FIPS_MODULE - TLS_GROUP_ENTRY("brainpoolP256r1tls13", "brainpoolP256r1", "EC", 30), - TLS_GROUP_ENTRY("brainpoolP384r1tls13", "brainpoolP384r1", "EC", 31), - TLS_GROUP_ENTRY("brainpoolP512r1tls13", "brainpoolP512r1", "EC", 32), -# endif # endif /* OPENSSL_NO_EC */ # ifndef OPENSSL_NO_DH /* Security bit values for FFDHE groups are as per RFC 7919 */ - TLS_GROUP_ENTRY("ffdhe2048", "ffdhe2048", "DH", 33), - TLS_GROUP_ENTRY("ffdhe3072", "ffdhe3072", "DH", 34), - TLS_GROUP_ENTRY("ffdhe4096", "ffdhe4096", "DH", 35), - TLS_GROUP_ENTRY("ffdhe6144", "ffdhe6144", "DH", 36), - TLS_GROUP_ENTRY("ffdhe8192", "ffdhe8192", "DH", 37), + TLS_GROUP_ENTRY("ffdhe2048", "ffdhe2048", "DH", 30), + TLS_GROUP_ENTRY("ffdhe3072", "ffdhe3072", "DH", 31), + TLS_GROUP_ENTRY("ffdhe4096", "ffdhe4096", "DH", 32), + TLS_GROUP_ENTRY("ffdhe6144", "ffdhe6144", "DH", 33), + TLS_GROUP_ENTRY("ffdhe8192", "ffdhe8192", "DH", 34), +# endif +# if !defined(OPENSSL_NO_SM2) && !defined(FIPS_MODULE) + TLS_GROUP_ENTRY("curveSM2", "SM2", "SM2", 35), # endif }; #endif /* !defined(OPENSSL_NO_EC) || !defined(OPENSSL_NO_DH) */ diff --git a/openssl/src/providers/common/der/der_rsa_key.c b/openssl/src/providers/common/der/der_rsa_key.c index 893970575..81ab0346c 100644 --- a/openssl/src/providers/common/der/der_rsa_key.c +++ b/openssl/src/providers/common/der/der_rsa_key.c @@ -305,15 +305,6 @@ int ossl_DER_w_RSASSA_PSS_params(WPACKET *pkt, int tag, saltlen = ossl_rsa_pss_params_30_saltlen(pss); trailerfield = ossl_rsa_pss_params_30_trailerfield(pss); - if (saltlen < 0) { - ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_SALT_LENGTH); - return 0; - } - if (trailerfield != 1) { - ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_TRAILER); - return 0; - } - /* Getting default values */ default_hashalg_nid = ossl_rsa_pss_params_30_hashalg(NULL); default_saltlen = ossl_rsa_pss_params_30_saltlen(NULL); @@ -347,8 +338,8 @@ int ossl_DER_w_RSASSA_PSS_params(WPACKET *pkt, int tag, return ossl_DER_w_begin_sequence(pkt, tag) && (trailerfield == default_trailerfield - || ossl_DER_w_uint32(pkt, 3, (uint32_t)trailerfield)) - && (saltlen == default_saltlen || ossl_DER_w_uint32(pkt, 2, (uint32_t)saltlen)) + || ossl_DER_w_ulong(pkt, 3, trailerfield)) + && (saltlen == default_saltlen || ossl_DER_w_ulong(pkt, 2, saltlen)) && DER_w_MaskGenAlgorithm(pkt, 1, pss) && (hashalg_nid == default_hashalg_nid || ossl_DER_w_precompiled(pkt, 0, hashalg, hashalg_sz)) diff --git a/openssl/src/providers/common/der/der_rsa_sig.c b/openssl/src/providers/common/der/der_rsa_sig.c index 08d00641e..a94b6b78e 100644 --- a/openssl/src/providers/common/der/der_rsa_sig.c +++ b/openssl/src/providers/common/der/der_rsa_sig.c @@ -21,8 +21,6 @@ ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_384 #define ossl_der_oid_sha3_512WithRSAEncryption \ ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512 -#define ossl_der_oid_mdc2WithRSAEncryption \ - ossl_der_oid_mdc2WithRSASignature #define MD_with_RSA_CASE(name, var) \ case NID_##name: \ @@ -38,11 +36,7 @@ int ossl_DER_w_algorithmIdentifier_MDWithRSAEncryption(WPACKET *pkt, int tag, switch (mdnid) { #ifndef FIPS_MODULE - MD_with_RSA_CASE(md2, precompiled); MD_with_RSA_CASE(md5, precompiled); - MD_with_RSA_CASE(md4, precompiled); - MD_with_RSA_CASE(ripemd160, precompiled); - MD_with_RSA_CASE(mdc2, precompiled); #endif MD_with_RSA_CASE(sha1, precompiled); MD_with_RSA_CASE(sha224, precompiled); diff --git a/openssl/src/providers/common/der/gen/darwin_arm64/der_digests_gen.c b/openssl/src/providers/common/der/gen/darwin_arm64/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/darwin_arm64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/darwin_arm64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/darwin_arm64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/darwin_arm64/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/darwin_arm64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/darwin_arm64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/darwin_x64/der_digests_gen.c b/openssl/src/providers/common/der/gen/darwin_x64/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/darwin_x64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/darwin_x64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/darwin_x64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/darwin_x64/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/darwin_x64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/darwin_x64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/linux_arm/der_digests_gen.c b/openssl/src/providers/common/der/gen/linux_arm/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/linux_arm/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/linux_arm/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/linux_arm/der_rsa_gen.c b/openssl/src/providers/common/der/gen/linux_arm/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/linux_arm/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/linux_arm/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/linux_arm64/der_digests_gen.c b/openssl/src/providers/common/der/gen/linux_arm64/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/linux_arm64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/linux_arm64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/linux_arm64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/linux_arm64/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/linux_arm64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/linux_arm64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/linux_ia32/der_digests_gen.c b/openssl/src/providers/common/der/gen/linux_ia32/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/linux_ia32/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/linux_ia32/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/linux_ia32/der_rsa_gen.c b/openssl/src/providers/common/der/gen/linux_ia32/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/linux_ia32/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/linux_ia32/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/linux_loong64/der_digests_gen.c b/openssl/src/providers/common/der/gen/linux_loong64/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/linux_loong64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/linux_loong64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/linux_loong64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/linux_loong64/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/linux_loong64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/linux_loong64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/linux_mips64/der_digests_gen.c b/openssl/src/providers/common/der/gen/linux_mips64/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/linux_mips64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/linux_mips64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/linux_mips64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/linux_mips64/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/linux_mips64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/linux_mips64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/linux_ppc64/der_digests_gen.c b/openssl/src/providers/common/der/gen/linux_ppc64/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/linux_ppc64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/linux_ppc64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/linux_ppc64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/linux_ppc64/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/linux_ppc64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/linux_ppc64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/linux_riscv64/der_digests_gen.c b/openssl/src/providers/common/der/gen/linux_riscv64/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/linux_riscv64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/linux_riscv64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/linux_riscv64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/linux_riscv64/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/linux_riscv64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/linux_riscv64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/linux_x64/der_digests_gen.c b/openssl/src/providers/common/der/gen/linux_x64/der_digests_gen.c index 5125408e5..33ce5643c 100644 --- a/openssl/src/providers/common/der/gen/linux_x64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/linux_x64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/linux_x64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/linux_x64/der_rsa_gen.c index 173c6a4c6..0a1df54a7 100644 --- a/openssl/src/providers/common/der/gen/linux_x64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/linux_x64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/windows_arm64/der_digests_gen.c b/openssl/src/providers/common/der/gen/windows_arm64/der_digests_gen.c index 5951f8c13..065e22abd 100644 --- a/openssl/src/providers/common/der/gen/windows_arm64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/windows_arm64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/windows_arm64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/windows_arm64/der_rsa_gen.c index ec323c4a1..e2aae54a6 100644 --- a/openssl/src/providers/common/der/gen/windows_arm64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/windows_arm64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/windows_ia32/der_digests_gen.c b/openssl/src/providers/common/der/gen/windows_ia32/der_digests_gen.c index 5951f8c13..065e22abd 100644 --- a/openssl/src/providers/common/der/gen/windows_ia32/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/windows_ia32/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/windows_ia32/der_rsa_gen.c b/openssl/src/providers/common/der/gen/windows_ia32/der_rsa_gen.c index ec323c4a1..e2aae54a6 100644 --- a/openssl/src/providers/common/der/gen/windows_ia32/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/windows_ia32/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/gen/windows_x64/der_digests_gen.c b/openssl/src/providers/common/der/gen/windows_x64/der_digests_gen.c index 5951f8c13..065e22abd 100644 --- a/openssl/src/providers/common/der/gen/windows_x64/der_digests_gen.c +++ b/openssl/src/providers/common/der/gen/windows_x64/der_digests_gen.c @@ -30,14 +30,6 @@ const unsigned char ossl_der_oid_id_sha1[DER_OID_SZ_id_sha1] = { DER_OID_V_id_sha1 }; -/* - * id-md2 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 2 } - */ -const unsigned char ossl_der_oid_id_md2[DER_OID_SZ_id_md2] = { - DER_OID_V_id_md2 -}; - /* * id-md5 OBJECT IDENTIFIER ::= { * iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } diff --git a/openssl/src/providers/common/der/gen/windows_x64/der_rsa_gen.c b/openssl/src/providers/common/der/gen/windows_x64/der_rsa_gen.c index ec323c4a1..e2aae54a6 100644 --- a/openssl/src/providers/common/der/gen/windows_x64/der_rsa_gen.c +++ b/openssl/src/providers/common/der/gen/windows_x64/der_rsa_gen.c @@ -49,13 +49,6 @@ const unsigned char ossl_der_oid_id_RSASSA_PSS[DER_OID_SZ_id_RSASSA_PSS] = { DER_OID_V_id_RSASSA_PSS }; -/* - * md2WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 2 } - */ -const unsigned char ossl_der_oid_md2WithRSAEncryption[DER_OID_SZ_md2WithRSAEncryption] = { - DER_OID_V_md2WithRSAEncryption -}; - /* * md5WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 4 } */ @@ -147,28 +140,3 @@ const unsigned char ossl_der_oid_id_rsassa_pkcs1_v1_5_with_sha3_512[DER_OID_SZ_i DER_OID_V_id_rsassa_pkcs1_v1_5_with_sha3_512 }; -/* - * md4WithRSAEncryption OBJECT IDENTIFIER ::= { pkcs-1 3 } - */ -const unsigned char ossl_der_oid_md4WithRSAEncryption[DER_OID_SZ_md4WithRSAEncryption] = { - DER_OID_V_md4WithRSAEncryption -}; - -/* - * ripemd160WithRSAEncryption OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) teletrust(36) algorithm(3) signatureAlgorithm(3) rsaSignature(1) 2 - * } - */ -const unsigned char ossl_der_oid_ripemd160WithRSAEncryption[DER_OID_SZ_ripemd160WithRSAEncryption] = { - DER_OID_V_ripemd160WithRSAEncryption -}; - -/* - * mdc2WithRSASignature OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) mdc2WithRSASignature(14) - * } - */ -const unsigned char ossl_der_oid_mdc2WithRSASignature[DER_OID_SZ_mdc2WithRSASignature] = { - DER_OID_V_mdc2WithRSASignature -}; - diff --git a/openssl/src/providers/common/der/local.h b/openssl/src/providers/common/der/local.h new file mode 100644 index 000000000..6dd63389d --- /dev/null +++ b/openssl/src/providers/common/der/local.h @@ -0,0 +1,20 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +extern const unsigned char ossl_der_aid_sha1Identifier[]; +extern const unsigned char ossl_der_aid_sha224Identifier[]; +extern const unsigned char ossl_der_aid_sha256Identifier[]; +extern const unsigned char ossl_der_aid_sha384Identifier[]; +extern const unsigned char ossl_der_aid_sha512Identifier[]; +extern const unsigned char ossl_der_aid_sha512_224Identifier[]; +extern const unsigned char ossl_der_aid_sha512_256Identifier[]; diff --git a/openssl/src/providers/common/include/prov/fipscommon.h b/openssl/src/providers/common/include/prov/fipscommon.h deleted file mode 100644 index 45ed248e9..000000000 --- a/openssl/src/providers/common/include/prov/fipscommon.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifdef FIPS_MODULE -# include - -int FIPS_security_check_enabled(OSSL_LIB_CTX *libctx); -int FIPS_tls_prf_ems_check(OSSL_LIB_CTX *libctx); -int FIPS_restricted_drbg_digests_enabled(OSSL_LIB_CTX *libctx); - -#endif diff --git a/openssl/src/providers/common/include/prov/proverr.h b/openssl/src/providers/common/include/prov/proverr.h index 69e14465c..5084af201 100644 --- a/openssl/src/providers/common/include/prov/proverr.h +++ b/openssl/src/providers/common/include/prov/proverr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/providers/common/include/prov/provider_util.h b/openssl/src/providers/common/include/prov/provider_util.h index 5511565e7..dfe91f29b 100644 --- a/openssl/src/providers/common/include/prov/provider_util.h +++ b/openssl/src/providers/common/include/prov/provider_util.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -136,7 +136,3 @@ typedef struct ag_capable_st { */ void ossl_prov_cache_exported_algorithms(const OSSL_ALGORITHM_CAPABLE *in, OSSL_ALGORITHM *out); - -/* Duplicate a lump of memory safely */ -int ossl_prov_memdup(const void *src, size_t src_len, - unsigned char **dest, size_t *dest_len); diff --git a/openssl/src/providers/common/include/prov/securitycheck.h b/openssl/src/providers/common/include/prov/securitycheck.h index 611c6d531..4a7f85f71 100644 --- a/openssl/src/providers/common/include/prov/securitycheck.h +++ b/openssl/src/providers/common/include/prov/securitycheck.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -28,4 +28,3 @@ int ossl_digest_get_approved_nid(const EVP_MD *md); int ossl_digest_rsa_sign_get_md_nid(OSSL_LIB_CTX *ctx, const EVP_MD *md, int sha1_allowed); int ossl_securitycheck_enabled(OSSL_LIB_CTX *libctx); -int ossl_tls1_prf_ems_check_enabled(OSSL_LIB_CTX *libctx); diff --git a/openssl/src/providers/common/provider_err.c b/openssl/src/providers/common/provider_err.c index 611ec847c..344c12211 100644 --- a/openssl/src/providers/common/provider_err.c +++ b/openssl/src/providers/common/provider_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,7 +33,6 @@ static const ERR_STRING_DATA PROV_str_reasons[] = { "derivation function init failed"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_DIGEST_NOT_ALLOWED), "digest not allowed"}, - {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_EMS_NOT_ENABLED), "ems not enabled"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_ENTROPY_SOURCE_STRENGTH_TOO_WEAK), "entropy source strength too weak"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_ERROR_INSTANTIATING_DRBG), @@ -68,7 +67,6 @@ static const ERR_STRING_DATA PROV_str_reasons[] = { {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INSUFFICIENT_DRBG_STRENGTH), "insufficient drbg strength"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_AAD), "invalid aad"}, - {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_AEAD), "invalid aead"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_CONFIG_DATA), "invalid config data"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_CONSTANT_LENGTH), @@ -87,13 +85,10 @@ static const ERR_STRING_DATA PROV_str_reasons[] = { {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_ITERATION_COUNT), "invalid iteration count"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_IV_LENGTH), "invalid iv length"}, - {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_KDF), "invalid kdf"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_KEY), "invalid key"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_KEY_LENGTH), "invalid key length"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_MAC), "invalid mac"}, - {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_MEMORY_SIZE), - "invalid memory size"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_MGF1_MD), "invalid mgf1 md"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_MODE), "invalid mode"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_OUTPUT_LENGTH), @@ -111,8 +106,6 @@ static const ERR_STRING_DATA PROV_str_reasons[] = { {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_TAG), "invalid tag"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_TAG_LENGTH), "invalid tag length"}, - {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_THREAD_POOL_SIZE), - "invalid thread pool size"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_UKM_LENGTH), "invalid ukm length"}, {ERR_PACK(ERR_LIB_PROV, 0, PROV_R_INVALID_X931_DIGEST), diff --git a/openssl/src/providers/common/provider_seeding.c b/openssl/src/providers/common/provider_seeding.c index 544344f30..0edbb8763 100644 --- a/openssl/src/providers/common/provider_seeding.c +++ b/openssl/src/providers/common/provider_seeding.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,34 +9,11 @@ #include #include "prov/seeding.h" -#include "prov/providercommon.h" static OSSL_FUNC_get_entropy_fn *c_get_entropy = NULL; -static OSSL_FUNC_get_user_entropy_fn *c_get_user_entropy = NULL; static OSSL_FUNC_cleanup_entropy_fn *c_cleanup_entropy = NULL; -static OSSL_FUNC_cleanup_user_entropy_fn *c_cleanup_user_entropy = NULL; static OSSL_FUNC_get_nonce_fn *c_get_nonce = NULL; -static OSSL_FUNC_get_user_nonce_fn *c_get_user_nonce = NULL; static OSSL_FUNC_cleanup_nonce_fn *c_cleanup_nonce = NULL; -static OSSL_FUNC_cleanup_user_nonce_fn *c_cleanup_user_nonce = NULL; - -#ifdef FIPS_MODULE -/* - * The FIPS provider uses an internal library context which is what the - * passed provider context references. Since the seed source is external - * to the FIPS provider, this is the wrong one. We need to convert this - * to the correct core handle before up-calling libcrypto. - */ -# define CORE_HANDLE(provctx) \ - FIPS_get_core_handle(ossl_prov_ctx_get0_libctx(provctx)) -#else -/* - * The non-FIPS path *should* be unused because the full DRBG chain including - * seed source is instantiated. However, that might not apply for third - * party providers, so this is retained for compatibility. - */ -# define CORE_HANDLE(provctx) ossl_prov_ctx_get0_handle(provctx) -#endif int ossl_prov_seeding_from_dispatch(const OSSL_DISPATCH *fns) { @@ -52,27 +29,15 @@ int ossl_prov_seeding_from_dispatch(const OSSL_DISPATCH *fns) case OSSL_FUNC_GET_ENTROPY: set_func(c_get_entropy, OSSL_FUNC_get_entropy(fns)); break; - case OSSL_FUNC_GET_USER_ENTROPY: - set_func(c_get_user_entropy, OSSL_FUNC_get_user_entropy(fns)); - break; case OSSL_FUNC_CLEANUP_ENTROPY: set_func(c_cleanup_entropy, OSSL_FUNC_cleanup_entropy(fns)); break; - case OSSL_FUNC_CLEANUP_USER_ENTROPY: - set_func(c_cleanup_user_entropy, OSSL_FUNC_cleanup_user_entropy(fns)); - break; case OSSL_FUNC_GET_NONCE: set_func(c_get_nonce, OSSL_FUNC_get_nonce(fns)); break; - case OSSL_FUNC_GET_USER_NONCE: - set_func(c_get_user_nonce, OSSL_FUNC_get_user_nonce(fns)); - break; case OSSL_FUNC_CLEANUP_NONCE: set_func(c_cleanup_nonce, OSSL_FUNC_cleanup_nonce(fns)); break; - case OSSL_FUNC_CLEANUP_USER_NONCE: - set_func(c_cleanup_user_nonce, OSSL_FUNC_cleanup_user_nonce(fns)); - break; } #undef set_func } @@ -82,45 +47,31 @@ int ossl_prov_seeding_from_dispatch(const OSSL_DISPATCH *fns) size_t ossl_prov_get_entropy(PROV_CTX *prov_ctx, unsigned char **pout, int entropy, size_t min_len, size_t max_len) { - const OSSL_CORE_HANDLE *handle = CORE_HANDLE(prov_ctx); - - if (c_get_user_entropy != NULL) - return c_get_user_entropy(handle, pout, entropy, min_len, max_len); - if (c_get_entropy != NULL) - return c_get_entropy(handle, pout, entropy, min_len, max_len); - return 0; + if (c_get_entropy == NULL) + return 0; + return c_get_entropy(ossl_prov_ctx_get0_handle(prov_ctx), + pout, entropy, min_len, max_len); } void ossl_prov_cleanup_entropy(PROV_CTX *prov_ctx, unsigned char *buf, size_t len) { - const OSSL_CORE_HANDLE *handle = CORE_HANDLE(prov_ctx); - - if (c_cleanup_user_entropy != NULL) - c_cleanup_user_entropy(handle, buf, len); - else if (c_cleanup_entropy != NULL) - c_cleanup_entropy(handle, buf, len); + if (c_cleanup_entropy != NULL) + c_cleanup_entropy(ossl_prov_ctx_get0_handle(prov_ctx), buf, len); } size_t ossl_prov_get_nonce(PROV_CTX *prov_ctx, unsigned char **pout, size_t min_len, size_t max_len, - const void *salt, size_t salt_len) + const void *salt,size_t salt_len) { - const OSSL_CORE_HANDLE *handle = CORE_HANDLE(prov_ctx); - - if (c_get_user_nonce != NULL) - return c_get_user_nonce(handle, pout, min_len, max_len, salt, salt_len); - if (c_get_nonce != NULL) - return c_get_nonce(handle, pout, min_len, max_len, salt, salt_len); - return 0; + if (c_get_nonce == NULL) + return 0; + return c_get_nonce(ossl_prov_ctx_get0_handle(prov_ctx), pout, + min_len, max_len, salt, salt_len); } void ossl_prov_cleanup_nonce(PROV_CTX *prov_ctx, unsigned char *buf, size_t len) { - const OSSL_CORE_HANDLE *handle = CORE_HANDLE(prov_ctx); - - if (c_cleanup_user_nonce != NULL) - c_cleanup_user_nonce(handle, buf, len); - else if (c_cleanup_nonce != NULL) - c_cleanup_nonce(handle, buf, len); + if (c_cleanup_nonce != NULL) + c_cleanup_nonce(ossl_prov_ctx_get0_handle(prov_ctx), buf, len); } diff --git a/openssl/src/providers/common/provider_util.c b/openssl/src/providers/common/provider_util.c index 2473754d2..58d4db337 100644 --- a/openssl/src/providers/common/provider_util.c +++ b/openssl/src/providers/common/provider_util.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -19,6 +19,7 @@ # include "crypto/evp.h" #endif #include "prov/provider_util.h" +#include "internal/nelem.h" void ossl_prov_cipher_reset(PROV_CIPHER *pc) { @@ -164,7 +165,7 @@ int ossl_prov_digest_copy(PROV_DIGEST *dst, const PROV_DIGEST *src) } const EVP_MD *ossl_prov_digest_fetch(PROV_DIGEST *pd, OSSL_LIB_CTX *libctx, - const char *mdname, const char *propquery) + const char *mdname, const char *propquery) { EVP_MD_free(pd->alloc_md); pd->md = pd->alloc_md = EVP_MD_fetch(libctx, mdname, propquery); @@ -350,18 +351,3 @@ void ossl_prov_cache_exported_algorithms(const OSSL_ALGORITHM_CAPABLE *in, out[j++] = in[i].alg; } } - -/* Duplicate a lump of memory safely */ -int ossl_prov_memdup(const void *src, size_t src_len, - unsigned char **dest, size_t *dest_len) -{ - if (src != NULL) { - if ((*dest = OPENSSL_memdup(src, src_len)) == NULL) - return 0; - *dest_len = src_len; - } else { - *dest = NULL; - *dest_len = 0; - } - return 1; -} diff --git a/openssl/src/providers/common/securitycheck.c b/openssl/src/providers/common/securitycheck.c index 0d3acdbe5..699ada7c5 100644 --- a/openssl/src/providers/common/securitycheck.c +++ b/openssl/src/providers/common/securitycheck.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -167,25 +167,17 @@ int ossl_dsa_check_key(OSSL_LIB_CTX *ctx, const DSA *dsa, int sign) /* * For Digital signature verification DSA keys with < 112 bits of - * security strength, are still allowed for legacy - * use. The bounds given in SP 800-131Ar2 - Table 2 are - * (512 <= L < 2048 or 160 <= N < 224). - * - * We are a little stricter and insist that both minimums are met. - * For example a L = 256, N = 160 key *would* be allowed by SP 800-131Ar2 - * but we don't. + * security strength (i.e L < 2048 bits), are still allowed for legacy + * use. The bounds given in SP800 131Ar2 - Table 2 are + * (512 <= L < 2048 and 160 <= N < 224) */ - if (!sign) { - if (L < 512 || N < 160) - return 0; - if (L < 2048 || N < 224) - return 1; - } + if (!sign && L < 2048) + return (L >= 512 && N >= 160 && N < 224); /* Valid sizes for both sign and verify */ - if (L == 2048 && (N == 224 || N == 256)) /* 112 bits */ + if (L == 2048 && (N == 224 || N == 256)) return 1; - return (L == 3072 && N == 256); /* 128 bits */ + return (L == 3072 && N == 256); } # endif /* OPENSSL_NO_FIPS_SECURITYCHECKS */ return 1; diff --git a/openssl/src/providers/common/securitycheck_default.c b/openssl/src/providers/common/securitycheck_default.c index 246323493..ef82aab9c 100644 --- a/openssl/src/providers/common/securitycheck_default.c +++ b/openssl/src/providers/common/securitycheck_default.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,12 +22,6 @@ int ossl_securitycheck_enabled(OSSL_LIB_CTX *libctx) return 0; } -/* Disable the ems check in the default provider */ -int ossl_tls1_prf_ems_check_enabled(OSSL_LIB_CTX *libctx) -{ - return 0; -} - int ossl_digest_rsa_sign_get_md_nid(OSSL_LIB_CTX *ctx, const EVP_MD *md, ossl_unused int sha1_allowed) { @@ -36,10 +30,6 @@ int ossl_digest_rsa_sign_get_md_nid(OSSL_LIB_CTX *ctx, const EVP_MD *md, static const OSSL_ITEM name_to_nid[] = { { NID_md5, OSSL_DIGEST_NAME_MD5 }, { NID_md5_sha1, OSSL_DIGEST_NAME_MD5_SHA1 }, - { NID_md2, OSSL_DIGEST_NAME_MD2 }, - { NID_md4, OSSL_DIGEST_NAME_MD4 }, - { NID_mdc2, OSSL_DIGEST_NAME_MDC2 }, - { NID_ripemd160, OSSL_DIGEST_NAME_RIPEMD160 }, }; mdnid = ossl_digest_get_approved_nid_with_sha1(ctx, md, 1); diff --git a/openssl/src/providers/common/securitycheck_fips.c b/openssl/src/providers/common/securitycheck_fips.c index d1262d879..b7659bd39 100644 --- a/openssl/src/providers/common/securitycheck_fips.c +++ b/openssl/src/providers/common/securitycheck_fips.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,7 +18,8 @@ #include #include #include "prov/securitycheck.h" -#include "prov/fipscommon.h" + +int FIPS_security_check_enabled(OSSL_LIB_CTX *libctx); int ossl_securitycheck_enabled(OSSL_LIB_CTX *libctx) { @@ -29,11 +30,6 @@ int ossl_securitycheck_enabled(OSSL_LIB_CTX *libctx) #endif /* OPENSSL_NO_FIPS_SECURITYCHECKS */ } -int ossl_tls1_prf_ems_check_enabled(OSSL_LIB_CTX *libctx) -{ - return FIPS_tls_prf_ems_check(libctx); -} - int ossl_digest_rsa_sign_get_md_nid(OSSL_LIB_CTX *ctx, const EVP_MD *md, int sha1_allowed) { diff --git a/openssl/src/providers/decoders.inc b/openssl/src/providers/decoders.inc index 0191aa771..e2524b6b6 100644 --- a/openssl/src/providers/decoders.inc +++ b/openssl/src/providers/decoders.inc @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,6 +22,7 @@ #define DECODER_STRUCTURE_DHX "dhx" #define DECODER_STRUCTURE_DSA "dsa" #define DECODER_STRUCTURE_EC "ec" +#define DECODER_STRUCTURE_SM2 "sm2" #define DECODER_STRUCTURE_RSA "rsa" /* Arguments are prefixed with '_' to avoid build breaks on certain platforms */ @@ -58,7 +59,6 @@ DECODER_w_structure("EC", der, PrivateKeyInfo, ec, yes), DECODER_w_structure("EC", der, SubjectPublicKeyInfo, ec, yes), DECODER_w_structure("EC", der, type_specific_no_pub, ec, yes), DECODER_w_structure("EC", der, EC, ec, yes), -# ifndef OPENSSL_NO_ECX DECODER_w_structure("ED25519", der, PrivateKeyInfo, ed25519, yes), DECODER_w_structure("ED25519", der, SubjectPublicKeyInfo, ed25519, yes), DECODER_w_structure("ED448", der, PrivateKeyInfo, ed448, yes), @@ -67,7 +67,6 @@ DECODER_w_structure("X25519", der, PrivateKeyInfo, x25519, yes), DECODER_w_structure("X25519", der, SubjectPublicKeyInfo, x25519, yes), DECODER_w_structure("X448", der, PrivateKeyInfo, x448, yes), DECODER_w_structure("X448", der, SubjectPublicKeyInfo, x448, yes), -# endif # ifndef OPENSSL_NO_SM2 DECODER_w_structure("SM2", der, PrivateKeyInfo, sm2, no), DECODER_w_structure("SM2", der, SubjectPublicKeyInfo, sm2, no), diff --git a/openssl/src/providers/defltprov.c b/openssl/src/providers/defltprov.c index f02e04835..5487a0cad 100644 --- a/openssl/src/providers/defltprov.c +++ b/openssl/src/providers/defltprov.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -78,7 +78,7 @@ static int deflt_get_params(void *provctx, OSSL_PARAM params[]) * ALGNAME[VERSION?][-SUBNAME[VERSION?]?][-SIZE?][-MODE?] * * VERSION is only present if there are multiple versions of - * an alg (MD2, MD4, MD5). It may be omitted if there is only + * an alg (MD series). It may be omitted if there is only * one version (if a subsequent version is released in the future, * we can always change the canonical name, and add the old name * as an alias). @@ -103,7 +103,6 @@ static const OSSL_ALGORITHM deflt_digests[] = { { PROV_NAMES_SHA1, "provider=default", ossl_sha1_functions }, { PROV_NAMES_SHA2_224, "provider=default", ossl_sha224_functions }, { PROV_NAMES_SHA2_256, "provider=default", ossl_sha256_functions }, - { PROV_NAMES_SHA2_256_192, "provider=default", ossl_sha256_192_functions }, { PROV_NAMES_SHA2_384, "provider=default", ossl_sha384_functions }, { PROV_NAMES_SHA2_512, "provider=default", ossl_sha512_functions }, { PROV_NAMES_SHA2_512_224, "provider=default", ossl_sha512_224_functions }, @@ -115,11 +114,6 @@ static const OSSL_ALGORITHM deflt_digests[] = { { PROV_NAMES_SHA3_384, "provider=default", ossl_sha3_384_functions }, { PROV_NAMES_SHA3_512, "provider=default", ossl_sha3_512_functions }, - { PROV_NAMES_KECCAK_224, "provider=default", ossl_keccak_224_functions }, - { PROV_NAMES_KECCAK_256, "provider=default", ossl_keccak_256_functions }, - { PROV_NAMES_KECCAK_384, "provider=default", ossl_keccak_384_functions }, - { PROV_NAMES_KECCAK_512, "provider=default", ossl_keccak_512_functions }, - /* * KECCAK-KMAC-128 and KECCAK-KMAC-256 as hashes are mostly useful for * the KMAC-128 and KMAC-256. @@ -133,18 +127,6 @@ static const OSSL_ALGORITHM deflt_digests[] = { { PROV_NAMES_SHAKE_128, "provider=default", ossl_shake_128_functions }, { PROV_NAMES_SHAKE_256, "provider=default", ossl_shake_256_functions }, -#ifndef OPENSSL_NO_BLAKE2 - /* - * https://blake2.net/ doesn't specify size variants, - * but mentions that Bouncy Castle uses the names - * BLAKE2b-160, BLAKE2b-256, BLAKE2b-384, and BLAKE2b-512 - * If we assume that "2b" and "2s" are versions, that pattern - * fits with ours. We also add our historical names. - */ - { PROV_NAMES_BLAKE2S_256, "provider=default", ossl_blake2s256_functions }, - { PROV_NAMES_BLAKE2B_512, "provider=default", ossl_blake2b512_functions }, -#endif /* OPENSSL_NO_BLAKE2 */ - #ifndef OPENSSL_NO_SM3 { PROV_NAMES_SM3, "provider=default", ossl_sm3_functions }, #endif /* OPENSSL_NO_SM3 */ @@ -154,10 +136,6 @@ static const OSSL_ALGORITHM deflt_digests[] = { { PROV_NAMES_MD5_SHA1, "provider=default", ossl_md5_sha1_functions }, #endif /* OPENSSL_NO_MD5 */ -#ifndef OPENSSL_NO_RMD160 - { PROV_NAMES_RIPEMD_160, "provider=default", ossl_ripemd160_functions }, -#endif /* OPENSSL_NO_RMD160 */ - { PROV_NAMES_NULL, "provider=default", ossl_nullmd_functions }, { NULL, NULL, NULL } }; @@ -199,9 +177,6 @@ static const OSSL_ALGORITHM_CAPABLE deflt_ciphers[] = { ALG(PROV_NAMES_AES_128_SIV, ossl_aes128siv_functions), ALG(PROV_NAMES_AES_192_SIV, ossl_aes192siv_functions), ALG(PROV_NAMES_AES_256_SIV, ossl_aes256siv_functions), - ALG(PROV_NAMES_AES_128_GCM_SIV, ossl_aes128gcm_siv_functions), - ALG(PROV_NAMES_AES_192_GCM_SIV, ossl_aes192gcm_siv_functions), - ALG(PROV_NAMES_AES_256_GCM_SIV, ossl_aes256gcm_siv_functions), #endif /* OPENSSL_NO_SIV */ ALG(PROV_NAMES_AES_256_GCM, ossl_aes256gcm_functions), ALG(PROV_NAMES_AES_192_GCM, ossl_aes192gcm_functions), @@ -229,61 +204,6 @@ static const OSSL_ALGORITHM_CAPABLE deflt_ciphers[] = { ossl_cipher_capable_aes_cbc_hmac_sha256), ALGC(PROV_NAMES_AES_256_CBC_HMAC_SHA256, ossl_aes256cbc_hmac_sha256_functions, ossl_cipher_capable_aes_cbc_hmac_sha256), -#ifndef OPENSSL_NO_ARIA - ALG(PROV_NAMES_ARIA_256_GCM, ossl_aria256gcm_functions), - ALG(PROV_NAMES_ARIA_192_GCM, ossl_aria192gcm_functions), - ALG(PROV_NAMES_ARIA_128_GCM, ossl_aria128gcm_functions), - ALG(PROV_NAMES_ARIA_256_CCM, ossl_aria256ccm_functions), - ALG(PROV_NAMES_ARIA_192_CCM, ossl_aria192ccm_functions), - ALG(PROV_NAMES_ARIA_128_CCM, ossl_aria128ccm_functions), - ALG(PROV_NAMES_ARIA_256_ECB, ossl_aria256ecb_functions), - ALG(PROV_NAMES_ARIA_192_ECB, ossl_aria192ecb_functions), - ALG(PROV_NAMES_ARIA_128_ECB, ossl_aria128ecb_functions), - ALG(PROV_NAMES_ARIA_256_CBC, ossl_aria256cbc_functions), - ALG(PROV_NAMES_ARIA_192_CBC, ossl_aria192cbc_functions), - ALG(PROV_NAMES_ARIA_128_CBC, ossl_aria128cbc_functions), - ALG(PROV_NAMES_ARIA_256_OFB, ossl_aria256ofb_functions), - ALG(PROV_NAMES_ARIA_192_OFB, ossl_aria192ofb_functions), - ALG(PROV_NAMES_ARIA_128_OFB, ossl_aria128ofb_functions), - ALG(PROV_NAMES_ARIA_256_CFB, ossl_aria256cfb_functions), - ALG(PROV_NAMES_ARIA_192_CFB, ossl_aria192cfb_functions), - ALG(PROV_NAMES_ARIA_128_CFB, ossl_aria128cfb_functions), - ALG(PROV_NAMES_ARIA_256_CFB1, ossl_aria256cfb1_functions), - ALG(PROV_NAMES_ARIA_192_CFB1, ossl_aria192cfb1_functions), - ALG(PROV_NAMES_ARIA_128_CFB1, ossl_aria128cfb1_functions), - ALG(PROV_NAMES_ARIA_256_CFB8, ossl_aria256cfb8_functions), - ALG(PROV_NAMES_ARIA_192_CFB8, ossl_aria192cfb8_functions), - ALG(PROV_NAMES_ARIA_128_CFB8, ossl_aria128cfb8_functions), - ALG(PROV_NAMES_ARIA_256_CTR, ossl_aria256ctr_functions), - ALG(PROV_NAMES_ARIA_192_CTR, ossl_aria192ctr_functions), - ALG(PROV_NAMES_ARIA_128_CTR, ossl_aria128ctr_functions), -#endif /* OPENSSL_NO_ARIA */ -#ifndef OPENSSL_NO_CAMELLIA - ALG(PROV_NAMES_CAMELLIA_256_ECB, ossl_camellia256ecb_functions), - ALG(PROV_NAMES_CAMELLIA_192_ECB, ossl_camellia192ecb_functions), - ALG(PROV_NAMES_CAMELLIA_128_ECB, ossl_camellia128ecb_functions), - ALG(PROV_NAMES_CAMELLIA_256_CBC, ossl_camellia256cbc_functions), - ALG(PROV_NAMES_CAMELLIA_192_CBC, ossl_camellia192cbc_functions), - ALG(PROV_NAMES_CAMELLIA_128_CBC, ossl_camellia128cbc_functions), - ALG(PROV_NAMES_CAMELLIA_128_CBC_CTS, ossl_camellia128cbc_cts_functions), - ALG(PROV_NAMES_CAMELLIA_192_CBC_CTS, ossl_camellia192cbc_cts_functions), - ALG(PROV_NAMES_CAMELLIA_256_CBC_CTS, ossl_camellia256cbc_cts_functions), - ALG(PROV_NAMES_CAMELLIA_256_OFB, ossl_camellia256ofb_functions), - ALG(PROV_NAMES_CAMELLIA_192_OFB, ossl_camellia192ofb_functions), - ALG(PROV_NAMES_CAMELLIA_128_OFB, ossl_camellia128ofb_functions), - ALG(PROV_NAMES_CAMELLIA_256_CFB, ossl_camellia256cfb_functions), - ALG(PROV_NAMES_CAMELLIA_192_CFB, ossl_camellia192cfb_functions), - ALG(PROV_NAMES_CAMELLIA_128_CFB, ossl_camellia128cfb_functions), - ALG(PROV_NAMES_CAMELLIA_256_CFB1, ossl_camellia256cfb1_functions), - ALG(PROV_NAMES_CAMELLIA_192_CFB1, ossl_camellia192cfb1_functions), - ALG(PROV_NAMES_CAMELLIA_128_CFB1, ossl_camellia128cfb1_functions), - ALG(PROV_NAMES_CAMELLIA_256_CFB8, ossl_camellia256cfb8_functions), - ALG(PROV_NAMES_CAMELLIA_192_CFB8, ossl_camellia192cfb8_functions), - ALG(PROV_NAMES_CAMELLIA_128_CFB8, ossl_camellia128cfb8_functions), - ALG(PROV_NAMES_CAMELLIA_256_CTR, ossl_camellia256ctr_functions), - ALG(PROV_NAMES_CAMELLIA_192_CTR, ossl_camellia192ctr_functions), - ALG(PROV_NAMES_CAMELLIA_128_CTR, ossl_camellia128ctr_functions), -#endif /* OPENSSL_NO_CAMELLIA */ #ifndef OPENSSL_NO_DES ALG(PROV_NAMES_DES_EDE3_ECB, ossl_tdes_ede3_ecb_functions), ALG(PROV_NAMES_DES_EDE3_CBC, ossl_tdes_ede3_cbc_functions), @@ -298,14 +218,13 @@ static const OSSL_ALGORITHM_CAPABLE deflt_ciphers[] = { ALG(PROV_NAMES_DES_EDE_CFB, ossl_tdes_ede2_cfb_functions), #endif /* OPENSSL_NO_DES */ #ifndef OPENSSL_NO_SM4 - ALG(PROV_NAMES_SM4_GCM, ossl_sm4128gcm_functions), - ALG(PROV_NAMES_SM4_CCM, ossl_sm4128ccm_functions), ALG(PROV_NAMES_SM4_ECB, ossl_sm4128ecb_functions), ALG(PROV_NAMES_SM4_CBC, ossl_sm4128cbc_functions), ALG(PROV_NAMES_SM4_CTR, ossl_sm4128ctr_functions), ALG(PROV_NAMES_SM4_OFB, ossl_sm4128ofb128_functions), ALG(PROV_NAMES_SM4_CFB, ossl_sm4128cfb128_functions), - ALG(PROV_NAMES_SM4_XTS, ossl_sm4128xts_functions), + ALG(PROV_NAMES_SM4_GCM, ossl_sm4128gcm_functions), + ALG(PROV_NAMES_SM4_CCM, ossl_sm4128ccm_functions), #endif /* OPENSSL_NO_SM4 */ #ifndef OPENSSL_NO_CHACHA ALG(PROV_NAMES_ChaCha20, ossl_chacha20_functions), @@ -313,15 +232,14 @@ static const OSSL_ALGORITHM_CAPABLE deflt_ciphers[] = { ALG(PROV_NAMES_ChaCha20_Poly1305, ossl_chacha20_ossl_poly1305_functions), # endif /* OPENSSL_NO_POLY1305 */ #endif /* OPENSSL_NO_CHACHA */ +#ifndef OPENSSL_NO_ZUC + ALG(PROV_NAMES_ZUC_128_EEA3, ossl_zuc_128_eea3_functions), +#endif /* OPENSSL_NO_ZUC */ { { NULL, NULL, NULL }, NULL } }; static OSSL_ALGORITHM exported_ciphers[OSSL_NELEM(deflt_ciphers)]; static const OSSL_ALGORITHM deflt_macs[] = { -#ifndef OPENSSL_NO_BLAKE2 - { PROV_NAMES_BLAKE2BMAC, "provider=default", ossl_blake2bmac_functions }, - { PROV_NAMES_BLAKE2SMAC, "provider=default", ossl_blake2smac_functions }, -#endif #ifndef OPENSSL_NO_CMAC { PROV_NAMES_CMAC, "provider=default", ossl_cmac_functions }, #endif @@ -334,6 +252,9 @@ static const OSSL_ALGORITHM deflt_macs[] = { #endif #ifndef OPENSSL_NO_POLY1305 { PROV_NAMES_POLY1305, "provider=default", ossl_poly1305_functions }, +#endif +#ifndef OPENSSL_NO_ZUC + { PROV_NAMES_EIA3, "provider=default", ossl_eia3_functions }, #endif { NULL, NULL, NULL } }; @@ -354,13 +275,6 @@ static const OSSL_ALGORITHM deflt_kdfs[] = { { PROV_NAMES_SCRYPT, "provider=default", ossl_kdf_scrypt_functions }, #endif { PROV_NAMES_KRB5KDF, "provider=default", ossl_kdf_krb5kdf_functions }, - { PROV_NAMES_HMAC_DRBG_KDF, "provider=default", - ossl_kdf_hmac_drbg_functions }, -#ifndef OPENSSL_NO_ARGON2 - { PROV_NAMES_ARGON2I, "provider=default", ossl_kdf_argon2i_functions }, - { PROV_NAMES_ARGON2D, "provider=default", ossl_kdf_argon2d_functions }, - { PROV_NAMES_ARGON2ID, "provider=default", ossl_kdf_argon2id_functions }, -#endif { NULL, NULL, NULL } }; @@ -370,10 +284,11 @@ static const OSSL_ALGORITHM deflt_keyexch[] = { #endif #ifndef OPENSSL_NO_EC { PROV_NAMES_ECDH, "provider=default", ossl_ecdh_keyexch_functions }, -# ifndef OPENSSL_NO_ECX { PROV_NAMES_X25519, "provider=default", ossl_x25519_keyexch_functions }, { PROV_NAMES_X448, "provider=default", ossl_x448_keyexch_functions }, -# endif +#endif +#ifndef OPENSSL_NO_SM2 + { PROV_NAMES_SM2DH, "provider=default", ossl_sm2dh_keyexch_functions }, #endif { PROV_NAMES_TLS1_PRF, "provider=default", ossl_kdf_tls1_prf_keyexch_functions }, { PROV_NAMES_HKDF, "provider=default", ossl_kdf_hkdf_keyexch_functions }, @@ -397,10 +312,8 @@ static const OSSL_ALGORITHM deflt_signature[] = { #endif { PROV_NAMES_RSA, "provider=default", ossl_rsa_signature_functions }, #ifndef OPENSSL_NO_EC -# ifndef OPENSSL_NO_ECX { PROV_NAMES_ED25519, "provider=default", ossl_ed25519_signature_functions }, { PROV_NAMES_ED448, "provider=default", ossl_ed448_signature_functions }, -# endif { PROV_NAMES_ECDSA, "provider=default", ossl_ecdsa_signature_functions }, # ifndef OPENSSL_NO_SM2 { PROV_NAMES_SM2, "provider=default", ossl_sm2_signature_functions }, @@ -413,6 +326,10 @@ static const OSSL_ALGORITHM deflt_signature[] = { { PROV_NAMES_POLY1305, "provider=default", ossl_mac_legacy_poly1305_signature_functions }, #endif +#ifndef OPENSSL_NO_ZUC + { PROV_NAMES_EIA3, "provider=default", + ossl_mac_legacy_eia3_signature_functions }, +#endif #ifndef OPENSSL_NO_CMAC { PROV_NAMES_CMAC, "provider=default", ossl_mac_legacy_cmac_signature_functions }, #endif @@ -429,13 +346,6 @@ static const OSSL_ALGORITHM deflt_asym_cipher[] = { static const OSSL_ALGORITHM deflt_asym_kem[] = { { PROV_NAMES_RSA, "provider=default", ossl_rsa_asym_kem_functions }, -#ifndef OPENSSL_NO_EC -# ifndef OPENSSL_NO_ECX - { PROV_NAMES_X25519, "provider=default", ossl_ecx_asym_kem_functions }, - { PROV_NAMES_X448, "provider=default", ossl_ecx_asym_kem_functions }, -# endif - { PROV_NAMES_EC, "provider=default", ossl_ec_asym_kem_functions }, -#endif { NULL, NULL, NULL } }; @@ -457,7 +367,6 @@ static const OSSL_ALGORITHM deflt_keymgmt[] = { #ifndef OPENSSL_NO_EC { PROV_NAMES_EC, "provider=default", ossl_ec_keymgmt_functions, PROV_DESCS_EC }, -# ifndef OPENSSL_NO_ECX { PROV_NAMES_X25519, "provider=default", ossl_x25519_keymgmt_functions, PROV_DESCS_X25519 }, { PROV_NAMES_X448, "provider=default", ossl_x448_keymgmt_functions, @@ -466,7 +375,6 @@ static const OSSL_ALGORITHM deflt_keymgmt[] = { PROV_DESCS_ED25519 }, { PROV_NAMES_ED448, "provider=default", ossl_ed448_keymgmt_functions, PROV_DESCS_ED448 }, -# endif #endif { PROV_NAMES_TLS1_PRF, "provider=default", ossl_kdf_keymgmt_functions, PROV_DESCS_TLS1_PRF_SIGN }, @@ -482,6 +390,10 @@ static const OSSL_ALGORITHM deflt_keymgmt[] = { { PROV_NAMES_POLY1305, "provider=default", ossl_mac_legacy_keymgmt_functions, PROV_DESCS_POLY1305_SIGN }, #endif +#ifndef OPENSSL_NO_ZUC + { PROV_NAMES_EIA3, "provider=default", ossl_mac_legacy_keymgmt_functions, + PROV_DESCS_EIA3_SIGN }, +#endif #ifndef OPENSSL_NO_CMAC { PROV_NAMES_CMAC, "provider=default", ossl_cmac_legacy_keymgmt_functions, PROV_DESCS_CMAC_SIGN }, @@ -566,7 +478,7 @@ static const OSSL_DISPATCH deflt_dispatch_table[] = { { OSSL_FUNC_PROVIDER_QUERY_OPERATION, (void (*)(void))deflt_query }, { OSSL_FUNC_PROVIDER_GET_CAPABILITIES, (void (*)(void))ossl_prov_get_capabilities }, - OSSL_DISPATCH_END + { 0, NULL } }; OSSL_provider_init_fn ossl_default_provider_init; diff --git a/openssl/src/providers/encoders.inc b/openssl/src/providers/encoders.inc index cd0d1137b..d7e28cb5d 100644 --- a/openssl/src/providers/encoders.inc +++ b/openssl/src/providers/encoders.inc @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,6 +22,7 @@ #define ENCODER_STRUCTURE_DHX "dhx" #define ENCODER_STRUCTURE_DSA "dsa" #define ENCODER_STRUCTURE_EC "ec" +#define ENCODER_STRUCTURE_SM2 "sm2" #define ENCODER_STRUCTURE_RSA "rsa" #define ENCODER_STRUCTURE_PKCS1 "pkcs1" #define ENCODER_STRUCTURE_PKCS3 "pkcs3" @@ -57,12 +58,10 @@ ENCODER_TEXT("DSA", dsa, yes), #endif #ifndef OPENSSL_NO_EC ENCODER_TEXT("EC", ec, yes), -# ifndef OPENSSL_NO_ECX ENCODER_TEXT("ED25519", ed25519, yes), ENCODER_TEXT("ED448", ed448, yes), ENCODER_TEXT("X25519", x25519, yes), ENCODER_TEXT("X448", x448, yes), -# endif # ifndef OPENSSL_NO_SM2 ENCODER_TEXT("SM2", sm2, no), # endif @@ -183,7 +182,6 @@ ENCODER_w_structure("EC", ec, yes, pem, PrivateKeyInfo), ENCODER_w_structure("EC", ec, yes, der, SubjectPublicKeyInfo), ENCODER_w_structure("EC", ec, yes, pem, SubjectPublicKeyInfo), -#ifndef OPENSSL_NO_ECX ENCODER_w_structure("X25519", x25519, yes, der, EncryptedPrivateKeyInfo), ENCODER_w_structure("X25519", x25519, yes, pem, EncryptedPrivateKeyInfo), ENCODER_w_structure("X25519", x25519, yes, der, PrivateKeyInfo), @@ -211,7 +209,6 @@ ENCODER_w_structure("ED448", ed448, yes, der, PrivateKeyInfo), ENCODER_w_structure("ED448", ed448, yes, pem, PrivateKeyInfo), ENCODER_w_structure("ED448", ed448, yes, der, SubjectPublicKeyInfo), ENCODER_w_structure("ED448", ed448, yes, pem, SubjectPublicKeyInfo), -# endif # ifndef OPENSSL_NO_SM2 ENCODER_w_structure("SM2", sm2, no, der, EncryptedPrivateKeyInfo), diff --git a/openssl/src/providers/fips/fipsprov.c b/openssl/src/providers/fips/fipsprov.c index 86c18de28..66ac9257e 100644 --- a/openssl/src/providers/fips/fipsprov.c +++ b/openssl/src/providers/fips/fipsprov.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,10 +21,7 @@ #include "prov/providercommon.h" #include "prov/provider_util.h" #include "prov/seeding.h" -#include "prov/fipscommon.h" -#include "internal/nelem.h" #include "self_test.h" -#include "crypto/context.h" #include "internal/core.h" static const char FIPS_DEFAULT_PROPERTIES[] = "provider=fips,fips=yes"; @@ -39,14 +36,11 @@ static OSSL_FUNC_provider_gettable_params_fn fips_gettable_params; static OSSL_FUNC_provider_get_params_fn fips_get_params; static OSSL_FUNC_provider_query_operation_fn fips_query; -#define ALGC(NAMES, FUNC, CHECK) \ - { { NAMES, FIPS_DEFAULT_PROPERTIES, FUNC }, CHECK } -#define UNAPPROVED_ALGC(NAMES, FUNC, CHECK) \ - { { NAMES, FIPS_UNAPPROVED_PROPERTIES, FUNC }, CHECK } +#define ALGC(NAMES, FUNC, CHECK) { { NAMES, FIPS_DEFAULT_PROPERTIES, FUNC }, CHECK } #define ALG(NAMES, FUNC) ALGC(NAMES, FUNC, NULL) -#define UNAPPROVED_ALG(NAMES, FUNC) UNAPPROVED_ALGC(NAMES, FUNC, NULL) extern OSSL_FUNC_core_thread_start_fn *c_thread_start; +int FIPS_security_check_enabled(OSSL_LIB_CTX *libctx); /* * Should these function pointers be stored in the provider side provctx? Could @@ -78,42 +72,37 @@ static OSSL_FUNC_BIO_vsnprintf_fn *c_BIO_vsnprintf; static OSSL_FUNC_self_test_cb_fn *c_stcbfn = NULL; static OSSL_FUNC_core_get_libctx_fn *c_get_libctx = NULL; -typedef struct { - const char *option; - unsigned char enabled; -} FIPS_OPTION; - typedef struct fips_global_st { const OSSL_CORE_HANDLE *handle; SELF_TEST_POST_PARAMS selftest_params; - FIPS_OPTION fips_security_checks; - FIPS_OPTION fips_tls1_prf_ems_check; - FIPS_OPTION fips_restricted_drgb_digests; + int fips_security_checks; + const char *fips_security_check_option; } FIPS_GLOBAL; -static void init_fips_option(FIPS_OPTION *opt, int enabled) -{ - opt->enabled = enabled; - opt->option = enabled ? "1" : "0"; -} - -void *ossl_fips_prov_ossl_ctx_new(OSSL_LIB_CTX *libctx) +static void *fips_prov_ossl_ctx_new(OSSL_LIB_CTX *libctx) { FIPS_GLOBAL *fgbl = OPENSSL_zalloc(sizeof(*fgbl)); if (fgbl == NULL) return NULL; - init_fips_option(&fgbl->fips_security_checks, 1); - init_fips_option(&fgbl->fips_tls1_prf_ems_check, 0); /* Disabled by default */ - init_fips_option(&fgbl->fips_restricted_drgb_digests, 0); + fgbl->fips_security_checks = 1; + fgbl->fips_security_check_option = "1"; + return fgbl; } -void ossl_fips_prov_ossl_ctx_free(void *fgbl) +static void fips_prov_ossl_ctx_free(void *fgbl) { OPENSSL_free(fgbl); } +static const OSSL_LIB_CTX_METHOD fips_prov_ossl_ctx_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + fips_prov_ossl_ctx_new, + fips_prov_ossl_ctx_free, +}; + + /* Parameters we provide to the core */ static const OSSL_PARAM fips_param_types[] = { OSSL_PARAM_DEFN(OSSL_PROV_PARAM_NAME, OSSL_PARAM_UTF8_PTR, NULL, 0), @@ -121,8 +110,6 @@ static const OSSL_PARAM fips_param_types[] = { OSSL_PARAM_DEFN(OSSL_PROV_PARAM_BUILDINFO, OSSL_PARAM_UTF8_PTR, NULL, 0), OSSL_PARAM_DEFN(OSSL_PROV_PARAM_STATUS, OSSL_PARAM_INTEGER, NULL, 0), OSSL_PARAM_DEFN(OSSL_PROV_PARAM_SECURITY_CHECKS, OSSL_PARAM_INTEGER, NULL, 0), - OSSL_PARAM_DEFN(OSSL_PROV_PARAM_TLS1_PRF_EMS_CHECK, OSSL_PARAM_INTEGER, NULL, 0), - OSSL_PARAM_DEFN(OSSL_PROV_PARAM_DRBG_TRUNC_DIGEST, OSSL_PARAM_INTEGER, NULL, 0), OSSL_PARAM_END }; @@ -133,10 +120,9 @@ static int fips_get_params_from_core(FIPS_GLOBAL *fgbl) * NOTE: inside core_get_params() these will be loaded from config items * stored inside prov->parameters (except for * OSSL_PROV_PARAM_CORE_MODULE_FILENAME). - * OSSL_PROV_FIPS_PARAM_SECURITY_CHECKS and - * OSSL_PROV_FIPS_PARAM_TLS1_PRF_EMS_CHECK are not self test parameters. + * OSSL_PROV_FIPS_PARAM_SECURITY_CHECKS is not a self test parameter. */ - OSSL_PARAM core_params[10], *p = core_params; + OSSL_PARAM core_params[8], *p = core_params; *p++ = OSSL_PARAM_construct_utf8_ptr( OSSL_PROV_PARAM_CORE_MODULE_FILENAME, @@ -162,21 +148,10 @@ static int fips_get_params_from_core(FIPS_GLOBAL *fgbl) OSSL_PROV_FIPS_PARAM_CONDITIONAL_ERRORS, (char **)&fgbl->selftest_params.conditional_error_check, sizeof(fgbl->selftest_params.conditional_error_check)); - -/* FIPS features can be enabled or disabled independently */ -#define FIPS_FEATURE_OPTION(fgbl, pname, field) \ - *p++ = OSSL_PARAM_construct_utf8_ptr( \ - pname, (char **)&fgbl->field.option, \ - sizeof(fgbl->field.option)) - - FIPS_FEATURE_OPTION(fgbl, OSSL_PROV_FIPS_PARAM_SECURITY_CHECKS, - fips_security_checks); - FIPS_FEATURE_OPTION(fgbl, OSSL_PROV_FIPS_PARAM_TLS1_PRF_EMS_CHECK, - fips_tls1_prf_ems_check); - FIPS_FEATURE_OPTION(fgbl, OSSL_PROV_FIPS_PARAM_DRBG_TRUNC_DIGEST, - fips_restricted_drgb_digests); -#undef FIPS_FEATURE_OPTION - + *p++ = OSSL_PARAM_construct_utf8_ptr( + OSSL_PROV_FIPS_PARAM_SECURITY_CHECKS, + (char **)&fgbl->fips_security_check_option, + sizeof(fgbl->fips_security_check_option)); *p = OSSL_PARAM_construct_end(); if (!c_get_params(fgbl->handle, core_params)) { @@ -196,7 +171,8 @@ static int fips_get_params(void *provctx, OSSL_PARAM params[]) { OSSL_PARAM *p; FIPS_GLOBAL *fgbl = ossl_lib_ctx_get_data(ossl_prov_ctx_get0_libctx(provctx), - OSSL_LIB_CTX_FIPS_PROV_INDEX); + OSSL_LIB_CTX_FIPS_PROV_INDEX, + &fips_prov_ossl_ctx_method); p = OSSL_PARAM_locate(params, OSSL_PROV_PARAM_NAME); if (p != NULL && !OSSL_PARAM_set_utf8_ptr(p, "OpenSSL FIPS Provider")) @@ -210,19 +186,9 @@ static int fips_get_params(void *provctx, OSSL_PARAM params[]) p = OSSL_PARAM_locate(params, OSSL_PROV_PARAM_STATUS); if (p != NULL && !OSSL_PARAM_set_int(p, ossl_prov_is_running())) return 0; - -#define FIPS_FEATURE_GET(fgbl, pname, field) \ - p = OSSL_PARAM_locate(params, pname); \ - if (p != NULL && !OSSL_PARAM_set_int(p, fgbl->field.enabled)) \ - return 0 - - FIPS_FEATURE_GET(fgbl, OSSL_PROV_PARAM_SECURITY_CHECKS, - fips_security_checks); - FIPS_FEATURE_GET(fgbl, OSSL_PROV_PARAM_TLS1_PRF_EMS_CHECK, - fips_tls1_prf_ems_check); - FIPS_FEATURE_GET(fgbl, OSSL_PROV_PARAM_DRBG_TRUNC_DIGEST, - fips_restricted_drgb_digests); -#undef FIPS_FEATURE_GET + p = OSSL_PARAM_locate(params, OSSL_PROV_PARAM_SECURITY_CHECKS); + if (p != NULL && !OSSL_PARAM_set_int(p, fgbl->fips_security_checks)) + return 0; return 1; } @@ -243,7 +209,8 @@ static void set_self_test_cb(FIPS_GLOBAL *fgbl) static int fips_self_test(void *provctx) { FIPS_GLOBAL *fgbl = ossl_lib_ctx_get_data(ossl_prov_ctx_get0_libctx(provctx), - OSSL_LIB_CTX_FIPS_PROV_INDEX); + OSSL_LIB_CTX_FIPS_PROV_INDEX, + &fips_prov_ossl_ctx_method); set_self_test_cb(fgbl); return SELF_TEST_post(&fgbl->selftest_params, 1) ? 1 : 0; @@ -256,7 +223,7 @@ static int fips_self_test(void *provctx) * ALGNAME[VERSION?][-SUBNAME[VERSION?]?][-SIZE?][-MODE?] * * VERSION is only present if there are multiple versions of - * an alg (MD2, MD4, MD5). It may be omitted if there is only + * an alg (MD series). It may be omitted if there is only * one version (if a subsequent version is released in the future, * we can always change the canonical name, and add the old name * as an alias). @@ -360,8 +327,8 @@ static const OSSL_ALGORITHM_CAPABLE fips_ciphers[] = { ALGC(PROV_NAMES_AES_256_CBC_HMAC_SHA256, ossl_aes256cbc_hmac_sha256_functions, ossl_cipher_capable_aes_cbc_hmac_sha256), #ifndef OPENSSL_NO_DES - UNAPPROVED_ALG(PROV_NAMES_DES_EDE3_ECB, ossl_tdes_ede3_ecb_functions), - UNAPPROVED_ALG(PROV_NAMES_DES_EDE3_CBC, ossl_tdes_ede3_cbc_functions), + ALG(PROV_NAMES_DES_EDE3_ECB, ossl_tdes_ede3_ecb_functions), + ALG(PROV_NAMES_DES_EDE3_CBC, ossl_tdes_ede3_cbc_functions), #endif /* OPENSSL_NO_DES */ { { NULL, NULL, NULL }, NULL } }; @@ -409,10 +376,8 @@ static const OSSL_ALGORITHM fips_keyexch[] = { #endif #ifndef OPENSSL_NO_EC { PROV_NAMES_ECDH, FIPS_DEFAULT_PROPERTIES, ossl_ecdh_keyexch_functions }, -# ifndef OPENSSL_NO_ECX { PROV_NAMES_X25519, FIPS_DEFAULT_PROPERTIES, ossl_x25519_keyexch_functions }, { PROV_NAMES_X448, FIPS_DEFAULT_PROPERTIES, ossl_x448_keyexch_functions }, -# endif #endif { PROV_NAMES_TLS1_PRF, FIPS_DEFAULT_PROPERTIES, ossl_kdf_tls1_prf_keyexch_functions }, @@ -426,11 +391,8 @@ static const OSSL_ALGORITHM fips_signature[] = { #endif { PROV_NAMES_RSA, FIPS_DEFAULT_PROPERTIES, ossl_rsa_signature_functions }, #ifndef OPENSSL_NO_EC -# ifndef OPENSSL_NO_ECX - { PROV_NAMES_ED25519, FIPS_UNAPPROVED_PROPERTIES, - ossl_ed25519_signature_functions }, - { PROV_NAMES_ED448, FIPS_UNAPPROVED_PROPERTIES, ossl_ed448_signature_functions }, -# endif + { PROV_NAMES_ED25519, FIPS_DEFAULT_PROPERTIES, ossl_ed25519_signature_functions }, + { PROV_NAMES_ED448, FIPS_DEFAULT_PROPERTIES, ossl_ed448_signature_functions }, { PROV_NAMES_ECDSA, FIPS_DEFAULT_PROPERTIES, ossl_ecdsa_signature_functions }, #endif { PROV_NAMES_HMAC, FIPS_DEFAULT_PROPERTIES, @@ -470,16 +432,14 @@ static const OSSL_ALGORITHM fips_keymgmt[] = { #ifndef OPENSSL_NO_EC { PROV_NAMES_EC, FIPS_DEFAULT_PROPERTIES, ossl_ec_keymgmt_functions, PROV_DESCS_EC }, -# ifndef OPENSSL_NO_ECX { PROV_NAMES_X25519, FIPS_DEFAULT_PROPERTIES, ossl_x25519_keymgmt_functions, PROV_DESCS_X25519 }, { PROV_NAMES_X448, FIPS_DEFAULT_PROPERTIES, ossl_x448_keymgmt_functions, PROV_DESCS_X448 }, - { PROV_NAMES_ED25519, FIPS_UNAPPROVED_PROPERTIES, ossl_ed25519_keymgmt_functions, + { PROV_NAMES_ED25519, FIPS_DEFAULT_PROPERTIES, ossl_ed25519_keymgmt_functions, PROV_DESCS_ED25519 }, - { PROV_NAMES_ED448, FIPS_UNAPPROVED_PROPERTIES, ossl_ed448_keymgmt_functions, + { PROV_NAMES_ED448, FIPS_DEFAULT_PROPERTIES, ossl_ed448_keymgmt_functions, PROV_DESCS_ED448 }, -# endif #endif { PROV_NAMES_TLS1_PRF, FIPS_DEFAULT_PROPERTIES, ossl_kdf_keymgmt_functions, PROV_DESCS_TLS1_PRF_SIGN }, @@ -551,32 +511,17 @@ static const OSSL_DISPATCH fips_dispatch_table[] = { { OSSL_FUNC_PROVIDER_GET_CAPABILITIES, (void (*)(void))ossl_prov_get_capabilities }, { OSSL_FUNC_PROVIDER_SELF_TEST, (void (*)(void))fips_self_test }, - OSSL_DISPATCH_END + { 0, NULL } }; /* Functions we provide to ourself */ static const OSSL_DISPATCH intern_dispatch_table[] = { { OSSL_FUNC_PROVIDER_TEARDOWN, (void (*)(void))fips_intern_teardown }, { OSSL_FUNC_PROVIDER_QUERY_OPERATION, (void (*)(void))fips_query }, - OSSL_DISPATCH_END + { 0, NULL } }; -/* - * On VMS, the provider init function name is expected to be uppercase, - * see the pragmas in . Let's do the same with this - * internal name. This is how symbol names are treated by default - * by the compiler if nothing else is said, but since this is part - * of libfips, and we build our libraries with mixed case symbol names, - * we must switch back to this default explicitly here. - */ -#ifdef __VMS -# pragma names save -# pragma names uppercase,truncated -#endif OSSL_provider_init_fn OSSL_provider_init_int; -#ifdef __VMS -# pragma names restore -#endif int OSSL_provider_init_int(const OSSL_CORE_HANDLE *handle, const OSSL_DISPATCH *in, const OSSL_DISPATCH **out, @@ -589,7 +534,7 @@ int OSSL_provider_init_int(const OSSL_CORE_HANDLE *handle, memset(&selftest_params, 0, sizeof(selftest_params)); if (!ossl_prov_seeding_from_dispatch(in)) - goto err; + return 0; for (; in->function_id != 0; in++) { /* * We do not support the scenario of an application linked against @@ -695,27 +640,24 @@ int OSSL_provider_init_int(const OSSL_CORE_HANDLE *handle, } } - OPENSSL_cpuid_setup(); - /* Create a context. */ if ((*provctx = ossl_prov_ctx_new()) == NULL - || (libctx = OSSL_LIB_CTX_new()) == NULL) + || (libctx = OSSL_LIB_CTX_new()) == NULL) { + /* + * We free libctx separately here and only here because it hasn't + * been attached to *provctx. All other error paths below rely + * solely on fips_teardown. + */ + OSSL_LIB_CTX_free(libctx); goto err; + } - if ((fgbl = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_FIPS_PROV_INDEX)) == NULL) + if ((fgbl = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_FIPS_PROV_INDEX, + &fips_prov_ossl_ctx_method)) == NULL) goto err; fgbl->handle = handle; - /* - * We need to register this thread to receive thread lifecycle callbacks. - * This wouldn't matter if the current thread is also the same thread that - * closes the FIPS provider down. But if that happens on a different thread - * then memory leaks could otherwise occur. - */ - if (!ossl_thread_register_fips(libctx)) - goto err; - /* * We did initial set up of selftest_params in a local copy, because we * could not create fgbl until c_CRYPTO_zalloc was defined in the loop @@ -739,21 +681,10 @@ int OSSL_provider_init_int(const OSSL_CORE_HANDLE *handle, && strcmp(fgbl->selftest_params.conditional_error_check, "0") == 0) SELF_TEST_disable_conditional_error_state(); - /* Enable or disable FIPS provider options */ -#define FIPS_SET_OPTION(fgbl, field) \ - if (fgbl->field.option != NULL) { \ - if (strcmp(fgbl->field.option, "1") == 0) \ - fgbl->field.enabled = 1; \ - else if (strcmp(fgbl->field.option, "0") == 0) \ - fgbl->field.enabled = 0; \ - else \ - goto err; \ - } - - FIPS_SET_OPTION(fgbl, fips_security_checks); - FIPS_SET_OPTION(fgbl, fips_tls1_prf_ems_check); - FIPS_SET_OPTION(fgbl, fips_restricted_drgb_digests); -#undef FIPS_SET_OPTION + /* Disable the security check if it's disabled in the fips config file. */ + if (fgbl->fips_security_check_option != NULL + && strcmp(fgbl->fips_security_check_option, "0") == 0) + fgbl->fips_security_checks = 0; ossl_prov_cache_exported_algorithms(fips_ciphers, exported_fips_ciphers); @@ -867,7 +798,8 @@ int ERR_pop_to_mark(void) const OSSL_CORE_HANDLE *FIPS_get_core_handle(OSSL_LIB_CTX *libctx) { FIPS_GLOBAL *fgbl = ossl_lib_ctx_get_data(libctx, - OSSL_LIB_CTX_FIPS_PROV_INDEX); + OSSL_LIB_CTX_FIPS_PROV_INDEX, + &fips_prov_ossl_ctx_method); if (fgbl == NULL) return NULL; @@ -942,19 +874,14 @@ int BIO_snprintf(char *buf, size_t n, const char *format, ...) return ret; } -#define FIPS_FEATURE_CHECK(fname, field) \ - int fname(OSSL_LIB_CTX *libctx) \ - { \ - FIPS_GLOBAL *fgbl = \ - ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_FIPS_PROV_INDEX); \ - return fgbl->field.enabled; \ - } +int FIPS_security_check_enabled(OSSL_LIB_CTX *libctx) +{ + FIPS_GLOBAL *fgbl = ossl_lib_ctx_get_data(libctx, + OSSL_LIB_CTX_FIPS_PROV_INDEX, + &fips_prov_ossl_ctx_method); -FIPS_FEATURE_CHECK(FIPS_security_check_enabled, fips_security_checks) -FIPS_FEATURE_CHECK(FIPS_tls_prf_ems_check, fips_tls1_prf_ems_check) -FIPS_FEATURE_CHECK(FIPS_restricted_drbg_digests_enabled, - fips_restricted_drgb_digests) -#undef FIPS_FEATURE_CHECK + return fgbl->fips_security_checks; +} void OSSL_SELF_TEST_get_callback(OSSL_LIB_CTX *libctx, OSSL_CALLBACK **cb, void **cbarg) diff --git a/openssl/src/providers/fips/self_test.c b/openssl/src/providers/fips/self_test.c index b8dc9817b..53eac7e93 100644 --- a/openssl/src/providers/fips/self_test.c +++ b/openssl/src/providers/fips/self_test.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,11 +15,8 @@ #include #include #include -#include #include "internal/e_os.h" -#include "internal/tsan_assist.h" #include "prov/providercommon.h" -#include "crypto/rand.h" /* * We're cheating here. Normally we don't allow RUN_ONCE usage inside the FIPS @@ -50,6 +47,7 @@ static int FIPS_conditional_error_check = 1; static CRYPTO_RWLOCK *self_test_lock = NULL; +static CRYPTO_RWLOCK *fips_state_lock = NULL; static unsigned char fixed_key[32] = { FIPS_KEY_ELEMENTS }; static CRYPTO_ONCE fips_self_test_init = CRYPTO_ONCE_STATIC_INIT; @@ -61,6 +59,7 @@ DEFINE_RUN_ONCE_STATIC(do_fips_self_test_init) * platform then we just leak it deliberately. */ self_test_lock = CRYPTO_THREAD_lock_new(); + fips_state_lock = CRYPTO_THREAD_lock_new(); return self_test_lock != NULL; } @@ -105,7 +104,7 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) return TRUE; } -#elif defined(__GNUC__) && !defined(_AIX) +#elif defined(__GNUC__) # undef DEP_INIT_ATTRIBUTE # undef DEP_FINI_ATTRIBUTE # define DEP_INIT_ATTRIBUTE static __attribute__((constructor)) @@ -115,7 +114,7 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) # pragma init(init) # pragma fini(cleanup) -#elif defined(_AIX) && !defined(__GNUC__) +#elif defined(_AIX) void _init(void); void _cleanup(void); # pragma init(_init) @@ -129,10 +128,6 @@ void _cleanup(void) cleanup(); } -#elif defined(__hpux) -# pragma init "init" -# pragma fini "cleanup" - #elif defined(__TANDEM) /* Method automatically called by the NonStop OS when the DLL loads */ void __INIT__init(void) { @@ -156,12 +151,12 @@ void __TERM__cleanup(void) { # define DEP_INITIAL_STATE FIPS_STATE_SELFTEST #endif -static TSAN_QUALIFIER int FIPS_state = DEP_INITIAL_STATE; +static int FIPS_state = DEP_INITIAL_STATE; #if defined(DEP_INIT_ATTRIBUTE) DEP_INIT_ATTRIBUTE void init(void) { - tsan_store(&FIPS_state, FIPS_STATE_SELFTEST); + FIPS_state = FIPS_STATE_SELFTEST; } #endif @@ -169,67 +164,10 @@ DEP_INIT_ATTRIBUTE void init(void) DEP_FINI_ATTRIBUTE void cleanup(void) { CRYPTO_THREAD_lock_free(self_test_lock); + CRYPTO_THREAD_lock_free(fips_state_lock); } #endif -/* - * We need an explicit HMAC-SHA-256 KAT even though it is also - * checked as part of the KDF KATs. Refer IG 10.3. - */ -static const unsigned char hmac_kat_pt[] = { - 0xdd, 0x0c, 0x30, 0x33, 0x35, 0xf9, 0xe4, 0x2e, - 0xc2, 0xef, 0xcc, 0xbf, 0x07, 0x95, 0xee, 0xa2 -}; -static const unsigned char hmac_kat_key[] = { - 0xf4, 0x55, 0x66, 0x50, 0xac, 0x31, 0xd3, 0x54, - 0x61, 0x61, 0x0b, 0xac, 0x4e, 0xd8, 0x1b, 0x1a, - 0x18, 0x1b, 0x2d, 0x8a, 0x43, 0xea, 0x28, 0x54, - 0xcb, 0xae, 0x22, 0xca, 0x74, 0x56, 0x08, 0x13 -}; -static const unsigned char hmac_kat_digest[] = { - 0xf5, 0xf5, 0xe5, 0xf2, 0x66, 0x49, 0xe2, 0x40, - 0xfc, 0x9e, 0x85, 0x7f, 0x2b, 0x9a, 0xbe, 0x28, - 0x20, 0x12, 0x00, 0x92, 0x82, 0x21, 0x3e, 0x51, - 0x44, 0x5d, 0xe3, 0x31, 0x04, 0x01, 0x72, 0x6b -}; - -static int integrity_self_test(OSSL_SELF_TEST *ev, OSSL_LIB_CTX *libctx) -{ - int ok = 0; - unsigned char out[EVP_MAX_MD_SIZE]; - size_t out_len = 0; - - OSSL_PARAM params[2]; - EVP_MAC *mac = EVP_MAC_fetch(libctx, MAC_NAME, NULL); - EVP_MAC_CTX *ctx = EVP_MAC_CTX_new(mac); - - OSSL_SELF_TEST_onbegin(ev, OSSL_SELF_TEST_TYPE_KAT_INTEGRITY, - OSSL_SELF_TEST_DESC_INTEGRITY_HMAC); - - params[0] = OSSL_PARAM_construct_utf8_string("digest", DIGEST_NAME, 0); - params[1] = OSSL_PARAM_construct_end(); - - if (ctx == NULL - || mac == NULL - || !EVP_MAC_init(ctx, hmac_kat_key, sizeof(hmac_kat_key), params) - || !EVP_MAC_update(ctx, hmac_kat_pt, sizeof(hmac_kat_pt)) - || !EVP_MAC_final(ctx, out, &out_len, MAX_MD_SIZE)) - goto err; - - /* Optional corruption */ - OSSL_SELF_TEST_oncorrupt_byte(ev, out); - - if (out_len != sizeof(hmac_kat_digest) - || memcmp(out, hmac_kat_digest, out_len) != 0) - goto err; - ok = 1; -err: - OSSL_SELF_TEST_onend(ev, ok); - EVP_MAC_free(mac); - EVP_MAC_CTX_free(ctx); - return ok; -} - /* * Calculate the HMAC SHA256 of data read using a BIO and read_cb, and verify * the result matches the expected value. @@ -248,9 +186,6 @@ static int verify_integrity(OSSL_CORE_BIO *bio, OSSL_FUNC_BIO_read_ex_fn read_ex EVP_MAC_CTX *ctx = NULL; OSSL_PARAM params[2], *p = params; - if (!integrity_self_test(ev, libctx)) - goto err; - OSSL_SELF_TEST_onbegin(ev, event_type, OSSL_SELF_TEST_DESC_INTEGRITY_HMAC); mac = EVP_MAC_fetch(libctx, MAC_NAME, NULL); @@ -290,7 +225,10 @@ static int verify_integrity(OSSL_CORE_BIO *bio, OSSL_FUNC_BIO_read_ex_fn read_ex static void set_fips_state(int state) { - tsan_store(&FIPS_state, state); + if (ossl_assert(CRYPTO_THREAD_write_lock(fips_state_lock) != 0)) { + FIPS_state = state; + CRYPTO_THREAD_unlock(fips_state_lock); + } } /* This API is triggered either on loading of the FIPS module or on demand */ @@ -304,13 +242,14 @@ int SELF_TEST_post(SELF_TEST_POST_PARAMS *st, int on_demand_test) unsigned char *indicator_checksum = NULL; int loclstate; OSSL_SELF_TEST *ev = NULL; - EVP_RAND *testrand = NULL; - EVP_RAND_CTX *rng; if (!RUN_ONCE(&fips_self_test_init, do_fips_self_test_init)) return 0; - loclstate = tsan_load(&FIPS_state); + if (!CRYPTO_THREAD_read_lock(fips_state_lock)) + return 0; + loclstate = FIPS_state; + CRYPTO_THREAD_unlock(fips_state_lock); if (loclstate == FIPS_STATE_RUNNING) { if (!on_demand_test) @@ -322,17 +261,24 @@ int SELF_TEST_post(SELF_TEST_POST_PARAMS *st, int on_demand_test) if (!CRYPTO_THREAD_write_lock(self_test_lock)) return 0; - loclstate = tsan_load(&FIPS_state); - if (loclstate == FIPS_STATE_RUNNING) { + if (!CRYPTO_THREAD_read_lock(fips_state_lock)) { + CRYPTO_THREAD_unlock(self_test_lock); + return 0; + } + if (FIPS_state == FIPS_STATE_RUNNING) { + CRYPTO_THREAD_unlock(fips_state_lock); if (!on_demand_test) { CRYPTO_THREAD_unlock(self_test_lock); return 1; } set_fips_state(FIPS_STATE_SELFTEST); - } else if (loclstate != FIPS_STATE_SELFTEST) { + } else if (FIPS_state != FIPS_STATE_SELFTEST) { + CRYPTO_THREAD_unlock(fips_state_lock); CRYPTO_THREAD_unlock(self_test_lock); ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_STATE); return 0; + } else { + CRYPTO_THREAD_unlock(fips_state_lock); } if (st == NULL @@ -405,20 +351,8 @@ int SELF_TEST_post(SELF_TEST_POST_PARAMS *st, int on_demand_test) goto end; } } - - /* Verify that the RNG has been restored properly */ - rng = ossl_rand_get0_private_noncreating(st->libctx); - if (rng != NULL) - if ((testrand = EVP_RAND_fetch(st->libctx, "TEST-RAND", NULL)) == NULL - || strcmp(EVP_RAND_get0_name(EVP_RAND_CTX_get0_rand(rng)), - EVP_RAND_get0_name(testrand)) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_SELF_TEST_KAT_FAILURE); - goto end; - } - ok = 1; end: - EVP_RAND_free(testrand); OSSL_SELF_TEST_free(ev); OPENSSL_free(module_checksum); OPENSSL_free(indicator_checksum); @@ -455,13 +389,20 @@ void ossl_set_error_state(const char *type) int ossl_prov_is_running(void) { - int res, loclstate; - static TSAN_QUALIFIER unsigned int rate_limit = 0; + int res; + static unsigned int rate_limit = 0; - loclstate = tsan_load(&FIPS_state); - res = loclstate == FIPS_STATE_RUNNING || loclstate == FIPS_STATE_SELFTEST; - if (loclstate == FIPS_STATE_ERROR) - if (tsan_counter(&rate_limit) < FIPS_ERROR_REPORTING_RATE_LIMIT) + if (!CRYPTO_THREAD_read_lock(fips_state_lock)) + return 0; + res = FIPS_state == FIPS_STATE_RUNNING + || FIPS_state == FIPS_STATE_SELFTEST; + if (FIPS_state == FIPS_STATE_ERROR) { + CRYPTO_THREAD_unlock(fips_state_lock); + if (!CRYPTO_THREAD_write_lock(fips_state_lock)) + return 0; + if (rate_limit++ < FIPS_ERROR_REPORTING_RATE_LIMIT) ERR_raise(ERR_LIB_PROV, PROV_R_FIPS_MODULE_IN_ERROR_STATE); + } + CRYPTO_THREAD_unlock(fips_state_lock); return res; } diff --git a/openssl/src/providers/fips/self_test_data.inc b/openssl/src/providers/fips/self_test_data.inc index d2a4778e9..5f057d567 100644 --- a/openssl/src/providers/fips/self_test_data.inc +++ b/openssl/src/providers/fips/self_test_data.inc @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -106,12 +106,6 @@ typedef struct st_kat_sign_st { const char *algorithm; const char *mdalgorithm; const ST_KAT_PARAM *key; - const unsigned char *entropy; - size_t entropy_len; - const unsigned char *nonce; - size_t nonce_len; - const unsigned char *persstr; - size_t persstr_len; const unsigned char *sig_expected; /* Set to NULL if this value changes */ size_t sig_expected_len; } ST_KAT_SIGN; @@ -241,6 +235,19 @@ static const unsigned char aes_128_ecb_ct[] = { }; static const ST_KAT_CIPHER st_kat_cipher_tests[] = { +#ifndef OPENSSL_NO_DES + { + { + OSSL_SELF_TEST_DESC_CIPHER_TDES, + "DES-EDE3-CBC", + ITM(des_ede3_cbc_pt), + ITM(des_ede3_cbc_ct) + }, + CIPHER_MODE_ENCRYPT | CIPHER_MODE_DECRYPT, + ITM(des_ede3_cbc_key), + ITM(des_ede3_cbc_iv), + }, +#endif { { OSSL_SELF_TEST_DESC_CIPHER_AES_GCM, @@ -354,29 +361,19 @@ static const ST_KAT_PARAM x963kdf_params[] = { }; static const char pbkdf2_digest[] = "SHA256"; -/* - * Input parameters from RFC 6070, vector 5 (because it is the only one with - * a salt >= 16 bytes, which NIST SP 800-132 section 5.1 requires). The - * expected output is taken from - * https://github.com/brycx/Test-Vector-Generation/blob/master/PBKDF2/pbkdf2-hmac-sha2-test-vectors.md, - * which ran these test vectors with SHA-256. - */ static const unsigned char pbkdf2_password[] = { - 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x50, 0x41, 0x53, 0x53, - 0x57, 0x4f, 0x52, 0x44, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64 + 0x70, 0x61, 0x73, 0x73, 0x00, 0x77, 0x6f, 0x72, + 0x64 }; static const unsigned char pbkdf2_salt[] = { - 0x73, 0x61, 0x6c, 0x74, 0x53, 0x41, 0x4c, 0x54, 0x73, 0x61, 0x6c, 0x74, - 0x53, 0x41, 0x4c, 0x54, 0x73, 0x61, 0x6c, 0x74, 0x53, 0x41, 0x4c, 0x54, - 0x73, 0x61, 0x6c, 0x74, 0x53, 0x41, 0x4c, 0x54, 0x73, 0x61, 0x6c, 0x74 + 0x73, 0x61, 0x00, 0x6c, 0x74 }; static const unsigned char pbkdf2_expected[] = { - 0x34, 0x8c, 0x89, 0xdb, 0xcb, 0xd3, 0x2b, 0x2f, 0x32, 0xd8, 0x14, 0xb8, - 0x11, 0x6e, 0x84, 0xcf, 0x2b, 0x17, 0x34, 0x7e, 0xbc, 0x18, 0x00, 0x18, - 0x1c + 0x89, 0xb6, 0x9d, 0x05, 0x16, 0xf8, 0x29, 0x89, + 0x3c, 0x69, 0x62, 0x26, 0x65, 0x0a, 0x86, 0x87, }; static int pbkdf2_iterations = 4096; -static int pbkdf2_pkcs5 = 0; +static int pbkdf2_pkcs5 = 1; static const ST_KAT_PARAM pbkdf2_params[] = { ST_KAT_PARAM_UTF8STRING(OSSL_KDF_PARAM_DIGEST, pbkdf2_digest), ST_KAT_PARAM_OCTET(OSSL_KDF_PARAM_PASSWORD, pbkdf2_password), @@ -497,33 +494,6 @@ static const ST_KAT_PARAM kbkdf_params[] = { ST_KAT_PARAM_END() }; -static const char kbkdf_kmac_mac[] = "KMAC128"; -static unsigned char kbkdf_kmac_label[] = { - 0xB5, 0xB5, 0xF3, 0x71, 0x9F, 0xBE, 0x5B, 0x3D, - 0x7B, 0x8D, 0x05, 0xA1, 0xD3, 0x25, 0x19, 0x50, -}; -static unsigned char kbkdf_kmac_context[] = { - 0x36, 0x60, 0x0E, 0xF3, 0xC3, 0x70, 0xB5, 0xEF, - 0x58, 0xBE, 0xF1, 0xBA, 0x1C, 0xF2, 0x74, 0xCB, -}; -static unsigned char kbkdf_kmac_key[] = { - 0xB2, 0x51, 0x4C, 0xC1, 0xD5, 0xCD, 0x7B, 0x6B, - 0xA3, 0x3C, 0x90, 0x05, 0xBD, 0xAC, 0x32, 0x2A, -}; -static unsigned char kbkdf_kmac_expected[] = { - 0xB1, 0x58, 0xEE, 0xB1, 0x34, 0xA4, 0xDD, 0x9D, - 0xAC, 0x52, 0xBD, 0x9E, 0x30, 0xE8, 0x0D, 0x76, - 0x42, 0x57, 0x01, 0x89, 0x5F, 0x82, 0x74, 0xB9, - 0xEB, 0x3E, 0x84, 0xD8, 0xA5, 0xDE, 0x6E, 0x54, -}; -static const ST_KAT_PARAM kbkdf_kmac_params[] = { - ST_KAT_PARAM_UTF8STRING(OSSL_KDF_PARAM_MAC, kbkdf_kmac_mac), - ST_KAT_PARAM_OCTET(OSSL_KDF_PARAM_KEY, kbkdf_kmac_key), - ST_KAT_PARAM_OCTET(OSSL_KDF_PARAM_SALT, kbkdf_kmac_label), - ST_KAT_PARAM_OCTET(OSSL_KDF_PARAM_INFO, kbkdf_kmac_context), - ST_KAT_PARAM_END() -}; - static const char tls13_kdf_digest[] = "SHA256"; static int tls13_kdf_extract_mode = EVP_KDF_HKDF_MODE_EXTRACT_ONLY; static int tls13_kdf_expand_mode = EVP_KDF_HKDF_MODE_EXPAND_ONLY; @@ -614,12 +584,6 @@ static const ST_KAT_KDF st_kat_kdf_tests[] = kbkdf_params, ITM(kbkdf_expected) }, - { - OSSL_SELF_TEST_DESC_KDF_KBKDF_KMAC, - OSSL_KDF_NAME_KBKDF, - kbkdf_kmac_params, - ITM(kbkdf_kmac_expected) - }, { OSSL_SELF_TEST_DESC_KDF_HKDF, OSSL_KDF_NAME_HKDF, @@ -1306,11 +1270,11 @@ static const ST_KAT_PARAM rsa_crt_key[] = { ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_N, rsa_n), ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_E, rsa_e), ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_D, rsa_d), - ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_FACTOR1, rsa_p), - ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_FACTOR2, rsa_q), - ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_EXPONENT1, rsa_dp), - ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_EXPONENT2, rsa_dq), - ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_COEFFICIENT1, rsa_qInv), + ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_FACTOR, rsa_p), + ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_FACTOR, rsa_q), + ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_EXPONENT, rsa_dp), + ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_EXPONENT, rsa_dq), + ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_RSA_COEFFICIENT, rsa_qInv), ST_KAT_PARAM_END() }; @@ -1432,16 +1396,6 @@ static const unsigned char ecd_prime_pub[] = { 0xc4, 0xb7, 0x33, 0x68, 0xe4, 0x24, 0xa9, 0x12, 0x82 }; -static const unsigned char ecdsa_prime_expected_sig[] = { - 0x30, 0x3d, 0x02, 0x1c, 0x48, 0x4f, 0x3c, 0x97, - 0x5b, 0xfa, 0x40, 0x6c, 0xdb, 0xd6, 0x70, 0xb5, - 0xbd, 0x2d, 0xd0, 0xc6, 0x22, 0x93, 0x5a, 0x88, - 0x56, 0xd0, 0xaf, 0x0a, 0x94, 0x92, 0x20, 0x01, - 0x02, 0x1d, 0x00, 0xa4, 0x80, 0xe0, 0x47, 0x88, - 0x8a, 0xef, 0x2a, 0x47, 0x9d, 0x81, 0x9a, 0xbf, - 0x45, 0xc3, 0x6f, 0x9e, 0x2e, 0xc1, 0x44, 0x9f, - 0xfd, 0x79, 0xdb, 0x90, 0x3e, 0xb9, 0xb2 -}; static const ST_KAT_PARAM ecdsa_prime_key[] = { ST_KAT_PARAM_UTF8STRING(OSSL_PKEY_PARAM_GROUP_NAME, ecd_prime_curve_name), ST_KAT_PARAM_OCTET(OSSL_PKEY_PARAM_PUB_KEY, ecd_prime_pub), @@ -1467,17 +1421,6 @@ static const unsigned char ecd_bin_pub[] = { 0x02, 0xa8, 0xe9, 0x6f, 0x54, 0xfd, 0x3a, 0x6b, 0x99, 0xb6, 0x8f, 0x80, 0x46 }; -static const unsigned char ecdsa_bin_expected_sig[] = { - 0x30, 0x3f, 0x02, 0x1d, 0x58, 0xe9, 0xd0, 0x84, - 0x5c, 0xad, 0x29, 0x03, 0xf6, 0xa6, 0xbc, 0xe0, - 0x24, 0x6d, 0x9e, 0x79, 0x5d, 0x1e, 0xe8, 0x5a, - 0xc3, 0x31, 0x0a, 0xa9, 0xfb, 0xe3, 0x99, 0x54, - 0x11, 0x02, 0x1e, 0x00, 0xa3, 0x44, 0x28, 0xa3, - 0x70, 0x97, 0x98, 0x17, 0xd7, 0xa6, 0xad, 0x91, - 0xaf, 0x41, 0x69, 0xb6, 0x06, 0x99, 0x39, 0xc7, - 0x63, 0xa4, 0x6a, 0x81, 0xe4, 0x9a, 0x9d, 0x15, - 0x8b -}; static const ST_KAT_PARAM ecdsa_bin_key[] = { ST_KAT_PARAM_UTF8STRING(OSSL_PKEY_PARAM_GROUP_NAME, ecd_bin_curve_name), ST_KAT_PARAM_OCTET(OSSL_PKEY_PARAM_PUB_KEY, ecd_bin_pub), @@ -1603,16 +1546,6 @@ static const unsigned char dsa_priv[] = { 0x66, 0x35, 0xba, 0xc3, 0x94, 0x23, 0x50, 0x5e, 0x40, 0x7e, 0x5c, 0xb7 }; -static const unsigned char dsa_expected_sig[] = { - 0x30, 0x3c, 0x02, 0x1c, 0x69, 0xc6, 0xd6, 0x9e, - 0x2b, 0x91, 0xea, 0x72, 0xb3, 0x8b, 0x7c, 0x57, - 0x48, 0x75, 0xb7, 0x65, 0xc0, 0xb4, 0xf7, 0xbb, - 0x08, 0xa4, 0x95, 0x77, 0xfc, 0xa7, 0xed, 0x31, - 0x02, 0x1c, 0x4c, 0x2c, 0xff, 0xc6, 0x55, 0xeb, - 0x8f, 0xa7, 0x4f, 0x27, 0xd8, 0xec, 0xfd, 0x62, - 0x73, 0xf2, 0xd1, 0x55, 0xa5, 0xf0, 0x41, 0x68, - 0x34, 0x8d, 0x9e, 0x88, 0x08, 0x06 -}; static const ST_KAT_PARAM dsa_key[] = { ST_KAT_PARAM_BIGNUM(OSSL_PKEY_PARAM_FFC_P, dsa_p), @@ -1624,31 +1557,12 @@ static const ST_KAT_PARAM dsa_key[] = { }; #endif /* OPENSSL_NO_DSA */ -/* Hash DRBG inputs for signature KATs */ -static const unsigned char sig_kat_entropyin[] = { - 0x06, 0x6d, 0xc8, 0xce, 0x75, 0xb2, 0x89, 0x66, 0xa6, 0x85, 0x16, 0x3f, - 0xe2, 0xa4, 0xd4, 0x27, 0xfb, 0xdb, 0x61, 0x66, 0x50, 0x61, 0x6b, 0xa2, - 0x82, 0xfc, 0x33, 0x2b, 0x4e, 0x6f, 0x12, 0x20 -}; -static const unsigned char sig_kat_nonce[] = { - 0x55, 0x9f, 0x7c, 0x64, 0x89, 0x70, 0x83, 0xec, 0x2d, 0x73, 0x70, 0xd9, - 0xf0, 0xe5, 0x07, 0x1f -}; -static const unsigned char sig_kat_persstr[] = { - 0x88, 0x6f, 0x54, 0x9a, 0xad, 0x1a, 0xc6, 0x3d, 0x18, 0xcb, 0xcc, 0x66, - 0x85, 0xda, 0xa2, 0xc2, 0xf7, 0x9e, 0xb0, 0x89, 0x4c, 0xb4, 0xae, 0xf1, - 0xac, 0x54, 0x4f, 0xce, 0x57, 0xf1, 0x5e, 0x11 -}; - static const ST_KAT_SIGN st_kat_sign_tests[] = { { OSSL_SELF_TEST_DESC_SIGN_RSA, "RSA", "SHA-256", rsa_crt_key, - ITM(sig_kat_entropyin), - ITM(sig_kat_nonce), - ITM(sig_kat_persstr), ITM(rsa_expected_sig) }, #ifndef OPENSSL_NO_EC @@ -1657,10 +1571,10 @@ static const ST_KAT_SIGN st_kat_sign_tests[] = { "EC", "SHA-256", ecdsa_prime_key, - ITM(sig_kat_entropyin), - ITM(sig_kat_nonce), - ITM(sig_kat_persstr), - ITM(ecdsa_prime_expected_sig) + /* + * The ECDSA signature changes each time due to it using a random k. + * So there is no expected KAT for this case. + */ }, # ifndef OPENSSL_NO_EC2M { @@ -1668,10 +1582,10 @@ static const ST_KAT_SIGN st_kat_sign_tests[] = { "EC", "SHA-256", ecdsa_bin_key, - ITM(sig_kat_entropyin), - ITM(sig_kat_nonce), - ITM(sig_kat_persstr), - ITM(ecdsa_bin_expected_sig) + /* + * The ECDSA signature changes each time due to it using a random k. + * So there is no expected KAT for this case. + */ }, # endif #endif /* OPENSSL_NO_EC */ @@ -1681,10 +1595,10 @@ static const ST_KAT_SIGN st_kat_sign_tests[] = { "DSA", "SHA-256", dsa_key, - ITM(sig_kat_entropyin), - ITM(sig_kat_nonce), - ITM(sig_kat_persstr), - ITM(dsa_expected_sig) + /* + * The DSA signature changes each time due to it using a random k. + * So there is no expected KAT for this case. + */ }, #endif /* OPENSSL_NO_DSA */ }; diff --git a/openssl/src/providers/fips/self_test_kats.c b/openssl/src/providers/fips/self_test_kats.c index f13c41abd..94a0cf842 100644 --- a/openssl/src/providers/fips/self_test_kats.c +++ b/openssl/src/providers/fips/self_test_kats.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,19 +12,11 @@ #include #include #include -#include -#include "crypto/rand.h" #include "internal/cryptlib.h" #include "internal/nelem.h" #include "self_test.h" #include "self_test_data.inc" -static int set_kat_drbg(OSSL_LIB_CTX *ctx, - const unsigned char *entropy, size_t entropy_len, - const unsigned char *nonce, size_t nonce_len, - const unsigned char *persstr, size_t persstr_len); -static int reset_main_drbg(OSSL_LIB_CTX *ctx); - static int self_test_digest(const ST_KAT_DIGEST *t, OSSL_SELF_TEST *st, OSSL_LIB_CTX *libctx) { @@ -80,10 +72,10 @@ static int cipher_init(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, in_tag = (unsigned char *)t->tag; return EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, enc) - && (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, t->iv_len, NULL) > 0) + && EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, t->iv_len, NULL) && (in_tag == NULL || EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, t->tag_len, - in_tag) > 0) + in_tag)) && EVP_CipherInit_ex(ctx, NULL, NULL, t->key, t->iv, enc) && EVP_CIPHER_CTX_set_padding(ctx, pad) && EVP_CipherUpdate(ctx, NULL, &tmp, t->aad, t->aad_len); @@ -125,8 +117,8 @@ static int self_test_cipher(const ST_KAT_CIPHER *t, OSSL_SELF_TEST *st, if (t->tag != NULL) { unsigned char tag[16] = { 0 }; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, t->tag_len, - tag) <= 0 + if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, t->tag_len, + tag) || memcmp(tag, t->tag, t->tag_len) != 0) goto err; } @@ -445,7 +437,7 @@ static int self_test_ka(const ST_KAT_KAS *t, #endif /* !defined(OPENSSL_NO_DH) || !defined(OPENSSL_NO_EC) */ static int self_test_sign(const ST_KAT_SIGN *t, - OSSL_SELF_TEST *st, OSSL_LIB_CTX *libctx) + OSSL_SELF_TEST *st, OSSL_LIB_CTX *libctx) { int ret = 0; OSSL_PARAM *params = NULL, *params_sig = NULL; @@ -507,6 +499,10 @@ static int self_test_sign(const ST_KAT_SIGN *t, || EVP_PKEY_CTX_set_params(sctx, params_sig) <= 0) goto err; + /* + * Used by RSA, for other key types where the signature changes, we + * can only use the verify. + */ if (t->sig_expected != NULL && (siglen != t->sig_expected_len || memcmp(sig, t->sig_expected, t->sig_expected_len) != 0)) @@ -693,161 +689,14 @@ static int self_test_kas(OSSL_SELF_TEST *st, OSSL_LIB_CTX *libctx) static int self_test_signatures(OSSL_SELF_TEST *st, OSSL_LIB_CTX *libctx) { int i, ret = 1; - const ST_KAT_SIGN *t; - - for (i = 0; ret && i < (int)OSSL_NELEM(st_kat_sign_tests); ++i) { - t = st_kat_sign_tests + i; - if (!set_kat_drbg(libctx, t->entropy, t->entropy_len, - t->nonce, t->nonce_len, t->persstr, t->persstr_len)) - return 0; - if (!self_test_sign(t, st, libctx)) - ret = 0; - if (!reset_main_drbg(libctx)) - ret = 0; - } - return ret; -} - -/* - * Swap the library context DRBG for KAT testing - * - * In FIPS 140-3, the asymmetric POST must be a KAT, not a PCT. For DSA and ECDSA, - * the sign operation includes the random value 'k'. For a KAT to work, we - * have to have control of the DRBG to make sure it is in a "test" state, where - * its output is truly deterministic. - * - */ - -/* - * Replacement "random" sources - * main_rand is used for most tests and it's set to generate mode. - * kat_rand is used for KATs where specific input is mandated. - */ -static EVP_RAND_CTX *kat_rand = NULL; -static EVP_RAND_CTX *main_rand = NULL; - -static int set_kat_drbg(OSSL_LIB_CTX *ctx, - const unsigned char *entropy, size_t entropy_len, - const unsigned char *nonce, size_t nonce_len, - const unsigned char *persstr, size_t persstr_len) { - EVP_RAND *rand; - unsigned int strength = 256; - EVP_RAND_CTX *parent_rand = NULL; - OSSL_PARAM drbg_params[3] = { - OSSL_PARAM_END, OSSL_PARAM_END, OSSL_PARAM_END - }; - - /* If not NULL, we didn't cleanup from last call: BAD */ - if (kat_rand != NULL) - return 0; - - rand = EVP_RAND_fetch(ctx, "TEST-RAND", NULL); - if (rand == NULL) - return 0; - - parent_rand = EVP_RAND_CTX_new(rand, NULL); - EVP_RAND_free(rand); - if (parent_rand == NULL) - goto err; - - drbg_params[0] = OSSL_PARAM_construct_uint(OSSL_RAND_PARAM_STRENGTH, - &strength); - if (!EVP_RAND_CTX_set_params(parent_rand, drbg_params)) - goto err; - - rand = EVP_RAND_fetch(ctx, "HASH-DRBG", NULL); - if (rand == NULL) - goto err; - - kat_rand = EVP_RAND_CTX_new(rand, parent_rand); - EVP_RAND_free(rand); - if (kat_rand == NULL) - goto err; - - drbg_params[0] = OSSL_PARAM_construct_utf8_string("digest", "SHA256", 0); - if (!EVP_RAND_CTX_set_params(kat_rand, drbg_params)) - goto err; - - /* Instantiate the RNGs */ - drbg_params[0] = - OSSL_PARAM_construct_octet_string(OSSL_RAND_PARAM_TEST_ENTROPY, - (void *)entropy, entropy_len); - drbg_params[1] = - OSSL_PARAM_construct_octet_string(OSSL_RAND_PARAM_TEST_NONCE, - (void *)nonce, nonce_len); - if (!EVP_RAND_instantiate(parent_rand, strength, 0, NULL, 0, drbg_params)) - goto err; - - EVP_RAND_CTX_free(parent_rand); - parent_rand = NULL; - - if (!EVP_RAND_instantiate(kat_rand, strength, 0, persstr, persstr_len, NULL)) - goto err; - /* When we set the new private generator this one is freed, so upref it */ - if (!EVP_RAND_CTX_up_ref(main_rand)) - goto err; - - /* Update the library context DRBG */ - if (RAND_set0_private(ctx, kat_rand) > 0) { - /* Keeping a copy to verify zeroization */ - if (EVP_RAND_CTX_up_ref(kat_rand)) - return 1; - RAND_set0_private(ctx, main_rand); - } - - err: - EVP_RAND_CTX_free(parent_rand); - EVP_RAND_CTX_free(kat_rand); - kat_rand = NULL; - return 0; -} - -static int reset_main_drbg(OSSL_LIB_CTX *ctx) { - int ret = 1; - - if (!RAND_set0_private(ctx, main_rand)) - ret = 0; - if (kat_rand != NULL) { - if (!EVP_RAND_uninstantiate(kat_rand) - || !EVP_RAND_verify_zeroization(kat_rand)) + for (i = 0; i < (int)OSSL_NELEM(st_kat_sign_tests); ++i) { + if (!self_test_sign(&st_kat_sign_tests[i], st, libctx)) ret = 0; - EVP_RAND_CTX_free(kat_rand); - kat_rand = NULL; } return ret; } -static int setup_main_random(OSSL_LIB_CTX *libctx) -{ - OSSL_PARAM drbg_params[3] = { - OSSL_PARAM_END, OSSL_PARAM_END, OSSL_PARAM_END - }; - unsigned int strength = 256, generate = 1; - EVP_RAND *rand; - - rand = EVP_RAND_fetch(libctx, "TEST-RAND", NULL); - if (rand == NULL) - return 0; - - main_rand = EVP_RAND_CTX_new(rand, NULL); - EVP_RAND_free(rand); - if (main_rand == NULL) - goto err; - - drbg_params[0] = OSSL_PARAM_construct_uint(OSSL_RAND_PARAM_GENERATE, - &generate); - drbg_params[1] = OSSL_PARAM_construct_uint(OSSL_RAND_PARAM_STRENGTH, - &strength); - - if (!EVP_RAND_instantiate(main_rand, strength, 0, NULL, 0, drbg_params)) - goto err; - return 1; - err: - EVP_RAND_CTX_free(main_rand); - return 0; -} - /* * Run the algorithm KAT's. * Return 1 is successful, otherwise return 0. @@ -855,15 +704,8 @@ static int setup_main_random(OSSL_LIB_CTX *libctx) */ int SELF_TEST_kats(OSSL_SELF_TEST *st, OSSL_LIB_CTX *libctx) { - EVP_RAND_CTX *saved_rand = ossl_rand_get0_private_noncreating(libctx); int ret = 1; - if (!setup_main_random(libctx) - || !RAND_set0_private(libctx, main_rand)) { - EVP_RAND_CTX_free(main_rand); - return 0; - } - if (!self_test_digests(st, libctx)) ret = 0; if (!self_test_ciphers(st, libctx)) @@ -879,7 +721,5 @@ int SELF_TEST_kats(OSSL_SELF_TEST *st, OSSL_LIB_CTX *libctx) if (!self_test_asym_ciphers(st, libctx)) ret = 0; - RAND_set0_private(libctx, saved_rand); return ret; } - diff --git a/openssl/src/providers/implementations/asymciphers/rsa_enc.c b/openssl/src/providers/implementations/asymciphers/rsa_enc.c index 71bfa344d..ce5ddff65 100644 --- a/openssl/src/providers/implementations/asymciphers/rsa_enc.c +++ b/openssl/src/providers/implementations/asymciphers/rsa_enc.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -75,8 +75,6 @@ typedef struct { /* TLS padding */ unsigned int client_version; unsigned int alt_version; - /* PKCS#1 v1.5 decryption mode */ - unsigned int implicit_rejection; } PROV_RSA_CTX; static void *rsa_newctx(void *provctx) @@ -109,7 +107,6 @@ static int rsa_init(void *vprsactx, void *vrsa, const OSSL_PARAM params[], RSA_free(prsactx->rsa); prsactx->rsa = vrsa; prsactx->operation = operation; - prsactx->implicit_rejection = 1; switch (RSA_test_flags(prsactx->rsa, RSA_FLAG_TYPE_MASK)) { case RSA_FLAG_TYPE_RSA: @@ -159,15 +156,15 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen, int rsasize = RSA_size(prsactx->rsa); unsigned char *tbuf; - if ((tbuf = OPENSSL_malloc(rsasize)) == NULL) + if ((tbuf = OPENSSL_malloc(rsasize)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } if (prsactx->oaep_md == NULL) { + OPENSSL_free(tbuf); prsactx->oaep_md = EVP_MD_fetch(prsactx->libctx, "SHA-1", NULL); - if (prsactx->oaep_md == NULL) { - OPENSSL_free(tbuf); - ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR); - return 0; - } + ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR); + return 0; } ret = ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(prsactx->libctx, tbuf, @@ -200,7 +197,6 @@ static int rsa_decrypt(void *vprsactx, unsigned char *out, size_t *outlen, { PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx; int ret; - int pad_mode; size_t len = RSA_size(prsactx->rsa); if (!ossl_prov_is_running()) @@ -235,8 +231,10 @@ static int rsa_decrypt(void *vprsactx, unsigned char *out, size_t *outlen, || prsactx->pad_mode == RSA_PKCS1_WITH_TLS_PADDING) { unsigned char *tbuf; - if ((tbuf = OPENSSL_malloc(len)) == NULL) + if ((tbuf = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } ret = RSA_private_decrypt(inlen, in, tbuf, prsactx->rsa, RSA_NO_PADDING); /* @@ -276,12 +274,8 @@ static int rsa_decrypt(void *vprsactx, unsigned char *out, size_t *outlen, } OPENSSL_free(tbuf); } else { - if ((prsactx->implicit_rejection == 0) && - (prsactx->pad_mode == RSA_PKCS1_PADDING)) - pad_mode = RSA_PKCS1_NO_IMPLICIT_REJECT_PADDING; - else - pad_mode = prsactx->pad_mode; - ret = RSA_private_decrypt(inlen, in, out, prsactx->rsa, pad_mode); + ret = RSA_private_decrypt(inlen, in, out, prsactx->rsa, + prsactx->pad_mode); } *outlen = constant_time_select_s(constant_time_msb_s(ret), *outlen, ret); ret = constant_time_select_int(constant_time_msb(ret), 0, 1); @@ -405,10 +399,6 @@ static int rsa_get_ctx_params(void *vprsactx, OSSL_PARAM *params) if (p != NULL && !OSSL_PARAM_set_uint(p, prsactx->alt_version)) return 0; - p = OSSL_PARAM_locate(params, OSSL_ASYM_CIPHER_PARAM_IMPLICIT_REJECTION); - if (p != NULL && !OSSL_PARAM_set_uint(p, prsactx->implicit_rejection)) - return 0; - return 1; } @@ -420,7 +410,6 @@ static const OSSL_PARAM known_gettable_ctx_params[] = { NULL, 0), OSSL_PARAM_uint(OSSL_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION, NULL), OSSL_PARAM_uint(OSSL_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION, NULL), - OSSL_PARAM_uint(OSSL_ASYM_CIPHER_PARAM_IMPLICIT_REJECTION, NULL), OSSL_PARAM_END }; @@ -436,7 +425,7 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[]) const OSSL_PARAM *p; char mdname[OSSL_MAX_NAME_SIZE]; char mdprops[OSSL_MAX_PROPQUERY_SIZE] = { '\0' }; - char *str = NULL; + char *str = mdname; if (prsactx == NULL) return 0; @@ -445,14 +434,13 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[]) p = OSSL_PARAM_locate_const(params, OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST); if (p != NULL) { - str = mdname; if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdname))) return 0; + str = mdprops; p = OSSL_PARAM_locate_const(params, OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS); if (p != NULL) { - str = mdprops; if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdprops))) return 0; } @@ -508,14 +496,13 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[]) p = OSSL_PARAM_locate_const(params, OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST); if (p != NULL) { - str = mdname; if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdname))) return 0; + str = mdprops; p = OSSL_PARAM_locate_const(params, OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST_PROPS); if (p != NULL) { - str = mdprops; if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdprops))) return 0; } else { @@ -558,28 +545,18 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[]) return 0; prsactx->alt_version = alt_version; } - p = OSSL_PARAM_locate_const(params, OSSL_ASYM_CIPHER_PARAM_IMPLICIT_REJECTION); - if (p != NULL) { - unsigned int implicit_rejection; - - if (!OSSL_PARAM_get_uint(p, &implicit_rejection)) - return 0; - prsactx->implicit_rejection = implicit_rejection; - } return 1; } static const OSSL_PARAM known_settable_ctx_params[] = { OSSL_PARAM_utf8_string(OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST, NULL, 0), - OSSL_PARAM_utf8_string(OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS, NULL, 0), OSSL_PARAM_utf8_string(OSSL_ASYM_CIPHER_PARAM_PAD_MODE, NULL, 0), OSSL_PARAM_utf8_string(OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST, NULL, 0), OSSL_PARAM_utf8_string(OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST_PROPS, NULL, 0), OSSL_PARAM_octet_string(OSSL_ASYM_CIPHER_PARAM_OAEP_LABEL, NULL, 0), OSSL_PARAM_uint(OSSL_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION, NULL), OSSL_PARAM_uint(OSSL_ASYM_CIPHER_PARAM_TLS_NEGOTIATED_VERSION, NULL), - OSSL_PARAM_uint(OSSL_ASYM_CIPHER_PARAM_IMPLICIT_REJECTION, NULL), OSSL_PARAM_END }; @@ -605,5 +582,5 @@ const OSSL_DISPATCH ossl_rsa_asym_cipher_functions[] = { (void (*)(void))rsa_set_ctx_params }, { OSSL_FUNC_ASYM_CIPHER_SETTABLE_CTX_PARAMS, (void (*)(void))rsa_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/asymciphers/sm2_enc.c b/openssl/src/providers/implementations/asymciphers/sm2_enc.c index a9d652be3..9577d16e8 100644 --- a/openssl/src/providers/implementations/asymciphers/sm2_enc.c +++ b/openssl/src/providers/implementations/asymciphers/sm2_enc.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -138,8 +138,6 @@ static void *sm2_dupctx(void *vpsm2ctx) return NULL; *dstctx = *srcctx; - memset(&dstctx->md, 0, sizeof(dstctx->md)); - if (dstctx->key != NULL && !EC_KEY_up_ref(dstctx->key)) { OPENSSL_free(dstctx); return NULL; @@ -229,5 +227,5 @@ const OSSL_DISPATCH ossl_sm2_asym_cipher_functions[] = { (void (*)(void))sm2_set_ctx_params }, { OSSL_FUNC_ASYM_CIPHER_SETTABLE_CTX_PARAMS, (void (*)(void))sm2_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes.c b/openssl/src/providers/implementations/ciphers/cipher_aes.c index 280be2ddd..2f469c131 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes.c @@ -40,8 +40,10 @@ static void *aes_dupctx(void *ctx) return NULL; ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } in->base.hw->copyctx(&ret->base, &in->base); return ret; diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes.h b/openssl/src/providers/implementations/ciphers/cipher_aes.h index c62ac5e7e..7eaf76c8c 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes.h +++ b/openssl/src/providers/implementations/ciphers/cipher_aes.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -44,6 +44,7 @@ typedef struct prov_aes_ctx_st { /* KMO-AES/KMF-AES parameter block - end */ } param; unsigned int fc; + int res; } s390x; #endif /* defined(OPENSSL_CPUID_OBJ) && defined(__s390__) */ } plat; diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha.c b/openssl/src/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha.c index 28d3909c4..f9a8a5804 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,7 +26,7 @@ #ifndef AES_CBC_HMAC_SHA_CAPABLE # define IMPLEMENT_CIPHER(nm, sub, kbits, blkbits, ivbits, flags) \ const OSSL_DISPATCH ossl_##nm##kbits##sub##_functions[] = { \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; #else @@ -334,16 +334,6 @@ static void *aes_cbc_hmac_sha1_newctx(void *provctx, size_t kbits, return ctx; } -static void *aes_cbc_hmac_sha1_dupctx(void *provctx) -{ - PROV_AES_HMAC_SHA1_CTX *ctx = provctx; - - if (ctx == NULL) - return NULL; - - return OPENSSL_memdup(ctx, sizeof(*ctx)); -} - static void aes_cbc_hmac_sha1_freectx(void *vctx) { PROV_AES_HMAC_SHA1_CTX *ctx = (PROV_AES_HMAC_SHA1_CTX *)vctx; @@ -371,13 +361,6 @@ static void *aes_cbc_hmac_sha256_newctx(void *provctx, size_t kbits, return ctx; } -static void *aes_cbc_hmac_sha256_dupctx(void *provctx) -{ - PROV_AES_HMAC_SHA256_CTX *ctx = provctx; - - return OPENSSL_memdup(ctx, sizeof(*ctx)); -} - static void aes_cbc_hmac_sha256_freectx(void *vctx) { PROV_AES_HMAC_SHA256_CTX *ctx = (PROV_AES_HMAC_SHA256_CTX *)vctx; @@ -403,7 +386,6 @@ static int nm##_##kbits##_##sub##_get_params(OSSL_PARAM params[]) \ const OSSL_DISPATCH ossl_##nm##kbits##sub##_functions[] = { \ { OSSL_FUNC_CIPHER_NEWCTX, (void (*)(void))nm##_##kbits##_##sub##_newctx },\ { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))nm##_##sub##_freectx }, \ - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void))nm##_##sub##_dupctx}, \ { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))nm##_einit }, \ { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))nm##_dinit }, \ { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))nm##_update }, \ @@ -421,7 +403,7 @@ const OSSL_DISPATCH ossl_##nm##kbits##sub##_functions[] = { \ (void (*)(void))nm##_set_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))nm##_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; #endif /* AES_CBC_HMAC_SHA_CAPABLE */ diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_ccm.c b/openssl/src/providers/implementations/ciphers/cipher_aes_ccm.c index 8c9632809..bb4b1e1e6 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_ccm.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_ccm.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,26 +33,6 @@ static void *aes_ccm_newctx(void *provctx, size_t keybits) return ctx; } -static void *aes_ccm_dupctx(void *provctx) -{ - PROV_AES_CCM_CTX *ctx = provctx; - PROV_AES_CCM_CTX *dupctx = NULL; - - if (ctx == NULL) - return NULL; - dupctx = OPENSSL_memdup(provctx, sizeof(*ctx)); - if (dupctx == NULL) - return NULL; - /* - * ossl_cm_initctx, via the ossl_prov_aes_hw_ccm functions assign a - * provctx->ccm.ks.ks to the ccm context key so we need to point it to - * the memduped copy - */ - dupctx->base.ccm_ctx.key = &dupctx->ccm.ks.ks; - - return dupctx; -} - static OSSL_FUNC_cipher_freectx_fn aes_ccm_freectx; static void aes_ccm_freectx(void *vctx) { diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c index b050cf3ed..263d19028 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -61,10 +61,6 @@ static const PROV_CCM_HW aes_ccm = { # include "cipher_aes_ccm_hw_aesni.inc" #elif defined(SPARC_AES_CAPABLE) # include "cipher_aes_ccm_hw_t4.inc" -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 -# include "cipher_aes_ccm_hw_rv64i.inc" -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 -# include "cipher_aes_ccm_hw_rv32i.inc" #else const PROV_CCM_HW *ossl_prov_aes_hw_ccm(size_t keybits) { diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw_rv32i.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw_rv32i.inc deleted file mode 100644 index 7cfe0fc4c..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw_rv32i.inc +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RISC-V 32 ZKND ZKNE support for AES CCM. - * This file is included by cipher_aes_ccm_hw.c - */ - -static int ccm_rv32i_zknd_zkne_initkey(PROV_CCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_AES_CCM_CTX *actx = (PROV_AES_CCM_CTX *)ctx; - - AES_HW_CCM_SET_KEY_FN(rv32i_zkne_set_encrypt_key, rv32i_zkne_encrypt, - NULL, NULL); - return 1; -} - -static int ccm_rv32i_zbkb_zknd_zkne_initkey(PROV_CCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_AES_CCM_CTX *actx = (PROV_AES_CCM_CTX *)ctx; - - AES_HW_CCM_SET_KEY_FN(rv32i_zbkb_zkne_set_encrypt_key, rv32i_zkne_encrypt, - NULL, NULL); - return 1; -} - -static const PROV_CCM_HW rv32i_zknd_zkne_ccm = { - ccm_rv32i_zknd_zkne_initkey, - ossl_ccm_generic_setiv, - ossl_ccm_generic_setaad, - ossl_ccm_generic_auth_encrypt, - ossl_ccm_generic_auth_decrypt, - ossl_ccm_generic_gettag -}; - -static const PROV_CCM_HW rv32i_zbkb_zknd_zkne_ccm = { - ccm_rv32i_zbkb_zknd_zkne_initkey, - ossl_ccm_generic_setiv, - ossl_ccm_generic_setaad, - ossl_ccm_generic_auth_encrypt, - ossl_ccm_generic_auth_decrypt, - ossl_ccm_generic_gettag -}; - -const PROV_CCM_HW *ossl_prov_aes_hw_ccm(size_t keybits) -{ - if (RISCV_HAS_ZBKB_AND_ZKND_AND_ZKNE()) - return &rv32i_zbkb_zknd_zkne_ccm; - if (RISCV_HAS_ZKND_AND_ZKNE()) - return &rv32i_zknd_zkne_ccm; - return &aes_ccm; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw_rv64i.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw_rv64i.inc deleted file mode 100644 index f2353bb3b..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_ccm_hw_rv64i.inc +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RISC-V 64 ZKND ZKNE support for AES CCM. - * This file is included by cipher_aes_ccm_hw.c - */ - -static int ccm_rv64i_zknd_zkne_initkey(PROV_CCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_AES_CCM_CTX *actx = (PROV_AES_CCM_CTX *)ctx; - - AES_HW_CCM_SET_KEY_FN(rv64i_zkne_set_encrypt_key, rv64i_zkne_encrypt, - NULL, NULL); - return 1; -} - -static const PROV_CCM_HW rv64i_zknd_zkne_ccm = { - ccm_rv64i_zknd_zkne_initkey, - ossl_ccm_generic_setiv, - ossl_ccm_generic_setaad, - ossl_ccm_generic_auth_encrypt, - ossl_ccm_generic_auth_decrypt, - ossl_ccm_generic_gettag -}; - -/*- - * RISC-V RV64 ZVKNED support for AES CCM. - * This file is included by cipher_aes_ccm_hw.c - */ - -static int ccm_rv64i_zvkned_initkey(PROV_CCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_AES_CCM_CTX *actx = (PROV_AES_CCM_CTX *)ctx; - - /* Zvkned only supports 128 and 256 bit keys for key schedule generation. */ - if (keylen * 8 == 128 || keylen * 8 == 256) { - AES_HW_CCM_SET_KEY_FN(rv64i_zvkned_set_encrypt_key, rv64i_zvkned_encrypt, - NULL, NULL); - } else { - AES_HW_CCM_SET_KEY_FN(AES_set_encrypt_key, rv64i_zvkned_encrypt, NULL, NULL) - } - return 1; -} - -static const PROV_CCM_HW rv64i_zvkned_ccm = { - ccm_rv64i_zvkned_initkey, - ossl_ccm_generic_setiv, - ossl_ccm_generic_setaad, - ossl_ccm_generic_auth_encrypt, - ossl_ccm_generic_auth_decrypt, - ossl_ccm_generic_gettag -}; - -const PROV_CCM_HW *ossl_prov_aes_hw_ccm(size_t keybits) -{ - if (RISCV_HAS_ZVKNED() && riscv_vlen() >= 128) - return &rv64i_zvkned_ccm; - else if (RISCV_HAS_ZKND_AND_ZKNE()) - return &rv64i_zknd_zkne_ccm; - else - return &aes_ccm; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm.c b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm.c index 1114bd874..0081ca6cd 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -34,21 +34,6 @@ static void *aes_gcm_newctx(void *provctx, size_t keybits) return ctx; } -static void *aes_gcm_dupctx(void *provctx) -{ - PROV_AES_GCM_CTX *ctx = provctx; - PROV_AES_GCM_CTX *dctx = NULL; - - if (ctx == NULL) - return NULL; - - dctx = OPENSSL_memdup(ctx, sizeof(*ctx)); - if (dctx != NULL && dctx->base.gcm.key != NULL) - dctx->base.gcm.key = &dctx->ks.ks; - - return dctx; -} - static OSSL_FUNC_cipher_freectx_fn aes_gcm_freectx; static void aes_gcm_freectx(void *vctx) { diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c index 207a16bc7..44fa9d4d7 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -55,6 +55,7 @@ static int aes_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, GCM_HW_SET_KEY_CTR_FN(ks, AES_set_encrypt_key, AES_encrypt, NULL); # endif /* AES_CTR_ASM */ } + ctx->key_set = 1; return 1; } @@ -99,7 +100,7 @@ static int generic_aes_gcm_cipher_update(PROV_GCM_CTX *ctx, const unsigned char size_t res = (16 - ctx->gcm.mres) % 16; if (CRYPTO_gcm128_decrypt(&ctx->gcm, in, out, res)) - return 0; + return -1; bulk = AES_gcm_decrypt(in + res, out + res, len - res, ctx->gcm.key, @@ -140,12 +141,6 @@ static const PROV_GCM_HW aes_gcm = { # include "cipher_aes_gcm_hw_t4.inc" #elif defined(AES_PMULL_CAPABLE) && defined(AES_GCM_ASM) # include "cipher_aes_gcm_hw_armv8.inc" -#elif defined(PPC_AES_GCM_CAPABLE) && defined(_ARCH_PPC64) -# include "cipher_aes_gcm_hw_ppc.inc" -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 -# include "cipher_aes_gcm_hw_rv64i.inc" -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 -# include "cipher_aes_gcm_hw_rv32i.inc" #else const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits) { diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_aesni.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_aesni.inc index 92f41b8cd..e6aa0479d 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_aesni.inc +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_aesni.inc @@ -1,5 +1,5 @@ /* - * Copyright 2001-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -31,17 +31,8 @@ static const PROV_GCM_HW aesni_gcm = { ossl_gcm_one_shot }; -#include "cipher_aes_gcm_hw_vaes_avx512.inc" - const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits) { -#ifdef VAES_GCM_ENABLED - if (ossl_vaes_vpclmulqdq_capable()) - return &vaes_gcm; - else -#endif - if (AESNI_CAPABLE) - return &aesni_gcm; - else - return &aes_gcm; + return AESNI_CAPABLE ? &aesni_gcm : &aes_gcm; } + diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_armv8.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_armv8.inc index cc2407150..310f4470d 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_armv8.inc +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_armv8.inc @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,7 +8,7 @@ */ /* - * Crypto extension support for AES GCM. + * Crypto extention support for AES GCM. * This file is included by cipher_aes_gcm_hw.c */ @@ -22,25 +22,13 @@ size_t armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t switch(aes_key->rounds) { case 10: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_enc_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_enc_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } + aes_gcm_enc_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); break; case 12: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_enc_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_enc_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } + aes_gcm_enc_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); break; case 14: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_enc_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_enc_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } + aes_gcm_enc_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); break; } return align_bytes; @@ -56,25 +44,13 @@ size_t armv8_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t switch(aes_key->rounds) { case 10: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_dec_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_dec_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } + aes_gcm_dec_128_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); break; case 12: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_dec_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_dec_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } + aes_gcm_dec_192_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); break; case 14: - if (IS_CPU_SUPPORT_UNROLL8_EOR3()) { - unroll8_eor3_aes_gcm_dec_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } else { - aes_gcm_dec_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); - } + aes_gcm_dec_256_kernel(in, align_bytes * 8, out, (uint64_t *)Xi, ivec, key); break; } return align_bytes; @@ -86,13 +62,8 @@ static int armv8_aes_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; AES_KEY *ks = &actx->ks.ks; - if (AES_UNROLL12_EOR3_CAPABLE) { - GCM_HW_SET_KEY_CTR_FN(ks, aes_v8_set_encrypt_key, aes_v8_encrypt, - aes_v8_ctr32_encrypt_blocks_unroll12_eor3); - } else { - GCM_HW_SET_KEY_CTR_FN(ks, aes_v8_set_encrypt_key, aes_v8_encrypt, - aes_v8_ctr32_encrypt_blocks); - } + GCM_HW_SET_KEY_CTR_FN(ks, aes_v8_set_encrypt_key, aes_v8_encrypt, + aes_v8_ctr32_encrypt_blocks); return 1; } diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_ppc.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_ppc.inc deleted file mode 100644 index 153eb7989..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_ppc.inc +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright 2001-2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * PPC support for AES GCM. - * This file is included by cipher_aes_gcm_hw.c - */ - -static int aes_ppc_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; - AES_KEY *ks = &actx->ks.ks; - - GCM_HW_SET_KEY_CTR_FN(ks, aes_p8_set_encrypt_key, aes_p8_encrypt, - aes_p8_ctr32_encrypt_blocks); - return 1; -} - -static inline u32 UTO32(unsigned char *buf) -{ - return ((u32) buf[0] << 24) | ((u32) buf[1] << 16) | ((u32) buf[2] << 8) | ((u32) buf[3]); -} - -static inline u32 add32TOU(unsigned char buf[4], u32 n) -{ - u32 r; - - r = UTO32(buf); - r += n; - buf[0] = (unsigned char) (r >> 24) & 0xFF; - buf[1] = (unsigned char) (r >> 16) & 0xFF; - buf[2] = (unsigned char) (r >> 8) & 0xFF; - buf[3] = (unsigned char) r & 0xFF; - return r; -} - -static size_t ppc_aes_gcm_crypt(const unsigned char *in, unsigned char *out, size_t len, - const void *key, unsigned char ivec[16], u64 *Xi, int encrypt) -{ - int s = 0; - int ndone = 0; - int ctr_reset = 0; - u64 blocks_unused; - u64 nb = len / 16; - u64 next_ctr = 0; - unsigned char ctr_saved[12]; - - memcpy(ctr_saved, ivec, 12); - - while (nb) { - blocks_unused = (u64) 0xffffffffU + 1 - (u64) UTO32 (ivec + 12); - if (nb > blocks_unused) { - len = blocks_unused * 16; - nb -= blocks_unused; - next_ctr = blocks_unused; - ctr_reset = 1; - } else { - len = nb * 16; - next_ctr = nb; - nb = 0; - } - - s = encrypt ? ppc_aes_gcm_encrypt(in, out, len, key, ivec, Xi) - : ppc_aes_gcm_decrypt(in, out, len, key, ivec, Xi); - - /* add counter to ivec */ - add32TOU(ivec + 12, (u32) next_ctr); - if (ctr_reset) { - ctr_reset = 0; - in += len; - out += len; - } - memcpy(ivec, ctr_saved, 12); - ndone += s; - } - - return ndone; -} - -static int ppc_aes_gcm_cipher_update(PROV_GCM_CTX *ctx, const unsigned char *in, - size_t len, unsigned char *out) -{ - if (ctx->enc) { - if (ctx->ctr != NULL) { - size_t bulk = 0; - - if (len >= AES_GCM_ENC_BYTES && AES_GCM_ASM_PPC(ctx)) { - size_t res = (16 - ctx->gcm.mres) % 16; - - if (CRYPTO_gcm128_encrypt(&ctx->gcm, in, out, res)) - return 0; - - bulk = ppc_aes_gcm_crypt(in + res, out + res, len - res, - ctx->gcm.key, - ctx->gcm.Yi.c, ctx->gcm.Xi.u, 1); - - ctx->gcm.len.u[1] += bulk; - bulk += res; - } - if (CRYPTO_gcm128_encrypt_ctr32(&ctx->gcm, in + bulk, out + bulk, - len - bulk, ctx->ctr)) - return 0; - } else { - if (CRYPTO_gcm128_encrypt(&ctx->gcm, in, out, len)) - return 0; - } - } else { - if (ctx->ctr != NULL) { - size_t bulk = 0; - - if (len >= AES_GCM_DEC_BYTES && AES_GCM_ASM_PPC(ctx)) { - size_t res = (16 - ctx->gcm.mres) % 16; - - if (CRYPTO_gcm128_decrypt(&ctx->gcm, in, out, res)) - return -1; - - bulk = ppc_aes_gcm_crypt(in + res, out + res, len - res, - ctx->gcm.key, - ctx->gcm.Yi.c, ctx->gcm.Xi.u, 0); - - ctx->gcm.len.u[1] += bulk; - bulk += res; - } - if (CRYPTO_gcm128_decrypt_ctr32(&ctx->gcm, in + bulk, out + bulk, - len - bulk, ctx->ctr)) - return 0; - } else { - if (CRYPTO_gcm128_decrypt(&ctx->gcm, in, out, len)) - return 0; - } - } - return 1; -} - -static const PROV_GCM_HW aes_ppc_gcm = { - aes_ppc_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - ppc_aes_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; - -const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits) -{ - return PPC_AES_GCM_CAPABLE ? &aes_ppc_gcm : &aes_gcm; -} - diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_rv32i.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_rv32i.inc deleted file mode 100644 index bf3f98df1..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_rv32i.inc +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RISC-V 32 ZKND ZKNE support for AES GCM. - * This file is included by cipher_aes_gcm_hw.c - */ - -static int rv32i_zknd_zkne_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; - AES_KEY *ks = &actx->ks.ks; - - GCM_HW_SET_KEY_CTR_FN(ks, rv32i_zkne_set_encrypt_key, rv32i_zkne_encrypt, - NULL); - return 1; -} - -static int rv32i_zbkb_zknd_zkne_gcm_initkey(PROV_GCM_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; - AES_KEY *ks = &actx->ks.ks; - - GCM_HW_SET_KEY_CTR_FN(ks, rv32i_zbkb_zkne_set_encrypt_key, rv32i_zkne_encrypt, - NULL); - return 1; -} - -static const PROV_GCM_HW rv32i_zknd_zkne_gcm = { - rv32i_zknd_zkne_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - generic_aes_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; - -static const PROV_GCM_HW rv32i_zbkb_zknd_zkne_gcm = { - rv32i_zbkb_zknd_zkne_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - generic_aes_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; - -const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits) -{ - if (RISCV_HAS_ZBKB_AND_ZKND_AND_ZKNE()) - return &rv32i_zbkb_zknd_zkne_gcm; - if (RISCV_HAS_ZKND_AND_ZKNE()) - return &rv32i_zknd_zkne_gcm; - return &aes_gcm; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_rv64i.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_rv64i.inc deleted file mode 100644 index 105ca58fd..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_rv64i.inc +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RISC-V 64 support for AES GCM. - * This file is included by cipher_aes_gcm_hw.c - */ - -/*- - * RISC-V 64 ZKND and ZKNE support for AES GCM. - */ -static int rv64i_zknd_zkne_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; - AES_KEY *ks = &actx->ks.ks; - GCM_HW_SET_KEY_CTR_FN(ks, rv64i_zkne_set_encrypt_key, rv64i_zkne_encrypt, - NULL); - return 1; -} - -static const PROV_GCM_HW rv64i_zknd_zkne_gcm = { - rv64i_zknd_zkne_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - generic_aes_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; - -/*- - * RISC-V RV64 ZVKNED support for AES GCM. - */ -static int rv64i_zvkned_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; - AES_KEY *ks = &actx->ks.ks; - - /* - * Zvkned only supports 128 and 256 bit keys for key schedule generation. - * For AES-192 case, we could fallback to `AES_set_encrypt_key`. - */ - if (keylen * 8 == 128 || keylen * 8 == 256) { - GCM_HW_SET_KEY_CTR_FN(ks, rv64i_zvkned_set_encrypt_key, - rv64i_zvkned_encrypt, NULL); - } else { - GCM_HW_SET_KEY_CTR_FN(ks, AES_set_encrypt_key, - rv64i_zvkned_encrypt, NULL); - } - - return 1; -} - -static const PROV_GCM_HW rv64i_zvkned_gcm = { - rv64i_zvkned_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - generic_aes_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; - -/*- - * RISC-V RV64 ZVKB, ZVKG and ZVKNED support for AES GCM. - */ -static int rv64i_zvkb_zvkg_zvkned_gcm_initkey(PROV_GCM_CTX *ctx, - const unsigned char *key, - size_t keylen) { - PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; - AES_KEY *ks = &actx->ks.ks; - - /* - * Zvkned only supports 128 and 256 bit keys for key schedule generation. - * For AES-192 case, we could fallback to `AES_set_encrypt_key`. - */ - if (keylen * 8 == 128 || keylen * 8 == 256) { - GCM_HW_SET_KEY_CTR_FN(ks, rv64i_zvkned_set_encrypt_key, - rv64i_zvkned_encrypt, - rv64i_zvkb_zvkned_ctr32_encrypt_blocks); - } else { - GCM_HW_SET_KEY_CTR_FN(ks, AES_set_encrypt_key, - rv64i_zvkned_encrypt, - rv64i_zvkb_zvkned_ctr32_encrypt_blocks); - } - - return 1; -} - -static const PROV_GCM_HW rv64i_zvkb_zvkg_zvkned_gcm = { - rv64i_zvkb_zvkg_zvkned_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - generic_aes_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; - -const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits) { - if (RISCV_HAS_ZVKNED()) { - if (RISCV_HAS_ZVKB() && RISCV_HAS_ZVKG() && riscv_vlen() >= 128) { - return &rv64i_zvkb_zvkg_zvkned_gcm; - } - return &rv64i_zvkned_gcm; - } - - if (RISCV_HAS_ZKND_AND_ZKNE()) { - return &rv64i_zknd_zkne_gcm; - } - - return &aes_gcm; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_vaes_avx512.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_vaes_avx512.inc deleted file mode 100644 index c892c0754..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_hw_vaes_avx512.inc +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. - * Copyright (c) 2021, Intel Corporation. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * AVX512 VAES + VPCLMULDQD support for AES GCM. - * This file is included by cipher_aes_gcm_hw_aesni.inc - */ - -#undef VAES_GCM_ENABLED -#if (defined(__x86_64) || defined(__x86_64__) || \ - defined(_M_AMD64) || defined(_M_X64)) -# define VAES_GCM_ENABLED - -/* Returns non-zero when AVX512F + VAES + VPCLMULDQD combination is available */ -int ossl_vaes_vpclmulqdq_capable(void); - -# define OSSL_AES_GCM_UPDATE(direction) \ - void ossl_aes_gcm_ ## direction ## _avx512(const void *ks, \ - void *gcm128ctx, \ - unsigned int *pblocklen, \ - const unsigned char *in, \ - size_t len, \ - unsigned char *out); - -OSSL_AES_GCM_UPDATE(encrypt) -OSSL_AES_GCM_UPDATE(decrypt) - -void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx); -void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx, - const unsigned char *iv, size_t ivlen); -void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad, - size_t aadlen); -void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen); - -void ossl_gcm_gmult_avx512(u64 Xi[2], const void *gcm128ctx); - -static int vaes_gcm_setkey(PROV_GCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - GCM128_CONTEXT *gcmctx = &ctx->gcm; - PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; - AES_KEY *ks = &actx->ks.ks; - - aesni_set_encrypt_key(key, keylen * 8, ks); - memset(gcmctx, 0, sizeof(*gcmctx)); - gcmctx->key = ks; - ctx->key_set = 1; - - ossl_aes_gcm_init_avx512(ks, gcmctx); - - return 1; -} - -static int vaes_gcm_setiv(PROV_GCM_CTX *ctx, const unsigned char *iv, - size_t ivlen) -{ - GCM128_CONTEXT *gcmctx = &ctx->gcm; - - gcmctx->Yi.u[0] = 0; /* Current counter */ - gcmctx->Yi.u[1] = 0; - gcmctx->Xi.u[0] = 0; /* AAD hash */ - gcmctx->Xi.u[1] = 0; - gcmctx->len.u[0] = 0; /* AAD length */ - gcmctx->len.u[1] = 0; /* Message length */ - gcmctx->ares = 0; - gcmctx->mres = 0; - - /* IV is limited by 2^64 bits, thus 2^61 bytes */ - if (ivlen > (U64(1) << 61)) - return 0; - - ossl_aes_gcm_setiv_avx512(gcmctx->key, gcmctx, iv, ivlen); - - return 1; -} - -static int vaes_gcm_aadupdate(PROV_GCM_CTX *ctx, - const unsigned char *aad, - size_t aad_len) -{ - GCM128_CONTEXT *gcmctx = &ctx->gcm; - u64 alen = gcmctx->len.u[0]; - unsigned int ares; - size_t i, lenBlks; - - /* Bad sequence: call of AAD update after message processing */ - if (gcmctx->len.u[1] > 0) - return 0; - - alen += aad_len; - /* AAD is limited by 2^64 bits, thus 2^61 bytes */ - if ((alen > (U64(1) << 61)) || (alen < aad_len)) - return 0; - - gcmctx->len.u[0] = alen; - - ares = gcmctx->ares; - /* Partial AAD block left from previous AAD update calls */ - if (ares > 0) { - /* - * Fill partial block buffer till full block - * (note, the hash is stored reflected) - */ - while (ares > 0 && aad_len > 0) { - gcmctx->Xi.c[15 - ares] ^= *(aad++); - --aad_len; - ares = (ares + 1) % AES_BLOCK_SIZE; - } - /* Full block gathered */ - if (ares == 0) { - ossl_gcm_gmult_avx512(gcmctx->Xi.u, gcmctx); - } else { /* no more AAD */ - gcmctx->ares = ares; - return 1; - } - } - - /* Bulk AAD processing */ - lenBlks = aad_len & ((size_t)(-AES_BLOCK_SIZE)); - if (lenBlks > 0) { - ossl_aes_gcm_update_aad_avx512(gcmctx, aad, lenBlks); - aad += lenBlks; - aad_len -= lenBlks; - } - - /* Add remaining AAD to the hash (note, the hash is stored reflected) */ - if (aad_len > 0) { - ares = aad_len; - for (i = 0; i < aad_len; i++) - gcmctx->Xi.c[15 - i] ^= aad[i]; - } - - gcmctx->ares = ares; - - return 1; -} - -static int vaes_gcm_cipherupdate(PROV_GCM_CTX *ctx, const unsigned char *in, - size_t len, unsigned char *out) -{ - GCM128_CONTEXT *gcmctx = &ctx->gcm; - u64 mlen = gcmctx->len.u[1]; - - mlen += len; - if (mlen > ((U64(1) << 36) - 32) || (mlen < len)) - return 0; - - gcmctx->len.u[1] = mlen; - - /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */ - if (gcmctx->ares > 0) { - ossl_gcm_gmult_avx512(gcmctx->Xi.u, gcmctx); - gcmctx->ares = 0; - } - - if (ctx->enc) - ossl_aes_gcm_encrypt_avx512(gcmctx->key, gcmctx, &gcmctx->mres, in, len, out); - else - ossl_aes_gcm_decrypt_avx512(gcmctx->key, gcmctx, &gcmctx->mres, in, len, out); - - return 1; -} - -static int vaes_gcm_cipherfinal(PROV_GCM_CTX *ctx, unsigned char *tag) -{ - GCM128_CONTEXT *gcmctx = &ctx->gcm; - unsigned int *res = &gcmctx->mres; - - /* Finalize AAD processing */ - if (gcmctx->ares > 0) - res = &gcmctx->ares; - - ossl_aes_gcm_finalize_avx512(gcmctx, *res); - - if (ctx->enc) { - ctx->taglen = GCM_TAG_MAX_SIZE; - memcpy(tag, gcmctx->Xi.c, - ctx->taglen <= sizeof(gcmctx->Xi.c) ? ctx->taglen : - sizeof(gcmctx->Xi.c)); - *res = 0; - } else { - return !CRYPTO_memcmp(gcmctx->Xi.c, tag, ctx->taglen); - } - - return 1; -} - -static const PROV_GCM_HW vaes_gcm = { - vaes_gcm_setkey, - vaes_gcm_setiv, - vaes_gcm_aadupdate, - vaes_gcm_cipherupdate, - vaes_gcm_cipherfinal, - ossl_gcm_one_shot -}; - -#endif diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c deleted file mode 100644 index 2d4fd8865..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv.c +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for AES SIV mode */ - -/* - * This file uses the low level AES functions (which are deprecated for - * non-internal use) in order to implement provider AES ciphers. - */ -#include "internal/deprecated.h" - -#include -#include "prov/implementations.h" -#include "prov/providercommon.h" -#include "prov/ciphercommon_aead.h" -#include "prov/provider_ctx.h" -#include "cipher_aes_gcm_siv.h" - -static int ossl_aes_gcm_siv_set_ctx_params(void *vctx, const OSSL_PARAM params[]); - -static void *ossl_aes_gcm_siv_newctx(void *provctx, size_t keybits) -{ - PROV_AES_GCM_SIV_CTX *ctx; - - if (!ossl_prov_is_running()) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx != NULL) { - ctx->key_len = keybits / 8; - ctx->hw = ossl_prov_cipher_hw_aes_gcm_siv(keybits); - ctx->libctx = PROV_LIBCTX_OF(provctx); - ctx->provctx = provctx; - } - return ctx; -} - -static void ossl_aes_gcm_siv_freectx(void *vctx) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - - if (ctx == NULL) - return; - - OPENSSL_clear_free(ctx->aad, ctx->aad_len); - ctx->hw->clean_ctx(ctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *ossl_aes_gcm_siv_dupctx(void *vctx) -{ - PROV_AES_GCM_SIV_CTX *in = (PROV_AES_GCM_SIV_CTX *)vctx; - PROV_AES_GCM_SIV_CTX *ret; - - if (!ossl_prov_is_running()) - return NULL; - - if (in->hw == NULL) - return NULL; - - ret = OPENSSL_memdup(in, sizeof(*in)); - if (ret == NULL) - return NULL; - /* NULL-out these things we create later */ - ret->aad = NULL; - ret->ecb_ctx = NULL; - - if (in->aad != NULL) { - if ((ret->aad = OPENSSL_memdup(in->aad, UP16(ret->aad_len))) == NULL) - goto err; - } - - if (!in->hw->dup_ctx(ret, in)) - goto err; - - return ret; - err: - if (ret != NULL) { - OPENSSL_clear_free(ret->aad, ret->aad_len); - OPENSSL_free(ret); - } - return NULL; -} - -static int ossl_aes_gcm_siv_init(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[], int enc) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - - if (!ossl_prov_is_running()) - return 0; - - ctx->enc = enc; - - if (key != NULL) { - if (keylen != ctx->key_len) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); - return 0; - } - memcpy(ctx->key_gen_key, key, ctx->key_len); - } - if (iv != NULL) { - if (ivlen != sizeof(ctx->nonce)) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); - return 0; - } - memcpy(ctx->nonce, iv, sizeof(ctx->nonce)); - } - - if (!ctx->hw->initkey(ctx)) - return 0; - - return ossl_aes_gcm_siv_set_ctx_params(ctx, params); -} - -static int ossl_aes_gcm_siv_einit(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) -{ - return ossl_aes_gcm_siv_init(vctx, key, keylen, iv, ivlen, params, 1); -} - -static int ossl_aes_gcm_siv_dinit(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) -{ - return ossl_aes_gcm_siv_init(vctx, key, keylen, iv, ivlen, params, 0); -} - -#define ossl_aes_gcm_siv_stream_update ossl_aes_gcm_siv_cipher -static int ossl_aes_gcm_siv_cipher(void *vctx, unsigned char *out, size_t *outl, - size_t outsize, const unsigned char *in, size_t inl) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - int error = 0; - - if (!ossl_prov_is_running()) - return 0; - - /* The RFC has a test case for this, but we don't try to do anything */ - if (inl == 0) { - if (outl != NULL) - *outl = 0; - return 1; - } - - if (outsize < inl) { - ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); - return 0; - } - - error |= !ctx->hw->cipher(ctx, out, in, inl); - - if (outl != NULL && !error) - *outl = inl; - return !error; -} - -static int ossl_aes_gcm_siv_stream_final(void *vctx, unsigned char *out, size_t *outl, - size_t outsize) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - int error = 0; - - if (!ossl_prov_is_running()) - return 0; - - error |= !ctx->hw->cipher(vctx, out, NULL, 0); - - if (outl != NULL && !error) - *outl = 0; - return !error; -} - -static int ossl_aes_gcm_siv_get_ctx_params(void *vctx, OSSL_PARAM params[]) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - OSSL_PARAM *p; - - p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_AEAD_TAG); - if (p != NULL && p->data_type == OSSL_PARAM_OCTET_STRING) { - if (!ctx->enc || !ctx->generated_tag - || p->data_size != sizeof(ctx->tag) - || !OSSL_PARAM_set_octet_string(p, ctx->tag, sizeof(ctx->tag))) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - } - p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_AEAD_TAGLEN); - if (p != NULL && !OSSL_PARAM_set_size_t(p, sizeof(ctx->tag))) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_KEYLEN); - if (p != NULL && !OSSL_PARAM_set_size_t(p, ctx->key_len)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - return 1; -} - -static const OSSL_PARAM aes_gcm_siv_known_gettable_ctx_params[] = { - OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_KEYLEN, NULL), - OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_AEAD_TAGLEN, NULL), - OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_AEAD_TAG, NULL, 0), - OSSL_PARAM_END -}; - -static const OSSL_PARAM *ossl_aes_gcm_siv_gettable_ctx_params(ossl_unused void *cctx, - ossl_unused void *provctx) -{ - return aes_gcm_siv_known_gettable_ctx_params; -} - -static int ossl_aes_gcm_siv_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - const OSSL_PARAM *p; - unsigned int speed = 0; - - if (params == NULL) - return 1; - - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TAG); - if (p != NULL) { - if (p->data_type != OSSL_PARAM_OCTET_STRING - || p->data_size != sizeof(ctx->user_tag)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - if (!ctx->enc) { - memcpy(ctx->user_tag, p->data, sizeof(ctx->tag)); - ctx->have_user_tag = 1; - } - } - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_SPEED); - if (p != NULL) { - if (!OSSL_PARAM_get_uint(p, &speed)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - ctx->speed = !!speed; - } - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_KEYLEN); - if (p != NULL) { - size_t key_len; - - if (!OSSL_PARAM_get_size_t(p, &key_len)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - /* The key length can not be modified */ - if (key_len != ctx->key_len) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); - return 0; - } - } - return 1; -} - -static const OSSL_PARAM aes_gcm_siv_known_settable_ctx_params[] = { - OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_KEYLEN, NULL), - OSSL_PARAM_uint(OSSL_CIPHER_PARAM_SPEED, NULL), - OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_AEAD_TAG, NULL, 0), - OSSL_PARAM_END -}; -static const OSSL_PARAM *ossl_aes_gcm_siv_settable_ctx_params(ossl_unused void *cctx, - ossl_unused void *provctx) -{ - return aes_gcm_siv_known_settable_ctx_params; -} - -#define IMPLEMENT_cipher(alg, lc, UCMODE, flags, kbits, blkbits, ivbits) \ -static OSSL_FUNC_cipher_newctx_fn ossl_##alg##kbits##_##lc##_newctx; \ -static OSSL_FUNC_cipher_freectx_fn ossl_##alg##_##lc##_freectx; \ -static OSSL_FUNC_cipher_dupctx_fn ossl_##alg##_##lc##_dupctx; \ -static OSSL_FUNC_cipher_encrypt_init_fn ossl_##alg##_##lc##_einit; \ -static OSSL_FUNC_cipher_decrypt_init_fn ossl_##alg##_##lc##_dinit; \ -static OSSL_FUNC_cipher_update_fn ossl_##alg##_##lc##_stream_update; \ -static OSSL_FUNC_cipher_final_fn ossl_##alg##_##lc##_stream_final; \ -static OSSL_FUNC_cipher_cipher_fn ossl_##alg##_##lc##_cipher; \ -static OSSL_FUNC_cipher_get_params_fn ossl_##alg##_##kbits##_##lc##_get_params; \ -static OSSL_FUNC_cipher_get_ctx_params_fn ossl_##alg##_##lc##_get_ctx_params; \ -static OSSL_FUNC_cipher_gettable_ctx_params_fn ossl_##alg##_##lc##_gettable_ctx_params; \ -static OSSL_FUNC_cipher_set_ctx_params_fn ossl_##alg##_##lc##_set_ctx_params; \ -static OSSL_FUNC_cipher_settable_ctx_params_fn ossl_##alg##_##lc##_settable_ctx_params; \ -static int ossl_##alg##_##kbits##_##lc##_get_params(OSSL_PARAM params[]) \ -{ \ - return ossl_cipher_generic_get_params(params, EVP_CIPH_##UCMODE##_MODE, \ - flags, kbits, blkbits, ivbits); \ -} \ -static void *ossl_##alg##kbits##_##lc##_newctx(void *provctx) \ -{ \ - return ossl_##alg##_##lc##_newctx(provctx, kbits); \ -} \ -const OSSL_DISPATCH ossl_##alg##kbits##lc##_functions[] = { \ - { OSSL_FUNC_CIPHER_NEWCTX, (void (*)(void))ossl_##alg##kbits##_##lc##_newctx }, \ - { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))ossl_##alg##_##lc##_freectx }, \ - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void))ossl_##alg##_##lc##_dupctx }, \ - { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))ossl_##alg##_##lc##_einit }, \ - { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))ossl_##alg##_##lc##_dinit }, \ - { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))ossl_##alg##_##lc##_stream_update }, \ - { OSSL_FUNC_CIPHER_FINAL, (void (*)(void))ossl_##alg##_##lc##_stream_final }, \ - { OSSL_FUNC_CIPHER_CIPHER, (void (*)(void))ossl_##alg##_##lc##_cipher }, \ - { OSSL_FUNC_CIPHER_GET_PARAMS, (void (*)(void))ossl_##alg##_##kbits##_##lc##_get_params }, \ - { OSSL_FUNC_CIPHER_GETTABLE_PARAMS, (void (*)(void))ossl_cipher_generic_gettable_params }, \ - { OSSL_FUNC_CIPHER_GET_CTX_PARAMS, (void (*)(void))ossl_##alg##_##lc##_get_ctx_params }, \ - { OSSL_FUNC_CIPHER_GETTABLE_CTX_PARAMS, (void (*)(void))ossl_##alg##_##lc##_gettable_ctx_params }, \ - { OSSL_FUNC_CIPHER_SET_CTX_PARAMS, (void (*)(void))ossl_##alg##_##lc##_set_ctx_params }, \ - { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, (void (*)(void))ossl_##alg##_##lc##_settable_ctx_params }, \ - OSSL_DISPATCH_END \ -} - -IMPLEMENT_cipher(aes, gcm_siv, GCM_SIV, AEAD_FLAGS, 128, 8, 96); -IMPLEMENT_cipher(aes, gcm_siv, GCM_SIV, AEAD_FLAGS, 192, 8, 96); -IMPLEMENT_cipher(aes, gcm_siv, GCM_SIV, AEAD_FLAGS, 256, 8, 96); diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv.h b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv.h deleted file mode 100644 index 37d1e3326..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "prov/ciphercommon.h" -#include "crypto/aes_platform.h" - -#define BLOCK_SIZE 16 -#define NONCE_SIZE 12 -#define TAG_SIZE 16 - -/* AAD manipulation macros */ -#define UP16(x) (((x) + 15) & ~0x0F) -#define DOWN16(x) ((x) & ~0x0F) -#define REMAINDER16(x) ((x) & 0x0F) -#define IS16(x) (((x) & 0x0F) == 0) - -typedef struct prov_cipher_hw_aes_gcm_siv_st { - int (*initkey)(void *vctx); - int (*cipher)(void *vctx, unsigned char *out, const unsigned char *in, - size_t len); - int (*dup_ctx)(void *vdst, void *vsrc); - void (*clean_ctx)(void *vctx); -} PROV_CIPHER_HW_AES_GCM_SIV; - -/* Arranged for alignment purposes */ -typedef struct prov_aes_gcm_siv_ctx_st { - EVP_CIPHER_CTX *ecb_ctx; - const PROV_CIPHER_HW_AES_GCM_SIV *hw; /* maybe not used, yet? */ - uint8_t *aad; /* Allocated, rounded up to 16 bytes, from user */ - OSSL_LIB_CTX *libctx; - OSSL_PROVIDER *provctx; - size_t aad_len; /* actual AAD length */ - size_t key_len; - uint8_t key_gen_key[32]; /* from user */ - uint8_t msg_enc_key[32]; /* depends on key size */ - uint8_t msg_auth_key[BLOCK_SIZE]; - uint8_t tag[TAG_SIZE]; /* generated tag, given to user or compared to user */ - uint8_t user_tag[TAG_SIZE]; /* from user */ - uint8_t nonce[NONCE_SIZE]; /* from user */ - u128 Htable[16]; /* Polyval calculations via ghash */ - unsigned int enc : 1; /* Set to 1 if we are encrypting or 0 otherwise */ - unsigned int have_user_tag : 1; - unsigned int generated_tag : 1; - unsigned int used_enc : 1; - unsigned int used_dec : 1; - unsigned int speed : 1; -} PROV_AES_GCM_SIV_CTX; - -const PROV_CIPHER_HW_AES_GCM_SIV *ossl_prov_cipher_hw_aes_gcm_siv(size_t keybits); - -void ossl_polyval_ghash_init(u128 Htable[16], const uint64_t H[2]); -void ossl_polyval_ghash_hash(const u128 Htable[16], uint8_t *tag, const uint8_t *inp, size_t len); - -/* Define GSWAP8/GSWAP4 - used for BOTH little and big endian architectures */ -static ossl_inline uint32_t GSWAP4(uint32_t n) -{ - return (((n & 0x000000FF) << 24) - | ((n & 0x0000FF00) << 8) - | ((n & 0x00FF0000) >> 8) - | ((n & 0xFF000000) >> 24)); -} -static ossl_inline uint64_t GSWAP8(uint64_t n) -{ - uint64_t result; - - result = GSWAP4(n & 0x0FFFFFFFF); - result <<= 32; - return result | GSWAP4(n >> 32); -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c deleted file mode 100644 index 9887e1c3a..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv_hw.c +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * AES low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include "cipher_aes_gcm_siv.h" - -static int aes_gcm_siv_ctr32(PROV_AES_GCM_SIV_CTX *ctx, const unsigned char *init_counter, - unsigned char *out, const unsigned char *in, size_t len); - -static int aes_gcm_siv_initkey(void *vctx) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - uint8_t output[BLOCK_SIZE]; - uint32_t counter = 0x0; - size_t i; - union { - uint32_t counter; - uint8_t block[BLOCK_SIZE]; - } data; - int out_len; - EVP_CIPHER *ecb = NULL; - DECLARE_IS_ENDIAN; - - switch (ctx->key_len) { - case 16: - ecb = EVP_CIPHER_fetch(ctx->libctx, "AES-128-ECB", NULL); - break; - case 24: - ecb = EVP_CIPHER_fetch(ctx->libctx, "AES-192-ECB", NULL); - break; - case 32: - ecb = EVP_CIPHER_fetch(ctx->libctx, "AES-256-ECB", NULL); - break; - default: - goto err; - } - - if (ctx->ecb_ctx == NULL && (ctx->ecb_ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - if (!EVP_EncryptInit_ex2(ctx->ecb_ctx, ecb, ctx->key_gen_key, NULL, NULL)) - goto err; - - memset(&data, 0, sizeof(data)); - memcpy(&data.block[sizeof(data.counter)], ctx->nonce, NONCE_SIZE); - - /* msg_auth_key is always 16 bytes in size, regardless of AES128/AES256 */ - /* counter is stored little-endian */ - for (i = 0; i < BLOCK_SIZE; i += 8) { - if (IS_LITTLE_ENDIAN) { - data.counter = counter; - } else { - data.counter = GSWAP4(counter); - } - /* Block size is 16 (128 bits), but only 8 bytes are used */ - out_len = BLOCK_SIZE; - if (!EVP_EncryptUpdate(ctx->ecb_ctx, output, &out_len, data.block, BLOCK_SIZE)) - goto err; - memcpy(&ctx->msg_auth_key[i], output, 8); - counter++; - } - - /* msg_enc_key length is directly tied to key length AES128/AES256 */ - for (i = 0; i < ctx->key_len; i += 8) { - if (IS_LITTLE_ENDIAN) { - data.counter = counter; - } else { - data.counter = GSWAP4(counter); - } - /* Block size is 16 bytes (128 bits), but only 8 bytes are used */ - out_len = BLOCK_SIZE; - if (!EVP_EncryptUpdate(ctx->ecb_ctx, output, &out_len, data.block, BLOCK_SIZE)) - goto err; - memcpy(&ctx->msg_enc_key[i], output, 8); - counter++; - } - - if (!EVP_EncryptInit_ex2(ctx->ecb_ctx, ecb, ctx->msg_enc_key, NULL, NULL)) - goto err; - - /* Freshen up the state */ - ctx->used_enc = 0; - ctx->used_dec = 0; - EVP_CIPHER_free(ecb); - return 1; - err: - EVP_CIPHER_CTX_free(ctx->ecb_ctx); - EVP_CIPHER_free(ecb); - ctx->ecb_ctx = NULL; - return 0; -} - -static int aes_gcm_siv_aad(PROV_AES_GCM_SIV_CTX *ctx, - const unsigned char *aad, size_t len) -{ - size_t to_alloc; - uint8_t *ptr; - uint64_t len64; - - /* length of 0 resets the AAD */ - if (len == 0) { - OPENSSL_free(ctx->aad); - ctx->aad = NULL; - ctx->aad_len = 0; - return 1; - } - to_alloc = UP16(ctx->aad_len + len); - /* need to check the size of the AAD per RFC8452 */ - len64 = to_alloc; - if (len64 > ((uint64_t)1 << 36)) - return 0; - ptr = OPENSSL_realloc(ctx->aad, to_alloc); - if (ptr == NULL) - return 0; - ctx->aad = ptr; - memcpy(&ctx->aad[ctx->aad_len], aad, len); - ctx->aad_len += len; - if (to_alloc > ctx->aad_len) - memset(&ctx->aad[ctx->aad_len], 0, to_alloc - ctx->aad_len); - return 1; -} - -static int aes_gcm_siv_finish(PROV_AES_GCM_SIV_CTX *ctx) -{ - int ret = 0; - - if (ctx->enc) - return ctx->generated_tag; - ret = !CRYPTO_memcmp(ctx->tag, ctx->user_tag, sizeof(ctx->tag)); - ret &= ctx->have_user_tag; - return ret; -} - -static int aes_gcm_siv_encrypt(PROV_AES_GCM_SIV_CTX *ctx, const unsigned char *in, - unsigned char *out, size_t len) -{ - uint64_t len_blk[2]; - uint8_t S_s[TAG_SIZE]; - uint8_t counter_block[TAG_SIZE]; - uint8_t padding[BLOCK_SIZE]; - size_t i; - int64_t len64 = len; - int out_len; - int error = 0; - DECLARE_IS_ENDIAN; - - ctx->generated_tag = 0; - if (!ctx->speed && ctx->used_enc) - return 0; - /* need to check the size of the input! */ - if (len64 > ((int64_t)1 << 36) || len == 0) - return 0; - - if (IS_LITTLE_ENDIAN) { - len_blk[0] = (uint64_t)ctx->aad_len * 8; - len_blk[1] = (uint64_t)len * 8; - } else { - len_blk[0] = GSWAP8((uint64_t)ctx->aad_len * 8); - len_blk[1] = GSWAP8((uint64_t)len * 8); - } - memset(S_s, 0, TAG_SIZE); - ossl_polyval_ghash_init(ctx->Htable, (const uint64_t*)ctx->msg_auth_key); - - if (ctx->aad != NULL) { - /* AAD is allocated with padding, but need to adjust length */ - ossl_polyval_ghash_hash(ctx->Htable, S_s, ctx->aad, UP16(ctx->aad_len)); - } - if (DOWN16(len) > 0) - ossl_polyval_ghash_hash(ctx->Htable, S_s, (uint8_t *) in, DOWN16(len)); - if (!IS16(len)) { - /* deal with padding - probably easier to memset the padding first rather than calculate */ - memset(padding, 0, sizeof(padding)); - memcpy(padding, &in[DOWN16(len)], REMAINDER16(len)); - ossl_polyval_ghash_hash(ctx->Htable, S_s, padding, sizeof(padding)); - } - ossl_polyval_ghash_hash(ctx->Htable, S_s, (uint8_t *) len_blk, sizeof(len_blk)); - - for (i = 0; i < NONCE_SIZE; i++) - S_s[i] ^= ctx->nonce[i]; - - S_s[TAG_SIZE - 1] &= 0x7f; - out_len = sizeof(ctx->tag); - error |= !EVP_EncryptUpdate(ctx->ecb_ctx, ctx->tag, &out_len, S_s, sizeof(S_s)); - memcpy(counter_block, ctx->tag, TAG_SIZE); - counter_block[TAG_SIZE - 1] |= 0x80; - - error |= !aes_gcm_siv_ctr32(ctx, counter_block, out, in, len); - - ctx->generated_tag = !error; - /* Regardless of error */ - ctx->used_enc = 1; - return !error; -} - -static int aes_gcm_siv_decrypt(PROV_AES_GCM_SIV_CTX *ctx, const unsigned char *in, - unsigned char *out, size_t len) -{ - uint8_t counter_block[TAG_SIZE]; - uint64_t len_blk[2]; - uint8_t S_s[TAG_SIZE]; - size_t i; - uint64_t padding[2]; - int64_t len64 = len; - int out_len; - int error = 0; - DECLARE_IS_ENDIAN; - - ctx->generated_tag = 0; - if (!ctx->speed && ctx->used_dec) - return 0; - /* need to check the size of the input! */ - if (len64 > ((int64_t)1 << 36) || len == 0) - return 0; - - memcpy(counter_block, ctx->user_tag, sizeof(counter_block)); - counter_block[TAG_SIZE - 1] |= 0x80; - - error |= !aes_gcm_siv_ctr32(ctx, counter_block, out, in, len); - - if (IS_LITTLE_ENDIAN) { - len_blk[0] = (uint64_t)ctx->aad_len * 8; - len_blk[1] = (uint64_t)len * 8; - } else { - len_blk[0] = GSWAP8((uint64_t)ctx->aad_len * 8); - len_blk[1] = GSWAP8((uint64_t)len * 8); - } - memset(S_s, 0, TAG_SIZE); - ossl_polyval_ghash_init(ctx->Htable, (const uint64_t*)ctx->msg_auth_key); - if (ctx->aad != NULL) { - /* AAD allocated with padding, but need to adjust length */ - ossl_polyval_ghash_hash(ctx->Htable, S_s, ctx->aad, UP16(ctx->aad_len)); - } - if (DOWN16(len) > 0) - ossl_polyval_ghash_hash(ctx->Htable, S_s, out, DOWN16(len)); - if (!IS16(len)) { - /* deal with padding - probably easier to "memset" the padding first rather than calculate */ - padding[0] = padding[1] = 0; - memcpy(padding, &out[DOWN16(len)], REMAINDER16(len)); - ossl_polyval_ghash_hash(ctx->Htable, S_s, (uint8_t *)padding, sizeof(padding)); - } - ossl_polyval_ghash_hash(ctx->Htable, S_s, (uint8_t *)len_blk, TAG_SIZE); - - for (i = 0; i < NONCE_SIZE; i++) - S_s[i] ^= ctx->nonce[i]; - - S_s[TAG_SIZE - 1] &= 0x7f; - - /* - * In the ctx, user_tag is the one received/set by the user, - * and tag is generated from the input - */ - out_len = sizeof(ctx->tag); - error |= !EVP_EncryptUpdate(ctx->ecb_ctx, ctx->tag, &out_len, S_s, sizeof(S_s)); - ctx->generated_tag = !error; - /* Regardless of error */ - ctx->used_dec = 1; - return !error; -} - -static int aes_gcm_siv_cipher(void *vctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - - /* EncryptFinal or DecryptFinal */ - if (in == NULL) - return aes_gcm_siv_finish(ctx); - - /* Deal with associated data */ - if (out == NULL) - return aes_gcm_siv_aad(ctx, in, len); - - if (ctx->enc) - return aes_gcm_siv_encrypt(ctx, in, out, len); - - return aes_gcm_siv_decrypt(ctx, in, out, len); -} - -static void aes_gcm_siv_clean_ctx(void *vctx) -{ - PROV_AES_GCM_SIV_CTX *ctx = (PROV_AES_GCM_SIV_CTX *)vctx; - - EVP_CIPHER_CTX_free(ctx->ecb_ctx); - ctx->ecb_ctx = NULL; -} - -static int aes_gcm_siv_dup_ctx(void *vdst, void *vsrc) -{ - PROV_AES_GCM_SIV_CTX *dst = (PROV_AES_GCM_SIV_CTX *)vdst; - PROV_AES_GCM_SIV_CTX *src = (PROV_AES_GCM_SIV_CTX *)vsrc; - - dst->ecb_ctx = NULL; - if (src->ecb_ctx != NULL) { - if ((dst->ecb_ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - if (!EVP_CIPHER_CTX_copy(dst->ecb_ctx, src->ecb_ctx)) - goto err; - } - return 1; - - err: - EVP_CIPHER_CTX_free(dst->ecb_ctx); - dst->ecb_ctx = NULL; - return 0; -} - -static const PROV_CIPHER_HW_AES_GCM_SIV aes_gcm_siv_hw = -{ - aes_gcm_siv_initkey, - aes_gcm_siv_cipher, - aes_gcm_siv_dup_ctx, - aes_gcm_siv_clean_ctx, -}; - -const PROV_CIPHER_HW_AES_GCM_SIV *ossl_prov_cipher_hw_aes_gcm_siv(size_t keybits) -{ - return &aes_gcm_siv_hw; -} - -/* AES-GCM-SIV needs AES-CTR32, which is different than the AES-CTR implementation */ -static int aes_gcm_siv_ctr32(PROV_AES_GCM_SIV_CTX *ctx, const unsigned char *init_counter, - unsigned char *out, const unsigned char *in, size_t len) -{ - uint8_t keystream[BLOCK_SIZE]; - int out_len; - size_t i; - size_t j; - size_t todo; - uint32_t counter; - int error = 0; - union { - uint32_t x32[BLOCK_SIZE / sizeof(uint32_t)]; - uint8_t x8[BLOCK_SIZE]; - } block; - DECLARE_IS_ENDIAN; - - memcpy(&block, init_counter, sizeof(block)); - if (IS_BIG_ENDIAN) { - counter = GSWAP4(block.x32[0]); - } - - for (i = 0; i < len; i += sizeof(block)) { - out_len = BLOCK_SIZE; - error |= !EVP_EncryptUpdate(ctx->ecb_ctx, keystream, &out_len, (uint8_t*)&block, sizeof(block)); - if (IS_LITTLE_ENDIAN) { - block.x32[0]++; - } else { - counter++; - block.x32[0] = GSWAP4(counter); - } - todo = len - i; - if (todo > sizeof(keystream)) - todo = sizeof(keystream); - /* Non optimal, but avoids alignment issues */ - for (j = 0; j < todo; j++) - out[i + j] = in[i + j] ^ keystream[j]; - } - return !error; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c b/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c deleted file mode 100644 index fead51dd3..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_gcm_siv_polyval.c +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * AES low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include "cipher_aes_gcm_siv.h" - -static ossl_inline void mulx_ghash(uint64_t *a) -{ - uint64_t t[2], mask; - DECLARE_IS_ENDIAN; - - if (IS_LITTLE_ENDIAN) { - t[0] = GSWAP8(a[0]); - t[1] = GSWAP8(a[1]); - } else { - t[0] = a[0]; - t[1] = a[1]; - } - mask = -(int64_t)(t[1] & 1) & 0xe1; - mask <<= 56; - - if (IS_LITTLE_ENDIAN) { - a[1] = GSWAP8((t[1] >> 1) ^ (t[0] << 63)); - a[0] = GSWAP8((t[0] >> 1) ^ mask); - } else { - a[1] = (t[1] >> 1) ^ (t[0] << 63); - a[0] = (t[0] >> 1) ^ mask; - } -} - -#define aligned64(p) (((uintptr_t)p & 0x07) == 0) -static ossl_inline void byte_reverse16(uint8_t *out, const uint8_t *in) -{ - if (aligned64(out) && aligned64(in)) { - ((uint64_t *)out)[0] = GSWAP8(((uint64_t *)in)[1]); - ((uint64_t *)out)[1] = GSWAP8(((uint64_t *)in)[0]); - } else { - int i; - - for (i = 0; i < 16; i++) - out[i] = in[15 - i]; - } -} - -/* Initialization of POLYVAL via existing GHASH implementation */ -void ossl_polyval_ghash_init(u128 Htable[16], const uint64_t H[2]) -{ - uint64_t tmp[2]; - DECLARE_IS_ENDIAN; - - byte_reverse16((uint8_t *)tmp, (const uint8_t *)H); - mulx_ghash(tmp); - if (IS_LITTLE_ENDIAN) { - /* "H is stored in host byte order" */ - tmp[0] = GSWAP8(tmp[0]); - tmp[1] = GSWAP8(tmp[1]); - } - - ossl_gcm_init_4bit(Htable, (u64*)tmp); -} - -/* Implementation of POLYVAL via existing GHASH implementation */ -void ossl_polyval_ghash_hash(const u128 Htable[16], uint8_t *tag, const uint8_t *inp, size_t len) -{ - uint64_t out[2]; - uint64_t tmp[2]; - size_t i; - - byte_reverse16((uint8_t *)out, (uint8_t *)tag); - - /* - * This implementation doesn't deal with partials, callers do, - * so, len is a multiple of 16 - */ - for (i = 0; i < len; i += 16) { - byte_reverse16((uint8_t *)tmp, &inp[i]); - ossl_gcm_ghash_4bit((u64*)out, Htable, (uint8_t *)tmp, 16); - } - byte_reverse16(tag, (uint8_t *)out); -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aes_hw.c index a3b72d9f7..596cdba8d 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2001-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -142,12 +142,6 @@ const PROV_CIPHER_HW *ossl_prov_cipher_hw_aes_##mode(size_t keybits) \ # include "cipher_aes_hw_t4.inc" #elif defined(S390X_aes_128_CAPABLE) # include "cipher_aes_hw_s390x.inc" -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 -# include "cipher_aes_hw_rv64i.inc" -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 -# include "cipher_aes_hw_rv32i.inc" -#elif defined (ARMv8_HWAES_CAPABLE) -# include "cipher_aes_hw_armv8.inc" #else /* The generic case */ # define PROV_CIPHER_HW_declare(mode) diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_hw_armv8.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_hw_armv8.inc deleted file mode 100644 index 3f73c7929..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_hw_armv8.inc +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Crypto extension support for AES modes ecb, cbc, ofb, cfb, ctr. - * This file is included by cipher_aes_hw.c - */ - -static int cipher_hw_aes_arm_initkey(PROV_CIPHER_CTX *dat, - const unsigned char *key, - size_t keylen) -{ - int ret = cipher_hw_aes_initkey(dat, key, keylen); - if (AES_UNROLL12_EOR3_CAPABLE && dat->mode == EVP_CIPH_CTR_MODE) - dat->stream.ctr = (ctr128_f)HWAES_ctr32_encrypt_blocks_unroll12_eor3; - - return ret; -} - -#define PROV_CIPHER_HW_declare(mode) \ -static const PROV_CIPHER_HW aes_arm_##mode = { \ - cipher_hw_aes_arm_initkey, \ - ossl_cipher_hw_generic_##mode, \ - cipher_hw_aes_copyctx \ -}; -#define PROV_CIPHER_HW_select(mode) \ -if (ARMv8_HWAES_CAPABLE) \ - return &aes_arm_##mode; diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_hw_rv32i.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_hw_rv32i.inc deleted file mode 100644 index f6c652c32..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_hw_rv32i.inc +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RISC-V 32 ZKND ZKNE support for AES modes ecb, cbc, ofb, cfb, ctr. - * This file is included by cipher_aes_hw.c - */ - -#define cipher_hw_rv32i_zknd_zkne_cbc ossl_cipher_hw_generic_cbc -#define cipher_hw_rv32i_zknd_zkne_ecb ossl_cipher_hw_generic_ecb -#define cipher_hw_rv32i_zknd_zkne_ofb128 ossl_cipher_hw_generic_ofb128 -#define cipher_hw_rv32i_zknd_zkne_cfb128 ossl_cipher_hw_generic_cfb128 -#define cipher_hw_rv32i_zknd_zkne_cfb8 ossl_cipher_hw_generic_cfb8 -#define cipher_hw_rv32i_zknd_zkne_cfb1 ossl_cipher_hw_generic_cfb1 -#define cipher_hw_rv32i_zknd_zkne_ctr ossl_cipher_hw_generic_ctr - -#define cipher_hw_rv32i_zbkb_zknd_zkne_cbc ossl_cipher_hw_generic_cbc -#define cipher_hw_rv32i_zbkb_zknd_zkne_ecb ossl_cipher_hw_generic_ecb -#define cipher_hw_rv32i_zbkb_zknd_zkne_ofb128 ossl_cipher_hw_generic_ofb128 -#define cipher_hw_rv32i_zbkb_zknd_zkne_cfb128 ossl_cipher_hw_generic_cfb128 -#define cipher_hw_rv32i_zbkb_zknd_zkne_cfb8 ossl_cipher_hw_generic_cfb8 -#define cipher_hw_rv32i_zbkb_zknd_zkne_cfb1 ossl_cipher_hw_generic_cfb1 -#define cipher_hw_rv32i_zbkb_zknd_zkne_ctr ossl_cipher_hw_generic_ctr - -static int cipher_hw_rv32i_zknd_zkne_initkey(PROV_CIPHER_CTX *dat, - const unsigned char *key, size_t keylen) -{ - int ret; - PROV_AES_CTX *adat = (PROV_AES_CTX *)dat; - AES_KEY *ks = &adat->ks.ks; - - dat->ks = ks; - - if ((dat->mode == EVP_CIPH_ECB_MODE || dat->mode == EVP_CIPH_CBC_MODE) - && !dat->enc) { - ret = rv32i_zknd_zkne_set_decrypt_key(key, keylen * 8, ks); - dat->block = (block128_f) rv32i_zknd_decrypt; - dat->stream.cbc = NULL; - } else { - ret = rv32i_zkne_set_encrypt_key(key, keylen * 8, ks); - dat->block = (block128_f) rv32i_zkne_encrypt; - dat->stream.cbc = NULL; - } - - if (ret < 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_KEY_SETUP_FAILED); - return 0; - } - - return 1; -} - -static int cipher_hw_rv32i_zbkb_zknd_zkne_initkey(PROV_CIPHER_CTX *dat, - const unsigned char *key, size_t keylen) -{ - int ret; - PROV_AES_CTX *adat = (PROV_AES_CTX *)dat; - AES_KEY *ks = &adat->ks.ks; - - dat->ks = ks; - - if ((dat->mode == EVP_CIPH_ECB_MODE || dat->mode == EVP_CIPH_CBC_MODE) - && !dat->enc) { - ret = rv32i_zbkb_zknd_zkne_set_decrypt_key(key, keylen * 8, ks); - dat->block = (block128_f) rv32i_zknd_decrypt; - dat->stream.cbc = NULL; - } else { - ret = rv32i_zbkb_zkne_set_encrypt_key(key, keylen * 8, ks); - dat->block = (block128_f) rv32i_zkne_encrypt; - dat->stream.cbc = NULL; - } - - if (ret < 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_KEY_SETUP_FAILED); - return 0; - } - - return 1; -} - -#define PROV_CIPHER_HW_declare(mode) \ -static const PROV_CIPHER_HW rv32i_zknd_zkne_##mode = { \ - cipher_hw_rv32i_zknd_zkne_initkey, \ - cipher_hw_rv32i_zknd_zkne_##mode, \ - cipher_hw_aes_copyctx \ -}; \ -static const PROV_CIPHER_HW rv32i_zbkb_zknd_zkne_##mode = { \ - cipher_hw_rv32i_zbkb_zknd_zkne_initkey, \ - cipher_hw_rv32i_zbkb_zknd_zkne_##mode, \ - cipher_hw_aes_copyctx \ -}; -#define PROV_CIPHER_HW_select(mode) \ -if (RISCV_HAS_ZBKB_AND_ZKND_AND_ZKNE()) \ - return &rv32i_zbkb_zknd_zkne_##mode; \ -if (RISCV_HAS_ZKND_AND_ZKNE()) \ - return &rv32i_zknd_zkne_##mode; diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_hw_rv64i.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_hw_rv64i.inc deleted file mode 100644 index 07d479303..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_hw_rv64i.inc +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RISC-V 64 ZKND ZKNE support for AES modes ecb, cbc, ofb, cfb, ctr. - * This file is included by cipher_aes_hw.c - */ - -#define cipher_hw_rv64i_zknd_zkne_cbc ossl_cipher_hw_generic_cbc -#define cipher_hw_rv64i_zknd_zkne_ecb ossl_cipher_hw_generic_ecb -#define cipher_hw_rv64i_zknd_zkne_ofb128 ossl_cipher_hw_generic_ofb128 -#define cipher_hw_rv64i_zknd_zkne_cfb128 ossl_cipher_hw_generic_cfb128 -#define cipher_hw_rv64i_zknd_zkne_cfb8 ossl_cipher_hw_generic_cfb8 -#define cipher_hw_rv64i_zknd_zkne_cfb1 ossl_cipher_hw_generic_cfb1 -#define cipher_hw_rv64i_zknd_zkne_ctr ossl_cipher_hw_generic_ctr - -static int cipher_hw_rv64i_zknd_zkne_initkey(PROV_CIPHER_CTX *dat, - const unsigned char *key, size_t keylen) -{ - int ret; - PROV_AES_CTX *adat = (PROV_AES_CTX *)dat; - AES_KEY *ks = &adat->ks.ks; - - dat->ks = ks; - - if ((dat->mode == EVP_CIPH_ECB_MODE || dat->mode == EVP_CIPH_CBC_MODE) - && !dat->enc) { - ret = rv64i_zknd_set_decrypt_key(key, keylen * 8, ks); - dat->block = (block128_f) rv64i_zknd_decrypt; - dat->stream.cbc = NULL; - } else { - ret = rv64i_zkne_set_encrypt_key(key, keylen * 8, ks); - dat->block = (block128_f) rv64i_zkne_encrypt; - dat->stream.cbc = NULL; - } - - if (ret < 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_KEY_SETUP_FAILED); - return 0; - } - - return 1; -} - -/*- - * RISC-V RV64 ZVKNED support for AES modes ecb, cbc, ofb, cfb, ctr. - * This file is included by cipher_aes_hw.c - */ - -#define cipher_hw_rv64i_zvkned_cbc ossl_cipher_hw_generic_cbc -#define cipher_hw_rv64i_zvkned_ecb ossl_cipher_hw_generic_ecb -#define cipher_hw_rv64i_zvkned_ofb128 ossl_cipher_hw_generic_ofb128 -#define cipher_hw_rv64i_zvkned_cfb128 ossl_cipher_hw_generic_cfb128 -#define cipher_hw_rv64i_zvkned_cfb8 ossl_cipher_hw_generic_cfb8 -#define cipher_hw_rv64i_zvkned_cfb1 ossl_cipher_hw_generic_cfb1 -#define cipher_hw_rv64i_zvkned_ctr ossl_cipher_hw_generic_ctr - -static int cipher_hw_rv64i_zvkned_initkey(PROV_CIPHER_CTX *dat, - const unsigned char *key, - size_t keylen) -{ - int ret; - PROV_AES_CTX *adat = (PROV_AES_CTX *)dat; - AES_KEY *ks = &adat->ks.ks; - - dat->ks = ks; - - /* - * Zvkned only supports 128 and 256 bit keys for key schedule generation. - * For AES-192 case, we could fallback to `AES_set_encrypt_key`. - * All Zvkned-based implementations use the same `encrypt-key` scheduling - * for both encryption and decryption. - */ - if (keylen * 8 == 128 || keylen * 8 == 256) { - ret = rv64i_zvkned_set_encrypt_key(key, keylen * 8, ks); - } else { - ret = AES_set_encrypt_key(key, keylen * 8, ks); - } - - if (dat->mode == EVP_CIPH_CBC_MODE) { - if (dat->enc) { - dat->stream.cbc = (cbc128_f) rv64i_zvkned_cbc_encrypt; - } else { - dat->stream.cbc = (cbc128_f) rv64i_zvkned_cbc_decrypt; - } - } else if (dat->mode == EVP_CIPH_CTR_MODE) { - if (RISCV_HAS_ZVKB()) { - dat->stream.ctr = (ctr128_f) rv64i_zvkb_zvkned_ctr32_encrypt_blocks; - } - } else if (dat->mode == EVP_CIPH_ECB_MODE) { - if (dat->enc) { - dat->stream.ecb = (ecb128_f) rv64i_zvkned_ecb_encrypt; - } else { - dat->stream.ecb = (ecb128_f) rv64i_zvkned_ecb_decrypt; - } - } - - /* Zvkned supports aes-128/192/256 encryption and decryption. */ - if ((dat->mode == EVP_CIPH_ECB_MODE || dat->mode == EVP_CIPH_CBC_MODE) && - !dat->enc) { - dat->block = (block128_f) rv64i_zvkned_decrypt; - } else { - dat->block = (block128_f) rv64i_zvkned_encrypt; - } - - if (ret < 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_KEY_SETUP_FAILED); - return 0; - } - - return 1; -} - -#define PROV_CIPHER_HW_declare(mode) \ -static const PROV_CIPHER_HW rv64i_zknd_zkne_##mode = { \ - cipher_hw_rv64i_zknd_zkne_initkey, \ - cipher_hw_rv64i_zknd_zkne_##mode, \ - cipher_hw_aes_copyctx \ -}; \ -static const PROV_CIPHER_HW rv64i_zvkned_##mode = { \ - cipher_hw_rv64i_zvkned_initkey, \ - cipher_hw_rv64i_zvkned_##mode, \ - cipher_hw_aes_copyctx \ -}; -#define PROV_CIPHER_HW_select(mode) \ -if (RISCV_HAS_ZVKNED() && riscv_vlen() >= 128) \ - return &rv64i_zvkned_##mode; \ -else if (RISCV_HAS_ZKND_AND_ZKNE()) \ - return &rv64i_zknd_zkne_##mode; diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_hw_s390x.inc b/openssl/src/providers/implementations/ciphers/cipher_aes_hw_s390x.inc index 6c4a4cc99..c8282dbd0 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_hw_s390x.inc +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_hw_s390x.inc @@ -1,5 +1,5 @@ /* - * Copyright 2001-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -58,6 +58,7 @@ static int s390x_aes_ofb128_initkey(PROV_CIPHER_CTX *dat, memcpy(adat->plat.s390x.param.kmo_kmf.k, key, keylen); adat->plat.s390x.fc = S390X_AES_FC(keylen); + adat->plat.s390x.res = 0; return 1; } @@ -65,7 +66,7 @@ static int s390x_aes_ofb128_cipher_hw(PROV_CIPHER_CTX *dat, unsigned char *out, const unsigned char *in, size_t len) { PROV_AES_CTX *adat = (PROV_AES_CTX *)dat; - int n = dat->num; + int n = adat->plat.s390x.res; int rem; memcpy(adat->plat.s390x.param.kmo_kmf.cv, dat->iv, dat->ivlen); @@ -101,7 +102,7 @@ static int s390x_aes_ofb128_cipher_hw(PROV_CIPHER_CTX *dat, unsigned char *out, } memcpy(dat->iv, adat->plat.s390x.param.kmo_kmf.cv, dat->ivlen); - dat->num = n; + adat->plat.s390x.res = n; return 1; } @@ -112,6 +113,7 @@ static int s390x_aes_cfb128_initkey(PROV_CIPHER_CTX *dat, adat->plat.s390x.fc = S390X_AES_FC(keylen); adat->plat.s390x.fc |= 16 << 24; /* 16 bytes cipher feedback */ + adat->plat.s390x.res = 0; memcpy(adat->plat.s390x.param.kmo_kmf.k, key, keylen); return 1; } @@ -121,7 +123,7 @@ static int s390x_aes_cfb128_cipher_hw(PROV_CIPHER_CTX *dat, unsigned char *out, { PROV_AES_CTX *adat = (PROV_AES_CTX *)dat; unsigned int modifier = adat->base.enc ? 0 : S390X_DECRYPT; - int n = dat->num; + int n = adat->plat.s390x.res; int rem; unsigned char tmp; @@ -162,7 +164,7 @@ static int s390x_aes_cfb128_cipher_hw(PROV_CIPHER_CTX *dat, unsigned char *out, } memcpy(dat->iv, adat->plat.s390x.param.kmo_kmf.cv, dat->ivlen); - dat->num = n; + adat->plat.s390x.res = n; return 1; } diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_ocb.c b/openssl/src/providers/implementations/ciphers/cipher_aes_ocb.c index aec988e44..ce377ad57 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_ocb.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_ocb.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -340,8 +340,10 @@ static void *aes_ocb_dupctx(void *vctx) return NULL; ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } *ret = *in; if (!aes_generic_ocb_copy_ctx(ret, in)) { OPENSSL_free(ret); @@ -385,10 +387,7 @@ static int aes_ocb_set_ctx_params(void *vctx, const OSSL_PARAM params[]) /* IV len must be 1 to 15 */ if (sz < OCB_MIN_IV_LEN || sz > OCB_MAX_IV_LEN) return 0; - if (ctx->base.ivlen != sz) { - ctx->base.ivlen = sz; - ctx->iv_state = IV_STATE_UNINITIALISED; - } + ctx->base.ivlen = sz; } p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_KEYLEN); if (p != NULL) { @@ -552,7 +551,7 @@ const OSSL_DISPATCH ossl_##aes##kbits##mode##_functions[] = { \ (void (*)(void))cipher_ocb_gettable_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))cipher_ocb_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } IMPLEMENT_cipher(ocb, OCB, AES_OCB_FLAGS, 256, 128, OCB_DEFAULT_IV_LEN * 8); diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c index 00920408b..7aa97dc77 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_ocb_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -103,93 +103,6 @@ static const PROV_CIPHER_HW aes_t4_ocb = { \ # define PROV_CIPHER_HW_select() \ if (SPARC_AES_CAPABLE) \ return &aes_t4_ocb; - -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 - -static int cipher_hw_aes_ocb_rv64i_zknd_zkne_initkey(PROV_CIPHER_CTX *vctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_OCB_CTX *ctx = (PROV_AES_OCB_CTX *)vctx; - - OCB_SET_KEY_FN(rv64i_zkne_set_encrypt_key, rv64i_zknd_set_decrypt_key, - rv64i_zkne_encrypt, rv64i_zknd_decrypt, NULL, NULL); - return 1; -} - -static int cipher_hw_aes_ocb_rv64i_zvkned_initkey(PROV_CIPHER_CTX *vctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_OCB_CTX *ctx = (PROV_AES_OCB_CTX *)vctx; - - /* Zvkned only supports 128 and 256 bit keys. */ - if (keylen * 8 == 128 || keylen * 8 == 256) { - OCB_SET_KEY_FN(rv64i_zvkned_set_encrypt_key, - rv64i_zvkned_set_decrypt_key, - rv64i_zvkned_encrypt, rv64i_zvkned_decrypt, - NULL, NULL); - } else { - OCB_SET_KEY_FN(AES_set_encrypt_key, AES_set_encrypt_key, - rv64i_zvkned_encrypt, rv64i_zvkned_decrypt, - NULL, NULL); - } - return 1; -} - -# define PROV_CIPHER_HW_declare() \ -static const PROV_CIPHER_HW aes_rv64i_zknd_zkne_ocb = { \ - cipher_hw_aes_ocb_rv64i_zknd_zkne_initkey, \ - NULL \ -}; \ -static const PROV_CIPHER_HW aes_rv64i_zvkned_ocb = { \ - cipher_hw_aes_ocb_rv64i_zvkned_initkey, \ - NULL \ -}; -# define PROV_CIPHER_HW_select() \ - if (RISCV_HAS_ZVKNED() && riscv_vlen() >= 128) \ - return &aes_rv64i_zvkned_ocb; \ - else if (RISCV_HAS_ZKND_AND_ZKNE()) \ - return &aes_rv64i_zknd_zkne_ocb; - -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 - -static int cipher_hw_aes_ocb_rv32i_zknd_zkne_initkey(PROV_CIPHER_CTX *vctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_OCB_CTX *ctx = (PROV_AES_OCB_CTX *)vctx; - - OCB_SET_KEY_FN(rv32i_zkne_set_encrypt_key, rv32i_zknd_zkne_set_decrypt_key, - rv32i_zkne_encrypt, rv32i_zknd_decrypt, NULL, NULL); - return 1; -} - -static int cipher_hw_aes_ocb_rv32i_zbkb_zknd_zkne_initkey(PROV_CIPHER_CTX *vctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_OCB_CTX *ctx = (PROV_AES_OCB_CTX *)vctx; - - OCB_SET_KEY_FN(rv32i_zbkb_zkne_set_encrypt_key, rv32i_zbkb_zknd_zkne_set_decrypt_key, - rv32i_zkne_encrypt, rv32i_zknd_decrypt, NULL, NULL); - return 1; -} - -# define PROV_CIPHER_HW_declare() \ -static const PROV_CIPHER_HW aes_rv32i_zknd_zkne_ocb = { \ - cipher_hw_aes_ocb_rv32i_zknd_zkne_initkey, \ - NULL \ -}; \ -static const PROV_CIPHER_HW aes_rv32i_zbkb_zknd_zkne_ocb = { \ - cipher_hw_aes_ocb_rv32i_zbkb_zknd_zkne_initkey, \ - NULL \ -}; -# define PROV_CIPHER_HW_select() \ - if (RISCV_HAS_ZBKB_AND_ZKND_AND_ZKNE()) \ - return &aes_rv32i_zbkb_zknd_zkne_ocb; \ - if (RISCV_HAS_ZKND_AND_ZKNE()) \ - return &aes_rv32i_zknd_zkne_ocb; #else # define PROV_CIPHER_HW_declare() # define PROV_CIPHER_HW_select() diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_siv.c b/openssl/src/providers/implementations/ciphers/cipher_aes_siv.c index bcbc17a48..b396c8651 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_siv.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_siv.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -65,8 +65,10 @@ static void *siv_dupctx(void *vctx) return NULL; ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } if (!in->hw->dupctx(in, ret)) { OPENSSL_free(ret); ret = NULL; @@ -271,7 +273,7 @@ static int alg##_##kbits##_##lc##_get_params(OSSL_PARAM params[]) \ return ossl_cipher_generic_get_params(params, EVP_CIPH_##UCMODE##_MODE, \ flags, 2*kbits, blkbits, ivbits); \ } \ -static void *alg##kbits##lc##_newctx(void *provctx) \ +static void * alg##kbits##lc##_newctx(void *provctx) \ { \ return alg##_##lc##_newctx(provctx, 2*kbits, EVP_CIPH_##UCMODE##_MODE, \ flags); \ @@ -297,7 +299,7 @@ const OSSL_DISPATCH ossl_##alg##kbits##lc##_functions[] = { \ (void (*)(void)) alg##_##lc##_set_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void)) alg##_##lc##_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; IMPLEMENT_cipher(aes, siv, SIV, SIV_FLAGS, 128, 8, 0) diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_wrp.c b/openssl/src/providers/implementations/ciphers/cipher_aes_wrp.c index 912a47e3f..8bddf475e 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_wrp.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_wrp.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -66,26 +66,6 @@ static void *aes_wrap_newctx(size_t kbits, size_t blkbits, return wctx; } -static void *aes_wrap_dupctx(void *wctx) -{ - PROV_AES_WRAP_CTX *ctx = wctx; - PROV_AES_WRAP_CTX *dctx = wctx; - - if (ctx == NULL) - return NULL; - dctx = OPENSSL_memdup(ctx, sizeof(*ctx)); - - if (dctx != NULL && dctx->base.tlsmac != NULL && dctx->base.alloced) { - dctx->base.tlsmac = OPENSSL_memdup(dctx->base.tlsmac, - dctx->base.tlsmacsize); - if (dctx->base.tlsmac == NULL) { - OPENSSL_free(dctx); - dctx = NULL; - } - } - return dctx; -} - static void aes_wrap_freectx(void *vctx) { PROV_AES_WRAP_CTX *wctx = (PROV_AES_WRAP_CTX *)vctx; @@ -301,7 +281,6 @@ static int aes_wrap_set_ctx_params(void *vctx, const OSSL_PARAM params[]) { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))aes_##mode##_cipher }, \ { OSSL_FUNC_CIPHER_FINAL, (void (*)(void))aes_##mode##_final }, \ { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))aes_##mode##_freectx }, \ - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void))aes_##mode##_dupctx }, \ { OSSL_FUNC_CIPHER_GET_PARAMS, \ (void (*)(void))aes_##kbits##_##fname##_get_params }, \ { OSSL_FUNC_CIPHER_GETTABLE_PARAMS, \ @@ -314,7 +293,7 @@ static int aes_wrap_set_ctx_params(void *vctx, const OSSL_PARAM params[]) (void (*)(void))ossl_cipher_generic_gettable_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))ossl_cipher_generic_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } IMPLEMENT_cipher(wrap, wrap, WRAP, WRAP_FLAGS, 256, 64, AES_WRAP_NOPAD_IVLEN * 8); diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_xts.c b/openssl/src/providers/implementations/ciphers/cipher_aes_xts.c index cce2537ea..dce203298 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_xts.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_xts.c @@ -1,6 +1,6 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -146,8 +146,10 @@ static void *aes_xts_dupctx(void *vctx) return NULL; } ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } in->base.hw->copyctx(&ret->base, &in->base); return ret; } @@ -285,7 +287,7 @@ const OSSL_DISPATCH ossl_aes##kbits##xts_functions[] = { \ (void (*)(void))aes_xts_set_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))aes_xts_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } IMPLEMENT_cipher(xts, XTS, 256, AES_XTS_FLAGS); diff --git a/openssl/src/providers/implementations/ciphers/cipher_aes_xts_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aes_xts_hw.c index 3163234c3..c71492f51 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_aes_xts_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_aes_xts_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -158,135 +158,6 @@ static const PROV_CIPHER_HW aes_xts_t4 = { \ # define PROV_CIPHER_HW_select_xts() \ if (SPARC_AES_CAPABLE) \ return &aes_xts_t4; - -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64 - -static int cipher_hw_aes_xts_rv64i_zknd_zkne_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_XTS_CTX *xctx = (PROV_AES_XTS_CTX *)ctx; - OSSL_xts_stream_fn stream_enc = NULL; - OSSL_xts_stream_fn stream_dec = NULL; - - XTS_SET_KEY_FN(rv64i_zkne_set_encrypt_key, rv64i_zknd_set_decrypt_key, - rv64i_zkne_encrypt, rv64i_zknd_decrypt, - stream_enc, stream_dec); - return 1; -} - -static int cipher_hw_aes_xts_rv64i_zvbb_zvkg_zvkned_initkey( - PROV_CIPHER_CTX *ctx, const unsigned char *key, size_t keylen) -{ - PROV_AES_XTS_CTX *xctx = (PROV_AES_XTS_CTX *)ctx; - OSSL_xts_stream_fn stream_enc = NULL; - OSSL_xts_stream_fn stream_dec = NULL; - - /* Zvkned only supports 128 and 256 bit keys. */ - if (keylen * 8 == 128 * 2 || keylen * 8 == 256 * 2) { - XTS_SET_KEY_FN(rv64i_zvkned_set_encrypt_key, - rv64i_zvkned_set_decrypt_key, rv64i_zvkned_encrypt, - rv64i_zvkned_decrypt, - rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt, - rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt); - } else { - XTS_SET_KEY_FN(AES_set_encrypt_key, AES_set_encrypt_key, - rv64i_zvkned_encrypt, rv64i_zvkned_decrypt, - stream_enc, stream_dec); - } - return 1; -} - -static int cipher_hw_aes_xts_rv64i_zvkned_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_XTS_CTX *xctx = (PROV_AES_XTS_CTX *)ctx; - OSSL_xts_stream_fn stream_enc = NULL; - OSSL_xts_stream_fn stream_dec = NULL; - - /* Zvkned only supports 128 and 256 bit keys. */ - if (keylen * 8 == 128 * 2 || keylen * 8 == 256 * 2) { - XTS_SET_KEY_FN(rv64i_zvkned_set_encrypt_key, - rv64i_zvkned_set_decrypt_key, - rv64i_zvkned_encrypt, rv64i_zvkned_decrypt, - stream_enc, stream_dec); - } else { - XTS_SET_KEY_FN(AES_set_encrypt_key, AES_set_encrypt_key, - rv64i_zvkned_encrypt, rv64i_zvkned_decrypt, - stream_enc, stream_dec); - } - return 1; -} - -# define PROV_CIPHER_HW_declare_xts() \ -static const PROV_CIPHER_HW aes_xts_rv64i_zknd_zkne = { \ - cipher_hw_aes_xts_rv64i_zknd_zkne_initkey, \ - NULL, \ - cipher_hw_aes_xts_copyctx \ -}; \ -static const PROV_CIPHER_HW aes_xts_rv64i_zvkned = { \ - cipher_hw_aes_xts_rv64i_zvkned_initkey, \ - NULL, \ - cipher_hw_aes_xts_copyctx \ -}; \ -static const PROV_CIPHER_HW aes_xts_rv64i_zvbb_zvkg_zvkned = { \ - cipher_hw_aes_xts_rv64i_zvbb_zvkg_zvkned_initkey, \ - NULL, \ - cipher_hw_aes_xts_copyctx \ -}; - -# define PROV_CIPHER_HW_select_xts() \ -if (RISCV_HAS_ZVBB() && RISCV_HAS_ZVKG() && RISCV_HAS_ZVKNED() && \ - riscv_vlen() >= 128) \ - return &aes_xts_rv64i_zvbb_zvkg_zvkned; \ -if (RISCV_HAS_ZVKNED() && riscv_vlen() >= 128) \ - return &aes_xts_rv64i_zvkned; \ -else if (RISCV_HAS_ZKND_AND_ZKNE()) \ - return &aes_xts_rv64i_zknd_zkne; - -#elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 - -static int cipher_hw_aes_xts_rv32i_zknd_zkne_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_XTS_CTX *xctx = (PROV_AES_XTS_CTX *)ctx; - - XTS_SET_KEY_FN(rv32i_zkne_set_encrypt_key, rv32i_zknd_zkne_set_decrypt_key, - rv32i_zkne_encrypt, rv32i_zknd_decrypt, - NULL, NULL); - return 1; -} - -static int cipher_hw_aes_xts_rv32i_zbkb_zknd_zkne_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_AES_XTS_CTX *xctx = (PROV_AES_XTS_CTX *)ctx; - - XTS_SET_KEY_FN(rv32i_zbkb_zkne_set_encrypt_key, rv32i_zbkb_zknd_zkne_set_decrypt_key, - rv32i_zkne_encrypt, rv32i_zknd_decrypt, - NULL, NULL); - return 1; -} - -# define PROV_CIPHER_HW_declare_xts() \ -static const PROV_CIPHER_HW aes_xts_rv32i_zknd_zkne = { \ - cipher_hw_aes_xts_rv32i_zknd_zkne_initkey, \ - NULL, \ - cipher_hw_aes_xts_copyctx \ -}; \ -static const PROV_CIPHER_HW aes_xts_rv32i_zbkb_zknd_zkne = { \ - cipher_hw_aes_xts_rv32i_zbkb_zknd_zkne_initkey, \ - NULL, \ - cipher_hw_aes_xts_copyctx \ -}; -# define PROV_CIPHER_HW_select_xts() \ -if (RISCV_HAS_ZBKB_AND_ZKND_AND_ZKNE()) \ - return &aes_xts_rv32i_zbkb_zknd_zkne; \ -if (RISCV_HAS_ZKND_AND_ZKNE()) \ - return &aes_xts_rv32i_zknd_zkne; # else /* The generic case */ # define PROV_CIPHER_HW_declare_xts() diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria.c b/openssl/src/providers/implementations/ciphers/cipher_aria.c deleted file mode 100644 index ce4938d44..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria.c +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for ARIA cipher modes ecb, cbc, ofb, cfb, ctr */ - -#include "cipher_aria.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -static OSSL_FUNC_cipher_freectx_fn aria_freectx; -static OSSL_FUNC_cipher_dupctx_fn aria_dupctx; - -static void aria_freectx(void *vctx) -{ - PROV_ARIA_CTX *ctx = (PROV_ARIA_CTX *)vctx; - - ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *aria_dupctx(void *ctx) -{ - PROV_ARIA_CTX *in = (PROV_ARIA_CTX *)ctx; - PROV_ARIA_CTX *ret; - - if (!ossl_prov_is_running()) - return NULL; - - ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - in->base.hw->copyctx(&ret->base, &in->base); - - return ret; -} - -/* ossl_aria256ecb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ecb, ECB, 0, 256, 128, 0, block) -/* ossl_aria192ecb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ecb, ECB, 0, 192, 128, 0, block) -/* ossl_aria128ecb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ecb, ECB, 0, 128, 128, 0, block) -/* ossl_aria256cbc_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cbc, CBC, 0, 256, 128, 128, block) -/* ossl_aria192cbc_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cbc, CBC, 0, 192, 128, 128, block) -/* ossl_aria128cbc_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cbc, CBC, 0, 128, 128, 128, block) -/* ossl_aria256ofb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ofb, OFB, 0, 256, 8, 128, stream) -/* ossl_aria192ofb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ofb, OFB, 0, 192, 8, 128, stream) -/* ossl_aria128ofb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ofb, OFB, 0, 128, 8, 128, stream) -/* ossl_aria256cfb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb, CFB, 0, 256, 8, 128, stream) -/* ossl_aria192cfb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb, CFB, 0, 192, 8, 128, stream) -/* ossl_aria128cfb_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb, CFB, 0, 128, 8, 128, stream) -/* ossl_aria256cfb1_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb1, CFB, 0, 256, 8, 128, stream) -/* ossl_aria192cfb1_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb1, CFB, 0, 192, 8, 128, stream) -/* ossl_aria128cfb1_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb1, CFB, 0, 128, 8, 128, stream) -/* ossl_aria256cfb8_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb8, CFB, 0, 256, 8, 128, stream) -/* ossl_aria192cfb8_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb8, CFB, 0, 192, 8, 128, stream) -/* ossl_aria128cfb8_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, cfb8, CFB, 0, 128, 8, 128, stream) -/* ossl_aria256ctr_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ctr, CTR, 0, 256, 8, 128, stream) -/* ossl_aria192ctr_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ctr, CTR, 0, 192, 8, 128, stream) -/* ossl_aria128ctr_functions */ -IMPLEMENT_generic_cipher(aria, ARIA, ctr, CTR, 0, 128, 8, 128, stream) diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria.h b/openssl/src/providers/implementations/ciphers/cipher_aria.h deleted file mode 100644 index 39f84d3b4..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "crypto/aria.h" -#include "prov/ciphercommon.h" - -typedef struct prov_aria_ctx_st { - PROV_CIPHER_CTX base; /* Must be first */ - union { - OSSL_UNION_ALIGN; - ARIA_KEY ks; - } ks; -} PROV_ARIA_CTX; - - -#define ossl_prov_cipher_hw_aria_ofb ossl_prov_cipher_hw_aria_ofb128 -#define ossl_prov_cipher_hw_aria_cfb ossl_prov_cipher_hw_aria_cfb128 -const PROV_CIPHER_HW *ossl_prov_cipher_hw_aria_ecb(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_aria_cbc(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_aria_ofb128(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_aria_cfb128(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_aria_cfb1(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_aria_cfb8(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_aria_ctr(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria_ccm.c b/openssl/src/providers/implementations/ciphers/cipher_aria_ccm.c deleted file mode 100644 index 0a0f52cdc..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria_ccm.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for ARIA CCM mode */ - -#include "cipher_aria_ccm.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -static OSSL_FUNC_cipher_freectx_fn aria_ccm_freectx; - -static void *aria_ccm_newctx(void *provctx, size_t keybits) -{ - PROV_ARIA_CCM_CTX *ctx; - - if (!ossl_prov_is_running()) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx != NULL) - ossl_ccm_initctx(&ctx->base, keybits, ossl_prov_aria_hw_ccm(keybits)); - return ctx; -} - -static void *aria_ccm_dupctx(void *provctx) -{ - PROV_ARIA_CCM_CTX *ctx = provctx; - PROV_ARIA_CCM_CTX *dctx = NULL; - - if (ctx == NULL) - return NULL; - - dctx = OPENSSL_memdup(ctx, sizeof(*ctx)); - if (dctx != NULL && dctx->base.ccm_ctx.key != NULL) - dctx->base.ccm_ctx.key = &dctx->ks.ks; - - return dctx; -} - -static void aria_ccm_freectx(void *vctx) -{ - PROV_ARIA_CCM_CTX *ctx = (PROV_ARIA_CCM_CTX *)vctx; - - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -/* aria128ccm functions */ -IMPLEMENT_aead_cipher(aria, ccm, CCM, AEAD_FLAGS, 128, 8, 96); -/* aria192ccm functions */ -IMPLEMENT_aead_cipher(aria, ccm, CCM, AEAD_FLAGS, 192, 8, 96); -/* aria256ccm functions */ -IMPLEMENT_aead_cipher(aria, ccm, CCM, AEAD_FLAGS, 256, 8, 96); - diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria_ccm.h b/openssl/src/providers/implementations/ciphers/cipher_aria_ccm.h deleted file mode 100644 index 558da4973..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria_ccm.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "crypto/aria.h" -#include "prov/ciphercommon.h" -#include "prov/ciphercommon_ccm.h" - -typedef struct prov_aria_ccm_ctx_st { - PROV_CCM_CTX base; /* Must be first */ - union { - OSSL_UNION_ALIGN; - ARIA_KEY ks; - } ks; /* ARIA key schedule to use */ -} PROV_ARIA_CCM_CTX; - -const PROV_CCM_HW *ossl_prov_aria_hw_ccm(size_t keylen); diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c deleted file mode 100644 index e56ec8fb0..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria_ccm_hw.c +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * Generic support for ARIA CCM. - */ - -#include "cipher_aria_ccm.h" - -static int ccm_aria_initkey(PROV_CCM_CTX *ctx, - const unsigned char *key, size_t keylen) -{ - PROV_ARIA_CCM_CTX *actx = (PROV_ARIA_CCM_CTX *)ctx; - - ossl_aria_set_encrypt_key(key, keylen * 8, &actx->ks.ks); - CRYPTO_ccm128_init(&ctx->ccm_ctx, ctx->m, ctx->l, &actx->ks.ks, - (block128_f)ossl_aria_encrypt); - ctx->str = NULL; - ctx->key_set = 1; - return 1; -} - -static const PROV_CCM_HW ccm_aria = { - ccm_aria_initkey, - ossl_ccm_generic_setiv, - ossl_ccm_generic_setaad, - ossl_ccm_generic_auth_encrypt, - ossl_ccm_generic_auth_decrypt, - ossl_ccm_generic_gettag -}; -const PROV_CCM_HW *ossl_prov_aria_hw_ccm(size_t keybits) -{ - return &ccm_aria; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria_gcm.c b/openssl/src/providers/implementations/ciphers/cipher_aria_gcm.c deleted file mode 100644 index e794a80a0..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria_gcm.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for ARIA GCM mode */ - -#include "cipher_aria_gcm.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -static void *aria_gcm_newctx(void *provctx, size_t keybits) -{ - PROV_ARIA_GCM_CTX *ctx; - - if (!ossl_prov_is_running()) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx != NULL) - ossl_gcm_initctx(provctx, &ctx->base, keybits, - ossl_prov_aria_hw_gcm(keybits)); - return ctx; -} - -static void *aria_gcm_dupctx(void *provctx) -{ - PROV_ARIA_GCM_CTX *ctx = provctx; - PROV_ARIA_GCM_CTX *dctx = NULL; - - if (ctx == NULL) - return NULL; - - dctx = OPENSSL_memdup(ctx, sizeof(*ctx)); - if (dctx != NULL && dctx->base.gcm.key != NULL) - dctx->base.gcm.key = &dctx->ks.ks; - - return dctx; -} - -static OSSL_FUNC_cipher_freectx_fn aria_gcm_freectx; -static void aria_gcm_freectx(void *vctx) -{ - PROV_ARIA_GCM_CTX *ctx = (PROV_ARIA_GCM_CTX *)vctx; - - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -/* ossl_aria128gcm_functions */ -IMPLEMENT_aead_cipher(aria, gcm, GCM, AEAD_FLAGS, 128, 8, 96); -/* ossl_aria192gcm_functions */ -IMPLEMENT_aead_cipher(aria, gcm, GCM, AEAD_FLAGS, 192, 8, 96); -/* ossl_aria256gcm_functions */ -IMPLEMENT_aead_cipher(aria, gcm, GCM, AEAD_FLAGS, 256, 8, 96); - diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria_gcm.h b/openssl/src/providers/implementations/ciphers/cipher_aria_gcm.h deleted file mode 100644 index 6251e8322..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria_gcm.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "crypto/aria.h" -#include "prov/ciphercommon.h" -#include "prov/ciphercommon_gcm.h" - -typedef struct prov_aria_gcm_ctx_st { - PROV_GCM_CTX base; /* must be first entry in struct */ - union { - OSSL_UNION_ALIGN; - ARIA_KEY ks; - } ks; -} PROV_ARIA_GCM_CTX; - -const PROV_GCM_HW *ossl_prov_aria_hw_gcm(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c deleted file mode 100644 index 927327c29..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria_gcm_hw.c +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * Generic support for ARIA GCM. - */ - -#include "cipher_aria_gcm.h" - -static int aria_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, - size_t keylen) -{ - PROV_ARIA_GCM_CTX *actx = (PROV_ARIA_GCM_CTX *)ctx; - ARIA_KEY *ks = &actx->ks.ks; - - GCM_HW_SET_KEY_CTR_FN(ks, ossl_aria_set_encrypt_key, ossl_aria_encrypt, NULL); - return 1; -} - -static const PROV_GCM_HW aria_gcm = { - aria_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - ossl_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; -const PROV_GCM_HW *ossl_prov_aria_hw_gcm(size_t keybits) -{ - return &aria_gcm; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_aria_hw.c b/openssl/src/providers/implementations/ciphers/cipher_aria_hw.c deleted file mode 100644 index 425d87a65..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_aria_hw.c +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "cipher_aria.h" - -static int cipher_hw_aria_initkey(PROV_CIPHER_CTX *dat, - const unsigned char *key, size_t keylen) -{ - int ret, mode = dat->mode; - PROV_ARIA_CTX *adat = (PROV_ARIA_CTX *)dat; - ARIA_KEY *ks = &adat->ks.ks; - - if (dat->enc || (mode != EVP_CIPH_ECB_MODE && mode != EVP_CIPH_CBC_MODE)) - ret = ossl_aria_set_encrypt_key(key, keylen * 8, ks); - else - ret = ossl_aria_set_decrypt_key(key, keylen * 8, ks); - if (ret < 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_KEY_SETUP_FAILED); - return 0; - } - dat->ks = ks; - dat->block = (block128_f)ossl_aria_encrypt; - return 1; -} - -IMPLEMENT_CIPHER_HW_COPYCTX(cipher_hw_aria_copyctx, PROV_ARIA_CTX) - -# define PROV_CIPHER_HW_aria_mode(mode) \ -static const PROV_CIPHER_HW aria_##mode = { \ - cipher_hw_aria_initkey, \ - ossl_cipher_hw_chunked_##mode, \ - cipher_hw_aria_copyctx \ -}; \ -const PROV_CIPHER_HW *ossl_prov_cipher_hw_aria_##mode(size_t keybits) \ -{ \ - return &aria_##mode; \ -} - -PROV_CIPHER_HW_aria_mode(cbc) -PROV_CIPHER_HW_aria_mode(ecb) -PROV_CIPHER_HW_aria_mode(ofb128) -PROV_CIPHER_HW_aria_mode(cfb128) -PROV_CIPHER_HW_aria_mode(cfb1) -PROV_CIPHER_HW_aria_mode(cfb8) -PROV_CIPHER_HW_aria_mode(ctr) diff --git a/openssl/src/providers/implementations/ciphers/cipher_blowfish.c b/openssl/src/providers/implementations/ciphers/cipher_blowfish.c deleted file mode 100644 index 9f17f1200..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_blowfish.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for Blowfish cipher modes ecb, cbc, ofb, cfb */ - -/* - * BF low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include "cipher_blowfish.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -#define BF_FLAGS PROV_CIPHER_FLAG_VARIABLE_LENGTH - -static OSSL_FUNC_cipher_freectx_fn blowfish_freectx; -static OSSL_FUNC_cipher_dupctx_fn blowfish_dupctx; - -static void blowfish_freectx(void *vctx) -{ - PROV_BLOWFISH_CTX *ctx = (PROV_BLOWFISH_CTX *)vctx; - - ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *blowfish_dupctx(void *ctx) -{ - PROV_BLOWFISH_CTX *in = (PROV_BLOWFISH_CTX *)ctx; - PROV_BLOWFISH_CTX *ret; - - if (!ossl_prov_is_running()) - return NULL; - - ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - *ret = *in; - - return ret; -} - -/* bf_ecb_functions */ -IMPLEMENT_var_keylen_cipher(blowfish, BLOWFISH, ecb, ECB, BF_FLAGS, 128, 64, 0, block) -/* bf_cbc_functions */ -IMPLEMENT_var_keylen_cipher(blowfish, BLOWFISH, cbc, CBC, BF_FLAGS, 128, 64, 64, block) -/* bf_ofb_functions */ -IMPLEMENT_var_keylen_cipher(blowfish, BLOWFISH, ofb64, OFB, BF_FLAGS, 128, 8, 64, stream) -/* bf_cfb_functions */ -IMPLEMENT_var_keylen_cipher(blowfish, BLOWFISH, cfb64, CFB, BF_FLAGS, 128, 8, 64, stream) diff --git a/openssl/src/providers/implementations/ciphers/cipher_blowfish.h b/openssl/src/providers/implementations/ciphers/cipher_blowfish.h deleted file mode 100644 index bbdc9da37..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_blowfish.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "prov/ciphercommon.h" - -typedef struct prov_blowfish_ctx_st { - PROV_CIPHER_CTX base; /* Must be first */ - union { - OSSL_UNION_ALIGN; - BF_KEY ks; - } ks; -} PROV_BLOWFISH_CTX; - -const PROV_CIPHER_HW *ossl_prov_cipher_hw_blowfish_cbc(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_blowfish_ecb(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_blowfish_ofb64(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_blowfish_cfb64(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_blowfish_hw.c b/openssl/src/providers/implementations/ciphers/cipher_blowfish_hw.c deleted file mode 100644 index 4855a71f6..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_blowfish_hw.c +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * BF low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include "cipher_blowfish.h" - -static int cipher_hw_blowfish_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, size_t keylen) -{ - PROV_BLOWFISH_CTX *bctx = (PROV_BLOWFISH_CTX *)ctx; - - BF_set_key(&bctx->ks.ks, keylen, key); - return 1; -} - -# define PROV_CIPHER_HW_blowfish_mode(mode, UCMODE) \ -IMPLEMENT_CIPHER_HW_##UCMODE(mode, blowfish, PROV_BLOWFISH_CTX, BF_KEY, \ - BF_##mode) \ -static const PROV_CIPHER_HW bf_##mode = { \ - cipher_hw_blowfish_initkey, \ - cipher_hw_blowfish_##mode##_cipher \ -}; \ -const PROV_CIPHER_HW *ossl_prov_cipher_hw_blowfish_##mode(size_t keybits) \ -{ \ - return &bf_##mode; \ -} - -PROV_CIPHER_HW_blowfish_mode(cbc, CBC) -PROV_CIPHER_HW_blowfish_mode(ecb, ECB) -PROV_CIPHER_HW_blowfish_mode(ofb64, OFB) -PROV_CIPHER_HW_blowfish_mode(cfb64, CFB) diff --git a/openssl/src/providers/implementations/ciphers/cipher_camellia.c b/openssl/src/providers/implementations/ciphers/cipher_camellia.c deleted file mode 100644 index c550af3f8..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_camellia.c +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -/* Dispatch functions for CAMELLIA cipher modes ecb, cbc, ofb, cfb, ctr */ - -#include "cipher_camellia.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -static OSSL_FUNC_cipher_freectx_fn camellia_freectx; -static OSSL_FUNC_cipher_dupctx_fn camellia_dupctx; - -static void camellia_freectx(void *vctx) -{ - PROV_CAMELLIA_CTX *ctx = (PROV_CAMELLIA_CTX *)vctx; - - ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *camellia_dupctx(void *ctx) -{ - PROV_CAMELLIA_CTX *in = (PROV_CAMELLIA_CTX *)ctx; - PROV_CAMELLIA_CTX *ret; - - if (!ossl_prov_is_running()) - return NULL; - - ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - in->base.hw->copyctx(&ret->base, &in->base); - - return ret; -} - -/* ossl_camellia256ecb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ecb, ECB, 0, 256, 128, 0, block) -/* ossl_camellia192ecb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ecb, ECB, 0, 192, 128, 0, block) -/* ossl_camellia128ecb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ecb, ECB, 0, 128, 128, 0, block) -/* ossl_camellia256cbc_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cbc, CBC, 0, 256, 128, 128, block) -/* ossl_camellia192cbc_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cbc, CBC, 0, 192, 128, 128, block) -/* ossl_camellia128cbc_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cbc, CBC, 0, 128, 128, 128, block) -/* ossl_camellia256ofb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ofb, OFB, 0, 256, 8, 128, stream) -/* ossl_camellia192ofb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ofb, OFB, 0, 192, 8, 128, stream) -/* ossl_camellia128ofb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ofb, OFB, 0, 128, 8, 128, stream) -/* ossl_camellia256cfb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb, CFB, 0, 256, 8, 128, stream) -/* ossl_camellia192cfb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb, CFB, 0, 192, 8, 128, stream) -/* ossl_camellia128cfb_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb, CFB, 0, 128, 8, 128, stream) -/* ossl_camellia256cfb1_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb1, CFB, 0, 256, 8, 128, stream) -/* ossl_camellia192cfb1_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb1, CFB, 0, 192, 8, 128, stream) -/* ossl_camellia128cfb1_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb1, CFB, 0, 128, 8, 128, stream) -/* ossl_camellia256cfb8_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb8, CFB, 0, 256, 8, 128, stream) -/* ossl_camellia192cfb8_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb8, CFB, 0, 192, 8, 128, stream) -/* ossl_camellia128cfb8_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, cfb8, CFB, 0, 128, 8, 128, stream) -/* ossl_camellia256ctr_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ctr, CTR, 0, 256, 8, 128, stream) -/* ossl_camellia192ctr_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ctr, CTR, 0, 192, 8, 128, stream) -/* ossl_camellia128ctr_functions */ -IMPLEMENT_generic_cipher(camellia, CAMELLIA, ctr, CTR, 0, 128, 8, 128, stream) - -#include "cipher_camellia_cts.inc" diff --git a/openssl/src/providers/implementations/ciphers/cipher_camellia.h b/openssl/src/providers/implementations/ciphers/cipher_camellia.h deleted file mode 100644 index 953ea74c0..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_camellia.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "prov/ciphercommon.h" -#include "crypto/cmll_platform.h" - -typedef struct prov_camellia_ctx_st { - PROV_CIPHER_CTX base; /* Must be first */ - union { - OSSL_UNION_ALIGN; - CAMELLIA_KEY ks; - } ks; -} PROV_CAMELLIA_CTX; - -#define ossl_prov_cipher_hw_camellia_ofb ossl_prov_cipher_hw_camellia_ofb128 -#define ossl_prov_cipher_hw_camellia_cfb ossl_prov_cipher_hw_camellia_cfb128 -const PROV_CIPHER_HW *ossl_prov_cipher_hw_camellia_ecb(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_camellia_cbc(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_camellia_ofb128(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_camellia_cfb128(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_camellia_cfb1(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_camellia_cfb8(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_camellia_ctr(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_camellia_cts.inc b/openssl/src/providers/implementations/ciphers/cipher_camellia_cts.inc deleted file mode 100644 index 84ea992b8..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_camellia_cts.inc +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for CAMELLIA CBC CTS ciphers */ - -#include -#include "cipher_cts.h" - -#define CTS_FLAGS PROV_CIPHER_FLAG_CTS - -static OSSL_FUNC_cipher_encrypt_init_fn camellia_cbc_cts_einit; -static OSSL_FUNC_cipher_decrypt_init_fn camellia_cbc_cts_dinit; -static OSSL_FUNC_cipher_get_ctx_params_fn camellia_cbc_cts_get_ctx_params; -static OSSL_FUNC_cipher_set_ctx_params_fn camellia_cbc_cts_set_ctx_params; -static OSSL_FUNC_cipher_gettable_ctx_params_fn camellia_cbc_cts_gettable_ctx_params; -static OSSL_FUNC_cipher_settable_ctx_params_fn camellia_cbc_cts_settable_ctx_params; - -CIPHER_DEFAULT_GETTABLE_CTX_PARAMS_START(camellia_cbc_cts) -OSSL_PARAM_utf8_string(OSSL_CIPHER_PARAM_CTS_MODE, NULL, 0), -CIPHER_DEFAULT_GETTABLE_CTX_PARAMS_END(camellia_cbc_cts) - -static int camellia_cbc_cts_einit(void *ctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) -{ - if (!ossl_cipher_generic_einit(ctx, key, keylen, iv, ivlen, NULL)) - return 0; - return camellia_cbc_cts_set_ctx_params(ctx, params); -} - -static int camellia_cbc_cts_dinit(void *ctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) -{ - if (!ossl_cipher_generic_dinit(ctx, key, keylen, iv, ivlen, NULL)) - return 0; - return camellia_cbc_cts_set_ctx_params(ctx, params); -} - -static int camellia_cbc_cts_get_ctx_params(void *vctx, OSSL_PARAM params[]) -{ - PROV_CIPHER_CTX *ctx = (PROV_CIPHER_CTX *)vctx; - OSSL_PARAM *p; - - p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_CTS_MODE); - if (p != NULL) { - const char *name = ossl_cipher_cbc_cts_mode_id2name(ctx->cts_mode); - - if (name == NULL || !OSSL_PARAM_set_utf8_string(p, name)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - } - return ossl_cipher_generic_get_ctx_params(vctx, params); -} - -CIPHER_DEFAULT_SETTABLE_CTX_PARAMS_START(camellia_cbc_cts) -OSSL_PARAM_utf8_string(OSSL_CIPHER_PARAM_CTS_MODE, NULL, 0), -CIPHER_DEFAULT_SETTABLE_CTX_PARAMS_END(camellia_cbc_cts) - -static int camellia_cbc_cts_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - PROV_CIPHER_CTX *ctx = (PROV_CIPHER_CTX *)vctx; - const OSSL_PARAM *p; - int id; - - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_CTS_MODE); - if (p != NULL) { - if (p->data_type != OSSL_PARAM_UTF8_STRING) - goto err; - id = ossl_cipher_cbc_cts_mode_name2id(p->data); - if (id < 0) - goto err; - - ctx->cts_mode = (unsigned int)id; - } - return ossl_cipher_generic_set_ctx_params(vctx, params); -err: - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; -} - -/* ossl_camellia256cbc_cts_functions */ -IMPLEMENT_cts_cipher(camellia, CAMELLIA, cbc, CBC, CTS_FLAGS, 256, 128, 128, block) -/* ossl_camellia192cbc_cts_functions */ -IMPLEMENT_cts_cipher(camellia, CAMELLIA, cbc, CBC, CTS_FLAGS, 192, 128, 128, block) -/* ossl_camellia128cbc_cts_functions */ -IMPLEMENT_cts_cipher(camellia, CAMELLIA, cbc, CBC, CTS_FLAGS, 128, 128, 128, block) diff --git a/openssl/src/providers/implementations/ciphers/cipher_camellia_hw.c b/openssl/src/providers/implementations/ciphers/cipher_camellia_hw.c deleted file mode 100644 index 3ebf5b8d4..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_camellia_hw.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Camellia low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include "cipher_camellia.h" - -static int cipher_hw_camellia_initkey(PROV_CIPHER_CTX *dat, - const unsigned char *key, size_t keylen) -{ - int ret, mode = dat->mode; - PROV_CAMELLIA_CTX *adat = (PROV_CAMELLIA_CTX *)dat; - CAMELLIA_KEY *ks = &adat->ks.ks; - - dat->ks = ks; - ret = Camellia_set_key(key, keylen * 8, ks); - if (ret < 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_KEY_SETUP_FAILED); - return 0; - } - if (dat->enc || (mode != EVP_CIPH_ECB_MODE && mode != EVP_CIPH_CBC_MODE)) { - dat->block = (block128_f) Camellia_encrypt; - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? - (cbc128_f) Camellia_cbc_encrypt : NULL; - } else { - dat->block = (block128_f) Camellia_decrypt; - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? - (cbc128_f) Camellia_cbc_encrypt : NULL; - } - return 1; -} - -IMPLEMENT_CIPHER_HW_COPYCTX(cipher_hw_camellia_copyctx, PROV_CAMELLIA_CTX) - -# if defined(SPARC_CMLL_CAPABLE) -# include "cipher_camellia_hw_t4.inc" -# else -/* The generic case */ -# define PROV_CIPHER_HW_declare(mode) -# define PROV_CIPHER_HW_select(mode) -# endif /* SPARC_CMLL_CAPABLE */ - -#define PROV_CIPHER_HW_camellia_mode(mode) \ -static const PROV_CIPHER_HW camellia_##mode = { \ - cipher_hw_camellia_initkey, \ - ossl_cipher_hw_generic_##mode, \ - cipher_hw_camellia_copyctx \ -}; \ -PROV_CIPHER_HW_declare(mode) \ -const PROV_CIPHER_HW *ossl_prov_cipher_hw_camellia_##mode(size_t keybits) \ -{ \ - PROV_CIPHER_HW_select(mode) \ - return &camellia_##mode; \ -} - -PROV_CIPHER_HW_camellia_mode(cbc) -PROV_CIPHER_HW_camellia_mode(ecb) -PROV_CIPHER_HW_camellia_mode(ofb128) -PROV_CIPHER_HW_camellia_mode(cfb128) -PROV_CIPHER_HW_camellia_mode(cfb1) -PROV_CIPHER_HW_camellia_mode(cfb8) -PROV_CIPHER_HW_camellia_mode(ctr) diff --git a/openssl/src/providers/implementations/ciphers/cipher_camellia_hw_t4.inc b/openssl/src/providers/implementations/ciphers/cipher_camellia_hw_t4.inc deleted file mode 100644 index 2dcf3fa18..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_camellia_hw_t4.inc +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2001-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * Fujitsu SPARC64 X support for camellia modes. - * This file is included by cipher_camellia_hw.c - */ - -static int cipher_hw_camellia_t4_initkey(PROV_CIPHER_CTX *dat, - const unsigned char *key, - size_t keylen) -{ - int ret = 0, bits, mode = dat->mode; - PROV_CAMELLIA_CTX *adat = (PROV_CAMELLIA_CTX *)dat; - CAMELLIA_KEY *ks = &adat->ks.ks; - - dat->ks = ks; - bits = keylen * 8; - - cmll_t4_set_key(key, bits, ks); - - if (dat->enc || (mode != EVP_CIPH_ECB_MODE && mode != EVP_CIPH_CBC_MODE)) { - dat->block = (block128_f) cmll_t4_encrypt; - switch (bits) { - case 128: - if (mode == EVP_CIPH_CBC_MODE) - dat->stream.cbc = (cbc128_f) cmll128_t4_cbc_encrypt; - else if (mode == EVP_CIPH_CTR_MODE) - dat->stream.ctr = (ctr128_f) cmll128_t4_ctr32_encrypt; - else - dat->stream.cbc = NULL; - break; - case 192: - case 256: - if (mode == EVP_CIPH_CBC_MODE) - dat->stream.cbc = (cbc128_f) cmll256_t4_cbc_encrypt; - else if (mode == EVP_CIPH_CTR_MODE) - dat->stream.ctr = (ctr128_f) cmll256_t4_ctr32_encrypt; - else - dat->stream.cbc = NULL; - break; - default: - ret = -1; - break; - } - } else { - dat->block = (block128_f) cmll_t4_decrypt; - switch (bits) { - case 128: - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? - (cbc128_f) cmll128_t4_cbc_decrypt : NULL; - break; - case 192: - case 256: - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? - (cbc128_f) cmll256_t4_cbc_decrypt : NULL; - break; - default: - ret = -1; - break; - } - } - if (ret < 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_KEY_SETUP_FAILED); - return 0; - } - return 1; -} - -#define PROV_CIPHER_HW_declare(mode) \ -static const PROV_CIPHER_HW t4_camellia_##mode = { \ - cipher_hw_camellia_t4_initkey, \ - ossl_cipher_hw_generic_##mode, \ - cipher_hw_camellia_copyctx \ -}; -#define PROV_CIPHER_HW_select(mode) \ -if (SPARC_CMLL_CAPABLE) \ - return &t4_camellia_##mode; diff --git a/openssl/src/providers/implementations/ciphers/cipher_cast.h b/openssl/src/providers/implementations/ciphers/cipher_cast.h deleted file mode 100644 index 84b58621c..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_cast.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "prov/ciphercommon.h" - -typedef struct prov_cast_ctx_st { - PROV_CIPHER_CTX base; /* Must be first */ - union { - OSSL_UNION_ALIGN; - CAST_KEY ks; - } ks; -} PROV_CAST_CTX; - -const PROV_CIPHER_HW *ossl_prov_cipher_hw_cast5_cbc(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_cast5_ecb(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_cast5_ofb64(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_cast5_cfb64(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_cast5.c b/openssl/src/providers/implementations/ciphers/cipher_cast5.c deleted file mode 100644 index 84c88793b..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_cast5.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * CAST low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -/* Dispatch functions for cast cipher modes ecb, cbc, ofb, cfb */ - -#include -#include "cipher_cast.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -#define CAST5_FLAGS PROV_CIPHER_FLAG_VARIABLE_LENGTH - -static OSSL_FUNC_cipher_freectx_fn cast5_freectx; -static OSSL_FUNC_cipher_dupctx_fn cast5_dupctx; - -static void cast5_freectx(void *vctx) -{ - PROV_CAST_CTX *ctx = (PROV_CAST_CTX *)vctx; - - ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *cast5_dupctx(void *ctx) -{ - PROV_CAST_CTX *in = (PROV_CAST_CTX *)ctx; - PROV_CAST_CTX *ret; - - if (!ossl_prov_is_running()) - return NULL; - - ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - *ret = *in; - - return ret; -} - -/* ossl_cast5128ecb_functions */ -IMPLEMENT_var_keylen_cipher(cast5, CAST, ecb, ECB, CAST5_FLAGS, 128, 64, 0, block) -/* ossl_cast5128cbc_functions */ -IMPLEMENT_var_keylen_cipher(cast5, CAST, cbc, CBC, CAST5_FLAGS, 128, 64, 64, block) -/* ossl_cast5128ofb64_functions */ -IMPLEMENT_var_keylen_cipher(cast5, CAST, ofb64, OFB, CAST5_FLAGS, 128, 8, 64, stream) -/* ossl_cast5128cfb64_functions */ -IMPLEMENT_var_keylen_cipher(cast5, CAST, cfb64, CFB, CAST5_FLAGS, 128, 8, 64, stream) diff --git a/openssl/src/providers/implementations/ciphers/cipher_cast5_hw.c b/openssl/src/providers/implementations/ciphers/cipher_cast5_hw.c deleted file mode 100644 index 73f0628e5..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_cast5_hw.c +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * CAST low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include "cipher_cast.h" - -static int cipher_hw_cast5_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, size_t keylen) -{ - PROV_CAST_CTX *bctx = (PROV_CAST_CTX *)ctx; - - CAST_set_key(&(bctx->ks.ks), keylen, key); - return 1; -} - -# define PROV_CIPHER_HW_cast_mode(mode, UCMODE) \ -IMPLEMENT_CIPHER_HW_##UCMODE(mode, cast5, PROV_CAST_CTX, CAST_KEY, \ - CAST_##mode) \ -static const PROV_CIPHER_HW cast5_##mode = { \ - cipher_hw_cast5_initkey, \ - cipher_hw_cast5_##mode##_cipher \ -}; \ -const PROV_CIPHER_HW *ossl_prov_cipher_hw_cast5_##mode(size_t keybits) \ -{ \ - return &cast5_##mode; \ -} - -PROV_CIPHER_HW_cast_mode(cbc, CBC) -PROV_CIPHER_HW_cast_mode(ecb, ECB) -PROV_CIPHER_HW_cast_mode(ofb64, OFB) -PROV_CIPHER_HW_cast_mode(cfb64, CFB) diff --git a/openssl/src/providers/implementations/ciphers/cipher_chacha20.c b/openssl/src/providers/implementations/ciphers/cipher_chacha20.c index 5e2ad9144..386c865d8 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_chacha20.c +++ b/openssl/src/providers/implementations/ciphers/cipher_chacha20.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,7 +21,6 @@ static OSSL_FUNC_cipher_newctx_fn chacha20_newctx; static OSSL_FUNC_cipher_freectx_fn chacha20_freectx; -static OSSL_FUNC_cipher_dupctx_fn chacha20_dupctx; static OSSL_FUNC_cipher_get_params_fn chacha20_get_params; static OSSL_FUNC_cipher_get_ctx_params_fn chacha20_get_ctx_params; static OSSL_FUNC_cipher_set_ctx_params_fn chacha20_set_ctx_params; @@ -65,25 +64,6 @@ static void chacha20_freectx(void *vctx) } } -static void *chacha20_dupctx(void *vctx) -{ - PROV_CHACHA20_CTX *ctx = (PROV_CHACHA20_CTX *)vctx; - PROV_CHACHA20_CTX *dupctx = NULL; - - if (ctx != NULL) { - dupctx = OPENSSL_memdup(ctx, sizeof(*dupctx)); - if (dupctx != NULL && dupctx->base.tlsmac != NULL && dupctx->base.alloced) { - dupctx->base.tlsmac = OPENSSL_memdup(dupctx->base.tlsmac, - dupctx->base.tlsmacsize); - if (dupctx->base.tlsmac == NULL) { - OPENSSL_free(dupctx); - dupctx = NULL; - } - } - } - return dupctx; -} - static int chacha20_get_params(OSSL_PARAM params[]) { return ossl_cipher_generic_get_params(params, 0, CHACHA20_FLAGS, @@ -207,20 +187,19 @@ int ossl_chacha20_dinit(void *vctx, const unsigned char *key, size_t keylen, const OSSL_DISPATCH ossl_chacha20_functions[] = { { OSSL_FUNC_CIPHER_NEWCTX, (void (*)(void))chacha20_newctx }, { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))chacha20_freectx }, - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void))chacha20_dupctx }, { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))ossl_chacha20_einit }, { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))ossl_chacha20_dinit }, { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))chacha20_update }, { OSSL_FUNC_CIPHER_FINAL, (void (*)(void))chacha20_final }, { OSSL_FUNC_CIPHER_CIPHER, (void (*)(void))chacha20_cipher}, { OSSL_FUNC_CIPHER_GET_PARAMS, (void (*)(void))chacha20_get_params }, - { OSSL_FUNC_CIPHER_GETTABLE_PARAMS, (void (*)(void))chacha20_gettable_params }, + { OSSL_FUNC_CIPHER_GETTABLE_PARAMS,(void (*)(void))chacha20_gettable_params }, { OSSL_FUNC_CIPHER_GET_CTX_PARAMS, (void (*)(void))chacha20_get_ctx_params }, { OSSL_FUNC_CIPHER_GETTABLE_CTX_PARAMS, (void (*)(void))chacha20_gettable_ctx_params }, { OSSL_FUNC_CIPHER_SET_CTX_PARAMS, (void (*)(void))chacha20_set_ctx_params }, { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, (void (*)(void))chacha20_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c b/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c index d5d4e1a25..0ba748378 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c +++ b/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,6 +14,7 @@ #include "prov/implementations.h" #include "prov/providercommon.h" + #define CHACHA20_POLY1305_KEYLEN CHACHA_KEY_SIZE #define CHACHA20_POLY1305_BLKLEN 1 #define CHACHA20_POLY1305_MAX_IVLEN 12 @@ -23,7 +24,6 @@ static OSSL_FUNC_cipher_newctx_fn chacha20_poly1305_newctx; static OSSL_FUNC_cipher_freectx_fn chacha20_poly1305_freectx; -static OSSL_FUNC_cipher_dupctx_fn chacha20_poly1305_dupctx; static OSSL_FUNC_cipher_encrypt_init_fn chacha20_poly1305_einit; static OSSL_FUNC_cipher_decrypt_init_fn chacha20_poly1305_dinit; static OSSL_FUNC_cipher_get_params_fn chacha20_poly1305_get_params; @@ -53,31 +53,13 @@ static void *chacha20_poly1305_newctx(void *provctx) ossl_prov_cipher_hw_chacha20_poly1305( CHACHA20_POLY1305_KEYLEN * 8), NULL); + ctx->nonce_len = CHACHA20_POLY1305_IVLEN; ctx->tls_payload_length = NO_TLS_PAYLOAD_LENGTH; ossl_chacha20_initctx(&ctx->chacha); } return ctx; } -static void *chacha20_poly1305_dupctx(void *provctx) -{ - PROV_CHACHA20_POLY1305_CTX *ctx = provctx; - PROV_CHACHA20_POLY1305_CTX *dctx = NULL; - - if (ctx == NULL) - return NULL; - dctx = OPENSSL_memdup(ctx, sizeof(*ctx)); - if (dctx != NULL && dctx->base.tlsmac != NULL && dctx->base.alloced) { - dctx->base.tlsmac = OPENSSL_memdup(dctx->base.tlsmac, - dctx->base.tlsmacsize); - if (dctx->base.tlsmac == NULL) { - OPENSSL_free(dctx); - dctx = NULL; - } - } - return dctx; -} - static void chacha20_poly1305_freectx(void *vctx) { PROV_CHACHA20_POLY1305_CTX *ctx = (PROV_CHACHA20_POLY1305_CTX *)vctx; @@ -103,7 +85,7 @@ static int chacha20_poly1305_get_ctx_params(void *vctx, OSSL_PARAM params[]) p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_IVLEN); if (p != NULL) { - if (!OSSL_PARAM_set_size_t(p, CHACHA20_POLY1305_IVLEN)) { + if (!OSSL_PARAM_set_size_t(p, ctx->nonce_len)) { ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); return 0; } @@ -187,10 +169,11 @@ static int chacha20_poly1305_set_ctx_params(void *vctx, ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); return 0; } - if (len != CHACHA20_POLY1305_MAX_IVLEN) { + if (len == 0 || len > CHACHA20_POLY1305_MAX_IVLEN) { ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); return 0; } + ctx->nonce_len = len; } p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TAG); @@ -330,7 +313,6 @@ static int chacha20_poly1305_final(void *vctx, unsigned char *out, size_t *outl, const OSSL_DISPATCH ossl_chacha20_ossl_poly1305_functions[] = { { OSSL_FUNC_CIPHER_NEWCTX, (void (*)(void))chacha20_poly1305_newctx }, { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))chacha20_poly1305_freectx }, - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void))chacha20_poly1305_dupctx }, { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))chacha20_poly1305_einit }, { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))chacha20_poly1305_dinit }, { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))chacha20_poly1305_update }, @@ -348,6 +330,6 @@ const OSSL_DISPATCH ossl_chacha20_ossl_poly1305_functions[] = { (void (*)(void))chacha20_poly1305_set_ctx_params }, { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, (void (*)(void))chacha20_poly1305_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305.h b/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305.h index f2ea26a77..1f6f0066d 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305.h +++ b/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,7 +25,7 @@ typedef struct { struct { uint64_t aad, text; } len; unsigned int aad : 1; unsigned int mac_inited : 1; - size_t tag_len; + size_t tag_len, nonce_len; size_t tls_payload_length; size_t tls_aad_pad_sz; } PROV_CHACHA20_POLY1305_CTX; diff --git a/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305_hw.c b/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305_hw.c index 8173663e5..1533a3869 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_chacha20_poly1305_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -55,6 +55,7 @@ static int chacha_poly1305_tls_iv_set_fixed(PROV_CIPHER_CTX *bctx, return 1; } + static int chacha20_poly1305_initkey(PROV_CIPHER_CTX *bctx, const unsigned char *key, size_t keylen) { @@ -77,7 +78,6 @@ static int chacha20_poly1305_initiv(PROV_CIPHER_CTX *bctx) PROV_CHACHA20_POLY1305_CTX *ctx = (PROV_CHACHA20_POLY1305_CTX *)bctx; unsigned char tempiv[CHACHA_CTR_SIZE] = { 0 }; int ret = 1; - size_t noncelen = CHACHA20_POLY1305_IVLEN; ctx->len.aad = 0; ctx->len.text = 0; @@ -85,20 +85,22 @@ static int chacha20_poly1305_initiv(PROV_CIPHER_CTX *bctx) ctx->mac_inited = 0; ctx->tls_payload_length = NO_TLS_PAYLOAD_LENGTH; - /* pad on the left */ - memcpy(tempiv + CHACHA_CTR_SIZE - noncelen, bctx->oiv, - noncelen); - - if (bctx->enc) - ret = ossl_chacha20_einit(&ctx->chacha, NULL, 0, - tempiv, sizeof(tempiv), NULL); - else - ret = ossl_chacha20_dinit(&ctx->chacha, NULL, 0, - tempiv, sizeof(tempiv), NULL); - ctx->nonce[0] = ctx->chacha.counter[1]; - ctx->nonce[1] = ctx->chacha.counter[2]; - ctx->nonce[2] = ctx->chacha.counter[3]; - bctx->iv_set = 1; + /* pad on the left */ + if (ctx->nonce_len <= CHACHA_CTR_SIZE) { + memcpy(tempiv + CHACHA_CTR_SIZE - ctx->nonce_len, bctx->oiv, + ctx->nonce_len); + + if (bctx->enc) + ret = ossl_chacha20_einit(&ctx->chacha, NULL, 0, + tempiv, sizeof(tempiv), NULL); + else + ret = ossl_chacha20_dinit(&ctx->chacha, NULL, 0, + tempiv, sizeof(tempiv), NULL); + ctx->nonce[0] = ctx->chacha.counter[1]; + ctx->nonce[1] = ctx->chacha.counter[2]; + ctx->nonce[2] = ctx->chacha.counter[3]; + bctx->iv_set = 1; + } return ret; } diff --git a/openssl/src/providers/implementations/ciphers/cipher_cts.c b/openssl/src/providers/implementations/ciphers/cipher_cts.c index 6a596508d..72cc58ff8 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_cts.c +++ b/openssl/src/providers/implementations/ciphers/cipher_cts.c @@ -11,7 +11,7 @@ * Helper functions for 128 bit CBC CTS ciphers (Currently AES and Camellia). * * The function dispatch tables are embedded into cipher_aes.c - * and cipher_camellia.c using cipher_aes_cts.inc and cipher_camellia_cts.inc + * using cipher_aes_cts.inc. */ /* diff --git a/openssl/src/providers/implementations/ciphers/cipher_cts.h b/openssl/src/providers/implementations/ciphers/cipher_cts.h index a26e5a9e0..9473fbde8 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_cts.h +++ b/openssl/src/providers/implementations/ciphers/cipher_cts.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -42,7 +42,7 @@ const OSSL_DISPATCH ossl_##alg##kbits##lcmode##_cts_functions[] = { \ (void (*)(void)) alg##_cbc_cts_gettable_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void)) alg##_cbc_cts_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; OSSL_FUNC_cipher_update_fn ossl_cipher_cbc_cts_block_update; diff --git a/openssl/src/providers/implementations/ciphers/cipher_des.c b/openssl/src/providers/implementations/ciphers/cipher_des.c index e2c890979..c6d13466f 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_des.c +++ b/openssl/src/providers/implementations/ciphers/cipher_des.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -53,8 +53,10 @@ static void *des_dupctx(void *ctx) return NULL; ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } in->base.hw->copyctx(&ret->base, &in->base); return ret; @@ -96,7 +98,6 @@ static int des_init(void *vctx, const unsigned char *key, size_t keylen, } if (!ctx->hw->init(ctx, key, keylen)) return 0; - ctx->key_set = 1; } return ossl_cipher_generic_set_ctx_params(ctx, params); } @@ -184,7 +185,7 @@ const OSSL_DISPATCH ossl_##des_##lcmode##_functions[] = { \ (void (*)(void))ossl_cipher_generic_set_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))ossl_cipher_generic_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } /* ossl_des_ecb_functions */ diff --git a/openssl/src/providers/implementations/ciphers/cipher_des_hw.c b/openssl/src/providers/implementations/ciphers/cipher_des_hw.c index a2d54b46b..a77fcc681 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_des_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_des_hw.c @@ -136,8 +136,7 @@ static int cipher_hw_des_cfb1_cipher(PROV_CIPHER_CTX *ctx, unsigned char *out, { size_t n, chunk = MAXCHUNK / 8; DES_key_schedule *key = &(((PROV_DES_CTX *)ctx)->dks.ks); - unsigned char c[1]; - unsigned char d[1] = { 0 }; + unsigned char c[1], d[1]; if (inl < chunk) chunk = inl; diff --git a/openssl/src/providers/implementations/ciphers/cipher_idea.c b/openssl/src/providers/implementations/ciphers/cipher_idea.c deleted file mode 100644 index c69c6ac09..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_idea.c +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * IDEA low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -/* Dispatch functions for Idea cipher modes ecb, cbc, ofb, cfb */ - -#include "cipher_idea.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -static OSSL_FUNC_cipher_freectx_fn idea_freectx; -static OSSL_FUNC_cipher_dupctx_fn idea_dupctx; - -static void idea_freectx(void *vctx) -{ - PROV_IDEA_CTX *ctx = (PROV_IDEA_CTX *)vctx; - - ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *idea_dupctx(void *ctx) -{ - PROV_IDEA_CTX *in = (PROV_IDEA_CTX *)ctx; - PROV_IDEA_CTX *ret; - - if (!ossl_prov_is_running()) - return NULL; - - ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - *ret = *in; - - return ret; -} - -/* ossl_idea128ecb_functions */ -IMPLEMENT_generic_cipher(idea, IDEA, ecb, ECB, 0, 128, 64, 0, block) -/* ossl_idea128cbc_functions */ -IMPLEMENT_generic_cipher(idea, IDEA, cbc, CBC, 0, 128, 64, 64, block) -/* ossl_idea128ofb64_functions */ -IMPLEMENT_generic_cipher(idea, IDEA, ofb64, OFB, 0, 128, 8, 64, stream) -/* ossl_idea128cfb64_functions */ -IMPLEMENT_generic_cipher(idea, IDEA, cfb64, CFB, 0, 128, 8, 64, stream) diff --git a/openssl/src/providers/implementations/ciphers/cipher_idea.h b/openssl/src/providers/implementations/ciphers/cipher_idea.h deleted file mode 100644 index 212efa8af..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_idea.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "prov/ciphercommon.h" - -typedef struct prov_idea_ctx_st { - PROV_CIPHER_CTX base; /* Must be first */ - union { - OSSL_UNION_ALIGN; - IDEA_KEY_SCHEDULE ks; - } ks; -} PROV_IDEA_CTX; - -const PROV_CIPHER_HW *ossl_prov_cipher_hw_idea_cbc(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_idea_ecb(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_idea_ofb64(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_idea_cfb64(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_idea_hw.c b/openssl/src/providers/implementations/ciphers/cipher_idea_hw.c deleted file mode 100644 index 1c451b77e..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_idea_hw.c +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * IDEA low level APIs are deprecated for public use, but still ok for internal - * use where we're using them to implement the higher level EVP interface, as is - * the case here. - */ -#include "internal/deprecated.h" - -#include "cipher_idea.h" - -static int cipher_hw_idea_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, size_t keylen) -{ - PROV_IDEA_CTX *ictx = (PROV_IDEA_CTX *)ctx; - IDEA_KEY_SCHEDULE *ks = &(ictx->ks.ks); - - if (ctx->enc - || ctx->mode == EVP_CIPH_OFB_MODE - || ctx->mode == EVP_CIPH_CFB_MODE) { - IDEA_set_encrypt_key(key, ks); - } else { - IDEA_KEY_SCHEDULE tmp; - - IDEA_set_encrypt_key(key, &tmp); - IDEA_set_decrypt_key(&tmp, ks); - OPENSSL_cleanse((unsigned char *)&tmp, sizeof(IDEA_KEY_SCHEDULE)); - } - return 1; -} - -# define PROV_CIPHER_HW_idea_mode_ex(mode, UCMODE, fname) \ -IMPLEMENT_CIPHER_HW_##UCMODE(mode, idea, PROV_IDEA_CTX, IDEA_KEY_SCHEDULE, \ - fname) \ -static const PROV_CIPHER_HW idea_##mode = { \ - cipher_hw_idea_initkey, \ - cipher_hw_idea_##mode##_cipher \ -}; \ -const PROV_CIPHER_HW *ossl_prov_cipher_hw_idea_##mode(size_t keybits) \ -{ \ - return &idea_##mode; \ -} - -# define PROV_CIPHER_HW_idea_mode(mode, UCMODE) \ - PROV_CIPHER_HW_idea_mode_ex(mode, UCMODE, IDEA_##mode) - -PROV_CIPHER_HW_idea_mode(cbc, CBC) -PROV_CIPHER_HW_idea_mode(ofb64, OFB) -PROV_CIPHER_HW_idea_mode(cfb64, CFB) -/* - * IDEA_ecb_encrypt() does not have a enc parameter - so we create a macro - * that ignores this parameter when IMPLEMENT_CIPHER_HW_ecb() is called. - */ -#define IDEA2_ecb_encrypt(in, out, ks, enc) IDEA_ecb_encrypt(in, out, ks) - -PROV_CIPHER_HW_idea_mode_ex(ecb, ECB, IDEA2_ecb) diff --git a/openssl/src/providers/implementations/ciphers/cipher_null.c b/openssl/src/providers/implementations/ciphers/cipher_null.c index c911049e2..0df97a7f8 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_null.c +++ b/openssl/src/providers/implementations/ciphers/cipher_null.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -193,5 +193,5 @@ const OSSL_DISPATCH ossl_null_functions[] = { { OSSL_FUNC_CIPHER_SET_CTX_PARAMS, (void (*)(void))null_set_ctx_params }, { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, (void (*)(void))null_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/ciphers/cipher_rc2.c b/openssl/src/providers/implementations/ciphers/cipher_rc2.c deleted file mode 100644 index a4cd6bd53..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_rc2.c +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for RC2 cipher modes ecb, cbc, ofb, cfb */ - -/* - * RC2 low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include -#include "cipher_rc2.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -#define RC2_40_MAGIC 0xa0 -#define RC2_64_MAGIC 0x78 -#define RC2_128_MAGIC 0x3a -#define RC2_FLAGS PROV_CIPHER_FLAG_VARIABLE_LENGTH - -static OSSL_FUNC_cipher_encrypt_init_fn rc2_einit; -static OSSL_FUNC_cipher_decrypt_init_fn rc2_dinit; -static OSSL_FUNC_cipher_freectx_fn rc2_freectx; -static OSSL_FUNC_cipher_dupctx_fn rc2_dupctx; -static OSSL_FUNC_cipher_gettable_ctx_params_fn rc2_gettable_ctx_params; -static OSSL_FUNC_cipher_settable_ctx_params_fn rc2_settable_ctx_params; -static OSSL_FUNC_cipher_set_ctx_params_fn rc2_set_ctx_params; - -static void rc2_freectx(void *vctx) -{ - PROV_RC2_CTX *ctx = (PROV_RC2_CTX *)vctx; - - ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *rc2_dupctx(void *ctx) -{ - PROV_RC2_CTX *in = (PROV_RC2_CTX *)ctx; - PROV_RC2_CTX *ret; - - if (!ossl_prov_is_running()) - return NULL; - - ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - *ret = *in; - - return ret; -} - -static int rc2_keybits_to_magic(int keybits) -{ - switch (keybits) { - case 128: - return RC2_128_MAGIC; - case 64: - return RC2_64_MAGIC; - case 40: - return RC2_40_MAGIC; - } - ERR_raise(ERR_LIB_PROV, PROV_R_UNSUPPORTED_KEY_SIZE); - return 0; -} - -static int rc2_magic_to_keybits(int magic) -{ - switch (magic) { - case RC2_128_MAGIC: - return 128; - case RC2_64_MAGIC: - return 64; - case RC2_40_MAGIC: - return 40; - } - ERR_raise(ERR_LIB_PROV, PROV_R_UNSUPPORTED_KEY_SIZE); - return 0; -} - -static int rc2_einit(void *ctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) -{ - if (!ossl_cipher_generic_einit(ctx, key, keylen, iv, ivlen, NULL)) - return 0; - return rc2_set_ctx_params(ctx, params); -} - -static int rc2_dinit(void *ctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) -{ - if (!ossl_cipher_generic_dinit(ctx, key, keylen, iv, ivlen, NULL)) - return 0; - return rc2_set_ctx_params(ctx, params); -} - -static int rc2_get_ctx_params(void *vctx, OSSL_PARAM params[]) -{ - PROV_RC2_CTX *ctx = (PROV_RC2_CTX *)vctx; - OSSL_PARAM *p; - - if (!ossl_cipher_generic_get_ctx_params(vctx, params)) - return 0; - p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_RC2_KEYBITS); - if (p != NULL && !OSSL_PARAM_set_size_t(p, ctx->key_bits)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_ALGORITHM_ID_PARAMS); - if (p != NULL) { - long num; - int i; - ASN1_TYPE *type; - unsigned char *d = p->data; - unsigned char **dd = d == NULL ? NULL : &d; - - if (p->data_type != OSSL_PARAM_OCTET_STRING) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - if ((type = ASN1_TYPE_new()) == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); - return 0; - } - - /* Is this the original IV or the running IV? */ - num = rc2_keybits_to_magic(ctx->key_bits); - if (!ASN1_TYPE_set_int_octetstring(type, num, - ctx->base.iv, ctx->base.ivlen)) { - ASN1_TYPE_free(type); - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); - return 0; - } - /* - * IF the caller has a buffer, we pray to the gods they got the - * size right. There's no way to tell the i2d functions... - */ - i = i2d_ASN1_TYPE(type, dd); - if (i >= 0) - p->return_size = (size_t)i; - - ASN1_TYPE_free(type); - if (i < 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - } - return 1; -} - -static int rc2_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - PROV_RC2_CTX *ctx = (PROV_RC2_CTX *)vctx; - const OSSL_PARAM *p; - - if (params == NULL) - return 1; - - if (!ossl_cipher_var_keylen_set_ctx_params(vctx, params)) - return 0; - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_RC2_KEYBITS); - if (p != NULL) { - if (!OSSL_PARAM_get_size_t(p, &ctx->key_bits)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - } - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_ALGORITHM_ID_PARAMS); - if (p != NULL) { - ASN1_TYPE *type = NULL; - long num = 0; - const unsigned char *d = p->data; - int ret = 1; - unsigned char iv[16]; - - if (p->data_type != OSSL_PARAM_OCTET_STRING - || ctx->base.ivlen > sizeof(iv) - || (type = d2i_ASN1_TYPE(NULL, &d, p->data_size)) == NULL - || ((size_t)ASN1_TYPE_get_int_octetstring(type, &num, iv, - ctx->base.ivlen) - != ctx->base.ivlen) - || !ossl_cipher_generic_initiv(&ctx->base, iv, ctx->base.ivlen) - || (ctx->key_bits = rc2_magic_to_keybits(num)) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - ret = 0; - } - ASN1_TYPE_free(type); - if (ret == 0) - return 0; - /* - * This code assumes that the caller will call - * EVP_CipherInit_ex() with a non NULL key in order to setup a key that - * uses the keylen and keybits that were set here. - */ - ctx->base.keylen = ctx->key_bits / 8; - } - return 1; -} - -CIPHER_DEFAULT_GETTABLE_CTX_PARAMS_START(rc2) -OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_RC2_KEYBITS, NULL), -OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_ALGORITHM_ID_PARAMS, NULL, 0), -CIPHER_DEFAULT_GETTABLE_CTX_PARAMS_END(rc2) - -CIPHER_DEFAULT_SETTABLE_CTX_PARAMS_START(rc2) -OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_KEYLEN, NULL), -OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_RC2_KEYBITS, NULL), -OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_ALGORITHM_ID_PARAMS, NULL, 0), -CIPHER_DEFAULT_SETTABLE_CTX_PARAMS_END(rc2) - -#define IMPLEMENT_cipher(alg, UCALG, lcmode, UCMODE, flags, kbits, blkbits, \ - ivbits, typ) \ -static OSSL_FUNC_cipher_get_params_fn alg##_##kbits##_##lcmode##_get_params; \ -static int alg##_##kbits##_##lcmode##_get_params(OSSL_PARAM params[]) \ -{ \ - return ossl_cipher_generic_get_params(params, EVP_CIPH_##UCMODE##_MODE, \ - flags, kbits, blkbits, ivbits); \ -} \ -static OSSL_FUNC_cipher_newctx_fn alg##_##kbits##_##lcmode##_newctx; \ -static void *alg##_##kbits##_##lcmode##_newctx(void *provctx) \ -{ \ - PROV_##UCALG##_CTX *ctx; \ - if (!ossl_prov_is_running()) \ - return NULL; \ - ctx = OPENSSL_zalloc(sizeof(*ctx)); \ - if (ctx != NULL) { \ - ossl_cipher_generic_initkey(ctx, kbits, blkbits, ivbits, \ - EVP_CIPH_##UCMODE##_MODE, flags, \ - ossl_prov_cipher_hw_##alg##_##lcmode(kbits), \ - NULL); \ - ctx->key_bits = kbits; \ - } \ - return ctx; \ -} \ -const OSSL_DISPATCH ossl_##alg##kbits##lcmode##_functions[] = { \ - { OSSL_FUNC_CIPHER_NEWCTX, \ - (void (*)(void)) alg##_##kbits##_##lcmode##_newctx }, \ - { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void)) alg##_freectx }, \ - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void)) alg##_dupctx }, \ - { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))rc2_einit }, \ - { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))rc2_dinit }, \ - { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))ossl_cipher_generic_##typ##_update },\ - { OSSL_FUNC_CIPHER_FINAL, (void (*)(void))ossl_cipher_generic_##typ##_final }, \ - { OSSL_FUNC_CIPHER_CIPHER, (void (*)(void))ossl_cipher_generic_cipher }, \ - { OSSL_FUNC_CIPHER_GET_PARAMS, \ - (void (*)(void)) alg##_##kbits##_##lcmode##_get_params }, \ - { OSSL_FUNC_CIPHER_GETTABLE_PARAMS, \ - (void (*)(void))ossl_cipher_generic_gettable_params }, \ - { OSSL_FUNC_CIPHER_GET_CTX_PARAMS, \ - (void (*)(void))rc2_get_ctx_params }, \ - { OSSL_FUNC_CIPHER_GETTABLE_CTX_PARAMS, \ - (void (*)(void))rc2_gettable_ctx_params }, \ - { OSSL_FUNC_CIPHER_SET_CTX_PARAMS, \ - (void (*)(void))rc2_set_ctx_params }, \ - { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ - (void (*)(void))rc2_settable_ctx_params }, \ - OSSL_DISPATCH_END \ -}; - -/* ossl_rc2128ecb_functions */ -IMPLEMENT_cipher(rc2, RC2, ecb, ECB, RC2_FLAGS, 128, 64, 0, block) -/* ossl_rc2128cbc_functions */ -IMPLEMENT_cipher(rc2, RC2, cbc, CBC, RC2_FLAGS, 128, 64, 64, block) -/* ossl_rc240cbc_functions */ -IMPLEMENT_cipher(rc2, RC2, cbc, CBC, RC2_FLAGS, 40, 64, 64, block) -/* ossl_rc264cbc_functions */ -IMPLEMENT_cipher(rc2, RC2, cbc, CBC, RC2_FLAGS, 64, 64, 64, block) - -/* ossl_rc2128ofb128_functions */ -IMPLEMENT_cipher(rc2, RC2, ofb128, OFB, RC2_FLAGS, 128, 8, 64, stream) -/* ossl_rc2128cfb128_functions */ -IMPLEMENT_cipher(rc2, RC2, cfb128, CFB, RC2_FLAGS, 128, 8, 64, stream) diff --git a/openssl/src/providers/implementations/ciphers/cipher_rc2.h b/openssl/src/providers/implementations/ciphers/cipher_rc2.h deleted file mode 100644 index 7a4bea5ac..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_rc2.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "prov/ciphercommon.h" - -typedef struct prov_rc2_ctx_st { - PROV_CIPHER_CTX base; /* Must be first */ - union { - OSSL_UNION_ALIGN; - RC2_KEY ks; - } ks; - size_t key_bits; -} PROV_RC2_CTX; - -#define ossl_prov_cipher_hw_rc2_ofb128 ossl_prov_cipher_hw_rc2_ofb64 -#define ossl_prov_cipher_hw_rc2_cfb128 ossl_prov_cipher_hw_rc2_cfb64 - -const PROV_CIPHER_HW *ossl_prov_cipher_hw_rc2_cbc(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_rc2_ecb(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_rc2_ofb64(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_rc2_cfb64(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_rc2_hw.c b/openssl/src/providers/implementations/ciphers/cipher_rc2_hw.c deleted file mode 100644 index da9ff729c..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_rc2_hw.c +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RC2 low level APIs are deprecated for public use, but still ok for internal - * use. - */ -#include "internal/deprecated.h" - -#include "cipher_rc2.h" - -static int cipher_hw_rc2_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, size_t keylen) -{ - PROV_RC2_CTX *rctx = (PROV_RC2_CTX *)ctx; - RC2_KEY *ks = &(rctx->ks.ks); - - RC2_set_key(ks, (int)ctx->keylen, key, (int)rctx->key_bits); - return 1; -} - -# define PROV_CIPHER_HW_rc2_mode(mode, UCMODE) \ -IMPLEMENT_CIPHER_HW_##UCMODE(mode, rc2, PROV_RC2_CTX, RC2_KEY, \ - RC2_##mode) \ -static const PROV_CIPHER_HW rc2_##mode = { \ - cipher_hw_rc2_initkey, \ - cipher_hw_rc2_##mode##_cipher \ -}; \ -const PROV_CIPHER_HW *ossl_prov_cipher_hw_rc2_##mode(size_t keybits) \ -{ \ - return &rc2_##mode; \ -} - -PROV_CIPHER_HW_rc2_mode(cbc, CBC) -PROV_CIPHER_HW_rc2_mode(ecb, ECB) -PROV_CIPHER_HW_rc2_mode(ofb64, OFB) -PROV_CIPHER_HW_rc2_mode(cfb64, CFB) diff --git a/openssl/src/providers/implementations/ciphers/cipher_rc4.c b/openssl/src/providers/implementations/ciphers/cipher_rc4.c index 733524d36..a548beafa 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_rc4.c +++ b/openssl/src/providers/implementations/ciphers/cipher_rc4.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -43,8 +43,10 @@ static void *rc4_dupctx(void *ctx) return NULL; ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } *ret = *in; return ret; @@ -76,7 +78,7 @@ static int alg##_##kbits##_get_params(OSSL_PARAM params[]) \ kbits, blkbits, ivbits); \ } \ static OSSL_FUNC_cipher_newctx_fn alg##_##kbits##_newctx; \ -static void *alg##_##kbits##_newctx(void *provctx) \ +static void * alg##_##kbits##_newctx(void *provctx) \ { \ PROV_##UCALG##_CTX *ctx; \ if (!ossl_prov_is_running()) \ @@ -110,7 +112,7 @@ const OSSL_DISPATCH ossl_##alg##kbits##_functions[] = { \ (void (*)(void))ossl_cipher_generic_gettable_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))ossl_cipher_var_keylen_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; /* ossl_rc440_functions */ diff --git a/openssl/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c b/openssl/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c index 82ef7890b..c46c6eab6 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c +++ b/openssl/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -34,7 +34,6 @@ static OSSL_FUNC_cipher_encrypt_init_fn rc4_hmac_md5_einit; static OSSL_FUNC_cipher_decrypt_init_fn rc4_hmac_md5_dinit; static OSSL_FUNC_cipher_newctx_fn rc4_hmac_md5_newctx; static OSSL_FUNC_cipher_freectx_fn rc4_hmac_md5_freectx; -static OSSL_FUNC_cipher_dupctx_fn rc4_hmac_md5_dupctx; static OSSL_FUNC_cipher_get_ctx_params_fn rc4_hmac_md5_get_ctx_params; static OSSL_FUNC_cipher_gettable_ctx_params_fn rc4_hmac_md5_gettable_ctx_params; static OSSL_FUNC_cipher_set_ctx_params_fn rc4_hmac_md5_set_ctx_params; @@ -72,15 +71,6 @@ static void rc4_hmac_md5_freectx(void *vctx) OPENSSL_clear_free(ctx, sizeof(*ctx)); } -static void *rc4_hmac_md5_dupctx(void *vctx) -{ - PROV_RC4_HMAC_MD5_CTX *ctx = vctx; - - if (ctx == NULL) - return NULL; - return OPENSSL_memdup(ctx, sizeof(*ctx)); -} - static int rc4_hmac_md5_einit(void *ctx, const unsigned char *key, size_t keylen, const unsigned char *iv, size_t ivlen, const OSSL_PARAM params[]) @@ -224,7 +214,6 @@ static int rc4_hmac_md5_get_params(OSSL_PARAM params[]) const OSSL_DISPATCH ossl_rc4_hmac_ossl_md5_functions[] = { { OSSL_FUNC_CIPHER_NEWCTX, (void (*)(void))rc4_hmac_md5_newctx }, { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))rc4_hmac_md5_freectx }, - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void))rc4_hmac_md5_dupctx }, { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))rc4_hmac_md5_einit }, { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))rc4_hmac_md5_dinit }, { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))rc4_hmac_md5_update }, @@ -241,5 +230,5 @@ const OSSL_DISPATCH ossl_rc4_hmac_ossl_md5_functions[] = { (void (*)(void))rc4_hmac_md5_set_ctx_params }, { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, (void (*)(void))rc4_hmac_md5_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.h b/openssl/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.h index 4a1d154a7..1697aabbf 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.h +++ b/openssl/src/providers/implementations/ciphers/cipher_rc4_hmac_md5.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -31,6 +31,3 @@ typedef struct prov_cipher_hw_rc4_hmac_md5_st { } PROV_CIPHER_HW_RC4_HMAC_MD5; const PROV_CIPHER_HW *ossl_prov_cipher_hw_rc4_hmac_md5(size_t keybits); - -void rc4_md5_enc(RC4_KEY *key, const void *in0, void *out, - MD5_CTX *ctx, const void *inp, size_t blocks); diff --git a/openssl/src/providers/implementations/ciphers/cipher_rc5.c b/openssl/src/providers/implementations/ciphers/cipher_rc5.c index 090b0488e..5c7d2b172 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_rc5.c +++ b/openssl/src/providers/implementations/ciphers/cipher_rc5.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -47,8 +47,10 @@ static void *rc5_dupctx(void *ctx) return NULL; ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } *ret = *in; return ret; @@ -136,7 +138,7 @@ static int alg##_##kbits##_##lcmode##_get_params(OSSL_PARAM params[]) \ flags, kbits, blkbits, ivbits); \ } \ static OSSL_FUNC_cipher_newctx_fn alg##_##kbits##_##lcmode##_newctx; \ -static void *alg##_##kbits##_##lcmode##_newctx(void *provctx) \ +static void * alg##_##kbits##_##lcmode##_newctx(void *provctx) \ { \ PROV_##UCALG##_CTX *ctx; \ if (!ossl_prov_is_running()) \ @@ -173,7 +175,7 @@ const OSSL_DISPATCH ossl_##alg##kbits##lcmode##_functions[] = { \ (void (*)(void))rc5_set_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))rc5_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; /* ossl_rc5128ecb_functions */ diff --git a/openssl/src/providers/implementations/ciphers/cipher_seed.c b/openssl/src/providers/implementations/ciphers/cipher_seed.c deleted file mode 100644 index 3644cb5e2..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_seed.c +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for Seed cipher modes ecb, cbc, ofb, cfb */ - -/* - * SEED low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include "cipher_seed.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -static OSSL_FUNC_cipher_freectx_fn seed_freectx; -static OSSL_FUNC_cipher_dupctx_fn seed_dupctx; - -static void seed_freectx(void *vctx) -{ - PROV_SEED_CTX *ctx = (PROV_SEED_CTX *)vctx; - - ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *seed_dupctx(void *ctx) -{ - PROV_SEED_CTX *in = (PROV_SEED_CTX *)ctx; - PROV_SEED_CTX *ret; - - if (!ossl_prov_is_running()) - return NULL; - - ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - *ret = *in; - - return ret; -} - -/* ossl_seed128ecb_functions */ -IMPLEMENT_generic_cipher(seed, SEED, ecb, ECB, 0, 128, 128, 0, block) -/* ossl_seed128cbc_functions */ -IMPLEMENT_generic_cipher(seed, SEED, cbc, CBC, 0, 128, 128, 128, block) -/* ossl_seed128ofb128_functions */ -IMPLEMENT_generic_cipher(seed, SEED, ofb128, OFB, 0, 128, 8, 128, stream) -/* ossl_seed128cfb128_functions */ -IMPLEMENT_generic_cipher(seed, SEED, cfb128, CFB, 0, 128, 8, 128, stream) diff --git a/openssl/src/providers/implementations/ciphers/cipher_seed.h b/openssl/src/providers/implementations/ciphers/cipher_seed.h deleted file mode 100644 index 9006a9183..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_seed.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "prov/ciphercommon.h" - -typedef struct prov_seed_ctx_st { - PROV_CIPHER_CTX base; /* Must be first */ - union { - OSSL_UNION_ALIGN; - SEED_KEY_SCHEDULE ks; - } ks; -} PROV_SEED_CTX; - -const PROV_CIPHER_HW *ossl_prov_cipher_hw_seed_cbc(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_seed_ecb(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_seed_ofb128(size_t keybits); -const PROV_CIPHER_HW *ossl_prov_cipher_hw_seed_cfb128(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_seed_hw.c b/openssl/src/providers/implementations/ciphers/cipher_seed_hw.c deleted file mode 100644 index 2d1dba92b..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_seed_hw.c +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * SEED low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include "cipher_seed.h" - -static int cipher_hw_seed_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, size_t keylen) -{ - PROV_SEED_CTX *sctx = (PROV_SEED_CTX *)ctx; - - SEED_set_key(key, &(sctx->ks.ks)); - return 1; -} - -# define PROV_CIPHER_HW_seed_mode(mode, UCMODE) \ -IMPLEMENT_CIPHER_HW_##UCMODE(mode, seed, PROV_SEED_CTX, SEED_KEY_SCHEDULE, \ - SEED_##mode) \ -static const PROV_CIPHER_HW seed_##mode = { \ - cipher_hw_seed_initkey, \ - cipher_hw_seed_##mode##_cipher \ -}; \ -const PROV_CIPHER_HW *ossl_prov_cipher_hw_seed_##mode(size_t keybits) \ -{ \ - return &seed_##mode; \ -} - -PROV_CIPHER_HW_seed_mode(cbc, CBC) -PROV_CIPHER_HW_seed_mode(ecb, ECB) -PROV_CIPHER_HW_seed_mode(ofb128, OFB) -PROV_CIPHER_HW_seed_mode(cfb128, CFB) diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4.c b/openssl/src/providers/implementations/ciphers/cipher_sm4.c index 863c9997f..6cf2731c6 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4.c +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4.c @@ -33,8 +33,10 @@ static void *sm4_dupctx(void *ctx) return NULL; ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } in->base.hw->copyctx(&ret->base, &in->base); return ret; diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4.h b/openssl/src/providers/implementations/ciphers/cipher_sm4.h index 9ab49e327..01a031a74 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4.h +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm.c b/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm.c index 3af84d85b..8582a8be1 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm.c +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm.c @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,14 +7,14 @@ * https://www.openssl.org/source/license.html */ +#include "internal/deprecated.h" + /* Dispatch functions for SM4 CCM mode */ #include "cipher_sm4_ccm.h" #include "prov/implementations.h" #include "prov/providercommon.h" -static OSSL_FUNC_cipher_freectx_fn sm4_ccm_freectx; - static void *sm4_ccm_newctx(void *provctx, size_t keybits) { PROV_SM4_CCM_CTX *ctx; @@ -28,21 +28,7 @@ static void *sm4_ccm_newctx(void *provctx, size_t keybits) return ctx; } -static void *sm4_ccm_dupctx(void *provctx) -{ - PROV_SM4_CCM_CTX *ctx = provctx; - PROV_SM4_CCM_CTX *dctx = NULL; - - if (ctx == NULL) - return NULL; - - dctx = OPENSSL_memdup(ctx, sizeof(*ctx)); - if (dctx != NULL && dctx->base.ccm_ctx.key != NULL) - dctx->base.ccm_ctx.key = &dctx->ks.ks; - - return dctx; -} - +static OSSL_FUNC_cipher_freectx_fn sm4_ccm_freectx; static void sm4_ccm_freectx(void *vctx) { PROV_SM4_CCM_CTX *ctx = (PROV_SM4_CCM_CTX *)vctx; @@ -50,5 +36,5 @@ static void sm4_ccm_freectx(void *vctx) OPENSSL_clear_free(ctx, sizeof(*ctx)); } -/* sm4128ccm functions */ +/* ossl_sm4128ccm_functions */ IMPLEMENT_aead_cipher(sm4, ccm, CCM, AEAD_FLAGS, 128, 8, 96); diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm.h b/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm.h index 561437993..15bacbae9 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm.h +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm.h @@ -7,17 +7,16 @@ * https://www.openssl.org/source/license.html */ -#include "crypto/sm4.h" +#include #include "prov/ciphercommon.h" #include "prov/ciphercommon_ccm.h" -#include "crypto/sm4_platform.h" typedef struct prov_sm4_ccm_ctx_st { - PROV_CCM_CTX base; /* Must be first */ + PROV_CCM_CTX base; /* must be first entry in struct */ union { OSSL_UNION_ALIGN; SM4_KEY ks; } ks; /* SM4 key schedule to use */ } PROV_SM4_CCM_CTX; -const PROV_CCM_HW *ossl_prov_sm4_hw_ccm(size_t keylen); +const PROV_CCM_HW *ossl_prov_sm4_hw_ccm(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c b/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c index 34f0e751e..cddb93ee7 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,51 +7,44 @@ * https://www.openssl.org/source/license.html */ -/*- - * Generic support for SM4 CCM. +/* Dispatch functions for SM4 CCM mode */ + +/* + * This file uses the low level SM4 functions (which are deprecated for + * non-internal use) in order to implement provider SM4 ciphers. */ +#include "internal/deprecated.h" #include "cipher_sm4_ccm.h" #include "crypto/sm4_platform.h" -#define SM4_HW_CCM_SET_KEY_FN(fn_set_enc_key, fn_blk, fn_ccm_enc, fn_ccm_dec) \ - fn_set_enc_key(key, &actx->ks.ks); \ - CRYPTO_ccm128_init(&ctx->ccm_ctx, ctx->m, ctx->l, &actx->ks.ks, \ - (block128_f)fn_blk); \ - ctx->str = ctx->enc ? (ccm128_f)fn_ccm_enc : (ccm128_f)fn_ccm_dec; \ - ctx->key_set = 1; - -static int ccm_sm4_initkey(PROV_CCM_CTX *ctx, - const unsigned char *key, size_t keylen) +static int sm4_ccm_initkey(PROV_CCM_CTX *ctx, const unsigned char *key, + size_t keylen) { PROV_SM4_CCM_CTX *actx = (PROV_SM4_CCM_CTX *)ctx; + SM4_KEY *ks = &actx->ks.ks; -#ifdef HWSM4_CAPABLE +# ifdef HWSM4_CAPABLE if (HWSM4_CAPABLE) { - SM4_HW_CCM_SET_KEY_FN(HWSM4_set_encrypt_key, HWSM4_encrypt, NULL, NULL); - } else -#endif /* HWSM4_CAPABLE */ - -#ifdef VPSM4_EX_CAPABLE - if (VPSM4_EX_CAPABLE) { - SM4_HW_CCM_SET_KEY_FN(vpsm4_ex_set_encrypt_key, vpsm4_ex_encrypt, NULL, - NULL); + HWSM4_set_encrypt_key(key, ks); + CRYPTO_ccm128_init(&ctx->ccm_ctx, ctx->m, ctx->l, &actx->ks.ks, + (block128_f) HWSM4_encrypt); + ctx->str = (ccm128_f)NULL; } else -#endif /* VPSM4_EX_CAPABLE */ - -#ifdef VPSM4_CAPABLE - if (VPSM4_CAPABLE) { - SM4_HW_CCM_SET_KEY_FN(vpsm4_set_encrypt_key, vpsm4_encrypt, NULL, NULL); - } else -#endif /* VPSM4_CAPABLE */ +# endif /* HWSM4_CAPABLE */ { - SM4_HW_CCM_SET_KEY_FN(ossl_sm4_set_key, ossl_sm4_encrypt, NULL, NULL); + ossl_sm4_set_key(key, ks); + CRYPTO_ccm128_init(&ctx->ccm_ctx, ctx->m, ctx->l, &actx->ks.ks, + (block128_f) ossl_sm4_encrypt); + ctx->str = (ccm128_f)NULL; } - return 1; + ctx->key_set = 1; + + return 1; } -static const PROV_CCM_HW ccm_sm4 = { - ccm_sm4_initkey, +static const PROV_CCM_HW sm4_ccm = { + sm4_ccm_initkey, ossl_ccm_generic_setiv, ossl_ccm_generic_setaad, ossl_ccm_generic_auth_encrypt, @@ -59,11 +52,8 @@ static const PROV_CCM_HW ccm_sm4 = { ossl_ccm_generic_gettag }; -#if defined(__riscv) && __riscv_xlen == 64 -# include "cipher_sm4_ccm_hw_rv64i.inc" -#else + const PROV_CCM_HW *ossl_prov_sm4_hw_ccm(size_t keybits) { - return &ccm_sm4; + return &sm4_ccm; } -#endif diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm_hw_rv64i.inc b/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm_hw_rv64i.inc deleted file mode 100644 index c20c9f688..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_ccm_hw_rv64i.inc +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RV64I ZVKSED support for SM4 CCM. - * This file is included by cipher_sm4_ccm_hw.c - */ - -static int rv64i_zvksed_sm4_ccm_initkey(PROV_CCM_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_SM4_CCM_CTX *actx = (PROV_SM4_CCM_CTX *)ctx; - - SM4_HW_CCM_SET_KEY_FN(rv64i_zvksed_sm4_set_encrypt_key, - rv64i_zvksed_sm4_encrypt, NULL, NULL); - return 1; -} - -static const PROV_CCM_HW rv64i_zvksed_sm4_ccm = { - rv64i_zvksed_sm4_ccm_initkey, - ossl_ccm_generic_setiv, - ossl_ccm_generic_setaad, - ossl_ccm_generic_auth_encrypt, - ossl_ccm_generic_auth_decrypt, - ossl_ccm_generic_gettag -}; - -const PROV_CCM_HW *ossl_prov_sm4_hw_ccm(size_t keybits) -{ - if (RISCV_HAS_ZVKB_AND_ZVKSED() && riscv_vlen() >= 128) - return &rv64i_zvksed_sm4_ccm; - else - return &ccm_sm4; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm.c b/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm.c index 1128f6593..9b574fb5f 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm.c +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm.c @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,14 +7,19 @@ * https://www.openssl.org/source/license.html */ -/* Dispatch functions for SM4 GCM mode */ +/* + * AES low level APIs are deprecated for public use, but still ok for internal + * use where we're using them to implement the higher level EVP interface, as is + * the case here. + */ +#include "internal/deprecated.h" + +/* Dispatch functions for AES GCM mode */ #include "cipher_sm4_gcm.h" #include "prov/implementations.h" #include "prov/providercommon.h" -static OSSL_FUNC_cipher_freectx_fn sm4_gcm_freectx; - static void *sm4_gcm_newctx(void *provctx, size_t keybits) { PROV_SM4_GCM_CTX *ctx; @@ -29,21 +34,7 @@ static void *sm4_gcm_newctx(void *provctx, size_t keybits) return ctx; } -static void *sm4_gcm_dupctx(void *provctx) -{ - PROV_SM4_GCM_CTX *ctx = provctx; - PROV_SM4_GCM_CTX *dctx = NULL; - - if (ctx == NULL) - return NULL; - - dctx = OPENSSL_memdup(ctx, sizeof(*ctx)); - if (dctx != NULL && dctx->base.gcm.key != NULL) - dctx->base.gcm.key = &dctx->ks.ks; - - return dctx; -} - +static OSSL_FUNC_cipher_freectx_fn sm4_gcm_freectx; static void sm4_gcm_freectx(void *vctx) { PROV_SM4_GCM_CTX *ctx = (PROV_SM4_GCM_CTX *)vctx; diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm.h b/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm.h index 2b6b5f3ec..3754286cc 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm.h +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm.h @@ -7,16 +7,16 @@ * https://www.openssl.org/source/license.html */ -#include "crypto/sm4.h" +#include #include "prov/ciphercommon.h" #include "prov/ciphercommon_gcm.h" typedef struct prov_sm4_gcm_ctx_st { - PROV_GCM_CTX base; /* must be first entry in struct */ + PROV_GCM_CTX base; /* must be first entry in struct */ union { OSSL_UNION_ALIGN; SM4_KEY ks; - } ks; + } ks; /* SM4 key schedule to use */ } PROV_SM4_GCM_CTX; const PROV_GCM_HW *ossl_prov_sm4_hw_gcm(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c b/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c index 06ca45078..b134985fa 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,58 +7,48 @@ * https://www.openssl.org/source/license.html */ -/*- - * Generic support for SM4 GCM. +/* Dispatch functions for SM4 GCM mode */ + +/* + * This file uses the low level SM4 functions (which are deprecated for + * non-internal use) in order to implement provider SM4 ciphers. */ +#include "internal/deprecated.h" #include "cipher_sm4_gcm.h" #include "crypto/sm4_platform.h" -# define SM4_GCM_HW_SET_KEY_CTR_FN(ks, fn_set_enc_key, fn_block, fn_ctr) \ - fn_set_enc_key(key, ks); \ - CRYPTO_gcm128_init(&ctx->gcm, ks, (block128_f)fn_block); \ - ctx->ctr = (ctr128_f)fn_ctr; \ - ctx->key_set = 1; - static int sm4_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, size_t keylen) { PROV_SM4_GCM_CTX *actx = (PROV_SM4_GCM_CTX *)ctx; SM4_KEY *ks = &actx->ks.ks; + ctx->ks = ks; # ifdef HWSM4_CAPABLE if (HWSM4_CAPABLE) { + HWSM4_set_encrypt_key(key, ks); + CRYPTO_gcm128_init(&ctx->gcm, ks, (block128_f) HWSM4_encrypt); # ifdef HWSM4_ctr32_encrypt_blocks - SM4_GCM_HW_SET_KEY_CTR_FN(ks, HWSM4_set_encrypt_key, HWSM4_encrypt, - HWSM4_ctr32_encrypt_blocks); + ctx->ctr = (ctr128_f) HWSM4_ctr32_encrypt_blocks; # else /* HWSM4_ctr32_encrypt_blocks */ - SM4_GCM_HW_SET_KEY_CTR_FN(ks, HWSM4_set_encrypt_key, HWSM4_encrypt, NULL); + ctx->ctr = (ctr128_f)NULL; # endif } else # endif /* HWSM4_CAPABLE */ - -#ifdef VPSM4_EX_CAPABLE - if (VPSM4_EX_CAPABLE) { - SM4_GCM_HW_SET_KEY_CTR_FN(ks, vpsm4_ex_set_encrypt_key, vpsm4_ex_encrypt, - vpsm4_ex_ctr32_encrypt_blocks); - } else -#endif /* VPSM4_EX_CAPABLE */ - -# ifdef VPSM4_CAPABLE - if (VPSM4_CAPABLE) { - SM4_GCM_HW_SET_KEY_CTR_FN(ks, vpsm4_set_encrypt_key, vpsm4_encrypt, - vpsm4_ctr32_encrypt_blocks); - } else -# endif /* VPSM4_CAPABLE */ { - SM4_GCM_HW_SET_KEY_CTR_FN(ks, ossl_sm4_set_key, ossl_sm4_encrypt, NULL); + ossl_sm4_set_key(key, ks); + CRYPTO_gcm128_init(&ctx->gcm, ks, (block128_f)ossl_sm4_encrypt); + ctx->ctr = (ctr128_f)NULL; } + ctx->key_set = 1; return 1; } -static int hw_gcm_cipher_update(PROV_GCM_CTX *ctx, const unsigned char *in, - size_t len, unsigned char *out) +static int generic_sm4_gcm_cipher_update(PROV_GCM_CTX *ctx, + const unsigned char *in, + size_t len, unsigned char *out) { if (ctx->enc) { if (ctx->ctr != NULL) { @@ -84,16 +74,13 @@ static const PROV_GCM_HW sm4_gcm = { sm4_gcm_initkey, ossl_gcm_setiv, ossl_gcm_aad_update, - hw_gcm_cipher_update, + generic_sm4_gcm_cipher_update, ossl_gcm_cipher_final, ossl_gcm_one_shot }; -#if defined(__riscv) && __riscv_xlen == 64 -# include "cipher_sm4_gcm_hw_rv64i.inc" -#else + const PROV_GCM_HW *ossl_prov_sm4_hw_gcm(size_t keybits) { return &sm4_gcm; } -#endif diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm_hw_rv64i.inc b/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm_hw_rv64i.inc deleted file mode 100644 index 109d13b43..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_gcm_hw_rv64i.inc +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RISC-V 64 ZVKSED support for SM4 GCM. - * This file is included by cipher_sm4_gcm_hw.c - */ - -static int rv64i_zvksed_sm4_gcm_initkey(PROV_GCM_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_SM4_GCM_CTX *actx = (PROV_SM4_GCM_CTX *)ctx; - SM4_KEY *ks = &actx->ks.ks; - - SM4_GCM_HW_SET_KEY_CTR_FN(ks, rv64i_zvksed_sm4_set_encrypt_key, - rv64i_zvksed_sm4_encrypt, NULL); - return 1; -} - -static const PROV_GCM_HW rv64i_zvksed_sm4_gcm = { - rv64i_zvksed_sm4_gcm_initkey, - ossl_gcm_setiv, - ossl_gcm_aad_update, - hw_gcm_cipher_update, - ossl_gcm_cipher_final, - ossl_gcm_one_shot -}; - -const PROV_GCM_HW *ossl_prov_sm4_hw_gcm(size_t keybits) -{ - if (RISCV_HAS_ZVKB_AND_ZVKSED() && riscv_vlen() >= 128) - return &rv64i_zvksed_sm4_gcm; - else - return &sm4_gcm; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_hw.c b/openssl/src/providers/implementations/ciphers/cipher_sm4_hw.c index c4f2f97cc..4cd3d3d66 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_sm4_hw.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -41,32 +41,6 @@ static int cipher_hw_sm4_initkey(PROV_CIPHER_CTX *ctx, #endif (void)0; /* terminate potentially open 'else' */ } else -#endif -#ifdef VPSM4_EX_CAPABLE - if (VPSM4_EX_CAPABLE) { - vpsm4_ex_set_encrypt_key(key, ks); - ctx->block = (block128_f)vpsm4_ex_encrypt; - ctx->stream.cbc = NULL; - if (ctx->mode == EVP_CIPH_CBC_MODE) - ctx->stream.cbc = (cbc128_f)vpsm4_ex_cbc_encrypt; - else if (ctx->mode == EVP_CIPH_ECB_MODE) - ctx->stream.ecb = (ecb128_f)vpsm4_ex_ecb_encrypt; - else if (ctx->mode == EVP_CIPH_CTR_MODE) - ctx->stream.ctr = (ctr128_f)vpsm4_ex_ctr32_encrypt_blocks; - } else -#endif -#ifdef VPSM4_CAPABLE - if (VPSM4_CAPABLE) { - vpsm4_set_encrypt_key(key, ks); - ctx->block = (block128_f)vpsm4_encrypt; - ctx->stream.cbc = NULL; - if (ctx->mode == EVP_CIPH_CBC_MODE) - ctx->stream.cbc = (cbc128_f)vpsm4_cbc_encrypt; - else if (ctx->mode == EVP_CIPH_ECB_MODE) - ctx->stream.ecb = (ecb128_f)vpsm4_ecb_encrypt; - else if (ctx->mode == EVP_CIPH_CTR_MODE) - ctx->stream.ctr = (ctr128_f)vpsm4_ctr32_encrypt_blocks; - } else #endif { ossl_sm4_set_key(key, ks); @@ -87,28 +61,6 @@ static int cipher_hw_sm4_initkey(PROV_CIPHER_CTX *ctx, ctx->stream.ecb = (ecb128_f)HWSM4_ecb_encrypt; #endif } else -#endif -#ifdef VPSM4_EX_CAPABLE - if (VPSM4_EX_CAPABLE) { - vpsm4_ex_set_decrypt_key(key, ks); - ctx->block = (block128_f)vpsm4_ex_decrypt; - ctx->stream.cbc = NULL; - if (ctx->mode == EVP_CIPH_CBC_MODE) - ctx->stream.cbc = (cbc128_f)vpsm4_ex_cbc_encrypt; - else if (ctx->mode == EVP_CIPH_ECB_MODE) - ctx->stream.ecb = (ecb128_f)vpsm4_ex_ecb_encrypt; - } else -#endif -#ifdef VPSM4_CAPABLE - if (VPSM4_CAPABLE) { - vpsm4_set_decrypt_key(key, ks); - ctx->block = (block128_f)vpsm4_decrypt; - ctx->stream.cbc = NULL; - if (ctx->mode == EVP_CIPH_CBC_MODE) - ctx->stream.cbc = (cbc128_f)vpsm4_cbc_encrypt; - else if (ctx->mode == EVP_CIPH_ECB_MODE) - ctx->stream.ecb = (ecb128_f)vpsm4_ecb_encrypt; - } else #endif { ossl_sm4_set_key(key, ks); @@ -127,21 +79,11 @@ static const PROV_CIPHER_HW sm4_##mode = { \ ossl_cipher_hw_generic_##mode, \ cipher_hw_sm4_copyctx \ }; \ -PROV_CIPHER_HW_declare(mode) \ const PROV_CIPHER_HW *ossl_prov_cipher_hw_sm4_##mode(size_t keybits) \ { \ - PROV_CIPHER_HW_select(mode) \ return &sm4_##mode; \ } -#if defined(__riscv) && __riscv_xlen == 64 -# include "cipher_sm4_hw_rv64i.inc" -#else -/* The generic case */ -# define PROV_CIPHER_HW_declare(mode) -# define PROV_CIPHER_HW_select(mode) -#endif - PROV_CIPHER_HW_sm4_mode(cbc) PROV_CIPHER_HW_sm4_mode(ecb) PROV_CIPHER_HW_sm4_mode(ofb128) diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_hw_rv64i.inc b/openssl/src/providers/implementations/ciphers/cipher_sm4_hw_rv64i.inc deleted file mode 100644 index 763d9d09d..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_hw_rv64i.inc +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RV64 ZVKSED support for AES modes ecb, cbc, ofb, cfb, ctr. - * This file is included by cipher_sm4_hw.c - */ - -#define cipher_hw_rv64i_zvksed_sm4_cbc ossl_cipher_hw_generic_cbc -#define cipher_hw_rv64i_zvksed_sm4_ecb ossl_cipher_hw_generic_ecb -#define cipher_hw_rv64i_zvksed_sm4_ofb128 ossl_cipher_hw_generic_ofb128 -#define cipher_hw_rv64i_zvksed_sm4_cfb128 ossl_cipher_hw_generic_cfb128 -#define cipher_hw_rv64i_zvksed_sm4_ctr ossl_cipher_hw_generic_ctr - -static int cipher_hw_rv64i_zvksed_sm4_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_SM4_CTX *sctx = (PROV_SM4_CTX *)ctx; - SM4_KEY *ks = &sctx->ks.ks; - - ctx->ks = ks; - if (ctx->enc - || (ctx->mode != EVP_CIPH_ECB_MODE - && ctx->mode != EVP_CIPH_CBC_MODE)) { - rv64i_zvksed_sm4_set_encrypt_key(key, ks); - ctx->block = (block128_f) rv64i_zvksed_sm4_encrypt; - ctx->stream.cbc = NULL; - } else { - rv64i_zvksed_sm4_set_decrypt_key(key, ks); - ctx->block = (block128_f) rv64i_zvksed_sm4_decrypt; - ctx->stream.cbc = NULL; - } - - return 1; -} - -#define PROV_CIPHER_HW_declare(mode) \ -static const PROV_CIPHER_HW rv64i_zvksed_sm4_##mode = { \ - cipher_hw_rv64i_zvksed_sm4_initkey, \ - cipher_hw_rv64i_zvksed_sm4_##mode, \ - cipher_hw_sm4_copyctx \ -}; -#define PROV_CIPHER_HW_select(mode) \ -if (RISCV_HAS_ZVKB_AND_ZVKSED() && riscv_vlen() >= 128) \ - return &rv64i_zvksed_sm4_##mode; diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_xts.c b/openssl/src/providers/implementations/ciphers/cipher_sm4_xts.c deleted file mode 100644 index e8c28e266..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_xts.c +++ /dev/null @@ -1,281 +0,0 @@ - -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Dispatch functions for SM4 XTS mode */ - -#include -#include "cipher_sm4_xts.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -#define SM4_XTS_FLAGS PROV_CIPHER_FLAG_CUSTOM_IV -#define SM4_XTS_IV_BITS 128 -#define SM4_XTS_BLOCK_BITS 8 - -/* forward declarations */ -static OSSL_FUNC_cipher_encrypt_init_fn sm4_xts_einit; -static OSSL_FUNC_cipher_decrypt_init_fn sm4_xts_dinit; -static OSSL_FUNC_cipher_update_fn sm4_xts_stream_update; -static OSSL_FUNC_cipher_final_fn sm4_xts_stream_final; -static OSSL_FUNC_cipher_cipher_fn sm4_xts_cipher; -static OSSL_FUNC_cipher_freectx_fn sm4_xts_freectx; -static OSSL_FUNC_cipher_dupctx_fn sm4_xts_dupctx; -static OSSL_FUNC_cipher_set_ctx_params_fn sm4_xts_set_ctx_params; -static OSSL_FUNC_cipher_settable_ctx_params_fn sm4_xts_settable_ctx_params; - -/*- - * Provider dispatch functions - */ -static int sm4_xts_init(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[], int enc) -{ - PROV_SM4_XTS_CTX *xctx = (PROV_SM4_XTS_CTX *)vctx; - PROV_CIPHER_CTX *ctx = &xctx->base; - - if (!ossl_prov_is_running()) - return 0; - - ctx->enc = enc; - - if (iv != NULL) { - if (!ossl_cipher_generic_initiv(vctx, iv, ivlen)) - return 0; - } - if (key != NULL) { - if (keylen != ctx->keylen) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); - return 0; - } - if (!ctx->hw->init(ctx, key, keylen)) - return 0; - } - return sm4_xts_set_ctx_params(xctx, params); -} - -static int sm4_xts_einit(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) -{ - return sm4_xts_init(vctx, key, keylen, iv, ivlen, params, 1); -} - -static int sm4_xts_dinit(void *vctx, const unsigned char *key, size_t keylen, - const unsigned char *iv, size_t ivlen, - const OSSL_PARAM params[]) -{ - return sm4_xts_init(vctx, key, keylen, iv, ivlen, params, 0); -} - -static void *sm4_xts_newctx(void *provctx, unsigned int mode, uint64_t flags, - size_t kbits, size_t blkbits, size_t ivbits) -{ - PROV_SM4_XTS_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx)); - - if (ctx != NULL) { - ossl_cipher_generic_initkey(&ctx->base, kbits, blkbits, ivbits, mode, - flags, ossl_prov_cipher_hw_sm4_xts(kbits), - NULL); - } - return ctx; -} - -static void sm4_xts_freectx(void *vctx) -{ - PROV_SM4_XTS_CTX *ctx = (PROV_SM4_XTS_CTX *)vctx; - - ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); - OPENSSL_clear_free(ctx, sizeof(*ctx)); -} - -static void *sm4_xts_dupctx(void *vctx) -{ - PROV_SM4_XTS_CTX *in = (PROV_SM4_XTS_CTX *)vctx; - PROV_SM4_XTS_CTX *ret = NULL; - - if (!ossl_prov_is_running()) - return NULL; - - if (in->xts.key1 != NULL) { - if (in->xts.key1 != &in->ks1) - return NULL; - } - if (in->xts.key2 != NULL) { - if (in->xts.key2 != &in->ks2) - return NULL; - } - ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) - return NULL; - in->base.hw->copyctx(&ret->base, &in->base); - return ret; -} - -static int sm4_xts_cipher(void *vctx, unsigned char *out, size_t *outl, - size_t outsize, const unsigned char *in, size_t inl) -{ - PROV_SM4_XTS_CTX *ctx = (PROV_SM4_XTS_CTX *)vctx; - - if (!ossl_prov_is_running() - || ctx->xts.key1 == NULL - || ctx->xts.key2 == NULL - || !ctx->base.iv_set - || out == NULL - || in == NULL - || inl < SM4_BLOCK_SIZE) - return 0; - - /* - * Impose a limit of 2^20 blocks per data unit as specified by - * IEEE Std 1619-2018. The earlier and obsolete IEEE Std 1619-2007 - * indicated that this was a SHOULD NOT rather than a MUST NOT. - * NIST SP 800-38E mandates the same limit. - */ - if (inl > XTS_MAX_BLOCKS_PER_DATA_UNIT * SM4_BLOCK_SIZE) { - ERR_raise(ERR_LIB_PROV, PROV_R_XTS_DATA_UNIT_IS_TOO_LARGE); - return 0; - } - if (ctx->xts_standard) { - if (ctx->stream != NULL) - (*ctx->stream)(in, out, inl, ctx->xts.key1, ctx->xts.key2, - ctx->base.iv, ctx->base.enc); - else if (CRYPTO_xts128_encrypt(&ctx->xts, ctx->base.iv, in, out, inl, - ctx->base.enc)) - return 0; - } else { - if (ctx->stream_gb != NULL) - (*ctx->stream_gb)(in, out, inl, ctx->xts.key1, ctx->xts.key2, - ctx->base.iv, ctx->base.enc); - else if (ossl_crypto_xts128gb_encrypt(&ctx->xts, ctx->base.iv, in, out, - inl, ctx->base.enc)) - return 0; - } - *outl = inl; - return 1; -} - -static int sm4_xts_stream_update(void *vctx, unsigned char *out, size_t *outl, - size_t outsize, const unsigned char *in, - size_t inl) -{ - PROV_SM4_XTS_CTX *ctx = (PROV_SM4_XTS_CTX *)vctx; - - if (outsize < inl) { - ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); - return 0; - } - - if (!sm4_xts_cipher(ctx, out, outl, outsize, in, inl)) { - ERR_raise(ERR_LIB_PROV, PROV_R_CIPHER_OPERATION_FAILED); - return 0; - } - - return 1; -} - -static int sm4_xts_stream_final(void *vctx, unsigned char *out, size_t *outl, - size_t outsize) -{ - if (!ossl_prov_is_running()) - return 0; - *outl = 0; - return 1; -} - -static const OSSL_PARAM sm4_xts_known_settable_ctx_params[] = { - OSSL_PARAM_utf8_string(OSSL_CIPHER_PARAM_XTS_STANDARD, NULL, 0), - OSSL_PARAM_END -}; - -static const OSSL_PARAM *sm4_xts_settable_ctx_params(ossl_unused void *cctx, - ossl_unused void *provctx) -{ - return sm4_xts_known_settable_ctx_params; -} - -static int sm4_xts_set_ctx_params(void *vxctx, const OSSL_PARAM params[]) -{ - PROV_SM4_XTS_CTX *xctx = (PROV_SM4_XTS_CTX *)vxctx; - const OSSL_PARAM *p; - - if (params == NULL) - return 1; - - /*- - * Sets the XTS standard to use with SM4-XTS algorithm. - * - * Must be utf8 string "GB" or "IEEE", - * "GB" means the GB/T 17964-2021 standard - * "IEEE" means the IEEE Std 1619-2007 standard - */ - p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_XTS_STANDARD); - - if (p != NULL) { - const char *xts_standard = NULL; - - if (p->data_type != OSSL_PARAM_UTF8_STRING) - return 0; - - if (!OSSL_PARAM_get_utf8_string_ptr(p, &xts_standard)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - if (OPENSSL_strcasecmp(xts_standard, "GB") == 0) { - xctx->xts_standard = 0; - } else if (OPENSSL_strcasecmp(xts_standard, "IEEE") == 0) { - xctx->xts_standard = 1; - } else { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - } - - return 1; -} - -#define IMPLEMENT_cipher(lcmode, UCMODE, kbits, flags) \ -static OSSL_FUNC_cipher_get_params_fn sm4_##kbits##_##lcmode##_get_params; \ -static int sm4_##kbits##_##lcmode##_get_params(OSSL_PARAM params[]) \ -{ \ - return ossl_cipher_generic_get_params(params, EVP_CIPH_##UCMODE##_MODE, \ - flags, 2 * kbits, SM4_XTS_BLOCK_BITS,\ - SM4_XTS_IV_BITS); \ -} \ -static OSSL_FUNC_cipher_newctx_fn sm4_##kbits##_xts_newctx; \ -static void *sm4_##kbits##_xts_newctx(void *provctx) \ -{ \ - return sm4_xts_newctx(provctx, EVP_CIPH_##UCMODE##_MODE, flags, 2 * kbits, \ - SM4_XTS_BLOCK_BITS, SM4_XTS_IV_BITS); \ -} \ -const OSSL_DISPATCH ossl_sm4##kbits##xts_functions[] = { \ - { OSSL_FUNC_CIPHER_NEWCTX, (void (*)(void))sm4_##kbits##_xts_newctx }, \ - { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))sm4_xts_einit }, \ - { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))sm4_xts_dinit }, \ - { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))sm4_xts_stream_update }, \ - { OSSL_FUNC_CIPHER_FINAL, (void (*)(void))sm4_xts_stream_final }, \ - { OSSL_FUNC_CIPHER_CIPHER, (void (*)(void))sm4_xts_cipher }, \ - { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))sm4_xts_freectx }, \ - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void))sm4_xts_dupctx }, \ - { OSSL_FUNC_CIPHER_GET_PARAMS, \ - (void (*)(void))sm4_##kbits##_##lcmode##_get_params }, \ - { OSSL_FUNC_CIPHER_GETTABLE_PARAMS, \ - (void (*)(void))ossl_cipher_generic_gettable_params }, \ - { OSSL_FUNC_CIPHER_GET_CTX_PARAMS, \ - (void (*)(void))ossl_cipher_generic_get_ctx_params }, \ - { OSSL_FUNC_CIPHER_GETTABLE_CTX_PARAMS, \ - (void (*)(void))ossl_cipher_generic_gettable_ctx_params }, \ - { OSSL_FUNC_CIPHER_SET_CTX_PARAMS, \ - (void (*)(void))sm4_xts_set_ctx_params }, \ - { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ - (void (*)(void))sm4_xts_settable_ctx_params }, \ - OSSL_DISPATCH_END \ -} -/* ossl_sm4128xts_functions */ -IMPLEMENT_cipher(xts, XTS, 128, SM4_XTS_FLAGS); diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_xts.h b/openssl/src/providers/implementations/ciphers/cipher_sm4_xts.h deleted file mode 100644 index 43d9a212e..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_xts.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "prov/ciphercommon.h" -#include "crypto/sm4_platform.h" - -PROV_CIPHER_FUNC(void, xts_stream, - (const unsigned char *in, unsigned char *out, size_t len, - const SM4_KEY *key1, const SM4_KEY *key2, - const unsigned char iv[16], const int enc)); - -typedef struct prov_sm4_xts_ctx_st { - /* Must be first */ - PROV_CIPHER_CTX base; - - /* SM4 key schedules to use */ - union { - OSSL_UNION_ALIGN; - SM4_KEY ks; - } ks1, ks2; - - /*- - * XTS standard to use with SM4-XTS algorithm - * - * Must be 0 or 1, - * 0 for XTS mode specified by GB/T 17964-2021 - * 1 for XTS mode specified by IEEE Std 1619-2007 - */ - int xts_standard; - - XTS128_CONTEXT xts; - - /* Stream function for XTS mode specified by GB/T 17964-2021 */ - OSSL_xts_stream_fn stream_gb; - /* Stream function for XTS mode specified by IEEE Std 1619-2007 */ - OSSL_xts_stream_fn stream; -} PROV_SM4_XTS_CTX; - -const PROV_CIPHER_HW *ossl_prov_cipher_hw_sm4_xts(size_t keybits); diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c b/openssl/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c deleted file mode 100644 index 6cf58e851..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_xts_hw.c +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "cipher_sm4_xts.h" - -#define XTS_SET_KEY_FN(fn_set_enc_key, fn_set_dec_key, \ - fn_block_enc, fn_block_dec, \ - fn_stream, fn_stream_gb) { \ - size_t bytes = keylen / 2; \ - \ - if (ctx->enc) { \ - fn_set_enc_key(key, &xctx->ks1.ks); \ - xctx->xts.block1 = (block128_f)fn_block_enc; \ - } else { \ - fn_set_dec_key(key, &xctx->ks1.ks); \ - xctx->xts.block1 = (block128_f)fn_block_dec; \ - } \ - fn_set_enc_key(key + bytes, &xctx->ks2.ks); \ - xctx->xts.block2 = (block128_f)fn_block_enc; \ - xctx->xts.key1 = &xctx->ks1; \ - xctx->xts.key2 = &xctx->ks2; \ - xctx->stream = fn_stream; \ - xctx->stream_gb = fn_stream_gb; \ -} - -static int cipher_hw_sm4_xts_generic_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_SM4_XTS_CTX *xctx = (PROV_SM4_XTS_CTX *)ctx; - OSSL_xts_stream_fn stream = NULL; - OSSL_xts_stream_fn stream_gb = NULL; -#ifdef HWSM4_CAPABLE - if (HWSM4_CAPABLE) { - XTS_SET_KEY_FN(HWSM4_set_encrypt_key, HWSM4_set_decrypt_key, - HWSM4_encrypt, HWSM4_decrypt, stream, stream_gb); - return 1; - } else -#endif /* HWSM4_CAPABLE */ -#ifdef VPSM4_EX_CAPABLE - if (VPSM4_EX_CAPABLE) { - stream = vpsm4_ex_xts_encrypt; - stream_gb = vpsm4_ex_xts_encrypt_gb; - XTS_SET_KEY_FN(vpsm4_ex_set_encrypt_key, vpsm4_ex_set_decrypt_key, - vpsm4_ex_encrypt, vpsm4_ex_decrypt, stream, stream_gb); - return 1; - } else -#endif /* VPSM4_EX_CAPABLE */ -#ifdef VPSM4_CAPABLE - if (VPSM4_CAPABLE) { - stream = vpsm4_xts_encrypt; - stream_gb = vpsm4_xts_encrypt_gb; - XTS_SET_KEY_FN(vpsm4_set_encrypt_key, vpsm4_set_decrypt_key, - vpsm4_encrypt, vpsm4_decrypt, stream, stream_gb); - return 1; - } else -#endif /* VPSM4_CAPABLE */ - { - (void)0; - } - { - XTS_SET_KEY_FN(ossl_sm4_set_key, ossl_sm4_set_key, ossl_sm4_encrypt, - ossl_sm4_decrypt, stream, stream_gb); - } - return 1; -} - -static void cipher_hw_sm4_xts_copyctx(PROV_CIPHER_CTX *dst, - const PROV_CIPHER_CTX *src) -{ - PROV_SM4_XTS_CTX *sctx = (PROV_SM4_XTS_CTX *)src; - PROV_SM4_XTS_CTX *dctx = (PROV_SM4_XTS_CTX *)dst; - - *dctx = *sctx; - dctx->xts.key1 = &dctx->ks1.ks; - dctx->xts.key2 = &dctx->ks2.ks; -} - - -static const PROV_CIPHER_HW sm4_generic_xts = { - cipher_hw_sm4_xts_generic_initkey, - NULL, - cipher_hw_sm4_xts_copyctx -}; - -#if defined(__riscv) && __riscv_xlen == 64 -# include "cipher_sm4_xts_hw_rv64i.inc" -#else -const PROV_CIPHER_HW *ossl_prov_cipher_hw_sm4_xts(size_t keybits) -{ - return &sm4_generic_xts; -} -#endif diff --git a/openssl/src/providers/implementations/ciphers/cipher_sm4_xts_hw_rv64i.inc b/openssl/src/providers/implementations/ciphers/cipher_sm4_xts_hw_rv64i.inc deleted file mode 100644 index 2ab15269c..000000000 --- a/openssl/src/providers/implementations/ciphers/cipher_sm4_xts_hw_rv64i.inc +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/*- - * RISC-V 64 ZVKSED support for SM4 GCM. - * This file is included by cipher_sm4_gcm_hw.c - */ - -static int rv64i_zvksed_sm4_xts_initkey(PROV_CIPHER_CTX *ctx, - const unsigned char *key, - size_t keylen) -{ - PROV_SM4_XTS_CTX *xctx = (PROV_SM4_XTS_CTX *)ctx; - OSSL_xts_stream_fn stream_fn = NULL; - OSSL_xts_stream_fn stream_gb_fn = NULL; - - XTS_SET_KEY_FN(rv64i_zvksed_sm4_set_encrypt_key, - rv64i_zvksed_sm4_set_decrypt_key, - rv64i_zvksed_sm4_encrypt, - rv64i_zvksed_sm4_decrypt, - stream_fn, stream_gb_fn); - return 1; -} - -static const PROV_CIPHER_HW rv64i_zvksed_sm4_xts = { - rv64i_zvksed_sm4_xts_initkey, - NULL, - cipher_hw_sm4_xts_copyctx -}; - -const PROV_CIPHER_HW *ossl_prov_cipher_hw_sm4_xts(size_t keybits) -{ - if (RISCV_HAS_ZVKB_AND_ZVKSED() && riscv_vlen() >= 128) - return &rv64i_zvksed_sm4_xts; - else - return &sm4_generic_xts; -} diff --git a/openssl/src/providers/implementations/ciphers/cipher_tdes.h b/openssl/src/providers/implementations/ciphers/cipher_tdes.h index 3c98ed241..93f9d1744 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_tdes.h +++ b/openssl/src/providers/implementations/ciphers/cipher_tdes.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -67,7 +67,7 @@ const OSSL_DISPATCH ossl_tdes_##type##_##lcmode##_functions[] = { \ (void (*)(void))ossl_cipher_generic_set_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))ossl_cipher_generic_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } void *ossl_tdes_newctx(void *provctx, int mode, size_t kbits, size_t blkbits, diff --git a/openssl/src/providers/implementations/ciphers/cipher_tdes_common.c b/openssl/src/providers/implementations/ciphers/cipher_tdes_common.c index c80d9f16b..346aec05a 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_tdes_common.c +++ b/openssl/src/providers/implementations/ciphers/cipher_tdes_common.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -44,8 +44,10 @@ void *ossl_tdes_dupctx(void *ctx) return NULL; ret = OPENSSL_malloc(sizeof(*ret)); - if (ret == NULL) + if (ret == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } in->base.hw->copyctx(&ret->base, &in->base); return ret; @@ -90,7 +92,6 @@ static int tdes_init(void *vctx, const unsigned char *key, size_t keylen, } if (!ctx->hw->init(ctx, key, ctx->keylen)) return 0; - ctx->key_set = 1; } return ossl_cipher_generic_set_ctx_params(ctx, params); } @@ -122,12 +123,13 @@ static int tdes_generatekey(PROV_CIPHER_CTX *ctx, void *ptr) if (kl == 0 || RAND_priv_bytes_ex(ctx->libctx, ptr, kl, 0) <= 0) return 0; DES_set_odd_parity(deskey); - if (kl >= 16) { + if (kl >= 16) DES_set_odd_parity(deskey + 1); - if (kl >= 24) - DES_set_odd_parity(deskey + 2); + if (kl >= 24) { + DES_set_odd_parity(deskey + 2); + return 1; } - return 1; + return 0; } int ossl_tdes_get_ctx_params(void *vctx, OSSL_PARAM params[]) diff --git a/openssl/src/providers/implementations/ciphers/cipher_tdes_default_hw.c b/openssl/src/providers/implementations/ciphers/cipher_tdes_default_hw.c index ccdf3941c..53cbbad57 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_tdes_default_hw.c +++ b/openssl/src/providers/implementations/ciphers/cipher_tdes_default_hw.c @@ -99,8 +99,7 @@ static int ossl_cipher_hw_tdes_cfb1(PROV_CIPHER_CTX *ctx, unsigned char *out, { PROV_TDES_CTX *tctx = (PROV_TDES_CTX *)ctx; size_t n; - unsigned char c[1]; - unsigned char d[1] = { 0 }; + unsigned char c[1], d[1]; if (ctx->use_bits == 0) inl *= 8; diff --git a/openssl/src/providers/implementations/ciphers/cipher_tdes_wrap.c b/openssl/src/providers/implementations/ciphers/cipher_tdes_wrap.c index 391383b55..1b4539a64 100644 --- a/openssl/src/providers/implementations/ciphers/cipher_tdes_wrap.c +++ b/openssl/src/providers/implementations/ciphers/cipher_tdes_wrap.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -203,7 +203,7 @@ const OSSL_DISPATCH ossl_tdes_wrap_cbc_functions[] = \ (void (*)(void))ossl_cipher_generic_set_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))ossl_cipher_generic_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } /* ossl_tdes_wrap_cbc_functions */ diff --git a/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3.c b/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3.c new file mode 100644 index 000000000..b79fc3e80 --- /dev/null +++ b/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3.c @@ -0,0 +1,222 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* Dispatch functions for zuc_128_eea3 cipher */ + +#include +#include "cipher_zuc_eea3.h" +#include "prov/implementations.h" +#include "prov/providercommon.h" + +#define ZUC_EEA3_KEYLEN (ZUC_KEY_SIZE) +#define ZUC_EEA3_BLKLEN (1) +#define ZUC_EEA3_IVLEN (ZUC_CTR_SIZE) +#define ZUC_EEA3_FLAGS (PROV_CIPHER_FLAG_VARIABLE_LENGTH) + +static OSSL_FUNC_cipher_newctx_fn zuc_128_eea3_newctx; +static OSSL_FUNC_cipher_freectx_fn zuc_128_eea3_freectx; +static OSSL_FUNC_cipher_get_params_fn zuc_128_eea3_get_params; +static OSSL_FUNC_cipher_get_ctx_params_fn zuc_128_eea3_get_ctx_params; +static OSSL_FUNC_cipher_set_ctx_params_fn zuc_128_eea3_set_ctx_params; +static OSSL_FUNC_cipher_gettable_ctx_params_fn zuc_128_eea3_gettable_ctx_params; +static OSSL_FUNC_cipher_settable_ctx_params_fn zuc_128_eea3_settable_ctx_params; +#define zuc_128_eea3_cipher ossl_cipher_generic_cipher +#define zuc_128_eea3_update ossl_cipher_generic_stream_update +#define zuc_128_eea3_final ossl_cipher_generic_stream_final +#define zuc_128_eea3_gettable_params ossl_cipher_generic_gettable_params + +static void *zuc_128_eea3_newctx(void *provctx) +{ + PROV_ZUC_EEA3_CTX *ctx; + + if (!ossl_prov_is_running()) + return NULL; + + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx != NULL) + ossl_cipher_generic_initkey(ctx, ZUC_EEA3_KEYLEN * 8, + ZUC_EEA3_BLKLEN * 8, + ZUC_EEA3_IVLEN * 8, + 0, ZUC_EEA3_FLAGS, + ossl_prov_cipher_hw_zuc_128_eea3(ZUC_EEA3_KEYLEN * 8), + NULL); + return ctx; +} + +static void zuc_128_eea3_freectx(void *vctx) +{ + PROV_CIPHER_HW_ZUC_EEA3 *hw; + PROV_ZUC_EEA3_CTX *ctx = (PROV_ZUC_EEA3_CTX *)vctx; + + if (ctx != NULL) { + hw = (PROV_CIPHER_HW_ZUC_EEA3 *)((PROV_CIPHER_CTX *)vctx)->hw; + hw->cleanup(ctx); + ossl_cipher_generic_reset_ctx((PROV_CIPHER_CTX *)vctx); + OPENSSL_clear_free(ctx, sizeof(*ctx)); + } +} + +static int zuc_128_eea3_get_params(OSSL_PARAM params[]) +{ + return ossl_cipher_generic_get_params(params, 0, ZUC_EEA3_FLAGS, + ZUC_EEA3_KEYLEN * 8, + ZUC_EEA3_BLKLEN * 8, + ZUC_EEA3_IVLEN * 8); +} + +static int zuc_128_eea3_get_ctx_params(void *vctx, OSSL_PARAM params[]) +{ + OSSL_PARAM *p; + PROV_ZUC_EEA3_CTX *ctx = (PROV_ZUC_EEA3_CTX *)vctx; + + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_IVLEN); + if (p != NULL && !OSSL_PARAM_set_size_t(p, ZUC_EEA3_IVLEN)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_KEYLEN); + if (p != NULL && !OSSL_PARAM_set_size_t(p, ZUC_EEA3_KEYLEN)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_UPDATED_IV); + if (p != NULL) { + if (p->data_size < ZUC_EEA3_IVLEN) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); + return 0; + } + if (!OSSL_PARAM_set_octet_string(p, ctx->base.iv, ZUC_EEA3_IVLEN) + && !OSSL_PARAM_set_octet_ptr(p, &ctx->base.iv, ZUC_EEA3_IVLEN)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + } + + return 1; +} + +static const OSSL_PARAM zuc_128_eea3_known_gettable_ctx_params[] = { + OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_KEYLEN, NULL), + OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_IVLEN, NULL), + OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_UPDATED_IV, NULL), + OSSL_PARAM_END +}; +const OSSL_PARAM *zuc_128_eea3_gettable_ctx_params(ossl_unused void *cctx, + ossl_unused void *provctx) +{ + return zuc_128_eea3_known_gettable_ctx_params; +} + +static int zuc_128_eea3_set_ctx_params(void *vctx, const OSSL_PARAM params[]) +{ + const OSSL_PARAM *p; + size_t len; + + if (params == NULL) + return 1; + + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_KEYLEN); + if (p != NULL) { + if (!OSSL_PARAM_get_size_t(p, &len)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); + return 0; + } + if (len != ZUC_EEA3_KEYLEN) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); + return 0; + } + } + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_IVLEN); + if (p != NULL) { + if (!OSSL_PARAM_get_size_t(p, &len)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); + return 0; + } + if (len != ZUC_EEA3_IVLEN) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); + return 0; + } + } + return 1; +} + +static const OSSL_PARAM zuc_128_eea3_known_settable_ctx_params[] = { + OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_KEYLEN, NULL), + OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_IVLEN, NULL), + OSSL_PARAM_END +}; +const OSSL_PARAM *zuc_128_eea3_settable_ctx_params(ossl_unused void *cctx, + ossl_unused void *provctx) +{ + return zuc_128_eea3_known_settable_ctx_params; +} + +int ossl_zuc_128_eea3_einit(void *vctx, const unsigned char *key, size_t keylen, + const unsigned char *iv, size_t ivlen, + const OSSL_PARAM params[]) +{ + int ret; + PROV_CIPHER_CTX *ctx; + PROV_CIPHER_HW_ZUC_EEA3 *hw; + + /* The generic function checks for ossl_prov_is_running() */ + ret = ossl_cipher_generic_einit(vctx, key, keylen, iv, ivlen, NULL); + if (ret && iv != NULL) { + ctx = (PROV_CIPHER_CTX *)vctx; + hw = (PROV_CIPHER_HW_ZUC_EEA3 *)ctx->hw; + hw->initiv(ctx); + } + if (ret && !zuc_128_eea3_set_ctx_params(vctx, params)) + ret = 0; + return ret; +} + +int ossl_zuc_128_eea3_dinit(void *vctx, const unsigned char *key, size_t keylen, + const unsigned char *iv, size_t ivlen, + const OSSL_PARAM params[]) +{ + int ret; + PROV_CIPHER_CTX *ctx; + PROV_CIPHER_HW_ZUC_EEA3 *hw; + + /* The generic function checks for ossl_prov_is_running() */ + ret = ossl_cipher_generic_dinit(vctx, key, keylen, iv, ivlen, NULL); + if (ret && iv != NULL) { + ctx = (PROV_CIPHER_CTX *)vctx; + hw = (PROV_CIPHER_HW_ZUC_EEA3 *)ctx->hw; + hw->initiv(ctx); + } + if (ret && !zuc_128_eea3_set_ctx_params(vctx, params)) + ret = 0; + return ret; +} + +/* ossl_zuc_128_eea3_functions */ +const OSSL_DISPATCH ossl_zuc_128_eea3_functions[] = { + { OSSL_FUNC_CIPHER_NEWCTX, (void (*)(void))zuc_128_eea3_newctx }, + { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))zuc_128_eea3_freectx }, + { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))ossl_zuc_128_eea3_einit }, + { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))ossl_zuc_128_eea3_dinit }, + { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))zuc_128_eea3_update }, + { OSSL_FUNC_CIPHER_FINAL, (void (*)(void))zuc_128_eea3_final }, + { OSSL_FUNC_CIPHER_CIPHER, (void (*)(void))zuc_128_eea3_cipher}, + { OSSL_FUNC_CIPHER_GET_PARAMS, (void (*)(void))zuc_128_eea3_get_params }, + { OSSL_FUNC_CIPHER_GETTABLE_PARAMS, + (void (*)(void))zuc_128_eea3_gettable_params }, + { OSSL_FUNC_CIPHER_GET_CTX_PARAMS, + (void (*)(void))zuc_128_eea3_get_ctx_params }, + { OSSL_FUNC_CIPHER_GETTABLE_CTX_PARAMS, + (void (*)(void))zuc_128_eea3_gettable_ctx_params }, + { OSSL_FUNC_CIPHER_SET_CTX_PARAMS, + (void (*)(void))zuc_128_eea3_set_ctx_params }, + { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, + (void (*)(void))zuc_128_eea3_settable_ctx_params }, + { 0, NULL } +}; + diff --git a/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3.h b/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3.h new file mode 100644 index 000000000..a450e68f1 --- /dev/null +++ b/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3.h @@ -0,0 +1,31 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include "include/crypto/zuc.h" +#include "prov/ciphercommon.h" + +typedef struct { + PROV_CIPHER_CTX base; /* must be first */ + union { + OSSL_UNION_ALIGN; + ZUC_KEY ks; + } ks; +} PROV_ZUC_EEA3_CTX; + +typedef struct prov_cipher_hw_zuc_eea3_st { + PROV_CIPHER_HW base; /* must be first */ + int (*initiv)(PROV_CIPHER_CTX *ctx); + void (*cleanup)(PROV_ZUC_EEA3_CTX *ctx); +} PROV_CIPHER_HW_ZUC_EEA3; + +const PROV_CIPHER_HW *ossl_prov_cipher_hw_zuc_128_eea3(size_t keybits); + +OSSL_FUNC_cipher_encrypt_init_fn ossl_zuc_128_eea3_einit; +OSSL_FUNC_cipher_decrypt_init_fn ossl_zuc_128_eea3_dinit; +void ossl_zuc_128_eea3_initctx(PROV_ZUC_EEA3_CTX *ctx); diff --git a/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c b/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c new file mode 100644 index 000000000..3c1d00260 --- /dev/null +++ b/openssl/src/providers/implementations/ciphers/cipher_zuc_eea3_hw.c @@ -0,0 +1,131 @@ +/* + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* zuc_128_eea3 cipher implementation */ + +#include "cipher_zuc_eea3.h" + +static int zuc_128_eea3_initiv(PROV_CIPHER_CTX *vctx); + +static int zuc_128_eea3_initkey(PROV_CIPHER_CTX *vctx, const uint8_t *key, + size_t keylen) +{ + PROV_ZUC_EEA3_CTX *ctx = (PROV_ZUC_EEA3_CTX *)vctx; + ZUC_KEY *zk = &ctx->ks.ks; + + zk->k = key; + + zuc_128_eea3_initiv(vctx); + + return 1; +} + +static int zuc_128_eea3_initiv(PROV_CIPHER_CTX *vctx) +{ + PROV_ZUC_EEA3_CTX *ctx = (PROV_ZUC_EEA3_CTX *)vctx; + ZUC_KEY *zk = &ctx->ks.ks; + uint32_t count; + uint32_t bearer; + uint32_t direction; + unsigned char *iv = &vctx->oiv[0]; + + /* + * This is a lazy approach: we 'borrow' the 'iv' parameter + * to use it as a place of transfer the EEA3 iv params - + * count, bearer and direction. + * + * count is 32 bits, bearer is 5 bits and direction is 1 + * bit so we read the first 38 bits of iv. And the whole + * iv is set to 5 bytes (40 bits). + */ + + /* IV is a 'must' */ + if (!vctx->iv_set || !zk->k) + return 0; + + count = ((long)iv[0] << 24) | (iv[1] << 16) | (iv[2] << 8) | iv[3]; + bearer = (iv[4] & 0xF8) >> 3; + direction = (iv[4] & 0x4) >> 2; + + zk->iv[0] = (count >> 24) & 0xFF; + zk->iv[1] = (count >> 16) & 0xFF; + zk->iv[2] = (count >> 8) & 0xFF; + zk->iv[3] = count & 0xFF; + + zk->iv[4] = ((bearer << 3) | ((direction & 1) << 2)) & 0xFC; + zk->iv[5] = zk->iv[6] = zk->iv[7] = 0; + + zk->iv[8] = zk->iv[0]; + zk->iv[9] = zk->iv[1]; + zk->iv[10] = zk->iv[2]; + zk->iv[11] = zk->iv[3]; + zk->iv[12] = zk->iv[4]; + zk->iv[13] = zk->iv[5]; + zk->iv[14] = zk->iv[6]; + zk->iv[15] = zk->iv[7]; + + zk->keystream_len = 0; + zk->inited = 0; + + ZUC_init(zk); + + return 1; +} + +static int zuc_128_eea3_cipher(PROV_CIPHER_CTX *vctx, unsigned char *out, + const unsigned char *in, size_t inl) +{ + PROV_ZUC_EEA3_CTX *ctx = (PROV_ZUC_EEA3_CTX *)vctx; + ZUC_KEY *zk = &ctx->ks.ks; + unsigned int i, k, n, num = vctx->num; + + if (num >= zk->keystream_len && !ZUC_generate_keystream(zk)) + return 0; + + n = zk->L * sizeof(uint32_t); + + /* + * EEA3 is based on 'bits', but we can only handle 'bytes'. + * + * So we choose to output a final whole byte, even if there are some + * bits at the end of the input. Those trailing bits in the last byte + * should be discarded by caller. + */ + for (i = 0; i < inl; i++) { + k = num + i; + if (k >= zk->keystream_len) { + if (!ZUC_generate_keystream(zk)) + return 0; + } + + out[i] = in[i] ^ zk->keystream[k % n]; + } + + /* num always points to next key byte to use */ + vctx->num += inl; + + return 1; +} + +static void zuc_128_eea3_cleanup(PROV_ZUC_EEA3_CTX *ctx) +{ + ZUC_destroy_keystream(&ctx->ks.ks); +} + +static const PROV_CIPHER_HW_ZUC_EEA3 zuc_128_eea3_hw = { + { zuc_128_eea3_initkey, zuc_128_eea3_cipher }, + zuc_128_eea3_initiv, + zuc_128_eea3_cleanup +}; + +const PROV_CIPHER_HW *ossl_prov_cipher_hw_zuc_128_eea3(size_t keybits) +{ + return (PROV_CIPHER_HW *)&zuc_128_eea3_hw; +} + diff --git a/openssl/src/providers/implementations/ciphers/ciphercommon.c b/openssl/src/providers/implementations/ciphers/ciphercommon.c index 7ad3eb0a1..fa383165d 100644 --- a/openssl/src/providers/implementations/ciphers/ciphercommon.c +++ b/openssl/src/providers/implementations/ciphers/ciphercommon.c @@ -128,10 +128,7 @@ int ossl_cipher_var_keylen_set_ctx_params(void *vctx, const OSSL_PARAM params[]) ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); return 0; } - if (ctx->keylen != keylen) { - ctx->keylen = keylen; - ctx->key_set = 0; - } + ctx->keylen = keylen; } return 1; } @@ -220,7 +217,6 @@ static int cipher_generic_init_internal(PROV_CIPHER_CTX *ctx, } if (!ctx->hw->init(ctx, key, ctx->keylen)) return 0; - ctx->key_set = 1; } return ossl_cipher_generic_set_ctx_params(ctx, params); } @@ -253,11 +249,6 @@ int ossl_cipher_generic_block_update(void *vctx, unsigned char *out, size_t blksz = ctx->blocksize; size_t nextblocks; - if (!ctx->key_set) { - ERR_raise(ERR_LIB_PROV, PROV_R_NO_KEY_SET); - return 0; - } - if (ctx->tlsversion > 0) { /* * Each update call corresponds to a TLS record and is individually @@ -399,11 +390,6 @@ int ossl_cipher_generic_block_final(void *vctx, unsigned char *out, if (!ossl_prov_is_running()) return 0; - if (!ctx->key_set) { - ERR_raise(ERR_LIB_PROV, PROV_R_NO_KEY_SET); - return 0; - } - if (ctx->tlsversion > 0) { /* We never finalize TLS, so this is an error */ ERR_raise(ERR_LIB_PROV, PROV_R_CIPHER_OPERATION_FAILED); @@ -470,11 +456,6 @@ int ossl_cipher_generic_stream_update(void *vctx, unsigned char *out, { PROV_CIPHER_CTX *ctx = (PROV_CIPHER_CTX *)vctx; - if (!ctx->key_set) { - ERR_raise(ERR_LIB_PROV, PROV_R_NO_KEY_SET); - return 0; - } - if (inl == 0) { *outl = 0; return 1; @@ -529,16 +510,9 @@ int ossl_cipher_generic_stream_update(void *vctx, unsigned char *out, int ossl_cipher_generic_stream_final(void *vctx, unsigned char *out, size_t *outl, size_t outsize) { - PROV_CIPHER_CTX *ctx = (PROV_CIPHER_CTX *)vctx; - if (!ossl_prov_is_running()) return 0; - if (!ctx->key_set) { - ERR_raise(ERR_LIB_PROV, PROV_R_NO_KEY_SET); - return 0; - } - *outl = 0; return 1; } @@ -552,11 +526,6 @@ int ossl_cipher_generic_cipher(void *vctx, unsigned char *out, size_t *outl, if (!ossl_prov_is_running()) return 0; - if (!ctx->key_set) { - ERR_raise(ERR_LIB_PROV, PROV_R_NO_KEY_SET); - return 0; - } - if (outsize < inl) { ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); return 0; diff --git a/openssl/src/providers/implementations/ciphers/ciphercommon_block.c b/openssl/src/providers/implementations/ciphers/ciphercommon_block.c index cfc78e077..388eb9611 100644 --- a/openssl/src/providers/implementations/ciphers/ciphercommon_block.c +++ b/openssl/src/providers/implementations/ciphers/ciphercommon_block.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,9 +13,26 @@ #include #include #include "internal/constant_time.h" -#include "internal/ssl3_cbc.h" #include "ciphercommon_local.h" +/* Functions defined in ssl/tls_pad.c */ +int ssl3_cbc_remove_padding_and_mac(size_t *reclen, + size_t origreclen, + unsigned char *recdata, + unsigned char **mac, + int *alloced, + size_t block_size, size_t mac_size, + OSSL_LIB_CTX *libctx); + +int tls1_cbc_remove_padding_and_mac(size_t *reclen, + size_t origreclen, + unsigned char *recdata, + unsigned char **mac, + int *alloced, + size_t block_size, size_t mac_size, + int aead, + OSSL_LIB_CTX *libctx); + /* * Fills a single block of buffered data from the input, and returns the amount * of data remaining in the input that is a multiple of the blocksize. The buffer @@ -93,7 +110,7 @@ int ossl_cipher_unpadblock(unsigned char *buf, size_t *buflen, size_t blocksize) size_t pad, i; size_t len = *buflen; - if (len != blocksize) { + if(len != blocksize) { ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR); return 0; } @@ -152,6 +169,9 @@ int ossl_cipher_tlsunpadblock(OSSL_LIB_CTX *libctx, unsigned int tlsversion, alloced, blocksize, macsize, libctx); +#ifndef OPENSSL_NO_NTLS + case NTLS1_1_VERSION: +#endif case TLS1_2_VERSION: case DTLS1_2_VERSION: case TLS1_1_VERSION: diff --git a/openssl/src/providers/implementations/ciphers/ciphercommon_ccm.c b/openssl/src/providers/implementations/ciphers/ciphercommon_ccm.c index 33105911e..ce3f7527f 100644 --- a/openssl/src/providers/implementations/ciphers/ciphercommon_ccm.c +++ b/openssl/src/providers/implementations/ciphers/ciphercommon_ccm.c @@ -109,10 +109,7 @@ int ossl_ccm_set_ctx_params(void *vctx, const OSSL_PARAM params[]) ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); return 0; } - if (ctx->l != ivlen) { - ctx->l = ivlen; - ctx->iv_set = 0; - } + ctx->l = ivlen; } p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TLS1_AAD); diff --git a/openssl/src/providers/implementations/ciphers/ciphercommon_gcm.c b/openssl/src/providers/implementations/ciphers/ciphercommon_gcm.c index fe24b450a..c4301f6b8 100644 --- a/openssl/src/providers/implementations/ciphers/ciphercommon_gcm.c +++ b/openssl/src/providers/implementations/ciphers/ciphercommon_gcm.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,7 +15,6 @@ #include "prov/ciphercommon_gcm.h" #include "prov/providercommon.h" #include "prov/provider_ctx.h" -#include "internal/param_names.h" static int gcm_tls_init(PROV_GCM_CTX *dat, unsigned char *aad, size_t aad_len); static int gcm_tls_iv_set_fixed(PROV_GCM_CTX *ctx, unsigned char *iv, @@ -26,10 +25,6 @@ static int gcm_cipher_internal(PROV_GCM_CTX *ctx, unsigned char *out, size_t *padlen, const unsigned char *in, size_t len); -/* - * Called from EVP_CipherInit when there is currently no context via - * the new_ctx() function - */ void ossl_gcm_initctx(void *provctx, PROV_GCM_CTX *ctx, size_t keybits, const PROV_GCM_HW *hw) { @@ -43,9 +38,6 @@ void ossl_gcm_initctx(void *provctx, PROV_GCM_CTX *ctx, size_t keybits, ctx->libctx = PROV_LIBCTX_OF(provctx); } -/* - * Called by EVP_CipherInit via the _einit and _dinit functions - */ static int gcm_init(void *vctx, const unsigned char *key, size_t keylen, const unsigned char *iv, size_t ivlen, const OSSL_PARAM params[], int enc) @@ -74,7 +66,6 @@ static int gcm_init(void *vctx, const unsigned char *key, size_t keylen, } if (!ctx->hw->setkey(ctx, key, ctx->keylen)) return 0; - ctx->tls_enc_records = 0; } return ossl_gcm_set_ctx_params(ctx, params); } @@ -146,98 +137,85 @@ int ossl_gcm_get_ctx_params(void *vctx, OSSL_PARAM params[]) PROV_GCM_CTX *ctx = (PROV_GCM_CTX *)vctx; OSSL_PARAM *p; size_t sz; - int type; - - for (p = params; p->key != NULL; p++) { - type = ossl_param_find_pidx(p->key); - switch (type) { - default: - break; - - case PIDX_CIPHER_PARAM_IVLEN: - if (!OSSL_PARAM_set_size_t(p, ctx->ivlen)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - break; - - case PIDX_CIPHER_PARAM_KEYLEN: - if (!OSSL_PARAM_set_size_t(p, ctx->keylen)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - break; - - case PIDX_CIPHER_PARAM_AEAD_TAGLEN: - { - size_t taglen = (ctx->taglen != UNINITIALISED_SIZET) ? ctx->taglen : - GCM_TAG_MAX_SIZE; - - if (!OSSL_PARAM_set_size_t(p, taglen)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - } - break; - - case PIDX_CIPHER_PARAM_IV: - if (ctx->iv_state == IV_STATE_UNINITIALISED) - return 0; - if (ctx->ivlen > p->data_size) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); - return 0; - } - if (!OSSL_PARAM_set_octet_string(p, ctx->iv, ctx->ivlen) - && !OSSL_PARAM_set_octet_ptr(p, &ctx->iv, ctx->ivlen)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - break; - - case PIDX_CIPHER_PARAM_UPDATED_IV: - if (ctx->iv_state == IV_STATE_UNINITIALISED) - return 0; - if (ctx->ivlen > p->data_size) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); - return 0; - } - if (!OSSL_PARAM_set_octet_string(p, ctx->iv, ctx->ivlen) - && !OSSL_PARAM_set_octet_ptr(p, &ctx->iv, ctx->ivlen)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - break; - - case PIDX_CIPHER_PARAM_AEAD_TLS1_AAD_PAD: - if (!OSSL_PARAM_set_size_t(p, ctx->tls_aad_pad_sz)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - break; - - case PIDX_CIPHER_PARAM_AEAD_TAG: - sz = p->data_size; - if (sz == 0 - || sz > EVP_GCM_TLS_TAG_LEN - || !ctx->enc - || ctx->taglen == UNINITIALISED_SIZET) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_TAG); - return 0; - } - if (!OSSL_PARAM_set_octet_string(p, ctx->buf, sz)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); - return 0; - } - break; - - case PIDX_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN: - if (p->data == NULL - || p->data_type != OSSL_PARAM_OCTET_STRING - || !getivgen(ctx, p->data, p->data_size)) - return 0; - break; + + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_IVLEN); + if (p != NULL && !OSSL_PARAM_set_size_t(p, ctx->ivlen)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_KEYLEN); + if (p != NULL && !OSSL_PARAM_set_size_t(p, ctx->keylen)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_AEAD_TAGLEN); + if (p != NULL) { + size_t taglen = (ctx->taglen != UNINITIALISED_SIZET) ? ctx->taglen : + GCM_TAG_MAX_SIZE; + + if (!OSSL_PARAM_set_size_t(p, taglen)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + } + + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_IV); + if (p != NULL) { + if (ctx->iv_state == IV_STATE_UNINITIALISED) + return 0; + if (ctx->ivlen > p->data_size) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); + return 0; + } + if (!OSSL_PARAM_set_octet_string(p, ctx->iv, ctx->ivlen) + && !OSSL_PARAM_set_octet_ptr(p, &ctx->iv, ctx->ivlen)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + } + + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_UPDATED_IV); + if (p != NULL) { + if (ctx->iv_state == IV_STATE_UNINITIALISED) + return 0; + if (ctx->ivlen > p->data_size) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); + return 0; + } + if (!OSSL_PARAM_set_octet_string(p, ctx->iv, ctx->ivlen) + && !OSSL_PARAM_set_octet_ptr(p, &ctx->iv, ctx->ivlen)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + } + + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_AEAD_TLS1_AAD_PAD); + if (p != NULL && !OSSL_PARAM_set_size_t(p, ctx->tls_aad_pad_sz)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_AEAD_TAG); + if (p != NULL) { + sz = p->data_size; + if (sz == 0 + || sz > EVP_GCM_TLS_TAG_LEN + || !ctx->enc + || ctx->taglen == UNINITIALISED_SIZET) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_TAG); + return 0; + } + if (!OSSL_PARAM_set_octet_string(p, ctx->buf, sz)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; } } + p = OSSL_PARAM_locate(params, OSSL_CIPHER_PARAM_AEAD_TLS1_GET_IV_GEN); + if (p != NULL) { + if (p->data == NULL + || p->data_type != OSSL_PARAM_OCTET_STRING + || !getivgen(ctx, p->data, p->data_size)) + return 0; + } return 1; } @@ -247,80 +225,71 @@ int ossl_gcm_set_ctx_params(void *vctx, const OSSL_PARAM params[]) const OSSL_PARAM *p; size_t sz; void *vp; - int type; if (params == NULL) return 1; - for (p = params; p->key != NULL; p++) { - type = ossl_param_find_pidx(p->key); - switch (type) { - default: - break; - - case PIDX_CIPHER_PARAM_AEAD_TAG: - vp = ctx->buf; - if (!OSSL_PARAM_get_octet_string(p, &vp, EVP_GCM_TLS_TAG_LEN, &sz)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - if (sz == 0 || ctx->enc) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_TAG); - return 0; - } - ctx->taglen = sz; - break; - - case PIDX_CIPHER_PARAM_AEAD_IVLEN: - if (!OSSL_PARAM_get_size_t(p, &sz)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - if (sz == 0 || sz > sizeof(ctx->iv)) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); - return 0; - } - if (ctx->ivlen != sz) { - /* If the iv was already set or autogenerated, it is invalid. */ - if (ctx->iv_state != IV_STATE_UNINITIALISED) - ctx->iv_state = IV_STATE_FINISHED; - ctx->ivlen = sz; - } - break; - - case PIDX_CIPHER_PARAM_AEAD_TLS1_AAD: - if (p->data_type != OSSL_PARAM_OCTET_STRING) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - sz = gcm_tls_init(ctx, p->data, p->data_size); - if (sz == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_AAD); - return 0; - } - ctx->tls_aad_pad_sz = sz; - break; - - case PIDX_CIPHER_PARAM_AEAD_TLS1_IV_FIXED: - if (p->data_type != OSSL_PARAM_OCTET_STRING) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - if (gcm_tls_iv_set_fixed(ctx, p->data, p->data_size) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - break; - - case PIDX_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV: - if (p->data == NULL - || p->data_type != OSSL_PARAM_OCTET_STRING - || !setivinv(ctx, p->data, p->data_size)) - return 0; - break; + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TAG); + if (p != NULL) { + vp = ctx->buf; + if (!OSSL_PARAM_get_octet_string(p, &vp, EVP_GCM_TLS_TAG_LEN, &sz)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); + return 0; + } + if (sz == 0 || ctx->enc) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_TAG); + return 0; + } + ctx->taglen = sz; + } + + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_IVLEN); + if (p != NULL) { + if (!OSSL_PARAM_get_size_t(p, &sz)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); + return 0; + } + if (sz == 0 || sz > sizeof(ctx->iv)) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); + return 0; } + ctx->ivlen = sz; } + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TLS1_AAD); + if (p != NULL) { + if (p->data_type != OSSL_PARAM_OCTET_STRING) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); + return 0; + } + sz = gcm_tls_init(ctx, p->data, p->data_size); + if (sz == 0) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_AAD); + return 0; + } + ctx->tls_aad_pad_sz = sz; + } + + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TLS1_IV_FIXED); + if (p != NULL) { + if (p->data_type != OSSL_PARAM_OCTET_STRING) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); + return 0; + } + if (gcm_tls_iv_set_fixed(ctx, p->data, p->data_size) == 0) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); + return 0; + } + } + p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_AEAD_TLS1_SET_IV_INV); + if (p != NULL) { + if (p->data == NULL + || p->data_type != OSSL_PARAM_OCTET_STRING + || !setivinv(ctx, p->data, p->data_size)) + return 0; + } + + return 1; } @@ -478,6 +447,7 @@ static int gcm_tls_init(PROV_GCM_CTX *dat, unsigned char *aad, size_t aad_len) buf = dat->buf; memcpy(buf, aad, aad_len); dat->tls_aad_len = aad_len; + dat->tls_enc_records = 0; len = buf[aad_len - 2] << 8 | buf[aad_len - 1]; /* Correct length for explicit iv. */ diff --git a/openssl/src/providers/implementations/digests/blake2_impl.h b/openssl/src/providers/implementations/digests/blake2_impl.h deleted file mode 100644 index e7c31474a..000000000 --- a/openssl/src/providers/implementations/digests/blake2_impl.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Derived from the BLAKE2 reference implementation written by Samuel Neves. - * Copyright 2012, Samuel Neves - * More information about the BLAKE2 hash function and its implementations - * can be found at https://blake2.net. - */ - -#include -#include "internal/endian.h" - -static ossl_inline uint32_t load32(const uint8_t *src) -{ - DECLARE_IS_ENDIAN; - - if (IS_LITTLE_ENDIAN) { - uint32_t w; - memcpy(&w, src, sizeof(w)); - return w; - } else { - uint32_t w = ((uint32_t)src[0]) - | ((uint32_t)src[1] << 8) - | ((uint32_t)src[2] << 16) - | ((uint32_t)src[3] << 24); - return w; - } -} - -static ossl_inline uint64_t load64(const uint8_t *src) -{ - DECLARE_IS_ENDIAN; - - if (IS_LITTLE_ENDIAN) { - uint64_t w; - memcpy(&w, src, sizeof(w)); - return w; - } else { - uint64_t w = ((uint64_t)src[0]) - | ((uint64_t)src[1] << 8) - | ((uint64_t)src[2] << 16) - | ((uint64_t)src[3] << 24) - | ((uint64_t)src[4] << 32) - | ((uint64_t)src[5] << 40) - | ((uint64_t)src[6] << 48) - | ((uint64_t)src[7] << 56); - return w; - } -} - -static ossl_inline void store32(uint8_t *dst, uint32_t w) -{ - DECLARE_IS_ENDIAN; - - if (IS_LITTLE_ENDIAN) { - memcpy(dst, &w, sizeof(w)); - } else { - uint8_t *p = (uint8_t *)dst; - int i; - - for (i = 0; i < 4; i++) - p[i] = (uint8_t)(w >> (8 * i)); - } -} - -static ossl_inline void store64(uint8_t *dst, uint64_t w) -{ - DECLARE_IS_ENDIAN; - - if (IS_LITTLE_ENDIAN) { - memcpy(dst, &w, sizeof(w)); - } else { - uint8_t *p = (uint8_t *)dst; - int i; - - for (i = 0; i < 8; i++) - p[i] = (uint8_t)(w >> (8 * i)); - } -} - -static ossl_inline uint64_t load48(const uint8_t *src) -{ - uint64_t w = ((uint64_t)src[0]) - | ((uint64_t)src[1] << 8) - | ((uint64_t)src[2] << 16) - | ((uint64_t)src[3] << 24) - | ((uint64_t)src[4] << 32) - | ((uint64_t)src[5] << 40); - return w; -} - -static ossl_inline void store48(uint8_t *dst, uint64_t w) -{ - uint8_t *p = (uint8_t *)dst; - p[0] = (uint8_t)w; - p[1] = (uint8_t)(w>>8); - p[2] = (uint8_t)(w>>16); - p[3] = (uint8_t)(w>>24); - p[4] = (uint8_t)(w>>32); - p[5] = (uint8_t)(w>>40); -} - -static ossl_inline uint32_t rotr32(const uint32_t w, const unsigned int c) -{ - return (w >> c) | (w << (32 - c)); -} - -static ossl_inline uint64_t rotr64(const uint64_t w, const unsigned int c) -{ - return (w >> c) | (w << (64 - c)); -} diff --git a/openssl/src/providers/implementations/digests/blake2_prov.c b/openssl/src/providers/implementations/digests/blake2_prov.c deleted file mode 100644 index 37c3e7038..000000000 --- a/openssl/src/providers/implementations/digests/blake2_prov.c +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include "prov/blake2.h" -#include "prov/digestcommon.h" -#include "prov/implementations.h" - -#define IMPLEMENT_BLAKE_functions(variant, VARIANT, variantsize) \ -static const OSSL_PARAM known_blake##variant##_ctx_params[] = { \ - {OSSL_DIGEST_PARAM_SIZE, OSSL_PARAM_UNSIGNED_INTEGER, NULL, 0, 0}, \ - OSSL_PARAM_END \ -}; \ - \ -const OSSL_PARAM *ossl_blake##variant##_gettable_ctx_params(ossl_unused void *ctx, \ - ossl_unused void *pctx) \ -{ \ - return known_blake##variant##_ctx_params; \ -} \ - \ -const OSSL_PARAM *ossl_blake##variant##_settable_ctx_params(ossl_unused void *ctx, \ - ossl_unused void *pctx) \ -{ \ - return known_blake##variant##_ctx_params; \ -} \ - \ -int ossl_blake##variant##_get_ctx_params(void *vctx, OSSL_PARAM params[]) \ -{ \ - struct blake##variant##_md_data_st *mdctx = vctx; \ - OSSL_PARAM *p; \ - \ - BLAKE##VARIANT##_CTX *ctx = &mdctx->ctx; \ - \ - if (ctx == NULL) \ - return 0; \ - if (params == NULL) \ - return 1; \ - \ - p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_SIZE); \ - if (p != NULL \ - && !OSSL_PARAM_set_uint(p, (unsigned int)mdctx->params.digest_length)) { \ - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); \ - return 0; \ - } \ - \ - return 1; \ -} \ - \ -int ossl_blake##variant##_set_ctx_params(void *vctx, const OSSL_PARAM params[]) \ -{ \ - size_t size; \ - struct blake##variant##_md_data_st *mdctx = vctx; \ - const OSSL_PARAM *p; \ - \ - BLAKE##VARIANT##_CTX *ctx = &mdctx->ctx; \ - \ - if (ctx == NULL) \ - return 0; \ - if (params == NULL) \ - return 1; \ - \ - p = OSSL_PARAM_locate_const(params, OSSL_DIGEST_PARAM_SIZE); \ - if (p != NULL) { \ - if (!OSSL_PARAM_get_size_t(p, &size)) { \ - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); \ - return 0; \ - } \ - if (size < 1 || size > BLAKE##VARIANT##_OUTBYTES) { \ - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_SIZE); \ - return 0; \ - } \ - ossl_blake##variant##_param_set_digest_length(&mdctx->params, (uint8_t)size); \ - } \ - \ - return 1; \ -} \ - \ -static int ossl_blake##variantsize##_init(void *ctx) \ -{ \ - struct blake##variant##_md_data_st *mdctx = ctx; \ - uint8_t digest_length = mdctx->params.digest_length; \ - \ - ossl_blake##variant##_param_init(&mdctx->params); \ - if (digest_length != 0) \ - mdctx->params.digest_length = digest_length; \ - return ossl_blake##variant##_init(&mdctx->ctx, &mdctx->params); \ -} \ - \ -static OSSL_FUNC_digest_init_fn blake##variantsize##_internal_init; \ -static OSSL_FUNC_digest_newctx_fn blake##variantsize##_newctx; \ -static OSSL_FUNC_digest_freectx_fn blake##variantsize##_freectx; \ -static OSSL_FUNC_digest_dupctx_fn blake##variantsize##_dupctx; \ -static OSSL_FUNC_digest_final_fn blake##variantsize##_internal_final; \ -static OSSL_FUNC_digest_get_params_fn blake##variantsize##_get_params; \ - \ -static int blake##variantsize##_internal_init(void *ctx, const OSSL_PARAM params[]) \ -{ \ - return ossl_prov_is_running() && ossl_blake##variant##_set_ctx_params(ctx, params) \ - && ossl_blake##variantsize##_init(ctx); \ -} \ - \ -static void *blake##variantsize##_newctx(void *prov_ctx) \ -{ \ - struct blake##variant##_md_data_st *ctx; \ - \ - ctx = ossl_prov_is_running() ? OPENSSL_zalloc(sizeof(*ctx)) : NULL; \ - return ctx; \ -} \ - \ -static void blake##variantsize##_freectx(void *vctx) \ -{ \ - struct blake##variant##_md_data_st *ctx; \ - \ - ctx = (struct blake##variant##_md_data_st *)vctx; \ - OPENSSL_clear_free(ctx, sizeof(*ctx)); \ -} \ - \ -static void *blake##variantsize##_dupctx(void *ctx) \ -{ \ - struct blake##variant##_md_data_st *in, *ret; \ - \ - in = (struct blake##variant##_md_data_st *)ctx; \ - ret = ossl_prov_is_running()? OPENSSL_malloc(sizeof(*ret)) : NULL; \ - if (ret != NULL) \ - *ret = *in; \ - return ret; \ -} \ - \ -static int blake##variantsize##_internal_final(void *ctx, unsigned char *out, \ - size_t *outl, size_t outsz) \ -{ \ - struct blake##variant##_md_data_st *b_ctx; \ - \ - b_ctx = (struct blake##variant##_md_data_st *)ctx; \ - \ - if (!ossl_prov_is_running()) \ - return 0; \ - \ - *outl = b_ctx->ctx.outlen; \ - \ - if (outsz == 0) \ - return 1; \ - \ - if (outsz < *outl) { \ - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_SIZE); \ - return 0; \ - } \ - \ - return ossl_blake##variant##_final(out, ctx); \ -} \ - \ -static int blake##variantsize##_get_params(OSSL_PARAM params[]) \ -{ \ - return ossl_digest_default_get_params(params, BLAKE##VARIANT##_BLOCKBYTES, BLAKE##VARIANT##_OUTBYTES, 0); \ -} \ - \ -const OSSL_DISPATCH ossl_blake##variantsize##_functions[] = { \ - {OSSL_FUNC_DIGEST_NEWCTX, (void (*)(void))blake##variantsize##_newctx}, \ - {OSSL_FUNC_DIGEST_UPDATE, (void (*)(void))ossl_blake##variant##_update}, \ - {OSSL_FUNC_DIGEST_FINAL, (void (*)(void))blake##variantsize##_internal_final}, \ - {OSSL_FUNC_DIGEST_FREECTX, (void (*)(void))blake##variantsize##_freectx}, \ - {OSSL_FUNC_DIGEST_DUPCTX, (void (*)(void))blake##variantsize##_dupctx}, \ - {OSSL_FUNC_DIGEST_GET_PARAMS, (void (*)(void))blake##variantsize##_get_params}, \ - {OSSL_FUNC_DIGEST_GETTABLE_PARAMS, \ - (void (*)(void))ossl_digest_default_gettable_params}, \ - {OSSL_FUNC_DIGEST_INIT, (void (*)(void))blake##variantsize##_internal_init}, \ - {OSSL_FUNC_DIGEST_GETTABLE_CTX_PARAMS, \ - (void (*)(void))ossl_blake##variant##_gettable_ctx_params}, \ - {OSSL_FUNC_DIGEST_SETTABLE_CTX_PARAMS, \ - (void (*)(void))ossl_blake##variant##_settable_ctx_params}, \ - {OSSL_FUNC_DIGEST_GET_CTX_PARAMS, \ - (void (*)(void))ossl_blake##variant##_get_ctx_params}, \ - {OSSL_FUNC_DIGEST_SET_CTX_PARAMS, \ - (void (*)(void))ossl_blake##variant##_set_ctx_params}, \ - {0, NULL} \ -}; - -IMPLEMENT_BLAKE_functions(2s, 2S, 2s256) -IMPLEMENT_BLAKE_functions(2b, 2B, 2b512) diff --git a/openssl/src/providers/implementations/digests/blake2b_prov.c b/openssl/src/providers/implementations/digests/blake2b_prov.c deleted file mode 100644 index 6ef7fac00..000000000 --- a/openssl/src/providers/implementations/digests/blake2b_prov.c +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Derived from the BLAKE2 reference implementation written by Samuel Neves. - * Copyright 2012, Samuel Neves - * More information about the BLAKE2 hash function and its implementations - * can be found at https://blake2.net. - */ - -#include -#include -#include -#include "internal/numbers.h" -#include "blake2_impl.h" -#include "prov/blake2.h" - -static const uint64_t blake2b_IV[8] = -{ - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - -static const uint8_t blake2b_sigma[12][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } -}; - -/* Set that it's the last block we'll compress */ -static ossl_inline void blake2b_set_lastblock(BLAKE2B_CTX *S) -{ - S->f[0] = -1; -} - -/* Initialize the hashing state. */ -static ossl_inline void blake2b_init0(BLAKE2B_CTX *S) -{ - int i; - - memset(S, 0, sizeof(BLAKE2B_CTX)); - for (i = 0; i < 8; ++i) { - S->h[i] = blake2b_IV[i]; - } -} - -/* init xors IV with input parameter block and sets the output length */ -static void blake2b_init_param(BLAKE2B_CTX *S, const BLAKE2B_PARAM *P) -{ - size_t i; - const uint8_t *p = (const uint8_t *)(P); - - blake2b_init0(S); - S->outlen = P->digest_length; - - /* The param struct is carefully hand packed, and should be 64 bytes on - * every platform. */ - assert(sizeof(BLAKE2B_PARAM) == 64); - /* IV XOR ParamBlock */ - for (i = 0; i < 8; ++i) { - S->h[i] ^= load64(p + sizeof(S->h[i]) * i); - } -} - -/* Initialize the parameter block with default values */ -void ossl_blake2b_param_init(BLAKE2B_PARAM *P) -{ - P->digest_length = BLAKE2B_DIGEST_LENGTH; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32(P->leaf_length, 0); - store64(P->node_offset, 0); - P->node_depth = 0; - P->inner_length = 0; - memset(P->reserved, 0, sizeof(P->reserved)); - memset(P->salt, 0, sizeof(P->salt)); - memset(P->personal, 0, sizeof(P->personal)); -} - -void ossl_blake2b_param_set_digest_length(BLAKE2B_PARAM *P, uint8_t outlen) -{ - P->digest_length = outlen; -} - -void ossl_blake2b_param_set_key_length(BLAKE2B_PARAM *P, uint8_t keylen) -{ - P->key_length = keylen; -} - -void ossl_blake2b_param_set_personal(BLAKE2B_PARAM *P, const uint8_t *personal, - size_t len) -{ - memcpy(P->personal, personal, len); - memset(P->personal + len, 0, BLAKE2B_PERSONALBYTES - len); -} - -void ossl_blake2b_param_set_salt(BLAKE2B_PARAM *P, const uint8_t *salt, - size_t len) -{ - memcpy(P->salt, salt, len); - memset(P->salt + len, 0, BLAKE2B_SALTBYTES - len); -} - -/* - * Initialize the hashing context with the given parameter block. - * Always returns 1. - */ -int ossl_blake2b_init(BLAKE2B_CTX *c, const BLAKE2B_PARAM *P) -{ - blake2b_init_param(c, P); - return 1; -} - -/* - * Initialize the hashing context with the given parameter block and key. - * Always returns 1. - */ -int ossl_blake2b_init_key(BLAKE2B_CTX *c, const BLAKE2B_PARAM *P, - const void *key) -{ - blake2b_init_param(c, P); - - /* Pad the key to form first data block */ - { - uint8_t block[BLAKE2B_BLOCKBYTES] = {0}; - - memcpy(block, key, P->key_length); - ossl_blake2b_update(c, block, BLAKE2B_BLOCKBYTES); - OPENSSL_cleanse(block, BLAKE2B_BLOCKBYTES); - } - - return 1; -} - -/* Permute the state while xoring in the block of data. */ -static void blake2b_compress(BLAKE2B_CTX *S, - const uint8_t *blocks, - size_t len) -{ - uint64_t m[16]; - uint64_t v[16]; - int i; - size_t increment; - - /* - * There are two distinct usage vectors for this function: - * - * a) BLAKE2b_Update uses it to process complete blocks, - * possibly more than one at a time; - * - * b) BLAK2b_Final uses it to process last block, always - * single but possibly incomplete, in which case caller - * pads input with zeros. - */ - assert(len < BLAKE2B_BLOCKBYTES || len % BLAKE2B_BLOCKBYTES == 0); - - /* - * Since last block is always processed with separate call, - * |len| not being multiple of complete blocks can be observed - * only with |len| being less than BLAKE2B_BLOCKBYTES ("less" - * including even zero), which is why following assignment doesn't - * have to reside inside the main loop below. - */ - increment = len < BLAKE2B_BLOCKBYTES ? len : BLAKE2B_BLOCKBYTES; - - for (i = 0; i < 8; ++i) { - v[i] = S->h[i]; - } - - do { - for (i = 0; i < 16; ++i) { - m[i] = load64(blocks + i * sizeof(m[i])); - } - - /* blake2b_increment_counter */ - S->t[0] += increment; - S->t[1] += (S->t[0] < increment); - - v[8] = blake2b_IV[0]; - v[9] = blake2b_IV[1]; - v[10] = blake2b_IV[2]; - v[11] = blake2b_IV[3]; - v[12] = S->t[0] ^ blake2b_IV[4]; - v[13] = S->t[1] ^ blake2b_IV[5]; - v[14] = S->f[0] ^ blake2b_IV[6]; - v[15] = S->f[1] ^ blake2b_IV[7]; -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2b_sigma[r][2*i+0]]; \ - d = rotr64(d ^ a, 32); \ - c = c + d; \ - b = rotr64(b ^ c, 24); \ - a = a + b + m[blake2b_sigma[r][2*i+1]]; \ - d = rotr64(d ^ a, 16); \ - c = c + d; \ - b = rotr64(b ^ c, 63); \ - } while (0) -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while (0) -#if defined(OPENSSL_SMALL_FOOTPRINT) - /* 3x size reduction on x86_64, almost 7x on ARMv8, 9x on ARMv4 */ - for (i = 0; i < 12; i++) { - ROUND(i); - } -#else - ROUND(0); - ROUND(1); - ROUND(2); - ROUND(3); - ROUND(4); - ROUND(5); - ROUND(6); - ROUND(7); - ROUND(8); - ROUND(9); - ROUND(10); - ROUND(11); -#endif - - for (i = 0; i < 8; ++i) { - S->h[i] = v[i] ^= v[i + 8] ^ S->h[i]; - } -#undef G -#undef ROUND - blocks += increment; - len -= increment; - } while (len); -} - -/* Absorb the input data into the hash state. Always returns 1. */ -int ossl_blake2b_update(BLAKE2B_CTX *c, const void *data, size_t datalen) -{ - const uint8_t *in = data; - size_t fill; - - /* - * Intuitively one would expect intermediate buffer, c->buf, to - * store incomplete blocks. But in this case we are interested to - * temporarily stash even complete blocks, because last one in the - * stream has to be treated in special way, and at this point we - * don't know if last block in *this* call is last one "ever". This - * is the reason for why |datalen| is compared as >, and not >=. - */ - fill = sizeof(c->buf) - c->buflen; - if (datalen > fill) { - if (c->buflen) { - memcpy(c->buf + c->buflen, in, fill); /* Fill buffer */ - blake2b_compress(c, c->buf, BLAKE2B_BLOCKBYTES); - c->buflen = 0; - in += fill; - datalen -= fill; - } - if (datalen > BLAKE2B_BLOCKBYTES) { - size_t stashlen = datalen % BLAKE2B_BLOCKBYTES; - /* - * If |datalen| is a multiple of the blocksize, stash - * last complete block, it can be final one... - */ - stashlen = stashlen ? stashlen : BLAKE2B_BLOCKBYTES; - datalen -= stashlen; - blake2b_compress(c, in, datalen); - in += datalen; - datalen = stashlen; - } - } - - assert(datalen <= BLAKE2B_BLOCKBYTES); - - memcpy(c->buf + c->buflen, in, datalen); - c->buflen += datalen; /* Be lazy, do not compress */ - - return 1; -} - -/* - * Calculate the final hash and save it in md. - * Always returns 1. - */ -int ossl_blake2b_final(unsigned char *md, BLAKE2B_CTX *c) -{ - uint8_t outbuffer[BLAKE2B_OUTBYTES] = {0}; - uint8_t *target = outbuffer; - int iter = (c->outlen + 7) / 8; - int i; - - /* Avoid writing to the temporary buffer if possible */ - if ((c->outlen % sizeof(c->h[0])) == 0) - target = md; - - blake2b_set_lastblock(c); - /* Padding */ - memset(c->buf + c->buflen, 0, sizeof(c->buf) - c->buflen); - blake2b_compress(c, c->buf, c->buflen); - - /* Output full hash to buffer */ - for (i = 0; i < iter; ++i) - store64(target + sizeof(c->h[i]) * i, c->h[i]); - - if (target != md) { - memcpy(md, target, c->outlen); - OPENSSL_cleanse(target, sizeof(outbuffer)); - } - - OPENSSL_cleanse(c, sizeof(BLAKE2B_CTX)); - return 1; -} diff --git a/openssl/src/providers/implementations/digests/blake2s_prov.c b/openssl/src/providers/implementations/digests/blake2s_prov.c deleted file mode 100644 index 72cab1e9a..000000000 --- a/openssl/src/providers/implementations/digests/blake2s_prov.c +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Derived from the BLAKE2 reference implementation written by Samuel Neves. - * Copyright 2012, Samuel Neves - * More information about the BLAKE2 hash function and its implementations - * can be found at https://blake2.net. - */ - -#include -#include -#include -#include "blake2_impl.h" -#include "prov/blake2.h" - -static const uint32_t blake2s_IV[8] = -{ - 0x6A09E667U, 0xBB67AE85U, 0x3C6EF372U, 0xA54FF53AU, - 0x510E527FU, 0x9B05688CU, 0x1F83D9ABU, 0x5BE0CD19U -}; - -static const uint8_t blake2s_sigma[10][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , -}; - -/* Set that it's the last block we'll compress */ -static ossl_inline void blake2s_set_lastblock(BLAKE2S_CTX *S) -{ - S->f[0] = -1; -} - -/* Initialize the hashing state. */ -static ossl_inline void blake2s_init0(BLAKE2S_CTX *S) -{ - int i; - - memset(S, 0, sizeof(BLAKE2S_CTX)); - for (i = 0; i < 8; ++i) { - S->h[i] = blake2s_IV[i]; - } -} - -/* init xors IV with input parameter block and sets the output length */ -static void blake2s_init_param(BLAKE2S_CTX *S, const BLAKE2S_PARAM *P) -{ - size_t i; - const uint8_t *p = (const uint8_t *)(P); - - blake2s_init0(S); - S->outlen = P->digest_length; - - /* The param struct is carefully hand packed, and should be 32 bytes on - * every platform. */ - assert(sizeof(BLAKE2S_PARAM) == 32); - /* IV XOR ParamBlock */ - for (i = 0; i < 8; ++i) { - S->h[i] ^= load32(&p[i*4]); - } -} - -void ossl_blake2s_param_init(BLAKE2S_PARAM *P) -{ - P->digest_length = BLAKE2S_DIGEST_LENGTH; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32(P->leaf_length, 0); - store48(P->node_offset, 0); - P->node_depth = 0; - P->inner_length = 0; - memset(P->salt, 0, sizeof(P->salt)); - memset(P->personal, 0, sizeof(P->personal)); -} - -void ossl_blake2s_param_set_digest_length(BLAKE2S_PARAM *P, uint8_t outlen) -{ - P->digest_length = outlen; -} - -void ossl_blake2s_param_set_key_length(BLAKE2S_PARAM *P, uint8_t keylen) -{ - P->key_length = keylen; -} - -void ossl_blake2s_param_set_personal(BLAKE2S_PARAM *P, const uint8_t *personal, - size_t len) -{ - memcpy(P->personal, personal, len); - memset(P->personal + len, 0, BLAKE2S_PERSONALBYTES - len); -} - -void ossl_blake2s_param_set_salt(BLAKE2S_PARAM *P, const uint8_t *salt, - size_t len) -{ - memcpy(P->salt, salt, len); - memset(P->salt + len, 0, BLAKE2S_SALTBYTES - len);} - -/* - * Initialize the hashing context with the given parameter block. - * Always returns 1. - */ -int ossl_blake2s_init(BLAKE2S_CTX *c, const BLAKE2S_PARAM *P) -{ - blake2s_init_param(c, P); - return 1; -} - -/* - * Initialize the hashing context with the given parameter block and key. - * Always returns 1. - */ -int ossl_blake2s_init_key(BLAKE2S_CTX *c, const BLAKE2S_PARAM *P, - const void *key) -{ - blake2s_init_param(c, P); - - /* Pad the key to form first data block */ - { - uint8_t block[BLAKE2S_BLOCKBYTES] = {0}; - - memcpy(block, key, P->key_length); - ossl_blake2s_update(c, block, BLAKE2S_BLOCKBYTES); - OPENSSL_cleanse(block, BLAKE2S_BLOCKBYTES); - } - - return 1; -} - -/* Permute the state while xoring in the block of data. */ -static void blake2s_compress(BLAKE2S_CTX *S, - const uint8_t *blocks, - size_t len) -{ - uint32_t m[16]; - uint32_t v[16]; - size_t i; - size_t increment; - - /* - * There are two distinct usage vectors for this function: - * - * a) BLAKE2s_Update uses it to process complete blocks, - * possibly more than one at a time; - * - * b) BLAK2s_Final uses it to process last block, always - * single but possibly incomplete, in which case caller - * pads input with zeros. - */ - assert(len < BLAKE2S_BLOCKBYTES || len % BLAKE2S_BLOCKBYTES == 0); - - /* - * Since last block is always processed with separate call, - * |len| not being multiple of complete blocks can be observed - * only with |len| being less than BLAKE2S_BLOCKBYTES ("less" - * including even zero), which is why following assignment doesn't - * have to reside inside the main loop below. - */ - increment = len < BLAKE2S_BLOCKBYTES ? len : BLAKE2S_BLOCKBYTES; - - for (i = 0; i < 8; ++i) { - v[i] = S->h[i]; - } - - do { - for (i = 0; i < 16; ++i) { - m[i] = load32(blocks + i * sizeof(m[i])); - } - - /* blake2s_increment_counter */ - S->t[0] += increment; - S->t[1] += (S->t[0] < increment); - - v[ 8] = blake2s_IV[0]; - v[ 9] = blake2s_IV[1]; - v[10] = blake2s_IV[2]; - v[11] = blake2s_IV[3]; - v[12] = S->t[0] ^ blake2s_IV[4]; - v[13] = S->t[1] ^ blake2s_IV[5]; - v[14] = S->f[0] ^ blake2s_IV[6]; - v[15] = S->f[1] ^ blake2s_IV[7]; -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2s_sigma[r][2*i+0]]; \ - d = rotr32(d ^ a, 16); \ - c = c + d; \ - b = rotr32(b ^ c, 12); \ - a = a + b + m[blake2s_sigma[r][2*i+1]]; \ - d = rotr32(d ^ a, 8); \ - c = c + d; \ - b = rotr32(b ^ c, 7); \ - } while (0) -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while (0) -#if defined(OPENSSL_SMALL_FOOTPRINT) - /* almost 3x reduction on x86_64, 4.5x on ARMv8, 4x on ARMv4 */ - for (i = 0; i < 10; i++) { - ROUND(i); - } -#else - ROUND(0); - ROUND(1); - ROUND(2); - ROUND(3); - ROUND(4); - ROUND(5); - ROUND(6); - ROUND(7); - ROUND(8); - ROUND(9); -#endif - - for (i = 0; i < 8; ++i) { - S->h[i] = v[i] ^= v[i + 8] ^ S->h[i]; - } -#undef G -#undef ROUND - blocks += increment; - len -= increment; - } while (len); -} - -/* Absorb the input data into the hash state. Always returns 1. */ -int ossl_blake2s_update(BLAKE2S_CTX *c, const void *data, size_t datalen) -{ - const uint8_t *in = data; - size_t fill; - - /* - * Intuitively one would expect intermediate buffer, c->buf, to - * store incomplete blocks. But in this case we are interested to - * temporarily stash even complete blocks, because last one in the - * stream has to be treated in special way, and at this point we - * don't know if last block in *this* call is last one "ever". This - * is the reason for why |datalen| is compared as >, and not >=. - */ - fill = sizeof(c->buf) - c->buflen; - if (datalen > fill) { - if (c->buflen) { - memcpy(c->buf + c->buflen, in, fill); /* Fill buffer */ - blake2s_compress(c, c->buf, BLAKE2S_BLOCKBYTES); - c->buflen = 0; - in += fill; - datalen -= fill; - } - if (datalen > BLAKE2S_BLOCKBYTES) { - size_t stashlen = datalen % BLAKE2S_BLOCKBYTES; - /* - * If |datalen| is a multiple of the blocksize, stash - * last complete block, it can be final one... - */ - stashlen = stashlen ? stashlen : BLAKE2S_BLOCKBYTES; - datalen -= stashlen; - blake2s_compress(c, in, datalen); - in += datalen; - datalen = stashlen; - } - } - - assert(datalen <= BLAKE2S_BLOCKBYTES); - - memcpy(c->buf + c->buflen, in, datalen); - c->buflen += datalen; /* Be lazy, do not compress */ - - return 1; -} - -/* - * Calculate the final hash and save it in md. - * Always returns 1. - */ -int ossl_blake2s_final(unsigned char *md, BLAKE2S_CTX *c) -{ - uint8_t outbuffer[BLAKE2S_OUTBYTES] = {0}; - uint8_t *target = outbuffer; - int iter = (c->outlen + 3) / 4; - int i; - - /* Avoid writing to the temporary buffer if possible */ - if ((c->outlen % sizeof(c->h[0])) == 0) - target = md; - - blake2s_set_lastblock(c); - /* Padding */ - memset(c->buf + c->buflen, 0, sizeof(c->buf) - c->buflen); - blake2s_compress(c, c->buf, c->buflen); - - /* Output full hash to buffer */ - for (i = 0; i < iter; ++i) - store32(target + sizeof(c->h[i]) * i, c->h[i]); - - if (target != md) { - memcpy(md, target, c->outlen); - OPENSSL_cleanse(target, sizeof(outbuffer)); - } - - OPENSSL_cleanse(c, sizeof(BLAKE2S_CTX)); - return 1; -} diff --git a/openssl/src/providers/implementations/digests/md2_prov.c b/openssl/src/providers/implementations/digests/md2_prov.c deleted file mode 100644 index a41a02c19..000000000 --- a/openssl/src/providers/implementations/digests/md2_prov.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD2 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include "prov/digestcommon.h" -#include "prov/implementations.h" - -/* ossl_md2_functions */ -IMPLEMENT_digest_functions(md2, MD2_CTX, - MD2_BLOCK, MD2_DIGEST_LENGTH, 0, - MD2_Init, MD2_Update, MD2_Final) diff --git a/openssl/src/providers/implementations/digests/md4_prov.c b/openssl/src/providers/implementations/digests/md4_prov.c deleted file mode 100644 index 97f73018c..000000000 --- a/openssl/src/providers/implementations/digests/md4_prov.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MD4 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include "prov/digestcommon.h" -#include "prov/implementations.h" - -/* ossl_md4_functions */ -IMPLEMENT_digest_functions(md4, MD4_CTX, - MD4_CBLOCK, MD4_DIGEST_LENGTH, 0, - MD4_Init, MD4_Update, MD4_Final) diff --git a/openssl/src/providers/implementations/digests/mdc2_prov.c b/openssl/src/providers/implementations/digests/mdc2_prov.c deleted file mode 100644 index de39f8a10..000000000 --- a/openssl/src/providers/implementations/digests/mdc2_prov.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * MDC2 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include -#include -#include -#include -#include "prov/digestcommon.h" -#include "prov/implementations.h" - -static OSSL_FUNC_digest_set_ctx_params_fn mdc2_set_ctx_params; -static OSSL_FUNC_digest_settable_ctx_params_fn mdc2_settable_ctx_params; - -static const OSSL_PARAM known_mdc2_settable_ctx_params[] = { - OSSL_PARAM_uint(OSSL_DIGEST_PARAM_PAD_TYPE, NULL), - OSSL_PARAM_END -}; - -static const OSSL_PARAM *mdc2_settable_ctx_params(ossl_unused void *ctx, - ossl_unused void *provctx) -{ - return known_mdc2_settable_ctx_params; -} - -static int mdc2_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - const OSSL_PARAM *p; - MDC2_CTX *ctx = (MDC2_CTX *)vctx; - - if (ctx == NULL) - return 0; - if (params == NULL) - return 1; - - p = OSSL_PARAM_locate_const(params, OSSL_DIGEST_PARAM_PAD_TYPE); - if (p != NULL && !OSSL_PARAM_get_uint(p, &ctx->pad_type)) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER); - return 0; - } - return 1; -} - -/* ossl_mdc2_functions */ -IMPLEMENT_digest_functions_with_settable_ctx( - mdc2, MDC2_CTX, MDC2_BLOCK, MDC2_DIGEST_LENGTH, 0, - MDC2_Init, MDC2_Update, MDC2_Final, - mdc2_settable_ctx_params, mdc2_set_ctx_params) diff --git a/openssl/src/providers/implementations/digests/ripemd_prov.c b/openssl/src/providers/implementations/digests/ripemd_prov.c deleted file mode 100644 index 526706c06..000000000 --- a/openssl/src/providers/implementations/digests/ripemd_prov.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * RIPEMD160 low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include "prov/digestcommon.h" -#include "prov/implementations.h" - -/* ossl_ripemd160_functions */ -IMPLEMENT_digest_functions(ripemd160, RIPEMD160_CTX, - RIPEMD160_CBLOCK, RIPEMD160_DIGEST_LENGTH, 0, - RIPEMD160_Init, RIPEMD160_Update, RIPEMD160_Final) diff --git a/openssl/src/providers/implementations/digests/sha2_prov.c b/openssl/src/providers/implementations/digests/sha2_prov.c index 60f1912cc..3b731796b 100644 --- a/openssl/src/providers/implementations/digests/sha2_prov.c +++ b/openssl/src/providers/implementations/digests/sha2_prov.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include "prov/digestcommon.h" @@ -71,12 +72,7 @@ IMPLEMENT_digest_functions(sha224, SHA256_CTX, IMPLEMENT_digest_functions(sha256, SHA256_CTX, SHA256_CBLOCK, SHA256_DIGEST_LENGTH, SHA2_FLAGS, SHA256_Init, SHA256_Update, SHA256_Final) -#ifndef FIPS_MODULE -/* ossl_sha256_192_functions */ -IMPLEMENT_digest_functions(sha256_192, SHA256_CTX, - SHA256_CBLOCK, SHA256_192_DIGEST_LENGTH, SHA2_FLAGS, - ossl_sha256_192_init, SHA256_Update, SHA256_Final) -#endif + /* ossl_sha384_functions */ IMPLEMENT_digest_functions(sha384, SHA512_CTX, SHA512_CBLOCK, SHA384_DIGEST_LENGTH, SHA2_FLAGS, @@ -96,3 +92,4 @@ IMPLEMENT_digest_functions(sha512_224, SHA512_CTX, IMPLEMENT_digest_functions(sha512_256, SHA512_CTX, SHA512_CBLOCK, SHA256_DIGEST_LENGTH, SHA2_FLAGS, sha512_256_init, SHA512_Update, SHA512_Final) + diff --git a/openssl/src/providers/implementations/digests/sha3_prov.c b/openssl/src/providers/implementations/digests/sha3_prov.c index 2fd0f928e..168825d47 100644 --- a/openssl/src/providers/implementations/digests/sha3_prov.c +++ b/openssl/src/providers/implementations/digests/sha3_prov.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,12 +33,10 @@ static OSSL_FUNC_digest_update_fn keccak_update; static OSSL_FUNC_digest_final_fn keccak_final; static OSSL_FUNC_digest_freectx_fn keccak_freectx; static OSSL_FUNC_digest_dupctx_fn keccak_dupctx; -static OSSL_FUNC_digest_squeeze_fn shake_squeeze; static OSSL_FUNC_digest_set_ctx_params_fn shake_set_ctx_params; static OSSL_FUNC_digest_settable_ctx_params_fn shake_settable_ctx_params; static sha3_absorb_fn generic_sha3_absorb; static sha3_final_fn generic_sha3_final; -static sha3_squeeze_fn generic_sha3_squeeze; #if defined(OPENSSL_CPUID_OBJ) && defined(__s390__) && defined(KECCAK1600_ASM) /* @@ -105,37 +103,20 @@ static int keccak_update(void *vctx, const unsigned char *inp, size_t len) } static int keccak_final(void *vctx, unsigned char *out, size_t *outl, - size_t outlen) + size_t outsz) { int ret = 1; KECCAK1600_CTX *ctx = vctx; if (!ossl_prov_is_running()) return 0; - if (outlen > 0) - ret = ctx->meth.final(ctx, out, ctx->md_size); + if (outsz > 0) + ret = ctx->meth.final(out, ctx); *outl = ctx->md_size; return ret; } -static int shake_squeeze(void *vctx, unsigned char *out, size_t *outl, - size_t outlen) -{ - int ret = 1; - KECCAK1600_CTX *ctx = vctx; - - if (!ossl_prov_is_running()) - return 0; - if (ctx->meth.squeeze == NULL) - return 0; - if (outlen > 0) - ret = ctx->meth.squeeze(ctx, out, outlen); - - *outl = outlen; - return ret; -} - /*- * Generic software version of the absorb() and final(). */ @@ -143,35 +124,18 @@ static size_t generic_sha3_absorb(void *vctx, const void *inp, size_t len) { KECCAK1600_CTX *ctx = vctx; - if (!(ctx->xof_state == XOF_STATE_INIT || - ctx->xof_state == XOF_STATE_ABSORB)) - return 0; - ctx->xof_state = XOF_STATE_ABSORB; return SHA3_absorb(ctx->A, inp, len, ctx->block_size); } -static int generic_sha3_final(void *vctx, unsigned char *out, size_t outlen) -{ - return ossl_sha3_final((KECCAK1600_CTX *)vctx, out, outlen); -} - -static int generic_sha3_squeeze(void *vctx, unsigned char *out, size_t outlen) +static int generic_sha3_final(unsigned char *md, void *vctx) { - return ossl_sha3_squeeze((KECCAK1600_CTX *)vctx, out, outlen); + return ossl_sha3_final(md, (KECCAK1600_CTX *)vctx); } static PROV_SHA3_METHOD sha3_generic_md = { generic_sha3_absorb, - generic_sha3_final, - NULL -}; - -static PROV_SHA3_METHOD shake_generic_md = -{ - generic_sha3_absorb, - generic_sha3_final, - generic_sha3_squeeze + generic_sha3_final }; #if defined(S390_SHA3) @@ -188,211 +152,43 @@ static size_t s390x_sha3_absorb(void *vctx, const void *inp, size_t len) KECCAK1600_CTX *ctx = vctx; size_t rem = len % ctx->block_size; - if (!(ctx->xof_state == XOF_STATE_INIT || - ctx->xof_state == XOF_STATE_ABSORB)) - return 0; - ctx->xof_state = XOF_STATE_ABSORB; s390x_kimd(inp, len - rem, ctx->pad, ctx->A); return rem; } -static int s390x_sha3_final(void *vctx, unsigned char *out, size_t outlen) +static int s390x_sha3_final(unsigned char *md, void *vctx) { KECCAK1600_CTX *ctx = vctx; if (!ossl_prov_is_running()) return 0; - if (!(ctx->xof_state == XOF_STATE_INIT || - ctx->xof_state == XOF_STATE_ABSORB)) - return 0; - ctx->xof_state = XOF_STATE_FINAL; s390x_klmd(ctx->buf, ctx->bufsz, NULL, 0, ctx->pad, ctx->A); - memcpy(out, ctx->A, outlen); + memcpy(md, ctx->A, ctx->md_size); return 1; } -static int s390x_shake_final(void *vctx, unsigned char *out, size_t outlen) +static int s390x_shake_final(unsigned char *md, void *vctx) { KECCAK1600_CTX *ctx = vctx; if (!ossl_prov_is_running()) return 0; - if (!(ctx->xof_state == XOF_STATE_INIT || - ctx->xof_state == XOF_STATE_ABSORB)) - return 0; - ctx->xof_state = XOF_STATE_FINAL; - s390x_klmd(ctx->buf, ctx->bufsz, out, outlen, ctx->pad, ctx->A); + s390x_klmd(ctx->buf, ctx->bufsz, md, ctx->md_size, ctx->pad, ctx->A); return 1; } -static int s390x_shake_squeeze(void *vctx, unsigned char *out, size_t outlen) -{ - KECCAK1600_CTX *ctx = vctx; - size_t len; - - if (!ossl_prov_is_running()) - return 0; - if (ctx->xof_state == XOF_STATE_FINAL) - return 0; - /* - * On the first squeeze call, finish the absorb process (incl. padding). - */ - if (ctx->xof_state != XOF_STATE_SQUEEZE) { - ctx->xof_state = XOF_STATE_SQUEEZE; - s390x_klmd(ctx->buf, ctx->bufsz, out, outlen, ctx->pad, ctx->A); - ctx->bufsz = outlen % ctx->block_size; - /* reuse ctx->bufsz to count bytes squeezed from current sponge */ - return 1; - } - ctx->xof_state = XOF_STATE_SQUEEZE; - if (ctx->bufsz != 0) { - len = ctx->block_size - ctx->bufsz; - if (outlen < len) - len = outlen; - memcpy(out, (char *)ctx->A + ctx->bufsz, len); - out += len; - outlen -= len; - ctx->bufsz += len; - if (ctx->bufsz == ctx->block_size) - ctx->bufsz = 0; - } - if (outlen == 0) - return 1; - s390x_klmd(NULL, 0, out, outlen, ctx->pad | S390X_KLMD_PS, ctx->A); - ctx->bufsz = outlen % ctx->block_size; - - return 1; -} - -static int s390x_keccakc_final(void *vctx, unsigned char *out, size_t outlen, - int padding) -{ - KECCAK1600_CTX *ctx = vctx; - size_t bsz = ctx->block_size; - size_t num = ctx->bufsz; - size_t needed = outlen; - - if (!ossl_prov_is_running()) - return 0; - if (!(ctx->xof_state == XOF_STATE_INIT || - ctx->xof_state == XOF_STATE_ABSORB)) - return 0; - ctx->xof_state = XOF_STATE_FINAL; - if (outlen == 0) - return 1; - memset(ctx->buf + num, 0, bsz - num); - ctx->buf[num] = padding; - ctx->buf[bsz - 1] |= 0x80; - s390x_kimd(ctx->buf, bsz, ctx->pad, ctx->A); - num = needed > bsz ? bsz : needed; - memcpy(out, ctx->A, num); - needed -= num; - if (needed > 0) - s390x_klmd(NULL, 0, out + bsz, needed, ctx->pad | S390X_KLMD_PS, ctx->A); - - return 1; -} - -static int s390x_keccak_final(void *vctx, unsigned char *out, size_t outlen) -{ - return s390x_keccakc_final(vctx, out, outlen, 0x01); -} - -static int s390x_kmac_final(void *vctx, unsigned char *out, size_t outlen) -{ - return s390x_keccakc_final(vctx, out, outlen, 0x04); -} - -static int s390x_keccakc_squeeze(void *vctx, unsigned char *out, size_t outlen, - int padding) -{ - KECCAK1600_CTX *ctx = vctx; - size_t len; - - if (!ossl_prov_is_running()) - return 0; - if (ctx->xof_state == XOF_STATE_FINAL) - return 0; - /* - * On the first squeeze call, finish the absorb process - * by adding the trailing padding and then doing - * a final absorb. - */ - if (ctx->xof_state != XOF_STATE_SQUEEZE) { - len = ctx->block_size - ctx->bufsz; - memset(ctx->buf + ctx->bufsz, 0, len); - ctx->buf[ctx->bufsz] = padding; - ctx->buf[ctx->block_size - 1] |= 0x80; - s390x_kimd(ctx->buf, ctx->block_size, ctx->pad, ctx->A); - ctx->bufsz = 0; - /* reuse ctx->bufsz to count bytes squeezed from current sponge */ - } - if (ctx->bufsz != 0 || ctx->xof_state != XOF_STATE_SQUEEZE) { - len = ctx->block_size - ctx->bufsz; - if (outlen < len) - len = outlen; - memcpy(out, (char *)ctx->A + ctx->bufsz, len); - out += len; - outlen -= len; - ctx->bufsz += len; - if (ctx->bufsz == ctx->block_size) - ctx->bufsz = 0; - } - ctx->xof_state = XOF_STATE_SQUEEZE; - if (outlen == 0) - return 1; - s390x_klmd(NULL, 0, out, outlen, ctx->pad | S390X_KLMD_PS, ctx->A); - ctx->bufsz = outlen % ctx->block_size; - - return 1; -} - -static int s390x_keccak_squeeze(void *vctx, unsigned char *out, size_t outlen) -{ - return s390x_keccakc_squeeze(vctx, out, outlen, 0x01); -} - -static int s390x_kmac_squeeze(void *vctx, unsigned char *out, size_t outlen) -{ - return s390x_keccakc_squeeze(vctx, out, outlen, 0x04); -} - static PROV_SHA3_METHOD sha3_s390x_md = { s390x_sha3_absorb, - s390x_sha3_final, - NULL, -}; - -static PROV_SHA3_METHOD keccak_s390x_md = -{ - s390x_sha3_absorb, - s390x_keccak_final, - s390x_keccak_squeeze, + s390x_sha3_final }; static PROV_SHA3_METHOD shake_s390x_md = { s390x_sha3_absorb, - s390x_shake_final, - s390x_shake_squeeze, -}; - -static PROV_SHA3_METHOD kmac_s390x_md = -{ - s390x_sha3_absorb, - s390x_kmac_final, - s390x_kmac_squeeze, + s390x_shake_final }; -# define SHAKE_SET_MD(uname, typ) \ - if (S390_SHA3_CAPABLE(uname)) { \ - ctx->pad = S390X_##uname; \ - ctx->meth = typ##_s390x_md; \ - } else { \ - ctx->meth = shake_generic_md; \ - } - # define SHA3_SET_MD(uname, typ) \ if (S390_SHA3_CAPABLE(uname)) { \ ctx->pad = S390X_##uname; \ @@ -400,64 +196,8 @@ static PROV_SHA3_METHOD kmac_s390x_md = } else { \ ctx->meth = sha3_generic_md; \ } -# define KMAC_SET_MD(bitlen) \ - if (S390_SHA3_CAPABLE(SHAKE_##bitlen)) { \ - ctx->pad = S390X_SHAKE_##bitlen; \ - ctx->meth = kmac_s390x_md; \ - } else { \ - ctx->meth = sha3_generic_md; \ - } -#elif defined(__aarch64__) && defined(KECCAK1600_ASM) -# include "arm_arch.h" - -static sha3_absorb_fn armsha3_sha3_absorb; - -size_t SHA3_absorb_cext(uint64_t A[5][5], const unsigned char *inp, size_t len, - size_t r); -/*- - * Hardware-assisted ARMv8.2 SHA3 extension version of the absorb() - */ -static size_t armsha3_sha3_absorb(void *vctx, const void *inp, size_t len) -{ - KECCAK1600_CTX *ctx = vctx; - - return SHA3_absorb_cext(ctx->A, inp, len, ctx->block_size); -} - -static PROV_SHA3_METHOD sha3_ARMSHA3_md = -{ - armsha3_sha3_absorb, - generic_sha3_final -}; -static PROV_SHA3_METHOD shake_ARMSHA3_md = -{ - armsha3_sha3_absorb, - generic_sha3_final, - generic_sha3_squeeze -}; -# define SHAKE_SET_MD(uname, typ) \ - if (OPENSSL_armcap_P & ARMV8_HAVE_SHA3_AND_WORTH_USING) { \ - ctx->meth = shake_ARMSHA3_md; \ - } else { \ - ctx->meth = shake_generic_md; \ - } - -# define SHA3_SET_MD(uname, typ) \ - if (OPENSSL_armcap_P & ARMV8_HAVE_SHA3_AND_WORTH_USING) { \ - ctx->meth = sha3_ARMSHA3_md; \ - } else { \ - ctx->meth = sha3_generic_md; \ - } -# define KMAC_SET_MD(bitlen) \ - if (OPENSSL_armcap_P & ARMV8_HAVE_SHA3_AND_WORTH_USING) { \ - ctx->meth = sha3_ARMSHA3_md; \ - } else { \ - ctx->meth = sha3_generic_md; \ - } #else # define SHA3_SET_MD(uname, typ) ctx->meth = sha3_generic_md; -# define KMAC_SET_MD(bitlen) ctx->meth = sha3_generic_md; -# define SHAKE_SET_MD(uname, typ) ctx->meth = shake_generic_md; #endif /* S390_SHA3 */ #define SHA3_newctx(typ, uname, name, bitlen, pad) \ @@ -474,20 +214,6 @@ static void *name##_newctx(void *provctx) \ return ctx; \ } -#define SHAKE_newctx(typ, uname, name, bitlen, pad) \ -static OSSL_FUNC_digest_newctx_fn name##_newctx; \ -static void *name##_newctx(void *provctx) \ -{ \ - KECCAK1600_CTX *ctx = ossl_prov_is_running() ? OPENSSL_zalloc(sizeof(*ctx))\ - : NULL; \ - \ - if (ctx == NULL) \ - return NULL; \ - ossl_sha3_init(ctx, pad, bitlen); \ - SHAKE_SET_MD(uname, typ) \ - return ctx; \ -} - #define KMAC_newctx(uname, bitlen, pad) \ static OSSL_FUNC_digest_newctx_fn uname##_newctx; \ static void *uname##_newctx(void *provctx) \ @@ -498,7 +224,7 @@ static void *uname##_newctx(void *provctx) \ if (ctx == NULL) \ return NULL; \ ossl_keccak_kmac_init(ctx, pad, bitlen); \ - KMAC_SET_MD(bitlen) \ + ctx->meth = sha3_generic_md; \ return ctx; \ } @@ -519,7 +245,6 @@ const OSSL_DISPATCH ossl_##name##_functions[] = { \ #define PROV_FUNC_SHAKE_DIGEST(name, bitlen, blksize, dgstsize, flags) \ PROV_FUNC_SHA3_DIGEST_COMMON(name, bitlen, blksize, dgstsize, flags), \ - { OSSL_FUNC_DIGEST_SQUEEZE, (void (*)(void))shake_squeeze }, \ { OSSL_FUNC_DIGEST_INIT, (void (*)(void))keccak_init_params }, \ { OSSL_FUNC_DIGEST_SET_CTX_PARAMS, (void (*)(void))shake_set_ctx_params }, \ { OSSL_FUNC_DIGEST_SETTABLE_CTX_PARAMS, \ @@ -537,7 +262,7 @@ static void *keccak_dupctx(void *ctx) { KECCAK1600_CTX *in = (KECCAK1600_CTX *)ctx; KECCAK1600_CTX *ret = ossl_prov_is_running() ? OPENSSL_malloc(sizeof(*ret)) - : NULL; + : NULL; if (ret != NULL) *ret = *in; @@ -578,14 +303,8 @@ static int shake_set_ctx_params(void *vctx, const OSSL_PARAM params[]) SHA3_BLOCKSIZE(bitlen), SHA3_MDSIZE(bitlen), \ SHA3_FLAGS) -#define IMPLEMENT_KECCAK_functions(bitlen) \ - SHA3_newctx(keccak, KECCAK_##bitlen, keccak_##bitlen, bitlen, '\x01') \ - PROV_FUNC_SHA3_DIGEST(keccak_##bitlen, bitlen, \ - SHA3_BLOCKSIZE(bitlen), SHA3_MDSIZE(bitlen), \ - SHA3_FLAGS) - #define IMPLEMENT_SHAKE_functions(bitlen) \ - SHAKE_newctx(shake, SHAKE_##bitlen, shake_##bitlen, bitlen, '\x1f') \ + SHA3_newctx(shake, SHAKE_##bitlen, shake_##bitlen, bitlen, '\x1f') \ PROV_FUNC_SHAKE_DIGEST(shake_##bitlen, bitlen, \ SHA3_BLOCKSIZE(bitlen), SHA3_MDSIZE(bitlen), \ SHAKE_FLAGS) @@ -603,14 +322,6 @@ IMPLEMENT_SHA3_functions(256) IMPLEMENT_SHA3_functions(384) /* ossl_sha3_512_functions */ IMPLEMENT_SHA3_functions(512) -/* ossl_keccak_224_functions */ -IMPLEMENT_KECCAK_functions(224) -/* ossl_keccak_256_functions */ -IMPLEMENT_KECCAK_functions(256) -/* ossl_keccak_384_functions */ -IMPLEMENT_KECCAK_functions(384) -/* ossl_keccak_512_functions */ -IMPLEMENT_KECCAK_functions(512) /* ossl_shake_128_functions */ IMPLEMENT_SHAKE_functions(128) /* ossl_shake_256_functions */ diff --git a/openssl/src/providers/implementations/digests/sm3_prov.c b/openssl/src/providers/implementations/digests/sm3_prov.c index 9d6de5b6a..436a47683 100644 --- a/openssl/src/providers/implementations/digests/sm3_prov.c +++ b/openssl/src/providers/implementations/digests/sm3_prov.c @@ -6,13 +6,14 @@ * in the file LICENSE in the source distribution or at * https://www.openssl.org/source/license.html */ +#include "internal/deprecated.h" #include -#include "internal/sm3.h" +#include #include "prov/digestcommon.h" #include "prov/implementations.h" /* ossl_sm3_functions */ IMPLEMENT_digest_functions(sm3, SM3_CTX, SM3_CBLOCK, SM3_DIGEST_LENGTH, 0, - ossl_sm3_init, ossl_sm3_update, ossl_sm3_final) + SM3_Init, SM3_Update, SM3_Final) diff --git a/openssl/src/providers/implementations/digests/wp_prov.c b/openssl/src/providers/implementations/digests/wp_prov.c deleted file mode 100644 index 2af70b337..000000000 --- a/openssl/src/providers/implementations/digests/wp_prov.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * Whirlpool low level APIs are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - -#include -#include -#include "prov/digestcommon.h" -#include "prov/implementations.h" - -/* ossl_wp_functions */ -IMPLEMENT_digest_functions(wp, WHIRLPOOL_CTX, - WHIRLPOOL_BBLOCK / 8, WHIRLPOOL_DIGEST_LENGTH, 0, - WHIRLPOOL_Init, WHIRLPOOL_Update, WHIRLPOOL_Final) diff --git a/openssl/src/providers/implementations/encode_decode/decode_der2key.c b/openssl/src/providers/implementations/encode_decode/decode_der2key.c index b0d4e0ecf..cfe10fa41 100644 --- a/openssl/src/providers/implementations/encode_decode/decode_der2key.c +++ b/openssl/src/providers/implementations/encode_decode/decode_der2key.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -35,7 +35,6 @@ #include "prov/bio.h" #include "prov/implementations.h" #include "endecoder_local.h" -#include "internal/nelem.h" struct der2key_ctx_st; /* Forward declaration */ typedef int check_key_fn(void *, struct der2key_ctx_st *ctx); @@ -89,7 +88,6 @@ struct keytype_desc_st { */ struct der2key_ctx_st { PROV_CTX *provctx; - char propq[OSSL_MAX_PROPQUERY_SIZE]; const struct keytype_desc_st *desc; /* The selection that is passed to der2key_decode() */ int selection; @@ -107,10 +105,16 @@ static void *der2key_decode_p8(const unsigned char **input_der, const X509_ALGOR *alg = NULL; void *key = NULL; - if ((p8inf = d2i_PKCS8_PRIV_KEY_INFO(NULL, input_der, input_der_len)) != NULL - && PKCS8_pkey_get0(NULL, NULL, NULL, &alg, p8inf) - && OBJ_obj2nid(alg->algorithm) == ctx->desc->evp_type) - key = key_from_pkcs8(p8inf, PROV_LIBCTX_OF(ctx->provctx), ctx->propq); + if ((p8inf = d2i_PKCS8_PRIV_KEY_INFO(NULL, input_der, input_der_len)) + != NULL && PKCS8_pkey_get0(NULL, NULL, NULL, &alg, p8inf)) { + int nid = OBJ_obj2nid(alg->algorithm); + + /* Note: support sm2 p8 key with algorithm id-ecPublicKey */ + if (nid == ctx->desc->evp_type + || (nid == EVP_PKEY_EC + && ctx->desc->evp_type == EVP_PKEY_SM2)) + key = key_from_pkcs8(p8inf, PROV_LIBCTX_OF(ctx->provctx), NULL); + } PKCS8_PRIV_KEY_INFO_free(p8inf); return key; @@ -121,8 +125,6 @@ static void *der2key_decode_p8(const unsigned char **input_der, static OSSL_FUNC_decoder_freectx_fn der2key_freectx; static OSSL_FUNC_decoder_decode_fn der2key_decode; static OSSL_FUNC_decoder_export_object_fn der2key_export_object; -static OSSL_FUNC_decoder_settable_ctx_params_fn der2key_settable_ctx_params; -static OSSL_FUNC_decoder_set_ctx_params_fn der2key_set_ctx_params; static struct der2key_ctx_st * der2key_newctx(void *provctx, const struct keytype_desc_st *desc) @@ -136,28 +138,6 @@ der2key_newctx(void *provctx, const struct keytype_desc_st *desc) return ctx; } -static const OSSL_PARAM *der2key_settable_ctx_params(ossl_unused void *provctx) -{ - static const OSSL_PARAM settables[] = { - OSSL_PARAM_utf8_string(OSSL_DECODER_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_END - }; - return settables; -} - -static int der2key_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - struct der2key_ctx_st *ctx = vctx; - const OSSL_PARAM *p; - char *str = ctx->propq; - - p = OSSL_PARAM_locate_const(params, OSSL_DECODER_PARAM_PROPERTIES); - if (p != NULL && !OSSL_PARAM_get_utf8_string(p, &str, sizeof(ctx->propq))) - return 0; - - return 1; -} - static void der2key_freectx(void *vctx) { struct der2key_ctx_st *ctx = vctx; @@ -253,7 +233,7 @@ static int der2key_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, derp = der; if (ctx->desc->d2i_PUBKEY != NULL) key = ctx->desc->d2i_PUBKEY(NULL, &derp, der_len); - else if (ctx->desc->d2i_public_key != NULL) + else key = ctx->desc->d2i_public_key(NULL, &derp, der_len); if (key == NULL && ctx->selection != 0) { ERR_clear_last_mark(); @@ -342,14 +322,10 @@ static int der2key_export_object(void *vctx, void *keydata; if (reference_sz == sizeof(keydata) && export != NULL) { - int selection = ctx->selection; - - if (selection == 0) - selection = OSSL_KEYMGMT_SELECT_ALL; /* The contents of the reference is the address to our object */ keydata = *(void **)reference; - return export(keydata, selection, export_cb, export_cbarg); + return export(keydata, ctx->selection, export_cb, export_cbarg); } return 0; } @@ -404,7 +380,7 @@ static void *dsa_d2i_PKCS8(void **key, const unsigned char **der, long der_len, (key_from_pkcs8_t *)ossl_dsa_key_from_pkcs8); } -# define dsa_d2i_PUBKEY (d2i_of_void *)ossl_d2i_DSA_PUBKEY +# define dsa_d2i_PUBKEY (d2i_of_void *)d2i_DSA_PUBKEY # define dsa_free (free_key_fn *)DSA_free # define dsa_check NULL @@ -446,7 +422,6 @@ static void ec_adjust(void *key, struct der2key_ctx_st *ctx) ossl_ec_key_set0_libctx(key, PROV_LIBCTX_OF(ctx->provctx)); } -# ifndef OPENSSL_NO_ECX /* * ED25519, ED448, X25519, X448 only implement PKCS#8 and SubjectPublicKeyInfo, * so no d2i functions to be had. @@ -464,46 +439,45 @@ static void ecx_key_adjust(void *key, struct der2key_ctx_st *ctx) ossl_ecx_key_set0_libctx(key, PROV_LIBCTX_OF(ctx->provctx)); } -# define ed25519_evp_type EVP_PKEY_ED25519 -# define ed25519_d2i_private_key NULL -# define ed25519_d2i_public_key NULL -# define ed25519_d2i_key_params NULL -# define ed25519_d2i_PKCS8 ecx_d2i_PKCS8 -# define ed25519_d2i_PUBKEY (d2i_of_void *)ossl_d2i_ED25519_PUBKEY -# define ed25519_free (free_key_fn *)ossl_ecx_key_free -# define ed25519_check NULL -# define ed25519_adjust ecx_key_adjust - -# define ed448_evp_type EVP_PKEY_ED448 -# define ed448_d2i_private_key NULL -# define ed448_d2i_public_key NULL -# define ed448_d2i_key_params NULL -# define ed448_d2i_PKCS8 ecx_d2i_PKCS8 -# define ed448_d2i_PUBKEY (d2i_of_void *)ossl_d2i_ED448_PUBKEY -# define ed448_free (free_key_fn *)ossl_ecx_key_free -# define ed448_check NULL -# define ed448_adjust ecx_key_adjust - -# define x25519_evp_type EVP_PKEY_X25519 -# define x25519_d2i_private_key NULL -# define x25519_d2i_public_key NULL -# define x25519_d2i_key_params NULL -# define x25519_d2i_PKCS8 ecx_d2i_PKCS8 -# define x25519_d2i_PUBKEY (d2i_of_void *)ossl_d2i_X25519_PUBKEY -# define x25519_free (free_key_fn *)ossl_ecx_key_free -# define x25519_check NULL -# define x25519_adjust ecx_key_adjust - -# define x448_evp_type EVP_PKEY_X448 -# define x448_d2i_private_key NULL -# define x448_d2i_public_key NULL -# define x448_d2i_key_params NULL -# define x448_d2i_PKCS8 ecx_d2i_PKCS8 -# define x448_d2i_PUBKEY (d2i_of_void *)ossl_d2i_X448_PUBKEY -# define x448_free (free_key_fn *)ossl_ecx_key_free -# define x448_check NULL -# define x448_adjust ecx_key_adjust -# endif /* OPENSSL_NO_ECX */ +# define ed25519_evp_type EVP_PKEY_ED25519 +# define ed25519_d2i_private_key NULL +# define ed25519_d2i_public_key NULL +# define ed25519_d2i_key_params NULL +# define ed25519_d2i_PKCS8 ecx_d2i_PKCS8 +# define ed25519_d2i_PUBKEY (d2i_of_void *)ossl_d2i_ED25519_PUBKEY +# define ed25519_free (free_key_fn *)ossl_ecx_key_free +# define ed25519_check NULL +# define ed25519_adjust ecx_key_adjust + +# define ed448_evp_type EVP_PKEY_ED448 +# define ed448_d2i_private_key NULL +# define ed448_d2i_public_key NULL +# define ed448_d2i_key_params NULL +# define ed448_d2i_PKCS8 ecx_d2i_PKCS8 +# define ed448_d2i_PUBKEY (d2i_of_void *)ossl_d2i_ED448_PUBKEY +# define ed448_free (free_key_fn *)ossl_ecx_key_free +# define ed448_check NULL +# define ed448_adjust ecx_key_adjust + +# define x25519_evp_type EVP_PKEY_X25519 +# define x25519_d2i_private_key NULL +# define x25519_d2i_public_key NULL +# define x25519_d2i_key_params NULL +# define x25519_d2i_PKCS8 ecx_d2i_PKCS8 +# define x25519_d2i_PUBKEY (d2i_of_void *)ossl_d2i_X25519_PUBKEY +# define x25519_free (free_key_fn *)ossl_ecx_key_free +# define x25519_check NULL +# define x25519_adjust ecx_key_adjust + +# define x448_evp_type EVP_PKEY_X448 +# define x448_d2i_private_key NULL +# define x448_d2i_public_key NULL +# define x448_d2i_key_params NULL +# define x448_d2i_PKCS8 ecx_d2i_PKCS8 +# define x448_d2i_PUBKEY (d2i_of_void *)ossl_d2i_X448_PUBKEY +# define x448_free (free_key_fn *)ossl_ecx_key_free +# define x448_check NULL +# define x448_adjust ecx_key_adjust # ifndef OPENSSL_NO_SM2 # define sm2_evp_type EVP_PKEY_SM2 @@ -518,9 +492,13 @@ static void *sm2_d2i_PKCS8(void **key, const unsigned char **der, long der_len, (key_from_pkcs8_t *)ossl_ec_key_from_pkcs8); } +static int sm2_check(void *key, struct der2key_ctx_st *ctx) +{ + return (EC_KEY_get_flags(key) & EC_FLAG_SM2_RANGE) != 0; +} + # define sm2_d2i_PUBKEY (d2i_of_void *)d2i_EC_PUBKEY # define sm2_free (free_key_fn *)EC_KEY_free -# define sm2_check ec_check # define sm2_adjust ec_adjust # endif #endif @@ -780,11 +758,7 @@ static void rsa_adjust(void *key, struct der2key_ctx_st *ctx) (void (*)(void))der2key_decode }, \ { OSSL_FUNC_DECODER_EXPORT_OBJECT, \ (void (*)(void))der2key_export_object }, \ - { OSSL_FUNC_DECODER_SETTABLE_CTX_PARAMS, \ - (void (*)(void))der2key_settable_ctx_params }, \ - { OSSL_FUNC_DECODER_SET_CTX_PARAMS, \ - (void (*)(void))der2key_set_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } #ifndef OPENSSL_NO_DH @@ -808,7 +782,6 @@ MAKE_DECODER("EC", ec, ec, PrivateKeyInfo); MAKE_DECODER("EC", ec, ec, SubjectPublicKeyInfo); MAKE_DECODER("EC", ec, ec, type_specific_no_pub); MAKE_DECODER("EC", ec, ec, EC); -# ifndef OPENSSL_NO_ECX MAKE_DECODER("X25519", x25519, ecx, PrivateKeyInfo); MAKE_DECODER("X25519", x25519, ecx, SubjectPublicKeyInfo); MAKE_DECODER("X448", x448, ecx, PrivateKeyInfo); @@ -817,11 +790,10 @@ MAKE_DECODER("ED25519", ed25519, ecx, PrivateKeyInfo); MAKE_DECODER("ED25519", ed25519, ecx, SubjectPublicKeyInfo); MAKE_DECODER("ED448", ed448, ecx, PrivateKeyInfo); MAKE_DECODER("ED448", ed448, ecx, SubjectPublicKeyInfo); -# endif # ifndef OPENSSL_NO_SM2 MAKE_DECODER("SM2", sm2, ec, PrivateKeyInfo); MAKE_DECODER("SM2", sm2, ec, SubjectPublicKeyInfo); -MAKE_DECODER("SM2", sm2, sm2, type_specific_no_pub); +MAKE_DECODER("SM2", sm2, ec, type_specific_no_pub); # endif #endif MAKE_DECODER("RSA", rsa, rsa, PrivateKeyInfo); diff --git a/openssl/src/providers/implementations/encode_decode/decode_epki2pki.c b/openssl/src/providers/implementations/encode_decode/decode_epki2pki.c index 37d9bd185..9cea80b61 100644 --- a/openssl/src/providers/implementations/encode_decode/decode_epki2pki.c +++ b/openssl/src/providers/implementations/encode_decode/decode_epki2pki.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -26,15 +26,12 @@ static OSSL_FUNC_decoder_newctx_fn epki2pki_newctx; static OSSL_FUNC_decoder_freectx_fn epki2pki_freectx; static OSSL_FUNC_decoder_decode_fn epki2pki_decode; -static OSSL_FUNC_decoder_settable_ctx_params_fn epki2pki_settable_ctx_params; -static OSSL_FUNC_decoder_set_ctx_params_fn epki2pki_set_ctx_params; /* * Context used for EncryptedPrivateKeyInfo to PrivateKeyInfo decoding. */ struct epki2pki_ctx_st { PROV_CTX *provctx; - char propq[OSSL_MAX_PROPQUERY_SIZE]; }; static void *epki2pki_newctx(void *provctx) @@ -53,28 +50,6 @@ static void epki2pki_freectx(void *vctx) OPENSSL_free(ctx); } -static const OSSL_PARAM *epki2pki_settable_ctx_params(ossl_unused void *provctx) -{ - static const OSSL_PARAM settables[] = { - OSSL_PARAM_utf8_string(OSSL_DECODER_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_END - }; - return settables; -} - -static int epki2pki_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - struct epki2pki_ctx_st *ctx = vctx; - const OSSL_PARAM *p; - char *str = ctx->propq; - - p = OSSL_PARAM_locate_const(params, OSSL_DECODER_PARAM_PROPERTIES); - if (p != NULL && !OSSL_PARAM_get_utf8_string(p, &str, sizeof(ctx->propq))) - return 0; - - return 1; -} - /* * The selection parameter in epki2pki_decode() is not used by this function * because it's not relevant just to decode EncryptedPrivateKeyInfo to @@ -129,8 +104,7 @@ static int epki2pki_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, if (!PKCS12_pbe_crypt_ex(alg, pbuf, plen, oct->data, oct->length, &new_der, &new_der_len, 0, - PROV_LIBCTX_OF(ctx->provctx), - ctx->propq)) { + PROV_LIBCTX_OF(ctx->provctx), NULL)) { ok = 0; } else { OPENSSL_free(der); @@ -180,9 +154,5 @@ const OSSL_DISPATCH ossl_EncryptedPrivateKeyInfo_der_to_der_decoder_functions[] { OSSL_FUNC_DECODER_NEWCTX, (void (*)(void))epki2pki_newctx }, { OSSL_FUNC_DECODER_FREECTX, (void (*)(void))epki2pki_freectx }, { OSSL_FUNC_DECODER_DECODE, (void (*)(void))epki2pki_decode }, - { OSSL_FUNC_DECODER_SETTABLE_CTX_PARAMS, - (void (*)(void))epki2pki_settable_ctx_params }, - { OSSL_FUNC_DECODER_SET_CTX_PARAMS, - (void (*)(void))epki2pki_set_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/encode_decode/decode_msblob2key.c b/openssl/src/providers/implementations/encode_decode/decode_msblob2key.c index df327210f..501957fab 100644 --- a/openssl/src/providers/implementations/encode_decode/decode_msblob2key.c +++ b/openssl/src/providers/implementations/encode_decode/decode_msblob2key.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -79,18 +79,6 @@ static void msblob2key_freectx(void *vctx) OPENSSL_free(ctx); } -static int msblob2key_does_selection(void *provctx, int selection) -{ - if (selection == 0) - return 1; - - if ((selection & (OSSL_KEYMGMT_SELECT_PRIVATE_KEY - | OSSL_KEYMGMT_SELECT_PUBLIC_KEY)) != 0) - return 1; - - return 0; -} - static int msblob2key_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, OSSL_CALLBACK *data_cb, void *data_cbarg, OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg) @@ -132,8 +120,10 @@ static int msblob2key_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, goto next; } buf = OPENSSL_malloc(length); - if (buf == NULL) + if (buf == NULL) { + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto end; + } p = buf; if (BIO_read(in, buf, length) != (int)length) { ERR_raise(ERR_LIB_PEM, PEM_R_KEYBLOB_TOO_SHORT); @@ -221,14 +211,10 @@ msblob2key_export_object(void *vctx, void *keydata; if (reference_sz == sizeof(keydata) && export != NULL) { - int selection = ctx->selection; - - if (selection == 0) - selection = OSSL_KEYMGMT_SELECT_ALL; /* The contents of the reference is the address to our object */ keydata = *(void **)reference; - return export(keydata, selection, export_cb, export_cbarg); + return export(keydata, ctx->selection, export_cb, export_cbarg); } return 0; } @@ -274,13 +260,11 @@ static void rsa_adjust(void *key, struct msblob2key_ctx_st *ctx) (void (*)(void))msblob2##keytype##_newctx }, \ { OSSL_FUNC_DECODER_FREECTX, \ (void (*)(void))msblob2key_freectx }, \ - { OSSL_FUNC_DECODER_DOES_SELECTION, \ - (void (*)(void))msblob2key_does_selection }, \ { OSSL_FUNC_DECODER_DECODE, \ (void (*)(void))msblob2key_decode }, \ { OSSL_FUNC_DECODER_EXPORT_OBJECT, \ (void (*)(void))msblob2key_export_object }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } #ifndef OPENSSL_NO_DSA diff --git a/openssl/src/providers/implementations/encode_decode/decode_pem2der.c b/openssl/src/providers/implementations/encode_decode/decode_pem2der.c index ea6eb7f96..1f7ddb562 100644 --- a/openssl/src/providers/implementations/encode_decode/decode_pem2der.c +++ b/openssl/src/providers/implementations/encode_decode/decode_pem2der.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -119,6 +119,7 @@ static int pem2der_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, { PEM_STRING_DSAPARAMS, OSSL_OBJECT_PKEY, "DSA", "type-specific" }, { PEM_STRING_ECPRIVATEKEY, OSSL_OBJECT_PKEY, "EC", "type-specific" }, { PEM_STRING_ECPARAMETERS, OSSL_OBJECT_PKEY, "EC", "type-specific" }, + { PEM_STRING_SM2PRIVATEKEY, OSSL_OBJECT_PKEY, "SM2", "type-specific" }, { PEM_STRING_SM2PARAMETERS, OSSL_OBJECT_PKEY, "SM2", "type-specific" }, { PEM_STRING_RSA, OSSL_OBJECT_PKEY, "RSA", "type-specific" }, { PEM_STRING_RSA_PUBLIC, OSSL_OBJECT_PKEY, "RSA", "type-specific" }, @@ -216,5 +217,5 @@ const OSSL_DISPATCH ossl_pem_to_der_decoder_functions[] = { { OSSL_FUNC_DECODER_NEWCTX, (void (*)(void))pem2der_newctx }, { OSSL_FUNC_DECODER_FREECTX, (void (*)(void))pem2der_freectx }, { OSSL_FUNC_DECODER_DECODE, (void (*)(void))pem2der_decode }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/encode_decode/decode_pvk2key.c b/openssl/src/providers/implementations/encode_decode/decode_pvk2key.c index ea4585d93..c6424165b 100644 --- a/openssl/src/providers/implementations/encode_decode/decode_pvk2key.c +++ b/openssl/src/providers/implementations/encode_decode/decode_pvk2key.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,7 +24,6 @@ #include /* For public PVK functions */ #include #include "internal/passphrase.h" -#include "internal/sizes.h" #include "crypto/pem.h" /* For internal PVK and "blob" headers */ #include "crypto/rsa.h" #include "prov/bio.h" @@ -50,15 +49,12 @@ struct keytype_desc_st { static OSSL_FUNC_decoder_freectx_fn pvk2key_freectx; static OSSL_FUNC_decoder_decode_fn pvk2key_decode; static OSSL_FUNC_decoder_export_object_fn pvk2key_export_object; -static OSSL_FUNC_decoder_settable_ctx_params_fn pvk2key_settable_ctx_params; -static OSSL_FUNC_decoder_set_ctx_params_fn pvk2key_set_ctx_params; /* * Context used for DER to key decoding. */ struct pvk2key_ctx_st { PROV_CTX *provctx; - char propq[OSSL_MAX_PROPQUERY_SIZE]; const struct keytype_desc_st *desc; /* The selection that is passed to der2key_decode() */ int selection; @@ -83,39 +79,6 @@ static void pvk2key_freectx(void *vctx) OPENSSL_free(ctx); } -static const OSSL_PARAM *pvk2key_settable_ctx_params(ossl_unused void *provctx) -{ - static const OSSL_PARAM settables[] = { - OSSL_PARAM_utf8_string(OSSL_DECODER_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_END, - }; - return settables; -} - -static int pvk2key_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - struct pvk2key_ctx_st *ctx = vctx; - const OSSL_PARAM *p; - char *str = ctx->propq; - - p = OSSL_PARAM_locate_const(params, OSSL_DECODER_PARAM_PROPERTIES); - if (p != NULL && !OSSL_PARAM_get_utf8_string(p, &str, sizeof(ctx->propq))) - return 0; - - return 1; -} - -static int pvk2key_does_selection(void *provctx, int selection) -{ - if (selection == 0) - return 1; - - if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) - return 1; - - return 0; -} - static int pvk2key_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, OSSL_CALLBACK *data_cb, void *data_cbarg, OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg) @@ -141,8 +104,7 @@ static int pvk2key_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, goto end; key = ctx->desc->read_private_key(in, ossl_pw_pvk_password, &pwdata, - PROV_LIBCTX_OF(ctx->provctx), - ctx->propq); + PROV_LIBCTX_OF(ctx->provctx), NULL); /* * Because the PVK API doesn't have a separate decrypt call, we need @@ -217,14 +179,10 @@ static int pvk2key_export_object(void *vctx, void *keydata; if (reference_sz == sizeof(keydata) && export != NULL) { - int selection = ctx->selection; - - if (selection == 0) - selection = OSSL_KEYMGMT_SELECT_ALL; /* The contents of the reference is the address to our object */ keydata = *(void **)reference; - return export(keydata, selection, export_cb, export_cbarg); + return export(keydata, ctx->selection, export_cb, export_cbarg); } return 0; } @@ -268,17 +226,11 @@ static void rsa_adjust(void *key, struct pvk2key_ctx_st *ctx) (void (*)(void))pvk2##keytype##_newctx }, \ { OSSL_FUNC_DECODER_FREECTX, \ (void (*)(void))pvk2key_freectx }, \ - { OSSL_FUNC_DECODER_DOES_SELECTION, \ - (void (*)(void))pvk2key_does_selection }, \ { OSSL_FUNC_DECODER_DECODE, \ (void (*)(void))pvk2key_decode }, \ { OSSL_FUNC_DECODER_EXPORT_OBJECT, \ (void (*)(void))pvk2key_export_object }, \ - { OSSL_FUNC_DECODER_SETTABLE_CTX_PARAMS, \ - (void (*)(void))pvk2key_settable_ctx_params }, \ - { OSSL_FUNC_DECODER_SET_CTX_PARAMS, \ - (void (*)(void))pvk2key_set_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } #ifndef OPENSSL_NO_DSA diff --git a/openssl/src/providers/implementations/encode_decode/decode_spki2typespki.c b/openssl/src/providers/implementations/encode_decode/decode_spki2typespki.c index 7074be93d..a5dbbb31a 100644 --- a/openssl/src/providers/implementations/encode_decode/decode_spki2typespki.c +++ b/openssl/src/providers/implementations/encode_decode/decode_spki2typespki.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,8 +23,6 @@ static OSSL_FUNC_decoder_newctx_fn spki2typespki_newctx; static OSSL_FUNC_decoder_freectx_fn spki2typespki_freectx; static OSSL_FUNC_decoder_decode_fn spki2typespki_decode; -static OSSL_FUNC_decoder_settable_ctx_params_fn spki2typespki_settable_ctx_params; -static OSSL_FUNC_decoder_set_ctx_params_fn spki2typespki_set_ctx_params; /* * Context used for SubjectPublicKeyInfo to Type specific SubjectPublicKeyInfo @@ -32,7 +30,6 @@ static OSSL_FUNC_decoder_set_ctx_params_fn spki2typespki_set_ctx_params; */ struct spki2typespki_ctx_st { PROV_CTX *provctx; - char propq[OSSL_MAX_PROPQUERY_SIZE]; }; static void *spki2typespki_newctx(void *provctx) @@ -51,28 +48,6 @@ static void spki2typespki_freectx(void *vctx) OPENSSL_free(ctx); } -static const OSSL_PARAM *spki2typespki_settable_ctx_params(ossl_unused void *provctx) -{ - static const OSSL_PARAM settables[] = { - OSSL_PARAM_utf8_string(OSSL_DECODER_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_END - }; - return settables; -} - -static int spki2typespki_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - struct spki2typespki_ctx_st *ctx = vctx; - const OSSL_PARAM *p; - char *str = ctx->propq; - - p = OSSL_PARAM_locate_const(params, OSSL_DECODER_PARAM_PROPERTIES); - if (p != NULL && !OSSL_PARAM_get_utf8_string(p, &str, sizeof(ctx->propq))) - return 0; - - return 1; -} - static int spki2typespki_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, OSSL_CALLBACK *data_cb, void *data_cbarg, OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg) @@ -92,8 +67,8 @@ static int spki2typespki_decode(void *vctx, OSSL_CORE_BIO *cin, int selection, return 1; derp = der; xpub = ossl_d2i_X509_PUBKEY_INTERNAL((const unsigned char **)&derp, len, - PROV_LIBCTX_OF(ctx->provctx), - ctx->propq); + PROV_LIBCTX_OF(ctx->provctx)); + if (xpub == NULL) { /* We return "empty handed". This is not an error. */ @@ -145,9 +120,5 @@ const OSSL_DISPATCH ossl_SubjectPublicKeyInfo_der_to_der_decoder_functions[] = { { OSSL_FUNC_DECODER_NEWCTX, (void (*)(void))spki2typespki_newctx }, { OSSL_FUNC_DECODER_FREECTX, (void (*)(void))spki2typespki_freectx }, { OSSL_FUNC_DECODER_DECODE, (void (*)(void))spki2typespki_decode }, - { OSSL_FUNC_DECODER_SETTABLE_CTX_PARAMS, - (void (*)(void))spki2typespki_settable_ctx_params }, - { OSSL_FUNC_DECODER_SET_CTX_PARAMS, - (void (*)(void))spki2typespki_set_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/encode_decode/encode_key2any.c b/openssl/src/providers/implementations/encode_decode/encode_key2any.c index 2b39bf039..2452b9c0e 100644 --- a/openssl/src/providers/implementations/encode_decode/encode_key2any.c +++ b/openssl/src/providers/implementations/encode_decode/encode_key2any.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -67,7 +67,7 @@ typedef int write_bio_of_void_fn(BIO *bp, const void *x); /* Free the blob allocated during key_to_paramstring_fn */ static void free_asn1_data(int type, void *data) { - switch (type) { + switch(type) { case V_ASN1_OBJECT: ASN1_OBJECT_free(data); break; @@ -91,7 +91,7 @@ static PKCS8_PRIV_KEY_INFO *key_to_p8info(const void *key, int key_nid, || (derlen = k2d(key, &der)) <= 0 || !PKCS8_pkey_set0(p8info, OBJ_nid2obj(key_nid), 0, params_type, params, der, derlen)) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); PKCS8_PRIV_KEY_INFO_free(p8info); OPENSSL_free(der); p8info = NULL; @@ -154,7 +154,7 @@ static X509_PUBKEY *key_to_pubkey(const void *key, int key_nid, || (derlen = k2d(key, &der)) <= 0 || !X509_PUBKEY_set0_param(xpk, OBJ_nid2obj(key_nid), params_type, params, der, derlen)) { - ERR_raise(ERR_LIB_PROV, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); X509_PUBKEY_free(xpk); OPENSSL_free(der); xpk = NULL; @@ -168,7 +168,7 @@ static X509_PUBKEY *key_to_pubkey(const void *key, int key_nid, * EncryptedPrivateKeyInfo structure (defined by PKCS#8). They require * that there's an intent to encrypt, anything else is an error. * - * key_to_pki_* primarily produce encoded output with the private key data + * key_to_pki_* primarly produce encoded output with the private key data * in a PrivateKeyInfo structure (also defined by PKCS#8). However, if * there is an intent to encrypt the data, the corresponding key_to_epki_* * function is used instead. @@ -380,7 +380,7 @@ static int key_to_type_specific_der_bio(BIO *out, const void *key, int ret; if ((derlen = k2d(key, &der)) <= 0) { - ERR_raise(ERR_LIB_PROV, ERR_R_PROV_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; } @@ -446,7 +446,7 @@ static int prepare_dh_params(const void *dh, int nid, int save, ASN1_STRING *params = ASN1_STRING_new(); if (params == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; } @@ -456,7 +456,7 @@ static int prepare_dh_params(const void *dh, int nid, int save, params->length = i2d_DHparams(dh, ¶ms->data); if (params->length <= 0) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); ASN1_STRING_free(params); return 0; } @@ -535,8 +535,6 @@ static int dh_check_key_type(const void *dh, int expected_type) # define dh_evp_type EVP_PKEY_DH # define dhx_evp_type EVP_PKEY_DHX -# define dh_input_type "DH" -# define dhx_input_type "DHX" # define dh_pem_type "DH" # define dhx_pem_type "X9.42 DH" #endif @@ -550,14 +548,14 @@ static int encode_dsa_params(const void *dsa, int nid, ASN1_STRING *params = ASN1_STRING_new(); if (params == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; } params->length = i2d_DSAparams(dsa, ¶ms->data); if (params->length <= 0) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); ASN1_STRING_free(params); return 0; } @@ -632,7 +630,6 @@ static int dsa_pki_priv_to_der(const void *dsa, unsigned char **pder) # define dsa_check_key_type NULL # define dsa_evp_type EVP_PKEY_DSA -# define dsa_input_type "DSA" # define dsa_pem_type "DSA" #endif @@ -645,13 +642,13 @@ static int prepare_ec_explicit_params(const void *eckey, ASN1_STRING *params = ASN1_STRING_new(); if (params == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; } params->length = i2d_ECParameters(eckey, ¶ms->data); if (params->length <= 0) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); ASN1_STRING_free(params); return 0; } @@ -736,27 +733,18 @@ static int ec_pki_priv_to_der(const void *veckey, unsigned char **pder) # define ec_check_key_type NULL # define ec_evp_type EVP_PKEY_EC -# define ec_input_type "EC" # define ec_pem_type "EC" # ifndef OPENSSL_NO_SM2 -/* - * Albeit SM2 is a slightly different algorithm than ECDSA, the key type - * encoding (in all places where an AlgorithmIdentifier is produced, such - * as PrivateKeyInfo and SubjectPublicKeyInfo) is the same as for ECC keys - * according to the example in GM/T 0015-2012, appendix D.2. - * This leaves the distinction of SM2 keys to the EC group (which is found - * in AlgorithmIdentified.params). - */ -# define sm2_evp_type ec_evp_type -# define sm2_input_type "SM2" -# define sm2_pem_type "SM2" +/* Keep SM2 wrap in EC to be compatible with common implementations */ +# define sm2_evp_type EVP_PKEY_EC +# define sm2_pem_type "EC" # endif #endif /* ---------------------------------------------------------------------- */ -#ifndef OPENSSL_NO_ECX +#ifndef OPENSSL_NO_EC # define prepare_ecx_params NULL static int ecx_spki_pub_to_der(const void *vecxkey, unsigned char **pder) @@ -770,8 +758,10 @@ static int ecx_spki_pub_to_der(const void *vecxkey, unsigned char **pder) } keyblob = OPENSSL_memdup(ecxkey->pubkey, ecxkey->keylen); - if (keyblob == NULL) + if (keyblob == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } *pder = keyblob; return ecxkey->keylen; @@ -794,7 +784,7 @@ static int ecx_pki_priv_to_der(const void *vecxkey, unsigned char **pder) keybloblen = i2d_ASN1_OCTET_STRING(&oct, pder); if (keybloblen < 0) { - ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; } @@ -814,10 +804,6 @@ static int ecx_pki_priv_to_der(const void *vecxkey, unsigned char **pder) # define ed448_evp_type EVP_PKEY_ED448 # define x25519_evp_type EVP_PKEY_X25519 # define x448_evp_type EVP_PKEY_X448 -# define ed25519_input_type "ED25519" -# define ed448_input_type "ED448" -# define x25519_input_type "X25519" -# define x448_input_type "X448" # define ed25519_pem_type "ED25519" # define ed448_pem_type "ED448" # define x25519_pem_type "X25519" @@ -864,17 +850,14 @@ static int prepare_rsa_params(const void *rsa, int nid, int save, case 1: if ((str = OPENSSL_malloc(str_sz)) == NULL || !WPACKET_init_der(&pkt, str, str_sz)) { - WPACKET_cleanup(&pkt); goto err; } break; } if (!ossl_DER_w_RSASSA_PSS_params(&pkt, -1, pss) || !WPACKET_finish(&pkt) - || !WPACKET_get_total_written(&pkt, &str_sz)) { - WPACKET_cleanup(&pkt); + || !WPACKET_get_total_written(&pkt, &str_sz)) goto err; - } WPACKET_cleanup(&pkt); /* @@ -930,8 +913,6 @@ static int rsa_check_key_type(const void *rsa, int expected_type) #define rsa_evp_type EVP_PKEY_RSA #define rsapss_evp_type EVP_PKEY_RSA_PSS -#define rsa_input_type "RSA" -#define rsapss_input_type "RSA-PSS" #define rsa_pem_type "RSA" #define rsapss_pem_type "RSA-PSS" @@ -1304,7 +1285,7 @@ static int key2any_encode(struct key2any_ctx_st *ctx, OSSL_CORE_BIO *cout, (void (*)(void))impl##_to_##kind##_##output##_free_object }, \ { OSSL_FUNC_ENCODER_ENCODE, \ (void (*)(void))impl##_to_##kind##_##output##_encode }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } /* @@ -1402,7 +1383,6 @@ MAKE_ENCODER(sm2, ec, EVP_PKEY_EC, PrivateKeyInfo, pem); MAKE_ENCODER(sm2, ec, EVP_PKEY_EC, SubjectPublicKeyInfo, der); MAKE_ENCODER(sm2, ec, EVP_PKEY_EC, SubjectPublicKeyInfo, pem); # endif -# ifndef OPENSSL_NO_ECX MAKE_ENCODER(ed25519, ecx, EVP_PKEY_ED25519, EncryptedPrivateKeyInfo, der); MAKE_ENCODER(ed25519, ecx, EVP_PKEY_ED25519, EncryptedPrivateKeyInfo, pem); MAKE_ENCODER(ed25519, ecx, EVP_PKEY_ED25519, PrivateKeyInfo, der); @@ -1427,7 +1407,6 @@ MAKE_ENCODER(x448, ecx, EVP_PKEY_ED448, PrivateKeyInfo, der); MAKE_ENCODER(x448, ecx, EVP_PKEY_ED448, PrivateKeyInfo, pem); MAKE_ENCODER(x448, ecx, EVP_PKEY_ED448, SubjectPublicKeyInfo, der); MAKE_ENCODER(x448, ecx, EVP_PKEY_ED448, SubjectPublicKeyInfo, pem); -# endif #endif /* diff --git a/openssl/src/providers/implementations/encode_decode/encode_key2blob.c b/openssl/src/providers/implementations/encode_decode/encode_key2blob.c index 29e72faa6..550bceb09 100644 --- a/openssl/src/providers/implementations/encode_decode/encode_key2blob.c +++ b/openssl/src/providers/implementations/encode_decode/encode_key2blob.c @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -168,7 +168,7 @@ static int key2blob_encode(void *vctx, const void *key, int selection, (void (*)(void))impl##2blob_free_object }, \ { OSSL_FUNC_ENCODER_ENCODE, \ (void (*)(void))impl##2blob_encode }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } #ifndef OPENSSL_NO_EC diff --git a/openssl/src/providers/implementations/encode_decode/encode_key2ms.c b/openssl/src/providers/implementations/encode_decode/encode_key2ms.c index 1f21a5129..fe8c2dce4 100644 --- a/openssl/src/providers/implementations/encode_decode/encode_key2ms.c +++ b/openssl/src/providers/implementations/encode_decode/encode_key2ms.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -222,7 +222,7 @@ static int key2pvk_encode(void *vctx, const void *key, int selection, (void (*)(void))impl##2##output##_free_object }, \ { OSSL_FUNC_ENCODER_ENCODE, \ (void (*)(void))impl##2##output##_encode }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } #ifndef OPENSSL_NO_DSA diff --git a/openssl/src/providers/implementations/encode_decode/encode_key2text.c b/openssl/src/providers/implementations/encode_decode/encode_key2text.c index c0c292328..4c6b41f78 100644 --- a/openssl/src/providers/implementations/encode_decode/encode_key2text.c +++ b/openssl/src/providers/implementations/encode_decode/encode_key2text.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -109,7 +109,7 @@ static int print_labeled_bignum(BIO *out, const char *label, const BIGNUM *bn) if ((bytes % 15) == 0 && bytes > 0) { if (BIO_printf(out, ":\n%s", spaces) <= 0) goto err; - use_sep = 0; /* The first byte on the next line doesn't have a : */ + use_sep = 0; /* The first byte on the next line doesnt have a : */ } if (BIO_printf(out, "%s%c%c", use_sep ? ":" : "", tolower(p[0]), tolower(p[1])) <= 0) @@ -220,7 +220,6 @@ static int dh_to_text(BIO *out, const void *key, int selection) const BIGNUM *priv_key = NULL, *pub_key = NULL; const FFC_PARAMS *params = NULL; const BIGNUM *p = NULL; - long length; if (out == NULL || dh == NULL) { ERR_raise(ERR_LIB_PROV, ERR_R_PASSED_NULL_PARAMETER); @@ -241,7 +240,7 @@ static int dh_to_text(BIO *out, const void *key, int selection) return 0; } } - if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) { + if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) { pub_key = DH_get0_pub_key(dh); if (pub_key == NULL) { ERR_raise(ERR_LIB_PROV, PROV_R_NOT_A_PUBLIC_KEY); @@ -273,11 +272,6 @@ static int dh_to_text(BIO *out, const void *key, int selection) if (params != NULL && !ffc_params_to_text(out, params)) return 0; - length = DH_get_length(dh); - if (length > 0 - && BIO_printf(out, "recommended-private-length: %ld bits\n", - length) <= 0) - return 0; return 1; } @@ -316,7 +310,7 @@ static int dsa_to_text(BIO *out, const void *key, int selection) return 0; } } - if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) { + if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) { pub_key = DSA_get0_pub_key(dsa); if (pub_key == NULL) { ERR_raise(ERR_LIB_PROV, PROV_R_NOT_A_PUBLIC_KEY); @@ -526,7 +520,7 @@ static int ec_to_text(BIO *out, const void *key, int selection) if (priv_len == 0) goto err; } - if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) { + if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) { const EC_POINT *pub_pt = EC_KEY_get0_public_key(ec); if (pub_pt == NULL) { @@ -566,7 +560,7 @@ static int ec_to_text(BIO *out, const void *key, int selection) /* ---------------------------------------------------------------------- */ -#ifndef OPENSSL_NO_ECX +#ifndef OPENSSL_NO_EC static int ecx_to_text(BIO *out, const void *key, int selection) { const ECX_KEY *ecx = key; @@ -577,31 +571,26 @@ static int ecx_to_text(BIO *out, const void *key, int selection) return 0; } - switch (ecx->type) { - case ECX_KEY_TYPE_X25519: - type_label = "X25519"; - break; - case ECX_KEY_TYPE_X448: - type_label = "X448"; - break; - case ECX_KEY_TYPE_ED25519: - type_label = "ED25519"; - break; - case ECX_KEY_TYPE_ED448: - type_label = "ED448"; - break; - } - if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) { if (ecx->privkey == NULL) { ERR_raise(ERR_LIB_PROV, PROV_R_NOT_A_PRIVATE_KEY); return 0; } - if (BIO_printf(out, "%s Private-Key:\n", type_label) <= 0) - return 0; - if (!print_labeled_buf(out, "priv:", ecx->privkey, ecx->keylen)) - return 0; + switch (ecx->type) { + case ECX_KEY_TYPE_X25519: + type_label = "X25519 Private-Key"; + break; + case ECX_KEY_TYPE_X448: + type_label = "X448 Private-Key"; + break; + case ECX_KEY_TYPE_ED25519: + type_label = "ED25519 Private-Key"; + break; + case ECX_KEY_TYPE_ED448: + type_label = "ED448 Private-Key"; + break; + } } else if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) { /* ecx->pubkey is an array, not a pointer... */ if (!ecx->haspubkey) { @@ -609,11 +598,29 @@ static int ecx_to_text(BIO *out, const void *key, int selection) return 0; } - if (BIO_printf(out, "%s Public-Key:\n", type_label) <= 0) - return 0; + switch (ecx->type) { + case ECX_KEY_TYPE_X25519: + type_label = "X25519 Public-Key"; + break; + case ECX_KEY_TYPE_X448: + type_label = "X448 Public-Key"; + break; + case ECX_KEY_TYPE_ED25519: + type_label = "ED25519 Public-Key"; + break; + case ECX_KEY_TYPE_ED448: + type_label = "ED448 Public-Key"; + break; + } } - if (!print_labeled_buf(out, "pub:", ecx->pubkey, ecx->keylen)) + if (BIO_printf(out, "%s:\n", type_label) <= 0) + return 0; + if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0 + && !print_labeled_buf(out, "priv:", ecx->privkey, ecx->keylen)) + return 0; + if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0 + && !print_labeled_buf(out, "pub:", ecx->pubkey, ecx->keylen)) return 0; return 1; @@ -651,7 +658,7 @@ static int rsa_to_text(BIO *out, const void *key, int selection) coeffs = sk_BIGNUM_const_new_null(); if (factors == NULL || exps == NULL || coeffs == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -854,7 +861,7 @@ static int key2text_encode(void *vctx, const void *key, int selection, (void (*)(void))impl##2text_free_object }, \ { OSSL_FUNC_ENCODER_ENCODE, \ (void (*)(void))impl##2text_encode }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } #ifndef OPENSSL_NO_DH @@ -869,12 +876,10 @@ MAKE_TEXT_ENCODER(ec, ec); # ifndef OPENSSL_NO_SM2 MAKE_TEXT_ENCODER(sm2, ec); # endif -# ifndef OPENSSL_NO_ECX MAKE_TEXT_ENCODER(ed25519, ecx); MAKE_TEXT_ENCODER(ed448, ecx); MAKE_TEXT_ENCODER(x25519, ecx); MAKE_TEXT_ENCODER(x448, ecx); -# endif #endif MAKE_TEXT_ENCODER(rsa, rsa); MAKE_TEXT_ENCODER(rsapss, rsa); diff --git a/openssl/src/providers/implementations/exchange/dh_exch.c b/openssl/src/providers/implementations/exchange/dh_exch.c index 20b8fa007..1d8a2e27b 100644 --- a/openssl/src/providers/implementations/exchange/dh_exch.c +++ b/openssl/src/providers/implementations/exchange/dh_exch.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -194,8 +194,10 @@ static int dh_X9_42_kdf_derive(void *vpdhctx, unsigned char *secret, } if (!dh_plain_derive(pdhctx, NULL, &stmplen, 0, 1)) return 0; - if ((stmp = OPENSSL_secure_malloc(stmplen)) == NULL) + if ((stmp = OPENSSL_secure_malloc(stmplen)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } if (!dh_plain_derive(pdhctx, stmp, &stmplen, stmplen, 1)) goto err; @@ -349,13 +351,12 @@ static int dh_set_ctx_params(void *vpdhctx, const OSSL_PARAM params[]) EVP_MD_free(pdhctx->kdf_md); pdhctx->kdf_md = EVP_MD_fetch(pdhctx->libctx, name, mdprops); - if (pdhctx->kdf_md == NULL) - return 0; if (!ossl_digest_is_allowed(pdhctx->libctx, pdhctx->kdf_md)) { EVP_MD_free(pdhctx->kdf_md); pdhctx->kdf_md = NULL; - return 0; } + if (pdhctx->kdf_md == NULL) + return 0; } p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_KDF_OUTLEN); @@ -472,7 +473,7 @@ static int dh_get_ctx_params(void *vpdhctx, OSSL_PARAM params[]) if (p != NULL && !OSSL_PARAM_set_utf8_string(p, pdhctx->kdf_md == NULL ? "" - : EVP_MD_get0_name(pdhctx->kdf_md))) { + : EVP_MD_get0_name(pdhctx->kdf_md))){ return 0; } @@ -507,5 +508,5 @@ const OSSL_DISPATCH ossl_dh_keyexch_functions[] = { { OSSL_FUNC_KEYEXCH_GET_CTX_PARAMS, (void (*)(void))dh_get_ctx_params }, { OSSL_FUNC_KEYEXCH_GETTABLE_CTX_PARAMS, (void (*)(void))dh_gettable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/exchange/ecdh_exch.c b/openssl/src/providers/implementations/exchange/ecdh_exch.c index 5b8412aba..35d665fb9 100644 --- a/openssl/src/providers/implementations/exchange/ecdh_exch.c +++ b/openssl/src/providers/implementations/exchange/ecdh_exch.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -126,7 +126,7 @@ int ecdh_match_params(const EC_KEY *priv, const EC_KEY *peer) ctx = BN_CTX_new_ex(ossl_ec_key_get_libctx(priv)); if (ctx == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_BN_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; } ret = group_priv != NULL @@ -283,13 +283,12 @@ int ecdh_set_ctx_params(void *vpecdhctx, const OSSL_PARAM params[]) EVP_MD_free(pectx->kdf_md); pectx->kdf_md = EVP_MD_fetch(pectx->libctx, name, mdprops); - if (pectx->kdf_md == NULL) - return 0; if (!ossl_digest_is_allowed(pectx->libctx, pectx->kdf_md)) { EVP_MD_free(pectx->kdf_md); pectx->kdf_md = NULL; - return 0; } + if (pectx->kdf_md == NULL) + return 0; } p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_KDF_OUTLEN); @@ -378,7 +377,7 @@ int ecdh_get_ctx_params(void *vpecdhctx, OSSL_PARAM params[]) if (p != NULL && !OSSL_PARAM_set_utf8_string(p, pectx->kdf_md == NULL ? "" - : EVP_MD_get0_name(pectx->kdf_md))) { + : EVP_MD_get0_name(pectx->kdf_md))){ return 0; } @@ -451,7 +450,7 @@ int ecdh_plain_derive(void *vpecdhctx, unsigned char *secret, } if ((group = EC_KEY_get0_group(pecdhctx->k)) == NULL - || (cofactor = EC_GROUP_get0_cofactor(group)) == NULL) + || (cofactor = EC_GROUP_get0_cofactor(group)) == NULL ) return 0; /* @@ -524,8 +523,10 @@ int ecdh_X9_63_kdf_derive(void *vpecdhctx, unsigned char *secret, } if (!ecdh_plain_derive(vpecdhctx, NULL, &stmplen, 0)) return 0; - if ((stmp = OPENSSL_secure_malloc(stmplen)) == NULL) + if ((stmp = OPENSSL_secure_malloc(stmplen)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } if (!ecdh_plain_derive(vpecdhctx, stmp, &stmplen, stmplen)) goto err; @@ -575,5 +576,5 @@ const OSSL_DISPATCH ossl_ecdh_keyexch_functions[] = { { OSSL_FUNC_KEYEXCH_GET_CTX_PARAMS, (void (*)(void))ecdh_get_ctx_params }, { OSSL_FUNC_KEYEXCH_GETTABLE_CTX_PARAMS, (void (*)(void))ecdh_gettable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/exchange/ecx_exch.c b/openssl/src/providers/implementations/exchange/ecx_exch.c index ccf39462e..2ba9090c8 100644 --- a/openssl/src/providers/implementations/exchange/ecx_exch.c +++ b/openssl/src/providers/implementations/exchange/ecx_exch.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,6 +17,9 @@ #include "crypto/ecx.h" #include "prov/implementations.h" #include "prov/providercommon.h" +#ifdef S390X_EC_ASM +# include "s390x_arch.h" +#endif static OSSL_FUNC_keyexch_newctx_fn x25519_newctx; static OSSL_FUNC_keyexch_newctx_fn x448_newctx; @@ -46,8 +49,10 @@ static void *ecx_newctx(void *provctx, size_t keylen) return NULL; ctx = OPENSSL_zalloc(sizeof(PROV_ECX_CTX)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->keylen = keylen; @@ -115,8 +120,65 @@ static int ecx_derive(void *vecxctx, unsigned char *secret, size_t *secretlen, if (!ossl_prov_is_running()) return 0; - return ossl_ecx_compute_key(ecxctx->peerkey, ecxctx->key, ecxctx->keylen, - secret, secretlen, outlen); + + if (ecxctx->key == NULL + || ecxctx->key->privkey == NULL + || ecxctx->peerkey == NULL) { + ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_KEY); + return 0; + } + + if (!ossl_assert(ecxctx->keylen == X25519_KEYLEN + || ecxctx->keylen == X448_KEYLEN)) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); + return 0; + } + + if (secret == NULL) { + *secretlen = ecxctx->keylen; + return 1; + } + if (outlen < ecxctx->keylen) { + ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); + return 0; + } + + if (ecxctx->keylen == X25519_KEYLEN) { +#ifdef S390X_EC_ASM + if (OPENSSL_s390xcap_P.pcc[1] + & S390X_CAPBIT(S390X_SCALAR_MULTIPLY_X25519)) { + if (s390x_x25519_mul(secret, ecxctx->peerkey->pubkey, + ecxctx->key->privkey) == 0) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); + return 0; + } + } else +#endif + if (ossl_x25519(secret, ecxctx->key->privkey, + ecxctx->peerkey->pubkey) == 0) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); + return 0; + } + } else { +#ifdef S390X_EC_ASM + if (OPENSSL_s390xcap_P.pcc[1] + & S390X_CAPBIT(S390X_SCALAR_MULTIPLY_X448)) { + if (s390x_x448_mul(secret, ecxctx->peerkey->pubkey, + ecxctx->key->privkey) == 0) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); + return 0; + } + } else +#endif + if (ossl_x448(secret, ecxctx->key->privkey, + ecxctx->peerkey->pubkey) == 0) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_DURING_DERIVATION); + return 0; + } + } + + *secretlen = ecxctx->keylen; + return 1; } static void ecx_freectx(void *vecxctx) @@ -138,8 +200,10 @@ static void *ecx_dupctx(void *vecxctx) return NULL; dstctx = OPENSSL_zalloc(sizeof(*srcctx)); - if (dstctx == NULL) + if (dstctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } *dstctx = *srcctx; if (dstctx->key != NULL && !ossl_ecx_key_up_ref(dstctx->key)) { @@ -165,7 +229,7 @@ const OSSL_DISPATCH ossl_x25519_keyexch_functions[] = { { OSSL_FUNC_KEYEXCH_SET_PEER, (void (*)(void))ecx_set_peer }, { OSSL_FUNC_KEYEXCH_FREECTX, (void (*)(void))ecx_freectx }, { OSSL_FUNC_KEYEXCH_DUPCTX, (void (*)(void))ecx_dupctx }, - OSSL_DISPATCH_END + { 0, NULL } }; const OSSL_DISPATCH ossl_x448_keyexch_functions[] = { @@ -175,5 +239,5 @@ const OSSL_DISPATCH ossl_x448_keyexch_functions[] = { { OSSL_FUNC_KEYEXCH_SET_PEER, (void (*)(void))ecx_set_peer }, { OSSL_FUNC_KEYEXCH_FREECTX, (void (*)(void))ecx_freectx }, { OSSL_FUNC_KEYEXCH_DUPCTX, (void (*)(void))ecx_dupctx }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/exchange/kdf_exch.c b/openssl/src/providers/implementations/exchange/kdf_exch.c index 340a2663c..38631b45d 100644 --- a/openssl/src/providers/implementations/exchange/kdf_exch.c +++ b/openssl/src/providers/implementations/exchange/kdf_exch.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -11,10 +11,7 @@ #include #include #include -#include -#include #include -#include "internal/numbers.h" #include "prov/implementations.h" #include "prov/provider_ctx.h" #include "prov/kdfexchange.h" @@ -28,13 +25,9 @@ static OSSL_FUNC_keyexch_derive_fn kdf_derive; static OSSL_FUNC_keyexch_freectx_fn kdf_freectx; static OSSL_FUNC_keyexch_dupctx_fn kdf_dupctx; static OSSL_FUNC_keyexch_set_ctx_params_fn kdf_set_ctx_params; -static OSSL_FUNC_keyexch_get_ctx_params_fn kdf_get_ctx_params; static OSSL_FUNC_keyexch_settable_ctx_params_fn kdf_tls1_prf_settable_ctx_params; static OSSL_FUNC_keyexch_settable_ctx_params_fn kdf_hkdf_settable_ctx_params; static OSSL_FUNC_keyexch_settable_ctx_params_fn kdf_scrypt_settable_ctx_params; -static OSSL_FUNC_keyexch_gettable_ctx_params_fn kdf_tls1_prf_gettable_ctx_params; -static OSSL_FUNC_keyexch_gettable_ctx_params_fn kdf_hkdf_gettable_ctx_params; -static OSSL_FUNC_keyexch_gettable_ctx_params_fn kdf_scrypt_gettable_ctx_params; typedef struct { void *provctx; @@ -99,33 +92,16 @@ static int kdf_derive(void *vpkdfctx, unsigned char *secret, size_t *secretlen, size_t outlen) { PROV_KDF_CTX *pkdfctx = (PROV_KDF_CTX *)vpkdfctx; - size_t kdfsize; - int ret; if (!ossl_prov_is_running()) return 0; - kdfsize = EVP_KDF_CTX_get_kdf_size(pkdfctx->kdfctx); - if (secret == NULL) { - *secretlen = kdfsize; + *secretlen = EVP_KDF_CTX_get_kdf_size(pkdfctx->kdfctx); return 1; } - if (kdfsize != SIZE_MAX) { - if (outlen < kdfsize) { - ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); - return 0; - } - outlen = kdfsize; - } - - ret = EVP_KDF_derive(pkdfctx->kdfctx, secret, outlen, NULL); - if (ret <= 0) - return 0; - - *secretlen = outlen; - return 1; + return EVP_KDF_derive(pkdfctx->kdfctx, secret, outlen, NULL); } static void kdf_freectx(void *vpkdfctx) @@ -173,13 +149,6 @@ static int kdf_set_ctx_params(void *vpkdfctx, const OSSL_PARAM params[]) return EVP_KDF_CTX_set_params(pkdfctx->kdfctx, params); } -static int kdf_get_ctx_params(void *vpkdfctx, OSSL_PARAM params[]) -{ - PROV_KDF_CTX *pkdfctx = (PROV_KDF_CTX *)vpkdfctx; - - return EVP_KDF_CTX_get_params(pkdfctx->kdfctx, params); -} - static const OSSL_PARAM *kdf_settable_ctx_params(ossl_unused void *vpkdfctx, void *provctx, const char *kdfname) @@ -208,34 +177,6 @@ KDF_SETTABLE_CTX_PARAMS(tls1_prf, "TLS1-PRF") KDF_SETTABLE_CTX_PARAMS(hkdf, "HKDF") KDF_SETTABLE_CTX_PARAMS(scrypt, "SCRYPT") -static const OSSL_PARAM *kdf_gettable_ctx_params(ossl_unused void *vpkdfctx, - void *provctx, - const char *kdfname) -{ - EVP_KDF *kdf = EVP_KDF_fetch(PROV_LIBCTX_OF(provctx), kdfname, - NULL); - const OSSL_PARAM *params; - - if (kdf == NULL) - return NULL; - - params = EVP_KDF_gettable_ctx_params(kdf); - EVP_KDF_free(kdf); - - return params; -} - -#define KDF_GETTABLE_CTX_PARAMS(funcname, kdfname) \ - static const OSSL_PARAM *kdf_##funcname##_gettable_ctx_params(void *vpkdfctx, \ - void *provctx) \ - { \ - return kdf_gettable_ctx_params(vpkdfctx, provctx, kdfname); \ - } - -KDF_GETTABLE_CTX_PARAMS(tls1_prf, "TLS1-PRF") -KDF_GETTABLE_CTX_PARAMS(hkdf, "HKDF") -KDF_GETTABLE_CTX_PARAMS(scrypt, "SCRYPT") - #define KDF_KEYEXCH_FUNCTIONS(funcname) \ const OSSL_DISPATCH ossl_kdf_##funcname##_keyexch_functions[] = { \ { OSSL_FUNC_KEYEXCH_NEWCTX, (void (*)(void))kdf_##funcname##_newctx }, \ @@ -244,12 +185,9 @@ KDF_GETTABLE_CTX_PARAMS(scrypt, "SCRYPT") { OSSL_FUNC_KEYEXCH_FREECTX, (void (*)(void))kdf_freectx }, \ { OSSL_FUNC_KEYEXCH_DUPCTX, (void (*)(void))kdf_dupctx }, \ { OSSL_FUNC_KEYEXCH_SET_CTX_PARAMS, (void (*)(void))kdf_set_ctx_params }, \ - { OSSL_FUNC_KEYEXCH_GET_CTX_PARAMS, (void (*)(void))kdf_get_ctx_params }, \ { OSSL_FUNC_KEYEXCH_SETTABLE_CTX_PARAMS, \ (void (*)(void))kdf_##funcname##_settable_ctx_params }, \ - { OSSL_FUNC_KEYEXCH_GETTABLE_CTX_PARAMS, \ - (void (*)(void))kdf_##funcname##_gettable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; KDF_KEYEXCH_FUNCTIONS(tls1_prf) diff --git a/openssl/src/providers/implementations/exchange/sm2dh_exch.c b/openssl/src/providers/implementations/exchange/sm2dh_exch.c new file mode 100644 index 000000000..bd6977a24 --- /dev/null +++ b/openssl/src/providers/implementations/exchange/sm2dh_exch.c @@ -0,0 +1,490 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * SM2DH low level APIs are deprecated for public use, but still ok for + * internal use. + */ +#include "internal/deprecated.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include "prov/provider_ctx.h" +#include "prov/providercommon.h" +#include "prov/implementations.h" +#include "prov/securitycheck.h" +#include "crypto/evp.h" +#include "crypto/ec.h" +#include "crypto/sm2.h" + +static OSSL_FUNC_keyexch_newctx_fn sm2dh_newctx; +static OSSL_FUNC_keyexch_init_fn sm2dh_init; +static OSSL_FUNC_keyexch_set_peer_fn sm2dh_set_peer; +static OSSL_FUNC_keyexch_derive_fn sm2dh_derive; +static OSSL_FUNC_keyexch_freectx_fn sm2dh_freectx; +static OSSL_FUNC_keyexch_dupctx_fn sm2dh_dupctx; +static OSSL_FUNC_keyexch_set_ctx_params_fn sm2dh_set_ctx_params; +static OSSL_FUNC_keyexch_settable_ctx_params_fn sm2dh_settable_ctx_params; +static OSSL_FUNC_keyexch_get_ctx_params_fn sm2dh_get_ctx_params; +static OSSL_FUNC_keyexch_gettable_ctx_params_fn sm2dh_gettable_ctx_params; + +#define OSSL_EXCHANGE_PARAM_INITIATOR "sm2-initiator" +#define OSSL_EXCHANGE_PARAM_SELF_ID "self-id" +#define OSSL_EXCHANGE_PARAM_PEER_ID "peer-id" +#define OSSL_EXCHANGE_PARAM_SELF_ENC_KEY "self-enc-key" +#define OSSL_EXCHANGE_PARAM_PEER_ENC_KEY "peer-enc-key" + +#define OSSL_EXCHANGE_PARAM_DIGEST OSSL_PKEY_PARAM_DIGEST +#define OSSL_EXCHANGE_PARAM_DIGEST_PROPS "digest-props" /* utf8_string */ +#define OSSL_EXCHANGE_PARAM_OUTLEN "outlen" /* size_t */ + +#define OSSL_PKEY_PARAM_SM2_ZA "sm2-za" + +/* + * What's passed as an actual key is defined by the KEYMGMT interface. + * We happen to know that our KEYMGMT simply passes EC_KEY structures, so + * we use that here too. + */ + +typedef struct { + OSSL_LIB_CTX *libctx; + + EC_KEY *k; + EC_KEY *peerk; + + /* private key in self encryption certificate */ + EC_KEY *enc_k; + /* public key in peer encryption certificate */ + EC_KEY *enc_peerk; + + uint8_t *id; + size_t id_len; + + uint8_t *peer_id; + size_t peer_id_len; + + int initiator; + + EVP_MD *md; + + size_t outlen; +} PROV_SM2DH_CTX; + +static +void *sm2dh_newctx(void *provctx) +{ + PROV_SM2DH_CTX *pectx; + + if (!ossl_prov_is_running()) + return NULL; + + pectx = OPENSSL_zalloc(sizeof(*pectx)); + if (pectx == NULL) + return NULL; + + pectx->libctx = PROV_LIBCTX_OF(provctx); + + return (void *)pectx; +} + +static +int ecdh_match_params(const EC_KEY *priv, const EC_KEY *peer) +{ + int ret; + BN_CTX *ctx = NULL; + const EC_GROUP *group_priv = EC_KEY_get0_group(priv); + const EC_GROUP *group_peer = EC_KEY_get0_group(peer); + + ctx = BN_CTX_new_ex(ossl_ec_key_get_libctx(priv)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); + return 0; + } + ret = group_priv != NULL + && group_peer != NULL + && EC_GROUP_cmp(group_priv, group_peer, ctx) == 0; + if (!ret) + ERR_raise(ERR_LIB_PROV, PROV_R_MISMATCHING_DOMAIN_PARAMETERS); + BN_CTX_free(ctx); + return ret; +} + +static +int sm2dh_init(void *vpecdhctx, void *vecdh, const OSSL_PARAM params[]) +{ + PROV_SM2DH_CTX *pecdhctx = (PROV_SM2DH_CTX *)vpecdhctx; + + if (!ossl_prov_is_running() + || pecdhctx == NULL + || vecdh == NULL + || !EC_KEY_up_ref(vecdh)) + return 0; + + EC_KEY_free(pecdhctx->k); + pecdhctx->k = vecdh; + + return sm2dh_set_ctx_params(pecdhctx, params) + && ossl_ec_check_key(pecdhctx->libctx, vecdh, 1) + && pecdhctx->enc_k != NULL + && pecdhctx->enc_peerk != NULL + && ecdh_match_params(pecdhctx->enc_k, pecdhctx->enc_peerk) + && ossl_ec_check_key(pecdhctx->libctx, pecdhctx->enc_k, 1) + && ossl_ec_check_key(pecdhctx->libctx, pecdhctx->enc_peerk, 1); +} + +static +int sm2dh_set_peer(void *vpecdhctx, void *vecdh) +{ + PROV_SM2DH_CTX *pecdhctx = (PROV_SM2DH_CTX *)vpecdhctx; + + if (!ossl_prov_is_running() + || pecdhctx == NULL + || vecdh == NULL + || !ecdh_match_params(pecdhctx->k, vecdh) + || !ossl_ec_check_key(pecdhctx->libctx, vecdh, 1) + || !EC_KEY_up_ref(vecdh)) + return 0; + + EC_KEY_free(pecdhctx->peerk); + pecdhctx->peerk = vecdh; + return 1; +} + +static +void sm2dh_freectx(void *vpecdhctx) +{ + PROV_SM2DH_CTX *pecdhctx = (PROV_SM2DH_CTX *)vpecdhctx; + + EC_KEY_free(pecdhctx->k); + EC_KEY_free(pecdhctx->peerk); + + EC_KEY_free(pecdhctx->enc_k); + EC_KEY_free(pecdhctx->enc_peerk); + + OPENSSL_free(pecdhctx->id); + OPENSSL_free(pecdhctx->peer_id); + + EVP_MD_free(pecdhctx->md); + + OPENSSL_free(pecdhctx); +} + +static +void *sm2dh_dupctx(void *vpecdhctx) +{ + PROV_SM2DH_CTX *srcctx = (PROV_SM2DH_CTX *)vpecdhctx; + PROV_SM2DH_CTX *dstctx; + + if (!ossl_prov_is_running()) + return NULL; + + dstctx = OPENSSL_zalloc(sizeof(*srcctx)); + if (dstctx == NULL) + return NULL; + + *dstctx = *srcctx; + + /* clear all pointers */ + dstctx->k= NULL; + dstctx->peerk = NULL; + dstctx->enc_k = NULL; + dstctx->enc_peerk = NULL; + dstctx->md = NULL; + dstctx->id = NULL; + dstctx->peer_id = NULL; + + /* up-ref all ref-counted objects referenced in dstctx */ + if (srcctx->k != NULL && !EC_KEY_up_ref(srcctx->k)) + goto err; + else + dstctx->k = srcctx->k; + + if (srcctx->peerk != NULL && !EC_KEY_up_ref(srcctx->peerk)) + goto err; + else + dstctx->peerk = srcctx->peerk; + + if (srcctx->k != NULL && !EC_KEY_up_ref(srcctx->k)) + goto err; + else + dstctx->k = srcctx->k; + + if (srcctx->peerk != NULL && !EC_KEY_up_ref(srcctx->peerk)) + goto err; + else + dstctx->peerk = srcctx->peerk; + + if (srcctx->md != NULL && !EVP_MD_up_ref(srcctx->md)) + goto err; + else + dstctx->md = srcctx->md; + + if (srcctx->id != NULL && srcctx->id_len > 0) { + dstctx->id = OPENSSL_memdup(srcctx->id, srcctx->id_len); + if (dstctx->id == NULL) + goto err; + } + + if (srcctx->peer_id != NULL && srcctx->peer_id_len > 0) { + dstctx->peer_id = OPENSSL_memdup(srcctx->peer_id, srcctx->peer_id_len); + if (dstctx->peer_id == NULL) + goto err; + } + + return dstctx; + + err: + sm2dh_freectx(dstctx); + return NULL; +} + +static +int sm2dh_set_ctx_params(void *vpecdhctx, const OSSL_PARAM params[]) +{ + PROV_SM2DH_CTX *pectx = (PROV_SM2DH_CTX *)vpecdhctx; + const OSSL_PARAM *p; + EVP_PKEY *key = NULL; + void *tmp_id = NULL; + size_t tmp_len; + char name[80] = { '\0' }; /* should be big enough */ + char *str = NULL; + + if (pectx == NULL) + return 0; + if (params == NULL) + return 1; + + p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_INITIATOR); + if (p != NULL) { + int initiator; + + if (!OSSL_PARAM_get_int(p, &initiator)) + return 0; + + pectx->initiator = initiator; + } + + p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_SELF_ENC_KEY); + if (p != NULL) { + EVP_KEYMGMT *keymgmt = NULL; + + if (!OSSL_PARAM_get_octet_ptr(p, (const void **)&key, NULL)) + return 0; + + if (key == NULL) + return 0; + + EC_KEY_free(pectx->enc_k); + + pectx->enc_k = (EC_KEY *)evp_pkey_export_to_provider(key, pectx->libctx, + &keymgmt, NULL); + EVP_KEYMGMT_free(keymgmt); + + if (pectx->enc_k == NULL) + return 0; + else + EC_KEY_up_ref(pectx->enc_k); + } + + p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_PEER_ENC_KEY); + if (p != NULL) { + EVP_KEYMGMT *keymgmt = NULL; + + if (!OSSL_PARAM_get_octet_ptr(p, (const void **)&key, NULL)) + return 0; + + if (key == NULL) + return 0; + + EC_KEY_free(pectx->enc_peerk); + pectx->enc_peerk = (EC_KEY *)evp_pkey_export_to_provider(key, + pectx->libctx, + &keymgmt, + NULL); + EVP_KEYMGMT_free(keymgmt); + + if (pectx->enc_peerk == NULL) + return 0; + else + EC_KEY_up_ref(pectx->enc_peerk); + } + + p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_OUTLEN); + if (p != NULL) { + size_t outlen; + + if (!OSSL_PARAM_get_size_t(p, &outlen)) + return 0; + pectx->outlen = outlen; + } + + p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_DIGEST); + if (p != NULL) { + char mdprops[80] = { '\0' }; /* should be big enough */ + + str = name; + if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(name))) + return 0; + + str = mdprops; + p = OSSL_PARAM_locate_const(params, + OSSL_EXCHANGE_PARAM_DIGEST_PROPS); + + if (p != NULL) { + if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdprops))) + return 0; + } + + EVP_MD_free(pectx->md); + pectx->md = EVP_MD_fetch(pectx->libctx, name, mdprops); + if (!ossl_digest_is_allowed(pectx->libctx, pectx->md)) { + EVP_MD_free(pectx->md); + pectx->md = NULL; + } + if (pectx->md == NULL) + return 0; + } + + p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_SELF_ID); + if (p != NULL) { + tmp_id = NULL; + if (!OSSL_PARAM_get_octet_string(p, &tmp_id, 0, &tmp_len)) + return 0; + + OPENSSL_free(pectx->id); + pectx->id = tmp_id; + pectx->id_len = tmp_len; + } + + p = OSSL_PARAM_locate_const(params, OSSL_EXCHANGE_PARAM_PEER_ID); + if (p != NULL) { + tmp_id = NULL; + if (!OSSL_PARAM_get_octet_string(p, &tmp_id, 0, &tmp_len)) + return 0; + + OPENSSL_free(pectx->peer_id); + pectx->peer_id = tmp_id; + pectx->peer_id_len = tmp_len; + } + + return 1; +} + +static const OSSL_PARAM known_settable_ctx_params[] = { + OSSL_PARAM_int(OSSL_EXCHANGE_PARAM_INITIATOR, NULL), + OSSL_PARAM_octet_string(OSSL_EXCHANGE_PARAM_SELF_ID, NULL, 0), + OSSL_PARAM_octet_string(OSSL_EXCHANGE_PARAM_PEER_ID, NULL, 0), + OSSL_PARAM_octet_ptr(OSSL_EXCHANGE_PARAM_SELF_ENC_KEY, NULL, 0), + OSSL_PARAM_octet_ptr(OSSL_EXCHANGE_PARAM_PEER_ENC_KEY, NULL, 0), + OSSL_PARAM_utf8_string(OSSL_EXCHANGE_PARAM_DIGEST, NULL, 0), + OSSL_PARAM_utf8_string(OSSL_EXCHANGE_PARAM_DIGEST_PROPS, NULL, 0), + OSSL_PARAM_size_t(OSSL_EXCHANGE_PARAM_OUTLEN, NULL), + OSSL_PARAM_END +}; + +static +const OSSL_PARAM *sm2dh_settable_ctx_params(ossl_unused void *vpecdhctx, + ossl_unused void *provctx) +{ + return known_settable_ctx_params; +} + +static +int sm2dh_get_ctx_params(void *vpecdhctx, + OSSL_PARAM params[]) +{ + PROV_SM2DH_CTX *pectx = (PROV_SM2DH_CTX *)vpecdhctx; + OSSL_PARAM *p; + + if (pectx == NULL) + return 0; + + p = OSSL_PARAM_locate(params, OSSL_EXCHANGE_PARAM_DIGEST); + if (p != NULL + && !OSSL_PARAM_set_utf8_string(p, pectx->md == NULL + ? "" + : EVP_MD_get0_name(pectx->md))){ + return 0; + } + + p = OSSL_PARAM_locate(params, OSSL_EXCHANGE_PARAM_OUTLEN); + if (p != NULL && !OSSL_PARAM_set_size_t(p, pectx->outlen)) + return 0; + + return 1; +} + +static const OSSL_PARAM known_gettable_ctx_params[] = { + OSSL_PARAM_utf8_string(OSSL_EXCHANGE_PARAM_DIGEST, NULL, 0), + OSSL_PARAM_size_t(OSSL_EXCHANGE_PARAM_OUTLEN, NULL), + OSSL_PARAM_END +}; + +static +const OSSL_PARAM *sm2dh_gettable_ctx_params(ossl_unused void *vpecdhctx, + ossl_unused void *provctx) +{ + return known_gettable_ctx_params; +} + +static +int sm2dh_derive(void *vpecdhctx, unsigned char *secret, + size_t *psecretlen, size_t outlen) +{ + PROV_SM2DH_CTX *pecdhctx = (PROV_SM2DH_CTX *)vpecdhctx; + + if (secret == NULL) { + *psecretlen = pecdhctx->outlen; + return 1; + } + + if (pecdhctx->outlen > outlen) { + ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); + return 0; + } + + if (pecdhctx->k == NULL || pecdhctx->peerk == NULL) { + ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_KEY); + return 0; + } + + if (SM2_compute_key(secret, pecdhctx->outlen, pecdhctx->initiator, + pecdhctx->peer_id, pecdhctx->peer_id_len, + pecdhctx->id, pecdhctx->id_len, + pecdhctx->peerk, pecdhctx->k, + pecdhctx->enc_peerk, pecdhctx->enc_k, + pecdhctx->md, pecdhctx->libctx, NULL) <= 0) + return 0; + + *psecretlen = pecdhctx->outlen; + + return 1; +} + +const OSSL_DISPATCH ossl_sm2dh_keyexch_functions[] = { + { OSSL_FUNC_KEYEXCH_NEWCTX, (void (*)(void))sm2dh_newctx }, + { OSSL_FUNC_KEYEXCH_INIT, (void (*)(void))sm2dh_init }, + { OSSL_FUNC_KEYEXCH_DERIVE, (void (*)(void))sm2dh_derive }, + { OSSL_FUNC_KEYEXCH_SET_PEER, (void (*)(void))sm2dh_set_peer }, + { OSSL_FUNC_KEYEXCH_FREECTX, (void (*)(void))sm2dh_freectx }, + { OSSL_FUNC_KEYEXCH_DUPCTX, (void (*)(void))sm2dh_dupctx }, + { OSSL_FUNC_KEYEXCH_SET_CTX_PARAMS, (void (*)(void))sm2dh_set_ctx_params }, + { OSSL_FUNC_KEYEXCH_SETTABLE_CTX_PARAMS, + (void (*)(void))sm2dh_settable_ctx_params }, + { OSSL_FUNC_KEYEXCH_GET_CTX_PARAMS, (void (*)(void))sm2dh_get_ctx_params }, + { OSSL_FUNC_KEYEXCH_GETTABLE_CTX_PARAMS, + (void (*)(void))sm2dh_gettable_ctx_params }, + { 0, NULL } +}; diff --git a/openssl/src/providers/implementations/include/prov/blake2.h b/openssl/src/providers/implementations/include/prov/blake2.h deleted file mode 100644 index 42229e2d7..000000000 --- a/openssl/src/providers/implementations/include/prov/blake2.h +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_PROV_BLAKE2_H -# define OSSL_PROV_BLAKE2_H - -# include - -# include -# include -# include - -# define BLAKE2S_BLOCKBYTES 64 -# define BLAKE2S_OUTBYTES 32 -# define BLAKE2S_KEYBYTES 32 -# define BLAKE2S_SALTBYTES 8 -# define BLAKE2S_PERSONALBYTES 8 - -# define BLAKE2B_BLOCKBYTES 128 -# define BLAKE2B_OUTBYTES 64 -# define BLAKE2B_KEYBYTES 64 -# define BLAKE2B_SALTBYTES 16 -# define BLAKE2B_PERSONALBYTES 16 - -struct blake2s_param_st { - uint8_t digest_length; /* 1 */ - uint8_t key_length; /* 2 */ - uint8_t fanout; /* 3 */ - uint8_t depth; /* 4 */ - uint8_t leaf_length[4];/* 8 */ - uint8_t node_offset[6];/* 14 */ - uint8_t node_depth; /* 15 */ - uint8_t inner_length; /* 16 */ - uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ - uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ -}; - -typedef struct blake2s_param_st BLAKE2S_PARAM; - -struct blake2s_ctx_st { - uint32_t h[8]; - uint32_t t[2]; - uint32_t f[2]; - uint8_t buf[BLAKE2S_BLOCKBYTES]; - size_t buflen; - size_t outlen; -}; - -struct blake2b_param_st { - uint8_t digest_length; /* 1 */ - uint8_t key_length; /* 2 */ - uint8_t fanout; /* 3 */ - uint8_t depth; /* 4 */ - uint8_t leaf_length[4];/* 8 */ - uint8_t node_offset[8];/* 16 */ - uint8_t node_depth; /* 17 */ - uint8_t inner_length; /* 18 */ - uint8_t reserved[14]; /* 32 */ - uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ - uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ -}; - -typedef struct blake2b_param_st BLAKE2B_PARAM; - -struct blake2b_ctx_st { - uint64_t h[8]; - uint64_t t[2]; - uint64_t f[2]; - uint8_t buf[BLAKE2B_BLOCKBYTES]; - size_t buflen; - size_t outlen; -}; - -#define BLAKE2B_DIGEST_LENGTH 64 -#define BLAKE2S_DIGEST_LENGTH 32 - -typedef struct blake2s_ctx_st BLAKE2S_CTX; -typedef struct blake2b_ctx_st BLAKE2B_CTX; - -struct blake2b_md_data_st { - BLAKE2B_CTX ctx; - BLAKE2B_PARAM params; -}; - -struct blake2s_md_data_st { - BLAKE2S_CTX ctx; - BLAKE2S_PARAM params; -}; - -int ossl_blake2b_init(BLAKE2B_CTX *c, const BLAKE2B_PARAM *P); -int ossl_blake2b_init_key(BLAKE2B_CTX *c, const BLAKE2B_PARAM *P, - const void *key); -int ossl_blake2b_update(BLAKE2B_CTX *c, const void *data, size_t datalen); -int ossl_blake2b_final(unsigned char *md, BLAKE2B_CTX *c); - -OSSL_FUNC_digest_get_ctx_params_fn ossl_blake2b_get_ctx_params; -OSSL_FUNC_digest_set_ctx_params_fn ossl_blake2b_set_ctx_params; -OSSL_FUNC_digest_gettable_ctx_params_fn ossl_blake2b_gettable_ctx_params; -OSSL_FUNC_digest_settable_ctx_params_fn ossl_blake2b_settable_ctx_params; - -/* - * These setters are internal and do not check the validity of their parameters. - * See blake2b_mac_ctrl for validation logic. - */ - -void ossl_blake2b_param_init(BLAKE2B_PARAM *P); -void ossl_blake2b_param_set_digest_length(BLAKE2B_PARAM *P, uint8_t outlen); -void ossl_blake2b_param_set_key_length(BLAKE2B_PARAM *P, uint8_t keylen); -void ossl_blake2b_param_set_personal(BLAKE2B_PARAM *P, const uint8_t *personal, - size_t length); -void ossl_blake2b_param_set_salt(BLAKE2B_PARAM *P, const uint8_t *salt, - size_t length); -int ossl_blake2s_init(BLAKE2S_CTX *c, const BLAKE2S_PARAM *P); -int ossl_blake2s_init_key(BLAKE2S_CTX *c, const BLAKE2S_PARAM *P, - const void *key); -int ossl_blake2s_update(BLAKE2S_CTX *c, const void *data, size_t datalen); -int ossl_blake2s_final(unsigned char *md, BLAKE2S_CTX *c); - -void ossl_blake2s_param_init(BLAKE2S_PARAM *P); -void ossl_blake2s_param_set_digest_length(BLAKE2S_PARAM *P, uint8_t outlen); -void ossl_blake2s_param_set_key_length(BLAKE2S_PARAM *P, uint8_t keylen); -void ossl_blake2s_param_set_personal(BLAKE2S_PARAM *P, const uint8_t *personal, - size_t length); -void ossl_blake2s_param_set_salt(BLAKE2S_PARAM *P, const uint8_t *salt, - size_t length); - -OSSL_FUNC_digest_get_ctx_params_fn ossl_blake2s_get_ctx_params; -OSSL_FUNC_digest_set_ctx_params_fn ossl_blake2s_set_ctx_params; -OSSL_FUNC_digest_gettable_ctx_params_fn ossl_blake2s_gettable_ctx_params; -OSSL_FUNC_digest_settable_ctx_params_fn ossl_blake2s_settable_ctx_params; - -#endif /* OSSL_PROV_BLAKE2_H */ diff --git a/openssl/src/providers/implementations/include/prov/ciphercommon.h b/openssl/src/providers/implementations/include/prov/ciphercommon.h index 45002ad59..91c4c914b 100644 --- a/openssl/src/providers/implementations/include/prov/ciphercommon.h +++ b/openssl/src/providers/implementations/include/prov/ciphercommon.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,27 +7,23 @@ * https://www.openssl.org/source/license.html */ -#ifndef OSSL_PROV_CIPHERCOMMON_H -# define OSSL_PROV_CIPHERCOMMON_H -# pragma once +#include +#include +#include +#include +#include "internal/cryptlib.h" +#include "crypto/modes.h" -# include -# include -# include -# include -# include "internal/cryptlib.h" -# include "crypto/modes.h" +#define MAXCHUNK ((size_t)1 << (sizeof(long) * 8 - 2)) +#define MAXBITCHUNK ((size_t)1 << (sizeof(size_t) * 8 - 4)) -# define MAXCHUNK ((size_t)1 << 30) -# define MAXBITCHUNK ((size_t)1 << (sizeof(size_t) * 8 - 4)) +#define GENERIC_BLOCK_SIZE 16 +#define IV_STATE_UNINITIALISED 0 /* initial state is not initialized */ +#define IV_STATE_BUFFERED 1 /* iv has been copied to the iv buffer */ +#define IV_STATE_COPIED 2 /* iv has been copied from the iv buffer */ +#define IV_STATE_FINISHED 3 /* the iv has been used - so don't reuse it */ -# define GENERIC_BLOCK_SIZE 16 -# define IV_STATE_UNINITIALISED 0 /* initial state is not initialized */ -# define IV_STATE_BUFFERED 1 /* iv has been copied to the iv buffer */ -# define IV_STATE_COPIED 2 /* iv has been copied from the iv buffer */ -# define IV_STATE_FINISHED 3 /* the iv has been used - so don't reuse it */ - -# define PROV_CIPHER_FUNC(type, name, args) typedef type (* OSSL_##name##_fn)args +#define PROV_CIPHER_FUNC(type, name, args) typedef type (* OSSL_##name##_fn)args typedef struct prov_cipher_hw_st PROV_CIPHER_HW; typedef struct prov_cipher_ctx_st PROV_CIPHER_CTX; @@ -36,23 +32,16 @@ typedef int (PROV_CIPHER_HW_FN)(PROV_CIPHER_CTX *dat, unsigned char *out, const unsigned char *in, size_t len); /* Internal flags that can be queried */ -# define PROV_CIPHER_FLAG_AEAD 0x0001 -# define PROV_CIPHER_FLAG_CUSTOM_IV 0x0002 -# define PROV_CIPHER_FLAG_CTS 0x0004 -# define PROV_CIPHER_FLAG_TLS1_MULTIBLOCK 0x0008 -# define PROV_CIPHER_FLAG_RAND_KEY 0x0010 +#define PROV_CIPHER_FLAG_AEAD 0x0001 +#define PROV_CIPHER_FLAG_CUSTOM_IV 0x0002 +#define PROV_CIPHER_FLAG_CTS 0x0004 +#define PROV_CIPHER_FLAG_TLS1_MULTIBLOCK 0x0008 +#define PROV_CIPHER_FLAG_RAND_KEY 0x0010 /* Internal flags that are only used within the provider */ -# define PROV_CIPHER_FLAG_VARIABLE_LENGTH 0x0100 -# define PROV_CIPHER_FLAG_INVERSE_CIPHER 0x0200 +#define PROV_CIPHER_FLAG_VARIABLE_LENGTH 0x0100 +#define PROV_CIPHER_FLAG_INVERSE_CIPHER 0x0200 struct prov_cipher_ctx_st { - /* place buffer at the beginning for memory alignment */ - /* The original value of the iv */ - unsigned char oiv[GENERIC_BLOCK_SIZE]; - /* Buffer of partial blocks processed via update calls */ - unsigned char buf[GENERIC_BLOCK_SIZE]; - unsigned char iv[GENERIC_BLOCK_SIZE]; - block128_f block; union { cbc128_f cbc; @@ -69,7 +58,6 @@ struct prov_cipher_ctx_st { unsigned int pad : 1; /* Whether padding should be used or not */ unsigned int enc : 1; /* Set to 1 for encrypt, or 0 otherwise */ unsigned int iv_set : 1; /* Set when the iv is copied to the iv/oiv buffers */ - unsigned int key_set : 1; /* Set when key is set on the context */ unsigned int updated : 1; /* Set to 1 during update for one shot ciphers */ unsigned int variable_keylength : 1; unsigned int inverse_cipher : 1; /* set to 1 to use inverse cipher */ @@ -94,6 +82,12 @@ struct prov_cipher_ctx_st { * manage partial blocks themselves. */ unsigned int num; + + /* The original value of the iv */ + unsigned char oiv[GENERIC_BLOCK_SIZE]; + /* Buffer of partial blocks processed via update calls */ + unsigned char buf[GENERIC_BLOCK_SIZE]; + unsigned char iv[GENERIC_BLOCK_SIZE]; const PROV_CIPHER_HW *hw; /* hardware specific functions */ const void *ks; /* Pointer to algorithm specific key data */ OSSL_LIB_CTX *libctx; @@ -131,7 +125,7 @@ void ossl_cipher_generic_initkey(void *vctx, size_t kbits, size_t blkbits, uint64_t flags, const PROV_CIPHER_HW *hw, void *provctx); -# define IMPLEMENT_generic_cipher_func(alg, UCALG, lcmode, UCMODE, flags, kbits,\ +#define IMPLEMENT_generic_cipher_func(alg, UCALG, lcmode, UCMODE, flags, kbits,\ blkbits, ivbits, typ) \ const OSSL_DISPATCH ossl_##alg##kbits##lcmode##_functions[] = { \ { OSSL_FUNC_CIPHER_NEWCTX, \ @@ -155,10 +149,10 @@ const OSSL_DISPATCH ossl_##alg##kbits##lcmode##_functions[] = { \ (void (*)(void))ossl_cipher_generic_gettable_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))ossl_cipher_generic_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; -# define IMPLEMENT_var_keylen_cipher_func(alg, UCALG, lcmode, UCMODE, flags, \ +#define IMPLEMENT_var_keylen_cipher_func(alg, UCALG, lcmode, UCMODE, flags, \ kbits, blkbits, ivbits, typ) \ const OSSL_DISPATCH ossl_##alg##kbits##lcmode##_functions[] = { \ { OSSL_FUNC_CIPHER_NEWCTX, \ @@ -182,11 +176,11 @@ const OSSL_DISPATCH ossl_##alg##kbits##lcmode##_functions[] = { \ (void (*)(void))ossl_cipher_generic_gettable_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))ossl_cipher_var_keylen_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; -# define IMPLEMENT_generic_cipher_genfn(alg, UCALG, lcmode, UCMODE, flags, \ +#define IMPLEMENT_generic_cipher_genfn(alg, UCALG, lcmode, UCMODE, flags, \ kbits, blkbits, ivbits, typ) \ static OSSL_FUNC_cipher_get_params_fn alg##_##kbits##_##lcmode##_get_params; \ static int alg##_##kbits##_##lcmode##_get_params(OSSL_PARAM params[]) \ @@ -208,14 +202,14 @@ static void * alg##_##kbits##_##lcmode##_newctx(void *provctx) \ return ctx; \ } \ -# define IMPLEMENT_generic_cipher(alg, UCALG, lcmode, UCMODE, flags, kbits, \ +#define IMPLEMENT_generic_cipher(alg, UCALG, lcmode, UCMODE, flags, kbits, \ blkbits, ivbits, typ) \ IMPLEMENT_generic_cipher_genfn(alg, UCALG, lcmode, UCMODE, flags, kbits, \ blkbits, ivbits, typ) \ IMPLEMENT_generic_cipher_func(alg, UCALG, lcmode, UCMODE, flags, kbits, \ blkbits, ivbits, typ) -# define IMPLEMENT_var_keylen_cipher(alg, UCALG, lcmode, UCMODE, flags, kbits, \ +#define IMPLEMENT_var_keylen_cipher(alg, UCALG, lcmode, UCMODE, flags, kbits, \ blkbits, ivbits, typ) \ IMPLEMENT_generic_cipher_genfn(alg, UCALG, lcmode, UCMODE, flags, kbits, \ blkbits, ivbits, typ) \ @@ -233,11 +227,11 @@ PROV_CIPHER_HW_FN ossl_cipher_hw_chunked_cbc; PROV_CIPHER_HW_FN ossl_cipher_hw_chunked_cfb8; PROV_CIPHER_HW_FN ossl_cipher_hw_chunked_cfb128; PROV_CIPHER_HW_FN ossl_cipher_hw_chunked_ofb128; -# define ossl_cipher_hw_chunked_ecb ossl_cipher_hw_generic_ecb -# define ossl_cipher_hw_chunked_ctr ossl_cipher_hw_generic_ctr -# define ossl_cipher_hw_chunked_cfb1 ossl_cipher_hw_generic_cfb1 +#define ossl_cipher_hw_chunked_ecb ossl_cipher_hw_generic_ecb +#define ossl_cipher_hw_chunked_ctr ossl_cipher_hw_generic_ctr +#define ossl_cipher_hw_chunked_cfb1 ossl_cipher_hw_generic_cfb1 -# define IMPLEMENT_CIPHER_HW_OFB(MODE, NAME, CTX_NAME, KEY_NAME, FUNC_PREFIX) \ +#define IMPLEMENT_CIPHER_HW_OFB(MODE, NAME, CTX_NAME, KEY_NAME, FUNC_PREFIX) \ static int cipher_hw_##NAME##_##MODE##_cipher(PROV_CIPHER_CTX *ctx, \ unsigned char *out, \ const unsigned char *in, size_t len) \ @@ -258,7 +252,7 @@ static int cipher_hw_##NAME##_##MODE##_cipher(PROV_CIPHER_CTX *ctx, \ return 1; \ } -# define IMPLEMENT_CIPHER_HW_ECB(MODE, NAME, CTX_NAME, KEY_NAME, FUNC_PREFIX) \ +#define IMPLEMENT_CIPHER_HW_ECB(MODE, NAME, CTX_NAME, KEY_NAME, FUNC_PREFIX) \ static int cipher_hw_##NAME##_##MODE##_cipher(PROV_CIPHER_CTX *ctx, \ unsigned char *out, \ const unsigned char *in, size_t len) \ @@ -273,7 +267,7 @@ static int cipher_hw_##NAME##_##MODE##_cipher(PROV_CIPHER_CTX *ctx, \ return 1; \ } -# define IMPLEMENT_CIPHER_HW_CBC(MODE, NAME, CTX_NAME, KEY_NAME, FUNC_PREFIX) \ +#define IMPLEMENT_CIPHER_HW_CBC(MODE, NAME, CTX_NAME, KEY_NAME, FUNC_PREFIX) \ static int cipher_hw_##NAME##_##MODE##_cipher(PROV_CIPHER_CTX *ctx, \ unsigned char *out, \ const unsigned char *in, size_t len) \ @@ -291,7 +285,7 @@ static int cipher_hw_##NAME##_##MODE##_cipher(PROV_CIPHER_CTX *ctx, \ return 1; \ } -# define IMPLEMENT_CIPHER_HW_CFB(MODE, NAME, CTX_NAME, KEY_NAME, FUNC_PREFIX) \ +#define IMPLEMENT_CIPHER_HW_CFB(MODE, NAME, CTX_NAME, KEY_NAME, FUNC_PREFIX) \ static int cipher_hw_##NAME##_##MODE##_cipher(PROV_CIPHER_CTX *ctx, \ unsigned char *out, \ const unsigned char *in, size_t len) \ @@ -315,7 +309,7 @@ static int cipher_hw_##NAME##_##MODE##_cipher(PROV_CIPHER_CTX *ctx, \ return 1; \ } -# define IMPLEMENT_CIPHER_HW_COPYCTX(name, CTX_TYPE) \ +#define IMPLEMENT_CIPHER_HW_COPYCTX(name, CTX_TYPE) \ static void name(PROV_CIPHER_CTX *dst, const PROV_CIPHER_CTX *src) \ { \ CTX_TYPE *sctx = (CTX_TYPE *)src; \ @@ -325,7 +319,7 @@ static void name(PROV_CIPHER_CTX *dst, const PROV_CIPHER_CTX *src) \ dst->ks = &dctx->ks.ks; \ } -# define CIPHER_DEFAULT_GETTABLE_CTX_PARAMS_START(name) \ +#define CIPHER_DEFAULT_GETTABLE_CTX_PARAMS_START(name) \ static const OSSL_PARAM name##_known_gettable_ctx_params[] = { \ OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_KEYLEN, NULL), \ OSSL_PARAM_size_t(OSSL_CIPHER_PARAM_IVLEN, NULL), \ @@ -334,7 +328,7 @@ static const OSSL_PARAM name##_known_gettable_ctx_params[] = { \ OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_IV, NULL, 0), \ OSSL_PARAM_octet_string(OSSL_CIPHER_PARAM_UPDATED_IV, NULL, 0), -# define CIPHER_DEFAULT_GETTABLE_CTX_PARAMS_END(name) \ +#define CIPHER_DEFAULT_GETTABLE_CTX_PARAMS_END(name) \ OSSL_PARAM_END \ }; \ const OSSL_PARAM * name##_gettable_ctx_params(ossl_unused void *cctx, \ @@ -343,11 +337,11 @@ const OSSL_PARAM * name##_gettable_ctx_params(ossl_unused void *cctx, \ return name##_known_gettable_ctx_params; \ } -# define CIPHER_DEFAULT_SETTABLE_CTX_PARAMS_START(name) \ +#define CIPHER_DEFAULT_SETTABLE_CTX_PARAMS_START(name) \ static const OSSL_PARAM name##_known_settable_ctx_params[] = { \ OSSL_PARAM_uint(OSSL_CIPHER_PARAM_PADDING, NULL), \ OSSL_PARAM_uint(OSSL_CIPHER_PARAM_NUM, NULL), -# define CIPHER_DEFAULT_SETTABLE_CTX_PARAMS_END(name) \ +#define CIPHER_DEFAULT_SETTABLE_CTX_PARAMS_END(name) \ OSSL_PARAM_END \ }; \ const OSSL_PARAM * name##_settable_ctx_params(ossl_unused void *cctx, \ @@ -365,5 +359,3 @@ size_t ossl_cipher_fillblock(unsigned char *buf, size_t *buflen, int ossl_cipher_trailingdata(unsigned char *buf, size_t *buflen, size_t blocksize, const unsigned char **in, size_t *inlen); - -#endif diff --git a/openssl/src/providers/implementations/include/prov/ciphercommon_aead.h b/openssl/src/providers/implementations/include/prov/ciphercommon_aead.h index 8d709f10e..1d017175d 100644 --- a/openssl/src/providers/implementations/include/prov/ciphercommon_aead.h +++ b/openssl/src/providers/implementations/include/prov/ciphercommon_aead.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,15 +7,11 @@ * https://www.openssl.org/source/license.html */ -#ifndef OSSL_PROV_CIPHERCOMMON_AEAD_H -# define OSSL_PROV_CIPHERCOMMON_AEAD_H -# pragma once +#define UNINITIALISED_SIZET ((size_t)-1) -# define UNINITIALISED_SIZET ((size_t)-1) +#define AEAD_FLAGS (PROV_CIPHER_FLAG_AEAD | PROV_CIPHER_FLAG_CUSTOM_IV) -# define AEAD_FLAGS (PROV_CIPHER_FLAG_AEAD | PROV_CIPHER_FLAG_CUSTOM_IV) - -# define IMPLEMENT_aead_cipher(alg, lc, UCMODE, flags, kbits, blkbits, ivbits) \ +#define IMPLEMENT_aead_cipher(alg, lc, UCMODE, flags, kbits, blkbits, ivbits) \ static OSSL_FUNC_cipher_get_params_fn alg##_##kbits##_##lc##_get_params; \ static int alg##_##kbits##_##lc##_get_params(OSSL_PARAM params[]) \ { \ @@ -27,14 +23,9 @@ static void * alg##kbits##lc##_newctx(void *provctx) \ { \ return alg##_##lc##_newctx(provctx, kbits); \ } \ -static void * alg##kbits##lc##_dupctx(void *src) \ -{ \ - return alg##_##lc##_dupctx(src); \ -} \ const OSSL_DISPATCH ossl_##alg##kbits##lc##_functions[] = { \ { OSSL_FUNC_CIPHER_NEWCTX, (void (*)(void))alg##kbits##lc##_newctx }, \ { OSSL_FUNC_CIPHER_FREECTX, (void (*)(void))alg##_##lc##_freectx }, \ - { OSSL_FUNC_CIPHER_DUPCTX, (void (*)(void))alg##kbits##lc##_dupctx }, \ { OSSL_FUNC_CIPHER_ENCRYPT_INIT, (void (*)(void))ossl_##lc##_einit }, \ { OSSL_FUNC_CIPHER_DECRYPT_INIT, (void (*)(void))ossl_##lc##_dinit }, \ { OSSL_FUNC_CIPHER_UPDATE, (void (*)(void))ossl_##lc##_stream_update }, \ @@ -52,7 +43,5 @@ const OSSL_DISPATCH ossl_##alg##kbits##lc##_functions[] = { \ (void (*)(void))ossl_cipher_aead_gettable_ctx_params }, \ { OSSL_FUNC_CIPHER_SETTABLE_CTX_PARAMS, \ (void (*)(void))ossl_cipher_aead_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } - -#endif diff --git a/openssl/src/providers/implementations/include/prov/ciphercommon_ccm.h b/openssl/src/providers/implementations/include/prov/ciphercommon_ccm.h index ce1a2aa0e..4c184b395 100644 --- a/openssl/src/providers/implementations/include/prov/ciphercommon_ccm.h +++ b/openssl/src/providers/implementations/include/prov/ciphercommon_ccm.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,15 +7,11 @@ * https://www.openssl.org/source/license.html */ -#ifndef OSSL_PROV_CIPHERCOMMON_CCM_H -# define OSSL_PROV_CIPHERCOMMON_CCM_H -# pragma once - -# include "ciphercommon_aead.h" +#include "ciphercommon_aead.h" typedef struct prov_ccm_hw_st PROV_CCM_HW; -# if defined(OPENSSL_CPUID_OBJ) && defined(__s390__) +#if defined(OPENSSL_CPUID_OBJ) && defined(__s390__) /*- * KMAC-AES parameter block - begin * (see z/Architecture Principles of Operation >= SA22-7832-08) @@ -28,7 +24,7 @@ typedef struct S390X_kmac_params_st { unsigned char k[32]; } S390X_KMAC_PARAMS; /* KMAC-AES parameter block - end */ -# endif +#endif /* Base structure that is shared by AES & ARIA for CCM MODE */ typedef struct prov_ccm_st { @@ -102,5 +98,3 @@ int ossl_ccm_generic_auth_encrypt(PROV_CCM_CTX *ctx, const unsigned char *in, int ossl_ccm_generic_auth_decrypt(PROV_CCM_CTX *ctx, const unsigned char *in, unsigned char *out, size_t len, unsigned char *expected_tag, size_t taglen); - -#endif diff --git a/openssl/src/providers/implementations/include/prov/ciphercommon_gcm.h b/openssl/src/providers/implementations/include/prov/ciphercommon_gcm.h index ee0b23b92..7c4a548f9 100644 --- a/openssl/src/providers/implementations/include/prov/ciphercommon_gcm.h +++ b/openssl/src/providers/implementations/include/prov/ciphercommon_gcm.h @@ -1,6 +1,6 @@ /* - * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,20 +8,16 @@ * https://www.openssl.org/source/license.html */ -#ifndef OSSL_PROV_CIPHERCOMMON_GCM_H -# define OSSL_PROV_CIPHERCOMMON_GCM_H -# pragma once - -# include -# include "ciphercommon_aead.h" +#include +#include "ciphercommon_aead.h" typedef struct prov_gcm_hw_st PROV_GCM_HW; -# define GCM_IV_DEFAULT_SIZE 12 /* IV's for AES_GCM should normally be 12 bytes */ -# define GCM_IV_MAX_SIZE (1024 / 8) -# define GCM_TAG_MAX_SIZE 16 +#define GCM_IV_DEFAULT_SIZE 12 /* IV's for AES_GCM should normally be 12 bytes */ +#define GCM_IV_MAX_SIZE (1024 / 8) +#define GCM_TAG_MAX_SIZE 16 -# if defined(OPENSSL_CPUID_OBJ) && defined(__s390__) +#if defined(OPENSSL_CPUID_OBJ) && defined(__s390__) /*- * KMA-GCM-AES parameter block - begin * (see z/Architecture Principles of Operation >= SA22-7832-11) @@ -46,7 +42,7 @@ typedef struct S390X_kma_params_st { unsigned char k[32]; /* key */ } S390X_KMA_PARAMS; -# endif +#endif typedef struct prov_gcm_ctx_st { unsigned int mode; /* The mode that we are using */ @@ -79,6 +75,7 @@ typedef struct prov_gcm_ctx_st { const PROV_GCM_HW *hw; /* hardware specific methods */ GCM128_CONTEXT gcm; ctr128_f ctr; + const void *ks; } PROV_GCM_CTX; PROV_CIPHER_FUNC(int, GCM_setkey, (PROV_GCM_CTX *ctx, const unsigned char *key, @@ -124,10 +121,9 @@ int ossl_gcm_one_shot(PROV_GCM_CTX *ctx, unsigned char *aad, size_t aad_len, int ossl_gcm_cipher_update(PROV_GCM_CTX *ctx, const unsigned char *in, size_t len, unsigned char *out); -# define GCM_HW_SET_KEY_CTR_FN(ks, fn_set_enc_key, fn_block, fn_ctr) \ +#define GCM_HW_SET_KEY_CTR_FN(ks, fn_set_enc_key, fn_block, fn_ctr) \ + ctx->ks = ks; \ fn_set_enc_key(key, keylen * 8, ks); \ CRYPTO_gcm128_init(&ctx->gcm, ks, (block128_f)fn_block); \ ctx->ctr = (ctr128_f)fn_ctr; \ ctx->key_set = 1; - -#endif diff --git a/openssl/src/providers/implementations/include/prov/ecx.h b/openssl/src/providers/implementations/include/prov/ecx.h deleted file mode 100644 index 3427d154a..000000000 --- a/openssl/src/providers/implementations/include/prov/ecx.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "crypto/types.h" - -#ifndef OPENSSL_NO_EC - -/* RFC 9180 Labels used for Extract and Expand operations */ - -/* ASCII: "eae_prk", in hex for EBCDIC compatibility */ -#define OSSL_DHKEM_LABEL_EAE_PRK "\x65\x61\x65\x5F\x70\x72\x6B" -/* ASCII: "shared_secret", in hex for EBCDIC compatibility */ -#define OSSL_DHKEM_LABEL_SHARED_SECRET "\x73\x68\x61\x72\x65\x64\x5F\x73\x65\x63\x72\x65\x74" -/* ASCII: "dkp_prk", in hex for EBCDIC compatibility */ -#define OSSL_DHKEM_LABEL_DKP_PRK "\x64\x6B\x70\x5F\x70\x72\x6B" -/* ASCII: "candidate", in hex for EBCDIC compatibility */ -#define OSSL_DHKEM_LABEL_CANDIDATE "\x63\x61\x6E\x64\x69\x64\x61\x74\x65" -/* ASCII: "sk", in hex for EBCDIC compatibility */ -#define OSSL_DHKEM_LABEL_SK "\x73\x6B" - -int ossl_ecx_dhkem_derive_private(ECX_KEY *ecx, unsigned char *privout, - const unsigned char *ikm, size_t ikmlen); -int ossl_ec_dhkem_derive_private(EC_KEY *ec, BIGNUM *privout, - const unsigned char *ikm, size_t ikmlen); -#endif diff --git a/openssl/src/providers/implementations/include/prov/hmac_drbg.h b/openssl/src/providers/implementations/include/prov/hmac_drbg.h deleted file mode 100644 index 28aa5bc1a..000000000 --- a/openssl/src/providers/implementations/include/prov/hmac_drbg.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_PROV_HMAC_DRBG_H -# define OSSL_PROV_HMAC_DRBG_H -# pragma once - -#include -#include "prov/provider_util.h" - -typedef struct drbg_hmac_st { - EVP_MAC_CTX *ctx; /* H(x) = HMAC_hash OR H(x) = KMAC */ - PROV_DIGEST digest; /* H(x) = hash(x) */ - size_t blocklen; - unsigned char K[EVP_MAX_MD_SIZE]; - unsigned char V[EVP_MAX_MD_SIZE]; -} PROV_DRBG_HMAC; - -int ossl_drbg_hmac_init(PROV_DRBG_HMAC *drbg, - const unsigned char *ent, size_t ent_len, - const unsigned char *nonce, size_t nonce_len, - const unsigned char *pstr, size_t pstr_len); -int ossl_drbg_hmac_generate(PROV_DRBG_HMAC *hmac, - unsigned char *out, size_t outlen, - const unsigned char *adin, size_t adin_len); - -#endif /* OSSL_PROV_HMAC_DRBG_H */ diff --git a/openssl/src/providers/implementations/include/prov/implementations.h b/openssl/src/providers/implementations/include/prov/implementations.h index 80b544c42..85e52074e 100644 --- a/openssl/src/providers/implementations/include/prov/implementations.h +++ b/openssl/src/providers/implementations/include/prov/implementations.h @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,7 +14,6 @@ extern const OSSL_DISPATCH ossl_sha1_functions[]; extern const OSSL_DISPATCH ossl_sha224_functions[]; extern const OSSL_DISPATCH ossl_sha256_functions[]; -extern const OSSL_DISPATCH ossl_sha256_192_functions[]; extern const OSSL_DISPATCH ossl_sha384_functions[]; extern const OSSL_DISPATCH ossl_sha512_functions[]; extern const OSSL_DISPATCH ossl_sha512_224_functions[]; @@ -23,24 +22,14 @@ extern const OSSL_DISPATCH ossl_sha3_224_functions[]; extern const OSSL_DISPATCH ossl_sha3_256_functions[]; extern const OSSL_DISPATCH ossl_sha3_384_functions[]; extern const OSSL_DISPATCH ossl_sha3_512_functions[]; -extern const OSSL_DISPATCH ossl_keccak_224_functions[]; -extern const OSSL_DISPATCH ossl_keccak_256_functions[]; -extern const OSSL_DISPATCH ossl_keccak_384_functions[]; -extern const OSSL_DISPATCH ossl_keccak_512_functions[]; extern const OSSL_DISPATCH ossl_keccak_kmac_128_functions[]; extern const OSSL_DISPATCH ossl_keccak_kmac_256_functions[]; extern const OSSL_DISPATCH ossl_shake_128_functions[]; extern const OSSL_DISPATCH ossl_shake_256_functions[]; -extern const OSSL_DISPATCH ossl_blake2s256_functions[]; -extern const OSSL_DISPATCH ossl_blake2b512_functions[]; extern const OSSL_DISPATCH ossl_md5_functions[]; extern const OSSL_DISPATCH ossl_md5_sha1_functions[]; extern const OSSL_DISPATCH ossl_sm3_functions[]; -extern const OSSL_DISPATCH ossl_md2_functions[]; -extern const OSSL_DISPATCH ossl_md4_functions[]; -extern const OSSL_DISPATCH ossl_mdc2_functions[]; extern const OSSL_DISPATCH ossl_wp_functions[]; -extern const OSSL_DISPATCH ossl_ripemd160_functions[]; extern const OSSL_DISPATCH ossl_nullmd_functions[]; /* Ciphers */ @@ -99,94 +88,14 @@ extern const OSSL_DISPATCH ossl_aes128cbc_hmac_sha1_functions[]; extern const OSSL_DISPATCH ossl_aes256cbc_hmac_sha256_functions[]; extern const OSSL_DISPATCH ossl_aes128cbc_hmac_sha256_functions[]; -#ifndef OPENSSL_NO_ARIA -extern const OSSL_DISPATCH ossl_aria256gcm_functions[]; -extern const OSSL_DISPATCH ossl_aria192gcm_functions[]; -extern const OSSL_DISPATCH ossl_aria128gcm_functions[]; -extern const OSSL_DISPATCH ossl_aria256ccm_functions[]; -extern const OSSL_DISPATCH ossl_aria192ccm_functions[]; -extern const OSSL_DISPATCH ossl_aria128ccm_functions[]; -extern const OSSL_DISPATCH ossl_aria256ecb_functions[]; -extern const OSSL_DISPATCH ossl_aria192ecb_functions[]; -extern const OSSL_DISPATCH ossl_aria128ecb_functions[]; -extern const OSSL_DISPATCH ossl_aria256cbc_functions[]; -extern const OSSL_DISPATCH ossl_aria192cbc_functions[]; -extern const OSSL_DISPATCH ossl_aria128cbc_functions[]; -extern const OSSL_DISPATCH ossl_aria256ofb_functions[]; -extern const OSSL_DISPATCH ossl_aria192ofb_functions[]; -extern const OSSL_DISPATCH ossl_aria128ofb_functions[]; -extern const OSSL_DISPATCH ossl_aria256cfb_functions[]; -extern const OSSL_DISPATCH ossl_aria192cfb_functions[]; -extern const OSSL_DISPATCH ossl_aria128cfb_functions[]; -extern const OSSL_DISPATCH ossl_aria256cfb1_functions[]; -extern const OSSL_DISPATCH ossl_aria192cfb1_functions[]; -extern const OSSL_DISPATCH ossl_aria128cfb1_functions[]; -extern const OSSL_DISPATCH ossl_aria256cfb8_functions[]; -extern const OSSL_DISPATCH ossl_aria192cfb8_functions[]; -extern const OSSL_DISPATCH ossl_aria128cfb8_functions[]; -extern const OSSL_DISPATCH ossl_aria256ctr_functions[]; -extern const OSSL_DISPATCH ossl_aria192ctr_functions[]; -extern const OSSL_DISPATCH ossl_aria128ctr_functions[]; -#endif /* OPENSSL_NO_ARIA */ -#ifndef OPENSSL_NO_CAMELLIA -extern const OSSL_DISPATCH ossl_camellia256ecb_functions[]; -extern const OSSL_DISPATCH ossl_camellia192ecb_functions[]; -extern const OSSL_DISPATCH ossl_camellia128ecb_functions[]; -extern const OSSL_DISPATCH ossl_camellia256cbc_functions[]; -extern const OSSL_DISPATCH ossl_camellia192cbc_functions[]; -extern const OSSL_DISPATCH ossl_camellia128cbc_functions[]; -extern const OSSL_DISPATCH ossl_camellia256cbc_cts_functions[]; -extern const OSSL_DISPATCH ossl_camellia192cbc_cts_functions[]; -extern const OSSL_DISPATCH ossl_camellia128cbc_cts_functions[]; -extern const OSSL_DISPATCH ossl_camellia256ofb_functions[]; -extern const OSSL_DISPATCH ossl_camellia192ofb_functions[]; -extern const OSSL_DISPATCH ossl_camellia128ofb_functions[]; -extern const OSSL_DISPATCH ossl_camellia256cfb_functions[]; -extern const OSSL_DISPATCH ossl_camellia192cfb_functions[]; -extern const OSSL_DISPATCH ossl_camellia128cfb_functions[]; -extern const OSSL_DISPATCH ossl_camellia256cfb1_functions[]; -extern const OSSL_DISPATCH ossl_camellia192cfb1_functions[]; -extern const OSSL_DISPATCH ossl_camellia128cfb1_functions[]; -extern const OSSL_DISPATCH ossl_camellia256cfb8_functions[]; -extern const OSSL_DISPATCH ossl_camellia192cfb8_functions[]; -extern const OSSL_DISPATCH ossl_camellia128cfb8_functions[]; -extern const OSSL_DISPATCH ossl_camellia256ctr_functions[]; -extern const OSSL_DISPATCH ossl_camellia192ctr_functions[]; -extern const OSSL_DISPATCH ossl_camellia128ctr_functions[]; -#endif /* OPENSSL_NO_CAMELLIA */ -#ifndef OPENSSL_NO_BF -extern const OSSL_DISPATCH ossl_blowfish128ecb_functions[]; -extern const OSSL_DISPATCH ossl_blowfish128cbc_functions[]; -extern const OSSL_DISPATCH ossl_blowfish128ofb64_functions[]; -extern const OSSL_DISPATCH ossl_blowfish128cfb64_functions[]; -#endif /* OPENSSL_NO_BF */ -#ifndef OPENSSL_NO_IDEA -extern const OSSL_DISPATCH ossl_idea128ecb_functions[]; -extern const OSSL_DISPATCH ossl_idea128cbc_functions[]; -extern const OSSL_DISPATCH ossl_idea128ofb64_functions[]; -extern const OSSL_DISPATCH ossl_idea128cfb64_functions[]; -#endif /* OPENSSL_NO_IDEA */ -#ifndef OPENSSL_NO_CAST -extern const OSSL_DISPATCH ossl_cast5128ecb_functions[]; -extern const OSSL_DISPATCH ossl_cast5128cbc_functions[]; -extern const OSSL_DISPATCH ossl_cast5128ofb64_functions[]; -extern const OSSL_DISPATCH ossl_cast5128cfb64_functions[]; -#endif /* OPENSSL_NO_CAST */ -#ifndef OPENSSL_NO_SEED -extern const OSSL_DISPATCH ossl_seed128ecb_functions[]; -extern const OSSL_DISPATCH ossl_seed128cbc_functions[]; -extern const OSSL_DISPATCH ossl_seed128ofb128_functions[]; -extern const OSSL_DISPATCH ossl_seed128cfb128_functions[]; -#endif /* OPENSSL_NO_SEED */ #ifndef OPENSSL_NO_SM4 -extern const OSSL_DISPATCH ossl_sm4128gcm_functions[]; -extern const OSSL_DISPATCH ossl_sm4128ccm_functions[]; extern const OSSL_DISPATCH ossl_sm4128ecb_functions[]; extern const OSSL_DISPATCH ossl_sm4128cbc_functions[]; extern const OSSL_DISPATCH ossl_sm4128ctr_functions[]; extern const OSSL_DISPATCH ossl_sm4128ofb128_functions[]; extern const OSSL_DISPATCH ossl_sm4128cfb128_functions[]; -extern const OSSL_DISPATCH ossl_sm4128xts_functions[]; +extern const OSSL_DISPATCH ossl_sm4128gcm_functions[]; +extern const OSSL_DISPATCH ossl_sm4128ccm_functions[]; #endif /* OPENSSL_NO_SM4 */ #ifndef OPENSSL_NO_RC5 extern const OSSL_DISPATCH ossl_rc5128ecb_functions[]; @@ -194,14 +103,6 @@ extern const OSSL_DISPATCH ossl_rc5128cbc_functions[]; extern const OSSL_DISPATCH ossl_rc5128ofb64_functions[]; extern const OSSL_DISPATCH ossl_rc5128cfb64_functions[]; #endif /* OPENSSL_NO_RC5 */ -#ifndef OPENSSL_NO_RC2 -extern const OSSL_DISPATCH ossl_rc2128ecb_functions[]; -extern const OSSL_DISPATCH ossl_rc2128cbc_functions[]; -extern const OSSL_DISPATCH ossl_rc240cbc_functions[]; -extern const OSSL_DISPATCH ossl_rc264cbc_functions[]; -extern const OSSL_DISPATCH ossl_rc2128cfb128_functions[]; -extern const OSSL_DISPATCH ossl_rc2128ofb128_functions[]; -#endif /* OPENSSL_NO_RC2 */ #ifndef OPENSSL_NO_DES extern const OSSL_DISPATCH ossl_tdes_ede3_ecb_functions[]; extern const OSSL_DISPATCH ossl_tdes_ede3_cbc_functions[]; @@ -242,19 +143,17 @@ extern const OSSL_DISPATCH ossl_chacha20_ossl_poly1305_functions[]; # endif /* OPENSSL_NO_POLY1305 */ #endif /* OPENSSL_NO_CHACHA */ +#ifndef OPENSSL_NO_ZUC +extern const OSSL_DISPATCH ossl_zuc_128_eea3_functions[]; +#endif /* OPENSSL_NO_ZUC */ #ifndef OPENSSL_NO_SIV extern const OSSL_DISPATCH ossl_aes128siv_functions[]; extern const OSSL_DISPATCH ossl_aes192siv_functions[]; extern const OSSL_DISPATCH ossl_aes256siv_functions[]; -extern const OSSL_DISPATCH ossl_aes128gcm_siv_functions[]; -extern const OSSL_DISPATCH ossl_aes192gcm_siv_functions[]; -extern const OSSL_DISPATCH ossl_aes256gcm_siv_functions[]; #endif /* OPENSSL_NO_SIV */ /* MACs */ -extern const OSSL_DISPATCH ossl_blake2bmac_functions[]; -extern const OSSL_DISPATCH ossl_blake2smac_functions[]; extern const OSSL_DISPATCH ossl_cmac_functions[]; extern const OSSL_DISPATCH ossl_gmac_functions[]; extern const OSSL_DISPATCH ossl_hmac_functions[]; @@ -262,11 +161,13 @@ extern const OSSL_DISPATCH ossl_kmac128_functions[]; extern const OSSL_DISPATCH ossl_kmac256_functions[]; extern const OSSL_DISPATCH ossl_siphash_functions[]; extern const OSSL_DISPATCH ossl_poly1305_functions[]; +#ifndef OPENSSL_NO_ZUC +extern const OSSL_DISPATCH ossl_eia3_functions[]; +#endif /* KDFs / PRFs */ extern const OSSL_DISPATCH ossl_kdf_pbkdf1_functions[]; extern const OSSL_DISPATCH ossl_kdf_pbkdf2_functions[]; -extern const OSSL_DISPATCH ossl_kdf_pvk_functions[]; extern const OSSL_DISPATCH ossl_kdf_pkcs12_functions[]; #ifndef OPENSSL_NO_SCRYPT extern const OSSL_DISPATCH ossl_kdf_scrypt_functions[]; @@ -280,12 +181,6 @@ extern const OSSL_DISPATCH ossl_kdf_x963_kdf_functions[]; extern const OSSL_DISPATCH ossl_kdf_kbkdf_functions[]; extern const OSSL_DISPATCH ossl_kdf_x942_kdf_functions[]; extern const OSSL_DISPATCH ossl_kdf_krb5kdf_functions[]; -extern const OSSL_DISPATCH ossl_kdf_hmac_drbg_functions[]; -#ifndef OPENSSL_NO_ARGON2 -extern const OSSL_DISPATCH ossl_kdf_argon2i_functions[]; -extern const OSSL_DISPATCH ossl_kdf_argon2d_functions[]; -extern const OSSL_DISPATCH ossl_kdf_argon2id_functions[]; -#endif /* RNGs */ extern const OSSL_DISPATCH ossl_test_rng_functions[]; @@ -301,12 +196,10 @@ extern const OSSL_DISPATCH ossl_dhx_keymgmt_functions[]; extern const OSSL_DISPATCH ossl_dsa_keymgmt_functions[]; extern const OSSL_DISPATCH ossl_rsa_keymgmt_functions[]; extern const OSSL_DISPATCH ossl_rsapss_keymgmt_functions[]; -#ifndef OPENSSL_NO_ECX extern const OSSL_DISPATCH ossl_x25519_keymgmt_functions[]; extern const OSSL_DISPATCH ossl_x448_keymgmt_functions[]; extern const OSSL_DISPATCH ossl_ed25519_keymgmt_functions[]; extern const OSSL_DISPATCH ossl_ed448_keymgmt_functions[]; -#endif extern const OSSL_DISPATCH ossl_ec_keymgmt_functions[]; extern const OSSL_DISPATCH ossl_kdf_keymgmt_functions[]; extern const OSSL_DISPATCH ossl_mac_legacy_keymgmt_functions[]; @@ -320,6 +213,9 @@ extern const OSSL_DISPATCH ossl_dh_keyexch_functions[]; extern const OSSL_DISPATCH ossl_x25519_keyexch_functions[]; extern const OSSL_DISPATCH ossl_x448_keyexch_functions[]; extern const OSSL_DISPATCH ossl_ecdh_keyexch_functions[]; +#ifndef OPENSSL_NO_SM2 +extern const OSSL_DISPATCH ossl_sm2dh_keyexch_functions[]; +#endif extern const OSSL_DISPATCH ossl_kdf_tls1_prf_keyexch_functions[]; extern const OSSL_DISPATCH ossl_kdf_hkdf_keyexch_functions[]; extern const OSSL_DISPATCH ossl_kdf_scrypt_keyexch_functions[]; @@ -334,6 +230,7 @@ extern const OSSL_DISPATCH ossl_mac_legacy_hmac_signature_functions[]; extern const OSSL_DISPATCH ossl_mac_legacy_siphash_signature_functions[]; extern const OSSL_DISPATCH ossl_mac_legacy_poly1305_signature_functions[]; extern const OSSL_DISPATCH ossl_mac_legacy_cmac_signature_functions[]; +extern const OSSL_DISPATCH ossl_mac_legacy_eia3_signature_functions[]; extern const OSSL_DISPATCH ossl_sm2_signature_functions[]; /* Asym Cipher */ @@ -344,8 +241,6 @@ extern const OSSL_DISPATCH ossl_sm2_asym_cipher_functions[]; /* Asym Key encapsulation */ extern const OSSL_DISPATCH ossl_rsa_asym_kem_functions[]; -extern const OSSL_DISPATCH ossl_ecx_asym_kem_functions[]; -extern const OSSL_DISPATCH ossl_ec_asym_kem_functions[]; /* Encoders */ extern const OSSL_DISPATCH ossl_rsa_to_PKCS1_der_encoder_functions[]; @@ -538,4 +433,3 @@ extern const OSSL_DISPATCH ossl_SubjectPublicKeyInfo_der_to_der_decoder_function extern const OSSL_DISPATCH ossl_pem_to_der_decoder_functions[]; extern const OSSL_DISPATCH ossl_file_store_functions[]; -extern const OSSL_DISPATCH ossl_winstore_store_functions[]; diff --git a/openssl/src/providers/implementations/include/prov/kdfexchange.h b/openssl/src/providers/implementations/include/prov/kdfexchange.h index cf08f785e..bfedd3afd 100644 --- a/openssl/src/providers/implementations/include/prov/kdfexchange.h +++ b/openssl/src/providers/implementations/include/prov/kdfexchange.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,6 +14,7 @@ struct kdf_data_st { OSSL_LIB_CTX *libctx; CRYPTO_REF_COUNT refcnt; + CRYPTO_RWLOCK *lock; }; typedef struct kdf_data_st KDF_DATA; diff --git a/openssl/src/providers/implementations/include/prov/macsignature.h b/openssl/src/providers/implementations/include/prov/macsignature.h index e13ff362c..9bfaaf9b6 100644 --- a/openssl/src/providers/implementations/include/prov/macsignature.h +++ b/openssl/src/providers/implementations/include/prov/macsignature.h @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,6 +13,7 @@ #include "prov/provider_util.h" struct mac_key_st { + CRYPTO_RWLOCK *lock; OSSL_LIB_CTX *libctx; CRYPTO_REF_COUNT refcnt; unsigned char *priv_key; diff --git a/openssl/src/providers/implementations/include/prov/names.h b/openssl/src/providers/implementations/include/prov/names.h index f0ad43534..2b672c29a 100644 --- a/openssl/src/providers/implementations/include/prov/names.h +++ b/openssl/src/providers/implementations/include/prov/names.h @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,7 +22,7 @@ * ALGNAME[VERSION?][-SUBNAME[VERSION?]?][-SIZE?][-MODE?] * * VERSION is only present if there are multiple versions of - * an alg (MD2, MD4, MD5). It may be omitted if there is only + * an alg (MD series). It may be omitted if there is only * one version (if a subsequent version is released in the future, * we can always change the canonical name, and add the old name * as an alias). @@ -97,60 +97,6 @@ #define PROV_NAMES_AES_128_SIV "AES-128-SIV" #define PROV_NAMES_AES_192_SIV "AES-192-SIV" #define PROV_NAMES_AES_256_SIV "AES-256-SIV" -#define PROV_NAMES_AES_128_GCM_SIV "AES-128-GCM-SIV" -#define PROV_NAMES_AES_192_GCM_SIV "AES-192-GCM-SIV" -#define PROV_NAMES_AES_256_GCM_SIV "AES-256-GCM-SIV" -#define PROV_NAMES_ARIA_256_GCM "ARIA-256-GCM:1.2.410.200046.1.1.36" -#define PROV_NAMES_ARIA_192_GCM "ARIA-192-GCM:1.2.410.200046.1.1.35" -#define PROV_NAMES_ARIA_128_GCM "ARIA-128-GCM:1.2.410.200046.1.1.34" -#define PROV_NAMES_ARIA_256_CCM "ARIA-256-CCM:1.2.410.200046.1.1.39" -#define PROV_NAMES_ARIA_192_CCM "ARIA-192-CCM:1.2.410.200046.1.1.38" -#define PROV_NAMES_ARIA_128_CCM "ARIA-128-CCM:1.2.410.200046.1.1.37" -#define PROV_NAMES_ARIA_256_ECB "ARIA-256-ECB:1.2.410.200046.1.1.11" -#define PROV_NAMES_ARIA_192_ECB "ARIA-192-ECB:1.2.410.200046.1.1.6" -#define PROV_NAMES_ARIA_128_ECB "ARIA-128-ECB:1.2.410.200046.1.1.1" -#define PROV_NAMES_ARIA_256_CBC "ARIA-256-CBC:ARIA256:1.2.410.200046.1.1.12" -#define PROV_NAMES_ARIA_192_CBC "ARIA-192-CBC:ARIA192:1.2.410.200046.1.1.7" -#define PROV_NAMES_ARIA_128_CBC "ARIA-128-CBC:ARIA128:1.2.410.200046.1.1.2" -#define PROV_NAMES_ARIA_256_OFB "ARIA-256-OFB:1.2.410.200046.1.1.14" -#define PROV_NAMES_ARIA_192_OFB "ARIA-192-OFB:1.2.410.200046.1.1.9" -#define PROV_NAMES_ARIA_128_OFB "ARIA-128-OFB:1.2.410.200046.1.1.4" -#define PROV_NAMES_ARIA_256_CFB "ARIA-256-CFB:1.2.410.200046.1.1.13" -#define PROV_NAMES_ARIA_192_CFB "ARIA-192-CFB:1.2.410.200046.1.1.8" -#define PROV_NAMES_ARIA_128_CFB "ARIA-128-CFB:1.2.410.200046.1.1.3" -#define PROV_NAMES_ARIA_256_CFB1 "ARIA-256-CFB1" -#define PROV_NAMES_ARIA_192_CFB1 "ARIA-192-CFB1" -#define PROV_NAMES_ARIA_128_CFB1 "ARIA-128-CFB1" -#define PROV_NAMES_ARIA_256_CFB8 "ARIA-256-CFB8" -#define PROV_NAMES_ARIA_192_CFB8 "ARIA-192-CFB8" -#define PROV_NAMES_ARIA_128_CFB8 "ARIA-128-CFB8" -#define PROV_NAMES_ARIA_256_CTR "ARIA-256-CTR:1.2.410.200046.1.1.15" -#define PROV_NAMES_ARIA_192_CTR "ARIA-192-CTR:1.2.410.200046.1.1.10" -#define PROV_NAMES_ARIA_128_CTR "ARIA-128-CTR:1.2.410.200046.1.1.5" -#define PROV_NAMES_CAMELLIA_256_ECB "CAMELLIA-256-ECB:0.3.4401.5.3.1.9.41" -#define PROV_NAMES_CAMELLIA_192_ECB "CAMELLIA-192-ECB:0.3.4401.5.3.1.9.21" -#define PROV_NAMES_CAMELLIA_128_ECB "CAMELLIA-128-ECB:0.3.4401.5.3.1.9.1" -#define PROV_NAMES_CAMELLIA_256_CBC "CAMELLIA-256-CBC:CAMELLIA256:1.2.392.200011.61.1.1.1.4" -#define PROV_NAMES_CAMELLIA_192_CBC "CAMELLIA-192-CBC:CAMELLIA192:1.2.392.200011.61.1.1.1.3" -#define PROV_NAMES_CAMELLIA_128_CBC "CAMELLIA-128-CBC:CAMELLIA128:1.2.392.200011.61.1.1.1.2" -#define PROV_NAMES_CAMELLIA_256_CBC_CTS "CAMELLIA-256-CBC-CTS" -#define PROV_NAMES_CAMELLIA_192_CBC_CTS "CAMELLIA-192-CBC-CTS" -#define PROV_NAMES_CAMELLIA_128_CBC_CTS "CAMELLIA-128-CBC-CTS" -#define PROV_NAMES_CAMELLIA_256_OFB "CAMELLIA-256-OFB:0.3.4401.5.3.1.9.43" -#define PROV_NAMES_CAMELLIA_192_OFB "CAMELLIA-192-OFB:0.3.4401.5.3.1.9.23" -#define PROV_NAMES_CAMELLIA_128_OFB "CAMELLIA-128-OFB:0.3.4401.5.3.1.9.3" -#define PROV_NAMES_CAMELLIA_256_CFB "CAMELLIA-256-CFB:0.3.4401.5.3.1.9.44" -#define PROV_NAMES_CAMELLIA_192_CFB "CAMELLIA-192-CFB:0.3.4401.5.3.1.9.24" -#define PROV_NAMES_CAMELLIA_128_CFB "CAMELLIA-128-CFB:0.3.4401.5.3.1.9.4" -#define PROV_NAMES_CAMELLIA_256_CFB1 "CAMELLIA-256-CFB1" -#define PROV_NAMES_CAMELLIA_192_CFB1 "CAMELLIA-192-CFB1" -#define PROV_NAMES_CAMELLIA_128_CFB1 "CAMELLIA-128-CFB1" -#define PROV_NAMES_CAMELLIA_256_CFB8 "CAMELLIA-256-CFB8" -#define PROV_NAMES_CAMELLIA_192_CFB8 "CAMELLIA-192-CFB8" -#define PROV_NAMES_CAMELLIA_128_CFB8 "CAMELLIA-128-CFB8" -#define PROV_NAMES_CAMELLIA_256_CTR "CAMELLIA-256-CTR:0.3.4401.5.3.1.9.49" -#define PROV_NAMES_CAMELLIA_192_CTR "CAMELLIA-192-CTR:0.3.4401.5.3.1.9.29" -#define PROV_NAMES_CAMELLIA_128_CTR "CAMELLIA-128-CTR:0.3.4401.5.3.1.9.9" #define PROV_NAMES_DES_EDE3_OFB "DES-EDE3-OFB" #define PROV_NAMES_DES_EDE3_CFB "DES-EDE3-CFB" #define PROV_NAMES_DES_EDE3_CFB8 "DES-EDE3-CFB8" @@ -167,31 +113,8 @@ #define PROV_NAMES_SM4_CFB "SM4-CFB:SM4-CFB128:1.2.156.10197.1.104.4" #define PROV_NAMES_SM4_GCM "SM4-GCM:1.2.156.10197.1.104.8" #define PROV_NAMES_SM4_CCM "SM4-CCM:1.2.156.10197.1.104.9" -#define PROV_NAMES_SM4_XTS "SM4-XTS:1.2.156.10197.1.104.10" #define PROV_NAMES_ChaCha20 "ChaCha20" #define PROV_NAMES_ChaCha20_Poly1305 "ChaCha20-Poly1305" -#define PROV_NAMES_CAST5_ECB "CAST5-ECB" -#define PROV_NAMES_CAST5_CBC "CAST5-CBC:CAST-CBC:CAST:1.2.840.113533.7.66.10" -#define PROV_NAMES_CAST5_OFB "CAST5-OFB" -#define PROV_NAMES_CAST5_CFB "CAST5-CFB" -#define PROV_NAMES_BF_ECB "BF-ECB" -#define PROV_NAMES_BF_CBC "BF-CBC:BF:BLOWFISH:1.3.6.1.4.1.3029.1.2" -#define PROV_NAMES_BF_OFB "BF-OFB" -#define PROV_NAMES_BF_CFB "BF-CFB" -#define PROV_NAMES_IDEA_ECB "IDEA-ECB" -#define PROV_NAMES_IDEA_CBC "IDEA-CBC:IDEA:1.3.6.1.4.1.188.7.1.1.2" -#define PROV_NAMES_IDEA_OFB "IDEA-OFB:IDEA-OFB64" -#define PROV_NAMES_IDEA_CFB "IDEA-CFB:IDEA-CFB64" -#define PROV_NAMES_SEED_ECB "SEED-ECB:1.2.410.200004.1.3" -#define PROV_NAMES_SEED_CBC "SEED-CBC:SEED:1.2.410.200004.1.4" -#define PROV_NAMES_SEED_OFB "SEED-OFB:SEED-OFB128:1.2.410.200004.1.6" -#define PROV_NAMES_SEED_CFB "SEED-CFB:SEED-CFB128:1.2.410.200004.1.5" -#define PROV_NAMES_RC2_ECB "RC2-ECB" -#define PROV_NAMES_RC2_CBC "RC2-CBC:RC2:RC2-128:1.2.840.113549.3.2" -#define PROV_NAMES_RC2_40_CBC "RC2-40-CBC:RC2-40" -#define PROV_NAMES_RC2_64_CBC "RC2-64-CBC:RC2-64" -#define PROV_NAMES_RC2_CFB "RC2-CFB" -#define PROV_NAMES_RC2_OFB "RC2-OFB" #define PROV_NAMES_RC4 "RC4:1.2.840.113549.3.4" #define PROV_NAMES_RC4_40 "RC4-40" #define PROV_NAMES_RC4_HMAC_MD5 "RC4-HMAC-MD5" @@ -206,6 +129,7 @@ #define PROV_NAMES_DES_CFB "DES-CFB:1.3.14.3.2.9" #define PROV_NAMES_DES_CFB1 "DES-CFB1" #define PROV_NAMES_DES_CFB8 "DES-CFB8" +#define PROV_NAMES_ZUC_128_EEA3 "ZUC-128-EEA3" /*- * Digests @@ -214,7 +138,6 @@ #define PROV_NAMES_SHA1 "SHA1:SHA-1:SSL3-SHA1:1.3.14.3.2.26" #define PROV_NAMES_SHA2_224 "SHA2-224:SHA-224:SHA224:2.16.840.1.101.3.4.2.4" #define PROV_NAMES_SHA2_256 "SHA2-256:SHA-256:SHA256:2.16.840.1.101.3.4.2.1" -#define PROV_NAMES_SHA2_256_192 "SHA2-256/192:SHA-256/192:SHA256-192" #define PROV_NAMES_SHA2_384 "SHA2-384:SHA-384:SHA384:2.16.840.1.101.3.4.2.2" #define PROV_NAMES_SHA2_512 "SHA2-512:SHA-512:SHA512:2.16.840.1.101.3.4.2.3" #define PROV_NAMES_SHA2_512_224 "SHA2-512/224:SHA-512/224:SHA512-224:2.16.840.1.101.3.4.2.5" @@ -226,37 +149,18 @@ #define PROV_NAMES_SHA3_384 "SHA3-384:2.16.840.1.101.3.4.2.9" #define PROV_NAMES_SHA3_512 "SHA3-512:2.16.840.1.101.3.4.2.10" -#define PROV_NAMES_KECCAK_224 "KECCAK-224" -#define PROV_NAMES_KECCAK_256 "KECCAK-256" -#define PROV_NAMES_KECCAK_384 "KECCAK-384" -#define PROV_NAMES_KECCAK_512 "KECCAK-512" - #define PROV_NAMES_SHAKE_128 "SHAKE-128:SHAKE128:2.16.840.1.101.3.4.2.11" #define PROV_NAMES_SHAKE_256 "SHAKE-256:SHAKE256:2.16.840.1.101.3.4.2.12" /* - * KECCAK-KMAC-128 and KECCAK-KMAC-256 as hashes are mostly useful for + * KECCAK-KMAC-128 and KECCAK-KMAC-256 as hashes are mostly useful for * KMAC128 and KMAC256. */ #define PROV_NAMES_KECCAK_KMAC_128 "KECCAK-KMAC-128:KECCAK-KMAC128" #define PROV_NAMES_KECCAK_KMAC_256 "KECCAK-KMAC-256:KECCAK-KMAC256" -/* - * https://blake2.net/ doesn't specify size variants, but mentions that - * Bouncy Castle uses the names BLAKE2b-160, BLAKE2b-256, BLAKE2b-384, and - * BLAKE2b-512 - * If we assume that "2b" and "2s" are versions, that pattern fits with ours. - * We also add our historical names. - */ -#define PROV_NAMES_BLAKE2S_256 "BLAKE2S-256:BLAKE2s256:1.3.6.1.4.1.1722.12.2.2.8" -#define PROV_NAMES_BLAKE2B_512 "BLAKE2B-512:BLAKE2b512:1.3.6.1.4.1.1722.12.2.1.16" #define PROV_NAMES_SM3 "SM3:1.2.156.10197.1.401" #define PROV_NAMES_MD5 "MD5:SSL3-MD5:1.2.840.113549.2.5" #define PROV_NAMES_MD5_SHA1 "MD5-SHA1" -#define PROV_NAMES_MD2 "MD2:1.2.840.113549.2.2" -#define PROV_NAMES_MD4 "MD4:1.2.840.113549.2.4" -#define PROV_NAMES_MDC2 "MDC2:2.5.8.3.101" -#define PROV_NAMES_WHIRLPOOL "WHIRLPOOL:1.0.10118.3.0.55" -#define PROV_NAMES_RIPEMD_160 "RIPEMD-160:RIPEMD160:RIPEMD:RMD160:1.3.36.3.2.1" /*- * KDFs / PRFs @@ -268,7 +172,6 @@ #define PROV_NAMES_SSKDF "SSKDF" #define PROV_NAMES_PBKDF1 "PBKDF1" #define PROV_NAMES_PBKDF2 "PBKDF2:1.2.840.113549.1.5.12" -#define PROV_NAMES_PVKKDF "PVKKDF" #define PROV_NAMES_SSHKDF "SSHKDF" #define PROV_NAMES_X963KDF "X963KDF:X942KDF-CONCAT" #define PROV_NAMES_X942KDF_ASN1 "X942KDF-ASN1:X942KDF" @@ -279,10 +182,6 @@ #define PROV_NAMES_SCRYPT "SCRYPT:id-scrypt:1.3.6.1.4.1.11591.4.11" #define PROV_DESCS_SCRYPT_SIGN "OpenSSL SCRYPT via EVP_PKEY implementation" #define PROV_NAMES_KRB5KDF "KRB5KDF" -#define PROV_NAMES_HMAC_DRBG_KDF "HMAC-DRBG-KDF" -#define PROV_NAMES_ARGON2I "ARGON2I" -#define PROV_NAMES_ARGON2D "ARGON2D" -#define PROV_NAMES_ARGON2ID "ARGON2ID" /*- * MACs @@ -299,8 +198,8 @@ #define PROV_NAMES_GMAC "GMAC:1.0.9797.3.4" #define PROV_NAMES_KMAC_128 "KMAC-128:KMAC128:2.16.840.1.101.3.4.2.19" #define PROV_NAMES_KMAC_256 "KMAC-256:KMAC256:2.16.840.1.101.3.4.2.20" -#define PROV_NAMES_BLAKE2BMAC "BLAKE2BMAC:1.3.6.1.4.1.1722.12.2.1" -#define PROV_NAMES_BLAKE2SMAC "BLAKE2SMAC:1.3.6.1.4.1.1722.12.2.2" +#define PROV_NAMES_EIA3 "EIA3" +#define PROV_DESCS_EIA3_SIGN "Tongsuo EIA3 via EVP_PKEY implementation" /*- * RANDs @@ -342,3 +241,5 @@ #define PROV_DESCS_RSA_PSS "OpenSSL RSA-PSS implementation" #define PROV_NAMES_SM2 "SM2:1.2.156.10197.1.301" #define PROV_DESCS_SM2 "OpenSSL SM2 implementation" +#define PROV_NAMES_SM2DH "SM2DH" +#define PROV_DESCS_SM2DH "Tongsuo SM2DH implementation" diff --git a/openssl/src/providers/implementations/include/prov/seeding.h b/openssl/src/providers/implementations/include/prov/seeding.h index af6cb79fb..637b921b2 100644 --- a/openssl/src/providers/implementations/include/prov/seeding.h +++ b/openssl/src/providers/implementations/include/prov/seeding.h @@ -14,6 +14,17 @@ size_t ossl_prov_acquire_entropy_from_tsc(RAND_POOL *pool); size_t ossl_prov_acquire_entropy_from_cpu(RAND_POOL *pool); +/* + * Add some platform specific additional data + * + * This function is platform specific and adds some random noise to the + * additional data used for generating random bytes and for reseeding + * the drbg. + * + * Returns 1 on success and 0 on failure. + */ +int ossl_rand_pool_add_additional_data(RAND_POOL *pool); + /* * External seeding functions from the core dispatch table. */ diff --git a/openssl/src/providers/implementations/kdfs/argon2.c b/openssl/src/providers/implementations/kdfs/argon2.c deleted file mode 100644 index fe84ab54c..000000000 --- a/openssl/src/providers/implementations/kdfs/argon2.c +++ /dev/null @@ -1,1560 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - * - * RFC 9106 Argon2 (see https://www.rfc-editor.org/rfc/rfc9106.txt) - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "internal/thread.h" -#include "internal/numbers.h" -#include "internal/endian.h" -#include "crypto/evp.h" -#include "prov/implementations.h" -#include "prov/provider_ctx.h" -#include "prov/providercommon.h" -#include "prov/blake2.h" - -#if defined(OPENSSL_NO_DEFAULT_THREAD_POOL) && defined(OPENSSL_NO_THREAD_POOL) -# define ARGON2_NO_THREADS -#endif - -#if !defined(OPENSSL_THREADS) -# define ARGON2_NO_THREADS -#endif - -#ifndef OPENSSL_NO_ARGON2 - -# define ARGON2_MIN_LANES 1u -# define ARGON2_MAX_LANES 0xFFFFFFu -# define ARGON2_MIN_THREADS 1u -# define ARGON2_MAX_THREADS 0xFFFFFFu -# define ARGON2_SYNC_POINTS 4u -# define ARGON2_MIN_OUT_LENGTH 4u -# define ARGON2_MAX_OUT_LENGTH 0xFFFFFFFFu -# define ARGON2_MIN_MEMORY (2 * ARGON2_SYNC_POINTS) -# define ARGON2_MIN(a, b) ((a) < (b) ? (a) : (b)) -# define ARGON2_MAX_MEMORY 0xFFFFFFFFu -# define ARGON2_MIN_TIME 1u -# define ARGON2_MAX_TIME 0xFFFFFFFFu -# define ARGON2_MIN_PWD_LENGTH 0u -# define ARGON2_MAX_PWD_LENGTH 0xFFFFFFFFu -# define ARGON2_MIN_AD_LENGTH 0u -# define ARGON2_MAX_AD_LENGTH 0xFFFFFFFFu -# define ARGON2_MIN_SALT_LENGTH 8u -# define ARGON2_MAX_SALT_LENGTH 0xFFFFFFFFu -# define ARGON2_MIN_SECRET 0u -# define ARGON2_MAX_SECRET 0xFFFFFFFFu -# define ARGON2_BLOCK_SIZE 1024 -# define ARGON2_QWORDS_IN_BLOCK ((ARGON2_BLOCK_SIZE) / 8) -# define ARGON2_OWORDS_IN_BLOCK ((ARGON2_BLOCK_SIZE) / 16) -# define ARGON2_HWORDS_IN_BLOCK ((ARGON2_BLOCK_SIZE) / 32) -# define ARGON2_512BIT_WORDS_IN_BLOCK ((ARGON2_BLOCK_SIZE) / 64) -# define ARGON2_ADDRESSES_IN_BLOCK 128 -# define ARGON2_PREHASH_DIGEST_LENGTH 64 -# define ARGON2_PREHASH_SEED_LENGTH \ - (ARGON2_PREHASH_DIGEST_LENGTH + (2 * sizeof(uint32_t))) - -# define ARGON2_DEFAULT_OUTLEN 64u -# define ARGON2_DEFAULT_T_COST 3u -# define ARGON2_DEFAULT_M_COST ARGON2_MIN_MEMORY -# define ARGON2_DEFAULT_LANES 1u -# define ARGON2_DEFAULT_THREADS 1u -# define ARGON2_DEFAULT_VERSION ARGON2_VERSION_NUMBER - -# undef G -# define G(a, b, c, d) \ - do { \ - a = a + b + 2 * mul_lower(a, b); \ - d = rotr64(d ^ a, 32); \ - c = c + d + 2 * mul_lower(c, d); \ - b = rotr64(b ^ c, 24); \ - a = a + b + 2 * mul_lower(a, b); \ - d = rotr64(d ^ a, 16); \ - c = c + d + 2 * mul_lower(c, d); \ - b = rotr64(b ^ c, 63); \ - } while ((void)0, 0) - -# undef PERMUTATION_P -# define PERMUTATION_P(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, \ - v12, v13, v14, v15) \ - do { \ - G(v0, v4, v8, v12); \ - G(v1, v5, v9, v13); \ - G(v2, v6, v10, v14); \ - G(v3, v7, v11, v15); \ - G(v0, v5, v10, v15); \ - G(v1, v6, v11, v12); \ - G(v2, v7, v8, v13); \ - G(v3, v4, v9, v14); \ - } while ((void)0, 0) - -# undef PERMUTATION_P_COLUMN -# define PERMUTATION_P_COLUMN(x, i) \ - do { \ - uint64_t *base = &x[16 * i]; \ - PERMUTATION_P( \ - *base, *(base + 1), *(base + 2), *(base + 3), \ - *(base + 4), *(base + 5), *(base + 6), *(base + 7), \ - *(base + 8), *(base + 9), *(base + 10), *(base + 11), \ - *(base + 12), *(base + 13), *(base + 14), *(base + 15) \ - ); \ - } while ((void)0, 0) - -# undef PERMUTATION_P_ROW -# define PERMUTATION_P_ROW(x, i) \ - do { \ - uint64_t *base = &x[2 * i]; \ - PERMUTATION_P( \ - *base, *(base + 1), *(base + 16), *(base + 17), \ - *(base + 32), *(base + 33), *(base + 48), *(base + 49), \ - *(base + 64), *(base + 65), *(base + 80), *(base + 81), \ - *(base + 96), *(base + 97), *(base + 112), *(base + 113) \ - ); \ - } while ((void)0, 0) - -typedef struct { - uint64_t v[ARGON2_QWORDS_IN_BLOCK]; -} BLOCK; - -typedef enum { - ARGON2_VERSION_10 = 0x10, - ARGON2_VERSION_13 = 0x13, - ARGON2_VERSION_NUMBER = ARGON2_VERSION_13 -} ARGON2_VERSION; - -typedef enum { - ARGON2_D = 0, - ARGON2_I = 1, - ARGON2_ID = 2 -} ARGON2_TYPE; - -typedef struct { - uint32_t pass; - uint32_t lane; - uint8_t slice; - uint32_t index; -} ARGON2_POS; - -typedef struct { - void *provctx; - uint32_t outlen; - uint8_t *pwd; - uint32_t pwdlen; - uint8_t *salt; - uint32_t saltlen; - uint8_t *secret; - uint32_t secretlen; - uint8_t *ad; - uint32_t adlen; - uint32_t t_cost; - uint32_t m_cost; - uint32_t lanes; - uint32_t threads; - uint32_t version; - uint32_t early_clean; - ARGON2_TYPE type; - BLOCK *memory; - uint32_t passes; - uint32_t memory_blocks; - uint32_t segment_length; - uint32_t lane_length; - OSSL_LIB_CTX *libctx; - EVP_MD *md; - EVP_MAC *mac; - char *propq; -} KDF_ARGON2; - -typedef struct { - ARGON2_POS pos; - KDF_ARGON2 *ctx; -} ARGON2_THREAD_DATA; - -static OSSL_FUNC_kdf_newctx_fn kdf_argon2i_new; -static OSSL_FUNC_kdf_newctx_fn kdf_argon2d_new; -static OSSL_FUNC_kdf_newctx_fn kdf_argon2id_new; -static OSSL_FUNC_kdf_freectx_fn kdf_argon2_free; -static OSSL_FUNC_kdf_reset_fn kdf_argon2_reset; -static OSSL_FUNC_kdf_derive_fn kdf_argon2_derive; -static OSSL_FUNC_kdf_settable_ctx_params_fn kdf_argon2_settable_ctx_params; -static OSSL_FUNC_kdf_set_ctx_params_fn kdf_argon2_set_ctx_params; - -static void kdf_argon2_init(KDF_ARGON2 *ctx, ARGON2_TYPE t); -static void *kdf_argon2d_new(void *provctx); -static void *kdf_argon2i_new(void *provctx); -static void *kdf_argon2id_new(void *provctx); -static void kdf_argon2_free(void *vctx); -static int kdf_argon2_derive(void *vctx, unsigned char *out, size_t outlen, - const OSSL_PARAM params[]); -static void kdf_argon2_reset(void *vctx); -static int kdf_argon2_ctx_set_threads(KDF_ARGON2 *ctx, uint32_t threads); -static int kdf_argon2_ctx_set_lanes(KDF_ARGON2 *ctx, uint32_t lanes); -static int kdf_argon2_ctx_set_t_cost(KDF_ARGON2 *ctx, uint32_t t_cost); -static int kdf_argon2_ctx_set_m_cost(KDF_ARGON2 *ctx, uint32_t m_cost); -static int kdf_argon2_ctx_set_out_length(KDF_ARGON2 *ctx, uint32_t outlen); -static int kdf_argon2_ctx_set_secret(KDF_ARGON2 *ctx, const OSSL_PARAM *p); -static int kdf_argon2_ctx_set_pwd(KDF_ARGON2 *ctx, const OSSL_PARAM *p); -static int kdf_argon2_ctx_set_salt(KDF_ARGON2 *ctx, const OSSL_PARAM *p); -static int kdf_argon2_ctx_set_ad(KDF_ARGON2 *ctx, const OSSL_PARAM *p); -static int kdf_argon2_set_ctx_params(void *vctx, const OSSL_PARAM params[]); -static int kdf_argon2_get_ctx_params(void *vctx, OSSL_PARAM params[]); -static int kdf_argon2_ctx_set_version(KDF_ARGON2 *ctx, uint32_t version); -static const OSSL_PARAM *kdf_argon2_settable_ctx_params(ossl_unused void *ctx, - ossl_unused void *p_ctx); -static const OSSL_PARAM *kdf_argon2_gettable_ctx_params(ossl_unused void *ctx, - ossl_unused void *p_ctx); - -static ossl_inline uint64_t load64(const uint8_t *src); -static ossl_inline void store32(uint8_t *dst, uint32_t w); -static ossl_inline void store64(uint8_t *dst, uint64_t w); -static ossl_inline uint64_t rotr64(const uint64_t w, const unsigned int c); -static ossl_inline uint64_t mul_lower(uint64_t x, uint64_t y); - -static void init_block_value(BLOCK *b, uint8_t in); -static void copy_block(BLOCK *dst, const BLOCK *src); -static void xor_block(BLOCK *dst, const BLOCK *src); -static void load_block(BLOCK *dst, const void *input); -static void store_block(void *output, const BLOCK *src); -static void fill_first_blocks(uint8_t *blockhash, const KDF_ARGON2 *ctx); -static void fill_block(const BLOCK *prev, const BLOCK *ref, BLOCK *next, - int with_xor); - -static void next_addresses(BLOCK *address_block, BLOCK *input_block, - const BLOCK *zero_block); -static int data_indep_addressing(const KDF_ARGON2 *ctx, uint32_t pass, - uint8_t slice); -static uint32_t index_alpha(const KDF_ARGON2 *ctx, uint32_t pass, - uint8_t slice, uint32_t index, - uint32_t pseudo_rand, int same_lane); - -static void fill_segment(const KDF_ARGON2 *ctx, uint32_t pass, uint32_t lane, - uint8_t slice); - -# if !defined(ARGON2_NO_THREADS) -static uint32_t fill_segment_thr(void *thread_data); -static int fill_mem_blocks_mt(KDF_ARGON2 *ctx); -# endif - -static int fill_mem_blocks_st(KDF_ARGON2 *ctx); -static ossl_inline int fill_memory_blocks(KDF_ARGON2 *ctx); - -static void initial_hash(uint8_t *blockhash, KDF_ARGON2 *ctx); -static int initialize(KDF_ARGON2 *ctx); -static void finalize(const KDF_ARGON2 *ctx, void *out); - -static int blake2b(EVP_MD *md, EVP_MAC *mac, void *out, size_t outlen, - const void *in, size_t inlen, const void *key, - size_t keylen); -static int blake2b_long(EVP_MD *md, EVP_MAC *mac, unsigned char *out, - size_t outlen, const void *in, size_t inlen); - -static ossl_inline uint64_t load64(const uint8_t *src) -{ - return - (((uint64_t)src[0]) << 0) - | (((uint64_t)src[1]) << 8) - | (((uint64_t)src[2]) << 16) - | (((uint64_t)src[3]) << 24) - | (((uint64_t)src[4]) << 32) - | (((uint64_t)src[5]) << 40) - | (((uint64_t)src[6]) << 48) - | (((uint64_t)src[7]) << 56); -} - -static ossl_inline void store32(uint8_t *dst, uint32_t w) -{ - dst[0] = (uint8_t)(w >> 0); - dst[1] = (uint8_t)(w >> 8); - dst[2] = (uint8_t)(w >> 16); - dst[3] = (uint8_t)(w >> 24); -} - -static ossl_inline void store64(uint8_t *dst, uint64_t w) -{ - dst[0] = (uint8_t)(w >> 0); - dst[1] = (uint8_t)(w >> 8); - dst[2] = (uint8_t)(w >> 16); - dst[3] = (uint8_t)(w >> 24); - dst[4] = (uint8_t)(w >> 32); - dst[5] = (uint8_t)(w >> 40); - dst[6] = (uint8_t)(w >> 48); - dst[7] = (uint8_t)(w >> 56); -} - -static ossl_inline uint64_t rotr64(const uint64_t w, const unsigned int c) -{ - return (w >> c) | (w << (64 - c)); -} - -static ossl_inline uint64_t mul_lower(uint64_t x, uint64_t y) -{ - const uint64_t m = 0xFFFFFFFFUL; - return (x & m) * (y & m); -} - -static void init_block_value(BLOCK *b, uint8_t in) -{ - memset(b->v, in, sizeof(b->v)); -} - -static void copy_block(BLOCK *dst, const BLOCK *src) -{ - memcpy(dst->v, src->v, sizeof(uint64_t) * ARGON2_QWORDS_IN_BLOCK); -} - -static void xor_block(BLOCK *dst, const BLOCK *src) -{ - int i; - - for (i = 0; i < ARGON2_QWORDS_IN_BLOCK; ++i) - dst->v[i] ^= src->v[i]; -} - -static void load_block(BLOCK *dst, const void *input) -{ - unsigned i; - - for (i = 0; i < ARGON2_QWORDS_IN_BLOCK; ++i) - dst->v[i] = load64((const uint8_t *)input + i * sizeof(dst->v[i])); -} - -static void store_block(void *output, const BLOCK *src) -{ - unsigned i; - - for (i = 0; i < ARGON2_QWORDS_IN_BLOCK; ++i) - store64((uint8_t *)output + i * sizeof(src->v[i]), src->v[i]); -} - -static void fill_first_blocks(uint8_t *blockhash, const KDF_ARGON2 *ctx) -{ - uint32_t l; - uint8_t blockhash_bytes[ARGON2_BLOCK_SIZE]; - - /* - * Make the first and second block in each lane as G(H0||0||i) - * or G(H0||1||i). - */ - for (l = 0; l < ctx->lanes; ++l) { - store32(blockhash + ARGON2_PREHASH_DIGEST_LENGTH, 0); - store32(blockhash + ARGON2_PREHASH_DIGEST_LENGTH + 4, l); - blake2b_long(ctx->md, ctx->mac, blockhash_bytes, ARGON2_BLOCK_SIZE, - blockhash, ARGON2_PREHASH_SEED_LENGTH); - load_block(&ctx->memory[l * ctx->lane_length + 0], - blockhash_bytes); - store32(blockhash + ARGON2_PREHASH_DIGEST_LENGTH, 1); - blake2b_long(ctx->md, ctx->mac, blockhash_bytes, ARGON2_BLOCK_SIZE, - blockhash, ARGON2_PREHASH_SEED_LENGTH); - load_block(&ctx->memory[l * ctx->lane_length + 1], - blockhash_bytes); - } - OPENSSL_cleanse(blockhash_bytes, ARGON2_BLOCK_SIZE); -} - -static void fill_block(const BLOCK *prev, const BLOCK *ref, - BLOCK *next, int with_xor) -{ - BLOCK blockR, tmp; - unsigned i; - - copy_block(&blockR, ref); - xor_block(&blockR, prev); - copy_block(&tmp, &blockR); - - if (with_xor) - xor_block(&tmp, next); - - for (i = 0; i < 8; ++i) - PERMUTATION_P_COLUMN(blockR.v, i); - - for (i = 0; i < 8; ++i) - PERMUTATION_P_ROW(blockR.v, i); - - copy_block(next, &tmp); - xor_block(next, &blockR); -} - -static void next_addresses(BLOCK *address_block, BLOCK *input_block, - const BLOCK *zero_block) -{ - input_block->v[6]++; - fill_block(zero_block, input_block, address_block, 0); - fill_block(zero_block, address_block, address_block, 0); -} - -static int data_indep_addressing(const KDF_ARGON2 *ctx, uint32_t pass, - uint8_t slice) -{ - switch (ctx->type) { - case ARGON2_I: - return 1; - case ARGON2_ID: - return (pass == 0) && (slice < ARGON2_SYNC_POINTS / 2); - case ARGON2_D: - default: - return 0; - } -} - -/* - * Pass 0 (pass = 0): - * This lane: all already finished segments plus already constructed blocks - * in this segment - * Other lanes: all already finished segments - * - * Pass 1+: - * This lane: (SYNC_POINTS - 1) last segments plus already constructed - * blocks in this segment - * Other lanes: (SYNC_POINTS - 1) last segments - */ -static uint32_t index_alpha(const KDF_ARGON2 *ctx, uint32_t pass, - uint8_t slice, uint32_t index, - uint32_t pseudo_rand, int same_lane) -{ - uint32_t ref_area_sz; - uint64_t rel_pos; - uint32_t start_pos, abs_pos; - - start_pos = 0; - switch (pass) { - case 0: - if (slice == 0) - ref_area_sz = index - 1; - else if (same_lane) - ref_area_sz = slice * ctx->segment_length + index - 1; - else - ref_area_sz = slice * ctx->segment_length + - ((index == 0) ? (-1) : 0); - break; - default: - if (same_lane) - ref_area_sz = ctx->lane_length - ctx->segment_length + index - 1; - else - ref_area_sz = ctx->lane_length - ctx->segment_length + - ((index == 0) ? (-1) : 0); - if (slice != ARGON2_SYNC_POINTS - 1) - start_pos = (slice + 1) * ctx->segment_length; - break; - } - - rel_pos = pseudo_rand; - rel_pos = rel_pos * rel_pos >> 32; - rel_pos = ref_area_sz - 1 - (ref_area_sz * rel_pos >> 32); - abs_pos = (start_pos + rel_pos) % ctx->lane_length; - - return abs_pos; -} - -static void fill_segment(const KDF_ARGON2 *ctx, uint32_t pass, uint32_t lane, - uint8_t slice) -{ - BLOCK *ref_block = NULL, *curr_block = NULL; - BLOCK address_block, input_block, zero_block; - uint64_t rnd, ref_index, ref_lane; - uint32_t prev_offset; - uint32_t start_idx; - uint32_t j; - uint32_t curr_offset; /* Offset of the current block */ - - memset(&input_block, 0, sizeof(BLOCK)); - - if (ctx == NULL) - return; - - if (data_indep_addressing(ctx, pass, slice)) { - init_block_value(&zero_block, 0); - init_block_value(&input_block, 0); - - input_block.v[0] = pass; - input_block.v[1] = lane; - input_block.v[2] = slice; - input_block.v[3] = ctx->memory_blocks; - input_block.v[4] = ctx->passes; - input_block.v[5] = ctx->type; - } - - start_idx = 0; - - /* We've generated the first two blocks. Generate the 1st block of addrs. */ - if ((pass == 0) && (slice == 0)) { - start_idx = 2; - if (data_indep_addressing(ctx, pass, slice)) - next_addresses(&address_block, &input_block, &zero_block); - } - - curr_offset = lane * ctx->lane_length + slice * ctx->segment_length - + start_idx; - - if ((curr_offset % ctx->lane_length) == 0) - prev_offset = curr_offset + ctx->lane_length - 1; - else - prev_offset = curr_offset - 1; - - for (j = start_idx; j < ctx->segment_length; ++j, ++curr_offset, ++prev_offset) { - if (curr_offset % ctx->lane_length == 1) - prev_offset = curr_offset - 1; - - /* Taking pseudo-random value from the previous block. */ - if (data_indep_addressing(ctx, pass, slice)) { - if (j % ARGON2_ADDRESSES_IN_BLOCK == 0) - next_addresses(&address_block, &input_block, &zero_block); - rnd = address_block.v[j % ARGON2_ADDRESSES_IN_BLOCK]; - } else { - rnd = ctx->memory[prev_offset].v[0]; - } - - /* Computing the lane of the reference block */ - ref_lane = ((rnd >> 32)) % ctx->lanes; - /* Can not reference other lanes yet */ - if ((pass == 0) && (slice == 0)) - ref_lane = lane; - - /* Computing the number of possible reference block within the lane. */ - ref_index = index_alpha(ctx, pass, slice, j, rnd & 0xFFFFFFFF, - ref_lane == lane); - - /* Creating a new block */ - ref_block = ctx->memory + ctx->lane_length * ref_lane + ref_index; - curr_block = ctx->memory + curr_offset; - if (ARGON2_VERSION_10 == ctx->version) { - /* Version 1.2.1 and earlier: overwrite, not XOR */ - fill_block(ctx->memory + prev_offset, ref_block, curr_block, 0); - continue; - } - - fill_block(ctx->memory + prev_offset, ref_block, curr_block, - pass == 0 ? 0 : 1); - } -} - -# if !defined(ARGON2_NO_THREADS) - -static uint32_t fill_segment_thr(void *thread_data) -{ - ARGON2_THREAD_DATA *my_data; - - my_data = (ARGON2_THREAD_DATA *) thread_data; - fill_segment(my_data->ctx, my_data->pos.pass, my_data->pos.lane, - my_data->pos.slice); - - return 0; -} - -static int fill_mem_blocks_mt(KDF_ARGON2 *ctx) -{ - uint32_t r, s, l, ll; - void **t; - ARGON2_THREAD_DATA *t_data; - - t = OPENSSL_zalloc(sizeof(void *)*ctx->lanes); - t_data = OPENSSL_zalloc(ctx->lanes * sizeof(ARGON2_THREAD_DATA)); - - if (t == NULL || t_data == NULL) - goto fail; - - for (r = 0; r < ctx->passes; ++r) { - for (s = 0; s < ARGON2_SYNC_POINTS; ++s) { - for (l = 0; l < ctx->lanes; ++l) { - ARGON2_POS p; - if (l >= ctx->threads) { - if (ossl_crypto_thread_join(t[l - ctx->threads], NULL) == 0) - goto fail; - if (ossl_crypto_thread_clean(t[l - ctx->threads]) == 0) - goto fail; - t[l] = NULL; - } - - p.pass = r; - p.lane = l; - p.slice = (uint8_t)s; - p.index = 0; - - t_data[l].ctx = ctx; - memcpy(&(t_data[l].pos), &p, sizeof(ARGON2_POS)); - t[l] = ossl_crypto_thread_start(ctx->libctx, &fill_segment_thr, - (void *) &t_data[l]); - if (t[l] == NULL) { - for (ll = 0; ll < l; ++ll) { - if (ossl_crypto_thread_join(t[ll], NULL) == 0) - goto fail; - if (ossl_crypto_thread_clean(t[ll]) == 0) - goto fail; - t[ll] = NULL; - } - goto fail; - } - } - for (l = ctx->lanes - ctx->threads; l < ctx->lanes; ++l) { - if (ossl_crypto_thread_join(t[l], NULL) == 0) - goto fail; - if (ossl_crypto_thread_clean(t[l]) == 0) - goto fail; - t[l] = NULL; - } - } - } - - OPENSSL_free(t_data); - OPENSSL_free(t); - - return 1; - -fail: - if (t_data != NULL) - OPENSSL_free(t_data); - if (t != NULL) - OPENSSL_free(t); - return 0; -} - -# endif /* !defined(ARGON2_NO_THREADS) */ - -static int fill_mem_blocks_st(KDF_ARGON2 *ctx) -{ - uint32_t r, s, l; - - for (r = 0; r < ctx->passes; ++r) - for (s = 0; s < ARGON2_SYNC_POINTS; ++s) - for (l = 0; l < ctx->lanes; ++l) - fill_segment(ctx, r, l, s); - return 1; -} - -static ossl_inline int fill_memory_blocks(KDF_ARGON2 *ctx) -{ -# if !defined(ARGON2_NO_THREADS) - return ctx->threads == 1 ? fill_mem_blocks_st(ctx) : fill_mem_blocks_mt(ctx); -# else - return ctx->threads == 1 ? fill_mem_blocks_st(ctx) : 0; -# endif -} - -static void initial_hash(uint8_t *blockhash, KDF_ARGON2 *ctx) -{ - EVP_MD_CTX *mdctx; - uint8_t value[sizeof(uint32_t)]; - unsigned int tmp; - uint32_t args[7]; - - if (ctx == NULL || blockhash == NULL) - return; - - args[0] = ctx->lanes; - args[1] = ctx->outlen; - args[2] = ctx->m_cost; - args[3] = ctx->t_cost; - args[4] = ctx->version; - args[5] = (uint32_t) ctx->type; - args[6] = ctx->pwdlen; - - mdctx = EVP_MD_CTX_create(); - if (mdctx == NULL || EVP_DigestInit_ex(mdctx, ctx->md, NULL) != 1) - goto fail; - - for (tmp = 0; tmp < sizeof(args) / sizeof(uint32_t); ++tmp) { - store32((uint8_t *) &value, args[tmp]); - if (EVP_DigestUpdate(mdctx, &value, sizeof(value)) != 1) - goto fail; - } - - if (ctx->pwd != NULL) { - if (EVP_DigestUpdate(mdctx, ctx->pwd, ctx->pwdlen) != 1) - goto fail; - if (ctx->early_clean) { - OPENSSL_cleanse(ctx->pwd, ctx->pwdlen); - ctx->pwdlen = 0; - } - } - - store32((uint8_t *) &value, ctx->saltlen); - - if (EVP_DigestUpdate(mdctx, &value, sizeof(value)) != 1) - goto fail; - - if (ctx->salt != NULL) - if (EVP_DigestUpdate(mdctx, ctx->salt, ctx->saltlen) != 1) - goto fail; - - store32((uint8_t *) &value, ctx->secretlen); - if (EVP_DigestUpdate(mdctx, &value, sizeof(value)) != 1) - goto fail; - - if (ctx->secret != NULL) { - if (EVP_DigestUpdate(mdctx, ctx->secret, ctx->secretlen) != 1) - goto fail; - if (ctx->early_clean) { - OPENSSL_cleanse(ctx->secret, ctx->secretlen); - ctx->secretlen = 0; - } - } - - store32((uint8_t *) &value, ctx->adlen); - if (EVP_DigestUpdate(mdctx, &value, sizeof(value)) != 1) - goto fail; - - if (ctx->ad != NULL) - if (EVP_DigestUpdate(mdctx, ctx->ad, ctx->adlen) != 1) - goto fail; - - tmp = ARGON2_PREHASH_DIGEST_LENGTH; - if (EVP_DigestFinal_ex(mdctx, blockhash, &tmp) != 1) - goto fail; - -fail: - EVP_MD_CTX_destroy(mdctx); -} - -static int initialize(KDF_ARGON2 *ctx) -{ - uint8_t blockhash[ARGON2_PREHASH_SEED_LENGTH]; - - if (ctx == NULL) - return 0; - - if (ctx->memory_blocks * sizeof(BLOCK) / sizeof(BLOCK) != ctx->memory_blocks) - return 0; - - if (ctx->type != ARGON2_D) - ctx->memory = OPENSSL_secure_zalloc(ctx->memory_blocks * - sizeof(BLOCK)); - else - ctx->memory = OPENSSL_zalloc(ctx->memory_blocks * - sizeof(BLOCK)); - - if (ctx->memory == NULL) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_MEMORY_SIZE, - "cannot allocate required memory"); - return 0; - } - - initial_hash(blockhash, ctx); - OPENSSL_cleanse(blockhash + ARGON2_PREHASH_DIGEST_LENGTH, - ARGON2_PREHASH_SEED_LENGTH - ARGON2_PREHASH_DIGEST_LENGTH); - fill_first_blocks(blockhash, ctx); - OPENSSL_cleanse(blockhash, ARGON2_PREHASH_SEED_LENGTH); - - return 1; -} - -static void finalize(const KDF_ARGON2 *ctx, void *out) -{ - BLOCK blockhash; - uint8_t blockhash_bytes[ARGON2_BLOCK_SIZE]; - uint32_t last_block_in_lane; - uint32_t l; - - if (ctx == NULL) - return; - - copy_block(&blockhash, ctx->memory + ctx->lane_length - 1); - - /* XOR the last blocks */ - for (l = 1; l < ctx->lanes; ++l) { - last_block_in_lane = l * ctx->lane_length + (ctx->lane_length - 1); - xor_block(&blockhash, ctx->memory + last_block_in_lane); - } - - /* Hash the result */ - store_block(blockhash_bytes, &blockhash); - blake2b_long(ctx->md, ctx->mac, out, ctx->outlen, blockhash_bytes, - ARGON2_BLOCK_SIZE); - OPENSSL_cleanse(blockhash.v, ARGON2_BLOCK_SIZE); - OPENSSL_cleanse(blockhash_bytes, ARGON2_BLOCK_SIZE); - - if (ctx->type != ARGON2_D) - OPENSSL_secure_clear_free(ctx->memory, - ctx->memory_blocks * sizeof(BLOCK)); - else - OPENSSL_clear_free(ctx->memory, - ctx->memory_blocks * sizeof(BLOCK)); -} - -static int blake2b_mac(EVP_MAC *mac, void *out, size_t outlen, const void *in, - size_t inlen, const void *key, size_t keylen) -{ - int ret = 0; - size_t par_n = 0, out_written; - EVP_MAC_CTX *ctx = NULL; - OSSL_PARAM par[3]; - - if ((ctx = EVP_MAC_CTX_new(mac)) == NULL) - goto fail; - - par[par_n++] = OSSL_PARAM_construct_octet_string(OSSL_MAC_PARAM_KEY, - (void *) key, keylen); - par[par_n++] = OSSL_PARAM_construct_size_t(OSSL_MAC_PARAM_SIZE, &outlen); - par[par_n++] = OSSL_PARAM_construct_end(); - - ret = EVP_MAC_CTX_set_params(ctx, par) == 1 - && EVP_MAC_init(ctx, NULL, 0, NULL) == 1 - && EVP_MAC_update(ctx, in, inlen) == 1 - && EVP_MAC_final(ctx, out, (size_t *) &out_written, outlen) == 1; - -fail: - EVP_MAC_CTX_free(ctx); - return ret; -} - -static int blake2b_md(EVP_MD *md, void *out, size_t outlen, const void *in, - size_t inlen) -{ - int ret = 0; - EVP_MD_CTX *ctx = NULL; - OSSL_PARAM par[2]; - - if ((ctx = EVP_MD_CTX_create()) == NULL) - return 0; - - par[0] = OSSL_PARAM_construct_size_t(OSSL_DIGEST_PARAM_SIZE, &outlen); - par[1] = OSSL_PARAM_construct_end(); - - ret = EVP_DigestInit_ex2(ctx, md, par) == 1 - && EVP_DigestUpdate(ctx, in, inlen) == 1 - && EVP_DigestFinal_ex(ctx, out, NULL) == 1; - - EVP_MD_CTX_free(ctx); - return ret; -} - -static int blake2b(EVP_MD *md, EVP_MAC *mac, void *out, size_t outlen, - const void *in, size_t inlen, const void *key, size_t keylen) -{ - if (out == NULL || outlen == 0) - return 0; - - if (key == NULL || keylen == 0) - return blake2b_md(md, out, outlen, in, inlen); - - return blake2b_mac(mac, out, outlen, in, inlen, key, keylen); -} - -static int blake2b_long(EVP_MD *md, EVP_MAC *mac, unsigned char *out, - size_t outlen, const void *in, size_t inlen) -{ - int ret = 0; - EVP_MD_CTX *ctx = NULL; - uint32_t outlen_curr; - uint8_t outbuf[BLAKE2B_OUTBYTES]; - uint8_t inbuf[BLAKE2B_OUTBYTES]; - uint8_t outlen_bytes[sizeof(uint32_t)] = {0}; - OSSL_PARAM par[2]; - size_t outlen_md; - - if (out == NULL || outlen == 0) - return 0; - - /* Ensure little-endian byte order */ - store32(outlen_bytes, (uint32_t)outlen); - - if ((ctx = EVP_MD_CTX_create()) == NULL) - return 0; - - outlen_md = (outlen <= BLAKE2B_OUTBYTES) ? outlen : BLAKE2B_OUTBYTES; - par[0] = OSSL_PARAM_construct_size_t(OSSL_DIGEST_PARAM_SIZE, &outlen_md); - par[1] = OSSL_PARAM_construct_end(); - - ret = EVP_DigestInit_ex2(ctx, md, par) == 1 - && EVP_DigestUpdate(ctx, outlen_bytes, sizeof(outlen_bytes)) == 1 - && EVP_DigestUpdate(ctx, in, inlen) == 1 - && EVP_DigestFinal_ex(ctx, (outlen > BLAKE2B_OUTBYTES) ? outbuf : out, - NULL) == 1; - - if (ret == 0) - goto fail; - - if (outlen > BLAKE2B_OUTBYTES) { - memcpy(out, outbuf, BLAKE2B_OUTBYTES / 2); - out += BLAKE2B_OUTBYTES / 2; - outlen_curr = (uint32_t) outlen - BLAKE2B_OUTBYTES / 2; - - while (outlen_curr > BLAKE2B_OUTBYTES) { - memcpy(inbuf, outbuf, BLAKE2B_OUTBYTES); - if (blake2b(md, mac, outbuf, BLAKE2B_OUTBYTES, inbuf, - BLAKE2B_OUTBYTES, NULL, 0) != 1) - goto fail; - memcpy(out, outbuf, BLAKE2B_OUTBYTES / 2); - out += BLAKE2B_OUTBYTES / 2; - outlen_curr -= BLAKE2B_OUTBYTES / 2; - } - - memcpy(inbuf, outbuf, BLAKE2B_OUTBYTES); - if (blake2b(md, mac, outbuf, outlen_curr, inbuf, BLAKE2B_OUTBYTES, - NULL, 0) != 1) - goto fail; - memcpy(out, outbuf, outlen_curr); - } - ret = 1; - -fail: - EVP_MD_CTX_free(ctx); - return ret; -} - -static void kdf_argon2_init(KDF_ARGON2 *c, ARGON2_TYPE type) -{ - OSSL_LIB_CTX *libctx; - - libctx = c->libctx; - memset(c, 0, sizeof(*c)); - - c->libctx = libctx; - c->outlen = ARGON2_DEFAULT_OUTLEN; - c->t_cost = ARGON2_DEFAULT_T_COST; - c->m_cost = ARGON2_DEFAULT_M_COST; - c->lanes = ARGON2_DEFAULT_LANES; - c->threads = ARGON2_DEFAULT_THREADS; - c->version = ARGON2_DEFAULT_VERSION; - c->type = type; -} - -static void *kdf_argon2d_new(void *provctx) -{ - KDF_ARGON2 *ctx; - - if (!ossl_prov_is_running()) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); - return NULL; - } - - ctx->libctx = PROV_LIBCTX_OF(provctx); - - kdf_argon2_init(ctx, ARGON2_D); - return ctx; -} - -static void *kdf_argon2i_new(void *provctx) -{ - KDF_ARGON2 *ctx; - - if (!ossl_prov_is_running()) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); - return NULL; - } - - ctx->libctx = PROV_LIBCTX_OF(provctx); - - kdf_argon2_init(ctx, ARGON2_I); - return ctx; -} - -static void *kdf_argon2id_new(void *provctx) -{ - KDF_ARGON2 *ctx; - - if (!ossl_prov_is_running()) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); - return NULL; - } - - ctx->libctx = PROV_LIBCTX_OF(provctx); - - kdf_argon2_init(ctx, ARGON2_ID); - return ctx; -} - -static void kdf_argon2_free(void *vctx) -{ - KDF_ARGON2 *ctx = (KDF_ARGON2 *)vctx; - - if (ctx == NULL) - return; - - if (ctx->pwd != NULL) - OPENSSL_clear_free(ctx->pwd, ctx->pwdlen); - - if (ctx->salt != NULL) - OPENSSL_clear_free(ctx->salt, ctx->saltlen); - - if (ctx->secret != NULL) - OPENSSL_clear_free(ctx->secret, ctx->secretlen); - - if (ctx->ad != NULL) - OPENSSL_clear_free(ctx->ad, ctx->adlen); - - EVP_MD_free(ctx->md); - EVP_MAC_free(ctx->mac); - - OPENSSL_free(ctx->propq); - - memset(ctx, 0, sizeof(*ctx)); - - OPENSSL_free(ctx); -} - -static int kdf_argon2_derive(void *vctx, unsigned char *out, size_t outlen, - const OSSL_PARAM params[]) -{ - KDF_ARGON2 *ctx; - uint32_t memory_blocks, segment_length; - - ctx = (KDF_ARGON2 *)vctx; - - if (!ossl_prov_is_running() || !kdf_argon2_set_ctx_params(vctx, params)) - return 0; - - if (ctx->mac == NULL) - ctx->mac = EVP_MAC_fetch(ctx->libctx, "blake2bmac", ctx->propq); - if (ctx->mac == NULL) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_MISSING_MAC, - "cannot fetch blake2bmac"); - return 0; - } - - if (ctx->md == NULL) - ctx->md = EVP_MD_fetch(ctx->libctx, "blake2b512", ctx->propq); - if (ctx->md == NULL) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_MISSING_MESSAGE_DIGEST, - "cannot fetch blake2b512"); - return 0; - } - - if (ctx->salt == NULL || ctx->saltlen == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_SALT); - return 0; - } - - if (outlen != ctx->outlen) { - if (OSSL_PARAM_locate((OSSL_PARAM *)params, "size") != NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); - return 0; - } - if (!kdf_argon2_ctx_set_out_length(ctx, (uint32_t) outlen)) - return 0; - } - - switch (ctx->type) { - case ARGON2_D: - case ARGON2_I: - case ARGON2_ID: - break; - default: - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_MODE, "invalid Argon2 type"); - return 0; - } - - if (ctx->threads > 1) { -# ifdef ARGON2_NO_THREADS - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_THREAD_POOL_SIZE, - "requested %u threads, single-threaded mode supported only", - ctx->threads); - return 0; -# else - if (ctx->threads > ossl_get_avail_threads(ctx->libctx)) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_THREAD_POOL_SIZE, - "requested %u threads, available: 1", - ossl_get_avail_threads(ctx->libctx)); - return 0; - } -# endif - if (ctx->threads > ctx->lanes) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_THREAD_POOL_SIZE, - "requested more threads (%u) than lanes (%u)", - ctx->threads, ctx->lanes); - return 0; - } - } - - if (ctx->m_cost < 8 * ctx->lanes) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_MEMORY_SIZE, - "m_cost must be greater or equal than 8 times the number of lanes"); - return 0; - } - - memory_blocks = ctx->m_cost; - if (memory_blocks < 2 * ARGON2_SYNC_POINTS * ctx->lanes) - memory_blocks = 2 * ARGON2_SYNC_POINTS * ctx->lanes; - - /* Ensure that all segments have equal length */ - segment_length = memory_blocks / (ctx->lanes * ARGON2_SYNC_POINTS); - memory_blocks = segment_length * (ctx->lanes * ARGON2_SYNC_POINTS); - - ctx->memory = NULL; - ctx->memory_blocks = memory_blocks; - ctx->segment_length = segment_length; - ctx->passes = ctx->t_cost; - ctx->lane_length = segment_length * ARGON2_SYNC_POINTS; - - if (initialize(ctx) != 1) - return 0; - - if (fill_memory_blocks(ctx) != 1) - return 0; - - finalize(ctx, out); - - return 1; -} - -static void kdf_argon2_reset(void *vctx) -{ - OSSL_LIB_CTX *libctx; - KDF_ARGON2 *ctx; - ARGON2_TYPE type; - - ctx = (KDF_ARGON2 *) vctx; - type = ctx->type; - libctx = ctx->libctx; - - EVP_MD_free(ctx->md); - EVP_MAC_free(ctx->mac); - - OPENSSL_free(ctx->propq); - - if (ctx->pwd != NULL) - OPENSSL_clear_free(ctx->pwd, ctx->pwdlen); - - if (ctx->salt != NULL) - OPENSSL_clear_free(ctx->salt, ctx->saltlen); - - if (ctx->secret != NULL) - OPENSSL_clear_free(ctx->secret, ctx->secretlen); - - if (ctx->ad != NULL) - OPENSSL_clear_free(ctx->ad, ctx->adlen); - - memset(ctx, 0, sizeof(*ctx)); - ctx->libctx = libctx; - kdf_argon2_init(ctx, type); -} - -static int kdf_argon2_ctx_set_threads(KDF_ARGON2 *ctx, uint32_t threads) -{ - if (threads < ARGON2_MIN_THREADS) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_THREAD_POOL_SIZE, - "min threads: %u", ARGON2_MIN_THREADS); - return 0; - } - - if (threads > ARGON2_MAX_THREADS) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_THREAD_POOL_SIZE, - "max threads: %u", ARGON2_MAX_THREADS); - return 0; - } - - ctx->threads = threads; - return 1; -} - -static int kdf_argon2_ctx_set_lanes(KDF_ARGON2 *ctx, uint32_t lanes) -{ - if (lanes > ARGON2_MAX_LANES) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER, - "max lanes: %u", ARGON2_MAX_LANES); - return 0; - } - - if (lanes < ARGON2_MIN_LANES) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER, - "min lanes: %u", ARGON2_MIN_LANES); - return 0; - } - - ctx->lanes = lanes; - return 1; -} - -static int kdf_argon2_ctx_set_t_cost(KDF_ARGON2 *ctx, uint32_t t_cost) -{ - /* ARGON2_MAX_MEMORY == max m_cost value, so skip check */ - - if (t_cost < ARGON2_MIN_TIME) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_ITERATION_COUNT, - "min: %u", ARGON2_MIN_TIME); - return 0; - } - - ctx->t_cost = t_cost; - return 1; -} - -static int kdf_argon2_ctx_set_m_cost(KDF_ARGON2 *ctx, uint32_t m_cost) -{ - /* ARGON2_MAX_MEMORY == max m_cost value, so skip check */ - - if (m_cost < ARGON2_MIN_MEMORY) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_MEMORY_SIZE, "min: %u", - ARGON2_MIN_MEMORY); - return 0; - } - - ctx->m_cost = m_cost; - return 1; -} - -static int kdf_argon2_ctx_set_out_length(KDF_ARGON2 *ctx, uint32_t outlen) -{ - /* - * ARGON2_MAX_OUT_LENGTH == max outlen value, so upper bounds checks - * are always satisfied; to suppress compiler if statement tautology - * warnings, these checks are skipped. - */ - - if (outlen < ARGON2_MIN_OUT_LENGTH) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_OUTPUT_LENGTH, "min: %u", - ARGON2_MIN_OUT_LENGTH); - return 0; - } - - ctx->outlen = outlen; - return 1; -} - -static int kdf_argon2_ctx_set_secret(KDF_ARGON2 *ctx, const OSSL_PARAM *p) -{ - size_t buflen; - - if (p->data == NULL) - return 0; - - if (ctx->secret != NULL) { - OPENSSL_clear_free(ctx->secret, ctx->secretlen); - ctx->secret = NULL; - ctx->secretlen = 0U; - } - - if (!OSSL_PARAM_get_octet_string(p, (void **)&ctx->secret, 0, &buflen)) - return 0; - - if (buflen > ARGON2_MAX_SECRET) { - OPENSSL_free(ctx->secret); - ctx->secret = NULL; - ctx->secretlen = 0U; - return 0; - } - - ctx->secretlen = (uint32_t) buflen; - return 1; -} - -static int kdf_argon2_ctx_set_pwd(KDF_ARGON2 *ctx, const OSSL_PARAM *p) -{ - size_t buflen; - - if (p->data == NULL) - return 0; - - if (ctx->pwd != NULL) { - OPENSSL_clear_free(ctx->pwd, ctx->pwdlen); - ctx->pwd = NULL; - ctx->pwdlen = 0U; - } - - if (!OSSL_PARAM_get_octet_string(p, (void **)&ctx->pwd, 0, &buflen)) - return 0; - - if (buflen > ARGON2_MAX_PWD_LENGTH) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH, "max: %u", - ARGON2_MAX_PWD_LENGTH); - goto fail; - } - - ctx->pwdlen = (uint32_t) buflen; - return 1; - -fail: - OPENSSL_free(ctx->pwd); - ctx->pwd = NULL; - ctx->pwdlen = 0U; - return 0; -} - -static int kdf_argon2_ctx_set_salt(KDF_ARGON2 *ctx, const OSSL_PARAM *p) -{ - size_t buflen; - - if (p->data == NULL) - return 0; - - if (ctx->salt != NULL) { - OPENSSL_clear_free(ctx->salt, ctx->saltlen); - ctx->salt = NULL; - ctx->saltlen = 0U; - } - - if (!OSSL_PARAM_get_octet_string(p, (void **)&ctx->salt, 0, &buflen)) - return 0; - - if (buflen < ARGON2_MIN_SALT_LENGTH) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH, "min: %u", - ARGON2_MIN_SALT_LENGTH); - goto fail; - } - - if (buflen > ARGON2_MAX_SALT_LENGTH) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH, "max: %u", - ARGON2_MAX_SALT_LENGTH); - goto fail; - } - - ctx->saltlen = (uint32_t) buflen; - return 1; - -fail: - OPENSSL_free(ctx->salt); - ctx->salt = NULL; - ctx->saltlen = 0U; - return 0; -} - -static int kdf_argon2_ctx_set_ad(KDF_ARGON2 *ctx, const OSSL_PARAM *p) -{ - size_t buflen; - - if (p->data == NULL) - return 0; - - if (ctx->ad != NULL) { - OPENSSL_clear_free(ctx->ad, ctx->adlen); - ctx->ad = NULL; - ctx->adlen = 0U; - } - - if (!OSSL_PARAM_get_octet_string(p, (void **)&ctx->ad, 0, &buflen)) - return 0; - - if (buflen > ARGON2_MAX_AD_LENGTH) { - OPENSSL_free(ctx->ad); - ctx->ad = NULL; - ctx->adlen = 0U; - return 0; - } - - ctx->adlen = (uint32_t) buflen; - return 1; -} - -static void kdf_argon2_ctx_set_flag_early_clean(KDF_ARGON2 *ctx, uint32_t f) -{ - ctx->early_clean = !!(f); -} - -static int kdf_argon2_ctx_set_version(KDF_ARGON2 *ctx, uint32_t version) -{ - switch (version) { - case ARGON2_VERSION_10: - case ARGON2_VERSION_13: - ctx->version = version; - return 1; - default: - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_MODE, - "invalid Argon2 version"); - return 0; - } -} - -static int set_property_query(KDF_ARGON2 *ctx, const char *propq) -{ - OPENSSL_free(ctx->propq); - ctx->propq = NULL; - if (propq != NULL) { - ctx->propq = OPENSSL_strdup(propq); - if (ctx->propq == NULL) - return 0; - } - EVP_MD_free(ctx->md); - ctx->md = NULL; - EVP_MAC_free(ctx->mac); - ctx->mac = NULL; - return 1; -} - -static int kdf_argon2_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - const OSSL_PARAM *p; - KDF_ARGON2 *ctx; - uint32_t u32_value; - - if (params == NULL) - return 1; - - ctx = (KDF_ARGON2 *) vctx; - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_PASSWORD)) != NULL) - if (!kdf_argon2_ctx_set_pwd(ctx, p)) - return 0; - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SALT)) != NULL) - if (!kdf_argon2_ctx_set_salt(ctx, p)) - return 0; - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SECRET)) != NULL) - if (!kdf_argon2_ctx_set_secret(ctx, p)) - return 0; - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_ARGON2_AD)) != NULL) - if (!kdf_argon2_ctx_set_ad(ctx, p)) - return 0; - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SIZE)) != NULL) { - if (!OSSL_PARAM_get_uint32(p, &u32_value)) - return 0; - if (!kdf_argon2_ctx_set_out_length(ctx, u32_value)) - return 0; - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_ITER)) != NULL) { - if (!OSSL_PARAM_get_uint32(p, &u32_value)) - return 0; - if (!kdf_argon2_ctx_set_t_cost(ctx, u32_value)) - return 0; - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_THREADS)) != NULL) { - if (!OSSL_PARAM_get_uint32(p, &u32_value)) - return 0; - if (!kdf_argon2_ctx_set_threads(ctx, u32_value)) - return 0; - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_ARGON2_LANES)) != NULL) { - if (!OSSL_PARAM_get_uint32(p, &u32_value)) - return 0; - if (!kdf_argon2_ctx_set_lanes(ctx, u32_value)) - return 0; - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_ARGON2_MEMCOST)) != NULL) { - if (!OSSL_PARAM_get_uint32(p, &u32_value)) - return 0; - if (!kdf_argon2_ctx_set_m_cost(ctx, u32_value)) - return 0; - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_EARLY_CLEAN)) != NULL) { - if (!OSSL_PARAM_get_uint32(p, &u32_value)) - return 0; - kdf_argon2_ctx_set_flag_early_clean(ctx, u32_value); - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_ARGON2_VERSION)) != NULL) { - if (!OSSL_PARAM_get_uint32(p, &u32_value)) - return 0; - if (!kdf_argon2_ctx_set_version(ctx, u32_value)) - return 0; - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_PROPERTIES)) != NULL) { - if (p->data_type != OSSL_PARAM_UTF8_STRING - || !set_property_query(ctx, p->data)) - return 0; - } - - return 1; -} - -static const OSSL_PARAM *kdf_argon2_settable_ctx_params(ossl_unused void *ctx, - ossl_unused void *p_ctx) -{ - static const OSSL_PARAM known_settable_ctx_params[] = { - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_PASSWORD, NULL, 0), - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_SALT, NULL, 0), - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_SECRET, NULL, 0), - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_ARGON2_AD, NULL, 0), - OSSL_PARAM_uint32(OSSL_KDF_PARAM_SIZE, NULL), - OSSL_PARAM_uint32(OSSL_KDF_PARAM_ITER, NULL), - OSSL_PARAM_uint32(OSSL_KDF_PARAM_THREADS, NULL), - OSSL_PARAM_uint32(OSSL_KDF_PARAM_ARGON2_LANES, NULL), - OSSL_PARAM_uint32(OSSL_KDF_PARAM_ARGON2_MEMCOST, NULL), - OSSL_PARAM_uint32(OSSL_KDF_PARAM_EARLY_CLEAN, NULL), - OSSL_PARAM_uint32(OSSL_KDF_PARAM_ARGON2_VERSION, NULL), - OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_END - }; - - return known_settable_ctx_params; -} - -static int kdf_argon2_get_ctx_params(void *vctx, OSSL_PARAM params[]) -{ - OSSL_PARAM *p; - - (void) vctx; - if ((p = OSSL_PARAM_locate(params, OSSL_KDF_PARAM_SIZE)) != NULL) - return OSSL_PARAM_set_size_t(p, SIZE_MAX); - - return -2; -} - -static const OSSL_PARAM *kdf_argon2_gettable_ctx_params(ossl_unused void *ctx, - ossl_unused void *p_ctx) -{ - static const OSSL_PARAM known_gettable_ctx_params[] = { - OSSL_PARAM_size_t(OSSL_KDF_PARAM_SIZE, NULL), - OSSL_PARAM_END - }; - - return known_gettable_ctx_params; -} - -const OSSL_DISPATCH ossl_kdf_argon2i_functions[] = { - { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_argon2i_new }, - { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_argon2_free }, - { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_argon2_reset }, - { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_argon2_derive }, - { OSSL_FUNC_KDF_SETTABLE_CTX_PARAMS, - (void(*)(void))kdf_argon2_settable_ctx_params }, - { OSSL_FUNC_KDF_SET_CTX_PARAMS, (void(*)(void))kdf_argon2_set_ctx_params }, - { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, - (void(*)(void))kdf_argon2_gettable_ctx_params }, - { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_argon2_get_ctx_params }, - OSSL_DISPATCH_END -}; - -const OSSL_DISPATCH ossl_kdf_argon2d_functions[] = { - { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_argon2d_new }, - { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_argon2_free }, - { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_argon2_reset }, - { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_argon2_derive }, - { OSSL_FUNC_KDF_SETTABLE_CTX_PARAMS, - (void(*)(void))kdf_argon2_settable_ctx_params }, - { OSSL_FUNC_KDF_SET_CTX_PARAMS, (void(*)(void))kdf_argon2_set_ctx_params }, - { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, - (void(*)(void))kdf_argon2_gettable_ctx_params }, - { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_argon2_get_ctx_params }, - OSSL_DISPATCH_END -}; - -const OSSL_DISPATCH ossl_kdf_argon2id_functions[] = { - { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_argon2id_new }, - { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_argon2_free }, - { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_argon2_reset }, - { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_argon2_derive }, - { OSSL_FUNC_KDF_SETTABLE_CTX_PARAMS, - (void(*)(void))kdf_argon2_settable_ctx_params }, - { OSSL_FUNC_KDF_SET_CTX_PARAMS, (void(*)(void))kdf_argon2_set_ctx_params }, - { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, - (void(*)(void))kdf_argon2_gettable_ctx_params }, - { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_argon2_get_ctx_params }, - OSSL_DISPATCH_END -}; - -#endif diff --git a/openssl/src/providers/implementations/kdfs/hkdf.c b/openssl/src/providers/implementations/kdfs/hkdf.c index 4a24013bf..69686d0d1 100644 --- a/openssl/src/providers/implementations/kdfs/hkdf.c +++ b/openssl/src/providers/implementations/kdfs/hkdf.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -30,13 +30,10 @@ #include "prov/implementations.h" #include "prov/provider_util.h" #include "internal/e_os.h" -#include "internal/params.h" #define HKDF_MAXBUF 2048 -#define HKDF_MAXINFO (32*1024) static OSSL_FUNC_kdf_newctx_fn kdf_hkdf_new; -static OSSL_FUNC_kdf_dupctx_fn kdf_hkdf_dup; static OSSL_FUNC_kdf_freectx_fn kdf_hkdf_free; static OSSL_FUNC_kdf_reset_fn kdf_hkdf_reset; static OSSL_FUNC_kdf_derive_fn kdf_hkdf_derive; @@ -85,7 +82,7 @@ typedef struct { size_t label_len; unsigned char *data; size_t data_len; - unsigned char *info; + unsigned char info[HKDF_MAXBUF]; size_t info_len; } KDF_HKDF; @@ -96,7 +93,9 @@ static void *kdf_hkdf_new(void *provctx) if (!ossl_prov_is_running()) return NULL; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) != NULL) + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); + else ctx->provctx = provctx; return ctx; } @@ -122,41 +121,11 @@ static void kdf_hkdf_reset(void *vctx) OPENSSL_free(ctx->label); OPENSSL_clear_free(ctx->data, ctx->data_len); OPENSSL_clear_free(ctx->key, ctx->key_len); - OPENSSL_clear_free(ctx->info, ctx->info_len); + OPENSSL_cleanse(ctx->info, ctx->info_len); memset(ctx, 0, sizeof(*ctx)); ctx->provctx = provctx; } -static void *kdf_hkdf_dup(void *vctx) -{ - const KDF_HKDF *src = (const KDF_HKDF *)vctx; - KDF_HKDF *dest; - - dest = kdf_hkdf_new(src->provctx); - if (dest != NULL) { - if (!ossl_prov_memdup(src->salt, src->salt_len, &dest->salt, - &dest->salt_len) - || !ossl_prov_memdup(src->key, src->key_len, - &dest->key , &dest->key_len) - || !ossl_prov_memdup(src->prefix, src->prefix_len, - &dest->prefix, &dest->prefix_len) - || !ossl_prov_memdup(src->label, src->label_len, - &dest->label, &dest->label_len) - || !ossl_prov_memdup(src->data, src->data_len, - &dest->data, &dest->data_len) - || !ossl_prov_memdup(src->info, src->info_len, - &dest->info, &dest->info_len) - || !ossl_prov_digest_copy(&dest->digest, &src->digest)) - goto err; - dest->mode = src->mode; - } - return dest; - - err: - kdf_hkdf_free(dest); - return NULL; -} - static size_t kdf_hkdf_size(KDF_HKDF *ctx) { int sz; @@ -277,6 +246,7 @@ static int hkdf_common_set_ctx_params(KDF_HKDF *ctx, const OSSL_PARAM params[]) static int kdf_hkdf_set_ctx_params(void *vctx, const OSSL_PARAM params[]) { + const OSSL_PARAM *p; KDF_HKDF *ctx = vctx; if (params == NULL) @@ -285,11 +255,23 @@ static int kdf_hkdf_set_ctx_params(void *vctx, const OSSL_PARAM params[]) if (!hkdf_common_set_ctx_params(ctx, params)) return 0; - if (ossl_param_get1_concat_octet_string(params, OSSL_KDF_PARAM_INFO, - &ctx->info, &ctx->info_len, - HKDF_MAXINFO) == 0) - return 0; - + /* The info fields concatenate, so process them all */ + if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_INFO)) != NULL) { + ctx->info_len = 0; + for (; p != NULL; p = OSSL_PARAM_locate_const(p + 1, + OSSL_KDF_PARAM_INFO)) { + const void *q = ctx->info + ctx->info_len; + size_t sz = 0; + + if (p->data_size != 0 + && p->data != NULL + && !OSSL_PARAM_get_octet_string(p, (void **)&q, + HKDF_MAXBUF - ctx->info_len, + &sz)) + return 0; + ctx->info_len += sz; + } + } return 1; } @@ -316,13 +298,6 @@ static int kdf_hkdf_get_ctx_params(void *vctx, OSSL_PARAM params[]) return 0; return OSSL_PARAM_set_size_t(p, sz); } - if ((p = OSSL_PARAM_locate(params, OSSL_KDF_PARAM_INFO)) != NULL) { - if (ctx->info == NULL || ctx->info_len == 0) { - p->return_size = 0; - return 1; - } - return OSSL_PARAM_set_octet_string(p, ctx->info, ctx->info_len); - } return -2; } @@ -331,7 +306,6 @@ static const OSSL_PARAM *kdf_hkdf_gettable_ctx_params(ossl_unused void *ctx, { static const OSSL_PARAM known_gettable_ctx_params[] = { OSSL_PARAM_size_t(OSSL_KDF_PARAM_SIZE, NULL), - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_INFO, NULL, 0), OSSL_PARAM_END }; return known_gettable_ctx_params; @@ -339,7 +313,6 @@ static const OSSL_PARAM *kdf_hkdf_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_hkdf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_hkdf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_hkdf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_hkdf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_hkdf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_hkdf_derive }, @@ -349,7 +322,7 @@ const OSSL_DISPATCH ossl_kdf_hkdf_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))kdf_hkdf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_hkdf_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; /* @@ -539,7 +512,7 @@ static int HKDF_Expand(const EVP_MD *evp_md, if (!HMAC_Final(hmac, prev, NULL)) goto err; - copy_len = (dig_len > okm_len - done_len) ? + copy_len = (done_len + dig_len > okm_len) ? okm_len - done_len : dig_len; @@ -755,7 +728,6 @@ static const OSSL_PARAM *kdf_tls1_3_settable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_tls1_3_kdf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_hkdf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_hkdf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_hkdf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_hkdf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_tls1_3_derive }, @@ -765,5 +737,5 @@ const OSSL_DISPATCH ossl_kdf_tls1_3_kdf_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))kdf_hkdf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_hkdf_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/kdfs/hmacdrbg_kdf.c b/openssl/src/providers/implementations/kdfs/hmacdrbg_kdf.c deleted file mode 100644 index 30f1dfbd2..000000000 --- a/openssl/src/providers/implementations/kdfs/hmacdrbg_kdf.c +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include -#include -#include -#include "prov/providercommon.h" -#include "prov/implementations.h" -#include "prov/hmac_drbg.h" -#include "prov/provider_ctx.h" - -static OSSL_FUNC_kdf_newctx_fn hmac_drbg_kdf_new; -static OSSL_FUNC_kdf_dupctx_fn hmac_drbg_kdf_dup; -static OSSL_FUNC_kdf_freectx_fn hmac_drbg_kdf_free; -static OSSL_FUNC_kdf_reset_fn hmac_drbg_kdf_reset; -static OSSL_FUNC_kdf_derive_fn hmac_drbg_kdf_derive; -static OSSL_FUNC_kdf_settable_ctx_params_fn hmac_drbg_kdf_settable_ctx_params; -static OSSL_FUNC_kdf_set_ctx_params_fn hmac_drbg_kdf_set_ctx_params; -static OSSL_FUNC_kdf_gettable_ctx_params_fn hmac_drbg_kdf_gettable_ctx_params; -static OSSL_FUNC_kdf_get_ctx_params_fn hmac_drbg_kdf_get_ctx_params; - -typedef struct { - PROV_DRBG_HMAC base; - void *provctx; - unsigned char *entropy, *nonce; - size_t entropylen, noncelen; - int init; -} KDF_HMAC_DRBG; - -static void *hmac_drbg_kdf_new(void *provctx) -{ - KDF_HMAC_DRBG *ctx; - - if (!ossl_prov_is_running()) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); - return NULL; - } - ctx->provctx = provctx; - return ctx; -} - -static void hmac_drbg_kdf_reset(void *vctx) -{ - KDF_HMAC_DRBG *ctx = (KDF_HMAC_DRBG *)vctx; - PROV_DRBG_HMAC *drbg = &ctx->base; - void *provctx = ctx->provctx; - - EVP_MAC_CTX_free(drbg->ctx); - ossl_prov_digest_reset(&drbg->digest); - OPENSSL_clear_free(ctx->entropy, ctx->entropylen); - OPENSSL_clear_free(ctx->nonce, ctx->noncelen); - OPENSSL_cleanse(ctx, sizeof(*ctx)); - ctx->provctx = provctx; -} - -static void hmac_drbg_kdf_free(void *vctx) -{ - KDF_HMAC_DRBG *ctx = (KDF_HMAC_DRBG *)vctx; - - if (ctx != NULL) { - hmac_drbg_kdf_reset(ctx); - OPENSSL_free(ctx); - } -} - -static int ossl_drbg_hmac_dup(PROV_DRBG_HMAC *dst, const PROV_DRBG_HMAC *src) { - if (src->ctx != NULL) { - dst->ctx = EVP_MAC_CTX_dup(src->ctx); - if (dst->ctx == NULL) - return 0; - } - if (!ossl_prov_digest_copy(&dst->digest, &src->digest)) - return 0; - memcpy(dst->K, src->K, sizeof(dst->K)); - memcpy(dst->V, src->V, sizeof(dst->V)); - dst->blocklen = src->blocklen; - return 1; -} - -static void *hmac_drbg_kdf_dup(void *vctx) -{ - const KDF_HMAC_DRBG *src = (const KDF_HMAC_DRBG *)vctx; - KDF_HMAC_DRBG *dst; - - dst = hmac_drbg_kdf_new(src->provctx); - if (dst != NULL) { - if (!ossl_drbg_hmac_dup(&dst->base, &src->base) - || !ossl_prov_memdup(src->entropy, src->entropylen, - &dst->entropy , &dst->entropylen) - || !ossl_prov_memdup(src->nonce, src->noncelen, - &dst->nonce, &dst->noncelen)) - goto err; - dst->init = src->init; - } - return dst; - - err: - hmac_drbg_kdf_free(dst); - return NULL; -} - -static int hmac_drbg_kdf_derive(void *vctx, unsigned char *out, size_t outlen, - const OSSL_PARAM params[]) -{ - KDF_HMAC_DRBG *ctx = (KDF_HMAC_DRBG *)vctx; - PROV_DRBG_HMAC *drbg = &ctx->base; - - if (!ossl_prov_is_running() - || !hmac_drbg_kdf_set_ctx_params(vctx, params)) - return 0; - if (!ctx->init) { - if (ctx->entropy == NULL - || ctx->entropylen == 0 - || ctx->nonce == NULL - || ctx->noncelen == 0 - || !ossl_drbg_hmac_init(drbg, ctx->entropy, ctx->entropylen, - ctx->nonce, ctx->noncelen, NULL, 0)) - return 0; - ctx->init = 1; - } - - return ossl_drbg_hmac_generate(drbg, out, outlen, NULL, 0); -} - -static int hmac_drbg_kdf_get_ctx_params(void *vctx, OSSL_PARAM params[]) -{ - KDF_HMAC_DRBG *hmac = (KDF_HMAC_DRBG *)vctx; - PROV_DRBG_HMAC *drbg = &hmac->base; - const char *name; - const EVP_MD *md; - OSSL_PARAM *p; - - p = OSSL_PARAM_locate(params, OSSL_KDF_PARAM_MAC); - if (p != NULL) { - if (drbg->ctx == NULL) - return 0; - name = EVP_MAC_get0_name(EVP_MAC_CTX_get0_mac(drbg->ctx)); - if (!OSSL_PARAM_set_utf8_string(p, name)) - return 0; - } - - p = OSSL_PARAM_locate(params, OSSL_KDF_PARAM_DIGEST); - if (p != NULL) { - md = ossl_prov_digest_md(&drbg->digest); - if (md == NULL || !OSSL_PARAM_set_utf8_string(p, EVP_MD_get0_name(md))) - return 0; - } - return 1; -} - -static const OSSL_PARAM *hmac_drbg_kdf_gettable_ctx_params( - ossl_unused void *vctx, ossl_unused void *p_ctx) -{ - static const OSSL_PARAM known_gettable_ctx_params[] = { - OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_MAC, NULL, 0), - OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_DIGEST, NULL, 0), - OSSL_PARAM_END - }; - return known_gettable_ctx_params; -} - -static int hmac_drbg_kdf_set_ctx_params(void *vctx, - const OSSL_PARAM params[]) -{ - KDF_HMAC_DRBG *hmac = (KDF_HMAC_DRBG *)vctx; - PROV_DRBG_HMAC *drbg = &hmac->base; - OSSL_LIB_CTX *libctx = PROV_LIBCTX_OF(hmac->provctx); - const EVP_MD *md; - const OSSL_PARAM *p; - void *ptr = NULL; - size_t size = 0; - - if (params == NULL) - return 1; - - p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_HMACDRBG_ENTROPY); - if (p != NULL) { - if (!OSSL_PARAM_get_octet_string(p, &ptr, 0, &size)) - return 0; - OPENSSL_free(hmac->entropy); - hmac->entropy = ptr; - hmac->entropylen = size; - hmac->init = 0; - ptr = NULL; - } - - p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_HMACDRBG_NONCE); - if (p != NULL) { - if (!OSSL_PARAM_get_octet_string(p, &ptr, 0, &size)) - return 0; - OPENSSL_free(hmac->nonce); - hmac->nonce = ptr; - hmac->noncelen = size; - hmac->init = 0; - } - - p = OSSL_PARAM_locate_const(params, OSSL_ALG_PARAM_DIGEST); - if (p != NULL) { - if (!ossl_prov_digest_load_from_params(&drbg->digest, params, libctx)) - return 0; - - /* Confirm digest is allowed. Allow all digests that are not XOF */ - md = ossl_prov_digest_md(&drbg->digest); - if (md != NULL) { - if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED); - return 0; - } - drbg->blocklen = EVP_MD_get_size(md); - } - return ossl_prov_macctx_load_from_params(&drbg->ctx, params, - "HMAC", NULL, NULL, libctx); - } - return 1; -} - -static const OSSL_PARAM *hmac_drbg_kdf_settable_ctx_params( - ossl_unused void *vctx, ossl_unused void *p_ctx) -{ - static const OSSL_PARAM known_settable_ctx_params[] = { - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_HMACDRBG_ENTROPY, NULL, 0), - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_HMACDRBG_NONCE, NULL, 0), - OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_DIGEST, NULL, 0), - OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_END - }; - return known_settable_ctx_params; -} - -const OSSL_DISPATCH ossl_kdf_hmac_drbg_functions[] = { - { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))hmac_drbg_kdf_new }, - { OSSL_FUNC_KDF_FREECTX, (void(*)(void))hmac_drbg_kdf_free }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))hmac_drbg_kdf_dup }, - { OSSL_FUNC_KDF_RESET, (void(*)(void))hmac_drbg_kdf_reset }, - { OSSL_FUNC_KDF_DERIVE, (void(*)(void))hmac_drbg_kdf_derive }, - { OSSL_FUNC_KDF_SETTABLE_CTX_PARAMS, - (void(*)(void))hmac_drbg_kdf_settable_ctx_params }, - { OSSL_FUNC_KDF_SET_CTX_PARAMS, - (void(*)(void))hmac_drbg_kdf_set_ctx_params }, - { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, - (void(*)(void))hmac_drbg_kdf_gettable_ctx_params }, - { OSSL_FUNC_KDF_GET_CTX_PARAMS, - (void(*)(void))hmac_drbg_kdf_get_ctx_params }, - OSSL_DISPATCH_END -}; diff --git a/openssl/src/providers/implementations/kdfs/kbkdf.c b/openssl/src/providers/implementations/kdfs/kbkdf.c index e6855d573..1f4db114a 100644 --- a/openssl/src/providers/implementations/kdfs/kbkdf.c +++ b/openssl/src/providers/implementations/kdfs/kbkdf.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2019 Red Hat, Inc. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -45,7 +45,6 @@ #include "prov/providercommon.h" #include "internal/e_os.h" -#include "internal/params.h" #define ossl_min(a, b) ((a) < (b)) ? (a) : (b) @@ -61,7 +60,6 @@ typedef struct { EVP_MAC_CTX *ctx_init; /* Names are lowercased versions of those found in SP800-108. */ - int r; unsigned char *ki; size_t ki_len; unsigned char *label; @@ -71,13 +69,11 @@ typedef struct { unsigned char *iv; size_t iv_len; int use_l; - int is_kmac; int use_separator; } KBKDF; /* Definitions needed for typechecking. */ static OSSL_FUNC_kdf_newctx_fn kbkdf_new; -static OSSL_FUNC_kdf_dupctx_fn kbkdf_dup; static OSSL_FUNC_kdf_freectx_fn kbkdf_free; static OSSL_FUNC_kdf_reset_fn kbkdf_reset; static OSSL_FUNC_kdf_derive_fn kbkdf_derive; @@ -104,10 +100,8 @@ static uint32_t be32(uint32_t host) static void init(KBKDF *ctx) { - ctx->r = 32; ctx->use_l = 1; ctx->use_separator = 1; - ctx->is_kmac = 0; } static void *kbkdf_new(void *provctx) @@ -118,8 +112,10 @@ static void *kbkdf_new(void *provctx) return NULL; ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->provctx = provctx; init(ctx); @@ -151,43 +147,12 @@ static void kbkdf_reset(void *vctx) init(ctx); } -static void *kbkdf_dup(void *vctx) -{ - const KBKDF *src = (const KBKDF *)vctx; - KBKDF *dest; - - dest = kbkdf_new(src->provctx); - if (dest != NULL) { - dest->ctx_init = EVP_MAC_CTX_dup(src->ctx_init); - if (dest->ctx_init == NULL - || !ossl_prov_memdup(src->ki, src->ki_len, - &dest->ki, &dest->ki_len) - || !ossl_prov_memdup(src->label, src->label_len, - &dest->label, &dest->label_len) - || !ossl_prov_memdup(src->context, src->context_len, - &dest->context, &dest->context_len) - || !ossl_prov_memdup(src->iv, src->iv_len, - &dest->iv, &dest->iv_len)) - goto err; - dest->mode = src->mode; - dest->r = src->r; - dest->use_l = src->use_l; - dest->use_separator = src->use_separator; - dest->is_kmac = src->is_kmac; - } - return dest; - - err: - kbkdf_free(dest); - return NULL; -} - /* SP800-108 section 5.1 or section 5.2 depending on mode. */ static int derive(EVP_MAC_CTX *ctx_init, kbkdf_mode mode, unsigned char *iv, size_t iv_len, unsigned char *label, size_t label_len, unsigned char *context, size_t context_len, unsigned char *k_i, size_t h, uint32_t l, int has_separator, - unsigned char *ko, size_t ko_len, int r) + unsigned char *ko, size_t ko_len) { int ret = 0; EVP_MAC_CTX *ctx = NULL; @@ -221,7 +186,7 @@ static int derive(EVP_MAC_CTX *ctx_init, kbkdf_mode mode, unsigned char *iv, if (mode == FEEDBACK && !EVP_MAC_update(ctx, k_i, k_i_len)) goto done; - if (!EVP_MAC_update(ctx, 4 - (r / 8) + (unsigned char *)&i, r / 8) + if (!EVP_MAC_update(ctx, (unsigned char *)&i, 4) || !EVP_MAC_update(ctx, label, label_len) || (has_separator && !EVP_MAC_update(ctx, &zero, 1)) || !EVP_MAC_update(ctx, context, context_len) @@ -244,31 +209,6 @@ static int derive(EVP_MAC_CTX *ctx_init, kbkdf_mode mode, unsigned char *iv, return ret; } -/* This must be run before the key is set */ -static int kmac_init(EVP_MAC_CTX *ctx, const unsigned char *custom, size_t customlen) -{ - OSSL_PARAM params[2]; - - if (custom == NULL || customlen == 0) - return 1; - params[0] = OSSL_PARAM_construct_octet_string(OSSL_MAC_PARAM_CUSTOM, - (void *)custom, customlen); - params[1] = OSSL_PARAM_construct_end(); - return EVP_MAC_CTX_set_params(ctx, params) > 0; -} - -static int kmac_derive(EVP_MAC_CTX *ctx, unsigned char *out, size_t outlen, - const unsigned char *context, size_t contextlen) -{ - OSSL_PARAM params[2]; - - params[0] = OSSL_PARAM_construct_size_t(OSSL_MAC_PARAM_SIZE, &outlen); - params[1] = OSSL_PARAM_construct_end(); - return EVP_MAC_CTX_set_params(ctx, params) > 0 - && EVP_MAC_update(ctx, context, contextlen) - && EVP_MAC_final(ctx, out, NULL, outlen); -} - static int kbkdf_derive(void *vctx, unsigned char *key, size_t keylen, const OSSL_PARAM params[]) { @@ -277,7 +217,6 @@ static int kbkdf_derive(void *vctx, unsigned char *key, size_t keylen, unsigned char *k_i = NULL; uint32_t l = 0; size_t h = 0; - uint64_t counter_max; if (!ossl_prov_is_running() || !kbkdf_set_ctx_params(ctx, params)) return 0; @@ -301,30 +240,14 @@ static int kbkdf_derive(void *vctx, unsigned char *key, size_t keylen, return 0; } - if (ctx->is_kmac) { - ret = kmac_derive(ctx->ctx_init, key, keylen, - ctx->context, ctx->context_len); - goto done; - } - h = EVP_MAC_CTX_get_mac_size(ctx->ctx_init); if (h == 0) goto done; - if (ctx->iv_len != 0 && ctx->iv_len != h) { ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SEED_LENGTH); goto done; } - if (ctx->mode == COUNTER) { - /* Fail if keylen is too large for r */ - counter_max = (uint64_t)1 << (uint64_t)ctx->r; - if ((uint64_t)(keylen / h) >= counter_max) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); - goto done; - } - } - if (ctx->use_l != 0) l = be32(keylen * 8); @@ -334,7 +257,7 @@ static int kbkdf_derive(void *vctx, unsigned char *key, size_t keylen, ret = derive(ctx->ctx_init, ctx->mode, ctx->iv, ctx->iv_len, ctx->label, ctx->label_len, ctx->context, ctx->context_len, k_i, h, l, - ctx->use_separator, key, keylen, ctx->r); + ctx->use_separator, key, keylen); done: if (ret != 1) OPENSSL_cleanse(key, keylen); @@ -342,6 +265,17 @@ static int kbkdf_derive(void *vctx, unsigned char *key, size_t keylen, return ret; } +static int kbkdf_set_buffer(unsigned char **out, size_t *out_len, + const OSSL_PARAM *p) +{ + if (p->data == NULL || p->data_size == 0) + return 1; + + OPENSSL_clear_free(*out, *out_len); + *out = NULL; + return OSSL_PARAM_get_octet_string(p, (void **)out, 0, out_len); +} + static int kbkdf_set_ctx_params(void *vctx, const OSSL_PARAM params[]) { KBKDF *ctx = (KBKDF *)vctx; @@ -354,19 +288,13 @@ static int kbkdf_set_ctx_params(void *vctx, const OSSL_PARAM params[]) if (!ossl_prov_macctx_load_from_params(&ctx->ctx_init, params, NULL, NULL, NULL, libctx)) return 0; - else if (ctx->ctx_init != NULL) { - if (EVP_MAC_is_a(EVP_MAC_CTX_get0_mac(ctx->ctx_init), - OSSL_MAC_NAME_KMAC128) - || EVP_MAC_is_a(EVP_MAC_CTX_get0_mac(ctx->ctx_init), - OSSL_MAC_NAME_KMAC256)) { - ctx->is_kmac = 1; - } else if (!EVP_MAC_is_a(EVP_MAC_CTX_get0_mac(ctx->ctx_init), - OSSL_MAC_NAME_HMAC) - && !EVP_MAC_is_a(EVP_MAC_CTX_get0_mac(ctx->ctx_init), - OSSL_MAC_NAME_CMAC)) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MAC); - return 0; - } + else if (ctx->ctx_init != NULL + && !EVP_MAC_is_a(EVP_MAC_CTX_get0_mac(ctx->ctx_init), + OSSL_MAC_NAME_HMAC) + && !EVP_MAC_is_a(EVP_MAC_CTX_get0_mac(ctx->ctx_init), + OSSL_MAC_NAME_CMAC)) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MAC); + return 0; } p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_MODE); @@ -381,48 +309,34 @@ static int kbkdf_set_ctx_params(void *vctx, const OSSL_PARAM params[]) return 0; } - if (ossl_param_get1_octet_string(params, OSSL_KDF_PARAM_KEY, - &ctx->ki, &ctx->ki_len) == 0) - return 0; + p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_KEY); + if (p != NULL && !kbkdf_set_buffer(&ctx->ki, &ctx->ki_len, p)) + return 0; - if (ossl_param_get1_octet_string(params, OSSL_KDF_PARAM_SALT, - &ctx->label, &ctx->label_len) == 0) - return 0; + p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SALT); + if (p != NULL && !kbkdf_set_buffer(&ctx->label, &ctx->label_len, p)) + return 0; - if (ossl_param_get1_concat_octet_string(params, OSSL_KDF_PARAM_INFO, - &ctx->context, &ctx->context_len, - 0) == 0) + p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_INFO); + if (p != NULL && !kbkdf_set_buffer(&ctx->context, &ctx->context_len, p)) return 0; - if (ossl_param_get1_octet_string(params, OSSL_KDF_PARAM_SEED, - &ctx->iv, &ctx->iv_len) == 0) - return 0; + p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SEED); + if (p != NULL && !kbkdf_set_buffer(&ctx->iv, &ctx->iv_len, p)) + return 0; p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_KBKDF_USE_L); if (p != NULL && !OSSL_PARAM_get_int(p, &ctx->use_l)) return 0; - p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_KBKDF_R); - if (p != NULL) { - int new_r = 0; - - if (!OSSL_PARAM_get_int(p, &new_r)) - return 0; - if (new_r != 8 && new_r != 16 && new_r != 24 && new_r != 32) - return 0; - ctx->r = new_r; - } - p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_KBKDF_USE_SEPARATOR); if (p != NULL && !OSSL_PARAM_get_int(p, &ctx->use_separator)) return 0; /* Set up digest context, if we can. */ - if (ctx->ctx_init != NULL && ctx->ki_len != 0) { - if ((ctx->is_kmac && !kmac_init(ctx->ctx_init, ctx->label, ctx->label_len)) - || !EVP_MAC_init(ctx->ctx_init, ctx->ki, ctx->ki_len, NULL)) + if (ctx->ctx_init != NULL && ctx->ki_len != 0 + && !EVP_MAC_init(ctx->ctx_init, ctx->ki, ctx->ki_len, NULL)) return 0; - } return 1; } @@ -441,7 +355,6 @@ static const OSSL_PARAM *kbkdf_settable_ctx_params(ossl_unused void *ctx, OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_PROPERTIES, NULL, 0), OSSL_PARAM_int(OSSL_KDF_PARAM_KBKDF_USE_L, NULL), OSSL_PARAM_int(OSSL_KDF_PARAM_KBKDF_USE_SEPARATOR, NULL), - OSSL_PARAM_int(OSSL_KDF_PARAM_KBKDF_R, NULL), OSSL_PARAM_END, }; return known_settable_ctx_params; @@ -469,7 +382,6 @@ static const OSSL_PARAM *kbkdf_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_kbkdf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kbkdf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kbkdf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kbkdf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kbkdf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kbkdf_derive }, @@ -479,5 +391,5 @@ const OSSL_DISPATCH ossl_kdf_kbkdf_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))kbkdf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kbkdf_get_ctx_params }, - OSSL_DISPATCH_END, + { 0, NULL }, }; diff --git a/openssl/src/providers/implementations/kdfs/krb5kdf.c b/openssl/src/providers/implementations/kdfs/krb5kdf.c index bc951f741..2c887f0eb 100644 --- a/openssl/src/providers/implementations/kdfs/krb5kdf.c +++ b/openssl/src/providers/implementations/kdfs/krb5kdf.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -34,7 +34,6 @@ /* KRB5 KDF defined in RFC 3961, Section 5.1 */ static OSSL_FUNC_kdf_newctx_fn krb5kdf_new; -static OSSL_FUNC_kdf_dupctx_fn krb5kdf_dup; static OSSL_FUNC_kdf_freectx_fn krb5kdf_free; static OSSL_FUNC_kdf_reset_fn krb5kdf_reset; static OSSL_FUNC_kdf_derive_fn krb5kdf_derive; @@ -64,8 +63,10 @@ static void *krb5kdf_new(void *provctx) if (!ossl_prov_is_running()) return NULL; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->provctx = provctx; return ctx; } @@ -101,27 +102,6 @@ static int krb5kdf_set_membuf(unsigned char **dst, size_t *dst_len, return OSSL_PARAM_get_octet_string(p, (void **)dst, 0, dst_len); } -static void *krb5kdf_dup(void *vctx) -{ - const KRB5KDF_CTX *src = (const KRB5KDF_CTX *)vctx; - KRB5KDF_CTX *dest; - - dest = krb5kdf_new(src->provctx); - if (dest != NULL) { - if (!ossl_prov_memdup(src->key, src->key_len, - &dest->key, &dest->key_len) - || !ossl_prov_memdup(src->constant, src->constant_len, - &dest->constant , &dest->constant_len) - || !ossl_prov_cipher_copy(&dest->cipher, &src->cipher)) - goto err; - } - return dest; - - err: - krb5kdf_free(dest); - return NULL; -} - static int krb5kdf_derive(void *vctx, unsigned char *key, size_t keylen, const OSSL_PARAM params[]) { @@ -218,7 +198,6 @@ static const OSSL_PARAM *krb5kdf_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_krb5kdf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))krb5kdf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))krb5kdf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))krb5kdf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))krb5kdf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))krb5kdf_derive }, @@ -230,7 +209,7 @@ const OSSL_DISPATCH ossl_kdf_krb5kdf_functions[] = { (void(*)(void))krb5kdf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))krb5kdf_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; #ifndef OPENSSL_NO_DES @@ -357,10 +336,8 @@ static int cipher_init(EVP_CIPHER_CTX *ctx, klen = EVP_CIPHER_CTX_get_key_length(ctx); if (key_len != (size_t)klen) { ret = EVP_CIPHER_CTX_set_key_length(ctx, key_len); - if (ret <= 0) { - ret = 0; + if (!ret) goto out; - } } /* we never want padding, either the length requested is a multiple of * the cipher block size or we are passed a cipher that can cope with @@ -416,12 +393,6 @@ static int KRB5KDF(const EVP_CIPHER *cipher, ENGINE *engine, /* Initialize input block */ blocksize = EVP_CIPHER_CTX_get_block_size(ctx); - if (blocksize == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_CIPHER); - ret = 0; - goto out; - } - if (constant_len > blocksize) { ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CONSTANT_LENGTH); ret = 0; diff --git a/openssl/src/providers/implementations/kdfs/pbkdf1.c b/openssl/src/providers/implementations/kdfs/pbkdf1.c index 69d3f7cb2..1a042bac9 100644 --- a/openssl/src/providers/implementations/kdfs/pbkdf1.c +++ b/openssl/src/providers/implementations/kdfs/pbkdf1.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,7 +24,6 @@ #include "prov/provider_util.h" static OSSL_FUNC_kdf_newctx_fn kdf_pbkdf1_new; -static OSSL_FUNC_kdf_dupctx_fn kdf_pbkdf1_dup; static OSSL_FUNC_kdf_freectx_fn kdf_pbkdf1_free; static OSSL_FUNC_kdf_reset_fn kdf_pbkdf1_reset; static OSSL_FUNC_kdf_derive_fn kdf_pbkdf1_derive; @@ -60,7 +59,7 @@ static int kdf_pbkdf1_do_derive(const unsigned char *pass, size_t passlen, ctx = EVP_MD_CTX_new(); if (ctx == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -72,11 +71,6 @@ static int kdf_pbkdf1_do_derive(const unsigned char *pass, size_t passlen, mdsize = EVP_MD_size(md_type); if (mdsize < 0) goto err; - if (n > (size_t)mdsize) { - ERR_raise(ERR_LIB_PROV, PROV_R_LENGTH_TOO_LARGE); - goto err; - } - for (i = 1; i < iter; i++) { if (!EVP_DigestInit_ex(ctx, md_type, NULL)) goto err; @@ -89,7 +83,6 @@ static int kdf_pbkdf1_do_derive(const unsigned char *pass, size_t passlen, memcpy(out, md_tmp, n); ret = 1; err: - OPENSSL_cleanse(md_tmp, EVP_MAX_MD_SIZE); EVP_MD_CTX_free(ctx); return ret; } @@ -102,8 +95,10 @@ static void *kdf_pbkdf1_new(void *provctx) return NULL; ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->provctx = provctx; return ctx; } @@ -135,28 +130,6 @@ static void kdf_pbkdf1_reset(void *vctx) ctx->provctx = provctx; } -static void *kdf_pbkdf1_dup(void *vctx) -{ - const KDF_PBKDF1 *src = (const KDF_PBKDF1 *)vctx; - KDF_PBKDF1 *dest; - - dest = kdf_pbkdf1_new(src->provctx); - if (dest != NULL) { - if (!ossl_prov_memdup(src->salt, src->salt_len, - &dest->salt, &dest->salt_len) - || !ossl_prov_memdup(src->pass, src->pass_len, - &dest->pass , &dest->pass_len) - || !ossl_prov_digest_copy(&dest->digest, &src->digest)) - goto err; - dest->iter = src->iter; - } - return dest; - - err: - kdf_pbkdf1_free(dest); - return NULL; -} - static int kdf_pbkdf1_set_membuf(unsigned char **buffer, size_t *buflen, const OSSL_PARAM *p) { @@ -165,8 +138,10 @@ static int kdf_pbkdf1_set_membuf(unsigned char **buffer, size_t *buflen, *buflen = 0; if (p->data_size == 0) { - if ((*buffer = OPENSSL_malloc(1)) == NULL) + if ((*buffer = OPENSSL_malloc(1)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } } else if (p->data != NULL) { if (!OSSL_PARAM_get_octet_string(p, (void **)buffer, 0, buflen)) return 0; @@ -212,7 +187,7 @@ static int kdf_pbkdf1_set_ctx_params(void *vctx, const OSSL_PARAM params[]) return 0; if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SALT)) != NULL) - if (!kdf_pbkdf1_set_membuf(&ctx->salt, &ctx->salt_len, p)) + if (!kdf_pbkdf1_set_membuf(&ctx->salt, &ctx->salt_len,p)) return 0; if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_ITER)) != NULL) @@ -256,7 +231,6 @@ static const OSSL_PARAM *kdf_pbkdf1_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_pbkdf1_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_pbkdf1_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_pbkdf1_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_pbkdf1_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_pbkdf1_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_pbkdf1_derive }, @@ -266,5 +240,5 @@ const OSSL_DISPATCH ossl_kdf_pbkdf1_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))kdf_pbkdf1_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_pbkdf1_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/kdfs/pbkdf2.c b/openssl/src/providers/implementations/kdfs/pbkdf2.c index f2d190c30..2a0ae63ac 100644 --- a/openssl/src/providers/implementations/kdfs/pbkdf2.c +++ b/openssl/src/providers/implementations/kdfs/pbkdf2.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -37,7 +37,6 @@ #define KDF_PBKDF2_MIN_SALT_LEN (128 / 8) static OSSL_FUNC_kdf_newctx_fn kdf_pbkdf2_new; -static OSSL_FUNC_kdf_dupctx_fn kdf_pbkdf2_dup; static OSSL_FUNC_kdf_freectx_fn kdf_pbkdf2_free; static OSSL_FUNC_kdf_reset_fn kdf_pbkdf2_reset; static OSSL_FUNC_kdf_derive_fn kdf_pbkdf2_derive; @@ -46,10 +45,10 @@ static OSSL_FUNC_kdf_set_ctx_params_fn kdf_pbkdf2_set_ctx_params; static OSSL_FUNC_kdf_gettable_ctx_params_fn kdf_pbkdf2_gettable_ctx_params; static OSSL_FUNC_kdf_get_ctx_params_fn kdf_pbkdf2_get_ctx_params; -static int pbkdf2_derive(const char *pass, size_t passlen, - const unsigned char *salt, int saltlen, uint64_t iter, - const EVP_MD *digest, unsigned char *key, - size_t keylen, int extra_checks); +static int pbkdf2_derive(const char *pass, size_t passlen, + const unsigned char *salt, int saltlen, uint64_t iter, + const EVP_MD *digest, unsigned char *key, + size_t keylen, int extra_checks); typedef struct { void *provctx; @@ -64,7 +63,7 @@ typedef struct { static void kdf_pbkdf2_init(KDF_PBKDF2 *ctx); -static void *kdf_pbkdf2_new_no_init(void *provctx) +static void *kdf_pbkdf2_new(void *provctx) { KDF_PBKDF2 *ctx; @@ -72,18 +71,12 @@ static void *kdf_pbkdf2_new_no_init(void *provctx) return NULL; ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->provctx = provctx; - return ctx; -} - -static void *kdf_pbkdf2_new(void *provctx) -{ - KDF_PBKDF2 *ctx = kdf_pbkdf2_new_no_init(provctx); - - if (ctx != NULL) - kdf_pbkdf2_init(ctx); + kdf_pbkdf2_init(ctx); return ctx; } @@ -115,30 +108,6 @@ static void kdf_pbkdf2_reset(void *vctx) kdf_pbkdf2_init(ctx); } -static void *kdf_pbkdf2_dup(void *vctx) -{ - const KDF_PBKDF2 *src = (const KDF_PBKDF2 *)vctx; - KDF_PBKDF2 *dest; - - /* We need a new PBKDF2 object but uninitialised since we're filling it */ - dest = kdf_pbkdf2_new_no_init(src->provctx); - if (dest != NULL) { - if (!ossl_prov_memdup(src->salt, src->salt_len, - &dest->salt, &dest->salt_len) - || !ossl_prov_memdup(src->pass, src->pass_len, - &dest->pass, &dest->pass_len) - || !ossl_prov_digest_copy(&dest->digest, &src->digest)) - goto err; - dest->iter = src->iter; - dest->lower_bound_checks = src->lower_bound_checks; - } - return dest; - - err: - kdf_pbkdf2_free(dest); - return NULL; -} - static void kdf_pbkdf2_init(KDF_PBKDF2 *ctx) { OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END }; @@ -161,8 +130,10 @@ static int pbkdf2_set_membuf(unsigned char **buffer, size_t *buflen, *buflen = 0; if (p->data_size == 0) { - if ((*buffer = OPENSSL_malloc(1)) == NULL) + if ((*buffer = OPENSSL_malloc(1)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } } else if (p->data != NULL) { if (!OSSL_PARAM_get_octet_string(p, (void **)buffer, 0, buflen)) return 0; @@ -225,7 +196,7 @@ static int kdf_pbkdf2_set_ctx_params(void *vctx, const OSSL_PARAM params[]) ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH); return 0; } - if (!pbkdf2_set_membuf(&ctx->salt, &ctx->salt_len, p)) + if (!pbkdf2_set_membuf(&ctx->salt, &ctx->salt_len,p)) return 0; } @@ -278,7 +249,6 @@ static const OSSL_PARAM *kdf_pbkdf2_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_pbkdf2_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_pbkdf2_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_pbkdf2_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_pbkdf2_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_pbkdf2_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_pbkdf2_derive }, @@ -288,7 +258,7 @@ const OSSL_DISPATCH ossl_kdf_pbkdf2_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))kdf_pbkdf2_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_pbkdf2_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; /* diff --git a/openssl/src/providers/implementations/kdfs/pbkdf2_fips.c b/openssl/src/providers/implementations/kdfs/pbkdf2_fips.c index e43ef1645..69f76a556 100644 --- a/openssl/src/providers/implementations/kdfs/pbkdf2_fips.c +++ b/openssl/src/providers/implementations/kdfs/pbkdf2_fips.c @@ -7,6 +7,7 @@ * https://www.openssl.org/source/license.html */ +#include #include "pbkdf2.h" /* diff --git a/openssl/src/providers/implementations/kdfs/pkcs12kdf.c b/openssl/src/providers/implementations/kdfs/pkcs12kdf.c index 0679c05f9..3218daa78 100644 --- a/openssl/src/providers/implementations/kdfs/pkcs12kdf.c +++ b/openssl/src/providers/implementations/kdfs/pkcs12kdf.c @@ -1,5 +1,5 @@ /* - * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,7 +24,6 @@ #include "prov/provider_util.h" static OSSL_FUNC_kdf_newctx_fn kdf_pkcs12_new; -static OSSL_FUNC_kdf_dupctx_fn kdf_pkcs12_dup; static OSSL_FUNC_kdf_freectx_fn kdf_pkcs12_free; static OSSL_FUNC_kdf_reset_fn kdf_pkcs12_reset; static OSSL_FUNC_kdf_derive_fn kdf_pkcs12_derive; @@ -60,7 +59,7 @@ static int pkcs12kdf_derive(const unsigned char *pass, size_t passlen, ctx = EVP_MD_CTX_new(); if (ctx == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto end; } vi = EVP_MD_get_block_size(md_type); @@ -81,8 +80,10 @@ static int pkcs12kdf_derive(const unsigned char *pass, size_t passlen, Plen = 0; Ilen = Slen + Plen; I = OPENSSL_malloc(Ilen); - if (D == NULL || Ai == NULL || B == NULL || I == NULL) + if (D == NULL || Ai == NULL || B == NULL || I == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto end; + } for (i = 0; i < v; i++) D[i] = id; p = I; @@ -142,8 +143,10 @@ static void *kdf_pkcs12_new(void *provctx) return NULL; ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } ctx->provctx = provctx; return ctx; } @@ -175,29 +178,6 @@ static void kdf_pkcs12_reset(void *vctx) ctx->provctx = provctx; } -static void *kdf_pkcs12_dup(void *vctx) -{ - const KDF_PKCS12 *src = (const KDF_PKCS12 *)vctx; - KDF_PKCS12 *dest; - - dest = kdf_pkcs12_new(src->provctx); - if (dest != NULL) { - if (!ossl_prov_memdup(src->salt, src->salt_len, - &dest->salt, &dest->salt_len) - || !ossl_prov_memdup(src->pass, src->pass_len, - &dest->pass , &dest->pass_len) - || !ossl_prov_digest_copy(&dest->digest, &src->digest)) - goto err; - dest->iter = src->iter; - dest->id = src->id; - } - return dest; - - err: - kdf_pkcs12_free(dest); - return NULL; -} - static int pkcs12kdf_set_membuf(unsigned char **buffer, size_t *buflen, const OSSL_PARAM *p) { @@ -206,8 +186,10 @@ static int pkcs12kdf_set_membuf(unsigned char **buffer, size_t *buflen, *buflen = 0; if (p->data_size == 0) { - if ((*buffer = OPENSSL_malloc(1)) == NULL) + if ((*buffer = OPENSSL_malloc(1)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } } else if (p->data != NULL) { if (!OSSL_PARAM_get_octet_string(p, (void **)buffer, 0, buflen)) return 0; @@ -256,7 +238,7 @@ static int kdf_pkcs12_set_ctx_params(void *vctx, const OSSL_PARAM params[]) return 0; if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SALT)) != NULL) - if (!pkcs12kdf_set_membuf(&ctx->salt, &ctx->salt_len, p)) + if (!pkcs12kdf_set_membuf(&ctx->salt, &ctx->salt_len,p)) return 0; if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_PKCS12_ID)) != NULL) @@ -305,7 +287,6 @@ static const OSSL_PARAM *kdf_pkcs12_gettable_ctx_params( const OSSL_DISPATCH ossl_kdf_pkcs12_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_pkcs12_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_pkcs12_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_pkcs12_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_pkcs12_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_pkcs12_derive }, @@ -315,5 +296,5 @@ const OSSL_DISPATCH ossl_kdf_pkcs12_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))kdf_pkcs12_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_pkcs12_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/kdfs/pvkkdf.c b/openssl/src/providers/implementations/kdfs/pvkkdf.c deleted file mode 100644 index 85a250ff7..000000000 --- a/openssl/src/providers/implementations/kdfs/pvkkdf.c +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include "internal/numbers.h" /* SIZE_MAX */ -#include "prov/provider_ctx.h" -#include "prov/providercommon.h" -#include "prov/implementations.h" -#include "prov/provider_util.h" - -static OSSL_FUNC_kdf_newctx_fn kdf_pvk_new; -static OSSL_FUNC_kdf_dupctx_fn kdf_pvk_dup; -static OSSL_FUNC_kdf_freectx_fn kdf_pvk_free; -static OSSL_FUNC_kdf_reset_fn kdf_pvk_reset; -static OSSL_FUNC_kdf_derive_fn kdf_pvk_derive; -static OSSL_FUNC_kdf_settable_ctx_params_fn kdf_pvk_settable_ctx_params; -static OSSL_FUNC_kdf_set_ctx_params_fn kdf_pvk_set_ctx_params; -static OSSL_FUNC_kdf_gettable_ctx_params_fn kdf_pvk_gettable_ctx_params; -static OSSL_FUNC_kdf_get_ctx_params_fn kdf_pvk_get_ctx_params; - -typedef struct { - void *provctx; - unsigned char *pass; - size_t pass_len; - unsigned char *salt; - size_t salt_len; - PROV_DIGEST digest; -} KDF_PVK; - -static void kdf_pvk_init(KDF_PVK *ctx); - -static void *kdf_pvk_new(void *provctx) -{ - KDF_PVK *ctx; - - if (!ossl_prov_is_running()) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) - return NULL; - ctx->provctx = provctx; - kdf_pvk_init(ctx); - return ctx; -} - -static void kdf_pvk_cleanup(KDF_PVK *ctx) -{ - ossl_prov_digest_reset(&ctx->digest); - OPENSSL_free(ctx->salt); - OPENSSL_clear_free(ctx->pass, ctx->pass_len); - OPENSSL_cleanse(ctx, sizeof(*ctx)); -} - -static void kdf_pvk_free(void *vctx) -{ - KDF_PVK *ctx = (KDF_PVK *)vctx; - - if (ctx != NULL) { - kdf_pvk_cleanup(ctx); - OPENSSL_free(ctx); - } -} - -static void *kdf_pvk_dup(void *vctx) -{ - const KDF_PVK *src = (const KDF_PVK *)vctx; - KDF_PVK *dest; - - dest = kdf_pvk_new(src->provctx); - if (dest != NULL) - if (!ossl_prov_memdup(src->salt, src->salt_len, - &dest->salt, &dest->salt_len) - || !ossl_prov_memdup(src->pass, src->pass_len, - &dest->pass , &dest->pass_len) - || !ossl_prov_digest_copy(&dest->digest, &src->digest)) - goto err; - return dest; - - err: - kdf_pvk_free(dest); - return NULL; -} - -static void kdf_pvk_reset(void *vctx) -{ - KDF_PVK *ctx = (KDF_PVK *)vctx; - void *provctx = ctx->provctx; - - kdf_pvk_cleanup(ctx); - ctx->provctx = provctx; - kdf_pvk_init(ctx); -} - -static void kdf_pvk_init(KDF_PVK *ctx) -{ - OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END }; - OSSL_LIB_CTX *provctx = PROV_LIBCTX_OF(ctx->provctx); - - params[0] = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST, - SN_sha1, 0); - if (!ossl_prov_digest_load_from_params(&ctx->digest, params, provctx)) - /* This is an error, but there is no way to indicate such directly */ - ossl_prov_digest_reset(&ctx->digest); -} - -static int pvk_set_membuf(unsigned char **buffer, size_t *buflen, - const OSSL_PARAM *p) -{ - OPENSSL_clear_free(*buffer, *buflen); - *buffer = NULL; - *buflen = 0; - - if (p->data_size == 0) { - if ((*buffer = OPENSSL_malloc(1)) == NULL) - return 0; - } else if (p->data != NULL) { - if (!OSSL_PARAM_get_octet_string(p, (void **)buffer, 0, buflen)) - return 0; - } - return 1; -} - -static int kdf_pvk_derive(void *vctx, unsigned char *key, size_t keylen, - const OSSL_PARAM params[]) -{ - KDF_PVK *ctx = (KDF_PVK *)vctx; - const EVP_MD *md; - EVP_MD_CTX *mctx; - int res; - - if (!ossl_prov_is_running() || !kdf_pvk_set_ctx_params(ctx, params)) - return 0; - - if (ctx->pass == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_PASS); - return 0; - } - - if (ctx->salt == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_SALT); - return 0; - } - - md = ossl_prov_digest_md(&ctx->digest); - if (md == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST); - return 0; - } - res = EVP_MD_get_size(md); - if (res <= 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_BAD_LENGTH); - return 0; - } - if ((size_t)res > keylen) { - ERR_raise(ERR_LIB_PROV, PROV_R_LENGTH_TOO_LARGE); - return 0; - } - - mctx = EVP_MD_CTX_new(); - res = mctx != NULL - && EVP_DigestInit_ex(mctx, md, NULL) - && EVP_DigestUpdate(mctx, ctx->salt, ctx->salt_len) - && EVP_DigestUpdate(mctx, ctx->pass, ctx->pass_len) - && EVP_DigestFinal_ex(mctx, key, NULL); - EVP_MD_CTX_free(mctx); - return res; -} - -static int kdf_pvk_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - const OSSL_PARAM *p; - KDF_PVK *ctx = vctx; - OSSL_LIB_CTX *provctx = PROV_LIBCTX_OF(ctx->provctx); - - if (params == NULL) - return 1; - - if (!ossl_prov_digest_load_from_params(&ctx->digest, params, provctx)) - return 0; - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_PASSWORD)) != NULL) - if (!pvk_set_membuf(&ctx->pass, &ctx->pass_len, p)) - return 0; - - if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SALT)) != NULL) { - if (!pvk_set_membuf(&ctx->salt, &ctx->salt_len, p)) - return 0; - } - - return 1; -} - -static const OSSL_PARAM *kdf_pvk_settable_ctx_params(ossl_unused void *ctx, - ossl_unused void *p_ctx) -{ - static const OSSL_PARAM known_settable_ctx_params[] = { - OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_DIGEST, NULL, 0), - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_PASSWORD, NULL, 0), - OSSL_PARAM_octet_string(OSSL_KDF_PARAM_SALT, NULL, 0), - OSSL_PARAM_END - }; - return known_settable_ctx_params; -} - -static int kdf_pvk_get_ctx_params(void *vctx, OSSL_PARAM params[]) -{ - OSSL_PARAM *p; - - if ((p = OSSL_PARAM_locate(params, OSSL_KDF_PARAM_SIZE)) != NULL) - return OSSL_PARAM_set_size_t(p, SIZE_MAX); - return -2; -} - -static const OSSL_PARAM *kdf_pvk_gettable_ctx_params(ossl_unused void *ctx, - ossl_unused void *p_ctx) -{ - static const OSSL_PARAM known_gettable_ctx_params[] = { - OSSL_PARAM_size_t(OSSL_KDF_PARAM_SIZE, NULL), - OSSL_PARAM_END - }; - return known_gettable_ctx_params; -} - -const OSSL_DISPATCH ossl_kdf_pvk_functions[] = { - { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_pvk_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_pvk_dup }, - { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_pvk_free }, - { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_pvk_reset }, - { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_pvk_derive }, - { OSSL_FUNC_KDF_SETTABLE_CTX_PARAMS, - (void(*)(void))kdf_pvk_settable_ctx_params }, - { OSSL_FUNC_KDF_SET_CTX_PARAMS, (void(*)(void))kdf_pvk_set_ctx_params }, - { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, - (void(*)(void))kdf_pvk_gettable_ctx_params }, - { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_pvk_get_ctx_params }, - OSSL_DISPATCH_END -}; diff --git a/openssl/src/providers/implementations/kdfs/scrypt.c b/openssl/src/providers/implementations/kdfs/scrypt.c index ee2d4a7d3..a7072f785 100644 --- a/openssl/src/providers/implementations/kdfs/scrypt.c +++ b/openssl/src/providers/implementations/kdfs/scrypt.c @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,12 +20,11 @@ #include "prov/implementations.h" #include "prov/provider_ctx.h" #include "prov/providercommon.h" -#include "prov/provider_util.h" +#include "prov/implementations.h" #ifndef OPENSSL_NO_SCRYPT static OSSL_FUNC_kdf_newctx_fn kdf_scrypt_new; -static OSSL_FUNC_kdf_dupctx_fn kdf_scrypt_dup; static OSSL_FUNC_kdf_freectx_fn kdf_scrypt_free; static OSSL_FUNC_kdf_reset_fn kdf_scrypt_reset; static OSSL_FUNC_kdf_derive_fn kdf_scrypt_derive; @@ -55,7 +54,7 @@ typedef struct { static void kdf_scrypt_init(KDF_SCRYPT *ctx); -static void *kdf_scrypt_new_inner(OSSL_LIB_CTX *libctx) +static void *kdf_scrypt_new(void *provctx) { KDF_SCRYPT *ctx; @@ -63,18 +62,15 @@ static void *kdf_scrypt_new_inner(OSSL_LIB_CTX *libctx) return NULL; ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) + if (ctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; - ctx->libctx = libctx; + } + ctx->libctx = PROV_LIBCTX_OF(provctx); kdf_scrypt_init(ctx); return ctx; } -static void *kdf_scrypt_new(void *provctx) -{ - return kdf_scrypt_new_inner(PROV_LIBCTX_OF(provctx)); -} - static void kdf_scrypt_free(void *vctx) { KDF_SCRYPT *ctx = (KDF_SCRYPT *)vctx; @@ -96,38 +92,6 @@ static void kdf_scrypt_reset(void *vctx) kdf_scrypt_init(ctx); } -static void *kdf_scrypt_dup(void *vctx) -{ - const KDF_SCRYPT *src = (const KDF_SCRYPT *)vctx; - KDF_SCRYPT *dest; - - dest = kdf_scrypt_new_inner(src->libctx); - if (dest != NULL) { - if (src->sha256 != NULL && !EVP_MD_up_ref(src->sha256)) - goto err; - if (src->propq != NULL) { - dest->propq = OPENSSL_strdup(src->propq); - if (dest->propq == NULL) - goto err; - } - if (!ossl_prov_memdup(src->salt, src->salt_len, - &dest->salt, &dest->salt_len) - || !ossl_prov_memdup(src->pass, src->pass_len, - &dest->pass , &dest->pass_len)) - goto err; - dest->N = src->N; - dest->r = src->r; - dest->p = src->p; - dest->maxmem_bytes = src->maxmem_bytes; - dest->sha256 = src->sha256; - } - return dest; - - err: - kdf_scrypt_free(dest); - return NULL; -} - static void kdf_scrypt_init(KDF_SCRYPT *ctx) { /* Default values are the most conservative recommendation given in the @@ -148,8 +112,10 @@ static int scrypt_set_membuf(unsigned char **buffer, size_t *buflen, *buflen = 0; if (p->data_size == 0) { - if ((*buffer = OPENSSL_malloc(1)) == NULL) + if ((*buffer = OPENSSL_malloc(1)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } } else if (p->data != NULL) { if (!OSSL_PARAM_get_octet_string(p, (void **)buffer, 0, buflen)) return 0; @@ -175,8 +141,10 @@ static int set_property_query(KDF_SCRYPT *ctx, const char *propq) ctx->propq = NULL; if (propq != NULL) { ctx->propq = OPENSSL_strdup(propq); - if (ctx->propq == NULL) + if (ctx->propq == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } } return 1; } @@ -307,7 +275,6 @@ static const OSSL_PARAM *kdf_scrypt_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_scrypt_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_scrypt_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_scrypt_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_scrypt_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_scrypt_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_scrypt_derive }, @@ -317,7 +284,7 @@ const OSSL_DISPATCH ossl_kdf_scrypt_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))kdf_scrypt_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_scrypt_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; #define R(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) @@ -521,8 +488,10 @@ static int scrypt_alg(const char *pass, size_t passlen, return 1; B = OPENSSL_malloc((size_t)(Blen + Vlen)); - if (B == NULL) + if (B == NULL) { + ERR_raise(ERR_LIB_EVP, ERR_R_MALLOC_FAILURE); return 0; + } X = (uint32_t *)(B + Blen); T = X + 32 * r; V = T + 32 * r; diff --git a/openssl/src/providers/implementations/kdfs/sshkdf.c b/openssl/src/providers/implementations/kdfs/sshkdf.c index 90986bc76..c592ba72f 100644 --- a/openssl/src/providers/implementations/kdfs/sshkdf.c +++ b/openssl/src/providers/implementations/kdfs/sshkdf.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,7 +24,6 @@ /* See RFC 4253, Section 7.2 */ static OSSL_FUNC_kdf_newctx_fn kdf_sshkdf_new; -static OSSL_FUNC_kdf_dupctx_fn kdf_sshkdf_dup; static OSSL_FUNC_kdf_freectx_fn kdf_sshkdf_free; static OSSL_FUNC_kdf_reset_fn kdf_sshkdf_reset; static OSSL_FUNC_kdf_derive_fn kdf_sshkdf_derive; @@ -58,7 +57,9 @@ static void *kdf_sshkdf_new(void *provctx) if (!ossl_prov_is_running()) return NULL; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) != NULL) + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); + else ctx->provctx = provctx; return ctx; } @@ -86,30 +87,6 @@ static void kdf_sshkdf_reset(void *vctx) ctx->provctx = provctx; } -static void *kdf_sshkdf_dup(void *vctx) -{ - const KDF_SSHKDF *src = (const KDF_SSHKDF *)vctx; - KDF_SSHKDF *dest; - - dest = kdf_sshkdf_new(src->provctx); - if (dest != NULL) { - if (!ossl_prov_memdup(src->key, src->key_len, - &dest->key, &dest->key_len) - || !ossl_prov_memdup(src->xcghash, src->xcghash_len, - &dest->xcghash , &dest->xcghash_len) - || !ossl_prov_memdup(src->session_id, src->session_id_len, - &dest->session_id , &dest->session_id_len) - || !ossl_prov_digest_copy(&dest->digest, &src->digest)) - goto err; - dest->type = src->type; - } - return dest; - - err: - kdf_sshkdf_free(dest); - return NULL; -} - static int sshkdf_set_membuf(unsigned char **dst, size_t *dst_len, const OSSL_PARAM *p) { @@ -235,7 +212,6 @@ static const OSSL_PARAM *kdf_sshkdf_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_sshkdf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_sshkdf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_sshkdf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_sshkdf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_sshkdf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_sshkdf_derive }, @@ -245,7 +221,7 @@ const OSSL_DISPATCH ossl_kdf_sshkdf_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))kdf_sshkdf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_sshkdf_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; static int SSHKDF(const EVP_MD *evp_md, diff --git a/openssl/src/providers/implementations/kdfs/sskdf.c b/openssl/src/providers/implementations/kdfs/sskdf.c index db750a4f2..297ddcdc2 100644 --- a/openssl/src/providers/implementations/kdfs/sskdf.c +++ b/openssl/src/providers/implementations/kdfs/sskdf.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -50,7 +50,6 @@ #include "prov/providercommon.h" #include "prov/implementations.h" #include "prov/provider_util.h" -#include "internal/params.h" typedef struct { void *provctx; @@ -63,7 +62,6 @@ typedef struct { unsigned char *salt; size_t salt_len; size_t out_len; /* optional KMAC parameter */ - int is_kmac; } KDF_SSKDF; #define SSKDF_MAX_INLEN (1<<30) @@ -74,7 +72,6 @@ typedef struct { static const unsigned char kmac_custom_str[] = { 0x4B, 0x44, 0x46 }; static OSSL_FUNC_kdf_newctx_fn sskdf_new; -static OSSL_FUNC_kdf_dupctx_fn sskdf_dup; static OSSL_FUNC_kdf_freectx_fn sskdf_free; static OSSL_FUNC_kdf_reset_fn sskdf_reset; static OSSL_FUNC_kdf_derive_fn sskdf_derive; @@ -292,8 +289,9 @@ static void *sskdf_new(void *provctx) if (!ossl_prov_is_running()) return NULL; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) != NULL) - ctx->provctx = provctx; + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); + ctx->provctx = provctx; return ctx; } @@ -321,45 +319,21 @@ static void sskdf_free(void *vctx) } } -static void *sskdf_dup(void *vctx) +static int sskdf_set_buffer(unsigned char **out, size_t *out_len, + const OSSL_PARAM *p) { - const KDF_SSKDF *src = (const KDF_SSKDF *)vctx; - KDF_SSKDF *dest; - - dest = sskdf_new(src->provctx); - if (dest != NULL) { - if (src->macctx != NULL) { - dest->macctx = EVP_MAC_CTX_dup(src->macctx); - if (dest->macctx == NULL) - goto err; - } - if (!ossl_prov_memdup(src->info, src->info_len, - &dest->info, &dest->info_len) - || !ossl_prov_memdup(src->salt, src->salt_len, - &dest->salt , &dest->salt_len) - || !ossl_prov_memdup(src->secret, src->secret_len, - &dest->secret, &dest->secret_len) - || !ossl_prov_digest_copy(&dest->digest, &src->digest)) - goto err; - dest->out_len = src->out_len; - dest->is_kmac = src->is_kmac; - } - return dest; - - err: - sskdf_free(dest); - return NULL; + if (p->data == NULL || p->data_size == 0) + return 1; + OPENSSL_free(*out); + *out = NULL; + return OSSL_PARAM_get_octet_string(p, (void **)out, 0, out_len); } static size_t sskdf_size(KDF_SSKDF *ctx) { int len; - const EVP_MD *md = NULL; + const EVP_MD *md = ossl_prov_digest_md(&ctx->digest); - if (ctx->is_kmac) - return SIZE_MAX; - - md = ossl_prov_digest_md(&ctx->digest); if (md == NULL) { ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_MESSAGE_DIGEST); return 0; @@ -399,7 +373,8 @@ static int sskdf_derive(void *vctx, unsigned char *key, size_t keylen, default_salt_len = EVP_MD_get_size(md); if (default_salt_len <= 0) return 0; - } else if (ctx->is_kmac) { + } else if (EVP_MAC_is_a(mac, OSSL_MAC_NAME_KMAC128) + || EVP_MAC_is_a(mac, OSSL_MAC_NAME_KMAC256)) { /* H(x) = KMACzzz(x, salt, custom) */ custom = kmac_custom_str; custom_len = sizeof(kmac_custom_str); @@ -414,8 +389,10 @@ static int sskdf_derive(void *vctx, unsigned char *key, size_t keylen, /* If no salt is set then use a default_salt of zeros */ if (ctx->salt == NULL || ctx->salt_len <= 0) { ctx->salt = OPENSSL_zalloc(default_salt_len); - if (ctx->salt == NULL) + if (ctx->salt == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } ctx->salt_len = default_salt_len; } ret = SSKDF_mac_kdm(ctx->macctx, @@ -471,40 +448,28 @@ static int sskdf_set_ctx_params(void *vctx, const OSSL_PARAM params[]) KDF_SSKDF *ctx = vctx; OSSL_LIB_CTX *libctx = PROV_LIBCTX_OF(ctx->provctx); size_t sz; - int r; if (params == NULL) return 1; - if (!ossl_prov_macctx_load_from_params(&ctx->macctx, params, - NULL, NULL, NULL, libctx)) - return 0; - if (ctx->macctx != NULL) { - if (EVP_MAC_is_a(EVP_MAC_CTX_get0_mac(ctx->macctx), - OSSL_MAC_NAME_KMAC128) - || EVP_MAC_is_a(EVP_MAC_CTX_get0_mac(ctx->macctx), - OSSL_MAC_NAME_KMAC256)) { - ctx->is_kmac = 1; - } - } - if (!ossl_prov_digest_load_from_params(&ctx->digest, params, libctx)) return 0; - r = ossl_param_get1_octet_string(params, OSSL_KDF_PARAM_SECRET, - &ctx->secret, &ctx->secret_len); - if (r == -1) - r = ossl_param_get1_octet_string(params, OSSL_KDF_PARAM_KEY, - &ctx->secret, &ctx->secret_len); - if (r == 0) + if (!ossl_prov_macctx_load_from_params(&ctx->macctx, params, + NULL, NULL, NULL, libctx)) return 0; - if (ossl_param_get1_concat_octet_string(params, OSSL_KDF_PARAM_INFO, - &ctx->info, &ctx->info_len, 0) == 0) - return 0; + if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SECRET)) != NULL + || (p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_KEY)) != NULL) + if (!sskdf_set_buffer(&ctx->secret, &ctx->secret_len, p)) + return 0; + + if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_INFO)) != NULL) + if (!sskdf_set_buffer(&ctx->info, &ctx->info_len, p)) + return 0; - if (ossl_param_get1_octet_string(params, OSSL_KDF_PARAM_SALT, - &ctx->salt, &ctx->salt_len) == 0) + if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SALT)) != NULL) + if (!sskdf_set_buffer(&ctx->salt, &ctx->salt_len, p)) return 0; if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_MAC_SIZE)) @@ -555,7 +520,6 @@ static const OSSL_PARAM *sskdf_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_sskdf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))sskdf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))sskdf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))sskdf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))sskdf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))sskdf_derive }, @@ -565,12 +529,11 @@ const OSSL_DISPATCH ossl_kdf_sskdf_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))sskdf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))sskdf_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; const OSSL_DISPATCH ossl_kdf_x963_kdf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))sskdf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))sskdf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))sskdf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))sskdf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))x963kdf_derive }, @@ -580,5 +543,5 @@ const OSSL_DISPATCH ossl_kdf_x963_kdf_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))sskdf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))sskdf_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/kdfs/tls1_prf.c b/openssl/src/providers/implementations/kdfs/tls1_prf.c index 279248692..fa0251577 100644 --- a/openssl/src/providers/implementations/kdfs/tls1_prf.c +++ b/openssl/src/providers/implementations/kdfs/tls1_prf.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -45,13 +45,6 @@ * A(0) = seed * A(i) = HMAC_(secret, A(i-1)) */ - -/* - * Low level APIs (such as DH) are deprecated for public use, but still ok for - * internal use. - */ -#include "internal/deprecated.h" - #include #include #include @@ -67,14 +60,9 @@ #include "prov/providercommon.h" #include "prov/implementations.h" #include "prov/provider_util.h" -#include "prov/securitycheck.h" #include "internal/e_os.h" -#include "internal/safe_math.h" - -OSSL_SAFE_MATH_UNSIGNED(size_t, size_t) static OSSL_FUNC_kdf_newctx_fn kdf_tls1_prf_new; -static OSSL_FUNC_kdf_dupctx_fn kdf_tls1_prf_dup; static OSSL_FUNC_kdf_freectx_fn kdf_tls1_prf_free; static OSSL_FUNC_kdf_reset_fn kdf_tls1_prf_reset; static OSSL_FUNC_kdf_derive_fn kdf_tls1_prf_derive; @@ -88,8 +76,7 @@ static int tls1_prf_alg(EVP_MAC_CTX *mdctx, EVP_MAC_CTX *sha1ctx, const unsigned char *seed, size_t seed_len, unsigned char *out, size_t olen); -#define TLS_MD_MASTER_SECRET_CONST "\x6d\x61\x73\x74\x65\x72\x20\x73\x65\x63\x72\x65\x74" -#define TLS_MD_MASTER_SECRET_CONST_SIZE 13 +#define TLS1_PRF_MAXBUF 1024 /* TLS KDF kdf context structure */ typedef struct { @@ -103,8 +90,8 @@ typedef struct { /* Secret value to use for PRF */ unsigned char *sec; size_t seclen; - /* Concatenated seed data */ - unsigned char *seed; + /* Buffer of concatenated seed data */ + unsigned char seed[TLS1_PRF_MAXBUF]; size_t seedlen; } TLS1_PRF; @@ -115,8 +102,9 @@ static void *kdf_tls1_prf_new(void *provctx) if (!ossl_prov_is_running()) return NULL; - if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) != NULL) - ctx->provctx = provctx; + if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); + ctx->provctx = provctx; return ctx; } @@ -138,42 +126,15 @@ static void kdf_tls1_prf_reset(void *vctx) EVP_MAC_CTX_free(ctx->P_hash); EVP_MAC_CTX_free(ctx->P_sha1); OPENSSL_clear_free(ctx->sec, ctx->seclen); - OPENSSL_clear_free(ctx->seed, ctx->seedlen); + OPENSSL_cleanse(ctx->seed, ctx->seedlen); memset(ctx, 0, sizeof(*ctx)); ctx->provctx = provctx; } -static void *kdf_tls1_prf_dup(void *vctx) -{ - const TLS1_PRF *src = (const TLS1_PRF *)vctx; - TLS1_PRF *dest; - - dest = kdf_tls1_prf_new(src->provctx); - if (dest != NULL) { - if (src->P_hash != NULL - && (dest->P_hash = EVP_MAC_CTX_dup(src->P_hash)) == NULL) - goto err; - if (src->P_sha1 != NULL - && (dest->P_sha1 = EVP_MAC_CTX_dup(src->P_sha1)) == NULL) - goto err; - if (!ossl_prov_memdup(src->sec, src->seclen, &dest->sec, &dest->seclen)) - goto err; - if (!ossl_prov_memdup(src->seed, src->seedlen, &dest->seed, - &dest->seedlen)) - goto err; - } - return dest; - - err: - kdf_tls1_prf_free(dest); - return NULL; -} - static int kdf_tls1_prf_derive(void *vctx, unsigned char *key, size_t keylen, const OSSL_PARAM params[]) { TLS1_PRF *ctx = (TLS1_PRF *)vctx; - OSSL_LIB_CTX *libctx = PROV_LIBCTX_OF(ctx->provctx); if (!ossl_prov_is_running() || !kdf_tls1_prf_set_ctx_params(ctx, params)) return 0; @@ -195,21 +156,6 @@ static int kdf_tls1_prf_derive(void *vctx, unsigned char *key, size_t keylen, return 0; } - /* - * The seed buffer is prepended with a label. - * If EMS mode is enforced then the label "master secret" is not allowed, - * We do the check this way since the PRF is used for other purposes, as well - * as "extended master secret". - */ - if (ossl_tls1_prf_ems_check_enabled(libctx)) { - if (ctx->seedlen >= TLS_MD_MASTER_SECRET_CONST_SIZE - && memcmp(ctx->seed, TLS_MD_MASTER_SECRET_CONST, - TLS_MD_MASTER_SECRET_CONST_SIZE) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_EMS_NOT_ENABLED); - return 0; - } - } - return tls1_prf_alg(ctx->P_hash, ctx->P_sha1, ctx->sec, ctx->seclen, ctx->seed, ctx->seedlen, @@ -253,29 +199,16 @@ static int kdf_tls1_prf_set_ctx_params(void *vctx, const OSSL_PARAM params[]) if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SEED)) != NULL) { for (; p != NULL; p = OSSL_PARAM_locate_const(p + 1, OSSL_KDF_PARAM_SEED)) { - if (p->data_size != 0 && p->data != NULL) { - const void *val = NULL; - size_t sz = 0; - unsigned char *seed; - size_t seedlen; - int err = 0; - - if (!OSSL_PARAM_get_octet_string_ptr(p, &val, &sz)) - return 0; - - seedlen = safe_add_size_t(ctx->seedlen, sz, &err); - if (err) - return 0; - - seed = OPENSSL_clear_realloc(ctx->seed, ctx->seedlen, seedlen); - if (!seed) - return 0; - - ctx->seed = seed; - if (ossl_assert(sz != 0)) - memcpy(ctx->seed + ctx->seedlen, val, sz); - ctx->seedlen = seedlen; - } + const void *q = ctx->seed + ctx->seedlen; + size_t sz = 0; + + if (p->data_size != 0 + && p->data != NULL + && !OSSL_PARAM_get_octet_string(p, (void **)&q, + TLS1_PRF_MAXBUF - ctx->seedlen, + &sz)) + return 0; + ctx->seedlen += sz; } } return 1; @@ -315,7 +248,6 @@ static const OSSL_PARAM *kdf_tls1_prf_gettable_ctx_params( const OSSL_DISPATCH ossl_kdf_tls1_prf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))kdf_tls1_prf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))kdf_tls1_prf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))kdf_tls1_prf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))kdf_tls1_prf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))kdf_tls1_prf_derive }, @@ -327,7 +259,7 @@ const OSSL_DISPATCH ossl_kdf_tls1_prf_functions[] = { (void(*)(void))kdf_tls1_prf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))kdf_tls1_prf_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; /* @@ -453,8 +385,10 @@ static int tls1_prf_alg(EVP_MAC_CTX *mdctx, EVP_MAC_CTX *sha1ctx, seed, seed_len, out, olen)) return 0; - if ((tmp = OPENSSL_malloc(olen)) == NULL) + if ((tmp = OPENSSL_malloc(olen)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } if (!tls1_prf_P_hash(sha1ctx, sec + slen - L_S2, L_S2, seed, seed_len, tmp, olen)) { diff --git a/openssl/src/providers/implementations/kdfs/x942kdf.c b/openssl/src/providers/implementations/kdfs/x942kdf.c index 19b54493e..2b4577b01 100644 --- a/openssl/src/providers/implementations/kdfs/x942kdf.c +++ b/openssl/src/providers/implementations/kdfs/x942kdf.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -17,7 +17,6 @@ #include #include "internal/packet.h" #include "internal/der.h" -#include "internal/nelem.h" #include "prov/provider_ctx.h" #include "prov/providercommon.h" #include "prov/implementations.h" @@ -27,7 +26,6 @@ #define X942KDF_MAX_INLEN (1 << 30) static OSSL_FUNC_kdf_newctx_fn x942kdf_new; -static OSSL_FUNC_kdf_dupctx_fn x942kdf_dup; static OSSL_FUNC_kdf_freectx_fn x942kdf_free; static OSSL_FUNC_kdf_reset_fn x942kdf_reset; static OSSL_FUNC_kdf_derive_fn x942kdf_derive; @@ -171,7 +169,7 @@ static int der_encode_sharedinfo(WPACKET *pkt, unsigned char *buf, size_t buflen * |cek_oidlen| The length (in bytes) of the key wrapping algorithm oid, * |acvp| is the optional blob of DER data representing one or more of the * OtherInfo fields related to |partyu|, |partyv|, |supp_pub| and |supp_priv|. - * This field should normally be NULL. If |acvp| is non NULL then |partyu|, + * This field should noramlly be NULL. If |acvp| is non NULL then |partyu|, * |partyv|, |supp_pub| and |supp_priv| should all be NULL. * |acvp_len| is the |acvp| length (in bytes). * |partyu| is the optional public info contributed by the initiator. @@ -239,7 +237,7 @@ x942_encode_otherinfo(size_t keylen, goto err; /* * Since we allocated the exact size required, the buffer should point to the - * start of the allocated buffer at this point. + * start of the alllocated buffer at this point. */ if (WPACKET_get_curr(&pkt) != der_buf) goto err; @@ -334,10 +332,10 @@ static void *x942kdf_new(void *provctx) KDF_X942 *ctx; if (!ossl_prov_is_running()) - return NULL; + return 0; if ((ctx = OPENSSL_zalloc(sizeof(*ctx))) == NULL) - return NULL; + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); ctx->provctx = provctx; ctx->use_keybits = 1; return ctx; @@ -370,41 +368,6 @@ static void x942kdf_free(void *vctx) } } -static void *x942kdf_dup(void *vctx) -{ - const KDF_X942 *src = (const KDF_X942 *)vctx; - KDF_X942 *dest; - - dest = x942kdf_new(src->provctx); - if (dest != NULL) { - if (!ossl_prov_memdup(src->secret, src->secret_len, - &dest->secret , &dest->secret_len) - || !ossl_prov_memdup(src->acvpinfo, src->acvpinfo_len, - &dest->acvpinfo , &dest->acvpinfo_len) - || !ossl_prov_memdup(src->partyuinfo, src->partyuinfo_len, - &dest->partyuinfo , &dest->partyuinfo_len) - || !ossl_prov_memdup(src->partyvinfo, src->partyvinfo_len, - &dest->partyvinfo , &dest->partyvinfo_len) - || !ossl_prov_memdup(src->supp_pubinfo, src->supp_pubinfo_len, - &dest->supp_pubinfo, - &dest->supp_pubinfo_len) - || !ossl_prov_memdup(src->supp_privinfo, src->supp_privinfo_len, - &dest->supp_privinfo, - &dest->supp_privinfo_len) - || !ossl_prov_digest_copy(&dest->digest, &src->digest)) - goto err; - dest->cek_oid = src->cek_oid; - dest->cek_oid_len = src->cek_oid_len; - dest->dkm_len = src->dkm_len; - dest->use_keybits = src->use_keybits; - } - return dest; - - err: - x942kdf_free(dest); - return NULL; -} - static int x942kdf_set_buffer(unsigned char **out, size_t *out_len, const OSSL_PARAM *p) { @@ -616,7 +579,6 @@ static const OSSL_PARAM *x942kdf_gettable_ctx_params(ossl_unused void *ctx, const OSSL_DISPATCH ossl_kdf_x942_kdf_functions[] = { { OSSL_FUNC_KDF_NEWCTX, (void(*)(void))x942kdf_new }, - { OSSL_FUNC_KDF_DUPCTX, (void(*)(void))x942kdf_dup }, { OSSL_FUNC_KDF_FREECTX, (void(*)(void))x942kdf_free }, { OSSL_FUNC_KDF_RESET, (void(*)(void))x942kdf_reset }, { OSSL_FUNC_KDF_DERIVE, (void(*)(void))x942kdf_derive }, @@ -626,5 +588,5 @@ const OSSL_DISPATCH ossl_kdf_x942_kdf_functions[] = { { OSSL_FUNC_KDF_GETTABLE_CTX_PARAMS, (void(*)(void))x942kdf_gettable_ctx_params }, { OSSL_FUNC_KDF_GET_CTX_PARAMS, (void(*)(void))x942kdf_get_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/kem/ec_kem.c b/openssl/src/providers/implementations/kem/ec_kem.c deleted file mode 100644 index b82f90366..000000000 --- a/openssl/src/providers/implementations/kem/ec_kem.c +++ /dev/null @@ -1,814 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * The following implementation is part of RFC 9180 related to DHKEM using - * EC keys (i.e. P-256, P-384 and P-521) - * References to Sections in the comments below refer to RFC 9180. - */ - -#include "internal/deprecated.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "prov/provider_ctx.h" -#include "prov/implementations.h" -#include "prov/securitycheck.h" -#include "prov/providercommon.h" - -#include -#include "internal/hpke_util.h" -#include "crypto/ec.h" -#include "prov/ecx.h" -#include "eckem.h" - -typedef struct { - EC_KEY *recipient_key; - EC_KEY *sender_authkey; - OSSL_LIB_CTX *libctx; - char *propq; - unsigned int mode; - unsigned int op; - unsigned char *ikm; - size_t ikmlen; - const char *kdfname; - const OSSL_HPKE_KEM_INFO *info; -} PROV_EC_CTX; - -static OSSL_FUNC_kem_newctx_fn eckem_newctx; -static OSSL_FUNC_kem_encapsulate_init_fn eckem_encapsulate_init; -static OSSL_FUNC_kem_auth_encapsulate_init_fn eckem_auth_encapsulate_init; -static OSSL_FUNC_kem_encapsulate_fn eckem_encapsulate; -static OSSL_FUNC_kem_decapsulate_init_fn eckem_decapsulate_init; -static OSSL_FUNC_kem_auth_decapsulate_init_fn eckem_auth_decapsulate_init; -static OSSL_FUNC_kem_decapsulate_fn eckem_decapsulate; -static OSSL_FUNC_kem_freectx_fn eckem_freectx; -static OSSL_FUNC_kem_set_ctx_params_fn eckem_set_ctx_params; -static OSSL_FUNC_kem_settable_ctx_params_fn eckem_settable_ctx_params; - -/* ASCII: "KEM", in hex for EBCDIC compatibility */ -static const char LABEL_KEM[] = "\x4b\x45\x4d"; - -static int eckey_check(const EC_KEY *ec, int requires_privatekey) -{ - int rv = 0; - BN_CTX *bnctx = NULL; - BIGNUM *rem = NULL; - const BIGNUM *priv = EC_KEY_get0_private_key(ec); - const EC_POINT *pub = EC_KEY_get0_public_key(ec); - - /* Keys always require a public component */ - if (pub == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_NOT_A_PUBLIC_KEY); - return 0; - } - if (priv == NULL) { - return (requires_privatekey == 0); - } else { - /* If there is a private key, check that is non zero (mod order) */ - const EC_GROUP *group = EC_KEY_get0_group(ec); - const BIGNUM *order = EC_GROUP_get0_order(group); - - bnctx = BN_CTX_new_ex(ossl_ec_key_get_libctx(ec)); - rem = BN_new(); - - if (order != NULL && rem != NULL && bnctx != NULL) { - rv = BN_mod(rem, priv, order, bnctx) - && !BN_is_zero(rem); - } - } - BN_free(rem); - BN_CTX_free(bnctx); - return rv; -} - -/* Returns NULL if the curve is not supported */ -static const char *ec_curvename_get0(const EC_KEY *ec) -{ - const EC_GROUP *group = EC_KEY_get0_group(ec); - - return EC_curve_nid2nist(EC_GROUP_get_curve_name(group)); -} - -/* - * Set the recipient key, and free any existing key. - * ec can be NULL. - * The ec key may have only a private or public component - * (but it must have a group). - */ -static int recipient_key_set(PROV_EC_CTX *ctx, EC_KEY *ec) -{ - EC_KEY_free(ctx->recipient_key); - ctx->recipient_key = NULL; - - if (ec != NULL) { - const char *curve = ec_curvename_get0(ec); - - if (curve == NULL) - return -2; - ctx->info = ossl_HPKE_KEM_INFO_find_curve(curve); - if (ctx->info == NULL) - return -2; - if (!EC_KEY_up_ref(ec)) - return 0; - ctx->recipient_key = ec; - ctx->kdfname = "HKDF"; - } - return 1; -} - -/* - * Set the senders auth key, and free any existing auth key. - * ec can be NULL. - */ -static int sender_authkey_set(PROV_EC_CTX *ctx, EC_KEY *ec) -{ - EC_KEY_free(ctx->sender_authkey); - ctx->sender_authkey = NULL; - - if (ec != NULL) { - if (!EC_KEY_up_ref(ec)) - return 0; - ctx->sender_authkey = ec; - } - return 1; -} - -/* - * Serializes a encoded public key buffer into a EC public key. - * Params: - * in Contains the group. - * pubbuf The encoded public key buffer - * Returns: The created public EC key, or NULL if there is an error. - */ -static EC_KEY *eckey_frompub(EC_KEY *in, - const unsigned char *pubbuf, size_t pubbuflen) -{ - EC_KEY *key; - - key = EC_KEY_new_ex(ossl_ec_key_get_libctx(in), ossl_ec_key_get0_propq(in)); - if (key == NULL) - goto err; - if (!EC_KEY_set_group(key, EC_KEY_get0_group(in))) - goto err; - if (!EC_KEY_oct2key(key, pubbuf, pubbuflen, NULL)) - goto err; - return key; -err: - EC_KEY_free(key); - return NULL; -} - -/* - * Deserialises a EC public key into a encoded byte array. - * Returns: 1 if successful or 0 otherwise. - */ -static int ecpubkey_todata(const EC_KEY *ec, unsigned char *out, size_t *outlen, - size_t maxoutlen) -{ - const EC_POINT *pub; - const EC_GROUP *group; - - group = EC_KEY_get0_group(ec); - pub = EC_KEY_get0_public_key(ec); - *outlen = EC_POINT_point2oct(group, pub, POINT_CONVERSION_UNCOMPRESSED, - out, maxoutlen, NULL); - return *outlen != 0; -} - -static void *eckem_newctx(void *provctx) -{ - PROV_EC_CTX *ctx = OPENSSL_zalloc(sizeof(PROV_EC_CTX)); - - if (ctx == NULL) - return NULL; - ctx->libctx = PROV_LIBCTX_OF(provctx); - - return ctx; -} - -static void eckem_freectx(void *vectx) -{ - PROV_EC_CTX *ctx = (PROV_EC_CTX *)vectx; - - OPENSSL_clear_free(ctx->ikm, ctx->ikmlen); - recipient_key_set(ctx, NULL); - sender_authkey_set(ctx, NULL); - OPENSSL_free(ctx); -} - -static int ossl_ec_match_params(const EC_KEY *key1, const EC_KEY *key2) -{ - int ret; - BN_CTX *ctx = NULL; - const EC_GROUP *group1 = EC_KEY_get0_group(key1); - const EC_GROUP *group2 = EC_KEY_get0_group(key2); - - ctx = BN_CTX_new_ex(ossl_ec_key_get_libctx(key1)); - if (ctx == NULL) - return 0; - - ret = group1 != NULL - && group2 != NULL - && EC_GROUP_cmp(group1, group2, ctx) == 0; - if (!ret) - ERR_raise(ERR_LIB_PROV, PROV_R_MISMATCHING_DOMAIN_PARAMETERS); - BN_CTX_free(ctx); - return ret; -} - -static int eckem_init(void *vctx, int operation, void *vec, void *vauth, - const OSSL_PARAM params[]) -{ - int rv; - PROV_EC_CTX *ctx = (PROV_EC_CTX *)vctx; - EC_KEY *ec = vec; - EC_KEY *auth = vauth; - - if (!ossl_prov_is_running()) - return 0; - - if (!eckey_check(ec, operation == EVP_PKEY_OP_DECAPSULATE)) - return 0; - rv = recipient_key_set(ctx, ec); - if (rv <= 0) - return rv; - - if (auth != NULL) { - if (!ossl_ec_match_params(ec, auth) - || !eckey_check(auth, operation == EVP_PKEY_OP_ENCAPSULATE) - || !sender_authkey_set(ctx, auth)) - return 0; - } - - ctx->op = operation; - return eckem_set_ctx_params(vctx, params); -} - -static int eckem_encapsulate_init(void *vctx, void *vec, - const OSSL_PARAM params[]) -{ - return eckem_init(vctx, EVP_PKEY_OP_ENCAPSULATE, vec, NULL, params); -} - -static int eckem_decapsulate_init(void *vctx, void *vec, - const OSSL_PARAM params[]) -{ - return eckem_init(vctx, EVP_PKEY_OP_DECAPSULATE, vec, NULL, params); -} - -static int eckem_auth_encapsulate_init(void *vctx, void *vecx, void *vauthpriv, - const OSSL_PARAM params[]) -{ - return eckem_init(vctx, EVP_PKEY_OP_ENCAPSULATE, vecx, vauthpriv, params); -} - -static int eckem_auth_decapsulate_init(void *vctx, void *vecx, void *vauthpub, - const OSSL_PARAM params[]) -{ - return eckem_init(vctx, EVP_PKEY_OP_DECAPSULATE, vecx, vauthpub, params); -} - -static int eckem_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - PROV_EC_CTX *ctx = (PROV_EC_CTX *)vctx; - const OSSL_PARAM *p; - int mode; - - if (params == NULL) - return 1; - - p = OSSL_PARAM_locate_const(params, OSSL_KEM_PARAM_IKME); - if (p != NULL) { - void *tmp = NULL; - size_t tmplen = 0; - - if (p->data != NULL && p->data_size != 0) { - if (!OSSL_PARAM_get_octet_string(p, &tmp, 0, &tmplen)) - return 0; - } - OPENSSL_clear_free(ctx->ikm, ctx->ikmlen); - /* Set the ephemeral seed */ - ctx->ikm = tmp; - ctx->ikmlen = tmplen; - } - - p = OSSL_PARAM_locate_const(params, OSSL_KEM_PARAM_OPERATION); - if (p != NULL) { - if (p->data_type != OSSL_PARAM_UTF8_STRING) - return 0; - mode = ossl_eckem_modename2id(p->data); - if (mode == KEM_MODE_UNDEFINED) - return 0; - ctx->mode = mode; - } - return 1; -} - -static const OSSL_PARAM known_settable_eckem_ctx_params[] = { - OSSL_PARAM_utf8_string(OSSL_KEM_PARAM_OPERATION, NULL, 0), - OSSL_PARAM_octet_string(OSSL_KEM_PARAM_IKME, NULL, 0), - OSSL_PARAM_END -}; - -static const OSSL_PARAM *eckem_settable_ctx_params(ossl_unused void *vctx, - ossl_unused void *provctx) -{ - return known_settable_eckem_ctx_params; -} - -/* - * See Section 4.1 DH-Based KEM (DHKEM) ExtractAndExpand - */ -static int dhkem_extract_and_expand(EVP_KDF_CTX *kctx, - unsigned char *okm, size_t okmlen, - uint16_t kemid, - const unsigned char *dhkm, size_t dhkmlen, - const unsigned char *kemctx, - size_t kemctxlen) -{ - uint8_t suiteid[2]; - uint8_t prk[EVP_MAX_MD_SIZE]; - size_t prklen = okmlen; - int ret; - - if (prklen > sizeof(prk)) - return 0; - - suiteid[0] = (kemid >> 8) & 0xff; - suiteid[1] = kemid & 0xff; - - ret = ossl_hpke_labeled_extract(kctx, prk, prklen, - NULL, 0, LABEL_KEM, suiteid, sizeof(suiteid), - OSSL_DHKEM_LABEL_EAE_PRK, dhkm, dhkmlen) - && ossl_hpke_labeled_expand(kctx, okm, okmlen, prk, prklen, - LABEL_KEM, suiteid, sizeof(suiteid), - OSSL_DHKEM_LABEL_SHARED_SECRET, - kemctx, kemctxlen); - OPENSSL_cleanse(prk, prklen); - return ret; -} - -/* - * See Section 7.1.3 DeriveKeyPair. - * - * This function is used by ec keygen. - * (For this reason it does not use any of the state stored in PROV_EC_CTX). - * - * Params: - * ec An initialized ec key. - * priv The buffer to store the generated private key into (it is assumed - * this is of length alg->encodedprivlen). - * ikm buffer containing the input key material (seed). This must be set. - * ikmlen size of the ikm buffer in bytes - * Returns: - * 1 if successful or 0 otherwise. - */ -int ossl_ec_dhkem_derive_private(EC_KEY *ec, BIGNUM *priv, - const unsigned char *ikm, size_t ikmlen) -{ - int ret = 0; - EVP_KDF_CTX *kdfctx = NULL; - uint8_t suiteid[2]; - unsigned char prk[OSSL_HPKE_MAX_SECRET]; - unsigned char privbuf[OSSL_HPKE_MAX_PRIVATE]; - const BIGNUM *order; - unsigned char counter = 0; - const char *curve = ec_curvename_get0(ec); - const OSSL_HPKE_KEM_INFO *info; - - if (curve == NULL) - return -2; - - info = ossl_HPKE_KEM_INFO_find_curve(curve); - if (info == NULL) - return -2; - - kdfctx = ossl_kdf_ctx_create("HKDF", info->mdname, - ossl_ec_key_get_libctx(ec), - ossl_ec_key_get0_propq(ec)); - if (kdfctx == NULL) - return 0; - - /* ikmlen should have a length of at least Nsk */ - if (ikmlen < info->Nsecret) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_INPUT_LENGTH, - "ikm length is :%zu, should be at least %zu", - ikmlen, info->Nsecret); - goto err; - } - - suiteid[0] = info->kem_id / 256; - suiteid[1] = info->kem_id % 256; - - if (!ossl_hpke_labeled_extract(kdfctx, prk, info->Nsecret, - NULL, 0, LABEL_KEM, suiteid, sizeof(suiteid), - OSSL_DHKEM_LABEL_DKP_PRK, ikm, ikmlen)) - goto err; - - order = EC_GROUP_get0_order(EC_KEY_get0_group(ec)); - do { - if (!ossl_hpke_labeled_expand(kdfctx, privbuf, info->Nsk, - prk, info->Nsecret, - LABEL_KEM, suiteid, sizeof(suiteid), - OSSL_DHKEM_LABEL_CANDIDATE, - &counter, 1)) - goto err; - privbuf[0] &= info->bitmask; - if (BN_bin2bn(privbuf, info->Nsk, priv) == NULL) - goto err; - if (counter == 0xFF) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GENERATE_KEY); - goto err; - } - counter++; - } while (BN_is_zero(priv) || BN_cmp(priv, order) >= 0); - ret = 1; -err: - OPENSSL_cleanse(prk, sizeof(prk)); - OPENSSL_cleanse(privbuf, sizeof(privbuf)); - EVP_KDF_CTX_free(kdfctx); - return ret; -} - -/* - * Do a keygen operation without having to use EVP_PKEY. - * Params: - * ctx Context object - * ikm The seed material - if this is NULL, then a random seed is used. - * Returns: - * The generated EC key, or NULL on failure. - */ -static EC_KEY *derivekey(PROV_EC_CTX *ctx, - const unsigned char *ikm, size_t ikmlen) -{ - int ret = 0; - EC_KEY *key; - unsigned char *seed = (unsigned char *)ikm; - size_t seedlen = ikmlen; - unsigned char tmpbuf[OSSL_HPKE_MAX_PRIVATE]; - - key = EC_KEY_new_ex(ctx->libctx, ctx->propq); - if (key == NULL) - goto err; - if (!EC_KEY_set_group(key, EC_KEY_get0_group(ctx->recipient_key))) - goto err; - - /* Generate a random seed if there is no input ikm */ - if (seed == NULL || seedlen == 0) { - seedlen = ctx->info->Nsk; - if (seedlen > sizeof(tmpbuf)) - goto err; - if (RAND_priv_bytes_ex(ctx->libctx, tmpbuf, seedlen, 0) <= 0) - goto err; - seed = tmpbuf; - } - ret = ossl_ec_generate_key_dhkem(key, seed, seedlen); -err: - if (seed != ikm) - OPENSSL_cleanse(seed, seedlen); - if (ret <= 0) { - EC_KEY_free(key); - key = NULL; - } - return key; -} - -/* - * Before doing a key exchange the public key of the peer needs to be checked - * Note that the group check is not done here as we have already checked - * that it only uses one of the approved curve names when the key was set. - * - * Returns 1 if the public key is valid, or 0 if it fails. - */ -static int check_publickey(const EC_KEY *pub) -{ - int ret = 0; - BN_CTX *bnctx = BN_CTX_new_ex(ossl_ec_key_get_libctx(pub)); - - if (bnctx == NULL) - return 0; - ret = ossl_ec_key_public_check(pub, bnctx); - BN_CTX_free(bnctx); - - return ret; -} - -/* - * Do an ecdh key exchange. - * dhkm = DH(sender, peer) - * - * NOTE: Instead of using EVP_PKEY_derive() API's, we use EC_KEY operations - * to avoid messy conversions back to EVP_PKEY. - * - * Returns the size of the secret if successful, or 0 otherwise, - */ -static int generate_ecdhkm(const EC_KEY *sender, const EC_KEY *peer, - unsigned char *out, size_t maxout, - unsigned int secretsz) -{ - const EC_GROUP *group = EC_KEY_get0_group(sender); - size_t secretlen = (EC_GROUP_get_degree(group) + 7) / 8; - - if (secretlen != secretsz || secretlen > maxout) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_BAD_LENGTH, "secretsz invalid"); - return 0; - } - - if (!check_publickey(peer)) - return 0; - return ECDH_compute_key(out, secretlen, EC_KEY_get0_public_key(peer), - sender, NULL) > 0; -} - -/* - * Derive a secret using ECDH (code is shared by the encap and decap) - * - * dhkm = Concat(ecdh(privkey1, peerkey1), ecdh(privkey2, peerkey2) - * kemctx = Concat(sender_pub, recipient_pub, ctx->sender_authkey) - * secret = dhkem_extract_and_expand(kemid, dhkm, kemctx); - * - * Params: - * ctx Object that contains algorithm state and constants. - * secret The returned secret (with a length ctx->alg->secretlen bytes). - * privkey1 A private key used for ECDH key derivation. - * peerkey1 A public key used for ECDH key derivation with privkey1 - * privkey2 A optional private key used for a second ECDH key derivation. - * It can be NULL. - * peerkey2 A optional public key used for a second ECDH key derivation - * with privkey2,. It can be NULL. - * sender_pub The senders public key in encoded form. - * recipient_pub The recipients public key in encoded form. - * Notes: - * The second ecdh() is only used for the HPKE auth modes when both privkey2 - * and peerkey2 are non NULL (i.e. ctx->sender_authkey is not NULL). - */ -static int derive_secret(PROV_EC_CTX *ctx, unsigned char *secret, - const EC_KEY *privkey1, const EC_KEY *peerkey1, - const EC_KEY *privkey2, const EC_KEY *peerkey2, - const unsigned char *sender_pub, - const unsigned char *recipient_pub) -{ - int ret = 0; - EVP_KDF_CTX *kdfctx = NULL; - unsigned char sender_authpub[OSSL_HPKE_MAX_PUBLIC]; - unsigned char dhkm[OSSL_HPKE_MAX_PRIVATE * 2]; - unsigned char kemctx[OSSL_HPKE_MAX_PUBLIC * 3]; - size_t sender_authpublen; - size_t kemctxlen = 0, dhkmlen = 0; - const OSSL_HPKE_KEM_INFO *info = ctx->info; - size_t encodedpublen = info->Npk; - size_t encodedprivlen = info->Nsk; - int auth = ctx->sender_authkey != NULL; - - if (!generate_ecdhkm(privkey1, peerkey1, dhkm, sizeof(dhkm), encodedprivlen)) - goto err; - dhkmlen = encodedprivlen; - kemctxlen = 2 * encodedpublen; - - /* Concat the optional second ECDH (used for Auth) */ - if (auth) { - /* Get the public key of the auth sender in encoded form */ - if (!ecpubkey_todata(ctx->sender_authkey, sender_authpub, - &sender_authpublen, sizeof(sender_authpub))) - goto err; - if (sender_authpublen != encodedpublen) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_KEY, - "Invalid sender auth public key"); - goto err; - } - if (!generate_ecdhkm(privkey2, peerkey2, - dhkm + dhkmlen, sizeof(dhkm) - dhkmlen, - encodedprivlen)) - goto err; - dhkmlen += encodedprivlen; - kemctxlen += encodedpublen; - } - if (kemctxlen > sizeof(kemctx)) - goto err; - - /* kemctx is the concat of both sides encoded public key */ - memcpy(kemctx, sender_pub, info->Npk); - memcpy(kemctx + info->Npk, recipient_pub, info->Npk); - if (auth) - memcpy(kemctx + 2 * encodedpublen, sender_authpub, encodedpublen); - kdfctx = ossl_kdf_ctx_create(ctx->kdfname, info->mdname, - ctx->libctx, ctx->propq); - if (kdfctx == NULL) - goto err; - if (!dhkem_extract_and_expand(kdfctx, secret, info->Nsecret, - info->kem_id, dhkm, dhkmlen, - kemctx, kemctxlen)) - goto err; - ret = 1; -err: - OPENSSL_cleanse(dhkm, dhkmlen); - EVP_KDF_CTX_free(kdfctx); - return ret; -} - -/* - * Do a DHKEM encapsulate operation. - * - * See Section 4.1 Encap() and AuthEncap() - * - * Params: - * ctx A context object holding the recipients public key and the - * optional senders auth private key. - * enc A buffer to return the senders ephemeral public key. - * Setting this to NULL allows the enclen and secretlen to return - * values, without calculating the secret. - * enclen Passes in the max size of the enc buffer and returns the - * encoded public key length. - * secret A buffer to return the calculated shared secret. - * secretlen Passes in the max size of the secret buffer and returns the - * secret length. - * Returns: 1 on success or 0 otherwise. - */ -static int dhkem_encap(PROV_EC_CTX *ctx, - unsigned char *enc, size_t *enclen, - unsigned char *secret, size_t *secretlen) -{ - int ret = 0; - EC_KEY *sender_ephemkey = NULL; - unsigned char sender_pub[OSSL_HPKE_MAX_PUBLIC]; - unsigned char recipient_pub[OSSL_HPKE_MAX_PUBLIC]; - size_t sender_publen, recipient_publen; - const OSSL_HPKE_KEM_INFO *info = ctx->info; - - if (enc == NULL) { - if (enclen == NULL && secretlen == NULL) - return 0; - if (enclen != NULL) - *enclen = info->Nenc; - if (secretlen != NULL) - *secretlen = info->Nsecret; - return 1; - } - - if (*secretlen < info->Nsecret) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_BAD_LENGTH, "*secretlen too small"); - return 0; - } - if (*enclen < info->Nenc) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_BAD_LENGTH, "*enclen too small"); - return 0; - } - - /* Create an ephemeral key */ - sender_ephemkey = derivekey(ctx, ctx->ikm, ctx->ikmlen); - if (sender_ephemkey == NULL) - goto err; - if (!ecpubkey_todata(sender_ephemkey, sender_pub, &sender_publen, - sizeof(sender_pub)) - || !ecpubkey_todata(ctx->recipient_key, recipient_pub, - &recipient_publen, sizeof(recipient_pub))) - goto err; - - if (sender_publen != info->Npk - || recipient_publen != sender_publen) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_KEY, "Invalid public key"); - goto err; - } - - if (!derive_secret(ctx, secret, - sender_ephemkey, ctx->recipient_key, - ctx->sender_authkey, ctx->recipient_key, - sender_pub, recipient_pub)) - goto err; - - /* Return the senders ephemeral public key in encoded form */ - memcpy(enc, sender_pub, sender_publen); - *enclen = sender_publen; - *secretlen = info->Nsecret; - ret = 1; -err: - EC_KEY_free(sender_ephemkey); - return ret; -} - -/* - * Do a DHKEM decapsulate operation. - * See Section 4.1 Decap() and Auth Decap() - * - * Params: - * ctx A context object holding the recipients private key and the - * optional senders auth public key. - * secret A buffer to return the calculated shared secret. Setting this to - * NULL can be used to return the secretlen. - * secretlen Passes in the max size of the secret buffer and returns the - * secret length. - * enc A buffer containing the senders ephemeral public key that was returned - * from dhkem_encap(). - * enclen The length in bytes of enc. - * Returns: 1 If the shared secret is returned or 0 on error. - */ -static int dhkem_decap(PROV_EC_CTX *ctx, - unsigned char *secret, size_t *secretlen, - const unsigned char *enc, size_t enclen) -{ - int ret = 0; - EC_KEY *sender_ephempubkey = NULL; - const OSSL_HPKE_KEM_INFO *info = ctx->info; - unsigned char recipient_pub[OSSL_HPKE_MAX_PUBLIC]; - size_t recipient_publen; - size_t encodedpublen = info->Npk; - - if (secret == NULL) { - *secretlen = info->Nsecret; - return 1; - } - - if (*secretlen < info->Nsecret) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_BAD_LENGTH, "*secretlen too small"); - return 0; - } - if (enclen != encodedpublen) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_KEY, "Invalid enc public key"); - return 0; - } - - sender_ephempubkey = eckey_frompub(ctx->recipient_key, enc, enclen); - if (sender_ephempubkey == NULL) - goto err; - if (!ecpubkey_todata(ctx->recipient_key, recipient_pub, &recipient_publen, - sizeof(recipient_pub))) - goto err; - if (recipient_publen != encodedpublen) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_KEY, "Invalid recipient public key"); - goto err; - } - - if (!derive_secret(ctx, secret, - ctx->recipient_key, sender_ephempubkey, - ctx->recipient_key, ctx->sender_authkey, - enc, recipient_pub)) - goto err; - *secretlen = info->Nsecret; - ret = 1; -err: - EC_KEY_free(sender_ephempubkey); - return ret; -} - -static int eckem_encapsulate(void *vctx, unsigned char *out, size_t *outlen, - unsigned char *secret, size_t *secretlen) -{ - PROV_EC_CTX *ctx = (PROV_EC_CTX *)vctx; - - switch (ctx->mode) { - case KEM_MODE_DHKEM: - return dhkem_encap(ctx, out, outlen, secret, secretlen); - default: - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MODE); - return -2; - } -} - -static int eckem_decapsulate(void *vctx, unsigned char *out, size_t *outlen, - const unsigned char *in, size_t inlen) -{ - PROV_EC_CTX *ctx = (PROV_EC_CTX *)vctx; - - switch (ctx->mode) { - case KEM_MODE_DHKEM: - return dhkem_decap(ctx, out, outlen, in, inlen); - default: - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MODE); - return -2; - } -} - -const OSSL_DISPATCH ossl_ec_asym_kem_functions[] = { - { OSSL_FUNC_KEM_NEWCTX, (void (*)(void))eckem_newctx }, - { OSSL_FUNC_KEM_ENCAPSULATE_INIT, - (void (*)(void))eckem_encapsulate_init }, - { OSSL_FUNC_KEM_ENCAPSULATE, (void (*)(void))eckem_encapsulate }, - { OSSL_FUNC_KEM_DECAPSULATE_INIT, - (void (*)(void))eckem_decapsulate_init }, - { OSSL_FUNC_KEM_DECAPSULATE, (void (*)(void))eckem_decapsulate }, - { OSSL_FUNC_KEM_FREECTX, (void (*)(void))eckem_freectx }, - { OSSL_FUNC_KEM_SET_CTX_PARAMS, - (void (*)(void))eckem_set_ctx_params }, - { OSSL_FUNC_KEM_SETTABLE_CTX_PARAMS, - (void (*)(void))eckem_settable_ctx_params }, - { OSSL_FUNC_KEM_AUTH_ENCAPSULATE_INIT, - (void (*)(void))eckem_auth_encapsulate_init }, - { OSSL_FUNC_KEM_AUTH_DECAPSULATE_INIT, - (void (*)(void))eckem_auth_decapsulate_init }, - OSSL_DISPATCH_END -}; diff --git a/openssl/src/providers/implementations/kem/ecx_kem.c b/openssl/src/providers/implementations/kem/ecx_kem.c deleted file mode 100644 index 4a762f215..000000000 --- a/openssl/src/providers/implementations/kem/ecx_kem.c +++ /dev/null @@ -1,704 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* - * The following implementation is part of RFC 9180 related to DHKEM using - * ECX keys (i.e. X25519 and X448) - * References to Sections in the comments below refer to RFC 9180. - */ - -#include "internal/deprecated.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "prov/provider_ctx.h" -#include "prov/implementations.h" -#include "prov/securitycheck.h" -#include "prov/providercommon.h" -#include "prov/ecx.h" -#include "crypto/ecx.h" -#include -#include "internal/hpke_util.h" -#include "eckem.h" - -#define MAX_ECX_KEYLEN X448_KEYLEN - -/* KEM identifiers from Section 7.1 "Table 2 KEM IDs" */ -#define KEMID_X25519_HKDF_SHA256 0x20 -#define KEMID_X448_HKDF_SHA512 0x21 - -/* ASCII: "KEM", in hex for EBCDIC compatibility */ -static const char LABEL_KEM[] = "\x4b\x45\x4d"; - -typedef struct { - ECX_KEY *recipient_key; - ECX_KEY *sender_authkey; - OSSL_LIB_CTX *libctx; - char *propq; - unsigned int mode; - unsigned int op; - unsigned char *ikm; - size_t ikmlen; - const char *kdfname; - const OSSL_HPKE_KEM_INFO *info; -} PROV_ECX_CTX; - -static OSSL_FUNC_kem_newctx_fn ecxkem_newctx; -static OSSL_FUNC_kem_encapsulate_init_fn ecxkem_encapsulate_init; -static OSSL_FUNC_kem_encapsulate_fn ecxkem_encapsulate; -static OSSL_FUNC_kem_decapsulate_init_fn ecxkem_decapsulate_init; -static OSSL_FUNC_kem_decapsulate_fn ecxkem_decapsulate; -static OSSL_FUNC_kem_freectx_fn ecxkem_freectx; -static OSSL_FUNC_kem_set_ctx_params_fn ecxkem_set_ctx_params; -static OSSL_FUNC_kem_auth_encapsulate_init_fn ecxkem_auth_encapsulate_init; -static OSSL_FUNC_kem_auth_decapsulate_init_fn ecxkem_auth_decapsulate_init; - -/* - * Set KEM values as specified in Section 7.1 "Table 2 KEM IDs" - * There is only one set of values for X25519 and X448. - * Additional values could be set via set_params if required. - */ -static const OSSL_HPKE_KEM_INFO *get_kem_info(ECX_KEY *ecx) -{ - const char *name = NULL; - - if (ecx->type == ECX_KEY_TYPE_X25519) - name = SN_X25519; - else - name = SN_X448; - return ossl_HPKE_KEM_INFO_find_curve(name); -} - -/* - * Set the recipient key, and free any existing key. - * ecx can be NULL. The ecx key may have only a private or public component. - */ -static int recipient_key_set(PROV_ECX_CTX *ctx, ECX_KEY *ecx) -{ - ossl_ecx_key_free(ctx->recipient_key); - ctx->recipient_key = NULL; - if (ecx != NULL) { - ctx->info = get_kem_info(ecx); - if (ctx->info == NULL) - return -2; - ctx->kdfname = "HKDF"; - if (!ossl_ecx_key_up_ref(ecx)) - return 0; - ctx->recipient_key = ecx; - } - return 1; -} - -/* - * Set the senders auth key, and free any existing auth key. - * ecx can be NULL. - */ -static int sender_authkey_set(PROV_ECX_CTX *ctx, ECX_KEY *ecx) -{ - ossl_ecx_key_free(ctx->sender_authkey); - ctx->sender_authkey = NULL; - - if (ecx != NULL) { - if (!ossl_ecx_key_up_ref(ecx)) - return 0; - ctx->sender_authkey = ecx; - } - return 1; -} - -/* - * Serialize a public key from byte array's for the encoded public keys. - * ctx is used to access the key type. - * Returns: The created ECX_KEY or NULL on error. - */ -static ECX_KEY *ecxkey_pubfromdata(PROV_ECX_CTX *ctx, - const unsigned char *pubbuf, size_t pubbuflen) -{ - ECX_KEY *ecx = NULL; - OSSL_PARAM params[2], *p = params; - - *p++ = OSSL_PARAM_construct_octet_string(OSSL_PKEY_PARAM_PUB_KEY, - (char *)pubbuf, pubbuflen); - *p = OSSL_PARAM_construct_end(); - - ecx = ossl_ecx_key_new(ctx->libctx, ctx->recipient_key->type, 1, ctx->propq); - if (ecx == NULL) - return NULL; - if (ossl_ecx_key_fromdata(ecx, params, 0) <= 0) { - ossl_ecx_key_free(ecx); - ecx = NULL; - } - return ecx; -} - -static unsigned char *ecx_pubkey(ECX_KEY *ecx) -{ - if (ecx == NULL || !ecx->haspubkey) { - ERR_raise(ERR_LIB_PROV, PROV_R_NOT_A_PUBLIC_KEY); - return 0; - } - return ecx->pubkey; -} - -static void *ecxkem_newctx(void *provctx) -{ - PROV_ECX_CTX *ctx = OPENSSL_zalloc(sizeof(PROV_ECX_CTX)); - - if (ctx == NULL) - return NULL; - ctx->libctx = PROV_LIBCTX_OF(provctx); - - return ctx; -} - -static void ecxkem_freectx(void *vectx) -{ - PROV_ECX_CTX *ctx = (PROV_ECX_CTX *)vectx; - - OPENSSL_clear_free(ctx->ikm, ctx->ikmlen); - recipient_key_set(ctx, NULL); - sender_authkey_set(ctx, NULL); - OPENSSL_free(ctx); -} - -static int ecx_match_params(const ECX_KEY *key1, const ECX_KEY *key2) -{ - return (key1->type == key2->type && key1->keylen == key2->keylen); -} - -static int ecx_key_check(const ECX_KEY *ecx, int requires_privatekey) -{ - if (ecx->privkey == NULL) - return (requires_privatekey == 0); - return 1; -} - -static int ecxkem_init(void *vecxctx, int operation, void *vecx, void *vauth, - ossl_unused const OSSL_PARAM params[]) -{ - int rv; - PROV_ECX_CTX *ctx = (PROV_ECX_CTX *)vecxctx; - ECX_KEY *ecx = vecx; - ECX_KEY *auth = vauth; - - if (!ossl_prov_is_running()) - return 0; - - if (!ecx_key_check(ecx, operation == EVP_PKEY_OP_DECAPSULATE)) - return 0; - rv = recipient_key_set(ctx, ecx); - if (rv <= 0) - return rv; - - if (auth != NULL) { - if (!ecx_match_params(auth, ctx->recipient_key) - || !ecx_key_check(auth, operation == EVP_PKEY_OP_ENCAPSULATE) - || !sender_authkey_set(ctx, auth)) - return 0; - } - - ctx->op = operation; - return ecxkem_set_ctx_params(vecxctx, params); -} - -static int ecxkem_encapsulate_init(void *vecxctx, void *vecx, - const OSSL_PARAM params[]) -{ - return ecxkem_init(vecxctx, EVP_PKEY_OP_ENCAPSULATE, vecx, NULL, params); -} - -static int ecxkem_decapsulate_init(void *vecxctx, void *vecx, - const OSSL_PARAM params[]) -{ - return ecxkem_init(vecxctx, EVP_PKEY_OP_DECAPSULATE, vecx, NULL, params); -} - -static int ecxkem_auth_encapsulate_init(void *vctx, void *vecx, void *vauthpriv, - const OSSL_PARAM params[]) -{ - return ecxkem_init(vctx, EVP_PKEY_OP_ENCAPSULATE, vecx, vauthpriv, params); -} - -static int ecxkem_auth_decapsulate_init(void *vctx, void *vecx, void *vauthpub, - const OSSL_PARAM params[]) -{ - return ecxkem_init(vctx, EVP_PKEY_OP_DECAPSULATE, vecx, vauthpub, params); -} - -static int ecxkem_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - PROV_ECX_CTX *ctx = (PROV_ECX_CTX *)vctx; - const OSSL_PARAM *p; - int mode; - - if (ctx == NULL) - return 0; - if (params == NULL) - return 1; - - p = OSSL_PARAM_locate_const(params, OSSL_KEM_PARAM_IKME); - if (p != NULL) { - void *tmp = NULL; - size_t tmplen = 0; - - if (p->data != NULL && p->data_size != 0) { - if (!OSSL_PARAM_get_octet_string(p, &tmp, 0, &tmplen)) - return 0; - } - OPENSSL_clear_free(ctx->ikm, ctx->ikmlen); - ctx->ikm = tmp; - ctx->ikmlen = tmplen; - } - p = OSSL_PARAM_locate_const(params, OSSL_KEM_PARAM_OPERATION); - if (p != NULL) { - if (p->data_type != OSSL_PARAM_UTF8_STRING) - return 0; - mode = ossl_eckem_modename2id(p->data); - if (mode == KEM_MODE_UNDEFINED) - return 0; - ctx->mode = mode; - } - return 1; -} - -static const OSSL_PARAM known_settable_ecxkem_ctx_params[] = { - OSSL_PARAM_utf8_string(OSSL_KEM_PARAM_OPERATION, NULL, 0), - OSSL_PARAM_octet_string(OSSL_KEM_PARAM_IKME, NULL, 0), - OSSL_PARAM_END -}; - -static const OSSL_PARAM *ecxkem_settable_ctx_params(ossl_unused void *vctx, - ossl_unused void *provctx) -{ - return known_settable_ecxkem_ctx_params; -} - -/* - * See Section 4.1 DH-Based KEM (DHKEM) ExtractAndExpand - */ -static int dhkem_extract_and_expand(EVP_KDF_CTX *kctx, - unsigned char *okm, size_t okmlen, - uint16_t kemid, - const unsigned char *dhkm, size_t dhkmlen, - const unsigned char *kemctx, - size_t kemctxlen) -{ - uint8_t suiteid[2]; - uint8_t prk[EVP_MAX_MD_SIZE]; - size_t prklen = okmlen; /* Nh */ - int ret; - - if (prklen > sizeof(prk)) - return 0; - - suiteid[0] = (kemid >> 8) &0xff; - suiteid[1] = kemid & 0xff; - - ret = ossl_hpke_labeled_extract(kctx, prk, prklen, - NULL, 0, LABEL_KEM, suiteid, sizeof(suiteid), - OSSL_DHKEM_LABEL_EAE_PRK, dhkm, dhkmlen) - && ossl_hpke_labeled_expand(kctx, okm, okmlen, prk, prklen, - LABEL_KEM, suiteid, sizeof(suiteid), - OSSL_DHKEM_LABEL_SHARED_SECRET, - kemctx, kemctxlen); - OPENSSL_cleanse(prk, prklen); - return ret; -} - -/* - * See Section 7.1.3 DeriveKeyPair. - * - * This function is used by ecx keygen. - * (For this reason it does not use any of the state stored in PROV_ECX_CTX). - * - * Params: - * ecx An initialized ecx key. - * privout The buffer to store the generated private key into (it is assumed - * this is of length ecx->keylen). - * ikm buffer containing the input key material (seed). This must be non NULL. - * ikmlen size of the ikm buffer in bytes - * Returns: - * 1 if successful or 0 otherwise. - */ -int ossl_ecx_dhkem_derive_private(ECX_KEY *ecx, unsigned char *privout, - const unsigned char *ikm, size_t ikmlen) -{ - int ret = 0; - EVP_KDF_CTX *kdfctx = NULL; - unsigned char prk[EVP_MAX_MD_SIZE]; - uint8_t suiteid[2]; - const OSSL_HPKE_KEM_INFO *info = get_kem_info(ecx); - - /* ikmlen should have a length of at least Nsk */ - if (ikmlen < info->Nsk) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_INPUT_LENGTH, - "ikm length is :%zu, should be at least %zu", - ikmlen, info->Nsk); - goto err; - } - - kdfctx = ossl_kdf_ctx_create("HKDF", info->mdname, ecx->libctx, ecx->propq); - if (kdfctx == NULL) - return 0; - - suiteid[0] = info->kem_id / 256; - suiteid[1] = info->kem_id % 256; - - if (!ossl_hpke_labeled_extract(kdfctx, prk, info->Nsecret, - NULL, 0, LABEL_KEM, suiteid, sizeof(suiteid), - OSSL_DHKEM_LABEL_DKP_PRK, ikm, ikmlen)) - goto err; - - if (!ossl_hpke_labeled_expand(kdfctx, privout, info->Nsk, prk, info->Nsecret, - LABEL_KEM, suiteid, sizeof(suiteid), - OSSL_DHKEM_LABEL_SK, NULL, 0)) - goto err; - ret = 1; -err: - OPENSSL_cleanse(prk, sizeof(prk)); - EVP_KDF_CTX_free(kdfctx); - return ret; -} - -/* - * Do a keygen operation without having to use EVP_PKEY. - * Params: - * ctx Context object - * ikm The seed material - if this is NULL, then a random seed is used. - * Returns: - * The generated ECX key, or NULL on failure. - */ -static ECX_KEY *derivekey(PROV_ECX_CTX *ctx, - const unsigned char *ikm, size_t ikmlen) -{ - int ok = 0; - ECX_KEY *key; - unsigned char *privkey; - unsigned char *seed = (unsigned char *)ikm; - size_t seedlen = ikmlen; - unsigned char tmpbuf[OSSL_HPKE_MAX_PRIVATE]; - const OSSL_HPKE_KEM_INFO *info = ctx->info; - - key = ossl_ecx_key_new(ctx->libctx, ctx->recipient_key->type, 0, ctx->propq); - if (key == NULL) - return NULL; - privkey = ossl_ecx_key_allocate_privkey(key); - if (privkey == NULL) - goto err; - - /* Generate a random seed if there is no input ikm */ - if (seed == NULL || seedlen == 0) { - if (info->Nsk > sizeof(tmpbuf)) - goto err; - if (RAND_priv_bytes_ex(ctx->libctx, tmpbuf, info->Nsk, 0) <= 0) - goto err; - seed = tmpbuf; - seedlen = info->Nsk; - } - if (!ossl_ecx_dhkem_derive_private(key, privkey, seed, seedlen)) - goto err; - if (!ossl_ecx_public_from_private(key)) - goto err; - key->haspubkey = 1; - ok = 1; -err: - if (!ok) { - ossl_ecx_key_free(key); - key = NULL; - } - if (seed != ikm) - OPENSSL_cleanse(seed, seedlen); - return key; -} - -/* - * Do an ecxdh key exchange. - * dhkm = DH(sender, peer) - * - * NOTE: Instead of using EVP_PKEY_derive() API's, we use ECX_KEY operations - * to avoid messy conversions back to EVP_PKEY. - * - * Returns the size of the secret if successful, or 0 otherwise, - */ -static int generate_ecxdhkm(const ECX_KEY *sender, const ECX_KEY *peer, - unsigned char *out, size_t maxout, - unsigned int secretsz) -{ - size_t len = 0; - - /* NOTE: ossl_ecx_compute_key checks for shared secret being all zeros */ - return ossl_ecx_compute_key((ECX_KEY *)peer, (ECX_KEY *)sender, - sender->keylen, out, &len, maxout); -} - -/* - * Derive a secret using ECXDH (code is shared by the encap and decap) - * - * dhkm = Concat(ecxdh(privkey1, peerkey1), ecdh(privkey2, peerkey2) - * kemctx = Concat(sender_pub, recipient_pub, ctx->sender_authkey) - * secret = dhkem_extract_and_expand(kemid, dhkm, kemctx); - * - * Params: - * ctx Object that contains algorithm state and constants. - * secret The returned secret (with a length ctx->alg->secretlen bytes). - * privkey1 A private key used for ECXDH key derivation. - * peerkey1 A public key used for ECXDH key derivation with privkey1 - * privkey2 A optional private key used for a second ECXDH key derivation. - * It can be NULL. - * peerkey2 A optional public key used for a second ECXDH key derivation - * with privkey2,. It can be NULL. - * sender_pub The senders public key in encoded form. - * recipient_pub The recipients public key in encoded form. - * Notes: - * The second ecdh() is only used for the HPKE auth modes when both privkey2 - * and peerkey2 are non NULL (i.e. ctx->sender_authkey is not NULL). - */ -static int derive_secret(PROV_ECX_CTX *ctx, unsigned char *secret, - const ECX_KEY *privkey1, const ECX_KEY *peerkey1, - const ECX_KEY *privkey2, const ECX_KEY *peerkey2, - const unsigned char *sender_pub, - const unsigned char *recipient_pub) -{ - int ret = 0; - EVP_KDF_CTX *kdfctx = NULL; - unsigned char *sender_authpub = NULL; - unsigned char dhkm[MAX_ECX_KEYLEN * 2]; - unsigned char kemctx[MAX_ECX_KEYLEN * 3]; - size_t kemctxlen = 0, dhkmlen = 0; - const OSSL_HPKE_KEM_INFO *info = ctx->info; - int auth = ctx->sender_authkey != NULL; - size_t encodedkeylen = info->Npk; - - if (!generate_ecxdhkm(privkey1, peerkey1, dhkm, sizeof(dhkm), encodedkeylen)) - goto err; - dhkmlen = encodedkeylen; - - /* Concat the optional second ECXDH (used for Auth) */ - if (auth) { - if (!generate_ecxdhkm(privkey2, peerkey2, - dhkm + dhkmlen, sizeof(dhkm) - dhkmlen, - encodedkeylen)) - goto err; - /* Get the public key of the auth sender in encoded form */ - sender_authpub = ecx_pubkey(ctx->sender_authkey); - if (sender_authpub == NULL) - goto err; - dhkmlen += encodedkeylen; - } - kemctxlen = encodedkeylen + dhkmlen; - if (kemctxlen > sizeof(kemctx)) - goto err; - - /* kemctx is the concat of both sides encoded public key */ - memcpy(kemctx, sender_pub, encodedkeylen); - memcpy(kemctx + encodedkeylen, recipient_pub, encodedkeylen); - if (auth) - memcpy(kemctx + 2 * encodedkeylen, sender_authpub, encodedkeylen); - kdfctx = ossl_kdf_ctx_create(ctx->kdfname, info->mdname, - ctx->libctx, ctx->propq); - if (kdfctx == NULL) - goto err; - if (!dhkem_extract_and_expand(kdfctx, secret, info->Nsecret, - info->kem_id, dhkm, dhkmlen, - kemctx, kemctxlen)) - goto err; - ret = 1; -err: - OPENSSL_cleanse(dhkm, dhkmlen); - EVP_KDF_CTX_free(kdfctx); - return ret; -} - -/* - * Do a DHKEM encapsulate operation. - * - * See Section 4.1 Encap() and AuthEncap() - * - * Params: - * ctx A context object holding the recipients public key and the - * optional senders auth private key. - * enc A buffer to return the senders ephemeral public key. - * Setting this to NULL allows the enclen and secretlen to return - * values, without calculating the secret. - * enclen Passes in the max size of the enc buffer and returns the - * encoded public key length. - * secret A buffer to return the calculated shared secret. - * secretlen Passes in the max size of the secret buffer and returns the - * secret length. - * Returns: 1 on success or 0 otherwise. - */ -static int dhkem_encap(PROV_ECX_CTX *ctx, - unsigned char *enc, size_t *enclen, - unsigned char *secret, size_t *secretlen) -{ - int ret = 0; - ECX_KEY *sender_ephemkey = NULL; - unsigned char *sender_ephempub, *recipient_pub; - const OSSL_HPKE_KEM_INFO *info = ctx->info; - - if (enc == NULL) { - if (enclen == NULL && secretlen == NULL) - return 0; - if (enclen != NULL) - *enclen = info->Nenc; - if (secretlen != NULL) - *secretlen = info->Nsecret; - return 1; - } - - if (*secretlen < info->Nsecret) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_BAD_LENGTH, "*secretlen too small"); - return 0; - } - if (*enclen < info->Nenc) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_BAD_LENGTH, "*enclen too small"); - return 0; - } - - /* Create an ephemeral key */ - sender_ephemkey = derivekey(ctx, ctx->ikm, ctx->ikmlen); - - sender_ephempub = ecx_pubkey(sender_ephemkey); - recipient_pub = ecx_pubkey(ctx->recipient_key); - if (sender_ephempub == NULL || recipient_pub == NULL) - goto err; - - if (!derive_secret(ctx, secret, - sender_ephemkey, ctx->recipient_key, - ctx->sender_authkey, ctx->recipient_key, - sender_ephempub, recipient_pub)) - goto err; - - /* Return the public part of the ephemeral key */ - memcpy(enc, sender_ephempub, info->Nenc); - *enclen = info->Nenc; - *secretlen = info->Nsecret; - ret = 1; -err: - ossl_ecx_key_free(sender_ephemkey); - return ret; -} - -/* - * Do a DHKEM decapsulate operation. - * See Section 4.1 Decap() and Auth Decap() - * - * Params: - * ctx A context object holding the recipients private key and the - * optional senders auth public key. - * secret A buffer to return the calculated shared secret. Setting this to - * NULL can be used to return the secretlen. - * secretlen Passes in the max size of the secret buffer and returns the - * secret length. - * enc A buffer containing the senders ephemeral public key that was returned - * from dhkem_encap(). - * enclen The length in bytes of enc. - * Returns: 1 If the shared secret is returned or 0 on error. - */ -static int dhkem_decap(PROV_ECX_CTX *ctx, - unsigned char *secret, size_t *secretlen, - const unsigned char *enc, size_t enclen) -{ - int ret = 0; - ECX_KEY *recipient_privkey = ctx->recipient_key; - ECX_KEY *sender_ephempubkey = NULL; - const OSSL_HPKE_KEM_INFO *info = ctx->info; - unsigned char *recipient_pub; - - if (secret == NULL) { - *secretlen = info->Nsecret; - return 1; - } - if (*secretlen < info->Nsecret) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_BAD_LENGTH, "*secretlen too small"); - return 0; - } - if (enclen != info->Nenc) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_KEY, "Invalid enc public key"); - return 0; - } - - /* Get the public part of the ephemeral key created by encap */ - sender_ephempubkey = ecxkey_pubfromdata(ctx, enc, enclen); - if (sender_ephempubkey == NULL) - goto err; - - recipient_pub = ecx_pubkey(recipient_privkey); - if (recipient_pub == NULL) - goto err; - - if (!derive_secret(ctx, secret, - ctx->recipient_key, sender_ephempubkey, - ctx->recipient_key, ctx->sender_authkey, - enc, recipient_pub)) - goto err; - - *secretlen = info->Nsecret; - ret = 1; -err: - ossl_ecx_key_free(sender_ephempubkey); - return ret; -} - -static int ecxkem_encapsulate(void *vctx, unsigned char *out, size_t *outlen, - unsigned char *secret, size_t *secretlen) -{ - PROV_ECX_CTX *ctx = (PROV_ECX_CTX *)vctx; - - switch (ctx->mode) { - case KEM_MODE_DHKEM: - return dhkem_encap(ctx, out, outlen, secret, secretlen); - default: - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MODE); - return -2; - } -} - -static int ecxkem_decapsulate(void *vctx, unsigned char *out, size_t *outlen, - const unsigned char *in, size_t inlen) -{ - PROV_ECX_CTX *ctx = (PROV_ECX_CTX *)vctx; - - switch (ctx->mode) { - case KEM_MODE_DHKEM: - return dhkem_decap(vctx, out, outlen, in, inlen); - default: - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MODE); - return -2; - } -} - -const OSSL_DISPATCH ossl_ecx_asym_kem_functions[] = { - { OSSL_FUNC_KEM_NEWCTX, (void (*)(void))ecxkem_newctx }, - { OSSL_FUNC_KEM_ENCAPSULATE_INIT, - (void (*)(void))ecxkem_encapsulate_init }, - { OSSL_FUNC_KEM_ENCAPSULATE, (void (*)(void))ecxkem_encapsulate }, - { OSSL_FUNC_KEM_DECAPSULATE_INIT, - (void (*)(void))ecxkem_decapsulate_init }, - { OSSL_FUNC_KEM_DECAPSULATE, (void (*)(void))ecxkem_decapsulate }, - { OSSL_FUNC_KEM_FREECTX, (void (*)(void))ecxkem_freectx }, - { OSSL_FUNC_KEM_SET_CTX_PARAMS, - (void (*)(void))ecxkem_set_ctx_params }, - { OSSL_FUNC_KEM_SETTABLE_CTX_PARAMS, - (void (*)(void))ecxkem_settable_ctx_params }, - { OSSL_FUNC_KEM_AUTH_ENCAPSULATE_INIT, - (void (*)(void))ecxkem_auth_encapsulate_init }, - { OSSL_FUNC_KEM_AUTH_DECAPSULATE_INIT, - (void (*)(void))ecxkem_auth_decapsulate_init }, - OSSL_DISPATCH_END -}; diff --git a/openssl/src/providers/implementations/kem/kem_util.c b/openssl/src/providers/implementations/kem/kem_util.c deleted file mode 100644 index 1fd52e1c2..000000000 --- a/openssl/src/providers/implementations/kem/kem_util.c +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include /* for memcpy() */ -#include -#include -#include "eckem.h" - -typedef struct { - unsigned int id; - const char *mode; -} KEM_MODE; - -static const KEM_MODE eckem_modename_id_map[] = { - { KEM_MODE_DHKEM, OSSL_KEM_PARAM_OPERATION_DHKEM }, - { 0, NULL } -}; - -int ossl_eckem_modename2id(const char *name) -{ - size_t i; - - if (name == NULL) - return KEM_MODE_UNDEFINED; - - for (i = 0; eckem_modename_id_map[i].mode != NULL; ++i) { - if (OPENSSL_strcasecmp(name, eckem_modename_id_map[i].mode) == 0) - return eckem_modename_id_map[i].id; - } - return KEM_MODE_UNDEFINED; -} diff --git a/openssl/src/providers/implementations/kem/rsa_kem.c b/openssl/src/providers/implementations/kem/rsa_kem.c index ff22ddffc..882cf1612 100644 --- a/openssl/src/providers/implementations/kem/rsa_kem.c +++ b/openssl/src/providers/implementations/kem/rsa_kem.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,7 +23,6 @@ #include #include "crypto/rsa.h" #include -#include "internal/nelem.h" #include "prov/provider_ctx.h" #include "prov/implementations.h" #include "prov/securitycheck.h" @@ -361,5 +360,5 @@ const OSSL_DISPATCH ossl_rsa_asym_kem_functions[] = { (void (*)(void))rsakem_set_ctx_params }, { OSSL_FUNC_KEM_SETTABLE_CTX_PARAMS, (void (*)(void))rsakem_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/keymgmt/dh_kmgmt.c b/openssl/src/providers/implementations/keymgmt/dh_kmgmt.c index 82c3093b1..88fddf51d 100644 --- a/openssl/src/providers/implementations/keymgmt/dh_kmgmt.c +++ b/openssl/src/providers/implementations/keymgmt/dh_kmgmt.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,6 @@ * internal use. */ #include "internal/deprecated.h" -#include "internal/common.h" #include /* strcmp */ #include @@ -199,8 +198,8 @@ static int dh_import(void *keydata, int selection, const OSSL_PARAM params[]) if ((selection & DH_POSSIBLE_SELECTIONS) == 0) return 0; - /* a key without parameters is meaningless */ - ok = ok && ossl_dh_params_fromdata(dh, params); + if ((selection & OSSL_KEYMGMT_SELECT_ALL_PARAMETERS) != 0) + ok = ok && ossl_dh_params_fromdata(dh, params); if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) { int include_private = @@ -223,9 +222,6 @@ static int dh_export(void *keydata, int selection, OSSL_CALLBACK *param_cb, if (!ossl_prov_is_running() || dh == NULL) return 0; - if ((selection & DH_POSSIBLE_SELECTIONS) == 0) - return 0; - tmpl = OSSL_PARAM_BLD_new(); if (tmpl == NULL) return 0; @@ -240,11 +236,11 @@ static int dh_export(void *keydata, int selection, OSSL_CALLBACK *param_cb, ok = ok && ossl_dh_key_todata(dh, tmpl, NULL, include_private); } - if (!ok || (params = OSSL_PARAM_BLD_to_param(tmpl)) == NULL) { + if (!ok + || (params = OSSL_PARAM_BLD_to_param(tmpl)) == NULL) { ok = 0; goto err; } - ok = param_cb(params, cbarg); OSSL_PARAM_free(params); err: @@ -393,7 +389,7 @@ static int dh_validate_public(const DH *dh, int checktype) && ossl_dh_is_named_safe_prime_group(dh)) return ossl_dh_check_pub_key_partial(dh, pub_key, &res); - return DH_check_pub_key_ex(dh, pub_key); + return DH_check_pub_key(dh, pub_key, &res); } static int dh_validate_private(const DH *dh) @@ -404,7 +400,7 @@ static int dh_validate_private(const DH *dh) DH_get0_key(dh, NULL, &priv_key); if (priv_key == NULL) return 0; - return ossl_dh_check_priv_key(dh, priv_key, &status); + return ossl_dh_check_priv_key(dh, priv_key, &status);; } static int dh_validate(const void *keydata, int selection, int checktype) @@ -525,7 +521,6 @@ static int dh_gen_common_set_params(void *genctx, const OSSL_PARAM params[]) { struct dh_gen_ctx *gctx = genctx; const OSSL_PARAM *p; - int gen_type = -1; if (gctx == NULL) return 0; @@ -535,13 +530,11 @@ static int dh_gen_common_set_params(void *genctx, const OSSL_PARAM params[]) p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_FFC_TYPE); if (p != NULL) { if (p->data_type != OSSL_PARAM_UTF8_STRING - || ((gen_type = + || ((gctx->gen_type = dh_gen_type_name2id_w_default(p->data, gctx->dh_type)) == -1)) { ERR_raise(ERR_LIB_PROV, ERR_R_PASSED_INVALID_ARGUMENT); return 0; } - if (gen_type != -1) - gctx->gen_type = gen_type; } p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_GROUP_NAME); if (p != NULL) { @@ -703,26 +696,13 @@ static void *dh_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) return NULL; /* - * If a group name is selected then the type is group regardless of what + * If a group name is selected then the type is group regardless of what the * the user selected. This overrides rather than errors for backwards * compatibility. */ if (gctx->group_nid != NID_undef) gctx->gen_type = DH_PARAMGEN_TYPE_GROUP; - /* - * Do a bounds check on context gen_type. Must be in range: - * DH_PARAMGEN_TYPE_GENERATOR <= gen_type <= DH_PARAMGEN_TYPE_GROUP - * Noted here as this needs to be adjusted if a new group type is - * added. - */ - if (!ossl_assert((gctx->gen_type >= DH_PARAMGEN_TYPE_GENERATOR) - && (gctx->gen_type <= DH_PARAMGEN_TYPE_GROUP))) { - ERR_raise_data(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR, - "gen_type set to unsupported value %d", gctx->gen_type); - return NULL; - } - /* For parameter generation - If there is a group name just create it */ if (gctx->gen_type == DH_PARAMGEN_TYPE_GROUP && gctx->ffc_params == NULL) { @@ -755,8 +735,10 @@ static void *dh_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) } else if (gctx->hindex != 0) { ossl_ffc_params_set_h(ffc, gctx->hindex); } - if (gctx->mdname != NULL) - ossl_ffc_set_digest(ffc, gctx->mdname, gctx->mdprops); + if (gctx->mdname != NULL) { + if (!ossl_ffc_set_digest(ffc, gctx->mdname, gctx->mdprops)) + goto end; + } gctx->cb = osslcb; gctx->cbarg = cbarg; gencb = BN_GENCB_new(); @@ -861,7 +843,7 @@ const OSSL_DISPATCH ossl_dh_keymgmt_functions[] = { { OSSL_FUNC_KEYMGMT_EXPORT, (void (*)(void))dh_export }, { OSSL_FUNC_KEYMGMT_EXPORT_TYPES, (void (*)(void))dh_export_types }, { OSSL_FUNC_KEYMGMT_DUP, (void (*)(void))dh_dup }, - OSSL_DISPATCH_END + { 0, NULL } }; /* For any DH key, we use the "DH" algorithms regardless of sub-type. */ @@ -895,5 +877,5 @@ const OSSL_DISPATCH ossl_dhx_keymgmt_functions[] = { { OSSL_FUNC_KEYMGMT_QUERY_OPERATION_NAME, (void (*)(void))dhx_query_operation_name }, { OSSL_FUNC_KEYMGMT_DUP, (void (*)(void))dh_dup }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/keymgmt/dsa_kmgmt.c b/openssl/src/providers/implementations/keymgmt/dsa_kmgmt.c index 88a2feda5..2ab69f5f3 100644 --- a/openssl/src/providers/implementations/keymgmt/dsa_kmgmt.c +++ b/openssl/src/providers/implementations/keymgmt/dsa_kmgmt.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -199,9 +199,8 @@ static int dsa_import(void *keydata, int selection, const OSSL_PARAM params[]) if ((selection & DSA_POSSIBLE_SELECTIONS) == 0) return 0; - /* a key without parameters is meaningless */ - ok = ok && ossl_dsa_ffc_params_fromdata(dsa, params); - + if ((selection & OSSL_KEYMGMT_SELECT_ALL_PARAMETERS) != 0) + ok = ok && ossl_dsa_ffc_params_fromdata(dsa, params); if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) { int include_private = selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY ? 1 : 0; @@ -223,9 +222,6 @@ static int dsa_export(void *keydata, int selection, OSSL_CALLBACK *param_cb, if (!ossl_prov_is_running() || dsa == NULL) return 0; - if ((selection & DSA_POSSIBLE_SELECTIONS) == 0) - return 0; - tmpl = OSSL_PARAM_BLD_new(); if (tmpl == NULL) return 0; @@ -239,10 +235,9 @@ static int dsa_export(void *keydata, int selection, OSSL_CALLBACK *param_cb, ok = ok && dsa_key_todata(dsa, tmpl, NULL, include_private); } - if (!ok || (params = OSSL_PARAM_BLD_to_param(tmpl)) == NULL) { - ok = 0; + if (!ok + || (params = OSSL_PARAM_BLD_to_param(tmpl)) == NULL) goto err; - } ok = param_cb(params, cbarg); OSSL_PARAM_free(params); @@ -462,7 +457,6 @@ static int dsa_gen_set_params(void *genctx, const OSSL_PARAM params[]) { struct dsa_gen_ctx *gctx = genctx; const OSSL_PARAM *p; - int gen_type = -1; if (gctx == NULL) return 0; @@ -473,18 +467,10 @@ static int dsa_gen_set_params(void *genctx, const OSSL_PARAM params[]) p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_FFC_TYPE); if (p != NULL) { if (p->data_type != OSSL_PARAM_UTF8_STRING - || ((gen_type = dsa_gen_type_name2id(p->data)) == -1)) { + || ((gctx->gen_type = dsa_gen_type_name2id(p->data)) == -1)) { ERR_raise(ERR_LIB_PROV, ERR_R_PASSED_INVALID_ARGUMENT); return 0; } - - /* - * Only assign context gen_type if it was set by dsa_gen_type_name2id - * must be in range: - * DSA_PARAMGEN_TYPE_FIPS_186_4 <= gen_type <= DSA_PARAMGEN_TYPE_FIPS_DEFAULT - */ - if (gen_type != -1) - gctx->gen_type = gen_type; } p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_FFC_GINDEX); if (p != NULL @@ -577,19 +563,6 @@ static void *dsa_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) gctx->gen_type = (gctx->pbits >= 2048 ? DSA_PARAMGEN_TYPE_FIPS_186_4 : DSA_PARAMGEN_TYPE_FIPS_186_2); - /* - * Do a bounds check on context gen_type. Must be in range: - * DSA_PARAMGEN_TYPE_FIPS_186_4 <= gen_type <= DSA_PARAMGEN_TYPE_FIPS_DEFAULT - * Noted here as this needs to be adjusted if a new type is - * added. - */ - if (!ossl_assert((gctx->gen_type >= DSA_PARAMGEN_TYPE_FIPS_186_4) - && (gctx->gen_type <= DSA_PARAMGEN_TYPE_FIPS_DEFAULT))) { - ERR_raise_data(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR, - "gen_type set to unsupported value %d", gctx->gen_type); - return NULL; - } - gctx->cb = osslcb; gctx->cbarg = cbarg; gencb = BN_GENCB_new(); @@ -612,9 +585,10 @@ static void *dsa_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) } else if (gctx->hindex != 0) { ossl_ffc_params_set_h(ffc, gctx->hindex); } - if (gctx->mdname != NULL) - ossl_ffc_set_digest(ffc, gctx->mdname, gctx->mdprops); - + if (gctx->mdname != NULL) { + if (!ossl_ffc_set_digest(ffc, gctx->mdname, gctx->mdprops)) + goto end; + } if ((gctx->selection & OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS) != 0) { if (ossl_dsa_generate_ffc_parameters(dsa, gctx->gen_type, @@ -697,5 +671,5 @@ const OSSL_DISPATCH ossl_dsa_keymgmt_functions[] = { { OSSL_FUNC_KEYMGMT_EXPORT, (void (*)(void))dsa_export }, { OSSL_FUNC_KEYMGMT_EXPORT_TYPES, (void (*)(void))dsa_export_types }, { OSSL_FUNC_KEYMGMT_DUP, (void (*)(void))dsa_dup }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/keymgmt/ec_kmgmt.c b/openssl/src/providers/implementations/keymgmt/ec_kmgmt.c index 939093539..17eeb4ffc 100644 --- a/openssl/src/providers/implementations/keymgmt/ec_kmgmt.c +++ b/openssl/src/providers/implementations/keymgmt/ec_kmgmt.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -92,6 +92,8 @@ static const char *sm2_query_operation_name(int operation_id) { switch (operation_id) { + case OSSL_OP_KEYEXCH: + return "SM2DH"; case OSSL_OP_SIGNATURE: return "SM2"; } @@ -147,10 +149,8 @@ int key_to_params(const EC_KEY *eckey, OSSL_PARAM_BLD *tmpl, if (p != NULL || tmpl != NULL) { /* convert pub_point to a octet string according to the SECG standard */ - point_conversion_form_t format = EC_KEY_get_conv_form(eckey); - if ((pub_key_len = EC_POINT_point2buf(ecg, pub_point, - format, + POINT_CONVERSION_COMPRESSED, pub_key, bnctx)) == 0 || !ossl_param_build_set_octet_string(tmpl, p, OSSL_PKEY_PARAM_PUB_KEY, @@ -158,16 +158,10 @@ int key_to_params(const EC_KEY *eckey, OSSL_PARAM_BLD *tmpl, goto err; } if (px != NULL || py != NULL) { - if (px != NULL) { + if (px != NULL) x = BN_CTX_get(bnctx); - if (x == NULL) - goto err; - } - if (py != NULL) { + if (py != NULL) y = BN_CTX_get(bnctx); - if (y == NULL) - goto err; - } if (!EC_POINT_get_affine_coordinates(ecg, pub_point, x, y, bnctx)) goto err; @@ -222,7 +216,7 @@ int key_to_params(const EC_KEY *eckey, OSSL_PARAM_BLD *tmpl, ecbits = EC_GROUP_order_bits(ecg); if (ecbits <= 0) goto err; - sz = (ecbits + 7) / 8; + sz = (ecbits + 7 ) / 8; if (!ossl_param_build_set_bn_pad(tmpl, params, OSSL_PKEY_PARAM_PRIV_KEY, @@ -504,14 +498,10 @@ int ec_export(void *keydata, int selection, OSSL_CALLBACK *param_cb, if ((selection & OSSL_KEYMGMT_SELECT_OTHER_PARAMETERS) != 0) ok = ok && otherparams_to_params(ec, tmpl, NULL); - if (!ok || (params = OSSL_PARAM_BLD_to_param(tmpl)) == NULL) { - ok = 0; - goto end; - } - - ok = param_cb(params, cbarg); - OSSL_PARAM_free(params); + if (ok && (params = OSSL_PARAM_BLD_to_param(tmpl)) != NULL) + ok = param_cb(params, cbarg); end: + OSSL_PARAM_free(params); OSSL_PARAM_BLD_free(tmpl); OPENSSL_free(pub_key); OPENSSL_free(genbuf); @@ -533,8 +523,7 @@ int ec_export(void *keydata, int selection, OSSL_CALLBACK *param_cb, OSSL_PARAM_octet_string(OSSL_PKEY_PARAM_EC_GENERATOR, NULL, 0), \ OSSL_PARAM_BN(OSSL_PKEY_PARAM_EC_ORDER, NULL, 0), \ OSSL_PARAM_BN(OSSL_PKEY_PARAM_EC_COFACTOR, NULL, 0), \ - OSSL_PARAM_octet_string(OSSL_PKEY_PARAM_EC_SEED, NULL, 0), \ - OSSL_PARAM_int(OSSL_PKEY_PARAM_EC_DECODED_FROM_EXPLICIT_PARAMS, NULL) + OSSL_PARAM_octet_string(OSSL_PKEY_PARAM_EC_SEED, NULL, 0) # define EC_IMEXPORTABLE_PUBLIC_KEY \ OSSL_PARAM_octet_string(OSSL_PKEY_PARAM_PUB_KEY, NULL, 0) @@ -645,10 +634,8 @@ int common_get_params(void *key, OSSL_PARAM params[], int sm2) BN_CTX *bnctx = NULL; ecg = EC_KEY_get0_group(eck); - if (ecg == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_NO_PARAMETERS_SET); + if (ecg == NULL) return 0; - } libctx = ossl_ec_key_get_libctx(eck); propq = ossl_ec_key_get0_propq(eck); @@ -737,15 +724,10 @@ int common_get_params(void *key, OSSL_PARAM params[], int sm2) } if ((p = OSSL_PARAM_locate(params, OSSL_PKEY_PARAM_ENCODED_PUBLIC_KEY)) != NULL) { - const EC_POINT *ecp = EC_KEY_get0_public_key(key); - - if (ecp == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_NOT_A_PUBLIC_KEY); - goto err; - } - p->return_size = EC_POINT_point2oct(ecg, ecp, + p->return_size = EC_POINT_point2oct(EC_KEY_get0_group(key), + EC_KEY_get0_public_key(key), POINT_CONVERSION_UNCOMPRESSED, - p->data, p->data_size, bnctx); + p->data, p->return_size, bnctx); if (p->return_size == 0) goto err; } @@ -954,7 +936,7 @@ int ec_validate(const void *keydata, int selection, int checktype) if ((flags & EC_FLAG_CHECK_NAMED_GROUP) != 0) ok = ok && EC_GROUP_check_named_curve(EC_KEY_get0_group(eck), - (flags & EC_FLAG_CHECK_NAMED_GROUP_NIST) != 0, ctx) > 0; + (flags & EC_FLAG_CHECK_NAMED_GROUP_NIST) != 0, ctx); else ok = ok && EC_GROUP_check(EC_KEY_get0_group(eck), ctx); } @@ -989,8 +971,6 @@ struct ec_gen_ctx { int selection; int ecdh_mode; EC_GROUP *gen_group; - unsigned char *dhkem_ikm; - size_t dhkem_ikmlen; }; static void *ec_gen_init(void *provctx, int selection, @@ -1006,10 +986,10 @@ static void *ec_gen_init(void *provctx, int selection, gctx->libctx = libctx; gctx->selection = selection; gctx->ecdh_mode = 0; - if (!ec_gen_set_params(gctx, params)) { - OPENSSL_free(gctx); - gctx = NULL; - } + } + if (!ec_gen_set_params(gctx, params)) { + OPENSSL_free(gctx); + gctx = NULL; } return gctx; } @@ -1026,6 +1006,7 @@ static void *sm2_gen_init(void *provctx, int selection, return gctx; if ((gctx->group_name = OPENSSL_strdup("sm2")) != NULL) return gctx; + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); ec_gen_cleanup(gctx); } return NULL; @@ -1123,9 +1104,6 @@ static int ec_gen_set_params(void *genctx, const OSSL_PARAM params[]) COPY_OCTET_PARAM(params, OSSL_PKEY_PARAM_EC_GENERATOR, gctx->gen, gctx->gen_len); - COPY_OCTET_PARAM(params, OSSL_PKEY_PARAM_DHKEM_IKM, gctx->dhkem_ikm, - gctx->dhkem_ikmlen); - ret = 1; err: EC_GROUP_free(group); @@ -1225,7 +1203,6 @@ static const OSSL_PARAM *ec_gen_settable_params(ossl_unused void *genctx, OSSL_PARAM_BN(OSSL_PKEY_PARAM_EC_ORDER, NULL, 0), OSSL_PARAM_BN(OSSL_PKEY_PARAM_EC_COFACTOR, NULL, 0), OSSL_PARAM_octet_string(OSSL_PKEY_PARAM_EC_SEED, NULL, 0), - OSSL_PARAM_octet_string(OSSL_PKEY_PARAM_DHKEM_IKM, NULL, 0), OSSL_PARAM_END }; @@ -1279,22 +1256,14 @@ static void *ec_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) ret = ec_gen_assign_group(ec, gctx->gen_group); /* Whether you want it or not, you get a keypair, not just one half */ - if ((gctx->selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) { -#ifndef FIPS_MODULE - if (gctx->dhkem_ikm != NULL && gctx->dhkem_ikmlen != 0) - ret = ret && ossl_ec_generate_key_dhkem(ec, gctx->dhkem_ikm, - gctx->dhkem_ikmlen); - else -#endif - ret = ret && EC_KEY_generate_key(ec); - } + if ((gctx->selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) + ret = ret && EC_KEY_generate_key(ec); if (gctx->ecdh_mode != -1) ret = ret && ossl_ec_set_ecdh_cofactor_mode(ec, gctx->ecdh_mode); if (gctx->group_check != NULL) - ret = ret && ossl_ec_set_check_group_type_from_name(ec, - gctx->group_check); + ret = ret && ossl_ec_set_check_group_type_from_name(ec, gctx->group_check); if (ret) return ec; err: @@ -1362,7 +1331,6 @@ static void ec_gen_cleanup(void *genctx) if (gctx == NULL) return; - OPENSSL_clear_free(gctx->dhkem_ikm, gctx->dhkem_ikmlen); EC_GROUP_free(gctx->gen_group); BN_free(gctx->p); BN_free(gctx->a); @@ -1444,7 +1412,7 @@ const OSSL_DISPATCH ossl_ec_keymgmt_functions[] = { { OSSL_FUNC_KEYMGMT_QUERY_OPERATION_NAME, (void (*)(void))ec_query_operation_name }, { OSSL_FUNC_KEYMGMT_DUP, (void (*)(void))ec_dup }, - OSSL_DISPATCH_END + { 0, NULL } }; #ifndef FIPS_MODULE @@ -1475,7 +1443,7 @@ const OSSL_DISPATCH ossl_sm2_keymgmt_functions[] = { { OSSL_FUNC_KEYMGMT_QUERY_OPERATION_NAME, (void (*)(void))sm2_query_operation_name }, { OSSL_FUNC_KEYMGMT_DUP, (void (*)(void))ec_dup }, - OSSL_DISPATCH_END + { 0, NULL } }; # endif #endif diff --git a/openssl/src/providers/implementations/keymgmt/ecx_kmgmt.c b/openssl/src/providers/implementations/keymgmt/ecx_kmgmt.c index ae11fd4bc..2a7f867aa 100644 --- a/openssl/src/providers/implementations/keymgmt/ecx_kmgmt.c +++ b/openssl/src/providers/implementations/keymgmt/ecx_kmgmt.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -16,14 +16,12 @@ #include #include #include -#include #include "internal/param_build_set.h" #include #include "crypto/ecx.h" #include "prov/implementations.h" #include "prov/providercommon.h" #include "prov/provider_ctx.h" -#include "prov/ecx.h" #ifdef S390X_EC_ASM # include "s390x_arch.h" # include /* For SHA512_DIGEST_LENGTH */ @@ -80,8 +78,6 @@ struct ecx_gen_ctx { char *propq; ECX_KEY_TYPE type; int selection; - unsigned char *dhkem_ikm; - size_t dhkem_ikmlen; }; #ifdef S390X_EC_ASM @@ -242,9 +238,6 @@ static int ecx_export(void *keydata, int selection, OSSL_CALLBACK *param_cb, if (!ossl_prov_is_running() || key == NULL) return 0; - if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) == 0) - return 0; - tmpl = OSSL_PARAM_BLD_new(); if (tmpl == NULL) return 0; @@ -388,8 +381,10 @@ static int set_property_query(ECX_KEY *ecxkey, const char *propq) ecxkey->propq = NULL; if (propq != NULL) { ecxkey->propq = OPENSSL_strdup(propq); - if (ecxkey->propq == NULL) + if (ecxkey->propq == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } } return 1; } @@ -563,16 +558,6 @@ static int ecx_gen_set_params(void *genctx, const OSSL_PARAM params[]) if (gctx->propq == NULL) return 0; } - p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_DHKEM_IKM); - if (p != NULL) { - if (p->data_size != 0 && p->data != NULL) { - OPENSSL_free(gctx->dhkem_ikm); - gctx->dhkem_ikm = NULL; - if (!OSSL_PARAM_get_octet_string(p, (void **)&gctx->dhkem_ikm, 0, - &gctx->dhkem_ikmlen)) - return 0; - } - } return 1; } @@ -583,80 +568,11 @@ static const OSSL_PARAM *ecx_gen_settable_params(ossl_unused void *genctx, static OSSL_PARAM settable[] = { OSSL_PARAM_utf8_string(OSSL_PKEY_PARAM_GROUP_NAME, NULL, 0), OSSL_PARAM_utf8_string(OSSL_KDF_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_octet_string(OSSL_PKEY_PARAM_DHKEM_IKM, NULL, 0), OSSL_PARAM_END }; return settable; } -#ifdef FIPS_MODULE -/* - * Refer: FIPS 140-3 IG 10.3.A Additional Comment 1 - * Perform a pairwise test for EDDSA by signing and verifying signature. - * - * The parameter `self_test` is used to indicate whether to create OSSL_SELF_TEST - * instance. - */ -static int ecd_fips140_pairwise_test(const ECX_KEY *ecx, int type, int self_test) -{ - int ret = 0; - OSSL_SELF_TEST *st = NULL; - OSSL_CALLBACK *cb = NULL; - void *cbarg = NULL; - - unsigned char msg[16] = {0}; - size_t msg_len = sizeof(msg); - unsigned char sig[ED448_SIGSIZE] = {0}; - - int is_ed25519 = (type == ECX_KEY_TYPE_ED25519) ? 1 : 0; - int operation_result = 0; - - /* - * The functions `OSSL_SELF_TEST_*` will return directly if parameter `st` - * is NULL. - */ - if (self_test) { - OSSL_SELF_TEST_get_callback(ecx->libctx, &cb, &cbarg); - - st = OSSL_SELF_TEST_new(cb, cbarg); - if (st == NULL) - return 0; - } - - OSSL_SELF_TEST_onbegin(st, OSSL_SELF_TEST_TYPE_PCT, - OSSL_SELF_TEST_DESC_PCT_EDDSA); - - if (is_ed25519) - operation_result = ossl_ed25519_sign(sig, msg, msg_len, ecx->pubkey, - ecx->privkey, 0, 0, 0, NULL, 0, - ecx->libctx, ecx->propq); - else - operation_result = ossl_ed448_sign(ecx->libctx, sig, msg, msg_len, - ecx->pubkey, ecx->privkey, NULL, 0, - 0, ecx->propq); - if (operation_result != 1) - goto err; - - OSSL_SELF_TEST_oncorrupt_byte(st, sig); - - if (is_ed25519) - operation_result = ossl_ed25519_verify(msg, msg_len, sig, ecx->pubkey, - 0, 0, 0, NULL, 0, ecx->libctx, - ecx->propq); - else - operation_result = ossl_ed448_verify(ecx->libctx, msg, msg_len, sig, - ecx->pubkey, NULL, 0, 0, ecx->propq); - if (operation_result != 1) - goto err; - - ret = 1; -err: - OSSL_SELF_TEST_onend(st, ret); - OSSL_SELF_TEST_free(st); - return ret; -} -#endif - static void *ecx_gen(struct ecx_gen_ctx *gctx) { ECX_KEY *key; @@ -666,7 +582,7 @@ static void *ecx_gen(struct ecx_gen_ctx *gctx) return NULL; if ((key = ossl_ecx_key_new(gctx->libctx, gctx->type, 0, gctx->propq)) == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; } @@ -675,24 +591,11 @@ static void *ecx_gen(struct ecx_gen_ctx *gctx) return key; if ((privkey = ossl_ecx_key_allocate_privkey(key)) == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } -#ifndef FIPS_MODULE - if (gctx->dhkem_ikm != NULL && gctx->dhkem_ikmlen != 0) { - if (gctx->type == ECX_KEY_TYPE_ED25519 - || gctx->type == ECX_KEY_TYPE_ED448) - goto err; - if (!ossl_ecx_dhkem_derive_private(key, privkey, - gctx->dhkem_ikm, gctx->dhkem_ikmlen)) - goto err; - } else -#endif - { - if (RAND_priv_bytes_ex(gctx->libctx, privkey, key->keylen, 0) <= 0) - goto err; - } - + if (RAND_priv_bytes_ex(gctx->libctx, privkey, key->keylen, 0) <= 0) + goto err; switch (gctx->type) { case ECX_KEY_TYPE_X25519: privkey[0] &= 248; @@ -753,7 +656,6 @@ static void *x448_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) static void *ed25519_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) { - ECX_KEY *key = NULL; struct ecx_gen_ctx *gctx = genctx; if (!ossl_prov_is_running()) @@ -763,31 +665,14 @@ static void *ed25519_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) if (OPENSSL_s390xcap_P.pcc[1] & S390X_CAPBIT(S390X_SCALAR_MULTIPLY_ED25519) && OPENSSL_s390xcap_P.kdsa[0] & S390X_CAPBIT(S390X_EDDSA_SIGN_ED25519) && OPENSSL_s390xcap_P.kdsa[0] - & S390X_CAPBIT(S390X_EDDSA_VERIFY_ED25519)) { - key = s390x_ecd_keygen25519(gctx); - } else -#endif - { - key = ecx_gen(gctx); - } - -#ifdef FIPS_MODULE - /* Exit if keygen failed OR we are doing parameter generation (blank key) */ - if (!key || ((gctx->selection & OSSL_KEYMGMT_SELECT_KEYPAIR) == 0)) - return key; - if (ecd_fips140_pairwise_test(key, ECX_KEY_TYPE_ED25519, 1) != 1) { - ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT); - ossl_ecx_key_free(key); - return NULL; - } + & S390X_CAPBIT(S390X_EDDSA_VERIFY_ED25519)) + return s390x_ecd_keygen25519(gctx); #endif - - return key; + return ecx_gen(gctx); } static void *ed448_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) { - ECX_KEY *key = NULL; struct ecx_gen_ctx *gctx = genctx; if (!ossl_prov_is_running()) @@ -796,33 +681,16 @@ static void *ed448_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg) #ifdef S390X_EC_ASM if (OPENSSL_s390xcap_P.pcc[1] & S390X_CAPBIT(S390X_SCALAR_MULTIPLY_ED448) && OPENSSL_s390xcap_P.kdsa[0] & S390X_CAPBIT(S390X_EDDSA_SIGN_ED448) - && OPENSSL_s390xcap_P.kdsa[0] & S390X_CAPBIT(S390X_EDDSA_VERIFY_ED448)) { - key = s390x_ecd_keygen448(gctx); - } else -#endif - { - key = ecx_gen(gctx); - } - -#ifdef FIPS_MODULE - /* Exit if keygen failed OR we are doing parameter generation (blank key) */ - if (!key || ((gctx->selection & OSSL_KEYMGMT_SELECT_KEYPAIR) == 0)) - return key; - if (ecd_fips140_pairwise_test(key, ECX_KEY_TYPE_ED448, 1) != 1) { - ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT); - ossl_ecx_key_free(key); - return NULL; - } + && OPENSSL_s390xcap_P.kdsa[0] & S390X_CAPBIT(S390X_EDDSA_VERIFY_ED448)) + return s390x_ecd_keygen448(gctx); #endif - - return key; + return ecx_gen(gctx); } static void ecx_gen_cleanup(void *genctx) { struct ecx_gen_ctx *gctx = genctx; - OPENSSL_clear_free(gctx->dhkem_ikm, gctx->dhkem_ikmlen); OPENSSL_free(gctx->propq); OPENSSL_free(gctx); } @@ -859,23 +727,6 @@ static int ecx_key_pairwise_check(const ECX_KEY *ecx, int type) case ECX_KEY_TYPE_X448: ossl_x448_public_from_private(pub, ecx->privkey); break; - default: - return 0; - } - return CRYPTO_memcmp(ecx->pubkey, pub, ecx->keylen) == 0; -} - -#ifdef FIPS_MODULE -static int ecd_key_pairwise_check(const ECX_KEY *ecx, int type) -{ - return ecd_fips140_pairwise_test(ecx, type, 0); -} -#else -static int ecd_key_pairwise_check(const ECX_KEY *ecx, int type) -{ - uint8_t pub[64]; - - switch (type) { case ECX_KEY_TYPE_ED25519: if (!ossl_ed25519_public_from_private(ecx->libctx, pub, ecx->privkey, ecx->propq)) @@ -891,7 +742,6 @@ static int ecd_key_pairwise_check(const ECX_KEY *ecx, int type) } return CRYPTO_memcmp(ecx->pubkey, pub, ecx->keylen) == 0; } -#endif static int ecx_validate(const void *keydata, int selection, int type, size_t keylen) { @@ -915,12 +765,7 @@ static int ecx_validate(const void *keydata, int selection, int type, size_t key if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) ok = ok && ecx->privkey != NULL; - if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != OSSL_KEYMGMT_SELECT_KEYPAIR) - return ok; - - if (type == ECX_KEY_TYPE_ED25519 || type == ECX_KEY_TYPE_ED448) - ok = ok && ecd_key_pairwise_check(ecx, type); - else + if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) == OSSL_KEYMGMT_SELECT_KEYPAIR) ok = ok && ecx_key_pairwise_check(ecx, type); return ok; @@ -969,7 +814,7 @@ static int ed448_validate(const void *keydata, int selection, int checktype) { OSSL_FUNC_KEYMGMT_GEN_CLEANUP, (void (*)(void))ecx_gen_cleanup }, \ { OSSL_FUNC_KEYMGMT_LOAD, (void (*)(void))ecx_load }, \ { OSSL_FUNC_KEYMGMT_DUP, (void (*)(void))ecx_dup }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; MAKE_KEYMGMT_FUNCTIONS(x25519) @@ -992,7 +837,7 @@ static void *s390x_ecx_keygen25519(struct ecx_gen_ctx *gctx) unsigned char *privkey = NULL, *pubkey; if (key == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -1004,23 +849,12 @@ static void *s390x_ecx_keygen25519(struct ecx_gen_ctx *gctx) privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } -#ifndef FIPS_MODULE - if (gctx->dhkem_ikm != NULL && gctx->dhkem_ikmlen != 0) { - if (gctx->type != ECX_KEY_TYPE_X25519) - goto err; - if (!ossl_ecx_dhkem_derive_private(key, privkey, - gctx->dhkem_ikm, gctx->dhkem_ikmlen)) - goto err; - } else -#endif - { - if (RAND_priv_bytes_ex(gctx->libctx, privkey, X25519_KEYLEN, 0) <= 0) - goto err; - } + if (RAND_priv_bytes_ex(gctx->libctx, privkey, X25519_KEYLEN, 0) <= 0) + goto err; privkey[0] &= 248; privkey[31] &= 127; @@ -1049,7 +883,7 @@ static void *s390x_ecx_keygen448(struct ecx_gen_ctx *gctx) unsigned char *privkey = NULL, *pubkey; if (key == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -1061,23 +895,12 @@ static void *s390x_ecx_keygen448(struct ecx_gen_ctx *gctx) privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } -#ifndef FIPS_MODULE - if (gctx->dhkem_ikm != NULL && gctx->dhkem_ikmlen != 0) { - if (gctx->type != ECX_KEY_TYPE_X448) - goto err; - if (!ossl_ecx_dhkem_derive_private(key, privkey, - gctx->dhkem_ikm, gctx->dhkem_ikmlen)) - goto err; - } else -#endif - { - if (RAND_priv_bytes_ex(gctx->libctx, privkey, X448_KEYLEN, 0) <= 0) - goto err; - } + if (RAND_priv_bytes_ex(gctx->libctx, privkey, X448_KEYLEN, 0) <= 0) + goto err; privkey[0] &= 252; privkey[55] |= 128; @@ -1112,7 +935,7 @@ static void *s390x_ecd_keygen25519(struct ecx_gen_ctx *gctx) int j; if (key == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -1124,7 +947,7 @@ static void *s390x_ecd_keygen25519(struct ecx_gen_ctx *gctx) privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -1179,7 +1002,7 @@ static void *s390x_ecd_keygen448(struct ecx_gen_ctx *gctx) EVP_MD *shake = NULL; if (key == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -1191,7 +1014,7 @@ static void *s390x_ecd_keygen448(struct ecx_gen_ctx *gctx) privkey = ossl_ecx_key_allocate_privkey(key); if (privkey == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EC_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } diff --git a/openssl/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c b/openssl/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c index deb496000..0b301c333 100644 --- a/openssl/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c +++ b/openssl/src/providers/implementations/keymgmt/kdf_legacy_kmgmt.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -37,11 +37,13 @@ KDF_DATA *ossl_kdf_data_new(void *provctx) if (kdfdata == NULL) return NULL; - if (!CRYPTO_NEW_REF(&kdfdata->refcnt, 1)) { + kdfdata->lock = CRYPTO_THREAD_lock_new(); + if (kdfdata->lock == NULL) { OPENSSL_free(kdfdata); return NULL; } kdfdata->libctx = PROV_LIBCTX_OF(provctx); + kdfdata->refcnt = 1; return kdfdata; } @@ -53,11 +55,11 @@ void ossl_kdf_data_free(KDF_DATA *kdfdata) if (kdfdata == NULL) return; - CRYPTO_DOWN_REF(&kdfdata->refcnt, &ref); + CRYPTO_DOWN_REF(&kdfdata->refcnt, &ref, kdfdata->lock); if (ref > 0) return; - CRYPTO_FREE_REF(&kdfdata->refcnt); + CRYPTO_THREAD_lock_free(kdfdata->lock); OPENSSL_free(kdfdata); } @@ -67,7 +69,7 @@ int ossl_kdf_data_up_ref(KDF_DATA *kdfdata) /* This is effectively doing a new operation on the KDF_DATA and should be * adequately guarded again modules' error states. However, both current - * calls here are guarded properly in exchange/kdf_exch.c. Thus, it + * calls here are guarded propery in exchange/kdf_exch.c. Thus, it * could be removed here. The concern is that something in the future * might call this function without adequate guards. It's a cheap call, * it seems best to leave it even though it is currently redundant. @@ -75,7 +77,7 @@ int ossl_kdf_data_up_ref(KDF_DATA *kdfdata) if (!ossl_prov_is_running()) return 0; - CRYPTO_UP_REF(&kdfdata->refcnt, &ref); + CRYPTO_UP_REF(&kdfdata->refcnt, &ref, kdfdata->lock); return 1; } @@ -98,5 +100,5 @@ const OSSL_DISPATCH ossl_kdf_keymgmt_functions[] = { { OSSL_FUNC_KEYMGMT_NEW, (void (*)(void))kdf_newdata }, { OSSL_FUNC_KEYMGMT_FREE, (void (*)(void))kdf_freedata }, { OSSL_FUNC_KEYMGMT_HAS, (void (*)(void))kdf_has }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c b/openssl/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c index 6f0f0313b..216c841da 100644 --- a/openssl/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c +++ b/openssl/src/providers/implementations/keymgmt/mac_legacy_kmgmt.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -72,11 +72,13 @@ MAC_KEY *ossl_mac_key_new(OSSL_LIB_CTX *libctx, int cmac) if (mackey == NULL) return NULL; - if (!CRYPTO_NEW_REF(&mackey->refcnt, 1)) { + mackey->lock = CRYPTO_THREAD_lock_new(); + if (mackey->lock == NULL) { OPENSSL_free(mackey); return NULL; } mackey->libctx = libctx; + mackey->refcnt = 1; mackey->cmac = cmac; return mackey; @@ -89,14 +91,14 @@ void ossl_mac_key_free(MAC_KEY *mackey) if (mackey == NULL) return; - CRYPTO_DOWN_REF(&mackey->refcnt, &ref); + CRYPTO_DOWN_REF(&mackey->refcnt, &ref, mackey->lock); if (ref > 0) return; OPENSSL_secure_clear_free(mackey->priv_key, mackey->priv_key_len); OPENSSL_free(mackey->properties); ossl_prov_cipher_reset(&mackey->cipher); - CRYPTO_FREE_REF(&mackey->refcnt); + CRYPTO_THREAD_lock_free(mackey->lock); OPENSSL_free(mackey); } @@ -106,7 +108,7 @@ int ossl_mac_key_up_ref(MAC_KEY *mackey) /* This is effectively doing a new operation on the MAC_KEY and should be * adequately guarded again modules' error states. However, both current - * calls here are guarded properly in signature/mac_legacy.c. Thus, it + * calls here are guarded propery in signature/mac_legacy.c. Thus, it * could be removed here. The concern is that something in the future * might call this function without adequate guards. It's a cheap call, * it seems best to leave it even though it is currently redundant. @@ -114,7 +116,7 @@ int ossl_mac_key_up_ref(MAC_KEY *mackey) if (!ossl_prov_is_running()) return 0; - CRYPTO_UP_REF(&mackey->refcnt, &ref); + CRYPTO_UP_REF(&mackey->refcnt, &ref, mackey->lock); return 1; } @@ -192,8 +194,10 @@ static int mac_key_fromdata(MAC_KEY *key, const OSSL_PARAM params[]) OPENSSL_secure_clear_free(key->priv_key, key->priv_key_len); /* allocate at least one byte to distinguish empty key from no key set */ key->priv_key = OPENSSL_secure_malloc(p->data_size > 0 ? p->data_size : 1); - if (key->priv_key == NULL) + if (key->priv_key == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(key->priv_key, p->data, p->data_size); key->priv_key_len = p->data_size; } @@ -206,8 +210,10 @@ static int mac_key_fromdata(MAC_KEY *key, const OSSL_PARAM params[]) } OPENSSL_free(key->properties); key->properties = OPENSSL_strdup(p->data); - if (key->properties == NULL) + if (key->properties == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } } if (key->cmac && !ossl_prov_cipher_load_from_params(&key->cipher, params, @@ -275,9 +281,6 @@ static int mac_export(void *keydata, int selection, OSSL_CALLBACK *param_cb, if (!ossl_prov_is_running() || key == NULL) return 0; - if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) == 0) - return 0; - tmpl = OSSL_PARAM_BLD_new(); if (tmpl == NULL) return 0; @@ -426,8 +429,10 @@ static int mac_gen_set_params(void *genctx, const OSSL_PARAM params[]) return 0; } gctx->priv_key = OPENSSL_secure_malloc(p->data_size ? p->data_size : 1); - if (gctx->priv_key == NULL) + if (gctx->priv_key == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(gctx->priv_key, p->data, p->data_size); gctx->priv_key_len = p->data_size; } @@ -481,7 +486,7 @@ static void *mac_gen(void *genctx, OSSL_CALLBACK *cb, void *cbarg) return NULL; if ((key = ossl_mac_key_new(gctx->libctx, 0)) == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_PROV_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; } @@ -543,7 +548,7 @@ const OSSL_DISPATCH ossl_mac_legacy_keymgmt_functions[] = { (void (*)(void))mac_gen_settable_params }, { OSSL_FUNC_KEYMGMT_GEN, (void (*)(void))mac_gen }, { OSSL_FUNC_KEYMGMT_GEN_CLEANUP, (void (*)(void))mac_gen_cleanup }, - OSSL_DISPATCH_END + { 0, NULL } }; const OSSL_DISPATCH ossl_cmac_legacy_keymgmt_functions[] = { @@ -565,6 +570,6 @@ const OSSL_DISPATCH ossl_cmac_legacy_keymgmt_functions[] = { (void (*)(void))cmac_gen_settable_params }, { OSSL_FUNC_KEYMGMT_GEN, (void (*)(void))mac_gen }, { OSSL_FUNC_KEYMGMT_GEN_CLEANUP, (void (*)(void))mac_gen_cleanup }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/keymgmt/rsa_kmgmt.c b/openssl/src/providers/implementations/keymgmt/rsa_kmgmt.c index c24cb8da8..1528e43ad 100644 --- a/openssl/src/providers/implementations/keymgmt/rsa_kmgmt.c +++ b/openssl/src/providers/implementations/keymgmt/rsa_kmgmt.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -124,9 +124,9 @@ static int rsa_has(const void *keydata, int selection) /* OSSL_KEYMGMT_SELECT_OTHER_PARAMETERS are always available even if empty */ if ((selection & OSSL_KEYMGMT_SELECT_KEYPAIR) != 0) - ok = ok && (RSA_get0_n(rsa) != NULL); - if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) ok = ok && (RSA_get0_e(rsa) != NULL); + if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0) + ok = ok && (RSA_get0_n(rsa) != NULL); if ((selection & OSSL_KEYMGMT_SELECT_PRIVATE_KEY) != 0) ok = ok && (RSA_get0_d(rsa) != NULL); return ok; @@ -229,10 +229,9 @@ static int rsa_export(void *keydata, int selection, ok = ok && ossl_rsa_todata(rsa, tmpl, NULL, include_private); } - if (!ok || (params = OSSL_PARAM_BLD_to_param(tmpl)) == NULL) { - ok = 0; + if (!ok + || (params = OSSL_PARAM_BLD_to_param(tmpl)) == NULL) goto err; - } ok = param_callback(params, cbarg); OSSL_PARAM_free(params); @@ -715,7 +714,7 @@ const OSSL_DISPATCH ossl_rsa_keymgmt_functions[] = { { OSSL_FUNC_KEYMGMT_EXPORT, (void (*)(void))rsa_export }, { OSSL_FUNC_KEYMGMT_EXPORT_TYPES, (void (*)(void))rsa_export_types }, { OSSL_FUNC_KEYMGMT_DUP, (void (*)(void))rsa_dup }, - OSSL_DISPATCH_END + { 0, NULL } }; const OSSL_DISPATCH ossl_rsapss_keymgmt_functions[] = { @@ -740,5 +739,5 @@ const OSSL_DISPATCH ossl_rsapss_keymgmt_functions[] = { { OSSL_FUNC_KEYMGMT_QUERY_OPERATION_NAME, (void (*)(void))rsa_query_operation_name }, { OSSL_FUNC_KEYMGMT_DUP, (void (*)(void))rsa_dup }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/macs/blake2_mac_impl.c b/openssl/src/providers/implementations/macs/blake2_mac_impl.c deleted file mode 100644 index ec22e607a..000000000 --- a/openssl/src/providers/implementations/macs/blake2_mac_impl.c +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include - -#include "prov/blake2.h" -#include "internal/cryptlib.h" -#include "prov/implementations.h" -#include "prov/providercommon.h" - -/* - * Forward declaration of everything implemented here. This is not strictly - * necessary for the compiler, but provides an assurance that the signatures - * of the functions in the dispatch table are correct. - */ -static OSSL_FUNC_mac_newctx_fn blake2_mac_new; -static OSSL_FUNC_mac_dupctx_fn blake2_mac_dup; -static OSSL_FUNC_mac_freectx_fn blake2_mac_free; -static OSSL_FUNC_mac_gettable_ctx_params_fn blake2_gettable_ctx_params; -static OSSL_FUNC_mac_get_ctx_params_fn blake2_get_ctx_params; -static OSSL_FUNC_mac_settable_ctx_params_fn blake2_mac_settable_ctx_params; -static OSSL_FUNC_mac_set_ctx_params_fn blake2_mac_set_ctx_params; -static OSSL_FUNC_mac_init_fn blake2_mac_init; -static OSSL_FUNC_mac_update_fn blake2_mac_update; -static OSSL_FUNC_mac_final_fn blake2_mac_final; - -struct blake2_mac_data_st { - BLAKE2_CTX ctx; - BLAKE2_PARAM params; - unsigned char key[BLAKE2_KEYBYTES]; -}; - -static void *blake2_mac_new(void *unused_provctx) -{ - struct blake2_mac_data_st *macctx; - - if (!ossl_prov_is_running()) - return NULL; - - macctx = OPENSSL_zalloc(sizeof(*macctx)); - if (macctx != NULL) { - BLAKE2_PARAM_INIT(&macctx->params); - /* ctx initialization is deferred to BLAKE2b_Init() */ - } - return macctx; -} - -static void *blake2_mac_dup(void *vsrc) -{ - struct blake2_mac_data_st *dst; - struct blake2_mac_data_st *src = vsrc; - - if (!ossl_prov_is_running()) - return NULL; - - dst = OPENSSL_zalloc(sizeof(*dst)); - if (dst == NULL) - return NULL; - - *dst = *src; - return dst; -} - -static void blake2_mac_free(void *vmacctx) -{ - struct blake2_mac_data_st *macctx = vmacctx; - - if (macctx != NULL) { - OPENSSL_cleanse(macctx->key, sizeof(macctx->key)); - OPENSSL_free(macctx); - } -} - -static size_t blake2_mac_size(void *vmacctx) -{ - struct blake2_mac_data_st *macctx = vmacctx; - - return macctx->params.digest_length; -} - -static int blake2_setkey(struct blake2_mac_data_st *macctx, - const unsigned char *key, size_t keylen) -{ - if (keylen > BLAKE2_KEYBYTES || keylen == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); - return 0; - } - memcpy(macctx->key, key, keylen); - /* Pad with zeroes at the end if required */ - if (keylen < BLAKE2_KEYBYTES) - memset(macctx->key + keylen, 0, BLAKE2_KEYBYTES - keylen); - BLAKE2_PARAM_SET_KEY_LENGTH(&macctx->params, (uint8_t)keylen); - return 1; -} - -static int blake2_mac_init(void *vmacctx, const unsigned char *key, - size_t keylen, const OSSL_PARAM params[]) -{ - struct blake2_mac_data_st *macctx = vmacctx; - - if (!ossl_prov_is_running() || !blake2_mac_set_ctx_params(macctx, params)) - return 0; - if (key != NULL) { - if (!blake2_setkey(macctx, key, keylen)) - return 0; - } else if (macctx->params.key_length == 0) { - /* Check key has been set */ - ERR_raise(ERR_LIB_PROV, PROV_R_NO_KEY_SET); - return 0; - } - return BLAKE2_INIT_KEY(&macctx->ctx, &macctx->params, macctx->key); -} - -static int blake2_mac_update(void *vmacctx, - const unsigned char *data, size_t datalen) -{ - struct blake2_mac_data_st *macctx = vmacctx; - - if (datalen == 0) - return 1; - - return BLAKE2_UPDATE(&macctx->ctx, data, datalen); -} - -static int blake2_mac_final(void *vmacctx, - unsigned char *out, size_t *outl, - size_t outsize) -{ - struct blake2_mac_data_st *macctx = vmacctx; - - if (!ossl_prov_is_running()) - return 0; - - *outl = blake2_mac_size(macctx); - return BLAKE2_FINAL(out, &macctx->ctx); -} - -static const OSSL_PARAM known_gettable_ctx_params[] = { - OSSL_PARAM_size_t(OSSL_MAC_PARAM_SIZE, NULL), - OSSL_PARAM_size_t(OSSL_MAC_PARAM_BLOCK_SIZE, NULL), - OSSL_PARAM_END -}; -static const OSSL_PARAM *blake2_gettable_ctx_params(ossl_unused void *ctx, - ossl_unused void *provctx) -{ - return known_gettable_ctx_params; -} - -static int blake2_get_ctx_params(void *vmacctx, OSSL_PARAM params[]) -{ - OSSL_PARAM *p; - - if ((p = OSSL_PARAM_locate(params, OSSL_MAC_PARAM_SIZE)) != NULL - && !OSSL_PARAM_set_size_t(p, blake2_mac_size(vmacctx))) - return 0; - - if ((p = OSSL_PARAM_locate(params, OSSL_MAC_PARAM_BLOCK_SIZE)) != NULL - && !OSSL_PARAM_set_size_t(p, BLAKE2_BLOCKBYTES)) - return 0; - - return 1; -} - -static const OSSL_PARAM known_settable_ctx_params[] = { - OSSL_PARAM_size_t(OSSL_MAC_PARAM_SIZE, NULL), - OSSL_PARAM_octet_string(OSSL_MAC_PARAM_KEY, NULL, 0), - OSSL_PARAM_octet_string(OSSL_MAC_PARAM_CUSTOM, NULL, 0), - OSSL_PARAM_octet_string(OSSL_MAC_PARAM_SALT, NULL, 0), - OSSL_PARAM_END -}; -static const OSSL_PARAM *blake2_mac_settable_ctx_params( - ossl_unused void *ctx, ossl_unused void *p_ctx) -{ - return known_settable_ctx_params; -} - -/* - * ALL parameters should be set before init(). - */ -static int blake2_mac_set_ctx_params(void *vmacctx, const OSSL_PARAM params[]) -{ - struct blake2_mac_data_st *macctx = vmacctx; - const OSSL_PARAM *p; - - if (params == NULL) - return 1; - - if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_SIZE)) != NULL) { - size_t size; - - if (!OSSL_PARAM_get_size_t(p, &size) - || size < 1 - || size > BLAKE2_OUTBYTES) { - ERR_raise(ERR_LIB_PROV, PROV_R_NOT_XOF_OR_INVALID_LENGTH); - return 0; - } - BLAKE2_PARAM_SET_DIGEST_LENGTH(&macctx->params, (uint8_t)size); - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_KEY)) != NULL - && !blake2_setkey(macctx, p->data, p->data_size)) - return 0; - - if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_CUSTOM)) - != NULL) { - /* - * The OSSL_PARAM API doesn't provide direct pointer use, so we - * must handle the OSSL_PARAM structure ourselves here - */ - if (p->data_size > BLAKE2_PERSONALBYTES) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_CUSTOM_LENGTH); - return 0; - } - BLAKE2_PARAM_SET_PERSONAL(&macctx->params, p->data, p->data_size); - } - - if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_SALT)) != NULL) { - /* - * The OSSL_PARAM API doesn't provide direct pointer use, so we - * must handle the OSSL_PARAM structure ourselves here as well - */ - if (p->data_size > BLAKE2_SALTBYTES) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH); - return 0; - } - BLAKE2_PARAM_SET_SALT(&macctx->params, p->data, p->data_size); - } - return 1; -} - -const OSSL_DISPATCH BLAKE2_FUNCTIONS[] = { - { OSSL_FUNC_MAC_NEWCTX, (void (*)(void))blake2_mac_new }, - { OSSL_FUNC_MAC_DUPCTX, (void (*)(void))blake2_mac_dup }, - { OSSL_FUNC_MAC_FREECTX, (void (*)(void))blake2_mac_free }, - { OSSL_FUNC_MAC_INIT, (void (*)(void))blake2_mac_init }, - { OSSL_FUNC_MAC_UPDATE, (void (*)(void))blake2_mac_update }, - { OSSL_FUNC_MAC_FINAL, (void (*)(void))blake2_mac_final }, - { OSSL_FUNC_MAC_GETTABLE_CTX_PARAMS, - (void (*)(void))blake2_gettable_ctx_params }, - { OSSL_FUNC_MAC_GET_CTX_PARAMS, (void (*)(void))blake2_get_ctx_params }, - { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, - (void (*)(void))blake2_mac_settable_ctx_params }, - { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))blake2_mac_set_ctx_params }, - OSSL_DISPATCH_END -}; diff --git a/openssl/src/providers/implementations/macs/blake2b_mac.c b/openssl/src/providers/implementations/macs/blake2b_mac.c deleted file mode 100644 index b445cbd57..000000000 --- a/openssl/src/providers/implementations/macs/blake2b_mac.c +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Constants */ -#define BLAKE2_CTX BLAKE2B_CTX -#define BLAKE2_PARAM BLAKE2B_PARAM -#define BLAKE2_KEYBYTES BLAKE2B_KEYBYTES -#define BLAKE2_OUTBYTES BLAKE2B_OUTBYTES -#define BLAKE2_PERSONALBYTES BLAKE2B_PERSONALBYTES -#define BLAKE2_SALTBYTES BLAKE2B_SALTBYTES -#define BLAKE2_BLOCKBYTES BLAKE2B_BLOCKBYTES - -/* Function names */ -#define BLAKE2_PARAM_INIT ossl_blake2b_param_init -#define BLAKE2_INIT_KEY ossl_blake2b_init_key -#define BLAKE2_UPDATE ossl_blake2b_update -#define BLAKE2_FINAL ossl_blake2b_final -#define BLAKE2_PARAM_SET_DIGEST_LENGTH ossl_blake2b_param_set_digest_length -#define BLAKE2_PARAM_SET_KEY_LENGTH ossl_blake2b_param_set_key_length -#define BLAKE2_PARAM_SET_PERSONAL ossl_blake2b_param_set_personal -#define BLAKE2_PARAM_SET_SALT ossl_blake2b_param_set_salt - -/* OSSL_DISPATCH symbol */ -#define BLAKE2_FUNCTIONS ossl_blake2bmac_functions - -#include "blake2_mac_impl.c" - diff --git a/openssl/src/providers/implementations/macs/blake2s_mac.c b/openssl/src/providers/implementations/macs/blake2s_mac.c deleted file mode 100644 index 6b3fa28bd..000000000 --- a/openssl/src/providers/implementations/macs/blake2s_mac.c +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -/* Constants */ -#define BLAKE2_CTX BLAKE2S_CTX -#define BLAKE2_PARAM BLAKE2S_PARAM -#define BLAKE2_KEYBYTES BLAKE2S_KEYBYTES -#define BLAKE2_OUTBYTES BLAKE2S_OUTBYTES -#define BLAKE2_PERSONALBYTES BLAKE2S_PERSONALBYTES -#define BLAKE2_SALTBYTES BLAKE2S_SALTBYTES -#define BLAKE2_BLOCKBYTES BLAKE2S_BLOCKBYTES - -/* Function names */ -#define BLAKE2_PARAM_INIT ossl_blake2s_param_init -#define BLAKE2_INIT_KEY ossl_blake2s_init_key -#define BLAKE2_UPDATE ossl_blake2s_update -#define BLAKE2_FINAL ossl_blake2s_final -#define BLAKE2_PARAM_SET_DIGEST_LENGTH ossl_blake2s_param_set_digest_length -#define BLAKE2_PARAM_SET_KEY_LENGTH ossl_blake2s_param_set_key_length -#define BLAKE2_PARAM_SET_PERSONAL ossl_blake2s_param_set_personal -#define BLAKE2_PARAM_SET_SALT ossl_blake2s_param_set_salt - -/* OSSL_DISPATCH symbol */ -#define BLAKE2_FUNCTIONS ossl_blake2smac_functions - -#include "blake2_mac_impl.c" diff --git a/openssl/src/providers/implementations/macs/cmac_prov.c b/openssl/src/providers/implementations/macs/cmac_prov.c index fa0b576b9..96da429e8 100644 --- a/openssl/src/providers/implementations/macs/cmac_prov.c +++ b/openssl/src/providers/implementations/macs/cmac_prov.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,8 +18,6 @@ #include #include #include -#include -#include #include "prov/implementations.h" #include "prov/provider_ctx.h" @@ -101,12 +99,8 @@ static void *cmac_dup(void *vsrc) static size_t cmac_size(void *vmacctx) { struct cmac_data_st *macctx = vmacctx; - const EVP_CIPHER_CTX *cipherctx = CMAC_CTX_get0_cipher_ctx(macctx->ctx); - if (EVP_CIPHER_CTX_get0_cipher(cipherctx) == NULL) - return 0; - - return EVP_CIPHER_CTX_get_block_size(cipherctx); + return EVP_CIPHER_CTX_get_block_size(CMAC_CTX_get0_cipher_ctx(macctx->ctx)); } static int cmac_setkey(struct cmac_data_st *macctx, @@ -201,16 +195,8 @@ static int cmac_set_ctx_params(void *vmacctx, const OSSL_PARAM params[]) if (params == NULL) return 1; - if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_CIPHER)) != NULL) { - if (!ossl_prov_cipher_load_from_params(&macctx->cipher, params, ctx)) - return 0; - - if (EVP_CIPHER_get_mode(ossl_prov_cipher_cipher(&macctx->cipher)) - != EVP_CIPH_CBC_MODE) { - ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_MODE); - return 0; - } - } + if (!ossl_prov_cipher_load_from_params(&macctx->cipher, params, ctx)) + return 0; if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_KEY)) != NULL) { if (p->data_type != OSSL_PARAM_OCTET_STRING) @@ -233,5 +219,5 @@ const OSSL_DISPATCH ossl_cmac_functions[] = { { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, (void (*)(void))cmac_settable_ctx_params }, { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))cmac_set_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/macs/eia3_prov.c b/openssl/src/providers/implementations/macs/eia3_prov.c new file mode 100644 index 000000000..50619b4a1 --- /dev/null +++ b/openssl/src/providers/implementations/macs/eia3_prov.c @@ -0,0 +1,247 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include +#include + +#include +#include "crypto/eia3/eia3_local.h" + +#include "prov/implementations.h" +#include "prov/providercommon.h" + +/* + * Forward declaration of everything implemented here. This is not strictly + * necessary for the compiler, but provides an assurance that the signatures + * of the functions in the dispatch table are correct. + */ +static OSSL_FUNC_mac_newctx_fn eia3_new; +static OSSL_FUNC_mac_dupctx_fn eia3_dup; +static OSSL_FUNC_mac_freectx_fn eia3_free; +static OSSL_FUNC_mac_gettable_params_fn eia3_gettable_params; +static OSSL_FUNC_mac_get_params_fn eia3_get_params; +static OSSL_FUNC_mac_get_ctx_params_fn eia3_get_ctx_params; +static OSSL_FUNC_mac_settable_ctx_params_fn eia3_settable_ctx_params; +static OSSL_FUNC_mac_set_ctx_params_fn eia3_set_ctx_params; +static OSSL_FUNC_mac_init_fn eia3_init; +static OSSL_FUNC_mac_update_fn eia3_update; +static OSSL_FUNC_mac_final_fn eia3_final; + +struct eia3_data_st { + void *provctx; + int updated; + unsigned char key[ZUC_KEY_SIZE]; + unsigned char iv[ZUC_CTR_SIZE]; + EIA3_CTX eia3; +}; + +static void *eia3_new(void *provctx) +{ + struct eia3_data_st *ctx; + + if (!ossl_prov_is_running()) + return NULL; + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx != NULL) + ctx->provctx = provctx; + return ctx; +} + +static void eia3_free(void *vmacctx) +{ + OPENSSL_free(vmacctx); +} + +static void *eia3_dup(void *vsrc) +{ + struct eia3_data_st *src = vsrc; + struct eia3_data_st *dst; + + if (!ossl_prov_is_running()) + return NULL; + dst = OPENSSL_malloc(sizeof(*dst)); + if (dst == NULL) + return NULL; + + *dst = *src; + return dst; +} + +static size_t eia3_size(void) +{ + return EIA3_DIGEST_SIZE; +} + +static int eia3_setkey(struct eia3_data_st *ctx, + const unsigned char *key, size_t keylen) +{ + if (keylen != EVP_ZUC_KEY_SIZE) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); + return 0; + } + EIA3_Init(&ctx->eia3, key, ctx->iv); + memcpy(&ctx->key, key, keylen); + ctx->updated = 0; + return 1; +} + +static int eia3_init(void *vmacctx, const unsigned char *key, + size_t keylen, const OSSL_PARAM params[]) +{ + struct eia3_data_st *ctx = vmacctx; + + /* initialize the context in MAC_ctrl function */ + if (!ossl_prov_is_running() || !eia3_set_ctx_params(ctx, params)) + return 0; + if (key != NULL) + return eia3_setkey(ctx, key, keylen); + /* no reinitialization of context with the same key is allowed */ + return ctx->updated == 0; +} + +static int eia3_update(void *vmacctx, const unsigned char *data, + size_t datalen) +{ + struct eia3_data_st *ctx = vmacctx; + + ctx->updated = 1; + if (datalen == 0) + return 1; + + /* eia3 has nothing to return in its update function */ + EIA3_Update(&ctx->eia3, data, datalen); + return 1; +} + +static int eia3_final(void *vmacctx, unsigned char *out, size_t *outl, + size_t outsize) +{ + struct eia3_data_st *ctx = vmacctx; + + if (!ossl_prov_is_running()) + return 0; + ctx->updated = 1; + EIA3_Final(&ctx->eia3, out); + *outl = eia3_size(); + return 1; +} + +static const OSSL_PARAM known_gettable_params[] = { + OSSL_PARAM_size_t(OSSL_MAC_PARAM_SIZE, NULL), + OSSL_PARAM_octet_string(OSSL_MAC_PARAM_KEY, NULL, 0), + OSSL_PARAM_octet_string(OSSL_MAC_PARAM_IV, NULL, 0), + OSSL_PARAM_END +}; +static const OSSL_PARAM *eia3_gettable_params(void *provctx) +{ + return known_gettable_params; +} + +static int eia3_get_params(OSSL_PARAM params[]) +{ + OSSL_PARAM *p; + + if ((p = OSSL_PARAM_locate(params, OSSL_MAC_PARAM_SIZE)) != NULL) + return OSSL_PARAM_set_size_t(p, eia3_size()); + + return 1; +} + +static const OSSL_PARAM known_settable_ctx_params[] = { + OSSL_PARAM_octet_string(OSSL_MAC_PARAM_KEY, NULL, 0), + OSSL_PARAM_octet_string(OSSL_MAC_PARAM_IV, NULL, 0), + OSSL_PARAM_END +}; +static const OSSL_PARAM *eia3_settable_ctx_params(ossl_unused void *ctx, + ossl_unused void *provctx) +{ + return known_settable_ctx_params; +} + +static int eia3_set_ctx_params(void *vmacctx, const OSSL_PARAM *params) +{ + struct eia3_data_st *ctx = vmacctx; + const OSSL_PARAM *p; + + if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_KEY)) != NULL + && p->data_type == OSSL_PARAM_OCTET_STRING + && !eia3_setkey(ctx, p->data, p->data_size)) + return 0; + + if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_IV)) != NULL + && p->data_type == OSSL_PARAM_OCTET_STRING) { + if (p->data_size != ZUC_CTR_SIZE) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); + return 0; + } + + memcpy(&ctx->iv, p->data, p->data_size); + + EIA3_Init(&ctx->eia3, ctx->key, ctx->iv); + ctx->updated = 0; + } + + return 1; +} + +static int eia3_get_ctx_params(void *vmacctx, OSSL_PARAM params[]) +{ + struct eia3_data_st *ctx = vmacctx; + OSSL_PARAM *p; + + if ((p = OSSL_PARAM_locate(params, OSSL_MAC_PARAM_SIZE)) != NULL + && !OSSL_PARAM_set_size_t(p, eia3_size())) + return 0; + + if ((p = OSSL_PARAM_locate(params, OSSL_MAC_PARAM_IV)) != NULL) { + if (p->data_size < ZUC_CTR_SIZE) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_IV_LENGTH); + return 0; + } + if (!OSSL_PARAM_set_octet_string(p, ctx->iv, ZUC_CTR_SIZE) + && !OSSL_PARAM_set_octet_ptr(p, &ctx->iv, ZUC_CTR_SIZE)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + } + + if ((p = OSSL_PARAM_locate(params, OSSL_MAC_PARAM_KEY)) != NULL) { + if (p->data_size < ZUC_KEY_SIZE) { + ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); + return 0; + } + if (!OSSL_PARAM_set_octet_string(p, ctx->key, ZUC_KEY_SIZE) + && !OSSL_PARAM_set_octet_ptr(p, &ctx->key, ZUC_KEY_SIZE)) { + ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER); + return 0; + } + } + + return 1; +} + +const OSSL_DISPATCH ossl_eia3_functions[] = { + { OSSL_FUNC_MAC_NEWCTX, (void (*)(void))eia3_new }, + { OSSL_FUNC_MAC_DUPCTX, (void (*)(void))eia3_dup }, + { OSSL_FUNC_MAC_FREECTX, (void (*)(void))eia3_free }, + { OSSL_FUNC_MAC_INIT, (void (*)(void))eia3_init }, + { OSSL_FUNC_MAC_UPDATE, (void (*)(void))eia3_update }, + { OSSL_FUNC_MAC_FINAL, (void (*)(void))eia3_final }, + { OSSL_FUNC_MAC_GETTABLE_PARAMS, (void (*)(void))eia3_gettable_params }, + { OSSL_FUNC_MAC_GET_PARAMS, (void (*)(void))eia3_get_params }, + { OSSL_FUNC_MAC_GET_CTX_PARAMS, (void (*)(void))eia3_get_ctx_params }, + { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, + (void (*)(void))eia3_settable_ctx_params }, + { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))eia3_set_ctx_params }, + { 0, NULL } +}; diff --git a/openssl/src/providers/implementations/macs/gmac_prov.c b/openssl/src/providers/implementations/macs/gmac_prov.c index 122df5f60..ad189d9c1 100644 --- a/openssl/src/providers/implementations/macs/gmac_prov.c +++ b/openssl/src/providers/implementations/macs/gmac_prov.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -235,8 +235,8 @@ static int gmac_set_ctx_params(void *vmacctx, const OSSL_PARAM params[]) if (p->data_type != OSSL_PARAM_OCTET_STRING) return 0; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, - p->data_size, NULL) <= 0 + if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, + p->data_size, NULL) || !EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, p->data)) return 0; } @@ -255,5 +255,5 @@ const OSSL_DISPATCH ossl_gmac_functions[] = { { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, (void (*)(void))gmac_settable_ctx_params }, { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))gmac_set_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/macs/hmac_prov.c b/openssl/src/providers/implementations/macs/hmac_prov.c index c72c1e6c0..2d6087b2c 100644 --- a/openssl/src/providers/implementations/macs/hmac_prov.c +++ b/openssl/src/providers/implementations/macs/hmac_prov.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,8 +21,6 @@ #include #include -#include "internal/ssl3_cbc.h" - #include "prov/implementations.h" #include "prov/provider_ctx.h" #include "prov/provider_util.h" @@ -61,6 +59,17 @@ struct hmac_data_st { size_t tls_mac_out_size; }; +/* Defined in ssl/s3_cbc.c */ +int ssl3_cbc_digest_record(const EVP_MD *md, + unsigned char *md_out, + size_t *md_out_size, + const unsigned char header[13], + const unsigned char *data, + size_t data_size, + size_t data_plus_mac_plus_padding_size, + const unsigned char *mac_secret, + size_t mac_secret_length, char is_sslv3); + static void *hmac_new(void *provctx) { struct hmac_data_st *macctx; @@ -106,7 +115,6 @@ static void *hmac_dup(void *vsrc) *dst = *src; dst->ctx = ctx; dst->key = NULL; - memset(&dst->digest, 0, sizeof(dst->digest)); if (!HMAC_CTX_copy(dst->ctx, src->ctx) || !ossl_prov_digest_copy(&dst->digest, &src->digest)) { @@ -274,6 +282,23 @@ static const OSSL_PARAM *hmac_settable_ctx_params(ossl_unused void *ctx, return known_settable_ctx_params; } +static int set_flag(const OSSL_PARAM params[], const char *key, int mask, + int *flags) +{ + const OSSL_PARAM *p = OSSL_PARAM_locate_const(params, key); + int flag = 0; + + if (p != NULL) { + if (!OSSL_PARAM_get_int(p, &flag)) + return 0; + if (flag == 0) + *flags &= ~mask; + else + *flags |= mask; + } + return 1; +} + /* * ALL parameters should be set before init(). */ @@ -282,6 +307,7 @@ static int hmac_set_ctx_params(void *vmacctx, const OSSL_PARAM params[]) struct hmac_data_st *macctx = vmacctx; OSSL_LIB_CTX *ctx = PROV_LIBCTX_OF(macctx->provctx); const OSSL_PARAM *p; + int flags = 0; if (params == NULL) return 1; @@ -289,6 +315,15 @@ static int hmac_set_ctx_params(void *vmacctx, const OSSL_PARAM params[]) if (!ossl_prov_digest_load_from_params(&macctx->digest, params, ctx)) return 0; + if (!set_flag(params, OSSL_MAC_PARAM_DIGEST_NOINIT, EVP_MD_CTX_FLAG_NO_INIT, + &flags)) + return 0; + if (!set_flag(params, OSSL_MAC_PARAM_DIGEST_ONESHOT, EVP_MD_CTX_FLAG_ONESHOT, + &flags)) + return 0; + if (flags) + HMAC_CTX_set_flags(macctx->ctx, flags); + if ((p = OSSL_PARAM_locate_const(params, OSSL_MAC_PARAM_KEY)) != NULL) { if (p->data_type != OSSL_PARAM_OCTET_STRING) return 0; @@ -317,5 +352,5 @@ const OSSL_DISPATCH ossl_hmac_functions[] = { { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, (void (*)(void))hmac_settable_ctx_params }, { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))hmac_set_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/macs/kmac_prov.c b/openssl/src/providers/implementations/macs/kmac_prov.c index ddaab4ba8..123c40f54 100644 --- a/openssl/src/providers/implementations/macs/kmac_prov.c +++ b/openssl/src/providers/implementations/macs/kmac_prov.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -91,21 +91,21 @@ static OSSL_FUNC_mac_final_fn kmac_final; * Restrict the maximum length of the customisation string. This must not * exceed 64 bits = 8k bytes. */ -#define KMAC_MAX_CUSTOM 512 +#define KMAC_MAX_CUSTOM 256 /* Maximum size of encoded custom string */ #define KMAC_MAX_CUSTOM_ENCODED (KMAC_MAX_CUSTOM + KMAC_MAX_ENCODED_HEADER_LEN) -/* Maximum key size in bytes = 512 (4096 bits) */ -#define KMAC_MAX_KEY 512 +/* Maximum key size in bytes = 256 (2048 bits) */ +#define KMAC_MAX_KEY 256 #define KMAC_MIN_KEY 4 /* * Maximum Encoded Key size will be padded to a multiple of the blocksize - * i.e KMAC_MAX_KEY + KMAC_MAX_ENCODED_HEADER_LEN = 512 + 4 + * i.e KMAC_MAX_KEY + KMAC_MAX_ENCODED_HEADER_LEN = 256 + 4 * Padded to a multiple of KMAC_MAX_BLOCKSIZE */ -#define KMAC_MAX_KEY_ENCODED (KMAC_MAX_BLOCKSIZE * 4) +#define KMAC_MAX_KEY_ENCODED (KMAC_MAX_BLOCKSIZE * 2) /* Fixed value of encode_string("KMAC") */ static const unsigned char kmac_string[] = { @@ -249,7 +249,7 @@ static int kmac_setkey(struct kmac_data_st *kctx, const unsigned char *key, ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH); return 0; } - if (w <= 0) { + if (w < 0) { ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH); return 0; } @@ -289,7 +289,7 @@ static int kmac_init(void *vmacctx, const unsigned char *key, return 0; t = EVP_MD_get_block_size(ossl_prov_digest_md(&kctx->digest)); - if (t <= 0) { + if (t < 0) { ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH); return 0; } @@ -310,8 +310,10 @@ static int kmac_init(void *vmacctx, const unsigned char *key, return 0; } out = OPENSSL_malloc(out_len); - if (out == NULL) + if (out == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } res = bytepad(out, NULL, kmac_string, sizeof(kmac_string), kctx->custom, kctx->custom_len, block_len) && EVP_DigestUpdate(ctx, out, out_len) @@ -605,7 +607,7 @@ const OSSL_DISPATCH ossl_kmac128_functions[] = { { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, (void (*)(void))kmac_settable_ctx_params }, { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))kmac_set_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; const OSSL_DISPATCH ossl_kmac256_functions[] = { @@ -621,5 +623,5 @@ const OSSL_DISPATCH ossl_kmac256_functions[] = { { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, (void (*)(void))kmac_settable_ctx_params }, { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))kmac_set_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/macs/poly1305_prov.c b/openssl/src/providers/implementations/macs/poly1305_prov.c index 19974f928..f922802ea 100644 --- a/openssl/src/providers/implementations/macs/poly1305_prov.c +++ b/openssl/src/providers/implementations/macs/poly1305_prov.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -183,5 +183,5 @@ const OSSL_DISPATCH ossl_poly1305_functions[] = { { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, (void (*)(void))poly1305_settable_ctx_params }, { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))poly1305_set_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/macs/siphash_prov.c b/openssl/src/providers/implementations/macs/siphash_prov.c index a54def3b8..a28cdb33a 100644 --- a/openssl/src/providers/implementations/macs/siphash_prov.c +++ b/openssl/src/providers/implementations/macs/siphash_prov.c @@ -1,5 +1,5 @@ /* - * Copyright 2018-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -233,5 +233,5 @@ const OSSL_DISPATCH ossl_siphash_functions[] = { { OSSL_FUNC_MAC_SETTABLE_CTX_PARAMS, (void (*)(void))siphash_settable_ctx_params }, { OSSL_FUNC_MAC_SET_CTX_PARAMS, (void (*)(void))siphash_set_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/rands/crngt.c b/openssl/src/providers/implementations/rands/crngt.c index fa4a2db14..4095994bd 100644 --- a/openssl/src/providers/implementations/rands/crngt.c +++ b/openssl/src/providers/implementations/rands/crngt.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -23,7 +23,6 @@ #include "crypto/rand_pool.h" #include "drbg_local.h" #include "prov/seeding.h" -#include "crypto/context.h" typedef struct crng_test_global_st { unsigned char crngt_prev[EVP_MAX_MD_SIZE]; @@ -53,7 +52,7 @@ static int crngt_get_entropy(PROV_CTX *provctx, const EVP_MD *digest, return 0; } -void ossl_rand_crng_ctx_free(void *vcrngt_glob) +static void rand_crng_ossl_ctx_free(void *vcrngt_glob) { CRNG_TEST_GLOBAL *crngt_glob = vcrngt_glob; @@ -62,7 +61,7 @@ void ossl_rand_crng_ctx_free(void *vcrngt_glob) OPENSSL_free(crngt_glob); } -void *ossl_rand_crng_ctx_new(OSSL_LIB_CTX *ctx) +static void *rand_crng_ossl_ctx_new(OSSL_LIB_CTX *ctx) { CRNG_TEST_GLOBAL *crngt_glob = OPENSSL_zalloc(sizeof(*crngt_glob)); @@ -83,6 +82,12 @@ void *ossl_rand_crng_ctx_new(OSSL_LIB_CTX *ctx) return crngt_glob; } +static const OSSL_LIB_CTX_METHOD rand_crng_ossl_ctx_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + rand_crng_ossl_ctx_new, + rand_crng_ossl_ctx_free, +}; + static int prov_crngt_compare_previous(const unsigned char *prev, const unsigned char *cur, size_t sz) @@ -108,7 +113,8 @@ size_t ossl_crngt_get_entropy(PROV_DRBG *drbg, int crng_test_pass = 1; OSSL_LIB_CTX *libctx = ossl_prov_ctx_get0_libctx(drbg->provctx); CRNG_TEST_GLOBAL *crngt_glob - = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_RAND_CRNGT_INDEX); + = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_RAND_CRNGT_INDEX, + &rand_crng_ossl_ctx_method); OSSL_CALLBACK *stcb = NULL; void *stcbarg = NULL; OSSL_SELF_TEST *st = NULL; diff --git a/openssl/src/providers/implementations/rands/drbg.c b/openssl/src/providers/implementations/rands/drbg.c index 46a056bc2..daa85f068 100644 --- a/openssl/src/providers/implementations/rands/drbg.c +++ b/openssl/src/providers/implementations/rands/drbg.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -21,8 +21,6 @@ #include "crypto/rand_pool.h" #include "prov/provider_ctx.h" #include "prov/providercommon.h" -#include "prov/fipscommon.h" -#include "crypto/context.h" /* * Support framework for NIST SP 800-90A DRBG @@ -45,20 +43,21 @@ static const OSSL_DISPATCH *find_call(const OSSL_DISPATCH *dispatch, static int rand_drbg_restart(PROV_DRBG *drbg); -/* - * We interpret a call to this function as a hint only and ignore it. This - * occurs when the EVP layer thinks we should do some locking. In practice - * however we manage for ourselves when we take a lock or not on the basis - * of whether drbg->lock is present or not. - */ int ossl_drbg_lock(void *vctx) { - return 1; + PROV_DRBG *drbg = vctx; + + if (drbg == NULL || drbg->lock == NULL) + return 1; + return CRYPTO_THREAD_write_lock(drbg->lock); } -/* Interpreted as a hint only and ignored as for ossl_drbg_lock() */ void ossl_drbg_unlock(void *vctx) { + PROV_DRBG *drbg = vctx; + + if (drbg != NULL && drbg->lock != NULL) + CRYPTO_THREAD_unlock(drbg->lock); } static int ossl_drbg_lock_parent(PROV_DRBG *drbg) @@ -149,7 +148,8 @@ size_t ossl_drbg_get_seed(void *vdrbg, unsigned char **pout, { PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; size_t bytes_needed; - unsigned char *buffer; + size_t chunk, outlen; + unsigned char *buffer, *out; /* Figure out how many bytes we need */ bytes_needed = entropy >= 0 ? (entropy + 7) / 8 : 0; @@ -160,25 +160,41 @@ size_t ossl_drbg_get_seed(void *vdrbg, unsigned char **pout, /* Allocate storage */ buffer = OPENSSL_secure_malloc(bytes_needed); - if (buffer == NULL) + if (buffer == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } - /* - * Get random data. Include our DRBG address as - * additional input, in order to provide a distinction between - * different DRBG child instances. - * - * Note: using the sizeof() operator on a pointer triggers - * a warning in some static code analyzers, but it's - * intentional and correct here. - */ - if (!ossl_prov_drbg_generate(drbg, buffer, bytes_needed, - drbg->strength, prediction_resistance, - (unsigned char *)&drbg, sizeof(drbg))) { - OPENSSL_secure_clear_free(buffer, bytes_needed); - ERR_raise(ERR_LIB_PROV, PROV_R_GENERATE_ERROR); - return 0; + out = buffer; + outlen = bytes_needed; + + for (; outlen > 0; outlen -= chunk, out += chunk) { + chunk = outlen > drbg->max_request ? drbg->max_request : outlen; + + /* + * Get random data. Include our DRBG address as + * additional input, in order to provide a distinction between + * different DRBG child instances. + * + * Note: using the sizeof() operator on a pointer triggers + * a warning in some static code analyzers, but it's + * intentional and correct here. + */ + if (!ossl_prov_drbg_generate(drbg, out, chunk, + drbg->strength, prediction_resistance, + (unsigned char *)&drbg, sizeof(drbg))) { + OPENSSL_secure_clear_free(buffer, bytes_needed); + ERR_raise(ERR_LIB_PROV, PROV_R_GENERATE_ERROR); + return 0; + } + + /* + * Prediction resistance is only relevant the first time around, + * subsequently, the DRBG has already been properly reseeded. + */ + prediction_resistance = 0; } + *pout = buffer; return bytes_needed; } @@ -255,7 +271,7 @@ static void cleanup_entropy(PROV_DRBG *drbg, unsigned char *out, size_t outlen) } else if (drbg->parent_clear_seed != NULL) { if (!ossl_drbg_lock_parent(drbg)) return; - drbg->parent_clear_seed(drbg->parent, out, outlen); + drbg->parent_clear_seed(drbg, out, outlen); ossl_drbg_unlock_parent(drbg); } } @@ -273,7 +289,7 @@ typedef struct prov_drbg_nonce_global_st { * to be in a different global data object. Otherwise we will go into an * infinite recursion loop. */ -void *ossl_prov_drbg_nonce_ctx_new(OSSL_LIB_CTX *libctx) +static void *prov_drbg_nonce_ossl_ctx_new(OSSL_LIB_CTX *libctx) { PROV_DRBG_NONCE_GLOBAL *dngbl = OPENSSL_zalloc(sizeof(*dngbl)); @@ -289,7 +305,7 @@ void *ossl_prov_drbg_nonce_ctx_new(OSSL_LIB_CTX *libctx) return dngbl; } -void ossl_prov_drbg_nonce_ctx_free(void *vdngbl) +static void prov_drbg_nonce_ossl_ctx_free(void *vdngbl) { PROV_DRBG_NONCE_GLOBAL *dngbl = vdngbl; @@ -301,6 +317,12 @@ void ossl_prov_drbg_nonce_ctx_free(void *vdngbl) OPENSSL_free(dngbl); } +static const OSSL_LIB_CTX_METHOD drbg_nonce_ossl_ctx_method = { + OSSL_LIB_CTX_METHOD_DEFAULT_PRIORITY, + prov_drbg_nonce_ossl_ctx_new, + prov_drbg_nonce_ossl_ctx_free, +}; + /* Get a nonce from the operating system */ static size_t prov_drbg_get_nonce(PROV_DRBG *drbg, unsigned char **pout, size_t min_len, size_t max_len) @@ -309,7 +331,8 @@ static size_t prov_drbg_get_nonce(PROV_DRBG *drbg, unsigned char **pout, unsigned char *buf = NULL; OSSL_LIB_CTX *libctx = ossl_prov_ctx_get0_libctx(drbg->provctx); PROV_DRBG_NONCE_GLOBAL *dngbl - = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DRBG_NONCE_INDEX); + = ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DRBG_NONCE_INDEX, + &drbg_nonce_ossl_ctx_method); struct { void *drbg; int count; @@ -335,9 +358,8 @@ static size_t prov_drbg_get_nonce(PROV_DRBG *drbg, unsigned char **pout, /* Use the built in nonce source plus some of our specifics */ memset(&data, 0, sizeof(data)); data.drbg = drbg; - if (!CRYPTO_atomic_add(&dngbl->rand_nonce_count, 1, &data.count, - dngbl->rand_nonce_lock)) - return 0; + CRYPTO_atomic_add(&dngbl->rand_nonce_count, 1, &data.count, + dngbl->rand_nonce_lock); return ossl_prov_get_nonce(drbg->provctx, pout, min_len, max_len, &data, sizeof(data)); } @@ -484,12 +506,16 @@ int ossl_prov_drbg_uninstantiate(PROV_DRBG *drbg) return 1; } -static int ossl_prov_drbg_reseed_unlocked(PROV_DRBG *drbg, - int prediction_resistance, - const unsigned char *ent, - size_t ent_len, - const unsigned char *adin, - size_t adinlen) +/* + * Reseed |drbg|, mixing in the specified data + * + * Requires that drbg->lock is already locked for write, if non-null. + * + * Returns 1 on success, 0 on failure. + */ +int ossl_prov_drbg_reseed(PROV_DRBG *drbg, int prediction_resistance, + const unsigned char *ent, size_t ent_len, + const unsigned char *adin, size_t adinlen) { unsigned char *entropy = NULL; size_t entropylen = 0; @@ -591,37 +617,12 @@ static int ossl_prov_drbg_reseed_unlocked(PROV_DRBG *drbg, return 0; } -/* - * Reseed |drbg|, mixing in the specified data - * - * Acquires the drbg->lock for writing, if non-null. - * - * Returns 1 on success, 0 on failure. - */ -int ossl_prov_drbg_reseed(PROV_DRBG *drbg, int prediction_resistance, - const unsigned char *ent, size_t ent_len, - const unsigned char *adin, size_t adinlen) -{ - int ret; - - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) - return 0; - - ret = ossl_prov_drbg_reseed_unlocked(drbg, prediction_resistance, ent, - ent_len, adin, adinlen); - - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; -} - /* * Generate |outlen| bytes into the buffer at |out|. Reseed if we need * to or if |prediction_resistance| is set. Additional input can be * sent in |adin| and |adinlen|. * - * Acquires the drbg->lock for writing if available + * Requires that drbg->lock is already locked for write, if non-null. * * Returns 1 on success, 0 on failure. * @@ -632,39 +633,35 @@ int ossl_prov_drbg_generate(PROV_DRBG *drbg, unsigned char *out, size_t outlen, { int fork_id; int reseed_required = 0; - int ret = 0; if (!ossl_prov_is_running()) return 0; - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) - return 0; - if (drbg->state != EVP_RAND_STATE_READY) { /* try to recover from previous errors */ rand_drbg_restart(drbg); if (drbg->state == EVP_RAND_STATE_ERROR) { ERR_raise(ERR_LIB_PROV, PROV_R_IN_ERROR_STATE); - goto err; + return 0; } if (drbg->state == EVP_RAND_STATE_UNINITIALISED) { ERR_raise(ERR_LIB_PROV, PROV_R_NOT_INSTANTIATED); - goto err; + return 0; } } if (strength > drbg->strength) { ERR_raise(ERR_LIB_PROV, PROV_R_INSUFFICIENT_DRBG_STRENGTH); - goto err; + return 0; } if (outlen > drbg->max_request) { ERR_raise(ERR_LIB_PROV, PROV_R_REQUEST_TOO_LARGE_FOR_DRBG); - goto err; + return 0; } if (adinlen > drbg->max_adinlen) { ERR_raise(ERR_LIB_PROV, PROV_R_ADDITIONAL_INPUT_TOO_LONG); - goto err; + return 0; } fork_id = openssl_get_fork_id(); @@ -689,10 +686,10 @@ int ossl_prov_drbg_generate(PROV_DRBG *drbg, unsigned char *out, size_t outlen, reseed_required = 1; if (reseed_required || prediction_resistance) { - if (!ossl_prov_drbg_reseed_unlocked(drbg, prediction_resistance, NULL, - 0, adin, adinlen)) { + if (!ossl_prov_drbg_reseed(drbg, prediction_resistance, NULL, 0, + adin, adinlen)) { ERR_raise(ERR_LIB_PROV, PROV_R_RESEED_ERROR); - goto err; + return 0; } adin = NULL; adinlen = 0; @@ -701,17 +698,12 @@ int ossl_prov_drbg_generate(PROV_DRBG *drbg, unsigned char *out, size_t outlen, if (!drbg->generate(drbg, out, outlen, adin, adinlen)) { drbg->state = EVP_RAND_STATE_ERROR; ERR_raise(ERR_LIB_PROV, PROV_R_GENERATE_ERROR); - goto err; + return 0; } drbg->generate_counter++; - ret = 1; - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; + return 1; } /* @@ -788,7 +780,6 @@ int ossl_drbg_enable_locking(void *vctx) PROV_DRBG *ossl_rand_drbg_new (void *provctx, void *parent, const OSSL_DISPATCH *p_dispatch, int (*dnew)(PROV_DRBG *ctx), - void (*dfree)(void *vctx), int (*instantiate)(PROV_DRBG *drbg, const unsigned char *entropy, size_t entropylen, const unsigned char *nonce, size_t noncelen, @@ -807,8 +798,10 @@ PROV_DRBG *ossl_rand_drbg_new return NULL; drbg = OPENSSL_zalloc(sizeof(*drbg)); - if (drbg == NULL) + if (drbg == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } drbg->provctx = provctx; drbg->instantiate = instantiate; @@ -866,7 +859,7 @@ PROV_DRBG *ossl_rand_drbg_new return drbg; err: - dfree(drbg); + ossl_rand_drbg_free(drbg); return NULL; } @@ -879,10 +872,6 @@ void ossl_rand_drbg_free(PROV_DRBG *drbg) OPENSSL_free(drbg); } -/* - * Helper function called by internal DRBG implementations. Assumes that at - * least a read lock has been taken on drbg->lock - */ int ossl_drbg_get_ctx_params(PROV_DRBG *drbg, OSSL_PARAM params[]) { OSSL_PARAM *p; @@ -895,6 +884,10 @@ int ossl_drbg_get_ctx_params(PROV_DRBG *drbg, OSSL_PARAM params[]) if (p != NULL && !OSSL_PARAM_set_int(p, drbg->strength)) return 0; + p = OSSL_PARAM_locate(params, OSSL_RAND_PARAM_MAX_REQUEST); + if (p != NULL && !OSSL_PARAM_set_size_t(p, drbg->max_request)) + return 0; + p = OSSL_PARAM_locate(params, OSSL_DRBG_PARAM_MIN_ENTROPYLEN); if (p != NULL && !OSSL_PARAM_set_size_t(p, drbg->min_entropylen)) return 0; @@ -931,43 +924,10 @@ int ossl_drbg_get_ctx_params(PROV_DRBG *drbg, OSSL_PARAM params[]) if (p != NULL && !OSSL_PARAM_set_time_t(p, drbg->reseed_time_interval)) return 0; - return 1; -} - -/* - * Helper function to get certain params that require no lock to obtain. Sets - * *complete to 1 if all the params were processed, or 0 otherwise - */ -int ossl_drbg_get_ctx_params_no_lock(PROV_DRBG *drbg, OSSL_PARAM params[], - int *complete) -{ - size_t cnt = 0; - OSSL_PARAM *p; - - /* This value never changes once set */ - p = OSSL_PARAM_locate(params, OSSL_RAND_PARAM_MAX_REQUEST); - if (p != NULL) { - if (!OSSL_PARAM_set_size_t(p, drbg->max_request)) - return 0; - cnt++; - } - - /* - * Can be changed by multiple threads, but we tolerate inaccuracies in this - * value. - */ p = OSSL_PARAM_locate(params, OSSL_DRBG_PARAM_RESEED_COUNTER); - if (p != NULL) { - if (!OSSL_PARAM_set_uint(p, tsan_load(&drbg->reseed_counter))) - return 0; - cnt++; - } - - if (params[cnt].key == NULL) - *complete = 1; - else - *complete = 0; - + if (p != NULL + && !OSSL_PARAM_set_uint(p, tsan_load(&drbg->reseed_counter))) + return 0; return 1; } @@ -987,31 +947,3 @@ int ossl_drbg_set_ctx_params(PROV_DRBG *drbg, const OSSL_PARAM params[]) return 0; return 1; } - -/* Confirm digest is allowed to be used with a DRBG */ -int ossl_drbg_verify_digest(ossl_unused OSSL_LIB_CTX *libctx, const EVP_MD *md) -{ -#ifdef FIPS_MODULE - /* FIPS 140-3 IG D.R limited DRBG digests to a specific set */ - static const char *const allowed_digests[] = { - "SHA1", /* SHA 1 allowed */ - "SHA2-256", "SHA2-512", /* non-truncated SHA2 allowed */ - "SHA3-256", "SHA3-512", /* non-truncated SHA3 allowed */ - }; - size_t i; - - if (FIPS_restricted_drbg_digests_enabled(libctx)) { - for (i = 0; i < OSSL_NELEM(allowed_digests); i++) - if (EVP_MD_is_a(md, allowed_digests[i])) - return 1; - ERR_raise(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED); - return 0; - } -#endif - /* Outside of FIPS, any digests that are not XOF are allowed */ - if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED); - return 0; - } - return 1; -} diff --git a/openssl/src/providers/implementations/rands/drbg_ctr.c b/openssl/src/providers/implementations/rands/drbg_ctr.c index 0c4553ad5..451113c4d 100644 --- a/openssl/src/providers/implementations/rands/drbg_ctr.c +++ b/openssl/src/providers/implementations/rands/drbg_ctr.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -33,8 +33,6 @@ static OSSL_FUNC_rand_gettable_ctx_params_fn drbg_ctr_gettable_ctx_params; static OSSL_FUNC_rand_get_ctx_params_fn drbg_ctr_get_ctx_params; static OSSL_FUNC_rand_verify_zeroization_fn drbg_ctr_verify_zeroization; -static int drbg_ctr_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[]); - /* * The state of a DRBG AES-CTR. */ @@ -332,20 +330,11 @@ static int drbg_ctr_instantiate_wrapper(void *vdrbg, unsigned int strength, const OSSL_PARAM params[]) { PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; - int ret = 0; - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) + if (!ossl_prov_is_running() || !drbg_ctr_set_ctx_params(drbg, params)) return 0; - - if (!ossl_prov_is_running() - || !drbg_ctr_set_ctx_params_locked(drbg, params)) - goto err; - ret = ossl_prov_drbg_instantiate(drbg, strength, prediction_resistance, - pstr, pstr_len); - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - return ret; + return ossl_prov_drbg_instantiate(drbg, strength, prediction_resistance, + pstr, pstr_len); } static int drbg_ctr_reseed(PROV_DRBG *drbg, @@ -484,41 +473,21 @@ static int drbg_ctr_uninstantiate(PROV_DRBG *drbg) static int drbg_ctr_uninstantiate_wrapper(void *vdrbg) { - PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; - int ret; - - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) - return 0; - - ret = drbg_ctr_uninstantiate(drbg); - - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; + return drbg_ctr_uninstantiate((PROV_DRBG *)vdrbg); } static int drbg_ctr_verify_zeroization(void *vdrbg) { PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; PROV_DRBG_CTR *ctr = (PROV_DRBG_CTR *)drbg->data; - int ret = 0; - if (drbg->lock != NULL && !CRYPTO_THREAD_read_lock(drbg->lock)) - return 0; - - PROV_DRBG_VERIFY_ZEROIZATION(ctr->K); - PROV_DRBG_VERIFY_ZEROIZATION(ctr->V); - PROV_DRBG_VERIFY_ZEROIZATION(ctr->bltmp); - PROV_DRBG_VERIFY_ZEROIZATION(ctr->KX); + PROV_DRBG_VERYIFY_ZEROIZATION(ctr->K); + PROV_DRBG_VERYIFY_ZEROIZATION(ctr->V); + PROV_DRBG_VERYIFY_ZEROIZATION(ctr->bltmp); + PROV_DRBG_VERYIFY_ZEROIZATION(ctr->KX); if (ctr->bltmp_pos != 0) - goto err; - - ret = 1; - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - return ret; + return 0; + return 1; } static int drbg_ctr_init_lengths(PROV_DRBG *drbg) @@ -569,7 +538,7 @@ static int drbg_ctr_init(PROV_DRBG *drbg) if (ctr->ctx_ctr == NULL) ctr->ctx_ctr = EVP_CIPHER_CTX_new(); if (ctr->ctx_ecb == NULL || ctr->ctx_ctr == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -596,7 +565,7 @@ static int drbg_ctr_init(PROV_DRBG *drbg) if (ctr->ctx_df == NULL) ctr->ctx_df = EVP_CIPHER_CTX_new(); if (ctr->ctx_df == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_EVP_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } /* Set key schedule for df_key */ @@ -612,7 +581,7 @@ static int drbg_ctr_init(PROV_DRBG *drbg) EVP_CIPHER_CTX_free(ctr->ctx_ecb); EVP_CIPHER_CTX_free(ctr->ctx_ctr); ctr->ctx_ecb = ctr->ctx_ctr = NULL; - return 0; + return 0; } static int drbg_ctr_new(PROV_DRBG *drbg) @@ -620,8 +589,10 @@ static int drbg_ctr_new(PROV_DRBG *drbg) PROV_DRBG_CTR *ctr; ctr = OPENSSL_secure_zalloc(sizeof(*ctr)); - if (ctr == NULL) + if (ctr == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } ctr->use_df = 1; drbg->data = ctr; @@ -631,8 +602,7 @@ static int drbg_ctr_new(PROV_DRBG *drbg) static void *drbg_ctr_new_wrapper(void *provctx, void *parent, const OSSL_DISPATCH *parent_dispatch) { - return ossl_rand_drbg_new(provctx, parent, parent_dispatch, - &drbg_ctr_new, &drbg_ctr_free, + return ossl_rand_drbg_new(provctx, parent, parent_dispatch, &drbg_ctr_new, &drbg_ctr_instantiate, &drbg_ctr_uninstantiate, &drbg_ctr_reseed, &drbg_ctr_generate); } @@ -659,35 +629,20 @@ static int drbg_ctr_get_ctx_params(void *vdrbg, OSSL_PARAM params[]) PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; PROV_DRBG_CTR *ctr = (PROV_DRBG_CTR *)drbg->data; OSSL_PARAM *p; - int ret = 0, complete = 0; - - if (!ossl_drbg_get_ctx_params_no_lock(drbg, params, &complete)) - return 0; - - if (complete) - return 1; - - if (drbg->lock != NULL && !CRYPTO_THREAD_read_lock(drbg->lock)) - return 0; p = OSSL_PARAM_locate(params, OSSL_DRBG_PARAM_USE_DF); if (p != NULL && !OSSL_PARAM_set_int(p, ctr->use_df)) - goto err; + return 0; p = OSSL_PARAM_locate(params, OSSL_DRBG_PARAM_CIPHER); if (p != NULL) { if (ctr->cipher_ctr == NULL || !OSSL_PARAM_set_utf8_string(p, EVP_CIPHER_get0_name(ctr->cipher_ctr))) - goto err; + return 0; } - ret = ossl_drbg_get_ctx_params(drbg, params); - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; + return ossl_drbg_get_ctx_params(drbg, params); } static const OSSL_PARAM *drbg_ctr_gettable_ctx_params(ossl_unused void *vctx, @@ -702,7 +657,7 @@ static const OSSL_PARAM *drbg_ctr_gettable_ctx_params(ossl_unused void *vctx, return known_gettable_ctx_params; } -static int drbg_ctr_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[]) +static int drbg_ctr_set_ctx_params(void *vctx, const OSSL_PARAM params[]) { PROV_DRBG *ctx = (PROV_DRBG *)vctx; PROV_DRBG_CTR *ctr = (PROV_DRBG_CTR *)ctx->data; @@ -738,8 +693,10 @@ static int drbg_ctr_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[]) ERR_raise(ERR_LIB_PROV, PROV_R_REQUIRE_CTR_MODE_CIPHER); return 0; } - if ((ecb = OPENSSL_strndup(base, p->data_size)) == NULL) + if ((ecb = OPENSSL_strndup(base, p->data_size)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } strcpy(ecb + p->data_size - ecb_str_len, "ECB"); EVP_CIPHER_free(ctr->cipher_ecb); EVP_CIPHER_free(ctr->cipher_ctr); @@ -759,22 +716,6 @@ static int drbg_ctr_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[]) return ossl_drbg_set_ctx_params(ctx, params); } -static int drbg_ctr_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - PROV_DRBG *drbg = (PROV_DRBG *)vctx; - int ret; - - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) - return 0; - - ret = drbg_ctr_set_ctx_params_locked(vctx, params); - - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; -} - static const OSSL_PARAM *drbg_ctr_settable_ctx_params(ossl_unused void *vctx, ossl_unused void *provctx) { @@ -810,5 +751,5 @@ const OSSL_DISPATCH ossl_drbg_ctr_functions[] = { (void(*)(void))drbg_ctr_verify_zeroization }, { OSSL_FUNC_RAND_GET_SEED, (void(*)(void))ossl_drbg_get_seed }, { OSSL_FUNC_RAND_CLEAR_SEED, (void(*)(void))ossl_drbg_clear_seed }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/rands/drbg_hash.c b/openssl/src/providers/implementations/rands/drbg_hash.c index b9854a9aa..617e75848 100644 --- a/openssl/src/providers/implementations/rands/drbg_hash.c +++ b/openssl/src/providers/implementations/rands/drbg_hash.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -35,8 +35,6 @@ static OSSL_FUNC_rand_gettable_ctx_params_fn drbg_hash_gettable_ctx_params; static OSSL_FUNC_rand_get_ctx_params_fn drbg_hash_get_ctx_params; static OSSL_FUNC_rand_verify_zeroization_fn drbg_hash_verify_zeroization; -static int drbg_hash_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[]); - /* 888 bits from SP800-90Ar1 10.1 table 2 */ #define HASH_PRNG_MAX_SEEDLEN (888/8) @@ -115,7 +113,7 @@ static int hash_df(PROV_DRBG *drbg, unsigned char *out, memcpy(out, vtmp, outlen); OPENSSL_cleanse(vtmp, hash->blocklen); break; - } else if (!EVP_DigestFinal(ctx, out, NULL)) { + } else if(!EVP_DigestFinal(ctx, out, NULL)) { return 0; } @@ -166,7 +164,7 @@ static int add_bytes(PROV_DRBG *drbg, unsigned char *dst, /* Add the carry to the top of the dst if inlen is not the same size */ for (i = drbg->seedlen - inlen; i > 0; --i, d--) { *d += 1; /* Carry can only be 1 */ - if (*d != 0) /* exit if carry doesn't propagate to the next byte */ + if (*d != 0) /* exit if carry doesnt propagate to the next byte */ break; } } @@ -214,7 +212,7 @@ static int hash_gen(PROV_DRBG *drbg, unsigned char *out, size_t outlen) if (outlen == 0) return 1; memcpy(hash->vtmp, hash->V, drbg->seedlen); - for (;;) { + for(;;) { if (!EVP_DigestInit_ex(hash->ctx, ossl_prov_digest_md(&hash->digest), NULL) || !EVP_DigestUpdate(hash->ctx, hash->vtmp, drbg->seedlen)) @@ -253,10 +251,16 @@ static int drbg_hash_instantiate(PROV_DRBG *drbg, const unsigned char *pstr, size_t pstr_len) { PROV_DRBG_HASH *hash = (PROV_DRBG_HASH *)drbg->data; + const EVP_MD *md; EVP_MD_CTX_free(hash->ctx); hash->ctx = EVP_MD_CTX_new(); + md = ossl_prov_digest_md(&hash->digest); + /* GM/T 0105-2021 B.1, requested_number_of_bits <= 256 bits */ + if (md != NULL && EVP_MD_is_a(md, "SM3")) + drbg->max_request = 256 / 8; + /* (Step 1-3) V = Hash_df(entropy||nonce||pers, seedlen) */ return hash->ctx != NULL && hash_df(drbg, hash->V, INBYTE_IGNORE, @@ -272,20 +276,11 @@ static int drbg_hash_instantiate_wrapper(void *vdrbg, unsigned int strength, const OSSL_PARAM params[]) { PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; - int ret = 0; - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) + if (!ossl_prov_is_running() || !drbg_hash_set_ctx_params(drbg, params)) return 0; - - if (!ossl_prov_is_running() - || !drbg_hash_set_ctx_params_locked(drbg, params)) - goto err; - ret = ossl_prov_drbg_instantiate(drbg, strength, prediction_resistance, - pstr, pstr_len); - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - return ret; + return ossl_prov_drbg_instantiate(drbg, strength, prediction_resistance, + pstr, pstr_len); } /* @@ -301,12 +296,26 @@ static int drbg_hash_reseed(PROV_DRBG *drbg, const unsigned char *adin, size_t adin_len) { PROV_DRBG_HASH *hash = (PROV_DRBG_HASH *)drbg->data; + const EVP_MD *md; + + md = ossl_prov_digest_md(&hash->digest); + + if (md != NULL && EVP_MD_is_a(md, "SM3")) { + /* + * GM/T 0105-2021 B.5 + * V = SM3_df(0x01 || entropy_input || V || additional_input) + */ + if (!hash_df(drbg, hash->C, 0x01, ent, ent_len, hash->V, drbg->seedlen, + adin, adin_len)) + return 0; + } else { + /* (Step 1-2) V = Hash_df(0x01 || V || entropy_input || additional_input) */ + /* V about to be updated so use C as output instead */ + if (!hash_df(drbg, hash->C, 0x01, hash->V, drbg->seedlen, ent, ent_len, + adin, adin_len)) + return 0; + } - /* (Step 1-2) V = Hash_df(0x01 || V || entropy_input || additional_input) */ - /* V about to be updated so use C as output instead */ - if (!hash_df(drbg, hash->C, 0x01, hash->V, drbg->seedlen, ent, ent_len, - adin, adin_len)) - return 0; memcpy(hash->V, hash->C, drbg->seedlen); /* (Step 4) C = Hash_df(0x00||V, seedlen) */ return hash_df1(drbg, hash->C, 0x00, hash->V, drbg->seedlen); @@ -382,38 +391,18 @@ static int drbg_hash_uninstantiate(PROV_DRBG *drbg) static int drbg_hash_uninstantiate_wrapper(void *vdrbg) { - PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; - int ret; - - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) - return 0; - - ret = drbg_hash_uninstantiate(drbg); - - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; + return drbg_hash_uninstantiate((PROV_DRBG *)vdrbg); } static int drbg_hash_verify_zeroization(void *vdrbg) { PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; PROV_DRBG_HASH *hash = (PROV_DRBG_HASH *)drbg->data; - int ret = 0; - - if (drbg->lock != NULL && !CRYPTO_THREAD_read_lock(drbg->lock)) - return 0; - PROV_DRBG_VERIFY_ZEROIZATION(hash->V); - PROV_DRBG_VERIFY_ZEROIZATION(hash->C); - PROV_DRBG_VERIFY_ZEROIZATION(hash->vtmp); - - ret = 1; - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - return ret; + PROV_DRBG_VERYIFY_ZEROIZATION(hash->V); + PROV_DRBG_VERYIFY_ZEROIZATION(hash->C); + PROV_DRBG_VERYIFY_ZEROIZATION(hash->vtmp); + return 1; } static int drbg_hash_new(PROV_DRBG *ctx) @@ -421,8 +410,10 @@ static int drbg_hash_new(PROV_DRBG *ctx) PROV_DRBG_HASH *hash; hash = OPENSSL_secure_zalloc(sizeof(*hash)); - if (hash == NULL) + if (hash == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } ctx->data = hash; ctx->seedlen = HASH_PRNG_MAX_SEEDLEN; @@ -439,8 +430,7 @@ static int drbg_hash_new(PROV_DRBG *ctx) static void *drbg_hash_new_wrapper(void *provctx, void *parent, const OSSL_DISPATCH *parent_dispatch) { - return ossl_rand_drbg_new(provctx, parent, parent_dispatch, - &drbg_hash_new, &drbg_hash_free, + return ossl_rand_drbg_new(provctx, parent, parent_dispatch, &drbg_hash_new, &drbg_hash_instantiate, &drbg_hash_uninstantiate, &drbg_hash_reseed, &drbg_hash_generate); } @@ -464,30 +454,15 @@ static int drbg_hash_get_ctx_params(void *vdrbg, OSSL_PARAM params[]) PROV_DRBG_HASH *hash = (PROV_DRBG_HASH *)drbg->data; const EVP_MD *md; OSSL_PARAM *p; - int ret = 0, complete = 0; - - if (!ossl_drbg_get_ctx_params_no_lock(drbg, params, &complete)) - return 0; - - if (complete) - return 1; - - if (drbg->lock != NULL && !CRYPTO_THREAD_read_lock(drbg->lock)) - return 0; p = OSSL_PARAM_locate(params, OSSL_DRBG_PARAM_DIGEST); if (p != NULL) { md = ossl_prov_digest_md(&hash->digest); if (md == NULL || !OSSL_PARAM_set_utf8_string(p, EVP_MD_get0_name(md))) - goto err; + return 0; } - ret = ossl_drbg_get_ctx_params(drbg, params); - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; + return ossl_drbg_get_ctx_params(drbg, params); } static const OSSL_PARAM *drbg_hash_gettable_ctx_params(ossl_unused void *vctx, @@ -501,7 +476,7 @@ static const OSSL_PARAM *drbg_hash_gettable_ctx_params(ossl_unused void *vctx, return known_gettable_ctx_params; } -static int drbg_hash_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[]) +static int drbg_hash_set_ctx_params(void *vctx, const OSSL_PARAM params[]) { PROV_DRBG *ctx = (PROV_DRBG *)vctx; PROV_DRBG_HASH *hash = (PROV_DRBG_HASH *)ctx->data; @@ -513,8 +488,10 @@ static int drbg_hash_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[] md = ossl_prov_digest_md(&hash->digest); if (md != NULL) { - if (!ossl_drbg_verify_digest(libctx, md)) - return 0; /* Error already raised for us */ + if ((EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) { + ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED); + return 0; + } /* These are taken from SP 800-90 10.1 Table 2 */ hash->blocklen = EVP_MD_get_size(md); @@ -534,22 +511,6 @@ static int drbg_hash_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[] return ossl_drbg_set_ctx_params(ctx, params); } -static int drbg_hash_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - PROV_DRBG *drbg = (PROV_DRBG *)vctx; - int ret; - - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) - return 0; - - ret = drbg_hash_set_ctx_params_locked(vctx, params); - - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; -} - static const OSSL_PARAM *drbg_hash_settable_ctx_params(ossl_unused void *vctx, ossl_unused void *p_ctx) { @@ -584,5 +545,5 @@ const OSSL_DISPATCH ossl_drbg_hash_functions[] = { (void(*)(void))drbg_hash_verify_zeroization }, { OSSL_FUNC_RAND_GET_SEED, (void(*)(void))ossl_drbg_get_seed }, { OSSL_FUNC_RAND_CLEAR_SEED, (void(*)(void))ossl_drbg_clear_seed }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/rands/drbg_hmac.c b/openssl/src/providers/implementations/rands/drbg_hmac.c index 03b43a3c3..e68465a78 100644 --- a/openssl/src/providers/implementations/rands/drbg_hmac.c +++ b/openssl/src/providers/implementations/rands/drbg_hmac.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,11 +13,11 @@ #include #include #include +#include "prov/provider_util.h" #include "internal/thread_once.h" #include "prov/providercommon.h" #include "prov/implementations.h" #include "prov/provider_ctx.h" -#include "prov/hmac_drbg.h" #include "drbg_local.h" static OSSL_FUNC_rand_newctx_fn drbg_hmac_new_wrapper; @@ -32,7 +32,13 @@ static OSSL_FUNC_rand_gettable_ctx_params_fn drbg_hmac_gettable_ctx_params; static OSSL_FUNC_rand_get_ctx_params_fn drbg_hmac_get_ctx_params; static OSSL_FUNC_rand_verify_zeroization_fn drbg_hmac_verify_zeroization; -static int drbg_hmac_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[]); +typedef struct rand_drbg_hmac_st { + EVP_MAC_CTX *ctx; /* H(x) = HMAC_hash OR H(x) = KMAC */ + PROV_DIGEST digest; /* H(x) = hash(x) */ + size_t blocklen; + unsigned char K[EVP_MAX_MD_SIZE]; + unsigned char V[EVP_MAX_MD_SIZE]; +} PROV_DRBG_HMAC; /* * Called twice by SP800-90Ar1 10.1.2.2 HMAC_DRBG_Update_Process. @@ -85,11 +91,13 @@ static int do_hmac(PROV_DRBG_HMAC *hmac, unsigned char inbyte, * * Returns zero if an error occurs otherwise it returns 1. */ -static int drbg_hmac_update(PROV_DRBG_HMAC *hmac, +static int drbg_hmac_update(PROV_DRBG *drbg, const unsigned char *in1, size_t in1len, const unsigned char *in2, size_t in2len, const unsigned char *in3, size_t in3len) { + PROV_DRBG_HMAC *hmac = (PROV_DRBG_HMAC *)drbg->data; + /* (Steps 1-2) K = HMAC(K, V||0x00||provided_data). V = HMAC(K,V) */ if (!do_hmac(hmac, 0x00, in1, in1len, in2, in2len, in3, in3len)) return 0; @@ -111,11 +119,13 @@ static int drbg_hmac_update(PROV_DRBG_HMAC *hmac, * * Returns zero if an error occurs otherwise it returns 1. */ -int ossl_drbg_hmac_init(PROV_DRBG_HMAC *hmac, - const unsigned char *ent, size_t ent_len, - const unsigned char *nonce, size_t nonce_len, - const unsigned char *pstr, size_t pstr_len) +static int drbg_hmac_instantiate(PROV_DRBG *drbg, + const unsigned char *ent, size_t ent_len, + const unsigned char *nonce, size_t nonce_len, + const unsigned char *pstr, size_t pstr_len) { + PROV_DRBG_HMAC *hmac = (PROV_DRBG_HMAC *)drbg->data; + if (hmac->ctx == NULL) { ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_MAC); return 0; @@ -126,17 +136,9 @@ int ossl_drbg_hmac_init(PROV_DRBG_HMAC *hmac, /* (Step 3) V = 0x01 01...01 */ memset(hmac->V, 0x01, hmac->blocklen); /* (Step 4) (K,V) = HMAC_DRBG_Update(entropy||nonce||pers string, K, V) */ - return drbg_hmac_update(hmac, ent, ent_len, nonce, nonce_len, pstr, + return drbg_hmac_update(drbg, ent, ent_len, nonce, nonce_len, pstr, pstr_len); } -static int drbg_hmac_instantiate(PROV_DRBG *drbg, - const unsigned char *ent, size_t ent_len, - const unsigned char *nonce, size_t nonce_len, - const unsigned char *pstr, size_t pstr_len) -{ - return ossl_drbg_hmac_init((PROV_DRBG_HMAC *)drbg->data, ent, ent_len, - nonce, nonce_len, pstr, pstr_len); -} static int drbg_hmac_instantiate_wrapper(void *vdrbg, unsigned int strength, int prediction_resistance, @@ -145,23 +147,13 @@ static int drbg_hmac_instantiate_wrapper(void *vdrbg, unsigned int strength, const OSSL_PARAM params[]) { PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; - int ret = 0; - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) + if (!ossl_prov_is_running() || !drbg_hmac_set_ctx_params(drbg, params)) return 0; - - if (!ossl_prov_is_running() - || !drbg_hmac_set_ctx_params_locked(drbg, params)) - goto err; - ret = ossl_prov_drbg_instantiate(drbg, strength, prediction_resistance, - pstr, pstr_len); - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - return ret; + return ossl_prov_drbg_instantiate(drbg, strength, prediction_resistance, + pstr, pstr_len); } - /* * SP800-90Ar1 10.1.2.4 HMAC_DRBG_Reseed_Process: * @@ -176,10 +168,8 @@ static int drbg_hmac_reseed(PROV_DRBG *drbg, const unsigned char *ent, size_t ent_len, const unsigned char *adin, size_t adin_len) { - PROV_DRBG_HMAC *hmac = (PROV_DRBG_HMAC *)drbg->data; - /* (Step 2) (K,V) = HMAC_DRBG_Update(entropy||additional_input, K, V) */ - return drbg_hmac_update(hmac, ent, ent_len, adin, adin_len, NULL, 0); + return drbg_hmac_update(drbg, ent, ent_len, adin, adin_len, NULL, 0); } static int drbg_hmac_reseed_wrapper(void *vdrbg, int prediction_resistance, @@ -201,17 +191,18 @@ static int drbg_hmac_reseed_wrapper(void *vdrbg, int prediction_resistance, * * Returns zero if an error occurs otherwise it returns 1. */ -int ossl_drbg_hmac_generate(PROV_DRBG_HMAC *hmac, - unsigned char *out, size_t outlen, - const unsigned char *adin, size_t adin_len) +static int drbg_hmac_generate(PROV_DRBG *drbg, + unsigned char *out, size_t outlen, + const unsigned char *adin, size_t adin_len) { + PROV_DRBG_HMAC *hmac = (PROV_DRBG_HMAC *)drbg->data; EVP_MAC_CTX *ctx = hmac->ctx; const unsigned char *temp = hmac->V; /* (Step 2) if adin != NULL then (K,V) = HMAC_DRBG_Update(adin, K, V) */ if (adin != NULL && adin_len > 0 - && !drbg_hmac_update(hmac, adin, adin_len, NULL, 0, NULL, 0)) + && !drbg_hmac_update(drbg, adin, adin_len, NULL, 0, NULL, 0)) return 0; /* @@ -240,22 +231,14 @@ int ossl_drbg_hmac_generate(PROV_DRBG_HMAC *hmac, outlen -= hmac->blocklen; } /* (Step 6) (K,V) = HMAC_DRBG_Update(adin, K, V) */ - if (!drbg_hmac_update(hmac, adin, adin_len, NULL, 0, NULL, 0)) + if (!drbg_hmac_update(drbg, adin, adin_len, NULL, 0, NULL, 0)) return 0; return 1; } -static int drbg_hmac_generate(PROV_DRBG *drbg, - unsigned char *out, size_t outlen, - const unsigned char *adin, size_t adin_len) -{ - return ossl_drbg_hmac_generate((PROV_DRBG_HMAC *)drbg->data, out, outlen, - adin, adin_len); -} - -static int drbg_hmac_generate_wrapper(void *vdrbg, - unsigned char *out, size_t outlen, unsigned int strength, +static int drbg_hmac_generate_wrapper + (void *vdrbg, unsigned char *out, size_t outlen, unsigned int strength, int prediction_resistance, const unsigned char *adin, size_t adin_len) { PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; @@ -275,37 +258,17 @@ static int drbg_hmac_uninstantiate(PROV_DRBG *drbg) static int drbg_hmac_uninstantiate_wrapper(void *vdrbg) { - PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; - int ret; - - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) - return 0; - - ret = drbg_hmac_uninstantiate(drbg); - - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; + return drbg_hmac_uninstantiate((PROV_DRBG *)vdrbg); } static int drbg_hmac_verify_zeroization(void *vdrbg) { PROV_DRBG *drbg = (PROV_DRBG *)vdrbg; PROV_DRBG_HMAC *hmac = (PROV_DRBG_HMAC *)drbg->data; - int ret = 0; - - if (drbg->lock != NULL && !CRYPTO_THREAD_read_lock(drbg->lock)) - return 0; - - PROV_DRBG_VERIFY_ZEROIZATION(hmac->K); - PROV_DRBG_VERIFY_ZEROIZATION(hmac->V); - ret = 1; - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - return ret; + PROV_DRBG_VERYIFY_ZEROIZATION(hmac->K); + PROV_DRBG_VERYIFY_ZEROIZATION(hmac->V); + return 1; } static int drbg_hmac_new(PROV_DRBG *drbg) @@ -313,8 +276,10 @@ static int drbg_hmac_new(PROV_DRBG *drbg) PROV_DRBG_HMAC *hmac; hmac = OPENSSL_secure_zalloc(sizeof(*hmac)); - if (hmac == NULL) + if (hmac == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } drbg->data = hmac; /* See SP800-57 Part1 Rev4 5.6.1 Table 3 */ @@ -331,8 +296,7 @@ static int drbg_hmac_new(PROV_DRBG *drbg) static void *drbg_hmac_new_wrapper(void *provctx, void *parent, const OSSL_DISPATCH *parent_dispatch) { - return ossl_rand_drbg_new(provctx, parent, parent_dispatch, - &drbg_hmac_new, &drbg_hmac_free, + return ossl_rand_drbg_new(provctx, parent, parent_dispatch, &drbg_hmac_new, &drbg_hmac_instantiate, &drbg_hmac_uninstantiate, &drbg_hmac_reseed, &drbg_hmac_generate); } @@ -357,39 +321,24 @@ static int drbg_hmac_get_ctx_params(void *vdrbg, OSSL_PARAM params[]) const char *name; const EVP_MD *md; OSSL_PARAM *p; - int ret = 0, complete = 0; - - if (!ossl_drbg_get_ctx_params_no_lock(drbg, params, &complete)) - return 0; - - if (complete) - return 1; - - if (drbg->lock != NULL && !CRYPTO_THREAD_read_lock(drbg->lock)) - return 0; p = OSSL_PARAM_locate(params, OSSL_DRBG_PARAM_MAC); if (p != NULL) { if (hmac->ctx == NULL) - goto err; + return 0; name = EVP_MAC_get0_name(EVP_MAC_CTX_get0_mac(hmac->ctx)); if (!OSSL_PARAM_set_utf8_string(p, name)) - goto err; + return 0; } p = OSSL_PARAM_locate(params, OSSL_DRBG_PARAM_DIGEST); if (p != NULL) { md = ossl_prov_digest_md(&hmac->digest); if (md == NULL || !OSSL_PARAM_set_utf8_string(p, EVP_MD_get0_name(md))) - goto err; + return 0; } - ret = ossl_drbg_get_ctx_params(drbg, params); - err: - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; + return ossl_drbg_get_ctx_params(drbg, params); } static const OSSL_PARAM *drbg_hmac_gettable_ctx_params(ossl_unused void *vctx, @@ -404,7 +353,7 @@ static const OSSL_PARAM *drbg_hmac_gettable_ctx_params(ossl_unused void *vctx, return known_gettable_ctx_params; } -static int drbg_hmac_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[]) +static int drbg_hmac_set_ctx_params(void *vctx, const OSSL_PARAM params[]) { PROV_DRBG *ctx = (PROV_DRBG *)vctx; PROV_DRBG_HMAC *hmac = (PROV_DRBG_HMAC *)ctx->data; @@ -414,15 +363,22 @@ static int drbg_hmac_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[] if (!ossl_prov_digest_load_from_params(&hmac->digest, params, libctx)) return 0; + /* + * Confirm digest is allowed. We allow all digests that are not XOF + * (such as SHAKE). In FIPS mode, the fetch will fail for non-approved + * digests. + */ md = ossl_prov_digest_md(&hmac->digest); - if (md != NULL && !ossl_drbg_verify_digest(libctx, md)) - return 0; /* Error already raised for us */ + if (md != NULL && (EVP_MD_get_flags(md) & EVP_MD_FLAG_XOF) != 0) { + ERR_raise(ERR_LIB_PROV, PROV_R_XOF_DIGESTS_NOT_ALLOWED); + return 0; + } if (!ossl_prov_macctx_load_from_params(&hmac->ctx, params, NULL, NULL, NULL, libctx)) return 0; - if (md != NULL && hmac->ctx != NULL) { + if (hmac->ctx != NULL) { /* These are taken from SP 800-90 10.1 Table 2 */ hmac->blocklen = EVP_MD_get_size(md); /* See SP800-57 Part1 Rev4 5.6.1 Table 3 */ @@ -437,22 +393,6 @@ static int drbg_hmac_set_ctx_params_locked(void *vctx, const OSSL_PARAM params[] return ossl_drbg_set_ctx_params(ctx, params); } -static int drbg_hmac_set_ctx_params(void *vctx, const OSSL_PARAM params[]) -{ - PROV_DRBG *drbg = (PROV_DRBG *)vctx; - int ret; - - if (drbg->lock != NULL && !CRYPTO_THREAD_write_lock(drbg->lock)) - return 0; - - ret = drbg_hmac_set_ctx_params_locked(vctx, params); - - if (drbg->lock != NULL) - CRYPTO_THREAD_unlock(drbg->lock); - - return ret; -} - static const OSSL_PARAM *drbg_hmac_settable_ctx_params(ossl_unused void *vctx, ossl_unused void *p_ctx) { @@ -488,5 +428,5 @@ const OSSL_DISPATCH ossl_drbg_ossl_hmac_functions[] = { (void(*)(void))drbg_hmac_verify_zeroization }, { OSSL_FUNC_RAND_GET_SEED, (void(*)(void))ossl_drbg_get_seed }, { OSSL_FUNC_RAND_CLEAR_SEED, (void(*)(void))ossl_drbg_clear_seed }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/rands/drbg_local.h b/openssl/src/providers/implementations/rands/drbg_local.h index 902dfc937..8bc5df89c 100644 --- a/openssl/src/providers/implementations/rands/drbg_local.h +++ b/openssl/src/providers/implementations/rands/drbg_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -50,8 +50,13 @@ # define DRBG_MAX_LENGTH INT32_MAX /* The default nonce */ -/* ASCII: "OpenSSL NIST SP 800-90A DRBG", in hex for EBCDIC compatibility */ -#define DRBG_DEFAULT_PERS_STRING "\x4f\x70\x65\x6e\x53\x53\x4c\x20\x4e\x49\x53\x54\x20\x53\x50\x20\x38\x30\x30\x2d\x39\x30\x41\x20\x44\x52\x42\x47" +#ifdef CHARSET_EBCDIC +# define DRBG_DEFAULT_PERS_STRING { 0x4f, 0x70, 0x65, 0x6e, 0x53, 0x53, \ + 0x4c, 0x20, 0x4e, 0x49, 0x53, 0x54, 0x20, 0x53, 0x50, 0x20, 0x38, 0x30, \ + 0x30, 0x2d, 0x39, 0x30, 0x41, 0x20, 0x44, 0x52, 0x42, 0x47, 0x00}; +#else +# define DRBG_DEFAULT_PERS_STRING "OpenSSL NIST SP 800-90A DRBG" +#endif typedef struct prov_drbg_st PROV_DRBG; @@ -176,7 +181,6 @@ struct prov_drbg_st { PROV_DRBG *ossl_rand_drbg_new (void *provctx, void *parent, const OSSL_DISPATCH *parent_dispatch, int (*dnew)(PROV_DRBG *ctx), - void (*dfree)(void *vctx), int (*instantiate)(PROV_DRBG *drbg, const unsigned char *entropy, size_t entropylen, const unsigned char *nonce, size_t noncelen, @@ -207,13 +211,13 @@ OSSL_FUNC_rand_get_seed_fn ossl_drbg_get_seed; OSSL_FUNC_rand_clear_seed_fn ossl_drbg_clear_seed; /* Verify that an array of numeric values is all zero */ -#define PROV_DRBG_VERIFY_ZEROIZATION(v) \ +#define PROV_DRBG_VERYIFY_ZEROIZATION(v) \ { \ size_t i; \ \ for (i = 0; i < OSSL_NELEM(v); i++) \ if ((v)[i] != 0) \ - goto err; \ + return 0; \ } /* locking api */ @@ -223,11 +227,9 @@ OSSL_FUNC_rand_unlock_fn ossl_drbg_unlock; /* Common parameters for all of our DRBGs */ int ossl_drbg_get_ctx_params(PROV_DRBG *drbg, OSSL_PARAM params[]); -int ossl_drbg_get_ctx_params_no_lock(PROV_DRBG *drbg, OSSL_PARAM params[], - int *complete); int ossl_drbg_set_ctx_params(PROV_DRBG *drbg, const OSSL_PARAM params[]); -#define OSSL_PARAM_DRBG_SETTABLE_CTX_COMMON \ +#define OSSL_PARAM_DRBG_SETTABLE_CTX_COMMON \ OSSL_PARAM_uint(OSSL_DRBG_PARAM_RESEED_REQUESTS, NULL), \ OSSL_PARAM_uint64(OSSL_DRBG_PARAM_RESEED_TIME_INTERVAL, NULL) @@ -254,7 +256,4 @@ size_t ossl_crngt_get_entropy(PROV_DRBG *drbg, void ossl_crngt_cleanup_entropy(PROV_DRBG *drbg, unsigned char *out, size_t outlen); -/* Confirm digest is allowed to be used with a DRBG */ -int ossl_drbg_verify_digest(ossl_unused OSSL_LIB_CTX *libctx, const EVP_MD *md); - #endif diff --git a/openssl/src/providers/implementations/rands/seed_src.c b/openssl/src/providers/implementations/rands/seed_src.c index e8f7ec9ef..7a4b780bb 100644 --- a/openssl/src/providers/implementations/rands/seed_src.c +++ b/openssl/src/providers/implementations/rands/seed_src.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -53,8 +53,10 @@ static void *seed_src_new(void *provctx, void *parent, } s = OPENSSL_zalloc(sizeof(*s)); - if (s == NULL) + if (s == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } s->provctx = provctx; s->state = EVP_RAND_STATE_UNINITIALISED; @@ -104,7 +106,7 @@ static int seed_src_generate(void *vseed, unsigned char *out, size_t outlen, pool = ossl_rand_pool_new(strength, 1, outlen, outlen); if (pool == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_RAND_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; } @@ -177,32 +179,35 @@ static size_t seed_get_seed(void *vseed, unsigned char **pout, int prediction_resistance, const unsigned char *adin, size_t adin_len) { - size_t ret = 0; - size_t entropy_available = 0; - size_t i; - RAND_POOL *pool; - - pool = ossl_rand_pool_new(entropy, 1, min_len, max_len); - if (pool == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_RAND_LIB); + size_t bytes_needed; + unsigned char *p; + + /* + * Figure out how many bytes we need. + * This assumes that the seed sources provide eight bits of entropy + * per byte. For lower quality sources, the formula will need to be + * different. + */ + bytes_needed = entropy >= 0 ? (entropy + 7) / 8 : 0; + if (bytes_needed < min_len) + bytes_needed = min_len; + if (bytes_needed > max_len) { + ERR_raise(ERR_LIB_PROV, PROV_R_ENTROPY_SOURCE_STRENGTH_TOO_WEAK); return 0; } - /* Get entropy by polling system entropy sources. */ - entropy_available = ossl_pool_acquire_entropy(pool); - - if (entropy_available > 0) { - ret = ossl_rand_pool_length(pool); - *pout = ossl_rand_pool_detach(pool); - - /* xor the additional data into the output */ - for (i = 0 ; i < adin_len ; ++i) - (*pout)[i % ret] ^= adin[i]; - } else { - ERR_raise(ERR_LIB_PROV, PROV_R_ENTROPY_SOURCE_STRENGTH_TOO_WEAK); + p = OPENSSL_secure_malloc(bytes_needed); + if (p == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); + return 0; } - ossl_rand_pool_free(pool); - return ret; + if (seed_src_generate(vseed, p, bytes_needed, 0, prediction_resistance, + adin, adin_len) != 0) { + *pout = p; + return bytes_needed; + } + OPENSSL_secure_clear_free(p, bytes_needed); + return 0; } static void seed_clear_seed(ossl_unused void *vdrbg, @@ -244,5 +249,5 @@ const OSSL_DISPATCH ossl_seed_src_functions[] = { (void(*)(void))seed_src_verify_zeroization }, { OSSL_FUNC_RAND_GET_SEED, (void(*)(void))seed_get_seed }, { OSSL_FUNC_RAND_CLEAR_SEED, (void(*)(void))seed_clear_seed }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/rands/seeding/rand_cpu_arm64.c b/openssl/src/providers/implementations/rands/seeding/rand_cpu_arm64.c deleted file mode 100644 index a8530e02b..000000000 --- a/openssl/src/providers/implementations/rands/seeding/rand_cpu_arm64.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/cryptlib.h" -#include -#include "crypto/rand_pool.h" -#include "prov/seeding.h" - - -#ifdef OPENSSL_RAND_SEED_RDCPU -#include "crypto/arm_arch.h" - -size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len); - -static size_t get_hardware_random_value(unsigned char *buf, size_t len); - -/* - * Acquire entropy using Arm-specific cpu instructions - * - * Uses the RNDRRS instruction. RNDR is never needed since - * RNDRRS will always be available if RNDR is an available - * instruction. - * - * Returns the total entropy count, if it exceeds the requested - * entropy count. Otherwise, returns an entropy count of 0. - */ -size_t ossl_prov_acquire_entropy_from_cpu(RAND_POOL *pool) -{ - size_t bytes_needed; - unsigned char *buffer; - - bytes_needed = ossl_rand_pool_bytes_needed(pool, 1 /*entropy_factor*/); - if (bytes_needed > 0) { - buffer = ossl_rand_pool_add_begin(pool, bytes_needed); - - if (buffer != NULL) { - if (get_hardware_random_value(buffer, bytes_needed) == bytes_needed) - ossl_rand_pool_add_end(pool, bytes_needed, 8 * bytes_needed); - else - ossl_rand_pool_add_end(pool, 0, 0); - } - } - - return ossl_rand_pool_entropy_available(pool); -} - -static size_t get_hardware_random_value(unsigned char *buf, size_t len) -{ - /* Always use RNDRRS or nothing */ - if (OPENSSL_armcap_P & ARMV8_RNG) { - if (OPENSSL_rndrrs_bytes(buf, len) != len) - return 0; - } else { - return 0; - } - return len; -} - -#else -NON_EMPTY_TRANSLATION_UNIT -#endif /* OPENSSL_RAND_SEED_RDCPU */ diff --git a/openssl/src/providers/implementations/rands/seeding/rand_unix.c b/openssl/src/providers/implementations/rands/seeding/rand_unix.c index 9a936d800..fede8441d 100644 --- a/openssl/src/providers/implementations/rands/seeding/rand_unix.c +++ b/openssl/src/providers/implementations/rands/seeding/rand_unix.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,8 +17,8 @@ #include #include "crypto/rand_pool.h" #include "crypto/rand.h" +#include #include "internal/dso.h" -#include "internal/nelem.h" #include "prov/seeding.h" #ifdef __linux @@ -50,6 +50,7 @@ # include static uint64_t get_time_stamp(void); +static uint64_t get_timer_bits(void); /* Macro to convert two thirty two bit values into a sixty four bit one */ # define TWO32TO64(a, b) ((((uint64_t)(a)) << 32) + (b)) @@ -107,7 +108,7 @@ static uint64_t get_time_stamp(void); #endif #if !(defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32) \ - || defined(OPENSSL_SYS_VMS) || defined(OPENSSL_SYS_VXWORKS) \ + || defined(OPENSSL_SYS_VXWORKS) \ || defined(OPENSSL_SYS_UEFI)) # if defined(OPENSSL_SYS_VOS) @@ -175,7 +176,7 @@ size_t ossl_pool_acquire_entropy(RAND_POOL *pool) /* Get wall clock time, take 8 bits. */ clock_gettime(CLOCK_REALTIME, &ts); v = (unsigned char)(ts.tv_nsec & 0xFF); - ossl_rand_pool_add(pool, arg, &v, sizeof(v), 2); + ossl_rand_pool_add(pool, arg, &v, sizeof(v) , 2); } return ossl_rand_pool_entropy_available(pool); } @@ -299,8 +300,6 @@ static ssize_t sysctl_random(char *buf, size_t buflen) # endif # elif defined(__hppa__) # define __NR_getrandom (__NR_Linux + 339) -# elif defined(__sparc__) -# define __NR_getrandom 347 # elif defined(__ia64__) # define __NR_getrandom 1339 # elif defined(__alpha__) @@ -319,7 +318,9 @@ static ssize_t sysctl_random(char *buf, size_t buflen) # define __NR_getrandom 352 # elif defined(__cris__) # define __NR_getrandom 356 -# else /* generic (f.e. aarch64, loongarch, loongarch64) */ +# elif defined(__aarch64__) +# define __NR_getrandom 278 +# else /* generic */ # define __NR_getrandom 278 # endif # endif @@ -354,7 +355,7 @@ static ssize_t syscall_random(void *buf, size_t buflen) * internally. So we need to check errno for ENOSYS */ # if !defined(__DragonFly__) && !defined(__NetBSD__) -# if defined(__GNUC__) && __GNUC__>=2 && defined(__ELF__) && !defined(__hpux) +# if defined(__GNUC__) && __GNUC__>=2 && defined(__ELF__) extern int getentropy(void *buffer, size_t length) __attribute__((weak)); if (getentropy != NULL) { @@ -395,10 +396,6 @@ static ssize_t syscall_random(void *buf, size_t buflen) # elif (defined(__DragonFly__) && __DragonFly_version >= 500700) \ || (defined(__NetBSD__) && __NetBSD_Version >= 1000000000) return getrandom(buf, buflen, 0); -# elif defined(__wasi__) - if (getentropy(buf, buflen) == 0) - return (ssize_t)buflen; - return -1; # else errno = ENOSYS; return -1; @@ -510,7 +507,7 @@ static int wait_random_seeded(void) * So the handle might have been closed or even reused for opening * another file. */ -static int check_random_device(struct random_device *rd) +static int check_random_device(struct random_device * rd) { struct stat st; @@ -528,7 +525,7 @@ static int check_random_device(struct random_device *rd) static int get_random_device(size_t n) { struct stat st; - struct random_device *rd = &random_devices[n]; + struct random_device * rd = &random_devices[n]; /* reuse existing file descriptor if it is (still) valid */ if (check_random_device(rd)) @@ -557,7 +554,7 @@ static int get_random_device(size_t n) */ static void close_random_device(size_t n) { - struct random_device *rd = &random_devices[n]; + struct random_device * rd = &random_devices[n]; if (check_random_device(rd)) close(rd->fd); @@ -775,6 +772,31 @@ int ossl_pool_add_nonce_data(RAND_POOL *pool) return ossl_rand_pool_add(pool, (unsigned char *)&data, sizeof(data), 0); } +int ossl_rand_pool_add_additional_data(RAND_POOL *pool) +{ + struct { + int fork_id; + CRYPTO_THREAD_ID tid; + uint64_t time; + } data; + + /* Erase the entire structure including any padding */ + memset(&data, 0, sizeof(data)); + + /* + * Add some noise from the thread id and a high resolution timer. + * The fork_id adds some extra fork-safety. + * The thread id adds a little randomness if the drbg is accessed + * concurrently (which is the case for the drbg). + */ + data.fork_id = openssl_get_fork_id(); + data.tid = CRYPTO_THREAD_get_current_id(); + data.time = get_timer_bits(); + + return ossl_rand_pool_add(pool, (unsigned char *)&data, sizeof(data), 0); +} + + /* * Get the current time with the highest possible resolution * @@ -804,5 +826,55 @@ static uint64_t get_time_stamp(void) return time(NULL); } +/* + * Get an arbitrary timer value of the highest possible resolution + * + * The timer value is added as random noise to the additional data, + * which is not considered a trusted entropy sourec, so any result + * is acceptable. + */ +static uint64_t get_timer_bits(void) +{ + uint64_t res = OPENSSL_rdtsc(); + + if (res != 0) + return res; + +# if defined(__sun) + return gethrtime(); +# elif defined(_AIX) + { + timebasestruct_t t; + + read_wall_time(&t, TIMEBASE_SZ); + return TWO32TO64(t.tb_high, t.tb_low); + } +# elif defined(OSSL_POSIX_TIMER_OKAY) + { + struct timespec ts; + +# ifdef CLOCK_BOOTTIME +# define CLOCK_TYPE CLOCK_BOOTTIME +# elif defined(_POSIX_MONOTONIC_CLOCK) +# define CLOCK_TYPE CLOCK_MONOTONIC +# else +# define CLOCK_TYPE CLOCK_REALTIME +# endif + + if (clock_gettime(CLOCK_TYPE, &ts) == 0) + return TWO32TO64(ts.tv_sec, ts.tv_nsec); + } +# endif +# if defined(__unix__) \ + || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) + { + struct timeval tv; + + if (gettimeofday(&tv, NULL) == 0) + return TWO32TO64(tv.tv_sec, tv.tv_usec); + } +# endif + return time(NULL); +} #endif /* (defined(OPENSSL_SYS_UNIX) && !defined(OPENSSL_SYS_VXWORKS)) || defined(__DJGPP__) */ diff --git a/openssl/src/providers/implementations/rands/seeding/rand_vms.c b/openssl/src/providers/implementations/rands/seeding/rand_vms.c deleted file mode 100644 index 4ff879491..000000000 --- a/openssl/src/providers/implementations/rands/seeding/rand_vms.c +++ /dev/null @@ -1,616 +0,0 @@ -/* - * Copyright 2001-2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/e_os.h" - -#define __NEW_STARLET 1 /* New starlet definitions since VMS 7.0 */ -#include -#include "internal/cryptlib.h" -#include "internal/nelem.h" -#include -#include "crypto/rand.h" -#include "crypto/rand_pool.h" -#include "prov/seeding.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef __DECC -# pragma message disable DOLLARID -#endif - -#include /* SYS$GET_ENTROPY presence */ - -#ifndef OPENSSL_RAND_SEED_OS -# error "Unsupported seeding method configured; must be os" -#endif - -/* - * DATA COLLECTION METHOD - * ====================== - * - * This is a method to get low quality entropy. - * It works by collecting all kinds of statistical data that - * VMS offers and using them as random seed. - */ - -/* We need to make sure we have the right size pointer in some cases */ -#if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size save -# pragma pointer_size 32 -#endif -typedef uint32_t *uint32_t__ptr32; -#if __INITIAL_POINTER_SIZE == 64 -# pragma pointer_size restore -#endif - -struct item_st { - short length, code; /* length is number of bytes */ -}; - -static const struct item_st DVI_item_data[] = { - {4, DVI$_ERRCNT}, - {4, DVI$_REFCNT}, -}; - -static const struct item_st JPI_item_data[] = { - {4, JPI$_BUFIO}, - {4, JPI$_CPUTIM}, - {4, JPI$_DIRIO}, - {4, JPI$_IMAGECOUNT}, - {4, JPI$_PAGEFLTS}, - {4, JPI$_PID}, - {4, JPI$_PPGCNT}, - {4, JPI$_WSPEAK}, - /* - * Note: the direct result is just a 32-bit address. However, it points - * to a list of 4 32-bit words, so we make extra space for them so we can - * do in-place replacement of values - */ - {16, JPI$_FINALEXC}, -}; - -static const struct item_st JPI_item_data_64bit[] = { - {8, JPI$_LAST_LOGIN_I}, - {8, JPI$_LOGINTIM}, -}; - -static const struct item_st RMI_item_data[] = { - {4, RMI$_COLPG}, - {4, RMI$_MWAIT}, - {4, RMI$_CEF}, - {4, RMI$_PFW}, - {4, RMI$_LEF}, - {4, RMI$_LEFO}, - {4, RMI$_HIB}, - {4, RMI$_HIBO}, - {4, RMI$_SUSP}, - {4, RMI$_SUSPO}, - {4, RMI$_FPG}, - {4, RMI$_COM}, - {4, RMI$_COMO}, - {4, RMI$_CUR}, -#if defined __alpha - {4, RMI$_FRLIST}, - {4, RMI$_MODLIST}, -#endif - {4, RMI$_FAULTS}, - {4, RMI$_PREADS}, - {4, RMI$_PWRITES}, - {4, RMI$_PWRITIO}, - {4, RMI$_PREADIO}, - {4, RMI$_GVALFLTS}, - {4, RMI$_WRTINPROG}, - {4, RMI$_FREFLTS}, - {4, RMI$_DZROFLTS}, - {4, RMI$_SYSFAULTS}, - {4, RMI$_ISWPCNT}, - {4, RMI$_DIRIO}, - {4, RMI$_BUFIO}, - {4, RMI$_MBREADS}, - {4, RMI$_MBWRITES}, - {4, RMI$_LOGNAM}, - {4, RMI$_FCPCALLS}, - {4, RMI$_FCPREAD}, - {4, RMI$_FCPWRITE}, - {4, RMI$_FCPCACHE}, - {4, RMI$_FCPCPU}, - {4, RMI$_FCPHIT}, - {4, RMI$_FCPSPLIT}, - {4, RMI$_FCPFAULT}, - {4, RMI$_ENQNEW}, - {4, RMI$_ENQCVT}, - {4, RMI$_DEQ}, - {4, RMI$_BLKAST}, - {4, RMI$_ENQWAIT}, - {4, RMI$_ENQNOTQD}, - {4, RMI$_DLCKSRCH}, - {4, RMI$_DLCKFND}, - {4, RMI$_NUMLOCKS}, - {4, RMI$_NUMRES}, - {4, RMI$_ARRLOCPK}, - {4, RMI$_DEPLOCPK}, - {4, RMI$_ARRTRAPK}, - {4, RMI$_TRCNGLOS}, - {4, RMI$_RCVBUFFL}, - {4, RMI$_ENQNEWLOC}, - {4, RMI$_ENQNEWIN}, - {4, RMI$_ENQNEWOUT}, - {4, RMI$_ENQCVTLOC}, - {4, RMI$_ENQCVTIN}, - {4, RMI$_ENQCVTOUT}, - {4, RMI$_DEQLOC}, - {4, RMI$_DEQIN}, - {4, RMI$_DEQOUT}, - {4, RMI$_BLKLOC}, - {4, RMI$_BLKIN}, - {4, RMI$_BLKOUT}, - {4, RMI$_DIRIN}, - {4, RMI$_DIROUT}, - /* We currently get a fault when trying these */ -#if 0 - {140, RMI$_MSCP_EVERYTHING}, /* 35 32-bit words */ - {152, RMI$_DDTM_ALL}, /* 38 32-bit words */ - {80, RMI$_TMSCP_EVERYTHING} /* 20 32-bit words */ -#endif - {4, RMI$_LPZ_PAGCNT}, - {4, RMI$_LPZ_HITS}, - {4, RMI$_LPZ_MISSES}, - {4, RMI$_LPZ_EXPCNT}, - {4, RMI$_LPZ_ALLOCF}, - {4, RMI$_LPZ_ALLOC2}, - {4, RMI$_ACCESS}, - {4, RMI$_ALLOC}, - {4, RMI$_FCPCREATE}, - {4, RMI$_VOLWAIT}, - {4, RMI$_FCPTURN}, - {4, RMI$_FCPERASE}, - {4, RMI$_OPENS}, - {4, RMI$_FIDHIT}, - {4, RMI$_FIDMISS}, - {4, RMI$_FILHDR_HIT}, - {4, RMI$_DIRFCB_HIT}, - {4, RMI$_DIRFCB_MISS}, - {4, RMI$_DIRDATA_HIT}, - {4, RMI$_EXTHIT}, - {4, RMI$_EXTMISS}, - {4, RMI$_QUOHIT}, - {4, RMI$_QUOMISS}, - {4, RMI$_STORAGMAP_HIT}, - {4, RMI$_VOLLCK}, - {4, RMI$_SYNCHLCK}, - {4, RMI$_SYNCHWAIT}, - {4, RMI$_ACCLCK}, - {4, RMI$_XQPCACHEWAIT}, - {4, RMI$_DIRDATA_MISS}, - {4, RMI$_FILHDR_MISS}, - {4, RMI$_STORAGMAP_MISS}, - {4, RMI$_PROCCNTMAX}, - {4, RMI$_PROCBATCNT}, - {4, RMI$_PROCINTCNT}, - {4, RMI$_PROCNETCNT}, - {4, RMI$_PROCSWITCHCNT}, - {4, RMI$_PROCBALSETCNT}, - {4, RMI$_PROCLOADCNT}, - {4, RMI$_BADFLTS}, - {4, RMI$_EXEFAULTS}, - {4, RMI$_HDRINSWAPS}, - {4, RMI$_HDROUTSWAPS}, - {4, RMI$_IOPAGCNT}, - {4, RMI$_ISWPCNTPG}, - {4, RMI$_OSWPCNT}, - {4, RMI$_OSWPCNTPG}, - {4, RMI$_RDFAULTS}, - {4, RMI$_TRANSFLTS}, - {4, RMI$_WRTFAULTS}, -#if defined __alpha - {4, RMI$_USERPAGES}, -#endif - {4, RMI$_VMSPAGES}, - {4, RMI$_TTWRITES}, - {4, RMI$_BUFOBJPAG}, - {4, RMI$_BUFOBJPAGPEAK}, - {4, RMI$_BUFOBJPAGS01}, - {4, RMI$_BUFOBJPAGS2}, - {4, RMI$_BUFOBJPAGMAXS01}, - {4, RMI$_BUFOBJPAGMAXS2}, - {4, RMI$_BUFOBJPAGPEAKS01}, - {4, RMI$_BUFOBJPAGPEAKS2}, - {4, RMI$_BUFOBJPGLTMAXS01}, - {4, RMI$_BUFOBJPGLTMAXS2}, - {4, RMI$_DLCK_INCMPLT}, - {4, RMI$_DLCKMSGS_IN}, - {4, RMI$_DLCKMSGS_OUT}, - {4, RMI$_MCHKERRS}, - {4, RMI$_MEMERRS}, -}; - -static const struct item_st RMI_item_data_64bit[] = { -#if defined __ia64 - {8, RMI$_FRLIST}, - {8, RMI$_MODLIST}, -#endif - {8, RMI$_LCKMGR_REQCNT}, - {8, RMI$_LCKMGR_REQTIME}, - {8, RMI$_LCKMGR_SPINCNT}, - {8, RMI$_LCKMGR_SPINTIME}, - {8, RMI$_CPUINTSTK}, - {8, RMI$_CPUMPSYNCH}, - {8, RMI$_CPUKERNEL}, - {8, RMI$_CPUEXEC}, - {8, RMI$_CPUSUPER}, - {8, RMI$_CPUUSER}, -#if defined __ia64 - {8, RMI$_USERPAGES}, -#endif - {8, RMI$_TQETOTAL}, - {8, RMI$_TQESYSUB}, - {8, RMI$_TQEUSRTIMR}, - {8, RMI$_TQEUSRWAKE}, -}; - -static const struct item_st SYI_item_data[] = { - {4, SYI$_PAGEFILE_FREE}, -}; - -/* - * Input: - * items_data - an array of lengths and codes - * items_data_num - number of elements in that array - * - * Output: - * items - pre-allocated ILE3 array to be filled. - * It's assumed to have items_data_num elements plus - * one extra for the terminating NULL element - * databuffer - pre-allocated 32-bit word array. - * - * Returns the number of elements used in databuffer - */ -static size_t prepare_item_list(const struct item_st *items_input, - size_t items_input_num, - ILE3 *items, - uint32_t__ptr32 databuffer) -{ - size_t data_sz = 0; - - for (; items_input_num-- > 0; items_input++, items++) { - - items->ile3$w_code = items_input->code; - /* Special treatment of JPI$_FINALEXC */ - if (items->ile3$w_code == JPI$_FINALEXC) - items->ile3$w_length = 4; - else - items->ile3$w_length = items_input->length; - - items->ile3$ps_bufaddr = databuffer; - items->ile3$ps_retlen_addr = 0; - - databuffer += items_input->length / sizeof(databuffer[0]); - data_sz += items_input->length; - } - /* Terminating NULL entry */ - items->ile3$w_length = items->ile3$w_code = 0; - items->ile3$ps_bufaddr = items->ile3$ps_retlen_addr = NULL; - - return data_sz / sizeof(databuffer[0]); -} - -static void massage_JPI(ILE3 *items) -{ - /* - * Special treatment of JPI$_FINALEXC - * The result of that item's data buffer is a 32-bit address to a list of - * 4 32-bit words. - */ - for (; items->ile3$w_length != 0; items++) { - if (items->ile3$w_code == JPI$_FINALEXC) { - uint32_t *data = items->ile3$ps_bufaddr; - uint32_t *ptr = (uint32_t *)*data; - size_t j; - - /* - * We know we made space for 4 32-bit words, so we can do in-place - * replacement. - */ - for (j = 0; j < 4; j++) - data[j] = ptr[j]; - - break; - } - } -} - -/* - * This number expresses how many bits of data contain 1 bit of entropy. - * - * For the moment, we assume about 0.05 entropy bits per data bit, or 1 - * bit of entropy per 20 data bits. - */ -#define ENTROPY_FACTOR 20 - -size_t data_collect_method(RAND_POOL *pool) -{ - ILE3 JPI_items_64bit[OSSL_NELEM(JPI_item_data_64bit) + 1]; - ILE3 RMI_items_64bit[OSSL_NELEM(RMI_item_data_64bit) + 1]; - ILE3 DVI_items[OSSL_NELEM(DVI_item_data) + 1]; - ILE3 JPI_items[OSSL_NELEM(JPI_item_data) + 1]; - ILE3 RMI_items[OSSL_NELEM(RMI_item_data) + 1]; - ILE3 SYI_items[OSSL_NELEM(SYI_item_data) + 1]; - union { - /* This ensures buffer starts at 64 bit boundary */ - uint64_t dummy; - uint32_t buffer[OSSL_NELEM(JPI_item_data_64bit) * 2 - + OSSL_NELEM(RMI_item_data_64bit) * 2 - + OSSL_NELEM(DVI_item_data) - + OSSL_NELEM(JPI_item_data) - + OSSL_NELEM(RMI_item_data) - + OSSL_NELEM(SYI_item_data) - + 4 /* For JPI$_FINALEXC */]; - } data; - size_t total_elems = 0; - size_t total_length = 0; - size_t bytes_needed = ossl_rand_pool_bytes_needed(pool, ENTROPY_FACTOR); - size_t bytes_remaining = ossl_rand_pool_bytes_remaining(pool); - - /* Take all the 64-bit items first, to ensure proper alignment of data */ - total_elems += - prepare_item_list(JPI_item_data_64bit, OSSL_NELEM(JPI_item_data_64bit), - JPI_items_64bit, &data.buffer[total_elems]); - total_elems += - prepare_item_list(RMI_item_data_64bit, OSSL_NELEM(RMI_item_data_64bit), - RMI_items_64bit, &data.buffer[total_elems]); - /* Now the 32-bit items */ - total_elems += prepare_item_list(DVI_item_data, OSSL_NELEM(DVI_item_data), - DVI_items, &data.buffer[total_elems]); - total_elems += prepare_item_list(JPI_item_data, OSSL_NELEM(JPI_item_data), - JPI_items, &data.buffer[total_elems]); - total_elems += prepare_item_list(RMI_item_data, OSSL_NELEM(RMI_item_data), - RMI_items, &data.buffer[total_elems]); - total_elems += prepare_item_list(SYI_item_data, OSSL_NELEM(SYI_item_data), - SYI_items, &data.buffer[total_elems]); - total_length = total_elems * sizeof(data.buffer[0]); - - /* Fill data.buffer with various info bits from this process */ - { - uint32_t status; - uint32_t efn; - IOSB iosb; - $DESCRIPTOR(SYSDEVICE, "SYS$SYSDEVICE:"); - - if ((status = sys$getdviw(EFN$C_ENF, 0, &SYSDEVICE, DVI_items, - 0, 0, 0, 0, 0)) != SS$_NORMAL) { - lib$signal(status); - return 0; - } - if ((status = sys$getjpiw(EFN$C_ENF, 0, 0, JPI_items_64bit, 0, 0, 0)) - != SS$_NORMAL) { - lib$signal(status); - return 0; - } - if ((status = sys$getjpiw(EFN$C_ENF, 0, 0, JPI_items, 0, 0, 0)) - != SS$_NORMAL) { - lib$signal(status); - return 0; - } - if ((status = sys$getsyiw(EFN$C_ENF, 0, 0, SYI_items, 0, 0, 0)) - != SS$_NORMAL) { - lib$signal(status); - return 0; - } - /* - * The RMI service is a bit special, as there is no synchronous - * variant, so we MUST create an event flag to synchronise on. - */ - if ((status = lib$get_ef(&efn)) != SS$_NORMAL) { - lib$signal(status); - return 0; - } - if ((status = sys$getrmi(efn, 0, 0, RMI_items_64bit, &iosb, 0, 0)) - != SS$_NORMAL) { - lib$signal(status); - return 0; - } - if ((status = sys$synch(efn, &iosb)) != SS$_NORMAL) { - lib$signal(status); - return 0; - } - if (iosb.iosb$l_getxxi_status != SS$_NORMAL) { - lib$signal(iosb.iosb$l_getxxi_status); - return 0; - } - if ((status = sys$getrmi(efn, 0, 0, RMI_items, &iosb, 0, 0)) - != SS$_NORMAL) { - lib$signal(status); - return 0; - } - if ((status = sys$synch(efn, &iosb)) != SS$_NORMAL) { - lib$signal(status); - return 0; - } - if (iosb.iosb$l_getxxi_status != SS$_NORMAL) { - lib$signal(iosb.iosb$l_getxxi_status); - return 0; - } - if ((status = lib$free_ef(&efn)) != SS$_NORMAL) { - lib$signal(status); - return 0; - } - } - - massage_JPI(JPI_items); - - /* - * If we can't feed the requirements from the caller, we're in deep trouble. - */ - if (!ossl_assert(total_length >= bytes_needed)) { - ERR_raise_data(ERR_LIB_RAND, RAND_R_RANDOM_POOL_UNDERFLOW, - "Needed: %zu, Available: %zu", - bytes_needed, total_length); - return 0; - } - - /* - * Try not to overfeed the pool - */ - if (total_length > bytes_remaining) - total_length = bytes_remaining; - - /* We give the pessimistic value for the amount of entropy */ - ossl_rand_pool_add(pool, (unsigned char *)data.buffer, total_length, - 8 * total_length / ENTROPY_FACTOR); - return ossl_rand_pool_entropy_available(pool); -} - -/* - * SYS$GET_ENTROPY METHOD - * ====================== - * - * This is a high entropy method based on a new system service that is - * based on getentropy() from FreeBSD 12. It's only used if available, - * and its availability is detected at run-time. - * - * We assume that this function provides full entropy random output. - */ -#define PUBLIC_VECTORS "SYS$LIBRARY:SYS$PUBLIC_VECTORS.EXE" -#define GET_ENTROPY "SYS$GET_ENTROPY" - -static int get_entropy_address_flag = 0; -static int (*get_entropy_address)(void *buffer, size_t buffer_size) = NULL; -static int init_get_entropy_address(void) -{ - if (get_entropy_address_flag == 0) - get_entropy_address = dlsym(dlopen(PUBLIC_VECTORS, 0), GET_ENTROPY); - get_entropy_address_flag = 1; - return get_entropy_address != NULL; -} - -size_t get_entropy_method(RAND_POOL *pool) -{ - /* - * The documentation says that SYS$GET_ENTROPY will give a maximum of - * 256 bytes of data. - */ - unsigned char buffer[256]; - size_t bytes_needed; - size_t bytes_to_get = 0; - uint32_t status; - - for (bytes_needed = ossl_rand_pool_bytes_needed(pool, 1); - bytes_needed > 0; - bytes_needed -= bytes_to_get) { - bytes_to_get = - bytes_needed > sizeof(buffer) ? sizeof(buffer) : bytes_needed; - - status = get_entropy_address(buffer, bytes_to_get); - if (status == SS$_RETRY) { - /* Set to zero so the loop doesn't diminish |bytes_needed| */ - bytes_to_get = 0; - /* Should sleep some amount of time */ - continue; - } - - if (status != SS$_NORMAL) { - lib$signal(status); - return 0; - } - - ossl_rand_pool_add(pool, buffer, bytes_to_get, 8 * bytes_to_get); - } - - return ossl_rand_pool_entropy_available(pool); -} - -/* - * MAIN ENTROPY ACQUISITION FUNCTIONS - * ================================== - * - * These functions are called by the RAND / DRBG functions - */ - -size_t ossl_pool_acquire_entropy(RAND_POOL *pool) -{ - if (init_get_entropy_address()) - return get_entropy_method(pool); - return data_collect_method(pool); -} - -int ossl_pool_add_nonce_data(RAND_POOL *pool) -{ - /* - * Two variables to ensure that two nonces won't ever be the same - */ - static unsigned __int64 last_time = 0; - static unsigned __int32 last_seq = 0; - - struct { - pid_t pid; - CRYPTO_THREAD_ID tid; - unsigned __int64 time; - unsigned __int32 seq; - } data; - - /* Erase the entire structure including any padding */ - memset(&data, 0, sizeof(data)); - - /* - * Add process id, thread id, a timestamp, and a sequence number in case - * the same time stamp is repeated, to ensure that the nonce is unique - * with high probability for different process instances. - * - * The normal OpenVMS time is specified to be high granularity (100ns), - * but the time update granularity given by sys$gettim() may be lower. - * - * OpenVMS version 8.4 (which is the latest for Alpha and Itanium) and - * on have sys$gettim_prec() as well, which is supposedly having a better - * time update granularity, but tests on Itanium (and even Alpha) have - * shown that compared with sys$gettim(), the difference is marginal, - * so of very little significance in terms of entropy. - * Given that, and that it's a high ask to expect everyone to have - * upgraded to OpenVMS version 8.4, only sys$gettim() is used, and a - * sequence number is added as well, in case sys$gettim() returns the - * same time value more than once. - * - * This function is assumed to be called under thread lock, and does - * therefore not take concurrency into account. - */ - data.pid = getpid(); - data.tid = CRYPTO_THREAD_get_current_id(); - data.seq = 0; - sys$gettim((void*)&data.time); - - if (data.time == last_time) { - data.seq = ++last_seq; - } else { - last_time = data.time; - last_seq = 0; - } - - return ossl_rand_pool_add(pool, (unsigned char *)&data, sizeof(data), 0); -} - -int ossl_rand_pool_init(void) -{ - return 1; -} - -void ossl_rand_pool_cleanup(void) -{ -} - -void ossl_rand_pool_keep_random_devices_open(int keep) -{ -} diff --git a/openssl/src/providers/implementations/rands/seeding/rand_vxworks.c b/openssl/src/providers/implementations/rands/seeding/rand_vxworks.c index a28fbd799..12be9357b 100644 --- a/openssl/src/providers/implementations/rands/seeding/rand_vxworks.c +++ b/openssl/src/providers/implementations/rands/seeding/rand_vxworks.c @@ -76,6 +76,26 @@ void ossl_rand_pool_keep_random_devices_open(int keep) { } +int ossl_rand_pool_add_additional_data(RAND_POOL *pool) +{ + struct { + CRYPTO_THREAD_ID tid; + uint64_t time; + } data; + + memset(&data, 0, sizeof(data)); + + /* + * Add some noise from the thread id and a high resolution timer. + * The thread id adds a little randomness if the drbg is accessed + * concurrently (which is the case for the drbg). + */ + data.tid = CRYPTO_THREAD_get_current_id(); + data.time = get_timer_bits(); + + return ossl_rand_pool_add(pool, (unsigned char *)&data, sizeof(data), 0); +} + int ossl_pool_add_nonce_data(RAND_POOL *pool) { struct { @@ -116,7 +136,7 @@ size_t ossl_pool_acquire_entropy(RAND_POOL *pool) RANDOM_NUM_GEN_STATUS status = randStatus(); if ((status == RANDOM_NUM_GEN_ENOUGH_ENTROPY) - || (status == RANDOM_NUM_GEN_MAX_ENTROPY)) { + || (status == RANDOM_NUM_GEN_MAX_ENTROPY) ) { result = randBytes(buffer, bytes_needed); if (result == OK) ossl_rand_pool_add_end(pool, bytes_needed, 8 * bytes_needed); diff --git a/openssl/src/providers/implementations/rands/seeding/rand_win.c b/openssl/src/providers/implementations/rands/seeding/rand_win.c index ee2d3e4d7..704705425 100644 --- a/openssl/src/providers/implementations/rands/seeding/rand_win.c +++ b/openssl/src/providers/implementations/rands/seeding/rand_win.c @@ -28,9 +28,7 @@ # ifdef USE_BCRYPTGENRANDOM # include -# ifdef _MSC_VER -# pragma comment(lib, "bcrypt.lib") -# endif +# pragma comment(lib, "bcrypt.lib") # ifndef STATUS_SUCCESS # define STATUS_SUCCESS ((NTSTATUS)0x00000000L) # endif @@ -147,6 +145,26 @@ int ossl_pool_add_nonce_data(RAND_POOL *pool) return ossl_rand_pool_add(pool, (unsigned char *)&data, sizeof(data), 0); } +int ossl_rand_pool_add_additional_data(RAND_POOL *pool) +{ + struct { + DWORD tid; + LARGE_INTEGER time; + } data; + + /* Erase the entire structure including any padding */ + memset(&data, 0, sizeof(data)); + + /* + * Add some noise from the thread id and a high resolution timer. + * The thread id adds a little randomness if the drbg is accessed + * concurrently (which is the case for the drbg). + */ + data.tid = GetCurrentThreadId(); + QueryPerformanceCounter(&data.time); + return ossl_rand_pool_add(pool, (unsigned char *)&data, sizeof(data), 0); +} + int ossl_rand_pool_init(void) { return 1; diff --git a/openssl/src/providers/implementations/rands/test_rng.c b/openssl/src/providers/implementations/rands/test_rng.c index 57b36469c..4e7fed0fc 100644 --- a/openssl/src/providers/implementations/rands/test_rng.c +++ b/openssl/src/providers/implementations/rands/test_rng.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -40,14 +39,12 @@ static OSSL_FUNC_rand_get_seed_fn test_rng_get_seed; typedef struct { void *provctx; - unsigned int generate; int state; unsigned int strength; size_t max_request; unsigned char *entropy, *nonce; size_t entropy_len, entropy_pos, nonce_len; CRYPTO_RWLOCK *lock; - uint32_t seed; } PROV_TEST_RNG; static void *test_rng_new(void *provctx, void *parent, @@ -89,7 +86,6 @@ static int test_rng_instantiate(void *vtest, unsigned int strength, t->state = EVP_RAND_STATE_READY; t->entropy_pos = 0; - t->seed = 221953166; /* Value doesn't matter, so long as it isn't zero */ return 1; } @@ -103,45 +99,16 @@ static int test_rng_uninstantiate(void *vtest) return 1; } -static unsigned char gen_byte(PROV_TEST_RNG *t) -{ - uint32_t n; - - /* - * Implement the 32 bit xorshift as suggested by George Marsaglia in: - * https://doi.org/10.18637/jss.v008.i14 - * - * This is a very fast PRNG so there is no need to extract bytes one at a - * time and use the entire value each time. - */ - n = t->seed; - n ^= n << 13; - n ^= n >> 17; - n ^= n << 5; - t->seed = n; - - return n & 0xff; -} - static int test_rng_generate(void *vtest, unsigned char *out, size_t outlen, unsigned int strength, int prediction_resistance, const unsigned char *adin, size_t adin_len) { PROV_TEST_RNG *t = (PROV_TEST_RNG *)vtest; - size_t i; - if (strength > t->strength) + if (strength > t->strength || t->entropy_len - t->entropy_pos < outlen) return 0; - if (t->generate) { - for (i = 0; i < outlen; i++) - out[i] = gen_byte(t); - } else { - if (t->entropy_len - t->entropy_pos < outlen) - return 0; - - memcpy(out, t->entropy + t->entropy_pos, outlen); - t->entropy_pos += outlen; - } + memcpy(out, t->entropy + t->entropy_pos, outlen); + t->entropy_pos += outlen; return 1; } @@ -156,23 +123,15 @@ static int test_rng_reseed(ossl_unused void *vtest, } static size_t test_rng_nonce(void *vtest, unsigned char *out, - unsigned int strength, size_t min_noncelen, + unsigned int strength, + ossl_unused size_t min_noncelen, ossl_unused size_t max_noncelen) { PROV_TEST_RNG *t = (PROV_TEST_RNG *)vtest; - size_t i; - if (strength > t->strength) + if (t->nonce == NULL || strength > t->strength) return 0; - if (t->generate) { - for (i = 0; i < min_noncelen; i++) - out[i] = gen_byte(t); - return min_noncelen; - } - - if (t->nonce == NULL) - return 0; if (out != NULL) memcpy(out, t->nonce, t->nonce_len); return t->nonce_len; @@ -194,10 +153,6 @@ static int test_rng_get_ctx_params(void *vtest, OSSL_PARAM params[]) p = OSSL_PARAM_locate(params, OSSL_RAND_PARAM_MAX_REQUEST); if (p != NULL && !OSSL_PARAM_set_size_t(p, t->max_request)) return 0; - - p = OSSL_PARAM_locate(params, OSSL_RAND_PARAM_GENERATE); - if (p != NULL && OSSL_PARAM_set_uint(p, t->generate)) - return 0; return 1; } @@ -208,7 +163,6 @@ static const OSSL_PARAM *test_rng_gettable_ctx_params(ossl_unused void *vtest, OSSL_PARAM_int(OSSL_RAND_PARAM_STATE, NULL), OSSL_PARAM_uint(OSSL_RAND_PARAM_STRENGTH, NULL), OSSL_PARAM_size_t(OSSL_RAND_PARAM_MAX_REQUEST, NULL), - OSSL_PARAM_uint(OSSL_RAND_PARAM_GENERATE, NULL), OSSL_PARAM_END }; return known_gettable_ctx_params; @@ -249,12 +203,9 @@ static int test_rng_set_ctx_params(void *vtest, const OSSL_PARAM params[]) } p = OSSL_PARAM_locate_const(params, OSSL_RAND_PARAM_MAX_REQUEST); - if (p != NULL && !OSSL_PARAM_get_size_t(p, &t->max_request)) + if (p != NULL && !OSSL_PARAM_get_size_t(p, &t->max_request)) return 0; - p = OSSL_PARAM_locate_const(params, OSSL_RAND_PARAM_GENERATE); - if (p != NULL && !OSSL_PARAM_get_uint(p, &t->generate)) - return 0; return 1; } @@ -266,7 +217,6 @@ static const OSSL_PARAM *test_rng_settable_ctx_params(ossl_unused void *vtest, OSSL_PARAM_octet_string(OSSL_RAND_PARAM_TEST_NONCE, NULL, 0), OSSL_PARAM_uint(OSSL_RAND_PARAM_STRENGTH, NULL), OSSL_PARAM_size_t(OSSL_RAND_PARAM_MAX_REQUEST, NULL), - OSSL_PARAM_uint(OSSL_RAND_PARAM_GENERATE, NULL), OSSL_PARAM_END }; return known_settable_ctx_params; @@ -342,5 +292,5 @@ const OSSL_DISPATCH ossl_test_rng_functions[] = { { OSSL_FUNC_RAND_VERIFY_ZEROIZATION, (void(*)(void))test_rng_verify_zeroization }, { OSSL_FUNC_RAND_GET_SEED, (void(*)(void))test_rng_get_seed }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/signature/dsa_sig.c b/openssl/src/providers/implementations/signature/dsa_sig.c index b89a0f683..28fd7c498 100644 --- a/openssl/src/providers/implementations/signature/dsa_sig.c +++ b/openssl/src/providers/implementations/signature/dsa_sig.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "internal/nelem.h" #include "internal/sizes.h" @@ -74,9 +75,6 @@ typedef struct { */ unsigned int flag_allow_md : 1; - /* If this is set to 1 then the generated k is not random */ - unsigned int nonce_type; - char mdname[OSSL_MAX_NAME_SIZE]; /* The Algorithm Identifier of the combined signature algorithm */ @@ -114,6 +112,7 @@ static void *dsa_newctx(void *provctx, const char *propq) if (propq != NULL && (pdsactx->propq = OPENSSL_strdup(propq)) == NULL) { OPENSSL_free(pdsactx); pdsactx = NULL; + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); } return pdsactx; } @@ -252,9 +251,7 @@ static int dsa_sign(void *vpdsactx, unsigned char *sig, size_t *siglen, if (mdsize != 0 && tbslen != mdsize) return 0; - ret = ossl_dsa_sign_int(0, tbs, tbslen, sig, &sltmp, pdsactx->dsa, - pdsactx->nonce_type, pdsactx->mdname, - pdsactx->libctx, pdsactx->propq); + ret = ossl_dsa_sign_int(0, tbs, tbslen, sig, &sltmp, pdsactx->dsa); if (ret <= 0) return 0; @@ -461,17 +458,12 @@ static int dsa_get_ctx_params(void *vpdsactx, OSSL_PARAM *params) if (p != NULL && !OSSL_PARAM_set_utf8_string(p, pdsactx->mdname)) return 0; - p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_NONCE_TYPE); - if (p != NULL && !OSSL_PARAM_set_uint(p, pdsactx->nonce_type)) - return 0; - return 1; } static const OSSL_PARAM known_gettable_ctx_params[] = { OSSL_PARAM_octet_string(OSSL_SIGNATURE_PARAM_ALGORITHM_ID, NULL, 0), OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_DIGEST, NULL, 0), - OSSL_PARAM_uint(OSSL_SIGNATURE_PARAM_NONCE_TYPE, NULL), OSSL_PARAM_END }; @@ -507,10 +499,6 @@ static int dsa_set_ctx_params(void *vpdsactx, const OSSL_PARAM params[]) if (!dsa_setup_md(pdsactx, mdname, mdprops)) return 0; } - p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_NONCE_TYPE); - if (p != NULL - && !OSSL_PARAM_get_uint(p, &pdsactx->nonce_type)) - return 0; return 1; } @@ -518,7 +506,6 @@ static int dsa_set_ctx_params(void *vpdsactx, const OSSL_PARAM params[]) static const OSSL_PARAM settable_ctx_params[] = { OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_DIGEST, NULL, 0), OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_uint(OSSL_SIGNATURE_PARAM_NONCE_TYPE, NULL), OSSL_PARAM_END }; @@ -610,5 +597,5 @@ const OSSL_DISPATCH ossl_dsa_signature_functions[] = { (void (*)(void))dsa_set_ctx_md_params }, { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_MD_PARAMS, (void (*)(void))dsa_settable_ctx_md_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/signature/ecdsa_sig.c b/openssl/src/providers/implementations/signature/ecdsa_sig.c index fe65ed8dc..865d49d10 100644 --- a/openssl/src/providers/implementations/signature/ecdsa_sig.c +++ b/openssl/src/providers/implementations/signature/ecdsa_sig.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,7 +25,6 @@ #include "internal/nelem.h" #include "internal/sizes.h" #include "internal/cryptlib.h" -#include "internal/deterministic_nonce.h" #include "prov/providercommon.h" #include "prov/implementations.h" #include "prov/provider_ctx.h" @@ -103,8 +102,6 @@ typedef struct { */ unsigned int kattest; #endif - /* If this is set then the generated k is not random */ - unsigned int nonce_type; } PROV_ECDSA_CTX; static void *ecdsa_newctx(void *provctx, const char *propq) @@ -123,6 +120,7 @@ static void *ecdsa_newctx(void *provctx, const char *propq) if (propq != NULL && (ctx->propq = OPENSSL_strdup(propq)) == NULL) { OPENSSL_free(ctx); ctx = NULL; + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); } return ctx; } @@ -195,15 +193,7 @@ static int ecdsa_sign(void *vctx, unsigned char *sig, size_t *siglen, if (ctx->mdsize != 0 && tbslen != ctx->mdsize) return 0; - if (ctx->nonce_type != 0) { - ret = ossl_ecdsa_deterministic_sign(tbs, tbslen, sig, &sltmp, - ctx->ec, ctx->nonce_type, - ctx->mdname, - ctx->libctx, ctx->propq); - } else { - ret = ECDSA_sign_ex(0, tbs, tbslen, sig, &sltmp, ctx->kinv, ctx->r, - ctx->ec); - } + ret = ECDSA_sign_ex(0, tbs, tbslen, sig, &sltmp, ctx->kinv, ctx->r, ctx->ec); if (ret <= 0) return 0; @@ -467,10 +457,6 @@ static int ecdsa_get_ctx_params(void *vctx, OSSL_PARAM *params) : EVP_MD_get0_name(ctx->md))) return 0; - p = OSSL_PARAM_locate(params, OSSL_SIGNATURE_PARAM_NONCE_TYPE); - if (p != NULL && !OSSL_PARAM_set_uint(p, ctx->nonce_type)) - return 0; - return 1; } @@ -478,7 +464,6 @@ static const OSSL_PARAM known_gettable_ctx_params[] = { OSSL_PARAM_octet_string(OSSL_SIGNATURE_PARAM_ALGORITHM_ID, NULL, 0), OSSL_PARAM_size_t(OSSL_SIGNATURE_PARAM_DIGEST_SIZE, NULL), OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_DIGEST, NULL, 0), - OSSL_PARAM_uint(OSSL_SIGNATURE_PARAM_NONCE_TYPE, NULL), OSSL_PARAM_END }; @@ -529,10 +514,6 @@ static int ecdsa_set_ctx_params(void *vctx, const OSSL_PARAM params[]) return 0; ctx->mdsize = mdsize; } - p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_NONCE_TYPE); - if (p != NULL - && !OSSL_PARAM_get_uint(p, &ctx->nonce_type)) - return 0; return 1; } @@ -542,7 +523,6 @@ static const OSSL_PARAM settable_ctx_params[] = { OSSL_PARAM_size_t(OSSL_SIGNATURE_PARAM_DIGEST_SIZE, NULL), OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_PROPERTIES, NULL, 0), OSSL_PARAM_uint(OSSL_SIGNATURE_PARAM_KAT, NULL), - OSSL_PARAM_uint(OSSL_SIGNATURE_PARAM_NONCE_TYPE, NULL), OSSL_PARAM_END }; @@ -635,5 +615,5 @@ const OSSL_DISPATCH ossl_ecdsa_signature_functions[] = { (void (*)(void))ecdsa_set_ctx_md_params }, { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_MD_PARAMS, (void (*)(void))ecdsa_settable_ctx_md_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/signature/eddsa_sig.c b/openssl/src/providers/implementations/signature/eddsa_sig.c index 8594af39e..eb1a76912 100644 --- a/openssl/src/providers/implementations/signature/eddsa_sig.c +++ b/openssl/src/providers/implementations/signature/eddsa_sig.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "internal/nelem.h" #include "internal/sizes.h" @@ -43,24 +44,6 @@ static int s390x_ed448_digestverify(const ECX_KEY *edkey, #endif /* S390X_EC_ASM */ -enum ID_EdDSA_INSTANCE { - ID_NOT_SET = 0, - ID_Ed25519, - ID_Ed25519ctx, - ID_Ed25519ph, - ID_Ed448, - ID_Ed448ph -}; - -#define SN_Ed25519 "Ed25519" -#define SN_Ed25519ph "Ed25519ph" -#define SN_Ed25519ctx "Ed25519ctx" -#define SN_Ed448 "Ed448" -#define SN_Ed448ph "Ed448ph" - -#define EDDSA_MAX_CONTEXT_STRING_LEN 255 -#define EDDSA_PREHASH_OUTPUT_LEN 64 - static OSSL_FUNC_signature_newctx_fn eddsa_newctx; static OSSL_FUNC_signature_digest_sign_init_fn eddsa_digest_signverify_init; static OSSL_FUNC_signature_digest_sign_fn ed25519_digest_sign; @@ -71,55 +54,6 @@ static OSSL_FUNC_signature_freectx_fn eddsa_freectx; static OSSL_FUNC_signature_dupctx_fn eddsa_dupctx; static OSSL_FUNC_signature_get_ctx_params_fn eddsa_get_ctx_params; static OSSL_FUNC_signature_gettable_ctx_params_fn eddsa_gettable_ctx_params; -static OSSL_FUNC_signature_set_ctx_params_fn eddsa_set_ctx_params; -static OSSL_FUNC_signature_settable_ctx_params_fn eddsa_settable_ctx_params; - -/* there are five EdDSA instances: - - Ed25519 - Ed25519ph - Ed25519ctx - Ed448 - Ed448ph - - Quoting from RFC 8032, Section 5.1: - - For Ed25519, dom2(f,c) is the empty string. The phflag value is - irrelevant. The context (if present at all) MUST be empty. This - causes the scheme to be one and the same with the Ed25519 scheme - published earlier. - - For Ed25519ctx, phflag=0. The context input SHOULD NOT be empty. - - For Ed25519ph, phflag=1 and PH is SHA512 instead. That is, the input - is hashed using SHA-512 before signing with Ed25519. - - Quoting from RFC 8032, Section 5.2: - - Ed448ph is the same but with PH being SHAKE256(x, 64) and phflag - being 1, i.e., the input is hashed before signing with Ed448 with a - hash constant modified. - - Value of context is set by signer and verifier (maximum of 255 - octets; the default is empty string) and has to match octet by octet - for verification to be successful. - - Quoting from RFC 8032, Section 2: - - dom2(x, y) The blank octet string when signing or verifying - Ed25519. Otherwise, the octet string: "SigEd25519 no - Ed25519 collisions" || octet(x) || octet(OLEN(y)) || - y, where x is in range 0-255 and y is an octet string - of at most 255 octets. "SigEd25519 no Ed25519 - collisions" is in ASCII (32 octets). - - dom4(x, y) The octet string "SigEd448" || octet(x) || - octet(OLEN(y)) || y, where x is in range 0-255 and y - is an octet string of at most 255 octets. "SigEd448" - is in ASCII (8 octets). - - Note above that x is the pre-hash flag, and y is the context string. -*/ typedef struct { OSSL_LIB_CTX *libctx; @@ -129,19 +63,6 @@ typedef struct { unsigned char aid_buf[OSSL_MAX_ALGORITHM_ID_SIZE]; unsigned char *aid; size_t aid_len; - - /* id indicating the EdDSA instance */ - int instance_id; - - unsigned int dom2_flag : 1; - unsigned int prehash_flag : 1; - - /* indicates that a non-empty context string is required, as in Ed25519ctx */ - unsigned int context_string_flag : 1; - - unsigned char context_string[EDDSA_MAX_CONTEXT_STRING_LEN]; - size_t context_string_len; - } PROV_EDDSA_CTX; static void *eddsa_newctx(void *provctx, const char *propq_unused) @@ -152,8 +73,10 @@ static void *eddsa_newctx(void *provctx, const char *propq_unused) return NULL; peddsactx = OPENSSL_zalloc(sizeof(PROV_EDDSA_CTX)); - if (peddsactx == NULL) + if (peddsactx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } peddsactx->libctx = PROV_LIBCTX_OF(provctx); @@ -162,7 +85,7 @@ static void *eddsa_newctx(void *provctx, const char *propq_unused) static int eddsa_digest_signverify_init(void *vpeddsactx, const char *mdname, void *vedkey, - const OSSL_PARAM params[]) + ossl_unused const OSSL_PARAM params[]) { PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx; ECX_KEY *edkey = (ECX_KEY *)vedkey; @@ -179,7 +102,8 @@ static int eddsa_digest_signverify_init(void *vpeddsactx, const char *mdname, if (edkey == NULL) { if (peddsactx->key != NULL) - return eddsa_set_ctx_params(peddsactx, params); + /* there is nothing to do on reinit */ + return 1; ERR_raise(ERR_LIB_PROV, PROV_R_NO_KEY_SET); return 0; } @@ -189,11 +113,6 @@ static int eddsa_digest_signverify_init(void *vpeddsactx, const char *mdname, return 0; } - peddsactx->dom2_flag = 0; - peddsactx->prehash_flag = 0; - peddsactx->context_string_flag = 0; - peddsactx->context_string_len = 0; - /* * We do not care about DER writing errors. * All it really means is that for some reason, there's no @@ -206,11 +125,9 @@ static int eddsa_digest_signverify_init(void *vpeddsactx, const char *mdname, switch (edkey->type) { case ECX_KEY_TYPE_ED25519: ret = ret && ossl_DER_w_algorithmIdentifier_ED25519(&pkt, -1, edkey); - peddsactx->instance_id = ID_Ed25519; break; case ECX_KEY_TYPE_ED448: ret = ret && ossl_DER_w_algorithmIdentifier_ED448(&pkt, -1, edkey); - peddsactx->instance_id = ID_Ed448; break; default: /* Should never happen */ @@ -226,9 +143,6 @@ static int eddsa_digest_signverify_init(void *vpeddsactx, const char *mdname, peddsactx->key = edkey; - if (!eddsa_set_ctx_params(peddsactx, params)) - return 0; - return 1; } @@ -238,8 +152,6 @@ int ed25519_digest_sign(void *vpeddsactx, unsigned char *sigret, { PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx; const ECX_KEY *edkey = peddsactx->key; - uint8_t md[EVP_MAX_MD_SIZE]; - size_t mdlen; if (!ossl_prov_is_running()) return 0; @@ -252,38 +164,12 @@ int ed25519_digest_sign(void *vpeddsactx, unsigned char *sigret, ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } - if (edkey->privkey == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_NOT_A_PRIVATE_KEY); - return 0; - } #ifdef S390X_EC_ASM - /* s390x_ed25519_digestsign() does not yet support dom2 or context-strings. - fall back to non-accelerated sign if those options are set. */ - if (S390X_CAN_SIGN(ED25519) - && !peddsactx->dom2_flag - && !peddsactx->context_string_flag - && peddsactx->context_string_len == 0) { - if (s390x_ed25519_digestsign(edkey, sigret, tbs, tbslen) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SIGN); - return 0; - } - *siglen = ED25519_SIGSIZE; - return 1; - } + if (S390X_CAN_SIGN(ED25519)) + return s390x_ed25519_digestsign(edkey, sigret, tbs, tbslen); #endif /* S390X_EC_ASM */ - - if (peddsactx->prehash_flag) { - if (!EVP_Q_digest(peddsactx->libctx, SN_sha512, NULL, tbs, tbslen, md, &mdlen) - || mdlen != EDDSA_PREHASH_OUTPUT_LEN) - return 0; - tbs = md; - tbslen = mdlen; - } - if (ossl_ed25519_sign(sigret, tbs, tbslen, edkey->pubkey, edkey->privkey, - peddsactx->dom2_flag, peddsactx->prehash_flag, peddsactx->context_string_flag, - peddsactx->context_string, peddsactx->context_string_len, - peddsactx->libctx, NULL) == 0) { + peddsactx->libctx, NULL) == 0) { ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SIGN); return 0; } @@ -291,41 +177,12 @@ int ed25519_digest_sign(void *vpeddsactx, unsigned char *sigret, return 1; } -/* EVP_Q_digest() does not allow variable output length for XOFs, - so we use this function */ -static int ed448_shake256(OSSL_LIB_CTX *libctx, - const char *propq, - const uint8_t *in, size_t inlen, - uint8_t *out, size_t outlen) -{ - int ret = 0; - EVP_MD_CTX *hash_ctx = EVP_MD_CTX_new(); - EVP_MD *shake256 = EVP_MD_fetch(libctx, SN_shake256, propq); - - if (hash_ctx == NULL || shake256 == NULL) - goto err; - - if (!EVP_DigestInit_ex(hash_ctx, shake256, NULL) - || !EVP_DigestUpdate(hash_ctx, in, inlen) - || !EVP_DigestFinalXOF(hash_ctx, out, outlen)) - goto err; - - ret = 1; - - err: - EVP_MD_CTX_free(hash_ctx); - EVP_MD_free(shake256); - return ret; -} - int ed448_digest_sign(void *vpeddsactx, unsigned char *sigret, size_t *siglen, size_t sigsize, const unsigned char *tbs, size_t tbslen) { PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx; const ECX_KEY *edkey = peddsactx->key; - uint8_t md[EDDSA_PREHASH_OUTPUT_LEN]; - size_t mdlen = sizeof(md); if (!ossl_prov_is_running()) return 0; @@ -338,36 +195,12 @@ int ed448_digest_sign(void *vpeddsactx, unsigned char *sigret, ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } - if (edkey->privkey == NULL) { - ERR_raise(ERR_LIB_PROV, PROV_R_NOT_A_PRIVATE_KEY); - return 0; - } #ifdef S390X_EC_ASM - /* s390x_ed448_digestsign() does not yet support context-strings or pre-hashing. - fall back to non-accelerated sign if a context-string or pre-hasing is provided. */ - if (S390X_CAN_SIGN(ED448) - && peddsactx->context_string_len == 0 - && peddsactx->prehash_flag == 0) { - if (s390x_ed448_digestsign(edkey, sigret, tbs, tbslen) == 0) { - ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SIGN); - return 0; - } - *siglen = ED448_SIGSIZE; - return 1; - } + if (S390X_CAN_SIGN(ED448)) + return s390x_ed448_digestsign(edkey, sigret, tbs, tbslen); #endif /* S390X_EC_ASM */ - - if (peddsactx->prehash_flag) { - if (!ed448_shake256(peddsactx->libctx, NULL, tbs, tbslen, md, mdlen)) - return 0; - tbs = md; - tbslen = mdlen; - } - - if (ossl_ed448_sign(peddsactx->libctx, sigret, tbs, tbslen, - edkey->pubkey, edkey->privkey, - peddsactx->context_string, peddsactx->context_string_len, - peddsactx->prehash_flag, edkey->propq) == 0) { + if (ossl_ed448_sign(peddsactx->libctx, sigret, tbs, tbslen, edkey->pubkey, + edkey->privkey, NULL, 0, edkey->propq) == 0) { ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SIGN); return 0; } @@ -381,34 +214,16 @@ int ed25519_digest_verify(void *vpeddsactx, const unsigned char *sig, { PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx; const ECX_KEY *edkey = peddsactx->key; - uint8_t md[EVP_MAX_MD_SIZE]; - size_t mdlen; if (!ossl_prov_is_running() || siglen != ED25519_SIGSIZE) return 0; #ifdef S390X_EC_ASM - /* s390x_ed25519_digestverify() does not yet support dom2 or context-strings. - fall back to non-accelerated verify if those options are set. */ - if (S390X_CAN_SIGN(ED25519) - && !peddsactx->dom2_flag - && !peddsactx->context_string_flag - && peddsactx->context_string_len == 0) { + if (S390X_CAN_SIGN(ED25519)) return s390x_ed25519_digestverify(edkey, sig, tbs, tbslen); - } #endif /* S390X_EC_ASM */ - if (peddsactx->prehash_flag) { - if (!EVP_Q_digest(peddsactx->libctx, SN_sha512, NULL, tbs, tbslen, md, &mdlen) - || mdlen != EDDSA_PREHASH_OUTPUT_LEN) - return 0; - tbs = md; - tbslen = mdlen; - } - return ossl_ed25519_verify(tbs, tbslen, sig, edkey->pubkey, - peddsactx->dom2_flag, peddsactx->prehash_flag, peddsactx->context_string_flag, - peddsactx->context_string, peddsactx->context_string_len, peddsactx->libctx, edkey->propq); } @@ -418,32 +233,17 @@ int ed448_digest_verify(void *vpeddsactx, const unsigned char *sig, { PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx; const ECX_KEY *edkey = peddsactx->key; - uint8_t md[EDDSA_PREHASH_OUTPUT_LEN]; - size_t mdlen = sizeof(md); if (!ossl_prov_is_running() || siglen != ED448_SIGSIZE) return 0; #ifdef S390X_EC_ASM - /* s390x_ed448_digestverify() does not yet support context-strings or pre-hashing. - fall back to non-accelerated verify if a context-string or pre-hasing is provided. */ - if (S390X_CAN_SIGN(ED448) - && peddsactx->context_string_len == 0 - && peddsactx->prehash_flag == 0) { + if (S390X_CAN_SIGN(ED448)) return s390x_ed448_digestverify(edkey, sig, tbs, tbslen); - } #endif /* S390X_EC_ASM */ - if (peddsactx->prehash_flag) { - if (!ed448_shake256(peddsactx->libctx, NULL, tbs, tbslen, md, mdlen)) - return 0; - tbs = md; - tbslen = mdlen; - } - return ossl_ed448_verify(peddsactx->libctx, tbs, tbslen, sig, edkey->pubkey, - peddsactx->context_string, peddsactx->context_string_len, - peddsactx->prehash_flag, edkey->propq); + NULL, 0, edkey->propq); } static void eddsa_freectx(void *vpeddsactx) @@ -500,8 +300,6 @@ static int eddsa_get_ctx_params(void *vpeddsactx, OSSL_PARAM *params) static const OSSL_PARAM known_gettable_ctx_params[] = { OSSL_PARAM_octet_string(OSSL_SIGNATURE_PARAM_ALGORITHM_ID, NULL, 0), - OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_INSTANCE, NULL, 0), - OSSL_PARAM_octet_string(OSSL_SIGNATURE_PARAM_CONTEXT_STRING, NULL, 0), OSSL_PARAM_END }; @@ -511,84 +309,6 @@ static const OSSL_PARAM *eddsa_gettable_ctx_params(ossl_unused void *vpeddsactx, return known_gettable_ctx_params; } -static int eddsa_set_ctx_params(void *vpeddsactx, const OSSL_PARAM params[]) -{ - PROV_EDDSA_CTX *peddsactx = (PROV_EDDSA_CTX *)vpeddsactx; - const OSSL_PARAM *p; - - if (peddsactx == NULL) - return 0; - if (params == NULL) - return 1; - - p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_INSTANCE); - if (p != NULL) { - char instance_name[OSSL_MAX_NAME_SIZE] = ""; - char *pinstance_name = instance_name; - - if (!OSSL_PARAM_get_utf8_string(p, &pinstance_name, sizeof(instance_name))) - return 0; - - if (OPENSSL_strcasecmp(pinstance_name, SN_Ed25519) == 0) { - peddsactx->instance_id = ID_Ed25519; - if (peddsactx->key->type != ECX_KEY_TYPE_ED25519) return 0; - peddsactx->dom2_flag = 0; - peddsactx->prehash_flag = 0; - peddsactx->context_string_flag = 0; - } else if (OPENSSL_strcasecmp(pinstance_name, SN_Ed25519ctx) == 0) { - peddsactx->instance_id = ID_Ed25519ctx; - if (peddsactx->key->type != ECX_KEY_TYPE_ED25519) return 0; - peddsactx->dom2_flag = 1; - peddsactx->prehash_flag = 0; - peddsactx->context_string_flag = 1; - } else if (OPENSSL_strcasecmp(pinstance_name, SN_Ed25519ph) == 0) { - peddsactx->instance_id = ID_Ed25519ph; - if (peddsactx->key->type != ECX_KEY_TYPE_ED25519) return 0; - peddsactx->dom2_flag = 1; - peddsactx->prehash_flag = 1; - peddsactx->context_string_flag = 0; - } else if (OPENSSL_strcasecmp(pinstance_name, SN_Ed448) == 0) { - peddsactx->instance_id = ID_Ed448; - if (peddsactx->key->type != ECX_KEY_TYPE_ED448) return 0; - peddsactx->prehash_flag = 0; - peddsactx->context_string_flag = 0; - } else if (OPENSSL_strcasecmp(pinstance_name, SN_Ed448ph) == 0) { - peddsactx->instance_id = ID_Ed448ph; - if (peddsactx->key->type != ECX_KEY_TYPE_ED448) return 0; - peddsactx->prehash_flag = 1; - peddsactx->context_string_flag = 0; - } else { - /* we did not recognize the instance */ - return 0; - } - - } - - p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_CONTEXT_STRING); - if (p != NULL) { - void *vp_context_string = peddsactx->context_string; - - if (!OSSL_PARAM_get_octet_string(p, &vp_context_string, sizeof(peddsactx->context_string), &(peddsactx->context_string_len))) { - peddsactx->context_string_len = 0; - return 0; - } - } - - return 1; -} - -static const OSSL_PARAM settable_ctx_params[] = { - OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_INSTANCE, NULL, 0), - OSSL_PARAM_octet_string(OSSL_SIGNATURE_PARAM_CONTEXT_STRING, NULL, 0), - OSSL_PARAM_END -}; - -static const OSSL_PARAM *eddsa_settable_ctx_params(ossl_unused void *vpeddsactx, - ossl_unused void *provctx) -{ - return settable_ctx_params; -} - const OSSL_DISPATCH ossl_ed25519_signature_functions[] = { { OSSL_FUNC_SIGNATURE_NEWCTX, (void (*)(void))eddsa_newctx }, { OSSL_FUNC_SIGNATURE_DIGEST_SIGN_INIT, @@ -604,10 +324,7 @@ const OSSL_DISPATCH ossl_ed25519_signature_functions[] = { { OSSL_FUNC_SIGNATURE_GET_CTX_PARAMS, (void (*)(void))eddsa_get_ctx_params }, { OSSL_FUNC_SIGNATURE_GETTABLE_CTX_PARAMS, (void (*)(void))eddsa_gettable_ctx_params }, - { OSSL_FUNC_SIGNATURE_SET_CTX_PARAMS, (void (*)(void))eddsa_set_ctx_params }, - { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_PARAMS, - (void (*)(void))eddsa_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; const OSSL_DISPATCH ossl_ed448_signature_functions[] = { @@ -625,10 +342,7 @@ const OSSL_DISPATCH ossl_ed448_signature_functions[] = { { OSSL_FUNC_SIGNATURE_GET_CTX_PARAMS, (void (*)(void))eddsa_get_ctx_params }, { OSSL_FUNC_SIGNATURE_GETTABLE_CTX_PARAMS, (void (*)(void))eddsa_gettable_ctx_params }, - { OSSL_FUNC_SIGNATURE_SET_CTX_PARAMS, (void (*)(void))eddsa_set_ctx_params }, - { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_PARAMS, - (void (*)(void))eddsa_settable_ctx_params }, - OSSL_DISPATCH_END + { 0, NULL } }; #ifdef S390X_EC_ASM diff --git a/openssl/src/providers/implementations/signature/mac_legacy_sig.c b/openssl/src/providers/implementations/signature/mac_legacy_sig.c index b25a74506..54fcf143f 100644 --- a/openssl/src/providers/implementations/signature/mac_legacy_sig.c +++ b/openssl/src/providers/implementations/signature/mac_legacy_sig.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -29,6 +29,7 @@ static OSSL_FUNC_signature_newctx_fn mac_hmac_newctx; static OSSL_FUNC_signature_newctx_fn mac_siphash_newctx; static OSSL_FUNC_signature_newctx_fn mac_poly1305_newctx; static OSSL_FUNC_signature_newctx_fn mac_cmac_newctx; +static OSSL_FUNC_signature_newctx_fn mac_eia3_newctx; static OSSL_FUNC_signature_digest_sign_init_fn mac_digest_sign_init; static OSSL_FUNC_signature_digest_sign_update_fn mac_digest_sign_update; static OSSL_FUNC_signature_digest_sign_final_fn mac_digest_sign_final; @@ -39,6 +40,7 @@ static OSSL_FUNC_signature_settable_ctx_params_fn mac_hmac_settable_ctx_params; static OSSL_FUNC_signature_settable_ctx_params_fn mac_siphash_settable_ctx_params; static OSSL_FUNC_signature_settable_ctx_params_fn mac_poly1305_settable_ctx_params; static OSSL_FUNC_signature_settable_ctx_params_fn mac_cmac_settable_ctx_params; +static OSSL_FUNC_signature_settable_ctx_params_fn mac_eia3_settable_ctx_params; typedef struct { OSSL_LIB_CTX *libctx; @@ -60,8 +62,10 @@ static void *mac_newctx(void *provctx, const char *propq, const char *macname) return NULL; pmacctx->libctx = PROV_LIBCTX_OF(provctx); - if (propq != NULL && (pmacctx->propq = OPENSSL_strdup(propq)) == NULL) + if (propq != NULL && (pmacctx->propq = OPENSSL_strdup(propq)) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; + } mac = EVP_MAC_fetch(pmacctx->libctx, macname, propq); if (mac == NULL) @@ -92,6 +96,7 @@ MAC_NEWCTX(hmac, "HMAC") MAC_NEWCTX(siphash, "SIPHASH") MAC_NEWCTX(poly1305, "POLY1305") MAC_NEWCTX(cmac, "CMAC") +MAC_NEWCTX(eia3, "EIA3") static int mac_digest_sign_init(void *vpmacctx, const char *mdname, void *vkey, const OSSL_PARAM params[]) @@ -240,6 +245,7 @@ MAC_SETTABLE_CTX_PARAMS(hmac, "HMAC") MAC_SETTABLE_CTX_PARAMS(siphash, "SIPHASH") MAC_SETTABLE_CTX_PARAMS(poly1305, "POLY1305") MAC_SETTABLE_CTX_PARAMS(cmac, "CMAC") +MAC_SETTABLE_CTX_PARAMS(eia3, "EIA3") #define MAC_SIGNATURE_FUNCTIONS(funcname) \ const OSSL_DISPATCH ossl_mac_legacy_##funcname##_signature_functions[] = { \ @@ -256,10 +262,11 @@ MAC_SETTABLE_CTX_PARAMS(cmac, "CMAC") (void (*)(void))mac_set_ctx_params }, \ { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_PARAMS, \ (void (*)(void))mac_##funcname##_settable_ctx_params }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ }; MAC_SIGNATURE_FUNCTIONS(hmac) MAC_SIGNATURE_FUNCTIONS(siphash) MAC_SIGNATURE_FUNCTIONS(poly1305) MAC_SIGNATURE_FUNCTIONS(cmac) +MAC_SIGNATURE_FUNCTIONS(eia3) diff --git a/openssl/src/providers/implementations/signature/rsa_sig.c b/openssl/src/providers/implementations/signature/rsa_sig.c index 76db37dd0..6e9b6d64b 100644 --- a/openssl/src/providers/implementations/signature/rsa_sig.c +++ b/openssl/src/providers/implementations/signature/rsa_sig.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -123,7 +123,7 @@ static int rsa_check_padding(const PROV_RSA_CTX *prsactx, const char *mdname, const char *mgf1_mdname, int mdnid) { - switch (prsactx->pad_mode) { + switch(prsactx->pad_mode) { case RSA_NO_PADDING: if (mdname != NULL || mdnid != NID_undef) { ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE); @@ -182,14 +182,15 @@ static void *rsa_newctx(void *provctx, const char *propq) || (propq != NULL && (propq_copy = OPENSSL_strdup(propq)) == NULL)) { OPENSSL_free(prsactx); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; } prsactx->libctx = PROV_LIBCTX_OF(provctx); prsactx->flag_allow_md = 1; prsactx->propq = propq_copy; - /* Maximum up to digest length for sign, auto for verify */ - prsactx->saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX; + /* Maximum for sign, auto for verify */ + prsactx->saltlen = RSA_PSS_SALTLEN_AUTO; prsactx->min_saltlen = -1; return prsactx; } @@ -197,27 +198,13 @@ static void *rsa_newctx(void *provctx, const char *propq) static int rsa_pss_compute_saltlen(PROV_RSA_CTX *ctx) { int saltlen = ctx->saltlen; - int saltlenMax = -1; - - /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection - * 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the - * salt (sLen) shall satisfy 0 <= sLen <= hLen, where hLen is the length of - * the hash function output block (in bytes)." - * - * Provide a way to use at most the digest length, so that the default does - * not violate FIPS 186-4. */ + if (saltlen == RSA_PSS_SALTLEN_DIGEST) { saltlen = EVP_MD_get_size(ctx->md); - } else if (saltlen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) { - saltlen = RSA_PSS_SALTLEN_MAX; - saltlenMax = EVP_MD_get_size(ctx->md); - } - if (saltlen == RSA_PSS_SALTLEN_MAX || saltlen == RSA_PSS_SALTLEN_AUTO) { + } else if (saltlen == RSA_PSS_SALTLEN_AUTO || saltlen == RSA_PSS_SALTLEN_MAX) { saltlen = RSA_size(ctx->rsa) - EVP_MD_get_size(ctx->md) - 2; if ((RSA_bits(ctx->rsa) & 0x7) == 1) saltlen--; - if (saltlenMax >= 0 && saltlen > saltlenMax) - saltlen = saltlenMax; } if (saltlen < 0) { ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR); @@ -243,11 +230,11 @@ static unsigned char *rsa_generate_signature_aid(PROV_RSA_CTX *ctx, int ret; if (!WPACKET_init_der(&pkt, aid_buf, buf_len)) { - ERR_raise(ERR_LIB_PROV, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; } - switch (ctx->pad_mode) { + switch(ctx->pad_mode) { case RSA_PKCS1_PADDING: ret = ossl_DER_w_algorithmIdentifier_MDWithRSAEncryption(&pkt, -1, ctx->mdnid); @@ -421,8 +408,8 @@ static int rsa_signverify_init(void *vprsactx, void *vrsa, prsactx->operation = operation; - /* Maximize up to digest length for sign, auto for verify */ - prsactx->saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX; + /* Maximum for sign, auto for verify */ + prsactx->saltlen = RSA_PSS_SALTLEN_AUTO; prsactx->min_saltlen = -1; switch (RSA_test_flags(prsactx->rsa, RSA_FLAG_TYPE_MASK)) { @@ -497,8 +484,10 @@ static int setup_tbuf(PROV_RSA_CTX *ctx) { if (ctx->tbuf != NULL) return 1; - if ((ctx->tbuf = OPENSSL_malloc(RSA_size(ctx->rsa))) == NULL) + if ((ctx->tbuf = OPENSSL_malloc(RSA_size(ctx->rsa))) == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; + } return 1; } @@ -550,26 +539,6 @@ static int rsa_sign(void *vprsactx, unsigned char *sig, size_t *siglen, return 0; } -#ifndef FIPS_MODULE - if (EVP_MD_is_a(prsactx->md, OSSL_DIGEST_NAME_MDC2)) { - unsigned int sltmp; - - if (prsactx->pad_mode != RSA_PKCS1_PADDING) { - ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_PADDING_MODE, - "only PKCS#1 padding supported with MDC2"); - return 0; - } - ret = RSA_sign_ASN1_OCTET_STRING(0, tbs, tbslen, sig, &sltmp, - prsactx->rsa); - - if (ret <= 0) { - ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB); - return 0; - } - ret = sltmp; - goto end; - } -#endif switch (prsactx->pad_mode) { case RSA_X931_PADDING: if ((size_t)RSA_size(prsactx->rsa) < tbslen + 1) { @@ -579,7 +548,7 @@ static int rsa_sign(void *vprsactx, unsigned char *sig, size_t *siglen, return 0; } if (!setup_tbuf(prsactx)) { - ERR_raise(ERR_LIB_PROV, ERR_R_PROV_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return 0; } memcpy(prsactx->tbuf, tbs, tbslen); @@ -656,9 +625,6 @@ static int rsa_sign(void *vprsactx, unsigned char *sig, size_t *siglen, prsactx->pad_mode); } -#ifndef FIPS_MODULE - end: -#endif if (ret <= 0) { ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB); return 0; @@ -834,17 +800,14 @@ static int rsa_verify(void *vprsactx, const unsigned char *sig, size_t siglen, return 0; } } else { - int ret; - if (!setup_tbuf(prsactx)) return 0; - ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa, - prsactx->pad_mode); - if (ret <= 0) { + rslen = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa, + prsactx->pad_mode); + if (rslen == 0) { ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB); return 0; } - rslen = (size_t)ret; } if ((rslen != tbslen) || memcmp(tbs, prsactx->tbuf, rslen)) @@ -1000,13 +963,14 @@ static void *rsa_dupctx(void *vprsactx) return NULL; dstctx = OPENSSL_zalloc(sizeof(*srcctx)); - if (dstctx == NULL) + if (dstctx == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } *dstctx = *srcctx; dstctx->rsa = NULL; dstctx->md = NULL; - dstctx->mgf1_md = NULL; dstctx->mdctx = NULL; dstctx->tbuf = NULL; dstctx->propq = NULL; @@ -1120,9 +1084,6 @@ static int rsa_get_ctx_params(void *vprsactx, OSSL_PARAM *params) case RSA_PSS_SALTLEN_AUTO: value = OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO; break; - case RSA_PSS_SALTLEN_AUTO_DIGEST_MAX: - value = OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX; - break; default: { int len = BIO_snprintf(p->data, p->data_size, "%d", @@ -1286,8 +1247,6 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[]) saltlen = RSA_PSS_SALTLEN_MAX; else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO) == 0) saltlen = RSA_PSS_SALTLEN_AUTO; - else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX) == 0) - saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX; else saltlen = atoi(p->data); break; @@ -1296,11 +1255,11 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[]) } /* - * RSA_PSS_SALTLEN_AUTO_DIGEST_MAX seems curiously named in this check. - * Contrary to what it's name suggests, it's the currently lowest - * saltlen number possible. + * RSA_PSS_SALTLEN_MAX seems curiously named in this check. + * Contrary to what it's name suggests, it's the currently + * lowest saltlen number possible. */ - if (saltlen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) { + if (saltlen < RSA_PSS_SALTLEN_MAX) { ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH); return 0; } @@ -1308,7 +1267,6 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[]) if (rsa_pss_restricted(prsactx)) { switch (saltlen) { case RSA_PSS_SALTLEN_AUTO: - case RSA_PSS_SALTLEN_AUTO_DIGEST_MAX: if (prsactx->operation == EVP_PKEY_OP_VERIFY) { ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH, "Cannot use autodetected salt length"); @@ -1489,5 +1447,5 @@ const OSSL_DISPATCH ossl_rsa_signature_functions[] = { (void (*)(void))rsa_set_ctx_md_params }, { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_MD_PARAMS, (void (*)(void))rsa_settable_ctx_md_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/signature/sm2_sig.c b/openssl/src/providers/implementations/signature/sm2_sig.c index 479e4eebe..1cca525f5 100644 --- a/openssl/src/providers/implementations/signature/sm2_sig.c +++ b/openssl/src/providers/implementations/signature/sm2_sig.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,7 +9,7 @@ /* * ECDSA low level APIs are deprecated for public use, but still ok for - * internal use - SM2 implementation uses ECDSA_size() function. + * internal use - SM2 implemetation uses ECDSA_size() function. */ #include "internal/deprecated.h" @@ -22,10 +22,10 @@ #include #include #include +#include #include "internal/nelem.h" #include "internal/sizes.h" #include "internal/cryptlib.h" -#include "internal/sm3.h" #include "prov/implementations.h" #include "prov/providercommon.h" #include "prov/provider_ctx.h" @@ -55,6 +55,8 @@ static OSSL_FUNC_signature_gettable_ctx_md_params_fn sm2sig_gettable_ctx_md_para static OSSL_FUNC_signature_set_ctx_md_params_fn sm2sig_set_ctx_md_params; static OSSL_FUNC_signature_settable_ctx_md_params_fn sm2sig_settable_ctx_md_params; +#define OSSL_PKEY_PARAM_SM2_ZA "sm2-za" + /* * What's passed as an actual key is defined by the KEYMGMT interface. * We happen to know that our KEYMGMT simply passes EC structures, so @@ -66,9 +68,9 @@ typedef struct { EC_KEY *ec; /* - * Flag to determine if the 'z' digest needs to be computed and fed to the + * Flag to termine if the 'z' digest needs to be computed and fed to the * hash function. - * This flag should be set on initialization and the computation should + * This flag should be set on initialization and the compuation should * be performed only once, on first update. */ unsigned int flag_compute_z_digest : 1; @@ -122,6 +124,7 @@ static void *sm2sig_newctx(void *provctx, const char *propq) ctx->libctx = PROV_LIBCTX_OF(provctx); if (propq != NULL && (ctx->propq = OPENSSL_strdup(propq)) == NULL) { OPENSSL_free(ctx); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; } ctx->mdsize = SM3_DIGEST_LENGTH; @@ -208,6 +211,9 @@ static int sm2sig_digest_signverify_init(void *vpsm2ctx, const char *mdname, WPACKET pkt; int ret = 0; + /* This default value must be assigned before it may be overridden */ + ctx->flag_compute_z_digest = 1; + if (!sm2sig_signature_init(vpsm2ctx, ec, params) || !sm2sig_set_mdname(ctx, mdname)) return ret; @@ -239,8 +245,6 @@ static int sm2sig_digest_signverify_init(void *vpsm2ctx, const char *mdname, if (!EVP_DigestInit_ex2(ctx->mdctx, ctx->md, params)) goto error; - ctx->flag_compute_z_digest = 1; - ret = 1; error: @@ -329,7 +333,6 @@ static void sm2sig_freectx(void *vpsm2ctx) free_md(ctx); EC_KEY_free(ctx->ec); - OPENSSL_free(ctx->propq); OPENSSL_free(ctx->id); OPENSSL_free(ctx); } @@ -345,21 +348,13 @@ static void *sm2sig_dupctx(void *vpsm2ctx) *dstctx = *srcctx; dstctx->ec = NULL; - dstctx->propq = NULL; dstctx->md = NULL; dstctx->mdctx = NULL; - dstctx->id = NULL; if (srcctx->ec != NULL && !EC_KEY_up_ref(srcctx->ec)) goto err; dstctx->ec = srcctx->ec; - if (srcctx->propq != NULL) { - dstctx->propq = OPENSSL_strdup(srcctx->propq); - if (dstctx->propq == NULL) - goto err; - } - if (srcctx->md != NULL && !EVP_MD_up_ref(srcctx->md)) goto err; dstctx->md = srcctx->md; @@ -435,6 +430,22 @@ static int sm2sig_set_ctx_params(void *vpsm2ctx, const OSSL_PARAM params[]) if (params == NULL) return 1; + p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_SM2_ZA); + if (p != NULL) { + char *v = NULL; + + if (!OSSL_PARAM_get_utf8_string(p, &v, 0)) + return 0; + + /* + * If 'sm2-za:no' is specified, omit computing the z digest + */ + if (OPENSSL_strcasecmp(v, "no") == 0) + psm2ctx->flag_compute_z_digest = 0; + + OPENSSL_free(v); + } + p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_DIST_ID); if (p != NULL) { void *tmp_id = NULL; @@ -485,6 +496,7 @@ static const OSSL_PARAM known_settable_ctx_params[] = { OSSL_PARAM_size_t(OSSL_SIGNATURE_PARAM_DIGEST_SIZE, NULL), OSSL_PARAM_utf8_string(OSSL_SIGNATURE_PARAM_DIGEST, NULL, 0), OSSL_PARAM_octet_string(OSSL_PKEY_PARAM_DIST_ID, NULL, 0), + OSSL_PARAM_utf8_string(OSSL_PKEY_PARAM_SM2_ZA, NULL, 0), OSSL_PARAM_END }; @@ -568,5 +580,5 @@ const OSSL_DISPATCH ossl_sm2_signature_functions[] = { (void (*)(void))sm2sig_set_ctx_md_params }, { OSSL_FUNC_SIGNATURE_SETTABLE_CTX_MD_PARAMS, (void (*)(void))sm2sig_settable_ctx_md_params }, - OSSL_DISPATCH_END + { 0, NULL } }; diff --git a/openssl/src/providers/implementations/storemgmt/file_store.c b/openssl/src/providers/implementations/storemgmt/file_store.c index 171c74d58..65a39437b 100644 --- a/openssl/src/providers/implementations/storemgmt/file_store.c +++ b/openssl/src/providers/implementations/storemgmt/file_store.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -55,7 +55,9 @@ static OSSL_FUNC_store_close_fn file_close; * passes that on to the data callback; this decoder is created with * internal OpenSSL functions, thereby bypassing the need for a surrounding * provider. This is ok, since this is a local decoder, not meant for - * public consumption. + * public consumption. It also uses the libcrypto internal decoder + * setup function ossl_decoder_ctx_setup_for_pkey(), to allow the + * last resort decoder to be added first (and thereby be executed last). * Finally, it sets up its own construct and cleanup functions. * * Essentially, that makes this implementation a kind of glorified decoder. @@ -153,7 +155,7 @@ static struct file_ctx_st *file_open_stream(BIO *source, const char *uri, struct file_ctx_st *ctx; if ((ctx = new_file_ctx(IS_FILE, uri, provctx)) == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_PROV_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -170,7 +172,7 @@ static void *file_open_dir(const char *path, const char *uri, void *provctx) struct file_ctx_st *ctx; if ((ctx = new_file_ctx(IS_DIR, uri, provctx)) == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_PROV_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; } @@ -199,7 +201,7 @@ static void *file_open(void *provctx, const char *uri) unsigned int check_absolute:1; } path_data[2]; size_t path_data_n = 0, i; - const char *path, *p = uri, *q; + const char *path; BIO *bio; ERR_set_mark(); @@ -211,18 +213,20 @@ static void *file_open(void *provctx, const char *uri) path_data[path_data_n++].path = uri; /* - * Second step, if the URI appears to start with the "file" scheme, + * Second step, if the URI appears to start with the 'file' scheme, * extract the path and make that the second path to check. * There's a special case if the URI also contains an authority, then * the full URI shouldn't be used as a path anywhere. */ - if (CHECK_AND_SKIP_CASE_PREFIX(p, "file:")) { - q = p; - if (CHECK_AND_SKIP_CASE_PREFIX(q, "//")) { + if (OPENSSL_strncasecmp(uri, "file:", 5) == 0) { + const char *p = &uri[5]; + + if (strncmp(&uri[5], "//", 2) == 0) { path_data_n--; /* Invalidate using the full URI */ - if (CHECK_AND_SKIP_CASE_PREFIX(q, "localhost/") - || CHECK_AND_SKIP_CASE_PREFIX(q, "/")) { - p = q - 1; + if (OPENSSL_strncasecmp(&uri[7], "localhost/", 10) == 0) { + p = &uri[16]; + } else if (uri[7] == '/') { + p = &uri[7]; } else { ERR_clear_last_mark(); ERR_raise(ERR_LIB_PROV, PROV_R_URI_AUTHORITY_UNSUPPORTED); @@ -232,7 +236,7 @@ static void *file_open(void *provctx, const char *uri) path_data[path_data_n].check_absolute = 1; #ifdef _WIN32 - /* Windows "file:" URIs with a drive letter start with a '/' */ + /* Windows file: URIs with a drive letter start with a / */ if (p[0] == '/' && p[2] == ':' && p[3] == '/') { char c = tolower(p[1]); @@ -420,7 +424,7 @@ static int file_setup_decoders(struct file_ctx_st *ctx) /* Setup for this session, so only if not already done */ if (ctx->_.file.decoderctx == NULL) { if ((ctx->_.file.decoderctx = OSSL_DECODER_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB); + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); goto err; } @@ -556,8 +560,10 @@ static char *file_name_to_uri(struct file_ctx_st *ctx, const char *name) + strlen(name) + 1 /* \0 */; data = OPENSSL_zalloc(calculated_length); - if (data == NULL) + if (data == NULL) { + ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE); return NULL; + } OPENSSL_strlcat(data, ctx->uri, calculated_length); OPENSSL_strlcat(data, pathsep, calculated_length); @@ -606,21 +612,11 @@ static int file_name_check(struct file_ctx_st *ctx, const char *name) * Last, check that the rest of the extension is a decimal number, at * least one digit long. */ - if (!isdigit((unsigned char)*p)) + if (!isdigit(*p)) return 0; - while (isdigit((unsigned char)*p)) + while (isdigit(*p)) p++; -#ifdef __VMS - /* - * One extra step here, check for a possible generation number. - */ - if (*p == ';') - for (p++; *p != '\0'; p++) - if (!ossl_isdigit((unsigned char)*p)) - break; -#endif - /* * If we've reached the end of the string at this point, we've successfully * found a fitting file name. @@ -780,5 +776,5 @@ const OSSL_DISPATCH ossl_file_store_functions[] = { { OSSL_FUNC_STORE_LOAD, (void (*)(void))file_load }, { OSSL_FUNC_STORE_EOF, (void (*)(void))file_eof }, { OSSL_FUNC_STORE_CLOSE, (void (*)(void))file_close }, - OSSL_DISPATCH_END, + { 0, NULL }, }; diff --git a/openssl/src/providers/implementations/storemgmt/file_store_any2obj.c b/openssl/src/providers/implementations/storemgmt/file_store_any2obj.c index b8fa59108..28601683b 100644 --- a/openssl/src/providers/implementations/storemgmt/file_store_any2obj.c +++ b/openssl/src/providers/implementations/storemgmt/file_store_any2obj.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -125,7 +125,7 @@ static int msblob2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection, mem_want = 16; /* The size of the MSBLOB header */ if ((mem = BUF_MEM_new()) == NULL || !BUF_MEM_grow(mem, mem_want)) { - ERR_raise(ERR_LIB_PEM, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; } @@ -147,7 +147,7 @@ static int msblob2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection, ok = 0; mem_want = ossl_blob_length(bitlen, isdss, ispub); if (!BUF_MEM_grow(mem, mem_len + mem_want)) { - ERR_raise(ERR_LIB_PEM, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; } @@ -192,7 +192,7 @@ static int pvk2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection, mem_want = 24; /* The size of the PVK header */ if ((mem = BUF_MEM_new()) == NULL || !BUF_MEM_grow(mem, mem_want)) { - ERR_raise(ERR_LIB_PEM, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; } @@ -214,7 +214,7 @@ static int pvk2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection, ok = 0; mem_want = saltlen + keylen; if (!BUF_MEM_grow(mem, mem_len + mem_want)) { - ERR_raise(ERR_LIB_PEM, ERR_R_BUF_LIB); + ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE); goto err; } @@ -246,7 +246,7 @@ static int pvk2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection, { OSSL_FUNC_DECODER_NEWCTX, (void (*)(void))any2obj_newctx }, \ { OSSL_FUNC_DECODER_FREECTX, (void (*)(void))any2obj_freectx }, \ { OSSL_FUNC_DECODER_DECODE, (void (*)(void))fromtype##2obj_decode }, \ - OSSL_DISPATCH_END \ + { 0, NULL } \ } MAKE_DECODER(der, OSSL_OBJECT_UNKNOWN); diff --git a/openssl/src/providers/implementations/storemgmt/winstore_store.c b/openssl/src/providers/implementations/storemgmt/winstore_store.c deleted file mode 100644 index e230101d7..000000000 --- a/openssl/src/providers/implementations/storemgmt/winstore_store.c +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include /* The OSSL_STORE_INFO type numbers */ -#include "internal/cryptlib.h" -#include "internal/o_dir.h" -#include "crypto/decoder.h" -#include "crypto/ctype.h" /* ossl_isdigit() */ -#include "prov/implementations.h" -#include "prov/bio.h" -#include "file_store_local.h" -#ifdef __CYGWIN__ -# include -#endif -#include - -enum { - STATE_IDLE, - STATE_READ, - STATE_EOF, -}; - -struct winstore_ctx_st { - void *provctx; - char *propq; - unsigned char *subject; - size_t subject_len; - - HCERTSTORE win_store; - const CERT_CONTEXT *win_ctx; - int state; - - OSSL_DECODER_CTX *dctx; -}; - -static void winstore_win_reset(struct winstore_ctx_st *ctx) -{ - if (ctx->win_ctx != NULL) { - CertFreeCertificateContext(ctx->win_ctx); - ctx->win_ctx = NULL; - } - - ctx->state = STATE_IDLE; -} - -static void winstore_win_advance(struct winstore_ctx_st *ctx) -{ - CERT_NAME_BLOB name = {0}; - - if (ctx->state == STATE_EOF) - return; - - name.cbData = ctx->subject_len; - name.pbData = ctx->subject; - - ctx->win_ctx = (name.cbData == 0 ? NULL : - CertFindCertificateInStore(ctx->win_store, - X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, - 0, CERT_FIND_SUBJECT_NAME, - &name, ctx->win_ctx)); - - ctx->state = (ctx->win_ctx == NULL) ? STATE_EOF : STATE_READ; -} - -static void *winstore_open(void *provctx, const char *uri) -{ - struct winstore_ctx_st *ctx = NULL; - - if (!HAS_CASE_PREFIX(uri, "org.openssl.winstore:")) - return NULL; - - ctx = OPENSSL_zalloc(sizeof(*ctx)); - if (ctx == NULL) - return NULL; - - ctx->provctx = provctx; - ctx->win_store = CertOpenSystemStoreW(0, L"ROOT"); - if (ctx->win_store == NULL) { - OPENSSL_free(ctx); - return NULL; - } - - winstore_win_reset(ctx); - return ctx; -} - -static void *winstore_attach(void *provctx, OSSL_CORE_BIO *cin) -{ - return NULL; /* not supported */ -} - -static const OSSL_PARAM *winstore_settable_ctx_params(void *loaderctx, const OSSL_PARAM params[]) -{ - static const OSSL_PARAM known_settable_ctx_params[] = { - OSSL_PARAM_octet_string(OSSL_STORE_PARAM_SUBJECT, NULL, 0), - OSSL_PARAM_utf8_string(OSSL_STORE_PARAM_PROPERTIES, NULL, 0), - OSSL_PARAM_END - }; - return known_settable_ctx_params; -} - -static int winstore_set_ctx_params(void *loaderctx, const OSSL_PARAM params[]) -{ - struct winstore_ctx_st *ctx = loaderctx; - const OSSL_PARAM *p; - int do_reset = 0; - - if (params == NULL) - return 1; - - p = OSSL_PARAM_locate_const(params, OSSL_STORE_PARAM_PROPERTIES); - if (p != NULL) { - do_reset = 1; - OPENSSL_free(ctx->propq); - ctx->propq = NULL; - if (!OSSL_PARAM_get_utf8_string(p, &ctx->propq, 0)) - return 0; - } - - p = OSSL_PARAM_locate_const(params, OSSL_STORE_PARAM_SUBJECT); - if (p != NULL) { - const unsigned char *der = NULL; - size_t der_len = 0; - - if (!OSSL_PARAM_get_octet_string_ptr(p, (const void **)&der, &der_len)) - return 0; - - do_reset = 1; - - OPENSSL_free(ctx->subject); - - ctx->subject = OPENSSL_malloc(der_len); - if (ctx->subject == NULL) { - ctx->subject_len = 0; - return 0; - } - - ctx->subject_len = der_len; - memcpy(ctx->subject, der, der_len); - } - - if (do_reset) { - winstore_win_reset(ctx); - winstore_win_advance(ctx); - } - - return 1; -} - -struct load_data_st { - OSSL_CALLBACK *object_cb; - void *object_cbarg; -}; - -static int load_construct(OSSL_DECODER_INSTANCE *decoder_inst, - const OSSL_PARAM *params, void *construct_data) -{ - struct load_data_st *data = construct_data; - return data->object_cb(params, data->object_cbarg); -} - -static void load_cleanup(void *construct_data) -{ - /* No-op. */ -} - -static int setup_decoder(struct winstore_ctx_st *ctx) -{ - OSSL_LIB_CTX *libctx = ossl_prov_ctx_get0_libctx(ctx->provctx); - const OSSL_ALGORITHM *to_algo = NULL; - - if (ctx->dctx != NULL) - return 1; - - ctx->dctx = OSSL_DECODER_CTX_new(); - if (ctx->dctx == NULL) { - ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB); - return 0; - } - - if (!OSSL_DECODER_CTX_set_input_type(ctx->dctx, "DER")) { - ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB); - goto err; - } - - if (!OSSL_DECODER_CTX_set_input_structure(ctx->dctx, "Certificate")) { - ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB); - goto err; - } - - for (to_algo = ossl_any_to_obj_algorithm; - to_algo->algorithm_names != NULL; - to_algo++) { - OSSL_DECODER *to_obj = NULL; - OSSL_DECODER_INSTANCE *to_obj_inst = NULL; - - /* - * Create the internal last resort decoder implementation - * together with a "decoder instance". - * The decoder doesn't need any identification or to be - * attached to any provider, since it's only used locally. - */ - to_obj = ossl_decoder_from_algorithm(0, to_algo, NULL); - if (to_obj != NULL) - to_obj_inst = ossl_decoder_instance_new(to_obj, ctx->provctx); - - OSSL_DECODER_free(to_obj); - if (to_obj_inst == NULL) - goto err; - - if (!ossl_decoder_ctx_add_decoder_inst(ctx->dctx, - to_obj_inst)) { - ossl_decoder_instance_free(to_obj_inst); - ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB); - goto err; - } - } - - if (!OSSL_DECODER_CTX_add_extra(ctx->dctx, libctx, ctx->propq)) { - ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB); - goto err; - } - - if (!OSSL_DECODER_CTX_set_construct(ctx->dctx, load_construct)) { - ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB); - goto err; - } - - if (!OSSL_DECODER_CTX_set_cleanup(ctx->dctx, load_cleanup)) { - ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB); - goto err; - } - - return 1; - -err: - OSSL_DECODER_CTX_free(ctx->dctx); - ctx->dctx = NULL; - return 0; -} - -static int winstore_load_using(struct winstore_ctx_st *ctx, - OSSL_CALLBACK *object_cb, void *object_cbarg, - OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg, - const void *der, size_t der_len) -{ - struct load_data_st data; - const unsigned char *der_ = der; - size_t der_len_ = der_len; - - if (setup_decoder(ctx) == 0) - return 0; - - data.object_cb = object_cb; - data.object_cbarg = object_cbarg; - - OSSL_DECODER_CTX_set_construct_data(ctx->dctx, &data); - OSSL_DECODER_CTX_set_passphrase_cb(ctx->dctx, pw_cb, pw_cbarg); - - if (OSSL_DECODER_from_data(ctx->dctx, &der_, &der_len_) == 0) - return 0; - - return 1; -} - -static int winstore_load(void *loaderctx, - OSSL_CALLBACK *object_cb, void *object_cbarg, - OSSL_PASSPHRASE_CALLBACK *pw_cb, void *pw_cbarg) -{ - int ret = 0; - struct winstore_ctx_st *ctx = loaderctx; - - if (ctx->state != STATE_READ) - return 0; - - ret = winstore_load_using(ctx, object_cb, object_cbarg, pw_cb, pw_cbarg, - ctx->win_ctx->pbCertEncoded, - ctx->win_ctx->cbCertEncoded); - - if (ret == 1) - winstore_win_advance(ctx); - - return ret; -} - -static int winstore_eof(void *loaderctx) -{ - struct winstore_ctx_st *ctx = loaderctx; - - return ctx->state != STATE_READ; -} - -static int winstore_close(void *loaderctx) -{ - struct winstore_ctx_st *ctx = loaderctx; - - winstore_win_reset(ctx); - CertCloseStore(ctx->win_store, 0); - OSSL_DECODER_CTX_free(ctx->dctx); - OPENSSL_free(ctx->propq); - OPENSSL_free(ctx->subject); - OPENSSL_free(ctx); - return 1; -} - -const OSSL_DISPATCH ossl_winstore_store_functions[] = { - { OSSL_FUNC_STORE_OPEN, (void (*)(void))winstore_open }, - { OSSL_FUNC_STORE_ATTACH, (void (*)(void))winstore_attach }, - { OSSL_FUNC_STORE_SETTABLE_CTX_PARAMS, (void (*)(void))winstore_settable_ctx_params }, - { OSSL_FUNC_STORE_SET_CTX_PARAMS, (void (*)(void))winstore_set_ctx_params }, - { OSSL_FUNC_STORE_LOAD, (void (*)(void))winstore_load }, - { OSSL_FUNC_STORE_EOF, (void (*)(void))winstore_eof }, - { OSSL_FUNC_STORE_CLOSE, (void (*)(void))winstore_close }, - OSSL_DISPATCH_END, -}; diff --git a/openssl/src/providers/legacyprov.c b/openssl/src/providers/legacyprov.c index 16e3639e7..43803b31c 100644 --- a/openssl/src/providers/legacyprov.c +++ b/openssl/src/providers/legacyprov.c @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,6 @@ #include #include #include -#include #include #include "prov/provider_ctx.h" #include "prov/implementations.h" @@ -34,22 +33,6 @@ OSSL_provider_init_fn ossl_legacy_provider_init; # define OSSL_provider_init ossl_legacy_provider_init #endif -#ifndef STATIC_LEGACY -/* - * Should these function pointers be stored in the provider side provctx? - * Could they ever be different from one init to the next? We assume not for - * now. - */ - -/* Functions provided by the core */ -static OSSL_FUNC_core_new_error_fn *c_new_error; -static OSSL_FUNC_core_set_error_debug_fn *c_set_error_debug; -static OSSL_FUNC_core_vset_error_fn *c_vset_error; -static OSSL_FUNC_core_set_error_mark_fn *c_set_error_mark; -static OSSL_FUNC_core_clear_last_error_mark_fn *c_clear_last_error_mark; -static OSSL_FUNC_core_pop_error_to_mark_fn *c_pop_error_to_mark; -#endif - /* Parameters we provide to the core */ static const OSSL_PARAM legacy_param_types[] = { OSSL_PARAM_DEFN(OSSL_PROV_PARAM_NAME, OSSL_PARAM_UTF8_PTR, NULL, 0), @@ -84,57 +67,10 @@ static int legacy_get_params(void *provctx, OSSL_PARAM params[]) } static const OSSL_ALGORITHM legacy_digests[] = { -#ifndef OPENSSL_NO_MD2 - ALG(PROV_NAMES_MD2, ossl_md2_functions), -#endif -#ifndef OPENSSL_NO_MD4 - ALG(PROV_NAMES_MD4, ossl_md4_functions), -#endif -#ifndef OPENSSL_NO_MDC2 - ALG(PROV_NAMES_MDC2, ossl_mdc2_functions), -#endif /* OPENSSL_NO_MDC2 */ -#ifndef OPENSSL_NO_WHIRLPOOL - ALG(PROV_NAMES_WHIRLPOOL, ossl_wp_functions), -#endif /* OPENSSL_NO_WHIRLPOOL */ -#ifndef OPENSSL_NO_RMD160 - ALG(PROV_NAMES_RIPEMD_160, ossl_ripemd160_functions), -#endif /* OPENSSL_NO_RMD160 */ { NULL, NULL, NULL } }; static const OSSL_ALGORITHM legacy_ciphers[] = { -#ifndef OPENSSL_NO_CAST - ALG(PROV_NAMES_CAST5_ECB, ossl_cast5128ecb_functions), - ALG(PROV_NAMES_CAST5_CBC, ossl_cast5128cbc_functions), - ALG(PROV_NAMES_CAST5_OFB, ossl_cast5128ofb64_functions), - ALG(PROV_NAMES_CAST5_CFB, ossl_cast5128cfb64_functions), -#endif /* OPENSSL_NO_CAST */ -#ifndef OPENSSL_NO_BF - ALG(PROV_NAMES_BF_ECB, ossl_blowfish128ecb_functions), - ALG(PROV_NAMES_BF_CBC, ossl_blowfish128cbc_functions), - ALG(PROV_NAMES_BF_OFB, ossl_blowfish128ofb64_functions), - ALG(PROV_NAMES_BF_CFB, ossl_blowfish128cfb64_functions), -#endif /* OPENSSL_NO_BF */ -#ifndef OPENSSL_NO_IDEA - ALG(PROV_NAMES_IDEA_ECB, ossl_idea128ecb_functions), - ALG(PROV_NAMES_IDEA_CBC, ossl_idea128cbc_functions), - ALG(PROV_NAMES_IDEA_OFB, ossl_idea128ofb64_functions), - ALG(PROV_NAMES_IDEA_CFB, ossl_idea128cfb64_functions), -#endif /* OPENSSL_NO_IDEA */ -#ifndef OPENSSL_NO_SEED - ALG(PROV_NAMES_SEED_ECB, ossl_seed128ecb_functions), - ALG(PROV_NAMES_SEED_CBC, ossl_seed128cbc_functions), - ALG(PROV_NAMES_SEED_OFB, ossl_seed128ofb128_functions), - ALG(PROV_NAMES_SEED_CFB, ossl_seed128cfb128_functions), -#endif /* OPENSSL_NO_SEED */ -#ifndef OPENSSL_NO_RC2 - ALG(PROV_NAMES_RC2_ECB, ossl_rc2128ecb_functions), - ALG(PROV_NAMES_RC2_CBC, ossl_rc2128cbc_functions), - ALG(PROV_NAMES_RC2_40_CBC, ossl_rc240cbc_functions), - ALG(PROV_NAMES_RC2_64_CBC, ossl_rc264cbc_functions), - ALG(PROV_NAMES_RC2_CFB, ossl_rc2128cfb128_functions), - ALG(PROV_NAMES_RC2_OFB, ossl_rc2128ofb128_functions), -#endif /* OPENSSL_NO_RC2 */ #ifndef OPENSSL_NO_RC4 ALG(PROV_NAMES_RC4, ossl_rc4128_functions), ALG(PROV_NAMES_RC4_40, ossl_rc440_functions), @@ -162,7 +98,6 @@ static const OSSL_ALGORITHM legacy_ciphers[] = { static const OSSL_ALGORITHM legacy_kdfs[] = { ALG(PROV_NAMES_PBKDF1, ossl_kdf_pbkdf1_functions), - ALG(PROV_NAMES_PVKKDF, ossl_kdf_pvk_functions), { NULL, NULL, NULL } }; @@ -193,7 +128,7 @@ static const OSSL_DISPATCH legacy_dispatch_table[] = { { OSSL_FUNC_PROVIDER_GETTABLE_PARAMS, (void (*)(void))legacy_gettable_params }, { OSSL_FUNC_PROVIDER_GET_PARAMS, (void (*)(void))legacy_get_params }, { OSSL_FUNC_PROVIDER_QUERY_OPERATION, (void (*)(void))legacy_query }, - OSSL_DISPATCH_END + { 0, NULL } }; int OSSL_provider_init(const OSSL_CORE_HANDLE *handle, @@ -202,41 +137,6 @@ int OSSL_provider_init(const OSSL_CORE_HANDLE *handle, void **provctx) { OSSL_LIB_CTX *libctx = NULL; -#ifndef STATIC_LEGACY - const OSSL_DISPATCH *tmp; -#endif - -#ifndef STATIC_LEGACY - for (tmp = in; tmp->function_id != 0; tmp++) { - /* - * We do not support the scenario of an application linked against - * multiple versions of libcrypto (e.g. one static and one dynamic), - * but sharing a single legacy.so. We do a simple sanity check here. - */ -#define set_func(c, f) if (c == NULL) c = f; else if (c != f) return 0; - switch (tmp->function_id) { - case OSSL_FUNC_CORE_NEW_ERROR: - set_func(c_new_error, OSSL_FUNC_core_new_error(tmp)); - break; - case OSSL_FUNC_CORE_SET_ERROR_DEBUG: - set_func(c_set_error_debug, OSSL_FUNC_core_set_error_debug(tmp)); - break; - case OSSL_FUNC_CORE_VSET_ERROR: - set_func(c_vset_error, OSSL_FUNC_core_vset_error(tmp)); - break; - case OSSL_FUNC_CORE_SET_ERROR_MARK: - set_func(c_set_error_mark, OSSL_FUNC_core_set_error_mark(tmp)); - break; - case OSSL_FUNC_CORE_CLEAR_LAST_ERROR_MARK: - set_func(c_clear_last_error_mark, - OSSL_FUNC_core_clear_last_error_mark(tmp)); - break; - case OSSL_FUNC_CORE_POP_ERROR_TO_MARK: - set_func(c_pop_error_to_mark, OSSL_FUNC_core_pop_error_to_mark(tmp)); - break; - } - } -#endif if ((*provctx = ossl_prov_ctx_new()) == NULL || (libctx = OSSL_LIB_CTX_new_child(handle, in)) == NULL) { @@ -252,53 +152,3 @@ int OSSL_provider_init(const OSSL_CORE_HANDLE *handle, return 1; } - -#ifndef STATIC_LEGACY -/* - * Provider specific implementation of libcrypto functions in terms of - * upcalls. - */ - -/* - * For ERR functions, we pass a NULL context. This is valid to do as long - * as only error codes that the calling libcrypto supports are used. - */ -void ERR_new(void) -{ - c_new_error(NULL); -} - -void ERR_set_debug(const char *file, int line, const char *func) -{ - c_set_error_debug(NULL, file, line, func); -} - -void ERR_set_error(int lib, int reason, const char *fmt, ...) -{ - va_list args; - - va_start(args, fmt); - c_vset_error(NULL, ERR_PACK(lib, 0, reason), fmt, args); - va_end(args); -} - -void ERR_vset_error(int lib, int reason, const char *fmt, va_list args) -{ - c_vset_error(NULL, ERR_PACK(lib, 0, reason), fmt, args); -} - -int ERR_set_mark(void) -{ - return c_set_error_mark(NULL); -} - -int ERR_clear_last_mark(void) -{ - return c_clear_last_error_mark(NULL); -} - -int ERR_pop_to_mark(void) -{ - return c_pop_error_to_mark(NULL); -} -#endif diff --git a/openssl/src/providers/local.h b/openssl/src/providers/local.h new file mode 100644 index 000000000..c44611f84 --- /dev/null +++ b/openssl/src/providers/local.h @@ -0,0 +1,16 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ + +OSSL_provider_init_fn ossl_null_provider_init; +OSSL_provider_init_fn ossl_default_provider_init; +OSSL_provider_init_fn ossl_base_provider_init; diff --git a/openssl/src/providers/nullprov.c b/openssl/src/providers/nullprov.c index bace75af1..d5b4c05d9 100644 --- a/openssl/src/providers/nullprov.c +++ b/openssl/src/providers/nullprov.c @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -64,7 +64,7 @@ static const OSSL_DISPATCH null_dispatch_table[] = { { OSSL_FUNC_PROVIDER_GETTABLE_PARAMS, (void (*)(void))null_gettable_params }, { OSSL_FUNC_PROVIDER_GET_PARAMS, (void (*)(void))null_get_params }, { OSSL_FUNC_PROVIDER_QUERY_OPERATION, (void (*)(void))null_query }, - OSSL_DISPATCH_END + { 0, NULL } }; int ossl_null_provider_init(const OSSL_CORE_HANDLE *handle, diff --git a/openssl/src/providers/stores.inc b/openssl/src/providers/stores.inc index 526532759..4c1ec8f28 100644 --- a/openssl/src/providers/stores.inc +++ b/openssl/src/providers/stores.inc @@ -12,6 +12,3 @@ #endif STORE("file", "yes", ossl_file_store_functions) -#ifndef OPENSSL_NO_WINSTORE -STORE("org.openssl.winstore", "yes", ossl_winstore_store_functions) -#endif diff --git a/openssl/src/ssl/bio_ssl.c b/openssl/src/ssl/bio_ssl.c index aabd047fe..0725c3b7a 100644 --- a/openssl/src/ssl/bio_ssl.c +++ b/openssl/src/ssl/bio_ssl.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -25,11 +25,7 @@ static int ssl_free(BIO *data); static long ssl_callback_ctrl(BIO *h, int cmd, BIO_info_cb *fp); typedef struct bio_ssl_st { SSL *ssl; /* The ssl handle :-) */ - /* - * Re-negotiate every time the total number of bytes is this size - * or when timeout expires. - * There is no proper support for TLS-1.3 or QUIC yet. - */ + /* re-negotiate every time the total number of bytes is this size */ int num_renegotiates; unsigned long renegotiate_count; size_t byte_count; @@ -61,8 +57,10 @@ static int ssl_new(BIO *bi) { BIO_SSL *bs = OPENSSL_zalloc(sizeof(*bs)); - if (bs == NULL) + if (bs == NULL) { + ERR_raise(ERR_LIB_BIO, ERR_R_MALLOC_FAILURE); return 0; + } BIO_set_init(bi, 0); BIO_set_data(bi, bs); /* Clear all flags */ @@ -140,6 +138,12 @@ static int ssl_read(BIO *b, char *buf, size_t size, size_t *readbytes) BIO_set_retry_special(b); retry_reason = BIO_RR_SSL_X509_LOOKUP; break; +# ifndef OPENSSL_NO_SESSION_LOOKUP + case SSL_ERROR_WANT_SESSION_LOOKUP: + BIO_set_retry_special(b); + retry_reason = BIO_RR_SSL_SESSION_LOOKUP; + break; +# endif case SSL_ERROR_WANT_ACCEPT: BIO_set_retry_special(b); retry_reason = BIO_RR_ACCEPT; @@ -174,7 +178,7 @@ static int ssl_write(BIO *b, const char *buf, size_t size, size_t *written) BIO_clear_retry_flags(b); - ret = ssl_write_internal(ssl, buf, size, 0, written); + ret = ssl_write_internal(ssl, buf, size, written); switch (SSL_get_error(ssl, ret)) { case SSL_ERROR_NONE: @@ -208,6 +212,12 @@ static int ssl_write(BIO *b, const char *buf, size_t size, size_t *written) BIO_set_retry_special(b); retry_reason = BIO_RR_SSL_X509_LOOKUP; break; +# ifndef OPENSSL_NO_SESSION_LOOKUP + case SSL_ERROR_WANT_SESSION_LOOKUP: + BIO_set_retry_special(b); + retry_reason = BIO_RR_SSL_SESSION_LOOKUP; + break; +# endif case SSL_ERROR_WANT_CONNECT: BIO_set_retry_special(b); retry_reason = BIO_RR_CONNECT; @@ -229,24 +239,19 @@ static long ssl_ctrl(BIO *b, int cmd, long num, void *ptr) BIO *dbio, *bio; long ret = 1; BIO *next; - SSL_CONNECTION *sc = NULL; bs = BIO_get_data(b); next = BIO_next(b); ssl = bs->ssl; - if (ssl == NULL && cmd != BIO_C_SET_SSL) + if ((ssl == NULL) && (cmd != BIO_C_SET_SSL)) return 0; switch (cmd) { case BIO_CTRL_RESET: - /* TODO(QUIC FUTURE): Add support when SSL_clear() is supported */ - if ((sc = SSL_CONNECTION_FROM_SSL_ONLY(ssl)) == NULL) - return 0; - SSL_shutdown(ssl); - if (sc->handshake_func == ssl->method->ssl_connect) + if (ssl->handshake_func == ssl->method->ssl_connect) SSL_set_connect_state(ssl); - else if (sc->handshake_func == ssl->method->ssl_accept) + else if (ssl->handshake_func == ssl->method->ssl_accept) SSL_set_accept_state(ssl); if (!SSL_clear(ssl)) { @@ -256,8 +261,8 @@ static long ssl_ctrl(BIO *b, int cmd, long num, void *ptr) if (next != NULL) ret = BIO_ctrl(next, cmd, num, ptr); - else if (sc->rbio != NULL) - ret = BIO_ctrl(sc->rbio, cmd, num, ptr); + else if (ssl->rbio != NULL) + ret = BIO_ctrl(ssl->rbio, cmd, num, ptr); else ret = 1; break; @@ -318,20 +323,20 @@ static long ssl_ctrl(BIO *b, int cmd, long num, void *ptr) BIO_set_shutdown(b, (int)num); break; case BIO_CTRL_WPENDING: - ret = BIO_ctrl(SSL_get_wbio(ssl), cmd, num, ptr); + ret = BIO_ctrl(ssl->wbio, cmd, num, ptr); break; case BIO_CTRL_PENDING: ret = SSL_pending(ssl); if (ret == 0) - ret = BIO_pending(SSL_get_rbio(ssl)); + ret = BIO_pending(ssl->rbio); break; case BIO_CTRL_FLUSH: BIO_clear_retry_flags(b); - ret = BIO_ctrl(SSL_get_wbio(ssl), cmd, num, ptr); + ret = BIO_ctrl(ssl->wbio, cmd, num, ptr); BIO_copy_next_retry(b); break; case BIO_CTRL_PUSH: - if ((next != NULL) && (next != SSL_get_rbio(ssl))) { + if ((next != NULL) && (next != ssl->rbio)) { /* * We are going to pass ownership of next to the SSL object...but * we don't own a reference to pass yet - so up ref @@ -368,6 +373,12 @@ static long ssl_ctrl(BIO *b, int cmd, long num, void *ptr) BIO_set_retry_special(b); BIO_set_retry_reason(b, BIO_RR_SSL_X509_LOOKUP); break; +# ifndef OPENSSL_NO_SESSION_LOOKUP + case SSL_ERROR_WANT_SESSION_LOOKUP: + BIO_set_retry_special(b); + BIO_set_retry_reason(b, BIO_RR_SSL_SESSION_LOOKUP); + break; +# endif default: break; } @@ -385,21 +396,13 @@ static long ssl_ctrl(BIO *b, int cmd, long num, void *ptr) ret = (dbs->ssl != NULL); break; case BIO_C_GET_FD: - ret = BIO_ctrl(SSL_get_rbio(ssl), cmd, num, ptr); + ret = BIO_ctrl(ssl->rbio, cmd, num, ptr); break; case BIO_CTRL_SET_CALLBACK: ret = 0; /* use callback ctrl */ break; - case BIO_CTRL_GET_RPOLL_DESCRIPTOR: - if (!SSL_get_rpoll_descriptor(ssl, (BIO_POLL_DESCRIPTOR *)ptr)) - ret = 0; - break; - case BIO_CTRL_GET_WPOLL_DESCRIPTOR: - if (!SSL_get_wpoll_descriptor(ssl, (BIO_POLL_DESCRIPTOR *)ptr)) - ret = 0; - break; default: - ret = BIO_ctrl(SSL_get_rbio(ssl), cmd, num, ptr); + ret = BIO_ctrl(ssl->rbio, cmd, num, ptr); break; } return ret; @@ -415,7 +418,7 @@ static long ssl_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp) ssl = bs->ssl; switch (cmd) { case BIO_CTRL_SET_CALLBACK: - ret = BIO_callback_ctrl(SSL_get_rbio(ssl), cmd, fp); + ret = BIO_callback_ctrl(ssl->rbio, cmd, fp); break; default: ret = 0; @@ -438,12 +441,6 @@ BIO *BIO_new_buffer_ssl_connect(SSL_CTX *ctx) #ifndef OPENSSL_NO_SOCK BIO *ret = NULL, *buf = NULL, *ssl = NULL; -# ifndef OPENSSL_NO_QUIC - if (ctx != NULL && IS_QUIC_CTX(ctx)) - /* Never use buffering for QUIC. */ - return BIO_new_ssl_connect(ctx); -# endif - if ((buf = BIO_new(BIO_f_buffer())) == NULL) return NULL; if ((ssl = BIO_new_ssl_connect(ctx)) == NULL) @@ -465,13 +462,6 @@ BIO *BIO_new_ssl_connect(SSL_CTX *ctx) if ((con = BIO_new(BIO_s_connect())) == NULL) return NULL; - -# ifndef OPENSSL_NO_QUIC - if (ctx != NULL && IS_QUIC_CTX(ctx)) - if (!BIO_set_sock_type(con, SOCK_DGRAM)) - goto err; -#endif - if ((ssl = BIO_new_ssl(ctx, 1)) == NULL) goto err; if ((ret = BIO_push(ssl, con)) == NULL) diff --git a/openssl/src/ssl/d1_lib.c b/openssl/src/ssl/d1_lib.c index 1ac0975d0..871c187a9 100644 --- a/openssl/src/ssl/d1_lib.c +++ b/openssl/src/ssl/d1_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 2005-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2005-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,15 +12,17 @@ #include #include #include "ssl_local.h" -#include "internal/time.h" -static int dtls1_handshake_write(SSL_CONNECTION *s); +static void get_current_time(struct timeval *t); +static int dtls1_handshake_write(SSL *s); static size_t dtls1_link_min_mtu(void); /* XDTLS: figure out the right values */ static const size_t g_probable_mtu[] = { 1500, 512, 256 }; const SSL3_ENC_METHOD DTLSv1_enc_data = { + tls1_enc, + tls1_mac, tls1_setup_key_block, tls1_generate_master_secret, tls1_change_cipher_state, @@ -36,6 +38,8 @@ const SSL3_ENC_METHOD DTLSv1_enc_data = { }; const SSL3_ENC_METHOD DTLSv1_2_enc_data = { + tls1_enc, + tls1_mac, tls1_setup_key_block, tls1_generate_master_secret, tls1_change_cipher_state, @@ -51,31 +55,27 @@ const SSL3_ENC_METHOD DTLSv1_2_enc_data = { dtls1_handshake_write }; -OSSL_TIME dtls1_default_timeout(void) +long dtls1_default_timeout(void) { /* * 2 hours, the 24 hours mentioned in the DTLSv1 spec is way too long for * http, the cache would over fill */ - return ossl_seconds2time(60 * 60 * 2); + return (60 * 60 * 2); } -int dtls1_new(SSL *ssl) +int dtls1_new(SSL *s) { DTLS1_STATE *d1; - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - if (s == NULL) - return 0; if (!DTLS_RECORD_LAYER_new(&s->rlayer)) { return 0; } - if (!ssl3_new(ssl)) + if (!ssl3_new(s)) return 0; if ((d1 = OPENSSL_zalloc(sizeof(*d1))) == NULL) { - ssl3_free(ssl); + ssl3_free(s); return 0; } @@ -93,25 +93,25 @@ int dtls1_new(SSL *ssl) pqueue_free(d1->buffered_messages); pqueue_free(d1->sent_messages); OPENSSL_free(d1); - ssl3_free(ssl); + ssl3_free(s); return 0; } s->d1 = d1; - if (!ssl->method->ssl_clear(ssl)) + if (!s->method->ssl_clear(s)) return 0; return 1; } -static void dtls1_clear_queues(SSL_CONNECTION *s) +static void dtls1_clear_queues(SSL *s) { dtls1_clear_received_buffer(s); dtls1_clear_sent_buffer(s); } -void dtls1_clear_received_buffer(SSL_CONNECTION *s) +void dtls1_clear_received_buffer(SSL *s) { pitem *item = NULL; hm_fragment *frag = NULL; @@ -123,36 +123,24 @@ void dtls1_clear_received_buffer(SSL_CONNECTION *s) } } -void dtls1_clear_sent_buffer(SSL_CONNECTION *s) +void dtls1_clear_sent_buffer(SSL *s) { pitem *item = NULL; hm_fragment *frag = NULL; while ((item = pqueue_pop(s->d1->sent_messages)) != NULL) { frag = (hm_fragment *)item->data; - - if (frag->msg_header.is_ccs - && frag->msg_header.saved_retransmit_state.wrlmethod != NULL - && s->rlayer.wrl != frag->msg_header.saved_retransmit_state.wrl) { - /* - * If we're freeing the CCS then we're done with the old wrl and it - * can bee freed - */ - frag->msg_header.saved_retransmit_state.wrlmethod->free(frag->msg_header.saved_retransmit_state.wrl); - } - dtls1_hm_fragment_free(frag); pitem_free(item); } } -void dtls1_free(SSL *ssl) +void dtls1_free(SSL *s) { - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); + DTLS_RECORD_LAYER_free(&s->rlayer); - if (s == NULL) - return; + ssl3_free(s); if (s->d1 != NULL) { dtls1_clear_queues(s); @@ -160,26 +148,17 @@ void dtls1_free(SSL *ssl) pqueue_free(s->d1->sent_messages); } - DTLS_RECORD_LAYER_free(&s->rlayer); - - ssl3_free(ssl); - OPENSSL_free(s->d1); s->d1 = NULL; } -int dtls1_clear(SSL *ssl) +int dtls1_clear(SSL *s) { pqueue *buffered_messages; pqueue *sent_messages; size_t mtu; size_t link_mtu; - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - if (s == NULL) - return 0; - DTLS_RECORD_LAYER_clear(&s->rlayer); if (s->d1) { @@ -201,7 +180,7 @@ int dtls1_clear(SSL *ssl) s->d1->cookie_len = sizeof(s->d1->cookie); } - if (SSL_get_options(ssl) & SSL_OP_NO_QUERY_MTU) { + if (SSL_get_options(s) & SSL_OP_NO_QUERY_MTU) { s->d1->mtu = mtu; s->d1->link_mtu = link_mtu; } @@ -210,34 +189,28 @@ int dtls1_clear(SSL *ssl) s->d1->sent_messages = sent_messages; } - if (!ssl3_clear(ssl)) + if (!ssl3_clear(s)) return 0; - if (ssl->method->version == DTLS_ANY_VERSION) + if (s->method->version == DTLS_ANY_VERSION) s->version = DTLS_MAX_VERSION_INTERNAL; #ifndef OPENSSL_NO_DTLS1_METHOD else if (s->options & SSL_OP_CISCO_ANYCONNECT) s->client_version = s->version = DTLS1_BAD_VER; #endif else - s->version = ssl->method->version; + s->version = s->method->version; return 1; } -long dtls1_ctrl(SSL *ssl, int cmd, long larg, void *parg) +long dtls1_ctrl(SSL *s, int cmd, long larg, void *parg) { int ret = 0; - OSSL_TIME t; - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - if (s == NULL) - return 0; switch (cmd) { case DTLS_CTRL_GET_TIMEOUT: - if (dtls1_get_timeout(s, &t)) { - *(struct timeval *)parg = ossl_time_to_timeval(t); + if (dtls1_get_timeout(s, (struct timeval *)parg) != NULL) { ret = 1; } break; @@ -261,28 +234,20 @@ long dtls1_ctrl(SSL *ssl, int cmd, long larg, void *parg) s->d1->mtu = larg; return larg; default: - ret = ssl3_ctrl(ssl, cmd, larg, parg); + ret = ssl3_ctrl(s, cmd, larg, parg); break; } return ret; } -static void dtls1_bio_set_next_timeout(BIO *bio, const DTLS1_STATE *d1) +void dtls1_start_timer(SSL *s) { - struct timeval tv = ossl_time_to_timeval(d1->next_timeout); - - BIO_ctrl(bio, BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT, 0, &tv); -} - -void dtls1_start_timer(SSL_CONNECTION *s) -{ - OSSL_TIME duration; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); + unsigned int sec, usec; #ifndef OPENSSL_NO_SCTP /* Disable timer for SCTP */ - if (BIO_dgram_is_sctp(SSL_get_wbio(ssl))) { - s->d1->next_timeout = ossl_time_zero(); + if (BIO_dgram_is_sctp(SSL_get_wbio(s))) { + memset(&s->d1->next_timeout, 0, sizeof(s->d1->next_timeout)); return; } #endif @@ -291,89 +256,122 @@ void dtls1_start_timer(SSL_CONNECTION *s) * If timer is not set, initialize duration with 1 second or * a user-specified value if the timer callback is installed. */ - if (ossl_time_is_zero(s->d1->next_timeout)) { + if (s->d1->next_timeout.tv_sec == 0 && s->d1->next_timeout.tv_usec == 0) { + if (s->d1->timer_cb != NULL) - s->d1->timeout_duration_us = s->d1->timer_cb(ssl, 0); + s->d1->timeout_duration_us = s->d1->timer_cb(s, 0); else s->d1->timeout_duration_us = 1000000; } - /* Set timeout to current time plus duration */ - duration = ossl_us2time(s->d1->timeout_duration_us); - s->d1->next_timeout = ossl_time_add(ossl_time_now(), duration); + /* Set timeout to current time */ + get_current_time(&(s->d1->next_timeout)); + + /* Add duration to current time */ + + sec = s->d1->timeout_duration_us / 1000000; + usec = s->d1->timeout_duration_us - (sec * 1000000); + + s->d1->next_timeout.tv_sec += sec; + s->d1->next_timeout.tv_usec += usec; - /* set s->d1->next_timeout into ssl->rbio interface */ - dtls1_bio_set_next_timeout(SSL_get_rbio(ssl), s->d1); + if (s->d1->next_timeout.tv_usec >= 1000000) { + s->d1->next_timeout.tv_sec++; + s->d1->next_timeout.tv_usec -= 1000000; + } + + BIO_ctrl(SSL_get_rbio(s), BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT, 0, + &(s->d1->next_timeout)); } -int dtls1_get_timeout(const SSL_CONNECTION *s, OSSL_TIME *timeleft) +struct timeval *dtls1_get_timeout(SSL *s, struct timeval *timeleft) { - OSSL_TIME timenow; + struct timeval timenow; /* If no timeout is set, just return NULL */ - if (ossl_time_is_zero(s->d1->next_timeout)) - return 0; + if (s->d1->next_timeout.tv_sec == 0 && s->d1->next_timeout.tv_usec == 0) { + return NULL; + } /* Get current time */ - timenow = ossl_time_now(); + get_current_time(&timenow); + + /* If timer already expired, set remaining time to 0 */ + if (s->d1->next_timeout.tv_sec < timenow.tv_sec || + (s->d1->next_timeout.tv_sec == timenow.tv_sec && + s->d1->next_timeout.tv_usec <= timenow.tv_usec)) { + memset(timeleft, 0, sizeof(*timeleft)); + return timeleft; + } + + /* Calculate time left until timer expires */ + memcpy(timeleft, &(s->d1->next_timeout), sizeof(struct timeval)); + timeleft->tv_sec -= timenow.tv_sec; + timeleft->tv_usec -= timenow.tv_usec; + if (timeleft->tv_usec < 0) { + timeleft->tv_sec--; + timeleft->tv_usec += 1000000; + } /* - * If timer already expired or if remaining time is less than 15 ms, - * set it to 0 to prevent issues because of small divergences with - * socket timeouts. + * If remaining time is less than 15 ms, set it to 0 to prevent issues + * because of small divergences with socket timeouts. */ - *timeleft = ossl_time_subtract(s->d1->next_timeout, timenow); - if (ossl_time_compare(*timeleft, ossl_ms2time(15)) <= 0) - *timeleft = ossl_time_zero(); - return 1; + if (timeleft->tv_sec == 0 && timeleft->tv_usec < 15000) { + memset(timeleft, 0, sizeof(*timeleft)); + } + + return timeleft; } -int dtls1_is_timer_expired(SSL_CONNECTION *s) +int dtls1_is_timer_expired(SSL *s) { - OSSL_TIME timeleft; + struct timeval timeleft; /* Get time left until timeout, return false if no timer running */ - if (!dtls1_get_timeout(s, &timeleft)) + if (dtls1_get_timeout(s, &timeleft) == NULL) { return 0; + } /* Return false if timer is not expired yet */ - if (!ossl_time_is_zero(timeleft)) + if (timeleft.tv_sec > 0 || timeleft.tv_usec > 0) { return 0; + } /* Timer expired, so return true */ return 1; } -static void dtls1_double_timeout(SSL_CONNECTION *s) +static void dtls1_double_timeout(SSL *s) { s->d1->timeout_duration_us *= 2; if (s->d1->timeout_duration_us > 60000000) s->d1->timeout_duration_us = 60000000; } -void dtls1_stop_timer(SSL_CONNECTION *s) +void dtls1_stop_timer(SSL *s) { /* Reset everything */ s->d1->timeout_num_alerts = 0; - s->d1->next_timeout = ossl_time_zero(); + memset(&s->d1->next_timeout, 0, sizeof(s->d1->next_timeout)); s->d1->timeout_duration_us = 1000000; - dtls1_bio_set_next_timeout(s->rbio, s->d1); + BIO_ctrl(SSL_get_rbio(s), BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT, 0, + &(s->d1->next_timeout)); /* Clear retransmission buffer */ dtls1_clear_sent_buffer(s); } -int dtls1_check_timeout_num(SSL_CONNECTION *s) +int dtls1_check_timeout_num(SSL *s) { size_t mtu; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); s->d1->timeout_num_alerts++; /* Reduce MTU after 2 unsuccessful retransmissions */ if (s->d1->timeout_num_alerts > 2 - && !(SSL_get_options(ssl) & SSL_OP_NO_QUERY_MTU)) { + && !(SSL_get_options(s) & SSL_OP_NO_QUERY_MTU)) { mtu = - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_GET_FALLBACK_MTU, 0, NULL); + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_GET_FALLBACK_MTU, 0, NULL); if (mtu < s->d1->mtu) s->d1->mtu = mtu; } @@ -387,7 +385,7 @@ int dtls1_check_timeout_num(SSL_CONNECTION *s) return 0; } -int dtls1_handle_timeout(SSL_CONNECTION *s) +int dtls1_handle_timeout(SSL *s) { /* if no timer is expired, don't do anything */ if (!dtls1_is_timer_expired(s)) { @@ -395,8 +393,7 @@ int dtls1_handle_timeout(SSL_CONNECTION *s) } if (s->d1->timer_cb != NULL) - s->d1->timeout_duration_us = s->d1->timer_cb(SSL_CONNECTION_GET_SSL(s), - s->d1->timeout_duration_us); + s->d1->timeout_duration_us = s->d1->timer_cb(s, s->d1->timeout_duration_us); else dtls1_double_timeout(s); @@ -410,40 +407,62 @@ int dtls1_handle_timeout(SSL_CONNECTION *s) return dtls1_retransmit_buffered_messages(s); } +static void get_current_time(struct timeval *t) +{ +#if defined(_WIN32) + SYSTEMTIME st; + union { + unsigned __int64 ul; + FILETIME ft; + } now; + + GetSystemTime(&st); + SystemTimeToFileTime(&st, &now.ft); + /* re-bias to 1/1/1970 */ +# ifdef __MINGW32__ + now.ul -= 116444736000000000ULL; +# else + /* *INDENT-OFF* */ + now.ul -= 116444736000000000UI64; + /* *INDENT-ON* */ +# endif + t->tv_sec = (long)(now.ul / 10000000); + t->tv_usec = ((int)(now.ul % 10000000)) / 10; +#else + gettimeofday(t, NULL); +#endif +} + #define LISTEN_SUCCESS 2 #define LISTEN_SEND_VERIFY_REQUEST 1 #ifndef OPENSSL_NO_SOCK -int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) +int DTLSv1_listen(SSL *s, BIO_ADDR *client) { int next, n, ret = 0; unsigned char cookie[DTLS1_COOKIE_LENGTH]; unsigned char seq[SEQ_NUM_SIZE]; const unsigned char *data; - unsigned char *buf = NULL, *wbuf; - size_t fragoff, fraglen, msglen; - unsigned int rectype, versmajor, versminor, msgseq, msgtype, clientvers, cookielen; + unsigned char *buf, *wbuf; + size_t fragoff, fraglen, msglen, reclen, align = 0; + unsigned int rectype, versmajor, msgseq, msgtype, clientvers, cookielen; BIO *rbio, *wbio; BIO_ADDR *tmpclient = NULL; PACKET pkt, msgpkt, msgpayload, session, cookiepkt; - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - if (s == NULL) - return -1; if (s->handshake_func == NULL) { /* Not properly initialized yet */ - SSL_set_accept_state(ssl); + SSL_set_accept_state(s); } /* Ensure there is no state left over from a previous invocation */ - if (!SSL_clear(ssl)) + if (!SSL_clear(s)) return -1; ERR_clear_error(); - rbio = SSL_get_rbio(ssl); - wbio = SSL_get_wbio(ssl); + rbio = SSL_get_rbio(s); + wbio = SSL_get_wbio(s); if (!rbio || !wbio) { ERR_raise(ERR_LIB_SSL, SSL_R_BIO_NOT_SET); @@ -462,14 +481,25 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) return -1; } - buf = OPENSSL_malloc(DTLS1_RT_HEADER_LENGTH + SSL3_RT_MAX_PLAIN_LENGTH); - if (buf == NULL) - return -1; - wbuf = OPENSSL_malloc(DTLS1_RT_HEADER_LENGTH + SSL3_RT_MAX_PLAIN_LENGTH); - if (wbuf == NULL) { - OPENSSL_free(buf); + if (!ssl3_setup_buffers(s)) { + /* ERR_raise() already called */ return -1; } + buf = RECORD_LAYER_get_rbuf(&s->rlayer)->buf; + wbuf = RECORD_LAYER_get_wbuf(&s->rlayer)[0].buf; +#if defined(SSL3_ALIGN_PAYLOAD) +# if SSL3_ALIGN_PAYLOAD != 0 + /* + * Using SSL3_RT_HEADER_LENGTH here instead of DTLS1_RT_HEADER_LENGTH for + * consistency with ssl3_read_n. In practice it should make no difference + * for sensible values of SSL3_ALIGN_PAYLOAD because the difference between + * SSL3_RT_HEADER_LENGTH and DTLS1_RT_HEADER_LENGTH is exactly 8 + */ + align = (size_t)buf + SSL3_RT_HEADER_LENGTH; + align = SSL3_ALIGN_PAYLOAD - 1 - ((align - 1) % SSL3_ALIGN_PAYLOAD); +# endif +#endif + buf += align; do { /* Get a packet */ @@ -482,14 +512,12 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) /* Non-blocking IO */ goto end; } - ret = -1; - goto end; + return -1; } if (!PACKET_buf_init(&pkt, buf, n)) { ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - ret = -1; - goto end; + return -1; } /* @@ -507,18 +535,17 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) goto end; } + if (s->msg_callback) + s->msg_callback(0, 0, SSL3_RT_HEADER, buf, + DTLS1_RT_HEADER_LENGTH, s, s->msg_callback_arg); + /* Get the record header */ if (!PACKET_get_1(&pkt, &rectype) - || !PACKET_get_1(&pkt, &versmajor) - || !PACKET_get_1(&pkt, &versminor)) { + || !PACKET_get_1(&pkt, &versmajor)) { ERR_raise(ERR_LIB_SSL, SSL_R_LENGTH_MISMATCH); goto end; } - if (s->msg_callback) - s->msg_callback(0, (versmajor << 8) | versminor, SSL3_RT_HEADER, buf, - DTLS1_RT_HEADER_LENGTH, ssl, s->msg_callback_arg); - if (rectype != SSL3_RT_HANDSHAKE) { ERR_raise(ERR_LIB_SSL, SSL_R_UNEXPECTED_MESSAGE); goto end; @@ -533,12 +560,14 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) goto end; } - /* Save the sequence number: 64 bits, with top 2 bytes = epoch */ - if (!PACKET_copy_bytes(&pkt, seq, SEQ_NUM_SIZE) + if (!PACKET_forward(&pkt, 1) + /* Save the sequence number: 64 bits, with top 2 bytes = epoch */ + || !PACKET_copy_bytes(&pkt, seq, SEQ_NUM_SIZE) || !PACKET_get_length_prefixed_2(&pkt, &msgpkt)) { ERR_raise(ERR_LIB_SSL, SSL_R_LENGTH_MISMATCH); goto end; } + reclen = PACKET_remaining(&msgpkt); /* * We allow data remaining at the end of the packet because there could * be a second record (but we ignore it) @@ -591,7 +620,7 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) if (s->msg_callback) s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, data, - fraglen + DTLS1_HM_HEADER_LENGTH, ssl, + fraglen + DTLS1_HM_HEADER_LENGTH, s, s->msg_callback_arg); if (!PACKET_get_net_2(&msgpayload, &clientvers)) { @@ -602,8 +631,8 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) /* * Verify client version is supported */ - if (DTLS_VERSION_LT(clientvers, (unsigned int)ssl->method->version) && - ssl->method->version != DTLS_ANY_VERSION) { + if (DTLS_VERSION_LT(clientvers, (unsigned int)s->method->version) && + s->method->version != DTLS_ANY_VERSION) { ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_VERSION_NUMBER); goto end; } @@ -629,13 +658,12 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) /* * We have a cookie, so lets check it. */ - if (ssl->ctx->app_verify_cookie_cb == NULL) { + if (s->ctx->app_verify_cookie_cb == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_NO_VERIFY_COOKIE_CALLBACK); /* This is fatal */ - ret = -1; - goto end; + return -1; } - if (ssl->ctx->app_verify_cookie_cb(ssl, PACKET_data(&cookiepkt), + if (s->ctx->app_verify_cookie_cb(s, PACKET_data(&cookiepkt), (unsigned int)PACKET_remaining(&cookiepkt)) == 0) { /* * We treat invalid cookies in the same was as no cookie as @@ -660,13 +688,12 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) */ /* Generate the cookie */ - if (ssl->ctx->app_gen_cookie_cb == NULL || - ssl->ctx->app_gen_cookie_cb(ssl, cookie, &cookielen) == 0 || + if (s->ctx->app_gen_cookie_cb == NULL || + s->ctx->app_gen_cookie_cb(s, cookie, &cookielen) == 0 || cookielen > 255) { ERR_raise(ERR_LIB_SSL, SSL_R_COOKIE_GEN_CALLBACK_FAILURE); /* This is fatal */ - ret = -1; - goto end; + return -1; } /* @@ -674,8 +701,8 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) * haven't decided which version to use yet send back using version * 1.0 header: otherwise some clients will ignore it. */ - version = (ssl->method->version == DTLS_ANY_VERSION) ? DTLS1_VERSION - : s->version; + version = (s->method->version == DTLS_ANY_VERSION) ? DTLS1_VERSION + : s->version; /* Construct the record and message headers */ if (!WPACKET_init_static_len(&wpkt, @@ -730,8 +757,7 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); WPACKET_cleanup(&wpkt); /* This is fatal */ - ret = -1; - goto end; + return -1; } /* @@ -747,11 +773,10 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) if (s->msg_callback) s->msg_callback(1, 0, SSL3_RT_HEADER, buf, - DTLS1_RT_HEADER_LENGTH, ssl, - s->msg_callback_arg); + DTLS1_RT_HEADER_LENGTH, s, s->msg_callback_arg); if ((tmpclient = BIO_ADDR_new()) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto end; } @@ -774,8 +799,7 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) */ goto end; } - ret = -1; - goto end; + return -1; } if (BIO_flush(wbio) <= 0) { @@ -786,8 +810,7 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) */ goto end; } - ret = -1; - goto end; + return -1; } } } while (next != LISTEN_SUCCESS); @@ -798,13 +821,13 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) s->d1->handshake_read_seq = 1; s->d1->handshake_write_seq = 1; s->d1->next_handshake_write_seq = 1; - s->rlayer.wrlmethod->increment_sequence_ctr(s->rlayer.wrl); + DTLS_RECORD_LAYER_set_write_sequence(&s->rlayer, seq); /* * We are doing cookie exchange, so make sure we set that option in the * SSL object */ - SSL_set_options(ssl, SSL_OP_COOKIE_EXCHANGE); + SSL_set_options(s, SSL_OP_COOKIE_EXCHANGE); /* * Tell the state machine that we've done the initial hello verify @@ -818,38 +841,18 @@ int DTLSv1_listen(SSL *ssl, BIO_ADDR *client) if (BIO_dgram_get_peer(rbio, client) <= 0) BIO_ADDR_clear(client); - /* Buffer the record for use by the record layer */ - if (BIO_write(s->rlayer.rrlnext, buf, n) != n) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - ret = -1; - goto end; - } - - /* - * Reset the record layer - but this time we can use the record we just - * buffered in s->rlayer.rrlnext - */ - if (!ssl_set_new_record_layer(s, - DTLS_ANY_VERSION, - OSSL_RECORD_DIRECTION_READ, - OSSL_RECORD_PROTECTION_LEVEL_NONE, NULL, 0, - NULL, 0, NULL, 0, NULL, 0, NULL, 0, - NID_undef, NULL, NULL, NULL)) { - /* SSLfatal already called */ - ret = -1; - goto end; - } + /* Buffer the record in the processed_rcds queue */ + if (!dtls_buffer_listen_record(s, reclen, seq, align)) + return -1; ret = 1; end: BIO_ADDR_free(tmpclient); - OPENSSL_free(buf); - OPENSSL_free(wbuf); return ret; } #endif -static int dtls1_handshake_write(SSL_CONNECTION *s) +static int dtls1_handshake_write(SSL *s) { return dtls1_do_write(s, SSL3_RT_HANDSHAKE); } @@ -859,14 +862,10 @@ int dtls1_shutdown(SSL *s) int ret; #ifndef OPENSSL_NO_SCTP BIO *wbio; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (s == NULL) - return -1; wbio = SSL_get_wbio(s); if (wbio != NULL && BIO_dgram_is_sctp(wbio) && - !(sc->shutdown & SSL_SENT_SHUTDOWN)) { + !(s->shutdown & SSL_SENT_SHUTDOWN)) { ret = BIO_dgram_sctp_wait_for_dry(wbio); if (ret < 0) return -1; @@ -883,21 +882,19 @@ int dtls1_shutdown(SSL *s) return ret; } -int dtls1_query_mtu(SSL_CONNECTION *s) +int dtls1_query_mtu(SSL *s) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - if (s->d1->link_mtu) { s->d1->mtu = - s->d1->link_mtu - BIO_dgram_get_mtu_overhead(SSL_get_wbio(ssl)); + s->d1->link_mtu - BIO_dgram_get_mtu_overhead(SSL_get_wbio(s)); s->d1->link_mtu = 0; } /* AHA! Figure out the MTU, and stick to the right size */ if (s->d1->mtu < dtls1_min_mtu(s)) { - if (!(SSL_get_options(ssl) & SSL_OP_NO_QUERY_MTU)) { + if (!(SSL_get_options(s) & SSL_OP_NO_QUERY_MTU)) { s->d1->mtu = - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_QUERY_MTU, 0, NULL); + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_QUERY_MTU, 0, NULL); /* * I've seen the kernel return bogus numbers when it doesn't know @@ -906,7 +903,7 @@ int dtls1_query_mtu(SSL_CONNECTION *s) if (s->d1->mtu < dtls1_min_mtu(s)) { /* Set to min mtu */ s->d1->mtu = dtls1_min_mtu(s); - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SET_MTU, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SET_MTU, (long)s->d1->mtu, NULL); } } else @@ -921,24 +918,16 @@ static size_t dtls1_link_min_mtu(void) sizeof(g_probable_mtu[0])) - 1]); } -size_t dtls1_min_mtu(SSL_CONNECTION *s) +size_t dtls1_min_mtu(SSL *s) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - - return dtls1_link_min_mtu() - BIO_dgram_get_mtu_overhead(SSL_get_wbio(ssl)); + return dtls1_link_min_mtu() - BIO_dgram_get_mtu_overhead(SSL_get_wbio(s)); } -size_t DTLS_get_data_mtu(const SSL *ssl) +size_t DTLS_get_data_mtu(const SSL *s) { size_t mac_overhead, int_overhead, blocksize, ext_overhead; - const SSL_CIPHER *ciph = SSL_get_current_cipher(ssl); - size_t mtu; - const SSL_CONNECTION *s = SSL_CONNECTION_FROM_CONST_SSL_ONLY(ssl); - - if (s == NULL) - return 0; - - mtu = s->d1->mtu; + const SSL_CIPHER *ciph = SSL_get_current_cipher(s); + size_t mtu = s->d1->mtu; if (ciph == NULL) return 0; @@ -970,12 +959,7 @@ size_t DTLS_get_data_mtu(const SSL *ssl) return mtu; } -void DTLS_set_timer_cb(SSL *ssl, DTLS_timer_cb cb) +void DTLS_set_timer_cb(SSL *s, DTLS_timer_cb cb) { - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - if (s == NULL) - return; - s->d1->timer_cb = cb; } diff --git a/openssl/src/ssl/d1_msg.c b/openssl/src/ssl/d1_msg.c index b1e1fad16..10438a395 100644 --- a/openssl/src/ssl/d1_msg.c +++ b/openssl/src/ssl/d1_msg.c @@ -1,5 +1,5 @@ /* - * Copyright 2005-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,17 +9,13 @@ #include "ssl_local.h" -int dtls1_write_app_data_bytes(SSL *s, uint8_t type, const void *buf_, - size_t len, size_t *written) +int dtls1_write_app_data_bytes(SSL *s, int type, const void *buf_, size_t len, + size_t *written) { int i; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - if (sc == NULL) - return -1; - - if (SSL_in_init(s) && !ossl_statem_get_in_handshake(sc)) { - i = sc->handshake_func(s); + if (SSL_in_init(s) && !ossl_statem_get_in_handshake(s)) { + i = s->handshake_func(s); if (i < 0) return i; if (i == 0) { @@ -33,46 +29,42 @@ int dtls1_write_app_data_bytes(SSL *s, uint8_t type, const void *buf_, return -1; } - return dtls1_write_bytes(sc, type, buf_, len, written); + return dtls1_write_bytes(s, type, buf_, len, written); } -int dtls1_dispatch_alert(SSL *ssl) +int dtls1_dispatch_alert(SSL *s) { int i, j; void (*cb) (const SSL *ssl, int type, int val) = NULL; unsigned char buf[DTLS1_AL_HEADER_LENGTH]; unsigned char *ptr = &buf[0]; size_t written; - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - if (s == NULL) - return 0; - s->s3.alert_dispatch = SSL_ALERT_DISPATCH_NONE; + s->s3.alert_dispatch = 0; memset(buf, 0, sizeof(buf)); *ptr++ = s->s3.send_alert[0]; *ptr++ = s->s3.send_alert[1]; - i = do_dtls1_write(s, SSL3_RT_ALERT, &buf[0], sizeof(buf), &written); + i = do_dtls1_write(s, SSL3_RT_ALERT, &buf[0], sizeof(buf), 0, &written); if (i <= 0) { s->s3.alert_dispatch = 1; - /* fprintf(stderr, "not done with alert\n"); */ + /* fprintf( stderr, "not done with alert\n" ); */ } else { (void)BIO_flush(s->wbio); if (s->msg_callback) s->msg_callback(1, s->version, SSL3_RT_ALERT, s->s3.send_alert, - 2, ssl, s->msg_callback_arg); + 2, s, s->msg_callback_arg); if (s->info_callback != NULL) cb = s->info_callback; - else if (ssl->ctx->info_callback != NULL) - cb = ssl->ctx->info_callback; + else if (s->ctx->info_callback != NULL) + cb = s->ctx->info_callback; if (cb != NULL) { j = (s->s3.send_alert[0] << 8) | s->s3.send_alert[1]; - cb(ssl, SSL_CB_WRITE_ALERT, j); + cb(s, SSL_CB_WRITE_ALERT, j); } } return i; diff --git a/openssl/src/ssl/d1_srtp.c b/openssl/src/ssl/d1_srtp.c index 155021ff5..230075338 100644 --- a/openssl/src/ssl/d1_srtp.c +++ b/openssl/src/ssl/d1_srtp.c @@ -1,5 +1,5 @@ /* - * Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -16,11 +16,10 @@ #include #include #include "ssl_local.h" -#include "quic/quic_local.h" #ifndef OPENSSL_NO_SRTP -static const SRTP_PROTECTION_PROFILE srtp_known_profiles[] = { +static SRTP_PROTECTION_PROFILE srtp_known_profiles[] = { { "SRTP_AES128_CM_SHA1_80", SRTP_AES128_CM_SHA1_80, @@ -37,45 +36,13 @@ static const SRTP_PROTECTION_PROFILE srtp_known_profiles[] = { "SRTP_AEAD_AES_256_GCM", SRTP_AEAD_AES_256_GCM, }, - { - "SRTP_DOUBLE_AEAD_AES_128_GCM_AEAD_AES_128_GCM", - SRTP_DOUBLE_AEAD_AES_128_GCM_AEAD_AES_128_GCM, - }, - { - "SRTP_DOUBLE_AEAD_AES_256_GCM_AEAD_AES_256_GCM", - SRTP_DOUBLE_AEAD_AES_256_GCM_AEAD_AES_256_GCM, - }, - { - "SRTP_ARIA_128_CTR_HMAC_SHA1_80", - SRTP_ARIA_128_CTR_HMAC_SHA1_80, - }, - { - "SRTP_ARIA_128_CTR_HMAC_SHA1_32", - SRTP_ARIA_128_CTR_HMAC_SHA1_32, - }, - { - "SRTP_ARIA_256_CTR_HMAC_SHA1_80", - SRTP_ARIA_256_CTR_HMAC_SHA1_80, - }, - { - "SRTP_ARIA_256_CTR_HMAC_SHA1_32", - SRTP_ARIA_256_CTR_HMAC_SHA1_32, - }, - { - "SRTP_AEAD_ARIA_128_GCM", - SRTP_AEAD_ARIA_128_GCM, - }, - { - "SRTP_AEAD_ARIA_256_GCM", - SRTP_AEAD_ARIA_256_GCM, - }, {0} }; static int find_profile_by_name(char *profile_name, - const SRTP_PROTECTION_PROFILE **pptr, size_t len) + SRTP_PROTECTION_PROFILE **pptr, size_t len) { - const SRTP_PROTECTION_PROFILE *p; + SRTP_PROTECTION_PROFILE *p; p = srtp_known_profiles; while (p->name) { @@ -98,7 +65,7 @@ static int ssl_ctx_make_profiles(const char *profiles_string, char *col; char *ptr = (char *)profiles_string; - const SRTP_PROTECTION_PROFILE *p; + SRTP_PROTECTION_PROFILE *p; if ((profiles = sk_SRTP_PROTECTION_PROFILE_new_null()) == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_SRTP_COULD_NOT_ALLOCATE_PROFILES); @@ -110,14 +77,12 @@ static int ssl_ctx_make_profiles(const char *profiles_string, if (!find_profile_by_name(ptr, &p, col ? (size_t)(col - ptr) : strlen(ptr))) { - if (sk_SRTP_PROTECTION_PROFILE_find(profiles, - (SRTP_PROTECTION_PROFILE *)p) >= 0) { + if (sk_SRTP_PROTECTION_PROFILE_find(profiles, p) >= 0) { ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST); goto err; } - if (!sk_SRTP_PROTECTION_PROFILE_push(profiles, - (SRTP_PROTECTION_PROFILE *)p)) { + if (!sk_SRTP_PROTECTION_PROFILE_push(profiles, p)) { ERR_raise(ERR_LIB_SSL, SSL_R_SRTP_COULD_NOT_ALLOCATE_PROFILES); goto err; } @@ -142,29 +107,19 @@ static int ssl_ctx_make_profiles(const char *profiles_string, int SSL_CTX_set_tlsext_use_srtp(SSL_CTX *ctx, const char *profiles) { - if (IS_QUIC_METHOD(ctx->method)) - return 1; - return ssl_ctx_make_profiles(profiles, &ctx->srtp_profiles); } int SSL_set_tlsext_use_srtp(SSL *s, const char *profiles) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 1; - - return ssl_ctx_make_profiles(profiles, &sc->srtp_profiles); + return ssl_ctx_make_profiles(profiles, &s->srtp_profiles); } STACK_OF(SRTP_PROTECTION_PROFILE) *SSL_get_srtp_profiles(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc != NULL) { - if (sc->srtp_profiles != NULL) { - return sc->srtp_profiles; + if (s != NULL) { + if (s->srtp_profiles != NULL) { + return s->srtp_profiles; } else if ((s->ctx != NULL) && (s->ctx->srtp_profiles != NULL)) { return s->ctx->srtp_profiles; } @@ -175,11 +130,6 @@ STACK_OF(SRTP_PROTECTION_PROFILE) *SSL_get_srtp_profiles(SSL *s) SRTP_PROTECTION_PROFILE *SSL_get_selected_srtp_profile(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - - return sc->srtp_profile; + return s->srtp_profile; } #endif diff --git a/openssl/src/ssl/event_queue.c b/openssl/src/ssl/event_queue.c deleted file mode 100644 index 20f4020db..000000000 --- a/openssl/src/ssl/event_queue.c +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/event_queue.h" -#include "ssl_local.h" - -struct ossl_event_queue_st { - PRIORITY_QUEUE_OF(OSSL_EVENT) *timed_events; - PRIORITY_QUEUE_OF(OSSL_EVENT) *now_events; -}; - -static int event_compare_times(const OSSL_EVENT *a, const OSSL_EVENT *b) -{ - return ossl_time_compare(a->when, b->when); -} - -static int event_compare_priority(const OSSL_EVENT *a, const OSSL_EVENT *b) -{ - if (a->priority > b->priority) - return -1; - if (a->priority < b->priority) - return 1; - return 0; -} - -OSSL_EVENT_QUEUE *ossl_event_queue_new(void) -{ - OSSL_EVENT_QUEUE *r = OPENSSL_malloc(sizeof(*r)); - - if (r != NULL) { - r->timed_events = ossl_pqueue_OSSL_EVENT_new(&event_compare_times); - r->now_events = ossl_pqueue_OSSL_EVENT_new(&event_compare_priority); - if (r->timed_events == NULL || r->now_events == NULL) { - ossl_event_queue_free(r); - return NULL; - } - } - return r; -} - -void ossl_event_free(OSSL_EVENT *event) -{ - if (event != NULL) { - if (event->flag_dynamic) - OPENSSL_free(event); - else - event->queue = NULL; - } -} - -static void event_queue_free(PRIORITY_QUEUE_OF(OSSL_EVENT) *queue) -{ - OSSL_EVENT *e; - - if (queue != NULL) { - while ((e = ossl_pqueue_OSSL_EVENT_pop(queue)) != NULL) - ossl_event_free(e); - ossl_pqueue_OSSL_EVENT_free(queue); - } -} - -void ossl_event_queue_free(OSSL_EVENT_QUEUE *queue) -{ - if (queue != NULL) { - event_queue_free(queue->now_events); - event_queue_free(queue->timed_events); - OPENSSL_free(queue); - } -} - -static ossl_inline -int event_queue_add(OSSL_EVENT_QUEUE *queue, OSSL_EVENT *event) -{ - PRIORITY_QUEUE_OF(OSSL_EVENT) *pq = - ossl_time_compare(event->when, ossl_time_now()) <= 0 - ? queue->now_events - : queue->timed_events; - - if (ossl_pqueue_OSSL_EVENT_push(pq, event, &event->ref)) { - event->queue = pq; - return 1; - } - return 0; -} - -static ossl_inline -void ossl_event_set(OSSL_EVENT *event, uint32_t type, uint32_t priority, - OSSL_TIME when, void *ctx, - void *payload, size_t payload_size) -{ - event->type = type; - event->priority = priority; - event->when = when; - event->ctx = ctx; - event->payload = payload; - event->payload_size = payload_size; -} - -OSSL_EVENT *ossl_event_queue_add_new(OSSL_EVENT_QUEUE *queue, - uint32_t type, uint32_t priority, - OSSL_TIME when, void *ctx, - void *payload, size_t payload_size) -{ - OSSL_EVENT *e = OPENSSL_malloc(sizeof(*e)); - - if (e == NULL || queue == NULL) { - OPENSSL_free(e); - return NULL; - } - - ossl_event_set(e, type, priority, when, ctx, payload, payload_size); - e->flag_dynamic = 1; - if (event_queue_add(queue, e)) - return e; - OPENSSL_free(e); - return NULL; -} - -int ossl_event_queue_add(OSSL_EVENT_QUEUE *queue, OSSL_EVENT *event, - uint32_t type, uint32_t priority, - OSSL_TIME when, void *ctx, - void *payload, size_t payload_size) -{ - if (event == NULL || queue == NULL) - return 0; - ossl_event_set(event, type, priority, when, ctx, payload, payload_size); - event->flag_dynamic = 0; - return event_queue_add(queue, event); -} - -int ossl_event_queue_remove(OSSL_EVENT_QUEUE *queue, OSSL_EVENT *event) -{ - if (event != NULL && event->queue != NULL) { - ossl_pqueue_OSSL_EVENT_remove(event->queue, event->ref); - event->queue = NULL; - } - return 1; -} - -OSSL_TIME ossl_event_time_until(const OSSL_EVENT *event) -{ - if (event == NULL) - return ossl_time_infinite(); - return ossl_time_subtract(event->when, ossl_time_now()); -} - -OSSL_TIME ossl_event_queue_time_until_next(const OSSL_EVENT_QUEUE *queue) -{ - if (queue == NULL) - return ossl_time_infinite(); - if (ossl_pqueue_OSSL_EVENT_num(queue->now_events) > 0) - return ossl_time_zero(); - return ossl_event_time_until(ossl_pqueue_OSSL_EVENT_peek(queue->timed_events)); -} - -int ossl_event_queue_postpone_until(OSSL_EVENT_QUEUE *queue, - OSSL_EVENT *event, - OSSL_TIME when) -{ - if (ossl_event_queue_remove(queue, event)) { - event->when = when; - return event_queue_add(queue, event); - } - return 0; -} - -int ossl_event_queue_get1_next_event(OSSL_EVENT_QUEUE *queue, - OSSL_EVENT **event) -{ - OSSL_TIME now = ossl_time_now(); - OSSL_EVENT *e; - - /* Check for expired timer based events and convert them to now events */ - while ((e = ossl_pqueue_OSSL_EVENT_peek(queue->timed_events)) != NULL - && ossl_time_compare(e->when, now) <= 0) { - e = ossl_pqueue_OSSL_EVENT_pop(queue->timed_events); - if (!ossl_pqueue_OSSL_EVENT_push(queue->now_events, e, &e->ref)) { - e->queue = NULL; - return 0; - } - } - - /* - * Get next event from the now queue. - * The pop returns NULL when there is none. - */ - *event = ossl_pqueue_OSSL_EVENT_pop(queue->now_events); - return 1; -} diff --git a/openssl/src/ssl/ktls.c b/openssl/src/ssl/ktls.c new file mode 100644 index 000000000..79d980959 --- /dev/null +++ b/openssl/src/ssl/ktls.c @@ -0,0 +1,244 @@ +/* + * Copyright 2018-2021 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "ssl_local.h" +#include "internal/ktls.h" + +#if defined(__FreeBSD__) +# include "crypto/cryptodev.h" + +/*- + * Check if a given cipher is supported by the KTLS interface. + * The kernel might still fail the setsockopt() if no suitable + * provider is found, but this checks if the socket option + * supports the cipher suite used at all. + */ +int ktls_check_supported_cipher(const SSL *s, const EVP_CIPHER *c, + const EVP_CIPHER_CTX *dd) +{ + + switch (s->version) { + case TLS1_VERSION: + case TLS1_1_VERSION: + case TLS1_2_VERSION: + case TLS1_3_VERSION: + break; + default: + return 0; + } + + switch (s->s3.tmp.new_cipher->algorithm_enc) { + case SSL_AES128GCM: + case SSL_AES256GCM: + return 1; + case SSL_AES128: + case SSL_AES256: + if (s->ext.use_etm) + return 0; + switch (s->s3.tmp.new_cipher->algorithm_mac) { + case SSL_SHA1: + case SSL_SHA256: + case SSL_SHA384: + return 1; + default: + return 0; + } + default: + return 0; + } +} + +/* Function to configure kernel TLS structure */ +int ktls_configure_crypto(const SSL *s, const EVP_CIPHER *c, EVP_CIPHER_CTX *dd, + void *rl_sequence, ktls_crypto_info_t *crypto_info, + unsigned char **rec_seq, unsigned char *iv, + unsigned char *key, unsigned char *mac_key, + size_t mac_secret_size) +{ + memset(crypto_info, 0, sizeof(*crypto_info)); + switch (s->s3.tmp.new_cipher->algorithm_enc) { + case SSL_AES128GCM: + case SSL_AES256GCM: + crypto_info->cipher_algorithm = CRYPTO_AES_NIST_GCM_16; + if (s->version == TLS1_3_VERSION) + crypto_info->iv_len = EVP_CIPHER_CTX_get_iv_length(dd); + else + crypto_info->iv_len = EVP_GCM_TLS_FIXED_IV_LEN; + break; + case SSL_AES128: + case SSL_AES256: + switch (s->s3.tmp.new_cipher->algorithm_mac) { + case SSL_SHA1: + crypto_info->auth_algorithm = CRYPTO_SHA1_HMAC; + break; + case SSL_SHA256: + crypto_info->auth_algorithm = CRYPTO_SHA2_256_HMAC; + break; + case SSL_SHA384: + crypto_info->auth_algorithm = CRYPTO_SHA2_384_HMAC; + break; + default: + return 0; + } + crypto_info->cipher_algorithm = CRYPTO_AES_CBC; + crypto_info->iv_len = EVP_CIPHER_get_iv_length(c); + crypto_info->auth_key = mac_key; + crypto_info->auth_key_len = mac_secret_size; + break; + default: + return 0; + } + crypto_info->cipher_key = key; + crypto_info->cipher_key_len = EVP_CIPHER_get_key_length(c); + crypto_info->iv = iv; + crypto_info->tls_vmajor = (s->version >> 8) & 0x000000ff; + crypto_info->tls_vminor = (s->version & 0x000000ff); +# ifdef TCP_RXTLS_ENABLE + memcpy(crypto_info->rec_seq, rl_sequence, sizeof(crypto_info->rec_seq)); + if (rec_seq != NULL) + *rec_seq = crypto_info->rec_seq; +# else + if (rec_seq != NULL) + *rec_seq = NULL; +# endif + return 1; +}; + +#endif /* __FreeBSD__ */ + +#if defined(OPENSSL_SYS_LINUX) + +/* Function to check supported ciphers in Linux */ +int ktls_check_supported_cipher(const SSL *s, const EVP_CIPHER *c, + const EVP_CIPHER_CTX *dd) +{ + switch (s->version) { + case TLS1_2_VERSION: + case TLS1_3_VERSION: + break; + default: + return 0; + } + + /* check that cipher is AES_GCM_128, AES_GCM_256, AES_CCM_128 + * or Chacha20-Poly1305 + */ +# ifdef OPENSSL_KTLS_AES_CCM_128 + if (EVP_CIPHER_is_a(c, "AES-128-CCM")) { + if (s->version == TLS_1_3_VERSION /* broken on 5.x kernels */ + || EVP_CIPHER_CTX_get_tag_length(dd) != EVP_CCM_TLS_TAG_LEN) + return 0; + return 1; + } else +# endif + if (0 +# ifdef OPENSSL_KTLS_AES_GCM_128 + || EVP_CIPHER_is_a(c, "AES-128-GCM") +# endif +# ifdef OPENSSL_KTLS_AES_GCM_256 + || EVP_CIPHER_is_a(c, "AES-256-GCM") +# endif +# ifdef OPENSSL_KTLS_CHACHA20_POLY1305 + || EVP_CIPHER_is_a(c, "ChaCha20-Poly1305") +# endif + ) { + return 1; + } + return 0; +} + +/* Function to configure kernel TLS structure */ +int ktls_configure_crypto(const SSL *s, const EVP_CIPHER *c, EVP_CIPHER_CTX *dd, + void *rl_sequence, ktls_crypto_info_t *crypto_info, + unsigned char **rec_seq, unsigned char *iv, + unsigned char *key, unsigned char *mac_key, + size_t mac_secret_size) +{ + unsigned char geniv[12]; + unsigned char *iiv = iv; + + if (s->version == TLS1_2_VERSION && + EVP_CIPHER_get_mode(c) == EVP_CIPH_GCM_MODE) { + if (!EVP_CIPHER_CTX_get_updated_iv(dd, geniv, + EVP_GCM_TLS_FIXED_IV_LEN + + EVP_GCM_TLS_EXPLICIT_IV_LEN)) + return 0; + iiv = geniv; + } + + memset(crypto_info, 0, sizeof(*crypto_info)); + switch (EVP_CIPHER_get_nid(c)) + { +# ifdef OPENSSL_KTLS_AES_GCM_128 + case NID_aes_128_gcm: + crypto_info->gcm128.info.cipher_type = TLS_CIPHER_AES_GCM_128; + crypto_info->gcm128.info.version = s->version; + crypto_info->tls_crypto_info_len = sizeof(crypto_info->gcm128); + memcpy(crypto_info->gcm128.iv, iiv + EVP_GCM_TLS_FIXED_IV_LEN, + TLS_CIPHER_AES_GCM_128_IV_SIZE); + memcpy(crypto_info->gcm128.salt, iiv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); + memcpy(crypto_info->gcm128.key, key, EVP_CIPHER_get_key_length(c)); + memcpy(crypto_info->gcm128.rec_seq, rl_sequence, + TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); + if (rec_seq != NULL) + *rec_seq = crypto_info->gcm128.rec_seq; + return 1; +# endif +# ifdef OPENSSL_KTLS_AES_GCM_256 + case NID_aes_256_gcm: + crypto_info->gcm256.info.cipher_type = TLS_CIPHER_AES_GCM_256; + crypto_info->gcm256.info.version = s->version; + crypto_info->tls_crypto_info_len = sizeof(crypto_info->gcm256); + memcpy(crypto_info->gcm256.iv, iiv + EVP_GCM_TLS_FIXED_IV_LEN, + TLS_CIPHER_AES_GCM_256_IV_SIZE); + memcpy(crypto_info->gcm256.salt, iiv, TLS_CIPHER_AES_GCM_256_SALT_SIZE); + memcpy(crypto_info->gcm256.key, key, EVP_CIPHER_get_key_length(c)); + memcpy(crypto_info->gcm256.rec_seq, rl_sequence, + TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE); + if (rec_seq != NULL) + *rec_seq = crypto_info->gcm256.rec_seq; + return 1; +# endif +# ifdef OPENSSL_KTLS_AES_CCM_128 + case NID_aes_128_ccm: + crypto_info->ccm128.info.cipher_type = TLS_CIPHER_AES_CCM_128; + crypto_info->ccm128.info.version = s->version; + crypto_info->tls_crypto_info_len = sizeof(crypto_info->ccm128); + memcpy(crypto_info->ccm128.iv, iiv + EVP_CCM_TLS_FIXED_IV_LEN, + TLS_CIPHER_AES_CCM_128_IV_SIZE); + memcpy(crypto_info->ccm128.salt, iiv, TLS_CIPHER_AES_CCM_128_SALT_SIZE); + memcpy(crypto_info->ccm128.key, key, EVP_CIPHER_get_key_length(c)); + memcpy(crypto_info->ccm128.rec_seq, rl_sequence, + TLS_CIPHER_AES_CCM_128_REC_SEQ_SIZE); + if (rec_seq != NULL) + *rec_seq = crypto_info->ccm128.rec_seq; + return 1; +# endif +# ifdef OPENSSL_KTLS_CHACHA20_POLY1305 + case NID_chacha20_poly1305: + crypto_info->chacha20poly1305.info.cipher_type = TLS_CIPHER_CHACHA20_POLY1305; + crypto_info->chacha20poly1305.info.version = s->version; + crypto_info->tls_crypto_info_len = sizeof(crypto_info->chacha20poly1305); + memcpy(crypto_info->chacha20poly1305.iv, iiv, + TLS_CIPHER_CHACHA20_POLY1305_IV_SIZE); + memcpy(crypto_info->chacha20poly1305.key, key, + EVP_CIPHER_get_key_length(c)); + memcpy(crypto_info->chacha20poly1305.rec_seq, rl_sequence, + TLS_CIPHER_CHACHA20_POLY1305_REC_SEQ_SIZE); + if (rec_seq != NULL) + *rec_seq = crypto_info->chacha20poly1305.rec_seq; + return 1; +# endif + default: + return 0; + } + +} + +#endif /* OPENSSL_SYS_LINUX */ diff --git a/openssl/src/ssl/methods.c b/openssl/src/ssl/methods.c index 525f59e91..896b08c9f 100644 --- a/openssl/src/ssl/methods.c +++ b/openssl/src/ssl/methods.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -173,6 +173,24 @@ IMPLEMENT_dtls1_meth_func(DTLS_ANY_VERSION, 0, 0, DTLS_client_method, ssl_undefined_function, ossl_statem_connect, DTLSv1_2_enc_data) + +#ifndef OPENSSL_NO_NTLS +IMPLEMENT_tls_meth_func(NTLS_VERSION, SSL_METHOD_NO_SUITEB, SSL_OP_NO_NTLS, + ntls_method, + ossl_statem_accept, + ossl_statem_connect, NTLS_enc_data) + +IMPLEMENT_tls_meth_func(NTLS_VERSION, SSL_METHOD_NO_SUITEB, SSL_OP_NO_NTLS, + ntls_server_method, + ossl_statem_accept, + ssl_undefined_function, NTLS_enc_data) + +IMPLEMENT_tls_meth_func(NTLS_VERSION, SSL_METHOD_NO_SUITEB, SSL_OP_NO_NTLS, + ntls_client_method, + ssl_undefined_function, + ossl_statem_connect, NTLS_enc_data) +#endif + #ifndef OPENSSL_NO_DEPRECATED_1_1_0 # ifndef OPENSSL_NO_TLS1_2_METHOD const SSL_METHOD *TLSv1_2_method(void) @@ -276,4 +294,21 @@ const SSL_METHOD *DTLSv1_client_method(void) } # endif +# ifndef OPENSSL_NO_NTLS +const SSL_METHOD *NTLS_method(void) +{ + return ntls_method(); +} + +const SSL_METHOD *NTLS_server_method(void) +{ + return ntls_server_method(); +} + +const SSL_METHOD *NTLS_client_method(void) +{ + return ntls_client_method(); +} +# endif + #endif diff --git a/openssl/src/ssl/pqueue.c b/openssl/src/ssl/pqueue.c index db161e25d..0852aceac 100644 --- a/openssl/src/ssl/pqueue.c +++ b/openssl/src/ssl/pqueue.c @@ -19,8 +19,10 @@ pitem *pitem_new(unsigned char *prio64be, void *data) { pitem *item = OPENSSL_malloc(sizeof(*item)); - if (item == NULL) + if (item == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return NULL; + } memcpy(item->priority, prio64be, sizeof(item->priority)); item->data = data; @@ -37,6 +39,9 @@ pqueue *pqueue_new(void) { pqueue *pq = OPENSSL_zalloc(sizeof(*pq)); + if (pq == NULL) + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + return pq; } diff --git a/openssl/src/ssl/priority_queue.c b/openssl/src/ssl/priority_queue.c deleted file mode 100644 index 5393c532a..000000000 --- a/openssl/src/ssl/priority_queue.c +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include "internal/priority_queue.h" -#include "internal/safe_math.h" -#include "internal/numbers.h" - -OSSL_SAFE_MATH_UNSIGNED(size_t, size_t) - -/* - * Fundamental operations: - * Binary Heap Fibonacci Heap - * Get smallest O(1) O(1) - * Delete any O(log n) O(log n) average but worst O(n) - * Insert O(log n) O(1) - * - * Not supported: - * Merge two structures O(log n) O(1) - * Decrease key O(log n) O(1) - * Increase key O(log n) ? - * - * The Fibonacci heap is quite a bit more complicated to implement and has - * larger overhead in practice. We favour the binary heap here. A multi-way - * (ternary or quaternary) heap might elicit a performance advantage via better - * cache access patterns. - */ - -struct pq_heap_st { - void *data; /* User supplied data pointer */ - size_t index; /* Constant index in elements[] */ -}; - -struct pq_elem_st { - size_t posn; /* Current index in heap[] or link in free list */ -#ifndef NDEBUG - int used; /* Debug flag indicating that this is in use */ -#endif -}; - -struct ossl_pqueue_st -{ - struct pq_heap_st *heap; - struct pq_elem_st *elements; - int (*compare)(const void *, const void *); - size_t htop; /* Highest used heap element */ - size_t hmax; /* Allocated heap & element space */ - size_t freelist; /* Index into elements[], start of free element list */ -}; - -/* - * The initial and maximum number of elements in the heap. - */ -static const size_t min_nodes = 8; -static const size_t max_nodes = - SIZE_MAX / (sizeof(struct pq_heap_st) > sizeof(struct pq_elem_st) - ? sizeof(struct pq_heap_st) : sizeof(struct pq_elem_st)); - -#ifndef NDEBUG -/* Some basic sanity checking of the data structure */ -# define ASSERT_USED(pq, idx) \ - assert(pq->elements[pq->heap[idx].index].used); \ - assert(pq->elements[pq->heap[idx].index].posn == idx) -# define ASSERT_ELEM_USED(pq, elem) \ - assert(pq->elements[elem].used) -#else -# define ASSERT_USED(pq, idx) -# define ASSERT_ELEM_USED(pq, elem) -#endif - -/* - * Calculate the array growth based on the target size. - * - * The growth factor is a rational number and is defined by a numerator - * and a denominator. According to Andrew Koenig in his paper "Why Are - * Vectors Efficient?" from JOOP 11(5) 1998, this factor should be less - * than the golden ratio (1.618...). - * - * We use an expansion factor of 8 / 5 = 1.6 - */ -static ossl_inline size_t compute_pqueue_growth(size_t target, size_t current) -{ - int err = 0; - - while (current < target) { - if (current >= max_nodes) - return 0; - - current = safe_muldiv_size_t(current, 8, 5, &err); - if (err) - return 0; - if (current >= max_nodes) - current = max_nodes; - } - return current; -} - -static ossl_inline void pqueue_swap_elem(OSSL_PQUEUE *pq, size_t i, size_t j) -{ - struct pq_heap_st *h = pq->heap, t_h; - struct pq_elem_st *e = pq->elements; - - ASSERT_USED(pq, i); - ASSERT_USED(pq, j); - - t_h = h[i]; - h[i] = h[j]; - h[j] = t_h; - - e[h[i].index].posn = i; - e[h[j].index].posn = j; -} - -static ossl_inline void pqueue_move_elem(OSSL_PQUEUE *pq, size_t from, size_t to) -{ - struct pq_heap_st *h = pq->heap; - struct pq_elem_st *e = pq->elements; - - ASSERT_USED(pq, from); - - h[to] = h[from]; - e[h[to].index].posn = to; -} - -/* - * Force the specified element to the front of the heap. This breaks - * the heap partial ordering pre-condition. - */ -static ossl_inline void pqueue_force_bottom(OSSL_PQUEUE *pq, size_t n) -{ - ASSERT_USED(pq, n); - while (n > 0) { - const size_t p = (n - 1) / 2; - - ASSERT_USED(pq, p); - pqueue_swap_elem(pq, n, p); - n = p; - } -} - -/* - * Move an element down to its correct position to restore the partial - * order pre-condition. - */ -static ossl_inline void pqueue_move_down(OSSL_PQUEUE *pq, size_t n) -{ - struct pq_heap_st *h = pq->heap; - - ASSERT_USED(pq, n); - while (n > 0) { - const size_t p = (n - 1) / 2; - - ASSERT_USED(pq, p); - if (pq->compare(h[n].data, h[p].data) >= 0) - break; - pqueue_swap_elem(pq, n, p); - n = p; - } -} - -/* - * Move an element up to its correct position to restore the partial - * order pre-condition. - */ -static ossl_inline void pqueue_move_up(OSSL_PQUEUE *pq, size_t n) -{ - struct pq_heap_st *h = pq->heap; - size_t p = n * 2 + 1; - - ASSERT_USED(pq, n); - if (pq->htop > p + 1) { - ASSERT_USED(pq, p); - ASSERT_USED(pq, p + 1); - if (pq->compare(h[p].data, h[p + 1].data) > 0) - p++; - } - while (pq->htop > p && pq->compare(h[p].data, h[n].data) < 0) { - ASSERT_USED(pq, p); - pqueue_swap_elem(pq, n, p); - n = p; - p = n * 2 + 1; - if (pq->htop > p + 1) { - ASSERT_USED(pq, p + 1); - if (pq->compare(h[p].data, h[p + 1].data) > 0) - p++; - } - } -} - -int ossl_pqueue_push(OSSL_PQUEUE *pq, void *data, size_t *elem) -{ - size_t n, m; - - if (!ossl_pqueue_reserve(pq, 1)) - return 0; - - n = pq->htop++; - m = pq->freelist; - pq->freelist = pq->elements[m].posn; - - pq->heap[n].data = data; - pq->heap[n].index = m; - - pq->elements[m].posn = n; -#ifndef NDEBUG - pq->elements[m].used = 1; -#endif - pqueue_move_down(pq, n); - if (elem != NULL) - *elem = m; - return 1; -} - -void *ossl_pqueue_peek(const OSSL_PQUEUE *pq) -{ - if (pq->htop > 0) { - ASSERT_USED(pq, 0); - return pq->heap->data; - } - return NULL; -} - -void *ossl_pqueue_pop(OSSL_PQUEUE *pq) -{ - void *res; - size_t elem; - - if (pq == NULL || pq->htop == 0) - return NULL; - - ASSERT_USED(pq, 0); - res = pq->heap->data; - elem = pq->heap->index; - - if (--pq->htop != 0) { - pqueue_move_elem(pq, pq->htop, 0); - pqueue_move_up(pq, 0); - } - - pq->elements[elem].posn = pq->freelist; - pq->freelist = elem; -#ifndef NDEBUG - pq->elements[elem].used = 0; -#endif - return res; -} - -void *ossl_pqueue_remove(OSSL_PQUEUE *pq, size_t elem) -{ - size_t n; - - if (pq == NULL || elem >= pq->hmax || pq->htop == 0) - return 0; - - ASSERT_ELEM_USED(pq, elem); - n = pq->elements[elem].posn; - - ASSERT_USED(pq, n); - - if (n == pq->htop - 1) { - pq->elements[elem].posn = pq->freelist; - pq->freelist = elem; -#ifndef NDEBUG - pq->elements[elem].used = 0; -#endif - return pq->heap[--pq->htop].data; - } - if (n > 0) - pqueue_force_bottom(pq, n); - return ossl_pqueue_pop(pq); -} - -static void pqueue_add_freelist(OSSL_PQUEUE *pq, size_t from) -{ - struct pq_elem_st *e = pq->elements; - size_t i; - -#ifndef NDEBUG - for (i = from; i < pq->hmax; i++) - e[i].used = 0; -#endif - e[from].posn = pq->freelist; - for (i = from + 1; i < pq->hmax; i++) - e[i].posn = i - 1; - pq->freelist = pq->hmax - 1; -} - -int ossl_pqueue_reserve(OSSL_PQUEUE *pq, size_t n) -{ - size_t new_max, cur_max; - struct pq_heap_st *h; - struct pq_elem_st *e; - - if (pq == NULL) - return 0; - cur_max = pq->hmax; - if (pq->htop + n < cur_max) - return 1; - - new_max = compute_pqueue_growth(n + cur_max, cur_max); - if (new_max == 0) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - h = OPENSSL_realloc(pq->heap, new_max * sizeof(*pq->heap)); - if (h == NULL) - return 0; - pq->heap = h; - - e = OPENSSL_realloc(pq->elements, new_max * sizeof(*pq->elements)); - if (e == NULL) - return 0; - pq->elements = e; - - pq->hmax = new_max; - pqueue_add_freelist(pq, cur_max); - return 1; -} - -OSSL_PQUEUE *ossl_pqueue_new(int (*compare)(const void *, const void *)) -{ - OSSL_PQUEUE *pq; - - if (compare == NULL) - return NULL; - - pq = OPENSSL_malloc(sizeof(*pq)); - if (pq == NULL) - return NULL; - pq->compare = compare; - pq->hmax = min_nodes; - pq->htop = 0; - pq->freelist = 0; - pq->heap = OPENSSL_malloc(sizeof(*pq->heap) * min_nodes); - pq->elements = OPENSSL_malloc(sizeof(*pq->elements) * min_nodes); - if (pq->heap == NULL || pq->elements == NULL) { - ossl_pqueue_free(pq); - return NULL; - } - pqueue_add_freelist(pq, 0); - return pq; -} - -void ossl_pqueue_free(OSSL_PQUEUE *pq) -{ - if (pq != NULL) { - OPENSSL_free(pq->heap); - OPENSSL_free(pq->elements); - OPENSSL_free(pq); - } -} - -void ossl_pqueue_pop_free(OSSL_PQUEUE *pq, void (*freefunc)(void *)) -{ - size_t i; - - if (pq != NULL) { - for (i = 0; i < pq->htop; i++) - (*freefunc)(pq->heap[i].data); - ossl_pqueue_free(pq); - } -} - -size_t ossl_pqueue_num(const OSSL_PQUEUE *pq) -{ - return pq != NULL ? pq->htop : 0; -} diff --git a/openssl/src/ssl/quic/cc_newreno.c b/openssl/src/ssl/quic/cc_newreno.c deleted file mode 100644 index 1fe37c276..000000000 --- a/openssl/src/ssl/quic/cc_newreno.c +++ /dev/null @@ -1,485 +0,0 @@ -#include "internal/quic_cc.h" -#include "internal/quic_types.h" -#include "internal/safe_math.h" - -OSSL_SAFE_MATH_UNSIGNED(u64, uint64_t) - -typedef struct ossl_cc_newreno_st { - /* Dependencies. */ - OSSL_TIME (*now_cb)(void *arg); - void *now_cb_arg; - - /* 'Constants' (which we allow to be configurable). */ - uint64_t k_init_wnd, k_min_wnd; - uint32_t k_loss_reduction_factor_num, k_loss_reduction_factor_den; - uint32_t persistent_cong_thresh; - - /* State. */ - size_t max_dgram_size; - uint64_t bytes_in_flight, cong_wnd, slow_start_thresh, bytes_acked; - OSSL_TIME cong_recovery_start_time; - - /* Unflushed state during multiple on-loss calls. */ - int processing_loss; /* 1 if not flushed */ - OSSL_TIME tx_time_of_last_loss; - - /* Diagnostic state. */ - int in_congestion_recovery; - - /* Diagnostic output locations. */ - size_t *p_diag_max_dgram_payload_len; - uint64_t *p_diag_cur_cwnd_size; - uint64_t *p_diag_min_cwnd_size; - uint64_t *p_diag_cur_bytes_in_flight; - uint32_t *p_diag_cur_state; -} OSSL_CC_NEWRENO; - -#define MIN_MAX_INIT_WND_SIZE 14720 /* RFC 9002 s. 7.2 */ - -/* TODO(QUIC FUTURE): Pacing support. */ - -static void newreno_set_max_dgram_size(OSSL_CC_NEWRENO *nr, - size_t max_dgram_size); -static void newreno_update_diag(OSSL_CC_NEWRENO *nr); - -static void newreno_reset(OSSL_CC_DATA *cc); - -static OSSL_CC_DATA *newreno_new(OSSL_TIME (*now_cb)(void *arg), - void *now_cb_arg) -{ - OSSL_CC_NEWRENO *nr; - - if ((nr = OPENSSL_zalloc(sizeof(*nr))) == NULL) - return NULL; - - nr->now_cb = now_cb; - nr->now_cb_arg = now_cb_arg; - - newreno_set_max_dgram_size(nr, QUIC_MIN_INITIAL_DGRAM_LEN); - newreno_reset((OSSL_CC_DATA *)nr); - - return (OSSL_CC_DATA *)nr; -} - -static void newreno_free(OSSL_CC_DATA *cc) -{ - OPENSSL_free(cc); -} - -static void newreno_set_max_dgram_size(OSSL_CC_NEWRENO *nr, - size_t max_dgram_size) -{ - size_t max_init_wnd; - int is_reduced = (max_dgram_size < nr->max_dgram_size); - - nr->max_dgram_size = max_dgram_size; - - max_init_wnd = 2 * max_dgram_size; - if (max_init_wnd < MIN_MAX_INIT_WND_SIZE) - max_init_wnd = MIN_MAX_INIT_WND_SIZE; - - nr->k_init_wnd = 10 * max_dgram_size; - if (nr->k_init_wnd > max_init_wnd) - nr->k_init_wnd = max_init_wnd; - - nr->k_min_wnd = 2 * max_dgram_size; - - if (is_reduced) - nr->cong_wnd = nr->k_init_wnd; - - newreno_update_diag(nr); -} - -static void newreno_reset(OSSL_CC_DATA *cc) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - nr->k_loss_reduction_factor_num = 1; - nr->k_loss_reduction_factor_den = 2; - nr->persistent_cong_thresh = 3; - - nr->cong_wnd = nr->k_init_wnd; - nr->bytes_in_flight = 0; - nr->bytes_acked = 0; - nr->slow_start_thresh = UINT64_MAX; - nr->cong_recovery_start_time = ossl_time_zero(); - - nr->processing_loss = 0; - nr->tx_time_of_last_loss = ossl_time_zero(); - nr->in_congestion_recovery = 0; -} - -static int newreno_set_input_params(OSSL_CC_DATA *cc, const OSSL_PARAM *params) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - const OSSL_PARAM *p; - size_t value; - - p = OSSL_PARAM_locate_const(params, OSSL_CC_OPTION_MAX_DGRAM_PAYLOAD_LEN); - if (p != NULL) { - if (!OSSL_PARAM_get_size_t(p, &value)) - return 0; - if (value < QUIC_MIN_INITIAL_DGRAM_LEN) - return 0; - - newreno_set_max_dgram_size(nr, value); - } - - return 1; -} - -static int bind_diag(OSSL_PARAM *params, const char *param_name, size_t len, - void **pp) -{ - const OSSL_PARAM *p = OSSL_PARAM_locate_const(params, param_name); - - *pp = NULL; - - if (p == NULL) - return 1; - - if (p->data_type != OSSL_PARAM_UNSIGNED_INTEGER - || p->data_size != len) - return 0; - - *pp = p->data; - return 1; -} - -static int newreno_bind_diagnostic(OSSL_CC_DATA *cc, OSSL_PARAM *params) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - size_t *new_p_max_dgram_payload_len; - uint64_t *new_p_cur_cwnd_size; - uint64_t *new_p_min_cwnd_size; - uint64_t *new_p_cur_bytes_in_flight; - uint32_t *new_p_cur_state; - - if (!bind_diag(params, OSSL_CC_OPTION_MAX_DGRAM_PAYLOAD_LEN, - sizeof(size_t), (void **)&new_p_max_dgram_payload_len) - || !bind_diag(params, OSSL_CC_OPTION_CUR_CWND_SIZE, - sizeof(uint64_t), (void **)&new_p_cur_cwnd_size) - || !bind_diag(params, OSSL_CC_OPTION_MIN_CWND_SIZE, - sizeof(uint64_t), (void **)&new_p_min_cwnd_size) - || !bind_diag(params, OSSL_CC_OPTION_CUR_BYTES_IN_FLIGHT, - sizeof(uint64_t), (void **)&new_p_cur_bytes_in_flight) - || !bind_diag(params, OSSL_CC_OPTION_CUR_STATE, - sizeof(uint32_t), (void **)&new_p_cur_state)) - return 0; - - if (new_p_max_dgram_payload_len != NULL) - nr->p_diag_max_dgram_payload_len = new_p_max_dgram_payload_len; - - if (new_p_cur_cwnd_size != NULL) - nr->p_diag_cur_cwnd_size = new_p_cur_cwnd_size; - - if (new_p_min_cwnd_size != NULL) - nr->p_diag_min_cwnd_size = new_p_min_cwnd_size; - - if (new_p_cur_bytes_in_flight != NULL) - nr->p_diag_cur_bytes_in_flight = new_p_cur_bytes_in_flight; - - if (new_p_cur_state != NULL) - nr->p_diag_cur_state = new_p_cur_state; - - newreno_update_diag(nr); - return 1; -} - -static void unbind_diag(OSSL_PARAM *params, const char *param_name, - void **pp) -{ - const OSSL_PARAM *p = OSSL_PARAM_locate_const(params, param_name); - - if (p != NULL) - *pp = NULL; -} - -static int newreno_unbind_diagnostic(OSSL_CC_DATA *cc, OSSL_PARAM *params) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - unbind_diag(params, OSSL_CC_OPTION_MAX_DGRAM_PAYLOAD_LEN, - (void **)&nr->p_diag_max_dgram_payload_len); - unbind_diag(params, OSSL_CC_OPTION_CUR_CWND_SIZE, - (void **)&nr->p_diag_cur_cwnd_size); - unbind_diag(params, OSSL_CC_OPTION_MIN_CWND_SIZE, - (void **)&nr->p_diag_min_cwnd_size); - unbind_diag(params, OSSL_CC_OPTION_CUR_BYTES_IN_FLIGHT, - (void **)&nr->p_diag_cur_bytes_in_flight); - unbind_diag(params, OSSL_CC_OPTION_CUR_STATE, - (void **)&nr->p_diag_cur_state); - return 1; -} - -static void newreno_update_diag(OSSL_CC_NEWRENO *nr) -{ - if (nr->p_diag_max_dgram_payload_len != NULL) - *nr->p_diag_max_dgram_payload_len = nr->max_dgram_size; - - if (nr->p_diag_cur_cwnd_size != NULL) - *nr->p_diag_cur_cwnd_size = nr->cong_wnd; - - if (nr->p_diag_min_cwnd_size != NULL) - *nr->p_diag_min_cwnd_size = nr->k_min_wnd; - - if (nr->p_diag_cur_bytes_in_flight != NULL) - *nr->p_diag_cur_bytes_in_flight = nr->bytes_in_flight; - - if (nr->p_diag_cur_state != NULL) { - if (nr->in_congestion_recovery) - *nr->p_diag_cur_state = 'R'; - else if (nr->cong_wnd < nr->slow_start_thresh) - *nr->p_diag_cur_state = 'S'; - else - *nr->p_diag_cur_state = 'A'; - } -} - -static int newreno_in_cong_recovery(OSSL_CC_NEWRENO *nr, OSSL_TIME tx_time) -{ - return ossl_time_compare(tx_time, nr->cong_recovery_start_time) <= 0; -} - -static void newreno_cong(OSSL_CC_NEWRENO *nr, OSSL_TIME tx_time) -{ - int err = 0; - - /* No reaction if already in a recovery period. */ - if (newreno_in_cong_recovery(nr, tx_time)) - return; - - /* Start a new recovery period. */ - nr->in_congestion_recovery = 1; - nr->cong_recovery_start_time = nr->now_cb(nr->now_cb_arg); - - /* slow_start_thresh = cong_wnd * loss_reduction_factor */ - nr->slow_start_thresh - = safe_muldiv_u64(nr->cong_wnd, - nr->k_loss_reduction_factor_num, - nr->k_loss_reduction_factor_den, - &err); - - if (err) - nr->slow_start_thresh = UINT64_MAX; - - nr->cong_wnd = nr->slow_start_thresh; - if (nr->cong_wnd < nr->k_min_wnd) - nr->cong_wnd = nr->k_min_wnd; -} - -static void newreno_flush(OSSL_CC_NEWRENO *nr, uint32_t flags) -{ - if (!nr->processing_loss) - return; - - newreno_cong(nr, nr->tx_time_of_last_loss); - - if ((flags & OSSL_CC_LOST_FLAG_PERSISTENT_CONGESTION) != 0) { - nr->cong_wnd = nr->k_min_wnd; - nr->cong_recovery_start_time = ossl_time_zero(); - } - - nr->processing_loss = 0; - newreno_update_diag(nr); -} - -static uint64_t newreno_get_tx_allowance(OSSL_CC_DATA *cc) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - if (nr->bytes_in_flight >= nr->cong_wnd) - return 0; - - return nr->cong_wnd - nr->bytes_in_flight; -} - -static OSSL_TIME newreno_get_wakeup_deadline(OSSL_CC_DATA *cc) -{ - if (newreno_get_tx_allowance(cc) > 0) { - /* We have TX allowance now so wakeup immediately */ - return ossl_time_zero(); - } else { - /* - * The NewReno congestion controller does not vary its state in time, - * only in response to stimulus. - */ - return ossl_time_infinite(); - } -} - -static int newreno_on_data_sent(OSSL_CC_DATA *cc, uint64_t num_bytes) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - nr->bytes_in_flight += num_bytes; - newreno_update_diag(nr); - return 1; -} - -static int newreno_is_cong_limited(OSSL_CC_NEWRENO *nr) -{ - uint64_t wnd_rem; - - /* We are congestion-limited if we are already at the congestion window. */ - if (nr->bytes_in_flight >= nr->cong_wnd) - return 1; - - wnd_rem = nr->cong_wnd - nr->bytes_in_flight; - - /* - * Consider ourselves congestion-limited if less than three datagrams' worth - * of congestion window remains to be spent, or if we are in slow start and - * have consumed half of our window. - */ - return (nr->cong_wnd < nr->slow_start_thresh && wnd_rem <= nr->cong_wnd / 2) - || wnd_rem <= 3 * nr->max_dgram_size; -} - -static int newreno_on_data_acked(OSSL_CC_DATA *cc, - const OSSL_CC_ACK_INFO *info) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - /* - * Packet has been acked. Firstly, remove it from the aggregate count of - * bytes in flight. - */ - nr->bytes_in_flight -= info->tx_size; - - /* - * We use acknowledgement of data as a signal that we are not at channel - * capacity and that it may be reasonable to increase the congestion window. - * However, acknowledgement is not a useful signal that there is further - * capacity if we are not actually saturating the congestion window that we - * already have (for example, if the application is not generating much data - * or we are limited by flow control). Therefore, we only expand the - * congestion window if we are consuming a significant fraction of the - * congestion window. - */ - if (!newreno_is_cong_limited(nr)) - goto out; - - /* - * We can handle acknowledgement of a packet in one of three ways - * depending on our current state: - * - * - Congestion Recovery: Do nothing. We don't start increasing - * the congestion window in response to acknowledgements until - * we are no longer in the Congestion Recovery state. - * - * - Slow Start: Increase the congestion window using the slow - * start scale. - * - * - Congestion Avoidance: Increase the congestion window using - * the congestion avoidance scale. - */ - if (newreno_in_cong_recovery(nr, info->tx_time)) { - /* Congestion recovery, do nothing. */ - } else if (nr->cong_wnd < nr->slow_start_thresh) { - /* When this condition is true we are in the Slow Start state. */ - nr->cong_wnd += info->tx_size; - nr->in_congestion_recovery = 0; - } else { - /* Otherwise, we are in the Congestion Avoidance state. */ - nr->bytes_acked += info->tx_size; - - /* - * Avoid integer division as per RFC 9002 s. B.5. / RFC3465 s. 2.1. - */ - if (nr->bytes_acked >= nr->cong_wnd) { - nr->bytes_acked -= nr->cong_wnd; - nr->cong_wnd += nr->max_dgram_size; - } - - nr->in_congestion_recovery = 0; - } - -out: - newreno_update_diag(nr); - return 1; -} - -static int newreno_on_data_lost(OSSL_CC_DATA *cc, - const OSSL_CC_LOSS_INFO *info) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - if (info->tx_size > nr->bytes_in_flight) - return 0; - - nr->bytes_in_flight -= info->tx_size; - - if (!nr->processing_loss) { - - if (ossl_time_compare(info->tx_time, nr->tx_time_of_last_loss) <= 0) - /* - * After triggering congestion due to a lost packet at time t, don't - * trigger congestion again due to any subsequently detected lost - * packet at a time s < t, as we've effectively already signalled - * congestion on loss of that and subsequent packets. - */ - goto out; - - nr->processing_loss = 1; - - /* - * Cancel any pending window increase in the Congestion Avoidance state. - */ - nr->bytes_acked = 0; - } - - nr->tx_time_of_last_loss - = ossl_time_max(nr->tx_time_of_last_loss, info->tx_time); - -out: - newreno_update_diag(nr); - return 1; -} - -static int newreno_on_data_lost_finished(OSSL_CC_DATA *cc, uint32_t flags) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - newreno_flush(nr, flags); - return 1; -} - -static int newreno_on_data_invalidated(OSSL_CC_DATA *cc, - uint64_t num_bytes) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - nr->bytes_in_flight -= num_bytes; - newreno_update_diag(nr); - return 1; -} - -static int newreno_on_ecn(OSSL_CC_DATA *cc, - const OSSL_CC_ECN_INFO *info) -{ - OSSL_CC_NEWRENO *nr = (OSSL_CC_NEWRENO *)cc; - - nr->processing_loss = 1; - nr->bytes_acked = 0; - nr->tx_time_of_last_loss = info->largest_acked_time; - newreno_flush(nr, 0); - return 1; -} - -const OSSL_CC_METHOD ossl_cc_newreno_method = { - newreno_new, - newreno_free, - newreno_reset, - newreno_set_input_params, - newreno_bind_diagnostic, - newreno_unbind_diagnostic, - newreno_get_tx_allowance, - newreno_get_wakeup_deadline, - newreno_on_data_sent, - newreno_on_data_acked, - newreno_on_data_lost, - newreno_on_data_lost_finished, - newreno_on_data_invalidated, - newreno_on_ecn, -}; diff --git a/openssl/src/ssl/quic/json_enc.c b/openssl/src/ssl/quic/json_enc.c deleted file mode 100644 index 3e9f715df..000000000 --- a/openssl/src/ssl/quic/json_enc.c +++ /dev/null @@ -1,766 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/json_enc.h" -#include "internal/nelem.h" -#include "internal/numbers.h" -#include -#include - -/* - * wbuf - * ==== - */ -static int wbuf_flush(struct json_write_buf *wbuf, int full); - -static int wbuf_init(struct json_write_buf *wbuf, BIO *bio, size_t alloc) -{ - wbuf->buf = OPENSSL_malloc(alloc); - if (wbuf->buf == NULL) - return 0; - - wbuf->cur = 0; - wbuf->alloc = alloc; - wbuf->bio = bio; - return 1; -} - -static void wbuf_cleanup(struct json_write_buf *wbuf) -{ - OPENSSL_free(wbuf->buf); - wbuf->buf = NULL; - wbuf->alloc = 0; -} - -static void wbuf_set0_bio(struct json_write_buf *wbuf, BIO *bio) -{ - wbuf->bio = bio; -} - -/* Empty write buffer. */ -static ossl_inline void wbuf_clean(struct json_write_buf *wbuf) -{ - wbuf->cur = 0; -} - -/* Available data remaining in buffer. */ -static ossl_inline size_t wbuf_avail(struct json_write_buf *wbuf) -{ - return wbuf->alloc - wbuf->cur; -} - -/* Add character to write buffer, returning 0 on flush failure. */ -static ossl_inline int wbuf_write_char(struct json_write_buf *wbuf, char c) -{ - if (wbuf_avail(wbuf) == 0) { - if (!wbuf_flush(wbuf, /*full=*/0)) - return 0; - } - - wbuf->buf[wbuf->cur++] = c; - return 1; -} - -/* - * Write zero-terminated string to write buffer, returning 0 on flush failure. - */ -static int wbuf_write_str(struct json_write_buf *wbuf, const char *s) -{ - char c; - - while ((c = *s++) != 0) - if (!wbuf_write_char(wbuf, c)) - return 0; - - return 1; -} - -/* Flush write buffer, returning 0 on I/O failure. */ -static int wbuf_flush(struct json_write_buf *wbuf, int full) -{ - size_t written = 0, total_written = 0; - - while (total_written < wbuf->cur) { - if (!BIO_write_ex(wbuf->bio, - wbuf->buf + total_written, - wbuf->cur - total_written, - &written)) { - memmove(wbuf->buf, - wbuf->buf + total_written, - wbuf->cur - total_written); - wbuf->cur = 0; - return 0; - } - - total_written += written; - } - - wbuf->cur = 0; - - if (full) - (void)BIO_flush(wbuf->bio); /* best effort */ - - return 1; -} - -/* - * OSSL_JSON_ENC: Stack Management - * =============================== - */ - -static int json_ensure_stack_size(OSSL_JSON_ENC *json, size_t num_bytes) -{ - unsigned char *stack; - - if (json->stack_bytes >= num_bytes) - return 1; - - if (num_bytes <= OSSL_NELEM(json->stack_small)) { - stack = json->stack_small; - } else { - if (json->stack == json->stack_small) - json->stack = NULL; - - stack = OPENSSL_realloc(json->stack, num_bytes); - if (stack == NULL) - return 0; - } - - json->stack = stack; - json->stack_bytes = num_bytes; - return 1; -} - -/* Push one bit onto the stack. Returns 0 on allocation failure. */ -static int json_push(OSSL_JSON_ENC *json, unsigned int v) -{ - if (v > 1) - return 0; - - if (json->stack_end_byte >= json->stack_bytes) { - size_t new_size - = (json->stack_bytes == 0) - ? OSSL_NELEM(json->stack_small) - : (json->stack_bytes * 2); - - if (!json_ensure_stack_size(json, new_size)) - return 0; - - json->stack_bytes = new_size; - } - - if (v > 0) - json->stack[json->stack_end_byte] |= (v << json->stack_end_bit); - else - json->stack[json->stack_end_byte] &= ~(1U << json->stack_end_bit); - - json->stack_end_bit = (json->stack_end_bit + 1) % CHAR_BIT; - if (json->stack_end_bit == 0) - ++json->stack_end_byte; - - return 1; -} - -/* - * Pop a bit from the stack. Returns 0 if stack is empty. Use json_peek() to get - * the value before calling this. - */ -static int json_pop(OSSL_JSON_ENC *json) -{ - if (json->stack_end_byte == 0 && json->stack_end_bit == 0) - return 0; - - if (json->stack_end_bit == 0) { - --json->stack_end_byte; - json->stack_end_bit = CHAR_BIT - 1; - } else { - --json->stack_end_bit; - } - - return 1; -} - -/* - * Returns the bit on the top of the stack, or -1 if the stack is empty. - */ -static int json_peek(OSSL_JSON_ENC *json) -{ - size_t obyte, obit; - - obyte = json->stack_end_byte; - obit = json->stack_end_bit; - if (obit == 0) { - if (obyte == 0) - return -1; - - --obyte; - obit = CHAR_BIT - 1; - } else { - --obit; - } - - return (json->stack[obyte] & (1U << obit)) != 0; -} - -/* - * OSSL_JSON_ENC: Initialisation - * ============================= - */ - -enum { - STATE_PRE_KEY, - STATE_PRE_ITEM, - STATE_PRE_COMMA -}; - -static ossl_inline int in_ijson(const OSSL_JSON_ENC *json) -{ - return (json->flags & OSSL_JSON_FLAG_IJSON) != 0; -} - -static ossl_inline int in_seq(const OSSL_JSON_ENC *json) -{ - return (json->flags & OSSL_JSON_FLAG_SEQ) != 0; -} - -static ossl_inline int in_pretty(const OSSL_JSON_ENC *json) -{ - return (json->flags & OSSL_JSON_FLAG_PRETTY) != 0; -} - -int ossl_json_init(OSSL_JSON_ENC *json, BIO *bio, uint32_t flags) -{ - memset(json, 0, sizeof(*json)); - json->flags = flags; - json->error = 0; - if (!wbuf_init(&json->wbuf, bio, 4096)) - return 0; - - json->state = STATE_PRE_COMMA; - return 1; -} - -void ossl_json_cleanup(OSSL_JSON_ENC *json) -{ - wbuf_cleanup(&json->wbuf); - - if (json->stack != json->stack_small) - OPENSSL_free(json->stack); - - json->stack = NULL; -} - -int ossl_json_flush_cleanup(OSSL_JSON_ENC *json) -{ - int ok = ossl_json_flush(json); - - ossl_json_cleanup(json); - return ok; -} - -int ossl_json_reset(OSSL_JSON_ENC *json) -{ - wbuf_clean(&json->wbuf); - json->stack_end_byte = 0; - json->stack_end_bit = 0; - json->error = 0; - return 1; -} - -int ossl_json_flush(OSSL_JSON_ENC *json) -{ - return wbuf_flush(&json->wbuf, /*full=*/1); -} - -int ossl_json_set0_sink(OSSL_JSON_ENC *json, BIO *bio) -{ - wbuf_set0_bio(&json->wbuf, bio); - return 1; -} - -int ossl_json_in_error(OSSL_JSON_ENC *json) -{ - return json->error; -} - -/* - * JSON Builder Calls - * ================== - */ - -static void json_write_qstring(OSSL_JSON_ENC *json, const char *str); -static void json_indent(OSSL_JSON_ENC *json); - -static void json_raise_error(OSSL_JSON_ENC *json) -{ - json->error = 1; -} - -static void json_undefer(OSSL_JSON_ENC *json) -{ - if (!json->defer_indent) - return; - - json_indent(json); -} - -static void json_write_char(OSSL_JSON_ENC *json, char ch) -{ - if (ossl_json_in_error(json)) - return; - - json_undefer(json); - if (!wbuf_write_char(&json->wbuf, ch)) - json_raise_error(json); -} - -static void json_write_str(OSSL_JSON_ENC *json, const char *s) -{ - if (ossl_json_in_error(json)) - return; - - json_undefer(json); - if (!wbuf_write_str(&json->wbuf, s)) - json_raise_error(json); -} - -static void json_indent(OSSL_JSON_ENC *json) -{ - size_t i, depth; - - json->defer_indent = 0; - - if (!in_pretty(json)) - return; - - json_write_char(json, '\n'); - - depth = json->stack_end_byte * 8 + json->stack_end_bit; - for (i = 0; i < depth * 4; ++i) - json_write_str(json, " "); -} - -static int json_pre_item(OSSL_JSON_ENC *json) -{ - int s; - - if (ossl_json_in_error(json)) - return 0; - - switch (json->state) { - case STATE_PRE_COMMA: - s = json_peek(json); - - if (s == 0) { - json_raise_error(json); - return 0; - } - - if (s == 1) { - json_write_char(json, ','); - if (ossl_json_in_error(json)) - return 0; - - json_indent(json); - } - - if (s < 0 && in_seq(json)) - json_write_char(json, '\x1E'); - - json->state = STATE_PRE_ITEM; - break; - - case STATE_PRE_ITEM: - break; - - case STATE_PRE_KEY: - default: - json_raise_error(json); - return 0; - } - - return 1; -} - -static void json_post_item(OSSL_JSON_ENC *json) -{ - int s = json_peek(json); - - json->state = STATE_PRE_COMMA; - - if (s < 0 && in_seq(json)) - json_write_char(json, '\n'); -} - -/* - * Begin a composite structure (object or array). - * - * type: 0=object, 1=array. - */ -static void composite_begin(OSSL_JSON_ENC *json, int type, char ch) -{ - if (!json_pre_item(json) - || !json_push(json, type)) - json_raise_error(json); - - json_write_char(json, ch); - json->defer_indent = 1; -} - -/* - * End a composite structure (object or array). - * - * type: 0=object, 1=array. Errors on mismatch. - */ -static void composite_end(OSSL_JSON_ENC *json, int type, char ch) -{ - int was_defer = json->defer_indent; - - if (ossl_json_in_error(json)) - return; - - json->defer_indent = 0; - - if (json_peek(json) != type) { - json_raise_error(json); - return; - } - - if (type == 0 && json->state == STATE_PRE_ITEM) { - json_raise_error(json); - return; - } - - if (!json_pop(json)) { - json_raise_error(json); - return; - } - - if (!was_defer) - json_indent(json); - - json_write_char(json, ch); - json_post_item(json); -} - -/* Begin a new JSON object. */ -void ossl_json_object_begin(OSSL_JSON_ENC *json) -{ - composite_begin(json, 0, '{'); - json->state = STATE_PRE_KEY; -} - -/* End a JSON object. Must be matched with a call to ossl_json_object_begin(). */ -void ossl_json_object_end(OSSL_JSON_ENC *json) -{ - composite_end(json, 0, '}'); -} - -/* Begin a new JSON array. */ -void ossl_json_array_begin(OSSL_JSON_ENC *json) -{ - composite_begin(json, 1, '['); - json->state = STATE_PRE_ITEM; -} - -/* End a JSON array. Must be matched with a call to ossl_json_array_begin(). */ -void ossl_json_array_end(OSSL_JSON_ENC *json) -{ - composite_end(json, 1, ']'); -} - -/* - * Encode a JSON key within an object. Pass a zero-terminated string, which can - * be freed immediately following the call to this function. - */ -void ossl_json_key(OSSL_JSON_ENC *json, const char *key) -{ - if (ossl_json_in_error(json)) - return; - - if (json_peek(json) != 0) { - /* Not in object */ - json_raise_error(json); - return; - } - - if (json->state == STATE_PRE_COMMA) { - json_write_char(json, ','); - json->state = STATE_PRE_KEY; - } - - json_indent(json); - if (json->state != STATE_PRE_KEY) { - json_raise_error(json); - return; - } - - json_write_qstring(json, key); - if (ossl_json_in_error(json)) - return; - - json_write_char(json, ':'); - if (in_pretty(json)) - json_write_char(json, ' '); - - json->state = STATE_PRE_ITEM; -} - -/* Encode a JSON 'null' value. */ -void ossl_json_null(OSSL_JSON_ENC *json) -{ - if (!json_pre_item(json)) - return; - - json_write_str(json, "null"); - json_post_item(json); -} - -void ossl_json_bool(OSSL_JSON_ENC *json, int v) -{ - if (!json_pre_item(json)) - return; - - json_write_str(json, v > 0 ? "true" : "false"); - json_post_item(json); -} - -#define POW_53 (((int64_t)1) << 53) - -/* Encode a JSON integer from a uint64_t. */ -static void json_u64(OSSL_JSON_ENC *json, uint64_t v, int noquote) -{ - char buf[22], *p = buf + sizeof(buf) - 1; - int quote = !noquote && in_ijson(json) && v > (uint64_t)(POW_53 - 1); - - if (!json_pre_item(json)) - return; - - if (quote) - json_write_char(json, '"'); - - if (v == 0) - p = "0"; - else - for (*p = '\0'; v > 0; v /= 10) - *--p = '0' + v % 10; - - json_write_str(json, p); - - if (quote) - json_write_char(json, '"'); - - json_post_item(json); -} - -void ossl_json_u64(OSSL_JSON_ENC *json, uint64_t v) -{ - json_u64(json, v, 0); -} - -/* Encode a JSON integer from an int64_t. */ -void ossl_json_i64(OSSL_JSON_ENC *json, int64_t value) -{ - uint64_t uv; - int quote; - - if (value >= 0) { - ossl_json_u64(json, (uint64_t)value); - return; - } - - if (!json_pre_item(json)) - return; - - quote = in_ijson(json) - && (value > POW_53 - 1 || value < -POW_53 + 1); - - if (quote) - json_write_char(json, '"'); - - json_write_char(json, '-'); - - uv = (value == INT64_MIN) - ? ((uint64_t)-(INT64_MIN + 1)) + 1 - : (uint64_t)-value; - json_u64(json, uv, /*noquote=*/1); - - if (quote && !ossl_json_in_error(json)) - json_write_char(json, '"'); -} - -/* Encode a JSON number from a 64-bit floating point value. */ -void ossl_json_f64(OSSL_JSON_ENC *json, double value) -{ - char buf[32]; - - if (!json_pre_item(json)) - return; - -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - { - int checks = isnan(value); -# if !defined(OPENSSL_SYS_VMS) - checks |= isinf(value); -# endif - - if (checks) { - json_raise_error(json); - return; - } - } -#endif - - BIO_snprintf(buf, sizeof(buf), "%1.17g", value); - json_write_str(json, buf); - json_post_item(json); -} - -/* - * Encode a JSON UTF-8 string from a zero-terminated string. The string passed - * can be freed immediately following the call to this function. - */ -static ossl_inline int hex_digit(int v) -{ - return v >= 10 ? 'a' + (v - 10) : '0' + v; -} - -static ossl_inline void -json_write_qstring_inner(OSSL_JSON_ENC *json, const char *str, size_t str_len, - int nul_term) -{ - char c, *o, obuf[7]; - unsigned char *u_str; - int i; - size_t j; - - if (ossl_json_in_error(json)) - return; - - json_write_char(json, '"'); - - for (j = nul_term ? strlen(str) : str_len; j > 0; str++, j--) { - c = *str; - u_str = (unsigned char*)str; - switch (c) { - case '\n': o = "\\n"; break; - case '\r': o = "\\r"; break; - case '\t': o = "\\t"; break; - case '\b': o = "\\b"; break; - case '\f': o = "\\f"; break; - case '"': o = "\\\""; break; - case '\\': o = "\\\\"; break; - default: - /* valid UTF-8 sequences according to RFC-3629 */ - if (u_str[0] >= 0xc2 && u_str[0] <= 0xdf && j >= 2 - && u_str[1] >= 0x80 && u_str[1] <= 0xbf) { - memcpy(obuf, str, 2); - obuf[2] = '\0'; - str++, j--; - o = obuf; - break; - } - if (u_str[0] >= 0xe0 && u_str[0] <= 0xef && j >= 3 - && u_str[1] >= 0x80 && u_str[1] <= 0xbf - && u_str[2] >= 0x80 && u_str[2] <= 0xbf - && !(u_str[0] == 0xe0 && u_str[1] <= 0x9f) - && !(u_str[0] == 0xed && u_str[1] >= 0xa0)) { - memcpy(obuf, str, 3); - obuf[3] = '\0'; - str += 2; - j -= 2; - o = obuf; - break; - } - if (u_str[0] >= 0xf0 && u_str[0] <= 0xf4 && j >= 4 - && u_str[1] >= 0x80 && u_str[1] <= 0xbf - && u_str[2] >= 0x80 && u_str[2] <= 0xbf - && u_str[3] >= 0x80 && u_str[3] <= 0xbf - && !(u_str[0] == 0xf0 && u_str[1] <= 0x8f) - && !(u_str[0] == 0xf4 && u_str[1] >= 0x90)) { - memcpy(obuf, str, 4); - obuf[4] = '\0'; - str += 3; - j -= 3; - o = obuf; - break; - } - if (u_str[0] < 0x20 || u_str[0] >= 0x7f) { - obuf[0] = '\\'; - obuf[1] = 'u'; - for (i = 0; i < 4; ++i) - obuf[2 + i] = hex_digit((u_str[0] >> ((3 - i) * 4)) & 0x0F); - obuf[6] = '\0'; - o = obuf; - } else { - json_write_char(json, c); - continue; - } - break; - } - - json_write_str(json, o); - } - - json_write_char(json, '"'); -} - -static void -json_write_qstring(OSSL_JSON_ENC *json, const char *str) -{ - json_write_qstring_inner(json, str, 0, 1); -} - -static void -json_write_qstring_len(OSSL_JSON_ENC *json, const char *str, size_t str_len) -{ - json_write_qstring_inner(json, str, str_len, 0); -} - -void ossl_json_str(OSSL_JSON_ENC *json, const char *str) -{ - if (!json_pre_item(json)) - return; - - json_write_qstring(json, str); - json_post_item(json); -} - -void ossl_json_str_len(OSSL_JSON_ENC *json, const char *str, size_t str_len) -{ - if (!json_pre_item(json)) - return; - - json_write_qstring_len(json, str, str_len); - json_post_item(json); -} - -/* - * Encode binary data as a lowercase hex string. data_len is the data length in - * bytes. - */ -void ossl_json_str_hex(OSSL_JSON_ENC *json, const void *data, size_t data_len) -{ - const unsigned char *b = data, *end = b + data_len; - unsigned char c; - - if (!json_pre_item(json)) - return; - - json_write_char(json, '"'); - - for (; b < end; ++b) { - c = *b; - json_write_char(json, hex_digit(c >> 4)); - json_write_char(json, hex_digit(c & 0x0F)); - } - - json_write_char(json, '"'); - json_post_item(json); -} diff --git a/openssl/src/ssl/quic/qlog.c b/openssl/src/ssl/quic/qlog.c deleted file mode 100644 index 3aadda046..000000000 --- a/openssl/src/ssl/quic/qlog.c +++ /dev/null @@ -1,728 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/qlog.h" -#include "internal/json_enc.h" -#include "internal/common.h" -#include "internal/cryptlib.h" -#include "crypto/ctype.h" - -#define BITS_PER_WORD (sizeof(size_t) * 8) -#define NUM_ENABLED_W ((QLOG_EVENT_TYPE_NUM + BITS_PER_WORD - 1) / BITS_PER_WORD) - -static ossl_unused ossl_inline int bit_get(const size_t *p, uint32_t bit_no) -{ - return p[bit_no / BITS_PER_WORD] & (((size_t)1) << (bit_no % BITS_PER_WORD)); -} - -static ossl_unused ossl_inline void bit_set(size_t *p, uint32_t bit_no, int enable) -{ - size_t mask = (((size_t)1) << (bit_no % BITS_PER_WORD)); - - if (enable) - p[bit_no / BITS_PER_WORD] |= mask; - else - p[bit_no / BITS_PER_WORD] &= ~mask; -} - -struct qlog_st { - QLOG_TRACE_INFO info; - - BIO *bio; - size_t enabled[NUM_ENABLED_W]; - uint32_t event_type; - const char *event_cat, *event_name, *event_combined_name; - OSSL_TIME event_time, prev_event_time; - OSSL_JSON_ENC json; - int header_done, first_event_done; -}; - -static OSSL_TIME default_now(void *arg) -{ - return ossl_time_now(); -} - -/* - * Construction - * ============ - */ -QLOG *ossl_qlog_new(const QLOG_TRACE_INFO *info) -{ - QLOG *qlog = OPENSSL_zalloc(sizeof(QLOG)); - - if (qlog == NULL) - return NULL; - - qlog->info.odcid = info->odcid; - qlog->info.is_server = info->is_server; - qlog->info.now_cb = info->now_cb; - qlog->info.now_cb_arg = info->now_cb_arg; - qlog->info.override_process_id = info->override_process_id; - - if (info->title != NULL - && (qlog->info.title = OPENSSL_strdup(info->title)) == NULL) - goto err; - - if (info->description != NULL - && (qlog->info.description = OPENSSL_strdup(info->description)) == NULL) - goto err; - - if (info->group_id != NULL - && (qlog->info.group_id = OPENSSL_strdup(info->group_id)) == NULL) - goto err; - - if (info->override_impl_name != NULL - && (qlog->info.override_impl_name - = OPENSSL_strdup(info->override_impl_name)) == NULL) - goto err; - - if (!ossl_json_init(&qlog->json, NULL, - OSSL_JSON_FLAG_IJSON | OSSL_JSON_FLAG_SEQ)) - goto err; - - if (qlog->info.now_cb == NULL) - qlog->info.now_cb = default_now; - - return qlog; - -err: - if (qlog != NULL) { - OPENSSL_free((char *)qlog->info.title); - OPENSSL_free((char *)qlog->info.description); - OPENSSL_free((char *)qlog->info.group_id); - OPENSSL_free((char *)qlog->info.override_impl_name); - OPENSSL_free(qlog); - } - return NULL; -} - -QLOG *ossl_qlog_new_from_env(const QLOG_TRACE_INFO *info) -{ - QLOG *qlog = NULL; - const char *qlogdir = ossl_safe_getenv("QLOGDIR"); - const char *qfilter = ossl_safe_getenv("OSSL_QFILTER"); - char qlogdir_sep, *filename = NULL; - size_t i, l, strl; - - if (info == NULL || qlogdir == NULL) - return NULL; - - l = strlen(qlogdir); - if (l == 0) - return NULL; - - qlogdir_sep = ossl_determine_dirsep(qlogdir); - - /* dir; [sep]; ODCID; _; strlen("client" / "server"); strlen(".sqlog"); NUL */ - strl = l + 1 + info->odcid.id_len * 2 + 1 + 6 + 6 + 1; - filename = OPENSSL_malloc(strl); - if (filename == NULL) - return NULL; - - memcpy(filename, qlogdir, l); - if (qlogdir_sep != '\0') - filename[l++] = qlogdir_sep; - - for (i = 0; i < info->odcid.id_len; ++i) - l += BIO_snprintf(filename + l, strl - l, "%02x", info->odcid.id[i]); - - l += BIO_snprintf(filename + l, strl - l, "_%s.sqlog", - info->is_server ? "server" : "client"); - - qlog = ossl_qlog_new(info); - if (qlog == NULL) - goto err; - - if (!ossl_qlog_set_sink_filename(qlog, filename)) - goto err; - - if (qfilter == NULL || qfilter[0] == '\0') - qfilter = "*"; - - if (!ossl_qlog_set_filter(qlog, qfilter)) - goto err; - - OPENSSL_free(filename); - return qlog; - -err: - OPENSSL_free(filename); - ossl_qlog_free(qlog); - return NULL; -} - -void ossl_qlog_free(QLOG *qlog) -{ - if (qlog == NULL) - return; - - ossl_json_flush_cleanup(&qlog->json); - BIO_free_all(qlog->bio); - OPENSSL_free((char *)qlog->info.title); - OPENSSL_free((char *)qlog->info.description); - OPENSSL_free((char *)qlog->info.group_id); - OPENSSL_free((char *)qlog->info.override_impl_name); - OPENSSL_free(qlog); -} - -/* - * Configuration - * ============= - */ -int ossl_qlog_set_sink_bio(QLOG *qlog, BIO *bio) -{ - if (qlog == NULL) - return 0; - - ossl_qlog_flush(qlog); /* best effort */ - BIO_free_all(qlog->bio); - qlog->bio = bio; - ossl_json_set0_sink(&qlog->json, bio); - return 1; -} - -#ifndef OPENSSL_NO_STDIO - -int ossl_qlog_set_sink_file(QLOG *qlog, FILE *f, int close_flag) -{ - BIO *bio; - - if (qlog == NULL) - return 0; - - bio = BIO_new_fp(f, BIO_CLOSE); - if (bio == NULL) - return 0; - - if (!ossl_qlog_set_sink_bio(qlog, bio)) { - BIO_free_all(bio); - return 0; - } - - return 1; -} - -#endif - -int ossl_qlog_set_sink_filename(QLOG *qlog, const char *filename) -{ - BIO *bio; - - if (qlog == NULL) - return 0; - - /* - * We supply our own text encoding as JSON requires UTF-8, so disable any - * OS-specific processing here. - */ - bio = BIO_new_file(filename, "wb"); - if (bio == NULL) - return 0; - - if (!ossl_qlog_set_sink_bio(qlog, bio)) { - BIO_free_all(bio); - return 0; - } - - return 1; -} - -int ossl_qlog_flush(QLOG *qlog) -{ - if (qlog == NULL) - return 1; - - return ossl_json_flush(&qlog->json); -} - -int ossl_qlog_set_event_type_enabled(QLOG *qlog, uint32_t event_type, - int enabled) -{ - if (qlog == NULL || event_type >= QLOG_EVENT_TYPE_NUM) - return 0; - - bit_set(qlog->enabled, event_type, enabled); - return 1; -} - -int ossl_qlog_enabled(QLOG *qlog, uint32_t event_type) -{ - if (qlog == NULL) - return 0; - - return bit_get(qlog->enabled, event_type) != 0; -} - -/* - * Event Lifecycle - * =============== - */ -static void write_str_once(QLOG *qlog, const char *key, char **p) -{ - if (*p == NULL) - return; - - ossl_json_key(&qlog->json, key); - ossl_json_str(&qlog->json, *p); - - OPENSSL_free(*p); - *p = NULL; -} - -static void qlog_event_seq_header(QLOG *qlog) -{ - if (qlog->header_done) - return; - - ossl_json_object_begin(&qlog->json); - { - ossl_json_key(&qlog->json, "qlog_version"); - ossl_json_str(&qlog->json, "0.3"); - - ossl_json_key(&qlog->json, "qlog_format"); - ossl_json_str(&qlog->json, "JSON-SEQ"); - - write_str_once(qlog, "title", (char **)&qlog->info.title); - write_str_once(qlog, "description", (char **)&qlog->info.description); - - ossl_json_key(&qlog->json, "trace"); - ossl_json_object_begin(&qlog->json); - { - ossl_json_key(&qlog->json, "common_fields"); - ossl_json_object_begin(&qlog->json); - { - ossl_json_key(&qlog->json, "time_format"); - ossl_json_str(&qlog->json, "delta"); - - ossl_json_key(&qlog->json, "protocol_type"); - ossl_json_array_begin(&qlog->json); - { - ossl_json_str(&qlog->json, "QUIC"); - } /* protocol_type */ - ossl_json_array_end(&qlog->json); - - write_str_once(qlog, "group_id", (char **)&qlog->info.group_id); - - ossl_json_key(&qlog->json, "system_info"); - ossl_json_object_begin(&qlog->json); - { - if (qlog->info.override_process_id != 0) { - ossl_json_key(&qlog->json, "process_id"); - ossl_json_u64(&qlog->json, qlog->info.override_process_id); - } else { -#if defined(OPENSSL_SYS_UNIX) - ossl_json_key(&qlog->json, "process_id"); - ossl_json_u64(&qlog->json, (uint64_t)getpid()); -#elif defined(OPENSSL_SYS_WINDOWS) - ossl_json_key(&qlog->json, "process_id"); - ossl_json_u64(&qlog->json, (uint64_t)GetCurrentProcessId()); -#endif - } - } /* system_info */ - ossl_json_object_end(&qlog->json); - } /* common_fields */ - ossl_json_object_end(&qlog->json); - - ossl_json_key(&qlog->json, "vantage_point"); - ossl_json_object_begin(&qlog->json); - { - char buf[128]; - const char *p = buf; - - if (qlog->info.override_impl_name != NULL) { - p = qlog->info.override_impl_name; - } else { - BIO_snprintf(buf, sizeof(buf), "OpenSSL/%s (%s)", - OpenSSL_version(OPENSSL_FULL_VERSION_STRING), - OpenSSL_version(OPENSSL_PLATFORM) + 10); - } - - ossl_json_key(&qlog->json, "type"); - ossl_json_str(&qlog->json, - qlog->info.is_server ? "server" : "client"); - - ossl_json_key(&qlog->json, "name"); - ossl_json_str(&qlog->json, p); - } /* vantage_point */ - ossl_json_object_end(&qlog->json); - } /* trace */ - ossl_json_object_end(&qlog->json); - } - ossl_json_object_end(&qlog->json); - - qlog->header_done = 1; -} - -static void qlog_event_prologue(QLOG *qlog) -{ - qlog_event_seq_header(qlog); - - ossl_json_object_begin(&qlog->json); - - ossl_json_key(&qlog->json, "name"); - ossl_json_str(&qlog->json, qlog->event_combined_name); - - ossl_json_key(&qlog->json, "data"); - ossl_json_object_begin(&qlog->json); -} - -static void qlog_event_epilogue(QLOG *qlog) -{ - ossl_json_object_end(&qlog->json); - - ossl_json_key(&qlog->json, "time"); - if (!qlog->first_event_done) { - ossl_json_u64(&qlog->json, ossl_time2ms(qlog->event_time)); - qlog->prev_event_time = qlog->event_time; - qlog->first_event_done = 1; - } else { - OSSL_TIME delta = ossl_time_subtract(qlog->event_time, - qlog->prev_event_time); - - ossl_json_u64(&qlog->json, ossl_time2ms(delta)); - qlog->prev_event_time = qlog->event_time; - } - - ossl_json_object_end(&qlog->json); -} - -int ossl_qlog_event_try_begin(QLOG *qlog, - uint32_t event_type, - const char *event_cat, - const char *event_name, - const char *event_combined_name) -{ - if (qlog == NULL) - return 0; - - if (!ossl_assert(qlog->event_type == QLOG_EVENT_TYPE_NONE) - || !ossl_qlog_enabled(qlog, event_type)) - return 0; - - qlog->event_type = event_type; - qlog->event_cat = event_cat; - qlog->event_name = event_name; - qlog->event_combined_name = event_combined_name; - qlog->event_time = qlog->info.now_cb(qlog->info.now_cb_arg); - - qlog_event_prologue(qlog); - return 1; -} - -void ossl_qlog_event_end(QLOG *qlog) -{ - if (!ossl_assert(qlog != NULL && qlog->event_type != QLOG_EVENT_TYPE_NONE)) - return; - - qlog_event_epilogue(qlog); - qlog->event_type = QLOG_EVENT_TYPE_NONE; -} - -/* - * Field Generators - * ================ - */ -void ossl_qlog_group_begin(QLOG *qlog, const char *name) -{ - if (name != NULL) - ossl_json_key(&qlog->json, name); - - ossl_json_object_begin(&qlog->json); -} - -void ossl_qlog_group_end(QLOG *qlog) -{ - ossl_json_object_end(&qlog->json); -} - -void ossl_qlog_array_begin(QLOG *qlog, const char *name) -{ - if (name != NULL) - ossl_json_key(&qlog->json, name); - - ossl_json_array_begin(&qlog->json); -} - -void ossl_qlog_array_end(QLOG *qlog) -{ - ossl_json_array_end(&qlog->json); -} - -void ossl_qlog_override_time(QLOG *qlog, OSSL_TIME event_time) -{ - qlog->event_time = event_time; -} - -void ossl_qlog_str(QLOG *qlog, const char *name, const char *value) -{ - if (name != NULL) - ossl_json_key(&qlog->json, name); - - ossl_json_str(&qlog->json, value); -} - -void ossl_qlog_str_len(QLOG *qlog, const char *name, - const char *value, size_t value_len) -{ - if (name != NULL) - ossl_json_key(&qlog->json, name); - - ossl_json_str_len(&qlog->json, value, value_len); -} - -void ossl_qlog_u64(QLOG *qlog, const char *name, uint64_t value) -{ - if (name != NULL) - ossl_json_key(&qlog->json, name); - - ossl_json_u64(&qlog->json, value); -} - -void ossl_qlog_i64(QLOG *qlog, const char *name, int64_t value) -{ - if (name != NULL) - ossl_json_key(&qlog->json, name); - - ossl_json_i64(&qlog->json, value); -} - -void ossl_qlog_bool(QLOG *qlog, const char *name, int value) -{ - if (name != NULL) - ossl_json_key(&qlog->json, name); - - ossl_json_bool(&qlog->json, value); -} - -void ossl_qlog_bin(QLOG *qlog, const char *name, - const void *value, size_t value_len) -{ - if (name != NULL) - ossl_json_key(&qlog->json, name); - - ossl_json_str_hex(&qlog->json, value, value_len); -} - -/* - * Filter Parsing - * ============== - */ -struct lexer { - const char *p, *term_end, *end; -}; - -static ossl_inline int is_term_sep_ws(char c) -{ - return c == ' ' || c == '\r' || c == '\n' || c == '\t'; -} - -static ossl_inline int is_name_char(char c) -{ - return ossl_isalpha(c) || ossl_isdigit(c) || c == '_' || c == '-'; -} - -static int lex_init(struct lexer *lex, const char *in, size_t in_len) -{ - if (in == NULL) - return 0; - - lex->p = in; - lex->term_end = in; - lex->end = in + in_len; - return 1; -} - -static int lex_do(struct lexer *lex) -{ - const char *p = lex->term_end, *end = lex->end, *term_end; - - for (; is_term_sep_ws(*p) && p < end; ++p); - - if (p == end) { - lex->p = end; - lex->term_end = end; - return 0; - } - - for (term_end = p; !is_term_sep_ws(*term_end) && term_end < end; ++term_end); - - lex->p = p; - lex->term_end = term_end; - return 1; -} - -static int lex_eot(struct lexer *lex) -{ - return lex->p == lex->term_end; -} - -static int lex_peek_char(struct lexer *lex) -{ - return lex_eot(lex) ? -1 : *lex->p; -} - -static int lex_skip_char(struct lexer *lex) -{ - if (lex_eot(lex)) - return 0; - - ++lex->p; - return 1; -} - -static int lex_match(struct lexer *lex, const char *s, size_t s_len) -{ - if ((size_t)(lex->term_end - lex->p) != s_len) - return 0; - - if (memcmp(lex->p, s, s_len)) - return 0; - - return 1; -} - -static void lex_get_rest(struct lexer *lex, const char **str, size_t *str_l) -{ - *str = lex->p; - *str_l = lex->term_end - lex->p; -} - -static int lex_extract_to(struct lexer *lex, char c, - const char **str, size_t *str_l) -{ - const char *p = lex->p, *term_end = lex->term_end, *s; - - for (s = p; s < term_end && *s != c; ++s); - if (s == term_end) - return 0; - - *str = p; - *str_l = s - p; - lex->p = ++s; - return 1; -} - -static int ossl_unused filter_match_event(const char *cat, size_t cat_l, - const char *event, size_t event_l, - const char *expect_cat, - const char *expect_event) -{ - size_t expect_cat_l = strlen(expect_cat); - size_t expect_event_l = strlen(expect_event); - - if ((cat != NULL && cat_l != expect_cat_l) - || (event != NULL && event_l != expect_event_l) - || (cat != NULL && memcmp(cat, expect_cat, expect_cat_l)) - || (event != NULL && memcmp(event, expect_event, expect_event_l))) - return 0; - - return 1; -} - -/* - * enabled: event enablement bitmask Array of size NUM_ENABLED_W. - * add: 1 to enable an event, 0 to disable. - * cat: Category name/length. Not necessarily zero terminated. - * NULL to match any. - * event: Event name/length. Not necessarily zero terminated. - * NULL to match any. - */ -static void filter_apply(size_t *enabled, int add, - const char *cat, size_t cat_l, - const char *event, size_t event_l) -{ - /* Find events which match the given filters. */ -# define QLOG_EVENT(e_cat, e_name) \ - if (filter_match_event(cat, cat_l, event, event_l, \ - #e_cat, #e_name)) \ - bit_set(enabled, QLOG_EVENT_TYPE_##e_cat##_##e_name, add); -# include "internal/qlog_events.h" -# undef QLOG_EVENT -} - -static int lex_fail(struct lexer *lex, const char *msg) -{ - /* - * TODO(QLOG FUTURE): Determine how to print log messages about bad filter - * strings - */ - lex->p = lex->term_end = lex->end; - return 0; -} - -static int validate_name(const char **p, size_t *l) -{ - const char *p_ = *p; - size_t i, l_ = *l; - - if (l_ == 1 && *p_ == '*') { - *p = NULL; - *l = 0; - return 1; - } - - if (l_ == 0) - return 0; - - for (i = 0; i < l_; ++i) - if (!is_name_char(p_[i])) - return 0; - - return 1; -} - -int ossl_qlog_set_filter(QLOG *qlog, const char *filter) -{ - struct lexer lex = {0}; - char c; - const char *cat, *event; - size_t cat_l, event_l, enabled[NUM_ENABLED_W]; - int add; - - memcpy(enabled, qlog->enabled, sizeof(enabled)); - - if (!lex_init(&lex, filter, strlen(filter))) - return 0; - - while (lex_do(&lex)) { - c = lex_peek_char(&lex); - if (c == '+' || c == '-') { - add = (c == '+'); - lex_skip_char(&lex); - - c = lex_peek_char(&lex); - if (!is_name_char(c) && c != '*') - return lex_fail(&lex, "expected alphanumeric name or '*'" - " after +/-"); - } else if (!is_name_char(c) && c != '*') { - return lex_fail(&lex, "expected +/- or alphanumeric name or '*'"); - } else { - add = 1; - } - - if (lex_match(&lex, "*", 1)) { - filter_apply(enabled, add, NULL, 0, NULL, 0); - continue; - } - - if (!lex_extract_to(&lex, ':', &cat, &cat_l)) - return lex_fail(&lex, "expected ':' after category name"); - - lex_get_rest(&lex, &event, &event_l); - if (!validate_name(&cat, &cat_l)) - return lex_fail(&lex, "expected alphanumeric category name or '*'"); - if (!validate_name(&event, &event_l)) - return lex_fail(&lex, "expected alphanumeric event name or '*'"); - - filter_apply(enabled, add, cat, cat_l, event, event_l); - } - - memcpy(qlog->enabled, enabled, sizeof(enabled)); - return 1; -} diff --git a/openssl/src/ssl/quic/qlog_event_helpers.c b/openssl/src/ssl/quic/qlog_event_helpers.c deleted file mode 100644 index 55cc28d9f..000000000 --- a/openssl/src/ssl/quic/qlog_event_helpers.c +++ /dev/null @@ -1,634 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/qlog_event_helpers.h" -#include "internal/common.h" -#include "internal/packet.h" -#include "internal/quic_channel.h" -#include "internal/quic_error.h" - -void ossl_qlog_event_connectivity_connection_started(QLOG *qlog, - const QUIC_CONN_ID *init_dcid) -{ -#ifndef OPENSSL_NO_QLOG - QLOG_EVENT_BEGIN(qlog, connectivity, connection_started) - QLOG_STR("protocol", "quic"); - QLOG_CID("dst_cid", init_dcid); - QLOG_EVENT_END() -#endif -} - -#ifndef OPENSSL_NO_QLOG -static const char *map_state_to_qlog(uint32_t state, - int handshake_complete, - int handshake_confirmed) -{ - switch (state) { - default: - case QUIC_CHANNEL_STATE_IDLE: - return NULL; - - case QUIC_CHANNEL_STATE_ACTIVE: - if (handshake_confirmed) - return "handshake_confirmed"; - else if (handshake_complete) - return "handshake_complete"; - else - return "attempted"; - - case QUIC_CHANNEL_STATE_TERMINATING_CLOSING: - return "closing"; - - case QUIC_CHANNEL_STATE_TERMINATING_DRAINING: - return "draining"; - - case QUIC_CHANNEL_STATE_TERMINATED: - return "closed"; - } -} -#endif - -void ossl_qlog_event_connectivity_connection_state_updated(QLOG *qlog, - uint32_t old_state, - uint32_t new_state, - int handshake_complete, - int handshake_confirmed) -{ -#ifndef OPENSSL_NO_QLOG - const char *state_s; - - QLOG_EVENT_BEGIN(qlog, connectivity, connection_state_updated) - state_s = map_state_to_qlog(new_state, - handshake_complete, - handshake_confirmed); - - if (state_s != NULL) - QLOG_STR("state", state_s); - QLOG_EVENT_END() -#endif -} - -#ifndef OPENSSL_NO_QLOG -static const char *quic_err_to_qlog(uint64_t error_code) -{ - switch (error_code) { - case OSSL_QUIC_ERR_INTERNAL_ERROR: - return "internal_error"; - case OSSL_QUIC_ERR_CONNECTION_REFUSED: - return "connection_refused"; - case OSSL_QUIC_ERR_FLOW_CONTROL_ERROR: - return "flow_control_error"; - case OSSL_QUIC_ERR_STREAM_LIMIT_ERROR: - return "stream_limit_error"; - case OSSL_QUIC_ERR_STREAM_STATE_ERROR: - return "stream_state_error"; - case OSSL_QUIC_ERR_FINAL_SIZE_ERROR: - return "final_size_error"; - case OSSL_QUIC_ERR_FRAME_ENCODING_ERROR: - return "frame_encoding_error"; - case OSSL_QUIC_ERR_TRANSPORT_PARAMETER_ERROR: - return "transport_parameter_error"; - case OSSL_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR: - return "connection_id_limit_error"; - case OSSL_QUIC_ERR_PROTOCOL_VIOLATION: - return "protocol_violation"; - case OSSL_QUIC_ERR_INVALID_TOKEN: - return "invalid_token"; - case OSSL_QUIC_ERR_APPLICATION_ERROR: - return "application_error"; - case OSSL_QUIC_ERR_CRYPTO_BUFFER_EXCEEDED: - return "crypto_buffer_exceeded"; - case OSSL_QUIC_ERR_KEY_UPDATE_ERROR: - return "key_update_error"; - case OSSL_QUIC_ERR_AEAD_LIMIT_REACHED: - return "aead_limit_reached"; - case OSSL_QUIC_ERR_NO_VIABLE_PATH: - return "no_viable_path"; - default: - return NULL; - } -} -#endif - -void ossl_qlog_event_connectivity_connection_closed(QLOG *qlog, - const QUIC_TERMINATE_CAUSE *tcause) -{ -#ifndef OPENSSL_NO_QLOG - QLOG_EVENT_BEGIN(qlog, connectivity, connection_closed) - QLOG_STR("owner", tcause->remote ? "remote" : "local"); - if (tcause->app) { - QLOG_U64("application_code", tcause->error_code); - } else { - const char *m = quic_err_to_qlog(tcause->error_code); - char ce[32]; - - if (tcause->error_code >= OSSL_QUIC_ERR_CRYPTO_ERR_BEGIN - && tcause->error_code <= OSSL_QUIC_ERR_CRYPTO_ERR_END) { - BIO_snprintf(ce, sizeof(ce), "crypto_error_0x%03llx", - (unsigned long long)tcause->error_code); - m = ce; - } - /* TODO(QLOG FUTURE): Consider adding ERR information in the output. */ - - if (m != NULL) - QLOG_STR("connection_code", m); - else - QLOG_U64("connection_code", tcause->error_code); - } - - QLOG_STR_LEN("reason", tcause->reason, tcause->reason_len); - QLOG_EVENT_END() -#endif -} - -#ifndef OPENSSL_NO_QLOG -static const char *quic_pkt_type_to_qlog(uint32_t pkt_type) -{ - switch (pkt_type) { - case QUIC_PKT_TYPE_INITIAL: - return "initial"; - case QUIC_PKT_TYPE_HANDSHAKE: - return "handshake"; - case QUIC_PKT_TYPE_0RTT: - return "0RTT"; - case QUIC_PKT_TYPE_1RTT: - return "1RTT"; - case QUIC_PKT_TYPE_VERSION_NEG: - return "version_negotiation"; - case QUIC_PKT_TYPE_RETRY: - return "retry"; - default: - return "unknown"; - } -} -#endif - -void ossl_qlog_event_recovery_packet_lost(QLOG *qlog, - const QUIC_TXPIM_PKT *tpkt) -{ -#ifndef OPENSSL_NO_QLOG - QLOG_EVENT_BEGIN(qlog, recovery, packet_lost) - QLOG_BEGIN("header") - QLOG_STR("packet_type", quic_pkt_type_to_qlog(tpkt->pkt_type)); - if (ossl_quic_pkt_type_has_pn(tpkt->pkt_type)) - QLOG_U64("packet_number", tpkt->ackm_pkt.pkt_num); - QLOG_END() - QLOG_EVENT_END() -#endif -} - -#ifndef OPENSSL_NO_QLOG -# define MAX_ACK_RANGES 32 - -static void ignore_res(int x) {} - -/* - * For logging received packets, we need to parse all the frames in the packet - * to log them. We should do this separately to the RXDP code because we want to - * log the packet and its contents before we start to actually process it in - * case it causes an error. We also in general don't want to do other - * non-logging related work in the middle of an event logging transaction. - * Reparsing packet data allows us to meet these needs while avoiding the need - * to keep around bookkeeping data on what frames were in a packet, etc. - * - * For logging transmitted packets, we actually reuse the same code and reparse - * the outgoing packet's payload. This again has the advantage that we only log - * a packet when it is actually queued for transmission (and not if something - * goes wrong before then) while avoiding the need to keep around bookkeeping - * data on what frames it contained. - */ -static int log_frame_actual(QLOG *qlog_instance, PACKET *pkt, - size_t *need_skip) -{ - uint64_t frame_type; - OSSL_QUIC_FRAME_ACK ack; - OSSL_QUIC_ACK_RANGE ack_ranges[MAX_ACK_RANGES]; - uint64_t num_ranges, total_ranges; - size_t i; - PACKET orig_pkt = *pkt; - - if (!ossl_quic_wire_peek_frame_header(pkt, &frame_type, NULL)) - return 0; - - /* - * If something goes wrong decoding a frame we cannot log it as that frame - * as we need to know how to decode it in order to be able to do so, but in - * that case we log it as an unknown frame to assist with diagnosis. - */ - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_PADDING: - QLOG_STR("frame_type", "padding"); - QLOG_U64("payload_length", - ossl_quic_wire_decode_padding(pkt)); - break; - case OSSL_QUIC_FRAME_TYPE_PING: - if (!ossl_quic_wire_decode_frame_ping(pkt)) - goto unknown; - - QLOG_STR("frame_type", "ping"); - break; - case OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN: - case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN: - if (!ossl_quic_wire_peek_frame_ack_num_ranges(pkt, &num_ranges)) - goto unknown; - - ack.ack_ranges = ack_ranges; - ack.num_ack_ranges = OSSL_NELEM(ack_ranges); - if (!ossl_quic_wire_decode_frame_ack(pkt, 3, &ack, &total_ranges)) - goto unknown; - - QLOG_STR("frame_type", "ack"); - QLOG_U64("ack_delay", ossl_time2ms(ack.delay_time)); - if (ack.ecn_present) { - QLOG_U64("ect1", ack.ect0); - QLOG_U64("ect0", ack.ect1); - QLOG_U64("ce", ack.ecnce); - } - QLOG_BEGIN_ARRAY("acked_ranges"); - for (i = 0; i < ack.num_ack_ranges; ++i) { - QLOG_BEGIN_ARRAY(NULL) - QLOG_U64(NULL, ack.ack_ranges[i].start); - if (ack.ack_ranges[i].end != ack.ack_ranges[i].start) - QLOG_U64(NULL, ack.ack_ranges[i].end); - QLOG_END_ARRAY() - } - QLOG_END_ARRAY() - break; - case OSSL_QUIC_FRAME_TYPE_RESET_STREAM: - { - OSSL_QUIC_FRAME_RESET_STREAM f; - - if (!ossl_quic_wire_decode_frame_reset_stream(pkt, &f)) - goto unknown; - - QLOG_STR("frame_type", "reset_stream"); - QLOG_U64("stream_id", f.stream_id); - QLOG_U64("error_code", f.app_error_code); - QLOG_U64("final_size", f.final_size); - } - break; - case OSSL_QUIC_FRAME_TYPE_STOP_SENDING: - { - OSSL_QUIC_FRAME_STOP_SENDING f; - - if (!ossl_quic_wire_decode_frame_stop_sending(pkt, &f)) - goto unknown; - - QLOG_STR("frame_type", "stop_sending"); - QLOG_U64("stream_id", f.stream_id); - QLOG_U64("error_code", f.app_error_code); - } - break; - case OSSL_QUIC_FRAME_TYPE_CRYPTO: - { - OSSL_QUIC_FRAME_CRYPTO f; - - if (!ossl_quic_wire_decode_frame_crypto(pkt, 1, &f)) - goto unknown; - - QLOG_STR("frame_type", "crypto"); - QLOG_U64("offset", f.offset); - QLOG_U64("payload_length", f.len); - *need_skip += (size_t)f.len; - } - break; - case OSSL_QUIC_FRAME_TYPE_STREAM: - case OSSL_QUIC_FRAME_TYPE_STREAM_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_LEN: - case OSSL_QUIC_FRAME_TYPE_STREAM_LEN_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN_FIN: - { - OSSL_QUIC_FRAME_STREAM f; - - if (!ossl_quic_wire_decode_frame_stream(pkt, 1, &f)) - goto unknown; - - QLOG_STR("frame_type", "stream"); - QLOG_U64("stream_id", f.stream_id); - QLOG_U64("offset", f.offset); - QLOG_U64("payload_length", f.len); - QLOG_BOOL("explicit_length", f.has_explicit_len); - if (f.is_fin) - QLOG_BOOL("fin", 1); - *need_skip = f.has_explicit_len - ? *need_skip + (size_t)f.len : SIZE_MAX; - } - break; - case OSSL_QUIC_FRAME_TYPE_MAX_DATA: - { - uint64_t x; - - if (!ossl_quic_wire_decode_frame_max_data(pkt, &x)) - goto unknown; - - QLOG_STR("frame_type", "max_data"); - QLOG_U64("maximum", x); - } - break; - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI: - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI: - { - uint64_t x; - - if (!ossl_quic_wire_decode_frame_max_streams(pkt, &x)) - goto unknown; - - QLOG_STR("frame_type", "max_streams"); - QLOG_STR("stream_type", - frame_type == OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI - ? "bidirectional" : "unidirectional"); - QLOG_U64("maximum", x); - } - break; - case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA: - { - uint64_t stream_id, max_data; - - if (!ossl_quic_wire_decode_frame_max_stream_data(pkt, &stream_id, - &max_data)) - goto unknown; - - QLOG_STR("frame_type", "max_stream_data"); - QLOG_U64("stream_id", stream_id); - QLOG_U64("maximum", max_data); - } - break; - case OSSL_QUIC_FRAME_TYPE_PATH_CHALLENGE: - { - uint64_t challenge; - - if (!ossl_quic_wire_decode_frame_path_challenge(pkt, &challenge)) - goto unknown; - - QLOG_STR("frame_type", "path_challenge"); - } - break; - case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE: - { - uint64_t challenge; - - if (!ossl_quic_wire_decode_frame_path_response(pkt, &challenge)) - goto unknown; - - QLOG_STR("frame_type", "path_response"); - } - break; - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_APP: - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT: - { - OSSL_QUIC_FRAME_CONN_CLOSE f; - - if (!ossl_quic_wire_decode_frame_conn_close(pkt, &f)) - goto unknown; - - QLOG_STR("frame_type", "connection_close"); - QLOG_STR("error_space", f.is_app ? "application" : "transport"); - QLOG_U64("error_code_value", f.error_code); - if (f.is_app) - QLOG_U64("error_code", f.error_code); - if (!f.is_app && f.frame_type != 0) - QLOG_U64("trigger_frame_type", f.frame_type); - QLOG_STR_LEN("reason", f.reason, f.reason_len); - } - break; - case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE: - { - if (!ossl_quic_wire_decode_frame_handshake_done(pkt)) - goto unknown; - - QLOG_STR("frame_type", "handshake_done"); - } - break; - case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID: - { - OSSL_QUIC_FRAME_NEW_CONN_ID f; - - if (!ossl_quic_wire_decode_frame_new_conn_id(pkt, &f)) - goto unknown; - - QLOG_STR("frame_type", "new_connection_id"); - QLOG_U64("sequence_number", f.seq_num); - QLOG_U64("retire_prior_to", f.retire_prior_to); - QLOG_CID("connection_id", &f.conn_id); - QLOG_BIN("stateless_reset_token", - f.stateless_reset.token, - sizeof(f.stateless_reset.token)); - } - break; - case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID: - { - uint64_t seq_num; - - if (!ossl_quic_wire_decode_frame_retire_conn_id(pkt, &seq_num)) - goto unknown; - - QLOG_STR("frame_type", "retire_connection_id"); - QLOG_U64("sequence_number", seq_num); - } - break; - case OSSL_QUIC_FRAME_TYPE_DATA_BLOCKED: - { - uint64_t x; - - if (!ossl_quic_wire_decode_frame_data_blocked(pkt, &x)) - goto unknown; - - QLOG_STR("frame_type", "data_blocked"); - QLOG_U64("limit", x); - } - break; - case OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED: - { - uint64_t stream_id, x; - - if (!ossl_quic_wire_decode_frame_stream_data_blocked(pkt, - &stream_id, - &x)) - goto unknown; - - QLOG_STR("frame_type", "stream_data_blocked"); - QLOG_U64("stream_id", stream_id); - QLOG_U64("limit", x); - } - break; - case OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI: - case OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_UNI: - { - uint64_t x; - - if (!ossl_quic_wire_decode_frame_streams_blocked(pkt, &x)) - goto unknown; - - QLOG_STR("frame_type", "streams_blocked"); - QLOG_STR("stream_type", - frame_type == OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI - ? "bidirectional" : "unidirectional"); - QLOG_U64("limit", x); - } - break; - case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN: - { - const unsigned char *token; - size_t token_len; - - if (!ossl_quic_wire_decode_frame_new_token(pkt, &token, &token_len)) - goto unknown; - - QLOG_STR("frame_type", "new_token"); - QLOG_BEGIN("token"); - QLOG_BEGIN("raw"); - QLOG_BIN("data", token, token_len); - QLOG_END(); - QLOG_END(); - } - break; - default: -unknown: - QLOG_STR("frame_type", "unknown"); - QLOG_U64("frame_type_value", frame_type); - - /* - * Can't continue scanning for frames in this case as the frame length - * is unknown. We log the entire body of the rest of the packet payload - * as the raw data of the frame. - */ - QLOG_BEGIN("raw"); - QLOG_BIN("data", PACKET_data(&orig_pkt), - PACKET_remaining(&orig_pkt)); - QLOG_END(); - ignore_res(PACKET_forward(pkt, PACKET_remaining(pkt))); - break; - } - - return 1; -} - -static void log_frame(QLOG *qlog_instance, PACKET *pkt, - size_t *need_skip) -{ - size_t rem_before, rem_after; - - rem_before = PACKET_remaining(pkt); - - if (!log_frame_actual(qlog_instance, pkt, need_skip)) - return; - - rem_after = PACKET_remaining(pkt); - QLOG_U64("length", rem_before - rem_after); -} - -static int log_frames(QLOG *qlog_instance, - const OSSL_QTX_IOVEC *iovec, - size_t num_iovec) -{ - size_t i; - PACKET pkt; - size_t need_skip = 0; - - for (i = 0; i < num_iovec; ++i) { - if (!PACKET_buf_init(&pkt, iovec[i].buf, iovec[i].buf_len)) - return 0; - - while (PACKET_remaining(&pkt) > 0) { - if (need_skip > 0) { - size_t adv = need_skip; - - if (adv < PACKET_remaining(&pkt)) - adv = PACKET_remaining(&pkt); - - if (!PACKET_forward(&pkt, adv)) - return 0; - - need_skip -= adv; - continue; - } - - QLOG_BEGIN(NULL) - { - log_frame(qlog_instance, &pkt, &need_skip); - } - QLOG_END() - } - } - - return 1; -} - -static void log_packet(QLOG *qlog_instance, - const QUIC_PKT_HDR *hdr, - QUIC_PN pn, - const OSSL_QTX_IOVEC *iovec, - size_t num_iovec, - uint64_t datagram_id) -{ - const char *type_s; - - QLOG_BEGIN("header") - type_s = quic_pkt_type_to_qlog(hdr->type); - if (type_s == NULL) - type_s = "unknown"; - - QLOG_STR("packet_type", type_s); - if (ossl_quic_pkt_type_has_pn(hdr->type)) - QLOG_U64("packet_number", pn); - - QLOG_CID("dcid", &hdr->dst_conn_id); - if (ossl_quic_pkt_type_has_scid(hdr->type)) - QLOG_CID("scid", &hdr->src_conn_id); - - if (hdr->token_len > 0) { - QLOG_BEGIN("token") - QLOG_BEGIN("raw") - QLOG_BIN("data", hdr->token, hdr->token_len); - QLOG_END() - QLOG_END() - } - /* TODO(QLOG FUTURE): flags, length */ - QLOG_END() - QLOG_U64("datagram_id", datagram_id); - - if (ossl_quic_pkt_type_is_encrypted(hdr->type)) { - QLOG_BEGIN_ARRAY("frames") - log_frames(qlog_instance, iovec, num_iovec); - QLOG_END_ARRAY() - } -} - -#endif - -void ossl_qlog_event_transport_packet_sent(QLOG *qlog, - const QUIC_PKT_HDR *hdr, - QUIC_PN pn, - const OSSL_QTX_IOVEC *iovec, - size_t num_iovec, - uint64_t datagram_id) -{ -#ifndef OPENSSL_NO_QLOG - QLOG_EVENT_BEGIN(qlog, transport, packet_sent) - log_packet(qlog, hdr, pn, iovec, num_iovec, datagram_id); - QLOG_EVENT_END() -#endif -} - -void ossl_qlog_event_transport_packet_received(QLOG *qlog, - const QUIC_PKT_HDR *hdr, - QUIC_PN pn, - const OSSL_QTX_IOVEC *iovec, - size_t num_iovec, - uint64_t datagram_id) -{ -#ifndef OPENSSL_NO_QLOG - QLOG_EVENT_BEGIN(qlog, transport, packet_received) - log_packet(qlog, hdr, pn, iovec, num_iovec, datagram_id); - QLOG_EVENT_END() -#endif -} diff --git a/openssl/src/ssl/quic/quic_ackm.c b/openssl/src/ssl/quic/quic_ackm.c deleted file mode 100644 index 75a1e5741..000000000 --- a/openssl/src/ssl/quic/quic_ackm.c +++ /dev/null @@ -1,1725 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_ackm.h" -#include "internal/uint_set.h" -#include "internal/common.h" -#include - -DEFINE_LIST_OF(tx_history, OSSL_ACKM_TX_PKT); - -/* - * TX Packet History - * ***************** - * - * The TX Packet History object tracks information about packets which have been - * sent for which we later expect to receive an ACK. It is essentially a simple - * database keeping a list of packet information structures in packet number - * order which can also be looked up directly by packet number. - * - * We currently only allow packets to be appended to the list (i.e. the packet - * numbers of the packets appended to the list must monotonically increase), as - * we should not currently need more general functionality such as a sorted list - * insert. - */ -struct tx_pkt_history_st { - /* A linked list of all our packets. */ - OSSL_LIST(tx_history) packets; - - /* - * Mapping from packet numbers (uint64_t) to (OSSL_ACKM_TX_PKT *) - * - * Invariant: A packet is in this map if and only if it is in the linked - * list. - */ - LHASH_OF(OSSL_ACKM_TX_PKT) *map; - - /* - * The lowest packet number which may currently be added to the history list - * (inclusive). We do not allow packet numbers to be added to the history - * list non-monotonically, so packet numbers must be greater than or equal - * to this value. - */ - uint64_t watermark; - - /* - * Packet number of the highest packet info structure we have yet appended - * to the list. This is usually one less than watermark, except when we have - * not added any packet yet. - */ - uint64_t highest_sent; -}; - -DEFINE_LHASH_OF_EX(OSSL_ACKM_TX_PKT); - -static unsigned long tx_pkt_info_hash(const OSSL_ACKM_TX_PKT *pkt) -{ - /* Using low bits of the packet number as the hash should be enough */ - return (unsigned long)pkt->pkt_num; -} - -static int tx_pkt_info_compare(const OSSL_ACKM_TX_PKT *a, - const OSSL_ACKM_TX_PKT *b) -{ - if (a->pkt_num < b->pkt_num) - return -1; - if (a->pkt_num > b->pkt_num) - return 1; - return 0; -} - -static int -tx_pkt_history_init(struct tx_pkt_history_st *h) -{ - ossl_list_tx_history_init(&h->packets); - h->watermark = 0; - h->highest_sent = 0; - - h->map = lh_OSSL_ACKM_TX_PKT_new(tx_pkt_info_hash, tx_pkt_info_compare); - if (h->map == NULL) - return 0; - - return 1; -} - -static void -tx_pkt_history_destroy(struct tx_pkt_history_st *h) -{ - lh_OSSL_ACKM_TX_PKT_free(h->map); - h->map = NULL; - ossl_list_tx_history_init(&h->packets); -} - -static int -tx_pkt_history_add_actual(struct tx_pkt_history_st *h, - OSSL_ACKM_TX_PKT *pkt) -{ - OSSL_ACKM_TX_PKT *existing; - - /* - * There should not be any existing packet with this number - * in our mapping. - */ - existing = lh_OSSL_ACKM_TX_PKT_retrieve(h->map, pkt); - if (!ossl_assert(existing == NULL)) - return 0; - - /* Should not already be in a list. */ - if (!ossl_assert(ossl_list_tx_history_next(pkt) == NULL - && ossl_list_tx_history_prev(pkt) == NULL)) - return 0; - - lh_OSSL_ACKM_TX_PKT_insert(h->map, pkt); - - ossl_list_tx_history_insert_tail(&h->packets, pkt); - return 1; -} - -/* Adds a packet information structure to the history list. */ -static int -tx_pkt_history_add(struct tx_pkt_history_st *h, - OSSL_ACKM_TX_PKT *pkt) -{ - if (!ossl_assert(pkt->pkt_num >= h->watermark)) - return 0; - - if (tx_pkt_history_add_actual(h, pkt) < 1) - return 0; - - h->watermark = pkt->pkt_num + 1; - h->highest_sent = pkt->pkt_num; - return 1; -} - -/* Retrieve a packet information structure by packet number. */ -static OSSL_ACKM_TX_PKT * -tx_pkt_history_by_pkt_num(struct tx_pkt_history_st *h, uint64_t pkt_num) -{ - OSSL_ACKM_TX_PKT key; - - key.pkt_num = pkt_num; - - return lh_OSSL_ACKM_TX_PKT_retrieve(h->map, &key); -} - -/* Remove a packet information structure from the history log. */ -static int -tx_pkt_history_remove(struct tx_pkt_history_st *h, uint64_t pkt_num) -{ - OSSL_ACKM_TX_PKT key, *pkt; - key.pkt_num = pkt_num; - - pkt = tx_pkt_history_by_pkt_num(h, pkt_num); - if (pkt == NULL) - return 0; - - ossl_list_tx_history_remove(&h->packets, pkt); - lh_OSSL_ACKM_TX_PKT_delete(h->map, &key); - return 1; -} - -/* - * RX Packet Number Tracking - * ************************* - * - * **Background.** The RX side of the ACK manager must track packets we have - * received for which we have to generate ACK frames. Broadly, this means we - * store a set of packet numbers which we have received but which we do not know - * for a fact that the transmitter knows we have received. - * - * This must handle various situations: - * - * 1. We receive a packet but have not sent an ACK yet, so the transmitter - * does not know whether we have received it or not yet. - * - * 2. We receive a packet and send an ACK which is lost. We do not - * immediately know that the ACK was lost and the transmitter does not know - * that we have received the packet. - * - * 3. We receive a packet and send an ACK which is received by the - * transmitter. The transmitter does not immediately respond with an ACK, - * or responds with an ACK which is lost. The transmitter knows that we - * have received the packet, but we do not know for sure that it knows, - * because the ACK we sent could have been lost. - * - * 4. We receive a packet and send an ACK which is received by the - * transmitter. The transmitter subsequently sends us an ACK which confirms - * its receipt of the ACK we sent, and we successfully receive that ACK, so - * we know that the transmitter knows, that we received the original - * packet. - * - * Only when we reach case (4) are we relieved of any need to track a given - * packet number we have received, because only in this case do we know for sure - * that the peer knows we have received the packet. Having reached case (4) we - * will never again need to generate an ACK containing the PN in question, but - * until we reach that point, we must keep track of the PN as not having been - * provably ACKed, as we may have to keep generating ACKs for the given PN not - * just until the transmitter receives one, but until we know that it has - * received one. This will be referred to herein as "provably ACKed". - * - * **Duplicate handling.** The above discusses the case where we have received a - * packet with a given PN but are at best unsure whether the sender knows we - * have received it or not. However, we must also handle the case where we have - * yet to receive a packet with a given PN in the first place. The reason for - * this is because of the requirement expressed by RFC 9000 s. 12.3: - * - * "A receiver MUST discard a newly unprotected packet unless it is certain - * that it has not processed another packet with the same packet number from - * the same packet number space." - * - * We must ensure we never process a duplicate PN. As such, each possible PN we - * can receive must exist in one of the following logical states: - * - * - We have never processed this PN before - * (so if we receive such a PN, it can be processed) - * - * - We have processed this PN but it has not yet been provably ACKed - * (and should therefore be in any future ACK frame generated; - * if we receive such a PN again, it must be ignored) - * - * - We have processed this PN and it has been provably ACKed - * (if we receive such a PN again, it must be ignored) - * - * However, if we were to track this state for every PN ever used in the history - * of a connection, the amount of state required would increase unboundedly as - * the connection goes on (for example, we would have to store a set of every PN - * ever received.) - * - * RFC 9000 s. 12.3 continues: - * - * "Endpoints that track all individual packets for the purposes of detecting - * duplicates are at risk of accumulating excessive state. The data required - * for detecting duplicates can be limited by maintaining a minimum packet - * number below which all packets are immediately dropped." - * - * Moreover, RFC 9000 s. 13.2.3 states that: - * - * "A receiver MUST retain an ACK Range unless it can ensure that it will not - * subsequently accept packets with numbers in that range. Maintaining a - * minimum packet number that increases as ranges are discarded is one way to - * achieve this with minimal state." - * - * This touches on a subtlety of the original requirement quoted above: the - * receiver MUST discard a packet unless it is certain that it has not processed - * another packet with the same PN. However, this does not forbid the receiver - * from also discarding some PNs even though it has not yet processed them. In - * other words, implementations must be conservative and err in the direction of - * assuming a packet is a duplicate, but it is acceptable for this to come at - * the cost of falsely identifying some packets as duplicates. - * - * This allows us to bound the amount of state we must keep, and we adopt the - * suggested strategy quoted above to do so. We define a watermark PN below - * which all PNs are in the same state. This watermark is only ever increased. - * Thus the PNs the state for which needs to be explicitly tracked is limited to - * only a small number of recent PNs, and all older PNs have an assumed state. - * - * Any given PN thus falls into one of the following states: - * - * - (A) The PN is above the watermark but we have not yet received it. - * - * If we receive such a PN, we should process it and record the PN as - * received. - * - * - (B) The PN is above the watermark and we have received it. - * - * The PN should be included in any future ACK frame we generate. - * If we receive such a PN again, we should ignore it. - * - * - (C) The PN is below the watermark. - * - * We do not know whether a packet with the given PN was received or - * not. To be safe, if we receive such a packet, it is not processed. - * - * Note that state (C) corresponds to both "we have processed this PN and it has - * been provably ACKed" logical state and a subset of the PNs in the "we have - * never processed this PN before" logical state (namely all PNs which were lost - * and never received, but which are not recent enough to be above the - * watermark). The reason we can merge these states and avoid tracking states - * for the PNs in this state is because the provably ACKed and never-received - * states are functionally identical in terms of how we need to handle them: we - * don't need to do anything for PNs in either of these states, so we don't have - * to care about PNs in this state nor do we have to care about distinguishing - * the two states for a given PN. - * - * Note that under this scheme provably ACKed PNs are by definition always below - * the watermark; therefore, it follows that when a PN becomes provably ACKed, - * the watermark must be immediately increased to exceed it (otherwise we would - * keep reporting it in future ACK frames). - * - * This is in line with RFC 9000 s. 13.2.4's suggested strategy on when - * to advance the watermark: - * - * "When a packet containing an ACK frame is sent, the Largest Acknowledged - * field in that frame can be saved. When a packet containing an ACK frame is - * acknowledged, the receiver can stop acknowledging packets less than or - * equal to the Largest Acknowledged field in the sent ACK frame." - * - * This is where our scheme's false positives arise. When a packet containing an - * ACK frame is itself ACK'd, PNs referenced in that ACK frame become provably - * acked, and the watermark is bumped accordingly. However, the Largest - * Acknowledged field does not imply that all lower PNs have been received, - * because there may be gaps expressed in the ranges of PNs expressed by that - * and previous ACK frames. Thus, some unreceived PNs may be moved below the - * watermark, and we may subsequently reject those PNs as possibly being - * duplicates even though we have not actually received those PNs. Since we bump - * the watermark when a PN becomes provably ACKed, it follows that an unreceived - * PN falls below the watermark (and thus becomes a false positive for the - * purposes of duplicate detection) when a higher-numbered PN becomes provably - * ACKed. - * - * Thus, when PN n becomes provably acked, any unreceived PNs in the range [0, - * n) will no longer be processed. Although datagrams may be reordered in the - * network, a PN we receive can only become provably ACKed after our own - * subsequently generated ACK frame is sent in a future TX packet, and then we - * receive another RX PN acknowledging that TX packet. This means that a given RX - * PN can only become provably ACKed at least 1 RTT after it is received; it is - * unlikely that any reordered datagrams will still be "in the network" (and not - * lost) by this time. If this does occur for whatever reason and a late PN is - * received, the packet will be discarded unprocessed and the PN is simply - * handled as though lost (a "written off" PN). - * - * **Data structure.** Our state for the RX handling side of the ACK manager, as - * discussed above, mainly comprises: - * - * a) a logical set of PNs, and - * b) a monotonically increasing PN counter (the watermark). - * - * For (a), we define a data structure which stores a logical set of PNs, which - * we use to keep track of which PNs we have received but which have not yet - * been provably ACKed, and thus will later need to generate an ACK frame for. - * - * The correspondence with the logical states discussed above is as follows. A - * PN is in state (C) if it is below the watermark; otherwise it is in state (B) - * if it is in the logical set of PNs, and in state (A) otherwise. - * - * Note that PNs are only removed from the PN set (when they become provably - * ACKed or written off) by virtue of advancement of the watermark. Removing PNs - * from the PN set any other way would be ambiguous as it would be - * indistinguishable from a PN we have not yet received and risk us processing a - * duplicate packet. In other words, for a given PN: - * - * - State (A) can transition to state (B) or (C) - * - State (B) can transition to state (C) only - * - State (C) is the terminal state - * - * We can query the logical set data structure for PNs which have been received - * but which have not been provably ACKed when we want to generate ACK frames. - * Since ACK frames can be lost and/or we might not know that the peer has - * successfully received them, we might generate multiple ACK frames covering a - * given PN until that PN becomes provably ACKed and we finally remove it from - * our set (by bumping the watermark) as no longer being our concern. - * - * The data structure used is the UINT_SET structure defined in uint_set.h, - * which is used as a PN set. We use the following operations of the structure: - * - * Insert Range: Used when we receive a new PN. - * - * Remove Range: Used when bumping the watermark. - * - * Query: Used to determine if a PN is in the set. - * - * **Possible duplicates.** A PN is considered a possible duplicate when either: - * - * a) its PN is already in the PN set (i.e. has already been received), or - * b) its PN is below the watermark (i.e. was provably ACKed or written off). - * - * A packet with a given PN is considered 'processable' when that PN is not - * considered a possible duplicate (see ossl_ackm_is_rx_pn_processable). - * - * **TX/RX interaction.** The watermark is bumped whenever an RX packet becomes - * provably ACKed. This occurs when an ACK frame is received by the TX side of - * the ACK manager; thus, there is necessary interaction between the TX and RX - * sides of the ACK manager. - * - * This is implemented as follows. When a packet is queued as sent in the TX - * side of the ACK manager, it may optionally have a Largest Acked value set on - * it. The user of the ACK manager should do this if the packet being - * transmitted contains an ACK frame, by setting the field to the Largest Acked - * field of that frame. Otherwise, this field should be set to QUIC_PN_INVALID. - * When a TX packet is eventually acknowledged which has this field set, it is - * used to update the state of the RX side of the ACK manager by bumping the - * watermark accordingly. - */ -struct rx_pkt_history_st { - UINT_SET set; - - /* - * Invariant: PNs below this are not in the set. - * Invariant: This is monotonic and only ever increases. - */ - QUIC_PN watermark; -}; - -static int rx_pkt_history_bump_watermark(struct rx_pkt_history_st *h, - QUIC_PN watermark); - -static void rx_pkt_history_init(struct rx_pkt_history_st *h) -{ - ossl_uint_set_init(&h->set); - h->watermark = 0; -} - -static void rx_pkt_history_destroy(struct rx_pkt_history_st *h) -{ - ossl_uint_set_destroy(&h->set); -} - -/* - * Limit the number of ACK ranges we store to prevent resource consumption DoS - * attacks. - */ -#define MAX_RX_ACK_RANGES 32 - -static void rx_pkt_history_trim_range_count(struct rx_pkt_history_st *h) -{ - QUIC_PN highest = QUIC_PN_INVALID; - - while (ossl_list_uint_set_num(&h->set) > MAX_RX_ACK_RANGES) { - UINT_RANGE r = ossl_list_uint_set_head(&h->set)->range; - - highest = (highest == QUIC_PN_INVALID) - ? r.end : ossl_quic_pn_max(highest, r.end); - - ossl_uint_set_remove(&h->set, &r); - } - - /* - * Bump watermark to cover all PNs we removed to avoid accidental - * reprocessing of packets. - */ - if (highest != QUIC_PN_INVALID) - rx_pkt_history_bump_watermark(h, highest + 1); -} - -static int rx_pkt_history_add_pn(struct rx_pkt_history_st *h, - QUIC_PN pn) -{ - UINT_RANGE r; - - r.start = pn; - r.end = pn; - - if (pn < h->watermark) - return 1; /* consider this a success case */ - - if (ossl_uint_set_insert(&h->set, &r) != 1) - return 0; - - rx_pkt_history_trim_range_count(h); - return 1; -} - -static int rx_pkt_history_bump_watermark(struct rx_pkt_history_st *h, - QUIC_PN watermark) -{ - UINT_RANGE r; - - if (watermark <= h->watermark) - return 1; - - /* Remove existing PNs below the watermark. */ - r.start = 0; - r.end = watermark - 1; - if (ossl_uint_set_remove(&h->set, &r) != 1) - return 0; - - h->watermark = watermark; - return 1; -} - -/* - * ACK Manager Implementation - * ************************** - * Implementation of the ACK manager proper. - */ - -/* Constants used by the ACK manager; see RFC 9002. */ -#define K_GRANULARITY (1 * OSSL_TIME_MS) -#define K_PKT_THRESHOLD 3 -#define K_TIME_THRESHOLD_NUM 9 -#define K_TIME_THRESHOLD_DEN 8 - -/* The maximum number of times we allow PTO to be doubled. */ -#define MAX_PTO_COUNT 16 - -/* Default maximum amount of time to leave an ACK-eliciting packet un-ACK'd. */ -#define DEFAULT_TX_MAX_ACK_DELAY ossl_ms2time(QUIC_DEFAULT_MAX_ACK_DELAY) - -struct ossl_ackm_st { - /* Our list of transmitted packets. Corresponds to RFC 9002 sent_packets. */ - struct tx_pkt_history_st tx_history[QUIC_PN_SPACE_NUM]; - - /* Our list of received PNs which are not yet provably acked. */ - struct rx_pkt_history_st rx_history[QUIC_PN_SPACE_NUM]; - - /* Polymorphic dependencies that we consume. */ - OSSL_TIME (*now)(void *arg); - void *now_arg; - OSSL_STATM *statm; - const OSSL_CC_METHOD *cc_method; - OSSL_CC_DATA *cc_data; - - /* RFC 9002 variables. */ - uint32_t pto_count; - QUIC_PN largest_acked_pkt[QUIC_PN_SPACE_NUM]; - OSSL_TIME time_of_last_ack_eliciting_pkt[QUIC_PN_SPACE_NUM]; - OSSL_TIME loss_time[QUIC_PN_SPACE_NUM]; - OSSL_TIME loss_detection_deadline; - - /* Lowest PN which is still not known to be ACKed. */ - QUIC_PN lowest_unacked_pkt[QUIC_PN_SPACE_NUM]; - - /* Time at which we got our first RTT sample, or 0. */ - OSSL_TIME first_rtt_sample; - - /* - * A packet's num_bytes are added to this if it is inflight, - * and removed again once ack'd/lost/discarded. - */ - uint64_t bytes_in_flight; - - /* - * A packet's num_bytes are added to this if it is both inflight and - * ack-eliciting, and removed again once ack'd/lost/discarded. - */ - uint64_t ack_eliciting_bytes_in_flight[QUIC_PN_SPACE_NUM]; - - /* Count of ECN-CE events. */ - uint64_t peer_ecnce[QUIC_PN_SPACE_NUM]; - - /* Set to 1 when the handshake is confirmed. */ - char handshake_confirmed; - - /* Set to 1 when the peer has completed address validation. */ - char peer_completed_addr_validation; - - /* Set to 1 when a PN space has been discarded. */ - char discarded[QUIC_PN_SPACE_NUM]; - - /* Set to 1 when we think an ACK frame should be generated. */ - char rx_ack_desired[QUIC_PN_SPACE_NUM]; - - /* Set to 1 if an ACK frame has ever been generated. */ - char rx_ack_generated[QUIC_PN_SPACE_NUM]; - - /* Probe request counts for reporting to the user. */ - OSSL_ACKM_PROBE_INFO pending_probe; - - /* Generated ACK frames for each PN space. */ - OSSL_QUIC_FRAME_ACK ack[QUIC_PN_SPACE_NUM]; - OSSL_QUIC_ACK_RANGE ack_ranges[QUIC_PN_SPACE_NUM][MAX_RX_ACK_RANGES]; - - /* Other RX state. */ - /* Largest PN we have RX'd. */ - QUIC_PN rx_largest_pn[QUIC_PN_SPACE_NUM]; - - /* Time at which the PN in rx_largest_pn was RX'd. */ - OSSL_TIME rx_largest_time[QUIC_PN_SPACE_NUM]; - - /* - * ECN event counters. Each time we receive a packet with a given ECN label, - * the corresponding ECN counter here is incremented. - */ - uint64_t rx_ect0[QUIC_PN_SPACE_NUM]; - uint64_t rx_ect1[QUIC_PN_SPACE_NUM]; - uint64_t rx_ecnce[QUIC_PN_SPACE_NUM]; - - /* - * Number of ACK-eliciting packets since last ACK. We use this to defer - * emitting ACK frames until a threshold number of ACK-eliciting packets - * have been received. - */ - uint32_t rx_ack_eliciting_pkts_since_last_ack[QUIC_PN_SPACE_NUM]; - - /* - * The ACK frame coalescing deadline at which we should flush any unsent ACK - * frames. - */ - OSSL_TIME rx_ack_flush_deadline[QUIC_PN_SPACE_NUM]; - - /* - * The RX maximum ACK delay (the maximum amount of time our peer might - * wait to send us an ACK after receiving an ACK-eliciting packet). - */ - OSSL_TIME rx_max_ack_delay; - - /* - * The TX maximum ACK delay (the maximum amount of time we allow ourselves - * to wait before generating an ACK after receiving an ACK-eliciting - * packet). - */ - OSSL_TIME tx_max_ack_delay; - - /* Callbacks for deadline updates. */ - void (*loss_detection_deadline_cb)(OSSL_TIME deadline, void *arg); - void *loss_detection_deadline_cb_arg; - - void (*ack_deadline_cb)(OSSL_TIME deadline, int pkt_space, void *arg); - void *ack_deadline_cb_arg; -}; - -static ossl_inline uint32_t min_u32(uint32_t x, uint32_t y) -{ - return x < y ? x : y; -} - -/* - * Get TX history for a given packet number space. Must not have been - * discarded. - */ -static struct tx_pkt_history_st *get_tx_history(OSSL_ACKM *ackm, int pkt_space) -{ - assert(!ackm->discarded[pkt_space]); - - return &ackm->tx_history[pkt_space]; -} - -/* - * Get RX history for a given packet number space. Must not have been - * discarded. - */ -static struct rx_pkt_history_st *get_rx_history(OSSL_ACKM *ackm, int pkt_space) -{ - assert(!ackm->discarded[pkt_space]); - - return &ackm->rx_history[pkt_space]; -} - -/* Does the newly-acknowledged list contain any ack-eliciting packet? */ -static int ack_includes_ack_eliciting(OSSL_ACKM_TX_PKT *pkt) -{ - for (; pkt != NULL; pkt = pkt->anext) - if (pkt->is_ack_eliciting) - return 1; - - return 0; -} - -/* Return number of ACK-eliciting bytes in flight across all PN spaces. */ -static uint64_t ackm_ack_eliciting_bytes_in_flight(OSSL_ACKM *ackm) -{ - int i; - uint64_t total = 0; - - for (i = 0; i < QUIC_PN_SPACE_NUM; ++i) - total += ackm->ack_eliciting_bytes_in_flight[i]; - - return total; -} - -/* Return 1 if the range contains the given PN. */ -static int range_contains(const OSSL_QUIC_ACK_RANGE *range, QUIC_PN pn) -{ - return pn >= range->start && pn <= range->end; -} - -/* - * Given a logical representation of an ACK frame 'ack', create a singly-linked - * list of the newly ACK'd frames; that is, of frames which are matched by the - * list of PN ranges contained in the ACK frame. The packet structures in the - * list returned are removed from the TX history list. Returns a pointer to the - * list head (or NULL) if empty. - */ -static OSSL_ACKM_TX_PKT *ackm_detect_and_remove_newly_acked_pkts(OSSL_ACKM *ackm, - const OSSL_QUIC_FRAME_ACK *ack, - int pkt_space) -{ - OSSL_ACKM_TX_PKT *acked_pkts = NULL, **fixup = &acked_pkts, *pkt, *pprev; - struct tx_pkt_history_st *h; - size_t ridx = 0; - - assert(ack->num_ack_ranges > 0); - - /* - * Our history list is a list of packets sorted in ascending order - * by packet number. - * - * ack->ack_ranges is a list of packet number ranges in descending order. - * - * Walk through our history list from the end in order to efficiently detect - * membership in the specified ack ranges. As an optimization, we use our - * hashtable to try and skip to the first matching packet. This may fail if - * the ACK ranges given include nonexistent packets. - */ - h = get_tx_history(ackm, pkt_space); - - pkt = tx_pkt_history_by_pkt_num(h, ack->ack_ranges[0].end); - if (pkt == NULL) - pkt = ossl_list_tx_history_tail(&h->packets); - - for (; pkt != NULL; pkt = pprev) { - /* - * Save prev value as it will be zeroed if we remove the packet from the - * history list below. - */ - pprev = ossl_list_tx_history_prev(pkt); - - for (;; ++ridx) { - if (ridx >= ack->num_ack_ranges) { - /* - * We have exhausted all ranges so stop here, even if there are - * more packets to look at. - */ - goto stop; - } - - if (range_contains(&ack->ack_ranges[ridx], pkt->pkt_num)) { - /* We have matched this range. */ - tx_pkt_history_remove(h, pkt->pkt_num); - - *fixup = pkt; - fixup = &pkt->anext; - *fixup = NULL; - break; - } else if (pkt->pkt_num > ack->ack_ranges[ridx].end) { - /* - * We have not reached this range yet in our list, so do not - * advance ridx. - */ - break; - } else { - /* - * We have moved beyond this range, so advance to the next range - * and try matching again. - */ - assert(pkt->pkt_num < ack->ack_ranges[ridx].start); - continue; - } - } - } -stop: - - return acked_pkts; -} - -/* - * Create a singly-linked list of newly detected-lost packets in the given - * packet number space. Returns the head of the list or NULL if no packets were - * detected lost. The packets in the list are removed from the TX history list. - */ -static OSSL_ACKM_TX_PKT *ackm_detect_and_remove_lost_pkts(OSSL_ACKM *ackm, - int pkt_space) -{ - OSSL_ACKM_TX_PKT *lost_pkts = NULL, **fixup = &lost_pkts, *pkt, *pnext; - OSSL_TIME loss_delay, lost_send_time, now; - OSSL_RTT_INFO rtt; - struct tx_pkt_history_st *h; - - assert(ackm->largest_acked_pkt[pkt_space] != QUIC_PN_INVALID); - - ossl_statm_get_rtt_info(ackm->statm, &rtt); - - ackm->loss_time[pkt_space] = ossl_time_zero(); - - loss_delay = ossl_time_multiply(ossl_time_max(rtt.latest_rtt, - rtt.smoothed_rtt), - K_TIME_THRESHOLD_NUM); - loss_delay = ossl_time_divide(loss_delay, K_TIME_THRESHOLD_DEN); - - /* Minimum time of K_GRANULARITY before packets are deemed lost. */ - loss_delay = ossl_time_max(loss_delay, ossl_ticks2time(K_GRANULARITY)); - - /* Packets sent before this time are deemed lost. */ - now = ackm->now(ackm->now_arg); - lost_send_time = ossl_time_subtract(now, loss_delay); - - h = get_tx_history(ackm, pkt_space); - pkt = ossl_list_tx_history_head(&h->packets); - - for (; pkt != NULL; pkt = pnext) { - assert(pkt_space == pkt->pkt_space); - - /* - * Save prev value as it will be zeroed if we remove the packet from the - * history list below. - */ - pnext = ossl_list_tx_history_next(pkt); - - if (pkt->pkt_num > ackm->largest_acked_pkt[pkt_space]) - continue; - - /* - * Mark packet as lost, or set time when it should be marked. - */ - if (ossl_time_compare(pkt->time, lost_send_time) <= 0 - || ackm->largest_acked_pkt[pkt_space] - >= pkt->pkt_num + K_PKT_THRESHOLD) { - tx_pkt_history_remove(h, pkt->pkt_num); - - *fixup = pkt; - fixup = &pkt->lnext; - *fixup = NULL; - } else { - if (ossl_time_is_zero(ackm->loss_time[pkt_space])) - ackm->loss_time[pkt_space] = - ossl_time_add(pkt->time, loss_delay); - else - ackm->loss_time[pkt_space] = - ossl_time_min(ackm->loss_time[pkt_space], - ossl_time_add(pkt->time, loss_delay)); - } - } - - return lost_pkts; -} - -static OSSL_TIME ackm_get_loss_time_and_space(OSSL_ACKM *ackm, int *pspace) -{ - OSSL_TIME time = ackm->loss_time[QUIC_PN_SPACE_INITIAL]; - int i, space = QUIC_PN_SPACE_INITIAL; - - for (i = space + 1; i < QUIC_PN_SPACE_NUM; ++i) - if (ossl_time_is_zero(time) - || ossl_time_compare(ackm->loss_time[i], time) == -1) { - time = ackm->loss_time[i]; - space = i; - } - - *pspace = space; - return time; -} - -static OSSL_TIME ackm_get_pto_time_and_space(OSSL_ACKM *ackm, int *space) -{ - OSSL_RTT_INFO rtt; - OSSL_TIME duration; - OSSL_TIME pto_timeout = ossl_time_infinite(), t; - int pto_space = QUIC_PN_SPACE_INITIAL, i; - - ossl_statm_get_rtt_info(ackm->statm, &rtt); - - duration - = ossl_time_add(rtt.smoothed_rtt, - ossl_time_max(ossl_time_multiply(rtt.rtt_variance, 4), - ossl_ticks2time(K_GRANULARITY))); - - duration - = ossl_time_multiply(duration, - (uint64_t)1 << min_u32(ackm->pto_count, - MAX_PTO_COUNT)); - - /* Anti-deadlock PTO starts from the current time. */ - if (ackm_ack_eliciting_bytes_in_flight(ackm) == 0) { - assert(!ackm->peer_completed_addr_validation); - - *space = ackm->discarded[QUIC_PN_SPACE_INITIAL] - ? QUIC_PN_SPACE_HANDSHAKE - : QUIC_PN_SPACE_INITIAL; - return ossl_time_add(ackm->now(ackm->now_arg), duration); - } - - for (i = QUIC_PN_SPACE_INITIAL; i < QUIC_PN_SPACE_NUM; ++i) { - if (ackm->ack_eliciting_bytes_in_flight[i] == 0) - continue; - - if (i == QUIC_PN_SPACE_APP) { - /* Skip application data until handshake confirmed. */ - if (!ackm->handshake_confirmed) - break; - - /* Include max_ack_delay and backoff for app data. */ - if (!ossl_time_is_infinite(ackm->rx_max_ack_delay)) { - uint64_t factor - = (uint64_t)1 << min_u32(ackm->pto_count, MAX_PTO_COUNT); - - duration - = ossl_time_add(duration, - ossl_time_multiply(ackm->rx_max_ack_delay, - factor)); - } - } - - t = ossl_time_add(ackm->time_of_last_ack_eliciting_pkt[i], duration); - if (ossl_time_compare(t, pto_timeout) < 0) { - pto_timeout = t; - pto_space = i; - } - } - - *space = pto_space; - return pto_timeout; -} - -static void ackm_set_loss_detection_timer_actual(OSSL_ACKM *ackm, - OSSL_TIME deadline) -{ - ackm->loss_detection_deadline = deadline; - - if (ackm->loss_detection_deadline_cb != NULL) - ackm->loss_detection_deadline_cb(deadline, - ackm->loss_detection_deadline_cb_arg); -} - -static int ackm_set_loss_detection_timer(OSSL_ACKM *ackm) -{ - int space; - OSSL_TIME earliest_loss_time, timeout; - - earliest_loss_time = ackm_get_loss_time_and_space(ackm, &space); - if (!ossl_time_is_zero(earliest_loss_time)) { - /* Time threshold loss detection. */ - ackm_set_loss_detection_timer_actual(ackm, earliest_loss_time); - return 1; - } - - if (ackm_ack_eliciting_bytes_in_flight(ackm) == 0 - && ackm->peer_completed_addr_validation) { - /* - * Nothing to detect lost, so no timer is set. However, the client - * needs to arm the timer if the server might be blocked by the - * anti-amplification limit. - */ - ackm_set_loss_detection_timer_actual(ackm, ossl_time_zero()); - return 1; - } - - timeout = ackm_get_pto_time_and_space(ackm, &space); - ackm_set_loss_detection_timer_actual(ackm, timeout); - return 1; -} - -static int ackm_in_persistent_congestion(OSSL_ACKM *ackm, - const OSSL_ACKM_TX_PKT *lpkt) -{ - /* TODO(QUIC FUTURE): Persistent congestion not currently implemented. */ - return 0; -} - -static void ackm_on_pkts_lost(OSSL_ACKM *ackm, int pkt_space, - const OSSL_ACKM_TX_PKT *lpkt, int pseudo) -{ - const OSSL_ACKM_TX_PKT *p, *pnext; - OSSL_RTT_INFO rtt; - QUIC_PN largest_pn_lost = 0; - OSSL_CC_LOSS_INFO loss_info = {0}; - uint32_t flags = 0; - - for (p = lpkt; p != NULL; p = pnext) { - pnext = p->lnext; - - if (p->is_inflight) { - ackm->bytes_in_flight -= p->num_bytes; - if (p->is_ack_eliciting) - ackm->ack_eliciting_bytes_in_flight[p->pkt_space] - -= p->num_bytes; - - if (p->pkt_num > largest_pn_lost) - largest_pn_lost = p->pkt_num; - - if (!pseudo) { - /* - * If this is pseudo-loss (e.g. during connection retry) we do not - * inform the CC as it is not a real loss and not reflective of - * network conditions. - */ - loss_info.tx_time = p->time; - loss_info.tx_size = p->num_bytes; - - ackm->cc_method->on_data_lost(ackm->cc_data, &loss_info); - } - } - - p->on_lost(p->cb_arg); - } - - /* - * Persistent congestion can only be considered if we have gotten at least - * one RTT sample. - */ - ossl_statm_get_rtt_info(ackm->statm, &rtt); - if (!ossl_time_is_zero(ackm->first_rtt_sample) - && ackm_in_persistent_congestion(ackm, lpkt)) - flags |= OSSL_CC_LOST_FLAG_PERSISTENT_CONGESTION; - - ackm->cc_method->on_data_lost_finished(ackm->cc_data, flags); -} - -static void ackm_on_pkts_acked(OSSL_ACKM *ackm, const OSSL_ACKM_TX_PKT *apkt) -{ - const OSSL_ACKM_TX_PKT *anext; - QUIC_PN last_pn_acked = 0; - OSSL_CC_ACK_INFO ainfo = {0}; - - for (; apkt != NULL; apkt = anext) { - if (apkt->is_inflight) { - ackm->bytes_in_flight -= apkt->num_bytes; - if (apkt->is_ack_eliciting) - ackm->ack_eliciting_bytes_in_flight[apkt->pkt_space] - -= apkt->num_bytes; - - if (apkt->pkt_num > last_pn_acked) - last_pn_acked = apkt->pkt_num; - - if (apkt->largest_acked != QUIC_PN_INVALID) - /* - * This can fail, but it is monotonic; worst case we try again - * next time. - */ - rx_pkt_history_bump_watermark(get_rx_history(ackm, - apkt->pkt_space), - apkt->largest_acked + 1); - } - - ainfo.tx_time = apkt->time; - ainfo.tx_size = apkt->num_bytes; - - anext = apkt->anext; - apkt->on_acked(apkt->cb_arg); /* may free apkt */ - - if (apkt->is_inflight) - ackm->cc_method->on_data_acked(ackm->cc_data, &ainfo); - } -} - -OSSL_ACKM *ossl_ackm_new(OSSL_TIME (*now)(void *arg), - void *now_arg, - OSSL_STATM *statm, - const OSSL_CC_METHOD *cc_method, - OSSL_CC_DATA *cc_data) -{ - OSSL_ACKM *ackm; - int i; - - ackm = OPENSSL_zalloc(sizeof(OSSL_ACKM)); - if (ackm == NULL) - return NULL; - - for (i = 0; i < (int)OSSL_NELEM(ackm->tx_history); ++i) { - ackm->largest_acked_pkt[i] = QUIC_PN_INVALID; - ackm->rx_ack_flush_deadline[i] = ossl_time_infinite(); - if (tx_pkt_history_init(&ackm->tx_history[i]) < 1) - goto err; - } - - for (i = 0; i < (int)OSSL_NELEM(ackm->rx_history); ++i) - rx_pkt_history_init(&ackm->rx_history[i]); - - ackm->now = now; - ackm->now_arg = now_arg; - ackm->statm = statm; - ackm->cc_method = cc_method; - ackm->cc_data = cc_data; - - ackm->rx_max_ack_delay = ossl_ms2time(QUIC_DEFAULT_MAX_ACK_DELAY); - ackm->tx_max_ack_delay = DEFAULT_TX_MAX_ACK_DELAY; - - return ackm; - -err: - while (--i >= 0) - tx_pkt_history_destroy(&ackm->tx_history[i]); - - OPENSSL_free(ackm); - return NULL; -} - -void ossl_ackm_free(OSSL_ACKM *ackm) -{ - size_t i; - - if (ackm == NULL) - return; - - for (i = 0; i < OSSL_NELEM(ackm->tx_history); ++i) - if (!ackm->discarded[i]) { - tx_pkt_history_destroy(&ackm->tx_history[i]); - rx_pkt_history_destroy(&ackm->rx_history[i]); - } - - OPENSSL_free(ackm); -} - -int ossl_ackm_on_tx_packet(OSSL_ACKM *ackm, OSSL_ACKM_TX_PKT *pkt) -{ - struct tx_pkt_history_st *h = get_tx_history(ackm, pkt->pkt_space); - - /* Time must be set and not move backwards. */ - if (ossl_time_is_zero(pkt->time) - || ossl_time_compare(ackm->time_of_last_ack_eliciting_pkt[pkt->pkt_space], - pkt->time) > 0) - return 0; - - /* Must have non-zero number of bytes. */ - if (pkt->num_bytes == 0) - return 0; - - /* Does not make any sense for a non-in-flight packet to be ACK-eliciting. */ - if (!pkt->is_inflight && pkt->is_ack_eliciting) - return 0; - - if (tx_pkt_history_add(h, pkt) == 0) - return 0; - - if (pkt->is_inflight) { - if (pkt->is_ack_eliciting) { - ackm->time_of_last_ack_eliciting_pkt[pkt->pkt_space] = pkt->time; - ackm->ack_eliciting_bytes_in_flight[pkt->pkt_space] - += pkt->num_bytes; - } - - ackm->bytes_in_flight += pkt->num_bytes; - ackm_set_loss_detection_timer(ackm); - - ackm->cc_method->on_data_sent(ackm->cc_data, pkt->num_bytes); - } - - return 1; -} - -int ossl_ackm_on_rx_datagram(OSSL_ACKM *ackm, size_t num_bytes) -{ - /* No-op on the client. */ - return 1; -} - -static void ackm_process_ecn(OSSL_ACKM *ackm, const OSSL_QUIC_FRAME_ACK *ack, - int pkt_space) -{ - struct tx_pkt_history_st *h; - OSSL_ACKM_TX_PKT *pkt; - OSSL_CC_ECN_INFO ecn_info = {0}; - - /* - * If the ECN-CE counter reported by the peer has increased, this could - * be a new congestion event. - */ - if (ack->ecnce > ackm->peer_ecnce[pkt_space]) { - ackm->peer_ecnce[pkt_space] = ack->ecnce; - - h = get_tx_history(ackm, pkt_space); - pkt = tx_pkt_history_by_pkt_num(h, ack->ack_ranges[0].end); - if (pkt == NULL) - return; - - ecn_info.largest_acked_time = pkt->time; - ackm->cc_method->on_ecn(ackm->cc_data, &ecn_info); - } -} - -int ossl_ackm_on_rx_ack_frame(OSSL_ACKM *ackm, const OSSL_QUIC_FRAME_ACK *ack, - int pkt_space, OSSL_TIME rx_time) -{ - OSSL_ACKM_TX_PKT *na_pkts, *lost_pkts; - int must_set_timer = 0; - - if (ackm->largest_acked_pkt[pkt_space] == QUIC_PN_INVALID) - ackm->largest_acked_pkt[pkt_space] = ack->ack_ranges[0].end; - else - ackm->largest_acked_pkt[pkt_space] - = ossl_quic_pn_max(ackm->largest_acked_pkt[pkt_space], - ack->ack_ranges[0].end); - - /* - * If we get an ACK in the handshake space, address validation is completed. - * Make sure we update the timer, even if no packets were ACK'd. - */ - if (!ackm->peer_completed_addr_validation - && pkt_space == QUIC_PN_SPACE_HANDSHAKE) { - ackm->peer_completed_addr_validation = 1; - must_set_timer = 1; - } - - /* - * Find packets that are newly acknowledged and remove them from the list. - */ - na_pkts = ackm_detect_and_remove_newly_acked_pkts(ackm, ack, pkt_space); - if (na_pkts == NULL) { - if (must_set_timer) - ackm_set_loss_detection_timer(ackm); - - return 1; - } - - /* - * Update the RTT if the largest acknowledged is newly acked and at least - * one ACK-eliciting packet was newly acked. - * - * First packet in the list is always the one with the largest PN. - */ - if (na_pkts->pkt_num == ack->ack_ranges[0].end && - ack_includes_ack_eliciting(na_pkts)) { - OSSL_TIME now = ackm->now(ackm->now_arg), ack_delay; - if (ossl_time_is_zero(ackm->first_rtt_sample)) - ackm->first_rtt_sample = now; - - /* Enforce maximum ACK delay. */ - ack_delay = ack->delay_time; - if (ackm->handshake_confirmed) - ack_delay = ossl_time_min(ack_delay, ackm->rx_max_ack_delay); - - ossl_statm_update_rtt(ackm->statm, ack_delay, - ossl_time_subtract(now, na_pkts->time)); - } - - /* - * Process ECN information if present. - * - * We deliberately do most ECN processing in the ACKM rather than the - * congestion controller to avoid having to give the congestion controller - * access to ACKM internal state. - */ - if (ack->ecn_present) - ackm_process_ecn(ackm, ack, pkt_space); - - /* Handle inferred loss. */ - lost_pkts = ackm_detect_and_remove_lost_pkts(ackm, pkt_space); - if (lost_pkts != NULL) - ackm_on_pkts_lost(ackm, pkt_space, lost_pkts, /*pseudo=*/0); - - ackm_on_pkts_acked(ackm, na_pkts); - - /* - * Reset pto_count unless the client is unsure if the server validated the - * client's address. - */ - if (ackm->peer_completed_addr_validation) - ackm->pto_count = 0; - - ackm_set_loss_detection_timer(ackm); - return 1; -} - -int ossl_ackm_on_pkt_space_discarded(OSSL_ACKM *ackm, int pkt_space) -{ - OSSL_ACKM_TX_PKT *pkt, *pnext; - uint64_t num_bytes_invalidated = 0; - - if (ackm->discarded[pkt_space]) - return 0; - - if (pkt_space == QUIC_PN_SPACE_HANDSHAKE) - ackm->peer_completed_addr_validation = 1; - - for (pkt = ossl_list_tx_history_head(&get_tx_history(ackm, pkt_space)->packets); - pkt != NULL; pkt = pnext) { - pnext = ossl_list_tx_history_next(pkt); - if (pkt->is_inflight) { - ackm->bytes_in_flight -= pkt->num_bytes; - num_bytes_invalidated += pkt->num_bytes; - } - - pkt->on_discarded(pkt->cb_arg); /* may free pkt */ - } - - tx_pkt_history_destroy(&ackm->tx_history[pkt_space]); - rx_pkt_history_destroy(&ackm->rx_history[pkt_space]); - - if (num_bytes_invalidated > 0) - ackm->cc_method->on_data_invalidated(ackm->cc_data, - num_bytes_invalidated); - - ackm->time_of_last_ack_eliciting_pkt[pkt_space] = ossl_time_zero(); - ackm->loss_time[pkt_space] = ossl_time_zero(); - ackm->pto_count = 0; - ackm->discarded[pkt_space] = 1; - ackm->ack_eliciting_bytes_in_flight[pkt_space] = 0; - ackm_set_loss_detection_timer(ackm); - return 1; -} - -int ossl_ackm_on_handshake_confirmed(OSSL_ACKM *ackm) -{ - ackm->handshake_confirmed = 1; - ackm->peer_completed_addr_validation = 1; - ackm_set_loss_detection_timer(ackm); - return 1; -} - -static void ackm_queue_probe_anti_deadlock_handshake(OSSL_ACKM *ackm) -{ - ++ackm->pending_probe.anti_deadlock_handshake; -} - -static void ackm_queue_probe_anti_deadlock_initial(OSSL_ACKM *ackm) -{ - ++ackm->pending_probe.anti_deadlock_initial; -} - -static void ackm_queue_probe(OSSL_ACKM *ackm, int pkt_space) -{ - /* - * TODO(QUIC FUTURE): We are allowed to send either one or two probe - * packets here. - * Determine a strategy for when we should send two probe packets. - */ - ++ackm->pending_probe.pto[pkt_space]; -} - -int ossl_ackm_on_timeout(OSSL_ACKM *ackm) -{ - int pkt_space; - OSSL_TIME earliest_loss_time; - OSSL_ACKM_TX_PKT *lost_pkts; - - earliest_loss_time = ackm_get_loss_time_and_space(ackm, &pkt_space); - if (!ossl_time_is_zero(earliest_loss_time)) { - /* Time threshold loss detection. */ - lost_pkts = ackm_detect_and_remove_lost_pkts(ackm, pkt_space); - if (lost_pkts != NULL) - ackm_on_pkts_lost(ackm, pkt_space, lost_pkts, /*pseudo=*/0); - ackm_set_loss_detection_timer(ackm); - return 1; - } - - if (ackm_ack_eliciting_bytes_in_flight(ackm) == 0) { - assert(!ackm->peer_completed_addr_validation); - /* - * Client sends an anti-deadlock packet: Initial is padded to earn more - * anti-amplification credit. A handshake packet proves address - * ownership. - */ - if (ackm->discarded[QUIC_PN_SPACE_INITIAL]) - ackm_queue_probe_anti_deadlock_handshake(ackm); - else - ackm_queue_probe_anti_deadlock_initial(ackm); - } else { - /* - * PTO. The user of the ACKM should send new data if available, else - * retransmit old data, or if neither is available, send a single PING - * frame. - */ - ackm_get_pto_time_and_space(ackm, &pkt_space); - ackm_queue_probe(ackm, pkt_space); - } - - ++ackm->pto_count; - ackm_set_loss_detection_timer(ackm); - return 1; -} - -OSSL_TIME ossl_ackm_get_loss_detection_deadline(OSSL_ACKM *ackm) -{ - return ackm->loss_detection_deadline; -} - -OSSL_ACKM_PROBE_INFO *ossl_ackm_get0_probe_request(OSSL_ACKM *ackm) -{ - return &ackm->pending_probe; -} - -int ossl_ackm_get_largest_unacked(OSSL_ACKM *ackm, int pkt_space, QUIC_PN *pn) -{ - struct tx_pkt_history_st *h; - OSSL_ACKM_TX_PKT *p; - - h = get_tx_history(ackm, pkt_space); - p = ossl_list_tx_history_tail(&h->packets); - if (p != NULL) { - *pn = p->pkt_num; - return 1; - } - - return 0; -} - -/* Number of ACK-eliciting packets RX'd before we always emit an ACK. */ -#define PKTS_BEFORE_ACK 2 - -/* - * Return 1 if emission of an ACK frame is currently desired. - * - * This occurs when one or more of the following conditions occurs: - * - * - We have flagged that we want to send an ACK frame - * (for example, due to the packet threshold count being exceeded), or - * - * - We have exceeded the ACK flush deadline, meaning that - * we have received at least one ACK-eliciting packet, but held off on - * sending an ACK frame immediately in the hope that more ACK-eliciting - * packets might come in, but not enough did and we are now requesting - * transmission of an ACK frame anyway. - * - */ -int ossl_ackm_is_ack_desired(OSSL_ACKM *ackm, int pkt_space) -{ - return ackm->rx_ack_desired[pkt_space] - || (!ossl_time_is_infinite(ackm->rx_ack_flush_deadline[pkt_space]) - && ossl_time_compare(ackm->now(ackm->now_arg), - ackm->rx_ack_flush_deadline[pkt_space]) >= 0); -} - -/* - * Returns 1 if an ACK frame matches a given packet number. - */ -static int ack_contains(const OSSL_QUIC_FRAME_ACK *ack, QUIC_PN pkt_num) -{ - size_t i; - - for (i = 0; i < ack->num_ack_ranges; ++i) - if (range_contains(&ack->ack_ranges[i], pkt_num)) - return 1; - - return 0; -} - -/* - * Returns 1 iff a PN (which we have just received) was previously reported as - * implied missing (by us, in an ACK frame we previously generated). - */ -static int ackm_is_missing(OSSL_ACKM *ackm, int pkt_space, QUIC_PN pkt_num) -{ - /* - * A PN is implied missing if it is not greater than the highest PN in our - * generated ACK frame, but is not matched by the frame. - */ - return ackm->ack[pkt_space].num_ack_ranges > 0 - && pkt_num <= ackm->ack[pkt_space].ack_ranges[0].end - && !ack_contains(&ackm->ack[pkt_space], pkt_num); -} - -/* - * Returns 1 iff our RX of a PN newly establishes the implication of missing - * packets. - */ -static int ackm_has_newly_missing(OSSL_ACKM *ackm, int pkt_space) -{ - struct rx_pkt_history_st *h; - - h = get_rx_history(ackm, pkt_space); - - if (ossl_list_uint_set_is_empty(&h->set)) - return 0; - - /* - * The second condition here establishes that the highest PN range in our RX - * history comprises only a single PN. If there is more than one, then this - * function will have returned 1 during a previous call to - * ossl_ackm_on_rx_packet assuming the third condition below was met. Thus - * we only return 1 when the missing PN condition is newly established. - * - * The third condition here establishes that the highest PN range in our RX - * history is beyond (and does not border) the highest PN we have yet - * reported in any ACK frame. Thus there is a gap of at least one PN between - * the PNs we have ACK'd previously and the PN we have just received. - */ - return ackm->ack[pkt_space].num_ack_ranges > 0 - && ossl_list_uint_set_tail(&h->set)->range.start - == ossl_list_uint_set_tail(&h->set)->range.end - && ossl_list_uint_set_tail(&h->set)->range.start - > ackm->ack[pkt_space].ack_ranges[0].end + 1; -} - -static void ackm_set_flush_deadline(OSSL_ACKM *ackm, int pkt_space, - OSSL_TIME deadline) -{ - ackm->rx_ack_flush_deadline[pkt_space] = deadline; - - if (ackm->ack_deadline_cb != NULL) - ackm->ack_deadline_cb(ossl_ackm_get_ack_deadline(ackm, pkt_space), - pkt_space, ackm->ack_deadline_cb_arg); -} - -/* Explicitly flags that we want to generate an ACK frame. */ -static void ackm_queue_ack(OSSL_ACKM *ackm, int pkt_space) -{ - ackm->rx_ack_desired[pkt_space] = 1; - - /* Cancel deadline. */ - ackm_set_flush_deadline(ackm, pkt_space, ossl_time_infinite()); -} - -static void ackm_on_rx_ack_eliciting(OSSL_ACKM *ackm, - OSSL_TIME rx_time, int pkt_space, - int was_missing) -{ - OSSL_TIME tx_max_ack_delay; - - if (ackm->rx_ack_desired[pkt_space]) - /* ACK generation already requested so nothing to do. */ - return; - - ++ackm->rx_ack_eliciting_pkts_since_last_ack[pkt_space]; - - if (!ackm->rx_ack_generated[pkt_space] - || was_missing - || ackm->rx_ack_eliciting_pkts_since_last_ack[pkt_space] - >= PKTS_BEFORE_ACK - || ackm_has_newly_missing(ackm, pkt_space)) { - /* - * Either: - * - * - We have never yet generated an ACK frame, meaning that this - * is the first ever packet received, which we should always - * acknowledge immediately, or - * - * - We previously reported the PN that we have just received as - * missing in a previous ACK frame (meaning that we should report - * the fact that we now have it to the peer immediately), or - * - * - We have exceeded the ACK-eliciting packet threshold count - * for the purposes of ACK coalescing, so request transmission - * of an ACK frame, or - * - * - The PN we just received and added to our PN RX history - * newly implies one or more missing PNs, in which case we should - * inform the peer by sending an ACK frame immediately. - * - * We do not test the ACK flush deadline here because it is tested - * separately in ossl_ackm_is_ack_desired. - */ - ackm_queue_ack(ackm, pkt_space); - return; - } - - /* - * Not emitting an ACK yet. - * - * Update the ACK flush deadline. - * - * RFC 9000 s. 13.2.1: "An endpoint MUST acknowledge all ack-eliciting - * Initial and Handshake packets immediately"; don't delay ACK generation if - * we are using the Initial or Handshake PN spaces. - */ - tx_max_ack_delay = ackm->tx_max_ack_delay; - if (pkt_space == QUIC_PN_SPACE_INITIAL - || pkt_space == QUIC_PN_SPACE_HANDSHAKE) - tx_max_ack_delay = ossl_time_zero(); - - if (ossl_time_is_infinite(ackm->rx_ack_flush_deadline[pkt_space])) - ackm_set_flush_deadline(ackm, pkt_space, - ossl_time_add(rx_time, tx_max_ack_delay)); - else - ackm_set_flush_deadline(ackm, pkt_space, - ossl_time_min(ackm->rx_ack_flush_deadline[pkt_space], - ossl_time_add(rx_time, - tx_max_ack_delay))); -} - -int ossl_ackm_on_rx_packet(OSSL_ACKM *ackm, const OSSL_ACKM_RX_PKT *pkt) -{ - struct rx_pkt_history_st *h = get_rx_history(ackm, pkt->pkt_space); - int was_missing; - - if (ossl_ackm_is_rx_pn_processable(ackm, pkt->pkt_num, pkt->pkt_space) != 1) - /* PN has already been processed or written off, no-op. */ - return 1; - - /* - * Record the largest PN we have RX'd and the time we received it. - * We use this to calculate the ACK delay field of ACK frames. - */ - if (pkt->pkt_num > ackm->rx_largest_pn[pkt->pkt_space]) { - ackm->rx_largest_pn[pkt->pkt_space] = pkt->pkt_num; - ackm->rx_largest_time[pkt->pkt_space] = pkt->time; - } - - /* - * If the PN we just received was previously implied missing by virtue of - * being omitted from a previous ACK frame generated, we skip any packet - * count thresholds or coalescing delays and emit a new ACK frame - * immediately. - */ - was_missing = ackm_is_missing(ackm, pkt->pkt_space, pkt->pkt_num); - - /* - * Add the packet number to our history list of PNs we have not yet provably - * acked. - */ - if (rx_pkt_history_add_pn(h, pkt->pkt_num) != 1) - return 0; - - /* - * Receiving this packet may or may not cause us to emit an ACK frame. - * We may not emit an ACK frame yet if we have not yet received a threshold - * number of packets. - */ - if (pkt->is_ack_eliciting) - ackm_on_rx_ack_eliciting(ackm, pkt->time, pkt->pkt_space, was_missing); - - /* Update the ECN counters according to which ECN signal we got, if any. */ - switch (pkt->ecn) { - case OSSL_ACKM_ECN_ECT0: - ++ackm->rx_ect0[pkt->pkt_space]; - break; - case OSSL_ACKM_ECN_ECT1: - ++ackm->rx_ect1[pkt->pkt_space]; - break; - case OSSL_ACKM_ECN_ECNCE: - ++ackm->rx_ecnce[pkt->pkt_space]; - break; - default: - break; - } - - return 1; -} - -static void ackm_fill_rx_ack_ranges(OSSL_ACKM *ackm, int pkt_space, - OSSL_QUIC_FRAME_ACK *ack) -{ - struct rx_pkt_history_st *h = get_rx_history(ackm, pkt_space); - UINT_SET_ITEM *x; - size_t i = 0; - - /* - * Copy out ranges from the PN set, starting at the end, until we reach our - * maximum number of ranges. - */ - for (x = ossl_list_uint_set_tail(&h->set); - x != NULL && i < OSSL_NELEM(ackm->ack_ranges); - x = ossl_list_uint_set_prev(x), ++i) { - ackm->ack_ranges[pkt_space][i].start = x->range.start; - ackm->ack_ranges[pkt_space][i].end = x->range.end; - } - - ack->ack_ranges = ackm->ack_ranges[pkt_space]; - ack->num_ack_ranges = i; -} - -const OSSL_QUIC_FRAME_ACK *ossl_ackm_get_ack_frame(OSSL_ACKM *ackm, - int pkt_space) -{ - OSSL_QUIC_FRAME_ACK *ack = &ackm->ack[pkt_space]; - OSSL_TIME now = ackm->now(ackm->now_arg); - - ackm_fill_rx_ack_ranges(ackm, pkt_space, ack); - - if (!ossl_time_is_zero(ackm->rx_largest_time[pkt_space]) - && ossl_time_compare(now, ackm->rx_largest_time[pkt_space]) > 0 - && pkt_space == QUIC_PN_SPACE_APP) - ack->delay_time = - ossl_time_subtract(now, ackm->rx_largest_time[pkt_space]); - else - ack->delay_time = ossl_time_zero(); - - ack->ect0 = ackm->rx_ect0[pkt_space]; - ack->ect1 = ackm->rx_ect1[pkt_space]; - ack->ecnce = ackm->rx_ecnce[pkt_space]; - ack->ecn_present = 1; - - ackm->rx_ack_eliciting_pkts_since_last_ack[pkt_space] = 0; - - ackm->rx_ack_generated[pkt_space] = 1; - ackm->rx_ack_desired[pkt_space] = 0; - ackm_set_flush_deadline(ackm, pkt_space, ossl_time_infinite()); - return ack; -} - - -OSSL_TIME ossl_ackm_get_ack_deadline(OSSL_ACKM *ackm, int pkt_space) -{ - if (ackm->rx_ack_desired[pkt_space]) - /* Already desired, deadline is now. */ - return ossl_time_zero(); - - return ackm->rx_ack_flush_deadline[pkt_space]; -} - -int ossl_ackm_is_rx_pn_processable(OSSL_ACKM *ackm, QUIC_PN pn, int pkt_space) -{ - struct rx_pkt_history_st *h = get_rx_history(ackm, pkt_space); - - return pn >= h->watermark && ossl_uint_set_query(&h->set, pn) == 0; -} - -void ossl_ackm_set_loss_detection_deadline_callback(OSSL_ACKM *ackm, - void (*fn)(OSSL_TIME deadline, - void *arg), - void *arg) -{ - ackm->loss_detection_deadline_cb = fn; - ackm->loss_detection_deadline_cb_arg = arg; -} - -void ossl_ackm_set_ack_deadline_callback(OSSL_ACKM *ackm, - void (*fn)(OSSL_TIME deadline, - int pkt_space, - void *arg), - void *arg) -{ - ackm->ack_deadline_cb = fn; - ackm->ack_deadline_cb_arg = arg; -} - -int ossl_ackm_mark_packet_pseudo_lost(OSSL_ACKM *ackm, - int pkt_space, QUIC_PN pn) -{ - struct tx_pkt_history_st *h = get_tx_history(ackm, pkt_space); - OSSL_ACKM_TX_PKT *pkt; - - pkt = tx_pkt_history_by_pkt_num(h, pn); - if (pkt == NULL) - return 0; - - tx_pkt_history_remove(h, pkt->pkt_num); - pkt->lnext = NULL; - ackm_on_pkts_lost(ackm, pkt_space, pkt, /*pseudo=*/1); - return 1; -} - -OSSL_TIME ossl_ackm_get_pto_duration(OSSL_ACKM *ackm) -{ - OSSL_TIME duration; - OSSL_RTT_INFO rtt; - - ossl_statm_get_rtt_info(ackm->statm, &rtt); - - duration = ossl_time_add(rtt.smoothed_rtt, - ossl_time_max(ossl_time_multiply(rtt.rtt_variance, 4), - ossl_ticks2time(K_GRANULARITY))); - if (!ossl_time_is_infinite(ackm->rx_max_ack_delay)) - duration = ossl_time_add(duration, ackm->rx_max_ack_delay); - - return duration; -} - -QUIC_PN ossl_ackm_get_largest_acked(OSSL_ACKM *ackm, int pkt_space) -{ - return ackm->largest_acked_pkt[pkt_space]; -} - -void ossl_ackm_set_rx_max_ack_delay(OSSL_ACKM *ackm, OSSL_TIME rx_max_ack_delay) -{ - ackm->rx_max_ack_delay = rx_max_ack_delay; -} - -void ossl_ackm_set_tx_max_ack_delay(OSSL_ACKM *ackm, OSSL_TIME tx_max_ack_delay) -{ - ackm->tx_max_ack_delay = tx_max_ack_delay; -} diff --git a/openssl/src/ssl/quic/quic_cfq.c b/openssl/src/ssl/quic/quic_cfq.c deleted file mode 100644 index 9b9999a82..000000000 --- a/openssl/src/ssl/quic/quic_cfq.c +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_cfq.h" -#include "internal/numbers.h" - -typedef struct quic_cfq_item_ex_st QUIC_CFQ_ITEM_EX; - -struct quic_cfq_item_ex_st { - QUIC_CFQ_ITEM public; - QUIC_CFQ_ITEM_EX *prev, *next; - unsigned char *encoded; - cfq_free_cb *free_cb; - void *free_cb_arg; - uint64_t frame_type; - size_t encoded_len; - uint32_t priority, pn_space, flags; - int state; -}; - -uint64_t ossl_quic_cfq_item_get_frame_type(const QUIC_CFQ_ITEM *item) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - return ex->frame_type; -} - -const unsigned char *ossl_quic_cfq_item_get_encoded(const QUIC_CFQ_ITEM *item) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - return ex->encoded; -} - -size_t ossl_quic_cfq_item_get_encoded_len(const QUIC_CFQ_ITEM *item) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - return ex->encoded_len; -} - -int ossl_quic_cfq_item_get_state(const QUIC_CFQ_ITEM *item) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - return ex->state; -} - -uint32_t ossl_quic_cfq_item_get_pn_space(const QUIC_CFQ_ITEM *item) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - return ex->pn_space; -} - -int ossl_quic_cfq_item_is_unreliable(const QUIC_CFQ_ITEM *item) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - return (ex->flags & QUIC_CFQ_ITEM_FLAG_UNRELIABLE) != 0; -} - -typedef struct quic_cfq_item_list_st { - QUIC_CFQ_ITEM_EX *head, *tail; -} QUIC_CFQ_ITEM_LIST; - -struct quic_cfq_st { - /* - * Invariant: A CFQ item is always in exactly one of these lists, never more - * or less than one. - * - * Invariant: The list the CFQ item is determined exactly by the state field - * of the item. - */ - QUIC_CFQ_ITEM_LIST new_list, tx_list, free_list; -}; - -static int compare(const QUIC_CFQ_ITEM_EX *a, const QUIC_CFQ_ITEM_EX *b) -{ - if (a->pn_space < b->pn_space) - return -1; - else if (a->pn_space > b->pn_space) - return 1; - - if (a->priority > b->priority) - return -1; - else if (a->priority < b->priority) - return 1; - - return 0; -} - -static void list_remove(QUIC_CFQ_ITEM_LIST *l, QUIC_CFQ_ITEM_EX *n) -{ - if (l->head == n) - l->head = n->next; - if (l->tail == n) - l->tail = n->prev; - if (n->prev != NULL) - n->prev->next = n->next; - if (n->next != NULL) - n->next->prev = n->prev; - n->prev = n->next = NULL; -} - -static void list_insert_head(QUIC_CFQ_ITEM_LIST *l, QUIC_CFQ_ITEM_EX *n) -{ - n->next = l->head; - n->prev = NULL; - l->head = n; - if (n->next != NULL) - n->next->prev = n; - if (l->tail == NULL) - l->tail = n; -} - -static void list_insert_tail(QUIC_CFQ_ITEM_LIST *l, QUIC_CFQ_ITEM_EX *n) -{ - n->prev = l->tail; - n->next = NULL; - l->tail = n; - if (n->prev != NULL) - n->prev->next = n; - if (l->head == NULL) - l->head = n; -} - -static void list_insert_after(QUIC_CFQ_ITEM_LIST *l, - QUIC_CFQ_ITEM_EX *ref, - QUIC_CFQ_ITEM_EX *n) -{ - n->prev = ref; - n->next = ref->next; - if (ref->next != NULL) - ref->next->prev = n; - ref->next = n; - if (l->tail == ref) - l->tail = n; -} - -static void list_insert_sorted(QUIC_CFQ_ITEM_LIST *l, QUIC_CFQ_ITEM_EX *n, - int (*cmp)(const QUIC_CFQ_ITEM_EX *a, - const QUIC_CFQ_ITEM_EX *b)) -{ - QUIC_CFQ_ITEM_EX *p = l->head, *pprev = NULL; - - if (p == NULL) { - l->head = l->tail = n; - n->prev = n->next = NULL; - return; - } - - for (; p != NULL && cmp(p, n) < 0; pprev = p, p = p->next); - - if (p == NULL) - list_insert_tail(l, n); - else if (pprev == NULL) - list_insert_head(l, n); - else - list_insert_after(l, pprev, n); -} - -QUIC_CFQ *ossl_quic_cfq_new(void) -{ - QUIC_CFQ *cfq = OPENSSL_zalloc(sizeof(*cfq)); - - if (cfq == NULL) - return NULL; - - return cfq; -} - -static void clear_item(QUIC_CFQ_ITEM_EX *item) -{ - if (item->free_cb != NULL) { - item->free_cb(item->encoded, item->encoded_len, item->free_cb_arg); - - item->free_cb = NULL; - item->encoded = NULL; - item->encoded_len = 0; - } - - item->state = -1; -} - -static void free_list_items(QUIC_CFQ_ITEM_LIST *l) -{ - QUIC_CFQ_ITEM_EX *p, *pnext; - - for (p = l->head; p != NULL; p = pnext) { - pnext = p->next; - clear_item(p); - OPENSSL_free(p); - } -} - -void ossl_quic_cfq_free(QUIC_CFQ *cfq) -{ - if (cfq == NULL) - return; - - free_list_items(&cfq->new_list); - free_list_items(&cfq->tx_list); - free_list_items(&cfq->free_list); - OPENSSL_free(cfq); -} - -static QUIC_CFQ_ITEM_EX *cfq_get_free(QUIC_CFQ *cfq) -{ - QUIC_CFQ_ITEM_EX *item = cfq->free_list.head; - - if (item != NULL) - return item; - - item = OPENSSL_zalloc(sizeof(*item)); - if (item == NULL) - return NULL; - - item->state = -1; - list_insert_tail(&cfq->free_list, item); - return item; -} - -QUIC_CFQ_ITEM *ossl_quic_cfq_add_frame(QUIC_CFQ *cfq, - uint32_t priority, - uint32_t pn_space, - uint64_t frame_type, - uint32_t flags, - const unsigned char *encoded, - size_t encoded_len, - cfq_free_cb *free_cb, - void *free_cb_arg) -{ - QUIC_CFQ_ITEM_EX *item = cfq_get_free(cfq); - - if (item == NULL) - return NULL; - - item->priority = priority; - item->frame_type = frame_type; - item->pn_space = pn_space; - item->encoded = (unsigned char *)encoded; - item->encoded_len = encoded_len; - item->free_cb = free_cb; - item->free_cb_arg = free_cb_arg; - - item->state = QUIC_CFQ_STATE_NEW; - item->flags = flags; - list_remove(&cfq->free_list, item); - list_insert_sorted(&cfq->new_list, item, compare); - return &item->public; -} - -void ossl_quic_cfq_mark_tx(QUIC_CFQ *cfq, QUIC_CFQ_ITEM *item) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - switch (ex->state) { - case QUIC_CFQ_STATE_NEW: - list_remove(&cfq->new_list, ex); - list_insert_tail(&cfq->tx_list, ex); - ex->state = QUIC_CFQ_STATE_TX; - break; - case QUIC_CFQ_STATE_TX: - break; /* nothing to do */ - default: - assert(0); /* invalid state (e.g. in free state) */ - break; - } -} - -void ossl_quic_cfq_mark_lost(QUIC_CFQ *cfq, QUIC_CFQ_ITEM *item, - uint32_t priority) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - if (ossl_quic_cfq_item_is_unreliable(item)) { - ossl_quic_cfq_release(cfq, item); - return; - } - - switch (ex->state) { - case QUIC_CFQ_STATE_NEW: - if (priority != UINT32_MAX && priority != ex->priority) { - list_remove(&cfq->new_list, ex); - ex->priority = priority; - list_insert_sorted(&cfq->new_list, ex, compare); - } - break; /* nothing to do */ - case QUIC_CFQ_STATE_TX: - if (priority != UINT32_MAX) - ex->priority = priority; - list_remove(&cfq->tx_list, ex); - list_insert_sorted(&cfq->new_list, ex, compare); - ex->state = QUIC_CFQ_STATE_NEW; - break; - default: - assert(0); /* invalid state (e.g. in free state) */ - break; - } -} - -/* - * Releases a CFQ item. The item may be in either state (NEW or TX) prior to the - * call. The QUIC_CFQ_ITEM pointer must not be used following this call. - */ -void ossl_quic_cfq_release(QUIC_CFQ *cfq, QUIC_CFQ_ITEM *item) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - switch (ex->state) { - case QUIC_CFQ_STATE_NEW: - list_remove(&cfq->new_list, ex); - list_insert_tail(&cfq->free_list, ex); - clear_item(ex); - break; - case QUIC_CFQ_STATE_TX: - list_remove(&cfq->tx_list, ex); - list_insert_tail(&cfq->free_list, ex); - clear_item(ex); - break; - default: - assert(0); /* invalid state (e.g. in free state) */ - break; - } -} - -QUIC_CFQ_ITEM *ossl_quic_cfq_get_priority_head(const QUIC_CFQ *cfq, - uint32_t pn_space) -{ - QUIC_CFQ_ITEM_EX *item = cfq->new_list.head; - - for (; item != NULL && item->pn_space != pn_space; item = item->next); - - if (item == NULL) - return NULL; - - return &item->public; -} - -QUIC_CFQ_ITEM *ossl_quic_cfq_item_get_priority_next(const QUIC_CFQ_ITEM *item, - uint32_t pn_space) -{ - QUIC_CFQ_ITEM_EX *ex = (QUIC_CFQ_ITEM_EX *)item; - - if (ex == NULL) - return NULL; - - ex = ex->next; - - for (; ex != NULL && ex->pn_space != pn_space; ex = ex->next); - - if (ex == NULL) - return NULL; /* ubsan */ - - return &ex->public; -} diff --git a/openssl/src/ssl/quic/quic_channel.c b/openssl/src/ssl/quic/quic_channel.c deleted file mode 100644 index 396cbe846..000000000 --- a/openssl/src/ssl/quic/quic_channel.c +++ /dev/null @@ -1,3704 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "internal/quic_channel.h" -#include "internal/quic_error.h" -#include "internal/quic_rx_depack.h" -#include "internal/quic_lcidm.h" -#include "internal/quic_srtm.h" -#include "internal/qlog_event_helpers.h" -#include "../ssl_local.h" -#include "quic_channel_local.h" -#include "quic_port_local.h" -#include "quic_engine_local.h" - -/* - * NOTE: While this channel implementation currently has basic server support, - * this functionality has been implemented for internal testing purposes and is - * not suitable for network use. In particular, it does not implement address - * validation, anti-amplification or retry logic. - * - * TODO(QUIC SERVER): Implement address validation and anti-amplification - * TODO(QUIC SERVER): Implement retry logic - */ - -#define INIT_CRYPTO_RECV_BUF_LEN 16384 -#define INIT_CRYPTO_SEND_BUF_LEN 16384 -#define INIT_APP_BUF_LEN 8192 - -/* - * Interval before we force a PING to ensure NATs don't timeout. This is based - * on the lowest commonly seen value of 30 seconds as cited in RFC 9000 s. - * 10.1.2. - */ -#define MAX_NAT_INTERVAL (ossl_ms2time(25000)) - -/* - * Our maximum ACK delay on the TX side. This is up to us to choose. Note that - * this could differ from QUIC_DEFAULT_MAX_DELAY in future as that is a protocol - * value which determines the value of the maximum ACK delay if the - * max_ack_delay transport parameter is not set. - */ -#define DEFAULT_MAX_ACK_DELAY QUIC_DEFAULT_MAX_ACK_DELAY - -DEFINE_LIST_OF_IMPL(ch, QUIC_CHANNEL); - -static void ch_save_err_state(QUIC_CHANNEL *ch); -static int ch_rx(QUIC_CHANNEL *ch, int channel_only); -static int ch_tx(QUIC_CHANNEL *ch); -static int ch_tick_tls(QUIC_CHANNEL *ch, int channel_only); -static void ch_rx_handle_packet(QUIC_CHANNEL *ch, int channel_only); -static OSSL_TIME ch_determine_next_tick_deadline(QUIC_CHANNEL *ch); -static int ch_retry(QUIC_CHANNEL *ch, - const unsigned char *retry_token, - size_t retry_token_len, - const QUIC_CONN_ID *retry_scid); -static void ch_cleanup(QUIC_CHANNEL *ch); -static int ch_generate_transport_params(QUIC_CHANNEL *ch); -static int ch_on_transport_params(const unsigned char *params, - size_t params_len, - void *arg); -static int ch_on_handshake_alert(void *arg, unsigned char alert_code); -static int ch_on_handshake_complete(void *arg); -static int ch_on_handshake_yield_secret(uint32_t enc_level, int direction, - uint32_t suite_id, EVP_MD *md, - const unsigned char *secret, - size_t secret_len, - void *arg); -static int ch_on_crypto_recv_record(const unsigned char **buf, - size_t *bytes_read, void *arg); -static int ch_on_crypto_release_record(size_t bytes_read, void *arg); -static int crypto_ensure_empty(QUIC_RSTREAM *rstream); -static int ch_on_crypto_send(const unsigned char *buf, size_t buf_len, - size_t *consumed, void *arg); -static OSSL_TIME get_time(void *arg); -static uint64_t get_stream_limit(int uni, void *arg); -static int rx_late_validate(QUIC_PN pn, int pn_space, void *arg); -static void rxku_detected(QUIC_PN pn, void *arg); -static int ch_retry(QUIC_CHANNEL *ch, - const unsigned char *retry_token, - size_t retry_token_len, - const QUIC_CONN_ID *retry_scid); -static void ch_update_idle(QUIC_CHANNEL *ch); -static int ch_discard_el(QUIC_CHANNEL *ch, - uint32_t enc_level); -static void ch_on_idle_timeout(QUIC_CHANNEL *ch); -static void ch_update_idle(QUIC_CHANNEL *ch); -static void ch_update_ping_deadline(QUIC_CHANNEL *ch); -static void ch_on_terminating_timeout(QUIC_CHANNEL *ch); -static void ch_start_terminating(QUIC_CHANNEL *ch, - const QUIC_TERMINATE_CAUSE *tcause, - int force_immediate); -static void ch_on_txp_ack_tx(const OSSL_QUIC_FRAME_ACK *ack, uint32_t pn_space, - void *arg); -static void ch_rx_handle_version_neg(QUIC_CHANNEL *ch, OSSL_QRX_PKT *pkt); -static void ch_raise_version_neg_failure(QUIC_CHANNEL *ch); -static void ch_record_state_transition(QUIC_CHANNEL *ch, uint32_t new_state); - -DEFINE_LHASH_OF_EX(QUIC_SRT_ELEM); - -QUIC_NEEDS_LOCK -static QLOG *ch_get_qlog(QUIC_CHANNEL *ch) -{ -#ifndef OPENSSL_NO_QLOG - QLOG_TRACE_INFO qti = {0}; - - if (ch->qlog != NULL) - return ch->qlog; - - if (!ch->use_qlog) - return NULL; - - if (ch->is_server && ch->init_dcid.id_len == 0) - return NULL; - - qti.odcid = ch->init_dcid; - qti.title = ch->qlog_title; - qti.description = NULL; - qti.group_id = NULL; - qti.is_server = ch->is_server; - qti.now_cb = get_time; - qti.now_cb_arg = ch; - if ((ch->qlog = ossl_qlog_new_from_env(&qti)) == NULL) { - ch->use_qlog = 0; /* don't try again */ - return NULL; - } - - return ch->qlog; -#else - return NULL; -#endif -} - -QUIC_NEEDS_LOCK -static QLOG *ch_get_qlog_cb(void *arg) -{ - QUIC_CHANNEL *ch = arg; - - return ch_get_qlog(ch); -} - -/* - * QUIC Channel Initialization and Teardown - * ======================================== - */ -#define DEFAULT_INIT_CONN_RXFC_WND (768 * 1024) -#define DEFAULT_CONN_RXFC_MAX_WND_MUL 20 - -#define DEFAULT_INIT_STREAM_RXFC_WND (512 * 1024) -#define DEFAULT_STREAM_RXFC_MAX_WND_MUL 12 - -#define DEFAULT_INIT_CONN_MAX_STREAMS 100 - -static int ch_init(QUIC_CHANNEL *ch) -{ - OSSL_QUIC_TX_PACKETISER_ARGS txp_args = {0}; - OSSL_QTX_ARGS qtx_args = {0}; - OSSL_QRX_ARGS qrx_args = {0}; - QUIC_TLS_ARGS tls_args = {0}; - uint32_t pn_space; - size_t rx_short_dcid_len; - size_t tx_init_dcid_len; - - if (ch->port == NULL || ch->lcidm == NULL || ch->srtm == NULL) - goto err; - - rx_short_dcid_len = ossl_quic_port_get_rx_short_dcid_len(ch->port); - tx_init_dcid_len = ossl_quic_port_get_tx_init_dcid_len(ch->port); - - /* For clients, generate our initial DCID. */ - if (!ch->is_server - && !ossl_quic_gen_rand_conn_id(ch->port->engine->libctx, tx_init_dcid_len, - &ch->init_dcid)) - goto err; - - /* We plug in a network write BIO to the QTX later when we get one. */ - qtx_args.libctx = ch->port->engine->libctx; - qtx_args.get_qlog_cb = ch_get_qlog_cb; - qtx_args.get_qlog_cb_arg = ch; - qtx_args.mdpl = QUIC_MIN_INITIAL_DGRAM_LEN; - ch->rx_max_udp_payload_size = qtx_args.mdpl; - - ch->ping_deadline = ossl_time_infinite(); - - ch->qtx = ossl_qtx_new(&qtx_args); - if (ch->qtx == NULL) - goto err; - - ch->txpim = ossl_quic_txpim_new(); - if (ch->txpim == NULL) - goto err; - - ch->cfq = ossl_quic_cfq_new(); - if (ch->cfq == NULL) - goto err; - - if (!ossl_quic_txfc_init(&ch->conn_txfc, NULL)) - goto err; - - /* - * Note: The TP we transmit governs what the peer can transmit and thus - * applies to the RXFC. - */ - ch->tx_init_max_stream_data_bidi_local = DEFAULT_INIT_STREAM_RXFC_WND; - ch->tx_init_max_stream_data_bidi_remote = DEFAULT_INIT_STREAM_RXFC_WND; - ch->tx_init_max_stream_data_uni = DEFAULT_INIT_STREAM_RXFC_WND; - - if (!ossl_quic_rxfc_init(&ch->conn_rxfc, NULL, - DEFAULT_INIT_CONN_RXFC_WND, - DEFAULT_CONN_RXFC_MAX_WND_MUL * - DEFAULT_INIT_CONN_RXFC_WND, - get_time, ch)) - goto err; - - for (pn_space = QUIC_PN_SPACE_INITIAL; pn_space < QUIC_PN_SPACE_NUM; ++pn_space) - if (!ossl_quic_rxfc_init_standalone(&ch->crypto_rxfc[pn_space], - INIT_CRYPTO_RECV_BUF_LEN, - get_time, ch)) - goto err; - - if (!ossl_quic_rxfc_init_standalone(&ch->max_streams_bidi_rxfc, - DEFAULT_INIT_CONN_MAX_STREAMS, - get_time, ch)) - goto err; - - if (!ossl_quic_rxfc_init_standalone(&ch->max_streams_uni_rxfc, - DEFAULT_INIT_CONN_MAX_STREAMS, - get_time, ch)) - goto err; - - if (!ossl_statm_init(&ch->statm)) - goto err; - - ch->have_statm = 1; - ch->cc_method = &ossl_cc_newreno_method; - if ((ch->cc_data = ch->cc_method->new(get_time, ch)) == NULL) - goto err; - - if ((ch->ackm = ossl_ackm_new(get_time, ch, &ch->statm, - ch->cc_method, ch->cc_data)) == NULL) - goto err; - - if (!ossl_quic_stream_map_init(&ch->qsm, get_stream_limit, ch, - &ch->max_streams_bidi_rxfc, - &ch->max_streams_uni_rxfc, - ch->is_server)) - goto err; - - ch->have_qsm = 1; - - if (!ch->is_server - && !ossl_quic_lcidm_generate_initial(ch->lcidm, ch, &txp_args.cur_scid)) - goto err; - - /* We use a zero-length SCID. */ - txp_args.cur_dcid = ch->init_dcid; - txp_args.ack_delay_exponent = 3; - txp_args.qtx = ch->qtx; - txp_args.txpim = ch->txpim; - txp_args.cfq = ch->cfq; - txp_args.ackm = ch->ackm; - txp_args.qsm = &ch->qsm; - txp_args.conn_txfc = &ch->conn_txfc; - txp_args.conn_rxfc = &ch->conn_rxfc; - txp_args.max_streams_bidi_rxfc = &ch->max_streams_bidi_rxfc; - txp_args.max_streams_uni_rxfc = &ch->max_streams_uni_rxfc; - txp_args.cc_method = ch->cc_method; - txp_args.cc_data = ch->cc_data; - txp_args.now = get_time; - txp_args.now_arg = ch; - txp_args.get_qlog_cb = ch_get_qlog_cb; - txp_args.get_qlog_cb_arg = ch; - - for (pn_space = QUIC_PN_SPACE_INITIAL; pn_space < QUIC_PN_SPACE_NUM; ++pn_space) { - ch->crypto_send[pn_space] = ossl_quic_sstream_new(INIT_CRYPTO_SEND_BUF_LEN); - if (ch->crypto_send[pn_space] == NULL) - goto err; - - txp_args.crypto[pn_space] = ch->crypto_send[pn_space]; - } - - ch->txp = ossl_quic_tx_packetiser_new(&txp_args); - if (ch->txp == NULL) - goto err; - - ossl_quic_tx_packetiser_set_ack_tx_cb(ch->txp, ch_on_txp_ack_tx, ch); - - qrx_args.libctx = ch->port->engine->libctx; - qrx_args.demux = ch->port->demux; - qrx_args.short_conn_id_len = rx_short_dcid_len; - qrx_args.max_deferred = 32; - - if ((ch->qrx = ossl_qrx_new(&qrx_args)) == NULL) - goto err; - - if (!ossl_qrx_set_late_validation_cb(ch->qrx, - rx_late_validate, - ch)) - goto err; - - if (!ossl_qrx_set_key_update_cb(ch->qrx, - rxku_detected, - ch)) - goto err; - - for (pn_space = QUIC_PN_SPACE_INITIAL; pn_space < QUIC_PN_SPACE_NUM; ++pn_space) { - ch->crypto_recv[pn_space] = ossl_quic_rstream_new(NULL, NULL, 0); - if (ch->crypto_recv[pn_space] == NULL) - goto err; - } - - /* Plug in the TLS handshake layer. */ - tls_args.s = ch->tls; - tls_args.crypto_send_cb = ch_on_crypto_send; - tls_args.crypto_send_cb_arg = ch; - tls_args.crypto_recv_rcd_cb = ch_on_crypto_recv_record; - tls_args.crypto_recv_rcd_cb_arg = ch; - tls_args.crypto_release_rcd_cb = ch_on_crypto_release_record; - tls_args.crypto_release_rcd_cb_arg = ch; - tls_args.yield_secret_cb = ch_on_handshake_yield_secret; - tls_args.yield_secret_cb_arg = ch; - tls_args.got_transport_params_cb = ch_on_transport_params; - tls_args.got_transport_params_cb_arg= ch; - tls_args.handshake_complete_cb = ch_on_handshake_complete; - tls_args.handshake_complete_cb_arg = ch; - tls_args.alert_cb = ch_on_handshake_alert; - tls_args.alert_cb_arg = ch; - tls_args.is_server = ch->is_server; - - if ((ch->qtls = ossl_quic_tls_new(&tls_args)) == NULL) - goto err; - - ch->tx_max_ack_delay = DEFAULT_MAX_ACK_DELAY; - ch->rx_max_ack_delay = QUIC_DEFAULT_MAX_ACK_DELAY; - ch->rx_ack_delay_exp = QUIC_DEFAULT_ACK_DELAY_EXP; - ch->rx_active_conn_id_limit = QUIC_MIN_ACTIVE_CONN_ID_LIMIT; - ch->tx_enc_level = QUIC_ENC_LEVEL_INITIAL; - ch->rx_enc_level = QUIC_ENC_LEVEL_INITIAL; - ch->txku_threshold_override = UINT64_MAX; - - ch->max_idle_timeout_local_req = QUIC_DEFAULT_IDLE_TIMEOUT; - ch->max_idle_timeout_remote_req = 0; - ch->max_idle_timeout = ch->max_idle_timeout_local_req; - - ossl_ackm_set_tx_max_ack_delay(ch->ackm, ossl_ms2time(ch->tx_max_ack_delay)); - ossl_ackm_set_rx_max_ack_delay(ch->ackm, ossl_ms2time(ch->rx_max_ack_delay)); - - ch_update_idle(ch); - ossl_list_ch_insert_tail(&ch->port->channel_list, ch); - ch->on_port_list = 1; - return 1; - -err: - ch_cleanup(ch); - return 0; -} - -static void ch_cleanup(QUIC_CHANNEL *ch) -{ - uint32_t pn_space; - - if (ch->ackm != NULL) - for (pn_space = QUIC_PN_SPACE_INITIAL; - pn_space < QUIC_PN_SPACE_NUM; - ++pn_space) - ossl_ackm_on_pkt_space_discarded(ch->ackm, pn_space); - - ossl_quic_lcidm_cull(ch->lcidm, ch); - ossl_quic_srtm_cull(ch->srtm, ch); - ossl_quic_tx_packetiser_free(ch->txp); - ossl_quic_txpim_free(ch->txpim); - ossl_quic_cfq_free(ch->cfq); - ossl_qtx_free(ch->qtx); - if (ch->cc_data != NULL) - ch->cc_method->free(ch->cc_data); - if (ch->have_statm) - ossl_statm_destroy(&ch->statm); - ossl_ackm_free(ch->ackm); - - if (ch->have_qsm) - ossl_quic_stream_map_cleanup(&ch->qsm); - - for (pn_space = QUIC_PN_SPACE_INITIAL; pn_space < QUIC_PN_SPACE_NUM; ++pn_space) { - ossl_quic_sstream_free(ch->crypto_send[pn_space]); - ossl_quic_rstream_free(ch->crypto_recv[pn_space]); - } - - ossl_qrx_pkt_release(ch->qrx_pkt); - ch->qrx_pkt = NULL; - - ossl_quic_tls_free(ch->qtls); - ossl_qrx_free(ch->qrx); - OPENSSL_free(ch->local_transport_params); - OPENSSL_free((char *)ch->terminate_cause.reason); - OSSL_ERR_STATE_free(ch->err_state); - OPENSSL_free(ch->ack_range_scratch); - - if (ch->on_port_list) { - ossl_list_ch_remove(&ch->port->channel_list, ch); - ch->on_port_list = 0; - } - -#ifndef OPENSSL_NO_QLOG - if (ch->qlog != NULL) - ossl_qlog_flush(ch->qlog); /* best effort */ - - OPENSSL_free(ch->qlog_title); - ossl_qlog_free(ch->qlog); -#endif -} - -QUIC_CHANNEL *ossl_quic_channel_new(const QUIC_CHANNEL_ARGS *args) -{ - QUIC_CHANNEL *ch = NULL; - - if ((ch = OPENSSL_zalloc(sizeof(*ch))) == NULL) - return NULL; - - ch->port = args->port; - ch->is_server = args->is_server; - ch->tls = args->tls; - ch->lcidm = args->lcidm; - ch->srtm = args->srtm; -#ifndef OPENSSL_NO_QLOG - ch->use_qlog = args->use_qlog; - - if (ch->use_qlog && args->qlog_title != NULL) { - if ((ch->qlog_title = OPENSSL_strdup(args->qlog_title)) == NULL) { - OPENSSL_free(ch); - return NULL; - } - } -#endif - - if (!ch_init(ch)) { - OPENSSL_free(ch); - return NULL; - } - - return ch; -} - -void ossl_quic_channel_free(QUIC_CHANNEL *ch) -{ - if (ch == NULL) - return; - - ch_cleanup(ch); - OPENSSL_free(ch); -} - -/* Set mutator callbacks for test framework support */ -int ossl_quic_channel_set_mutator(QUIC_CHANNEL *ch, - ossl_mutate_packet_cb mutatecb, - ossl_finish_mutate_cb finishmutatecb, - void *mutatearg) -{ - if (ch->qtx == NULL) - return 0; - - ossl_qtx_set_mutator(ch->qtx, mutatecb, finishmutatecb, mutatearg); - return 1; -} - -int ossl_quic_channel_get_peer_addr(QUIC_CHANNEL *ch, BIO_ADDR *peer_addr) -{ - if (!ch->addressed_mode) - return 0; - - *peer_addr = ch->cur_peer_addr; - return 1; -} - -int ossl_quic_channel_set_peer_addr(QUIC_CHANNEL *ch, const BIO_ADDR *peer_addr) -{ - if (ch->state != QUIC_CHANNEL_STATE_IDLE) - return 0; - - if (peer_addr == NULL || BIO_ADDR_family(peer_addr) == AF_UNSPEC) { - BIO_ADDR_clear(&ch->cur_peer_addr); - ch->addressed_mode = 0; - return 1; - } - - ch->cur_peer_addr = *peer_addr; - ch->addressed_mode = 1; - return 1; -} - -QUIC_REACTOR *ossl_quic_channel_get_reactor(QUIC_CHANNEL *ch) -{ - return ossl_quic_port_get0_reactor(ch->port); -} - -QUIC_STREAM_MAP *ossl_quic_channel_get_qsm(QUIC_CHANNEL *ch) -{ - return &ch->qsm; -} - -OSSL_STATM *ossl_quic_channel_get_statm(QUIC_CHANNEL *ch) -{ - return &ch->statm; -} - -QUIC_STREAM *ossl_quic_channel_get_stream_by_id(QUIC_CHANNEL *ch, - uint64_t stream_id) -{ - return ossl_quic_stream_map_get_by_id(&ch->qsm, stream_id); -} - -int ossl_quic_channel_is_active(const QUIC_CHANNEL *ch) -{ - return ch != NULL && ch->state == QUIC_CHANNEL_STATE_ACTIVE; -} - -int ossl_quic_channel_is_closing(const QUIC_CHANNEL *ch) -{ - return ch->state == QUIC_CHANNEL_STATE_TERMINATING_CLOSING; -} - -static int ossl_quic_channel_is_draining(const QUIC_CHANNEL *ch) -{ - return ch->state == QUIC_CHANNEL_STATE_TERMINATING_DRAINING; -} - -static int ossl_quic_channel_is_terminating(const QUIC_CHANNEL *ch) -{ - return ossl_quic_channel_is_closing(ch) - || ossl_quic_channel_is_draining(ch); -} - -int ossl_quic_channel_is_terminated(const QUIC_CHANNEL *ch) -{ - return ch->state == QUIC_CHANNEL_STATE_TERMINATED; -} - -int ossl_quic_channel_is_term_any(const QUIC_CHANNEL *ch) -{ - return ossl_quic_channel_is_terminating(ch) - || ossl_quic_channel_is_terminated(ch); -} - -const QUIC_TERMINATE_CAUSE * -ossl_quic_channel_get_terminate_cause(const QUIC_CHANNEL *ch) -{ - return ossl_quic_channel_is_term_any(ch) ? &ch->terminate_cause : NULL; -} - -int ossl_quic_channel_is_handshake_complete(const QUIC_CHANNEL *ch) -{ - return ch->handshake_complete; -} - -int ossl_quic_channel_is_handshake_confirmed(const QUIC_CHANNEL *ch) -{ - return ch->handshake_confirmed; -} - -QUIC_DEMUX *ossl_quic_channel_get0_demux(QUIC_CHANNEL *ch) -{ - return ch->port->demux; -} - -QUIC_PORT *ossl_quic_channel_get0_port(QUIC_CHANNEL *ch) -{ - return ch->port; -} - -QUIC_ENGINE *ossl_quic_channel_get0_engine(QUIC_CHANNEL *ch) -{ - return ossl_quic_port_get0_engine(ch->port); -} - -CRYPTO_MUTEX *ossl_quic_channel_get_mutex(QUIC_CHANNEL *ch) -{ - return ossl_quic_port_get0_mutex(ch->port); -} - -int ossl_quic_channel_has_pending(const QUIC_CHANNEL *ch) -{ - return ossl_quic_demux_has_pending(ch->port->demux) - || ossl_qrx_processed_read_pending(ch->qrx); -} - -/* - * QUIC Channel: Callbacks from Miscellaneous Subsidiary Components - * ================================================================ - */ - -/* Used by various components. */ -static OSSL_TIME get_time(void *arg) -{ - QUIC_CHANNEL *ch = arg; - - return ossl_quic_port_get_time(ch->port); -} - -/* Used by QSM. */ -static uint64_t get_stream_limit(int uni, void *arg) -{ - QUIC_CHANNEL *ch = arg; - - return uni ? ch->max_local_streams_uni : ch->max_local_streams_bidi; -} - -/* - * Called by QRX to determine if a packet is potentially invalid before trying - * to decrypt it. - */ -static int rx_late_validate(QUIC_PN pn, int pn_space, void *arg) -{ - QUIC_CHANNEL *ch = arg; - - /* Potential duplicates should not be processed. */ - if (!ossl_ackm_is_rx_pn_processable(ch->ackm, pn, pn_space)) - return 0; - - return 1; -} - -/* - * Triggers a TXKU (whether spontaneous or solicited). Does not check whether - * spontaneous TXKU is currently allowed. - */ -QUIC_NEEDS_LOCK -static void ch_trigger_txku(QUIC_CHANNEL *ch) -{ - uint64_t next_pn - = ossl_quic_tx_packetiser_get_next_pn(ch->txp, QUIC_PN_SPACE_APP); - - if (!ossl_quic_pn_valid(next_pn) - || !ossl_qtx_trigger_key_update(ch->qtx)) { - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_INTERNAL_ERROR, 0, - "key update"); - return; - } - - ch->txku_in_progress = 1; - ch->txku_pn = next_pn; - ch->rxku_expected = ch->ku_locally_initiated; -} - -QUIC_NEEDS_LOCK -static int txku_in_progress(QUIC_CHANNEL *ch) -{ - if (ch->txku_in_progress - && ossl_ackm_get_largest_acked(ch->ackm, QUIC_PN_SPACE_APP) >= ch->txku_pn) { - OSSL_TIME pto = ossl_ackm_get_pto_duration(ch->ackm); - - /* - * RFC 9001 s. 6.5: Endpoints SHOULD wait three times the PTO before - * initiating a key update after receiving an acknowledgment that - * confirms that the previous key update was received. - * - * Note that by the above wording, this period starts from when we get - * the ack for a TXKU-triggering packet, not when the TXKU is initiated. - * So we defer TXKU cooldown deadline calculation to this point. - */ - ch->txku_in_progress = 0; - ch->txku_cooldown_deadline = ossl_time_add(get_time(ch), - ossl_time_multiply(pto, 3)); - } - - return ch->txku_in_progress; -} - -QUIC_NEEDS_LOCK -static int txku_allowed(QUIC_CHANNEL *ch) -{ - return ch->tx_enc_level == QUIC_ENC_LEVEL_1RTT /* Sanity check. */ - /* Strict RFC 9001 criterion for TXKU. */ - && ch->handshake_confirmed - && !txku_in_progress(ch); -} - -QUIC_NEEDS_LOCK -static int txku_recommendable(QUIC_CHANNEL *ch) -{ - if (!txku_allowed(ch)) - return 0; - - return - /* Recommended RFC 9001 criterion for TXKU. */ - ossl_time_compare(get_time(ch), ch->txku_cooldown_deadline) >= 0 - /* Some additional sensible criteria. */ - && !ch->rxku_in_progress - && !ch->rxku_pending_confirm; -} - -QUIC_NEEDS_LOCK -static int txku_desirable(QUIC_CHANNEL *ch) -{ - uint64_t cur_pkt_count, max_pkt_count, thresh_pkt_count; - const uint32_t enc_level = QUIC_ENC_LEVEL_1RTT; - - /* Check AEAD limit to determine if we should perform a spontaneous TXKU. */ - cur_pkt_count = ossl_qtx_get_cur_epoch_pkt_count(ch->qtx, enc_level); - max_pkt_count = ossl_qtx_get_max_epoch_pkt_count(ch->qtx, enc_level); - - thresh_pkt_count = max_pkt_count / 2; - if (ch->txku_threshold_override != UINT64_MAX) - thresh_pkt_count = ch->txku_threshold_override; - - return cur_pkt_count >= thresh_pkt_count; -} - -QUIC_NEEDS_LOCK -static void ch_maybe_trigger_spontaneous_txku(QUIC_CHANNEL *ch) -{ - if (!txku_recommendable(ch) || !txku_desirable(ch)) - return; - - ch->ku_locally_initiated = 1; - ch_trigger_txku(ch); -} - -QUIC_NEEDS_LOCK -static int rxku_allowed(QUIC_CHANNEL *ch) -{ - /* - * RFC 9001 s. 6.1: An endpoint MUST NOT initiate a key update prior to - * having confirmed the handshake (Section 4.1.2). - * - * RFC 9001 s. 6.1: An endpoint MUST NOT initiate a subsequent key update - * unless it has received an acknowledgment for a packet that was sent - * protected with keys from the current key phase. - * - * RFC 9001 s. 6.2: If an endpoint detects a second update before it has - * sent any packets with updated keys containing an acknowledgment for the - * packet that initiated the key update, it indicates that its peer has - * updated keys twice without awaiting confirmation. An endpoint MAY treat - * such consecutive key updates as a connection error of type - * KEY_UPDATE_ERROR. - */ - return ch->handshake_confirmed && !ch->rxku_pending_confirm; -} - -/* - * Called when the QRX detects a new RX key update event. - */ -enum rxku_decision { - DECISION_RXKU_ONLY, - DECISION_PROTOCOL_VIOLATION, - DECISION_SOLICITED_TXKU -}; - -/* Called when the QRX detects a key update has occurred. */ -QUIC_NEEDS_LOCK -static void rxku_detected(QUIC_PN pn, void *arg) -{ - QUIC_CHANNEL *ch = arg; - enum rxku_decision decision; - OSSL_TIME pto; - - /* - * Note: rxku_in_progress is always 0 here as an RXKU cannot be detected - * when we are still in UPDATING or COOLDOWN (see quic_record_rx.h). - */ - assert(!ch->rxku_in_progress); - - if (!rxku_allowed(ch)) - /* Is RXKU even allowed at this time? */ - decision = DECISION_PROTOCOL_VIOLATION; - - else if (ch->ku_locally_initiated) - /* - * If this key update was locally initiated (meaning that this detected - * RXKU event is a result of our own spontaneous TXKU), we do not - * trigger another TXKU; after all, to do so would result in an infinite - * ping-pong of key updates. We still process it as an RXKU. - */ - decision = DECISION_RXKU_ONLY; - - else - /* - * Otherwise, a peer triggering a KU means we have to trigger a KU also. - */ - decision = DECISION_SOLICITED_TXKU; - - if (decision == DECISION_PROTOCOL_VIOLATION) { - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_KEY_UPDATE_ERROR, - 0, "RX key update again too soon"); - return; - } - - pto = ossl_ackm_get_pto_duration(ch->ackm); - - ch->ku_locally_initiated = 0; - ch->rxku_in_progress = 1; - ch->rxku_pending_confirm = 1; - ch->rxku_trigger_pn = pn; - ch->rxku_update_end_deadline = ossl_time_add(get_time(ch), pto); - ch->rxku_expected = 0; - - if (decision == DECISION_SOLICITED_TXKU) - /* NOT gated by usual txku_allowed() */ - ch_trigger_txku(ch); - - /* - * Ordinarily, we only generate ACK when some ACK-eliciting frame has been - * received. In some cases, this may not occur for a long time, for example - * if transmission of application data is going in only one direction and - * nothing else is happening with the connection. However, since the peer - * cannot initiate a subsequent (spontaneous) TXKU until its prior - * (spontaneous or solicited) TXKU has completed - meaning that prior - * TXKU's trigger packet (or subsequent packet) has been acknowledged, this - * can lead to very long times before a TXKU is considered 'completed'. - * Optimise this by forcing ACK generation after triggering TXKU. - * (Basically, we consider a RXKU event something that is 'ACK-eliciting', - * which it more or less should be; it is necessarily separate from ordinary - * processing of ACK-eliciting frames as key update is not indicated via a - * frame.) - */ - ossl_quic_tx_packetiser_schedule_ack(ch->txp, QUIC_PN_SPACE_APP); -} - -/* Called per tick to handle RXKU timer events. */ -QUIC_NEEDS_LOCK -static void ch_rxku_tick(QUIC_CHANNEL *ch) -{ - if (!ch->rxku_in_progress - || ossl_time_compare(get_time(ch), ch->rxku_update_end_deadline) < 0) - return; - - ch->rxku_update_end_deadline = ossl_time_infinite(); - ch->rxku_in_progress = 0; - - if (!ossl_qrx_key_update_timeout(ch->qrx, /*normal=*/1)) - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_INTERNAL_ERROR, 0, - "RXKU cooldown internal error"); -} - -QUIC_NEEDS_LOCK -static void ch_on_txp_ack_tx(const OSSL_QUIC_FRAME_ACK *ack, uint32_t pn_space, - void *arg) -{ - QUIC_CHANNEL *ch = arg; - - if (pn_space != QUIC_PN_SPACE_APP || !ch->rxku_pending_confirm - || !ossl_quic_frame_ack_contains_pn(ack, ch->rxku_trigger_pn)) - return; - - /* - * Defer clearing rxku_pending_confirm until TXP generate call returns - * successfully. - */ - ch->rxku_pending_confirm_done = 1; -} - -/* - * QUIC Channel: Handshake Layer Event Handling - * ============================================ - */ -static int ch_on_crypto_send(const unsigned char *buf, size_t buf_len, - size_t *consumed, void *arg) -{ - int ret; - QUIC_CHANNEL *ch = arg; - uint32_t enc_level = ch->tx_enc_level; - uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - QUIC_SSTREAM *sstream = ch->crypto_send[pn_space]; - - if (!ossl_assert(sstream != NULL)) - return 0; - - ret = ossl_quic_sstream_append(sstream, buf, buf_len, consumed); - return ret; -} - -static int crypto_ensure_empty(QUIC_RSTREAM *rstream) -{ - size_t avail = 0; - int is_fin = 0; - - if (rstream == NULL) - return 1; - - if (!ossl_quic_rstream_available(rstream, &avail, &is_fin)) - return 0; - - return avail == 0; -} - -static int ch_on_crypto_recv_record(const unsigned char **buf, - size_t *bytes_read, void *arg) -{ - QUIC_CHANNEL *ch = arg; - QUIC_RSTREAM *rstream; - int is_fin = 0; /* crypto stream is never finished, so we don't use this */ - uint32_t i; - - /* - * After we move to a later EL we must not allow our peer to send any new - * bytes in the crypto stream on a previous EL. Retransmissions of old bytes - * are allowed. - * - * In practice we will only move to a new EL when we have consumed all bytes - * which should be sent on the crypto stream at a previous EL. For example, - * the Handshake EL should not be provisioned until we have completely - * consumed a TLS 1.3 ServerHello. Thus when we provision an EL the output - * of ossl_quic_rstream_available() should be 0 for all lower ELs. Thus if a - * given EL is available we simply ensure we have not received any further - * bytes at a lower EL. - */ - for (i = QUIC_ENC_LEVEL_INITIAL; i < ch->rx_enc_level; ++i) - if (i != QUIC_ENC_LEVEL_0RTT && - !crypto_ensure_empty(ch->crypto_recv[ossl_quic_enc_level_to_pn_space(i)])) { - /* Protocol violation (RFC 9001 s. 4.1.3) */ - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - OSSL_QUIC_FRAME_TYPE_CRYPTO, - "crypto stream data in wrong EL"); - return 0; - } - - rstream = ch->crypto_recv[ossl_quic_enc_level_to_pn_space(ch->rx_enc_level)]; - if (rstream == NULL) - return 0; - - return ossl_quic_rstream_get_record(rstream, buf, bytes_read, - &is_fin); -} - -static int ch_on_crypto_release_record(size_t bytes_read, void *arg) -{ - QUIC_CHANNEL *ch = arg; - QUIC_RSTREAM *rstream; - OSSL_RTT_INFO rtt_info; - uint32_t rx_pn_space = ossl_quic_enc_level_to_pn_space(ch->rx_enc_level); - - rstream = ch->crypto_recv[rx_pn_space]; - if (rstream == NULL) - return 0; - - ossl_statm_get_rtt_info(ossl_quic_channel_get_statm(ch), &rtt_info); - if (!ossl_quic_rxfc_on_retire(&ch->crypto_rxfc[rx_pn_space], bytes_read, - rtt_info.smoothed_rtt)) - return 0; - - return ossl_quic_rstream_release_record(rstream, bytes_read); -} - -static int ch_on_handshake_yield_secret(uint32_t enc_level, int direction, - uint32_t suite_id, EVP_MD *md, - const unsigned char *secret, - size_t secret_len, - void *arg) -{ - QUIC_CHANNEL *ch = arg; - uint32_t i; - - if (enc_level < QUIC_ENC_LEVEL_HANDSHAKE || enc_level >= QUIC_ENC_LEVEL_NUM) - /* Invalid EL. */ - return 0; - - - if (direction) { - /* TX */ - if (enc_level <= ch->tx_enc_level) - /* - * Does not make sense for us to try and provision an EL we have already - * attained. - */ - return 0; - - if (!ossl_qtx_provide_secret(ch->qtx, enc_level, - suite_id, md, - secret, secret_len)) - return 0; - - ch->tx_enc_level = enc_level; - } else { - /* RX */ - if (enc_level <= ch->rx_enc_level) - /* - * Does not make sense for us to try and provision an EL we have already - * attained. - */ - return 0; - - /* - * Ensure all crypto streams for previous ELs are now empty of available - * data. - */ - for (i = QUIC_ENC_LEVEL_INITIAL; i < enc_level; ++i) - if (!crypto_ensure_empty(ch->crypto_recv[ossl_quic_enc_level_to_pn_space(i)])) { - /* Protocol violation (RFC 9001 s. 4.1.3) */ - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - OSSL_QUIC_FRAME_TYPE_CRYPTO, - "crypto stream data in wrong EL"); - return 0; - } - - if (!ossl_qrx_provide_secret(ch->qrx, enc_level, - suite_id, md, - secret, secret_len)) - return 0; - - ch->have_new_rx_secret = 1; - ch->rx_enc_level = enc_level; - } - - return 1; -} - -static int ch_on_handshake_complete(void *arg) -{ - QUIC_CHANNEL *ch = arg; - - if (!ossl_assert(!ch->handshake_complete)) - return 0; /* this should not happen twice */ - - if (!ossl_assert(ch->tx_enc_level == QUIC_ENC_LEVEL_1RTT)) - return 0; - - if (!ch->got_remote_transport_params) { - /* - * Was not a valid QUIC handshake if we did not get valid transport - * params. - */ - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_CRYPTO_MISSING_EXT, - OSSL_QUIC_FRAME_TYPE_CRYPTO, - "no transport parameters received"); - return 0; - } - - /* Don't need transport parameters anymore. */ - OPENSSL_free(ch->local_transport_params); - ch->local_transport_params = NULL; - - /* Tell the QRX it can now process 1-RTT packets. */ - ossl_qrx_allow_1rtt_processing(ch->qrx); - - /* Tell TXP the handshake is complete. */ - ossl_quic_tx_packetiser_notify_handshake_complete(ch->txp); - - ch->handshake_complete = 1; - - if (ch->is_server) { - /* - * On the server, the handshake is confirmed as soon as it is complete. - */ - ossl_quic_channel_on_handshake_confirmed(ch); - - ossl_quic_tx_packetiser_schedule_handshake_done(ch->txp); - } - - ch_record_state_transition(ch, ch->state); - return 1; -} - -static int ch_on_handshake_alert(void *arg, unsigned char alert_code) -{ - QUIC_CHANNEL *ch = arg; - - /* - * RFC 9001 s. 4.4: More specifically, servers MUST NOT send post-handshake - * TLS CertificateRequest messages, and clients MUST treat receipt of such - * messages as a connection error of type PROTOCOL_VIOLATION. - */ - if (alert_code == SSL_AD_UNEXPECTED_MESSAGE - && ch->handshake_complete - && ossl_quic_tls_is_cert_request(ch->qtls)) - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - 0, - "Post-handshake TLS " - "CertificateRequest received"); - /* - * RFC 9001 s. 4.6.1: Servers MUST NOT send the early_data extension with a - * max_early_data_size field set to any value other than 0xffffffff. A - * client MUST treat receipt of a NewSessionTicket that contains an - * early_data extension with any other value as a connection error of type - * PROTOCOL_VIOLATION. - */ - else if (alert_code == SSL_AD_ILLEGAL_PARAMETER - && ch->handshake_complete - && ossl_quic_tls_has_bad_max_early_data(ch->qtls)) - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - 0, - "Bad max_early_data received"); - else - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_CRYPTO_ERR_BEGIN - + alert_code, - 0, "handshake alert"); - - return 1; -} - -/* - * QUIC Channel: Transport Parameter Handling - * ========================================== - */ - -/* - * Called by handshake layer when we receive QUIC Transport Parameters from the - * peer. Note that these are not authenticated until the handshake is marked - * as complete. - */ -#define TP_REASON_SERVER_ONLY(x) \ - x " may not be sent by a client" -#define TP_REASON_DUP(x) \ - x " appears multiple times" -#define TP_REASON_MALFORMED(x) \ - x " is malformed" -#define TP_REASON_EXPECTED_VALUE(x) \ - x " does not match expected value" -#define TP_REASON_NOT_RETRY(x) \ - x " sent when not performing a retry" -#define TP_REASON_REQUIRED(x) \ - x " was not sent but is required" -#define TP_REASON_INTERNAL_ERROR(x) \ - x " encountered internal error" - -static void txfc_bump_cwm_bidi(QUIC_STREAM *s, void *arg) -{ - if (!ossl_quic_stream_is_bidi(s) - || ossl_quic_stream_is_server_init(s)) - return; - - ossl_quic_txfc_bump_cwm(&s->txfc, *(uint64_t *)arg); -} - -static void txfc_bump_cwm_uni(QUIC_STREAM *s, void *arg) -{ - if (ossl_quic_stream_is_bidi(s) - || ossl_quic_stream_is_server_init(s)) - return; - - ossl_quic_txfc_bump_cwm(&s->txfc, *(uint64_t *)arg); -} - -static void do_update(QUIC_STREAM *s, void *arg) -{ - QUIC_CHANNEL *ch = arg; - - ossl_quic_stream_map_update_state(&ch->qsm, s); -} - -static uint64_t min_u64_ignore_0(uint64_t a, uint64_t b) -{ - if (a == 0) - return b; - if (b == 0) - return a; - - return a < b ? a : b; -} - -static int ch_on_transport_params(const unsigned char *params, - size_t params_len, - void *arg) -{ - QUIC_CHANNEL *ch = arg; - PACKET pkt; - uint64_t id, v; - size_t len; - const unsigned char *body; - int got_orig_dcid = 0; - int got_initial_scid = 0; - int got_retry_scid = 0; - int got_initial_max_data = 0; - int got_initial_max_stream_data_bidi_local = 0; - int got_initial_max_stream_data_bidi_remote = 0; - int got_initial_max_stream_data_uni = 0; - int got_initial_max_streams_bidi = 0; - int got_initial_max_streams_uni = 0; - int got_stateless_reset_token = 0; - int got_preferred_addr = 0; - int got_ack_delay_exp = 0; - int got_max_ack_delay = 0; - int got_max_udp_payload_size = 0; - int got_max_idle_timeout = 0; - int got_active_conn_id_limit = 0; - int got_disable_active_migration = 0; - QUIC_CONN_ID cid; - const char *reason = "bad transport parameter"; - ossl_unused uint64_t rx_max_idle_timeout = 0; - ossl_unused const void *stateless_reset_token_p = NULL; - QUIC_PREFERRED_ADDR pfa; - - if (ch->got_remote_transport_params) { - reason = "multiple transport parameter extensions"; - goto malformed; - } - - if (!PACKET_buf_init(&pkt, params, params_len)) { - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_INTERNAL_ERROR, 0, - "internal error (packet buf init)"); - return 0; - } - - while (PACKET_remaining(&pkt) > 0) { - if (!ossl_quic_wire_peek_transport_param(&pkt, &id)) - goto malformed; - - switch (id) { - case QUIC_TPARAM_ORIG_DCID: - if (got_orig_dcid) { - reason = TP_REASON_DUP("ORIG_DCID"); - goto malformed; - } - - if (ch->is_server) { - reason = TP_REASON_SERVER_ONLY("ORIG_DCID"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_cid(&pkt, NULL, &cid)) { - reason = TP_REASON_MALFORMED("ORIG_DCID"); - goto malformed; - } - -#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - /* Must match our initial DCID. */ - if (!ossl_quic_conn_id_eq(&ch->init_dcid, &cid)) { - reason = TP_REASON_EXPECTED_VALUE("ORIG_DCID"); - goto malformed; - } -#endif - - got_orig_dcid = 1; - break; - - case QUIC_TPARAM_RETRY_SCID: - if (ch->is_server) { - reason = TP_REASON_SERVER_ONLY("RETRY_SCID"); - goto malformed; - } - - if (got_retry_scid) { - reason = TP_REASON_DUP("RETRY_SCID"); - goto malformed; - } - - if (!ch->doing_retry) { - reason = TP_REASON_NOT_RETRY("RETRY_SCID"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_cid(&pkt, NULL, &cid)) { - reason = TP_REASON_MALFORMED("RETRY_SCID"); - goto malformed; - } - - /* Must match Retry packet SCID. */ - if (!ossl_quic_conn_id_eq(&ch->retry_scid, &cid)) { - reason = TP_REASON_EXPECTED_VALUE("RETRY_SCID"); - goto malformed; - } - - got_retry_scid = 1; - break; - - case QUIC_TPARAM_INITIAL_SCID: - if (got_initial_scid) { - /* must not appear more than once */ - reason = TP_REASON_DUP("INITIAL_SCID"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_cid(&pkt, NULL, &cid)) { - reason = TP_REASON_MALFORMED("INITIAL_SCID"); - goto malformed; - } - - /* Must match SCID of first Initial packet from server. */ - if (!ossl_quic_conn_id_eq(&ch->init_scid, &cid)) { - reason = TP_REASON_EXPECTED_VALUE("INITIAL_SCID"); - goto malformed; - } - - got_initial_scid = 1; - break; - - case QUIC_TPARAM_INITIAL_MAX_DATA: - if (got_initial_max_data) { - /* must not appear more than once */ - reason = TP_REASON_DUP("INITIAL_MAX_DATA"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v)) { - reason = TP_REASON_MALFORMED("INITIAL_MAX_DATA"); - goto malformed; - } - - ossl_quic_txfc_bump_cwm(&ch->conn_txfc, v); - got_initial_max_data = 1; - break; - - case QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_BIDI_LOCAL: - if (got_initial_max_stream_data_bidi_local) { - /* must not appear more than once */ - reason = TP_REASON_DUP("INITIAL_MAX_STREAM_DATA_BIDI_LOCAL"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v)) { - reason = TP_REASON_MALFORMED("INITIAL_MAX_STREAM_DATA_BIDI_LOCAL"); - goto malformed; - } - - /* - * This is correct; the BIDI_LOCAL TP governs streams created by - * the endpoint which sends the TP, i.e., our peer. - */ - ch->rx_init_max_stream_data_bidi_remote = v; - got_initial_max_stream_data_bidi_local = 1; - break; - - case QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_BIDI_REMOTE: - if (got_initial_max_stream_data_bidi_remote) { - /* must not appear more than once */ - reason = TP_REASON_DUP("INITIAL_MAX_STREAM_DATA_BIDI_REMOTE"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v)) { - reason = TP_REASON_MALFORMED("INITIAL_MAX_STREAM_DATA_BIDI_REMOTE"); - goto malformed; - } - - /* - * This is correct; the BIDI_REMOTE TP governs streams created - * by the endpoint which receives the TP, i.e., us. - */ - ch->rx_init_max_stream_data_bidi_local = v; - - /* Apply to all existing streams. */ - ossl_quic_stream_map_visit(&ch->qsm, txfc_bump_cwm_bidi, &v); - got_initial_max_stream_data_bidi_remote = 1; - break; - - case QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_UNI: - if (got_initial_max_stream_data_uni) { - /* must not appear more than once */ - reason = TP_REASON_DUP("INITIAL_MAX_STREAM_DATA_UNI"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v)) { - reason = TP_REASON_MALFORMED("INITIAL_MAX_STREAM_DATA_UNI"); - goto malformed; - } - - ch->rx_init_max_stream_data_uni = v; - - /* Apply to all existing streams. */ - ossl_quic_stream_map_visit(&ch->qsm, txfc_bump_cwm_uni, &v); - got_initial_max_stream_data_uni = 1; - break; - - case QUIC_TPARAM_ACK_DELAY_EXP: - if (got_ack_delay_exp) { - /* must not appear more than once */ - reason = TP_REASON_DUP("ACK_DELAY_EXP"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v) - || v > QUIC_MAX_ACK_DELAY_EXP) { - reason = TP_REASON_MALFORMED("ACK_DELAY_EXP"); - goto malformed; - } - - ch->rx_ack_delay_exp = (unsigned char)v; - got_ack_delay_exp = 1; - break; - - case QUIC_TPARAM_MAX_ACK_DELAY: - if (got_max_ack_delay) { - /* must not appear more than once */ - reason = TP_REASON_DUP("MAX_ACK_DELAY"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v) - || v >= (((uint64_t)1) << 14)) { - reason = TP_REASON_MALFORMED("MAX_ACK_DELAY"); - goto malformed; - } - - ch->rx_max_ack_delay = v; - ossl_ackm_set_rx_max_ack_delay(ch->ackm, - ossl_ms2time(ch->rx_max_ack_delay)); - - got_max_ack_delay = 1; - break; - - case QUIC_TPARAM_INITIAL_MAX_STREAMS_BIDI: - if (got_initial_max_streams_bidi) { - /* must not appear more than once */ - reason = TP_REASON_DUP("INITIAL_MAX_STREAMS_BIDI"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v) - || v > (((uint64_t)1) << 60)) { - reason = TP_REASON_MALFORMED("INITIAL_MAX_STREAMS_BIDI"); - goto malformed; - } - - assert(ch->max_local_streams_bidi == 0); - ch->max_local_streams_bidi = v; - got_initial_max_streams_bidi = 1; - break; - - case QUIC_TPARAM_INITIAL_MAX_STREAMS_UNI: - if (got_initial_max_streams_uni) { - /* must not appear more than once */ - reason = TP_REASON_DUP("INITIAL_MAX_STREAMS_UNI"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v) - || v > (((uint64_t)1) << 60)) { - reason = TP_REASON_MALFORMED("INITIAL_MAX_STREAMS_UNI"); - goto malformed; - } - - assert(ch->max_local_streams_uni == 0); - ch->max_local_streams_uni = v; - got_initial_max_streams_uni = 1; - break; - - case QUIC_TPARAM_MAX_IDLE_TIMEOUT: - if (got_max_idle_timeout) { - /* must not appear more than once */ - reason = TP_REASON_DUP("MAX_IDLE_TIMEOUT"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v)) { - reason = TP_REASON_MALFORMED("MAX_IDLE_TIMEOUT"); - goto malformed; - } - - ch->max_idle_timeout_remote_req = v; - - ch->max_idle_timeout = min_u64_ignore_0(ch->max_idle_timeout_local_req, - ch->max_idle_timeout_remote_req); - - - ch_update_idle(ch); - got_max_idle_timeout = 1; - rx_max_idle_timeout = v; - break; - - case QUIC_TPARAM_MAX_UDP_PAYLOAD_SIZE: - if (got_max_udp_payload_size) { - /* must not appear more than once */ - reason = TP_REASON_DUP("MAX_UDP_PAYLOAD_SIZE"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v) - || v < QUIC_MIN_INITIAL_DGRAM_LEN) { - reason = TP_REASON_MALFORMED("MAX_UDP_PAYLOAD_SIZE"); - goto malformed; - } - - ch->rx_max_udp_payload_size = v; - got_max_udp_payload_size = 1; - break; - - case QUIC_TPARAM_ACTIVE_CONN_ID_LIMIT: - if (got_active_conn_id_limit) { - /* must not appear more than once */ - reason = TP_REASON_DUP("ACTIVE_CONN_ID_LIMIT"); - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_int(&pkt, &id, &v) - || v < QUIC_MIN_ACTIVE_CONN_ID_LIMIT) { - reason = TP_REASON_MALFORMED("ACTIVE_CONN_ID_LIMIT"); - goto malformed; - } - - ch->rx_active_conn_id_limit = v; - got_active_conn_id_limit = 1; - break; - - case QUIC_TPARAM_STATELESS_RESET_TOKEN: - if (got_stateless_reset_token) { - reason = TP_REASON_DUP("STATELESS_RESET_TOKEN"); - goto malformed; - } - - /* - * We must ensure a client doesn't send them because we don't have - * processing for them. - * - * TODO(QUIC SERVER): remove this restriction - */ - if (ch->is_server) { - reason = TP_REASON_SERVER_ONLY("STATELESS_RESET_TOKEN"); - goto malformed; - } - - body = ossl_quic_wire_decode_transport_param_bytes(&pkt, &id, &len); - if (body == NULL || len != QUIC_STATELESS_RESET_TOKEN_LEN) { - reason = TP_REASON_MALFORMED("STATELESS_RESET_TOKEN"); - goto malformed; - } - if (!ossl_quic_srtm_add(ch->srtm, ch, ch->cur_remote_seq_num, - (const QUIC_STATELESS_RESET_TOKEN *)body)) { - reason = TP_REASON_INTERNAL_ERROR("STATELESS_RESET_TOKEN"); - goto malformed; - } - - stateless_reset_token_p = body; - got_stateless_reset_token = 1; - break; - - case QUIC_TPARAM_PREFERRED_ADDR: - /* TODO(QUIC FUTURE): Handle preferred address. */ - if (got_preferred_addr) { - reason = TP_REASON_DUP("PREFERRED_ADDR"); - goto malformed; - } - - /* - * RFC 9000 s. 18.2: "A server that chooses a zero-length - * connection ID MUST NOT provide a preferred address. - * Similarly, a server MUST NOT include a zero-length connection - * ID in this transport parameter. A client MUST treat a - * violation of these requirements as a connection error of type - * TRANSPORT_PARAMETER_ERROR." - */ - if (ch->is_server) { - reason = TP_REASON_SERVER_ONLY("PREFERRED_ADDR"); - goto malformed; - } - - if (ch->cur_remote_dcid.id_len == 0) { - reason = "PREFERRED_ADDR provided for zero-length CID"; - goto malformed; - } - - if (!ossl_quic_wire_decode_transport_param_preferred_addr(&pkt, &pfa)) { - reason = TP_REASON_MALFORMED("PREFERRED_ADDR"); - goto malformed; - } - - if (pfa.cid.id_len == 0) { - reason = "zero-length CID in PREFERRED_ADDR"; - goto malformed; - } - - got_preferred_addr = 1; - break; - - case QUIC_TPARAM_DISABLE_ACTIVE_MIGRATION: - /* We do not currently handle migration, so nothing to do. */ - if (got_disable_active_migration) { - /* must not appear more than once */ - reason = TP_REASON_DUP("DISABLE_ACTIVE_MIGRATION"); - goto malformed; - } - - body = ossl_quic_wire_decode_transport_param_bytes(&pkt, &id, &len); - if (body == NULL || len > 0) { - reason = TP_REASON_MALFORMED("DISABLE_ACTIVE_MIGRATION"); - goto malformed; - } - - got_disable_active_migration = 1; - break; - - default: - /* - * Skip over and ignore. - * - * RFC 9000 s. 7.4: We SHOULD treat duplicated transport parameters - * as a connection error, but we are not required to. Currently, - * handle this programmatically by checking for duplicates in the - * parameters that we recognise, as above, but don't bother - * maintaining a list of duplicates for anything we don't recognise. - */ - body = ossl_quic_wire_decode_transport_param_bytes(&pkt, &id, - &len); - if (body == NULL) - goto malformed; - - break; - } - } - - if (!got_initial_scid) { - reason = TP_REASON_REQUIRED("INITIAL_SCID"); - goto malformed; - } - - if (!ch->is_server) { - if (!got_orig_dcid) { - reason = TP_REASON_REQUIRED("ORIG_DCID"); - goto malformed; - } - - if (ch->doing_retry && !got_retry_scid) { - reason = TP_REASON_REQUIRED("RETRY_SCID"); - goto malformed; - } - } - - ch->got_remote_transport_params = 1; - -#ifndef OPENSSL_NO_QLOG - QLOG_EVENT_BEGIN(ch_get_qlog(ch), transport, parameters_set) - QLOG_STR("owner", "remote"); - - if (got_orig_dcid) - QLOG_CID("original_destination_connection_id", - &ch->init_dcid); - if (got_initial_scid) - QLOG_CID("original_source_connection_id", - &ch->init_dcid); - if (got_retry_scid) - QLOG_CID("retry_source_connection_id", - &ch->retry_scid); - if (got_initial_max_data) - QLOG_U64("initial_max_data", - ossl_quic_txfc_get_cwm(&ch->conn_txfc)); - if (got_initial_max_stream_data_bidi_local) - QLOG_U64("initial_max_stream_data_bidi_local", - ch->rx_init_max_stream_data_bidi_local); - if (got_initial_max_stream_data_bidi_remote) - QLOG_U64("initial_max_stream_data_bidi_remote", - ch->rx_init_max_stream_data_bidi_remote); - if (got_initial_max_stream_data_uni) - QLOG_U64("initial_max_stream_data_uni", - ch->rx_init_max_stream_data_uni); - if (got_initial_max_streams_bidi) - QLOG_U64("initial_max_streams_bidi", - ch->max_local_streams_bidi); - if (got_initial_max_streams_uni) - QLOG_U64("initial_max_streams_uni", - ch->max_local_streams_uni); - if (got_ack_delay_exp) - QLOG_U64("ack_delay_exponent", ch->rx_ack_delay_exp); - if (got_max_ack_delay) - QLOG_U64("max_ack_delay", ch->rx_max_ack_delay); - if (got_max_udp_payload_size) - QLOG_U64("max_udp_payload_size", ch->rx_max_udp_payload_size); - if (got_max_idle_timeout) - QLOG_U64("max_idle_timeout", rx_max_idle_timeout); - if (got_active_conn_id_limit) - QLOG_U64("active_connection_id_limit", ch->rx_active_conn_id_limit); - if (got_stateless_reset_token) - QLOG_BIN("stateless_reset_token", stateless_reset_token_p, - QUIC_STATELESS_RESET_TOKEN_LEN); - if (got_preferred_addr) { - QLOG_BEGIN("preferred_addr") - QLOG_U64("port_v4", pfa.ipv4_port); - QLOG_U64("port_v6", pfa.ipv6_port); - QLOG_BIN("ip_v4", pfa.ipv4, sizeof(pfa.ipv4)); - QLOG_BIN("ip_v6", pfa.ipv6, sizeof(pfa.ipv6)); - QLOG_BIN("stateless_reset_token", pfa.stateless_reset.token, - sizeof(pfa.stateless_reset.token)); - QLOG_CID("connection_id", &pfa.cid); - QLOG_END() - } - QLOG_BOOL("disable_active_migration", got_disable_active_migration); - QLOG_EVENT_END() -#endif - - if (got_initial_max_data || got_initial_max_stream_data_bidi_remote - || got_initial_max_streams_bidi || got_initial_max_streams_uni) - /* - * If FC credit was bumped, we may now be able to send. Update all - * streams. - */ - ossl_quic_stream_map_visit(&ch->qsm, do_update, ch); - - /* If we are a server, we now generate our own transport parameters. */ - if (ch->is_server && !ch_generate_transport_params(ch)) { - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_INTERNAL_ERROR, 0, - "internal error"); - return 0; - } - - return 1; - -malformed: - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_TRANSPORT_PARAMETER_ERROR, - 0, reason); - return 0; -} - -/* - * Called when we want to generate transport parameters. This is called - * immediately at instantiation time for a client and after we receive the - * client's transport parameters for a server. - */ -static int ch_generate_transport_params(QUIC_CHANNEL *ch) -{ - int ok = 0; - BUF_MEM *buf_mem = NULL; - WPACKET wpkt; - int wpkt_valid = 0; - size_t buf_len = 0; - - if (ch->local_transport_params != NULL || ch->got_local_transport_params) - goto err; - - if ((buf_mem = BUF_MEM_new()) == NULL) - goto err; - - if (!WPACKET_init(&wpkt, buf_mem)) - goto err; - - wpkt_valid = 1; - - if (ossl_quic_wire_encode_transport_param_bytes(&wpkt, QUIC_TPARAM_DISABLE_ACTIVE_MIGRATION, - NULL, 0) == NULL) - goto err; - - if (ch->is_server) { - if (!ossl_quic_wire_encode_transport_param_cid(&wpkt, QUIC_TPARAM_ORIG_DCID, - &ch->init_dcid)) - goto err; - - if (!ossl_quic_wire_encode_transport_param_cid(&wpkt, QUIC_TPARAM_INITIAL_SCID, - &ch->cur_local_cid)) - goto err; - } else { - /* Client always uses an empty SCID. */ - if (ossl_quic_wire_encode_transport_param_bytes(&wpkt, QUIC_TPARAM_INITIAL_SCID, - NULL, 0) == NULL) - goto err; - } - - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_MAX_IDLE_TIMEOUT, - ch->max_idle_timeout_local_req)) - goto err; - - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_MAX_UDP_PAYLOAD_SIZE, - QUIC_MIN_INITIAL_DGRAM_LEN)) - goto err; - - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_ACTIVE_CONN_ID_LIMIT, - QUIC_MIN_ACTIVE_CONN_ID_LIMIT)) - goto err; - - if (ch->tx_max_ack_delay != QUIC_DEFAULT_MAX_ACK_DELAY - && !ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_MAX_ACK_DELAY, - ch->tx_max_ack_delay)) - goto err; - - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_INITIAL_MAX_DATA, - ossl_quic_rxfc_get_cwm(&ch->conn_rxfc))) - goto err; - - /* Send the default CWM for a new RXFC. */ - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_BIDI_LOCAL, - ch->tx_init_max_stream_data_bidi_local)) - goto err; - - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_BIDI_REMOTE, - ch->tx_init_max_stream_data_bidi_remote)) - goto err; - - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_INITIAL_MAX_STREAM_DATA_UNI, - ch->tx_init_max_stream_data_uni)) - goto err; - - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_INITIAL_MAX_STREAMS_BIDI, - ossl_quic_rxfc_get_cwm(&ch->max_streams_bidi_rxfc))) - goto err; - - if (!ossl_quic_wire_encode_transport_param_int(&wpkt, QUIC_TPARAM_INITIAL_MAX_STREAMS_UNI, - ossl_quic_rxfc_get_cwm(&ch->max_streams_uni_rxfc))) - goto err; - - if (!WPACKET_finish(&wpkt)) - goto err; - - wpkt_valid = 0; - - if (!WPACKET_get_total_written(&wpkt, &buf_len)) - goto err; - - ch->local_transport_params = (unsigned char *)buf_mem->data; - buf_mem->data = NULL; - - - if (!ossl_quic_tls_set_transport_params(ch->qtls, ch->local_transport_params, - buf_len)) - goto err; - -#ifndef OPENSSL_NO_QLOG - QLOG_EVENT_BEGIN(ch_get_qlog(ch), transport, parameters_set) - QLOG_STR("owner", "local"); - QLOG_BOOL("disable_active_migration", 1); - if (ch->is_server) { - QLOG_CID("original_destination_connection_id", &ch->init_dcid); - QLOG_CID("initial_source_connection_id", &ch->cur_local_cid); - } else { - QLOG_STR("initial_source_connection_id", ""); - } - QLOG_U64("max_idle_timeout", ch->max_idle_timeout); - QLOG_U64("max_udp_payload_size", QUIC_MIN_INITIAL_DGRAM_LEN); - QLOG_U64("active_connection_id_limit", QUIC_MIN_ACTIVE_CONN_ID_LIMIT); - QLOG_U64("max_ack_delay", ch->tx_max_ack_delay); - QLOG_U64("initial_max_data", ossl_quic_rxfc_get_cwm(&ch->conn_rxfc)); - QLOG_U64("initial_max_stream_data_bidi_local", - ch->tx_init_max_stream_data_bidi_local); - QLOG_U64("initial_max_stream_data_bidi_remote", - ch->tx_init_max_stream_data_bidi_remote); - QLOG_U64("initial_max_stream_data_uni", - ch->tx_init_max_stream_data_uni); - QLOG_U64("initial_max_streams_bidi", - ossl_quic_rxfc_get_cwm(&ch->max_streams_bidi_rxfc)); - QLOG_U64("initial_max_streams_uni", - ossl_quic_rxfc_get_cwm(&ch->max_streams_uni_rxfc)); - QLOG_EVENT_END() -#endif - - ch->got_local_transport_params = 1; - - ok = 1; -err: - if (wpkt_valid) - WPACKET_cleanup(&wpkt); - BUF_MEM_free(buf_mem); - return ok; -} - -/* - * QUIC Channel: Ticker-Mutator - * ============================ - */ - -/* - * The central ticker function called by the reactor. This does everything, or - * at least everything network I/O related. Best effort - not allowed to fail - * "loudly". - */ -void ossl_quic_channel_subtick(QUIC_CHANNEL *ch, QUIC_TICK_RESULT *res, - uint32_t flags) -{ - OSSL_TIME now, deadline; - int channel_only = (flags & QUIC_REACTOR_TICK_FLAG_CHANNEL_ONLY) != 0; - - /* - * When we tick the QUIC connection, we do everything we need to do - * periodically. Network I/O handling will already have been performed - * as necessary by the QUIC port. Thus, in order, we: - * - * - handle any packets the DEMUX has queued up for us; - * - handle any timer events which are due to fire (ACKM, etc.); - * - generate any packets which need to be sent; - * - determine the time at which we should next be ticked. - */ - - /* If we are in the TERMINATED state, there is nothing to do. */ - if (ossl_quic_channel_is_terminated(ch)) { - res->net_read_desired = 0; - res->net_write_desired = 0; - res->tick_deadline = ossl_time_infinite(); - return; - } - - /* - * If we are in the TERMINATING state, check if the terminating timer has - * expired. - */ - if (ossl_quic_channel_is_terminating(ch)) { - now = get_time(ch); - - if (ossl_time_compare(now, ch->terminate_deadline) >= 0) { - ch_on_terminating_timeout(ch); - res->net_read_desired = 0; - res->net_write_desired = 0; - res->tick_deadline = ossl_time_infinite(); - return; /* abort normal processing, nothing to do */ - } - } - - if (!ch->port->engine->inhibit_tick) { - /* Handle RXKU timeouts. */ - ch_rxku_tick(ch); - - do { - /* Process queued incoming packets. */ - ch->did_tls_tick = 0; - ch->have_new_rx_secret = 0; - ch_rx(ch, channel_only); - - /* - * Allow the handshake layer to check for any new incoming data and - * generate new outgoing data. - */ - if (!ch->did_tls_tick) - ch_tick_tls(ch, channel_only); - - /* - * If the handshake layer gave us a new secret, we need to do RX - * again because packets that were not previously processable and - * were deferred might now be processable. - * - * TODO(QUIC FUTURE): Consider handling this in the yield_secret callback. - */ - } while (ch->have_new_rx_secret); - } - - /* - * Handle any timer events which are due to fire; namely, the loss - * detection deadline and the idle timeout. - * - * ACKM ACK generation deadline is polled by TXP, so we don't need to - * handle it here. - */ - now = get_time(ch); - if (ossl_time_compare(now, ch->idle_deadline) >= 0) { - /* - * Idle timeout differs from normal protocol violation because we do - * not send a CONN_CLOSE frame; go straight to TERMINATED. - */ - if (!ch->port->engine->inhibit_tick) - ch_on_idle_timeout(ch); - - res->net_read_desired = 0; - res->net_write_desired = 0; - res->tick_deadline = ossl_time_infinite(); - return; - } - - if (!ch->port->engine->inhibit_tick) { - deadline = ossl_ackm_get_loss_detection_deadline(ch->ackm); - if (!ossl_time_is_zero(deadline) - && ossl_time_compare(now, deadline) >= 0) - ossl_ackm_on_timeout(ch->ackm); - - /* If a ping is due, inform TXP. */ - if (ossl_time_compare(now, ch->ping_deadline) >= 0) { - int pn_space = ossl_quic_enc_level_to_pn_space(ch->tx_enc_level); - - ossl_quic_tx_packetiser_schedule_ack_eliciting(ch->txp, pn_space); - - /* - * If we have no CC budget at this time we cannot process the above - * PING request immediately. In any case we have scheduled the - * request so bump the ping deadline. If we don't do this we will - * busy-loop endlessly as the above deadline comparison condition - * will still be met. - */ - ch_update_ping_deadline(ch); - } - - /* Queue any data to be sent for transmission. */ - ch_tx(ch); - - /* Do stream GC. */ - ossl_quic_stream_map_gc(&ch->qsm); - } - - /* Determine the time at which we should next be ticked. */ - res->tick_deadline = ch_determine_next_tick_deadline(ch); - - /* - * Always process network input unless we are now terminated. Although we - * had not terminated at the beginning of this tick, network errors in - * ch_tx() may have caused us to transition to the Terminated state. - */ - res->net_read_desired = !ossl_quic_channel_is_terminated(ch); - - /* We want to write to the network if we have any data in our TX queue. */ - res->net_write_desired - = (!ossl_quic_channel_is_terminated(ch) - && ossl_qtx_get_queue_len_datagrams(ch->qtx) > 0); -} - -static int ch_tick_tls(QUIC_CHANNEL *ch, int channel_only) -{ - uint64_t error_code; - const char *error_msg; - ERR_STATE *error_state = NULL; - - if (channel_only) - return 1; - - ch->did_tls_tick = 1; - ossl_quic_tls_tick(ch->qtls); - - if (ossl_quic_tls_get_error(ch->qtls, &error_code, &error_msg, - &error_state)) { - ossl_quic_channel_raise_protocol_error_state(ch, error_code, 0, - error_msg, error_state); - return 0; - } - - return 1; -} - -/* Check incoming forged packet limit and terminate connection if needed. */ -static void ch_rx_check_forged_pkt_limit(QUIC_CHANNEL *ch) -{ - uint32_t enc_level; - uint64_t limit = UINT64_MAX, l; - - for (enc_level = QUIC_ENC_LEVEL_INITIAL; - enc_level < QUIC_ENC_LEVEL_NUM; - ++enc_level) - { - /* - * Different ELs can have different AEADs which can in turn impose - * different limits, so use the lowest value of any currently valid EL. - */ - if ((ch->el_discarded & (1U << enc_level)) != 0) - continue; - - if (enc_level > ch->rx_enc_level) - break; - - l = ossl_qrx_get_max_forged_pkt_count(ch->qrx, enc_level); - if (l < limit) - limit = l; - } - - if (ossl_qrx_get_cur_forged_pkt_count(ch->qrx) < limit) - return; - - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_AEAD_LIMIT_REACHED, 0, - "forgery limit"); -} - -/* Process queued incoming packets and handle frames, if any. */ -static int ch_rx(QUIC_CHANNEL *ch, int channel_only) -{ - int handled_any = 0; - const int closing = ossl_quic_channel_is_closing(ch); - - if (!ch->is_server && !ch->have_sent_any_pkt) - /* - * We have not sent anything yet, therefore there is no need to check - * for incoming data. - */ - return 1; - - for (;;) { - assert(ch->qrx_pkt == NULL); - - if (!ossl_qrx_read_pkt(ch->qrx, &ch->qrx_pkt)) - break; - - /* Track the amount of data received while in the closing state */ - if (closing) - ossl_quic_tx_packetiser_record_received_closing_bytes( - ch->txp, ch->qrx_pkt->hdr->len); - - if (!handled_any) { - ch_update_idle(ch); - ch_update_ping_deadline(ch); - } - - ch_rx_handle_packet(ch, channel_only); /* best effort */ - - /* - * Regardless of the outcome of frame handling, unref the packet. - * This will free the packet unless something added another - * reference to it during frame processing. - */ - ossl_qrx_pkt_release(ch->qrx_pkt); - ch->qrx_pkt = NULL; - - ch->have_sent_ack_eliciting_since_rx = 0; - handled_any = 1; - } - - ch_rx_check_forged_pkt_limit(ch); - - /* - * When in TERMINATING - CLOSING, generate a CONN_CLOSE frame whenever we - * process one or more incoming packets. - */ - if (handled_any && closing) - ch->conn_close_queued = 1; - - return 1; -} - -static int bio_addr_eq(const BIO_ADDR *a, const BIO_ADDR *b) -{ - if (BIO_ADDR_family(a) != BIO_ADDR_family(b)) - return 0; - - switch (BIO_ADDR_family(a)) { - case AF_INET: - return !memcmp(&a->s_in.sin_addr, - &b->s_in.sin_addr, - sizeof(a->s_in.sin_addr)) - && a->s_in.sin_port == b->s_in.sin_port; -#if OPENSSL_USE_IPV6 - case AF_INET6: - return !memcmp(&a->s_in6.sin6_addr, - &b->s_in6.sin6_addr, - sizeof(a->s_in6.sin6_addr)) - && a->s_in6.sin6_port == b->s_in6.sin6_port; -#endif - default: - return 0; /* not supported */ - } - - return 1; -} - -/* Handles the packet currently in ch->qrx_pkt->hdr. */ -static void ch_rx_handle_packet(QUIC_CHANNEL *ch, int channel_only) -{ - uint32_t enc_level; - int old_have_processed_any_pkt = ch->have_processed_any_pkt; - OSSL_QTX_IOVEC iovec; - - assert(ch->qrx_pkt != NULL); - - /* - * RFC 9000 s. 10.2.1 Closing Connection State: - * An endpoint that is closing is not required to process any - * received frame. - */ - if (!ossl_quic_channel_is_active(ch)) - return; - - if (ossl_quic_pkt_type_is_encrypted(ch->qrx_pkt->hdr->type)) { - if (!ch->have_received_enc_pkt) { - ch->cur_remote_dcid = ch->init_scid = ch->qrx_pkt->hdr->src_conn_id; - ch->have_received_enc_pkt = 1; - - /* - * We change to using the SCID in the first Initial packet as the - * DCID. - */ - ossl_quic_tx_packetiser_set_cur_dcid(ch->txp, &ch->init_scid); - } - - enc_level = ossl_quic_pkt_type_to_enc_level(ch->qrx_pkt->hdr->type); - if ((ch->el_discarded & (1U << enc_level)) != 0) - /* Do not process packets from ELs we have already discarded. */ - return; - } - - /* - * RFC 9000 s. 9.6: "If a client receives packets from a new server address - * when the client has not initiated a migration to that address, the client - * SHOULD discard these packets." - * - * We need to be a bit careful here as due to the BIO abstraction layer an - * application is liable to be weird and lie to us about peer addresses. - * Only apply this check if we actually are using a real AF_INET or AF_INET6 - * address. - */ - if (!ch->is_server - && ch->qrx_pkt->peer != NULL - && ( - BIO_ADDR_family(&ch->cur_peer_addr) == AF_INET -#if OPENSSL_USE_IPV6 - || BIO_ADDR_family(&ch->cur_peer_addr) == AF_INET6 -#endif - ) - && !bio_addr_eq(ch->qrx_pkt->peer, &ch->cur_peer_addr)) - return; - - if (!ch->is_server - && ch->have_received_enc_pkt - && ossl_quic_pkt_type_has_scid(ch->qrx_pkt->hdr->type)) { - /* - * RFC 9000 s. 7.2: "Once a client has received a valid Initial packet - * from the server, it MUST discard any subsequent packet it receives on - * that connection with a different SCID." - */ - if (!ossl_quic_conn_id_eq(&ch->qrx_pkt->hdr->src_conn_id, - &ch->init_scid)) - return; - } - - if (ossl_quic_pkt_type_has_version(ch->qrx_pkt->hdr->type) - && ch->qrx_pkt->hdr->version != QUIC_VERSION_1) - /* - * RFC 9000 s. 5.2.1: If a client receives a packet that uses a - * different version than it initially selected, it MUST discard the - * packet. We only ever use v1, so require it. - */ - return; - - ch->have_processed_any_pkt = 1; - - /* - * RFC 9000 s. 17.2: "An endpoint MUST treat receipt of a packet that has a - * non-zero value for [the reserved bits] after removing both packet and - * header protection as a connection error of type PROTOCOL_VIOLATION." - */ - if (ossl_quic_pkt_type_is_encrypted(ch->qrx_pkt->hdr->type) - && ch->qrx_pkt->hdr->reserved != 0) { - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - 0, "packet header reserved bits"); - return; - } - - iovec.buf = ch->qrx_pkt->hdr->data; - iovec.buf_len = ch->qrx_pkt->hdr->len; - ossl_qlog_event_transport_packet_received(ch_get_qlog(ch), ch->qrx_pkt->hdr, - ch->qrx_pkt->pn, &iovec, 1, - ch->qrx_pkt->datagram_id); - - /* Handle incoming packet. */ - switch (ch->qrx_pkt->hdr->type) { - case QUIC_PKT_TYPE_RETRY: - if (ch->doing_retry || ch->is_server) - /* - * It is not allowed to ask a client to do a retry more than - * once. Clients may not send retries. - */ - return; - - /* - * RFC 9000 s 17.2.5.2: After the client has received and processed an - * Initial or Retry packet from the server, it MUST discard any - * subsequent Retry packets that it receives. - */ - if (ch->have_received_enc_pkt) - return; - - if (ch->qrx_pkt->hdr->len <= QUIC_RETRY_INTEGRITY_TAG_LEN) - /* Packets with zero-length Retry Tokens are invalid. */ - return; - - /* - * TODO(QUIC FUTURE): Theoretically this should probably be in the QRX. - * However because validation is dependent on context (namely the - * client's initial DCID) we can't do this cleanly. In the future we - * should probably add a callback to the QRX to let it call us (via - * the DEMUX) and ask us about the correct original DCID, rather - * than allow the QRX to emit a potentially malformed packet to the - * upper layers. However, special casing this will do for now. - */ - if (!ossl_quic_validate_retry_integrity_tag(ch->port->engine->libctx, - ch->port->engine->propq, - ch->qrx_pkt->hdr, - &ch->init_dcid)) - /* Malformed retry packet, ignore. */ - return; - - if (!ch_retry(ch, ch->qrx_pkt->hdr->data, - ch->qrx_pkt->hdr->len - QUIC_RETRY_INTEGRITY_TAG_LEN, - &ch->qrx_pkt->hdr->src_conn_id)) - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_INTERNAL_ERROR, - 0, "handling retry packet"); - break; - - case QUIC_PKT_TYPE_0RTT: - if (!ch->is_server) - /* Clients should never receive 0-RTT packets. */ - return; - - /* - * TODO(QUIC 0RTT): Implement 0-RTT on the server side. We currently - * do not need to implement this as a client can only do 0-RTT if we - * have given it permission to in a previous session. - */ - break; - - case QUIC_PKT_TYPE_INITIAL: - case QUIC_PKT_TYPE_HANDSHAKE: - case QUIC_PKT_TYPE_1RTT: - if (ch->is_server && ch->qrx_pkt->hdr->type == QUIC_PKT_TYPE_HANDSHAKE) - /* - * We automatically drop INITIAL EL keys when first successfully - * decrypting a HANDSHAKE packet, as per the RFC. - */ - ch_discard_el(ch, QUIC_ENC_LEVEL_INITIAL); - - if (ch->rxku_in_progress - && ch->qrx_pkt->hdr->type == QUIC_PKT_TYPE_1RTT - && ch->qrx_pkt->pn >= ch->rxku_trigger_pn - && ch->qrx_pkt->key_epoch < ossl_qrx_get_key_epoch(ch->qrx)) { - /* - * RFC 9001 s. 6.4: Packets with higher packet numbers MUST be - * protected with either the same or newer packet protection keys - * than packets with lower packet numbers. An endpoint that - * successfully removes protection with old keys when newer keys - * were used for packets with lower packet numbers MUST treat this - * as a connection error of type KEY_UPDATE_ERROR. - */ - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_KEY_UPDATE_ERROR, - 0, "new packet with old keys"); - break; - } - - if (!ch->is_server - && ch->qrx_pkt->hdr->type == QUIC_PKT_TYPE_INITIAL - && ch->qrx_pkt->hdr->token_len > 0) { - /* - * RFC 9000 s. 17.2.2: Clients that receive an Initial packet with a - * non-zero Token Length field MUST either discard the packet or - * generate a connection error of type PROTOCOL_VIOLATION. - * - * TODO(QUIC FUTURE): consider the implications of RFC 9000 s. 10.2.3 - * Immediate Close during the Handshake: - * However, at the cost of reducing feedback about - * errors for legitimate peers, some forms of denial of - * service can be made more difficult for an attacker - * if endpoints discard illegal packets rather than - * terminating a connection with CONNECTION_CLOSE. For - * this reason, endpoints MAY discard packets rather - * than immediately close if errors are detected in - * packets that lack authentication. - * I.e. should we drop this packet instead of closing the connection? - */ - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - 0, "client received initial token"); - break; - } - - /* This packet contains frames, pass to the RXDP. */ - ossl_quic_handle_frames(ch, ch->qrx_pkt); /* best effort */ - - if (ch->did_crypto_frame) - ch_tick_tls(ch, channel_only); - - break; - - case QUIC_PKT_TYPE_VERSION_NEG: - /* - * "A client MUST discard any Version Negotiation packet if it has - * received and successfully processed any other packet." - */ - if (!old_have_processed_any_pkt) - ch_rx_handle_version_neg(ch, ch->qrx_pkt); - - break; - - default: - assert(0); - break; - } -} - -static void ch_rx_handle_version_neg(QUIC_CHANNEL *ch, OSSL_QRX_PKT *pkt) -{ - /* - * We do not support version negotiation at this time. As per RFC 9000 s. - * 6.2., we MUST abandon the connection attempt if we receive a Version - * Negotiation packet, unless we have already successfully processed another - * incoming packet, or the packet lists the QUIC version we want to use. - */ - PACKET vpkt; - unsigned long v; - - if (!PACKET_buf_init(&vpkt, pkt->hdr->data, pkt->hdr->len)) - return; - - while (PACKET_remaining(&vpkt) > 0) { - if (!PACKET_get_net_4(&vpkt, &v)) - break; - - if ((uint32_t)v == QUIC_VERSION_1) - return; - } - - /* No match, this is a failure case. */ - ch_raise_version_neg_failure(ch); -} - -static void ch_raise_version_neg_failure(QUIC_CHANNEL *ch) -{ - QUIC_TERMINATE_CAUSE tcause = {0}; - - tcause.error_code = OSSL_QUIC_ERR_CONNECTION_REFUSED; - tcause.reason = "version negotiation failure"; - tcause.reason_len = strlen(tcause.reason); - - /* - * Skip TERMINATING state; this is not considered a protocol error and we do - * not send CONNECTION_CLOSE. - */ - ch_start_terminating(ch, &tcause, 1); -} - -/* Try to generate packets and if possible, flush them to the network. */ -static int ch_tx(QUIC_CHANNEL *ch) -{ - QUIC_TXP_STATUS status; - int res; - - /* - * RFC 9000 s. 10.2.2: Draining Connection State: - * While otherwise identical to the closing state, an endpoint - * in the draining state MUST NOT send any packets. - * and: - * An endpoint MUST NOT send further packets. - */ - if (ossl_quic_channel_is_draining(ch)) - return 0; - - if (ossl_quic_channel_is_closing(ch)) { - /* - * While closing, only send CONN_CLOSE if we've received more traffic - * from the peer. Once we tell the TXP to generate CONN_CLOSE, all - * future calls to it generate CONN_CLOSE frames, so otherwise we would - * just constantly generate CONN_CLOSE frames. - * - * Confirming to RFC 9000 s. 10.2.1 Closing Connection State: - * An endpoint SHOULD limit the rate at which it generates - * packets in the closing state. - */ - if (!ch->conn_close_queued) - return 0; - - ch->conn_close_queued = 0; - } - - /* Do TXKU if we need to. */ - ch_maybe_trigger_spontaneous_txku(ch); - - ch->rxku_pending_confirm_done = 0; - - /* Loop until we stop generating packets to send */ - do { - /* - * Send packet, if we need to. Best effort. The TXP consults the CC and - * applies any limitations imposed by it, so we don't need to do it here. - * - * Best effort. In particular if TXP fails for some reason we should - * still flush any queued packets which we already generated. - */ - res = ossl_quic_tx_packetiser_generate(ch->txp, &status); - if (status.sent_pkt > 0) { - ch->have_sent_any_pkt = 1; /* Packet(s) were sent */ - ch->port->have_sent_any_pkt = 1; - - /* - * RFC 9000 s. 10.1. 'An endpoint also restarts its idle timer when - * sending an ack-eliciting packet if no other ack-eliciting packets - * have been sent since last receiving and processing a packet.' - */ - if (status.sent_ack_eliciting - && !ch->have_sent_ack_eliciting_since_rx) { - ch_update_idle(ch); - ch->have_sent_ack_eliciting_since_rx = 1; - } - - if (!ch->is_server && status.sent_handshake) - /* - * RFC 9001 s. 4.9.1: A client MUST discard Initial keys when it - * first sends a Handshake packet. - */ - ch_discard_el(ch, QUIC_ENC_LEVEL_INITIAL); - - if (ch->rxku_pending_confirm_done) - ch->rxku_pending_confirm = 0; - - ch_update_ping_deadline(ch); - } - - if (!res) { - /* - * One case where TXP can fail is if we reach a TX PN of 2**62 - 1. - * As per RFC 9000 s. 12.3, if this happens we MUST close the - * connection without sending a CONNECTION_CLOSE frame. This is - * actually handled as an emergent consequence of our design, as the - * TX packetiser will never transmit another packet when the TX PN - * reaches the limit. - * - * Calling the below function terminates the connection; its attempt - * to schedule a CONNECTION_CLOSE frame will not actually cause a - * packet to be transmitted for this reason. - */ - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_INTERNAL_ERROR, - 0, - "internal error (txp generate)"); - break; - } - } while (status.sent_pkt > 0); - - /* Flush packets to network. */ - switch (ossl_qtx_flush_net(ch->qtx)) { - case QTX_FLUSH_NET_RES_OK: - case QTX_FLUSH_NET_RES_TRANSIENT_FAIL: - /* Best effort, done for now. */ - break; - - case QTX_FLUSH_NET_RES_PERMANENT_FAIL: - default: - /* Permanent underlying network BIO, start terminating. */ - ossl_quic_port_raise_net_error(ch->port, ch); - break; - } - - return 1; -} - -/* Determine next tick deadline. */ -static OSSL_TIME ch_determine_next_tick_deadline(QUIC_CHANNEL *ch) -{ - OSSL_TIME deadline; - int i; - - if (ossl_quic_channel_is_terminated(ch)) - return ossl_time_infinite(); - - deadline = ossl_ackm_get_loss_detection_deadline(ch->ackm); - if (ossl_time_is_zero(deadline)) - deadline = ossl_time_infinite(); - - /* - * Check the ack deadline for all enc_levels that are actually provisioned. - * ACKs aren't restricted by CC. - */ - for (i = 0; i < QUIC_ENC_LEVEL_NUM; i++) { - if (ossl_qtx_is_enc_level_provisioned(ch->qtx, i)) { - deadline = ossl_time_min(deadline, - ossl_ackm_get_ack_deadline(ch->ackm, - ossl_quic_enc_level_to_pn_space(i))); - } - } - - /* - * When do we need to send an ACK-eliciting packet to reset the idle - * deadline timer for the peer? - */ - if (!ossl_time_is_infinite(ch->ping_deadline)) - deadline = ossl_time_min(deadline, ch->ping_deadline); - - /* Apply TXP wakeup deadline. */ - deadline = ossl_time_min(deadline, - ossl_quic_tx_packetiser_get_deadline(ch->txp)); - - /* Is the terminating timer armed? */ - if (ossl_quic_channel_is_terminating(ch)) - deadline = ossl_time_min(deadline, - ch->terminate_deadline); - else if (!ossl_time_is_infinite(ch->idle_deadline)) - deadline = ossl_time_min(deadline, - ch->idle_deadline); - - /* When does the RXKU process complete? */ - if (ch->rxku_in_progress) - deadline = ossl_time_min(deadline, ch->rxku_update_end_deadline); - - return deadline; -} - -/* - * QUIC Channel: Lifecycle Events - * ============================== - */ - -/* - * Record a state transition. This is not necessarily a change to ch->state but - * also includes the handshake becoming complete or confirmed, etc. - */ -static void ch_record_state_transition(QUIC_CHANNEL *ch, uint32_t new_state) -{ - uint32_t old_state = ch->state; - - ch->state = new_state; - - ossl_qlog_event_connectivity_connection_state_updated(ch_get_qlog(ch), - old_state, - new_state, - ch->handshake_complete, - ch->handshake_confirmed); -} - -int ossl_quic_channel_start(QUIC_CHANNEL *ch) -{ - if (ch->is_server) - /* - * This is not used by the server. The server moves to active - * automatically on receiving an incoming connection. - */ - return 0; - - if (ch->state != QUIC_CHANNEL_STATE_IDLE) - /* Calls to connect are idempotent */ - return 1; - - /* Inform QTX of peer address. */ - if (!ossl_quic_tx_packetiser_set_peer(ch->txp, &ch->cur_peer_addr)) - return 0; - - /* Plug in secrets for the Initial EL. */ - if (!ossl_quic_provide_initial_secret(ch->port->engine->libctx, - ch->port->engine->propq, - &ch->init_dcid, - ch->is_server, - ch->qrx, ch->qtx)) - return 0; - - /* - * Determine the QUIC Transport Parameters and serialize the transport - * parameters block. (For servers, we do this later as we must defer - * generation until we have received the client's transport parameters.) - */ - if (!ch->is_server && !ch->got_local_transport_params - && !ch_generate_transport_params(ch)) - return 0; - - /* Change state. */ - ch_record_state_transition(ch, QUIC_CHANNEL_STATE_ACTIVE); - ch->doing_proactive_ver_neg = 0; /* not currently supported */ - - ossl_qlog_event_connectivity_connection_started(ch_get_qlog(ch), - &ch->init_dcid); - - /* Handshake layer: start (e.g. send CH). */ - if (!ch_tick_tls(ch, /*channel_only=*/0)) - return 0; - - ossl_quic_reactor_tick(ossl_quic_port_get0_reactor(ch->port), 0); /* best effort */ - return 1; -} - -/* Start a locally initiated connection shutdown. */ -void ossl_quic_channel_local_close(QUIC_CHANNEL *ch, uint64_t app_error_code, - const char *app_reason) -{ - QUIC_TERMINATE_CAUSE tcause = {0}; - - if (ossl_quic_channel_is_term_any(ch)) - return; - - tcause.app = 1; - tcause.error_code = app_error_code; - tcause.reason = app_reason; - tcause.reason_len = app_reason != NULL ? strlen(app_reason) : 0; - ch_start_terminating(ch, &tcause, 0); -} - -static void free_token(const unsigned char *buf, size_t buf_len, void *arg) -{ - OPENSSL_free((unsigned char *)buf); -} - -/* Called when a server asks us to do a retry. */ -static int ch_retry(QUIC_CHANNEL *ch, - const unsigned char *retry_token, - size_t retry_token_len, - const QUIC_CONN_ID *retry_scid) -{ - void *buf; - - /* - * RFC 9000 s. 17.2.5.1: "A client MUST discard a Retry packet that contains - * a SCID field that is identical to the DCID field of its initial packet." - */ - if (ossl_quic_conn_id_eq(&ch->init_dcid, retry_scid)) - return 1; - - /* We change to using the SCID in the Retry packet as the DCID. */ - if (!ossl_quic_tx_packetiser_set_cur_dcid(ch->txp, retry_scid)) - return 0; - - /* - * Now we retry. We will release the Retry packet immediately, so copy - * the token. - */ - if ((buf = OPENSSL_memdup(retry_token, retry_token_len)) == NULL) - return 0; - - if (!ossl_quic_tx_packetiser_set_initial_token(ch->txp, buf, - retry_token_len, - free_token, NULL)) { - /* - * This may fail if the token we receive is too big for us to ever be - * able to transmit in an outgoing Initial packet. - */ - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_INVALID_TOKEN, 0, - "received oversize token"); - OPENSSL_free(buf); - return 0; - } - - ch->retry_scid = *retry_scid; - ch->doing_retry = 1; - - /* - * We need to stimulate the Initial EL to generate the first CRYPTO frame - * again. We can do this most cleanly by simply forcing the ACKM to consider - * the first Initial packet as lost, which it effectively was as the server - * hasn't processed it. This also maintains the desired behaviour with e.g. - * PNs not resetting and so on. - * - * The PN we used initially is always zero, because QUIC does not allow - * repeated retries. - */ - if (!ossl_ackm_mark_packet_pseudo_lost(ch->ackm, QUIC_PN_SPACE_INITIAL, - /*PN=*/0)) - return 0; - - /* - * Plug in new secrets for the Initial EL. This is the only time we change - * the secrets for an EL after we already provisioned it. - */ - if (!ossl_quic_provide_initial_secret(ch->port->engine->libctx, - ch->port->engine->propq, - &ch->retry_scid, - /*is_server=*/0, - ch->qrx, ch->qtx)) - return 0; - - return 1; -} - -/* Called when an EL is to be discarded. */ -static int ch_discard_el(QUIC_CHANNEL *ch, - uint32_t enc_level) -{ - if (!ossl_assert(enc_level < QUIC_ENC_LEVEL_1RTT)) - return 0; - - if ((ch->el_discarded & (1U << enc_level)) != 0) - /* Already done. */ - return 1; - - /* Best effort for all of these. */ - ossl_quic_tx_packetiser_discard_enc_level(ch->txp, enc_level); - ossl_qrx_discard_enc_level(ch->qrx, enc_level); - ossl_qtx_discard_enc_level(ch->qtx, enc_level); - - if (enc_level != QUIC_ENC_LEVEL_0RTT) { - uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - - ossl_ackm_on_pkt_space_discarded(ch->ackm, pn_space); - - /* We should still have crypto streams at this point. */ - if (!ossl_assert(ch->crypto_send[pn_space] != NULL) - || !ossl_assert(ch->crypto_recv[pn_space] != NULL)) - return 0; - - /* Get rid of the crypto stream state for the EL. */ - ossl_quic_sstream_free(ch->crypto_send[pn_space]); - ch->crypto_send[pn_space] = NULL; - - ossl_quic_rstream_free(ch->crypto_recv[pn_space]); - ch->crypto_recv[pn_space] = NULL; - } - - ch->el_discarded |= (1U << enc_level); - return 1; -} - -/* Intended to be called by the RXDP. */ -int ossl_quic_channel_on_handshake_confirmed(QUIC_CHANNEL *ch) -{ - if (ch->handshake_confirmed) - return 1; - - if (!ch->handshake_complete) { - /* - * Does not make sense for handshake to be confirmed before it is - * completed. - */ - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE, - "handshake cannot be confirmed " - "before it is completed"); - return 0; - } - - ch_discard_el(ch, QUIC_ENC_LEVEL_HANDSHAKE); - ch->handshake_confirmed = 1; - ch_record_state_transition(ch, ch->state); - ossl_ackm_on_handshake_confirmed(ch->ackm); - return 1; -} - -/* - * Master function used when we want to start tearing down a connection: - * - * - If the connection is still IDLE we can go straight to TERMINATED; - * - * - If we are already TERMINATED this is a no-op. - * - * - If we are TERMINATING - CLOSING and we have now got a CONNECTION_CLOSE - * from the peer (tcause->remote == 1), we move to TERMINATING - DRAINING. - * - * - If we are TERMINATING - DRAINING, we remain here until the terminating - * timer expires. - * - * - Otherwise, we are in ACTIVE and move to TERMINATING - CLOSING. - * if we caused the termination (e.g. we have sent a CONNECTION_CLOSE). Note - * that we are considered to have caused a termination if we sent the first - * CONNECTION_CLOSE frame, even if it is caused by a peer protocol - * violation. If the peer sent the first CONNECTION_CLOSE frame, we move to - * TERMINATING - DRAINING. - * - * We record the termination cause structure passed on the first call only. - * Any successive calls have their termination cause data discarded; - * once we start sending a CONNECTION_CLOSE frame, we don't change the details - * in it. - * - * This conforms to RFC 9000 s. 10.2.1: Closing Connection State: - * To minimize the state that an endpoint maintains for a closing - * connection, endpoints MAY send the exact same packet in response - * to any received packet. - * - * We don't drop any connection state (specifically packet protection keys) - * even though we are permitted to. This conforms to RFC 9000 s. 10.2.1: - * Closing Connection State: - * An endpoint MAY retain packet protection keys for incoming - * packets to allow it to read and process a CONNECTION_CLOSE frame. - * - * Note that we do not conform to these two from the same section: - * An endpoint's selected connection ID and the QUIC version - * are sufficient information to identify packets for a closing - * connection; the endpoint MAY discard all other connection state. - * and: - * An endpoint MAY drop packet protection keys when entering the - * closing state and send a packet containing a CONNECTION_CLOSE - * frame in response to any UDP datagram that is received. - */ -static void copy_tcause(QUIC_TERMINATE_CAUSE *dst, - const QUIC_TERMINATE_CAUSE *src) -{ - dst->error_code = src->error_code; - dst->frame_type = src->frame_type; - dst->app = src->app; - dst->remote = src->remote; - - dst->reason = NULL; - dst->reason_len = 0; - - if (src->reason != NULL && src->reason_len > 0) { - size_t l = src->reason_len; - char *r; - - if (l >= SIZE_MAX) - --l; - - /* - * If this fails, dst->reason becomes NULL and we simply do not use a - * reason. This ensures termination is infallible. - */ - dst->reason = r = OPENSSL_memdup(src->reason, l + 1); - if (r == NULL) - return; - - r[l] = '\0'; - dst->reason_len = l; - } -} - -static void ch_start_terminating(QUIC_CHANNEL *ch, - const QUIC_TERMINATE_CAUSE *tcause, - int force_immediate) -{ - /* No point sending anything if we haven't sent anything yet. */ - if (!ch->have_sent_any_pkt) - force_immediate = 1; - - switch (ch->state) { - default: - case QUIC_CHANNEL_STATE_IDLE: - copy_tcause(&ch->terminate_cause, tcause); - ch_on_terminating_timeout(ch); - break; - - case QUIC_CHANNEL_STATE_ACTIVE: - copy_tcause(&ch->terminate_cause, tcause); - - ossl_qlog_event_connectivity_connection_closed(ch_get_qlog(ch), tcause); - - if (!force_immediate) { - ch_record_state_transition(ch, tcause->remote - ? QUIC_CHANNEL_STATE_TERMINATING_DRAINING - : QUIC_CHANNEL_STATE_TERMINATING_CLOSING); - /* - * RFC 9000 s. 10.2 Immediate Close - * These states SHOULD persist for at least three times - * the current PTO interval as defined in [QUIC-RECOVERY]. - */ - ch->terminate_deadline - = ossl_time_add(get_time(ch), - ossl_time_multiply(ossl_ackm_get_pto_duration(ch->ackm), - 3)); - - if (!tcause->remote) { - OSSL_QUIC_FRAME_CONN_CLOSE f = {0}; - - /* best effort */ - f.error_code = ch->terminate_cause.error_code; - f.frame_type = ch->terminate_cause.frame_type; - f.is_app = ch->terminate_cause.app; - f.reason = (char *)ch->terminate_cause.reason; - f.reason_len = ch->terminate_cause.reason_len; - ossl_quic_tx_packetiser_schedule_conn_close(ch->txp, &f); - /* - * RFC 9000 s. 10.2.2 Draining Connection State: - * An endpoint that receives a CONNECTION_CLOSE frame MAY - * send a single packet containing a CONNECTION_CLOSE - * frame before entering the draining state, using a - * NO_ERROR code if appropriate - */ - ch->conn_close_queued = 1; - } - } else { - ch_on_terminating_timeout(ch); - } - break; - - case QUIC_CHANNEL_STATE_TERMINATING_CLOSING: - if (force_immediate) - ch_on_terminating_timeout(ch); - else if (tcause->remote) - /* - * RFC 9000 s. 10.2.2 Draining Connection State: - * An endpoint MAY enter the draining state from the - * closing state if it receives a CONNECTION_CLOSE frame, - * which indicates that the peer is also closing or draining. - */ - ch_record_state_transition(ch, QUIC_CHANNEL_STATE_TERMINATING_DRAINING); - - break; - - case QUIC_CHANNEL_STATE_TERMINATING_DRAINING: - /* - * Other than in the force-immediate case, we remain here until the - * timeout expires. - */ - if (force_immediate) - ch_on_terminating_timeout(ch); - - break; - - case QUIC_CHANNEL_STATE_TERMINATED: - /* No-op. */ - break; - } -} - -/* For RXDP use. */ -void ossl_quic_channel_on_remote_conn_close(QUIC_CHANNEL *ch, - OSSL_QUIC_FRAME_CONN_CLOSE *f) -{ - QUIC_TERMINATE_CAUSE tcause = {0}; - - if (!ossl_quic_channel_is_active(ch)) - return; - - tcause.remote = 1; - tcause.app = f->is_app; - tcause.error_code = f->error_code; - tcause.frame_type = f->frame_type; - tcause.reason = f->reason; - tcause.reason_len = f->reason_len; - ch_start_terminating(ch, &tcause, 0); -} - -static void free_frame_data(unsigned char *buf, size_t buf_len, void *arg) -{ - OPENSSL_free(buf); -} - -static int ch_enqueue_retire_conn_id(QUIC_CHANNEL *ch, uint64_t seq_num) -{ - BUF_MEM *buf_mem = NULL; - WPACKET wpkt; - size_t l; - - ossl_quic_srtm_remove(ch->srtm, ch, seq_num); - - if ((buf_mem = BUF_MEM_new()) == NULL) - goto err; - - if (!WPACKET_init(&wpkt, buf_mem)) - goto err; - - if (!ossl_quic_wire_encode_frame_retire_conn_id(&wpkt, seq_num)) { - WPACKET_cleanup(&wpkt); - goto err; - } - - WPACKET_finish(&wpkt); - if (!WPACKET_get_total_written(&wpkt, &l)) - goto err; - - if (ossl_quic_cfq_add_frame(ch->cfq, 1, QUIC_PN_SPACE_APP, - OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID, 0, - (unsigned char *)buf_mem->data, l, - free_frame_data, NULL) == NULL) - goto err; - - buf_mem->data = NULL; - BUF_MEM_free(buf_mem); - return 1; - -err: - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID, - "internal error enqueueing retire conn id"); - BUF_MEM_free(buf_mem); - return 0; -} - -void ossl_quic_channel_on_new_conn_id(QUIC_CHANNEL *ch, - OSSL_QUIC_FRAME_NEW_CONN_ID *f) -{ - uint64_t new_remote_seq_num = ch->cur_remote_seq_num; - uint64_t new_retire_prior_to = ch->cur_retire_prior_to; - - if (!ossl_quic_channel_is_active(ch)) - return; - - /* We allow only two active connection ids; first check some constraints */ - if (ch->cur_remote_dcid.id_len == 0) { - /* Changing from 0 length connection id is disallowed */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID, - "zero length connection id in use"); - - return; - } - - if (f->seq_num > new_remote_seq_num) - new_remote_seq_num = f->seq_num; - if (f->retire_prior_to > new_retire_prior_to) - new_retire_prior_to = f->retire_prior_to; - - /* - * RFC 9000-5.1.1: An endpoint MUST NOT provide more connection IDs - * than the peer's limit. - * - * After processing a NEW_CONNECTION_ID frame and adding and retiring - * active connection IDs, if the number of active connection IDs exceeds - * the value advertised in its active_connection_id_limit transport - * parameter, an endpoint MUST close the connection with an error of - * type CONNECTION_ID_LIMIT_ERROR. - */ - if (new_remote_seq_num - new_retire_prior_to > 1) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR, - OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID, - "active_connection_id limit violated"); - return; - } - - /* - * RFC 9000-5.1.1: An endpoint MAY send connection IDs that temporarily - * exceed a peer's limit if the NEW_CONNECTION_ID frame also requires - * the retirement of any excess, by including a sufficiently large - * value in the Retire Prior To field. - * - * RFC 9000-5.1.2: An endpoint SHOULD allow for sending and tracking - * a number of RETIRE_CONNECTION_ID frames of at least twice the value - * of the active_connection_id_limit transport parameter. An endpoint - * MUST NOT forget a connection ID without retiring it, though it MAY - * choose to treat having connection IDs in need of retirement that - * exceed this limit as a connection error of type CONNECTION_ID_LIMIT_ERROR. - * - * We are a little bit more liberal than the minimum mandated. - */ - if (new_retire_prior_to - ch->cur_retire_prior_to > 10) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR, - OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID, - "retiring connection id limit violated"); - - return; - } - - if (new_remote_seq_num > ch->cur_remote_seq_num) { - /* Add new stateless reset token */ - if (!ossl_quic_srtm_add(ch->srtm, ch, new_remote_seq_num, - &f->stateless_reset)) { - ossl_quic_channel_raise_protocol_error( - ch, OSSL_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR, - OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID, - "unable to store stateless reset token"); - - return; - } - ch->cur_remote_seq_num = new_remote_seq_num; - ch->cur_remote_dcid = f->conn_id; - ossl_quic_tx_packetiser_set_cur_dcid(ch->txp, &ch->cur_remote_dcid); - } - - /* - * RFC 9000-5.1.2: Upon receipt of an increased Retire Prior To - * field, the peer MUST stop using the corresponding connection IDs - * and retire them with RETIRE_CONNECTION_ID frames before adding the - * newly provided connection ID to the set of active connection IDs. - */ - - /* - * Note: RFC 9000 s. 19.15 says: - * "An endpoint that receives a NEW_CONNECTION_ID frame with a sequence - * number smaller than the Retire Prior To field of a previously received - * NEW_CONNECTION_ID frame MUST send a corresponding - * RETIRE_CONNECTION_ID frame that retires the newly received connection - * ID, unless it has already done so for that sequence number." - * - * Since we currently always queue RETIRE_CONN_ID frames based on the Retire - * Prior To field of a NEW_CONNECTION_ID frame immediately upon receiving - * that NEW_CONNECTION_ID frame, by definition this will always be met. - * This may change in future when we change our CID handling. - */ - while (new_retire_prior_to > ch->cur_retire_prior_to) { - if (!ch_enqueue_retire_conn_id(ch, ch->cur_retire_prior_to)) - break; - ++ch->cur_retire_prior_to; - } -} - -static void ch_save_err_state(QUIC_CHANNEL *ch) -{ - if (ch->err_state == NULL) - ch->err_state = OSSL_ERR_STATE_new(); - - if (ch->err_state == NULL) - return; - - OSSL_ERR_STATE_save(ch->err_state); -} - -void ossl_quic_channel_inject(QUIC_CHANNEL *ch, QUIC_URXE *e) -{ - ossl_qrx_inject_urxe(ch->qrx, e); -} - -void ossl_quic_channel_on_stateless_reset(QUIC_CHANNEL *ch) -{ - QUIC_TERMINATE_CAUSE tcause = {0}; - - tcause.error_code = OSSL_QUIC_ERR_NO_ERROR; - tcause.remote = 1; - ch_start_terminating(ch, &tcause, 0); -} - -void ossl_quic_channel_raise_net_error(QUIC_CHANNEL *ch) -{ - QUIC_TERMINATE_CAUSE tcause = {0}; - - if (ch->net_error) - return; - - ch->net_error = 1; - - tcause.error_code = OSSL_QUIC_ERR_INTERNAL_ERROR; - tcause.reason = "network BIO I/O error"; - tcause.reason_len = strlen(tcause.reason); - - /* - * Skip Terminating state and go directly to Terminated, no point trying to - * send CONNECTION_CLOSE if we cannot communicate. - */ - ch_start_terminating(ch, &tcause, 1); -} - -int ossl_quic_channel_net_error(QUIC_CHANNEL *ch) -{ - return ch->net_error; -} - -void ossl_quic_channel_restore_err_state(QUIC_CHANNEL *ch) -{ - if (ch == NULL) - return; - - if (!ossl_quic_port_is_running(ch->port)) - ossl_quic_port_restore_err_state(ch->port); - else - OSSL_ERR_STATE_restore(ch->err_state); -} - -void ossl_quic_channel_raise_protocol_error_loc(QUIC_CHANNEL *ch, - uint64_t error_code, - uint64_t frame_type, - const char *reason, - ERR_STATE *err_state, - const char *src_file, - int src_line, - const char *src_func) -{ - QUIC_TERMINATE_CAUSE tcause = {0}; - int err_reason = error_code == OSSL_QUIC_ERR_INTERNAL_ERROR - ? ERR_R_INTERNAL_ERROR : SSL_R_QUIC_PROTOCOL_ERROR; - const char *err_str = ossl_quic_err_to_string(error_code); - const char *err_str_pfx = " (", *err_str_sfx = ")"; - const char *ft_str = NULL; - const char *ft_str_pfx = " (", *ft_str_sfx = ")"; - - if (ch->protocol_error) - /* Only the first call to this function matters. */ - return; - - if (err_str == NULL) { - err_str = ""; - err_str_pfx = ""; - err_str_sfx = ""; - } - - /* - * If we were provided an underlying error state, restore it and then append - * our ERR on top as a "cover letter" error. - */ - if (err_state != NULL) - OSSL_ERR_STATE_restore(err_state); - - if (frame_type != 0) { - ft_str = ossl_quic_frame_type_to_string(frame_type); - if (ft_str == NULL) { - ft_str = ""; - ft_str_pfx = ""; - ft_str_sfx = ""; - } - - ERR_raise_data(ERR_LIB_SSL, err_reason, - "QUIC error code: 0x%llx%s%s%s " - "(triggered by frame type: 0x%llx%s%s%s), reason: \"%s\"", - (unsigned long long) error_code, - err_str_pfx, err_str, err_str_sfx, - (unsigned long long) frame_type, - ft_str_pfx, ft_str, ft_str_sfx, - reason); - } else { - ERR_raise_data(ERR_LIB_SSL, err_reason, - "QUIC error code: 0x%llx%s%s%s, reason: \"%s\"", - (unsigned long long) error_code, - err_str_pfx, err_str, err_str_sfx, - reason); - } - - if (src_file != NULL) - ERR_set_debug(src_file, src_line, src_func); - - ch_save_err_state(ch); - - tcause.error_code = error_code; - tcause.frame_type = frame_type; - tcause.reason = reason; - tcause.reason_len = strlen(reason); - - ch->protocol_error = 1; - ch_start_terminating(ch, &tcause, 0); -} - -/* - * Called once the terminating timer expires, meaning we move from TERMINATING - * to TERMINATED. - */ -static void ch_on_terminating_timeout(QUIC_CHANNEL *ch) -{ - ch_record_state_transition(ch, QUIC_CHANNEL_STATE_TERMINATED); -} - -/* - * Determines the effective idle timeout duration. This is based on the idle - * timeout values that we and our peer signalled in transport parameters - * but have some limits applied. - */ -static OSSL_TIME ch_get_effective_idle_timeout_duration(QUIC_CHANNEL *ch) -{ - OSSL_TIME pto; - - if (ch->max_idle_timeout == 0) - return ossl_time_infinite(); - - /* - * RFC 9000 s. 10.1: Idle Timeout - * To avoid excessively small idle timeout periods, endpoints - * MUST increase the idle timeout period to be at least three - * times the current Probe Timeout (PTO). This allows for - * multiple PTOs to expire, and therefore multiple probes to - * be sent and lost, prior to idle timeout. - */ - pto = ossl_ackm_get_pto_duration(ch->ackm); - return ossl_time_max(ossl_ms2time(ch->max_idle_timeout), - ossl_time_multiply(pto, 3)); -} - -/* - * Updates our idle deadline. Called when an event happens which should bump the - * idle timeout. - */ -static void ch_update_idle(QUIC_CHANNEL *ch) -{ - ch->idle_deadline = ossl_time_add(get_time(ch), - ch_get_effective_idle_timeout_duration(ch)); -} - -/* - * Updates our ping deadline, which determines when we next generate a ping if - * we don't have any other ACK-eliciting frames to send. - */ -static void ch_update_ping_deadline(QUIC_CHANNEL *ch) -{ - OSSL_TIME max_span, idle_duration; - - idle_duration = ch_get_effective_idle_timeout_duration(ch); - if (ossl_time_is_infinite(idle_duration)) { - ch->ping_deadline = ossl_time_infinite(); - return; - } - - /* - * Maximum amount of time without traffic before we send a PING to keep - * the connection open. Usually we use max_idle_timeout/2, but ensure - * the period never exceeds the assumed NAT interval to ensure NAT - * devices don't have their state time out (RFC 9000 s. 10.1.2). - */ - max_span = ossl_time_divide(idle_duration, 2); - max_span = ossl_time_min(max_span, MAX_NAT_INTERVAL); - ch->ping_deadline = ossl_time_add(get_time(ch), max_span); -} - -/* Called when the idle timeout expires. */ -static void ch_on_idle_timeout(QUIC_CHANNEL *ch) -{ - /* - * Idle timeout does not have an error code associated with it because a - * CONN_CLOSE is never sent for it. We shouldn't use this data once we reach - * TERMINATED anyway. - */ - ch->terminate_cause.app = 0; - ch->terminate_cause.error_code = OSSL_QUIC_LOCAL_ERR_IDLE_TIMEOUT; - ch->terminate_cause.frame_type = 0; - - ch_record_state_transition(ch, QUIC_CHANNEL_STATE_TERMINATED); -} - -/* Called when we, as a server, get a new incoming connection. */ -int ossl_quic_channel_on_new_conn(QUIC_CHANNEL *ch, const BIO_ADDR *peer, - const QUIC_CONN_ID *peer_scid, - const QUIC_CONN_ID *peer_dcid) -{ - if (!ossl_assert(ch->state == QUIC_CHANNEL_STATE_IDLE && ch->is_server)) - return 0; - - /* Generate an Initial LCID we will use for the connection. */ - if (!ossl_quic_lcidm_generate_initial(ch->lcidm, ch, &ch->cur_local_cid)) - return 0; - - /* Note our newly learnt peer address and CIDs. */ - ch->cur_peer_addr = *peer; - ch->init_dcid = *peer_dcid; - ch->cur_remote_dcid = *peer_scid; - - /* Inform QTX of peer address. */ - if (!ossl_quic_tx_packetiser_set_peer(ch->txp, &ch->cur_peer_addr)) - return 0; - - /* Inform TXP of desired CIDs. */ - if (!ossl_quic_tx_packetiser_set_cur_dcid(ch->txp, &ch->cur_remote_dcid)) - return 0; - - if (!ossl_quic_tx_packetiser_set_cur_scid(ch->txp, &ch->cur_local_cid)) - return 0; - - /* Setup QLOG, which did not happen earlier due to lacking an Initial ODCID. */ - ossl_qtx_set_qlog_cb(ch->qtx, ch_get_qlog_cb, ch); - ossl_quic_tx_packetiser_set_qlog_cb(ch->txp, ch_get_qlog_cb, ch); - - /* Plug in secrets for the Initial EL. */ - if (!ossl_quic_provide_initial_secret(ch->port->engine->libctx, - ch->port->engine->propq, - &ch->init_dcid, - /*is_server=*/1, - ch->qrx, ch->qtx)) - return 0; - - /* Register the peer ODCID in the LCIDM. */ - if (!ossl_quic_lcidm_enrol_odcid(ch->lcidm, ch, &ch->init_dcid)) - return 0; - - /* Change state. */ - ch_record_state_transition(ch, QUIC_CHANNEL_STATE_ACTIVE); - ch->doing_proactive_ver_neg = 0; /* not currently supported */ - return 1; -} - -SSL *ossl_quic_channel_get0_ssl(QUIC_CHANNEL *ch) -{ - return ch->tls; -} - -static int ch_init_new_stream(QUIC_CHANNEL *ch, QUIC_STREAM *qs, - int can_send, int can_recv) -{ - uint64_t rxfc_wnd; - int server_init = ossl_quic_stream_is_server_init(qs); - int local_init = (ch->is_server == server_init); - int is_uni = !ossl_quic_stream_is_bidi(qs); - - if (can_send) - if ((qs->sstream = ossl_quic_sstream_new(INIT_APP_BUF_LEN)) == NULL) - goto err; - - if (can_recv) - if ((qs->rstream = ossl_quic_rstream_new(NULL, NULL, 0)) == NULL) - goto err; - - /* TXFC */ - if (!ossl_quic_txfc_init(&qs->txfc, &ch->conn_txfc)) - goto err; - - if (ch->got_remote_transport_params) { - /* - * If we already got peer TPs we need to apply the initial CWM credit - * now. If we didn't already get peer TPs this will be done - * automatically for all extant streams when we do. - */ - if (can_send) { - uint64_t cwm; - - if (is_uni) - cwm = ch->rx_init_max_stream_data_uni; - else if (local_init) - cwm = ch->rx_init_max_stream_data_bidi_local; - else - cwm = ch->rx_init_max_stream_data_bidi_remote; - - ossl_quic_txfc_bump_cwm(&qs->txfc, cwm); - } - } - - /* RXFC */ - if (!can_recv) - rxfc_wnd = 0; - else if (is_uni) - rxfc_wnd = ch->tx_init_max_stream_data_uni; - else if (local_init) - rxfc_wnd = ch->tx_init_max_stream_data_bidi_local; - else - rxfc_wnd = ch->tx_init_max_stream_data_bidi_remote; - - if (!ossl_quic_rxfc_init(&qs->rxfc, &ch->conn_rxfc, - rxfc_wnd, - DEFAULT_STREAM_RXFC_MAX_WND_MUL * rxfc_wnd, - get_time, ch)) - goto err; - - return 1; - -err: - ossl_quic_sstream_free(qs->sstream); - qs->sstream = NULL; - ossl_quic_rstream_free(qs->rstream); - qs->rstream = NULL; - return 0; -} - -static uint64_t *ch_get_local_stream_next_ordinal_ptr(QUIC_CHANNEL *ch, - int is_uni) -{ - return is_uni ? &ch->next_local_stream_ordinal_uni - : &ch->next_local_stream_ordinal_bidi; -} - -static const uint64_t *ch_get_local_stream_max_ptr(const QUIC_CHANNEL *ch, - int is_uni) -{ - return is_uni ? &ch->max_local_streams_uni - : &ch->max_local_streams_bidi; -} - -static const QUIC_RXFC *ch_get_remote_stream_count_rxfc(const QUIC_CHANNEL *ch, - int is_uni) -{ - return is_uni ? &ch->max_streams_uni_rxfc - : &ch->max_streams_bidi_rxfc; -} - -int ossl_quic_channel_is_new_local_stream_admissible(QUIC_CHANNEL *ch, - int is_uni) -{ - const uint64_t *p_next_ordinal = ch_get_local_stream_next_ordinal_ptr(ch, is_uni); - - return ossl_quic_stream_map_is_local_allowed_by_stream_limit(&ch->qsm, - *p_next_ordinal, - is_uni); -} - -uint64_t ossl_quic_channel_get_local_stream_count_avail(const QUIC_CHANNEL *ch, - int is_uni) -{ - const uint64_t *p_next_ordinal, *p_max; - - p_next_ordinal = ch_get_local_stream_next_ordinal_ptr((QUIC_CHANNEL *)ch, - is_uni); - p_max = ch_get_local_stream_max_ptr(ch, is_uni); - - return *p_max - *p_next_ordinal; -} - -uint64_t ossl_quic_channel_get_remote_stream_count_avail(const QUIC_CHANNEL *ch, - int is_uni) -{ - return ossl_quic_rxfc_get_credit(ch_get_remote_stream_count_rxfc(ch, is_uni)); -} - -QUIC_STREAM *ossl_quic_channel_new_stream_local(QUIC_CHANNEL *ch, int is_uni) -{ - QUIC_STREAM *qs; - int type; - uint64_t stream_id; - uint64_t *p_next_ordinal; - - type = ch->is_server ? QUIC_STREAM_INITIATOR_SERVER - : QUIC_STREAM_INITIATOR_CLIENT; - - p_next_ordinal = ch_get_local_stream_next_ordinal_ptr(ch, is_uni); - - if (is_uni) - type |= QUIC_STREAM_DIR_UNI; - else - type |= QUIC_STREAM_DIR_BIDI; - - if (*p_next_ordinal >= ((uint64_t)1) << 62) - return NULL; - - stream_id = ((*p_next_ordinal) << 2) | type; - - if ((qs = ossl_quic_stream_map_alloc(&ch->qsm, stream_id, type)) == NULL) - return NULL; - - /* Locally-initiated stream, so we always want a send buffer. */ - if (!ch_init_new_stream(ch, qs, /*can_send=*/1, /*can_recv=*/!is_uni)) - goto err; - - ++*p_next_ordinal; - return qs; - -err: - ossl_quic_stream_map_release(&ch->qsm, qs); - return NULL; -} - -QUIC_STREAM *ossl_quic_channel_new_stream_remote(QUIC_CHANNEL *ch, - uint64_t stream_id) -{ - uint64_t peer_role; - int is_uni; - QUIC_STREAM *qs; - - peer_role = ch->is_server - ? QUIC_STREAM_INITIATOR_CLIENT - : QUIC_STREAM_INITIATOR_SERVER; - - if ((stream_id & QUIC_STREAM_INITIATOR_MASK) != peer_role) - return NULL; - - is_uni = ((stream_id & QUIC_STREAM_DIR_MASK) == QUIC_STREAM_DIR_UNI); - - qs = ossl_quic_stream_map_alloc(&ch->qsm, stream_id, - stream_id & (QUIC_STREAM_INITIATOR_MASK - | QUIC_STREAM_DIR_MASK)); - if (qs == NULL) - return NULL; - - if (!ch_init_new_stream(ch, qs, /*can_send=*/!is_uni, /*can_recv=*/1)) - goto err; - - if (ch->incoming_stream_auto_reject) - ossl_quic_channel_reject_stream(ch, qs); - else - ossl_quic_stream_map_push_accept_queue(&ch->qsm, qs); - - return qs; - -err: - ossl_quic_stream_map_release(&ch->qsm, qs); - return NULL; -} - -void ossl_quic_channel_set_incoming_stream_auto_reject(QUIC_CHANNEL *ch, - int enable, - uint64_t aec) -{ - ch->incoming_stream_auto_reject = (enable != 0); - ch->incoming_stream_auto_reject_aec = aec; -} - -void ossl_quic_channel_reject_stream(QUIC_CHANNEL *ch, QUIC_STREAM *qs) -{ - ossl_quic_stream_map_stop_sending_recv_part(&ch->qsm, qs, - ch->incoming_stream_auto_reject_aec); - - ossl_quic_stream_map_reset_stream_send_part(&ch->qsm, qs, - ch->incoming_stream_auto_reject_aec); - qs->deleted = 1; - - ossl_quic_stream_map_update_state(&ch->qsm, qs); -} - -/* Replace local connection ID in TXP and DEMUX for testing purposes. */ -int ossl_quic_channel_replace_local_cid(QUIC_CHANNEL *ch, - const QUIC_CONN_ID *conn_id) -{ - /* Remove the current LCID from the LCIDM. */ - if (!ossl_quic_lcidm_debug_remove(ch->lcidm, &ch->cur_local_cid)) - return 0; - ch->cur_local_cid = *conn_id; - /* Set in the TXP, used only for long header packets. */ - if (!ossl_quic_tx_packetiser_set_cur_scid(ch->txp, &ch->cur_local_cid)) - return 0; - /* Add the new LCID to the LCIDM. */ - if (!ossl_quic_lcidm_debug_add(ch->lcidm, ch, &ch->cur_local_cid, - 100)) - return 0; - return 1; -} - -void ossl_quic_channel_set_msg_callback(QUIC_CHANNEL *ch, - ossl_msg_cb msg_callback, - SSL *msg_callback_ssl) -{ - ch->msg_callback = msg_callback; - ch->msg_callback_ssl = msg_callback_ssl; - ossl_qtx_set_msg_callback(ch->qtx, msg_callback, msg_callback_ssl); - ossl_quic_tx_packetiser_set_msg_callback(ch->txp, msg_callback, - msg_callback_ssl); - ossl_qrx_set_msg_callback(ch->qrx, msg_callback, msg_callback_ssl); -} - -void ossl_quic_channel_set_msg_callback_arg(QUIC_CHANNEL *ch, - void *msg_callback_arg) -{ - ch->msg_callback_arg = msg_callback_arg; - ossl_qtx_set_msg_callback_arg(ch->qtx, msg_callback_arg); - ossl_quic_tx_packetiser_set_msg_callback_arg(ch->txp, msg_callback_arg); - ossl_qrx_set_msg_callback_arg(ch->qrx, msg_callback_arg); -} - -void ossl_quic_channel_set_txku_threshold_override(QUIC_CHANNEL *ch, - uint64_t tx_pkt_threshold) -{ - ch->txku_threshold_override = tx_pkt_threshold; -} - -uint64_t ossl_quic_channel_get_tx_key_epoch(QUIC_CHANNEL *ch) -{ - return ossl_qtx_get_key_epoch(ch->qtx); -} - -uint64_t ossl_quic_channel_get_rx_key_epoch(QUIC_CHANNEL *ch) -{ - return ossl_qrx_get_key_epoch(ch->qrx); -} - -int ossl_quic_channel_trigger_txku(QUIC_CHANNEL *ch) -{ - if (!txku_allowed(ch)) - return 0; - - ch->ku_locally_initiated = 1; - ch_trigger_txku(ch); - return 1; -} - -int ossl_quic_channel_ping(QUIC_CHANNEL *ch) -{ - int pn_space = ossl_quic_enc_level_to_pn_space(ch->tx_enc_level); - - ossl_quic_tx_packetiser_schedule_ack_eliciting(ch->txp, pn_space); - - return 1; -} - -uint16_t ossl_quic_channel_get_diag_num_rx_ack(QUIC_CHANNEL *ch) -{ - return ch->diag_num_rx_ack; -} - -void ossl_quic_channel_get_diag_local_cid(QUIC_CHANNEL *ch, QUIC_CONN_ID *cid) -{ - *cid = ch->cur_local_cid; -} - -int ossl_quic_channel_have_generated_transport_params(const QUIC_CHANNEL *ch) -{ - return ch->got_local_transport_params; -} - -void ossl_quic_channel_set_max_idle_timeout_request(QUIC_CHANNEL *ch, uint64_t ms) -{ - ch->max_idle_timeout_local_req = ms; -} -uint64_t ossl_quic_channel_get_max_idle_timeout_request(const QUIC_CHANNEL *ch) -{ - return ch->max_idle_timeout_local_req; -} - -uint64_t ossl_quic_channel_get_max_idle_timeout_peer_request(const QUIC_CHANNEL *ch) -{ - return ch->max_idle_timeout_remote_req; -} - -uint64_t ossl_quic_channel_get_max_idle_timeout_actual(const QUIC_CHANNEL *ch) -{ - return ch->max_idle_timeout; -} diff --git a/openssl/src/ssl/quic/quic_channel_local.h b/openssl/src/ssl/quic/quic_channel_local.h deleted file mode 100644 index 16d96ef7d..000000000 --- a/openssl/src/ssl/quic/quic_channel_local.h +++ /dev/null @@ -1,453 +0,0 @@ -#ifndef OSSL_QUIC_CHANNEL_LOCAL_H -# define OSSL_QUIC_CHANNEL_LOCAL_H - -# include "internal/quic_channel.h" - -# ifndef OPENSSL_NO_QUIC - -# include -# include "internal/list.h" -# include "internal/quic_predef.h" -# include "internal/quic_fc.h" -# include "internal/quic_stream_map.h" - -/* - * QUIC Channel Structure - * ====================== - * - * QUIC channel internals. It is intended that only the QUIC_CHANNEL - * implementation and the RX depacketiser be allowed to access this structure - * directly. As the RX depacketiser has no state of its own and computes over a - * QUIC_CHANNEL structure, it can be viewed as an extension of the QUIC_CHANNEL - * implementation. While the RX depacketiser could be provided with adequate - * accessors to do what it needs, this would weaken the abstraction provided by - * the QUIC_CHANNEL to other components; moreover the coupling of the RX - * depacketiser to QUIC_CHANNEL internals is too deep and bespoke to make this - * desirable. - * - * Other components should not include this header. - */ -struct quic_channel_st { - QUIC_PORT *port; - - /* - * QUIC_PORT keeps the channels which belong to it on a list for bookkeeping - * purposes. - */ - OSSL_LIST_MEMBER(ch, struct quic_channel_st); - - /* - * The associated TLS 1.3 connection data. Used to provide the handshake - * layer; its 'network' side is plugged into the crypto stream for each EL - * (other than the 0-RTT EL). - */ - QUIC_TLS *qtls; - SSL *tls; - - /* Port LCIDM we use to register LCIDs. */ - QUIC_LCIDM *lcidm; - /* SRTM we register SRTs with. */ - QUIC_SRTM *srtm; - - /* Optional QLOG instance (or NULL). */ - QLOG *qlog; - - /* - * The transport parameter block we will send or have sent. - * Freed after sending or when connection is freed. - */ - unsigned char *local_transport_params; - - /* Our current L4 peer address, if any. */ - BIO_ADDR cur_peer_addr; - - /* - * Subcomponents of the connection. All of these components are instantiated - * and owned by us. - */ - OSSL_QUIC_TX_PACKETISER *txp; - QUIC_TXPIM *txpim; - QUIC_CFQ *cfq; - /* - * Connection level FC. The stream_count RXFCs is used to manage - * MAX_STREAMS signalling. - */ - QUIC_TXFC conn_txfc; - QUIC_RXFC conn_rxfc, crypto_rxfc[QUIC_PN_SPACE_NUM]; - QUIC_RXFC max_streams_bidi_rxfc, max_streams_uni_rxfc; - QUIC_STREAM_MAP qsm; - OSSL_STATM statm; - OSSL_CC_DATA *cc_data; - const OSSL_CC_METHOD *cc_method; - OSSL_ACKM *ackm; - - /* Record layers in the TX and RX directions. */ - OSSL_QTX *qtx; - OSSL_QRX *qrx; - - /* Message callback related arguments */ - ossl_msg_cb msg_callback; - void *msg_callback_arg; - SSL *msg_callback_ssl; - - /* - * Send and receive parts of the crypto streams. - * crypto_send[QUIC_PN_SPACE_APP] is the 1-RTT crypto stream. There is no - * 0-RTT crypto stream. - */ - QUIC_SSTREAM *crypto_send[QUIC_PN_SPACE_NUM]; - QUIC_RSTREAM *crypto_recv[QUIC_PN_SPACE_NUM]; - - /* Internal state. */ - /* - * Client: The DCID used in the first Initial packet we transmit as a client. - * Server: The DCID used in the first Initial packet the client transmitted. - * Randomly generated and required by RFC to be at least 8 bytes. - */ - QUIC_CONN_ID init_dcid; - - /* - * Client: The SCID found in the first Initial packet from the server. - * Not valid for servers. - * Valid if have_received_enc_pkt is set. - */ - QUIC_CONN_ID init_scid; - - /* - * Client only: The SCID found in an incoming Retry packet we handled. - * Not valid for servers. - */ - QUIC_CONN_ID retry_scid; - - /* Server only: The DCID we currently expect the peer to use to talk to us. */ - QUIC_CONN_ID cur_local_cid; - - /* - * The DCID we currently use to talk to the peer and its sequence num. - */ - QUIC_CONN_ID cur_remote_dcid; - uint64_t cur_remote_seq_num; - uint64_t cur_retire_prior_to; - - /* Transport parameter values we send to our peer. */ - uint64_t tx_init_max_stream_data_bidi_local; - uint64_t tx_init_max_stream_data_bidi_remote; - uint64_t tx_init_max_stream_data_uni; - uint64_t tx_max_ack_delay; /* ms */ - - /* Transport parameter values received from server. */ - uint64_t rx_init_max_stream_data_bidi_local; - uint64_t rx_init_max_stream_data_bidi_remote; - uint64_t rx_init_max_stream_data_uni; - uint64_t rx_max_ack_delay; /* ms */ - unsigned char rx_ack_delay_exp; - - /* Diagnostic counters for testing purposes only. May roll over. */ - uint16_t diag_num_rx_ack; /* Number of ACK frames received */ - - /* - * Temporary staging area to store information about the incoming packet we - * are currently processing. - */ - OSSL_QRX_PKT *qrx_pkt; - - /* - * Current limit on number of streams we may create. Set by transport - * parameters initially and then by MAX_STREAMS frames. - */ - uint64_t max_local_streams_bidi; - uint64_t max_local_streams_uni; - - /* The idle timeout values we and our peer requested. */ - uint64_t max_idle_timeout_local_req; - uint64_t max_idle_timeout_remote_req; - - /* The negotiated maximum idle timeout in milliseconds. */ - uint64_t max_idle_timeout; - - /* - * Maximum payload size in bytes for datagrams sent to our peer, as - * negotiated by transport parameters. - */ - uint64_t rx_max_udp_payload_size; - /* Maximum active CID limit, as negotiated by transport parameters. */ - uint64_t rx_active_conn_id_limit; - - /* - * Used to allocate stream IDs. This is a stream ordinal, i.e., a stream ID - * without the low two bits designating type and initiator. Shift and or in - * the type bits to convert to a stream ID. - */ - uint64_t next_local_stream_ordinal_bidi; - uint64_t next_local_stream_ordinal_uni; - - /* - * Used to track which stream ordinals within a given stream type have been - * used by the remote peer. This is an optimisation used to determine - * which streams should be implicitly created due to usage of a higher - * stream ordinal. - */ - uint64_t next_remote_stream_ordinal_bidi; - uint64_t next_remote_stream_ordinal_uni; - - /* - * Application error code to be used for STOP_SENDING/RESET_STREAM frames - * used to autoreject incoming streams. - */ - uint64_t incoming_stream_auto_reject_aec; - - /* - * Override packet count threshold at which we do a spontaneous TXKU. - * Usually UINT64_MAX in which case a suitable value is chosen based on AEAD - * limit advice from the QRL utility functions. This is intended for testing - * use only. Usually set to UINT64_MAX. - */ - uint64_t txku_threshold_override; - - /* Valid if we are in the TERMINATING or TERMINATED states. */ - QUIC_TERMINATE_CAUSE terminate_cause; - - /* - * Deadline at which we move to TERMINATING state. Valid if in the - * TERMINATING state. - */ - OSSL_TIME terminate_deadline; - - /* - * Deadline at which connection dies due to idle timeout if no further - * events occur. - */ - OSSL_TIME idle_deadline; - - /* - * Deadline at which we should send an ACK-eliciting packet to ensure - * idle timeout does not occur. - */ - OSSL_TIME ping_deadline; - - /* - * The deadline at which the period in which it is RECOMMENDED that we not - * initiate any spontaneous TXKU ends. This is zero if no such deadline - * applies. - */ - OSSL_TIME txku_cooldown_deadline; - - /* - * The deadline at which we take the QRX out of UPDATING and back to NORMAL. - * Valid if rxku_in_progress in 1. - */ - OSSL_TIME rxku_update_end_deadline; - - /* - * The first (application space) PN sent with a new key phase. Valid if the - * QTX key epoch is greater than 0. Once a packet we sent with a PN p (p >= - * txku_pn) is ACKed, the TXKU is considered completed and txku_in_progress - * becomes 0. For sanity's sake, such a PN p should also be <= the highest - * PN we have ever sent, of course. - */ - QUIC_PN txku_pn; - - /* - * The (application space) PN which triggered RXKU detection. Valid if - * rxku_pending_confirm. - */ - QUIC_PN rxku_trigger_pn; - - /* - * State tracking. QUIC connection-level state is best represented based on - * whether various things have happened yet or not, rather than as an - * explicit FSM. We do have a coarse state variable which tracks the basic - * state of the connection's lifecycle, but more fine-grained conditions of - * the Active state are tracked via flags below. For more details, see - * doc/designs/quic-design/connection-state-machine.md. We are in the Open - * state if the state is QUIC_CHANNEL_STATE_ACTIVE and handshake_confirmed is - * set. - */ - unsigned int state : 3; - - /* - * Have we received at least one encrypted packet from the peer? - * (If so, Retry and Version Negotiation messages should no longer - * be received and should be ignored if they do occur.) - */ - unsigned int have_received_enc_pkt : 1; - - /* - * Have we successfully processed any packet, including a Version - * Negotiation packet? If so, further Version Negotiation packets should be - * ignored. - */ - unsigned int have_processed_any_pkt : 1; - - /* - * Have we sent literally any packet yet? If not, there is no point polling - * RX. - */ - unsigned int have_sent_any_pkt : 1; - - /* - * Are we currently doing proactive version negotiation? - */ - unsigned int doing_proactive_ver_neg : 1; - - /* We have received transport parameters from the peer. */ - unsigned int got_remote_transport_params : 1; - /* We have generated our local transport parameters. */ - unsigned int got_local_transport_params : 1; - - /* - * This monotonically transitions to 1 once the TLS state machine is - * 'complete', meaning that it has both sent a Finished and successfully - * verified the peer's Finished (see RFC 9001 s. 4.1.1). Note that it - * does not transition to 1 at both peers simultaneously. - * - * Handshake completion is not the same as handshake confirmation (see - * below). - */ - unsigned int handshake_complete : 1; - - /* - * This monotonically transitions to 1 once the handshake is confirmed. - * This happens on the client when we receive a HANDSHAKE_DONE frame. - * At our option, we may also take acknowledgement of any 1-RTT packet - * we sent as a handshake confirmation. - */ - unsigned int handshake_confirmed : 1; - - /* - * We are sending Initial packets based on a Retry. This means we definitely - * should not receive another Retry, and if we do it is an error. - */ - unsigned int doing_retry : 1; - - /* - * We don't store the current EL here; the TXP asks the QTX which ELs - * are provisioned to determine which ELs to use. - */ - - /* Have statm, qsm been initialised? Used to track cleanup. */ - unsigned int have_statm : 1; - unsigned int have_qsm : 1; - - /* - * Preferred ELs for transmission and reception. This is not strictly needed - * as it can be inferred from what keys we have provisioned, but makes - * determining the current EL simpler and faster. A separate EL for - * transmission and reception is not strictly necessary but makes things - * easier for interoperation with the handshake layer, which likes to invoke - * the yield secret callback at different times for TX and RX. - */ - unsigned int tx_enc_level : 3; - unsigned int rx_enc_level : 3; - - /* If bit n is set, EL n has been discarded. */ - unsigned int el_discarded : 4; - - /* - * While in TERMINATING - CLOSING, set when we should generate a connection - * close frame. - */ - unsigned int conn_close_queued : 1; - - /* Are we in server mode? Never changes after instantiation. */ - unsigned int is_server : 1; - - /* - * Set temporarily when the handshake layer has given us a new RX secret. - * Used to determine if we need to check our RX queues again. - */ - unsigned int have_new_rx_secret : 1; - - /* Have we ever called QUIC_TLS yet during RX processing? */ - unsigned int did_tls_tick : 1; - /* Has any CRYPTO frame been processed during this tick? */ - unsigned int did_crypto_frame : 1; - - /* - * Have we sent an ack-eliciting packet since the last successful packet - * reception? Used to determine when to bump idle timer (see RFC 9000 s. - * 10.1). - */ - unsigned int have_sent_ack_eliciting_since_rx : 1; - - /* Should incoming streams automatically be rejected? */ - unsigned int incoming_stream_auto_reject : 1; - - /* - * 1 if a key update sequence was locally initiated, meaning we sent the - * TXKU first and the resultant RXKU shouldn't result in our triggering - * another TXKU. 0 if a key update sequence was initiated by the peer, - * meaning we detect a RXKU first and have to generate a TXKU in response. - */ - unsigned int ku_locally_initiated : 1; - - /* - * 1 if we have triggered TXKU (whether spontaneous or solicited) but are - * waiting for any PN using that new KP to be ACKed. While this is set, we - * are not allowed to trigger spontaneous TXKU (but solicited TXKU is - * potentially still possible). - */ - unsigned int txku_in_progress : 1; - - /* - * We have received an RXKU event and currently are going through - * UPDATING/COOLDOWN on the QRX. COOLDOWN is currently not used. Since RXKU - * cannot be detected in this state, this doesn't cause a protocol error or - * anything similar if a peer tries TXKU in this state. That traffic would - * simply be dropped. It's only used to track that our UPDATING timer is - * active so we know when to take the QRX out of UPDATING and back to - * NORMAL. - */ - unsigned int rxku_in_progress : 1; - - /* - * We have received an RXKU but have yet to send an ACK for it, which means - * no further RXKUs are allowed yet. Note that we cannot detect further - * RXKUs anyway while the QRX remains in the UPDATING/COOLDOWN states, so - * this restriction comes into play if we take more than PTO time to send - * an ACK for it (not likely). - */ - unsigned int rxku_pending_confirm : 1; - - /* Temporary variable indicating rxku_pending_confirm is to become 0. */ - unsigned int rxku_pending_confirm_done : 1; - - /* - * If set, RXKU is expected (because we initiated a spontaneous TXKU). - */ - unsigned int rxku_expected : 1; - - /* Permanent net error encountered */ - unsigned int net_error : 1; - - /* - * Protocol error encountered. Note that you should refer to the state field - * rather than this. This is only used so we can ignore protocol errors - * after the first protocol error, but still record the first protocol error - * if it happens during the TERMINATING state. - */ - unsigned int protocol_error : 1; - - /* Are we using addressed mode? */ - unsigned int addressed_mode : 1; - - /* Are we on the QUIC_PORT linked list of channels? */ - unsigned int on_port_list : 1; - - /* Has qlog been requested? */ - unsigned int use_qlog : 1; - - /* Saved error stack in case permanent error was encountered */ - ERR_STATE *err_state; - - /* Scratch area for use by RXDP to store decoded ACK ranges. */ - OSSL_QUIC_ACK_RANGE *ack_range_scratch; - size_t num_ack_range_scratch; - - /* Title for qlog purposes. We own this copy. */ - char *qlog_title; -}; - -# endif - -#endif diff --git a/openssl/src/ssl/quic/quic_demux.c b/openssl/src/ssl/quic/quic_demux.c deleted file mode 100644 index e3b5ca191..000000000 --- a/openssl/src/ssl/quic/quic_demux.c +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_demux.h" -#include "internal/quic_wire_pkt.h" -#include "internal/common.h" -#include -#include - -#define URXE_DEMUX_STATE_FREE 0 /* on urx_free list */ -#define URXE_DEMUX_STATE_PENDING 1 /* on urx_pending list */ -#define URXE_DEMUX_STATE_ISSUED 2 /* on neither list */ - -#define DEMUX_MAX_MSGS_PER_CALL 32 - -#define DEMUX_DEFAULT_MTU 1500 - -struct quic_demux_st { - /* The underlying transport BIO with datagram semantics. */ - BIO *net_bio; - - /* - * QUIC short packets do not contain the length of the connection ID field, - * therefore it must be known contextually. The demuxer requires connection - * IDs of the same length to be used for all incoming packets. - */ - size_t short_conn_id_len; - - /* - * Our current understanding of the upper bound on an incoming datagram size - * in bytes. - */ - size_t mtu; - - /* The datagram_id to use for the next datagram we receive. */ - uint64_t next_datagram_id; - - /* Time retrieval callback. */ - OSSL_TIME (*now)(void *arg); - void *now_arg; - - /* The default packet handler, if any. */ - ossl_quic_demux_cb_fn *default_cb; - void *default_cb_arg; - - /* - * List of URXEs which are not currently in use (i.e., not filled with - * unconsumed data). These are moved to the pending list as they are filled. - */ - QUIC_URXE_LIST urx_free; - - /* - * List of URXEs which are filled with received encrypted data. These are - * removed from this list as we invoke the callbacks for each of them. They - * are then not on any list managed by us; we forget about them until our - * user calls ossl_quic_demux_release_urxe to return the URXE to us, at - * which point we add it to the free list. - */ - QUIC_URXE_LIST urx_pending; - - /* Whether to use local address support. */ - char use_local_addr; -}; - -QUIC_DEMUX *ossl_quic_demux_new(BIO *net_bio, - size_t short_conn_id_len, - OSSL_TIME (*now)(void *arg), - void *now_arg) -{ - QUIC_DEMUX *demux; - - demux = OPENSSL_zalloc(sizeof(QUIC_DEMUX)); - if (demux == NULL) - return NULL; - - demux->net_bio = net_bio; - demux->short_conn_id_len = short_conn_id_len; - /* We update this if possible when we get a BIO. */ - demux->mtu = DEMUX_DEFAULT_MTU; - demux->now = now; - demux->now_arg = now_arg; - - if (net_bio != NULL - && BIO_dgram_get_local_addr_cap(net_bio) - && BIO_dgram_set_local_addr_enable(net_bio, 1)) - demux->use_local_addr = 1; - - return demux; -} - -static void demux_free_urxl(QUIC_URXE_LIST *l) -{ - QUIC_URXE *e, *enext; - - for (e = ossl_list_urxe_head(l); e != NULL; e = enext) { - enext = ossl_list_urxe_next(e); - ossl_list_urxe_remove(l, e); - OPENSSL_free(e); - } -} - -void ossl_quic_demux_free(QUIC_DEMUX *demux) -{ - if (demux == NULL) - return; - - /* Free all URXEs we are holding. */ - demux_free_urxl(&demux->urx_free); - demux_free_urxl(&demux->urx_pending); - - OPENSSL_free(demux); -} - -void ossl_quic_demux_set_bio(QUIC_DEMUX *demux, BIO *net_bio) -{ - unsigned int mtu; - - demux->net_bio = net_bio; - - if (net_bio != NULL) { - /* - * Try to determine our MTU if possible. The BIO is not required to - * support this, in which case we remain at the last known MTU, or our - * initial default. - */ - mtu = BIO_dgram_get_mtu(net_bio); - if (mtu >= QUIC_MIN_INITIAL_DGRAM_LEN) - ossl_quic_demux_set_mtu(demux, mtu); /* best effort */ - } -} - -int ossl_quic_demux_set_mtu(QUIC_DEMUX *demux, unsigned int mtu) -{ - if (mtu < QUIC_MIN_INITIAL_DGRAM_LEN) - return 0; - - demux->mtu = mtu; - return 1; -} - -void ossl_quic_demux_set_default_handler(QUIC_DEMUX *demux, - ossl_quic_demux_cb_fn *cb, - void *cb_arg) -{ - demux->default_cb = cb; - demux->default_cb_arg = cb_arg; -} - -static QUIC_URXE *demux_alloc_urxe(size_t alloc_len) -{ - QUIC_URXE *e; - - if (alloc_len >= SIZE_MAX - sizeof(QUIC_URXE)) - return NULL; - - e = OPENSSL_malloc(sizeof(QUIC_URXE) + alloc_len); - if (e == NULL) - return NULL; - - ossl_list_urxe_init_elem(e); - e->alloc_len = alloc_len; - e->data_len = 0; - return e; -} - -static QUIC_URXE *demux_resize_urxe(QUIC_DEMUX *demux, QUIC_URXE *e, - size_t new_alloc_len) -{ - QUIC_URXE *e2, *prev; - - if (!ossl_assert(e->demux_state == URXE_DEMUX_STATE_FREE)) - /* Never attempt to resize a URXE which is not on the free list. */ - return NULL; - - prev = ossl_list_urxe_prev(e); - ossl_list_urxe_remove(&demux->urx_free, e); - - e2 = OPENSSL_realloc(e, sizeof(QUIC_URXE) + new_alloc_len); - if (e2 == NULL) { - /* Failed to resize, abort. */ - if (prev == NULL) - ossl_list_urxe_insert_head(&demux->urx_free, e); - else - ossl_list_urxe_insert_after(&demux->urx_free, prev, e); - - return NULL; - } - - if (prev == NULL) - ossl_list_urxe_insert_head(&demux->urx_free, e2); - else - ossl_list_urxe_insert_after(&demux->urx_free, prev, e2); - - e2->alloc_len = new_alloc_len; - return e2; -} - -static QUIC_URXE *demux_reserve_urxe(QUIC_DEMUX *demux, QUIC_URXE *e, - size_t alloc_len) -{ - return e->alloc_len < alloc_len ? demux_resize_urxe(demux, e, alloc_len) : e; -} - -static int demux_ensure_free_urxe(QUIC_DEMUX *demux, size_t min_num_free) -{ - QUIC_URXE *e; - - while (ossl_list_urxe_num(&demux->urx_free) < min_num_free) { - e = demux_alloc_urxe(demux->mtu); - if (e == NULL) - return 0; - - ossl_list_urxe_insert_tail(&demux->urx_free, e); - e->demux_state = URXE_DEMUX_STATE_FREE; - } - - return 1; -} - -/* - * Receive datagrams from network, placing them into URXEs. - * - * Returns 1 on success or 0 on failure. - * - * Precondition: at least one URXE is free - * Precondition: there are no pending URXEs - */ -static int demux_recv(QUIC_DEMUX *demux) -{ - BIO_MSG msg[DEMUX_MAX_MSGS_PER_CALL]; - size_t rd, i; - QUIC_URXE *urxe = ossl_list_urxe_head(&demux->urx_free), *unext; - OSSL_TIME now; - - /* This should never be called when we have any pending URXE. */ - assert(ossl_list_urxe_head(&demux->urx_pending) == NULL); - assert(urxe->demux_state == URXE_DEMUX_STATE_FREE); - - if (demux->net_bio == NULL) - /* - * If no BIO is plugged in, treat this as no datagram being available. - */ - return QUIC_DEMUX_PUMP_RES_TRANSIENT_FAIL; - - /* - * Opportunistically receive as many messages as possible in a single - * syscall, determined by how many free URXEs are available. - */ - for (i = 0; i < (ossl_ssize_t)OSSL_NELEM(msg); - ++i, urxe = ossl_list_urxe_next(urxe)) { - if (urxe == NULL) { - /* We need at least one URXE to receive into. */ - if (!ossl_assert(i > 0)) - return QUIC_DEMUX_PUMP_RES_PERMANENT_FAIL; - - break; - } - - /* Ensure the URXE is big enough. */ - urxe = demux_reserve_urxe(demux, urxe, demux->mtu); - if (urxe == NULL) - /* Allocation error, fail. */ - return QUIC_DEMUX_PUMP_RES_PERMANENT_FAIL; - - /* Ensure we zero any fields added to BIO_MSG at a later date. */ - memset(&msg[i], 0, sizeof(BIO_MSG)); - msg[i].data = ossl_quic_urxe_data(urxe); - msg[i].data_len = urxe->alloc_len; - msg[i].peer = &urxe->peer; - BIO_ADDR_clear(&urxe->peer); - if (demux->use_local_addr) - msg[i].local = &urxe->local; - else - BIO_ADDR_clear(&urxe->local); - } - - ERR_set_mark(); - if (!BIO_recvmmsg(demux->net_bio, msg, sizeof(BIO_MSG), i, 0, &rd)) { - if (BIO_err_is_non_fatal(ERR_peek_last_error())) { - /* Transient error, clear the error and stop. */ - ERR_pop_to_mark(); - return QUIC_DEMUX_PUMP_RES_TRANSIENT_FAIL; - } else { - /* Non-transient error, do not clear the error. */ - ERR_clear_last_mark(); - return QUIC_DEMUX_PUMP_RES_PERMANENT_FAIL; - } - } - - ERR_clear_last_mark(); - now = demux->now != NULL ? demux->now(demux->now_arg) : ossl_time_zero(); - - urxe = ossl_list_urxe_head(&demux->urx_free); - for (i = 0; i < rd; ++i, urxe = unext) { - unext = ossl_list_urxe_next(urxe); - /* Set URXE with actual length of received datagram. */ - urxe->data_len = msg[i].data_len; - /* Time we received datagram. */ - urxe->time = now; - urxe->datagram_id = demux->next_datagram_id++; - /* Move from free list to pending list. */ - ossl_list_urxe_remove(&demux->urx_free, urxe); - ossl_list_urxe_insert_tail(&demux->urx_pending, urxe); - urxe->demux_state = URXE_DEMUX_STATE_PENDING; - } - - return QUIC_DEMUX_PUMP_RES_OK; -} - -/* Extract destination connection ID from the first packet in a datagram. */ -static int demux_identify_conn_id(QUIC_DEMUX *demux, - QUIC_URXE *e, - QUIC_CONN_ID *dst_conn_id) -{ - return ossl_quic_wire_get_pkt_hdr_dst_conn_id(ossl_quic_urxe_data(e), - e->data_len, - demux->short_conn_id_len, - dst_conn_id); -} - -/* - * Process a single pending URXE. - * Returning 1 on success, 0 on failure. - */ -static int demux_process_pending_urxe(QUIC_DEMUX *demux, QUIC_URXE *e) -{ - QUIC_CONN_ID dst_conn_id; - int dst_conn_id_ok = 0; - - /* The next URXE we process should be at the head of the pending list. */ - if (!ossl_assert(e == ossl_list_urxe_head(&demux->urx_pending))) - return 0; - - assert(e->demux_state == URXE_DEMUX_STATE_PENDING); - - /* Determine the DCID of the first packet in the datagram. */ - dst_conn_id_ok = demux_identify_conn_id(demux, e, &dst_conn_id); - - ossl_list_urxe_remove(&demux->urx_pending, e); - if (demux->default_cb != NULL) { - /* - * Pass to default handler for routing. The URXE now belongs to the - * callback. - */ - e->demux_state = URXE_DEMUX_STATE_ISSUED; - demux->default_cb(e, demux->default_cb_arg, - dst_conn_id_ok ? &dst_conn_id : NULL); - } else { - /* Discard. */ - ossl_list_urxe_insert_tail(&demux->urx_free, e); - e->demux_state = URXE_DEMUX_STATE_FREE; - } - - return 1; /* keep processing pending URXEs */ -} - -/* Process pending URXEs to generate callbacks. */ -static int demux_process_pending_urxl(QUIC_DEMUX *demux) -{ - QUIC_URXE *e; - int ret; - - while ((e = ossl_list_urxe_head(&demux->urx_pending)) != NULL) - if ((ret = demux_process_pending_urxe(demux, e)) <= 0) - return ret; - - return 1; -} - -/* - * Drain the pending URXE list, processing any pending URXEs by making their - * callbacks. If no URXEs are pending, a network read is attempted first. - */ -int ossl_quic_demux_pump(QUIC_DEMUX *demux) -{ - int ret; - - if (ossl_list_urxe_head(&demux->urx_pending) == NULL) { - ret = demux_ensure_free_urxe(demux, DEMUX_MAX_MSGS_PER_CALL); - if (ret != 1) - return QUIC_DEMUX_PUMP_RES_PERMANENT_FAIL; - - ret = demux_recv(demux); - if (ret != QUIC_DEMUX_PUMP_RES_OK) - return ret; - - /* - * If demux_recv returned successfully, we should always have something. - */ - assert(ossl_list_urxe_head(&demux->urx_pending) != NULL); - } - - if ((ret = demux_process_pending_urxl(demux)) <= 0) - return QUIC_DEMUX_PUMP_RES_PERMANENT_FAIL; - - return QUIC_DEMUX_PUMP_RES_OK; -} - -/* Artificially inject a packet into the demuxer for testing purposes. */ -int ossl_quic_demux_inject(QUIC_DEMUX *demux, - const unsigned char *buf, - size_t buf_len, - const BIO_ADDR *peer, - const BIO_ADDR *local) -{ - int ret; - QUIC_URXE *urxe; - - ret = demux_ensure_free_urxe(demux, 1); - if (ret != 1) - return 0; - - urxe = ossl_list_urxe_head(&demux->urx_free); - - assert(urxe->demux_state == URXE_DEMUX_STATE_FREE); - - urxe = demux_reserve_urxe(demux, urxe, buf_len); - if (urxe == NULL) - return 0; - - memcpy(ossl_quic_urxe_data(urxe), buf, buf_len); - urxe->data_len = buf_len; - - if (peer != NULL) - urxe->peer = *peer; - else - BIO_ADDR_clear(&urxe->peer); - - if (local != NULL) - urxe->local = *local; - else - BIO_ADDR_clear(&urxe->local); - - urxe->time - = demux->now != NULL ? demux->now(demux->now_arg) : ossl_time_zero(); - - /* Move from free list to pending list. */ - ossl_list_urxe_remove(&demux->urx_free, urxe); - ossl_list_urxe_insert_tail(&demux->urx_pending, urxe); - urxe->demux_state = URXE_DEMUX_STATE_PENDING; - - return demux_process_pending_urxl(demux) > 0; -} - -/* Called by our user to return a URXE to the free list. */ -void ossl_quic_demux_release_urxe(QUIC_DEMUX *demux, - QUIC_URXE *e) -{ - assert(ossl_list_urxe_prev(e) == NULL && ossl_list_urxe_next(e) == NULL); - assert(e->demux_state == URXE_DEMUX_STATE_ISSUED); - ossl_list_urxe_insert_tail(&demux->urx_free, e); - e->demux_state = URXE_DEMUX_STATE_FREE; -} - -void ossl_quic_demux_reinject_urxe(QUIC_DEMUX *demux, - QUIC_URXE *e) -{ - assert(ossl_list_urxe_prev(e) == NULL && ossl_list_urxe_next(e) == NULL); - assert(e->demux_state == URXE_DEMUX_STATE_ISSUED); - ossl_list_urxe_insert_head(&demux->urx_pending, e); - e->demux_state = URXE_DEMUX_STATE_PENDING; -} - -int ossl_quic_demux_has_pending(const QUIC_DEMUX *demux) -{ - return ossl_list_urxe_head(&demux->urx_pending) != NULL; -} diff --git a/openssl/src/ssl/quic/quic_engine.c b/openssl/src/ssl/quic/quic_engine.c deleted file mode 100644 index 3bcb5d681..000000000 --- a/openssl/src/ssl/quic/quic_engine.c +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_engine.h" -#include "internal/quic_port.h" -#include "quic_engine_local.h" -#include "quic_port_local.h" -#include "../ssl_local.h" - -/* - * QUIC Engine - * =========== - */ -static int qeng_init(QUIC_ENGINE *qeng); -static void qeng_cleanup(QUIC_ENGINE *qeng); -static void qeng_tick(QUIC_TICK_RESULT *res, void *arg, uint32_t flags); - -DEFINE_LIST_OF_IMPL(port, QUIC_PORT); - -QUIC_ENGINE *ossl_quic_engine_new(const QUIC_ENGINE_ARGS *args) -{ - QUIC_ENGINE *qeng; - - if ((qeng = OPENSSL_zalloc(sizeof(QUIC_ENGINE))) == NULL) - return NULL; - - qeng->libctx = args->libctx; - qeng->propq = args->propq; - qeng->mutex = args->mutex; - qeng->now_cb = args->now_cb; - qeng->now_cb_arg = args->now_cb_arg; - - if (!qeng_init(qeng)) { - OPENSSL_free(qeng); - return NULL; - } - - return qeng; -} - -void ossl_quic_engine_free(QUIC_ENGINE *qeng) -{ - if (qeng == NULL) - return; - - qeng_cleanup(qeng); - OPENSSL_free(qeng); -} - -static int qeng_init(QUIC_ENGINE *qeng) -{ - ossl_quic_reactor_init(&qeng->rtor, qeng_tick, qeng, ossl_time_zero()); - return 1; -} - -static void qeng_cleanup(QUIC_ENGINE *qeng) -{ - assert(ossl_list_port_num(&qeng->port_list) == 0); -} - -QUIC_REACTOR *ossl_quic_engine_get0_reactor(QUIC_ENGINE *qeng) -{ - return &qeng->rtor; -} - -CRYPTO_MUTEX *ossl_quic_engine_get0_mutex(QUIC_ENGINE *qeng) -{ - return qeng->mutex; -} - -OSSL_TIME ossl_quic_engine_get_time(QUIC_ENGINE *qeng) -{ - if (qeng->now_cb == NULL) - return ossl_time_now(); - - return qeng->now_cb(qeng->now_cb_arg); -} - -void ossl_quic_engine_set_inhibit_tick(QUIC_ENGINE *qeng, int inhibit) -{ - qeng->inhibit_tick = (inhibit != 0); -} - -/* - * QUIC Engine: Child Object Lifecycle Management - * ============================================== - */ - -QUIC_PORT *ossl_quic_engine_create_port(QUIC_ENGINE *qeng, - const QUIC_PORT_ARGS *args) -{ - QUIC_PORT_ARGS largs = *args; - - if (ossl_list_port_num(&qeng->port_list) > 0) - /* TODO(QUIC MULTIPORT): We currently support only one port. */ - return NULL; - - if (largs.engine != NULL) - return NULL; - - largs.engine = qeng; - return ossl_quic_port_new(&largs); -} - -/* - * QUIC Engine: Ticker-Mutator - * ========================== - */ - -/* - * The central ticker function called by the reactor. This does everything, or - * at least everything network I/O related. Best effort - not allowed to fail - * "loudly". - */ -static void qeng_tick(QUIC_TICK_RESULT *res, void *arg, uint32_t flags) -{ - QUIC_ENGINE *qeng = arg; - QUIC_PORT *port; - - res->net_read_desired = 0; - res->net_write_desired = 0; - res->tick_deadline = ossl_time_infinite(); - - if (qeng->inhibit_tick) - return; - - /* Iterate through all ports and service them. */ - LIST_FOREACH(port, port, &qeng->port_list) { - QUIC_TICK_RESULT subr = {0}; - - ossl_quic_port_subtick(port, &subr, flags); - ossl_quic_tick_result_merge_into(res, &subr); - } -} diff --git a/openssl/src/ssl/quic/quic_engine_local.h b/openssl/src/ssl/quic/quic_engine_local.h deleted file mode 100644 index 280fd31dd..000000000 --- a/openssl/src/ssl/quic/quic_engine_local.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_ENGINE_LOCAL_H -# define OSSL_QUIC_ENGINE_LOCAL_H - -# include "internal/quic_engine.h" -# include "internal/quic_reactor.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Engine Structure - * ===================== - * - * QUIC engine internals. It is intended that only the QUIC_ENGINE, QUIC_PORT - * and QUIC_CHANNEL implementations be allowed to access this structure - * directly. - * - * Other components should not include this header. - */ -DECLARE_LIST_OF(port, QUIC_PORT); - -struct quic_engine_st { - /* All objects in a QUIC event domain share the same (libctx, propq). */ - OSSL_LIB_CTX *libctx; - const char *propq; - - /* - * Master synchronisation mutex for the entire QUIC event domain. Used for - * thread assisted mode synchronisation. We don't own this; the instantiator - * of the engine passes it to us and is responsible for freeing it after - * engine destruction. - */ - CRYPTO_MUTEX *mutex; - - /* Callback used to get the current time. */ - OSSL_TIME (*now_cb)(void *arg); - void *now_cb_arg; - - /* Asynchronous I/O reactor. */ - QUIC_REACTOR rtor; - - /* List of all child ports. */ - OSSL_LIST(port) port_list; - - /* Inhibit tick for testing purposes? */ - unsigned int inhibit_tick : 1; -}; - -# endif - -#endif diff --git a/openssl/src/ssl/quic/quic_fc.c b/openssl/src/ssl/quic/quic_fc.c deleted file mode 100644 index 64ef31780..000000000 --- a/openssl/src/ssl/quic/quic_fc.c +++ /dev/null @@ -1,411 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_fc.h" -#include "internal/quic_error.h" -#include "internal/common.h" -#include "internal/safe_math.h" -#include - -OSSL_SAFE_MATH_UNSIGNED(uint64_t, uint64_t) - -/* - * TX Flow Controller (TXFC) - * ========================= - */ - -int ossl_quic_txfc_init(QUIC_TXFC *txfc, QUIC_TXFC *conn_txfc) -{ - if (conn_txfc != NULL && conn_txfc->parent != NULL) - return 0; - - txfc->swm = 0; - txfc->cwm = 0; - txfc->parent = conn_txfc; - txfc->has_become_blocked = 0; - return 1; -} - -QUIC_TXFC *ossl_quic_txfc_get_parent(QUIC_TXFC *txfc) -{ - return txfc->parent; -} - -int ossl_quic_txfc_bump_cwm(QUIC_TXFC *txfc, uint64_t cwm) -{ - if (cwm <= txfc->cwm) - return 0; - - txfc->cwm = cwm; - return 1; -} - -uint64_t ossl_quic_txfc_get_credit_local(QUIC_TXFC *txfc, uint64_t consumed) -{ - assert((txfc->swm + consumed) <= txfc->cwm); - return txfc->cwm - (consumed + txfc->swm); -} - -uint64_t ossl_quic_txfc_get_credit(QUIC_TXFC *txfc, uint64_t consumed) -{ - uint64_t r, conn_r; - - r = ossl_quic_txfc_get_credit_local(txfc, 0); - - if (txfc->parent != NULL) { - assert(txfc->parent->parent == NULL); - conn_r = ossl_quic_txfc_get_credit_local(txfc->parent, consumed); - if (conn_r < r) - r = conn_r; - } - - return r; -} - -int ossl_quic_txfc_consume_credit_local(QUIC_TXFC *txfc, uint64_t num_bytes) -{ - int ok = 1; - uint64_t credit = ossl_quic_txfc_get_credit_local(txfc, 0); - - if (num_bytes > credit) { - ok = 0; - num_bytes = credit; - } - - if (num_bytes > 0 && num_bytes == credit) - txfc->has_become_blocked = 1; - - txfc->swm += num_bytes; - return ok; -} - -int ossl_quic_txfc_consume_credit(QUIC_TXFC *txfc, uint64_t num_bytes) -{ - int ok = ossl_quic_txfc_consume_credit_local(txfc, num_bytes); - - if (txfc->parent != NULL) { - assert(txfc->parent->parent == NULL); - if (!ossl_quic_txfc_consume_credit_local(txfc->parent, num_bytes)) - return 0; - } - - return ok; -} - -int ossl_quic_txfc_has_become_blocked(QUIC_TXFC *txfc, int clear) -{ - int r = txfc->has_become_blocked; - - if (clear) - txfc->has_become_blocked = 0; - - return r; -} - -uint64_t ossl_quic_txfc_get_cwm(QUIC_TXFC *txfc) -{ - return txfc->cwm; -} - -uint64_t ossl_quic_txfc_get_swm(QUIC_TXFC *txfc) -{ - return txfc->swm; -} - -/* - * RX Flow Controller (RXFC) - * ========================= - */ - -int ossl_quic_rxfc_init(QUIC_RXFC *rxfc, QUIC_RXFC *conn_rxfc, - uint64_t initial_window_size, - uint64_t max_window_size, - OSSL_TIME (*now)(void *now_arg), - void *now_arg) -{ - if (conn_rxfc != NULL && conn_rxfc->parent != NULL) - return 0; - - rxfc->swm = 0; - rxfc->cwm = initial_window_size; - rxfc->rwm = 0; - rxfc->esrwm = 0; - rxfc->hwm = 0; - rxfc->cur_window_size = initial_window_size; - rxfc->max_window_size = max_window_size; - rxfc->parent = conn_rxfc; - rxfc->error_code = 0; - rxfc->has_cwm_changed = 0; - rxfc->epoch_start = ossl_time_zero(); - rxfc->now = now; - rxfc->now_arg = now_arg; - rxfc->is_fin = 0; - rxfc->standalone = 0; - return 1; -} - -int ossl_quic_rxfc_init_standalone(QUIC_RXFC *rxfc, - uint64_t initial_window_size, - OSSL_TIME (*now)(void *arg), - void *now_arg) -{ - if (!ossl_quic_rxfc_init(rxfc, NULL, - initial_window_size, initial_window_size, - now, now_arg)) - return 0; - - rxfc->standalone = 1; - return 1; -} - -QUIC_RXFC *ossl_quic_rxfc_get_parent(QUIC_RXFC *rxfc) -{ - return rxfc->parent; -} - -void ossl_quic_rxfc_set_max_window_size(QUIC_RXFC *rxfc, - size_t max_window_size) -{ - rxfc->max_window_size = max_window_size; -} - -static void rxfc_start_epoch(QUIC_RXFC *rxfc) -{ - rxfc->epoch_start = rxfc->now(rxfc->now_arg); - rxfc->esrwm = rxfc->rwm; -} - -static int on_rx_controlled_bytes(QUIC_RXFC *rxfc, uint64_t num_bytes) -{ - int ok = 1; - uint64_t credit = rxfc->cwm - rxfc->swm; - - if (num_bytes > credit) { - ok = 0; - num_bytes = credit; - rxfc->error_code = OSSL_QUIC_ERR_FLOW_CONTROL_ERROR; - } - - rxfc->swm += num_bytes; - return ok; -} - -int ossl_quic_rxfc_on_rx_stream_frame(QUIC_RXFC *rxfc, uint64_t end, int is_fin) -{ - uint64_t delta; - - if (!rxfc->standalone && rxfc->parent == NULL) - return 0; - - if (rxfc->is_fin && ((is_fin && rxfc->hwm != end) || end > rxfc->hwm)) { - /* Stream size cannot change after the stream is finished */ - rxfc->error_code = OSSL_QUIC_ERR_FINAL_SIZE_ERROR; - return 1; /* not a caller error */ - } - - if (is_fin) - rxfc->is_fin = 1; - - if (end > rxfc->hwm) { - delta = end - rxfc->hwm; - rxfc->hwm = end; - - on_rx_controlled_bytes(rxfc, delta); /* result ignored */ - if (rxfc->parent != NULL) - on_rx_controlled_bytes(rxfc->parent, delta); /* result ignored */ - } else if (end < rxfc->hwm && is_fin) { - rxfc->error_code = OSSL_QUIC_ERR_FINAL_SIZE_ERROR; - return 1; /* not a caller error */ - } - - return 1; -} - -/* threshold = 3/4 */ -#define WINDOW_THRESHOLD_NUM 3 -#define WINDOW_THRESHOLD_DEN 4 - -static int rxfc_cwm_bump_desired(QUIC_RXFC *rxfc) -{ - int err = 0; - uint64_t window_rem = rxfc->cwm - rxfc->rwm; - uint64_t threshold - = safe_muldiv_uint64_t(rxfc->cur_window_size, - WINDOW_THRESHOLD_NUM, WINDOW_THRESHOLD_DEN, &err); - - if (err) - /* - * Extremely large window should never occur, but if it does, just use - * 1/2 as the threshold. - */ - threshold = rxfc->cur_window_size / 2; - - /* - * No point emitting a new MAX_STREAM_DATA frame if the stream has a final - * size. - */ - return !rxfc->is_fin && window_rem <= threshold; -} - -static int rxfc_should_bump_window_size(QUIC_RXFC *rxfc, OSSL_TIME rtt) -{ - /* - * dt: time since start of epoch - * b: bytes of window consumed since start of epoch - * dw: proportion of window consumed since start of epoch - * T_window: time it will take to use up the entire window, based on dt, dw - * RTT: The current estimated RTT. - * - * b = rwm - esrwm - * dw = b / window_size - * T_window = dt / dw - * T_window = dt / (b / window_size) - * T_window = (dt * window_size) / b - * - * We bump the window size if T_window < 4 * RTT. - * - * We leave the division by b on the LHS to reduce the risk of overflowing - * our 64-bit nanosecond representation, which will afford plenty of - * precision left over after the division anyway. - */ - uint64_t b = rxfc->rwm - rxfc->esrwm; - OSSL_TIME now, dt, t_window; - - if (b == 0) - return 0; - - now = rxfc->now(rxfc->now_arg); - dt = ossl_time_subtract(now, rxfc->epoch_start); - t_window = ossl_time_muldiv(dt, rxfc->cur_window_size, b); - - return ossl_time_compare(t_window, ossl_time_multiply(rtt, 4)) < 0; -} - -static void rxfc_adjust_window_size(QUIC_RXFC *rxfc, uint64_t min_window_size, - OSSL_TIME rtt) -{ - /* Are we sending updates too often? */ - uint64_t new_window_size; - - new_window_size = rxfc->cur_window_size; - - if (rxfc_should_bump_window_size(rxfc, rtt)) - new_window_size *= 2; - - if (new_window_size < min_window_size) - new_window_size = min_window_size; - if (new_window_size > rxfc->max_window_size) /* takes precedence over min size */ - new_window_size = rxfc->max_window_size; - - rxfc->cur_window_size = new_window_size; - rxfc_start_epoch(rxfc); -} - -static void rxfc_update_cwm(QUIC_RXFC *rxfc, uint64_t min_window_size, - OSSL_TIME rtt) -{ - uint64_t new_cwm; - - if (!rxfc_cwm_bump_desired(rxfc)) - return; - - rxfc_adjust_window_size(rxfc, min_window_size, rtt); - - new_cwm = rxfc->rwm + rxfc->cur_window_size; - if (new_cwm > rxfc->cwm) { - rxfc->cwm = new_cwm; - rxfc->has_cwm_changed = 1; - } -} - -static int rxfc_on_retire(QUIC_RXFC *rxfc, uint64_t num_bytes, - uint64_t min_window_size, - OSSL_TIME rtt) -{ - if (ossl_time_is_zero(rxfc->epoch_start)) - /* This happens when we retire our first ever bytes. */ - rxfc_start_epoch(rxfc); - - rxfc->rwm += num_bytes; - rxfc_update_cwm(rxfc, min_window_size, rtt); - return 1; -} - -int ossl_quic_rxfc_on_retire(QUIC_RXFC *rxfc, - uint64_t num_bytes, - OSSL_TIME rtt) -{ - if (rxfc->parent == NULL && !rxfc->standalone) - return 0; - - if (num_bytes == 0) - return 1; - - if (rxfc->rwm + num_bytes > rxfc->swm) - /* Impossible for us to retire more bytes than we have received. */ - return 0; - - rxfc_on_retire(rxfc, num_bytes, 0, rtt); - - if (!rxfc->standalone) - rxfc_on_retire(rxfc->parent, num_bytes, rxfc->cur_window_size, rtt); - - return 1; -} - -uint64_t ossl_quic_rxfc_get_cwm(const QUIC_RXFC *rxfc) -{ - return rxfc->cwm; -} - -uint64_t ossl_quic_rxfc_get_swm(const QUIC_RXFC *rxfc) -{ - return rxfc->swm; -} - -uint64_t ossl_quic_rxfc_get_rwm(const QUIC_RXFC *rxfc) -{ - return rxfc->rwm; -} - -uint64_t ossl_quic_rxfc_get_credit(const QUIC_RXFC *rxfc) -{ - return ossl_quic_rxfc_get_cwm(rxfc) - ossl_quic_rxfc_get_swm(rxfc); -} - -int ossl_quic_rxfc_has_cwm_changed(QUIC_RXFC *rxfc, int clear) -{ - int r = rxfc->has_cwm_changed; - - if (clear) - rxfc->has_cwm_changed = 0; - - return r; -} - -int ossl_quic_rxfc_get_error(QUIC_RXFC *rxfc, int clear) -{ - int r = rxfc->error_code; - - if (clear) - rxfc->error_code = 0; - - return r; -} - -int ossl_quic_rxfc_get_final_size(const QUIC_RXFC *rxfc, uint64_t *final_size) -{ - if (!rxfc->is_fin) - return 0; - - if (final_size != NULL) - *final_size = rxfc->hwm; - - return 1; -} diff --git a/openssl/src/ssl/quic/quic_fifd.c b/openssl/src/ssl/quic/quic_fifd.c deleted file mode 100644 index 1d1bcc11c..000000000 --- a/openssl/src/ssl/quic/quic_fifd.c +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_fifd.h" -#include "internal/quic_wire.h" -#include "internal/qlog_event_helpers.h" - -DEFINE_LIST_OF(tx_history, OSSL_ACKM_TX_PKT); - -int ossl_quic_fifd_init(QUIC_FIFD *fifd, - QUIC_CFQ *cfq, - OSSL_ACKM *ackm, - QUIC_TXPIM *txpim, - /* stream_id is UINT64_MAX for the crypto stream */ - QUIC_SSTREAM *(*get_sstream_by_id)(uint64_t stream_id, - uint32_t pn_space, - void *arg), - void *get_sstream_by_id_arg, - /* stream_id is UINT64_MAX if not applicable */ - void (*regen_frame)(uint64_t frame_type, - uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, - void *arg), - void *regen_frame_arg, - void (*confirm_frame)(uint64_t frame_type, - uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, - void *arg), - void *confirm_frame_arg, - void (*sstream_updated)(uint64_t stream_id, - void *arg), - void *sstream_updated_arg, - QLOG *(*get_qlog_cb)(void *arg), - void *get_qlog_cb_arg) -{ - if (cfq == NULL || ackm == NULL || txpim == NULL - || get_sstream_by_id == NULL || regen_frame == NULL) - return 0; - - fifd->cfq = cfq; - fifd->ackm = ackm; - fifd->txpim = txpim; - fifd->get_sstream_by_id = get_sstream_by_id; - fifd->get_sstream_by_id_arg = get_sstream_by_id_arg; - fifd->regen_frame = regen_frame; - fifd->regen_frame_arg = regen_frame_arg; - fifd->confirm_frame = confirm_frame; - fifd->confirm_frame_arg = confirm_frame_arg; - fifd->sstream_updated = sstream_updated; - fifd->sstream_updated_arg = sstream_updated_arg; - fifd->get_qlog_cb = get_qlog_cb; - fifd->get_qlog_cb_arg = get_qlog_cb_arg; - return 1; -} - -void ossl_quic_fifd_cleanup(QUIC_FIFD *fifd) -{ - /* No-op. */ -} - -static void on_acked(void *arg) -{ - QUIC_TXPIM_PKT *pkt = arg; - QUIC_FIFD *fifd = pkt->fifd; - const QUIC_TXPIM_CHUNK *chunks = ossl_quic_txpim_pkt_get_chunks(pkt); - size_t i, num_chunks = ossl_quic_txpim_pkt_get_num_chunks(pkt); - QUIC_SSTREAM *sstream; - QUIC_CFQ_ITEM *cfq_item, *cfq_item_next; - - /* STREAM and CRYPTO stream chunks, FINs and stream FC frames */ - for (i = 0; i < num_chunks; ++i) { - sstream = fifd->get_sstream_by_id(chunks[i].stream_id, - pkt->ackm_pkt.pkt_space, - fifd->get_sstream_by_id_arg); - if (sstream == NULL) - continue; - - if (chunks[i].end >= chunks[i].start) - /* coverity[check_return]: Best effort - we cannot fail here. */ - ossl_quic_sstream_mark_acked(sstream, - chunks[i].start, chunks[i].end); - - if (chunks[i].has_fin && chunks[i].stream_id != UINT64_MAX) - ossl_quic_sstream_mark_acked_fin(sstream); - - if (chunks[i].has_stop_sending && chunks[i].stream_id != UINT64_MAX) - fifd->confirm_frame(OSSL_QUIC_FRAME_TYPE_STOP_SENDING, - chunks[i].stream_id, pkt, - fifd->confirm_frame_arg); - - if (chunks[i].has_reset_stream && chunks[i].stream_id != UINT64_MAX) - fifd->confirm_frame(OSSL_QUIC_FRAME_TYPE_RESET_STREAM, - chunks[i].stream_id, pkt, - fifd->confirm_frame_arg); - - if (ossl_quic_sstream_is_totally_acked(sstream)) - fifd->sstream_updated(chunks[i].stream_id, fifd->sstream_updated_arg); - } - - /* GCR */ - for (cfq_item = pkt->retx_head; cfq_item != NULL; cfq_item = cfq_item_next) { - cfq_item_next = cfq_item->pkt_next; - ossl_quic_cfq_release(fifd->cfq, cfq_item); - } - - ossl_quic_txpim_pkt_release(fifd->txpim, pkt); -} - -static QLOG *fifd_get_qlog(QUIC_FIFD *fifd) -{ - if (fifd->get_qlog_cb == NULL) - return NULL; - - return fifd->get_qlog_cb(fifd->get_qlog_cb_arg); -} - -static void on_lost(void *arg) -{ - QUIC_TXPIM_PKT *pkt = arg; - QUIC_FIFD *fifd = pkt->fifd; - const QUIC_TXPIM_CHUNK *chunks = ossl_quic_txpim_pkt_get_chunks(pkt); - size_t i, num_chunks = ossl_quic_txpim_pkt_get_num_chunks(pkt); - QUIC_SSTREAM *sstream; - QUIC_CFQ_ITEM *cfq_item, *cfq_item_next; - int sstream_updated; - - ossl_qlog_event_recovery_packet_lost(fifd_get_qlog(fifd), pkt); - - /* STREAM and CRYPTO stream chunks, FIN and stream FC frames */ - for (i = 0; i < num_chunks; ++i) { - sstream = fifd->get_sstream_by_id(chunks[i].stream_id, - pkt->ackm_pkt.pkt_space, - fifd->get_sstream_by_id_arg); - if (sstream == NULL) - continue; - - sstream_updated = 0; - - if (chunks[i].end >= chunks[i].start) { - /* - * Note: If the stream is being reset, we do not need to retransmit - * old data as this is pointless. In this case this will be handled - * by (sstream == NULL) above as the QSM will free the QUIC_SSTREAM - * and our call to get_sstream_by_id above will return NULL. - */ - ossl_quic_sstream_mark_lost(sstream, - chunks[i].start, chunks[i].end); - sstream_updated = 1; - } - - if (chunks[i].has_fin && chunks[i].stream_id != UINT64_MAX) { - ossl_quic_sstream_mark_lost_fin(sstream); - sstream_updated = 1; - } - - if (chunks[i].has_stop_sending && chunks[i].stream_id != UINT64_MAX) - fifd->regen_frame(OSSL_QUIC_FRAME_TYPE_STOP_SENDING, - chunks[i].stream_id, pkt, - fifd->regen_frame_arg); - - if (chunks[i].has_reset_stream && chunks[i].stream_id != UINT64_MAX) - fifd->regen_frame(OSSL_QUIC_FRAME_TYPE_RESET_STREAM, - chunks[i].stream_id, pkt, - fifd->regen_frame_arg); - - /* - * Inform caller that stream needs an FC frame. - * - * Note: We could track whether an FC frame was sent originally for the - * stream to determine if it really needs to be regenerated or not. - * However, if loss has occurred, it's probably better to ensure the - * peer has up-to-date flow control data for the stream. Given that - * these frames are extremely small, we may as well always send it when - * handling loss. - */ - fifd->regen_frame(OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA, - chunks[i].stream_id, - pkt, - fifd->regen_frame_arg); - - if (sstream_updated && chunks[i].stream_id != UINT64_MAX) - fifd->sstream_updated(chunks[i].stream_id, - fifd->sstream_updated_arg); - } - - /* GCR */ - for (cfq_item = pkt->retx_head; cfq_item != NULL; cfq_item = cfq_item_next) { - cfq_item_next = cfq_item->pkt_next; - ossl_quic_cfq_mark_lost(fifd->cfq, cfq_item, UINT32_MAX); - } - - /* Regenerate flag frames */ - if (pkt->had_handshake_done_frame) - fifd->regen_frame(OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE, - UINT64_MAX, pkt, - fifd->regen_frame_arg); - - if (pkt->had_max_data_frame) - fifd->regen_frame(OSSL_QUIC_FRAME_TYPE_MAX_DATA, - UINT64_MAX, pkt, - fifd->regen_frame_arg); - - if (pkt->had_max_streams_bidi_frame) - fifd->regen_frame(OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI, - UINT64_MAX, pkt, - fifd->regen_frame_arg); - - if (pkt->had_max_streams_uni_frame) - fifd->regen_frame(OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI, - UINT64_MAX, pkt, - fifd->regen_frame_arg); - - if (pkt->had_ack_frame) - /* - * We always use the ACK_WITH_ECN frame type to represent the ACK frame - * type in our callback; we assume it is the caller's job to decide - * whether it wants to send ECN data or not. - */ - fifd->regen_frame(OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN, - UINT64_MAX, pkt, - fifd->regen_frame_arg); - - ossl_quic_txpim_pkt_release(fifd->txpim, pkt); -} - -static void on_discarded(void *arg) -{ - QUIC_TXPIM_PKT *pkt = arg; - QUIC_FIFD *fifd = pkt->fifd; - QUIC_CFQ_ITEM *cfq_item, *cfq_item_next; - - /* - * Don't need to do anything to SSTREAMs for STREAM and CRYPTO streams, as - * we assume caller will clean them up. - */ - - /* GCR */ - for (cfq_item = pkt->retx_head; cfq_item != NULL; cfq_item = cfq_item_next) { - cfq_item_next = cfq_item->pkt_next; - ossl_quic_cfq_release(fifd->cfq, cfq_item); - } - - ossl_quic_txpim_pkt_release(fifd->txpim, pkt); -} - -int ossl_quic_fifd_pkt_commit(QUIC_FIFD *fifd, QUIC_TXPIM_PKT *pkt) -{ - QUIC_CFQ_ITEM *cfq_item; - const QUIC_TXPIM_CHUNK *chunks; - size_t i, num_chunks; - QUIC_SSTREAM *sstream; - - pkt->fifd = fifd; - - pkt->ackm_pkt.on_lost = on_lost; - pkt->ackm_pkt.on_acked = on_acked; - pkt->ackm_pkt.on_discarded = on_discarded; - pkt->ackm_pkt.cb_arg = pkt; - - ossl_list_tx_history_init_elem(&pkt->ackm_pkt); - pkt->ackm_pkt.anext = pkt->ackm_pkt.lnext = NULL; - - /* - * Mark the CFQ items which have been added to this packet as having been - * transmitted. - */ - for (cfq_item = pkt->retx_head; - cfq_item != NULL; - cfq_item = cfq_item->pkt_next) - ossl_quic_cfq_mark_tx(fifd->cfq, cfq_item); - - /* - * Mark the send stream chunks which have been added to the packet as having - * been transmitted. - */ - chunks = ossl_quic_txpim_pkt_get_chunks(pkt); - num_chunks = ossl_quic_txpim_pkt_get_num_chunks(pkt); - for (i = 0; i < num_chunks; ++i) { - sstream = fifd->get_sstream_by_id(chunks[i].stream_id, - pkt->ackm_pkt.pkt_space, - fifd->get_sstream_by_id_arg); - if (sstream == NULL) - continue; - - if (chunks[i].end >= chunks[i].start - && !ossl_quic_sstream_mark_transmitted(sstream, - chunks[i].start, - chunks[i].end)) - return 0; - - if (chunks[i].has_fin - && !ossl_quic_sstream_mark_transmitted_fin(sstream, - chunks[i].end + 1)) - return 0; - } - - /* Inform the ACKM. */ - return ossl_ackm_on_tx_packet(fifd->ackm, &pkt->ackm_pkt); -} - -void ossl_quic_fifd_set_qlog_cb(QUIC_FIFD *fifd, QLOG *(*get_qlog_cb)(void *arg), - void *get_qlog_cb_arg) -{ - fifd->get_qlog_cb = get_qlog_cb; - fifd->get_qlog_cb_arg = get_qlog_cb_arg; -} diff --git a/openssl/src/ssl/quic/quic_impl.c b/openssl/src/ssl/quic/quic_impl.c deleted file mode 100644 index c77230a19..000000000 --- a/openssl/src/ssl/quic/quic_impl.c +++ /dev/null @@ -1,4182 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include "quic_local.h" -#include "internal/quic_tls.h" -#include "internal/quic_rx_depack.h" -#include "internal/quic_error.h" -#include "internal/quic_engine.h" -#include "internal/quic_port.h" -#include "internal/time.h" - -typedef struct qctx_st QCTX; - -static void aon_write_finish(QUIC_XSO *xso); -static int create_channel(QUIC_CONNECTION *qc); -static QUIC_XSO *create_xso_from_stream(QUIC_CONNECTION *qc, QUIC_STREAM *qs); -static int qc_try_create_default_xso_for_write(QCTX *ctx); -static int qc_wait_for_default_xso_for_read(QCTX *ctx, int peek); -static void quic_lock(QUIC_CONNECTION *qc); -static void quic_unlock(QUIC_CONNECTION *qc); -static void quic_lock_for_io(QCTX *ctx); -static int quic_do_handshake(QCTX *ctx); -static void qc_update_reject_policy(QUIC_CONNECTION *qc); -static void qc_touch_default_xso(QUIC_CONNECTION *qc); -static void qc_set_default_xso(QUIC_CONNECTION *qc, QUIC_XSO *xso, int touch); -static void qc_set_default_xso_keep_ref(QUIC_CONNECTION *qc, QUIC_XSO *xso, - int touch, QUIC_XSO **old_xso); -static SSL *quic_conn_stream_new(QCTX *ctx, uint64_t flags, int need_lock); -static int quic_validate_for_write(QUIC_XSO *xso, int *err); -static int quic_mutation_allowed(QUIC_CONNECTION *qc, int req_active); -static int qc_blocking_mode(const QUIC_CONNECTION *qc); -static int xso_blocking_mode(const QUIC_XSO *xso); -static void qctx_maybe_autotick(QCTX *ctx); -static int qctx_should_autotick(QCTX *ctx); - -/* - * QUIC Front-End I/O API: Common Utilities - * ======================================== - */ - -/* - * Block until a predicate is met. - * - * Precondition: Must have a channel. - * Precondition: Must hold channel lock (unchecked). - */ -QUIC_NEEDS_LOCK -static int block_until_pred(QUIC_CONNECTION *qc, - int (*pred)(void *arg), void *pred_arg, - uint32_t flags) -{ - QUIC_REACTOR *rtor; - - assert(qc->ch != NULL); - - /* - * Any attempt to block auto-disables tick inhibition as otherwise we will - * hang around forever. - */ - ossl_quic_engine_set_inhibit_tick(qc->engine, 0); - - rtor = ossl_quic_channel_get_reactor(qc->ch); - return ossl_quic_reactor_block_until_pred(rtor, pred, pred_arg, flags, - qc->mutex); -} - -static OSSL_TIME get_time(QUIC_CONNECTION *qc) -{ - if (qc->override_now_cb != NULL) - return qc->override_now_cb(qc->override_now_cb_arg); - else - return ossl_time_now(); -} - -static OSSL_TIME get_time_cb(void *arg) -{ - QUIC_CONNECTION *qc = arg; - - return get_time(qc); -} - -/* - * QCTX is a utility structure which provides information we commonly wish to - * unwrap upon an API call being dispatched to us, namely: - * - * - a pointer to the QUIC_CONNECTION (regardless of whether a QCSO or QSSO - * was passed); - * - a pointer to any applicable QUIC_XSO (e.g. if a QSSO was passed, or if - * a QCSO with a default stream was passed); - * - whether a QSSO was passed (xso == NULL must not be used to determine this - * because it may be non-NULL when a QCSO is passed if that QCSO has a - * default stream); - * - whether we are in "I/O context", meaning that non-normal errors can - * be reported via SSL_get_error() as well as via ERR. Functions such as - * SSL_read(), SSL_write() and SSL_do_handshake() are "I/O context" - * functions which are allowed to change the value returned by - * SSL_get_error. However, other functions (including functions which call - * SSL_do_handshake() implicitly) are not allowed to change the return value - * of SSL_get_error. - */ -struct qctx_st { - QUIC_CONNECTION *qc; - QUIC_XSO *xso; - int is_stream, in_io; -}; - -QUIC_NEEDS_LOCK -static void quic_set_last_error(QCTX *ctx, int last_error) -{ - if (!ctx->in_io) - return; - - if (ctx->is_stream && ctx->xso != NULL) - ctx->xso->last_error = last_error; - else if (!ctx->is_stream && ctx->qc != NULL) - ctx->qc->last_error = last_error; -} - -/* - * Raise a 'normal' error, meaning one that can be reported via SSL_get_error() - * rather than via ERR. Note that normal errors must always be raised while - * holding a lock. - */ -QUIC_NEEDS_LOCK -static int quic_raise_normal_error(QCTX *ctx, - int err) -{ - assert(ctx->in_io); - quic_set_last_error(ctx, err); - - return 0; -} - -/* - * Raise a 'non-normal' error, meaning any error that is not reported via - * SSL_get_error() and must be reported via ERR. - * - * qc should be provided if available. In exceptional circumstances when qc is - * not known NULL may be passed. This should generally only happen when an - * expect_...() function defined below fails, which generally indicates a - * dispatch error or caller error. - * - * ctx should be NULL if the connection lock is not held. - */ -static int quic_raise_non_normal_error(QCTX *ctx, - const char *file, - int line, - const char *func, - int reason, - const char *fmt, - ...) -{ - va_list args; - - if (ctx != NULL) { - quic_set_last_error(ctx, SSL_ERROR_SSL); - - if (reason == SSL_R_PROTOCOL_IS_SHUTDOWN && ctx->qc != NULL) - ossl_quic_channel_restore_err_state(ctx->qc->ch); - } - - ERR_new(); - ERR_set_debug(file, line, func); - - va_start(args, fmt); - ERR_vset_error(ERR_LIB_SSL, reason, fmt, args); - va_end(args); - - return 0; -} - -#define QUIC_RAISE_NORMAL_ERROR(ctx, err) \ - quic_raise_normal_error((ctx), (err)) - -#define QUIC_RAISE_NON_NORMAL_ERROR(ctx, reason, msg) \ - quic_raise_non_normal_error((ctx), \ - OPENSSL_FILE, OPENSSL_LINE, \ - OPENSSL_FUNC, \ - (reason), \ - (msg)) - -/* - * Given a QCSO or QSSO, initialises a QCTX, determining the contextually - * applicable QUIC_CONNECTION pointer and, if applicable, QUIC_XSO pointer. - * - * After this returns 1, all fields of the passed QCTX are initialised. - * Returns 0 on failure. This function is intended to be used to provide API - * semantics and as such, it invokes QUIC_RAISE_NON_NORMAL_ERROR() on failure. - */ -static int expect_quic(const SSL *s, QCTX *ctx) -{ - QUIC_CONNECTION *qc; - QUIC_XSO *xso; - - ctx->qc = NULL; - ctx->xso = NULL; - ctx->is_stream = 0; - - if (s == NULL) - return QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_PASSED_NULL_PARAMETER, NULL); - - switch (s->type) { - case SSL_TYPE_QUIC_CONNECTION: - qc = (QUIC_CONNECTION *)s; - ctx->qc = qc; - ctx->xso = qc->default_xso; - ctx->is_stream = 0; - ctx->in_io = 0; - return 1; - - case SSL_TYPE_QUIC_XSO: - xso = (QUIC_XSO *)s; - ctx->qc = xso->conn; - ctx->xso = xso; - ctx->is_stream = 1; - ctx->in_io = 0; - return 1; - - default: - return QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_INTERNAL_ERROR, NULL); - } -} - -/* - * Like expect_quic(), but requires a QUIC_XSO be contextually available. In - * other words, requires that the passed QSO be a QSSO or a QCSO with a default - * stream. - * - * remote_init determines if we expect the default XSO to be remotely created or - * not. If it is -1, do not instantiate a default XSO if one does not yet exist. - * - * Channel mutex is acquired and retained on success. - */ -QUIC_ACQUIRES_LOCK -static int ossl_unused expect_quic_with_stream_lock(const SSL *s, int remote_init, - int in_io, QCTX *ctx) -{ - if (!expect_quic(s, ctx)) - return 0; - - if (in_io) - quic_lock_for_io(ctx); - else - quic_lock(ctx->qc); - - if (ctx->xso == NULL && remote_init >= 0) { - if (!quic_mutation_allowed(ctx->qc, /*req_active=*/0)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - goto err; - } - - /* If we haven't finished the handshake, try to advance it. */ - if (quic_do_handshake(ctx) < 1) - /* ossl_quic_do_handshake raised error here */ - goto err; - - if (remote_init == 0) { - if (!qc_try_create_default_xso_for_write(ctx)) - goto err; - } else { - if (!qc_wait_for_default_xso_for_read(ctx, /*peek=*/0)) - goto err; - } - - ctx->xso = ctx->qc->default_xso; - } - - if (ctx->xso == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_NO_STREAM, NULL); - goto err; - } - - return 1; /* coverity[missing_unlock]: lock held */ - -err: - quic_unlock(ctx->qc); - return 0; -} - -/* - * Like expect_quic(), but fails if called on a QUIC_XSO. ctx->xso may still - * be non-NULL if the QCSO has a default stream. - */ -static int ossl_unused expect_quic_conn_only(const SSL *s, QCTX *ctx) -{ - if (!expect_quic(s, ctx)) - return 0; - - if (ctx->is_stream) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_CONN_USE_ONLY, NULL); - - return 1; -} - -/* - * Ensures that the channel mutex is held for a method which touches channel - * state. - * - * Precondition: Channel mutex is not held (unchecked) - */ -static void quic_lock(QUIC_CONNECTION *qc) -{ -#if defined(OPENSSL_THREADS) - ossl_crypto_mutex_lock(qc->mutex); -#endif -} - -static void quic_lock_for_io(QCTX *ctx) -{ - quic_lock(ctx->qc); - ctx->in_io = 1; - - /* - * We are entering an I/O function so we must update the values returned by - * SSL_get_error and SSL_want. Set no error. This will be overridden later - * if a call to QUIC_RAISE_NORMAL_ERROR or QUIC_RAISE_NON_NORMAL_ERROR - * occurs during the API call. - */ - quic_set_last_error(ctx, SSL_ERROR_NONE); -} - -/* Precondition: Channel mutex is held (unchecked) */ -QUIC_NEEDS_LOCK -static void quic_unlock(QUIC_CONNECTION *qc) -{ -#if defined(OPENSSL_THREADS) - ossl_crypto_mutex_unlock(qc->mutex); -#endif -} - -/* - * This predicate is the criterion which should determine API call rejection for - * *most* mutating API calls, particularly stream-related operations for send - * parts. - * - * A call is rejected (this function returns 0) if shutdown is in progress - * (stream flushing), or we are in a TERMINATING or TERMINATED state. If - * req_active=1, the connection must be active (i.e., the IDLE state is also - * rejected). - */ -static int quic_mutation_allowed(QUIC_CONNECTION *qc, int req_active) -{ - if (qc->shutting_down || ossl_quic_channel_is_term_any(qc->ch)) - return 0; - - if (req_active && !ossl_quic_channel_is_active(qc->ch)) - return 0; - - return 1; -} - -/* - * QUIC Front-End I/O API: Initialization - * ====================================== - * - * SSL_new => ossl_quic_new - * ossl_quic_init - * SSL_reset => ossl_quic_reset - * SSL_clear => ossl_quic_clear - * ossl_quic_deinit - * SSL_free => ossl_quic_free - * - * SSL_set_options => ossl_quic_set_options - * SSL_get_options => ossl_quic_get_options - * SSL_clear_options => ossl_quic_clear_options - * - */ - -/* SSL_new */ -SSL *ossl_quic_new(SSL_CTX *ctx) -{ - QUIC_CONNECTION *qc = NULL; - SSL *ssl_base = NULL; - SSL_CONNECTION *sc = NULL; - - qc = OPENSSL_zalloc(sizeof(*qc)); - if (qc == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_CRYPTO_LIB, NULL); - return NULL; - } -#if defined(OPENSSL_THREADS) - if ((qc->mutex = ossl_crypto_mutex_new()) == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_CRYPTO_LIB, NULL); - goto err; - } -#endif - - /* Initialise the QUIC_CONNECTION's stub header. */ - ssl_base = &qc->ssl; - if (!ossl_ssl_init(ssl_base, ctx, ctx->method, SSL_TYPE_QUIC_CONNECTION)) { - ssl_base = NULL; - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_INTERNAL_ERROR, NULL); - goto err; - } - - qc->tls = ossl_ssl_connection_new_int(ctx, TLS_method()); - if (qc->tls == NULL || (sc = SSL_CONNECTION_FROM_SSL(qc->tls)) == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_INTERNAL_ERROR, NULL); - goto err; - } - - /* override the user_ssl of the inner connection */ - sc->s3.flags |= TLS1_FLAGS_QUIC; - - /* Restrict options derived from the SSL_CTX. */ - sc->options &= OSSL_QUIC_PERMITTED_OPTIONS_CONN; - sc->pha_enabled = 0; - -#if !defined(OPENSSL_NO_QUIC_THREAD_ASSIST) - qc->is_thread_assisted - = (ssl_base->method == OSSL_QUIC_client_thread_method()); -#endif - - qc->as_server = 0; /* TODO(QUIC SERVER): add server support */ - qc->as_server_state = qc->as_server; - - qc->default_stream_mode = SSL_DEFAULT_STREAM_MODE_AUTO_BIDI; - qc->default_ssl_mode = qc->ssl.ctx->mode; - qc->default_ssl_options = qc->ssl.ctx->options & OSSL_QUIC_PERMITTED_OPTIONS; - qc->desires_blocking = 1; - qc->blocking = 0; - qc->incoming_stream_policy = SSL_INCOMING_STREAM_POLICY_AUTO; - qc->last_error = SSL_ERROR_NONE; - - if (!create_channel(qc)) - goto err; - - ossl_quic_channel_set_msg_callback(qc->ch, ctx->msg_callback, ssl_base); - ossl_quic_channel_set_msg_callback_arg(qc->ch, ctx->msg_callback_arg); - - qc_update_reject_policy(qc); - - /* - * We do not create the default XSO yet. The reason for this is that the - * stream ID of the default XSO will depend on whether the stream is client - * or server-initiated, which depends on who transmits first. Since we do - * not know whether the application will be using a client-transmits-first - * or server-transmits-first protocol, we defer default XSO creation until - * the client calls SSL_read() or SSL_write(). If it calls SSL_read() first, - * we take that as a cue that the client is expecting a server-initiated - * stream, and vice versa if SSL_write() is called first. - */ - return ssl_base; - -err: - if (ssl_base == NULL) { -#if defined(OPENSSL_THREADS) - ossl_crypto_mutex_free(&qc->mutex); -#endif - OPENSSL_free(qc); - } else { - SSL_free(ssl_base); - } - return NULL; -} - -/* SSL_free */ -QUIC_TAKES_LOCK -void ossl_quic_free(SSL *s) -{ - QCTX ctx; - int is_default; - - /* We should never be called on anything but a QSO. */ - if (!expect_quic(s, &ctx)) - return; - - quic_lock(ctx.qc); - - if (ctx.is_stream) { - /* - * When a QSSO is freed, the XSO is freed immediately, because the XSO - * itself only contains API personality layer data. However the - * underlying QUIC_STREAM is not freed immediately but is instead marked - * as deleted for later collection. - */ - - assert(ctx.qc->num_xso > 0); - --ctx.qc->num_xso; - - /* If a stream's send part has not been finished, auto-reset it. */ - if (( ctx.xso->stream->send_state == QUIC_SSTREAM_STATE_READY - || ctx.xso->stream->send_state == QUIC_SSTREAM_STATE_SEND) - && !ossl_quic_sstream_get_final_size(ctx.xso->stream->sstream, NULL)) - ossl_quic_stream_map_reset_stream_send_part(ossl_quic_channel_get_qsm(ctx.qc->ch), - ctx.xso->stream, 0); - - /* Do STOP_SENDING for the receive part, if applicable. */ - if ( ctx.xso->stream->recv_state == QUIC_RSTREAM_STATE_RECV - || ctx.xso->stream->recv_state == QUIC_RSTREAM_STATE_SIZE_KNOWN) - ossl_quic_stream_map_stop_sending_recv_part(ossl_quic_channel_get_qsm(ctx.qc->ch), - ctx.xso->stream, 0); - - /* Update stream state. */ - ctx.xso->stream->deleted = 1; - ossl_quic_stream_map_update_state(ossl_quic_channel_get_qsm(ctx.qc->ch), - ctx.xso->stream); - - is_default = (ctx.xso == ctx.qc->default_xso); - quic_unlock(ctx.qc); - - /* - * Unref the connection in most cases; the XSO has a ref to the QC and - * not vice versa. But for a default XSO, to avoid circular references, - * the QC refs the XSO but the XSO does not ref the QC. If we are the - * default XSO, we only get here when the QC is being torn down anyway, - * so don't call SSL_free(qc) as we are already in it. - */ - if (!is_default) - SSL_free(&ctx.qc->ssl); - - /* Note: SSL_free calls OPENSSL_free(xso) for us */ - return; - } - - /* - * Free the default XSO, if any. The QUIC_STREAM is not deleted at this - * stage, but is freed during the channel free when the whole QSM is freed. - */ - if (ctx.qc->default_xso != NULL) { - QUIC_XSO *xso = ctx.qc->default_xso; - - quic_unlock(ctx.qc); - SSL_free(&xso->ssl); - quic_lock(ctx.qc); - ctx.qc->default_xso = NULL; - } - - /* Ensure we have no remaining XSOs. */ - assert(ctx.qc->num_xso == 0); - -#if !defined(OPENSSL_NO_QUIC_THREAD_ASSIST) - if (ctx.qc->is_thread_assisted && ctx.qc->started) { - ossl_quic_thread_assist_wait_stopped(&ctx.qc->thread_assist); - ossl_quic_thread_assist_cleanup(&ctx.qc->thread_assist); - } -#endif - - SSL_free(ctx.qc->tls); - - ossl_quic_channel_free(ctx.qc->ch); - ossl_quic_port_free(ctx.qc->port); - ossl_quic_engine_free(ctx.qc->engine); - - BIO_free_all(ctx.qc->net_rbio); - BIO_free_all(ctx.qc->net_wbio); - - quic_unlock(ctx.qc); /* tsan doesn't like freeing locked mutexes */ -#if defined(OPENSSL_THREADS) - ossl_crypto_mutex_free(&ctx.qc->mutex); -#endif - - /* - * Note: SSL_free (that called this function) calls OPENSSL_free(ctx.qc) for - * us - */ -} - -/* SSL method init */ -int ossl_quic_init(SSL *s) -{ - /* Same op as SSL_clear, forward the call. */ - return ossl_quic_clear(s); -} - -/* SSL method deinit */ -void ossl_quic_deinit(SSL *s) -{ - /* No-op. */ -} - -/* SSL_clear (ssl_reset method) */ -int ossl_quic_reset(SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - ERR_raise(ERR_LIB_SSL, ERR_R_UNSUPPORTED); - return 0; -} - -/* ssl_clear method (unused) */ -int ossl_quic_clear(SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - ERR_raise(ERR_LIB_SSL, ERR_R_UNSUPPORTED); - return 0; -} - -int ossl_quic_conn_set_override_now_cb(SSL *s, - OSSL_TIME (*now_cb)(void *arg), - void *now_cb_arg) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - - ctx.qc->override_now_cb = now_cb; - ctx.qc->override_now_cb_arg = now_cb_arg; - - quic_unlock(ctx.qc); - return 1; -} - -void ossl_quic_conn_force_assist_thread_wake(SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return; - -#if !defined(OPENSSL_NO_QUIC_THREAD_ASSIST) - if (ctx.qc->is_thread_assisted && ctx.qc->started) - ossl_quic_thread_assist_notify_deadline_changed(&ctx.qc->thread_assist); -#endif -} - -QUIC_NEEDS_LOCK -static void qc_touch_default_xso(QUIC_CONNECTION *qc) -{ - qc->default_xso_created = 1; - qc_update_reject_policy(qc); -} - -/* - * Changes default XSO. Allows caller to keep reference to the old default XSO - * (if any). Reference to new XSO is transferred from caller. - */ -QUIC_NEEDS_LOCK -static void qc_set_default_xso_keep_ref(QUIC_CONNECTION *qc, QUIC_XSO *xso, - int touch, - QUIC_XSO **old_xso) -{ - int refs; - - *old_xso = NULL; - - if (qc->default_xso != xso) { - *old_xso = qc->default_xso; /* transfer old XSO ref to caller */ - - qc->default_xso = xso; - - if (xso == NULL) { - /* - * Changing to not having a default XSO. XSO becomes standalone and - * now has a ref to the QC. - */ - if (!ossl_assert(SSL_up_ref(&qc->ssl))) - return; - } else { - /* - * Changing from not having a default XSO to having one. The new XSO - * will have had a reference to the QC we need to drop to avoid a - * circular reference. - * - * Currently we never change directly from one default XSO to - * another, though this function would also still be correct if this - * weren't the case. - */ - assert(*old_xso == NULL); - - CRYPTO_DOWN_REF(&qc->ssl.references, &refs); - assert(refs > 0); - } - } - - if (touch) - qc_touch_default_xso(qc); -} - -/* - * Changes default XSO, releasing the reference to any previous default XSO. - * Reference to new XSO is transferred from caller. - */ -QUIC_NEEDS_LOCK -static void qc_set_default_xso(QUIC_CONNECTION *qc, QUIC_XSO *xso, int touch) -{ - QUIC_XSO *old_xso = NULL; - - qc_set_default_xso_keep_ref(qc, xso, touch, &old_xso); - - if (old_xso != NULL) - SSL_free(&old_xso->ssl); -} - -QUIC_NEEDS_LOCK -static void xso_update_options(QUIC_XSO *xso) -{ - int cleanse = ((xso->ssl_options & SSL_OP_CLEANSE_PLAINTEXT) != 0); - - if (xso->stream->rstream != NULL) - ossl_quic_rstream_set_cleanse(xso->stream->rstream, cleanse); - - if (xso->stream->sstream != NULL) - ossl_quic_sstream_set_cleanse(xso->stream->sstream, cleanse); -} - -/* - * SSL_set_options - * --------------- - * - * Setting options on a QCSO - * - configures the handshake-layer options; - * - configures the default data-plane options for new streams; - * - configures the data-plane options on the default XSO, if there is one. - * - * Setting options on a QSSO - * - configures data-plane options for that stream only. - */ -QUIC_TAKES_LOCK -static uint64_t quic_mask_or_options(SSL *ssl, uint64_t mask_value, uint64_t or_value) -{ - QCTX ctx; - uint64_t hs_mask_value, hs_or_value, ret; - - if (!expect_quic(ssl, &ctx)) - return 0; - - quic_lock(ctx.qc); - - if (!ctx.is_stream) { - /* - * If we were called on the connection, we apply any handshake option - * changes. - */ - hs_mask_value = (mask_value & OSSL_QUIC_PERMITTED_OPTIONS_CONN); - hs_or_value = (or_value & OSSL_QUIC_PERMITTED_OPTIONS_CONN); - - SSL_clear_options(ctx.qc->tls, hs_mask_value); - SSL_set_options(ctx.qc->tls, hs_or_value); - - /* Update defaults for new streams. */ - ctx.qc->default_ssl_options - = ((ctx.qc->default_ssl_options & ~mask_value) | or_value) - & OSSL_QUIC_PERMITTED_OPTIONS; - } - - if (ctx.xso != NULL) { - ctx.xso->ssl_options - = ((ctx.xso->ssl_options & ~mask_value) | or_value) - & OSSL_QUIC_PERMITTED_OPTIONS_STREAM; - - xso_update_options(ctx.xso); - } - - ret = ctx.is_stream ? ctx.xso->ssl_options : ctx.qc->default_ssl_options; - - quic_unlock(ctx.qc); - return ret; -} - -uint64_t ossl_quic_set_options(SSL *ssl, uint64_t options) -{ - return quic_mask_or_options(ssl, 0, options); -} - -/* SSL_clear_options */ -uint64_t ossl_quic_clear_options(SSL *ssl, uint64_t options) -{ - return quic_mask_or_options(ssl, options, 0); -} - -/* SSL_get_options */ -uint64_t ossl_quic_get_options(const SSL *ssl) -{ - return quic_mask_or_options((SSL *)ssl, 0, 0); -} - -/* - * QUIC Front-End I/O API: Network BIO Configuration - * ================================================= - * - * Handling the different BIOs is difficult: - * - * - It is more or less a requirement that we use non-blocking network I/O; - * we need to be able to have timeouts on recv() calls, and make best effort - * (non blocking) send() and recv() calls. - * - * The only sensible way to do this is to configure the socket into - * non-blocking mode. We could try to do select() before calling send() or - * recv() to get a guarantee that the call will not block, but this will - * probably run into issues with buggy OSes which generate spurious socket - * readiness events. In any case, relying on this to work reliably does not - * seem sane. - * - * Timeouts could be handled via setsockopt() socket timeout options, but - * this depends on OS support and adds another syscall to every network I/O - * operation. It also has obvious thread safety concerns if we want to move - * to concurrent use of a single socket at some later date. - * - * Some OSes support a MSG_DONTWAIT flag which allows a single I/O option to - * be made non-blocking. However some OSes (e.g. Windows) do not support - * this, so we cannot rely on this. - * - * As such, we need to configure any FD in non-blocking mode. This may - * confound users who pass a blocking socket to libssl. However, in practice - * it would be extremely strange for a user of QUIC to pass an FD to us, - * then also try and send receive traffic on the same socket(!). Thus the - * impact of this should be limited, and can be documented. - * - * - We support both blocking and non-blocking operation in terms of the API - * presented to the user. One prospect is to set the blocking mode based on - * whether the socket passed to us was already in blocking mode. However, - * Windows has no API for determining if a socket is in blocking mode (!), - * therefore this cannot be done portably. Currently therefore we expose an - * explicit API call to set this, and default to blocking mode. - * - * - We need to determine our initial destination UDP address. The "natural" - * way for a user to do this is to set the peer variable on a BIO_dgram. - * However, this has problems because BIO_dgram's peer variable is used for - * both transmission and reception. This means it can be constantly being - * changed to a malicious value (e.g. if some random unrelated entity on the - * network starts sending traffic to us) on every read call. This is not a - * direct issue because we use the 'stateless' BIO_sendmmsg and BIO_recvmmsg - * calls only, which do not use this variable. However, we do need to let - * the user specify the peer in a 'normal' manner. The compromise here is - * that we grab the current peer value set at the time the write BIO is set - * and do not read the value again. - * - * - We also need to support memory BIOs (e.g. BIO_dgram_pair) or custom BIOs. - * Currently we do this by only supporting non-blocking mode. - * - */ - -/* - * Determines what initial destination UDP address we should use, if possible. - * If this fails the client must set the destination address manually, or use a - * BIO which does not need a destination address. - */ -static int csm_analyse_init_peer_addr(BIO *net_wbio, BIO_ADDR *peer) -{ - if (BIO_dgram_detect_peer_addr(net_wbio, peer) <= 0) - return 0; - - return 1; -} - -static int qc_can_support_blocking_cached(QUIC_CONNECTION *qc) -{ - QUIC_REACTOR *rtor = ossl_quic_channel_get_reactor(qc->ch); - - return ossl_quic_reactor_can_poll_r(rtor) - && ossl_quic_reactor_can_poll_w(rtor); -} - -static void qc_update_can_support_blocking(QUIC_CONNECTION *qc) -{ - ossl_quic_port_update_poll_descriptors(qc->port); /* best effort */ -} - -static void qc_update_blocking_mode(QUIC_CONNECTION *qc) -{ - qc->blocking = qc->desires_blocking && qc_can_support_blocking_cached(qc); -} - -void ossl_quic_conn_set0_net_rbio(SSL *s, BIO *net_rbio) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return; - - if (ctx.qc->net_rbio == net_rbio) - return; - - if (!ossl_quic_port_set_net_rbio(ctx.qc->port, net_rbio)) - return; - - BIO_free_all(ctx.qc->net_rbio); - ctx.qc->net_rbio = net_rbio; - - if (net_rbio != NULL) - BIO_set_nbio(net_rbio, 1); /* best effort autoconfig */ - - /* - * Determine if the current pair of read/write BIOs now set allows blocking - * mode to be supported. - */ - qc_update_can_support_blocking(ctx.qc); - qc_update_blocking_mode(ctx.qc); -} - -void ossl_quic_conn_set0_net_wbio(SSL *s, BIO *net_wbio) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return; - - if (ctx.qc->net_wbio == net_wbio) - return; - - if (!ossl_quic_port_set_net_wbio(ctx.qc->port, net_wbio)) - return; - - BIO_free_all(ctx.qc->net_wbio); - ctx.qc->net_wbio = net_wbio; - - if (net_wbio != NULL) - BIO_set_nbio(net_wbio, 1); /* best effort autoconfig */ - - /* - * Determine if the current pair of read/write BIOs now set allows blocking - * mode to be supported. - */ - qc_update_can_support_blocking(ctx.qc); - qc_update_blocking_mode(ctx.qc); -} - -BIO *ossl_quic_conn_get_net_rbio(const SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return NULL; - - return ctx.qc->net_rbio; -} - -BIO *ossl_quic_conn_get_net_wbio(const SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return NULL; - - return ctx.qc->net_wbio; -} - -int ossl_quic_conn_get_blocking_mode(const SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - if (ctx.is_stream) - return xso_blocking_mode(ctx.xso); - - return qc_blocking_mode(ctx.qc); -} - -QUIC_TAKES_LOCK -int ossl_quic_conn_set_blocking_mode(SSL *s, int blocking) -{ - int ret = 0; - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - - /* Sanity check - can we support the request given the current network BIO? */ - if (blocking) { - /* - * If called directly on a QCSO, update our information on network BIO - * capabilities. - */ - if (!ctx.is_stream) - qc_update_can_support_blocking(ctx.qc); - - /* Cannot enable blocking mode if we do not have pollable FDs. */ - if (!qc_can_support_blocking_cached(ctx.qc)) { - ret = QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_UNSUPPORTED, NULL); - goto out; - } - } - - if (!ctx.is_stream) - /* - * If called directly on a QCSO, update default and connection-level - * blocking modes. - */ - ctx.qc->desires_blocking = (blocking != 0); - - if (ctx.xso != NULL) { - /* - * If called on a QSSO or a QCSO with a default XSO, update the blocking - * mode. - */ - ctx.xso->desires_blocking = (blocking != 0); - ctx.xso->desires_blocking_set = 1; - } - - ret = 1; -out: - qc_update_blocking_mode(ctx.qc); - quic_unlock(ctx.qc); - return ret; -} - -int ossl_quic_conn_set_initial_peer_addr(SSL *s, - const BIO_ADDR *peer_addr) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - if (ctx.qc->started) - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED, - NULL); - - if (peer_addr == NULL) { - BIO_ADDR_clear(&ctx.qc->init_peer_addr); - return 1; - } - - ctx.qc->init_peer_addr = *peer_addr; - return 1; -} - -/* - * QUIC Front-End I/O API: Asynchronous I/O Management - * =================================================== - * - * (BIO/)SSL_handle_events => ossl_quic_handle_events - * (BIO/)SSL_get_event_timeout => ossl_quic_get_event_timeout - * (BIO/)SSL_get_poll_fd => ossl_quic_get_poll_fd - * - */ - -/* Returns 1 if the connection is being used in blocking mode. */ -static int qc_blocking_mode(const QUIC_CONNECTION *qc) -{ - return qc->blocking; -} - -static int xso_blocking_mode(const QUIC_XSO *xso) -{ - if (xso->desires_blocking_set) - return xso->desires_blocking && qc_can_support_blocking_cached(xso->conn); - else - /* Only ever set if we can support blocking. */ - return xso->conn->blocking; -} - -/* SSL_handle_events; performs QUIC I/O and timeout processing. */ -QUIC_TAKES_LOCK -int ossl_quic_handle_events(SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - ossl_quic_reactor_tick(ossl_quic_channel_get_reactor(ctx.qc->ch), 0); - quic_unlock(ctx.qc); - return 1; -} - -/* - * SSL_get_event_timeout. Get the time in milliseconds until the SSL object - * should next have events handled by the application by calling - * SSL_handle_events(). tv is set to 0 if the object should have events handled - * immediately. If no timeout is currently active, *is_infinite is set to 1 and - * the value of *tv is undefined. - */ -QUIC_TAKES_LOCK -int ossl_quic_get_event_timeout(SSL *s, struct timeval *tv, int *is_infinite) -{ - QCTX ctx; - OSSL_TIME deadline = ossl_time_infinite(); - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - - deadline - = ossl_quic_reactor_get_tick_deadline(ossl_quic_channel_get_reactor(ctx.qc->ch)); - - if (ossl_time_is_infinite(deadline)) { - *is_infinite = 1; - - /* - * Robustness against faulty applications that don't check *is_infinite; - * harmless long timeout. - */ - tv->tv_sec = 1000000; - tv->tv_usec = 0; - - quic_unlock(ctx.qc); - return 1; - } - - *tv = ossl_time_to_timeval(ossl_time_subtract(deadline, get_time(ctx.qc))); - *is_infinite = 0; - quic_unlock(ctx.qc); - return 1; -} - -/* SSL_get_rpoll_descriptor */ -int ossl_quic_get_rpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *desc) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - if (desc == NULL || ctx.qc->net_rbio == NULL) - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_PASSED_INVALID_ARGUMENT, - NULL); - - return BIO_get_rpoll_descriptor(ctx.qc->net_rbio, desc); -} - -/* SSL_get_wpoll_descriptor */ -int ossl_quic_get_wpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *desc) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - if (desc == NULL || ctx.qc->net_wbio == NULL) - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_PASSED_INVALID_ARGUMENT, - NULL); - - return BIO_get_wpoll_descriptor(ctx.qc->net_wbio, desc); -} - -/* SSL_net_read_desired */ -QUIC_TAKES_LOCK -int ossl_quic_get_net_read_desired(SSL *s) -{ - QCTX ctx; - int ret; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - ret = ossl_quic_reactor_net_read_desired(ossl_quic_channel_get_reactor(ctx.qc->ch)); - quic_unlock(ctx.qc); - return ret; -} - -/* SSL_net_write_desired */ -QUIC_TAKES_LOCK -int ossl_quic_get_net_write_desired(SSL *s) -{ - int ret; - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - ret = ossl_quic_reactor_net_write_desired(ossl_quic_channel_get_reactor(ctx.qc->ch)); - quic_unlock(ctx.qc); - return ret; -} - -/* - * QUIC Front-End I/O API: Connection Lifecycle Operations - * ======================================================= - * - * SSL_do_handshake => ossl_quic_do_handshake - * SSL_set_connect_state => ossl_quic_set_connect_state - * SSL_set_accept_state => ossl_quic_set_accept_state - * SSL_shutdown => ossl_quic_shutdown - * SSL_ctrl => ossl_quic_ctrl - * (BIO/)SSL_connect => ossl_quic_connect - * (BIO/)SSL_accept => ossl_quic_accept - * - */ - -QUIC_NEEDS_LOCK -static void qc_shutdown_flush_init(QUIC_CONNECTION *qc) -{ - QUIC_STREAM_MAP *qsm; - - if (qc->shutting_down) - return; - - qsm = ossl_quic_channel_get_qsm(qc->ch); - - ossl_quic_stream_map_begin_shutdown_flush(qsm); - qc->shutting_down = 1; -} - -/* Returns 1 if all shutdown-flush streams have been done with. */ -QUIC_NEEDS_LOCK -static int qc_shutdown_flush_finished(QUIC_CONNECTION *qc) -{ - QUIC_STREAM_MAP *qsm = ossl_quic_channel_get_qsm(qc->ch); - - return qc->shutting_down - && ossl_quic_stream_map_is_shutdown_flush_finished(qsm); -} - -/* SSL_shutdown */ -static int quic_shutdown_wait(void *arg) -{ - QUIC_CONNECTION *qc = arg; - - return ossl_quic_channel_is_terminated(qc->ch); -} - -/* Returns 1 if shutdown flush process has finished or is inapplicable. */ -static int quic_shutdown_flush_wait(void *arg) -{ - QUIC_CONNECTION *qc = arg; - - return ossl_quic_channel_is_term_any(qc->ch) - || qc_shutdown_flush_finished(qc); -} - -static int quic_shutdown_peer_wait(void *arg) -{ - QUIC_CONNECTION *qc = arg; - return ossl_quic_channel_is_term_any(qc->ch); -} - -QUIC_TAKES_LOCK -int ossl_quic_conn_shutdown(SSL *s, uint64_t flags, - const SSL_SHUTDOWN_EX_ARGS *args, - size_t args_len) -{ - int ret; - QCTX ctx; - int stream_flush = ((flags & SSL_SHUTDOWN_FLAG_NO_STREAM_FLUSH) == 0); - int no_block = ((flags & SSL_SHUTDOWN_FLAG_NO_BLOCK) != 0); - int wait_peer = ((flags & SSL_SHUTDOWN_FLAG_WAIT_PEER) != 0); - - if (!expect_quic(s, &ctx)) - return -1; - - if (ctx.is_stream) { - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, SSL_R_CONN_USE_ONLY, NULL); - return -1; - } - - quic_lock(ctx.qc); - - if (ossl_quic_channel_is_terminated(ctx.qc->ch)) { - quic_unlock(ctx.qc); - return 1; - } - - /* Phase 1: Stream Flushing */ - if (!wait_peer && stream_flush) { - qc_shutdown_flush_init(ctx.qc); - - if (!qc_shutdown_flush_finished(ctx.qc)) { - if (!no_block && qc_blocking_mode(ctx.qc)) { - ret = block_until_pred(ctx.qc, quic_shutdown_flush_wait, ctx.qc, 0); - if (ret < 1) { - ret = 0; - goto err; - } - } else { - qctx_maybe_autotick(&ctx); - } - } - - if (!qc_shutdown_flush_finished(ctx.qc)) { - quic_unlock(ctx.qc); - return 0; /* ongoing */ - } - } - - /* Phase 2: Connection Closure */ - if (wait_peer && !ossl_quic_channel_is_term_any(ctx.qc->ch)) { - if (!no_block && qc_blocking_mode(ctx.qc)) { - ret = block_until_pred(ctx.qc, quic_shutdown_peer_wait, ctx.qc, 0); - if (ret < 1) { - ret = 0; - goto err; - } - } else { - qctx_maybe_autotick(&ctx); - } - - if (!ossl_quic_channel_is_term_any(ctx.qc->ch)) { - ret = 0; /* peer hasn't closed yet - still not done */ - goto err; - } - - /* - * We are at least terminating - go through the normal process of - * waiting until we are in the TERMINATED state. - */ - } - - /* Block mutation ops regardless of if we did stream flush. */ - ctx.qc->shutting_down = 1; - - /* - * This call is a no-op if we are already terminating, so it doesn't - * affect the wait_peer case. - */ - ossl_quic_channel_local_close(ctx.qc->ch, - args != NULL ? args->quic_error_code : 0, - args != NULL ? args->quic_reason : NULL); - - SSL_set_shutdown(ctx.qc->tls, SSL_SENT_SHUTDOWN); - - if (ossl_quic_channel_is_terminated(ctx.qc->ch)) { - quic_unlock(ctx.qc); - return 1; - } - - /* Phase 3: Terminating Wait Time */ - if (!no_block && qc_blocking_mode(ctx.qc) - && (flags & SSL_SHUTDOWN_FLAG_RAPID) == 0) { - ret = block_until_pred(ctx.qc, quic_shutdown_wait, ctx.qc, 0); - if (ret < 1) { - ret = 0; - goto err; - } - } else { - qctx_maybe_autotick(&ctx); - } - - ret = ossl_quic_channel_is_terminated(ctx.qc->ch); -err: - quic_unlock(ctx.qc); - return ret; -} - -/* SSL_ctrl */ -long ossl_quic_ctrl(SSL *s, int cmd, long larg, void *parg) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - switch (cmd) { - case SSL_CTRL_MODE: - /* If called on a QCSO, update the default mode. */ - if (!ctx.is_stream) - ctx.qc->default_ssl_mode |= (uint32_t)larg; - - /* - * If we were called on a QSSO or have a default stream, we also update - * that. - */ - if (ctx.xso != NULL) { - /* Cannot enable EPW while AON write in progress. */ - if (ctx.xso->aon_write_in_progress) - larg &= ~SSL_MODE_ENABLE_PARTIAL_WRITE; - - ctx.xso->ssl_mode |= (uint32_t)larg; - return ctx.xso->ssl_mode; - } - - return ctx.qc->default_ssl_mode; - case SSL_CTRL_CLEAR_MODE: - if (!ctx.is_stream) - ctx.qc->default_ssl_mode &= ~(uint32_t)larg; - - if (ctx.xso != NULL) { - ctx.xso->ssl_mode &= ~(uint32_t)larg; - return ctx.xso->ssl_mode; - } - - return ctx.qc->default_ssl_mode; - - case SSL_CTRL_SET_MSG_CALLBACK_ARG: - ossl_quic_channel_set_msg_callback_arg(ctx.qc->ch, parg); - /* This ctrl also needs to be passed to the internal SSL object */ - return SSL_ctrl(ctx.qc->tls, cmd, larg, parg); - - case DTLS_CTRL_GET_TIMEOUT: /* DTLSv1_get_timeout */ - { - int is_infinite; - - if (!ossl_quic_get_event_timeout(s, parg, &is_infinite)) - return 0; - - return !is_infinite; - } - case DTLS_CTRL_HANDLE_TIMEOUT: /* DTLSv1_handle_timeout */ - /* For legacy compatibility with DTLS calls. */ - return ossl_quic_handle_events(s) == 1 ? 1 : -1; - - /* Mask ctrls we shouldn't support for QUIC. */ - case SSL_CTRL_GET_READ_AHEAD: - case SSL_CTRL_SET_READ_AHEAD: - case SSL_CTRL_SET_MAX_SEND_FRAGMENT: - case SSL_CTRL_SET_SPLIT_SEND_FRAGMENT: - case SSL_CTRL_SET_MAX_PIPELINES: - return 0; - - default: - /* - * Probably a TLS related ctrl. Send back to the frontend SSL_ctrl - * implementation. Either SSL_ctrl will handle it itself by direct - * access into handshake layer state, or failing that, it will be passed - * to the handshake layer via the SSL_METHOD vtable. If the ctrl is not - * supported by anything, the handshake layer's ctrl method will finally - * return 0. - */ - return ossl_ctrl_internal(&ctx.qc->ssl, cmd, larg, parg, /*no_quic=*/1); - } -} - -/* SSL_set_connect_state */ -void ossl_quic_set_connect_state(SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return; - - /* Cannot be changed after handshake started */ - if (ctx.qc->started || ctx.is_stream) - return; - - ctx.qc->as_server_state = 0; -} - -/* SSL_set_accept_state */ -void ossl_quic_set_accept_state(SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return; - - /* Cannot be changed after handshake started */ - if (ctx.qc->started || ctx.is_stream) - return; - - ctx.qc->as_server_state = 1; -} - -/* SSL_do_handshake */ -struct quic_handshake_wait_args { - QUIC_CONNECTION *qc; -}; - -static int tls_wants_non_io_retry(QUIC_CONNECTION *qc) -{ - int want = SSL_want(qc->tls); - - if (want == SSL_X509_LOOKUP - || want == SSL_CLIENT_HELLO_CB - || want == SSL_RETRY_VERIFY) - return 1; - - return 0; -} - -static int quic_handshake_wait(void *arg) -{ - struct quic_handshake_wait_args *args = arg; - - if (!quic_mutation_allowed(args->qc, /*req_active=*/1)) - return -1; - - if (ossl_quic_channel_is_handshake_complete(args->qc->ch)) - return 1; - - if (tls_wants_non_io_retry(args->qc)) - return 1; - - return 0; -} - -static int configure_channel(QUIC_CONNECTION *qc) -{ - assert(qc->ch != NULL); - - if (!ossl_quic_port_set_net_rbio(qc->port, qc->net_rbio) - || !ossl_quic_port_set_net_wbio(qc->port, qc->net_wbio) - || !ossl_quic_channel_set_peer_addr(qc->ch, &qc->init_peer_addr)) - return 0; - - return 1; -} - -QUIC_NEEDS_LOCK -static int create_channel(QUIC_CONNECTION *qc) -{ - QUIC_ENGINE_ARGS engine_args = {0}; - QUIC_PORT_ARGS port_args = {0}; - - engine_args.libctx = qc->ssl.ctx->libctx; - engine_args.propq = qc->ssl.ctx->propq; - engine_args.mutex = qc->mutex; - engine_args.now_cb = get_time_cb; - engine_args.now_cb_arg = qc; - qc->engine = ossl_quic_engine_new(&engine_args); - if (qc->engine == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_INTERNAL_ERROR, NULL); - return 0; - } - - port_args.channel_ctx = qc->ssl.ctx; - qc->port = ossl_quic_engine_create_port(qc->engine, &port_args); - if (qc->port == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_INTERNAL_ERROR, NULL); - ossl_quic_engine_free(qc->engine); - return 0; - } - - qc->ch = ossl_quic_port_create_outgoing(qc->port, qc->tls); - if (qc->ch == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_INTERNAL_ERROR, NULL); - ossl_quic_port_free(qc->port); - ossl_quic_engine_free(qc->engine); - return 0; - } - - return 1; -} - -/* - * Configures a channel with the information we have accumulated via calls made - * to us from the application prior to starting a handshake attempt. - */ -QUIC_NEEDS_LOCK -static int ensure_channel_started(QCTX *ctx) -{ - QUIC_CONNECTION *qc = ctx->qc; - - if (!qc->started) { - if (!configure_channel(qc)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, - "failed to configure channel"); - return 0; - } - - if (!ossl_quic_channel_start(qc->ch)) { - ossl_quic_channel_restore_err_state(qc->ch); - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, - "failed to start channel"); - return 0; - } - -#if !defined(OPENSSL_NO_QUIC_THREAD_ASSIST) - if (qc->is_thread_assisted) - if (!ossl_quic_thread_assist_init_start(&qc->thread_assist, qc->ch, - qc->override_now_cb, - qc->override_now_cb_arg)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, - "failed to start assist thread"); - return 0; - } -#endif - } - - qc->started = 1; - return 1; -} - -QUIC_NEEDS_LOCK -static int quic_do_handshake(QCTX *ctx) -{ - int ret; - QUIC_CONNECTION *qc = ctx->qc; - - if (ossl_quic_channel_is_handshake_complete(qc->ch)) - /* Handshake already completed. */ - return 1; - - if (!quic_mutation_allowed(qc, /*req_active=*/0)) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - - if (qc->as_server != qc->as_server_state) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_PASSED_INVALID_ARGUMENT, NULL); - return -1; /* Non-protocol error */ - } - - if (qc->net_rbio == NULL || qc->net_wbio == NULL) { - /* Need read and write BIOs. */ - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_BIO_NOT_SET, NULL); - return -1; /* Non-protocol error */ - } - - /* - * We need to determine our addressing mode. There are basically two - * ways we can use L4 addresses: - * - * - Addressed mode, in which our BIO_sendmmsg calls have destination - * addresses attached to them which we expect the underlying network BIO - * to handle; - * - * - Unaddressed mode, in which the BIO provided to us on the - * network side neither provides us with L4 addresses nor is capable of - * honouring ones we provide. We don't know where the QUIC traffic we - * send ends up exactly and trust the application to know what it is - * doing. - * - * Addressed mode is preferred because it enables support for connection - * migration, multipath, etc. in the future. Addressed mode is automatically - * enabled if we are using e.g. BIO_s_datagram, with or without - * BIO_s_connect. - * - * If we are passed a BIO_s_dgram_pair (or some custom BIO) we may have to - * use unaddressed mode unless that BIO supports capability flags indicating - * it can provide and honour L4 addresses. - * - * Our strategy for determining address mode is simple: we probe the - * underlying network BIOs for their capabilities. If the network BIOs - * support what we need, we use addressed mode. Otherwise, we use - * unaddressed mode. - * - * If addressed mode is chosen, we require an initial peer address to be - * set. If this is not set, we fail. If unaddressed mode is used, we do not - * require this, as such an address is superfluous, though it can be set if - * desired. - */ - if (!qc->started && !qc->addressing_probe_done) { - long rcaps = BIO_dgram_get_effective_caps(qc->net_rbio); - long wcaps = BIO_dgram_get_effective_caps(qc->net_wbio); - - qc->addressed_mode_r = ((rcaps & BIO_DGRAM_CAP_PROVIDES_SRC_ADDR) != 0); - qc->addressed_mode_w = ((wcaps & BIO_DGRAM_CAP_HANDLES_DST_ADDR) != 0); - qc->addressing_probe_done = 1; - } - - if (!qc->started && qc->addressed_mode_w - && BIO_ADDR_family(&qc->init_peer_addr) == AF_UNSPEC) { - /* - * We are trying to connect and are using addressed mode, which means we - * need an initial peer address; if we do not have a peer address yet, - * we should try to autodetect one. - * - * We do this as late as possible because some BIOs (e.g. BIO_s_connect) - * may not be able to provide us with a peer address until they have - * finished their own processing. They may not be able to perform this - * processing until an application has finished configuring that BIO - * (e.g. with setter calls), which might happen after SSL_set_bio is - * called. - */ - if (!csm_analyse_init_peer_addr(qc->net_wbio, &qc->init_peer_addr)) - /* best effort */ - BIO_ADDR_clear(&qc->init_peer_addr); - else - ossl_quic_channel_set_peer_addr(qc->ch, &qc->init_peer_addr); - } - - if (!qc->started - && qc->addressed_mode_w - && BIO_ADDR_family(&qc->init_peer_addr) == AF_UNSPEC) { - /* - * If we still don't have a peer address in addressed mode, we can't do - * anything. - */ - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_REMOTE_PEER_ADDRESS_NOT_SET, NULL); - return -1; /* Non-protocol error */ - } - - /* - * Start connection process. Note we may come here multiple times in - * non-blocking mode, which is fine. - */ - if (!ensure_channel_started(ctx)) /* raises on failure */ - return -1; /* Non-protocol error */ - - if (ossl_quic_channel_is_handshake_complete(qc->ch)) - /* The handshake is now done. */ - return 1; - - if (!qc_blocking_mode(qc)) { - /* Try to advance the reactor. */ - qctx_maybe_autotick(ctx); - - if (ossl_quic_channel_is_handshake_complete(qc->ch)) - /* The handshake is now done. */ - return 1; - - if (ossl_quic_channel_is_term_any(qc->ch)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - return 0; - } else if (qc->desires_blocking) { - /* - * As a special case when doing a handshake when blocking mode is - * desired yet not available, see if the network BIOs have become - * poll descriptor-enabled. This supports BIOs such as BIO_s_connect - * which do late creation of socket FDs and therefore cannot expose - * a poll descriptor until after a network BIO is set on the QCSO. - */ - assert(!qc->blocking); - qc_update_can_support_blocking(qc); - qc_update_blocking_mode(qc); - } - } - - /* - * We are either in blocking mode or just entered it due to the code above. - */ - if (qc_blocking_mode(qc)) { - /* In blocking mode, wait for the handshake to complete. */ - struct quic_handshake_wait_args args; - - args.qc = qc; - - ret = block_until_pred(qc, quic_handshake_wait, &args, 0); - if (!quic_mutation_allowed(qc, /*req_active=*/1)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - return 0; /* Shutdown before completion */ - } else if (ret <= 0) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - return -1; /* Non-protocol error */ - } - - if (tls_wants_non_io_retry(qc)) { - QUIC_RAISE_NORMAL_ERROR(ctx, SSL_get_error(qc->tls, 0)); - return -1; - } - - assert(ossl_quic_channel_is_handshake_complete(qc->ch)); - return 1; - } - - if (tls_wants_non_io_retry(qc)) { - QUIC_RAISE_NORMAL_ERROR(ctx, SSL_get_error(qc->tls, 0)); - return -1; - } - - /* - * Otherwise, indicate that the handshake isn't done yet. - * We can only get here in non-blocking mode. - */ - QUIC_RAISE_NORMAL_ERROR(ctx, SSL_ERROR_WANT_READ); - return -1; /* Non-protocol error */ -} - -QUIC_TAKES_LOCK -int ossl_quic_do_handshake(SSL *s) -{ - int ret; - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock_for_io(&ctx); - - ret = quic_do_handshake(&ctx); - quic_unlock(ctx.qc); - return ret; -} - -/* SSL_connect */ -int ossl_quic_connect(SSL *s) -{ - /* Ensure we are in connect state (no-op if non-idle). */ - ossl_quic_set_connect_state(s); - - /* Begin or continue the handshake */ - return ossl_quic_do_handshake(s); -} - -/* SSL_accept */ -int ossl_quic_accept(SSL *s) -{ - /* Ensure we are in accept state (no-op if non-idle). */ - ossl_quic_set_accept_state(s); - - /* Begin or continue the handshake */ - return ossl_quic_do_handshake(s); -} - -/* - * QUIC Front-End I/O API: Stream Lifecycle Operations - * =================================================== - * - * SSL_stream_new => ossl_quic_conn_stream_new - * - */ - -/* - * Try to create the default XSO if it doesn't already exist. Returns 1 if the - * default XSO was created. Returns 0 if it was not (e.g. because it already - * exists). Note that this is NOT an error condition. - */ -QUIC_NEEDS_LOCK -static int qc_try_create_default_xso_for_write(QCTX *ctx) -{ - uint64_t flags = 0; - QUIC_CONNECTION *qc = ctx->qc; - - if (qc->default_xso_created - || qc->default_stream_mode == SSL_DEFAULT_STREAM_MODE_NONE) - /* - * We only do this once. If the user detaches a previously created - * default XSO we don't auto-create another one. - */ - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_NO_STREAM, NULL); - - /* Create a locally-initiated stream. */ - if (qc->default_stream_mode == SSL_DEFAULT_STREAM_MODE_AUTO_UNI) - flags |= SSL_STREAM_FLAG_UNI; - - qc_set_default_xso(qc, (QUIC_XSO *)quic_conn_stream_new(ctx, flags, - /*needs_lock=*/0), - /*touch=*/0); - if (qc->default_xso == NULL) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - - qc_touch_default_xso(qc); - return 1; -} - -struct quic_wait_for_stream_args { - QUIC_CONNECTION *qc; - QUIC_STREAM *qs; - QCTX *ctx; - uint64_t expect_id; -}; - -QUIC_NEEDS_LOCK -static int quic_wait_for_stream(void *arg) -{ - struct quic_wait_for_stream_args *args = arg; - - if (!quic_mutation_allowed(args->qc, /*req_active=*/1)) { - /* If connection is torn down due to an error while blocking, stop. */ - QUIC_RAISE_NON_NORMAL_ERROR(args->ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - return -1; - } - - args->qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(args->qc->ch), - args->expect_id | QUIC_STREAM_DIR_BIDI); - if (args->qs == NULL) - args->qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(args->qc->ch), - args->expect_id | QUIC_STREAM_DIR_UNI); - - if (args->qs != NULL) - return 1; /* stream now exists */ - - return 0; /* did not get a stream, keep trying */ -} - -QUIC_NEEDS_LOCK -static int qc_wait_for_default_xso_for_read(QCTX *ctx, int peek) -{ - /* Called on a QCSO and we don't currently have a default stream. */ - uint64_t expect_id; - QUIC_CONNECTION *qc = ctx->qc; - QUIC_STREAM *qs; - int res; - struct quic_wait_for_stream_args wargs; - OSSL_RTT_INFO rtt_info; - - /* - * If default stream functionality is disabled or we already detached - * one, don't make another default stream and just fail. - */ - if (qc->default_xso_created - || qc->default_stream_mode == SSL_DEFAULT_STREAM_MODE_NONE) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_NO_STREAM, NULL); - - /* - * The peer may have opened a stream since we last ticked. So tick and - * see if the stream with ordinal 0 (remote, bidi/uni based on stream - * mode) exists yet. QUIC stream IDs must be allocated in order, so the - * first stream created by a peer must have an ordinal of 0. - */ - expect_id = qc->as_server - ? QUIC_STREAM_INITIATOR_CLIENT - : QUIC_STREAM_INITIATOR_SERVER; - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(qc->ch), - expect_id | QUIC_STREAM_DIR_BIDI); - if (qs == NULL) - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(qc->ch), - expect_id | QUIC_STREAM_DIR_UNI); - - if (qs == NULL) { - qctx_maybe_autotick(ctx); - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(qc->ch), - expect_id); - } - - if (qs == NULL) { - if (peek) - return 0; - - if (!qc_blocking_mode(qc)) - /* Non-blocking mode, so just bail immediately. */ - return QUIC_RAISE_NORMAL_ERROR(ctx, SSL_ERROR_WANT_READ); - - /* Block until we have a stream. */ - wargs.qc = qc; - wargs.qs = NULL; - wargs.ctx = ctx; - wargs.expect_id = expect_id; - - res = block_until_pred(qc, quic_wait_for_stream, &wargs, 0); - if (res == 0) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - else if (res < 0 || wargs.qs == NULL) - /* quic_wait_for_stream raised error here */ - return 0; - - qs = wargs.qs; - } - - /* - * We now have qs != NULL. Remove it from the incoming stream queue so that - * it isn't also returned by any future SSL_accept_stream calls. - */ - ossl_statm_get_rtt_info(ossl_quic_channel_get_statm(qc->ch), &rtt_info); - ossl_quic_stream_map_remove_from_accept_queue(ossl_quic_channel_get_qsm(qc->ch), - qs, rtt_info.smoothed_rtt); - - /* - * Now make qs the default stream, creating the necessary XSO. - */ - qc_set_default_xso(qc, create_xso_from_stream(qc, qs), /*touch=*/0); - if (qc->default_xso == NULL) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - - qc_touch_default_xso(qc); /* inhibits default XSO */ - return 1; -} - -QUIC_NEEDS_LOCK -static QUIC_XSO *create_xso_from_stream(QUIC_CONNECTION *qc, QUIC_STREAM *qs) -{ - QUIC_XSO *xso = NULL; - - if ((xso = OPENSSL_zalloc(sizeof(*xso))) == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_CRYPTO_LIB, NULL); - goto err; - } - - if (!ossl_ssl_init(&xso->ssl, qc->ssl.ctx, qc->ssl.method, SSL_TYPE_QUIC_XSO)) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_INTERNAL_ERROR, NULL); - goto err; - } - - /* XSO refs QC */ - if (!SSL_up_ref(&qc->ssl)) { - QUIC_RAISE_NON_NORMAL_ERROR(NULL, ERR_R_SSL_LIB, NULL); - goto err; - } - - xso->conn = qc; - xso->ssl_mode = qc->default_ssl_mode; - xso->ssl_options - = qc->default_ssl_options & OSSL_QUIC_PERMITTED_OPTIONS_STREAM; - xso->last_error = SSL_ERROR_NONE; - - xso->stream = qs; - - ++qc->num_xso; - xso_update_options(xso); - return xso; - -err: - OPENSSL_free(xso); - return NULL; -} - -struct quic_new_stream_wait_args { - QUIC_CONNECTION *qc; - int is_uni; -}; - -static int quic_new_stream_wait(void *arg) -{ - struct quic_new_stream_wait_args *args = arg; - QUIC_CONNECTION *qc = args->qc; - - if (!quic_mutation_allowed(qc, /*req_active=*/1)) - return -1; - - if (ossl_quic_channel_is_new_local_stream_admissible(qc->ch, args->is_uni)) - return 1; - - return 0; -} - -/* locking depends on need_lock */ -static SSL *quic_conn_stream_new(QCTX *ctx, uint64_t flags, int need_lock) -{ - int ret; - QUIC_CONNECTION *qc = ctx->qc; - QUIC_XSO *xso = NULL; - QUIC_STREAM *qs = NULL; - int is_uni = ((flags & SSL_STREAM_FLAG_UNI) != 0); - int no_blocking = ((flags & SSL_STREAM_FLAG_NO_BLOCK) != 0); - int advance = ((flags & SSL_STREAM_FLAG_ADVANCE) != 0); - - if (need_lock) - quic_lock(qc); - - if (!quic_mutation_allowed(qc, /*req_active=*/0)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - goto err; - } - - if (!advance - && !ossl_quic_channel_is_new_local_stream_admissible(qc->ch, is_uni)) { - struct quic_new_stream_wait_args args; - - /* - * Stream count flow control currently doesn't permit this stream to be - * opened. - */ - if (no_blocking || !qc_blocking_mode(qc)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_STREAM_COUNT_LIMITED, NULL); - goto err; - } - - args.qc = qc; - args.is_uni = is_uni; - - /* Blocking mode - wait until we can get a stream. */ - ret = block_until_pred(ctx->qc, quic_new_stream_wait, &args, 0); - if (!quic_mutation_allowed(qc, /*req_active=*/1)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - goto err; /* Shutdown before completion */ - } else if (ret <= 0) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - goto err; /* Non-protocol error */ - } - } - - qs = ossl_quic_channel_new_stream_local(qc->ch, is_uni); - if (qs == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - goto err; - } - - xso = create_xso_from_stream(qc, qs); - if (xso == NULL) - goto err; - - qc_touch_default_xso(qc); /* inhibits default XSO */ - if (need_lock) - quic_unlock(qc); - - return &xso->ssl; - -err: - OPENSSL_free(xso); - ossl_quic_stream_map_release(ossl_quic_channel_get_qsm(qc->ch), qs); - if (need_lock) - quic_unlock(qc); - - return NULL; - -} - -QUIC_TAKES_LOCK -SSL *ossl_quic_conn_stream_new(SSL *s, uint64_t flags) -{ - QCTX ctx; - - if (!expect_quic_conn_only(s, &ctx)) - return NULL; - - return quic_conn_stream_new(&ctx, flags, /*need_lock=*/1); -} - -/* - * QUIC Front-End I/O API: Steady-State Operations - * =============================================== - * - * Here we dispatch calls to the steady-state front-end I/O API functions; that - * is, the functions used during the established phase of a QUIC connection - * (e.g. SSL_read, SSL_write). - * - * Each function must handle both blocking and non-blocking modes. As discussed - * above, all QUIC I/O is implemented using non-blocking mode internally. - * - * SSL_get_error => partially implemented by ossl_quic_get_error - * SSL_want => ossl_quic_want - * (BIO/)SSL_read => ossl_quic_read - * (BIO/)SSL_write => ossl_quic_write - * SSL_pending => ossl_quic_pending - * SSL_stream_conclude => ossl_quic_conn_stream_conclude - * SSL_key_update => ossl_quic_key_update - */ - -/* SSL_get_error */ -int ossl_quic_get_error(const SSL *s, int i) -{ - QCTX ctx; - int net_error, last_error; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - net_error = ossl_quic_channel_net_error(ctx.qc->ch); - last_error = ctx.is_stream ? ctx.xso->last_error : ctx.qc->last_error; - quic_unlock(ctx.qc); - - if (net_error) - return SSL_ERROR_SYSCALL; - - return last_error; -} - -/* Converts a code returned by SSL_get_error to a code returned by SSL_want. */ -static int error_to_want(int error) -{ - switch (error) { - case SSL_ERROR_WANT_CONNECT: /* never used - UDP is connectionless */ - case SSL_ERROR_WANT_ACCEPT: /* never used - UDP is connectionless */ - case SSL_ERROR_ZERO_RETURN: - default: - return SSL_NOTHING; - - case SSL_ERROR_WANT_READ: - return SSL_READING; - - case SSL_ERROR_WANT_WRITE: - return SSL_WRITING; - - case SSL_ERROR_WANT_RETRY_VERIFY: - return SSL_RETRY_VERIFY; - - case SSL_ERROR_WANT_CLIENT_HELLO_CB: - return SSL_CLIENT_HELLO_CB; - - case SSL_ERROR_WANT_X509_LOOKUP: - return SSL_X509_LOOKUP; - } -} - -/* SSL_want */ -int ossl_quic_want(const SSL *s) -{ - QCTX ctx; - int w; - - if (!expect_quic(s, &ctx)) - return SSL_NOTHING; - - quic_lock(ctx.qc); - - w = error_to_want(ctx.is_stream ? ctx.xso->last_error : ctx.qc->last_error); - - quic_unlock(ctx.qc); - return w; -} - -/* - * SSL_write - * --------- - * - * The set of functions below provide the implementation of the public SSL_write - * function. We must handle: - * - * - both blocking and non-blocking operation at the application level, - * depending on how we are configured; - * - * - SSL_MODE_ENABLE_PARTIAL_WRITE being on or off; - * - * - SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER. - * - */ -QUIC_NEEDS_LOCK -static void quic_post_write(QUIC_XSO *xso, int did_append, - int did_append_all, uint64_t flags, - int do_tick) -{ - /* - * We have appended at least one byte to the stream. - * Potentially mark stream as active, depending on FC. - */ - if (did_append) - ossl_quic_stream_map_update_state(ossl_quic_channel_get_qsm(xso->conn->ch), - xso->stream); - - if (did_append_all && (flags & SSL_WRITE_FLAG_CONCLUDE) != 0) - ossl_quic_sstream_fin(xso->stream->sstream); - - /* - * Try and send. - * - * TODO(QUIC FUTURE): It is probably inefficient to try and do this - * immediately, plus we should eventually consider Nagle's algorithm. - */ - if (do_tick) - ossl_quic_reactor_tick(ossl_quic_channel_get_reactor(xso->conn->ch), 0); -} - -struct quic_write_again_args { - QUIC_XSO *xso; - const unsigned char *buf; - size_t len; - size_t total_written; - int err; - uint64_t flags; -}; - -/* - * Absolute maximum write buffer size, enforced to prevent a rogue peer from - * deliberately inducing DoS. This has been chosen based on the optimal buffer - * size for an RTT of 500ms and a bandwidth of 100 Mb/s. - */ -#define MAX_WRITE_BUF_SIZE (6 * 1024 * 1024) - -/* - * Ensure spare buffer space available (up until a limit, at least). - */ -QUIC_NEEDS_LOCK -static int sstream_ensure_spare(QUIC_SSTREAM *sstream, uint64_t spare) -{ - size_t cur_sz = ossl_quic_sstream_get_buffer_size(sstream); - size_t avail = ossl_quic_sstream_get_buffer_avail(sstream); - size_t spare_ = (spare > SIZE_MAX) ? SIZE_MAX : (size_t)spare; - size_t new_sz, growth; - - if (spare_ <= avail || cur_sz == MAX_WRITE_BUF_SIZE) - return 1; - - growth = spare_ - avail; - if (cur_sz + growth > MAX_WRITE_BUF_SIZE) - new_sz = MAX_WRITE_BUF_SIZE; - else - new_sz = cur_sz + growth; - - return ossl_quic_sstream_set_buffer_size(sstream, new_sz); -} - -/* - * Append to a QUIC_STREAM's QUIC_SSTREAM, ensuring buffer space is expanded - * as needed according to flow control. - */ -QUIC_NEEDS_LOCK -static int xso_sstream_append(QUIC_XSO *xso, const unsigned char *buf, - size_t len, size_t *actual_written) -{ - QUIC_SSTREAM *sstream = xso->stream->sstream; - uint64_t cur = ossl_quic_sstream_get_cur_size(sstream); - uint64_t cwm = ossl_quic_txfc_get_cwm(&xso->stream->txfc); - uint64_t permitted = (cwm >= cur ? cwm - cur : 0); - - if (len > permitted) - len = (size_t)permitted; - - if (!sstream_ensure_spare(sstream, len)) - return 0; - - return ossl_quic_sstream_append(sstream, buf, len, actual_written); -} - -QUIC_NEEDS_LOCK -static int quic_write_again(void *arg) -{ - struct quic_write_again_args *args = arg; - size_t actual_written = 0; - - if (!quic_mutation_allowed(args->xso->conn, /*req_active=*/1)) - /* If connection is torn down due to an error while blocking, stop. */ - return -2; - - if (!quic_validate_for_write(args->xso, &args->err)) - /* - * Stream may have become invalid for write due to connection events - * while we blocked. - */ - return -2; - - args->err = ERR_R_INTERNAL_ERROR; - if (!xso_sstream_append(args->xso, args->buf, args->len, &actual_written)) - return -2; - - quic_post_write(args->xso, actual_written > 0, - args->len == actual_written, args->flags, 0); - - args->buf += actual_written; - args->len -= actual_written; - args->total_written += actual_written; - - if (args->len == 0) - /* Written everything, done. */ - return 1; - - /* Not written everything yet, keep trying. */ - return 0; -} - -QUIC_NEEDS_LOCK -static int quic_write_blocking(QCTX *ctx, const void *buf, size_t len, - uint64_t flags, size_t *written) -{ - int res; - QUIC_XSO *xso = ctx->xso; - struct quic_write_again_args args; - size_t actual_written = 0; - - /* First make a best effort to append as much of the data as possible. */ - if (!xso_sstream_append(xso, buf, len, &actual_written)) { - /* Stream already finished or allocation error. */ - *written = 0; - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - } - - quic_post_write(xso, actual_written > 0, actual_written == len, flags, 1); - - if (actual_written == len) { - /* Managed to append everything on the first try. */ - *written = actual_written; - return 1; - } - - /* - * We did not manage to append all of the data immediately, so the stream - * buffer has probably filled up. This means we need to block until some of - * it is freed up. - */ - args.xso = xso; - args.buf = (const unsigned char *)buf + actual_written; - args.len = len - actual_written; - args.total_written = 0; - args.err = ERR_R_INTERNAL_ERROR; - args.flags = flags; - - res = block_until_pred(xso->conn, quic_write_again, &args, 0); - if (res <= 0) { - if (!quic_mutation_allowed(xso->conn, /*req_active=*/1)) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - else - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, args.err, NULL); - } - - *written = args.total_written; - return 1; -} - -/* - * Functions to manage All-or-Nothing (AON) (that is, non-ENABLE_PARTIAL_WRITE) - * write semantics. - */ -static void aon_write_begin(QUIC_XSO *xso, const unsigned char *buf, - size_t buf_len, size_t already_sent) -{ - assert(!xso->aon_write_in_progress); - - xso->aon_write_in_progress = 1; - xso->aon_buf_base = buf; - xso->aon_buf_pos = already_sent; - xso->aon_buf_len = buf_len; -} - -static void aon_write_finish(QUIC_XSO *xso) -{ - xso->aon_write_in_progress = 0; - xso->aon_buf_base = NULL; - xso->aon_buf_pos = 0; - xso->aon_buf_len = 0; -} - -QUIC_NEEDS_LOCK -static int quic_write_nonblocking_aon(QCTX *ctx, const void *buf, - size_t len, uint64_t flags, - size_t *written) -{ - QUIC_XSO *xso = ctx->xso; - const void *actual_buf; - size_t actual_len, actual_written = 0; - int accept_moving_buffer - = ((xso->ssl_mode & SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) != 0); - - if (xso->aon_write_in_progress) { - /* - * We are in the middle of an AON write (i.e., a previous write did not - * manage to append all data to the SSTREAM and we have Enable Partial - * Write (EPW) mode disabled.) - */ - if ((!accept_moving_buffer && xso->aon_buf_base != buf) - || len != xso->aon_buf_len) - /* - * Pointer must not have changed if we are not in accept moving - * buffer mode. Length must never change. - */ - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_BAD_WRITE_RETRY, NULL); - - actual_buf = (unsigned char *)buf + xso->aon_buf_pos; - actual_len = len - xso->aon_buf_pos; - assert(actual_len > 0); - } else { - actual_buf = buf; - actual_len = len; - } - - /* First make a best effort to append as much of the data as possible. */ - if (!xso_sstream_append(xso, actual_buf, actual_len, &actual_written)) { - /* Stream already finished or allocation error. */ - *written = 0; - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - } - - quic_post_write(xso, actual_written > 0, actual_written == actual_len, - flags, qctx_should_autotick(ctx)); - - if (actual_written == actual_len) { - /* We have sent everything. */ - if (xso->aon_write_in_progress) { - /* - * We have sent everything, and we were in the middle of an AON - * write. The output write length is the total length of the AON - * buffer, not however many bytes we managed to write to the stream - * in this call. - */ - *written = xso->aon_buf_len; - aon_write_finish(xso); - } else { - *written = actual_written; - } - - return 1; - } - - if (xso->aon_write_in_progress) { - /* - * AON write is in progress but we have not written everything yet. We - * may have managed to send zero bytes, or some number of bytes less - * than the total remaining which need to be appended during this - * AON operation. - */ - xso->aon_buf_pos += actual_written; - assert(xso->aon_buf_pos < xso->aon_buf_len); - return QUIC_RAISE_NORMAL_ERROR(ctx, SSL_ERROR_WANT_WRITE); - } - - /* - * Not in an existing AON operation but partial write is not enabled, so we - * need to begin a new AON operation. However we needn't bother if we didn't - * actually append anything. - */ - if (actual_written > 0) - aon_write_begin(xso, buf, len, actual_written); - - /* - * AON - We do not publicly admit to having appended anything until AON - * completes. - */ - *written = 0; - return QUIC_RAISE_NORMAL_ERROR(ctx, SSL_ERROR_WANT_WRITE); -} - -QUIC_NEEDS_LOCK -static int quic_write_nonblocking_epw(QCTX *ctx, const void *buf, size_t len, - uint64_t flags, size_t *written) -{ - QUIC_XSO *xso = ctx->xso; - - /* Simple best effort operation. */ - if (!xso_sstream_append(xso, buf, len, written)) { - /* Stream already finished or allocation error. */ - *written = 0; - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - } - - quic_post_write(xso, *written > 0, *written == len, flags, - qctx_should_autotick(ctx)); - - if (*written == 0) - /* SSL_write_ex returns 0 if it didn't read anything. */ - return QUIC_RAISE_NORMAL_ERROR(ctx, SSL_ERROR_WANT_READ); - - return 1; -} - -QUIC_NEEDS_LOCK -static int quic_validate_for_write(QUIC_XSO *xso, int *err) -{ - QUIC_STREAM_MAP *qsm; - - if (xso == NULL || xso->stream == NULL) { - *err = ERR_R_INTERNAL_ERROR; - return 0; - } - - switch (xso->stream->send_state) { - default: - case QUIC_SSTREAM_STATE_NONE: - *err = SSL_R_STREAM_RECV_ONLY; - return 0; - - case QUIC_SSTREAM_STATE_READY: - qsm = ossl_quic_channel_get_qsm(xso->conn->ch); - - if (!ossl_quic_stream_map_ensure_send_part_id(qsm, xso->stream)) { - *err = ERR_R_INTERNAL_ERROR; - return 0; - } - - /* FALLTHROUGH */ - case QUIC_SSTREAM_STATE_SEND: - case QUIC_SSTREAM_STATE_DATA_SENT: - case QUIC_SSTREAM_STATE_DATA_RECVD: - if (ossl_quic_sstream_get_final_size(xso->stream->sstream, NULL)) { - *err = SSL_R_STREAM_FINISHED; - return 0; - } - - return 1; - - case QUIC_SSTREAM_STATE_RESET_SENT: - case QUIC_SSTREAM_STATE_RESET_RECVD: - *err = SSL_R_STREAM_RESET; - return 0; - } -} - -QUIC_TAKES_LOCK -int ossl_quic_write_flags(SSL *s, const void *buf, size_t len, - uint64_t flags, size_t *written) -{ - int ret; - QCTX ctx; - int partial_write, err; - - *written = 0; - - if (len == 0) { - /* Do not autocreate default XSO for zero-length writes. */ - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock_for_io(&ctx); - } else { - if (!expect_quic_with_stream_lock(s, /*remote_init=*/0, /*io=*/1, &ctx)) - return 0; - } - - partial_write = ((ctx.xso != NULL) - ? ((ctx.xso->ssl_mode & SSL_MODE_ENABLE_PARTIAL_WRITE) != 0) : 0); - - if ((flags & ~SSL_WRITE_FLAG_CONCLUDE) != 0) { - ret = QUIC_RAISE_NON_NORMAL_ERROR(&ctx, SSL_R_UNSUPPORTED_WRITE_FLAG, NULL); - goto out; - } - - if (!quic_mutation_allowed(ctx.qc, /*req_active=*/0)) { - ret = QUIC_RAISE_NON_NORMAL_ERROR(&ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - goto out; - } - - /* - * If we haven't finished the handshake, try to advance it. - * We don't accept writes until the handshake is completed. - */ - if (quic_do_handshake(&ctx) < 1) { - ret = 0; - goto out; - } - - /* Ensure correct stream state, stream send part not concluded, etc. */ - if (len > 0 && !quic_validate_for_write(ctx.xso, &err)) { - ret = QUIC_RAISE_NON_NORMAL_ERROR(&ctx, err, NULL); - goto out; - } - - if (len == 0) { - if ((flags & SSL_WRITE_FLAG_CONCLUDE) != 0) - quic_post_write(ctx.xso, 0, 1, flags, - qctx_should_autotick(&ctx)); - - ret = 1; - goto out; - } - - if (xso_blocking_mode(ctx.xso)) - ret = quic_write_blocking(&ctx, buf, len, flags, written); - else if (partial_write) - ret = quic_write_nonblocking_epw(&ctx, buf, len, flags, written); - else - ret = quic_write_nonblocking_aon(&ctx, buf, len, flags, written); - -out: - quic_unlock(ctx.qc); - return ret; -} - -QUIC_TAKES_LOCK -int ossl_quic_write(SSL *s, const void *buf, size_t len, size_t *written) -{ - return ossl_quic_write_flags(s, buf, len, 0, written); -} - -/* - * SSL_read - * -------- - */ -struct quic_read_again_args { - QCTX *ctx; - QUIC_STREAM *stream; - void *buf; - size_t len; - size_t *bytes_read; - int peek; -}; - -QUIC_NEEDS_LOCK -static int quic_validate_for_read(QUIC_XSO *xso, int *err, int *eos) -{ - QUIC_STREAM_MAP *qsm; - - *eos = 0; - - if (xso == NULL || xso->stream == NULL) { - *err = ERR_R_INTERNAL_ERROR; - return 0; - } - - switch (xso->stream->recv_state) { - default: - case QUIC_RSTREAM_STATE_NONE: - *err = SSL_R_STREAM_SEND_ONLY; - return 0; - - case QUIC_RSTREAM_STATE_RECV: - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - case QUIC_RSTREAM_STATE_DATA_RECVD: - return 1; - - case QUIC_RSTREAM_STATE_DATA_READ: - *eos = 1; - return 0; - - case QUIC_RSTREAM_STATE_RESET_RECVD: - qsm = ossl_quic_channel_get_qsm(xso->conn->ch); - ossl_quic_stream_map_notify_app_read_reset_recv_part(qsm, xso->stream); - - /* FALLTHROUGH */ - case QUIC_RSTREAM_STATE_RESET_READ: - *err = SSL_R_STREAM_RESET; - return 0; - } -} - -QUIC_NEEDS_LOCK -static int quic_read_actual(QCTX *ctx, - QUIC_STREAM *stream, - void *buf, size_t buf_len, - size_t *bytes_read, - int peek) -{ - int is_fin = 0, err, eos; - QUIC_CONNECTION *qc = ctx->qc; - - if (!quic_validate_for_read(ctx->xso, &err, &eos)) { - if (eos) { - ctx->xso->retired_fin = 1; - return QUIC_RAISE_NORMAL_ERROR(ctx, SSL_ERROR_ZERO_RETURN); - } else { - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, err, NULL); - } - } - - if (peek) { - if (!ossl_quic_rstream_peek(stream->rstream, buf, buf_len, - bytes_read, &is_fin)) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - - } else { - if (!ossl_quic_rstream_read(stream->rstream, buf, buf_len, - bytes_read, &is_fin)) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - } - - if (!peek) { - if (*bytes_read > 0) { - /* - * We have read at least one byte from the stream. Inform stream-level - * RXFC of the retirement of controlled bytes. Update the active stream - * status (the RXFC may now want to emit a frame granting more credit to - * the peer). - */ - OSSL_RTT_INFO rtt_info; - - ossl_statm_get_rtt_info(ossl_quic_channel_get_statm(qc->ch), &rtt_info); - - if (!ossl_quic_rxfc_on_retire(&stream->rxfc, *bytes_read, - rtt_info.smoothed_rtt)) - return QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_INTERNAL_ERROR, NULL); - } - - if (is_fin && !peek) { - QUIC_STREAM_MAP *qsm = ossl_quic_channel_get_qsm(ctx->qc->ch); - - ossl_quic_stream_map_notify_totally_read(qsm, ctx->xso->stream); - } - - if (*bytes_read > 0) - ossl_quic_stream_map_update_state(ossl_quic_channel_get_qsm(qc->ch), - stream); - } - - if (*bytes_read == 0 && is_fin) { - ctx->xso->retired_fin = 1; - return QUIC_RAISE_NORMAL_ERROR(ctx, SSL_ERROR_ZERO_RETURN); - } - - return 1; -} - -QUIC_NEEDS_LOCK -static int quic_read_again(void *arg) -{ - struct quic_read_again_args *args = arg; - - if (!quic_mutation_allowed(args->ctx->qc, /*req_active=*/1)) { - /* If connection is torn down due to an error while blocking, stop. */ - QUIC_RAISE_NON_NORMAL_ERROR(args->ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - return -1; - } - - if (!quic_read_actual(args->ctx, args->stream, - args->buf, args->len, args->bytes_read, - args->peek)) - return -1; - - if (*args->bytes_read > 0) - /* got at least one byte, the SSL_read op can finish now */ - return 1; - - return 0; /* did not read anything, keep trying */ -} - -QUIC_TAKES_LOCK -static int quic_read(SSL *s, void *buf, size_t len, size_t *bytes_read, int peek) -{ - int ret, res; - QCTX ctx; - struct quic_read_again_args args; - - *bytes_read = 0; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock_for_io(&ctx); - - if (!quic_mutation_allowed(ctx.qc, /*req_active=*/0)) { - ret = QUIC_RAISE_NON_NORMAL_ERROR(&ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - goto out; - } - - /* If we haven't finished the handshake, try to advance it. */ - if (quic_do_handshake(&ctx) < 1) { - ret = 0; /* ossl_quic_do_handshake raised error here */ - goto out; - } - - if (ctx.xso == NULL) { - /* - * Called on a QCSO and we don't currently have a default stream. - * - * Wait until we get a stream initiated by the peer (blocking mode) or - * fail if we don't have one yet (non-blocking mode). - */ - if (!qc_wait_for_default_xso_for_read(&ctx, /*peek=*/0)) { - ret = 0; /* error already raised here */ - goto out; - } - - ctx.xso = ctx.qc->default_xso; - } - - if (!quic_read_actual(&ctx, ctx.xso->stream, buf, len, bytes_read, peek)) { - ret = 0; /* quic_read_actual raised error here */ - goto out; - } - - if (*bytes_read > 0) { - /* - * Even though we succeeded, tick the reactor here to ensure we are - * handling other aspects of the QUIC connection. - */ - qctx_maybe_autotick(&ctx); - ret = 1; - } else if (xso_blocking_mode(ctx.xso)) { - /* - * We were not able to read anything immediately, so our stream - * buffer is empty. This means we need to block until we get - * at least one byte. - */ - args.ctx = &ctx; - args.stream = ctx.xso->stream; - args.buf = buf; - args.len = len; - args.bytes_read = bytes_read; - args.peek = peek; - - res = block_until_pred(ctx.qc, quic_read_again, &args, 0); - if (res == 0) { - ret = QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_INTERNAL_ERROR, NULL); - goto out; - } else if (res < 0) { - ret = 0; /* quic_read_again raised error here */ - goto out; - } - - ret = 1; - } else { - /* - * We did not get any bytes and are not in blocking mode. - * Tick to see if this delivers any more. - */ - qctx_maybe_autotick(&ctx); - - /* Try the read again. */ - if (!quic_read_actual(&ctx, ctx.xso->stream, buf, len, bytes_read, peek)) { - ret = 0; /* quic_read_actual raised error here */ - goto out; - } - - if (*bytes_read > 0) - ret = 1; /* Succeeded this time. */ - else - ret = QUIC_RAISE_NORMAL_ERROR(&ctx, SSL_ERROR_WANT_READ); - } - -out: - quic_unlock(ctx.qc); - return ret; -} - -int ossl_quic_read(SSL *s, void *buf, size_t len, size_t *bytes_read) -{ - return quic_read(s, buf, len, bytes_read, 0); -} - -int ossl_quic_peek(SSL *s, void *buf, size_t len, size_t *bytes_read) -{ - return quic_read(s, buf, len, bytes_read, 1); -} - -/* - * SSL_pending - * ----------- - */ - -QUIC_TAKES_LOCK -static size_t ossl_quic_pending_int(const SSL *s, int check_channel) -{ - QCTX ctx; - size_t avail = 0; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - - if (ctx.xso == NULL) { - /* No XSO yet, but there might be a default XSO eligible to be created. */ - if (qc_wait_for_default_xso_for_read(&ctx, /*peek=*/1)) { - ctx.xso = ctx.qc->default_xso; - } else { - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, SSL_R_NO_STREAM, NULL); - goto out; - } - } - - if (ctx.xso->stream == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_INTERNAL_ERROR, NULL); - goto out; - } - - if (check_channel) - avail = ossl_quic_stream_recv_pending(ctx.xso->stream, - /*include_fin=*/1) - || ossl_quic_channel_has_pending(ctx.qc->ch) - || ossl_quic_channel_is_term_any(ctx.qc->ch); - else - avail = ossl_quic_stream_recv_pending(ctx.xso->stream, - /*include_fin=*/0); - -out: - quic_unlock(ctx.qc); - return avail; -} - -size_t ossl_quic_pending(const SSL *s) -{ - return ossl_quic_pending_int(s, /*check_channel=*/0); -} - -int ossl_quic_has_pending(const SSL *s) -{ - /* Do we have app-side pending data or pending URXEs or RXEs? */ - return ossl_quic_pending_int(s, /*check_channel=*/1) > 0; -} - -/* - * SSL_stream_conclude - * ------------------- - */ -QUIC_TAKES_LOCK -int ossl_quic_conn_stream_conclude(SSL *s) -{ - QCTX ctx; - QUIC_STREAM *qs; - int err; - - if (!expect_quic_with_stream_lock(s, /*remote_init=*/0, /*io=*/0, &ctx)) - return 0; - - qs = ctx.xso->stream; - - if (!quic_mutation_allowed(ctx.qc, /*req_active=*/1)) { - quic_unlock(ctx.qc); - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - } - - if (!quic_validate_for_write(ctx.xso, &err)) { - quic_unlock(ctx.qc); - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, err, NULL); - } - - if (ossl_quic_sstream_get_final_size(qs->sstream, NULL)) { - quic_unlock(ctx.qc); - return 1; - } - - ossl_quic_sstream_fin(qs->sstream); - quic_post_write(ctx.xso, 1, 0, 0, qctx_should_autotick(&ctx)); - quic_unlock(ctx.qc); - return 1; -} - -/* - * SSL_inject_net_dgram - * -------------------- - */ -QUIC_TAKES_LOCK -int SSL_inject_net_dgram(SSL *s, const unsigned char *buf, - size_t buf_len, - const BIO_ADDR *peer, - const BIO_ADDR *local) -{ - int ret; - QCTX ctx; - QUIC_DEMUX *demux; - - if (!expect_quic(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - - demux = ossl_quic_channel_get0_demux(ctx.qc->ch); - ret = ossl_quic_demux_inject(demux, buf, buf_len, peer, local); - - quic_unlock(ctx.qc); - return ret; -} - -/* - * SSL_get0_connection - * ------------------- - */ -SSL *ossl_quic_get0_connection(SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return NULL; - - return &ctx.qc->ssl; -} - -/* - * SSL_get_stream_type - * ------------------- - */ -int ossl_quic_get_stream_type(SSL *s) -{ - QCTX ctx; - - if (!expect_quic(s, &ctx)) - return SSL_STREAM_TYPE_BIDI; - - if (ctx.xso == NULL) { - /* - * If deferred XSO creation has yet to occur, proceed according to the - * default stream mode. If AUTO_BIDI or AUTO_UNI is set, we cannot know - * what kind of stream will be created yet, so return BIDI on the basis - * that at this time, the client still has the option of calling - * SSL_read() or SSL_write() first. - */ - if (ctx.qc->default_xso_created - || ctx.qc->default_stream_mode == SSL_DEFAULT_STREAM_MODE_NONE) - return SSL_STREAM_TYPE_NONE; - else - return SSL_STREAM_TYPE_BIDI; - } - - if (ossl_quic_stream_is_bidi(ctx.xso->stream)) - return SSL_STREAM_TYPE_BIDI; - - if (ossl_quic_stream_is_server_init(ctx.xso->stream) != ctx.qc->as_server) - return SSL_STREAM_TYPE_READ; - else - return SSL_STREAM_TYPE_WRITE; -} - -/* - * SSL_get_stream_id - * ----------------- - */ -QUIC_TAKES_LOCK -uint64_t ossl_quic_get_stream_id(SSL *s) -{ - QCTX ctx; - uint64_t id; - - if (!expect_quic_with_stream_lock(s, /*remote_init=*/-1, /*io=*/0, &ctx)) - return UINT64_MAX; - - id = ctx.xso->stream->id; - quic_unlock(ctx.qc); - - return id; -} - -/* - * SSL_is_stream_local - * ------------------- - */ -QUIC_TAKES_LOCK -int ossl_quic_is_stream_local(SSL *s) -{ - QCTX ctx; - int is_local; - - if (!expect_quic_with_stream_lock(s, /*remote_init=*/-1, /*io=*/0, &ctx)) - return -1; - - is_local = ossl_quic_stream_is_local_init(ctx.xso->stream); - quic_unlock(ctx.qc); - - return is_local; -} - -/* - * SSL_set_default_stream_mode - * --------------------------- - */ -QUIC_TAKES_LOCK -int ossl_quic_set_default_stream_mode(SSL *s, uint32_t mode) -{ - QCTX ctx; - - if (!expect_quic_conn_only(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - - if (ctx.qc->default_xso_created) { - quic_unlock(ctx.qc); - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED, - "too late to change default stream mode"); - } - - switch (mode) { - case SSL_DEFAULT_STREAM_MODE_NONE: - case SSL_DEFAULT_STREAM_MODE_AUTO_BIDI: - case SSL_DEFAULT_STREAM_MODE_AUTO_UNI: - ctx.qc->default_stream_mode = mode; - break; - default: - quic_unlock(ctx.qc); - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_PASSED_INVALID_ARGUMENT, - "bad default stream type"); - } - - quic_unlock(ctx.qc); - return 1; -} - -/* - * SSL_detach_stream - * ----------------- - */ -QUIC_TAKES_LOCK -SSL *ossl_quic_detach_stream(SSL *s) -{ - QCTX ctx; - QUIC_XSO *xso = NULL; - - if (!expect_quic_conn_only(s, &ctx)) - return NULL; - - quic_lock(ctx.qc); - - /* Calling this function inhibits default XSO autocreation. */ - /* QC ref to any default XSO is transferred to us and to caller. */ - qc_set_default_xso_keep_ref(ctx.qc, NULL, /*touch=*/1, &xso); - - quic_unlock(ctx.qc); - - return xso != NULL ? &xso->ssl : NULL; -} - -/* - * SSL_attach_stream - * ----------------- - */ -QUIC_TAKES_LOCK -int ossl_quic_attach_stream(SSL *conn, SSL *stream) -{ - QCTX ctx; - QUIC_XSO *xso; - int nref; - - if (!expect_quic_conn_only(conn, &ctx)) - return 0; - - if (stream == NULL || stream->type != SSL_TYPE_QUIC_XSO) - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_PASSED_NULL_PARAMETER, - "stream to attach must be a valid QUIC stream"); - - xso = (QUIC_XSO *)stream; - - quic_lock(ctx.qc); - - if (ctx.qc->default_xso != NULL) { - quic_unlock(ctx.qc); - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED, - "connection already has a default stream"); - } - - /* - * It is a caller error for the XSO being attached as a default XSO to have - * more than one ref. - */ - if (!CRYPTO_GET_REF(&xso->ssl.references, &nref)) { - quic_unlock(ctx.qc); - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_INTERNAL_ERROR, - "ref"); - } - - if (nref != 1) { - quic_unlock(ctx.qc); - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_PASSED_INVALID_ARGUMENT, - "stream being attached must have " - "only 1 reference"); - } - - /* Caller's reference to the XSO is transferred to us. */ - /* Calling this function inhibits default XSO autocreation. */ - qc_set_default_xso(ctx.qc, xso, /*touch=*/1); - - quic_unlock(ctx.qc); - return 1; -} - -/* - * SSL_set_incoming_stream_policy - * ------------------------------ - */ -QUIC_NEEDS_LOCK -static int qc_get_effective_incoming_stream_policy(QUIC_CONNECTION *qc) -{ - switch (qc->incoming_stream_policy) { - case SSL_INCOMING_STREAM_POLICY_AUTO: - if ((qc->default_xso == NULL && !qc->default_xso_created) - || qc->default_stream_mode == SSL_DEFAULT_STREAM_MODE_NONE) - return SSL_INCOMING_STREAM_POLICY_ACCEPT; - else - return SSL_INCOMING_STREAM_POLICY_REJECT; - - default: - return qc->incoming_stream_policy; - } -} - -QUIC_NEEDS_LOCK -static void qc_update_reject_policy(QUIC_CONNECTION *qc) -{ - int policy = qc_get_effective_incoming_stream_policy(qc); - int enable_reject = (policy == SSL_INCOMING_STREAM_POLICY_REJECT); - - ossl_quic_channel_set_incoming_stream_auto_reject(qc->ch, - enable_reject, - qc->incoming_stream_aec); -} - -QUIC_TAKES_LOCK -int ossl_quic_set_incoming_stream_policy(SSL *s, int policy, - uint64_t aec) -{ - int ret = 1; - QCTX ctx; - - if (!expect_quic_conn_only(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - - switch (policy) { - case SSL_INCOMING_STREAM_POLICY_AUTO: - case SSL_INCOMING_STREAM_POLICY_ACCEPT: - case SSL_INCOMING_STREAM_POLICY_REJECT: - ctx.qc->incoming_stream_policy = policy; - ctx.qc->incoming_stream_aec = aec; - break; - - default: - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_PASSED_INVALID_ARGUMENT, NULL); - ret = 0; - break; - } - - qc_update_reject_policy(ctx.qc); - quic_unlock(ctx.qc); - return ret; -} - -/* - * SSL_get_value, SSL_set_value - * ---------------------------- - */ -QUIC_TAKES_LOCK -static int qc_getset_idle_timeout(QCTX *ctx, uint32_t class_, - uint64_t *p_value_out, uint64_t *p_value_in) -{ - int ret = 0; - uint64_t value_out = 0, value_in; - - quic_lock(ctx->qc); - - switch (class_) { - case SSL_VALUE_CLASS_FEATURE_REQUEST: - value_out = ossl_quic_channel_get_max_idle_timeout_request(ctx->qc->ch); - - if (p_value_in != NULL) { - value_in = *p_value_in; - if (value_in > OSSL_QUIC_VLINT_MAX) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_PASSED_INVALID_ARGUMENT, - NULL); - goto err; - } - - if (ossl_quic_channel_have_generated_transport_params(ctx->qc->ch)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_FEATURE_NOT_RENEGOTIABLE, - NULL); - goto err; - } - - ossl_quic_channel_set_max_idle_timeout_request(ctx->qc->ch, value_in); - } - break; - - case SSL_VALUE_CLASS_FEATURE_PEER_REQUEST: - case SSL_VALUE_CLASS_FEATURE_NEGOTIATED: - if (p_value_in != NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_UNSUPPORTED_CONFIG_VALUE_OP, - NULL); - goto err; - } - - if (!ossl_quic_channel_is_handshake_complete(ctx->qc->ch)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_FEATURE_NEGOTIATION_NOT_COMPLETE, - NULL); - goto err; - } - - value_out = (class_ == SSL_VALUE_CLASS_FEATURE_NEGOTIATED) - ? ossl_quic_channel_get_max_idle_timeout_actual(ctx->qc->ch) - : ossl_quic_channel_get_max_idle_timeout_peer_request(ctx->qc->ch); - break; - - default: - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS, - NULL); - goto err; - } - - ret = 1; -err: - quic_unlock(ctx->qc); - if (ret && p_value_out != NULL) - *p_value_out = value_out; - - return ret; -} - -QUIC_TAKES_LOCK -static int qc_get_stream_avail(QCTX *ctx, uint32_t class_, - int is_uni, int is_remote, - uint64_t *value) -{ - int ret = 0; - - if (class_ != SSL_VALUE_CLASS_GENERIC) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS, - NULL); - return 0; - } - - quic_lock(ctx->qc); - - *value = is_remote - ? ossl_quic_channel_get_remote_stream_count_avail(ctx->qc->ch, is_uni) - : ossl_quic_channel_get_local_stream_count_avail(ctx->qc->ch, is_uni); - - ret = 1; - quic_unlock(ctx->qc); - return ret; -} - -QUIC_NEEDS_LOCK -static int qctx_should_autotick(QCTX *ctx) -{ - int event_handling_mode; - - if (ctx->is_stream) { - event_handling_mode = ctx->xso->event_handling_mode; - if (event_handling_mode != SSL_VALUE_EVENT_HANDLING_MODE_INHERIT) - return event_handling_mode != SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT; - } - - event_handling_mode = ctx->qc->event_handling_mode; - return event_handling_mode != SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT; -} - -QUIC_NEEDS_LOCK -static void qctx_maybe_autotick(QCTX *ctx) -{ - if (!qctx_should_autotick(ctx)) - return; - - ossl_quic_reactor_tick(ossl_quic_channel_get_reactor(ctx->qc->ch), 0); -} - -QUIC_TAKES_LOCK -static int qc_getset_event_handling(QCTX *ctx, uint32_t class_, - uint64_t *p_value_out, - uint64_t *p_value_in) -{ - int ret = 0; - uint64_t value_out = 0; - - quic_lock(ctx->qc); - - if (class_ != SSL_VALUE_CLASS_GENERIC) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS, - NULL); - goto err; - } - - if (p_value_in != NULL) { - switch (*p_value_in) { - case SSL_VALUE_EVENT_HANDLING_MODE_INHERIT: - case SSL_VALUE_EVENT_HANDLING_MODE_IMPLICIT: - case SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT: - break; - default: - QUIC_RAISE_NON_NORMAL_ERROR(ctx, ERR_R_PASSED_INVALID_ARGUMENT, - NULL); - goto err; - } - - value_out = *p_value_in; - if (ctx->is_stream) - ctx->xso->event_handling_mode = (int)value_out; - else - ctx->qc->event_handling_mode = (int)value_out; - } else { - value_out = ctx->is_stream - ? ctx->xso->event_handling_mode - : ctx->qc->event_handling_mode; - } - - ret = 1; -err: - quic_unlock(ctx->qc); - if (ret && p_value_out != NULL) - *p_value_out = value_out; - - return ret; -} - -QUIC_TAKES_LOCK -static int qc_get_stream_write_buf_stat(QCTX *ctx, uint32_t class_, - uint64_t *p_value_out, - size_t (*getter)(QUIC_SSTREAM *sstream)) -{ - int ret = 0; - size_t value = 0; - - quic_lock(ctx->qc); - - if (class_ != SSL_VALUE_CLASS_GENERIC) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS, - NULL); - goto err; - } - - if (ctx->xso == NULL) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_NO_STREAM, NULL); - goto err; - } - - if (!ossl_quic_stream_has_send(ctx->xso->stream)) { - QUIC_RAISE_NON_NORMAL_ERROR(ctx, SSL_R_STREAM_RECV_ONLY, NULL); - goto err; - } - - if (ossl_quic_stream_has_send_buffer(ctx->xso->stream)) - value = getter(ctx->xso->stream->sstream); - - ret = 1; -err: - quic_unlock(ctx->qc); - *p_value_out = (uint64_t)value; - return ret; -} - -QUIC_NEEDS_LOCK -static int expect_quic_for_value(SSL *s, QCTX *ctx, uint32_t id) -{ - switch (id) { - case SSL_VALUE_EVENT_HANDLING_MODE: - case SSL_VALUE_STREAM_WRITE_BUF_SIZE: - case SSL_VALUE_STREAM_WRITE_BUF_USED: - case SSL_VALUE_STREAM_WRITE_BUF_AVAIL: - return expect_quic(s, ctx); - default: - return expect_quic_conn_only(s, ctx); - } -} - -QUIC_TAKES_LOCK -int ossl_quic_get_value_uint(SSL *s, uint32_t class_, uint32_t id, - uint64_t *value) -{ - QCTX ctx; - - if (!expect_quic_for_value(s, &ctx, id)) - return 0; - - if (value == NULL) - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, - ERR_R_PASSED_INVALID_ARGUMENT, NULL); - - switch (id) { - case SSL_VALUE_QUIC_IDLE_TIMEOUT: - return qc_getset_idle_timeout(&ctx, class_, value, NULL); - - case SSL_VALUE_QUIC_STREAM_BIDI_LOCAL_AVAIL: - return qc_get_stream_avail(&ctx, class_, /*uni=*/0, /*remote=*/0, value); - case SSL_VALUE_QUIC_STREAM_BIDI_REMOTE_AVAIL: - return qc_get_stream_avail(&ctx, class_, /*uni=*/0, /*remote=*/1, value); - case SSL_VALUE_QUIC_STREAM_UNI_LOCAL_AVAIL: - return qc_get_stream_avail(&ctx, class_, /*uni=*/1, /*remote=*/0, value); - case SSL_VALUE_QUIC_STREAM_UNI_REMOTE_AVAIL: - return qc_get_stream_avail(&ctx, class_, /*uni=*/1, /*remote=*/1, value); - - case SSL_VALUE_EVENT_HANDLING_MODE: - return qc_getset_event_handling(&ctx, class_, value, NULL); - - case SSL_VALUE_STREAM_WRITE_BUF_SIZE: - return qc_get_stream_write_buf_stat(&ctx, class_, value, - ossl_quic_sstream_get_buffer_size); - case SSL_VALUE_STREAM_WRITE_BUF_USED: - return qc_get_stream_write_buf_stat(&ctx, class_, value, - ossl_quic_sstream_get_buffer_used); - case SSL_VALUE_STREAM_WRITE_BUF_AVAIL: - return qc_get_stream_write_buf_stat(&ctx, class_, value, - ossl_quic_sstream_get_buffer_avail); - - default: - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, - SSL_R_UNSUPPORTED_CONFIG_VALUE, NULL); - } - - return 1; -} - -QUIC_TAKES_LOCK -int ossl_quic_set_value_uint(SSL *s, uint32_t class_, uint32_t id, - uint64_t value) -{ - QCTX ctx; - - if (!expect_quic_for_value(s, &ctx, id)) - return 0; - - switch (id) { - case SSL_VALUE_QUIC_IDLE_TIMEOUT: - return qc_getset_idle_timeout(&ctx, class_, NULL, &value); - - case SSL_VALUE_EVENT_HANDLING_MODE: - return qc_getset_event_handling(&ctx, class_, NULL, &value); - - default: - return QUIC_RAISE_NON_NORMAL_ERROR(&ctx, - SSL_R_UNSUPPORTED_CONFIG_VALUE, NULL); - } - - return 1; -} - -/* - * SSL_accept_stream - * ----------------- - */ -struct wait_for_incoming_stream_args { - QCTX *ctx; - QUIC_STREAM *qs; -}; - -QUIC_NEEDS_LOCK -static int wait_for_incoming_stream(void *arg) -{ - struct wait_for_incoming_stream_args *args = arg; - QUIC_CONNECTION *qc = args->ctx->qc; - QUIC_STREAM_MAP *qsm = ossl_quic_channel_get_qsm(qc->ch); - - if (!quic_mutation_allowed(qc, /*req_active=*/1)) { - /* If connection is torn down due to an error while blocking, stop. */ - QUIC_RAISE_NON_NORMAL_ERROR(args->ctx, SSL_R_PROTOCOL_IS_SHUTDOWN, NULL); - return -1; - } - - args->qs = ossl_quic_stream_map_peek_accept_queue(qsm); - if (args->qs != NULL) - return 1; /* got a stream */ - - return 0; /* did not get a stream, keep trying */ -} - -QUIC_TAKES_LOCK -SSL *ossl_quic_accept_stream(SSL *s, uint64_t flags) -{ - QCTX ctx; - int ret; - SSL *new_s = NULL; - QUIC_STREAM_MAP *qsm; - QUIC_STREAM *qs; - QUIC_XSO *xso; - OSSL_RTT_INFO rtt_info; - - if (!expect_quic_conn_only(s, &ctx)) - return NULL; - - quic_lock(ctx.qc); - - if (qc_get_effective_incoming_stream_policy(ctx.qc) - == SSL_INCOMING_STREAM_POLICY_REJECT) { - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED, NULL); - goto out; - } - - qsm = ossl_quic_channel_get_qsm(ctx.qc->ch); - - qs = ossl_quic_stream_map_peek_accept_queue(qsm); - if (qs == NULL) { - if (qc_blocking_mode(ctx.qc) - && (flags & SSL_ACCEPT_STREAM_NO_BLOCK) == 0) { - struct wait_for_incoming_stream_args args; - - args.ctx = &ctx; - args.qs = NULL; - - ret = block_until_pred(ctx.qc, wait_for_incoming_stream, &args, 0); - if (ret == 0) { - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_INTERNAL_ERROR, NULL); - goto out; - } else if (ret < 0 || args.qs == NULL) { - goto out; - } - - qs = args.qs; - } else { - goto out; - } - } - - xso = create_xso_from_stream(ctx.qc, qs); - if (xso == NULL) - goto out; - - ossl_statm_get_rtt_info(ossl_quic_channel_get_statm(ctx.qc->ch), &rtt_info); - ossl_quic_stream_map_remove_from_accept_queue(qsm, qs, - rtt_info.smoothed_rtt); - new_s = &xso->ssl; - - /* Calling this function inhibits default XSO autocreation. */ - qc_touch_default_xso(ctx.qc); /* inhibits default XSO */ - -out: - quic_unlock(ctx.qc); - return new_s; -} - -/* - * SSL_get_accept_stream_queue_len - * ------------------------------- - */ -QUIC_TAKES_LOCK -size_t ossl_quic_get_accept_stream_queue_len(SSL *s) -{ - QCTX ctx; - size_t v; - - if (!expect_quic_conn_only(s, &ctx)) - return 0; - - quic_lock(ctx.qc); - - v = ossl_quic_stream_map_get_total_accept_queue_len(ossl_quic_channel_get_qsm(ctx.qc->ch)); - - quic_unlock(ctx.qc); - return v; -} - -/* - * SSL_stream_reset - * ---------------- - */ -int ossl_quic_stream_reset(SSL *ssl, - const SSL_STREAM_RESET_ARGS *args, - size_t args_len) -{ - QCTX ctx; - QUIC_STREAM_MAP *qsm; - QUIC_STREAM *qs; - uint64_t error_code; - int ok, err; - - if (!expect_quic_with_stream_lock(ssl, /*remote_init=*/0, /*io=*/0, &ctx)) - return 0; - - qsm = ossl_quic_channel_get_qsm(ctx.qc->ch); - qs = ctx.xso->stream; - error_code = (args != NULL ? args->quic_error_code : 0); - - if (!quic_validate_for_write(ctx.xso, &err)) { - ok = QUIC_RAISE_NON_NORMAL_ERROR(&ctx, err, NULL); - goto err; - } - - ok = ossl_quic_stream_map_reset_stream_send_part(qsm, qs, error_code); - if (ok) - ctx.xso->requested_reset = 1; - -err: - quic_unlock(ctx.qc); - return ok; -} - -/* - * SSL_get_stream_read_state - * ------------------------- - */ -static void quic_classify_stream(QUIC_CONNECTION *qc, - QUIC_STREAM *qs, - int is_write, - int *state, - uint64_t *app_error_code) -{ - int local_init; - uint64_t final_size; - - local_init = (ossl_quic_stream_is_server_init(qs) == qc->as_server); - - if (app_error_code != NULL) - *app_error_code = UINT64_MAX; - else - app_error_code = &final_size; /* throw away value */ - - if (!ossl_quic_stream_is_bidi(qs) && local_init != is_write) { - /* - * Unidirectional stream and this direction of transmission doesn't - * exist. - */ - *state = SSL_STREAM_STATE_WRONG_DIR; - } else if (ossl_quic_channel_is_term_any(qc->ch)) { - /* Connection already closed. */ - *state = SSL_STREAM_STATE_CONN_CLOSED; - } else if (!is_write && qs->recv_state == QUIC_RSTREAM_STATE_DATA_READ) { - /* Application has read a FIN. */ - *state = SSL_STREAM_STATE_FINISHED; - } else if ((!is_write && qs->stop_sending) - || (is_write && ossl_quic_stream_send_is_reset(qs))) { - /* - * Stream has been reset locally. FIN takes precedence over this for the - * read case as the application need not care if the stream is reset - * after a FIN has been successfully processed. - */ - *state = SSL_STREAM_STATE_RESET_LOCAL; - *app_error_code = !is_write - ? qs->stop_sending_aec - : qs->reset_stream_aec; - } else if ((!is_write && ossl_quic_stream_recv_is_reset(qs)) - || (is_write && qs->peer_stop_sending)) { - /* - * Stream has been reset remotely. */ - *state = SSL_STREAM_STATE_RESET_REMOTE; - *app_error_code = !is_write - ? qs->peer_reset_stream_aec - : qs->peer_stop_sending_aec; - } else if (is_write && ossl_quic_sstream_get_final_size(qs->sstream, - &final_size)) { - /* - * Stream has been finished. Stream reset takes precedence over this for - * the write case as peer may not have received all data. - */ - *state = SSL_STREAM_STATE_FINISHED; - } else { - /* Stream still healthy. */ - *state = SSL_STREAM_STATE_OK; - } -} - -static int quic_get_stream_state(SSL *ssl, int is_write) -{ - QCTX ctx; - int state; - - if (!expect_quic_with_stream_lock(ssl, /*remote_init=*/-1, /*io=*/0, &ctx)) - return SSL_STREAM_STATE_NONE; - - quic_classify_stream(ctx.qc, ctx.xso->stream, is_write, &state, NULL); - quic_unlock(ctx.qc); - return state; -} - -int ossl_quic_get_stream_read_state(SSL *ssl) -{ - return quic_get_stream_state(ssl, /*is_write=*/0); -} - -/* - * SSL_get_stream_write_state - * -------------------------- - */ -int ossl_quic_get_stream_write_state(SSL *ssl) -{ - return quic_get_stream_state(ssl, /*is_write=*/1); -} - -/* - * SSL_get_stream_read_error_code - * ------------------------------ - */ -static int quic_get_stream_error_code(SSL *ssl, int is_write, - uint64_t *app_error_code) -{ - QCTX ctx; - int state; - - if (!expect_quic_with_stream_lock(ssl, /*remote_init=*/-1, /*io=*/0, &ctx)) - return -1; - - quic_classify_stream(ctx.qc, ctx.xso->stream, /*is_write=*/0, - &state, app_error_code); - - quic_unlock(ctx.qc); - switch (state) { - case SSL_STREAM_STATE_FINISHED: - return 0; - case SSL_STREAM_STATE_RESET_LOCAL: - case SSL_STREAM_STATE_RESET_REMOTE: - return 1; - default: - return -1; - } -} - -int ossl_quic_get_stream_read_error_code(SSL *ssl, uint64_t *app_error_code) -{ - return quic_get_stream_error_code(ssl, /*is_write=*/0, app_error_code); -} - -/* - * SSL_get_stream_write_error_code - * ------------------------------- - */ -int ossl_quic_get_stream_write_error_code(SSL *ssl, uint64_t *app_error_code) -{ - return quic_get_stream_error_code(ssl, /*is_write=*/1, app_error_code); -} - -/* - * Write buffer size mutation - * -------------------------- - */ -int ossl_quic_set_write_buffer_size(SSL *ssl, size_t size) -{ - int ret = 0; - QCTX ctx; - - if (!expect_quic_with_stream_lock(ssl, /*remote_init=*/-1, /*io=*/0, &ctx)) - return 0; - - if (!ossl_quic_stream_has_send(ctx.xso->stream)) { - /* Called on a unidirectional receive-only stream - error. */ - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED, NULL); - goto out; - } - - if (!ossl_quic_stream_has_send_buffer(ctx.xso->stream)) { - /* - * If the stream has a send part but we have disposed of it because we - * no longer need it, this is a no-op. - */ - ret = 1; - goto out; - } - - if (!ossl_quic_sstream_set_buffer_size(ctx.xso->stream->sstream, size)) { - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_INTERNAL_ERROR, NULL); - goto out; - } - - ret = 1; - -out: - quic_unlock(ctx.qc); - return ret; -} - -/* - * SSL_get_conn_close_info - * ----------------------- - */ -int ossl_quic_get_conn_close_info(SSL *ssl, - SSL_CONN_CLOSE_INFO *info, - size_t info_len) -{ - QCTX ctx; - const QUIC_TERMINATE_CAUSE *tc; - - if (!expect_quic_conn_only(ssl, &ctx)) - return -1; - - tc = ossl_quic_channel_get_terminate_cause(ctx.qc->ch); - if (tc == NULL) - return 0; - - info->error_code = tc->error_code; - info->frame_type = tc->frame_type; - info->reason = tc->reason; - info->reason_len = tc->reason_len; - info->flags = 0; - if (!tc->remote) - info->flags |= SSL_CONN_CLOSE_FLAG_LOCAL; - if (!tc->app) - info->flags |= SSL_CONN_CLOSE_FLAG_TRANSPORT; - return 1; -} - -/* - * SSL_key_update - * -------------- - */ -int ossl_quic_key_update(SSL *ssl, int update_type) -{ - QCTX ctx; - - if (!expect_quic_conn_only(ssl, &ctx)) - return 0; - - switch (update_type) { - case SSL_KEY_UPDATE_NOT_REQUESTED: - /* - * QUIC signals peer key update implicily by triggering a local - * spontaneous TXKU. Silently upgrade this to SSL_KEY_UPDATE_REQUESTED. - */ - case SSL_KEY_UPDATE_REQUESTED: - break; - - default: - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, ERR_R_PASSED_INVALID_ARGUMENT, NULL); - return 0; - } - - quic_lock(ctx.qc); - - /* Attempt to perform a TXKU. */ - if (!ossl_quic_channel_trigger_txku(ctx.qc->ch)) { - QUIC_RAISE_NON_NORMAL_ERROR(&ctx, SSL_R_TOO_MANY_KEY_UPDATES, NULL); - quic_unlock(ctx.qc); - return 0; - } - - quic_unlock(ctx.qc); - return 1; -} - -/* - * SSL_get_key_update_type - * ----------------------- - */ -int ossl_quic_get_key_update_type(const SSL *s) -{ - /* - * We always handle key updates immediately so a key update is never - * pending. - */ - return SSL_KEY_UPDATE_NONE; -} - -/* - * QUIC Front-End I/O API: SSL_CTX Management - * ========================================== - */ - -long ossl_quic_ctx_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg) -{ - switch (cmd) { - default: - return ssl3_ctx_ctrl(ctx, cmd, larg, parg); - } -} - -long ossl_quic_callback_ctrl(SSL *s, int cmd, void (*fp) (void)) -{ - QCTX ctx; - - if (!expect_quic_conn_only(s, &ctx)) - return 0; - - switch (cmd) { - case SSL_CTRL_SET_MSG_CALLBACK: - ossl_quic_channel_set_msg_callback(ctx.qc->ch, (ossl_msg_cb)fp, - &ctx.qc->ssl); - /* This callback also needs to be set on the internal SSL object */ - return ssl3_callback_ctrl(ctx.qc->tls, cmd, fp);; - - default: - /* Probably a TLS related ctrl. Defer to our internal SSL object */ - return ssl3_callback_ctrl(ctx.qc->tls, cmd, fp); - } -} - -long ossl_quic_ctx_callback_ctrl(SSL_CTX *ctx, int cmd, void (*fp) (void)) -{ - return ssl3_ctx_callback_ctrl(ctx, cmd, fp); -} - -int ossl_quic_renegotiate_check(SSL *ssl, int initok) -{ - /* We never do renegotiation. */ - return 0; -} - -const SSL_CIPHER *ossl_quic_get_cipher_by_char(const unsigned char *p) -{ - const SSL_CIPHER *ciph = ssl3_get_cipher_by_char(p); - - if ((ciph->algorithm2 & SSL_QUIC) == 0) - return NULL; - - return ciph; -} - -/* - * These functions define the TLSv1.2 (and below) ciphers that are supported by - * the SSL_METHOD. Since QUIC only supports TLSv1.3 we don't support any. - */ - -int ossl_quic_num_ciphers(void) -{ - return 0; -} - -const SSL_CIPHER *ossl_quic_get_cipher(unsigned int u) -{ - return NULL; -} - -/* - * SSL_get_shutdown() - * ------------------ - */ -int ossl_quic_get_shutdown(const SSL *s) -{ - QCTX ctx; - int shut = 0; - - if (!expect_quic_conn_only(s, &ctx)) - return 0; - - if (ossl_quic_channel_is_term_any(ctx.qc->ch)) { - shut |= SSL_SENT_SHUTDOWN; - if (!ossl_quic_channel_is_closing(ctx.qc->ch)) - shut |= SSL_RECEIVED_SHUTDOWN; - } - - return shut; -} - -/* - * QUIC Polling Support APIs - * ========================= - */ - -/* Do we have the R (read) condition? */ -QUIC_NEEDS_LOCK -static int test_poll_event_r(QUIC_XSO *xso) -{ - int fin = 0; - size_t avail = 0; - - return ossl_quic_stream_has_recv_buffer(xso->stream) - && ossl_quic_rstream_available(xso->stream->rstream, &avail, &fin) - && (avail > 0 || (fin && !xso->retired_fin)); -} - -/* Do we have the ER (exception: read) condition? */ -QUIC_NEEDS_LOCK -static int test_poll_event_er(QUIC_XSO *xso) -{ - return ossl_quic_stream_has_recv(xso->stream) - && ossl_quic_stream_recv_is_reset(xso->stream) - && !xso->retired_fin; -} - -/* Do we have the W (write) condition? */ -QUIC_NEEDS_LOCK -static int test_poll_event_w(QUIC_XSO *xso) -{ - return !xso->conn->shutting_down - && ossl_quic_stream_has_send_buffer(xso->stream) - && ossl_quic_sstream_get_buffer_avail(xso->stream->sstream) - && !ossl_quic_sstream_get_final_size(xso->stream->sstream, NULL) - && quic_mutation_allowed(xso->conn, /*req_active=*/1); -} - -/* Do we have the EW (exception: write) condition? */ -QUIC_NEEDS_LOCK -static int test_poll_event_ew(QUIC_XSO *xso) -{ - return ossl_quic_stream_has_send(xso->stream) - && xso->stream->peer_stop_sending - && !xso->requested_reset - && !xso->conn->shutting_down; -} - -/* Do we have the EC (exception: connection) condition? */ -QUIC_NEEDS_LOCK -static int test_poll_event_ec(QUIC_CONNECTION *qc) -{ - return ossl_quic_channel_is_term_any(qc->ch); -} - -/* Do we have the ECD (exception: connection drained) condition? */ -QUIC_NEEDS_LOCK -static int test_poll_event_ecd(QUIC_CONNECTION *qc) -{ - return ossl_quic_channel_is_terminated(qc->ch); -} - -/* Do we have the IS (incoming: stream) condition? */ -QUIC_NEEDS_LOCK -static int test_poll_event_is(QUIC_CONNECTION *qc, int is_uni) -{ - return ossl_quic_stream_map_get_accept_queue_len(ossl_quic_channel_get_qsm(qc->ch), - is_uni); -} - -/* Do we have the OS (outgoing: stream) condition? */ -QUIC_NEEDS_LOCK -static int test_poll_event_os(QUIC_CONNECTION *qc, int is_uni) -{ - /* Is it currently possible for us to make an outgoing stream? */ - return quic_mutation_allowed(qc, /*req_active=*/1) - && ossl_quic_channel_get_local_stream_count_avail(qc->ch, is_uni) > 0; -} - -QUIC_TAKES_LOCK -int ossl_quic_conn_poll_events(SSL *ssl, uint64_t events, int do_tick, - uint64_t *p_revents) -{ - QCTX ctx; - uint64_t revents = 0; - - if (!expect_quic(ssl, &ctx)) - return 0; - - quic_lock(ctx.qc); - - if (do_tick) - ossl_quic_reactor_tick(ossl_quic_channel_get_reactor(ctx.qc->ch), 0); - - if (ctx.xso != NULL) { - /* SSL object has a stream component. */ - - if ((events & SSL_POLL_EVENT_R) != 0 - && test_poll_event_r(ctx.xso)) - revents |= SSL_POLL_EVENT_R; - - if ((events & SSL_POLL_EVENT_ER) != 0 - && test_poll_event_er(ctx.xso)) - revents |= SSL_POLL_EVENT_ER; - - if ((events & SSL_POLL_EVENT_W) != 0 - && test_poll_event_w(ctx.xso)) - revents |= SSL_POLL_EVENT_W; - - if ((events & SSL_POLL_EVENT_EW) != 0 - && test_poll_event_ew(ctx.xso)) - revents |= SSL_POLL_EVENT_EW; - } - - if (!ctx.is_stream) { - if ((events & SSL_POLL_EVENT_EC) != 0 - && test_poll_event_ec(ctx.qc)) - revents |= SSL_POLL_EVENT_EC; - - if ((events & SSL_POLL_EVENT_ECD) != 0 - && test_poll_event_ecd(ctx.qc)) - revents |= SSL_POLL_EVENT_ECD; - - if ((events & SSL_POLL_EVENT_ISB) != 0 - && test_poll_event_is(ctx.qc, /*uni=*/0)) - revents |= SSL_POLL_EVENT_ISB; - - if ((events & SSL_POLL_EVENT_ISU) != 0 - && test_poll_event_is(ctx.qc, /*uni=*/1)) - revents |= SSL_POLL_EVENT_ISU; - - if ((events & SSL_POLL_EVENT_OSB) != 0 - && test_poll_event_os(ctx.qc, /*uni=*/0)) - revents |= SSL_POLL_EVENT_OSB; - - if ((events & SSL_POLL_EVENT_OSU) != 0 - && test_poll_event_os(ctx.qc, /*uni=*/1)) - revents |= SSL_POLL_EVENT_OSU; - } - - quic_unlock(ctx.qc); - *p_revents = revents; - return 1; -} - -/* - * Internal Testing APIs - * ===================== - */ - -QUIC_CHANNEL *ossl_quic_conn_get_channel(SSL *s) -{ - QCTX ctx; - - if (!expect_quic_conn_only(s, &ctx)) - return NULL; - - return ctx.qc->ch; -} - -int ossl_quic_set_diag_title(SSL_CTX *ctx, const char *title) -{ -#ifndef OPENSSL_NO_QLOG - OPENSSL_free(ctx->qlog_title); - ctx->qlog_title = NULL; - - if (title == NULL) - return 1; - - if ((ctx->qlog_title = OPENSSL_strdup(title)) == NULL) - return 0; -#endif - - return 1; -} diff --git a/openssl/src/ssl/quic/quic_lcidm.c b/openssl/src/ssl/quic/quic_lcidm.c deleted file mode 100644 index e5948b95e..000000000 --- a/openssl/src/ssl/quic/quic_lcidm.c +++ /dev/null @@ -1,556 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_lcidm.h" -#include "internal/quic_types.h" -#include "internal/quic_vlint.h" -#include "internal/common.h" -#include -#include -#include - -/* - * QUIC Local Connection ID Manager - * ================================ - */ - -typedef struct quic_lcidm_conn_st QUIC_LCIDM_CONN; - -enum { - LCID_TYPE_ODCID, /* This LCID is the ODCID from the peer */ - LCID_TYPE_INITIAL, /* This is our Initial SCID */ - LCID_TYPE_NCID /* This LCID was issued via a NCID frame */ -}; - -typedef struct quic_lcid_st { - QUIC_CONN_ID cid; - uint64_t seq_num; - - /* Back-pointer to the owning QUIC_LCIDM_CONN structure. */ - QUIC_LCIDM_CONN *conn; - - /* LCID_TYPE_* */ - unsigned int type : 2; -} QUIC_LCID; - -DEFINE_LHASH_OF_EX(QUIC_LCID); -DEFINE_LHASH_OF_EX(QUIC_LCIDM_CONN); - -struct quic_lcidm_conn_st { - size_t num_active_lcid; - LHASH_OF(QUIC_LCID) *lcids; - void *opaque; - QUIC_LCID *odcid_lcid_obj; - uint64_t next_seq_num; - - /* Have we enrolled an ODCID? */ - unsigned int done_odcid : 1; -}; - -struct quic_lcidm_st { - OSSL_LIB_CTX *libctx; - LHASH_OF(QUIC_LCID) *lcids; /* (QUIC_CONN_ID) -> (QUIC_LCID *) */ - LHASH_OF(QUIC_LCIDM_CONN) *conns; /* (void *opaque) -> (QUIC_LCIDM_CONN *) */ - size_t lcid_len; /* Length in bytes for all LCIDs */ -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - QUIC_CONN_ID next_lcid; -#endif -}; - -static unsigned long bin_hash(const unsigned char *buf, size_t buf_len) -{ - unsigned long hash = 0; - size_t i; - - for (i = 0; i < buf_len; ++i) - hash ^= ((unsigned long)buf[i]) << (8 * (i % sizeof(unsigned long))); - - return hash; -} - -static unsigned long lcid_hash(const QUIC_LCID *lcid_obj) -{ - return bin_hash(lcid_obj->cid.id, lcid_obj->cid.id_len); -} - -static int lcid_comp(const QUIC_LCID *a, const QUIC_LCID *b) -{ - return !ossl_quic_conn_id_eq(&a->cid, &b->cid); -} - -static unsigned long lcidm_conn_hash(const QUIC_LCIDM_CONN *conn) -{ - return (unsigned long)(uintptr_t)conn->opaque; -} - -static int lcidm_conn_comp(const QUIC_LCIDM_CONN *a, const QUIC_LCIDM_CONN *b) -{ - return a->opaque != b->opaque; -} - -QUIC_LCIDM *ossl_quic_lcidm_new(OSSL_LIB_CTX *libctx, size_t lcid_len) -{ - QUIC_LCIDM *lcidm = NULL; - - if (lcid_len > QUIC_MAX_CONN_ID_LEN) - goto err; - - if ((lcidm = OPENSSL_zalloc(sizeof(*lcidm))) == NULL) - goto err; - - if ((lcidm->lcids = lh_QUIC_LCID_new(lcid_hash, lcid_comp)) == NULL) - goto err; - - if ((lcidm->conns = lh_QUIC_LCIDM_CONN_new(lcidm_conn_hash, - lcidm_conn_comp)) == NULL) - goto err; - - lcidm->libctx = libctx; - lcidm->lcid_len = lcid_len; - return lcidm; - -err: - if (lcidm != NULL) { - lh_QUIC_LCID_free(lcidm->lcids); - lh_QUIC_LCIDM_CONN_free(lcidm->conns); - OPENSSL_free(lcidm); - } - return NULL; -} - -static void lcidm_delete_conn(QUIC_LCIDM *lcidm, QUIC_LCIDM_CONN *conn); - -static void lcidm_delete_conn_(QUIC_LCIDM_CONN *conn, void *arg) -{ - lcidm_delete_conn((QUIC_LCIDM *)arg, conn); -} - -void ossl_quic_lcidm_free(QUIC_LCIDM *lcidm) -{ - if (lcidm == NULL) - return; - - /* - * Calling OPENSSL_lh_delete during a doall call is unsafe with our - * current LHASH implementation for several reasons: - * - * - firstly, because deletes can cause the hashtable to be contracted, - * resulting in rehashing which might cause items in later buckets to - * move to earlier buckets, which might cause doall to skip an item, - * resulting in a memory leak; - * - * - secondly, because doall in general is not safe across hashtable - * size changes, as it caches hashtable size and pointer values - * while operating. - * - * The fix for this is to disable hashtable contraction using the following - * call, which guarantees that no rehashing will occur so long as we only - * call delete and not insert. - */ - lh_QUIC_LCIDM_CONN_set_down_load(lcidm->conns, 0); - - lh_QUIC_LCIDM_CONN_doall_arg(lcidm->conns, lcidm_delete_conn_, lcidm); - - lh_QUIC_LCID_free(lcidm->lcids); - lh_QUIC_LCIDM_CONN_free(lcidm->conns); - OPENSSL_free(lcidm); -} - -static QUIC_LCID *lcidm_get0_lcid(const QUIC_LCIDM *lcidm, const QUIC_CONN_ID *lcid) -{ - QUIC_LCID key; - - key.cid = *lcid; - - if (key.cid.id_len > QUIC_MAX_CONN_ID_LEN) - return NULL; - - return lh_QUIC_LCID_retrieve(lcidm->lcids, &key); -} - -static QUIC_LCIDM_CONN *lcidm_get0_conn(const QUIC_LCIDM *lcidm, void *opaque) -{ - QUIC_LCIDM_CONN key; - - key.opaque = opaque; - - return lh_QUIC_LCIDM_CONN_retrieve(lcidm->conns, &key); -} - -static QUIC_LCIDM_CONN *lcidm_upsert_conn(const QUIC_LCIDM *lcidm, void *opaque) -{ - QUIC_LCIDM_CONN *conn = lcidm_get0_conn(lcidm, opaque); - - if (conn != NULL) - return conn; - - if ((conn = OPENSSL_zalloc(sizeof(*conn))) == NULL) - goto err; - - if ((conn->lcids = lh_QUIC_LCID_new(lcid_hash, lcid_comp)) == NULL) - goto err; - - conn->opaque = opaque; - - lh_QUIC_LCIDM_CONN_insert(lcidm->conns, conn); - if (lh_QUIC_LCIDM_CONN_error(lcidm->conns)) - goto err; - - return conn; - -err: - if (conn != NULL) { - lh_QUIC_LCID_free(conn->lcids); - OPENSSL_free(conn); - } - return NULL; -} - -static void lcidm_delete_conn_lcid(QUIC_LCIDM *lcidm, QUIC_LCID *lcid_obj) -{ - lh_QUIC_LCID_delete(lcidm->lcids, lcid_obj); - lh_QUIC_LCID_delete(lcid_obj->conn->lcids, lcid_obj); - assert(lcid_obj->conn->num_active_lcid > 0); - --lcid_obj->conn->num_active_lcid; - OPENSSL_free(lcid_obj); -} - -/* doall_arg wrapper */ -static void lcidm_delete_conn_lcid_(QUIC_LCID *lcid_obj, void *arg) -{ - lcidm_delete_conn_lcid((QUIC_LCIDM *)arg, lcid_obj); -} - -static void lcidm_delete_conn(QUIC_LCIDM *lcidm, QUIC_LCIDM_CONN *conn) -{ - /* See comment in ossl_quic_lcidm_free */ - lh_QUIC_LCID_set_down_load(conn->lcids, 0); - - lh_QUIC_LCID_doall_arg(conn->lcids, lcidm_delete_conn_lcid_, lcidm); - lh_QUIC_LCIDM_CONN_delete(lcidm->conns, conn); - lh_QUIC_LCID_free(conn->lcids); - OPENSSL_free(conn); -} - -static QUIC_LCID *lcidm_conn_new_lcid(QUIC_LCIDM *lcidm, QUIC_LCIDM_CONN *conn, - const QUIC_CONN_ID *lcid) -{ - QUIC_LCID *lcid_obj = NULL; - - if (lcid->id_len > QUIC_MAX_CONN_ID_LEN) - return NULL; - - if ((lcid_obj = OPENSSL_zalloc(sizeof(*lcid_obj))) == NULL) - goto err; - - lcid_obj->cid = *lcid; - lcid_obj->conn = conn; - - lh_QUIC_LCID_insert(conn->lcids, lcid_obj); - if (lh_QUIC_LCID_error(conn->lcids)) - goto err; - - lh_QUIC_LCID_insert(lcidm->lcids, lcid_obj); - if (lh_QUIC_LCID_error(lcidm->lcids)) { - lh_QUIC_LCID_delete(conn->lcids, lcid_obj); - goto err; - } - - ++conn->num_active_lcid; - return lcid_obj; - -err: - OPENSSL_free(lcid_obj); - return NULL; -} - -size_t ossl_quic_lcidm_get_lcid_len(const QUIC_LCIDM *lcidm) -{ - return lcidm->lcid_len; -} - -size_t ossl_quic_lcidm_get_num_active_lcid(const QUIC_LCIDM *lcidm, - void *opaque) -{ - QUIC_LCIDM_CONN *conn; - - conn = lcidm_get0_conn(lcidm, opaque); - if (conn == NULL) - return 0; - - return conn->num_active_lcid; -} - -static int lcidm_generate_cid(QUIC_LCIDM *lcidm, - QUIC_CONN_ID *cid) -{ -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - int i; - - lcidm->next_lcid.id_len = (unsigned char)lcidm->lcid_len; - *cid = lcidm->next_lcid; - - for (i = lcidm->lcid_len - 1; i >= 0; --i) - if (++lcidm->next_lcid.id[i] != 0) - break; - - return 1; -#else - return ossl_quic_gen_rand_conn_id(lcidm->libctx, lcidm->lcid_len, cid); -#endif -} - -static int lcidm_generate(QUIC_LCIDM *lcidm, - void *opaque, - unsigned int type, - QUIC_CONN_ID *lcid_out, - uint64_t *seq_num) -{ - QUIC_LCIDM_CONN *conn; - QUIC_LCID key, *lcid_obj; - size_t i; -#define MAX_RETRIES 8 - - if ((conn = lcidm_upsert_conn(lcidm, opaque)) == NULL) - return 0; - - if ((type == LCID_TYPE_INITIAL && conn->next_seq_num > 0) - || conn->next_seq_num > OSSL_QUIC_VLINT_MAX) - return 0; - - i = 0; - do { - if (i++ >= MAX_RETRIES) - /* - * Too many retries; should not happen but if it does, don't loop - * endlessly. - */ - return 0; - - if (!lcidm_generate_cid(lcidm, lcid_out)) - return 0; - - key.cid = *lcid_out; - /* If a collision occurs, retry. */ - } while (lh_QUIC_LCID_retrieve(lcidm->lcids, &key) != NULL); - - if ((lcid_obj = lcidm_conn_new_lcid(lcidm, conn, lcid_out)) == NULL) - return 0; - - lcid_obj->seq_num = conn->next_seq_num; - lcid_obj->type = type; - - if (seq_num != NULL) - *seq_num = lcid_obj->seq_num; - - ++conn->next_seq_num; - return 1; -} - -int ossl_quic_lcidm_enrol_odcid(QUIC_LCIDM *lcidm, - void *opaque, - const QUIC_CONN_ID *initial_odcid) -{ - QUIC_LCIDM_CONN *conn; - QUIC_LCID key, *lcid_obj; - - if (initial_odcid == NULL || initial_odcid->id_len < QUIC_MIN_ODCID_LEN - || initial_odcid->id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - - if ((conn = lcidm_upsert_conn(lcidm, opaque)) == NULL) - return 0; - - if (conn->done_odcid) - return 0; - - key.cid = *initial_odcid; - if (lh_QUIC_LCID_retrieve(lcidm->lcids, &key) != NULL) - return 0; - - if ((lcid_obj = lcidm_conn_new_lcid(lcidm, conn, initial_odcid)) == NULL) - return 0; - - lcid_obj->seq_num = LCIDM_ODCID_SEQ_NUM; - lcid_obj->type = LCID_TYPE_ODCID; - - conn->odcid_lcid_obj = lcid_obj; - conn->done_odcid = 1; - return 1; -} - -int ossl_quic_lcidm_generate_initial(QUIC_LCIDM *lcidm, - void *opaque, - QUIC_CONN_ID *initial_lcid) -{ - return lcidm_generate(lcidm, opaque, LCID_TYPE_INITIAL, - initial_lcid, NULL); -} - -int ossl_quic_lcidm_generate(QUIC_LCIDM *lcidm, - void *opaque, - OSSL_QUIC_FRAME_NEW_CONN_ID *ncid_frame) -{ - ncid_frame->seq_num = 0; - ncid_frame->retire_prior_to = 0; - - return lcidm_generate(lcidm, opaque, LCID_TYPE_NCID, - &ncid_frame->conn_id, - &ncid_frame->seq_num); -} - -int ossl_quic_lcidm_retire_odcid(QUIC_LCIDM *lcidm, void *opaque) -{ - QUIC_LCIDM_CONN *conn; - - if ((conn = lcidm_upsert_conn(lcidm, opaque)) == NULL) - return 0; - - if (conn->odcid_lcid_obj == NULL) - return 0; - - lcidm_delete_conn_lcid(lcidm, conn->odcid_lcid_obj); - conn->odcid_lcid_obj = NULL; - return 1; -} - -struct retire_args { - QUIC_LCID *earliest_seq_num_lcid_obj; - uint64_t earliest_seq_num, retire_prior_to; -}; - -static void retire_for_conn(QUIC_LCID *lcid_obj, void *arg) -{ - struct retire_args *args = arg; - - /* ODCID LCID cannot be retired via this API */ - if (lcid_obj->type == LCID_TYPE_ODCID - || lcid_obj->seq_num >= args->retire_prior_to) - return; - - if (lcid_obj->seq_num < args->earliest_seq_num) { - args->earliest_seq_num = lcid_obj->seq_num; - args->earliest_seq_num_lcid_obj = lcid_obj; - } -} - -int ossl_quic_lcidm_retire(QUIC_LCIDM *lcidm, - void *opaque, - uint64_t retire_prior_to, - const QUIC_CONN_ID *containing_pkt_dcid, - QUIC_CONN_ID *retired_lcid, - uint64_t *retired_seq_num, - int *did_retire) -{ - QUIC_LCIDM_CONN key, *conn; - struct retire_args args = {0}; - - key.opaque = opaque; - - if (did_retire == NULL) - return 0; - - *did_retire = 0; - if ((conn = lh_QUIC_LCIDM_CONN_retrieve(lcidm->conns, &key)) == NULL) - return 1; - - args.retire_prior_to = retire_prior_to; - args.earliest_seq_num = UINT64_MAX; - - lh_QUIC_LCID_doall_arg(conn->lcids, retire_for_conn, &args); - if (args.earliest_seq_num_lcid_obj == NULL) - return 1; - - if (containing_pkt_dcid != NULL - && ossl_quic_conn_id_eq(&args.earliest_seq_num_lcid_obj->cid, - containing_pkt_dcid)) - return 0; - - *did_retire = 1; - if (retired_lcid != NULL) - *retired_lcid = args.earliest_seq_num_lcid_obj->cid; - if (retired_seq_num != NULL) - *retired_seq_num = args.earliest_seq_num_lcid_obj->seq_num; - - lcidm_delete_conn_lcid(lcidm, args.earliest_seq_num_lcid_obj); - return 1; -} - -int ossl_quic_lcidm_cull(QUIC_LCIDM *lcidm, void *opaque) -{ - QUIC_LCIDM_CONN key, *conn; - - key.opaque = opaque; - - if ((conn = lh_QUIC_LCIDM_CONN_retrieve(lcidm->conns, &key)) == NULL) - return 0; - - lcidm_delete_conn(lcidm, conn); - return 1; -} - -int ossl_quic_lcidm_lookup(QUIC_LCIDM *lcidm, - const QUIC_CONN_ID *lcid, - uint64_t *seq_num, - void **opaque) -{ - QUIC_LCID *lcid_obj; - - if (lcid == NULL) - return 0; - - if ((lcid_obj = lcidm_get0_lcid(lcidm, lcid)) == NULL) - return 0; - - if (seq_num != NULL) - *seq_num = lcid_obj->seq_num; - - if (opaque != NULL) - *opaque = lcid_obj->conn->opaque; - - return 1; -} - -int ossl_quic_lcidm_debug_remove(QUIC_LCIDM *lcidm, - const QUIC_CONN_ID *lcid) -{ - QUIC_LCID key, *lcid_obj; - - key.cid = *lcid; - if ((lcid_obj = lh_QUIC_LCID_retrieve(lcidm->lcids, &key)) == NULL) - return 0; - - lcidm_delete_conn_lcid(lcidm, lcid_obj); - return 1; -} - -int ossl_quic_lcidm_debug_add(QUIC_LCIDM *lcidm, void *opaque, - const QUIC_CONN_ID *lcid, - uint64_t seq_num) -{ - QUIC_LCIDM_CONN *conn; - QUIC_LCID key, *lcid_obj; - - if (lcid == NULL || lcid->id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - - if ((conn = lcidm_upsert_conn(lcidm, opaque)) == NULL) - return 0; - - key.cid = *lcid; - if (lh_QUIC_LCID_retrieve(lcidm->lcids, &key) != NULL) - return 0; - - if ((lcid_obj = lcidm_conn_new_lcid(lcidm, conn, lcid)) == NULL) - return 0; - - lcid_obj->seq_num = seq_num; - lcid_obj->type = LCID_TYPE_NCID; - return 1; -} diff --git a/openssl/src/ssl/quic/quic_local.h b/openssl/src/ssl/quic/quic_local.h deleted file mode 100644 index d6518fd6b..000000000 --- a/openssl/src/ssl/quic/quic_local.h +++ /dev/null @@ -1,361 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_LOCAL_H -# define OSSL_QUIC_LOCAL_H - -# include -# include "internal/quic_ssl.h" /* QUIC_CONNECTION */ -# include "internal/quic_txp.h" -# include "internal/quic_statm.h" -# include "internal/quic_demux.h" -# include "internal/quic_record_rx.h" -# include "internal/quic_tls.h" -# include "internal/quic_fc.h" -# include "internal/quic_stream.h" -# include "internal/quic_channel.h" -# include "internal/quic_reactor.h" -# include "internal/quic_thread_assist.h" -# include "../ssl_local.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC stream SSL object (QSSO) type. This implements the API personality layer - * for QSSO objects, wrapping the QUIC-native QUIC_STREAM object and tracking - * state required by the libssl API personality. - */ -struct quic_xso_st { - /* SSL object common header. */ - struct ssl_st ssl; - - /* The connection this stream is associated with. Always non-NULL. */ - QUIC_CONNECTION *conn; - - /* The stream object. Always non-NULL for as long as the XSO exists. */ - QUIC_STREAM *stream; - - /* - * Has this stream been logically configured into blocking mode? Only - * meaningful if desires_blocking_set is 1. Ignored if blocking is not - * currently possible given QUIC_CONNECTION configuration. - */ - unsigned int desires_blocking : 1; - - /* - * Has SSL_set_blocking_mode been called on this stream? If not set, we - * inherit from the QUIC_CONNECTION blocking state. - */ - unsigned int desires_blocking_set : 1; - - /* The application has retired a FIN (i.e. SSL_ERROR_ZERO_RETURN). */ - unsigned int retired_fin : 1; - - /* - * The application has requested a reset. Not set for reflexive - * STREAM_RESETs caused by peer STOP_SENDING. - */ - unsigned int requested_reset : 1; - - /* - * This state tracks SSL_write all-or-nothing (AON) write semantics - * emulation. - * - * Example chronology: - * - * t=0: aon_write_in_progress=0 - * t=1: SSL_write(ssl, b1, l1) called; - * too big to enqueue into sstream at once, SSL_ERROR_WANT_WRITE; - * aon_write_in_progress=1; aon_buf_base=b1; aon_buf_len=l1; - * aon_buf_pos < l1 (depends on how much room was in sstream); - * t=2: SSL_write(ssl, b2, l2); - * b2 must equal b1 (validated unless ACCEPT_MOVING_WRITE_BUFFER) - * l2 must equal l1 (always validated) - * append into sstream from [b2 + aon_buf_pos, b2 + aon_buf_len) - * if done, aon_write_in_progress=0 - * - */ - /* Is an AON write in progress? */ - unsigned int aon_write_in_progress : 1; - - /* Event handling mode. One of SSL_QUIC_VALUE_EVENT_HANDLING. */ - unsigned int event_handling_mode : 2; - - /* - * The base buffer pointer the caller passed us for the initial AON write - * call. We use this for validation purposes unless - * ACCEPT_MOVING_WRITE_BUFFER is enabled. - * - * NOTE: We never dereference this, as the caller might pass a different - * (but identical) buffer if using ACCEPT_MOVING_WRITE_BUFFER. It is for - * validation by pointer comparison only. - */ - const unsigned char *aon_buf_base; - /* The total length of the AON buffer being sent, in bytes. */ - size_t aon_buf_len; - /* - * The position in the AON buffer up to which we have successfully sent data - * so far. - */ - size_t aon_buf_pos; - - /* SSL_set_mode */ - uint32_t ssl_mode; - - /* SSL_set_options */ - uint64_t ssl_options; - - /* - * Last 'normal' error during an app-level I/O operation, used by - * SSL_get_error(); used to track data-path errors like SSL_ERROR_WANT_READ - * and SSL_ERROR_WANT_WRITE. - */ - int last_error; -}; - -struct quic_conn_st { - /* - * ssl_st is a common header for ordinary SSL objects, QUIC connection - * objects and QUIC stream objects, allowing objects of these different - * types to be disambiguated at runtime and providing some common fields. - * - * Note: This must come first in the QUIC_CONNECTION structure. - */ - struct ssl_st ssl; - - SSL *tls; - - /* The QUIC engine representing the QUIC event domain. */ - QUIC_ENGINE *engine; - - /* The QUIC port representing the QUIC listener and socket. */ - QUIC_PORT *port; - - /* - * The QUIC channel providing the core QUIC connection implementation. Note - * that this is not instantiated until we actually start trying to do the - * handshake. This is to allow us to gather information like whether we are - * going to be in client or server mode before committing to instantiating - * the channel, since we want to determine the channel arguments based on - * that. - * - * The channel remains available after connection termination until the SSL - * object is freed, thus (ch != NULL) iff (started == 1). - */ - QUIC_CHANNEL *ch; - - /* - * The mutex used to synchronise access to the QUIC_CHANNEL. We own this but - * provide it to the channel. - */ - CRYPTO_MUTEX *mutex; - - /* - * If we have a default stream attached, this is the internal XSO - * object. If there is no default stream, this is NULL. - */ - QUIC_XSO *default_xso; - - /* The network read and write BIOs. */ - BIO *net_rbio, *net_wbio; - - /* Initial peer L4 address. */ - BIO_ADDR init_peer_addr; - -# ifndef OPENSSL_NO_QUIC_THREAD_ASSIST - /* Manages thread for QUIC thread assisted mode. */ - QUIC_THREAD_ASSIST thread_assist; -# endif - - /* If non-NULL, used instead of ossl_time_now(). Used for testing. */ - OSSL_TIME (*override_now_cb)(void *arg); - void *override_now_cb_arg; - - /* Number of XSOs allocated. Includes the default XSO, if any. */ - size_t num_xso; - - /* Have we started? */ - unsigned int started : 1; - - /* - * This is 1 if we were instantiated using a QUIC server method - * (for future use). - */ - unsigned int as_server : 1; - - /* - * Has the application called SSL_set_accept_state? We require this to be - * congruent with the value of as_server. - */ - unsigned int as_server_state : 1; - - /* Are we using thread assisted mode? Never changes after init. */ - unsigned int is_thread_assisted : 1; - - /* Do connection-level operations (e.g. handshakes) run in blocking mode? */ - unsigned int blocking : 1; - - /* Does the application want blocking mode? */ - unsigned int desires_blocking : 1; - - /* Have we created a default XSO yet? */ - unsigned int default_xso_created : 1; - - /* - * Pre-TERMINATING shutdown phase in which we are flushing streams. - * Monotonically transitions to 1. - * New streams cannot be created in this state. - */ - unsigned int shutting_down : 1; - - /* Have we probed the BIOs for addressing support? */ - unsigned int addressing_probe_done : 1; - - /* Are we using addressed mode (BIO_sendmmsg with non-NULL peer)? */ - unsigned int addressed_mode_w : 1; - unsigned int addressed_mode_r : 1; - - /* Event handling mode. One of SSL_QUIC_VALUE_EVENT_HANDLING. */ - unsigned int event_handling_mode : 2; - - /* Default stream type. Defaults to SSL_DEFAULT_STREAM_MODE_AUTO_BIDI. */ - uint32_t default_stream_mode; - - /* SSL_set_mode. This is not used directly but inherited by new XSOs. */ - uint32_t default_ssl_mode; - - /* SSL_set_options. This is not used directly but inherited by new XSOs. */ - uint64_t default_ssl_options; - - /* SSL_set_incoming_stream_policy. */ - int incoming_stream_policy; - uint64_t incoming_stream_aec; - - /* - * Last 'normal' error during an app-level I/O operation, used by - * SSL_get_error(); used to track data-path errors like SSL_ERROR_WANT_READ - * and SSL_ERROR_WANT_WRITE. - */ - int last_error; -}; - -/* Internal calls to the QUIC CSM which come from various places. */ -int ossl_quic_conn_on_handshake_confirmed(QUIC_CONNECTION *qc); - -/* - * To be called when a protocol violation occurs. The connection is torn down - * with the given error code, which should be a OSSL_QUIC_ERR_* value. Reason - * string is optional and copied if provided. frame_type should be 0 if not - * applicable. - */ -void ossl_quic_conn_raise_protocol_error(QUIC_CONNECTION *qc, - uint64_t error_code, - uint64_t frame_type, - const char *reason); - -void ossl_quic_conn_on_remote_conn_close(QUIC_CONNECTION *qc, - OSSL_QUIC_FRAME_CONN_CLOSE *f); - -int ossl_quic_trace(int write_p, int version, int content_type, - const void *buf, size_t msglen, SSL *ssl, void *arg); - -# define OSSL_QUIC_ANY_VERSION 0xFFFFF -# define IS_QUIC_METHOD(m) \ - ((m) == OSSL_QUIC_client_method() || \ - (m) == OSSL_QUIC_client_thread_method()) -# define IS_QUIC_CTX(ctx) IS_QUIC_METHOD((ctx)->method) - -# define QUIC_CONNECTION_FROM_SSL_int(ssl, c) \ - ((ssl) == NULL ? NULL \ - : ((ssl)->type == SSL_TYPE_QUIC_CONNECTION \ - ? (c QUIC_CONNECTION *)(ssl) \ - : NULL)) - -# define QUIC_XSO_FROM_SSL_int(ssl, c) \ - ((ssl) == NULL \ - ? NULL \ - : (((ssl)->type == SSL_TYPE_QUIC_XSO \ - ? (c QUIC_XSO *)(ssl) \ - : ((ssl)->type == SSL_TYPE_QUIC_CONNECTION \ - ? (c QUIC_XSO *)((QUIC_CONNECTION *)(ssl))->default_xso \ - : NULL)))) - -# define SSL_CONNECTION_FROM_QUIC_SSL_int(ssl, c) \ - ((ssl) == NULL ? NULL \ - : ((ssl)->type == SSL_TYPE_QUIC_CONNECTION \ - ? (c SSL_CONNECTION *)((c QUIC_CONNECTION *)(ssl))->tls \ - : NULL)) - -# define IS_QUIC(ssl) ((ssl) != NULL \ - && ((ssl)->type == SSL_TYPE_QUIC_CONNECTION \ - || (ssl)->type == SSL_TYPE_QUIC_XSO)) -# else -# define QUIC_CONNECTION_FROM_SSL_int(ssl, c) NULL -# define QUIC_XSO_FROM_SSL_int(ssl, c) NULL -# define SSL_CONNECTION_FROM_QUIC_SSL_int(ssl, c) NULL -# define IS_QUIC(ssl) 0 -# define IS_QUIC_CTX(ctx) 0 -# define IS_QUIC_METHOD(m) 0 -# endif - -# define QUIC_CONNECTION_FROM_SSL(ssl) \ - QUIC_CONNECTION_FROM_SSL_int(ssl, SSL_CONNECTION_NO_CONST) -# define QUIC_CONNECTION_FROM_CONST_SSL(ssl) \ - QUIC_CONNECTION_FROM_SSL_int(ssl, const) -# define QUIC_XSO_FROM_SSL(ssl) \ - QUIC_XSO_FROM_SSL_int(ssl, SSL_CONNECTION_NO_CONST) -# define QUIC_XSO_FROM_CONST_SSL(ssl) \ - QUIC_XSO_FROM_SSL_int(ssl, const) -# define SSL_CONNECTION_FROM_QUIC_SSL(ssl) \ - SSL_CONNECTION_FROM_QUIC_SSL_int(ssl, SSL_CONNECTION_NO_CONST) -# define SSL_CONNECTION_FROM_CONST_QUIC_SSL(ssl) \ - SSL_CONNECTION_FROM_CONST_QUIC_SSL_int(ssl, const) - -# define IMPLEMENT_quic_meth_func(version, func_name, q_accept, \ - q_connect, enc_data) \ -const SSL_METHOD *func_name(void) \ - { \ - static const SSL_METHOD func_name##_data= { \ - version, \ - 0, \ - 0, \ - ossl_quic_new, \ - ossl_quic_free, \ - ossl_quic_reset, \ - ossl_quic_init, \ - NULL /* clear */, \ - ossl_quic_deinit, \ - q_accept, \ - q_connect, \ - ossl_quic_read, \ - ossl_quic_peek, \ - ossl_quic_write, \ - NULL /* shutdown */, \ - NULL /* renegotiate */, \ - ossl_quic_renegotiate_check, \ - NULL /* read_bytes */, \ - NULL /* write_bytes */, \ - NULL /* dispatch_alert */, \ - ossl_quic_ctrl, \ - ossl_quic_ctx_ctrl, \ - ossl_quic_get_cipher_by_char, \ - NULL /* put_cipher_by_char */, \ - ossl_quic_pending, \ - ossl_quic_num_ciphers, \ - ossl_quic_get_cipher, \ - tls1_default_timeout, \ - &enc_data, \ - ssl_undefined_void_function, \ - ossl_quic_callback_ctrl, \ - ossl_quic_ctx_callback_ctrl, \ - }; \ - return &func_name##_data; \ - } - -#endif diff --git a/openssl/src/ssl/quic/quic_method.c b/openssl/src/ssl/quic/quic_method.c deleted file mode 100644 index 2882a40f3..000000000 --- a/openssl/src/ssl/quic/quic_method.c +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "quic_local.h" - -IMPLEMENT_quic_meth_func(OSSL_QUIC_ANY_VERSION, - OSSL_QUIC_client_method, - ssl_undefined_function, - ossl_quic_connect, ssl3_undef_enc_method) - -IMPLEMENT_quic_meth_func(OSSL_QUIC_ANY_VERSION, - OSSL_QUIC_client_thread_method, - ssl_undefined_function, - ossl_quic_connect, ssl3_undef_enc_method) diff --git a/openssl/src/ssl/quic/quic_port.c b/openssl/src/ssl/quic/quic_port.c deleted file mode 100644 index 96c289f7e..000000000 --- a/openssl/src/ssl/quic/quic_port.c +++ /dev/null @@ -1,615 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_port.h" -#include "internal/quic_channel.h" -#include "internal/quic_lcidm.h" -#include "internal/quic_srtm.h" -#include "quic_port_local.h" -#include "quic_channel_local.h" -#include "quic_engine_local.h" -#include "../ssl_local.h" - -/* - * QUIC Port Structure - * =================== - */ -#define INIT_DCID_LEN 8 - -static int port_init(QUIC_PORT *port); -static void port_cleanup(QUIC_PORT *port); -static OSSL_TIME get_time(void *arg); -static void port_default_packet_handler(QUIC_URXE *e, void *arg, - const QUIC_CONN_ID *dcid); -static void port_rx_pre(QUIC_PORT *port); - -DEFINE_LIST_OF_IMPL(ch, QUIC_CHANNEL); -DEFINE_LIST_OF_IMPL(port, QUIC_PORT); - -QUIC_PORT *ossl_quic_port_new(const QUIC_PORT_ARGS *args) -{ - QUIC_PORT *port; - - if ((port = OPENSSL_zalloc(sizeof(QUIC_PORT))) == NULL) - return NULL; - - port->engine = args->engine; - port->channel_ctx = args->channel_ctx; - port->is_multi_conn = args->is_multi_conn; - - if (!port_init(port)) { - OPENSSL_free(port); - return NULL; - } - - return port; -} - -void ossl_quic_port_free(QUIC_PORT *port) -{ - if (port == NULL) - return; - - port_cleanup(port); - OPENSSL_free(port); -} - -static int port_init(QUIC_PORT *port) -{ - size_t rx_short_dcid_len = (port->is_multi_conn ? INIT_DCID_LEN : 0); - - if (port->engine == NULL || port->channel_ctx == NULL) - goto err; - - if ((port->err_state = OSSL_ERR_STATE_new()) == NULL) - goto err; - - if ((port->demux = ossl_quic_demux_new(/*BIO=*/NULL, - /*Short CID Len=*/rx_short_dcid_len, - get_time, port)) == NULL) - goto err; - - ossl_quic_demux_set_default_handler(port->demux, - port_default_packet_handler, - port); - - if ((port->srtm = ossl_quic_srtm_new(port->engine->libctx, - port->engine->propq)) == NULL) - goto err; - - if ((port->lcidm = ossl_quic_lcidm_new(port->engine->libctx, - rx_short_dcid_len)) == NULL) - goto err; - - port->rx_short_dcid_len = (unsigned char)rx_short_dcid_len; - port->tx_init_dcid_len = INIT_DCID_LEN; - port->state = QUIC_PORT_STATE_RUNNING; - - ossl_list_port_insert_tail(&port->engine->port_list, port); - port->on_engine_list = 1; - return 1; - -err: - port_cleanup(port); - return 0; -} - -static void port_cleanup(QUIC_PORT *port) -{ - assert(ossl_list_ch_num(&port->channel_list) == 0); - - ossl_quic_demux_free(port->demux); - port->demux = NULL; - - ossl_quic_srtm_free(port->srtm); - port->srtm = NULL; - - ossl_quic_lcidm_free(port->lcidm); - port->lcidm = NULL; - - OSSL_ERR_STATE_free(port->err_state); - port->err_state = NULL; - - if (port->on_engine_list) { - ossl_list_port_remove(&port->engine->port_list, port); - port->on_engine_list = 0; - } -} - -static void port_transition_failed(QUIC_PORT *port) -{ - if (port->state == QUIC_PORT_STATE_FAILED) - return; - - port->state = QUIC_PORT_STATE_FAILED; -} - -int ossl_quic_port_is_running(const QUIC_PORT *port) -{ - return port->state == QUIC_PORT_STATE_RUNNING; -} - -QUIC_ENGINE *ossl_quic_port_get0_engine(QUIC_PORT *port) -{ - return port->engine; -} - -QUIC_REACTOR *ossl_quic_port_get0_reactor(QUIC_PORT *port) -{ - return ossl_quic_engine_get0_reactor(port->engine); -} - -QUIC_DEMUX *ossl_quic_port_get0_demux(QUIC_PORT *port) -{ - return port->demux; -} - -CRYPTO_MUTEX *ossl_quic_port_get0_mutex(QUIC_PORT *port) -{ - return ossl_quic_engine_get0_mutex(port->engine); -} - -OSSL_TIME ossl_quic_port_get_time(QUIC_PORT *port) -{ - return ossl_quic_engine_get_time(port->engine); -} - -static OSSL_TIME get_time(void *port) -{ - return ossl_quic_port_get_time((QUIC_PORT *)port); -} - -int ossl_quic_port_get_rx_short_dcid_len(const QUIC_PORT *port) -{ - return port->rx_short_dcid_len; -} - -int ossl_quic_port_get_tx_init_dcid_len(const QUIC_PORT *port) -{ - return port->tx_init_dcid_len; -} - -/* - * QUIC Port: Network BIO Configuration - * ==================================== - */ - -/* Determines whether we can support a given poll descriptor. */ -static int validate_poll_descriptor(const BIO_POLL_DESCRIPTOR *d) -{ - if (d->type == BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD && d->value.fd < 0) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - return 1; -} - -BIO *ossl_quic_port_get_net_rbio(QUIC_PORT *port) -{ - return port->net_rbio; -} - -BIO *ossl_quic_port_get_net_wbio(QUIC_PORT *port) -{ - return port->net_wbio; -} - -static int port_update_poll_desc(QUIC_PORT *port, BIO *net_bio, int for_write) -{ - BIO_POLL_DESCRIPTOR d = {0}; - - if (net_bio == NULL - || (!for_write && !BIO_get_rpoll_descriptor(net_bio, &d)) - || (for_write && !BIO_get_wpoll_descriptor(net_bio, &d))) - /* Non-pollable BIO */ - d.type = BIO_POLL_DESCRIPTOR_TYPE_NONE; - - if (!validate_poll_descriptor(&d)) - return 0; - - /* - * TODO(QUIC MULTIPORT): We currently only support one port per - * engine/domain. This is necessitated because QUIC_REACTOR only supports a - * single pollable currently. In the future, once complete polling - * infrastructure has been implemented, this limitation can be removed. - * - * For now, just update the descriptor on the the engine's reactor as we are - * guaranteed to be the only port under it. - */ - if (for_write) - ossl_quic_reactor_set_poll_w(&port->engine->rtor, &d); - else - ossl_quic_reactor_set_poll_r(&port->engine->rtor, &d); - - return 1; -} - -int ossl_quic_port_update_poll_descriptors(QUIC_PORT *port) -{ - int ok = 1; - - if (!port_update_poll_desc(port, port->net_rbio, /*for_write=*/0)) - ok = 0; - - if (!port_update_poll_desc(port, port->net_wbio, /*for_write=*/1)) - ok = 0; - - return ok; -} - -/* - * QUIC_PORT does not ref any BIO it is provided with, nor is any ref - * transferred to it. The caller (e.g., QUIC_CONNECTION) is responsible for - * ensuring the BIO lasts until the channel is freed or the BIO is switched out - * for another BIO by a subsequent successful call to this function. - */ -int ossl_quic_port_set_net_rbio(QUIC_PORT *port, BIO *net_rbio) -{ - if (port->net_rbio == net_rbio) - return 1; - - if (!port_update_poll_desc(port, net_rbio, /*for_write=*/0)) - return 0; - - ossl_quic_demux_set_bio(port->demux, net_rbio); - port->net_rbio = net_rbio; - return 1; -} - -int ossl_quic_port_set_net_wbio(QUIC_PORT *port, BIO *net_wbio) -{ - QUIC_CHANNEL *ch; - - if (port->net_wbio == net_wbio) - return 1; - - if (!port_update_poll_desc(port, net_wbio, /*for_write=*/1)) - return 0; - - LIST_FOREACH(ch, ch, &port->channel_list) - ossl_qtx_set_bio(ch->qtx, net_wbio); - - port->net_wbio = net_wbio; - return 1; -} - -/* - * QUIC Port: Channel Lifecycle - * ============================ - */ - -static SSL *port_new_handshake_layer(QUIC_PORT *port) -{ - SSL *tls = NULL; - SSL_CONNECTION *tls_conn = NULL; - - tls = ossl_ssl_connection_new_int(port->channel_ctx, TLS_method()); - if (tls == NULL || (tls_conn = SSL_CONNECTION_FROM_SSL(tls)) == NULL) - return NULL; - - /* Override the user_ssl of the inner connection. */ - tls_conn->s3.flags |= TLS1_FLAGS_QUIC; - - /* Restrict options derived from the SSL_CTX. */ - tls_conn->options &= OSSL_QUIC_PERMITTED_OPTIONS_CONN; - tls_conn->pha_enabled = 0; - return tls; -} - -static QUIC_CHANNEL *port_make_channel(QUIC_PORT *port, SSL *tls, int is_server) -{ - QUIC_CHANNEL_ARGS args = {0}; - QUIC_CHANNEL *ch; - - args.port = port; - args.is_server = is_server; - args.tls = (tls != NULL ? tls : port_new_handshake_layer(port)); - args.lcidm = port->lcidm; - args.srtm = port->srtm; - if (args.tls == NULL) - return NULL; - -#ifndef OPENSSL_NO_QLOG - args.use_qlog = 1; /* disabled if env not set */ - args.qlog_title = args.tls->ctx->qlog_title; -#endif - - ch = ossl_quic_channel_new(&args); - if (ch == NULL) { - if (tls == NULL) - SSL_free(args.tls); - - return NULL; - } - - return ch; -} - -QUIC_CHANNEL *ossl_quic_port_create_outgoing(QUIC_PORT *port, SSL *tls) -{ - return port_make_channel(port, tls, /*is_server=*/0); -} - -QUIC_CHANNEL *ossl_quic_port_create_incoming(QUIC_PORT *port, SSL *tls) -{ - QUIC_CHANNEL *ch; - - assert(port->tserver_ch == NULL); - - ch = port_make_channel(port, tls, /*is_server=*/1); - port->tserver_ch = ch; - port->is_server = 1; - return ch; -} - -/* - * QUIC Port: Ticker-Mutator - * ========================= - */ - -/* - * Tick function for this port. This does everything related to network I/O for - * this port's network BIOs, and services child channels. - */ -void ossl_quic_port_subtick(QUIC_PORT *port, QUIC_TICK_RESULT *res, - uint32_t flags) -{ - QUIC_CHANNEL *ch; - - res->net_read_desired = 0; - res->net_write_desired = 0; - res->tick_deadline = ossl_time_infinite(); - - if (!port->engine->inhibit_tick) { - /* Handle any incoming data from network. */ - if (ossl_quic_port_is_running(port)) - port_rx_pre(port); - - /* Iterate through all channels and service them. */ - LIST_FOREACH(ch, ch, &port->channel_list) { - QUIC_TICK_RESULT subr = {0}; - - ossl_quic_channel_subtick(ch, &subr, flags); - ossl_quic_tick_result_merge_into(res, &subr); - } - } -} - -/* Process incoming datagrams, if any. */ -static void port_rx_pre(QUIC_PORT *port) -{ - int ret; - - /* - * Originally, this check (don't RX before we have sent anything if we are - * not a server, because there can't be anything) was just intended as a - * minor optimisation. However, it is actually required on Windows, and - * removing this check will cause Windows to break. - * - * The reason is that under Win32, recvfrom() does not work on a UDP socket - * which has not had bind() called (???). However, calling sendto() will - * automatically bind an unbound UDP socket. Therefore, if we call a Winsock - * recv-type function before calling a Winsock send-type function, that call - * will fail with WSAEINVAL, which we will regard as a permanent network - * error. - * - * Therefore, this check is essential as we do not require our API users to - * bind a socket first when using the API in client mode. - */ - if (!port->is_server && !port->have_sent_any_pkt) - return; - - /* - * Get DEMUX to BIO_recvmmsg from the network and queue incoming datagrams - * to the appropriate QRX instances. - */ - ret = ossl_quic_demux_pump(port->demux); - if (ret == QUIC_DEMUX_PUMP_RES_PERMANENT_FAIL) - /* - * We don't care about transient failure, but permanent failure means we - * should tear down the port. All connections skip straight to the - * Terminated state as there is no point trying to send CONNECTION_CLOSE - * frames if the network BIO is not operating correctly. - */ - ossl_quic_port_raise_net_error(port, NULL); -} - -/* - * Handles an incoming connection request and potentially decides to make a - * connection from it. If a new connection is made, the new channel is written - * to *new_ch. - */ -static void port_on_new_conn(QUIC_PORT *port, const BIO_ADDR *peer, - const QUIC_CONN_ID *scid, - const QUIC_CONN_ID *dcid, - QUIC_CHANNEL **new_ch) -{ - if (port->tserver_ch != NULL) { - /* Specially assign to existing channel */ - if (!ossl_quic_channel_on_new_conn(port->tserver_ch, peer, scid, dcid)) - return; - - *new_ch = port->tserver_ch; - port->tserver_ch = NULL; - return; - } -} - -static int port_try_handle_stateless_reset(QUIC_PORT *port, const QUIC_URXE *e) -{ - size_t i; - const unsigned char *data = ossl_quic_urxe_data(e); - void *opaque = NULL; - - /* - * Perform some fast and cheap checks for a packet not being a stateless - * reset token. RFC 9000 s. 10.3 specifies this layout for stateless - * reset packets: - * - * Stateless Reset { - * Fixed Bits (2) = 1, - * Unpredictable Bits (38..), - * Stateless Reset Token (128), - * } - * - * It also specifies: - * However, endpoints MUST treat any packet ending in a valid - * stateless reset token as a Stateless Reset, as other QUIC - * versions might allow the use of a long header. - * - * We can rapidly check for the minimum length and that the first pair - * of bits in the first byte are 01 or 11. - * - * The function returns 1 if it is a stateless reset packet, 0 if it isn't - * and -1 if an error was encountered. - */ - if (e->data_len < QUIC_STATELESS_RESET_TOKEN_LEN + 5 - || (0100 & *data) != 0100) - return 0; - - for (i = 0;; ++i) { - if (!ossl_quic_srtm_lookup(port->srtm, - (QUIC_STATELESS_RESET_TOKEN *)(data + e->data_len - - sizeof(QUIC_STATELESS_RESET_TOKEN)), - i, &opaque, NULL)) - break; - - assert(opaque != NULL); - ossl_quic_channel_on_stateless_reset((QUIC_CHANNEL *)opaque); - } - - return i > 0; -} - -/* - * This is called by the demux when we get a packet not destined for any known - * DCID. - */ -static void port_default_packet_handler(QUIC_URXE *e, void *arg, - const QUIC_CONN_ID *dcid) -{ - QUIC_PORT *port = arg; - PACKET pkt; - QUIC_PKT_HDR hdr; - QUIC_CHANNEL *ch = NULL, *new_ch = NULL; - - /* Don't handle anything if we are no longer running. */ - if (!ossl_quic_port_is_running(port)) - goto undesirable; - - if (port_try_handle_stateless_reset(port, e)) - goto undesirable; - - if (dcid != NULL - && ossl_quic_lcidm_lookup(port->lcidm, dcid, NULL, - (void **)&ch)) { - assert(ch != NULL); - ossl_quic_channel_inject(ch, e); - return; - } - - /* - * If we have an incoming packet which doesn't match any existing connection - * we assume this is an attempt to make a new connection. Currently we - * require our caller to have precreated a latent 'incoming' channel via - * TSERVER which then gets turned into the new connection. - * - * TODO(QUIC SERVER): In the future we will construct channels dynamically - * in this case. - */ - if (port->tserver_ch == NULL) - goto undesirable; - - /* - * We have got a packet for an unknown DCID. This might be an attempt to - * open a new connection. - */ - if (e->data_len < QUIC_MIN_INITIAL_DGRAM_LEN) - goto undesirable; - - if (!PACKET_buf_init(&pkt, ossl_quic_urxe_data(e), e->data_len)) - goto undesirable; - - /* - * We set short_conn_id_len to SIZE_MAX here which will cause the decode - * operation to fail if we get a 1-RTT packet. This is fine since we only - * care about Initial packets. - */ - if (!ossl_quic_wire_decode_pkt_hdr(&pkt, SIZE_MAX, 1, 0, &hdr, NULL)) - goto undesirable; - - switch (hdr.version) { - case QUIC_VERSION_1: - break; - - case QUIC_VERSION_NONE: - default: - /* Unknown version or proactive version negotiation request, bail. */ - /* TODO(QUIC SERVER): Handle version negotiation on server side */ - goto undesirable; - } - - /* - * We only care about Initial packets which might be trying to establish a - * connection. - */ - if (hdr.type != QUIC_PKT_TYPE_INITIAL) - goto undesirable; - - /* - * Try to process this as a valid attempt to initiate a connection. - * - * The channel will do all the LCID registration needed, but as an - * optimization inject this packet directly into the channel's QRX for - * processing without going through the DEMUX again. - */ - port_on_new_conn(port, &e->peer, &hdr.src_conn_id, &hdr.dst_conn_id, - &new_ch); - if (new_ch != NULL) - ossl_qrx_inject_urxe(new_ch->qrx, e); - - return; - -undesirable: - ossl_quic_demux_release_urxe(port->demux, e); -} - -void ossl_quic_port_raise_net_error(QUIC_PORT *port, - QUIC_CHANNEL *triggering_ch) -{ - QUIC_CHANNEL *ch; - - if (!ossl_quic_port_is_running(port)) - return; - - /* - * Immediately capture any triggering error on the error stack, with a - * cover error. - */ - ERR_raise_data(ERR_LIB_SSL, SSL_R_QUIC_NETWORK_ERROR, - "port failed due to network BIO I/O error"); - OSSL_ERR_STATE_save(port->err_state); - - port_transition_failed(port); - - /* Give the triggering channel (if any) the first notification. */ - if (triggering_ch != NULL) - ossl_quic_channel_raise_net_error(triggering_ch); - - LIST_FOREACH(ch, ch, &port->channel_list) - if (ch != triggering_ch) - ossl_quic_channel_raise_net_error(ch); -} - -void ossl_quic_port_restore_err_state(const QUIC_PORT *port) -{ - ERR_clear_error(); - OSSL_ERR_STATE_restore(port->err_state); -} diff --git a/openssl/src/ssl/quic/quic_port_local.h b/openssl/src/ssl/quic/quic_port_local.h deleted file mode 100644 index b5e120636..000000000 --- a/openssl/src/ssl/quic/quic_port_local.h +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_PORT_LOCAL_H -# define OSSL_QUIC_PORT_LOCAL_H - -# include "internal/quic_port.h" -# include "internal/quic_reactor.h" -# include "internal/list.h" - -# ifndef OPENSSL_NO_QUIC - -/* - * QUIC Port Structure - * =================== - * - * QUIC port internals. It is intended that only the QUIC_PORT and QUIC_CHANNEL - * implementation be allowed to access this structure directly. - * - * Other components should not include this header. - */ -DECLARE_LIST_OF(ch, QUIC_CHANNEL); - -/* A port is always in one of the following states: */ -enum { - /* Initial and steady state. */ - QUIC_PORT_STATE_RUNNING, - - /* - * Terminal state indicating port is no longer functioning. There are no - * transitions out of this state. May be triggered by e.g. a permanent - * network BIO error. - */ - QUIC_PORT_STATE_FAILED -}; - -struct quic_port_st { - /* The engine which this port is a child of. */ - QUIC_ENGINE *engine; - - /* - * QUIC_ENGINE keeps the ports which belong to it on a list for bookkeeping - * purposes. - */ - OSSL_LIST_MEMBER(port, QUIC_PORT); - - /* Used to create handshake layer objects inside newly created channels. */ - SSL_CTX *channel_ctx; - - /* Network-side read and write BIOs. */ - BIO *net_rbio, *net_wbio; - - /* RX demuxer. We register incoming DCIDs with this. */ - QUIC_DEMUX *demux; - - /* List of all child channels. */ - OSSL_LIST(ch) channel_list; - - /* Special TSERVER channel. To be removed in the future. */ - QUIC_CHANNEL *tserver_ch; - - /* LCIDM used for incoming packet routing by DCID. */ - QUIC_LCIDM *lcidm; - - /* SRTM used for incoming packet routing by SRT. */ - QUIC_SRTM *srtm; - - /* Port-level permanent errors (causing failure state) are stored here. */ - ERR_STATE *err_state; - - /* DCID length used for incoming short header packets. */ - unsigned char rx_short_dcid_len; - /* For clients, CID length used for outgoing Initial packets. */ - unsigned char tx_init_dcid_len; - - /* Port state (QUIC_PORT_STATE_*). */ - unsigned int state : 1; - - /* Is this port created to support multiple connections? */ - unsigned int is_multi_conn : 1; - - /* Has this port sent any packet of any kind yet? */ - unsigned int have_sent_any_pkt : 1; - - /* Does this port allow incoming connections? */ - unsigned int is_server : 1; - - /* Are we on the QUIC_ENGINE linked list of ports? */ - unsigned int on_engine_list : 1; -}; - -# endif - -#endif diff --git a/openssl/src/ssl/quic/quic_rcidm.c b/openssl/src/ssl/quic/quic_rcidm.c deleted file mode 100644 index 77d8cbfcc..000000000 --- a/openssl/src/ssl/quic/quic_rcidm.c +++ /dev/null @@ -1,688 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_rcidm.h" -#include "internal/priority_queue.h" -#include "internal/list.h" -#include "internal/common.h" - -/* - * QUIC Remote Connection ID Manager - * ================================= - * - * We can receive an arbitrary number of RCIDs via NCID frames. Periodically, we - * may desire (for example for anti-connection fingerprinting reasons, etc.) - * to switch to a new RCID according to some arbitrary policy such as the number - * of packets we have sent. - * - * When we do this we should move to the next RCID in the sequence of received - * RCIDs ordered by sequence number. For example, if a peer sends us three NCID - * frames with sequence numbers 10, 11, 12, we should seek to consume these - * RCIDs in order. - * - * However, due to the possibility of packet reordering in the network, NCID - * frames might be received out of order. Thus if a peer sends us NCID frames - * with sequence numbers 12, 10, 11, we should still consume the RCID with - * sequence number 10 before consuming the RCIDs with sequence numbers 11 or 12. - * - * We use a priority queue for this purpose. - */ -static void rcidm_update(QUIC_RCIDM *rcidm); -static void rcidm_set_preferred_rcid(QUIC_RCIDM *rcidm, - const QUIC_CONN_ID *rcid); - -#define PACKETS_PER_RCID 10000 - -#define INITIAL_SEQ_NUM 0 -#define PREF_ADDR_SEQ_NUM 1 - -/* - * RCID - * ==== - * - * The RCID structure is used to track RCIDs which have sequence numbers (i.e., - * INITIAL, PREF_ADDR and NCID type RCIDs). The RCIDs without sequence numbers - * (Initial ODCIDs and Retry ODCIDs), hereafter referred to as unnumbered RCIDs, - * can logically be viewed as their own type of RCID but are tracked separately - * as singletons without needing a discrete structure. - * - * At any given time an RCID object is in one of these states: - * - * - * (start) - * | - * [add] - * | - * _____v_____ ___________ ____________ - * | | | | | | - * | PENDING | --[select]--> | CURRENT | --[retire]--> | RETIRING | - * |___________| |___________| |____________| - * | - * [pop] - * | - * v - * (fin) - * - * The transition through the states is monotonic and irreversible. - * The RCID object is freed when it is popped. - * - * PENDING - * Invariants: - * rcid->state == RCID_STATE_PENDING; - * rcid->pq_idx != SIZE_MAX (debug assert only); - * the RCID is not the current RCID, rcidm->cur_rcid != rcid; - * the RCID is in the priority queue; - * the RCID is not in the retiring_list. - * - * CURRENT - * Invariants: - * rcid->state == RCID_STATE_CUR; - * rcid->pq_idx == SIZE_MAX (debug assert only); - * the RCID is the current RCID, rcidm->cur_rcid == rcid; - * the RCID is not in the priority queue; - * the RCID is not in the retiring_list. - * - * RETIRING - * Invariants: - * rcid->state == RCID_STATE_RETIRING; - * rcid->pq_idx == SIZE_MAX (debug assert only); - * the RCID is not the current RCID, rcidm->cur_rcid != rcid; - * the RCID is not in the priority queue; - * the RCID is in the retiring_list. - * - * Invariant: At most one RCID object is in the CURRENT state at any one time. - * - * (If no RCID object is in the CURRENT state, this means either - * an unnumbered RCID is being used as the preferred RCID - * or we currently have no preferred RCID.) - * - * All of the above states can be considered substates of the 'ACTIVE' state - * for an RCID as specified in RFC 9000. A CID only ceases to be active - * when we send a RETIRE_CONN_ID frame, which is the responsibility of the - * user of the RCIDM and happens after the above state machine is terminated. - */ -enum { - RCID_STATE_PENDING, - RCID_STATE_CUR, - RCID_STATE_RETIRING -}; - -enum { - RCID_TYPE_INITIAL, /* CID is from an peer INITIAL packet (seq 0) */ - RCID_TYPE_PREF_ADDR, /* CID is from a preferred_address TPARAM (seq 1) */ - RCID_TYPE_NCID /* CID is from a NCID frame */ - /* - * INITIAL_ODCID and RETRY_ODCID also conceptually exist but are tracked - * separately. - */ -}; - -typedef struct rcid_st { - OSSL_LIST_MEMBER(retiring, struct rcid_st); /* valid iff RETIRING */ - - QUIC_CONN_ID cid; /* The actual CID string for this RCID */ - uint64_t seq_num; - size_t pq_idx; /* Index of entry into priority queue */ - unsigned int state : 2; /* RCID_STATE_* */ - unsigned int type : 2; /* RCID_TYPE_* */ -} RCID; - -DEFINE_PRIORITY_QUEUE_OF(RCID); -DEFINE_LIST_OF(retiring, RCID); - -/* - * RCID Manager - * ============ - * - * The following "business logic" invariants also apply to the RCIDM - * as a whole: - * - * Invariant: An RCID of INITIAL type has a sequence number of 0. - * Invariant: An RCID of PREF_ADDR type has a sequence number of 1. - * - * Invariant: There is never more than one Initial ODCID - * added throughout the lifetime of an RCIDM. - * Invariant: There is never more than one Retry ODCID - * added throughout the lifetime of an RCIDM. - * Invariant: There is never more than one INITIAL RCID created - * throughout the lifetime of an RCIDM. - * Invariant: There is never more than one PREF_ADDR RCID created - * throughout the lifetime of an RCIDM. - * Invariant: No INITIAL or PREF_ADDR RCID may be added after - * the handshake is completed. - * - */ -struct quic_rcidm_st { - /* - * The current RCID we prefer to use (value undefined if - * !have_preferred_rcid). - * - * This is preferentially set to a numbered RCID (represented by an RCID - * object) if we have one (in which case preferred_rcid == cur_rcid->cid); - * otherwise it is set to one of the unnumbered RCIDs (the Initial ODCID or - * Retry ODCID) if available (and cur_rcid == NULL). - */ - QUIC_CONN_ID preferred_rcid; - - /* - * These are initialized if the corresponding added_ flags are set. - */ - QUIC_CONN_ID initial_odcid, retry_odcid; - - /* - * Total number of packets sent since we last made a packet count-based RCID - * update decision. - */ - uint64_t packets_sent; - - /* Number of post-handshake RCID changes we have performed. */ - uint64_t num_changes; - - /* - * The Retire Prior To watermark value; max(retire_prior_to) of all received - * NCID frames. - */ - uint64_t retire_prior_to; - - /* (SORT BY seq_num ASC) -> (RCID *) */ - PRIORITY_QUEUE_OF(RCID) *rcids; - - /* - * Current RCID object we are using. This may differ from the first item in - * the priority queue if we received NCID frames out of order. For example - * if we get seq 5, switch to it immediately, then get seq 4, we want to - * keep using seq 5 until we decide to roll again rather than immediately - * switch to seq 4. Never points to an object on the retiring_list. - */ - RCID *cur_rcid; - - /* - * When a RCID becomes pending-retirement, it is moved to the retiring_list, - * then freed when it is popped from the retired queue. We use a list for - * this rather than a priority queue as the order in which items are freed - * does not matter. We always append to the tail of the list in order to - * maintain the guarantee that the head (if present) only changes when a - * caller calls pop(). - */ - OSSL_LIST(retiring) retiring_list; - - /* Number of entries on the retiring_list. */ - size_t num_retiring; - - /* preferred_rcid has been changed? */ - unsigned int preferred_rcid_changed : 1; - - /* Do we have any RCID we can use currently? */ - unsigned int have_preferred_rcid : 1; - - /* QUIC handshake has been completed? */ - unsigned int handshake_complete : 1; - - /* odcid was set (not necessarily still valid as a RCID)? */ - unsigned int added_initial_odcid : 1; - /* retry_odcid was set (not necessarily still valid as a RCID?) */ - unsigned int added_retry_odcid : 1; - /* An initial RCID was added as an RCID structure? */ - unsigned int added_initial_rcid : 1; - /* Has a RCID roll been manually requested? */ - unsigned int roll_requested : 1; -}; - -/* - * Caller must periodically pop retired RCIDs and handle them. If the caller - * fails to do so, fail safely rather than start exhibiting integer rollover. - * Limit the total number of numbered RCIDs to an implausibly large but safe - * value. - */ -#define MAX_NUMBERED_RCIDS (SIZE_MAX / 2) - -static void rcidm_transition_rcid(QUIC_RCIDM *rcidm, RCID *rcid, - unsigned int state); - -/* Check invariants of an RCID */ -static void rcidm_check_rcid(QUIC_RCIDM *rcidm, RCID *rcid) -{ - assert(rcid->state == RCID_STATE_PENDING - || rcid->state == RCID_STATE_CUR - || rcid->state == RCID_STATE_RETIRING); - assert((rcid->state == RCID_STATE_PENDING) - == (rcid->pq_idx != SIZE_MAX)); - assert((rcid->state == RCID_STATE_CUR) - == (rcidm->cur_rcid == rcid)); - assert((ossl_list_retiring_next(rcid) != NULL - || ossl_list_retiring_prev(rcid) != NULL - || ossl_list_retiring_head(&rcidm->retiring_list) == rcid) - == (rcid->state == RCID_STATE_RETIRING)); - assert(rcid->type != RCID_TYPE_INITIAL || rcid->seq_num == 0); - assert(rcid->type != RCID_TYPE_PREF_ADDR || rcid->seq_num == 1); - assert(rcid->seq_num <= OSSL_QUIC_VLINT_MAX); - assert(rcid->cid.id_len > 0 && rcid->cid.id_len <= QUIC_MAX_CONN_ID_LEN); - assert(rcid->seq_num >= rcidm->retire_prior_to - || rcid->state == RCID_STATE_RETIRING); - assert(rcidm->num_changes == 0 || rcidm->handshake_complete); - assert(rcid->state != RCID_STATE_RETIRING || rcidm->num_retiring > 0); -} - -static int rcid_cmp(const RCID *a, const RCID *b) -{ - if (a->seq_num < b->seq_num) - return -1; - if (a->seq_num > b->seq_num) - return 1; - return 0; -} - -QUIC_RCIDM *ossl_quic_rcidm_new(const QUIC_CONN_ID *initial_odcid) -{ - QUIC_RCIDM *rcidm; - - if ((rcidm = OPENSSL_zalloc(sizeof(*rcidm))) == NULL) - return NULL; - - if ((rcidm->rcids = ossl_pqueue_RCID_new(rcid_cmp)) == NULL) { - OPENSSL_free(rcidm); - return NULL; - } - - if (initial_odcid != NULL) { - rcidm->initial_odcid = *initial_odcid; - rcidm->added_initial_odcid = 1; - } - - rcidm_update(rcidm); - return rcidm; -} - -void ossl_quic_rcidm_free(QUIC_RCIDM *rcidm) -{ - RCID *rcid, *rnext; - - if (rcidm == NULL) - return; - - OPENSSL_free(rcidm->cur_rcid); - while ((rcid = ossl_pqueue_RCID_pop(rcidm->rcids)) != NULL) - OPENSSL_free(rcid); - - LIST_FOREACH_DELSAFE(rcid, rnext, retiring, &rcidm->retiring_list) - OPENSSL_free(rcid); - - ossl_pqueue_RCID_free(rcidm->rcids); - OPENSSL_free(rcidm); -} - -static void rcidm_set_preferred_rcid(QUIC_RCIDM *rcidm, - const QUIC_CONN_ID *rcid) -{ - if (rcid == NULL) { - rcidm->preferred_rcid_changed = 1; - rcidm->have_preferred_rcid = 0; - return; - } - - if (ossl_quic_conn_id_eq(&rcidm->preferred_rcid, rcid)) - return; - - rcidm->preferred_rcid = *rcid; - rcidm->preferred_rcid_changed = 1; - rcidm->have_preferred_rcid = 1; -} - -/* - * RCID Lifecycle Management - * ========================= - */ -static RCID *rcidm_create_rcid(QUIC_RCIDM *rcidm, uint64_t seq_num, - const QUIC_CONN_ID *cid, - unsigned int type) -{ - RCID *rcid; - - if (cid->id_len < 1 || cid->id_len > QUIC_MAX_CONN_ID_LEN - || seq_num > OSSL_QUIC_VLINT_MAX - || ossl_pqueue_RCID_num(rcidm->rcids) + rcidm->num_retiring - > MAX_NUMBERED_RCIDS) - return NULL; - - if ((rcid = OPENSSL_zalloc(sizeof(*rcid))) == NULL) - return NULL; - - rcid->seq_num = seq_num; - rcid->cid = *cid; - rcid->type = type; - - if (rcid->seq_num >= rcidm->retire_prior_to) { - rcid->state = RCID_STATE_PENDING; - - if (!ossl_pqueue_RCID_push(rcidm->rcids, rcid, &rcid->pq_idx)) { - OPENSSL_free(rcid); - return NULL; - } - } else { - /* RCID is immediately retired upon creation. */ - rcid->state = RCID_STATE_RETIRING; - rcid->pq_idx = SIZE_MAX; - ossl_list_retiring_insert_tail(&rcidm->retiring_list, rcid); - ++rcidm->num_retiring; - } - - rcidm_check_rcid(rcidm, rcid); - return rcid; -} - -static void rcidm_transition_rcid(QUIC_RCIDM *rcidm, RCID *rcid, - unsigned int state) -{ - unsigned int old_state = rcid->state; - - assert(state >= old_state && state <= RCID_STATE_RETIRING); - rcidm_check_rcid(rcidm, rcid); - if (state == old_state) - return; - - if (rcidm->cur_rcid != NULL && state == RCID_STATE_CUR) { - rcidm_transition_rcid(rcidm, rcidm->cur_rcid, RCID_STATE_RETIRING); - assert(rcidm->cur_rcid == NULL); - } - - if (old_state == RCID_STATE_PENDING) { - ossl_pqueue_RCID_remove(rcidm->rcids, rcid->pq_idx); - rcid->pq_idx = SIZE_MAX; - } - - rcid->state = state; - - if (state == RCID_STATE_CUR) { - rcidm->cur_rcid = rcid; - } else if (state == RCID_STATE_RETIRING) { - if (old_state == RCID_STATE_CUR) - rcidm->cur_rcid = NULL; - - ossl_list_retiring_insert_tail(&rcidm->retiring_list, rcid); - ++rcidm->num_retiring; - } - - rcidm_check_rcid(rcidm, rcid); -} - -static void rcidm_free_rcid(QUIC_RCIDM *rcidm, RCID *rcid) -{ - if (rcid == NULL) - return; - - rcidm_check_rcid(rcidm, rcid); - - switch (rcid->state) { - case RCID_STATE_PENDING: - ossl_pqueue_RCID_remove(rcidm->rcids, rcid->pq_idx); - break; - case RCID_STATE_CUR: - rcidm->cur_rcid = NULL; - break; - case RCID_STATE_RETIRING: - ossl_list_retiring_remove(&rcidm->retiring_list, rcid); - --rcidm->num_retiring; - break; - default: - assert(0); - break; - } - - OPENSSL_free(rcid); -} - -static void rcidm_handle_retire_prior_to(QUIC_RCIDM *rcidm, - uint64_t retire_prior_to) -{ - RCID *rcid; - - if (retire_prior_to <= rcidm->retire_prior_to) - return; - - /* - * Retire the current RCID (if any) if it is affected. - */ - if (rcidm->cur_rcid != NULL && rcidm->cur_rcid->seq_num < retire_prior_to) - rcidm_transition_rcid(rcidm, rcidm->cur_rcid, RCID_STATE_RETIRING); - - /* - * Any other RCIDs needing retirement will be at the start of the priority - * queue, so just stop once we see a higher sequence number exceeding the - * threshold. - */ - while ((rcid = ossl_pqueue_RCID_peek(rcidm->rcids)) != NULL - && rcid->seq_num < retire_prior_to) - rcidm_transition_rcid(rcidm, rcid, RCID_STATE_RETIRING); - - rcidm->retire_prior_to = retire_prior_to; -} - -/* - * Decision Logic - * ============== - */ - -static void rcidm_roll(QUIC_RCIDM *rcidm) -{ - RCID *rcid; - - if ((rcid = ossl_pqueue_RCID_peek(rcidm->rcids)) == NULL) - return; - - rcidm_transition_rcid(rcidm, rcid, RCID_STATE_CUR); - - ++rcidm->num_changes; - rcidm->roll_requested = 0; - - if (rcidm->packets_sent >= PACKETS_PER_RCID) - rcidm->packets_sent %= PACKETS_PER_RCID; - else - rcidm->packets_sent = 0; -} - -static void rcidm_update(QUIC_RCIDM *rcidm) -{ - RCID *rcid; - - /* - * If we have no current numbered RCID but have one or more pending, use it. - */ - if (rcidm->cur_rcid == NULL - && (rcid = ossl_pqueue_RCID_peek(rcidm->rcids)) != NULL) { - rcidm_transition_rcid(rcidm, rcid, RCID_STATE_CUR); - assert(rcidm->cur_rcid != NULL); - } - - /* Prefer use of any current numbered RCID we have, if possible. */ - if (rcidm->cur_rcid != NULL) { - rcidm_check_rcid(rcidm, rcidm->cur_rcid); - rcidm_set_preferred_rcid(rcidm, &rcidm->cur_rcid->cid); - return; - } - - /* - * If there are no RCIDs from NCID frames we can use, go through the various - * kinds of bootstrapping RCIDs we can use in order of priority. - */ - if (rcidm->added_retry_odcid && !rcidm->handshake_complete) { - rcidm_set_preferred_rcid(rcidm, &rcidm->retry_odcid); - return; - } - - if (rcidm->added_initial_odcid && !rcidm->handshake_complete) { - rcidm_set_preferred_rcid(rcidm, &rcidm->initial_odcid); - return; - } - - /* We don't know of any usable RCIDs */ - rcidm_set_preferred_rcid(rcidm, NULL); -} - -static int rcidm_should_roll(QUIC_RCIDM *rcidm) -{ - /* - * Always switch as soon as possible if handshake completes; - * and every n packets after handshake completes or the last roll; and - * whenever manually requested. - */ - return rcidm->handshake_complete - && (rcidm->num_changes == 0 - || rcidm->packets_sent >= PACKETS_PER_RCID - || rcidm->roll_requested); -} - -static void rcidm_tick(QUIC_RCIDM *rcidm) -{ - if (rcidm_should_roll(rcidm)) - rcidm_roll(rcidm); - - rcidm_update(rcidm); -} - -/* - * Events - * ====== - */ -void ossl_quic_rcidm_on_handshake_complete(QUIC_RCIDM *rcidm) -{ - if (rcidm->handshake_complete) - return; - - rcidm->handshake_complete = 1; - rcidm_tick(rcidm); -} - -void ossl_quic_rcidm_on_packet_sent(QUIC_RCIDM *rcidm, uint64_t num_packets) -{ - if (num_packets == 0) - return; - - rcidm->packets_sent += num_packets; - rcidm_tick(rcidm); -} - -void ossl_quic_rcidm_request_roll(QUIC_RCIDM *rcidm) -{ - rcidm->roll_requested = 1; - rcidm_tick(rcidm); -} - -/* - * Mutation Operations - * =================== - */ -int ossl_quic_rcidm_add_from_initial(QUIC_RCIDM *rcidm, - const QUIC_CONN_ID *rcid) -{ - RCID *rcid_obj; - - if (rcidm->added_initial_rcid || rcidm->handshake_complete) - return 0; - - rcid_obj = rcidm_create_rcid(rcidm, INITIAL_SEQ_NUM, - rcid, RCID_TYPE_INITIAL); - if (rcid_obj == NULL) - return 0; - - rcidm->added_initial_rcid = 1; - rcidm_tick(rcidm); - return 1; -} - -int ossl_quic_rcidm_add_from_server_retry(QUIC_RCIDM *rcidm, - const QUIC_CONN_ID *retry_odcid) -{ - if (rcidm->added_retry_odcid || rcidm->handshake_complete) - return 0; - - rcidm->retry_odcid = *retry_odcid; - rcidm->added_retry_odcid = 1; - rcidm_tick(rcidm); - return 1; -} - -int ossl_quic_rcidm_add_from_ncid(QUIC_RCIDM *rcidm, - const OSSL_QUIC_FRAME_NEW_CONN_ID *ncid) -{ - RCID *rcid; - - rcid = rcidm_create_rcid(rcidm, ncid->seq_num, &ncid->conn_id, RCID_TYPE_NCID); - if (rcid == NULL) - return 0; - - rcidm_handle_retire_prior_to(rcidm, ncid->retire_prior_to); - rcidm_tick(rcidm); - return 1; -} - -/* - * Queries - * ======= - */ - -static int rcidm_get_retire(QUIC_RCIDM *rcidm, uint64_t *seq_num, int peek) -{ - RCID *rcid = ossl_list_retiring_head(&rcidm->retiring_list); - - if (rcid == NULL) - return 0; - - if (seq_num != NULL) - *seq_num = rcid->seq_num; - - if (!peek) - rcidm_free_rcid(rcidm, rcid); - - return 1; -} - -int ossl_quic_rcidm_pop_retire_seq_num(QUIC_RCIDM *rcidm, - uint64_t *seq_num) -{ - return rcidm_get_retire(rcidm, seq_num, /*peek=*/0); -} - -int ossl_quic_rcidm_peek_retire_seq_num(QUIC_RCIDM *rcidm, - uint64_t *seq_num) -{ - return rcidm_get_retire(rcidm, seq_num, /*peek=*/1); -} - -int ossl_quic_rcidm_get_preferred_tx_dcid(QUIC_RCIDM *rcidm, - QUIC_CONN_ID *tx_dcid) -{ - if (!rcidm->have_preferred_rcid) - return 0; - - *tx_dcid = rcidm->preferred_rcid; - return 1; -} - -int ossl_quic_rcidm_get_preferred_tx_dcid_changed(QUIC_RCIDM *rcidm, - int clear) -{ - int r = rcidm->preferred_rcid_changed; - - if (clear) - rcidm->preferred_rcid_changed = 0; - - return r; -} - -size_t ossl_quic_rcidm_get_num_active(const QUIC_RCIDM *rcidm) -{ - return ossl_pqueue_RCID_num(rcidm->rcids) - + (rcidm->cur_rcid != NULL ? 1 : 0) - + ossl_quic_rcidm_get_num_retiring(rcidm); -} - -size_t ossl_quic_rcidm_get_num_retiring(const QUIC_RCIDM *rcidm) -{ - return rcidm->num_retiring; -} diff --git a/openssl/src/ssl/quic/quic_reactor.c b/openssl/src/ssl/quic/quic_reactor.c deleted file mode 100644 index 3975b8771..000000000 --- a/openssl/src/ssl/quic/quic_reactor.c +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#include "internal/quic_reactor.h" -#include "internal/common.h" -#include "internal/thread_arch.h" - -/* - * Core I/O Reactor Framework - * ========================== - */ -void ossl_quic_reactor_init(QUIC_REACTOR *rtor, - void (*tick_cb)(QUIC_TICK_RESULT *res, void *arg, - uint32_t flags), - void *tick_cb_arg, - OSSL_TIME initial_tick_deadline) -{ - rtor->poll_r.type = BIO_POLL_DESCRIPTOR_TYPE_NONE; - rtor->poll_w.type = BIO_POLL_DESCRIPTOR_TYPE_NONE; - rtor->net_read_desired = 0; - rtor->net_write_desired = 0; - rtor->can_poll_r = 0; - rtor->can_poll_w = 0; - rtor->tick_deadline = initial_tick_deadline; - - rtor->tick_cb = tick_cb; - rtor->tick_cb_arg = tick_cb_arg; -} - -void ossl_quic_reactor_set_poll_r(QUIC_REACTOR *rtor, const BIO_POLL_DESCRIPTOR *r) -{ - if (r == NULL) - rtor->poll_r.type = BIO_POLL_DESCRIPTOR_TYPE_NONE; - else - rtor->poll_r = *r; - - rtor->can_poll_r - = ossl_quic_reactor_can_support_poll_descriptor(rtor, &rtor->poll_r); -} - -void ossl_quic_reactor_set_poll_w(QUIC_REACTOR *rtor, const BIO_POLL_DESCRIPTOR *w) -{ - if (w == NULL) - rtor->poll_w.type = BIO_POLL_DESCRIPTOR_TYPE_NONE; - else - rtor->poll_w = *w; - - rtor->can_poll_w - = ossl_quic_reactor_can_support_poll_descriptor(rtor, &rtor->poll_w); -} - -const BIO_POLL_DESCRIPTOR *ossl_quic_reactor_get_poll_r(const QUIC_REACTOR *rtor) -{ - return &rtor->poll_r; -} - -const BIO_POLL_DESCRIPTOR *ossl_quic_reactor_get_poll_w(const QUIC_REACTOR *rtor) -{ - return &rtor->poll_w; -} - -int ossl_quic_reactor_can_support_poll_descriptor(const QUIC_REACTOR *rtor, - const BIO_POLL_DESCRIPTOR *d) -{ - return d->type == BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD; -} - -int ossl_quic_reactor_can_poll_r(const QUIC_REACTOR *rtor) -{ - return rtor->can_poll_r; -} - -int ossl_quic_reactor_can_poll_w(const QUIC_REACTOR *rtor) -{ - return rtor->can_poll_w; -} - -int ossl_quic_reactor_net_read_desired(QUIC_REACTOR *rtor) -{ - return rtor->net_read_desired; -} - -int ossl_quic_reactor_net_write_desired(QUIC_REACTOR *rtor) -{ - return rtor->net_write_desired; -} - -OSSL_TIME ossl_quic_reactor_get_tick_deadline(QUIC_REACTOR *rtor) -{ - return rtor->tick_deadline; -} - -int ossl_quic_reactor_tick(QUIC_REACTOR *rtor, uint32_t flags) -{ - QUIC_TICK_RESULT res = {0}; - - /* - * Note that the tick callback cannot fail; this is intentional. Arguably it - * does not make that much sense for ticking to 'fail' (in the sense of an - * explicit error indicated to the user) because ticking is by its nature - * best effort. If something fatal happens with a connection we can report - * it on the next actual application I/O call. - */ - rtor->tick_cb(&res, rtor->tick_cb_arg, flags); - - rtor->net_read_desired = res.net_read_desired; - rtor->net_write_desired = res.net_write_desired; - rtor->tick_deadline = res.tick_deadline; - return 1; -} - -/* - * Blocking I/O Adaptation Layer - * ============================= - */ - -/* - * Utility which can be used to poll on up to two FDs. This is designed to - * support use of split FDs (e.g. with SSL_set_rfd and SSL_set_wfd where - * different FDs are used for read and write). - * - * Generally use of poll(2) is preferred where available. Windows, however, - * hasn't traditionally offered poll(2), only select(2). WSAPoll() was - * introduced in Vista but has seemingly been buggy until relatively recent - * versions of Windows 10. Moreover we support XP so this is not a suitable - * target anyway. However, the traditional issues with select(2) turn out not to - * be an issue on Windows; whereas traditional *NIX select(2) uses a bitmap of - * FDs (and thus is limited in the magnitude of the FDs expressible), Windows - * select(2) is very different. In Windows, socket handles are not allocated - * contiguously from zero and thus this bitmap approach was infeasible. Thus in - * adapting the Berkeley sockets API to Windows a different approach was taken - * whereby the fd_set contains a fixed length array of socket handles and an - * integer indicating how many entries are valid; thus Windows select() - * ironically is actually much more like *NIX poll(2) than *NIX select(2). In - * any case, this means that the relevant limit for Windows select() is the - * number of FDs being polled, not the magnitude of those FDs. Since we only - * poll for two FDs here, this limit does not concern us. - * - * Usage: rfd and wfd may be the same or different. Either or both may also be - * -1. If rfd_want_read is 1, rfd is polled for readability, and if - * wfd_want_write is 1, wfd is polled for writability. Note that since any - * passed FD is always polled for error conditions, setting rfd_want_read=0 and - * wfd_want_write=0 is not the same as passing -1 for both FDs. - * - * deadline is a timestamp to return at. If it is ossl_time_infinite(), the call - * never times out. - * - * Returns 0 on error and 1 on success. Timeout expiry is considered a success - * condition. We don't elaborate our return values here because the way we are - * actually using this doesn't currently care. - * - * If mutex is non-NULL, it is assumed to be held for write and is unlocked for - * the duration of the call. - * - * Precondition: mutex is NULL or is held for write (unchecked) - * Postcondition: mutex is NULL or is held for write (unless - * CRYPTO_THREAD_write_lock fails) - */ -static int poll_two_fds(int rfd, int rfd_want_read, - int wfd, int wfd_want_write, - OSSL_TIME deadline, - CRYPTO_MUTEX *mutex) -{ -#if defined(OPENSSL_SYS_WINDOWS) || !defined(POLLIN) - fd_set rfd_set, wfd_set, efd_set; - OSSL_TIME now, timeout; - struct timeval tv, *ptv; - int maxfd, pres; - -# ifndef OPENSSL_SYS_WINDOWS - /* - * On Windows there is no relevant limit to the magnitude of a fd value (see - * above). On *NIX the fd_set uses a bitmap and we must check the limit. - */ - if (rfd >= FD_SETSIZE || wfd >= FD_SETSIZE) - return 0; -# endif - - FD_ZERO(&rfd_set); - FD_ZERO(&wfd_set); - FD_ZERO(&efd_set); - - if (rfd != -1 && rfd_want_read) - openssl_fdset(rfd, &rfd_set); - if (wfd != -1 && wfd_want_write) - openssl_fdset(wfd, &wfd_set); - - /* Always check for error conditions. */ - if (rfd != -1) - openssl_fdset(rfd, &efd_set); - if (wfd != -1) - openssl_fdset(wfd, &efd_set); - - maxfd = rfd; - if (wfd > maxfd) - maxfd = wfd; - - if (!ossl_assert(rfd != -1 || wfd != -1 - || !ossl_time_is_infinite(deadline))) - /* Do not block forever; should not happen. */ - return 0; - -# if defined(OPENSSL_THREADS) - if (mutex != NULL) - ossl_crypto_mutex_unlock(mutex); -# endif - - do { - /* - * select expects a timeout, not a deadline, so do the conversion. - * Update for each call to ensure the correct value is used if we repeat - * due to EINTR. - */ - if (ossl_time_is_infinite(deadline)) { - ptv = NULL; - } else { - now = ossl_time_now(); - /* - * ossl_time_subtract saturates to zero so we don't need to check if - * now > deadline. - */ - timeout = ossl_time_subtract(deadline, now); - tv = ossl_time_to_timeval(timeout); - ptv = &tv; - } - - pres = select(maxfd + 1, &rfd_set, &wfd_set, &efd_set, ptv); - } while (pres == -1 && get_last_socket_error_is_eintr()); - -# if defined(OPENSSL_THREADS) - if (mutex != NULL) - ossl_crypto_mutex_lock(mutex); -# endif - - return pres < 0 ? 0 : 1; -#else - int pres, timeout_ms; - OSSL_TIME now, timeout; - struct pollfd pfds[2] = {0}; - size_t npfd = 0; - - if (rfd == wfd) { - pfds[npfd].fd = rfd; - pfds[npfd].events = (rfd_want_read ? POLLIN : 0) - | (wfd_want_write ? POLLOUT : 0); - if (rfd >= 0 && pfds[npfd].events != 0) - ++npfd; - } else { - pfds[npfd].fd = rfd; - pfds[npfd].events = (rfd_want_read ? POLLIN : 0); - if (rfd >= 0 && pfds[npfd].events != 0) - ++npfd; - - pfds[npfd].fd = wfd; - pfds[npfd].events = (wfd_want_write ? POLLOUT : 0); - if (wfd >= 0 && pfds[npfd].events != 0) - ++npfd; - } - - if (!ossl_assert(npfd != 0 || !ossl_time_is_infinite(deadline))) - /* Do not block forever; should not happen. */ - return 0; - -# if defined(OPENSSL_THREADS) - if (mutex != NULL) - ossl_crypto_mutex_unlock(mutex); -# endif - - do { - if (ossl_time_is_infinite(deadline)) { - timeout_ms = -1; - } else { - now = ossl_time_now(); - timeout = ossl_time_subtract(deadline, now); - timeout_ms = ossl_time2ms(timeout); - } - - pres = poll(pfds, npfd, timeout_ms); - } while (pres == -1 && get_last_socket_error_is_eintr()); - -# if defined(OPENSSL_THREADS) - if (mutex != NULL) - ossl_crypto_mutex_lock(mutex); -# endif - - return pres < 0 ? 0 : 1; -#endif -} - -static int poll_descriptor_to_fd(const BIO_POLL_DESCRIPTOR *d, int *fd) -{ - if (d == NULL || d->type == BIO_POLL_DESCRIPTOR_TYPE_NONE) { - *fd = INVALID_SOCKET; - return 1; - } - - if (d->type != BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD - || d->value.fd == INVALID_SOCKET) - return 0; - - *fd = d->value.fd; - return 1; -} - -/* - * Poll up to two abstract poll descriptors. Currently we only support - * poll descriptors which represent FDs. - * - * If mutex is non-NULL, it is assumed be a lock currently held for write and is - * unlocked for the duration of any wait. - * - * Precondition: mutex is NULL or is held for write (unchecked) - * Postcondition: mutex is NULL or is held for write (unless - * CRYPTO_THREAD_write_lock fails) - */ -static int poll_two_descriptors(const BIO_POLL_DESCRIPTOR *r, int r_want_read, - const BIO_POLL_DESCRIPTOR *w, int w_want_write, - OSSL_TIME deadline, - CRYPTO_MUTEX *mutex) -{ - int rfd, wfd; - - if (!poll_descriptor_to_fd(r, &rfd) - || !poll_descriptor_to_fd(w, &wfd)) - return 0; - - return poll_two_fds(rfd, r_want_read, wfd, w_want_write, deadline, mutex); -} - -/* - * Block until a predicate function evaluates to true. - * - * If mutex is non-NULL, it is assumed be a lock currently held for write and is - * unlocked for the duration of any wait. - * - * Precondition: Must hold channel write lock (unchecked) - * Precondition: mutex is NULL or is held for write (unchecked) - * Postcondition: mutex is NULL or is held for write (unless - * CRYPTO_THREAD_write_lock fails) - */ -int ossl_quic_reactor_block_until_pred(QUIC_REACTOR *rtor, - int (*pred)(void *arg), void *pred_arg, - uint32_t flags, - CRYPTO_MUTEX *mutex) -{ - int res; - - for (;;) { - if ((flags & SKIP_FIRST_TICK) != 0) - flags &= ~SKIP_FIRST_TICK; - else - /* best effort */ - ossl_quic_reactor_tick(rtor, 0); - - if ((res = pred(pred_arg)) != 0) - return res; - - if (!poll_two_descriptors(ossl_quic_reactor_get_poll_r(rtor), - ossl_quic_reactor_net_read_desired(rtor), - ossl_quic_reactor_get_poll_w(rtor), - ossl_quic_reactor_net_write_desired(rtor), - ossl_quic_reactor_get_tick_deadline(rtor), - mutex)) - /* - * We don't actually care why the call succeeded (timeout, FD - * readiness), we just call reactor_tick and start trying to do I/O - * things again. If poll_two_fds returns 0, this is some other - * non-timeout failure and we should stop here. - * - * TODO(QUIC FUTURE): In the future we could avoid unnecessary - * syscalls by not retrying network I/O that isn't ready based - * on the result of the poll call. However this might be difficult - * because it requires we do the call to poll(2) or equivalent - * syscall ourselves, whereas in the general case the application - * does the polling and just calls SSL_handle_events(). - * Implementing this optimisation in the future will probably - * therefore require API changes. - */ - return 0; - } -} diff --git a/openssl/src/ssl/quic/quic_record_rx.c b/openssl/src/ssl/quic/quic_record_rx.c deleted file mode 100644 index 722b957a4..000000000 --- a/openssl/src/ssl/quic/quic_record_rx.c +++ /dev/null @@ -1,1357 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/quic_record_rx.h" -#include "quic_record_shared.h" -#include "internal/common.h" -#include "internal/list.h" -#include "../ssl_local.h" - -/* - * Mark a packet in a bitfield. - * - * pkt_idx: index of packet within datagram. - */ -static ossl_inline void pkt_mark(uint64_t *bitf, size_t pkt_idx) -{ - assert(pkt_idx < QUIC_MAX_PKT_PER_URXE); - *bitf |= ((uint64_t)1) << pkt_idx; -} - -/* Returns 1 if a packet is in the bitfield. */ -static ossl_inline int pkt_is_marked(const uint64_t *bitf, size_t pkt_idx) -{ - assert(pkt_idx < QUIC_MAX_PKT_PER_URXE); - return (*bitf & (((uint64_t)1) << pkt_idx)) != 0; -} - -/* - * RXE - * === - * - * RX Entries (RXEs) store processed (i.e., decrypted) data received from the - * network. One RXE is used per received QUIC packet. - */ -typedef struct rxe_st RXE; - -struct rxe_st { - OSSL_QRX_PKT pkt; - OSSL_LIST_MEMBER(rxe, RXE); - size_t data_len, alloc_len, refcount; - - /* Extra fields for per-packet information. */ - QUIC_PKT_HDR hdr; /* data/len are decrypted payload */ - - /* Decoded packet number. */ - QUIC_PN pn; - - /* Addresses copied from URXE. */ - BIO_ADDR peer, local; - - /* Time we received the packet (not when we processed it). */ - OSSL_TIME time; - - /* Total length of the datagram which contained this packet. */ - size_t datagram_len; - - /* - * The key epoch the packet was received with. Always 0 for non-1-RTT - * packets. - */ - uint64_t key_epoch; - - /* - * Monotonically increases with each datagram received. - * For diagnostic use only. - */ - uint64_t datagram_id; - - /* - * alloc_len allocated bytes (of which data_len bytes are valid) follow this - * structure. - */ -}; - -DEFINE_LIST_OF(rxe, RXE); -typedef OSSL_LIST(rxe) RXE_LIST; - -static ossl_inline unsigned char *rxe_data(const RXE *e) -{ - return (unsigned char *)(e + 1); -} - -/* - * QRL - * === - */ -struct ossl_qrx_st { - OSSL_LIB_CTX *libctx; - const char *propq; - - /* Demux to receive datagrams from. */ - QUIC_DEMUX *demux; - - /* Length of connection IDs used in short-header packets in bytes. */ - size_t short_conn_id_len; - - /* Maximum number of deferred datagrams buffered at any one time. */ - size_t max_deferred; - - /* Current count of deferred datagrams. */ - size_t num_deferred; - - /* - * List of URXEs which are filled with received encrypted data. - * These are returned to the DEMUX's free list as they are processed. - */ - QUIC_URXE_LIST urx_pending; - - /* - * List of URXEs which we could not decrypt immediately and which are being - * kept in case they can be decrypted later. - */ - QUIC_URXE_LIST urx_deferred; - - /* - * List of RXEs which are not currently in use. These are moved - * to the pending list as they are filled. - */ - RXE_LIST rx_free; - - /* - * List of RXEs which are filled with decrypted packets ready to be passed - * to the user. A RXE is removed from all lists inside the QRL when passed - * to the user, then returned to the free list when the user returns it. - */ - RXE_LIST rx_pending; - - /* Largest PN we have received and processed in a given PN space. */ - QUIC_PN largest_pn[QUIC_PN_SPACE_NUM]; - - /* Per encryption-level state. */ - OSSL_QRL_ENC_LEVEL_SET el_set; - - /* Bytes we have received since this counter was last cleared. */ - uint64_t bytes_received; - - /* - * Number of forged packets we have received since the QRX was instantiated. - * Note that as per RFC 9001, this is connection-level state; it is not per - * EL and is not reset by a key update. - */ - uint64_t forged_pkt_count; - - /* - * The PN the current key epoch started at, inclusive. - */ - uint64_t cur_epoch_start_pn; - - /* Validation callback. */ - ossl_qrx_late_validation_cb *validation_cb; - void *validation_cb_arg; - - /* Key update callback. */ - ossl_qrx_key_update_cb *key_update_cb; - void *key_update_cb_arg; - - /* Initial key phase. For debugging use only; always 0 in real use. */ - unsigned char init_key_phase_bit; - - /* Are we allowed to process 1-RTT packets yet? */ - unsigned char allow_1rtt; - - /* Message callback related arguments */ - ossl_msg_cb msg_callback; - void *msg_callback_arg; - SSL *msg_callback_ssl; -}; - -OSSL_QRX *ossl_qrx_new(const OSSL_QRX_ARGS *args) -{ - OSSL_QRX *qrx; - size_t i; - - if (args->demux == NULL || args->max_deferred == 0) - return NULL; - - qrx = OPENSSL_zalloc(sizeof(OSSL_QRX)); - if (qrx == NULL) - return NULL; - - for (i = 0; i < OSSL_NELEM(qrx->largest_pn); ++i) - qrx->largest_pn[i] = args->init_largest_pn[i]; - - qrx->libctx = args->libctx; - qrx->propq = args->propq; - qrx->demux = args->demux; - qrx->short_conn_id_len = args->short_conn_id_len; - qrx->init_key_phase_bit = args->init_key_phase_bit; - qrx->max_deferred = args->max_deferred; - return qrx; -} - -static void qrx_cleanup_rxl(RXE_LIST *l) -{ - RXE *e, *enext; - - for (e = ossl_list_rxe_head(l); e != NULL; e = enext) { - enext = ossl_list_rxe_next(e); - ossl_list_rxe_remove(l, e); - OPENSSL_free(e); - } -} - -static void qrx_cleanup_urxl(OSSL_QRX *qrx, QUIC_URXE_LIST *l) -{ - QUIC_URXE *e, *enext; - - for (e = ossl_list_urxe_head(l); e != NULL; e = enext) { - enext = ossl_list_urxe_next(e); - ossl_list_urxe_remove(l, e); - ossl_quic_demux_release_urxe(qrx->demux, e); - } -} - -void ossl_qrx_free(OSSL_QRX *qrx) -{ - uint32_t i; - - if (qrx == NULL) - return; - - /* Free RXE queue data. */ - qrx_cleanup_rxl(&qrx->rx_free); - qrx_cleanup_rxl(&qrx->rx_pending); - qrx_cleanup_urxl(qrx, &qrx->urx_pending); - qrx_cleanup_urxl(qrx, &qrx->urx_deferred); - - /* Drop keying material and crypto resources. */ - for (i = 0; i < QUIC_ENC_LEVEL_NUM; ++i) - ossl_qrl_enc_level_set_discard(&qrx->el_set, i); - - OPENSSL_free(qrx); -} - -void ossl_qrx_inject_urxe(OSSL_QRX *qrx, QUIC_URXE *urxe) -{ - /* Initialize our own fields inside the URXE and add to the pending list. */ - urxe->processed = 0; - urxe->hpr_removed = 0; - urxe->deferred = 0; - ossl_list_urxe_insert_tail(&qrx->urx_pending, urxe); - - if (qrx->msg_callback != NULL) - qrx->msg_callback(0, OSSL_QUIC1_VERSION, SSL3_RT_QUIC_DATAGRAM, urxe + 1, - urxe->data_len, qrx->msg_callback_ssl, - qrx->msg_callback_arg); -} - -static void qrx_requeue_deferred(OSSL_QRX *qrx) -{ - QUIC_URXE *e; - - while ((e = ossl_list_urxe_head(&qrx->urx_deferred)) != NULL) { - ossl_list_urxe_remove(&qrx->urx_deferred, e); - ossl_list_urxe_insert_tail(&qrx->urx_pending, e); - } -} - -int ossl_qrx_provide_secret(OSSL_QRX *qrx, uint32_t enc_level, - uint32_t suite_id, EVP_MD *md, - const unsigned char *secret, size_t secret_len) -{ - if (enc_level >= QUIC_ENC_LEVEL_NUM) - return 0; - - if (!ossl_qrl_enc_level_set_provide_secret(&qrx->el_set, - qrx->libctx, - qrx->propq, - enc_level, - suite_id, - md, - secret, - secret_len, - qrx->init_key_phase_bit, - /*is_tx=*/0)) - return 0; - - /* - * Any packets we previously could not decrypt, we may now be able to - * decrypt, so move any datagrams containing deferred packets from the - * deferred to the pending queue. - */ - qrx_requeue_deferred(qrx); - return 1; -} - -int ossl_qrx_discard_enc_level(OSSL_QRX *qrx, uint32_t enc_level) -{ - if (enc_level >= QUIC_ENC_LEVEL_NUM) - return 0; - - ossl_qrl_enc_level_set_discard(&qrx->el_set, enc_level); - return 1; -} - -/* Returns 1 if there are one or more pending RXEs. */ -int ossl_qrx_processed_read_pending(OSSL_QRX *qrx) -{ - return !ossl_list_rxe_is_empty(&qrx->rx_pending); -} - -/* Returns 1 if there are yet-unprocessed packets. */ -int ossl_qrx_unprocessed_read_pending(OSSL_QRX *qrx) -{ - return !ossl_list_urxe_is_empty(&qrx->urx_pending) - || !ossl_list_urxe_is_empty(&qrx->urx_deferred); -} - -/* Pop the next pending RXE. Returns NULL if no RXE is pending. */ -static RXE *qrx_pop_pending_rxe(OSSL_QRX *qrx) -{ - RXE *rxe = ossl_list_rxe_head(&qrx->rx_pending); - - if (rxe == NULL) - return NULL; - - ossl_list_rxe_remove(&qrx->rx_pending, rxe); - return rxe; -} - -/* Allocate a new RXE. */ -static RXE *qrx_alloc_rxe(size_t alloc_len) -{ - RXE *rxe; - - if (alloc_len >= SIZE_MAX - sizeof(RXE)) - return NULL; - - rxe = OPENSSL_malloc(sizeof(RXE) + alloc_len); - if (rxe == NULL) - return NULL; - - ossl_list_rxe_init_elem(rxe); - rxe->alloc_len = alloc_len; - rxe->data_len = 0; - rxe->refcount = 0; - return rxe; -} - -/* - * Ensures there is at least one RXE in the RX free list, allocating a new entry - * if necessary. The returned RXE is in the RX free list; it is not popped. - * - * alloc_len is a hint which may be used to determine the RXE size if allocation - * is necessary. Returns NULL on allocation failure. - */ -static RXE *qrx_ensure_free_rxe(OSSL_QRX *qrx, size_t alloc_len) -{ - RXE *rxe; - - if (ossl_list_rxe_head(&qrx->rx_free) != NULL) - return ossl_list_rxe_head(&qrx->rx_free); - - rxe = qrx_alloc_rxe(alloc_len); - if (rxe == NULL) - return NULL; - - ossl_list_rxe_insert_tail(&qrx->rx_free, rxe); - return rxe; -} - -/* - * Resize the data buffer attached to an RXE to be n bytes in size. The address - * of the RXE might change; the new address is returned, or NULL on failure, in - * which case the original RXE remains valid. - */ -static RXE *qrx_resize_rxe(RXE_LIST *rxl, RXE *rxe, size_t n) -{ - RXE *rxe2, *p; - - /* Should never happen. */ - if (rxe == NULL) - return NULL; - - if (n >= SIZE_MAX - sizeof(RXE)) - return NULL; - - /* Remove the item from the list to avoid accessing freed memory */ - p = ossl_list_rxe_prev(rxe); - ossl_list_rxe_remove(rxl, rxe); - - /* Should never resize an RXE which has been handed out. */ - if (!ossl_assert(rxe->refcount == 0)) - return NULL; - - /* - * NOTE: We do not clear old memory, although it does contain decrypted - * data. - */ - rxe2 = OPENSSL_realloc(rxe, sizeof(RXE) + n); - if (rxe2 == NULL) { - /* Resize failed, restore old allocation. */ - if (p == NULL) - ossl_list_rxe_insert_head(rxl, rxe); - else - ossl_list_rxe_insert_after(rxl, p, rxe); - return NULL; - } - - if (p == NULL) - ossl_list_rxe_insert_head(rxl, rxe2); - else - ossl_list_rxe_insert_after(rxl, p, rxe2); - - rxe2->alloc_len = n; - return rxe2; -} - -/* - * Ensure the data buffer attached to an RXE is at least n bytes in size. - * Returns NULL on failure. - */ -static RXE *qrx_reserve_rxe(RXE_LIST *rxl, - RXE *rxe, size_t n) -{ - if (rxe->alloc_len >= n) - return rxe; - - return qrx_resize_rxe(rxl, rxe, n); -} - -/* Return a RXE handed out to the user back to our freelist. */ -static void qrx_recycle_rxe(OSSL_QRX *qrx, RXE *rxe) -{ - /* RXE should not be in any list */ - assert(ossl_list_rxe_prev(rxe) == NULL && ossl_list_rxe_next(rxe) == NULL); - rxe->pkt.hdr = NULL; - rxe->pkt.peer = NULL; - rxe->pkt.local = NULL; - ossl_list_rxe_insert_tail(&qrx->rx_free, rxe); -} - -/* - * Given a pointer to a pointer pointing to a buffer and the size of that - * buffer, copy the buffer into *prxe, expanding the RXE if necessary (its - * pointer may change due to realloc). *pi is the offset in bytes to copy the - * buffer to, and on success is updated to be the offset pointing after the - * copied buffer. *pptr is updated to point to the new location of the buffer. - */ -static int qrx_relocate_buffer(OSSL_QRX *qrx, RXE **prxe, size_t *pi, - const unsigned char **pptr, size_t buf_len) -{ - RXE *rxe; - unsigned char *dst; - - if (!buf_len) - return 1; - - if ((rxe = qrx_reserve_rxe(&qrx->rx_free, *prxe, *pi + buf_len)) == NULL) - return 0; - - *prxe = rxe; - dst = (unsigned char *)rxe_data(rxe) + *pi; - - memcpy(dst, *pptr, buf_len); - *pi += buf_len; - *pptr = dst; - return 1; -} - -static uint32_t qrx_determine_enc_level(const QUIC_PKT_HDR *hdr) -{ - switch (hdr->type) { - case QUIC_PKT_TYPE_INITIAL: - return QUIC_ENC_LEVEL_INITIAL; - case QUIC_PKT_TYPE_HANDSHAKE: - return QUIC_ENC_LEVEL_HANDSHAKE; - case QUIC_PKT_TYPE_0RTT: - return QUIC_ENC_LEVEL_0RTT; - case QUIC_PKT_TYPE_1RTT: - return QUIC_ENC_LEVEL_1RTT; - - default: - assert(0); - case QUIC_PKT_TYPE_RETRY: - case QUIC_PKT_TYPE_VERSION_NEG: - return QUIC_ENC_LEVEL_INITIAL; /* not used */ - } -} - -static uint32_t rxe_determine_pn_space(RXE *rxe) -{ - uint32_t enc_level; - - enc_level = qrx_determine_enc_level(&rxe->hdr); - return ossl_quic_enc_level_to_pn_space(enc_level); -} - -static int qrx_validate_hdr_early(OSSL_QRX *qrx, RXE *rxe, - const QUIC_CONN_ID *first_dcid) -{ - /* Ensure version is what we want. */ - if (rxe->hdr.version != QUIC_VERSION_1 - && rxe->hdr.version != QUIC_VERSION_NONE) - return 0; - - /* Clients should never receive 0-RTT packets. */ - if (rxe->hdr.type == QUIC_PKT_TYPE_0RTT) - return 0; - - /* Version negotiation and retry packets must be the first packet. */ - if (first_dcid != NULL && !ossl_quic_pkt_type_can_share_dgram(rxe->hdr.type)) - return 0; - - /* - * If this is not the first packet in a datagram, the destination connection - * ID must match the one in that packet. - */ - if (first_dcid != NULL) { - if (!ossl_assert(first_dcid->id_len < QUIC_MAX_CONN_ID_LEN) - || !ossl_quic_conn_id_eq(first_dcid, - &rxe->hdr.dst_conn_id)) - return 0; - } - - return 1; -} - -/* Validate header and decode PN. */ -static int qrx_validate_hdr(OSSL_QRX *qrx, RXE *rxe) -{ - int pn_space = rxe_determine_pn_space(rxe); - - if (!ossl_quic_wire_decode_pkt_hdr_pn(rxe->hdr.pn, rxe->hdr.pn_len, - qrx->largest_pn[pn_space], - &rxe->pn)) - return 0; - - return 1; -} - -/* Late packet header validation. */ -static int qrx_validate_hdr_late(OSSL_QRX *qrx, RXE *rxe) -{ - int pn_space = rxe_determine_pn_space(rxe); - - /* - * Allow our user to decide whether to discard the packet before we try and - * decrypt it. - */ - if (qrx->validation_cb != NULL - && !qrx->validation_cb(rxe->pn, pn_space, qrx->validation_cb_arg)) - return 0; - - return 1; -} - -/* - * Retrieves the correct cipher context for an EL and key phase. Writes the key - * epoch number actually used for packet decryption to *rx_key_epoch. - */ -static size_t qrx_get_cipher_ctx_idx(OSSL_QRX *qrx, OSSL_QRL_ENC_LEVEL *el, - uint32_t enc_level, - unsigned char key_phase_bit, - uint64_t *rx_key_epoch, - int *is_old_key) -{ - size_t idx; - - *is_old_key = 0; - - if (enc_level != QUIC_ENC_LEVEL_1RTT) { - *rx_key_epoch = 0; - return 0; - } - - if (!ossl_assert(key_phase_bit <= 1)) - return SIZE_MAX; - - /* - * RFC 9001 requires that we not create timing channels which could reveal - * the decrypted value of the Key Phase bit. We usually handle this by - * keeping the cipher contexts for both the current and next key epochs - * around, so that we just select a cipher context blindly using the key - * phase bit, which is time-invariant. - * - * In the COOLDOWN state, we only have one keyslot/cipher context. RFC 9001 - * suggests an implementation strategy to avoid creating a timing channel in - * this case: - * - * Endpoints can use randomized packet protection keys in place of - * discarded keys when key updates are not yet permitted. - * - * Rather than use a randomised key, we simply use our existing key as it - * will fail AEAD verification anyway. This avoids the need to keep around a - * dedicated garbage key. - * - * Note: Accessing different cipher contexts is technically not - * timing-channel safe due to microarchitectural side channels, but this is - * the best we can reasonably do and appears to be directly suggested by the - * RFC. - */ - idx = (el->state == QRL_EL_STATE_PROV_COOLDOWN ? el->key_epoch & 1 - : key_phase_bit); - - /* - * We also need to determine the key epoch number which this index - * corresponds to. This is so we can report the key epoch number in the - * OSSL_QRX_PKT structure, which callers need to validate whether it was OK - * for a packet to be sent using a given key epoch's keys. - */ - switch (el->state) { - case QRL_EL_STATE_PROV_NORMAL: - /* - * If we are in the NORMAL state, usually the KP bit will match the LSB - * of our key epoch, meaning no new key update is being signalled. If it - * does not match, this means the packet (purports to) belong to - * the next key epoch. - * - * IMPORTANT: The AEAD tag has not been verified yet when this function - * is called, so this code must be timing-channel safe, hence use of - * XOR. Moreover, the value output below is not yet authenticated. - */ - *rx_key_epoch - = el->key_epoch + ((el->key_epoch & 1) ^ (uint64_t)key_phase_bit); - break; - - case QRL_EL_STATE_PROV_UPDATING: - /* - * If we are in the UPDATING state, usually the KP bit will match the - * LSB of our key epoch. If it does not match, this means that the - * packet (purports to) belong to the previous key epoch. - * - * As above, must be timing-channel safe. - */ - *is_old_key = (el->key_epoch & 1) ^ (uint64_t)key_phase_bit; - *rx_key_epoch = el->key_epoch - (uint64_t)*is_old_key; - break; - - case QRL_EL_STATE_PROV_COOLDOWN: - /* - * If we are in COOLDOWN, there is only one key epoch we can possibly - * decrypt with, so just try that. If AEAD decryption fails, the - * value we output here isn't used anyway. - */ - *rx_key_epoch = el->key_epoch; - break; - } - - return idx; -} - -/* - * Tries to decrypt a packet payload. - * - * Returns 1 on success or 0 on failure (which is permanent). The payload is - * decrypted from src and written to dst. The buffer dst must be of at least - * src_len bytes in length. The actual length of the output in bytes is written - * to *dec_len on success, which will always be equal to or less than (usually - * less than) src_len. - */ -static int qrx_decrypt_pkt_body(OSSL_QRX *qrx, unsigned char *dst, - const unsigned char *src, - size_t src_len, size_t *dec_len, - const unsigned char *aad, size_t aad_len, - QUIC_PN pn, uint32_t enc_level, - unsigned char key_phase_bit, - uint64_t *rx_key_epoch) -{ - int l = 0, l2 = 0, is_old_key, nonce_len; - unsigned char nonce[EVP_MAX_IV_LENGTH]; - size_t i, cctx_idx; - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(&qrx->el_set, - enc_level, 1); - EVP_CIPHER_CTX *cctx; - - if (src_len > INT_MAX || aad_len > INT_MAX) - return 0; - - /* We should not have been called if we do not have key material. */ - if (!ossl_assert(el != NULL)) - return 0; - - if (el->tag_len >= src_len) - return 0; - - /* - * If we have failed to authenticate a certain number of ciphertexts, refuse - * to decrypt any more ciphertexts. - */ - if (qrx->forged_pkt_count >= ossl_qrl_get_suite_max_forged_pkt(el->suite_id)) - return 0; - - cctx_idx = qrx_get_cipher_ctx_idx(qrx, el, enc_level, key_phase_bit, - rx_key_epoch, &is_old_key); - if (!ossl_assert(cctx_idx < OSSL_NELEM(el->cctx))) - return 0; - - if (is_old_key && pn >= qrx->cur_epoch_start_pn) - /* - * RFC 9001 s. 5.5: Once an endpoint successfully receives a packet with - * a given PN, it MUST discard all packets in the same PN space with - * higher PNs if they cannot be successfully unprotected with the same - * key, or -- if there is a key update -- a subsequent packet protection - * key. - * - * In other words, once a PN x triggers a KU, it is invalid for us to - * receive a packet with a newer PN y (y > x) using the old keys. - */ - return 0; - - cctx = el->cctx[cctx_idx]; - - /* Construct nonce (nonce=IV ^ PN). */ - nonce_len = EVP_CIPHER_CTX_get_iv_length(cctx); - if (!ossl_assert(nonce_len >= (int)sizeof(QUIC_PN))) - return 0; - - memcpy(nonce, el->iv[cctx_idx], nonce_len); - for (i = 0; i < sizeof(QUIC_PN); ++i) - nonce[nonce_len - i - 1] ^= (unsigned char)(pn >> (i * 8)); - - /* type and key will already have been setup; feed the IV. */ - if (EVP_CipherInit_ex(cctx, NULL, - NULL, NULL, nonce, /*enc=*/0) != 1) - return 0; - - /* Feed the AEAD tag we got so the cipher can validate it. */ - if (EVP_CIPHER_CTX_ctrl(cctx, EVP_CTRL_AEAD_SET_TAG, - el->tag_len, - (unsigned char *)src + src_len - el->tag_len) != 1) - return 0; - - /* Feed AAD data. */ - if (EVP_CipherUpdate(cctx, NULL, &l, aad, aad_len) != 1) - return 0; - - /* Feed encrypted packet body. */ - if (EVP_CipherUpdate(cctx, dst, &l, src, src_len - el->tag_len) != 1) - return 0; - -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - /* - * Throw away what we just decrypted and just use the ciphertext instead - * (which should be unencrypted) - */ - memcpy(dst, src, l); - - /* Pretend to authenticate the tag but ignore it */ - if (EVP_CipherFinal_ex(cctx, NULL, &l2) != 1) { - /* We don't care */ - } -#else - /* Ensure authentication succeeded. */ - if (EVP_CipherFinal_ex(cctx, NULL, &l2) != 1) { - /* Authentication failed, increment failed auth counter. */ - ++qrx->forged_pkt_count; - return 0; - } -#endif - - *dec_len = l; - return 1; -} - -static ossl_inline void ignore_res(int x) -{ - /* No-op. */ -} - -static void qrx_key_update_initiated(OSSL_QRX *qrx, QUIC_PN pn) -{ - if (!ossl_qrl_enc_level_set_key_update(&qrx->el_set, QUIC_ENC_LEVEL_1RTT)) - /* We are already in RXKU, so we don't call the callback again. */ - return; - - qrx->cur_epoch_start_pn = pn; - - if (qrx->key_update_cb != NULL) - qrx->key_update_cb(pn, qrx->key_update_cb_arg); -} - -/* Process a single packet in a datagram. */ -static int qrx_process_pkt(OSSL_QRX *qrx, QUIC_URXE *urxe, - PACKET *pkt, size_t pkt_idx, - QUIC_CONN_ID *first_dcid, - size_t datagram_len) -{ - RXE *rxe; - const unsigned char *eop = NULL; - size_t i, aad_len = 0, dec_len = 0; - PACKET orig_pkt = *pkt; - const unsigned char *sop = PACKET_data(pkt); - unsigned char *dst; - char need_second_decode = 0, already_processed = 0; - QUIC_PKT_HDR_PTRS ptrs; - uint32_t pn_space, enc_level; - OSSL_QRL_ENC_LEVEL *el = NULL; - uint64_t rx_key_epoch = UINT64_MAX; - - /* - * Get a free RXE. If we need to allocate a new one, use the packet length - * as a good ballpark figure. - */ - rxe = qrx_ensure_free_rxe(qrx, PACKET_remaining(pkt)); - if (rxe == NULL) - return 0; - - /* Have we already processed this packet? */ - if (pkt_is_marked(&urxe->processed, pkt_idx)) - already_processed = 1; - - /* - * Decode the header into the RXE structure. We first decrypt and read the - * unprotected part of the packet header (unless we already removed header - * protection, in which case we decode all of it). - */ - need_second_decode = !pkt_is_marked(&urxe->hpr_removed, pkt_idx); - if (!ossl_quic_wire_decode_pkt_hdr(pkt, - qrx->short_conn_id_len, - need_second_decode, 0, &rxe->hdr, &ptrs)) - goto malformed; - - /* - * Our successful decode above included an intelligible length and the - * PACKET is now pointing to the end of the QUIC packet. - */ - eop = PACKET_data(pkt); - - /* - * Make a note of the first packet's DCID so we can later ensure the - * destination connection IDs of all packets in a datagram match. - */ - if (pkt_idx == 0) - *first_dcid = rxe->hdr.dst_conn_id; - - /* - * Early header validation. Since we now know the packet length, we can also - * now skip over it if we already processed it. - */ - if (already_processed - || !qrx_validate_hdr_early(qrx, rxe, pkt_idx == 0 ? NULL : first_dcid)) - /* - * Already processed packets are handled identically to malformed - * packets; i.e., they are ignored. - */ - goto malformed; - - if (!ossl_quic_pkt_type_is_encrypted(rxe->hdr.type)) { - /* - * Version negotiation and retry packets are a special case. They do not - * contain a payload which needs decrypting and have no header - * protection. - */ - - /* Just copy the payload from the URXE to the RXE. */ - if ((rxe = qrx_reserve_rxe(&qrx->rx_free, rxe, rxe->hdr.len)) == NULL) - /* - * Allocation failure. EOP will be pointing to the end of the - * datagram so processing of this datagram will end here. - */ - goto malformed; - - /* We are now committed to returning the packet. */ - memcpy(rxe_data(rxe), rxe->hdr.data, rxe->hdr.len); - pkt_mark(&urxe->processed, pkt_idx); - - rxe->hdr.data = rxe_data(rxe); - rxe->pn = QUIC_PN_INVALID; - - rxe->data_len = rxe->hdr.len; - rxe->datagram_len = datagram_len; - rxe->key_epoch = 0; - rxe->peer = urxe->peer; - rxe->local = urxe->local; - rxe->time = urxe->time; - rxe->datagram_id = urxe->datagram_id; - - /* Move RXE to pending. */ - ossl_list_rxe_remove(&qrx->rx_free, rxe); - ossl_list_rxe_insert_tail(&qrx->rx_pending, rxe); - return 0; /* success, did not defer */ - } - - /* Determine encryption level of packet. */ - enc_level = qrx_determine_enc_level(&rxe->hdr); - - /* If we do not have keying material for this encryption level yet, defer. */ - switch (ossl_qrl_enc_level_set_have_el(&qrx->el_set, enc_level)) { - case 1: - /* We have keys. */ - if (enc_level == QUIC_ENC_LEVEL_1RTT && !qrx->allow_1rtt) - /* - * But we cannot process 1-RTT packets until the handshake is - * completed (RFC 9000 s. 5.7). - */ - goto cannot_decrypt; - - break; - case 0: - /* No keys yet. */ - goto cannot_decrypt; - default: - /* We already discarded keys for this EL, we will never process this.*/ - goto malformed; - } - - /* - * We will copy any token included in the packet to the start of our RXE - * data buffer (so that we don't reference the URXE buffer any more and can - * recycle it). Track our position in the RXE buffer by index instead of - * pointer as the pointer may change as reallocs occur. - */ - i = 0; - - /* - * rxe->hdr.data is now pointing at the (encrypted) packet payload. rxe->hdr - * also has fields pointing into the PACKET buffer which will be going away - * soon (the URXE will be reused for another incoming packet). - * - * Firstly, relocate some of these fields into the RXE as needed. - * - * Relocate token buffer and fix pointer. - */ - if (rxe->hdr.type == QUIC_PKT_TYPE_INITIAL) { - const unsigned char *token = rxe->hdr.token; - - /* - * This may change the value of rxe and change the value of the token - * pointer as well. So we must make a temporary copy of the pointer to - * the token, and then copy it back into the new location of the rxe - */ - if (!qrx_relocate_buffer(qrx, &rxe, &i, &token, rxe->hdr.token_len)) - goto malformed; - - rxe->hdr.token = token; - } - - /* Now remove header protection. */ - *pkt = orig_pkt; - - el = ossl_qrl_enc_level_set_get(&qrx->el_set, enc_level, 1); - assert(el != NULL); /* Already checked above */ - - if (need_second_decode) { - if (!ossl_quic_hdr_protector_decrypt(&el->hpr, &ptrs)) - goto malformed; - - /* - * We have removed header protection, so don't attempt to do it again if - * the packet gets deferred and processed again. - */ - pkt_mark(&urxe->hpr_removed, pkt_idx); - - /* Decode the now unprotected header. */ - if (ossl_quic_wire_decode_pkt_hdr(pkt, qrx->short_conn_id_len, - 0, 0, &rxe->hdr, NULL) != 1) - goto malformed; - } - - /* Validate header and decode PN. */ - if (!qrx_validate_hdr(qrx, rxe)) - goto malformed; - - if (qrx->msg_callback != NULL) - qrx->msg_callback(0, OSSL_QUIC1_VERSION, SSL3_RT_QUIC_PACKET, sop, - eop - sop - rxe->hdr.len, qrx->msg_callback_ssl, - qrx->msg_callback_arg); - - /* - * The AAD data is the entire (unprotected) packet header including the PN. - * The packet header has been unprotected in place, so we can just reuse the - * PACKET buffer. The header ends where the payload begins. - */ - aad_len = rxe->hdr.data - sop; - - /* Ensure the RXE buffer size is adequate for our payload. */ - if ((rxe = qrx_reserve_rxe(&qrx->rx_free, rxe, rxe->hdr.len + i)) == NULL) { - /* - * Allocation failure, treat as malformed and do not bother processing - * any further packets in the datagram as they are likely to also - * encounter allocation failures. - */ - eop = NULL; - goto malformed; - } - - /* - * We decrypt the packet body to immediately after the token at the start of - * the RXE buffer (where present). - * - * Do the decryption from the PACKET (which points into URXE memory) to our - * RXE payload (single-copy decryption), then fixup the pointers in the - * header to point to our new buffer. - * - * If decryption fails this is considered a permanent error; we defer - * packets we don't yet have decryption keys for above, so if this fails, - * something has gone wrong with the handshake process or a packet has been - * corrupted. - */ - dst = (unsigned char *)rxe_data(rxe) + i; - if (!qrx_decrypt_pkt_body(qrx, dst, rxe->hdr.data, rxe->hdr.len, - &dec_len, sop, aad_len, rxe->pn, enc_level, - rxe->hdr.key_phase, &rx_key_epoch)) - goto malformed; - - /* - * ----------------------------------------------------- - * IMPORTANT: ANYTHING ABOVE THIS LINE IS UNVERIFIED - * AND MUST BE TIMING-CHANNEL SAFE. - * ----------------------------------------------------- - * - * At this point, we have successfully authenticated the AEAD tag and no - * longer need to worry about exposing the PN, PN length or Key Phase bit in - * timing channels. Invoke any configured validation callback to allow for - * rejection of duplicate PNs. - */ - if (!qrx_validate_hdr_late(qrx, rxe)) - goto malformed; - - /* Check for a Key Phase bit differing from our expectation. */ - if (rxe->hdr.type == QUIC_PKT_TYPE_1RTT - && rxe->hdr.key_phase != (el->key_epoch & 1)) - qrx_key_update_initiated(qrx, rxe->pn); - - /* - * We have now successfully decrypted the packet payload. If there are - * additional packets in the datagram, it is possible we will fail to - * decrypt them and need to defer them until we have some key material we - * don't currently possess. If this happens, the URXE will be moved to the - * deferred queue. Since a URXE corresponds to one datagram, which may - * contain multiple packets, we must ensure any packets we have already - * processed in the URXE are not processed again (this is an RFC - * requirement). We do this by marking the nth packet in the datagram as - * processed. - * - * We are now committed to returning this decrypted packet to the user, - * meaning we now consider the packet processed and must mark it - * accordingly. - */ - pkt_mark(&urxe->processed, pkt_idx); - - /* - * Update header to point to the decrypted buffer, which may be shorter - * due to AEAD tags, block padding, etc. - */ - rxe->hdr.data = dst; - rxe->hdr.len = dec_len; - rxe->data_len = dec_len; - rxe->datagram_len = datagram_len; - rxe->key_epoch = rx_key_epoch; - - /* We processed the PN successfully, so update largest processed PN. */ - pn_space = rxe_determine_pn_space(rxe); - if (rxe->pn > qrx->largest_pn[pn_space]) - qrx->largest_pn[pn_space] = rxe->pn; - - /* Copy across network addresses and RX time from URXE to RXE. */ - rxe->peer = urxe->peer; - rxe->local = urxe->local; - rxe->time = urxe->time; - rxe->datagram_id = urxe->datagram_id; - - /* Move RXE to pending. */ - ossl_list_rxe_remove(&qrx->rx_free, rxe); - ossl_list_rxe_insert_tail(&qrx->rx_pending, rxe); - return 0; /* success, did not defer; not distinguished from failure */ - -cannot_decrypt: - /* - * We cannot process this packet right now (but might be able to later). We - * MUST attempt to process any other packets in the datagram, so defer it - * and skip over it. - */ - assert(eop != NULL && eop >= PACKET_data(pkt)); - /* - * We don't care if this fails as it will just result in the packet being at - * the end of the datagram buffer. - */ - ignore_res(PACKET_forward(pkt, eop - PACKET_data(pkt))); - return 1; /* deferred */ - -malformed: - if (eop != NULL) { - /* - * This packet cannot be processed and will never be processable. We - * were at least able to decode its header and determine its length, so - * we can skip over it and try to process any subsequent packets in the - * datagram. - * - * Mark as processed as an optimization. - */ - assert(eop >= PACKET_data(pkt)); - pkt_mark(&urxe->processed, pkt_idx); - /* We don't care if this fails (see above) */ - ignore_res(PACKET_forward(pkt, eop - PACKET_data(pkt))); - } else { - /* - * This packet cannot be processed and will never be processable. - * Because even its header is not intelligible, we cannot examine any - * further packets in the datagram because its length cannot be - * discerned. - * - * Advance over the entire remainder of the datagram, and mark it as - * processed as an optimization. - */ - pkt_mark(&urxe->processed, pkt_idx); - /* We don't care if this fails (see above) */ - ignore_res(PACKET_forward(pkt, PACKET_remaining(pkt))); - } - return 0; /* failure, did not defer; not distinguished from success */ -} - -/* Process a datagram which was received. */ -static int qrx_process_datagram(OSSL_QRX *qrx, QUIC_URXE *e, - const unsigned char *data, - size_t data_len) -{ - int have_deferred = 0; - PACKET pkt; - size_t pkt_idx = 0; - QUIC_CONN_ID first_dcid = { 255 }; - - qrx->bytes_received += data_len; - - if (!PACKET_buf_init(&pkt, data, data_len)) - return 0; - - for (; PACKET_remaining(&pkt) > 0; ++pkt_idx) { - /* - * A packet smaller than the minimum possible QUIC packet size is not - * considered valid. We also ignore more than a certain number of - * packets within the same datagram. - */ - if (PACKET_remaining(&pkt) < QUIC_MIN_VALID_PKT_LEN - || pkt_idx >= QUIC_MAX_PKT_PER_URXE) - break; - - /* - * We note whether packet processing resulted in a deferral since - * this means we need to move the URXE to the deferred list rather - * than the free list after we're finished dealing with it for now. - * - * However, we don't otherwise care here whether processing succeeded or - * failed, as the RFC says even if a packet in a datagram is malformed, - * we should still try to process any packets following it. - * - * In the case where the packet is so malformed we can't determine its - * length, qrx_process_pkt will take care of advancing to the end of - * the packet, so we will exit the loop automatically in this case. - */ - if (qrx_process_pkt(qrx, e, &pkt, pkt_idx, &first_dcid, data_len)) - have_deferred = 1; - } - - /* Only report whether there were any deferrals. */ - return have_deferred; -} - -/* Process a single pending URXE. */ -static int qrx_process_one_urxe(OSSL_QRX *qrx, QUIC_URXE *e) -{ - int was_deferred; - - /* The next URXE we process should be at the head of the pending list. */ - if (!ossl_assert(e == ossl_list_urxe_head(&qrx->urx_pending))) - return 0; - - /* - * Attempt to process the datagram. The return value indicates only if - * processing of the datagram was deferred. If we failed to process the - * datagram, we do not attempt to process it again and silently eat the - * error. - */ - was_deferred = qrx_process_datagram(qrx, e, ossl_quic_urxe_data(e), - e->data_len); - - /* - * Remove the URXE from the pending list and return it to - * either the free or deferred list. - */ - ossl_list_urxe_remove(&qrx->urx_pending, e); - if (was_deferred > 0 && - (e->deferred || qrx->num_deferred < qrx->max_deferred)) { - ossl_list_urxe_insert_tail(&qrx->urx_deferred, e); - if (!e->deferred) { - e->deferred = 1; - ++qrx->num_deferred; - } - } else { - if (e->deferred) { - e->deferred = 0; - --qrx->num_deferred; - } - ossl_quic_demux_release_urxe(qrx->demux, e); - } - - return 1; -} - -/* Process any pending URXEs to generate pending RXEs. */ -static int qrx_process_pending_urxl(OSSL_QRX *qrx) -{ - QUIC_URXE *e; - - while ((e = ossl_list_urxe_head(&qrx->urx_pending)) != NULL) - if (!qrx_process_one_urxe(qrx, e)) - return 0; - - return 1; -} - -int ossl_qrx_read_pkt(OSSL_QRX *qrx, OSSL_QRX_PKT **ppkt) -{ - RXE *rxe; - - if (!ossl_qrx_processed_read_pending(qrx)) { - if (!qrx_process_pending_urxl(qrx)) - return 0; - - if (!ossl_qrx_processed_read_pending(qrx)) - return 0; - } - - rxe = qrx_pop_pending_rxe(qrx); - if (!ossl_assert(rxe != NULL)) - return 0; - - assert(rxe->refcount == 0); - rxe->refcount = 1; - - rxe->pkt.hdr = &rxe->hdr; - rxe->pkt.pn = rxe->pn; - rxe->pkt.time = rxe->time; - rxe->pkt.datagram_len = rxe->datagram_len; - rxe->pkt.peer - = BIO_ADDR_family(&rxe->peer) != AF_UNSPEC ? &rxe->peer : NULL; - rxe->pkt.local - = BIO_ADDR_family(&rxe->local) != AF_UNSPEC ? &rxe->local : NULL; - rxe->pkt.key_epoch = rxe->key_epoch; - rxe->pkt.datagram_id = rxe->datagram_id; - rxe->pkt.qrx = qrx; - *ppkt = &rxe->pkt; - - return 1; -} - -void ossl_qrx_pkt_release(OSSL_QRX_PKT *pkt) -{ - RXE *rxe; - - if (pkt == NULL) - return; - - rxe = (RXE *)pkt; - assert(rxe->refcount > 0); - if (--rxe->refcount == 0) - qrx_recycle_rxe(pkt->qrx, rxe); -} - -void ossl_qrx_pkt_up_ref(OSSL_QRX_PKT *pkt) -{ - RXE *rxe = (RXE *)pkt; - - assert(rxe->refcount > 0); - ++rxe->refcount; -} - -uint64_t ossl_qrx_get_bytes_received(OSSL_QRX *qrx, int clear) -{ - uint64_t v = qrx->bytes_received; - - if (clear) - qrx->bytes_received = 0; - - return v; -} - -int ossl_qrx_set_late_validation_cb(OSSL_QRX *qrx, - ossl_qrx_late_validation_cb *cb, - void *cb_arg) -{ - qrx->validation_cb = cb; - qrx->validation_cb_arg = cb_arg; - return 1; -} - -int ossl_qrx_set_key_update_cb(OSSL_QRX *qrx, - ossl_qrx_key_update_cb *cb, - void *cb_arg) -{ - qrx->key_update_cb = cb; - qrx->key_update_cb_arg = cb_arg; - return 1; -} - -uint64_t ossl_qrx_get_key_epoch(OSSL_QRX *qrx) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(&qrx->el_set, - QUIC_ENC_LEVEL_1RTT, 1); - - return el == NULL ? UINT64_MAX : el->key_epoch; -} - -int ossl_qrx_key_update_timeout(OSSL_QRX *qrx, int normal) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(&qrx->el_set, - QUIC_ENC_LEVEL_1RTT, 1); - - if (el == NULL) - return 0; - - if (el->state == QRL_EL_STATE_PROV_UPDATING - && !ossl_qrl_enc_level_set_key_update_done(&qrx->el_set, - QUIC_ENC_LEVEL_1RTT)) - return 0; - - if (normal && el->state == QRL_EL_STATE_PROV_COOLDOWN - && !ossl_qrl_enc_level_set_key_cooldown_done(&qrx->el_set, - QUIC_ENC_LEVEL_1RTT)) - return 0; - - return 1; -} - -uint64_t ossl_qrx_get_cur_forged_pkt_count(OSSL_QRX *qrx) -{ - return qrx->forged_pkt_count; -} - -uint64_t ossl_qrx_get_max_forged_pkt_count(OSSL_QRX *qrx, - uint32_t enc_level) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(&qrx->el_set, - enc_level, 1); - - return el == NULL ? UINT64_MAX - : ossl_qrl_get_suite_max_forged_pkt(el->suite_id); -} - -void ossl_qrx_allow_1rtt_processing(OSSL_QRX *qrx) -{ - if (qrx->allow_1rtt) - return; - - qrx->allow_1rtt = 1; - qrx_requeue_deferred(qrx); -} - -void ossl_qrx_set_msg_callback(OSSL_QRX *qrx, ossl_msg_cb msg_callback, - SSL *msg_callback_ssl) -{ - qrx->msg_callback = msg_callback; - qrx->msg_callback_ssl = msg_callback_ssl; -} - -void ossl_qrx_set_msg_callback_arg(OSSL_QRX *qrx, void *msg_callback_arg) -{ - qrx->msg_callback_arg = msg_callback_arg; -} diff --git a/openssl/src/ssl/quic/quic_record_shared.c b/openssl/src/ssl/quic/quic_record_shared.c deleted file mode 100644 index a3fd51db6..000000000 --- a/openssl/src/ssl/quic/quic_record_shared.c +++ /dev/null @@ -1,489 +0,0 @@ -#include "quic_record_shared.h" -#include "internal/quic_record_util.h" -#include "internal/common.h" -#include "../ssl_local.h" - -/* Constants used for key derivation in QUIC v1. */ -static const unsigned char quic_v1_iv_label[] = { - 0x71, 0x75, 0x69, 0x63, 0x20, 0x69, 0x76 /* "quic iv" */ -}; -static const unsigned char quic_v1_key_label[] = { - 0x71, 0x75, 0x69, 0x63, 0x20, 0x6b, 0x65, 0x79 /* "quic key" */ -}; -static const unsigned char quic_v1_hp_label[] = { - 0x71, 0x75, 0x69, 0x63, 0x20, 0x68, 0x70 /* "quic hp" */ -}; -static const unsigned char quic_v1_ku_label[] = { - 0x71, 0x75, 0x69, 0x63, 0x20, 0x6b, 0x75 /* "quic ku" */ -}; - -OSSL_QRL_ENC_LEVEL *ossl_qrl_enc_level_set_get(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level, - int require_prov) -{ - OSSL_QRL_ENC_LEVEL *el; - - if (!ossl_assert(enc_level < QUIC_ENC_LEVEL_NUM)) - return NULL; - - el = &els->el[enc_level]; - - if (require_prov) - switch (el->state) { - case QRL_EL_STATE_PROV_NORMAL: - case QRL_EL_STATE_PROV_UPDATING: - case QRL_EL_STATE_PROV_COOLDOWN: - break; - default: - return NULL; - } - - return el; -} - -int ossl_qrl_enc_level_set_have_el(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - - switch (el->state) { - case QRL_EL_STATE_UNPROV: - return 0; - case QRL_EL_STATE_PROV_NORMAL: - case QRL_EL_STATE_PROV_UPDATING: - case QRL_EL_STATE_PROV_COOLDOWN: - return 1; - default: - case QRL_EL_STATE_DISCARDED: - return -1; - } -} - -int ossl_qrl_enc_level_set_has_keyslot(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level, - unsigned char tgt_state, - size_t keyslot) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - - if (!ossl_assert(el != NULL && keyslot < 2)) - return 0; - - switch (tgt_state) { - case QRL_EL_STATE_PROV_NORMAL: - case QRL_EL_STATE_PROV_UPDATING: - return enc_level == QUIC_ENC_LEVEL_1RTT || keyslot == 0; - case QRL_EL_STATE_PROV_COOLDOWN: - assert(enc_level == QUIC_ENC_LEVEL_1RTT); - return keyslot == (el->key_epoch & 1); - default: - return 0; - } -} - -static void el_teardown_keyslot(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level, - size_t keyslot) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - - if (!ossl_qrl_enc_level_set_has_keyslot(els, enc_level, el->state, keyslot)) - return; - - if (el->cctx[keyslot] != NULL) { - EVP_CIPHER_CTX_free(el->cctx[keyslot]); - el->cctx[keyslot] = NULL; - } - - OPENSSL_cleanse(el->iv[keyslot], sizeof(el->iv[keyslot])); -} - -static int el_setup_keyslot(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level, - unsigned char tgt_state, - size_t keyslot, - const unsigned char *secret, - size_t secret_len) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - unsigned char key[EVP_MAX_KEY_LENGTH]; - size_t key_len = 0, iv_len = 0; - const char *cipher_name = NULL; - EVP_CIPHER *cipher = NULL; - EVP_CIPHER_CTX *cctx = NULL; - - if (!ossl_assert(el != NULL - && ossl_qrl_enc_level_set_has_keyslot(els, enc_level, - tgt_state, keyslot))) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - cipher_name = ossl_qrl_get_suite_cipher_name(el->suite_id); - iv_len = ossl_qrl_get_suite_cipher_iv_len(el->suite_id); - key_len = ossl_qrl_get_suite_cipher_key_len(el->suite_id); - if (cipher_name == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (secret_len != ossl_qrl_get_suite_secret_len(el->suite_id) - || secret_len > EVP_MAX_KEY_LENGTH) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - assert(el->cctx[keyslot] == NULL); - - /* Derive "quic iv" key. */ - if (!tls13_hkdf_expand_ex(el->libctx, el->propq, - el->md, - secret, - quic_v1_iv_label, - sizeof(quic_v1_iv_label), - NULL, 0, - el->iv[keyslot], iv_len, 1)) - goto err; - - /* Derive "quic key" key. */ - if (!tls13_hkdf_expand_ex(el->libctx, el->propq, - el->md, - secret, - quic_v1_key_label, - sizeof(quic_v1_key_label), - NULL, 0, - key, key_len, 1)) - goto err; - - /* Create and initialise cipher context. */ - if ((cipher = EVP_CIPHER_fetch(el->libctx, cipher_name, el->propq)) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - goto err; - } - - if ((cctx = EVP_CIPHER_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - goto err; - } - - if (!ossl_assert(iv_len == (size_t)EVP_CIPHER_get_iv_length(cipher)) - || !ossl_assert(key_len == (size_t)EVP_CIPHER_get_key_length(cipher))) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* IV will be changed on RX/TX so we don't need to use a real value here. */ - if (!EVP_CipherInit_ex(cctx, cipher, NULL, key, el->iv[keyslot], 0)) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - goto err; - } - - el->cctx[keyslot] = cctx; - - /* Zeroize intermediate keys. */ - OPENSSL_cleanse(key, sizeof(key)); - EVP_CIPHER_free(cipher); - return 1; - - err: - EVP_CIPHER_CTX_free(cctx); - EVP_CIPHER_free(cipher); - OPENSSL_cleanse(el->iv[keyslot], sizeof(el->iv[keyslot])); - OPENSSL_cleanse(key, sizeof(key)); - return 0; -} - -int ossl_qrl_enc_level_set_provide_secret(OSSL_QRL_ENC_LEVEL_SET *els, - OSSL_LIB_CTX *libctx, - const char *propq, - uint32_t enc_level, - uint32_t suite_id, - EVP_MD *md, - const unsigned char *secret, - size_t secret_len, - unsigned char init_key_phase_bit, - int is_tx) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - unsigned char ku_key[EVP_MAX_KEY_LENGTH], hpr_key[EVP_MAX_KEY_LENGTH]; - int have_ks0 = 0, have_ks1 = 0, own_md = 0; - const char *md_name = ossl_qrl_get_suite_md_name(suite_id); - size_t hpr_key_len, init_keyslot; - - if (el == NULL - || md_name == NULL - || init_key_phase_bit > 1 || is_tx < 0 || is_tx > 1 - || (init_key_phase_bit > 0 && enc_level != QUIC_ENC_LEVEL_1RTT)) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - if (enc_level == QUIC_ENC_LEVEL_INITIAL - && el->state == QRL_EL_STATE_PROV_NORMAL) { - /* - * Sometimes the INITIAL EL needs to be reprovisioned, namely if a - * connection retry occurs. Exceptionally, if the caller wants to - * reprovision the INITIAL EL, tear it down as usual and then override - * the state so it can be provisioned again. - */ - ossl_qrl_enc_level_set_discard(els, enc_level); - el->state = QRL_EL_STATE_UNPROV; - } - - if (el->state != QRL_EL_STATE_UNPROV) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - init_keyslot = is_tx ? 0 : init_key_phase_bit; - hpr_key_len = ossl_qrl_get_suite_hdr_prot_key_len(suite_id); - if (hpr_key_len == 0) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (md == NULL) { - md = EVP_MD_fetch(libctx, md_name, propq); - if (md == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - own_md = 1; - } - - el->libctx = libctx; - el->propq = propq; - el->md = md; - el->suite_id = suite_id; - el->tag_len = ossl_qrl_get_suite_cipher_tag_len(suite_id); - el->op_count = 0; - el->key_epoch = (uint64_t)init_key_phase_bit; - el->is_tx = (unsigned char)is_tx; - - /* Derive "quic hp" key. */ - if (!tls13_hkdf_expand_ex(libctx, propq, - md, - secret, - quic_v1_hp_label, - sizeof(quic_v1_hp_label), - NULL, 0, - hpr_key, hpr_key_len, 1)) - goto err; - - /* Setup KS0 (or KS1 if init_key_phase_bit), our initial keyslot. */ - if (!el_setup_keyslot(els, enc_level, QRL_EL_STATE_PROV_NORMAL, - init_keyslot, secret, secret_len)) - goto err; - - have_ks0 = 1; - - if (enc_level == QUIC_ENC_LEVEL_1RTT) { - /* Derive "quic ku" key (the epoch 1 secret). */ - if (!tls13_hkdf_expand_ex(libctx, propq, - md, - secret, - quic_v1_ku_label, - sizeof(quic_v1_ku_label), - NULL, 0, - is_tx ? el->ku : ku_key, secret_len, 1)) - goto err; - - if (!is_tx) { - /* Setup KS1 (or KS0 if init_key_phase_bit), our next keyslot. */ - if (!el_setup_keyslot(els, enc_level, QRL_EL_STATE_PROV_NORMAL, - !init_keyslot, ku_key, secret_len)) - goto err; - - have_ks1 = 1; - - /* Derive NEXT "quic ku" key (the epoch 2 secret). */ - if (!tls13_hkdf_expand_ex(libctx, propq, - md, - ku_key, - quic_v1_ku_label, - sizeof(quic_v1_ku_label), - NULL, 0, - el->ku, secret_len, 1)) - goto err; - } - } - - /* Setup header protection context. */ - if (!ossl_quic_hdr_protector_init(&el->hpr, - libctx, propq, - ossl_qrl_get_suite_hdr_prot_cipher_id(suite_id), - hpr_key, hpr_key_len)) - goto err; - - /* - * We are now provisioned: KS0 has our current key (for key epoch 0), KS1 - * has our next key (for key epoch 1, in the case of the 1-RTT EL only), and - * el->ku has the secret which will be used to generate keys for key epoch - * 2. - */ - OPENSSL_cleanse(hpr_key, sizeof(hpr_key)); - OPENSSL_cleanse(ku_key, sizeof(ku_key)); - el->state = QRL_EL_STATE_PROV_NORMAL; - return 1; - - err: - el->suite_id = 0; - el->md = NULL; - OPENSSL_cleanse(hpr_key, sizeof(hpr_key)); - OPENSSL_cleanse(ku_key, sizeof(ku_key)); - OPENSSL_cleanse(el->ku, sizeof(el->ku)); - if (have_ks0) - el_teardown_keyslot(els, enc_level, init_keyslot); - if (have_ks1) - el_teardown_keyslot(els, enc_level, !init_keyslot); - if (own_md) - EVP_MD_free(md); - return 0; -} - -int ossl_qrl_enc_level_set_key_update(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - size_t secret_len; - unsigned char new_ku[EVP_MAX_KEY_LENGTH]; - - if (el == NULL || !ossl_assert(enc_level == QUIC_ENC_LEVEL_1RTT)) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - if (el->state != QRL_EL_STATE_PROV_NORMAL) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (!el->is_tx) { - /* - * We already have the key for the next epoch, so just move to using it. - */ - ++el->key_epoch; - el->state = QRL_EL_STATE_PROV_UPDATING; - return 1; - } - - /* - * TX case. For the TX side we use only keyslot 0; it replaces the old key - * immediately. - */ - secret_len = ossl_qrl_get_suite_secret_len(el->suite_id); - - /* Derive NEXT "quic ku" key (the epoch n+1 secret). */ - if (!tls13_hkdf_expand_ex(el->libctx, el->propq, - el->md, el->ku, - quic_v1_ku_label, - sizeof(quic_v1_ku_label), - NULL, 0, - new_ku, secret_len, 1)) - return 0; - - el_teardown_keyslot(els, enc_level, 0); - - /* Setup keyslot for CURRENT "quic ku" key. */ - if (!el_setup_keyslot(els, enc_level, QRL_EL_STATE_PROV_NORMAL, - 0, el->ku, secret_len)) - return 0; - - ++el->key_epoch; - el->op_count = 0; - memcpy(el->ku, new_ku, secret_len); - /* Remain in PROV_NORMAL state */ - return 1; -} - -/* Transitions from PROV_UPDATING to PROV_COOLDOWN. */ -int ossl_qrl_enc_level_set_key_update_done(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - - if (el == NULL || !ossl_assert(enc_level == QUIC_ENC_LEVEL_1RTT)) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - /* No new key yet, but erase key material to aid PFS. */ - el_teardown_keyslot(els, enc_level, ~el->key_epoch & 1); - el->state = QRL_EL_STATE_PROV_COOLDOWN; - return 1; -} - -/* - * Transitions from PROV_COOLDOWN to PROV_NORMAL. (If in PROV_UPDATING, - * auto-transitions to PROV_COOLDOWN first.) - */ -int ossl_qrl_enc_level_set_key_cooldown_done(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - size_t secret_len; - unsigned char new_ku[EVP_MAX_KEY_LENGTH]; - - if (el == NULL || !ossl_assert(enc_level == QUIC_ENC_LEVEL_1RTT)) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - if (el->state == QRL_EL_STATE_PROV_UPDATING - && !ossl_qrl_enc_level_set_key_update_done(els, enc_level)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (el->state != QRL_EL_STATE_PROV_COOLDOWN) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - secret_len = ossl_qrl_get_suite_secret_len(el->suite_id); - - if (!el_setup_keyslot(els, enc_level, QRL_EL_STATE_PROV_NORMAL, - ~el->key_epoch & 1, el->ku, secret_len)) - return 0; - - /* Derive NEXT "quic ku" key (the epoch n+1 secret). */ - if (!tls13_hkdf_expand_ex(el->libctx, el->propq, - el->md, - el->ku, - quic_v1_ku_label, - sizeof(quic_v1_ku_label), - NULL, 0, - new_ku, secret_len, 1)) { - el_teardown_keyslot(els, enc_level, ~el->key_epoch & 1); - return 0; - } - - memcpy(el->ku, new_ku, secret_len); - el->state = QRL_EL_STATE_PROV_NORMAL; - return 1; -} - -/* - * Discards keying material for a given encryption level. Transitions from any - * state to DISCARDED. - */ -void ossl_qrl_enc_level_set_discard(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level) -{ - OSSL_QRL_ENC_LEVEL *el = ossl_qrl_enc_level_set_get(els, enc_level, 0); - - if (el == NULL || el->state == QRL_EL_STATE_DISCARDED) - return; - - if (ossl_qrl_enc_level_set_have_el(els, enc_level) == 1) { - ossl_quic_hdr_protector_cleanup(&el->hpr); - - el_teardown_keyslot(els, enc_level, 0); - el_teardown_keyslot(els, enc_level, 1); - } - - EVP_MD_free(el->md); - el->md = NULL; - el->state = QRL_EL_STATE_DISCARDED; -} diff --git a/openssl/src/ssl/quic/quic_record_shared.h b/openssl/src/ssl/quic/quic_record_shared.h deleted file mode 100644 index e8c9e28e9..000000000 --- a/openssl/src/ssl/quic/quic_record_shared.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef OSSL_QUIC_RECORD_SHARED_H -# define OSSL_QUIC_RECORD_SHARED_H - -# include -# include "internal/quic_types.h" -# include "internal/quic_wire_pkt.h" - -/* - * QUIC Record Layer EL Management Utilities - * ========================================= - * - * This defines a structure for managing the cryptographic state at a given - * encryption level, as this functionality is shared between QRX and QTX. For - * QRL use only. - */ - -/* - * States an EL can be in. The Updating and Cooldown states are used by RX only; - * a TX EL in the Provisioned state is always in the Normal substate. - * - * Key material is available if in the Provisioned state. - */ -#define QRL_EL_STATE_UNPROV 0 /* Unprovisioned (initial state) */ -#define QRL_EL_STATE_PROV_NORMAL 1 /* Provisioned - Normal */ -#define QRL_EL_STATE_PROV_UPDATING 2 /* Provisioned - Updating */ -#define QRL_EL_STATE_PROV_COOLDOWN 3 /* Provisioned - Cooldown */ -#define QRL_EL_STATE_DISCARDED 4 /* Discarded (terminal state) */ - -typedef struct ossl_qrl_enc_level_st { - /* - * Cryptographic context used to apply and remove header protection from - * packet headers. - */ - QUIC_HDR_PROTECTOR hpr; - - /* Hash function used for key derivation. */ - EVP_MD *md; - - /* Context used for packet body ciphering. One for each keyslot. */ - EVP_CIPHER_CTX *cctx[2]; - - OSSL_LIB_CTX *libctx; - const char *propq; - - /* - * Key epoch, essentially the number of times we have done a key update. - * - * The least significant bit of this is therefore by definition the current - * Key Phase bit value. - */ - uint64_t key_epoch; - - /* Usage counter. The caller maintains this. Used by TX side only. */ - uint64_t op_count; - - /* QRL_SUITE_* value. */ - uint32_t suite_id; - - /* Length of authentication tag. */ - uint32_t tag_len; - - /* Current EL state. */ - unsigned char state; /* QRL_EL_STATE_* */ - - /* 1 if for TX, else RX. Initialised when secret provided. */ - unsigned char is_tx; - - /* IV used to construct nonces used for AEAD packet body ciphering. */ - unsigned char iv[2][EVP_MAX_IV_LENGTH]; - - /* - * Secret for next key epoch. - */ - unsigned char ku[EVP_MAX_KEY_LENGTH]; -} OSSL_QRL_ENC_LEVEL; - -typedef struct ossl_qrl_enc_level_set_st { - OSSL_QRL_ENC_LEVEL el[QUIC_ENC_LEVEL_NUM]; -} OSSL_QRL_ENC_LEVEL_SET; - -/* - * Returns 1 if we have key material for a given encryption level (that is, if - * we are in the PROVISIONED state), 0 if we do not yet have material (we are in - * the UNPROVISIONED state) and -1 if the EL is discarded (we are in the - * DISCARDED state). - */ -int ossl_qrl_enc_level_set_have_el(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level); - -/* - * Returns EL in a set. If enc_level is not a valid QUIC_ENC_LEVEL_* value, - * returns NULL. If require_prov is 1, returns NULL if the EL is not in - * the PROVISIONED state; otherwise, the returned EL may be in any state. - */ -OSSL_QRL_ENC_LEVEL *ossl_qrl_enc_level_set_get(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level, - int require_prov); - -/* Provide secret to an EL. md may be NULL. */ -int ossl_qrl_enc_level_set_provide_secret(OSSL_QRL_ENC_LEVEL_SET *els, - OSSL_LIB_CTX *libctx, - const char *propq, - uint32_t enc_level, - uint32_t suite_id, - EVP_MD *md, - const unsigned char *secret, - size_t secret_len, - unsigned char init_key_phase_bit, - int is_tx); - -/* - * Returns 1 if the given keyslot index is currently valid for a given EL and EL - * state. - */ -int ossl_qrl_enc_level_set_has_keyslot(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level, - unsigned char tgt_state, - size_t keyslot); - -/* Perform a key update. Transitions from PROV_NORMAL to PROV_UPDATING. */ -int ossl_qrl_enc_level_set_key_update(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level); - -/* Transitions from PROV_UPDATING to PROV_COOLDOWN. */ -int ossl_qrl_enc_level_set_key_update_done(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level); - -/* - * Transitions from PROV_COOLDOWN to PROV_NORMAL. (If in PROV_UPDATING, - * auto-transitions to PROV_COOLDOWN first.) - */ -int ossl_qrl_enc_level_set_key_cooldown_done(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level); - -/* - * Discard an EL. No secret can be provided for the EL ever again. - */ -void ossl_qrl_enc_level_set_discard(OSSL_QRL_ENC_LEVEL_SET *els, - uint32_t enc_level); - -#endif diff --git a/openssl/src/ssl/quic/quic_record_tx.c b/openssl/src/ssl/quic/quic_record_tx.c deleted file mode 100644 index cda684245..000000000 --- a/openssl/src/ssl/quic/quic_record_tx.c +++ /dev/null @@ -1,1101 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_record_tx.h" -#include "internal/qlog_event_helpers.h" -#include "internal/bio_addr.h" -#include "internal/common.h" -#include "quic_record_shared.h" -#include "internal/list.h" -#include "../ssl_local.h" - -/* - * TXE - * === - * Encrypted packets awaiting transmission are kept in TX Entries (TXEs), which - * are queued in linked lists just like TXEs. - */ -typedef struct txe_st TXE; - -struct txe_st { - OSSL_LIST_MEMBER(txe, TXE); - size_t data_len, alloc_len; - - /* - * Destination and local addresses, as applicable. Both of these are only - * used if the family is not AF_UNSPEC. - */ - BIO_ADDR peer, local; - - /* - * alloc_len allocated bytes (of which data_len bytes are valid) follow this - * structure. - */ -}; - -DEFINE_LIST_OF(txe, TXE); -typedef OSSL_LIST(txe) TXE_LIST; - -static ossl_inline unsigned char *txe_data(const TXE *e) -{ - return (unsigned char *)(e + 1); -} - -/* - * QTX - * === - */ -struct ossl_qtx_st { - OSSL_LIB_CTX *libctx; - const char *propq; - - /* Per encryption-level state. */ - OSSL_QRL_ENC_LEVEL_SET el_set; - - /* TX BIO. */ - BIO *bio; - - /* QLOG instance retrieval callback if in use, or NULL. */ - QLOG *(*get_qlog_cb)(void *arg); - void *get_qlog_cb_arg; - - /* TX maximum datagram payload length. */ - size_t mdpl; - - /* - * List of TXEs which are not currently in use. These are moved to the - * pending list (possibly via tx_cons first) as they are filled. - */ - TXE_LIST free; - - /* - * List of TXEs which are filled with completed datagrams ready to be - * transmitted. - */ - TXE_LIST pending; - size_t pending_count; /* items in list */ - size_t pending_bytes; /* sum(txe->data_len) in pending */ - - /* - * TXE which is under construction for coalescing purposes, if any. - * This TXE is neither on the free nor pending list. Once the datagram - * is completed, it is moved to the pending list. - */ - TXE *cons; - size_t cons_count; /* num packets */ - - /* - * Number of packets transmitted in this key epoch. Used to enforce AEAD - * confidentiality limit. - */ - uint64_t epoch_pkt_count; - - /* Datagram counter. Increases monotonically per datagram (not per packet). */ - uint64_t datagram_count; - - ossl_mutate_packet_cb mutatecb; - ossl_finish_mutate_cb finishmutatecb; - void *mutatearg; - - /* Message callback related arguments */ - ossl_msg_cb msg_callback; - void *msg_callback_arg; - SSL *msg_callback_ssl; -}; - -/* Instantiates a new QTX. */ -OSSL_QTX *ossl_qtx_new(const OSSL_QTX_ARGS *args) -{ - OSSL_QTX *qtx; - - if (args->mdpl < QUIC_MIN_INITIAL_DGRAM_LEN) - return 0; - - qtx = OPENSSL_zalloc(sizeof(OSSL_QTX)); - if (qtx == NULL) - return 0; - - qtx->libctx = args->libctx; - qtx->propq = args->propq; - qtx->bio = args->bio; - qtx->mdpl = args->mdpl; - qtx->get_qlog_cb = args->get_qlog_cb; - qtx->get_qlog_cb_arg = args->get_qlog_cb_arg; - - return qtx; -} - -static void qtx_cleanup_txl(TXE_LIST *l) -{ - TXE *e, *enext; - - for (e = ossl_list_txe_head(l); e != NULL; e = enext) { - enext = ossl_list_txe_next(e); - OPENSSL_free(e); - } -} - -/* Frees the QTX. */ -void ossl_qtx_free(OSSL_QTX *qtx) -{ - uint32_t i; - - if (qtx == NULL) - return; - - /* Free TXE queue data. */ - qtx_cleanup_txl(&qtx->pending); - qtx_cleanup_txl(&qtx->free); - OPENSSL_free(qtx->cons); - - /* Drop keying material and crypto resources. */ - for (i = 0; i < QUIC_ENC_LEVEL_NUM; ++i) - ossl_qrl_enc_level_set_discard(&qtx->el_set, i); - - OPENSSL_free(qtx); -} - -/* Set mutator callbacks for test framework support */ -void ossl_qtx_set_mutator(OSSL_QTX *qtx, ossl_mutate_packet_cb mutatecb, - ossl_finish_mutate_cb finishmutatecb, void *mutatearg) -{ - qtx->mutatecb = mutatecb; - qtx->finishmutatecb = finishmutatecb; - qtx->mutatearg = mutatearg; -} - -void ossl_qtx_set_qlog_cb(OSSL_QTX *qtx, QLOG *(*get_qlog_cb)(void *arg), - void *get_qlog_cb_arg) -{ - qtx->get_qlog_cb = get_qlog_cb; - qtx->get_qlog_cb_arg = get_qlog_cb_arg; -} - -int ossl_qtx_provide_secret(OSSL_QTX *qtx, - uint32_t enc_level, - uint32_t suite_id, - EVP_MD *md, - const unsigned char *secret, - size_t secret_len) -{ - if (enc_level >= QUIC_ENC_LEVEL_NUM) - return 0; - - return ossl_qrl_enc_level_set_provide_secret(&qtx->el_set, - qtx->libctx, - qtx->propq, - enc_level, - suite_id, - md, - secret, - secret_len, - 0, - /*is_tx=*/1); -} - -int ossl_qtx_discard_enc_level(OSSL_QTX *qtx, uint32_t enc_level) -{ - if (enc_level >= QUIC_ENC_LEVEL_NUM) - return 0; - - ossl_qrl_enc_level_set_discard(&qtx->el_set, enc_level); - return 1; -} - -int ossl_qtx_is_enc_level_provisioned(OSSL_QTX *qtx, uint32_t enc_level) -{ - return ossl_qrl_enc_level_set_get(&qtx->el_set, enc_level, 1) != NULL; -} - -/* Allocate a new TXE. */ -static TXE *qtx_alloc_txe(size_t alloc_len) -{ - TXE *txe; - - if (alloc_len >= SIZE_MAX - sizeof(TXE)) - return NULL; - - txe = OPENSSL_malloc(sizeof(TXE) + alloc_len); - if (txe == NULL) - return NULL; - - ossl_list_txe_init_elem(txe); - txe->alloc_len = alloc_len; - txe->data_len = 0; - return txe; -} - -/* - * Ensures there is at least one TXE in the free list, allocating a new entry - * if necessary. The returned TXE is in the free list; it is not popped. - * - * alloc_len is a hint which may be used to determine the TXE size if allocation - * is necessary. Returns NULL on allocation failure. - */ -static TXE *qtx_ensure_free_txe(OSSL_QTX *qtx, size_t alloc_len) -{ - TXE *txe; - - txe = ossl_list_txe_head(&qtx->free); - if (txe != NULL) - return txe; - - txe = qtx_alloc_txe(alloc_len); - if (txe == NULL) - return NULL; - - ossl_list_txe_insert_tail(&qtx->free, txe); - return txe; -} - -/* - * Resize the data buffer attached to an TXE to be n bytes in size. The address - * of the TXE might change; the new address is returned, or NULL on failure, in - * which case the original TXE remains valid. - */ -static TXE *qtx_resize_txe(OSSL_QTX *qtx, TXE_LIST *txl, TXE *txe, size_t n) -{ - TXE *txe2, *p; - - /* Should never happen. */ - if (txe == NULL) - return NULL; - - if (n >= SIZE_MAX - sizeof(TXE)) - return NULL; - - /* Remove the item from the list to avoid accessing freed memory */ - p = ossl_list_txe_prev(txe); - ossl_list_txe_remove(txl, txe); - - /* - * NOTE: We do not clear old memory, although it does contain decrypted - * data. - */ - txe2 = OPENSSL_realloc(txe, sizeof(TXE) + n); - if (txe2 == NULL || txe == txe2) { - if (p == NULL) - ossl_list_txe_insert_head(txl, txe); - else - ossl_list_txe_insert_after(txl, p, txe); - return txe2; - } - - if (p == NULL) - ossl_list_txe_insert_head(txl, txe2); - else - ossl_list_txe_insert_after(txl, p, txe2); - - if (qtx->cons == txe) - qtx->cons = txe2; - - txe2->alloc_len = n; - return txe2; -} - -/* - * Ensure the data buffer attached to an TXE is at least n bytes in size. - * Returns NULL on failure. - */ -static TXE *qtx_reserve_txe(OSSL_QTX *qtx, TXE_LIST *txl, - TXE *txe, size_t n) -{ - if (txe->alloc_len >= n) - return txe; - - return qtx_resize_txe(qtx, txl, txe, n); -} - -/* Move a TXE from pending to free. */ -static void qtx_pending_to_free(OSSL_QTX *qtx) -{ - TXE *txe = ossl_list_txe_head(&qtx->pending); - - assert(txe != NULL); - ossl_list_txe_remove(&qtx->pending, txe); - --qtx->pending_count; - qtx->pending_bytes -= txe->data_len; - ossl_list_txe_insert_tail(&qtx->free, txe); -} - -/* Add a TXE not currently in any list to the pending list. */ -static void qtx_add_to_pending(OSSL_QTX *qtx, TXE *txe) -{ - ossl_list_txe_insert_tail(&qtx->pending, txe); - ++qtx->pending_count; - qtx->pending_bytes += txe->data_len; -} - -struct iovec_cur { - const OSSL_QTX_IOVEC *iovec; - size_t num_iovec, idx, byte_off, bytes_remaining; -}; - -static size_t iovec_total_bytes(const OSSL_QTX_IOVEC *iovec, - size_t num_iovec) -{ - size_t i, l = 0; - - for (i = 0; i < num_iovec; ++i) - l += iovec[i].buf_len; - - return l; -} - -static void iovec_cur_init(struct iovec_cur *cur, - const OSSL_QTX_IOVEC *iovec, - size_t num_iovec) -{ - cur->iovec = iovec; - cur->num_iovec = num_iovec; - cur->idx = 0; - cur->byte_off = 0; - cur->bytes_remaining = iovec_total_bytes(iovec, num_iovec); -} - -/* - * Get an extent of bytes from the iovec cursor. *buf is set to point to the - * buffer and the number of bytes in length of the buffer is returned. This - * value may be less than the max_buf_len argument. If no more data is - * available, returns 0. - */ -static size_t iovec_cur_get_buffer(struct iovec_cur *cur, - const unsigned char **buf, - size_t max_buf_len) -{ - size_t l; - - if (max_buf_len == 0) { - *buf = NULL; - return 0; - } - - for (;;) { - if (cur->idx >= cur->num_iovec) - return 0; - - l = cur->iovec[cur->idx].buf_len - cur->byte_off; - if (l > max_buf_len) - l = max_buf_len; - - if (l > 0) { - *buf = cur->iovec[cur->idx].buf + cur->byte_off; - cur->byte_off += l; - cur->bytes_remaining -= l; - return l; - } - - /* - * Zero-length iovec entry or we already consumed all of it, try the - * next iovec. - */ - ++cur->idx; - cur->byte_off = 0; - } -} - -/* Determines the size of the AEAD output given the input size. */ -int ossl_qtx_calculate_ciphertext_payload_len(OSSL_QTX *qtx, uint32_t enc_level, - size_t plaintext_len, - size_t *ciphertext_len) -{ - OSSL_QRL_ENC_LEVEL *el - = ossl_qrl_enc_level_set_get(&qtx->el_set, enc_level, 1); - size_t tag_len; - - if (el == NULL) { - *ciphertext_len = 0; - return 0; - } - - /* - * We currently only support ciphers with a 1:1 mapping between plaintext - * and ciphertext size, save for authentication tag. - */ - tag_len = ossl_qrl_get_suite_cipher_tag_len(el->suite_id); - - *ciphertext_len = plaintext_len + tag_len; - return 1; -} - -/* Determines the size of the AEAD input given the output size. */ -int ossl_qtx_calculate_plaintext_payload_len(OSSL_QTX *qtx, uint32_t enc_level, - size_t ciphertext_len, - size_t *plaintext_len) -{ - OSSL_QRL_ENC_LEVEL *el - = ossl_qrl_enc_level_set_get(&qtx->el_set, enc_level, 1); - size_t tag_len; - - if (el == NULL) { - *plaintext_len = 0; - return 0; - } - - tag_len = ossl_qrl_get_suite_cipher_tag_len(el->suite_id); - - if (ciphertext_len <= tag_len) { - *plaintext_len = 0; - return 0; - } - - *plaintext_len = ciphertext_len - tag_len; - return 1; -} - -/* Any other error (including packet being too big for MDPL). */ -#define QTX_FAIL_GENERIC (-1) - -/* - * Returned where there is insufficient room in the datagram to write the - * packet. - */ -#define QTX_FAIL_INSUFFICIENT_LEN (-2) - -static int qtx_write_hdr(OSSL_QTX *qtx, const QUIC_PKT_HDR *hdr, TXE *txe, - QUIC_PKT_HDR_PTRS *ptrs) -{ - WPACKET wpkt; - size_t l = 0; - unsigned char *data = txe_data(txe) + txe->data_len; - - if (!WPACKET_init_static_len(&wpkt, data, txe->alloc_len - txe->data_len, 0)) - return 0; - - if (!ossl_quic_wire_encode_pkt_hdr(&wpkt, hdr->dst_conn_id.id_len, - hdr, ptrs) - || !WPACKET_get_total_written(&wpkt, &l)) { - WPACKET_finish(&wpkt); - return 0; - } - WPACKET_finish(&wpkt); - - if (qtx->msg_callback != NULL) - qtx->msg_callback(1, OSSL_QUIC1_VERSION, SSL3_RT_QUIC_PACKET, data, l, - qtx->msg_callback_ssl, qtx->msg_callback_arg); - - txe->data_len += l; - - return 1; -} - -static int qtx_encrypt_into_txe(OSSL_QTX *qtx, struct iovec_cur *cur, TXE *txe, - uint32_t enc_level, QUIC_PN pn, - const unsigned char *hdr, size_t hdr_len, - QUIC_PKT_HDR_PTRS *ptrs) -{ - int l = 0, l2 = 0, nonce_len; - OSSL_QRL_ENC_LEVEL *el - = ossl_qrl_enc_level_set_get(&qtx->el_set, enc_level, 1); - unsigned char nonce[EVP_MAX_IV_LENGTH]; - size_t i; - EVP_CIPHER_CTX *cctx = NULL; - - /* We should not have been called if we do not have key material. */ - if (!ossl_assert(el != NULL)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - /* - * Have we already encrypted the maximum number of packets using the current - * key? - */ - if (el->op_count >= ossl_qrl_get_suite_max_pkt(el->suite_id)) { - ERR_raise(ERR_LIB_SSL, SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED); - return 0; - } - - /* - * TX key update is simpler than for RX; once we initiate a key update, we - * never need the old keys, as we never deliberately send a packet with old - * keys. Thus the EL always uses keyslot 0 for the TX side. - */ - cctx = el->cctx[0]; - if (!ossl_assert(cctx != NULL)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - /* Construct nonce (nonce=IV ^ PN). */ - nonce_len = EVP_CIPHER_CTX_get_iv_length(cctx); - if (!ossl_assert(nonce_len >= (int)sizeof(QUIC_PN))) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - memcpy(nonce, el->iv[0], (size_t)nonce_len); - for (i = 0; i < sizeof(QUIC_PN); ++i) - nonce[nonce_len - i - 1] ^= (unsigned char)(pn >> (i * 8)); - - /* type and key will already have been setup; feed the IV. */ - if (EVP_CipherInit_ex(cctx, NULL, NULL, NULL, nonce, /*enc=*/1) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - /* Feed AAD data. */ - if (EVP_CipherUpdate(cctx, NULL, &l, hdr, hdr_len) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - /* Encrypt plaintext directly into TXE. */ - for (;;) { - const unsigned char *src; - size_t src_len; - - src_len = iovec_cur_get_buffer(cur, &src, SIZE_MAX); - if (src_len == 0) - break; - - if (EVP_CipherUpdate(cctx, txe_data(txe) + txe->data_len, - &l, src, src_len) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - /* Ignore what we just encrypted and overwrite it with the plaintext */ - memcpy(txe_data(txe) + txe->data_len, src, l); -#endif - - assert(l > 0 && src_len == (size_t)l); - txe->data_len += src_len; - } - - /* Finalise and get tag. */ - if (EVP_CipherFinal_ex(cctx, NULL, &l2) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - if (EVP_CIPHER_CTX_ctrl(cctx, EVP_CTRL_AEAD_GET_TAG, - el->tag_len, txe_data(txe) + txe->data_len) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - txe->data_len += el->tag_len; - - /* Apply header protection. */ - if (!ossl_quic_hdr_protector_encrypt(&el->hpr, ptrs)) - return 0; - - ++el->op_count; - return 1; -} - -/* - * Append a packet to the TXE buffer, serializing and encrypting it in the - * process. - */ -static int qtx_write(OSSL_QTX *qtx, const OSSL_QTX_PKT *pkt, TXE *txe, - uint32_t enc_level, QUIC_PKT_HDR *hdr, - const OSSL_QTX_IOVEC *iovec, size_t num_iovec) -{ - int ret, needs_encrypt; - size_t hdr_len, pred_hdr_len, payload_len, pkt_len, space_left; - size_t min_len, orig_data_len; - struct iovec_cur cur; - QUIC_PKT_HDR_PTRS ptrs; - unsigned char *hdr_start; - OSSL_QRL_ENC_LEVEL *el = NULL; - - /* - * Determine if the packet needs encryption and the minimum conceivable - * serialization length. - */ - if (!ossl_quic_pkt_type_is_encrypted(hdr->type)) { - needs_encrypt = 0; - min_len = QUIC_MIN_VALID_PKT_LEN; - } else { - needs_encrypt = 1; - min_len = QUIC_MIN_VALID_PKT_LEN_CRYPTO; - el = ossl_qrl_enc_level_set_get(&qtx->el_set, enc_level, 1); - if (!ossl_assert(el != NULL)) /* should already have been checked */ - return 0; - } - - orig_data_len = txe->data_len; - space_left = txe->alloc_len - txe->data_len; - if (space_left < min_len) { - /* Not even a possibility of it fitting. */ - ret = QTX_FAIL_INSUFFICIENT_LEN; - goto err; - } - - /* Set some fields in the header we are responsible for. */ - if (hdr->type == QUIC_PKT_TYPE_1RTT) - hdr->key_phase = (unsigned char)(el->key_epoch & 1); - - /* Walk the iovecs to determine actual input payload length. */ - iovec_cur_init(&cur, iovec, num_iovec); - - if (cur.bytes_remaining == 0) { - /* No zero-length payloads allowed. */ - ret = QTX_FAIL_GENERIC; - goto err; - } - - /* Determine encrypted payload length. */ - if (needs_encrypt) - ossl_qtx_calculate_ciphertext_payload_len(qtx, enc_level, - cur.bytes_remaining, - &payload_len); - else - payload_len = cur.bytes_remaining; - - /* Determine header length. */ - hdr->data = NULL; - hdr->len = payload_len; - pred_hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(hdr->dst_conn_id.id_len, - hdr); - if (pred_hdr_len == 0) { - ret = QTX_FAIL_GENERIC; - goto err; - } - - /* We now definitively know our packet length. */ - pkt_len = pred_hdr_len + payload_len; - - if (pkt_len > space_left) { - ret = QTX_FAIL_INSUFFICIENT_LEN; - goto err; - } - - if (ossl_quic_pkt_type_has_pn(hdr->type)) { - if (!ossl_quic_wire_encode_pkt_hdr_pn(pkt->pn, - hdr->pn, - hdr->pn_len)) { - ret = QTX_FAIL_GENERIC; - goto err; - } - } - - /* Append the header to the TXE. */ - hdr_start = txe_data(txe) + txe->data_len; - if (!qtx_write_hdr(qtx, hdr, txe, &ptrs)) { - ret = QTX_FAIL_GENERIC; - goto err; - } - - hdr_len = (txe_data(txe) + txe->data_len) - hdr_start; - assert(hdr_len == pred_hdr_len); - - if (!needs_encrypt) { - /* Just copy the payload across. */ - const unsigned char *src; - size_t src_len; - - for (;;) { - /* Buffer length has already been checked above. */ - src_len = iovec_cur_get_buffer(&cur, &src, SIZE_MAX); - if (src_len == 0) - break; - - memcpy(txe_data(txe) + txe->data_len, src, src_len); - txe->data_len += src_len; - } - } else { - /* Encrypt into TXE. */ - if (!qtx_encrypt_into_txe(qtx, &cur, txe, enc_level, pkt->pn, - hdr_start, hdr_len, &ptrs)) { - ret = QTX_FAIL_GENERIC; - goto err; - } - - assert(txe->data_len - orig_data_len == pkt_len); - } - - return 1; - -err: - /* - * Restore original length so we don't leave a half-written packet in the - * TXE. - */ - txe->data_len = orig_data_len; - return ret; -} - -static TXE *qtx_ensure_cons(OSSL_QTX *qtx) -{ - TXE *txe = qtx->cons; - - if (txe != NULL) - return txe; - - txe = qtx_ensure_free_txe(qtx, qtx->mdpl); - if (txe == NULL) - return NULL; - - ossl_list_txe_remove(&qtx->free, txe); - qtx->cons = txe; - qtx->cons_count = 0; - txe->data_len = 0; - return txe; -} - -static QLOG *qtx_get_qlog(OSSL_QTX *qtx) -{ - if (qtx->get_qlog_cb == NULL) - return NULL; - - return qtx->get_qlog_cb(qtx->get_qlog_cb_arg); -} - -static int qtx_mutate_write(OSSL_QTX *qtx, const OSSL_QTX_PKT *pkt, TXE *txe, - uint32_t enc_level) -{ - int ret; - QUIC_PKT_HDR *hdr; - const OSSL_QTX_IOVEC *iovec; - size_t num_iovec; - - /* If we are running tests then mutate_packet may be non NULL */ - if (qtx->mutatecb != NULL) { - if (!qtx->mutatecb(pkt->hdr, pkt->iovec, pkt->num_iovec, &hdr, - &iovec, &num_iovec, qtx->mutatearg)) - return QTX_FAIL_GENERIC; - } else { - hdr = pkt->hdr; - iovec = pkt->iovec; - num_iovec = pkt->num_iovec; - } - - ret = qtx_write(qtx, pkt, txe, enc_level, - hdr, iovec, num_iovec); - if (ret == 1) - ossl_qlog_event_transport_packet_sent(qtx_get_qlog(qtx), hdr, pkt->pn, - iovec, num_iovec, - qtx->datagram_count); - - if (qtx->finishmutatecb != NULL) - qtx->finishmutatecb(qtx->mutatearg); - - return ret; -} - -static int addr_eq(const BIO_ADDR *a, const BIO_ADDR *b) -{ - return ((a == NULL || BIO_ADDR_family(a) == AF_UNSPEC) - && (b == NULL || BIO_ADDR_family(b) == AF_UNSPEC)) - || (a != NULL && b != NULL && memcmp(a, b, sizeof(*a)) == 0); -} - -int ossl_qtx_write_pkt(OSSL_QTX *qtx, const OSSL_QTX_PKT *pkt) -{ - int ret; - int coalescing = (pkt->flags & OSSL_QTX_PKT_FLAG_COALESCE) != 0; - int was_coalescing; - TXE *txe; - uint32_t enc_level; - - /* Must have EL configured, must have header. */ - if (pkt->hdr == NULL) - return 0; - - enc_level = ossl_quic_pkt_type_to_enc_level(pkt->hdr->type); - - /* Some packet types must be in a packet all by themselves. */ - if (!ossl_quic_pkt_type_can_share_dgram(pkt->hdr->type)) - ossl_qtx_finish_dgram(qtx); - else if (enc_level >= QUIC_ENC_LEVEL_NUM - || ossl_qrl_enc_level_set_have_el(&qtx->el_set, enc_level) != 1) { - /* All other packet types are encrypted. */ - return 0; - } - - was_coalescing = (qtx->cons != NULL && qtx->cons->data_len > 0); - if (was_coalescing) - if (!addr_eq(&qtx->cons->peer, pkt->peer) - || !addr_eq(&qtx->cons->local, pkt->local)) { - /* Must stop coalescing if addresses have changed */ - ossl_qtx_finish_dgram(qtx); - was_coalescing = 0; - } - - for (;;) { - /* - * Start a new coalescing session or continue using the existing one and - * serialize/encrypt the packet. We always encrypt packets as soon as - * our caller gives them to us, which relieves the caller of any need to - * keep the plaintext around. - */ - txe = qtx_ensure_cons(qtx); - if (txe == NULL) - return 0; /* allocation failure */ - - /* - * Ensure TXE has at least MDPL bytes allocated. This should only be - * possible if the MDPL has increased. - */ - if (!qtx_reserve_txe(qtx, NULL, txe, qtx->mdpl)) - return 0; - - if (!was_coalescing) { - /* Set addresses in TXE. */ - if (pkt->peer != NULL) - txe->peer = *pkt->peer; - else - BIO_ADDR_clear(&txe->peer); - - if (pkt->local != NULL) - txe->local = *pkt->local; - else - BIO_ADDR_clear(&txe->local); - } - - ret = qtx_mutate_write(qtx, pkt, txe, enc_level); - if (ret == 1) { - break; - } else if (ret == QTX_FAIL_INSUFFICIENT_LEN) { - if (was_coalescing) { - /* - * We failed due to insufficient length, so end the current - * datagram and try again. - */ - ossl_qtx_finish_dgram(qtx); - was_coalescing = 0; - } else { - /* - * We failed due to insufficient length, but we were not - * coalescing/started with an empty datagram, so any future - * attempt to write this packet must also fail. - */ - return 0; - } - } else { - return 0; /* other error */ - } - } - - ++qtx->cons_count; - - /* - * Some packet types cannot have another packet come after them. - */ - if (ossl_quic_pkt_type_must_be_last(pkt->hdr->type)) - coalescing = 0; - - if (!coalescing) - ossl_qtx_finish_dgram(qtx); - - return 1; -} - -/* - * Finish any incomplete datagrams for transmission which were flagged for - * coalescing. If there is no current coalescing datagram, this is a no-op. - */ -void ossl_qtx_finish_dgram(OSSL_QTX *qtx) -{ - TXE *txe = qtx->cons; - - if (txe == NULL) - return; - - if (txe->data_len == 0) - /* - * If we did not put anything in the datagram, just move it back to the - * free list. - */ - ossl_list_txe_insert_tail(&qtx->free, txe); - else - qtx_add_to_pending(qtx, txe); - - qtx->cons = NULL; - qtx->cons_count = 0; - ++qtx->datagram_count; -} - -static void txe_to_msg(TXE *txe, BIO_MSG *msg) -{ - msg->data = txe_data(txe); - msg->data_len = txe->data_len; - msg->flags = 0; - msg->peer - = BIO_ADDR_family(&txe->peer) != AF_UNSPEC ? &txe->peer : NULL; - msg->local - = BIO_ADDR_family(&txe->local) != AF_UNSPEC ? &txe->local : NULL; -} - -#define MAX_MSGS_PER_SEND 32 - -int ossl_qtx_flush_net(OSSL_QTX *qtx) -{ - BIO_MSG msg[MAX_MSGS_PER_SEND]; - size_t wr, i, total_written = 0; - TXE *txe; - int res; - - if (ossl_list_txe_head(&qtx->pending) == NULL) - return QTX_FLUSH_NET_RES_OK; /* Nothing to send. */ - - if (qtx->bio == NULL) - return QTX_FLUSH_NET_RES_PERMANENT_FAIL; - - for (;;) { - for (txe = ossl_list_txe_head(&qtx->pending), i = 0; - txe != NULL && i < OSSL_NELEM(msg); - txe = ossl_list_txe_next(txe), ++i) - txe_to_msg(txe, &msg[i]); - - if (!i) - /* Nothing to send. */ - break; - - ERR_set_mark(); - res = BIO_sendmmsg(qtx->bio, msg, sizeof(BIO_MSG), i, 0, &wr); - if (res && wr == 0) { - /* - * Treat 0 messages sent as a transient error and just stop for now. - */ - ERR_clear_last_mark(); - break; - } else if (!res) { - /* - * We did not get anything, so further calls will probably not - * succeed either. - */ - if (BIO_err_is_non_fatal(ERR_peek_last_error())) { - /* Transient error, just stop for now, clearing the error. */ - ERR_pop_to_mark(); - break; - } else { - /* Non-transient error, fail and do not clear the error. */ - ERR_clear_last_mark(); - return QTX_FLUSH_NET_RES_PERMANENT_FAIL; - } - } - - ERR_clear_last_mark(); - - /* - * Remove everything which was successfully sent from the pending queue. - */ - for (i = 0; i < wr; ++i) { - if (qtx->msg_callback != NULL) - qtx->msg_callback(1, OSSL_QUIC1_VERSION, SSL3_RT_QUIC_DATAGRAM, - msg[i].data, msg[i].data_len, - qtx->msg_callback_ssl, - qtx->msg_callback_arg); - qtx_pending_to_free(qtx); - } - - total_written += wr; - } - - return total_written > 0 - ? QTX_FLUSH_NET_RES_OK - : QTX_FLUSH_NET_RES_TRANSIENT_FAIL; -} - -int ossl_qtx_pop_net(OSSL_QTX *qtx, BIO_MSG *msg) -{ - TXE *txe = ossl_list_txe_head(&qtx->pending); - - if (txe == NULL) - return 0; - - txe_to_msg(txe, msg); - qtx_pending_to_free(qtx); - return 1; -} - -void ossl_qtx_set_bio(OSSL_QTX *qtx, BIO *bio) -{ - qtx->bio = bio; -} - -int ossl_qtx_set_mdpl(OSSL_QTX *qtx, size_t mdpl) -{ - if (mdpl < QUIC_MIN_INITIAL_DGRAM_LEN) - return 0; - - qtx->mdpl = mdpl; - return 1; -} - -size_t ossl_qtx_get_mdpl(OSSL_QTX *qtx) -{ - return qtx->mdpl; -} - -size_t ossl_qtx_get_queue_len_datagrams(OSSL_QTX *qtx) -{ - return qtx->pending_count; -} - -size_t ossl_qtx_get_queue_len_bytes(OSSL_QTX *qtx) -{ - return qtx->pending_bytes; -} - -size_t ossl_qtx_get_cur_dgram_len_bytes(OSSL_QTX *qtx) -{ - return qtx->cons != NULL ? qtx->cons->data_len : 0; -} - -size_t ossl_qtx_get_unflushed_pkt_count(OSSL_QTX *qtx) -{ - return qtx->cons_count; -} - -int ossl_qtx_trigger_key_update(OSSL_QTX *qtx) -{ - return ossl_qrl_enc_level_set_key_update(&qtx->el_set, - QUIC_ENC_LEVEL_1RTT); -} - -uint64_t ossl_qtx_get_cur_epoch_pkt_count(OSSL_QTX *qtx, uint32_t enc_level) -{ - OSSL_QRL_ENC_LEVEL *el; - - el = ossl_qrl_enc_level_set_get(&qtx->el_set, enc_level, 1); - if (el == NULL) - return UINT64_MAX; - - return el->op_count; -} - -uint64_t ossl_qtx_get_max_epoch_pkt_count(OSSL_QTX *qtx, uint32_t enc_level) -{ - OSSL_QRL_ENC_LEVEL *el; - - el = ossl_qrl_enc_level_set_get(&qtx->el_set, enc_level, 1); - if (el == NULL) - return UINT64_MAX; - - return ossl_qrl_get_suite_max_pkt(el->suite_id); -} - -void ossl_qtx_set_msg_callback(OSSL_QTX *qtx, ossl_msg_cb msg_callback, - SSL *msg_callback_ssl) -{ - qtx->msg_callback = msg_callback; - qtx->msg_callback_ssl = msg_callback_ssl; -} - -void ossl_qtx_set_msg_callback_arg(OSSL_QTX *qtx, void *msg_callback_arg) -{ - qtx->msg_callback_arg = msg_callback_arg; -} - -uint64_t ossl_qtx_get_key_epoch(OSSL_QTX *qtx) -{ - OSSL_QRL_ENC_LEVEL *el; - - el = ossl_qrl_enc_level_set_get(&qtx->el_set, QUIC_ENC_LEVEL_1RTT, 1); - if (el == NULL) - return 0; - - return el->key_epoch; -} diff --git a/openssl/src/ssl/quic/quic_record_util.c b/openssl/src/ssl/quic/quic_record_util.c deleted file mode 100644 index e95a84c39..000000000 --- a/openssl/src/ssl/quic/quic_record_util.c +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_record_util.h" -#include "internal/quic_record_rx.h" -#include "internal/quic_record_tx.h" -#include "internal/quic_wire_pkt.h" -#include "../ssl_local.h" -#include -#include - -/* - * QUIC Key Derivation Utilities - * ============================= - */ -int ossl_quic_hkdf_extract(OSSL_LIB_CTX *libctx, - const char *propq, - const EVP_MD *md, - const unsigned char *salt, size_t salt_len, - const unsigned char *ikm, size_t ikm_len, - unsigned char *out, size_t out_len) -{ - int ret = 0; - EVP_KDF *kdf = NULL; - EVP_KDF_CTX *kctx = NULL; - OSSL_PARAM params[7], *p = params; - int mode = EVP_PKEY_HKDEF_MODE_EXTRACT_ONLY; - const char *md_name; - - if ((md_name = EVP_MD_get0_name(md)) == NULL - || (kdf = EVP_KDF_fetch(libctx, OSSL_KDF_NAME_HKDF, propq)) == NULL - || (kctx = EVP_KDF_CTX_new(kdf)) == NULL) - goto err; - - *p++ = OSSL_PARAM_construct_int(OSSL_KDF_PARAM_MODE, &mode); - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST, - (char *)md_name, 0); - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_SALT, - (unsigned char *)salt, salt_len); - *p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_KEY, - (unsigned char *)ikm, ikm_len); - *p++ = OSSL_PARAM_construct_end(); - - ret = EVP_KDF_derive(kctx, out, out_len, params); - -err: - EVP_KDF_CTX_free(kctx); - EVP_KDF_free(kdf); - return ret; -} - -/* Constants used for key derivation in QUIC v1. */ -static const unsigned char quic_client_in_label[] = { - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x69, 0x6e /* "client in" */ -}; -static const unsigned char quic_server_in_label[] = { - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x69, 0x6e /* "server in" */ -}; - -/* Salt used to derive Initial packet protection keys (RFC 9001 Section 5.2). */ -static const unsigned char quic_v1_initial_salt[] = { - 0x38, 0x76, 0x2c, 0xf7, 0xf5, 0x59, 0x34, 0xb3, 0x4d, 0x17, - 0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad, 0xcc, 0xbb, 0x7f, 0x0a -}; - -int ossl_quic_provide_initial_secret(OSSL_LIB_CTX *libctx, - const char *propq, - const QUIC_CONN_ID *dst_conn_id, - int is_server, - struct ossl_qrx_st *qrx, - struct ossl_qtx_st *qtx) -{ - unsigned char initial_secret[32]; - unsigned char client_initial_secret[32], server_initial_secret[32]; - unsigned char *rx_secret, *tx_secret; - EVP_MD *sha256; - - if (qrx == NULL && qtx == NULL) - return 1; - - /* Initial encryption always uses SHA-256. */ - if ((sha256 = EVP_MD_fetch(libctx, "SHA256", propq)) == NULL) - return 0; - - if (is_server) { - rx_secret = client_initial_secret; - tx_secret = server_initial_secret; - } else { - rx_secret = server_initial_secret; - tx_secret = client_initial_secret; - } - - /* Derive initial secret from destination connection ID. */ - if (!ossl_quic_hkdf_extract(libctx, propq, - sha256, - quic_v1_initial_salt, - sizeof(quic_v1_initial_salt), - dst_conn_id->id, - dst_conn_id->id_len, - initial_secret, - sizeof(initial_secret))) - goto err; - - /* Derive "client in" secret. */ - if (((qtx != NULL && tx_secret == client_initial_secret) - || (qrx != NULL && rx_secret == client_initial_secret)) - && !tls13_hkdf_expand_ex(libctx, propq, - sha256, - initial_secret, - quic_client_in_label, - sizeof(quic_client_in_label), - NULL, 0, - client_initial_secret, - sizeof(client_initial_secret), 1)) - goto err; - - /* Derive "server in" secret. */ - if (((qtx != NULL && tx_secret == server_initial_secret) - || (qrx != NULL && rx_secret == server_initial_secret)) - && !tls13_hkdf_expand_ex(libctx, propq, - sha256, - initial_secret, - quic_server_in_label, - sizeof(quic_server_in_label), - NULL, 0, - server_initial_secret, - sizeof(server_initial_secret), 1)) - goto err; - - /* Setup RX EL. Initial encryption always uses AES-128-GCM. */ - if (qrx != NULL - && !ossl_qrx_provide_secret(qrx, QUIC_ENC_LEVEL_INITIAL, - QRL_SUITE_AES128GCM, - sha256, - rx_secret, - sizeof(server_initial_secret))) - goto err; - - /* - * ossl_qrx_provide_secret takes ownership of our ref to SHA256, so if we - * are initialising both sides, get a new ref for the following call for the - * TX side. - */ - if (qrx != NULL && qtx != NULL && !EVP_MD_up_ref(sha256)) { - sha256 = NULL; - goto err; - } - - /* Setup TX cipher. */ - if (qtx != NULL - && !ossl_qtx_provide_secret(qtx, QUIC_ENC_LEVEL_INITIAL, - QRL_SUITE_AES128GCM, - sha256, - tx_secret, - sizeof(server_initial_secret))) - goto err; - - return 1; - -err: - EVP_MD_free(sha256); - return 0; -} - -/* - * QUIC Record Layer Ciphersuite Info - * ================================== - */ - -struct suite_info { - const char *cipher_name, *md_name; - uint32_t secret_len, cipher_key_len, cipher_iv_len, cipher_tag_len; - uint32_t hdr_prot_key_len, hdr_prot_cipher_id; - uint64_t max_pkt, max_forged_pkt; -}; - -static const struct suite_info suite_aes128gcm = { - "AES-128-GCM", "SHA256", 32, 16, 12, 16, 16, - QUIC_HDR_PROT_CIPHER_AES_128, - ((uint64_t)1) << 23, /* Limits as prescribed by RFC 9001 */ - ((uint64_t)1) << 52, -}; - -static const struct suite_info suite_aes256gcm = { - "AES-256-GCM", "SHA384", 48, 32, 12, 16, 32, - QUIC_HDR_PROT_CIPHER_AES_256, - ((uint64_t)1) << 23, /* Limits as prescribed by RFC 9001 */ - ((uint64_t)1) << 52, -}; - -static const struct suite_info suite_chacha20poly1305 = { - "ChaCha20-Poly1305", "SHA256", 32, 32, 12, 16, 32, - QUIC_HDR_PROT_CIPHER_CHACHA, - /* Do not use UINT64_MAX here as this represents an invalid value */ - UINT64_MAX - 1, /* No applicable limit for this suite (RFC 9001) */ - ((uint64_t)1) << 36, /* Limit as prescribed by RFC 9001 */ -}; - -static const struct suite_info *get_suite(uint32_t suite_id) -{ - switch (suite_id) { - case QRL_SUITE_AES128GCM: - return &suite_aes128gcm; - case QRL_SUITE_AES256GCM: - return &suite_aes256gcm; - case QRL_SUITE_CHACHA20POLY1305: - return &suite_chacha20poly1305; - default: - return NULL; - } -} - -const char *ossl_qrl_get_suite_cipher_name(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->cipher_name : NULL; -} - -const char *ossl_qrl_get_suite_md_name(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->md_name : NULL; -} - -uint32_t ossl_qrl_get_suite_secret_len(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->secret_len : 0; -} - -uint32_t ossl_qrl_get_suite_cipher_key_len(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->cipher_key_len : 0; -} - -uint32_t ossl_qrl_get_suite_cipher_iv_len(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->cipher_iv_len : 0; -} - -uint32_t ossl_qrl_get_suite_cipher_tag_len(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->cipher_tag_len : 0; -} - -uint32_t ossl_qrl_get_suite_hdr_prot_cipher_id(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->hdr_prot_cipher_id : 0; -} - -uint32_t ossl_qrl_get_suite_hdr_prot_key_len(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->hdr_prot_key_len : 0; -} - -uint64_t ossl_qrl_get_suite_max_pkt(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->max_pkt : UINT64_MAX; -} - -uint64_t ossl_qrl_get_suite_max_forged_pkt(uint32_t suite_id) -{ - const struct suite_info *c = get_suite(suite_id); - return c != NULL ? c->max_forged_pkt : UINT64_MAX; -} diff --git a/openssl/src/ssl/quic/quic_rstream.c b/openssl/src/ssl/quic/quic_rstream.c deleted file mode 100644 index dd3dbf756..000000000 --- a/openssl/src/ssl/quic/quic_rstream.c +++ /dev/null @@ -1,295 +0,0 @@ -/* -* Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ -#include -#include "internal/common.h" -#include "internal/time.h" -#include "internal/quic_stream.h" -#include "internal/quic_sf_list.h" -#include "internal/ring_buf.h" - -struct quic_rstream_st { - SFRAME_LIST fl; - QUIC_RXFC *rxfc; - OSSL_STATM *statm; - UINT_RANGE head_range; - struct ring_buf rbuf; -}; - -QUIC_RSTREAM *ossl_quic_rstream_new(QUIC_RXFC *rxfc, - OSSL_STATM *statm, size_t rbuf_size) -{ - QUIC_RSTREAM *ret = OPENSSL_zalloc(sizeof(*ret)); - - if (ret == NULL) - return NULL; - - ring_buf_init(&ret->rbuf); - if (!ring_buf_resize(&ret->rbuf, rbuf_size, 0)) { - OPENSSL_free(ret); - return NULL; - } - - ossl_sframe_list_init(&ret->fl); - ret->rxfc = rxfc; - ret->statm = statm; - return ret; -} - -void ossl_quic_rstream_free(QUIC_RSTREAM *qrs) -{ - int cleanse; - - if (qrs == NULL) - return; - - cleanse = qrs->fl.cleanse; - ossl_sframe_list_destroy(&qrs->fl); - ring_buf_destroy(&qrs->rbuf, cleanse); - OPENSSL_free(qrs); -} - -int ossl_quic_rstream_queue_data(QUIC_RSTREAM *qrs, OSSL_QRX_PKT *pkt, - uint64_t offset, - const unsigned char *data, uint64_t data_len, - int fin) -{ - UINT_RANGE range; - - if ((data == NULL && data_len != 0) || (data_len == 0 && fin == 0)) { - /* empty frame allowed only at the end of the stream */ - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - range.start = offset; - range.end = offset + data_len; - - return ossl_sframe_list_insert(&qrs->fl, &range, pkt, data, fin); -} - -static int read_internal(QUIC_RSTREAM *qrs, unsigned char *buf, size_t size, - size_t *readbytes, int *fin, int drop) -{ - void *iter = NULL; - UINT_RANGE range; - const unsigned char *data; - uint64_t offset = 0; - size_t readbytes_ = 0; - int fin_ = 0, ret = 1; - - while (ossl_sframe_list_peek(&qrs->fl, &iter, &range, &data, &fin_)) { - size_t l = (size_t)(range.end - range.start); - - if (l > size) { - l = size; - fin_ = 0; - } - offset = range.start + l; - if (l == 0) - break; - - if (data == NULL) { - size_t max_len; - - data = ring_buf_get_ptr(&qrs->rbuf, range.start, &max_len); - if (!ossl_assert(data != NULL)) - return 0; - if (max_len < l) { - memcpy(buf, data, max_len); - size -= max_len; - buf += max_len; - readbytes_ += max_len; - l -= max_len; - data = ring_buf_get_ptr(&qrs->rbuf, range.start + max_len, - &max_len); - if (!ossl_assert(data != NULL) || !ossl_assert(max_len > l)) - return 0; - } - } - - memcpy(buf, data, l); - size -= l; - buf += l; - readbytes_ += l; - if (size == 0) - break; - } - - if (drop && offset != 0) { - ret = ossl_sframe_list_drop_frames(&qrs->fl, offset); - ring_buf_cpop_range(&qrs->rbuf, 0, offset - 1, qrs->fl.cleanse); - } - - if (ret) { - *readbytes = readbytes_; - *fin = fin_; - } - - return ret; -} - -static OSSL_TIME get_rtt(QUIC_RSTREAM *qrs) -{ - OSSL_TIME rtt; - - if (qrs->statm != NULL) { - OSSL_RTT_INFO rtt_info; - - ossl_statm_get_rtt_info(qrs->statm, &rtt_info); - rtt = rtt_info.smoothed_rtt; - } else { - rtt = ossl_time_zero(); - } - return rtt; -} - -int ossl_quic_rstream_read(QUIC_RSTREAM *qrs, unsigned char *buf, size_t size, - size_t *readbytes, int *fin) -{ - OSSL_TIME rtt = get_rtt(qrs); - - if (!read_internal(qrs, buf, size, readbytes, fin, 1)) - return 0; - - if (qrs->rxfc != NULL - && !ossl_quic_rxfc_on_retire(qrs->rxfc, *readbytes, rtt)) - return 0; - - return 1; -} - -int ossl_quic_rstream_peek(QUIC_RSTREAM *qrs, unsigned char *buf, size_t size, - size_t *readbytes, int *fin) -{ - return read_internal(qrs, buf, size, readbytes, fin, 0); -} - -int ossl_quic_rstream_available(QUIC_RSTREAM *qrs, size_t *avail, int *fin) -{ - void *iter = NULL; - UINT_RANGE range; - const unsigned char *data; - uint64_t avail_ = 0; - - while (ossl_sframe_list_peek(&qrs->fl, &iter, &range, &data, fin)) - avail_ += range.end - range.start; - -#if SIZE_MAX < UINT64_MAX - *avail = avail_ > SIZE_MAX ? SIZE_MAX : (size_t)avail_; -#else - *avail = (size_t)avail_; -#endif - return 1; -} - -int ossl_quic_rstream_get_record(QUIC_RSTREAM *qrs, - const unsigned char **record, size_t *rec_len, - int *fin) -{ - const unsigned char *record_ = NULL; - size_t rec_len_, max_len; - - if (!ossl_sframe_list_lock_head(&qrs->fl, &qrs->head_range, &record_, fin)) { - /* No head frame to lock and return */ - *record = NULL; - *rec_len = 0; - return 1; - } - - /* if final empty frame, we drop it immediately */ - if (qrs->head_range.end == qrs->head_range.start) { - if (!ossl_assert(*fin)) - return 0; - if (!ossl_sframe_list_drop_frames(&qrs->fl, qrs->head_range.end)) - return 0; - } - - rec_len_ = (size_t)(qrs->head_range.end - qrs->head_range.start); - - if (record_ == NULL && rec_len_ != 0) { - record_ = ring_buf_get_ptr(&qrs->rbuf, qrs->head_range.start, - &max_len); - if (!ossl_assert(record_ != NULL)) - return 0; - if (max_len < rec_len_) { - rec_len_ = max_len; - qrs->head_range.end = qrs->head_range.start + max_len; - } - } - - *rec_len = rec_len_; - *record = record_; - return 1; -} - - -int ossl_quic_rstream_release_record(QUIC_RSTREAM *qrs, size_t read_len) -{ - uint64_t offset; - - if (!ossl_sframe_list_is_head_locked(&qrs->fl)) - return 0; - - if (read_len > qrs->head_range.end - qrs->head_range.start) { - if (read_len != SIZE_MAX) - return 0; - offset = qrs->head_range.end; - } else { - offset = qrs->head_range.start + read_len; - } - - if (!ossl_sframe_list_drop_frames(&qrs->fl, offset)) - return 0; - - if (offset > 0) - ring_buf_cpop_range(&qrs->rbuf, 0, offset - 1, qrs->fl.cleanse); - - if (qrs->rxfc != NULL) { - OSSL_TIME rtt = get_rtt(qrs); - - if (!ossl_quic_rxfc_on_retire(qrs->rxfc, offset, rtt)) - return 0; - } - - return 1; -} - -static int write_at_ring_buf_cb(uint64_t logical_offset, - const unsigned char *buf, - size_t buf_len, - void *cb_arg) -{ - struct ring_buf *rbuf = cb_arg; - - return ring_buf_write_at(rbuf, logical_offset, buf, buf_len); -} - -int ossl_quic_rstream_move_to_rbuf(QUIC_RSTREAM *qrs) -{ - if (ring_buf_avail(&qrs->rbuf) == 0) - return 0; - return ossl_sframe_list_move_data(&qrs->fl, - write_at_ring_buf_cb, &qrs->rbuf); -} - -int ossl_quic_rstream_resize_rbuf(QUIC_RSTREAM *qrs, size_t rbuf_size) -{ - if (ossl_sframe_list_is_head_locked(&qrs->fl)) - return 0; - - if (!ring_buf_resize(&qrs->rbuf, rbuf_size, qrs->fl.cleanse)) - return 0; - - return 1; -} - -void ossl_quic_rstream_set_cleanse(QUIC_RSTREAM *qrs, int cleanse) -{ - qrs->fl.cleanse = cleanse; -} diff --git a/openssl/src/ssl/quic/quic_rx_depack.c b/openssl/src/ssl/quic/quic_rx_depack.c deleted file mode 100644 index 58a8edf03..000000000 --- a/openssl/src/ssl/quic/quic_rx_depack.c +++ /dev/null @@ -1,1467 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/packet_quic.h" -#include "internal/nelem.h" -#include "internal/quic_wire.h" -#include "internal/quic_record_rx.h" -#include "internal/quic_ackm.h" -#include "internal/quic_rx_depack.h" -#include "internal/quic_error.h" -#include "internal/quic_fc.h" -#include "internal/quic_channel.h" -#include "internal/sockets.h" - -#include "quic_local.h" -#include "quic_channel_local.h" -#include "../ssl_local.h" - -/* - * Helper functions to process different frame types. - * - * Typically, those that are ACK eliciting will take an OSSL_ACKM_RX_PKT - * pointer argument, the few that aren't ACK eliciting will not. This makes - * them a verifiable pattern against tables where this is specified. - */ -static int depack_do_implicit_stream_create(QUIC_CHANNEL *ch, - uint64_t stream_id, - uint64_t frame_type, - QUIC_STREAM **result); - -static int depack_do_frame_padding(PACKET *pkt) -{ - /* We ignore this frame */ - ossl_quic_wire_decode_padding(pkt); - return 1; -} - -static int depack_do_frame_ping(PACKET *pkt, QUIC_CHANNEL *ch, - uint32_t enc_level, - OSSL_ACKM_RX_PKT *ackm_data) -{ - /* We ignore this frame, apart from eliciting an ACK */ - if (!ossl_quic_wire_decode_frame_ping(pkt)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_PING, - "decode error"); - return 0; - } - - ossl_quic_tx_packetiser_schedule_ack_eliciting(ch->txp, enc_level); - return 1; -} - -static int depack_do_frame_ack(PACKET *pkt, QUIC_CHANNEL *ch, - int packet_space, OSSL_TIME received, - uint64_t frame_type, - OSSL_QRX_PKT *qpacket) -{ - OSSL_QUIC_FRAME_ACK ack; - OSSL_QUIC_ACK_RANGE *p; - uint64_t total_ranges = 0; - uint32_t ack_delay_exp = ch->rx_ack_delay_exp; - - if (!ossl_quic_wire_peek_frame_ack_num_ranges(pkt, &total_ranges) - /* In case sizeof(uint64_t) > sizeof(size_t) */ - || total_ranges > SIZE_MAX / sizeof(OSSL_QUIC_ACK_RANGE)) - goto malformed; - - if (ch->num_ack_range_scratch < (size_t)total_ranges) { - if ((p = OPENSSL_realloc(ch->ack_range_scratch, - sizeof(OSSL_QUIC_ACK_RANGE) - * (size_t)total_ranges)) == NULL) - goto malformed; - - ch->ack_range_scratch = p; - ch->num_ack_range_scratch = (size_t)total_ranges; - } - - ack.ack_ranges = ch->ack_range_scratch; - ack.num_ack_ranges = (size_t)total_ranges; - - if (!ossl_quic_wire_decode_frame_ack(pkt, ack_delay_exp, &ack, NULL)) - goto malformed; - - if (qpacket->hdr->type == QUIC_PKT_TYPE_1RTT - && (qpacket->key_epoch < ossl_qrx_get_key_epoch(ch->qrx) - || ch->rxku_expected) - && ack.ack_ranges[0].end >= ch->txku_pn) { - /* - * RFC 9001 s. 6.2: An endpoint that receives an acknowledgment that is - * carried in a packet protected with old keys where any acknowledged - * packet was protected with newer keys MAY treat that as a connection - * error of type KEY_UPDATE_ERROR. - * - * Two cases to handle here: - * - * - We did spontaneous TXKU, the peer has responded in kind and we - * have detected RXKU; !ch->rxku_expected, but then it sent a packet - * with old keys acknowledging a packet in the new key epoch. - * - * This also covers the case where we got RXKU and triggered - * solicited TXKU, and then for some reason the peer sent an ACK of - * a PN in our new TX key epoch with old keys. - * - * - We did spontaneous TXKU; ch->txku_pn is the starting PN of our - * new TX key epoch; the peer has not initiated a solicited TXKU in - * response (so we have not detected RXKU); in this case the RX key - * epoch has not incremented and ch->rxku_expected is still 1. - */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_KEY_UPDATE_ERROR, - frame_type, - "acked packet which initiated a " - "key update without a " - "corresponding key update"); - return 0; - } - - if (!ossl_ackm_on_rx_ack_frame(ch->ackm, &ack, - packet_space, received)) - goto malformed; - - ++ch->diag_num_rx_ack; - return 1; - -malformed: - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - frame_type, - "decode error"); - return 0; -} - -static int depack_do_frame_reset_stream(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - OSSL_QUIC_FRAME_RESET_STREAM frame_data; - QUIC_STREAM *stream = NULL; - uint64_t fce; - - if (!ossl_quic_wire_decode_frame_reset_stream(pkt, &frame_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_RESET_STREAM, - "decode error"); - return 0; - } - - if (!depack_do_implicit_stream_create(ch, frame_data.stream_id, - OSSL_QUIC_FRAME_TYPE_RESET_STREAM, - &stream)) - return 0; /* error already raised for us */ - - if (stream == NULL) - return 1; /* old deleted stream, not a protocol violation, ignore */ - - if (!ossl_quic_stream_has_recv(stream)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_STREAM_STATE_ERROR, - OSSL_QUIC_FRAME_TYPE_RESET_STREAM, - "RESET_STREAM frame for " - "TX only stream"); - return 0; - } - - /* - * The final size field of the RESET_STREAM frame must be used to determine - * how much flow control credit the aborted stream was considered to have - * consumed. - * - * We also need to ensure that if we already have a final size for the - * stream, the RESET_STREAM frame's Final Size field matches this; we SHOULD - * terminate the connection otherwise (RFC 9000 s. 4.5). The RXFC takes care - * of this for us. - */ - if (!ossl_quic_rxfc_on_rx_stream_frame(&stream->rxfc, - frame_data.final_size, /*is_fin=*/1)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - OSSL_QUIC_FRAME_TYPE_RESET_STREAM, - "internal error (flow control)"); - return 0; - } - - /* Has a flow control error occurred? */ - fce = ossl_quic_rxfc_get_error(&stream->rxfc, 0); - if (fce != OSSL_QUIC_ERR_NO_ERROR) { - ossl_quic_channel_raise_protocol_error(ch, - fce, - OSSL_QUIC_FRAME_TYPE_RESET_STREAM, - "flow control violation"); - return 0; - } - - /* - * Depending on the receive part state this is handled either as a reset - * transition or a no-op (e.g. if a reset has already been received before, - * or the application already retired a FIN). Best effort - there are no - * protocol error conditions we need to check for here. - */ - ossl_quic_stream_map_notify_reset_recv_part(&ch->qsm, stream, - frame_data.app_error_code, - frame_data.final_size); - - ossl_quic_stream_map_update_state(&ch->qsm, stream); - return 1; -} - -static int depack_do_frame_stop_sending(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - OSSL_QUIC_FRAME_STOP_SENDING frame_data; - QUIC_STREAM *stream = NULL; - - if (!ossl_quic_wire_decode_frame_stop_sending(pkt, &frame_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_STOP_SENDING, - "decode error"); - return 0; - } - - if (!depack_do_implicit_stream_create(ch, frame_data.stream_id, - OSSL_QUIC_FRAME_TYPE_STOP_SENDING, - &stream)) - return 0; /* error already raised for us */ - - if (stream == NULL) - return 1; /* old deleted stream, not a protocol violation, ignore */ - - if (!ossl_quic_stream_has_send(stream)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_STREAM_STATE_ERROR, - OSSL_QUIC_FRAME_TYPE_STOP_SENDING, - "STOP_SENDING frame for " - "RX only stream"); - return 0; - } - - stream->peer_stop_sending = 1; - stream->peer_stop_sending_aec = frame_data.app_error_code; - - /* - * RFC 9000 s. 3.5: Receiving a STOP_SENDING frame means we must respond in - * turn with a RESET_STREAM frame for the same part of the stream. The other - * part is unaffected. - */ - ossl_quic_stream_map_reset_stream_send_part(&ch->qsm, stream, - frame_data.app_error_code); - return 1; -} - -static int depack_do_frame_crypto(PACKET *pkt, QUIC_CHANNEL *ch, - OSSL_QRX_PKT *parent_pkt, - OSSL_ACKM_RX_PKT *ackm_data, - uint64_t *datalen) -{ - OSSL_QUIC_FRAME_CRYPTO f; - QUIC_RSTREAM *rstream; - QUIC_RXFC *rxfc; - - *datalen = 0; - - if (!ossl_quic_wire_decode_frame_crypto(pkt, 0, &f)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_CRYPTO, - "decode error"); - return 0; - } - - if (f.len == 0) - return 1; /* nothing to do */ - - rstream = ch->crypto_recv[ackm_data->pkt_space]; - if (!ossl_assert(rstream != NULL)) - /* - * This should not happen; we should only have a NULL stream here if - * the EL has been discarded, and if the EL has been discarded we - * shouldn't be here. - */ - return 0; - - rxfc = &ch->crypto_rxfc[ackm_data->pkt_space]; - - if (!ossl_quic_rxfc_on_rx_stream_frame(rxfc, f.offset + f.len, - /*is_fin=*/0)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - OSSL_QUIC_FRAME_TYPE_CRYPTO, - "internal error (crypto RXFC)"); - return 0; - } - - if (ossl_quic_rxfc_get_error(rxfc, 0) != OSSL_QUIC_ERR_NO_ERROR) { - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_CRYPTO_BUFFER_EXCEEDED, - OSSL_QUIC_FRAME_TYPE_CRYPTO, - "exceeded maximum crypto buffer"); - return 0; - } - - if (!ossl_quic_rstream_queue_data(rstream, parent_pkt, - f.offset, f.data, f.len, 0)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - OSSL_QUIC_FRAME_TYPE_CRYPTO, - "internal error (rstream queue)"); - return 0; - } - - ch->did_crypto_frame = 1; - *datalen = f.len; - - return 1; -} - -static int depack_do_frame_new_token(PACKET *pkt, QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - const uint8_t *token; - size_t token_len; - - if (!ossl_quic_wire_decode_frame_new_token(pkt, &token, &token_len)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_NEW_TOKEN, - "decode error"); - return 0; - } - - if (token_len == 0) { - /* - * RFC 9000 s. 19.7: "A client MUST treat receipt of a NEW_TOKEN frame - * with an empty Token field as a connection error of type - * FRAME_ENCODING_ERROR." - */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_NEW_TOKEN, - "zero-length NEW_TOKEN"); - return 0; - } - - /* TODO(QUIC FUTURE): ADD CODE to send |token| to the session manager */ - - return 1; -} - -/* - * Returns 1 if no protocol violation has occurred. In this case *result will be - * non-NULL unless this is an old deleted stream and we should ignore the frame - * causing this function to be called. Returns 0 on protocol violation. - */ -static int depack_do_implicit_stream_create(QUIC_CHANNEL *ch, - uint64_t stream_id, - uint64_t frame_type, - QUIC_STREAM **result) -{ - QUIC_STREAM *stream; - uint64_t peer_role, stream_ordinal; - uint64_t *p_next_ordinal_local, *p_next_ordinal_remote; - QUIC_RXFC *max_streams_fc; - int is_uni, is_remote_init; - - stream = ossl_quic_stream_map_get_by_id(&ch->qsm, stream_id); - if (stream != NULL) { - *result = stream; - return 1; - } - - /* - * If we do not yet have a stream with the given ID, there are three - * possibilities: - * - * (a) The stream ID is for a remotely-created stream and the peer - * is creating a stream. - * - * (b) The stream ID is for a locally-created stream which has - * previously been deleted. - * - * (c) The stream ID is for a locally-created stream which does - * not exist yet. This is a protocol violation and we must - * terminate the connection in this case. - * - * We distinguish between (b) and (c) using the stream ID allocator - * variable. Since stream ordinals are allocated monotonically, we - * simply determine if the stream ordinal is in the future. - */ - peer_role = ch->is_server - ? QUIC_STREAM_INITIATOR_CLIENT - : QUIC_STREAM_INITIATOR_SERVER; - - is_remote_init = ((stream_id & QUIC_STREAM_INITIATOR_MASK) == peer_role); - is_uni = ((stream_id & QUIC_STREAM_DIR_MASK) == QUIC_STREAM_DIR_UNI); - - stream_ordinal = stream_id >> 2; - - if (is_remote_init) { - /* - * Peer-created stream which does not yet exist. Create it. QUIC stream - * ordinals within a given stream type MUST be used in sequence and - * receiving a STREAM frame for ordinal n must implicitly create streams - * with ordinals [0, n) within that stream type even if no explicit - * STREAM frames are received for those ordinals. - */ - p_next_ordinal_remote = is_uni - ? &ch->next_remote_stream_ordinal_uni - : &ch->next_remote_stream_ordinal_bidi; - - /* Check this isn't violating stream count flow control. */ - max_streams_fc = is_uni - ? &ch->max_streams_uni_rxfc - : &ch->max_streams_bidi_rxfc; - - if (!ossl_quic_rxfc_on_rx_stream_frame(max_streams_fc, - stream_ordinal + 1, - /*is_fin=*/0)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - frame_type, - "internal error (stream count RXFC)"); - return 0; - } - - if (ossl_quic_rxfc_get_error(max_streams_fc, 0) != OSSL_QUIC_ERR_NO_ERROR) { - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_STREAM_LIMIT_ERROR, - frame_type, - "exceeded maximum allowed streams"); - return 0; - } - - /* - * Create the named stream and any streams coming before it yet to be - * created. - */ - while (*p_next_ordinal_remote <= stream_ordinal) { - uint64_t cur_stream_id = (*p_next_ordinal_remote << 2) | - (stream_id - & (QUIC_STREAM_DIR_MASK | QUIC_STREAM_INITIATOR_MASK)); - - stream = ossl_quic_channel_new_stream_remote(ch, cur_stream_id); - if (stream == NULL) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - frame_type, - "internal error (stream allocation)"); - return 0; - } - - ++*p_next_ordinal_remote; - } - - *result = stream; - } else { - /* Locally-created stream which does not yet exist. */ - p_next_ordinal_local = is_uni - ? &ch->next_local_stream_ordinal_uni - : &ch->next_local_stream_ordinal_bidi; - - if (stream_ordinal >= *p_next_ordinal_local) { - /* - * We never created this stream yet, this is a protocol - * violation. - */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_STREAM_STATE_ERROR, - frame_type, - "STREAM frame for nonexistent " - "stream"); - return 0; - } - - /* - * Otherwise this is for an old locally-initiated stream which we - * have subsequently deleted. Ignore the data; it may simply be a - * retransmission. We already take care of notifying the peer of the - * termination of the stream during the stream deletion lifecycle. - */ - *result = NULL; - } - - return 1; -} - -static int depack_do_frame_stream(PACKET *pkt, QUIC_CHANNEL *ch, - OSSL_QRX_PKT *parent_pkt, - OSSL_ACKM_RX_PKT *ackm_data, - uint64_t frame_type, - uint64_t *datalen) -{ - OSSL_QUIC_FRAME_STREAM frame_data; - QUIC_STREAM *stream; - uint64_t fce; - size_t rs_avail; - int rs_fin = 0; - - *datalen = 0; - - if (!ossl_quic_wire_decode_frame_stream(pkt, 0, &frame_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - frame_type, - "decode error"); - return 0; - } - - if (!depack_do_implicit_stream_create(ch, frame_data.stream_id, - frame_type, &stream)) - return 0; /* protocol error raised by above call */ - - if (stream == NULL) - /* - * Data for old stream which is not a protocol violation but should be - * ignored, so stop here. - */ - return 1; - - if (!ossl_quic_stream_has_recv(stream)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_STREAM_STATE_ERROR, - frame_type, - "STREAM frame for TX only " - "stream"); - return 0; - } - - /* Notify stream flow controller. */ - if (!ossl_quic_rxfc_on_rx_stream_frame(&stream->rxfc, - frame_data.offset + frame_data.len, - frame_data.is_fin)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - frame_type, - "internal error (flow control)"); - return 0; - } - - /* Has a flow control error occurred? */ - fce = ossl_quic_rxfc_get_error(&stream->rxfc, 0); - if (fce != OSSL_QUIC_ERR_NO_ERROR) { - ossl_quic_channel_raise_protocol_error(ch, - fce, - frame_type, - "flow control violation"); - return 0; - } - - switch (stream->recv_state) { - case QUIC_RSTREAM_STATE_RECV: - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - /* - * It only makes sense to process incoming STREAM frames in these - * states. - */ - break; - - case QUIC_RSTREAM_STATE_DATA_RECVD: - case QUIC_RSTREAM_STATE_DATA_READ: - case QUIC_RSTREAM_STATE_RESET_RECVD: - case QUIC_RSTREAM_STATE_RESET_READ: - default: - /* - * We have no use for STREAM frames once the receive part reaches any of - * these states, so just ignore. - */ - return 1; - } - - /* If we are in RECV, auto-transition to SIZE_KNOWN on FIN. */ - if (frame_data.is_fin - && !ossl_quic_stream_recv_get_final_size(stream, NULL)) { - - /* State was already checked above, so can't fail. */ - ossl_quic_stream_map_notify_size_known_recv_part(&ch->qsm, stream, - frame_data.offset - + frame_data.len); - } - - /* - * If we requested STOP_SENDING do not bother buffering the data. Note that - * this must happen after RXFC checks above as even if we sent STOP_SENDING - * we must still enforce correct flow control (RFC 9000 s. 3.5). - */ - if (stream->stop_sending) - return 1; /* not an error - packet reordering, etc. */ - - /* - * The receive stream buffer may or may not choose to consume the data - * without copying by reffing the OSSL_QRX_PKT. In this case - * ossl_qrx_pkt_release() will be eventually called when the data is no - * longer needed. - * - * It is OK for the peer to send us a zero-length non-FIN STREAM frame, - * which is a no-op, aside from the fact that it ensures the stream exists. - * In this case we have nothing to report to the receive buffer. - */ - if ((frame_data.len > 0 || frame_data.is_fin) - && !ossl_quic_rstream_queue_data(stream->rstream, parent_pkt, - frame_data.offset, - frame_data.data, - frame_data.len, - frame_data.is_fin)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - frame_type, - "internal error (rstream queue)"); - return 0; - } - - /* - * rs_fin will be 1 only if we can read all data up to and including the FIN - * without any gaps before it; this implies we have received all data. Avoid - * calling ossl_quic_rstream_available() where it is not necessary as it is - * more expensive. - */ - if (stream->recv_state == QUIC_RSTREAM_STATE_SIZE_KNOWN - && !ossl_quic_rstream_available(stream->rstream, &rs_avail, &rs_fin)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - frame_type, - "internal error (rstream available)"); - return 0; - } - - if (rs_fin) - ossl_quic_stream_map_notify_totally_received(&ch->qsm, stream); - - *datalen = frame_data.len; - - return 1; -} - -static void update_streams(QUIC_STREAM *s, void *arg) -{ - QUIC_CHANNEL *ch = arg; - - ossl_quic_stream_map_update_state(&ch->qsm, s); -} - -static void update_streams_bidi(QUIC_STREAM *s, void *arg) -{ - QUIC_CHANNEL *ch = arg; - - if (!ossl_quic_stream_is_bidi(s)) - return; - - ossl_quic_stream_map_update_state(&ch->qsm, s); -} - -static void update_streams_uni(QUIC_STREAM *s, void *arg) -{ - QUIC_CHANNEL *ch = arg; - - if (ossl_quic_stream_is_bidi(s)) - return; - - ossl_quic_stream_map_update_state(&ch->qsm, s); -} - -static int depack_do_frame_max_data(PACKET *pkt, QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - uint64_t max_data = 0; - - if (!ossl_quic_wire_decode_frame_max_data(pkt, &max_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_MAX_DATA, - "decode error"); - return 0; - } - - ossl_quic_txfc_bump_cwm(&ch->conn_txfc, max_data); - ossl_quic_stream_map_visit(&ch->qsm, update_streams, ch); - return 1; -} - -static int depack_do_frame_max_stream_data(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - uint64_t stream_id = 0; - uint64_t max_stream_data = 0; - QUIC_STREAM *stream; - - if (!ossl_quic_wire_decode_frame_max_stream_data(pkt, &stream_id, - &max_stream_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA, - "decode error"); - return 0; - } - - if (!depack_do_implicit_stream_create(ch, stream_id, - OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA, - &stream)) - return 0; /* error already raised for us */ - - if (stream == NULL) - return 1; /* old deleted stream, not a protocol violation, ignore */ - - if (!ossl_quic_stream_has_send(stream)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_STREAM_STATE_ERROR, - OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA, - "MAX_STREAM_DATA for TX only " - "stream"); - return 0; - } - - ossl_quic_txfc_bump_cwm(&stream->txfc, max_stream_data); - ossl_quic_stream_map_update_state(&ch->qsm, stream); - return 1; -} - -static int depack_do_frame_max_streams(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data, - uint64_t frame_type) -{ - uint64_t max_streams = 0; - - if (!ossl_quic_wire_decode_frame_max_streams(pkt, &max_streams)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - frame_type, - "decode error"); - return 0; - } - - if (max_streams > (((uint64_t)1) << 60)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - frame_type, - "invalid max streams value"); - return 0; - } - - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI: - if (max_streams > ch->max_local_streams_bidi) - ch->max_local_streams_bidi = max_streams; - - /* Some streams may now be able to send. */ - ossl_quic_stream_map_visit(&ch->qsm, update_streams_bidi, ch); - break; - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI: - if (max_streams > ch->max_local_streams_uni) - ch->max_local_streams_uni = max_streams; - - /* Some streams may now be able to send. */ - ossl_quic_stream_map_visit(&ch->qsm, update_streams_uni, ch); - break; - default: - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - frame_type, - "decode error"); - return 0; - } - - return 1; -} - -static int depack_do_frame_data_blocked(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - uint64_t max_data = 0; - - if (!ossl_quic_wire_decode_frame_data_blocked(pkt, &max_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_DATA_BLOCKED, - "decode error"); - return 0; - } - - /* No-op - informative/debugging frame. */ - return 1; -} - -static int depack_do_frame_stream_data_blocked(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - uint64_t stream_id = 0; - uint64_t max_data = 0; - QUIC_STREAM *stream; - - if (!ossl_quic_wire_decode_frame_stream_data_blocked(pkt, &stream_id, - &max_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED, - "decode error"); - return 0; - } - - /* - * This is an informative/debugging frame, so we don't have to do anything, - * but it does trigger stream creation. - */ - if (!depack_do_implicit_stream_create(ch, stream_id, - OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED, - &stream)) - return 0; /* error already raised for us */ - - if (stream == NULL) - return 1; /* old deleted stream, not a protocol violation, ignore */ - - if (!ossl_quic_stream_has_recv(stream)) { - /* - * RFC 9000 s. 19.14: "An endpoint that receives a STREAM_DATA_BLOCKED - * frame for a send-only stream MUST terminate the connection with error - * STREAM_STATE_ERROR." - */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_STREAM_STATE_ERROR, - OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED, - "STREAM_DATA_BLOCKED frame for " - "TX only stream"); - return 0; - } - - /* No-op - informative/debugging frame. */ - return 1; -} - -static int depack_do_frame_streams_blocked(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data, - uint64_t frame_type) -{ - uint64_t max_data = 0; - - if (!ossl_quic_wire_decode_frame_streams_blocked(pkt, &max_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - frame_type, - "decode error"); - return 0; - } - - if (max_data > (((uint64_t)1) << 60)) { - /* - * RFC 9000 s. 19.14: "This value cannot exceed 2**60, as it is not - * possible to encode stream IDs larger than 2**62 - 1. Receipt of a - * frame that encodes a larger stream ID MUST be treated as a connection - * error of type STREAM_LIMIT_ERROR or FRAME_ENCODING_ERROR." - */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_STREAM_LIMIT_ERROR, - frame_type, - "invalid stream count limit"); - return 0; - } - - /* No-op - informative/debugging frame. */ - return 1; -} - -static int depack_do_frame_new_conn_id(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - OSSL_QUIC_FRAME_NEW_CONN_ID frame_data; - - if (!ossl_quic_wire_decode_frame_new_conn_id(pkt, &frame_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID, - "decode error"); - return 0; - } - - ossl_quic_channel_on_new_conn_id(ch, &frame_data); - - return 1; -} - -static int depack_do_frame_retire_conn_id(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - uint64_t seq_num; - - if (!ossl_quic_wire_decode_frame_retire_conn_id(pkt, &seq_num)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID, - "decode error"); - return 0; - } - - /* - * RFC 9000 s. 19.16: "An endpoint cannot send this frame if it was provided - * with a zero-length connection ID by its peer. An endpoint that provides a - * zero-length connection ID MUST treat receipt of a RETIRE_CONNECTION_ID - * frame as a connection error of type PROTOCOL_VIOLATION." - * - * Since we always use a zero-length SCID as a client, there is no case - * where it is valid for a server to send this. Our server support is - * currently non-conformant and for internal testing use; simply handle it - * as a no-op in this case. - * - * TODO(QUIC SERVER): Revise and implement correctly for server support. - */ - if (!ch->is_server) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID, - "conn has zero-length CID"); - return 0; - } - - return 1; -} - -static void free_path_response(unsigned char *buf, size_t buf_len, void *arg) -{ - OPENSSL_free(buf); -} - -static int depack_do_frame_path_challenge(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - uint64_t frame_data = 0; - unsigned char *encoded = NULL; - size_t encoded_len; - WPACKET wpkt; - - if (!ossl_quic_wire_decode_frame_path_challenge(pkt, &frame_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_PATH_CHALLENGE, - "decode error"); - return 0; - } - - /* - * RFC 9000 s. 8.2.2: On receiving a PATH_CHALLENGE frame, an endpoint MUST - * respond by echoing the data contained in the PATH_CHALLENGE frame in a - * PATH_RESPONSE frame. - * - * TODO(QUIC FUTURE): We should try to avoid allocation here in the future. - */ - encoded_len = sizeof(uint64_t) + 1; - if ((encoded = OPENSSL_malloc(encoded_len)) == NULL) - goto err; - - if (!WPACKET_init_static_len(&wpkt, encoded, encoded_len, 0)) - goto err; - - if (!ossl_quic_wire_encode_frame_path_response(&wpkt, frame_data)) { - WPACKET_cleanup(&wpkt); - goto err; - } - - WPACKET_finish(&wpkt); - - if (!ossl_quic_cfq_add_frame(ch->cfq, 0, QUIC_PN_SPACE_APP, - OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE, - QUIC_CFQ_ITEM_FLAG_UNRELIABLE, - encoded, encoded_len, - free_path_response, NULL)) - goto err; - - return 1; - -err: - OPENSSL_free(encoded); - ossl_quic_channel_raise_protocol_error(ch, OSSL_QUIC_ERR_INTERNAL_ERROR, - OSSL_QUIC_FRAME_TYPE_PATH_CHALLENGE, - "internal error"); - return 0; -} - -static int depack_do_frame_path_response(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - uint64_t frame_data = 0; - - if (!ossl_quic_wire_decode_frame_path_response(pkt, &frame_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE, - "decode error"); - return 0; - } - - /* TODO(QUIC MULTIPATH): ADD CODE to send |frame_data| to the ch manager */ - - return 1; -} - -static int depack_do_frame_conn_close(PACKET *pkt, QUIC_CHANNEL *ch, - uint64_t frame_type) -{ - OSSL_QUIC_FRAME_CONN_CLOSE frame_data; - - if (!ossl_quic_wire_decode_frame_conn_close(pkt, &frame_data)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - frame_type, - "decode error"); - return 0; - } - - ossl_quic_channel_on_remote_conn_close(ch, &frame_data); - return 1; -} - -static int depack_do_frame_handshake_done(PACKET *pkt, - QUIC_CHANNEL *ch, - OSSL_ACKM_RX_PKT *ackm_data) -{ - if (!ossl_quic_wire_decode_frame_handshake_done(pkt)) { - /* This can fail only with an internal error. */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_INTERNAL_ERROR, - OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE, - "internal error (decode frame handshake done)"); - return 0; - } - - ossl_quic_channel_on_handshake_confirmed(ch); - return 1; -} - -/* Main frame processor */ - -static int depack_process_frames(QUIC_CHANNEL *ch, PACKET *pkt, - OSSL_QRX_PKT *parent_pkt, uint32_t enc_level, - OSSL_TIME received, OSSL_ACKM_RX_PKT *ackm_data) -{ - uint32_t pkt_type = parent_pkt->hdr->type; - uint32_t packet_space = ossl_quic_enc_level_to_pn_space(enc_level); - - if (PACKET_remaining(pkt) == 0) { - /* - * RFC 9000 s. 12.4: An endpoint MUST treat receipt of a packet - * containing no frames as a connection error of type - * PROTOCOL_VIOLATION. - */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - 0, - "empty packet payload"); - return 0; - } - - while (PACKET_remaining(pkt) > 0) { - int was_minimal; - uint64_t frame_type; - const unsigned char *sof = NULL; - uint64_t datalen = 0; - - if (ch->msg_callback != NULL) - sof = PACKET_data(pkt); - - if (!ossl_quic_wire_peek_frame_header(pkt, &frame_type, &was_minimal)) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - 0, - "malformed frame header"); - return 0; - } - - if (!was_minimal) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "non-minimal frame type encoding"); - return 0; - } - - /* - * There are only a few frame types which are not ACK-eliciting. Handle - * these centrally to make error handling cases more resilient, as we - * should tell the ACKM about an ACK-eliciting frame even if it was not - * successfully handled. - */ - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_PADDING: - case OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN: - case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN: - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT: - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_APP: - break; - default: - ackm_data->is_ack_eliciting = 1; - break; - } - - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_PING: - /* Allowed in all packet types */ - if (!depack_do_frame_ping(pkt, ch, enc_level, ackm_data)) - return 0; - break; - case OSSL_QUIC_FRAME_TYPE_PADDING: - /* Allowed in all packet types */ - if (!depack_do_frame_padding(pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN: - case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN: - /* ACK frames are valid everywhere except in 0RTT packets */ - if (pkt_type == QUIC_PKT_TYPE_0RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "ACK not valid in 0-RTT"); - return 0; - } - if (!depack_do_frame_ack(pkt, ch, packet_space, received, - frame_type, parent_pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_RESET_STREAM: - /* RESET_STREAM frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "RESET_STREAM not valid in " - "INITIAL/HANDSHAKE"); - return 0; - } - if (!depack_do_frame_reset_stream(pkt, ch, ackm_data)) - return 0; - break; - case OSSL_QUIC_FRAME_TYPE_STOP_SENDING: - /* STOP_SENDING frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "STOP_SENDING not valid in " - "INITIAL/HANDSHAKE"); - return 0; - } - if (!depack_do_frame_stop_sending(pkt, ch, ackm_data)) - return 0; - break; - case OSSL_QUIC_FRAME_TYPE_CRYPTO: - /* CRYPTO frames are valid everywhere except in 0RTT packets */ - if (pkt_type == QUIC_PKT_TYPE_0RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "CRYPTO frame not valid in 0-RTT"); - return 0; - } - if (!depack_do_frame_crypto(pkt, ch, parent_pkt, ackm_data, &datalen)) - return 0; - break; - case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN: - /* NEW_TOKEN frames are valid in 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "NEW_TOKEN valid only in 1-RTT"); - return 0; - } - if (!depack_do_frame_new_token(pkt, ch, ackm_data)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM: - case OSSL_QUIC_FRAME_TYPE_STREAM_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_LEN: - case OSSL_QUIC_FRAME_TYPE_STREAM_LEN_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN_FIN: - /* STREAM frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "STREAM valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_stream(pkt, ch, parent_pkt, ackm_data, - frame_type, &datalen)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_MAX_DATA: - /* MAX_DATA frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "MAX_DATA valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_max_data(pkt, ch, ackm_data)) - return 0; - break; - case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA: - /* MAX_STREAM_DATA frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "MAX_STREAM_DATA valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_max_stream_data(pkt, ch, ackm_data)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI: - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI: - /* MAX_STREAMS frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "MAX_STREAMS valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_max_streams(pkt, ch, ackm_data, - frame_type)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_DATA_BLOCKED: - /* DATA_BLOCKED frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "DATA_BLOCKED valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_data_blocked(pkt, ch, ackm_data)) - return 0; - break; - case OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED: - /* STREAM_DATA_BLOCKED frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "STREAM_DATA_BLOCKED valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_stream_data_blocked(pkt, ch, ackm_data)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI: - case OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_UNI: - /* STREAMS_BLOCKED frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "STREAMS valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_streams_blocked(pkt, ch, ackm_data, - frame_type)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID: - /* NEW_CONN_ID frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "NEW_CONN_ID valid only in 0/1-RTT"); - } - if (!depack_do_frame_new_conn_id(pkt, ch, ackm_data)) - return 0; - break; - case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID: - /* RETIRE_CONN_ID frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "RETIRE_CONN_ID valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_retire_conn_id(pkt, ch, ackm_data)) - return 0; - break; - case OSSL_QUIC_FRAME_TYPE_PATH_CHALLENGE: - /* PATH_CHALLENGE frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "PATH_CHALLENGE valid only in 0/1-RTT"); - return 0; - } - if (!depack_do_frame_path_challenge(pkt, ch, ackm_data)) - return 0; - - break; - case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE: - /* PATH_RESPONSE frames are valid in 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "PATH_CHALLENGE valid only in 1-RTT"); - return 0; - } - if (!depack_do_frame_path_response(pkt, ch, ackm_data)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_APP: - /* CONN_CLOSE_APP frames are valid in 0RTT and 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_0RTT - && pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "CONN_CLOSE (APP) valid only in 0/1-RTT"); - return 0; - } - /* FALLTHRU */ - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT: - /* CONN_CLOSE_TRANSPORT frames are valid in all packets */ - if (!depack_do_frame_conn_close(pkt, ch, frame_type)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE: - /* HANDSHAKE_DONE frames are valid in 1RTT packets */ - if (pkt_type != QUIC_PKT_TYPE_1RTT) { - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_PROTOCOL_VIOLATION, - frame_type, - "HANDSHAKE_DONE valid only in 1-RTT"); - return 0; - } - if (!depack_do_frame_handshake_done(pkt, ch, ackm_data)) - return 0; - break; - - default: - /* Unknown frame type */ - ossl_quic_channel_raise_protocol_error(ch, - OSSL_QUIC_ERR_FRAME_ENCODING_ERROR, - frame_type, - "Unknown frame type received"); - return 0; - } - - if (ch->msg_callback != NULL) { - int ctype = SSL3_RT_QUIC_FRAME_FULL; - - size_t framelen = PACKET_data(pkt) - sof; - - if (frame_type == OSSL_QUIC_FRAME_TYPE_PADDING) { - ctype = SSL3_RT_QUIC_FRAME_PADDING; - } else if (OSSL_QUIC_FRAME_TYPE_IS_STREAM(frame_type) - || frame_type == OSSL_QUIC_FRAME_TYPE_CRYPTO) { - ctype = SSL3_RT_QUIC_FRAME_HEADER; - framelen -= (size_t)datalen; - } - - ch->msg_callback(0, OSSL_QUIC1_VERSION, ctype, sof, framelen, - ch->msg_callback_ssl, ch->msg_callback_arg); - } - } - - return 1; -} - -QUIC_NEEDS_LOCK -int ossl_quic_handle_frames(QUIC_CHANNEL *ch, OSSL_QRX_PKT *qpacket) -{ - PACKET pkt; - OSSL_ACKM_RX_PKT ackm_data; - uint32_t enc_level; - - /* - * ok has three states: - * -1 error with ackm_data uninitialized - * 0 error with ackm_data initialized - * 1 success (ackm_data initialized) - */ - int ok = -1; /* Assume the worst */ - - if (ch == NULL) - goto end; - - ch->did_crypto_frame = 0; - - /* Initialize |ackm_data| (and reinitialize |ok|)*/ - memset(&ackm_data, 0, sizeof(ackm_data)); - /* - * ASSUMPTION: All packets that aren't special case have a - * packet number. - */ - ackm_data.pkt_num = qpacket->pn; - ackm_data.time = qpacket->time; - enc_level = ossl_quic_pkt_type_to_enc_level(qpacket->hdr->type); - if (enc_level >= QUIC_ENC_LEVEL_NUM) - /* - * Retry and Version Negotiation packets should not be passed to this - * function. - */ - goto end; - - ok = 0; /* Still assume the worst */ - ackm_data.pkt_space = ossl_quic_enc_level_to_pn_space(enc_level); - - /* Now that special cases are out of the way, parse frames */ - if (!PACKET_buf_init(&pkt, qpacket->hdr->data, qpacket->hdr->len) - || !depack_process_frames(ch, &pkt, qpacket, - enc_level, - qpacket->time, - &ackm_data)) - goto end; - - ok = 1; - end: - /* - * ASSUMPTION: If this function is called at all, |qpacket| is - * a legitimate packet, even if its contents aren't. - * Therefore, we call ossl_ackm_on_rx_packet() unconditionally, as long as - * |ackm_data| has at least been initialized. - */ - if (ok >= 0) - ossl_ackm_on_rx_packet(ch->ackm, &ackm_data); - - return ok > 0; -} diff --git a/openssl/src/ssl/quic/quic_sf_list.c b/openssl/src/ssl/quic/quic_sf_list.c deleted file mode 100644 index 0541a2ab6..000000000 --- a/openssl/src/ssl/quic/quic_sf_list.c +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/uint_set.h" -#include "internal/common.h" -#include "internal/quic_sf_list.h" - -struct stream_frame_st { - struct stream_frame_st *prev, *next; - UINT_RANGE range; - OSSL_QRX_PKT *pkt; - const unsigned char *data; -}; - -static void stream_frame_free(SFRAME_LIST *fl, STREAM_FRAME *sf) -{ - if (fl->cleanse && sf->data != NULL) - OPENSSL_cleanse((unsigned char *)sf->data, - (size_t)(sf->range.end - sf->range.start)); - ossl_qrx_pkt_release(sf->pkt); - OPENSSL_free(sf); -} - -static STREAM_FRAME *stream_frame_new(UINT_RANGE *range, OSSL_QRX_PKT *pkt, - const unsigned char *data) -{ - STREAM_FRAME *sf = OPENSSL_zalloc(sizeof(*sf)); - - if (sf == NULL) - return NULL; - - if (pkt != NULL) - ossl_qrx_pkt_up_ref(pkt); - - sf->range = *range; - sf->pkt = pkt; - sf->data = data; - - return sf; -} - -void ossl_sframe_list_init(SFRAME_LIST *fl) -{ - memset(fl, 0, sizeof(*fl)); -} - -void ossl_sframe_list_destroy(SFRAME_LIST *fl) -{ - STREAM_FRAME *sf, *next_frame; - - for (sf = fl->head; sf != NULL; sf = next_frame) { - next_frame = sf->next; - stream_frame_free(fl, sf); - } -} - -static int append_frame(SFRAME_LIST *fl, UINT_RANGE *range, - OSSL_QRX_PKT *pkt, - const unsigned char *data) -{ - STREAM_FRAME *new_frame; - - if ((new_frame = stream_frame_new(range, pkt, data)) == NULL) - return 0; - new_frame->prev = fl->tail; - if (fl->tail != NULL) - fl->tail->next = new_frame; - fl->tail = new_frame; - ++fl->num_frames; - return 1; -} - -int ossl_sframe_list_insert(SFRAME_LIST *fl, UINT_RANGE *range, - OSSL_QRX_PKT *pkt, - const unsigned char *data, int fin) -{ - STREAM_FRAME *sf, *new_frame, *prev_frame, *next_frame; -#ifndef NDEBUG - uint64_t curr_end = fl->tail != NULL ? fl->tail->range.end - : fl->offset; - - /* This check for FINAL_SIZE_ERROR is handled by QUIC FC already */ - assert((!fin || curr_end <= range->end) - && (!fl->fin || curr_end >= range->end)); -#endif - - if (fl->offset >= range->end) - goto end; - - /* nothing there yet */ - if (fl->tail == NULL) { - fl->tail = fl->head = stream_frame_new(range, pkt, data); - if (fl->tail == NULL) - return 0; - - ++fl->num_frames; - goto end; - } - - /* optimize insertion at the end */ - if (fl->tail->range.start < range->start) { - if (fl->tail->range.end >= range->end) - goto end; - - if (!append_frame(fl, range, pkt, data)) - return 0; - goto end; - } - - prev_frame = NULL; - for (sf = fl->head; sf != NULL && sf->range.start < range->start; - sf = sf->next) - prev_frame = sf; - - if (!ossl_assert(sf != NULL)) - /* frame list invariant broken */ - return 0; - - if (prev_frame != NULL && prev_frame->range.end >= range->end) - goto end; - - /* - * Now we must create a new frame although in the end we might drop it, - * because we will be potentially dropping existing overlapping frames. - */ - new_frame = stream_frame_new(range, pkt, data); - if (new_frame == NULL) - return 0; - - for (next_frame = sf; - next_frame != NULL && next_frame->range.end <= range->end;) { - STREAM_FRAME *drop_frame = next_frame; - - next_frame = next_frame->next; - if (next_frame != NULL) - next_frame->prev = drop_frame->prev; - if (prev_frame != NULL) - prev_frame->next = drop_frame->next; - if (fl->head == drop_frame) - fl->head = next_frame; - if (fl->tail == drop_frame) - fl->tail = prev_frame; - --fl->num_frames; - stream_frame_free(fl, drop_frame); - } - - if (next_frame != NULL) { - /* check whether the new_frame is redundant because there is no gap */ - if (prev_frame != NULL - && next_frame->range.start <= prev_frame->range.end) { - stream_frame_free(fl, new_frame); - goto end; - } - next_frame->prev = new_frame; - } else { - fl->tail = new_frame; - } - - new_frame->next = next_frame; - new_frame->prev = prev_frame; - - if (prev_frame != NULL) - prev_frame->next = new_frame; - else - fl->head = new_frame; - - ++fl->num_frames; - - end: - fl->fin = fin || fl->fin; - - return 1; -} - -int ossl_sframe_list_peek(const SFRAME_LIST *fl, void **iter, - UINT_RANGE *range, const unsigned char **data, - int *fin) -{ - STREAM_FRAME *sf = *iter; - uint64_t start; - - if (sf == NULL) { - start = fl->offset; - sf = fl->head; - } else { - start = sf->range.end; - sf = sf->next; - } - - range->start = start; - - if (sf == NULL || sf->range.start > start - || !ossl_assert(start < sf->range.end)) { - range->end = start; - *data = NULL; - *iter = NULL; - /* set fin only if we are at the end */ - *fin = sf == NULL ? fl->fin : 0; - return 0; - } - - range->end = sf->range.end; - if (sf->data != NULL) - *data = sf->data + (start - sf->range.start); - else - *data = NULL; - *fin = sf->next == NULL ? fl->fin : 0; - *iter = sf; - return 1; -} - -int ossl_sframe_list_drop_frames(SFRAME_LIST *fl, uint64_t limit) -{ - STREAM_FRAME *sf; - - /* offset cannot move back or past the data received */ - if (!ossl_assert(limit >= fl->offset) - || !ossl_assert(fl->tail == NULL - || limit <= fl->tail->range.end) - || !ossl_assert(fl->tail != NULL - || limit == fl->offset)) - return 0; - - fl->offset = limit; - - for (sf = fl->head; sf != NULL && sf->range.end <= limit;) { - STREAM_FRAME *drop_frame = sf; - - sf = sf->next; - --fl->num_frames; - stream_frame_free(fl, drop_frame); - } - fl->head = sf; - - if (sf != NULL) - sf->prev = NULL; - else - fl->tail = NULL; - - fl->head_locked = 0; - - return 1; -} - -int ossl_sframe_list_lock_head(SFRAME_LIST *fl, UINT_RANGE *range, - const unsigned char **data, - int *fin) -{ - int ret; - void *iter = NULL; - - if (fl->head_locked) - return 0; - - ret = ossl_sframe_list_peek(fl, &iter, range, data, fin); - if (ret) - fl->head_locked = 1; - return ret; -} - -int ossl_sframe_list_is_head_locked(SFRAME_LIST *fl) -{ - return fl->head_locked; -} - -int ossl_sframe_list_move_data(SFRAME_LIST *fl, - sframe_list_write_at_cb *write_at_cb, - void *cb_arg) -{ - STREAM_FRAME *sf = fl->head, *prev_frame = NULL; - uint64_t limit = fl->offset; - - if (sf == NULL) - return 1; - - if (fl->head_locked) - sf = sf->next; - - for (; sf != NULL; sf = sf->next) { - size_t len; - const unsigned char *data = sf->data; - - if (limit < sf->range.start) - limit = sf->range.start; - - if (data != NULL) { - if (limit > sf->range.start) - data += (size_t)(limit - sf->range.start); - len = (size_t)(sf->range.end - limit); - - if (!write_at_cb(limit, data, len, cb_arg)) - /* data did not fit */ - return 0; - - if (fl->cleanse) - OPENSSL_cleanse((unsigned char *)sf->data, - (size_t)(sf->range.end - sf->range.start)); - - /* release the packet */ - sf->data = NULL; - ossl_qrx_pkt_release(sf->pkt); - sf->pkt = NULL; - } - - limit = sf->range.end; - - /* merge contiguous frames */ - if (prev_frame != NULL - && prev_frame->range.end >= sf->range.start) { - prev_frame->range.end = sf->range.end; - prev_frame->next = sf->next; - - if (sf->next != NULL) - sf->next->prev = prev_frame; - else - fl->tail = prev_frame; - - --fl->num_frames; - stream_frame_free(fl, sf); - sf = prev_frame; - continue; - } - - prev_frame = sf; - } - - return 1; -} diff --git a/openssl/src/ssl/quic/quic_srt_gen.c b/openssl/src/ssl/quic/quic_srt_gen.c deleted file mode 100644 index 233e4aa62..000000000 --- a/openssl/src/ssl/quic/quic_srt_gen.c +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#include "internal/quic_srt_gen.h" -#include -#include - -struct quic_srt_gen_st { - EVP_MAC *mac; - EVP_MAC_CTX *mac_ctx; -}; - -/* - * Simple HMAC-SHA256-based stateless reset token generator. - */ - -QUIC_SRT_GEN *ossl_quic_srt_gen_new(OSSL_LIB_CTX *libctx, const char *propq, - const unsigned char *key, size_t key_len) -{ - QUIC_SRT_GEN *srt_gen; - OSSL_PARAM params[3], *p = params; - - if ((srt_gen = OPENSSL_zalloc(sizeof(*srt_gen))) == NULL) - return NULL; - - if ((srt_gen->mac = EVP_MAC_fetch(libctx, "HMAC", propq)) == NULL) - goto err; - - if ((srt_gen->mac_ctx = EVP_MAC_CTX_new(srt_gen->mac)) == NULL) - goto err; - - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_MAC_PARAM_DIGEST, "SHA256", 7); - if (propq != NULL) - *p++ = OSSL_PARAM_construct_utf8_string(OSSL_MAC_PARAM_PROPERTIES, - (char *)propq, 0); - *p++ = OSSL_PARAM_construct_end(); - - if (!EVP_MAC_init(srt_gen->mac_ctx, key, key_len, params)) - goto err; - - return srt_gen; - -err: - ossl_quic_srt_gen_free(srt_gen); - return NULL; -} - -void ossl_quic_srt_gen_free(QUIC_SRT_GEN *srt_gen) -{ - if (srt_gen == NULL) - return; - - EVP_MAC_CTX_free(srt_gen->mac_ctx); - EVP_MAC_free(srt_gen->mac); - OPENSSL_free(srt_gen); -} - -int ossl_quic_srt_gen_calculate_token(QUIC_SRT_GEN *srt_gen, - const QUIC_CONN_ID *dcid, - QUIC_STATELESS_RESET_TOKEN *token) -{ - size_t outl = 0; - unsigned char mac[SHA256_DIGEST_LENGTH]; - - if (!EVP_MAC_init(srt_gen->mac_ctx, NULL, 0, NULL)) - return 0; - - if (!EVP_MAC_update(srt_gen->mac_ctx, (const unsigned char *)dcid->id, - dcid->id_len)) - return 0; - - if (!EVP_MAC_final(srt_gen->mac_ctx, mac, &outl, sizeof(mac)) - || outl != sizeof(mac)) - return 0; - - assert(sizeof(mac) >= sizeof(token->token)); - memcpy(token->token, mac, sizeof(token->token)); - return 1; -} diff --git a/openssl/src/ssl/quic/quic_srtm.c b/openssl/src/ssl/quic/quic_srtm.c deleted file mode 100644 index 3d0bfd97c..000000000 --- a/openssl/src/ssl/quic/quic_srtm.c +++ /dev/null @@ -1,565 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_srtm.h" -#include "internal/common.h" -#include -#include -#include - -/* - * QUIC Stateless Reset Token Manager - * ================================== - */ -typedef struct srtm_item_st SRTM_ITEM; - -#define BLINDED_SRT_LEN 16 - -DEFINE_LHASH_OF_EX(SRTM_ITEM); - -/* - * The SRTM is implemented using two LHASH instances, one matching opaque pointers to - * an item structure, and another matching a SRT-derived value to an item - * structure. Multiple items with different seq_num values under a given opaque, - * and duplicate SRTs, are handled using sorted singly-linked lists. - * - * The O(n) insert and lookup performance is tolerated on the basis that the - * total number of entries for a given opaque (total number of extant CIDs for a - * connection) should be quite small, and the QUIC protocol allows us to place a - * hard limit on this via the active_connection_id_limit TPARAM. Thus there is - * no risk of a large number of SRTs needing to be registered under a given - * opaque. - * - * It is expected one SRTM will exist per QUIC_PORT and track all SRTs across - * all connections for that QUIC_PORT. - */ -struct srtm_item_st { - SRTM_ITEM *next_by_srt_blinded; /* SORT BY opaque DESC */ - SRTM_ITEM *next_by_seq_num; /* SORT BY seq_num DESC */ - void *opaque; /* \__ unique identity for item */ - uint64_t seq_num; /* / */ - QUIC_STATELESS_RESET_TOKEN srt; - unsigned char srt_blinded[BLINDED_SRT_LEN]; /* H(srt) */ - -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - uint32_t debug_token; -#endif -}; - -struct quic_srtm_st { - /* Crypto context used to calculate blinded SRTs H(srt). */ - EVP_CIPHER_CTX *blind_ctx; /* kept with key */ - - LHASH_OF(SRTM_ITEM) *items_fwd; /* (opaque) -> SRTM_ITEM */ - LHASH_OF(SRTM_ITEM) *items_rev; /* (H(srt)) -> SRTM_ITEM */ - - /* - * Monotonically transitions to 1 in event of allocation failure. The only - * valid operation on such an object is to free it. - */ - unsigned int alloc_failed : 1; -}; - -static unsigned long items_fwd_hash(const SRTM_ITEM *item) -{ - return (unsigned long)(uintptr_t)item->opaque; -} - -static int items_fwd_cmp(const SRTM_ITEM *a, const SRTM_ITEM *b) -{ - return a->opaque != b->opaque; -} - -static unsigned long items_rev_hash(const SRTM_ITEM *item) -{ - /* - * srt_blinded has already been through a crypto-grade hash function, so we - * can just use bits from that. - */ - unsigned long l; - - memcpy(&l, item->srt_blinded, sizeof(l)); - return l; -} - -static int items_rev_cmp(const SRTM_ITEM *a, const SRTM_ITEM *b) -{ - /* - * We don't need to use CRYPTO_memcmp here as the relationship of - * srt_blinded to srt is already cryptographically obfuscated. - */ - return memcmp(a->srt_blinded, b->srt_blinded, sizeof(a->srt_blinded)); -} - -static int srtm_check_lh(QUIC_SRTM *srtm, LHASH_OF(SRTM_ITEM) *lh) -{ - if (lh_SRTM_ITEM_error(lh)) { - srtm->alloc_failed = 1; - return 0; - } - - return 1; -} - -QUIC_SRTM *ossl_quic_srtm_new(OSSL_LIB_CTX *libctx, const char *propq) -{ - QUIC_SRTM *srtm = NULL; - unsigned char key[16]; - EVP_CIPHER *ecb = NULL; - - if (RAND_priv_bytes_ex(libctx, key, sizeof(key), sizeof(key) * 8) != 1) - goto err; - - if ((srtm = OPENSSL_zalloc(sizeof(*srtm))) == NULL) - return NULL; - - /* Use AES-128-ECB as a permutation over 128-bit SRTs. */ - if ((ecb = EVP_CIPHER_fetch(libctx, "AES-128-ECB", propq)) == NULL) - goto err; - - if ((srtm->blind_ctx = EVP_CIPHER_CTX_new()) == NULL) - goto err; - - if (!EVP_EncryptInit_ex2(srtm->blind_ctx, ecb, key, NULL, NULL)) - goto err; - - EVP_CIPHER_free(ecb); - ecb = NULL; - - /* Create mappings. */ - if ((srtm->items_fwd = lh_SRTM_ITEM_new(items_fwd_hash, items_fwd_cmp)) == NULL - || (srtm->items_rev = lh_SRTM_ITEM_new(items_rev_hash, items_rev_cmp)) == NULL) - goto err; - - return srtm; - -err: - /* - * No cleansing of key needed as blinding exists only for side channel - * mitigation. - */ - ossl_quic_srtm_free(srtm); - EVP_CIPHER_free(ecb); - return NULL; -} - -static void srtm_free_each(SRTM_ITEM *ihead) -{ - SRTM_ITEM *inext, *item = ihead; - - for (item = item->next_by_seq_num; item != NULL; item = inext) { - inext = item->next_by_seq_num; - OPENSSL_free(item); - } - - OPENSSL_free(ihead); -} - -void ossl_quic_srtm_free(QUIC_SRTM *srtm) -{ - if (srtm == NULL) - return; - - lh_SRTM_ITEM_free(srtm->items_rev); - if (srtm->items_fwd != NULL) { - lh_SRTM_ITEM_doall(srtm->items_fwd, srtm_free_each); - lh_SRTM_ITEM_free(srtm->items_fwd); - } - - EVP_CIPHER_CTX_free(srtm->blind_ctx); - OPENSSL_free(srtm); -} - -/* - * Find a SRTM_ITEM by (opaque, seq_num). Returns NULL if no match. - * If head is non-NULL, writes the head of the relevant opaque list to *head if - * there is one. - * If prev is non-NULL, writes the previous node to *prev or NULL if it is - * the first item. - */ -static SRTM_ITEM *srtm_find(QUIC_SRTM *srtm, void *opaque, uint64_t seq_num, - SRTM_ITEM **head_p, SRTM_ITEM **prev_p) -{ - SRTM_ITEM key, *item = NULL, *prev = NULL; - - key.opaque = opaque; - - item = lh_SRTM_ITEM_retrieve(srtm->items_fwd, &key); - if (head_p != NULL) - *head_p = item; - - for (; item != NULL; prev = item, item = item->next_by_seq_num) - if (item->seq_num == seq_num) { - break; - } else if (item->seq_num < seq_num) { - /* - * List is sorted in descending order so there can't be any match - * after this. - */ - item = NULL; - break; - } - - if (prev_p != NULL) - *prev_p = prev; - - return item; -} - -/* - * Inserts a SRTM_ITEM into the singly-linked by-sequence-number linked list. - * The new head pointer is written to *new_head (which may or may not be - * unchanged). - */ -static void sorted_insert_seq_num(SRTM_ITEM *head, SRTM_ITEM *item, SRTM_ITEM **new_head) -{ - uint64_t seq_num = item->seq_num; - SRTM_ITEM *cur = head, **fixup = new_head; - - *new_head = head; - - while (cur != NULL && cur->seq_num > seq_num) { - fixup = &cur->next_by_seq_num; - cur = cur->next_by_seq_num; - } - - item->next_by_seq_num = *fixup; - *fixup = item; -} - -/* - * Inserts a SRTM_ITEM into the singly-linked by-SRT list. - * The new head pointer is written to *new_head (which may or may not be - * unchanged). - */ -static void sorted_insert_srt(SRTM_ITEM *head, SRTM_ITEM *item, SRTM_ITEM **new_head) -{ - uintptr_t opaque = (uintptr_t)item->opaque; - SRTM_ITEM *cur = head, **fixup = new_head; - - *new_head = head; - - while (cur != NULL && (uintptr_t)cur->opaque > opaque) { - fixup = &cur->next_by_srt_blinded; - cur = cur->next_by_srt_blinded; - } - - item->next_by_srt_blinded = *fixup; - *fixup = item; -} - -/* - * Computes the blinded SRT value used for internal lookup for side channel - * mitigation purposes. We compute this once as a cached value when an SRTM_ITEM - * is formed. - */ -static int srtm_compute_blinded(QUIC_SRTM *srtm, SRTM_ITEM *item, - const QUIC_STATELESS_RESET_TOKEN *token) -{ - int outl = 0; - - /* - * We use AES-128-ECB as a permutation using a random key to facilitate - * blinding for side-channel purposes. Encrypt the token as a single AES - * block. - */ - if (!EVP_EncryptUpdate(srtm->blind_ctx, item->srt_blinded, &outl, - (const unsigned char *)token, sizeof(*token))) - return 0; - - if (!ossl_assert(outl == sizeof(*token))) - return 0; - - return 1; -} - -int ossl_quic_srtm_add(QUIC_SRTM *srtm, void *opaque, uint64_t seq_num, - const QUIC_STATELESS_RESET_TOKEN *token) -{ - SRTM_ITEM *item = NULL, *head = NULL, *new_head, *r_item; - - if (srtm->alloc_failed) - return 0; - - /* (opaque, seq_num) duplicates not allowed */ - if ((item = srtm_find(srtm, opaque, seq_num, &head, NULL)) != NULL) - return 0; - - if ((item = OPENSSL_zalloc(sizeof(*item))) == NULL) - return 0; - - item->opaque = opaque; - item->seq_num = seq_num; - item->srt = *token; - if (!srtm_compute_blinded(srtm, item, &item->srt)) { - OPENSSL_free(item); - return 0; - } - - /* Add to forward mapping. */ - if (head == NULL) { - /* First item under this opaque */ - lh_SRTM_ITEM_insert(srtm->items_fwd, item); - if (!srtm_check_lh(srtm, srtm->items_fwd)) { - OPENSSL_free(item); - return 0; - } - } else { - sorted_insert_seq_num(head, item, &new_head); - if (new_head != head) { /* head changed, update in lhash */ - lh_SRTM_ITEM_insert(srtm->items_fwd, new_head); - if (!srtm_check_lh(srtm, srtm->items_fwd)) { - OPENSSL_free(item); - return 0; - } - } - } - - /* Add to reverse mapping. */ - r_item = lh_SRTM_ITEM_retrieve(srtm->items_rev, item); - if (r_item == NULL) { - /* First item under this blinded SRT */ - lh_SRTM_ITEM_insert(srtm->items_rev, item); - if (!srtm_check_lh(srtm, srtm->items_rev)) - /* - * Can't free the item now as we would have to undo the insertion - * into the forward mapping which would require an insert operation - * to restore the previous value. which might also fail. However, - * the item will be freed OK when we free the entire SRTM. - */ - return 0; - } else { - sorted_insert_srt(r_item, item, &new_head); - if (new_head != r_item) { /* head changed, update in lhash */ - lh_SRTM_ITEM_insert(srtm->items_rev, new_head); - if (!srtm_check_lh(srtm, srtm->items_rev)) - /* As above. */ - return 0; - } - } - - return 1; -} - -/* Remove item from reverse mapping. */ -static int srtm_remove_from_rev(QUIC_SRTM *srtm, SRTM_ITEM *item) -{ - SRTM_ITEM *rh_item; - - rh_item = lh_SRTM_ITEM_retrieve(srtm->items_rev, item); - assert(rh_item != NULL); - if (rh_item == item) { - /* - * Change lhash to point to item after this one, or remove the entry if - * this is the last one. - */ - if (item->next_by_srt_blinded != NULL) { - lh_SRTM_ITEM_insert(srtm->items_rev, item->next_by_srt_blinded); - if (!srtm_check_lh(srtm, srtm->items_rev)) - return 0; - } else { - lh_SRTM_ITEM_delete(srtm->items_rev, item); - } - } else { - /* Find our entry in the SRT list */ - for (; rh_item->next_by_srt_blinded != item; - rh_item = rh_item->next_by_srt_blinded); - rh_item->next_by_srt_blinded = item->next_by_srt_blinded; - } - - return 1; -} - -int ossl_quic_srtm_remove(QUIC_SRTM *srtm, void *opaque, uint64_t seq_num) -{ - SRTM_ITEM *item, *prev = NULL; - - if (srtm->alloc_failed) - return 0; - - if ((item = srtm_find(srtm, opaque, seq_num, NULL, &prev)) == NULL) - /* No match */ - return 0; - - /* Remove from forward mapping. */ - if (prev == NULL) { - /* - * Change lhash to point to item after this one, or remove the entry if - * this is the last one. - */ - if (item->next_by_seq_num != NULL) { - lh_SRTM_ITEM_insert(srtm->items_fwd, item->next_by_seq_num); - if (!srtm_check_lh(srtm, srtm->items_fwd)) - return 0; - } else { - lh_SRTM_ITEM_delete(srtm->items_fwd, item); - } - } else { - prev->next_by_seq_num = item->next_by_seq_num; - } - - /* Remove from reverse mapping. */ - if (!srtm_remove_from_rev(srtm, item)) - return 0; - - OPENSSL_free(item); - return 1; -} - -int ossl_quic_srtm_cull(QUIC_SRTM *srtm, void *opaque) -{ - SRTM_ITEM key, *item = NULL, *inext, *ihead; - - key.opaque = opaque; - - if (srtm->alloc_failed) - return 0; - - if ((ihead = lh_SRTM_ITEM_retrieve(srtm->items_fwd, &key)) == NULL) - return 1; /* nothing removed is a success condition */ - - for (item = ihead; item != NULL; item = inext) { - inext = item->next_by_seq_num; - if (item != ihead) { - srtm_remove_from_rev(srtm, item); - OPENSSL_free(item); - } - } - - lh_SRTM_ITEM_delete(srtm->items_fwd, ihead); - srtm_remove_from_rev(srtm, ihead); - OPENSSL_free(ihead); - return 1; -} - -int ossl_quic_srtm_lookup(QUIC_SRTM *srtm, - const QUIC_STATELESS_RESET_TOKEN *token, - size_t idx, - void **opaque, uint64_t *seq_num) -{ - SRTM_ITEM key, *item; - - if (srtm->alloc_failed) - return 0; - - if (!srtm_compute_blinded(srtm, &key, token)) - return 0; - - item = lh_SRTM_ITEM_retrieve(srtm->items_rev, &key); - for (; idx > 0 && item != NULL; --idx, item = item->next_by_srt_blinded); - if (item == NULL) - return 0; - - if (opaque != NULL) - *opaque = item->opaque; - if (seq_num != NULL) - *seq_num = item->seq_num; - - return 1; -} - -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - -static uint32_t token_next = 0x5eadbeef; -static size_t tokens_seen; - -struct check_args { - uint32_t token; - int mode; -}; - -static void check_mark(SRTM_ITEM *item, void *arg) -{ - struct check_args *arg_ = arg; - uint32_t token = arg_->token; - uint64_t prev_seq_num = 0; - void *prev_opaque = NULL; - int have_prev = 0; - - assert(item != NULL); - - while (item != NULL) { - if (have_prev) { - assert(!(item->opaque == prev_opaque && item->seq_num == prev_seq_num)); - if (!arg_->mode) - assert(item->opaque != prev_opaque || item->seq_num < prev_seq_num); - } - - ++tokens_seen; - item->debug_token = token; - prev_opaque = item->opaque; - prev_seq_num = item->seq_num; - have_prev = 1; - - if (arg_->mode) - item = item->next_by_srt_blinded; - else - item = item->next_by_seq_num; - } -} - -static void check_count(SRTM_ITEM *item, void *arg) -{ - struct check_args *arg_ = arg; - uint32_t token = arg_->token; - - assert(item != NULL); - - while (item != NULL) { - ++tokens_seen; - assert(item->debug_token == token); - - if (arg_->mode) - item = item->next_by_seq_num; - else - item = item->next_by_srt_blinded; - } -} - -#endif - -void ossl_quic_srtm_check(const QUIC_SRTM *srtm) -{ -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - struct check_args args = {0}; - size_t tokens_expected, tokens_expected_old; - - args.token = token_next; - ++token_next; - - assert(srtm != NULL); - assert(srtm->blind_ctx != NULL); - assert(srtm->items_fwd != NULL); - assert(srtm->items_rev != NULL); - - tokens_seen = 0; - lh_SRTM_ITEM_doall_arg(srtm->items_fwd, check_mark, &args); - - tokens_expected = tokens_seen; - tokens_seen = 0; - lh_SRTM_ITEM_doall_arg(srtm->items_rev, check_count, &args); - - assert(tokens_seen == tokens_expected); - tokens_expected_old = tokens_expected; - - args.token = token_next; - ++token_next; - - args.mode = 1; - tokens_seen = 0; - lh_SRTM_ITEM_doall_arg(srtm->items_rev, check_mark, &args); - - tokens_expected = tokens_seen; - tokens_seen = 0; - lh_SRTM_ITEM_doall_arg(srtm->items_fwd, check_count, &args); - - assert(tokens_seen == tokens_expected); - assert(tokens_seen == tokens_expected_old); -#endif -} diff --git a/openssl/src/ssl/quic/quic_sstream.c b/openssl/src/ssl/quic/quic_sstream.c deleted file mode 100644 index 1f0b5497f..000000000 --- a/openssl/src/ssl/quic/quic_sstream.c +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_stream.h" -#include "internal/uint_set.h" -#include "internal/common.h" -#include "internal/ring_buf.h" - -/* - * ================================================================== - * QUIC Send Stream - */ -struct quic_sstream_st { - struct ring_buf ring_buf; - - /* - * Any logical byte in the stream is in one of these states: - * - * - NEW: The byte has not yet been transmitted, or has been lost and is - * in need of retransmission. - * - * - IN_FLIGHT: The byte has been transmitted but is awaiting - * acknowledgement. We continue to store the data in case we return - * to the NEW state. - * - * - ACKED: The byte has been acknowledged and we can cease storing it. - * We do not necessarily cull it immediately, so there may be a delay - * between reaching the ACKED state and the buffer space actually being - * recycled. - * - * A logical byte in the stream is - * - * - in the NEW state if it is in new_set; - * - is in the ACKED state if it is in acked_set - * (and may or may not have been culled); - * - is in the IN_FLIGHT state otherwise. - * - * Invariant: No logical byte is ever in both new_set and acked_set. - */ - UINT_SET new_set, acked_set; - - /* - * The current size of the stream is ring_buf.head_offset. If - * have_final_size is true, this is also the final size of the stream. - */ - unsigned int have_final_size : 1; - unsigned int sent_final_size : 1; - unsigned int acked_final_size : 1; - unsigned int cleanse : 1; -}; - -static void qss_cull(QUIC_SSTREAM *qss); - -QUIC_SSTREAM *ossl_quic_sstream_new(size_t init_buf_size) -{ - QUIC_SSTREAM *qss; - - qss = OPENSSL_zalloc(sizeof(QUIC_SSTREAM)); - if (qss == NULL) - return NULL; - - ring_buf_init(&qss->ring_buf); - if (!ring_buf_resize(&qss->ring_buf, init_buf_size, 0)) { - ring_buf_destroy(&qss->ring_buf, 0); - OPENSSL_free(qss); - return NULL; - } - - ossl_uint_set_init(&qss->new_set); - ossl_uint_set_init(&qss->acked_set); - return qss; -} - -void ossl_quic_sstream_free(QUIC_SSTREAM *qss) -{ - if (qss == NULL) - return; - - ossl_uint_set_destroy(&qss->new_set); - ossl_uint_set_destroy(&qss->acked_set); - ring_buf_destroy(&qss->ring_buf, qss->cleanse); - OPENSSL_free(qss); -} - -int ossl_quic_sstream_get_stream_frame(QUIC_SSTREAM *qss, - size_t skip, - OSSL_QUIC_FRAME_STREAM *hdr, - OSSL_QTX_IOVEC *iov, - size_t *num_iov) -{ - size_t num_iov_ = 0, src_len = 0, total_len = 0, i; - uint64_t max_len; - const unsigned char *src = NULL; - UINT_SET_ITEM *range = ossl_list_uint_set_head(&qss->new_set); - - if (*num_iov < 2) - return 0; - - for (i = 0; i < skip && range != NULL; ++i) - range = ossl_list_uint_set_next(range); - - if (range == NULL) { - if (i < skip) - /* Don't return FIN for infinitely increasing skip */ - return 0; - - /* No new bytes to send, but we might have a FIN */ - if (!qss->have_final_size || qss->sent_final_size) - return 0; - - hdr->offset = qss->ring_buf.head_offset; - hdr->len = 0; - hdr->is_fin = 1; - *num_iov = 0; - return 1; - } - - /* - * We can only send a contiguous range of logical bytes in a single - * stream frame, so limit ourselves to the range of the first set entry. - * - * Set entries never have 'adjacent' entries so we don't have to worry - * about them here. - */ - max_len = range->range.end - range->range.start + 1; - - for (i = 0;; ++i) { - if (total_len >= max_len) - break; - - if (!ring_buf_get_buf_at(&qss->ring_buf, - range->range.start + total_len, - &src, &src_len)) - return 0; - - if (src_len == 0) - break; - - assert(i < 2); - - if (total_len + src_len > max_len) - src_len = (size_t)(max_len - total_len); - - iov[num_iov_].buf = src; - iov[num_iov_].buf_len = src_len; - - total_len += src_len; - ++num_iov_; - } - - hdr->offset = range->range.start; - hdr->len = total_len; - hdr->is_fin = qss->have_final_size - && hdr->offset + hdr->len == qss->ring_buf.head_offset; - - *num_iov = num_iov_; - return 1; -} - -int ossl_quic_sstream_has_pending(QUIC_SSTREAM *qss) -{ - OSSL_QUIC_FRAME_STREAM shdr; - OSSL_QTX_IOVEC iov[2]; - size_t num_iov = OSSL_NELEM(iov); - - return ossl_quic_sstream_get_stream_frame(qss, 0, &shdr, iov, &num_iov); -} - -uint64_t ossl_quic_sstream_get_cur_size(QUIC_SSTREAM *qss) -{ - return qss->ring_buf.head_offset; -} - -int ossl_quic_sstream_mark_transmitted(QUIC_SSTREAM *qss, - uint64_t start, - uint64_t end) -{ - UINT_RANGE r; - - r.start = start; - r.end = end; - - if (!ossl_uint_set_remove(&qss->new_set, &r)) - return 0; - - return 1; -} - -int ossl_quic_sstream_mark_transmitted_fin(QUIC_SSTREAM *qss, - uint64_t final_size) -{ - /* - * We do not really need final_size since we already know the size of the - * stream, but this serves as a sanity check. - */ - if (!qss->have_final_size || final_size != qss->ring_buf.head_offset) - return 0; - - qss->sent_final_size = 1; - return 1; -} - -int ossl_quic_sstream_mark_lost(QUIC_SSTREAM *qss, - uint64_t start, - uint64_t end) -{ - UINT_RANGE r; - r.start = start; - r.end = end; - - /* - * We lost a range of stream data bytes, so reinsert them into the new set, - * so that they are returned once more by ossl_quic_sstream_get_stream_frame. - */ - if (!ossl_uint_set_insert(&qss->new_set, &r)) - return 0; - - return 1; -} - -int ossl_quic_sstream_mark_lost_fin(QUIC_SSTREAM *qss) -{ - if (qss->acked_final_size) - /* Does not make sense to lose a FIN after it has been ACKed */ - return 0; - - /* FIN was lost, so we need to transmit it again. */ - qss->sent_final_size = 0; - return 1; -} - -int ossl_quic_sstream_mark_acked(QUIC_SSTREAM *qss, - uint64_t start, - uint64_t end) -{ - UINT_RANGE r; - r.start = start; - r.end = end; - - if (!ossl_uint_set_insert(&qss->acked_set, &r)) - return 0; - - qss_cull(qss); - return 1; -} - -int ossl_quic_sstream_mark_acked_fin(QUIC_SSTREAM *qss) -{ - if (!qss->have_final_size) - /* Cannot ack final size before we have a final size */ - return 0; - - qss->acked_final_size = 1; - return 1; -} - -void ossl_quic_sstream_fin(QUIC_SSTREAM *qss) -{ - if (qss->have_final_size) - return; - - qss->have_final_size = 1; -} - -int ossl_quic_sstream_get_final_size(QUIC_SSTREAM *qss, uint64_t *final_size) -{ - if (!qss->have_final_size) - return 0; - - if (final_size != NULL) - *final_size = qss->ring_buf.head_offset; - - return 1; -} - -int ossl_quic_sstream_append(QUIC_SSTREAM *qss, - const unsigned char *buf, - size_t buf_len, - size_t *consumed) -{ - size_t l, consumed_ = 0; - UINT_RANGE r; - struct ring_buf old_ring_buf = qss->ring_buf; - - if (qss->have_final_size) { - *consumed = 0; - return 0; - } - - /* - * Note: It is assumed that ossl_quic_sstream_append will be called during a - * call to e.g. SSL_write and this function is therefore designed to support - * such semantics. In particular, the buffer pointed to by buf is only - * assumed to be valid for the duration of this call, therefore we must copy - * the data here. We will later copy-and-encrypt the data during packet - * encryption, so this is a two-copy design. Supporting a one-copy design in - * the future will require applications to use a different kind of API. - * Supporting such changes in future will require corresponding enhancements - * to this code. - */ - while (buf_len > 0) { - l = ring_buf_push(&qss->ring_buf, buf, buf_len); - if (l == 0) - break; - - buf += l; - buf_len -= l; - consumed_ += l; - } - - if (consumed_ > 0) { - r.start = old_ring_buf.head_offset; - r.end = r.start + consumed_ - 1; - assert(r.end + 1 == qss->ring_buf.head_offset); - if (!ossl_uint_set_insert(&qss->new_set, &r)) { - qss->ring_buf = old_ring_buf; - *consumed = 0; - return 0; - } - } - - *consumed = consumed_; - return 1; -} - -static void qss_cull(QUIC_SSTREAM *qss) -{ - UINT_SET_ITEM *h = ossl_list_uint_set_head(&qss->acked_set); - - /* - * Potentially cull data from our ring buffer. This can happen once data has - * been ACKed and we know we are never going to have to transmit it again. - * - * Since we use a ring buffer design for simplicity, we cannot cull byte n + - * k (for k > 0) from the ring buffer until byte n has also been culled. - * This means if parts of the stream get acknowledged out of order we might - * keep around some data we technically don't need to for a while. The - * impact of this is likely to be small and limited to quite a short - * duration, and doesn't justify the use of a more complex design. - */ - - /* - * We only need to check the first range entry in the integer set because we - * can only cull contiguous areas at the start of the ring buffer anyway. - */ - if (h != NULL) - ring_buf_cpop_range(&qss->ring_buf, h->range.start, h->range.end, - qss->cleanse); -} - -int ossl_quic_sstream_set_buffer_size(QUIC_SSTREAM *qss, size_t num_bytes) -{ - return ring_buf_resize(&qss->ring_buf, num_bytes, qss->cleanse); -} - -size_t ossl_quic_sstream_get_buffer_size(QUIC_SSTREAM *qss) -{ - return qss->ring_buf.alloc; -} - -size_t ossl_quic_sstream_get_buffer_used(QUIC_SSTREAM *qss) -{ - return ring_buf_used(&qss->ring_buf); -} - -size_t ossl_quic_sstream_get_buffer_avail(QUIC_SSTREAM *qss) -{ - return ring_buf_avail(&qss->ring_buf); -} - -int ossl_quic_sstream_is_totally_acked(QUIC_SSTREAM *qss) -{ - UINT_RANGE r; - uint64_t cur_size; - - if (qss->have_final_size && !qss->acked_final_size) - return 0; - - if (ossl_quic_sstream_get_cur_size(qss) == 0) - return 1; - - if (ossl_list_uint_set_num(&qss->acked_set) != 1) - return 0; - - r = ossl_list_uint_set_head(&qss->acked_set)->range; - cur_size = qss->ring_buf.head_offset; - - /* - * The invariants of UINT_SET guarantee a single list element if we have a - * single contiguous range, which is what we should have if everything has - * been acked. - */ - assert(r.end + 1 <= cur_size); - return r.start == 0 && r.end + 1 == cur_size; -} - -void ossl_quic_sstream_adjust_iov(size_t len, - OSSL_QTX_IOVEC *iov, - size_t num_iov) -{ - size_t running = 0, i, iovlen; - - for (i = 0, running = 0; i < num_iov; ++i) { - iovlen = iov[i].buf_len; - - if (running >= len) - iov[i].buf_len = 0; - else if (running + iovlen > len) - iov[i].buf_len = len - running; - - running += iovlen; - } -} - -void ossl_quic_sstream_set_cleanse(QUIC_SSTREAM *qss, int cleanse) -{ - qss->cleanse = cleanse; -} diff --git a/openssl/src/ssl/quic/quic_statm.c b/openssl/src/ssl/quic/quic_statm.c deleted file mode 100644 index f1e0f6591..000000000 --- a/openssl/src/ssl/quic/quic_statm.c +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_statm.h" - -void ossl_statm_update_rtt(OSSL_STATM *statm, - OSSL_TIME ack_delay, - OSSL_TIME override_latest_rtt) -{ - OSSL_TIME adjusted_rtt, latest_rtt = override_latest_rtt; - - /* Use provided RTT value, or else last RTT value. */ - if (ossl_time_is_zero(latest_rtt)) - latest_rtt = statm->latest_rtt; - else - statm->latest_rtt = latest_rtt; - - if (!statm->have_first_sample) { - statm->min_rtt = latest_rtt; - statm->smoothed_rtt = latest_rtt; - statm->rtt_variance = ossl_time_divide(latest_rtt, 2); - statm->have_first_sample = 1; - return; - } - - /* Update minimum RTT. */ - if (ossl_time_compare(latest_rtt, statm->min_rtt) < 0) - statm->min_rtt = latest_rtt; - - /* - * Enforcement of max_ack_delay is the responsibility of - * the caller as it is context-dependent. - */ - - adjusted_rtt = latest_rtt; - if (ossl_time_compare(latest_rtt, ossl_time_add(statm->min_rtt, ack_delay)) >= 0) - adjusted_rtt = ossl_time_subtract(latest_rtt, ack_delay); - - statm->rtt_variance = ossl_time_divide(ossl_time_add(ossl_time_multiply(statm->rtt_variance, 3), - ossl_time_abs_difference(statm->smoothed_rtt, - adjusted_rtt)), 4); - statm->smoothed_rtt = ossl_time_divide(ossl_time_add(ossl_time_multiply(statm->smoothed_rtt, 7), - adjusted_rtt), 8); -} - -/* RFC 9002 kInitialRtt value. RFC recommended value. */ -#define K_INITIAL_RTT ossl_ms2time(333) - -int ossl_statm_init(OSSL_STATM *statm) -{ - statm->smoothed_rtt = K_INITIAL_RTT; - statm->latest_rtt = ossl_time_zero(); - statm->min_rtt = ossl_time_infinite(); - statm->rtt_variance = ossl_time_divide(K_INITIAL_RTT, 2); - statm->have_first_sample = 0; - return 1; -} - -void ossl_statm_destroy(OSSL_STATM *statm) -{ - /* No-op. */ -} - -void ossl_statm_get_rtt_info(OSSL_STATM *statm, OSSL_RTT_INFO *rtt_info) -{ - rtt_info->min_rtt = statm->min_rtt; - rtt_info->latest_rtt = statm->latest_rtt; - rtt_info->smoothed_rtt = statm->smoothed_rtt; - rtt_info->rtt_variance = statm->rtt_variance; -} diff --git a/openssl/src/ssl/quic/quic_stream_map.c b/openssl/src/ssl/quic/quic_stream_map.c deleted file mode 100644 index c836721f7..000000000 --- a/openssl/src/ssl/quic/quic_stream_map.c +++ /dev/null @@ -1,861 +0,0 @@ -/* -* Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. -* -* Licensed under the Apache License 2.0 (the "License"). You may not use -* this file except in compliance with the License. You can obtain a copy -* in the file LICENSE in the source distribution or at -* https://www.openssl.org/source/license.html -*/ - -#include "internal/quic_stream_map.h" -#include "internal/nelem.h" - -/* - * QUIC Stream Map - * =============== - */ -DEFINE_LHASH_OF_EX(QUIC_STREAM); - -static void shutdown_flush_done(QUIC_STREAM_MAP *qsm, QUIC_STREAM *qs); - -/* Circular list management. */ -static void list_insert_tail(QUIC_STREAM_LIST_NODE *l, - QUIC_STREAM_LIST_NODE *n) -{ - /* Must not be in list. */ - assert(n->prev == NULL && n->next == NULL - && l->prev != NULL && l->next != NULL); - - n->prev = l->prev; - n->prev->next = n; - l->prev = n; - n->next = l; -} - -static void list_remove(QUIC_STREAM_LIST_NODE *l, - QUIC_STREAM_LIST_NODE *n) -{ - assert(n->prev != NULL && n->next != NULL - && n->prev != n && n->next != n); - - n->prev->next = n->next; - n->next->prev = n->prev; - n->next = n->prev = NULL; -} - -static QUIC_STREAM *list_next(QUIC_STREAM_LIST_NODE *l, QUIC_STREAM_LIST_NODE *n, - size_t off) -{ - assert(n->prev != NULL && n->next != NULL - && (n == l || (n->prev != n && n->next != n)) - && l->prev != NULL && l->next != NULL); - - n = n->next; - - if (n == l) - n = n->next; - if (n == l) - return NULL; - - assert(n != NULL); - - return (QUIC_STREAM *)(((char *)n) - off); -} - -#define active_next(l, s) list_next((l), &(s)->active_node, \ - offsetof(QUIC_STREAM, active_node)) -#define accept_next(l, s) list_next((l), &(s)->accept_node, \ - offsetof(QUIC_STREAM, accept_node)) -#define ready_for_gc_next(l, s) list_next((l), &(s)->ready_for_gc_node, \ - offsetof(QUIC_STREAM, ready_for_gc_node)) -#define accept_head(l) list_next((l), (l), \ - offsetof(QUIC_STREAM, accept_node)) -#define ready_for_gc_head(l) list_next((l), (l), \ - offsetof(QUIC_STREAM, ready_for_gc_node)) - -static unsigned long hash_stream(const QUIC_STREAM *s) -{ - return (unsigned long)s->id; -} - -static int cmp_stream(const QUIC_STREAM *a, const QUIC_STREAM *b) -{ - if (a->id < b->id) - return -1; - if (a->id > b->id) - return 1; - return 0; -} - -int ossl_quic_stream_map_init(QUIC_STREAM_MAP *qsm, - uint64_t (*get_stream_limit_cb)(int uni, void *arg), - void *get_stream_limit_cb_arg, - QUIC_RXFC *max_streams_bidi_rxfc, - QUIC_RXFC *max_streams_uni_rxfc, - int is_server) -{ - qsm->map = lh_QUIC_STREAM_new(hash_stream, cmp_stream); - qsm->active_list.prev = qsm->active_list.next = &qsm->active_list; - qsm->accept_list.prev = qsm->accept_list.next = &qsm->accept_list; - qsm->ready_for_gc_list.prev = qsm->ready_for_gc_list.next - = &qsm->ready_for_gc_list; - qsm->rr_stepping = 1; - qsm->rr_counter = 0; - qsm->rr_cur = NULL; - - qsm->num_accept_bidi = 0; - qsm->num_accept_uni = 0; - qsm->num_shutdown_flush = 0; - - qsm->get_stream_limit_cb = get_stream_limit_cb; - qsm->get_stream_limit_cb_arg = get_stream_limit_cb_arg; - qsm->max_streams_bidi_rxfc = max_streams_bidi_rxfc; - qsm->max_streams_uni_rxfc = max_streams_uni_rxfc; - qsm->is_server = is_server; - return 1; -} - -static void release_each(QUIC_STREAM *stream, void *arg) -{ - QUIC_STREAM_MAP *qsm = arg; - - ossl_quic_stream_map_release(qsm, stream); -} - -void ossl_quic_stream_map_cleanup(QUIC_STREAM_MAP *qsm) -{ - ossl_quic_stream_map_visit(qsm, release_each, qsm); - - lh_QUIC_STREAM_free(qsm->map); - qsm->map = NULL; -} - -void ossl_quic_stream_map_visit(QUIC_STREAM_MAP *qsm, - void (*visit_cb)(QUIC_STREAM *stream, void *arg), - void *visit_cb_arg) -{ - lh_QUIC_STREAM_doall_arg(qsm->map, visit_cb, visit_cb_arg); -} - -QUIC_STREAM *ossl_quic_stream_map_alloc(QUIC_STREAM_MAP *qsm, - uint64_t stream_id, - int type) -{ - QUIC_STREAM *s; - QUIC_STREAM key; - - key.id = stream_id; - - s = lh_QUIC_STREAM_retrieve(qsm->map, &key); - if (s != NULL) - return NULL; - - s = OPENSSL_zalloc(sizeof(*s)); - if (s == NULL) - return NULL; - - s->id = stream_id; - s->type = type; - s->as_server = qsm->is_server; - s->send_state = (ossl_quic_stream_is_local_init(s) - || ossl_quic_stream_is_bidi(s)) - ? QUIC_SSTREAM_STATE_READY - : QUIC_SSTREAM_STATE_NONE; - s->recv_state = (!ossl_quic_stream_is_local_init(s) - || ossl_quic_stream_is_bidi(s)) - ? QUIC_RSTREAM_STATE_RECV - : QUIC_RSTREAM_STATE_NONE; - - s->send_final_size = UINT64_MAX; - - lh_QUIC_STREAM_insert(qsm->map, s); - return s; -} - -void ossl_quic_stream_map_release(QUIC_STREAM_MAP *qsm, QUIC_STREAM *stream) -{ - if (stream == NULL) - return; - - if (stream->active_node.next != NULL) - list_remove(&qsm->active_list, &stream->active_node); - if (stream->accept_node.next != NULL) - list_remove(&qsm->accept_list, &stream->accept_node); - if (stream->ready_for_gc_node.next != NULL) - list_remove(&qsm->ready_for_gc_list, &stream->ready_for_gc_node); - - ossl_quic_sstream_free(stream->sstream); - stream->sstream = NULL; - - ossl_quic_rstream_free(stream->rstream); - stream->rstream = NULL; - - lh_QUIC_STREAM_delete(qsm->map, stream); - OPENSSL_free(stream); -} - -QUIC_STREAM *ossl_quic_stream_map_get_by_id(QUIC_STREAM_MAP *qsm, - uint64_t stream_id) -{ - QUIC_STREAM key; - - key.id = stream_id; - - return lh_QUIC_STREAM_retrieve(qsm->map, &key); -} - -static void stream_map_mark_active(QUIC_STREAM_MAP *qsm, QUIC_STREAM *s) -{ - if (s->active) - return; - - list_insert_tail(&qsm->active_list, &s->active_node); - - if (qsm->rr_cur == NULL) - qsm->rr_cur = s; - - s->active = 1; -} - -static void stream_map_mark_inactive(QUIC_STREAM_MAP *qsm, QUIC_STREAM *s) -{ - if (!s->active) - return; - - if (qsm->rr_cur == s) - qsm->rr_cur = active_next(&qsm->active_list, s); - if (qsm->rr_cur == s) - qsm->rr_cur = NULL; - - list_remove(&qsm->active_list, &s->active_node); - - s->active = 0; -} - -void ossl_quic_stream_map_set_rr_stepping(QUIC_STREAM_MAP *qsm, size_t stepping) -{ - qsm->rr_stepping = stepping; - qsm->rr_counter = 0; -} - -static int stream_has_data_to_send(QUIC_STREAM *s) -{ - OSSL_QUIC_FRAME_STREAM shdr; - OSSL_QTX_IOVEC iov[2]; - size_t num_iov; - uint64_t fc_credit, fc_swm, fc_limit; - - switch (s->send_state) { - case QUIC_SSTREAM_STATE_READY: - case QUIC_SSTREAM_STATE_SEND: - case QUIC_SSTREAM_STATE_DATA_SENT: - /* - * We can still have data to send in DATA_SENT due to retransmissions, - * etc. - */ - break; - default: - return 0; /* Nothing to send. */ - } - - /* - * We cannot determine if we have data to send simply by checking if - * ossl_quic_txfc_get_credit() is zero, because we may also have older - * stream data we need to retransmit. The SSTREAM returns older data first, - * so we do a simple comparison of the next chunk the SSTREAM wants to send - * against the TXFC CWM. - */ - num_iov = OSSL_NELEM(iov); - if (!ossl_quic_sstream_get_stream_frame(s->sstream, 0, &shdr, iov, - &num_iov)) - return 0; - - fc_credit = ossl_quic_txfc_get_credit(&s->txfc, 0); - fc_swm = ossl_quic_txfc_get_swm(&s->txfc); - fc_limit = fc_swm + fc_credit; - - return (shdr.is_fin && shdr.len == 0) || shdr.offset < fc_limit; -} - -static ossl_unused int qsm_send_part_permits_gc(const QUIC_STREAM *qs) -{ - switch (qs->send_state) { - case QUIC_SSTREAM_STATE_NONE: - case QUIC_SSTREAM_STATE_DATA_RECVD: - case QUIC_SSTREAM_STATE_RESET_RECVD: - return 1; - default: - return 0; - } -} - -static int qsm_ready_for_gc(QUIC_STREAM_MAP *qsm, QUIC_STREAM *qs) -{ - int recv_stream_fully_drained = 0; /* TODO(QUIC FUTURE): Optimisation */ - - /* - * If sstream has no FIN, we auto-reset it at marked-for-deletion time, so - * we don't need to worry about that here. - */ - assert(!qs->deleted - || !ossl_quic_stream_has_send(qs) - || ossl_quic_stream_send_is_reset(qs) - || ossl_quic_stream_send_get_final_size(qs, NULL)); - - return - qs->deleted - && (!ossl_quic_stream_has_recv(qs) - || recv_stream_fully_drained - || qs->acked_stop_sending) - && (!ossl_quic_stream_has_send(qs) - || qs->send_state == QUIC_SSTREAM_STATE_DATA_RECVD - || qs->send_state == QUIC_SSTREAM_STATE_RESET_RECVD); -} - -int ossl_quic_stream_map_is_local_allowed_by_stream_limit(QUIC_STREAM_MAP *qsm, - uint64_t stream_ordinal, - int is_uni) -{ - uint64_t stream_limit; - - if (qsm->get_stream_limit_cb == NULL) - return 1; - - stream_limit = qsm->get_stream_limit_cb(is_uni, qsm->get_stream_limit_cb_arg); - return stream_ordinal < stream_limit; -} - -void ossl_quic_stream_map_update_state(QUIC_STREAM_MAP *qsm, QUIC_STREAM *s) -{ - int should_be_active, allowed_by_stream_limit = 1; - - if (ossl_quic_stream_is_server_init(s) == qsm->is_server) { - int is_uni = !ossl_quic_stream_is_bidi(s); - uint64_t stream_ordinal = s->id >> 2; - - allowed_by_stream_limit - = ossl_quic_stream_map_is_local_allowed_by_stream_limit(qsm, - stream_ordinal, - is_uni); - } - - if (s->send_state == QUIC_SSTREAM_STATE_DATA_SENT - && ossl_quic_sstream_is_totally_acked(s->sstream)) - ossl_quic_stream_map_notify_totally_acked(qsm, s); - else if (s->shutdown_flush - && s->send_state == QUIC_SSTREAM_STATE_SEND - && ossl_quic_sstream_is_totally_acked(s->sstream)) - shutdown_flush_done(qsm, s); - - if (!s->ready_for_gc) { - s->ready_for_gc = qsm_ready_for_gc(qsm, s); - if (s->ready_for_gc) - list_insert_tail(&qsm->ready_for_gc_list, &s->ready_for_gc_node); - } - - should_be_active - = allowed_by_stream_limit - && !s->ready_for_gc - && ((ossl_quic_stream_has_recv(s) - && !ossl_quic_stream_recv_is_reset(s) - && (s->recv_state == QUIC_RSTREAM_STATE_RECV - && (s->want_max_stream_data - || ossl_quic_rxfc_has_cwm_changed(&s->rxfc, 0)))) - || s->want_stop_sending - || s->want_reset_stream - || (!s->peer_stop_sending && stream_has_data_to_send(s))); - - if (should_be_active) - stream_map_mark_active(qsm, s); - else - stream_map_mark_inactive(qsm, s); -} - -/* - * Stream Send Part State Management - * ================================= - */ - -int ossl_quic_stream_map_ensure_send_part_id(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs) -{ - switch (qs->send_state) { - case QUIC_SSTREAM_STATE_NONE: - /* Stream without send part - caller error. */ - return 0; - - case QUIC_SSTREAM_STATE_READY: - /* - * We always allocate a stream ID upfront, so we don't need to do it - * here. - */ - qs->send_state = QUIC_SSTREAM_STATE_SEND; - return 1; - - default: - /* Nothing to do. */ - return 1; - } -} - -int ossl_quic_stream_map_notify_all_data_sent(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs) -{ - switch (qs->send_state) { - default: - /* Wrong state - caller error. */ - case QUIC_SSTREAM_STATE_NONE: - /* Stream without send part - caller error. */ - return 0; - - case QUIC_SSTREAM_STATE_SEND: - if (!ossl_quic_sstream_get_final_size(qs->sstream, &qs->send_final_size)) - return 0; - - qs->send_state = QUIC_SSTREAM_STATE_DATA_SENT; - return 1; - } -} - -static void shutdown_flush_done(QUIC_STREAM_MAP *qsm, QUIC_STREAM *qs) -{ - if (!qs->shutdown_flush) - return; - - assert(qsm->num_shutdown_flush > 0); - qs->shutdown_flush = 0; - --qsm->num_shutdown_flush; -} - -int ossl_quic_stream_map_notify_totally_acked(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs) -{ - switch (qs->send_state) { - default: - /* Wrong state - caller error. */ - case QUIC_SSTREAM_STATE_NONE: - /* Stream without send part - caller error. */ - return 0; - - case QUIC_SSTREAM_STATE_DATA_SENT: - qs->send_state = QUIC_SSTREAM_STATE_DATA_RECVD; - /* We no longer need a QUIC_SSTREAM in this state. */ - ossl_quic_sstream_free(qs->sstream); - qs->sstream = NULL; - - shutdown_flush_done(qsm, qs); - return 1; - } -} - -int ossl_quic_stream_map_reset_stream_send_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs, - uint64_t aec) -{ - switch (qs->send_state) { - default: - case QUIC_SSTREAM_STATE_NONE: - /* - * RESET_STREAM pertains to sending part only, so we cannot reset a - * receive-only stream. - */ - case QUIC_SSTREAM_STATE_DATA_RECVD: - /* - * RFC 9000 s. 3.3: A sender MUST NOT [...] send RESET_STREAM from a - * terminal state. If the stream has already finished normally and the - * peer has acknowledged this, we cannot reset it. - */ - return 0; - - case QUIC_SSTREAM_STATE_READY: - if (!ossl_quic_stream_map_ensure_send_part_id(qsm, qs)) - return 0; - - /* FALLTHROUGH */ - case QUIC_SSTREAM_STATE_SEND: - /* - * If we already have a final size (e.g. because we are coming from - * DATA_SENT), we have to be consistent with that, so don't change it. - * If we don't already have a final size, determine a final size value. - * This is the value which we will end up using for a RESET_STREAM frame - * for flow control purposes. We could send the stream size (total - * number of bytes appended to QUIC_SSTREAM by the application), but it - * is in our interest to exclude any bytes we have not actually - * transmitted yet, to avoid unnecessarily consuming flow control - * credit. We can get this from the TXFC. - */ - qs->send_final_size = ossl_quic_txfc_get_swm(&qs->txfc); - - /* FALLTHROUGH */ - case QUIC_SSTREAM_STATE_DATA_SENT: - qs->reset_stream_aec = aec; - qs->want_reset_stream = 1; - qs->send_state = QUIC_SSTREAM_STATE_RESET_SENT; - - ossl_quic_sstream_free(qs->sstream); - qs->sstream = NULL; - - shutdown_flush_done(qsm, qs); - ossl_quic_stream_map_update_state(qsm, qs); - return 1; - - case QUIC_SSTREAM_STATE_RESET_SENT: - case QUIC_SSTREAM_STATE_RESET_RECVD: - /* - * Idempotent - no-op. In any case, do not send RESET_STREAM again - as - * mentioned, we must not send it from a terminal state. - */ - return 1; - } -} - -int ossl_quic_stream_map_notify_reset_stream_acked(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs) -{ - switch (qs->send_state) { - default: - /* Wrong state - caller error. */ - case QUIC_SSTREAM_STATE_NONE: - /* Stream without send part - caller error. */ - return 0; - - case QUIC_SSTREAM_STATE_RESET_SENT: - qs->send_state = QUIC_SSTREAM_STATE_RESET_RECVD; - return 1; - - case QUIC_SSTREAM_STATE_RESET_RECVD: - /* Already in the correct state. */ - return 1; - } -} - -/* - * Stream Receive Part State Management - * ==================================== - */ - -int ossl_quic_stream_map_notify_size_known_recv_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs, - uint64_t final_size) -{ - switch (qs->recv_state) { - default: - /* Wrong state - caller error. */ - case QUIC_RSTREAM_STATE_NONE: - /* Stream without receive part - caller error. */ - return 0; - - case QUIC_RSTREAM_STATE_RECV: - qs->recv_state = QUIC_RSTREAM_STATE_SIZE_KNOWN; - return 1; - } -} - -int ossl_quic_stream_map_notify_totally_received(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs) -{ - switch (qs->recv_state) { - default: - /* Wrong state - caller error. */ - case QUIC_RSTREAM_STATE_NONE: - /* Stream without receive part - caller error. */ - return 0; - - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - qs->recv_state = QUIC_RSTREAM_STATE_DATA_RECVD; - qs->want_stop_sending = 0; - return 1; - } -} - -int ossl_quic_stream_map_notify_totally_read(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs) -{ - switch (qs->recv_state) { - default: - /* Wrong state - caller error. */ - case QUIC_RSTREAM_STATE_NONE: - /* Stream without receive part - caller error. */ - return 0; - - case QUIC_RSTREAM_STATE_DATA_RECVD: - qs->recv_state = QUIC_RSTREAM_STATE_DATA_READ; - - /* QUIC_RSTREAM is no longer needed */ - ossl_quic_rstream_free(qs->rstream); - qs->rstream = NULL; - return 1; - } -} - -int ossl_quic_stream_map_notify_reset_recv_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs, - uint64_t app_error_code, - uint64_t final_size) -{ - uint64_t prev_final_size; - - switch (qs->recv_state) { - default: - case QUIC_RSTREAM_STATE_NONE: - /* Stream without receive part - caller error. */ - return 0; - - case QUIC_RSTREAM_STATE_RECV: - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - case QUIC_RSTREAM_STATE_DATA_RECVD: - if (ossl_quic_stream_recv_get_final_size(qs, &prev_final_size) - && prev_final_size != final_size) - /* Cannot change previous final size. */ - return 0; - - qs->recv_state = QUIC_RSTREAM_STATE_RESET_RECVD; - qs->peer_reset_stream_aec = app_error_code; - - /* RFC 9000 s. 3.3: No point sending STOP_SENDING if already reset. */ - qs->want_stop_sending = 0; - - /* QUIC_RSTREAM is no longer needed */ - ossl_quic_rstream_free(qs->rstream); - qs->rstream = NULL; - - ossl_quic_stream_map_update_state(qsm, qs); - return 1; - - case QUIC_RSTREAM_STATE_DATA_READ: - /* - * If we already retired the FIN to the application this is moot - * - just ignore. - */ - case QUIC_RSTREAM_STATE_RESET_RECVD: - case QUIC_RSTREAM_STATE_RESET_READ: - /* Could be a reordered/retransmitted frame - just ignore. */ - return 1; - } -} - -int ossl_quic_stream_map_notify_app_read_reset_recv_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs) -{ - switch (qs->recv_state) { - default: - /* Wrong state - caller error. */ - case QUIC_RSTREAM_STATE_NONE: - /* Stream without receive part - caller error. */ - return 0; - - case QUIC_RSTREAM_STATE_RESET_RECVD: - qs->recv_state = QUIC_RSTREAM_STATE_RESET_READ; - return 1; - } -} - -int ossl_quic_stream_map_stop_sending_recv_part(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *qs, - uint64_t aec) -{ - if (qs->stop_sending) - return 0; - - switch (qs->recv_state) { - default: - case QUIC_RSTREAM_STATE_NONE: - /* Send-only stream, so this makes no sense. */ - case QUIC_RSTREAM_STATE_DATA_RECVD: - case QUIC_RSTREAM_STATE_DATA_READ: - /* - * Not really any point in STOP_SENDING if we already received all data. - */ - case QUIC_RSTREAM_STATE_RESET_RECVD: - case QUIC_RSTREAM_STATE_RESET_READ: - /* - * RFC 9000 s. 3.5: "STOP_SENDING SHOULD only be sent for a stream that - * has not been reset by the peer." - * - * No point in STOP_SENDING if the peer already reset their send part. - */ - return 0; - - case QUIC_RSTREAM_STATE_RECV: - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - /* - * RFC 9000 s. 3.5: "If the stream is in the Recv or Size Known state, - * the transport SHOULD signal this by sending a STOP_SENDING frame to - * prompt closure of the stream in the opposite direction." - * - * Note that it does make sense to send STOP_SENDING for a receive part - * of a stream which has a known size (because we have received a FIN) - * but which still has other (previous) stream data yet to be received. - */ - break; - } - - qs->stop_sending = 1; - qs->stop_sending_aec = aec; - return ossl_quic_stream_map_schedule_stop_sending(qsm, qs); -} - -/* Called to mark STOP_SENDING for generation, or regeneration after loss. */ -int ossl_quic_stream_map_schedule_stop_sending(QUIC_STREAM_MAP *qsm, QUIC_STREAM *qs) -{ - if (!qs->stop_sending) - return 0; - - /* - * Ignore the call as a no-op if already scheduled, or in a state - * where it makes no sense to send STOP_SENDING. - */ - if (qs->want_stop_sending) - return 1; - - switch (qs->recv_state) { - default: - return 1; /* ignore */ - case QUIC_RSTREAM_STATE_RECV: - case QUIC_RSTREAM_STATE_SIZE_KNOWN: - /* - * RFC 9000 s. 3.5: "An endpoint is expected to send another - * STOP_SENDING frame if a packet containing a previous STOP_SENDING is - * lost. However, once either all stream data or a RESET_STREAM frame - * has been received for the stream -- that is, the stream is in any - * state other than "Recv" or "Size Known" -- sending a STOP_SENDING - * frame is unnecessary." - */ - break; - } - - qs->want_stop_sending = 1; - ossl_quic_stream_map_update_state(qsm, qs); - return 1; -} - -QUIC_STREAM *ossl_quic_stream_map_peek_accept_queue(QUIC_STREAM_MAP *qsm) -{ - return accept_head(&qsm->accept_list); -} - -void ossl_quic_stream_map_push_accept_queue(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *s) -{ - list_insert_tail(&qsm->accept_list, &s->accept_node); - if (ossl_quic_stream_is_bidi(s)) - ++qsm->num_accept_bidi; - else - ++qsm->num_accept_uni; -} - -static QUIC_RXFC *qsm_get_max_streams_rxfc(QUIC_STREAM_MAP *qsm, QUIC_STREAM *s) -{ - return ossl_quic_stream_is_bidi(s) - ? qsm->max_streams_bidi_rxfc - : qsm->max_streams_uni_rxfc; -} - -void ossl_quic_stream_map_remove_from_accept_queue(QUIC_STREAM_MAP *qsm, - QUIC_STREAM *s, - OSSL_TIME rtt) -{ - QUIC_RXFC *max_streams_rxfc; - - list_remove(&qsm->accept_list, &s->accept_node); - if (ossl_quic_stream_is_bidi(s)) - --qsm->num_accept_bidi; - else - --qsm->num_accept_uni; - - if ((max_streams_rxfc = qsm_get_max_streams_rxfc(qsm, s)) != NULL) - ossl_quic_rxfc_on_retire(max_streams_rxfc, 1, rtt); -} - -size_t ossl_quic_stream_map_get_accept_queue_len(QUIC_STREAM_MAP *qsm, int is_uni) -{ - return is_uni ? qsm->num_accept_uni : qsm->num_accept_bidi; -} - -size_t ossl_quic_stream_map_get_total_accept_queue_len(QUIC_STREAM_MAP *qsm) -{ - return ossl_quic_stream_map_get_accept_queue_len(qsm, /*is_uni=*/0) - + ossl_quic_stream_map_get_accept_queue_len(qsm, /*is_uni=*/1); -} - -void ossl_quic_stream_map_gc(QUIC_STREAM_MAP *qsm) -{ - QUIC_STREAM *qs, *qs_head, *qsn = NULL; - - for (qs = qs_head = ready_for_gc_head(&qsm->ready_for_gc_list); - qs != NULL && qs != qs_head; - qs = qsn) - { - qsn = ready_for_gc_next(&qsm->ready_for_gc_list, qs); - - ossl_quic_stream_map_release(qsm, qs); - } -} - -static int eligible_for_shutdown_flush(QUIC_STREAM *qs) -{ - /* - * We only care about servicing the send part of a stream (if any) during - * shutdown flush. We make sure we flush a stream if it is either - * non-terminated or was terminated normally such as via - * SSL_stream_conclude. A stream which was terminated via a reset is not - * flushed, and we will have thrown away the send buffer in that case - * anyway. - */ - switch (qs->send_state) { - case QUIC_SSTREAM_STATE_SEND: - case QUIC_SSTREAM_STATE_DATA_SENT: - return !ossl_quic_sstream_is_totally_acked(qs->sstream); - default: - return 0; - } -} - -static void begin_shutdown_flush_each(QUIC_STREAM *qs, void *arg) -{ - QUIC_STREAM_MAP *qsm = arg; - - if (!eligible_for_shutdown_flush(qs) || qs->shutdown_flush) - return; - - qs->shutdown_flush = 1; - ++qsm->num_shutdown_flush; -} - -void ossl_quic_stream_map_begin_shutdown_flush(QUIC_STREAM_MAP *qsm) -{ - qsm->num_shutdown_flush = 0; - - ossl_quic_stream_map_visit(qsm, begin_shutdown_flush_each, qsm); -} - -int ossl_quic_stream_map_is_shutdown_flush_finished(QUIC_STREAM_MAP *qsm) -{ - return qsm->num_shutdown_flush == 0; -} - -/* - * QUIC Stream Iterator - * ==================== - */ -void ossl_quic_stream_iter_init(QUIC_STREAM_ITER *it, QUIC_STREAM_MAP *qsm, - int advance_rr) -{ - it->qsm = qsm; - it->stream = it->first_stream = qsm->rr_cur; - if (advance_rr && it->stream != NULL - && ++qsm->rr_counter >= qsm->rr_stepping) { - qsm->rr_counter = 0; - qsm->rr_cur = active_next(&qsm->active_list, qsm->rr_cur); - } -} - -void ossl_quic_stream_iter_next(QUIC_STREAM_ITER *it) -{ - if (it->stream == NULL) - return; - - it->stream = active_next(&it->qsm->active_list, it->stream); - if (it->stream == it->first_stream) - it->stream = NULL; -} diff --git a/openssl/src/ssl/quic/quic_thread_assist.c b/openssl/src/ssl/quic/quic_thread_assist.c deleted file mode 100644 index 26c738cb5..000000000 --- a/openssl/src/ssl/quic/quic_thread_assist.c +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "quic_local.h" -#include "internal/time.h" -#include "internal/thread.h" -#include "internal/thread_arch.h" -#include "internal/quic_thread_assist.h" - -#if !defined(OPENSSL_NO_QUIC_THREAD_ASSIST) - -/* Main loop for the QUIC assist thread. */ -static unsigned int assist_thread_main(void *arg) -{ - QUIC_THREAD_ASSIST *qta = arg; - CRYPTO_MUTEX *m = ossl_quic_channel_get_mutex(qta->ch); - QUIC_REACTOR *rtor; - - ossl_crypto_mutex_lock(m); - - rtor = ossl_quic_channel_get_reactor(qta->ch); - - for (;;) { - OSSL_TIME deadline; - - if (qta->teardown) - break; - - deadline = ossl_quic_reactor_get_tick_deadline(rtor); - if (qta->now_cb != NULL - && !ossl_time_is_zero(deadline) - && !ossl_time_is_infinite(deadline)) { - /* - * ossl_crypto_condvar_wait_timeout needs to use real time for the - * deadline - */ - deadline = ossl_time_add(ossl_time_subtract(deadline, - qta->now_cb(qta->now_cb_arg)), - ossl_time_now()); - } - ossl_crypto_condvar_wait_timeout(qta->cv, m, deadline); - - /* - * We have now been woken up. This can be for one of the following - * reasons: - * - * - We have been asked to teardown (qta->teardown is set); - * - The tick deadline has passed. - * - The tick deadline has changed. - * - * For robustness, this loop also handles spurious wakeups correctly - * (which does not require any extra code). - */ - if (qta->teardown) - break; - - ossl_quic_reactor_tick(rtor, QUIC_REACTOR_TICK_FLAG_CHANNEL_ONLY); - } - - ossl_crypto_mutex_unlock(m); - return 1; -} - -int ossl_quic_thread_assist_init_start(QUIC_THREAD_ASSIST *qta, - QUIC_CHANNEL *ch, - OSSL_TIME (*now_cb)(void *arg), - void *now_cb_arg) -{ - CRYPTO_MUTEX *mutex = ossl_quic_channel_get_mutex(ch); - - if (mutex == NULL) - return 0; - - qta->ch = ch; - qta->teardown = 0; - qta->joined = 0; - qta->now_cb = now_cb; - qta->now_cb_arg = now_cb_arg; - - qta->cv = ossl_crypto_condvar_new(); - if (qta->cv == NULL) - return 0; - - qta->t = ossl_crypto_thread_native_start(assist_thread_main, - qta, /*joinable=*/1); - if (qta->t == NULL) { - ossl_crypto_condvar_free(&qta->cv); - return 0; - } - - return 1; -} - -int ossl_quic_thread_assist_stop_async(QUIC_THREAD_ASSIST *qta) -{ - if (!qta->teardown) { - qta->teardown = 1; - ossl_crypto_condvar_signal(qta->cv); - } - - return 1; -} - -int ossl_quic_thread_assist_wait_stopped(QUIC_THREAD_ASSIST *qta) -{ - CRYPTO_THREAD_RETVAL rv; - CRYPTO_MUTEX *m = ossl_quic_channel_get_mutex(qta->ch); - - if (qta->joined) - return 1; - - if (!ossl_quic_thread_assist_stop_async(qta)) - return 0; - - ossl_crypto_mutex_unlock(m); - - if (!ossl_crypto_thread_native_join(qta->t, &rv)) { - ossl_crypto_mutex_lock(m); - return 0; - } - - qta->joined = 1; - - ossl_crypto_mutex_lock(m); - return 1; -} - -int ossl_quic_thread_assist_cleanup(QUIC_THREAD_ASSIST *qta) -{ - if (!ossl_assert(qta->joined)) - return 0; - - ossl_crypto_condvar_free(&qta->cv); - ossl_crypto_thread_native_clean(qta->t); - - qta->ch = NULL; - qta->t = NULL; - return 1; -} - -int ossl_quic_thread_assist_notify_deadline_changed(QUIC_THREAD_ASSIST *qta) -{ - if (qta->teardown) - return 0; - - ossl_crypto_condvar_signal(qta->cv); - return 1; -} - -#endif diff --git a/openssl/src/ssl/quic/quic_tls.c b/openssl/src/ssl/quic/quic_tls.c deleted file mode 100644 index bd560c9a9..000000000 --- a/openssl/src/ssl/quic/quic_tls.c +++ /dev/null @@ -1,879 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ -#include -#include "internal/recordmethod.h" -#include "internal/quic_tls.h" -#include "../ssl_local.h" -#include "internal/quic_error.h" - -#define QUIC_TLS_FATAL(rl, ad, err) \ - do { \ - if ((rl) != NULL) (rl)->alert = (ad); \ - ERR_raise(ERR_LIB_SSL, (err)); \ - if ((rl) != NULL) (rl)->qtls->inerror = 1; \ - } while(0) - -struct quic_tls_st { - QUIC_TLS_ARGS args; - - /* - * Transport parameters which client should send. Buffer lifetime must - * exceed the lifetime of the QUIC_TLS object. - */ - const unsigned char *local_transport_params; - size_t local_transport_params_len; - - ERR_STATE *error_state; - - /* - * QUIC error code (usually in the TLS Alert-mapped CRYPTO_ERR range). Valid - * only if inerror is 1. - */ - uint64_t error_code; - - /* - * Error message with static storage duration. Valid only if inerror is 1. - * Should be suitable for encapsulation in a CONNECTION_CLOSE frame. - */ - const char *error_msg; - - /* Whether our SSL object for TLS has been configured for use in QUIC */ - unsigned int configured : 1; - - /* Set if we have hit any error state */ - unsigned int inerror : 1; - - /* Set if the handshake has completed */ - unsigned int complete : 1; -}; - -struct ossl_record_layer_st { - QUIC_TLS *qtls; - - /* Protection level */ - int level; - - /* Only used for retry flags */ - BIO *dummybio; - - /* Number of bytes written so far if we are part way through a write */ - size_t written; - - /* If we are part way through a write, a copy of the template */ - OSSL_RECORD_TEMPLATE template; - - /* - * If we hit an error, what alert code should be used - */ - int alert; - - /* Amount of crypto stream data we read in the last call to quic_read_record */ - size_t recread; - - /* Amount of crypto stream data read but not yet released */ - size_t recunreleased; - - /* Callbacks */ - OSSL_FUNC_rlayer_msg_callback_fn *msg_callback; - void *cbarg; -}; - -static int quic_set1_bio(OSSL_RECORD_LAYER *rl, BIO *bio); -static int quic_free(OSSL_RECORD_LAYER *r); - -static int -quic_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers, - int role, int direction, int level, uint16_t epoch, - unsigned char *secret, size_t secretlen, - unsigned char *key, size_t keylen, unsigned char *iv, - size_t ivlen, unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, size_t taglen, - int mactype, - const EVP_MD *md, COMP_METHOD *comp, - const EVP_MD *kdfdigest, BIO *prev, BIO *transport, - BIO *next, BIO_ADDR *local, BIO_ADDR *peer, - const OSSL_PARAM *settings, const OSSL_PARAM *options, - const OSSL_DISPATCH *fns, void *cbarg, void *rlarg, - OSSL_RECORD_LAYER **retrl) -{ - OSSL_RECORD_LAYER *rl = OPENSSL_zalloc(sizeof(*rl)); - uint32_t enc_level; - int qdir; - uint32_t suite_id = 0; - - if (rl == NULL) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - rl->qtls = (QUIC_TLS *)rlarg; - rl->level = level; - if (!quic_set1_bio(rl, transport)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - rl->cbarg = cbarg; - *retrl = rl; - - if (fns != NULL) { - for (; fns->function_id != 0; fns++) { - switch (fns->function_id) { - break; - case OSSL_FUNC_RLAYER_MSG_CALLBACK: - rl->msg_callback = OSSL_FUNC_rlayer_msg_callback(fns); - break; - default: - /* Just ignore anything we don't understand */ - break; - } - } - } - - switch (level) { - case OSSL_RECORD_PROTECTION_LEVEL_NONE: - return 1; - - case OSSL_RECORD_PROTECTION_LEVEL_EARLY: - enc_level = QUIC_ENC_LEVEL_0RTT; - break; - - case OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE: - enc_level = QUIC_ENC_LEVEL_HANDSHAKE; - break; - - case OSSL_RECORD_PROTECTION_LEVEL_APPLICATION: - enc_level = QUIC_ENC_LEVEL_1RTT; - break; - - default: - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - if (direction == OSSL_RECORD_DIRECTION_READ) - qdir = 0; - else - qdir = 1; - - if (EVP_CIPHER_is_a(ciph, "AES-128-GCM")) { - suite_id = QRL_SUITE_AES128GCM; - } else if (EVP_CIPHER_is_a(ciph, "AES-256-GCM")) { - suite_id = QRL_SUITE_AES256GCM; - } else if (EVP_CIPHER_is_a(ciph, "CHACHA20-POLY1305")) { - suite_id = QRL_SUITE_CHACHA20POLY1305; - } else { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, SSL_R_UNKNOWN_CIPHER_TYPE); - goto err; - } - - /* We pass a ref to the md in a successful yield_secret_cb call */ - /* TODO(QUIC FUTURE): This cast is horrible. We should try and remove it */ - if (!EVP_MD_up_ref((EVP_MD *)kdfdigest)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - if (!rl->qtls->args.yield_secret_cb(enc_level, qdir, suite_id, - (EVP_MD *)kdfdigest, secret, secretlen, - rl->qtls->args.yield_secret_cb_arg)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - EVP_MD_free((EVP_MD *)kdfdigest); - goto err; - } - - return 1; - err: - *retrl = NULL; - quic_free(rl); - return 0; -} - -static int quic_free(OSSL_RECORD_LAYER *rl) -{ - if (rl == NULL) - return 1; - - BIO_free(rl->dummybio); - OPENSSL_free(rl); - return 1; -} - -static int quic_unprocessed_read_pending(OSSL_RECORD_LAYER *rl) -{ - /* - * Read ahead isn't really a thing for QUIC so we never have unprocessed - * data pending - */ - return 0; -} - -static int quic_processed_read_pending(OSSL_RECORD_LAYER *rl) -{ - /* - * This is currently only ever used by: - * - SSL_has_pending() - * - to check whether we have more records that we want to supply to the - * upper layers - * - * We only ever supply 1 record at a time to the upper layers, and - * SSL_has_pending() will go via the QUIC method not the TLS method so that - * use case doesn't apply here. - * Therefore we can ignore this for now and always return 0. We might - * eventually want to change this to check in the receive buffers to see if - * we have any more data pending. - */ - return 0; -} - -static size_t quic_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type, - size_t len, - size_t maxfrag, size_t *preffrag) -{ - return 1; -} - -static int quic_write_records(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *template, - size_t numtempl) -{ - size_t consumed; - unsigned char alert; - - if (!ossl_assert(numtempl == 1)) { - /* How could this be? quic_get_max_records() always returns 1 */ - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - BIO_clear_retry_flags(rl->dummybio); - - if (rl->msg_callback != NULL) { - unsigned char dummyrec[SSL3_RT_HEADER_LENGTH]; - - /* - * For the purposes of the callback we "pretend" to be normal TLS, - * and manufacture a dummy record header - */ - dummyrec[0] = (rl->level == OSSL_RECORD_PROTECTION_LEVEL_NONE) - ? template->type - : SSL3_RT_APPLICATION_DATA; - dummyrec[1] = (unsigned char)((template->version >> 8) & 0xff); - dummyrec[2] = (unsigned char)(template->version & 0xff); - /* - * We assume that buflen is always <= UINT16_MAX. Since this is - * generated by libssl itself we actually expect it to never - * exceed SSL3_RT_MAX_PLAIN_LENGTH - so it should be a safe - * assumption - */ - dummyrec[3] = (unsigned char)((template->buflen >> 8) & 0xff); - dummyrec[4] = (unsigned char)(template->buflen & 0xff); - - rl->msg_callback(1, TLS1_3_VERSION, SSL3_RT_HEADER, dummyrec, - SSL3_RT_HEADER_LENGTH, rl->cbarg); - - if (rl->level != OSSL_RECORD_PROTECTION_LEVEL_NONE) { - rl->msg_callback(1, TLS1_3_VERSION, SSL3_RT_INNER_CONTENT_TYPE, - &template->type, 1, rl->cbarg); - } - } - - switch (template->type) { - case SSL3_RT_ALERT: - if (template->buflen != 2) { - /* - * We assume that libssl always sends both bytes of an alert to - * us in one go, and never fragments it. If we ever get more - * or less bytes than exactly 2 then this is very unexpected. - */ - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_VALUE); - return OSSL_RECORD_RETURN_FATAL; - } - /* - * Byte 0 is the alert level (we ignore it) and byte 1 is the alert - * description that we are actually interested in. - */ - alert = template->buf[1]; - - if (!rl->qtls->args.alert_cb(rl->qtls->args.alert_cb_arg, alert)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - break; - - case SSL3_RT_HANDSHAKE: - /* - * We expect this to only fail on some fatal error (e.g. malloc - * failure) - */ - if (!rl->qtls->args.crypto_send_cb(template->buf + rl->written, - template->buflen - rl->written, - &consumed, - rl->qtls->args.crypto_send_cb_arg)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - /* - * We might have written less than we wanted to if we have filled the - * send stream buffer. - */ - if (consumed + rl->written != template->buflen) { - if (!ossl_assert(consumed + rl->written < template->buflen)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - /* - * We've not written everything we wanted to. Take a copy of the - * template, remember how much we wrote so far and signal a retry. - * The buffer supplied in the template is guaranteed to be the same - * on a retry for handshake data - */ - rl->written += consumed; - rl->template = *template; - BIO_set_retry_write(rl->dummybio); - - return OSSL_RECORD_RETURN_RETRY; - } - rl->written = 0; - break; - - default: - /* Anything else is unexpected and an error */ - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - return OSSL_RECORD_RETURN_SUCCESS; -} - -static int quic_retry_write_records(OSSL_RECORD_LAYER *rl) -{ - return quic_write_records(rl, &rl->template, 1); -} - -static int quic_read_record(OSSL_RECORD_LAYER *rl, void **rechandle, - int *rversion, uint8_t *type, const unsigned char **data, - size_t *datalen, uint16_t *epoch, - unsigned char *seq_num) -{ - if (rl->recread != 0 || rl->recunreleased != 0) - return OSSL_RECORD_RETURN_FATAL; - - BIO_clear_retry_flags(rl->dummybio); - - if (!rl->qtls->args.crypto_recv_rcd_cb(data, datalen, - rl->qtls->args.crypto_recv_rcd_cb_arg)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - if (*datalen == 0) { - BIO_set_retry_read(rl->dummybio); - return OSSL_RECORD_RETURN_RETRY; - } - - *rechandle = rl; - *rversion = TLS1_3_VERSION; - *type = SSL3_RT_HANDSHAKE; - rl->recread = rl->recunreleased = *datalen; - /* epoch/seq_num are not relevant for TLS */ - - if (rl->msg_callback != NULL) { - unsigned char dummyrec[SSL3_RT_HEADER_LENGTH]; - - /* - * For the purposes of the callback we "pretend" to be normal TLS, - * and manufacture a dummy record header - */ - dummyrec[0] = (rl->level == OSSL_RECORD_PROTECTION_LEVEL_NONE) - ? SSL3_RT_HANDSHAKE - : SSL3_RT_APPLICATION_DATA; - dummyrec[1] = (unsigned char)((TLS1_2_VERSION >> 8) & 0xff); - dummyrec[2] = (unsigned char)(TLS1_2_VERSION & 0xff); - /* - * *datalen will always fit into 2 bytes because our original buffer - * size is less than that. - */ - dummyrec[3] = (unsigned char)((*datalen >> 8) & 0xff); - dummyrec[4] = (unsigned char)(*datalen & 0xff); - - rl->msg_callback(0, TLS1_3_VERSION, SSL3_RT_HEADER, dummyrec, - SSL3_RT_HEADER_LENGTH, rl->cbarg); - rl->msg_callback(0, TLS1_3_VERSION, SSL3_RT_INNER_CONTENT_TYPE, type, 1, - rl->cbarg); - } - - return OSSL_RECORD_RETURN_SUCCESS; -} - -static int quic_release_record(OSSL_RECORD_LAYER *rl, void *rechandle, - size_t length) -{ - if (!ossl_assert(rl->recread > 0) - || !ossl_assert(rl->recunreleased <= rl->recread) - || !ossl_assert(rl == rechandle) - || !ossl_assert(length <= rl->recunreleased)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - rl->recunreleased -= length; - - if (rl->recunreleased > 0) - return OSSL_RECORD_RETURN_SUCCESS; - - if (!rl->qtls->args.crypto_release_rcd_cb(rl->recread, - rl->qtls->args.crypto_release_rcd_cb_arg)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - rl->recread = 0; - return OSSL_RECORD_RETURN_SUCCESS; -} - -static int quic_get_alert_code(OSSL_RECORD_LAYER *rl) -{ - return rl->alert; -} - -static int quic_set_protocol_version(OSSL_RECORD_LAYER *rl, int version) -{ - /* We only support TLSv1.3, so its bad if we negotiate anything else */ - if (!ossl_assert(version == TLS1_3_VERSION)) { - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - return 1; -} - -static void quic_set_plain_alerts(OSSL_RECORD_LAYER *rl, int allow) -{ - /* We don't care */ -} - -static void quic_set_first_handshake(OSSL_RECORD_LAYER *rl, int first) -{ - /* We don't care */ -} - -static void quic_set_max_pipelines(OSSL_RECORD_LAYER *rl, size_t max_pipelines) -{ - /* We don't care */ -} - -static void quic_get_state(OSSL_RECORD_LAYER *rl, const char **shortstr, - const char **longstr) -{ - /* - * According to the docs, valid read state strings are: "RH"/"read header", - * "RB"/"read body", and "unknown"/"unknown". We don't read records in quite - * that way, so we report every "normal" state as "read header". In the - * event of error then we report "unknown". - */ - - if (rl->qtls->inerror) { - if (shortstr != NULL) - *shortstr = "unknown"; - if (longstr != NULL) - *longstr = "unknown"; - } else { - if (shortstr != NULL) - *shortstr = "RH"; - if (longstr != NULL) - *longstr = "read header"; - } -} - -static int quic_set_options(OSSL_RECORD_LAYER *rl, const OSSL_PARAM *options) -{ - /* - * We don't support any options yet - but we might do at some point so - * this could be useful. - */ - return 1; -} - -static const COMP_METHOD *quic_get_compression(OSSL_RECORD_LAYER *rl) -{ - /* We only support TLSv1.3 which doesn't have compression */ - return NULL; -} - -static void quic_set_max_frag_len(OSSL_RECORD_LAYER *rl, size_t max_frag_len) -{ - /* This really doesn't make any sense for QUIC. Ignore it */ -} - -static int quic_alloc_buffers(OSSL_RECORD_LAYER *rl) -{ - /* - * This is a hint only. We don't support it (yet), so just ignore the - * request - */ - return 1; -} - -static int quic_free_buffers(OSSL_RECORD_LAYER *rl) -{ - /* - * This is a hint only. We don't support it (yet), so just ignore the - * request - */ - return 1; -} - -static int quic_set1_bio(OSSL_RECORD_LAYER *rl, BIO *bio) -{ - if (bio != NULL && !BIO_up_ref(bio)) - return 0; - BIO_free(rl->dummybio); - rl->dummybio = bio; - - return 1; -} - -/* - * Never called functions - * - * Due to the way we are configured and used we never expect any of the next set - * of functions to be called. Therefore we set them to always fail. - */ - -static size_t quic_app_data_pending(OSSL_RECORD_LAYER *rl) -{ - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return (size_t)ossl_assert(0); -} - -static size_t quic_get_max_record_overhead(OSSL_RECORD_LAYER *rl) -{ - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return (size_t)ossl_assert(0); -} - -static int quic_increment_sequence_ctr(OSSL_RECORD_LAYER *rl) -{ - QUIC_TLS_FATAL(rl, SSL_AD_INTERNAL_ERROR, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return ossl_assert(0); -} - -/* End of never called functions */ - -static const OSSL_RECORD_METHOD quic_tls_record_method = { - quic_new_record_layer, - quic_free, - quic_unprocessed_read_pending, - quic_processed_read_pending, - quic_app_data_pending, /* Never called */ - quic_get_max_records, - quic_write_records, - quic_retry_write_records, - quic_read_record, - quic_release_record, - quic_get_alert_code, - quic_set1_bio, - quic_set_protocol_version, - quic_set_plain_alerts, - quic_set_first_handshake, - quic_set_max_pipelines, - NULL, /* set_in_init: Optional - we don't need it */ - quic_get_state, - quic_set_options, - quic_get_compression, - quic_set_max_frag_len, - quic_get_max_record_overhead, /* Never called */ - quic_increment_sequence_ctr, /* Never called */ - quic_alloc_buffers, - quic_free_buffers -}; - -static int add_transport_params_cb(SSL *s, unsigned int ext_type, - unsigned int context, - const unsigned char **out, size_t *outlen, - X509 *x, size_t chainidx, int *al, - void *add_arg) -{ - QUIC_TLS *qtls = add_arg; - - *out = qtls->local_transport_params; - *outlen = qtls->local_transport_params_len; - return 1; -} - -static void free_transport_params_cb(SSL *s, unsigned int ext_type, - unsigned int context, - const unsigned char *out, - void *add_arg) -{ -} - -static int parse_transport_params_cb(SSL *s, unsigned int ext_type, - unsigned int context, - const unsigned char *in, - size_t inlen, X509 *x, - size_t chainidx, - int *al, void *parse_arg) -{ - QUIC_TLS *qtls = parse_arg; - - return qtls->args.got_transport_params_cb(in, inlen, - qtls->args.got_transport_params_cb_arg); -} - -QUIC_TLS *ossl_quic_tls_new(const QUIC_TLS_ARGS *args) -{ - QUIC_TLS *qtls; - - if (args->crypto_send_cb == NULL - || args->crypto_recv_rcd_cb == NULL - || args->crypto_release_rcd_cb == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - - qtls = OPENSSL_zalloc(sizeof(*qtls)); - if (qtls == NULL) - return NULL; - - if ((qtls->error_state = OSSL_ERR_STATE_new()) == NULL) { - OPENSSL_free(qtls); - return NULL; - } - - qtls->args = *args; - return qtls; -} - -void ossl_quic_tls_free(QUIC_TLS *qtls) -{ - if (qtls == NULL) - return; - OSSL_ERR_STATE_free(qtls->error_state); - OPENSSL_free(qtls); -} - -static int raise_error(QUIC_TLS *qtls, uint64_t error_code, - const char *error_msg, - const char *src_file, - int src_line, - const char *src_func) -{ - /* - * When QTLS fails, add a "cover letter" error with information, potentially - * with any underlying libssl errors underneath it (but our cover error may - * be the only error in some cases). Then capture this into an ERR_STATE so - * we can report it later if need be when the QUIC_CHANNEL asks for it. - */ - ERR_new(); - ERR_set_debug(src_file, src_line, src_func); - ERR_set_error(ERR_LIB_SSL, SSL_R_QUIC_HANDSHAKE_LAYER_ERROR, - "handshake layer error, error code %llu (0x%llx) (\"%s\")", - error_code, error_code, error_msg); - OSSL_ERR_STATE_save_to_mark(qtls->error_state); - - /* - * We record the error information reported via the QUIC protocol - * separately. - */ - qtls->error_code = error_code; - qtls->error_msg = error_msg; - qtls->inerror = 1; - - ERR_pop_to_mark(); - return 0; -} - -#define RAISE_ERROR(qtls, error_code, error_msg) \ - raise_error((qtls), (error_code), (error_msg), \ - OPENSSL_FILE, OPENSSL_LINE, OPENSSL_FUNC) - -#define RAISE_INTERNAL_ERROR(qtls) \ - RAISE_ERROR((qtls), OSSL_QUIC_ERR_INTERNAL_ERROR, "internal error") - -int ossl_quic_tls_tick(QUIC_TLS *qtls) -{ - int ret, err; - const unsigned char *alpn; - unsigned int alpnlen; - - if (qtls->inerror) - return 0; - - /* - * SSL_get_error does not truly know what the cause of an SSL_read failure - * is and to some extent guesses based on contextual information. In - * particular, if there is _any_ ERR on the error stack, SSL_ERROR_SSL or - * SSL_ERROR_SYSCALL will be returned no matter what and there is no - * possibility of SSL_ERROR_WANT_READ/WRITE being returned, even if that was - * the actual cause of the SSL_read() failure. - * - * This means that ordinarily, the below code might not work right if the - * application has any ERR on the error stack. In order to make this code - * perform correctly regardless of prior ERR state, we use a variant of - * SSL_get_error() which ignores the error stack. However, some ERRs are - * raised by SSL_read() and actually indicate that something has gone wrong - * during the call to SSL_read(). We therefore adopt a strategy of marking - * the ERR stack and seeing if any errors get appended during the call to - * SSL_read(). If they are, we assume SSL_read() has raised an error and - * that we should use normal SSL_get_error() handling. - * - * NOTE: Ensure all escape paths from this function call - * ERR_clear_to_mark(). The RAISE macros handle this in failure cases. - */ - ERR_set_mark(); - - if (!qtls->configured) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(qtls->args.s); - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(sc); - BIO *nullbio; - - /* - * No matter how the user has configured us, there are certain - * requirements for QUIC-TLS that we enforce - */ - - /* ALPN is a requirement for QUIC and must be set */ - if (qtls->args.is_server) { - if (sctx->ext.alpn_select_cb == NULL) - return RAISE_INTERNAL_ERROR(qtls); - } else { - if (sc->ext.alpn == NULL || sc->ext.alpn_len == 0) - return RAISE_ERROR(qtls, OSSL_QUIC_ERR_CRYPTO_NO_APP_PROTO, - "ALPN must be configured when using QUIC"); - } - if (!SSL_set_min_proto_version(qtls->args.s, TLS1_3_VERSION)) - return RAISE_INTERNAL_ERROR(qtls); - - SSL_clear_options(qtls->args.s, SSL_OP_ENABLE_MIDDLEBOX_COMPAT); - ossl_ssl_set_custom_record_layer(sc, &quic_tls_record_method, qtls); - - if (!ossl_tls_add_custom_ext_intern(NULL, &sc->cert->custext, - qtls->args.is_server ? ENDPOINT_SERVER - : ENDPOINT_CLIENT, - TLSEXT_TYPE_quic_transport_parameters, - SSL_EXT_TLS1_3_ONLY - | SSL_EXT_CLIENT_HELLO - | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS, - add_transport_params_cb, - free_transport_params_cb, qtls, - parse_transport_params_cb, qtls)) - return RAISE_INTERNAL_ERROR(qtls); - - nullbio = BIO_new(BIO_s_null()); - if (nullbio == NULL) - return RAISE_INTERNAL_ERROR(qtls); - - /* - * Our custom record layer doesn't use the BIO - but libssl generally - * expects one to be present. - */ - SSL_set_bio(qtls->args.s, nullbio, nullbio); - - if (qtls->args.is_server) - SSL_set_accept_state(qtls->args.s); - else - SSL_set_connect_state(qtls->args.s); - - qtls->configured = 1; - } - - if (qtls->complete) - /* - * There should never be app data to read, but calling SSL_read() will - * ensure any post-handshake messages are processed. - */ - ret = SSL_read(qtls->args.s, NULL, 0); - else - ret = SSL_do_handshake(qtls->args.s); - - if (ret <= 0) { - err = ossl_ssl_get_error(qtls->args.s, ret, - /*check_err=*/ERR_count_to_mark() > 0); - - switch (err) { - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - case SSL_ERROR_WANT_CLIENT_HELLO_CB: - case SSL_ERROR_WANT_X509_LOOKUP: - case SSL_ERROR_WANT_RETRY_VERIFY: - ERR_pop_to_mark(); - return 1; - - default: - return RAISE_INTERNAL_ERROR(qtls); - } - } - - if (!qtls->complete) { - /* Validate that we have ALPN */ - SSL_get0_alpn_selected(qtls->args.s, &alpn, &alpnlen); - if (alpn == NULL || alpnlen == 0) - return RAISE_ERROR(qtls, OSSL_QUIC_ERR_CRYPTO_NO_APP_PROTO, - "no application protocol negotiated"); - - qtls->complete = 1; - ERR_pop_to_mark(); - return qtls->args.handshake_complete_cb(qtls->args.handshake_complete_cb_arg); - } - - ERR_pop_to_mark(); - return 1; -} - -int ossl_quic_tls_set_transport_params(QUIC_TLS *qtls, - const unsigned char *transport_params, - size_t transport_params_len) -{ - qtls->local_transport_params = transport_params; - qtls->local_transport_params_len = transport_params_len; - return 1; -} - -int ossl_quic_tls_get_error(QUIC_TLS *qtls, - uint64_t *error_code, - const char **error_msg, - ERR_STATE **error_state) -{ - if (qtls->inerror) { - *error_code = qtls->error_code; - *error_msg = qtls->error_msg; - *error_state = qtls->error_state; - } - - return qtls->inerror; -} - -/* - * Returns true if the last handshake record message we processed was a - * CertificateRequest - */ -int ossl_quic_tls_is_cert_request(QUIC_TLS *qtls) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(qtls->args.s); - - return sc->s3.tmp.message_type == SSL3_MT_CERTIFICATE_REQUEST; -} - -/* - * Returns true if the last session associated with the connection has an - * invalid max_early_data value for QUIC. - */ -int ossl_quic_tls_has_bad_max_early_data(QUIC_TLS *qtls) -{ - uint32_t max_early_data = SSL_get0_session(qtls->args.s)->ext.max_early_data; - - /* - * If max_early_data was present we always ensure a non-zero value is - * stored in the session for QUIC. Therefore if max_early_data == 0 here - * we can be confident that it was not present in the NewSessionTicket - */ - return max_early_data != 0xffffffff && max_early_data != 0; -} diff --git a/openssl/src/ssl/quic/quic_trace.c b/openssl/src/ssl/quic/quic_trace.c deleted file mode 100644 index 5a6d79bf4..000000000 --- a/openssl/src/ssl/quic/quic_trace.c +++ /dev/null @@ -1,641 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "../ssl_local.h" -#include "internal/quic_wire_pkt.h" - -static const char *packet_type(int type) -{ - switch (type) { - case QUIC_PKT_TYPE_INITIAL: - return "Initial"; - - case QUIC_PKT_TYPE_0RTT: - return "0RTT"; - - case QUIC_PKT_TYPE_HANDSHAKE: - return "Handshake"; - - case QUIC_PKT_TYPE_RETRY: - return "Retry"; - - case QUIC_PKT_TYPE_1RTT: - return "1RTT"; - - case QUIC_PKT_TYPE_VERSION_NEG: - return "VersionNeg"; - - default: - return "Unknown"; - } -} - -/* Print a non-NUL terminated string to BIO */ -static void put_str(BIO *bio, char *str, size_t slen) -{ - size_t i; - - for (i = 0; i < slen; i++) - BIO_printf(bio, "%c", str[i]); -} - -static void put_data(BIO *bio, const uint8_t *data, size_t datalen) -{ - size_t i; - - for (i = 0; i < datalen; i++) - BIO_printf(bio, "%02x", data[i]); -} - -static void put_conn_id(BIO *bio, QUIC_CONN_ID *id) -{ - if (id->id_len == 0) { - BIO_puts(bio, ""); - return; - } - - BIO_puts(bio, "0x"); - put_data(bio, id->id, id->id_len); -} - -static void put_token(BIO *bio, const uint8_t *token, size_t token_len) -{ - if (token_len == 0) - BIO_puts(bio, ""); - else - put_data(bio, token, token_len); -} - -static int frame_ack(BIO *bio, PACKET *pkt) -{ - OSSL_QUIC_FRAME_ACK ack; - OSSL_QUIC_ACK_RANGE *ack_ranges = NULL; - uint64_t total_ranges = 0; - uint64_t i; - - if (!ossl_quic_wire_peek_frame_ack_num_ranges(pkt, &total_ranges) - /* In case sizeof(uint64_t) > sizeof(size_t) */ - || total_ranges > SIZE_MAX / sizeof(ack_ranges[0]) - || (ack_ranges = OPENSSL_zalloc(sizeof(ack_ranges[0]) - * (size_t)total_ranges)) == NULL) - return 0; - - ack.ack_ranges = ack_ranges; - ack.num_ack_ranges = (size_t)total_ranges; - - /* Ack delay exponent is 0, so we can get the raw delay time below */ - if (!ossl_quic_wire_decode_frame_ack(pkt, 0, &ack, NULL)) - return 0; - - BIO_printf(bio, " Largest acked: %llu\n", - (unsigned long long)ack.ack_ranges[0].end); - BIO_printf(bio, " Ack delay (raw) %llu\n", - (unsigned long long)ossl_time2ticks(ack.delay_time)); - BIO_printf(bio, " Ack range count: %llu\n", - (unsigned long long)total_ranges - 1); - BIO_printf(bio, " First ack range: %llu\n", - (unsigned long long)(ack.ack_ranges[0].end - - ack.ack_ranges[0].start)); - for (i = 1; i < total_ranges; i++) { - BIO_printf(bio, " Gap: %llu\n", - (unsigned long long)(ack.ack_ranges[i - 1].start - - ack.ack_ranges[i].end - 2)); - BIO_printf(bio, " Ack range len: %llu\n", - (unsigned long long)(ack.ack_ranges[i].end - - ack.ack_ranges[i].start)); - } - - OPENSSL_free(ack_ranges); - return 1; -} - -static int frame_reset_stream(BIO *bio, PACKET *pkt) -{ - OSSL_QUIC_FRAME_RESET_STREAM frame_data; - - if (!ossl_quic_wire_decode_frame_reset_stream(pkt, &frame_data)) - return 0; - - BIO_printf(bio, " Stream id: %llu\n", - (unsigned long long)frame_data.stream_id); - BIO_printf(bio, " App Protocol Error Code: %llu\n", - (unsigned long long)frame_data.app_error_code); - BIO_printf(bio, " Final size: %llu\n", - (unsigned long long)frame_data.final_size); - - return 1; -} - -static int frame_stop_sending(BIO *bio, PACKET *pkt) -{ - OSSL_QUIC_FRAME_STOP_SENDING frame_data; - - if (!ossl_quic_wire_decode_frame_stop_sending(pkt, &frame_data)) - return 0; - - BIO_printf(bio, " Stream id: %llu\n", - (unsigned long long)frame_data.stream_id); - BIO_printf(bio, " App Protocol Error Code: %llu\n", - (unsigned long long)frame_data.app_error_code); - - return 1; -} - -static int frame_crypto(BIO *bio, PACKET *pkt) -{ - OSSL_QUIC_FRAME_CRYPTO frame_data; - - if (!ossl_quic_wire_decode_frame_crypto(pkt, 1, &frame_data)) - return 0; - - BIO_printf(bio, " Offset: %llu\n", (unsigned long long)frame_data.offset); - BIO_printf(bio, " Len: %llu\n", (unsigned long long)frame_data.len); - - return 1; -} - -static int frame_new_token(BIO *bio, PACKET *pkt) -{ - const uint8_t *token; - size_t token_len; - - if (!ossl_quic_wire_decode_frame_new_token(pkt, &token, &token_len)) - return 0; - - BIO_puts(bio, " Token: "); - put_token(bio, token, token_len); - BIO_puts(bio, "\n"); - - return 1; -} - -static int frame_stream(BIO *bio, PACKET *pkt, uint64_t frame_type) -{ - - OSSL_QUIC_FRAME_STREAM frame_data; - - BIO_puts(bio, "Stream"); - switch(frame_type) { - case OSSL_QUIC_FRAME_TYPE_STREAM: - BIO_puts(bio, "\n"); - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM_FIN: - BIO_puts(bio, " (Fin)\n"); - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM_LEN: - BIO_puts(bio, " (Len)\n"); - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM_LEN_FIN: - BIO_puts(bio, " (Len, Fin)\n"); - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF: - BIO_puts(bio, " (Off)\n"); - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_FIN: - BIO_puts(bio, " (Off, Fin)\n"); - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN: - BIO_puts(bio, " (Off, Len)\n"); - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN_FIN: - BIO_puts(bio, " (Off, Len, Fin)\n"); - break; - - default: - return 0; - } - - if (!ossl_quic_wire_decode_frame_stream(pkt, 1, &frame_data)) - return 0; - - BIO_printf(bio, " Stream id: %llu\n", - (unsigned long long)frame_data.stream_id); - BIO_printf(bio, " Offset: %llu\n", - (unsigned long long)frame_data.offset); - /* - * It would be nice to find a way of passing the implicit length through - * to the msg_callback. But this is not currently possible. - */ - if (frame_data.has_explicit_len) - BIO_printf(bio, " Len: %llu\n", (unsigned long long)frame_data.len); - else - BIO_puts(bio, " Len: \n"); - - return 1; -} - -static int frame_max_data(BIO *bio, PACKET *pkt) -{ - uint64_t max_data = 0; - - if (!ossl_quic_wire_decode_frame_max_data(pkt, &max_data)) - return 0; - - BIO_printf(bio, " Max Data: %llu\n", (unsigned long long)max_data); - - return 1; -} - -static int frame_max_stream_data(BIO *bio, PACKET *pkt) -{ - uint64_t stream_id = 0; - uint64_t max_stream_data = 0; - - if (!ossl_quic_wire_decode_frame_max_stream_data(pkt, &stream_id, - &max_stream_data)) - return 0; - - BIO_printf(bio, " Max Stream Data: %llu\n", - (unsigned long long)max_stream_data); - - return 1; -} - -static int frame_max_streams(BIO *bio, PACKET *pkt) -{ - uint64_t max_streams = 0; - - if (!ossl_quic_wire_decode_frame_max_streams(pkt, &max_streams)) - return 0; - - BIO_printf(bio, " Max Streams: %llu\n", (unsigned long long)max_streams); - - return 1; -} - -static int frame_data_blocked(BIO *bio, PACKET *pkt) -{ - uint64_t max_data = 0; - - if (!ossl_quic_wire_decode_frame_data_blocked(pkt, &max_data)) - return 0; - - BIO_printf(bio, " Max Data: %llu\n", (unsigned long long)max_data); - - return 1; -} - -static int frame_stream_data_blocked(BIO *bio, PACKET *pkt) -{ - uint64_t stream_id = 0; - uint64_t max_data = 0; - - if (!ossl_quic_wire_decode_frame_stream_data_blocked(pkt, &stream_id, - &max_data)) - return 0; - - BIO_printf(bio, " Stream id: %llu\n", (unsigned long long)stream_id); - BIO_printf(bio, " Max Data: %llu\n", (unsigned long long)max_data); - - return 1; -} - -static int frame_streams_blocked(BIO *bio, PACKET *pkt) -{ - uint64_t max_data = 0; - - if (!ossl_quic_wire_decode_frame_streams_blocked(pkt, &max_data)) - return 0; - - BIO_printf(bio, " Max Data: %llu\n", (unsigned long long)max_data); - - return 1; -} - -static int frame_new_conn_id(BIO *bio, PACKET *pkt) -{ - OSSL_QUIC_FRAME_NEW_CONN_ID frame_data; - - if (!ossl_quic_wire_decode_frame_new_conn_id(pkt, &frame_data)) - return 0; - - BIO_printf(bio, " Sequence Number: %llu\n", - (unsigned long long)frame_data.seq_num); - BIO_printf(bio, " Retire prior to: %llu\n", - (unsigned long long)frame_data.retire_prior_to); - BIO_puts(bio, " Connection id: "); - put_conn_id(bio, &frame_data.conn_id); - BIO_puts(bio, "\n Stateless Reset Token: "); - put_data(bio, frame_data.stateless_reset.token, - sizeof(frame_data.stateless_reset.token)); - BIO_puts(bio, "\n"); - - return 1; -} - -static int frame_retire_conn_id(BIO *bio, PACKET *pkt) -{ - uint64_t seq_num; - - if (!ossl_quic_wire_decode_frame_retire_conn_id(pkt, &seq_num)) - return 0; - - BIO_printf(bio, " Sequence Number: %llu\n", (unsigned long long)seq_num); - - return 1; -} - -static int frame_path_challenge(BIO *bio, PACKET *pkt) -{ - uint64_t data = 0; - - if (!ossl_quic_wire_decode_frame_path_challenge(pkt, &data)) - return 0; - - BIO_printf(bio, " Data: %016llx\n", (unsigned long long)data); - - return 1; -} - -static int frame_path_response(BIO *bio, PACKET *pkt) -{ - uint64_t data = 0; - - if (!ossl_quic_wire_decode_frame_path_response(pkt, &data)) - return 0; - - BIO_printf(bio, " Data: %016llx\n", (unsigned long long)data); - - return 1; -} - -static int frame_conn_closed(BIO *bio, PACKET *pkt) -{ - OSSL_QUIC_FRAME_CONN_CLOSE frame_data; - - if (!ossl_quic_wire_decode_frame_conn_close(pkt, &frame_data)) - return 0; - - BIO_printf(bio, " Error Code: %llu\n", - (unsigned long long)frame_data.error_code); - BIO_puts(bio, " Reason: "); - put_str(bio, frame_data.reason, frame_data.reason_len); - BIO_puts(bio, "\n"); - - return 1; -} - -static int trace_frame_data(BIO *bio, PACKET *pkt) -{ - uint64_t frame_type; - - if (!ossl_quic_wire_peek_frame_header(pkt, &frame_type, NULL)) - return 0; - - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_PING: - BIO_puts(bio, "Ping\n"); - if (!ossl_quic_wire_decode_frame_ping(pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_PADDING: - BIO_puts(bio, "Padding\n"); - ossl_quic_wire_decode_padding(pkt); - break; - - case OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN: - case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN: - BIO_puts(bio, "Ack "); - if (frame_type == OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN) - BIO_puts(bio, " (with ECN)\n"); - else - BIO_puts(bio, " (without ECN)\n"); - if (!frame_ack(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_RESET_STREAM: - BIO_puts(bio, "Reset stream\n"); - if (!frame_reset_stream(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_STOP_SENDING: - BIO_puts(bio, "Stop sending\n"); - if (!frame_stop_sending(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_CRYPTO: - BIO_puts(bio, "Crypto\n"); - if (!frame_crypto(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN: - BIO_puts(bio, "New token\n"); - if (!frame_new_token(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM: - case OSSL_QUIC_FRAME_TYPE_STREAM_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_LEN: - case OSSL_QUIC_FRAME_TYPE_STREAM_LEN_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_FIN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN: - case OSSL_QUIC_FRAME_TYPE_STREAM_OFF_LEN_FIN: - /* frame_stream() prints the frame type string */ - if (!frame_stream(bio, pkt, frame_type)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_MAX_DATA: - BIO_puts(bio, "Max data\n"); - if (!frame_max_data(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA: - BIO_puts(bio, "Max stream data\n"); - if (!frame_max_stream_data(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI: - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI: - BIO_puts(bio, "Max streams "); - if (frame_type == OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI) - BIO_puts(bio, " (Bidi)\n"); - else - BIO_puts(bio, " (Uni)\n"); - if (!frame_max_streams(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_DATA_BLOCKED: - BIO_puts(bio, "Data blocked\n"); - if (!frame_data_blocked(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED: - BIO_puts(bio, "Stream data blocked\n"); - if (!frame_stream_data_blocked(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI: - case OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_UNI: - BIO_puts(bio, "Streams blocked"); - if (frame_type == OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI) - BIO_puts(bio, " (Bidi)\n"); - else - BIO_puts(bio, " (Uni)\n"); - if (!frame_streams_blocked(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID: - BIO_puts(bio, "New conn id\n"); - if (!frame_new_conn_id(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID: - BIO_puts(bio, "Retire conn id\n"); - if (!frame_retire_conn_id(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_PATH_CHALLENGE: - BIO_puts(bio, "Path challenge\n"); - if (!frame_path_challenge(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE: - BIO_puts(bio, "Path response\n"); - if (!frame_path_response(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_APP: - case OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT: - BIO_puts(bio, "Connection close"); - if (frame_type == OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_APP) - BIO_puts(bio, " (app)\n"); - else - BIO_puts(bio, " (transport)\n"); - if (!frame_conn_closed(bio, pkt)) - return 0; - break; - - case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE: - BIO_puts(bio, "Handshake done\n"); - if (!ossl_quic_wire_decode_frame_handshake_done(pkt)) - return 0; - break; - - default: - return 0; - } - - if (PACKET_remaining(pkt) != 0) - BIO_puts(bio, " \n"); - - return 1; -} - -int ossl_quic_trace(int write_p, int version, int content_type, - const void *buf, size_t msglen, SSL *ssl, void *arg) -{ - BIO *bio = arg; - PACKET pkt; - - switch (content_type) { - case SSL3_RT_QUIC_DATAGRAM: - BIO_puts(bio, write_p ? "Sent" : "Received"); - /* - * Unfortunately there is no way of receiving auxiliary information - * about the datagram through the msg_callback API such as the peer - * address - */ - BIO_printf(bio, " Datagram\n Length: %zu\n", msglen); - break; - - case SSL3_RT_QUIC_PACKET: - { - QUIC_PKT_HDR hdr; - size_t i; - - if (!PACKET_buf_init(&pkt, buf, msglen)) - return 0; - /* Decode the packet header */ - /* - * TODO(QUIC SERVER): We need to query the short connection id len - * here, e.g. via some API SSL_get_short_conn_id_len() - */ - if (ossl_quic_wire_decode_pkt_hdr(&pkt, 0, 0, 1, &hdr, NULL) != 1) - return 0; - - BIO_puts(bio, write_p ? "Sent" : "Received"); - BIO_puts(bio, " Packet\n"); - BIO_printf(bio, " Packet Type: %s\n", packet_type(hdr.type)); - if (hdr.type != QUIC_PKT_TYPE_1RTT) - BIO_printf(bio, " Version: 0x%08lx\n", - (unsigned long)hdr.version); - BIO_puts(bio, " Destination Conn Id: "); - put_conn_id(bio, &hdr.dst_conn_id); - BIO_puts(bio, "\n"); - if (hdr.type != QUIC_PKT_TYPE_1RTT) { - BIO_puts(bio, " Source Conn Id: "); - put_conn_id(bio, &hdr.src_conn_id); - BIO_puts(bio, "\n"); - } - BIO_printf(bio, " Payload length: %zu\n", hdr.len); - if (hdr.type == QUIC_PKT_TYPE_INITIAL) { - BIO_puts(bio, " Token: "); - put_token(bio, hdr.token, hdr.token_len); - BIO_puts(bio, "\n"); - } - if (hdr.type != QUIC_PKT_TYPE_VERSION_NEG - && hdr.type != QUIC_PKT_TYPE_RETRY) { - BIO_puts(bio, " Packet Number: 0x"); - /* Will always be at least 1 byte */ - for (i = 0; i < hdr.pn_len; i++) - BIO_printf(bio, "%02x", hdr.pn[i]); - BIO_puts(bio, "\n"); - } - break; - } - - case SSL3_RT_QUIC_FRAME_PADDING: - case SSL3_RT_QUIC_FRAME_FULL: - case SSL3_RT_QUIC_FRAME_HEADER: - { - BIO_puts(bio, write_p ? "Sent" : "Received"); - BIO_puts(bio, " Frame: "); - - if (!PACKET_buf_init(&pkt, buf, msglen)) - return 0; - if (!trace_frame_data(bio, &pkt)) { - BIO_puts(bio, " \n"); - return 0; - } - } - break; - - default: - /* Unrecognised content_type. We defer to SSL_trace */ - return 0; - } - - return 1; -} diff --git a/openssl/src/ssl/quic/quic_tserver.c b/openssl/src/ssl/quic/quic_tserver.c deleted file mode 100644 index b9de60aea..000000000 --- a/openssl/src/ssl/quic/quic_tserver.c +++ /dev/null @@ -1,583 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_tserver.h" -#include "internal/quic_channel.h" -#include "internal/quic_statm.h" -#include "internal/quic_port.h" -#include "internal/quic_engine.h" -#include "internal/common.h" -#include "internal/time.h" -#include "quic_local.h" - -/* - * QUIC Test Server Module - * ======================= - */ -struct quic_tserver_st { - QUIC_TSERVER_ARGS args; - - /* Dummy SSL object for this QUIC connection for use by msg_callback */ - SSL *ssl; - - /* - * The QUIC engine, port and channel providing the core QUIC connection - * implementation. - */ - QUIC_ENGINE *engine; - QUIC_PORT *port; - QUIC_CHANNEL *ch; - - /* The mutex we give to the QUIC channel. */ - CRYPTO_MUTEX *mutex; - - /* SSL_CTX for creating the underlying TLS connection */ - SSL_CTX *ctx; - - /* SSL for the underlying TLS connection */ - SSL *tls; - - /* The current peer L4 address. AF_UNSPEC if we do not have a peer yet. */ - BIO_ADDR cur_peer_addr; - - /* Are we connected to a peer? */ - unsigned int connected : 1; -}; - -static int alpn_select_cb(SSL *ssl, const unsigned char **out, - unsigned char *outlen, const unsigned char *in, - unsigned int inlen, void *arg) -{ - QUIC_TSERVER *srv = arg; - static const unsigned char alpndeflt[] = { - 8, 'o', 's', 's', 'l', 't', 'e', 's', 't' - }; - const unsigned char *alpn; - size_t alpnlen; - - if (srv->args.alpn == NULL) { - alpn = alpndeflt; - alpnlen = sizeof(alpn); - } else { - alpn = srv->args.alpn; - alpnlen = srv->args.alpnlen; - } - - if (SSL_select_next_proto((unsigned char **)out, outlen, alpn, alpnlen, - in, inlen) != OPENSSL_NPN_NEGOTIATED) - return SSL_TLSEXT_ERR_ALERT_FATAL; - - return SSL_TLSEXT_ERR_OK; -} - -QUIC_TSERVER *ossl_quic_tserver_new(const QUIC_TSERVER_ARGS *args, - const char *certfile, const char *keyfile) -{ - QUIC_TSERVER *srv = NULL; - QUIC_ENGINE_ARGS engine_args = {0}; - QUIC_PORT_ARGS port_args = {0}; - QUIC_CONNECTION *qc = NULL; - - if (args->net_rbio == NULL || args->net_wbio == NULL) - goto err; - - if ((srv = OPENSSL_zalloc(sizeof(*srv))) == NULL) - goto err; - - srv->args = *args; - -#if defined(OPENSSL_THREADS) - if ((srv->mutex = ossl_crypto_mutex_new()) == NULL) - goto err; -#endif - - if (args->ctx != NULL) - srv->ctx = args->ctx; - else - srv->ctx = SSL_CTX_new_ex(srv->args.libctx, srv->args.propq, - TLS_method()); - if (srv->ctx == NULL) - goto err; - - if (certfile != NULL - && SSL_CTX_use_certificate_file(srv->ctx, certfile, SSL_FILETYPE_PEM) <= 0) - goto err; - - if (keyfile != NULL - && SSL_CTX_use_PrivateKey_file(srv->ctx, keyfile, SSL_FILETYPE_PEM) <= 0) - goto err; - - SSL_CTX_set_alpn_select_cb(srv->ctx, alpn_select_cb, srv); - - srv->tls = SSL_new(srv->ctx); - if (srv->tls == NULL) - goto err; - - engine_args.libctx = srv->args.libctx; - engine_args.propq = srv->args.propq; - engine_args.mutex = srv->mutex; - engine_args.now_cb = srv->args.now_cb; - engine_args.now_cb_arg = srv->args.now_cb_arg; - - if ((srv->engine = ossl_quic_engine_new(&engine_args)) == NULL) - goto err; - - port_args.channel_ctx = srv->ctx; - port_args.is_multi_conn = 1; - - if ((srv->port = ossl_quic_engine_create_port(srv->engine, &port_args)) == NULL) - goto err; - - if ((srv->ch = ossl_quic_port_create_incoming(srv->port, srv->tls)) == NULL) - goto err; - - if (!ossl_quic_port_set_net_rbio(srv->port, srv->args.net_rbio) - || !ossl_quic_port_set_net_wbio(srv->port, srv->args.net_wbio)) - goto err; - - qc = OPENSSL_zalloc(sizeof(*qc)); - if (qc == NULL) - goto err; - srv->ssl = (SSL *)qc; - qc->ch = srv->ch; - srv->ssl->type = SSL_TYPE_QUIC_CONNECTION; - - return srv; - -err: - if (srv != NULL) { - if (args->ctx == NULL) - SSL_CTX_free(srv->ctx); - SSL_free(srv->tls); - ossl_quic_channel_free(srv->ch); - ossl_quic_port_free(srv->port); - ossl_quic_engine_free(srv->engine); -#if defined(OPENSSL_THREADS) - ossl_crypto_mutex_free(&srv->mutex); -#endif - OPENSSL_free(qc); - } - - OPENSSL_free(srv); - return NULL; -} - -void ossl_quic_tserver_free(QUIC_TSERVER *srv) -{ - if (srv == NULL) - return; - - SSL_free(srv->tls); - ossl_quic_channel_free(srv->ch); - ossl_quic_port_free(srv->port); - ossl_quic_engine_free(srv->engine); - BIO_free_all(srv->args.net_rbio); - BIO_free_all(srv->args.net_wbio); - OPENSSL_free(srv->ssl); - SSL_CTX_free(srv->ctx); -#if defined(OPENSSL_THREADS) - ossl_crypto_mutex_free(&srv->mutex); -#endif - OPENSSL_free(srv); -} - -/* Set mutator callbacks for test framework support */ -int ossl_quic_tserver_set_plain_packet_mutator(QUIC_TSERVER *srv, - ossl_mutate_packet_cb mutatecb, - ossl_finish_mutate_cb finishmutatecb, - void *mutatearg) -{ - return ossl_quic_channel_set_mutator(srv->ch, mutatecb, finishmutatecb, - mutatearg); -} - -int ossl_quic_tserver_set_handshake_mutator(QUIC_TSERVER *srv, - ossl_statem_mutate_handshake_cb mutate_handshake_cb, - ossl_statem_finish_mutate_handshake_cb finish_mutate_handshake_cb, - void *mutatearg) -{ - return ossl_statem_set_mutator(ossl_quic_channel_get0_ssl(srv->ch), - mutate_handshake_cb, - finish_mutate_handshake_cb, - mutatearg); -} - -int ossl_quic_tserver_tick(QUIC_TSERVER *srv) -{ - ossl_quic_reactor_tick(ossl_quic_channel_get_reactor(srv->ch), 0); - - if (ossl_quic_channel_is_active(srv->ch)) - srv->connected = 1; - - return 1; -} - -int ossl_quic_tserver_is_connected(QUIC_TSERVER *srv) -{ - return ossl_quic_channel_is_active(srv->ch); -} - -/* Returns 1 if the server is in any terminating or terminated state */ -int ossl_quic_tserver_is_term_any(const QUIC_TSERVER *srv) -{ - return ossl_quic_channel_is_term_any(srv->ch); -} - -const QUIC_TERMINATE_CAUSE * -ossl_quic_tserver_get_terminate_cause(const QUIC_TSERVER *srv) -{ - return ossl_quic_channel_get_terminate_cause(srv->ch); -} - -/* Returns 1 if the server is in a terminated state */ -int ossl_quic_tserver_is_terminated(const QUIC_TSERVER *srv) -{ - return ossl_quic_channel_is_terminated(srv->ch); -} - -int ossl_quic_tserver_is_handshake_confirmed(const QUIC_TSERVER *srv) -{ - return ossl_quic_channel_is_handshake_confirmed(srv->ch); -} - -int ossl_quic_tserver_read(QUIC_TSERVER *srv, - uint64_t stream_id, - unsigned char *buf, - size_t buf_len, - size_t *bytes_read) -{ - int is_fin = 0; - QUIC_STREAM *qs; - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(srv->ch), - stream_id); - if (qs == NULL) { - int is_client_init - = ((stream_id & QUIC_STREAM_INITIATOR_MASK) - == QUIC_STREAM_INITIATOR_CLIENT); - - /* - * A client-initiated stream might spontaneously come into existence, so - * allow trying to read on a client-initiated stream before it exists, - * assuming the connection is still active. - * Otherwise, fail. - */ - if (!is_client_init || !ossl_quic_channel_is_active(srv->ch)) - return 0; - - *bytes_read = 0; - return 1; - } - - if (qs->recv_state == QUIC_RSTREAM_STATE_DATA_READ - || !ossl_quic_stream_has_recv_buffer(qs)) - return 0; - - if (!ossl_quic_rstream_read(qs->rstream, buf, buf_len, - bytes_read, &is_fin)) - return 0; - - if (*bytes_read > 0) { - /* - * We have read at least one byte from the stream. Inform stream-level - * RXFC of the retirement of controlled bytes. Update the active stream - * status (the RXFC may now want to emit a frame granting more credit to - * the peer). - */ - OSSL_RTT_INFO rtt_info; - - ossl_statm_get_rtt_info(ossl_quic_channel_get_statm(srv->ch), &rtt_info); - - if (!ossl_quic_rxfc_on_retire(&qs->rxfc, *bytes_read, - rtt_info.smoothed_rtt)) - return 0; - } - - if (is_fin) - ossl_quic_stream_map_notify_totally_read(ossl_quic_channel_get_qsm(srv->ch), - qs); - - if (*bytes_read > 0) - ossl_quic_stream_map_update_state(ossl_quic_channel_get_qsm(srv->ch), qs); - - return 1; -} - -int ossl_quic_tserver_has_read_ended(QUIC_TSERVER *srv, uint64_t stream_id) -{ - QUIC_STREAM *qs; - unsigned char buf[1]; - size_t bytes_read = 0; - int is_fin = 0; - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(srv->ch), - stream_id); - - if (qs == NULL) - return 0; - - if (qs->recv_state == QUIC_RSTREAM_STATE_DATA_READ) - return 1; - - if (!ossl_quic_stream_has_recv_buffer(qs)) - return 0; - - /* - * If we do not have the DATA_READ, it is possible we should still return 1 - * if there is a lone FIN (but no more data) remaining to be retired from - * the RSTREAM, for example because ossl_quic_tserver_read() has not been - * called since the FIN was received. - */ - if (!ossl_quic_rstream_peek(qs->rstream, buf, sizeof(buf), - &bytes_read, &is_fin)) - return 0; - - if (is_fin && bytes_read == 0) { - /* If we have a FIN awaiting retirement and no data before it... */ - /* Let RSTREAM know we've consumed this FIN. */ - if (!ossl_quic_rstream_read(qs->rstream, buf, sizeof(buf), - &bytes_read, &is_fin)) - return 0; - - assert(is_fin && bytes_read == 0); - assert(qs->recv_state == QUIC_RSTREAM_STATE_DATA_RECVD); - - ossl_quic_stream_map_notify_totally_read(ossl_quic_channel_get_qsm(srv->ch), - qs); - ossl_quic_stream_map_update_state(ossl_quic_channel_get_qsm(srv->ch), qs); - return 1; - } - - return 0; -} - -int ossl_quic_tserver_write(QUIC_TSERVER *srv, - uint64_t stream_id, - const unsigned char *buf, - size_t buf_len, - size_t *bytes_written) -{ - QUIC_STREAM *qs; - - if (!ossl_quic_channel_is_active(srv->ch)) - return 0; - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(srv->ch), - stream_id); - if (qs == NULL || !ossl_quic_stream_has_send_buffer(qs)) - return 0; - - if (!ossl_quic_sstream_append(qs->sstream, - buf, buf_len, bytes_written)) - return 0; - - if (*bytes_written > 0) - /* - * We have appended at least one byte to the stream. Potentially mark - * the stream as active, depending on FC. - */ - ossl_quic_stream_map_update_state(ossl_quic_channel_get_qsm(srv->ch), qs); - - /* Try and send. */ - ossl_quic_tserver_tick(srv); - return 1; -} - -int ossl_quic_tserver_conclude(QUIC_TSERVER *srv, uint64_t stream_id) -{ - QUIC_STREAM *qs; - - if (!ossl_quic_channel_is_active(srv->ch)) - return 0; - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(srv->ch), - stream_id); - if (qs == NULL || !ossl_quic_stream_has_send_buffer(qs)) - return 0; - - if (!ossl_quic_sstream_get_final_size(qs->sstream, NULL)) { - ossl_quic_sstream_fin(qs->sstream); - ossl_quic_stream_map_update_state(ossl_quic_channel_get_qsm(srv->ch), qs); - } - - ossl_quic_tserver_tick(srv); - return 1; -} - -int ossl_quic_tserver_stream_new(QUIC_TSERVER *srv, - int is_uni, - uint64_t *stream_id) -{ - QUIC_STREAM *qs; - - if (!ossl_quic_channel_is_active(srv->ch)) - return 0; - - if ((qs = ossl_quic_channel_new_stream_local(srv->ch, is_uni)) == NULL) - return 0; - - *stream_id = qs->id; - return 1; -} - -BIO *ossl_quic_tserver_get0_rbio(QUIC_TSERVER *srv) -{ - return srv->args.net_rbio; -} - -SSL_CTX *ossl_quic_tserver_get0_ssl_ctx(QUIC_TSERVER *srv) -{ - return srv->ctx; -} - -int ossl_quic_tserver_stream_has_peer_stop_sending(QUIC_TSERVER *srv, - uint64_t stream_id, - uint64_t *app_error_code) -{ - QUIC_STREAM *qs; - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(srv->ch), - stream_id); - if (qs == NULL) - return 0; - - if (qs->peer_stop_sending && app_error_code != NULL) - *app_error_code = qs->peer_stop_sending_aec; - - return qs->peer_stop_sending; -} - -int ossl_quic_tserver_stream_has_peer_reset_stream(QUIC_TSERVER *srv, - uint64_t stream_id, - uint64_t *app_error_code) -{ - QUIC_STREAM *qs; - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(srv->ch), - stream_id); - if (qs == NULL) - return 0; - - if (ossl_quic_stream_recv_is_reset(qs) && app_error_code != NULL) - *app_error_code = qs->peer_reset_stream_aec; - - return ossl_quic_stream_recv_is_reset(qs); -} - -int ossl_quic_tserver_set_new_local_cid(QUIC_TSERVER *srv, - const QUIC_CONN_ID *conn_id) -{ - /* Replace existing local connection ID in the QUIC_CHANNEL */ - return ossl_quic_channel_replace_local_cid(srv->ch, conn_id); -} - -uint64_t ossl_quic_tserver_pop_incoming_stream(QUIC_TSERVER *srv) -{ - QUIC_STREAM_MAP *qsm = ossl_quic_channel_get_qsm(srv->ch); - QUIC_STREAM *qs = ossl_quic_stream_map_peek_accept_queue(qsm); - - if (qs == NULL) - return UINT64_MAX; - - ossl_quic_stream_map_remove_from_accept_queue(qsm, qs, ossl_time_zero()); - - return qs->id; -} - -int ossl_quic_tserver_is_stream_totally_acked(QUIC_TSERVER *srv, - uint64_t stream_id) -{ - QUIC_STREAM *qs; - - qs = ossl_quic_stream_map_get_by_id(ossl_quic_channel_get_qsm(srv->ch), - stream_id); - if (qs == NULL) - return 1; - - return ossl_quic_sstream_is_totally_acked(qs->sstream); -} - -int ossl_quic_tserver_get_net_read_desired(QUIC_TSERVER *srv) -{ - return ossl_quic_reactor_net_read_desired( - ossl_quic_channel_get_reactor(srv->ch)); -} - -int ossl_quic_tserver_get_net_write_desired(QUIC_TSERVER *srv) -{ - return ossl_quic_reactor_net_write_desired( - ossl_quic_channel_get_reactor(srv->ch)); -} - -OSSL_TIME ossl_quic_tserver_get_deadline(QUIC_TSERVER *srv) -{ - return ossl_quic_reactor_get_tick_deadline( - ossl_quic_channel_get_reactor(srv->ch)); -} - -int ossl_quic_tserver_shutdown(QUIC_TSERVER *srv, uint64_t app_error_code) -{ - ossl_quic_channel_local_close(srv->ch, app_error_code, NULL); - - /* TODO(QUIC SERVER): !SSL_SHUTDOWN_FLAG_NO_STREAM_FLUSH */ - - if (ossl_quic_channel_is_terminated(srv->ch)) - return 1; - - ossl_quic_reactor_tick(ossl_quic_channel_get_reactor(srv->ch), 0); - - return ossl_quic_channel_is_terminated(srv->ch); -} - -int ossl_quic_tserver_ping(QUIC_TSERVER *srv) -{ - if (ossl_quic_channel_is_terminated(srv->ch)) - return 0; - - if (!ossl_quic_channel_ping(srv->ch)) - return 0; - - ossl_quic_reactor_tick(ossl_quic_channel_get_reactor(srv->ch), 0); - return 1; -} - -QUIC_CHANNEL *ossl_quic_tserver_get_channel(QUIC_TSERVER *srv) -{ - return srv->ch; -} - -void ossl_quic_tserver_set_msg_callback(QUIC_TSERVER *srv, - void (*f)(int write_p, int version, - int content_type, - const void *buf, size_t len, - SSL *ssl, void *arg), - void *arg) -{ - ossl_quic_channel_set_msg_callback(srv->ch, f, srv->ssl); - ossl_quic_channel_set_msg_callback_arg(srv->ch, arg); - SSL_set_msg_callback(srv->tls, f); - SSL_set_msg_callback_arg(srv->tls, arg); -} - -int ossl_quic_tserver_new_ticket(QUIC_TSERVER *srv) -{ - return SSL_new_session_ticket(srv->tls); -} - -int ossl_quic_tserver_set_max_early_data(QUIC_TSERVER *srv, - uint32_t max_early_data) -{ - return SSL_set_max_early_data(srv->tls, max_early_data); -} - -void ossl_quic_tserver_set_psk_find_session_cb(QUIC_TSERVER *srv, - SSL_psk_find_session_cb_func cb) -{ - SSL_set_psk_find_session_callback(srv->tls, cb); -} diff --git a/openssl/src/ssl/quic/quic_txp.c b/openssl/src/ssl/quic/quic_txp.c deleted file mode 100644 index 2532d1edc..000000000 --- a/openssl/src/ssl/quic/quic_txp.c +++ /dev/null @@ -1,3155 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_txp.h" -#include "internal/quic_fifd.h" -#include "internal/quic_stream_map.h" -#include "internal/quic_error.h" -#include "internal/common.h" -#include - -#define MIN_CRYPTO_HDR_SIZE 3 - -#define MIN_FRAME_SIZE_HANDSHAKE_DONE 1 -#define MIN_FRAME_SIZE_MAX_DATA 2 -#define MIN_FRAME_SIZE_ACK 5 -#define MIN_FRAME_SIZE_CRYPTO (MIN_CRYPTO_HDR_SIZE + 1) -#define MIN_FRAME_SIZE_STREAM 3 /* minimum useful size (for non-FIN) */ -#define MIN_FRAME_SIZE_MAX_STREAMS_BIDI 2 -#define MIN_FRAME_SIZE_MAX_STREAMS_UNI 2 - -/* - * Packet Archetypes - * ================= - */ - -/* Generate normal packets containing most frame types, subject to EL. */ -#define TX_PACKETISER_ARCHETYPE_NORMAL 0 - -/* - * A probe packet is different in that: - * - It bypasses CC, but *is* counted as in flight for purposes of CC; - * - It must be ACK-eliciting. - */ -#define TX_PACKETISER_ARCHETYPE_PROBE 1 - -/* - * An ACK-only packet is different in that: - * - It bypasses CC, and is considered a 'non-inflight' packet; - * - It may not contain anything other than an ACK frame, not even padding. - */ -#define TX_PACKETISER_ARCHETYPE_ACK_ONLY 2 - -#define TX_PACKETISER_ARCHETYPE_NUM 3 - -struct ossl_quic_tx_packetiser_st { - OSSL_QUIC_TX_PACKETISER_ARGS args; - - /* - * Opaque initial token blob provided by caller. TXP frees using the - * callback when it is no longer needed. - */ - const unsigned char *initial_token; - size_t initial_token_len; - ossl_quic_initial_token_free_fn *initial_token_free_cb; - void *initial_token_free_cb_arg; - - /* Subcomponents of the TXP that we own. */ - QUIC_FIFD fifd; /* QUIC Frame-in-Flight Dispatcher */ - - /* Internal state. */ - uint64_t next_pn[QUIC_PN_SPACE_NUM]; /* Next PN to use in given PN space. */ - OSSL_TIME last_tx_time; /* Last time a packet was generated, or 0. */ - - /* Internal state - frame (re)generation flags. */ - unsigned int want_handshake_done : 1; - unsigned int want_max_data : 1; - unsigned int want_max_streams_bidi : 1; - unsigned int want_max_streams_uni : 1; - - /* Internal state - frame (re)generation flags - per PN space. */ - unsigned int want_ack : QUIC_PN_SPACE_NUM; - unsigned int force_ack_eliciting : QUIC_PN_SPACE_NUM; - - /* - * Internal state - connection close terminal state. - * Once this is set, it is not unset unlike other want_ flags - we keep - * sending it in every packet. - */ - unsigned int want_conn_close : 1; - - /* Has the handshake been completed? */ - unsigned int handshake_complete : 1; - - OSSL_QUIC_FRAME_CONN_CLOSE conn_close_frame; - - /* - * Counts of the number of bytes received and sent while in the closing - * state. - */ - uint64_t closing_bytes_recv; - uint64_t closing_bytes_xmit; - - /* Internal state - packet assembly. */ - struct txp_el { - unsigned char *scratch; /* scratch buffer for packet assembly */ - size_t scratch_len; /* number of bytes allocated for scratch */ - OSSL_QTX_IOVEC *iovec; /* scratch iovec array for use with QTX */ - size_t alloc_iovec; /* size of iovec array */ - } el[QUIC_ENC_LEVEL_NUM]; - - /* Message callback related arguments */ - ossl_msg_cb msg_callback; - void *msg_callback_arg; - SSL *msg_callback_ssl; - - /* Callbacks. */ - void (*ack_tx_cb)(const OSSL_QUIC_FRAME_ACK *ack, - uint32_t pn_space, - void *arg); - void *ack_tx_cb_arg; -}; - -/* - * The TX helper records state used while generating frames into packets. It - * enables serialization into the packet to be done "transactionally" where - * serialization of a frame can be rolled back if it fails midway (e.g. if it - * does not fit). - */ -struct tx_helper { - OSSL_QUIC_TX_PACKETISER *txp; - /* - * The Maximum Packet Payload Length in bytes. This is the amount of - * space we have to generate frames into. - */ - size_t max_ppl; - /* - * Number of bytes we have generated so far. - */ - size_t bytes_appended; - /* - * Number of scratch bytes in txp->scratch we have used so far. Some iovecs - * will reference this scratch buffer. When we need to use more of it (e.g. - * when we need to put frame headers somewhere), we append to the scratch - * buffer, resizing if necessary, and increase this accordingly. - */ - size_t scratch_bytes; - /* - * Bytes reserved in the MaxPPL budget. We keep this number of bytes spare - * until reserve_allowed is set to 1. Currently this is always at most 1, as - * a PING frame takes up one byte and this mechanism is only used to ensure - * we can encode a PING frame if we have been asked to ensure a packet is - * ACK-eliciting and we are unusure if we are going to add any other - * ACK-eliciting frames before we reach our MaxPPL budget. - */ - size_t reserve; - /* - * Number of iovecs we have currently appended. This is the number of - * entries valid in txp->iovec. - */ - size_t num_iovec; - /* The EL this TX helper is being used for. */ - uint32_t enc_level; - /* - * Whether we are allowed to make use of the reserve bytes in our MaxPPL - * budget. This is used to ensure we have room to append a PING frame later - * if we need to. Once we know we will not need to append a PING frame, this - * is set to 1. - */ - unsigned int reserve_allowed : 1; - /* - * Set to 1 if we have appended a STREAM frame with an implicit length. If - * this happens we should never append another frame after that frame as it - * cannot be validly encoded. This is just a safety check. - */ - unsigned int done_implicit : 1; - struct { - /* - * The fields in this structure are valid if active is set, which means - * that a serialization transaction is currently in progress. - */ - unsigned char *data; - WPACKET wpkt; - unsigned int active : 1; - } txn; -}; - -static void tx_helper_rollback(struct tx_helper *h); -static int txp_el_ensure_iovec(struct txp_el *el, size_t num); - -/* Initialises the TX helper. */ -static int tx_helper_init(struct tx_helper *h, OSSL_QUIC_TX_PACKETISER *txp, - uint32_t enc_level, size_t max_ppl, size_t reserve) -{ - if (reserve > max_ppl) - return 0; - - h->txp = txp; - h->enc_level = enc_level; - h->max_ppl = max_ppl; - h->reserve = reserve; - h->num_iovec = 0; - h->bytes_appended = 0; - h->scratch_bytes = 0; - h->reserve_allowed = 0; - h->done_implicit = 0; - h->txn.data = NULL; - h->txn.active = 0; - - if (max_ppl > h->txp->el[enc_level].scratch_len) { - unsigned char *scratch; - - scratch = OPENSSL_realloc(h->txp->el[enc_level].scratch, max_ppl); - if (scratch == NULL) - return 0; - - h->txp->el[enc_level].scratch = scratch; - h->txp->el[enc_level].scratch_len = max_ppl; - } - - return 1; -} - -static void tx_helper_cleanup(struct tx_helper *h) -{ - if (h->txn.active) - tx_helper_rollback(h); - - h->txp = NULL; -} - -static void tx_helper_unrestrict(struct tx_helper *h) -{ - h->reserve_allowed = 1; -} - -/* - * Append an extent of memory to the iovec list. The memory must remain - * allocated until we finish generating the packet and call the QTX. - * - * In general, the buffers passed to this function will be from one of two - * ranges: - * - * - Application data contained in stream buffers managed elsewhere - * in the QUIC stack; or - * - * - Control frame data appended into txp->scratch using tx_helper_begin and - * tx_helper_commit. - * - */ -static int tx_helper_append_iovec(struct tx_helper *h, - const unsigned char *buf, - size_t buf_len) -{ - struct txp_el *el = &h->txp->el[h->enc_level]; - - if (buf_len == 0) - return 1; - - if (!ossl_assert(!h->done_implicit)) - return 0; - - if (!txp_el_ensure_iovec(el, h->num_iovec + 1)) - return 0; - - el->iovec[h->num_iovec].buf = buf; - el->iovec[h->num_iovec].buf_len = buf_len; - - ++h->num_iovec; - h->bytes_appended += buf_len; - return 1; -} - -/* - * How many more bytes of space do we have left in our plaintext packet payload? - */ -static size_t tx_helper_get_space_left(struct tx_helper *h) -{ - return h->max_ppl - - (h->reserve_allowed ? 0 : h->reserve) - h->bytes_appended; -} - -/* - * Begin a control frame serialization transaction. This allows the - * serialization of the control frame to be backed out if it turns out it won't - * fit. Write the control frame to the returned WPACKET. Ensure you always - * call tx_helper_rollback or tx_helper_commit (or tx_helper_cleanup). Returns - * NULL on failure. - */ -static WPACKET *tx_helper_begin(struct tx_helper *h) -{ - size_t space_left, len; - unsigned char *data; - struct txp_el *el = &h->txp->el[h->enc_level]; - - if (!ossl_assert(!h->txn.active)) - return NULL; - - if (!ossl_assert(!h->done_implicit)) - return NULL; - - data = (unsigned char *)el->scratch + h->scratch_bytes; - len = el->scratch_len - h->scratch_bytes; - - space_left = tx_helper_get_space_left(h); - if (!ossl_assert(space_left <= len)) - return NULL; - - if (!WPACKET_init_static_len(&h->txn.wpkt, data, len, 0)) - return NULL; - - if (!WPACKET_set_max_size(&h->txn.wpkt, space_left)) { - WPACKET_cleanup(&h->txn.wpkt); - return NULL; - } - - h->txn.data = data; - h->txn.active = 1; - return &h->txn.wpkt; -} - -static void tx_helper_end(struct tx_helper *h, int success) -{ - if (success) - WPACKET_finish(&h->txn.wpkt); - else - WPACKET_cleanup(&h->txn.wpkt); - - h->txn.active = 0; - h->txn.data = NULL; -} - -/* Abort a control frame serialization transaction. */ -static void tx_helper_rollback(struct tx_helper *h) -{ - if (!h->txn.active) - return; - - tx_helper_end(h, 0); -} - -/* Commit a control frame. */ -static int tx_helper_commit(struct tx_helper *h) -{ - size_t l = 0; - - if (!h->txn.active) - return 0; - - if (!WPACKET_get_total_written(&h->txn.wpkt, &l)) { - tx_helper_end(h, 0); - return 0; - } - - if (!tx_helper_append_iovec(h, h->txn.data, l)) { - tx_helper_end(h, 0); - return 0; - } - - if (h->txp->msg_callback != NULL && l > 0) { - uint64_t ftype; - int ctype = SSL3_RT_QUIC_FRAME_FULL; - PACKET pkt; - - if (!PACKET_buf_init(&pkt, h->txn.data, l) - || !ossl_quic_wire_peek_frame_header(&pkt, &ftype, NULL)) { - tx_helper_end(h, 0); - return 0; - } - - if (ftype == OSSL_QUIC_FRAME_TYPE_PADDING) - ctype = SSL3_RT_QUIC_FRAME_PADDING; - else if (OSSL_QUIC_FRAME_TYPE_IS_STREAM(ftype) - || ftype == OSSL_QUIC_FRAME_TYPE_CRYPTO) - ctype = SSL3_RT_QUIC_FRAME_HEADER; - - h->txp->msg_callback(1, OSSL_QUIC1_VERSION, ctype, h->txn.data, l, - h->txp->msg_callback_ssl, - h->txp->msg_callback_arg); - } - - h->scratch_bytes += l; - tx_helper_end(h, 1); - return 1; -} - -struct archetype_data { - unsigned int allow_ack : 1; - unsigned int allow_ping : 1; - unsigned int allow_crypto : 1; - unsigned int allow_handshake_done : 1; - unsigned int allow_path_challenge : 1; - unsigned int allow_path_response : 1; - unsigned int allow_new_conn_id : 1; - unsigned int allow_retire_conn_id : 1; - unsigned int allow_stream_rel : 1; - unsigned int allow_conn_fc : 1; - unsigned int allow_conn_close : 1; - unsigned int allow_cfq_other : 1; - unsigned int allow_new_token : 1; - unsigned int allow_force_ack_eliciting : 1; - unsigned int allow_padding : 1; - unsigned int require_ack_eliciting : 1; - unsigned int bypass_cc : 1; -}; - -struct txp_pkt_geom { - size_t cmpl, cmppl, hwm, pkt_overhead; - uint32_t archetype; - struct archetype_data adata; -}; - -struct txp_pkt { - struct tx_helper h; - int h_valid; - QUIC_TXPIM_PKT *tpkt; - QUIC_STREAM *stream_head; - QUIC_PKT_HDR phdr; - struct txp_pkt_geom geom; - int force_pad; -}; - -static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space, - void *arg); -static void on_regen_notify(uint64_t frame_type, uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, void *arg); -static void on_confirm_notify(uint64_t frame_type, uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, void *arg); -static void on_sstream_updated(uint64_t stream_id, void *arg); -static int sstream_is_pending(QUIC_SSTREAM *sstream); -static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t enc_level, - uint32_t archetype, - uint64_t cc_limit, - uint32_t *conn_close_enc_level); -static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp); -static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp, - size_t pl, - uint32_t enc_level, - size_t hdr_len, - size_t *r); -static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp); -static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp, - struct txp_pkt *pkt, - int chosen_for_conn_close); -static int txp_pkt_init(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp, - uint32_t enc_level, uint32_t archetype, - size_t running_total); -static void txp_pkt_cleanup(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp); -static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt *pkt, - OSSL_QUIC_TX_PACKETISER *txp); -static int txp_pkt_append_padding(struct txp_pkt *pkt, - OSSL_QUIC_TX_PACKETISER *txp, size_t num_bytes); -static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER *txp, struct txp_pkt *pkt, - uint32_t archetype, int *txpim_pkt_reffed); -static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER *txp, - uint64_t cc_limit); - -OSSL_QUIC_TX_PACKETISER *ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS *args) -{ - OSSL_QUIC_TX_PACKETISER *txp; - - if (args == NULL - || args->qtx == NULL - || args->txpim == NULL - || args->cfq == NULL - || args->ackm == NULL - || args->qsm == NULL - || args->conn_txfc == NULL - || args->conn_rxfc == NULL - || args->max_streams_bidi_rxfc == NULL - || args->max_streams_uni_rxfc == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); - return NULL; - } - - txp = OPENSSL_zalloc(sizeof(*txp)); - if (txp == NULL) - return NULL; - - txp->args = *args; - txp->last_tx_time = ossl_time_zero(); - - if (!ossl_quic_fifd_init(&txp->fifd, - txp->args.cfq, txp->args.ackm, txp->args.txpim, - get_sstream_by_id, txp, - on_regen_notify, txp, - on_confirm_notify, txp, - on_sstream_updated, txp, - args->get_qlog_cb, - args->get_qlog_cb_arg)) { - OPENSSL_free(txp); - return NULL; - } - - return txp; -} - -void ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER *txp) -{ - uint32_t enc_level; - - if (txp == NULL) - return; - - ossl_quic_tx_packetiser_set_initial_token(txp, NULL, 0, NULL, NULL); - ossl_quic_fifd_cleanup(&txp->fifd); - OPENSSL_free(txp->conn_close_frame.reason); - - for (enc_level = QUIC_ENC_LEVEL_INITIAL; - enc_level < QUIC_ENC_LEVEL_NUM; - ++enc_level) { - OPENSSL_free(txp->el[enc_level].iovec); - OPENSSL_free(txp->el[enc_level].scratch); - } - - OPENSSL_free(txp); -} - -/* - * Determine if an Initial packet token length is reasonable based on the - * current MDPL, returning 1 if it is OK. - * - * The real PMTU to the peer could differ from our (pessimistic) understanding - * of the PMTU, therefore it is possible we could receive an Initial token from - * a server in a Retry packet which is bigger than the MDPL. In this case it is - * impossible for us ever to make forward progress and we need to error out - * and fail the connection attempt. - * - * The specific boundary condition is complex: for example, after the size of - * the Initial token, there are the Initial packet header overheads and then - * encryption/AEAD tag overheads. After that, the minimum room for frame data in - * order to guarantee forward progress must be guaranteed. For example, a crypto - * stream needs to always be able to serialize at least one byte in a CRYPTO - * frame in order to make forward progress. Because the offset field of a CRYPTO - * frame uses a variable-length integer, the number of bytes needed to ensure - * this also varies. - * - * Rather than trying to get this boundary condition check actually right, - * require a reasonable amount of slack to avoid pathological behaviours. (After - * all, transmitting a CRYPTO stream one byte at a time is probably not - * desirable anyway.) - * - * We choose 160 bytes as the required margin, which is double the rough - * estimation of the minimum we would require to guarantee forward progress - * under worst case packet overheads. - */ -#define TXP_REQUIRED_TOKEN_MARGIN 160 - -static int txp_check_token_len(size_t token_len, size_t mdpl) -{ - if (token_len == 0) - return 1; - - if (token_len >= mdpl) - return 0; - - if (TXP_REQUIRED_TOKEN_MARGIN >= mdpl) - /* (should not be possible because MDPL must be at least 1200) */ - return 0; - - if (token_len > mdpl - TXP_REQUIRED_TOKEN_MARGIN) - return 0; - - return 1; -} - -int ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER *txp, - const unsigned char *token, - size_t token_len, - ossl_quic_initial_token_free_fn *free_cb, - void *free_cb_arg) -{ - if (!txp_check_token_len(token_len, txp_get_mdpl(txp))) - return 0; - - if (txp->initial_token != NULL && txp->initial_token_free_cb != NULL) - txp->initial_token_free_cb(txp->initial_token, txp->initial_token_len, - txp->initial_token_free_cb_arg); - - txp->initial_token = token; - txp->initial_token_len = token_len; - txp->initial_token_free_cb = free_cb; - txp->initial_token_free_cb_arg = free_cb_arg; - return 1; -} - -int ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER *txp, - const QUIC_CONN_ID *dcid) -{ - if (dcid == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - txp->args.cur_dcid = *dcid; - return 1; -} - -int ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER *txp, - const QUIC_CONN_ID *scid) -{ - if (scid == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); - return 0; - } - - txp->args.cur_scid = *scid; - return 1; -} - -/* Change the destination L4 address the TXP uses to send datagrams. */ -int ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER *txp, - const BIO_ADDR *peer) -{ - if (peer == NULL) { - BIO_ADDR_clear(&txp->args.peer); - return 1; - } - - txp->args.peer = *peer; - return 1; -} - -void ossl_quic_tx_packetiser_set_ack_tx_cb(OSSL_QUIC_TX_PACKETISER *txp, - void (*cb)(const OSSL_QUIC_FRAME_ACK *ack, - uint32_t pn_space, - void *arg), - void *cb_arg) -{ - txp->ack_tx_cb = cb; - txp->ack_tx_cb_arg = cb_arg; -} - -void ossl_quic_tx_packetiser_set_qlog_cb(OSSL_QUIC_TX_PACKETISER *txp, - QLOG *(*get_qlog_cb)(void *arg), - void *get_qlog_cb_arg) -{ - ossl_quic_fifd_set_qlog_cb(&txp->fifd, get_qlog_cb, get_qlog_cb_arg); - -} - -int ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t enc_level) -{ - if (enc_level >= QUIC_ENC_LEVEL_NUM) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - if (enc_level != QUIC_ENC_LEVEL_0RTT) - txp->args.crypto[ossl_quic_enc_level_to_pn_space(enc_level)] = NULL; - - return 1; -} - -void ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER *txp) -{ - txp->handshake_complete = 1; -} - -void ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER *txp) -{ - txp->want_handshake_done = 1; -} - -void ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t pn_space) -{ - txp->force_ack_eliciting |= (1UL << pn_space); -} - -void ossl_quic_tx_packetiser_schedule_ack(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t pn_space) -{ - txp->want_ack |= (1UL << pn_space); -} - -#define TXP_ERR_INTERNAL 0 /* Internal (e.g. alloc) error */ -#define TXP_ERR_SUCCESS 1 /* Success */ -#define TXP_ERR_SPACE 2 /* Not enough room for another packet */ -#define TXP_ERR_INPUT 3 /* Invalid/malformed input */ - -/* - * Generates a datagram by polling the various ELs to determine if they want to - * generate any frames, and generating a datagram which coalesces packets for - * any ELs which do. - */ -int ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER *txp, - QUIC_TXP_STATUS *status) -{ - /* - * Called to generate one or more datagrams, each containing one or more - * packets. - * - * There are some tricky things to note here: - * - * - The TXP is only concerned with generating encrypted packets; - * other packets use a different path. - * - * - Any datagram containing an Initial packet must have a payload length - * (DPL) of at least 1200 bytes. This padding need not necessarily be - * found in the Initial packet. - * - * - It is desirable to be able to coalesce an Initial packet - * with a Handshake packet. Since, before generating the Handshake - * packet, we do not know how long it will be, we cannot know the - * correct amount of padding to ensure a DPL of at least 1200 bytes. - * Thus this padding must added to the Handshake packet (or whatever - * packet is the last in the datagram). - * - * - However, at the time that we generate the Initial packet, - * we do not actually know for sure that we will be followed - * in the datagram by another packet. For example, suppose we have - * some queued data (e.g. crypto stream data for the HANDSHAKE EL) - * it looks like we will want to send on the HANDSHAKE EL. - * We could assume padding will be placed in the Handshake packet - * subsequently and avoid adding any padding to the Initial packet - * (which would leave no room for the Handshake packet in the - * datagram). - * - * However, this is not actually a safe assumption. Suppose that we - * are using a link with a MDPL of 1200 bytes, the minimum allowed by - * QUIC. Suppose that the Initial packet consumes 1195 bytes in total. - * Since it is not possible to fit a Handshake packet in just 5 bytes, - * upon trying to add a Handshake packet after generating the Initial - * packet, we will discover we have no room to fit it! This is not a - * problem in itself as another datagram can be sent subsequently, but - * it is a problem because we were counting to use that packet to hold - * the essential padding. But if we have already finished encrypting - * the Initial packet, we cannot go and add padding to it anymore. - * This leaves us stuck. - * - * Because of this, we have to plan multiple packets simultaneously, such - * that we can start generating a Handshake (or 0-RTT or 1-RTT, or so on) - * packet while still having the option to go back and add padding to the - * Initial packet if it turns out to be needed. - * - * Trying to predict ahead of time (e.g. during Initial packet generation) - * whether we will successfully generate a subsequent packet is fraught with - * error as it relies on a large number of variables: - * - * - Do we have room to fit a packet header? (Consider that due to - * variable-length integer encoding this is highly variable and can even - * depend on payload length due to a variable-length Length field.) - * - * - Can we fit even a single one of the frames we want to put in this - * packet in the packet? (Each frame type has a bespoke encoding. While - * our encodings of some frame types are adaptive based on the available - * room - e.g. STREAM frames - ultimately all frame types have some - * absolute minimum number of bytes to be successfully encoded. For - * example, if after an Initial packet there is enough room to encode - * only one byte of frame data, it is quite likely we can't send any of - * the frames we wanted to send.) While this is not strictly a problem - * because we could just fill the packet with padding frames, this is a - * pointless packet and is wasteful. - * - * Thus we adopt a multi-phase architecture: - * - * 1. Archetype Selection: Determine desired packet archetype. - * - * 2. Packet Staging: Generation of packet information and packet payload - * data (frame data) into staging areas. - * - * 3. Packet Adjustment: Adjustment of staged packets, adding padding to - * the staged packets if needed. - * - * 4. Commit: The packets are sent to the QTX and recorded as having been - * sent to the FIFM. - * - */ - int res = 0, rc; - uint32_t archetype, enc_level; - uint32_t conn_close_enc_level = QUIC_ENC_LEVEL_NUM; - struct txp_pkt pkt[QUIC_ENC_LEVEL_NUM]; - size_t pkts_done = 0; - uint64_t cc_limit = txp->args.cc_method->get_tx_allowance(txp->args.cc_data); - int need_padding = 0, txpim_pkt_reffed; - - for (enc_level = QUIC_ENC_LEVEL_INITIAL; - enc_level < QUIC_ENC_LEVEL_NUM; - ++enc_level) - pkt[enc_level].h_valid = 0; - - memset(status, 0, sizeof(*status)); - - /* - * Should not be needed, but a sanity check in case anyone else has been - * using the QTX. - */ - ossl_qtx_finish_dgram(txp->args.qtx); - - /* 1. Archetype Selection */ - archetype = txp_determine_archetype(txp, cc_limit); - - /* 2. Packet Staging */ - for (enc_level = QUIC_ENC_LEVEL_INITIAL; - enc_level < QUIC_ENC_LEVEL_NUM; - ++enc_level) { - size_t running_total = (enc_level > QUIC_ENC_LEVEL_INITIAL) - ? pkt[enc_level - 1].geom.hwm : 0; - - pkt[enc_level].geom.hwm = running_total; - - if (!txp_should_try_staging(txp, enc_level, archetype, cc_limit, - &conn_close_enc_level)) - continue; - - if (!txp_pkt_init(&pkt[enc_level], txp, enc_level, archetype, - running_total)) - /* - * If this fails this is not a fatal error - it means the geometry - * planning determined there was not enough space for another - * packet. So just proceed with what we've already planned for. - */ - break; - - rc = txp_generate_for_el(txp, &pkt[enc_level], - conn_close_enc_level == enc_level); - if (rc != TXP_ERR_SUCCESS) - goto out; - - if (pkt[enc_level].force_pad) - /* - * txp_generate_for_el emitted a frame which forces packet padding. - */ - need_padding = 1; - - pkt[enc_level].geom.hwm = running_total - + pkt[enc_level].h.bytes_appended - + pkt[enc_level].geom.pkt_overhead; - } - - /* 3. Packet Adjustment */ - if (pkt[QUIC_ENC_LEVEL_INITIAL].h_valid - && pkt[QUIC_ENC_LEVEL_INITIAL].h.bytes_appended > 0) - /* - * We have an Initial packet in this datagram, so we need to make sure - * the total size of the datagram is adequate. - */ - need_padding = 1; - - if (need_padding) { - size_t total_dgram_size = 0; - const size_t min_dpl = QUIC_MIN_INITIAL_DGRAM_LEN; - uint32_t pad_el = QUIC_ENC_LEVEL_NUM; - - for (enc_level = QUIC_ENC_LEVEL_INITIAL; - enc_level < QUIC_ENC_LEVEL_NUM; - ++enc_level) - if (pkt[enc_level].h_valid && pkt[enc_level].h.bytes_appended > 0) { - if (pad_el == QUIC_ENC_LEVEL_NUM - /* - * We might not be able to add padding, for example if we - * are using the ACK_ONLY archetype. - */ - && pkt[enc_level].geom.adata.allow_padding - && !pkt[enc_level].h.done_implicit) - pad_el = enc_level; - - txp_pkt_postgen_update_pkt_overhead(&pkt[enc_level], txp); - total_dgram_size += pkt[enc_level].geom.pkt_overhead - + pkt[enc_level].h.bytes_appended; - } - - if (pad_el != QUIC_ENC_LEVEL_NUM && total_dgram_size < min_dpl) { - size_t deficit = min_dpl - total_dgram_size; - - if (!txp_pkt_append_padding(&pkt[pad_el], txp, deficit)) - goto out; - - total_dgram_size += deficit; - - /* - * Padding frames make a packet ineligible for being a non-inflight - * packet. - */ - pkt[pad_el].tpkt->ackm_pkt.is_inflight = 1; - } - - /* - * If we have failed to make a datagram of adequate size, for example - * because we have a padding requirement but are using the ACK_ONLY - * archetype (because we are CC limited), which precludes us from - * sending padding, give up on generating the datagram - there is - * nothing we can do. - */ - if (total_dgram_size < min_dpl) { - res = 1; - goto out; - } - } - - /* 4. Commit */ - for (enc_level = QUIC_ENC_LEVEL_INITIAL; - enc_level < QUIC_ENC_LEVEL_NUM; - ++enc_level) { - - if (!pkt[enc_level].h_valid) - /* Did not attempt to generate a packet for this EL. */ - continue; - - if (pkt[enc_level].h.bytes_appended == 0) - /* Nothing was generated for this EL, so skip. */ - continue; - - rc = txp_pkt_commit(txp, &pkt[enc_level], archetype, - &txpim_pkt_reffed); - if (rc) { - status->sent_ack_eliciting - = status->sent_ack_eliciting - || pkt[enc_level].tpkt->ackm_pkt.is_ack_eliciting; - - if (enc_level == QUIC_ENC_LEVEL_HANDSHAKE) - status->sent_handshake - = (pkt[enc_level].h_valid - && pkt[enc_level].h.bytes_appended > 0); - } - - if (txpim_pkt_reffed) - pkt[enc_level].tpkt = NULL; /* don't free */ - - if (!rc) - goto out; - - ++pkts_done; - } - - /* Flush & Cleanup */ - res = 1; -out: - ossl_qtx_finish_dgram(txp->args.qtx); - - for (enc_level = QUIC_ENC_LEVEL_INITIAL; - enc_level < QUIC_ENC_LEVEL_NUM; - ++enc_level) - txp_pkt_cleanup(&pkt[enc_level], txp); - - status->sent_pkt = pkts_done; - - return res; -} - -static const struct archetype_data archetypes[QUIC_ENC_LEVEL_NUM][TX_PACKETISER_ARCHETYPE_NUM] = { - /* EL 0(INITIAL) */ - { - /* EL 0(INITIAL) - Archetype 0(NORMAL) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 1, - /*allow_crypto =*/ 1, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 0, - /*allow_retire_conn_id =*/ 0, - /*allow_stream_rel =*/ 0, - /*allow_conn_fc =*/ 0, - /*allow_conn_close =*/ 1, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 1, - /*require_ack_eliciting =*/ 0, - /*bypass_cc =*/ 0, - }, - /* EL 0(INITIAL) - Archetype 1(PROBE) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 1, - /*allow_crypto =*/ 1, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 0, - /*allow_retire_conn_id =*/ 0, - /*allow_stream_rel =*/ 0, - /*allow_conn_fc =*/ 0, - /*allow_conn_close =*/ 1, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 1, - /*require_ack_eliciting =*/ 1, - /*bypass_cc =*/ 1, - }, - /* EL 0(INITIAL) - Archetype 2(ACK_ONLY) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 0, - /*allow_crypto =*/ 0, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 0, - /*allow_retire_conn_id =*/ 0, - /*allow_stream_rel =*/ 0, - /*allow_conn_fc =*/ 0, - /*allow_conn_close =*/ 0, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 0, - /*require_ack_eliciting =*/ 0, - /*bypass_cc =*/ 1, - }, - }, - /* EL 1(HANDSHAKE) */ - { - /* EL 1(HANDSHAKE) - Archetype 0(NORMAL) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 1, - /*allow_crypto =*/ 1, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 0, - /*allow_retire_conn_id =*/ 0, - /*allow_stream_rel =*/ 0, - /*allow_conn_fc =*/ 0, - /*allow_conn_close =*/ 1, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 1, - /*require_ack_eliciting =*/ 0, - /*bypass_cc =*/ 0, - }, - /* EL 1(HANDSHAKE) - Archetype 1(PROBE) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 1, - /*allow_crypto =*/ 1, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 0, - /*allow_retire_conn_id =*/ 0, - /*allow_stream_rel =*/ 0, - /*allow_conn_fc =*/ 0, - /*allow_conn_close =*/ 1, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 1, - /*require_ack_eliciting =*/ 1, - /*bypass_cc =*/ 1, - }, - /* EL 1(HANDSHAKE) - Archetype 2(ACK_ONLY) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 0, - /*allow_crypto =*/ 0, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 0, - /*allow_retire_conn_id =*/ 0, - /*allow_stream_rel =*/ 0, - /*allow_conn_fc =*/ 0, - /*allow_conn_close =*/ 0, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 0, - /*require_ack_eliciting =*/ 0, - /*bypass_cc =*/ 1, - }, - }, - /* EL 2(0RTT) */ - { - /* EL 2(0RTT) - Archetype 0(NORMAL) */ - { - /*allow_ack =*/ 0, - /*allow_ping =*/ 1, - /*allow_crypto =*/ 0, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 1, - /*allow_retire_conn_id =*/ 1, - /*allow_stream_rel =*/ 1, - /*allow_conn_fc =*/ 1, - /*allow_conn_close =*/ 1, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 0, - /*allow_padding =*/ 1, - /*require_ack_eliciting =*/ 0, - /*bypass_cc =*/ 0, - }, - /* EL 2(0RTT) - Archetype 1(PROBE) */ - { - /*allow_ack =*/ 0, - /*allow_ping =*/ 1, - /*allow_crypto =*/ 0, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 1, - /*allow_retire_conn_id =*/ 1, - /*allow_stream_rel =*/ 1, - /*allow_conn_fc =*/ 1, - /*allow_conn_close =*/ 1, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 0, - /*allow_padding =*/ 1, - /*require_ack_eliciting =*/ 1, - /*bypass_cc =*/ 1, - }, - /* EL 2(0RTT) - Archetype 2(ACK_ONLY) */ - { - /*allow_ack =*/ 0, - /*allow_ping =*/ 0, - /*allow_crypto =*/ 0, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 0, - /*allow_retire_conn_id =*/ 0, - /*allow_stream_rel =*/ 0, - /*allow_conn_fc =*/ 0, - /*allow_conn_close =*/ 0, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 0, - /*allow_padding =*/ 0, - /*require_ack_eliciting =*/ 0, - /*bypass_cc =*/ 1, - }, - }, - /* EL 3(1RTT) */ - { - /* EL 3(1RTT) - Archetype 0(NORMAL) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 1, - /*allow_crypto =*/ 1, - /*allow_handshake_done =*/ 1, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 1, - /*allow_new_conn_id =*/ 1, - /*allow_retire_conn_id =*/ 1, - /*allow_stream_rel =*/ 1, - /*allow_conn_fc =*/ 1, - /*allow_conn_close =*/ 1, - /*allow_cfq_other =*/ 1, - /*allow_new_token =*/ 1, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 1, - /*require_ack_eliciting =*/ 0, - /*bypass_cc =*/ 0, - }, - /* EL 3(1RTT) - Archetype 1(PROBE) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 1, - /*allow_crypto =*/ 1, - /*allow_handshake_done =*/ 1, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 1, - /*allow_new_conn_id =*/ 1, - /*allow_retire_conn_id =*/ 1, - /*allow_stream_rel =*/ 1, - /*allow_conn_fc =*/ 1, - /*allow_conn_close =*/ 1, - /*allow_cfq_other =*/ 1, - /*allow_new_token =*/ 1, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 1, - /*require_ack_eliciting =*/ 1, - /*bypass_cc =*/ 1, - }, - /* EL 3(1RTT) - Archetype 2(ACK_ONLY) */ - { - /*allow_ack =*/ 1, - /*allow_ping =*/ 0, - /*allow_crypto =*/ 0, - /*allow_handshake_done =*/ 0, - /*allow_path_challenge =*/ 0, - /*allow_path_response =*/ 0, - /*allow_new_conn_id =*/ 0, - /*allow_retire_conn_id =*/ 0, - /*allow_stream_rel =*/ 0, - /*allow_conn_fc =*/ 0, - /*allow_conn_close =*/ 0, - /*allow_cfq_other =*/ 0, - /*allow_new_token =*/ 0, - /*allow_force_ack_eliciting =*/ 1, - /*allow_padding =*/ 0, - /*require_ack_eliciting =*/ 0, - /*bypass_cc =*/ 1, - } - } -}; - -static int txp_get_archetype_data(uint32_t enc_level, - uint32_t archetype, - struct archetype_data *a) -{ - if (enc_level >= QUIC_ENC_LEVEL_NUM - || archetype >= TX_PACKETISER_ARCHETYPE_NUM) - return 0; - - /* No need to avoid copying this as it should not exceed one int in size. */ - *a = archetypes[enc_level][archetype]; - return 1; -} - -static int txp_determine_geometry(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t archetype, - uint32_t enc_level, - size_t running_total, - QUIC_PKT_HDR *phdr, - struct txp_pkt_geom *geom) -{ - size_t mdpl, cmpl, hdr_len; - - /* Get information about packet archetype. */ - if (!txp_get_archetype_data(enc_level, archetype, &geom->adata)) - return 0; - - /* Assemble packet header. */ - phdr->type = ossl_quic_enc_level_to_pkt_type(enc_level); - phdr->spin_bit = 0; - phdr->pn_len = txp_determine_pn_len(txp); - phdr->partial = 0; - phdr->fixed = 1; - phdr->reserved = 0; - phdr->version = QUIC_VERSION_1; - phdr->dst_conn_id = txp->args.cur_dcid; - phdr->src_conn_id = txp->args.cur_scid; - - /* - * We need to know the length of the payload to get an accurate header - * length for non-1RTT packets, because the Length field found in - * Initial/Handshake/0-RTT packets uses a variable-length encoding. However, - * we don't have a good idea of the length of our payload, because the - * length of the payload depends on the room in the datagram after fitting - * the header, which depends on the size of the header. - * - * In general, it does not matter if a packet is slightly shorter (because - * e.g. we predicted use of a 2-byte length field, but ended up only needing - * a 1-byte length field). However this does matter for Initial packets - * which must be at least 1200 bytes, which is also the assumed default MTU; - * therefore in many cases Initial packets will be padded to 1200 bytes, - * which means if we overestimated the header size, we will be short by a - * few bytes and the server will ignore the packet for being too short. In - * this case, however, such packets always *will* be padded to meet 1200 - * bytes, which requires a 2-byte length field, so we don't actually need to - * worry about this. Thus we estimate the header length assuming a 2-byte - * length field here, which should in practice work well in all cases. - */ - phdr->len = OSSL_QUIC_VLINT_2B_MAX - phdr->pn_len; - - if (enc_level == QUIC_ENC_LEVEL_INITIAL) { - phdr->token = txp->initial_token; - phdr->token_len = txp->initial_token_len; - } else { - phdr->token = NULL; - phdr->token_len = 0; - } - - hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(phdr->dst_conn_id.id_len, - phdr); - if (hdr_len == 0) - return 0; - - /* MDPL: Maximum datagram payload length. */ - mdpl = txp_get_mdpl(txp); - - /* - * CMPL: Maximum encoded packet size we can put into this datagram given any - * previous packets coalesced into it. - */ - if (running_total > mdpl) - /* Should not be possible, but if it happens: */ - cmpl = 0; - else - cmpl = mdpl - running_total; - - /* CMPPL: Maximum amount we can put into the current packet payload */ - if (!txp_determine_ppl_from_pl(txp, cmpl, enc_level, hdr_len, &geom->cmppl)) - return 0; - - geom->cmpl = cmpl; - geom->pkt_overhead = cmpl - geom->cmppl; - geom->archetype = archetype; - return 1; -} - -static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER *txp, - uint64_t cc_limit) -{ - OSSL_ACKM_PROBE_INFO *probe_info - = ossl_ackm_get0_probe_request(txp->args.ackm); - uint32_t pn_space; - - /* - * If ACKM has requested probe generation (e.g. due to PTO), we generate a - * Probe-archetype packet. Actually, we determine archetype on a - * per-datagram basis, so if any EL wants a probe, do a pass in which - * we try and generate a probe (if needed) for all ELs. - */ - if (probe_info->anti_deadlock_initial > 0 - || probe_info->anti_deadlock_handshake > 0) - return TX_PACKETISER_ARCHETYPE_PROBE; - - for (pn_space = QUIC_PN_SPACE_INITIAL; - pn_space < QUIC_PN_SPACE_NUM; - ++pn_space) - if (probe_info->pto[pn_space] > 0) - return TX_PACKETISER_ARCHETYPE_PROBE; - - /* - * If we are out of CC budget, we cannot send a normal packet, - * but we can do an ACK-only packet (potentially, if we - * want to send an ACK). - */ - if (cc_limit == 0) - return TX_PACKETISER_ARCHETYPE_ACK_ONLY; - - /* All other packets. */ - return TX_PACKETISER_ARCHETYPE_NORMAL; -} - -static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t enc_level, - uint32_t archetype, - uint64_t cc_limit, - uint32_t *conn_close_enc_level) -{ - struct archetype_data a; - uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - QUIC_CFQ_ITEM *cfq_item; - - if (!ossl_qtx_is_enc_level_provisioned(txp->args.qtx, enc_level)) - return 0; - - if (!txp_get_archetype_data(enc_level, archetype, &a)) - return 0; - - if (!a.bypass_cc && cc_limit == 0) - /* CC not allowing us to send. */ - return 0; - - /* - * We can produce CONNECTION_CLOSE frames on any EL in principle, which - * means we need to choose which EL we would prefer to use. After a - * connection is fully established we have only one provisioned EL and this - * is a non-issue. Where multiple ELs are provisioned, it is possible the - * peer does not have the keys for the EL yet, which suggests in general it - * is preferable to use the lowest EL which is still provisioned. - * - * However (RFC 9000 s. 10.2.3 & 12.5) we are also required to not send - * application CONNECTION_CLOSE frames in non-1-RTT ELs, so as to not - * potentially leak application data on a connection which has yet to be - * authenticated. Thus when we have an application CONNECTION_CLOSE frame - * queued and need to send it on a non-1-RTT EL, we have to convert it - * into a transport CONNECTION_CLOSE frame which contains no application - * data. Since this loses information, it suggests we should use the 1-RTT - * EL to avoid this if possible, even if a lower EL is also available. - * - * At the same time, just because we have the 1-RTT EL provisioned locally - * does not necessarily mean the peer does, for example if a handshake - * CRYPTO frame has been lost. It is fairly important that CONNECTION_CLOSE - * is signalled in a way we know our peer can decrypt, as we stop processing - * connection retransmission logic for real after connection close and - * simply 'blindly' retransmit the same CONNECTION_CLOSE frame. - * - * This is not a major concern for clients, since if a client has a 1-RTT EL - * provisioned the server is guaranteed to also have a 1-RTT EL provisioned. - * - * TODO(QUIC SERVER): Revisit this when server support is added. - */ - if (*conn_close_enc_level > enc_level - && *conn_close_enc_level != QUIC_ENC_LEVEL_1RTT) - *conn_close_enc_level = enc_level; - - /* Do we need to send a PTO probe? */ - if (a.allow_force_ack_eliciting) { - OSSL_ACKM_PROBE_INFO *probe_info - = ossl_ackm_get0_probe_request(txp->args.ackm); - - if ((enc_level == QUIC_ENC_LEVEL_INITIAL - && probe_info->anti_deadlock_initial > 0) - || (enc_level == QUIC_ENC_LEVEL_HANDSHAKE - && probe_info->anti_deadlock_handshake > 0) - || probe_info->pto[pn_space] > 0) - return 1; - } - - /* Does the crypto stream for this EL want to produce anything? */ - if (a.allow_crypto && sstream_is_pending(txp->args.crypto[pn_space])) - return 1; - - /* Does the ACKM for this PN space want to produce anything? */ - if (a.allow_ack && (ossl_ackm_is_ack_desired(txp->args.ackm, pn_space) - || (txp->want_ack & (1UL << pn_space)) != 0)) - return 1; - - /* Do we need to force emission of an ACK-eliciting packet? */ - if (a.allow_force_ack_eliciting - && (txp->force_ack_eliciting & (1UL << pn_space)) != 0) - return 1; - - /* Does the connection-level RXFC want to produce a frame? */ - if (a.allow_conn_fc && (txp->want_max_data - || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0))) - return 1; - - /* Do we want to produce a MAX_STREAMS frame? */ - if (a.allow_conn_fc - && (txp->want_max_streams_bidi - || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, - 0) - || txp->want_max_streams_uni - || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, - 0))) - return 1; - - /* Do we want to produce a HANDSHAKE_DONE frame? */ - if (a.allow_handshake_done && txp->want_handshake_done) - return 1; - - /* Do we want to produce a CONNECTION_CLOSE frame? */ - if (a.allow_conn_close && txp->want_conn_close && - *conn_close_enc_level == enc_level) - /* - * This is a bit of a special case since CONNECTION_CLOSE can appear in - * most packet types, and when we decide we want to send it this status - * isn't tied to a specific EL. So if we want to send it, we send it - * only on the lowest non-dropped EL. - */ - return 1; - - /* Does the CFQ have any frames queued for this PN space? */ - if (enc_level != QUIC_ENC_LEVEL_0RTT) - for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space); - cfq_item != NULL; - cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) { - uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item); - - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID: - if (a.allow_new_conn_id) - return 1; - break; - case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID: - if (a.allow_retire_conn_id) - return 1; - break; - case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN: - if (a.allow_new_token) - return 1; - break; - case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE: - if (a.allow_path_response) - return 1; - break; - default: - if (a.allow_cfq_other) - return 1; - break; - } - } - - if (a.allow_stream_rel && txp->handshake_complete) { - QUIC_STREAM_ITER it; - - /* If there are any active streams, 0/1-RTT wants to produce a packet. - * Whether a stream is on the active list is required to be precise - * (i.e., a stream is never on the active list if we cannot produce a - * frame for it), and all stream-related frames are governed by - * a.allow_stream_rel (i.e., if we can send one type of stream-related - * frame, we can send any of them), so we don't need to inspect - * individual streams on the active list, just confirm that the active - * list is non-empty. - */ - ossl_quic_stream_iter_init(&it, txp->args.qsm, 0); - if (it.stream != NULL) - return 1; - } - - return 0; -} - -static int sstream_is_pending(QUIC_SSTREAM *sstream) -{ - OSSL_QUIC_FRAME_STREAM hdr; - OSSL_QTX_IOVEC iov[2]; - size_t num_iov = OSSL_NELEM(iov); - - return ossl_quic_sstream_get_stream_frame(sstream, 0, &hdr, iov, &num_iov); -} - -/* Determine how many bytes we should use for the encoded PN. */ -static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp) -{ - return 4; /* TODO(QUIC FUTURE) */ -} - -/* Determine plaintext packet payload length from payload length. */ -static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp, - size_t pl, - uint32_t enc_level, - size_t hdr_len, - size_t *r) -{ - if (pl < hdr_len) - return 0; - - pl -= hdr_len; - - if (!ossl_qtx_calculate_plaintext_payload_len(txp->args.qtx, enc_level, - pl, &pl)) - return 0; - - *r = pl; - return 1; -} - -static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp) -{ - return ossl_qtx_get_mdpl(txp->args.qtx); -} - -static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space, - void *arg) -{ - OSSL_QUIC_TX_PACKETISER *txp = arg; - QUIC_STREAM *s; - - if (stream_id == UINT64_MAX) - return txp->args.crypto[pn_space]; - - s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id); - if (s == NULL) - return NULL; - - return s->sstream; -} - -static void on_regen_notify(uint64_t frame_type, uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, void *arg) -{ - OSSL_QUIC_TX_PACKETISER *txp = arg; - - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE: - txp->want_handshake_done = 1; - break; - case OSSL_QUIC_FRAME_TYPE_MAX_DATA: - txp->want_max_data = 1; - break; - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI: - txp->want_max_streams_bidi = 1; - break; - case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI: - txp->want_max_streams_uni = 1; - break; - case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN: - txp->want_ack |= (1UL << pkt->ackm_pkt.pkt_space); - break; - case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA: - { - QUIC_STREAM *s - = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id); - - if (s == NULL) - return; - - s->want_max_stream_data = 1; - ossl_quic_stream_map_update_state(txp->args.qsm, s); - } - break; - case OSSL_QUIC_FRAME_TYPE_STOP_SENDING: - { - QUIC_STREAM *s - = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id); - - if (s == NULL) - return; - - ossl_quic_stream_map_schedule_stop_sending(txp->args.qsm, s); - } - break; - case OSSL_QUIC_FRAME_TYPE_RESET_STREAM: - { - QUIC_STREAM *s - = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id); - - if (s == NULL) - return; - - s->want_reset_stream = 1; - ossl_quic_stream_map_update_state(txp->args.qsm, s); - } - break; - default: - assert(0); - break; - } -} - -static int txp_need_ping(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t pn_space, - const struct archetype_data *adata) -{ - return adata->allow_ping - && (adata->require_ack_eliciting - || (txp->force_ack_eliciting & (1UL << pn_space)) != 0); -} - -static int txp_pkt_init(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp, - uint32_t enc_level, uint32_t archetype, - size_t running_total) -{ - uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - - if (!txp_determine_geometry(txp, archetype, enc_level, - running_total, &pkt->phdr, &pkt->geom)) - return 0; - - /* - * Initialise TX helper. If we must be ACK eliciting, reserve 1 byte for - * PING. - */ - if (!tx_helper_init(&pkt->h, txp, enc_level, - pkt->geom.cmppl, - txp_need_ping(txp, pn_space, &pkt->geom.adata) ? 1 : 0)) - return 0; - - pkt->h_valid = 1; - pkt->tpkt = NULL; - pkt->stream_head = NULL; - pkt->force_pad = 0; - return 1; -} - -static void txp_pkt_cleanup(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp) -{ - if (!pkt->h_valid) - return; - - tx_helper_cleanup(&pkt->h); - pkt->h_valid = 0; - - if (pkt->tpkt != NULL) { - ossl_quic_txpim_pkt_release(txp->args.txpim, pkt->tpkt); - pkt->tpkt = NULL; - } -} - -static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt *pkt, - OSSL_QUIC_TX_PACKETISER *txp) -{ - /* - * After we have staged and generated our packets, but before we commit - * them, it is possible for the estimated packet overhead (packet header + - * AEAD tag size) to shrink slightly because we generated a short packet - * whose which can be represented in fewer bytes as a variable-length - * integer than we were (pessimistically) budgeting for. We need to account - * for this to ensure that we get our padding calculation exactly right. - * - * Update pkt_overhead to be accurate now that we know how much data is - * going in a packet. - */ - size_t hdr_len, ciphertext_len; - - if (pkt->h.enc_level == QUIC_ENC_LEVEL_INITIAL) - /* - * Don't update overheads for the INITIAL EL - we have not finished - * appending padding to it and would potentially miscalculate the - * correct padding if we now update the pkt_overhead field to switch to - * e.g. a 1-byte length field in the packet header. Since we are padding - * to QUIC_MIN_INITIAL_DGRAM_LEN which requires a 2-byte length field, - * this is guaranteed to be moot anyway. See comment in - * txp_determine_geometry for more information. - */ - return 1; - - if (!ossl_qtx_calculate_ciphertext_payload_len(txp->args.qtx, pkt->h.enc_level, - pkt->h.bytes_appended, - &ciphertext_len)) - return 0; - - pkt->phdr.len = ciphertext_len; - - hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(pkt->phdr.dst_conn_id.id_len, - &pkt->phdr); - - pkt->geom.pkt_overhead = hdr_len + ciphertext_len - pkt->h.bytes_appended; - return 1; -} - -static void on_confirm_notify(uint64_t frame_type, uint64_t stream_id, - QUIC_TXPIM_PKT *pkt, void *arg) -{ - OSSL_QUIC_TX_PACKETISER *txp = arg; - - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_STOP_SENDING: - { - QUIC_STREAM *s - = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id); - - if (s == NULL) - return; - - s->acked_stop_sending = 1; - ossl_quic_stream_map_update_state(txp->args.qsm, s); - } - break; - case OSSL_QUIC_FRAME_TYPE_RESET_STREAM: - { - QUIC_STREAM *s - = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id); - - if (s == NULL) - return; - - /* - * We must already be in RESET_SENT or RESET_RECVD if we are - * here, so we don't need to check state here. - */ - ossl_quic_stream_map_notify_reset_stream_acked(txp->args.qsm, s); - ossl_quic_stream_map_update_state(txp->args.qsm, s); - } - break; - default: - assert(0); - break; - } -} - -static int txp_pkt_append_padding(struct txp_pkt *pkt, - OSSL_QUIC_TX_PACKETISER *txp, size_t num_bytes) -{ - WPACKET *wpkt; - - if (num_bytes == 0) - return 1; - - if (!ossl_assert(pkt->h_valid)) - return 0; - - if (!ossl_assert(pkt->tpkt != NULL)) - return 0; - - wpkt = tx_helper_begin(&pkt->h); - if (wpkt == NULL) - return 0; - - if (!ossl_quic_wire_encode_padding(wpkt, num_bytes)) { - tx_helper_rollback(&pkt->h); - return 0; - } - - if (!tx_helper_commit(&pkt->h)) - return 0; - - pkt->tpkt->ackm_pkt.num_bytes += num_bytes; - /* Cannot be non-inflight if we have a PADDING frame */ - pkt->tpkt->ackm_pkt.is_inflight = 1; - return 1; -} - -static void on_sstream_updated(uint64_t stream_id, void *arg) -{ - OSSL_QUIC_TX_PACKETISER *txp = arg; - QUIC_STREAM *s; - - s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id); - if (s == NULL) - return; - - ossl_quic_stream_map_update_state(txp->args.qsm, s); -} - -/* - * Returns 1 if we can send that many bytes in closing state, 0 otherwise. - * Also maintains the bytes sent state if it returns a success. - */ -static int try_commit_conn_close(OSSL_QUIC_TX_PACKETISER *txp, size_t n) -{ - int res; - - /* We can always send the first connection close frame */ - if (txp->closing_bytes_recv == 0) - return 1; - - /* - * RFC 9000 s. 10.2.1 Closing Connection State: - * To avoid being used for an amplification attack, such - * endpoints MUST limit the cumulative size of packets it sends - * to three times the cumulative size of the packets that are - * received and attributed to the connection. - * and: - * An endpoint in the closing state MUST either discard packets - * received from an unvalidated address or limit the cumulative - * size of packets it sends to an unvalidated address to three - * times the size of packets it receives from that address. - */ - res = txp->closing_bytes_xmit + n <= txp->closing_bytes_recv * 3; - - /* - * Attribute the bytes to the connection, if we are allowed to send them - * and this isn't the first closing frame. - */ - if (res && txp->closing_bytes_recv != 0) - txp->closing_bytes_xmit += n; - return res; -} - -void ossl_quic_tx_packetiser_record_received_closing_bytes( - OSSL_QUIC_TX_PACKETISER *txp, size_t n) -{ - txp->closing_bytes_recv += n; -} - -static int txp_generate_pre_token(OSSL_QUIC_TX_PACKETISER *txp, - struct txp_pkt *pkt, - int chosen_for_conn_close, - int *can_be_non_inflight) -{ - const uint32_t enc_level = pkt->h.enc_level; - const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - const struct archetype_data *a = &pkt->geom.adata; - QUIC_TXPIM_PKT *tpkt = pkt->tpkt; - struct tx_helper *h = &pkt->h; - const OSSL_QUIC_FRAME_ACK *ack; - OSSL_QUIC_FRAME_ACK ack2; - - tpkt->ackm_pkt.largest_acked = QUIC_PN_INVALID; - - /* ACK Frames (Regenerate) */ - if (a->allow_ack - && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_ACK - && (((txp->want_ack & (1UL << pn_space)) != 0) - || ossl_ackm_is_ack_desired(txp->args.ackm, pn_space)) - && (ack = ossl_ackm_get_ack_frame(txp->args.ackm, pn_space)) != NULL) { - WPACKET *wpkt = tx_helper_begin(h); - - if (wpkt == NULL) - return 0; - - /* We do not currently support ECN */ - ack2 = *ack; - ack2.ecn_present = 0; - - if (ossl_quic_wire_encode_frame_ack(wpkt, - txp->args.ack_delay_exponent, - &ack2)) { - if (!tx_helper_commit(h)) - return 0; - - tpkt->had_ack_frame = 1; - - if (ack->num_ack_ranges > 0) - tpkt->ackm_pkt.largest_acked = ack->ack_ranges[0].end; - - if (txp->ack_tx_cb != NULL) - txp->ack_tx_cb(&ack2, pn_space, txp->ack_tx_cb_arg); - } else { - tx_helper_rollback(h); - } - } - - /* CONNECTION_CLOSE Frames (Regenerate) */ - if (a->allow_conn_close && txp->want_conn_close && chosen_for_conn_close) { - WPACKET *wpkt = tx_helper_begin(h); - OSSL_QUIC_FRAME_CONN_CLOSE f, *pf = &txp->conn_close_frame; - size_t l; - - if (wpkt == NULL) - return 0; - - /* - * Application CONNECTION_CLOSE frames may only be sent in the - * Application PN space, as otherwise they may be sent before a - * connection is authenticated and leak application data. Therefore, if - * we need to send a CONNECTION_CLOSE frame in another PN space and were - * given an application CONNECTION_CLOSE frame, convert it into a - * transport CONNECTION_CLOSE frame, removing any sensitive application - * data. - * - * RFC 9000 s. 10.2.3: "A CONNECTION_CLOSE of type 0x1d MUST be replaced - * by a CONNECTION_CLOSE of type 0x1c when sending the frame in Initial - * or Handshake packets. Otherwise, information about the application - * state might be revealed. Endpoints MUST clear the value of the Reason - * Phrase field and SHOULD use the APPLICATION_ERROR code when - * converting to a CONNECTION_CLOSE of type 0x1c." - */ - if (pn_space != QUIC_PN_SPACE_APP && pf->is_app) { - pf = &f; - pf->is_app = 0; - pf->frame_type = 0; - pf->error_code = OSSL_QUIC_ERR_APPLICATION_ERROR; - pf->reason = NULL; - pf->reason_len = 0; - } - - if (ossl_quic_wire_encode_frame_conn_close(wpkt, pf) - && WPACKET_get_total_written(wpkt, &l) - && try_commit_conn_close(txp, l)) { - if (!tx_helper_commit(h)) - return 0; - - tpkt->had_conn_close = 1; - *can_be_non_inflight = 0; - } else { - tx_helper_rollback(h); - } - } - - return 1; -} - -static int try_len(size_t space_left, size_t orig_len, - size_t base_hdr_len, size_t lenbytes, - uint64_t maxn, size_t *hdr_len, size_t *payload_len) -{ - size_t n; - size_t maxn_ = maxn > SIZE_MAX ? SIZE_MAX : (size_t)maxn; - - *hdr_len = base_hdr_len + lenbytes; - - if (orig_len == 0 && space_left >= *hdr_len) { - *payload_len = 0; - return 1; - } - - n = orig_len; - if (n > maxn_) - n = maxn_; - if (n + *hdr_len > space_left) - n = (space_left >= *hdr_len) ? space_left - *hdr_len : 0; - - *payload_len = n; - return n > 0; -} - -static int determine_len(size_t space_left, size_t orig_len, - size_t base_hdr_len, - uint64_t *hlen, uint64_t *len) -{ - int ok = 0; - size_t chosen_payload_len = 0; - size_t chosen_hdr_len = 0; - size_t payload_len[4], hdr_len[4]; - int i, valid[4] = {0}; - - valid[0] = try_len(space_left, orig_len, base_hdr_len, - 1, OSSL_QUIC_VLINT_1B_MAX, - &hdr_len[0], &payload_len[0]); - valid[1] = try_len(space_left, orig_len, base_hdr_len, - 2, OSSL_QUIC_VLINT_2B_MAX, - &hdr_len[1], &payload_len[1]); - valid[2] = try_len(space_left, orig_len, base_hdr_len, - 4, OSSL_QUIC_VLINT_4B_MAX, - &hdr_len[2], &payload_len[2]); - valid[3] = try_len(space_left, orig_len, base_hdr_len, - 8, OSSL_QUIC_VLINT_8B_MAX, - &hdr_len[3], &payload_len[3]); - - for (i = OSSL_NELEM(valid) - 1; i >= 0; --i) - if (valid[i] && payload_len[i] >= chosen_payload_len) { - chosen_payload_len = payload_len[i]; - chosen_hdr_len = hdr_len[i]; - ok = 1; - } - - *hlen = chosen_hdr_len; - *len = chosen_payload_len; - return ok; -} - -/* - * Given a CRYPTO frame header with accurate chdr->len and a budget - * (space_left), try to find the optimal value of chdr->len to fill as much of - * the budget as possible. This is slightly hairy because larger values of - * chdr->len cause larger encoded sizes of the length field of the frame, which - * in turn mean less space available for payload data. We check all possible - * encodings and choose the optimal encoding. - */ -static int determine_crypto_len(struct tx_helper *h, - OSSL_QUIC_FRAME_CRYPTO *chdr, - size_t space_left, - uint64_t *hlen, - uint64_t *len) -{ - size_t orig_len; - size_t base_hdr_len; /* CRYPTO header length without length field */ - - if (chdr->len > SIZE_MAX) - return 0; - - orig_len = (size_t)chdr->len; - - chdr->len = 0; - base_hdr_len = ossl_quic_wire_get_encoded_frame_len_crypto_hdr(chdr); - chdr->len = orig_len; - if (base_hdr_len == 0) - return 0; - - --base_hdr_len; - - return determine_len(space_left, orig_len, base_hdr_len, hlen, len); -} - -static int determine_stream_len(struct tx_helper *h, - OSSL_QUIC_FRAME_STREAM *shdr, - size_t space_left, - uint64_t *hlen, - uint64_t *len) -{ - size_t orig_len; - size_t base_hdr_len; /* STREAM header length without length field */ - - if (shdr->len > SIZE_MAX) - return 0; - - orig_len = (size_t)shdr->len; - - shdr->len = 0; - base_hdr_len = ossl_quic_wire_get_encoded_frame_len_stream_hdr(shdr); - shdr->len = orig_len; - if (base_hdr_len == 0) - return 0; - - if (shdr->has_explicit_len) - --base_hdr_len; - - return determine_len(space_left, orig_len, base_hdr_len, hlen, len); -} - -static int txp_generate_crypto_frames(OSSL_QUIC_TX_PACKETISER *txp, - struct txp_pkt *pkt, - int *have_ack_eliciting) -{ - const uint32_t enc_level = pkt->h.enc_level; - const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - QUIC_TXPIM_PKT *tpkt = pkt->tpkt; - struct tx_helper *h = &pkt->h; - size_t num_stream_iovec; - OSSL_QUIC_FRAME_STREAM shdr = {0}; - OSSL_QUIC_FRAME_CRYPTO chdr = {0}; - OSSL_QTX_IOVEC iov[2]; - uint64_t hdr_bytes; - WPACKET *wpkt; - QUIC_TXPIM_CHUNK chunk = {0}; - size_t i, space_left; - - for (i = 0;; ++i) { - space_left = tx_helper_get_space_left(h); - - if (space_left < MIN_FRAME_SIZE_CRYPTO) - return 1; /* no point trying */ - - /* Do we have any CRYPTO data waiting? */ - num_stream_iovec = OSSL_NELEM(iov); - if (!ossl_quic_sstream_get_stream_frame(txp->args.crypto[pn_space], - i, &shdr, iov, - &num_stream_iovec)) - return 1; /* nothing to do */ - - /* Convert STREAM frame header to CRYPTO frame header */ - chdr.offset = shdr.offset; - chdr.len = shdr.len; - - if (chdr.len == 0) - return 1; /* nothing to do */ - - /* Find best fit (header length, payload length) combination. */ - if (!determine_crypto_len(h, &chdr, space_left, &hdr_bytes, - &chdr.len)) - return 1; /* can't fit anything */ - - /* - * Truncate IOVs to match our chosen length. - * - * The length cannot be more than SIZE_MAX because this length comes - * from our send stream buffer. - */ - ossl_quic_sstream_adjust_iov((size_t)chdr.len, iov, num_stream_iovec); - - /* - * Ensure we have enough iovecs allocated (1 for the header, up to 2 for - * the stream data.) - */ - if (!txp_el_ensure_iovec(&txp->el[enc_level], h->num_iovec + 3)) - return 0; /* alloc error */ - - /* Encode the header. */ - wpkt = tx_helper_begin(h); - if (wpkt == NULL) - return 0; /* alloc error */ - - if (!ossl_quic_wire_encode_frame_crypto_hdr(wpkt, &chdr)) { - tx_helper_rollback(h); - return 1; /* can't fit */ - } - - if (!tx_helper_commit(h)) - return 0; /* alloc error */ - - /* Add payload iovecs to the helper (infallible). */ - for (i = 0; i < num_stream_iovec; ++i) - tx_helper_append_iovec(h, iov[i].buf, iov[i].buf_len); - - *have_ack_eliciting = 1; - tx_helper_unrestrict(h); /* no longer need PING */ - - /* Log chunk to TXPIM. */ - chunk.stream_id = UINT64_MAX; /* crypto stream */ - chunk.start = chdr.offset; - chunk.end = chdr.offset + chdr.len - 1; - chunk.has_fin = 0; /* Crypto stream never ends */ - if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk)) - return 0; /* alloc error */ - } -} - -struct chunk_info { - OSSL_QUIC_FRAME_STREAM shdr; - uint64_t orig_len; - OSSL_QTX_IOVEC iov[2]; - size_t num_stream_iovec; - int valid; -}; - -static int txp_plan_stream_chunk(OSSL_QUIC_TX_PACKETISER *txp, - struct tx_helper *h, - QUIC_SSTREAM *sstream, - QUIC_TXFC *stream_txfc, - size_t skip, - struct chunk_info *chunk, - uint64_t consumed) -{ - uint64_t fc_credit, fc_swm, fc_limit; - - chunk->num_stream_iovec = OSSL_NELEM(chunk->iov); - chunk->valid = ossl_quic_sstream_get_stream_frame(sstream, skip, - &chunk->shdr, - chunk->iov, - &chunk->num_stream_iovec); - if (!chunk->valid) - return 1; - - if (!ossl_assert(chunk->shdr.len > 0 || chunk->shdr.is_fin)) - /* Should only have 0-length chunk if FIN */ - return 0; - - chunk->orig_len = chunk->shdr.len; - - /* Clamp according to connection and stream-level TXFC. */ - fc_credit = ossl_quic_txfc_get_credit(stream_txfc, consumed); - fc_swm = ossl_quic_txfc_get_swm(stream_txfc); - fc_limit = fc_swm + fc_credit; - - if (chunk->shdr.len > 0 && chunk->shdr.offset + chunk->shdr.len > fc_limit) { - chunk->shdr.len = (fc_limit <= chunk->shdr.offset) - ? 0 : fc_limit - chunk->shdr.offset; - chunk->shdr.is_fin = 0; - } - - if (chunk->shdr.len == 0 && !chunk->shdr.is_fin) { - /* - * Nothing to do due to TXFC. Since SSTREAM returns chunks in ascending - * order of offset we don't need to check any later chunks, so stop - * iterating here. - */ - chunk->valid = 0; - return 1; - } - - return 1; -} - -/* - * Returns 0 on fatal error (e.g. allocation failure), 1 on success. - * *packet_full is set to 1 if there is no longer enough room for another STREAM - * frame. - */ -static int txp_generate_stream_frames(OSSL_QUIC_TX_PACKETISER *txp, - struct txp_pkt *pkt, - uint64_t id, - QUIC_SSTREAM *sstream, - QUIC_TXFC *stream_txfc, - QUIC_STREAM *next_stream, - int *have_ack_eliciting, - int *packet_full, - uint64_t *new_credit_consumed, - uint64_t conn_consumed) -{ - int rc = 0; - struct chunk_info chunks[2] = {0}; - const uint32_t enc_level = pkt->h.enc_level; - QUIC_TXPIM_PKT *tpkt = pkt->tpkt; - struct tx_helper *h = &pkt->h; - OSSL_QUIC_FRAME_STREAM *shdr; - WPACKET *wpkt; - QUIC_TXPIM_CHUNK chunk; - size_t i, j, space_left; - int can_fill_payload, use_explicit_len; - int could_have_following_chunk; - uint64_t orig_len; - uint64_t hdr_len_implicit, payload_len_implicit; - uint64_t hdr_len_explicit, payload_len_explicit; - uint64_t fc_swm, fc_new_hwm; - - fc_swm = ossl_quic_txfc_get_swm(stream_txfc); - fc_new_hwm = fc_swm; - - /* - * Load the first two chunks if any offered by the send stream. We retrieve - * the next chunk in advance so we can determine if we need to send any more - * chunks from the same stream after this one, which is needed when - * determining when we can use an implicit length in a STREAM frame. - */ - for (i = 0; i < 2; ++i) { - if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i, &chunks[i], - conn_consumed)) - goto err; - - if (i == 0 && !chunks[i].valid) { - /* No chunks, nothing to do. */ - rc = 1; - goto err; - } - } - - for (i = 0;; ++i) { - space_left = tx_helper_get_space_left(h); - - if (!chunks[i % 2].valid) { - /* Out of chunks; we're done. */ - rc = 1; - goto err; - } - - if (space_left < MIN_FRAME_SIZE_STREAM) { - *packet_full = 1; - rc = 1; - goto err; - } - - if (!ossl_assert(!h->done_implicit)) - /* - * Logic below should have ensured we didn't append an - * implicit-length unless we filled the packet or didn't have - * another stream to handle, so this should not be possible. - */ - goto err; - - shdr = &chunks[i % 2].shdr; - orig_len = chunks[i % 2].orig_len; - if (i > 0) - /* Load next chunk for lookahead. */ - if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i + 1, - &chunks[(i + 1) % 2], conn_consumed)) - goto err; - - /* - * Find best fit (header length, payload length) combination for if we - * use an implicit length. - */ - shdr->has_explicit_len = 0; - hdr_len_implicit = payload_len_implicit = 0; - if (!determine_stream_len(h, shdr, space_left, - &hdr_len_implicit, &payload_len_implicit)) { - *packet_full = 1; - rc = 1; - goto err; /* can't fit anything */ - } - - /* - * If there is a next stream, we don't use the implicit length so we can - * add more STREAM frames after this one, unless there is enough data - * for this STREAM frame to fill the packet. - */ - can_fill_payload = (hdr_len_implicit + payload_len_implicit - >= space_left); - - /* - * Is there is a stream after this one, or another chunk pending - * transmission in this stream? - */ - could_have_following_chunk - = (next_stream != NULL || chunks[(i + 1) % 2].valid); - - /* Choose between explicit or implicit length representations. */ - use_explicit_len = !((can_fill_payload || !could_have_following_chunk) - && !pkt->force_pad); - - if (use_explicit_len) { - /* - * Find best fit (header length, payload length) combination for if - * we use an explicit length. - */ - shdr->has_explicit_len = 1; - hdr_len_explicit = payload_len_explicit = 0; - if (!determine_stream_len(h, shdr, space_left, - &hdr_len_explicit, &payload_len_explicit)) { - *packet_full = 1; - rc = 1; - goto err; /* can't fit anything */ - } - - shdr->len = payload_len_explicit; - } else { - *packet_full = 1; - shdr->has_explicit_len = 0; - shdr->len = payload_len_implicit; - } - - /* If this is a FIN, don't keep filling the packet with more FINs. */ - if (shdr->is_fin) - chunks[(i + 1) % 2].valid = 0; - - /* - * We are now committed to our length (shdr->len can't change). - * If we truncated the chunk, clear the FIN bit. - */ - if (shdr->len < orig_len) - shdr->is_fin = 0; - - /* Truncate IOVs to match our chosen length. */ - ossl_quic_sstream_adjust_iov((size_t)shdr->len, chunks[i % 2].iov, - chunks[i % 2].num_stream_iovec); - - /* - * Ensure we have enough iovecs allocated (1 for the header, up to 2 for - * the stream data.) - */ - if (!txp_el_ensure_iovec(&txp->el[enc_level], h->num_iovec + 3)) - goto err; /* alloc error */ - - /* Encode the header. */ - wpkt = tx_helper_begin(h); - if (wpkt == NULL) - goto err; /* alloc error */ - - shdr->stream_id = id; - if (!ossl_assert(ossl_quic_wire_encode_frame_stream_hdr(wpkt, shdr))) { - /* (Should not be possible.) */ - tx_helper_rollback(h); - *packet_full = 1; - rc = 1; - goto err; /* can't fit */ - } - - if (!tx_helper_commit(h)) - goto err; /* alloc error */ - - /* Add payload iovecs to the helper (infallible). */ - for (j = 0; j < chunks[i % 2].num_stream_iovec; ++j) - tx_helper_append_iovec(h, chunks[i % 2].iov[j].buf, - chunks[i % 2].iov[j].buf_len); - - *have_ack_eliciting = 1; - tx_helper_unrestrict(h); /* no longer need PING */ - if (!shdr->has_explicit_len) - h->done_implicit = 1; - - /* Log new TXFC credit which was consumed. */ - if (shdr->len > 0 && shdr->offset + shdr->len > fc_new_hwm) - fc_new_hwm = shdr->offset + shdr->len; - - /* Log chunk to TXPIM. */ - chunk.stream_id = shdr->stream_id; - chunk.start = shdr->offset; - chunk.end = shdr->offset + shdr->len - 1; - chunk.has_fin = shdr->is_fin; - chunk.has_stop_sending = 0; - chunk.has_reset_stream = 0; - if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk)) - goto err; /* alloc error */ - - if (shdr->len < orig_len) { - /* - * If we did not serialize all of this chunk we definitely do not - * want to try the next chunk - */ - rc = 1; - goto err; - } - } - -err: - *new_credit_consumed = fc_new_hwm - fc_swm; - return rc; -} - -static void txp_enlink_tmp(QUIC_STREAM **tmp_head, QUIC_STREAM *stream) -{ - stream->txp_next = *tmp_head; - *tmp_head = stream; -} - -static int txp_generate_stream_related(OSSL_QUIC_TX_PACKETISER *txp, - struct txp_pkt *pkt, - int *have_ack_eliciting, - QUIC_STREAM **tmp_head) -{ - QUIC_STREAM_ITER it; - WPACKET *wpkt; - uint64_t cwm; - QUIC_STREAM *stream, *snext; - struct tx_helper *h = &pkt->h; - uint64_t conn_consumed = 0; - - for (ossl_quic_stream_iter_init(&it, txp->args.qsm, 1); - it.stream != NULL;) { - - stream = it.stream; - ossl_quic_stream_iter_next(&it); - snext = it.stream; - - stream->txp_sent_fc = 0; - stream->txp_sent_stop_sending = 0; - stream->txp_sent_reset_stream = 0; - stream->txp_blocked = 0; - stream->txp_txfc_new_credit_consumed = 0; - - /* Stream Abort Frames (STOP_SENDING, RESET_STREAM) */ - if (stream->want_stop_sending) { - OSSL_QUIC_FRAME_STOP_SENDING f; - - wpkt = tx_helper_begin(h); - if (wpkt == NULL) - return 0; /* alloc error */ - - f.stream_id = stream->id; - f.app_error_code = stream->stop_sending_aec; - if (!ossl_quic_wire_encode_frame_stop_sending(wpkt, &f)) { - tx_helper_rollback(h); /* can't fit */ - txp_enlink_tmp(tmp_head, stream); - break; - } - - if (!tx_helper_commit(h)) - return 0; /* alloc error */ - - *have_ack_eliciting = 1; - tx_helper_unrestrict(h); /* no longer need PING */ - stream->txp_sent_stop_sending = 1; - } - - if (stream->want_reset_stream) { - OSSL_QUIC_FRAME_RESET_STREAM f; - - if (!ossl_assert(stream->send_state == QUIC_SSTREAM_STATE_RESET_SENT)) - return 0; - - wpkt = tx_helper_begin(h); - if (wpkt == NULL) - return 0; /* alloc error */ - - f.stream_id = stream->id; - f.app_error_code = stream->reset_stream_aec; - if (!ossl_quic_stream_send_get_final_size(stream, &f.final_size)) - return 0; /* should not be possible */ - - if (!ossl_quic_wire_encode_frame_reset_stream(wpkt, &f)) { - tx_helper_rollback(h); /* can't fit */ - txp_enlink_tmp(tmp_head, stream); - break; - } - - if (!tx_helper_commit(h)) - return 0; /* alloc error */ - - *have_ack_eliciting = 1; - tx_helper_unrestrict(h); /* no longer need PING */ - stream->txp_sent_reset_stream = 1; - - /* - * The final size of the stream as indicated by RESET_STREAM is used - * to ensure a consistent view of flow control state by both - * parties; if we happen to send a RESET_STREAM that consumes more - * flow control credit, make sure we account for that. - */ - if (!ossl_assert(f.final_size <= ossl_quic_txfc_get_swm(&stream->txfc))) - return 0; - - stream->txp_txfc_new_credit_consumed - = f.final_size - ossl_quic_txfc_get_swm(&stream->txfc); - } - - /* - * Stream Flow Control Frames (MAX_STREAM_DATA) - * - * RFC 9000 s. 13.3: "An endpoint SHOULD stop sending MAX_STREAM_DATA - * frames when the receiving part of the stream enters a "Size Known" or - * "Reset Recvd" state." -- In practice, RECV is the only state - * in which it makes sense to generate more MAX_STREAM_DATA frames. - */ - if (stream->recv_state == QUIC_RSTREAM_STATE_RECV - && (stream->want_max_stream_data - || ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 0))) { - - wpkt = tx_helper_begin(h); - if (wpkt == NULL) - return 0; /* alloc error */ - - cwm = ossl_quic_rxfc_get_cwm(&stream->rxfc); - - if (!ossl_quic_wire_encode_frame_max_stream_data(wpkt, stream->id, - cwm)) { - tx_helper_rollback(h); /* can't fit */ - txp_enlink_tmp(tmp_head, stream); - break; - } - - if (!tx_helper_commit(h)) - return 0; /* alloc error */ - - *have_ack_eliciting = 1; - tx_helper_unrestrict(h); /* no longer need PING */ - stream->txp_sent_fc = 1; - } - - /* - * Stream Data Frames (STREAM) - * - * RFC 9000 s. 3.3: A sender MUST NOT send a STREAM [...] frame for a - * stream in the "Reset Sent" state [or any terminal state]. We don't - * send any more STREAM frames if we are sending, have sent, or are - * planning to send, RESET_STREAM. The other terminal state is Data - * Recvd, but txp_generate_stream_frames() is guaranteed to generate - * nothing in this case. - */ - if (ossl_quic_stream_has_send_buffer(stream) - && !ossl_quic_stream_send_is_reset(stream)) { - int packet_full = 0; - - if (!ossl_assert(!stream->want_reset_stream)) - return 0; - - if (!txp_generate_stream_frames(txp, pkt, - stream->id, stream->sstream, - &stream->txfc, - snext, - have_ack_eliciting, - &packet_full, - &stream->txp_txfc_new_credit_consumed, - conn_consumed)) { - /* Fatal error (allocation, etc.) */ - txp_enlink_tmp(tmp_head, stream); - return 0; - } - conn_consumed += stream->txp_txfc_new_credit_consumed; - - if (packet_full) { - txp_enlink_tmp(tmp_head, stream); - break; - } - } - - txp_enlink_tmp(tmp_head, stream); - } - - return 1; -} - -static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp, - struct txp_pkt *pkt, - int chosen_for_conn_close) -{ - int rc = TXP_ERR_SUCCESS; - const uint32_t enc_level = pkt->h.enc_level; - const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - int have_ack_eliciting = 0, done_pre_token = 0; - const struct archetype_data a = pkt->geom.adata; - /* - * Cleared if we encode any non-ACK-eliciting frame type which rules out the - * packet being a non-inflight frame. This means any non-ACK ACK-eliciting - * frame, even PADDING frames. ACK eliciting frames always cause a packet to - * become ineligible for non-inflight treatment so it is not necessary to - * clear this in cases where have_ack_eliciting is set, as it is ignored in - * that case. - */ - int can_be_non_inflight = 1; - QUIC_CFQ_ITEM *cfq_item; - QUIC_TXPIM_PKT *tpkt = NULL; - struct tx_helper *h = &pkt->h; - - /* Maximum PN reached? */ - if (!ossl_quic_pn_valid(txp->next_pn[pn_space])) - goto fatal_err; - - if (!ossl_assert(pkt->tpkt == NULL)) - goto fatal_err; - - if ((pkt->tpkt = tpkt = ossl_quic_txpim_pkt_alloc(txp->args.txpim)) == NULL) - goto fatal_err; - - /* - * Frame Serialization - * =================== - * - * We now serialize frames into the packet in descending order of priority. - */ - - /* HANDSHAKE_DONE (Regenerate) */ - if (a.allow_handshake_done && txp->want_handshake_done - && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_HANDSHAKE_DONE) { - WPACKET *wpkt = tx_helper_begin(h); - - if (wpkt == NULL) - goto fatal_err; - - if (ossl_quic_wire_encode_frame_handshake_done(wpkt)) { - tpkt->had_handshake_done_frame = 1; - have_ack_eliciting = 1; - - if (!tx_helper_commit(h)) - goto fatal_err; - - tx_helper_unrestrict(h); /* no longer need PING */ - } else { - tx_helper_rollback(h); - } - } - - /* MAX_DATA (Regenerate) */ - if (a.allow_conn_fc - && (txp->want_max_data - || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0)) - && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_DATA) { - WPACKET *wpkt = tx_helper_begin(h); - uint64_t cwm = ossl_quic_rxfc_get_cwm(txp->args.conn_rxfc); - - if (wpkt == NULL) - goto fatal_err; - - if (ossl_quic_wire_encode_frame_max_data(wpkt, cwm)) { - tpkt->had_max_data_frame = 1; - have_ack_eliciting = 1; - - if (!tx_helper_commit(h)) - goto fatal_err; - - tx_helper_unrestrict(h); /* no longer need PING */ - } else { - tx_helper_rollback(h); - } - } - - /* MAX_STREAMS_BIDI (Regenerate) */ - if (a.allow_conn_fc - && (txp->want_max_streams_bidi - || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, 0)) - && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_STREAMS_BIDI) { - WPACKET *wpkt = tx_helper_begin(h); - uint64_t max_streams - = ossl_quic_rxfc_get_cwm(txp->args.max_streams_bidi_rxfc); - - if (wpkt == NULL) - goto fatal_err; - - if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/0, - max_streams)) { - tpkt->had_max_streams_bidi_frame = 1; - have_ack_eliciting = 1; - - if (!tx_helper_commit(h)) - goto fatal_err; - - tx_helper_unrestrict(h); /* no longer need PING */ - } else { - tx_helper_rollback(h); - } - } - - /* MAX_STREAMS_UNI (Regenerate) */ - if (a.allow_conn_fc - && (txp->want_max_streams_uni - || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, 0)) - && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_STREAMS_UNI) { - WPACKET *wpkt = tx_helper_begin(h); - uint64_t max_streams - = ossl_quic_rxfc_get_cwm(txp->args.max_streams_uni_rxfc); - - if (wpkt == NULL) - goto fatal_err; - - if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/1, - max_streams)) { - tpkt->had_max_streams_uni_frame = 1; - have_ack_eliciting = 1; - - if (!tx_helper_commit(h)) - goto fatal_err; - - tx_helper_unrestrict(h); /* no longer need PING */ - } else { - tx_helper_rollback(h); - } - } - - /* GCR Frames */ - for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space); - cfq_item != NULL; - cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) { - uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item); - const unsigned char *encoded = ossl_quic_cfq_item_get_encoded(cfq_item); - size_t encoded_len = ossl_quic_cfq_item_get_encoded_len(cfq_item); - - switch (frame_type) { - case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID: - if (!a.allow_new_conn_id) - continue; - break; - case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID: - if (!a.allow_retire_conn_id) - continue; - break; - case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN: - if (!a.allow_new_token) - continue; - - /* - * NEW_TOKEN frames are handled via GCR, but some - * Regenerate-strategy frames should come before them (namely - * ACK, CONNECTION_CLOSE, PATH_CHALLENGE and PATH_RESPONSE). If - * we find a NEW_TOKEN frame, do these now. If there are no - * NEW_TOKEN frames in the GCR queue we will handle these below. - */ - if (!done_pre_token) - if (txp_generate_pre_token(txp, pkt, - chosen_for_conn_close, - &can_be_non_inflight)) - done_pre_token = 1; - - break; - case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE: - if (!a.allow_path_response) - continue; - - /* - * RFC 9000 s. 8.2.2: An endpoint MUST expand datagrams that - * contain a PATH_RESPONSE frame to at least the smallest - * allowed maximum datagram size of 1200 bytes. - */ - pkt->force_pad = 1; - break; - default: - if (!a.allow_cfq_other) - continue; - break; - } - - /* - * If the frame is too big, don't try to schedule any more GCR frames in - * this packet rather than sending subsequent ones out of order. - */ - if (encoded_len > tx_helper_get_space_left(h)) - break; - - if (!tx_helper_append_iovec(h, encoded, encoded_len)) - goto fatal_err; - - ossl_quic_txpim_pkt_add_cfq_item(tpkt, cfq_item); - - if (ossl_quic_frame_type_is_ack_eliciting(frame_type)) { - have_ack_eliciting = 1; - tx_helper_unrestrict(h); /* no longer need PING */ - } - } - - /* - * If we didn't generate ACK, CONNECTION_CLOSE, PATH_CHALLENGE or - * PATH_RESPONSE (as desired) before, do so now. - */ - if (!done_pre_token) - if (txp_generate_pre_token(txp, pkt, - chosen_for_conn_close, - &can_be_non_inflight)) - done_pre_token = 1; - - /* CRYPTO Frames */ - if (a.allow_crypto) - if (!txp_generate_crypto_frames(txp, pkt, &have_ack_eliciting)) - goto fatal_err; - - /* Stream-specific frames */ - if (a.allow_stream_rel && txp->handshake_complete) - if (!txp_generate_stream_related(txp, pkt, - &have_ack_eliciting, - &pkt->stream_head)) - goto fatal_err; - - /* PING */ - tx_helper_unrestrict(h); - - if (!have_ack_eliciting && txp_need_ping(txp, pn_space, &a)) { - WPACKET *wpkt; - - assert(h->reserve > 0); - wpkt = tx_helper_begin(h); - if (wpkt == NULL) - goto fatal_err; - - if (!ossl_quic_wire_encode_frame_ping(wpkt) - || !tx_helper_commit(h)) - /* - * We treat a request to be ACK-eliciting as a requirement, so this - * is an error. - */ - goto fatal_err; - - have_ack_eliciting = 1; - } - - /* PADDING is added by ossl_quic_tx_packetiser_generate(). */ - - /* - * ACKM Data - * ========= - */ - if (have_ack_eliciting) - can_be_non_inflight = 0; - - /* ACKM Data */ - tpkt->ackm_pkt.num_bytes = h->bytes_appended + pkt->geom.pkt_overhead; - tpkt->ackm_pkt.pkt_num = txp->next_pn[pn_space]; - /* largest_acked is set in txp_generate_pre_token */ - tpkt->ackm_pkt.pkt_space = pn_space; - tpkt->ackm_pkt.is_inflight = !can_be_non_inflight; - tpkt->ackm_pkt.is_ack_eliciting = have_ack_eliciting; - tpkt->ackm_pkt.is_pto_probe = 0; - tpkt->ackm_pkt.is_mtu_probe = 0; - tpkt->ackm_pkt.time = txp->args.now(txp->args.now_arg); - tpkt->pkt_type = pkt->phdr.type; - - /* Done. */ - return rc; - -fatal_err: - /* - * Handler for fatal errors, i.e. errors causing us to abort the entire - * packet rather than just one frame. Examples of such errors include - * allocation errors. - */ - if (tpkt != NULL) { - ossl_quic_txpim_pkt_release(txp->args.txpim, tpkt); - pkt->tpkt = NULL; - } - return TXP_ERR_INTERNAL; -} - -/* - * Commits and queues a packet for transmission. There is no backing out after - * this. - * - * This: - * - * - Sends the packet to the QTX for encryption and transmission; - * - * - Records the packet as having been transmitted in FIFM. ACKM is informed, - * etc. and the TXPIM record is filed. - * - * - Informs various subsystems of frames that were sent and clears frame - * wanted flags so that we do not generate the same frames again. - * - * Assumptions: - * - * - pkt is a txp_pkt for the correct EL; - * - * - pkt->tpkt is valid; - * - * - pkt->tpkt->ackm_pkt has been fully filled in; - * - * - Stream chunk records have been appended to pkt->tpkt for STREAM and - * CRYPTO frames, but not for RESET_STREAM or STOP_SENDING frames; - * - * - The chosen stream list for the packet can be fully walked from - * pkt->stream_head using stream->txp_next; - * - * - pkt->has_ack_eliciting is set correctly. - * - */ -static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER *txp, - struct txp_pkt *pkt, - uint32_t archetype, - int *txpim_pkt_reffed) -{ - int rc = 1; - uint32_t enc_level = pkt->h.enc_level; - uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - QUIC_TXPIM_PKT *tpkt = pkt->tpkt; - QUIC_STREAM *stream; - OSSL_QTX_PKT txpkt; - struct archetype_data a; - - *txpim_pkt_reffed = 0; - - /* Cannot send a packet with an empty payload. */ - if (pkt->h.bytes_appended == 0) - return 0; - - if (!txp_get_archetype_data(enc_level, archetype, &a)) - return 0; - - /* Packet Information for QTX */ - txpkt.hdr = &pkt->phdr; - txpkt.iovec = txp->el[enc_level].iovec; - txpkt.num_iovec = pkt->h.num_iovec; - txpkt.local = NULL; - txpkt.peer = BIO_ADDR_family(&txp->args.peer) == AF_UNSPEC - ? NULL : &txp->args.peer; - txpkt.pn = txp->next_pn[pn_space]; - txpkt.flags = OSSL_QTX_PKT_FLAG_COALESCE; /* always try to coalesce */ - - /* Generate TXPIM chunks representing STOP_SENDING and RESET_STREAM frames. */ - for (stream = pkt->stream_head; stream != NULL; stream = stream->txp_next) - if (stream->txp_sent_stop_sending || stream->txp_sent_reset_stream) { - /* Log STOP_SENDING/RESET_STREAM chunk to TXPIM. */ - QUIC_TXPIM_CHUNK chunk; - - chunk.stream_id = stream->id; - chunk.start = UINT64_MAX; - chunk.end = 0; - chunk.has_fin = 0; - chunk.has_stop_sending = stream->txp_sent_stop_sending; - chunk.has_reset_stream = stream->txp_sent_reset_stream; - if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk)) - return 0; /* alloc error */ - } - - /* Dispatch to FIFD. */ - if (!ossl_quic_fifd_pkt_commit(&txp->fifd, tpkt)) - return 0; - - /* - * Transmission and Post-Packet Generation Bookkeeping - * =================================================== - * - * No backing out anymore - at this point the ACKM has recorded the packet - * as having been sent, so we need to increment our next PN counter, or - * the ACKM will complain when we try to record a duplicate packet with - * the same PN later. At this point actually sending the packet may still - * fail. In this unlikely event it will simply be handled as though it - * were a lost packet. - */ - ++txp->next_pn[pn_space]; - *txpim_pkt_reffed = 1; - - /* Send the packet. */ - if (!ossl_qtx_write_pkt(txp->args.qtx, &txpkt)) - return 0; - - /* - * Record FC and stream abort frames as sent; deactivate streams which no - * longer have anything to do. - */ - for (stream = pkt->stream_head; stream != NULL; stream = stream->txp_next) { - if (stream->txp_sent_fc) { - stream->want_max_stream_data = 0; - ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 1); - } - - if (stream->txp_sent_stop_sending) - stream->want_stop_sending = 0; - - if (stream->txp_sent_reset_stream) - stream->want_reset_stream = 0; - - if (stream->txp_txfc_new_credit_consumed > 0) { - if (!ossl_assert(ossl_quic_txfc_consume_credit(&stream->txfc, - stream->txp_txfc_new_credit_consumed))) - /* - * Should not be possible, but we should continue with our - * bookkeeping as we have already committed the packet to the - * FIFD. Just change the value we return. - */ - rc = 0; - - stream->txp_txfc_new_credit_consumed = 0; - } - - /* - * If we no longer need to generate any flow control (MAX_STREAM_DATA), - * STOP_SENDING or RESET_STREAM frames, nor any STREAM frames (because - * the stream is drained of data or TXFC-blocked), we can mark the - * stream as inactive. - */ - ossl_quic_stream_map_update_state(txp->args.qsm, stream); - - if (ossl_quic_stream_has_send_buffer(stream) - && !ossl_quic_sstream_has_pending(stream->sstream) - && ossl_quic_sstream_get_final_size(stream->sstream, NULL)) - /* - * Transition to DATA_SENT if stream has a final size and we have - * sent all data. - */ - ossl_quic_stream_map_notify_all_data_sent(txp->args.qsm, stream); - } - - /* We have now sent the packet, so update state accordingly. */ - if (tpkt->ackm_pkt.is_ack_eliciting) - txp->force_ack_eliciting &= ~(1UL << pn_space); - - if (tpkt->had_handshake_done_frame) - txp->want_handshake_done = 0; - - if (tpkt->had_max_data_frame) { - txp->want_max_data = 0; - ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 1); - } - - if (tpkt->had_max_streams_bidi_frame) { - txp->want_max_streams_bidi = 0; - ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, 1); - } - - if (tpkt->had_max_streams_uni_frame) { - txp->want_max_streams_uni = 0; - ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, 1); - } - - if (tpkt->had_ack_frame) - txp->want_ack &= ~(1UL << pn_space); - - if (tpkt->had_conn_close) - txp->want_conn_close = 0; - - /* - * Decrement probe request counts if we have sent a packet that meets - * the requirement of a probe, namely being ACK-eliciting. - */ - if (tpkt->ackm_pkt.is_ack_eliciting) { - OSSL_ACKM_PROBE_INFO *probe_info - = ossl_ackm_get0_probe_request(txp->args.ackm); - - if (enc_level == QUIC_ENC_LEVEL_INITIAL - && probe_info->anti_deadlock_initial > 0) - --probe_info->anti_deadlock_initial; - - if (enc_level == QUIC_ENC_LEVEL_HANDSHAKE - && probe_info->anti_deadlock_handshake > 0) - --probe_info->anti_deadlock_handshake; - - if (a.allow_force_ack_eliciting /* (i.e., not for 0-RTT) */ - && probe_info->pto[pn_space] > 0) - --probe_info->pto[pn_space]; - } - - return rc; -} - -/* Ensure the iovec array is at least num elements long. */ -static int txp_el_ensure_iovec(struct txp_el *el, size_t num) -{ - OSSL_QTX_IOVEC *iovec; - - if (el->alloc_iovec >= num) - return 1; - - num = el->alloc_iovec != 0 ? el->alloc_iovec * 2 : 8; - - iovec = OPENSSL_realloc(el->iovec, sizeof(OSSL_QTX_IOVEC) * num); - if (iovec == NULL) - return 0; - - el->iovec = iovec; - el->alloc_iovec = num; - return 1; -} - -int ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER *txp, - const OSSL_QUIC_FRAME_CONN_CLOSE *f) -{ - char *reason = NULL; - size_t reason_len = f->reason_len; - size_t max_reason_len = txp_get_mdpl(txp) / 2; - - if (txp->want_conn_close) - return 0; - - /* - * Arbitrarily limit the length of the reason length string to half of the - * MDPL. - */ - if (reason_len > max_reason_len) - reason_len = max_reason_len; - - if (reason_len > 0) { - reason = OPENSSL_memdup(f->reason, reason_len); - if (reason == NULL) - return 0; - } - - txp->conn_close_frame = *f; - txp->conn_close_frame.reason = reason; - txp->conn_close_frame.reason_len = reason_len; - txp->want_conn_close = 1; - return 1; -} - -void ossl_quic_tx_packetiser_set_msg_callback(OSSL_QUIC_TX_PACKETISER *txp, - ossl_msg_cb msg_callback, - SSL *msg_callback_ssl) -{ - txp->msg_callback = msg_callback; - txp->msg_callback_ssl = msg_callback_ssl; -} - -void ossl_quic_tx_packetiser_set_msg_callback_arg(OSSL_QUIC_TX_PACKETISER *txp, - void *msg_callback_arg) -{ - txp->msg_callback_arg = msg_callback_arg; -} - -QUIC_PN ossl_quic_tx_packetiser_get_next_pn(OSSL_QUIC_TX_PACKETISER *txp, - uint32_t pn_space) -{ - if (pn_space >= QUIC_PN_SPACE_NUM) - return UINT64_MAX; - - return txp->next_pn[pn_space]; -} - -OSSL_TIME ossl_quic_tx_packetiser_get_deadline(OSSL_QUIC_TX_PACKETISER *txp) -{ - /* - * TXP-specific deadline computations which rely on TXP innards. This is in - * turn relied on by the QUIC_CHANNEL code to determine the channel event - * handling deadline. - */ - OSSL_TIME deadline = ossl_time_infinite(); - uint32_t enc_level, pn_space; - - /* - * ACK generation is not CC-gated - packets containing only ACKs are allowed - * to bypass CC. We want to generate ACK frames even if we are currently - * restricted by CC so the peer knows we have received data. The generate - * call will take care of selecting the correct packet archetype. - */ - for (enc_level = QUIC_ENC_LEVEL_INITIAL; - enc_level < QUIC_ENC_LEVEL_NUM; - ++enc_level) - if (ossl_qtx_is_enc_level_provisioned(txp->args.qtx, enc_level)) { - pn_space = ossl_quic_enc_level_to_pn_space(enc_level); - deadline = ossl_time_min(deadline, - ossl_ackm_get_ack_deadline(txp->args.ackm, pn_space)); - } - - /* When will CC let us send more? */ - if (txp->args.cc_method->get_tx_allowance(txp->args.cc_data) == 0) - deadline = ossl_time_min(deadline, - txp->args.cc_method->get_wakeup_deadline(txp->args.cc_data)); - - return deadline; -} diff --git a/openssl/src/ssl/quic/quic_txpim.c b/openssl/src/ssl/quic/quic_txpim.c deleted file mode 100644 index 04b25ee47..000000000 --- a/openssl/src/ssl/quic/quic_txpim.c +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_txpim.h" -#include - -typedef struct quic_txpim_pkt_ex_st QUIC_TXPIM_PKT_EX; - -struct quic_txpim_pkt_ex_st { - QUIC_TXPIM_PKT public; - QUIC_TXPIM_PKT_EX *prev, *next; - QUIC_TXPIM_CHUNK *chunks; - size_t num_chunks, alloc_chunks; - unsigned int chunks_need_sort : 1; -}; - -typedef struct quic_txpim_pkt_ex_list { - QUIC_TXPIM_PKT_EX *head, *tail; -} QUIC_TXPIM_PKT_EX_LIST; - -struct quic_txpim_st { - QUIC_TXPIM_PKT_EX_LIST free_list; - size_t in_use; -}; - -#define MAX_ALLOC_CHUNKS 512 - -QUIC_TXPIM *ossl_quic_txpim_new(void) -{ - QUIC_TXPIM *txpim = OPENSSL_zalloc(sizeof(*txpim)); - - if (txpim == NULL) - return NULL; - - return txpim; -} - -static void free_list(QUIC_TXPIM_PKT_EX_LIST *l) -{ - QUIC_TXPIM_PKT_EX *n, *nnext; - - for (n = l->head; n != NULL; n = nnext) { - nnext = n->next; - - OPENSSL_free(n->chunks); - OPENSSL_free(n); - } - - l->head = l->tail = NULL; -} - -void ossl_quic_txpim_free(QUIC_TXPIM *txpim) -{ - if (txpim == NULL) - return; - - assert(txpim->in_use == 0); - free_list(&txpim->free_list); - OPENSSL_free(txpim); -} - -static void list_remove(QUIC_TXPIM_PKT_EX_LIST *l, QUIC_TXPIM_PKT_EX *n) -{ - if (l->head == n) - l->head = n->next; - if (l->tail == n) - l->tail = n->prev; - if (n->prev != NULL) - n->prev->next = n->next; - if (n->next != NULL) - n->next->prev = n->prev; - n->prev = n->next = NULL; -} - -static void list_insert_tail(QUIC_TXPIM_PKT_EX_LIST *l, QUIC_TXPIM_PKT_EX *n) -{ - n->prev = l->tail; - n->next = NULL; - l->tail = n; - if (n->prev != NULL) - n->prev->next = n; - if (l->head == NULL) - l->head = n; -} - -static QUIC_TXPIM_PKT_EX *txpim_get_free(QUIC_TXPIM *txpim) -{ - QUIC_TXPIM_PKT_EX *ex = txpim->free_list.head; - - if (ex != NULL) - return ex; - - ex = OPENSSL_zalloc(sizeof(*ex)); - if (ex == NULL) - return NULL; - - list_insert_tail(&txpim->free_list, ex); - return ex; -} - -static void txpim_clear(QUIC_TXPIM_PKT_EX *ex) -{ - memset(&ex->public.ackm_pkt, 0, sizeof(ex->public.ackm_pkt)); - ossl_quic_txpim_pkt_clear_chunks(&ex->public); - ex->public.retx_head = NULL; - ex->public.fifd = NULL; - ex->public.had_handshake_done_frame = 0; - ex->public.had_max_data_frame = 0; - ex->public.had_max_streams_bidi_frame = 0; - ex->public.had_max_streams_uni_frame = 0; - ex->public.had_ack_frame = 0; - ex->public.had_conn_close = 0; -} - -QUIC_TXPIM_PKT *ossl_quic_txpim_pkt_alloc(QUIC_TXPIM *txpim) -{ - QUIC_TXPIM_PKT_EX *ex = txpim_get_free(txpim); - - if (ex == NULL) - return NULL; - - txpim_clear(ex); - list_remove(&txpim->free_list, ex); - ++txpim->in_use; - return &ex->public; -} - -void ossl_quic_txpim_pkt_release(QUIC_TXPIM *txpim, QUIC_TXPIM_PKT *fpkt) -{ - QUIC_TXPIM_PKT_EX *ex = (QUIC_TXPIM_PKT_EX *)fpkt; - - assert(txpim->in_use > 0); - --txpim->in_use; - list_insert_tail(&txpim->free_list, ex); -} - -void ossl_quic_txpim_pkt_add_cfq_item(QUIC_TXPIM_PKT *fpkt, - QUIC_CFQ_ITEM *item) -{ - item->pkt_next = fpkt->retx_head; - item->pkt_prev = NULL; - fpkt->retx_head = item; -} - -void ossl_quic_txpim_pkt_clear_chunks(QUIC_TXPIM_PKT *fpkt) -{ - QUIC_TXPIM_PKT_EX *ex = (QUIC_TXPIM_PKT_EX *)fpkt; - - ex->num_chunks = 0; -} - -int ossl_quic_txpim_pkt_append_chunk(QUIC_TXPIM_PKT *fpkt, - const QUIC_TXPIM_CHUNK *chunk) -{ - QUIC_TXPIM_PKT_EX *ex = (QUIC_TXPIM_PKT_EX *)fpkt; - QUIC_TXPIM_CHUNK *new_chunk; - size_t new_alloc_chunks = ex->alloc_chunks; - - if (ex->num_chunks == ex->alloc_chunks) { - new_alloc_chunks = (ex->alloc_chunks == 0) ? 4 : ex->alloc_chunks * 8 / 5; - if (new_alloc_chunks > MAX_ALLOC_CHUNKS) - new_alloc_chunks = MAX_ALLOC_CHUNKS; - if (ex->num_chunks == new_alloc_chunks) - return 0; - - new_chunk = OPENSSL_realloc(ex->chunks, - new_alloc_chunks * sizeof(QUIC_TXPIM_CHUNK)); - if (new_chunk == NULL) - return 0; - - ex->chunks = new_chunk; - ex->alloc_chunks = new_alloc_chunks; - } - - ex->chunks[ex->num_chunks++] = *chunk; - ex->chunks_need_sort = 1; - return 1; -} - -static int compare(const void *a, const void *b) -{ - const QUIC_TXPIM_CHUNK *ac = a, *bc = b; - - if (ac->stream_id < bc->stream_id) - return -1; - else if (ac->stream_id > bc->stream_id) - return 1; - - if (ac->start < bc->start) - return -1; - else if (ac->start > bc->start) - return 1; - - return 0; -} - -const QUIC_TXPIM_CHUNK *ossl_quic_txpim_pkt_get_chunks(const QUIC_TXPIM_PKT *fpkt) -{ - QUIC_TXPIM_PKT_EX *ex = (QUIC_TXPIM_PKT_EX *)fpkt; - - if (ex->chunks_need_sort) { - /* - * List of chunks will generally be very small so there is no issue - * simply sorting here. - */ - qsort(ex->chunks, ex->num_chunks, sizeof(QUIC_TXPIM_CHUNK), compare); - ex->chunks_need_sort = 0; - } - - return ex->chunks; -} - -size_t ossl_quic_txpim_pkt_get_num_chunks(const QUIC_TXPIM_PKT *fpkt) -{ - QUIC_TXPIM_PKT_EX *ex = (QUIC_TXPIM_PKT_EX *)fpkt; - - return ex->num_chunks; -} - -size_t ossl_quic_txpim_get_in_use(const QUIC_TXPIM *txpim) -{ - return txpim->in_use; -} diff --git a/openssl/src/ssl/quic/quic_types.c b/openssl/src/ssl/quic/quic_types.c deleted file mode 100644 index 4ff3ae658..000000000 --- a/openssl/src/ssl/quic/quic_types.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/quic_types.h" -#include -#include - -int ossl_quic_gen_rand_conn_id(OSSL_LIB_CTX *libctx, size_t len, - QUIC_CONN_ID *cid) -{ - if (len > QUIC_MAX_CONN_ID_LEN) - return 0; - - cid->id_len = (unsigned char)len; - - if (RAND_bytes_ex(libctx, cid->id, len, len * 8) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_RAND_LIB); - cid->id_len = 0; - return 0; - } - - return 1; -} diff --git a/openssl/src/ssl/quic/quic_wire.c b/openssl/src/ssl/quic/quic_wire.c deleted file mode 100644 index a7c766779..000000000 --- a/openssl/src/ssl/quic/quic_wire.c +++ /dev/null @@ -1,1078 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "internal/quic_ssl.h" -#include "internal/quic_vlint.h" -#include "internal/quic_wire.h" -#include "internal/quic_error.h" - -OSSL_SAFE_MATH_UNSIGNED(uint64_t, uint64_t) - -int ossl_quic_frame_ack_contains_pn(const OSSL_QUIC_FRAME_ACK *ack, QUIC_PN pn) -{ - size_t i; - - for (i = 0; i < ack->num_ack_ranges; ++i) - if (pn >= ack->ack_ranges[i].start - && pn <= ack->ack_ranges[i].end) - return 1; - - return 0; -} - -/* - * QUIC Wire Format Encoding - * ========================= - */ - -int ossl_quic_wire_encode_padding(WPACKET *pkt, size_t num_bytes) -{ - /* - * PADDING is frame type zero, which as a variable-length integer is - * represented as a single zero byte. As an optimisation, just use memset. - */ - return WPACKET_memset(pkt, 0, num_bytes); -} - -static int encode_frame_hdr(WPACKET *pkt, uint64_t frame_type) -{ - return WPACKET_quic_write_vlint(pkt, frame_type); -} - -int ossl_quic_wire_encode_frame_ping(WPACKET *pkt) -{ - return encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_PING); -} - -int ossl_quic_wire_encode_frame_ack(WPACKET *pkt, - uint32_t ack_delay_exponent, - const OSSL_QUIC_FRAME_ACK *ack) -{ - uint64_t frame_type = ack->ecn_present ? OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN - : OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN; - - uint64_t largest_ackd, first_ack_range, ack_delay_enc; - uint64_t i, num_ack_ranges = ack->num_ack_ranges; - OSSL_TIME delay; - - if (num_ack_ranges == 0) - return 0; - - delay = ossl_time_divide(ossl_time_divide(ack->delay_time, OSSL_TIME_US), - (uint64_t)1 << ack_delay_exponent); - ack_delay_enc = ossl_time2ticks(delay); - - largest_ackd = ack->ack_ranges[0].end; - first_ack_range = ack->ack_ranges[0].end - ack->ack_ranges[0].start; - - if (!encode_frame_hdr(pkt, frame_type) - || !WPACKET_quic_write_vlint(pkt, largest_ackd) - || !WPACKET_quic_write_vlint(pkt, ack_delay_enc) - || !WPACKET_quic_write_vlint(pkt, num_ack_ranges - 1) - || !WPACKET_quic_write_vlint(pkt, first_ack_range)) - return 0; - - for (i = 1; i < num_ack_ranges; ++i) { - uint64_t gap, range_len; - - gap = ack->ack_ranges[i - 1].start - ack->ack_ranges[i].end - 2; - range_len = ack->ack_ranges[i].end - ack->ack_ranges[i].start; - - if (!WPACKET_quic_write_vlint(pkt, gap) - || !WPACKET_quic_write_vlint(pkt, range_len)) - return 0; - } - - if (ack->ecn_present) - if (!WPACKET_quic_write_vlint(pkt, ack->ect0) - || !WPACKET_quic_write_vlint(pkt, ack->ect1) - || !WPACKET_quic_write_vlint(pkt, ack->ecnce)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_reset_stream(WPACKET *pkt, - const OSSL_QUIC_FRAME_RESET_STREAM *f) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_RESET_STREAM) - || !WPACKET_quic_write_vlint(pkt, f->stream_id) - || !WPACKET_quic_write_vlint(pkt, f->app_error_code) - || !WPACKET_quic_write_vlint(pkt, f->final_size)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_stop_sending(WPACKET *pkt, - const OSSL_QUIC_FRAME_STOP_SENDING *f) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_STOP_SENDING) - || !WPACKET_quic_write_vlint(pkt, f->stream_id) - || !WPACKET_quic_write_vlint(pkt, f->app_error_code)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_crypto_hdr(WPACKET *pkt, - const OSSL_QUIC_FRAME_CRYPTO *f) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_CRYPTO) - || !WPACKET_quic_write_vlint(pkt, f->offset) - || !WPACKET_quic_write_vlint(pkt, f->len)) - return 0; - - return 1; -} - -size_t ossl_quic_wire_get_encoded_frame_len_crypto_hdr(const OSSL_QUIC_FRAME_CRYPTO *f) -{ - size_t a, b, c; - - a = ossl_quic_vlint_encode_len(OSSL_QUIC_FRAME_TYPE_CRYPTO); - b = ossl_quic_vlint_encode_len(f->offset); - c = ossl_quic_vlint_encode_len(f->len); - if (a == 0 || b == 0 || c == 0) - return 0; - - return a + b + c; -} - -void *ossl_quic_wire_encode_frame_crypto(WPACKET *pkt, - const OSSL_QUIC_FRAME_CRYPTO *f) -{ - unsigned char *p = NULL; - - if (!ossl_quic_wire_encode_frame_crypto_hdr(pkt, f) - || f->len > SIZE_MAX /* sizeof(uint64_t) > sizeof(size_t)? */ - || !WPACKET_allocate_bytes(pkt, (size_t)f->len, &p)) - return NULL; - - if (f->data != NULL) - memcpy(p, f->data, (size_t)f->len); - - return p; -} - -int ossl_quic_wire_encode_frame_new_token(WPACKET *pkt, - const unsigned char *token, - size_t token_len) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_NEW_TOKEN) - || !WPACKET_quic_write_vlint(pkt, token_len) - || !WPACKET_memcpy(pkt, token, token_len)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_stream_hdr(WPACKET *pkt, - const OSSL_QUIC_FRAME_STREAM *f) -{ - uint64_t frame_type = OSSL_QUIC_FRAME_TYPE_STREAM; - - if (f->offset != 0) - frame_type |= OSSL_QUIC_FRAME_FLAG_STREAM_OFF; - if (f->has_explicit_len) - frame_type |= OSSL_QUIC_FRAME_FLAG_STREAM_LEN; - if (f->is_fin) - frame_type |= OSSL_QUIC_FRAME_FLAG_STREAM_FIN; - - if (!encode_frame_hdr(pkt, frame_type) - || !WPACKET_quic_write_vlint(pkt, f->stream_id)) - return 0; - - if (f->offset != 0 && !WPACKET_quic_write_vlint(pkt, f->offset)) - return 0; - - if (f->has_explicit_len && !WPACKET_quic_write_vlint(pkt, f->len)) - return 0; - - return 1; -} - -size_t ossl_quic_wire_get_encoded_frame_len_stream_hdr(const OSSL_QUIC_FRAME_STREAM *f) -{ - size_t a, b, c, d; - - a = ossl_quic_vlint_encode_len(OSSL_QUIC_FRAME_TYPE_STREAM); - b = ossl_quic_vlint_encode_len(f->stream_id); - if (a == 0 || b == 0) - return 0; - - if (f->offset > 0) { - c = ossl_quic_vlint_encode_len(f->offset); - if (c == 0) - return 0; - } else { - c = 0; - } - - if (f->has_explicit_len) { - d = ossl_quic_vlint_encode_len(f->len); - if (d == 0) - return 0; - } else { - d = 0; - } - - return a + b + c + d; -} - -void *ossl_quic_wire_encode_frame_stream(WPACKET *pkt, - const OSSL_QUIC_FRAME_STREAM *f) -{ - - unsigned char *p = NULL; - - if (!ossl_quic_wire_encode_frame_stream_hdr(pkt, f) - || f->len > SIZE_MAX /* sizeof(uint64_t) > sizeof(size_t)? */) - return NULL; - - if (!WPACKET_allocate_bytes(pkt, (size_t)f->len, &p)) - return NULL; - - if (f->data != NULL) - memcpy(p, f->data, (size_t)f->len); - - return p; -} - -int ossl_quic_wire_encode_frame_max_data(WPACKET *pkt, - uint64_t max_data) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_MAX_DATA) - || !WPACKET_quic_write_vlint(pkt, max_data)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_max_stream_data(WPACKET *pkt, - uint64_t stream_id, - uint64_t max_data) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA) - || !WPACKET_quic_write_vlint(pkt, stream_id) - || !WPACKET_quic_write_vlint(pkt, max_data)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_max_streams(WPACKET *pkt, - char is_uni, - uint64_t max_streams) -{ - if (!encode_frame_hdr(pkt, is_uni ? OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI - : OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI) - || !WPACKET_quic_write_vlint(pkt, max_streams)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_data_blocked(WPACKET *pkt, - uint64_t max_data) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_DATA_BLOCKED) - || !WPACKET_quic_write_vlint(pkt, max_data)) - return 0; - - return 1; -} - - -int ossl_quic_wire_encode_frame_stream_data_blocked(WPACKET *pkt, - uint64_t stream_id, - uint64_t max_stream_data) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED) - || !WPACKET_quic_write_vlint(pkt, stream_id) - || !WPACKET_quic_write_vlint(pkt, max_stream_data)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_streams_blocked(WPACKET *pkt, - char is_uni, - uint64_t max_streams) -{ - if (!encode_frame_hdr(pkt, is_uni ? OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_UNI - : OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI) - || !WPACKET_quic_write_vlint(pkt, max_streams)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_new_conn_id(WPACKET *pkt, - const OSSL_QUIC_FRAME_NEW_CONN_ID *f) -{ - if (f->conn_id.id_len < 1 - || f->conn_id.id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID) - || !WPACKET_quic_write_vlint(pkt, f->seq_num) - || !WPACKET_quic_write_vlint(pkt, f->retire_prior_to) - || !WPACKET_put_bytes_u8(pkt, f->conn_id.id_len) - || !WPACKET_memcpy(pkt, f->conn_id.id, f->conn_id.id_len) - || !WPACKET_memcpy(pkt, f->stateless_reset.token, - sizeof(f->stateless_reset.token))) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_retire_conn_id(WPACKET *pkt, - uint64_t seq_num) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID) - || !WPACKET_quic_write_vlint(pkt, seq_num)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_path_challenge(WPACKET *pkt, - uint64_t data) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_PATH_CHALLENGE) - || !WPACKET_put_bytes_u64(pkt, data)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_path_response(WPACKET *pkt, - uint64_t data) -{ - if (!encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE) - || !WPACKET_put_bytes_u64(pkt, data)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_conn_close(WPACKET *pkt, - const OSSL_QUIC_FRAME_CONN_CLOSE *f) -{ - if (!encode_frame_hdr(pkt, f->is_app ? OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_APP - : OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT) - || !WPACKET_quic_write_vlint(pkt, f->error_code)) - return 0; - - /* - * RFC 9000 s. 19.19: The application-specific variant of CONNECTION_CLOSE - * (type 0x1d) does not include this field. - */ - if (!f->is_app && !WPACKET_quic_write_vlint(pkt, f->frame_type)) - return 0; - - if (!WPACKET_quic_write_vlint(pkt, f->reason_len) - || !WPACKET_memcpy(pkt, f->reason, f->reason_len)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_frame_handshake_done(WPACKET *pkt) -{ - return encode_frame_hdr(pkt, OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE); -} - -unsigned char *ossl_quic_wire_encode_transport_param_bytes(WPACKET *pkt, - uint64_t id, - const unsigned char *value, - size_t value_len) -{ - unsigned char *b = NULL; - - if (!WPACKET_quic_write_vlint(pkt, id) - || !WPACKET_quic_write_vlint(pkt, value_len)) - return NULL; - - if (value_len == 0) - b = WPACKET_get_curr(pkt); - else if (!WPACKET_allocate_bytes(pkt, value_len, (unsigned char **)&b)) - return NULL; - - if (value != NULL) - memcpy(b, value, value_len); - - return b; -} - -int ossl_quic_wire_encode_transport_param_int(WPACKET *pkt, - uint64_t id, - uint64_t value) -{ - if (!WPACKET_quic_write_vlint(pkt, id) - || !WPACKET_quic_write_vlint(pkt, ossl_quic_vlint_encode_len(value)) - || !WPACKET_quic_write_vlint(pkt, value)) - return 0; - - return 1; -} - -int ossl_quic_wire_encode_transport_param_cid(WPACKET *wpkt, - uint64_t id, - const QUIC_CONN_ID *cid) -{ - if (cid->id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - - if (ossl_quic_wire_encode_transport_param_bytes(wpkt, id, - cid->id, - cid->id_len) == NULL) - return 0; - - return 1; -} - -/* - * QUIC Wire Format Decoding - * ========================= - */ -int ossl_quic_wire_peek_frame_header(PACKET *pkt, uint64_t *type, - int *was_minimal) -{ - return PACKET_peek_quic_vlint_ex(pkt, type, was_minimal); -} - -int ossl_quic_wire_skip_frame_header(PACKET *pkt, uint64_t *type) -{ - return PACKET_get_quic_vlint(pkt, type); -} - -static int expect_frame_header_mask(PACKET *pkt, - uint64_t expected_frame_type, - uint64_t mask_bits, - uint64_t *actual_frame_type) -{ - uint64_t actual_frame_type_; - - if (!ossl_quic_wire_skip_frame_header(pkt, &actual_frame_type_) - || (actual_frame_type_ & ~mask_bits) != expected_frame_type) - return 0; - - if (actual_frame_type != NULL) - *actual_frame_type = actual_frame_type_; - - return 1; -} - -static int expect_frame_header(PACKET *pkt, uint64_t expected_frame_type) -{ - uint64_t actual_frame_type; - - if (!ossl_quic_wire_skip_frame_header(pkt, &actual_frame_type) - || actual_frame_type != expected_frame_type) - return 0; - - return 1; -} - -int ossl_quic_wire_peek_frame_ack_num_ranges(const PACKET *orig_pkt, - uint64_t *total_ranges) -{ - PACKET pkt = *orig_pkt; - uint64_t ack_range_count, i; - - if (!expect_frame_header_mask(&pkt, OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN, - 1, NULL) - || !PACKET_skip_quic_vlint(&pkt) - || !PACKET_skip_quic_vlint(&pkt) - || !PACKET_get_quic_vlint(&pkt, &ack_range_count)) - return 0; - - /* - * Ensure the specified number of ack ranges listed in the ACK frame header - * actually are available in the frame data. This naturally bounds the - * number of ACK ranges which can be requested by the MDPL, and therefore by - * the MTU. This ensures we do not allocate memory for an excessive number - * of ACK ranges. - */ - for (i = 0; i < ack_range_count; ++i) - if (!PACKET_skip_quic_vlint(&pkt) - || !PACKET_skip_quic_vlint(&pkt)) - return 0; - - /* (cannot overflow because QUIC vlints can only encode up to 2**62-1) */ - *total_ranges = ack_range_count + 1; - return 1; -} - -int ossl_quic_wire_decode_frame_ack(PACKET *pkt, - uint32_t ack_delay_exponent, - OSSL_QUIC_FRAME_ACK *ack, - uint64_t *total_ranges) { - uint64_t frame_type, largest_ackd, ack_delay_raw; - uint64_t ack_range_count, first_ack_range, start, end, i; - - /* This call matches both ACK_WITHOUT_ECN and ACK_WITH_ECN. */ - if (!expect_frame_header_mask(pkt, OSSL_QUIC_FRAME_TYPE_ACK_WITHOUT_ECN, - 1, &frame_type) - || !PACKET_get_quic_vlint(pkt, &largest_ackd) - || !PACKET_get_quic_vlint(pkt, &ack_delay_raw) - || !PACKET_get_quic_vlint(pkt, &ack_range_count) - || !PACKET_get_quic_vlint(pkt, &first_ack_range)) - return 0; - - if (first_ack_range > largest_ackd) - return 0; - - if (ack_range_count > SIZE_MAX /* sizeof(uint64_t) > sizeof(size_t)? */) - return 0; - - start = largest_ackd - first_ack_range; - - if (ack != NULL) { - int err = 0; - ack->delay_time - = ossl_time_multiply(ossl_ticks2time(OSSL_TIME_US), - safe_mul_uint64_t(ack_delay_raw, - (uint64_t)1 << ack_delay_exponent, - &err)); - if (err) - ack->delay_time = ossl_time_infinite(); - - if (ack->num_ack_ranges > 0) { - ack->ack_ranges[0].end = largest_ackd; - ack->ack_ranges[0].start = start; - } - } - - for (i = 0; i < ack_range_count; ++i) { - uint64_t gap, len; - - if (!PACKET_get_quic_vlint(pkt, &gap) - || !PACKET_get_quic_vlint(pkt, &len)) - return 0; - - end = start - gap - 2; - if (start < gap + 2 || len > end) - return 0; - - if (ack != NULL && i + 1 < ack->num_ack_ranges) { - ack->ack_ranges[i + 1].start = start = end - len; - ack->ack_ranges[i + 1].end = end; - } - } - - if (ack != NULL && ack_range_count + 1 < ack->num_ack_ranges) - ack->num_ack_ranges = (size_t)ack_range_count + 1; - - if (total_ranges != NULL) - *total_ranges = ack_range_count + 1; - - if (frame_type == OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN) { - uint64_t ect0, ect1, ecnce; - - if (!PACKET_get_quic_vlint(pkt, &ect0) - || !PACKET_get_quic_vlint(pkt, &ect1) - || !PACKET_get_quic_vlint(pkt, &ecnce)) - return 0; - - if (ack != NULL) { - ack->ect0 = ect0; - ack->ect1 = ect1; - ack->ecnce = ecnce; - ack->ecn_present = 1; - } - } else if (ack != NULL) { - ack->ecn_present = 0; - } - - return 1; -} - -int ossl_quic_wire_decode_frame_reset_stream(PACKET *pkt, - OSSL_QUIC_FRAME_RESET_STREAM *f) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_RESET_STREAM) - || !PACKET_get_quic_vlint(pkt, &f->stream_id) - || !PACKET_get_quic_vlint(pkt, &f->app_error_code) - || !PACKET_get_quic_vlint(pkt, &f->final_size)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_stop_sending(PACKET *pkt, - OSSL_QUIC_FRAME_STOP_SENDING *f) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_STOP_SENDING) - || !PACKET_get_quic_vlint(pkt, &f->stream_id) - || !PACKET_get_quic_vlint(pkt, &f->app_error_code)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_crypto(PACKET *pkt, - int nodata, - OSSL_QUIC_FRAME_CRYPTO *f) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_CRYPTO) - || !PACKET_get_quic_vlint(pkt, &f->offset) - || !PACKET_get_quic_vlint(pkt, &f->len) - || f->len > SIZE_MAX /* sizeof(uint64_t) > sizeof(size_t)? */) - return 0; - - if (f->offset + f->len > (((uint64_t)1) << 62) - 1) - /* RFC 9000 s. 19.6 */ - return 0; - - if (nodata) { - f->data = NULL; - } else { - if (PACKET_remaining(pkt) < f->len) - return 0; - - f->data = PACKET_data(pkt); - - if (!PACKET_forward(pkt, (size_t)f->len)) - return 0; - } - - return 1; -} - -int ossl_quic_wire_decode_frame_new_token(PACKET *pkt, - const unsigned char **token, - size_t *token_len) -{ - uint64_t token_len_; - - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_NEW_TOKEN) - || !PACKET_get_quic_vlint(pkt, &token_len_)) - return 0; - - if (token_len_ > SIZE_MAX) - return 0; - - *token = PACKET_data(pkt); - *token_len = (size_t)token_len_; - - if (!PACKET_forward(pkt, (size_t)token_len_)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_stream(PACKET *pkt, - int nodata, - OSSL_QUIC_FRAME_STREAM *f) -{ - uint64_t frame_type; - - /* This call matches all STREAM values (low 3 bits are masked). */ - if (!expect_frame_header_mask(pkt, OSSL_QUIC_FRAME_TYPE_STREAM, - OSSL_QUIC_FRAME_FLAG_STREAM_MASK, - &frame_type) - || !PACKET_get_quic_vlint(pkt, &f->stream_id)) - return 0; - - if ((frame_type & OSSL_QUIC_FRAME_FLAG_STREAM_OFF) != 0) { - if (!PACKET_get_quic_vlint(pkt, &f->offset)) - return 0; - } else { - f->offset = 0; - } - - f->has_explicit_len = ((frame_type & OSSL_QUIC_FRAME_FLAG_STREAM_LEN) != 0); - f->is_fin = ((frame_type & OSSL_QUIC_FRAME_FLAG_STREAM_FIN) != 0); - - if (f->has_explicit_len) { - if (!PACKET_get_quic_vlint(pkt, &f->len)) - return 0; - } else { - if (nodata) - f->len = 0; - else - f->len = PACKET_remaining(pkt); - } - - /* - * RFC 9000 s. 19.8: "The largest offset delivered on a stream -- the sum of - * the offset and data length -- cannot exceed 2**62 - 1, as it is not - * possible to provide flow control credit for that data." - */ - if (f->offset + f->len > (((uint64_t)1) << 62) - 1) - return 0; - - if (nodata) { - f->data = NULL; - } else { - f->data = PACKET_data(pkt); - - if (f->len > SIZE_MAX /* sizeof(uint64_t) > sizeof(size_t)? */ - || !PACKET_forward(pkt, (size_t)f->len)) - return 0; - } - - return 1; -} - -int ossl_quic_wire_decode_frame_max_data(PACKET *pkt, - uint64_t *max_data) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_MAX_DATA) - || !PACKET_get_quic_vlint(pkt, max_data)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_max_stream_data(PACKET *pkt, - uint64_t *stream_id, - uint64_t *max_stream_data) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA) - || !PACKET_get_quic_vlint(pkt, stream_id) - || !PACKET_get_quic_vlint(pkt, max_stream_data)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_max_streams(PACKET *pkt, - uint64_t *max_streams) -{ - /* This call matches both MAX_STREAMS_BIDI and MAX_STREAMS_UNI. */ - if (!expect_frame_header_mask(pkt, OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI, - 1, NULL) - || !PACKET_get_quic_vlint(pkt, max_streams)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_data_blocked(PACKET *pkt, - uint64_t *max_data) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_DATA_BLOCKED) - || !PACKET_get_quic_vlint(pkt, max_data)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_stream_data_blocked(PACKET *pkt, - uint64_t *stream_id, - uint64_t *max_stream_data) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_STREAM_DATA_BLOCKED) - || !PACKET_get_quic_vlint(pkt, stream_id) - || !PACKET_get_quic_vlint(pkt, max_stream_data)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_streams_blocked(PACKET *pkt, - uint64_t *max_streams) -{ - /* This call matches both STREAMS_BLOCKED_BIDI and STREAMS_BLOCKED_UNI. */ - if (!expect_frame_header_mask(pkt, OSSL_QUIC_FRAME_TYPE_STREAMS_BLOCKED_BIDI, - 1, NULL) - || !PACKET_get_quic_vlint(pkt, max_streams)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_new_conn_id(PACKET *pkt, - OSSL_QUIC_FRAME_NEW_CONN_ID *f) -{ - unsigned int len; - - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID) - || !PACKET_get_quic_vlint(pkt, &f->seq_num) - || !PACKET_get_quic_vlint(pkt, &f->retire_prior_to) - || f->seq_num < f->retire_prior_to - || !PACKET_get_1(pkt, &len) - || len < 1 - || len > QUIC_MAX_CONN_ID_LEN) - return 0; - - f->conn_id.id_len = (unsigned char)len; - if (!PACKET_copy_bytes(pkt, f->conn_id.id, len)) - return 0; - - /* Clear unused bytes to allow consistent memcmp. */ - if (len < QUIC_MAX_CONN_ID_LEN) - memset(f->conn_id.id + len, 0, QUIC_MAX_CONN_ID_LEN - len); - - if (!PACKET_copy_bytes(pkt, f->stateless_reset.token, - sizeof(f->stateless_reset.token))) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_retire_conn_id(PACKET *pkt, - uint64_t *seq_num) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID) - || !PACKET_get_quic_vlint(pkt, seq_num)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_path_challenge(PACKET *pkt, - uint64_t *data) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_PATH_CHALLENGE) - || !PACKET_get_net_8(pkt, data)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_path_response(PACKET *pkt, - uint64_t *data) -{ - if (!expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE) - || !PACKET_get_net_8(pkt, data)) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_frame_conn_close(PACKET *pkt, - OSSL_QUIC_FRAME_CONN_CLOSE *f) -{ - uint64_t frame_type, reason_len; - - /* This call matches both CONN_CLOSE_TRANSPORT and CONN_CLOSE_APP. */ - if (!expect_frame_header_mask(pkt, OSSL_QUIC_FRAME_TYPE_CONN_CLOSE_TRANSPORT, - 1, &frame_type) - || !PACKET_get_quic_vlint(pkt, &f->error_code)) - return 0; - - f->is_app = ((frame_type & 1) != 0); - - if (!f->is_app) { - if (!PACKET_get_quic_vlint(pkt, &f->frame_type)) - return 0; - } else { - f->frame_type = 0; - } - - if (!PACKET_get_quic_vlint(pkt, &reason_len) - || reason_len > SIZE_MAX) - return 0; - - if (!PACKET_get_bytes(pkt, (const unsigned char **)&f->reason, - (size_t)reason_len)) - return 0; - - f->reason_len = (size_t)reason_len; - return 1; -} - -size_t ossl_quic_wire_decode_padding(PACKET *pkt) -{ - const unsigned char *start = PACKET_data(pkt), *end = PACKET_end(pkt), - *p = start; - - while (p < end && *p == 0) - ++p; - - if (!PACKET_forward(pkt, p - start)) - return 0; - - return p - start; -} - -int ossl_quic_wire_decode_frame_ping(PACKET *pkt) -{ - return expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_PING); -} - -int ossl_quic_wire_decode_frame_handshake_done(PACKET *pkt) -{ - return expect_frame_header(pkt, OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE); -} - -int ossl_quic_wire_peek_transport_param(PACKET *pkt, uint64_t *id) -{ - return PACKET_peek_quic_vlint(pkt, id); -} - -const unsigned char *ossl_quic_wire_decode_transport_param_bytes(PACKET *pkt, - uint64_t *id, - size_t *len) -{ - uint64_t len_; - const unsigned char *b = NULL; - uint64_t id_; - - if (!PACKET_get_quic_vlint(pkt, &id_) - || !PACKET_get_quic_vlint(pkt, &len_)) - return NULL; - - if (len_ > SIZE_MAX - || !PACKET_get_bytes(pkt, (const unsigned char **)&b, (size_t)len_)) - return NULL; - - *len = (size_t)len_; - if (id != NULL) - *id = id_; - return b; -} - -int ossl_quic_wire_decode_transport_param_int(PACKET *pkt, - uint64_t *id, - uint64_t *value) -{ - PACKET sub; - - sub.curr = ossl_quic_wire_decode_transport_param_bytes(pkt, - id, &sub.remaining); - if (sub.curr == NULL) - return 0; - - if (!PACKET_get_quic_vlint(&sub, value)) - return 0; - - if (PACKET_remaining(&sub) > 0) - return 0; - - return 1; -} - -int ossl_quic_wire_decode_transport_param_cid(PACKET *pkt, - uint64_t *id, - QUIC_CONN_ID *cid) -{ - const unsigned char *body; - size_t len = 0; - - body = ossl_quic_wire_decode_transport_param_bytes(pkt, id, &len); - if (body == NULL || len > QUIC_MAX_CONN_ID_LEN) - return 0; - - cid->id_len = (unsigned char)len; - memcpy(cid->id, body, cid->id_len); - return 1; -} - -int ossl_quic_wire_decode_transport_param_preferred_addr(PACKET *pkt, - QUIC_PREFERRED_ADDR *p) -{ - const unsigned char *body; - uint64_t id; - size_t len = 0; - PACKET pkt2; - unsigned int ipv4_port, ipv6_port, cidl; - - body = ossl_quic_wire_decode_transport_param_bytes(pkt, &id, &len); - if (body == NULL - || len < QUIC_MIN_ENCODED_PREFERRED_ADDR_LEN - || len > QUIC_MAX_ENCODED_PREFERRED_ADDR_LEN - || id != QUIC_TPARAM_PREFERRED_ADDR) - return 0; - - if (!PACKET_buf_init(&pkt2, body, len)) - return 0; - - if (!PACKET_copy_bytes(&pkt2, p->ipv4, sizeof(p->ipv4)) - || !PACKET_get_net_2(&pkt2, &ipv4_port) - || !PACKET_copy_bytes(&pkt2, p->ipv6, sizeof(p->ipv6)) - || !PACKET_get_net_2(&pkt2, &ipv6_port) - || !PACKET_get_1(&pkt2, &cidl) - || cidl > QUIC_MAX_CONN_ID_LEN - || !PACKET_copy_bytes(&pkt2, p->cid.id, cidl) - || !PACKET_copy_bytes(&pkt2, p->stateless_reset.token, - sizeof(p->stateless_reset.token))) - return 0; - - p->ipv4_port = (uint16_t)ipv4_port; - p->ipv6_port = (uint16_t)ipv6_port; - p->cid.id_len = (unsigned char)cidl; - return 1; -} - -const char * -ossl_quic_frame_type_to_string(uint64_t frame_type) -{ - switch (frame_type) { -#define X(name) case OSSL_QUIC_FRAME_TYPE_##name: return #name; - X(PADDING) - X(PING) - X(ACK_WITHOUT_ECN) - X(ACK_WITH_ECN) - X(RESET_STREAM) - X(STOP_SENDING) - X(CRYPTO) - X(NEW_TOKEN) - X(MAX_DATA) - X(MAX_STREAM_DATA) - X(MAX_STREAMS_BIDI) - X(MAX_STREAMS_UNI) - X(DATA_BLOCKED) - X(STREAM_DATA_BLOCKED) - X(STREAMS_BLOCKED_BIDI) - X(STREAMS_BLOCKED_UNI) - X(NEW_CONN_ID) - X(RETIRE_CONN_ID) - X(PATH_CHALLENGE) - X(PATH_RESPONSE) - X(CONN_CLOSE_TRANSPORT) - X(CONN_CLOSE_APP) - X(HANDSHAKE_DONE) - X(STREAM) - X(STREAM_FIN) - X(STREAM_LEN) - X(STREAM_LEN_FIN) - X(STREAM_OFF) - X(STREAM_OFF_FIN) - X(STREAM_OFF_LEN) - X(STREAM_OFF_LEN_FIN) -#undef X - default: - return NULL; - } -} - -const char *ossl_quic_err_to_string(uint64_t error_code) -{ - switch (error_code) { -#define X(name) case OSSL_QUIC_ERR_##name: return #name; - X(NO_ERROR) - X(INTERNAL_ERROR) - X(CONNECTION_REFUSED) - X(FLOW_CONTROL_ERROR) - X(STREAM_LIMIT_ERROR) - X(STREAM_STATE_ERROR) - X(FINAL_SIZE_ERROR) - X(FRAME_ENCODING_ERROR) - X(TRANSPORT_PARAMETER_ERROR) - X(CONNECTION_ID_LIMIT_ERROR) - X(PROTOCOL_VIOLATION) - X(INVALID_TOKEN) - X(APPLICATION_ERROR) - X(CRYPTO_BUFFER_EXCEEDED) - X(KEY_UPDATE_ERROR) - X(AEAD_LIMIT_REACHED) - X(NO_VIABLE_PATH) -#undef X - default: - return NULL; - } -} diff --git a/openssl/src/ssl/quic/quic_wire_pkt.c b/openssl/src/ssl/quic/quic_wire_pkt.c deleted file mode 100644 index acb926ad3..000000000 --- a/openssl/src/ssl/quic/quic_wire_pkt.c +++ /dev/null @@ -1,945 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "internal/common.h" -#include "internal/quic_wire_pkt.h" - -int ossl_quic_hdr_protector_init(QUIC_HDR_PROTECTOR *hpr, - OSSL_LIB_CTX *libctx, - const char *propq, - uint32_t cipher_id, - const unsigned char *quic_hp_key, - size_t quic_hp_key_len) -{ - const char *cipher_name = NULL; - - switch (cipher_id) { - case QUIC_HDR_PROT_CIPHER_AES_128: - cipher_name = "AES-128-ECB"; - break; - case QUIC_HDR_PROT_CIPHER_AES_256: - cipher_name = "AES-256-ECB"; - break; - case QUIC_HDR_PROT_CIPHER_CHACHA: - cipher_name = "ChaCha20"; - break; - default: - ERR_raise(ERR_LIB_SSL, ERR_R_UNSUPPORTED); - return 0; - } - - hpr->cipher_ctx = EVP_CIPHER_CTX_new(); - if (hpr->cipher_ctx == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - hpr->cipher = EVP_CIPHER_fetch(libctx, cipher_name, propq); - if (hpr->cipher == NULL - || quic_hp_key_len != (size_t)EVP_CIPHER_get_key_length(hpr->cipher)) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - goto err; - } - - if (!EVP_CipherInit_ex(hpr->cipher_ctx, hpr->cipher, NULL, - quic_hp_key, NULL, 1)) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - goto err; - } - - hpr->libctx = libctx; - hpr->propq = propq; - hpr->cipher_id = cipher_id; - return 1; - -err: - ossl_quic_hdr_protector_cleanup(hpr); - return 0; -} - -void ossl_quic_hdr_protector_cleanup(QUIC_HDR_PROTECTOR *hpr) -{ - EVP_CIPHER_CTX_free(hpr->cipher_ctx); - hpr->cipher_ctx = NULL; - - EVP_CIPHER_free(hpr->cipher); - hpr->cipher = NULL; -} - -static int hdr_generate_mask(QUIC_HDR_PROTECTOR *hpr, - const unsigned char *sample, size_t sample_len, - unsigned char *mask) -{ - int l = 0; - unsigned char dst[16]; - static const unsigned char zeroes[5] = {0}; - size_t i; - - if (hpr->cipher_id == QUIC_HDR_PROT_CIPHER_AES_128 - || hpr->cipher_id == QUIC_HDR_PROT_CIPHER_AES_256) { - if (sample_len < 16) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - if (!EVP_CipherInit_ex(hpr->cipher_ctx, NULL, NULL, NULL, NULL, 1) - || !EVP_CipherUpdate(hpr->cipher_ctx, dst, &l, sample, 16)) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - for (i = 0; i < 5; ++i) - mask[i] = dst[i]; - } else if (hpr->cipher_id == QUIC_HDR_PROT_CIPHER_CHACHA) { - if (sample_len < 16) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } - - if (!EVP_CipherInit_ex(hpr->cipher_ctx, NULL, NULL, NULL, sample, 1) - || !EVP_CipherUpdate(hpr->cipher_ctx, mask, &l, - zeroes, sizeof(zeroes))) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - } else { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - assert(0); - return 0; - } - -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - /* No matter what we did above we use the same mask in fuzzing mode */ - memset(mask, 0, 5); -#endif - - return 1; -} - -int ossl_quic_hdr_protector_decrypt(QUIC_HDR_PROTECTOR *hpr, - QUIC_PKT_HDR_PTRS *ptrs) -{ - return ossl_quic_hdr_protector_decrypt_fields(hpr, - ptrs->raw_sample, - ptrs->raw_sample_len, - ptrs->raw_start, - ptrs->raw_pn); -} - -int ossl_quic_hdr_protector_decrypt_fields(QUIC_HDR_PROTECTOR *hpr, - const unsigned char *sample, - size_t sample_len, - unsigned char *first_byte, - unsigned char *pn_bytes) -{ - unsigned char mask[5], pn_len, i; - - if (!hdr_generate_mask(hpr, sample, sample_len, mask)) - return 0; - - *first_byte ^= mask[0] & ((*first_byte & 0x80) != 0 ? 0xf : 0x1f); - pn_len = (*first_byte & 0x3) + 1; - - for (i = 0; i < pn_len; ++i) - pn_bytes[i] ^= mask[i + 1]; - - return 1; -} - -int ossl_quic_hdr_protector_encrypt(QUIC_HDR_PROTECTOR *hpr, - QUIC_PKT_HDR_PTRS *ptrs) -{ - return ossl_quic_hdr_protector_encrypt_fields(hpr, - ptrs->raw_sample, - ptrs->raw_sample_len, - ptrs->raw_start, - ptrs->raw_pn); -} - -int ossl_quic_hdr_protector_encrypt_fields(QUIC_HDR_PROTECTOR *hpr, - const unsigned char *sample, - size_t sample_len, - unsigned char *first_byte, - unsigned char *pn_bytes) -{ - unsigned char mask[5], pn_len, i; - - if (!hdr_generate_mask(hpr, sample, sample_len, mask)) - return 0; - - pn_len = (*first_byte & 0x3) + 1; - for (i = 0; i < pn_len; ++i) - pn_bytes[i] ^= mask[i + 1]; - - *first_byte ^= mask[0] & ((*first_byte & 0x80) != 0 ? 0xf : 0x1f); - return 1; -} - -int ossl_quic_wire_decode_pkt_hdr(PACKET *pkt, - size_t short_conn_id_len, - int partial, - int nodata, - QUIC_PKT_HDR *hdr, - QUIC_PKT_HDR_PTRS *ptrs) -{ - unsigned int b0; - unsigned char *pn = NULL; - size_t l = PACKET_remaining(pkt); - - if (ptrs != NULL) { - ptrs->raw_start = (unsigned char *)PACKET_data(pkt); - ptrs->raw_sample = NULL; - ptrs->raw_sample_len = 0; - ptrs->raw_pn = NULL; - } - - if (l < QUIC_MIN_VALID_PKT_LEN - || !PACKET_get_1(pkt, &b0)) - return 0; - - hdr->partial = partial; - hdr->unused = 0; - hdr->reserved = 0; - - if ((b0 & 0x80) == 0) { - /* Short header. */ - if (short_conn_id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - - if ((b0 & 0x40) == 0 /* fixed bit not set? */ - || l < QUIC_MIN_VALID_PKT_LEN_CRYPTO) - return 0; - - hdr->type = QUIC_PKT_TYPE_1RTT; - hdr->fixed = 1; - hdr->spin_bit = (b0 & 0x20) != 0; - if (partial) { - hdr->key_phase = 0; /* protected, zero for now */ - hdr->pn_len = 0; /* protected, zero for now */ - hdr->reserved = 0; /* protected, zero for now */ - } else { - hdr->key_phase = (b0 & 0x04) != 0; - hdr->pn_len = (b0 & 0x03) + 1; - hdr->reserved = (b0 & 0x18) >> 3; - } - - /* Copy destination connection ID field to header structure. */ - if (!PACKET_copy_bytes(pkt, hdr->dst_conn_id.id, short_conn_id_len)) - return 0; - - hdr->dst_conn_id.id_len = (unsigned char)short_conn_id_len; - - /* - * Skip over the PN. If this is a partial decode, the PN length field - * currently has header protection applied. Thus we do not know the - * length of the PN but we are allowed to assume it is 4 bytes long at - * this stage. - */ - memset(hdr->pn, 0, sizeof(hdr->pn)); - pn = (unsigned char *)PACKET_data(pkt); - if (partial) { - if (!PACKET_forward(pkt, sizeof(hdr->pn))) - return 0; - } else { - if (!PACKET_copy_bytes(pkt, hdr->pn, hdr->pn_len)) - return 0; - } - - /* Fields not used in short-header packets. */ - hdr->version = 0; - hdr->src_conn_id.id_len = 0; - hdr->token = NULL; - hdr->token_len = 0; - - /* - * Short-header packets always come last in a datagram, the length - * is the remainder of the buffer. - */ - hdr->len = PACKET_remaining(pkt); - hdr->data = PACKET_data(pkt); - - /* - * Skip over payload. Since this is a short header packet, which cannot - * be followed by any other kind of packet, this advances us to the end - * of the datagram. - */ - if (!PACKET_forward(pkt, hdr->len)) - return 0; - } else { - /* Long header. */ - unsigned long version; - unsigned int dst_conn_id_len, src_conn_id_len, raw_type; - - if (!PACKET_get_net_4(pkt, &version)) - return 0; - - /* - * All QUIC packets must have the fixed bit set, except exceptionally - * for Version Negotiation packets. - */ - if (version != 0 && (b0 & 0x40) == 0) - return 0; - - if (!PACKET_get_1(pkt, &dst_conn_id_len) - || dst_conn_id_len > QUIC_MAX_CONN_ID_LEN - || !PACKET_copy_bytes(pkt, hdr->dst_conn_id.id, dst_conn_id_len) - || !PACKET_get_1(pkt, &src_conn_id_len) - || src_conn_id_len > QUIC_MAX_CONN_ID_LEN - || !PACKET_copy_bytes(pkt, hdr->src_conn_id.id, src_conn_id_len)) - return 0; - - hdr->version = (uint32_t)version; - hdr->dst_conn_id.id_len = (unsigned char)dst_conn_id_len; - hdr->src_conn_id.id_len = (unsigned char)src_conn_id_len; - - if (version == 0) { - /* - * Version negotiation packet. Version negotiation packets are - * identified by a version field of 0 and the type bits in the first - * byte are ignored (they may take any value, and we ignore them). - */ - hdr->type = QUIC_PKT_TYPE_VERSION_NEG; - hdr->fixed = (b0 & 0x40) != 0; - - hdr->data = PACKET_data(pkt); - hdr->len = PACKET_remaining(pkt); - - /* - * Version negotiation packets must contain an array of u32s, so it - * is invalid for their payload length to not be divisible by 4. - */ - if ((hdr->len % 4) != 0) - return 0; - - /* Version negotiation packets are always fully decoded. */ - hdr->partial = 0; - - /* Fields not used in version negotiation packets. */ - hdr->pn_len = 0; - hdr->spin_bit = 0; - hdr->key_phase = 0; - hdr->token = NULL; - hdr->token_len = 0; - memset(hdr->pn, 0, sizeof(hdr->pn)); - - if (!PACKET_forward(pkt, hdr->len)) - return 0; - } else if (version != QUIC_VERSION_1) { - /* Unknown version, do not decode. */ - return 0; - } else { - if (l < QUIC_MIN_VALID_PKT_LEN_CRYPTO) - return 0; - - /* Get long packet type and decode to QUIC_PKT_TYPE_*. */ - raw_type = ((b0 >> 4) & 0x3); - - switch (raw_type) { - case 0: - hdr->type = QUIC_PKT_TYPE_INITIAL; - break; - case 1: - hdr->type = QUIC_PKT_TYPE_0RTT; - break; - case 2: - hdr->type = QUIC_PKT_TYPE_HANDSHAKE; - break; - case 3: - hdr->type = QUIC_PKT_TYPE_RETRY; - break; - } - - hdr->pn_len = 0; - hdr->fixed = 1; - - /* Fields not used in long-header packets. */ - hdr->spin_bit = 0; - hdr->key_phase = 0; - - if (hdr->type == QUIC_PKT_TYPE_INITIAL) { - /* Initial packet. */ - uint64_t token_len; - - if (!PACKET_get_quic_vlint(pkt, &token_len) - || token_len > SIZE_MAX - || !PACKET_get_bytes(pkt, &hdr->token, (size_t)token_len)) - return 0; - - hdr->token_len = (size_t)token_len; - if (token_len == 0) - hdr->token = NULL; - } else { - hdr->token = NULL; - hdr->token_len = 0; - } - - if (hdr->type == QUIC_PKT_TYPE_RETRY) { - /* Retry packet. */ - hdr->data = PACKET_data(pkt); - hdr->len = PACKET_remaining(pkt); - - /* Retry packets are always fully decoded. */ - hdr->partial = 0; - - /* Unused bits in Retry header. */ - hdr->unused = b0 & 0x0f; - - /* Fields not used in Retry packets. */ - memset(hdr->pn, 0, sizeof(hdr->pn)); - - if (!PACKET_forward(pkt, hdr->len)) - return 0; - } else { - /* Initial, 0-RTT or Handshake packet. */ - uint64_t len; - - hdr->pn_len = partial ? 0 : ((b0 & 0x03) + 1); - hdr->reserved = partial ? 0 : ((b0 & 0x0C) >> 2); - - if (!PACKET_get_quic_vlint(pkt, &len) - || len < sizeof(hdr->pn)) - return 0; - - if (!nodata && len > PACKET_remaining(pkt)) - return 0; - - /* - * Skip over the PN. If this is a partial decode, the PN length - * field currently has header protection applied. Thus we do not - * know the length of the PN but we are allowed to assume it is - * 4 bytes long at this stage. - */ - pn = (unsigned char *)PACKET_data(pkt); - memset(hdr->pn, 0, sizeof(hdr->pn)); - if (partial) { - if (!PACKET_forward(pkt, sizeof(hdr->pn))) - return 0; - - hdr->len = (size_t)(len - sizeof(hdr->pn)); - } else { - if (!PACKET_copy_bytes(pkt, hdr->pn, hdr->pn_len)) - return 0; - - hdr->len = (size_t)(len - hdr->pn_len); - } - - if (nodata) { - hdr->data = NULL; - } else { - hdr->data = PACKET_data(pkt); - - /* Skip over packet body. */ - if (!PACKET_forward(pkt, hdr->len)) - return 0; - } - } - } - } - - if (ptrs != NULL) { - ptrs->raw_pn = pn; - if (pn != NULL) { - ptrs->raw_sample = pn + 4; - ptrs->raw_sample_len = PACKET_end(pkt) - ptrs->raw_sample; - } - } - - return 1; -} - -int ossl_quic_wire_encode_pkt_hdr(WPACKET *pkt, - size_t short_conn_id_len, - const QUIC_PKT_HDR *hdr, - QUIC_PKT_HDR_PTRS *ptrs) -{ - unsigned char b0; - size_t off_start, off_sample, off_pn; - unsigned char *start = WPACKET_get_curr(pkt); - - if (!WPACKET_get_total_written(pkt, &off_start)) - return 0; - - if (ptrs != NULL) { - /* ptrs would not be stable on non-static WPACKET */ - if (!ossl_assert(pkt->staticbuf != NULL)) - return 0; - ptrs->raw_start = NULL; - ptrs->raw_sample = NULL; - ptrs->raw_sample_len = 0; - ptrs->raw_pn = 0; - } - - /* Cannot serialize a partial header, or one whose DCID length is wrong. */ - if (hdr->partial - || (hdr->type == QUIC_PKT_TYPE_1RTT - && hdr->dst_conn_id.id_len != short_conn_id_len)) - return 0; - - if (hdr->type == QUIC_PKT_TYPE_1RTT) { - /* Short header. */ - - /* - * Cannot serialize a header whose DCID length is wrong, or with an - * invalid PN length. - */ - if (hdr->dst_conn_id.id_len != short_conn_id_len - || short_conn_id_len > QUIC_MAX_CONN_ID_LEN - || hdr->pn_len < 1 || hdr->pn_len > 4) - return 0; - - b0 = (hdr->spin_bit << 5) - | (hdr->key_phase << 2) - | (hdr->pn_len - 1) - | (hdr->reserved << 3) - | 0x40; /* fixed bit */ - - if (!WPACKET_put_bytes_u8(pkt, b0) - || !WPACKET_memcpy(pkt, hdr->dst_conn_id.id, short_conn_id_len) - || !WPACKET_get_total_written(pkt, &off_pn) - || !WPACKET_memcpy(pkt, hdr->pn, hdr->pn_len)) - return 0; - } else { - /* Long header. */ - unsigned int raw_type; - - if (hdr->dst_conn_id.id_len > QUIC_MAX_CONN_ID_LEN - || hdr->src_conn_id.id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - - if (ossl_quic_pkt_type_has_pn(hdr->type) - && (hdr->pn_len < 1 || hdr->pn_len > 4)) - return 0; - - switch (hdr->type) { - case QUIC_PKT_TYPE_VERSION_NEG: - if (hdr->version != 0) - return 0; - - /* Version negotiation packets use zero for the type bits */ - raw_type = 0; - break; - - case QUIC_PKT_TYPE_INITIAL: raw_type = 0; break; - case QUIC_PKT_TYPE_0RTT: raw_type = 1; break; - case QUIC_PKT_TYPE_HANDSHAKE: raw_type = 2; break; - case QUIC_PKT_TYPE_RETRY: raw_type = 3; break; - default: - return 0; - } - - b0 = (raw_type << 4) | 0x80; /* long */ - if (hdr->type != QUIC_PKT_TYPE_VERSION_NEG || hdr->fixed) - b0 |= 0x40; /* fixed */ - if (ossl_quic_pkt_type_has_pn(hdr->type)) { - b0 |= hdr->pn_len - 1; - b0 |= (hdr->reserved << 2); - } - if (hdr->type == QUIC_PKT_TYPE_RETRY) - b0 |= hdr->unused; - - if (!WPACKET_put_bytes_u8(pkt, b0) - || !WPACKET_put_bytes_u32(pkt, hdr->version) - || !WPACKET_put_bytes_u8(pkt, hdr->dst_conn_id.id_len) - || !WPACKET_memcpy(pkt, hdr->dst_conn_id.id, - hdr->dst_conn_id.id_len) - || !WPACKET_put_bytes_u8(pkt, hdr->src_conn_id.id_len) - || !WPACKET_memcpy(pkt, hdr->src_conn_id.id, - hdr->src_conn_id.id_len)) - return 0; - - if (hdr->type == QUIC_PKT_TYPE_VERSION_NEG - || hdr->type == QUIC_PKT_TYPE_RETRY) { - if (hdr->len > 0 && !WPACKET_reserve_bytes(pkt, hdr->len, NULL)) - return 0; - - return 1; - } - - if (hdr->type == QUIC_PKT_TYPE_INITIAL) { - if (!WPACKET_quic_write_vlint(pkt, hdr->token_len) - || !WPACKET_memcpy(pkt, hdr->token, hdr->token_len)) - return 0; - } - - if (!WPACKET_quic_write_vlint(pkt, hdr->len + hdr->pn_len) - || !WPACKET_get_total_written(pkt, &off_pn) - || !WPACKET_memcpy(pkt, hdr->pn, hdr->pn_len)) - return 0; - } - - if (hdr->len > 0 && !WPACKET_reserve_bytes(pkt, hdr->len, NULL)) - return 0; - - off_sample = off_pn + 4; - - if (ptrs != NULL) { - ptrs->raw_start = start; - ptrs->raw_sample = start + (off_sample - off_start); - ptrs->raw_sample_len - = WPACKET_get_curr(pkt) + hdr->len - ptrs->raw_sample; - ptrs->raw_pn = start + (off_pn - off_start); - } - - return 1; -} - -int ossl_quic_wire_get_encoded_pkt_hdr_len(size_t short_conn_id_len, - const QUIC_PKT_HDR *hdr) -{ - size_t len = 0, enclen; - - /* Cannot serialize a partial header, or one whose DCID length is wrong. */ - if (hdr->partial - || (hdr->type == QUIC_PKT_TYPE_1RTT - && hdr->dst_conn_id.id_len != short_conn_id_len)) - return 0; - - if (hdr->type == QUIC_PKT_TYPE_1RTT) { - /* Short header. */ - - /* - * Cannot serialize a header whose DCID length is wrong, or with an - * invalid PN length. - */ - if (hdr->dst_conn_id.id_len != short_conn_id_len - || short_conn_id_len > QUIC_MAX_CONN_ID_LEN - || hdr->pn_len < 1 || hdr->pn_len > 4) - return 0; - - return 1 + short_conn_id_len + hdr->pn_len; - } else { - /* Long header. */ - if (hdr->dst_conn_id.id_len > QUIC_MAX_CONN_ID_LEN - || hdr->src_conn_id.id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - - len += 1 /* Initial byte */ + 4 /* Version */ - + 1 + hdr->dst_conn_id.id_len /* DCID Len, DCID */ - + 1 + hdr->src_conn_id.id_len /* SCID Len, SCID */ - ; - - if (ossl_quic_pkt_type_has_pn(hdr->type)) { - if (hdr->pn_len < 1 || hdr->pn_len > 4) - return 0; - - len += hdr->pn_len; - } - - if (hdr->type == QUIC_PKT_TYPE_INITIAL) { - enclen = ossl_quic_vlint_encode_len(hdr->token_len); - if (!enclen) - return 0; - - len += enclen + hdr->token_len; - } - - if (!ossl_quic_pkt_type_must_be_last(hdr->type)) { - enclen = ossl_quic_vlint_encode_len(hdr->len + hdr->pn_len); - if (!enclen) - return 0; - - len += enclen; - } - - return len; - } -} - -int ossl_quic_wire_get_pkt_hdr_dst_conn_id(const unsigned char *buf, - size_t buf_len, - size_t short_conn_id_len, - QUIC_CONN_ID *dst_conn_id) -{ - unsigned char b0; - size_t blen; - - if (buf_len < QUIC_MIN_VALID_PKT_LEN - || short_conn_id_len > QUIC_MAX_CONN_ID_LEN) - return 0; - - b0 = buf[0]; - if ((b0 & 0x80) != 0) { - /* - * Long header. We need 6 bytes (initial byte, 4 version bytes, DCID - * length byte to begin with). This is covered by the buf_len test - * above. - */ - - /* - * If the version field is non-zero (meaning that this is not a Version - * Negotiation packet), the fixed bit must be set. - */ - if ((buf[1] || buf[2] || buf[3] || buf[4]) && (b0 & 0x40) == 0) - return 0; - - blen = (size_t)buf[5]; /* DCID Length */ - if (blen > QUIC_MAX_CONN_ID_LEN - || buf_len < QUIC_MIN_VALID_PKT_LEN + blen) - return 0; - - dst_conn_id->id_len = (unsigned char)blen; - memcpy(dst_conn_id->id, buf + 6, blen); - return 1; - } else { - /* Short header. */ - if ((b0 & 0x40) == 0) - /* Fixed bit not set, not a valid QUIC packet header. */ - return 0; - - if (buf_len < QUIC_MIN_VALID_PKT_LEN_CRYPTO + short_conn_id_len) - return 0; - - dst_conn_id->id_len = (unsigned char)short_conn_id_len; - memcpy(dst_conn_id->id, buf + 1, short_conn_id_len); - return 1; - } -} - -int ossl_quic_wire_decode_pkt_hdr_pn(const unsigned char *enc_pn, - size_t enc_pn_len, - QUIC_PN largest_pn, - QUIC_PN *res_pn) -{ - int64_t expected_pn, truncated_pn, candidate_pn, pn_win, pn_hwin, pn_mask; - - switch (enc_pn_len) { - case 1: - truncated_pn = enc_pn[0]; - break; - case 2: - truncated_pn = ((QUIC_PN)enc_pn[0] << 8) - | (QUIC_PN)enc_pn[1]; - break; - case 3: - truncated_pn = ((QUIC_PN)enc_pn[0] << 16) - | ((QUIC_PN)enc_pn[1] << 8) - | (QUIC_PN)enc_pn[2]; - break; - case 4: - truncated_pn = ((QUIC_PN)enc_pn[0] << 24) - | ((QUIC_PN)enc_pn[1] << 16) - | ((QUIC_PN)enc_pn[2] << 8) - | (QUIC_PN)enc_pn[3]; - break; - default: - return 0; - } - - /* Implemented as per RFC 9000 Section A.3. */ - expected_pn = largest_pn + 1; - pn_win = ((int64_t)1) << (enc_pn_len * 8); - pn_hwin = pn_win / 2; - pn_mask = pn_win - 1; - candidate_pn = (expected_pn & ~pn_mask) | truncated_pn; - if (candidate_pn <= expected_pn - pn_hwin - && candidate_pn < (((int64_t)1) << 62) - pn_win) - *res_pn = candidate_pn + pn_win; - else if (candidate_pn > expected_pn + pn_hwin - && candidate_pn >= pn_win) - *res_pn = candidate_pn - pn_win; - else - *res_pn = candidate_pn; - return 1; -} - -/* From RFC 9000 Section A.2. Simplified implementation. */ -int ossl_quic_wire_determine_pn_len(QUIC_PN pn, - QUIC_PN largest_acked) -{ - uint64_t num_unacked - = (largest_acked == QUIC_PN_INVALID) ? pn + 1 : pn - largest_acked; - - /* - * num_unacked \in [ 0, 2** 7] -> 1 byte - * num_unacked \in (2** 7, 2**15] -> 2 bytes - * num_unacked \in (2**15, 2**23] -> 3 bytes - * num_unacked \in (2**23, ] -> 4 bytes - */ - - if (num_unacked <= (1U<<7)) return 1; - if (num_unacked <= (1U<<15)) return 2; - if (num_unacked <= (1U<<23)) return 3; - return 4; -} - -int ossl_quic_wire_encode_pkt_hdr_pn(QUIC_PN pn, - unsigned char *enc_pn, - size_t enc_pn_len) -{ - switch (enc_pn_len) { - case 1: - enc_pn[0] = (unsigned char)pn; - break; - case 2: - enc_pn[1] = (unsigned char)pn; - enc_pn[0] = (unsigned char)(pn >> 8); - break; - case 3: - enc_pn[2] = (unsigned char)pn; - enc_pn[1] = (unsigned char)(pn >> 8); - enc_pn[0] = (unsigned char)(pn >> 16); - break; - case 4: - enc_pn[3] = (unsigned char)pn; - enc_pn[2] = (unsigned char)(pn >> 8); - enc_pn[1] = (unsigned char)(pn >> 16); - enc_pn[0] = (unsigned char)(pn >> 24); - break; - default: - return 0; - } - - return 1; -} - -int ossl_quic_validate_retry_integrity_tag(OSSL_LIB_CTX *libctx, - const char *propq, - const QUIC_PKT_HDR *hdr, - const QUIC_CONN_ID *client_initial_dcid) -{ - unsigned char expected_tag[QUIC_RETRY_INTEGRITY_TAG_LEN]; - const unsigned char *actual_tag; - - if (hdr == NULL || hdr->len < QUIC_RETRY_INTEGRITY_TAG_LEN) - return 0; - - if (!ossl_quic_calculate_retry_integrity_tag(libctx, propq, - hdr, client_initial_dcid, - expected_tag)) - return 0; - - actual_tag = hdr->data + hdr->len - QUIC_RETRY_INTEGRITY_TAG_LEN; - - return !CRYPTO_memcmp(expected_tag, actual_tag, - QUIC_RETRY_INTEGRITY_TAG_LEN); -} - -/* RFC 9001 s. 5.8 */ -static const unsigned char retry_integrity_key[] = { - 0xbe, 0x0c, 0x69, 0x0b, 0x9f, 0x66, 0x57, 0x5a, - 0x1d, 0x76, 0x6b, 0x54, 0xe3, 0x68, 0xc8, 0x4e -}; - -static const unsigned char retry_integrity_nonce[] = { - 0x46, 0x15, 0x99, 0xd3, 0x5d, 0x63, 0x2b, 0xf2, - 0x23, 0x98, 0x25, 0xbb -}; - -int ossl_quic_calculate_retry_integrity_tag(OSSL_LIB_CTX *libctx, - const char *propq, - const QUIC_PKT_HDR *hdr, - const QUIC_CONN_ID *client_initial_dcid, - unsigned char *tag) -{ - EVP_CIPHER *cipher = NULL; - EVP_CIPHER_CTX *cctx = NULL; - int ok = 0, l = 0, l2 = 0, wpkt_valid = 0; - WPACKET wpkt; - /* Worst case length of the Retry Psuedo-Packet header is 68 bytes. */ - unsigned char buf[128]; - QUIC_PKT_HDR hdr2; - size_t hdr_enc_len = 0; - - if (hdr->type != QUIC_PKT_TYPE_RETRY || hdr->version == 0 - || hdr->len < QUIC_RETRY_INTEGRITY_TAG_LEN - || hdr->data == NULL - || client_initial_dcid == NULL || tag == NULL - || client_initial_dcid->id_len > QUIC_MAX_CONN_ID_LEN) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - - /* - * Do not reserve packet body in WPACKET. Retry packet header - * does not contain a Length field so this does not affect - * the serialized packet header. - */ - hdr2 = *hdr; - hdr2.len = 0; - - /* Assemble retry psuedo-packet. */ - if (!WPACKET_init_static_len(&wpkt, buf, sizeof(buf), 0)) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); - goto err; - } - - wpkt_valid = 1; - - /* Prepend original DCID to the packet. */ - if (!WPACKET_put_bytes_u8(&wpkt, client_initial_dcid->id_len) - || !WPACKET_memcpy(&wpkt, client_initial_dcid->id, - client_initial_dcid->id_len)) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); - goto err; - } - - /* Encode main retry header. */ - if (!ossl_quic_wire_encode_pkt_hdr(&wpkt, hdr2.dst_conn_id.id_len, - &hdr2, NULL)) - goto err; - - if (!WPACKET_get_total_written(&wpkt, &hdr_enc_len)) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); - return 0; - } - - /* Create and initialise cipher context. */ - /* TODO(QUIC FUTURE): Cipher fetch caching. */ - if ((cipher = EVP_CIPHER_fetch(libctx, "AES-128-GCM", propq)) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - goto err; - } - - if ((cctx = EVP_CIPHER_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - goto err; - } - - if (!EVP_CipherInit_ex(cctx, cipher, NULL, - retry_integrity_key, retry_integrity_nonce, /*enc=*/1)) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - goto err; - } - - /* Feed packet header as AAD data. */ - if (EVP_CipherUpdate(cctx, NULL, &l, buf, hdr_enc_len) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - /* Feed packet body as AAD data. */ - if (EVP_CipherUpdate(cctx, NULL, &l, hdr->data, - hdr->len - QUIC_RETRY_INTEGRITY_TAG_LEN) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - /* Finalise and get tag. */ - if (EVP_CipherFinal_ex(cctx, NULL, &l2) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - if (EVP_CIPHER_CTX_ctrl(cctx, EVP_CTRL_AEAD_GET_TAG, - QUIC_RETRY_INTEGRITY_TAG_LEN, - tag) != 1) { - ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); - return 0; - } - - ok = 1; -err: - EVP_CIPHER_free(cipher); - EVP_CIPHER_CTX_free(cctx); - if (wpkt_valid) - WPACKET_finish(&wpkt); - - return ok; -} diff --git a/openssl/src/ssl/quic/uint_set.c b/openssl/src/ssl/quic/uint_set.c deleted file mode 100644 index faca90600..000000000 --- a/openssl/src/ssl/quic/uint_set.c +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/uint_set.h" -#include "internal/common.h" -#include - -/* - * uint64_t Integer Sets - * ===================== - * - * This data structure supports the following operations: - * - * Insert Range: Adds an inclusive range of integers [start, end] - * to the set. Equivalent to Insert for each number - * in the range. - * - * Remove Range: Removes an inclusive range of integers [start, end] - * from the set. Not all of the range need already be in - * the set, but any part of the range in the set is removed. - * - * Query: Is an integer in the data structure? - * - * The data structure can be iterated. - * - * For greater efficiency in tracking large numbers of contiguous integers, we - * track integer ranges rather than individual integers. The data structure - * manages a list of integer ranges [[start, end]...]. Internally this is - * implemented as a doubly linked sorted list of range structures, which are - * automatically split and merged as necessary. - * - * This data structure requires O(n) traversal of the list for insertion, - * removal and query when we are not adding/removing ranges which are near the - * beginning or end of the set of ranges. For the applications for which this - * data structure is used (e.g. QUIC PN tracking for ACK generation), it is - * expected that the number of integer ranges needed at any given time will - * generally be small and that most operations will be close to the beginning or - * end of the range. - * - * Invariant: The data structure is always sorted in ascending order by value. - * - * Invariant: No two adjacent ranges ever 'border' one another (have no - * numerical gap between them) as the data structure always ensures - * such ranges are merged. - * - * Invariant: No two ranges ever overlap. - * - * Invariant: No range [a, b] ever has a > b. - * - * Invariant: Since ranges are represented using inclusive bounds, no range - * item inside the data structure can represent a span of zero - * integers. - */ -void ossl_uint_set_init(UINT_SET *s) -{ - ossl_list_uint_set_init(s); -} - -void ossl_uint_set_destroy(UINT_SET *s) -{ - UINT_SET_ITEM *x, *xnext; - - for (x = ossl_list_uint_set_head(s); x != NULL; x = xnext) { - xnext = ossl_list_uint_set_next(x); - OPENSSL_free(x); - } -} - -/* Possible merge of x, prev(x) */ -static void uint_set_merge_adjacent(UINT_SET *s, UINT_SET_ITEM *x) -{ - UINT_SET_ITEM *xprev = ossl_list_uint_set_prev(x); - - if (xprev == NULL) - return; - - if (x->range.start - 1 != xprev->range.end) - return; - - x->range.start = xprev->range.start; - ossl_list_uint_set_remove(s, xprev); - OPENSSL_free(xprev); -} - -static uint64_t u64_min(uint64_t x, uint64_t y) -{ - return x < y ? x : y; -} - -static uint64_t u64_max(uint64_t x, uint64_t y) -{ - return x > y ? x : y; -} - -/* - * Returns 1 if there exists an integer x which falls within both ranges a and - * b. - */ -static int uint_range_overlaps(const UINT_RANGE *a, - const UINT_RANGE *b) -{ - return u64_min(a->end, b->end) - >= u64_max(a->start, b->start); -} - -static UINT_SET_ITEM *create_set_item(uint64_t start, uint64_t end) -{ - UINT_SET_ITEM *x = OPENSSL_malloc(sizeof(UINT_SET_ITEM)); - - if (x == NULL) - return NULL; - - ossl_list_uint_set_init_elem(x); - x->range.start = start; - x->range.end = end; - return x; -} - -int ossl_uint_set_insert(UINT_SET *s, const UINT_RANGE *range) -{ - UINT_SET_ITEM *x, *xnext, *z, *zprev, *f; - uint64_t start = range->start, end = range->end; - - if (!ossl_assert(start <= end)) - return 0; - - if (ossl_list_uint_set_is_empty(s)) { - /* Nothing in the set yet, so just add this range. */ - x = create_set_item(start, end); - if (x == NULL) - return 0; - ossl_list_uint_set_insert_head(s, x); - return 1; - } - - z = ossl_list_uint_set_tail(s); - if (start > z->range.end) { - /* - * Range is after the latest range in the set, so append. - * - * Note: The case where the range is before the earliest range in the - * set is handled as a degenerate case of the final case below. See - * optimization note (*) below. - */ - if (z->range.end + 1 == start) { - z->range.end = end; - return 1; - } - - x = create_set_item(start, end); - if (x == NULL) - return 0; - ossl_list_uint_set_insert_tail(s, x); - return 1; - } - - f = ossl_list_uint_set_head(s); - if (start <= f->range.start && end >= z->range.end) { - /* - * New range dwarfs all ranges in our set. - * - * Free everything except the first range in the set, which we scavenge - * and reuse. - */ - x = ossl_list_uint_set_head(s); - x->range.start = start; - x->range.end = end; - for (x = ossl_list_uint_set_next(x); x != NULL; x = xnext) { - xnext = ossl_list_uint_set_next(x); - ossl_list_uint_set_remove(s, x); - } - return 1; - } - - /* - * Walk backwards since we will most often be inserting at the end. As an - * optimization, test the head node first and skip iterating over the - * entire list if we are inserting at the start. The assumption is that - * insertion at the start and end of the space will be the most common - * operations. (*) - */ - z = end < f->range.start ? f : z; - - for (; z != NULL; z = zprev) { - zprev = ossl_list_uint_set_prev(z); - - /* An existing range dwarfs our new range (optimisation). */ - if (z->range.start <= start && z->range.end >= end) - return 1; - - if (uint_range_overlaps(&z->range, range)) { - /* - * Our new range overlaps an existing range, or possibly several - * existing ranges. - */ - UINT_SET_ITEM *ovend = z; - - ovend->range.end = u64_max(end, z->range.end); - - /* Get earliest overlapping range. */ - while (zprev != NULL && uint_range_overlaps(&zprev->range, range)) { - z = zprev; - zprev = ossl_list_uint_set_prev(z); - } - - ovend->range.start = u64_min(start, z->range.start); - - /* Replace sequence of nodes z..ovend with updated ovend only. */ - while (z != ovend) { - z = ossl_list_uint_set_next(x = z); - ossl_list_uint_set_remove(s, x); - OPENSSL_free(x); - } - break; - } else if (end < z->range.start - && (zprev == NULL || start > zprev->range.end)) { - if (z->range.start == end + 1) { - /* We can extend the following range backwards. */ - z->range.start = start; - - /* - * If this closes a gap we now need to merge - * consecutive nodes. - */ - uint_set_merge_adjacent(s, z); - } else if (zprev != NULL && zprev->range.end + 1 == start) { - /* We can extend the preceding range forwards. */ - zprev->range.end = end; - - /* - * If this closes a gap we now need to merge - * consecutive nodes. - */ - uint_set_merge_adjacent(s, z); - } else { - /* - * The new interval is between intervals without overlapping or - * touching them, so insert between, preserving sort. - */ - x = create_set_item(start, end); - if (x == NULL) - return 0; - ossl_list_uint_set_insert_before(s, z, x); - } - break; - } - } - - return 1; -} - -int ossl_uint_set_remove(UINT_SET *s, const UINT_RANGE *range) -{ - UINT_SET_ITEM *z, *zprev, *y; - uint64_t start = range->start, end = range->end; - - if (!ossl_assert(start <= end)) - return 0; - - /* Walk backwards since we will most often be removing at the end. */ - for (z = ossl_list_uint_set_tail(s); z != NULL; z = zprev) { - zprev = ossl_list_uint_set_prev(z); - - if (start > z->range.end) - /* No overlapping ranges can exist beyond this point, so stop. */ - break; - - if (start <= z->range.start && end >= z->range.end) { - /* - * The range being removed dwarfs this range, so it should be - * removed. - */ - ossl_list_uint_set_remove(s, z); - OPENSSL_free(z); - } else if (start <= z->range.start && end >= z->range.start) { - /* - * The range being removed includes start of this range, but does - * not cover the entire range (as this would be caught by the case - * above). Shorten the range. - */ - assert(end < z->range.end); - z->range.start = end + 1; - } else if (end >= z->range.end) { - /* - * The range being removed includes the end of this range, but does - * not cover the entire range (as this would be caught by the case - * above). Shorten the range. We can also stop iterating. - */ - assert(start > z->range.start); - assert(start > 0); - z->range.end = start - 1; - break; - } else if (start > z->range.start && end < z->range.end) { - /* - * The range being removed falls entirely in this range, so cut it - * into two. Cases where a zero-length range would be created are - * handled by the above cases. - */ - y = create_set_item(end + 1, z->range.end); - ossl_list_uint_set_insert_after(s, z, y); - z->range.end = start - 1; - break; - } else { - /* Assert no partial overlap; all cases should be covered above. */ - assert(!uint_range_overlaps(&z->range, range)); - } - } - - return 1; -} - -int ossl_uint_set_query(const UINT_SET *s, uint64_t v) -{ - UINT_SET_ITEM *x; - - if (ossl_list_uint_set_is_empty(s)) - return 0; - - for (x = ossl_list_uint_set_tail(s); x != NULL; x = ossl_list_uint_set_prev(x)) - if (x->range.start <= v && x->range.end >= v) - return 1; - else if (x->range.end < v) - return 0; - - return 0; -} diff --git a/openssl/src/ssl/record/dtls1_bitmap.c b/openssl/src/ssl/record/dtls1_bitmap.c new file mode 100644 index 000000000..4733a62a9 --- /dev/null +++ b/openssl/src/ssl/record/dtls1_bitmap.c @@ -0,0 +1,78 @@ +/* + * Copyright 2005-2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../ssl_local.h" +#include "record_local.h" + +/* mod 128 saturating subtract of two 64-bit values in big-endian order */ +static int satsub64be(const unsigned char *v1, const unsigned char *v2) +{ + int64_t ret; + uint64_t l1, l2; + + n2l8(v1, l1); + n2l8(v2, l2); + + ret = l1 - l2; + + /* We do not permit wrap-around */ + if (l1 > l2 && ret < 0) + return 128; + else if (l2 > l1 && ret > 0) + return -128; + + if (ret > 128) + return 128; + else if (ret < -128) + return -128; + else + return (int)ret; +} + +int dtls1_record_replay_check(SSL *s, DTLS1_BITMAP *bitmap) +{ + int cmp; + unsigned int shift; + const unsigned char *seq = s->rlayer.read_sequence; + + cmp = satsub64be(seq, bitmap->max_seq_num); + if (cmp > 0) { + SSL3_RECORD_set_seq_num(RECORD_LAYER_get_rrec(&s->rlayer), seq); + return 1; /* this record in new */ + } + shift = -cmp; + if (shift >= sizeof(bitmap->map) * 8) + return 0; /* stale, outside the window */ + else if (bitmap->map & (1UL << shift)) + return 0; /* record previously received */ + + SSL3_RECORD_set_seq_num(RECORD_LAYER_get_rrec(&s->rlayer), seq); + return 1; +} + +void dtls1_record_bitmap_update(SSL *s, DTLS1_BITMAP *bitmap) +{ + int cmp; + unsigned int shift; + const unsigned char *seq = RECORD_LAYER_get_read_sequence(&s->rlayer); + + cmp = satsub64be(seq, bitmap->max_seq_num); + if (cmp > 0) { + shift = cmp; + if (shift < sizeof(bitmap->map) * 8) + bitmap->map <<= shift, bitmap->map |= 1UL; + else + bitmap->map = 1UL; + memcpy(bitmap->max_seq_num, seq, SEQ_NUM_SIZE); + } else { + shift = -cmp; + if (shift < sizeof(bitmap->map) * 8) + bitmap->map |= 1UL << shift; + } +} diff --git a/openssl/src/ssl/record/methods/dtls_meth.c b/openssl/src/ssl/record/methods/dtls_meth.c deleted file mode 100644 index a5e6c8234..000000000 --- a/openssl/src/ssl/record/methods/dtls_meth.c +++ /dev/null @@ -1,797 +0,0 @@ -/* - * Copyright 2018-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "../../ssl_local.h" -#include "../record_local.h" -#include "recmethod_local.h" - -/* mod 128 saturating subtract of two 64-bit values in big-endian order */ -static int satsub64be(const unsigned char *v1, const unsigned char *v2) -{ - int64_t ret; - uint64_t l1, l2; - - n2l8(v1, l1); - n2l8(v2, l2); - - ret = l1 - l2; - - /* We do not permit wrap-around */ - if (l1 > l2 && ret < 0) - return 128; - else if (l2 > l1 && ret > 0) - return -128; - - if (ret > 128) - return 128; - else if (ret < -128) - return -128; - else - return (int)ret; -} - -static int dtls_record_replay_check(OSSL_RECORD_LAYER *rl, DTLS_BITMAP *bitmap) -{ - int cmp; - unsigned int shift; - const unsigned char *seq = rl->sequence; - - cmp = satsub64be(seq, bitmap->max_seq_num); - if (cmp > 0) { - ossl_tls_rl_record_set_seq_num(&rl->rrec[0], seq); - return 1; /* this record in new */ - } - shift = -cmp; - if (shift >= sizeof(bitmap->map) * 8) - return 0; /* stale, outside the window */ - else if (bitmap->map & ((uint64_t)1 << shift)) - return 0; /* record previously received */ - - ossl_tls_rl_record_set_seq_num(&rl->rrec[0], seq); - return 1; -} - -static void dtls_record_bitmap_update(OSSL_RECORD_LAYER *rl, - DTLS_BITMAP *bitmap) -{ - int cmp; - unsigned int shift; - const unsigned char *seq = rl->sequence; - - cmp = satsub64be(seq, bitmap->max_seq_num); - if (cmp > 0) { - shift = cmp; - if (shift < sizeof(bitmap->map) * 8) - bitmap->map <<= shift, bitmap->map |= 1UL; - else - bitmap->map = 1UL; - memcpy(bitmap->max_seq_num, seq, SEQ_NUM_SIZE); - } else { - shift = -cmp; - if (shift < sizeof(bitmap->map) * 8) - bitmap->map |= (uint64_t)1 << shift; - } -} - -static DTLS_BITMAP *dtls_get_bitmap(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rr, - unsigned int *is_next_epoch) -{ - *is_next_epoch = 0; - - /* In current epoch, accept HM, CCS, DATA, & ALERT */ - if (rr->epoch == rl->epoch) - return &rl->bitmap; - - /* - * Check if the message is from the next epoch - */ - else if (rr->epoch == rl->epoch + 1) { - *is_next_epoch = 1; - return &rl->next_bitmap; - } - - return NULL; -} - -static void dtls_set_in_init(OSSL_RECORD_LAYER *rl, int in_init) -{ - rl->in_init = in_init; -} - -static int dtls_process_record(OSSL_RECORD_LAYER *rl, DTLS_BITMAP *bitmap) -{ - int i; - int enc_err; - TLS_RL_RECORD *rr; - int imac_size; - size_t mac_size = 0; - unsigned char md[EVP_MAX_MD_SIZE]; - SSL_MAC_BUF macbuf = { NULL, 0 }; - int ret = 0; - - rr = &rl->rrec[0]; - - /* - * At this point, rl->packet_length == DTLS1_RT_HEADER_LENGTH + rr->length, - * and we have that many bytes in rl->packet - */ - rr->input = &(rl->packet[DTLS1_RT_HEADER_LENGTH]); - - /* - * ok, we can now read from 'rl->packet' data into 'rr'. rr->input - * points at rr->length bytes, which need to be copied into rr->data by - * either the decryption or by the decompression. When the data is 'copied' - * into the rr->data buffer, rr->input will be pointed at the new buffer - */ - - /* - * We now have - encrypted [ MAC [ compressed [ plain ] ] ] rr->length - * bytes of encrypted compressed stuff. - */ - - /* check is not needed I believe */ - if (rr->length > SSL3_RT_MAX_ENCRYPTED_LENGTH) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, SSL_R_ENCRYPTED_LENGTH_TOO_LONG); - return 0; - } - - /* decrypt in place in 'rr->input' */ - rr->data = rr->input; - rr->orig_len = rr->length; - - if (rl->md_ctx != NULL) { - const EVP_MD *tmpmd = EVP_MD_CTX_get0_md(rl->md_ctx); - - if (tmpmd != NULL) { - imac_size = EVP_MD_get_size(tmpmd); - if (!ossl_assert(imac_size >= 0 && imac_size <= EVP_MAX_MD_SIZE)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - return 0; - } - mac_size = (size_t)imac_size; - } - } - - if (rl->use_etm && rl->md_ctx != NULL) { - unsigned char *mac; - - if (rr->orig_len < mac_size) { - RLAYERfatal(rl, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); - return 0; - } - rr->length -= mac_size; - mac = rr->data + rr->length; - i = rl->funcs->mac(rl, rr, md, 0 /* not send */); - if (i == 0 || CRYPTO_memcmp(md, mac, (size_t)mac_size) != 0) { - RLAYERfatal(rl, SSL_AD_BAD_RECORD_MAC, - SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); - return 0; - } - /* - * We've handled the mac now - there is no MAC inside the encrypted - * record - */ - mac_size = 0; - } - - /* - * Set a mark around the packet decryption attempt. This is DTLS, so - * bad packets are just ignored, and we don't want to leave stray - * errors in the queue from processing bogus junk that we ignored. - */ - ERR_set_mark(); - enc_err = rl->funcs->cipher(rl, rr, 1, 0, &macbuf, mac_size); - - /*- - * enc_err is: - * 0: if the record is publicly invalid, or an internal error, or AEAD - * decryption failed, or ETM decryption failed. - * 1: Success or MTE decryption failed (MAC will be randomised) - */ - if (enc_err == 0) { - ERR_pop_to_mark(); - if (rl->alert != SSL_AD_NO_ALERT) { - /* RLAYERfatal() already called */ - goto end; - } - /* For DTLS we simply ignore bad packets. */ - rr->length = 0; - rl->packet_length = 0; - goto end; - } - ERR_clear_last_mark(); - OSSL_TRACE_BEGIN(TLS) { - BIO_printf(trc_out, "dec %zd\n", rr->length); - BIO_dump_indent(trc_out, rr->data, rr->length, 4); - } OSSL_TRACE_END(TLS); - - /* r->length is now the compressed data plus mac */ - if (!rl->use_etm - && (rl->enc_ctx != NULL) - && (EVP_MD_CTX_get0_md(rl->md_ctx) != NULL)) { - /* rl->md_ctx != NULL => mac_size != -1 */ - - i = rl->funcs->mac(rl, rr, md, 0 /* not send */); - if (i == 0 || macbuf.mac == NULL - || CRYPTO_memcmp(md, macbuf.mac, mac_size) != 0) - enc_err = 0; - if (rr->length > SSL3_RT_MAX_COMPRESSED_LENGTH + mac_size) - enc_err = 0; - } - - if (enc_err == 0) { - /* decryption failed, silently discard message */ - rr->length = 0; - rl->packet_length = 0; - goto end; - } - - /* r->length is now just compressed */ - if (rl->compctx != NULL) { - if (rr->length > SSL3_RT_MAX_COMPRESSED_LENGTH) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, - SSL_R_COMPRESSED_LENGTH_TOO_LONG); - goto end; - } - if (!tls_do_uncompress(rl, rr)) { - RLAYERfatal(rl, SSL_AD_DECOMPRESSION_FAILURE, SSL_R_BAD_DECOMPRESSION); - goto end; - } - } - - /* - * Check if the received packet overflows the current Max Fragment - * Length setting. - */ - if (rr->length > rl->max_frag_len) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, SSL_R_DATA_LENGTH_TOO_LONG); - goto end; - } - - rr->off = 0; - /*- - * So at this point the following is true - * ssl->s3.rrec.type is the type of record - * ssl->s3.rrec.length == number of bytes in record - * ssl->s3.rrec.off == offset to first valid byte - * ssl->s3.rrec.data == where to take bytes from, increment - * after use :-). - */ - - /* we have pulled in a full packet so zero things */ - rl->packet_length = 0; - - /* Mark receipt of record. */ - dtls_record_bitmap_update(rl, bitmap); - - ret = 1; - end: - if (macbuf.alloced) - OPENSSL_free(macbuf.mac); - return ret; -} - -static int dtls_rlayer_buffer_record(OSSL_RECORD_LAYER *rl, struct pqueue_st *queue, - unsigned char *priority) -{ - DTLS_RLAYER_RECORD_DATA *rdata; - pitem *item; - - /* Limit the size of the queue to prevent DOS attacks */ - if (pqueue_size(queue) >= 100) - return 0; - - rdata = OPENSSL_malloc(sizeof(*rdata)); - item = pitem_new(priority, rdata); - if (rdata == NULL || item == NULL) { - OPENSSL_free(rdata); - pitem_free(item); - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return -1; - } - - rdata->packet = rl->packet; - rdata->packet_length = rl->packet_length; - memcpy(&(rdata->rbuf), &rl->rbuf, sizeof(TLS_BUFFER)); - memcpy(&(rdata->rrec), &rl->rrec[0], sizeof(TLS_RL_RECORD)); - - item->data = rdata; - - rl->packet = NULL; - rl->packet_length = 0; - memset(&rl->rbuf, 0, sizeof(TLS_BUFFER)); - memset(&rl->rrec[0], 0, sizeof(rl->rrec[0])); - - if (!tls_setup_read_buffer(rl)) { - /* RLAYERfatal() already called */ - OPENSSL_free(rdata->rbuf.buf); - OPENSSL_free(rdata); - pitem_free(item); - return -1; - } - - if (pqueue_insert(queue, item) == NULL) { - /* Must be a duplicate so ignore it */ - OPENSSL_free(rdata->rbuf.buf); - OPENSSL_free(rdata); - pitem_free(item); - } - - return 1; -} - -/* copy buffered record into OSSL_RECORD_LAYER structure */ -static int dtls_copy_rlayer_record(OSSL_RECORD_LAYER *rl, pitem *item) -{ - DTLS_RLAYER_RECORD_DATA *rdata; - - rdata = (DTLS_RLAYER_RECORD_DATA *)item->data; - - ossl_tls_buffer_release(&rl->rbuf); - - rl->packet = rdata->packet; - rl->packet_length = rdata->packet_length; - memcpy(&rl->rbuf, &(rdata->rbuf), sizeof(TLS_BUFFER)); - memcpy(&rl->rrec[0], &(rdata->rrec), sizeof(TLS_RL_RECORD)); - - /* Set proper sequence number for mac calculation */ - memcpy(&(rl->sequence[2]), &(rdata->packet[5]), 6); - - return 1; -} - -static int dtls_retrieve_rlayer_buffered_record(OSSL_RECORD_LAYER *rl, - struct pqueue_st *queue) -{ - pitem *item; - - item = pqueue_pop(queue); - if (item) { - dtls_copy_rlayer_record(rl, item); - - OPENSSL_free(item->data); - pitem_free(item); - - return 1; - } - - return 0; -} - -/*- - * Call this to get a new input record. - * It will return <= 0 if more data is needed, normally due to an error - * or non-blocking IO. - * When it finishes, one packet has been decoded and can be found in - * ssl->s3.rrec.type - is the type of record - * ssl->s3.rrec.data - data - * ssl->s3.rrec.length - number of bytes - */ -int dtls_get_more_records(OSSL_RECORD_LAYER *rl) -{ - int ssl_major, ssl_minor; - int rret; - size_t more, n; - TLS_RL_RECORD *rr; - unsigned char *p = NULL; - DTLS_BITMAP *bitmap; - unsigned int is_next_epoch; - - rl->num_recs = 0; - rl->curr_rec = 0; - rl->num_released = 0; - - rr = rl->rrec; - - if (rl->rbuf.buf == NULL) { - if (!tls_setup_read_buffer(rl)) { - /* RLAYERfatal() already called */ - return OSSL_RECORD_RETURN_FATAL; - } - } - - again: - /* if we're renegotiating, then there may be buffered records */ - if (dtls_retrieve_rlayer_buffered_record(rl, rl->processed_rcds)) { - rl->num_recs = 1; - return OSSL_RECORD_RETURN_SUCCESS; - } - - /* get something from the wire */ - - /* check if we have the header */ - if ((rl->rstate != SSL_ST_READ_BODY) || - (rl->packet_length < DTLS1_RT_HEADER_LENGTH)) { - rret = rl->funcs->read_n(rl, DTLS1_RT_HEADER_LENGTH, - TLS_BUFFER_get_len(&rl->rbuf), 0, 1, &n); - /* read timeout is handled by dtls1_read_bytes */ - if (rret < OSSL_RECORD_RETURN_SUCCESS) { - /* RLAYERfatal() already called if appropriate */ - return rret; /* error or non-blocking */ - } - - /* this packet contained a partial record, dump it */ - if (rl->packet_length != DTLS1_RT_HEADER_LENGTH) { - rl->packet_length = 0; - goto again; - } - - rl->rstate = SSL_ST_READ_BODY; - - p = rl->packet; - - /* Pull apart the header into the DTLS1_RECORD */ - rr->type = *(p++); - ssl_major = *(p++); - ssl_minor = *(p++); - rr->rec_version = (ssl_major << 8) | ssl_minor; - - /* sequence number is 64 bits, with top 2 bytes = epoch */ - n2s(p, rr->epoch); - - memcpy(&(rl->sequence[2]), p, 6); - p += 6; - - n2s(p, rr->length); - - if (rl->msg_callback != NULL) - rl->msg_callback(0, rr->rec_version, SSL3_RT_HEADER, rl->packet, DTLS1_RT_HEADER_LENGTH, - rl->cbarg); - - /* - * Lets check the version. We tolerate alerts that don't have the exact - * version number (e.g. because of protocol version errors) - */ - if (!rl->is_first_record && rr->type != SSL3_RT_ALERT) { - if (rr->rec_version != rl->version) { - /* unexpected version, silently discard */ - rr->length = 0; - rl->packet_length = 0; - goto again; - } - } - - if (ssl_major != - (rl->version == DTLS_ANY_VERSION ? DTLS1_VERSION_MAJOR - : rl->version >> 8)) { - /* wrong version, silently discard record */ - rr->length = 0; - rl->packet_length = 0; - goto again; - } - - if (rr->length > SSL3_RT_MAX_ENCRYPTED_LENGTH) { - /* record too long, silently discard it */ - rr->length = 0; - rl->packet_length = 0; - goto again; - } - - /* - * If received packet overflows maximum possible fragment length then - * silently discard it - */ - if (rr->length > rl->max_frag_len + SSL3_RT_MAX_ENCRYPTED_OVERHEAD) { - /* record too long, silently discard it */ - rr->length = 0; - rl->packet_length = 0; - goto again; - } - - /* now rl->rstate == SSL_ST_READ_BODY */ - } - - /* rl->rstate == SSL_ST_READ_BODY, get and decode the data */ - - if (rr->length > rl->packet_length - DTLS1_RT_HEADER_LENGTH) { - /* now rl->packet_length == DTLS1_RT_HEADER_LENGTH */ - more = rr->length; - rret = rl->funcs->read_n(rl, more, more, 1, 1, &n); - /* this packet contained a partial record, dump it */ - if (rret < OSSL_RECORD_RETURN_SUCCESS || n != more) { - if (rl->alert != SSL_AD_NO_ALERT) { - /* read_n() called RLAYERfatal() */ - return OSSL_RECORD_RETURN_FATAL; - } - rr->length = 0; - rl->packet_length = 0; - goto again; - } - - /* - * now n == rr->length, - * and rl->packet_length == DTLS1_RT_HEADER_LENGTH + rr->length - */ - } - /* set state for later operations */ - rl->rstate = SSL_ST_READ_HEADER; - - /* match epochs. NULL means the packet is dropped on the floor */ - bitmap = dtls_get_bitmap(rl, rr, &is_next_epoch); - if (bitmap == NULL) { - rr->length = 0; - rl->packet_length = 0; /* dump this record */ - goto again; /* get another record */ - } -#ifndef OPENSSL_NO_SCTP - /* Only do replay check if no SCTP bio */ - if (!BIO_dgram_is_sctp(rl->bio)) { -#endif - /* Check whether this is a repeat, or aged record. */ - if (!dtls_record_replay_check(rl, bitmap)) { - rr->length = 0; - rl->packet_length = 0; /* dump this record */ - goto again; /* get another record */ - } -#ifndef OPENSSL_NO_SCTP - } -#endif - - /* just read a 0 length packet */ - if (rr->length == 0) - goto again; - - /* - * If this record is from the next epoch (either HM or ALERT), and a - * handshake is currently in progress, buffer it since it cannot be - * processed at this time. - */ - if (is_next_epoch) { - if (rl->in_init) { - if (dtls_rlayer_buffer_record(rl, rl->unprocessed_rcds, - rr->seq_num) < 0) { - /* RLAYERfatal() already called */ - return OSSL_RECORD_RETURN_FATAL; - } - } - rr->length = 0; - rl->packet_length = 0; - goto again; - } - - if (!dtls_process_record(rl, bitmap)) { - if (rl->alert != SSL_AD_NO_ALERT) { - /* dtls_process_record() called RLAYERfatal */ - return OSSL_RECORD_RETURN_FATAL; - } - rr->length = 0; - rl->packet_length = 0; /* dump this record */ - goto again; /* get another record */ - } - - if (rl->funcs->post_process_record && !rl->funcs->post_process_record(rl, rr)) { - /* RLAYERfatal already called */ - return OSSL_RECORD_RETURN_FATAL; - } - - rl->num_recs = 1; - return OSSL_RECORD_RETURN_SUCCESS; -} - -static int dtls_free(OSSL_RECORD_LAYER *rl) -{ - TLS_BUFFER *rbuf; - size_t left, written; - pitem *item; - DTLS_RLAYER_RECORD_DATA *rdata; - int ret = 1; - - rbuf = &rl->rbuf; - - left = rbuf->left; - if (left > 0) { - /* - * This record layer is closing but we still have data left in our - * buffer. It must be destined for the next epoch - so push it there. - */ - ret = BIO_write_ex(rl->next, rbuf->buf + rbuf->offset, left, &written); - rbuf->left = 0; - } - - if (rl->unprocessed_rcds != NULL) { - while ((item = pqueue_pop(rl->unprocessed_rcds)) != NULL) { - rdata = (DTLS_RLAYER_RECORD_DATA *)item->data; - /* Push to the next record layer */ - ret &= BIO_write_ex(rl->next, rdata->packet, rdata->packet_length, - &written); - OPENSSL_free(rdata->rbuf.buf); - OPENSSL_free(item->data); - pitem_free(item); - } - pqueue_free(rl->unprocessed_rcds); - } - - if (rl->processed_rcds!= NULL) { - while ((item = pqueue_pop(rl->processed_rcds)) != NULL) { - rdata = (DTLS_RLAYER_RECORD_DATA *)item->data; - OPENSSL_free(rdata->rbuf.buf); - OPENSSL_free(item->data); - pitem_free(item); - } - pqueue_free(rl->processed_rcds); - } - - return tls_free(rl) && ret; -} - -static int -dtls_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers, - int role, int direction, int level, uint16_t epoch, - unsigned char *secret, size_t secretlen, - unsigned char *key, size_t keylen, unsigned char *iv, - size_t ivlen, unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, size_t taglen, - int mactype, - const EVP_MD *md, COMP_METHOD *comp, - const EVP_MD *kdfdigest, BIO *prev, BIO *transport, - BIO *next, BIO_ADDR *local, BIO_ADDR *peer, - const OSSL_PARAM *settings, const OSSL_PARAM *options, - const OSSL_DISPATCH *fns, void *cbarg, void *rlarg, - OSSL_RECORD_LAYER **retrl) -{ - int ret; - - ret = tls_int_new_record_layer(libctx, propq, vers, role, direction, level, - ciph, taglen, md, comp, prev, - transport, next, settings, - options, fns, cbarg, retrl); - - if (ret != OSSL_RECORD_RETURN_SUCCESS) - return ret; - - (*retrl)->unprocessed_rcds = pqueue_new(); - (*retrl)->processed_rcds = pqueue_new(); - - if ((*retrl)->unprocessed_rcds == NULL - || (*retrl)->processed_rcds == NULL) { - dtls_free(*retrl); - *retrl = NULL; - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); - return OSSL_RECORD_RETURN_FATAL; - } - - (*retrl)->isdtls = 1; - (*retrl)->epoch = epoch; - (*retrl)->in_init = 1; - - switch (vers) { - case DTLS_ANY_VERSION: - (*retrl)->funcs = &dtls_any_funcs; - break; - case DTLS1_2_VERSION: - case DTLS1_VERSION: - case DTLS1_BAD_VER: - (*retrl)->funcs = &dtls_1_funcs; - break; - default: - /* Should not happen */ - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - ret = OSSL_RECORD_RETURN_FATAL; - goto err; - } - - ret = (*retrl)->funcs->set_crypto_state(*retrl, level, key, keylen, iv, - ivlen, mackey, mackeylen, ciph, - taglen, mactype, md, comp); - - err: - if (ret != OSSL_RECORD_RETURN_SUCCESS) { - dtls_free(*retrl); - *retrl = NULL; - } - return ret; -} - -int dtls_prepare_record_header(OSSL_RECORD_LAYER *rl, - WPACKET *thispkt, - OSSL_RECORD_TEMPLATE *templ, - uint8_t rectype, - unsigned char **recdata) -{ - size_t maxcomplen; - - *recdata = NULL; - - maxcomplen = templ->buflen; - if (rl->compctx != NULL) - maxcomplen += SSL3_RT_MAX_COMPRESSED_OVERHEAD; - - if (!WPACKET_put_bytes_u8(thispkt, rectype) - || !WPACKET_put_bytes_u16(thispkt, templ->version) - || !WPACKET_put_bytes_u16(thispkt, rl->epoch) - || !WPACKET_memcpy(thispkt, &(rl->sequence[2]), 6) - || !WPACKET_start_sub_packet_u16(thispkt) - || (rl->eivlen > 0 - && !WPACKET_allocate_bytes(thispkt, rl->eivlen, NULL)) - || (maxcomplen > 0 - && !WPACKET_reserve_bytes(thispkt, maxcomplen, - recdata))) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - return 1; -} - -int dtls_post_encryption_processing(OSSL_RECORD_LAYER *rl, - size_t mac_size, - OSSL_RECORD_TEMPLATE *thistempl, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr) -{ - if (!tls_post_encryption_processing_default(rl, mac_size, thistempl, - thispkt, thiswr)) { - /* RLAYERfatal() already called */ - return 0; - } - - return tls_increment_sequence_ctr(rl); -} - -static size_t dtls_get_max_record_overhead(OSSL_RECORD_LAYER *rl) -{ - size_t blocksize = 0; - - if (rl->enc_ctx != NULL && - (EVP_CIPHER_CTX_get_mode(rl->enc_ctx) == EVP_CIPH_CBC_MODE)) - blocksize = EVP_CIPHER_CTX_get_block_size(rl->enc_ctx); - - /* - * If we have a cipher in place then the tag is mandatory. If the cipher is - * CBC mode then an explicit IV is also mandatory. If we know the digest, - * then we check it is consistent with the taglen. In the case of stitched - * ciphers or AEAD ciphers we don't now the digest (or there isn't one) so - * we just trust that the taglen is correct. - */ - assert(rl->enc_ctx == NULL || ((blocksize == 0 || rl->eivlen > 0) - && rl->taglen > 0)); - assert(rl->md == NULL || (int)rl->taglen == EVP_MD_size(rl->md)); - - /* - * Record overhead consists of the record header, the explicit IV, any - * expansion due to cbc padding, and the mac/tag len. There could be - * further expansion due to compression - but we don't know what this will - * be without knowing the length of the data. However when this function is - * called we don't know what the length will be yet - so this is a catch-22. - * We *could* use SSL_3_RT_MAX_COMPRESSED_OVERHEAD which is an upper limit - * for the maximum record size. But this value is larger than our fallback - * MTU size - so isn't very helpful. We just ignore potential expansion - * due to compression. - */ - return DTLS1_RT_HEADER_LENGTH + rl->eivlen + blocksize + rl->taglen; -} - -const OSSL_RECORD_METHOD ossl_dtls_record_method = { - dtls_new_record_layer, - dtls_free, - tls_unprocessed_read_pending, - tls_processed_read_pending, - tls_app_data_pending, - tls_get_max_records, - tls_write_records, - tls_retry_write_records, - tls_read_record, - tls_release_record, - tls_get_alert_code, - tls_set1_bio, - tls_set_protocol_version, - NULL, - tls_set_first_handshake, - tls_set_max_pipelines, - dtls_set_in_init, - tls_get_state, - tls_set_options, - tls_get_compression, - tls_set_max_frag_len, - dtls_get_max_record_overhead, - tls_increment_sequence_ctr, - tls_alloc_buffers, - tls_free_buffers -}; diff --git a/openssl/src/ssl/record/methods/ktls_meth.c b/openssl/src/ssl/record/methods/ktls_meth.c deleted file mode 100644 index 33c7140e1..000000000 --- a/openssl/src/ssl/record/methods/ktls_meth.c +++ /dev/null @@ -1,610 +0,0 @@ -/* - * Copyright 2018-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include "../../ssl_local.h" -#include "../record_local.h" -#include "recmethod_local.h" -#include "internal/ktls.h" - -static struct record_functions_st ossl_ktls_funcs; - -#if defined(__FreeBSD__) -# include "crypto/cryptodev.h" - -/*- - * Check if a given cipher is supported by the KTLS interface. - * The kernel might still fail the setsockopt() if no suitable - * provider is found, but this checks if the socket option - * supports the cipher suite used at all. - */ -static int ktls_int_check_supported_cipher(OSSL_RECORD_LAYER *rl, - const EVP_CIPHER *c, - const EVP_MD *md, - size_t taglen) -{ - switch (rl->version) { - case TLS1_VERSION: - case TLS1_1_VERSION: - case TLS1_2_VERSION: -#ifdef OPENSSL_KTLS_TLS13 - case TLS1_3_VERSION: -#endif - break; - default: - return 0; - } - - if (EVP_CIPHER_is_a(c, "AES-128-GCM") - || EVP_CIPHER_is_a(c, "AES-256-GCM") -# ifdef OPENSSL_KTLS_CHACHA20_POLY1305 - || EVP_CIPHER_is_a(c, "CHACHA20-POLY1305") -# endif - ) - return 1; - - if (!EVP_CIPHER_is_a(c, "AES-128-CBC") - && !EVP_CIPHER_is_a(c, "AES-256-CBC")) - return 0; - - if (rl->use_etm) - return 0; - - if (md == NULL) - return 0; - - if (EVP_MD_is_a(md, "SHA1") - || EVP_MD_is_a(md, "SHA2-256") - || EVP_MD_is_a(md, "SHA2-384")) - return 1; - - return 0; -} - -/* Function to configure kernel TLS structure */ -static -int ktls_configure_crypto(OSSL_LIB_CTX *libctx, int version, const EVP_CIPHER *c, - EVP_MD *md, void *rl_sequence, - ktls_crypto_info_t *crypto_info, int is_tx, - unsigned char *iv, size_t ivlen, - unsigned char *key, size_t keylen, - unsigned char *mac_key, size_t mac_secret_size) -{ - memset(crypto_info, 0, sizeof(*crypto_info)); - if (EVP_CIPHER_is_a(c, "AES-128-GCM") - || EVP_CIPHER_is_a(c, "AES-256-GCM")) { - crypto_info->cipher_algorithm = CRYPTO_AES_NIST_GCM_16; - crypto_info->iv_len = ivlen; - } else -# ifdef OPENSSL_KTLS_CHACHA20_POLY1305 - if (EVP_CIPHER_is_a(c, "CHACHA20-POLY1305")) { - crypto_info->cipher_algorithm = CRYPTO_CHACHA20_POLY1305; - crypto_info->iv_len = ivlen; - } else -# endif - if (EVP_CIPHER_is_a(c, "AES-128-CBC") || EVP_CIPHER_is_a(c, "AES-256-CBC")) { - if (md == NULL) - return 0; - if (EVP_MD_is_a(md, "SHA1")) - crypto_info->auth_algorithm = CRYPTO_SHA1_HMAC; - else if (EVP_MD_is_a(md, "SHA2-256")) - crypto_info->auth_algorithm = CRYPTO_SHA2_256_HMAC; - else if (EVP_MD_is_a(md, "SHA2-384")) - crypto_info->auth_algorithm = CRYPTO_SHA2_384_HMAC; - else - return 0; - crypto_info->cipher_algorithm = CRYPTO_AES_CBC; - crypto_info->iv_len = ivlen; - crypto_info->auth_key = mac_key; - crypto_info->auth_key_len = mac_secret_size; - } else { - return 0; - } - crypto_info->cipher_key = key; - crypto_info->cipher_key_len = keylen; - crypto_info->iv = iv; - crypto_info->tls_vmajor = (version >> 8) & 0x000000ff; - crypto_info->tls_vminor = (version & 0x000000ff); -# ifdef TCP_RXTLS_ENABLE - memcpy(crypto_info->rec_seq, rl_sequence, sizeof(crypto_info->rec_seq)); -# else - if (!is_tx) - return 0; -# endif - return 1; -}; - -#endif /* __FreeBSD__ */ - -#if defined(OPENSSL_SYS_LINUX) -/* Function to check supported ciphers in Linux */ -static int ktls_int_check_supported_cipher(OSSL_RECORD_LAYER *rl, - const EVP_CIPHER *c, - const EVP_MD *md, - size_t taglen) -{ - switch (rl->version) { - case TLS1_2_VERSION: -#ifdef OPENSSL_KTLS_TLS13 - case TLS1_3_VERSION: -#endif - break; - default: - return 0; - } - - /* - * Check that cipher is AES_GCM_128, AES_GCM_256, AES_CCM_128 - * or Chacha20-Poly1305 - */ -# ifdef OPENSSL_KTLS_AES_CCM_128 - if (EVP_CIPHER_is_a(c, "AES-128-CCM")) { - if (taglen != EVP_CCM_TLS_TAG_LEN) - return 0; - return 1; - } else -# endif - if (0 -# ifdef OPENSSL_KTLS_AES_GCM_128 - || EVP_CIPHER_is_a(c, "AES-128-GCM") -# endif -# ifdef OPENSSL_KTLS_AES_GCM_256 - || EVP_CIPHER_is_a(c, "AES-256-GCM") -# endif -# ifdef OPENSSL_KTLS_CHACHA20_POLY1305 - || EVP_CIPHER_is_a(c, "ChaCha20-Poly1305") -# endif - ) { - return 1; - } - return 0; -} - -/* Function to configure kernel TLS structure */ -static -int ktls_configure_crypto(OSSL_LIB_CTX *libctx, int version, const EVP_CIPHER *c, - const EVP_MD *md, void *rl_sequence, - ktls_crypto_info_t *crypto_info, int is_tx, - unsigned char *iv, size_t ivlen, - unsigned char *key, size_t keylen, - unsigned char *mac_key, size_t mac_secret_size) -{ - unsigned char geniv[EVP_GCM_TLS_EXPLICIT_IV_LEN]; - unsigned char *eiv = NULL; - -# ifdef OPENSSL_NO_KTLS_RX - if (!is_tx) - return 0; -# endif - - if (EVP_CIPHER_get_mode(c) == EVP_CIPH_GCM_MODE - || EVP_CIPHER_get_mode(c) == EVP_CIPH_CCM_MODE) { - if (!ossl_assert(EVP_GCM_TLS_FIXED_IV_LEN == EVP_CCM_TLS_FIXED_IV_LEN) - || !ossl_assert(EVP_GCM_TLS_EXPLICIT_IV_LEN - == EVP_CCM_TLS_EXPLICIT_IV_LEN)) - return 0; - if (version == TLS1_2_VERSION) { - if (!ossl_assert(ivlen == EVP_GCM_TLS_FIXED_IV_LEN)) - return 0; - if (is_tx) { - if (RAND_bytes_ex(libctx, geniv, - EVP_GCM_TLS_EXPLICIT_IV_LEN, 0) <= 0) - return 0; - } else { - memset(geniv, 0, EVP_GCM_TLS_EXPLICIT_IV_LEN); - } - eiv = geniv; - } else { - if (!ossl_assert(ivlen == EVP_GCM_TLS_FIXED_IV_LEN - + EVP_GCM_TLS_EXPLICIT_IV_LEN)) - return 0; - eiv = iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE; - } - } - - memset(crypto_info, 0, sizeof(*crypto_info)); - switch (EVP_CIPHER_get_nid(c)) { -# ifdef OPENSSL_KTLS_AES_GCM_128 - case NID_aes_128_gcm: - if (!ossl_assert(TLS_CIPHER_AES_GCM_128_SALT_SIZE - == EVP_GCM_TLS_FIXED_IV_LEN) - || !ossl_assert(TLS_CIPHER_AES_GCM_128_IV_SIZE - == EVP_GCM_TLS_EXPLICIT_IV_LEN)) - return 0; - crypto_info->gcm128.info.cipher_type = TLS_CIPHER_AES_GCM_128; - crypto_info->gcm128.info.version = version; - crypto_info->tls_crypto_info_len = sizeof(crypto_info->gcm128); - memcpy(crypto_info->gcm128.iv, eiv, TLS_CIPHER_AES_GCM_128_IV_SIZE); - memcpy(crypto_info->gcm128.salt, iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); - memcpy(crypto_info->gcm128.key, key, keylen); - memcpy(crypto_info->gcm128.rec_seq, rl_sequence, - TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); - return 1; -# endif -# ifdef OPENSSL_KTLS_AES_GCM_256 - case NID_aes_256_gcm: - if (!ossl_assert(TLS_CIPHER_AES_GCM_256_SALT_SIZE - == EVP_GCM_TLS_FIXED_IV_LEN) - || !ossl_assert(TLS_CIPHER_AES_GCM_256_IV_SIZE - == EVP_GCM_TLS_EXPLICIT_IV_LEN)) - return 0; - crypto_info->gcm256.info.cipher_type = TLS_CIPHER_AES_GCM_256; - crypto_info->gcm256.info.version = version; - crypto_info->tls_crypto_info_len = sizeof(crypto_info->gcm256); - memcpy(crypto_info->gcm256.iv, eiv, TLS_CIPHER_AES_GCM_256_IV_SIZE); - memcpy(crypto_info->gcm256.salt, iv, TLS_CIPHER_AES_GCM_256_SALT_SIZE); - memcpy(crypto_info->gcm256.key, key, keylen); - memcpy(crypto_info->gcm256.rec_seq, rl_sequence, - TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE); - - return 1; -# endif -# ifdef OPENSSL_KTLS_AES_CCM_128 - case NID_aes_128_ccm: - if (!ossl_assert(TLS_CIPHER_AES_CCM_128_SALT_SIZE - == EVP_CCM_TLS_FIXED_IV_LEN) - || !ossl_assert(TLS_CIPHER_AES_CCM_128_IV_SIZE - == EVP_CCM_TLS_EXPLICIT_IV_LEN)) - return 0; - crypto_info->ccm128.info.cipher_type = TLS_CIPHER_AES_CCM_128; - crypto_info->ccm128.info.version = version; - crypto_info->tls_crypto_info_len = sizeof(crypto_info->ccm128); - memcpy(crypto_info->ccm128.iv, eiv, TLS_CIPHER_AES_CCM_128_IV_SIZE); - memcpy(crypto_info->ccm128.salt, iv, TLS_CIPHER_AES_CCM_128_SALT_SIZE); - memcpy(crypto_info->ccm128.key, key, keylen); - memcpy(crypto_info->ccm128.rec_seq, rl_sequence, - TLS_CIPHER_AES_CCM_128_REC_SEQ_SIZE); - return 1; -# endif -# ifdef OPENSSL_KTLS_CHACHA20_POLY1305 - case NID_chacha20_poly1305: - if (!ossl_assert(ivlen == TLS_CIPHER_CHACHA20_POLY1305_IV_SIZE)) - return 0; - crypto_info->chacha20poly1305.info.cipher_type - = TLS_CIPHER_CHACHA20_POLY1305; - crypto_info->chacha20poly1305.info.version = version; - crypto_info->tls_crypto_info_len = sizeof(crypto_info->chacha20poly1305); - memcpy(crypto_info->chacha20poly1305.iv, iv, ivlen); - memcpy(crypto_info->chacha20poly1305.key, key, keylen); - memcpy(crypto_info->chacha20poly1305.rec_seq, rl_sequence, - TLS_CIPHER_CHACHA20_POLY1305_REC_SEQ_SIZE); - return 1; -# endif - default: - return 0; - } - -} - -#endif /* OPENSSL_SYS_LINUX */ - -static int ktls_set_crypto_state(OSSL_RECORD_LAYER *rl, int level, - unsigned char *key, size_t keylen, - unsigned char *iv, size_t ivlen, - unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, - size_t taglen, - int mactype, - const EVP_MD *md, - COMP_METHOD *comp) -{ - ktls_crypto_info_t crypto_info; - - /* - * Check if we are suitable for KTLS. If not suitable we return - * OSSL_RECORD_RETURN_NON_FATAL_ERR so that other record layers can be tried - * instead - */ - - if (comp != NULL) - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - - /* ktls supports only the maximum fragment size */ - if (rl->max_frag_len != SSL3_RT_MAX_PLAIN_LENGTH) - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - - /* check that cipher is supported */ - if (!ktls_int_check_supported_cipher(rl, ciph, md, taglen)) - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - - /* All future data will get encrypted by ktls. Flush the BIO or skip ktls */ - if (rl->direction == OSSL_RECORD_DIRECTION_WRITE) { - if (BIO_flush(rl->bio) <= 0) - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - - /* KTLS does not support record padding */ - if (rl->padding != NULL || rl->block_padding > 0) - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - } - - if (!ktls_configure_crypto(rl->libctx, rl->version, ciph, md, rl->sequence, - &crypto_info, - rl->direction == OSSL_RECORD_DIRECTION_WRITE, - iv, ivlen, key, keylen, mackey, mackeylen)) - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - - if (!BIO_set_ktls(rl->bio, &crypto_info, rl->direction)) - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - - if (rl->direction == OSSL_RECORD_DIRECTION_WRITE && - (rl->options & SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE) != 0) - /* Ignore errors. The application opts in to using the zerocopy - * optimization. If the running kernel doesn't support it, just - * continue without the optimization. - */ - BIO_set_ktls_tx_zerocopy_sendfile(rl->bio); - - return OSSL_RECORD_RETURN_SUCCESS; -} - -static int ktls_read_n(OSSL_RECORD_LAYER *rl, size_t n, size_t max, int extend, - int clearold, size_t *readbytes) -{ - int ret; - - ret = tls_default_read_n(rl, n, max, extend, clearold, readbytes); - - if (ret < OSSL_RECORD_RETURN_RETRY) { - switch (errno) { - case EBADMSG: - RLAYERfatal(rl, SSL_AD_BAD_RECORD_MAC, - SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); - break; - case EMSGSIZE: - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, - SSL_R_PACKET_LENGTH_TOO_LONG); - break; - case EINVAL: - RLAYERfatal(rl, SSL_AD_PROTOCOL_VERSION, - SSL_R_WRONG_VERSION_NUMBER); - break; - default: - break; - } - } - - return ret; -} - -static int ktls_cipher(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *inrecs, - size_t n_recs, int sending, SSL_MAC_BUF *mac, - size_t macsize) -{ - return 1; -} - -static int ktls_validate_record_header(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec) -{ - if (rec->rec_version != TLS1_2_VERSION) { - RLAYERfatal(rl, SSL_AD_DECODE_ERROR, SSL_R_WRONG_VERSION_NUMBER); - return 0; - } - - return 1; -} - -static int ktls_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec) -{ - if (rl->version == TLS1_3_VERSION) - return tls13_common_post_process_record(rl, rec); - - return 1; -} - -static int -ktls_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers, - int role, int direction, int level, uint16_t epoch, - unsigned char *secret, size_t secretlen, - unsigned char *key, size_t keylen, unsigned char *iv, - size_t ivlen, unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, size_t taglen, - int mactype, - const EVP_MD *md, COMP_METHOD *comp, - const EVP_MD *kdfdigest, BIO *prev, BIO *transport, - BIO *next, BIO_ADDR *local, BIO_ADDR *peer, - const OSSL_PARAM *settings, const OSSL_PARAM *options, - const OSSL_DISPATCH *fns, void *cbarg, void *rlarg, - OSSL_RECORD_LAYER **retrl) -{ - int ret; - - ret = tls_int_new_record_layer(libctx, propq, vers, role, direction, level, - ciph, taglen, md, comp, prev, - transport, next, settings, - options, fns, cbarg, retrl); - - if (ret != OSSL_RECORD_RETURN_SUCCESS) - return ret; - - (*retrl)->funcs = &ossl_ktls_funcs; - - ret = (*retrl)->funcs->set_crypto_state(*retrl, level, key, keylen, iv, - ivlen, mackey, mackeylen, ciph, - taglen, mactype, md, comp); - - if (ret != OSSL_RECORD_RETURN_SUCCESS) { - OPENSSL_free(*retrl); - *retrl = NULL; - } else { - /* - * With KTLS we always try and read as much as possible and fill the - * buffer - */ - (*retrl)->read_ahead = 1; - } - return ret; -} - -static int ktls_allocate_write_buffers(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, size_t *prefix) -{ - if (!ossl_assert(numtempl == 1)) - return 0; - - /* - * We just use the end application buffer in the case of KTLS, so nothing - * to do. We pretend we set up one buffer. - */ - rl->numwpipes = 1; - - return 1; -} - -static int ktls_initialise_write_packets(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, - OSSL_RECORD_TEMPLATE *prefixtempl, - WPACKET *pkt, - TLS_BUFFER *bufs, - size_t *wpinited) -{ - TLS_BUFFER *wb; - - /* - * We just use the application buffer directly and don't use any WPACKET - * structures - */ - wb = &bufs[0]; - wb->type = templates[0].type; - - /* - * ktls doesn't modify the buffer, but to avoid a warning we need - * to discard the const qualifier. - * This doesn't leak memory because the buffers have never been allocated - * with KTLS - */ - TLS_BUFFER_set_buf(wb, (unsigned char *)templates[0].buf); - TLS_BUFFER_set_offset(wb, 0); - TLS_BUFFER_set_app_buffer(wb, 1); - - return 1; -} - -static int ktls_prepare_record_header(OSSL_RECORD_LAYER *rl, - WPACKET *thispkt, - OSSL_RECORD_TEMPLATE *templ, - uint8_t rectype, - unsigned char **recdata) -{ - /* The kernel writes the record header, so nothing to do */ - *recdata = NULL; - - return 1; -} - -static int ktls_prepare_for_encryption(OSSL_RECORD_LAYER *rl, - size_t mac_size, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr) -{ - /* No encryption, so nothing to do */ - return 1; -} - -static int ktls_post_encryption_processing(OSSL_RECORD_LAYER *rl, - size_t mac_size, - OSSL_RECORD_TEMPLATE *templ, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr) -{ - /* The kernel does anything that is needed, so nothing to do here */ - return 1; -} - -static int ktls_prepare_write_bio(OSSL_RECORD_LAYER *rl, int type) -{ - /* - * To prevent coalescing of control and data messages, - * such as in buffer_write, we flush the BIO - */ - if (type != SSL3_RT_APPLICATION_DATA) { - int ret, i = BIO_flush(rl->bio); - - if (i <= 0) { - if (BIO_should_retry(rl->bio)) - ret = OSSL_RECORD_RETURN_RETRY; - else - ret = OSSL_RECORD_RETURN_FATAL; - return ret; - } - BIO_set_ktls_ctrl_msg(rl->bio, type); - } - - return OSSL_RECORD_RETURN_SUCCESS; -} - -static int ktls_alloc_buffers(OSSL_RECORD_LAYER *rl) -{ - /* We use the application buffer directly for writing */ - if (rl->direction == OSSL_RECORD_DIRECTION_WRITE) - return 1; - - return tls_alloc_buffers(rl); -} - -static int ktls_free_buffers(OSSL_RECORD_LAYER *rl) -{ - /* We use the application buffer directly for writing */ - if (rl->direction == OSSL_RECORD_DIRECTION_WRITE) - return 1; - - return tls_free_buffers(rl); -} - -static struct record_functions_st ossl_ktls_funcs = { - ktls_set_crypto_state, - ktls_cipher, - NULL, - tls_default_set_protocol_version, - ktls_read_n, - tls_get_more_records, - ktls_validate_record_header, - ktls_post_process_record, - tls_get_max_records_default, - tls_write_records_default, - ktls_allocate_write_buffers, - ktls_initialise_write_packets, - NULL, - ktls_prepare_record_header, - NULL, - ktls_prepare_for_encryption, - ktls_post_encryption_processing, - ktls_prepare_write_bio -}; - -const OSSL_RECORD_METHOD ossl_ktls_record_method = { - ktls_new_record_layer, - tls_free, - tls_unprocessed_read_pending, - tls_processed_read_pending, - tls_app_data_pending, - tls_get_max_records, - tls_write_records, - tls_retry_write_records, - tls_read_record, - tls_release_record, - tls_get_alert_code, - tls_set1_bio, - tls_set_protocol_version, - tls_set_plain_alerts, - tls_set_first_handshake, - tls_set_max_pipelines, - NULL, - tls_get_state, - tls_set_options, - tls_get_compression, - tls_set_max_frag_len, - NULL, - tls_increment_sequence_ctr, - ktls_alloc_buffers, - ktls_free_buffers -}; diff --git a/openssl/src/ssl/record/methods/recmethod_local.h b/openssl/src/ssl/record/methods/recmethod_local.h deleted file mode 100644 index fe9dce153..000000000 --- a/openssl/src/ssl/record/methods/recmethod_local.h +++ /dev/null @@ -1,537 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include "../../ssl_local.h" -#include "../record_local.h" - -typedef struct dtls_bitmap_st { - /* Track 64 packets */ - uint64_t map; - /* Max record number seen so far, 64-bit value in big-endian encoding */ - unsigned char max_seq_num[SEQ_NUM_SIZE]; -} DTLS_BITMAP; - -typedef struct ssl_mac_buf_st { - unsigned char *mac; - int alloced; -} SSL_MAC_BUF; - -typedef struct tls_buffer_st { - /* at least SSL3_RT_MAX_PACKET_SIZE bytes */ - unsigned char *buf; - /* default buffer size (or 0 if no default set) */ - size_t default_len; - /* buffer size */ - size_t len; - /* where to 'copy from' */ - size_t offset; - /* how many bytes left */ - size_t left; - /* 'buf' is from application for KTLS */ - int app_buffer; - /* The type of data stored in this buffer. Only used for writing */ - int type; -} TLS_BUFFER; - -typedef struct tls_rl_record_st { - /* Record layer version */ - /* r */ - int rec_version; - /* type of record */ - /* r */ - int type; - /* How many bytes available */ - /* rw */ - size_t length; - /* - * How many bytes were available before padding was removed? This is used - * to implement the MAC check in constant time for CBC records. - */ - /* rw */ - size_t orig_len; - /* read/write offset into 'buf' */ - /* r */ - size_t off; - /* pointer to the record data */ - /* rw */ - unsigned char *data; - /* where the decode bytes are */ - /* rw */ - unsigned char *input; - /* only used with decompression - malloc()ed */ - /* r */ - unsigned char *comp; - /* epoch number, needed by DTLS1 */ - /* r */ - uint16_t epoch; - /* sequence number, needed by DTLS1 */ - /* r */ - unsigned char seq_num[SEQ_NUM_SIZE]; -} TLS_RL_RECORD; - -/* Macros/functions provided by the TLS_RL_RECORD component */ - -#define TLS_RL_RECORD_set_type(r, t) ((r)->type = (t)) -#define TLS_RL_RECORD_set_rec_version(r, v) ((r)->rec_version = (v)) -#define TLS_RL_RECORD_get_length(r) ((r)->length) -#define TLS_RL_RECORD_set_length(r, l) ((r)->length = (l)) -#define TLS_RL_RECORD_add_length(r, l) ((r)->length += (l)) -#define TLS_RL_RECORD_set_data(r, d) ((r)->data = (d)) -#define TLS_RL_RECORD_set_input(r, i) ((r)->input = (i)) -#define TLS_RL_RECORD_reset_input(r) ((r)->input = (r)->data) - - -/* Protocol version specific function pointers */ -struct record_functions_st -{ - /* - * Returns either OSSL_RECORD_RETURN_SUCCESS, OSSL_RECORD_RETURN_FATAL or - * OSSL_RECORD_RETURN_NON_FATAL_ERR if we can keep trying to find an - * alternative record layer. - */ - int (*set_crypto_state)(OSSL_RECORD_LAYER *rl, int level, - unsigned char *key, size_t keylen, - unsigned char *iv, size_t ivlen, - unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, - size_t taglen, - int mactype, - const EVP_MD *md, - COMP_METHOD *comp); - - /* - * Returns: - * 0: if the record is publicly invalid, or an internal error, or AEAD - * decryption failed, or EtM decryption failed. - * 1: Success or MtE decryption failed (MAC will be randomised) - */ - int (*cipher)(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *recs, size_t n_recs, - int sending, SSL_MAC_BUF *macs, size_t macsize); - /* Returns 1 for success or 0 for error */ - int (*mac)(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec, unsigned char *md, - int sending); - - /* Return 1 for success or 0 for error */ - int (*set_protocol_version)(OSSL_RECORD_LAYER *rl, int version); - - /* Read related functions */ - - int (*read_n)(OSSL_RECORD_LAYER *rl, size_t n, size_t max, int extend, - int clearold, size_t *readbytes); - - int (*get_more_records)(OSSL_RECORD_LAYER *rl); - - /* Return 1 for success or 0 for error */ - int (*validate_record_header)(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec); - - /* Return 1 for success or 0 for error */ - int (*post_process_record)(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec); - - /* Write related functions */ - - size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len, - size_t maxfrag, size_t *preffrag); - - /* Return 1 for success or 0 for error */ - int (*write_records)(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, - size_t numtempl); - - /* Allocate the rl->wbuf buffers. Return 1 for success or 0 for error */ - int (*allocate_write_buffers)(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, size_t *prefix); - - /* - * Initialise the packets in the |pkt| array using the buffers in |rl->wbuf|. - * Some protocol versions may use the space in |prefixtempl| to add - * an artificial template in front of the |templates| array and hence may - * initialise 1 more WPACKET than there are templates. |*wpinited| - * returns the number of WPACKETs in |pkt| that were successfully - * initialised. This must be 0 on entry and will be filled in even on error. - */ - int (*initialise_write_packets)(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, - OSSL_RECORD_TEMPLATE *prefixtempl, - WPACKET *pkt, - TLS_BUFFER *bufs, - size_t *wpinited); - - /* Get the actual record type to be used for a given template */ - uint8_t (*get_record_type)(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *template); - - /* Write the record header data to the WPACKET */ - int (*prepare_record_header)(OSSL_RECORD_LAYER *rl, WPACKET *thispkt, - OSSL_RECORD_TEMPLATE *templ, - uint8_t rectype, - unsigned char **recdata); - - int (*add_record_padding)(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *thistempl, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr); - - /* - * This applies any mac that might be necessary, ensures that we have enough - * space in the WPACKET to perform the encryption and sets up the - * TLS_RL_RECORD ready for that encryption. - */ - int (*prepare_for_encryption)(OSSL_RECORD_LAYER *rl, - size_t mac_size, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr); - - /* - * Any updates required to the record after encryption has been applied. For - * example, adding a MAC if using encrypt-then-mac - */ - int (*post_encryption_processing)(OSSL_RECORD_LAYER *rl, - size_t mac_size, - OSSL_RECORD_TEMPLATE *thistempl, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr); - - /* - * Some record layer implementations need to do some custom preparation of - * the BIO before we write to it. KTLS does this to prevent coalescing of - * control and data messages. - */ - int (*prepare_write_bio)(OSSL_RECORD_LAYER *rl, int type); -}; - -struct ossl_record_layer_st -{ - OSSL_LIB_CTX *libctx; - const char *propq; - int isdtls; - int version; - int role; - int direction; - int level; - const EVP_MD *md; - /* DTLS only */ - uint16_t epoch; - - /* - * A BIO containing any data read in the previous epoch that was destined - * for this epoch - */ - BIO *prev; - - /* The transport BIO */ - BIO *bio; - - /* - * A BIO where we will send any data read by us that is destined for the - * next epoch. - */ - BIO *next; - - /* Types match the equivalent fields in the SSL object */ - uint64_t options; - uint32_t mode; - - /* write IO goes into here */ - TLS_BUFFER wbuf[SSL_MAX_PIPELINES + 1]; - - /* Next wbuf with pending data still to write */ - size_t nextwbuf; - - /* How many pipelines can be used to write data */ - size_t numwpipes; - - /* read IO goes into here */ - TLS_BUFFER rbuf; - /* each decoded record goes in here */ - TLS_RL_RECORD rrec[SSL_MAX_PIPELINES]; - - /* How many records have we got available in the rrec buffer */ - size_t num_recs; - - /* The record number in the rrec buffer that can be read next */ - size_t curr_rec; - - /* The number of records that have been released via tls_release_record */ - size_t num_released; - - /* where we are when reading */ - int rstate; - - /* used internally to point at a raw packet */ - unsigned char *packet; - size_t packet_length; - - /* Sequence number for the next record */ - unsigned char sequence[SEQ_NUM_SIZE]; - - /* Alert code to be used if an error occurs */ - int alert; - - /* - * Read as many input bytes as possible (for non-blocking reads) - */ - int read_ahead; - - /* The number of consecutive empty records we have received */ - size_t empty_record_count; - - /* - * Do we need to send a prefix empty record before application data as a - * countermeasure against known-IV weakness (necessary for SSLv3 and - * TLSv1.0) - */ - int need_empty_fragments; - - /* cryptographic state */ - EVP_CIPHER_CTX *enc_ctx; - - /* Explicit IV length */ - size_t eivlen; - - /* used for mac generation */ - EVP_MD_CTX *md_ctx; - - /* compress/uncompress */ - COMP_CTX *compctx; - - /* Set to 1 if this is the first handshake. 0 otherwise */ - int is_first_handshake; - - /* - * The smaller of the configured and negotiated maximum fragment length - * or SSL3_RT_MAX_PLAIN_LENGTH if none - */ - unsigned int max_frag_len; - - /* The maximum amount of early data we can receive/send */ - uint32_t max_early_data; - - /* The amount of early data that we have sent/received */ - size_t early_data_count; - - /* TLSv1.3 record padding */ - size_t block_padding; - - /* Only used by SSLv3 */ - unsigned char mac_secret[EVP_MAX_MD_SIZE]; - - /* TLSv1.0/TLSv1.1/TLSv1.2 */ - int use_etm; - - /* Flags for GOST ciphers */ - int stream_mac; - int tlstree; - - /* TLSv1.3 fields */ - /* static IV */ - unsigned char iv[EVP_MAX_IV_LENGTH]; - int allow_plain_alerts; - - /* TLS "any" fields */ - /* Set to true if this is the first record in a connection */ - unsigned int is_first_record; - - size_t taglen; - - /* DTLS received handshake records (processed and unprocessed) */ - struct pqueue_st *unprocessed_rcds; - struct pqueue_st *processed_rcds; - - /* records being received in the current epoch */ - DTLS_BITMAP bitmap; - /* renegotiation starts a new set of sequence numbers */ - DTLS_BITMAP next_bitmap; - - /* - * Whether we are currently in a handshake or not. Only maintained for DTLS - */ - int in_init; - - /* Callbacks */ - void *cbarg; - OSSL_FUNC_rlayer_skip_early_data_fn *skip_early_data; - OSSL_FUNC_rlayer_msg_callback_fn *msg_callback; - OSSL_FUNC_rlayer_security_fn *security; - OSSL_FUNC_rlayer_padding_fn *padding; - - size_t max_pipelines; - - /* Function pointers for version specific functions */ - const struct record_functions_st *funcs; -}; - -typedef struct dtls_rlayer_record_data_st { - unsigned char *packet; - size_t packet_length; - TLS_BUFFER rbuf; - TLS_RL_RECORD rrec; -} DTLS_RLAYER_RECORD_DATA; - -extern const struct record_functions_st ssl_3_0_funcs; -extern const struct record_functions_st tls_1_funcs; -extern const struct record_functions_st tls_1_3_funcs; -extern const struct record_functions_st tls_any_funcs; -extern const struct record_functions_st dtls_1_funcs; -extern const struct record_functions_st dtls_any_funcs; - -void ossl_rlayer_fatal(OSSL_RECORD_LAYER *rl, int al, int reason, - const char *fmt, ...); - -#define RLAYERfatal(rl, al, r) RLAYERfatal_data((rl), (al), (r), NULL) -#define RLAYERfatal_data \ - (ERR_new(), \ - ERR_set_debug(OPENSSL_FILE, OPENSSL_LINE, OPENSSL_FUNC), \ - ossl_rlayer_fatal) - -#define RLAYER_USE_EXPLICIT_IV(rl) ((rl)->version == TLS1_1_VERSION \ - || (rl)->version == TLS1_2_VERSION \ - || (rl)->version == DTLS1_BAD_VER \ - || (rl)->version == DTLS1_VERSION \ - || (rl)->version == DTLS1_2_VERSION) - -void ossl_tls_rl_record_set_seq_num(TLS_RL_RECORD *r, - const unsigned char *seq_num); - -int ossl_set_tls_provider_parameters(OSSL_RECORD_LAYER *rl, - EVP_CIPHER_CTX *ctx, - const EVP_CIPHER *ciph, - const EVP_MD *md); - -int tls_increment_sequence_ctr(OSSL_RECORD_LAYER *rl); -int tls_alloc_buffers(OSSL_RECORD_LAYER *rl); -int tls_free_buffers(OSSL_RECORD_LAYER *rl); - -int tls_default_read_n(OSSL_RECORD_LAYER *rl, size_t n, size_t max, int extend, - int clearold, size_t *readbytes); -int tls_get_more_records(OSSL_RECORD_LAYER *rl); -int dtls_get_more_records(OSSL_RECORD_LAYER *rl); - -int dtls_prepare_record_header(OSSL_RECORD_LAYER *rl, - WPACKET *thispkt, - OSSL_RECORD_TEMPLATE *templ, - uint8_t rectype, - unsigned char **recdata); -int dtls_post_encryption_processing(OSSL_RECORD_LAYER *rl, - size_t mac_size, - OSSL_RECORD_TEMPLATE *thistempl, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr); - -int tls_default_set_protocol_version(OSSL_RECORD_LAYER *rl, int version); -int tls_default_validate_record_header(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *re); -int tls_do_compress(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *wr); -int tls_do_uncompress(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec); -int tls_default_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec); -int tls13_common_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec); - -int -tls_int_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers, - int role, int direction, int level, - const EVP_CIPHER *ciph, size_t taglen, - const EVP_MD *md, COMP_METHOD *comp, BIO *prev, - BIO *transport, BIO *next, - const OSSL_PARAM *settings, const OSSL_PARAM *options, - const OSSL_DISPATCH *fns, void *cbarg, - OSSL_RECORD_LAYER **retrl); -int tls_free(OSSL_RECORD_LAYER *rl); -int tls_unprocessed_read_pending(OSSL_RECORD_LAYER *rl); -int tls_processed_read_pending(OSSL_RECORD_LAYER *rl); -size_t tls_app_data_pending(OSSL_RECORD_LAYER *rl); -size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len, - size_t maxfrag, size_t *preffrag); -int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, - size_t numtempl); -int tls_retry_write_records(OSSL_RECORD_LAYER *rl); -int tls_get_alert_code(OSSL_RECORD_LAYER *rl); -int tls_set1_bio(OSSL_RECORD_LAYER *rl, BIO *bio); -int tls_read_record(OSSL_RECORD_LAYER *rl, void **rechandle, int *rversion, - uint8_t *type, const unsigned char **data, size_t *datalen, - uint16_t *epoch, unsigned char *seq_num); -int tls_release_record(OSSL_RECORD_LAYER *rl, void *rechandle, size_t length); -int tls_default_set_protocol_version(OSSL_RECORD_LAYER *rl, int version); -int tls_set_protocol_version(OSSL_RECORD_LAYER *rl, int version); -void tls_set_plain_alerts(OSSL_RECORD_LAYER *rl, int allow); -void tls_set_first_handshake(OSSL_RECORD_LAYER *rl, int first); -void tls_set_max_pipelines(OSSL_RECORD_LAYER *rl, size_t max_pipelines); -void tls_get_state(OSSL_RECORD_LAYER *rl, const char **shortstr, - const char **longstr); -int tls_set_options(OSSL_RECORD_LAYER *rl, const OSSL_PARAM *options); -const COMP_METHOD *tls_get_compression(OSSL_RECORD_LAYER *rl); -void tls_set_max_frag_len(OSSL_RECORD_LAYER *rl, size_t max_frag_len); -int tls_setup_read_buffer(OSSL_RECORD_LAYER *rl); -int tls_setup_write_buffer(OSSL_RECORD_LAYER *rl, size_t numwpipes, - size_t firstlen, size_t nextlen); - -int tls_write_records_multiblock(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl); - -size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, uint8_t type, - size_t len, - size_t maxfrag, size_t *preffrag); -size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, uint8_t type, - size_t len, size_t maxfrag, - size_t *preffrag); -int tls_allocate_write_buffers_default(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, size_t *prefix); -int tls_initialise_write_packets_default(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, - OSSL_RECORD_TEMPLATE *prefixtempl, - WPACKET *pkt, - TLS_BUFFER *bufs, - size_t *wpinited); -int tls1_allocate_write_buffers(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, size_t *prefix); -int tls1_initialise_write_packets(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, - OSSL_RECORD_TEMPLATE *prefixtempl, - WPACKET *pkt, - TLS_BUFFER *bufs, - size_t *wpinited); -int tls_prepare_record_header_default(OSSL_RECORD_LAYER *rl, - WPACKET *thispkt, - OSSL_RECORD_TEMPLATE *templ, - uint8_t rectype, - unsigned char **recdata); -int tls_prepare_for_encryption_default(OSSL_RECORD_LAYER *rl, - size_t mac_size, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr); -int tls_post_encryption_processing_default(OSSL_RECORD_LAYER *rl, - size_t mac_size, - OSSL_RECORD_TEMPLATE *thistempl, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr); -int tls_write_records_default(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl); - -/* Macros/functions provided by the TLS_BUFFER component */ - -#define TLS_BUFFER_get_buf(b) ((b)->buf) -#define TLS_BUFFER_set_buf(b, n) ((b)->buf = (n)) -#define TLS_BUFFER_get_len(b) ((b)->len) -#define TLS_BUFFER_get_left(b) ((b)->left) -#define TLS_BUFFER_set_left(b, l) ((b)->left = (l)) -#define TLS_BUFFER_sub_left(b, l) ((b)->left -= (l)) -#define TLS_BUFFER_get_offset(b) ((b)->offset) -#define TLS_BUFFER_set_offset(b, o) ((b)->offset = (o)) -#define TLS_BUFFER_add_offset(b, o) ((b)->offset += (o)) -#define TLS_BUFFER_set_app_buffer(b, l) ((b)->app_buffer = (l)) -#define TLS_BUFFER_is_app_buffer(b) ((b)->app_buffer) - -void ossl_tls_buffer_release(TLS_BUFFER *b); diff --git a/openssl/src/ssl/record/methods/ssl3_meth.c b/openssl/src/ssl/record/methods/ssl3_meth.c deleted file mode 100644 index 6b5a1bed2..000000000 --- a/openssl/src/ssl/record/methods/ssl3_meth.c +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "internal/ssl3_cbc.h" -#include "../../ssl_local.h" -#include "../record_local.h" -#include "recmethod_local.h" - -static int ssl3_set_crypto_state(OSSL_RECORD_LAYER *rl, int level, - unsigned char *key, size_t keylen, - unsigned char *iv, size_t ivlen, - unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, - size_t taglen, - int mactype, - const EVP_MD *md, - COMP_METHOD *comp) -{ - EVP_CIPHER_CTX *ciph_ctx; - int enc = (rl->direction == OSSL_RECORD_DIRECTION_WRITE) ? 1 : 0; - - if (md == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - if ((rl->enc_ctx = EVP_CIPHER_CTX_new()) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - ciph_ctx = rl->enc_ctx; - - rl->md_ctx = EVP_MD_CTX_new(); - if (rl->md_ctx == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - if ((md != NULL && EVP_DigestInit_ex(rl->md_ctx, md, NULL) <= 0)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - -#ifndef OPENSSL_NO_COMP - if (comp != NULL) { - rl->compctx = COMP_CTX_new(comp); - if (rl->compctx == NULL) { - ERR_raise(ERR_LIB_SSL, SSL_R_COMPRESSION_LIBRARY_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - } -#endif - - if (!EVP_CipherInit_ex(ciph_ctx, ciph, NULL, key, iv, enc)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - /* - * The cipher we actually ended up using in the EVP_CIPHER_CTX may be - * different to that in ciph if we have an ENGINE in use - */ - if (EVP_CIPHER_get0_provider(EVP_CIPHER_CTX_get0_cipher(ciph_ctx)) != NULL - && !ossl_set_tls_provider_parameters(rl, ciph_ctx, ciph, md)) { - /* ERR_raise already called */ - return OSSL_RECORD_RETURN_FATAL; - } - - if (mackeylen > sizeof(rl->mac_secret)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - memcpy(rl->mac_secret, mackey, mackeylen); - - return OSSL_RECORD_RETURN_SUCCESS; -} - -/* - * ssl3_cipher encrypts/decrypts |n_recs| records in |inrecs|. Calls RLAYERfatal - * on internal error, but not otherwise. It is the responsibility of the caller - * to report a bad_record_mac - * - * Returns: - * 0: if the record is publicly invalid, or an internal error - * 1: Success or Mac-then-encrypt decryption failed (MAC will be randomised) - */ -static int ssl3_cipher(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *inrecs, - size_t n_recs, int sending, SSL_MAC_BUF *mac, - size_t macsize) -{ - TLS_RL_RECORD *rec; - EVP_CIPHER_CTX *ds; - size_t l, i; - size_t bs; - const EVP_CIPHER *enc; - int provided; - - rec = inrecs; - /* - * We shouldn't ever be called with more than one record in the SSLv3 case - */ - if (n_recs != 1) - return 0; - - ds = rl->enc_ctx; - if (ds == NULL || (enc = EVP_CIPHER_CTX_get0_cipher(ds)) == NULL) - return 0; - - provided = (EVP_CIPHER_get0_provider(enc) != NULL); - - l = rec->length; - bs = EVP_CIPHER_CTX_get_block_size(ds); - - if (bs == 0) - return 0; - - /* COMPRESS */ - - if ((bs != 1) && sending && !provided) { - /* - * We only do this for legacy ciphers. Provided ciphers add the - * padding on the provider side. - */ - i = bs - (l % bs); - - /* we need to add 'i-1' padding bytes */ - l += i; - /* - * the last of these zero bytes will be overwritten with the - * padding length. - */ - memset(&rec->input[rec->length], 0, i); - rec->length += i; - rec->input[l - 1] = (unsigned char)(i - 1); - } - - if (!sending) { - if (l == 0 || l % bs != 0) { - /* Publicly invalid */ - return 0; - } - /* otherwise, rec->length >= bs */ - } - - if (provided) { - int outlen; - - if (!EVP_CipherUpdate(ds, rec->data, &outlen, rec->input, - (unsigned int)l)) - return 0; - rec->length = outlen; - - if (!sending && mac != NULL) { - /* Now get a pointer to the MAC */ - OSSL_PARAM params[2], *p = params; - - /* Get the MAC */ - mac->alloced = 0; - - *p++ = OSSL_PARAM_construct_octet_ptr(OSSL_CIPHER_PARAM_TLS_MAC, - (void **)&mac->mac, - macsize); - *p = OSSL_PARAM_construct_end(); - - if (!EVP_CIPHER_CTX_get_params(ds, params)) { - /* Shouldn't normally happen */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - } - } else { - if (EVP_Cipher(ds, rec->data, rec->input, (unsigned int)l) < 1) { - /* Shouldn't happen */ - RLAYERfatal(rl, SSL_AD_BAD_RECORD_MAC, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (!sending) - return ssl3_cbc_remove_padding_and_mac(&rec->length, - rec->orig_len, - rec->data, - (mac != NULL) ? &mac->mac : NULL, - (mac != NULL) ? &mac->alloced : NULL, - bs, - macsize, - rl->libctx); - } - - return 1; -} - -static const unsigned char ssl3_pad_1[48] = { - 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, - 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, - 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, - 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, - 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, - 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36 -}; - -static const unsigned char ssl3_pad_2[48] = { - 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, - 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, - 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, - 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, - 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, - 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c -}; - -static int ssl3_mac(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec, unsigned char *md, - int sending) -{ - unsigned char *mac_sec, *seq = rl->sequence; - const EVP_MD_CTX *hash; - unsigned char *p, rec_char; - size_t md_size; - size_t npad; - int t; - - mac_sec = &(rl->mac_secret[0]); - hash = rl->md_ctx; - - t = EVP_MD_CTX_get_size(hash); - if (t <= 0) - return 0; - md_size = t; - npad = (48 / md_size) * md_size; - - if (!sending - && EVP_CIPHER_CTX_get_mode(rl->enc_ctx) == EVP_CIPH_CBC_MODE - && ssl3_cbc_record_digest_supported(hash)) { -#ifdef OPENSSL_NO_DEPRECATED_3_0 - return 0; -#else - /* - * This is a CBC-encrypted record. We must avoid leaking any - * timing-side channel information about how many blocks of data we - * are hashing because that gives an attacker a timing-oracle. - */ - - /*- - * npad is, at most, 48 bytes and that's with MD5: - * 16 + 48 + 8 (sequence bytes) + 1 + 2 = 75. - * - * With SHA-1 (the largest hash speced for SSLv3) the hash size - * goes up 4, but npad goes down by 8, resulting in a smaller - * total size. - */ - unsigned char header[75]; - size_t j = 0; - memcpy(header + j, mac_sec, md_size); - j += md_size; - memcpy(header + j, ssl3_pad_1, npad); - j += npad; - memcpy(header + j, seq, 8); - j += 8; - header[j++] = rec->type; - header[j++] = (unsigned char)(rec->length >> 8); - header[j++] = (unsigned char)(rec->length & 0xff); - - /* Final param == is SSLv3 */ - if (ssl3_cbc_digest_record(EVP_MD_CTX_get0_md(hash), - md, &md_size, - header, rec->input, - rec->length, rec->orig_len, - mac_sec, md_size, 1) <= 0) - return 0; -#endif - } else { - unsigned int md_size_u; - /* Chop the digest off the end :-) */ - EVP_MD_CTX *md_ctx = EVP_MD_CTX_new(); - - if (md_ctx == NULL) - return 0; - - rec_char = rec->type; - p = md; - s2n(rec->length, p); - if (EVP_MD_CTX_copy_ex(md_ctx, hash) <= 0 - || EVP_DigestUpdate(md_ctx, mac_sec, md_size) <= 0 - || EVP_DigestUpdate(md_ctx, ssl3_pad_1, npad) <= 0 - || EVP_DigestUpdate(md_ctx, seq, 8) <= 0 - || EVP_DigestUpdate(md_ctx, &rec_char, 1) <= 0 - || EVP_DigestUpdate(md_ctx, md, 2) <= 0 - || EVP_DigestUpdate(md_ctx, rec->input, rec->length) <= 0 - || EVP_DigestFinal_ex(md_ctx, md, NULL) <= 0 - || EVP_MD_CTX_copy_ex(md_ctx, hash) <= 0 - || EVP_DigestUpdate(md_ctx, mac_sec, md_size) <= 0 - || EVP_DigestUpdate(md_ctx, ssl3_pad_2, npad) <= 0 - || EVP_DigestUpdate(md_ctx, md, md_size) <= 0 - || EVP_DigestFinal_ex(md_ctx, md, &md_size_u) <= 0) { - EVP_MD_CTX_free(md_ctx); - return 0; - } - - EVP_MD_CTX_free(md_ctx); - } - - if (!tls_increment_sequence_ctr(rl)) - return 0; - - return 1; -} - -const struct record_functions_st ssl_3_0_funcs = { - ssl3_set_crypto_state, - ssl3_cipher, - ssl3_mac, - tls_default_set_protocol_version, - tls_default_read_n, - tls_get_more_records, - tls_default_validate_record_header, - tls_default_post_process_record, - tls_get_max_records_default, - tls_write_records_default, - /* These 2 functions are defined in tls1_meth.c */ - tls1_allocate_write_buffers, - tls1_initialise_write_packets, - NULL, - tls_prepare_record_header_default, - NULL, - tls_prepare_for_encryption_default, - tls_post_encryption_processing_default, - NULL -}; diff --git a/openssl/src/ssl/record/methods/tls13_meth.c b/openssl/src/ssl/record/methods/tls13_meth.c deleted file mode 100644 index d782c327e..000000000 --- a/openssl/src/ssl/record/methods/tls13_meth.c +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include "../../ssl_local.h" -#include "../record_local.h" -#include "recmethod_local.h" - -static int tls13_set_crypto_state(OSSL_RECORD_LAYER *rl, int level, - unsigned char *key, size_t keylen, - unsigned char *iv, size_t ivlen, - unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, - size_t taglen, - int mactype, - const EVP_MD *md, - COMP_METHOD *comp) -{ - EVP_CIPHER_CTX *ciph_ctx; - int mode; - int enc = (rl->direction == OSSL_RECORD_DIRECTION_WRITE) ? 1 : 0; - - if (ivlen > sizeof(rl->iv)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - memcpy(rl->iv, iv, ivlen); - - ciph_ctx = rl->enc_ctx = EVP_CIPHER_CTX_new(); - if (ciph_ctx == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - mode = EVP_CIPHER_get_mode(ciph); - - if (EVP_CipherInit_ex(ciph_ctx, ciph, NULL, NULL, NULL, enc) <= 0 - || EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_AEAD_SET_IVLEN, ivlen, - NULL) <= 0 - || (mode == EVP_CIPH_CCM_MODE - && EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_AEAD_SET_TAG, taglen, - NULL) <= 0) - || EVP_CipherInit_ex(ciph_ctx, NULL, NULL, key, NULL, enc) <= 0) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - return OSSL_RECORD_RETURN_SUCCESS; -} - -static int tls13_cipher(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *recs, - size_t n_recs, int sending, SSL_MAC_BUF *mac, - size_t macsize) -{ - EVP_CIPHER_CTX *ctx; - unsigned char iv[EVP_MAX_IV_LENGTH], recheader[SSL3_RT_HEADER_LENGTH]; - size_t ivlen, offset, loop, hdrlen; - unsigned char *staticiv; - unsigned char *seq = rl->sequence; - int lenu, lenf; - TLS_RL_RECORD *rec = &recs[0]; - WPACKET wpkt; - const EVP_CIPHER *cipher; - int mode; - - if (n_recs != 1) { - /* Should not happen */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - ctx = rl->enc_ctx; - staticiv = rl->iv; - - cipher = EVP_CIPHER_CTX_get0_cipher(ctx); - if (cipher == NULL) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - mode = EVP_CIPHER_get_mode(cipher); - - /* - * If we're sending an alert and ctx != NULL then we must be forcing - * plaintext alerts. If we're reading and ctx != NULL then we allow - * plaintext alerts at certain points in the handshake. If we've got this - * far then we have already validated that a plaintext alert is ok here. - */ - if (ctx == NULL || rec->type == SSL3_RT_ALERT) { - memmove(rec->data, rec->input, rec->length); - rec->input = rec->data; - return 1; - } - - ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); - - if (!sending) { - /* - * Take off tag. There must be at least one byte of content type as - * well as the tag - */ - if (rec->length < rl->taglen + 1) - return 0; - rec->length -= rl->taglen; - } - - /* Set up IV */ - if (ivlen < SEQ_NUM_SIZE) { - /* Should not happen */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - offset = ivlen - SEQ_NUM_SIZE; - memcpy(iv, staticiv, offset); - for (loop = 0; loop < SEQ_NUM_SIZE; loop++) - iv[offset + loop] = staticiv[offset + loop] ^ seq[loop]; - - if (!tls_increment_sequence_ctr(rl)) { - /* RLAYERfatal already called */ - return 0; - } - - if (EVP_CipherInit_ex(ctx, NULL, NULL, NULL, iv, sending) <= 0 - || (!sending && EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, - rl->taglen, - rec->data + rec->length) <= 0)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - /* Set up the AAD */ - if (!WPACKET_init_static_len(&wpkt, recheader, sizeof(recheader), 0) - || !WPACKET_put_bytes_u8(&wpkt, rec->type) - || !WPACKET_put_bytes_u16(&wpkt, rec->rec_version) - || !WPACKET_put_bytes_u16(&wpkt, rec->length + rl->taglen) - || !WPACKET_get_total_written(&wpkt, &hdrlen) - || hdrlen != SSL3_RT_HEADER_LENGTH - || !WPACKET_finish(&wpkt)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - WPACKET_cleanup(&wpkt); - return 0; - } - - /* - * For CCM we must explicitly set the total plaintext length before we add - * any AAD. - */ - if ((mode == EVP_CIPH_CCM_MODE - && EVP_CipherUpdate(ctx, NULL, &lenu, NULL, - (unsigned int)rec->length) <= 0) - || EVP_CipherUpdate(ctx, NULL, &lenu, recheader, - sizeof(recheader)) <= 0 - || EVP_CipherUpdate(ctx, rec->data, &lenu, rec->input, - (unsigned int)rec->length) <= 0 - || EVP_CipherFinal_ex(ctx, rec->data + lenu, &lenf) <= 0 - || (size_t)(lenu + lenf) != rec->length) { - return 0; - } - if (sending) { - /* Add the tag */ - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, rl->taglen, - rec->data + rec->length) <= 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - rec->length += rl->taglen; - } - - return 1; -} - -static int tls13_validate_record_header(OSSL_RECORD_LAYER *rl, - TLS_RL_RECORD *rec) -{ - if (rec->type != SSL3_RT_APPLICATION_DATA - && (rec->type != SSL3_RT_CHANGE_CIPHER_SPEC - || !rl->is_first_handshake) - && (rec->type != SSL3_RT_ALERT || !rl->allow_plain_alerts)) { - RLAYERfatal(rl, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_BAD_RECORD_TYPE); - return 0; - } - - if (rec->rec_version != TLS1_2_VERSION) { - RLAYERfatal(rl, SSL_AD_DECODE_ERROR, SSL_R_WRONG_VERSION_NUMBER); - return 0; - } - - if (rec->length > SSL3_RT_MAX_TLS13_ENCRYPTED_LENGTH) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, - SSL_R_ENCRYPTED_LENGTH_TOO_LONG); - return 0; - } - return 1; -} - -static int tls13_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec) -{ - /* Skip this if we've received a plaintext alert */ - if (rec->type != SSL3_RT_ALERT) { - size_t end; - - if (rec->length == 0 - || rec->type != SSL3_RT_APPLICATION_DATA) { - RLAYERfatal(rl, SSL_AD_UNEXPECTED_MESSAGE, - SSL_R_BAD_RECORD_TYPE); - return 0; - } - - /* Strip trailing padding */ - for (end = rec->length - 1; end > 0 && rec->data[end] == 0; end--) - continue; - - rec->length = end; - rec->type = rec->data[end]; - } - - if (rec->length > SSL3_RT_MAX_PLAIN_LENGTH) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, SSL_R_DATA_LENGTH_TOO_LONG); - return 0; - } - - if (!tls13_common_post_process_record(rl, rec)) { - /* RLAYERfatal already called */ - return 0; - } - - return 1; -} - -static uint8_t tls13_get_record_type(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *template) -{ - if (rl->allow_plain_alerts && template->type == SSL3_RT_ALERT) - return SSL3_RT_ALERT; - - /* - * Aside from the above case we always use the application data record type - * when encrypting in TLSv1.3. The "inner" record type encodes the "real" - * record type from the template. - */ - return SSL3_RT_APPLICATION_DATA; -} - -static int tls13_add_record_padding(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *thistempl, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr) -{ - size_t rlen; - - /* Nothing to be done in the case of a plaintext alert */ - if (rl->allow_plain_alerts && thistempl->type != SSL3_RT_ALERT) - return 1; - - if (!WPACKET_put_bytes_u8(thispkt, thistempl->type)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - TLS_RL_RECORD_add_length(thiswr, 1); - - /* Add TLS1.3 padding */ - rlen = TLS_RL_RECORD_get_length(thiswr); - if (rlen < rl->max_frag_len) { - size_t padding = 0; - size_t max_padding = rl->max_frag_len - rlen; - - if (rl->padding != NULL) { - padding = rl->padding(rl->cbarg, thistempl->type, rlen); - } else if (rl->block_padding > 0) { - size_t mask = rl->block_padding - 1; - size_t remainder; - - /* optimize for power of 2 */ - if ((rl->block_padding & mask) == 0) - remainder = rlen & mask; - else - remainder = rlen % rl->block_padding; - /* don't want to add a block of padding if we don't have to */ - if (remainder == 0) - padding = 0; - else - padding = rl->block_padding - remainder; - } - if (padding > 0) { - /* do not allow the record to exceed max plaintext length */ - if (padding > max_padding) - padding = max_padding; - if (!WPACKET_memset(thispkt, 0, padding)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, - ERR_R_INTERNAL_ERROR); - return 0; - } - TLS_RL_RECORD_add_length(thiswr, padding); - } - } - - return 1; -} - -const struct record_functions_st tls_1_3_funcs = { - tls13_set_crypto_state, - tls13_cipher, - NULL, - tls_default_set_protocol_version, - tls_default_read_n, - tls_get_more_records, - tls13_validate_record_header, - tls13_post_process_record, - tls_get_max_records_default, - tls_write_records_default, - tls_allocate_write_buffers_default, - tls_initialise_write_packets_default, - tls13_get_record_type, - tls_prepare_record_header_default, - tls13_add_record_padding, - tls_prepare_for_encryption_default, - tls_post_encryption_processing_default, - NULL -}; diff --git a/openssl/src/ssl/record/methods/tls1_meth.c b/openssl/src/ssl/record/methods/tls1_meth.c deleted file mode 100644 index 9275e19fb..000000000 --- a/openssl/src/ssl/record/methods/tls1_meth.c +++ /dev/null @@ -1,700 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include "internal/ssl3_cbc.h" -#include "../../ssl_local.h" -#include "../record_local.h" -#include "recmethod_local.h" - -static int tls1_set_crypto_state(OSSL_RECORD_LAYER *rl, int level, - unsigned char *key, size_t keylen, - unsigned char *iv, size_t ivlen, - unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, - size_t taglen, - int mactype, - const EVP_MD *md, - COMP_METHOD *comp) -{ - EVP_CIPHER_CTX *ciph_ctx; - EVP_PKEY *mac_key; - int enc = (rl->direction == OSSL_RECORD_DIRECTION_WRITE) ? 1 : 0; - - if (level != OSSL_RECORD_PROTECTION_LEVEL_APPLICATION) - return OSSL_RECORD_RETURN_FATAL; - - if ((rl->enc_ctx = EVP_CIPHER_CTX_new()) == NULL) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - return OSSL_RECORD_RETURN_FATAL; - } - - ciph_ctx = rl->enc_ctx; - - rl->md_ctx = EVP_MD_CTX_new(); - if (rl->md_ctx == NULL) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } -#ifndef OPENSSL_NO_COMP - if (comp != NULL) { - rl->compctx = COMP_CTX_new(comp); - if (rl->compctx == NULL) { - ERR_raise(ERR_LIB_SSL, SSL_R_COMPRESSION_LIBRARY_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - } -#endif - - /* - * If we have an AEAD Cipher, then there is no separate MAC, so we can skip - * setting up the MAC key. - */ - if ((EVP_CIPHER_get_flags(ciph) & EVP_CIPH_FLAG_AEAD_CIPHER) == 0) { - if (mactype == EVP_PKEY_HMAC) { - mac_key = EVP_PKEY_new_raw_private_key_ex(rl->libctx, "HMAC", - rl->propq, mackey, - mackeylen); - } else { - /* - * If its not HMAC then the only other types of MAC we support are - * the GOST MACs, so we need to use the old style way of creating - * a MAC key. - */ - mac_key = EVP_PKEY_new_mac_key(mactype, NULL, mackey, - (int)mackeylen); - } - if (mac_key == NULL - || EVP_DigestSignInit_ex(rl->md_ctx, NULL, EVP_MD_get0_name(md), - rl->libctx, rl->propq, mac_key, - NULL) <= 0) { - EVP_PKEY_free(mac_key); - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - EVP_PKEY_free(mac_key); - } - - if (EVP_CIPHER_get_mode(ciph) == EVP_CIPH_GCM_MODE) { - if (!EVP_CipherInit_ex(ciph_ctx, ciph, NULL, key, NULL, enc) - || EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_GCM_SET_IV_FIXED, - (int)ivlen, iv) <= 0) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - } else if (EVP_CIPHER_get_mode(ciph) == EVP_CIPH_CCM_MODE) { - if (!EVP_CipherInit_ex(ciph_ctx, ciph, NULL, NULL, NULL, enc) - || EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_AEAD_SET_IVLEN, 12, - NULL) <= 0 - || EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_AEAD_SET_TAG, - (int)taglen, NULL) <= 0 - || EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_CCM_SET_IV_FIXED, - (int)ivlen, iv) <= 0 - || !EVP_CipherInit_ex(ciph_ctx, NULL, NULL, key, NULL, enc)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - } else { - if (!EVP_CipherInit_ex(ciph_ctx, ciph, NULL, key, iv, enc)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - } - /* Needed for "composite" AEADs, such as RC4-HMAC-MD5 */ - if ((EVP_CIPHER_get_flags(ciph) & EVP_CIPH_FLAG_AEAD_CIPHER) != 0 - && mackeylen != 0 - && EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_AEAD_SET_MAC_KEY, - (int)mackeylen, mackey) <= 0) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - /* - * The cipher we actually ended up using in the EVP_CIPHER_CTX may be - * different to that in ciph if we have an ENGINE in use - */ - if (EVP_CIPHER_get0_provider(EVP_CIPHER_CTX_get0_cipher(ciph_ctx)) != NULL - && !ossl_set_tls_provider_parameters(rl, ciph_ctx, ciph, md)) { - /* ERR_raise already called */ - return OSSL_RECORD_RETURN_FATAL; - } - - /* Calculate the explicit IV length */ - if (RLAYER_USE_EXPLICIT_IV(rl)) { - int mode = EVP_CIPHER_CTX_get_mode(ciph_ctx); - int eivlen = 0; - - if (mode == EVP_CIPH_CBC_MODE) { - eivlen = EVP_CIPHER_CTX_get_iv_length(ciph_ctx); - if (eivlen < 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_LIBRARY_BUG); - return OSSL_RECORD_RETURN_FATAL; - } - if (eivlen <= 1) - eivlen = 0; - } else if (mode == EVP_CIPH_GCM_MODE) { - /* Need explicit part of IV for GCM mode */ - eivlen = EVP_GCM_TLS_EXPLICIT_IV_LEN; - } else if (mode == EVP_CIPH_CCM_MODE) { - eivlen = EVP_CCM_TLS_EXPLICIT_IV_LEN; - } - rl->eivlen = (size_t)eivlen; - } - - return OSSL_RECORD_RETURN_SUCCESS; -} - -#define MAX_PADDING 256 -/*- - * tls1_cipher encrypts/decrypts |n_recs| in |recs|. Calls RLAYERfatal on - * internal error, but not otherwise. It is the responsibility of the caller to - * report a bad_record_mac - if appropriate (DTLS just drops the record). - * - * Returns: - * 0: if the record is publicly invalid, or an internal error, or AEAD - * decryption failed, or Encrypt-then-mac decryption failed. - * 1: Success or Mac-then-encrypt decryption failed (MAC will be randomised) - */ -static int tls1_cipher(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *recs, - size_t n_recs, int sending, SSL_MAC_BUF *macs, - size_t macsize) -{ - EVP_CIPHER_CTX *ds; - size_t reclen[SSL_MAX_PIPELINES]; - unsigned char buf[SSL_MAX_PIPELINES][EVP_AEAD_TLS1_AAD_LEN]; - unsigned char *data[SSL_MAX_PIPELINES]; - int pad = 0, tmpr, provided; - size_t bs, ctr, padnum, loop; - unsigned char padval; - const EVP_CIPHER *enc; - - if (n_recs == 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (EVP_MD_CTX_get0_md(rl->md_ctx)) { - int n = EVP_MD_CTX_get_size(rl->md_ctx); - - if (!ossl_assert(n >= 0)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - } - ds = rl->enc_ctx; - if (!ossl_assert(rl->enc_ctx != NULL)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - enc = EVP_CIPHER_CTX_get0_cipher(rl->enc_ctx); - - if (sending) { - int ivlen; - - /* For TLSv1.1 and later explicit IV */ - if (RLAYER_USE_EXPLICIT_IV(rl) - && EVP_CIPHER_get_mode(enc) == EVP_CIPH_CBC_MODE) - ivlen = EVP_CIPHER_get_iv_length(enc); - else - ivlen = 0; - if (ivlen > 1) { - for (ctr = 0; ctr < n_recs; ctr++) { - if (recs[ctr].data != recs[ctr].input) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } else if (RAND_bytes_ex(rl->libctx, recs[ctr].input, - ivlen, 0) <= 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - } - } - } - if (!ossl_assert(enc != NULL)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - provided = (EVP_CIPHER_get0_provider(enc) != NULL); - - bs = EVP_CIPHER_get_block_size(EVP_CIPHER_CTX_get0_cipher(ds)); - - if (bs == 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_CIPHER); - return 0; - } - - if (n_recs > 1) { - if ((EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(ds)) - & EVP_CIPH_FLAG_PIPELINE) == 0) { - /* - * We shouldn't have been called with pipeline data if the - * cipher doesn't support pipelining - */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_PIPELINE_FAILURE); - return 0; - } - } - for (ctr = 0; ctr < n_recs; ctr++) { - reclen[ctr] = recs[ctr].length; - - if ((EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(ds)) - & EVP_CIPH_FLAG_AEAD_CIPHER) != 0) { - unsigned char *seq; - - seq = rl->sequence; - - if (rl->isdtls) { - unsigned char dtlsseq[8], *p = dtlsseq; - - s2n(rl->epoch, p); - memcpy(p, &seq[2], 6); - memcpy(buf[ctr], dtlsseq, 8); - } else { - memcpy(buf[ctr], seq, 8); - if (!tls_increment_sequence_ctr(rl)) { - /* RLAYERfatal already called */ - return 0; - } - } - - buf[ctr][8] = recs[ctr].type; - buf[ctr][9] = (unsigned char)(rl->version >> 8); - buf[ctr][10] = (unsigned char)(rl->version); - buf[ctr][11] = (unsigned char)(recs[ctr].length >> 8); - buf[ctr][12] = (unsigned char)(recs[ctr].length & 0xff); - pad = EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_AEAD_TLS1_AAD, - EVP_AEAD_TLS1_AAD_LEN, buf[ctr]); - if (pad <= 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (sending) { - reclen[ctr] += pad; - recs[ctr].length += pad; - } - } else if ((bs != 1) && sending && !provided) { - /* - * We only do this for legacy ciphers. Provided ciphers add the - * padding on the provider side. - */ - padnum = bs - (reclen[ctr] % bs); - - /* Add weird padding of up to 256 bytes */ - - if (padnum > MAX_PADDING) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - /* we need to add 'padnum' padding bytes of value padval */ - padval = (unsigned char)(padnum - 1); - for (loop = reclen[ctr]; loop < reclen[ctr] + padnum; loop++) - recs[ctr].input[loop] = padval; - reclen[ctr] += padnum; - recs[ctr].length += padnum; - } - - if (!sending) { - if (reclen[ctr] == 0 || reclen[ctr] % bs != 0) { - /* Publicly invalid */ - return 0; - } - } - } - if (n_recs > 1) { - /* Set the output buffers */ - for (ctr = 0; ctr < n_recs; ctr++) - data[ctr] = recs[ctr].data; - - if (EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_SET_PIPELINE_OUTPUT_BUFS, - (int)n_recs, data) <= 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_PIPELINE_FAILURE); - return 0; - } - /* Set the input buffers */ - for (ctr = 0; ctr < n_recs; ctr++) - data[ctr] = recs[ctr].input; - - if (EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_SET_PIPELINE_INPUT_BUFS, - (int)n_recs, data) <= 0 - || EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_SET_PIPELINE_INPUT_LENS, - (int)n_recs, reclen) <= 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_PIPELINE_FAILURE); - return 0; - } - } - - if (!rl->isdtls && rl->tlstree) { - int decrement_seq = 0; - - /* - * When sending, seq is incremented after MAC calculation. - * So if we are in ETM mode, we use seq 'as is' in the ctrl-function. - * Otherwise we have to decrease it in the implementation - */ - if (sending && !rl->use_etm) - decrement_seq = 1; - - if (EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_TLSTREE, decrement_seq, - rl->sequence) <= 0) { - - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - } - - if (provided) { - int outlen; - - /* Provided cipher - we do not support pipelining on this path */ - if (n_recs > 1) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (!EVP_CipherUpdate(ds, recs[0].data, &outlen, recs[0].input, - (unsigned int)reclen[0])) - return 0; - recs[0].length = outlen; - - /* - * The length returned from EVP_CipherUpdate above is the actual - * payload length. We need to adjust the data/input ptr to skip over - * any explicit IV - */ - if (!sending) { - if (EVP_CIPHER_get_mode(enc) == EVP_CIPH_GCM_MODE) { - recs[0].data += EVP_GCM_TLS_EXPLICIT_IV_LEN; - recs[0].input += EVP_GCM_TLS_EXPLICIT_IV_LEN; - } else if (EVP_CIPHER_get_mode(enc) == EVP_CIPH_CCM_MODE) { - recs[0].data += EVP_CCM_TLS_EXPLICIT_IV_LEN; - recs[0].input += EVP_CCM_TLS_EXPLICIT_IV_LEN; - } else if (bs != 1 && RLAYER_USE_EXPLICIT_IV(rl)) { - recs[0].data += bs; - recs[0].input += bs; - recs[0].orig_len -= bs; - } - - /* Now get a pointer to the MAC (if applicable) */ - if (macs != NULL) { - OSSL_PARAM params[2], *p = params; - - /* Get the MAC */ - macs[0].alloced = 0; - - *p++ = OSSL_PARAM_construct_octet_ptr(OSSL_CIPHER_PARAM_TLS_MAC, - (void **)&macs[0].mac, - macsize); - *p = OSSL_PARAM_construct_end(); - - if (!EVP_CIPHER_CTX_get_params(ds, params)) { - /* Shouldn't normally happen */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, - ERR_R_INTERNAL_ERROR); - return 0; - } - } - } - } else { - /* Legacy cipher */ - - tmpr = EVP_Cipher(ds, recs[0].data, recs[0].input, - (unsigned int)reclen[0]); - if ((EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(ds)) - & EVP_CIPH_FLAG_CUSTOM_CIPHER) != 0 - ? (tmpr < 0) - : (tmpr == 0)) { - /* AEAD can fail to verify MAC */ - return 0; - } - - if (!sending) { - for (ctr = 0; ctr < n_recs; ctr++) { - /* Adjust the record to remove the explicit IV/MAC/Tag */ - if (EVP_CIPHER_get_mode(enc) == EVP_CIPH_GCM_MODE) { - recs[ctr].data += EVP_GCM_TLS_EXPLICIT_IV_LEN; - recs[ctr].input += EVP_GCM_TLS_EXPLICIT_IV_LEN; - recs[ctr].length -= EVP_GCM_TLS_EXPLICIT_IV_LEN; - } else if (EVP_CIPHER_get_mode(enc) == EVP_CIPH_CCM_MODE) { - recs[ctr].data += EVP_CCM_TLS_EXPLICIT_IV_LEN; - recs[ctr].input += EVP_CCM_TLS_EXPLICIT_IV_LEN; - recs[ctr].length -= EVP_CCM_TLS_EXPLICIT_IV_LEN; - } else if (bs != 1 && RLAYER_USE_EXPLICIT_IV(rl)) { - if (recs[ctr].length < bs) - return 0; - recs[ctr].data += bs; - recs[ctr].input += bs; - recs[ctr].length -= bs; - recs[ctr].orig_len -= bs; - } - - /* - * If using Mac-then-encrypt, then this will succeed but - * with a random MAC if padding is invalid - */ - if (!tls1_cbc_remove_padding_and_mac(&recs[ctr].length, - recs[ctr].orig_len, - recs[ctr].data, - (macs != NULL) ? &macs[ctr].mac : NULL, - (macs != NULL) ? &macs[ctr].alloced - : NULL, - bs, - pad ? (size_t)pad : macsize, - (EVP_CIPHER_get_flags(enc) - & EVP_CIPH_FLAG_AEAD_CIPHER) != 0, - rl->libctx)) - return 0; - } - } - } - return 1; -} - -static int tls1_mac(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec, unsigned char *md, - int sending) -{ - unsigned char *seq = rl->sequence; - EVP_MD_CTX *hash; - size_t md_size; - EVP_MD_CTX *hmac = NULL, *mac_ctx; - unsigned char header[13]; - int t; - int ret = 0; - - hash = rl->md_ctx; - - t = EVP_MD_CTX_get_size(hash); - if (!ossl_assert(t >= 0)) - return 0; - md_size = t; - - if (rl->stream_mac) { - mac_ctx = hash; - } else { - hmac = EVP_MD_CTX_new(); - if (hmac == NULL || !EVP_MD_CTX_copy(hmac, hash)) { - goto end; - } - mac_ctx = hmac; - } - - if (!rl->isdtls - && rl->tlstree - && EVP_MD_CTX_ctrl(mac_ctx, EVP_MD_CTRL_TLSTREE, 0, seq) <= 0) - goto end; - - if (rl->isdtls) { - unsigned char dtlsseq[8], *p = dtlsseq; - - s2n(rl->epoch, p); - memcpy(p, &seq[2], 6); - - memcpy(header, dtlsseq, 8); - } else { - memcpy(header, seq, 8); - } - - header[8] = rec->type; - header[9] = (unsigned char)(rl->version >> 8); - header[10] = (unsigned char)(rl->version); - header[11] = (unsigned char)(rec->length >> 8); - header[12] = (unsigned char)(rec->length & 0xff); - - if (!sending && !rl->use_etm - && EVP_CIPHER_CTX_get_mode(rl->enc_ctx) == EVP_CIPH_CBC_MODE - && ssl3_cbc_record_digest_supported(mac_ctx)) { - OSSL_PARAM tls_hmac_params[2], *p = tls_hmac_params; - - *p++ = OSSL_PARAM_construct_size_t(OSSL_MAC_PARAM_TLS_DATA_SIZE, - &rec->orig_len); - *p++ = OSSL_PARAM_construct_end(); - - if (!EVP_PKEY_CTX_set_params(EVP_MD_CTX_get_pkey_ctx(mac_ctx), - tls_hmac_params)) - goto end; - } - - if (EVP_DigestSignUpdate(mac_ctx, header, sizeof(header)) <= 0 - || EVP_DigestSignUpdate(mac_ctx, rec->input, rec->length) <= 0 - || EVP_DigestSignFinal(mac_ctx, md, &md_size) <= 0) - goto end; - - OSSL_TRACE_BEGIN(TLS) { - BIO_printf(trc_out, "seq:\n"); - BIO_dump_indent(trc_out, seq, 8, 4); - BIO_printf(trc_out, "rec:\n"); - BIO_dump_indent(trc_out, rec->data, rec->length, 4); - } OSSL_TRACE_END(TLS); - - if (!rl->isdtls && !tls_increment_sequence_ctr(rl)) { - /* RLAYERfatal already called */ - goto end; - } - - OSSL_TRACE_BEGIN(TLS) { - BIO_printf(trc_out, "md:\n"); - BIO_dump_indent(trc_out, md, md_size, 4); - } OSSL_TRACE_END(TLS); - ret = 1; - end: - EVP_MD_CTX_free(hmac); - return ret; -} - -#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD != 0 -# ifndef OPENSSL_NO_COMP -# define MAX_PREFIX_LEN ((SSL3_ALIGN_PAYLOAD - 1) \ - + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD \ - + SSL3_RT_HEADER_LENGTH \ - + SSL3_RT_MAX_COMPRESSED_OVERHEAD) -# else -# define MAX_PREFIX_LEN ((SSL3_ALIGN_PAYLOAD - 1) \ - + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD \ - + SSL3_RT_HEADER_LENGTH) -# endif /* OPENSSL_NO_COMP */ -#else -# ifndef OPENSSL_NO_COMP -# define MAX_PREFIX_LEN (SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD \ - + SSL3_RT_HEADER_LENGTH \ - + SSL3_RT_MAX_COMPRESSED_OVERHEAD) -# else -# define MAX_PREFIX_LEN (SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD \ - + SSL3_RT_HEADER_LENGTH) -# endif /* OPENSSL_NO_COMP */ -#endif - -/* This function is also used by the SSLv3 implementation */ -int tls1_allocate_write_buffers(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, size_t *prefix) -{ - /* Do we need to add an empty record prefix? */ - *prefix = rl->need_empty_fragments - && templates[0].type == SSL3_RT_APPLICATION_DATA; - - /* - * In the prefix case we can allocate a much smaller buffer. Otherwise we - * just allocate the default buffer size - */ - if (!tls_setup_write_buffer(rl, numtempl + *prefix, - *prefix ? MAX_PREFIX_LEN : 0, 0)) { - /* RLAYERfatal() already called */ - return 0; - } - - return 1; -} - -/* This function is also used by the SSLv3 implementation */ -int tls1_initialise_write_packets(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, - OSSL_RECORD_TEMPLATE *prefixtempl, - WPACKET *pkt, - TLS_BUFFER *bufs, - size_t *wpinited) -{ - size_t align = 0; - TLS_BUFFER *wb; - size_t prefix; - - /* Do we need to add an empty record prefix? */ - prefix = rl->need_empty_fragments - && templates[0].type == SSL3_RT_APPLICATION_DATA; - - if (prefix) { - /* - * countermeasure against known-IV weakness in CBC ciphersuites (see - * http://www.openssl.org/~bodo/tls-cbc.txt) - */ - prefixtempl->buf = NULL; - prefixtempl->version = templates[0].version; - prefixtempl->buflen = 0; - prefixtempl->type = SSL3_RT_APPLICATION_DATA; - - wb = &bufs[0]; - -#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD != 0 - align = (size_t)TLS_BUFFER_get_buf(wb) + SSL3_RT_HEADER_LENGTH; - align = SSL3_ALIGN_PAYLOAD - 1 - - ((align - 1) % SSL3_ALIGN_PAYLOAD); -#endif - TLS_BUFFER_set_offset(wb, align); - - if (!WPACKET_init_static_len(&pkt[0], TLS_BUFFER_get_buf(wb), - TLS_BUFFER_get_len(wb), 0)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - *wpinited = 1; - if (!WPACKET_allocate_bytes(&pkt[0], align, NULL)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - } - - return tls_initialise_write_packets_default(rl, templates, numtempl, - NULL, - pkt + prefix, bufs + prefix, - wpinited); -} - -/* TLSv1.0, TLSv1.1 and TLSv1.2 all use the same funcs */ -const struct record_functions_st tls_1_funcs = { - tls1_set_crypto_state, - tls1_cipher, - tls1_mac, - tls_default_set_protocol_version, - tls_default_read_n, - tls_get_more_records, - tls_default_validate_record_header, - tls_default_post_process_record, - tls_get_max_records_multiblock, - tls_write_records_multiblock, /* Defined in tls_multib.c */ - tls1_allocate_write_buffers, - tls1_initialise_write_packets, - NULL, - tls_prepare_record_header_default, - NULL, - tls_prepare_for_encryption_default, - tls_post_encryption_processing_default, - NULL -}; - -const struct record_functions_st dtls_1_funcs = { - tls1_set_crypto_state, - tls1_cipher, - tls1_mac, - tls_default_set_protocol_version, - tls_default_read_n, - dtls_get_more_records, - NULL, - NULL, - NULL, - tls_write_records_default, - /* - * Don't use tls1_allocate_write_buffers since that handles empty fragment - * records which aren't needed in DTLS. We just use the default allocation - * instead. - */ - tls_allocate_write_buffers_default, - /* Don't use tls1_initialise_write_packets for same reason as above */ - tls_initialise_write_packets_default, - NULL, - dtls_prepare_record_header, - NULL, - tls_prepare_for_encryption_default, - dtls_post_encryption_processing, - NULL -}; diff --git a/openssl/src/ssl/record/methods/tls_common.c b/openssl/src/ssl/record/methods/tls_common.c deleted file mode 100644 index 6cb8e8870..000000000 --- a/openssl/src/ssl/record/methods/tls_common.c +++ /dev/null @@ -1,2170 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include -#include -#include -#include -#include -#include -#include "internal/e_os.h" -#include "internal/packet.h" -#include "internal/ssl3_cbc.h" -#include "../../ssl_local.h" -#include "../record_local.h" -#include "recmethod_local.h" - -static void tls_int_free(OSSL_RECORD_LAYER *rl); - -void ossl_tls_buffer_release(TLS_BUFFER *b) -{ - OPENSSL_free(b->buf); - b->buf = NULL; -} - -static void TLS_RL_RECORD_release(TLS_RL_RECORD *r, size_t num_recs) -{ - size_t i; - - for (i = 0; i < num_recs; i++) { - OPENSSL_free(r[i].comp); - r[i].comp = NULL; - } -} - -void ossl_tls_rl_record_set_seq_num(TLS_RL_RECORD *r, - const unsigned char *seq_num) -{ - memcpy(r->seq_num, seq_num, SEQ_NUM_SIZE); -} - -void ossl_rlayer_fatal(OSSL_RECORD_LAYER *rl, int al, int reason, - const char *fmt, ...) -{ - va_list args; - - va_start(args, fmt); - ERR_vset_error(ERR_LIB_SSL, reason, fmt, args); - va_end(args); - - rl->alert = al; -} - -int ossl_set_tls_provider_parameters(OSSL_RECORD_LAYER *rl, - EVP_CIPHER_CTX *ctx, - const EVP_CIPHER *ciph, - const EVP_MD *md) -{ - /* - * Provided cipher, the TLS padding/MAC removal is performed provider - * side so we need to tell the ctx about our TLS version and mac size - */ - OSSL_PARAM params[3], *pprm = params; - size_t macsize = 0; - int imacsize = -1; - - if ((EVP_CIPHER_get_flags(ciph) & EVP_CIPH_FLAG_AEAD_CIPHER) == 0 - && !rl->use_etm) - imacsize = EVP_MD_get_size(md); - if (imacsize >= 0) - macsize = (size_t)imacsize; - - *pprm++ = OSSL_PARAM_construct_int(OSSL_CIPHER_PARAM_TLS_VERSION, - &rl->version); - *pprm++ = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_TLS_MAC_SIZE, - &macsize); - *pprm = OSSL_PARAM_construct_end(); - - if (!EVP_CIPHER_CTX_set_params(ctx, params)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - return 1; -} - -/* - * ssl3_cbc_record_digest_supported returns 1 iff |ctx| uses a hash function - * which ssl3_cbc_digest_record supports. - */ -char ssl3_cbc_record_digest_supported(const EVP_MD_CTX *ctx) -{ - switch (EVP_MD_CTX_get_type(ctx)) { - case NID_md5: - case NID_sha1: - case NID_sha224: - case NID_sha256: - case NID_sha384: - case NID_sha512: - return 1; - default: - return 0; - } -} - -#ifndef OPENSSL_NO_COMP -static int tls_allow_compression(OSSL_RECORD_LAYER *rl) -{ - if (rl->options & SSL_OP_NO_COMPRESSION) - return 0; - - return rl->security == NULL - || rl->security(rl->cbarg, SSL_SECOP_COMPRESSION, 0, 0, NULL); -} -#endif - -static void tls_release_write_buffer_int(OSSL_RECORD_LAYER *rl, size_t start) -{ - TLS_BUFFER *wb; - size_t pipes; - - pipes = rl->numwpipes; - - while (pipes > start) { - wb = &rl->wbuf[pipes - 1]; - - if (TLS_BUFFER_is_app_buffer(wb)) - TLS_BUFFER_set_app_buffer(wb, 0); - else - OPENSSL_free(wb->buf); - wb->buf = NULL; - pipes--; - } -} - -int tls_setup_write_buffer(OSSL_RECORD_LAYER *rl, size_t numwpipes, - size_t firstlen, size_t nextlen) -{ - unsigned char *p; - size_t align = 0, headerlen; - TLS_BUFFER *wb; - size_t currpipe; - size_t defltlen = 0; - size_t contenttypelen = 0; - - if (firstlen == 0 || (numwpipes > 1 && nextlen == 0)) { - if (rl->isdtls) - headerlen = DTLS1_RT_HEADER_LENGTH + 1; - else - headerlen = SSL3_RT_HEADER_LENGTH; - - /* TLSv1.3 adds an extra content type byte after payload data */ - if (rl->version == TLS1_3_VERSION) - contenttypelen = 1; - -#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD != 0 - align = SSL3_ALIGN_PAYLOAD - 1; -#endif - - defltlen = align + headerlen + rl->eivlen + rl->max_frag_len - + contenttypelen + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD; -#ifndef OPENSSL_NO_COMP - if (tls_allow_compression(rl)) - defltlen += SSL3_RT_MAX_COMPRESSED_OVERHEAD; -#endif - /* - * We don't need to add eivlen here since empty fragments only occur - * when we don't have an explicit IV. The contenttype byte will also - * always be 0 in these protocol versions - */ - if ((rl->options & SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) == 0) - defltlen += headerlen + align + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD; - } - - wb = rl->wbuf; - for (currpipe = 0; currpipe < numwpipes; currpipe++) { - TLS_BUFFER *thiswb = &wb[currpipe]; - size_t len = (currpipe == 0) ? firstlen : nextlen; - - if (len == 0) - len = defltlen; - - if (thiswb->len != len) { - OPENSSL_free(thiswb->buf); - thiswb->buf = NULL; /* force reallocation */ - } - - p = thiswb->buf; - if (p == NULL) { - p = OPENSSL_malloc(len); - if (p == NULL) { - if (rl->numwpipes < currpipe) - rl->numwpipes = currpipe; - /* - * We've got a malloc failure, and we're still initialising - * buffers. We assume we're so doomed that we won't even be able - * to send an alert. - */ - RLAYERfatal(rl, SSL_AD_NO_ALERT, ERR_R_CRYPTO_LIB); - return 0; - } - } - memset(thiswb, 0, sizeof(TLS_BUFFER)); - thiswb->buf = p; - thiswb->len = len; - } - - /* Free any previously allocated buffers that we are no longer using */ - tls_release_write_buffer_int(rl, currpipe); - - rl->numwpipes = numwpipes; - - return 1; -} - -static void tls_release_write_buffer(OSSL_RECORD_LAYER *rl) -{ - tls_release_write_buffer_int(rl, 0); - - rl->numwpipes = 0; -} - -int tls_setup_read_buffer(OSSL_RECORD_LAYER *rl) -{ - unsigned char *p; - size_t len, align = 0, headerlen; - TLS_BUFFER *b; - - b = &rl->rbuf; - - if (rl->isdtls) - headerlen = DTLS1_RT_HEADER_LENGTH; - else - headerlen = SSL3_RT_HEADER_LENGTH; - -#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD != 0 - align = (-SSL3_RT_HEADER_LENGTH) & (SSL3_ALIGN_PAYLOAD - 1); -#endif - - if (b->buf == NULL) { - len = rl->max_frag_len - + SSL3_RT_MAX_ENCRYPTED_OVERHEAD + headerlen + align; -#ifndef OPENSSL_NO_COMP - if (tls_allow_compression(rl)) - len += SSL3_RT_MAX_COMPRESSED_OVERHEAD; -#endif - - /* Ensure our buffer is large enough to support all our pipelines */ - if (rl->max_pipelines > 1) - len *= rl->max_pipelines; - - if (b->default_len > len) - len = b->default_len; - - if ((p = OPENSSL_malloc(len)) == NULL) { - /* - * We've got a malloc failure, and we're still initialising buffers. - * We assume we're so doomed that we won't even be able to send an - * alert. - */ - RLAYERfatal(rl, SSL_AD_NO_ALERT, ERR_R_CRYPTO_LIB); - return 0; - } - b->buf = p; - b->len = len; - } - - return 1; -} - -static int tls_release_read_buffer(OSSL_RECORD_LAYER *rl) -{ - TLS_BUFFER *b; - - b = &rl->rbuf; - if ((rl->options & SSL_OP_CLEANSE_PLAINTEXT) != 0) - OPENSSL_cleanse(b->buf, b->len); - OPENSSL_free(b->buf); - b->buf = NULL; - rl->packet = NULL; - rl->packet_length = 0; - return 1; -} - -/* - * Return values are as per SSL_read() - */ -int tls_default_read_n(OSSL_RECORD_LAYER *rl, size_t n, size_t max, int extend, - int clearold, size_t *readbytes) -{ - /* - * If extend == 0, obtain new n-byte packet; if extend == 1, increase - * packet by another n bytes. The packet will be in the sub-array of - * rl->rbuf.buf specified by rl->packet and rl->packet_length. (If - * rl->read_ahead is set, 'max' bytes may be stored in rbuf [plus - * rl->packet_length bytes if extend == 1].) if clearold == 1, move the - * packet to the start of the buffer; if clearold == 0 then leave any old - * packets where they were - */ - size_t len, left, align = 0; - unsigned char *pkt; - TLS_BUFFER *rb; - - if (n == 0) - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - - rb = &rl->rbuf; - left = rb->left; -#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD != 0 - align = (size_t)rb->buf + SSL3_RT_HEADER_LENGTH; - align = SSL3_ALIGN_PAYLOAD - 1 - ((align - 1) % SSL3_ALIGN_PAYLOAD); -#endif - - if (!extend) { - /* start with empty packet ... */ - if (left == 0) - rb->offset = align; - - rl->packet = rb->buf + rb->offset; - rl->packet_length = 0; - /* ... now we can act as if 'extend' was set */ - } - - if (!ossl_assert(rl->packet != NULL)) { - /* does not happen */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - len = rl->packet_length; - pkt = rb->buf + align; - /* - * Move any available bytes to front of buffer: 'len' bytes already - * pointed to by 'packet', 'left' extra ones at the end - */ - if (rl->packet != pkt && clearold == 1) { - memmove(pkt, rl->packet, len + left); - rl->packet = pkt; - rb->offset = len + align; - } - - /* - * For DTLS/UDP reads should not span multiple packets because the read - * operation returns the whole packet at once (as long as it fits into - * the buffer). - */ - if (rl->isdtls) { - if (left == 0 && extend) { - /* - * We received a record with a header but no body data. This will - * get dumped. - */ - return OSSL_RECORD_RETURN_NON_FATAL_ERR; - } - if (left > 0 && n > left) - n = left; - } - - /* if there is enough in the buffer from a previous read, take some */ - if (left >= n) { - rl->packet_length += n; - rb->left = left - n; - rb->offset += n; - *readbytes = n; - return OSSL_RECORD_RETURN_SUCCESS; - } - - /* else we need to read more data */ - - if (n > rb->len - rb->offset) { - /* does not happen */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - /* We always act like read_ahead is set for DTLS */ - if (!rl->read_ahead && !rl->isdtls) { - /* ignore max parameter */ - max = n; - } else { - if (max < n) - max = n; - if (max > rb->len - rb->offset) - max = rb->len - rb->offset; - } - - while (left < n) { - size_t bioread = 0; - int ret; - BIO *bio = rl->prev != NULL ? rl->prev : rl->bio; - - /* - * Now we have len+left bytes at the front of rl->rbuf.buf and - * need to read in more until we have len + n (up to len + max if - * possible) - */ - - clear_sys_error(); - if (bio != NULL) { - ret = BIO_read(bio, pkt + len + left, max - left); - if (ret > 0) { - bioread = ret; - ret = OSSL_RECORD_RETURN_SUCCESS; - } else if (BIO_should_retry(bio)) { - if (rl->prev != NULL) { - /* - * We were reading from the previous epoch. Now there is no - * more data, so swap to the actual transport BIO - */ - BIO_free(rl->prev); - rl->prev = NULL; - continue; - } - ret = OSSL_RECORD_RETURN_RETRY; - } else if (BIO_eof(bio)) { - ret = OSSL_RECORD_RETURN_EOF; - } else { - ret = OSSL_RECORD_RETURN_FATAL; - } - } else { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_READ_BIO_NOT_SET); - ret = OSSL_RECORD_RETURN_FATAL; - } - - if (ret <= OSSL_RECORD_RETURN_RETRY) { - rb->left = left; - if ((rl->mode & SSL_MODE_RELEASE_BUFFERS) != 0 && !rl->isdtls) - if (len + left == 0) - tls_release_read_buffer(rl); - return ret; - } - left += bioread; - /* - * reads should *never* span multiple packets for DTLS because the - * underlying transport protocol is message oriented as opposed to - * byte oriented as in the TLS case. - */ - if (rl->isdtls) { - if (n > left) - n = left; /* makes the while condition false */ - } - } - - /* done reading, now the book-keeping */ - rb->offset += n; - rb->left = left - n; - rl->packet_length += n; - *readbytes = n; - return OSSL_RECORD_RETURN_SUCCESS; -} - -/* - * Peeks ahead into "read_ahead" data to see if we have a whole record waiting - * for us in the buffer. - */ -static int tls_record_app_data_waiting(OSSL_RECORD_LAYER *rl) -{ - TLS_BUFFER *rbuf; - size_t left, len; - unsigned char *p; - - rbuf = &rl->rbuf; - - p = TLS_BUFFER_get_buf(rbuf); - if (p == NULL) - return 0; - - left = TLS_BUFFER_get_left(rbuf); - - if (left < SSL3_RT_HEADER_LENGTH) - return 0; - - p += TLS_BUFFER_get_offset(rbuf); - - /* - * We only check the type and record length, we will sanity check version - * etc later - */ - if (*p != SSL3_RT_APPLICATION_DATA) - return 0; - - p += 3; - n2s(p, len); - - if (left < SSL3_RT_HEADER_LENGTH + len) - return 0; - - return 1; -} - -static int rlayer_early_data_count_ok(OSSL_RECORD_LAYER *rl, size_t length, - size_t overhead, int send) -{ - uint32_t max_early_data = rl->max_early_data; - - if (max_early_data == 0) { - RLAYERfatal(rl, send ? SSL_AD_INTERNAL_ERROR : SSL_AD_UNEXPECTED_MESSAGE, - SSL_R_TOO_MUCH_EARLY_DATA); - return 0; - } - - /* If we are dealing with ciphertext we need to allow for the overhead */ - max_early_data += overhead; - - if (rl->early_data_count + length > max_early_data) { - RLAYERfatal(rl, send ? SSL_AD_INTERNAL_ERROR : SSL_AD_UNEXPECTED_MESSAGE, - SSL_R_TOO_MUCH_EARLY_DATA); - return 0; - } - rl->early_data_count += length; - - return 1; -} - -/* - * MAX_EMPTY_RECORDS defines the number of consecutive, empty records that - * will be processed per call to tls_get_more_records. Without this limit an - * attacker could send empty records at a faster rate than we can process and - * cause tls_get_more_records to loop forever. - */ -#define MAX_EMPTY_RECORDS 32 - -#define SSL2_RT_HEADER_LENGTH 2 - -/*- - * Call this to buffer new input records in rl->rrec. - * It will return a OSSL_RECORD_RETURN_* value. - * When it finishes successfully (OSSL_RECORD_RETURN_SUCCESS), |rl->num_recs| - * records have been decoded. For each record 'i': - * rrec[i].type - is the type of record - * rrec[i].data, - data - * rrec[i].length, - number of bytes - * Multiple records will only be returned if the record types are all - * SSL3_RT_APPLICATION_DATA. The number of records returned will always be <= - * |max_pipelines| - */ -int tls_get_more_records(OSSL_RECORD_LAYER *rl) -{ - int enc_err, rret; - int i; - size_t more, n; - TLS_RL_RECORD *rr, *thisrr; - TLS_BUFFER *rbuf; - unsigned char *p; - unsigned char md[EVP_MAX_MD_SIZE]; - unsigned int version; - size_t mac_size = 0; - int imac_size; - size_t num_recs = 0, max_recs, j; - PACKET pkt, sslv2pkt; - SSL_MAC_BUF *macbufs = NULL; - int ret = OSSL_RECORD_RETURN_FATAL; - - rr = rl->rrec; - rbuf = &rl->rbuf; - if (rbuf->buf == NULL) { - if (!tls_setup_read_buffer(rl)) { - /* RLAYERfatal() already called */ - return OSSL_RECORD_RETURN_FATAL; - } - } - - max_recs = rl->max_pipelines; - - if (max_recs == 0) - max_recs = 1; - - do { - thisrr = &rr[num_recs]; - - /* check if we have the header */ - if ((rl->rstate != SSL_ST_READ_BODY) || - (rl->packet_length < SSL3_RT_HEADER_LENGTH)) { - size_t sslv2len; - unsigned int type; - - rret = rl->funcs->read_n(rl, SSL3_RT_HEADER_LENGTH, - TLS_BUFFER_get_len(rbuf), 0, - num_recs == 0 ? 1 : 0, &n); - - if (rret < OSSL_RECORD_RETURN_SUCCESS) - return rret; /* error or non-blocking */ - - rl->rstate = SSL_ST_READ_BODY; - - p = rl->packet; - if (!PACKET_buf_init(&pkt, p, rl->packet_length)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - sslv2pkt = pkt; - if (!PACKET_get_net_2_len(&sslv2pkt, &sslv2len) - || !PACKET_get_1(&sslv2pkt, &type)) { - RLAYERfatal(rl, SSL_AD_DECODE_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - /* - * The first record received by the server may be a V2ClientHello. - */ - if (rl->role == OSSL_RECORD_ROLE_SERVER - && rl->is_first_record - && (sslv2len & 0x8000) != 0 - && (type == SSL2_MT_CLIENT_HELLO)) { - /* - * SSLv2 style record - * - * |num_recs| here will actually always be 0 because - * |num_recs > 0| only ever occurs when we are processing - * multiple app data records - which we know isn't the case here - * because it is an SSLv2ClientHello. We keep it using - * |num_recs| for the sake of consistency - */ - thisrr->type = SSL3_RT_HANDSHAKE; - thisrr->rec_version = SSL2_VERSION; - - thisrr->length = sslv2len & 0x7fff; - - if (thisrr->length > TLS_BUFFER_get_len(rbuf) - - SSL2_RT_HEADER_LENGTH) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, - SSL_R_PACKET_LENGTH_TOO_LONG); - return OSSL_RECORD_RETURN_FATAL; - } - } else { - /* SSLv3+ style record */ - - /* Pull apart the header into the TLS_RL_RECORD */ - if (!PACKET_get_1(&pkt, &type) - || !PACKET_get_net_2(&pkt, &version) - || !PACKET_get_net_2_len(&pkt, &thisrr->length)) { - if (rl->msg_callback != NULL) - rl->msg_callback(0, 0, SSL3_RT_HEADER, p, 5, rl->cbarg); - RLAYERfatal(rl, SSL_AD_DECODE_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - thisrr->type = type; - thisrr->rec_version = version; - - /* - * When we call validate_record_header() only records actually - * received in SSLv2 format should have the record version set - * to SSL2_VERSION. This way validate_record_header() can know - * what format the record was in based on the version. - */ - if (thisrr->rec_version == SSL2_VERSION) { - RLAYERfatal(rl, SSL_AD_PROTOCOL_VERSION, - SSL_R_WRONG_VERSION_NUMBER); - return OSSL_RECORD_RETURN_FATAL; - } - - if (rl->msg_callback != NULL) - rl->msg_callback(0, version, SSL3_RT_HEADER, p, 5, rl->cbarg); - - if (thisrr->length > - TLS_BUFFER_get_len(rbuf) - SSL3_RT_HEADER_LENGTH) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, - SSL_R_PACKET_LENGTH_TOO_LONG); - return OSSL_RECORD_RETURN_FATAL; - } - } - - if (!rl->funcs->validate_record_header(rl, thisrr)) { - /* RLAYERfatal already called */ - return OSSL_RECORD_RETURN_FATAL; - } - - /* now rl->rstate == SSL_ST_READ_BODY */ - } - - /* - * rl->rstate == SSL_ST_READ_BODY, get and decode the data. Calculate - * how much more data we need to read for the rest of the record - */ - if (thisrr->rec_version == SSL2_VERSION) { - more = thisrr->length + SSL2_RT_HEADER_LENGTH - - SSL3_RT_HEADER_LENGTH; - } else { - more = thisrr->length; - } - - if (more > 0) { - /* now rl->packet_length == SSL3_RT_HEADER_LENGTH */ - - rret = rl->funcs->read_n(rl, more, more, 1, 0, &n); - if (rret < OSSL_RECORD_RETURN_SUCCESS) - return rret; /* error or non-blocking io */ - } - - /* set state for later operations */ - rl->rstate = SSL_ST_READ_HEADER; - - /* - * At this point, rl->packet_length == SSL3_RT_HEADER_LENGTH - * + thisrr->length, or rl->packet_length == SSL2_RT_HEADER_LENGTH - * + thisrr->length and we have that many bytes in rl->packet - */ - if (thisrr->rec_version == SSL2_VERSION) - thisrr->input = &(rl->packet[SSL2_RT_HEADER_LENGTH]); - else - thisrr->input = &(rl->packet[SSL3_RT_HEADER_LENGTH]); - - /* - * ok, we can now read from 'rl->packet' data into 'thisrr'. - * thisrr->input points at thisrr->length bytes, which need to be copied - * into thisrr->data by either the decryption or by the decompression. - * When the data is 'copied' into the thisrr->data buffer, - * thisrr->input will be updated to point at the new buffer - */ - - /* - * We now have - encrypted [ MAC [ compressed [ plain ] ] ] - * thisrr->length bytes of encrypted compressed stuff. - */ - - /* decrypt in place in 'thisrr->input' */ - thisrr->data = thisrr->input; - thisrr->orig_len = thisrr->length; - - num_recs++; - - /* we have pulled in a full packet so zero things */ - rl->packet_length = 0; - rl->is_first_record = 0; - } while (num_recs < max_recs - && thisrr->type == SSL3_RT_APPLICATION_DATA - && RLAYER_USE_EXPLICIT_IV(rl) - && rl->enc_ctx != NULL - && (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(rl->enc_ctx)) - & EVP_CIPH_FLAG_PIPELINE) != 0 - && tls_record_app_data_waiting(rl)); - - if (num_recs == 1 - && thisrr->type == SSL3_RT_CHANGE_CIPHER_SPEC - /* The following can happen in tlsany_meth after HRR */ - && rl->version == TLS1_3_VERSION - && rl->is_first_handshake) { - /* - * CCS messages must be exactly 1 byte long, containing the value 0x01 - */ - if (thisrr->length != 1 || thisrr->data[0] != 0x01) { - RLAYERfatal(rl, SSL_AD_ILLEGAL_PARAMETER, - SSL_R_INVALID_CCS_MESSAGE); - return OSSL_RECORD_RETURN_FATAL; - } - /* - * CCS messages are ignored in TLSv1.3. We treat it like an empty - * handshake record - */ - thisrr->type = SSL3_RT_HANDSHAKE; - if (++(rl->empty_record_count) > MAX_EMPTY_RECORDS) { - RLAYERfatal(rl, SSL_AD_UNEXPECTED_MESSAGE, - SSL_R_UNEXPECTED_CCS_MESSAGE); - return OSSL_RECORD_RETURN_FATAL; - } - rl->num_recs = 0; - rl->curr_rec = 0; - rl->num_released = 0; - - return OSSL_RECORD_RETURN_SUCCESS; - } - - if (rl->md_ctx != NULL) { - const EVP_MD *tmpmd = EVP_MD_CTX_get0_md(rl->md_ctx); - - if (tmpmd != NULL) { - imac_size = EVP_MD_get_size(tmpmd); - if (!ossl_assert(imac_size >= 0 && imac_size <= EVP_MAX_MD_SIZE)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - return OSSL_RECORD_RETURN_FATAL; - } - mac_size = (size_t)imac_size; - } - } - - /* - * If in encrypt-then-mac mode calculate mac from encrypted record. All - * the details below are public so no timing details can leak. - */ - if (rl->use_etm && rl->md_ctx != NULL) { - unsigned char *mac; - - for (j = 0; j < num_recs; j++) { - thisrr = &rr[j]; - - if (thisrr->length < mac_size) { - RLAYERfatal(rl, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); - return OSSL_RECORD_RETURN_FATAL; - } - thisrr->length -= mac_size; - mac = thisrr->data + thisrr->length; - i = rl->funcs->mac(rl, thisrr, md, 0 /* not send */); - if (i == 0 || CRYPTO_memcmp(md, mac, mac_size) != 0) { - RLAYERfatal(rl, SSL_AD_BAD_RECORD_MAC, - SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); - return OSSL_RECORD_RETURN_FATAL; - } - } - /* - * We've handled the mac now - there is no MAC inside the encrypted - * record - */ - mac_size = 0; - } - - if (mac_size > 0) { - macbufs = OPENSSL_zalloc(sizeof(*macbufs) * num_recs); - if (macbufs == NULL) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); - return OSSL_RECORD_RETURN_FATAL; - } - } - - ERR_set_mark(); - enc_err = rl->funcs->cipher(rl, rr, num_recs, 0, macbufs, mac_size); - - /*- - * enc_err is: - * 0: if the record is publicly invalid, or an internal error, or AEAD - * decryption failed, or ETM decryption failed. - * 1: Success or MTE decryption failed (MAC will be randomised) - */ - if (enc_err == 0) { - if (rl->alert != SSL_AD_NO_ALERT) { - /* RLAYERfatal() already got called */ - ERR_clear_last_mark(); - goto end; - } - if (num_recs == 1 - && rl->skip_early_data != NULL - && rl->skip_early_data(rl->cbarg)) { - /* - * Valid early_data that we cannot decrypt will fail here. We treat - * it like an empty record. - */ - - /* - * Remove any errors from the stack. Decryption failures are normal - * behaviour. - */ - ERR_pop_to_mark(); - - thisrr = &rr[0]; - - if (!rlayer_early_data_count_ok(rl, thisrr->length, - EARLY_DATA_CIPHERTEXT_OVERHEAD, 0)) { - /* RLAYERfatal() already called */ - goto end; - } - - thisrr->length = 0; - rl->num_recs = 0; - rl->curr_rec = 0; - rl->num_released = 0; - /* Reset the read sequence */ - memset(rl->sequence, 0, sizeof(rl->sequence)); - ret = 1; - goto end; - } - ERR_clear_last_mark(); - RLAYERfatal(rl, SSL_AD_BAD_RECORD_MAC, - SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); - goto end; - } else { - ERR_clear_last_mark(); - } - OSSL_TRACE_BEGIN(TLS) { - BIO_printf(trc_out, "dec %lu\n", (unsigned long)rr[0].length); - BIO_dump_indent(trc_out, rr[0].data, rr[0].length, 4); - } OSSL_TRACE_END(TLS); - - /* r->length is now the compressed data plus mac */ - if (rl->enc_ctx != NULL - && !rl->use_etm - && EVP_MD_CTX_get0_md(rl->md_ctx) != NULL) { - for (j = 0; j < num_recs; j++) { - SSL_MAC_BUF *thismb = &macbufs[j]; - - thisrr = &rr[j]; - - i = rl->funcs->mac(rl, thisrr, md, 0 /* not send */); - if (i == 0 || thismb == NULL || thismb->mac == NULL - || CRYPTO_memcmp(md, thismb->mac, (size_t)mac_size) != 0) - enc_err = 0; - if (thisrr->length > SSL3_RT_MAX_COMPRESSED_LENGTH + mac_size) - enc_err = 0; -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (enc_err == 0 && mac_size > 0 && thismb != NULL && - thismb->mac != NULL && (md[0] ^ thismb->mac[0]) != 0xFF) { - enc_err = 1; - } -#endif - } - } - - if (enc_err == 0) { - if (rl->alert != SSL_AD_NO_ALERT) { - /* We already called RLAYERfatal() */ - goto end; - } - /* - * A separate 'decryption_failed' alert was introduced with TLS 1.0, - * SSL 3.0 only has 'bad_record_mac'. But unless a decryption - * failure is directly visible from the ciphertext anyway, we should - * not reveal which kind of error occurred -- this might become - * visible to an attacker (e.g. via a logfile) - */ - RLAYERfatal(rl, SSL_AD_BAD_RECORD_MAC, - SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); - goto end; - } - - for (j = 0; j < num_recs; j++) { - thisrr = &rr[j]; - - if (!rl->funcs->post_process_record(rl, thisrr)) { - /* RLAYERfatal already called */ - goto end; - } - - /* - * Record overflow checking (e.g. checking if - * thisrr->length > SSL3_RT_MAX_PLAIN_LENGTH) is the responsibility of - * the post_process_record() function above. However we check here if - * the received packet overflows the current Max Fragment Length setting - * if there is one. - * Note: rl->max_frag_len != SSL3_RT_MAX_PLAIN_LENGTH and KTLS are - * mutually exclusive. Also note that with KTLS thisrr->length can - * be > SSL3_RT_MAX_PLAIN_LENGTH (and rl->max_frag_len must be ignored) - */ - if (rl->max_frag_len != SSL3_RT_MAX_PLAIN_LENGTH - && thisrr->length > rl->max_frag_len) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, SSL_R_DATA_LENGTH_TOO_LONG); - goto end; - } - - thisrr->off = 0; - /*- - * So at this point the following is true - * thisrr->type is the type of record - * thisrr->length == number of bytes in record - * thisrr->off == offset to first valid byte - * thisrr->data == where to take bytes from, increment after use :-). - */ - - /* just read a 0 length packet */ - if (thisrr->length == 0) { - if (++(rl->empty_record_count) > MAX_EMPTY_RECORDS) { - RLAYERfatal(rl, SSL_AD_UNEXPECTED_MESSAGE, - SSL_R_RECORD_TOO_SMALL); - goto end; - } - } else { - rl->empty_record_count = 0; - } - } - - if (rl->level == OSSL_RECORD_PROTECTION_LEVEL_EARLY) { - thisrr = &rr[0]; - if (thisrr->type == SSL3_RT_APPLICATION_DATA - && !rlayer_early_data_count_ok(rl, thisrr->length, 0, 0)) { - /* RLAYERfatal already called */ - goto end; - } - } - - rl->num_recs = num_recs; - rl->curr_rec = 0; - rl->num_released = 0; - ret = OSSL_RECORD_RETURN_SUCCESS; - end: - if (macbufs != NULL) { - for (j = 0; j < num_recs; j++) { - if (macbufs[j].alloced) - OPENSSL_free(macbufs[j].mac); - } - OPENSSL_free(macbufs); - } - return ret; -} - -/* Shared by ssl3_meth and tls1_meth */ -int tls_default_validate_record_header(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec) -{ - size_t len = SSL3_RT_MAX_ENCRYPTED_LENGTH; - - if (rec->rec_version != rl->version) { - RLAYERfatal(rl, SSL_AD_PROTOCOL_VERSION, SSL_R_WRONG_VERSION_NUMBER); - return 0; - } - -#ifndef OPENSSL_NO_COMP - /* - * If OPENSSL_NO_COMP is defined then SSL3_RT_MAX_ENCRYPTED_LENGTH - * does not include the compression overhead anyway. - */ - if (rl->compctx == NULL) - len -= SSL3_RT_MAX_COMPRESSED_OVERHEAD; -#endif - - if (rec->length > len) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, - SSL_R_ENCRYPTED_LENGTH_TOO_LONG); - return 0; - } - - return 1; -} - -int tls_do_compress(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *wr) -{ -#ifndef OPENSSL_NO_COMP - int i; - - i = COMP_compress_block(rl->compctx, wr->data, - (int)(wr->length + SSL3_RT_MAX_COMPRESSED_OVERHEAD), - wr->input, (int)wr->length); - if (i < 0) - return 0; - - wr->length = i; - wr->input = wr->data; - return 1; -#else - return 0; -#endif -} - -int tls_do_uncompress(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec) -{ -#ifndef OPENSSL_NO_COMP - int i; - - if (rec->comp == NULL) { - rec->comp = (unsigned char *) - OPENSSL_malloc(SSL3_RT_MAX_ENCRYPTED_LENGTH); - } - if (rec->comp == NULL) - return 0; - - i = COMP_expand_block(rl->compctx, rec->comp, SSL3_RT_MAX_PLAIN_LENGTH, - rec->data, (int)rec->length); - if (i < 0) - return 0; - else - rec->length = i; - rec->data = rec->comp; - return 1; -#else - return 0; -#endif -} - -/* Shared by tlsany_meth, ssl3_meth and tls1_meth */ -int tls_default_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec) -{ - if (rl->compctx != NULL) { - if (rec->length > SSL3_RT_MAX_COMPRESSED_LENGTH) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, - SSL_R_COMPRESSED_LENGTH_TOO_LONG); - return 0; - } - if (!tls_do_uncompress(rl, rec)) { - RLAYERfatal(rl, SSL_AD_DECOMPRESSION_FAILURE, - SSL_R_BAD_DECOMPRESSION); - return 0; - } - } - - if (rec->length > SSL3_RT_MAX_PLAIN_LENGTH) { - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, SSL_R_DATA_LENGTH_TOO_LONG); - return 0; - } - - return 1; -} - -/* Shared by tls13_meth and ktls_meth */ -int tls13_common_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec) -{ - if (rec->type != SSL3_RT_APPLICATION_DATA - && rec->type != SSL3_RT_ALERT - && rec->type != SSL3_RT_HANDSHAKE) { - RLAYERfatal(rl, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_BAD_RECORD_TYPE); - return 0; - } - - if (rl->msg_callback != NULL) - rl->msg_callback(0, rl->version, SSL3_RT_INNER_CONTENT_TYPE, &rec->type, - 1, rl->cbarg); - - /* - * TLSv1.3 alert and handshake records are required to be non-zero in - * length. - */ - if ((rec->type == SSL3_RT_HANDSHAKE || rec->type == SSL3_RT_ALERT) - && rec->length == 0) { - RLAYERfatal(rl, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_BAD_LENGTH); - return 0; - } - - return 1; -} - -int tls_read_record(OSSL_RECORD_LAYER *rl, void **rechandle, int *rversion, - uint8_t *type, const unsigned char **data, size_t *datalen, - uint16_t *epoch, unsigned char *seq_num) -{ - TLS_RL_RECORD *rec; - - /* - * tls_get_more_records() can return success without actually reading - * anything useful (i.e. if empty records are read). We loop here until - * we have something useful. tls_get_more_records() will eventually fail if - * too many sequential empty records are read. - */ - while (rl->curr_rec >= rl->num_recs) { - int ret; - - if (rl->num_released != rl->num_recs) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_RECORDS_NOT_RELEASED); - return OSSL_RECORD_RETURN_FATAL; - } - - ret = rl->funcs->get_more_records(rl); - - if (ret != OSSL_RECORD_RETURN_SUCCESS) - return ret; - } - - /* - * We have now got rl->num_recs records buffered in rl->rrec. rl->curr_rec - * points to the next one to read. - */ - rec = &rl->rrec[rl->curr_rec++]; - - *rechandle = rec; - *rversion = rec->rec_version; - *type = rec->type; - *data = rec->data + rec->off; - *datalen = rec->length; - if (rl->isdtls) { - *epoch = rec->epoch; - memcpy(seq_num, rec->seq_num, sizeof(rec->seq_num)); - } - - return OSSL_RECORD_RETURN_SUCCESS; -} - -int tls_release_record(OSSL_RECORD_LAYER *rl, void *rechandle, size_t length) -{ - TLS_RL_RECORD *rec = &rl->rrec[rl->num_released]; - - if (!ossl_assert(rl->num_released < rl->curr_rec) - || !ossl_assert(rechandle == rec)) { - /* Should not happen */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_INVALID_RECORD); - return OSSL_RECORD_RETURN_FATAL; - } - - if (rec->length < length) { - /* Should not happen */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - if ((rl->options & SSL_OP_CLEANSE_PLAINTEXT) != 0) - OPENSSL_cleanse(rec->data + rec->off, length); - - rec->off += length; - rec->length -= length; - - if (rec->length > 0) - return OSSL_RECORD_RETURN_SUCCESS; - - rl->num_released++; - - if (rl->curr_rec == rl->num_released - && (rl->mode & SSL_MODE_RELEASE_BUFFERS) != 0 - && TLS_BUFFER_get_left(&rl->rbuf) == 0) - tls_release_read_buffer(rl); - - return OSSL_RECORD_RETURN_SUCCESS; -} - -int tls_set_options(OSSL_RECORD_LAYER *rl, const OSSL_PARAM *options) -{ - const OSSL_PARAM *p; - - p = OSSL_PARAM_locate_const(options, OSSL_LIBSSL_RECORD_LAYER_PARAM_OPTIONS); - if (p != NULL && !OSSL_PARAM_get_uint64(p, &rl->options)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - return 0; - } - - p = OSSL_PARAM_locate_const(options, OSSL_LIBSSL_RECORD_LAYER_PARAM_MODE); - if (p != NULL && !OSSL_PARAM_get_uint32(p, &rl->mode)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - return 0; - } - - if (rl->direction == OSSL_RECORD_DIRECTION_READ) { - p = OSSL_PARAM_locate_const(options, - OSSL_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN); - if (p != NULL && !OSSL_PARAM_get_size_t(p, &rl->rbuf.default_len)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - return 0; - } - } else { - p = OSSL_PARAM_locate_const(options, - OSSL_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING); - if (p != NULL && !OSSL_PARAM_get_size_t(p, &rl->block_padding)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - return 0; - } - } - - if (rl->level == OSSL_RECORD_PROTECTION_LEVEL_APPLICATION) { - /* - * We ignore any read_ahead setting prior to the application protection - * level. Otherwise we may read ahead data in a lower protection level - * that is destined for a higher protection level. To simplify the logic - * we don't support that at this stage. - */ - p = OSSL_PARAM_locate_const(options, - OSSL_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD); - if (p != NULL && !OSSL_PARAM_get_int(p, &rl->read_ahead)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - return 0; - } - } - - return 1; -} - -int -tls_int_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers, - int role, int direction, int level, - const EVP_CIPHER *ciph, size_t taglen, - const EVP_MD *md, COMP_METHOD *comp, BIO *prev, - BIO *transport, BIO *next, const OSSL_PARAM *settings, - const OSSL_PARAM *options, - const OSSL_DISPATCH *fns, void *cbarg, - OSSL_RECORD_LAYER **retrl) -{ - OSSL_RECORD_LAYER *rl = OPENSSL_zalloc(sizeof(*rl)); - const OSSL_PARAM *p; - - *retrl = NULL; - - if (rl == NULL) - return OSSL_RECORD_RETURN_FATAL; - - /* - * Default the value for max_frag_len. This may be overridden by the - * settings - */ - rl->max_frag_len = SSL3_RT_MAX_PLAIN_LENGTH; - - /* Loop through all the settings since they must all be understood */ - if (settings != NULL) { - for (p = settings; p->key != NULL; p++) { - if (strcmp(p->key, OSSL_LIBSSL_RECORD_LAYER_PARAM_USE_ETM) == 0) { - if (!OSSL_PARAM_get_int(p, &rl->use_etm)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - goto err; - } - } else if (strcmp(p->key, - OSSL_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN) == 0) { - if (!OSSL_PARAM_get_uint(p, &rl->max_frag_len)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - goto err; - } - } else if (strcmp(p->key, - OSSL_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA) == 0) { - if (!OSSL_PARAM_get_uint32(p, &rl->max_early_data)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - goto err; - } - } else if (strcmp(p->key, - OSSL_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC) == 0) { - if (!OSSL_PARAM_get_int(p, &rl->stream_mac)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - goto err; - } - } else if (strcmp(p->key, - OSSL_LIBSSL_RECORD_LAYER_PARAM_TLSTREE) == 0) { - if (!OSSL_PARAM_get_int(p, &rl->tlstree)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - goto err; - } - } else { - ERR_raise(ERR_LIB_SSL, SSL_R_UNKNOWN_MANDATORY_PARAMETER); - goto err; - } - } - } - - rl->libctx = libctx; - rl->propq = propq; - - rl->version = vers; - rl->role = role; - rl->direction = direction; - rl->level = level; - rl->taglen = taglen; - rl->md = md; - - rl->alert = SSL_AD_NO_ALERT; - rl->rstate = SSL_ST_READ_HEADER; - - if (level == OSSL_RECORD_PROTECTION_LEVEL_NONE) - rl->is_first_record = 1; - - if (!tls_set1_bio(rl, transport)) - goto err; - - if (prev != NULL && !BIO_up_ref(prev)) - goto err; - rl->prev = prev; - - if (next != NULL && !BIO_up_ref(next)) - goto err; - rl->next = next; - - rl->cbarg = cbarg; - if (fns != NULL) { - for (; fns->function_id != 0; fns++) { - switch (fns->function_id) { - case OSSL_FUNC_RLAYER_SKIP_EARLY_DATA: - rl->skip_early_data = OSSL_FUNC_rlayer_skip_early_data(fns); - break; - case OSSL_FUNC_RLAYER_MSG_CALLBACK: - rl->msg_callback = OSSL_FUNC_rlayer_msg_callback(fns); - break; - case OSSL_FUNC_RLAYER_SECURITY: - rl->security = OSSL_FUNC_rlayer_security(fns); - break; - case OSSL_FUNC_RLAYER_PADDING: - rl->padding = OSSL_FUNC_rlayer_padding(fns); - default: - /* Just ignore anything we don't understand */ - break; - } - } - } - - if (!tls_set_options(rl, options)) { - ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER); - goto err; - } - - if ((rl->options & SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) == 0 - && rl->version <= TLS1_VERSION - && !EVP_CIPHER_is_a(ciph, "NULL") - && !EVP_CIPHER_is_a(ciph, "RC4")) { - /* - * Enable vulnerability countermeasure for CBC ciphers with known-IV - * problem (http://www.openssl.org/~bodo/tls-cbc.txt) - */ - rl->need_empty_fragments = 1; - } - - *retrl = rl; - return OSSL_RECORD_RETURN_SUCCESS; - err: - tls_int_free(rl); - return OSSL_RECORD_RETURN_FATAL; -} - -static int -tls_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers, - int role, int direction, int level, uint16_t epoch, - unsigned char *secret, size_t secretlen, - unsigned char *key, size_t keylen, unsigned char *iv, - size_t ivlen, unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, size_t taglen, - int mactype, - const EVP_MD *md, COMP_METHOD *comp, - const EVP_MD *kdfdigest, BIO *prev, BIO *transport, - BIO *next, BIO_ADDR *local, BIO_ADDR *peer, - const OSSL_PARAM *settings, const OSSL_PARAM *options, - const OSSL_DISPATCH *fns, void *cbarg, void *rlarg, - OSSL_RECORD_LAYER **retrl) -{ - int ret; - - ret = tls_int_new_record_layer(libctx, propq, vers, role, direction, level, - ciph, taglen, md, comp, prev, - transport, next, settings, - options, fns, cbarg, retrl); - - if (ret != OSSL_RECORD_RETURN_SUCCESS) - return ret; - - switch (vers) { - case TLS_ANY_VERSION: - (*retrl)->funcs = &tls_any_funcs; - break; - case TLS1_3_VERSION: - (*retrl)->funcs = &tls_1_3_funcs; - break; - case TLS1_2_VERSION: - case TLS1_1_VERSION: - case TLS1_VERSION: - (*retrl)->funcs = &tls_1_funcs; - break; - case SSL3_VERSION: - (*retrl)->funcs = &ssl_3_0_funcs; - break; - default: - /* Should not happen */ - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - ret = OSSL_RECORD_RETURN_FATAL; - goto err; - } - - ret = (*retrl)->funcs->set_crypto_state(*retrl, level, key, keylen, iv, - ivlen, mackey, mackeylen, ciph, - taglen, mactype, md, comp); - - err: - if (ret != OSSL_RECORD_RETURN_SUCCESS) { - tls_int_free(*retrl); - *retrl = NULL; - } - return ret; -} - -static void tls_int_free(OSSL_RECORD_LAYER *rl) -{ - BIO_free(rl->prev); - BIO_free(rl->bio); - BIO_free(rl->next); - ossl_tls_buffer_release(&rl->rbuf); - - tls_release_write_buffer(rl); - - EVP_CIPHER_CTX_free(rl->enc_ctx); - EVP_MD_CTX_free(rl->md_ctx); -#ifndef OPENSSL_NO_COMP - COMP_CTX_free(rl->compctx); -#endif - - if (rl->version == SSL3_VERSION) - OPENSSL_cleanse(rl->mac_secret, sizeof(rl->mac_secret)); - - TLS_RL_RECORD_release(rl->rrec, SSL_MAX_PIPELINES); - - OPENSSL_free(rl); -} - -int tls_free(OSSL_RECORD_LAYER *rl) -{ - TLS_BUFFER *rbuf; - size_t left, written; - int ret = 1; - - if (rl == NULL) - return 1; - - rbuf = &rl->rbuf; - - left = TLS_BUFFER_get_left(rbuf); - if (left > 0) { - /* - * This record layer is closing but we still have data left in our - * buffer. It must be destined for the next epoch - so push it there. - */ - ret = BIO_write_ex(rl->next, rbuf->buf + rbuf->offset, left, &written); - } - tls_int_free(rl); - - return ret; -} - -int tls_unprocessed_read_pending(OSSL_RECORD_LAYER *rl) -{ - return TLS_BUFFER_get_left(&rl->rbuf) != 0; -} - -int tls_processed_read_pending(OSSL_RECORD_LAYER *rl) -{ - return rl->curr_rec < rl->num_recs; -} - -size_t tls_app_data_pending(OSSL_RECORD_LAYER *rl) -{ - size_t i; - size_t num = 0; - - for (i = rl->curr_rec; i < rl->num_recs; i++) { - if (rl->rrec[i].type != SSL3_RT_APPLICATION_DATA) - return num; - num += rl->rrec[i].length; - } - return num; -} - -size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, uint8_t type, - size_t len, - size_t maxfrag, size_t *preffrag) -{ - /* - * If we have a pipeline capable cipher, and we have been configured to use - * it, then return the preferred number of pipelines. - */ - if (rl->max_pipelines > 0 - && rl->enc_ctx != NULL - && (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(rl->enc_ctx)) - & EVP_CIPH_FLAG_PIPELINE) != 0 - && RLAYER_USE_EXPLICIT_IV(rl)) { - size_t pipes; - - if (len == 0) - return 1; - pipes = ((len - 1) / *preffrag) + 1; - - return (pipes < rl->max_pipelines) ? pipes : rl->max_pipelines; - } - - return 1; -} - -size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len, - size_t maxfrag, size_t *preffrag) -{ - return rl->funcs->get_max_records(rl, type, len, maxfrag, preffrag); -} - -int tls_allocate_write_buffers_default(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, - size_t *prefix) -{ - if (!tls_setup_write_buffer(rl, numtempl, 0, 0)) { - /* RLAYERfatal() already called */ - return 0; - } - - return 1; -} - -int tls_initialise_write_packets_default(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl, - OSSL_RECORD_TEMPLATE *prefixtempl, - WPACKET *pkt, - TLS_BUFFER *bufs, - size_t *wpinited) -{ - WPACKET *thispkt; - size_t j, align; - TLS_BUFFER *wb; - - for (j = 0; j < numtempl; j++) { - thispkt = &pkt[j]; - wb = &bufs[j]; - - wb->type = templates[j].type; - -#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD != 0 - align = (size_t)TLS_BUFFER_get_buf(wb); - align += rl->isdtls ? DTLS1_RT_HEADER_LENGTH : SSL3_RT_HEADER_LENGTH; - align = SSL3_ALIGN_PAYLOAD - 1 - - ((align - 1) % SSL3_ALIGN_PAYLOAD); -#endif - TLS_BUFFER_set_offset(wb, align); - - if (!WPACKET_init_static_len(thispkt, TLS_BUFFER_get_buf(wb), - TLS_BUFFER_get_len(wb), 0)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - (*wpinited)++; - if (!WPACKET_allocate_bytes(thispkt, align, NULL)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - } - - return 1; -} - -int tls_prepare_record_header_default(OSSL_RECORD_LAYER *rl, - WPACKET *thispkt, - OSSL_RECORD_TEMPLATE *templ, - uint8_t rectype, - unsigned char **recdata) -{ - size_t maxcomplen; - - *recdata = NULL; - - maxcomplen = templ->buflen; - if (rl->compctx != NULL) - maxcomplen += SSL3_RT_MAX_COMPRESSED_OVERHEAD; - - if (!WPACKET_put_bytes_u8(thispkt, rectype) - || !WPACKET_put_bytes_u16(thispkt, templ->version) - || !WPACKET_start_sub_packet_u16(thispkt) - || (rl->eivlen > 0 - && !WPACKET_allocate_bytes(thispkt, rl->eivlen, NULL)) - || (maxcomplen > 0 - && !WPACKET_reserve_bytes(thispkt, maxcomplen, - recdata))) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - return 1; -} - -int tls_prepare_for_encryption_default(OSSL_RECORD_LAYER *rl, - size_t mac_size, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr) -{ - size_t len; - unsigned char *recordstart; - - /* - * we should still have the output to thiswr->data and the input from - * wr->input. Length should be thiswr->length. thiswr->data still points - * in the wb->buf - */ - - if (!rl->use_etm && mac_size != 0) { - unsigned char *mac; - - if (!WPACKET_allocate_bytes(thispkt, mac_size, &mac) - || !rl->funcs->mac(rl, thiswr, mac, 1)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - } - - /* - * Reserve some bytes for any growth that may occur during encryption. If - * we are adding the MAC independently of the cipher algorithm, then the - * max encrypted overhead does not need to include an allocation for that - * MAC - */ - if (!WPACKET_reserve_bytes(thispkt, SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD - - mac_size, NULL) - /* - * We also need next the amount of bytes written to this - * sub-packet - */ - || !WPACKET_get_length(thispkt, &len)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - /* Get a pointer to the start of this record excluding header */ - recordstart = WPACKET_get_curr(thispkt) - len; - TLS_RL_RECORD_set_data(thiswr, recordstart); - TLS_RL_RECORD_reset_input(thiswr); - TLS_RL_RECORD_set_length(thiswr, len); - - return 1; -} - -int tls_post_encryption_processing_default(OSSL_RECORD_LAYER *rl, - size_t mac_size, - OSSL_RECORD_TEMPLATE *thistempl, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr) -{ - size_t origlen, len; - size_t headerlen = rl->isdtls ? DTLS1_RT_HEADER_LENGTH - : SSL3_RT_HEADER_LENGTH; - - /* Allocate bytes for the encryption overhead */ - if (!WPACKET_get_length(thispkt, &origlen) - /* Check we allowed enough room for the encryption growth */ - || !ossl_assert(origlen + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD - - mac_size >= thiswr->length) - /* Encryption should never shrink the data! */ - || origlen > thiswr->length - || (thiswr->length > origlen - && !WPACKET_allocate_bytes(thispkt, - thiswr->length - origlen, - NULL))) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - if (rl->use_etm && mac_size != 0) { - unsigned char *mac; - - if (!WPACKET_allocate_bytes(thispkt, mac_size, &mac) - || !rl->funcs->mac(rl, thiswr, mac, 1)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - TLS_RL_RECORD_add_length(thiswr, mac_size); - } - - if (!WPACKET_get_length(thispkt, &len) - || !WPACKET_close(thispkt)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (rl->msg_callback != NULL) { - unsigned char *recordstart; - - recordstart = WPACKET_get_curr(thispkt) - len - headerlen; - rl->msg_callback(1, thiswr->rec_version, SSL3_RT_HEADER, recordstart, - headerlen, rl->cbarg); - - if (rl->version == TLS1_3_VERSION && rl->enc_ctx != NULL) { - unsigned char ctype = thistempl->type; - - rl->msg_callback(1, thiswr->rec_version, SSL3_RT_INNER_CONTENT_TYPE, - &ctype, 1, rl->cbarg); - } - } - - if (!WPACKET_finish(thispkt)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - TLS_RL_RECORD_add_length(thiswr, headerlen); - - return 1; -} - -int tls_write_records_default(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl) -{ - WPACKET pkt[SSL_MAX_PIPELINES + 1]; - TLS_RL_RECORD wr[SSL_MAX_PIPELINES + 1]; - WPACKET *thispkt; - TLS_RL_RECORD *thiswr; - int mac_size = 0, ret = 0; - size_t wpinited = 0; - size_t j, prefix = 0; - OSSL_RECORD_TEMPLATE prefixtempl; - OSSL_RECORD_TEMPLATE *thistempl; - - if (rl->md_ctx != NULL && EVP_MD_CTX_get0_md(rl->md_ctx) != NULL) { - mac_size = EVP_MD_CTX_get_size(rl->md_ctx); - if (mac_size < 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - } - - if (!rl->funcs->allocate_write_buffers(rl, templates, numtempl, &prefix)) { - /* RLAYERfatal() already called */ - goto err; - } - - if (!rl->funcs->initialise_write_packets(rl, templates, numtempl, - &prefixtempl, pkt, rl->wbuf, - &wpinited)) { - /* RLAYERfatal() already called */ - goto err; - } - - /* Clear our TLS_RL_RECORD structures */ - memset(wr, 0, sizeof(wr)); - for (j = 0; j < numtempl + prefix; j++) { - unsigned char *compressdata = NULL; - uint8_t rectype; - - thispkt = &pkt[j]; - thiswr = &wr[j]; - thistempl = (j < prefix) ? &prefixtempl : &templates[j - prefix]; - - /* - * Default to the record type as specified in the template unless the - * protocol implementation says differently. - */ - if (rl->funcs->get_record_type != NULL) - rectype = rl->funcs->get_record_type(rl, thistempl); - else - rectype = thistempl->type; - - TLS_RL_RECORD_set_type(thiswr, rectype); - TLS_RL_RECORD_set_rec_version(thiswr, thistempl->version); - - if (!rl->funcs->prepare_record_header(rl, thispkt, thistempl, rectype, - &compressdata)) { - /* RLAYERfatal() already called */ - goto err; - } - - /* lets setup the record stuff. */ - TLS_RL_RECORD_set_data(thiswr, compressdata); - TLS_RL_RECORD_set_length(thiswr, thistempl->buflen); - - TLS_RL_RECORD_set_input(thiswr, (unsigned char *)thistempl->buf); - - /* - * we now 'read' from thiswr->input, thiswr->length bytes into - * thiswr->data - */ - - /* first we compress */ - if (rl->compctx != NULL) { - if (!tls_do_compress(rl, thiswr) - || !WPACKET_allocate_bytes(thispkt, thiswr->length, NULL)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_COMPRESSION_FAILURE); - goto err; - } - } else if (compressdata != NULL) { - if (!WPACKET_memcpy(thispkt, thiswr->input, thiswr->length)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - TLS_RL_RECORD_reset_input(&wr[j]); - } - - if (rl->funcs->add_record_padding != NULL - && !rl->funcs->add_record_padding(rl, thistempl, thispkt, - thiswr)) { - /* RLAYERfatal() already called */ - goto err; - } - - if (!rl->funcs->prepare_for_encryption(rl, mac_size, thispkt, thiswr)) { - /* RLAYERfatal() already called */ - goto err; - } - } - - if (prefix) { - if (rl->funcs->cipher(rl, wr, 1, 1, NULL, mac_size) < 1) { - if (rl->alert == SSL_AD_NO_ALERT) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - } - goto err; - } - } - - if (rl->funcs->cipher(rl, wr + prefix, numtempl, 1, NULL, mac_size) < 1) { - if (rl->alert == SSL_AD_NO_ALERT) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - } - goto err; - } - - for (j = 0; j < numtempl + prefix; j++) { - thispkt = &pkt[j]; - thiswr = &wr[j]; - thistempl = (j < prefix) ? &prefixtempl : &templates[j - prefix]; - - if (!rl->funcs->post_encryption_processing(rl, mac_size, thistempl, - thispkt, thiswr)) { - /* RLAYERfatal() already called */ - goto err; - } - - /* now let's set up wb */ - TLS_BUFFER_set_left(&rl->wbuf[j], TLS_RL_RECORD_get_length(thiswr)); - } - - ret = 1; - err: - for (j = 0; j < wpinited; j++) - WPACKET_cleanup(&pkt[j]); - return ret; -} - -int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, - size_t numtempl) -{ - /* Check we don't have pending data waiting to write */ - if (!ossl_assert(rl->nextwbuf >= rl->numwpipes - || TLS_BUFFER_get_left(&rl->wbuf[rl->nextwbuf]) == 0)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return OSSL_RECORD_RETURN_FATAL; - } - - if (!rl->funcs->write_records(rl, templates, numtempl)) { - /* RLAYERfatal already called */ - return OSSL_RECORD_RETURN_FATAL; - } - - rl->nextwbuf = 0; - /* we now just need to write the buffers */ - return tls_retry_write_records(rl); -} - -int tls_retry_write_records(OSSL_RECORD_LAYER *rl) -{ - int i, ret; - TLS_BUFFER *thiswb; - size_t tmpwrit = 0; - - if (rl->nextwbuf >= rl->numwpipes) - return OSSL_RECORD_RETURN_SUCCESS; - - for (;;) { - thiswb = &rl->wbuf[rl->nextwbuf]; - - clear_sys_error(); - if (rl->bio != NULL) { - if (rl->funcs->prepare_write_bio != NULL) { - ret = rl->funcs->prepare_write_bio(rl, thiswb->type); - if (ret != OSSL_RECORD_RETURN_SUCCESS) - return ret; - } - i = BIO_write(rl->bio, (char *) - &(TLS_BUFFER_get_buf(thiswb) - [TLS_BUFFER_get_offset(thiswb)]), - (unsigned int)TLS_BUFFER_get_left(thiswb)); - if (i >= 0) { - tmpwrit = i; - if (i == 0 && BIO_should_retry(rl->bio)) - ret = OSSL_RECORD_RETURN_RETRY; - else - ret = OSSL_RECORD_RETURN_SUCCESS; - } else { - if (BIO_should_retry(rl->bio)) - ret = OSSL_RECORD_RETURN_RETRY; - else - ret = OSSL_RECORD_RETURN_FATAL; - } - } else { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_BIO_NOT_SET); - ret = OSSL_RECORD_RETURN_FATAL; - i = -1; - } - - /* - * When an empty fragment is sent on a connection using KTLS, - * it is sent as a write of zero bytes. If this zero byte - * write succeeds, i will be 0 rather than a non-zero value. - * Treat i == 0 as success rather than an error for zero byte - * writes to permit this case. - */ - if (i >= 0 && tmpwrit == TLS_BUFFER_get_left(thiswb)) { - TLS_BUFFER_set_left(thiswb, 0); - TLS_BUFFER_add_offset(thiswb, tmpwrit); - if (++(rl->nextwbuf) < rl->numwpipes) - continue; - - if (rl->nextwbuf == rl->numwpipes - && (rl->mode & SSL_MODE_RELEASE_BUFFERS) != 0) - tls_release_write_buffer(rl); - return OSSL_RECORD_RETURN_SUCCESS; - } else if (i <= 0) { - if (rl->isdtls) { - /* - * For DTLS, just drop it. That's kind of the whole point in - * using a datagram service - */ - TLS_BUFFER_set_left(thiswb, 0); - if (++(rl->nextwbuf) == rl->numwpipes - && (rl->mode & SSL_MODE_RELEASE_BUFFERS) != 0) - tls_release_write_buffer(rl); - - } - return ret; - } - TLS_BUFFER_add_offset(thiswb, tmpwrit); - TLS_BUFFER_sub_left(thiswb, tmpwrit); - } -} - -int tls_get_alert_code(OSSL_RECORD_LAYER *rl) -{ - return rl->alert; -} - -int tls_set1_bio(OSSL_RECORD_LAYER *rl, BIO *bio) -{ - if (bio != NULL && !BIO_up_ref(bio)) - return 0; - BIO_free(rl->bio); - rl->bio = bio; - - return 1; -} - -/* Shared by most methods except tlsany_meth */ -int tls_default_set_protocol_version(OSSL_RECORD_LAYER *rl, int version) -{ - if (rl->version != version) - return 0; - - return 1; -} - -int tls_set_protocol_version(OSSL_RECORD_LAYER *rl, int version) -{ - return rl->funcs->set_protocol_version(rl, version); -} - -void tls_set_plain_alerts(OSSL_RECORD_LAYER *rl, int allow) -{ - rl->allow_plain_alerts = allow; -} - -void tls_set_first_handshake(OSSL_RECORD_LAYER *rl, int first) -{ - rl->is_first_handshake = first; -} - -void tls_set_max_pipelines(OSSL_RECORD_LAYER *rl, size_t max_pipelines) -{ - rl->max_pipelines = max_pipelines; - if (max_pipelines > 1) - rl->read_ahead = 1; -} - -void tls_get_state(OSSL_RECORD_LAYER *rl, const char **shortstr, - const char **longstr) -{ - const char *shrt, *lng; - - switch (rl->rstate) { - case SSL_ST_READ_HEADER: - shrt = "RH"; - lng = "read header"; - break; - case SSL_ST_READ_BODY: - shrt = "RB"; - lng = "read body"; - break; - default: - shrt = lng = "unknown"; - break; - } - if (shortstr != NULL) - *shortstr = shrt; - if (longstr != NULL) - *longstr = lng; -} - -const COMP_METHOD *tls_get_compression(OSSL_RECORD_LAYER *rl) -{ -#ifndef OPENSSL_NO_COMP - return (rl->compctx == NULL) ? NULL : COMP_CTX_get_method(rl->compctx); -#else - return NULL; -#endif -} - -void tls_set_max_frag_len(OSSL_RECORD_LAYER *rl, size_t max_frag_len) -{ - rl->max_frag_len = max_frag_len; - /* - * We don't need to adjust buffer sizes. Write buffer sizes are - * automatically checked anyway. We should only be changing the read buffer - * size during the handshake, so we will create a new buffer when we create - * the new record layer. We can't change the existing buffer because it may - * already have data in it. - */ -} - -int tls_increment_sequence_ctr(OSSL_RECORD_LAYER *rl) -{ - int i; - - /* Increment the sequence counter */ - for (i = SEQ_NUM_SIZE; i > 0; i--) { - ++(rl->sequence[i - 1]); - if (rl->sequence[i - 1] != 0) - break; - } - if (i == 0) { - /* Sequence has wrapped */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, SSL_R_SEQUENCE_CTR_WRAPPED); - return 0; - } - return 1; -} - -int tls_alloc_buffers(OSSL_RECORD_LAYER *rl) -{ - if (rl->direction == OSSL_RECORD_DIRECTION_WRITE) { - /* If we have a pending write then buffers are already allocated */ - if (rl->nextwbuf < rl->numwpipes) - return 1; - /* - * We assume 1 pipe with default sized buffer. If what we need ends up - * being a different size to that then it will be reallocated on demand. - * If we need more than 1 pipe then that will also be allocated on - * demand - */ - if (!tls_setup_write_buffer(rl, 1, 0, 0)) - return 0; - - /* - * Normally when we allocate write buffers we immediately write - * something into it. In this case we're not doing that so mark the - * buffer as empty. - */ - TLS_BUFFER_set_left(&rl->wbuf[0], 0); - return 1; - } - - /* Read direction */ - - /* If we have pending data to be read then buffers are already allocated */ - if (rl->curr_rec < rl->num_recs || TLS_BUFFER_get_left(&rl->rbuf) != 0) - return 1; - return tls_setup_read_buffer(rl); -} - -int tls_free_buffers(OSSL_RECORD_LAYER *rl) -{ - if (rl->direction == OSSL_RECORD_DIRECTION_WRITE) { - if (rl->nextwbuf < rl->numwpipes) { - /* - * We may have pending data. If we've just got one empty buffer - * allocated then it has probably just been alloc'd via - * tls_alloc_buffers, and it is fine to free it. Otherwise this - * looks like real pending data and it is an error. - */ - if (rl->nextwbuf != 0 - || rl->numwpipes != 1 - || TLS_BUFFER_get_left(&rl->wbuf[0]) != 0) - return 0; - } - tls_release_write_buffer(rl); - return 1; - } - - /* Read direction */ - - /* If we have pending data to be read then fail */ - if (rl->curr_rec < rl->num_recs - || rl->curr_rec != rl->num_released - || TLS_BUFFER_get_left(&rl->rbuf) != 0 - || rl->rstate == SSL_ST_READ_BODY) - return 0; - - return tls_release_read_buffer(rl); -} - -const OSSL_RECORD_METHOD ossl_tls_record_method = { - tls_new_record_layer, - tls_free, - tls_unprocessed_read_pending, - tls_processed_read_pending, - tls_app_data_pending, - tls_get_max_records, - tls_write_records, - tls_retry_write_records, - tls_read_record, - tls_release_record, - tls_get_alert_code, - tls_set1_bio, - tls_set_protocol_version, - tls_set_plain_alerts, - tls_set_first_handshake, - tls_set_max_pipelines, - NULL, - tls_get_state, - tls_set_options, - tls_get_compression, - tls_set_max_frag_len, - NULL, - tls_increment_sequence_ctr, - tls_alloc_buffers, - tls_free_buffers -}; diff --git a/openssl/src/ssl/record/methods/tls_multib.c b/openssl/src/ssl/record/methods/tls_multib.c deleted file mode 100644 index 3c2c30ef1..000000000 --- a/openssl/src/ssl/record/methods/tls_multib.c +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "../../ssl_local.h" -#include "../record_local.h" -#include "recmethod_local.h" - -#if defined(OPENSSL_SMALL_FOOTPRINT) \ - || !(defined(AES_ASM) && (defined(__x86_64) \ - || defined(__x86_64__) \ - || defined(_M_AMD64) \ - || defined(_M_X64))) -# undef EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK -# define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0 -#endif - -static int tls_is_multiblock_capable(OSSL_RECORD_LAYER *rl, uint8_t type, - size_t len, size_t fraglen) -{ -#if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK - if (type == SSL3_RT_APPLICATION_DATA - && len >= 4 * fraglen - && rl->compctx == NULL - && rl->msg_callback == NULL - && !rl->use_etm - && RLAYER_USE_EXPLICIT_IV(rl) - && !BIO_get_ktls_send(rl->bio) - && (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(rl->enc_ctx)) - & EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK) != 0) - return 1; -#endif - return 0; -} - -size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, uint8_t type, - size_t len, size_t maxfrag, - size_t *preffrag) -{ - if (tls_is_multiblock_capable(rl, type, len, *preffrag)) { - /* minimize address aliasing conflicts */ - if ((*preffrag & 0xfff) == 0) - *preffrag -= 512; - - if (len >= 8 * (*preffrag)) - return 8; - - return 4; - } - - return tls_get_max_records_default(rl, type, len, maxfrag, preffrag); -} - -/* - * Write records using the multiblock method. - * - * Returns 1 on success, 0 if multiblock isn't suitable (non-fatal error), or - * -1 on fatal error. - */ -static int tls_write_records_multiblock_int(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl) -{ -#if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK - size_t i; - size_t totlen; - TLS_BUFFER *wb; - unsigned char aad[13]; - EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM mb_param; - size_t packlen; - int packleni; - - if (numtempl != 4 && numtempl != 8) - return 0; - - /* - * Check templates have contiguous buffers and are all the same type and - * length - */ - for (i = 1; i < numtempl; i++) { - if (templates[i - 1].type != templates[i].type - || templates[i - 1].buflen != templates[i].buflen - || templates[i - 1].buf + templates[i - 1].buflen - != templates[i].buf) - return 0; - } - - totlen = templates[0].buflen * numtempl; - if (!tls_is_multiblock_capable(rl, templates[0].type, totlen, - templates[0].buflen)) - return 0; - - /* - * If we get this far, then multiblock is suitable - * Depending on platform multi-block can deliver several *times* - * better performance. Downside is that it has to allocate - * jumbo buffer to accommodate up to 8 records, but the - * compromise is considered worthy. - */ - - /* - * Allocate jumbo buffer. This will get freed next time we do a non - * multiblock write in the call to tls_setup_write_buffer() - the different - * buffer sizes will be spotted and the buffer reallocated. - */ - packlen = EVP_CIPHER_CTX_ctrl(rl->enc_ctx, - EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE, - (int)templates[0].buflen, NULL); - packlen *= numtempl; - if (!tls_setup_write_buffer(rl, 1, packlen, packlen)) { - /* RLAYERfatal() already called */ - return -1; - } - wb = &rl->wbuf[0]; - - mb_param.interleave = numtempl; - memcpy(aad, rl->sequence, 8); - aad[8] = templates[0].type; - aad[9] = (unsigned char)(templates[0].version >> 8); - aad[10] = (unsigned char)(templates[0].version); - aad[11] = 0; - aad[12] = 0; - mb_param.out = NULL; - mb_param.inp = aad; - mb_param.len = totlen; - - packleni = EVP_CIPHER_CTX_ctrl(rl->enc_ctx, - EVP_CTRL_TLS1_1_MULTIBLOCK_AAD, - sizeof(mb_param), &mb_param); - packlen = (size_t)packleni; - if (packleni <= 0 || packlen > wb->len) { /* never happens */ - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return -1; - } - - mb_param.out = wb->buf; - mb_param.inp = templates[0].buf; - mb_param.len = totlen; - - if (EVP_CIPHER_CTX_ctrl(rl->enc_ctx, - EVP_CTRL_TLS1_1_MULTIBLOCK_ENCRYPT, - sizeof(mb_param), &mb_param) <= 0) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return -1; - } - - rl->sequence[7] += mb_param.interleave; - if (rl->sequence[7] < mb_param.interleave) { - int j = 6; - while (j >= 0 && (++rl->sequence[j--]) == 0) ; - } - - wb->offset = 0; - wb->left = packlen; - - return 1; -#else /* !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK */ - return 0; -#endif -} - -int tls_write_records_multiblock(OSSL_RECORD_LAYER *rl, - OSSL_RECORD_TEMPLATE *templates, - size_t numtempl) -{ - int ret; - - ret = tls_write_records_multiblock_int(rl, templates, numtempl); - if (ret < 0) { - /* RLAYERfatal already called */ - return 0; - } - if (ret == 0) { - /* Multiblock wasn't suitable so just do a standard write */ - if (!tls_write_records_default(rl, templates, numtempl)) { - /* RLAYERfatal already called */ - return 0; - } - } - - return 1; -} diff --git a/openssl/src/ssl/record/methods/tlsany_meth.c b/openssl/src/ssl/record/methods/tlsany_meth.c deleted file mode 100644 index 3f73f9ebd..000000000 --- a/openssl/src/ssl/record/methods/tlsany_meth.c +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "../../ssl_local.h" -#include "../record_local.h" -#include "recmethod_local.h" - -#define MIN_SSL2_RECORD_LEN 9 - -static int tls_any_set_crypto_state(OSSL_RECORD_LAYER *rl, int level, - unsigned char *key, size_t keylen, - unsigned char *iv, size_t ivlen, - unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, - size_t taglen, - int mactype, - const EVP_MD *md, - COMP_METHOD *comp) -{ - if (level != OSSL_RECORD_PROTECTION_LEVEL_NONE) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return OSSL_RECORD_RETURN_FATAL; - } - - /* No crypto protection at the "NONE" level so nothing to be done */ - - return OSSL_RECORD_RETURN_SUCCESS; -} - -static int tls_any_cipher(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *recs, - size_t n_recs, int sending, SSL_MAC_BUF *macs, - size_t macsize) -{ - return 1; -} - -static int tls_validate_record_header(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec) -{ - if (rec->rec_version == SSL2_VERSION) { - /* SSLv2 format ClientHello */ - if (!ossl_assert(rl->version == TLS_ANY_VERSION)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - if (rec->length < MIN_SSL2_RECORD_LEN) { - RLAYERfatal(rl, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); - return 0; - } - } else { - if (rl->version == TLS_ANY_VERSION) { - if ((rec->rec_version >> 8) != SSL3_VERSION_MAJOR) { - if (rl->is_first_record) { - unsigned char *p; - - /* - * Go back to start of packet, look at the five bytes that - * we have. - */ - p = rl->packet; - if (HAS_PREFIX((char *)p, "GET ") || - HAS_PREFIX((char *)p, "POST ") || - HAS_PREFIX((char *)p, "HEAD ") || - HAS_PREFIX((char *)p, "PUT ")) { - RLAYERfatal(rl, SSL_AD_NO_ALERT, SSL_R_HTTP_REQUEST); - return 0; - } else if (HAS_PREFIX((char *)p, "CONNE")) { - RLAYERfatal(rl, SSL_AD_NO_ALERT, - SSL_R_HTTPS_PROXY_REQUEST); - return 0; - } - - /* Doesn't look like TLS - don't send an alert */ - RLAYERfatal(rl, SSL_AD_NO_ALERT, - SSL_R_WRONG_VERSION_NUMBER); - return 0; - } else { - RLAYERfatal(rl, SSL_AD_PROTOCOL_VERSION, - SSL_R_WRONG_VERSION_NUMBER); - return 0; - } - } - } else if (rl->version == TLS1_3_VERSION) { - /* - * In this case we know we are going to negotiate TLSv1.3, but we've - * had an HRR, so we haven't actually done so yet. In TLSv1.3 we - * must ignore the legacy record version in plaintext records. - */ - } else if (rec->rec_version != rl->version) { - if ((rl->version & 0xFF00) == (rec->rec_version & 0xFF00)) { - if (rec->type == SSL3_RT_ALERT) { - /* - * The record is using an incorrect version number, - * but what we've got appears to be an alert. We - * haven't read the body yet to check whether its a - * fatal or not - but chances are it is. We probably - * shouldn't send a fatal alert back. We'll just - * end. - */ - RLAYERfatal(rl, SSL_AD_NO_ALERT, - SSL_R_WRONG_VERSION_NUMBER); - return 0; - } - /* Send back error using their minor version number */ - rl->version = (unsigned short)rec->rec_version; - } - RLAYERfatal(rl, SSL_AD_PROTOCOL_VERSION, - SSL_R_WRONG_VERSION_NUMBER); - return 0; - } - } - if (rec->length > SSL3_RT_MAX_PLAIN_LENGTH) { - /* - * We use SSL_R_DATA_LENGTH_TOO_LONG instead of - * SSL_R_ENCRYPTED_LENGTH_TOO_LONG here because we are the "any" method - * and we know that we are dealing with plaintext data - */ - RLAYERfatal(rl, SSL_AD_RECORD_OVERFLOW, SSL_R_DATA_LENGTH_TOO_LONG); - return 0; - } - return 1; -} - -static int tls_any_set_protocol_version(OSSL_RECORD_LAYER *rl, int vers) -{ - if (rl->version != TLS_ANY_VERSION && rl->version != vers) - return 0; - rl->version = vers; - - return 1; -} - -static int tls_any_prepare_for_encryption(OSSL_RECORD_LAYER *rl, - size_t mac_size, - WPACKET *thispkt, - TLS_RL_RECORD *thiswr) -{ - /* No encryption, so nothing to do */ - return 1; -} - -const struct record_functions_st tls_any_funcs = { - tls_any_set_crypto_state, - tls_any_cipher, - NULL, - tls_any_set_protocol_version, - tls_default_read_n, - tls_get_more_records, - tls_validate_record_header, - tls_default_post_process_record, - tls_get_max_records_default, - tls_write_records_default, - tls_allocate_write_buffers_default, - tls_initialise_write_packets_default, - NULL, - tls_prepare_record_header_default, - NULL, - tls_any_prepare_for_encryption, - tls_post_encryption_processing_default, - NULL -}; - -static int dtls_any_set_protocol_version(OSSL_RECORD_LAYER *rl, int vers) -{ - if (rl->version != DTLS_ANY_VERSION && rl->version != vers) - return 0; - rl->version = vers; - - return 1; -} - -const struct record_functions_st dtls_any_funcs = { - tls_any_set_crypto_state, - tls_any_cipher, - NULL, - dtls_any_set_protocol_version, - tls_default_read_n, - dtls_get_more_records, - NULL, - NULL, - NULL, - tls_write_records_default, - tls_allocate_write_buffers_default, - tls_initialise_write_packets_default, - NULL, - dtls_prepare_record_header, - NULL, - tls_prepare_for_encryption_default, - dtls_post_encryption_processing, - NULL -}; diff --git a/openssl/src/ssl/record/rec_layer_d1.c b/openssl/src/ssl/record/rec_layer_d1.c index ee45f8117..775e5afca 100644 --- a/openssl/src/ssl/record/rec_layer_d1.c +++ b/openssl/src/ssl/record/rec_layer_d1.c @@ -1,5 +1,5 @@ /* - * Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -20,14 +20,22 @@ int DTLS_RECORD_LAYER_new(RECORD_LAYER *rl) { DTLS_RECORD_LAYER *d; - if ((d = OPENSSL_malloc(sizeof(*d))) == NULL) + if ((d = OPENSSL_malloc(sizeof(*d))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } rl->d = d; - d->buffered_app_data = pqueue_new(); + d->unprocessed_rcds.q = pqueue_new(); + d->processed_rcds.q = pqueue_new(); + d->buffered_app_data.q = pqueue_new(); - if (d->buffered_app_data == NULL) { + if (d->unprocessed_rcds.q == NULL || d->processed_rcds.q == NULL + || d->buffered_app_data.q == NULL) { + pqueue_free(d->unprocessed_rcds.q); + pqueue_free(d->processed_rcds.q); + pqueue_free(d->buffered_app_data.q); OPENSSL_free(d); rl->d = NULL; return 0; @@ -42,7 +50,9 @@ void DTLS_RECORD_LAYER_free(RECORD_LAYER *rl) return; DTLS_RECORD_LAYER_clear(rl); - pqueue_free(rl->d->buffered_app_data); + pqueue_free(rl->d->unprocessed_rcds.q); + pqueue_free(rl->d->processed_rcds.q); + pqueue_free(rl->d->buffered_app_data.q); OPENSSL_free(rl->d); rl->d = NULL; } @@ -51,42 +61,99 @@ void DTLS_RECORD_LAYER_clear(RECORD_LAYER *rl) { DTLS_RECORD_LAYER *d; pitem *item = NULL; - TLS_RECORD *rec; + DTLS1_RECORD_DATA *rdata; + pqueue *unprocessed_rcds; + pqueue *processed_rcds; pqueue *buffered_app_data; d = rl->d; - while ((item = pqueue_pop(d->buffered_app_data)) != NULL) { - rec = (TLS_RECORD *)item->data; + while ((item = pqueue_pop(d->unprocessed_rcds.q)) != NULL) { + rdata = (DTLS1_RECORD_DATA *)item->data; + OPENSSL_free(rdata->rbuf.buf); + OPENSSL_free(item->data); + pitem_free(item); + } + while ((item = pqueue_pop(d->processed_rcds.q)) != NULL) { + rdata = (DTLS1_RECORD_DATA *)item->data; if (rl->s->options & SSL_OP_CLEANSE_PLAINTEXT) - OPENSSL_cleanse(rec->allocdata, rec->length); - OPENSSL_free(rec->allocdata); + OPENSSL_cleanse(rdata->rbuf.buf, rdata->rbuf.len); + OPENSSL_free(rdata->rbuf.buf); OPENSSL_free(item->data); pitem_free(item); } - buffered_app_data = d->buffered_app_data; + while ((item = pqueue_pop(d->buffered_app_data.q)) != NULL) { + rdata = (DTLS1_RECORD_DATA *)item->data; + if (rl->s->options & SSL_OP_CLEANSE_PLAINTEXT) + OPENSSL_cleanse(rdata->rbuf.buf, rdata->rbuf.len); + OPENSSL_free(rdata->rbuf.buf); + OPENSSL_free(item->data); + pitem_free(item); + } + + unprocessed_rcds = d->unprocessed_rcds.q; + processed_rcds = d->processed_rcds.q; + buffered_app_data = d->buffered_app_data.q; memset(d, 0, sizeof(*d)); - d->buffered_app_data = buffered_app_data; + d->unprocessed_rcds.q = unprocessed_rcds; + d->processed_rcds.q = processed_rcds; + d->buffered_app_data.q = buffered_app_data; +} + +void DTLS_RECORD_LAYER_set_saved_w_epoch(RECORD_LAYER *rl, unsigned short e) +{ + if (e == rl->d->w_epoch - 1) { + memcpy(rl->d->curr_write_sequence, + rl->write_sequence, sizeof(rl->write_sequence)); + memcpy(rl->write_sequence, + rl->d->last_write_sequence, sizeof(rl->write_sequence)); + } else if (e == rl->d->w_epoch + 1) { + memcpy(rl->d->last_write_sequence, + rl->write_sequence, sizeof(unsigned char[8])); + memcpy(rl->write_sequence, + rl->d->curr_write_sequence, sizeof(rl->write_sequence)); + } + rl->d->w_epoch = e; +} + +void DTLS_RECORD_LAYER_set_write_sequence(RECORD_LAYER *rl, unsigned char *seq) +{ + memcpy(rl->write_sequence, seq, SEQ_NUM_SIZE); +} + +/* copy buffered record into SSL structure */ +static int dtls1_copy_record(SSL *s, pitem *item) +{ + DTLS1_RECORD_DATA *rdata; + + rdata = (DTLS1_RECORD_DATA *)item->data; + + SSL3_BUFFER_release(&s->rlayer.rbuf); + + s->rlayer.packet = rdata->packet; + s->rlayer.packet_length = rdata->packet_length; + memcpy(&s->rlayer.rbuf, &(rdata->rbuf), sizeof(SSL3_BUFFER)); + memcpy(&s->rlayer.rrec, &(rdata->rrec), sizeof(SSL3_RECORD)); + + /* Set proper sequence number for mac calculation */ + memcpy(&(s->rlayer.read_sequence[2]), &(rdata->packet[5]), 6); + + return 1; } -static int dtls_buffer_record(SSL_CONNECTION *s, TLS_RECORD *rec) +int dtls1_buffer_record(SSL *s, record_pqueue *queue, unsigned char *priority) { - TLS_RECORD *rdata; + DTLS1_RECORD_DATA *rdata; pitem *item; - struct pqueue_st *queue = s->rlayer.d->buffered_app_data; /* Limit the size of the queue to prevent DOS attacks */ - if (pqueue_size(queue) >= 100) + if (pqueue_size(queue->q) >= 100) return 0; - /* We don't buffer partially read records */ - if (!ossl_assert(rec->off == 0)) - return -1; - rdata = OPENSSL_malloc(sizeof(*rdata)); - item = pitem_new(rec->seq_num, rdata); + item = pitem_new(priority, rdata); if (rdata == NULL || item == NULL) { OPENSSL_free(rdata); pitem_free(item); @@ -94,40 +161,39 @@ static int dtls_buffer_record(SSL_CONNECTION *s, TLS_RECORD *rec) return -1; } - *rdata = *rec; - /* - * We will release the record from the record layer soon, so we take a copy - * now. Copying data isn't good - but this should be infrequent so we - * accept it here. - */ - rdata->data = rdata->allocdata = OPENSSL_memdup(rec->data, rec->length); - if (rdata->data == NULL) { - OPENSSL_free(rdata); - pitem_free(item); - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); - return -1; - } - /* - * We use a NULL rechandle to indicate that the data field has been - * allocated by us. - */ - rdata->rechandle = NULL; + rdata->packet = s->rlayer.packet; + rdata->packet_length = s->rlayer.packet_length; + memcpy(&(rdata->rbuf), &s->rlayer.rbuf, sizeof(SSL3_BUFFER)); + memcpy(&(rdata->rrec), &s->rlayer.rrec, sizeof(SSL3_RECORD)); item->data = rdata; #ifndef OPENSSL_NO_SCTP /* Store bio_dgram_sctp_rcvinfo struct */ - if (BIO_dgram_is_sctp(s->rbio) && - (ossl_statem_get_state(s) == TLS_ST_SR_FINISHED - || ossl_statem_get_state(s) == TLS_ST_CR_FINISHED)) { - BIO_ctrl(s->rbio, BIO_CTRL_DGRAM_SCTP_GET_RCVINFO, + if (BIO_dgram_is_sctp(SSL_get_rbio(s)) && + (SSL_get_state(s) == TLS_ST_SR_FINISHED + || SSL_get_state(s) == TLS_ST_CR_FINISHED)) { + BIO_ctrl(SSL_get_rbio(s), BIO_CTRL_DGRAM_SCTP_GET_RCVINFO, sizeof(rdata->recordinfo), &rdata->recordinfo); } #endif - if (pqueue_insert(queue, item) == NULL) { + s->rlayer.packet = NULL; + s->rlayer.packet_length = 0; + memset(&s->rlayer.rbuf, 0, sizeof(s->rlayer.rbuf)); + memset(&s->rlayer.rrec, 0, sizeof(s->rlayer.rrec)); + + if (!ssl3_setup_buffers(s)) { + /* SSLfatal() already called */ + OPENSSL_free(rdata->rbuf.buf); + OPENSSL_free(rdata); + pitem_free(item); + return -1; + } + + if (pqueue_insert(queue->q, item) == NULL) { /* Must be a duplicate so ignore it */ - OPENSSL_free(rdata->allocdata); + OPENSSL_free(rdata->rbuf.buf); OPENSSL_free(rdata); pitem_free(item); } @@ -135,42 +201,121 @@ static int dtls_buffer_record(SSL_CONNECTION *s, TLS_RECORD *rec) return 1; } -/* Unbuffer a previously buffered TLS_RECORD structure if any */ -static void dtls_unbuffer_record(SSL_CONNECTION *s) +int dtls1_retrieve_buffered_record(SSL *s, record_pqueue *queue) { - TLS_RECORD *rdata; pitem *item; - /* If we already have records to handle then do nothing */ - if (s->rlayer.curr_rec < s->rlayer.num_recs) - return; + item = pqueue_pop(queue->q); + if (item) { + dtls1_copy_record(s, item); + + OPENSSL_free(item->data); + pitem_free(item); - item = pqueue_pop(s->rlayer.d->buffered_app_data); - if (item != NULL) { - rdata = (TLS_RECORD *)item->data; + return 1; + } - s->rlayer.tlsrecs[0] = *rdata; - s->rlayer.num_recs = 1; - s->rlayer.curr_rec = 0; + return 0; +} -#ifndef OPENSSL_NO_SCTP - /* Restore bio_dgram_sctp_rcvinfo struct */ - if (BIO_dgram_is_sctp(s->rbio)) { - BIO_ctrl(s->rbio, BIO_CTRL_DGRAM_SCTP_SET_RCVINFO, - sizeof(rdata->recordinfo), &rdata->recordinfo); +/* + * retrieve a buffered record that belongs to the new epoch, i.e., not + * processed yet + */ +#define dtls1_get_unprocessed_record(s) \ + dtls1_retrieve_buffered_record((s), \ + &((s)->rlayer.d->unprocessed_rcds)) + +int dtls1_process_buffered_records(SSL *s) +{ + pitem *item; + SSL3_BUFFER *rb; + SSL3_RECORD *rr; + DTLS1_BITMAP *bitmap; + unsigned int is_next_epoch; + int replayok = 1; + + item = pqueue_peek(s->rlayer.d->unprocessed_rcds.q); + if (item) { + /* Check if epoch is current. */ + if (s->rlayer.d->unprocessed_rcds.epoch != s->rlayer.d->r_epoch) + return 1; /* Nothing to do. */ + + rr = RECORD_LAYER_get_rrec(&s->rlayer); + + rb = RECORD_LAYER_get_rbuf(&s->rlayer); + + if (SSL3_BUFFER_get_left(rb) > 0) { + /* + * We've still got data from the current packet to read. There could + * be a record from the new epoch in it - so don't overwrite it + * with the unprocessed records yet (we'll do it when we've + * finished reading the current packet). + */ + return 1; } + + /* Process all the records. */ + while (pqueue_peek(s->rlayer.d->unprocessed_rcds.q)) { + dtls1_get_unprocessed_record(s); + bitmap = dtls1_get_bitmap(s, rr, &is_next_epoch); + if (bitmap == NULL) { + /* + * Should not happen. This will only ever be NULL when the + * current record is from a different epoch. But that cannot + * be the case because we already checked the epoch above + */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } +#ifndef OPENSSL_NO_SCTP + /* Only do replay check if no SCTP bio */ + if (!BIO_dgram_is_sctp(SSL_get_rbio(s))) #endif + { + /* + * Check whether this is a repeat, or aged record. We did this + * check once already when we first received the record - but + * we might have updated the window since then due to + * records we subsequently processed. + */ + replayok = dtls1_record_replay_check(s, bitmap); + } - OPENSSL_free(item->data); - pitem_free(item); + if (!replayok || !dtls1_process_record(s, bitmap)) { + if (ossl_statem_in_error(s)) { + /* dtls1_process_record called SSLfatal() */ + return -1; + } + /* dump this record */ + rr->length = 0; + RECORD_LAYER_reset_packet_length(&s->rlayer); + continue; + } + + if (dtls1_buffer_record(s, &(s->rlayer.d->processed_rcds), + SSL3_RECORD_get_seq_num(s->rlayer.rrec)) < 0) { + /* SSLfatal() already called */ + return 0; + } + } } + + /* + * sync epoch numbers once all the unprocessed records have been + * processed + */ + s->rlayer.d->processed_rcds.epoch = s->rlayer.d->r_epoch; + s->rlayer.d->unprocessed_rcds.epoch = s->rlayer.d->r_epoch + 1; + + return 1; } /*- * Return up to 'len' payload bytes received in 'type' records. * 'type' is one of the following: * - * - SSL3_RT_HANDSHAKE + * - SSL3_RT_HANDSHAKE (when ssl3_get_message calls us) * - SSL3_RT_APPLICATION_DATA (when ssl3_read calls us) * - 0 (during a shutdown, no data has to be returned) * @@ -195,29 +340,32 @@ static void dtls_unbuffer_record(SSL_CONNECTION *s) * Application data protocol * none of our business */ -int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, - unsigned char *buf, size_t len, - int peek, size_t *readbytes) +int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf, + size_t len, int peek, size_t *readbytes) { - int i, j, ret; + int i, j, iret; size_t n; - TLS_RECORD *rr; + SSL3_RECORD *rr; void (*cb) (const SSL *ssl, int type2, int val) = NULL; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - if (sc == NULL) - return -1; + if (!SSL3_BUFFER_is_initialised(&s->rlayer.rbuf)) { + /* Not initialized yet */ + if (!ssl3_setup_buffers(s)) { + /* SSLfatal() already called */ + return -1; + } + } if ((type && (type != SSL3_RT_APPLICATION_DATA) && (type != SSL3_RT_HANDSHAKE)) || (peek && (type != SSL3_RT_APPLICATION_DATA))) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return -1; } - if (!ossl_statem_get_in_handshake(sc) && SSL_in_init(s)) { + if (!ossl_statem_get_in_handshake(s) && SSL_in_init(s)) { /* type == SSL3_RT_APPLICATION_DATA */ - i = sc->handshake_func(s); + i = s->handshake_func(s); /* SSLfatal() already called if appropriate */ if (i < 0) return i; @@ -226,76 +374,92 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, } start: - sc->rwstate = SSL_NOTHING; + s->rwstate = SSL_NOTHING; + + /*- + * s->s3.rrec.type - is the type of record + * s->s3.rrec.data, - data + * s->s3.rrec.off, - offset into 'data' for next read + * s->s3.rrec.length, - number of bytes. + */ + rr = s->rlayer.rrec; /* * We are not handshaking and have no data yet, so process data buffered * during the last handshake in advance, if any. */ - if (SSL_is_init_finished(s)) - dtls_unbuffer_record(sc); + if (SSL_is_init_finished(s) && SSL3_RECORD_get_length(rr) == 0) { + pitem *item; + item = pqueue_pop(s->rlayer.d->buffered_app_data.q); + if (item) { +#ifndef OPENSSL_NO_SCTP + /* Restore bio_dgram_sctp_rcvinfo struct */ + if (BIO_dgram_is_sctp(SSL_get_rbio(s))) { + DTLS1_RECORD_DATA *rdata = (DTLS1_RECORD_DATA *)item->data; + BIO_ctrl(SSL_get_rbio(s), BIO_CTRL_DGRAM_SCTP_SET_RCVINFO, + sizeof(rdata->recordinfo), &rdata->recordinfo); + } +#endif + + dtls1_copy_record(s, item); + + OPENSSL_free(item->data); + pitem_free(item); + } + } /* Check for timeout */ - if (dtls1_handle_timeout(sc) > 0) { + if (dtls1_handle_timeout(s) > 0) { goto start; - } else if (ossl_statem_in_error(sc)) { + } else if (ossl_statem_in_error(s)) { /* dtls1_handle_timeout() has failed with a fatal error */ return -1; } /* get new packet if necessary */ - if (sc->rlayer.curr_rec >= sc->rlayer.num_recs) { - sc->rlayer.curr_rec = sc->rlayer.num_recs = 0; - do { - rr = &sc->rlayer.tlsrecs[sc->rlayer.num_recs]; - - ret = HANDLE_RLAYER_READ_RETURN(sc, - sc->rlayer.rrlmethod->read_record(sc->rlayer.rrl, - &rr->rechandle, - &rr->version, &rr->type, - &rr->data, &rr->length, - &rr->epoch, rr->seq_num)); - if (ret <= 0) { - ret = dtls1_read_failed(sc, ret); - /* - * Anything other than a timeout is an error. SSLfatal() already - * called if appropriate. - */ - if (ret <= 0) - return ret; - else - goto start; - } - rr->off = 0; - sc->rlayer.num_recs++; - } while (sc->rlayer.rrlmethod->processed_read_pending(sc->rlayer.rrl) - && sc->rlayer.num_recs < SSL_MAX_PIPELINES); + if ((SSL3_RECORD_get_length(rr) == 0) + || (s->rlayer.rstate == SSL_ST_READ_BODY)) { + RECORD_LAYER_set_numrpipes(&s->rlayer, 0); + iret = dtls1_get_record(s); + if (iret <= 0) { + iret = dtls1_read_failed(s, iret); + /* + * Anything other than a timeout is an error. SSLfatal() already + * called if appropriate. + */ + if (iret <= 0) + return iret; + else + goto start; + } + RECORD_LAYER_set_numrpipes(&s->rlayer, 1); } - rr = &sc->rlayer.tlsrecs[sc->rlayer.curr_rec]; /* * Reset the count of consecutive warning alerts if we've got a non-empty * record that isn't an alert. */ - if (rr->type != SSL3_RT_ALERT && rr->length != 0) - sc->rlayer.alert_count = 0; + if (SSL3_RECORD_get_type(rr) != SSL3_RT_ALERT + && SSL3_RECORD_get_length(rr) != 0) + s->rlayer.alert_count = 0; /* we now have a packet which can be read and processed */ - if (sc->s3.change_cipher_spec /* set when we receive ChangeCipherSpec, + if (s->s3.change_cipher_spec /* set when we receive ChangeCipherSpec, * reset by ssl3_get_finished */ - && (rr->type != SSL3_RT_HANDSHAKE)) { + && (SSL3_RECORD_get_type(rr) != SSL3_RT_HANDSHAKE)) { /* * We now have application data between CCS and Finished. Most likely * the packets were reordered on their way, so buffer the application * data for later processing rather than dropping the connection. */ - if (dtls_buffer_record(sc, rr) < 0) { + if (dtls1_buffer_record(s, &(s->rlayer.d->buffered_app_data), + SSL3_RECORD_get_seq_num(rr)) < 0) { /* SSLfatal() already called */ return -1; } - if (!ssl_release_record(sc, rr, 0)) - return -1; + SSL3_RECORD_set_length(rr, 0); + SSL3_RECORD_set_read(rr); goto start; } @@ -303,15 +467,15 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * If the other end has shut down, throw anything we read away (even in * 'peek' mode) */ - if (sc->shutdown & SSL_RECEIVED_SHUTDOWN) { - if (!ssl_release_record(sc, rr, 0)) - return -1; - sc->rwstate = SSL_NOTHING; + if (s->shutdown & SSL_RECEIVED_SHUTDOWN) { + SSL3_RECORD_set_length(rr, 0); + SSL3_RECORD_set_read(rr); + s->rwstate = SSL_NOTHING; return 0; } - if (type == rr->type - || (rr->type == SSL3_RT_CHANGE_CIPHER_SPEC + if (type == SSL3_RECORD_get_type(rr) + || (SSL3_RECORD_get_type(rr) == SSL3_RT_CHANGE_CIPHER_SPEC && type == SSL3_RT_HANDSHAKE && recvd_type != NULL)) { /* * SSL3_RT_APPLICATION_DATA or @@ -322,39 +486,46 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * make sure that we are not getting application data when we are * doing a handshake for the first time */ - if (SSL_in_init(s) && (type == SSL3_RT_APPLICATION_DATA) - && (SSL_IS_FIRST_HANDSHAKE(sc))) { - SSLfatal(sc, SSL_AD_UNEXPECTED_MESSAGE, + if (SSL_in_init(s) && (type == SSL3_RT_APPLICATION_DATA) && + (s->enc_read_ctx == NULL)) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_APP_DATA_IN_HANDSHAKE); return -1; } if (recvd_type != NULL) - *recvd_type = rr->type; + *recvd_type = SSL3_RECORD_get_type(rr); if (len == 0) { /* - * Release a zero length record. This ensures multiple calls to + * Mark a zero length record as read. This ensures multiple calls to * SSL_read() with a zero length buffer will eventually cause * SSL_pending() to report data as being available. */ - if (rr->length == 0 && !ssl_release_record(sc, rr, 0)) - return -1; + if (SSL3_RECORD_get_length(rr) == 0) + SSL3_RECORD_set_read(rr); return 0; } - if (len > rr->length) - n = rr->length; + if (len > SSL3_RECORD_get_length(rr)) + n = SSL3_RECORD_get_length(rr); else n = len; - memcpy(buf, &(rr->data[rr->off]), n); + memcpy(buf, &(SSL3_RECORD_get_data(rr)[SSL3_RECORD_get_off(rr)]), n); if (peek) { - if (rr->length == 0 && !ssl_release_record(sc, rr, 0)) - return -1; + if (SSL3_RECORD_get_length(rr) == 0) + SSL3_RECORD_set_read(rr); } else { - if (!ssl_release_record(sc, rr, n)) - return -1; + if (s->options & SSL_OP_CLEANSE_PLAINTEXT) + OPENSSL_cleanse(&(SSL3_RECORD_get_data(rr)[SSL3_RECORD_get_off(rr)]), n); + SSL3_RECORD_sub_length(rr, n); + SSL3_RECORD_add_off(rr, n); + if (SSL3_RECORD_get_length(rr) == 0) { + s->rlayer.rstate = SSL_ST_READ_HEADER; + SSL3_RECORD_set_off(rr, 0); + SSL3_RECORD_set_read(rr); + } } #ifndef OPENSSL_NO_SCTP /* @@ -363,9 +534,9 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * anymore, finally set shutdown. */ if (BIO_dgram_is_sctp(SSL_get_rbio(s)) && - sc->d1->shutdown_received - && BIO_dgram_sctp_msg_waiting(SSL_get_rbio(s)) <= 0) { - sc->shutdown |= SSL_RECEIVED_SHUTDOWN; + s->d1->shutdown_received + && !BIO_dgram_sctp_msg_waiting(SSL_get_rbio(s))) { + s->shutdown |= SSL_RECEIVED_SHUTDOWN; return 0; } #endif @@ -378,25 +549,28 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * then it was unexpected (Hello Request or Client Hello). */ - if (rr->type == SSL3_RT_ALERT) { + if (SSL3_RECORD_get_type(rr) == SSL3_RT_ALERT) { unsigned int alert_level, alert_descr; - const unsigned char *alert_bytes = rr->data + rr->off; + unsigned char *alert_bytes = SSL3_RECORD_get_data(rr) + + SSL3_RECORD_get_off(rr); PACKET alert; - if (!PACKET_buf_init(&alert, alert_bytes, rr->length) + if (!PACKET_buf_init(&alert, alert_bytes, SSL3_RECORD_get_length(rr)) || !PACKET_get_1(&alert, &alert_level) || !PACKET_get_1(&alert, &alert_descr) || PACKET_remaining(&alert) != 0) { - SSLfatal(sc, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_INVALID_ALERT); + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_INVALID_ALERT); return -1; } - if (sc->msg_callback) - sc->msg_callback(0, sc->version, SSL3_RT_ALERT, alert_bytes, 2, s, - sc->msg_callback_arg); + s->s3.alert_level = alert_level; - if (sc->info_callback != NULL) - cb = sc->info_callback; + if (s->msg_callback) + s->msg_callback(0, s->version, SSL3_RT_ALERT, alert_bytes, 2, s, + s->msg_callback_arg); + + if (s->info_callback != NULL) + cb = s->info_callback; else if (s->ctx->info_callback != NULL) cb = s->ctx->info_callback; @@ -406,13 +580,12 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, } if (alert_level == SSL3_AL_WARNING) { - sc->s3.warn_alert = alert_descr; - if (!ssl_release_record(sc, rr, 0)) - return -1; + s->s3.warn_alert = alert_descr; + SSL3_RECORD_set_read(rr); - sc->rlayer.alert_count++; - if (sc->rlayer.alert_count == MAX_WARN_ALERT_COUNT) { - SSLfatal(sc, SSL_AD_UNEXPECTED_MESSAGE, + s->rlayer.alert_count++; + if (s->rlayer.alert_count == MAX_WARN_ALERT_COUNT) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_TOO_MANY_WARN_ALERTS); return -1; } @@ -425,68 +598,68 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * that nothing gets discarded. */ if (BIO_dgram_is_sctp(SSL_get_rbio(s)) && - BIO_dgram_sctp_msg_waiting(SSL_get_rbio(s)) > 0) { - sc->d1->shutdown_received = 1; - sc->rwstate = SSL_READING; + BIO_dgram_sctp_msg_waiting(SSL_get_rbio(s))) { + s->d1->shutdown_received = 1; + s->rwstate = SSL_READING; BIO_clear_retry_flags(SSL_get_rbio(s)); BIO_set_retry_read(SSL_get_rbio(s)); return -1; } #endif - sc->shutdown |= SSL_RECEIVED_SHUTDOWN; + s->shutdown |= SSL_RECEIVED_SHUTDOWN; return 0; } } else if (alert_level == SSL3_AL_FATAL) { - sc->rwstate = SSL_NOTHING; - sc->s3.fatal_alert = alert_descr; - SSLfatal_data(sc, SSL_AD_NO_ALERT, + s->rwstate = SSL_NOTHING; + s->s3.fatal_alert = alert_descr; + SSLfatal_data(s, SSL_AD_NO_ALERT, SSL_AD_REASON_OFFSET + alert_descr, "SSL alert number %d", alert_descr); - sc->shutdown |= SSL_RECEIVED_SHUTDOWN; - if (!ssl_release_record(sc, rr, 0)) - return -1; - SSL_CTX_remove_session(sc->session_ctx, sc->session); + s->shutdown |= SSL_RECEIVED_SHUTDOWN; + SSL3_RECORD_set_read(rr); + SSL_CTX_remove_session(s->session_ctx, s->session); return 0; } else { - SSLfatal(sc, SSL_AD_ILLEGAL_PARAMETER, SSL_R_UNKNOWN_ALERT_TYPE); + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_UNKNOWN_ALERT_TYPE); return -1; } goto start; } - if (sc->shutdown & SSL_SENT_SHUTDOWN) { /* but we have not received a + if (s->shutdown & SSL_SENT_SHUTDOWN) { /* but we have not received a * shutdown */ - sc->rwstate = SSL_NOTHING; - if (!ssl_release_record(sc, rr, 0)) - return -1; + s->rwstate = SSL_NOTHING; + SSL3_RECORD_set_length(rr, 0); + SSL3_RECORD_set_read(rr); return 0; } - if (rr->type == SSL3_RT_CHANGE_CIPHER_SPEC) { + if (SSL3_RECORD_get_type(rr) == SSL3_RT_CHANGE_CIPHER_SPEC) { /* * We can't process a CCS now, because previous handshake messages * are still missing, so just drop it. */ - if (!ssl_release_record(sc, rr, 0)) - return -1; + SSL3_RECORD_set_length(rr, 0); + SSL3_RECORD_set_read(rr); goto start; } /* * Unexpected handshake message (Client Hello, or protocol violation) */ - if (rr->type == SSL3_RT_HANDSHAKE && !ossl_statem_get_in_handshake(sc)) { + if ((SSL3_RECORD_get_type(rr) == SSL3_RT_HANDSHAKE) && + !ossl_statem_get_in_handshake(s)) { struct hm_header_st msg_hdr; /* * This may just be a stale retransmit. Also sanity check that we have * at least enough record bytes for a message header */ - if (rr->epoch != sc->rlayer.d->r_epoch - || rr->length < DTLS1_HM_HEADER_LENGTH) { - if (!ssl_release_record(sc, rr, 0)) - return -1; + if (SSL3_RECORD_get_epoch(rr) != s->rlayer.d->r_epoch + || SSL3_RECORD_get_length(rr) < DTLS1_HM_HEADER_LENGTH) { + SSL3_RECORD_set_length(rr, 0); + SSL3_RECORD_set_read(rr); goto start; } @@ -497,24 +670,24 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * here, then retransmit our CCS and FINISHED. */ if (msg_hdr.type == SSL3_MT_FINISHED) { - if (dtls1_check_timeout_num(sc) < 0) { + if (dtls1_check_timeout_num(s) < 0) { /* SSLfatal) already called */ return -1; } - if (dtls1_retransmit_buffered_messages(sc) <= 0) { + if (dtls1_retransmit_buffered_messages(s) <= 0) { /* Fail if we encountered a fatal error */ - if (ossl_statem_in_error(sc)) + if (ossl_statem_in_error(s)) return -1; } - if (!ssl_release_record(sc, rr, 0)) - return -1; - if (!(sc->mode & SSL_MODE_AUTO_RETRY)) { - if (!sc->rlayer.rrlmethod->unprocessed_read_pending(sc->rlayer.rrl)) { + SSL3_RECORD_set_length(rr, 0); + SSL3_RECORD_set_read(rr); + if (!(s->mode & SSL_MODE_AUTO_RETRY)) { + if (SSL3_BUFFER_get_left(&s->rlayer.rbuf) == 0) { /* no read-ahead left? */ BIO *bio; - sc->rwstate = SSL_READING; + s->rwstate = SSL_READING; bio = SSL_get_rbio(s); BIO_clear_retry_flags(bio); BIO_set_retry_read(bio); @@ -531,22 +704,22 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * finished */ if (!ossl_assert(SSL_is_init_finished(s))) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return -1; } /* We found handshake data, so we're going back into init */ - ossl_statem_set_in_init(sc, 1); + ossl_statem_set_in_init(s, 1); - i = sc->handshake_func(s); + i = s->handshake_func(s); /* SSLfatal() called if appropriate */ if (i < 0) return i; if (i == 0) return -1; - if (!(sc->mode & SSL_MODE_AUTO_RETRY)) { - if (!sc->rlayer.rrlmethod->unprocessed_read_pending(sc->rlayer.rrl)) { + if (!(s->mode & SSL_MODE_AUTO_RETRY)) { + if (SSL3_BUFFER_get_left(&s->rlayer.rbuf) == 0) { /* no read-ahead left? */ BIO *bio; /* @@ -555,7 +728,7 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * option set. Otherwise renegotiation may cause nasty * problems in the blocking world */ - sc->rwstate = SSL_READING; + s->rwstate = SSL_READING; bio = SSL_get_rbio(s); BIO_clear_retry_flags(bio); BIO_set_retry_read(bio); @@ -565,9 +738,9 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, goto start; } - switch (rr->type) { + switch (SSL3_RECORD_get_type(rr)) { default: - SSLfatal(sc, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_RECORD); + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_RECORD); return -1; case SSL3_RT_CHANGE_CIPHER_SPEC: case SSL3_RT_ALERT: @@ -577,7 +750,7 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * SSL3_RT_HANDSHAKE when ossl_statem_get_in_handshake(s) is true, but * that should not happen when type != rr->type */ - SSLfatal(sc, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); return -1; case SSL3_RT_APPLICATION_DATA: /* @@ -587,13 +760,13 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * application data at this point (session renegotiation not yet * started), we will indulge it. */ - if (sc->s3.in_read_app_data && - (sc->s3.total_renegotiations != 0) && - ossl_statem_app_data_allowed(sc)) { - sc->s3.in_read_app_data = 2; + if (s->s3.in_read_app_data && + (s->s3.total_renegotiations != 0) && + ossl_statem_app_data_allowed(s)) { + s->s3.in_read_app_data = 2; return -1; } else { - SSLfatal(sc, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_RECORD); + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_RECORD); return -1; } } @@ -604,8 +777,8 @@ int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, * Call this to write data in records of type 'type' It will return <= 0 if * not all data has been sent or non-blocking IO. */ -int dtls1_write_bytes(SSL_CONNECTION *s, uint8_t type, const void *buf, - size_t len, size_t *written) +int dtls1_write_bytes(SSL *s, int type, const void *buf, size_t len, + size_t *written) { int i; @@ -614,61 +787,253 @@ int dtls1_write_bytes(SSL_CONNECTION *s, uint8_t type, const void *buf, return -1; } s->rwstate = SSL_NOTHING; - i = do_dtls1_write(s, type, buf, len, written); + i = do_dtls1_write(s, type, buf, len, 0, written); return i; } -int do_dtls1_write(SSL_CONNECTION *sc, uint8_t type, const unsigned char *buf, - size_t len, size_t *written) +int do_dtls1_write(SSL *s, int type, const unsigned char *buf, + size_t len, int create_empty_fragment, size_t *written) { - int i; - OSSL_RECORD_TEMPLATE tmpl; - SSL *s = SSL_CONNECTION_GET_SSL(sc); - int ret; + unsigned char *p, *pseq; + int i, mac_size, clear = 0; + size_t prefix_len = 0; + int eivlen; + SSL3_RECORD wr; + SSL3_BUFFER *wb; + SSL_SESSION *sess; + + wb = &s->rlayer.wbuf[0]; + + /* + * DTLS writes whole datagrams, so there can't be anything left in + * the buffer. + */ + if (!ossl_assert(SSL3_BUFFER_get_left(wb) == 0)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } /* If we have an alert to send, lets send it */ - if (sc->s3.alert_dispatch > 0) { + if (s->s3.alert_dispatch) { i = s->method->ssl_dispatch_alert(s); if (i <= 0) return i; /* if it went, fall through and send more stuff */ } - if (len == 0) + if (len == 0 && !create_empty_fragment) return 0; - if (len > ssl_get_max_send_fragment(sc)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE); + if (len > ssl_get_max_send_fragment(s)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE); return 0; } - tmpl.type = type; + sess = s->session; + + if ((sess == NULL) + || (s->enc_write_ctx == NULL) + || (EVP_MD_CTX_get0_md(s->write_hash) == NULL)) + clear = 1; + + if (clear) + mac_size = 0; + else { + mac_size = EVP_MD_CTX_get_size(s->write_hash); + if (mac_size < 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, + SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE); + return -1; + } + } + + p = SSL3_BUFFER_get_buf(wb) + prefix_len; + + /* write the header */ + + *(p++) = type & 0xff; + SSL3_RECORD_set_type(&wr, type); /* * Special case: for hello verify request, client version 1.0 and we * haven't decided which version to use yet send back using version 1.0 * header: otherwise some clients will ignore it. */ - if (s->method->version == DTLS_ANY_VERSION - && sc->max_proto_version != DTLS1_BAD_VER) - tmpl.version = DTLS1_VERSION; - else - tmpl.version = sc->version; - tmpl.buf = buf; - tmpl.buflen = len; + if (s->method->version == DTLS_ANY_VERSION && + s->max_proto_version != DTLS1_BAD_VER) { + *(p++) = DTLS1_VERSION >> 8; + *(p++) = DTLS1_VERSION & 0xff; + } else { + *(p++) = s->version >> 8; + *(p++) = s->version & 0xff; + } + + /* field where we are to write out packet epoch, seq num and len */ + pseq = p; + p += 10; + + /* Explicit IV length, block ciphers appropriate version flag */ + if (s->enc_write_ctx) { + int mode = EVP_CIPHER_CTX_get_mode(s->enc_write_ctx); + if (mode == EVP_CIPH_CBC_MODE) { + eivlen = EVP_CIPHER_CTX_get_iv_length(s->enc_write_ctx); + if (eivlen <= 1) + eivlen = 0; + } + /* Need explicit part of IV for GCM mode */ + else if (mode == EVP_CIPH_GCM_MODE) + eivlen = EVP_GCM_TLS_EXPLICIT_IV_LEN; + else if (mode == EVP_CIPH_CCM_MODE) + eivlen = EVP_CCM_TLS_EXPLICIT_IV_LEN; + else + eivlen = 0; + } else + eivlen = 0; + + /* lets setup the record stuff. */ + SSL3_RECORD_set_data(&wr, p + eivlen); /* make room for IV in case of CBC */ + SSL3_RECORD_set_length(&wr, len); + SSL3_RECORD_set_input(&wr, (unsigned char *)buf); + + /* + * we now 'read' from wr.input, wr.length bytes into wr.data + */ + + /* first we compress */ + if (s->compress != NULL) { + if (!ssl3_do_compress(s, &wr)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_COMPRESSION_FAILURE); + return -1; + } + } else { + memcpy(SSL3_RECORD_get_data(&wr), SSL3_RECORD_get_input(&wr), + SSL3_RECORD_get_length(&wr)); + SSL3_RECORD_reset_input(&wr); + } + + /* + * we should still have the output to wr.data and the input from + * wr.input. Length should be wr.length. wr.data still points in the + * wb->buf + */ + + if (!SSL_WRITE_ETM(s) && mac_size != 0) { + if (!s->method->ssl3_enc->mac(s, &wr, + &(p[SSL3_RECORD_get_length(&wr) + eivlen]), + 1)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + SSL3_RECORD_add_length(&wr, mac_size); + } + + /* this is true regardless of mac size */ + SSL3_RECORD_set_data(&wr, p); + SSL3_RECORD_reset_input(&wr); + + if (eivlen) + SSL3_RECORD_add_length(&wr, eivlen); + + if (s->method->ssl3_enc->enc(s, &wr, 1, 1, NULL, mac_size) < 1) { + if (!ossl_statem_in_error(s)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + } + return -1; + } + + if (SSL_WRITE_ETM(s) && mac_size != 0) { + if (!s->method->ssl3_enc->mac(s, &wr, + &(p[SSL3_RECORD_get_length(&wr)]), 1)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + SSL3_RECORD_add_length(&wr, mac_size); + } + + /* record length after mac and block padding */ + + /* there's only one epoch between handshake and app data */ + + s2n(s->rlayer.d->w_epoch, pseq); + + memcpy(pseq, &(s->rlayer.write_sequence[2]), 6); + pseq += 6; + s2n(SSL3_RECORD_get_length(&wr), pseq); + + if (s->msg_callback) + s->msg_callback(1, 0, SSL3_RT_HEADER, pseq - DTLS1_RT_HEADER_LENGTH, + DTLS1_RT_HEADER_LENGTH, s, s->msg_callback_arg); + + /* + * we should now have wr.data pointing to the encrypted data, which is + * wr->length long + */ + SSL3_RECORD_set_type(&wr, type); /* not needed but helps for debugging */ + SSL3_RECORD_add_length(&wr, DTLS1_RT_HEADER_LENGTH); + + ssl3_record_sequence_update(&(s->rlayer.write_sequence[0])); + + if (create_empty_fragment) { + /* + * we are in a recursive call; just return the length, don't write + * out anything here + */ + *written = wr.length; + return 1; + } + + /* now let's set up wb */ + SSL3_BUFFER_set_left(wb, prefix_len + SSL3_RECORD_get_length(&wr)); + SSL3_BUFFER_set_offset(wb, 0); + + /* + * memorize arguments so that ssl3_write_pending can detect bad write + * retries later + */ + s->rlayer.wpend_tot = len; + s->rlayer.wpend_buf = buf; + s->rlayer.wpend_type = type; + s->rlayer.wpend_ret = len; - ret = HANDLE_RLAYER_WRITE_RETURN(sc, - sc->rlayer.wrlmethod->write_records(sc->rlayer.wrl, &tmpl, 1)); + /* we now just need to write the buffer. Calls SSLfatal() as required. */ + return ssl3_write_pending(s, type, buf, len, written); +} + +DTLS1_BITMAP *dtls1_get_bitmap(SSL *s, SSL3_RECORD *rr, + unsigned int *is_next_epoch) +{ - if (ret > 0) - *written = (int)len; + *is_next_epoch = 0; + + /* In current epoch, accept HM, CCS, DATA, & ALERT */ + if (rr->epoch == s->rlayer.d->r_epoch) + return &s->rlayer.d->bitmap; + + /* + * Only HM and ALERT messages can be from the next epoch and only if we + * have already processed all of the unprocessed records from the last + * epoch + */ + else if (rr->epoch == (unsigned long)(s->rlayer.d->r_epoch + 1) && + s->rlayer.d->unprocessed_rcds.epoch != s->rlayer.d->r_epoch && + (rr->type == SSL3_RT_HANDSHAKE || rr->type == SSL3_RT_ALERT)) { + *is_next_epoch = 1; + return &s->rlayer.d->next_bitmap; + } - return ret; + return NULL; } -void dtls1_increment_epoch(SSL_CONNECTION *s, int rw) +void dtls1_reset_seq_numbers(SSL *s, int rw) { + unsigned char *seq; + unsigned int seq_bytes = sizeof(s->rlayer.read_sequence); + if (rw & SSL3_CC_READ) { + seq = s->rlayer.read_sequence; s->rlayer.d->r_epoch++; + memcpy(&s->rlayer.d->bitmap, &s->rlayer.d->next_bitmap, + sizeof(s->rlayer.d->bitmap)); + memset(&s->rlayer.d->next_bitmap, 0, sizeof(s->rlayer.d->next_bitmap)); /* * We must not use any buffered messages received from the previous @@ -676,17 +1041,11 @@ void dtls1_increment_epoch(SSL_CONNECTION *s, int rw) */ dtls1_clear_received_buffer(s); } else { + seq = s->rlayer.write_sequence; + memcpy(s->rlayer.d->last_write_sequence, seq, + sizeof(s->rlayer.write_sequence)); s->rlayer.d->w_epoch++; } -} - -uint16_t dtls1_get_epoch(SSL_CONNECTION *s, int rw) { - uint16_t epoch; - - if (rw & SSL3_CC_READ) - epoch = s->rlayer.d->r_epoch; - else - epoch = s->rlayer.d->w_epoch; - return epoch; + memset(seq, 0, seq_bytes); } diff --git a/openssl/src/ssl/record/rec_layer_s3.c b/openssl/src/ssl/record/rec_layer_s3.c index 6a31efe1c..fe1b3f3c1 100644 --- a/openssl/src/ssl/record/rec_layer_s3.c +++ b/openssl/src/ssl/record/rec_layer_s3.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -10,190 +10,118 @@ #include #include #include -#include #include "../ssl_local.h" -#include "../quic/quic_local.h" #include #include #include -#include #include "record_local.h" #include "internal/packet.h" -void RECORD_LAYER_init(RECORD_LAYER *rl, SSL_CONNECTION *s) +#if defined(OPENSSL_SMALL_FOOTPRINT) || \ + !( defined(AES_ASM) && ( \ + defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_AMD64) || defined(_M_X64) ) \ + ) +# undef EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK +# define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0 +#endif + +void RECORD_LAYER_init(RECORD_LAYER *rl, SSL *s) { rl->s = s; + RECORD_LAYER_set_first_record(&s->rlayer); + SSL3_RECORD_clear(rl->rrec, SSL_MAX_PIPELINES); } -int RECORD_LAYER_clear(RECORD_LAYER *rl) +void RECORD_LAYER_clear(RECORD_LAYER *rl) { - int ret = 1; - - /* Clear any buffered records we no longer need */ - while (rl->curr_rec < rl->num_recs) - ret &= ssl_release_record(rl->s, - &(rl->tlsrecs[rl->curr_rec++]), - 0); + rl->rstate = SSL_ST_READ_HEADER; + /* + * Do I need to clear read_ahead? As far as I can tell read_ahead did not + * previously get reset by SSL_clear...so I'll keep it that way..but is + * that right? + */ + rl->packet = NULL; + rl->packet_length = 0; rl->wnum = 0; memset(rl->handshake_fragment, 0, sizeof(rl->handshake_fragment)); rl->handshake_fragment_len = 0; rl->wpend_tot = 0; rl->wpend_type = 0; + rl->wpend_ret = 0; rl->wpend_buf = NULL; - rl->alert_count = 0; - rl->num_recs = 0; - rl->curr_rec = 0; - - BIO_free(rl->rrlnext); - rl->rrlnext = NULL; - - if (rl->rrlmethod != NULL) - rl->rrlmethod->free(rl->rrl); /* Ignore return value */ - if (rl->wrlmethod != NULL) - rl->wrlmethod->free(rl->wrl); /* Ignore return value */ - BIO_free(rl->rrlnext); - rl->rrlmethod = NULL; - rl->wrlmethod = NULL; - rl->rrlnext = NULL; - rl->rrl = NULL; - rl->wrl = NULL; + + SSL3_BUFFER_clear(&rl->rbuf); + ssl3_release_write_buffer(rl->s); + rl->numrpipes = 0; + SSL3_RECORD_clear(rl->rrec, SSL_MAX_PIPELINES); + + RECORD_LAYER_reset_read_sequence(rl); + RECORD_LAYER_reset_write_sequence(rl); if (rl->d) DTLS_RECORD_LAYER_clear(rl); - - return ret; } -int RECORD_LAYER_reset(RECORD_LAYER *rl) +void RECORD_LAYER_release(RECORD_LAYER *rl) { - int ret; - - ret = RECORD_LAYER_clear(rl); - - /* We try and reset both record layers even if one fails */ - ret &= ssl_set_new_record_layer(rl->s, - SSL_CONNECTION_IS_DTLS(rl->s) - ? DTLS_ANY_VERSION : TLS_ANY_VERSION, - OSSL_RECORD_DIRECTION_READ, - OSSL_RECORD_PROTECTION_LEVEL_NONE, NULL, 0, - NULL, 0, NULL, 0, NULL, 0, NULL, 0, - NID_undef, NULL, NULL, NULL); - - ret &= ssl_set_new_record_layer(rl->s, - SSL_CONNECTION_IS_DTLS(rl->s) - ? DTLS_ANY_VERSION : TLS_ANY_VERSION, - OSSL_RECORD_DIRECTION_WRITE, - OSSL_RECORD_PROTECTION_LEVEL_NONE, NULL, 0, - NULL, 0, NULL, 0, NULL, 0, NULL, 0, - NID_undef, NULL, NULL, NULL); - - /* SSLfatal already called in the event of failure */ - return ret; + if (SSL3_BUFFER_is_initialised(&rl->rbuf)) + ssl3_release_read_buffer(rl->s); + if (rl->numwpipes > 0) + ssl3_release_write_buffer(rl->s); + SSL3_RECORD_release(rl->rrec, SSL_MAX_PIPELINES); } /* Checks if we have unprocessed read ahead data pending */ int RECORD_LAYER_read_pending(const RECORD_LAYER *rl) { - return rl->rrlmethod->unprocessed_read_pending(rl->rrl); + return SSL3_BUFFER_get_left(&rl->rbuf) != 0; } /* Checks if we have decrypted unread record data pending */ int RECORD_LAYER_processed_read_pending(const RECORD_LAYER *rl) { - return (rl->curr_rec < rl->num_recs) - || rl->rrlmethod->processed_read_pending(rl->rrl); + size_t curr_rec = 0, num_recs = RECORD_LAYER_get_numrpipes(rl); + const SSL3_RECORD *rr = rl->rrec; + + while (curr_rec < num_recs && SSL3_RECORD_is_read(&rr[curr_rec])) + curr_rec++; + + return curr_rec < num_recs; } int RECORD_LAYER_write_pending(const RECORD_LAYER *rl) { - return rl->wpend_tot > 0; + return (rl->numwpipes > 0) + && SSL3_BUFFER_get_left(&rl->wbuf[rl->numwpipes - 1]) != 0; } -static uint32_t ossl_get_max_early_data(SSL_CONNECTION *s) +void RECORD_LAYER_reset_read_sequence(RECORD_LAYER *rl) { - uint32_t max_early_data; - SSL_SESSION *sess = s->session; - - /* - * If we are a client then we always use the max_early_data from the - * session/psksession. Otherwise we go with the lowest out of the max early - * data set in the session and the configured max_early_data. - */ - if (!s->server && sess->ext.max_early_data == 0) { - if (!ossl_assert(s->psksession != NULL - && s->psksession->ext.max_early_data > 0)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - sess = s->psksession; - } - - if (!s->server) - max_early_data = sess->ext.max_early_data; - else if (s->ext.early_data != SSL_EARLY_DATA_ACCEPTED) - max_early_data = s->recv_max_early_data; - else - max_early_data = s->recv_max_early_data < sess->ext.max_early_data - ? s->recv_max_early_data : sess->ext.max_early_data; - - return max_early_data; + memset(rl->read_sequence, 0, sizeof(rl->read_sequence)); } -static int ossl_early_data_count_ok(SSL_CONNECTION *s, size_t length, - size_t overhead, int send) +void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl) { - uint32_t max_early_data; - - max_early_data = ossl_get_max_early_data(s); - - if (max_early_data == 0) { - SSLfatal(s, send ? SSL_AD_INTERNAL_ERROR : SSL_AD_UNEXPECTED_MESSAGE, - SSL_R_TOO_MUCH_EARLY_DATA); - return 0; - } - - /* If we are dealing with ciphertext we need to allow for the overhead */ - max_early_data += overhead; - - if (s->early_data_count + length > max_early_data) { - SSLfatal(s, send ? SSL_AD_INTERNAL_ERROR : SSL_AD_UNEXPECTED_MESSAGE, - SSL_R_TOO_MUCH_EARLY_DATA); - return 0; - } - s->early_data_count += length; - - return 1; + memset(rl->write_sequence, 0, sizeof(rl->write_sequence)); } size_t ssl3_pending(const SSL *s) { size_t i, num = 0; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - if (sc == NULL) + if (s->rlayer.rstate == SSL_ST_READ_BODY) return 0; - if (SSL_CONNECTION_IS_DTLS(sc)) { - TLS_RECORD *rdata; - pitem *item, *iter; - - iter = pqueue_iterator(sc->rlayer.d->buffered_app_data); - while ((item = pqueue_next(&iter)) != NULL) { - rdata = item->data; - num += rdata->length; - } - } - - for (i = 0; i < sc->rlayer.num_recs; i++) { - if (sc->rlayer.tlsrecs[i].type != SSL3_RT_APPLICATION_DATA) - return num; - num += sc->rlayer.tlsrecs[i].length; + for (i = 0; i < RECORD_LAYER_get_numrpipes(&s->rlayer); i++) { + if (SSL3_RECORD_get_type(&s->rlayer.rrec[i]) + != SSL3_RT_APPLICATION_DATA) + return 0; + num += SSL3_RECORD_get_length(&s->rlayer.rrec[i]); } - num += sc->rlayer.rrlmethod->app_data_pending(sc->rlayer.rrl); - return num; } @@ -204,59 +132,213 @@ void SSL_CTX_set_default_read_buffer_len(SSL_CTX *ctx, size_t len) void SSL_set_default_read_buffer_len(SSL *s, size_t len) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL || IS_QUIC(s)) - return; - sc->rlayer.default_read_buf_len = len; + SSL3_BUFFER_set_default_len(RECORD_LAYER_get_rbuf(&s->rlayer), len); } const char *SSL_rstate_string_long(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - const char *lng; - - if (sc == NULL) - return NULL; - - if (sc->rlayer.rrlmethod == NULL || sc->rlayer.rrl == NULL) + switch (s->rlayer.rstate) { + case SSL_ST_READ_HEADER: + return "read header"; + case SSL_ST_READ_BODY: + return "read body"; + case SSL_ST_READ_DONE: + return "read done"; + default: return "unknown"; - - sc->rlayer.rrlmethod->get_state(sc->rlayer.rrl, NULL, &lng); - - return lng; + } } const char *SSL_rstate_string(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - const char *shrt; - - if (sc == NULL) - return NULL; - - if (sc->rlayer.rrlmethod == NULL || sc->rlayer.rrl == NULL) + switch (s->rlayer.rstate) { + case SSL_ST_READ_HEADER: + return "RH"; + case SSL_ST_READ_BODY: + return "RB"; + case SSL_ST_READ_DONE: + return "RD"; + default: return "unknown"; - - sc->rlayer.rrlmethod->get_state(sc->rlayer.rrl, &shrt, NULL); - - return shrt; + } } -static int tls_write_check_pending(SSL_CONNECTION *s, uint8_t type, - const unsigned char *buf, size_t len) +/* + * Return values are as per SSL_read() + */ +int ssl3_read_n(SSL *s, size_t n, size_t max, int extend, int clearold, + size_t *readbytes) { - if (s->rlayer.wpend_tot == 0) + /* + * If extend == 0, obtain new n-byte packet; if extend == 1, increase + * packet by another n bytes. The packet will be in the sub-array of + * s->rlayer.rbuf.buf specified by s->rlayer.packet and + * s->rlayer.packet_length. (If s->rlayer.read_ahead is set, 'max' bytes may + * be stored in rbuf [plus s->rlayer.packet_length bytes if extend == 1].) + * if clearold == 1, move the packet to the start of the buffer; if + * clearold == 0 then leave any old packets where they were + */ + size_t len, left, align = 0; + unsigned char *pkt; + SSL3_BUFFER *rb; + + if (n == 0) return 0; - /* We have pending data, so do some sanity checks */ - if ((s->rlayer.wpend_tot > len) - || (!(s->mode & SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) - && (s->rlayer.wpend_buf != buf)) - || (s->rlayer.wpend_type != type)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_WRITE_RETRY); + rb = &s->rlayer.rbuf; + if (rb->buf == NULL) + if (!ssl3_setup_read_buffer(s)) { + /* SSLfatal() already called */ + return -1; + } + + left = rb->left; +#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0 + align = (size_t)rb->buf + SSL3_RT_HEADER_LENGTH; + align = SSL3_ALIGN_PAYLOAD - 1 - ((align - 1) % SSL3_ALIGN_PAYLOAD); +#endif + + if (!extend) { + /* start with empty packet ... */ + if (left == 0) + rb->offset = align; + else if (align != 0 && left >= SSL3_RT_HEADER_LENGTH) { + /* + * check if next packet length is large enough to justify payload + * alignment... + */ + pkt = rb->buf + rb->offset; + if (pkt[0] == SSL3_RT_APPLICATION_DATA + && (pkt[3] << 8 | pkt[4]) >= 128) { + /* + * Note that even if packet is corrupted and its length field + * is insane, we can only be led to wrong decision about + * whether memmove will occur or not. Header values has no + * effect on memmove arguments and therefore no buffer + * overrun can be triggered. + */ + memmove(rb->buf + align, pkt, left); + rb->offset = align; + } + } + s->rlayer.packet = rb->buf + rb->offset; + s->rlayer.packet_length = 0; + /* ... now we can act as if 'extend' was set */ + } + + len = s->rlayer.packet_length; + pkt = rb->buf + align; + /* + * Move any available bytes to front of buffer: 'len' bytes already + * pointed to by 'packet', 'left' extra ones at the end + */ + if (s->rlayer.packet != pkt && clearold == 1) { + memmove(pkt, s->rlayer.packet, len + left); + s->rlayer.packet = pkt; + rb->offset = len + align; + } + + /* + * For DTLS/UDP reads should not span multiple packets because the read + * operation returns the whole packet at once (as long as it fits into + * the buffer). + */ + if (SSL_IS_DTLS(s)) { + if (left == 0 && extend) + return 0; + if (left > 0 && n > left) + n = left; + } + + /* if there is enough in the buffer from a previous read, take some */ + if (left >= n) { + s->rlayer.packet_length += n; + rb->left = left - n; + rb->offset += n; + *readbytes = n; + return 1; + } + + /* else we need to read more data */ + + if (n > rb->len - rb->offset) { + /* does not happen */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return -1; } + + /* + * Ktls always reads full records. + * Also, we always act like read_ahead is set for DTLS. + */ + if (!BIO_get_ktls_recv(s->rbio) && !s->rlayer.read_ahead + && !SSL_IS_DTLS(s)) { + /* ignore max parameter */ + max = n; + } else { + if (max < n) + max = n; + if (max > rb->len - rb->offset) + max = rb->len - rb->offset; + } + + while (left < n) { + size_t bioread = 0; + int ret; + + /* + * Now we have len+left bytes at the front of s->s3.rbuf.buf and + * need to read in more until we have len+n (up to len+max if + * possible) + */ + + clear_sys_error(); + if (s->rbio != NULL) { + s->rwstate = SSL_READING; + ret = BIO_read(s->rbio, pkt + len + left, max - left); + if (ret >= 0) + bioread = ret; + if (ret <= 0 + && !BIO_should_retry(s->rbio) + && BIO_eof(s->rbio)) { + if (s->options & SSL_OP_IGNORE_UNEXPECTED_EOF) { + SSL_set_shutdown(s, SSL_RECEIVED_SHUTDOWN); + s->s3.warn_alert = SSL_AD_CLOSE_NOTIFY; + } else { + SSLfatal(s, SSL_AD_DECODE_ERROR, + SSL_R_UNEXPECTED_EOF_WHILE_READING); + } + } + } else { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_READ_BIO_NOT_SET); + ret = -1; + } + + if (ret <= 0) { + rb->left = left; + if (s->mode & SSL_MODE_RELEASE_BUFFERS && !SSL_IS_DTLS(s)) + if (len + left == 0) + ssl3_release_read_buffer(s); + return ret; + } + left += bioread; + /* + * reads should *never* span multiple packets for DTLS because the + * underlying transport protocol is message oriented as opposed to + * byte oriented as in the TLS case. + */ + if (SSL_IS_DTLS(s)) { + if (n > left) + n = left; /* makes the while condition false */ + } + } + + /* done reading, now the book-keeping */ + rb->offset += n; + rb->left = left - n; + s->rlayer.packet_length += n; + s->rwstate = SSL_NOTHING; + *readbytes = n; return 1; } @@ -264,19 +346,18 @@ static int tls_write_check_pending(SSL_CONNECTION *s, uint8_t type, * Call this to write data in records of type 'type' It will return <= 0 if * not all data has been sent or non-blocking IO. */ -int ssl3_write_bytes(SSL *ssl, uint8_t type, const void *buf_, size_t len, +int ssl3_write_bytes(SSL *s, int type, const void *buf_, size_t len, size_t *written) { const unsigned char *buf = buf_; size_t tot; size_t n, max_send_fragment, split_send_fragment, maxpipes; +#if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK + size_t nw; +#endif + SSL3_BUFFER *wb = &s->rlayer.wbuf[0]; int i; - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - OSSL_RECORD_TEMPLATE tmpls[SSL_MAX_PIPELINES]; - unsigned int recversion; - - if (s == NULL) - return -1; + size_t tmpwrit; s->rwstate = SSL_NOTHING; tot = s->rlayer.wnum; @@ -284,20 +365,19 @@ int ssl3_write_bytes(SSL *ssl, uint8_t type, const void *buf_, size_t len, * ensure that if we end up with a smaller value of data to write out * than the original len from a write which didn't complete for * non-blocking I/O and also somehow ended up avoiding the check for - * this in tls_write_check_pending/SSL_R_BAD_WRITE_RETRY as it must never be + * this in ssl3_write_pending/SSL_R_BAD_WRITE_RETRY as it must never be * possible to end up with (len-tot) as a large number that will then * promptly send beyond the end of the users buffer ... so we trap and * report the error in a way the user will notice */ if ((len < s->rlayer.wnum) - || ((s->rlayer.wpend_tot != 0) - && (len < (s->rlayer.wnum + s->rlayer.wpend_tot)))) { + || ((wb->left != 0) && (len < (s->rlayer.wnum + s->rlayer.wpend_tot)))) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_LENGTH); return -1; } if (s->early_data_state == SSL_EARLY_DATA_WRITING - && !ossl_early_data_count_ok(s, len, 0, 1)) { + && !early_data_count_ok(s, len, 0, 1)) { /* SSLfatal() already called */ return -1; } @@ -309,8 +389,8 @@ int ssl3_write_bytes(SSL *ssl, uint8_t type, const void *buf_, size_t len, * into init unless we have writes pending - in which case we should finish * doing that first. */ - if (s->rlayer.wpend_tot == 0 && (s->key_update != SSL_KEY_UPDATE_NONE - || s->ext.extra_tickets_expected > 0)) + if (wb->left == 0 && (s->key_update != SSL_KEY_UPDATE_NONE + || s->ext.extra_tickets_expected > 0)) ossl_statem_set_in_init(s, 1); /* @@ -318,9 +398,9 @@ int ssl3_write_bytes(SSL *ssl, uint8_t type, const void *buf_, size_t len, * between receiving the EoED and the CF - but we don't want to handle those * messages yet. */ - if (SSL_in_init(ssl) && !ossl_statem_get_in_handshake(s) + if (SSL_in_init(s) && !ossl_statem_get_in_handshake(s) && s->early_data_state != SSL_EARLY_DATA_UNAUTH_WRITING) { - i = s->handshake_func(ssl); + i = s->handshake_func(s); /* SSLfatal() already called */ if (i < 0) return i; @@ -329,53 +409,188 @@ int ssl3_write_bytes(SSL *ssl, uint8_t type, const void *buf_, size_t len, } } - i = tls_write_check_pending(s, type, buf, len); - if (i < 0) { - /* SSLfatal() already called */ - return i; - } else if (i > 0) { - /* Retry needed */ - i = HANDLE_RLAYER_WRITE_RETURN(s, - s->rlayer.wrlmethod->retry_write_records(s->rlayer.wrl)); + /* + * first check if there is a SSL3_BUFFER still being written out. This + * will happen with non blocking IO + */ + if (wb->left != 0) { + /* SSLfatal() already called if appropriate */ + i = ssl3_write_pending(s, type, &buf[tot], s->rlayer.wpend_tot, + &tmpwrit); if (i <= 0) { + /* XXX should we ssl3_release_write_buffer if i<0? */ s->rlayer.wnum = tot; return i; } - tot += s->rlayer.wpend_tot; - s->rlayer.wpend_tot = 0; - } /* else no retry required */ - - if (tot == 0) { - /* - * We've not previously sent any data for this write so memorize - * arguments so that we can detect bad write retries later - */ - s->rlayer.wpend_tot = 0; - s->rlayer.wpend_type = type; - s->rlayer.wpend_buf = buf; + tot += tmpwrit; /* this might be last fragment */ } +#if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK + /* + * Depending on platform multi-block can deliver several *times* + * better performance. Downside is that it has to allocate + * jumbo buffer to accommodate up to 8 records, but the + * compromise is considered worthy. + */ + if (type == SSL3_RT_APPLICATION_DATA + && len >= 4 * (max_send_fragment = ssl_get_max_send_fragment(s)) + && s->compress == NULL + && s->msg_callback == NULL + && !SSL_WRITE_ETM(s) + && SSL_USE_EXPLICIT_IV(s) + && BIO_get_ktls_send(s->wbio) == 0 + && (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(s->enc_write_ctx)) + & EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK) != 0) { + unsigned char aad[13]; + EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM mb_param; + size_t packlen; + int packleni; + + /* minimize address aliasing conflicts */ + if ((max_send_fragment & 0xfff) == 0) + max_send_fragment -= 512; + + if (tot == 0 || wb->buf == NULL) { /* allocate jumbo buffer */ + ssl3_release_write_buffer(s); + + packlen = EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, + EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE, + (int)max_send_fragment, NULL); + + if (len >= 8 * max_send_fragment) + packlen *= 8; + else + packlen *= 4; + + if (!ssl3_setup_write_buffer(s, 1, packlen)) { + /* SSLfatal() already called */ + return -1; + } + } else if (tot == len) { /* done? */ + /* free jumbo buffer */ + ssl3_release_write_buffer(s); + *written = tot; + return 1; + } + + n = (len - tot); + for (;;) { + if (n < 4 * max_send_fragment) { + /* free jumbo buffer */ + ssl3_release_write_buffer(s); + break; + } + + if (s->s3.alert_dispatch) { + i = s->method->ssl_dispatch_alert(s); + if (i <= 0) { + /* SSLfatal() already called if appropriate */ + s->rlayer.wnum = tot; + return i; + } + } + + if (n >= 8 * max_send_fragment) + nw = max_send_fragment * (mb_param.interleave = 8); + else + nw = max_send_fragment * (mb_param.interleave = 4); + + memcpy(aad, s->rlayer.write_sequence, 8); + aad[8] = type; + aad[9] = (unsigned char)(s->version >> 8); + aad[10] = (unsigned char)(s->version); + aad[11] = 0; + aad[12] = 0; + mb_param.out = NULL; + mb_param.inp = aad; + mb_param.len = nw; + + packleni = EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, + EVP_CTRL_TLS1_1_MULTIBLOCK_AAD, + sizeof(mb_param), &mb_param); + packlen = (size_t)packleni; + if (packleni <= 0 || packlen > wb->len) { /* never happens */ + /* free jumbo buffer */ + ssl3_release_write_buffer(s); + break; + } + mb_param.out = wb->buf; + mb_param.inp = &buf[tot]; + mb_param.len = nw; + + if (EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, + EVP_CTRL_TLS1_1_MULTIBLOCK_ENCRYPT, + sizeof(mb_param), &mb_param) <= 0) + return -1; + + s->rlayer.write_sequence[7] += mb_param.interleave; + if (s->rlayer.write_sequence[7] < mb_param.interleave) { + int j = 6; + while (j >= 0 && (++s->rlayer.write_sequence[j--]) == 0) ; + } + + wb->offset = 0; + wb->left = packlen; + + s->rlayer.wpend_tot = nw; + s->rlayer.wpend_buf = &buf[tot]; + s->rlayer.wpend_type = type; + s->rlayer.wpend_ret = nw; + + i = ssl3_write_pending(s, type, &buf[tot], nw, &tmpwrit); + if (i <= 0) { + /* SSLfatal() already called if appropriate */ + if (i < 0 && (!s->wbio || !BIO_should_retry(s->wbio))) { + /* free jumbo buffer */ + ssl3_release_write_buffer(s); + } + s->rlayer.wnum = tot; + return i; + } + if (tmpwrit == n) { + /* free jumbo buffer */ + ssl3_release_write_buffer(s); + *written = tot + tmpwrit; + return 1; + } + n -= tmpwrit; + tot += tmpwrit; + } + } else +#endif /* !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK */ if (tot == len) { /* done? */ + if (s->mode & SSL_MODE_RELEASE_BUFFERS && !SSL_IS_DTLS(s)) + ssl3_release_write_buffer(s); + *written = tot; return 1; } - /* If we have an alert to send, lets send it */ - if (s->s3.alert_dispatch > 0) { - i = ssl->method->ssl_dispatch_alert(ssl); - if (i <= 0) { - /* SSLfatal() already called if appropriate */ - s->rlayer.wnum = tot; - return i; - } - /* if it went, fall through and send more stuff */ - } - n = (len - tot); max_send_fragment = ssl_get_max_send_fragment(s); split_send_fragment = ssl_get_split_send_fragment(s); - + /* + * If max_pipelines is 0 then this means "undefined" and we default to + * 1 pipeline. Similarly if the cipher does not support pipelined + * processing then we also only use 1 pipeline, or if we're not using + * explicit IVs + */ + maxpipes = s->max_pipelines; + if (maxpipes > SSL_MAX_PIPELINES) { + /* + * We should have prevented this when we set max_pipelines so we + * shouldn't get here + */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + if (maxpipes == 0 + || s->enc_write_ctx == NULL + || (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(s->enc_write_ctx)) + & EVP_CIPH_FLAG_PIPELINE) == 0 + || !SSL_USE_EXPLICIT_IV(s)) + maxpipes = 1; if (max_send_fragment == 0 || split_send_fragment == 0 || split_send_fragment > max_send_fragment) { @@ -387,203 +602,642 @@ int ssl3_write_bytes(SSL *ssl, uint8_t type, const void *buf_, size_t len, return -1; } - /* - * Some servers hang if initial client hello is larger than 256 bytes - * and record version number > TLS 1.0 - */ - recversion = (s->version == TLS1_3_VERSION) ? TLS1_2_VERSION : s->version; - if (SSL_get_state(ssl) == TLS_ST_CW_CLNT_HELLO - && !s->renegotiate - && TLS1_get_version(ssl) > TLS1_VERSION - && s->hello_retry_request == SSL_HRR_NONE) - recversion = TLS1_VERSION; - for (;;) { - size_t tmppipelen, remain; - size_t j, lensofar = 0; + size_t pipelens[SSL_MAX_PIPELINES], tmppipelen, remain; + size_t numpipes, j; - /* - * Ask the record layer how it would like to split the amount of data - * that we have, and how many of those records it would like in one go. - */ - maxpipes = s->rlayer.wrlmethod->get_max_records(s->rlayer.wrl, type, n, - max_send_fragment, - &split_send_fragment); - /* - * If max_pipelines is 0 then this means "undefined" and we default to - * whatever the record layer wants to do. Otherwise we use the smallest - * value from the number requested by the record layer, and max number - * configured by the user. - */ - if (s->max_pipelines > 0 && maxpipes > s->max_pipelines) - maxpipes = s->max_pipelines; - - if (maxpipes > SSL_MAX_PIPELINES) - maxpipes = SSL_MAX_PIPELINES; - - if (split_send_fragment > max_send_fragment) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return -1; - } + if (n == 0) + numpipes = 1; + else + numpipes = ((n - 1) / split_send_fragment) + 1; + if (numpipes > maxpipes) + numpipes = maxpipes; - if (n / maxpipes >= split_send_fragment) { + if (n / numpipes >= max_send_fragment) { /* * We have enough data to completely fill all available * pipelines */ - for (j = 0; j < maxpipes; j++) { - tmpls[j].type = type; - tmpls[j].version = recversion; - tmpls[j].buf = &(buf[tot]) + (j * split_send_fragment); - tmpls[j].buflen = split_send_fragment; + for (j = 0; j < numpipes; j++) { + pipelens[j] = max_send_fragment; } - /* Remember how much data we are going to be sending */ - s->rlayer.wpend_tot = maxpipes * split_send_fragment; } else { /* We can partially fill all available pipelines */ - tmppipelen = n / maxpipes; - remain = n % maxpipes; - /* - * If there is a remainder we add an extra byte to the first few - * pipelines - */ - if (remain > 0) - tmppipelen++; - for (j = 0; j < maxpipes; j++) { - tmpls[j].type = type; - tmpls[j].version = recversion; - tmpls[j].buf = &(buf[tot]) + lensofar; - tmpls[j].buflen = tmppipelen; - lensofar += tmppipelen; - if (j + 1 == remain) - tmppipelen--; + tmppipelen = n / numpipes; + remain = n % numpipes; + for (j = 0; j < numpipes; j++) { + pipelens[j] = tmppipelen; + if (j < remain) + pipelens[j]++; } - /* Remember how much data we are going to be sending */ - s->rlayer.wpend_tot = n; } - i = HANDLE_RLAYER_WRITE_RETURN(s, - s->rlayer.wrlmethod->write_records(s->rlayer.wrl, tmpls, maxpipes)); + i = do_ssl3_write(s, type, &(buf[tot]), pipelens, numpipes, 0, + &tmpwrit); if (i <= 0) { /* SSLfatal() already called if appropriate */ + /* XXX should we ssl3_release_write_buffer if i<0? */ s->rlayer.wnum = tot; return i; } - if (s->rlayer.wpend_tot == n - || (type == SSL3_RT_APPLICATION_DATA - && (s->mode & SSL_MODE_ENABLE_PARTIAL_WRITE) != 0)) { - *written = tot + s->rlayer.wpend_tot; - s->rlayer.wpend_tot = 0; + if (tmpwrit == n || + (type == SSL3_RT_APPLICATION_DATA && + (s->mode & SSL_MODE_ENABLE_PARTIAL_WRITE))) { + /* + * next chunk of data should get another prepended empty fragment + * in ciphersuites with known-IV weakness: + */ + s->s3.empty_fragment_done = 0; + + if (tmpwrit == n + && (s->mode & SSL_MODE_RELEASE_BUFFERS) != 0 + && !SSL_IS_DTLS(s)) + ssl3_release_write_buffer(s); + + *written = tot + tmpwrit; return 1; } - n -= s->rlayer.wpend_tot; - tot += s->rlayer.wpend_tot; + n -= tmpwrit; + tot += tmpwrit; } } -int ossl_tls_handle_rlayer_return(SSL_CONNECTION *s, int writing, int ret, - char *file, int line) +int do_ssl3_write(SSL *s, int type, const unsigned char *buf, + size_t *pipelens, size_t numpipes, + int create_empty_fragment, size_t *written) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); + WPACKET pkt[SSL_MAX_PIPELINES]; + SSL3_RECORD wr[SSL_MAX_PIPELINES]; + WPACKET *thispkt; + SSL3_RECORD *thiswr; + unsigned char *recordstart; + int i, mac_size, clear = 0; + size_t prefix_len = 0; + int eivlen = 0; + size_t align = 0; + SSL3_BUFFER *wb; + SSL_SESSION *sess; + size_t totlen = 0, len, wpinited = 0; + size_t j; + + for (j = 0; j < numpipes; j++) + totlen += pipelens[j]; + /* + * first check if there is a SSL3_BUFFER still being written out. This + * will happen with non blocking IO + */ + if (RECORD_LAYER_write_pending(&s->rlayer)) { + /* Calls SSLfatal() as required */ + return ssl3_write_pending(s, type, buf, totlen, written); + } + + /* If we have an alert to send, lets send it */ + if (s->s3.alert_dispatch) { + i = s->method->ssl_dispatch_alert(s); + if (i <= 0) { + /* SSLfatal() already called if appropriate */ + return i; + } + /* if it went, fall through and send more stuff */ + } + + if (s->rlayer.numwpipes < numpipes) { + if (!ssl3_setup_write_buffer(s, numpipes, 0)) { + /* SSLfatal() already called */ + return -1; + } + } + + if (totlen == 0 && !create_empty_fragment) + return 0; + + sess = s->session; - if (ret == OSSL_RECORD_RETURN_RETRY) { - s->rwstate = writing ? SSL_WRITING : SSL_READING; - ret = -1; + if ((sess == NULL) + || (s->enc_write_ctx == NULL) + || (EVP_MD_CTX_get0_md(s->write_hash) == NULL)) { + clear = s->enc_write_ctx ? 0 : 1; /* must be AEAD cipher */ + mac_size = 0; } else { - s->rwstate = SSL_NOTHING; - if (ret == OSSL_RECORD_RETURN_EOF) { - if (writing) { - /* - * This shouldn't happen with a writing operation. We treat it - * as fatal. - */ - ERR_new(); - ERR_set_debug(file, line, 0); - ossl_statem_fatal(s, SSL_AD_INTERNAL_ERROR, - ERR_R_INTERNAL_ERROR, NULL); - ret = OSSL_RECORD_RETURN_FATAL; - } else if ((s->options & SSL_OP_IGNORE_UNEXPECTED_EOF) != 0) { - SSL_set_shutdown(ssl, SSL_RECEIVED_SHUTDOWN); - s->s3.warn_alert = SSL_AD_CLOSE_NOTIFY; + mac_size = EVP_MD_CTX_get_size(s->write_hash); + if (mac_size < 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } + + /* + * 'create_empty_fragment' is true only when this function calls itself + */ + if (!clear && !create_empty_fragment && !s->s3.empty_fragment_done) { + /* + * countermeasure against known-IV weakness in CBC ciphersuites (see + * http://www.openssl.org/~bodo/tls-cbc.txt) + */ + + if (s->s3.need_empty_fragments && type == SSL3_RT_APPLICATION_DATA) { + /* + * recursive function call with 'create_empty_fragment' set; this + * prepares and buffers the data for an empty fragment (these + * 'prefix_len' bytes are sent out later together with the actual + * payload) + */ + size_t tmppipelen = 0; + int ret; + + ret = do_ssl3_write(s, type, buf, &tmppipelen, 1, 1, &prefix_len); + if (ret <= 0) { + /* SSLfatal() already called if appropriate */ + goto err; + } + + if (prefix_len > + (SSL3_RT_HEADER_LENGTH + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD)) { + /* insufficient space */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } + + s->s3.empty_fragment_done = 1; + } + + if (BIO_get_ktls_send(s->wbio)) { + /* + * ktls doesn't modify the buffer, but to avoid a warning we need to + * discard the const qualifier. + * This doesn't leak memory because the buffers have been released when + * switching to ktls. + */ + SSL3_BUFFER_set_buf(&s->rlayer.wbuf[0], (unsigned char *)buf); + SSL3_BUFFER_set_offset(&s->rlayer.wbuf[0], 0); + SSL3_BUFFER_set_app_buffer(&s->rlayer.wbuf[0], 1); + goto wpacket_init_complete; + } + + if (create_empty_fragment) { + wb = &s->rlayer.wbuf[0]; +#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0 + /* + * extra fragment would be couple of cipher blocks, which would be + * multiple of SSL3_ALIGN_PAYLOAD, so if we want to align the real + * payload, then we can just pretend we simply have two headers. + */ + align = (size_t)SSL3_BUFFER_get_buf(wb) + 2 * SSL3_RT_HEADER_LENGTH; + align = SSL3_ALIGN_PAYLOAD - 1 - ((align - 1) % SSL3_ALIGN_PAYLOAD); +#endif + SSL3_BUFFER_set_offset(wb, align); + if (!WPACKET_init_static_len(&pkt[0], SSL3_BUFFER_get_buf(wb), + SSL3_BUFFER_get_len(wb), 0) + || !WPACKET_allocate_bytes(&pkt[0], align, NULL)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + wpinited = 1; + } else if (prefix_len) { + wb = &s->rlayer.wbuf[0]; + if (!WPACKET_init_static_len(&pkt[0], + SSL3_BUFFER_get_buf(wb), + SSL3_BUFFER_get_len(wb), 0) + || !WPACKET_allocate_bytes(&pkt[0], SSL3_BUFFER_get_offset(wb) + + prefix_len, NULL)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + wpinited = 1; + } else { + for (j = 0; j < numpipes; j++) { + thispkt = &pkt[j]; + + wb = &s->rlayer.wbuf[j]; +#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD != 0 + align = (size_t)SSL3_BUFFER_get_buf(wb) + SSL3_RT_HEADER_LENGTH; + align = SSL3_ALIGN_PAYLOAD - 1 - ((align - 1) % SSL3_ALIGN_PAYLOAD); +#endif + SSL3_BUFFER_set_offset(wb, align); + if (!WPACKET_init_static_len(thispkt, SSL3_BUFFER_get_buf(wb), + SSL3_BUFFER_get_len(wb), 0) + || !WPACKET_allocate_bytes(thispkt, align, NULL)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + wpinited++; + } + } + + /* Explicit IV length, block ciphers appropriate version flag */ + if (s->enc_write_ctx && SSL_USE_EXPLICIT_IV(s) && !SSL_TREAT_AS_TLS13(s)) { + int mode = EVP_CIPHER_CTX_get_mode(s->enc_write_ctx); + if (mode == EVP_CIPH_CBC_MODE) { + eivlen = EVP_CIPHER_CTX_get_iv_length(s->enc_write_ctx); + if (eivlen <= 1) + eivlen = 0; + } else if (mode == EVP_CIPH_GCM_MODE) { + /* Need explicit part of IV for GCM mode */ + eivlen = EVP_GCM_TLS_EXPLICIT_IV_LEN; + } else if (mode == EVP_CIPH_CCM_MODE) { + eivlen = EVP_CCM_TLS_EXPLICIT_IV_LEN; + } + } + + wpacket_init_complete: + + totlen = 0; + /* Clear our SSL3_RECORD structures */ + memset(wr, 0, sizeof(wr)); + for (j = 0; j < numpipes; j++) { + unsigned int version = (s->version == TLS1_3_VERSION) ? TLS1_2_VERSION + : s->version; + unsigned char *compressdata = NULL; + size_t maxcomplen; + unsigned int rectype; + + thispkt = &pkt[j]; + thiswr = &wr[j]; + + /* + * In TLSv1.3, once encrypting, we always use application data for the + * record type + */ + if (SSL_TREAT_AS_TLS13(s) + && s->enc_write_ctx != NULL + && (s->statem.enc_write_state != ENC_WRITE_STATE_WRITE_PLAIN_ALERTS + || type != SSL3_RT_ALERT)) + rectype = SSL3_RT_APPLICATION_DATA; + else + rectype = type; + SSL3_RECORD_set_type(thiswr, rectype); + + /* + * Some servers hang if initial client hello is larger than 256 bytes + * and record version number > TLS 1.0 + */ + if (SSL_get_state(s) == TLS_ST_CW_CLNT_HELLO + && !s->renegotiate + && TLS1_get_version(s) > TLS1_VERSION + && s->hello_retry_request == SSL_HRR_NONE) + version = TLS1_VERSION; + SSL3_RECORD_set_rec_version(thiswr, version); + + maxcomplen = pipelens[j]; + if (s->compress != NULL) + maxcomplen += SSL3_RT_MAX_COMPRESSED_OVERHEAD; + + /* + * When using offload kernel will write the header. + * Otherwise write the header now + */ + if (!BIO_get_ktls_send(s->wbio) + && (!WPACKET_put_bytes_u8(thispkt, rectype) + || !WPACKET_put_bytes_u16(thispkt, version) + || !WPACKET_start_sub_packet_u16(thispkt) + || (eivlen > 0 + && !WPACKET_allocate_bytes(thispkt, eivlen, NULL)) + || (maxcomplen > 0 + && !WPACKET_reserve_bytes(thispkt, maxcomplen, + &compressdata)))) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* lets setup the record stuff. */ + SSL3_RECORD_set_data(thiswr, compressdata); + SSL3_RECORD_set_length(thiswr, pipelens[j]); + SSL3_RECORD_set_input(thiswr, (unsigned char *)&buf[totlen]); + totlen += pipelens[j]; + + /* + * we now 'read' from thiswr->input, thiswr->length bytes into + * thiswr->data + */ + + /* first we compress */ + if (s->compress != NULL) { + if (!ssl3_do_compress(s, thiswr) + || !WPACKET_allocate_bytes(thispkt, thiswr->length, NULL)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_COMPRESSION_FAILURE); + goto err; + } + } else { + if (BIO_get_ktls_send(s->wbio)) { + SSL3_RECORD_reset_data(&wr[j]); } else { - ERR_new(); - ERR_set_debug(file, line, 0); + if (!WPACKET_memcpy(thispkt, thiswr->input, thiswr->length)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + SSL3_RECORD_reset_input(&wr[j]); + } + } + + if (SSL_TREAT_AS_TLS13(s) + && !BIO_get_ktls_send(s->wbio) + && s->enc_write_ctx != NULL + && (s->statem.enc_write_state != ENC_WRITE_STATE_WRITE_PLAIN_ALERTS + || type != SSL3_RT_ALERT)) { + size_t rlen, max_send_fragment; + + if (!WPACKET_put_bytes_u8(thispkt, type)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + SSL3_RECORD_add_length(thiswr, 1); + + /* Add TLS1.3 padding */ + max_send_fragment = ssl_get_max_send_fragment(s); + rlen = SSL3_RECORD_get_length(thiswr); + if (rlen < max_send_fragment) { + size_t padding = 0; + size_t max_padding = max_send_fragment - rlen; + if (s->record_padding_cb != NULL) { + padding = s->record_padding_cb(s, type, rlen, s->record_padding_arg); + } else if (s->block_padding > 0) { + size_t mask = s->block_padding - 1; + size_t remainder; + + /* optimize for power of 2 */ + if ((s->block_padding & mask) == 0) + remainder = rlen & mask; + else + remainder = rlen % s->block_padding; + /* don't want to add a block of padding if we don't have to */ + if (remainder == 0) + padding = 0; + else + padding = s->block_padding - remainder; + } + if (padding > 0) { + /* do not allow the record to exceed max plaintext length */ + if (padding > max_padding) + padding = max_padding; + if (!WPACKET_memset(thispkt, 0, padding)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, + ERR_R_INTERNAL_ERROR); + goto err; + } + SSL3_RECORD_add_length(thiswr, padding); + } + } + } + + /* + * we should still have the output to thiswr->data and the input from + * wr->input. Length should be thiswr->length. thiswr->data still points + * in the wb->buf + */ + + if (!BIO_get_ktls_send(s->wbio) && !SSL_WRITE_ETM(s) && mac_size != 0) { + unsigned char *mac; + + if (!WPACKET_allocate_bytes(thispkt, mac_size, &mac) + || !s->method->ssl3_enc->mac(s, thiswr, mac, 1)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } + + /* + * Reserve some bytes for any growth that may occur during encryption. + * This will be at most one cipher block or the tag length if using + * AEAD. SSL_RT_MAX_CIPHER_BLOCK_SIZE covers either case. + */ + if (!BIO_get_ktls_send(s->wbio)) { + if (!WPACKET_reserve_bytes(thispkt, + SSL_RT_MAX_CIPHER_BLOCK_SIZE, + NULL) /* - * This reason code is part of the API and may be used by - * applications for control flow decisions. + * We also need next the amount of bytes written to this + * sub-packet */ - ossl_statem_fatal(s, SSL_AD_DECODE_ERROR, - SSL_R_UNEXPECTED_EOF_WHILE_READING, NULL); + || !WPACKET_get_length(thispkt, &len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* Get a pointer to the start of this record excluding header */ + recordstart = WPACKET_get_curr(thispkt) - len; + SSL3_RECORD_set_data(thiswr, recordstart); + SSL3_RECORD_reset_input(thiswr); + SSL3_RECORD_set_length(thiswr, len); + } + } + + if (s->statem.enc_write_state == ENC_WRITE_STATE_WRITE_PLAIN_ALERTS) { + /* + * We haven't actually negotiated the version yet, but we're trying to + * send early data - so we need to use the tls13enc function. + */ + if (tls13_enc(s, wr, numpipes, 1, NULL, mac_size) < 1) { + if (!ossl_statem_in_error(s)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + } + goto err; + } + } else { + if (!BIO_get_ktls_send(s->wbio)) { + if (s->method->ssl3_enc->enc(s, wr, numpipes, 1, NULL, + mac_size) < 1) { + if (!ossl_statem_in_error(s)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + } + goto err; + } + } + } + + for (j = 0; j < numpipes; j++) { + size_t origlen; + + thispkt = &pkt[j]; + thiswr = &wr[j]; + + if (BIO_get_ktls_send(s->wbio)) + goto mac_done; + + /* Allocate bytes for the encryption overhead */ + if (!WPACKET_get_length(thispkt, &origlen) + /* Encryption should never shrink the data! */ + || origlen > thiswr->length + || (thiswr->length > origlen + && !WPACKET_allocate_bytes(thispkt, + thiswr->length - origlen, + NULL))) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + if (SSL_WRITE_ETM(s) && mac_size != 0) { + unsigned char *mac; + + if (!WPACKET_allocate_bytes(thispkt, mac_size, &mac) + || !s->method->ssl3_enc->mac(s, thiswr, mac, 1)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; } - } else if (ret == OSSL_RECORD_RETURN_FATAL) { - int al = s->rlayer.rrlmethod->get_alert_code(s->rlayer.rrl); + SSL3_RECORD_add_length(thiswr, mac_size); + } + + if (!WPACKET_get_length(thispkt, &len) + || !WPACKET_close(thispkt)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } - if (al != SSL_AD_NO_ALERT) { - ERR_new(); - ERR_set_debug(file, line, 0); - ossl_statem_fatal(s, al, SSL_R_RECORD_LAYER_FAILURE, NULL); + if (s->msg_callback) { + recordstart = WPACKET_get_curr(thispkt) - len + - SSL3_RT_HEADER_LENGTH; + s->msg_callback(1, thiswr->rec_version, SSL3_RT_HEADER, recordstart, + SSL3_RT_HEADER_LENGTH, s, + s->msg_callback_arg); + + if (SSL_TREAT_AS_TLS13(s) && s->enc_write_ctx != NULL) { + unsigned char ctype = type; + + s->msg_callback(1, thiswr->rec_version, SSL3_RT_INNER_CONTENT_TYPE, + &ctype, 1, s, s->msg_callback_arg); } + } + + if (!WPACKET_finish(thispkt)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* header is added by the kernel when using offload */ + SSL3_RECORD_add_length(&wr[j], SSL3_RT_HEADER_LENGTH); + + if (create_empty_fragment) { /* - * else some failure but there is no alert code. We don't log an - * error for this. The record layer should have logged an error - * already or, if not, its due to some sys call error which will be - * reported via SSL_ERROR_SYSCALL and errno. + * we are in a recursive call; just return the length, don't write + * out anything here */ + if (j > 0) { + /* We should never be pipelining an empty fragment!! */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + *written = SSL3_RECORD_get_length(thiswr); + return 1; } + + mac_done: /* - * The record layer distinguishes the cases of EOF, non-fatal - * err and retry. Upper layers do not. - * If we got a retry or success then *ret is already correct, - * otherwise we need to convert the return value. + * we should now have thiswr->data pointing to the encrypted data, which + * is thiswr->length long */ - if (ret == OSSL_RECORD_RETURN_NON_FATAL_ERR || ret == OSSL_RECORD_RETURN_EOF) - ret = 0; - else if (ret < OSSL_RECORD_RETURN_NON_FATAL_ERR) - ret = -1; + SSL3_RECORD_set_type(thiswr, type); /* not needed but helps for + * debugging */ + + /* now let's set up wb */ + SSL3_BUFFER_set_left(&s->rlayer.wbuf[j], + prefix_len + SSL3_RECORD_get_length(thiswr)); } - return ret; + /* + * memorize arguments so that ssl3_write_pending can detect bad write + * retries later + */ + s->rlayer.wpend_tot = totlen; + s->rlayer.wpend_buf = buf; + s->rlayer.wpend_type = type; + s->rlayer.wpend_ret = totlen; + + /* we now just need to write the buffer */ + return ssl3_write_pending(s, type, buf, totlen, written); + err: + for (j = 0; j < wpinited; j++) + WPACKET_cleanup(&pkt[j]); + return -1; } -int ssl_release_record(SSL_CONNECTION *s, TLS_RECORD *rr, size_t length) +/* if s->s3.wbuf.left != 0, we need to call this + * + * Return values are as per SSL_write() + */ +int ssl3_write_pending(SSL *s, int type, const unsigned char *buf, size_t len, + size_t *written) { - assert(rr->length >= length); - if (rr->rechandle != NULL) { - if (length == 0) - length = rr->length; - /* The record layer allocated the buffers for this record */ - if (HANDLE_RLAYER_READ_RETURN(s, - s->rlayer.rrlmethod->release_record(s->rlayer.rrl, - rr->rechandle, - length)) <= 0) { - /* RLAYER_fatal already called */ - return 0; - } + int i; + SSL3_BUFFER *wb = s->rlayer.wbuf; + size_t currbuf = 0; + size_t tmpwrit = 0; - if (length == rr->length) - s->rlayer.curr_rec++; - } else if (length == 0 || length == rr->length) { - /* We allocated the buffers for this record (only happens with DTLS) */ - OPENSSL_free(rr->allocdata); - rr->allocdata = NULL; + if ((s->rlayer.wpend_tot > len) + || (!(s->mode & SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) + && (s->rlayer.wpend_buf != buf)) + || (s->rlayer.wpend_type != type)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_WRITE_RETRY); + return -1; } - rr->length -= length; - if (rr->length > 0) - rr->off += length; - else - rr->off = 0; - return 1; + for (;;) { + /* Loop until we find a buffer we haven't written out yet */ + if (SSL3_BUFFER_get_left(&wb[currbuf]) == 0 + && currbuf < s->rlayer.numwpipes - 1) { + currbuf++; + continue; + } + clear_sys_error(); + if (s->wbio != NULL) { + s->rwstate = SSL_WRITING; + + /* + * To prevent coalescing of control and data messages, + * such as in buffer_write, we flush the BIO + */ + if (BIO_get_ktls_send(s->wbio) && type != SSL3_RT_APPLICATION_DATA) { + i = BIO_flush(s->wbio); + if (i <= 0) + return i; + BIO_set_ktls_ctrl_msg(s->wbio, type); + } + i = BIO_write(s->wbio, (char *) + &(SSL3_BUFFER_get_buf(&wb[currbuf]) + [SSL3_BUFFER_get_offset(&wb[currbuf])]), + (unsigned int)SSL3_BUFFER_get_left(&wb[currbuf])); + if (i >= 0) + tmpwrit = i; + } else { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BIO_NOT_SET); + i = -1; + } + + /* + * When an empty fragment is sent on a connection using KTLS, + * it is sent as a write of zero bytes. If this zero byte + * write succeeds, i will be 0 rather than a non-zero value. + * Treat i == 0 as success rather than an error for zero byte + * writes to permit this case. + */ + if (i >= 0 && tmpwrit == SSL3_BUFFER_get_left(&wb[currbuf])) { + SSL3_BUFFER_set_left(&wb[currbuf], 0); + SSL3_BUFFER_add_offset(&wb[currbuf], tmpwrit); + if (currbuf + 1 < s->rlayer.numwpipes) + continue; + s->rwstate = SSL_NOTHING; + *written = s->rlayer.wpend_ret; + return 1; + } else if (i <= 0) { + if (SSL_IS_DTLS(s)) { + /* + * For DTLS, just drop it. That's kind of the whole point in + * using a datagram service + */ + SSL3_BUFFER_set_left(&wb[currbuf], 0); + } + return i; + } + SSL3_BUFFER_add_offset(&wb[currbuf], tmpwrit); + SSL3_BUFFER_sub_left(&wb[currbuf], tmpwrit); + } } /*- * Return up to 'len' payload bytes received in 'type' records. * 'type' is one of the following: * - * - SSL3_RT_HANDSHAKE (when tls_get_message_header and tls_get_message_body - * call us) + * - SSL3_RT_HANDSHAKE (when ssl3_get_message calls us) * - SSL3_RT_APPLICATION_DATA (when ssl3_read calls us) * - 0 (during a shutdown, no data has to be returned) * @@ -608,23 +1262,30 @@ int ssl_release_record(SSL_CONNECTION *s, TLS_RECORD *rr, size_t length) * Application data protocol * none of our business */ -int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, - unsigned char *buf, size_t len, - int peek, size_t *readbytes) +int ssl3_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf, + size_t len, int peek, size_t *readbytes) { int i, j, ret; - size_t n, curr_rec, totalbytes; - TLS_RECORD *rr; + size_t n, curr_rec, num_recs, totalbytes; + SSL3_RECORD *rr; + SSL3_BUFFER *rbuf; void (*cb) (const SSL *ssl, int type2, int val) = NULL; - int is_tls13; - SSL_CONNECTION *s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); + int is_tls13 = SSL_IS_TLS13(s); + + rbuf = &s->rlayer.rbuf; - is_tls13 = SSL_CONNECTION_IS_TLS13(s); + if (!SSL3_BUFFER_is_initialised(rbuf)) { + /* Not initialized yet */ + if (!ssl3_setup_read_buffer(s)) { + /* SSLfatal() already called */ + return -1; + } + } - if ((type != 0 - && (type != SSL3_RT_APPLICATION_DATA) - && (type != SSL3_RT_HANDSHAKE)) - || (peek && (type != SSL3_RT_APPLICATION_DATA))) { + if ((type && (type != SSL3_RT_APPLICATION_DATA) + && (type != SSL3_RT_HANDSHAKE)) || (peek + && (type != + SSL3_RT_APPLICATION_DATA))) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return -1; } @@ -659,9 +1320,9 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * Now s->rlayer.handshake_fragment_len == 0 if type == SSL3_RT_HANDSHAKE. */ - if (!ossl_statem_get_in_handshake(s) && SSL_in_init(ssl)) { + if (!ossl_statem_get_in_handshake(s) && SSL_in_init(s)) { /* type == SSL3_RT_APPLICATION_DATA */ - i = s->handshake_func(ssl); + i = s->handshake_func(s); /* SSLfatal() already called */ if (i < 0) return i; @@ -678,32 +1339,39 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * rr[i].off, - offset into 'data' for next read * rr[i].length, - number of bytes. */ - /* get new records if necessary */ - if (s->rlayer.curr_rec >= s->rlayer.num_recs) { - s->rlayer.curr_rec = s->rlayer.num_recs = 0; - do { - rr = &s->rlayer.tlsrecs[s->rlayer.num_recs]; - - ret = HANDLE_RLAYER_READ_RETURN(s, - s->rlayer.rrlmethod->read_record(s->rlayer.rrl, - &rr->rechandle, - &rr->version, &rr->type, - &rr->data, &rr->length, - NULL, NULL)); + rr = s->rlayer.rrec; + num_recs = RECORD_LAYER_get_numrpipes(&s->rlayer); + + do { + /* get new records if necessary */ + if (num_recs == 0) { + ret = ssl3_get_record(s); if (ret <= 0) { /* SSLfatal() already called if appropriate */ return ret; } - rr->off = 0; - s->rlayer.num_recs++; - } while (s->rlayer.rrlmethod->processed_read_pending(s->rlayer.rrl) - && s->rlayer.num_recs < SSL_MAX_PIPELINES); - } - rr = &s->rlayer.tlsrecs[s->rlayer.curr_rec]; + num_recs = RECORD_LAYER_get_numrpipes(&s->rlayer); + if (num_recs == 0) { + /* Shouldn't happen */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + } + /* Skip over any records we have already read */ + for (curr_rec = 0; + curr_rec < num_recs && SSL3_RECORD_is_read(&rr[curr_rec]); + curr_rec++) ; + if (curr_rec == num_recs) { + RECORD_LAYER_set_numrpipes(&s->rlayer, 0); + num_recs = 0; + curr_rec = 0; + } + } while (num_recs == 0); + rr = &rr[curr_rec]; if (s->rlayer.handshake_fragment_len > 0 - && rr->type != SSL3_RT_HANDSHAKE - && SSL_CONNECTION_IS_TLS13(s)) { + && SSL3_RECORD_get_type(rr) != SSL3_RT_HANDSHAKE + && SSL_IS_TLS13(s)) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA); return -1; @@ -713,14 +1381,15 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * Reset the count of consecutive warning alerts if we've got a non-empty * record that isn't an alert. */ - if (rr->type != SSL3_RT_ALERT && rr->length != 0) + if (SSL3_RECORD_get_type(rr) != SSL3_RT_ALERT + && SSL3_RECORD_get_length(rr) != 0) s->rlayer.alert_count = 0; /* we now have a packet which can be read and processed */ if (s->s3.change_cipher_spec /* set when we receive ChangeCipherSpec, * reset by ssl3_get_finished */ - && (rr->type != SSL3_RT_HANDSHAKE)) { + && (SSL3_RECORD_get_type(rr) != SSL3_RT_HANDSHAKE)) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_DATA_BETWEEN_CCS_AND_FINISHED); return -1; @@ -731,13 +1400,13 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * 'peek' mode) */ if (s->shutdown & SSL_RECEIVED_SHUTDOWN) { - s->rlayer.curr_rec++; + SSL3_RECORD_set_length(rr, 0); s->rwstate = SSL_NOTHING; return 0; } - if (type == rr->type - || (rr->type == SSL3_RT_CHANGE_CIPHER_SPEC + if (type == SSL3_RECORD_get_type(rr) + || (SSL3_RECORD_get_type(rr) == SSL3_RT_CHANGE_CIPHER_SPEC && type == SSL3_RT_HANDSHAKE && recvd_type != NULL && !is_tls13)) { /* @@ -749,39 +1418,37 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * make sure that we are not getting application data when we are * doing a handshake for the first time */ - if (SSL_in_init(ssl) && type == SSL3_RT_APPLICATION_DATA - && SSL_IS_FIRST_HANDSHAKE(s)) { + if (SSL_in_init(s) && (type == SSL3_RT_APPLICATION_DATA) && + (s->enc_read_ctx == NULL)) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_APP_DATA_IN_HANDSHAKE); return -1; } if (type == SSL3_RT_HANDSHAKE - && rr->type == SSL3_RT_CHANGE_CIPHER_SPEC + && SSL3_RECORD_get_type(rr) == SSL3_RT_CHANGE_CIPHER_SPEC && s->rlayer.handshake_fragment_len > 0) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_CCS_RECEIVED_EARLY); return -1; } if (recvd_type != NULL) - *recvd_type = rr->type; + *recvd_type = SSL3_RECORD_get_type(rr); if (len == 0) { /* - * Skip a zero length record. This ensures multiple calls to + * Mark a zero length record as read. This ensures multiple calls to * SSL_read() with a zero length buffer will eventually cause * SSL_pending() to report data as being available. */ - if (rr->length == 0 && !ssl_release_record(s, rr, 0)) - return -1; - + if (SSL3_RECORD_get_length(rr) == 0) + SSL3_RECORD_set_read(rr); return 0; } totalbytes = 0; - curr_rec = s->rlayer.curr_rec; do { - if (len - totalbytes > rr->length) - n = rr->length; + if (len - totalbytes > SSL3_RECORD_get_length(rr)) + n = SSL3_RECORD_get_length(rr); else n = len - totalbytes; @@ -789,25 +1456,35 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, buf += n; if (peek) { /* Mark any zero length record as consumed CVE-2016-6305 */ - if (rr->length == 0 && !ssl_release_record(s, rr, 0)) - return -1; + if (SSL3_RECORD_get_length(rr) == 0) + SSL3_RECORD_set_read(rr); } else { - if (!ssl_release_record(s, rr, n)) - return -1; + if (s->options & SSL_OP_CLEANSE_PLAINTEXT) + OPENSSL_cleanse(&(rr->data[rr->off]), n); + SSL3_RECORD_sub_length(rr, n); + SSL3_RECORD_add_off(rr, n); + if (SSL3_RECORD_get_length(rr) == 0) { + s->rlayer.rstate = SSL_ST_READ_HEADER; + SSL3_RECORD_set_off(rr, 0); + SSL3_RECORD_set_read(rr); + } } - if (rr->length == 0 - || (peek && n == rr->length)) { - rr++; + if (SSL3_RECORD_get_length(rr) == 0 + || (peek && n == SSL3_RECORD_get_length(rr))) { curr_rec++; + rr++; } totalbytes += n; - } while (type == SSL3_RT_APPLICATION_DATA - && curr_rec < s->rlayer.num_recs - && totalbytes < len); + } while (type == SSL3_RT_APPLICATION_DATA && curr_rec < num_recs + && totalbytes < len); if (totalbytes == 0) { /* We must have read empty records. Get more data */ goto start; } + if (!peek && curr_rec == num_recs + && (s->mode & SSL_MODE_RELEASE_BUFFERS) + && SSL3_BUFFER_get_left(rbuf) == 0) + ssl3_release_read_buffer(s); *readbytes = totalbytes; return 1; } @@ -821,7 +1498,7 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, /* * Lets just double check that we've not got an SSLv2 record */ - if (rr->version == SSL2_VERSION) { + if (rr->rec_version == SSL2_VERSION) { /* * Should never happen. ssl3_get_record() should only give us an SSLv2 * record back if this is the first packet and we are looking for an @@ -832,7 +1509,7 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, return -1; } - if (ssl->method->version == TLS_ANY_VERSION + if (s->method->version == TLS_ANY_VERSION && (s->server || rr->type != SSL3_RT_ALERT)) { /* * If we've got this far and still haven't decided on what version @@ -840,7 +1517,7 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * with. We shouldn't be receiving anything other than a ClientHello * if we are a server. */ - s->version = rr->version; + s->version = rr->rec_version; SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_MESSAGE); return -1; } @@ -850,12 +1527,13 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * (Possibly rr is 'empty' now, i.e. rr->length may be 0.) */ - if (rr->type == SSL3_RT_ALERT) { + if (SSL3_RECORD_get_type(rr) == SSL3_RT_ALERT) { unsigned int alert_level, alert_descr; - const unsigned char *alert_bytes = rr->data + rr->off; + unsigned char *alert_bytes = SSL3_RECORD_get_data(rr) + + SSL3_RECORD_get_off(rr); PACKET alert; - if (!PACKET_buf_init(&alert, alert_bytes, rr->length) + if (!PACKET_buf_init(&alert, alert_bytes, SSL3_RECORD_get_length(rr)) || !PACKET_get_1(&alert, &alert_level) || !PACKET_get_1(&alert, &alert_descr) || PACKET_remaining(&alert) != 0) { @@ -863,25 +1541,26 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, return -1; } + s->s3.alert_level = alert_level; + if (s->msg_callback) - s->msg_callback(0, s->version, SSL3_RT_ALERT, alert_bytes, 2, ssl, + s->msg_callback(0, s->version, SSL3_RT_ALERT, alert_bytes, 2, s, s->msg_callback_arg); if (s->info_callback != NULL) cb = s->info_callback; - else if (ssl->ctx->info_callback != NULL) - cb = ssl->ctx->info_callback; + else if (s->ctx->info_callback != NULL) + cb = s->ctx->info_callback; if (cb != NULL) { j = (alert_level << 8) | alert_descr; - cb(ssl, SSL_CB_READ_ALERT, j); + cb(s, SSL_CB_READ_ALERT, j); } - if ((!is_tls13 && alert_level == SSL3_AL_WARNING) + if (alert_level == SSL3_AL_WARNING || (is_tls13 && alert_descr == SSL_AD_USER_CANCELLED)) { s->s3.warn_alert = alert_descr; - if (!ssl_release_record(s, rr, 0)) - return -1; + SSL3_RECORD_set_read(rr); s->rlayer.alert_count++; if (s->rlayer.alert_count == MAX_WARN_ALERT_COUNT) { @@ -908,8 +1587,7 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, SSL_AD_REASON_OFFSET + alert_descr, "SSL alert number %d", alert_descr); s->shutdown |= SSL_RECEIVED_SHUTDOWN; - if (!ssl_release_record(s, rr, 0)) - return -1; + SSL3_RECORD_set_read(rr); SSL_CTX_remove_session(s->session_ctx, s->session); return 0; } else if (alert_descr == SSL_AD_NO_RENEGOTIATION) { @@ -933,7 +1611,7 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, } if ((s->shutdown & SSL_SENT_SHUTDOWN) != 0) { - if (rr->type == SSL3_RT_HANDSHAKE) { + if (SSL3_RECORD_get_type(rr) == SSL3_RT_HANDSHAKE) { BIO *rbio; /* @@ -943,15 +1621,15 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * because we are unable to write any response due to having already * sent close_notify. */ - if (!SSL_CONNECTION_IS_TLS13(s)) { - if (!ssl_release_record(s, rr, 0)) - return -1; + if (!SSL_IS_TLS13(s)) { + SSL3_RECORD_set_length(rr, 0); + SSL3_RECORD_set_read(rr); if ((s->mode & SSL_MODE_AUTO_RETRY) != 0) goto start; s->rwstate = SSL_READING; - rbio = SSL_get_rbio(ssl); + rbio = SSL_get_rbio(s); BIO_clear_retry_flags(rbio); BIO_set_retry_read(rbio); return -1; @@ -964,8 +1642,8 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * above. * No alert sent because we already sent close_notify */ - if (!ssl_release_record(s, rr, 0)) - return -1; + SSL3_RECORD_set_length(rr, 0); + SSL3_RECORD_set_read(rr); SSLfatal(s, SSL_AD_NO_ALERT, SSL_R_APPLICATION_DATA_AFTER_CLOSE_NOTIFY); return -1; @@ -978,32 +1656,29 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * "SHUTDOWN" code above to avoid filling the fragment storage with data * that we're just going to discard. */ - if (rr->type == SSL3_RT_HANDSHAKE) { + if (SSL3_RECORD_get_type(rr) == SSL3_RT_HANDSHAKE) { size_t dest_maxlen = sizeof(s->rlayer.handshake_fragment); unsigned char *dest = s->rlayer.handshake_fragment; size_t *dest_len = &s->rlayer.handshake_fragment_len; n = dest_maxlen - *dest_len; /* available space in 'dest' */ - if (rr->length < n) - n = rr->length; /* available bytes */ + if (SSL3_RECORD_get_length(rr) < n) + n = SSL3_RECORD_get_length(rr); /* available bytes */ /* now move 'n' bytes: */ - if (n > 0) { - memcpy(dest + *dest_len, rr->data + rr->off, n); - *dest_len += n; - } - /* - * We release the number of bytes consumed, or the whole record if it - * is zero length - */ - if ((n > 0 || rr->length == 0) && !ssl_release_record(s, rr, n)) - return -1; + memcpy(dest + *dest_len, + SSL3_RECORD_get_data(rr) + SSL3_RECORD_get_off(rr), n); + SSL3_RECORD_add_off(rr, n); + SSL3_RECORD_sub_length(rr, n); + *dest_len += n; + if (SSL3_RECORD_get_length(rr) == 0) + SSL3_RECORD_set_read(rr); if (*dest_len < dest_maxlen) goto start; /* fragment was too small */ } - if (rr->type == SSL3_RT_CHANGE_CIPHER_SPEC) { + if (SSL3_RECORD_get_type(rr) == SSL3_RT_CHANGE_CIPHER_SPEC) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_CCS_RECEIVED_EARLY); return -1; } @@ -1019,7 +1694,7 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, /* We found handshake data, so we're going back into init */ ossl_statem_set_in_init(s, 1); - i = s->handshake_func(ssl); + i = s->handshake_func(s); /* SSLfatal() already called if appropriate */ if (i < 0) return i; @@ -1036,7 +1711,8 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, return -1; if (!(s->mode & SSL_MODE_AUTO_RETRY)) { - if (!RECORD_LAYER_read_pending(&s->rlayer)) { + if (SSL3_BUFFER_get_left(rbuf) == 0) { + /* no read-ahead left? */ BIO *bio; /* * In the case where we try to read application data, but we @@ -1045,7 +1721,7 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * problems in the blocking world */ s->rwstate = SSL_READING; - bio = SSL_get_rbio(ssl); + bio = SSL_get_rbio(s); BIO_clear_retry_flags(bio); BIO_set_retry_read(bio); return -1; @@ -1054,7 +1730,7 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, goto start; } - switch (rr->type) { + switch (SSL3_RECORD_get_type(rr)) { default: /* * TLS 1.0 and 1.1 say you SHOULD ignore unrecognised record types, but @@ -1096,13 +1772,12 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, * decryption was applied. We just skip it and move on to the next * record. */ - if (!ossl_early_data_count_ok(s, rr->length, - EARLY_DATA_CIPHERTEXT_OVERHEAD, 0)) { + if (!early_data_count_ok(s, rr->length, + EARLY_DATA_CIPHERTEXT_OVERHEAD, 0)) { /* SSLfatal() already called */ return -1; } - if (!ssl_release_record(s, rr, 0)) - return -1; + SSL3_RECORD_set_read(rr); goto start; } else { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_RECORD); @@ -1111,367 +1786,30 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, } } -/* - * Returns true if the current rrec was sent in SSLv2 backwards compatible - * format and false otherwise. - */ -int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl) -{ - if (SSL_CONNECTION_IS_DTLS(rl->s)) - return 0; - return rl->tlsrecs[0].version == SSL2_VERSION; -} - -static OSSL_FUNC_rlayer_msg_callback_fn rlayer_msg_callback_wrapper; -static void rlayer_msg_callback_wrapper(int write_p, int version, - int content_type, const void *buf, - size_t len, void *cbarg) -{ - SSL_CONNECTION *s = cbarg; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - - if (s->msg_callback != NULL) - s->msg_callback(write_p, version, content_type, buf, len, ssl, - s->msg_callback_arg); -} - -static OSSL_FUNC_rlayer_security_fn rlayer_security_wrapper; -static int rlayer_security_wrapper(void *cbarg, int op, int bits, int nid, - void *other) -{ - SSL_CONNECTION *s = cbarg; - - return ssl_security(s, op, bits, nid, other); -} - -static OSSL_FUNC_rlayer_padding_fn rlayer_padding_wrapper; -static size_t rlayer_padding_wrapper(void *cbarg, int type, size_t len) -{ - SSL_CONNECTION *s = cbarg; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - - return s->rlayer.record_padding_cb(ssl, type, len, - s->rlayer.record_padding_arg); -} - -static const OSSL_DISPATCH rlayer_dispatch[] = { - { OSSL_FUNC_RLAYER_SKIP_EARLY_DATA, (void (*)(void))ossl_statem_skip_early_data }, - { OSSL_FUNC_RLAYER_MSG_CALLBACK, (void (*)(void))rlayer_msg_callback_wrapper }, - { OSSL_FUNC_RLAYER_SECURITY, (void (*)(void))rlayer_security_wrapper }, - { OSSL_FUNC_RLAYER_PADDING, (void (*)(void))rlayer_padding_wrapper }, - OSSL_DISPATCH_END -}; - -void ossl_ssl_set_custom_record_layer(SSL_CONNECTION *s, - const OSSL_RECORD_METHOD *meth, - void *rlarg) -{ - s->rlayer.custom_rlmethod = meth; - s->rlayer.rlarg = rlarg; -} - -static const OSSL_RECORD_METHOD *ssl_select_next_record_layer(SSL_CONNECTION *s, - int direction, - int level) +void ssl3_record_sequence_update(unsigned char *seq) { - if (s->rlayer.custom_rlmethod != NULL) - return s->rlayer.custom_rlmethod; - - if (level == OSSL_RECORD_PROTECTION_LEVEL_NONE) { - if (SSL_CONNECTION_IS_DTLS(s)) - return &ossl_dtls_record_method; + int i; - return &ossl_tls_record_method; + for (i = 7; i >= 0; i--) { + ++seq[i]; + if (seq[i] != 0) + break; } - -#ifndef OPENSSL_NO_KTLS - /* KTLS does not support renegotiation */ - if (level == OSSL_RECORD_PROTECTION_LEVEL_APPLICATION - && (s->options & SSL_OP_ENABLE_KTLS) != 0 - && (SSL_CONNECTION_IS_TLS13(s) || SSL_IS_FIRST_HANDSHAKE(s))) - return &ossl_ktls_record_method; -#endif - - /* Default to the current OSSL_RECORD_METHOD */ - return direction == OSSL_RECORD_DIRECTION_READ ? s->rlayer.rrlmethod - : s->rlayer.wrlmethod; } -static int ssl_post_record_layer_select(SSL_CONNECTION *s, int direction) +/* + * Returns true if the current rrec was sent in SSLv2 backwards compatible + * format and false otherwise. + */ +int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl) { - const OSSL_RECORD_METHOD *thismethod; - OSSL_RECORD_LAYER *thisrl; - - if (direction == OSSL_RECORD_DIRECTION_READ) { - thismethod = s->rlayer.rrlmethod; - thisrl = s->rlayer.rrl; - } else { - thismethod = s->rlayer.wrlmethod; - thisrl = s->rlayer.wrl; - } - -#ifndef OPENSSL_NO_KTLS - { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - - if (s->rlayer.rrlmethod == &ossl_ktls_record_method) { - /* KTLS does not support renegotiation so disallow it */ - SSL_set_options(ssl, SSL_OP_NO_RENEGOTIATION); - } - } -#endif - if (SSL_IS_FIRST_HANDSHAKE(s) && thismethod->set_first_handshake != NULL) - thismethod->set_first_handshake(thisrl, 1); - - if (s->max_pipelines != 0 && thismethod->set_max_pipelines != NULL) - thismethod->set_max_pipelines(thisrl, s->max_pipelines); - - return 1; + return SSL3_RECORD_is_sslv2_record(&rl->rrec[0]); } -int ssl_set_new_record_layer(SSL_CONNECTION *s, int version, - int direction, int level, - unsigned char *secret, size_t secretlen, - unsigned char *key, size_t keylen, - unsigned char *iv, size_t ivlen, - unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, size_t taglen, - int mactype, const EVP_MD *md, - const SSL_COMP *comp, const EVP_MD *kdfdigest) -{ - OSSL_PARAM options[5], *opts = options; - OSSL_PARAM settings[6], *set = settings; - const OSSL_RECORD_METHOD **thismethod; - OSSL_RECORD_LAYER **thisrl, *newrl = NULL; - BIO *thisbio; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - const OSSL_RECORD_METHOD *meth; - int use_etm, stream_mac = 0, tlstree = 0; - unsigned int maxfrag = (direction == OSSL_RECORD_DIRECTION_WRITE) - ? ssl_get_max_send_fragment(s) - : SSL3_RT_MAX_PLAIN_LENGTH; - int use_early_data = 0; - uint32_t max_early_data; - COMP_METHOD *compm = (comp == NULL) ? NULL : comp->method; - - meth = ssl_select_next_record_layer(s, direction, level); - - if (direction == OSSL_RECORD_DIRECTION_READ) { - thismethod = &s->rlayer.rrlmethod; - thisrl = &s->rlayer.rrl; - thisbio = s->rbio; - } else { - thismethod = &s->rlayer.wrlmethod; - thisrl = &s->rlayer.wrl; - thisbio = s->wbio; - } - - if (meth == NULL) - meth = *thismethod; - - if (!ossl_assert(meth != NULL)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); - return 0; - } - - /* Parameters that *may* be supported by a record layer if passed */ - *opts++ = OSSL_PARAM_construct_uint64(OSSL_LIBSSL_RECORD_LAYER_PARAM_OPTIONS, - &s->options); - *opts++ = OSSL_PARAM_construct_uint32(OSSL_LIBSSL_RECORD_LAYER_PARAM_MODE, - &s->mode); - if (direction == OSSL_RECORD_DIRECTION_READ) { - *opts++ = OSSL_PARAM_construct_size_t(OSSL_LIBSSL_RECORD_LAYER_READ_BUFFER_LEN, - &s->rlayer.default_read_buf_len); - *opts++ = OSSL_PARAM_construct_int(OSSL_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD, - &s->rlayer.read_ahead); - } else { - *opts++ = OSSL_PARAM_construct_size_t(OSSL_LIBSSL_RECORD_LAYER_PARAM_BLOCK_PADDING, - &s->rlayer.block_padding); - } - *opts = OSSL_PARAM_construct_end(); - - /* Parameters that *must* be supported by a record layer if passed */ - if (direction == OSSL_RECORD_DIRECTION_READ) { - use_etm = SSL_READ_ETM(s) ? 1 : 0; - if ((s->mac_flags & SSL_MAC_FLAG_READ_MAC_STREAM) != 0) - stream_mac = 1; - - if ((s->mac_flags & SSL_MAC_FLAG_READ_MAC_TLSTREE) != 0) - tlstree = 1; - } else { - use_etm = SSL_WRITE_ETM(s) ? 1 : 0; - if ((s->mac_flags & SSL_MAC_FLAG_WRITE_MAC_STREAM) != 0) - stream_mac = 1; - - if ((s->mac_flags & SSL_MAC_FLAG_WRITE_MAC_TLSTREE) != 0) - tlstree = 1; - } - - if (use_etm) - *set++ = OSSL_PARAM_construct_int(OSSL_LIBSSL_RECORD_LAYER_PARAM_USE_ETM, - &use_etm); - - if (stream_mac) - *set++ = OSSL_PARAM_construct_int(OSSL_LIBSSL_RECORD_LAYER_PARAM_STREAM_MAC, - &stream_mac); - - if (tlstree) - *set++ = OSSL_PARAM_construct_int(OSSL_LIBSSL_RECORD_LAYER_PARAM_TLSTREE, - &tlstree); - - /* - * We only need to do this for the read side. The write side should already - * have the correct value due to the ssl_get_max_send_fragment() call above - */ - if (direction == OSSL_RECORD_DIRECTION_READ - && s->session != NULL - && USE_MAX_FRAGMENT_LENGTH_EXT(s->session)) - maxfrag = GET_MAX_FRAGMENT_LENGTH(s->session); - - - if (maxfrag != SSL3_RT_MAX_PLAIN_LENGTH) - *set++ = OSSL_PARAM_construct_uint(OSSL_LIBSSL_RECORD_LAYER_PARAM_MAX_FRAG_LEN, - &maxfrag); - - /* - * The record layer must check the amount of early data sent or received - * using the early keys. A server also needs to worry about rejected early - * data that might arrive when the handshake keys are in force. - */ - if (s->server && direction == OSSL_RECORD_DIRECTION_READ) { - use_early_data = (level == OSSL_RECORD_PROTECTION_LEVEL_EARLY - || level == OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE); - } else if (!s->server && direction == OSSL_RECORD_DIRECTION_WRITE) { - use_early_data = (level == OSSL_RECORD_PROTECTION_LEVEL_EARLY); - } - if (use_early_data) { - max_early_data = ossl_get_max_early_data(s); - - if (max_early_data != 0) - *set++ = OSSL_PARAM_construct_uint32(OSSL_LIBSSL_RECORD_LAYER_PARAM_MAX_EARLY_DATA, - &max_early_data); - } - - *set = OSSL_PARAM_construct_end(); - - for (;;) { - int rlret; - BIO *prev = NULL; - BIO *next = NULL; - unsigned int epoch = 0; - OSSL_DISPATCH rlayer_dispatch_tmp[OSSL_NELEM(rlayer_dispatch)]; - size_t i, j; - - if (direction == OSSL_RECORD_DIRECTION_READ) { - prev = s->rlayer.rrlnext; - if (SSL_CONNECTION_IS_DTLS(s) - && level != OSSL_RECORD_PROTECTION_LEVEL_NONE) - epoch = dtls1_get_epoch(s, SSL3_CC_READ); /* new epoch */ - -#ifndef OPENSSL_NO_DGRAM - if (SSL_CONNECTION_IS_DTLS(s)) - next = BIO_new(BIO_s_dgram_mem()); - else -#endif - next = BIO_new(BIO_s_mem()); - - if (next == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - s->rlayer.rrlnext = next; - } else { - if (SSL_CONNECTION_IS_DTLS(s) - && level != OSSL_RECORD_PROTECTION_LEVEL_NONE) - epoch = dtls1_get_epoch(s, SSL3_CC_WRITE); /* new epoch */ - } - - /* - * Create a copy of the dispatch array, missing out wrappers for - * callbacks that we don't need. - */ - for (i = 0, j = 0; i < OSSL_NELEM(rlayer_dispatch); i++) { - switch (rlayer_dispatch[i].function_id) { - case OSSL_FUNC_RLAYER_MSG_CALLBACK: - if (s->msg_callback == NULL) - continue; - break; - case OSSL_FUNC_RLAYER_PADDING: - if (s->rlayer.record_padding_cb == NULL) - continue; - break; - default: - break; - } - rlayer_dispatch_tmp[j++] = rlayer_dispatch[i]; - } - - rlret = meth->new_record_layer(sctx->libctx, sctx->propq, version, - s->server, direction, level, epoch, - secret, secretlen, key, keylen, iv, - ivlen, mackey, mackeylen, ciph, taglen, - mactype, md, compm, kdfdigest, prev, - thisbio, next, NULL, NULL, settings, - options, rlayer_dispatch_tmp, s, - s->rlayer.rlarg, &newrl); - BIO_free(prev); - switch (rlret) { - case OSSL_RECORD_RETURN_FATAL: - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_RECORD_LAYER_FAILURE); - return 0; - - case OSSL_RECORD_RETURN_NON_FATAL_ERR: - if (*thismethod != meth && *thismethod != NULL) { - /* - * We tried a new record layer method, but it didn't work out, - * so we fallback to the original method and try again - */ - meth = *thismethod; - continue; - } - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_NO_SUITABLE_RECORD_LAYER); - return 0; - - case OSSL_RECORD_RETURN_SUCCESS: - break; - - default: - /* Should not happen */ - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - break; - } - - /* - * Free the old record layer if we have one except in the case of DTLS when - * writing and there are still buffered sent messages in our queue. In that - * case the record layer is still referenced by those buffered messages for - * potential retransmit. Only when those buffered messages get freed do we - * free the record layer object (see dtls1_hm_fragment_free) - */ - if (!SSL_CONNECTION_IS_DTLS(s) - || direction == OSSL_RECORD_DIRECTION_READ - || pqueue_peek(s->d1->sent_messages) == NULL) { - if (*thismethod != NULL && !(*thismethod)->free(*thisrl)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - } - - *thisrl = newrl; - *thismethod = meth; - - return ssl_post_record_layer_select(s, direction); -} - -int ssl_set_record_protocol_version(SSL_CONNECTION *s, int vers) +/* + * Returns the length in bytes of the current rrec + */ +size_t RECORD_LAYER_get_rrec_length(RECORD_LAYER *rl) { - if (!ossl_assert(s->rlayer.rrlmethod != NULL) - || !ossl_assert(s->rlayer.wrlmethod != NULL)) - return 0; - s->rlayer.rrlmethod->set_protocol_version(s->rlayer.rrl, s->version); - s->rlayer.wrlmethod->set_protocol_version(s->rlayer.wrl, s->version); - - return 1; + return SSL3_RECORD_get_length(&rl->rrec[0]); } diff --git a/openssl/src/ssl/record/record.h b/openssl/src/ssl/record/record.h index 9a076a1fb..234656bf9 100644 --- a/openssl/src/ssl/record/record.h +++ b/openssl/src/ssl/record/record.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,9 +7,6 @@ * https://www.openssl.org/source/license.html */ -#include -#include "internal/recordmethod.h" - /***************************************************************************** * * * These structures should be considered PRIVATE to the record layer. No * @@ -17,31 +14,83 @@ * * *****************************************************************************/ +typedef struct ssl3_buffer_st { + /* at least SSL3_RT_MAX_PACKET_SIZE bytes, see ssl3_setup_buffers() */ + unsigned char *buf; + /* default buffer size (or 0 if no default set) */ + size_t default_len; + /* buffer size */ + size_t len; + /* where to 'copy from' */ + size_t offset; + /* how many bytes left */ + size_t left; + /* 'buf' is from application for KTLS */ + int app_buffer; +} SSL3_BUFFER; + #define SEQ_NUM_SIZE 8 -typedef struct tls_record_st { - void *rechandle; - int version; - uint8_t type; - /* The data buffer containing bytes from the record */ - const unsigned char *data; +typedef struct ssl3_record_st { + /* Record layer version */ + /* r */ + int rec_version; + /* type of record */ + /* r */ + int type; + /* How many bytes available */ + /* rw */ + size_t length; /* - * Buffer that we allocated to store data. If non NULL always the same as - * data (but non-const) + * How many bytes were available before padding was removed? This is used + * to implement the MAC check in constant time for CBC records. */ - unsigned char *allocdata; - /* Number of remaining to be read in the data buffer */ - size_t length; - /* Offset into the data buffer where to start reading */ + /* rw */ + size_t orig_len; + /* read/write offset into 'buf' */ + /* r */ size_t off; - /* epoch number. DTLS only */ - uint16_t epoch; - /* sequence number. DTLS only */ + /* pointer to the record data */ + /* rw */ + unsigned char *data; + /* where the decode bytes are */ + /* rw */ + unsigned char *input; + /* only used with decompression - malloc()ed */ + /* r */ + unsigned char *comp; + /* Whether the data from this record has already been read or not */ + /* r */ + unsigned int read; + /* epoch number, needed by DTLS1 */ + /* r */ + unsigned long epoch; + /* sequence number, needed by DTLS1 */ + /* r */ unsigned char seq_num[SEQ_NUM_SIZE]; +} SSL3_RECORD; + +typedef struct dtls1_bitmap_st { + /* Track 32 packets on 32-bit systems and 64 - on 64-bit systems */ + unsigned long map; + /* Max record number seen so far, 64-bit value in big-endian encoding */ + unsigned char max_seq_num[SEQ_NUM_SIZE]; +} DTLS1_BITMAP; + +typedef struct record_pqueue_st { + unsigned short epoch; + struct pqueue_st *q; +} record_pqueue; + +typedef struct dtls1_record_data_st { + unsigned char *packet; + size_t packet_length; + SSL3_BUFFER rbuf; + SSL3_RECORD rrec; #ifndef OPENSSL_NO_SCTP struct bio_dgram_sctp_rcvinfo recordinfo; #endif -} TLS_RECORD; +} DTLS1_RECORD_DATA; typedef struct dtls_record_layer_st { /* @@ -49,15 +98,24 @@ typedef struct dtls_record_layer_st { * undefined, and starts at zero once the initial handshake is * completed */ - uint16_t r_epoch; - uint16_t w_epoch; - + unsigned short r_epoch; + unsigned short w_epoch; + /* records being received in the current epoch */ + DTLS1_BITMAP bitmap; + /* renegotiation starts a new set of sequence numbers */ + DTLS1_BITMAP next_bitmap; + /* Received handshake records (processed and unprocessed) */ + record_pqueue unprocessed_rcds; + record_pqueue processed_rcds; /* * Buffered application records. Only for records between CCS and * Finished to prevent either protocol violation or unnecessary message * loss. */ - struct pqueue_st *buffered_app_data; + record_pqueue buffered_app_data; + /* save last and current sequence numbers for retransmissions */ + unsigned char last_write_sequence[8]; + unsigned char curr_write_sequence[8]; } DTLS_RECORD_LAYER; /***************************************************************************** @@ -69,58 +127,48 @@ typedef struct dtls_record_layer_st { *****************************************************************************/ typedef struct record_layer_st { - /* The parent SSL_CONNECTION structure */ - SSL_CONNECTION *s; - - /* Custom record layer: always selected if set */ - const OSSL_RECORD_METHOD *custom_rlmethod; - /* Record layer specific argument */ - void *rlarg; - /* Method to use for the read record layer*/ - const OSSL_RECORD_METHOD *rrlmethod; - /* Method to use for the write record layer*/ - const OSSL_RECORD_METHOD *wrlmethod; - /* The read record layer object itself */ - OSSL_RECORD_LAYER *rrl; - /* The write record layer object itself */ - OSSL_RECORD_LAYER *wrl; - /* BIO to store data destined for the next read record layer epoch */ - BIO *rrlnext; - /* Default read buffer length to be passed to the record layer */ - size_t default_read_buf_len; - + /* The parent SSL structure */ + SSL *s; /* * Read as many input bytes as possible (for * non-blocking reads) */ int read_ahead; - + /* where we are when reading */ + int rstate; + /* How many pipelines can be used to read data */ + size_t numrpipes; + /* How many pipelines can be used to write data */ + size_t numwpipes; + /* read IO goes into here */ + SSL3_BUFFER rbuf; + /* write IO goes into here */ + SSL3_BUFFER wbuf[SSL_MAX_PIPELINES]; + /* each decoded record goes in here */ + SSL3_RECORD rrec[SSL_MAX_PIPELINES]; + /* used internally to point at a raw packet */ + unsigned char *packet; + size_t packet_length; /* number of bytes sent so far */ size_t wnum; unsigned char handshake_fragment[4]; size_t handshake_fragment_len; + /* The number of consecutive empty records we have received */ + size_t empty_record_count; /* partial write - check the numbers match */ /* number bytes written */ size_t wpend_tot; - uint8_t wpend_type; + int wpend_type; + /* number of bytes submitted */ + size_t wpend_ret; const unsigned char *wpend_buf; - + unsigned char read_sequence[SEQ_NUM_SIZE]; + unsigned char write_sequence[SEQ_NUM_SIZE]; + /* Set to true if this is the first record in a connection */ + unsigned int is_first_record; /* Count of the number of consecutive warning alerts received */ unsigned int alert_count; DTLS_RECORD_LAYER *d; - - /* TLS1.3 padding callback */ - size_t (*record_padding_cb)(SSL *s, int type, size_t len, void *arg); - void *record_padding_arg; - size_t block_padding; - - /* How many records we have read from the record layer */ - size_t num_recs; - /* The next record from the record layer that we need to process */ - size_t curr_rec; - /* Record layer data to be processed */ - TLS_RECORD tlsrecs[SSL_MAX_PIPELINES]; - } RECORD_LAYER; /***************************************************************************** @@ -130,66 +178,70 @@ typedef struct record_layer_st { * * *****************************************************************************/ +struct ssl_mac_buf_st { + unsigned char *mac; + int alloced; +}; +typedef struct ssl_mac_buf_st SSL_MAC_BUF; + +#define MIN_SSL2_RECORD_LEN 9 + #define RECORD_LAYER_set_read_ahead(rl, ra) ((rl)->read_ahead = (ra)) #define RECORD_LAYER_get_read_ahead(rl) ((rl)->read_ahead) - -void RECORD_LAYER_init(RECORD_LAYER *rl, SSL_CONNECTION *s); -int RECORD_LAYER_clear(RECORD_LAYER *rl); -int RECORD_LAYER_reset(RECORD_LAYER *rl); +#define RECORD_LAYER_get_packet(rl) ((rl)->packet) +#define RECORD_LAYER_get_packet_length(rl) ((rl)->packet_length) +#define RECORD_LAYER_add_packet_length(rl, inc) ((rl)->packet_length += (inc)) +#define DTLS_RECORD_LAYER_get_w_epoch(rl) ((rl)->d->w_epoch) +#define DTLS_RECORD_LAYER_get_processed_rcds(rl) \ + ((rl)->d->processed_rcds) +#define DTLS_RECORD_LAYER_get_unprocessed_rcds(rl) \ + ((rl)->d->unprocessed_rcds) +#define RECORD_LAYER_get_rbuf(rl) (&(rl)->rbuf) +#define RECORD_LAYER_get_wbuf(rl) ((rl)->wbuf) + +void RECORD_LAYER_init(RECORD_LAYER *rl, SSL *s); +void RECORD_LAYER_clear(RECORD_LAYER *rl); +void RECORD_LAYER_release(RECORD_LAYER *rl); int RECORD_LAYER_read_pending(const RECORD_LAYER *rl); int RECORD_LAYER_processed_read_pending(const RECORD_LAYER *rl); int RECORD_LAYER_write_pending(const RECORD_LAYER *rl); +void RECORD_LAYER_reset_read_sequence(RECORD_LAYER *rl); +void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl); int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl); +size_t RECORD_LAYER_get_rrec_length(RECORD_LAYER *rl); __owur size_t ssl3_pending(const SSL *s); -__owur int ssl3_write_bytes(SSL *s, uint8_t type, const void *buf, size_t len, +__owur int ssl3_write_bytes(SSL *s, int type, const void *buf, size_t len, size_t *written); -__owur int ssl3_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, +int do_ssl3_write(SSL *s, int type, const unsigned char *buf, + size_t *pipelens, size_t numpipes, + int create_empty_fragment, size_t *written); +__owur int ssl3_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf, size_t len, int peek, size_t *readbytes); - +__owur int ssl3_setup_buffers(SSL *s); +__owur int ssl3_enc(SSL *s, SSL3_RECORD *inrecs, size_t n_recs, int send, + SSL_MAC_BUF *mac, size_t macsize); +__owur int n_ssl3_mac(SSL *ssl, SSL3_RECORD *rec, unsigned char *md, int send); +__owur int ssl3_write_pending(SSL *s, int type, const unsigned char *buf, size_t len, + size_t *written); +__owur int tls1_enc(SSL *s, SSL3_RECORD *recs, size_t n_recs, int sending, + SSL_MAC_BUF *mac, size_t macsize); +__owur int tls1_mac(SSL *ssl, SSL3_RECORD *rec, unsigned char *md, int send); +__owur int tls13_enc(SSL *s, SSL3_RECORD *recs, size_t n_recs, int send, + SSL_MAC_BUF *mac, size_t macsize); int DTLS_RECORD_LAYER_new(RECORD_LAYER *rl); void DTLS_RECORD_LAYER_free(RECORD_LAYER *rl); void DTLS_RECORD_LAYER_clear(RECORD_LAYER *rl); -__owur int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type, +void DTLS_RECORD_LAYER_set_saved_w_epoch(RECORD_LAYER *rl, unsigned short e); +void DTLS_RECORD_LAYER_clear(RECORD_LAYER *rl); +void DTLS_RECORD_LAYER_set_write_sequence(RECORD_LAYER *rl, unsigned char *seq); +__owur int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf, size_t len, int peek, size_t *readbytes); -__owur int dtls1_write_bytes(SSL_CONNECTION *s, uint8_t type, const void *buf, - size_t len, size_t *written); -int do_dtls1_write(SSL_CONNECTION *s, uint8_t type, const unsigned char *buf, - size_t len, size_t *written); -void dtls1_increment_epoch(SSL_CONNECTION *s, int rw); -uint16_t dtls1_get_epoch(SSL_CONNECTION *s, int rw); -int ssl_release_record(SSL_CONNECTION *s, TLS_RECORD *rr, size_t length); - -# define HANDLE_RLAYER_READ_RETURN(s, ret) \ - ossl_tls_handle_rlayer_return(s, 0, ret, OPENSSL_FILE, OPENSSL_LINE) - -# define HANDLE_RLAYER_WRITE_RETURN(s, ret) \ - ossl_tls_handle_rlayer_return(s, 1, ret, OPENSSL_FILE, OPENSSL_LINE) - -int ossl_tls_handle_rlayer_return(SSL_CONNECTION *s, int writing, int ret, - char *file, int line); - -int ssl_set_new_record_layer(SSL_CONNECTION *s, int version, - int direction, int level, - unsigned char *secret, size_t secretlen, - unsigned char *key, size_t keylen, - unsigned char *iv, size_t ivlen, - unsigned char *mackey, size_t mackeylen, - const EVP_CIPHER *ciph, size_t taglen, - int mactype, const EVP_MD *md, - const SSL_COMP *comp, const EVP_MD *kdfdigest); -int ssl_set_record_protocol_version(SSL_CONNECTION *s, int vers); - -# define OSSL_FUNC_RLAYER_SKIP_EARLY_DATA 1 -OSSL_CORE_MAKE_FUNC(int, rlayer_skip_early_data, (void *cbarg)) -# define OSSL_FUNC_RLAYER_MSG_CALLBACK 2 -OSSL_CORE_MAKE_FUNC(void, rlayer_msg_callback, (int write_p, int version, - int content_type, - const void *buf, size_t len, - void *cbarg)) -# define OSSL_FUNC_RLAYER_SECURITY 3 -OSSL_CORE_MAKE_FUNC(int, rlayer_security, (void *cbarg, int op, int bits, - int nid, void *other)) -# define OSSL_FUNC_RLAYER_PADDING 4 -OSSL_CORE_MAKE_FUNC(size_t, rlayer_padding, (void *cbarg, int type, size_t len)) +__owur int dtls1_write_bytes(SSL *s, int type, const void *buf, size_t len, + size_t *written); +int do_dtls1_write(SSL *s, int type, const unsigned char *buf, + size_t len, int create_empty_fragment, size_t *written); +void dtls1_reset_seq_numbers(SSL *s, int rw); +int dtls_buffer_listen_record(SSL *s, size_t len, unsigned char *seq, + size_t off); diff --git a/openssl/src/ssl/record/record_local.h b/openssl/src/ssl/record/record_local.h index 1acb588f8..0a929c696 100644 --- a/openssl/src/ssl/record/record_local.h +++ b/openssl/src/ssl/record/record_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -15,3 +15,113 @@ *****************************************************************************/ #define MAX_WARN_ALERT_COUNT 5 + +/* Functions/macros provided by the RECORD_LAYER component */ + +#define RECORD_LAYER_get_rrec(rl) ((rl)->rrec) +#define RECORD_LAYER_set_packet(rl, p) ((rl)->packet = (p)) +#define RECORD_LAYER_reset_packet_length(rl) ((rl)->packet_length = 0) +#define RECORD_LAYER_get_rstate(rl) ((rl)->rstate) +#define RECORD_LAYER_set_rstate(rl, st) ((rl)->rstate = (st)) +#define RECORD_LAYER_get_read_sequence(rl) ((rl)->read_sequence) +#define RECORD_LAYER_get_write_sequence(rl) ((rl)->write_sequence) +#define RECORD_LAYER_get_numrpipes(rl) ((rl)->numrpipes) +#define RECORD_LAYER_set_numrpipes(rl, n) ((rl)->numrpipes = (n)) +#define RECORD_LAYER_inc_empty_record_count(rl) ((rl)->empty_record_count++) +#define RECORD_LAYER_reset_empty_record_count(rl) \ + ((rl)->empty_record_count = 0) +#define RECORD_LAYER_get_empty_record_count(rl) ((rl)->empty_record_count) +#define RECORD_LAYER_is_first_record(rl) ((rl)->is_first_record) +#define RECORD_LAYER_set_first_record(rl) ((rl)->is_first_record = 1) +#define RECORD_LAYER_clear_first_record(rl) ((rl)->is_first_record = 0) +#define DTLS_RECORD_LAYER_get_r_epoch(rl) ((rl)->d->r_epoch) + +__owur int ssl3_read_n(SSL *s, size_t n, size_t max, int extend, int clearold, + size_t *readbytes); + +DTLS1_BITMAP *dtls1_get_bitmap(SSL *s, SSL3_RECORD *rr, + unsigned int *is_next_epoch); +int dtls1_process_buffered_records(SSL *s); +int dtls1_retrieve_buffered_record(SSL *s, record_pqueue *queue); +int dtls1_buffer_record(SSL *s, record_pqueue *q, unsigned char *priority); +void ssl3_record_sequence_update(unsigned char *seq); + +/* Functions provided by the DTLS1_BITMAP component */ + +int dtls1_record_replay_check(SSL *s, DTLS1_BITMAP *bitmap); +void dtls1_record_bitmap_update(SSL *s, DTLS1_BITMAP *bitmap); + +/* Macros/functions provided by the SSL3_BUFFER component */ + +#define SSL3_BUFFER_get_buf(b) ((b)->buf) +#define SSL3_BUFFER_set_buf(b, n) ((b)->buf = (n)) +#define SSL3_BUFFER_get_len(b) ((b)->len) +#define SSL3_BUFFER_set_len(b, l) ((b)->len = (l)) +#define SSL3_BUFFER_get_left(b) ((b)->left) +#define SSL3_BUFFER_set_left(b, l) ((b)->left = (l)) +#define SSL3_BUFFER_sub_left(b, l) ((b)->left -= (l)) +#define SSL3_BUFFER_get_offset(b) ((b)->offset) +#define SSL3_BUFFER_set_offset(b, o) ((b)->offset = (o)) +#define SSL3_BUFFER_add_offset(b, o) ((b)->offset += (o)) +#define SSL3_BUFFER_is_initialised(b) ((b)->buf != NULL) +#define SSL3_BUFFER_set_default_len(b, l) ((b)->default_len = (l)) +#define SSL3_BUFFER_set_app_buffer(b, l) ((b)->app_buffer = (l)) +#define SSL3_BUFFER_is_app_buffer(b) ((b)->app_buffer) + +void SSL3_BUFFER_clear(SSL3_BUFFER *b); +void SSL3_BUFFER_set_data(SSL3_BUFFER *b, const unsigned char *d, size_t n); +void SSL3_BUFFER_release(SSL3_BUFFER *b); +__owur int ssl3_setup_read_buffer(SSL *s); +__owur int ssl3_setup_write_buffer(SSL *s, size_t numwpipes, size_t len); +int ssl3_release_read_buffer(SSL *s); +int ssl3_release_write_buffer(SSL *s); + +/* Macros/functions provided by the SSL3_RECORD component */ + +#define SSL3_RECORD_get_type(r) ((r)->type) +#define SSL3_RECORD_set_type(r, t) ((r)->type = (t)) +#define SSL3_RECORD_set_rec_version(r, v) ((r)->rec_version = (v)) +#define SSL3_RECORD_get_length(r) ((r)->length) +#define SSL3_RECORD_set_length(r, l) ((r)->length = (l)) +#define SSL3_RECORD_add_length(r, l) ((r)->length += (l)) +#define SSL3_RECORD_sub_length(r, l) ((r)->length -= (l)) +#define SSL3_RECORD_get_data(r) ((r)->data) +#define SSL3_RECORD_set_data(r, d) ((r)->data = (d)) +#define SSL3_RECORD_get_input(r) ((r)->input) +#define SSL3_RECORD_set_input(r, i) ((r)->input = (i)) +#define SSL3_RECORD_reset_input(r) ((r)->input = (r)->data) +#define SSL3_RECORD_reset_data(r) ((r)->data = (r)->input) +#define SSL3_RECORD_get_seq_num(r) ((r)->seq_num) +#define SSL3_RECORD_get_off(r) ((r)->off) +#define SSL3_RECORD_set_off(r, o) ((r)->off = (o)) +#define SSL3_RECORD_add_off(r, o) ((r)->off += (o)) +#define SSL3_RECORD_get_epoch(r) ((r)->epoch) +#define SSL3_RECORD_is_sslv2_record(r) \ + ((r)->rec_version == SSL2_VERSION) +#define SSL3_RECORD_is_read(r) ((r)->read) +#define SSL3_RECORD_set_read(r) ((r)->read = 1) + +void SSL3_RECORD_clear(SSL3_RECORD *r, size_t); +void SSL3_RECORD_release(SSL3_RECORD *r, size_t num_recs); +void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num); +int ssl3_get_record(SSL *s); +__owur int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr); +__owur int ssl3_do_uncompress(SSL *ssl, SSL3_RECORD *rr); +__owur int ssl3_cbc_remove_padding_and_mac(size_t *reclen, + size_t origreclen, + unsigned char *recdata, + unsigned char **mac, + int *alloced, + size_t block_size, size_t mac_size, + OSSL_LIB_CTX *libctx); +__owur int tls1_cbc_remove_padding_and_mac(size_t *reclen, + size_t origreclen, + unsigned char *recdata, + unsigned char **mac, + int *alloced, + size_t block_size, size_t mac_size, + int aead, + OSSL_LIB_CTX *libctx); +int dtls1_process_record(SSL *s, DTLS1_BITMAP *bitmap); +__owur int dtls1_get_record(SSL *s); +int early_data_count_ok(SSL *s, size_t length, size_t overhead, int send); diff --git a/openssl/src/ssl/record/ssl3_buffer.c b/openssl/src/ssl/record/ssl3_buffer.c new file mode 100644 index 000000000..01c553ebf --- /dev/null +++ b/openssl/src/ssl/record/ssl3_buffer.c @@ -0,0 +1,185 @@ +/* + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../ssl_local.h" +#include "record_local.h" + +void SSL3_BUFFER_set_data(SSL3_BUFFER *b, const unsigned char *d, size_t n) +{ + if (d != NULL) + memcpy(b->buf, d, n); + b->left = n; + b->offset = 0; +} + +/* + * Clear the contents of an SSL3_BUFFER but retain any memory allocated. Also + * retains the default_len setting + */ +void SSL3_BUFFER_clear(SSL3_BUFFER *b) +{ + b->offset = 0; + b->left = 0; +} + +void SSL3_BUFFER_release(SSL3_BUFFER *b) +{ + OPENSSL_free(b->buf); + b->buf = NULL; +} + +int ssl3_setup_read_buffer(SSL *s) +{ + unsigned char *p; + size_t len, align = 0, headerlen; + SSL3_BUFFER *b; + + b = RECORD_LAYER_get_rbuf(&s->rlayer); + + if (SSL_IS_DTLS(s)) + headerlen = DTLS1_RT_HEADER_LENGTH; + else + headerlen = SSL3_RT_HEADER_LENGTH; + +#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0 + align = (-SSL3_RT_HEADER_LENGTH) & (SSL3_ALIGN_PAYLOAD - 1); +#endif + + if (b->buf == NULL) { + len = SSL3_RT_MAX_PLAIN_LENGTH + + SSL3_RT_MAX_ENCRYPTED_OVERHEAD + headerlen + align; +#ifndef OPENSSL_NO_COMP + if (ssl_allow_compression(s)) + len += SSL3_RT_MAX_COMPRESSED_OVERHEAD; +#endif + if (b->default_len > len) + len = b->default_len; + if ((p = OPENSSL_malloc(len)) == NULL) { + /* + * We've got a malloc failure, and we're still initialising buffers. + * We assume we're so doomed that we won't even be able to send an + * alert. + */ + SSLfatal(s, SSL_AD_NO_ALERT, ERR_R_MALLOC_FAILURE); + return 0; + } + b->buf = p; + b->len = len; + } + + return 1; +} + +int ssl3_setup_write_buffer(SSL *s, size_t numwpipes, size_t len) +{ + unsigned char *p; + size_t align = 0, headerlen; + SSL3_BUFFER *wb; + size_t currpipe; + + s->rlayer.numwpipes = numwpipes; + + if (len == 0) { + if (SSL_IS_DTLS(s)) + headerlen = DTLS1_RT_HEADER_LENGTH + 1; + else + headerlen = SSL3_RT_HEADER_LENGTH; + +#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0 + align = SSL3_ALIGN_PAYLOAD - 1; +#endif + + len = ssl_get_max_send_fragment(s) + + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD + headerlen + align; +#ifndef OPENSSL_NO_COMP + if (ssl_allow_compression(s)) + len += SSL3_RT_MAX_COMPRESSED_OVERHEAD; +#endif + if (!(s->options & SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS)) + len += headerlen + align + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD; + } + + wb = RECORD_LAYER_get_wbuf(&s->rlayer); + for (currpipe = 0; currpipe < numwpipes; currpipe++) { + SSL3_BUFFER *thiswb = &wb[currpipe]; + + if (thiswb->len != len) { + OPENSSL_free(thiswb->buf); + thiswb->buf = NULL; /* force reallocation */ + } + + if (thiswb->buf == NULL) { + if (s->wbio == NULL || !BIO_get_ktls_send(s->wbio)) { + p = OPENSSL_malloc(len); + if (p == NULL) { + s->rlayer.numwpipes = currpipe; + /* + * We've got a malloc failure, and we're still initialising + * buffers. We assume we're so doomed that we won't even be able + * to send an alert. + */ + SSLfatal(s, SSL_AD_NO_ALERT, ERR_R_MALLOC_FAILURE); + return 0; + } + } else { + p = NULL; + } + memset(thiswb, 0, sizeof(SSL3_BUFFER)); + thiswb->buf = p; + thiswb->len = len; + } + } + + return 1; +} + +int ssl3_setup_buffers(SSL *s) +{ + if (!ssl3_setup_read_buffer(s)) { + /* SSLfatal() already called */ + return 0; + } + if (!ssl3_setup_write_buffer(s, 1, 0)) { + /* SSLfatal() already called */ + return 0; + } + return 1; +} + +int ssl3_release_write_buffer(SSL *s) +{ + SSL3_BUFFER *wb; + size_t pipes; + + pipes = s->rlayer.numwpipes; + while (pipes > 0) { + wb = &RECORD_LAYER_get_wbuf(&s->rlayer)[pipes - 1]; + + if (SSL3_BUFFER_is_app_buffer(wb)) + SSL3_BUFFER_set_app_buffer(wb, 0); + else + OPENSSL_free(wb->buf); + wb->buf = NULL; + pipes--; + } + s->rlayer.numwpipes = 0; + return 1; +} + +int ssl3_release_read_buffer(SSL *s) +{ + SSL3_BUFFER *b; + + b = RECORD_LAYER_get_rbuf(&s->rlayer); + if (s->options & SSL_OP_CLEANSE_PLAINTEXT) + OPENSSL_cleanse(b->buf, b->len); + OPENSSL_free(b->buf); + b->buf = NULL; + return 1; +} diff --git a/openssl/src/ssl/record/ssl3_record.c b/openssl/src/ssl/record/ssl3_record.c new file mode 100644 index 000000000..08d50bc6a --- /dev/null +++ b/openssl/src/ssl/record/ssl3_record.c @@ -0,0 +1,1909 @@ +/* + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../ssl_local.h" +#include +#include +#include +#include "record_local.h" +#include "internal/cryptlib.h" + +static const unsigned char ssl3_pad_1[48] = { + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36 +}; + +static const unsigned char ssl3_pad_2[48] = { + 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, + 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, + 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, + 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, + 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, + 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c +}; + +/* + * Clear the contents of an SSL3_RECORD but retain any memory allocated + */ +void SSL3_RECORD_clear(SSL3_RECORD *r, size_t num_recs) +{ + unsigned char *comp; + size_t i; + + for (i = 0; i < num_recs; i++) { + comp = r[i].comp; + + memset(&r[i], 0, sizeof(*r)); + r[i].comp = comp; + } +} + +void SSL3_RECORD_release(SSL3_RECORD *r, size_t num_recs) +{ + size_t i; + + for (i = 0; i < num_recs; i++) { + OPENSSL_free(r[i].comp); + r[i].comp = NULL; + } +} + +void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num) +{ + memcpy(r->seq_num, seq_num, SEQ_NUM_SIZE); +} + +/* + * Peeks ahead into "read_ahead" data to see if we have a whole record waiting + * for us in the buffer. + */ +static int ssl3_record_app_data_waiting(SSL *s) +{ + SSL3_BUFFER *rbuf; + size_t left, len; + unsigned char *p; + + rbuf = RECORD_LAYER_get_rbuf(&s->rlayer); + + p = SSL3_BUFFER_get_buf(rbuf); + if (p == NULL) + return 0; + + left = SSL3_BUFFER_get_left(rbuf); + + if (left < SSL3_RT_HEADER_LENGTH) + return 0; + + p += SSL3_BUFFER_get_offset(rbuf); + + /* + * We only check the type and record length, we will sanity check version + * etc later + */ + if (*p != SSL3_RT_APPLICATION_DATA) + return 0; + + p += 3; + n2s(p, len); + + if (left < SSL3_RT_HEADER_LENGTH + len) + return 0; + + return 1; +} + +int early_data_count_ok(SSL *s, size_t length, size_t overhead, int send) +{ + uint32_t max_early_data; + SSL_SESSION *sess = s->session; + + /* + * If we are a client then we always use the max_early_data from the + * session/psksession. Otherwise we go with the lowest out of the max early + * data set in the session and the configured max_early_data. + */ + if (!s->server && sess->ext.max_early_data == 0) { + if (!ossl_assert(s->psksession != NULL + && s->psksession->ext.max_early_data > 0)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + sess = s->psksession; + } + + if (!s->server) + max_early_data = sess->ext.max_early_data; + else if (s->ext.early_data != SSL_EARLY_DATA_ACCEPTED) + max_early_data = s->recv_max_early_data; + else + max_early_data = s->recv_max_early_data < sess->ext.max_early_data + ? s->recv_max_early_data : sess->ext.max_early_data; + + if (max_early_data == 0) { + SSLfatal(s, send ? SSL_AD_INTERNAL_ERROR : SSL_AD_UNEXPECTED_MESSAGE, + SSL_R_TOO_MUCH_EARLY_DATA); + return 0; + } + + /* If we are dealing with ciphertext we need to allow for the overhead */ + max_early_data += overhead; + + if (s->early_data_count + length > max_early_data) { + SSLfatal(s, send ? SSL_AD_INTERNAL_ERROR : SSL_AD_UNEXPECTED_MESSAGE, + SSL_R_TOO_MUCH_EARLY_DATA); + return 0; + } + s->early_data_count += length; + + return 1; +} + +/* + * MAX_EMPTY_RECORDS defines the number of consecutive, empty records that + * will be processed per call to ssl3_get_record. Without this limit an + * attacker could send empty records at a faster rate than we can process and + * cause ssl3_get_record to loop forever. + */ +#define MAX_EMPTY_RECORDS 32 + +#define SSL2_RT_HEADER_LENGTH 2 +/*- + * Call this to get new input records. + * It will return <= 0 if more data is needed, normally due to an error + * or non-blocking IO. + * When it finishes, |numrpipes| records have been decoded. For each record 'i': + * rr[i].type - is the type of record + * rr[i].data, - data + * rr[i].length, - number of bytes + * Multiple records will only be returned if the record types are all + * SSL3_RT_APPLICATION_DATA. The number of records returned will always be <= + * |max_pipelines| + */ +/* used only by ssl3_read_bytes */ +int ssl3_get_record(SSL *s) +{ + int enc_err, rret; + int i; + size_t more, n; + SSL3_RECORD *rr, *thisrr; + SSL3_BUFFER *rbuf; + SSL_SESSION *sess; + unsigned char *p; + unsigned char md[EVP_MAX_MD_SIZE]; + unsigned int version; + size_t mac_size = 0; + int imac_size; + size_t num_recs = 0, max_recs, j; + PACKET pkt, sslv2pkt; + int is_ktls_left; + SSL_MAC_BUF *macbufs = NULL; + int ret = -1; + + rr = RECORD_LAYER_get_rrec(&s->rlayer); + rbuf = RECORD_LAYER_get_rbuf(&s->rlayer); + is_ktls_left = (SSL3_BUFFER_get_left(rbuf) > 0); + max_recs = s->max_pipelines; + if (max_recs == 0) + max_recs = 1; + sess = s->session; + + do { + thisrr = &rr[num_recs]; + + /* check if we have the header */ + if ((RECORD_LAYER_get_rstate(&s->rlayer) != SSL_ST_READ_BODY) || + (RECORD_LAYER_get_packet_length(&s->rlayer) + < SSL3_RT_HEADER_LENGTH)) { + size_t sslv2len; + unsigned int type; + + rret = ssl3_read_n(s, SSL3_RT_HEADER_LENGTH, + SSL3_BUFFER_get_len(rbuf), 0, + num_recs == 0 ? 1 : 0, &n); + if (rret <= 0) { +#ifndef OPENSSL_NO_KTLS + if (!BIO_get_ktls_recv(s->rbio) || rret == 0) + return rret; /* error or non-blocking */ + switch (errno) { + case EBADMSG: + SSLfatal(s, SSL_AD_BAD_RECORD_MAC, + SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); + break; + case EMSGSIZE: + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, + SSL_R_PACKET_LENGTH_TOO_LONG); + break; + case EINVAL: + SSLfatal(s, SSL_AD_PROTOCOL_VERSION, + SSL_R_WRONG_VERSION_NUMBER); + break; + default: + break; + } +#endif + return rret; + } + RECORD_LAYER_set_rstate(&s->rlayer, SSL_ST_READ_BODY); + + p = RECORD_LAYER_get_packet(&s->rlayer); + +# ifndef OPENSSL_NO_STATUS + /*record client protocol*/ + if (s->status_param.ssl_status_enable) { + s->status_param.type = SSL_CLIENT_RPOTOCOL; + if (s->status_callback(p, 2, &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + return -1; + } + } +# endif + + if (!PACKET_buf_init(&pkt, RECORD_LAYER_get_packet(&s->rlayer), + RECORD_LAYER_get_packet_length(&s->rlayer))) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + sslv2pkt = pkt; + if (!PACKET_get_net_2_len(&sslv2pkt, &sslv2len) + || !PACKET_get_1(&sslv2pkt, &type)) { + SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + /* + * The first record received by the server may be a V2ClientHello. + */ + if (s->server && RECORD_LAYER_is_first_record(&s->rlayer) + && (sslv2len & 0x8000) != 0 + && (type == SSL2_MT_CLIENT_HELLO)) { + /* + * SSLv2 style record + * + * |num_recs| here will actually always be 0 because + * |num_recs > 0| only ever occurs when we are processing + * multiple app data records - which we know isn't the case here + * because it is an SSLv2ClientHello. We keep it using + * |num_recs| for the sake of consistency + */ + thisrr->type = SSL3_RT_HANDSHAKE; + thisrr->rec_version = SSL2_VERSION; + + thisrr->length = sslv2len & 0x7fff; + + if (thisrr->length > SSL3_BUFFER_get_len(rbuf) + - SSL2_RT_HEADER_LENGTH) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, + SSL_R_PACKET_LENGTH_TOO_LONG); + return -1; + } + + if (thisrr->length < MIN_SSL2_RECORD_LEN) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); + return -1; + } + } else { + /* SSLv3+ style record */ + + /* Pull apart the header into the SSL3_RECORD */ + if (!PACKET_get_1(&pkt, &type) + || !PACKET_get_net_2(&pkt, &version) + || !PACKET_get_net_2_len(&pkt, &thisrr->length)) { + if (s->msg_callback) + s->msg_callback(0, 0, SSL3_RT_HEADER, p, 5, s, + s->msg_callback_arg); + SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + thisrr->type = type; + thisrr->rec_version = version; + + if (s->msg_callback) + s->msg_callback(0, version, SSL3_RT_HEADER, p, 5, s, + s->msg_callback_arg); + + /* + * Lets check version. In TLSv1.3 we only check this field + * when encryption is occurring (see later check). For the + * ServerHello after an HRR we haven't actually selected TLSv1.3 + * yet, but we still treat it as TLSv1.3, so we must check for + * that explicitly + */ + if (!s->first_packet && !SSL_IS_TLS13(s) + && s->hello_retry_request != SSL_HRR_PENDING + && version != (unsigned int)s->version) { + if ((s->version & 0xFF00) == (version & 0xFF00) + && !s->enc_write_ctx && !s->write_hash) { + if (thisrr->type == SSL3_RT_ALERT) { + /* + * The record is using an incorrect version number, + * but what we've got appears to be an alert. We + * haven't read the body yet to check whether its a + * fatal or not - but chances are it is. We probably + * shouldn't send a fatal alert back. We'll just + * end. + */ + SSLfatal(s, SSL_AD_NO_ALERT, + SSL_R_WRONG_VERSION_NUMBER); + return -1; + } + /* + * Send back error using their minor version number :-) + */ + s->version = (unsigned short)version; + } + SSLfatal(s, SSL_AD_PROTOCOL_VERSION, + SSL_R_WRONG_VERSION_NUMBER); + return -1; + } + +#ifndef OPENSSL_NO_NTLS + if (s->enable_ntls == 1 + && version == NTLS_VERSION) { + /* do nothing */ + } else +#endif + if ((version >> 8) != SSL3_VERSION_MAJOR) { + if (RECORD_LAYER_is_first_record(&s->rlayer)) { + /* Go back to start of packet, look at the five bytes + * that we have. */ + p = RECORD_LAYER_get_packet(&s->rlayer); + if (strncmp((char *)p, "GET ", 4) == 0 || + strncmp((char *)p, "POST ", 5) == 0 || + strncmp((char *)p, "HEAD ", 5) == 0 || + strncmp((char *)p, "PUT ", 4) == 0) { + SSLfatal(s, SSL_AD_NO_ALERT, SSL_R_HTTP_REQUEST); + return -1; + } else if (strncmp((char *)p, "CONNE", 5) == 0) { + SSLfatal(s, SSL_AD_NO_ALERT, + SSL_R_HTTPS_PROXY_REQUEST); + return -1; + } + + /* Doesn't look like TLS - don't send an alert */ + SSLfatal(s, SSL_AD_NO_ALERT, + SSL_R_WRONG_VERSION_NUMBER); + return -1; + } else { + SSLfatal(s, SSL_AD_PROTOCOL_VERSION, + SSL_R_WRONG_VERSION_NUMBER); + return -1; + } + } + + if (SSL_IS_TLS13(s) && s->enc_read_ctx != NULL) { + if (thisrr->type != SSL3_RT_APPLICATION_DATA + && (thisrr->type != SSL3_RT_CHANGE_CIPHER_SPEC + || !SSL_IS_FIRST_HANDSHAKE(s)) + && (thisrr->type != SSL3_RT_ALERT + || s->statem.enc_read_state + != ENC_READ_STATE_ALLOW_PLAIN_ALERTS)) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, + SSL_R_BAD_RECORD_TYPE); + return -1; + } + if (thisrr->rec_version != TLS1_2_VERSION) { + SSLfatal(s, SSL_AD_DECODE_ERROR, + SSL_R_WRONG_VERSION_NUMBER); + return -1; + } + } + + if (thisrr->length > + SSL3_BUFFER_get_len(rbuf) - SSL3_RT_HEADER_LENGTH) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, + SSL_R_PACKET_LENGTH_TOO_LONG); + return -1; + } + } + + /* now s->rlayer.rstate == SSL_ST_READ_BODY */ + } + + if (SSL_IS_TLS13(s)) { + if (thisrr->length > SSL3_RT_MAX_TLS13_ENCRYPTED_LENGTH) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, + SSL_R_ENCRYPTED_LENGTH_TOO_LONG); + return -1; + } + } else { + size_t len = SSL3_RT_MAX_ENCRYPTED_LENGTH; + +#ifndef OPENSSL_NO_COMP + /* + * If OPENSSL_NO_COMP is defined then SSL3_RT_MAX_ENCRYPTED_LENGTH + * does not include the compression overhead anyway. + */ + if (s->expand == NULL) + len -= SSL3_RT_MAX_COMPRESSED_OVERHEAD; +#endif + + /* KTLS may use all of the buffer */ + if (BIO_get_ktls_recv(s->rbio) && !is_ktls_left) + len = SSL3_BUFFER_get_left(rbuf); + + if (thisrr->length > len) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, + SSL_R_ENCRYPTED_LENGTH_TOO_LONG); + return -1; + } + } + + /* + * s->rlayer.rstate == SSL_ST_READ_BODY, get and decode the data. + * Calculate how much more data we need to read for the rest of the + * record + */ + if (thisrr->rec_version == SSL2_VERSION) { + more = thisrr->length + SSL2_RT_HEADER_LENGTH + - SSL3_RT_HEADER_LENGTH; + } else { + more = thisrr->length; + } + + if (more > 0) { + /* now s->rlayer.packet_length == SSL3_RT_HEADER_LENGTH */ + + rret = ssl3_read_n(s, more, more, 1, 0, &n); + if (rret <= 0) + return rret; /* error or non-blocking io */ + } + + /* set state for later operations */ + RECORD_LAYER_set_rstate(&s->rlayer, SSL_ST_READ_HEADER); + + /* + * At this point, s->rlayer.packet_length == SSL3_RT_HEADER_LENGTH + * + thisrr->length, or s->rlayer.packet_length == SSL2_RT_HEADER_LENGTH + * + thisrr->length and we have that many bytes in s->rlayer.packet + */ + if (thisrr->rec_version == SSL2_VERSION) { + thisrr->input = + &(RECORD_LAYER_get_packet(&s->rlayer)[SSL2_RT_HEADER_LENGTH]); + } else { + thisrr->input = + &(RECORD_LAYER_get_packet(&s->rlayer)[SSL3_RT_HEADER_LENGTH]); + } + + /* + * ok, we can now read from 's->rlayer.packet' data into 'thisrr'. + * thisrr->input points at thisrr->length bytes, which need to be copied + * into thisrr->data by either the decryption or by the decompression. + * When the data is 'copied' into the thisrr->data buffer, + * thisrr->input will be updated to point at the new buffer + */ + + /* + * We now have - encrypted [ MAC [ compressed [ plain ] ] ] + * thisrr->length bytes of encrypted compressed stuff. + */ + + /* decrypt in place in 'thisrr->input' */ + thisrr->data = thisrr->input; + thisrr->orig_len = thisrr->length; + + /* Mark this record as not read by upper layers yet */ + thisrr->read = 0; + + num_recs++; + + /* we have pulled in a full packet so zero things */ + RECORD_LAYER_reset_packet_length(&s->rlayer); + RECORD_LAYER_clear_first_record(&s->rlayer); + } while (num_recs < max_recs + && thisrr->type == SSL3_RT_APPLICATION_DATA + && SSL_USE_EXPLICIT_IV(s) + && s->enc_read_ctx != NULL + && (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(s->enc_read_ctx)) + & EVP_CIPH_FLAG_PIPELINE) != 0 + && ssl3_record_app_data_waiting(s)); + + if (num_recs == 1 + && thisrr->type == SSL3_RT_CHANGE_CIPHER_SPEC + && (SSL_IS_TLS13(s) || s->hello_retry_request != SSL_HRR_NONE) + && SSL_IS_FIRST_HANDSHAKE(s)) { + /* + * CCS messages must be exactly 1 byte long, containing the value 0x01 + */ + if (thisrr->length != 1 || thisrr->data[0] != 0x01) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_INVALID_CCS_MESSAGE); + return -1; + } + /* + * CCS messages are ignored in TLSv1.3. We treat it like an empty + * handshake record + */ + thisrr->type = SSL3_RT_HANDSHAKE; + RECORD_LAYER_inc_empty_record_count(&s->rlayer); + if (RECORD_LAYER_get_empty_record_count(&s->rlayer) + > MAX_EMPTY_RECORDS) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, + SSL_R_UNEXPECTED_CCS_MESSAGE); + return -1; + } + thisrr->read = 1; + RECORD_LAYER_set_numrpipes(&s->rlayer, 1); + + return 1; + } + + /* + * KTLS reads full records. If there is any data left, + * then it is from before enabling ktls + */ + if (BIO_get_ktls_recv(s->rbio) && !is_ktls_left) + goto skip_decryption; + + if (s->read_hash != NULL) { + const EVP_MD *tmpmd = EVP_MD_CTX_get0_md(s->read_hash); + + if (tmpmd != NULL) { + imac_size = EVP_MD_get_size(tmpmd); + if (!ossl_assert(imac_size >= 0 && imac_size <= EVP_MAX_MD_SIZE)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + return -1; + } + mac_size = (size_t)imac_size; + } + } + + /* + * If in encrypt-then-mac mode calculate mac from encrypted record. All + * the details below are public so no timing details can leak. + */ + if (SSL_READ_ETM(s) && s->read_hash) { + unsigned char *mac; + + for (j = 0; j < num_recs; j++) { + thisrr = &rr[j]; + + if (thisrr->length < mac_size) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); + return -1; + } + thisrr->length -= mac_size; + mac = thisrr->data + thisrr->length; + i = s->method->ssl3_enc->mac(s, thisrr, md, 0 /* not send */ ); + if (i == 0 || CRYPTO_memcmp(md, mac, mac_size) != 0) { + SSLfatal(s, SSL_AD_BAD_RECORD_MAC, + SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); + return -1; + } + } + /* + * We've handled the mac now - there is no MAC inside the encrypted + * record + */ + mac_size = 0; + } + + if (mac_size > 0) { + macbufs = OPENSSL_zalloc(sizeof(*macbufs) * num_recs); + if (macbufs == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return -1; + } + } + + enc_err = s->method->ssl3_enc->enc(s, rr, num_recs, 0, macbufs, mac_size); + + /*- + * enc_err is: + * 0: if the record is publicly invalid, or an internal error, or AEAD + * decryption failed, or ETM decryption failed. + * 1: Success or MTE decryption failed (MAC will be randomised) + */ + if (enc_err == 0) { + if (ossl_statem_in_error(s)) { + /* SSLfatal() already got called */ + goto end; + } + if (num_recs == 1 && ossl_statem_skip_early_data(s)) { + /* + * Valid early_data that we cannot decrypt will fail here. We treat + * it like an empty record. + */ + + thisrr = &rr[0]; + + if (!early_data_count_ok(s, thisrr->length, + EARLY_DATA_CIPHERTEXT_OVERHEAD, 0)) { + /* SSLfatal() already called */ + goto end; + } + + thisrr->length = 0; + thisrr->read = 1; + RECORD_LAYER_set_numrpipes(&s->rlayer, 1); + RECORD_LAYER_reset_read_sequence(&s->rlayer); + ret = 1; + goto end; + } + SSLfatal(s, SSL_AD_BAD_RECORD_MAC, + SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); + goto end; + } + OSSL_TRACE_BEGIN(TLS) { + BIO_printf(trc_out, "dec %lu\n", (unsigned long)rr[0].length); + BIO_dump_indent(trc_out, rr[0].data, rr[0].length, 4); + } OSSL_TRACE_END(TLS); + + /* r->length is now the compressed data plus mac */ + if ((sess != NULL) + && (s->enc_read_ctx != NULL) + && (!SSL_READ_ETM(s) && EVP_MD_CTX_get0_md(s->read_hash) != NULL)) { + /* s->read_hash != NULL => mac_size != -1 */ + + for (j = 0; j < num_recs; j++) { + SSL_MAC_BUF *thismb = &macbufs[j]; + thisrr = &rr[j]; + + i = s->method->ssl3_enc->mac(s, thisrr, md, 0 /* not send */ ); + if (i == 0 || thismb == NULL || thismb->mac == NULL + || CRYPTO_memcmp(md, thismb->mac, (size_t)mac_size) != 0) + enc_err = 0; + if (thisrr->length > SSL3_RT_MAX_COMPRESSED_LENGTH + mac_size) + enc_err = 0; + } + } + + if (enc_err == 0) { + if (ossl_statem_in_error(s)) { + /* We already called SSLfatal() */ + goto end; + } + /* + * A separate 'decryption_failed' alert was introduced with TLS 1.0, + * SSL 3.0 only has 'bad_record_mac'. But unless a decryption + * failure is directly visible from the ciphertext anyway, we should + * not reveal which kind of error occurred -- this might become + * visible to an attacker (e.g. via a logfile) + */ + SSLfatal(s, SSL_AD_BAD_RECORD_MAC, + SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); + goto end; + } + + skip_decryption: + + for (j = 0; j < num_recs; j++) { + thisrr = &rr[j]; + + /* thisrr->length is now just compressed */ + if (s->expand != NULL) { + if (thisrr->length > SSL3_RT_MAX_COMPRESSED_LENGTH) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, + SSL_R_COMPRESSED_LENGTH_TOO_LONG); + goto end; + } + if (!ssl3_do_uncompress(s, thisrr)) { + SSLfatal(s, SSL_AD_DECOMPRESSION_FAILURE, + SSL_R_BAD_DECOMPRESSION); + goto end; + } + } + + if (SSL_IS_TLS13(s) + && s->enc_read_ctx != NULL + && thisrr->type != SSL3_RT_ALERT) { + size_t end; + + if (thisrr->length == 0 + || thisrr->type != SSL3_RT_APPLICATION_DATA) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_BAD_RECORD_TYPE); + goto end; + } + + /* Strip trailing padding */ + for (end = thisrr->length - 1; end > 0 && thisrr->data[end] == 0; + end--) + continue; + + thisrr->length = end; + thisrr->type = thisrr->data[end]; + if (thisrr->type != SSL3_RT_APPLICATION_DATA + && thisrr->type != SSL3_RT_ALERT + && thisrr->type != SSL3_RT_HANDSHAKE) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_BAD_RECORD_TYPE); + goto end; + } + if (s->msg_callback) + s->msg_callback(0, s->version, SSL3_RT_INNER_CONTENT_TYPE, + &thisrr->data[end], 1, s, s->msg_callback_arg); + } + + /* + * TLSv1.3 alert and handshake records are required to be non-zero in + * length. + */ + if (SSL_IS_TLS13(s) + && (thisrr->type == SSL3_RT_HANDSHAKE + || thisrr->type == SSL3_RT_ALERT) + && thisrr->length == 0) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_BAD_LENGTH); + goto end; + } + + /* + * Usually thisrr->length is the length of a single record, but when + * KTLS handles the decryption, thisrr->length may be larger than + * SSL3_RT_MAX_PLAIN_LENGTH because the kernel may have coalesced + * multiple records. + * Therefore we have to rely on KTLS to check the plaintext length + * limit in the kernel. + */ + if (thisrr->length > SSL3_RT_MAX_PLAIN_LENGTH + && (!BIO_get_ktls_recv(s->rbio) || is_ktls_left)) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, SSL_R_DATA_LENGTH_TOO_LONG); + goto end; + } + + /* + * Check if the received packet overflows the current + * Max Fragment Length setting. + * Note: USE_MAX_FRAGMENT_LENGTH_EXT and KTLS are mutually exclusive. + */ + if (s->session != NULL && USE_MAX_FRAGMENT_LENGTH_EXT(s->session) + && thisrr->length > GET_MAX_FRAGMENT_LENGTH(s->session)) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, SSL_R_DATA_LENGTH_TOO_LONG); + goto end; + } + + thisrr->off = 0; + /*- + * So at this point the following is true + * thisrr->type is the type of record + * thisrr->length == number of bytes in record + * thisrr->off == offset to first valid byte + * thisrr->data == where to take bytes from, increment after use :-). + */ + + /* just read a 0 length packet */ + if (thisrr->length == 0) { + RECORD_LAYER_inc_empty_record_count(&s->rlayer); + if (RECORD_LAYER_get_empty_record_count(&s->rlayer) + > MAX_EMPTY_RECORDS) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_RECORD_TOO_SMALL); + goto end; + } + } else { + RECORD_LAYER_reset_empty_record_count(&s->rlayer); + } + } + + if (s->early_data_state == SSL_EARLY_DATA_READING) { + thisrr = &rr[0]; + if (thisrr->type == SSL3_RT_APPLICATION_DATA + && !early_data_count_ok(s, thisrr->length, 0, 0)) { + /* SSLfatal already called */ + goto end; + } + } + + RECORD_LAYER_set_numrpipes(&s->rlayer, num_recs); + ret = 1; + end: + if (macbufs != NULL) { + for (j = 0; j < num_recs; j++) { + if (macbufs[j].alloced) + OPENSSL_free(macbufs[j].mac); + } + OPENSSL_free(macbufs); + } + return ret; +} + +int ssl3_do_uncompress(SSL *ssl, SSL3_RECORD *rr) +{ +#ifndef OPENSSL_NO_COMP + int i; + + if (rr->comp == NULL) { + rr->comp = (unsigned char *) + OPENSSL_malloc(SSL3_RT_MAX_ENCRYPTED_LENGTH); + } + if (rr->comp == NULL) + return 0; + + i = COMP_expand_block(ssl->expand, rr->comp, + SSL3_RT_MAX_PLAIN_LENGTH, rr->data, (int)rr->length); + if (i < 0) + return 0; + else + rr->length = i; + rr->data = rr->comp; +#endif + return 1; +} + +int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr) +{ +#ifndef OPENSSL_NO_COMP + int i; + + i = COMP_compress_block(ssl->compress, wr->data, + (int)(wr->length + SSL3_RT_MAX_COMPRESSED_OVERHEAD), + wr->input, (int)wr->length); + if (i < 0) + return 0; + else + wr->length = i; + + wr->input = wr->data; +#endif + return 1; +} + +/*- + * ssl3_enc encrypts/decrypts |n_recs| records in |inrecs|. Calls SSLfatal on + * internal error, but not otherwise. It is the responsibility of the caller to + * report a bad_record_mac + * + * Returns: + * 0: if the record is publicly invalid, or an internal error + * 1: Success or Mac-then-encrypt decryption failed (MAC will be randomised) + */ +int ssl3_enc(SSL *s, SSL3_RECORD *inrecs, size_t n_recs, int sending, + SSL_MAC_BUF *mac, size_t macsize) +{ + SSL3_RECORD *rec; + EVP_CIPHER_CTX *ds; + size_t l, i; + size_t bs; + const EVP_CIPHER *enc; + + rec = inrecs; + /* + * We shouldn't ever be called with more than one record in the SSLv3 case + */ + if (n_recs != 1) + return 0; + if (sending) { + ds = s->enc_write_ctx; + if (s->enc_write_ctx == NULL) + enc = NULL; + else + enc = EVP_CIPHER_CTX_get0_cipher(s->enc_write_ctx); + } else { + ds = s->enc_read_ctx; + if (s->enc_read_ctx == NULL) + enc = NULL; + else + enc = EVP_CIPHER_CTX_get0_cipher(s->enc_read_ctx); + } + + if ((s->session == NULL) || (ds == NULL) || (enc == NULL)) { + memmove(rec->data, rec->input, rec->length); + rec->input = rec->data; + } else { + int provided = (EVP_CIPHER_get0_provider(enc) != NULL); + + l = rec->length; + bs = EVP_CIPHER_CTX_get_block_size(ds); + + /* COMPRESS */ + + if ((bs != 1) && sending && !provided) { + /* + * We only do this for legacy ciphers. Provided ciphers add the + * padding on the provider side. + */ + i = bs - (l % bs); + + /* we need to add 'i-1' padding bytes */ + l += i; + /* + * the last of these zero bytes will be overwritten with the + * padding length. + */ + memset(&rec->input[rec->length], 0, i); + rec->length += i; + rec->input[l - 1] = (unsigned char)(i - 1); + } + + if (!sending) { + if (l == 0 || l % bs != 0) { + /* Publicly invalid */ + return 0; + } + /* otherwise, rec->length >= bs */ + } + + if (EVP_CIPHER_get0_provider(enc) != NULL) { + int outlen; + + if (!EVP_CipherUpdate(ds, rec->data, &outlen, rec->input, + (unsigned int)l)) + return 0; + rec->length = outlen; + + if (!sending && mac != NULL) { + /* Now get a pointer to the MAC */ + OSSL_PARAM params[2], *p = params; + + /* Get the MAC */ + mac->alloced = 0; + + *p++ = OSSL_PARAM_construct_octet_ptr(OSSL_CIPHER_PARAM_TLS_MAC, + (void **)&mac->mac, + macsize); + *p = OSSL_PARAM_construct_end(); + + if (!EVP_CIPHER_CTX_get_params(ds, params)) { + /* Shouldn't normally happen */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + } else { + if (EVP_Cipher(ds, rec->data, rec->input, (unsigned int)l) < 1) { + /* Shouldn't happen */ + SSLfatal(s, SSL_AD_BAD_RECORD_MAC, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!sending) + return ssl3_cbc_remove_padding_and_mac(&rec->length, + rec->orig_len, + rec->data, + (mac != NULL) ? &mac->mac : NULL, + (mac != NULL) ? &mac->alloced : NULL, + bs, + macsize, + s->ctx->libctx); + } + } + return 1; +} + +#define MAX_PADDING 256 +/*- + * tls1_enc encrypts/decrypts |n_recs| in |recs|. Calls SSLfatal on internal + * error, but not otherwise. It is the responsibility of the caller to report + * a bad_record_mac - if appropriate (DTLS just drops the record). + * + * Returns: + * 0: if the record is publicly invalid, or an internal error, or AEAD + * decryption failed, or Encrypt-then-mac decryption failed. + * 1: Success or Mac-then-encrypt decryption failed (MAC will be randomised) + */ +int tls1_enc(SSL *s, SSL3_RECORD *recs, size_t n_recs, int sending, + SSL_MAC_BUF *macs, size_t macsize) +{ + EVP_CIPHER_CTX *ds; + size_t reclen[SSL_MAX_PIPELINES]; + unsigned char buf[SSL_MAX_PIPELINES][EVP_AEAD_TLS1_AAD_LEN]; + int i, pad = 0, tmpr; + size_t bs, ctr, padnum, loop; + unsigned char padval; + const EVP_CIPHER *enc; + + if (n_recs == 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (sending) { + if (EVP_MD_CTX_get0_md(s->write_hash)) { + int n = EVP_MD_CTX_get_size(s->write_hash); + if (!ossl_assert(n >= 0)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + ds = s->enc_write_ctx; + if (s->enc_write_ctx == NULL) + enc = NULL; + else { + int ivlen; + + enc = EVP_CIPHER_CTX_get0_cipher(s->enc_write_ctx); + /* For TLSv1.1 and later explicit IV */ + if (SSL_USE_EXPLICIT_IV(s) + && EVP_CIPHER_get_mode(enc) == EVP_CIPH_CBC_MODE) + ivlen = EVP_CIPHER_get_iv_length(enc); + else + ivlen = 0; + if (ivlen > 1) { + for (ctr = 0; ctr < n_recs; ctr++) { + if (recs[ctr].data != recs[ctr].input) { + /* + * we can't write into the input stream: Can this ever + * happen?? (steve) + */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } else if (RAND_bytes_ex(s->ctx->libctx, recs[ctr].input, + ivlen, 0) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + } + } + } else { + if (EVP_MD_CTX_get0_md(s->read_hash)) { + int n = EVP_MD_CTX_get_size(s->read_hash); + if (!ossl_assert(n >= 0)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + ds = s->enc_read_ctx; + if (s->enc_read_ctx == NULL) + enc = NULL; + else + enc = EVP_CIPHER_CTX_get0_cipher(s->enc_read_ctx); + } + + if ((s->session == NULL) || (ds == NULL) || (enc == NULL)) { + for (ctr = 0; ctr < n_recs; ctr++) { + memmove(recs[ctr].data, recs[ctr].input, recs[ctr].length); + recs[ctr].input = recs[ctr].data; + } + } else { + int provided = (EVP_CIPHER_get0_provider(enc) != NULL); + + bs = EVP_CIPHER_get_block_size(EVP_CIPHER_CTX_get0_cipher(ds)); + + if (n_recs > 1) { + if ((EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(ds)) + & EVP_CIPH_FLAG_PIPELINE) == 0) { + /* + * We shouldn't have been called with pipeline data if the + * cipher doesn't support pipelining + */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_PIPELINE_FAILURE); + return 0; + } + } + for (ctr = 0; ctr < n_recs; ctr++) { + reclen[ctr] = recs[ctr].length; + + if ((EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(ds)) + & EVP_CIPH_FLAG_AEAD_CIPHER) != 0) { + unsigned char *seq; + + seq = sending ? RECORD_LAYER_get_write_sequence(&s->rlayer) + : RECORD_LAYER_get_read_sequence(&s->rlayer); + + if (SSL_IS_DTLS(s)) { + /* DTLS does not support pipelining */ + unsigned char dtlsseq[8], *p = dtlsseq; + + s2n(sending ? DTLS_RECORD_LAYER_get_w_epoch(&s->rlayer) : + DTLS_RECORD_LAYER_get_r_epoch(&s->rlayer), p); + memcpy(p, &seq[2], 6); + memcpy(buf[ctr], dtlsseq, 8); + } else { + memcpy(buf[ctr], seq, 8); + for (i = 7; i >= 0; i--) { /* increment */ + ++seq[i]; + if (seq[i] != 0) + break; + } + } + + buf[ctr][8] = recs[ctr].type; + buf[ctr][9] = (unsigned char)(s->version >> 8); + buf[ctr][10] = (unsigned char)(s->version); + buf[ctr][11] = (unsigned char)(recs[ctr].length >> 8); + buf[ctr][12] = (unsigned char)(recs[ctr].length & 0xff); + pad = EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_AEAD_TLS1_AAD, + EVP_AEAD_TLS1_AAD_LEN, buf[ctr]); + if (pad <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (sending) { + reclen[ctr] += pad; + recs[ctr].length += pad; + } + + } else if ((bs != 1) && sending && !provided) { + /* + * We only do this for legacy ciphers. Provided ciphers add the + * padding on the provider side. + */ + padnum = bs - (reclen[ctr] % bs); + + /* Add weird padding of up to 256 bytes */ + + if (padnum > MAX_PADDING) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + /* we need to add 'padnum' padding bytes of value padval */ + padval = (unsigned char)(padnum - 1); + for (loop = reclen[ctr]; loop < reclen[ctr] + padnum; loop++) + recs[ctr].input[loop] = padval; + reclen[ctr] += padnum; + recs[ctr].length += padnum; + } + + if (!sending) { + if (reclen[ctr] == 0 || reclen[ctr] % bs != 0) { + /* Publicly invalid */ + return 0; + } + } + } + if (n_recs > 1) { + unsigned char *data[SSL_MAX_PIPELINES]; + + /* Set the output buffers */ + for (ctr = 0; ctr < n_recs; ctr++) { + data[ctr] = recs[ctr].data; + } + if (EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_SET_PIPELINE_OUTPUT_BUFS, + (int)n_recs, data) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_PIPELINE_FAILURE); + return 0; + } + /* Set the input buffers */ + for (ctr = 0; ctr < n_recs; ctr++) { + data[ctr] = recs[ctr].input; + } + if (EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_SET_PIPELINE_INPUT_BUFS, + (int)n_recs, data) <= 0 + || EVP_CIPHER_CTX_ctrl(ds, EVP_CTRL_SET_PIPELINE_INPUT_LENS, + (int)n_recs, reclen) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_PIPELINE_FAILURE); + return 0; + } + } + + if (provided) { + int outlen; + + /* Provided cipher - we do not support pipelining on this path */ + if (n_recs > 1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!EVP_CipherUpdate(ds, recs[0].data, &outlen, recs[0].input, + (unsigned int)reclen[0])) + return 0; + recs[0].length = outlen; + + /* + * The length returned from EVP_CipherUpdate above is the actual + * payload length. We need to adjust the data/input ptr to skip over + * any explicit IV + */ + if (!sending) { + if (EVP_CIPHER_get_mode(enc) == EVP_CIPH_GCM_MODE) { + recs[0].data += EVP_GCM_TLS_EXPLICIT_IV_LEN; + recs[0].input += EVP_GCM_TLS_EXPLICIT_IV_LEN; + } else if (EVP_CIPHER_get_mode(enc) == EVP_CIPH_CCM_MODE) { + recs[0].data += EVP_CCM_TLS_EXPLICIT_IV_LEN; + recs[0].input += EVP_CCM_TLS_EXPLICIT_IV_LEN; + } else if (bs != 1 && SSL_USE_EXPLICIT_IV(s)) { + recs[0].data += bs; + recs[0].input += bs; + recs[0].orig_len -= bs; + } + + /* Now get a pointer to the MAC (if applicable) */ + if (macs != NULL) { + OSSL_PARAM params[2], *p = params; + + /* Get the MAC */ + macs[0].alloced = 0; + + *p++ = OSSL_PARAM_construct_octet_ptr(OSSL_CIPHER_PARAM_TLS_MAC, + (void **)&macs[0].mac, + macsize); + *p = OSSL_PARAM_construct_end(); + + if (!EVP_CIPHER_CTX_get_params(ds, params)) { + /* Shouldn't normally happen */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, + ERR_R_INTERNAL_ERROR); + return 0; + } + } + } + } else { + /* Legacy cipher */ + + tmpr = EVP_Cipher(ds, recs[0].data, recs[0].input, + (unsigned int)reclen[0]); + if ((EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(ds)) + & EVP_CIPH_FLAG_CUSTOM_CIPHER) != 0 + ? (tmpr < 0) + : (tmpr == 0)) { + /* AEAD can fail to verify MAC */ + return 0; + } + + if (!sending) { + for (ctr = 0; ctr < n_recs; ctr++) { + /* Adjust the record to remove the explicit IV/MAC/Tag */ + if (EVP_CIPHER_get_mode(enc) == EVP_CIPH_GCM_MODE) { + recs[ctr].data += EVP_GCM_TLS_EXPLICIT_IV_LEN; + recs[ctr].input += EVP_GCM_TLS_EXPLICIT_IV_LEN; + recs[ctr].length -= EVP_GCM_TLS_EXPLICIT_IV_LEN; + } else if (EVP_CIPHER_get_mode(enc) == EVP_CIPH_CCM_MODE) { + recs[ctr].data += EVP_CCM_TLS_EXPLICIT_IV_LEN; + recs[ctr].input += EVP_CCM_TLS_EXPLICIT_IV_LEN; + recs[ctr].length -= EVP_CCM_TLS_EXPLICIT_IV_LEN; + } else if (bs != 1 && SSL_USE_EXPLICIT_IV(s)) { + if (recs[ctr].length < bs) + return 0; + recs[ctr].data += bs; + recs[ctr].input += bs; + recs[ctr].length -= bs; + recs[ctr].orig_len -= bs; + } + + /* + * If using Mac-then-encrypt, then this will succeed but + * with a random MAC if padding is invalid + */ + if (!tls1_cbc_remove_padding_and_mac(&recs[ctr].length, + recs[ctr].orig_len, + recs[ctr].data, + (macs != NULL) ? &macs[ctr].mac : NULL, + (macs != NULL) ? &macs[ctr].alloced + : NULL, + bs, + pad ? (size_t)pad : macsize, + (EVP_CIPHER_get_flags(enc) + & EVP_CIPH_FLAG_AEAD_CIPHER) != 0, + s->ctx->libctx)) + return 0; + } + } + } + } + return 1; +} + +/* + * ssl3_cbc_record_digest_supported returns 1 iff |ctx| uses a hash function + * which ssl3_cbc_digest_record supports. + */ +char ssl3_cbc_record_digest_supported(const EVP_MD_CTX *ctx) +{ + switch (EVP_MD_CTX_get_type(ctx)) { + case NID_md5: + case NID_sha1: + case NID_sha224: + case NID_sha256: + case NID_sha384: + case NID_sha512: +#ifndef OPENSSL_NO_SM3 + case NID_sm3: +#endif + return 1; + default: + return 0; + } +} + +int n_ssl3_mac(SSL *ssl, SSL3_RECORD *rec, unsigned char *md, int sending) +{ + unsigned char *mac_sec, *seq; + const EVP_MD_CTX *hash; + unsigned char *p, rec_char; + size_t md_size; + size_t npad; + int t; + + if (sending) { + mac_sec = &(ssl->s3.write_mac_secret[0]); + seq = RECORD_LAYER_get_write_sequence(&ssl->rlayer); + hash = ssl->write_hash; + } else { + mac_sec = &(ssl->s3.read_mac_secret[0]); + seq = RECORD_LAYER_get_read_sequence(&ssl->rlayer); + hash = ssl->read_hash; + } + + t = EVP_MD_CTX_get_size(hash); + if (t < 0) + return 0; + md_size = t; + npad = (48 / md_size) * md_size; + + if (!sending + && EVP_CIPHER_CTX_get_mode(ssl->enc_read_ctx) == EVP_CIPH_CBC_MODE + && ssl3_cbc_record_digest_supported(hash)) { +#ifdef OPENSSL_NO_DEPRECATED_3_0 + return 0; +#else + /* + * This is a CBC-encrypted record. We must avoid leaking any + * timing-side channel information about how many blocks of data we + * are hashing because that gives an attacker a timing-oracle. + */ + + /*- + * npad is, at most, 48 bytes and that's with MD5: + * 16 + 48 + 8 (sequence bytes) + 1 + 2 = 75. + * + * With SHA-1 (the largest hash speced for SSLv3) the hash size + * goes up 4, but npad goes down by 8, resulting in a smaller + * total size. + */ + unsigned char header[75]; + size_t j = 0; + memcpy(header + j, mac_sec, md_size); + j += md_size; + memcpy(header + j, ssl3_pad_1, npad); + j += npad; + memcpy(header + j, seq, 8); + j += 8; + header[j++] = rec->type; + header[j++] = (unsigned char)(rec->length >> 8); + header[j++] = (unsigned char)(rec->length & 0xff); + + /* Final param == is SSLv3 */ + if (ssl3_cbc_digest_record(EVP_MD_CTX_get0_md(hash), + md, &md_size, + header, rec->input, + rec->length, rec->orig_len, + mac_sec, md_size, 1) <= 0) + return 0; +#endif + } else { + unsigned int md_size_u; + /* Chop the digest off the end :-) */ + EVP_MD_CTX *md_ctx = EVP_MD_CTX_new(); + + if (md_ctx == NULL) + return 0; + + rec_char = rec->type; + p = md; + s2n(rec->length, p); + if (EVP_MD_CTX_copy_ex(md_ctx, hash) <= 0 + || EVP_DigestUpdate(md_ctx, mac_sec, md_size) <= 0 + || EVP_DigestUpdate(md_ctx, ssl3_pad_1, npad) <= 0 + || EVP_DigestUpdate(md_ctx, seq, 8) <= 0 + || EVP_DigestUpdate(md_ctx, &rec_char, 1) <= 0 + || EVP_DigestUpdate(md_ctx, md, 2) <= 0 + || EVP_DigestUpdate(md_ctx, rec->input, rec->length) <= 0 + || EVP_DigestFinal_ex(md_ctx, md, NULL) <= 0 + || EVP_MD_CTX_copy_ex(md_ctx, hash) <= 0 + || EVP_DigestUpdate(md_ctx, mac_sec, md_size) <= 0 + || EVP_DigestUpdate(md_ctx, ssl3_pad_2, npad) <= 0 + || EVP_DigestUpdate(md_ctx, md, md_size) <= 0 + || EVP_DigestFinal_ex(md_ctx, md, &md_size_u) <= 0) { + EVP_MD_CTX_free(md_ctx); + return 0; + } + + EVP_MD_CTX_free(md_ctx); + } + + ssl3_record_sequence_update(seq); + return 1; +} + +int tls1_mac(SSL *ssl, SSL3_RECORD *rec, unsigned char *md, int sending) +{ + unsigned char *seq; + EVP_MD_CTX *hash; + size_t md_size; + int i; + EVP_MD_CTX *hmac = NULL, *mac_ctx; + unsigned char header[13]; + int t; + int ret = 0; + + if (sending) { + seq = RECORD_LAYER_get_write_sequence(&ssl->rlayer); + hash = ssl->write_hash; + } else { + seq = RECORD_LAYER_get_read_sequence(&ssl->rlayer); + hash = ssl->read_hash; + } + + t = EVP_MD_CTX_get_size(hash); + if (!ossl_assert(t >= 0)) + return 0; + md_size = t; + + hmac = EVP_MD_CTX_new(); + if (hmac == NULL || !EVP_MD_CTX_copy(hmac, hash)) { + goto end; + } + mac_ctx = hmac; + + if (SSL_IS_DTLS(ssl)) { + unsigned char dtlsseq[8], *p = dtlsseq; + + s2n(sending ? DTLS_RECORD_LAYER_get_w_epoch(&ssl->rlayer) : + DTLS_RECORD_LAYER_get_r_epoch(&ssl->rlayer), p); + memcpy(p, &seq[2], 6); + + memcpy(header, dtlsseq, 8); + } else + memcpy(header, seq, 8); + + header[8] = rec->type; + header[9] = (unsigned char)(ssl->version >> 8); + header[10] = (unsigned char)(ssl->version); + header[11] = (unsigned char)(rec->length >> 8); + header[12] = (unsigned char)(rec->length & 0xff); + + if (!sending && !SSL_READ_ETM(ssl) + && EVP_CIPHER_CTX_get_mode(ssl->enc_read_ctx) == EVP_CIPH_CBC_MODE + && ssl3_cbc_record_digest_supported(mac_ctx)) { + OSSL_PARAM tls_hmac_params[2], *p = tls_hmac_params; + + *p++ = OSSL_PARAM_construct_size_t(OSSL_MAC_PARAM_TLS_DATA_SIZE, + &rec->orig_len); + *p++ = OSSL_PARAM_construct_end(); + + if (!EVP_PKEY_CTX_set_params(EVP_MD_CTX_get_pkey_ctx(mac_ctx), + tls_hmac_params)) { + goto end; + } + } + + if (EVP_DigestSignUpdate(mac_ctx, header, sizeof(header)) <= 0 + || EVP_DigestSignUpdate(mac_ctx, rec->input, rec->length) <= 0 + || EVP_DigestSignFinal(mac_ctx, md, &md_size) <= 0) { + goto end; + } + + OSSL_TRACE_BEGIN(TLS) { + BIO_printf(trc_out, "seq:\n"); + BIO_dump_indent(trc_out, seq, 8, 4); + BIO_printf(trc_out, "rec:\n"); + BIO_dump_indent(trc_out, rec->data, rec->length, 4); + } OSSL_TRACE_END(TLS); + + if (!SSL_IS_DTLS(ssl)) { + for (i = 7; i >= 0; i--) { + ++seq[i]; + if (seq[i] != 0) + break; + } + } + OSSL_TRACE_BEGIN(TLS) { + BIO_printf(trc_out, "md:\n"); + BIO_dump_indent(trc_out, md, md_size, 4); + } OSSL_TRACE_END(TLS); + ret = 1; + end: + EVP_MD_CTX_free(hmac); + return ret; +} + +int dtls1_process_record(SSL *s, DTLS1_BITMAP *bitmap) +{ + int i; + int enc_err; + SSL_SESSION *sess; + SSL3_RECORD *rr; + int imac_size; + size_t mac_size = 0; + unsigned char md[EVP_MAX_MD_SIZE]; + size_t max_plain_length = SSL3_RT_MAX_PLAIN_LENGTH; + SSL_MAC_BUF macbuf = { NULL, 0 }; + int ret = 0; + + rr = RECORD_LAYER_get_rrec(&s->rlayer); + sess = s->session; + + /* + * At this point, s->rlayer.packet_length == SSL3_RT_HEADER_LNGTH + rr->length, + * and we have that many bytes in s->rlayer.packet + */ + rr->input = &(RECORD_LAYER_get_packet(&s->rlayer)[DTLS1_RT_HEADER_LENGTH]); + + /* + * ok, we can now read from 's->rlayer.packet' data into 'rr'. rr->input + * points at rr->length bytes, which need to be copied into rr->data by + * either the decryption or by the decompression. When the data is 'copied' + * into the rr->data buffer, rr->input will be pointed at the new buffer + */ + + /* + * We now have - encrypted [ MAC [ compressed [ plain ] ] ] rr->length + * bytes of encrypted compressed stuff. + */ + + /* check is not needed I believe */ + if (rr->length > SSL3_RT_MAX_ENCRYPTED_LENGTH) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, SSL_R_ENCRYPTED_LENGTH_TOO_LONG); + return 0; + } + + /* decrypt in place in 'rr->input' */ + rr->data = rr->input; + rr->orig_len = rr->length; + + if (s->read_hash != NULL) { + const EVP_MD *tmpmd = EVP_MD_CTX_get0_md(s->read_hash); + + if (tmpmd != NULL) { + imac_size = EVP_MD_get_size(tmpmd); + if (!ossl_assert(imac_size >= 0 && imac_size <= EVP_MAX_MD_SIZE)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + return -1; + } + mac_size = (size_t)imac_size; + } + } + + if (SSL_READ_ETM(s) && s->read_hash) { + unsigned char *mac; + + if (rr->orig_len < mac_size) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); + return 0; + } + rr->length -= mac_size; + mac = rr->data + rr->length; + i = s->method->ssl3_enc->mac(s, rr, md, 0 /* not send */ ); + if (i == 0 || CRYPTO_memcmp(md, mac, (size_t)mac_size) != 0) { + SSLfatal(s, SSL_AD_BAD_RECORD_MAC, + SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); + return 0; + } + /* + * We've handled the mac now - there is no MAC inside the encrypted + * record + */ + mac_size = 0; + } + + /* + * Set a mark around the packet decryption attempt. This is DTLS, so + * bad packets are just ignored, and we don't want to leave stray + * errors in the queue from processing bogus junk that we ignored. + */ + ERR_set_mark(); + enc_err = s->method->ssl3_enc->enc(s, rr, 1, 0, &macbuf, mac_size); + + /*- + * enc_err is: + * 0: if the record is publicly invalid, or an internal error, or AEAD + * decryption failed, or ETM decryption failed. + * 1: Success or MTE decryption failed (MAC will be randomised) + */ + if (enc_err == 0) { + ERR_pop_to_mark(); + if (ossl_statem_in_error(s)) { + /* SSLfatal() got called */ + goto end; + } + /* For DTLS we simply ignore bad packets. */ + rr->length = 0; + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto end; + } + ERR_clear_last_mark(); + OSSL_TRACE_BEGIN(TLS) { + BIO_printf(trc_out, "dec %zd\n", rr->length); + BIO_dump_indent(trc_out, rr->data, rr->length, 4); + } OSSL_TRACE_END(TLS); + + /* r->length is now the compressed data plus mac */ + if ((sess != NULL) + && !SSL_READ_ETM(s) + && (s->enc_read_ctx != NULL) + && (EVP_MD_CTX_get0_md(s->read_hash) != NULL)) { + /* s->read_hash != NULL => mac_size != -1 */ + + i = s->method->ssl3_enc->mac(s, rr, md, 0 /* not send */ ); + if (i == 0 || macbuf.mac == NULL + || CRYPTO_memcmp(md, macbuf.mac, mac_size) != 0) + enc_err = 0; + if (rr->length > SSL3_RT_MAX_COMPRESSED_LENGTH + mac_size) + enc_err = 0; + } + + if (enc_err == 0) { + /* decryption failed, silently discard message */ + rr->length = 0; + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto end; + } + + /* r->length is now just compressed */ + if (s->expand != NULL) { + if (rr->length > SSL3_RT_MAX_COMPRESSED_LENGTH) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, + SSL_R_COMPRESSED_LENGTH_TOO_LONG); + goto end; + } + if (!ssl3_do_uncompress(s, rr)) { + SSLfatal(s, SSL_AD_DECOMPRESSION_FAILURE, SSL_R_BAD_DECOMPRESSION); + goto end; + } + } + + /* use current Max Fragment Length setting if applicable */ + if (s->session != NULL && USE_MAX_FRAGMENT_LENGTH_EXT(s->session)) + max_plain_length = GET_MAX_FRAGMENT_LENGTH(s->session); + + /* send overflow if the plaintext is too long now it has passed MAC */ + if (rr->length > max_plain_length) { + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, SSL_R_DATA_LENGTH_TOO_LONG); + goto end; + } + + rr->off = 0; + /*- + * So at this point the following is true + * ssl->s3.rrec.type is the type of record + * ssl->s3.rrec.length == number of bytes in record + * ssl->s3.rrec.off == offset to first valid byte + * ssl->s3.rrec.data == where to take bytes from, increment + * after use :-). + */ + + /* we have pulled in a full packet so zero things */ + RECORD_LAYER_reset_packet_length(&s->rlayer); + + /* Mark receipt of record. */ + dtls1_record_bitmap_update(s, bitmap); + + ret = 1; + end: + if (macbuf.alloced) + OPENSSL_free(macbuf.mac); + return ret; +} + +/* + * Retrieve a buffered record that belongs to the current epoch, i.e. processed + */ +#define dtls1_get_processed_record(s) \ + dtls1_retrieve_buffered_record((s), \ + &(DTLS_RECORD_LAYER_get_processed_rcds(&s->rlayer))) + +/*- + * Call this to get a new input record. + * It will return <= 0 if more data is needed, normally due to an error + * or non-blocking IO. + * When it finishes, one packet has been decoded and can be found in + * ssl->s3.rrec.type - is the type of record + * ssl->s3.rrec.data - data + * ssl->s3.rrec.length - number of bytes + */ +/* used only by dtls1_read_bytes */ +int dtls1_get_record(SSL *s) +{ + int ssl_major, ssl_minor; + int rret; + size_t more, n; + SSL3_RECORD *rr; + unsigned char *p = NULL; + unsigned short version; + DTLS1_BITMAP *bitmap; + unsigned int is_next_epoch; + + rr = RECORD_LAYER_get_rrec(&s->rlayer); + + again: + /* + * The epoch may have changed. If so, process all the pending records. + * This is a non-blocking operation. + */ + if (!dtls1_process_buffered_records(s)) { + /* SSLfatal() already called */ + return -1; + } + + /* if we're renegotiating, then there may be buffered records */ + if (dtls1_get_processed_record(s)) + return 1; + + /* get something from the wire */ + + /* check if we have the header */ + if ((RECORD_LAYER_get_rstate(&s->rlayer) != SSL_ST_READ_BODY) || + (RECORD_LAYER_get_packet_length(&s->rlayer) < DTLS1_RT_HEADER_LENGTH)) { + rret = ssl3_read_n(s, DTLS1_RT_HEADER_LENGTH, + SSL3_BUFFER_get_len(&s->rlayer.rbuf), 0, 1, &n); + /* read timeout is handled by dtls1_read_bytes */ + if (rret <= 0) { + /* SSLfatal() already called if appropriate */ + return rret; /* error or non-blocking */ + } + + /* this packet contained a partial record, dump it */ + if (RECORD_LAYER_get_packet_length(&s->rlayer) != + DTLS1_RT_HEADER_LENGTH) { + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto again; + } + + RECORD_LAYER_set_rstate(&s->rlayer, SSL_ST_READ_BODY); + + p = RECORD_LAYER_get_packet(&s->rlayer); + + if (s->msg_callback) + s->msg_callback(0, 0, SSL3_RT_HEADER, p, DTLS1_RT_HEADER_LENGTH, + s, s->msg_callback_arg); + + /* Pull apart the header into the DTLS1_RECORD */ + rr->type = *(p++); + ssl_major = *(p++); + ssl_minor = *(p++); + version = (ssl_major << 8) | ssl_minor; + + /* sequence number is 64 bits, with top 2 bytes = epoch */ + n2s(p, rr->epoch); + + memcpy(&(RECORD_LAYER_get_read_sequence(&s->rlayer)[2]), p, 6); + p += 6; + + n2s(p, rr->length); + rr->read = 0; + + /* + * Lets check the version. We tolerate alerts that don't have the exact + * version number (e.g. because of protocol version errors) + */ + if (!s->first_packet && rr->type != SSL3_RT_ALERT) { + if (version != s->version) { + /* unexpected version, silently discard */ + rr->length = 0; + rr->read = 1; + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto again; + } + } + + if ((version & 0xff00) != (s->version & 0xff00)) { + /* wrong version, silently discard record */ + rr->length = 0; + rr->read = 1; + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto again; + } + + if (rr->length > SSL3_RT_MAX_ENCRYPTED_LENGTH) { + /* record too long, silently discard it */ + rr->length = 0; + rr->read = 1; + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto again; + } + + /* If received packet overflows own-client Max Fragment Length setting */ + if (s->session != NULL && USE_MAX_FRAGMENT_LENGTH_EXT(s->session) + && rr->length > GET_MAX_FRAGMENT_LENGTH(s->session) + SSL3_RT_MAX_ENCRYPTED_OVERHEAD) { + /* record too long, silently discard it */ + rr->length = 0; + rr->read = 1; + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto again; + } + + /* now s->rlayer.rstate == SSL_ST_READ_BODY */ + } + + /* s->rlayer.rstate == SSL_ST_READ_BODY, get and decode the data */ + + if (rr->length > + RECORD_LAYER_get_packet_length(&s->rlayer) - DTLS1_RT_HEADER_LENGTH) { + /* now s->rlayer.packet_length == DTLS1_RT_HEADER_LENGTH */ + more = rr->length; + rret = ssl3_read_n(s, more, more, 1, 1, &n); + /* this packet contained a partial record, dump it */ + if (rret <= 0 || n != more) { + if (ossl_statem_in_error(s)) { + /* ssl3_read_n() called SSLfatal() */ + return -1; + } + rr->length = 0; + rr->read = 1; + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto again; + } + + /* + * now n == rr->length, and s->rlayer.packet_length == + * DTLS1_RT_HEADER_LENGTH + rr->length + */ + } + /* set state for later operations */ + RECORD_LAYER_set_rstate(&s->rlayer, SSL_ST_READ_HEADER); + + /* match epochs. NULL means the packet is dropped on the floor */ + bitmap = dtls1_get_bitmap(s, rr, &is_next_epoch); + if (bitmap == NULL) { + rr->length = 0; + RECORD_LAYER_reset_packet_length(&s->rlayer); /* dump this record */ + goto again; /* get another record */ + } +#ifndef OPENSSL_NO_SCTP + /* Only do replay check if no SCTP bio */ + if (!BIO_dgram_is_sctp(SSL_get_rbio(s))) { +#endif + /* Check whether this is a repeat, or aged record. */ + if (!dtls1_record_replay_check(s, bitmap)) { + rr->length = 0; + rr->read = 1; + RECORD_LAYER_reset_packet_length(&s->rlayer); /* dump this record */ + goto again; /* get another record */ + } +#ifndef OPENSSL_NO_SCTP + } +#endif + + /* just read a 0 length packet */ + if (rr->length == 0) { + rr->read = 1; + goto again; + } + + /* + * If this record is from the next epoch (either HM or ALERT), and a + * handshake is currently in progress, buffer it since it cannot be + * processed at this time. + */ + if (is_next_epoch) { + if ((SSL_in_init(s) || ossl_statem_get_in_handshake(s))) { + if (dtls1_buffer_record (s, + &(DTLS_RECORD_LAYER_get_unprocessed_rcds(&s->rlayer)), + rr->seq_num) < 0) { + /* SSLfatal() already called */ + return -1; + } + } + rr->length = 0; + rr->read = 1; + RECORD_LAYER_reset_packet_length(&s->rlayer); + goto again; + } + + if (!dtls1_process_record(s, bitmap)) { + if (ossl_statem_in_error(s)) { + /* dtls1_process_record() called SSLfatal */ + return -1; + } + rr->length = 0; + rr->read = 1; + RECORD_LAYER_reset_packet_length(&s->rlayer); /* dump this record */ + goto again; /* get another record */ + } + + return 1; + +} + +int dtls_buffer_listen_record(SSL *s, size_t len, unsigned char *seq, size_t off) +{ + SSL3_RECORD *rr; + + rr = RECORD_LAYER_get_rrec(&s->rlayer); + memset(rr, 0, sizeof(SSL3_RECORD)); + + rr->length = len; + rr->type = SSL3_RT_HANDSHAKE; + memcpy(rr->seq_num, seq, sizeof(rr->seq_num)); + rr->off = off; + + s->rlayer.packet = RECORD_LAYER_get_rbuf(&s->rlayer)->buf; + s->rlayer.packet_length = DTLS1_RT_HEADER_LENGTH + len; + rr->data = s->rlayer.packet + DTLS1_RT_HEADER_LENGTH; + + if (dtls1_buffer_record(s, &(s->rlayer.d->processed_rcds), + SSL3_RECORD_get_seq_num(s->rlayer.rrec)) <= 0) { + /* SSLfatal() already called */ + return 0; + } + + return 1; +} diff --git a/openssl/src/ssl/record/ssl3_record_tls13.c b/openssl/src/ssl/record/ssl3_record_tls13.c new file mode 100644 index 000000000..6aa2fb980 --- /dev/null +++ b/openssl/src/ssl/record/ssl3_record_tls13.c @@ -0,0 +1,194 @@ +/* + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../ssl_local.h" +#include "record_local.h" +#include "internal/cryptlib.h" + +/*- + * tls13_enc encrypts/decrypts |n_recs| in |recs|. Calls SSLfatal on internal + * error, but not otherwise. It is the responsibility of the caller to report + * a bad_record_mac. + * + * Returns: + * 0: On failure + * 1: if the record encryption/decryption was successful. + */ +int tls13_enc(SSL *s, SSL3_RECORD *recs, size_t n_recs, int sending, + ossl_unused SSL_MAC_BUF *mac, ossl_unused size_t macsize) +{ + EVP_CIPHER_CTX *ctx; + unsigned char iv[EVP_MAX_IV_LENGTH], recheader[SSL3_RT_HEADER_LENGTH]; + size_t ivlen, taglen, offset, loop, hdrlen; + unsigned char *staticiv; + unsigned char *seq; + int lenu, lenf; + SSL3_RECORD *rec = &recs[0]; + uint32_t alg_enc; + WPACKET wpkt; + + if (n_recs != 1) { + /* Should not happen */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (sending) { + ctx = s->enc_write_ctx; + staticiv = s->write_iv; + seq = RECORD_LAYER_get_write_sequence(&s->rlayer); + } else { + ctx = s->enc_read_ctx; + staticiv = s->read_iv; + seq = RECORD_LAYER_get_read_sequence(&s->rlayer); + } + + /* + * If we're sending an alert and ctx != NULL then we must be forcing + * plaintext alerts. If we're reading and ctx != NULL then we allow + * plaintext alerts at certain points in the handshake. If we've got this + * far then we have already validated that a plaintext alert is ok here. + */ + if (ctx == NULL || rec->type == SSL3_RT_ALERT) { + memmove(rec->data, rec->input, rec->length); + rec->input = rec->data; + return 1; + } + + ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); + + if (s->early_data_state == SSL_EARLY_DATA_WRITING + || s->early_data_state == SSL_EARLY_DATA_WRITE_RETRY) { + if (s->session != NULL && s->session->ext.max_early_data > 0) { + alg_enc = s->session->cipher->algorithm_enc; + } else { + if (!ossl_assert(s->psksession != NULL + && s->psksession->ext.max_early_data > 0)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + alg_enc = s->psksession->cipher->algorithm_enc; + } + } else { + /* + * To get here we must have selected a ciphersuite - otherwise ctx would + * be NULL + */ + if (!ossl_assert(s->s3.tmp.new_cipher != NULL)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + alg_enc = s->s3.tmp.new_cipher->algorithm_enc; + } + + if (alg_enc & SSL_AESCCM) { + if (alg_enc & (SSL_AES128CCM8 | SSL_AES256CCM8)) + taglen = EVP_CCM8_TLS_TAG_LEN; + else + taglen = EVP_CCM_TLS_TAG_LEN; + if (sending && EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, taglen, + NULL) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } else if (alg_enc & SSL_AESGCM) { + taglen = EVP_GCM_TLS_TAG_LEN; + } else if (alg_enc & SSL_CHACHA20) { + taglen = EVP_CHACHAPOLY_TLS_TAG_LEN; +#ifndef OPENSSL_NO_SM4 + } else if (alg_enc & SSL_SM4GCM) { + taglen = EVP_GCM_TLS_TAG_LEN; + } else if (alg_enc & SSL_SM4CCM) { + taglen = EVP_CCM_TLS_TAG_LEN; +#endif + } else { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!sending) { + /* + * Take off tag. There must be at least one byte of content type as + * well as the tag + */ + if (rec->length < taglen + 1) + return 0; + rec->length -= taglen; + } + + /* Set up IV */ + if (ivlen < SEQ_NUM_SIZE) { + /* Should not happen */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + offset = ivlen - SEQ_NUM_SIZE; + memcpy(iv, staticiv, offset); + for (loop = 0; loop < SEQ_NUM_SIZE; loop++) + iv[offset + loop] = staticiv[offset + loop] ^ seq[loop]; + + /* Increment the sequence counter */ + for (loop = SEQ_NUM_SIZE; loop > 0; loop--) { + ++seq[loop - 1]; + if (seq[loop - 1] != 0) + break; + } + if (loop == 0) { + /* Sequence has wrapped */ + return 0; + } + + if (EVP_CipherInit_ex(ctx, NULL, NULL, NULL, iv, sending) <= 0 + || (!sending && EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, + taglen, + rec->data + rec->length) <= 0)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* Set up the AAD */ + if (!WPACKET_init_static_len(&wpkt, recheader, sizeof(recheader), 0) + || !WPACKET_put_bytes_u8(&wpkt, rec->type) + || !WPACKET_put_bytes_u16(&wpkt, rec->rec_version) + || !WPACKET_put_bytes_u16(&wpkt, rec->length + taglen) + || !WPACKET_get_total_written(&wpkt, &hdrlen) + || hdrlen != SSL3_RT_HEADER_LENGTH + || !WPACKET_finish(&wpkt)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + WPACKET_cleanup(&wpkt); + return 0; + } + + /* + * For CCM we must explicitly set the total plaintext length before we add + * any AAD. + */ + if (((alg_enc & SSL_AESCCM || alg_enc & SSL_SM4CCM) != 0 + && EVP_CipherUpdate(ctx, NULL, &lenu, NULL, + (unsigned int)rec->length) <= 0) + || EVP_CipherUpdate(ctx, NULL, &lenu, recheader, + sizeof(recheader)) <= 0 + || EVP_CipherUpdate(ctx, rec->data, &lenu, rec->input, + (unsigned int)rec->length) <= 0 + || EVP_CipherFinal_ex(ctx, rec->data + lenu, &lenf) <= 0 + || (size_t)(lenu + lenf) != rec->length) { + return 0; + } + if (sending) { + /* Add the tag */ + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, taglen, + rec->data + rec->length) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + rec->length += taglen; + } + + return 1; +} diff --git a/openssl/src/ssl/record/methods/tls_pad.c b/openssl/src/ssl/record/tls_pad.c similarity index 90% rename from openssl/src/ssl/record/methods/tls_pad.c rename to openssl/src/ssl/record/tls_pad.c index 23198f3a4..e55935046 100644 --- a/openssl/src/ssl/record/methods/tls_pad.c +++ b/openssl/src/ssl/record/tls_pad.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -9,10 +9,8 @@ #include #include - #include "internal/constant_time.h" #include "internal/cryptlib.h" -#include "internal/ssl3_cbc.h" /* * This file has no dependencies on the rest of libssl because it is shared @@ -33,6 +31,23 @@ static int ssl3_cbc_copy_mac(size_t *reclen, size_t good, OSSL_LIB_CTX *libctx); +int ssl3_cbc_remove_padding_and_mac(size_t *reclen, + size_t origreclen, + unsigned char *recdata, + unsigned char **mac, + int *alloced, + size_t block_size, size_t mac_size, + OSSL_LIB_CTX *libctx); + +int tls1_cbc_remove_padding_and_mac(size_t *reclen, + size_t origreclen, + unsigned char *recdata, + unsigned char **mac, + int *alloced, + size_t block_size, size_t mac_size, + int aead, + OSSL_LIB_CTX *libctx); + /*- * ssl3_cbc_remove_padding removes padding from the decrypted, SSLv3, CBC * record in |recdata| by updating |reclen| in constant time. It also extracts @@ -192,7 +207,6 @@ static int ssl3_cbc_copy_mac(size_t *reclen, #if defined(CBC_MAC_ROTATE_IN_PLACE) unsigned char rotated_mac_buf[64 + EVP_MAX_MD_SIZE]; unsigned char *rotated_mac; - char aux1, aux2, aux3, mask; #else unsigned char rotated_mac[EVP_MAX_MD_SIZE]; #endif @@ -274,19 +288,12 @@ static int ssl3_cbc_copy_mac(size_t *reclen, #if defined(CBC_MAC_ROTATE_IN_PLACE) j = 0; for (i = 0; i < mac_size; i++) { - /* - * in case cache-line is 32 bytes, - * load from both lines and select appropriately - */ - aux1 = rotated_mac[rotate_offset & ~32]; - aux2 = rotated_mac[rotate_offset | 32]; - mask = constant_time_eq_8(rotate_offset & ~32, rotate_offset); - aux3 = constant_time_select_8(mask, aux1, aux2); - rotate_offset++; + /* in case cache-line is 32 bytes, touch second line */ + ((volatile unsigned char *)rotated_mac)[rotate_offset ^ 32]; /* If the padding wasn't good we emit a random MAC */ out[j++] = constant_time_select_8((unsigned char)(good & 0xff), - aux3, + rotated_mac[rotate_offset++], randmac[i]); rotate_offset &= constant_time_lt_s(rotate_offset, mac_size); } diff --git a/openssl/src/ssl/rio/poll_immediate.c b/openssl/src/ssl/rio/poll_immediate.c deleted file mode 100644 index 66e613a7d..000000000 --- a/openssl/src/ssl/rio/poll_immediate.c +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2024 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "internal/common.h" -#include -#include -#include "../ssl_local.h" - -#define ITEM_N(items, stride, n) \ - (*(SSL_POLL_ITEM *)((char *)(items) + (n)*(stride))) - -#define FAIL_FROM(n) \ - do { \ - size_t j; \ - \ - for (j = (n); j < num_items; ++j) \ - ITEM_N(items, stride, j).revents = 0; \ - \ - ok = 0; \ - goto out; \ - } while (0) - -#define FAIL_ITEM(i) \ - do { \ - ITEM_N(items, stride, i).revents = SSL_POLL_EVENT_F; \ - ++result_count; \ - FAIL_FROM(i + 1); \ - } while (0) - -int SSL_poll(SSL_POLL_ITEM *items, - size_t num_items, - size_t stride, - const struct timeval *timeout, - uint64_t flags, - size_t *p_result_count) -{ - int ok = 1; - size_t i, result_count = 0; - SSL_POLL_ITEM *item; - SSL *ssl; - uint64_t revents; - ossl_unused uint64_t events; - ossl_unused int do_tick = ((flags & SSL_POLL_FLAG_NO_HANDLE_EVENTS) == 0); - int is_immediate - = (timeout != NULL - && timeout->tv_sec == 0 && timeout->tv_usec == 0); - - /* - * Prevent calls which use SSL_poll functionality which is not currently - * supported. - */ - if (!is_immediate) { - ERR_raise_data(ERR_LIB_SSL, SSL_R_POLL_REQUEST_NOT_SUPPORTED, - "SSL_poll does not currently support blocking " - "operation"); - FAIL_FROM(0); - } - - /* Trivial case. */ - if (num_items == 0) - goto out; - - /* Poll current state of each item. */ - for (i = 0; i < num_items; ++i) { - item = &ITEM_N(items, stride, i); - events = item->events; - revents = 0; - - switch (item->desc.type) { - case BIO_POLL_DESCRIPTOR_TYPE_SSL: - ssl = item->desc.value.ssl; - if (ssl == NULL) - /* NULL items are no-ops and have revents reported as 0 */ - break; - - switch (ssl->type) { -#ifndef OPENSSL_NO_QUIC - case SSL_TYPE_QUIC_CONNECTION: - case SSL_TYPE_QUIC_XSO: - if (!ossl_quic_conn_poll_events(ssl, events, do_tick, &revents)) - /* above call raises ERR */ - FAIL_ITEM(i); - - if (revents != 0) - ++result_count; - - break; -#endif - - default: - ERR_raise_data(ERR_LIB_SSL, SSL_R_POLL_REQUEST_NOT_SUPPORTED, - "SSL_poll currently only supports QUIC SSL " - "objects"); - FAIL_ITEM(i); - } - break; - case BIO_POLL_DESCRIPTOR_TYPE_SOCK_FD: - ERR_raise_data(ERR_LIB_SSL, SSL_R_POLL_REQUEST_NOT_SUPPORTED, - "SSL_poll currently does not support polling " - "sockets"); - FAIL_ITEM(i); - default: - ERR_raise_data(ERR_LIB_SSL, SSL_R_POLL_REQUEST_NOT_SUPPORTED, - "SSL_poll does not support unknown poll descriptor " - "type %d", item->desc.type); - FAIL_ITEM(i); - } - - item->revents = revents; - } - - /* TODO(QUIC POLLING): Blocking mode */ - /* TODO(QUIC POLLING): Support for polling FDs */ - -out: - if (p_result_count != NULL) - *p_result_count = result_count; - - return ok; -} diff --git a/openssl/src/ssl/record/methods/ssl3_cbc.c b/openssl/src/ssl/s3_cbc.c similarity index 83% rename from openssl/src/ssl/record/methods/ssl3_cbc.c rename to openssl/src/ssl/s3_cbc.c index a8282989e..a3f97ed77 100644 --- a/openssl/src/ssl/record/methods/ssl3_cbc.c +++ b/openssl/src/ssl/s3_cbc.c @@ -1,5 +1,5 @@ /* - * Copyright 2012-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2012-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -16,21 +16,54 @@ * moved out of libssl. */ + /* * MD5 and SHA-1 low level APIs are deprecated for public use, but still ok for * internal use. */ #include "internal/deprecated.h" +#include "internal/constant_time.h" +#include "internal/cryptlib.h" + #include #ifndef FIPS_MODULE # include +# include #endif #include -#include "internal/ssl3_cbc.h" -#include "internal/constant_time.h" -#include "internal/cryptlib.h" +char ssl3_cbc_record_digest_supported(const EVP_MD_CTX *ctx); +int ssl3_cbc_digest_record(const EVP_MD *md, + unsigned char *md_out, + size_t *md_out_size, + const unsigned char *header, + const unsigned char *data, + size_t data_size, + size_t data_plus_mac_plus_padding_size, + const unsigned char *mac_secret, + size_t mac_secret_length, char is_sslv3); + +# define l2n(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff)) + +# define l2n6(l,c) (*((c)++)=(unsigned char)(((l)>>40)&0xff), \ + *((c)++)=(unsigned char)(((l)>>32)&0xff), \ + *((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff)) + +# define l2n8(l,c) (*((c)++)=(unsigned char)(((l)>>56)&0xff), \ + *((c)++)=(unsigned char)(((l)>>48)&0xff), \ + *((c)++)=(unsigned char)(((l)>>40)&0xff), \ + *((c)++)=(unsigned char)(((l)>>32)&0xff), \ + *((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff)) /* * MAX_HASH_BIT_COUNT_BYTES is the maximum number of bytes in the hash's @@ -51,10 +84,10 @@ * little-endian order. The value of p is advanced by four. */ # define u32toLE(n, p) \ - (*((p)++) = (unsigned char)(n ), \ - *((p)++) = (unsigned char)(n >> 8), \ - *((p)++) = (unsigned char)(n >> 16), \ - *((p)++) = (unsigned char)(n >> 24)) + (*((p)++)=(unsigned char)(n), \ + *((p)++)=(unsigned char)(n>>8), \ + *((p)++)=(unsigned char)(n>>16), \ + *((p)++)=(unsigned char)(n>>24)) /* * These functions serialize the state of a hash and thus perform the @@ -64,18 +97,31 @@ static void tls1_md5_final_raw(void *ctx, unsigned char *md_out) { MD5_CTX *md5 = ctx; - u32toLE(md5->A, md_out); u32toLE(md5->B, md_out); u32toLE(md5->C, md_out); u32toLE(md5->D, md_out); } +# ifndef OPENSSL_NO_SM3 +static void tls1_sm3_final_raw(void *ctx, unsigned char *md_out) +{ + SM3_CTX *sm3 = ctx; + + l2n(sm3->A, md_out); + l2n(sm3->B, md_out); + l2n(sm3->C, md_out); + l2n(sm3->D, md_out); + l2n(sm3->E, md_out); + l2n(sm3->F, md_out); + l2n(sm3->G, md_out); + l2n(sm3->H, md_out); +} +# endif #endif /* FIPS_MODULE */ static void tls1_sha1_final_raw(void *ctx, unsigned char *md_out) { SHA_CTX *sha1 = ctx; - l2n(sha1->h0, md_out); l2n(sha1->h1, md_out); l2n(sha1->h2, md_out); @@ -88,8 +134,9 @@ static void tls1_sha256_final_raw(void *ctx, unsigned char *md_out) SHA256_CTX *sha256 = ctx; unsigned i; - for (i = 0; i < 8; i++) + for (i = 0; i < 8; i++) { l2n(sha256->h[i], md_out); + } } static void tls1_sha512_final_raw(void *ctx, unsigned char *md_out) @@ -97,8 +144,9 @@ static void tls1_sha512_final_raw(void *ctx, unsigned char *md_out) SHA512_CTX *sha512 = ctx; unsigned i; - for (i = 0; i < 8; i++) + for (i = 0; i < 8; i++) { l2n8(sha512->h[i], md_out); + } } #undef LARGEST_DIGEST_CTX @@ -194,14 +242,14 @@ int ssl3_cbc_digest_record(const EVP_MD *md, md_transform = (void (*)(void *ctx, const unsigned char *block))SHA256_Transform; md_size = 224 / 8; - } else if (EVP_MD_is_a(md, "SHA2-256")) { + } else if (EVP_MD_is_a(md, "SHA2-256")) { if (SHA256_Init((SHA256_CTX *)md_state.c) <= 0) return 0; md_final_raw = tls1_sha256_final_raw; md_transform = (void (*)(void *ctx, const unsigned char *block))SHA256_Transform; md_size = 32; - } else if (EVP_MD_is_a(md, "SHA2-384")) { + } else if (EVP_MD_is_a(md, "SHA2-384")) { if (SHA384_Init((SHA512_CTX *)md_state.c) <= 0) return 0; md_final_raw = tls1_sha512_final_raw; @@ -219,6 +267,19 @@ int ssl3_cbc_digest_record(const EVP_MD *md, md_size = 64; md_block_size = 128; md_length_size = 16; +#ifndef OPENSSL_NO_SM3 + } else if (EVP_MD_is_a(md, "SM3")) { +# ifdef FIPS_MODULE + return 0; +# else + if (SM3_Init((SM3_CTX *)md_state.c) <= 0) + return 0; + md_final_raw = tls1_sm3_final_raw; + md_transform = + (void (*)(void *ctx, const unsigned char *block))SM3_Transform; + md_size = 32; +# endif +#endif } else { /* * ssl3_cbc_record_digest_supported should have been called first to @@ -236,11 +297,10 @@ int ssl3_cbc_digest_record(const EVP_MD *md, header_length = 13; if (is_sslv3) { - header_length = mac_secret_length - + sslv3_pad_length - + 8 /* sequence number */ - + 1 /* record type */ - + 2; /* record length */ + header_length = mac_secret_length + sslv3_pad_length + 8 /* sequence + * number */ + + 1 /* record type */ + + 2 /* record length */ ; } /* @@ -258,9 +318,7 @@ int ssl3_cbc_digest_record(const EVP_MD *md, * short and there obviously cannot be this many blocks then * variance_blocks can be reduced. */ - variance_blocks = is_sslv3 ? 2 - : (((255 + 1 + md_size + md_block_size - 1) - / md_block_size) + 1); + variance_blocks = is_sslv3 ? 2 : ( ((255 + 1 + md_size + md_block_size - 1) / md_block_size) + 1); /* * From now on we're dealing with the MAC, which conceptually has 13 * bytes of `header' before the start of the data (TLS) or 71/75 bytes @@ -402,10 +460,8 @@ int ssl3_cbc_digest_record(const EVP_MD *md, unsigned char block[MAX_HASH_BLOCK_SIZE]; unsigned char is_block_a = constant_time_eq_8_s(i, index_a); unsigned char is_block_b = constant_time_eq_8_s(i, index_b); - for (j = 0; j < md_block_size; j++) { unsigned char b = 0, is_past_c, is_past_cp1; - if (k < header_length) b = header[k]; else if (k < data_plus_mac_plus_padding_size + header_length) @@ -456,7 +512,7 @@ int ssl3_cbc_digest_record(const EVP_MD *md, if (md_ctx == NULL) goto err; - if (EVP_DigestInit_ex(md_ctx, md, NULL /* engine */) <= 0) + if (EVP_DigestInit_ex(md_ctx, md, NULL /* engine */ ) <= 0) goto err; if (is_sslv3) { /* We repurpose |hmac_pad| to contain the SSLv3 pad2 block. */ diff --git a/openssl/src/ssl/s3_enc.c b/openssl/src/ssl/s3_enc.c index 878556b06..2ca3f74ae 100644 --- a/openssl/src/ssl/s3_enc.c +++ b/openssl/src/ssl/s3_enc.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2005 Nokia. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -15,7 +15,7 @@ #include #include "internal/cryptlib.h" -static int ssl3_generate_key_block(SSL_CONNECTION *s, unsigned char *km, int num) +static int ssl3_generate_key_block(SSL *s, unsigned char *km, int num) { const EVP_MD *md5 = NULL, *sha1 = NULL; EVP_MD_CTX *m5; @@ -24,18 +24,17 @@ static int ssl3_generate_key_block(SSL_CONNECTION *s, unsigned char *km, int num unsigned char c = 'A'; unsigned int i, k; int ret = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); #ifdef CHARSET_EBCDIC c = os_toascii[c]; /* 'A' in ASCII */ #endif k = 0; - md5 = ssl_evp_md_fetch(sctx->libctx, NID_md5, sctx->propq); - sha1 = ssl_evp_md_fetch(sctx->libctx, NID_sha1, sctx->propq); + md5 = ssl_evp_md_fetch(s->ctx->libctx, NID_md5, s->ctx->propq); + sha1 = ssl_evp_md_fetch(s->ctx->libctx, NID_sha1, s->ctx->propq); m5 = EVP_MD_CTX_new(); s1 = EVP_MD_CTX_new(); if (md5 == NULL || sha1 == NULL || m5 == NULL || s1 == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } for (i = 0; (int)i < num; i += MD5_DIGEST_LENGTH) { @@ -87,56 +86,131 @@ static int ssl3_generate_key_block(SSL_CONNECTION *s, unsigned char *km, int num return ret; } -int ssl3_change_cipher_state(SSL_CONNECTION *s, int which) +int ssl3_change_cipher_state(SSL *s, int which) { unsigned char *p, *mac_secret; - size_t md_len; - unsigned char *key, *iv; - const EVP_CIPHER *ciph; - const SSL_COMP *comp = NULL; - const EVP_MD *md; + unsigned char *ms, *key, *iv; + EVP_CIPHER_CTX *dd; + const EVP_CIPHER *c; +#ifndef OPENSSL_NO_COMP + COMP_METHOD *comp; +#endif + const EVP_MD *m; int mdi; - size_t n, iv_len, key_len; - int direction = (which & SSL3_CC_READ) != 0 ? OSSL_RECORD_DIRECTION_READ - : OSSL_RECORD_DIRECTION_WRITE; + size_t n, i, j, k, cl; + int reuse_dd = 0; - ciph = s->s3.tmp.new_sym_enc; - md = s->s3.tmp.new_hash; + c = s->s3.tmp.new_sym_enc; + m = s->s3.tmp.new_hash; /* m == NULL will lead to a crash later */ - if (!ossl_assert(md != NULL)) { + if (!ossl_assert(m != NULL)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } #ifndef OPENSSL_NO_COMP - comp = s->s3.tmp.new_compression; + if (s->s3.tmp.new_compression == NULL) + comp = NULL; + else + comp = s->s3.tmp.new_compression->method; #endif + if (which & SSL3_CC_READ) { + if (s->enc_read_ctx != NULL) { + reuse_dd = 1; + } else if ((s->enc_read_ctx = EVP_CIPHER_CTX_new()) == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } else { + /* + * make sure it's initialised in case we exit later with an error + */ + EVP_CIPHER_CTX_reset(s->enc_read_ctx); + } + dd = s->enc_read_ctx; + + if (ssl_replace_hash(&s->read_hash, m) == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } +#ifndef OPENSSL_NO_COMP + /* COMPRESS */ + COMP_CTX_free(s->expand); + s->expand = NULL; + if (comp != NULL) { + s->expand = COMP_CTX_new(comp); + if (s->expand == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, + SSL_R_COMPRESSION_LIBRARY_ERROR); + goto err; + } + } +#endif + RECORD_LAYER_reset_read_sequence(&s->rlayer); + mac_secret = &(s->s3.read_mac_secret[0]); + } else { + s->statem.enc_write_state = ENC_WRITE_STATE_INVALID; + if (s->enc_write_ctx != NULL) { + reuse_dd = 1; + } else if ((s->enc_write_ctx = EVP_CIPHER_CTX_new()) == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } else { + /* + * make sure it's initialised in case we exit later with an error + */ + EVP_CIPHER_CTX_reset(s->enc_write_ctx); + } + dd = s->enc_write_ctx; + if (ssl_replace_hash(&s->write_hash, m) == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } +#ifndef OPENSSL_NO_COMP + /* COMPRESS */ + COMP_CTX_free(s->compress); + s->compress = NULL; + if (comp != NULL) { + s->compress = COMP_CTX_new(comp); + if (s->compress == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, + SSL_R_COMPRESSION_LIBRARY_ERROR); + goto err; + } + } +#endif + RECORD_LAYER_reset_write_sequence(&s->rlayer); + mac_secret = &(s->s3.write_mac_secret[0]); + } + + if (reuse_dd) + EVP_CIPHER_CTX_reset(dd); + p = s->s3.tmp.key_block; - mdi = EVP_MD_get_size(md); + mdi = EVP_MD_get_size(m); if (mdi < 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } - md_len = (size_t)mdi; - key_len = EVP_CIPHER_get_key_length(ciph); - iv_len = EVP_CIPHER_get_iv_length(ciph); - + i = mdi; + cl = EVP_CIPHER_get_key_length(c); + j = cl; + k = EVP_CIPHER_get_iv_length(c); if ((which == SSL3_CHANGE_CIPHER_CLIENT_WRITE) || (which == SSL3_CHANGE_CIPHER_SERVER_READ)) { - mac_secret = &(p[0]); - n = md_len + md_len; + ms = &(p[0]); + n = i + i; key = &(p[n]); - n += key_len + key_len; + n += j + j; iv = &(p[n]); - n += iv_len + iv_len; + n += k + k; } else { - n = md_len; - mac_secret = &(p[n]); - n += md_len + key_len; + n = i; + ms = &(p[n]); + n += i + j; key = &(p[n]); - n += key_len + iv_len; + n += j + k; iv = &(p[n]); - n += iv_len; + n += k; } if (n > s->s3.tmp.key_block_length) { @@ -144,21 +218,26 @@ int ssl3_change_cipher_state(SSL_CONNECTION *s, int which) goto err; } - if (!ssl_set_new_record_layer(s, SSL3_VERSION, - direction, - OSSL_RECORD_PROTECTION_LEVEL_APPLICATION, - NULL, 0, key, key_len, iv, iv_len, mac_secret, - md_len, ciph, 0, NID_undef, md, comp, NULL)) { + memcpy(mac_secret, ms, i); + + if (!EVP_CipherInit_ex(dd, c, NULL, key, iv, (which & SSL3_CC_WRITE))) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (EVP_CIPHER_get0_provider(c) != NULL + && !tls_provider_set_tls_params(s, dd, c, m)) { /* SSLfatal already called */ goto err; } + s->statem.enc_write_state = ENC_WRITE_STATE_VALID; return 1; err: return 0; } -int ssl3_setup_key_block(SSL_CONNECTION *s) +int ssl3_setup_key_block(SSL *s) { unsigned char *p; const EVP_CIPHER *c; @@ -170,8 +249,8 @@ int ssl3_setup_key_block(SSL_CONNECTION *s) if (s->s3.tmp.key_block_length != 0) return 1; - if (!ssl_cipher_get_evp(SSL_CONNECTION_GET_CTX(s), s->session, &c, &hash, - NULL, NULL, &comp, 0)) { + if (!ssl_cipher_get_evp(s->ctx, s->session, &c, &hash, NULL, NULL, &comp, + 0)) { /* Error is already recorded */ SSLfatal_alert(s, SSL_AD_INTERNAL_ERROR); return 0; @@ -197,7 +276,7 @@ int ssl3_setup_key_block(SSL_CONNECTION *s) ssl3_cleanup_key_block(s); if ((p = OPENSSL_malloc(num)) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } @@ -207,22 +286,38 @@ int ssl3_setup_key_block(SSL_CONNECTION *s) /* Calls SSLfatal() as required */ ret = ssl3_generate_key_block(s, p, num); + if (!(s->options & SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS)) { + /* + * enable vulnerability countermeasure for CBC ciphers with known-IV + * problem (http://www.openssl.org/~bodo/tls-cbc.txt) + */ + s->s3.need_empty_fragments = 1; + + if (s->session->cipher != NULL) { + if (s->session->cipher->algorithm_enc == SSL_eNULL) + s->s3.need_empty_fragments = 0; + + if (s->session->cipher->algorithm_enc == SSL_RC4) + s->s3.need_empty_fragments = 0; + } + } + return ret; } -void ssl3_cleanup_key_block(SSL_CONNECTION *s) +void ssl3_cleanup_key_block(SSL *s) { OPENSSL_clear_free(s->s3.tmp.key_block, s->s3.tmp.key_block_length); s->s3.tmp.key_block = NULL; s->s3.tmp.key_block_length = 0; } -int ssl3_init_finished_mac(SSL_CONNECTION *s) +int ssl3_init_finished_mac(SSL *s) { BIO *buf = BIO_new(BIO_s_mem()); if (buf == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_BIO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } ssl3_free_digest_list(s); @@ -236,7 +331,7 @@ int ssl3_init_finished_mac(SSL_CONNECTION *s) * together. */ -void ssl3_free_digest_list(SSL_CONNECTION *s) +void ssl3_free_digest_list(SSL *s) { BIO_free(s->s3.handshake_buffer); s->s3.handshake_buffer = NULL; @@ -244,7 +339,7 @@ void ssl3_free_digest_list(SSL_CONNECTION *s) s->s3.handshake_dgst = NULL; } -int ssl3_finish_mac(SSL_CONNECTION *s, const unsigned char *buf, size_t len) +int ssl3_finish_mac(SSL *s, const unsigned char *buf, size_t len) { int ret; @@ -269,7 +364,7 @@ int ssl3_finish_mac(SSL_CONNECTION *s, const unsigned char *buf, size_t len) return 1; } -int ssl3_digest_cached_records(SSL_CONNECTION *s, int keep) +int ssl3_digest_cached_records(SSL *s, int keep) { const EVP_MD *md; long hdatalen; @@ -284,7 +379,7 @@ int ssl3_digest_cached_records(SSL_CONNECTION *s, int keep) s->s3.handshake_dgst = EVP_MD_CTX_new(); if (s->s3.handshake_dgst == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } @@ -318,7 +413,7 @@ void ssl3_digest_master_key_set_params(const SSL_SESSION *session, params[n++] = OSSL_PARAM_construct_end(); } -size_t ssl3_final_finish_mac(SSL_CONNECTION *s, const char *sender, size_t len, +size_t ssl3_final_finish_mac(SSL *s, const char *sender, size_t len, unsigned char *p) { int ret; @@ -336,7 +431,7 @@ size_t ssl3_final_finish_mac(SSL_CONNECTION *s, const char *sender, size_t len, ctx = EVP_MD_CTX_new(); if (ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } if (!EVP_MD_CTX_copy_ex(ctx, s->s3.handshake_dgst)) { @@ -371,11 +466,10 @@ size_t ssl3_final_finish_mac(SSL_CONNECTION *s, const char *sender, size_t len, return ret; } -int ssl3_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, - unsigned char *p, +int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p, size_t len, size_t *secret_size) { - static const unsigned char *const salt[3] = { + static const unsigned char *salt[3] = { #ifndef CHARSET_EBCDIC (const unsigned char *)"A", (const unsigned char *)"BB", @@ -393,11 +487,11 @@ int ssl3_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, size_t ret_secret_size = 0; if (ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } for (i = 0; i < 3; i++) { - if (EVP_DigestInit_ex(ctx, SSL_CONNECTION_GET_CTX(s)->sha1, NULL) <= 0 + if (EVP_DigestInit_ex(ctx, s->ctx->sha1, NULL) <= 0 || EVP_DigestUpdate(ctx, salt[i], strlen((const char *)salt[i])) <= 0 || EVP_DigestUpdate(ctx, p, len) <= 0 @@ -406,7 +500,7 @@ int ssl3_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, || EVP_DigestUpdate(ctx, &(s->s3.server_random[0]), SSL3_RANDOM_SIZE) <= 0 || EVP_DigestFinal_ex(ctx, buf, &n) <= 0 - || EVP_DigestInit_ex(ctx, SSL_CONNECTION_GET_CTX(s)->md5, NULL) <= 0 + || EVP_DigestInit_ex(ctx, s->ctx->md5, NULL) <= 0 || EVP_DigestUpdate(ctx, p, len) <= 0 || EVP_DigestUpdate(ctx, buf, n) <= 0 || EVP_DigestFinal_ex(ctx, out, &n) <= 0) { diff --git a/openssl/src/ssl/s3_lib.c b/openssl/src/ssl/s3_lib.c index 2bc5e79fd..4cbe0e000 100644 --- a/openssl/src/ssl/s3_lib.c +++ b/openssl/src/ssl/s3_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -47,7 +47,7 @@ static SSL_CIPHER tls13_ciphers[] = { TLS1_3_VERSION, TLS1_3_VERSION, 0, 0, SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | SSL_QUIC, + SSL_HANDSHAKE_MAC_SHA256, 128, 128, }, { @@ -62,7 +62,7 @@ static SSL_CIPHER tls13_ciphers[] = { TLS1_3_VERSION, TLS1_3_VERSION, 0, 0, SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | SSL_QUIC, + SSL_HANDSHAKE_MAC_SHA384, 256, 256, }, @@ -78,7 +78,7 @@ static SSL_CIPHER tls13_ciphers[] = { TLS1_3_VERSION, TLS1_3_VERSION, 0, 0, SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | SSL_QUIC, + SSL_HANDSHAKE_MAC_SHA256, 256, 256, }, @@ -108,11 +108,46 @@ static SSL_CIPHER tls13_ciphers[] = { SSL_AEAD, TLS1_3_VERSION, TLS1_3_VERSION, 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 128, + 128, + }, +#if (!defined OPENSSL_NO_SM2) && (!defined OPENSSL_NO_SM3) \ + && (!defined OPENSSL_NO_SM4) + /* Cipher 0x00C6 and 0x00C7, Reference to RFC 8998 */ + { + 1, + TLS1_3_RFC_SM4_GCM_SM3, + TLS1_3_RFC_SM4_GCM_SM3, + TLS1_3_CK_SM4_GCM_SM3, + SSL_kANY, + SSL_aANY, + SSL_SM4GCM, + SSL_AEAD, + TLS1_3_VERSION,TLS1_3_VERSION, + 0, 0, + SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, + 128, + 128, + }, { + 1, + TLS1_3_RFC_SM4_CCM_SM3, + TLS1_3_RFC_SM4_CCM_SM3, + TLS1_3_CK_SM4_CCM_SM3, + SSL_kANY, + SSL_aANY, + SSL_SM4CCM, + SSL_AEAD, + TLS1_3_VERSION,TLS1_3_VERSION, + 0, 0, + SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, + 128, 128, } +#endif }; /* @@ -122,7 +157,7 @@ static SSL_CIPHER tls13_ciphers[] = { * EC * PSK * SRP (within that: RSA EC PSK) - * Cipher families: Chacha/poly, Camellia, Gost, IDEA, SEED + * Chacha/poly * Weak ciphers */ static SSL_CIPHER ssl3_ciphers[] = { @@ -699,9 +734,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 128, 128, }, { @@ -715,9 +750,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 256, 256, }, { @@ -731,9 +766,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 128, 128, }, { @@ -747,9 +782,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 256, 256, }, { @@ -827,9 +862,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 128, 128, }, { @@ -843,9 +878,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 256, 256, }, { @@ -859,9 +894,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 128, 128, }, { @@ -875,9 +910,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 256, 256, }, { @@ -923,9 +958,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 128, 128, }, { @@ -939,9 +974,9 @@ static SSL_CIPHER ssl3_ciphers[] = { SSL_AEAD, TLS1_2_VERSION, TLS1_2_VERSION, DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_NOT_DEFAULT | SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 64, /* CCM8 uses a short tag, so we have a low security strength */ + 256, 256, }, { @@ -2147,1071 +2182,305 @@ static SSL_CIPHER ssl3_ciphers[] = { 256, }, +#ifndef OPENSSL_NO_WEAK_SSL_CIPHERS { 1, - TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_RSA_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_RSA_WITH_CAMELLIA_128_CBC_SHA256, + SSL3_TXT_RSA_RC4_128_MD5, + SSL3_RFC_RSA_RC4_128_MD5, + SSL3_CK_RSA_RC4_128_MD5, SSL_kRSA, SSL_aRSA, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 128, - 128, - }, - { - 1, - TLS1_TXT_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256, - SSL_kDHE, - SSL_aDSS, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + SSL_RC4, + SSL_MD5, + SSL3_VERSION, TLS1_2_VERSION, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, 128, 128, }, { 1, - TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, - SSL_kDHE, + SSL3_TXT_RSA_RC4_128_SHA, + SSL3_RFC_RSA_RC4_128_SHA, + SSL3_CK_RSA_RC4_128_SHA, + SSL_kRSA, SSL_aRSA, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + SSL_RC4, + SSL_SHA1, + SSL3_VERSION, TLS1_2_VERSION, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, 128, 128, }, { 1, - TLS1_TXT_ADH_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_ADH_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_ADH_WITH_CAMELLIA_128_CBC_SHA256, + SSL3_TXT_ADH_RC4_128_MD5, + SSL3_RFC_ADH_RC4_128_MD5, + SSL3_CK_ADH_RC4_128_MD5, SSL_kDHE, SSL_aNULL, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + SSL_RC4, + SSL_MD5, + SSL3_VERSION, TLS1_2_VERSION, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, 128, 128, }, { 1, - TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA256, - TLS1_RFC_RSA_WITH_CAMELLIA_256_CBC_SHA256, - TLS1_CK_RSA_WITH_CAMELLIA_256_CBC_SHA256, - SSL_kRSA, - SSL_aRSA, - SSL_CAMELLIA256, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 256, - 256, - }, - { - 1, - TLS1_TXT_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256, - TLS1_RFC_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256, - TLS1_CK_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256, - SSL_kDHE, - SSL_aDSS, - SSL_CAMELLIA256, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 256, - 256, - }, - { - 1, - TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256, - TLS1_RFC_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256, - TLS1_CK_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256, - SSL_kDHE, - SSL_aRSA, - SSL_CAMELLIA256, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 256, - 256, + TLS1_TXT_ECDHE_PSK_WITH_RC4_128_SHA, + TLS1_RFC_ECDHE_PSK_WITH_RC4_128_SHA, + TLS1_CK_ECDHE_PSK_WITH_RC4_128_SHA, + SSL_kECDHEPSK, + SSL_aPSK, + SSL_RC4, + SSL_SHA1, + TLS1_VERSION, TLS1_2_VERSION, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, + SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, + 128, + 128, }, { 1, - TLS1_TXT_ADH_WITH_CAMELLIA_256_CBC_SHA256, - TLS1_RFC_ADH_WITH_CAMELLIA_256_CBC_SHA256, - TLS1_CK_ADH_WITH_CAMELLIA_256_CBC_SHA256, - SSL_kDHE, + TLS1_TXT_ECDH_anon_WITH_RC4_128_SHA, + TLS1_RFC_ECDH_anon_WITH_RC4_128_SHA, + TLS1_CK_ECDH_anon_WITH_RC4_128_SHA, + SSL_kECDHE, SSL_aNULL, - SSL_CAMELLIA256, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 256, - 256, - }, - { - 1, - TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA, - TLS1_RFC_RSA_WITH_CAMELLIA_256_CBC_SHA, - TLS1_CK_RSA_WITH_CAMELLIA_256_CBC_SHA, - SSL_kRSA, - SSL_aRSA, - SSL_CAMELLIA256, + SSL_RC4, SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, + TLS1_VERSION, TLS1_2_VERSION, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 256, - 256, + 128, + 128, }, { 1, - TLS1_TXT_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA, - TLS1_RFC_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA, - TLS1_CK_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA, - SSL_kDHE, - SSL_aDSS, - SSL_CAMELLIA256, + TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA, + TLS1_RFC_ECDHE_ECDSA_WITH_RC4_128_SHA, + TLS1_CK_ECDHE_ECDSA_WITH_RC4_128_SHA, + SSL_kECDHE, + SSL_aECDSA, + SSL_RC4, SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, + TLS1_VERSION, TLS1_2_VERSION, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 256, - 256, + 128, + 128, }, { 1, - TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA, - TLS1_RFC_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA, - TLS1_CK_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA, - SSL_kDHE, + TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA, + TLS1_RFC_ECDHE_RSA_WITH_RC4_128_SHA, + TLS1_CK_ECDHE_RSA_WITH_RC4_128_SHA, + SSL_kECDHE, SSL_aRSA, - SSL_CAMELLIA256, + SSL_RC4, SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, + TLS1_VERSION, TLS1_2_VERSION, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 256, - 256, + 128, + 128, }, { 1, - TLS1_TXT_ADH_WITH_CAMELLIA_256_CBC_SHA, - TLS1_RFC_ADH_WITH_CAMELLIA_256_CBC_SHA, - TLS1_CK_ADH_WITH_CAMELLIA_256_CBC_SHA, - SSL_kDHE, - SSL_aNULL, - SSL_CAMELLIA256, + TLS1_TXT_PSK_WITH_RC4_128_SHA, + TLS1_RFC_PSK_WITH_RC4_128_SHA, + TLS1_CK_PSK_WITH_RC4_128_SHA, + SSL_kPSK, + SSL_aPSK, + SSL_RC4, SSL_SHA1, SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 256, - 256, + 128, + 128, }, { 1, - TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA, - TLS1_RFC_RSA_WITH_CAMELLIA_128_CBC_SHA, - TLS1_CK_RSA_WITH_CAMELLIA_128_CBC_SHA, - SSL_kRSA, + TLS1_TXT_RSA_PSK_WITH_RC4_128_SHA, + TLS1_RFC_RSA_PSK_WITH_RC4_128_SHA, + TLS1_CK_RSA_PSK_WITH_RC4_128_SHA, + SSL_kRSAPSK, SSL_aRSA, - SSL_CAMELLIA128, + SSL_RC4, SSL_SHA1, SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, 128, 128, }, { 1, - TLS1_TXT_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA, - TLS1_RFC_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA, - TLS1_CK_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA, - SSL_kDHE, - SSL_aDSS, - SSL_CAMELLIA128, + TLS1_TXT_DHE_PSK_WITH_RC4_128_SHA, + TLS1_RFC_DHE_PSK_WITH_RC4_128_SHA, + TLS1_CK_DHE_PSK_WITH_RC4_128_SHA, + SSL_kDHEPSK, + SSL_aPSK, + SSL_RC4, SSL_SHA1, SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, + 0, 0, + SSL_NOT_DEFAULT | SSL_MEDIUM, SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, 128, 128, }, - { - 1, - TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA, - TLS1_RFC_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA, - TLS1_CK_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA, - SSL_kDHE, - SSL_aRSA, - SSL_CAMELLIA128, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_ADH_WITH_CAMELLIA_128_CBC_SHA, - TLS1_RFC_ADH_WITH_CAMELLIA_128_CBC_SHA, - TLS1_CK_ADH_WITH_CAMELLIA_128_CBC_SHA, - SSL_kDHE, - SSL_aNULL, - SSL_CAMELLIA128, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256, - SSL_kECDHE, - SSL_aECDSA, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 128, - 128, - }, - { - 1, - TLS1_TXT_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_RFC_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_CK_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384, - SSL_kECDHE, - SSL_aECDSA, - SSL_CAMELLIA256, - SSL_SHA384, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - { - 1, - TLS1_TXT_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256, - SSL_kECDHE, - SSL_aRSA, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, - 128, - 128, - }, - { - 1, - TLS1_TXT_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_RFC_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_CK_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384, - SSL_kECDHE, - SSL_aRSA, - SSL_CAMELLIA256, - SSL_SHA384, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - { - 1, - TLS1_TXT_PSK_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_PSK_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_PSK_WITH_CAMELLIA_128_CBC_SHA256, - SSL_kPSK, - SSL_aPSK, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_PSK_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_RFC_PSK_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_CK_PSK_WITH_CAMELLIA_256_CBC_SHA384, - SSL_kPSK, - SSL_aPSK, - SSL_CAMELLIA256, - SSL_SHA384, - TLS1_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - { - 1, - TLS1_TXT_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, - SSL_kDHEPSK, - SSL_aPSK, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_RFC_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_CK_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, - SSL_kDHEPSK, - SSL_aPSK, - SSL_CAMELLIA256, - SSL_SHA384, - TLS1_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - { - 1, - TLS1_TXT_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256, - SSL_kRSAPSK, - SSL_aRSA, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_RFC_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_CK_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384, - SSL_kRSAPSK, - SSL_aRSA, - SSL_CAMELLIA256, - SSL_SHA384, - TLS1_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - { - 1, - TLS1_TXT_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_RFC_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, - TLS1_CK_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256, - SSL_kECDHEPSK, - SSL_aPSK, - SSL_CAMELLIA128, - SSL_SHA256, - TLS1_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_RFC_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, - TLS1_CK_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384, - SSL_kECDHEPSK, - SSL_aPSK, - SSL_CAMELLIA256, - SSL_SHA384, - TLS1_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - -#ifndef OPENSSL_NO_GOST - { - 1, - "GOST2001-GOST89-GOST89", - "TLS_GOSTR341001_WITH_28147_CNT_IMIT", - 0x3000081, - SSL_kGOST, - SSL_aGOST01, - SSL_eGOST2814789CNT, - SSL_GOST89MAC, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_HIGH, - SSL_HANDSHAKE_MAC_GOST94 | TLS1_PRF_GOST94 | TLS1_STREAM_MAC, - 256, - 256, - }, - { - 1, - "GOST2001-NULL-GOST94", - "TLS_GOSTR341001_WITH_NULL_GOSTR3411", - 0x3000083, - SSL_kGOST, - SSL_aGOST01, - SSL_eNULL, - SSL_GOST94, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_STRONG_NONE, - SSL_HANDSHAKE_MAC_GOST94 | TLS1_PRF_GOST94, - 0, - 0, - }, - { - 1, - "IANA-GOST2012-GOST8912-GOST8912", - NULL, - 0x0300c102, - SSL_kGOST, - SSL_aGOST12 | SSL_aGOST01, - SSL_eGOST2814789CNT12, - SSL_GOST89MAC12, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_HIGH, - SSL_HANDSHAKE_MAC_GOST12_256 | TLS1_PRF_GOST12_256 | TLS1_STREAM_MAC, - 256, - 256, - }, - { - 1, - "LEGACY-GOST2012-GOST8912-GOST8912", - NULL, - 0x0300ff85, - SSL_kGOST, - SSL_aGOST12 | SSL_aGOST01, - SSL_eGOST2814789CNT12, - SSL_GOST89MAC12, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_HIGH, - SSL_HANDSHAKE_MAC_GOST12_256 | TLS1_PRF_GOST12_256 | TLS1_STREAM_MAC, - 256, - 256, - }, - { - 1, - "GOST2012-NULL-GOST12", - NULL, - 0x0300ff87, - SSL_kGOST, - SSL_aGOST12 | SSL_aGOST01, - SSL_eNULL, - SSL_GOST12_256, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_STRONG_NONE, - SSL_HANDSHAKE_MAC_GOST12_256 | TLS1_PRF_GOST12_256 | TLS1_STREAM_MAC, - 0, - 0, - }, - { - 1, - "GOST2012-KUZNYECHIK-KUZNYECHIKOMAC", - NULL, - 0x0300C100, - SSL_kGOST18, - SSL_aGOST12, - SSL_KUZNYECHIK, - SSL_KUZNYECHIKOMAC, - TLS1_2_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_HIGH, - SSL_HANDSHAKE_MAC_GOST12_256 | TLS1_PRF_GOST12_256 | TLS1_TLSTREE, - 256, - 256, - }, - { - 1, - "GOST2012-MAGMA-MAGMAOMAC", - NULL, - 0x0300C101, - SSL_kGOST18, - SSL_aGOST12, - SSL_MAGMA, - SSL_MAGMAOMAC, - TLS1_2_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_HIGH, - SSL_HANDSHAKE_MAC_GOST12_256 | TLS1_PRF_GOST12_256 | TLS1_TLSTREE, - 256, - 256, - }, -#endif /* OPENSSL_NO_GOST */ - - { - 1, - SSL3_TXT_RSA_IDEA_128_SHA, - SSL3_RFC_RSA_IDEA_128_SHA, - SSL3_CK_RSA_IDEA_128_SHA, - SSL_kRSA, - SSL_aRSA, - SSL_IDEA, - SSL_SHA1, - SSL3_VERSION, TLS1_1_VERSION, - DTLS1_BAD_VER, DTLS1_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - - { - 1, - TLS1_TXT_RSA_WITH_SEED_SHA, - TLS1_RFC_RSA_WITH_SEED_SHA, - TLS1_CK_RSA_WITH_SEED_SHA, - SSL_kRSA, - SSL_aRSA, - SSL_SEED, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_DHE_DSS_WITH_SEED_SHA, - TLS1_RFC_DHE_DSS_WITH_SEED_SHA, - TLS1_CK_DHE_DSS_WITH_SEED_SHA, - SSL_kDHE, - SSL_aDSS, - SSL_SEED, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_DHE_RSA_WITH_SEED_SHA, - TLS1_RFC_DHE_RSA_WITH_SEED_SHA, - TLS1_CK_DHE_RSA_WITH_SEED_SHA, - SSL_kDHE, - SSL_aRSA, - SSL_SEED, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - { - 1, - TLS1_TXT_ADH_WITH_SEED_SHA, - TLS1_RFC_ADH_WITH_SEED_SHA, - TLS1_CK_ADH_WITH_SEED_SHA, - SSL_kDHE, - SSL_aNULL, - SSL_SEED, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - DTLS1_BAD_VER, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 128, - 128, - }, - -#ifndef OPENSSL_NO_WEAK_SSL_CIPHERS - { - 1, - SSL3_TXT_RSA_RC4_128_MD5, - SSL3_RFC_RSA_RC4_128_MD5, - SSL3_CK_RSA_RC4_128_MD5, - SSL_kRSA, - SSL_aRSA, - SSL_RC4, - SSL_MD5, - SSL3_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - SSL3_TXT_RSA_RC4_128_SHA, - SSL3_RFC_RSA_RC4_128_SHA, - SSL3_CK_RSA_RC4_128_SHA, - SSL_kRSA, - SSL_aRSA, - SSL_RC4, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - SSL3_TXT_ADH_RC4_128_MD5, - SSL3_RFC_ADH_RC4_128_MD5, - SSL3_CK_ADH_RC4_128_MD5, - SSL_kDHE, - SSL_aNULL, - SSL_RC4, - SSL_MD5, - SSL3_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - TLS1_TXT_ECDHE_PSK_WITH_RC4_128_SHA, - TLS1_RFC_ECDHE_PSK_WITH_RC4_128_SHA, - TLS1_CK_ECDHE_PSK_WITH_RC4_128_SHA, - SSL_kECDHEPSK, - SSL_aPSK, - SSL_RC4, - SSL_SHA1, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - TLS1_TXT_ECDH_anon_WITH_RC4_128_SHA, - TLS1_RFC_ECDH_anon_WITH_RC4_128_SHA, - TLS1_CK_ECDH_anon_WITH_RC4_128_SHA, - SSL_kECDHE, - SSL_aNULL, - SSL_RC4, - SSL_SHA1, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA, - TLS1_RFC_ECDHE_ECDSA_WITH_RC4_128_SHA, - TLS1_CK_ECDHE_ECDSA_WITH_RC4_128_SHA, - SSL_kECDHE, - SSL_aECDSA, - SSL_RC4, - SSL_SHA1, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA, - TLS1_RFC_ECDHE_RSA_WITH_RC4_128_SHA, - TLS1_CK_ECDHE_RSA_WITH_RC4_128_SHA, - SSL_kECDHE, - SSL_aRSA, - SSL_RC4, - SSL_SHA1, - TLS1_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - TLS1_TXT_PSK_WITH_RC4_128_SHA, - TLS1_RFC_PSK_WITH_RC4_128_SHA, - TLS1_CK_PSK_WITH_RC4_128_SHA, - SSL_kPSK, - SSL_aPSK, - SSL_RC4, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - TLS1_TXT_RSA_PSK_WITH_RC4_128_SHA, - TLS1_RFC_RSA_PSK_WITH_RC4_128_SHA, - TLS1_CK_RSA_PSK_WITH_RC4_128_SHA, - SSL_kRSAPSK, - SSL_aRSA, - SSL_RC4, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, - { - 1, - TLS1_TXT_DHE_PSK_WITH_RC4_128_SHA, - TLS1_RFC_DHE_PSK_WITH_RC4_128_SHA, - TLS1_CK_DHE_PSK_WITH_RC4_128_SHA, - SSL_kDHEPSK, - SSL_aPSK, - SSL_RC4, - SSL_SHA1, - SSL3_VERSION, TLS1_2_VERSION, - 0, 0, - SSL_NOT_DEFAULT | SSL_MEDIUM, - SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF, - 80, - 128, - }, #endif /* OPENSSL_NO_WEAK_SSL_CIPHERS */ +#ifndef OPENSSL_NO_NTLS +# ifndef OPENSSL_NO_SM4 +# ifndef OPENSSL_NO_SM3 +# ifndef OPENSSL_NO_SM2 { 1, - TLS1_TXT_RSA_WITH_ARIA_128_GCM_SHA256, - TLS1_RFC_RSA_WITH_ARIA_128_GCM_SHA256, - TLS1_CK_RSA_WITH_ARIA_128_GCM_SHA256, - SSL_kRSA, - SSL_aRSA, - SSL_ARIA128GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + NTLS_TXT_ECDHE_SM2_SM4_CBC_SM3, + NTLS_GB_ECDHE_SM2_SM4_CBC_SM3, + NTLS_CK_ECDHE_SM2_SM4_CBC_SM3, + SSL_kSM2DHE, + SSL_aSM2, + SSL_SM4, + SSL_SM3, + NTLS_VERSION, NTLS_VERSION, + 0, 0, + SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, 128, 128, }, { 1, - TLS1_TXT_RSA_WITH_ARIA_256_GCM_SHA384, - TLS1_RFC_RSA_WITH_ARIA_256_GCM_SHA384, - TLS1_CK_RSA_WITH_ARIA_256_GCM_SHA384, - SSL_kRSA, - SSL_aRSA, - SSL_ARIA256GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - { - 1, - TLS1_TXT_DHE_RSA_WITH_ARIA_128_GCM_SHA256, - TLS1_RFC_DHE_RSA_WITH_ARIA_128_GCM_SHA256, - TLS1_CK_DHE_RSA_WITH_ARIA_128_GCM_SHA256, - SSL_kDHE, - SSL_aRSA, - SSL_ARIA128GCM, + NTLS_TXT_ECDHE_SM2_SM4_GCM_SM3, + NTLS_GB_ECDHE_SM2_SM4_GCM_SM3, + NTLS_CK_ECDHE_SM2_SM4_GCM_SM3, + SSL_kSM2DHE, + SSL_aSM2, + SSL_SM4GCM, SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + NTLS_VERSION, NTLS_VERSION, + 0, 0, + SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, 128, 128, - }, - { - 1, - TLS1_TXT_DHE_RSA_WITH_ARIA_256_GCM_SHA384, - TLS1_RFC_DHE_RSA_WITH_ARIA_256_GCM_SHA384, - TLS1_CK_DHE_RSA_WITH_ARIA_256_GCM_SHA384, - SSL_kDHE, - SSL_aRSA, - SSL_ARIA256GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, + }, { 1, - TLS1_TXT_DHE_DSS_WITH_ARIA_128_GCM_SHA256, - TLS1_RFC_DHE_DSS_WITH_ARIA_128_GCM_SHA256, - TLS1_CK_DHE_DSS_WITH_ARIA_128_GCM_SHA256, - SSL_kDHE, - SSL_aDSS, - SSL_ARIA128GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + NTLS_TXT_ECC_SM2_SM4_CBC_SM3, + NTLS_GB_ECC_SM2_SM4_CBC_SM3, + NTLS_CK_ECC_SM2_SM4_CBC_SM3, + SSL_kSM2, + SSL_aSM2, + SSL_SM4, + SSL_SM3, + NTLS_VERSION, NTLS_VERSION, + 0, 0, + SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, 128, 128, }, { 1, - TLS1_TXT_DHE_DSS_WITH_ARIA_256_GCM_SHA384, - TLS1_RFC_DHE_DSS_WITH_ARIA_256_GCM_SHA384, - TLS1_CK_DHE_DSS_WITH_ARIA_256_GCM_SHA384, - SSL_kDHE, - SSL_aDSS, - SSL_ARIA256GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - { - 1, - TLS1_TXT_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256, - TLS1_RFC_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256, - TLS1_CK_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256, - SSL_kECDHE, - SSL_aECDSA, - SSL_ARIA128GCM, + NTLS_TXT_ECC_SM2_SM4_GCM_SM3, + NTLS_GB_ECC_SM2_SM4_GCM_SM3, + NTLS_CK_ECC_SM2_SM4_GCM_SM3, + SSL_kSM2, + SSL_aSM2, + SSL_SM4GCM, SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + NTLS_VERSION, NTLS_VERSION, + 0, 0, + SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, 128, 128, - }, - { - 1, - TLS1_TXT_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384, - TLS1_RFC_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384, - TLS1_CK_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384, - SSL_kECDHE, - SSL_aECDSA, - SSL_ARIA256GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, + }, +# endif /* OPENSSL_NO_SM2 */ { 1, - TLS1_TXT_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256, - TLS1_RFC_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256, - TLS1_CK_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256, - SSL_kECDHE, + NTLS_TXT_RSA_SM4_CBC_SM3, + NTLS_GB_RSA_SM4_CBC_SM3, + NTLS_CK_RSA_SM4_CBC_SM3, + SSL_kRSA, SSL_aRSA, - SSL_ARIA128GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + SSL_SM4, + SSL_SM3, + NTLS_VERSION, NTLS_VERSION, + 0, 0, + SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, 128, 128, - }, + }, { 1, - TLS1_TXT_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384, - TLS1_RFC_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384, - TLS1_CK_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384, - SSL_kECDHE, + NTLS_TXT_RSA_SM4_GCM_SM3, + NTLS_GB_RSA_SM4_GCM_SM3, + NTLS_CK_RSA_SM4_GCM_SM3, + SSL_kRSA, SSL_aRSA, - SSL_ARIA256GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, - { - 1, - TLS1_TXT_PSK_WITH_ARIA_128_GCM_SHA256, - TLS1_RFC_PSK_WITH_ARIA_128_GCM_SHA256, - TLS1_CK_PSK_WITH_ARIA_128_GCM_SHA256, - SSL_kPSK, - SSL_aPSK, - SSL_ARIA128GCM, + SSL_SM4GCM, SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, + NTLS_VERSION, NTLS_VERSION, + 0, 0, + SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, 128, 128, - }, - { - 1, - TLS1_TXT_PSK_WITH_ARIA_256_GCM_SHA384, - TLS1_RFC_PSK_WITH_ARIA_256_GCM_SHA384, - TLS1_CK_PSK_WITH_ARIA_256_GCM_SHA384, - SSL_kPSK, - SSL_aPSK, - SSL_ARIA256GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, + }, +# endif /* OPENSSL_NO_SM3 */ { 1, - TLS1_TXT_DHE_PSK_WITH_ARIA_128_GCM_SHA256, - TLS1_RFC_DHE_PSK_WITH_ARIA_128_GCM_SHA256, - TLS1_CK_DHE_PSK_WITH_ARIA_128_GCM_SHA256, - SSL_kDHEPSK, - SSL_aPSK, - SSL_ARIA128GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, + NTLS_TXT_RSA_SM4_CBC_SHA256, + NTLS_GB_RSA_SM4_CBC_SHA256, + NTLS_CK_RSA_SM4_CBC_SHA256, + SSL_kRSA, + SSL_aRSA, + SSL_SM4, + SSL_SHA256, + NTLS_VERSION, NTLS_VERSION, + 0, 0, + SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, 128, 128, - }, - { - 1, - TLS1_TXT_DHE_PSK_WITH_ARIA_256_GCM_SHA384, - TLS1_RFC_DHE_PSK_WITH_ARIA_256_GCM_SHA384, - TLS1_CK_DHE_PSK_WITH_ARIA_256_GCM_SHA384, - SSL_kDHEPSK, - SSL_aPSK, - SSL_ARIA256GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, + }, { 1, - TLS1_TXT_RSA_PSK_WITH_ARIA_128_GCM_SHA256, - TLS1_RFC_RSA_PSK_WITH_ARIA_128_GCM_SHA256, - TLS1_CK_RSA_PSK_WITH_ARIA_128_GCM_SHA256, - SSL_kRSAPSK, + NTLS_TXT_RSA_SM4_GCM_SHA256, + NTLS_GB_RSA_SM4_GCM_SHA256, + NTLS_CK_RSA_SM4_GCM_SHA256, + SSL_kRSA, SSL_aRSA, - SSL_ARIA128GCM, + SSL_SM4GCM, SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, + NTLS_VERSION, NTLS_VERSION, + 0, 0, + SSL_HIGH, SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256, 128, 128, - }, - { - 1, - TLS1_TXT_RSA_PSK_WITH_ARIA_256_GCM_SHA384, - TLS1_RFC_RSA_PSK_WITH_ARIA_256_GCM_SHA384, - TLS1_CK_RSA_PSK_WITH_ARIA_256_GCM_SHA384, - SSL_kRSAPSK, - SSL_aRSA, - SSL_ARIA256GCM, - SSL_AEAD, - TLS1_2_VERSION, TLS1_2_VERSION, - DTLS1_2_VERSION, DTLS1_2_VERSION, - SSL_NOT_DEFAULT | SSL_HIGH, - SSL_HANDSHAKE_MAC_SHA384 | TLS1_PRF_SHA384, - 256, - 256, - }, + }, +# endif /* OPENSSL_NO_SM4 */ +#endif /* OPENSSL_NO_NTLS */ }; /* @@ -3255,9 +2524,9 @@ void ssl_sort_cipher_list(void) qsort(ssl3_scsvs, SSL3_NUM_SCSVS, sizeof(ssl3_scsvs[0]), cipher_compare); } -static int sslcon_undefined_function_1(SSL_CONNECTION *sc, unsigned char *r, - size_t s, const char *t, size_t u, - const unsigned char *v, size_t w, int x) +static int ssl_undefined_function_1(SSL *ssl, unsigned char *r, size_t s, + const char * t, size_t u, + const unsigned char * v, size_t w, int x) { (void)r; (void)s; @@ -3266,10 +2535,12 @@ static int sslcon_undefined_function_1(SSL_CONNECTION *sc, unsigned char *r, (void)v; (void)w; (void)x; - return ssl_undefined_function(SSL_CONNECTION_GET_SSL(sc)); + return ssl_undefined_function(ssl); } const SSL3_ENC_METHOD SSLv3_enc_data = { + ssl3_enc, + n_ssl3_mac, ssl3_setup_key_block, ssl3_generate_master_secret, ssl3_change_cipher_state, @@ -3277,20 +2548,20 @@ const SSL3_ENC_METHOD SSLv3_enc_data = { SSL3_MD_CLIENT_FINISHED_CONST, 4, SSL3_MD_SERVER_FINISHED_CONST, 4, ssl3_alert_code, - sslcon_undefined_function_1, + ssl_undefined_function_1, 0, ssl3_set_handshake_header, tls_close_construct_packet, ssl3_handshake_write }; -OSSL_TIME ssl3_default_timeout(void) +long ssl3_default_timeout(void) { /* * 2 hours, the 24 hours mentioned in the SSLv3 spec is way too long for * http, the cache would over fill */ - return ossl_seconds2time(60 * 60 * 2); + return (60 * 60 * 2); } int ssl3_num_ciphers(void) @@ -3306,7 +2577,7 @@ const SSL_CIPHER *ssl3_get_cipher(unsigned int u) return NULL; } -int ssl3_set_handshake_header(SSL_CONNECTION *s, WPACKET *pkt, int htype) +int ssl3_set_handshake_header(SSL *s, WPACKET *pkt, int htype) { /* No header in the event of a CCS */ if (htype == SSL3_MT_CHANGE_CIPHER_SPEC) @@ -3320,7 +2591,7 @@ int ssl3_set_handshake_header(SSL_CONNECTION *s, WPACKET *pkt, int htype) return 1; } -int ssl3_handshake_write(SSL_CONNECTION *s) +int ssl3_handshake_write(SSL *s) { return ssl3_do_write(s, SSL3_RT_HANDSHAKE); } @@ -3328,12 +2599,7 @@ int ssl3_handshake_write(SSL_CONNECTION *s) int ssl3_new(SSL *s) { #ifndef OPENSSL_NO_SRP - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (!ssl_srp_ctx_init_intern(sc)) + if (!ssl_srp_ctx_init_intern(s)) return 0; #endif @@ -3345,84 +2611,68 @@ int ssl3_new(SSL *s) void ssl3_free(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) + if (s == NULL) return; - ssl3_cleanup_key_block(sc); - - EVP_PKEY_free(sc->s3.peer_tmp); - sc->s3.peer_tmp = NULL; - EVP_PKEY_free(sc->s3.tmp.pkey); - sc->s3.tmp.pkey = NULL; + ssl3_cleanup_key_block(s); - ssl_evp_cipher_free(sc->s3.tmp.new_sym_enc); - ssl_evp_md_free(sc->s3.tmp.new_hash); + EVP_PKEY_free(s->s3.peer_tmp); + s->s3.peer_tmp = NULL; + EVP_PKEY_free(s->s3.tmp.pkey); + s->s3.tmp.pkey = NULL; - OPENSSL_free(sc->s3.tmp.ctype); - sk_X509_NAME_pop_free(sc->s3.tmp.peer_ca_names, X509_NAME_free); - OPENSSL_free(sc->s3.tmp.ciphers_raw); - OPENSSL_clear_free(sc->s3.tmp.pms, sc->s3.tmp.pmslen); - OPENSSL_free(sc->s3.tmp.peer_sigalgs); - OPENSSL_free(sc->s3.tmp.peer_cert_sigalgs); - OPENSSL_free(sc->s3.tmp.valid_flags); - ssl3_free_digest_list(sc); - OPENSSL_free(sc->s3.alpn_selected); - OPENSSL_free(sc->s3.alpn_proposed); + ssl_evp_cipher_free(s->s3.tmp.new_sym_enc); + ssl_evp_md_free(s->s3.tmp.new_hash); -#ifndef OPENSSL_NO_PSK - OPENSSL_free(sc->s3.tmp.psk); + OPENSSL_free(s->s3.tmp.ctype); + sk_X509_NAME_pop_free(s->s3.tmp.peer_ca_names, X509_NAME_free); + OPENSSL_free(s->s3.tmp.ciphers_raw); + OPENSSL_clear_free(s->s3.tmp.pms, s->s3.tmp.pmslen); + OPENSSL_free(s->s3.tmp.peer_sigalgs); + OPENSSL_free(s->s3.tmp.peer_cert_sigalgs); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + OPENSSL_free(s->s3.tmp.peer_dc_sigalgs); #endif + ssl3_free_digest_list(s); + OPENSSL_free(s->s3.alpn_selected); + OPENSSL_free(s->s3.alpn_proposed); #ifndef OPENSSL_NO_SRP - ssl_srp_ctx_free_intern(sc); + ssl_srp_ctx_free_intern(s); #endif - memset(&sc->s3, 0, sizeof(sc->s3)); + memset(&s->s3, 0, sizeof(s->s3)); } int ssl3_clear(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - int flags; - - if (sc == NULL) - return 0; - - ssl3_cleanup_key_block(sc); - OPENSSL_free(sc->s3.tmp.ctype); - sk_X509_NAME_pop_free(sc->s3.tmp.peer_ca_names, X509_NAME_free); - OPENSSL_free(sc->s3.tmp.ciphers_raw); - OPENSSL_clear_free(sc->s3.tmp.pms, sc->s3.tmp.pmslen); - OPENSSL_free(sc->s3.tmp.peer_sigalgs); - OPENSSL_free(sc->s3.tmp.peer_cert_sigalgs); - OPENSSL_free(sc->s3.tmp.valid_flags); + ssl3_cleanup_key_block(s); + OPENSSL_free(s->s3.tmp.ctype); + sk_X509_NAME_pop_free(s->s3.tmp.peer_ca_names, X509_NAME_free); + OPENSSL_free(s->s3.tmp.ciphers_raw); + OPENSSL_clear_free(s->s3.tmp.pms, s->s3.tmp.pmslen); + OPENSSL_free(s->s3.tmp.peer_sigalgs); + OPENSSL_free(s->s3.tmp.peer_cert_sigalgs); - EVP_PKEY_free(sc->s3.tmp.pkey); - EVP_PKEY_free(sc->s3.peer_tmp); + EVP_PKEY_free(s->s3.tmp.pkey); + EVP_PKEY_free(s->s3.peer_tmp); - ssl3_free_digest_list(sc); + ssl3_free_digest_list(s); - OPENSSL_free(sc->s3.alpn_selected); - OPENSSL_free(sc->s3.alpn_proposed); + OPENSSL_free(s->s3.alpn_selected); + OPENSSL_free(s->s3.alpn_proposed); - /* - * NULL/zero-out everything in the s3 struct, but remember if we are doing - * QUIC. - */ - flags = sc->s3.flags & TLS1_FLAGS_QUIC; - memset(&sc->s3, 0, sizeof(sc->s3)); - sc->s3.flags |= flags; + /* NULL/zero-out everything in the s3 struct */ + memset(&s->s3, 0, sizeof(s->s3)); - if (!ssl_free_wbio_buffer(sc)) + if (!ssl_free_wbio_buffer(s)) return 0; - sc->version = SSL3_VERSION; + s->version = SSL3_VERSION; #if !defined(OPENSSL_NO_NEXTPROTONEG) - OPENSSL_free(sc->ext.npn); - sc->ext.npn = NULL; - sc->ext.npn_len = 0; + OPENSSL_free(s->ext.npn); + s->ext.npn = NULL; + s->ext.npn_len = 0; #endif return 1; @@ -3431,12 +2681,7 @@ int ssl3_clear(SSL *s) #ifndef OPENSSL_NO_SRP static char *srp_password_from_info_cb(SSL *s, void *arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - return OPENSSL_strdup(sc->srp_ctx.info); + return OPENSSL_strdup(s->srp_ctx.info); } #endif @@ -3445,26 +2690,22 @@ static int ssl3_set_req_cert_type(CERT *c, const unsigned char *p, size_t len); long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) { int ret = 0; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return ret; switch (cmd) { case SSL_CTRL_GET_CLIENT_CERT_REQUEST: break; case SSL_CTRL_GET_NUM_RENEGOTIATIONS: - ret = sc->s3.num_renegotiations; + ret = s->s3.num_renegotiations; break; case SSL_CTRL_CLEAR_NUM_RENEGOTIATIONS: - ret = sc->s3.num_renegotiations; - sc->s3.num_renegotiations = 0; + ret = s->s3.num_renegotiations; + s->s3.num_renegotiations = 0; break; case SSL_CTRL_GET_TOTAL_RENEGOTIATIONS: - ret = sc->s3.total_renegotiations; + ret = s->s3.total_renegotiations; break; case SSL_CTRL_GET_FLAGS: - ret = (int)(sc->s3.flags); + ret = (int)(s->s3.flags); break; #if !defined(OPENSSL_NO_DEPRECATED_3_0) case SSL_CTRL_SET_TMP_DH: @@ -3476,7 +2717,7 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) } pkdh = ssl_dh_to_pkey(parg); if (pkdh == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_DH_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } if (!SSL_set0_tmp_dh_pkey(s, pkdh)) { @@ -3493,7 +2734,7 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) } #endif case SSL_CTRL_SET_DH_AUTO: - sc->cert->dh_tmp_auto = larg; + s->cert->dh_tmp_auto = larg; return 1; #if !defined(OPENSSL_NO_DEPRECATED_3_0) case SSL_CTRL_SET_TMP_ECDH: @@ -3502,8 +2743,8 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); return 0; } - return ssl_set_tmp_ecdh_groups(&sc->ext.supportedgroups, - &sc->ext.supportedgroups_len, + return ssl_set_tmp_ecdh_groups(&s->ext.supportedgroups, + &s->ext.supportedgroups_len, parg); } #endif /* !OPENSSL_NO_DEPRECATED_3_0 */ @@ -3519,8 +2760,8 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) if (larg == TLSEXT_NAMETYPE_host_name) { size_t len; - OPENSSL_free(sc->ext.hostname); - sc->ext.hostname = NULL; + OPENSSL_free(s->ext.hostname); + s->ext.hostname = NULL; ret = 1; if (parg == NULL) @@ -3530,7 +2771,7 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) ERR_raise(ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_SERVERNAME); return 0; } - if ((sc->ext.hostname = OPENSSL_strdup((char *)parg)) == NULL) { + if ((s->ext.hostname = OPENSSL_strdup((char *)parg)) == NULL) { ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); return 0; } @@ -3540,79 +2781,79 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) } break; case SSL_CTRL_SET_TLSEXT_DEBUG_ARG: - sc->ext.debug_arg = parg; + s->ext.debug_arg = parg; ret = 1; break; case SSL_CTRL_GET_TLSEXT_STATUS_REQ_TYPE: - ret = sc->ext.status_type; + ret = s->ext.status_type; break; case SSL_CTRL_SET_TLSEXT_STATUS_REQ_TYPE: - sc->ext.status_type = larg; + s->ext.status_type = larg; ret = 1; break; case SSL_CTRL_GET_TLSEXT_STATUS_REQ_EXTS: - *(STACK_OF(X509_EXTENSION) **)parg = sc->ext.ocsp.exts; + *(STACK_OF(X509_EXTENSION) **)parg = s->ext.ocsp.exts; ret = 1; break; case SSL_CTRL_SET_TLSEXT_STATUS_REQ_EXTS: - sc->ext.ocsp.exts = parg; + s->ext.ocsp.exts = parg; ret = 1; break; case SSL_CTRL_GET_TLSEXT_STATUS_REQ_IDS: - *(STACK_OF(OCSP_RESPID) **)parg = sc->ext.ocsp.ids; + *(STACK_OF(OCSP_RESPID) **)parg = s->ext.ocsp.ids; ret = 1; break; case SSL_CTRL_SET_TLSEXT_STATUS_REQ_IDS: - sc->ext.ocsp.ids = parg; + s->ext.ocsp.ids = parg; ret = 1; break; case SSL_CTRL_GET_TLSEXT_STATUS_REQ_OCSP_RESP: - *(unsigned char **)parg = sc->ext.ocsp.resp; - if (sc->ext.ocsp.resp_len == 0 - || sc->ext.ocsp.resp_len > LONG_MAX) + *(unsigned char **)parg = s->ext.ocsp.resp; + if (s->ext.ocsp.resp_len == 0 + || s->ext.ocsp.resp_len > LONG_MAX) return -1; - return (long)sc->ext.ocsp.resp_len; + return (long)s->ext.ocsp.resp_len; case SSL_CTRL_SET_TLSEXT_STATUS_REQ_OCSP_RESP: - OPENSSL_free(sc->ext.ocsp.resp); - sc->ext.ocsp.resp = parg; - sc->ext.ocsp.resp_len = larg; + OPENSSL_free(s->ext.ocsp.resp); + s->ext.ocsp.resp = parg; + s->ext.ocsp.resp_len = larg; ret = 1; break; case SSL_CTRL_CHAIN: if (larg) - return ssl_cert_set1_chain(sc, NULL, (STACK_OF(X509) *)parg); + return ssl_cert_set1_chain(s, NULL, (STACK_OF(X509) *)parg); else - return ssl_cert_set0_chain(sc, NULL, (STACK_OF(X509) *)parg); + return ssl_cert_set0_chain(s, NULL, (STACK_OF(X509) *)parg); case SSL_CTRL_CHAIN_CERT: if (larg) - return ssl_cert_add1_chain_cert(sc, NULL, (X509 *)parg); + return ssl_cert_add1_chain_cert(s, NULL, (X509 *)parg); else - return ssl_cert_add0_chain_cert(sc, NULL, (X509 *)parg); + return ssl_cert_add0_chain_cert(s, NULL, (X509 *)parg); case SSL_CTRL_GET_CHAIN_CERTS: - *(STACK_OF(X509) **)parg = sc->cert->key->chain; + *(STACK_OF(X509) **)parg = s->cert->key->chain; ret = 1; break; case SSL_CTRL_SELECT_CURRENT_CERT: - return ssl_cert_select_current(sc->cert, (X509 *)parg); + return ssl_cert_select_current(s->cert, (X509 *)parg); case SSL_CTRL_SET_CURRENT_CERT: if (larg == SSL_CERT_SET_SERVER) { const SSL_CIPHER *cipher; - if (!sc->server) + if (!s->server) return 0; - cipher = sc->s3.tmp.new_cipher; + cipher = s->s3.tmp.new_cipher; if (cipher == NULL) return 0; /* @@ -3621,22 +2862,22 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) */ if (cipher->algorithm_auth & (SSL_aNULL | SSL_aSRP)) return 2; - if (sc->s3.tmp.cert == NULL) + if (s->s3.tmp.cert == NULL) return 0; - sc->cert->key = sc->s3.tmp.cert; + s->cert->key = s->s3.tmp.cert; return 1; } - return ssl_cert_set_current(sc->cert, larg); + return ssl_cert_set_current(s->cert, larg); case SSL_CTRL_GET_GROUPS: { uint16_t *clist; size_t clistlen; - if (!sc->session) + if (!s->session) return 0; - clist = sc->ext.peer_supportedgroups; - clistlen = sc->ext.peer_supportedgroups_len; + clist = s->ext.peer_supportedgroups; + clistlen = s->ext.peer_supportedgroups_len; if (parg) { size_t i; int *cptr = parg; @@ -3655,16 +2896,16 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) } case SSL_CTRL_SET_GROUPS: - return tls1_set_groups(&sc->ext.supportedgroups, - &sc->ext.supportedgroups_len, parg, larg); + return tls1_set_groups(&s->ext.supportedgroups, + &s->ext.supportedgroups_len, parg, larg); case SSL_CTRL_SET_GROUPS_LIST: - return tls1_set_groups_list(s->ctx, &sc->ext.supportedgroups, - &sc->ext.supportedgroups_len, parg); + return tls1_set_groups_list(s->ctx, &s->ext.supportedgroups, + &s->ext.supportedgroups_len, parg); case SSL_CTRL_GET_SHARED_GROUP: { - uint16_t id = tls1_shared_group(sc, larg); + uint16_t id = tls1_shared_group(s, larg); if (larg != -1) return tls1_group_id2nid(id, 1); @@ -3674,82 +2915,82 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) { unsigned int id; - if (SSL_CONNECTION_IS_TLS13(sc) && sc->s3.did_kex) - id = sc->s3.group_id; + if (SSL_IS_TLS13(s) && s->s3.did_kex) + id = s->s3.group_id; else - id = sc->session->kex_group; + id = s->session->kex_group; ret = tls1_group_id2nid(id, 1); break; } case SSL_CTRL_SET_SIGALGS: - return tls1_set_sigalgs(sc->cert, parg, larg, 0); + return tls1_set_sigalgs(s->cert, parg, larg, 0); case SSL_CTRL_SET_SIGALGS_LIST: - return tls1_set_sigalgs_list(s->ctx, sc->cert, parg, 0); + return tls1_set_sigalgs_list(s->cert, parg, 0); case SSL_CTRL_SET_CLIENT_SIGALGS: - return tls1_set_sigalgs(sc->cert, parg, larg, 1); + return tls1_set_sigalgs(s->cert, parg, larg, 1); case SSL_CTRL_SET_CLIENT_SIGALGS_LIST: - return tls1_set_sigalgs_list(s->ctx, sc->cert, parg, 1); + return tls1_set_sigalgs_list(s->cert, parg, 1); case SSL_CTRL_GET_CLIENT_CERT_TYPES: { const unsigned char **pctype = parg; - if (sc->server || !sc->s3.tmp.cert_req) + if (s->server || !s->s3.tmp.cert_req) return 0; if (pctype) - *pctype = sc->s3.tmp.ctype; - return sc->s3.tmp.ctype_len; + *pctype = s->s3.tmp.ctype; + return s->s3.tmp.ctype_len; } case SSL_CTRL_SET_CLIENT_CERT_TYPES: - if (!sc->server) + if (!s->server) return 0; - return ssl3_set_req_cert_type(sc->cert, parg, larg); + return ssl3_set_req_cert_type(s->cert, parg, larg); case SSL_CTRL_BUILD_CERT_CHAIN: - return ssl_build_cert_chain(sc, NULL, larg); + return ssl_build_cert_chain(s, NULL, larg); case SSL_CTRL_SET_VERIFY_CERT_STORE: - return ssl_cert_set_cert_store(sc->cert, parg, 0, larg); + return ssl_cert_set_cert_store(s->cert, parg, 0, larg); case SSL_CTRL_SET_CHAIN_CERT_STORE: - return ssl_cert_set_cert_store(sc->cert, parg, 1, larg); + return ssl_cert_set_cert_store(s->cert, parg, 1, larg); case SSL_CTRL_GET_VERIFY_CERT_STORE: - return ssl_cert_get_cert_store(sc->cert, parg, 0); + return ssl_cert_get_cert_store(s->cert, parg, 0); case SSL_CTRL_GET_CHAIN_CERT_STORE: - return ssl_cert_get_cert_store(sc->cert, parg, 1); + return ssl_cert_get_cert_store(s->cert, parg, 1); case SSL_CTRL_GET_PEER_SIGNATURE_NID: - if (sc->s3.tmp.peer_sigalg == NULL) + if (s->s3.tmp.peer_sigalg == NULL) return 0; - *(int *)parg = sc->s3.tmp.peer_sigalg->hash; + *(int *)parg = s->s3.tmp.peer_sigalg->hash; return 1; case SSL_CTRL_GET_SIGNATURE_NID: - if (sc->s3.tmp.sigalg == NULL) + if (s->s3.tmp.sigalg == NULL) return 0; - *(int *)parg = sc->s3.tmp.sigalg->hash; + *(int *)parg = s->s3.tmp.sigalg->hash; return 1; case SSL_CTRL_GET_PEER_TMP_KEY: - if (sc->session == NULL || sc->s3.peer_tmp == NULL) { + if (s->session == NULL || s->s3.peer_tmp == NULL) { return 0; } else { - EVP_PKEY_up_ref(sc->s3.peer_tmp); - *(EVP_PKEY **)parg = sc->s3.peer_tmp; + EVP_PKEY_up_ref(s->s3.peer_tmp); + *(EVP_PKEY **)parg = s->s3.peer_tmp; return 1; } case SSL_CTRL_GET_TMP_KEY: - if (sc->session == NULL || sc->s3.tmp.pkey == NULL) { + if (s->session == NULL || s->s3.tmp.pkey == NULL) { return 0; } else { - EVP_PKEY_up_ref(sc->s3.tmp.pkey); - *(EVP_PKEY **)parg = sc->s3.tmp.pkey; + EVP_PKEY_up_ref(s->s3.tmp.pkey); + *(EVP_PKEY **)parg = s->s3.tmp.pkey; return 1; } @@ -3757,24 +2998,12 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) { const unsigned char **pformat = parg; - if (sc->ext.peer_ecpointformats == NULL) + if (s->ext.peer_ecpointformats == NULL) return 0; - *pformat = sc->ext.peer_ecpointformats; - return (int)sc->ext.peer_ecpointformats_len; - } - - case SSL_CTRL_GET_IANA_GROUPS: - { - if (parg != NULL) { - *(uint16_t **)parg = (uint16_t *)sc->ext.peer_supportedgroups; - } - return (int)sc->ext.peer_supportedgroups_len; + *pformat = s->ext.peer_ecpointformats; + return (int)s->ext.peer_ecpointformats_len; } - case SSL_CTRL_SET_MSG_CALLBACK_ARG: - sc->msg_callback_arg = parg; - return 1; - default: break; } @@ -3784,32 +3013,24 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg) long ssl3_callback_ctrl(SSL *s, int cmd, void (*fp) (void)) { int ret = 0; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return ret; switch (cmd) { #if !defined(OPENSSL_NO_DEPRECATED_3_0) case SSL_CTRL_SET_TMP_DH_CB: - sc->cert->dh_tmp_cb = (DH *(*)(SSL *, int, int))fp; + s->cert->dh_tmp_cb = (DH *(*)(SSL *, int, int))fp; ret = 1; break; #endif case SSL_CTRL_SET_TLSEXT_DEBUG_CB: - sc->ext.debug_cb = (void (*)(SSL *, int, int, - const unsigned char *, int, void *))fp; + s->ext.debug_cb = (void (*)(SSL *, int, int, + const unsigned char *, int, void *))fp; ret = 1; break; case SSL_CTRL_SET_NOT_RESUMABLE_SESS_CB: - sc->not_resumable_session_cb = (int (*)(SSL *, int))fp; + s->not_resumable_session_cb = (int (*)(SSL *, int))fp; ret = 1; break; - - case SSL_CTRL_SET_MSG_CALLBACK: - sc->msg_callback = (ossl_msg_cb)fp; - return 1; default: break; } @@ -3829,7 +3050,7 @@ long ssl3_ctx_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg) } pkdh = ssl_dh_to_pkey(parg); if (pkdh == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_DH_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } if (!SSL_CTX_set0_tmp_dh_pkey(ctx, pkdh)) { @@ -3968,13 +3189,13 @@ long ssl3_ctx_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg) return tls1_set_sigalgs(ctx->cert, parg, larg, 0); case SSL_CTRL_SET_SIGALGS_LIST: - return tls1_set_sigalgs_list(ctx, ctx->cert, parg, 0); + return tls1_set_sigalgs_list(ctx->cert, parg, 0); case SSL_CTRL_SET_CLIENT_SIGALGS: return tls1_set_sigalgs(ctx->cert, parg, larg, 1); case SSL_CTRL_SET_CLIENT_SIGALGS_LIST: - return tls1_set_sigalgs_list(ctx, ctx->cert, parg, 1); + return tls1_set_sigalgs_list(ctx->cert, parg, 1); case SSL_CTRL_SET_CLIENT_CERT_TYPES: return ssl3_set_req_cert_type(ctx->cert, parg, larg); @@ -3998,12 +3219,12 @@ long ssl3_ctx_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg) case SSL_CTRL_EXTRA_CHAIN_CERT: if (ctx->extra_certs == NULL) { if ((ctx->extra_certs = sk_X509_new_null()) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } } if (!sk_X509_push(ctx->extra_certs, (X509 *)parg)) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } break; @@ -4016,7 +3237,7 @@ long ssl3_ctx_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg) break; case SSL_CTRL_CLEAR_EXTRA_CHAIN_CERTS: - OSSL_STACK_OF_X509_free(ctx->extra_certs); + sk_X509_pop_free(ctx->extra_certs, X509_free); ctx->extra_certs = NULL; break; @@ -4069,9 +3290,9 @@ long ssl3_ctx_callback_ctrl(SSL_CTX *ctx, int cmd, void (*fp) (void)) # ifndef OPENSSL_NO_DEPRECATED_3_0 case SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB: ctx->ext.ticket_key_cb = (int (*)(SSL *, unsigned char *, - unsigned char *, - EVP_CIPHER_CTX *, - HMAC_CTX *, int))fp; + unsigned char *, + EVP_CIPHER_CTX *, + HMAC_CTX *, int))fp; break; #endif @@ -4178,7 +3399,7 @@ int ssl3_put_cipher_by_char(const SSL_CIPHER *c, WPACKET *pkt, size_t *len) * * Returns the selected cipher or NULL when no common ciphers. */ -const SSL_CIPHER *ssl3_choose_cipher(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *clnt, +const SSL_CIPHER *ssl3_choose_cipher(SSL *s, STACK_OF(SSL_CIPHER) *clnt, STACK_OF(SSL_CIPHER) *srvr) { const SSL_CIPHER *c, *ret = NULL; @@ -4186,6 +3407,9 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *cl int i, ii, ok, prefer_sha256 = 0; unsigned long alg_k = 0, alg_a = 0, mask_k = 0, mask_a = 0; STACK_OF(SSL_CIPHER) *prio_chacha = NULL; +#ifndef OPENSSL_NO_OPTIMIZE_CHACHA_CHOOSE + int use_chacha = 0; +#endif /* Let's see which ciphers we can support */ @@ -4262,9 +3486,9 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *cl allow = srvr; } - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { #ifndef OPENSSL_NO_PSK - size_t j; + int j; /* * If we allow "old" style PSK callbacks, and we have no certificate (so @@ -4274,8 +3498,8 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *cl * that. */ if (s->psk_server_callback != NULL) { - for (j = 0; j < s->ssl_pkey_num && !ssl_has_cert(s, j); j++); - if (j == s->ssl_pkey_num) { + for (j = 0; j < SSL_PKEY_NUM && !ssl_has_cert(s, j); j++); + if (j == SSL_PKEY_NUM) { /* There are no certificates */ prefer_sha256 = 1; } @@ -4286,23 +3510,46 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *cl ssl_set_masks(s); } +#ifndef OPENSSL_NO_OPTIMIZE_CHACHA_CHOOSE +retry: +#endif for (i = 0; i < sk_SSL_CIPHER_num(prio); i++) { - int minversion, maxversion; - c = sk_SSL_CIPHER_value(prio, i); - minversion = SSL_CONNECTION_IS_DTLS(s) ? c->min_dtls : c->min_tls; - maxversion = SSL_CONNECTION_IS_DTLS(s) ? c->max_dtls : c->max_tls; /* Skip ciphers not supported by the protocol version */ - if (ssl_version_cmp(s, s->version, minversion) < 0 - || ssl_version_cmp(s, s->version, maxversion) > 0) + if (!SSL_IS_DTLS(s) && + ((s->version < c->min_tls) || (s->version > c->max_tls))) + continue; + if (SSL_IS_DTLS(s) && + (DTLS_VERSION_LT(s->version, c->min_dtls) || + DTLS_VERSION_GT(s->version, c->max_dtls))) + continue; + +#ifndef OPENSSL_NO_OPTIMIZE_CHACHA_CHOOSE + if ((s->options & SSL_OP_PRIORITIZE_CHACHA) && prio_chacha == NULL && + c->algorithm_enc == SSL_CHACHA20POLY1305 && use_chacha == 0) continue; +#endif + +#ifndef OPENSSL_NO_SM2 + /* + * RFC 8998 demand that server can use + * "TLS_SM4_GCM_SM3" and "TLS_SM4_CCM_SM3" with sm2 cert only + */ + if (s->enable_sm_tls13_strict) { + if (c->id == TLS1_3_CK_SM4_GCM_SM3 + || c->id == TLS1_3_CK_SM4_CCM_SM3) { + if (!ssl_has_cert(s, SSL_PKEY_SM2)) + continue; + } + } +#endif /* * Since TLS 1.3 ciphersuites can be used with any auth or * key exchange scheme skip tests. */ - if (!SSL_CONNECTION_IS_TLS13(s)) { + if (!SSL_IS_TLS13(s)) { mask_k = s->s3.tmp.mask_k; mask_a = s->s3.tmp.mask_a; #ifndef OPENSSL_NO_SRP @@ -4352,11 +3599,9 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *cl if (prefer_sha256) { const SSL_CIPHER *tmp = sk_SSL_CIPHER_value(allow, ii); - const EVP_MD *md = ssl_md(SSL_CONNECTION_GET_CTX(s), - tmp->algorithm2); - if (md != NULL - && EVP_MD_is_a(md, OSSL_DIGEST_NAME_SHA2_256)) { + if (EVP_MD_is_a(ssl_md(s->ctx, tmp->algorithm2), + OSSL_DIGEST_NAME_SHA2_256)) { ret = tmp; break; } @@ -4369,12 +3614,20 @@ const SSL_CIPHER *ssl3_choose_cipher(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *cl } } +#ifndef OPENSSL_NO_OPTIMIZE_CHACHA_CHOOSE + if (ret == NULL && s->options & SSL_OP_PRIORITIZE_CHACHA && + prio_chacha == NULL && use_chacha == 0) { + use_chacha = 1; + goto retry; + } +#endif + sk_SSL_CIPHER_free(prio_chacha); return ret; } -int ssl3_get_req_cert_type(SSL_CONNECTION *s, WPACKET *pkt) +int ssl3_get_req_cert_type(SSL *s, WPACKET *pkt) { uint32_t alg_k, alg_a = 0; @@ -4386,21 +3639,6 @@ int ssl3_get_req_cert_type(SSL_CONNECTION *s, WPACKET *pkt) alg_k = s->s3.tmp.new_cipher->algorithm_mkey; -#ifndef OPENSSL_NO_GOST - if (s->version >= TLS1_VERSION && (alg_k & SSL_kGOST)) - if (!WPACKET_put_bytes_u8(pkt, TLS_CT_GOST01_SIGN) - || !WPACKET_put_bytes_u8(pkt, TLS_CT_GOST12_IANA_SIGN) - || !WPACKET_put_bytes_u8(pkt, TLS_CT_GOST12_IANA_512_SIGN) - || !WPACKET_put_bytes_u8(pkt, TLS_CT_GOST12_LEGACY_SIGN) - || !WPACKET_put_bytes_u8(pkt, TLS_CT_GOST12_LEGACY_512_SIGN)) - return 0; - - if (s->version >= TLS1_2_VERSION && (alg_k & SSL_kGOST18)) - if (!WPACKET_put_bytes_u8(pkt, TLS_CT_GOST12_IANA_SIGN) - || !WPACKET_put_bytes_u8(pkt, TLS_CT_GOST12_IANA_512_SIGN)) - return 0; -#endif - if ((s->version == SSL3_VERSION) && (alg_k & SSL_kDHE)) { if (!WPACKET_put_bytes_u8(pkt, SSL3_CT_RSA_EPHEMERAL_DH)) return 0; @@ -4410,14 +3648,23 @@ int ssl3_get_req_cert_type(SSL_CONNECTION *s, WPACKET *pkt) } if (!(alg_a & SSL_aRSA) && !WPACKET_put_bytes_u8(pkt, SSL3_CT_RSA_SIGN)) return 0; - if (!(alg_a & SSL_aDSS) && !WPACKET_put_bytes_u8(pkt, SSL3_CT_DSS_SIGN)) + if (!(alg_a & SSL_aDSS) +#ifndef OPENSSL_NO_NTLS + /* TLCP not define DSS sign */ + && !SSL_IS_NTLS(s) +#endif + && !WPACKET_put_bytes_u8(pkt, SSL3_CT_DSS_SIGN)) return 0; /* * ECDSA certs can be used with RSA cipher suites too so we don't * need to check for SSL_kECDH or SSL_kECDHE */ - if (s->version >= TLS1_VERSION + if ((s->version >= TLS1_VERSION +#if (!defined OPENSSL_NO_NTLS) && (!defined OPENSSL_NO_SM2) + || SSL_IS_NTLS(s) +#endif + ) && !(alg_a & SSL_aECDSA) && !WPACKET_put_bytes_u8(pkt, TLS_CT_ECDSA_SIGN)) return 0; @@ -4444,30 +3691,26 @@ static int ssl3_set_req_cert_type(CERT *c, const unsigned char *p, size_t len) int ssl3_shutdown(SSL *s) { int ret; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; /* * Don't do anything much if we have not done the handshake or we don't * want to send messages :-) */ - if (sc->quiet_shutdown || SSL_in_before(s)) { - sc->shutdown = (SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN); + if (s->quiet_shutdown || SSL_in_before(s)) { + s->shutdown = (SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN); return 1; } - if (!(sc->shutdown & SSL_SENT_SHUTDOWN)) { - sc->shutdown |= SSL_SENT_SHUTDOWN; - ssl3_send_alert(sc, SSL3_AL_WARNING, SSL_AD_CLOSE_NOTIFY); + if (!(s->shutdown & SSL_SENT_SHUTDOWN)) { + s->shutdown |= SSL_SENT_SHUTDOWN; + ssl3_send_alert(s, SSL3_AL_WARNING, SSL_AD_CLOSE_NOTIFY); /* * our shutdown alert has been sent now, and if it still needs to be - * written, s->s3.alert_dispatch will be > 0 + * written, s->s3.alert_dispatch will be true */ - if (sc->s3.alert_dispatch > 0) + if (s->s3.alert_dispatch) return -1; /* return WANT_WRITE */ - } else if (sc->s3.alert_dispatch > 0) { + } else if (s->s3.alert_dispatch) { /* resend it if not sent */ ret = s->method->ssl_dispatch_alert(s); if (ret == -1) { @@ -4478,19 +3721,19 @@ int ssl3_shutdown(SSL *s) */ return ret; } - } else if (!(sc->shutdown & SSL_RECEIVED_SHUTDOWN)) { + } else if (!(s->shutdown & SSL_RECEIVED_SHUTDOWN)) { size_t readbytes; /* * If we are waiting for a close from our peer, we are closed */ s->method->ssl_read_bytes(s, 0, NULL, NULL, 0, 0, &readbytes); - if (!(sc->shutdown & SSL_RECEIVED_SHUTDOWN)) { + if (!(s->shutdown & SSL_RECEIVED_SHUTDOWN)) { return -1; /* return WANT_READ */ } } - if ((sc->shutdown == (SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN)) - && sc->s3.alert_dispatch == SSL_ALERT_DISPATCH_NONE) + if ((s->shutdown == (SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN)) && + !s->s3.alert_dispatch) return 1; else return 0; @@ -4498,13 +3741,8 @@ int ssl3_shutdown(SSL *s) int ssl3_write(SSL *s, const void *buf, size_t len, size_t *written) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - clear_sys_error(); - if (sc->s3.renegotiate) + if (s->s3.renegotiate) ssl3_renegotiate_check(s, 0); return s->method->ssl_write_bytes(s, SSL3_RT_APPLICATION_DATA, buf, len, @@ -4515,19 +3753,15 @@ static int ssl3_read_internal(SSL *s, void *buf, size_t len, int peek, size_t *readbytes) { int ret; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; clear_sys_error(); - if (sc->s3.renegotiate) + if (s->s3.renegotiate) ssl3_renegotiate_check(s, 0); - sc->s3.in_read_app_data = 1; + s->s3.in_read_app_data = 1; ret = s->method->ssl_read_bytes(s, SSL3_RT_APPLICATION_DATA, NULL, buf, len, peek, readbytes); - if ((ret == -1) && (sc->s3.in_read_app_data == 2)) { + if ((ret == -1) && (s->s3.in_read_app_data == 2)) { /* * ssl3_read_bytes decided to call s->handshake_func, which called * ssl3_read_bytes to read handshake data. However, ssl3_read_bytes @@ -4535,13 +3769,13 @@ static int ssl3_read_internal(SSL *s, void *buf, size_t len, int peek, * makes sense here; so disable handshake processing and try to read * application data again. */ - ossl_statem_set_in_handshake(sc, 1); + ossl_statem_set_in_handshake(s, 1); ret = s->method->ssl_read_bytes(s, SSL3_RT_APPLICATION_DATA, NULL, buf, len, peek, readbytes); - ossl_statem_set_in_handshake(sc, 0); + ossl_statem_set_in_handshake(s, 0); } else - sc->s3.in_read_app_data = 0; + s->s3.in_read_app_data = 0; return ret; } @@ -4558,15 +3792,10 @@ int ssl3_peek(SSL *s, void *buf, size_t len, size_t *readbytes) int ssl3_renegotiate(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - - if (sc->handshake_func == NULL) + if (s->handshake_func == NULL) return 1; - sc->s3.renegotiate = 1; + s->s3.renegotiate = 1; return 1; } @@ -4581,24 +3810,20 @@ int ssl3_renegotiate(SSL *s) int ssl3_renegotiate_check(SSL *s, int initok) { int ret = 0; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - if (sc == NULL) - return 0; - - if (sc->s3.renegotiate) { - if (!RECORD_LAYER_read_pending(&sc->rlayer) - && !RECORD_LAYER_write_pending(&sc->rlayer) + if (s->s3.renegotiate) { + if (!RECORD_LAYER_read_pending(&s->rlayer) + && !RECORD_LAYER_write_pending(&s->rlayer) && (initok || !SSL_in_init(s))) { /* * if we are the server, and we have sent a 'RENEGOTIATE' * message, we need to set the state machine into the renegotiate * state. */ - ossl_statem_set_renegotiate(sc); - sc->s3.renegotiate = 0; - sc->s3.num_renegotiations++; - sc->s3.total_renegotiations++; + ossl_statem_set_renegotiate(s); + s->s3.renegotiate = 0; + s->s3.num_renegotiations++; + s->s3.total_renegotiations++; ret = 1; } } @@ -4611,15 +3836,13 @@ int ssl3_renegotiate_check(SSL *s, int initok) * * If PSK and using SHA384 for TLS < 1.2 switch to default. */ -long ssl_get_algorithm2(SSL_CONNECTION *s) +long ssl_get_algorithm2(SSL *s) { long alg2; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - if (s->s3.tmp.new_cipher == NULL) return -1; alg2 = s->s3.tmp.new_cipher->algorithm2; - if (ssl->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_SHA256_PRF) { + if (s->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_SHA256_PRF) { if (alg2 == (SSL_HANDSHAKE_MAC_DEFAULT | TLS1_PRF)) return SSL_HANDSHAKE_MAC_SHA256 | TLS1_PRF_SHA256; } else if (s->s3.tmp.new_cipher->algorithm_mkey & SSL_PSK) { @@ -4633,8 +3856,7 @@ long ssl_get_algorithm2(SSL_CONNECTION *s) * Fill a ClientRandom or ServerRandom field of length len. Returns <= 0 on * failure, 1 on success. */ -int ssl_fill_hello_random(SSL_CONNECTION *s, int server, - unsigned char *result, size_t len, +int ssl_fill_hello_random(SSL *s, int server, unsigned char *result, size_t len, DOWNGRADE dgrd) { int send_time = 0, ret; @@ -4650,9 +3872,9 @@ int ssl_fill_hello_random(SSL_CONNECTION *s, int server, unsigned char *p = result; l2n(Time, p); - ret = RAND_bytes_ex(SSL_CONNECTION_GET_CTX(s)->libctx, p, len - 4, 0); + ret = RAND_bytes_ex(s->ctx->libctx, p, len - 4, 0); } else { - ret = RAND_bytes_ex(SSL_CONNECTION_GET_CTX(s)->libctx, result, len, 0); + ret = RAND_bytes_ex(s->ctx->libctx, result, len, 0); } if (ret > 0) { @@ -4670,12 +3892,11 @@ int ssl_fill_hello_random(SSL_CONNECTION *s, int server, return ret; } -int ssl_generate_master_secret(SSL_CONNECTION *s, unsigned char *pms, - size_t pmslen, int free_pms) +int ssl_generate_master_secret(SSL *s, unsigned char *pms, size_t pmslen, + int free_pms) { unsigned long alg_k = s->s3.tmp.new_cipher->algorithm_mkey; int ret = 0; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if (alg_k & SSL_PSK) { #ifndef OPENSSL_NO_PSK @@ -4706,7 +3927,7 @@ int ssl_generate_master_secret(SSL_CONNECTION *s, unsigned char *pms, OPENSSL_clear_free(s->s3.tmp.psk, psklen); s->s3.tmp.psk = NULL; s->s3.tmp.psklen = 0; - if (!ssl->method->ssl3_enc->generate_master_secret(s, + if (!s->method->ssl3_enc->generate_master_secret(s, s->session->master_key, pskpms, pskpmslen, &s->session->master_key_length)) { OPENSSL_clear_free(pskpms, pskpmslen); @@ -4719,7 +3940,7 @@ int ssl_generate_master_secret(SSL_CONNECTION *s, unsigned char *pms, goto err; #endif } else { - if (!ssl->method->ssl3_enc->generate_master_secret(s, + if (!s->method->ssl3_enc->generate_master_secret(s, s->session->master_key, pms, pmslen, &s->session->master_key_length)) { /* SSLfatal() already called */ @@ -4743,15 +3964,14 @@ int ssl_generate_master_secret(SSL_CONNECTION *s, unsigned char *pms, } /* Generate a private key from parameters */ -EVP_PKEY *ssl_generate_pkey(SSL_CONNECTION *s, EVP_PKEY *pm) +EVP_PKEY *ssl_generate_pkey(SSL *s, EVP_PKEY *pm) { EVP_PKEY_CTX *pctx = NULL; EVP_PKEY *pkey = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (pm == NULL) return NULL; - pctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, pm, sctx->propq); + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, pm, s->ctx->propq); if (pctx == NULL) goto err; if (EVP_PKEY_keygen_init(pctx) <= 0) @@ -4767,10 +3987,9 @@ EVP_PKEY *ssl_generate_pkey(SSL_CONNECTION *s, EVP_PKEY *pm) } /* Generate a private key from a group ID */ -EVP_PKEY *ssl_generate_pkey_group(SSL_CONNECTION *s, uint16_t id) +EVP_PKEY *ssl_generate_pkey_group(SSL *s, uint16_t id) { - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - const TLS_GROUP_INFO *ginf = tls1_group_id_lookup(sctx, id); + const TLS_GROUP_INFO *ginf = tls1_group_id_lookup(s->ctx, id); EVP_PKEY_CTX *pctx = NULL; EVP_PKEY *pkey = NULL; @@ -4779,18 +3998,22 @@ EVP_PKEY *ssl_generate_pkey_group(SSL_CONNECTION *s, uint16_t id) goto err; } - pctx = EVP_PKEY_CTX_new_from_name(sctx->libctx, ginf->algorithm, - sctx->propq); + if (!SSL_is_server(s) && id == TLSEXT_curve_SM2) + pctx = EVP_PKEY_CTX_new_from_name(s->ctx->libctx, "EC", + s->ctx->propq); + else + pctx = EVP_PKEY_CTX_new_from_name(s->ctx->libctx, ginf->algorithm, + s->ctx->propq); if (pctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } if (EVP_PKEY_keygen_init(pctx) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); goto err; } - if (EVP_PKEY_CTX_set_group_name(pctx, ginf->realname) <= 0) { + if (!EVP_PKEY_CTX_set_group_name(pctx, ginf->realname)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); goto err; } @@ -4808,24 +4031,27 @@ EVP_PKEY *ssl_generate_pkey_group(SSL_CONNECTION *s, uint16_t id) /* * Generate parameters from a group ID */ -EVP_PKEY *ssl_generate_param_group(SSL_CONNECTION *s, uint16_t id) +EVP_PKEY *ssl_generate_param_group(SSL *s, uint16_t id) { - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); EVP_PKEY_CTX *pctx = NULL; EVP_PKEY *pkey = NULL; - const TLS_GROUP_INFO *ginf = tls1_group_id_lookup(sctx, id); + const TLS_GROUP_INFO *ginf = tls1_group_id_lookup(s->ctx, id); if (ginf == NULL) goto err; - pctx = EVP_PKEY_CTX_new_from_name(sctx->libctx, ginf->algorithm, - sctx->propq); + if (SSL_IS_TLS13(s) && id == TLSEXT_curve_SM2) + pctx = EVP_PKEY_CTX_new_from_name(s->ctx->libctx, "EC", + s->ctx->propq); + else + pctx = EVP_PKEY_CTX_new_from_name(s->ctx->libctx, ginf->algorithm, + s->ctx->propq); if (pctx == NULL) goto err; if (EVP_PKEY_paramgen_init(pctx) <= 0) goto err; - if (EVP_PKEY_CTX_set_group_name(pctx, ginf->realname) <= 0) { + if (!EVP_PKEY_CTX_set_group_name(pctx, ginf->realname)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); goto err; } @@ -4840,12 +4066,12 @@ EVP_PKEY *ssl_generate_param_group(SSL_CONNECTION *s, uint16_t id) } /* Generate secrets from pms */ -int ssl_gensecret(SSL_CONNECTION *s, unsigned char *pms, size_t pmslen) +int ssl_gensecret(SSL *s, unsigned char *pms, size_t pmslen) { int rv = 0; /* SSLfatal() called as appropriate in the below functions */ - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* * If we are resuming then we already generated the early secret * when we created the ClientHello, so don't recreate it. @@ -4866,20 +4092,19 @@ int ssl_gensecret(SSL_CONNECTION *s, unsigned char *pms, size_t pmslen) } /* Derive secrets for ECDH/DH */ -int ssl_derive(SSL_CONNECTION *s, EVP_PKEY *privkey, EVP_PKEY *pubkey, int gensecret) +int ssl_derive(SSL *s, EVP_PKEY *privkey, EVP_PKEY *pubkey, int gensecret) { int rv = 0; unsigned char *pms = NULL; size_t pmslen = 0; - EVP_PKEY_CTX *pctx; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); + EVP_PKEY_CTX *pctx = NULL; if (privkey == NULL || pubkey == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - pctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, privkey, sctx->propq); + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, privkey, s->ctx->propq); if (EVP_PKEY_derive_init(pctx) <= 0 || EVP_PKEY_derive_set_peer(pctx, pubkey) <= 0 @@ -4888,12 +4113,12 @@ int ssl_derive(SSL_CONNECTION *s, EVP_PKEY *privkey, EVP_PKEY *pubkey, int gense goto err; } - if (SSL_CONNECTION_IS_TLS13(s) && EVP_PKEY_is_a(privkey, "DH")) + if (SSL_IS_TLS13(s) && EVP_PKEY_is_a(privkey, "DH")) EVP_PKEY_CTX_set_dh_pad(pctx, 1); pms = OPENSSL_malloc(pmslen); if (pms == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -4920,7 +4145,7 @@ int ssl_derive(SSL_CONNECTION *s, EVP_PKEY *privkey, EVP_PKEY *pubkey, int gense } /* Decapsulate secrets for KEM */ -int ssl_decapsulate(SSL_CONNECTION *s, EVP_PKEY *privkey, +int ssl_decapsulate(SSL *s, EVP_PKEY *privkey, const unsigned char *ct, size_t ctlen, int gensecret) { @@ -4928,14 +4153,13 @@ int ssl_decapsulate(SSL_CONNECTION *s, EVP_PKEY *privkey, unsigned char *pms = NULL; size_t pmslen = 0; EVP_PKEY_CTX *pctx; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (privkey == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - pctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, privkey, sctx->propq); + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, privkey, s->ctx->propq); if (EVP_PKEY_decapsulate_init(pctx, NULL) <= 0 || EVP_PKEY_decapsulate(pctx, NULL, &pmslen, ct, ctlen) <= 0) { @@ -4945,7 +4169,7 @@ int ssl_decapsulate(SSL_CONNECTION *s, EVP_PKEY *privkey, pms = OPENSSL_malloc(pmslen); if (pms == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -4971,7 +4195,7 @@ int ssl_decapsulate(SSL_CONNECTION *s, EVP_PKEY *privkey, return rv; } -int ssl_encapsulate(SSL_CONNECTION *s, EVP_PKEY *pubkey, +int ssl_encapsulate(SSL *s, EVP_PKEY *pubkey, unsigned char **ctp, size_t *ctlenp, int gensecret) { @@ -4979,14 +4203,13 @@ int ssl_encapsulate(SSL_CONNECTION *s, EVP_PKEY *pubkey, unsigned char *pms = NULL, *ct = NULL; size_t pmslen = 0, ctlen = 0; EVP_PKEY_CTX *pctx; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (pubkey == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - pctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, pubkey, sctx->propq); + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, pubkey, s->ctx->propq); if (EVP_PKEY_encapsulate_init(pctx, NULL) <= 0 || EVP_PKEY_encapsulate(pctx, NULL, &ctlen, NULL, &pmslen) <= 0 @@ -4998,7 +4221,7 @@ int ssl_encapsulate(SSL_CONNECTION *s, EVP_PKEY *pubkey, pms = OPENSSL_malloc(pmslen); ct = OPENSSL_malloc(ctlen); if (pms == NULL || ct == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -5032,22 +4255,6 @@ int ssl_encapsulate(SSL_CONNECTION *s, EVP_PKEY *pubkey, return rv; } -const char *SSL_get0_group_name(SSL *s) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - unsigned int id; - - if (sc == NULL) - return NULL; - - if (SSL_CONNECTION_IS_TLS13(sc) && sc->s3.did_kex) - id = sc->s3.group_id; - else - id = sc->session->kex_group; - - return tls1_group_id2name(s->ctx, id); -} - const char *SSL_group_to_name(SSL *s, int nid) { int group_id = 0; const TLS_GROUP_INFO *cinf = NULL; diff --git a/openssl/src/ssl/s3_msg.c b/openssl/src/ssl/s3_msg.c index 3fcea15e2..dd2fe040e 100644 --- a/openssl/src/ssl/s3_msg.c +++ b/openssl/src/ssl/s3_msg.c @@ -9,10 +9,9 @@ #include "ssl_local.h" -int ssl3_do_change_cipher_spec(SSL_CONNECTION *s) +int ssl3_do_change_cipher_spec(SSL *s) { int i; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if (s->server) i = SSL3_CHANGE_CIPHER_SERVER_READ; @@ -27,13 +26,13 @@ int ssl3_do_change_cipher_spec(SSL_CONNECTION *s) } s->session->cipher = s->s3.tmp.new_cipher; - if (!ssl->method->ssl3_enc->setup_key_block(s)) { + if (!s->method->ssl3_enc->setup_key_block(s)) { /* SSLfatal() already called */ return 0; } } - if (!ssl->method->ssl3_enc->change_cipher_state(s, i)) { + if (!s->method->ssl3_enc->change_cipher_state(s, i)) { /* SSLfatal() already called */ return 0; } @@ -41,15 +40,13 @@ int ssl3_do_change_cipher_spec(SSL_CONNECTION *s) return 1; } -int ssl3_send_alert(SSL_CONNECTION *s, int level, int desc) +int ssl3_send_alert(SSL *s, int level, int desc) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - /* Map tls/ssl alert value to correct one */ - if (SSL_CONNECTION_TREAT_AS_TLS13(s)) + if (SSL_TREAT_AS_TLS13(s)) desc = tls13_alert_code(desc); else - desc = ssl->method->ssl3_enc->alert_value(desc); + desc = s->method->ssl3_enc->alert_value(desc); if (s->version == SSL3_VERSION && desc == SSL_AD_PROTOCOL_VERSION) desc = SSL_AD_HANDSHAKE_FAILURE; /* SSL 3.0 does not have * protocol_version alerts */ @@ -61,12 +58,12 @@ int ssl3_send_alert(SSL_CONNECTION *s, int level, int desc) if ((level == SSL3_AL_FATAL) && (s->session != NULL)) SSL_CTX_remove_session(s->session_ctx, s->session); - s->s3.alert_dispatch = SSL_ALERT_DISPATCH_PENDING; + s->s3.alert_dispatch = 1; s->s3.send_alert[0] = level; s->s3.send_alert[1] = desc; if (!RECORD_LAYER_write_pending(&s->rlayer)) { /* data still being written out? */ - return ssl->method->ssl_dispatch_alert(ssl); + return s->method->ssl_dispatch_alert(s); } /* * else data is still being written out, we will get written some time in @@ -78,80 +75,44 @@ int ssl3_send_alert(SSL_CONNECTION *s, int level, int desc) int ssl3_dispatch_alert(SSL *s) { int i, j; + size_t alertlen; void (*cb) (const SSL *ssl, int type, int val) = NULL; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - OSSL_RECORD_TEMPLATE templ; - - if (sc == NULL) - return -1; - - if (sc->rlayer.wrlmethod == NULL) { - /* No write record layer so we can't sent and alert. We just ignore it */ - sc->s3.alert_dispatch = SSL_ALERT_DISPATCH_NONE; - return 1; - } - - templ.type = SSL3_RT_ALERT; - templ.version = (sc->version == TLS1_3_VERSION) ? TLS1_2_VERSION - : sc->version; - if (SSL_get_state(s) == TLS_ST_CW_CLNT_HELLO - && !sc->renegotiate - && TLS1_get_version(s) > TLS1_VERSION - && sc->hello_retry_request == SSL_HRR_NONE) { - templ.version = TLS1_VERSION; - } - templ.buf = &sc->s3.send_alert[0]; - templ.buflen = 2; - - if (RECORD_LAYER_write_pending(&sc->rlayer)) { - if (sc->s3.alert_dispatch != SSL_ALERT_DISPATCH_RETRY) { - /* - * We have a write pending but it wasn't from a previous call to - * this function! Can we ever get here? Maybe via API misuse?? - * Give up. - */ - sc->s3.alert_dispatch = SSL_ALERT_DISPATCH_NONE; - return -1; - } - /* Retry what we've already got pending */ - i = HANDLE_RLAYER_WRITE_RETURN(sc, - sc->rlayer.wrlmethod->retry_write_records(sc->rlayer.wrl)); - if (i <= 0) { - /* Could be NBIO. Keep alert_dispatch as SSL_ALERT_DISPATCH_RETRY */ - return -1; + size_t written; + + s->s3.alert_dispatch = 0; + alertlen = 2; +#ifndef OPENSSL_NO_QUIC + if (SSL_IS_QUIC(s)) { + if (!s->quic_method->send_alert(s, s->quic_write_level, + s->s3.send_alert[1])) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + return 0; } - sc->rlayer.wpend_tot = 0; - sc->s3.alert_dispatch = SSL_ALERT_DISPATCH_NONE; - return 1; - } - - i = HANDLE_RLAYER_WRITE_RETURN(sc, - sc->rlayer.wrlmethod->write_records(sc->rlayer.wrl, &templ, 1)); - + i = 1; + } else +#endif + i = do_ssl3_write(s, SSL3_RT_ALERT, &s->s3.send_alert[0], &alertlen, 1, 0, + &written); if (i <= 0) { - sc->s3.alert_dispatch = SSL_ALERT_DISPATCH_RETRY; - sc->rlayer.wpend_tot = templ.buflen; - sc->rlayer.wpend_type = templ.type; - sc->rlayer.wpend_buf = templ.buf; + s->s3.alert_dispatch = 1; } else { /* * Alert sent to BIO - now flush. If the message does not get sent due * to non-blocking IO, we will not worry too much. */ - (void)BIO_flush(sc->wbio); - sc->s3.alert_dispatch = SSL_ALERT_DISPATCH_NONE; + (void)BIO_flush(s->wbio); - if (sc->msg_callback) - sc->msg_callback(1, sc->version, SSL3_RT_ALERT, sc->s3.send_alert, - 2, s, sc->msg_callback_arg); + if (s->msg_callback) + s->msg_callback(1, s->version, SSL3_RT_ALERT, s->s3.send_alert, + 2, s, s->msg_callback_arg); - if (sc->info_callback != NULL) - cb = sc->info_callback; + if (s->info_callback != NULL) + cb = s->info_callback; else if (s->ctx->info_callback != NULL) cb = s->ctx->info_callback; if (cb != NULL) { - j = (sc->s3.send_alert[0] << 8) | sc->s3.send_alert[1]; + j = (s->s3.send_alert[0] << 8) | s->s3.send_alert[1]; cb(s, SSL_CB_WRITE_ALERT, j); } } diff --git a/openssl/src/ssl/ssl_asn1.c b/openssl/src/ssl/ssl_asn1.c index 9964a8c1e..6721b2840 100644 --- a/openssl/src/ssl/ssl_asn1.c +++ b/openssl/src/ssl/ssl_asn1.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2005 Nokia. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -12,7 +12,6 @@ #include #include "ssl_local.h" #include -#include #include typedef struct { @@ -45,7 +44,10 @@ typedef struct { uint32_t tlsext_max_fragment_len_mode; ASN1_OCTET_STRING *ticket_appdata; uint32_t kex_group; - ASN1_OCTET_STRING *peer_rpk; +#ifndef OPENSSL_NO_QUIC + uint32_t is_quic; + ASN1_OCTET_STRING *quic_early_data_context; +#endif } SSL_SESSION_ASN1; ASN1_SEQUENCE(SSL_SESSION_ASN1) = { @@ -78,7 +80,10 @@ ASN1_SEQUENCE(SSL_SESSION_ASN1) = { ASN1_EXP_OPT_EMBED(SSL_SESSION_ASN1, tlsext_max_fragment_len_mode, ZUINT32, 17), ASN1_EXP_OPT(SSL_SESSION_ASN1, ticket_appdata, ASN1_OCTET_STRING, 18), ASN1_EXP_OPT_EMBED(SSL_SESSION_ASN1, kex_group, UINT32, 19), - ASN1_EXP_OPT(SSL_SESSION_ASN1, peer_rpk, ASN1_OCTET_STRING, 20) +#ifndef OPENSSL_NO_QUIC + ASN1_EXP_OPT_EMBED(SSL_SESSION_ASN1, is_quic, ZUINT32, 20), + ASN1_EXP_OPT(SSL_SESSION_ASN1, quic_early_data_context, ASN1_OCTET_STRING, 21), +#endif } static_ASN1_SEQUENCE_END(SSL_SESSION_ASN1) IMPLEMENT_STATIC_ASN1_ENCODE_FUNCTIONS(SSL_SESSION_ASN1) @@ -128,10 +133,10 @@ int i2d_SSL_SESSION(const SSL_SESSION *in, unsigned char **pp) #endif ASN1_OCTET_STRING alpn_selected; ASN1_OCTET_STRING ticket_appdata; - ASN1_OCTET_STRING peer_rpk; - +#ifndef OPENSSL_NO_QUIC + ASN1_OCTET_STRING quic_early_data_context; +#endif long l; - int ret; if ((in == NULL) || ((in->cipher == NULL) && (in->cipher_id == 0))) return 0; @@ -168,20 +173,12 @@ int i2d_SSL_SESSION(const SSL_SESSION *in, unsigned char **pp) ssl_session_oinit(&as.session_id_context, &sid_ctx, in->sid_ctx, in->sid_ctx_length); - as.time = (int64_t)ossl_time_to_time_t(in->time); - as.timeout = (int64_t)ossl_time2seconds(in->timeout); + as.time = (int64_t)in->time; + as.timeout = (int64_t)in->timeout; as.verify_result = in->verify_result; as.peer = in->peer; - as.peer_rpk = NULL; - peer_rpk.data = NULL; - if (in->peer_rpk != NULL) { - peer_rpk.length = i2d_PUBKEY(in->peer_rpk, &peer_rpk.data); - if (peer_rpk.length > 0 && peer_rpk.data != NULL) - as.peer_rpk = &peer_rpk; - } - ssl_session_sinit(&as.tlsext_hostname, &tlsext_hostname, in->ext.hostname); if (in->ext.tick) { @@ -217,9 +214,18 @@ int i2d_SSL_SESSION(const SSL_SESSION *in, unsigned char **pp) ssl_session_oinit(&as.ticket_appdata, &ticket_appdata, in->ticket_appdata, in->ticket_appdata_len); - ret = i2d_SSL_SESSION_ASN1(&as, pp); - OPENSSL_free(peer_rpk.data); - return ret; +#ifndef OPENSSL_NO_QUIC + as.is_quic = in->is_quic; + + if (in->quic_early_data_context == NULL) + as.quic_early_data_context = NULL; + else + ssl_session_oinit(&as.quic_early_data_context, &quic_early_data_context, + in->quic_early_data_context, in->quic_early_data_context_len); +#endif + + return i2d_SSL_SESSION_ASN1(&as, pp); + } /* Utility functions for d2i_SSL_SESSION */ @@ -256,12 +262,6 @@ static int ssl_session_memcpy(unsigned char *dst, size_t *pdstlen, SSL_SESSION *d2i_SSL_SESSION(SSL_SESSION **a, const unsigned char **pp, long length) -{ - return d2i_SSL_SESSION_ex(a, pp, length, NULL, NULL); -} -SSL_SESSION *d2i_SSL_SESSION_ex(SSL_SESSION **a, const unsigned char **pp, - long length, OSSL_LIB_CTX *libctx, - const char *propq) { long id; size_t tmpl; @@ -288,6 +288,9 @@ SSL_SESSION *d2i_SSL_SESSION_ex(SSL_SESSION **a, const unsigned char **pp, } if ((as->ssl_version >> 8) != SSL3_VERSION_MAJOR +#ifndef OPENSSL_NO_NTLS + && as->ssl_version != NTLS_VERSION +#endif && (as->ssl_version >> 8) != DTLS1_VERSION_MAJOR && as->ssl_version != DTLS1_BAD_VER) { ERR_raise(ERR_LIB_SSL, SSL_R_UNSUPPORTED_SSL_VERSION); @@ -322,33 +325,20 @@ SSL_SESSION *d2i_SSL_SESSION_ex(SSL_SESSION **a, const unsigned char **pp, ret->master_key_length = tmpl; if (as->time != 0) - ret->time = ossl_time_from_time_t(as->time); + ret->time = (time_t)as->time; else - ret->time = ossl_time_now(); + ret->time = time(NULL); if (as->timeout != 0) - ret->timeout = ossl_seconds2time(as->timeout); + ret->timeout = (time_t)as->timeout; else - ret->timeout = ossl_seconds2time(3); + ret->timeout = 3; ssl_session_calculate_timeout(ret); X509_free(ret->peer); ret->peer = as->peer; as->peer = NULL; - EVP_PKEY_free(ret->peer_rpk); - ret->peer_rpk = NULL; - if (as->peer_rpk != NULL) { - const unsigned char *data = as->peer_rpk->data; - - /* - * |data| is incremented; we don't want to lose original ptr - */ - ret->peer_rpk = d2i_PUBKEY_ex(NULL, &data, as->peer_rpk->length, libctx, propq); - if (ret->peer_rpk == NULL) - goto err; - } - if (!ssl_session_memcpy(ret->sid_ctx, &ret->sid_ctx_length, as->session_id_context, SSL_MAX_SID_CTX_LENGTH)) goto err; @@ -418,6 +408,20 @@ SSL_SESSION *d2i_SSL_SESSION_ex(SSL_SESSION **a, const unsigned char **pp, ret->ticket_appdata_len = 0; } +#ifndef OPENSSL_NO_QUIC + ret->is_quic = as->is_quic; + + OPENSSL_free(ret->quic_early_data_context); + if (as->quic_early_data_context != NULL) { + ret->quic_early_data_context = as->quic_early_data_context->data; + ret->quic_early_data_context_len = as->quic_early_data_context->length; + as->quic_early_data_context->data = NULL; + } else { + ret->quic_early_data_context = NULL; + ret->quic_early_data_context_len = 0; + } +#endif + M_ASN1_free_of(as, SSL_SESSION_ASN1); if ((a != NULL) && (*a == NULL)) diff --git a/openssl/src/ssl/ssl_cert.c b/openssl/src/ssl/ssl_cert.c index f11eb7582..2a879e8f3 100644 --- a/openssl/src/ssl/ssl_cert.c +++ b/openssl/src/ssl/ssl_cert.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -10,12 +10,13 @@ #include #include - +#include #include "internal/nelem.h" #include "internal/o_dir.h" #include #include #include +#include #include #include #include @@ -24,16 +25,6 @@ #include "ssl_local.h" #include "ssl_cert_table.h" #include "internal/thread_once.h" -#ifndef OPENSSL_NO_POSIX_IO -# include -# ifdef _WIN32 -# define stat _stat -# endif -# ifndef S_ISDIR -# define S_ISDIR(a) (((a) & S_IFMT) == S_IFDIR) -# endif -#endif - static int ssl_security_default_callback(const SSL *s, const SSL_CTX *ctx, int op, int bits, int nid, void *other, @@ -58,31 +49,23 @@ int SSL_get_ex_data_X509_STORE_CTX_idx(void) return ssl_x509_store_ctx_idx; } -CERT *ssl_cert_new(size_t ssl_pkey_num) +CERT *ssl_cert_new(void) { - CERT *ret = NULL; - - /* Should never happen */ - if (!ossl_assert(ssl_pkey_num >= SSL_PKEY_NUM)) - return NULL; - - ret = OPENSSL_zalloc(sizeof(*ret)); - if (ret == NULL) - return NULL; + CERT *ret = OPENSSL_zalloc(sizeof(*ret)); - ret->ssl_pkey_num = ssl_pkey_num; - ret->pkeys = OPENSSL_zalloc(ret->ssl_pkey_num * sizeof(CERT_PKEY)); - if (ret->pkeys == NULL) { - OPENSSL_free(ret); + if (ret == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return NULL; } ret->key = &(ret->pkeys[SSL_PKEY_RSA]); + ret->references = 1; ret->sec_cb = ssl_security_default_callback; ret->sec_level = OPENSSL_TLS_SECURITY_LEVEL; ret->sec_ex = NULL; - if (!CRYPTO_NEW_REF(&ret->references, 1)) { - OPENSSL_free(ret->pkeys); + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -93,24 +76,18 @@ CERT *ssl_cert_new(size_t ssl_pkey_num) CERT *ssl_cert_dup(CERT *cert) { CERT *ret = OPENSSL_zalloc(sizeof(*ret)); - size_t i; -#ifndef OPENSSL_NO_COMP_ALG - int j; -#endif - - if (ret == NULL) - return NULL; + int i; - ret->ssl_pkey_num = cert->ssl_pkey_num; - ret->pkeys = OPENSSL_zalloc(ret->ssl_pkey_num * sizeof(CERT_PKEY)); - if (ret->pkeys == NULL) { - OPENSSL_free(ret); + if (ret == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return NULL; } + ret->references = 1; ret->key = &ret->pkeys[cert->key - cert->pkeys]; - if (!CRYPTO_NEW_REF(&ret->references, 1)) { - OPENSSL_free(ret->pkeys); + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); OPENSSL_free(ret); return NULL; } @@ -123,10 +100,9 @@ CERT *ssl_cert_dup(CERT *cert) ret->dh_tmp_cb = cert->dh_tmp_cb; ret->dh_tmp_auto = cert->dh_tmp_auto; - for (i = 0; i < ret->ssl_pkey_num; i++) { + for (i = 0; i < SSL_PKEY_NUM; i++) { CERT_PKEY *cpk = cert->pkeys + i; CERT_PKEY *rpk = ret->pkeys + i; - if (cpk->x509 != NULL) { rpk->x509 = cpk->x509; X509_up_ref(rpk->x509); @@ -137,29 +113,37 @@ CERT *ssl_cert_dup(CERT *cert) EVP_PKEY_up_ref(cpk->privatekey); } +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + if (cert->dc_pkeys[i].dc) { + DC_up_ref(cert->dc_pkeys[i].dc); + ret->dc_pkeys[i].dc = cert->dc_pkeys[i].dc; + } + + if (cert->dc_pkeys[i].privatekey) { + EVP_PKEY_up_ref(cert->dc_pkeys[i].privatekey); + ret->dc_pkeys[i].privatekey = cert->dc_pkeys[i].privatekey; + } +#endif + if (cpk->chain) { rpk->chain = X509_chain_up_ref(cpk->chain); if (!rpk->chain) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } } - if (cpk->serverinfo != NULL) { + if (cert->pkeys[i].serverinfo != NULL) { /* Just copy everything. */ - rpk->serverinfo = OPENSSL_memdup(cpk->serverinfo, cpk->serverinfo_length); - if (rpk->serverinfo == NULL) + ret->pkeys[i].serverinfo = + OPENSSL_malloc(cert->pkeys[i].serverinfo_length); + if (ret->pkeys[i].serverinfo == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; - rpk->serverinfo_length = cpk->serverinfo_length; - } -#ifndef OPENSSL_NO_COMP_ALG - for (j = TLSEXT_comp_cert_none; j < TLSEXT_comp_cert_limit; j++) { - if (cpk->comp_cert[j] != NULL) { - if (!OSSL_COMP_CERT_up_ref(cpk->comp_cert[j])) - goto err; - rpk->comp_cert[j] = cpk->comp_cert[j]; } + ret->pkeys[i].serverinfo_length = cert->pkeys[i].serverinfo_length; + memcpy(ret->pkeys[i].serverinfo, + cert->pkeys[i].serverinfo, cert->pkeys[i].serverinfo_length); } -#endif } /* Configured sigalgs copied across */ @@ -232,30 +216,25 @@ CERT *ssl_cert_dup(CERT *cert) void ssl_cert_clear_certs(CERT *c) { - size_t i; -#ifndef OPENSSL_NO_COMP_ALG - int j; -#endif - + int i; if (c == NULL) return; - for (i = 0; i < c->ssl_pkey_num; i++) { + for (i = 0; i < SSL_PKEY_NUM; i++) { CERT_PKEY *cpk = c->pkeys + i; X509_free(cpk->x509); cpk->x509 = NULL; EVP_PKEY_free(cpk->privatekey); cpk->privatekey = NULL; - OSSL_STACK_OF_X509_free(cpk->chain); + sk_X509_pop_free(cpk->chain, X509_free); cpk->chain = NULL; OPENSSL_free(cpk->serverinfo); cpk->serverinfo = NULL; cpk->serverinfo_length = 0; -#ifndef OPENSSL_NO_COMP_ALG - for (j = 0; j < TLSEXT_comp_cert_limit; j++) { - OSSL_COMP_CERT_free(cpk->comp_cert[j]); - cpk->comp_cert[j] = NULL; - cpk->cert_comp_used = 0; - } +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + DC_free(c->dc_pkeys[i].dc); + c->dc_pkeys[i].dc = NULL; + EVP_PKEY_free(c->dc_pkeys[i].privatekey); + c->dc_pkeys[i].privatekey = NULL; #endif } } @@ -266,7 +245,7 @@ void ssl_cert_free(CERT *c) if (c == NULL) return; - CRYPTO_DOWN_REF(&c->references, &i); + CRYPTO_DOWN_REF(&c->references, &i, c->lock); REF_PRINT_COUNT("CERT", c); if (i > 0) return; @@ -284,12 +263,11 @@ void ssl_cert_free(CERT *c) #ifndef OPENSSL_NO_PSK OPENSSL_free(c->psk_identity_hint); #endif - OPENSSL_free(c->pkeys); - CRYPTO_FREE_REF(&c->references); + CRYPTO_THREAD_lock_free(c->lock); OPENSSL_free(c); } -int ssl_cert_set0_chain(SSL_CONNECTION *s, SSL_CTX *ctx, STACK_OF(X509) *chain) +int ssl_cert_set0_chain(SSL *s, SSL_CTX *ctx, STACK_OF(X509) *chain) { int i, r; CERT_PKEY *cpk = s != NULL ? s->cert->key : ctx->cert->key; @@ -305,32 +283,30 @@ int ssl_cert_set0_chain(SSL_CONNECTION *s, SSL_CTX *ctx, STACK_OF(X509) *chain) return 0; } } - OSSL_STACK_OF_X509_free(cpk->chain); + sk_X509_pop_free(cpk->chain, X509_free); cpk->chain = chain; return 1; } -int ssl_cert_set1_chain(SSL_CONNECTION *s, SSL_CTX *ctx, STACK_OF(X509) *chain) +int ssl_cert_set1_chain(SSL *s, SSL_CTX *ctx, STACK_OF(X509) *chain) { STACK_OF(X509) *dchain; - if (!chain) return ssl_cert_set0_chain(s, ctx, NULL); dchain = X509_chain_up_ref(chain); if (!dchain) return 0; if (!ssl_cert_set0_chain(s, ctx, dchain)) { - OSSL_STACK_OF_X509_free(dchain); + sk_X509_pop_free(dchain, X509_free); return 0; } return 1; } -int ssl_cert_add0_chain_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x) +int ssl_cert_add0_chain_cert(SSL *s, SSL_CTX *ctx, X509 *x) { int r; CERT_PKEY *cpk = s ? s->cert->key : ctx->cert->key; - if (!cpk) return 0; r = ssl_security_cert(s, ctx, x, 0, 0); @@ -345,7 +321,7 @@ int ssl_cert_add0_chain_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x) return 1; } -int ssl_cert_add1_chain_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x) +int ssl_cert_add1_chain_cert(SSL *s, SSL_CTX *ctx, X509 *x) { if (!ssl_cert_add0_chain_cert(s, ctx, x)) return 0; @@ -355,11 +331,10 @@ int ssl_cert_add1_chain_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x) int ssl_cert_select_current(CERT *c, X509 *x) { - size_t i; - + int i; if (x == NULL) return 0; - for (i = 0; i < c->ssl_pkey_num; i++) { + for (i = 0; i < SSL_PKEY_NUM; i++) { CERT_PKEY *cpk = c->pkeys + i; if (cpk->x509 == x && cpk->privatekey) { c->key = cpk; @@ -367,7 +342,7 @@ int ssl_cert_select_current(CERT *c, X509 *x) } } - for (i = 0; i < c->ssl_pkey_num; i++) { + for (i = 0; i < SSL_PKEY_NUM; i++) { CERT_PKEY *cpk = c->pkeys + i; if (cpk->privatekey && cpk->x509 && !X509_cmp(cpk->x509, x)) { c->key = cpk; @@ -379,19 +354,18 @@ int ssl_cert_select_current(CERT *c, X509 *x) int ssl_cert_set_current(CERT *c, long op) { - size_t i, idx; - + int i, idx; if (!c) return 0; if (op == SSL_CERT_SET_FIRST) idx = 0; else if (op == SSL_CERT_SET_NEXT) { - idx = (size_t)(c->key - c->pkeys + 1); - if (idx >= c->ssl_pkey_num) + idx = (int)(c->key - c->pkeys + 1); + if (idx >= SSL_PKEY_NUM) return 0; } else return 0; - for (i = idx; i < c->ssl_pkey_num; i++) { + for (i = idx; i < SSL_PKEY_NUM; i++) { CERT_PKEY *cpk = c->pkeys + i; if (cpk->x509 && cpk->privatekey) { c->key = cpk; @@ -407,53 +381,49 @@ void ssl_cert_set_cert_cb(CERT *c, int (*cb) (SSL *ssl, void *arg), void *arg) c->cert_cb_arg = arg; } +SSL_cert_cb_fn ssl_cert_get_cert_cb(CERT *c) +{ + return c->cert_cb; +} + +void *ssl_cert_get_cert_cb_arg(CERT *c) +{ + return c->cert_cb_arg; +} + /* - * Verify a certificate chain/raw public key + * Verify a certificate chain * Return codes: * 1: Verify success * 0: Verify failure or error * -1: Retry required */ -static int ssl_verify_internal(SSL_CONNECTION *s, STACK_OF(X509) *sk, EVP_PKEY *rpk) +int ssl_verify_cert_chain(SSL *s, STACK_OF(X509) *sk) { X509 *x; int i = 0; X509_STORE *verify_store; X509_STORE_CTX *ctx = NULL; X509_VERIFY_PARAM *param; - SSL_CTX *sctx; - /* Something must be passed in */ - if ((sk == NULL || sk_X509_num(sk) == 0) && rpk == NULL) + if ((sk == NULL) || (sk_X509_num(sk) == 0)) return 0; - /* Only one can be set */ - if (sk != NULL && rpk != NULL) - return 0; - - sctx = SSL_CONNECTION_GET_CTX(s); if (s->cert->verify_store) verify_store = s->cert->verify_store; else - verify_store = sctx->cert_store; + verify_store = s->ctx->cert_store; - ctx = X509_STORE_CTX_new_ex(sctx->libctx, sctx->propq); + ctx = X509_STORE_CTX_new_ex(s->ctx->libctx, s->ctx->propq); if (ctx == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } - if (sk != NULL) { - x = sk_X509_value(sk, 0); - if (!X509_STORE_CTX_init(ctx, verify_store, x, sk)) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); - goto end; - } - } else { - if (!X509_STORE_CTX_init_rpk(ctx, verify_store, rpk)) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); - goto end; - } + x = sk_X509_value(sk, 0); + if (!X509_STORE_CTX_init(ctx, verify_store, x, sk)) { + ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + goto end; } param = X509_STORE_CTX_get0_param(ctx); /* @@ -461,13 +431,12 @@ static int ssl_verify_internal(SSL_CONNECTION *s, STACK_OF(X509) *sk, EVP_PKEY * * point, for now a single @SECLEVEL sets the same policy for TLS crypto * and PKI authentication. */ - X509_VERIFY_PARAM_set_auth_level(param, - SSL_get_security_level(SSL_CONNECTION_GET_SSL(s))); + X509_VERIFY_PARAM_set_auth_level(param, SSL_get_security_level(s)); /* Set suite B flags if needed */ X509_STORE_CTX_set_flags(ctx, tls1_suiteb(s)); - if (!X509_STORE_CTX_set_ex_data(ctx, - SSL_get_ex_data_X509_STORE_CTX_idx(), s)) { + if (!X509_STORE_CTX_set_ex_data + (ctx, SSL_get_ex_data_X509_STORE_CTX_idx(), s)) { goto end; } @@ -490,8 +459,8 @@ static int ssl_verify_internal(SSL_CONNECTION *s, STACK_OF(X509) *sk, EVP_PKEY * if (s->verify_callback) X509_STORE_CTX_set_verify_cb(ctx, s->verify_callback); - if (sctx->app_verify_callback != NULL) { - i = sctx->app_verify_callback(ctx, sctx->app_verify_arg); + if (s->ctx->app_verify_callback != NULL) { + i = s->ctx->app_verify_callback(ctx, s->ctx->app_verify_arg); } else { i = X509_verify_cert(ctx); /* We treat an error in the same way as a failure to verify */ @@ -500,13 +469,12 @@ static int ssl_verify_internal(SSL_CONNECTION *s, STACK_OF(X509) *sk, EVP_PKEY * } s->verify_result = X509_STORE_CTX_get_error(ctx); - OSSL_STACK_OF_X509_free(s->verified_chain); + sk_X509_pop_free(s->verified_chain, X509_free); s->verified_chain = NULL; - - if (sk != NULL && X509_STORE_CTX_get0_chain(ctx) != NULL) { + if (X509_STORE_CTX_get0_chain(ctx) != NULL) { s->verified_chain = X509_STORE_CTX_get1_chain(ctx); if (s->verified_chain == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); i = 0; } } @@ -519,30 +487,6 @@ static int ssl_verify_internal(SSL_CONNECTION *s, STACK_OF(X509) *sk, EVP_PKEY * return i; } -/* - * Verify a raw public key - * Return codes: - * 1: Verify success - * 0: Verify failure or error - * -1: Retry required - */ -int ssl_verify_rpk(SSL_CONNECTION *s, EVP_PKEY *rpk) -{ - return ssl_verify_internal(s, NULL, rpk); -} - -/* - * Verify a certificate chain - * Return codes: - * 1: Verify success - * 0: Verify failure or error - * -1: Retry required - */ -int ssl_verify_cert_chain(SSL_CONNECTION *s, STACK_OF(X509) *sk) -{ - return ssl_verify_internal(s, sk, NULL); -} - static void set0_CA_list(STACK_OF(X509_NAME) **ca_list, STACK_OF(X509_NAME) *name_list) { @@ -559,13 +503,13 @@ STACK_OF(X509_NAME) *SSL_dup_CA_list(const STACK_OF(X509_NAME) *sk) ret = sk_X509_NAME_new_reserve(NULL, num); if (ret == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return NULL; } for (i = 0; i < num; i++) { name = X509_NAME_dup(sk_X509_NAME_value(sk, i)); if (name == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); sk_X509_NAME_pop_free(ret, X509_NAME_free); return NULL; } @@ -576,12 +520,7 @@ STACK_OF(X509_NAME) *SSL_dup_CA_list(const STACK_OF(X509_NAME) *sk) void SSL_set0_CA_list(SSL *s, STACK_OF(X509_NAME) *name_list) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - set0_CA_list(&sc->ca_names, name_list); + set0_CA_list(&s->ca_names, name_list); } void SSL_CTX_set0_CA_list(SSL_CTX *ctx, STACK_OF(X509_NAME) *name_list) @@ -596,12 +535,7 @@ const STACK_OF(X509_NAME) *SSL_CTX_get0_CA_list(const SSL_CTX *ctx) const STACK_OF(X509_NAME) *SSL_get0_CA_list(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - - return sc->ca_names != NULL ? sc->ca_names : s->ctx->ca_names; + return s->ca_names != NULL ? s->ca_names : s->ctx->ca_names; } void SSL_CTX_set_client_CA_list(SSL_CTX *ctx, STACK_OF(X509_NAME) *name_list) @@ -616,35 +550,20 @@ STACK_OF(X509_NAME) *SSL_CTX_get_client_CA_list(const SSL_CTX *ctx) void SSL_set_client_CA_list(SSL *s, STACK_OF(X509_NAME) *name_list) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - set0_CA_list(&sc->client_ca_names, name_list); + set0_CA_list(&s->client_ca_names, name_list); } const STACK_OF(X509_NAME) *SSL_get0_peer_CA_list(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - - return sc->s3.tmp.peer_ca_names; + return s->s3.tmp.peer_ca_names; } STACK_OF(X509_NAME) *SSL_get_client_CA_list(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - - if (!sc->server) - return sc->s3.tmp.peer_ca_names; - return sc->client_ca_names != NULL ? sc->client_ca_names - : s->ctx->client_ca_names; + if (!s->server) + return s->s3.tmp.peer_ca_names; + return s->client_ca_names != NULL ? s->client_ca_names + : s->ctx->client_ca_names; } static int add_ca_name(STACK_OF(X509_NAME) **sk, const X509 *x) @@ -668,12 +587,7 @@ static int add_ca_name(STACK_OF(X509_NAME) **sk, const X509 *x) int SSL_add1_to_CA_list(SSL *ssl, const X509 *x) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - - return add_ca_name(&sc->ca_names, x); + return add_ca_name(&ssl->ca_names, x); } int SSL_CTX_add1_to_CA_list(SSL_CTX *ctx, const X509 *x) @@ -687,12 +601,7 @@ int SSL_CTX_add1_to_CA_list(SSL_CTX *ctx, const X509 *x) */ int SSL_add_client_CA(SSL *ssl, X509 *x) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - - return add_ca_name(&sc->client_ca_names, x); + return add_ca_name(&ssl->client_ca_names, x); } int SSL_CTX_add_client_CA(SSL_CTX *ctx, X509 *x) @@ -746,18 +655,14 @@ STACK_OF(X509_NAME) *SSL_load_client_CA_file_ex(const char *file, LHASH_OF(X509_NAME) *name_hash = lh_X509_NAME_new(xname_hash, xname_cmp); OSSL_LIB_CTX *prev_libctx = NULL; - if (name_hash == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); - goto err; - } - if (in == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_BIO_LIB); + if ((name_hash == NULL) || (in == NULL)) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } x = X509_new_ex(libctx, propq); if (x == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } if (BIO_read_filename(in, file) <= 0) @@ -771,7 +676,7 @@ STACK_OF(X509_NAME) *SSL_load_client_CA_file_ex(const char *file, if (ret == NULL) { ret = sk_X509_NAME_new_null(); if (ret == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } } @@ -827,7 +732,7 @@ int SSL_add_file_cert_subjects_to_stack(STACK_OF(X509_NAME) *stack, in = BIO_new(BIO_s_file()); if (in == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_BIO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } @@ -875,28 +780,12 @@ int SSL_add_dir_cert_subjects_to_stack(STACK_OF(X509_NAME) *stack, while ((filename = OPENSSL_DIR_read(&d, dir))) { char buf[1024]; int r; -#ifndef OPENSSL_NO_POSIX_IO - struct stat st; -#else - /* Cannot use stat so just skip current and parent directories */ - if (strcmp(filename, ".") == 0 || strcmp(filename, "..") == 0) - continue; -#endif if (strlen(dir) + strlen(filename) + 2 > sizeof(buf)) { ERR_raise(ERR_LIB_SSL, SSL_R_PATH_TOO_LONG); goto err; } -#ifdef OPENSSL_SYS_VMS - r = BIO_snprintf(buf, sizeof(buf), "%s%s", dir, filename); -#else r = BIO_snprintf(buf, sizeof(buf), "%s/%s", dir, filename); -#endif -#ifndef OPENSSL_NO_POSIX_IO - /* Skip subdirectories */ - if (!stat(buf, &st) && S_ISDIR(st.st_mode)) - continue; -#endif if (r <= 0 || r >= (int)sizeof(buf)) goto err; if (!SSL_add_file_cert_subjects_to_stack(stack, buf)) @@ -985,18 +874,18 @@ int SSL_add_store_cert_subjects_to_stack(STACK_OF(X509_NAME) *stack, } /* Build a certificate chain for current certificate */ -int ssl_build_cert_chain(SSL_CONNECTION *s, SSL_CTX *ctx, int flags) +int ssl_build_cert_chain(SSL *s, SSL_CTX *ctx, int flags) { - CERT *c = s != NULL ? s->cert : ctx->cert; + CERT *c = s ? s->cert : ctx->cert; CERT_PKEY *cpk = c->key; X509_STORE *chain_store = NULL; X509_STORE_CTX *xs_ctx = NULL; STACK_OF(X509) *chain = NULL, *untrusted = NULL; X509 *x; - SSL_CTX *real_ctx = (s == NULL) ? ctx : SSL_CONNECTION_GET_CTX(s); + SSL_CTX *real_ctx = (s == NULL) ? ctx : s->ctx; int i, rv = 0; - if (cpk->x509 == NULL) { + if (!cpk->x509) { ERR_raise(ERR_LIB_SSL, SSL_R_NO_CERTIFICATE_SET); goto err; } @@ -1014,10 +903,12 @@ int ssl_build_cert_chain(SSL_CONNECTION *s, SSL_CTX *ctx, int flags) if (!X509_STORE_add_cert(chain_store, cpk->x509)) goto err; } else { - if (c->chain_store != NULL) + if (c->chain_store) chain_store = c->chain_store; + else if (s) + chain_store = s->ctx->cert_store; else - chain_store = real_ctx->cert_store; + chain_store = ctx->cert_store; if (flags & SSL_BUILD_CHAIN_FLAG_UNTRUSTED) untrusted = cpk->chain; @@ -1025,7 +916,7 @@ int ssl_build_cert_chain(SSL_CONNECTION *s, SSL_CTX *ctx, int flags) xs_ctx = X509_STORE_CTX_new_ex(real_ctx->libctx, real_ctx->propq); if (xs_ctx == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } if (!X509_STORE_CTX_init(xs_ctx, chain_store, cpk->x509, untrusted)) { @@ -1074,12 +965,12 @@ int ssl_build_cert_chain(SSL_CONNECTION *s, SSL_CTX *ctx, int flags) rv = ssl_security_cert(s, ctx, x, 0, 0); if (rv != 1) { ERR_raise(ERR_LIB_SSL, rv); - OSSL_STACK_OF_X509_free(chain); + sk_X509_pop_free(chain, X509_free); rv = 0; goto err; } } - OSSL_STACK_OF_X509_free(cpk->chain); + sk_X509_pop_free(cpk->chain, X509_free); cpk->chain = chain; if (rv == 0) rv = 1; @@ -1142,7 +1033,6 @@ static int ssl_security_default_callback(const SSL *s, const SSL_CTX *ctx, void *ex) { int level, minbits, pfs_mask; - const SSL_CONNECTION *sc; minbits = ssl_get_security_level_bits(s, ctx, &level); @@ -1173,23 +1063,39 @@ static int ssl_security_default_callback(const SSL *s, const SSL_CTX *ctx, /* SHA1 HMAC is 160 bits of security */ if (minbits > 160 && c->algorithm_mac & SSL_SHA1) return 0; + /* Level 2: no RC4 */ + if (level >= 2 && c->algorithm_enc == SSL_RC4) + return 0; /* Level 3: forward secure ciphersuites only */ +#ifndef OPENSSL_NO_NTLS + pfs_mask = SSL_kDHE | SSL_kECDHE | SSL_kDHEPSK | SSL_kECDHEPSK | SSL_kSM2DHE; +#else pfs_mask = SSL_kDHE | SSL_kECDHE | SSL_kDHEPSK | SSL_kECDHEPSK; +#endif if (level >= 3 && c->min_tls != TLS1_3_VERSION && !(c->algorithm_mkey & pfs_mask)) return 0; break; } case SSL_SECOP_VERSION: - if ((sc = SSL_CONNECTION_FROM_CONST_SSL(s)) == NULL) - return 0; - if (!SSL_CONNECTION_IS_DTLS(sc)) { - /* SSLv3, TLS v1.0 and TLS v1.1 only allowed at level 0 */ - if (nid <= TLS1_1_VERSION && level > 0) + if (!SSL_IS_DTLS(s)) { +#ifndef OPENSSL_NO_NTLS + /* NTLS v1.1 not allowed at level 3 */ + if (nid == NTLS_VERSION && level >= 3) + return 0; +#endif + /* SSLv3 not allowed at level 2 */ + if (nid <= SSL3_VERSION && level >= 2) + return 0; + /* TLS v1.1 and above only for level 3 */ + if (nid <= TLS1_VERSION && level >= 3) + return 0; + /* TLS v1.2 only for level 4 and above */ + if (nid <= TLS1_1_VERSION && level >= 4) return 0; } else { - /* DTLS v1.0 only allowed at level 0 */ - if (DTLS_VERSION_LT(nid, DTLS1_2_VERSION) && level > 0) + /* DTLS v1.2 only for level 4 and above */ + if (DTLS_VERSION_LT(nid, DTLS1_2_VERSION) && level >= 4) return 0; } break; @@ -1209,10 +1115,9 @@ static int ssl_security_default_callback(const SSL *s, const SSL_CTX *ctx, return 1; } -int ssl_security(const SSL_CONNECTION *s, int op, int bits, int nid, void *other) +int ssl_security(const SSL *s, int op, int bits, int nid, void *other) { - return s->cert->sec_cb(SSL_CONNECTION_GET_SSL(s), NULL, op, bits, nid, - other, s->cert->sec_ex); + return s->cert->sec_cb(s, NULL, op, bits, nid, other, s->cert->sec_ex); } int ssl_ctx_security(const SSL_CTX *ctx, int op, int bits, int nid, void *other) @@ -1221,7 +1126,7 @@ int ssl_ctx_security(const SSL_CTX *ctx, int op, int bits, int nid, void *other) ctx->cert->sec_ex); } -int ssl_cert_lookup_by_nid(int nid, size_t *pidx, SSL_CTX *ctx) +int ssl_cert_lookup_by_nid(int nid, size_t *pidx) { size_t i; @@ -1231,20 +1136,14 @@ int ssl_cert_lookup_by_nid(int nid, size_t *pidx, SSL_CTX *ctx) return 1; } } - for (i = 0; i < ctx->sigalg_list_len; i++) { - if (ctx->ssl_cert_info[i].nid == nid) { - *pidx = SSL_PKEY_NUM + i; - return 1; - } - } + return 0; } -const SSL_CERT_LOOKUP *ssl_cert_lookup_by_pkey(const EVP_PKEY *pk, size_t *pidx, SSL_CTX *ctx) +const SSL_CERT_LOOKUP *ssl_cert_lookup_by_pkey(const EVP_PKEY *pk, size_t *pidx) { size_t i; - /* check classic pk types */ for (i = 0; i < OSSL_NELEM(ssl_cert_info); i++) { const SSL_CERT_LOOKUP *tmp_lu = &ssl_cert_info[i]; @@ -1255,26 +1154,86 @@ const SSL_CERT_LOOKUP *ssl_cert_lookup_by_pkey(const EVP_PKEY *pk, size_t *pidx, return tmp_lu; } } - /* check provider-loaded pk types */ - for (i = 0; ctx->sigalg_list_len; i++) { - SSL_CERT_LOOKUP *tmp_lu = &(ctx->ssl_cert_info[i]); - - if (EVP_PKEY_is_a(pk, OBJ_nid2sn(tmp_lu->nid)) - || EVP_PKEY_is_a(pk, OBJ_nid2ln(tmp_lu->nid))) { - if (pidx != NULL) - *pidx = SSL_PKEY_NUM + i; - return &ctx->ssl_cert_info[i]; - } - } return NULL; } -const SSL_CERT_LOOKUP *ssl_cert_lookup_by_idx(size_t idx, SSL_CTX *ctx) +const SSL_CERT_LOOKUP *ssl_cert_lookup_by_idx(size_t idx) { - if (idx >= (OSSL_NELEM(ssl_cert_info) + ctx->sigalg_list_len)) + if (idx >= OSSL_NELEM(ssl_cert_info)) return NULL; - else if (idx >= (OSSL_NELEM(ssl_cert_info))) - return &(ctx->ssl_cert_info[idx - SSL_PKEY_NUM]); return &ssl_cert_info[idx]; } + +#ifndef OPENSSL_NO_CERT_COMPRESSION + +static int ssl_cert_add_compression_alg(STACK_OF(CERT_COMP) *cert_comp_algs, + int alg_id, + SSL_cert_compress_cb_fn compress, + SSL_cert_decompress_cb_fn decompress) +{ + CERT_COMP *comp = NULL; + + comp = OPENSSL_malloc(sizeof(*comp)); + if (comp == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + return 0; + } + + comp->alg_id = alg_id; + comp->compress = compress; + comp->decompress = decompress; + + if (!sk_CERT_COMP_push(cert_comp_algs, comp)) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + OPENSSL_free(comp); + return 0; + } + + return 1; +} + + +int SSL_add_cert_compression_alg(SSL *s, int alg_id, + SSL_cert_compress_cb_fn compress, + SSL_cert_decompress_cb_fn decompress) +{ + if (s->cert_comp_algs == NULL) { + s->cert_comp_algs = sk_CERT_COMP_new_null(); + + if (s->cert_comp_algs == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + return 0; + } + } + + return ssl_cert_add_compression_alg(s->cert_comp_algs, alg_id, + compress, decompress); +} +int SSL_CTX_add_cert_compression_alg(SSL_CTX *ctx, int alg_id, + SSL_cert_compress_cb_fn compress, + SSL_cert_decompress_cb_fn decompress) +{ + if (ctx->cert_comp_algs == NULL) { + ctx->cert_comp_algs = sk_CERT_COMP_new_null(); + + if (ctx->cert_comp_algs == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + return 0; + } + } + + return ssl_cert_add_compression_alg(ctx->cert_comp_algs, alg_id, + compress, decompress); +} + +int SSL_get_cert_compression_compress_id(SSL *s) +{ + return s->cert_comp_compress_id; +} + +int SSL_get_cert_compression_decompress_id(SSL *s) +{ + return s->cert_comp_decompress_id; +} +#endif diff --git a/openssl/src/ssl/ssl_cert_comp.c b/openssl/src/ssl/ssl_cert_comp.c deleted file mode 100644 index 639610a5f..000000000 --- a/openssl/src/ssl/ssl_cert_comp.c +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the Apache License 2.0 (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include -#include "ssl_local.h" -#include "internal/e_os.h" -#include "internal/refcount.h" - -size_t ossl_calculate_comp_expansion(int alg, size_t length) -{ - size_t ret; - /* - * Uncompressibility expansion: - * ZLIB: N + 11 + 5 * (N >> 14) - * Brotli: per RFC7932: N + 5 + 3 * (N >> 16) - * ZSTD: N + 4 + 14 + 3 * (N >> 17) + 4 - */ - - switch (alg) { - case TLSEXT_comp_cert_zlib: - ret = length + 11 + 5 * (length >> 14); - break; - case TLSEXT_comp_cert_brotli: - ret = length + 5 + 3 * (length >> 16); - break; - case TLSEXT_comp_cert_zstd: - ret = length + 22 + 3 * (length >> 17); - break; - default: - return 0; - } - /* Check for overflow */ - if (ret < length) - return 0; - return ret; -} - -int ossl_comp_has_alg(int a) -{ -#ifndef OPENSSL_NO_COMP_ALG - /* 0 means "any" algorithm */ - if ((a == 0 || a == TLSEXT_comp_cert_brotli) && BIO_f_brotli() != NULL) - return 1; - if ((a == 0 || a == TLSEXT_comp_cert_zstd) && BIO_f_zstd() != NULL) - return 1; - if ((a == 0 || a == TLSEXT_comp_cert_zlib) && BIO_f_zlib() != NULL) - return 1; -#endif - return 0; -} - -/* New operation Helper routine */ -#ifndef OPENSSL_NO_COMP_ALG -static OSSL_COMP_CERT *OSSL_COMP_CERT_new(unsigned char *data, size_t len, size_t orig_len, int alg) -{ - OSSL_COMP_CERT *ret = NULL; - - if (!ossl_comp_has_alg(alg) - || data == NULL - || (ret = OPENSSL_zalloc(sizeof(*ret))) == NULL - || !CRYPTO_NEW_REF(&ret->references, 1)) - goto err; - - ret->data = data; - ret->len = len; - ret->orig_len = orig_len; - ret->alg = alg; - return ret; - err: - ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); - OPENSSL_free(data); - OPENSSL_free(ret); - return NULL; -} - -__owur static OSSL_COMP_CERT *OSSL_COMP_CERT_from_compressed_data(unsigned char *data, size_t len, - size_t orig_len, int alg) -{ - return OSSL_COMP_CERT_new(OPENSSL_memdup(data, len), len, orig_len, alg); -} - -__owur static OSSL_COMP_CERT *OSSL_COMP_CERT_from_uncompressed_data(unsigned char *data, size_t len, - int alg) -{ - OSSL_COMP_CERT *ret = NULL; - size_t max_length; - int comp_length; - COMP_METHOD *method; - unsigned char *comp_data = NULL; - COMP_CTX *comp_ctx = NULL; - - switch (alg) { - case TLSEXT_comp_cert_brotli: - method = COMP_brotli_oneshot(); - break; - case TLSEXT_comp_cert_zlib: - method = COMP_zlib_oneshot(); - break; - case TLSEXT_comp_cert_zstd: - method = COMP_zstd_oneshot(); - break; - default: - goto err; - } - - if ((max_length = ossl_calculate_comp_expansion(alg, len)) == 0 - || method == NULL - || (comp_ctx = COMP_CTX_new(method)) == NULL - || (comp_data = OPENSSL_zalloc(max_length)) == NULL) - goto err; - - comp_length = COMP_compress_block(comp_ctx, comp_data, max_length, data, len); - if (comp_length <= 0) - goto err; - - ret = OSSL_COMP_CERT_new(comp_data, comp_length, len, alg); - comp_data = NULL; - - err: - OPENSSL_free(comp_data); - COMP_CTX_free(comp_ctx); - return ret; -} - -void OSSL_COMP_CERT_free(OSSL_COMP_CERT *cc) -{ - int i; - - if (cc == NULL) - return; - - CRYPTO_DOWN_REF(&cc->references, &i); - REF_PRINT_COUNT("OSSL_COMP_CERT", cc); - if (i > 0) - return; - REF_ASSERT_ISNT(i < 0); - - OPENSSL_free(cc->data); - CRYPTO_FREE_REF(&cc->references); - OPENSSL_free(cc); -} -int OSSL_COMP_CERT_up_ref(OSSL_COMP_CERT *cc) -{ - int i; - - if (CRYPTO_UP_REF(&cc->references, &i) <= 0) - return 0; - - REF_PRINT_COUNT("OSSL_COMP_CERT", cc); - REF_ASSERT_ISNT(i < 2); - return ((i > 1) ? 1 : 0); -} - -static int ssl_set_cert_comp_pref(int *prefs, int *algs, size_t len) -{ - size_t j = 0; - size_t i; - int found = 0; - int already_set[TLSEXT_comp_cert_limit]; - int tmp_prefs[TLSEXT_comp_cert_limit]; - - /* Note that |len| is the number of |algs| elements */ - /* clear all algorithms */ - if (len == 0 || algs == NULL) { - memset(prefs, 0, sizeof(tmp_prefs)); - return 1; - } - - /* This will 0-terminate the array */ - memset(tmp_prefs, 0, sizeof(tmp_prefs)); - memset(already_set, 0, sizeof(already_set)); - /* Include only those algorithms we support, ignoring duplicates and unknowns */ - for (i = 0; i < len; i++) { - if (algs[i] != 0 && ossl_comp_has_alg(algs[i])) { - /* Check for duplicate */ - if (already_set[algs[i]]) - return 0; - tmp_prefs[j++] = algs[i]; - already_set[algs[i]] = 1; - found = 1; - } - } - if (found) - memcpy(prefs, tmp_prefs, sizeof(tmp_prefs)); - return found; -} - -static size_t ssl_get_cert_to_compress(SSL *ssl, CERT_PKEY *cpk, unsigned char **data) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - WPACKET tmppkt; - BUF_MEM buf = { 0 }; - size_t ret = 0; - - if (sc == NULL - || cpk == NULL - || !sc->server - || !SSL_in_before(ssl)) - return 0; - - /* Use the |tmppkt| for the to-be-compressed data */ - if (!WPACKET_init(&tmppkt, &buf)) - goto out; - - /* no context present, add 0-length context */ - if (!WPACKET_put_bytes_u8(&tmppkt, 0)) - goto out; - - /* - * ssl3_output_cert_chain() may generate an SSLfatal() error, - * for this case, we want to ignore it, argument for_comp = 1 - */ - if (!ssl3_output_cert_chain(sc, &tmppkt, cpk, 1)) - goto out; - WPACKET_get_total_written(&tmppkt, &ret); - - out: - WPACKET_cleanup(&tmppkt); - if (ret != 0 && data != NULL) - *data = (unsigned char *)buf.data; - else - OPENSSL_free(buf.data); - return ret; -} - -static int ssl_compress_one_cert(SSL *ssl, CERT_PKEY *cpk, int alg) -{ - unsigned char *cert_data = NULL; - OSSL_COMP_CERT *comp_cert = NULL; - size_t length; - - if (cpk == NULL - || alg == TLSEXT_comp_cert_none - || !ossl_comp_has_alg(alg)) - return 0; - - if ((length = ssl_get_cert_to_compress(ssl, cpk, &cert_data)) == 0) - return 0; - comp_cert = OSSL_COMP_CERT_from_uncompressed_data(cert_data, length, alg); - OPENSSL_free(cert_data); - if (comp_cert == NULL) - return 0; - - OSSL_COMP_CERT_free(cpk->comp_cert[alg]); - cpk->comp_cert[alg] = comp_cert; - return 1; -} - -/* alg_in can be 0, meaning any/all algorithms */ -static int ssl_compress_certs(SSL *ssl, CERT_PKEY *cpks, int alg_in) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - int i; - int j; - int alg; - int count = 0; - - if (sc == NULL - || cpks == NULL - || !ossl_comp_has_alg(alg_in)) - return 0; - - /* Look through the preferences to see what we have */ - for (i = 0; i < TLSEXT_comp_cert_limit; i++) { - /* - * alg = 0 means compress for everything, but only for algorithms enabled - * alg != 0 means compress for that algorithm if enabled - */ - alg = sc->cert_comp_prefs[i]; - if ((alg_in == 0 && alg != TLSEXT_comp_cert_none) - || (alg_in != 0 && alg == alg_in)) { - - for (j = 0; j < SSL_PKEY_NUM; j++) { - /* No cert, move on */ - if (cpks[j].x509 == NULL) - continue; - - if (!ssl_compress_one_cert(ssl, &cpks[j], alg)) - return 0; - - /* if the cert expanded, set the value in the CERT_PKEY to NULL */ - if (cpks[j].comp_cert[alg]->len >= cpks[j].comp_cert[alg]->orig_len) { - OSSL_COMP_CERT_free(cpks[j].comp_cert[alg]); - cpks[j].comp_cert[alg] = NULL; - } else { - count++; - } - } - } - } - return (count > 0); -} - -static size_t ssl_get_compressed_cert(SSL *ssl, CERT_PKEY *cpk, int alg, unsigned char **data, - size_t *orig_len) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - size_t cert_len = 0; - size_t comp_len = 0; - unsigned char *cert_data = NULL; - OSSL_COMP_CERT *comp_cert = NULL; - - if (sc == NULL - || cpk == NULL - || data == NULL - || orig_len == NULL - || !sc->server - || !SSL_in_before(ssl) - || !ossl_comp_has_alg(alg)) - return 0; - - if ((cert_len = ssl_get_cert_to_compress(ssl, cpk, &cert_data)) == 0) - goto err; - - comp_cert = OSSL_COMP_CERT_from_uncompressed_data(cert_data, cert_len, alg); - OPENSSL_free(cert_data); - if (comp_cert == NULL) - goto err; - - comp_len = comp_cert->len; - *orig_len = comp_cert->orig_len; - *data = comp_cert->data; - comp_cert->data = NULL; - err: - OSSL_COMP_CERT_free(comp_cert); - return comp_len; -} - -static int ossl_set1_compressed_cert(CERT *cert, int algorithm, - unsigned char *comp_data, size_t comp_length, - size_t orig_length) -{ - OSSL_COMP_CERT *comp_cert; - - /* No explicit cert set */ - if (cert == NULL || cert->key == NULL) - return 0; - - comp_cert = OSSL_COMP_CERT_from_compressed_data(comp_data, comp_length, - orig_length, algorithm); - if (comp_cert == NULL) - return 0; - - OSSL_COMP_CERT_free(cert->key->comp_cert[algorithm]); - cert->key->comp_cert[algorithm] = comp_cert; - - return 1; -} -#endif - -/*- - * Public API - */ -int SSL_CTX_set1_cert_comp_preference(SSL_CTX *ctx, int *algs, size_t len) -{ -#ifndef OPENSSL_NO_COMP_ALG - return ssl_set_cert_comp_pref(ctx->cert_comp_prefs, algs, len); -#else - return 0; -#endif -} - -int SSL_set1_cert_comp_preference(SSL *ssl, int *algs, size_t len) -{ -#ifndef OPENSSL_NO_COMP_ALG - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - return ssl_set_cert_comp_pref(sc->cert_comp_prefs, algs, len); -#else - return 0; -#endif -} - -int SSL_compress_certs(SSL *ssl, int alg) -{ -#ifndef OPENSSL_NO_COMP_ALG - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL || sc->cert == NULL) - return 0; - - return ssl_compress_certs(ssl, sc->cert->pkeys, alg); -#endif - return 0; -} - -int SSL_CTX_compress_certs(SSL_CTX *ctx, int alg) -{ - int ret = 0; -#ifndef OPENSSL_NO_COMP_ALG - SSL *new = SSL_new(ctx); - - if (new == NULL) - return 0; - - ret = ssl_compress_certs(new, ctx->cert->pkeys, alg); - SSL_free(new); -#endif - return ret; -} - -size_t SSL_get1_compressed_cert(SSL *ssl, int alg, unsigned char **data, size_t *orig_len) -{ -#ifndef OPENSSL_NO_COMP_ALG - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - CERT_PKEY *cpk = NULL; - - if (sc->cert != NULL) - cpk = sc->cert->key; - else - cpk = ssl->ctx->cert->key; - - return ssl_get_compressed_cert(ssl, cpk, alg, data, orig_len); -#else - return 0; -#endif -} - -size_t SSL_CTX_get1_compressed_cert(SSL_CTX *ctx, int alg, unsigned char **data, size_t *orig_len) -{ -#ifndef OPENSSL_NO_COMP_ALG - size_t ret; - SSL *new = SSL_new(ctx); - - ret = ssl_get_compressed_cert(new, ctx->cert->key, alg, data, orig_len); - SSL_free(new); - return ret; -#else - return 0; -#endif -} - -int SSL_CTX_set1_compressed_cert(SSL_CTX *ctx, int algorithm, unsigned char *comp_data, - size_t comp_length, size_t orig_length) -{ -#ifndef OPENSSL_NO_COMP_ALG - return ossl_set1_compressed_cert(ctx->cert, algorithm, comp_data, comp_length, orig_length); -#else - return 0; -#endif -} - -int SSL_set1_compressed_cert(SSL *ssl, int algorithm, unsigned char *comp_data, - size_t comp_length, size_t orig_length) -{ -#ifndef OPENSSL_NO_COMP_ALG - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - /* Cannot set a pre-compressed certificate on a client */ - if (sc == NULL || !sc->server) - return 0; - - return ossl_set1_compressed_cert(sc->cert, algorithm, comp_data, comp_length, orig_length); -#else - return 0; -#endif -} diff --git a/openssl/src/ssl/ssl_cert_table.h b/openssl/src/ssl/ssl_cert_table.h index e4dc8063b..f2dbf9605 100644 --- a/openssl/src/ssl/ssl_cert_table.h +++ b/openssl/src/ssl/ssl_cert_table.h @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -14,10 +14,18 @@ static const SSL_CERT_LOOKUP ssl_cert_info [] = { {EVP_PKEY_RSA, SSL_aRSA}, /* SSL_PKEY_RSA */ {EVP_PKEY_RSA_PSS, SSL_aRSA}, /* SSL_PKEY_RSA_PSS_SIGN */ {EVP_PKEY_DSA, SSL_aDSS}, /* SSL_PKEY_DSA_SIGN */ +#ifndef OPENSSL_NO_SM2 + {EVP_PKEY_EC, SSL_aECDSA | SSL_aSM2}, /* SSL_PKEY_ECC */ +#else {EVP_PKEY_EC, SSL_aECDSA}, /* SSL_PKEY_ECC */ - {NID_id_GostR3410_2001, SSL_aGOST01}, /* SSL_PKEY_GOST01 */ - {NID_id_GostR3410_2012_256, SSL_aGOST12}, /* SSL_PKEY_GOST12_256 */ - {NID_id_GostR3410_2012_512, SSL_aGOST12}, /* SSL_PKEY_GOST12_512 */ +#endif {EVP_PKEY_ED25519, SSL_aECDSA}, /* SSL_PKEY_ED25519 */ - {EVP_PKEY_ED448, SSL_aECDSA} /* SSL_PKEY_ED448 */ + {EVP_PKEY_ED448, SSL_aECDSA}, /* SSL_PKEY_ED448 */ + {EVP_PKEY_SM2, SSL_aSM2}, /* SSL_PKEY_ECC SM2 */ +#ifndef OPENSSL_NO_NTLS + {EVP_PKEY_SM2, SSL_aSM2}, /* SSL_PKEY_SM2_SIGN */ + {EVP_PKEY_SM2, SSL_aSM2}, /* SSL_PKEY_SM2_ENC */ + {EVP_PKEY_RSA, SSL_aRSA}, /* SSL_PKEY_RSA_SIGN */ + {EVP_PKEY_RSA, SSL_aRSA}, /* SSL_PKEY_RSA_ENC */ +#endif }; diff --git a/openssl/src/ssl/ssl_ciph.c b/openssl/src/ssl/ssl_ciph.c index ddde21b96..3e20412d6 100644 --- a/openssl/src/ssl/ssl_ciph.c +++ b/openssl/src/ssl/ssl_ciph.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -9,6 +9,8 @@ * https://www.openssl.org/source/license.html */ +/* for secure_getenv */ +#define _GNU_SOURCE #include #include #include @@ -34,27 +36,21 @@ static const ssl_cipher_table ssl_cipher_table_cipher[SSL_ENC_NUM_IDX] = { {SSL_DES, NID_des_cbc}, /* SSL_ENC_DES_IDX 0 */ {SSL_3DES, NID_des_ede3_cbc}, /* SSL_ENC_3DES_IDX 1 */ {SSL_RC4, NID_rc4}, /* SSL_ENC_RC4_IDX 2 */ - {SSL_RC2, NID_rc2_cbc}, /* SSL_ENC_RC2_IDX 3 */ - {SSL_IDEA, NID_idea_cbc}, /* SSL_ENC_IDEA_IDX 4 */ - {SSL_eNULL, NID_undef}, /* SSL_ENC_NULL_IDX 5 */ - {SSL_AES128, NID_aes_128_cbc}, /* SSL_ENC_AES128_IDX 6 */ - {SSL_AES256, NID_aes_256_cbc}, /* SSL_ENC_AES256_IDX 7 */ - {SSL_CAMELLIA128, NID_camellia_128_cbc}, /* SSL_ENC_CAMELLIA128_IDX 8 */ - {SSL_CAMELLIA256, NID_camellia_256_cbc}, /* SSL_ENC_CAMELLIA256_IDX 9 */ - {SSL_eGOST2814789CNT, NID_gost89_cnt}, /* SSL_ENC_GOST89_IDX 10 */ - {SSL_SEED, NID_seed_cbc}, /* SSL_ENC_SEED_IDX 11 */ - {SSL_AES128GCM, NID_aes_128_gcm}, /* SSL_ENC_AES128GCM_IDX 12 */ - {SSL_AES256GCM, NID_aes_256_gcm}, /* SSL_ENC_AES256GCM_IDX 13 */ - {SSL_AES128CCM, NID_aes_128_ccm}, /* SSL_ENC_AES128CCM_IDX 14 */ - {SSL_AES256CCM, NID_aes_256_ccm}, /* SSL_ENC_AES256CCM_IDX 15 */ - {SSL_AES128CCM8, NID_aes_128_ccm}, /* SSL_ENC_AES128CCM8_IDX 16 */ - {SSL_AES256CCM8, NID_aes_256_ccm}, /* SSL_ENC_AES256CCM8_IDX 17 */ - {SSL_eGOST2814789CNT12, NID_gost89_cnt_12}, /* SSL_ENC_GOST8912_IDX 18 */ - {SSL_CHACHA20POLY1305, NID_chacha20_poly1305}, /* SSL_ENC_CHACHA_IDX 19 */ - {SSL_ARIA128GCM, NID_aria_128_gcm}, /* SSL_ENC_ARIA128GCM_IDX 20 */ - {SSL_ARIA256GCM, NID_aria_256_gcm}, /* SSL_ENC_ARIA256GCM_IDX 21 */ - {SSL_MAGMA, NID_magma_ctr_acpkm}, /* SSL_ENC_MAGMA_IDX */ - {SSL_KUZNYECHIK, NID_kuznyechik_ctr_acpkm}, /* SSL_ENC_KUZNYECHIK_IDX */ + {SSL_eNULL, NID_undef}, /* SSL_ENC_NULL_IDX 3 */ + {SSL_AES128, NID_aes_128_cbc}, /* SSL_ENC_AES128_IDX 4 */ + {SSL_AES256, NID_aes_256_cbc}, /* SSL_ENC_AES256_IDX 5 */ + {SSL_AES128GCM, NID_aes_128_gcm}, /* SSL_ENC_AES128GCM_IDX 6 */ + {SSL_AES256GCM, NID_aes_256_gcm}, /* SSL_ENC_AES256GCM_IDX 7 */ + {SSL_AES128CCM, NID_aes_128_ccm}, /* SSL_ENC_AES128CCM_IDX 8 */ + {SSL_AES256CCM, NID_aes_256_ccm}, /* SSL_ENC_AES256CCM_IDX 9 */ + {SSL_AES128CCM8, NID_aes_128_ccm}, /* SSL_ENC_AES128CCM8_IDX 10 */ + {SSL_AES256CCM8, NID_aes_256_ccm}, /* SSL_ENC_AES256CCM8_IDX 11 */ + {SSL_CHACHA20POLY1305, NID_chacha20_poly1305}, /* SSL_ENC_CHACHA_IDX 12 */ + {SSL_SM4GCM, NID_sm4_gcm}, /* SSL_ENC_SM4_GCM_IDX */ + {SSL_SM4CCM, NID_sm4_ccm}, /* SSL_ENC_SM4_CCM_IDX */ +#ifndef OPENSSL_NO_SM4 + {SSL_SM4, NID_sm4_cbc} /* SSL_ENC_SM4_IDX */ +#endif }; #define SSL_COMP_NULL_IDX 0 @@ -71,18 +67,12 @@ static CRYPTO_ONCE ssl_load_builtin_comp_once = CRYPTO_ONCE_STATIC_INIT; static const ssl_cipher_table ssl_cipher_table_mac[SSL_MD_NUM_IDX] = { {SSL_MD5, NID_md5}, /* SSL_MD_MD5_IDX 0 */ {SSL_SHA1, NID_sha1}, /* SSL_MD_SHA1_IDX 1 */ - {SSL_GOST94, NID_id_GostR3411_94}, /* SSL_MD_GOST94_IDX 2 */ - {SSL_GOST89MAC, NID_id_Gost28147_89_MAC}, /* SSL_MD_GOST89MAC_IDX 3 */ - {SSL_SHA256, NID_sha256}, /* SSL_MD_SHA256_IDX 4 */ - {SSL_SHA384, NID_sha384}, /* SSL_MD_SHA384_IDX 5 */ - {SSL_GOST12_256, NID_id_GostR3411_2012_256}, /* SSL_MD_GOST12_256_IDX 6 */ - {SSL_GOST89MAC12, NID_gost_mac_12}, /* SSL_MD_GOST89MAC12_IDX 7 */ - {SSL_GOST12_512, NID_id_GostR3411_2012_512}, /* SSL_MD_GOST12_512_IDX 8 */ - {0, NID_md5_sha1}, /* SSL_MD_MD5_SHA1_IDX 9 */ - {0, NID_sha224}, /* SSL_MD_SHA224_IDX 10 */ - {0, NID_sha512}, /* SSL_MD_SHA512_IDX 11 */ - {SSL_MAGMAOMAC, NID_magma_mac}, /* sSL_MD_MAGMAOMAC_IDX */ - {SSL_KUZNYECHIKOMAC, NID_kuznyechik_mac} /* SSL_MD_KUZNYECHIKOMAC_IDX */ + {SSL_SHA256, NID_sha256}, /* SSL_MD_SHA256_IDX 2 */ + {SSL_SHA384, NID_sha384}, /* SSL_MD_SHA384_IDX 3 */ + {0, NID_md5_sha1}, /* SSL_MD_MD5_SHA1_IDX 4 */ + {0, NID_sha224}, /* SSL_MD_SHA224_IDX 5 */ + {0, NID_sha512}, /* SSL_MD_SHA512_IDX 6 */ + {SSL_SM3, NID_sm3} /* SSL_MD_SM3 */ }; /* *INDENT-OFF* */ @@ -95,8 +85,8 @@ static const ssl_cipher_table ssl_cipher_table_kx[] = { {SSL_kRSAPSK, NID_kx_rsa_psk}, {SSL_kPSK, NID_kx_psk}, {SSL_kSRP, NID_kx_srp}, - {SSL_kGOST, NID_kx_gost}, - {SSL_kGOST18, NID_kx_gost18}, + {SSL_kSM2, NID_kx_sm2}, + {SSL_kSM2DHE, NID_kx_sm2dhe}, {SSL_kANY, NID_kx_any} }; @@ -105,16 +95,15 @@ static const ssl_cipher_table ssl_cipher_table_auth[] = { {SSL_aECDSA, NID_auth_ecdsa}, {SSL_aPSK, NID_auth_psk}, {SSL_aDSS, NID_auth_dss}, - {SSL_aGOST01, NID_auth_gost01}, - {SSL_aGOST12, NID_auth_gost12}, {SSL_aSRP, NID_auth_srp}, + {SSL_aSM2, NID_auth_sm2}, {SSL_aNULL, NID_auth_null}, {SSL_aANY, NID_auth_any} }; /* *INDENT-ON* */ /* Utility function for table lookup */ -static int ssl_cipher_info_find(const ssl_cipher_table *table, +static int ssl_cipher_info_find(const ssl_cipher_table * table, size_t table_cnt, uint32_t mask) { size_t i; @@ -128,20 +117,12 @@ static int ssl_cipher_info_find(const ssl_cipher_table *table, #define ssl_cipher_info_lookup(table, x) \ ssl_cipher_info_find(table, OSSL_NELEM(table), x) -/* - * PKEY_TYPE for GOST89MAC is known in advance, but, because implementation - * is engine-provided, we'll fill it only if corresponding EVP_PKEY_METHOD is - * found - */ +/* this shoud be kept to the same order with ssl_cipher_table_mac */ static const int default_mac_pkey_id[SSL_MD_NUM_IDX] = { - /* MD5, SHA, GOST94, MAC89 */ - EVP_PKEY_HMAC, EVP_PKEY_HMAC, EVP_PKEY_HMAC, NID_undef, - /* SHA256, SHA384, GOST2012_256, MAC89-12 */ - EVP_PKEY_HMAC, EVP_PKEY_HMAC, EVP_PKEY_HMAC, NID_undef, - /* GOST2012_512 */ - EVP_PKEY_HMAC, - /* MD5/SHA1, SHA224, SHA512, MAGMAOMAC, KUZNYECHIKOMAC */ - NID_undef, NID_undef, NID_undef, NID_undef, NID_undef + /* MD5, SHA, SHA256, SHA384*/ + EVP_PKEY_HMAC, EVP_PKEY_HMAC, EVP_PKEY_HMAC, EVP_PKEY_HMAC, + /* MD5/SHA1, SHA224, SHA512, SM3 */ + NID_undef, NID_undef, NID_undef, EVP_PKEY_HMAC }; #define CIPHER_ADD 1 @@ -194,8 +175,16 @@ static const SSL_CIPHER cipher_aliases[] = { {0, SSL_TXT_kECDHEPSK, NULL, 0, SSL_kECDHEPSK}, {0, SSL_TXT_kDHEPSK, NULL, 0, SSL_kDHEPSK}, {0, SSL_TXT_kSRP, NULL, 0, SSL_kSRP}, - {0, SSL_TXT_kGOST, NULL, 0, SSL_kGOST}, - {0, SSL_TXT_kGOST18, NULL, 0, SSL_kGOST18}, +#ifndef OPENSSL_NO_NTLS + {0, SSL_TXT_kSM2, NULL, 0, SSL_kSM2}, + {0, SSL_TXT_kSM2DHE, NULL, 0, SSL_kSM2DHE}, + {0, NTLS_TXT_SM2DHE_WITH_SM4_SM3, NULL, 0, SSL_kSM2DHE, SSL_aSM2, SSL_SM4, + SSL_SM3, NTLS_VERSION, NTLS_VERSION, 0, 0, SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, 128, 128}, + {0, NTLS_TXT_SM2_WITH_SM4_SM3, NULL, 0, SSL_kSM2, SSL_aSM2, SSL_SM4, + SSL_SM3, NTLS_VERSION, NTLS_VERSION, 0, 0, SSL_HIGH, + SSL_HANDSHAKE_MAC_SM3 | TLS1_PRF_SM3, 128, 128}, +#endif /* server authentication aliases */ {0, SSL_TXT_aRSA, NULL, 0, 0, SSL_aRSA}, @@ -205,10 +194,8 @@ static const SSL_CIPHER cipher_aliases[] = { {0, SSL_TXT_aECDSA, NULL, 0, 0, SSL_aECDSA}, {0, SSL_TXT_ECDSA, NULL, 0, 0, SSL_aECDSA}, {0, SSL_TXT_aPSK, NULL, 0, 0, SSL_aPSK}, - {0, SSL_TXT_aGOST01, NULL, 0, 0, SSL_aGOST01}, - {0, SSL_TXT_aGOST12, NULL, 0, 0, SSL_aGOST12}, - {0, SSL_TXT_aGOST, NULL, 0, 0, SSL_aGOST01 | SSL_aGOST12}, {0, SSL_TXT_aSRP, NULL, 0, 0, SSL_aSRP}, + {0, SSL_TXT_aSM2, NULL, 0, 0, SSL_aSM2}, /* aliases combining key exchange and server authentication */ {0, SSL_TXT_EDH, NULL, 0, SSL_kDHE, ~SSL_aNULL}, @@ -225,12 +212,7 @@ static const SSL_CIPHER cipher_aliases[] = { /* symmetric encryption aliases */ {0, SSL_TXT_3DES, NULL, 0, 0, 0, SSL_3DES}, {0, SSL_TXT_RC4, NULL, 0, 0, 0, SSL_RC4}, - {0, SSL_TXT_RC2, NULL, 0, 0, 0, SSL_RC2}, - {0, SSL_TXT_IDEA, NULL, 0, 0, 0, SSL_IDEA}, - {0, SSL_TXT_SEED, NULL, 0, 0, 0, SSL_SEED}, {0, SSL_TXT_eNULL, NULL, 0, 0, 0, SSL_eNULL}, - {0, SSL_TXT_GOST, NULL, 0, 0, 0, - SSL_eGOST2814789CNT | SSL_eGOST2814789CNT12 | SSL_MAGMA | SSL_KUZNYECHIK}, {0, SSL_TXT_AES128, NULL, 0, 0, 0, SSL_AES128 | SSL_AES128GCM | SSL_AES128CCM | SSL_AES128CCM8}, {0, SSL_TXT_AES256, NULL, 0, 0, 0, @@ -240,33 +222,26 @@ static const SSL_CIPHER cipher_aliases[] = { {0, SSL_TXT_AES_CCM, NULL, 0, 0, 0, SSL_AES128CCM | SSL_AES256CCM | SSL_AES128CCM8 | SSL_AES256CCM8}, {0, SSL_TXT_AES_CCM_8, NULL, 0, 0, 0, SSL_AES128CCM8 | SSL_AES256CCM8}, - {0, SSL_TXT_CAMELLIA128, NULL, 0, 0, 0, SSL_CAMELLIA128}, - {0, SSL_TXT_CAMELLIA256, NULL, 0, 0, 0, SSL_CAMELLIA256}, - {0, SSL_TXT_CAMELLIA, NULL, 0, 0, 0, SSL_CAMELLIA}, {0, SSL_TXT_CHACHA20, NULL, 0, 0, 0, SSL_CHACHA20}, - {0, SSL_TXT_GOST2012_GOST8912_GOST8912, NULL, 0, 0, 0, SSL_eGOST2814789CNT12}, - {0, SSL_TXT_ARIA, NULL, 0, 0, 0, SSL_ARIA}, - {0, SSL_TXT_ARIA_GCM, NULL, 0, 0, 0, SSL_ARIA128GCM | SSL_ARIA256GCM}, - {0, SSL_TXT_ARIA128, NULL, 0, 0, 0, SSL_ARIA128GCM}, - {0, SSL_TXT_ARIA256, NULL, 0, 0, 0, SSL_ARIA256GCM}, {0, SSL_TXT_CBC, NULL, 0, 0, 0, SSL_CBC}, - + {0, SSL_TXT_SM4, NULL, 0, 0, 0, SSL_SM4}, /* MAC aliases */ {0, SSL_TXT_MD5, NULL, 0, 0, 0, 0, SSL_MD5}, {0, SSL_TXT_SHA1, NULL, 0, 0, 0, 0, SSL_SHA1}, {0, SSL_TXT_SHA, NULL, 0, 0, 0, 0, SSL_SHA1}, - {0, SSL_TXT_GOST94, NULL, 0, 0, 0, 0, SSL_GOST94}, - {0, SSL_TXT_GOST89MAC, NULL, 0, 0, 0, 0, SSL_GOST89MAC | SSL_GOST89MAC12}, {0, SSL_TXT_SHA256, NULL, 0, 0, 0, 0, SSL_SHA256}, {0, SSL_TXT_SHA384, NULL, 0, 0, 0, 0, SSL_SHA384}, - {0, SSL_TXT_GOST12, NULL, 0, 0, 0, 0, SSL_GOST12_256}, + {0, SSL_TXT_SM3, NULL, 0, 0, 0, 0, SSL_SM3}, /* protocol version aliases */ {0, SSL_TXT_SSLV3, NULL, 0, 0, 0, 0, 0, SSL3_VERSION}, {0, SSL_TXT_TLSV1, NULL, 0, 0, 0, 0, 0, TLS1_VERSION}, {0, "TLSv1.0", NULL, 0, 0, 0, 0, 0, TLS1_VERSION}, {0, SSL_TXT_TLSV1_2, NULL, 0, 0, 0, 0, 0, TLS1_2_VERSION}, +#ifndef OPENSSL_NO_NTLS + {0, SSL_TXT_NTLSV1_1, NULL, 0, 0, 0, 0, 0, NTLS1_1_VERSION}, +#endif /* strength classes */ {0, SSL_TXT_LOW, NULL, 0, 0, 0, 0, 0, 0, 0, 0, 0, SSL_LOW}, @@ -283,42 +258,6 @@ static const SSL_CIPHER cipher_aliases[] = { }; -/* - * Search for public key algorithm with given name and return its pkey_id if - * it is available. Otherwise return 0 - */ -#ifdef OPENSSL_NO_ENGINE - -static int get_optional_pkey_id(const char *pkey_name) -{ - const EVP_PKEY_ASN1_METHOD *ameth; - int pkey_id = 0; - ameth = EVP_PKEY_asn1_find_str(NULL, pkey_name, -1); - if (ameth && EVP_PKEY_asn1_get0_info(&pkey_id, NULL, NULL, NULL, NULL, - ameth) > 0) - return pkey_id; - return 0; -} - -#else - -static int get_optional_pkey_id(const char *pkey_name) -{ - const EVP_PKEY_ASN1_METHOD *ameth; - ENGINE *tmpeng = NULL; - int pkey_id = 0; - ameth = EVP_PKEY_asn1_find_str(&tmpeng, pkey_name, -1); - if (ameth) { - if (EVP_PKEY_asn1_get0_info(&pkey_id, NULL, NULL, NULL, NULL, - ameth) <= 0) - pkey_id = 0; - } - tls_engine_finish(tmpeng); - return pkey_id; -} - -#endif - int ssl_load_ciphers(SSL_CTX *ctx) { size_t i; @@ -358,7 +297,7 @@ int ssl_load_ciphers(SSL_CTX *ctx) /* * We ignore any errors from the fetches below. They are expected to fail - * if these algorithms are not available. + * if theose algorithms are not available. */ ERR_set_mark(); sig = EVP_SIGNATURE_fetch(ctx->libctx, "DSA", ctx->propq); @@ -381,6 +320,16 @@ int ssl_load_ciphers(SSL_CTX *ctx) ctx->disabled_auth_mask |= SSL_aECDSA; else EVP_SIGNATURE_free(sig); + kex = EVP_KEYEXCH_fetch(ctx->libctx, "SM2DH", ctx->propq); + if (kex == NULL) + ctx->disabled_mkey_mask |= SSL_kSM2DHE; + else + EVP_KEYEXCH_free(kex); + sig = EVP_SIGNATURE_fetch(ctx->libctx, "SM2", ctx->propq); + if (sig == NULL) + ctx->disabled_auth_mask |= SSL_aSM2; + else + EVP_SIGNATURE_free(sig); ERR_pop_to_mark(); #ifdef OPENSSL_NO_PSK @@ -391,57 +340,9 @@ int ssl_load_ciphers(SSL_CTX *ctx) ctx->disabled_mkey_mask |= SSL_kSRP; #endif - /* - * Check for presence of GOST 34.10 algorithms, and if they are not - * present, disable appropriate auth and key exchange - */ memcpy(ctx->ssl_mac_pkey_id, default_mac_pkey_id, sizeof(ctx->ssl_mac_pkey_id)); - ctx->ssl_mac_pkey_id[SSL_MD_GOST89MAC_IDX] = - get_optional_pkey_id(SN_id_Gost28147_89_MAC); - if (ctx->ssl_mac_pkey_id[SSL_MD_GOST89MAC_IDX]) - ctx->ssl_mac_secret_size[SSL_MD_GOST89MAC_IDX] = 32; - else - ctx->disabled_mac_mask |= SSL_GOST89MAC; - - ctx->ssl_mac_pkey_id[SSL_MD_GOST89MAC12_IDX] = - get_optional_pkey_id(SN_gost_mac_12); - if (ctx->ssl_mac_pkey_id[SSL_MD_GOST89MAC12_IDX]) - ctx->ssl_mac_secret_size[SSL_MD_GOST89MAC12_IDX] = 32; - else - ctx->disabled_mac_mask |= SSL_GOST89MAC12; - - ctx->ssl_mac_pkey_id[SSL_MD_MAGMAOMAC_IDX] = - get_optional_pkey_id(SN_magma_mac); - if (ctx->ssl_mac_pkey_id[SSL_MD_MAGMAOMAC_IDX]) - ctx->ssl_mac_secret_size[SSL_MD_MAGMAOMAC_IDX] = 32; - else - ctx->disabled_mac_mask |= SSL_MAGMAOMAC; - - ctx->ssl_mac_pkey_id[SSL_MD_KUZNYECHIKOMAC_IDX] = - get_optional_pkey_id(SN_kuznyechik_mac); - if (ctx->ssl_mac_pkey_id[SSL_MD_KUZNYECHIKOMAC_IDX]) - ctx->ssl_mac_secret_size[SSL_MD_KUZNYECHIKOMAC_IDX] = 32; - else - ctx->disabled_mac_mask |= SSL_KUZNYECHIKOMAC; - - if (!get_optional_pkey_id(SN_id_GostR3410_2001)) - ctx->disabled_auth_mask |= SSL_aGOST01 | SSL_aGOST12; - if (!get_optional_pkey_id(SN_id_GostR3410_2012_256)) - ctx->disabled_auth_mask |= SSL_aGOST12; - if (!get_optional_pkey_id(SN_id_GostR3410_2012_512)) - ctx->disabled_auth_mask |= SSL_aGOST12; - /* - * Disable GOST key exchange if no GOST signature algs are available * - */ - if ((ctx->disabled_auth_mask & (SSL_aGOST01 | SSL_aGOST12)) == - (SSL_aGOST01 | SSL_aGOST12)) - ctx->disabled_mkey_mask |= SSL_kGOST; - - if ((ctx->disabled_auth_mask & SSL_aGOST12) == SSL_aGOST12) - ctx->disabled_mkey_mask |= SSL_kGOST18; - return 1; } @@ -465,8 +366,7 @@ DEFINE_RUN_ONCE_STATIC(do_load_builtin_compressions) comp->method = method; comp->id = SSL_COMP_ZLIB_IDX; comp->name = COMP_get_name(method); - if (!sk_SSL_COMP_push(ssl_comp_methods, comp)) - OPENSSL_free(comp); + sk_SSL_COMP_push(ssl_comp_methods, comp); sk_SSL_COMP_sort(ssl_comp_methods); } } @@ -533,8 +433,7 @@ int ssl_cipher_get_evp(SSL_CTX *ctx, const SSL_SESSION *s, ctmp.id = s->compress_meth; if (ssl_comp_methods != NULL) { i = sk_SSL_COMP_find(ssl_comp_methods, &ctmp); - if (i >= 0) - *comp = sk_SSL_COMP_value(ssl_comp_methods, i); + *comp = sk_SSL_COMP_value(ssl_comp_methods, i); } /* If were only interested in comp then return success */ if ((enc == NULL) && (md == NULL)) @@ -557,14 +456,11 @@ int ssl_cipher_get_evp(SSL_CTX *ctx, const SSL_SESSION *s, if (c->algorithm_mac == SSL_AEAD) mac_pkey_type = NULL; } else { - const EVP_MD *digest = ctx->ssl_digest_methods[i]; - - if (digest == NULL - || !ssl_evp_md_up_ref(digest)) { + if (!ssl_evp_md_up_ref(ctx->ssl_digest_methods[i])) { ssl_evp_cipher_free(*enc); return 0; } - *md = digest; + *md = ctx->ssl_digest_methods[i]; if (mac_pkey_type != NULL) *mac_pkey_type = ctx->ssl_mac_pkey_id[i]; if (mac_secret_size != NULL) @@ -572,7 +468,7 @@ int ssl_cipher_get_evp(SSL_CTX *ctx, const SSL_SESSION *s, } if ((*enc != NULL) - && (*md != NULL + && (*md != NULL || (EVP_CIPHER_get_flags(*enc) & EVP_CIPH_FLAG_AEAD_CIPHER)) && (!mac_pkey_type || *mac_pkey_type != NID_undef)) { const EVP_CIPHER *evp = NULL; @@ -627,15 +523,14 @@ const EVP_MD *ssl_md(SSL_CTX *ctx, int idx) return ctx->ssl_digest_methods[idx]; } -const EVP_MD *ssl_handshake_md(SSL_CONNECTION *s) +const EVP_MD *ssl_handshake_md(SSL *s) { - return ssl_md(SSL_CONNECTION_GET_CTX(s), ssl_get_algorithm2(s)); + return ssl_md(s->ctx, ssl_get_algorithm2(s)); } -const EVP_MD *ssl_prf_md(SSL_CONNECTION *s) +const EVP_MD *ssl_prf_md(SSL *s) { - return ssl_md(SSL_CONNECTION_GET_CTX(s), - ssl_get_algorithm2(s) >> TLS1_PRF_DGST_SHIFT); + return ssl_md(s->ctx, ssl_get_algorithm2(s) >> TLS1_PRF_DGST_SHIFT); } #define ITEM_SEP(a) \ @@ -818,12 +713,11 @@ static void ssl_cipher_apply_rule(uint32_t cipher_id, uint32_t alg_mkey, const SSL_CIPHER *cp; int reverse = 0; - OSSL_TRACE_BEGIN(TLS_CIPHER) { + OSSL_TRACE_BEGIN(TLS_CIPHER){ BIO_printf(trc_out, "Applying rule %d with %08x/%08x/%08x/%08x/%08x %08x (%d)\n", - rule, (unsigned int)alg_mkey, (unsigned int)alg_auth, - (unsigned int)alg_enc, (unsigned int)alg_mac, min_tls, - (unsigned int)algo_strength, (int)strength_bits); + rule, alg_mkey, alg_auth, alg_enc, alg_mac, min_tls, + algo_strength, strength_bits); } if (rule == CIPHER_DEL || rule == CIPHER_BUMP) @@ -867,13 +761,9 @@ static void ssl_cipher_apply_rule(uint32_t cipher_id, uint32_t alg_mkey, BIO_printf(trc_out, "\nName: %s:" "\nAlgo = %08x/%08x/%08x/%08x/%08x Algo_strength = %08x\n", - cp->name, - (unsigned int)cp->algorithm_mkey, - (unsigned int)cp->algorithm_auth, - (unsigned int)cp->algorithm_enc, - (unsigned int)cp->algorithm_mac, - cp->min_tls, - (unsigned int)cp->algo_strength); + cp->name, cp->algorithm_mkey, cp->algorithm_auth, + cp->algorithm_enc, cp->algorithm_mac, cp->min_tls, + cp->algo_strength); } if (cipher_id != 0 && (cipher_id != cp->id)) continue; @@ -971,8 +861,10 @@ static int ssl_cipher_strength_sort(CIPHER_ORDER **head_p, } number_uses = OPENSSL_zalloc(sizeof(int) * (max_strength_bits + 1)); - if (number_uses == NULL) + if (number_uses == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } /* * Now find the strength_bits values actually used @@ -1010,7 +902,7 @@ static int ssl_cipher_process_rulestr(const char *rule_str, retval = 1; l = rule_str; - for (;;) { + for ( ; ; ) { ch = *l; if (ch == '\0') @@ -1051,9 +943,9 @@ static int ssl_cipher_process_rulestr(const char *rule_str, while (((ch >= 'A') && (ch <= 'Z')) || ((ch >= '0') && (ch <= '9')) || ((ch >= 'a') && (ch <= 'z')) || - (ch == '-') || (ch == '_') || (ch == '.') || (ch == '=')) + (ch == '-') || (ch == '.') || (ch == '=')) #else - while (isalnum((unsigned char)ch) || (ch == '-') || (ch == '_') || (ch == '.') + while (isalnum((unsigned char)ch) || (ch == '-') || (ch == '.') || (ch == '=')) #endif { @@ -1068,7 +960,9 @@ static int ssl_cipher_process_rulestr(const char *rule_str, * alphanumeric, so we call this an error. */ ERR_raise(ERR_LIB_SSL, SSL_R_INVALID_COMMAND); - return 0; + retval = found = 0; + l++; + break; } if (rule == CIPHER_SPECIAL) { @@ -1102,11 +996,6 @@ static int ssl_cipher_process_rulestr(const char *rule_str, && (ca_list[j]->name[buflen] == '\0')) { found = 1; break; - } else if (ca_list[j]->stdname != NULL - && strncmp(buf, ca_list[j]->stdname, buflen) == 0 - && ca_list[j]->stdname[buflen] == '\0') { - found = 1; - break; } else j++; } @@ -1223,10 +1112,10 @@ static int ssl_cipher_process_rulestr(const char *rule_str, */ if (rule == CIPHER_SPECIAL) { /* special command */ ok = 0; - if ((buflen == 8) && HAS_PREFIX(buf, "STRENGTH")) { + if ((buflen == 8) && strncmp(buf, "STRENGTH", 8) == 0) { ok = ssl_cipher_strength_sort(head_p, tail_p); - } else if (buflen == 10 && CHECK_AND_SKIP_PREFIX(buf, "SECLEVEL=")) { - int level = *buf - '0'; + } else if (buflen == 10 && strncmp(buf, "SECLEVEL=", 9) == 0) { + int level = buf[9] - '0'; if (level < 0 || level > 5) { ERR_raise(ERR_LIB_SSL, SSL_R_INVALID_COMMAND); } else { @@ -1266,14 +1155,14 @@ static int check_suiteb_cipher_list(const SSL_METHOD *meth, CERT *c, const char **prule_str) { unsigned int suiteb_flags = 0, suiteb_comb2 = 0; - if (HAS_PREFIX(*prule_str, "SUITEB128ONLY")) { + if (strncmp(*prule_str, "SUITEB128ONLY", 13) == 0) { suiteb_flags = SSL_CERT_FLAG_SUITEB_128_LOS_ONLY; - } else if (HAS_PREFIX(*prule_str, "SUITEB128C2")) { + } else if (strncmp(*prule_str, "SUITEB128C2", 11) == 0) { suiteb_comb2 = 1; suiteb_flags = SSL_CERT_FLAG_SUITEB_128_LOS; - } else if (HAS_PREFIX(*prule_str, "SUITEB128")) { + } else if (strncmp(*prule_str, "SUITEB128", 9) == 0) { suiteb_flags = SSL_CERT_FLAG_SUITEB_128_LOS; - } else if (HAS_PREFIX(*prule_str, "SUITEB192")) { + } else if (strncmp(*prule_str, "SUITEB192", 9) == 0) { suiteb_flags = SSL_CERT_FLAG_SUITEB_192_LOS; } @@ -1435,26 +1324,66 @@ int SSL_CTX_set_ciphersuites(SSL_CTX *ctx, const char *str) int SSL_set_ciphersuites(SSL *s, const char *str) { STACK_OF(SSL_CIPHER) *cipher_list; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - int ret; - - if (sc == NULL) - return 0; + int ret = set_ciphersuites(&(s->tls13_ciphersuites), str); - ret = set_ciphersuites(&(sc->tls13_ciphersuites), str); - - if (sc->cipher_list == NULL) { + if (s->cipher_list == NULL) { if ((cipher_list = SSL_get_ciphers(s)) != NULL) - sc->cipher_list = sk_SSL_CIPHER_dup(cipher_list); + s->cipher_list = sk_SSL_CIPHER_dup(cipher_list); } - if (ret && sc->cipher_list != NULL) - return update_cipher_list(s->ctx, &sc->cipher_list, - &sc->cipher_list_by_id, - sc->tls13_ciphersuites); + if (ret && s->cipher_list != NULL) + return update_cipher_list(s->ctx, &s->cipher_list, &s->cipher_list_by_id, + s->tls13_ciphersuites); return ret; } +#ifdef SYSTEM_CIPHERS_FILE +static char *load_system_str(const char *suffix) +{ + FILE *fp; + char buf[1024]; + char *new_rules; + const char *ciphers_path; + unsigned len, slen; + + if ((ciphers_path = secure_getenv("OPENSSL_SYSTEM_CIPHERS_OVERRIDE")) == NULL) + ciphers_path = SYSTEM_CIPHERS_FILE; + fp = fopen(ciphers_path, "r"); + if (fp == NULL || fgets(buf, sizeof(buf), fp) == NULL) { + /* cannot open or file is empty */ + snprintf(buf, sizeof(buf), "%s", SSL_DEFAULT_CIPHER_LIST); + } + + if (fp) + fclose(fp); + + slen = strlen(suffix); + len = strlen(buf); + + if (buf[len - 1] == '\n') { + len--; + buf[len] = 0; + } + if (buf[len - 1] == '\r') { + len--; + buf[len] = 0; + } + + new_rules = OPENSSL_malloc(len + slen + 1); + if (new_rules == 0) + return NULL; + + memcpy(new_rules, buf, len); + if (slen > 0) { + memcpy(&new_rules[len], suffix, slen); + len += slen; + } + new_rules[len] = 0; + + return new_rules; +} +#endif + STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, STACK_OF(SSL_CIPHER) *tls13_ciphersuites, STACK_OF(SSL_CIPHER) **cipher_list, @@ -1469,15 +1398,25 @@ STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, CIPHER_ORDER *co_list = NULL, *head = NULL, *tail = NULL, *curr; const SSL_CIPHER **ca_list = NULL; const SSL_METHOD *ssl_method = ctx->method; +#ifdef SYSTEM_CIPHERS_FILE + char *new_rules = NULL; + + if (rule_str != NULL && strncmp(rule_str, "PROFILE=SYSTEM", 14) == 0) { + char *p = rule_str + 14; + + new_rules = load_system_str(p); + rule_str = new_rules; + } +#endif /* * Return with error if nothing to do. */ if (rule_str == NULL || cipher_list == NULL || cipher_list_by_id == NULL) - return NULL; + goto err; if (!check_suiteb_cipher_list(ssl_method, c, &rule_str)) - return NULL; + goto err; /* * To reduce the work to do we only want to process the compiled @@ -1496,10 +1435,10 @@ STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, */ num_of_ciphers = ssl_method->num_ciphers(); - if (num_of_ciphers > 0) { - co_list = OPENSSL_malloc(sizeof(*co_list) * num_of_ciphers); - if (co_list == NULL) - return NULL; /* Failure */ + co_list = OPENSSL_malloc(sizeof(*co_list) * num_of_ciphers); + if (co_list == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto err; } ssl_cipher_collect_ciphers(ssl_method, num_of_ciphers, @@ -1565,8 +1504,7 @@ STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, * in force within each class */ if (!ssl_cipher_strength_sort(&head, &tail)) { - OPENSSL_free(co_list); - return NULL; + goto err; } /* @@ -1610,8 +1548,8 @@ STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, num_of_alias_max = num_of_ciphers + num_of_group_aliases + 1; ca_list = OPENSSL_malloc(sizeof(*ca_list) * num_of_alias_max); if (ca_list == NULL) { - OPENSSL_free(co_list); - return NULL; /* Failure */ + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto err; } ssl_cipher_collect_aliases(ca_list, num_of_group_aliases, disabled_mkey, disabled_auth, disabled_enc, @@ -1623,7 +1561,7 @@ STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, */ ok = 1; rule_p = rule_str; - if (HAS_PREFIX(rule_str, "DEFAULT")) { + if (strncmp(rule_str, "DEFAULT", 7) == 0) { ok = ssl_cipher_process_rulestr(OSSL_default_cipher_list(), &head, &tail, ca_list, c); rule_p += 7; @@ -1637,8 +1575,7 @@ STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, OPENSSL_free(ca_list); /* Not needed anymore */ if (!ok) { /* Rule processing failure */ - OPENSSL_free(co_list); - return NULL; + goto err; } /* @@ -1646,10 +1583,13 @@ STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, * if we cannot get one. */ if ((cipherstack = sk_SSL_CIPHER_new_null()) == NULL) { - OPENSSL_free(co_list); - return NULL; + goto err; } +#ifdef SYSTEM_CIPHERS_FILE + OPENSSL_free(new_rules); /* Not needed anymore */ +#endif + /* Add TLSv1.3 ciphers first - we always prefer those if possible */ for (i = 0; i < sk_SSL_CIPHER_num(tls13_ciphersuites); i++) { const SSL_CIPHER *sslc = sk_SSL_CIPHER_value(tls13_ciphersuites, i); @@ -1701,6 +1641,13 @@ STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, *cipher_list = cipherstack; return cipherstack; + +err: + OPENSSL_free(co_list); +#ifdef SYSTEM_CIPHERS_FILE + OPENSSL_free(new_rules); +#endif + return NULL; } char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) @@ -1708,12 +1655,14 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) const char *ver; const char *kx, *au, *enc, *mac; uint32_t alg_mkey, alg_auth, alg_enc, alg_mac; - static const char *const format = "%-30s %-7s Kx=%-8s Au=%-5s Enc=%-22s Mac=%-4s\n"; + static const char *format = "%-30s %-7s Kx=%-8s Au=%-5s Enc=%-22s Mac=%-4s\n"; if (buf == NULL) { len = 128; - if ((buf = OPENSSL_malloc(len)) == NULL) + if ((buf = OPENSSL_malloc(len)) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return NULL; + } } else if (len < 128) { return NULL; } @@ -1750,11 +1699,11 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) case SSL_kSRP: kx = "SRP"; break; - case SSL_kGOST: - kx = "GOST"; + case SSL_kSM2: + kx = "SM2"; break; - case SSL_kGOST18: - kx = "GOST18"; + case SSL_kSM2DHE: + kx = "SM2DHE"; break; case SSL_kANY: kx = "any"; @@ -1782,12 +1731,8 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) case SSL_aSRP: au = "SRP"; break; - case SSL_aGOST01: - au = "GOST01"; - break; - /* New GOST ciphersuites have both SSL_aGOST12 and SSL_aGOST01 bits */ - case (SSL_aGOST12 | SSL_aGOST01): - au = "GOST12"; + case SSL_aSM2: + au = "SM2"; break; case SSL_aANY: au = "any"; @@ -1807,12 +1752,6 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) case SSL_RC4: enc = "RC4(128)"; break; - case SSL_RC2: - enc = "RC2(128)"; - break; - case SSL_IDEA: - enc = "IDEA(128)"; - break; case SSL_eNULL: enc = "None"; break; @@ -1840,33 +1779,17 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) case SSL_AES256CCM8: enc = "AESCCM8(256)"; break; - case SSL_CAMELLIA128: - enc = "Camellia(128)"; - break; - case SSL_CAMELLIA256: - enc = "Camellia(256)"; - break; - case SSL_ARIA128GCM: - enc = "ARIAGCM(128)"; - break; - case SSL_ARIA256GCM: - enc = "ARIAGCM(256)"; - break; - case SSL_SEED: - enc = "SEED(128)"; - break; - case SSL_eGOST2814789CNT: - case SSL_eGOST2814789CNT12: - enc = "GOST89(256)"; + case SSL_CHACHA20POLY1305: + enc = "CHACHA20/POLY1305(256)"; break; - case SSL_MAGMA: - enc = "MAGMA"; + case SSL_SM4CCM: + enc = "SM4-CCM(128)"; break; - case SSL_KUZNYECHIK: - enc = "KUZNYECHIK"; + case SSL_SM4GCM: + enc = "SM4-GCM(128)"; break; - case SSL_CHACHA20POLY1305: - enc = "CHACHA20/POLY1305(256)"; + case SSL_SM4: + enc = "SM4(128)"; break; default: enc = "unknown"; @@ -1889,16 +1812,8 @@ char *SSL_CIPHER_description(const SSL_CIPHER *cipher, char *buf, int len) case SSL_AEAD: mac = "AEAD"; break; - case SSL_GOST89MAC: - case SSL_GOST89MAC12: - mac = "GOST89"; - break; - case SSL_GOST94: - mac = "GOST94"; - break; - case SSL_GOST12_256: - case SSL_GOST12_512: - mac = "GOST2012"; + case SSL_SM3: + mac = "SM3"; break; default: mac = "unknown"; @@ -2055,8 +1970,10 @@ int SSL_COMP_add_compression_method(int id, COMP_METHOD *cm) } comp = OPENSSL_malloc(sizeof(*comp)); - if (comp == NULL) + if (comp == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 1; + } comp->id = id; comp->method = cm; @@ -2068,7 +1985,7 @@ int SSL_COMP_add_compression_method(int id, COMP_METHOD *cm) } if (ssl_comp_methods == NULL || !sk_SSL_COMP_push(ssl_comp_methods, comp)) { OPENSSL_free(comp); - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 1; } return 0; @@ -2102,11 +2019,10 @@ int SSL_COMP_get_id(const SSL_COMP *comp) #endif } -const SSL_CIPHER *ssl_get_cipher_by_char(SSL_CONNECTION *s, - const unsigned char *ptr, +const SSL_CIPHER *ssl_get_cipher_by_char(SSL *ssl, const unsigned char *ptr, int all) { - const SSL_CIPHER *c = SSL_CONNECTION_GET_SSL(s)->method->get_cipher_by_char(ptr); + const SSL_CIPHER *c = ssl->method->get_cipher_by_char(ptr); if (c == NULL || (!all && c->valid == 0)) return NULL; @@ -2156,16 +2072,6 @@ int SSL_CIPHER_get_auth_nid(const SSL_CIPHER *c) return ssl_cipher_table_auth[i].nid; } -int ssl_get_md_idx(int md_nid) { - int i; - - for(i = 0; i < SSL_MD_NUM_IDX; i++) { - if (md_nid == ssl_cipher_table_mac[i].nid) - return i; - } - return -1; -} - const EVP_MD *SSL_CIPHER_get_handshake_digest(const SSL_CIPHER *c) { int idx = c->algorithm2 & SSL_HANDSHAKE_MAC_MASK; @@ -2188,7 +2094,7 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead, /* Some hard-coded numbers for the CCM/Poly1305 MAC overhead * because there are no handy #defines for those. */ - if (c->algorithm_enc & (SSL_AESGCM | SSL_ARIAGCM)) { + if (c->algorithm_enc & SSL_AESGCM) { out = EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN; } else if (c->algorithm_enc & (SSL_AES128CCM | SSL_AES256CCM)) { out = EVP_CCM_TLS_EXPLICIT_IV_LEN + 16; @@ -2221,8 +2127,6 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead, in = 1; /* padding length byte */ out = EVP_CIPHER_get_iv_length(e_ciph); blk = EVP_CIPHER_get_block_size(e_ciph); - if (blk == 0) - return 0; } } @@ -2236,15 +2140,22 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead, int ssl_cert_is_disabled(SSL_CTX *ctx, size_t idx) { - const SSL_CERT_LOOKUP *cl; + const SSL_CERT_LOOKUP *cl = ssl_cert_lookup_by_idx(idx); + uint32_t amask; - /* A provider-loaded key type is always enabled */ - if (idx >= SSL_PKEY_NUM) - return 0; + if (cl == NULL) + return 1; + + amask = cl->amask; + +#ifndef OPENSSL_NO_SM2 + if (cl->nid == EVP_PKEY_EC && (ctx->disabled_auth_mask & SSL_aSM2) != 0) + amask &= ~SSL_aSM2; +#endif - cl = ssl_cert_lookup_by_idx(idx, ctx); - if (cl == NULL || (cl->amask & ctx->disabled_auth_mask) != 0) + if ((amask & ctx->disabled_auth_mask) != 0) return 1; + return 0; } @@ -2267,5 +2178,59 @@ const char *OSSL_default_ciphersuites(void) { return "TLS_AES_256_GCM_SHA384:" "TLS_CHACHA20_POLY1305_SHA256:" - "TLS_AES_128_GCM_SHA256"; + "TLS_AES_128_GCM_SHA256" +#if (!defined OPENSSL_NO_SM2) && (!defined OPENSSL_NO_SM3) \ + && (!defined OPENSSL_NO_SM4) + ":TLS_SM4_GCM_SM3" + ":TLS_SM4_CCM_SM3" +#endif + ; +} + +unsigned long BABASSL_CIPHER_get_mkey(const SSL_CIPHER *c) +{ + return c->algorithm_mkey; +} + +unsigned long BABASSL_CIPHER_get_auth(const SSL_CIPHER *c) +{ + return c->algorithm_auth; } + +unsigned long BABASSL_CIPHER_get_enc(const SSL_CIPHER *c) +{ + return c->algorithm_enc; +} + +unsigned long BABASSL_CIPHER_get_mac(const SSL_CIPHER *c) +{ + return c->algorithm_mac; +} + +#ifndef OPENSSL_NO_QUIC +int SSL_CIPHER_get_prf_nid(const SSL_CIPHER *c) +{ + switch (c->algorithm2 & (0xFF << TLS1_PRF_DGST_SHIFT)) { + default: + break; + case TLS1_PRF_SHA1_MD5: /* TLS1_PRF */ + return NID_md5_sha1; + case TLS1_PRF_SHA256: + return NID_sha256; + case TLS1_PRF_SHA384: + return NID_sha384; + } + /* TLSv1.3 ciphers don't specify separate PRF */ + switch (c->algorithm2 & SSL_HANDSHAKE_MAC_MASK) { + default: + break; + case SSL_HANDSHAKE_MAC_MD5_SHA1: /* SSL_HANDSHAKE_MAC_DEFAULT */ + return NID_md5_sha1; + case SSL_HANDSHAKE_MAC_SHA256: + return NID_sha256; + case SSL_HANDSHAKE_MAC_SHA384: + return NID_sha384; + } + return NID_undef; +} +#endif diff --git a/openssl/src/ssl/ssl_conf.c b/openssl/src/ssl/ssl_conf.c index 77de00542..bd223c29b 100644 --- a/openssl/src/ssl/ssl_conf.c +++ b/openssl/src/ssl/ssl_conf.c @@ -1,5 +1,5 @@ /* - * Copyright 2012-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2012-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -13,6 +13,7 @@ #include #include #include +#include #include "internal/nelem.h" /* @@ -241,7 +242,7 @@ static int cmd_ECDHParameters(SSL_CONF_CTX *cctx, const char *value) return 1; /* ECDHParameters accepts a single group name */ - if (strchr(value, ':') != NULL) + if (strstr(value, ":") != NULL) return 0; if (cctx->ctx) @@ -318,7 +319,11 @@ static int protocol_from_string(const char *value) {"TLSv1.2", TLS1_2_VERSION}, {"TLSv1.3", TLS1_3_VERSION}, {"DTLSv1", DTLS1_VERSION}, - {"DTLSv1.2", DTLS1_2_VERSION} + {"DTLSv1.2", DTLS1_2_VERSION}, +#ifndef OPENSSL_NO_NTLS + {"NTLS", NTLS_VERSION}, + {"MIN_VERSION_WITH_NTLS", MIN_VERSION_WITH_NTLS} +#endif }; size_t i; size_t n = OSSL_NELEM(versions); @@ -337,7 +342,7 @@ static int min_max_proto(SSL_CONF_CTX *cctx, const char *value, int *bound) if (cctx->ctx != NULL) method_version = cctx->ctx->method->version; else if (cctx->ssl != NULL) - method_version = cctx->ssl->defltmeth->version; + method_version = cctx->ssl->ctx->method->version; else return 0; if ((new_version = protocol_from_string(value)) < 0) @@ -384,25 +389,17 @@ static int cmd_Options(SSL_CONF_CTX *cctx, const char *value) SSL_FLAG_TBL_SRV("ECDHSingle", SSL_OP_SINGLE_ECDH_USE), SSL_FLAG_TBL("UnsafeLegacyRenegotiation", SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION), - SSL_FLAG_TBL("UnsafeLegacyServerConnect", - SSL_OP_LEGACY_SERVER_CONNECT), SSL_FLAG_TBL("ClientRenegotiation", SSL_OP_ALLOW_CLIENT_RENEGOTIATION), SSL_FLAG_TBL_INV("EncryptThenMac", SSL_OP_NO_ENCRYPT_THEN_MAC), SSL_FLAG_TBL("NoRenegotiation", SSL_OP_NO_RENEGOTIATION), SSL_FLAG_TBL("AllowNoDHEKEX", SSL_OP_ALLOW_NO_DHE_KEX), - SSL_FLAG_TBL("PreferNoDHEKEX", SSL_OP_PREFER_NO_DHE_KEX), SSL_FLAG_TBL("PrioritizeChaCha", SSL_OP_PRIORITIZE_CHACHA), SSL_FLAG_TBL("MiddleboxCompat", SSL_OP_ENABLE_MIDDLEBOX_COMPAT), SSL_FLAG_TBL_INV("AntiReplay", SSL_OP_NO_ANTI_REPLAY), SSL_FLAG_TBL_INV("ExtendedMasterSecret", SSL_OP_NO_EXTENDED_MASTER_SECRET), SSL_FLAG_TBL_INV("CANames", SSL_OP_DISABLE_TLSEXT_CA_NAMES), - SSL_FLAG_TBL("KTLS", SSL_OP_ENABLE_KTLS), - SSL_FLAG_TBL_CERT("StrictCertCheck", SSL_CERT_FLAG_TLS_STRICT), - SSL_FLAG_TBL_INV("TxCertificateCompression", SSL_OP_NO_TX_CERTIFICATE_COMPRESSION), - SSL_FLAG_TBL_INV("RxCertificateCompression", SSL_OP_NO_RX_CERTIFICATE_COMPRESSION), - SSL_FLAG_TBL("KTLSTxZerocopySendfile", SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE), - SSL_FLAG_TBL("IgnoreUnexpectedEOF", SSL_OP_IGNORE_UNEXPECTED_EOF), + SSL_FLAG_TBL("KTLS", SSL_OP_ENABLE_KTLS) }; if (value == NULL) return -3; @@ -436,23 +433,16 @@ static int cmd_Certificate(SSL_CONF_CTX *cctx, const char *value) { int rv = 1; CERT *c = NULL; - if (cctx->ctx != NULL) { + if (cctx->ctx) { rv = SSL_CTX_use_certificate_chain_file(cctx->ctx, value); c = cctx->ctx->cert; } - if (cctx->ssl != NULL) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(cctx->ssl); - - if (sc != NULL) { - rv = SSL_use_certificate_chain_file(cctx->ssl, value); - c = sc->cert; - } else { - rv = 0; - } + if (cctx->ssl) { + rv = SSL_use_certificate_chain_file(cctx->ssl, value); + c = cctx->ssl->cert; } - if (rv > 0 && c != NULL && cctx->flags & SSL_CONF_FLAG_REQUIRE_PRIVATE) { + if (rv > 0 && c && cctx->flags & SSL_CONF_FLAG_REQUIRE_PRIVATE) { char **pfilename = &cctx->cert_filename[c->key - c->pkeys]; - OPENSSL_free(*pfilename); *pfilename = OPENSSL_strdup(value); if (*pfilename == NULL) @@ -474,6 +464,89 @@ static int cmd_PrivateKey(SSL_CONF_CTX *cctx, const char *value) return rv > 0; } +#ifndef OPENSSL_NO_NTLS +static int cmd_EncCertificate(SSL_CONF_CTX *cctx, const char *value) +{ + int rv = 1; + CERT *c = NULL; + + if (cctx->ctx) { + /* FIXME: currently we assume all SM2 certs in PEM format */ + rv = SSL_CTX_use_enc_certificate_file(cctx->ctx, value, + SSL_FILETYPE_PEM); + c = cctx->ctx->cert; + } + + if (cctx->ssl) { + /* FIXME: setting certificates for SSL is not supported yet */ + rv = 0; + } + + if (rv > 0 && c && cctx->flags & SSL_CONF_FLAG_REQUIRE_PRIVATE) { + char **pfilename = &cctx->cert_filename[c->key - c->pkeys]; + OPENSSL_free(*pfilename); + *pfilename = OPENSSL_strdup(value); + if (!*pfilename) + rv = 0; + } + + return rv > 0; +} + +static int cmd_EncPrivateKey(SSL_CONF_CTX *cctx, const char *value) +{ + int rv = 1; + if (!(cctx->flags & SSL_CONF_FLAG_CERTIFICATE)) + return -2; + if (cctx->ctx) + rv = SSL_CTX_use_enc_PrivateKey_file(cctx->ctx, value, SSL_FILETYPE_PEM); + if (cctx->ssl) + rv = 0; + return rv > 0; +} + +static int cmd_SignCertificate(SSL_CONF_CTX *cctx, const char *value) +{ + int rv = 1; + CERT *c = NULL; + + if (cctx->ctx) { + /* FIXME: currently we assume all sign certs in PEM format */ + rv = SSL_CTX_use_sign_certificate_file(cctx->ctx, value, + SSL_FILETYPE_PEM); + c = cctx->ctx->cert; + } + + if (cctx->ssl) { + /* FIXME: setting certificates for SSL is not supported yet */ + rv = 0; + } + + if (rv > 0 && c && cctx->flags & SSL_CONF_FLAG_REQUIRE_PRIVATE) { + char **pfilename = &cctx->cert_filename[c->key - c->pkeys]; + OPENSSL_free(*pfilename); + *pfilename = OPENSSL_strdup(value); + if (!*pfilename) + rv = 0; + } + + return rv > 0; +} + +static int cmd_SignPrivateKey(SSL_CONF_CTX *cctx, const char *value) +{ + int rv = 1; + if (!(cctx->flags & SSL_CONF_FLAG_CERTIFICATE)) + return -2; + if (cctx->ctx) + rv = SSL_CTX_use_sign_PrivateKey_file(cctx->ctx, value, + SSL_FILETYPE_PEM); + if (cctx->ssl) + rv = 0; + return rv > 0; +} +#endif + static int cmd_ServerInfoFile(SSL_CONF_CTX *cctx, const char *value) { int rv = 1; @@ -496,12 +569,7 @@ static int do_store(SSL_CONF_CTX *cctx, cert = cctx->ctx->cert; ctx = cctx->ctx; } else if (cctx->ssl != NULL) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(cctx->ssl); - - if (sc == NULL) - return 0; - - cert = sc->cert; + cert = cctx->ssl->cert; ctx = cctx->ssl->ctx; } else { return 1; @@ -682,6 +750,152 @@ static int cmd_NumTickets(SSL_CONF_CTX *cctx, const char *value) return rv; } +#ifndef OPENSSL_NO_NTLS +static int cmd_Enable_ntls(SSL_CONF_CTX *cctx, const char *value) +{ + if (strcmp(value, "on") == 0) { + if (cctx->ctx) + SSL_CTX_enable_ntls(cctx->ctx); + if (cctx->ssl) + SSL_enable_ntls(cctx->ssl); + } else { + if (cctx->ctx) + SSL_CTX_disable_ntls(cctx->ctx); + if (cctx->ssl) + SSL_disable_ntls(cctx->ssl); + } + return 1; +} + +static int cmd_Enable_force_ntls(SSL_CONF_CTX *cctx, const char *value) +{ + if (strcmp(value, "on") == 0) { + if (cctx->ctx) + SSL_CTX_enable_force_ntls(cctx->ctx); + if (cctx->ssl) + SSL_enable_force_ntls(cctx->ssl); + } else { + if (cctx->ctx) + SSL_CTX_disable_force_ntls(cctx->ctx); + if (cctx->ssl) + SSL_disable_force_ntls(cctx->ssl); + } + return 1; +} +#endif + +#ifndef OPENSSL_NO_SM2 +static int cmd_Enable_sm_tls13_strict(SSL_CONF_CTX *cctx, const char *value) +{ + if (strcmp(value, "on") == 0) { + if (cctx->ctx) + SSL_CTX_enable_sm_tls13_strict(cctx->ctx); + if (cctx->ssl) + SSL_enable_sm_tls13_strict(cctx->ssl); + } else { + if (cctx->ctx) + SSL_CTX_disable_sm_tls13_strict(cctx->ctx); + if (cctx->ssl) + SSL_disable_sm_tls13_strict(cctx->ssl); + } + return 1; +} +#endif + +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +static int cmd_DC(SSL_CONF_CTX *cctx, const char *value) +{ + int rv = 1; + + if (cctx->ctx != NULL) + rv = SSL_CTX_use_dc_file(cctx->ctx, value, 0); + + if (cctx->ssl != NULL) + rv = SSL_use_dc_file(cctx->ssl, value, 0); + + return rv > 0; +} + +static int cmd_DCKey(SSL_CONF_CTX *cctx, const char *value) +{ + int rv = 1; + if (!(cctx->flags & SSL_CONF_FLAG_CERTIFICATE)) + return -2; + + if (cctx->ctx != NULL) + rv = SSL_CTX_use_dc_PrivateKey_file(cctx->ctx, value, SSL_FILETYPE_PEM); + + if (cctx->ssl != NULL) + rv = SSL_use_dc_PrivateKey_file(cctx->ssl, value, SSL_FILETYPE_PEM); + + return rv > 0; +} + +static int cmd_Enable_verify_peer_by_dc(SSL_CONF_CTX *cctx, const char *value) +{ + if (strcmp(value, "on") == 0) { + if (cctx->ctx != NULL) + SSL_CTX_enable_verify_peer_by_dc(cctx->ctx); + if (cctx->ssl != NULL) + SSL_enable_verify_peer_by_dc(cctx->ssl); + } else { + if (cctx->ctx != NULL) + SSL_CTX_disable_verify_peer_by_dc(cctx->ctx); + if (cctx->ssl != NULL) + SSL_disable_verify_peer_by_dc(cctx->ssl); + } + return 1; +} + +static int cmd_Enable_sign_by_dc(SSL_CONF_CTX *cctx, const char *value) +{ + if (strcmp(value, "on") == 0) { + if (cctx->ctx != NULL) + SSL_CTX_enable_sign_by_dc(cctx->ctx); + if (cctx->ssl != NULL) + SSL_enable_sign_by_dc(cctx->ssl); + } else { + if (cctx->ctx != NULL) + SSL_CTX_disable_sign_by_dc(cctx->ctx); + if (cctx->ssl != NULL) + SSL_disable_sign_by_dc(cctx->ssl); + } + return 1; +} +#endif + +#ifndef OPENSSL_NO_SSL_TRACE +static void trace_cb(int write_p, int version, int content_type, + const void *buf, size_t msglen, SSL *ssl, void *arg) +{ + BIO *bio = NULL; + if (arg == NULL) { + bio = BIO_new_fp(stdout, BIO_NOCLOSE | BIO_FP_TEXT); + if (bio == NULL) + return; + + arg = bio; + } + + SSL_trace(write_p, version, content_type, buf, msglen, ssl, arg); + + BIO_free(bio); +} + +static int cmd_Trace(SSL_CONF_CTX *cctx, const char *value) +{ + if (strcmp(value, "on") == 0) { + if (cctx->ctx) + SSL_CTX_set_msg_callback(cctx->ctx, trace_cb); + + if (cctx->ssl) + SSL_set_msg_callback(cctx->ssl, trace_cb); + } + + return 1; +} +#endif + typedef struct { int (*cmd) (SSL_CONF_CTX *cctx, const char *value); const char *str_file; @@ -709,13 +923,12 @@ static const ssl_conf_cmd_tbl ssl_conf_cmds[] = { SSL_CONF_CMD_SWITCH("no_tls1_1", 0), SSL_CONF_CMD_SWITCH("no_tls1_2", 0), SSL_CONF_CMD_SWITCH("no_tls1_3", 0), +#ifndef OPENSSL_NO_NTLS + SSL_CONF_CMD_SWITCH("no_ntls", 0), +#endif SSL_CONF_CMD_SWITCH("bugs", 0), SSL_CONF_CMD_SWITCH("no_comp", 0), SSL_CONF_CMD_SWITCH("comp", 0), - SSL_CONF_CMD_SWITCH("no_tx_cert_comp", 0), - SSL_CONF_CMD_SWITCH("tx_cert_comp", 0), - SSL_CONF_CMD_SWITCH("no_rx_cert_comp", 0), - SSL_CONF_CMD_SWITCH("rx_cert_comp", 0), SSL_CONF_CMD_SWITCH("ecdh_single", SSL_CONF_FLAG_SERVER), SSL_CONF_CMD_SWITCH("no_ticket", 0), SSL_CONF_CMD_SWITCH("serverpref", SSL_CONF_FLAG_SERVER), @@ -724,16 +937,14 @@ static const ssl_conf_cmd_tbl ssl_conf_cmds[] = { SSL_CONF_CMD_SWITCH("legacy_server_connect", SSL_CONF_FLAG_CLIENT), SSL_CONF_CMD_SWITCH("no_renegotiation", 0), SSL_CONF_CMD_SWITCH("no_resumption_on_reneg", SSL_CONF_FLAG_SERVER), - SSL_CONF_CMD_SWITCH("no_legacy_server_connect", SSL_CONF_FLAG_CLIENT), + SSL_CONF_CMD_SWITCH("no_legacy_server_connect", SSL_CONF_FLAG_SERVER), SSL_CONF_CMD_SWITCH("allow_no_dhe_kex", 0), - SSL_CONF_CMD_SWITCH("prefer_no_dhe_kex", 0), SSL_CONF_CMD_SWITCH("prioritize_chacha", SSL_CONF_FLAG_SERVER), SSL_CONF_CMD_SWITCH("strict", 0), SSL_CONF_CMD_SWITCH("no_middlebox", 0), SSL_CONF_CMD_SWITCH("anti_replay", SSL_CONF_FLAG_SERVER), SSL_CONF_CMD_SWITCH("no_anti_replay", SSL_CONF_FLAG_SERVER), SSL_CONF_CMD_SWITCH("no_etm", 0), - SSL_CONF_CMD_SWITCH("no_ems", 0), SSL_CONF_CMD_STRING(SignatureAlgorithms, "sigalgs", 0), SSL_CONF_CMD_STRING(ClientSignatureAlgorithms, "client_sigalgs", 0), SSL_CONF_CMD_STRING(Curves, "curves", 0), @@ -785,6 +996,31 @@ static const ssl_conf_cmd_tbl ssl_conf_cmds[] = { SSL_CONF_TYPE_FILE), SSL_CONF_CMD_STRING(RecordPadding, "record_padding", 0), SSL_CONF_CMD_STRING(NumTickets, "num_tickets", SSL_CONF_FLAG_SERVER), +#ifndef OPENSSL_NO_NTLS + SSL_CONF_CMD_STRING(Enable_ntls, "enable_ntls", 0), + SSL_CONF_CMD_STRING(Enable_force_ntls, "enable_force_ntls", 0), + SSL_CONF_CMD(EncCertificate, "enc_cert", SSL_CONF_FLAG_CERTIFICATE, + SSL_CONF_TYPE_FILE), + SSL_CONF_CMD(EncPrivateKey, "enc_key", SSL_CONF_FLAG_CERTIFICATE, + SSL_CONF_TYPE_FILE), + SSL_CONF_CMD(SignCertificate, "sign_cert", SSL_CONF_FLAG_CERTIFICATE, + SSL_CONF_TYPE_FILE), + SSL_CONF_CMD(SignPrivateKey, "sign_key", SSL_CONF_FLAG_CERTIFICATE, + SSL_CONF_TYPE_FILE), +#endif +#ifndef OPENSSL_NO_SM2 + SSL_CONF_CMD_STRING(Enable_sm_tls13_strict, "Enable_sm_tls13_strict", 0), +#endif +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + SSL_CONF_CMD(DC, "dc", SSL_CONF_FLAG_CERTIFICATE, SSL_CONF_TYPE_FILE), + SSL_CONF_CMD(DCKey, "dc_key", SSL_CONF_FLAG_CERTIFICATE, + SSL_CONF_TYPE_FILE), + SSL_CONF_CMD_STRING(Enable_verify_peer_by_dc, "Enable_verify_peer_by_dc", 0), + SSL_CONF_CMD_STRING(Enable_sign_by_dc, "Enable_sign_by_dc", 0), +#endif +#ifndef OPENSSL_NO_SSL_TRACE + SSL_CONF_CMD_STRING(Trace, "Trace", 0), +#endif }; /* Supported switches: must match order of switches in ssl_conf_cmds */ @@ -794,13 +1030,12 @@ static const ssl_switch_tbl ssl_cmd_switches[] = { {SSL_OP_NO_TLSv1_1, 0}, /* no_tls1_1 */ {SSL_OP_NO_TLSv1_2, 0}, /* no_tls1_2 */ {SSL_OP_NO_TLSv1_3, 0}, /* no_tls1_3 */ +#ifndef OPENSSL_NO_NTLS + {SSL_OP_NO_NTLS, 0}, /* no_ntls */ +#endif {SSL_OP_ALL, 0}, /* bugs */ {SSL_OP_NO_COMPRESSION, 0}, /* no_comp */ {SSL_OP_NO_COMPRESSION, SSL_TFLAG_INV}, /* comp */ - {SSL_OP_NO_TX_CERTIFICATE_COMPRESSION, 0}, /* no_tx_cert_comp */ - {SSL_OP_NO_TX_CERTIFICATE_COMPRESSION, SSL_TFLAG_INV}, /* tx_cert_comp */ - {SSL_OP_NO_RX_CERTIFICATE_COMPRESSION, 0}, /* no_rx_cert_comp */ - {SSL_OP_NO_RX_CERTIFICATE_COMPRESSION, SSL_TFLAG_INV}, /* rx_cert_comp */ {SSL_OP_SINGLE_ECDH_USE, 0}, /* ecdh_single */ {SSL_OP_NO_TICKET, 0}, /* no_ticket */ {SSL_OP_CIPHER_SERVER_PREFERENCE, 0}, /* serverpref */ @@ -818,8 +1053,6 @@ static const ssl_switch_tbl ssl_cmd_switches[] = { {SSL_OP_LEGACY_SERVER_CONNECT, SSL_TFLAG_INV}, /* allow_no_dhe_kex */ {SSL_OP_ALLOW_NO_DHE_KEX, 0}, - /* prefer_no_dhe_kex */ - {SSL_OP_PREFER_NO_DHE_KEX, 0}, /* chacha reprioritization */ {SSL_OP_PRIORITIZE_CHACHA, 0}, {SSL_CERT_FLAG_TLS_STRICT, SSL_TFLAG_CERT}, /* strict */ @@ -831,8 +1064,6 @@ static const ssl_switch_tbl ssl_cmd_switches[] = { {SSL_OP_NO_ANTI_REPLAY, 0}, /* no Encrypt-then-Mac */ {SSL_OP_NO_ENCRYPT_THEN_MAC, 0}, - /* no Extended master secret */ - {SSL_OP_NO_EXTENDED_MASTER_SECRET, 0}, }; static int ssl_conf_cmd_skip_prefix(SSL_CONF_CTX *cctx, const char **pcmd) @@ -859,7 +1090,7 @@ static int ssl_conf_cmd_skip_prefix(SSL_CONF_CTX *cctx, const char **pcmd) } /* Determine if a command is allowed according to cctx flags */ -static int ssl_conf_cmd_allowed(SSL_CONF_CTX *cctx, const ssl_conf_cmd_tbl *t) +static int ssl_conf_cmd_allowed(SSL_CONF_CTX *cctx, const ssl_conf_cmd_tbl * t) { unsigned int tfl = t->flags; unsigned int cfl = cctx->flags; @@ -897,17 +1128,14 @@ static const ssl_conf_cmd_tbl *ssl_conf_cmd_lookup(SSL_CONF_CTX *cctx, return NULL; } -static int ctrl_switch_option(SSL_CONF_CTX *cctx, const ssl_conf_cmd_tbl *cmd) +static int ctrl_switch_option(SSL_CONF_CTX *cctx, const ssl_conf_cmd_tbl * cmd) { /* Find index of command in table */ size_t idx = cmd - ssl_conf_cmds; const ssl_switch_tbl *scmd; - /* Sanity check index */ - if (idx >= OSSL_NELEM(ssl_cmd_switches)) { - ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + if (idx >= OSSL_NELEM(ssl_cmd_switches)) return 0; - } /* Obtain switches entry with same index */ scmd = ssl_cmd_switches + idx; ssl_set_option(cctx, scmd->name_flags, scmd->option_value, 1); @@ -923,33 +1151,28 @@ int SSL_CONF_cmd(SSL_CONF_CTX *cctx, const char *cmd, const char *value) } if (!ssl_conf_cmd_skip_prefix(cctx, &cmd)) - goto unknown_cmd; + return -2; runcmd = ssl_conf_cmd_lookup(cctx, cmd); if (runcmd) { - int rv = -3; - + int rv; if (runcmd->value_type == SSL_CONF_TYPE_NONE) { return ctrl_switch_option(cctx, runcmd); } if (value == NULL) - goto bad_value; + return -3; rv = runcmd->cmd(cctx, value); if (rv > 0) return 2; - if (rv != -2) - rv = 0; - - bad_value: + if (rv == -2) + return -2; if (cctx->flags & SSL_CONF_FLAG_SHOW_ERRORS) ERR_raise_data(ERR_LIB_SSL, SSL_R_BAD_VALUE, - "cmd=%s, value=%s", cmd, - value != NULL ? value : ""); - return rv; + "cmd=%s, value=%s", cmd, value); + return 0; } - unknown_cmd: if (cctx->flags & SSL_CONF_FLAG_SHOW_ERRORS) ERR_raise_data(ERR_LIB_SSL, SSL_R_UNKNOWN_CMD_NAME, "cmd=%s", cmd); @@ -1013,16 +1236,11 @@ int SSL_CONF_CTX_finish(SSL_CONF_CTX *cctx) /* See if any certificates are missing private keys */ size_t i; CERT *c = NULL; - - if (cctx->ctx != NULL) { + if (cctx->ctx) c = cctx->ctx->cert; - } else if (cctx->ssl != NULL) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(cctx->ssl); - - if (sc != NULL) - c = sc->cert; - } - if (c != NULL && cctx->flags & SSL_CONF_FLAG_REQUIRE_PRIVATE) { + else if (cctx->ssl) + c = cctx->ssl->cert; + if (c && cctx->flags & SSL_CONF_FLAG_REQUIRE_PRIVATE) { for (i = 0; i < SSL_PKEY_NUM; i++) { const char *p = cctx->cert_filename[i]; /* @@ -1091,16 +1309,12 @@ void SSL_CONF_CTX_set_ssl(SSL_CONF_CTX *cctx, SSL *ssl) { cctx->ssl = ssl; cctx->ctx = NULL; - if (ssl != NULL) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return; - cctx->poptions = &sc->options; - cctx->min_version = &sc->min_proto_version; - cctx->max_version = &sc->max_proto_version; - cctx->pcert_flags = &sc->cert->cert_flags; - cctx->pvfy_flags = &sc->verify_mode; + if (ssl) { + cctx->poptions = &ssl->options; + cctx->min_version = &ssl->min_proto_version; + cctx->max_version = &ssl->max_proto_version; + cctx->pcert_flags = &ssl->cert->cert_flags; + cctx->pvfy_flags = &ssl->verify_mode; } else { cctx->poptions = NULL; cctx->min_version = NULL; diff --git a/openssl/src/ssl/ssl_dc.c b/openssl/src/ssl/ssl_dc.c new file mode 100644 index 000000000..d65bcc739 --- /dev/null +++ b/openssl/src/ssl/ssl_dc.c @@ -0,0 +1,514 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#include +#include +#include +#include +#include +#include +#include "ssl_local.h" + +#define DELEGATED_CREDENTIAL_CLIENT_LABEL "TLS, client delegated credentials" +#define DELEGATED_CREDENTIAL_SERVER_LABEL "TLS, server delegated credentials" +#define DELEGATED_CREDENTIAL_SIGN_START_SIZE 64 + +#define W16(buf, value) {put_value(buf, value, 2); buf += 2;} +#define W24(buf, value) {put_value(buf, value, 3); buf += 3;} +#define W32(buf, value) {put_value(buf, value, 4); buf += 4;} + +static void put_value(unsigned char *buf, size_t value, size_t len) +{ + for (buf += len - 1; len > 0; len--) { + *buf = (unsigned char)(value & 0xff); + buf--; + value >>= 8; + } +} + +void SSL_CTX_enable_verify_peer_by_dc(SSL_CTX *ctx) +{ + ctx->enable_verify_peer_by_dc = 1; +} + +void SSL_CTX_disable_verify_peer_by_dc(SSL_CTX *ctx) +{ + ctx->enable_verify_peer_by_dc = 0; +} + +void SSL_enable_verify_peer_by_dc(SSL *s) +{ + s->enable_verify_peer_by_dc = 1; +} + +void SSL_disable_verify_peer_by_dc(SSL *s) +{ + s->enable_verify_peer_by_dc = 0; +} + +void SSL_CTX_enable_sign_by_dc(SSL_CTX *ctx) +{ + ctx->enable_sign_by_dc = 1; +} + +void SSL_CTX_disable_sign_by_dc(SSL_CTX *ctx) +{ + ctx->enable_sign_by_dc = 0; +} + +void SSL_enable_sign_by_dc(SSL *s) +{ + s->enable_sign_by_dc = 1; +} + +void SSL_disable_sign_by_dc(SSL *s) +{ + s->enable_sign_by_dc = 0; +} + +int SSL_get_delegated_credential_tag(SSL *s) +{ + return s->delegated_credential_tag; +} + +static int ssl_dc_tbs_data(unsigned char *parent_cert_raw, + long parent_cert_len, + DELEGATED_CREDENTIAL *dc, int is_server, + unsigned char **tbs, unsigned int *tbs_len) +{ + unsigned int sign_data_len; + unsigned int dc_cred_and_alg_len = 0; + unsigned char *index; + + if (dc == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + return 0; + } + + dc_cred_and_alg_len = DC_get_raw_byte_len(dc) - 2 - DC_get_dc_signature_len(dc); + + /* length of dc client label is equal to server label */ + sign_data_len = DELEGATED_CREDENTIAL_SIGN_START_SIZE + + sizeof(DELEGATED_CREDENTIAL_SERVER_LABEL) + + parent_cert_len + + dc_cred_and_alg_len; + + *tbs = OPENSSL_malloc(sign_data_len); + index = *tbs; + if (*tbs == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + return 0; + } + /* + * First part is a string that consists of octet 32 (0x20) repeated 64 times. + */ + memset(index, 32, DELEGATED_CREDENTIAL_SIGN_START_SIZE); + index += DELEGATED_CREDENTIAL_SIGN_START_SIZE; + + /* + * Second part is a context string "TLS, server delegated credentials" for + * servers and "TLS, client delegated credentials" for clients. + * Third part is a single 0 byte, which serves as the separator. + * '0' exists in DELEGATED_CREDENTIAL_SERVER_LABEL default terminator + */ + if (is_server) { + strcpy((char *)index, DELEGATED_CREDENTIAL_SERVER_LABEL); + index += sizeof(DELEGATED_CREDENTIAL_SERVER_LABEL); + } else { + strcpy((char *)index, DELEGATED_CREDENTIAL_CLIENT_LABEL); + index += sizeof(DELEGATED_CREDENTIAL_CLIENT_LABEL); + } + + /* + * Fourth part is the DER-encoded X.509 end-entity certificate used to sign the + * DelegatedCredential. + */ + memcpy(index, parent_cert_raw, parent_cert_len); + index += parent_cert_len; + + /* + * Fifth part is Credential in DelegatedCredential + * Sixth part is DelegatedCredential.algorithm. + * We can make a one-time copy from dc raw byte + */ + memcpy(index, DC_get0_raw_byte(dc), dc_cred_and_alg_len); + index += dc_cred_and_alg_len; + + if ((index - *tbs) != sign_data_len) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + return 0; + } + + *tbs_len = sign_data_len; + return 1; +} + +int SSL_verify_delegated_credential_signature(X509 *parent_cert, + DELEGATED_CREDENTIAL *dc, + int is_server) +{ + unsigned char *tbs = NULL; + unsigned int tbs_len; + int ret = 0; + EVP_MD_CTX *mctx = NULL; + EVP_PKEY_CTX *pctx = NULL; + int dc_sign_algo = 0; + unsigned char *parent_cert_raw = NULL; + unsigned char *parent_cert_raw_index = NULL; + long parent_cert_len; + const EVP_MD *md = NULL; + EVP_PKEY *pkey = NULL; + const SIGALG_LOOKUP *lu = NULL; + + if (parent_cert == NULL || dc == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + goto err; + } + + pkey = X509_get0_pubkey(parent_cert); + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + goto err; + } + + dc_sign_algo = DC_get_signature_sign_algorithm(dc); + + lu = ssl_sigalg_lookup(dc_sign_algo); + if (lu == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_SIGNATURE_ALGORITHMS_ERROR); + goto err; + } + + md = EVP_get_digestbynid(lu->hash); + + parent_cert_len = i2d_X509_AUX(parent_cert, NULL); + if (parent_cert_len <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + goto err; + } + if ((parent_cert_raw = OPENSSL_malloc(parent_cert_len)) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto err; + } + + parent_cert_raw_index = parent_cert_raw; + parent_cert_len = i2d_X509_AUX(parent_cert, &parent_cert_raw_index); + + if (!ssl_dc_tbs_data(parent_cert_raw, parent_cert_len, + dc, is_server, &tbs, &tbs_len)) { + goto err; + } + + mctx = EVP_MD_CTX_new(); + if (mctx == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (EVP_DigestVerifyInit(mctx, &pctx, md, NULL, pkey) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); + goto err; + } + + if (lu->sig == EVP_PKEY_RSA_PSS) { + if (EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_PSS_PADDING) <= 0 + || EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, + RSA_PSS_SALTLEN_DIGEST) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); + goto err; + } + } + + ret = EVP_DigestVerify(mctx, DC_get0_dc_signature(dc), + DC_get_dc_signature_len(dc), + (const unsigned char *)tbs, tbs_len); + +err: + EVP_MD_CTX_free(mctx); + OPENSSL_free(tbs); + OPENSSL_free(parent_cert_raw); + return ret; +} + +int DC_sign(DELEGATED_CREDENTIAL *dc, EVP_PKEY *dc_pkey, + unsigned int valid_time, int expect_verify_hash, + X509 *ee_cert, EVP_PKEY *ee_pkey, const EVP_MD *md, int is_server) +{ + int ret = 0; + int day, sec; + unsigned char *dc_pkey_raw_index = NULL; + uint32_t max_valid_time = 7 * 24 * 3600; + unsigned char *dc_buf, *index; + int dc_raw_len = 0; + unsigned char *tbs = NULL; + unsigned int tbs_len; + int res = 0; + EVP_MD_CTX *mctx = NULL; + EVP_PKEY_CTX *pctx = NULL; + ASN1_TIME *ee_cert_time = NULL, *curr_time = NULL; + uint32_t dc_pkey_raw_len; + unsigned char *dc_pkey_raw = NULL; + size_t dc_sign_len, dc_sign_result_len; + unsigned char *parent_cert_raw = NULL, *parent_cert_raw_index = NULL; + int ee_cert_len; + const SIGALG_LOOKUP *dc_verify_lu = NULL; + const SIGALG_LOOKUP *sig_lu = NULL; + + if (dc == NULL || dc_pkey == NULL + || ee_cert == NULL || ee_pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!DC_check_parent_cert_valid(ee_cert)) + goto end; + + dc_pkey_raw_len = i2d_PUBKEY(dc_pkey, NULL); + if (dc_pkey_raw_len <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + goto end; + } + + if ((dc_pkey_raw = OPENSSL_malloc(dc_pkey_raw_len)) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto end; + } + + dc_pkey_raw_index = dc_pkey_raw; + dc_pkey_raw_len = i2d_PUBKEY(dc_pkey, &dc_pkey_raw_index); + + dc_verify_lu = ssl_sigalg_lookup_by_pkey_and_hash(dc_pkey, + expect_verify_hash, 1); + if (dc_verify_lu == NULL) + goto end; + + sig_lu = ssl_sigalg_lookup_by_pkey_and_hash(ee_pkey, EVP_MD_type(md), 0); + if (sig_lu == NULL) + goto end; + + if (valid_time > max_valid_time) { + ERR_raise(ERR_LIB_SSL, SSL_R_DC_VALID_TIME_TOO_LARGE); + goto end; + } + + ee_cert_time = ASN1_STRING_dup(X509_get0_notBefore(ee_cert)); + if (ee_cert_time == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + goto end; + } + + curr_time = X509_time_adj(NULL, 0, NULL); + if (curr_time == NULL) + goto end; + + if (!ASN1_TIME_diff(&day, &sec, ee_cert_time, curr_time)) + goto end; + + if (day < 0 || sec < 0 ) + goto end; + + valid_time += day * 24 * 3600 + sec; + + dc_sign_len = EVP_PKEY_size(ee_pkey); + + dc_raw_len = sizeof(uint32_t) + sizeof(uint16_t) + 3 + dc_pkey_raw_len + + sizeof(uint16_t) + 2 + dc_sign_len; + + dc_buf = OPENSSL_malloc(dc_raw_len); + if (!dc_buf) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto end; + } + + index = dc_buf; + + /* uint32 valid_time */ + W32(index, valid_time); + + /* SignatureScheme expected_cert_verify_algorithm */ + W16(index, dc_verify_lu->sigalg); + + /* opaque ASN1_subjectPublicKeyInfo<1..2^24-1> */ + W24(index, dc_pkey_raw_len); + memcpy(index, dc_pkey_raw, dc_pkey_raw_len); + index += dc_pkey_raw_len; + + /* SignatureScheme algorithm */ + W16(index, sig_lu->sigalg); + + /* + * Actualy dc_sign_len is not the real sign result len, but function + * ssl_dc_tbs_data o nly need credential and sign + * algorithm. So we can get right result even if using a wrong + * dc_sign_len + */ + W16(index, dc_sign_len); + + DC_set_dc_signature_len(dc, dc_sign_len); + DC_set0_raw_byte(dc, dc_buf, dc_raw_len); + + ee_cert_len = i2d_X509_AUX(ee_cert, NULL); + if (ee_cert_len <= 0) + goto end; + + if ((parent_cert_raw = OPENSSL_malloc(ee_cert_len)) == NULL) + goto end; + + parent_cert_raw_index = parent_cert_raw; + ee_cert_len = i2d_X509_AUX(ee_cert, &parent_cert_raw_index); + + res = ssl_dc_tbs_data(parent_cert_raw, ee_cert_len, + dc, is_server, &tbs, &tbs_len); + if (res <= 0) + goto end; + + mctx = EVP_MD_CTX_new(); + if (mctx == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto end; + } + + if (!EVP_DigestSignInit_ex(mctx, &pctx, EVP_MD_name(md), NULL, + NULL, ee_pkey, NULL)) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + goto end; + } + + if (sig_lu->sig == EVP_PKEY_RSA_PSS) { + if (EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_PSS_PADDING) <= 0 + || EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, + RSA_PSS_SALTLEN_DIGEST) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); + goto end; + } + } + + dc_sign_result_len = dc_sign_len; + res = EVP_DigestSign(mctx, index, &dc_sign_result_len, + (const unsigned char *)tbs, tbs_len); + if (res <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_EVP_LIB); + goto end; + } + + index -= 2; + W16(index, dc_sign_result_len); + DC_set_dc_signature_len(dc, dc_sign_result_len); + + dc_raw_len = dc_raw_len - dc_sign_len + dc_sign_result_len; + DC_set0_raw_byte(dc, DC_get0_raw_byte(dc), dc_raw_len); + + ret = 1; +end: + OPENSSL_free(tbs); + EVP_MD_CTX_free(mctx); + OPENSSL_free(dc_pkey_raw); + OPENSSL_free(parent_cert_raw); + ASN1_STRING_clear_free(ee_cert_time); + ASN1_STRING_clear_free(curr_time); + + return ret; +} + +int DC_print(BIO *bp, DELEGATED_CREDENTIAL *dc) +{ + int ret = 0; + int indent = 0; + unsigned int i, siglen; + unsigned int sigalg; + const char *name; + unsigned char *sig; + const SIGALG_LOOKUP *lu; + + if (BIO_printf(bp, "DelegatedCredential:\n") <= 0) + goto end; + + indent += 4; + if (BIO_printf(bp, "%*sCredential:\n", indent, "") <= 0) + goto end; + + indent += 4; + if (BIO_printf(bp, "%*svalid_time: %lu\n", + indent, "", DC_get_valid_time(dc)) <= 0) + goto end; + + sigalg = DC_get_expected_cert_verify_algorithm(dc); + + lu = ssl_sigalg_lookup(sigalg); + if (lu == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); + goto end; + } + + name = lu->name; + if (BIO_printf(bp, "%*sexpected_cert_verify_algorithm: %s (0x%04x)\n", + indent, "", name ? name : "NULL", sigalg) <= 0) + goto end; + + if (BIO_printf(bp, "%*sSubject Public Key Info:\n", indent, "") <= 0) + goto end; + + indent += 4; + if (BIO_printf(bp, "%*sPublic Key Algorithm: ", indent, "") <= 0) + goto end; + + if (i2a_ASN1_OBJECT(bp, OBJ_nid2obj( + EVP_PKEY_id(DC_get0_publickey(dc)))) <= 0) + goto end; + + if (BIO_puts(bp, "\n") <= 0) + goto end; + + indent += 4; + if (EVP_PKEY_print_public(bp, DC_get0_publickey(dc), indent, NULL) <= 0) + goto end; + + indent = 4; + sigalg = DC_get_signature_sign_algorithm(dc); + + lu = ssl_sigalg_lookup(sigalg); + if (lu == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); + goto end; + } + + name = lu->name; + if (BIO_printf(bp, "%*sSignature Algorithm: %s (0x%04x)", + indent, "", name ? name : "unknown", sigalg) <= 0) + goto end; + + if (BIO_printf(bp, "\n%*sSignature:", indent, "") <= 0) + goto end; + + indent += 4; + + sig = DC_get0_dc_signature(dc); + siglen = DC_get_dc_signature_len(dc); + + for (i = 0; i < siglen; i++) { + if ((i % 18) == 0) { + if (BIO_write(bp, "\n", 1) <= 0) + goto end; + if (BIO_indent(bp, indent, indent) <= 0) + goto end; + } + + if (BIO_printf(bp, "%02x%s", sig[i], + ((i + 1) == siglen) ? "" : ":") <= 0) + goto end; + } + + if (BIO_write(bp, "\n", 1) <= 0) + goto end; + + ret = 1; + +end: + return ret; +} diff --git a/openssl/src/ssl/ssl_err.c b/openssl/src/ssl/ssl_err.c index f5fb4107f..0a289c9ea 100644 --- a/openssl/src/ssl/ssl_err.c +++ b/openssl/src/ssl/ssl_err.c @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -23,12 +23,13 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "attempt to reuse session in different context"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE), "at least (D)TLS 1.2 needed in Suite B mode"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_BAD_CERTIFICATE), "bad certificate"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_BAD_CERTIFICATE_SIGNATURE_TYPE), + "bad certificate signature type"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_BAD_CERTIFICATE_USAGE), + "bad certificate usage"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_BAD_CHANGE_CIPHER_SPEC), "bad change cipher spec"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_BAD_CIPHER), "bad cipher"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_BAD_COMPRESSION_ALGORITHM), - "bad compression algorithm"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_BAD_DATA), "bad data"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_BAD_DATA_RETURNED_BY_CALLBACK), "bad data returned by callback"}, @@ -104,7 +105,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "compression library error"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_CONNECTION_TYPE_NOT_SET), "connection type not set"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_CONN_USE_ONLY), "conn use only"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_CONTEXT_NOT_DANE_ENABLED), "context not dane enabled"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_COOKIE_GEN_CALLBACK_FAILURE), @@ -139,6 +139,8 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "data between ccs and finished"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_DATA_LENGTH_TOO_LONG), "data length too long"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_DC_VALID_TIME_TOO_LARGE), + "dc valid time too large"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_DECRYPTION_FAILED), "decryption failed"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC), "decryption failed or bad record mac"}, @@ -155,9 +157,8 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "ecc cert not for signing"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_ECDH_REQUIRED_FOR_SUITEB_MODE), "ecdh required for suiteb mode"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_EE_CERT_NOT_FOUND), "ee cert not found"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_EE_KEY_TOO_SMALL), "ee key too small"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_EMPTY_RAW_PUBLIC_KEY), - "empty raw public key"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST), "empty srtp protection profile list"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_ENCRYPTED_LENGTH_TOO_LONG), @@ -176,16 +177,14 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "extra data in message"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_EXT_LENGTH_MISMATCH), "ext length mismatch"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_FAILED_TO_GET_PARAMETER), - "failed to get parameter"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_FAILED_TO_INIT_ASYNC), "failed to init async"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_FEATURE_NEGOTIATION_NOT_COMPLETE), - "feature negotiation not complete"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_FEATURE_NOT_RENEGOTIABLE), - "feature not renegotiable"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_FAILED_TO_VERIFY_DC_SIGNATURE), + "failed to verify dc signature"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_FRAGMENTED_CLIENT_HELLO), "fragmented client hello"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_GET_SIG_AND_HASH_ERR), + "get sig and hash err"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_GOT_A_FIN_BEFORE_A_CCS), "got a fin before a ccs"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_HTTPS_PROXY_REQUEST), @@ -226,9 +225,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "invalid max early data"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_INVALID_NULL_CMD_NAME), "invalid null cmd name"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_INVALID_RAW_PUBLIC_KEY), - "invalid raw public key"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_INVALID_RECORD), "invalid record"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_INVALID_SEQUENCE_NUMBER), "invalid sequence number"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_INVALID_SERVERINFO_DATA), @@ -240,24 +236,24 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "invalid status response"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_INVALID_TICKET_KEYS_LENGTH), "invalid ticket keys length"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_LEGACY_SIGALG_DISALLOWED_OR_UNSUPPORTED), - "legacy sigalg disallowed or unsupported"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_LENGTH_MISMATCH), "length mismatch"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_LENGTH_TOO_LONG), "length too long"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_LENGTH_TOO_SHORT), "length too short"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_LIBRARY_BUG), "library bug"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_LIBRARY_HAS_NO_CIPHERS), "library has no ciphers"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED), - "maximum encrypted pkts reached"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_DSA_SIGNING_CERT), "missing dsa signing cert"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_ECDSA_SIGNING_CERT), "missing ecdsa signing cert"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_ENC_CERTIFICATE), + "missing encryption certificate"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_FATAL), "missing fatal"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_PARAMETERS), "missing parameters"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_PSK_KEX_MODES_EXTENSION), "missing psk kex modes extension"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION), + "missing quic transport parameters extension"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_RSA_CERTIFICATE), "missing rsa certificate"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_MISSING_RSA_ENCRYPTING_CERT), @@ -302,8 +298,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "no compression specified"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_COOKIE_CALLBACK_SET), "no cookie callback set"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER), - "Peer haven't sent GOST certificate, required for selected ciphersuite"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_METHOD_SPECIFIED), "no method specified"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_PEM_EXTENSIONS), "no pem extensions"}, @@ -318,14 +312,11 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SHARED_SIGNATURE_ALGORITHMS), "no shared signature algorithms"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SRTP_PROFILES), "no srtp profiles"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_STREAM), "no stream"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SUITABLE_DIGEST_ALGORITHM), "no suitable digest algorithm"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SUITABLE_GROUPS), "no suitable groups"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SUITABLE_KEY_SHARE), "no suitable key share"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SUITABLE_RECORD_LAYER), - "no suitable record layer"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM), "no suitable signature algorithm"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_NO_VALID_SCTS), "no valid scts"}, @@ -351,8 +342,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "pem name bad prefix"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_PEM_NAME_TOO_SHORT), "pem name too short"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_PIPELINE_FAILURE), "pipeline failure"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_POLL_REQUEST_NOT_SUPPORTED), - "poll request not supported"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR), "post handshake auth encoding err"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_PRIVATE_KEY_MISMATCH), @@ -363,23 +352,12 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "psk identity not found"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_PSK_NO_CLIENT_CB), "psk no client cb"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_PSK_NO_SERVER_CB), "psk no server cb"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_QUIC_HANDSHAKE_LAYER_ERROR), - "quic handshake layer error"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_QUIC_NETWORK_ERROR), "quic network error"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_QUIC_PROTOCOL_ERROR), - "quic protocol error"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_READ_BIO_NOT_SET), "read bio not set"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_READ_TIMEOUT_EXPIRED), "read timeout expired"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_RECORDS_NOT_RELEASED), - "records not released"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_RECORD_LAYER_FAILURE), - "record layer failure"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_RECORD_LENGTH_MISMATCH), "record length mismatch"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_RECORD_TOO_SMALL), "record too small"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_REMOTE_PEER_ADDRESS_NOT_SET), - "remote peer address not set"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_RENEGOTIATE_EXT_TOO_LONG), "renegotiate ext too long"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_RENEGOTIATION_ENCODING_ERR), @@ -396,8 +374,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "scsv received when renegotiating"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SCT_VERIFICATION_FAILED), "sct verification failed"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SEQUENCE_CTR_WRAPPED), - "sequence ctr wrapped"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SERVERHELLO_TLSEXT), "serverhello tlsext"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED), "session id context uninitialized"}, @@ -423,27 +399,27 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSL3_SESSION_ID_TOO_LONG), "ssl3 session id too long"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_BAD_CERTIFICATE), - "ssl/tls alert bad certificate"}, + "sslv3 alert bad certificate"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_BAD_RECORD_MAC), - "ssl/tls alert bad record mac"}, + "sslv3 alert bad record mac"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED), - "ssl/tls alert certificate expired"}, + "sslv3 alert certificate expired"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED), - "ssl/tls alert certificate revoked"}, + "sslv3 alert certificate revoked"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN), - "ssl/tls alert certificate unknown"}, + "sslv3 alert certificate unknown"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE), - "ssl/tls alert decompression failure"}, + "sslv3 alert decompression failure"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE), - "ssl/tls alert handshake failure"}, + "sslv3 alert handshake failure"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER), - "ssl/tls alert illegal parameter"}, + "sslv3 alert illegal parameter"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_NO_CERTIFICATE), - "ssl/tls alert no certificate"}, + "sslv3 alert no certificate"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE), - "ssl/tls alert unexpected message"}, + "sslv3 alert unexpected message"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE), - "ssl/tls alert unsupported certificate"}, + "sslv3 alert unsupported certificate"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSL_COMMAND_SECTION_EMPTY), "ssl command section empty"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSL_COMMAND_SECTION_NOT_FOUND), @@ -471,13 +447,9 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "ssl session id too long"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_SSL_SESSION_VERSION_MISMATCH), "ssl session version mismatch"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_STATUS_CALLBACK_ERROR), + "status callback error"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_STILL_IN_INIT), "still in init"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_STREAM_COUNT_LIMITED), - "stream count limited"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_STREAM_FINISHED), "stream finished"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_STREAM_RECV_ONLY), "stream recv only"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_STREAM_RESET), "stream reset"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_STREAM_SEND_ONLY), "stream send only"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED), "tlsv13 alert certificate required"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV13_ALERT_MISSING_EXTENSION), @@ -498,8 +470,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "tlsv1 alert insufficient security"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV1_ALERT_INTERNAL_ERROR), "tlsv1 alert internal error"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL), - "tlsv1 alert no application protocol"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV1_ALERT_NO_RENEGOTIATION), "tlsv1 alert no renegotiation"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV1_ALERT_PROTOCOL_VERSION), @@ -508,8 +478,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "tlsv1 alert record overflow"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV1_ALERT_UNKNOWN_CA), "tlsv1 alert unknown ca"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY), - "tlsv1 alert unknown psk identity"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV1_ALERT_USER_CANCELLED), "tlsv1 alert user cancelled"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE), @@ -540,6 +508,8 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "unable to load ssl3 md5 routines"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES), "unable to load ssl3 sha1 routines"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNABLE_TO_LOOKUP_CERT), + "unable to lookup cert"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNEXPECTED_CCS_MESSAGE), "unexpected ccs message"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNEXPECTED_END_OF_EARLY_DATA), @@ -561,8 +531,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNKNOWN_DIGEST), "unknown digest"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNKNOWN_KEY_EXCHANGE_TYPE), "unknown key exchange type"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNKNOWN_MANDATORY_PARAMETER), - "unknown mandatory parameter"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNKNOWN_PKEY_TYPE), "unknown pkey type"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNKNOWN_PROTOCOL), "unknown protocol"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNKNOWN_SSL_VERSION), @@ -574,12 +542,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "unsolicited extension"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM), "unsupported compression algorithm"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNSUPPORTED_CONFIG_VALUE), - "unsupported config value"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS), - "unsupported config value class"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNSUPPORTED_CONFIG_VALUE_OP), - "unsupported config value op"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNSUPPORTED_ELLIPTIC_CURVE), "unsupported elliptic curve"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNSUPPORTED_PROTOCOL), @@ -588,8 +550,6 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { "unsupported ssl version"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNSUPPORTED_STATUS_TYPE), "unsupported status type"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_UNSUPPORTED_WRITE_FLAG), - "unsupported write flag"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_USE_SRTP_NOT_NEGOTIATED), "use srtp not negotiated"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_VERSION_TOO_HIGH), "version too high"}, @@ -599,7 +559,10 @@ static const ERR_STRING_DATA SSL_str_reasons[] = { {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_WRONG_CIPHER_RETURNED), "wrong cipher returned"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_WRONG_CURVE), "wrong curve"}, - {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_WRONG_RPK_TYPE), "wrong rpk type"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_WRONG_ENCRYPTION_LEVEL_RECEIVED), + "wrong encryption level received"}, + {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_WRONG_PUBLIC_KEY_TYPE), + "wrong public key type"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_WRONG_SIGNATURE_LENGTH), "wrong signature length"}, {ERR_PACK(ERR_LIB_SSL, 0, SSL_R_WRONG_SIGNATURE_SIZE), diff --git a/openssl/src/ssl/ssl_init.c b/openssl/src/ssl/ssl_init.c index 70e567b72..e9d551bf5 100644 --- a/openssl/src/ssl/ssl_init.c +++ b/openssl/src/ssl/ssl_init.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -35,7 +35,7 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_ssl_base) SSL_COMP_get_compression_methods(); #endif ssl_sort_cipher_list(); - OSSL_TRACE(INIT, "ossl_init_ssl_base: SSL_add_ssl_module()\n"); + OSSL_TRACE(INIT,"ossl_init_ssl_base: SSL_add_ssl_module()\n"); /* * We ignore an error return here. Not much we can do - but not that bad * either. We can still safely continue. @@ -46,7 +46,7 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_ssl_base) } static CRYPTO_ONCE ssl_strings = CRYPTO_ONCE_STATIC_INIT; - +static int ssl_strings_inited = 0; DEFINE_RUN_ONCE_STATIC(ossl_init_load_ssl_strings) { /* @@ -56,6 +56,7 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_load_ssl_strings) #if !defined(OPENSSL_NO_ERR) && !defined(OPENSSL_NO_AUTOERRINIT) OSSL_TRACE(INIT, "ossl_init_load_ssl_strings: ossl_err_load_SSL_strings()\n"); ossl_err_load_SSL_strings(); + ssl_strings_inited = 1; #endif return 1; } @@ -81,6 +82,17 @@ static void ssl_library_stop(void) ssl_comp_free_compression_methods_int(); #endif } + + if (ssl_strings_inited) { + OSSL_TRACE(INIT, "ssl_library_stop: err_free_strings_int()\n"); + /* + * If both crypto and ssl error strings are inited we will end up + * calling err_free_strings_int() twice - but that's ok. The second + * time will be a no-op. It's easier to do that than to try and track + * between the two libraries whether they have both been inited. + */ + err_free_strings_int(); + } } /* @@ -88,7 +100,7 @@ static void ssl_library_stop(void) * called prior to any threads making calls to any OpenSSL functions, * i.e. passing a non-null settings value is assumed to be single-threaded. */ -int OPENSSL_init_ssl(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings) +int OPENSSL_init_ssl(uint64_t opts, const OPENSSL_INIT_SETTINGS * settings) { static int stoperrset = 0; diff --git a/openssl/src/ssl/ssl_lib.c b/openssl/src/ssl/ssl_lib.c index 5ec6ac4b6..a442bb8ef 100644 --- a/openssl/src/ssl/ssl_lib.c +++ b/openssl/src/ssl/ssl_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -21,28 +21,45 @@ #include #include #include -#include #include "internal/cryptlib.h" -#include "internal/nelem.h" #include "internal/refcount.h" #include "internal/ktls.h" -#include "quic/quic_local.h" -static int ssl_undefined_function_3(SSL_CONNECTION *sc, unsigned char *r, +#include "crypto/x509.h" +#include "crypto/x509/x509_local.h" + +#ifndef OPENSSL_NO_CERT_COMPRESSION +static CERT_COMP *CERT_COMP_copy(const CERT_COMP *p); +static void CERT_COMP_free(CERT_COMP *p); +#endif + +static int ssl_undefined_function_1(SSL *ssl, SSL3_RECORD *r, size_t s, int t, + SSL_MAC_BUF *mac, size_t macsize) +{ + return ssl_undefined_function(ssl); +} + +static int ssl_undefined_function_2(SSL *ssl, SSL3_RECORD *r, unsigned char *s, + int t) +{ + return ssl_undefined_function(ssl); +} + +static int ssl_undefined_function_3(SSL *ssl, unsigned char *r, unsigned char *s, size_t t, size_t *u) { - return ssl_undefined_function(SSL_CONNECTION_GET_SSL(sc)); + return ssl_undefined_function(ssl); } -static int ssl_undefined_function_4(SSL_CONNECTION *sc, int r) +static int ssl_undefined_function_4(SSL *ssl, int r) { - return ssl_undefined_function(SSL_CONNECTION_GET_SSL(sc)); + return ssl_undefined_function(ssl); } -static size_t ssl_undefined_function_5(SSL_CONNECTION *sc, const char *r, - size_t s, unsigned char *t) +static size_t ssl_undefined_function_5(SSL *ssl, const char *r, size_t s, + unsigned char *t) { - return ssl_undefined_function(SSL_CONNECTION_GET_SSL(sc)); + return ssl_undefined_function(ssl); } static int ssl_undefined_function_6(int r) @@ -50,20 +67,17 @@ static int ssl_undefined_function_6(int r) return ssl_undefined_function(NULL); } -static int ssl_undefined_function_7(SSL_CONNECTION *sc, unsigned char *r, - size_t s, const char *t, size_t u, +static int ssl_undefined_function_7(SSL *ssl, unsigned char *r, size_t s, + const char *t, size_t u, const unsigned char *v, size_t w, int x) { - return ssl_undefined_function(SSL_CONNECTION_GET_SSL(sc)); -} - -static int ssl_undefined_function_8(SSL_CONNECTION *sc) -{ - return ssl_undefined_function(SSL_CONNECTION_GET_SSL(sc)); + return ssl_undefined_function(ssl); } -const SSL3_ENC_METHOD ssl3_undef_enc_method = { - ssl_undefined_function_8, +SSL3_ENC_METHOD ssl3_undef_enc_method = { + ssl_undefined_function_1, + ssl_undefined_function_2, + ssl_undefined_function, ssl_undefined_function_3, ssl_undefined_function_4, ssl_undefined_function_5, @@ -120,6 +134,7 @@ static int dane_ctx_enable(struct dane_ctx_st *dctx) if (mdord == NULL || mdevp == NULL) { OPENSSL_free(mdord); OPENSSL_free(mdevp); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } @@ -165,7 +180,7 @@ static void dane_final(SSL_DANE *dane) sk_danetls_record_pop_free(dane->trecs, tlsa_free); dane->trecs = NULL; - OSSL_STACK_OF_X509_free(dane->certs); + sk_X509_pop_free(dane->certs, X509_free); dane->certs = NULL; X509_free(dane->mcert); @@ -178,7 +193,7 @@ static void dane_final(SSL_DANE *dane) /* * dane_copy - Copy dane configuration, sans verification state. */ -static int ssl_dane_dup(SSL_CONNECTION *to, SSL_CONNECTION *from) +static int ssl_dane_dup(SSL *to, SSL *from) { int num; int i; @@ -189,19 +204,19 @@ static int ssl_dane_dup(SSL_CONNECTION *to, SSL_CONNECTION *from) num = sk_danetls_record_num(from->dane.trecs); dane_final(&to->dane); to->dane.flags = from->dane.flags; - to->dane.dctx = &SSL_CONNECTION_GET_CTX(to)->dane; + to->dane.dctx = &to->ctx->dane; to->dane.trecs = sk_danetls_record_new_reserve(NULL, num); if (to->dane.trecs == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } for (i = 0; i < num; ++i) { danetls_record *t = sk_danetls_record_value(from->dane.trecs, i); - if (SSL_dane_tlsa_add(SSL_CONNECTION_GET_SSL(to), t->usage, - t->selector, t->mtype, t->data, t->dlen) <= 0) + if (SSL_dane_tlsa_add(to, t->usage, t->selector, t->mtype, + t->data, t->dlen) <= 0) return 0; } return 1; @@ -223,13 +238,17 @@ static int dane_mtype_set(struct dane_ctx_st *dctx, int n = ((int)mtype) + 1; mdevp = OPENSSL_realloc(dctx->mdevp, n * sizeof(*mdevp)); - if (mdevp == NULL) + if (mdevp == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return -1; + } dctx->mdevp = mdevp; mdord = OPENSSL_realloc(dctx->mdord, n * sizeof(*mdord)); - if (mdord == NULL) + if (mdord == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return -1; + } dctx->mdord = mdord; /* Zero-fill any gaps */ @@ -303,8 +322,10 @@ static int dane_tlsa_add(SSL_DANE *dane, return 0; } - if ((t = OPENSSL_zalloc(sizeof(*t))) == NULL) + if ((t = OPENSSL_zalloc(sizeof(*t))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return -1; + } t->usage = usage; t->selector = selector; @@ -312,6 +333,7 @@ static int dane_tlsa_add(SSL_DANE *dane, t->data = OPENSSL_malloc(dlen); if (t->data == NULL) { tlsa_free(t); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return -1; } memcpy(t->data, data, dlen); @@ -327,31 +349,17 @@ static int dane_tlsa_add(SSL_DANE *dane, case DANETLS_SELECTOR_CERT: if (!d2i_X509(&cert, &p, ilen) || p < data || dlen != (size_t)(p - data)) { - X509_free(cert); tlsa_free(t); ERR_raise(ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_CERTIFICATE); return 0; } if (X509_get0_pubkey(cert) == NULL) { - X509_free(cert); tlsa_free(t); ERR_raise(ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_CERTIFICATE); return 0; } if ((DANETLS_USAGE_BIT(usage) & DANETLS_TA_MASK) == 0) { - /* - * The Full(0) certificate decodes to a seemingly valid X.509 - * object with a plausible key, so the TLSA record is well - * formed. However, we don't actually need the certificate for - * usages PKIX-EE(1) or DANE-EE(3), because at least the EE - * certificate is always presented by the peer. We discard the - * certificate, and just use the TLSA data as an opaque blob - * for matching the raw presented DER octets. - * - * DO NOT FREE `t` here, it will be added to the TLSA record - * list below! - */ X509_free(cert); break; } @@ -366,7 +374,7 @@ static int dane_tlsa_add(SSL_DANE *dane, if ((dane->certs == NULL && (dane->certs = sk_X509_new_null()) == NULL) || !sk_X509_push(dane->certs, cert)) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); X509_free(cert); tlsa_free(t); return -1; @@ -376,7 +384,6 @@ static int dane_tlsa_add(SSL_DANE *dane, case DANETLS_SELECTOR_SPKI: if (!d2i_PUBKEY(&pkey, &p, ilen) || p < data || dlen != (size_t)(p - data)) { - EVP_PKEY_free(pkey); tlsa_free(t); ERR_raise(ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_PUBLIC_KEY); return 0; @@ -428,7 +435,7 @@ static int dane_tlsa_add(SSL_DANE *dane, if (!sk_danetls_record_insert(dane->trecs, t, i)) { tlsa_free(t); - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return -1; } dane->umask |= DANETLS_USAGE_BIT(usage); @@ -540,8 +547,13 @@ static int ssl_check_allowed_versions(int min_version, int max_version) #ifdef OPENSSL_NO_TLS1_3 || (min_version <= TLS1_3_VERSION && TLS1_3_VERSION <= max_version) #endif - ) + ) { +#ifndef OPENSSL_NO_NTLS + if (min_version == NTLS_VERSION || max_version == NTLS_VERSION) + return 1; +#endif return 0; + } } return 1; } @@ -558,6 +570,15 @@ static int ssl_check_allowed_versions(int min_version, int max_version) void OPENSSL_VPROC_FUNC(void) {} #endif + +static void clear_ciphers(SSL *s) +{ + /* clear the current cipher */ + ssl_clear_cipher_ctx(s); + ssl_clear_hash_ctx(&s->read_hash); + ssl_clear_hash_ctx(&s->write_hash); +} + int SSL_clear(SSL *s) { if (s->method == NULL) { @@ -565,86 +586,77 @@ int SSL_clear(SSL *s) return 0; } - return s->method->ssl_reset(s); -} - -int ossl_ssl_connection_reset(SSL *s) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (ssl_clear_bad_session(sc)) { - SSL_SESSION_free(sc->session); - sc->session = NULL; + if (ssl_clear_bad_session(s)) { + SSL_SESSION_free(s->session); + s->session = NULL; } - SSL_SESSION_free(sc->psksession); - sc->psksession = NULL; - OPENSSL_free(sc->psksession_id); - sc->psksession_id = NULL; - sc->psksession_id_len = 0; - sc->hello_retry_request = SSL_HRR_NONE; - sc->sent_tickets = 0; + SSL_SESSION_free(s->psksession); + s->psksession = NULL; + OPENSSL_free(s->psksession_id); + s->psksession_id = NULL; + s->psksession_id_len = 0; + s->hello_retry_request = 0; + s->sent_tickets = 0; - sc->error = 0; - sc->hit = 0; - sc->shutdown = 0; + s->error = 0; + s->hit = 0; + s->shutdown = 0; - if (sc->renegotiate) { + if (s->renegotiate) { ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); return 0; } - ossl_statem_clear(sc); + ossl_statem_clear(s); - sc->version = s->method->version; - sc->client_version = sc->version; - sc->rwstate = SSL_NOTHING; + s->version = s->method->version; + s->client_version = s->version; + s->rwstate = SSL_NOTHING; - BUF_MEM_free(sc->init_buf); - sc->init_buf = NULL; - sc->first_packet = 0; + BUF_MEM_free(s->init_buf); + s->init_buf = NULL; + clear_ciphers(s); + s->first_packet = 0; - sc->key_update = SSL_KEY_UPDATE_NONE; - memset(sc->ext.compress_certificate_from_peer, 0, - sizeof(sc->ext.compress_certificate_from_peer)); - sc->ext.compress_certificate_sent = 0; + s->key_update = SSL_KEY_UPDATE_NONE; - EVP_MD_CTX_free(sc->pha_dgst); - sc->pha_dgst = NULL; + EVP_MD_CTX_free(s->pha_dgst); + s->pha_dgst = NULL; /* Reset DANE verification result state */ - sc->dane.mdpth = -1; - sc->dane.pdpth = -1; - X509_free(sc->dane.mcert); - sc->dane.mcert = NULL; - sc->dane.mtlsa = NULL; + s->dane.mdpth = -1; + s->dane.pdpth = -1; + X509_free(s->dane.mcert); + s->dane.mcert = NULL; + s->dane.mtlsa = NULL; /* Clear the verification result peername */ - X509_VERIFY_PARAM_move_peername(sc->param, NULL); + X509_VERIFY_PARAM_move_peername(s->param, NULL); /* Clear any shared connection state */ - OPENSSL_free(sc->shared_sigalgs); - sc->shared_sigalgs = NULL; - sc->shared_sigalgslen = 0; - + OPENSSL_free(s->shared_sigalgs); + s->shared_sigalgs = NULL; + s->shared_sigalgslen = 0; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + OPENSSL_free(s->shared_dc_sigalgs); + s->shared_dc_sigalgs = NULL; + s->shared_dc_sigalgslen = 0; +#endif /* * Check to see if we were changed into a different method, if so, revert * back. */ - if (s->method != s->defltmeth) { - s->method->ssl_deinit(s); - s->method = s->defltmeth; - if (!s->method->ssl_init(s)) + if (s->method != s->ctx->method) { + s->method->ssl_free(s); + s->method = s->ctx->method; + if (!s->method->ssl_new(s)) return 0; } else { if (!s->method->ssl_clear(s)) return 0; } - if (!RECORD_LAYER_reset(&sc->rlayer)) - return 0; + RECORD_LAYER_clear(&s->rlayer); return 1; } @@ -655,11 +667,6 @@ int SSL_CTX_set_ssl_version(SSL_CTX *ctx, const SSL_METHOD *meth) { STACK_OF(SSL_CIPHER) *sk; - if (IS_QUIC_CTX(ctx)) { - ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_SSL_VERSION); - return 0; - } - ctx->method = meth; if (!SSL_CTX_set_ciphersuites(ctx, OSSL_default_ciphersuites())) { @@ -670,7 +677,7 @@ int SSL_CTX_set_ssl_version(SSL_CTX *ctx, const SSL_METHOD *meth) ctx->tls13_ciphersuites, &(ctx->cipher_list), &(ctx->cipher_list_by_id), - OSSL_default_cipher_list(), ctx->cert); + SSL_SYSTEM_DEFAULT_CIPHER_LIST, ctx->cert); if ((sk == NULL) || (sk_SSL_CIPHER_num(sk) <= 0)) { ERR_raise(ERR_LIB_SSL, SSL_R_SSL_LIBRARY_HAS_NO_CIPHERS); return 0; @@ -681,6 +688,8 @@ int SSL_CTX_set_ssl_version(SSL_CTX *ctx, const SSL_METHOD *meth) SSL *SSL_new(SSL_CTX *ctx) { + SSL *s; + if (ctx == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_NULL_SSL_CTX); return NULL; @@ -689,76 +698,47 @@ SSL *SSL_new(SSL_CTX *ctx) ERR_raise(ERR_LIB_SSL, SSL_R_SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION); return NULL; } - return ctx->method->ssl_new(ctx); -} - -int ossl_ssl_init(SSL *ssl, SSL_CTX *ctx, const SSL_METHOD *method, int type) -{ - ssl->type = type; - - ssl->lock = CRYPTO_THREAD_lock_new(); - if (ssl->lock == NULL) - return 0; - - if (!CRYPTO_NEW_REF(&ssl->references, 1)) { - CRYPTO_THREAD_lock_free(ssl->lock); - return 0; - } - - if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL, ssl, &ssl->ex_data)) { - CRYPTO_THREAD_lock_free(ssl->lock); - CRYPTO_FREE_REF(&ssl->references); - ssl->lock = NULL; - return 0; - } - - SSL_CTX_up_ref(ctx); - ssl->ctx = ctx; - - ssl->defltmeth = ssl->method = method; - - return 1; -} - -SSL *ossl_ssl_connection_new_int(SSL_CTX *ctx, const SSL_METHOD *method) -{ - SSL_CONNECTION *s; - SSL *ssl; s = OPENSSL_zalloc(sizeof(*s)); if (s == NULL) - return NULL; + goto err; - ssl = &s->ssl; - if (!ossl_ssl_init(ssl, ctx, method, SSL_TYPE_SSL_CONNECTION)) { + s->references = 1; + s->lock = CRYPTO_THREAD_lock_new(); + if (s->lock == NULL) { OPENSSL_free(s); s = NULL; - ssl = NULL; - goto sslerr; + goto err; } RECORD_LAYER_init(&s->rlayer, s); s->options = ctx->options; - s->dane.flags = ctx->dane.flags; - if (method->version == ctx->method->version) { - s->min_proto_version = ctx->min_proto_version; - s->max_proto_version = ctx->max_proto_version; - } - + s->min_proto_version = ctx->min_proto_version; + s->max_proto_version = ctx->max_proto_version; s->mode = ctx->mode; s->max_cert_list = ctx->max_cert_list; s->max_early_data = ctx->max_early_data; s->recv_max_early_data = ctx->recv_max_early_data; - s->num_tickets = ctx->num_tickets; s->pha_enabled = ctx->pha_enabled; - +#ifndef OPENSSL_NO_NTLS + s->enable_ntls = ctx->enable_ntls; + s->enable_force_ntls = ctx->enable_force_ntls; +#endif +#ifndef OPENSSL_NO_SM2 + s->enable_sm_tls13_strict = ctx->enable_sm_tls13_strict; +#endif +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + s->enable_verify_peer_by_dc = ctx->enable_verify_peer_by_dc; + s->enable_sign_by_dc = ctx->enable_sign_by_dc; + s->delegated_credential_tag = 0; +#endif /* Shallow copy of the ciphersuites stack */ s->tls13_ciphersuites = sk_SSL_CIPHER_dup(ctx->tls13_ciphersuites); if (s->tls13_ciphersuites == NULL) - goto cerr; + goto err; /* * Earlier library versions used to copy the pointer to the CERT, not @@ -771,16 +751,16 @@ SSL *ossl_ssl_connection_new_int(SSL_CTX *ctx, const SSL_METHOD *method) */ s->cert = ssl_cert_dup(ctx->cert); if (s->cert == NULL) - goto sslerr; + goto err; RECORD_LAYER_set_read_ahead(&s->rlayer, ctx->read_ahead); s->msg_callback = ctx->msg_callback; s->msg_callback_arg = ctx->msg_callback_arg; s->verify_mode = ctx->verify_mode; s->not_resumable_session_cb = ctx->not_resumable_session_cb; - s->rlayer.record_padding_cb = ctx->record_padding_cb; - s->rlayer.record_padding_arg = ctx->record_padding_arg; - s->rlayer.block_padding = ctx->block_padding; + s->record_padding_cb = ctx->record_padding_cb; + s->record_padding_arg = ctx->record_padding_arg; + s->block_padding = ctx->block_padding; s->sid_ctx_length = ctx->sid_ctx_length; if (!ossl_assert(s->sid_ctx_length <= sizeof(s->sid_ctx))) goto err; @@ -790,18 +770,21 @@ SSL *ossl_ssl_connection_new_int(SSL_CTX *ctx, const SSL_METHOD *method) s->param = X509_VERIFY_PARAM_new(); if (s->param == NULL) - goto asn1err; + goto err; X509_VERIFY_PARAM_inherit(s->param, ctx->param); - s->quiet_shutdown = IS_QUIC_CTX(ctx) ? 0 : ctx->quiet_shutdown; - - if (!IS_QUIC_CTX(ctx)) - s->ext.max_fragment_len_mode = ctx->ext.max_fragment_len_mode; + s->quiet_shutdown = ctx->quiet_shutdown; + s->ext.max_fragment_len_mode = ctx->ext.max_fragment_len_mode; s->max_send_fragment = ctx->max_send_fragment; s->split_send_fragment = ctx->split_send_fragment; s->max_pipelines = ctx->max_pipelines; - s->rlayer.default_read_buf_len = ctx->default_read_buf_len; + if (s->max_pipelines > 1) + RECORD_LAYER_set_read_ahead(&s->rlayer, 1); + if (ctx->default_read_buf_len > 0) + SSL_set_default_read_buffer_len(s, ctx->default_read_buf_len); + SSL_CTX_up_ref(ctx); + s->ctx = ctx; s->ext.debug_cb = 0; s->ext.debug_arg = NULL; s->ext.ticket_expected = 0; @@ -840,14 +823,14 @@ SSL *ossl_ssl_connection_new_int(SSL_CTX *ctx, const SSL_METHOD *method) s->ext.npn = NULL; #endif - if (ctx->ext.alpn != NULL) { - s->ext.alpn = OPENSSL_malloc(ctx->ext.alpn_len); + if (s->ctx->ext.alpn) { + s->ext.alpn = OPENSSL_malloc(s->ctx->ext.alpn_len); if (s->ext.alpn == NULL) { s->ext.alpn_len = 0; goto err; } - memcpy(s->ext.alpn, ctx->ext.alpn, ctx->ext.alpn_len); - s->ext.alpn_len = ctx->ext.alpn_len; + memcpy(s->ext.alpn, s->ctx->ext.alpn, s->ctx->ext.alpn_len); + s->ext.alpn_len = s->ctx->ext.alpn_len; } s->verified_chain = NULL; @@ -856,20 +839,23 @@ SSL *ossl_ssl_connection_new_int(SSL_CTX *ctx, const SSL_METHOD *method) s->default_passwd_callback = ctx->default_passwd_callback; s->default_passwd_callback_userdata = ctx->default_passwd_callback_userdata; + s->method = ctx->method; + s->key_update = SSL_KEY_UPDATE_NONE; - if (!IS_QUIC_CTX(ctx)) { - s->allow_early_data_cb = ctx->allow_early_data_cb; - s->allow_early_data_cb_data = ctx->allow_early_data_cb_data; - } + s->allow_early_data_cb = ctx->allow_early_data_cb; + s->allow_early_data_cb_data = ctx->allow_early_data_cb_data; - if (!method->ssl_init(ssl)) - goto sslerr; + if (!s->method->ssl_new(s)) + goto err; + + s->server = (ctx->method->ssl_accept == ssl_undefined_function) ? 0 : 1; - s->server = (method->ssl_accept == ssl_undefined_function) ? 0 : 1; + if (!SSL_clear(s)) + goto err; - if (!method->ssl_reset(ssl)) - goto sslerr; + if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL, s, &s->ex_data)) + goto err; #ifndef OPENSSL_NO_PSK s->psk_client_callback = ctx->psk_client_callback; @@ -883,94 +869,49 @@ SSL *ossl_ssl_connection_new_int(SSL_CTX *ctx, const SSL_METHOD *method) s->job = NULL; -#ifndef OPENSSL_NO_COMP_ALG - memcpy(s->cert_comp_prefs, ctx->cert_comp_prefs, sizeof(s->cert_comp_prefs)); +#ifndef OPENSSL_NO_QUIC + s->quic_method = ctx->quic_method; #endif - if (ctx->client_cert_type != NULL) { - s->client_cert_type = OPENSSL_memdup(ctx->client_cert_type, - ctx->client_cert_type_len); - if (s->client_cert_type == NULL) - goto sslerr; - s->client_cert_type_len = ctx->client_cert_type_len; - } - if (ctx->server_cert_type != NULL) { - s->server_cert_type = OPENSSL_memdup(ctx->server_cert_type, - ctx->server_cert_type_len); - if (s->server_cert_type == NULL) - goto sslerr; - s->server_cert_type_len = ctx->server_cert_type_len; - } #ifndef OPENSSL_NO_CT - if (!SSL_set_ct_validation_callback(ssl, ctx->ct_validation_callback, + if (!SSL_set_ct_validation_callback(s, ctx->ct_validation_callback, ctx->ct_validation_callback_arg)) - goto sslerr; + goto err; +#endif +#ifndef OPENSSL_NO_CERT_COMPRESSION + if (ctx->cert_comp_algs) { + s->cert_comp_algs = sk_CERT_COMP_deep_copy(ctx->cert_comp_algs, + CERT_COMP_copy, + CERT_COMP_free); + + if (s->cert_comp_algs == NULL) + goto err; + } #endif - s->ssl_pkey_num = SSL_PKEY_NUM + ctx->sigalg_list_len; - return ssl; - cerr: - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); - goto err; - asn1err: - ERR_raise(ERR_LIB_SSL, ERR_R_ASN1_LIB); - goto err; - sslerr: - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); + return s; err: - SSL_free(ssl); + SSL_free(s); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return NULL; } -SSL *ossl_ssl_connection_new(SSL_CTX *ctx) -{ - return ossl_ssl_connection_new_int(ctx, ctx->method); -} - int SSL_is_dtls(const SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (s->type == SSL_TYPE_QUIC_CONNECTION || s->type == SSL_TYPE_QUIC_XSO) - return 0; -#endif - - if (sc == NULL) - return 0; - - return SSL_CONNECTION_IS_DTLS(sc) ? 1 : 0; + return SSL_IS_DTLS(s) ? 1 : 0; } - -int SSL_is_tls(const SSL *s) +#ifndef OPENSSL_NO_NTLS +int SSL_is_ntls(const SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (s->type == SSL_TYPE_QUIC_CONNECTION || s->type == SSL_TYPE_QUIC_XSO) - return 0; -#endif - - if (sc == NULL) - return 0; - - return SSL_CONNECTION_IS_DTLS(sc) ? 0 : 1; + return SSL_IS_NTLS(s) ? 1 : 0; } - -int SSL_is_quic(const SSL *s) -{ -#ifndef OPENSSL_NO_QUIC - if (s->type == SSL_TYPE_QUIC_CONNECTION || s->type == SSL_TYPE_QUIC_XSO) - return 1; #endif - return 0; -} int SSL_up_ref(SSL *s) { int i; - if (CRYPTO_UP_REF(&s->references, &i) <= 0) + if (CRYPTO_UP_REF(&s->references, &i, s->lock) <= 0) return 0; REF_PRINT_COUNT("SSL", s); @@ -994,17 +935,12 @@ int SSL_CTX_set_session_id_context(SSL_CTX *ctx, const unsigned char *sid_ctx, int SSL_set_session_id_context(SSL *ssl, const unsigned char *sid_ctx, unsigned int sid_ctx_len) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - if (sid_ctx_len > SSL_MAX_SID_CTX_LENGTH) { ERR_raise(ERR_LIB_SSL, SSL_R_SSL_SESSION_ID_CONTEXT_TOO_LONG); return 0; } - sc->sid_ctx_length = sid_ctx_len; - memcpy(sc->sid_ctx, sid_ctx, sid_ctx_len); + ssl->sid_ctx_length = sid_ctx_len; + memcpy(ssl->sid_ctx, sid_ctx, sid_ctx_len); return 1; } @@ -1020,11 +956,9 @@ int SSL_CTX_set_generate_session_id(SSL_CTX *ctx, GEN_SESSION_CB cb) int SSL_set_generate_session_id(SSL *ssl, GEN_SESSION_CB cb) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL || !CRYPTO_THREAD_write_lock(ssl->lock)) + if (!CRYPTO_THREAD_write_lock(ssl->lock)) return 0; - sc->generate_session_id = cb; + ssl->generate_session_id = cb; CRYPTO_THREAD_unlock(ssl->lock); return 1; } @@ -1040,19 +974,18 @@ int SSL_has_matching_session_id(const SSL *ssl, const unsigned char *id, * by this SSL. */ SSL_SESSION r, *p; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(ssl); - if (sc == NULL || id_len > sizeof(r.session_id)) + if (id_len > sizeof(r.session_id)) return 0; - r.ssl_version = sc->version; + r.ssl_version = ssl->version; r.session_id_length = id_len; memcpy(r.session_id, id, id_len); - if (!CRYPTO_THREAD_read_lock(sc->session_ctx->lock)) + if (!CRYPTO_THREAD_read_lock(ssl->session_ctx->lock)) return 0; - p = lh_SSL_SESSION_retrieve(sc->session_ctx->sessions, &r); - CRYPTO_THREAD_unlock(sc->session_ctx->lock); + p = lh_SSL_SESSION_retrieve(ssl->session_ctx->sessions, &r); + CRYPTO_THREAD_unlock(ssl->session_ctx->lock); return (p != NULL); } @@ -1063,12 +996,7 @@ int SSL_CTX_set_purpose(SSL_CTX *s, int purpose) int SSL_set_purpose(SSL *s, int purpose) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - return X509_VERIFY_PARAM_set_purpose(sc->param, purpose); + return X509_VERIFY_PARAM_set_purpose(s->param, purpose); } int SSL_CTX_set_trust(SSL_CTX *s, int trust) @@ -1078,37 +1006,21 @@ int SSL_CTX_set_trust(SSL_CTX *s, int trust) int SSL_set_trust(SSL *s, int trust) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - return X509_VERIFY_PARAM_set_trust(sc->param, trust); + return X509_VERIFY_PARAM_set_trust(s->param, trust); } int SSL_set1_host(SSL *s, const char *hostname) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - /* If a hostname is provided and parses as an IP address, * treat it as such. */ - if (hostname != NULL - && X509_VERIFY_PARAM_set1_ip_asc(sc->param, hostname) == 1) + if (hostname && X509_VERIFY_PARAM_set1_ip_asc(s->param, hostname) == 1) return 1; - return X509_VERIFY_PARAM_set1_host(sc->param, hostname, 0); + return X509_VERIFY_PARAM_set1_host(s->param, hostname, 0); } int SSL_add1_host(SSL *s, const char *hostname) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - /* If a hostname is provided and parses as an IP address, * treat it as such. */ if (hostname) @@ -1121,7 +1033,7 @@ int SSL_add1_host(SSL *s, const char *hostname) /* We didn't want it; only to check if it *is* an IP address */ ASN1_OCTET_STRING_free(ip); - old_ip = X509_VERIFY_PARAM_get1_ip_asc(sc->param); + old_ip = X509_VERIFY_PARAM_get1_ip_asc(s->param); if (old_ip) { OPENSSL_free(old_ip); @@ -1129,31 +1041,21 @@ int SSL_add1_host(SSL *s, const char *hostname) return 0; } - return X509_VERIFY_PARAM_set1_ip_asc(sc->param, hostname); + return X509_VERIFY_PARAM_set1_ip_asc(s->param, hostname); } } - return X509_VERIFY_PARAM_add1_host(sc->param, hostname, 0); + return X509_VERIFY_PARAM_add1_host(s->param, hostname, 0); } void SSL_set_hostflags(SSL *s, unsigned int flags) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - X509_VERIFY_PARAM_set_hostflags(sc->param, flags); + X509_VERIFY_PARAM_set_hostflags(s->param, flags); } const char *SSL_get0_peername(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - return X509_VERIFY_PARAM_get0_peername(sc->param); + return X509_VERIFY_PARAM_get0_peername(s->param); } int SSL_CTX_dane_enable(SSL_CTX *ctx) @@ -1179,13 +1081,8 @@ unsigned long SSL_CTX_dane_clear_flags(SSL_CTX *ctx, unsigned long flags) int SSL_dane_enable(SSL *s, const char *basedomain) { - SSL_DANE *dane; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; + SSL_DANE *dane = &s->dane; - dane = &sc->dane; if (s->ctx->dane.mdmax == 0) { ERR_raise(ERR_LIB_SSL, SSL_R_CONTEXT_NOT_DANE_ENABLED); return 0; @@ -1197,10 +1094,10 @@ int SSL_dane_enable(SSL *s, const char *basedomain) /* * Default SNI name. This rejects empty names, while set1_host below - * accepts them and disables hostname checks. To avoid side-effects with + * accepts them and disables host name checks. To avoid side-effects with * invalid input, set the SNI name first. */ - if (sc->ext.hostname == NULL) { + if (s->ext.hostname == NULL) { if (!SSL_set_tlsext_host_name(s, basedomain)) { ERR_raise(ERR_LIB_SSL, SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN); return -1; @@ -1208,7 +1105,7 @@ int SSL_dane_enable(SSL *s, const char *basedomain) } /* Primary RFC6125 reference identifier */ - if (!X509_VERIFY_PARAM_set1_host(sc->param, basedomain, 0)) { + if (!X509_VERIFY_PARAM_set1_host(s->param, basedomain, 0)) { ERR_raise(ERR_LIB_SSL, SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN); return -1; } @@ -1219,7 +1116,7 @@ int SSL_dane_enable(SSL *s, const char *basedomain) dane->trecs = sk_danetls_record_new_null(); if (dane->trecs == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return -1; } return 1; @@ -1227,43 +1124,25 @@ int SSL_dane_enable(SSL *s, const char *basedomain) unsigned long SSL_dane_set_flags(SSL *ssl, unsigned long flags) { - unsigned long orig; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - - orig = sc->dane.flags; + unsigned long orig = ssl->dane.flags; - sc->dane.flags |= flags; + ssl->dane.flags |= flags; return orig; } unsigned long SSL_dane_clear_flags(SSL *ssl, unsigned long flags) { - unsigned long orig; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); + unsigned long orig = ssl->dane.flags; - if (sc == NULL) - return 0; - - orig = sc->dane.flags; - - sc->dane.flags &= ~flags; + ssl->dane.flags &= ~flags; return orig; } int SSL_get0_dane_authority(SSL *s, X509 **mcert, EVP_PKEY **mspki) { - SSL_DANE *dane; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return -1; - - dane = &sc->dane; + SSL_DANE *dane = &s->dane; - if (!DANETLS_ENABLED(dane) || sc->verify_result != X509_V_OK) + if (!DANETLS_ENABLED(dane) || s->verify_result != X509_V_OK) return -1; if (dane->mtlsa) { if (mcert) @@ -1277,15 +1156,9 @@ int SSL_get0_dane_authority(SSL *s, X509 **mcert, EVP_PKEY **mspki) int SSL_get0_dane_tlsa(SSL *s, uint8_t *usage, uint8_t *selector, uint8_t *mtype, const unsigned char **data, size_t *dlen) { - SSL_DANE *dane; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return -1; - - dane = &sc->dane; + SSL_DANE *dane = &s->dane; - if (!DANETLS_ENABLED(dane) || sc->verify_result != X509_V_OK) + if (!DANETLS_ENABLED(dane) || s->verify_result != X509_V_OK) return -1; if (dane->mtlsa) { if (usage) @@ -1304,23 +1177,13 @@ int SSL_get0_dane_tlsa(SSL *s, uint8_t *usage, uint8_t *selector, SSL_DANE *SSL_get0_dane(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - return &sc->dane; + return &s->dane; } int SSL_dane_tlsa_add(SSL *s, uint8_t usage, uint8_t selector, uint8_t mtype, const unsigned char *data, size_t dlen) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - return dane_tlsa_add(&sc->dane, usage, selector, mtype, data, dlen); + return dane_tlsa_add(&s->dane, usage, selector, mtype, data, dlen); } int SSL_CTX_dane_mtype_set(SSL_CTX *ctx, const EVP_MD *md, uint8_t mtype, @@ -1336,12 +1199,7 @@ int SSL_CTX_set1_param(SSL_CTX *ctx, X509_VERIFY_PARAM *vpm) int SSL_set1_param(SSL *ssl, X509_VERIFY_PARAM *vpm) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - - return X509_VERIFY_PARAM_set1(sc->param, vpm); + return X509_VERIFY_PARAM_set1(ssl->param, vpm); } X509_VERIFY_PARAM *SSL_CTX_get0_param(SSL_CTX *ctx) @@ -1351,22 +1209,12 @@ X509_VERIFY_PARAM *SSL_CTX_get0_param(SSL_CTX *ctx) X509_VERIFY_PARAM *SSL_get0_param(SSL *ssl) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return NULL; - - return sc->param; + return ssl->param; } void SSL_certs_clear(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - ssl_cert_clear_certs(sc->cert); + ssl_cert_clear_certs(s->cert); } void SSL_free(SSL *s) @@ -1375,40 +1223,25 @@ void SSL_free(SSL *s) if (s == NULL) return; - CRYPTO_DOWN_REF(&s->references, &i); + CRYPTO_DOWN_REF(&s->references, &i, s->lock); REF_PRINT_COUNT("SSL", s); if (i > 0) return; REF_ASSERT_ISNT(i < 0); - CRYPTO_free_ex_data(CRYPTO_EX_INDEX_SSL, s, &s->ex_data); - - if (s->method != NULL) - s->method->ssl_free(s); - - SSL_CTX_free(s->ctx); - CRYPTO_THREAD_lock_free(s->lock); - CRYPTO_FREE_REF(&s->references); - - OPENSSL_free(s); -} - -void ossl_ssl_connection_free(SSL *ssl) -{ - SSL_CONNECTION *s; - - s = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - if (s == NULL) - return; - X509_VERIFY_PARAM_free(s->param); dane_final(&s->dane); + CRYPTO_free_ex_data(CRYPTO_EX_INDEX_SSL, s, &s->ex_data); + + RECORD_LAYER_release(&s->rlayer); /* Ignore return value */ ssl_free_wbio_buffer(s); - /* Ignore return value */ - RECORD_LAYER_clear(&s->rlayer); + BIO_free_all(s->wbio); + s->wbio = NULL; + BIO_free_all(s->rbio); + s->rbio = NULL; BUF_MEM_free(s->init_buf); @@ -1426,8 +1259,13 @@ void ossl_ssl_connection_free(SSL *ssl) SSL_SESSION_free(s->psksession); OPENSSL_free(s->psksession_id); + clear_ciphers(s); + ssl_cert_free(s->cert); OPENSSL_free(s->shared_sigalgs); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + OPENSSL_free(s->shared_dc_sigalgs); +#endif /* Free up if allocated */ OPENSSL_free(s->ext.hostname); @@ -1453,16 +1291,30 @@ void ossl_ssl_connection_free(SSL *ssl) OPENSSL_free(s->pha_context); EVP_MD_CTX_free(s->pha_dgst); +#ifndef OPENSSL_NO_QUIC + OPENSSL_free(s->ext.quic_transport_params); + OPENSSL_free(s->ext.peer_quic_transport_params_draft); + OPENSSL_free(s->ext.peer_quic_transport_params); + BUF_MEM_free(s->quic_buf); + OPENSSL_free(s->quic_early_data_context); + while (s->quic_input_data_head != NULL) { + QUIC_DATA *qd; + + qd = s->quic_input_data_head; + s->quic_input_data_head = qd->next; + OPENSSL_free(qd); + } +#endif + sk_X509_NAME_pop_free(s->ca_names, X509_NAME_free); sk_X509_NAME_pop_free(s->client_ca_names, X509_NAME_free); - OPENSSL_free(s->client_cert_type); - OPENSSL_free(s->server_cert_type); + sk_X509_pop_free(s->verified_chain, X509_free); - OSSL_STACK_OF_X509_free(s->verified_chain); + if (s->method != NULL) + s->method->ssl_free(s); - if (ssl->method != NULL) - ssl->method->ssl_deinit(ssl); + SSL_CTX_free(s->ctx); ASYNC_WAIT_CTX_free(s->waitctx); @@ -1473,67 +1325,34 @@ void ossl_ssl_connection_free(SSL *ssl) #ifndef OPENSSL_NO_SRTP sk_SRTP_PROTECTION_PROFILE_free(s->srtp_profiles); #endif +#ifndef OPENSSL_NO_CERT_COMPRESSION + sk_CERT_COMP_pop_free(s->cert_comp_algs, CERT_COMP_free); +#endif + CRYPTO_THREAD_lock_free(s->lock); - /* - * We do this late. We want to ensure that any other references we held to - * these BIOs are freed first *before* we call BIO_free_all(), because - * BIO_free_all() will only free each BIO in the chain if the number of - * references to the first BIO have dropped to 0 - */ - BIO_free_all(s->wbio); - s->wbio = NULL; - BIO_free_all(s->rbio); - s->rbio = NULL; - OPENSSL_free(s->s3.tmp.valid_flags); + OPENSSL_free(s); } void SSL_set0_rbio(SSL *s, BIO *rbio) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) { - ossl_quic_conn_set0_net_rbio(s, rbio); - return; - } -#endif - - if (sc == NULL) - return; - - BIO_free_all(sc->rbio); - sc->rbio = rbio; - sc->rlayer.rrlmethod->set1_bio(sc->rlayer.rrl, sc->rbio); + BIO_free_all(s->rbio); + s->rbio = rbio; } void SSL_set0_wbio(SSL *s, BIO *wbio) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) { - ossl_quic_conn_set0_net_wbio(s, wbio); - return; - } -#endif - - if (sc == NULL) - return; - /* * If the output buffering BIO is still in place, remove it */ - if (sc->bbio != NULL) - sc->wbio = BIO_pop(sc->wbio); + if (s->bbio != NULL) + s->wbio = BIO_pop(s->wbio); - BIO_free_all(sc->wbio); - sc->wbio = wbio; + BIO_free_all(s->wbio); + s->wbio = wbio; /* Re-attach |bbio| to the new |wbio|. */ - if (sc->bbio != NULL) - sc->wbio = BIO_push(sc->bbio, sc->wbio); - - sc->rlayer.wrlmethod->set1_bio(sc->rlayer.wrl, sc->wbio); + if (s->bbio != NULL) + s->wbio = BIO_push(s->bbio, s->wbio); } void SSL_set_bio(SSL *s, BIO *rbio, BIO *wbio) @@ -1578,39 +1397,24 @@ void SSL_set_bio(SSL *s, BIO *rbio, BIO *wbio) BIO *SSL_get_rbio(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_conn_get_net_rbio(s); -#endif - - if (sc == NULL) - return NULL; - - return sc->rbio; + return s->rbio; } BIO *SSL_get_wbio(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_conn_get_net_wbio(s); -#endif - - if (sc == NULL) - return NULL; - - if (sc->bbio != NULL) { + if (s->bbio != NULL) { /* * If |bbio| is active, the true caller-configured BIO is its * |next_bio|. */ - return BIO_next(sc->bbio); + return BIO_next(s->bbio); } - return sc->wbio; + return s->wbio; +} + +BIO *BABASSL_get0_wbio(const SSL *s) +{ + return s->wbio; } int SSL_get_fd(const SSL *s) @@ -1643,27 +1447,12 @@ int SSL_get_wfd(const SSL *s) } #ifndef OPENSSL_NO_SOCK -static const BIO_METHOD *fd_method(SSL *s) -{ -#ifndef OPENSSL_NO_DGRAM - if (IS_QUIC(s)) - return BIO_s_datagram(); -#endif - - return BIO_s_socket(); -} - int SSL_set_fd(SSL *s, int fd) { int ret = 0; BIO *bio = NULL; - if (s->type == SSL_TYPE_QUIC_XSO) { - ERR_raise(ERR_LIB_SSL, SSL_R_CONN_USE_ONLY); - goto err; - } - - bio = BIO_new(fd_method(s)); + bio = BIO_new(BIO_s_socket()); if (bio == NULL) { ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); @@ -1688,16 +1477,10 @@ int SSL_set_fd(SSL *s, int fd) int SSL_set_wfd(SSL *s, int fd) { BIO *rbio = SSL_get_rbio(s); - int desired_type = IS_QUIC(s) ? BIO_TYPE_DGRAM : BIO_TYPE_SOCKET; - - if (s->type == SSL_TYPE_QUIC_XSO) { - ERR_raise(ERR_LIB_SSL, SSL_R_CONN_USE_ONLY); - return 0; - } - if (rbio == NULL || BIO_method_type(rbio) != desired_type + if (rbio == NULL || BIO_method_type(rbio) != BIO_TYPE_SOCKET || (int)BIO_get_fd(rbio, NULL) != fd) { - BIO *bio = BIO_new(fd_method(s)); + BIO *bio = BIO_new(BIO_s_socket()); if (bio == NULL) { ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); @@ -1724,16 +1507,10 @@ int SSL_set_wfd(SSL *s, int fd) int SSL_set_rfd(SSL *s, int fd) { BIO *wbio = SSL_get_wbio(s); - int desired_type = IS_QUIC(s) ? BIO_TYPE_DGRAM : BIO_TYPE_SOCKET; - - if (s->type == SSL_TYPE_QUIC_XSO) { - ERR_raise(ERR_LIB_SSL, SSL_R_CONN_USE_ONLY); - return 0; - } - if (wbio == NULL || BIO_method_type(wbio) != desired_type + if (wbio == NULL || BIO_method_type(wbio) != BIO_TYPE_SOCKET || ((int)BIO_get_fd(wbio, NULL) != fd)) { - BIO *bio = BIO_new(fd_method(s)); + BIO *bio = BIO_new(BIO_s_socket()); if (bio == NULL) { ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); @@ -1754,15 +1531,11 @@ int SSL_set_rfd(SSL *s, int fd) size_t SSL_get_finished(const SSL *s, void *buf, size_t count) { size_t ret = 0; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - ret = sc->s3.tmp.finish_md_len; + ret = s->s3.tmp.finish_md_len; if (count > ret) count = ret; - memcpy(buf, sc->s3.tmp.finish_md, count); + memcpy(buf, s->s3.tmp.finish_md, count); return ret; } @@ -1770,45 +1543,26 @@ size_t SSL_get_finished(const SSL *s, void *buf, size_t count) size_t SSL_get_peer_finished(const SSL *s, void *buf, size_t count) { size_t ret = 0; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - ret = sc->s3.tmp.peer_finish_md_len; + ret = s->s3.tmp.peer_finish_md_len; if (count > ret) count = ret; - memcpy(buf, sc->s3.tmp.peer_finish_md, count); + memcpy(buf, s->s3.tmp.peer_finish_md, count); return ret; } int SSL_get_verify_mode(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->verify_mode; + return s->verify_mode; } int SSL_get_verify_depth(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return X509_VERIFY_PARAM_get_depth(sc->param); + return X509_VERIFY_PARAM_get_depth(s->param); } int (*SSL_get_verify_callback(const SSL *s)) (int, X509_STORE_CTX *) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - - return sc->verify_callback; + return s->verify_callback; } int SSL_CTX_get_verify_mode(const SSL_CTX *ctx) @@ -1828,52 +1582,24 @@ int (*SSL_CTX_get_verify_callback(const SSL_CTX *ctx)) (int, X509_STORE_CTX *) { void SSL_set_verify(SSL *s, int mode, int (*callback) (int ok, X509_STORE_CTX *ctx)) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->verify_mode = mode; + s->verify_mode = mode; if (callback != NULL) - sc->verify_callback = callback; + s->verify_callback = callback; } void SSL_set_verify_depth(SSL *s, int depth) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - X509_VERIFY_PARAM_set_depth(sc->param, depth); + X509_VERIFY_PARAM_set_depth(s->param, depth); } void SSL_set_read_ahead(SSL *s, int yes) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - OSSL_PARAM options[2], *opts = options; - - if (sc == NULL) - return; - - RECORD_LAYER_set_read_ahead(&sc->rlayer, yes); - - *opts++ = OSSL_PARAM_construct_int(OSSL_LIBSSL_RECORD_LAYER_PARAM_READ_AHEAD, - &sc->rlayer.read_ahead); - *opts = OSSL_PARAM_construct_end(); - - /* Ignore return value */ - sc->rlayer.rrlmethod->set_options(sc->rlayer.rrl, options); + RECORD_LAYER_set_read_ahead(&s->rlayer, yes); } int SSL_get_read_ahead(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL_ONLY(s); - - if (sc == NULL) - return 0; - - return RECORD_LAYER_get_read_ahead(&sc->rlayer); + return RECORD_LAYER_get_read_ahead(&s->rlayer); } int SSL_pending(const SSL *s) @@ -1897,38 +1623,16 @@ int SSL_has_pending(const SSL *s) { /* * Similar to SSL_pending() but returns a 1 to indicate that we have - * processed or unprocessed data available or 0 otherwise (as opposed to the - * number of bytes available). Unlike SSL_pending() this will take into - * account read_ahead data. A 1 return simply indicates that we have data. - * That data may not result in any application data, or we may fail to parse - * the records for some reason. + * unprocessed data available or 0 otherwise (as opposed to the number of + * bytes available). Unlike SSL_pending() this will take into account + * read_ahead data. A 1 return simply indicates that we have unprocessed + * data. That data may not result in any application data, or we may fail + * to parse the records for some reason. */ - const SSL_CONNECTION *sc; - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_has_pending(s); -#endif - - sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - /* Check buffered app data if any first */ - if (SSL_CONNECTION_IS_DTLS(sc)) { - TLS_RECORD *rdata; - pitem *item, *iter; - - iter = pqueue_iterator(sc->rlayer.d->buffered_app_data); - while ((item = pqueue_next(&iter)) != NULL) { - rdata = item->data; - if (rdata->length > 0) - return 1; - } - } - - if (RECORD_LAYER_processed_read_pending(&sc->rlayer)) + if (RECORD_LAYER_processed_read_pending(&s->rlayer)) return 1; - return RECORD_LAYER_read_pending(&sc->rlayer); + return RECORD_LAYER_read_pending(&s->rlayer); } X509 *SSL_get1_peer_certificate(const SSL *s) @@ -1943,29 +1647,20 @@ X509 *SSL_get1_peer_certificate(const SSL *s) X509 *SSL_get0_peer_certificate(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - - if (sc->session == NULL) + if ((s == NULL) || (s->session == NULL)) return NULL; else - return sc->session->peer; + return s->session->peer; } STACK_OF(X509) *SSL_get_peer_cert_chain(const SSL *s) { STACK_OF(X509) *r; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - if (sc->session == NULL) + if ((s == NULL) || (s->session == NULL)) r = NULL; else - r = sc->session->peer_chain; + r = s->session->peer_chain; /* * If we are a client, cert_chain includes the peer's own certificate; if @@ -1982,13 +1677,6 @@ STACK_OF(X509) *SSL_get_peer_cert_chain(const SSL *s) int SSL_copy_session_id(SSL *t, const SSL *f) { int i; - /* TODO(QUIC FUTURE): Not allowed for QUIC currently. */ - SSL_CONNECTION *tsc = SSL_CONNECTION_FROM_SSL_ONLY(t); - const SSL_CONNECTION *fsc = SSL_CONNECTION_FROM_CONST_SSL_ONLY(f); - - if (tsc == NULL || fsc == NULL) - return 0; - /* Do we need to do SSL locking? */ if (!SSL_set_session(t, SSL_get_session(f))) { return 0; @@ -1998,16 +1686,16 @@ int SSL_copy_session_id(SSL *t, const SSL *f) * what if we are setup for one protocol version but want to talk another */ if (t->method != f->method) { - t->method->ssl_deinit(t); + t->method->ssl_free(t); t->method = f->method; - if (t->method->ssl_init(t) == 0) + if (t->method->ssl_new(t) == 0) return 0; } - CRYPTO_UP_REF(&fsc->cert->references, &i); - ssl_cert_free(tsc->cert); - tsc->cert = fsc->cert; - if (!SSL_set_session_id_context(t, fsc->sid_ctx, (int)fsc->sid_ctx_length)) { + CRYPTO_UP_REF(&f->cert->references, &i, f->cert->lock); + ssl_cert_free(t->cert); + t->cert = f->cert; + if (!SSL_set_session_id_context(t, f->sid_ctx, (int)f->sid_ctx_length)) { return 0; } @@ -2032,32 +1720,25 @@ int SSL_CTX_check_private_key(const SSL_CTX *ctx) /* Fix this function so that it takes an optional type parameter */ int SSL_check_private_key(const SSL *ssl) { - const SSL_CONNECTION *sc; - - if ((sc = SSL_CONNECTION_FROM_CONST_SSL(ssl)) == NULL) { + if (ssl == NULL) { ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); return 0; } - if (sc->cert->key->x509 == NULL) { + if (ssl->cert->key->x509 == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_NO_CERTIFICATE_ASSIGNED); return 0; } - if (sc->cert->key->privatekey == NULL) { + if (ssl->cert->key->privatekey == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_NO_PRIVATE_KEY_ASSIGNED); return 0; } - return X509_check_private_key(sc->cert->key->x509, - sc->cert->key->privatekey); + return X509_check_private_key(ssl->cert->key->x509, + ssl->cert->key->privatekey); } int SSL_waiting_for_async(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->job) + if (s->job) return 1; return 0; @@ -2065,13 +1746,9 @@ int SSL_waiting_for_async(SSL *s) int SSL_get_all_async_fds(SSL *s, OSSL_ASYNC_FD *fds, size_t *numfds) { - ASYNC_WAIT_CTX *ctx; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; + ASYNC_WAIT_CTX *ctx = s->waitctx; - if ((ctx = sc->waitctx) == NULL) + if (ctx == NULL) return 0; return ASYNC_WAIT_CTX_get_all_fds(ctx, fds, numfds); } @@ -2079,13 +1756,9 @@ int SSL_get_all_async_fds(SSL *s, OSSL_ASYNC_FD *fds, size_t *numfds) int SSL_get_changed_async_fds(SSL *s, OSSL_ASYNC_FD *addfd, size_t *numaddfds, OSSL_ASYNC_FD *delfd, size_t *numdelfds) { - ASYNC_WAIT_CTX *ctx; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; + ASYNC_WAIT_CTX *ctx = s->waitctx; - if ((ctx = sc->waitctx) == NULL) + if (ctx == NULL) return 0; return ASYNC_WAIT_CTX_get_changed_fds(ctx, addfd, numaddfds, delfd, numdelfds); @@ -2105,35 +1778,21 @@ int SSL_CTX_set_async_callback_arg(SSL_CTX *ctx, void *arg) int SSL_set_async_callback(SSL *s, SSL_async_callback_fn callback) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - sc->async_cb = callback; + s->async_cb = callback; return 1; } int SSL_set_async_callback_arg(SSL *s, void *arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - sc->async_cb_arg = arg; + s->async_cb_arg = arg; return 1; } int SSL_get_async_status(SSL *s, int *status) { - ASYNC_WAIT_CTX *ctx; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; + ASYNC_WAIT_CTX *ctx = s->waitctx; - if ((ctx = sc->waitctx) == NULL) + if (ctx == NULL) return 0; *status = ASYNC_WAIT_CTX_get_status(ctx); return 1; @@ -2141,17 +1800,7 @@ int SSL_get_async_status(SSL *s, int *status) int SSL_accept(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return s->method->ssl_accept(s); -#endif - - if (sc == NULL) - return 0; - - if (sc->handshake_func == NULL) { + if (s->handshake_func == NULL) { /* Not properly initialized yet */ SSL_set_accept_state(s); } @@ -2161,17 +1810,7 @@ int SSL_accept(SSL *s) int SSL_connect(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return s->method->ssl_connect(s); -#endif - - if (sc == NULL) - return 0; - - if (sc->handshake_func == NULL) { + if (s->handshake_func == NULL) { /* Not properly initialized yet */ SSL_set_connect_state(s); } @@ -2181,57 +1820,48 @@ int SSL_connect(SSL *s) long SSL_get_default_timeout(const SSL *s) { - return (long int)ossl_time2seconds(s->method->get_timeout()); + return s->method->get_timeout(); } static int ssl_async_wait_ctx_cb(void *arg) { SSL *s = (SSL *)arg; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - return sc->async_cb(s, sc->async_cb_arg); + return s->async_cb(s, s->async_cb_arg); } static int ssl_start_async_job(SSL *s, struct ssl_async_args *args, int (*func) (void *)) { int ret; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->waitctx == NULL) { - sc->waitctx = ASYNC_WAIT_CTX_new(); - if (sc->waitctx == NULL) + if (s->waitctx == NULL) { + s->waitctx = ASYNC_WAIT_CTX_new(); + if (s->waitctx == NULL) return -1; - if (sc->async_cb != NULL + if (s->async_cb != NULL && !ASYNC_WAIT_CTX_set_callback - (sc->waitctx, ssl_async_wait_ctx_cb, s)) + (s->waitctx, ssl_async_wait_ctx_cb, s)) return -1; } - sc->rwstate = SSL_NOTHING; - switch (ASYNC_start_job(&sc->job, sc->waitctx, &ret, func, args, + s->rwstate = SSL_NOTHING; + switch (ASYNC_start_job(&s->job, s->waitctx, &ret, func, args, sizeof(struct ssl_async_args))) { case ASYNC_ERR: - sc->rwstate = SSL_NOTHING; + s->rwstate = SSL_NOTHING; ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_INIT_ASYNC); return -1; case ASYNC_PAUSE: - sc->rwstate = SSL_ASYNC_PAUSED; + s->rwstate = SSL_ASYNC_PAUSED; return -1; case ASYNC_NO_JOBS: - sc->rwstate = SSL_ASYNC_NO_JOBS; + s->rwstate = SSL_ASYNC_NO_JOBS; return -1; case ASYNC_FINISH: - sc->job = NULL; + s->job = NULL; return ret; default: - sc->rwstate = SSL_NOTHING; + s->rwstate = SSL_NOTHING; ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); /* Shouldn't happen */ return -1; @@ -2244,20 +1874,16 @@ static int ssl_io_intern(void *vargs) SSL *s; void *buf; size_t num; - SSL_CONNECTION *sc; args = (struct ssl_async_args *)vargs; s = args->s; buf = args->buf; num = args->num; - if ((sc = SSL_CONNECTION_FROM_SSL(s)) == NULL) - return -1; - switch (args->type) { case READFUNC: - return args->f.func_read(s, buf, num, &sc->asyncrw); + return args->f.func_read(s, buf, num, &s->asyncrw); case WRITEFUNC: - return args->f.func_write(s, buf, num, &sc->asyncrw); + return args->f.func_write(s, buf, num, &s->asyncrw); case OTHERFUNC: return args->f.func_other(s); } @@ -2266,28 +1892,24 @@ static int ssl_io_intern(void *vargs) int ssl_read_internal(SSL *s, void *buf, size_t num, size_t *readbytes) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - #ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return s->method->ssl_read(s, buf, num, readbytes); -#endif - - if (sc == NULL) + if (SSL_IS_QUIC(s)) { + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); return -1; - - if (sc->handshake_func == NULL) { + } +#endif + if (s->handshake_func == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_UNINITIALIZED); return -1; } - if (sc->shutdown & SSL_RECEIVED_SHUTDOWN) { - sc->rwstate = SSL_NOTHING; + if (s->shutdown & SSL_RECEIVED_SHUTDOWN) { + s->rwstate = SSL_NOTHING; return 0; } - if (sc->early_data_state == SSL_EARLY_DATA_CONNECT_RETRY - || sc->early_data_state == SSL_EARLY_DATA_ACCEPT_RETRY) { + if (s->early_data_state == SSL_EARLY_DATA_CONNECT_RETRY + || s->early_data_state == SSL_EARLY_DATA_ACCEPT_RETRY) { ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); return 0; } @@ -2295,9 +1917,9 @@ int ssl_read_internal(SSL *s, void *buf, size_t num, size_t *readbytes) * If we are a client and haven't received the ServerHello etc then we * better do that */ - ossl_statem_check_finish_init(sc, 0); + ossl_statem_check_finish_init(s, 0); - if ((sc->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { + if ((s->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { struct ssl_async_args args; int ret; @@ -2308,7 +1930,7 @@ int ssl_read_internal(SSL *s, void *buf, size_t num, size_t *readbytes) args.f.func_read = s->method->ssl_read; ret = ssl_start_async_job(s, &args, ssl_io_intern); - *readbytes = sc->asyncrw; + *readbytes = s->asyncrw; return ret; } else { return s->method->ssl_read(s, buf, num, readbytes); @@ -2349,15 +1971,13 @@ int SSL_read_ex(SSL *s, void *buf, size_t num, size_t *readbytes) int SSL_read_early_data(SSL *s, void *buf, size_t num, size_t *readbytes) { int ret; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - /* TODO(QUIC 0RTT): 0-RTT support */ - if (sc == NULL || !sc->server) { + if (!s->server) { ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); return SSL_READ_EARLY_DATA_ERROR; } - switch (sc->early_data_state) { + switch (s->early_data_state) { case SSL_EARLY_DATA_NONE: if (!SSL_in_before(s)) { ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); @@ -2366,32 +1986,32 @@ int SSL_read_early_data(SSL *s, void *buf, size_t num, size_t *readbytes) /* fall through */ case SSL_EARLY_DATA_ACCEPT_RETRY: - sc->early_data_state = SSL_EARLY_DATA_ACCEPTING; + s->early_data_state = SSL_EARLY_DATA_ACCEPTING; ret = SSL_accept(s); if (ret <= 0) { /* NBIO or error */ - sc->early_data_state = SSL_EARLY_DATA_ACCEPT_RETRY; + s->early_data_state = SSL_EARLY_DATA_ACCEPT_RETRY; return SSL_READ_EARLY_DATA_ERROR; } /* fall through */ case SSL_EARLY_DATA_READ_RETRY: - if (sc->ext.early_data == SSL_EARLY_DATA_ACCEPTED) { - sc->early_data_state = SSL_EARLY_DATA_READING; + if (s->ext.early_data == SSL_EARLY_DATA_ACCEPTED) { + s->early_data_state = SSL_EARLY_DATA_READING; ret = SSL_read_ex(s, buf, num, readbytes); /* * State machine will update early_data_state to * SSL_EARLY_DATA_FINISHED_READING if we get an EndOfEarlyData * message */ - if (ret > 0 || (ret <= 0 && sc->early_data_state + if (ret > 0 || (ret <= 0 && s->early_data_state != SSL_EARLY_DATA_FINISHED_READING)) { - sc->early_data_state = SSL_EARLY_DATA_READ_RETRY; + s->early_data_state = SSL_EARLY_DATA_READ_RETRY; return ret > 0 ? SSL_READ_EARLY_DATA_SUCCESS : SSL_READ_EARLY_DATA_ERROR; } } else { - sc->early_data_state = SSL_EARLY_DATA_FINISHED_READING; + s->early_data_state = SSL_EARLY_DATA_FINISHED_READING; } *readbytes = 0; return SSL_READ_EARLY_DATA_FINISH; @@ -2404,36 +2024,26 @@ int SSL_read_early_data(SSL *s, void *buf, size_t num, size_t *readbytes) int SSL_get_early_data_status(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL_ONLY(s); - - /* TODO(QUIC 0RTT): 0-RTT support */ - if (sc == NULL) - return 0; - - return sc->ext.early_data; + return s->ext.early_data; } static int ssl_peek_internal(SSL *s, void *buf, size_t num, size_t *readbytes) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - #ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return s->method->ssl_peek(s, buf, num, readbytes); + if (SSL_IS_QUIC(s)) { + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return -1; + } #endif - - if (sc == NULL) - return 0; - - if (sc->handshake_func == NULL) { + if (s->handshake_func == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_UNINITIALIZED); return -1; } - if (sc->shutdown & SSL_RECEIVED_SHUTDOWN) { + if (s->shutdown & SSL_RECEIVED_SHUTDOWN) { return 0; } - if ((sc->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { + if ((s->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { struct ssl_async_args args; int ret; @@ -2444,7 +2054,7 @@ static int ssl_peek_internal(SSL *s, void *buf, size_t num, size_t *readbytes) args.f.func_read = s->method->ssl_peek; ret = ssl_start_async_job(s, &args, ssl_io_intern); - *readbytes = sc->asyncrw; + *readbytes = s->asyncrw; return ret; } else { return s->method->ssl_peek(s, buf, num, readbytes); @@ -2483,45 +2093,35 @@ int SSL_peek_ex(SSL *s, void *buf, size_t num, size_t *readbytes) return ret; } -int ssl_write_internal(SSL *s, const void *buf, size_t num, - uint64_t flags, size_t *written) +int ssl_write_internal(SSL *s, const void *buf, size_t num, size_t *written) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - #ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_write_flags(s, buf, num, flags, written); + if (SSL_IS_QUIC(s)) { + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return -1; + } #endif - - if (sc == NULL) - return 0; - - if (sc->handshake_func == NULL) { + if (s->handshake_func == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_UNINITIALIZED); return -1; } - if (sc->shutdown & SSL_SENT_SHUTDOWN) { - sc->rwstate = SSL_NOTHING; + if (s->shutdown & SSL_SENT_SHUTDOWN) { + s->rwstate = SSL_NOTHING; ERR_raise(ERR_LIB_SSL, SSL_R_PROTOCOL_IS_SHUTDOWN); return -1; } - if (flags != 0) { - ERR_raise(ERR_LIB_SSL, SSL_R_UNSUPPORTED_WRITE_FLAG); - return -1; - } - - if (sc->early_data_state == SSL_EARLY_DATA_CONNECT_RETRY - || sc->early_data_state == SSL_EARLY_DATA_ACCEPT_RETRY - || sc->early_data_state == SSL_EARLY_DATA_READ_RETRY) { + if (s->early_data_state == SSL_EARLY_DATA_CONNECT_RETRY + || s->early_data_state == SSL_EARLY_DATA_ACCEPT_RETRY + || s->early_data_state == SSL_EARLY_DATA_READ_RETRY) { ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); return 0; } /* If we are a client and haven't sent the Finished we better do that */ - ossl_statem_check_finish_init(sc, 1); + ossl_statem_check_finish_init(s, 1); - if ((sc->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { + if ((s->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { int ret; struct ssl_async_args args; @@ -2532,7 +2132,7 @@ int ssl_write_internal(SSL *s, const void *buf, size_t num, args.f.func_write = s->method->ssl_write; ret = ssl_start_async_job(s, &args, ssl_io_intern); - *written = sc->asyncrw; + *written = s->asyncrw; return ret; } else { return s->method->ssl_write(s, buf, num, written); @@ -2542,29 +2142,25 @@ int ssl_write_internal(SSL *s, const void *buf, size_t num, ossl_ssize_t SSL_sendfile(SSL *s, int fd, off_t offset, size_t size, int flags) { ossl_ssize_t ret; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - if (sc->handshake_func == NULL) { + if (s->handshake_func == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_UNINITIALIZED); return -1; } - if (sc->shutdown & SSL_SENT_SHUTDOWN) { - sc->rwstate = SSL_NOTHING; + if (s->shutdown & SSL_SENT_SHUTDOWN) { + s->rwstate = SSL_NOTHING; ERR_raise(ERR_LIB_SSL, SSL_R_PROTOCOL_IS_SHUTDOWN); return -1; } - if (!BIO_get_ktls_send(sc->wbio)) { + if (!BIO_get_ktls_send(s->wbio)) { ERR_raise(ERR_LIB_SSL, SSL_R_UNINITIALIZED); return -1; } /* If we have an alert to send, lets send it */ - if (sc->s3.alert_dispatch > 0) { + if (s->s3.alert_dispatch) { ret = (ossl_ssize_t)s->method->ssl_dispatch_alert(s); if (ret <= 0) { /* SSLfatal() already called if appropriate */ @@ -2573,10 +2169,10 @@ ossl_ssize_t SSL_sendfile(SSL *s, int fd, off_t offset, size_t size, int flags) /* if it went, fall through and send more stuff */ } - sc->rwstate = SSL_WRITING; - if (BIO_flush(sc->wbio) <= 0) { - if (!BIO_should_retry(sc->wbio)) { - sc->rwstate = SSL_NOTHING; + s->rwstate = SSL_WRITING; + if (BIO_flush(s->wbio) <= 0) { + if (!BIO_should_retry(s->wbio)) { + s->rwstate = SSL_NOTHING; } else { #ifdef EAGAIN set_sys_error(EAGAIN); @@ -2596,13 +2192,13 @@ ossl_ssize_t SSL_sendfile(SSL *s, int fd, off_t offset, size_t size, int flags) if ((get_last_sys_error() == EAGAIN) || (get_last_sys_error() == EINTR) || (get_last_sys_error() == EBUSY)) - BIO_set_retry_write(sc->wbio); + BIO_set_retry_write(s->wbio); else #endif ERR_raise(ERR_LIB_SSL, SSL_R_UNINITIALIZED); return ret; } - sc->rwstate = SSL_NOTHING; + s->rwstate = SSL_NOTHING; return ret; #endif } @@ -2617,7 +2213,7 @@ int SSL_write(SSL *s, const void *buf, int num) return -1; } - ret = ssl_write_internal(s, buf, (size_t)num, 0, &written); + ret = ssl_write_internal(s, buf, (size_t)num, &written); /* * The cast is safe here because ret should be <= INT_MAX because num is @@ -2631,13 +2227,7 @@ int SSL_write(SSL *s, const void *buf, int num) int SSL_write_ex(SSL *s, const void *buf, size_t num, size_t *written) { - return SSL_write_ex2(s, buf, num, 0, written); -} - -int SSL_write_ex2(SSL *s, const void *buf, size_t num, uint64_t flags, - size_t *written) -{ - int ret = ssl_write_internal(s, buf, num, flags, written); + int ret = ssl_write_internal(s, buf, num, written); if (ret < 0) ret = 0; @@ -2649,69 +2239,64 @@ int SSL_write_early_data(SSL *s, const void *buf, size_t num, size_t *written) int ret, early_data_state; size_t writtmp; uint32_t partialwrite; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - /* TODO(QUIC 0RTT): This will need special handling for QUIC */ - if (sc == NULL) - return 0; - switch (sc->early_data_state) { + switch (s->early_data_state) { case SSL_EARLY_DATA_NONE: - if (sc->server + if (s->server || !SSL_in_before(s) - || ((sc->session == NULL || sc->session->ext.max_early_data == 0) - && (sc->psk_use_session_cb == NULL))) { + || ((s->session == NULL || s->session->ext.max_early_data == 0) + && (s->psk_use_session_cb == NULL))) { ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); return 0; } /* fall through */ case SSL_EARLY_DATA_CONNECT_RETRY: - sc->early_data_state = SSL_EARLY_DATA_CONNECTING; + s->early_data_state = SSL_EARLY_DATA_CONNECTING; ret = SSL_connect(s); if (ret <= 0) { /* NBIO or error */ - sc->early_data_state = SSL_EARLY_DATA_CONNECT_RETRY; + s->early_data_state = SSL_EARLY_DATA_CONNECT_RETRY; return 0; } /* fall through */ case SSL_EARLY_DATA_WRITE_RETRY: - sc->early_data_state = SSL_EARLY_DATA_WRITING; + s->early_data_state = SSL_EARLY_DATA_WRITING; /* * We disable partial write for early data because we don't keep track * of how many bytes we've written between the SSL_write_ex() call and * the flush if the flush needs to be retried) */ - partialwrite = sc->mode & SSL_MODE_ENABLE_PARTIAL_WRITE; - sc->mode &= ~SSL_MODE_ENABLE_PARTIAL_WRITE; + partialwrite = s->mode & SSL_MODE_ENABLE_PARTIAL_WRITE; + s->mode &= ~SSL_MODE_ENABLE_PARTIAL_WRITE; ret = SSL_write_ex(s, buf, num, &writtmp); - sc->mode |= partialwrite; + s->mode |= partialwrite; if (!ret) { - sc->early_data_state = SSL_EARLY_DATA_WRITE_RETRY; + s->early_data_state = SSL_EARLY_DATA_WRITE_RETRY; return ret; } - sc->early_data_state = SSL_EARLY_DATA_WRITE_FLUSH; + s->early_data_state = SSL_EARLY_DATA_WRITE_FLUSH; /* fall through */ case SSL_EARLY_DATA_WRITE_FLUSH: /* The buffering BIO is still in place so we need to flush it */ - if (statem_flush(sc) != 1) + if (statem_flush(s) != 1) return 0; *written = num; - sc->early_data_state = SSL_EARLY_DATA_WRITE_RETRY; + s->early_data_state = SSL_EARLY_DATA_WRITE_RETRY; return 1; case SSL_EARLY_DATA_FINISHED_READING: case SSL_EARLY_DATA_READ_RETRY: - early_data_state = sc->early_data_state; + early_data_state = s->early_data_state; /* We are a server writing to an unauthenticated client */ - sc->early_data_state = SSL_EARLY_DATA_UNAUTH_WRITING; + s->early_data_state = SSL_EARLY_DATA_UNAUTH_WRITING; ret = SSL_write_ex(s, buf, num, written); /* The buffering BIO is still in place */ if (ret) - (void)BIO_flush(sc->wbio); - sc->early_data_state = early_data_state; + (void)BIO_flush(s->wbio); + s->early_data_state = early_data_state; return ret; default: @@ -2728,23 +2313,14 @@ int SSL_shutdown(SSL *s) * calling it once is usually not enough, even if blocking I/O is used * (see ssl3_shutdown). */ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_conn_shutdown(s, 0, NULL, 0); -#endif - if (sc == NULL) - return -1; - - if (sc->handshake_func == NULL) { + if (s->handshake_func == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_UNINITIALIZED); return -1; } if (!SSL_in_init(s)) { - if ((sc->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { + if ((s->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { struct ssl_async_args args; memset(&args, 0, sizeof(args)); @@ -2764,17 +2340,7 @@ int SSL_shutdown(SSL *s) int SSL_key_update(SSL *s, int updatetype) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_key_update(s, updatetype); -#endif - - if (sc == NULL) - return 0; - - if (!SSL_CONNECTION_IS_TLS13(sc)) { + if (!SSL_IS_TLS13(s)) { ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_SSL_VERSION); return 0; } @@ -2790,43 +2356,33 @@ int SSL_key_update(SSL *s, int updatetype) return 0; } - if (RECORD_LAYER_write_pending(&sc->rlayer)) { + if (RECORD_LAYER_write_pending(&s->rlayer)) { ERR_raise(ERR_LIB_SSL, SSL_R_BAD_WRITE_RETRY); return 0; } - ossl_statem_set_in_init(sc, 1); - sc->key_update = updatetype; + ossl_statem_set_in_init(s, 1); + s->key_update = updatetype; return 1; } int SSL_get_key_update_type(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_get_key_update_type(s); -#endif - - if (sc == NULL) - return 0; - - return sc->key_update; + return s->key_update; } /* * Can we accept a renegotiation request? If yes, set the flag and * return 1 if yes. If not, raise error and return 0. */ -static int can_renegotiate(const SSL_CONNECTION *sc) +static int can_renegotiate(const SSL *s) { - if (SSL_CONNECTION_IS_TLS13(sc)) { + if (SSL_IS_TLS13(s)) { ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_SSL_VERSION); return 0; } - if ((sc->options & SSL_OP_NO_RENEGOTIATION) != 0) { + if ((s->options & SSL_OP_NO_RENEGOTIATION) != 0) { ERR_raise(ERR_LIB_SSL, SSL_R_NO_RENEGOTIATION); return 0; } @@ -2836,203 +2392,153 @@ static int can_renegotiate(const SSL_CONNECTION *sc) int SSL_renegotiate(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - - if (!can_renegotiate(sc)) + if (!can_renegotiate(s)) return 0; - sc->renegotiate = 1; - sc->new_session = 1; + s->renegotiate = 1; + s->new_session = 1; return s->method->ssl_renegotiate(s); } int SSL_renegotiate_abbreviated(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - - if (!can_renegotiate(sc)) + if (!can_renegotiate(s)) return 0; - sc->renegotiate = 1; - sc->new_session = 0; + s->renegotiate = 1; + s->new_session = 0; return s->method->ssl_renegotiate(s); } int SSL_renegotiate_pending(const SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - /* * becomes true when negotiation is requested; false again once a * handshake has finished */ - return (sc->renegotiate != 0); + return (s->renegotiate != 0); } int SSL_new_session_ticket(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - /* If we are in init because we're sending tickets, okay to send more. */ - if ((SSL_in_init(s) && sc->ext.extra_tickets_expected == 0) - || SSL_IS_FIRST_HANDSHAKE(sc) || !sc->server - || !SSL_CONNECTION_IS_TLS13(sc)) + if ((SSL_in_init(s) && s->ext.extra_tickets_expected == 0) + || SSL_IS_FIRST_HANDSHAKE(s) || !s->server + || !SSL_IS_TLS13(s)) return 0; - sc->ext.extra_tickets_expected++; - if (!RECORD_LAYER_write_pending(&sc->rlayer) && !SSL_in_init(s)) - ossl_statem_set_in_init(sc, 1); + s->ext.extra_tickets_expected++; + if (!RECORD_LAYER_write_pending(&s->rlayer) && !SSL_in_init(s)) + ossl_statem_set_in_init(s, 1); return 1; } long SSL_ctrl(SSL *s, int cmd, long larg, void *parg) -{ - return ossl_ctrl_internal(s, cmd, larg, parg, /*no_quic=*/0); -} - -long ossl_ctrl_internal(SSL *s, int cmd, long larg, void *parg, int no_quic) { long l; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - /* - * Routing of ctrl calls for QUIC is a little counterintuitive: - * - * - Firstly (no_quic=0), we pass the ctrl directly to our QUIC - * implementation in case it wants to handle the ctrl specially. - * - * - If our QUIC implementation does not care about the ctrl, it - * will reenter this function with no_quic=1 and we will try to handle - * it directly using the QCSO SSL object stub (not the handshake layer - * SSL object). This is important for e.g. the version configuration - * ctrls below, which must use s->defltmeth (and not sc->defltmeth). - * - * - If we don't handle a ctrl here specially, then processing is - * redirected to the handshake layer SSL object. - */ - if (!no_quic && IS_QUIC(s)) - return s->method->ssl_ctrl(s, cmd, larg, parg); - - if (sc == NULL) - return 0; switch (cmd) { case SSL_CTRL_GET_READ_AHEAD: - return RECORD_LAYER_get_read_ahead(&sc->rlayer); + return RECORD_LAYER_get_read_ahead(&s->rlayer); case SSL_CTRL_SET_READ_AHEAD: - l = RECORD_LAYER_get_read_ahead(&sc->rlayer); - RECORD_LAYER_set_read_ahead(&sc->rlayer, larg); + l = RECORD_LAYER_get_read_ahead(&s->rlayer); + RECORD_LAYER_set_read_ahead(&s->rlayer, larg); return l; - case SSL_CTRL_MODE: - { - OSSL_PARAM options[2], *opts = options; - - sc->mode |= larg; - - *opts++ = OSSL_PARAM_construct_uint32(OSSL_LIBSSL_RECORD_LAYER_PARAM_MODE, - &sc->mode); - *opts = OSSL_PARAM_construct_end(); - - /* Ignore return value */ - sc->rlayer.rrlmethod->set_options(sc->rlayer.rrl, options); + case SSL_CTRL_SET_MSG_CALLBACK_ARG: + s->msg_callback_arg = parg; + return 1; - return sc->mode; - } + case SSL_CTRL_MODE: + return (s->mode |= larg); case SSL_CTRL_CLEAR_MODE: - return (sc->mode &= ~larg); + return (s->mode &= ~larg); case SSL_CTRL_GET_MAX_CERT_LIST: - return (long)sc->max_cert_list; + return (long)s->max_cert_list; case SSL_CTRL_SET_MAX_CERT_LIST: if (larg < 0) return 0; - l = (long)sc->max_cert_list; - sc->max_cert_list = (size_t)larg; + l = (long)s->max_cert_list; + s->max_cert_list = (size_t)larg; return l; case SSL_CTRL_SET_MAX_SEND_FRAGMENT: if (larg < 512 || larg > SSL3_RT_MAX_PLAIN_LENGTH) return 0; #ifndef OPENSSL_NO_KTLS - if (sc->wbio != NULL && BIO_get_ktls_send(sc->wbio)) + if (s->wbio != NULL && BIO_get_ktls_send(s->wbio)) return 0; #endif /* OPENSSL_NO_KTLS */ - sc->max_send_fragment = larg; - if (sc->max_send_fragment < sc->split_send_fragment) - sc->split_send_fragment = sc->max_send_fragment; - sc->rlayer.wrlmethod->set_max_frag_len(sc->rlayer.wrl, larg); + s->max_send_fragment = larg; + if (s->max_send_fragment < s->split_send_fragment) + s->split_send_fragment = s->max_send_fragment; return 1; case SSL_CTRL_SET_SPLIT_SEND_FRAGMENT: - if ((size_t)larg > sc->max_send_fragment || larg == 0) + if ((size_t)larg > s->max_send_fragment || larg == 0) return 0; - sc->split_send_fragment = larg; + s->split_send_fragment = larg; return 1; case SSL_CTRL_SET_MAX_PIPELINES: if (larg < 1 || larg > SSL_MAX_PIPELINES) return 0; - sc->max_pipelines = larg; - if (sc->rlayer.rrlmethod->set_max_pipelines != NULL) - sc->rlayer.rrlmethod->set_max_pipelines(sc->rlayer.rrl, (size_t)larg); + s->max_pipelines = larg; + if (larg > 1) + RECORD_LAYER_set_read_ahead(&s->rlayer, 1); return 1; case SSL_CTRL_GET_RI_SUPPORT: - return sc->s3.send_connection_binding; + return s->s3.send_connection_binding; case SSL_CTRL_SET_RETRY_VERIFY: - sc->rwstate = SSL_RETRY_VERIFY; + s->rwstate = SSL_RETRY_VERIFY; return 1; case SSL_CTRL_CERT_FLAGS: - return (sc->cert->cert_flags |= larg); + return (s->cert->cert_flags |= larg); case SSL_CTRL_CLEAR_CERT_FLAGS: - return (sc->cert->cert_flags &= ~larg); + return (s->cert->cert_flags &= ~larg); case SSL_CTRL_GET_RAW_CIPHERLIST: if (parg) { - if (sc->s3.tmp.ciphers_raw == NULL) + if (s->s3.tmp.ciphers_raw == NULL) return 0; - *(unsigned char **)parg = sc->s3.tmp.ciphers_raw; - return (int)sc->s3.tmp.ciphers_rawlen; + *(unsigned char **)parg = s->s3.tmp.ciphers_raw; + return (int)s->s3.tmp.ciphers_rawlen; } else { return TLS_CIPHER_LEN; } case SSL_CTRL_GET_EXTMS_SUPPORT: - if (!sc->session || SSL_in_init(s) || ossl_statem_get_in_handshake(sc)) + if (!s->session || SSL_in_init(s) || ossl_statem_get_in_handshake(s)) return -1; - if (sc->session->flags & SSL_SESS_FLAG_EXTMS) + if (s->session->flags & SSL_SESS_FLAG_EXTMS) return 1; else return 0; case SSL_CTRL_SET_MIN_PROTO_VERSION: - return ssl_check_allowed_versions(larg, sc->max_proto_version) - && ssl_set_version_bound(s->defltmeth->version, (int)larg, - &sc->min_proto_version); + return ssl_check_allowed_versions(larg, s->max_proto_version) + && ssl_set_version_bound(s->ctx->method->version, (int)larg, + &s->min_proto_version); case SSL_CTRL_GET_MIN_PROTO_VERSION: - return sc->min_proto_version; + return s->min_proto_version; case SSL_CTRL_SET_MAX_PROTO_VERSION: - return ssl_check_allowed_versions(sc->min_proto_version, larg) - && ssl_set_version_bound(s->defltmeth->version, (int)larg, - &sc->max_proto_version); + return ssl_check_allowed_versions(s->min_proto_version, larg) + && ssl_set_version_bound(s->ctx->method->version, (int)larg, + &s->max_proto_version); case SSL_CTRL_GET_MAX_PROTO_VERSION: - return sc->max_proto_version; + return s->max_proto_version; default: - if (IS_QUIC(s)) - return SSL_ctrl((SSL *)sc, cmd, larg, parg); - else - return s->method->ssl_ctrl(s, cmd, larg, parg); + return s->method->ssl_ctrl(s, cmd, larg, parg); } } long SSL_callback_ctrl(SSL *s, int cmd, void (*fp) (void)) { - return s->method->ssl_callback_ctrl(s, cmd, fp); + switch (cmd) { + case SSL_CTRL_SET_MSG_CALLBACK: + s->msg_callback = (void (*) + (int write_p, int version, int content_type, + const void *buf, size_t len, SSL *ssl, + void *arg))(fp); + return 1; + + default: + return s->method->ssl_callback_ctrl(s, cmd, fp); + } } LHASH_OF(SSL_SESSION) *SSL_CTX_sessions(SSL_CTX *ctx) @@ -3061,7 +2567,7 @@ long SSL_CTX_ctrl(SSL_CTX *ctx, int cmd, long larg, void *parg) return tls1_set_groups_list(ctx, NULL, NULL, parg); case SSL_CTRL_SET_SIGALGS_LIST: case SSL_CTRL_SET_CLIENT_SIGALGS_LIST: - return tls1_set_sigalgs_list(ctx, NULL, parg, 0); + return tls1_set_sigalgs_list(NULL, parg, 0); default: return 0; } @@ -3203,17 +2709,13 @@ int ssl_cipher_ptr_id_cmp(const SSL_CIPHER *const *ap, return 0; } -/* - * return a STACK of the ciphers available for the SSL and in order of - * preference - */ +/** return a STACK of the ciphers available for the SSL and in order of + * preference */ STACK_OF(SSL_CIPHER) *SSL_get_ciphers(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc != NULL) { - if (sc->cipher_list != NULL) { - return sc->cipher_list; + if (s != NULL) { + if (s->cipher_list != NULL) { + return s->cipher_list; } else if ((s->ctx != NULL) && (s->ctx->cipher_list != NULL)) { return s->ctx->cipher_list; } @@ -3223,30 +2725,24 @@ STACK_OF(SSL_CIPHER) *SSL_get_ciphers(const SSL *s) STACK_OF(SSL_CIPHER) *SSL_get_client_ciphers(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL || !sc->server) + if ((s == NULL) || !s->server) return NULL; - return sc->peer_ciphers; + return s->peer_ciphers; } STACK_OF(SSL_CIPHER) *SSL_get1_supported_ciphers(SSL *s) { STACK_OF(SSL_CIPHER) *sk = NULL, *ciphers; int i; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; ciphers = SSL_get_ciphers(s); if (!ciphers) return NULL; - if (!ssl_set_client_disabled(sc)) + if (!ssl_set_client_disabled(s)) return NULL; for (i = 0; i < sk_SSL_CIPHER_num(ciphers); i++) { const SSL_CIPHER *c = sk_SSL_CIPHER_value(ciphers, i); - if (!ssl_cipher_disabled(sc, c, SSL_SECOP_CIPHER_SUPPORTED, 0)) { + if (!ssl_cipher_disabled(s, c, SSL_SECOP_CIPHER_SUPPORTED, 0)) { if (!sk) sk = sk_SSL_CIPHER_new_null(); if (!sk) @@ -3262,14 +2758,14 @@ STACK_OF(SSL_CIPHER) *SSL_get1_supported_ciphers(SSL *s) /** return a STACK of the ciphers available for the SSL and in order of * algorithm id */ -STACK_OF(SSL_CIPHER) *ssl_get_ciphers_by_id(SSL_CONNECTION *s) +STACK_OF(SSL_CIPHER) *ssl_get_ciphers_by_id(SSL *s) { if (s != NULL) { - if (s->cipher_list_by_id != NULL) + if (s->cipher_list_by_id != NULL) { return s->cipher_list_by_id; - else if (s->ssl.ctx != NULL - && s->ssl.ctx->cipher_list_by_id != NULL) - return s->ssl.ctx->cipher_list_by_id; + } else if ((s->ctx != NULL) && (s->ctx->cipher_list_by_id != NULL)) { + return s->ctx->cipher_list_by_id; + } } return NULL; } @@ -3348,14 +2844,10 @@ int SSL_CTX_set_cipher_list(SSL_CTX *ctx, const char *str) int SSL_set_cipher_list(SSL *s, const char *str) { STACK_OF(SSL_CIPHER) *sk; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - sk = ssl_create_cipher_list(s->ctx, sc->tls13_ciphersuites, - &sc->cipher_list, &sc->cipher_list_by_id, str, - sc->cert); + sk = ssl_create_cipher_list(s->ctx, s->tls13_ciphersuites, + &s->cipher_list, &s->cipher_list_by_id, str, + s->cert); /* see comment in SSL_CTX_set_cipher_list */ if (sk == NULL) return 0; @@ -3372,18 +2864,14 @@ char *SSL_get_shared_ciphers(const SSL *s, char *buf, int size) STACK_OF(SSL_CIPHER) *clntsk, *srvrsk; const SSL_CIPHER *c; int i; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - if (!sc->server - || sc->peer_ciphers == NULL + if (!s->server + || s->peer_ciphers == NULL || size < 2) return NULL; p = buf; - clntsk = sc->peer_ciphers; + clntsk = s->peer_ciphers; srvrsk = SSL_get_ciphers(s); if (clntsk == NULL || srvrsk == NULL) return NULL; @@ -3398,14 +2886,14 @@ char *SSL_get_shared_ciphers(const SSL *s, char *buf, int size) if (sk_SSL_CIPHER_find(srvrsk, c) < 0) continue; - n = OPENSSL_strnlen(c->name, size); - if (n >= size) { + n = strlen(c->name); + if (n + 1 > size) { if (p != buf) --p; *p = '\0'; return buf; } - memcpy(p, c->name, n); + strcpy(p, c->name); p += n; *(p++) = ':'; size -= n + 1; @@ -3426,18 +2914,11 @@ char *SSL_get_shared_ciphers(const SSL *s, char *buf, int size) */ const char *SSL_get_servername(const SSL *s, const int type) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - int server; - - if (sc == NULL) - return NULL; - /* * If we don't know if we are the client or the server yet then we assume * client. */ - server = sc->handshake_func == NULL ? 0 : sc->server; - + int server = s->handshake_func == NULL ? 0 : s->server; if (type != TLSEXT_NAMETYPE_host_name) return NULL; @@ -3458,8 +2939,8 @@ const char *SSL_get_servername(const SSL *s, const int type) * - The function will return the servername requested by the client in * this handshake or NULL if none was requested. */ - if (sc->hit && !SSL_CONNECTION_IS_TLS13(sc)) - return sc->session->ext.hostname; + if (s->hit && !SSL_IS_TLS13(s)) + return s->session->ext.hostname; } else { /** * Client side @@ -3484,18 +2965,17 @@ const char *SSL_get_servername(const SSL *s, const int type) * (or NULL if it was not called). */ if (SSL_in_before(s)) { - if (sc->ext.hostname == NULL - && sc->session != NULL - && sc->session->ssl_version != TLS1_3_VERSION) - return sc->session->ext.hostname; + if (s->ext.hostname == NULL + && s->session != NULL + && s->session->ssl_version != TLS1_3_VERSION) + return s->session->ext.hostname; } else { - if (!SSL_CONNECTION_IS_TLS13(sc) && sc->hit - && sc->session->ext.hostname != NULL) - return sc->session->ext.hostname; + if (!SSL_IS_TLS13(s) && s->hit && s->session->ext.hostname != NULL) + return s->session->ext.hostname; } } - return sc->ext.hostname; + return s->ext.hostname; } int SSL_get_servername_type(const SSL *s) @@ -3575,20 +3055,11 @@ int SSL_select_next_proto(unsigned char **out, unsigned char *outlen, void SSL_get0_next_proto_negotiated(const SSL *s, const unsigned char **data, unsigned *len) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) { - /* We have no other way to indicate error */ - *data = NULL; - *len = 0; - return; - } - - *data = sc->ext.npn; + *data = s->ext.npn; if (*data == NULL) { *len = 0; } else { - *len = (unsigned int)sc->ext.npn_len; + *len = (unsigned int)s->ext.npn_len; } } @@ -3606,10 +3077,6 @@ void SSL_CTX_set_npn_advertised_cb(SSL_CTX *ctx, SSL_CTX_npn_advertised_cb_func cb, void *arg) { - if (IS_QUIC_CTX(ctx)) - /* NPN not allowed for QUIC */ - return; - ctx->ext.npn_advertised_cb = cb; ctx->ext.npn_advertised_cb_arg = arg; } @@ -3628,10 +3095,6 @@ void SSL_CTX_set_npn_select_cb(SSL_CTX *ctx, SSL_CTX_npn_select_cb_func cb, void *arg) { - if (IS_QUIC_CTX(ctx)) - /* NPN not allowed for QUIC */ - return; - ctx->ext.npn_select_cb = cb; ctx->ext.npn_select_cb_arg = arg; } @@ -3671,8 +3134,10 @@ int SSL_CTX_set_alpn_protos(SSL_CTX *ctx, const unsigned char *protos, return 1; alpn = OPENSSL_memdup(protos, protos_len); - if (alpn == NULL) + if (alpn == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 1; + } OPENSSL_free(ctx->ext.alpn); ctx->ext.alpn = alpn; ctx->ext.alpn_len = protos_len; @@ -3689,15 +3154,11 @@ int SSL_set_alpn_protos(SSL *ssl, const unsigned char *protos, unsigned int protos_len) { unsigned char *alpn; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 1; if (protos_len == 0 || protos == NULL) { - OPENSSL_free(sc->ext.alpn); - sc->ext.alpn = NULL; - sc->ext.alpn_len = 0; + OPENSSL_free(ssl->ext.alpn); + ssl->ext.alpn = NULL; + ssl->ext.alpn_len = 0; return 0; } /* Not valid per RFC */ @@ -3705,11 +3166,13 @@ int SSL_set_alpn_protos(SSL *ssl, const unsigned char *protos, return 1; alpn = OPENSSL_memdup(protos, protos_len); - if (alpn == NULL) + if (alpn == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 1; - OPENSSL_free(sc->ext.alpn); - sc->ext.alpn = alpn; - sc->ext.alpn_len = protos_len; + } + OPENSSL_free(ssl->ext.alpn); + ssl->ext.alpn = alpn; + ssl->ext.alpn_len = protos_len; return 0; } @@ -3736,20 +3199,11 @@ void SSL_CTX_set_alpn_select_cb(SSL_CTX *ctx, void SSL_get0_alpn_selected(const SSL *ssl, const unsigned char **data, unsigned int *len) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(ssl); - - if (sc == NULL) { - /* We have no other way to indicate error */ - *data = NULL; - *len = 0; - return; - } - - *data = sc->s3.alpn_selected; + *data = ssl->s3.alpn_selected; if (*data == NULL) *len = 0; else - *len = (unsigned int)sc->s3.alpn_selected_len; + *len = (unsigned int)ssl->s3.alpn_selected_len; } int SSL_export_keying_material(SSL *s, unsigned char *out, size_t olen, @@ -3757,19 +3211,13 @@ int SSL_export_keying_material(SSL *s, unsigned char *out, size_t olen, const unsigned char *context, size_t contextlen, int use_context) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) + if (s->session == NULL + || (s->version < TLS1_VERSION && s->version != DTLS1_BAD_VER)) return -1; - if (sc->session == NULL - || (sc->version < TLS1_VERSION && sc->version != DTLS1_BAD_VER)) - return -1; - - return sc->ssl.method->ssl3_enc->export_keying_material(sc, out, olen, label, - llen, context, - contextlen, - use_context); + return s->method->ssl3_enc->export_keying_material(s, out, olen, label, + llen, context, + contextlen, use_context); } int SSL_export_keying_material_early(SSL *s, unsigned char *out, size_t olen, @@ -3777,15 +3225,10 @@ int SSL_export_keying_material_early(SSL *s, unsigned char *out, size_t olen, const unsigned char *context, size_t contextlen) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return -1; - - if (sc->version != TLS1_3_VERSION) + if (s->version != TLS1_3_VERSION) return 0; - return tls13_export_keying_material_early(sc, out, olen, label, llen, + return tls13_export_keying_material_early(s, out, olen, label, llen, context, contextlen); } @@ -3836,9 +3279,6 @@ SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, const SSL_METHOD *meth) { SSL_CTX *ret = NULL; -#ifndef OPENSSL_NO_COMP_ALG - int i; -#endif if (meth == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_NULL_SSL_METHOD_PASSED); @@ -3848,32 +3288,27 @@ SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, if (!OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL)) return NULL; - /* Doing this for the run once effect */ if (SSL_get_ex_data_X509_STORE_CTX_idx() < 0) { ERR_raise(ERR_LIB_SSL, SSL_R_X509_VERIFICATION_SETUP_PROBLEMS); goto err; } - ret = OPENSSL_zalloc(sizeof(*ret)); if (ret == NULL) - return NULL; + goto err; /* Init the reference counting before any call to SSL_CTX_free */ - if (!CRYPTO_NEW_REF(&ret->references, 1)) { - OPENSSL_free(ret); - return NULL; - } - + ret->references = 1; ret->lock = CRYPTO_THREAD_lock_new(); if (ret->lock == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); - goto err; + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + OPENSSL_free(ret); + return NULL; } #ifdef TSAN_REQUIRES_LOCKING ret->tsan_lock = CRYPTO_THREAD_lock_new(); if (ret->tsan_lock == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } #endif @@ -3885,6 +3320,17 @@ SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, goto err; } +#ifndef OPENSSL_NO_NTLS + ret->enable_ntls = 0; + ret->enable_force_ntls = 0; +#endif +#ifndef OPENSSL_NO_SM2 + ret->enable_sm_tls13_strict = 0; +#endif +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + ret->enable_verify_peer_by_dc = 0; + ret->enable_sign_by_dc = 0; +#endif ret->method = meth; ret->min_proto_version = 0; ret->max_proto_version = 0; @@ -3895,72 +3341,47 @@ SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, ret->session_timeout = meth->get_timeout(); ret->max_cert_list = SSL_MAX_CERT_LIST_DEFAULT; ret->verify_mode = SSL_VERIFY_NONE; + if ((ret->cert = ssl_cert_new()) == NULL) + goto err; ret->sessions = lh_SSL_SESSION_new(ssl_session_hash, ssl_session_cmp); - if (ret->sessions == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + if (ret->sessions == NULL) goto err; - } ret->cert_store = X509_STORE_new(); - if (ret->cert_store == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + if (ret->cert_store == NULL) goto err; - } #ifndef OPENSSL_NO_CT ret->ctlog_store = CTLOG_STORE_new_ex(libctx, propq); - if (ret->ctlog_store == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CT_LIB); + if (ret->ctlog_store == NULL) goto err; - } #endif /* initialize cipher/digest methods table */ - if (!ssl_load_ciphers(ret)) { - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); - goto err; - } - - if (!ssl_load_groups(ret)) { - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); - goto err; - } - - /* load provider sigalgs */ - if (!ssl_load_sigalgs(ret)) { - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); - goto err; - } - + if (!ssl_load_ciphers(ret)) + goto err2; /* initialise sig algs */ - if (!ssl_setup_sigalgs(ret)) { - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); - goto err; - } + if (!ssl_setup_sig_algs(ret)) + goto err2; - if (!SSL_CTX_set_ciphersuites(ret, OSSL_default_ciphersuites())) { - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); - goto err; - } - if ((ret->cert = ssl_cert_new(SSL_PKEY_NUM + ret->sigalg_list_len)) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); + if (!ssl_load_groups(ret)) + goto err2; + + if (!SSL_CTX_set_ciphersuites(ret, OSSL_default_ciphersuites())) goto err; - } if (!ssl_create_cipher_list(ret, ret->tls13_ciphersuites, &ret->cipher_list, &ret->cipher_list_by_id, - OSSL_default_cipher_list(), ret->cert) + SSL_SYSTEM_DEFAULT_CIPHER_LIST, ret->cert) || sk_SSL_CIPHER_num(ret->cipher_list) <= 0) { ERR_raise(ERR_LIB_SSL, SSL_R_LIBRARY_HAS_NO_CIPHERS); - goto err; + goto err2; } ret->param = X509_VERIFY_PARAM_new(); - if (ret->param == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + if (ret->param == NULL) goto err; - } /* * If these aren't available from the provider we'll get NULL returns. @@ -3969,20 +3390,14 @@ SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, ret->md5 = ssl_evp_md_fetch(libctx, NID_md5, propq); ret->sha1 = ssl_evp_md_fetch(libctx, NID_sha1, propq); - if ((ret->ca_names = sk_X509_NAME_new_null()) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + if ((ret->ca_names = sk_X509_NAME_new_null()) == NULL) goto err; - } - if ((ret->client_ca_names = sk_X509_NAME_new_null()) == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + if ((ret->client_ca_names = sk_X509_NAME_new_null()) == NULL) goto err; - } - if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL_CTX, ret, &ret->ex_data)) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL_CTX, ret, &ret->ex_data)) goto err; - } if ((ret->ext.secure = OPENSSL_secure_zalloc(sizeof(*ret->ext.secure))) == NULL) goto err; @@ -4004,16 +3419,12 @@ SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, ret->options |= SSL_OP_NO_TICKET; if (RAND_priv_bytes_ex(libctx, ret->ext.cookie_hmac_key, - sizeof(ret->ext.cookie_hmac_key), 0) <= 0) { - ERR_raise(ERR_LIB_SSL, ERR_R_RAND_LIB); + sizeof(ret->ext.cookie_hmac_key), 0) <= 0) goto err; - } #ifndef OPENSSL_NO_SRP - if (!ssl_ctx_srp_ctx_init_intern(ret)) { - ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB); + if (!ssl_ctx_srp_ctx_init_intern(ret)) goto err; - } #endif #ifndef OPENSSL_NO_ENGINE # ifdef OPENSSL_SSL_CLIENT_ENGINE_AUTO @@ -4032,21 +3443,6 @@ SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, ERR_clear_error(); } # endif -#endif - -#ifndef OPENSSL_NO_COMP_ALG - /* - * Set the default order: brotli, zlib, zstd - * Including only those enabled algorithms - */ - memset(ret->cert_comp_prefs, 0, sizeof(ret->cert_comp_prefs)); - i = 0; - if (ossl_comp_has_alg(TLSEXT_comp_cert_brotli)) - ret->cert_comp_prefs[i++] = TLSEXT_comp_cert_brotli; - if (ossl_comp_has_alg(TLSEXT_comp_cert_zlib)) - ret->cert_comp_prefs[i++] = TLSEXT_comp_cert_zlib; - if (ossl_comp_has_alg(TLSEXT_comp_cert_zstd)) - ret->cert_comp_prefs[i++] = TLSEXT_comp_cert_zstd; #endif /* * Disable compression by default to prevent CRIME. Applications can @@ -4095,6 +3491,8 @@ SSL_CTX *SSL_CTX_new_ex(OSSL_LIB_CTX *libctx, const char *propq, return ret; err: + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + err2: SSL_CTX_free(ret); return NULL; } @@ -4108,7 +3506,7 @@ int SSL_CTX_up_ref(SSL_CTX *ctx) { int i; - if (CRYPTO_UP_REF(&ctx->references, &i) <= 0) + if (CRYPTO_UP_REF(&ctx->references, &i, ctx->lock) <= 0) return 0; REF_PRINT_COUNT("SSL_CTX", ctx); @@ -4124,7 +3522,7 @@ void SSL_CTX_free(SSL_CTX *a) if (a == NULL) return; - CRYPTO_DOWN_REF(&a->references, &i); + CRYPTO_DOWN_REF(&a->references, &i, a->lock); REF_PRINT_COUNT("SSL_CTX", a); if (i > 0) return; @@ -4157,7 +3555,7 @@ void SSL_CTX_free(SSL_CTX *a) ssl_cert_free(a->cert); sk_X509_NAME_pop_free(a->ca_names, X509_NAME_free); sk_X509_NAME_pop_free(a->client_ca_names, X509_NAME_free); - OSSL_STACK_OF_X509_free(a->extra_certs); + sk_X509_pop_free(a->extra_certs, X509_free); a->comp_methods = NULL; #ifndef OPENSSL_NO_SRTP sk_SRTP_PROTECTION_PROFILE_free(a->srtp_profiles); @@ -4175,6 +3573,10 @@ void SSL_CTX_free(SSL_CTX *a) OPENSSL_free(a->ext.alpn); OPENSSL_secure_free(a->ext.secure); +#ifndef OPENSSL_NO_CERT_COMPRESSION + sk_CERT_COMP_pop_free(a->cert_comp_algs, CERT_COMP_free); +#endif + ssl_evp_md_free(a->md5); ssl_evp_md_free(a->sha1); @@ -4188,36 +3590,15 @@ void SSL_CTX_free(SSL_CTX *a) OPENSSL_free(a->group_list[j].algorithm); } OPENSSL_free(a->group_list); - for (j = 0; j < a->sigalg_list_len; j++) { - OPENSSL_free(a->sigalg_list[j].name); - OPENSSL_free(a->sigalg_list[j].sigalg_name); - OPENSSL_free(a->sigalg_list[j].sigalg_oid); - OPENSSL_free(a->sigalg_list[j].sig_name); - OPENSSL_free(a->sigalg_list[j].sig_oid); - OPENSSL_free(a->sigalg_list[j].hash_name); - OPENSSL_free(a->sigalg_list[j].hash_oid); - OPENSSL_free(a->sigalg_list[j].keytype); - OPENSSL_free(a->sigalg_list[j].keytype_oid); - } - OPENSSL_free(a->sigalg_list); - OPENSSL_free(a->ssl_cert_info); OPENSSL_free(a->sigalg_lookup_cache); - OPENSSL_free(a->tls12_sigalgs); - - OPENSSL_free(a->client_cert_type); - OPENSSL_free(a->server_cert_type); CRYPTO_THREAD_lock_free(a->lock); - CRYPTO_FREE_REF(&a->references); #ifdef TSAN_REQUIRES_LOCKING CRYPTO_THREAD_lock_free(a->tsan_lock); #endif OPENSSL_free(a->propq); -#ifndef OPENSSL_NO_QLOG - OPENSSL_free(a->qlog_title); -#endif OPENSSL_free(a); } @@ -4244,42 +3625,22 @@ void *SSL_CTX_get_default_passwd_cb_userdata(SSL_CTX *ctx) void SSL_set_default_passwd_cb(SSL *s, pem_password_cb *cb) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->default_passwd_callback = cb; + s->default_passwd_callback = cb; } void SSL_set_default_passwd_cb_userdata(SSL *s, void *u) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->default_passwd_callback_userdata = u; + s->default_passwd_callback_userdata = u; } pem_password_cb *SSL_get_default_passwd_cb(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - return sc->default_passwd_callback; + return s->default_passwd_callback; } void *SSL_get_default_passwd_cb_userdata(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - return sc->default_passwd_callback_userdata; + return s->default_passwd_callback_userdata; } void SSL_CTX_set_cert_verify_callback(SSL_CTX *ctx, @@ -4307,23 +3668,42 @@ void SSL_CTX_set_cert_cb(SSL_CTX *c, int (*cb) (SSL *ssl, void *arg), void *arg) ssl_cert_set_cert_cb(c->cert, cb, arg); } +SSL_cert_cb_fn BABASSL_CTX_get_cert_cb(SSL_CTX *c) +{ + return ssl_cert_get_cert_cb(c->cert); +} + +void *BABASSL_CTX_get_cert_cb_arg(SSL_CTX *c) +{ + return ssl_cert_get_cert_cb_arg(c->cert); +} + void SSL_set_cert_cb(SSL *s, int (*cb) (SSL *ssl, void *arg), void *arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); + ssl_cert_set_cert_cb(s->cert, cb, arg); +} - if (sc == NULL) - return; +SSL_cert_cb_fn BABASSL_get_cert_cb(SSL *s) +{ + return ssl_cert_get_cert_cb(s->cert); +} - ssl_cert_set_cert_cb(sc->cert, cb, arg); +void *BABASSL_get_cert_cb_arg(SSL *s) +{ + return ssl_cert_get_cert_cb_arg(s->cert); } -void ssl_set_masks(SSL_CONNECTION *s) +void ssl_set_masks(SSL *s) { CERT *c = s->cert; uint32_t *pvalid = s->s3.tmp.valid_flags; int rsa_enc, rsa_sign, dh_tmp, dsa_sign; unsigned long mask_k, mask_a; int have_ecc_cert, ecdsa_ok; +#ifndef OPENSSL_NO_NTLS + int sm2_enc, sm2_sign; + int ntls_rsa_enc, ntls_rsa_sign; +#endif if (c == NULL) return; @@ -4336,27 +3716,18 @@ void ssl_set_masks(SSL_CONNECTION *s) rsa_sign = pvalid[SSL_PKEY_RSA] & CERT_PKEY_VALID; dsa_sign = pvalid[SSL_PKEY_DSA_SIGN] & CERT_PKEY_VALID; have_ecc_cert = pvalid[SSL_PKEY_ECC] & CERT_PKEY_VALID; +#ifndef OPENSSL_NO_NTLS + sm2_sign = pvalid[SSL_PKEY_SM2_SIGN] & CERT_PKEY_VALID; + sm2_enc = pvalid[SSL_PKEY_SM2_ENC] & CERT_PKEY_VALID; + ntls_rsa_sign = pvalid[SSL_PKEY_RSA_SIGN] & CERT_PKEY_VALID; + ntls_rsa_enc = pvalid[SSL_PKEY_RSA_ENC] & CERT_PKEY_VALID; +#endif mask_k = 0; mask_a = 0; OSSL_TRACE4(TLS_CIPHER, "dh_tmp=%d rsa_enc=%d rsa_sign=%d dsa_sign=%d\n", dh_tmp, rsa_enc, rsa_sign, dsa_sign); -#ifndef OPENSSL_NO_GOST - if (ssl_has_cert(s, SSL_PKEY_GOST12_512)) { - mask_k |= SSL_kGOST | SSL_kGOST18; - mask_a |= SSL_aGOST12; - } - if (ssl_has_cert(s, SSL_PKEY_GOST12_256)) { - mask_k |= SSL_kGOST | SSL_kGOST18; - mask_a |= SSL_aGOST12; - } - if (ssl_has_cert(s, SSL_PKEY_GOST01)) { - mask_k |= SSL_kGOST; - mask_a |= SSL_aGOST01; - } -#endif - if (rsa_enc) mask_k |= SSL_kRSA; @@ -4370,7 +3741,7 @@ void ssl_set_masks(SSL_CONNECTION *s) if (rsa_enc || rsa_sign || (ssl_has_cert(s, SSL_PKEY_RSA_PSS_SIGN) && pvalid[SSL_PKEY_RSA_PSS_SIGN] & CERT_PKEY_EXPLICIT_SIGN - && TLS1_get_version(&s->ssl) == TLS1_2_VERSION)) + && TLS1_get_version(s) == TLS1_2_VERSION)) mask_a |= SSL_aRSA; if (dsa_sign) { @@ -4379,24 +3750,6 @@ void ssl_set_masks(SSL_CONNECTION *s) mask_a |= SSL_aNULL; - /* - * You can do anything with an RPK key, since there's no cert to restrict it - * But we need to check for private keys - */ - if (pvalid[SSL_PKEY_RSA] & CERT_PKEY_RPK) { - mask_a |= SSL_aRSA; - mask_k |= SSL_kRSA; - } - if (pvalid[SSL_PKEY_ECC] & CERT_PKEY_RPK) - mask_a |= SSL_aECDSA; - if (TLS1_get_version(&s->ssl) == TLS1_2_VERSION) { - if (pvalid[SSL_PKEY_RSA_PSS_SIGN] & CERT_PKEY_RPK) - mask_a |= SSL_aRSA; - if (pvalid[SSL_PKEY_ED25519] & CERT_PKEY_RPK - || pvalid[SSL_PKEY_ED448] & CERT_PKEY_RPK) - mask_a |= SSL_aECDSA; - } - /* * An ECC certificate may be usable for ECDH and/or ECDSA cipher suites * depending on the key usage extension. @@ -4413,16 +3766,32 @@ void ssl_set_masks(SSL_CONNECTION *s) /* Allow Ed25519 for TLS 1.2 if peer supports it */ if (!(mask_a & SSL_aECDSA) && ssl_has_cert(s, SSL_PKEY_ED25519) && pvalid[SSL_PKEY_ED25519] & CERT_PKEY_EXPLICIT_SIGN - && TLS1_get_version(&s->ssl) == TLS1_2_VERSION) + && TLS1_get_version(s) == TLS1_2_VERSION) mask_a |= SSL_aECDSA; /* Allow Ed448 for TLS 1.2 if peer supports it */ if (!(mask_a & SSL_aECDSA) && ssl_has_cert(s, SSL_PKEY_ED448) && pvalid[SSL_PKEY_ED448] & CERT_PKEY_EXPLICIT_SIGN - && TLS1_get_version(&s->ssl) == TLS1_2_VERSION) + && TLS1_get_version(s) == TLS1_2_VERSION) mask_a |= SSL_aECDSA; +#ifndef OPENSSL_NO_NTLS + if (sm2_sign) + mask_a |= SSL_aSM2; + + if (sm2_enc) + mask_k |= SSL_kSM2 | SSL_kSM2DHE; + + if (ntls_rsa_sign) + mask_a |= SSL_aRSA; + + if (ntls_rsa_enc) + mask_k |= SSL_kRSA; +#endif + +#ifndef OPENSSL_NO_EC mask_k |= SSL_kECDHE; +#endif #ifndef OPENSSL_NO_PSK mask_k |= SSL_kPSK; @@ -4439,7 +3808,7 @@ void ssl_set_masks(SSL_CONNECTION *s) s->s3.tmp.mask_a = mask_a; } -int ssl_check_srvr_ecc_cert_and_alg(X509 *x, SSL_CONNECTION *s) +int ssl_check_srvr_ecc_cert_and_alg(X509 *x, SSL *s) { if (s->s3.tmp.new_cipher->algorithm_auth & SSL_aECDSA) { /* key usage, if present, must allow signing */ @@ -4451,8 +3820,7 @@ int ssl_check_srvr_ecc_cert_and_alg(X509 *x, SSL_CONNECTION *s) return 1; /* all checks are ok */ } -int ssl_get_server_cert_serverinfo(SSL_CONNECTION *s, - const unsigned char **serverinfo, +int ssl_get_server_cert_serverinfo(SSL *s, const unsigned char **serverinfo, size_t *serverinfo_length) { CERT_PKEY *cpk = s->s3.tmp.cert; @@ -4466,16 +3834,15 @@ int ssl_get_server_cert_serverinfo(SSL_CONNECTION *s, return 1; } -void ssl_update_cache(SSL_CONNECTION *s, int mode) +void ssl_update_cache(SSL *s, int mode) { int i; /* * If the session_id_length is 0, we are not supposed to cache it, and it - * would be rather hard to do anyway :-). Also if the session has already - * been marked as not_resumable we should not cache it for later reuse. + * would be rather hard to do anyway :-) */ - if (s->session->session_id_length == 0 || s->session->not_resumable) + if (s->session->session_id_length == 0) return; /* @@ -4493,7 +3860,7 @@ void ssl_update_cache(SSL_CONNECTION *s, int mode) i = s->session_ctx->session_cache_mode; if ((i & mode) != 0 - && (!s->hit || SSL_CONNECTION_IS_TLS13(s))) { + && (!s->hit || SSL_IS_TLS13(s))) { /* * Add the session to the internal cache. In server side TLSv1.3 we * normally don't do this because by default it's a full stateless ticket @@ -4506,7 +3873,7 @@ void ssl_update_cache(SSL_CONNECTION *s, int mode) * - SSL_OP_NO_TICKET is set in which case it is a stateful ticket */ if ((i & SSL_SESS_CACHE_NO_INTERNAL_STORE) == 0 - && (!SSL_CONNECTION_IS_TLS13(s) + && (!SSL_IS_TLS13(s) || !s->server || (s->max_early_data > 0 && (s->options & SSL_OP_NO_ANTI_REPLAY) == 0) @@ -4521,8 +3888,7 @@ void ssl_update_cache(SSL_CONNECTION *s, int mode) */ if (s->session_ctx->new_session_cb != NULL) { SSL_SESSION_up_ref(s->session); - if (!s->session_ctx->new_session_cb(SSL_CONNECTION_GET_SSL(s), - s->session)) + if (!s->session_ctx->new_session_cb(s, s->session)) SSL_SESSION_free(s->session); } } @@ -4553,126 +3919,104 @@ const SSL_METHOD *SSL_get_ssl_method(const SSL *s) int SSL_set_ssl_method(SSL *s, const SSL_METHOD *meth) { int ret = 1; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - /* Not allowed for QUIC */ - if (sc == NULL - || (s->type != SSL_TYPE_SSL_CONNECTION && s->method != meth) - || (s->type == SSL_TYPE_SSL_CONNECTION && IS_QUIC_METHOD(meth))) - return 0; if (s->method != meth) { const SSL_METHOD *sm = s->method; - int (*hf) (SSL *) = sc->handshake_func; + int (*hf) (SSL *) = s->handshake_func; if (sm->version == meth->version) s->method = meth; else { - sm->ssl_deinit(s); + sm->ssl_free(s); s->method = meth; - ret = s->method->ssl_init(s); + ret = s->method->ssl_new(s); } if (hf == sm->ssl_connect) - sc->handshake_func = meth->ssl_connect; + s->handshake_func = meth->ssl_connect; else if (hf == sm->ssl_accept) - sc->handshake_func = meth->ssl_accept; + s->handshake_func = meth->ssl_accept; } return ret; } int SSL_get_error(const SSL *s, int i) -{ - return ossl_ssl_get_error(s, i, /*check_err=*/1); -} - -int ossl_ssl_get_error(const SSL *s, int i, int check_err) { int reason; unsigned long l; BIO *bio; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); if (i > 0) return SSL_ERROR_NONE; -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) { - reason = ossl_quic_get_error(s, i); - if (reason != SSL_ERROR_NONE) - return reason; - } -#endif - - if (sc == NULL) - return SSL_ERROR_SSL; - /* * Make things return SSL_ERROR_SYSCALL when doing SSL_do_handshake etc, * where we do encode the error */ - if (check_err && (l = ERR_peek_error()) != 0) { + if ((l = ERR_peek_error()) != 0) { if (ERR_GET_LIB(l) == ERR_LIB_SYS) return SSL_ERROR_SYSCALL; else return SSL_ERROR_SSL; } + if (SSL_want_read(s)) { #ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) + if (SSL_IS_QUIC(s)) + return SSL_ERROR_WANT_READ; #endif - { - if (SSL_want_read(s)) { - bio = SSL_get_rbio(s); - if (BIO_should_read(bio)) - return SSL_ERROR_WANT_READ; - else if (BIO_should_write(bio)) - /* - * This one doesn't make too much sense ... We never try to - * write to the rbio, and an application program where rbio and - * wbio are separate couldn't even know what it should wait for. - * However if we ever set s->rwstate incorrectly (so that we - * have SSL_want_read(s) instead of SSL_want_write(s)) and rbio - * and wbio *are* the same, this test works around that bug; so - * it might be safer to keep it. - */ - return SSL_ERROR_WANT_WRITE; - else if (BIO_should_io_special(bio)) { - reason = BIO_get_retry_reason(bio); - if (reason == BIO_RR_CONNECT) - return SSL_ERROR_WANT_CONNECT; - else if (reason == BIO_RR_ACCEPT) - return SSL_ERROR_WANT_ACCEPT; - else - return SSL_ERROR_SYSCALL; /* unknown */ - } - } - if (SSL_want_write(s)) { + bio = SSL_get_rbio(s); +#ifndef OPENSSL_NO_NTLS + if (s->enable_ntls == 1 && s->enable_force_ntls == 0 + && SSL_IS_FIRST_HANDSHAKE(s) + && s->preread_len < sizeof(s->preread_buf) && !BIO_eof(bio)) + return SSL_ERROR_WANT_READ; +#endif + if (BIO_should_read(bio)) + return SSL_ERROR_WANT_READ; + else if (BIO_should_write(bio)) /* - * Access wbio directly - in order to use the buffered bio if - * present + * This one doesn't make too much sense ... We never try to write + * to the rbio, and an application program where rbio and wbio + * are separate couldn't even know what it should wait for. + * However if we ever set s->rwstate incorrectly (so that we have + * SSL_want_read(s) instead of SSL_want_write(s)) and rbio and + * wbio *are* the same, this test works around that bug; so it + * might be safer to keep it. */ - bio = sc->wbio; - if (BIO_should_write(bio)) - return SSL_ERROR_WANT_WRITE; - else if (BIO_should_read(bio)) - /* - * See above (SSL_want_read(s) with BIO_should_write(bio)) - */ - return SSL_ERROR_WANT_READ; - else if (BIO_should_io_special(bio)) { - reason = BIO_get_retry_reason(bio); - if (reason == BIO_RR_CONNECT) - return SSL_ERROR_WANT_CONNECT; - else if (reason == BIO_RR_ACCEPT) - return SSL_ERROR_WANT_ACCEPT; - else - return SSL_ERROR_SYSCALL; - } + return SSL_ERROR_WANT_WRITE; + else if (BIO_should_io_special(bio)) { + reason = BIO_get_retry_reason(bio); + if (reason == BIO_RR_CONNECT) + return SSL_ERROR_WANT_CONNECT; + else if (reason == BIO_RR_ACCEPT) + return SSL_ERROR_WANT_ACCEPT; + else + return SSL_ERROR_SYSCALL; /* unknown */ } } + if (SSL_want_write(s)) { + /* Access wbio directly - in order to use the buffered bio if present */ + bio = s->wbio; + if (BIO_should_write(bio)) + return SSL_ERROR_WANT_WRITE; + else if (BIO_should_read(bio)) + /* + * See above (SSL_want_read(s) with BIO_should_write(bio)) + */ + return SSL_ERROR_WANT_READ; + else if (BIO_should_io_special(bio)) { + reason = BIO_get_retry_reason(bio); + if (reason == BIO_RR_CONNECT) + return SSL_ERROR_WANT_CONNECT; + else if (reason == BIO_RR_ACCEPT) + return SSL_ERROR_WANT_ACCEPT; + else + return SSL_ERROR_SYSCALL; + } + } if (SSL_want_x509_lookup(s)) return SSL_ERROR_WANT_X509_LOOKUP; if (SSL_want_retry_verify(s)) @@ -4681,11 +4025,15 @@ int ossl_ssl_get_error(const SSL *s, int i, int check_err) return SSL_ERROR_WANT_ASYNC; if (SSL_want_async_job(s)) return SSL_ERROR_WANT_ASYNC_JOB; +#ifndef OPENSSL_NO_SESSION_LOOKUP + if (SSL_want_sess_lookup(s)) + return SSL_ERROR_WANT_SESSION_LOOKUP; +#endif if (SSL_want_client_hello_cb(s)) return SSL_ERROR_WANT_CLIENT_HELLO_CB; - if ((sc->shutdown & SSL_RECEIVED_SHUTDOWN) && - (sc->s3.warn_alert == SSL_AD_CLOSE_NOTIFY)) + if ((s->shutdown & SSL_RECEIVED_SHUTDOWN) && + (s->s3.warn_alert == SSL_AD_CLOSE_NOTIFY)) return SSL_ERROR_ZERO_RETURN; return SSL_ERROR_SYSCALL; @@ -4693,37 +4041,30 @@ int ossl_ssl_get_error(const SSL *s, int i, int check_err) static int ssl_do_handshake_intern(void *vargs) { - struct ssl_async_args *args = (struct ssl_async_args *)vargs; - SSL *s = args->s; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); + struct ssl_async_args *args; + SSL *s; - if (sc == NULL) - return -1; + args = (struct ssl_async_args *)vargs; + s = args->s; - return sc->handshake_func(s); + return s->handshake_func(s); } int SSL_do_handshake(SSL *s) { int ret = 1; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_do_handshake(s); -#endif - if (sc->handshake_func == NULL) { + if (s->handshake_func == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_CONNECTION_TYPE_NOT_SET); return -1; } - ossl_statem_check_finish_init(sc, -1); + ossl_statem_check_finish_init(s, -1); s->method->ssl_renegotiate_check(s, 0); if (SSL_in_init(s) || SSL_in_before(s)) { - if ((sc->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { + if ((s->mode & SSL_MODE_ASYNC) && ASYNC_get_current_job() == NULL) { struct ssl_async_args args; memset(&args, 0, sizeof(args)); @@ -4731,48 +4072,43 @@ int SSL_do_handshake(SSL *s) ret = ssl_start_async_job(s, &args, ssl_do_handshake_intern); } else { - ret = sc->handshake_func(s); + ret = s->handshake_func(s); + } + } +#ifndef OPENSSL_NO_QUIC + if (SSL_IS_QUIC(s) && ret == 1) { + if (s->server) { + if (s->early_data_state == SSL_EARLY_DATA_ACCEPTING) { + s->early_data_state = SSL_EARLY_DATA_FINISHED_READING; + s->rwstate = SSL_READING; + ret = 0; + } + } else if (s->early_data_state == SSL_EARLY_DATA_CONNECTING) { + s->early_data_state = SSL_EARLY_DATA_WRITE_RETRY; + s->rwstate = SSL_READING; + ret = 0; } } +#endif return ret; } void SSL_set_accept_state(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) { - ossl_quic_set_accept_state(s); - return; - } -#endif - - sc->server = 1; - sc->shutdown = 0; - ossl_statem_clear(sc); - sc->handshake_func = s->method->ssl_accept; - /* Ignore return value. Its a void public API function */ - RECORD_LAYER_reset(&sc->rlayer); + s->server = 1; + s->shutdown = 0; + ossl_statem_clear(s); + s->handshake_func = s->method->ssl_accept; + clear_ciphers(s); } void SSL_set_connect_state(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) { - ossl_quic_set_connect_state(s); - return; - } -#endif - - sc->server = 0; - sc->shutdown = 0; - ossl_statem_clear(sc); - sc->handshake_func = s->method->ssl_connect; - /* Ignore return value. Its a void public API function */ - RECORD_LAYER_reset(&sc->rlayer); + s->server = 0; + s->shutdown = 0; + ossl_statem_clear(s); + s->handshake_func = s->method->ssl_connect; + clear_ciphers(s); } int ssl_undefined_function(SSL *s) @@ -4792,9 +4128,15 @@ int ssl_undefined_const_function(const SSL *s) return 0; } +const SSL_METHOD *ssl_bad_method(int ver) +{ + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return NULL; +} + const char *ssl_protocol_to_string(int version) { - switch (version) + switch(version) { case TLS1_3_VERSION: return "TLSv1.3"; @@ -4820,6 +4162,11 @@ const char *ssl_protocol_to_string(int version) case DTLS1_2_VERSION: return "DTLSv1.2"; +#ifndef OPENSSL_NO_NTLS + case NTLS1_1_VERSION: + return "NTLSv1.1"; +#endif + default: return "unknown"; } @@ -4827,33 +4174,7 @@ const char *ssl_protocol_to_string(int version) const char *SSL_get_version(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - -#ifndef OPENSSL_NO_QUIC - /* We only support QUICv1 - so if its QUIC its QUICv1 */ - if (s->type == SSL_TYPE_QUIC_CONNECTION || s->type == SSL_TYPE_QUIC_XSO) - return "QUICv1"; -#endif - - if (sc == NULL) - return NULL; - - return ssl_protocol_to_string(sc->version); -} - -__owur int SSL_get_handshake_rtt(const SSL *s, uint64_t *rtt) -{ - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return -1; - if (sc->ts_msg_write.t <= 0 || sc->ts_msg_read.t <= 0) - return 0; /* data not (yet) available */ - if (sc->ts_msg_read.t < sc->ts_msg_write.t) - return -1; - - *rtt = ossl_time2us(ossl_time_subtract(sc->ts_msg_read, sc->ts_msg_write)); - return 1; + return ssl_protocol_to_string(s->version); } static int dup_ca_names(STACK_OF(X509_NAME) **dst, STACK_OF(X509_NAME) *src) @@ -4890,16 +4211,10 @@ SSL *SSL_dup(SSL *s) { SSL *ret; int i; - /* TODO(QUIC FUTURE): Add a SSL_METHOD function for duplication */ - SSL_CONNECTION *retsc; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return NULL; /* If we're not quiescent, just up_ref! */ if (!SSL_in_init(s) || !SSL_in_before(s)) { - CRYPTO_UP_REF(&s->references, &i); + CRYPTO_UP_REF(&s->references, &i, s->lock); return s; } @@ -4908,10 +4223,8 @@ SSL *SSL_dup(SSL *s) */ if ((ret = SSL_new(SSL_get_SSL_CTX(s))) == NULL) return NULL; - if ((retsc = SSL_CONNECTION_FROM_SSL_ONLY(ret)) == NULL) - goto err; - if (sc->session != NULL) { + if (s->session != NULL) { /* * Arranges to share the same session via up_ref. This "copies" * session-id, SSL_METHOD, sid_ctx, and 'cert' @@ -4928,32 +4241,32 @@ SSL *SSL_dup(SSL *s) if (!SSL_set_ssl_method(ret, s->method)) goto err; - if (sc->cert != NULL) { - ssl_cert_free(retsc->cert); - retsc->cert = ssl_cert_dup(sc->cert); - if (retsc->cert == NULL) + if (s->cert != NULL) { + ssl_cert_free(ret->cert); + ret->cert = ssl_cert_dup(s->cert); + if (ret->cert == NULL) goto err; } - if (!SSL_set_session_id_context(ret, sc->sid_ctx, - (int)sc->sid_ctx_length)) + if (!SSL_set_session_id_context(ret, s->sid_ctx, + (int)s->sid_ctx_length)) goto err; } - if (!ssl_dane_dup(retsc, sc)) + if (!ssl_dane_dup(ret, s)) goto err; - retsc->version = sc->version; - retsc->options = sc->options; - retsc->min_proto_version = sc->min_proto_version; - retsc->max_proto_version = sc->max_proto_version; - retsc->mode = sc->mode; + ret->version = s->version; + ret->options = s->options; + ret->min_proto_version = s->min_proto_version; + ret->max_proto_version = s->max_proto_version; + ret->mode = s->mode; SSL_set_max_cert_list(ret, SSL_get_max_cert_list(s)); SSL_set_read_ahead(ret, SSL_get_read_ahead(s)); - retsc->msg_callback = sc->msg_callback; - retsc->msg_callback_arg = sc->msg_callback_arg; + ret->msg_callback = s->msg_callback; + ret->msg_callback_arg = s->msg_callback_arg; SSL_set_verify(ret, SSL_get_verify_mode(s), SSL_get_verify_callback(s)); SSL_set_verify_depth(ret, SSL_get_verify_depth(s)); - retsc->generate_session_id = sc->generate_session_id; + ret->generate_session_id = s->generate_session_id; SSL_set_info_callback(ret, SSL_get_info_callback(s)); @@ -4961,34 +4274,34 @@ SSL *SSL_dup(SSL *s) if (!CRYPTO_dup_ex_data(CRYPTO_EX_INDEX_SSL, &ret->ex_data, &s->ex_data)) goto err; - retsc->server = sc->server; - if (sc->handshake_func) { - if (sc->server) + ret->server = s->server; + if (s->handshake_func) { + if (s->server) SSL_set_accept_state(ret); else SSL_set_connect_state(ret); } - retsc->shutdown = sc->shutdown; - retsc->hit = sc->hit; + ret->shutdown = s->shutdown; + ret->hit = s->hit; - retsc->default_passwd_callback = sc->default_passwd_callback; - retsc->default_passwd_callback_userdata = sc->default_passwd_callback_userdata; + ret->default_passwd_callback = s->default_passwd_callback; + ret->default_passwd_callback_userdata = s->default_passwd_callback_userdata; - X509_VERIFY_PARAM_inherit(retsc->param, sc->param); + X509_VERIFY_PARAM_inherit(ret->param, s->param); /* dup the cipher_list and cipher_list_by_id stacks */ - if (sc->cipher_list != NULL) { - if ((retsc->cipher_list = sk_SSL_CIPHER_dup(sc->cipher_list)) == NULL) + if (s->cipher_list != NULL) { + if ((ret->cipher_list = sk_SSL_CIPHER_dup(s->cipher_list)) == NULL) goto err; } - if (sc->cipher_list_by_id != NULL) - if ((retsc->cipher_list_by_id = sk_SSL_CIPHER_dup(sc->cipher_list_by_id)) + if (s->cipher_list_by_id != NULL) + if ((ret->cipher_list_by_id = sk_SSL_CIPHER_dup(s->cipher_list_by_id)) == NULL) goto err; /* Dup the client_CA list */ - if (!dup_ca_names(&retsc->ca_names, sc->ca_names) - || !dup_ca_names(&retsc->client_ca_names, sc->client_ca_names)) + if (!dup_ca_names(&ret->ca_names, s->ca_names) + || !dup_ca_names(&ret->client_ca_names, s->client_ca_names)) goto err; return ret; @@ -4998,130 +4311,500 @@ SSL *SSL_dup(SSL *s) return NULL; } -X509 *SSL_get_certificate(const SSL *s) +/* + * To be used only in certificate callback, which are not duplicated: + * + * 1) certificates and corresponding keys. + * 2) SNI callback, set to NULL + * 3) client cert engine stuff, set to NULL + * 4) PSK/SRP stuff... + * 5) something in cert + * + * This feature is in experimental status, use with caution! + */ +SSL_CTX *SSL_CTX_dup(SSL_CTX *ctx) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; + SSL_CTX *ret = NULL; - if (sc->cert != NULL) - return sc->cert->key->x509; - else + if (ctx == NULL) { return NULL; -} + } -EVP_PKEY *SSL_get_privatekey(const SSL *s) -{ - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); + ret = OPENSSL_zalloc(sizeof(*ret)); + if (ret == NULL) + goto err; - if (sc == NULL) + ret->references = 1; + ret->lock = CRYPTO_THREAD_lock_new(); + if (ret->lock == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + OPENSSL_free(ret); return NULL; + } - if (sc->cert != NULL) - return sc->cert->key->privatekey; - else - return NULL; -} +#ifdef TSAN_REQUIRES_LOCKING + ret->tsan_lock = CRYPTO_THREAD_lock_new(); + if (ret->tsan_lock == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto err; + } +#endif -X509 *SSL_CTX_get0_certificate(const SSL_CTX *ctx) -{ - if (ctx->cert != NULL) - return ctx->cert->key->x509; - else - return NULL; -} + ret->libctx = ctx->libctx; + if (ctx->propq != NULL) { + ret->propq = OPENSSL_strdup(ctx->propq); + if (ret->propq == NULL) + goto err; + } -EVP_PKEY *SSL_CTX_get0_privatekey(const SSL_CTX *ctx) -{ - if (ctx->cert != NULL) - return ctx->cert->key->privatekey; +#ifndef OPENSSL_NO_NTLS + /* Tag of NTLS */ + ret->enable_ntls = ctx->enable_ntls; + ret->enable_force_ntls = ctx->enable_force_ntls; +#endif +#ifndef OPENSSL_NO_SM2 + ret->enable_sm_tls13_strict = ctx->enable_sm_tls13_strict ; +#endif + ret->method = ctx->method; + ret->min_proto_version = ctx->min_proto_version; + ret->max_proto_version = ctx->max_proto_version; + ret->mode = ctx->mode; + ret->session_cache_mode = ctx->session_cache_mode; + ret->session_cache_size = ctx->session_cache_size; + ret->session_timeout = ctx->session_timeout; + ret->max_cert_list = ctx->max_cert_list; + ret->verify_mode = ctx->verify_mode; + + if (ctx->cert) + ret->cert = ssl_cert_dup(ctx->cert); else - return NULL; -} - -const SSL_CIPHER *SSL_get_current_cipher(const SSL *s) -{ - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); + ret->cert = ssl_cert_new(); - if (sc == NULL) - return NULL; + if(ret->cert == NULL) + goto err; - if ((sc->session != NULL) && (sc->session->cipher != NULL)) - return sc->session->cipher; - return NULL; -} + ret->sessions = lh_SSL_SESSION_new(ssl_session_hash, ssl_session_cmp); + if (ret->sessions == NULL) + goto err; -const SSL_CIPHER *SSL_get_pending_cipher(const SSL *s) -{ - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); + /* we don't really support internal session cache... */ + ret->session_cache_head = NULL; + ret->session_cache_tail = NULL; - if (sc == NULL) - return NULL; + /* dup cert_store */ + ret->cert_store = X509_STORE_new(); + if (ret->cert_store == NULL) + goto err; - return sc->s3.tmp.new_cipher; -} + if (ctx->cert_store && !X509_STORE_copy(ret->cert_store, ctx->cert_store)) + goto err; -const COMP_METHOD *SSL_get_current_compression(const SSL *s) -{ -#ifndef OPENSSL_NO_COMP - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL_ONLY(s); +#ifndef OPENSSL_NO_CT + ret->ctlog_store = CTLOG_STORE_new(); + if (ret->ctlog_store == NULL) + goto err; - if (sc == NULL) - return NULL; + /* TODO: logs*/ - return sc->rlayer.wrlmethod->get_compression(sc->rlayer.wrl); -#else - return NULL; + ret->ct_validation_callback = ctx->ct_validation_callback; + ret->ct_validation_callback_arg = ctx->ct_validation_callback_arg; #endif -} - -const COMP_METHOD *SSL_get_current_expansion(const SSL *s) -{ -#ifndef OPENSSL_NO_COMP - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL_ONLY(s); - - if (sc == NULL) - return NULL; - return sc->rlayer.rrlmethod->get_compression(sc->rlayer.rrl); -#else - return NULL; -#endif -} + /* initialize cipher/digest methods table */ + if (!ssl_load_ciphers(ret)) + goto err2; + /* initialise sig algs */ + if (!ssl_setup_sig_algs(ret)) + goto err2; -int ssl_init_wbio_buffer(SSL_CONNECTION *s) -{ - BIO *bbio; + if (!ssl_load_groups(ret)) + goto err2; - if (s->bbio != NULL) { - /* Already buffered. */ - return 1; + /* dup the cipher_list and cipher_list_by_id stacks */ + if (ctx->cipher_list) { + ret->cipher_list = sk_SSL_CIPHER_dup(ctx->cipher_list); + if (ret->cipher_list == NULL) + goto err; + } + if (ctx->cipher_list_by_id) { + ret->cipher_list_by_id = sk_SSL_CIPHER_dup(ctx->cipher_list_by_id); + if (ret->cipher_list_by_id == NULL) + goto err; } - bbio = BIO_new(BIO_f_buffer()); - if (bbio == NULL || BIO_set_read_buffer_size(bbio, 1) <= 0) { - BIO_free(bbio); - ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); - return 0; + if (ctx->tls13_ciphersuites) { + ret->tls13_ciphersuites = sk_SSL_CIPHER_dup(ctx->tls13_ciphersuites); + if (ret->tls13_ciphersuites == NULL) + goto err; } - s->bbio = bbio; - s->wbio = BIO_push(bbio, s->wbio); - s->rlayer.wrlmethod->set1_bio(s->rlayer.wrl, s->wbio); + if ((ret->param = X509_VERIFY_PARAM_new()) == NULL) + goto err; + X509_VERIFY_PARAM_inherit(ret->param, ctx->param); - return 1; -} + SSL_CTX_set_verify_depth(ret, SSL_CTX_get_verify_depth(ctx)); -int ssl_free_wbio_buffer(SSL_CONNECTION *s) -{ - /* callers ensure s is never null */ + /* + * If these aren't available from the provider we'll get NULL returns. + * That's fine but will cause errors later if SSLv3 is negotiated + */ + ret->md5 = ssl_evp_md_fetch(ctx->libctx, NID_md5, ctx->propq); + ret->sha1 = ssl_evp_md_fetch(ctx->libctx, NID_sha1, ctx->propq); + + if (ctx->extra_certs) + ret->extra_certs = sk_X509_dup(ctx->extra_certs); + + if (ctx->ca_names) + ret->ca_names = sk_X509_NAME_dup(ctx->ca_names); + else + ret->ca_names = sk_X509_NAME_new_null(); + + if (ret->ca_names == NULL) + goto err; + + if (ctx->client_ca_names) + ret->client_ca_names = sk_X509_NAME_dup(ctx->client_ca_names); + else + ret->client_ca_names = sk_X509_NAME_new_null(); + + if (ret->client_ca_names == NULL) + goto err; + + /* copy app data, a little dangerous perhaps */ + if (!CRYPTO_dup_ex_data(CRYPTO_EX_INDEX_SSL_CTX, &ret->ex_data, &ctx->ex_data)) + goto err; + + ret->ext.secure = OPENSSL_zalloc(sizeof(SSL_CTX_EXT_SECURE)); + if (ret->ext.secure == NULL) + goto err; + + /* No compression for DTLS */ + if (!(ctx->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_DTLS)) + ret->comp_methods = SSL_COMP_get_compression_methods(); + + ret->max_send_fragment = ctx->max_send_fragment; + ret->split_send_fragment = ctx->split_send_fragment; + + /* dup RFC4507 ticket keys */ + memcpy(ret->ext.tick_key_name, ctx->ext.tick_key_name, + sizeof(ret->ext.tick_key_name)); +# ifndef OPENSSL_NO_DEPRECATED_3_0 + ret->ext.ticket_key_cb = ctx->ext.ticket_key_cb; +# endif + memcpy(ret->ext.secure->tick_hmac_key, ctx->ext.secure->tick_hmac_key, + sizeof(ret->ext.secure->tick_hmac_key)); + memcpy(ret->ext.secure->tick_aes_key, ctx->ext.secure->tick_aes_key, + sizeof(ret->ext.secure->tick_aes_key)); + memcpy(ret->ext.cookie_hmac_key, ctx->ext.cookie_hmac_key, + sizeof(ret->ext.cookie_hmac_key)); + + ret->options = ctx->options; + +#ifndef OPENSSL_NO_SRP + /* TODO: copy srp */ + if (!ssl_ctx_srp_ctx_init_intern(ret)) + goto err; +#endif +#ifndef OPENSSL_NO_ENGINE +# ifdef OPENSSL_SSL_CLIENT_ENGINE_AUTO +# define eng_strx(x) #x +# define eng_str(x) eng_strx(x) + /* Use specific client engine automatically... ignore errors */ + { + ENGINE *eng; + eng = ENGINE_by_id(eng_str(OPENSSL_SSL_CLIENT_ENGINE_AUTO)); + if (!eng) { + ERR_clear_error(); + ENGINE_load_builtin_engines(); + eng = ENGINE_by_id(eng_str(OPENSSL_SSL_CLIENT_ENGINE_AUTO)); + } + if (!eng || !SSL_CTX_set_client_cert_engine(ret, eng)) + ERR_clear_error(); + } +# endif +#endif + + ret->ext.status_type = ctx->ext.status_type; + ret->ext.status_cb = ctx->ext.status_cb; + ret->ext.status_arg = ctx->ext.status_arg; + + ret->max_early_data = ctx->max_early_data; + ret->recv_max_early_data = ctx->recv_max_early_data; + + ret->num_tickets = ctx->num_tickets; + + ret->new_session_cb = ctx->new_session_cb; + ret->remove_session_cb = ctx->remove_session_cb; + ret->get_session_cb = ctx->get_session_cb; + + ret->stats = ctx->stats; + + ret->app_verify_callback = ctx->app_verify_callback; + ret->app_verify_arg = ctx->app_verify_arg; + + ret->default_passwd_callback = ctx->default_passwd_callback; + ret->default_passwd_callback_userdata = ctx->default_passwd_callback_userdata; + + ret->client_cert_cb = ctx->client_cert_cb; + ret->app_gen_cookie_cb = ctx->app_gen_cookie_cb; + ret->app_verify_cookie_cb = ctx->app_verify_cookie_cb; + + ret->gen_stateless_cookie_cb = ctx->gen_stateless_cookie_cb; + ret->verify_stateless_cookie_cb = ctx->verify_stateless_cookie_cb; + + ret->info_callback = ctx->info_callback; + + ret->read_ahead = ctx->read_ahead; + + ret->msg_callback = ctx->msg_callback; + ret->msg_callback_arg = ctx->msg_callback_arg; + + ret->sid_ctx_length = ctx->sid_ctx_length; + memcpy(ret->sid_ctx, ctx->sid_ctx, sizeof(ret->sid_ctx)); + ret->default_verify_callback = ctx->default_verify_callback; + + ret->generate_session_id = ctx->generate_session_id; + + ret->quiet_shutdown = ctx->quiet_shutdown; + + ret->max_pipelines = ctx->max_pipelines; + ret->default_read_buf_len = ctx->default_read_buf_len; + +#ifndef OPENSSL_NO_ENGINE + ret->client_cert_engine = NULL; +#endif + + ret->client_hello_cb = ctx->client_hello_cb; + ret->client_hello_cb_arg = ctx->client_hello_cb_arg; + +#ifndef OPENSSL_NO_TLSEXT + ret->ext.servername_cb = ctx->ext.servername_cb; + ret->ext.servername_arg = ctx->ext.servername_arg; + + memcpy(ret->ext.tick_key_name, ctx->ext.tick_key_name, + sizeof(ret->ext.tick_key_name)); + + ret->ext.ticket_key_evp_cb = ctx->ext.ticket_key_evp_cb; + ret->ext.max_fragment_len_mode = ctx->ext.max_fragment_len_mode; + +# ifndef OPENSSL_NO_EC + if (ctx->ext.ecpointformats) { + ret->ext.ecpointformats = OPENSSL_memdup(ctx->ext.ecpointformats, + ctx->ext.ecpointformats_len); + if (!ret->ext.ecpointformats) + goto err; + + ret->ext.ecpointformats_len = ctx->ext.ecpointformats_len; + } + if (ctx->ext.supportedgroups) { + ret->ext.supportedgroups = OPENSSL_memdup(ctx->ext.supportedgroups, + ctx->ext.supportedgroups_len + * sizeof(*ctx->ext.supportedgroups)); + if (!ret->ext.supportedgroups) + goto err; + + ret->ext.supportedgroups_len = ctx->ext.supportedgroups_len; + } +# endif + + /* XXX: for ALPN, no need to duplicate client side paras */ + ret->ext.alpn_select_cb = ctx->ext.alpn_select_cb; + ret->ext.alpn_select_cb_arg = ctx->ext.alpn_select_cb_arg; + + if (ctx->ext.alpn) { + ret->ext.alpn = OPENSSL_malloc(ctx->ext.alpn_len); + if (ret->ext.alpn == NULL) + goto err; + + memcpy(ret->ext.alpn, ctx->ext.alpn, ctx->ext.alpn_len); + ret->ext.alpn_len = ctx->ext.alpn_len; + } + +# ifndef OPENSSL_NO_NEXTPROTONEG + ret->ext.npn_advertised_cb = ctx->ext.npn_advertised_cb; + ret->ext.npn_advertised_cb_arg = ctx->ext.npn_advertised_cb_arg; + ret->ext.npn_select_cb = ctx->ext.npn_select_cb; + ret->ext.npn_select_cb_arg = ctx->ext.npn_select_cb_arg; +# endif +#endif + +#ifndef OPENSSL_NO_PSK + /* do we need to dup this char * ? */ + ret->psk_client_callback = ctx->psk_client_callback; + ret->psk_server_callback = ctx->psk_server_callback; +#endif + ret->psk_find_session_cb = ctx->psk_find_session_cb; + ret->psk_use_session_cb = ctx->psk_use_session_cb; + + /* TODO: srp profiles; dane */ + ret->dane = ctx->dane; + +# ifndef OPENSSL_NO_SRTP + /* SRTP profiles we are willing to do from RFC 5764 */ + if (ctx->srtp_profiles) + ret->srtp_profiles = sk_SRTP_PROTECTION_PROFILE_dup(ctx->srtp_profiles); + else + ret->srtp_profiles = sk_SRTP_PROTECTION_PROFILE_new_null(); + + if (ret->srtp_profiles == NULL) + goto err; +# endif + + ret->not_resumable_session_cb = ctx->not_resumable_session_cb; + + ret->keylog_callback = ctx->keylog_callback; + + ret->record_padding_cb = ctx->record_padding_cb; + ret->record_padding_arg = ctx->record_padding_arg; + ret->block_padding = ctx->block_padding; + + ret->generate_ticket_cb = ctx->generate_ticket_cb; + ret->decrypt_ticket_cb = ctx->decrypt_ticket_cb; + ret->ticket_cb_data = ctx->ticket_cb_data; + + ret->allow_early_data_cb = ctx->allow_early_data_cb; + ret->allow_early_data_cb_data = ctx->allow_early_data_cb_data; + + ret->pha_enabled = ctx->pha_enabled; + + ret->async_cb = ctx->async_cb; + ret->async_cb_arg = ctx->async_cb_arg; + +#ifndef OPENSSL_NO_SM2 + ret->enable_sm_tls13_strict = ctx->enable_sm_tls13_strict; +#endif + +#ifndef OPENSSL_NO_QUIC + ret->quic_method = ctx->quic_method; +#endif + +#ifndef OPENSSL_NO_CERT_COMPRESSION + if (ctx->cert_comp_algs) { + ret->cert_comp_algs = sk_CERT_COMP_deep_copy(ctx->cert_comp_algs, + CERT_COMP_copy, + CERT_COMP_free); + if (ret->cert_comp_algs == NULL) + goto err; + } +#endif + + return (ret); + err: + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + err2: + if (ret != NULL) + SSL_CTX_free(ret); + return (NULL); +} + +void ssl_clear_cipher_ctx(SSL *s) +{ + if (s->enc_read_ctx != NULL) { + EVP_CIPHER_CTX_free(s->enc_read_ctx); + s->enc_read_ctx = NULL; + } + if (s->enc_write_ctx != NULL) { + EVP_CIPHER_CTX_free(s->enc_write_ctx); + s->enc_write_ctx = NULL; + } +#ifndef OPENSSL_NO_COMP + COMP_CTX_free(s->expand); + s->expand = NULL; + COMP_CTX_free(s->compress); + s->compress = NULL; +#endif +} + +X509 *SSL_get_certificate(const SSL *s) +{ + if (s->cert != NULL) + return s->cert->key->x509; + else + return NULL; +} + +EVP_PKEY *SSL_get_privatekey(const SSL *s) +{ + if (s->cert != NULL) + return s->cert->key->privatekey; + else + return NULL; +} + +X509 *SSL_CTX_get0_certificate(const SSL_CTX *ctx) +{ + if (ctx->cert != NULL) + return ctx->cert->key->x509; + else + return NULL; +} + +EVP_PKEY *SSL_CTX_get0_privatekey(const SSL_CTX *ctx) +{ + if (ctx->cert != NULL) + return ctx->cert->key->privatekey; + else + return NULL; +} + +const SSL_CIPHER *SSL_get_current_cipher(const SSL *s) +{ + if ((s->session != NULL) && (s->session->cipher != NULL)) + return s->session->cipher; + return NULL; +} + +const SSL_CIPHER *SSL_get_pending_cipher(const SSL *s) +{ + return s->s3.tmp.new_cipher; +} + +const COMP_METHOD *SSL_get_current_compression(const SSL *s) +{ +#ifndef OPENSSL_NO_COMP + return s->compress ? COMP_CTX_get_method(s->compress) : NULL; +#else + return NULL; +#endif +} + +const COMP_METHOD *SSL_get_current_expansion(const SSL *s) +{ +#ifndef OPENSSL_NO_COMP + return s->expand ? COMP_CTX_get_method(s->expand) : NULL; +#else + return NULL; +#endif +} + +int ssl_init_wbio_buffer(SSL *s) +{ + BIO *bbio; + + if (s->bbio != NULL) { + /* Already buffered. */ + return 1; + } + + bbio = BIO_new(BIO_f_buffer()); + if (bbio == NULL || !BIO_set_read_buffer_size(bbio, 1)) { + BIO_free(bbio); + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + return 0; + } + s->bbio = bbio; + s->wbio = BIO_push(bbio, s->wbio); + + return 1; +} + +int ssl_free_wbio_buffer(SSL *s) +{ + /* callers ensure s is never null */ if (s->bbio == NULL) return 1; s->wbio = BIO_pop(s->wbio); - s->rlayer.wrlmethod->set1_bio(s->rlayer.wrl, s->wbio); - BIO_free(s->bbio); s->bbio = NULL; @@ -5140,81 +4823,32 @@ int SSL_CTX_get_quiet_shutdown(const SSL_CTX *ctx) void SSL_set_quiet_shutdown(SSL *s, int mode) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - /* Not supported with QUIC */ - if (sc == NULL) - return; - - sc->quiet_shutdown = mode; + s->quiet_shutdown = mode; } int SSL_get_quiet_shutdown(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL_ONLY(s); - - /* Not supported with QUIC */ - if (sc == NULL) - return 0; - - return sc->quiet_shutdown; + return s->quiet_shutdown; } void SSL_set_shutdown(SSL *s, int mode) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - /* Not supported with QUIC */ - if (sc == NULL) - return; - - sc->shutdown = mode; + s->shutdown = mode; } int SSL_get_shutdown(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL_ONLY(s); - -#ifndef OPENSSL_NO_QUIC - /* QUIC: Just indicate whether the connection was shutdown cleanly. */ - if (IS_QUIC(s)) - return ossl_quic_get_shutdown(s); -#endif - - if (sc == NULL) - return 0; - - return sc->shutdown; + return s->shutdown; } int SSL_version(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - -#ifndef OPENSSL_NO_QUIC - /* We only support QUICv1 - so if its QUIC its QUICv1 */ - if (s->type == SSL_TYPE_QUIC_CONNECTION || s->type == SSL_TYPE_QUIC_XSO) - return OSSL_QUIC1_VERSION; -#endif - if (sc == NULL) - return 0; - - return sc->version; + return s->version; } int SSL_client_version(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - -#ifndef OPENSSL_NO_QUIC - /* We only support QUICv1 - so if its QUIC its QUICv1 */ - if (s->type == SSL_TYPE_QUIC_CONNECTION || s->type == SSL_TYPE_QUIC_XSO) - return OSSL_QUIC1_VERSION; -#endif - if (sc == NULL) - return 0; - - return sc->client_version; + return s->client_version; } SSL_CTX *SSL_get_SSL_CTX(const SSL *ssl) @@ -5225,34 +4859,28 @@ SSL_CTX *SSL_get_SSL_CTX(const SSL *ssl) SSL_CTX *SSL_set_SSL_CTX(SSL *ssl, SSL_CTX *ctx) { CERT *new_cert; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - /* TODO(QUIC FUTURE): Add support for QUIC */ - if (sc == NULL) - return NULL; - if (ssl->ctx == ctx) return ssl->ctx; if (ctx == NULL) - ctx = sc->session_ctx; + ctx = ssl->session_ctx; new_cert = ssl_cert_dup(ctx->cert); if (new_cert == NULL) { return NULL; } - if (!custom_exts_copy_flags(&new_cert->custext, &sc->cert->custext)) { + if (!custom_exts_copy_flags(&new_cert->custext, &ssl->cert->custext)) { ssl_cert_free(new_cert); return NULL; } - ssl_cert_free(sc->cert); - sc->cert = new_cert; + ssl_cert_free(ssl->cert); + ssl->cert = new_cert; /* * Program invariant: |sid_ctx| has fixed size (SSL_MAX_SID_CTX_LENGTH), * so setter APIs must prevent invalid lengths from entering the system. */ - if (!ossl_assert(sc->sid_ctx_length <= sizeof(sc->sid_ctx))) + if (!ossl_assert(ssl->sid_ctx_length <= sizeof(ssl->sid_ctx))) return NULL; /* @@ -5262,10 +4890,10 @@ SSL_CTX *SSL_set_SSL_CTX(SSL *ssl, SSL_CTX *ctx) * leave it unchanged. */ if ((ssl->ctx != NULL) && - (sc->sid_ctx_length == ssl->ctx->sid_ctx_length) && - (memcmp(sc->sid_ctx, ssl->ctx->sid_ctx, sc->sid_ctx_length) == 0)) { - sc->sid_ctx_length = ctx->sid_ctx_length; - memcpy(&sc->sid_ctx, &ctx->sid_ctx, sizeof(sc->sid_ctx)); + (ssl->sid_ctx_length == ssl->ctx->sid_ctx_length) && + (memcmp(ssl->sid_ctx, ssl->ctx->sid_ctx, ssl->sid_ctx_length) == 0)) { + ssl->sid_ctx_length = ctx->sid_ctx_length; + memcpy(&ssl->sid_ctx, &ctx->sid_ctx, sizeof(ssl->sid_ctx)); } SSL_CTX_up_ref(ctx); @@ -5275,6 +4903,20 @@ SSL_CTX *SSL_set_SSL_CTX(SSL *ssl, SSL_CTX *ctx) return ssl->ctx; } +SSL_CTX *BABASSL_set_SESSION_CTX(SSL *ssl, SSL_CTX *ctx) +{ + if (ssl->session_ctx == ctx) + return ssl->session_ctx; + if (ctx == NULL) + ctx = ssl->session_ctx; + + SSL_CTX_up_ref(ctx); + SSL_CTX_free(ssl->session_ctx); /* decrement reference count */ + ssl->session_ctx = ctx; + + return ssl->session_ctx; +} + int SSL_CTX_set_default_verify_paths(SSL_CTX *ctx) { return X509_STORE_set_default_paths_ex(ctx->cert_store, ctx->libctx, @@ -5368,12 +5010,7 @@ int SSL_CTX_load_verify_locations(SSL_CTX *ctx, const char *CAfile, void SSL_set_info_callback(SSL *ssl, void (*cb) (const SSL *ssl, int type, int val)) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return; - - sc->info_callback = cb; + ssl->info_callback = cb; } /* @@ -5383,61 +5020,36 @@ void SSL_set_info_callback(SSL *ssl, void (*SSL_get_info_callback(const SSL *ssl)) (const SSL * /* ssl */ , int /* type */ , int /* val */ ) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(ssl); - - if (sc == NULL) - return NULL; - - return sc->info_callback; + return ssl->info_callback; } void SSL_set_verify_result(SSL *ssl, long arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return; - - sc->verify_result = arg; + ssl->verify_result = arg; } long SSL_get_verify_result(const SSL *ssl) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(ssl); - - if (sc == NULL) - return 0; - - return sc->verify_result; + return ssl->verify_result; } size_t SSL_get_client_random(const SSL *ssl, unsigned char *out, size_t outlen) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(ssl); - - if (sc == NULL) - return 0; - if (outlen == 0) - return sizeof(sc->s3.client_random); - if (outlen > sizeof(sc->s3.client_random)) - outlen = sizeof(sc->s3.client_random); - memcpy(out, sc->s3.client_random, outlen); + return sizeof(ssl->s3.client_random); + if (outlen > sizeof(ssl->s3.client_random)) + outlen = sizeof(ssl->s3.client_random); + memcpy(out, ssl->s3.client_random, outlen); return outlen; } size_t SSL_get_server_random(const SSL *ssl, unsigned char *out, size_t outlen) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(ssl); - - if (sc == NULL) - return 0; - if (outlen == 0) - return sizeof(sc->s3.server_random); - if (outlen > sizeof(sc->s3.server_random)) - outlen = sizeof(sc->s3.server_random); - memcpy(out, sc->s3.server_random, outlen); + return sizeof(ssl->s3.server_random); + if (outlen > sizeof(ssl->s3.server_random)) + outlen = sizeof(ssl->s3.server_random); + memcpy(out, ssl->s3.server_random, outlen); return outlen; } @@ -5504,18 +5116,8 @@ void SSL_CTX_set1_cert_store(SSL_CTX *ctx, X509_STORE *store) int SSL_want(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_want(s); -#endif - - if (sc == NULL) - return SSL_NOTHING; - - return sc->rwstate; -} + return s->rwstate; +} #ifndef OPENSSL_NO_PSK int SSL_CTX_use_psk_identity_hint(SSL_CTX *ctx, const char *identity_hint) @@ -5536,53 +5138,40 @@ int SSL_CTX_use_psk_identity_hint(SSL_CTX *ctx, const char *identity_hint) int SSL_use_psk_identity_hint(SSL *s, const char *identity_hint) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) + if (s == NULL) return 0; if (identity_hint != NULL && strlen(identity_hint) > PSK_MAX_IDENTITY_LEN) { ERR_raise(ERR_LIB_SSL, SSL_R_DATA_LENGTH_TOO_LONG); return 0; } - OPENSSL_free(sc->cert->psk_identity_hint); + OPENSSL_free(s->cert->psk_identity_hint); if (identity_hint != NULL) { - sc->cert->psk_identity_hint = OPENSSL_strdup(identity_hint); - if (sc->cert->psk_identity_hint == NULL) + s->cert->psk_identity_hint = OPENSSL_strdup(identity_hint); + if (s->cert->psk_identity_hint == NULL) return 0; } else - sc->cert->psk_identity_hint = NULL; + s->cert->psk_identity_hint = NULL; return 1; } const char *SSL_get_psk_identity_hint(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL || sc->session == NULL) + if (s == NULL || s->session == NULL) return NULL; - - return sc->session->psk_identity_hint; + return s->session->psk_identity_hint; } const char *SSL_get_psk_identity(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL || sc->session == NULL) + if (s == NULL || s->session == NULL) return NULL; - - return sc->session->psk_identity; + return s->session->psk_identity; } void SSL_set_psk_client_callback(SSL *s, SSL_psk_client_cb_func cb) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->psk_client_callback = cb; + s->psk_client_callback = cb; } void SSL_CTX_set_psk_client_callback(SSL_CTX *ctx, SSL_psk_client_cb_func cb) @@ -5592,12 +5181,7 @@ void SSL_CTX_set_psk_client_callback(SSL_CTX *ctx, SSL_psk_client_cb_func cb) void SSL_set_psk_server_callback(SSL *s, SSL_psk_server_cb_func cb) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->psk_server_callback = cb; + s->psk_server_callback = cb; } void SSL_CTX_set_psk_server_callback(SSL_CTX *ctx, SSL_psk_server_cb_func cb) @@ -5608,12 +5192,7 @@ void SSL_CTX_set_psk_server_callback(SSL_CTX *ctx, SSL_psk_server_cb_func cb) void SSL_set_psk_find_session_callback(SSL *s, SSL_psk_find_session_cb_func cb) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->psk_find_session_cb = cb; + s->psk_find_session_cb = cb; } void SSL_CTX_set_psk_find_session_callback(SSL_CTX *ctx, @@ -5624,12 +5203,7 @@ void SSL_CTX_set_psk_find_session_callback(SSL_CTX *ctx, void SSL_set_psk_use_session_callback(SSL *s, SSL_psk_use_session_cb_func cb) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->psk_use_session_cb = cb; + s->psk_use_session_cb = cb; } void SSL_CTX_set_psk_use_session_callback(SSL_CTX *ctx, @@ -5690,9 +5264,6 @@ void *SSL_CTX_get_record_padding_callback_arg(const SSL_CTX *ctx) int SSL_CTX_set_block_padding(SSL_CTX *ctx, size_t block_size) { - if (IS_QUIC_CTX(ctx) && block_size > 1) - return 0; - /* block size of 0 or 1 is basically no padding */ if (block_size == 1) ctx->block_padding = 0; @@ -5708,14 +5279,10 @@ int SSL_set_record_padding_callback(SSL *ssl, size_t len, void *arg)) { BIO *b; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - if (sc == NULL) - return 0; b = SSL_get_wbio(ssl); if (b == NULL || !BIO_get_ktls_send(b)) { - sc->rlayer.record_padding_cb = cb; + ssl->record_padding_cb = cb; return 1; } return 0; @@ -5723,36 +5290,21 @@ int SSL_set_record_padding_callback(SSL *ssl, void SSL_set_record_padding_callback_arg(SSL *ssl, void *arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return; - - sc->rlayer.record_padding_arg = arg; + ssl->record_padding_arg = arg; } void *SSL_get_record_padding_callback_arg(const SSL *ssl) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(ssl); - - if (sc == NULL) - return NULL; - - return sc->rlayer.record_padding_arg; + return ssl->record_padding_arg; } int SSL_set_block_padding(SSL *ssl, size_t block_size) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL || (IS_QUIC(ssl) && block_size > 1)) - return 0; - /* block size of 0 or 1 is basically no padding */ if (block_size == 1) - sc->rlayer.block_padding = 0; + ssl->block_padding = 0; else if (block_size <= SSL3_RT_MAX_PLAIN_LENGTH) - sc->rlayer.block_padding = block_size; + ssl->block_padding = block_size; else return 0; return 1; @@ -5760,24 +5312,14 @@ int SSL_set_block_padding(SSL *ssl, size_t block_size) int SSL_set_num_tickets(SSL *s, size_t num_tickets) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - sc->num_tickets = num_tickets; + s->num_tickets = num_tickets; return 1; } size_t SSL_get_num_tickets(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->num_tickets; + return s->num_tickets; } int SSL_CTX_set_num_tickets(SSL_CTX *ctx, size_t num_tickets) @@ -5792,9 +5334,34 @@ size_t SSL_CTX_get_num_tickets(const SSL_CTX *ctx) return ctx->num_tickets; } +/* + * Allocates new EVP_MD_CTX and sets pointer to it into given pointer + * variable, freeing EVP_MD_CTX previously stored in that variable, if any. + * If EVP_MD pointer is passed, initializes ctx with this |md|. + * Returns the newly allocated ctx; + */ + +EVP_MD_CTX *ssl_replace_hash(EVP_MD_CTX **hash, const EVP_MD *md) +{ + ssl_clear_hash_ctx(hash); + *hash = EVP_MD_CTX_new(); + if (*hash == NULL || (md && EVP_DigestInit_ex(*hash, md, NULL) <= 0)) { + EVP_MD_CTX_free(*hash); + *hash = NULL; + return NULL; + } + return *hash; +} + +void ssl_clear_hash_ctx(EVP_MD_CTX **hash) +{ + + EVP_MD_CTX_free(*hash); + *hash = NULL; +} + /* Retrieve handshake hashes */ -int ssl_handshake_hash(SSL_CONNECTION *s, - unsigned char *out, size_t outlen, +int ssl_handshake_hash(SSL *s, unsigned char *out, size_t outlen, size_t *hashlen) { EVP_MD_CTX *ctx = NULL; @@ -5829,22 +5396,12 @@ int ssl_handshake_hash(SSL_CONNECTION *s, int SSL_session_reused(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->hit; + return s->hit; } int SSL_is_server(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->server; + return s->server; } #ifndef OPENSSL_NO_DEPRECATED_1_1_0 @@ -5858,22 +5415,12 @@ void SSL_set_debug(SSL *s, int debug) void SSL_set_security_level(SSL *s, int level) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->cert->sec_level = level; + s->cert->sec_level = level; } int SSL_get_security_level(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->cert->sec_level; + return s->cert->sec_level; } void SSL_set_security_callback(SSL *s, @@ -5881,44 +5428,24 @@ void SSL_set_security_callback(SSL *s, int op, int bits, int nid, void *other, void *ex)) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->cert->sec_cb = cb; + s->cert->sec_cb = cb; } int (*SSL_get_security_callback(const SSL *s)) (const SSL *s, const SSL_CTX *ctx, int op, int bits, int nid, void *other, void *ex) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - - return sc->cert->sec_cb; + return s->cert->sec_cb; } void SSL_set0_security_ex_data(SSL *s, void *ex) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - sc->cert->sec_ex = ex; + s->cert->sec_ex = ex; } void *SSL_get0_security_ex_data(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - - return sc->cert->sec_ex; + return s->cert->sec_ex; } void SSL_CTX_set_security_level(SSL_CTX *ctx, int level) @@ -5965,17 +5492,7 @@ uint64_t SSL_CTX_get_options(const SSL_CTX *ctx) uint64_t SSL_get_options(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_get_options(s); -#endif - - if (sc == NULL) - return 0; - - return sc->options; + return s->options; } uint64_t SSL_CTX_set_options(SSL_CTX *ctx, uint64_t op) @@ -5985,29 +5502,7 @@ uint64_t SSL_CTX_set_options(SSL_CTX *ctx, uint64_t op) uint64_t SSL_set_options(SSL *s, uint64_t op) { - SSL_CONNECTION *sc; - OSSL_PARAM options[2], *opts = options; - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_set_options(s, op); -#endif - - sc = SSL_CONNECTION_FROM_SSL(s); - if (sc == NULL) - return 0; - - sc->options |= op; - - *opts++ = OSSL_PARAM_construct_uint64(OSSL_LIBSSL_RECORD_LAYER_PARAM_OPTIONS, - &sc->options); - *opts = OSSL_PARAM_construct_end(); - - /* Ignore return value */ - sc->rlayer.rrlmethod->set_options(sc->rlayer.rrl, options); - sc->rlayer.wrlmethod->set_options(sc->rlayer.wrl, options); - - return sc->options; + return s->options |= op; } uint64_t SSL_CTX_clear_options(SSL_CTX *ctx, uint64_t op) @@ -6017,38 +5512,12 @@ uint64_t SSL_CTX_clear_options(SSL_CTX *ctx, uint64_t op) uint64_t SSL_clear_options(SSL *s, uint64_t op) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - OSSL_PARAM options[2], *opts = options; - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_clear_options(s, op); -#endif - - if (sc == NULL) - return 0; - - sc->options &= ~op; - - *opts++ = OSSL_PARAM_construct_uint64(OSSL_LIBSSL_RECORD_LAYER_PARAM_OPTIONS, - &sc->options); - *opts = OSSL_PARAM_construct_end(); - - /* Ignore return value */ - sc->rlayer.rrlmethod->set_options(sc->rlayer.rrl, options); - sc->rlayer.wrlmethod->set_options(sc->rlayer.wrl, options); - - return sc->options; + return s->options &= ~op; } STACK_OF(X509) *SSL_get0_verified_chain(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return NULL; - - return sc->verified_chain; + return s->verified_chain; } IMPLEMENT_OBJ_BSEARCH_GLOBAL_CMP_FN(SSL_CIPHER, SSL_CIPHER, ssl_cipher_id); @@ -6061,8 +5530,6 @@ IMPLEMENT_OBJ_BSEARCH_GLOBAL_CMP_FN(SSL_CIPHER, SSL_CIPHER, ssl_cipher_id); * If |dst| points to a NULL pointer, a new stack will be created and owned by * the caller. * Returns the number of SCTs moved, or a negative integer if an error occurs. - * The |dst| stack is created and possibly partially populated even in case - * of error, likewise the |src| stack may be left in an intermediate state. */ static int ct_move_scts(STACK_OF(SCT) **dst, STACK_OF(SCT) *src, sct_source_t origin) @@ -6073,7 +5540,7 @@ static int ct_move_scts(STACK_OF(SCT) **dst, STACK_OF(SCT) *src, if (*dst == NULL) { *dst = sk_SCT_new_null(); if (*dst == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } } @@ -6082,14 +5549,15 @@ static int ct_move_scts(STACK_OF(SCT) **dst, STACK_OF(SCT) *src, if (SCT_set_source(sct, origin) != 1) goto err; - if (!sk_SCT_push(*dst, sct)) + if (sk_SCT_push(*dst, sct) <= 0) goto err; scts_moved += 1; } return scts_moved; err: - SCT_free(sct); + if (sct != NULL) + sk_SCT_push(src, sct); /* Put the SCT back */ return -1; } @@ -6097,7 +5565,7 @@ static int ct_move_scts(STACK_OF(SCT) **dst, STACK_OF(SCT) *src, * Look for data collected during ServerHello and parse if found. * Returns the number of SCTs extracted. */ -static int ct_extract_tls_extension_scts(SSL_CONNECTION *s) +static int ct_extract_tls_extension_scts(SSL *s) { int scts_extracted = 0; @@ -6121,7 +5589,7 @@ static int ct_extract_tls_extension_scts(SSL_CONNECTION *s) * - 0 if no OCSP response exists or it contains no SCTs. * - A negative integer if an error occurs. */ -static int ct_extract_ocsp_response_scts(SSL_CONNECTION *s) +static int ct_extract_ocsp_response_scts(SSL *s) { # ifndef OPENSSL_NO_OCSP int scts_extracted = 0; @@ -6172,7 +5640,7 @@ static int ct_extract_ocsp_response_scts(SSL_CONNECTION *s) * Return the number of SCTs extracted, or a negative integer if an error * occurs. */ -static int ct_extract_x509v3_extension_scts(SSL_CONNECTION *s) +static int ct_extract_x509v3_extension_scts(SSL *s) { int scts_extracted = 0; X509 *cert = s->session != NULL ? s->session->peer : NULL; @@ -6197,31 +5665,26 @@ static int ct_extract_x509v3_extension_scts(SSL_CONNECTION *s) */ const STACK_OF(SCT) *SSL_get0_peer_scts(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - if (!sc->scts_parsed) { - if (ct_extract_tls_extension_scts(sc) < 0 || - ct_extract_ocsp_response_scts(sc) < 0 || - ct_extract_x509v3_extension_scts(sc) < 0) + if (!s->scts_parsed) { + if (ct_extract_tls_extension_scts(s) < 0 || + ct_extract_ocsp_response_scts(s) < 0 || + ct_extract_x509v3_extension_scts(s) < 0) goto err; - sc->scts_parsed = 1; + s->scts_parsed = 1; } - return sc->scts; + return s->scts; err: return NULL; } -static int ct_permissive(const CT_POLICY_EVAL_CTX *ctx, +static int ct_permissive(const CT_POLICY_EVAL_CTX * ctx, const STACK_OF(SCT) *scts, void *unused_arg) { return 1; } -static int ct_strict(const CT_POLICY_EVAL_CTX *ctx, +static int ct_strict(const CT_POLICY_EVAL_CTX * ctx, const STACK_OF(SCT) *scts, void *unused_arg) { int count = scts != NULL ? sk_SCT_num(scts) : 0; @@ -6241,11 +5704,6 @@ static int ct_strict(const CT_POLICY_EVAL_CTX *ctx, int SSL_set_ct_validation_callback(SSL *s, ssl_ct_validation_cb callback, void *arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - /* * Since code exists that uses the custom extension handler for CT, look * for this and throw an error if they have already registered to use CT. @@ -6265,8 +5723,8 @@ int SSL_set_ct_validation_callback(SSL *s, ssl_ct_validation_cb callback, return 0; } - sc->ct_validation_callback = callback; - sc->ct_validation_callback_arg = arg; + s->ct_validation_callback = callback; + s->ct_validation_callback_arg = arg; return 1; } @@ -6292,12 +5750,7 @@ int SSL_CTX_set_ct_validation_callback(SSL_CTX *ctx, int SSL_ct_is_enabled(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->ct_validation_callback != NULL; + return s->ct_validation_callback != NULL; } int SSL_CTX_ct_is_enabled(const SSL_CTX *ctx) @@ -6305,7 +5758,7 @@ int SSL_CTX_ct_is_enabled(const SSL_CTX *ctx) return ctx->ct_validation_callback != NULL; } -int ssl_validate_ct(SSL_CONNECTION *s) +int ssl_validate_ct(SSL *s) { int ret = 0; X509 *cert = s->session != NULL ? s->session->peer : NULL; @@ -6342,22 +5795,20 @@ int ssl_validate_ct(SSL_CONNECTION *s) } } - ctx = CT_POLICY_EVAL_CTX_new_ex(SSL_CONNECTION_GET_CTX(s)->libctx, - SSL_CONNECTION_GET_CTX(s)->propq); + ctx = CT_POLICY_EVAL_CTX_new_ex(s->ctx->libctx, s->ctx->propq); if (ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CT_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto end; } issuer = sk_X509_value(s->verified_chain, 1); CT_POLICY_EVAL_CTX_set1_cert(ctx, cert); CT_POLICY_EVAL_CTX_set1_issuer(ctx, issuer); - CT_POLICY_EVAL_CTX_set_shared_CTLOG_STORE(ctx, - SSL_CONNECTION_GET_CTX(s)->ctlog_store); + CT_POLICY_EVAL_CTX_set_shared_CTLOG_STORE(ctx, s->ctx->ctlog_store); CT_POLICY_EVAL_CTX_set_time( - ctx, (uint64_t)SSL_SESSION_get_time(s->session) * 1000); + ctx, (uint64_t)SSL_SESSION_get_time(SSL_get0_session(s)) * 1000); - scts = SSL_get0_peer_scts(SSL_CONNECTION_GET_SSL(s)); + scts = SSL_get0_peer_scts(s); /* * This function returns success (> 0) only when all the SCTs are valid, 0 @@ -6442,7 +5893,7 @@ int SSL_CTX_set_ctlog_list_file(SSL_CTX *ctx, const char *path) return CTLOG_STORE_load_file(ctx->ctlog_store, path); } -void SSL_CTX_set0_ctlog_store(SSL_CTX *ctx, CTLOG_STORE *logs) +void SSL_CTX_set0_ctlog_store(SSL_CTX *ctx, CTLOG_STORE * logs) { CTLOG_STORE_free(ctx->ctlog_store); ctx->ctlog_store = logs; @@ -6464,82 +5915,52 @@ void SSL_CTX_set_client_hello_cb(SSL_CTX *c, SSL_client_hello_cb_fn cb, int SSL_client_hello_isv2(SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->clienthello == NULL) + if (s->clienthello == NULL) return 0; - return sc->clienthello->isv2; + return s->clienthello->isv2; } unsigned int SSL_client_hello_get0_legacy_version(SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->clienthello == NULL) + if (s->clienthello == NULL) return 0; - return sc->clienthello->legacy_version; + return s->clienthello->legacy_version; } size_t SSL_client_hello_get0_random(SSL *s, const unsigned char **out) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->clienthello == NULL) + if (s->clienthello == NULL) return 0; if (out != NULL) - *out = sc->clienthello->random; + *out = s->clienthello->random; return SSL3_RANDOM_SIZE; } size_t SSL_client_hello_get0_session_id(SSL *s, const unsigned char **out) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->clienthello == NULL) + if (s->clienthello == NULL) return 0; if (out != NULL) - *out = sc->clienthello->session_id; - return sc->clienthello->session_id_len; + *out = s->clienthello->session_id; + return s->clienthello->session_id_len; } size_t SSL_client_hello_get0_ciphers(SSL *s, const unsigned char **out) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->clienthello == NULL) + if (s->clienthello == NULL) return 0; if (out != NULL) - *out = PACKET_data(&sc->clienthello->ciphersuites); - return PACKET_remaining(&sc->clienthello->ciphersuites); + *out = PACKET_data(&s->clienthello->ciphersuites); + return PACKET_remaining(&s->clienthello->ciphersuites); } size_t SSL_client_hello_get0_compression_methods(SSL *s, const unsigned char **out) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->clienthello == NULL) + if (s->clienthello == NULL) return 0; if (out != NULL) - *out = sc->clienthello->compressions; - return sc->clienthello->compressions_len; + *out = s->clienthello->compressions; + return s->clienthello->compressions_len; } int SSL_client_hello_get1_extensions_present(SSL *s, int **out, size_t *outlen) @@ -6547,15 +5968,11 @@ int SSL_client_hello_get1_extensions_present(SSL *s, int **out, size_t *outlen) RAW_EXTENSION *ext; int *present; size_t num = 0, i; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - if (sc == NULL) + if (s->clienthello == NULL || out == NULL || outlen == NULL) return 0; - - if (sc->clienthello == NULL || out == NULL || outlen == NULL) - return 0; - for (i = 0; i < sc->clienthello->pre_proc_exts_len; i++) { - ext = sc->clienthello->pre_proc_exts + i; + for (i = 0; i < s->clienthello->pre_proc_exts_len; i++) { + ext = s->clienthello->pre_proc_exts + i; if (ext->present) num++; } @@ -6564,10 +5981,12 @@ int SSL_client_hello_get1_extensions_present(SSL *s, int **out, size_t *outlen) *outlen = 0; return 1; } - if ((present = OPENSSL_malloc(sizeof(*present) * num)) == NULL) + if ((present = OPENSSL_malloc(sizeof(*present) * num)) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; - for (i = 0; i < sc->clienthello->pre_proc_exts_len; i++) { - ext = sc->clienthello->pre_proc_exts + i; + } + for (i = 0; i < s->clienthello->pre_proc_exts_len; i++) { + ext = s->clienthello->pre_proc_exts + i; if (ext->present) { if (ext->received_order >= num) goto err; @@ -6582,42 +6001,49 @@ int SSL_client_hello_get1_extensions_present(SSL *s, int **out, size_t *outlen) return 0; } -int SSL_client_hello_get_extension_order(SSL *s, uint16_t *exts, size_t *num_exts) +int BABASSL_client_hello_get1_extensions(SSL *s, int **out, size_t *outlen) { - RAW_EXTENSION *ext; - size_t num = 0, i; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); + int *exts, i = 0; + size_t num = 0; - if (sc == NULL) - return 0; + PACKET extensions = s->clienthello->extensions; - if (sc->clienthello == NULL || num_exts == NULL) - return 0; - for (i = 0; i < sc->clienthello->pre_proc_exts_len; i++) { - ext = sc->clienthello->pre_proc_exts + i; - if (ext->present) - num++; - } - if (num == 0) { - *num_exts = 0; - return 1; - } - if (exts == NULL) { - *num_exts = num; - return 1; + while (PACKET_remaining(&extensions) > 0) { + unsigned int type; + PACKET extension; + + if (!PACKET_get_net_2(&extensions, &type) || + !PACKET_get_length_prefixed_2(&extensions, &extension)) { + return 0; + } + + num++; } - if (*num_exts < num) + + exts = OPENSSL_malloc(sizeof(*exts) * num); + if (exts == NULL) return 0; - for (i = 0; i < sc->clienthello->pre_proc_exts_len; i++) { - ext = sc->clienthello->pre_proc_exts + i; - if (ext->present) { - if (ext->received_order >= num) - return 0; - exts[ext->received_order] = ext->type; + + extensions = s->clienthello->extensions; + + while (PACKET_remaining(&extensions) > 0) { + unsigned int type; + PACKET extension; + + if (!PACKET_get_net_2(&extensions, &type) || + !PACKET_get_length_prefixed_2(&extensions, &extension)) { + goto err; } + + exts[i++] = type; } - *num_exts = num; + + *out = exts; + *outlen = num; return 1; + err: + OPENSSL_free(exts); + return 0; } int SSL_client_hello_get0_ext(SSL *s, unsigned int type, const unsigned char **out, @@ -6625,15 +6051,11 @@ int SSL_client_hello_get0_ext(SSL *s, unsigned int type, const unsigned char **o { size_t i; RAW_EXTENSION *r; - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - if (sc->clienthello == NULL) + if (s->clienthello == NULL) return 0; - for (i = 0; i < sc->clienthello->pre_proc_exts_len; ++i) { - r = sc->clienthello->pre_proc_exts + i; + for (i = 0; i < s->clienthello->pre_proc_exts_len; ++i) { + r = s->clienthello->pre_proc_exts + i; if (r->present && r->type == type) { if (out != NULL) *out = PACKET_data(&r->data); @@ -6647,34 +6069,18 @@ int SSL_client_hello_get0_ext(SSL *s, unsigned int type, const unsigned char **o int SSL_free_buffers(SSL *ssl) { - RECORD_LAYER *rl; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(ssl); + RECORD_LAYER *rl = &ssl->rlayer; - if (sc == NULL) + if (RECORD_LAYER_read_pending(rl) || RECORD_LAYER_write_pending(rl)) return 0; - rl = &sc->rlayer; - - return rl->rrlmethod->free_buffers(rl->rrl) - && rl->wrlmethod->free_buffers(rl->wrl); + RECORD_LAYER_release(rl); + return 1; } int SSL_alloc_buffers(SSL *ssl) { - RECORD_LAYER *rl; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - - /* QUIC always has buffers allocated. */ - if (IS_QUIC(ssl)) - return 1; - - rl = &sc->rlayer; - - return rl->rrlmethod->alloc_buffers(rl->rrl) - && rl->wrlmethod->alloc_buffers(rl->wrl); + return ssl3_setup_buffers(ssl); } void SSL_CTX_set_keylog_callback(SSL_CTX *ctx, SSL_CTX_keylog_cb_func cb) @@ -6688,7 +6094,7 @@ SSL_CTX_keylog_cb_func SSL_CTX_get_keylog_callback(const SSL_CTX *ctx) } static int nss_keylog_int(const char *prefix, - SSL_CONNECTION *sc, + SSL *ssl, const uint8_t *parameter_1, size_t parameter_1_len, const uint8_t *parameter_2, @@ -6699,9 +6105,8 @@ static int nss_keylog_int(const char *prefix, size_t out_len = 0; size_t i; size_t prefix_len; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(sc); - if (sctx->keylog_callback == NULL) + if (ssl->ctx->keylog_callback == NULL) return 1; /* @@ -6714,8 +6119,10 @@ static int nss_keylog_int(const char *prefix, */ prefix_len = strlen(prefix); out_len = prefix_len + (2 * parameter_1_len) + (2 * parameter_2_len) + 3; - if ((out = cursor = OPENSSL_malloc(out_len)) == NULL) + if ((out = cursor = OPENSSL_malloc(out_len)) == NULL) { + SSLfatal(ssl, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; + } strcpy(cursor, prefix); cursor += prefix_len; @@ -6733,40 +6140,40 @@ static int nss_keylog_int(const char *prefix, } *cursor = '\0'; - sctx->keylog_callback(SSL_CONNECTION_GET_SSL(sc), (const char *)out); + ssl->ctx->keylog_callback(ssl, (const char *)out); OPENSSL_clear_free(out, out_len); return 1; } -int ssl_log_rsa_client_key_exchange(SSL_CONNECTION *sc, +int ssl_log_rsa_client_key_exchange(SSL *ssl, const uint8_t *encrypted_premaster, size_t encrypted_premaster_len, const uint8_t *premaster, size_t premaster_len) { if (encrypted_premaster_len < 8) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(ssl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } /* We only want the first 8 bytes of the encrypted premaster as a tag. */ return nss_keylog_int("RSA", - sc, + ssl, encrypted_premaster, 8, premaster, premaster_len); } -int ssl_log_secret(SSL_CONNECTION *sc, +int ssl_log_secret(SSL *ssl, const char *label, const uint8_t *secret, size_t secret_len) { return nss_keylog_int(label, - sc, - sc->s3.client_random, + ssl, + ssl->s3.client_random, SSL3_RANDOM_SIZE, secret, secret_len); @@ -6774,7 +6181,7 @@ int ssl_log_secret(SSL_CONNECTION *sc, #define SSLV2_CIPHER_LEN 3 -int ssl_cache_cipherlist(SSL_CONNECTION *s, PACKET *cipher_suites, int sslv2format) +int ssl_cache_cipherlist(SSL *s, PACKET *cipher_suites, int sslv2format) { int n; @@ -6810,7 +6217,7 @@ int ssl_cache_cipherlist(SSL_CONNECTION *s, PACKET *cipher_suites, int sslv2form raw = OPENSSL_malloc(numciphers * TLS_CIPHER_LEN); s->s3.tmp.ciphers_raw = raw; if (raw == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } for (s->s3.tmp.ciphers_rawlen = 0; @@ -6844,20 +6251,16 @@ int SSL_bytes_to_cipher_list(SSL *s, const unsigned char *bytes, size_t len, STACK_OF(SSL_CIPHER) **scsvs) { PACKET pkt; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; if (!PACKET_buf_init(&pkt, bytes, len)) return 0; - return ossl_bytes_to_cipher_list(sc, &pkt, sk, scsvs, isv2format, 0); + return bytes_to_cipher_list(s, &pkt, sk, scsvs, isv2format, 0); } -int ossl_bytes_to_cipher_list(SSL_CONNECTION *s, PACKET *cipher_suites, - STACK_OF(SSL_CIPHER) **skp, - STACK_OF(SSL_CIPHER) **scsvs_out, - int sslv2format, int fatal) +int bytes_to_cipher_list(SSL *s, PACKET *cipher_suites, + STACK_OF(SSL_CIPHER) **skp, + STACK_OF(SSL_CIPHER) **scsvs_out, + int sslv2format, int fatal) { const SSL_CIPHER *c; STACK_OF(SSL_CIPHER) *sk = NULL; @@ -6889,9 +6292,9 @@ int ossl_bytes_to_cipher_list(SSL_CONNECTION *s, PACKET *cipher_suites, scsvs = sk_SSL_CIPHER_new_null(); if (sk == NULL || scsvs == NULL) { if (fatal) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); else - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } @@ -6910,9 +6313,9 @@ int ossl_bytes_to_cipher_list(SSL_CONNECTION *s, PACKET *cipher_suites, if ((c->valid && !sk_SSL_CIPHER_push(sk, c)) || (!c->valid && !sk_SSL_CIPHER_push(scsvs, c))) { if (fatal) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); else - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } } @@ -6954,24 +6357,14 @@ uint32_t SSL_CTX_get_max_early_data(const SSL_CTX *ctx) int SSL_set_max_early_data(SSL *s, uint32_t max_early_data) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - - sc->max_early_data = max_early_data; + s->max_early_data = max_early_data; return 1; } uint32_t SSL_get_max_early_data(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->max_early_data; + return s->max_early_data; } int SSL_CTX_set_recv_max_early_data(SSL_CTX *ctx, uint32_t recv_max_early_data) @@ -6988,58 +6381,44 @@ uint32_t SSL_CTX_get_recv_max_early_data(const SSL_CTX *ctx) int SSL_set_recv_max_early_data(SSL *s, uint32_t recv_max_early_data) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; - - sc->recv_max_early_data = recv_max_early_data; + s->recv_max_early_data = recv_max_early_data; return 1; } uint32_t SSL_get_recv_max_early_data(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->recv_max_early_data; + return s->recv_max_early_data; } -__owur unsigned int ssl_get_max_send_fragment(const SSL_CONNECTION *sc) +__owur unsigned int ssl_get_max_send_fragment(const SSL *ssl) { /* Return any active Max Fragment Len extension */ - if (sc->session != NULL && USE_MAX_FRAGMENT_LENGTH_EXT(sc->session)) - return GET_MAX_FRAGMENT_LENGTH(sc->session); + if (ssl->session != NULL && USE_MAX_FRAGMENT_LENGTH_EXT(ssl->session)) + return GET_MAX_FRAGMENT_LENGTH(ssl->session); /* return current SSL connection setting */ - return sc->max_send_fragment; + return ssl->max_send_fragment; } -__owur unsigned int ssl_get_split_send_fragment(const SSL_CONNECTION *sc) +__owur unsigned int ssl_get_split_send_fragment(const SSL *ssl) { /* Return a value regarding an active Max Fragment Len extension */ - if (sc->session != NULL && USE_MAX_FRAGMENT_LENGTH_EXT(sc->session) - && sc->split_send_fragment > GET_MAX_FRAGMENT_LENGTH(sc->session)) - return GET_MAX_FRAGMENT_LENGTH(sc->session); + if (ssl->session != NULL && USE_MAX_FRAGMENT_LENGTH_EXT(ssl->session) + && ssl->split_send_fragment > GET_MAX_FRAGMENT_LENGTH(ssl->session)) + return GET_MAX_FRAGMENT_LENGTH(ssl->session); /* else limit |split_send_fragment| to current |max_send_fragment| */ - if (sc->split_send_fragment > sc->max_send_fragment) - return sc->max_send_fragment; + if (ssl->split_send_fragment > ssl->max_send_fragment) + return ssl->max_send_fragment; /* return current SSL connection setting */ - return sc->split_send_fragment; + return ssl->split_send_fragment; } int SSL_stateless(SSL *s) { int ret; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - - if (sc == NULL) - return 0; /* Ensure there is no state left over from a previous invocation */ if (!SSL_clear(s)) @@ -7047,14 +6426,14 @@ int SSL_stateless(SSL *s) ERR_clear_error(); - sc->s3.flags |= TLS1_FLAGS_STATELESS; + s->s3.flags |= TLS1_FLAGS_STATELESS; ret = SSL_accept(s); - sc->s3.flags &= ~TLS1_FLAGS_STATELESS; + s->s3.flags &= ~TLS1_FLAGS_STATELESS; - if (ret > 0 && sc->ext.cookieok) + if (ret > 0 && s->ext.cookieok) return 1; - if (sc->hello_retry_request == SSL_HRR_PENDING && !ossl_statem_in_error(sc)) + if (s->hello_retry_request == SSL_HRR_PENDING && !ossl_statem_in_error(s)) return 0; return -1; @@ -7067,33 +6446,16 @@ void SSL_CTX_set_post_handshake_auth(SSL_CTX *ctx, int val) void SSL_set_post_handshake_auth(SSL *ssl, int val) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(ssl); - - if (sc == NULL) - return; - - sc->pha_enabled = val; + ssl->pha_enabled = val; } int SSL_verify_client_post_handshake(SSL *ssl) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(ssl)) { + if (!SSL_IS_TLS13(ssl)) { ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_SSL_VERSION); return 0; } -#endif - - if (sc == NULL) - return 0; - - if (!SSL_CONNECTION_IS_TLS13(sc)) { - ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_SSL_VERSION); - return 0; - } - if (!sc->server) { + if (!ssl->server) { ERR_raise(ERR_LIB_SSL, SSL_R_NOT_SERVER); return 0; } @@ -7103,7 +6465,7 @@ int SSL_verify_client_post_handshake(SSL *ssl) return 0; } - switch (sc->post_handshake_auth) { + switch (ssl->post_handshake_auth) { case SSL_PHA_NONE: ERR_raise(ERR_LIB_SSL, SSL_R_EXTENSION_NOT_RECEIVED); return 0; @@ -7121,16 +6483,16 @@ int SSL_verify_client_post_handshake(SSL *ssl) return 0; } - sc->post_handshake_auth = SSL_PHA_REQUEST_PENDING; + ssl->post_handshake_auth = SSL_PHA_REQUEST_PENDING; /* checks verify_mode and algorithm_auth */ - if (!send_certificate_request(sc)) { - sc->post_handshake_auth = SSL_PHA_EXT_RECEIVED; /* restore on error */ + if (!send_certificate_request(ssl)) { + ssl->post_handshake_auth = SSL_PHA_EXT_RECEIVED; /* restore on error */ ERR_raise(ERR_LIB_SSL, SSL_R_INVALID_CONFIG); return 0; } - ossl_statem_set_in_init(sc, 1); + ossl_statem_set_in_init(ssl, 1); return 1; } @@ -7157,15 +6519,51 @@ void SSL_set_allow_early_data_cb(SSL *s, SSL_allow_early_data_cb_fn cb, void *arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s); + s->allow_early_data_cb = cb; + s->allow_early_data_cb_data = arg; +} - if (sc == NULL) - return; +#ifndef OPENSSL_NO_NTLS +void SSL_CTX_enable_ntls(SSL_CTX *ctx) +{ + ctx->enable_ntls = 1; +} + +void SSL_CTX_disable_ntls(SSL_CTX *ctx) +{ + ctx->enable_ntls = 0; +} - sc->allow_early_data_cb = cb; - sc->allow_early_data_cb_data = arg; +void SSL_CTX_enable_force_ntls(SSL_CTX *ctx) +{ + ctx->enable_force_ntls = 1; +} + +void SSL_CTX_disable_force_ntls(SSL_CTX *ctx) +{ + ctx->enable_force_ntls = 0; +} + +void SSL_enable_ntls(SSL *s) +{ + s->enable_ntls = 1; } +void SSL_disable_ntls(SSL *s) +{ + s->enable_ntls = 0; +} + +void SSL_enable_force_ntls(SSL *s) +{ + s->enable_force_ntls = 1; +} + +void SSL_disable_force_ntls(SSL *s) +{ + s->enable_force_ntls = 0; +} +#endif const EVP_CIPHER *ssl_evp_cipher_fetch(OSSL_LIB_CTX *libctx, int nid, const char *properties) @@ -7260,18 +6658,13 @@ void ssl_evp_md_free(const EVP_MD *md) int SSL_set0_tmp_dh_pkey(SSL *s, EVP_PKEY *dhpkey) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (!ssl_security(sc, SSL_SECOP_TMP_DH, + if (!ssl_security(s, SSL_SECOP_TMP_DH, EVP_PKEY_get_security_bits(dhpkey), 0, dhpkey)) { ERR_raise(ERR_LIB_SSL, SSL_R_DH_KEY_TOO_SMALL); return 0; } - EVP_PKEY_free(sc->cert->dh_tmp); - sc->cert->dh_tmp = dhpkey; + EVP_PKEY_free(s->cert->dh_tmp); + s->cert->dh_tmp = dhpkey; return 1; } @@ -7287,549 +6680,355 @@ int SSL_CTX_set0_tmp_dh_pkey(SSL_CTX *ctx, EVP_PKEY *dhpkey) return 1; } -/* QUIC-specific methods which are supported on QUIC connections only. */ -int SSL_handle_events(SSL *s) +int BABASSL_SESSION_get_ref(SSL_SESSION *sess) { - SSL_CONNECTION *sc; - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_handle_events(s); -#endif - - sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - if (sc != NULL && SSL_CONNECTION_IS_DTLS(sc)) - /* - * DTLSv1_handle_timeout returns 0 if the timer wasn't expired yet, - * which we consider a success case. Theoretically DTLSv1_handle_timeout - * can also return 0 if s is NULL or not a DTLS object, but we've - * already ruled out those possibilities above, so this is not possible - * here. Thus the only failure cases are where DTLSv1_handle_timeout - * returns -1. - */ - return DTLSv1_handle_timeout(s) >= 0; - - return 1; + if(sess != NULL) + return sess->references; + else + return 0; } -int SSL_get_event_timeout(SSL *s, struct timeval *tv, int *is_infinite) +/* + * For a cipher return the index corresponding to the certificate type + */ +static int ssl_cipher_get_cert_index(const SSL_CIPHER *c) { - SSL_CONNECTION *sc; + uint32_t alg_a; -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_get_event_timeout(s, tv, is_infinite); -#endif + alg_a = c->algorithm_auth; - sc = SSL_CONNECTION_FROM_SSL_ONLY(s); - if (sc != NULL && SSL_CONNECTION_IS_DTLS(sc) - && DTLSv1_get_timeout(s, tv)) { - *is_infinite = 0; - return 1; - } + if (alg_a & SSL_aECDSA) + return SSL_PKEY_ECC; + else if (alg_a & SSL_aRSA) + return SSL_PKEY_RSA; + else if (alg_a & SSL_aSM2) + return SSL_PKEY_SM2; - tv->tv_sec = 1000000; - tv->tv_usec = 0; - *is_infinite = 1; - return 1; + return -1; } -int SSL_get_rpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *desc) +X509 *BABASSL_get_use_certificate(const SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_get_rpoll_descriptor(s, desc); -#endif - - if (sc == NULL || sc->rbio == NULL) - return 0; - - return BIO_get_rpoll_descriptor(sc->rbio, desc); -} + CERT *c = s->cert; + int idx; -int SSL_get_wpoll_descriptor(SSL *s, BIO_POLL_DESCRIPTOR *desc) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); + if (s->s3.tmp.new_cipher == NULL) + return NULL; -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_get_wpoll_descriptor(s, desc); +#ifdef OPENSSL_SSL_DEBUG_BROKEN_PROTOCOL + /* + * Broken protocol test: return last used certificate: which may mismatch + * the one expected. + */ + if (c->cert_flags & SSL_CERT_FLAG_BROKEN_PROTOCOL) + return c->key ? c->key->x509 : NULL; #endif - if (sc == NULL || sc->wbio == NULL) - return 0; - - return BIO_get_wpoll_descriptor(sc->wbio, desc); -} + if (SSL_IS_TLS13(s)) + return SSL_get_certificate(s); -int SSL_net_read_desired(SSL *s) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return SSL_want_read(s); + idx = ssl_cipher_get_cert_index(s->s3.tmp.new_cipher); + /* This may or may not be an error. */ + if (idx < 0) + return NULL; - return ossl_quic_get_net_read_desired(s); -#else - return SSL_want_read(s); -#endif + return c->pkeys[idx].x509; } -int SSL_net_write_desired(SSL *s) +#ifndef OPENSSL_NO_NTLS +X509 *BABASSL_get_sign_certificate_ntls(const SSL *s) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return SSL_want_write(s); + if (s != NULL && s->s3.tmp.sign_cert != NULL) + return s->s3.tmp.sign_cert->x509; - return ossl_quic_get_net_write_desired(s); -#else - return SSL_want_write(s); -#endif + return NULL; } -int SSL_set_blocking_mode(SSL *s, int blocking) +X509 *BABASSL_get_enc_certificate_ntls(const SSL *s) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return 0; + if (s != NULL && s->s3.tmp.enc_cert != NULL) + return s->s3.tmp.enc_cert->x509; - return ossl_quic_conn_set_blocking_mode(s, blocking); -#else - return 0; -#endif + return NULL; } - -int SSL_get_blocking_mode(SSL *s) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return -1; - - return ossl_quic_conn_get_blocking_mode(s); -#else - return -1; #endif -} -int SSL_set1_initial_peer_addr(SSL *s, const BIO_ADDR *peer_addr) +/* + * BABASSL_get0_alpn_proposed gets the proposed ALPN protocol (if any) from + * |ssl|. On return it sets |*data| to point to |*len| bytes of protocol name + * (not including the leading length-prefix byte). If the server didn't + * respond with a negotiated protocol then |*len| will be zero. + */ +void BABASSL_get0_alpn_proposed(const SSL *ssl, const unsigned char **data, + unsigned *len) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return 0; - - return ossl_quic_conn_set_initial_peer_addr(s, peer_addr); -#else - return 0; -#endif -} + if (ssl == NULL || data == NULL || len == NULL) + return; -int SSL_shutdown_ex(SSL *ssl, uint64_t flags, - const SSL_SHUTDOWN_EX_ARGS *args, - size_t args_len) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(ssl)) - return SSL_shutdown(ssl); + *data = ssl->s3.alpn_proposed; - return ossl_quic_conn_shutdown(ssl, flags, args, args_len); -#else - return SSL_shutdown(ssl); -#endif + if (*data == NULL) + *len = 0; + else + *len = ssl->s3.alpn_proposed_len; } -int SSL_stream_conclude(SSL *ssl, uint64_t flags) +/* TODO: condition opt */ +int BABASSL_get_master_key(SSL *s, unsigned char **master_key, + int *master_key_len) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(ssl)) + if (s == NULL || s->session == NULL || s->statem.in_handshake == 1) return 0; - return ossl_quic_conn_stream_conclude(ssl); -#else - return 0; -#endif -} + if (master_key != NULL) + *master_key = s->session->master_key; -SSL *SSL_new_stream(SSL *s, uint64_t flags) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return NULL; - - return ossl_quic_conn_stream_new(s, flags); -#else - return NULL; -#endif -} - -SSL *SSL_get0_connection(SSL *s) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return s; - - return ossl_quic_get0_connection(s); -#else - return s; -#endif -} + if (master_key_len != NULL) + *master_key_len = s->session->master_key_length; -int SSL_is_connection(SSL *s) -{ - return SSL_get0_connection(s) == s; + return 1; } -int SSL_get_stream_type(SSL *s) +void BABASSL_CTX_certs_clear(SSL_CTX *ctx) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return SSL_STREAM_TYPE_BIDI; - - return ossl_quic_get_stream_type(s); -#else - return SSL_STREAM_TYPE_BIDI; -#endif + ssl_cert_clear_certs(ctx->cert); } -uint64_t SSL_get_stream_id(SSL *s) +int BABASSL_check_tlsext_status(SSL *s) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return UINT64_MAX; - - return ossl_quic_get_stream_id(s); -#else - return UINT64_MAX; -#endif + if (s->ext.status_type != -1 && s->ctx && s->ctx->ext.status_cb) + return 1; + else + return 0; } -int SSL_is_stream_local(SSL *s) +void BABASSL_debug(SSL *s, unsigned char *str, int len) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return -1; + int i; + if (!str) + return; - return ossl_quic_is_stream_local(s); -#else - return -1; -#endif + for(i = 0; i < len; i++) + printf("%02x", str[i]); + printf("\n"); + printf("%d\n", len); } -int SSL_set_default_stream_mode(SSL *s, uint32_t mode) +int SSL_get_alert_level(SSL *ssl, int *level, int *desc) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) + if (ssl == NULL) return 0; - return ossl_quic_set_default_stream_mode(s, mode); -#else - return 0; -#endif -} + if (level != NULL) + *level = ssl->s3.alert_level; -int SSL_set_incoming_stream_policy(SSL *s, int policy, uint64_t aec) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return 0; + if (desc != NULL) { + if (ssl->s3.alert_level == SSL3_AL_WARNING) + *desc = ssl->s3.warn_alert; + else if (ssl->s3.alert_level == SSL3_AL_FATAL) + *desc = ssl->s3.fatal_alert; + } - return ossl_quic_set_incoming_stream_policy(s, policy, aec); -#else - return 0; -#endif + return 1; } -SSL *SSL_accept_stream(SSL *s, uint64_t flags) +#ifndef OPENSSL_NO_SKIP_SCSV +void SSL_set_skip_scsv(SSL *s, int skip_scsv) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return NULL; - - return ossl_quic_accept_stream(s, flags); -#else - return NULL; -#endif + if (s) + s->skip_scsv = skip_scsv; } - -size_t SSL_get_accept_stream_queue_len(SSL *s) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return 0; - - return ossl_quic_get_accept_stream_queue_len(s); -#else - return 0; #endif -} -int SSL_stream_reset(SSL *s, - const SSL_STREAM_RESET_ARGS *args, - size_t args_len) +#ifndef OPENSSL_NO_DYNAMIC_CIPHERS +int SSL_set_cipher_list2(SSL *s, STACK_OF(SSL_CIPHER) *cipher_list) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) + STACK_OF(SSL_CIPHER) *sk = NULL; + + if (s == NULL) return 0; - return ossl_quic_stream_reset(s, args, args_len); -#else - return 0; -#endif -} + if (cipher_list != NULL) { + sk = sk_SSL_CIPHER_dup(cipher_list); + if (sk == NULL) + return 0; + } -int SSL_get_stream_read_state(SSL *s) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return SSL_STREAM_STATE_NONE; + if (s->cipher_list != NULL) + sk_SSL_CIPHER_free(s->cipher_list); - return ossl_quic_get_stream_read_state(s); -#else - return SSL_STREAM_STATE_NONE; -#endif -} - -int SSL_get_stream_write_state(SSL *s) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return SSL_STREAM_STATE_NONE; + s->cipher_list = sk; - return ossl_quic_get_stream_write_state(s); -#else - return SSL_STREAM_STATE_NONE; -#endif + return 1; } -int SSL_get_stream_read_error_code(SSL *s, uint64_t *app_error_code) +int SSL_set_cipher_list_by_id(SSL *s, STACK_OF(SSL_CIPHER) *cipher_list_by_id) { -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return -1; + STACK_OF(SSL_CIPHER) *sk = NULL; - return ossl_quic_get_stream_read_error_code(s, app_error_code); -#else - return -1; -#endif -} + if (s == NULL) + return 0; -int SSL_get_stream_write_error_code(SSL *s, uint64_t *app_error_code) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return -1; + if (cipher_list_by_id != NULL) { + sk = sk_SSL_CIPHER_dup(cipher_list_by_id); + if (sk == NULL) + return 0; + } - return ossl_quic_get_stream_write_error_code(s, app_error_code); -#else - return -1; -#endif -} + if (s->cipher_list_by_id != NULL) + sk_SSL_CIPHER_free(s->cipher_list_by_id); -int SSL_get_conn_close_info(SSL *s, SSL_CONN_CLOSE_INFO *info, - size_t info_len) -{ -#ifndef OPENSSL_NO_QUIC - if (!IS_QUIC(s)) - return -1; + s->cipher_list_by_id = sk; - return ossl_quic_get_conn_close_info(s, info, info_len); -#else - return -1; -#endif + return 1; } -int SSL_get_value_uint(SSL *s, uint32_t class_, uint32_t id, - uint64_t *value) +STACK_OF(SSL_CIPHER) *SSL_dup_cipher_list(SSL *s) { -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_get_value_uint(s, class_, id, value); -#endif + if (s->cipher_list != NULL) + return sk_SSL_CIPHER_dup(s->cipher_list); - ERR_raise(ERR_LIB_SSL, SSL_R_UNSUPPORTED_PROTOCOL); - return 0; + return NULL; } -int SSL_set_value_uint(SSL *s, uint32_t class_, uint32_t id, - uint64_t value) +STACK_OF(SSL_CIPHER) *SSL_dup_cipher_list_by_id(SSL *s) { -#ifndef OPENSSL_NO_QUIC - if (IS_QUIC(s)) - return ossl_quic_set_value_uint(s, class_, id, value); -#endif + if (s->cipher_list_by_id != NULL) + return sk_SSL_CIPHER_dup(s->cipher_list_by_id); - ERR_raise(ERR_LIB_SSL, SSL_R_UNSUPPORTED_PROTOCOL); - return 0; + return NULL; } -int SSL_add_expected_rpk(SSL *s, EVP_PKEY *rpk) +int SSL_CTX_set_cipher_list2(SSL_CTX *ctx, STACK_OF(SSL_CIPHER) *cipher_list) { - unsigned char *data = NULL; - SSL_DANE *dane = SSL_get0_dane(s); - int ret; + STACK_OF(SSL_CIPHER) *sk = NULL; - if (dane == NULL || dane->dctx == NULL) - return 0; - if ((ret = i2d_PUBKEY(rpk, &data)) <= 0) + if (ctx == NULL) return 0; - ret = SSL_dane_tlsa_add(s, DANETLS_USAGE_DANE_EE, - DANETLS_SELECTOR_SPKI, - DANETLS_MATCHING_FULL, - data, (size_t)ret) > 0; - OPENSSL_free(data); - return ret; -} - -EVP_PKEY *SSL_get0_peer_rpk(const SSL *s) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL || sc->session == NULL) - return NULL; - return sc->session->peer_rpk; -} + if (cipher_list != NULL) { + sk = sk_SSL_CIPHER_dup(cipher_list); + if (sk == NULL) + return 0; + } -int SSL_get_negotiated_client_cert_type(const SSL *s) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); + if (ctx->cipher_list != NULL) + sk_SSL_CIPHER_free(ctx->cipher_list); - if (sc == NULL) - return 0; + ctx->cipher_list = sk; - return sc->ext.client_cert_type; + return 1; } -int SSL_get_negotiated_server_cert_type(const SSL *s) +STACK_OF(SSL_CIPHER) *SSL_CTX_get_cipher_list_by_id(const SSL_CTX *ctx) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - return sc->ext.server_cert_type; + if (ctx != NULL) + return ctx->cipher_list_by_id; + return NULL; } -static int validate_cert_type(const unsigned char *val, size_t len) +int SSL_CTX_set_cipher_list_by_id(SSL_CTX *ctx, + STACK_OF(SSL_CIPHER) *cipher_list_by_id) { - size_t i; - int saw_rpk = 0; - int saw_x509 = 0; - - if (val == NULL && len == 0) - return 1; + STACK_OF(SSL_CIPHER) *sk = NULL; - if (val == NULL || len == 0) + if (ctx == NULL) return 0; - for (i = 0; i < len; i++) { - switch (val[i]) { - case TLSEXT_cert_type_rpk: - if (saw_rpk) - return 0; - saw_rpk = 1; - break; - case TLSEXT_cert_type_x509: - if (saw_x509) - return 0; - saw_x509 = 1; - break; - case TLSEXT_cert_type_pgp: - case TLSEXT_cert_type_1609dot2: - default: + if (cipher_list_by_id != NULL) { + sk = sk_SSL_CIPHER_dup(cipher_list_by_id); + if (sk == NULL) return 0; - } } - return 1; -} -static int set_cert_type(unsigned char **cert_type, - size_t *cert_type_len, - const unsigned char *val, - size_t len) -{ - unsigned char *tmp = NULL; + if (ctx->cipher_list_by_id != NULL) + sk_SSL_CIPHER_free(ctx->cipher_list_by_id); - if (!validate_cert_type(val, len)) - return 0; + ctx->cipher_list_by_id = sk; - if (val != NULL && (tmp = OPENSSL_memdup(val, len)) == NULL) - return 0; - - OPENSSL_free(*cert_type); - *cert_type = tmp; - *cert_type_len = len; return 1; } +#endif -int SSL_set1_client_cert_type(SSL *s, const unsigned char *val, size_t len) +#ifndef OPENSSL_NO_VERIFY_SNI +void SSL_CTX_set_verify_cert_with_sni(SSL_CTX *ctx, int value) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - return set_cert_type(&sc->client_cert_type, &sc->client_cert_type_len, - val, len); + if (value == 1) + ctx->verify_mode |= SSL_VERIFY_FAIL_IF_SNI_NOT_MATCH_CERT; + else + ctx->verify_mode &= ~SSL_VERIFY_FAIL_IF_SNI_NOT_MATCH_CERT; } -int SSL_set1_server_cert_type(SSL *s, const unsigned char *val, size_t len) +int SSL_CTX_get_verify_cert_with_sni(SSL_CTX *ctx) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - return set_cert_type(&sc->server_cert_type, &sc->server_cert_type_len, - val, len); + return ctx->verify_mode & SSL_VERIFY_FAIL_IF_SNI_NOT_MATCH_CERT; } +#endif -int SSL_CTX_set1_client_cert_type(SSL_CTX *ctx, const unsigned char *val, size_t len) +#ifndef OPENSSL_NO_SESSION_REUSED_TYPE +int SSL_get_session_reused_type(SSL *s) { - return set_cert_type(&ctx->client_cert_type, &ctx->client_cert_type_len, - val, len); + return !s->hit ? SSL_SESSION_REUSED_TYPE_NOCACHE : s->session_reused_type; } +#endif -int SSL_CTX_set1_server_cert_type(SSL_CTX *ctx, const unsigned char *val, size_t len) +#ifndef OPENSSL_NO_STATUS +void SSL_set_status_callback(SSL *s, + int (*status_callback)(unsigned char *p, + unsigned int length, + SSL_status *param), + unsigned int ssl_status_enable, void *arg) { - return set_cert_type(&ctx->server_cert_type, &ctx->server_cert_type_len, - val, len); + if (s != NULL) { + s->status_callback = status_callback; + s->status_param.arg = arg; + s->status_param.ssl_status_enable = ssl_status_enable; + + if (status_callback == NULL) + s->status_param.ssl_status_enable = 0; + } } -int SSL_get0_client_cert_type(const SSL *s, unsigned char **t, size_t *len) +int (*SSL_get_status_callback(const SSL *s))(unsigned char *p, + unsigned int length, SSL_status *param) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (t == NULL || len == NULL) - return 0; - - *t = sc->client_cert_type; - *len = sc->client_cert_type_len; - return 1; + return s->status_callback; } +#endif -int SSL_get0_server_cert_type(const SSL *s, unsigned char **t, size_t *len) +#ifndef OPENSSL_NO_SM2 +void SSL_CTX_enable_sm_tls13_strict(SSL_CTX *ctx) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); + ctx->enable_sm_tls13_strict = 1; +} - if (t == NULL || len == NULL) - return 0; +void SSL_CTX_disable_sm_tls13_strict(SSL_CTX *ctx) +{ + ctx->enable_sm_tls13_strict = 0; +} - *t = sc->server_cert_type; - *len = sc->server_cert_type_len; - return 1; +void SSL_enable_sm_tls13_strict(SSL *s) +{ + s->enable_sm_tls13_strict = 1; } -int SSL_CTX_get0_client_cert_type(const SSL_CTX *ctx, unsigned char **t, size_t *len) +void SSL_disable_sm_tls13_strict(SSL *s) { - if (t == NULL || len == NULL) - return 0; + s->enable_sm_tls13_strict = 0; +} +#endif - *t = ctx->client_cert_type; - *len = ctx->client_cert_type_len; - return 1; +#ifndef OPENSSL_NO_CERT_COMPRESSION +static CERT_COMP *CERT_COMP_copy(const CERT_COMP *p) +{ + return OPENSSL_memdup(p, sizeof(*p)); } -int SSL_CTX_get0_server_cert_type(const SSL_CTX *ctx, unsigned char **t, size_t *len) +static void CERT_COMP_free(CERT_COMP *p) { - if (t == NULL || len == NULL) - return 0; + if (p == NULL) + return; - *t = ctx->server_cert_type; - *len = ctx->server_cert_type_len; - return 1; + OPENSSL_free(p); } +#endif diff --git a/openssl/src/ssl/ssl_local.h b/openssl/src/ssl/ssl_local.h index f448cfdbc..dd5eac9e9 100644 --- a/openssl/src/ssl/ssl_local.h +++ b/openssl/src/ssl/ssl_local.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -15,35 +15,125 @@ # include "internal/e_os.h" /* struct timeval for DTLS */ # include # include +# include # include -# include "internal/common.h" /* for HAS_PREFIX */ # include -# include # include +# include # include # include # include # include # include # include -# include "internal/recordmethod.h" -# include "internal/statem.h" +# include "record/record.h" +# include "statem/statem.h" # include "internal/packet.h" # include "internal/dane.h" # include "internal/refcount.h" # include "internal/tsan_assist.h" # include "internal/bio.h" # include "internal/ktls.h" -# include "internal/time.h" -# include "internal/ssl.h" -# include "record/record.h" # ifdef OPENSSL_BUILD_SHLIBSSL # undef OPENSSL_EXTERN # define OPENSSL_EXTERN OPENSSL_EXPORT # endif +# define SM2_DEFAULT_ID "1234567812345678" +# define SM2_DEFAULT_ID_LEN (sizeof(SM2_DEFAULT_ID) - 1) + +# define c2l(c,l) (l = ((unsigned long)(*((c)++))) , \ + l|=(((unsigned long)(*((c)++)))<< 8), \ + l|=(((unsigned long)(*((c)++)))<<16), \ + l|=(((unsigned long)(*((c)++)))<<24)) + +/* NOTE - c is not incremented as per c2l */ +# define c2ln(c,l1,l2,n) { \ + c+=n; \ + l1=l2=0; \ + switch (n) { \ + case 8: l2 =((unsigned long)(*(--(c))))<<24; \ + case 7: l2|=((unsigned long)(*(--(c))))<<16; \ + case 6: l2|=((unsigned long)(*(--(c))))<< 8; \ + case 5: l2|=((unsigned long)(*(--(c)))); \ + case 4: l1 =((unsigned long)(*(--(c))))<<24; \ + case 3: l1|=((unsigned long)(*(--(c))))<<16; \ + case 2: l1|=((unsigned long)(*(--(c))))<< 8; \ + case 1: l1|=((unsigned long)(*(--(c)))); \ + } \ + } + +# define l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>>24)&0xff)) + +# define n2l(c,l) (l =((unsigned long)(*((c)++)))<<24, \ + l|=((unsigned long)(*((c)++)))<<16, \ + l|=((unsigned long)(*((c)++)))<< 8, \ + l|=((unsigned long)(*((c)++)))) + +# define n2l8(c,l) (l =((uint64_t)(*((c)++)))<<56, \ + l|=((uint64_t)(*((c)++)))<<48, \ + l|=((uint64_t)(*((c)++)))<<40, \ + l|=((uint64_t)(*((c)++)))<<32, \ + l|=((uint64_t)(*((c)++)))<<24, \ + l|=((uint64_t)(*((c)++)))<<16, \ + l|=((uint64_t)(*((c)++)))<< 8, \ + l|=((uint64_t)(*((c)++)))) + + +# define l2n(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff)) + +# define l2n6(l,c) (*((c)++)=(unsigned char)(((l)>>40)&0xff), \ + *((c)++)=(unsigned char)(((l)>>32)&0xff), \ + *((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff)) + +# define l2n8(l,c) (*((c)++)=(unsigned char)(((l)>>56)&0xff), \ + *((c)++)=(unsigned char)(((l)>>48)&0xff), \ + *((c)++)=(unsigned char)(((l)>>40)&0xff), \ + *((c)++)=(unsigned char)(((l)>>32)&0xff), \ + *((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff)) + +/* NOTE - c is not incremented as per l2c */ +# define l2cn(l1,l2,c,n) { \ + c+=n; \ + switch (n) { \ + case 8: *(--(c))=(unsigned char)(((l2)>>24)&0xff); \ + case 7: *(--(c))=(unsigned char)(((l2)>>16)&0xff); \ + case 6: *(--(c))=(unsigned char)(((l2)>> 8)&0xff); \ + case 5: *(--(c))=(unsigned char)(((l2) )&0xff); \ + case 4: *(--(c))=(unsigned char)(((l1)>>24)&0xff); \ + case 3: *(--(c))=(unsigned char)(((l1)>>16)&0xff); \ + case 2: *(--(c))=(unsigned char)(((l1)>> 8)&0xff); \ + case 1: *(--(c))=(unsigned char)(((l1) )&0xff); \ + } \ + } + +# define n2s(c,s) ((s=(((unsigned int)((c)[0]))<< 8)| \ + (((unsigned int)((c)[1])) )),(c)+=2) +# define s2n(s,c) (((c)[0]=(unsigned char)(((s)>> 8)&0xff), \ + (c)[1]=(unsigned char)(((s) )&0xff)),(c)+=2) + +# define n2l3(c,l) ((l =(((unsigned long)((c)[0]))<<16)| \ + (((unsigned long)((c)[1]))<< 8)| \ + (((unsigned long)((c)[2])) )),(c)+=3) + +# define l2n3(l,c) (((c)[0]=(unsigned char)(((l)>>16)&0xff), \ + (c)[1]=(unsigned char)(((l)>> 8)&0xff), \ + (c)[2]=(unsigned char)(((l) )&0xff)),(c)+=3) + # define TLS_MAX_VERSION_INTERNAL TLS1_3_VERSION # define DTLS_MAX_VERSION_INTERNAL DTLS1_2_VERSION @@ -57,7 +147,6 @@ # define DTLS_VERSION_LT(v1, v2) (dtls_ver_ordinal(v1) > dtls_ver_ordinal(v2)) # define DTLS_VERSION_LE(v1, v2) (dtls_ver_ordinal(v1) >= dtls_ver_ordinal(v2)) -# define SSL_AD_NO_ALERT -1 /* * Define the Bitmasks for SSL_CIPHER.algorithms. @@ -86,16 +175,15 @@ # define SSL_kEECDH SSL_kECDHE /* PSK */ # define SSL_kPSK 0x00000008U -/* GOST key exchange */ -# define SSL_kGOST 0x00000010U /* SRP */ # define SSL_kSRP 0x00000020U # define SSL_kRSAPSK 0x00000040U # define SSL_kECDHEPSK 0x00000080U # define SSL_kDHEPSK 0x00000100U -/* GOST KDF key exchange, draft-smyshlyaev-tls12-gost-suites */ -# define SSL_kGOST18 0x00000200U + +# define SSL_kSM2 0x00000400U +# define SSL_kSM2DHE 0x00000800U /* all PSK */ @@ -115,70 +203,56 @@ # define SSL_aECDSA 0x00000008U /* PSK auth */ # define SSL_aPSK 0x00000010U -/* GOST R 34.10-2001 signature auth */ -# define SSL_aGOST01 0x00000020U /* SRP auth */ # define SSL_aSRP 0x00000040U -/* GOST R 34.10-2012 signature auth */ -# define SSL_aGOST12 0x00000080U /* Any appropriate signature auth (for TLS 1.3 ciphersuites) */ # define SSL_aANY 0x00000000U +/* SM2 auth */ +# define SSL_aSM2 0x00000100U /* All bits requiring a certificate */ #define SSL_aCERT \ - (SSL_aRSA | SSL_aDSS | SSL_aECDSA | SSL_aGOST01 | SSL_aGOST12) + (SSL_aRSA | SSL_aDSS | SSL_aECDSA | SSL_aSM2) /* Bits for algorithm_enc (symmetric encryption) */ # define SSL_DES 0x00000001U # define SSL_3DES 0x00000002U # define SSL_RC4 0x00000004U -# define SSL_RC2 0x00000008U -# define SSL_IDEA 0x00000010U # define SSL_eNULL 0x00000020U # define SSL_AES128 0x00000040U # define SSL_AES256 0x00000080U -# define SSL_CAMELLIA128 0x00000100U -# define SSL_CAMELLIA256 0x00000200U -# define SSL_eGOST2814789CNT 0x00000400U -# define SSL_SEED 0x00000800U +/* 0x100U & 0x200U are spared now due to the removal of Camellia */ +/* 0x400U is spared now due to the removal of GOST */ +/* 0x800U is spared now due to the removal of SEED */ # define SSL_AES128GCM 0x00001000U # define SSL_AES256GCM 0x00002000U # define SSL_AES128CCM 0x00004000U # define SSL_AES256CCM 0x00008000U # define SSL_AES128CCM8 0x00010000U # define SSL_AES256CCM8 0x00020000U -# define SSL_eGOST2814789CNT12 0x00040000U +/* 0x40000U is spared now due to the removal of GOST */ # define SSL_CHACHA20POLY1305 0x00080000U -# define SSL_ARIA128GCM 0x00100000U -# define SSL_ARIA256GCM 0x00200000U -# define SSL_MAGMA 0x00400000U -# define SSL_KUZNYECHIK 0x00800000U +/* 0x100000U & 0x200000U are spared now due to the removal of ARIA */ +/* 0x400000U & 0x800000U are spared now due to the removal of GOST */ +# define SSL_SM4CCM 0x01000000U +# define SSL_SM4GCM 0x02000000U +# define SSL_SM4 0x04000000U # define SSL_AESGCM (SSL_AES128GCM | SSL_AES256GCM) # define SSL_AESCCM (SSL_AES128CCM | SSL_AES256CCM | SSL_AES128CCM8 | SSL_AES256CCM8) # define SSL_AES (SSL_AES128|SSL_AES256|SSL_AESGCM|SSL_AESCCM) -# define SSL_CAMELLIA (SSL_CAMELLIA128|SSL_CAMELLIA256) # define SSL_CHACHA20 (SSL_CHACHA20POLY1305) -# define SSL_ARIAGCM (SSL_ARIA128GCM | SSL_ARIA256GCM) -# define SSL_ARIA (SSL_ARIAGCM) -# define SSL_CBC (SSL_DES | SSL_3DES | SSL_RC2 | SSL_IDEA \ - | SSL_AES128 | SSL_AES256 | SSL_CAMELLIA128 \ - | SSL_CAMELLIA256 | SSL_SEED) +# define SSL_CBC (SSL_DES | SSL_3DES \ + | SSL_AES128 | SSL_AES256) /* Bits for algorithm_mac (symmetric authentication) */ # define SSL_MD5 0x00000001U # define SSL_SHA1 0x00000002U -# define SSL_GOST94 0x00000004U -# define SSL_GOST89MAC 0x00000008U # define SSL_SHA256 0x00000010U # define SSL_SHA384 0x00000020U /* Not a real MAC, just an indication it is part of cipher */ # define SSL_AEAD 0x00000040U -# define SSL_GOST12_256 0x00000080U -# define SSL_GOST89MAC12 0x00000100U -# define SSL_GOST12_512 0x00000200U -# define SSL_MAGMAOMAC 0x00000400U -# define SSL_KUZNYECHIKOMAC 0x00000800U +# define SSL_SM3 0x00001000U /* * When adding new digest in the ssl_ciph.c and increment SSL_MD_NUM_IDX make @@ -187,19 +261,13 @@ # define SSL_MD_MD5_IDX 0 # define SSL_MD_SHA1_IDX 1 -# define SSL_MD_GOST94_IDX 2 -# define SSL_MD_GOST89MAC_IDX 3 -# define SSL_MD_SHA256_IDX 4 -# define SSL_MD_SHA384_IDX 5 -# define SSL_MD_GOST12_256_IDX 6 -# define SSL_MD_GOST89MAC12_IDX 7 -# define SSL_MD_GOST12_512_IDX 8 -# define SSL_MD_MD5_SHA1_IDX 9 -# define SSL_MD_SHA224_IDX 10 -# define SSL_MD_SHA512_IDX 11 -# define SSL_MD_MAGMAOMAC_IDX 12 -# define SSL_MD_KUZNYECHIKOMAC_IDX 13 -# define SSL_MAX_DIGEST 14 +# define SSL_MD_SHA256_IDX 2 +# define SSL_MD_SHA384_IDX 3 +# define SSL_MD_MD5_SHA1_IDX 4 +# define SSL_MD_SHA224_IDX 5 +# define SSL_MD_SHA512_IDX 6 +# define SSL_MD_SM3_IDX 7 +# define SSL_MAX_DIGEST 8 #define SSL_MD_NUM_IDX SSL_MAX_DIGEST @@ -210,34 +278,16 @@ # define SSL_HANDSHAKE_MAC_MD5_SHA1 SSL_MD_MD5_SHA1_IDX # define SSL_HANDSHAKE_MAC_SHA256 SSL_MD_SHA256_IDX # define SSL_HANDSHAKE_MAC_SHA384 SSL_MD_SHA384_IDX -# define SSL_HANDSHAKE_MAC_GOST94 SSL_MD_GOST94_IDX -# define SSL_HANDSHAKE_MAC_GOST12_256 SSL_MD_GOST12_256_IDX -# define SSL_HANDSHAKE_MAC_GOST12_512 SSL_MD_GOST12_512_IDX # define SSL_HANDSHAKE_MAC_DEFAULT SSL_HANDSHAKE_MAC_MD5_SHA1 +# define SSL_HANDSHAKE_MAC_SM3 SSL_MD_SM3_IDX /* Bits 8-15 bits are PRF */ # define TLS1_PRF_DGST_SHIFT 8 # define TLS1_PRF_SHA1_MD5 (SSL_MD_MD5_SHA1_IDX << TLS1_PRF_DGST_SHIFT) # define TLS1_PRF_SHA256 (SSL_MD_SHA256_IDX << TLS1_PRF_DGST_SHIFT) # define TLS1_PRF_SHA384 (SSL_MD_SHA384_IDX << TLS1_PRF_DGST_SHIFT) -# define TLS1_PRF_GOST94 (SSL_MD_GOST94_IDX << TLS1_PRF_DGST_SHIFT) -# define TLS1_PRF_GOST12_256 (SSL_MD_GOST12_256_IDX << TLS1_PRF_DGST_SHIFT) -# define TLS1_PRF_GOST12_512 (SSL_MD_GOST12_512_IDX << TLS1_PRF_DGST_SHIFT) # define TLS1_PRF (SSL_MD_MD5_SHA1_IDX << TLS1_PRF_DGST_SHIFT) - -/* - * Stream MAC for GOST ciphersuites from cryptopro draft (currently this also - * goes into algorithm2) - */ -# define TLS1_STREAM_MAC 0x10000 -/* - * TLSTREE cipher/mac key derivation from draft-smyshlyaev-tls12-gost-suites - * (currently this also goes into algorithm2) - */ -# define TLS1_TLSTREE 0x20000 - -/* Ciphersuite supported in QUIC */ -# define SSL_QUIC 0x00040000U +# define TLS1_PRF_SM3 (SSL_MD_SM3_IDX << TLS1_PRF_DGST_SHIFT) # define SSL_STRONG_MASK 0x0000001FU # define SSL_DEFAULT_MASK 0X00000020U @@ -254,48 +304,57 @@ /* Flag used on OpenSSL ciphersuite ids to indicate they are for SSLv3+ */ # define SSL3_CK_CIPHERSUITE_FLAG 0x03000000 +/* Check if an SSL structure is using QUIC (which uses TLSv1.3) */ +# ifndef OPENSSL_NO_QUIC +# define SSL_IS_QUIC(s) (s->quic_method != NULL) +# else +# define SSL_IS_QUIC(s) 0 +# endif + /* Check if an SSL structure is using DTLS */ -# define SSL_CONNECTION_IS_DTLS(s) \ - (SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_DTLS) +# define SSL_IS_DTLS(s) (s->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_DTLS) /* Check if we are using TLSv1.3 */ -# define SSL_CONNECTION_IS_TLS13(s) (!SSL_CONNECTION_IS_DTLS(s) \ - && SSL_CONNECTION_GET_SSL(s)->method->version >= TLS1_3_VERSION \ - && SSL_CONNECTION_GET_SSL(s)->method->version != TLS_ANY_VERSION) +# define SSL_IS_TLS13(s) (!SSL_IS_DTLS(s) \ + && (s)->method->version >= TLS1_3_VERSION \ + && (s)->method->version != TLS_ANY_VERSION) + +# ifndef OPENSSL_NO_NTLS +# define SSL_IS_NTLS(s) (s->version == NTLS_VERSION) +# endif -# define SSL_CONNECTION_TREAT_AS_TLS13(s) \ - (SSL_CONNECTION_IS_TLS13(s) \ - || (s)->early_data_state == SSL_EARLY_DATA_CONNECTING \ +# define SSL_TREAT_AS_TLS13(s) \ + (SSL_IS_TLS13(s) || (s)->early_data_state == SSL_EARLY_DATA_CONNECTING \ || (s)->early_data_state == SSL_EARLY_DATA_CONNECT_RETRY \ || (s)->early_data_state == SSL_EARLY_DATA_WRITING \ || (s)->early_data_state == SSL_EARLY_DATA_WRITE_RETRY \ || (s)->hello_retry_request == SSL_HRR_PENDING) -# define SSL_IS_FIRST_HANDSHAKE(s) ((s)->s3.tmp.finish_md_len == 0 \ +# define SSL_IS_FIRST_HANDSHAKE(S) ((s)->s3.tmp.finish_md_len == 0 \ || (s)->s3.tmp.peer_finish_md_len == 0) /* See if we need explicit IV */ # define SSL_USE_EXPLICIT_IV(s) \ - (SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_EXPLICIT_IV) + (s->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_EXPLICIT_IV) /* * See if we use signature algorithms extension and signature algorithm * before signatures. */ # define SSL_USE_SIGALGS(s) \ - (SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_SIGALGS) + (s->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_SIGALGS) /* * Allow TLS 1.2 ciphersuites: applies to DTLS 1.2 as well as TLS 1.2: may * apply to others in future. */ # define SSL_USE_TLS1_2_CIPHERS(s) \ - (SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_TLS1_2_CIPHERS) + (s->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_TLS1_2_CIPHERS) /* * Determine if a client can use TLS 1.2 ciphersuites: can't rely on method * flags because it may not be set to correct version yet. */ # define SSL_CLIENT_USE_TLS1_2_CIPHERS(s) \ - ((!SSL_CONNECTION_IS_DTLS(s) && s->client_version >= TLS1_2_VERSION) || \ - (SSL_CONNECTION_IS_DTLS(s) && DTLS_VERSION_GE(s->client_version, DTLS1_2_VERSION))) + ((!SSL_IS_DTLS(s) && s->client_version >= TLS1_2_VERSION) || \ + (SSL_IS_DTLS(s) && DTLS_VERSION_GE(s->client_version, DTLS1_2_VERSION))) /* * Determine if a client should send signature algorithms extension: * as with TLS1.2 cipher we can't rely on method flags. @@ -314,54 +373,45 @@ # define SSL_READ_ETM(s) (s->s3.flags & TLS1_FLAGS_ENCRYPT_THEN_MAC_READ) # define SSL_WRITE_ETM(s) (s->s3.flags & TLS1_FLAGS_ENCRYPT_THEN_MAC_WRITE) -# define SSL_IS_QUIC_HANDSHAKE(s) (((s)->s3.flags & TLS1_FLAGS_QUIC) != 0) - -/* alert_dispatch values */ - -/* No alert pending */ -# define SSL_ALERT_DISPATCH_NONE 0 -/* Alert pending */ -# define SSL_ALERT_DISPATCH_PENDING 1 -/* Pending alert write needs to be retried */ -# define SSL_ALERT_DISPATCH_RETRY 2 - /* Mostly for SSLv3 */ # define SSL_PKEY_RSA 0 # define SSL_PKEY_RSA_PSS_SIGN 1 # define SSL_PKEY_DSA_SIGN 2 # define SSL_PKEY_ECC 3 -# define SSL_PKEY_GOST01 4 -# define SSL_PKEY_GOST12_256 5 -# define SSL_PKEY_GOST12_512 6 -# define SSL_PKEY_ED25519 7 -# define SSL_PKEY_ED448 8 -# define SSL_PKEY_NUM 9 +# define SSL_PKEY_ED25519 4 +# define SSL_PKEY_ED448 5 +# define SSL_PKEY_SM2 6 +# ifndef OPENSSL_NO_NTLS +# define SSL_PKEY_SM2_SIGN 7 +# define SSL_PKEY_SM2_ENC 8 +# define SSL_PKEY_RSA_SIGN 9 +# define SSL_PKEY_RSA_ENC 10 +# define SSL_PKEY_NUM 11 +# else +# define SSL_PKEY_NUM 7 +# endif # define SSL_ENC_DES_IDX 0 # define SSL_ENC_3DES_IDX 1 # define SSL_ENC_RC4_IDX 2 -# define SSL_ENC_RC2_IDX 3 -# define SSL_ENC_IDEA_IDX 4 -# define SSL_ENC_NULL_IDX 5 -# define SSL_ENC_AES128_IDX 6 -# define SSL_ENC_AES256_IDX 7 -# define SSL_ENC_CAMELLIA128_IDX 8 -# define SSL_ENC_CAMELLIA256_IDX 9 -# define SSL_ENC_GOST89_IDX 10 -# define SSL_ENC_SEED_IDX 11 -# define SSL_ENC_AES128GCM_IDX 12 -# define SSL_ENC_AES256GCM_IDX 13 -# define SSL_ENC_AES128CCM_IDX 14 -# define SSL_ENC_AES256CCM_IDX 15 -# define SSL_ENC_AES128CCM8_IDX 16 -# define SSL_ENC_AES256CCM8_IDX 17 -# define SSL_ENC_GOST8912_IDX 18 -# define SSL_ENC_CHACHA_IDX 19 -# define SSL_ENC_ARIA128GCM_IDX 20 -# define SSL_ENC_ARIA256GCM_IDX 21 -# define SSL_ENC_MAGMA_IDX 22 -# define SSL_ENC_KUZNYECHIK_IDX 23 -# define SSL_ENC_NUM_IDX 24 +# define SSL_ENC_NULL_IDX 3 +# define SSL_ENC_AES128_IDX 4 +# define SSL_ENC_AES256_IDX 5 +# define SSL_ENC_AES128GCM_IDX 6 +# define SSL_ENC_AES256GCM_IDX 7 +# define SSL_ENC_AES128CCM_IDX 8 +# define SSL_ENC_AES256CCM_IDX 9 +# define SSL_ENC_AES128CCM8_IDX 10 +# define SSL_ENC_AES256CCM8_IDX 11 +# define SSL_ENC_CHACHA_IDX 12 +# define SSL_ENC_SM4_GCM_IDX 13 +# define SSL_ENC_SM4_CCM_IDX 14 +# ifndef OPENSSL_NO_SM4 +# define SSL_ENC_SM4_IDX 15 +# define SSL_ENC_NUM_IDX 16 +# else +# define SSL_ENC_NUM_IDX 15 +# endif /*- * SSL_kRSA <- RSA_ENC @@ -377,11 +427,6 @@ #define CERT_PRIVATE_KEY 2 */ -/* Certificate Type State */ -# define OSSL_CERT_TYPE_CTOS_NONE 0 -# define OSSL_CERT_TYPE_CTOS_GOOD 1 -# define OSSL_CERT_TYPE_CTOS_ERROR 2 - /* Post-Handshake Authentication state */ typedef enum { SSL_PHA_NONE = 0, @@ -391,6 +436,16 @@ typedef enum { SSL_PHA_REQUESTED /* request received by client, or sent by server */ } SSL_PHA_STATE; +# ifndef OPENSSL_NO_CERT_COMPRESSION +typedef struct cert_comp_st { + unsigned int alg_id; + SSL_cert_compress_cb_fn compress; + SSL_cert_decompress_cb_fn decompress; +} CERT_COMP; + +DEFINE_STACK_OF(CERT_COMP) +# endif + /* CipherSuite length. SSLv3 and all TLS versions. */ # define TLS_CIPHER_LEN 2 /* used to hold info on the particular ciphers used */ @@ -422,12 +477,9 @@ struct ssl_method_st { int version; unsigned flags; unsigned long mask; - SSL *(*ssl_new) (SSL_CTX *ctx); - void (*ssl_free) (SSL *s); - int (*ssl_reset) (SSL *s); - int (*ssl_init) (SSL *s); + int (*ssl_new) (SSL *s); int (*ssl_clear) (SSL *s); - void (*ssl_deinit) (SSL *s); + void (*ssl_free) (SSL *s); int (*ssl_accept) (SSL *s); int (*ssl_connect) (SSL *s); int (*ssl_read) (SSL *s, void *buf, size_t len, size_t *readbytes); @@ -436,10 +488,10 @@ struct ssl_method_st { int (*ssl_shutdown) (SSL *s); int (*ssl_renegotiate) (SSL *s); int (*ssl_renegotiate_check) (SSL *s, int); - int (*ssl_read_bytes) (SSL *s, uint8_t type, uint8_t *recvd_type, + int (*ssl_read_bytes) (SSL *s, int type, int *recvd_type, unsigned char *buf, size_t len, int peek, size_t *readbytes); - int (*ssl_write_bytes) (SSL *s, uint8_t type, const void *buf_, size_t len, + int (*ssl_write_bytes) (SSL *s, int type, const void *buf_, size_t len, size_t *written); int (*ssl_dispatch_alert) (SSL *s); long (*ssl_ctrl) (SSL *s, int cmd, long larg, void *parg); @@ -450,7 +502,7 @@ struct ssl_method_st { size_t (*ssl_pending) (const SSL *s); int (*num_ciphers) (void); const SSL_CIPHER *(*get_cipher) (unsigned ncipher); - OSSL_TIME (*get_timeout) (void); + long (*get_timeout) (void); const struct ssl3_enc_method *ssl3_enc; /* Extra SSLv3/TLS stuff */ int (*ssl_version) (void); long (*ssl_callback_ctrl) (SSL *s, int cb_id, void (*fp) (void)); @@ -521,10 +573,11 @@ struct ssl_session_st { * to disable session caching and tickets. */ int not_resumable; - /* Peer raw public key, if available */ - EVP_PKEY *peer_rpk; /* This is the cert and type for the other end. */ X509 *peer; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + DELEGATED_CREDENTIAL *peer_dc; +#endif /* Certificate chain peer sent. */ STACK_OF(X509) *peer_chain; /* @@ -533,9 +586,10 @@ struct ssl_session_st { */ long verify_result; /* only for servers */ CRYPTO_REF_COUNT references; - OSSL_TIME timeout; - OSSL_TIME time; - OSSL_TIME calc_timeout; + time_t timeout; + time_t time; + time_t calc_timeout; + int timeout_ovf; unsigned int compress_meth; /* Need to lookup the method */ const SSL_CIPHER *cipher; unsigned long cipher_id; /* when ASN.1 loaded, this needs to be used to @@ -576,6 +630,13 @@ struct ssl_session_st { size_t ticket_appdata_len; uint32_t flags; SSL_CTX *owner; + CRYPTO_RWLOCK *lock; +# ifndef OPENSSL_NO_QUIC + unsigned int is_quic : 1; + + uint8_t *quic_early_data_context; + size_t quic_early_data_context_len; +# endif }; /* Extended master secret support */ @@ -632,11 +693,11 @@ typedef enum { /* * The allowance we have between the client's calculated ticket age and our own. - * We allow for 10 seconds. If a ticket is presented and the + * We allow for 10 seconds (units are in ms). If a ticket is presented and the * client's age calculation is different by more than this than our own then we * do not allow that ticket for early_data. */ -# define TICKET_AGE_ALLOWANCE ossl_seconds2time(10) +# define TICKET_AGE_ALLOWANCE (10 * 1000) #define MAX_COMPRESSIONS_SIZE 255 @@ -673,6 +734,10 @@ typedef struct { PACKET extensions; size_t pre_proc_exts_len; RAW_EXTENSION *pre_proc_exts; +# if !defined(OPENSSL_NO_SESSION_LOOKUP) + /* for asynchronous session resumption */ + STACK_OF(SSL_CIPHER) *ciphers; +# endif } CLIENTHELLO_MSG; /* @@ -693,29 +758,29 @@ typedef enum tlsext_index_en { TLSEXT_IDX_use_srtp, TLSEXT_IDX_encrypt_then_mac, TLSEXT_IDX_signed_certificate_timestamp, + TLSEXT_IDX_delegated_credential, TLSEXT_IDX_extended_master_secret, TLSEXT_IDX_signature_algorithms_cert, TLSEXT_IDX_post_handshake_auth, - TLSEXT_IDX_client_cert_type, - TLSEXT_IDX_server_cert_type, TLSEXT_IDX_signature_algorithms, TLSEXT_IDX_supported_versions, TLSEXT_IDX_psk_kex_modes, TLSEXT_IDX_key_share, TLSEXT_IDX_cookie, - TLSEXT_IDX_cryptopro_bug, - TLSEXT_IDX_compress_certificate, TLSEXT_IDX_early_data, TLSEXT_IDX_certificate_authorities, + TLSEXT_IDX_quic_transport_params_draft, + TLSEXT_IDX_quic_transport_params, + TLSEXT_IDX_compress_certificate, TLSEXT_IDX_padding, TLSEXT_IDX_psk, /* Dummy index - must always be the last entry */ TLSEXT_IDX_num_builtins } TLSEXT_INDEX; -DEFINE_LHASH_OF_EX(SSL_SESSION); +DEFINE_LHASH_OF(SSL_SESSION); /* Needed in ssl_cert.c */ -DEFINE_LHASH_OF_EX(X509_NAME); +DEFINE_LHASH_OF(X509_NAME); # define TLSEXT_KEYNAME_LENGTH 16 # define TLSEXT_TICK_KEY_LENGTH 32 @@ -769,31 +834,6 @@ typedef struct tls_group_info_st { char is_kem; /* Mode for this Group: 0 is KEX, 1 is KEM */ } TLS_GROUP_INFO; -typedef struct tls_sigalg_info_st { - char *name; /* name as in IANA TLS specs */ - uint16_t code_point; /* IANA-specified code point of sigalg-name */ - char *sigalg_name; /* (combined) sigalg name */ - char *sigalg_oid; /* (combined) sigalg OID */ - char *sig_name; /* pure signature algorithm name */ - char *sig_oid; /* pure signature algorithm OID */ - char *hash_name; /* hash algorithm name */ - char *hash_oid; /* hash algorithm OID */ - char *keytype; /* keytype name */ - char *keytype_oid; /* keytype OID */ - unsigned int secbits; /* Bits of security (from SP800-57) */ - int mintls; /* Minimum TLS version, -1 unsupported */ - int maxtls; /* Maximum TLS version (or 0 for undefined) */ -} TLS_SIGALG_INFO; - -/* - * Structure containing table entry of certificate info corresponding to - * CERT_PKEY entries - */ -typedef struct { - int nid; /* NID of public key algorithm */ - uint32_t amask; /* authmask corresponding to key type */ -} SSL_CERT_LOOKUP; - /* flags values */ # define TLS_GROUP_TYPE 0x0000000FU /* Mask for group type */ # define TLS_GROUP_CURVE_PRIME 0x00000001U @@ -804,6 +844,10 @@ typedef struct { # define TLS_GROUP_FFDHE_FOR_TLS1_3 (TLS_GROUP_FFDHE|TLS_GROUP_ONLY_FOR_TLS1_3) +# ifndef OPENSSL_NO_NTLS +# define PREREAD_HEADER_LENGTH 3 +# endif + struct ssl_ctx_st { OSSL_LIB_CTX *libctx; @@ -833,7 +877,7 @@ struct ssl_ctx_st { * SSL_new() is called. This has been put in to make life easier to set * things up */ - OSSL_TIME session_timeout; + long session_timeout; /* * If this callback is not null, it will be called each time a session id * is added to the cache. If this function returns 1, it means that the @@ -867,9 +911,9 @@ struct ssl_ctx_st { * other processes - spooky * :-) */ } stats; -#ifdef TSAN_REQUIRES_LOCKING +# ifdef TSAN_REQUIRES_LOCKING CRYPTO_RWLOCK *tsan_lock; -#endif +# endif CRYPTO_REF_COUNT references; @@ -940,11 +984,11 @@ struct ssl_ctx_st { size_t max_cert_list; struct cert_st /* CERT */ *cert; - SSL_CERT_LOOKUP *ssl_cert_info; int read_ahead; /* callback that allows applications to peek at protocol messages */ - ossl_msg_cb msg_callback; + void (*msg_callback) (int write_p, int version, int content_type, + const void *buf, size_t len, SSL *ssl, void *arg); void *msg_callback_arg; uint32_t verify_mode; @@ -1011,7 +1055,7 @@ struct ssl_ctx_st { int (*ticket_key_cb) (SSL *ssl, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc); -#endif +# endif int (*ticket_key_evp_cb) (SSL *ssl, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, EVP_MAC_CTX *hctx, @@ -1159,67 +1203,69 @@ struct ssl_ctx_st { const EVP_MD *ssl_digest_methods[SSL_MD_NUM_IDX]; size_t ssl_mac_secret_size[SSL_MD_NUM_IDX]; - size_t tls12_sigalgs_len; /* Cache of all sigalgs we know and whether they are available or not */ struct sigalg_lookup_st *sigalg_lookup_cache; - /* List of all sigalgs (code points) available, incl. from providers */ - uint16_t *tls12_sigalgs; TLS_GROUP_INFO *group_list; size_t group_list_len; size_t group_list_max_len; - TLS_SIGALG_INFO *sigalg_list; - size_t sigalg_list_len; - size_t sigalg_list_max_len; - /* masks of disabled algorithms */ uint32_t disabled_enc_mask; uint32_t disabled_mac_mask; uint32_t disabled_mkey_mask; uint32_t disabled_auth_mask; -#ifndef OPENSSL_NO_COMP_ALG - /* certificate compression preferences */ - int cert_comp_prefs[TLSEXT_comp_cert_limit]; +#ifndef OPENSSL_NO_NTLS + /* Tag of NTLS */ + int enable_ntls; + int enable_force_ntls; +#endif +#ifndef OPENSSL_NO_SM2 + /* + * tag of determining whether we should strict follow RFC 8998, + * when this tag set to 1, we will reject "TLS_SM4_GCM_SM3" and "TLS_SM4_CCM_SM3" + * without sm2 cert at server. This tag set to 0 default + */ + int enable_sm_tls13_strict; +#endif +#ifndef OPENSSL_NO_QUIC + const SSL_QUIC_METHOD *quic_method; +#endif +#ifndef OPENSSL_NO_CERT_COMPRESSION + STACK_OF(CERT_COMP) *cert_comp_algs; +#endif +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + int enable_verify_peer_by_dc; + int enable_sign_by_dc; #endif - - /* Certificate Type stuff - for RPK vs X.509 */ - unsigned char *client_cert_type; - size_t client_cert_type_len; - unsigned char *server_cert_type; - size_t server_cert_type_len; - -# ifndef OPENSSL_NO_QLOG - char *qlog_title; /* Session title for qlog */ -# endif }; -typedef struct cert_pkey_st CERT_PKEY; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +typedef struct dc_pkey_st DC_PKEY; +#endif -#define SSL_TYPE_SSL_CONNECTION 0 -#define SSL_TYPE_QUIC_CONNECTION 1 -#define SSL_TYPE_QUIC_XSO 2 +typedef struct cert_pkey_st CERT_PKEY; -struct ssl_st { - int type; - SSL_CTX *ctx; - const SSL_METHOD *defltmeth; - const SSL_METHOD *method; - CRYPTO_REF_COUNT references; - CRYPTO_RWLOCK *lock; - /* extra application data */ - CRYPTO_EX_DATA ex_data; +#ifndef OPENSSL_NO_QUIC +struct quic_data_st { + struct quic_data_st *next; + OSSL_ENCRYPTION_LEVEL level; + size_t start; /* offset into quic_buf->data */ + size_t length; }; +typedef struct quic_data_st QUIC_DATA; +int quic_set_encryption_secrets(SSL *s, OSSL_ENCRYPTION_LEVEL level); +#endif -struct ssl_connection_st { - /* type identifier and common data */ - struct ssl_st ssl; +struct ssl_st { /* * protocol version (one of SSL2_VERSION, SSL3_VERSION, TLS1_VERSION, * DTLS1_VERSION) */ int version; + /* SSLv3 */ + const SSL_METHOD *method; /* * There are 2 BIO's even though they are normally both the same. This * is so data can be read and written to different handlers @@ -1257,25 +1303,26 @@ struct ssl_connection_st { int quiet_shutdown; /* we have shut things down, 0x01 sent, 0x02 for received */ int shutdown; - /* Timestamps used to calculate the handshake RTT */ - OSSL_TIME ts_msg_write; - OSSL_TIME ts_msg_read; /* where we are */ OSSL_STATEM statem; SSL_EARLY_DATA_STATE early_data_state; BUF_MEM *init_buf; /* buffer used during init */ void *init_msg; /* pointer to handshake message body, set by - * tls_get_message_header() */ + * ssl3_get_message() */ size_t init_num; /* amount read/written */ size_t init_off; /* amount read/written */ - size_t ssl_pkey_num; - struct { long flags; + size_t read_mac_secret_size; + unsigned char read_mac_secret[EVP_MAX_MD_SIZE]; + size_t write_mac_secret_size; + unsigned char write_mac_secret[EVP_MAX_MD_SIZE]; unsigned char server_random[SSL3_RANDOM_SIZE]; unsigned char client_random[SSL3_RANDOM_SIZE]; - + /* flags for countermeasure against known-IV weakness */ + int need_empty_fragments; + int empty_fragment_done; /* used during startup, digest all incoming/outgoing packets */ BIO *handshake_buffer; /* @@ -1291,6 +1338,7 @@ struct ssl_connection_st { int change_cipher_spec; int warn_alert; int fatal_alert; + int alert_level; /* * we allow one fatal and one warning alert to be outstanding, send close * alert via the warning alert @@ -1305,7 +1353,6 @@ struct ssl_connection_st { int total_renegotiations; int num_renegotiations; int in_read_app_data; - struct { /* actually only need to be 16+20 for SSLv3 and 12 for TLS */ unsigned char finish_md[EVP_MAX_MD_SIZE * 2]; @@ -1351,6 +1398,14 @@ struct ssl_connection_st { const struct sigalg_lookup_st *sigalg; /* Pointer to certificate we use */ CERT_PKEY *cert; +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + /* Pointer to dc we use */ + DC_PKEY *dc; +# endif +# ifndef OPENSSL_NO_NTLS + CERT_PKEY *sign_cert; + CERT_PKEY *enc_cert; +# endif /* * signature algorithms peer reports: e.g. supported signature * algorithms extension for server or as part of a certificate @@ -1362,6 +1417,10 @@ struct ssl_connection_st { /* Size of above arrays */ size_t peer_sigalgslen; size_t peer_cert_sigalgslen; +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + uint16_t *peer_dc_sigalgs; + size_t peer_dc_sigalgslen; +# endif /* Sigalg peer actually uses */ const struct sigalg_lookup_st *peer_sigalg; /* @@ -1369,7 +1428,7 @@ struct ssl_connection_st { * SSL session: e.g. appropriate curve, signature algorithms etc. * If zero it can't be used at all. */ - uint32_t *valid_flags; + uint32_t valid_flags[SSL_PKEY_NUM]; /* * For servers the following masks are for the key and auth algorithms * that are supported by the certs below. For clients they are masks of @@ -1471,9 +1530,21 @@ struct ssl_connection_st { unsigned char handshake_traffic_hash[EVP_MAX_MD_SIZE]; unsigned char client_app_traffic_secret[EVP_MAX_MD_SIZE]; unsigned char server_app_traffic_secret[EVP_MAX_MD_SIZE]; +# ifndef OPENSSL_NO_QUIC + unsigned char client_hand_traffic_secret[EVP_MAX_MD_SIZE]; + unsigned char server_hand_traffic_secret[EVP_MAX_MD_SIZE]; + unsigned char client_early_traffic_secret[EVP_MAX_MD_SIZE]; +# endif unsigned char exporter_master_secret[EVP_MAX_MD_SIZE]; unsigned char early_exporter_master_secret[EVP_MAX_MD_SIZE]; - + EVP_CIPHER_CTX *enc_read_ctx; /* cryptographic state */ + unsigned char read_iv[EVP_MAX_IV_LENGTH]; /* TLSv1.3 static read IV */ + EVP_MD_CTX *read_hash; /* used for mac generation */ + COMP_CTX *compress; /* compression */ + COMP_CTX *expand; /* uncompress */ + EVP_CIPHER_CTX *enc_write_ctx; /* cryptographic state */ + unsigned char write_iv[EVP_MAX_IV_LENGTH]; /* TLSv1.3 static write IV */ + EVP_MD_CTX *write_hash; /* used for mac generation */ /* session info */ /* client cert? */ /* This is used to hold the server certificate used */ @@ -1531,9 +1602,12 @@ struct ssl_connection_st { SSL_psk_find_session_cb_func psk_find_session_cb; SSL_psk_use_session_cb_func psk_use_session_cb; + SSL_CTX *ctx; /* Verified chain of peer */ STACK_OF(X509) *verified_chain; long verify_result; + /* extra application data */ + CRYPTO_EX_DATA ex_data; /* * What we put in certificate_authorities extension for TLS 1.3 * (ClientHello and CertificateRequest) or just client cert requests for @@ -1542,6 +1616,7 @@ struct ssl_connection_st { */ STACK_OF(X509_NAME) *ca_names; STACK_OF(X509_NAME) *client_ca_names; + CRYPTO_REF_COUNT references; /* protocol behaviour */ uint64_t options; /* API behaviour */ @@ -1673,17 +1748,36 @@ struct ssl_connection_st { */ int tick_identity; - /* This is the list of algorithms the peer supports that we also support */ - int compress_certificate_from_peer[TLSEXT_comp_cert_limit]; - /* indicate that we sent the extension, so we'll accept it */ - int compress_certificate_sent; - - uint8_t client_cert_type; - uint8_t client_cert_type_ctos; - uint8_t server_cert_type; - uint8_t server_cert_type_ctos; +#ifndef OPENSSL_NO_QUIC + uint8_t *quic_transport_params; + size_t quic_transport_params_len; + uint8_t *peer_quic_transport_params_draft; + size_t peer_quic_transport_params_draft_len; + uint8_t *peer_quic_transport_params; + size_t peer_quic_transport_params_len; +#endif } ext; +#ifndef OPENSSL_NO_QUIC + OSSL_ENCRYPTION_LEVEL quic_read_level; + OSSL_ENCRYPTION_LEVEL quic_write_level; + OSSL_ENCRYPTION_LEVEL quic_latest_level_received; + /* + * defaults to 0, but can be set to: + * - TLSEXT_TYPE_quic_transport_parameters_draft + * - TLSEXT_TYPE_quic_transport_parameters + * Client: if 0, send both + * Server: if 0, use same version as client sent + */ + int quic_transport_version; + BUF_MEM *quic_buf; /* buffer incoming handshake messages */ + QUIC_DATA *quic_input_data_head; + QUIC_DATA *quic_input_data_tail; + size_t quic_next_record_start; + const SSL_QUIC_METHOD *quic_method; + uint8_t *quic_early_data_context; + size_t quic_early_data_context_len; +#endif /* * Parsed form of the ClientHello, kept around across client_hello_cb * calls. @@ -1745,10 +1839,7 @@ struct ssl_connection_st { * basis, depending on the chosen cipher. */ int (*not_resumable_session_cb) (SSL *ssl, int is_forward_secure); - - /* Record layer data */ RECORD_LAYER rlayer; - /* Default password callback. */ pem_password_cb *default_passwd_callback; /* Default password callback user data. */ @@ -1776,6 +1867,13 @@ struct ssl_connection_st { */ uint32_t early_data_count; + /* TLS1.3 padding callback */ + size_t (*record_padding_cb)(SSL *s, int type, size_t len, void *arg); + void *record_padding_arg; + size_t block_padding; + + CRYPTO_RWLOCK *lock; + /* The number of TLS1.3 tickets to automatically send */ size_t num_tickets; /* The number of TLS1.3 tickets actually sent so far */ @@ -1798,49 +1896,56 @@ struct ssl_connection_st { const struct sigalg_lookup_st **shared_sigalgs; size_t shared_sigalgslen; -#ifndef OPENSSL_NO_COMP_ALG - /* certificate compression preferences */ - int cert_comp_prefs[TLSEXT_comp_cert_limit]; -#endif +# ifndef OPENSSL_NO_NTLS + int enable_ntls; + int enable_force_ntls; - /* Certificate Type stuff - for RPK vs X.509 */ - unsigned char *client_cert_type; - size_t client_cert_type_len; - unsigned char *server_cert_type; - size_t server_cert_type_len; -}; + uint8_t preread_buf[PREREAD_HEADER_LENGTH]; + size_t preread_len; +# endif +# ifndef OPENSSL_NO_SKIP_SCSV + int skip_scsv; +# endif -# define SSL_CONNECTION_FROM_SSL_ONLY_int(ssl, c) \ - ((ssl) == NULL ? NULL \ - : ((ssl)->type == SSL_TYPE_SSL_CONNECTION \ - ? (c SSL_CONNECTION *)(ssl) \ - : NULL)) -# define SSL_CONNECTION_NO_CONST -# define SSL_CONNECTION_FROM_SSL_ONLY(ssl) \ - SSL_CONNECTION_FROM_SSL_ONLY_int(ssl, SSL_CONNECTION_NO_CONST) -# define SSL_CONNECTION_FROM_CONST_SSL_ONLY(ssl) \ - SSL_CONNECTION_FROM_SSL_ONLY_int(ssl, const) -# define SSL_CONNECTION_GET_CTX(sc) ((sc)->ssl.ctx) -# define SSL_CONNECTION_GET_SSL(sc) (&(sc)->ssl) -# ifndef OPENSSL_NO_QUIC -# include "quic/quic_local.h" -# define SSL_CONNECTION_FROM_SSL_int(ssl, c) \ - ((ssl) == NULL ? NULL \ - : ((ssl)->type == SSL_TYPE_SSL_CONNECTION \ - ? (c SSL_CONNECTION *)(ssl) \ - : ((ssl)->type == SSL_TYPE_QUIC_CONNECTION \ - ? (c SSL_CONNECTION *)((c QUIC_CONNECTION *)(ssl))->tls \ - : NULL))) -# define SSL_CONNECTION_FROM_SSL(ssl) \ - SSL_CONNECTION_FROM_SSL_int(ssl, SSL_CONNECTION_NO_CONST) -# define SSL_CONNECTION_FROM_CONST_SSL(ssl) \ - SSL_CONNECTION_FROM_SSL_int(ssl, const) -# else -# define SSL_CONNECTION_FROM_SSL(ssl) \ - SSL_CONNECTION_FROM_SSL_ONLY_int(ssl, SSL_CONNECTION_NO_CONST) -# define SSL_CONNECTION_FROM_CONST_SSL(ssl) \ - SSL_CONNECTION_FROM_SSL_ONLY_int(ssl, const) +# ifndef OPENSSL_NO_SESSION_REUSED_TYPE + /* session reuse type: 0->nocache, 1->cache, 2->ticket */ + int session_reused_type; +# endif + +# ifndef OPENSSL_NO_STATUS + int (*status_callback)(unsigned char *p, unsigned int length, SSL_status* param); + SSL_status status_param; +# endif + +# ifndef OPENSSL_NO_SM2 + /* + * tag of determining whether we should strict follow RFC 8998, + * when this tag set to 1, we will reject "TLS_SM4_GCM_SM3" and "TLS_SM4_CCM_SM3" + * without sm2 cert at server. This tag set to 0 default + */ + int enable_sm_tls13_strict; +# endif + +# ifndef OPENSSL_NO_CERT_COMPRESSION + STACK_OF(CERT_COMP) *cert_comp_algs; + uint16_t cert_comp_compress_id; + uint16_t cert_comp_decompress_id; +# endif +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + int enable_verify_peer_by_dc; + int enable_sign_by_dc; + /* + * delegated_credential_tag is used to illustrate whether client/server has send + * a delegated_credential extension or used it for handshake, If the client receives + * a delegated credential without sending this extension, then the client MUST abort + * with an "unexpected_message" alert. + */ + int delegated_credential_tag; + + const struct sigalg_lookup_st **shared_dc_sigalgs; + size_t shared_dc_sigalgslen; # endif +}; /* * Structure containing table entry of values associated with the signature @@ -1867,6 +1972,15 @@ typedef struct sigalg_lookup_st { int enabled; } SIGALG_LOOKUP; +/* + * Structure containing table entry of certificate info corresponding to + * CERT_PKEY entries + */ +typedef struct { + int nid; /* NID of public key algorithm */ + uint32_t amask; /* authmask corresponding to key type */ +} SSL_CERT_LOOKUP; + /* DTLS structures */ # ifndef OPENSSL_NO_SCTP @@ -1883,8 +1997,11 @@ typedef struct sigalg_lookup_st { # define DTLS1_SKIP_RECORD_HEADER 2 struct dtls1_retransmit_state { - const OSSL_RECORD_METHOD *wrlmethod; - OSSL_RECORD_LAYER *wrl; + EVP_CIPHER_CTX *enc_write_ctx; /* cryptographic state */ + EVP_MD_CTX *write_hash; /* used for mac generation */ + COMP_CTX *compress; /* compression */ + SSL_SESSION *session; + unsigned short epoch; }; struct hm_header_st { @@ -1947,7 +2064,7 @@ typedef struct dtls1_state_st { /* * Indicates when the last handshake msg sent will timeout */ - OSSL_TIME next_timeout; + struct timeval next_timeout; /* Timeout duration */ unsigned int timeout_duration_us; @@ -1967,19 +2084,12 @@ typedef struct dtls1_state_st { # define EXPLICIT_CHAR2_CURVE_TYPE 2 # define NAMED_CURVE_TYPE 3 -# ifndef OPENSSL_NO_COMP_ALG -struct ossl_comp_cert_st { - unsigned char *data; - size_t len; - size_t orig_len; - CRYPTO_REF_COUNT references; - int alg; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +struct dc_pkey_st { + DELEGATED_CREDENTIAL *dc; + EVP_PKEY *privatekey; }; -typedef struct ossl_comp_cert_st OSSL_COMP_CERT; - -void OSSL_COMP_CERT_free(OSSL_COMP_CERT *c); -int OSSL_COMP_CERT_up_ref(OSSL_COMP_CERT *c); -# endif +#endif struct cert_pkey_st { X509 *x509; @@ -1995,11 +2105,6 @@ struct cert_pkey_st { */ unsigned char *serverinfo; size_t serverinfo_length; -# ifndef OPENSSL_NO_COMP_ALG - /* Compressed certificate data - index 0 is unused */ - OSSL_COMP_CERT *comp_cert[TLSEXT_comp_cert_limit]; - int cert_comp_used; -# endif }; /* Retrieve Suite B flags */ # define tls1_suiteb(s) (s->cert->cert_flags & SSL_CERT_FLAG_SUITEB_128_LOS) @@ -2063,8 +2168,10 @@ typedef struct cert_st { int dh_tmp_auto; /* Flags related to certificates */ uint32_t cert_flags; - CERT_PKEY *pkeys; - size_t ssl_pkey_num; + CERT_PKEY pkeys[SSL_PKEY_NUM]; +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + DC_PKEY dc_pkeys[SSL_PKEY_NUM]; +# endif /* Custom certificate types sent in certificate request message. */ uint8_t *ctype; size_t ctype_len; @@ -2114,6 +2221,7 @@ typedef struct cert_st { char *psk_identity_hint; # endif CRYPTO_REF_COUNT references; /* >1 only if SSL_copy_session_id is used */ + CRYPTO_RWLOCK *lock; } CERT; # define FP_ICC (int (*)(const void *,const void *)) @@ -2123,36 +2231,37 @@ typedef struct cert_st { * of a mess of functions, but hell, think of it as an opaque structure :-) */ typedef struct ssl3_enc_method { - int (*setup_key_block) (SSL_CONNECTION *); - int (*generate_master_secret) (SSL_CONNECTION *, unsigned char *, - unsigned char *, size_t, size_t *); - int (*change_cipher_state) (SSL_CONNECTION *, int); - size_t (*final_finish_mac) (SSL_CONNECTION *, const char *, size_t, - unsigned char *); + int (*enc) (SSL *, SSL3_RECORD *, size_t, int, SSL_MAC_BUF *, size_t); + int (*mac) (SSL *, SSL3_RECORD *, unsigned char *, int); + int (*setup_key_block) (SSL *); + int (*generate_master_secret) (SSL *, unsigned char *, unsigned char *, + size_t, size_t *); + int (*change_cipher_state) (SSL *, int); + size_t (*final_finish_mac) (SSL *, const char *, size_t, unsigned char *); const char *client_finished_label; size_t client_finished_label_len; const char *server_finished_label; size_t server_finished_label_len; int (*alert_value) (int); - int (*export_keying_material) (SSL_CONNECTION *, unsigned char *, size_t, + int (*export_keying_material) (SSL *, unsigned char *, size_t, const char *, size_t, const unsigned char *, size_t, int use_context); /* Various flags indicating protocol version requirements */ uint32_t enc_flags; /* Set the handshake header */ - int (*set_handshake_header) (SSL_CONNECTION *s, WPACKET *pkt, int type); + int (*set_handshake_header) (SSL *s, WPACKET *pkt, int type); /* Close construction of the handshake message */ - int (*close_construct_packet) (SSL_CONNECTION *s, WPACKET *pkt, int htype); + int (*close_construct_packet) (SSL *s, WPACKET *pkt, int htype); /* Write out handshake message */ - int (*do_write) (SSL_CONNECTION *s); + int (*do_write) (SSL *s); } SSL3_ENC_METHOD; # define ssl_set_handshake_header(s, pkt, htype) \ - SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->set_handshake_header((s), (pkt), (htype)) + s->method->ssl3_enc->set_handshake_header((s), (pkt), (htype)) # define ssl_close_construct_packet(s, pkt, htype) \ - SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->close_construct_packet((s), (pkt), (htype)) -# define ssl_do_write(s) SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->do_write(s) + s->method->ssl3_enc->close_construct_packet((s), (pkt), (htype)) +# define ssl_do_write(s) s->method->ssl3_enc->do_write(s) /* Values for enc_flags */ @@ -2170,6 +2279,16 @@ typedef struct ssl3_enc_method { */ # define SSL_ENC_FLAG_TLS1_2_CIPHERS 0x10 +# ifndef OPENSSL_NO_COMP +/* Used for holding the relevant compression methods loaded into SSL_CTX */ +typedef struct ssl3_comp_st { + int comp_id; /* The identifier byte for this compression + * type */ + char *name; /* Text name used for the compression type */ + COMP_METHOD *method; /* The method :-) */ +} SSL3_COMP; +# endif + typedef enum downgrade_en { DOWNGRADE_NONE, DOWNGRADE_TO_1_2, @@ -2188,6 +2307,7 @@ typedef enum downgrade_en { #define TLSEXT_SIGALG_ecdsa_secp521r1_sha512 0x0603 #define TLSEXT_SIGALG_ecdsa_sha224 0x0303 #define TLSEXT_SIGALG_ecdsa_sha1 0x0203 +#define TLSEXT_SIGALG_sm2sig_sm3 0x0708 #define TLSEXT_SIGALG_rsa_pss_rsae_sha256 0x0804 #define TLSEXT_SIGALG_rsa_pss_rsae_sha384 0x0805 #define TLSEXT_SIGALG_rsa_pss_rsae_sha512 0x0806 @@ -2204,17 +2324,9 @@ typedef enum downgrade_en { #define TLSEXT_SIGALG_dsa_sha512 0x0602 #define TLSEXT_SIGALG_dsa_sha224 0x0302 #define TLSEXT_SIGALG_dsa_sha1 0x0202 -#define TLSEXT_SIGALG_gostr34102012_256_intrinsic 0x0840 -#define TLSEXT_SIGALG_gostr34102012_512_intrinsic 0x0841 -#define TLSEXT_SIGALG_gostr34102012_256_gostr34112012_256 0xeeee -#define TLSEXT_SIGALG_gostr34102012_512_gostr34112012_512 0xefef -#define TLSEXT_SIGALG_gostr34102001_gostr3411 0xeded #define TLSEXT_SIGALG_ed25519 0x0807 #define TLSEXT_SIGALG_ed448 0x0808 -#define TLSEXT_SIGALG_ecdsa_brainpoolP256r1_sha256 0x081a -#define TLSEXT_SIGALG_ecdsa_brainpoolP384r1_sha384 0x081b -#define TLSEXT_SIGALG_ecdsa_brainpoolP512r1_sha512 0x081c /* Known PSK key exchange modes */ #define TLSEXT_KEX_MODE_KE 0x00 @@ -2237,8 +2349,9 @@ typedef enum downgrade_en { extern const unsigned char tls11downgrade[8]; extern const unsigned char tls12downgrade[8]; -extern const SSL3_ENC_METHOD ssl3_undef_enc_method; +extern SSL3_ENC_METHOD ssl3_undef_enc_method; +__owur const SSL_METHOD *ssl_bad_method(int ver); __owur const SSL_METHOD *sslv3_method(void); __owur const SSL_METHOD *sslv3_server_method(void); __owur const SSL_METHOD *sslv3_client_method(void); @@ -2261,6 +2374,11 @@ __owur const SSL_METHOD *dtls_bad_ver_client_method(void); __owur const SSL_METHOD *dtlsv1_2_method(void); __owur const SSL_METHOD *dtlsv1_2_server_method(void); __owur const SSL_METHOD *dtlsv1_2_client_method(void); +# ifndef OPENSSL_NO_NTLS +__owur const SSL_METHOD *ntls_method(void); +__owur const SSL_METHOD *ntls_server_method(void); +__owur const SSL_METHOD *ntls_client_method(void); +# endif extern const SSL3_ENC_METHOD TLSv1_enc_data; extern const SSL3_ENC_METHOD TLSv1_1_enc_data; @@ -2269,6 +2387,9 @@ extern const SSL3_ENC_METHOD TLSv1_3_enc_data; extern const SSL3_ENC_METHOD SSLv3_enc_data; extern const SSL3_ENC_METHOD DTLSv1_enc_data; extern const SSL3_ENC_METHOD DTLSv1_2_enc_data; +# ifndef OPENSSL_NO_NTLS +extern const SSL3_ENC_METHOD NTLS_enc_data; +# endif /* * Flags for SSL methods @@ -2284,9 +2405,6 @@ const SSL_METHOD *func_name(void) \ version, \ flags, \ mask, \ - ossl_ssl_connection_new, \ - ossl_ssl_connection_free, \ - ossl_ssl_connection_reset, \ tls1_new, \ tls1_clear, \ tls1_free, \ @@ -2324,9 +2442,6 @@ const SSL_METHOD *func_name(void) \ SSL3_VERSION, \ SSL_METHOD_NO_FIPS | SSL_METHOD_NO_SUITEB, \ SSL_OP_NO_SSLv3, \ - ossl_ssl_connection_new, \ - ossl_ssl_connection_free, \ - ossl_ssl_connection_reset, \ ssl3_new, \ ssl3_clear, \ ssl3_free, \ @@ -2365,9 +2480,6 @@ const SSL_METHOD *func_name(void) \ version, \ flags, \ mask, \ - ossl_ssl_connection_new, \ - ossl_ssl_connection_free, \ - ossl_ssl_connection_reset, \ dtls1_new, \ dtls1_clear, \ dtls1_free, \ @@ -2399,58 +2511,33 @@ const SSL_METHOD *func_name(void) \ } struct openssl_ssl_test_functions { - int (*p_ssl_init_wbio_buffer) (SSL_CONNECTION *s); + int (*p_ssl_init_wbio_buffer) (SSL *s); + int (*p_ssl3_setup_buffers) (SSL *s); }; const char *ssl_protocol_to_string(int version); -static ossl_inline int tls12_rpk_and_privkey(const SSL_CONNECTION *sc, int idx) -{ - /* - * This is to check for special cases when using RPK with just - * a private key, and NO CERTIFICATE - */ - return ((sc->server && sc->ext.server_cert_type == TLSEXT_cert_type_rpk) - || (!sc->server && sc->ext.client_cert_type == TLSEXT_cert_type_rpk)) - && sc->cert->pkeys[idx].privatekey != NULL - && sc->cert->pkeys[idx].x509 == NULL; -} - -static ossl_inline int ssl_has_cert_type(const SSL_CONNECTION *sc, unsigned char ct) +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +static ossl_inline int ssl_has_dc(const SSL *s, int idx) { - unsigned char *ptr; - size_t len; - - if (sc->server) { - ptr = sc->server_cert_type; - len = sc->server_cert_type_len; - } else { - ptr = sc->client_cert_type; - len = sc->client_cert_type_len; - } - - if (ptr == NULL) + if (idx < 0 || idx >= SSL_PKEY_NUM) return 0; - return memchr(ptr, ct, len) != NULL; + return s->cert->dc_pkeys[idx].dc != NULL + && s->cert->dc_pkeys[idx].privatekey != NULL; } +#endif /* Returns true if certificate and private key for 'idx' are present */ -static ossl_inline int ssl_has_cert(const SSL_CONNECTION *s, int idx) +static ossl_inline int ssl_has_cert(const SSL *s, int idx) { - if (idx < 0 || idx >= (int)s->ssl_pkey_num) + if (idx < 0 || idx >= SSL_PKEY_NUM) return 0; - - /* If RPK is enabled for this SSL... only require private key */ - if (ssl_has_cert_type(s, TLSEXT_cert_type_rpk)) - return s->cert->pkeys[idx].privatekey != NULL; - return s->cert->pkeys[idx].x509 != NULL && s->cert->pkeys[idx].privatekey != NULL; } -static ossl_inline void tls1_get_peer_groups(SSL_CONNECTION *s, - const uint16_t **pgroups, +static ossl_inline void tls1_get_peer_groups(SSL *s, const uint16_t **pgroups, size_t *pgroupslen) { *pgroups = s->ext.peer_supportedgroups; @@ -2459,27 +2546,19 @@ static ossl_inline void tls1_get_peer_groups(SSL_CONNECTION *s, # ifndef OPENSSL_UNIT_TEST -__owur int ossl_ssl_init(SSL *ssl, SSL_CTX *ctx, const SSL_METHOD *method, - int type); -__owur SSL *ossl_ssl_connection_new_int(SSL_CTX *ctx, const SSL_METHOD *method); -__owur SSL *ossl_ssl_connection_new(SSL_CTX *ctx); -void ossl_ssl_connection_free(SSL *ssl); -__owur int ossl_ssl_connection_reset(SSL *ssl); - __owur int ssl_read_internal(SSL *s, void *buf, size_t num, size_t *readbytes); -__owur int ssl_write_internal(SSL *s, const void *buf, size_t num, - uint64_t flags, size_t *written); -int ssl_clear_bad_session(SSL_CONNECTION *s); -__owur CERT *ssl_cert_new(size_t ssl_pkey_num); +__owur int ssl_write_internal(SSL *s, const void *buf, size_t num, size_t *written); +void ssl_clear_cipher_ctx(SSL *s); +int ssl_clear_bad_session(SSL *s); +__owur CERT *ssl_cert_new(void); __owur CERT *ssl_cert_dup(CERT *cert); void ssl_cert_clear_certs(CERT *c); void ssl_cert_free(CERT *c); -__owur int ssl_generate_session_id(SSL_CONNECTION *s, SSL_SESSION *ss); -__owur int ssl_get_new_session(SSL_CONNECTION *s, int session); -__owur SSL_SESSION *lookup_sess_in_cache(SSL_CONNECTION *s, - const unsigned char *sess_id, +__owur int ssl_generate_session_id(SSL *s, SSL_SESSION *ss); +__owur int ssl_get_new_session(SSL *s, int session); +__owur SSL_SESSION *lookup_sess_in_cache(SSL *s, const unsigned char *sess_id, size_t sess_id_len); -__owur int ssl_get_prev_session(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello); +__owur int ssl_get_prev_session(SSL *s, CLIENTHELLO_MSG *hello); __owur SSL_SESSION *ssl_session_dup(const SSL_SESSION *src, int ticket); __owur int ssl_cipher_id_cmp(const SSL_CIPHER *a, const SSL_CIPHER *b); DECLARE_OBJ_BSEARCH_GLOBAL_CMP_FN(SSL_CIPHER, SSL_CIPHER, ssl_cipher_id); @@ -2491,13 +2570,12 @@ __owur STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(SSL_CTX *ctx, STACK_OF(SSL_CIPHER) **cipher_list_by_id, const char *rule_str, CERT *c); -__owur int ssl_cache_cipherlist(SSL_CONNECTION *s, PACKET *cipher_suites, - int sslv2format); -__owur int ossl_bytes_to_cipher_list(SSL_CONNECTION *s, PACKET *cipher_suites, - STACK_OF(SSL_CIPHER) **skp, - STACK_OF(SSL_CIPHER) **scsvs, int sslv2format, - int fatal); -void ssl_update_cache(SSL_CONNECTION *s, int mode); +__owur int ssl_cache_cipherlist(SSL *s, PACKET *cipher_suites, int sslv2format); +__owur int bytes_to_cipher_list(SSL *s, PACKET *cipher_suites, + STACK_OF(SSL_CIPHER) **skp, + STACK_OF(SSL_CIPHER) **scsvs, int sslv2format, + int fatal); +void ssl_update_cache(SSL *s, int mode); __owur int ssl_cipher_get_evp_cipher(SSL_CTX *ctx, const SSL_CIPHER *sslc, const EVP_CIPHER **enc); __owur int ssl_cipher_get_evp(SSL_CTX *ctxc, const SSL_SESSION *s, @@ -2508,88 +2586,83 @@ __owur int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead, size_t *int_overhead, size_t *blocksize, size_t *ext_overhead); __owur int ssl_cert_is_disabled(SSL_CTX *ctx, size_t idx); -__owur const SSL_CIPHER *ssl_get_cipher_by_char(SSL_CONNECTION *ssl, +__owur const SSL_CIPHER *ssl_get_cipher_by_char(SSL *ssl, const unsigned char *ptr, int all); -__owur int ssl_cert_set0_chain(SSL_CONNECTION *s, SSL_CTX *ctx, - STACK_OF(X509) *chain); -__owur int ssl_cert_set1_chain(SSL_CONNECTION *s, SSL_CTX *ctx, - STACK_OF(X509) *chain); -__owur int ssl_cert_add0_chain_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x); -__owur int ssl_cert_add1_chain_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x); +__owur int ssl_cert_set0_chain(SSL *s, SSL_CTX *ctx, STACK_OF(X509) *chain); +__owur int ssl_cert_set1_chain(SSL *s, SSL_CTX *ctx, STACK_OF(X509) *chain); +__owur int ssl_cert_add0_chain_cert(SSL *s, SSL_CTX *ctx, X509 *x); +__owur int ssl_cert_add1_chain_cert(SSL *s, SSL_CTX *ctx, X509 *x); __owur int ssl_cert_select_current(CERT *c, X509 *x); __owur int ssl_cert_set_current(CERT *c, long arg); void ssl_cert_set_cert_cb(CERT *c, int (*cb) (SSL *ssl, void *arg), void *arg); +SSL_cert_cb_fn ssl_cert_get_cert_cb(CERT *c); +void *ssl_cert_get_cert_cb_arg(CERT *c); -__owur int ssl_verify_cert_chain(SSL_CONNECTION *s, STACK_OF(X509) *sk); -__owur int ssl_verify_rpk(SSL_CONNECTION *s, EVP_PKEY *rpk); -__owur int ssl_build_cert_chain(SSL_CONNECTION *s, SSL_CTX *ctx, int flags); +__owur int ssl_verify_cert_chain(SSL *s, STACK_OF(X509) *sk); +__owur int ssl_build_cert_chain(SSL *s, SSL_CTX *ctx, int flags); __owur int ssl_cert_set_cert_store(CERT *c, X509_STORE *store, int chain, int ref); __owur int ssl_cert_get_cert_store(CERT *c, X509_STORE **pstore, int chain); -__owur int ssl_security(const SSL_CONNECTION *s, int op, int bits, int nid, - void *other); +__owur int ssl_security(const SSL *s, int op, int bits, int nid, void *other); __owur int ssl_ctx_security(const SSL_CTX *ctx, int op, int bits, int nid, void *other); int ssl_get_security_level_bits(const SSL *s, const SSL_CTX *ctx, int *levelp); -__owur int ssl_cert_lookup_by_nid(int nid, size_t *pidx, SSL_CTX *ctx); +__owur int ssl_cert_lookup_by_nid(int nid, size_t *pidx); __owur const SSL_CERT_LOOKUP *ssl_cert_lookup_by_pkey(const EVP_PKEY *pk, - size_t *pidx, - SSL_CTX *ctx); -__owur const SSL_CERT_LOOKUP *ssl_cert_lookup_by_idx(size_t idx, SSL_CTX *ctx); + size_t *pidx); +__owur const SSL_CERT_LOOKUP *ssl_cert_lookup_by_idx(size_t idx); int ssl_undefined_function(SSL *s); __owur int ssl_undefined_void_function(void); __owur int ssl_undefined_const_function(const SSL *s); -__owur int ssl_get_server_cert_serverinfo(SSL_CONNECTION *s, +__owur int ssl_get_server_cert_serverinfo(SSL *s, const unsigned char **serverinfo, size_t *serverinfo_length); -void ssl_set_masks(SSL_CONNECTION *s); -__owur STACK_OF(SSL_CIPHER) *ssl_get_ciphers_by_id(SSL_CONNECTION *sc); +void ssl_set_masks(SSL *s); +__owur STACK_OF(SSL_CIPHER) *ssl_get_ciphers_by_id(SSL *s); __owur int ssl_x509err2alert(int type); void ssl_sort_cipher_list(void); int ssl_load_ciphers(SSL_CTX *ctx); -__owur int ssl_setup_sigalgs(SSL_CTX *ctx); +__owur int ssl_setup_sig_algs(SSL_CTX *ctx); int ssl_load_groups(SSL_CTX *ctx); -int ssl_load_sigalgs(SSL_CTX *ctx); -__owur int ssl_fill_hello_random(SSL_CONNECTION *s, int server, - unsigned char *field, size_t len, - DOWNGRADE dgrd); -__owur int ssl_generate_master_secret(SSL_CONNECTION *s, unsigned char *pms, - size_t pmslen, int free_pms); -__owur EVP_PKEY *ssl_generate_pkey(SSL_CONNECTION *s, EVP_PKEY *pm); -__owur int ssl_gensecret(SSL_CONNECTION *s, unsigned char *pms, size_t pmslen); -__owur int ssl_derive(SSL_CONNECTION *s, EVP_PKEY *privkey, EVP_PKEY *pubkey, +__owur int ssl_fill_hello_random(SSL *s, int server, unsigned char *field, + size_t len, DOWNGRADE dgrd); +__owur int ssl_generate_master_secret(SSL *s, unsigned char *pms, size_t pmslen, + int free_pms); +__owur EVP_PKEY *ssl_generate_pkey(SSL *s, EVP_PKEY *pm); +__owur int ssl_gensecret(SSL *s, unsigned char *pms, size_t pmslen); +__owur int ssl_derive(SSL *s, EVP_PKEY *privkey, EVP_PKEY *pubkey, int genmaster); -__owur int ssl_decapsulate(SSL_CONNECTION *s, EVP_PKEY *privkey, +__owur int ssl_decapsulate(SSL *s, EVP_PKEY *privkey, const unsigned char *ct, size_t ctlen, int gensecret); -__owur int ssl_encapsulate(SSL_CONNECTION *s, EVP_PKEY *pubkey, +__owur int ssl_encapsulate(SSL *s, EVP_PKEY *pubkey, unsigned char **ctp, size_t *ctlenp, int gensecret); __owur EVP_PKEY *ssl_dh_to_pkey(DH *dh); __owur int ssl_set_tmp_ecdh_groups(uint16_t **pext, size_t *pextlen, void *key); -__owur unsigned int ssl_get_max_send_fragment(const SSL_CONNECTION *sc); -__owur unsigned int ssl_get_split_send_fragment(const SSL_CONNECTION *sc); +__owur unsigned int ssl_get_max_send_fragment(const SSL *ssl); +__owur unsigned int ssl_get_split_send_fragment(const SSL *ssl); __owur const SSL_CIPHER *ssl3_get_cipher_by_id(uint32_t id); __owur const SSL_CIPHER *ssl3_get_cipher_by_std_name(const char *stdname); __owur const SSL_CIPHER *ssl3_get_cipher_by_char(const unsigned char *p); __owur int ssl3_put_cipher_by_char(const SSL_CIPHER *c, WPACKET *pkt, size_t *len); -int ssl3_init_finished_mac(SSL_CONNECTION *s); -__owur int ssl3_setup_key_block(SSL_CONNECTION *s); -__owur int ssl3_change_cipher_state(SSL_CONNECTION *s, int which); -void ssl3_cleanup_key_block(SSL_CONNECTION *s); -__owur int ssl3_do_write(SSL_CONNECTION *s, uint8_t type); -int ssl3_send_alert(SSL_CONNECTION *s, int level, int desc); -__owur int ssl3_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, +int ssl3_init_finished_mac(SSL *s); +__owur int ssl3_setup_key_block(SSL *s); +__owur int ssl3_change_cipher_state(SSL *s, int which); +void ssl3_cleanup_key_block(SSL *s); +__owur int ssl3_do_write(SSL *s, int type); +int ssl3_send_alert(SSL *s, int level, int desc); +__owur int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p, size_t len, size_t *secret_size); -__owur int ssl3_get_req_cert_type(SSL_CONNECTION *s, WPACKET *pkt); +__owur int ssl3_get_req_cert_type(SSL *s, WPACKET *pkt); __owur int ssl3_num_ciphers(void); __owur const SSL_CIPHER *ssl3_get_cipher(unsigned int u); int ssl3_renegotiate(SSL *ssl); @@ -2597,17 +2670,16 @@ int ssl3_renegotiate_check(SSL *ssl, int initok); void ssl3_digest_master_key_set_params(const SSL_SESSION *session, OSSL_PARAM params[]); __owur int ssl3_dispatch_alert(SSL *s); -__owur size_t ssl3_final_finish_mac(SSL_CONNECTION *s, const char *sender, - size_t slen, unsigned char *p); -__owur int ssl3_finish_mac(SSL_CONNECTION *s, const unsigned char *buf, - size_t len); -void ssl3_free_digest_list(SSL_CONNECTION *s); -__owur unsigned long ssl3_output_cert_chain(SSL_CONNECTION *s, WPACKET *pkt, - CERT_PKEY *cpk, int for_comp); -__owur const SSL_CIPHER *ssl3_choose_cipher(SSL_CONNECTION *s, +__owur size_t ssl3_final_finish_mac(SSL *s, const char *sender, size_t slen, + unsigned char *p); +__owur int ssl3_finish_mac(SSL *s, const unsigned char *buf, size_t len); +void ssl3_free_digest_list(SSL *s); +__owur unsigned long ssl3_output_cert_chain(SSL *s, WPACKET *pkt, + CERT_PKEY *cpk); +__owur const SSL_CIPHER *ssl3_choose_cipher(SSL *ssl, STACK_OF(SSL_CIPHER) *clnt, STACK_OF(SSL_CIPHER) *srvr); -__owur int ssl3_digest_cached_records(SSL_CONNECTION *s, int keep); +__owur int ssl3_digest_cached_records(SSL *s, int keep); __owur int ssl3_new(SSL *s); void ssl3_free(SSL *s); __owur int ssl3_read(SSL *s, void *buf, size_t len, size_t *readbytes); @@ -2620,65 +2692,62 @@ __owur long ssl3_ctx_ctrl(SSL_CTX *s, int cmd, long larg, void *parg); __owur long ssl3_callback_ctrl(SSL *s, int cmd, void (*fp) (void)); __owur long ssl3_ctx_callback_ctrl(SSL_CTX *s, int cmd, void (*fp) (void)); -__owur int ssl3_do_change_cipher_spec(SSL_CONNECTION *s); -__owur OSSL_TIME ssl3_default_timeout(void); +__owur int ssl3_do_change_cipher_spec(SSL *ssl); +__owur long ssl3_default_timeout(void); -__owur int ssl3_set_handshake_header(SSL_CONNECTION *s, WPACKET *pkt, - int htype); -__owur int tls_close_construct_packet(SSL_CONNECTION *s, WPACKET *pkt, int htype); -__owur int tls_setup_handshake(SSL_CONNECTION *s); -__owur int dtls1_set_handshake_header(SSL_CONNECTION *s, WPACKET *pkt, int htype); -__owur int dtls1_close_construct_packet(SSL_CONNECTION *s, WPACKET *pkt, int htype); -__owur int ssl3_handshake_write(SSL_CONNECTION *s); +__owur int ssl3_set_handshake_header(SSL *s, WPACKET *pkt, int htype); +__owur int tls_close_construct_packet(SSL *s, WPACKET *pkt, int htype); +__owur int tls_setup_handshake(SSL *s); +__owur int dtls1_set_handshake_header(SSL *s, WPACKET *pkt, int htype); +__owur int dtls1_close_construct_packet(SSL *s, WPACKET *pkt, int htype); +__owur int ssl3_handshake_write(SSL *s); -__owur int ssl_allow_compression(SSL_CONNECTION *s); +__owur int ssl_allow_compression(SSL *s); -__owur int ssl_version_cmp(const SSL_CONNECTION *s, int versiona, int versionb); -__owur int ssl_version_supported(const SSL_CONNECTION *s, int version, +__owur int ssl_version_supported(const SSL *s, int version, const SSL_METHOD **meth); -__owur int ssl_set_client_hello_version(SSL_CONNECTION *s); -__owur int ssl_check_version_downgrade(SSL_CONNECTION *s); +__owur int ssl_set_client_hello_version(SSL *s); +__owur int ssl_check_version_downgrade(SSL *s); __owur int ssl_set_version_bound(int method_version, int version, int *bound); -__owur int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, +__owur int ssl_choose_server_version(SSL *s, CLIENTHELLO_MSG *hello, DOWNGRADE *dgrd); -__owur int ssl_choose_client_version(SSL_CONNECTION *s, int version, +__owur int ssl_choose_client_version(SSL *s, int version, RAW_EXTENSION *extensions); -__owur int ssl_get_min_max_version(const SSL_CONNECTION *s, int *min_version, +__owur int ssl_get_min_max_version(const SSL *s, int *min_version, int *max_version, int *real_max); -__owur OSSL_TIME tls1_default_timeout(void); -__owur int dtls1_do_write(SSL_CONNECTION *s, uint8_t type); -void dtls1_set_message_header(SSL_CONNECTION *s, +__owur long tls1_default_timeout(void); +__owur int dtls1_do_write(SSL *s, int type); +void dtls1_set_message_header(SSL *s, unsigned char mt, size_t len, size_t frag_off, size_t frag_len); -int dtls1_write_app_data_bytes(SSL *s, uint8_t type, const void *buf_, - size_t len, size_t *written); +int dtls1_write_app_data_bytes(SSL *s, int type, const void *buf_, size_t len, + size_t *written); -__owur int dtls1_read_failed(SSL_CONNECTION *s, int code); -__owur int dtls1_buffer_message(SSL_CONNECTION *s, int ccs); -__owur int dtls1_retransmit_message(SSL_CONNECTION *s, unsigned short seq, - int *found); +__owur int dtls1_read_failed(SSL *s, int code); +__owur int dtls1_buffer_message(SSL *s, int ccs); +__owur int dtls1_retransmit_message(SSL *s, unsigned short seq, int *found); __owur int dtls1_get_queue_priority(unsigned short seq, int is_ccs); -int dtls1_retransmit_buffered_messages(SSL_CONNECTION *s); -void dtls1_clear_received_buffer(SSL_CONNECTION *s); -void dtls1_clear_sent_buffer(SSL_CONNECTION *s); -void dtls1_get_message_header(const unsigned char *data, +int dtls1_retransmit_buffered_messages(SSL *s); +void dtls1_clear_received_buffer(SSL *s); +void dtls1_clear_sent_buffer(SSL *s); +void dtls1_get_message_header(unsigned char *data, struct hm_header_st *msg_hdr); -__owur OSSL_TIME dtls1_default_timeout(void); -__owur int dtls1_get_timeout(const SSL_CONNECTION *s, OSSL_TIME *timeleft); -__owur int dtls1_check_timeout_num(SSL_CONNECTION *s); -__owur int dtls1_handle_timeout(SSL_CONNECTION *s); -void dtls1_start_timer(SSL_CONNECTION *s); -void dtls1_stop_timer(SSL_CONNECTION *s); -__owur int dtls1_is_timer_expired(SSL_CONNECTION *s); +__owur long dtls1_default_timeout(void); +__owur struct timeval *dtls1_get_timeout(SSL *s, struct timeval *timeleft); +__owur int dtls1_check_timeout_num(SSL *s); +__owur int dtls1_handle_timeout(SSL *s); +void dtls1_start_timer(SSL *s); +void dtls1_stop_timer(SSL *s); +__owur int dtls1_is_timer_expired(SSL *s); __owur int dtls_raw_hello_verify_request(WPACKET *pkt, unsigned char *cookie, size_t cookie_len); -__owur size_t dtls1_min_mtu(SSL_CONNECTION *s); +__owur size_t dtls1_min_mtu(SSL *s); void dtls1_hm_fragment_free(hm_fragment *frag); -__owur int dtls1_query_mtu(SSL_CONNECTION *s); +__owur int dtls1_query_mtu(SSL *s); __owur int tls1_new(SSL *s); void tls1_free(SSL *s); @@ -2692,161 +2761,152 @@ __owur int dtls1_shutdown(SSL *s); __owur int dtls1_dispatch_alert(SSL *s); -__owur int ssl_init_wbio_buffer(SSL_CONNECTION *s); -int ssl_free_wbio_buffer(SSL_CONNECTION *s); +__owur int ssl_init_wbio_buffer(SSL *s); +int ssl_free_wbio_buffer(SSL *s); -__owur int tls1_change_cipher_state(SSL_CONNECTION *s, int which); -__owur int tls1_setup_key_block(SSL_CONNECTION *s); -__owur size_t tls1_final_finish_mac(SSL_CONNECTION *s, const char *str, - size_t slen, unsigned char *p); -__owur int tls1_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, +__owur int tls1_change_cipher_state(SSL *s, int which); +__owur int tls1_setup_key_block(SSL *s); +__owur size_t tls1_final_finish_mac(SSL *s, const char *str, size_t slen, + unsigned char *p); +__owur int tls1_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p, size_t len, size_t *secret_size); -__owur int tls13_setup_key_block(SSL_CONNECTION *s); -__owur size_t tls13_final_finish_mac(SSL_CONNECTION *s, const char *str, size_t slen, +__owur int tls13_setup_key_block(SSL *s); +__owur size_t tls13_final_finish_mac(SSL *s, const char *str, size_t slen, unsigned char *p); -__owur int tls13_change_cipher_state(SSL_CONNECTION *s, int which); -__owur int tls13_update_key(SSL_CONNECTION *s, int send); -__owur int tls13_hkdf_expand(SSL_CONNECTION *s, - const EVP_MD *md, +__owur int tls13_change_cipher_state(SSL *s, int which); +__owur int tls13_update_key(SSL *s, int send); +__owur int tls13_hkdf_expand(SSL *s, const EVP_MD *md, const unsigned char *secret, const unsigned char *label, size_t labellen, const unsigned char *data, size_t datalen, unsigned char *out, size_t outlen, int fatal); -__owur int tls13_hkdf_expand_ex(OSSL_LIB_CTX *libctx, const char *propq, - const EVP_MD *md, - const unsigned char *secret, - const unsigned char *label, size_t labellen, - const unsigned char *data, size_t datalen, - unsigned char *out, size_t outlen, - int raise_error); -__owur int tls13_derive_key(SSL_CONNECTION *s, const EVP_MD *md, +__owur int tls13_derive_key(SSL *s, const EVP_MD *md, const unsigned char *secret, unsigned char *key, size_t keylen); -__owur int tls13_derive_iv(SSL_CONNECTION *s, const EVP_MD *md, +__owur int tls13_derive_iv(SSL *s, const EVP_MD *md, const unsigned char *secret, unsigned char *iv, size_t ivlen); -__owur int tls13_derive_finishedkey(SSL_CONNECTION *s, const EVP_MD *md, +__owur int tls13_derive_finishedkey(SSL *s, const EVP_MD *md, const unsigned char *secret, unsigned char *fin, size_t finlen); -int tls13_generate_secret(SSL_CONNECTION *s, const EVP_MD *md, +int tls13_generate_secret(SSL *s, const EVP_MD *md, const unsigned char *prevsecret, const unsigned char *insecret, size_t insecretlen, unsigned char *outsecret); -__owur int tls13_generate_handshake_secret(SSL_CONNECTION *s, +__owur int tls13_generate_handshake_secret(SSL *s, const unsigned char *insecret, size_t insecretlen); -__owur int tls13_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, +__owur int tls13_generate_master_secret(SSL *s, unsigned char *out, unsigned char *prev, size_t prevlen, size_t *secret_size); -__owur int tls1_export_keying_material(SSL_CONNECTION *s, - unsigned char *out, size_t olen, +__owur int tls1_export_keying_material(SSL *s, unsigned char *out, size_t olen, const char *label, size_t llen, const unsigned char *p, size_t plen, int use_context); -__owur int tls13_export_keying_material(SSL_CONNECTION *s, - unsigned char *out, size_t olen, +__owur int tls13_export_keying_material(SSL *s, unsigned char *out, size_t olen, const char *label, size_t llen, const unsigned char *context, size_t contextlen, int use_context); -__owur int tls13_export_keying_material_early(SSL_CONNECTION *s, - unsigned char *out, size_t olen, - const char *label, size_t llen, +__owur int tls13_export_keying_material_early(SSL *s, unsigned char *out, + size_t olen, const char *label, + size_t llen, const unsigned char *context, size_t contextlen); __owur int tls1_alert_code(int code); __owur int tls13_alert_code(int code); __owur int ssl3_alert_code(int code); +#ifndef OPENSSL_NO_NTLS +__owur int ntls_alert_code(int code); +int tls_choose_sigalg_ntls(SSL *s, int fatalerrs); +#endif -__owur int ssl_check_srvr_ecc_cert_and_alg(X509 *x, SSL_CONNECTION *s); +__owur int ssl_check_srvr_ecc_cert_and_alg(X509 *x, SSL *s); SSL_COMP *ssl3_comp_find(STACK_OF(SSL_COMP) *sk, int n); __owur const TLS_GROUP_INFO *tls1_group_id_lookup(SSL_CTX *ctx, uint16_t curve_id); -__owur const char *tls1_group_id2name(SSL_CTX *ctx, uint16_t group_id); __owur int tls1_group_id2nid(uint16_t group_id, int include_unknown); __owur uint16_t tls1_nid2group_id(int nid); -__owur int tls1_check_group_id(SSL_CONNECTION *s, uint16_t group_id, - int check_own_curves); -__owur uint16_t tls1_shared_group(SSL_CONNECTION *s, int nmatch); +__owur int tls1_check_group_id(SSL *s, uint16_t group_id, int check_own_curves); +__owur uint16_t tls1_shared_group(SSL *s, int nmatch); __owur int tls1_set_groups(uint16_t **pext, size_t *pextlen, int *curves, size_t ncurves); __owur int tls1_set_groups_list(SSL_CTX *ctx, uint16_t **pext, size_t *pextlen, const char *str); -__owur EVP_PKEY *ssl_generate_pkey_group(SSL_CONNECTION *s, uint16_t id); -__owur int tls_valid_group(SSL_CONNECTION *s, uint16_t group_id, int minversion, +__owur EVP_PKEY *ssl_generate_pkey_group(SSL *s, uint16_t id); +__owur int tls_valid_group(SSL *s, uint16_t group_id, int minversion, int maxversion, int isec, int *okfortls13); -__owur EVP_PKEY *ssl_generate_param_group(SSL_CONNECTION *s, uint16_t id); -void tls1_get_formatlist(SSL_CONNECTION *s, const unsigned char **pformats, +__owur EVP_PKEY *ssl_generate_param_group(SSL *s, uint16_t id); +void tls1_get_formatlist(SSL *s, const unsigned char **pformats, size_t *num_formats); -__owur int tls1_check_ec_tmp_key(SSL_CONNECTION *s, unsigned long id); +__owur int tls1_check_ec_tmp_key(SSL *s, unsigned long id); -__owur int tls_group_allowed(SSL_CONNECTION *s, uint16_t curve, int op); -void tls1_get_supported_groups(SSL_CONNECTION *s, const uint16_t **pgroups, +__owur int tls_group_allowed(SSL *s, uint16_t curve, int op); +void tls1_get_supported_groups(SSL *s, const uint16_t **pgroups, size_t *pgroupslen); -__owur int tls1_set_server_sigalgs(SSL_CONNECTION *s); +__owur int tls1_set_server_sigalgs(SSL *s); -__owur SSL_TICKET_STATUS tls_get_ticket_from_client(SSL_CONNECTION *s, - CLIENTHELLO_MSG *hello, +__owur SSL_TICKET_STATUS tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello, SSL_SESSION **ret); -__owur SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, - const unsigned char *etick, +__owur SSL_TICKET_STATUS tls_decrypt_ticket(SSL *s, const unsigned char *etick, size_t eticklen, const unsigned char *sess_id, size_t sesslen, SSL_SESSION **psess); -__owur int tls_use_ticket(SSL_CONNECTION *s); +__owur int tls_use_ticket(SSL *s); -void ssl_set_sig_mask(uint32_t *pmask_a, SSL_CONNECTION *s, int op); +void ssl_set_sig_mask(uint32_t *pmask_a, SSL *s, int op); -__owur int tls1_set_sigalgs_list(SSL_CTX *ctx, CERT *c, const char *str, int client); +__owur int tls1_set_sigalgs_list(CERT *c, const char *str, int client); __owur int tls1_set_raw_sigalgs(CERT *c, const uint16_t *psigs, size_t salglen, int client); __owur int tls1_set_sigalgs(CERT *c, const int *salg, size_t salglen, int client); -int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, - STACK_OF(X509) *chain, int idx); -void tls1_set_cert_validity(SSL_CONNECTION *s); +int tls1_check_chain(SSL *s, X509 *x, EVP_PKEY *pk, STACK_OF(X509) *chain, + int idx); +void tls1_set_cert_validity(SSL *s); # ifndef OPENSSL_NO_CT -__owur int ssl_validate_ct(SSL_CONNECTION *s); +__owur int ssl_validate_ct(SSL *s); # endif -__owur EVP_PKEY *ssl_get_auto_dh(SSL_CONNECTION *s); +__owur EVP_PKEY *ssl_get_auto_dh(SSL *s); -__owur int ssl_security_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x, int vfy, - int is_ee); -__owur int ssl_security_cert_chain(SSL_CONNECTION *s, STACK_OF(X509) *sk, - X509 *ex, int vfy); +__owur int ssl_security_cert(SSL *s, SSL_CTX *ctx, X509 *x, int vfy, int is_ee); +__owur int ssl_security_cert_chain(SSL *s, STACK_OF(X509) *sk, X509 *ex, + int vfy); -int tls_choose_sigalg(SSL_CONNECTION *s, int fatalerrs); +int tls_choose_sigalg(SSL *s, int fatalerrs); -__owur long ssl_get_algorithm2(SSL_CONNECTION *s); -__owur int tls12_copy_sigalgs(SSL_CONNECTION *s, WPACKET *pkt, +__owur EVP_MD_CTX *ssl_replace_hash(EVP_MD_CTX **hash, const EVP_MD *md); +void ssl_clear_hash_ctx(EVP_MD_CTX **hash); +__owur long ssl_get_algorithm2(SSL *s); +__owur int tls12_copy_sigalgs(SSL *s, WPACKET *pkt, const uint16_t *psig, size_t psiglen); __owur int tls1_save_u16(PACKET *pkt, uint16_t **pdest, size_t *pdestlen); -__owur int tls1_save_sigalgs(SSL_CONNECTION *s, PACKET *pkt, int cert); -__owur int tls1_process_sigalgs(SSL_CONNECTION *s); -__owur int tls1_set_peer_legacy_sigalg(SSL_CONNECTION *s, const EVP_PKEY *pkey); +__owur int tls1_save_sigalgs(SSL *s, PACKET *pkt, int cert); +__owur int tls1_process_sigalgs(SSL *s); +__owur int tls1_set_peer_legacy_sigalg(SSL *s, const EVP_PKEY *pkey); __owur int tls1_lookup_md(SSL_CTX *ctx, const SIGALG_LOOKUP *lu, const EVP_MD **pmd); -__owur size_t tls12_get_psigalgs(SSL_CONNECTION *s, int sent, - const uint16_t **psigs); -__owur int tls_check_sigalg_curve(const SSL_CONNECTION *s, int curve); -__owur int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t, EVP_PKEY *pkey); -__owur int ssl_set_client_disabled(SSL_CONNECTION *s); -__owur int ssl_cipher_disabled(const SSL_CONNECTION *s, const SSL_CIPHER *c, - int op, int echde); - -__owur int ssl_handshake_hash(SSL_CONNECTION *s, - unsigned char *out, size_t outlen, - size_t *hashlen); +__owur const SIGALG_LOOKUP *ssl_sigalg_lookup(uint16_t sigalg); +__owur const SIGALG_LOOKUP *ssl_sigalg_lookup_by_pkey_and_hash(EVP_PKEY *pkey, + int hash, + int is_tls13); +__owur size_t tls12_get_psigalgs(SSL *s, int sent, const uint16_t **psigs); +__owur int tls_check_sigalg_curve(const SSL *s, int curve); +__owur int tls12_check_peer_sigalg(SSL *s, uint16_t, EVP_PKEY *pkey); +__owur int ssl_set_client_disabled(SSL *s); +__owur int ssl_cipher_disabled(const SSL *s, const SSL_CIPHER *c, int op, int echde); + +__owur int ssl_handshake_hash(SSL *s, unsigned char *out, size_t outlen, + size_t *hashlen); __owur const EVP_MD *ssl_md(SSL_CTX *ctx, int idx); -int ssl_get_md_idx(int md_nid); -__owur const EVP_MD *ssl_handshake_md(SSL_CONNECTION *s); -__owur const EVP_MD *ssl_prf_md(SSL_CONNECTION *s); +__owur const EVP_MD *ssl_handshake_md(SSL *s); +__owur const EVP_MD *ssl_prf_md(SSL *s); /* * ssl_log_rsa_client_key_exchange logs |premaster| to the SSL_CTX associated @@ -2854,7 +2914,7 @@ __owur const EVP_MD *ssl_prf_md(SSL_CONNECTION *s); * failure. The entry is identified by the first 8 bytes of * |encrypted_premaster|. */ -__owur int ssl_log_rsa_client_key_exchange(SSL_CONNECTION *s, +__owur int ssl_log_rsa_client_key_exchange(SSL *ssl, const uint8_t *encrypted_premaster, size_t encrypted_premaster_len, const uint8_t *premaster, @@ -2865,7 +2925,7 @@ __owur int ssl_log_rsa_client_key_exchange(SSL_CONNECTION *s, * logging is available. It returns one on success and zero on failure. It tags * the entry with |label|. */ -__owur int ssl_log_secret(SSL_CONNECTION *s, const char *label, +__owur int ssl_log_secret(SSL *ssl, const char *label, const uint8_t *secret, size_t secret_len); #define MASTER_SECRET_LABEL "CLIENT_RANDOM" @@ -2873,19 +2933,40 @@ __owur int ssl_log_secret(SSL_CONNECTION *s, const char *label, #define CLIENT_HANDSHAKE_LABEL "CLIENT_HANDSHAKE_TRAFFIC_SECRET" #define SERVER_HANDSHAKE_LABEL "SERVER_HANDSHAKE_TRAFFIC_SECRET" #define CLIENT_APPLICATION_LABEL "CLIENT_TRAFFIC_SECRET_0" -#define CLIENT_APPLICATION_N_LABEL "CLIENT_TRAFFIC_SECRET_N" #define SERVER_APPLICATION_LABEL "SERVER_TRAFFIC_SECRET_0" -#define SERVER_APPLICATION_N_LABEL "SERVER_TRAFFIC_SECRET_N" #define EARLY_EXPORTER_SECRET_LABEL "EARLY_EXPORTER_SECRET" #define EXPORTER_SECRET_LABEL "EXPORTER_SECRET" -__owur int srp_generate_server_master_secret(SSL_CONNECTION *s); -__owur int srp_generate_client_master_secret(SSL_CONNECTION *s); -__owur int srp_verify_server_param(SSL_CONNECTION *s); +# ifndef OPENSSL_NO_KTLS +/* ktls.c */ +int ktls_check_supported_cipher(const SSL *s, const EVP_CIPHER *c, + const EVP_CIPHER_CTX *dd); +int ktls_configure_crypto(const SSL *s, const EVP_CIPHER *c, EVP_CIPHER_CTX *dd, + void *rl_sequence, ktls_crypto_info_t *crypto_info, + unsigned char **rec_seq, unsigned char *iv, + unsigned char *key, unsigned char *mac_key, + size_t mac_secret_size); +# endif + +/* s3_cbc.c */ +__owur char ssl3_cbc_record_digest_supported(const EVP_MD_CTX *ctx); +__owur int ssl3_cbc_digest_record(const EVP_MD *md, + unsigned char *md_out, + size_t *md_out_size, + const unsigned char *header, + const unsigned char *data, + size_t data_size, + size_t data_plus_mac_plus_padding_size, + const unsigned char *mac_secret, + size_t mac_secret_length, char is_sslv3); + +__owur int srp_generate_server_master_secret(SSL *s); +__owur int srp_generate_client_master_secret(SSL *s); +__owur int srp_verify_server_param(SSL *s); /* statem/statem_srvr.c */ -__owur int send_certificate_request(SSL_CONNECTION *s); +__owur int send_certificate_request(SSL *s); /* statem/extensions_cust.c */ @@ -2895,19 +2976,10 @@ custom_ext_method *custom_ext_find(const custom_ext_methods *exts, void custom_ext_init(custom_ext_methods *meths); -int ossl_tls_add_custom_ext_intern(SSL_CTX *ctx, custom_ext_methods *exts, - ENDPOINT role, unsigned int ext_type, - unsigned int context, - SSL_custom_ext_add_cb_ex add_cb, - SSL_custom_ext_free_cb_ex free_cb, - void *add_arg, - SSL_custom_ext_parse_cb_ex parse_cb, - void *parse_arg); -__owur int custom_ext_parse(SSL_CONNECTION *s, unsigned int context, - unsigned int ext_type, +__owur int custom_ext_parse(SSL *s, unsigned int context, unsigned int ext_type, const unsigned char *ext_data, size_t ext_size, X509 *x, size_t chainidx); -__owur int custom_ext_add(SSL_CONNECTION *s, int context, WPACKET *pkt, X509 *x, +__owur int custom_ext_add(SSL *s, int context, WPACKET *pkt, X509 *x, size_t chainidx, int maxversion); __owur int custom_exts_copy(custom_ext_methods *dst, @@ -2921,6 +2993,10 @@ void ssl_comp_free_compression_methods_int(void); /* ssl_mcnf.c */ void ssl_ctx_system_config(SSL_CTX *ctx); +# ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +int tls1_set_shared_dc_sigalgs(SSL *s); +# endif + const EVP_CIPHER *ssl_evp_cipher_fetch(OSSL_LIB_CTX *libctx, int nid, const char *properties); @@ -2932,11 +3008,14 @@ const EVP_MD *ssl_evp_md_fetch(OSSL_LIB_CTX *libctx, int ssl_evp_md_up_ref(const EVP_MD *md); void ssl_evp_md_free(const EVP_MD *md); +int tls_provider_set_tls_params(SSL *s, EVP_CIPHER_CTX *ctx, + const EVP_CIPHER *ciph, + const EVP_MD *md); + void tls_engine_finish(ENGINE *e); const EVP_CIPHER *tls_get_cipher_from_engine(int nid); const EVP_MD *tls_get_digest_from_engine(int nid); -int tls_engine_load_ssl_client_cert(SSL_CONNECTION *s, X509 **px509, - EVP_PKEY **ppkey); +int tls_engine_load_ssl_client_cert(SSL *s, X509 **px509, EVP_PKEY **ppkey); int ssl_hmac_old_new(SSL_HMAC *ret); void ssl_hmac_old_free(SSL_HMAC *ctx); int ssl_hmac_old_init(SSL_HMAC *ctx, void *key, size_t len, char *md); @@ -2946,17 +3025,18 @@ size_t ssl_hmac_old_size(const SSL_HMAC *ctx); int ssl_ctx_srp_ctx_free_intern(SSL_CTX *ctx); int ssl_ctx_srp_ctx_init_intern(SSL_CTX *ctx); -int ssl_srp_ctx_free_intern(SSL_CONNECTION *s); -int ssl_srp_ctx_init_intern(SSL_CONNECTION *s); +int ssl_srp_ctx_free_intern(SSL *s); +int ssl_srp_ctx_init_intern(SSL *s); -int ssl_srp_calc_a_param_intern(SSL_CONNECTION *s); -int ssl_srp_server_param_with_username_intern(SSL_CONNECTION *s, int *ad); +int ssl_srp_calc_a_param_intern(SSL *s); +int ssl_srp_server_param_with_username_intern(SSL *s, int *ad); -void ssl_session_calculate_timeout(SSL_SESSION *ss); +void ssl_session_calculate_timeout(SSL_SESSION* ss); # else /* OPENSSL_UNIT_TEST */ # define ssl_init_wbio_buffer SSL_test_functions()->p_ssl_init_wbio_buffer +# define ssl3_setup_buffers SSL_test_functions()->p_ssl3_setup_buffers # endif @@ -2986,86 +3066,4 @@ static ossl_unused ossl_inline void ssl_tsan_counter(const SSL_CTX *ctx, } } -int ossl_comp_has_alg(int a); -size_t ossl_calculate_comp_expansion(int alg, size_t length); - -void ossl_ssl_set_custom_record_layer(SSL_CONNECTION *s, - const OSSL_RECORD_METHOD *meth, - void *rlarg); - -long ossl_ctrl_internal(SSL *s, int cmd, long larg, void *parg, int no_quic); - -/* - * Options which no longer have any effect, but which can be implemented - * as no-ops for QUIC. - */ -#define OSSL_LEGACY_SSL_OPTIONS \ - (SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG | \ - SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER | \ - SSL_OP_SSLEAY_080_CLIENT_DH_BUG | \ - SSL_OP_TLS_D5_BUG | \ - SSL_OP_TLS_BLOCK_PADDING_BUG | \ - SSL_OP_MSIE_SSLV2_RSA_PADDING | \ - SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG | \ - SSL_OP_MICROSOFT_SESS_ID_BUG | \ - SSL_OP_NETSCAPE_CHALLENGE_BUG | \ - SSL_OP_PKCS1_CHECK_1 | \ - SSL_OP_PKCS1_CHECK_2 | \ - SSL_OP_SINGLE_DH_USE | \ - SSL_OP_SINGLE_ECDH_USE | \ - SSL_OP_EPHEMERAL_RSA ) - -/* This option is undefined in public headers with no-dtls1-method. */ -#ifndef SSL_OP_CISCO_ANYCONNECT -# define SSL_OP_CISCO_ANYCONNECT 0 -#endif -/* - * Options which are no-ops under QUIC or TLSv1.3 and which are therefore - * allowed but ignored under QUIC. - */ -#define OSSL_TLS1_2_OPTIONS \ - (SSL_OP_CRYPTOPRO_TLSEXT_BUG | \ - SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS | \ - SSL_OP_ALLOW_CLIENT_RENEGOTIATION | \ - SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION | \ - SSL_OP_NO_COMPRESSION | \ - SSL_OP_NO_SSLv3 | \ - SSL_OP_NO_TLSv1 | \ - SSL_OP_NO_TLSv1_1 | \ - SSL_OP_NO_TLSv1_2 | \ - SSL_OP_NO_DTLSv1 | \ - SSL_OP_NO_DTLSv1_2 | \ - SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | \ - SSL_OP_CISCO_ANYCONNECT | \ - SSL_OP_NO_RENEGOTIATION | \ - SSL_OP_NO_EXTENDED_MASTER_SECRET | \ - SSL_OP_NO_ENCRYPT_THEN_MAC | \ - SSL_OP_COOKIE_EXCHANGE | \ - SSL_OP_LEGACY_SERVER_CONNECT | \ - SSL_OP_IGNORE_UNEXPECTED_EOF ) - -/* Total mask of connection-level options permitted or ignored under QUIC. */ -#define OSSL_QUIC_PERMITTED_OPTIONS_CONN \ - (OSSL_LEGACY_SSL_OPTIONS | \ - OSSL_TLS1_2_OPTIONS | \ - SSL_OP_CIPHER_SERVER_PREFERENCE | \ - SSL_OP_DISABLE_TLSEXT_CA_NAMES | \ - SSL_OP_NO_TX_CERTIFICATE_COMPRESSION | \ - SSL_OP_NO_RX_CERTIFICATE_COMPRESSION | \ - SSL_OP_PRIORITIZE_CHACHA | \ - SSL_OP_NO_QUERY_MTU | \ - SSL_OP_NO_TICKET | \ - SSL_OP_NO_ANTI_REPLAY ) - -/* Total mask of stream-level options permitted or ignored under QUIC. */ -#define OSSL_QUIC_PERMITTED_OPTIONS_STREAM \ - (OSSL_LEGACY_SSL_OPTIONS | \ - OSSL_TLS1_2_OPTIONS | \ - SSL_OP_CLEANSE_PLAINTEXT ) - -/* Total mask of options permitted on either connections or streams. */ -#define OSSL_QUIC_PERMITTED_OPTIONS \ - (OSSL_QUIC_PERMITTED_OPTIONS_CONN | \ - OSSL_QUIC_PERMITTED_OPTIONS_STREAM) - #endif diff --git a/openssl/src/ssl/ssl_mcnf.c b/openssl/src/ssl/ssl_mcnf.c index 8bccce84d..c2366e41e 100644 --- a/openssl/src/ssl/ssl_mcnf.c +++ b/openssl/src/ssl/ssl_mcnf.c @@ -24,7 +24,7 @@ static int ssl_do_config(SSL *s, SSL_CTX *ctx, const char *name, int system) { SSL_CONF_CTX *cctx = NULL; size_t i, idx, cmd_count; - int err = 1; + int rv = 0; unsigned int flags; const SSL_METHOD *meth; const SSL_CONF_CMD *cmds; @@ -66,22 +66,24 @@ static int ssl_do_config(SSL *s, SSL_CTX *ctx, const char *name, int system) flags |= SSL_CONF_FLAG_CLIENT; SSL_CONF_CTX_set_flags(cctx, flags); prev_libctx = OSSL_LIB_CTX_set0_default(libctx); - err = 0; for (i = 0; i < cmd_count; i++) { char *cmdstr, *arg; - int rv; conf_ssl_get_cmd(cmds, i, &cmdstr, &arg); rv = SSL_CONF_cmd(cctx, cmdstr, arg); - if (rv <= 0) - ++err; + if (rv <= 0) { + int errcode = rv == -2 ? SSL_R_UNKNOWN_COMMAND : SSL_R_BAD_VALUE; + + ERR_raise_data(ERR_LIB_SSL, errcode, + "section=%s, cmd=%s, arg=%s", name, cmdstr, arg); + goto err; + } } - if (!SSL_CONF_CTX_finish(cctx)) - ++err; + rv = SSL_CONF_CTX_finish(cctx); err: OSSL_LIB_CTX_set0_default(prev_libctx); SSL_CONF_CTX_free(cctx); - return err == 0; + return rv <= 0 ? 0 : 1; } int SSL_config(SSL *s, const char *name) diff --git a/openssl/src/ssl/ssl_quic.c b/openssl/src/ssl/ssl_quic.c new file mode 100644 index 000000000..f2388d725 --- /dev/null +++ b/openssl/src/ssl/ssl_quic.c @@ -0,0 +1,395 @@ +/* + * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "ssl_local.h" +#include "internal/cryptlib.h" +#include "internal/refcount.h" + +#ifdef OPENSSL_NO_QUIC +NON_EMPTY_TRANSLATION_UNIT +#else + +int SSL_set_quic_transport_params(SSL *ssl, const uint8_t *params, + size_t params_len) +{ + uint8_t *tmp; + + if (params == NULL || params_len == 0) { + tmp = NULL; + params_len = 0; + } else { + tmp = OPENSSL_memdup(params, params_len); + if (tmp == NULL) + return 0; + } + + OPENSSL_free(ssl->ext.quic_transport_params); + ssl->ext.quic_transport_params = tmp; + ssl->ext.quic_transport_params_len = params_len; + return 1; +} + +void SSL_get_peer_quic_transport_params(const SSL *ssl, + const uint8_t **out_params, + size_t *out_params_len) +{ + if (ssl->ext.peer_quic_transport_params_len) { + *out_params = ssl->ext.peer_quic_transport_params; + *out_params_len = ssl->ext.peer_quic_transport_params_len; + } else { + *out_params = ssl->ext.peer_quic_transport_params_draft; + *out_params_len = ssl->ext.peer_quic_transport_params_draft_len; + } +} + +/* Returns the negotiated version, or -1 on error */ +int SSL_get_peer_quic_transport_version(const SSL *ssl) +{ + if (ssl->ext.peer_quic_transport_params_len != 0 + && ssl->ext.peer_quic_transport_params_draft_len != 0) + return -1; + if (ssl->ext.peer_quic_transport_params_len != 0) + return TLSEXT_TYPE_quic_transport_parameters; + if (ssl->ext.peer_quic_transport_params_draft_len != 0) + return TLSEXT_TYPE_quic_transport_parameters_draft; + + return -1; +} + +void SSL_set_quic_use_legacy_codepoint(SSL *ssl, int use_legacy) +{ + if (use_legacy) + ssl->quic_transport_version = TLSEXT_TYPE_quic_transport_parameters_draft; + else + ssl->quic_transport_version = TLSEXT_TYPE_quic_transport_parameters; +} + +void SSL_set_quic_transport_version(SSL *ssl, int version) +{ + ssl->quic_transport_version = version; +} + +int SSL_get_quic_transport_version(const SSL *ssl) +{ + return ssl->quic_transport_version; +} + +size_t SSL_quic_max_handshake_flight_len(const SSL *ssl, OSSL_ENCRYPTION_LEVEL level) +{ + /* + * Limits flights to 16K by default when there are no large + * (certificate-carrying) messages. + */ + static const size_t DEFAULT_FLIGHT_LIMIT = 16384; + + switch (level) { + case ssl_encryption_initial: + return DEFAULT_FLIGHT_LIMIT; + case ssl_encryption_early_data: + /* QUIC does not send EndOfEarlyData. */ + return 0; + case ssl_encryption_handshake: + if (ssl->server) { + /* + * Servers may receive Certificate message if configured to request + * client certificates. + */ + if ((ssl->verify_mode & SSL_VERIFY_PEER) + && ssl->max_cert_list > DEFAULT_FLIGHT_LIMIT) + return ssl->max_cert_list; + } else { + /* + * Clients may receive both Certificate message and a CertificateRequest + * message. + */ + if (2*ssl->max_cert_list > DEFAULT_FLIGHT_LIMIT) + return 2 * ssl->max_cert_list; + } + return DEFAULT_FLIGHT_LIMIT; + case ssl_encryption_application: + return DEFAULT_FLIGHT_LIMIT; + } + + return 0; +} + +OSSL_ENCRYPTION_LEVEL SSL_quic_read_level(const SSL *ssl) +{ + return ssl->quic_read_level; +} + +OSSL_ENCRYPTION_LEVEL SSL_quic_write_level(const SSL *ssl) +{ + return ssl->quic_write_level; +} + +int SSL_provide_quic_data(SSL *ssl, OSSL_ENCRYPTION_LEVEL level, + const uint8_t *data, size_t len) +{ + size_t l, offset; + + if (!SSL_IS_QUIC(ssl)) { + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return 0; + } + + /* Level can be different than the current read, but not less */ + if (level < ssl->quic_read_level + || (ssl->quic_input_data_tail != NULL && level < ssl->quic_input_data_tail->level) + || level < ssl->quic_latest_level_received) { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_ENCRYPTION_LEVEL_RECEIVED); + return 0; + } + + if (len == 0) + return 1; + + if (ssl->quic_buf == NULL) { + BUF_MEM *buf; + if ((buf = BUF_MEM_new()) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + return 0; + } + if (!BUF_MEM_grow(buf, SSL3_RT_MAX_PLAIN_LENGTH)) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + BUF_MEM_free(buf); + return 0; + } + ssl->quic_buf = buf; + /* We preallocated storage, but there's still no *data*. */ + ssl->quic_buf->length = 0; + buf = NULL; + } + + /* A TLS message must not cross an encryption level boundary */ + if (ssl->quic_buf->length != ssl->quic_next_record_start + && level != ssl->quic_latest_level_received) { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_ENCRYPTION_LEVEL_RECEIVED); + return 0; + } + ssl->quic_latest_level_received = level; + + offset = ssl->quic_buf->length; + if (!BUF_MEM_grow(ssl->quic_buf, offset + len)) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + return 0; + } + memcpy(ssl->quic_buf->data + offset, data, len); + + /* Split on handshake message boundaries */ + while (ssl->quic_buf->length > ssl->quic_next_record_start + + SSL3_HM_HEADER_LENGTH) { + QUIC_DATA *qd; + const uint8_t *p; + + /* TLS Handshake message header has 1-byte type and 3-byte length */ + p = (const uint8_t *)ssl->quic_buf->data + + ssl->quic_next_record_start + 1; + n2l3(p, l); + l += SSL3_HM_HEADER_LENGTH; + /* Don't allocate a QUIC_DATA if we don't have a full record */ + if (l > ssl->quic_buf->length - ssl->quic_next_record_start) + break; + + qd = OPENSSL_zalloc(sizeof(*qd)); + if (qd == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + return 0; + } + + qd->next = NULL; + qd->length = l; + qd->start = ssl->quic_next_record_start; + qd->level = level; + + if (ssl->quic_input_data_tail != NULL) + ssl->quic_input_data_tail->next = qd; + else + ssl->quic_input_data_head = qd; + ssl->quic_input_data_tail = qd; + ssl->quic_next_record_start += l; + } + + return 1; +} + +int SSL_CTX_set_quic_method(SSL_CTX *ctx, const SSL_QUIC_METHOD *quic_method) +{ + if (ctx->method->version != TLS_ANY_VERSION) + return 0; + ctx->quic_method = quic_method; + ctx->options &= ~SSL_OP_ENABLE_MIDDLEBOX_COMPAT; + return 1; +} + +int SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method) +{ + if (ssl->method->version != TLS_ANY_VERSION) + return 0; + ssl->quic_method = quic_method; + ssl->options &= ~SSL_OP_ENABLE_MIDDLEBOX_COMPAT; + return 1; +} + +int quic_set_encryption_secrets(SSL *s, OSSL_ENCRYPTION_LEVEL level) +{ + uint8_t *c2s_secret = NULL; + uint8_t *s2c_secret = NULL; + uint8_t *read_secret = NULL; + uint8_t *write_secret = NULL; + size_t len; + const EVP_MD *md; + const SSL_CIPHER *c = NULL; + + if (!SSL_IS_QUIC(s)) + return 1; + + /* secrets from the POV of the client */ + switch (level) { + case ssl_encryption_early_data: + c2s_secret = s->client_early_traffic_secret; + break; + case ssl_encryption_handshake: + c2s_secret = s->client_hand_traffic_secret; + s2c_secret = s->server_hand_traffic_secret; + break; + case ssl_encryption_application: + c2s_secret = s->client_app_traffic_secret; + s2c_secret = s->server_app_traffic_secret; + break; + default: + return 1; + } + + c = SSL_SESSION_get0_cipher(s->session); + + if (s->early_data_state == SSL_EARLY_DATA_CONNECTING + && s->max_early_data > 0 && s->session->ext.max_early_data == 0) { + /* + * If we are attempting to send early data, and we've decided to + * actually do it but max_early_data in s->session is 0 then we + * must be using an external PSK. + */ + if (!ossl_assert(s->psksession != NULL + && s->max_early_data + == s->psksession->ext.max_early_data)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + c = SSL_SESSION_get0_cipher(s->psksession); + } + + if (c == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + md = ssl_handshake_md(s); + if (md == NULL) { + md = SSL_CIPHER_get_handshake_digest(c); + } + + if ((len = EVP_MD_size(md)) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + read_secret = s->server ? c2s_secret : s2c_secret; + write_secret = s->server ? s2c_secret : c2s_secret; + + if (read_secret && + !s->quic_method->set_read_secret(s, level, c, read_secret, len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (write_secret && + !s->quic_method->set_write_secret(s, level, c, write_secret, len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +int SSL_process_quic_post_handshake(SSL *ssl) +{ + int ret; + + if (SSL_in_init(ssl) || !SSL_IS_QUIC(ssl)) { + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return 0; + } + + /* if there is no data, return success as BoringSSL */ + while (ssl->quic_input_data_head != NULL) { + /* + * This is always safe (we are sure to be at a record boundary) because + * SSL_read()/SSL_write() are never used for QUIC connections -- the + * application data is handled at the QUIC layer instead. + */ + ossl_statem_set_in_init(ssl, 1); + ret = ssl->handshake_func(ssl); + ossl_statem_set_in_init(ssl, 0); + + if (ret <= 0) + return 0; + } + return 1; +} + +int SSL_is_quic(SSL *ssl) +{ + return SSL_IS_QUIC(ssl); +} + +void SSL_set_quic_early_data_enabled(SSL *ssl, int enabled) +{ + if (!SSL_is_quic(ssl) || !SSL_in_before(ssl)) + return; + + if (!enabled) { + ssl->early_data_state = SSL_EARLY_DATA_NONE; + return; + } + + if (ssl->server) { + ssl->early_data_state = SSL_EARLY_DATA_ACCEPTING; + return; + } + + if ((ssl->session == NULL || ssl->session->ext.max_early_data == 0) + && ssl->psk_use_session_cb == NULL) + return; + + ssl->early_data_state = SSL_EARLY_DATA_CONNECTING; +} + +int SSL_set_quic_early_data_context(SSL *ssl, const uint8_t *context, + size_t context_len) + +{ + uint8_t *tmp; + + if (context == NULL || context_len == 0) { + tmp = NULL; + context_len = 0; + } else { + tmp = OPENSSL_memdup(context, context_len); + if (tmp == NULL) + return 0; + } + + OPENSSL_free(ssl->quic_early_data_context); + ssl->quic_early_data_context = tmp; + ssl->quic_early_data_context_len = context_len; + return 1; +} + +#endif diff --git a/openssl/src/ssl/ssl_rsa.c b/openssl/src/ssl/ssl_rsa.c index c245c2408..2265ec287 100644 --- a/openssl/src/ssl/ssl_rsa.c +++ b/openssl/src/ssl/ssl_rsa.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,37 +17,39 @@ #include #include -static int ssl_set_cert(CERT *c, X509 *x509, SSL_CTX *ctx); -static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey, SSL_CTX *ctx); +static int ssl_set_cert(CERT *c, X509 *x509); +static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey); +#ifndef OPENSSL_NO_NTLS +static int ssl_set_cert_idx(CERT *c, X509 *x, int i); +static int ssl_set_pkey_idx(CERT *c, EVP_PKEY *pkey, int i); +static int ssl_use_certificate_idx(SSL *ssl, X509 *x, int i); +static int ssl_use_certificate_file_ntls(SSL *ssl, const char *file, + int type, int tag); +static int ssl_use_PrivateKey_idx(SSL *ssl, EVP_PKEY *pkey, int i); +static int ssl_use_PrivateKey_file_ntls(SSL *ssl, const char *file, + int type, int tag); +#endif #define SYNTHV1CONTEXT (SSL_EXT_TLS1_2_AND_BELOW_ONLY \ | SSL_EXT_CLIENT_HELLO \ | SSL_EXT_TLS1_2_SERVER_HELLO \ | SSL_EXT_IGNORE_ON_RESUMPTION) -#define NAME_PREFIX1 "SERVERINFO FOR " -#define NAME_PREFIX2 "SERVERINFOV2 FOR " - int SSL_use_certificate(SSL *ssl, X509 *x) { int rv; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - if (x == NULL) { ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); return 0; } - rv = ssl_security_cert(sc, NULL, x, 0, 1); + rv = ssl_security_cert(ssl, NULL, x, 0, 1); if (rv != 1) { ERR_raise(ERR_LIB_SSL, rv); return 0; } - return ssl_set_cert(sc->cert, x, SSL_CONNECTION_GET_CTX(sc)); + return ssl_set_cert(ssl->cert, x); } int SSL_use_certificate_file(SSL *ssl, const char *file, int type) @@ -68,23 +70,22 @@ int SSL_use_certificate_file(SSL *ssl, const char *file, int type) goto end; } + if (type != SSL_FILETYPE_ASN1 && type != SSL_FILETYPE_PEM) { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } x = X509_new_ex(ssl->ctx->libctx, ssl->ctx->propq); if (x == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto end; } if (type == SSL_FILETYPE_ASN1) { j = ERR_R_ASN1_LIB; cert = d2i_X509_bio(in, &x); } else if (type == SSL_FILETYPE_PEM) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - goto end; - j = ERR_R_PEM_LIB; - cert = PEM_read_bio_X509(in, &x, sc->default_passwd_callback, - sc->default_passwd_callback_userdata); + cert = PEM_read_bio_X509(in, &x, ssl->default_passwd_callback, + ssl->default_passwd_callback_userdata); } else { ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); goto end; @@ -109,7 +110,7 @@ int SSL_use_certificate_ASN1(SSL *ssl, const unsigned char *d, int len) x = X509_new_ex(ssl->ctx->libctx, ssl->ctx->propq); if (x == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } @@ -124,11 +125,253 @@ int SSL_use_certificate_ASN1(SSL *ssl, const unsigned char *d, int len) return ret; } -static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey, SSL_CTX *ctx) +#ifndef OPENSSL_NO_NTLS +static int ssl_use_certificate_idx(SSL *ssl, X509 *x, int i) +{ + int rv; + if (x == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + rv = ssl_security_cert(ssl, NULL, x, 0, 1); + if (rv != 1) { + ERR_raise(ERR_LIB_SSL, rv); + return 0; + } + + return ssl_set_cert_idx(ssl->cert, x, i); +} + +static int ssl_use_certificate_file_ntls(SSL *ssl, const char *file, + int type, int tag) +{ + int i, j; + BIO *in; + int ret = 0; + X509 *x = NULL; + EVP_PKEY *pkey; + + in = BIO_new(BIO_s_file()); + if (in == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + goto end; + } + + if (BIO_read_filename(in, file) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); + goto end; + } + if (type == SSL_FILETYPE_ASN1) { + j = ERR_R_ASN1_LIB; + x = d2i_X509_bio(in, NULL); + } else if (type == SSL_FILETYPE_PEM) { + j = ERR_R_PEM_LIB; + x = PEM_read_bio_X509(in, NULL, ssl->default_passwd_callback, + ssl->default_passwd_callback_userdata); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + + if (x == NULL) { + ERR_raise(ERR_LIB_SSL, j); + goto end; + } + + pkey = X509_get0_pubkey(x); + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_X509_LIB); + goto end; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = (tag == SSL_SIGN_CERT) ? SSL_PKEY_SM2_SIGN : SSL_PKEY_SM2_ENC; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = (tag == SSL_SIGN_CERT) ? SSL_PKEY_RSA_SIGN : SSL_PKEY_RSA_ENC; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + goto end; + } + + ret = ssl_use_certificate_idx(ssl, x, i); + end: + X509_free(x); + BIO_free(in); + return ret; +} + +static int ssl_use_PrivateKey_idx(SSL *ssl, EVP_PKEY *pkey, int i) +{ + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + return ssl_set_pkey_idx(ssl->cert, pkey, i); +} + +static int ssl_use_PrivateKey_file_ntls(SSL *ssl, const char *file, + int type, int tag) +{ + int i, j, ret = 0; + BIO *in; + EVP_PKEY *pkey = NULL; + + in = BIO_new(BIO_s_file()); + if (in == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + goto end; + } + + if (BIO_read_filename(in, file) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); + goto end; + } + if (type == SSL_FILETYPE_PEM) { + j = ERR_R_PEM_LIB; + pkey = PEM_read_bio_PrivateKey(in, NULL, + ssl->default_passwd_callback, + ssl->default_passwd_callback_userdata); + } else if (type == SSL_FILETYPE_ASN1) { + j = ERR_R_ASN1_LIB; + pkey = d2i_PrivateKey_bio(in, NULL); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, j); + goto end; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = (tag == SSL_SIGN_CERT) ? SSL_PKEY_SM2_SIGN : SSL_PKEY_SM2_ENC; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = (tag == SSL_SIGN_CERT) ? SSL_PKEY_RSA_SIGN : SSL_PKEY_RSA_ENC; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + goto end; + } + + ret = ssl_use_PrivateKey_idx(ssl, pkey, i); + end: + EVP_PKEY_free(pkey); + BIO_free(in); + return ret; +} + +int SSL_use_sign_certificate(SSL *ssl, X509 *x) +{ + EVP_PKEY *pkey; + int i; + + pkey = X509_get0_pubkey(x); + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_X509_LIB); + return 0; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = SSL_PKEY_SM2_SIGN; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = SSL_PKEY_RSA_SIGN; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE); + return 0; + } + + return ssl_use_certificate_idx(ssl, x, i); +} + +int SSL_use_sign_certificate_file(SSL *ssl, const char *file, int type) +{ + return ssl_use_certificate_file_ntls(ssl, file, type, SSL_SIGN_CERT); +} + +int SSL_use_enc_certificate(SSL *ssl, X509 *x) +{ + EVP_PKEY *pkey; + int i; + + pkey = X509_get0_pubkey(x); + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_X509_LIB); + return 0; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = SSL_PKEY_SM2_ENC; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = SSL_PKEY_RSA_ENC; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + return 0; + } + + return ssl_use_certificate_idx(ssl, x, i); +} + +int SSL_use_enc_certificate_file(SSL *ssl, const char *file, int type) +{ + return ssl_use_certificate_file_ntls(ssl, file, type, SSL_ENC_CERT); +} + +int SSL_use_enc_PrivateKey(SSL *ssl, EVP_PKEY *pkey) +{ + int i; + + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = SSL_PKEY_SM2_ENC; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = SSL_PKEY_RSA_ENC; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + return 0; + } + + return ssl_use_PrivateKey_idx(ssl, pkey, i); +} + +int SSL_use_enc_PrivateKey_file(SSL *ssl, const char *file, int type) +{ + return ssl_use_PrivateKey_file_ntls(ssl, file, type, SSL_ENC_CERT); +} + +int SSL_use_sign_PrivateKey(SSL *ssl, EVP_PKEY *pkey) +{ + int i; + + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = SSL_PKEY_SM2_SIGN; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = SSL_PKEY_RSA_SIGN; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + return 0; + } + + return ssl_use_PrivateKey_idx(ssl, pkey, i); +} + +int SSL_use_sign_PrivateKey_file(SSL *ssl, const char *file, int type) +{ + return ssl_use_PrivateKey_file_ntls(ssl, file, type, SSL_SIGN_CERT); +} +#endif +static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey) { size_t i; - if (ssl_cert_lookup_by_pkey(pkey, &i, ctx) == NULL) { + if (ssl_cert_lookup_by_pkey(pkey, &i) == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE); return 0; } @@ -147,16 +390,12 @@ static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey, SSL_CTX *ctx) int SSL_use_PrivateKey(SSL *ssl, EVP_PKEY *pkey) { int ret; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; if (pkey == NULL) { ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); return 0; } - ret = ssl_set_pkey(sc->cert, pkey, SSL_CONNECTION_GET_CTX(sc)); + ret = ssl_set_pkey(ssl->cert, pkey); return ret; } @@ -177,15 +416,10 @@ int SSL_use_PrivateKey_file(SSL *ssl, const char *file, int type) goto end; } if (type == SSL_FILETYPE_PEM) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - goto end; - j = ERR_R_PEM_LIB; pkey = PEM_read_bio_PrivateKey_ex(in, NULL, - sc->default_passwd_callback, - sc->default_passwd_callback_userdata, + ssl->default_passwd_callback, + ssl->default_passwd_callback_userdata, ssl->ctx->libctx, ssl->ctx->propq); } else if (type == SSL_FILETYPE_ASN1) { @@ -239,10 +473,10 @@ int SSL_CTX_use_certificate(SSL_CTX *ctx, X509 *x) ERR_raise(ERR_LIB_SSL, rv); return 0; } - return ssl_set_cert(ctx->cert, x, ctx); + return ssl_set_cert(ctx->cert, x); } -static int ssl_set_cert(CERT *c, X509 *x, SSL_CTX *ctx) +static int ssl_set_cert(CERT *c, X509 *x) { EVP_PKEY *pkey; size_t i; @@ -253,7 +487,7 @@ static int ssl_set_cert(CERT *c, X509 *x, SSL_CTX *ctx) return 0; } - if (ssl_cert_lookup_by_pkey(pkey, &i, ctx) == NULL) { + if (ssl_cert_lookup_by_pkey(pkey, &i) == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE); return 0; } @@ -267,7 +501,6 @@ static int ssl_set_cert(CERT *c, X509 *x, SSL_CTX *ctx) /* * The return code from EVP_PKEY_copy_parameters is deliberately * ignored. Some EVP_PKEY types cannot do this. - * coverity[check_return] */ EVP_PKEY_copy_parameters(pkey, c->pkeys[i].privatekey); ERR_clear_error(); @@ -310,10 +543,13 @@ int SSL_CTX_use_certificate_file(SSL_CTX *ctx, const char *file, int type) ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); goto end; } - + if (type != SSL_FILETYPE_ASN1 && type != SSL_FILETYPE_PEM) { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } x = X509_new_ex(ctx->libctx, ctx->propq); if (x == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto end; } if (type == SSL_FILETYPE_ASN1) { @@ -323,9 +559,6 @@ int SSL_CTX_use_certificate_file(SSL_CTX *ctx, const char *file, int type) j = ERR_R_PEM_LIB; cert = PEM_read_bio_X509(in, &x, ctx->default_passwd_callback, ctx->default_passwd_callback_userdata); - } else { - ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); - goto end; } if (cert == NULL) { ERR_raise(ERR_LIB_SSL, j); @@ -346,7 +579,7 @@ int SSL_CTX_use_certificate_ASN1(SSL_CTX *ctx, int len, const unsigned char *d) x = X509_new_ex(ctx->libctx, ctx->propq); if (x == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; } @@ -367,7 +600,7 @@ int SSL_CTX_use_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey) ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); return 0; } - return ssl_set_pkey(ctx->cert, pkey, ctx); + return ssl_set_pkey(ctx->cert, pkey); } int SSL_CTX_use_PrivateKey_file(SSL_CTX *ctx, const char *file, int type) @@ -453,13 +686,8 @@ static int use_certificate_chain_file(SSL_CTX *ctx, SSL *ssl, const char *file) passwd_callback = ctx->default_passwd_callback; passwd_callback_userdata = ctx->default_passwd_callback_userdata; } else { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return 0; - - passwd_callback = sc->default_passwd_callback; - passwd_callback_userdata = sc->default_passwd_callback_userdata; + passwd_callback = ssl->default_passwd_callback; + passwd_callback_userdata = ssl->default_passwd_callback_userdata; } in = BIO_new(BIO_s_file()); @@ -475,7 +703,7 @@ static int use_certificate_chain_file(SSL_CTX *ctx, SSL *ssl, const char *file) x = X509_new_ex(real_ctx->libctx, real_ctx->propq); if (x == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto end; } if (PEM_read_bio_X509_AUX(in, &x, passwd_callback, @@ -514,7 +742,7 @@ static int use_certificate_chain_file(SSL_CTX *ctx, SSL *ssl, const char *file) while (1) { ca = X509_new_ex(real_ctx->libctx, real_ctx->propq); if (ca == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_ASN1_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto end; } if (PEM_read_bio_X509(in, &ca, passwd_callback, @@ -594,7 +822,7 @@ static int serverinfo_find_extension(const unsigned char *serverinfo, if (type == extension_type) { *extension_data = PACKET_data(&data); - *extension_length = PACKET_remaining(&data); + *extension_length = PACKET_remaining(&data);; return 1; /* Success */ } } @@ -632,19 +860,13 @@ static int serverinfoex_srv_add_cb(SSL *s, unsigned int ext_type, { const unsigned char *serverinfo = NULL; size_t serverinfo_length = 0; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) { - *al = SSL_AD_INTERNAL_ERROR; - return -1; - } /* We only support extensions for the first Certificate */ if ((context & SSL_EXT_TLS1_3_CERTIFICATE) != 0 && chainidx > 0) return 0; /* Is there serverinfo data for the chosen server cert? */ - if ((ssl_get_server_cert_serverinfo(sc, &serverinfo, + if ((ssl_get_server_cert_serverinfo(s, &serverinfo, &serverinfo_length)) != 0) { /* Find the relevant extension from the serverinfo */ int retval = serverinfo_find_extension(serverinfo, serverinfo_length, @@ -733,66 +955,16 @@ static int serverinfo_process_buffer(unsigned int version, return 1; } -static size_t extension_contextoff(unsigned int version) -{ - return version == SSL_SERVERINFOV1 ? 4 : 0; -} - -static size_t extension_append_length(unsigned int version, size_t extension_length) -{ - return extension_length + extension_contextoff(version); -} - -static void extension_append(unsigned int version, - const unsigned char *extension, - const size_t extension_length, - unsigned char *serverinfo) -{ - const size_t contextoff = extension_contextoff(version); - - if (contextoff > 0) { - /* We know this only uses the last 2 bytes */ - serverinfo[0] = 0; - serverinfo[1] = 0; - serverinfo[2] = (SYNTHV1CONTEXT >> 8) & 0xff; - serverinfo[3] = SYNTHV1CONTEXT & 0xff; - } - - memcpy(serverinfo + contextoff, extension, extension_length); -} - int SSL_CTX_use_serverinfo_ex(SSL_CTX *ctx, unsigned int version, const unsigned char *serverinfo, size_t serverinfo_length) { - unsigned char *new_serverinfo = NULL; + unsigned char *new_serverinfo; if (ctx == NULL || serverinfo == NULL || serverinfo_length == 0) { ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); return 0; } - if (version == SSL_SERVERINFOV1) { - /* - * Convert serverinfo version v1 to v2 and call yourself recursively - * over the converted serverinfo. - */ - const size_t sinfo_length = extension_append_length(SSL_SERVERINFOV1, - serverinfo_length); - unsigned char *sinfo; - int ret; - - sinfo = OPENSSL_malloc(sinfo_length); - if (sinfo == NULL) - return 0; - - extension_append(SSL_SERVERINFOV1, serverinfo, serverinfo_length, sinfo); - - ret = SSL_CTX_use_serverinfo_ex(ctx, SSL_SERVERINFOV2, sinfo, - sinfo_length); - - OPENSSL_free(sinfo); - return ret; - } if (!serverinfo_process_buffer(version, serverinfo, serverinfo_length, NULL)) { ERR_raise(ERR_LIB_SSL, SSL_R_INVALID_SERVERINFO_DATA); @@ -804,8 +976,10 @@ int SSL_CTX_use_serverinfo_ex(SSL_CTX *ctx, unsigned int version, } new_serverinfo = OPENSSL_realloc(ctx->cert->key->serverinfo, serverinfo_length); - if (new_serverinfo == NULL) + if (new_serverinfo == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } ctx->cert->key->serverinfo = new_serverinfo; memcpy(ctx->cert->key->serverinfo, serverinfo, serverinfo_length); ctx->cert->key->serverinfo_length = serverinfo_length; @@ -838,10 +1012,12 @@ int SSL_CTX_use_serverinfo_file(SSL_CTX *ctx, const char *file) long extension_length = 0; char *name = NULL; char *header = NULL; + static const char namePrefix1[] = "SERVERINFO FOR "; + static const char namePrefix2[] = "SERVERINFOV2 FOR "; unsigned int name_len; int ret = 0; BIO *bin = NULL; - size_t num_extensions = 0; + size_t num_extensions = 0, contextoff = 0; if (ctx == NULL || file == NULL) { ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); @@ -860,7 +1036,6 @@ int SSL_CTX_use_serverinfo_file(SSL_CTX *ctx, const char *file) for (num_extensions = 0;; num_extensions++) { unsigned int version; - size_t append_length; if (PEM_read_bio(bin, &name, &header, &extension, &extension_length) == 0) { @@ -875,18 +1050,18 @@ int SSL_CTX_use_serverinfo_file(SSL_CTX *ctx, const char *file) } /* Check that PEM name starts with "BEGIN SERVERINFO FOR " */ name_len = strlen(name); - if (name_len < sizeof(NAME_PREFIX1) - 1) { + if (name_len < sizeof(namePrefix1) - 1) { ERR_raise(ERR_LIB_SSL, SSL_R_PEM_NAME_TOO_SHORT); goto end; } - if (HAS_PREFIX(name, NAME_PREFIX1)) { + if (strncmp(name, namePrefix1, sizeof(namePrefix1) - 1) == 0) { version = SSL_SERVERINFOV1; } else { - if (name_len < sizeof(NAME_PREFIX2) - 1) { + if (name_len < sizeof(namePrefix2) - 1) { ERR_raise(ERR_LIB_SSL, SSL_R_PEM_NAME_TOO_SHORT); goto end; } - if (!HAS_PREFIX(name, NAME_PREFIX2)) { + if (strncmp(name, namePrefix2, sizeof(namePrefix2) - 1) != 0) { ERR_raise(ERR_LIB_SSL, SSL_R_PEM_NAME_BAD_PREFIX); goto end; } @@ -903,6 +1078,11 @@ int SSL_CTX_use_serverinfo_file(SSL_CTX *ctx, const char *file) ERR_raise(ERR_LIB_SSL, SSL_R_BAD_DATA); goto end; } + /* + * File does not have a context value so we must take account of + * this later. + */ + contextoff = 4; } else { /* 8 byte header: 4 bytes context, 2 bytes type, 2 bytes len */ if (extension_length < 8 @@ -913,14 +1093,25 @@ int SSL_CTX_use_serverinfo_file(SSL_CTX *ctx, const char *file) } } /* Append the decoded extension to the serverinfo buffer */ - append_length = extension_append_length(version, extension_length); - tmp = OPENSSL_realloc(serverinfo, serverinfo_length + append_length); - if (tmp == NULL) + tmp = OPENSSL_realloc(serverinfo, serverinfo_length + extension_length + + contextoff); + if (tmp == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto end; + } serverinfo = tmp; - extension_append(version, extension, extension_length, - serverinfo + serverinfo_length); - serverinfo_length += append_length; + if (contextoff > 0) { + unsigned char *sinfo = serverinfo + serverinfo_length; + + /* We know this only uses the last 2 bytes */ + sinfo[0] = 0; + sinfo[1] = 0; + sinfo[2] = (SYNTHV1CONTEXT >> 8) & 0xff; + sinfo[3] = SYNTHV1CONTEXT & 0xff; + } + memcpy(serverinfo + serverinfo_length + contextoff, + extension, extension_length); + serverinfo_length += extension_length + contextoff; OPENSSL_free(name); name = NULL; @@ -949,24 +1140,18 @@ static int ssl_set_cert_and_key(SSL *ssl, SSL_CTX *ctx, X509 *x509, EVP_PKEY *pr size_t i; int j; int rv; - CERT *c; + CERT *c = ssl != NULL ? ssl->cert : ctx->cert; STACK_OF(X509) *dup_chain = NULL; EVP_PKEY *pubkey = NULL; - SSL_CONNECTION *sc = NULL; - - if (ctx == NULL && - (sc = SSL_CONNECTION_FROM_SSL(ssl)) == NULL) - return 0; - c = sc != NULL ? sc->cert : ctx->cert; /* Do all security checks before anything else */ - rv = ssl_security_cert(sc, ctx, x509, 0, 1); + rv = ssl_security_cert(ssl, ctx, x509, 0, 1); if (rv != 1) { ERR_raise(ERR_LIB_SSL, rv); goto out; } for (j = 0; j < sk_X509_num(chain); j++) { - rv = ssl_security_cert(sc, ctx, sk_X509_value(chain, j), 0, 0); + rv = ssl_security_cert(ssl, ctx, sk_X509_value(chain, j), 0, 0); if (rv != 1) { ERR_raise(ERR_LIB_SSL, rv); goto out; @@ -987,17 +1172,11 @@ static int ssl_set_cert_and_key(SSL *ssl, SSL_CTX *ctx, X509 *x509, EVP_PKEY *pr goto out; } else { /* copy to privatekey from pubkey */ - if (!EVP_PKEY_copy_parameters(privatekey, pubkey)) { - ERR_raise(ERR_LIB_SSL, SSL_R_COPY_PARAMETERS_FAILED); - goto out; - } + EVP_PKEY_copy_parameters(privatekey, pubkey); } } else if (EVP_PKEY_missing_parameters(pubkey)) { /* copy to pubkey from privatekey */ - if (!EVP_PKEY_copy_parameters(pubkey, privatekey)) { - ERR_raise(ERR_LIB_SSL, SSL_R_COPY_PARAMETERS_FAILED); - goto out; - } + EVP_PKEY_copy_parameters(pubkey, privatekey); } /* else both have parameters */ /* check that key <-> cert match */ @@ -1006,7 +1185,7 @@ static int ssl_set_cert_and_key(SSL *ssl, SSL_CTX *ctx, X509 *x509, EVP_PKEY *pr goto out; } } - if (ssl_cert_lookup_by_pkey(pubkey, &i, ctx) == NULL) { + if (ssl_cert_lookup_by_pkey(pubkey, &i) == NULL) { ERR_raise(ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE); goto out; } @@ -1022,12 +1201,12 @@ static int ssl_set_cert_and_key(SSL *ssl, SSL_CTX *ctx, X509 *x509, EVP_PKEY *pr if (chain != NULL) { dup_chain = X509_chain_up_ref(chain); if (dup_chain == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto out; } } - OSSL_STACK_OF_X509_free(c->pkeys[i].chain); + sk_X509_pop_free(c->pkeys[i].chain, X509_free); c->pkeys[i].chain = dup_chain; X509_free(c->pkeys[i].x509); @@ -1057,3 +1236,657 @@ int SSL_CTX_use_cert_and_key(SSL_CTX *ctx, X509 *x509, EVP_PKEY *privatekey, { return ssl_set_cert_and_key(NULL, ctx, x509, privatekey, chain, override); } + +#ifndef OPENSSL_NO_NTLS +/* this is the explicitly cert setting version of ssl_set_cert */ +static int ssl_set_cert_idx(CERT *c, X509 *x, int i) +{ + EVP_PKEY *pkey; + + pkey = X509_get0_pubkey(x); + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_X509_LIB); + return 0; + } + + if (c->pkeys[i].privatekey != NULL) { + /* + * The return code from EVP_PKEY_copy_parameters is deliberately + * ignored. Some EVP_PKEY types cannot do this. + */ + EVP_PKEY_copy_parameters(pkey, c->pkeys[i].privatekey); + ERR_clear_error(); + + if (!X509_check_private_key(x, c->pkeys[i].privatekey)) { + /* + * don't fail for a cert/key mismatch, just free current private + * key (when switching to a different cert & key, first this + * function should be used, then ssl_set_pkey + */ + EVP_PKEY_free(c->pkeys[i].privatekey); + c->pkeys[i].privatekey = NULL; + /* clear error queue */ + ERR_clear_error(); + } + } + + X509_free(c->pkeys[i].x509); + X509_up_ref(x); + c->pkeys[i].x509 = x; + c->key = &(c->pkeys[i]); + + return 1; +} + +static int ssl_set_pkey_idx(CERT *c, EVP_PKEY *pkey, int i) +{ + if (c->pkeys[i].x509 != NULL) { + EVP_PKEY *pktmp; + pktmp = X509_get0_pubkey(c->pkeys[i].x509); + if (pktmp == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + return 0; + } + /* + * The return code from EVP_PKEY_copy_parameters is deliberately + * ignored. Some EVP_PKEY types cannot do this. + */ + EVP_PKEY_copy_parameters(pktmp, pkey); + ERR_clear_error(); + + if (!X509_check_private_key(c->pkeys[i].x509, pkey)) { + X509_free(c->pkeys[i].x509); + c->pkeys[i].x509 = NULL; + return 0; + } + } + + EVP_PKEY_free(c->pkeys[i].privatekey); + EVP_PKEY_up_ref(pkey); + c->pkeys[i].privatekey = pkey; + c->key = &c->pkeys[i]; + + return 1; +} + +int SSL_CTX_use_enc_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey) +{ + int i; + + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = SSL_PKEY_SM2_ENC; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = SSL_PKEY_RSA_ENC; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + return 0; + } + + return ssl_set_pkey_idx(ctx->cert, pkey, i); +} + +int SSL_CTX_use_enc_PrivateKey_file(SSL_CTX *ctx, const char *file, int type) +{ + int j, ret = 0; + BIO *in; + EVP_PKEY *pkey = NULL; + + in = BIO_new(BIO_s_file()); + if (in == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + goto end; + } + + if (BIO_read_filename(in, file) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); + goto end; + } + if (type == SSL_FILETYPE_PEM) { + j = ERR_R_PEM_LIB; + pkey = PEM_read_bio_PrivateKey(in, NULL, + ctx->default_passwd_callback, + ctx->default_passwd_callback_userdata); + } else if (type == SSL_FILETYPE_ASN1) { + j = ERR_R_ASN1_LIB; + pkey = d2i_PrivateKey_bio(in, NULL); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, j); + goto end; + } + ret = SSL_CTX_use_enc_PrivateKey(ctx, pkey); + EVP_PKEY_free(pkey); + end: + BIO_free(in); + return ret; +} + +int SSL_CTX_use_sign_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey) +{ + int i; + + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = SSL_PKEY_SM2_SIGN; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = SSL_PKEY_RSA_SIGN; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + return 0; + } + + return ssl_set_pkey_idx(ctx->cert, pkey, i); +} + +int SSL_CTX_use_sign_PrivateKey_file(SSL_CTX *ctx, const char *file, int type) +{ + int j, ret = 0; + BIO *in; + EVP_PKEY *pkey = NULL; + + in = BIO_new(BIO_s_file()); + if (in == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + goto end; + } + + if (BIO_read_filename(in, file) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); + goto end; + } + if (type == SSL_FILETYPE_PEM) { + j = ERR_R_PEM_LIB; + pkey = PEM_read_bio_PrivateKey(in, NULL, + ctx->default_passwd_callback, + ctx->default_passwd_callback_userdata); + } else if (type == SSL_FILETYPE_ASN1) { + j = ERR_R_ASN1_LIB; + pkey = d2i_PrivateKey_bio(in, NULL); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, j); + goto end; + } + ret = SSL_CTX_use_sign_PrivateKey(ctx, pkey); + EVP_PKEY_free(pkey); + end: + BIO_free(in); + return ret; +} + +/* This function is used to set SM2 enc function only */ +int SSL_CTX_use_enc_certificate(SSL_CTX *ctx, X509 *x) +{ + EVP_PKEY *pkey = NULL; + int i, rv; + + if (x == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!(X509_get_key_usage(x) & X509v3_KU_KEY_ENCIPHERMENT) + && !(X509_get_key_usage(x) & X509v3_KU_DATA_ENCIPHERMENT)) { + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return 0; + } + + rv = ssl_security_cert(NULL, ctx, x, 0, 1); + if (rv != 1) { + ERR_raise(ERR_LIB_SSL, rv); + return 0; + } + + pkey = X509_get0_pubkey(x); + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_X509_LIB); + return 0; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = SSL_PKEY_SM2_ENC; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = SSL_PKEY_RSA_ENC; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + return 0; + } + + return ssl_set_cert_idx(ctx->cert, x, i); +} + +int SSL_CTX_use_enc_certificate_file(SSL_CTX *ctx, const char *file, int type) +{ + int j; + BIO *in; + int ret = 0; + X509 *x = NULL; + + in = BIO_new(BIO_s_file()); + if (in == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + goto end; + } + + if (BIO_read_filename(in, file) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); + goto end; + } + + if (type == SSL_FILETYPE_ASN1) { + j = ERR_R_ASN1_LIB; + x = d2i_X509_bio(in, NULL); + } else if (type == SSL_FILETYPE_PEM) { + j = ERR_R_PEM_LIB; + x = PEM_read_bio_X509(in, NULL, ctx->default_passwd_callback, + ctx->default_passwd_callback_userdata); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + + if (x == NULL) { + ERR_raise(ERR_LIB_SSL, j); + goto end; + } + + ret = SSL_CTX_use_enc_certificate(ctx, x); + end: + X509_free(x); + BIO_free(in); + return ret; +} + +int SSL_CTX_use_sign_certificate(SSL_CTX *ctx, X509 *x) +{ + EVP_PKEY *pkey = NULL; + int i, rv; + + if (x == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (!(X509_get_key_usage(x) & X509v3_KU_DIGITAL_SIGNATURE) + && !(X509_get_key_usage(x) & X509v3_KU_KEY_CERT_SIGN) + && !(X509_get_key_usage(x) & X509v3_KU_CRL_SIGN)) { + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return 0; + } + + rv = ssl_security_cert(NULL, ctx, x, 0, 1); + if (rv != 1) { + ERR_raise(ERR_LIB_SSL, rv); + return 0; + } + + pkey = X509_get0_pubkey(x); + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_X509_LIB); + return 0; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) + i = SSL_PKEY_SM2_SIGN; + else if (EVP_PKEY_is_a(pkey, "RSA")) + i = SSL_PKEY_RSA_SIGN; + else { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_CERTIFICATE_TYPE); + return 0; + } + + return ssl_set_cert_idx(ctx->cert, x, i); +} + +int SSL_CTX_use_sign_certificate_file(SSL_CTX *ctx, const char *file, int type) +{ + int j; + BIO *in; + int ret = 0; + X509 *x = NULL; + + in = BIO_new(BIO_s_file()); + if (in == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + goto end; + } + + if (BIO_read_filename(in, file) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); + goto end; + } + + if (type == SSL_FILETYPE_ASN1) { + j = ERR_R_ASN1_LIB; + x = d2i_X509_bio(in, NULL); + } else if (type == SSL_FILETYPE_PEM) { + j = ERR_R_PEM_LIB; + x = PEM_read_bio_X509(in, NULL, ctx->default_passwd_callback, + ctx->default_passwd_callback_userdata); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + + if (x == NULL) { + ERR_raise(ERR_LIB_SSL, j); + goto end; + } + + ret = SSL_CTX_use_sign_certificate(ctx, x); + end: + X509_free(x); + BIO_free(in); + return ret; +} +#endif + +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +static int ssl_set_dc(CERT *c, DELEGATED_CREDENTIAL *dc, int is_server) +{ + EVP_PKEY *pkey; + uint16_t sigalg; + size_t i; + const SIGALG_LOOKUP *lu; + + pkey = DC_get0_publickey(dc); + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_X509_LIB); + return 0; + } + + /* DC public key MUST NOT use the rsaEncryption OID */ + if (EVP_PKEY_is_a(pkey, "RSA")) { + ERR_raise(ERR_LIB_SSL, SSL_R_WRONG_PUBLIC_KEY_TYPE); + return 0; + } + + if (ssl_cert_lookup_by_pkey(pkey, &i) == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE); + return 0; + } + + if (i == SSL_PKEY_ECC && !EVP_PKEY_can_sign(pkey)) { + ERR_raise(ERR_LIB_SSL, SSL_R_ECC_CERT_NOT_FOR_SIGNING); + return 0; + } + + sigalg = DC_get_signature_sign_algorithm(dc); + + lu = ssl_sigalg_lookup(sigalg); + if (lu == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); + return 0; + } + + if (c->pkeys[lu->sig_idx].x509 == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_EE_CERT_NOT_FOUND); + return 0; + } + + if (!DC_check_valid(c->pkeys[lu->sig_idx].x509 , dc)) { + ERR_raise(ERR_LIB_SSL, SSL_R_CERTIFICATE_VERIFY_FAILED); + return 0; + } + + if (SSL_verify_delegated_credential_signature(c->pkeys[lu->sig_idx].x509, + dc, is_server) <= 0) { + ERR_raise(ERR_LIB_SSL, SSL_R_FAILED_TO_VERIFY_DC_SIGNATURE); + return 0; + } + + if (c->dc_pkeys[i].privatekey != NULL) { + /* + * The return code from EVP_PKEY_copy_parameters is deliberately + * ignored. Some EVP_PKEY types cannot do this. + */ + EVP_PKEY_copy_parameters(pkey, c->dc_pkeys[i].privatekey); + ERR_clear_error(); + + if (!DC_check_private_key(dc, c->dc_pkeys[i].privatekey)) { + /* + * don't fail for a dc/key mismatch, just free current private + * key (when switching to a different dc & key, first this + * function should be used, then ssl_set_pkey + */ + EVP_PKEY_free(c->dc_pkeys[i].privatekey); + c->dc_pkeys[i].privatekey = NULL; + /* clear error queue */ + ERR_clear_error(); + } + } + + DC_free(c->dc_pkeys[i].dc); + DC_up_ref(dc); + c->dc_pkeys[i].dc = dc; + + return 1; +} + +static int ssl_set_dc_pkey(CERT *c, EVP_PKEY *pkey) +{ + size_t i; + + if (ssl_cert_lookup_by_pkey(pkey, &i) == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE); + return 0; + } + + if (c->dc_pkeys[i].dc != NULL + && !DC_check_private_key(c->dc_pkeys[i].dc, pkey)) + return 0; + + EVP_PKEY_free(c->dc_pkeys[i].privatekey); + EVP_PKEY_up_ref(pkey); + c->dc_pkeys[i].privatekey = pkey; + + return 1; +} + +int SSL_use_dc(SSL *ssl, DELEGATED_CREDENTIAL *dc) +{ + if (ssl == NULL || dc == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + return ssl_set_dc(ssl->cert, dc, SSL_is_server(ssl)); +} + +int SSL_use_dc_file(SSL *ssl, const char *file, int type) +{ + DELEGATED_CREDENTIAL *dc = NULL; + int ret = 0; + + if (ssl == NULL || file == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + /* only support raw type */ + if (type == DC_FILETYPE_RAW) { + dc = DC_load_from_file_ex(file, + ssl->ctx->libctx, + ssl->ctx->propq); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + + if (dc == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + + ret = ssl_set_dc(ssl->cert, dc, SSL_is_server(ssl)); +end: + DC_free(dc); + return ret; +} + +int SSL_CTX_use_dc(SSL_CTX *ctx, DELEGATED_CREDENTIAL *dc) +{ + int is_server; + + if (ctx == NULL || dc == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (ctx->method == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION); + return 0; + } + + is_server = (ctx->method->ssl_accept == ssl_undefined_function) ? 0 : 1; + + return ssl_set_dc(ctx->cert, dc, is_server); +} + +int SSL_CTX_use_dc_file(SSL_CTX *ctx, const char *file, int type) +{ + int ret = 0; + int is_server; + DELEGATED_CREDENTIAL *dc = NULL; + + if (ctx == NULL || file == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + if (type == DC_FILETYPE_RAW) { + dc = DC_load_from_file_ex(file, ctx->libctx, ctx->propq); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + + if (dc == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + + is_server = (ctx->method->ssl_accept == ssl_undefined_function) ? 0 : 1; + + ret = ssl_set_dc(ctx->cert, dc, is_server); +end: + DC_free(dc); + return ret; +} + +int SSL_use_dc_PrivateKey(SSL *ssl, EVP_PKEY *pkey) +{ + if (ssl == NULL || pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + return ssl_set_dc_pkey(ssl->cert, pkey); +} + +int SSL_use_dc_PrivateKey_file(SSL *ssl, const char *file, int type) +{ + BIO *in; + int j, ret = 0; + EVP_PKEY *pkey = NULL; + + in = BIO_new(BIO_s_file()); + if (in == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + goto end; + } + + if (BIO_read_filename(in, file) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); + goto end; + } + if (type == SSL_FILETYPE_PEM) { + j = ERR_R_PEM_LIB; + pkey = PEM_read_bio_PrivateKey(in, NULL, + ssl->default_passwd_callback, + ssl->default_passwd_callback_userdata); + } else if (type == SSL_FILETYPE_ASN1) { + j = ERR_R_ASN1_LIB; + pkey = d2i_PrivateKey_bio(in, NULL); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, j); + goto end; + } + + ret = ssl_set_dc_pkey(ssl->cert, pkey); + EVP_PKEY_free(pkey); +end: + BIO_free(in); + return ret; +} + +int SSL_CTX_use_dc_PrivateKey(SSL_CTX *ctx, EVP_PKEY *pkey) +{ + if (ctx == NULL || pkey == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER); + return 0; + } + + return ssl_set_dc_pkey(ctx->cert, pkey); +} + +int SSL_CTX_use_dc_PrivateKey_file(SSL_CTX *ctx, const char *file, int type) +{ + BIO *in; + int j, ret = 0; + EVP_PKEY *pkey = NULL; + + in = BIO_new(BIO_s_file()); + if (in == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_BUF_LIB); + goto end; + } + + if (BIO_read_filename(in, file) <= 0) { + ERR_raise(ERR_LIB_SSL, ERR_R_SYS_LIB); + goto end; + } + if (type == SSL_FILETYPE_PEM) { + j = ERR_R_PEM_LIB; + pkey = PEM_read_bio_PrivateKey_ex(in, NULL, + ctx->default_passwd_callback, + ctx->default_passwd_callback_userdata, + ctx->libctx, ctx->propq); + } else if (type == SSL_FILETYPE_ASN1) { + j = ERR_R_ASN1_LIB; + pkey = d2i_PrivateKey_ex_bio(in, NULL, ctx->libctx, ctx->propq); + } else { + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_SSL_FILETYPE); + goto end; + } + if (pkey == NULL) { + ERR_raise(ERR_LIB_SSL, j); + goto end; + } + + ret = ssl_set_dc_pkey(ctx->cert, pkey); + EVP_PKEY_free(pkey); +end: + BIO_free(in); + return ret; +} +#endif diff --git a/openssl/src/ssl/ssl_sess.c b/openssl/src/ssl/ssl_sess.c index 3857e027e..6bbd27d4e 100644 --- a/openssl/src/ssl/ssl_sess.c +++ b/openssl/src/ssl/ssl_sess.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2005 Nokia. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -15,38 +15,70 @@ #include #include #include +#include #include "internal/refcount.h" #include "internal/cryptlib.h" #include "ssl_local.h" #include "statem/statem_local.h" +#ifndef OPENSSL_NO_SESSION_LOOKUP +static const char g_pending_session_magic = 0; +#endif + static void SSL_SESSION_list_remove(SSL_CTX *ctx, SSL_SESSION *s); static void SSL_SESSION_list_add(SSL_CTX *ctx, SSL_SESSION *s); static int remove_session_lock(SSL_CTX *ctx, SSL_SESSION *c, int lck); DEFINE_STACK_OF(SSL_SESSION) -__owur static ossl_inline int sess_timedout(OSSL_TIME t, SSL_SESSION *ss) +__owur static int sess_timedout(time_t t, SSL_SESSION *ss) { - return ossl_time_compare(t, ss->calc_timeout) > 0; + /* if timeout overflowed, it can never timeout! */ + if (ss->timeout_ovf) + return 0; + return t > ss->calc_timeout; } /* * Returns -1/0/+1 as other XXXcmp-type functions - * Takes calculated timeout into consideration + * Takes overflow of calculated timeout into consideration */ -__owur static ossl_inline int timeoutcmp(SSL_SESSION *a, SSL_SESSION *b) +__owur static int timeoutcmp(SSL_SESSION *a, SSL_SESSION *b) { - return ossl_time_compare(a->calc_timeout, b->calc_timeout); + /* if only one overflowed, then it is greater */ + if (a->timeout_ovf && !b->timeout_ovf) + return 1; + if (!a->timeout_ovf && b->timeout_ovf) + return -1; + /* No overflow, or both overflowed, so straight compare is safe */ + if (a->calc_timeout < b->calc_timeout) + return -1; + if (a->calc_timeout > b->calc_timeout) + return 1; + return 0; } /* - * Calculates effective timeout + * Calculates effective timeout, saving overflow state * Locking must be done by the caller of this function */ void ssl_session_calculate_timeout(SSL_SESSION *ss) { - ss->calc_timeout = ossl_time_add(ss->time, ss->timeout); + /* Force positive timeout */ + if (ss->timeout < 0) + ss->timeout = 0; + ss->calc_timeout = ss->time + ss->timeout; + /* + * |timeout| is always zero or positive, so the check for + * overflow only needs to consider if |time| is positive + */ + ss->timeout_ovf = ss->time > 0 && ss->calc_timeout < ss->time; + /* + * N.B. Realistic overflow can only occur in our lifetimes on a + * 32-bit machine in January 2038. + * However, There are no controls to limit the |timeout| + * value, except to keep it positive. + */ } /* @@ -61,19 +93,13 @@ void ssl_session_calculate_timeout(SSL_SESSION *ss) SSL_SESSION *SSL_get_session(const SSL *ssl) /* aka SSL_get0_session; gets 0 objects, just returns a copy of the pointer */ { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL) - return NULL; - - return sc->session; + return ssl->session; } SSL_SESSION *SSL_get1_session(SSL *ssl) /* variant of SSL_get_session: caller really gets something */ { SSL_SESSION *sess; - /* * Need to lock this all up rather than just use CRYPTO_add so that * somebody doesn't free ssl->session between when we check it's non-null @@ -81,8 +107,8 @@ SSL_SESSION *SSL_get1_session(SSL *ssl) */ if (!CRYPTO_THREAD_read_lock(ssl->lock)) return NULL; - sess = SSL_get_session(ssl); - if (sess != NULL) + sess = ssl->session; + if (sess) SSL_SESSION_up_ref(sess); CRYPTO_THREAD_unlock(ssl->lock); return sess; @@ -106,38 +132,48 @@ SSL_SESSION *SSL_SESSION_new(void) return NULL; ss = OPENSSL_zalloc(sizeof(*ss)); - if (ss == NULL) + if (ss == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return NULL; + } ss->verify_result = 1; /* avoid 0 (= X509_V_OK) just in case */ - /* 5 minute timeout by default */ - ss->timeout = ossl_seconds2time(60 * 5 + 4); - ss->time = ossl_time_now(); + ss->references = 1; + ss->timeout = 60 * 5 + 4; /* 5 minute timeout by default */ + ss->time = time(NULL); ssl_session_calculate_timeout(ss); - if (!CRYPTO_NEW_REF(&ss->references, 1)) { + ss->lock = CRYPTO_THREAD_lock_new(); + if (ss->lock == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); OPENSSL_free(ss); return NULL; } if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL_SESSION, ss, &ss->ex_data)) { - CRYPTO_FREE_REF(&ss->references); + CRYPTO_THREAD_lock_free(ss->lock); OPENSSL_free(ss); return NULL; } return ss; } +SSL_SESSION *SSL_SESSION_dup(const SSL_SESSION *src) +{ + return ssl_session_dup(src, 1); +} + /* * Create a new SSL_SESSION and duplicate the contents of |src| into it. If * ticket == 0 then no ticket information is duplicated, otherwise it is. */ -static SSL_SESSION *ssl_session_dup_intern(const SSL_SESSION *src, int ticket) +SSL_SESSION *ssl_session_dup(const SSL_SESSION *src, int ticket) { SSL_SESSION *dest; dest = OPENSSL_malloc(sizeof(*dest)); - if (dest == NULL) - return NULL; + if (dest == NULL) { + goto err; + } memcpy(dest, src, sizeof(*dest)); /* @@ -156,70 +192,70 @@ static SSL_SESSION *ssl_session_dup_intern(const SSL_SESSION *src, int ticket) #endif dest->peer_chain = NULL; dest->peer = NULL; - dest->peer_rpk = NULL; dest->ticket_appdata = NULL; memset(&dest->ex_data, 0, sizeof(dest->ex_data)); - /* As the copy is not in the cache, we remove the associated pointers */ +#ifndef OPENSSL_NO_QUIC + dest->quic_early_data_context = NULL; + dest->quic_early_data_context_len = 0; +#endif + /* We deliberately don't copy the prev and next pointers */ dest->prev = NULL; dest->next = NULL; - dest->owner = NULL; - if (!CRYPTO_NEW_REF(&dest->references, 1)) { - OPENSSL_free(dest); - return NULL; - } + dest->references = 1; + + dest->lock = CRYPTO_THREAD_lock_new(); + if (dest->lock == NULL) + goto err; - if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL_SESSION, dest, &dest->ex_data)) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); + if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL_SESSION, dest, &dest->ex_data)) goto err; + +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + if (src->peer_dc != NULL) { + if (!DC_up_ref(src->peer_dc)) + goto err; + dest->peer_dc = src->peer_dc; } +#endif if (src->peer != NULL) { - if (!X509_up_ref(src->peer)) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); + if (!X509_up_ref(src->peer)) goto err; - } dest->peer = src->peer; } if (src->peer_chain != NULL) { dest->peer_chain = X509_chain_up_ref(src->peer_chain); - if (dest->peer_chain == NULL) { - ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB); - goto err; - } - } - - if (src->peer_rpk != NULL) { - if (!EVP_PKEY_up_ref(src->peer_rpk)) + if (dest->peer_chain == NULL) goto err; - dest->peer_rpk = src->peer_rpk; } - #ifndef OPENSSL_NO_PSK if (src->psk_identity_hint) { dest->psk_identity_hint = OPENSSL_strdup(src->psk_identity_hint); - if (dest->psk_identity_hint == NULL) + if (dest->psk_identity_hint == NULL) { goto err; + } } if (src->psk_identity) { dest->psk_identity = OPENSSL_strdup(src->psk_identity); - if (dest->psk_identity == NULL) + if (dest->psk_identity == NULL) { goto err; + } } #endif if (!CRYPTO_dup_ex_data(CRYPTO_EX_INDEX_SSL_SESSION, &dest->ex_data, &src->ex_data)) { - ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB); goto err; } if (src->ext.hostname) { dest->ext.hostname = OPENSSL_strdup(src->ext.hostname); - if (dest->ext.hostname == NULL) + if (dest->ext.hostname == NULL) { goto err; + } } if (ticket != 0 && src->ext.tick != NULL) { @@ -242,8 +278,9 @@ static SSL_SESSION *ssl_session_dup_intern(const SSL_SESSION *src, int ticket) #ifndef OPENSSL_NO_SRP if (src->srp_username) { dest->srp_username = OPENSSL_strdup(src->srp_username); - if (dest->srp_username == NULL) + if (dest->srp_username == NULL) { goto err; + } } #endif @@ -254,33 +291,25 @@ static SSL_SESSION *ssl_session_dup_intern(const SSL_SESSION *src, int ticket) goto err; } +#ifndef OPENSSL_NO_QUIC + if (src->quic_early_data_context) { + dest->quic_early_data_context = + OPENSSL_memdup(src->quic_early_data_context, + src->quic_early_data_context_len); + if (dest->quic_early_data_context == NULL) + goto err; + + dest->quic_early_data_context_len = src->quic_early_data_context_len; + } +#endif + return dest; err: + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); SSL_SESSION_free(dest); return NULL; } -SSL_SESSION *SSL_SESSION_dup(const SSL_SESSION *src) -{ - return ssl_session_dup_intern(src, 1); -} - -/* - * Used internally when duplicating a session which might be already shared. - * We will have resumed the original session. Subsequently we might have marked - * it as non-resumable (e.g. in another thread) - but this copy should be ok to - * resume from. - */ -SSL_SESSION *ssl_session_dup(const SSL_SESSION *src, int ticket) -{ - SSL_SESSION *sess = ssl_session_dup_intern(src, ticket); - - if (sess != NULL) - sess->not_resumable = 0; - - return sess; -} - const unsigned char *SSL_SESSION_get_id(const SSL_SESSION *s, unsigned int *len) { if (len) @@ -316,15 +345,10 @@ static int def_generate_session_id(SSL *ssl, unsigned char *id, unsigned int *id_len) { unsigned int retry = 0; - do { + do if (RAND_bytes_ex(ssl->ctx->libctx, id, *id_len, 0) <= 0) return 0; -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (retry > 0) { - id[0]++; - } -#endif - } while (SSL_has_matching_session_id(ssl, id, *id_len) && + while (SSL_has_matching_session_id(ssl, id, *id_len) && (++retry < MAX_SESS_ID_ATTEMPTS)) ; if (retry < MAX_SESS_ID_ATTEMPTS) return 1; @@ -340,11 +364,10 @@ static int def_generate_session_id(SSL *ssl, unsigned char *id, return 0; } -int ssl_generate_session_id(SSL_CONNECTION *s, SSL_SESSION *ss) +int ssl_generate_session_id(SSL *s, SSL_SESSION *ss) { unsigned int tmp; GEN_SESSION_CB cb = def_generate_session_id; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); switch (s->version) { case SSL3_VERSION: @@ -355,6 +378,9 @@ int ssl_generate_session_id(SSL_CONNECTION *s, SSL_SESSION *ss) case DTLS1_BAD_VER: case DTLS1_VERSION: case DTLS1_2_VERSION: +#ifndef OPENSSL_NO_NTLS + case NTLS1_1_VERSION: +#endif ss->session_id_length = SSL3_SSL_SESSION_ID_LENGTH; break; default: @@ -383,10 +409,10 @@ int ssl_generate_session_id(SSL_CONNECTION *s, SSL_SESSION *ss) } /* Choose which callback will set the session ID */ - if (!CRYPTO_THREAD_read_lock(SSL_CONNECTION_GET_SSL(s)->lock)) + if (!CRYPTO_THREAD_read_lock(s->lock)) return 0; if (!CRYPTO_THREAD_read_lock(s->session_ctx->lock)) { - CRYPTO_THREAD_unlock(ssl->lock); + CRYPTO_THREAD_unlock(s->lock); SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED); return 0; @@ -396,11 +422,11 @@ int ssl_generate_session_id(SSL_CONNECTION *s, SSL_SESSION *ss) else if (s->session_ctx->generate_session_id) cb = s->session_ctx->generate_session_id; CRYPTO_THREAD_unlock(s->session_ctx->lock); - CRYPTO_THREAD_unlock(ssl->lock); + CRYPTO_THREAD_unlock(s->lock); /* Choose a session ID */ memset(ss->session_id, 0, ss->session_id_length); tmp = (int)ss->session_id_length; - if (!cb(ssl, ss->session_id, &tmp)) { + if (!cb(s, ss->session_id, &tmp)) { /* The callback failed */ SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_SSL_SESSION_ID_CALLBACK_FAILED); @@ -418,7 +444,7 @@ int ssl_generate_session_id(SSL_CONNECTION *s, SSL_SESSION *ss) } ss->session_id_length = tmp; /* Finally, check for a conflict */ - if (SSL_has_matching_session_id(ssl, ss->session_id, + if (SSL_has_matching_session_id(s, ss->session_id, (unsigned int)ss->session_id_length)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_SSL_SESSION_ID_CONFLICT); return 0; @@ -427,20 +453,20 @@ int ssl_generate_session_id(SSL_CONNECTION *s, SSL_SESSION *ss) return 1; } -int ssl_get_new_session(SSL_CONNECTION *s, int session) +int ssl_get_new_session(SSL *s, int session) { /* This gets used by clients and servers. */ SSL_SESSION *ss = NULL; if ((ss = SSL_SESSION_new()) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } /* If the context has a default timeout, use it */ - if (ossl_time_is_zero(s->session_ctx->session_timeout)) - ss->timeout = SSL_CONNECTION_GET_SSL(s)->method->get_timeout(); + if (s->session_ctx->session_timeout == 0) + ss->timeout = SSL_get_default_timeout(s); else ss->timeout = s->session_ctx->session_timeout; ssl_session_calculate_timeout(ss); @@ -449,7 +475,7 @@ int ssl_get_new_session(SSL_CONNECTION *s, int session) s->session = NULL; if (session) { - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* * We generate the session id while constructing the * NewSessionTicket in TLSv1.3. @@ -475,7 +501,22 @@ int ssl_get_new_session(SSL_CONNECTION *s, int session) s->session = ss; ss->ssl_version = s->version; ss->verify_result = X509_V_OK; +#ifndef OPENSSL_NO_QUIC + ss->is_quic = (s->quic_method != NULL); + + if (s->quic_early_data_context) { + ss->quic_early_data_context = + OPENSSL_memdup(s->quic_early_data_context, + s->quic_early_data_context_len); + if (ss->quic_early_data_context == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + SSL_SESSION_free(ss); + return 0; + } + ss->quic_early_data_context_len = s->quic_early_data_context_len; + } +#endif /* If client supports extended master secret set it in session */ if (s->s3.flags & TLS1_FLAGS_RECEIVED_EXTMS) ss->flags |= SSL_SESS_FLAG_EXTMS; @@ -483,8 +524,7 @@ int ssl_get_new_session(SSL_CONNECTION *s, int session) return 1; } -SSL_SESSION *lookup_sess_in_cache(SSL_CONNECTION *s, - const unsigned char *sess_id, +SSL_SESSION *lookup_sess_in_cache(SSL *s, const unsigned char *sess_id, size_t sess_id_len) { SSL_SESSION *ret = NULL; @@ -515,16 +555,14 @@ SSL_SESSION *lookup_sess_in_cache(SSL_CONNECTION *s, if (ret == NULL && s->session_ctx->get_session_cb != NULL) { int copy = 1; - ret = s->session_ctx->get_session_cb(SSL_CONNECTION_GET_SSL(s), - sess_id, sess_id_len, ©); + ret = s->session_ctx->get_session_cb(s, sess_id, sess_id_len, ©); + +#ifndef OPENSSL_NO_SESSION_LOOKUP + if (ret == SSL_magic_pending_session_ptr()) + return ret; /* Retry later */ +#endif if (ret != NULL) { - if (ret->not_resumable) { - /* If its not resumable then ignore this session */ - if (!copy) - SSL_SESSION_free(ret); - return NULL; - } ssl_tsan_counter(s->session_ctx, &s->session_ctx->stats.sess_cb_hit); @@ -564,6 +602,7 @@ SSL_SESSION *lookup_sess_in_cache(SSL_CONNECTION *s, * hello: The parsed ClientHello data * * Returns: + * -2: want asynchronous session lookup by lua * -1: fatal error * 0: no session found * 1: a session may have been found. @@ -574,7 +613,7 @@ SSL_SESSION *lookup_sess_in_cache(SSL_CONNECTION *s, * - Both for new and resumed sessions, s->ext.ticket_expected is set to 1 * if the server should issue a new session ticket (to 0 otherwise). */ -int ssl_get_prev_session(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello) +int ssl_get_prev_session(SSL *s, CLIENTHELLO_MSG *hello) { /* This is used only by servers. */ @@ -583,7 +622,7 @@ int ssl_get_prev_session(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello) int try_session_cache = 0; SSL_TICKET_STATUS r; - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* * By default we will send a new ticket. This can be overridden in the * ticket processing. @@ -597,6 +636,10 @@ int ssl_get_prev_session(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello) return -1; ret = s->session; +# ifndef OPENSSL_NO_SESSION_REUSED_TYPE + if (ret != NULL) + s->session_reused_type = SSL_SESSION_REUSED_TYPE_TICKET; +# endif } else { /* sets s->ext.ticket_expected */ r = tls_get_ticket_from_client(s, hello, &ret); @@ -615,12 +658,22 @@ int ssl_get_prev_session(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello) } break; case SSL_TICKET_NO_DECRYPT: +# ifndef OPENSSL_NO_SESSION_REUSED_TYPE + break; +# endif case SSL_TICKET_SUCCESS: +# ifndef OPENSSL_NO_SESSION_REUSED_TYPE + s->session_reused_type = SSL_SESSION_REUSED_TYPE_TICKET; +# endif case SSL_TICKET_SUCCESS_RENEW: break; } } +#ifndef OPENSSL_NO_SESSION_LOOKUP + if (ret == SSL_magic_pending_session_ptr()) + return -2; /* Retry later */ +#endif if (ret == NULL) goto err; @@ -656,7 +709,7 @@ int ssl_get_prev_session(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello) goto err; } - if (sess_timedout(ossl_time_now(), ret)) { + if (sess_timedout(time(NULL), ret)) { ssl_tsan_counter(s->session_ctx, &s->session_ctx->stats.sess_timeout); if (try_session_cache) { /* session was from the cache, so remove it */ @@ -678,21 +731,30 @@ int ssl_get_prev_session(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello) goto err; } - if (!SSL_CONNECTION_IS_TLS13(s)) { + if (!SSL_IS_TLS13(s)) { /* We already did this for TLS1.3 */ SSL_SESSION_free(s->session); s->session = ret; } +# ifndef OPENSSL_NO_SESSION_REUSED_TYPE + if (try_session_cache) + s->session_reused_type = SSL_SESSION_REUSED_TYPE_CACHE; +# endif + ssl_tsan_counter(s->session_ctx, &s->session_ctx->stats.sess_hit); s->verify_result = s->session->verify_result; return 1; err: +# ifndef OPENSSL_NO_SESSION_REUSED_TYPE + s->session_reused_type = SSL_SESSION_REUSED_TYPE_NOCACHE; +# endif + if (ret != NULL) { SSL_SESSION_free(ret); /* In TLSv1.3 s->session was already set to ret, so we NULL it out */ - if (SSL_CONNECTION_IS_TLS13(s)) + if (SSL_IS_TLS13(s)) s->session = NULL; if (!try_session_cache) { @@ -759,20 +821,28 @@ int SSL_CTX_add_session(SSL_CTX *ctx, SSL_SESSION *c) /* Adjust last used time, and add back into the cache at the appropriate spot */ if (ctx->session_cache_mode & SSL_SESS_CACHE_UPDATE_TIME) { - c->time = ossl_time_now(); + c->time = time(NULL); ssl_session_calculate_timeout(c); } + SSL_SESSION_list_add(ctx, c); - if (s == NULL) { + if (s != NULL) { + /* + * existing cache entry -- decrement previously incremented reference + * count because it already takes into account the cache + */ + + SSL_SESSION_free(s); /* s == c */ + ret = 0; + } else { /* * new cache entry -- remove old ones if cache has become too large - * delete cache entry *before* add, so we don't remove the one we're adding! */ ret = 1; if (SSL_CTX_sess_get_cache_size(ctx) > 0) { - while (SSL_CTX_sess_number(ctx) >= SSL_CTX_sess_get_cache_size(ctx)) { + while (SSL_CTX_sess_number(ctx) > SSL_CTX_sess_get_cache_size(ctx)) { if (!remove_session_lock(ctx, ctx->session_cache_tail, 0)) break; else @@ -780,18 +850,6 @@ int SSL_CTX_add_session(SSL_CTX *ctx, SSL_SESSION *c) } } } - - SSL_SESSION_list_add(ctx, c); - - if (s != NULL) { - /* - * existing cache entry -- decrement previously incremented reference - * count because it already takes into account the cache - */ - - SSL_SESSION_free(s); /* s == c */ - ret = 0; - } CRYPTO_THREAD_unlock(ctx->lock); return ret; } @@ -836,7 +894,7 @@ void SSL_SESSION_free(SSL_SESSION *ss) if (ss == NULL) return; - CRYPTO_DOWN_REF(&ss->references, &i); + CRYPTO_DOWN_REF(&ss->references, &i, ss->lock); REF_PRINT_COUNT("SSL_SESSION", ss); if (i > 0) return; @@ -847,8 +905,10 @@ void SSL_SESSION_free(SSL_SESSION *ss) OPENSSL_cleanse(ss->master_key, sizeof(ss->master_key)); OPENSSL_cleanse(ss->session_id, sizeof(ss->session_id)); X509_free(ss->peer); - EVP_PKEY_free(ss->peer_rpk); - OSSL_STACK_OF_X509_free(ss->peer_chain); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + DC_free(ss->peer_dc); +#endif + sk_X509_pop_free(ss->peer_chain, X509_free); OPENSSL_free(ss->ext.hostname); OPENSSL_free(ss->ext.tick); #ifndef OPENSSL_NO_PSK @@ -857,10 +917,13 @@ void SSL_SESSION_free(SSL_SESSION *ss) #endif #ifndef OPENSSL_NO_SRP OPENSSL_free(ss->srp_username); +#endif +#ifndef OPENSSL_NO_QUIC + OPENSSL_free(ss->quic_early_data_context); #endif OPENSSL_free(ss->ext.alpn_selected); OPENSSL_free(ss->ticket_appdata); - CRYPTO_FREE_REF(&ss->references); + CRYPTO_THREAD_lock_free(ss->lock); OPENSSL_clear_free(ss, sizeof(*ss)); } @@ -868,7 +931,7 @@ int SSL_SESSION_up_ref(SSL_SESSION *ss) { int i; - if (CRYPTO_UP_REF(&ss->references, &i) <= 0) + if (CRYPTO_UP_REF(&ss->references, &i, ss->lock) <= 0) return 0; REF_PRINT_COUNT("SSL_SESSION", ss); @@ -878,23 +941,18 @@ int SSL_SESSION_up_ref(SSL_SESSION *ss) int SSL_set_session(SSL *s, SSL_SESSION *session) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - ssl_clear_bad_session(sc); - if (s->defltmeth != s->method) { - if (!SSL_set_ssl_method(s, s->defltmeth)) + ssl_clear_bad_session(s); + if (s->ctx->method != s->method) { + if (!SSL_set_ssl_method(s, s->ctx->method)) return 0; } if (session != NULL) { SSL_SESSION_up_ref(session); - sc->verify_result = session->verify_result; + s->verify_result = session->verify_result; } - SSL_SESSION_free(sc->session); - sc->session = session; + SSL_SESSION_free(s->session); + s->session = session; return 1; } @@ -907,15 +965,14 @@ int SSL_SESSION_set1_id(SSL_SESSION *s, const unsigned char *sid, return 0; } s->session_id_length = sid_len; - if (sid != s->session_id && sid_len > 0) + if (sid != s->session_id) memcpy(s->session_id, sid, sid_len); - return 1; } long SSL_SESSION_set_timeout(SSL_SESSION *s, long t) { - OSSL_TIME new_timeout = ossl_seconds2time(t); + time_t new_timeout = (time_t)t; if (s == NULL || t < 0) return 0; @@ -937,24 +994,19 @@ long SSL_SESSION_get_timeout(const SSL_SESSION *s) { if (s == NULL) return 0; - return (long)ossl_time_to_time_t(s->timeout); + return (long)s->timeout; } long SSL_SESSION_get_time(const SSL_SESSION *s) -{ - return (long) SSL_SESSION_get_time_ex(s); -} - -time_t SSL_SESSION_get_time_ex(const SSL_SESSION *s) { if (s == NULL) return 0; - return ossl_time_to_time_t(s->time); + return (long)s->time; } -time_t SSL_SESSION_set_time_ex(SSL_SESSION *s, time_t t) +long SSL_SESSION_set_time(SSL_SESSION *s, long t) { - OSSL_TIME new_time = ossl_time_from_time_t(t); + time_t new_time = (time_t)t; if (s == NULL) return 0; @@ -972,11 +1024,6 @@ time_t SSL_SESSION_set_time_ex(SSL_SESSION *s, time_t t) return t; } -long SSL_SESSION_set_time(SSL_SESSION *s, long t) -{ - return (long) SSL_SESSION_set_time_ex(s, (time_t) t); -} - int SSL_SESSION_get_protocol_version(const SSL_SESSION *s) { return s->ssl_version; @@ -1078,10 +1125,12 @@ X509 *SSL_SESSION_get0_peer(SSL_SESSION *s) return s->peer; } -EVP_PKEY *SSL_SESSION_get0_peer_rpk(SSL_SESSION *s) +#ifndef OPENSSL_NO_SESSION_LOOKUP +SSL_SESSION *SSL_magic_pending_session_ptr(void) { - return s->peer_rpk; + return (SSL_SESSION *) &g_pending_session_magic; } +#endif int SSL_SESSION_set1_id_context(SSL_SESSION *s, const unsigned char *sid_ctx, unsigned int sid_ctx_len) @@ -1110,11 +1159,10 @@ int SSL_SESSION_is_resumable(const SSL_SESSION *s) long SSL_CTX_set_timeout(SSL_CTX *s, long t) { long l; - if (s == NULL) return 0; - l = (long)ossl_time2seconds(s->session_timeout); - s->session_timeout = ossl_seconds2time(t); + l = s->session_timeout; + s->session_timeout = t; return l; } @@ -1122,58 +1170,49 @@ long SSL_CTX_get_timeout(const SSL_CTX *s) { if (s == NULL) return 0; - return (long)ossl_time2seconds(s->session_timeout); + return s->session_timeout; } int SSL_set_session_secret_cb(SSL *s, tls_session_secret_cb_fn tls_session_secret_cb, void *arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) + if (s == NULL) return 0; - - sc->ext.session_secret_cb = tls_session_secret_cb; - sc->ext.session_secret_cb_arg = arg; + s->ext.session_secret_cb = tls_session_secret_cb; + s->ext.session_secret_cb_arg = arg; return 1; } int SSL_set_session_ticket_ext_cb(SSL *s, tls_session_ticket_ext_cb_fn cb, void *arg) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) + if (s == NULL) return 0; - - sc->ext.session_ticket_cb = cb; - sc->ext.session_ticket_cb_arg = arg; + s->ext.session_ticket_cb = cb; + s->ext.session_ticket_cb_arg = arg; return 1; } int SSL_set_session_ticket_ext(SSL *s, void *ext_data, int ext_len) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->version >= TLS1_VERSION) { - OPENSSL_free(sc->ext.session_ticket); - sc->ext.session_ticket = NULL; - sc->ext.session_ticket = + if (s->version >= TLS1_VERSION) { + OPENSSL_free(s->ext.session_ticket); + s->ext.session_ticket = NULL; + s->ext.session_ticket = OPENSSL_malloc(sizeof(TLS_SESSION_TICKET_EXT) + ext_len); - if (sc->ext.session_ticket == NULL) + if (s->ext.session_ticket == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } if (ext_data != NULL) { - sc->ext.session_ticket->length = ext_len; - sc->ext.session_ticket->data = sc->ext.session_ticket + 1; - memcpy(sc->ext.session_ticket->data, ext_data, ext_len); + s->ext.session_ticket->length = ext_len; + s->ext.session_ticket->data = s->ext.session_ticket + 1; + memcpy(s->ext.session_ticket->data, ext_data, ext_len); } else { - sc->ext.session_ticket->length = 0; - sc->ext.session_ticket->data = NULL; + s->ext.session_ticket->length = 0; + s->ext.session_ticket->data = NULL; } return 1; @@ -1187,7 +1226,6 @@ void SSL_CTX_flush_sessions(SSL_CTX *s, long t) STACK_OF(SSL_SESSION) *sk; SSL_SESSION *current; unsigned long i; - const OSSL_TIME timeout = ossl_time_from_time_t(t); if (!CRYPTO_THREAD_write_lock(s->lock)) return; @@ -1205,7 +1243,7 @@ void SSL_CTX_flush_sessions(SSL_CTX *s, long t) */ while (s->session_cache_tail != NULL) { current = s->session_cache_tail; - if (t == 0 || sess_timedout(timeout, current)) { + if (t == 0 || sess_timedout((time_t)t, current)) { lh_SSL_SESSION_delete(s->sessions, current); SSL_SESSION_list_remove(s, current); current->not_resumable = 1; @@ -1231,12 +1269,11 @@ void SSL_CTX_flush_sessions(SSL_CTX *s, long t) sk_SSL_SESSION_pop_free(sk, SSL_SESSION_free); } -int ssl_clear_bad_session(SSL_CONNECTION *s) +int ssl_clear_bad_session(SSL *s) { if ((s->session != NULL) && !(s->shutdown & SSL_SENT_SHUTDOWN) && - !(SSL_in_init(SSL_CONNECTION_GET_SSL(s)) - || SSL_in_before(SSL_CONNECTION_GET_SSL(s)))) { + !(SSL_in_init(s) || SSL_in_before(s))) { SSL_CTX_remove_session(s->session_ctx, s->session); return 1; } else @@ -1345,7 +1382,7 @@ void (*SSL_CTX_sess_get_remove_cb(SSL_CTX *ctx)) (SSL_CTX *ctx, } void SSL_CTX_sess_set_get_cb(SSL_CTX *ctx, - SSL_SESSION *(*cb) (SSL *ssl, + SSL_SESSION *(*cb) (struct ssl_st *ssl, const unsigned char *data, int len, int *copy)) { diff --git a/openssl/src/ssl/ssl_stat.c b/openssl/src/ssl/ssl_stat.c index 8b93ccd4a..49f203402 100644 --- a/openssl/src/ssl/ssl_stat.c +++ b/openssl/src/ssl/ssl_stat.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2005 Nokia. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -13,9 +13,7 @@ const char *SSL_state_string_long(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL || ossl_statem_in_error(sc)) + if (ossl_statem_in_error(s)) return "error"; switch (SSL_get_state(s)) { @@ -37,8 +35,6 @@ const char *SSL_state_string_long(const SSL *s) return "SSLv3/TLS read server hello"; case TLS_ST_CR_CERT: return "SSLv3/TLS read server certificate"; - case TLS_ST_CR_COMP_CERT: - return "TLSv1.3 read server compressed certificate"; case TLS_ST_CR_KEY_EXCH: return "SSLv3/TLS read server key exchange"; case TLS_ST_CR_CERT_REQ: @@ -49,8 +45,6 @@ const char *SSL_state_string_long(const SSL *s) return "SSLv3/TLS read server done"; case TLS_ST_CW_CERT: return "SSLv3/TLS write client certificate"; - case TLS_ST_CW_COMP_CERT: - return "TLSv1.3 write client compressed certificate"; case TLS_ST_CW_KEY_EXCH: return "SSLv3/TLS write client key exchange"; case TLS_ST_CW_CERT_VRFY: @@ -75,8 +69,6 @@ const char *SSL_state_string_long(const SSL *s) return "SSLv3/TLS write server hello"; case TLS_ST_SW_CERT: return "SSLv3/TLS write certificate"; - case TLS_ST_SW_COMP_CERT: - return "TLSv1.3 write server compressed certificate"; case TLS_ST_SW_KEY_EXCH: return "SSLv3/TLS write key exchange"; case TLS_ST_SW_CERT_REQ: @@ -87,8 +79,6 @@ const char *SSL_state_string_long(const SSL *s) return "SSLv3/TLS write server done"; case TLS_ST_SR_CERT: return "SSLv3/TLS read client certificate"; - case TLS_ST_SR_COMP_CERT: - return "TLSv1.3 read client compressed certificate"; case TLS_ST_SR_KEY_EXCH: return "SSLv3/TLS read client key exchange"; case TLS_ST_SR_CERT_VRFY: @@ -130,9 +120,7 @@ const char *SSL_state_string_long(const SSL *s) const char *SSL_state_string(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL || ossl_statem_in_error(sc)) + if (ossl_statem_in_error(s)) return "SSLERR"; switch (SSL_get_state(s)) { @@ -158,8 +146,6 @@ const char *SSL_state_string(const SSL *s) return "TRSH"; case TLS_ST_CR_CERT: return "TRSC"; - case TLS_ST_CR_COMP_CERT: - return "TRSCC"; case TLS_ST_CR_KEY_EXCH: return "TRSKE"; case TLS_ST_CR_CERT_REQ: @@ -168,8 +154,6 @@ const char *SSL_state_string(const SSL *s) return "TRSD"; case TLS_ST_CW_CERT: return "TWCC"; - case TLS_ST_CW_COMP_CERT: - return "TWCCC"; case TLS_ST_CW_KEY_EXCH: return "TWCKE"; case TLS_ST_CW_CERT_VRFY: @@ -194,8 +178,6 @@ const char *SSL_state_string(const SSL *s) return "TWSH"; case TLS_ST_SW_CERT: return "TWSC"; - case TLS_ST_SW_COMP_CERT: - return "TWSCC"; case TLS_ST_SW_KEY_EXCH: return "TWSKE"; case TLS_ST_SW_CERT_REQ: @@ -204,8 +186,6 @@ const char *SSL_state_string(const SSL *s) return "TWSD"; case TLS_ST_SR_CERT: return "TRCC"; - case TLS_ST_SR_COMP_CERT: - return "TRCCC"; case TLS_ST_SR_KEY_EXCH: return "TRCKE"; case TLS_ST_SR_CERT_VRFY: @@ -332,6 +312,20 @@ const char *SSL_alert_desc_string(int value) return "BH"; case TLS1_AD_UNKNOWN_PSK_IDENTITY: return "UP"; +#ifndef OPENSSL_NO_NTLS + case NTLS_AD_UNSUPPORTED_SITE2SITE: + return "U2"; + case NTLS_AD_NO_AREA: + return "NA"; + case NTLS_AD_UNSUPPORTED_AREATYPE: + return "AT"; + case NTLS_AD_BAD_IBCPARAM: + return "BI"; + case NTLS_AD_UNSUPPORTED_IBCPARAM: + return "UI"; + case NTLS_AD_IDENTITY_NEED: + return "IN"; +#endif default: return "UK"; } @@ -402,6 +396,20 @@ const char *SSL_alert_desc_string_long(int value) return "unknown PSK identity"; case TLS1_AD_NO_APPLICATION_PROTOCOL: return "no application protocol"; +#ifndef OPENSSL_NO_NTLS + case NTLS_AD_UNSUPPORTED_SITE2SITE: + return "unsupported site2site"; + case NTLS_AD_NO_AREA: + return "no area"; + case NTLS_AD_UNSUPPORTED_AREATYPE: + return "unsupported areatype"; + case NTLS_AD_BAD_IBCPARAM: + return "bad ibc parameters"; + case NTLS_AD_UNSUPPORTED_IBCPARAM: + return "unsupported ibc parameters"; + case NTLS_AD_IDENTITY_NEED: + return "identity need"; +#endif default: return "unknown"; } diff --git a/openssl/src/ssl/ssl_txt.c b/openssl/src/ssl/ssl_txt.c index 9e9c2e10e..630fe25d4 100644 --- a/openssl/src/ssl/ssl_txt.c +++ b/openssl/src/ssl/ssl_txt.c @@ -128,14 +128,12 @@ int SSL_SESSION_print(BIO *bp, const SSL_SESSION *x) } } #endif - if (!ossl_time_is_zero(x->time)) { - if (BIO_printf(bp, "\n Start Time: %lld", - (long long)ossl_time_to_time_t(x->time)) <= 0) + if (x->time != 0L) { + if (BIO_printf(bp, "\n Start Time: %lld", (long long)x->time) <= 0) goto err; } - if (!ossl_time_is_zero(x->timeout)) { - if (BIO_printf(bp, "\n Timeout : %lld (sec)", - (long long)ossl_time2seconds(x->timeout)) <= 0) + if (x->timeout != 0L) { + if (BIO_printf(bp, "\n Timeout : %lld (sec)", (long long)x->timeout) <= 0) goto err; } if (BIO_puts(bp, "\n") <= 0) @@ -153,9 +151,21 @@ int SSL_SESSION_print(BIO *bp, const SSL_SESSION *x) if (istls13) { if (BIO_printf(bp, " Max Early Data: %u\n", - (unsigned int)x->ext.max_early_data) <= 0) + x->ext.max_early_data) <= 0) goto err; } +#ifndef OPENSSL_NO_QUIC + if (BIO_printf(bp, " QUIC: %s\n", x->is_quic ? "yes" : "no") <= 0) + goto err; + + if (x->quic_early_data_context) { + if (BIO_puts(bp, " QUIC early data ctx:\n") <= 0) + goto err; + if (BIO_dump_indent(bp, (const char *)x->quic_early_data_context, + (int)x->quic_early_data_context_len, 4) <= 0) + goto err; + } +#endif return 1; err: diff --git a/openssl/src/ssl/ssl_utst.c b/openssl/src/ssl/ssl_utst.c index 91be7398c..690db6d49 100644 --- a/openssl/src/ssl/ssl_utst.c +++ b/openssl/src/ssl/ssl_utst.c @@ -13,6 +13,7 @@ static const struct openssl_ssl_test_functions ssl_test_functions = { ssl_init_wbio_buffer, + ssl3_setup_buffers, }; const struct openssl_ssl_test_functions *SSL_test_functions(void) diff --git a/openssl/src/ssl/sslerr.h b/openssl/src/ssl/sslerr.h index 7d2bbd6c4..6715ac6a3 100644 --- a/openssl/src/ssl/sslerr.h +++ b/openssl/src/ssl/sslerr.h @@ -1,6 +1,6 @@ /* * Generated by util/mkerr.pl DO NOT EDIT - * Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy diff --git a/openssl/src/ssl/statem/extensions.c b/openssl/src/ssl/statem/extensions.c index 0a64ca224..112275133 100644 --- a/openssl/src/ssl/statem/extensions.c +++ b/openssl/src/ssl/statem/extensions.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -17,61 +17,54 @@ #include "internal/cryptlib.h" #include "../ssl_local.h" #include "statem_local.h" +#include "internal/cryptlib.h" -static int final_renegotiate(SSL_CONNECTION *s, unsigned int context, int sent); -static int init_server_name(SSL_CONNECTION *s, unsigned int context); -static int final_server_name(SSL_CONNECTION *s, unsigned int context, int sent); -static int final_ec_pt_formats(SSL_CONNECTION *s, unsigned int context, - int sent); -static int init_session_ticket(SSL_CONNECTION *s, unsigned int context); +static int final_renegotiate(SSL *s, unsigned int context, int sent); +static int init_server_name(SSL *s, unsigned int context); +static int final_server_name(SSL *s, unsigned int context, int sent); +static int final_ec_pt_formats(SSL *s, unsigned int context, int sent); +static int init_session_ticket(SSL *s, unsigned int context); #ifndef OPENSSL_NO_OCSP -static int init_status_request(SSL_CONNECTION *s, unsigned int context); +static int init_status_request(SSL *s, unsigned int context); #endif #ifndef OPENSSL_NO_NEXTPROTONEG -static int init_npn(SSL_CONNECTION *s, unsigned int context); +static int init_npn(SSL *s, unsigned int context); #endif -static int init_alpn(SSL_CONNECTION *s, unsigned int context); -static int final_alpn(SSL_CONNECTION *s, unsigned int context, int sent); -static int init_sig_algs_cert(SSL_CONNECTION *s, unsigned int context); -static int init_sig_algs(SSL_CONNECTION *s, unsigned int context); -static int init_server_cert_type(SSL_CONNECTION *sc, unsigned int context); -static int init_client_cert_type(SSL_CONNECTION *sc, unsigned int context); -static int init_certificate_authorities(SSL_CONNECTION *s, - unsigned int context); -static EXT_RETURN tls_construct_certificate_authorities(SSL_CONNECTION *s, - WPACKET *pkt, +static int init_alpn(SSL *s, unsigned int context); +static int final_alpn(SSL *s, unsigned int context, int sent); +static int init_sig_algs_cert(SSL *s, unsigned int context); +static int init_sig_algs(SSL *s, unsigned int context); +static int init_certificate_authorities(SSL *s, unsigned int context); +static EXT_RETURN tls_construct_certificate_authorities(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -static int tls_parse_certificate_authorities(SSL_CONNECTION *s, PACKET *pkt, +static int tls_parse_certificate_authorities(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_SRP -static int init_srp(SSL_CONNECTION *s, unsigned int context); +static int init_srp(SSL *s, unsigned int context); #endif -static int init_ec_point_formats(SSL_CONNECTION *s, unsigned int context); -static int init_etm(SSL_CONNECTION *s, unsigned int context); -static int init_ems(SSL_CONNECTION *s, unsigned int context); -static int final_ems(SSL_CONNECTION *s, unsigned int context, int sent); -static int init_psk_kex_modes(SSL_CONNECTION *s, unsigned int context); -static int final_key_share(SSL_CONNECTION *s, unsigned int context, int sent); +static int init_ec_point_formats(SSL *s, unsigned int context); +static int init_etm(SSL *s, unsigned int context); +static int init_ems(SSL *s, unsigned int context); +static int final_ems(SSL *s, unsigned int context, int sent); +static int init_psk_kex_modes(SSL *s, unsigned int context); +static int final_key_share(SSL *s, unsigned int context, int sent); #ifndef OPENSSL_NO_SRTP -static int init_srtp(SSL_CONNECTION *s, unsigned int context); +static int init_srtp(SSL *s, unsigned int context); +#endif +static int final_sig_algs(SSL *s, unsigned int context, int sent); +static int final_early_data(SSL *s, unsigned int context, int sent); +static int final_maxfragmentlen(SSL *s, unsigned int context, int sent); +static int init_post_handshake_auth(SSL *s, unsigned int context); +static int final_psk(SSL *s, unsigned int context, int sent); + +#ifndef OPENSSL_NO_QUIC +static int init_quic_transport_params(SSL *s, unsigned int context); +static int final_quic_transport_params_draft(SSL *s, unsigned int context, int sent); +static int final_quic_transport_params(SSL *s, unsigned int context, int sent); #endif -static int final_sig_algs(SSL_CONNECTION *s, unsigned int context, int sent); -static int final_early_data(SSL_CONNECTION *s, unsigned int context, int sent); -static int final_maxfragmentlen(SSL_CONNECTION *s, unsigned int context, - int sent); -static int init_post_handshake_auth(SSL_CONNECTION *s, unsigned int context); -static int final_psk(SSL_CONNECTION *s, unsigned int context, int sent); -static int tls_init_compress_certificate(SSL_CONNECTION *sc, unsigned int context); -static EXT_RETURN tls_construct_compress_certificate(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -static int tls_parse_compress_certificate(SSL_CONNECTION *sc, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); - /* Structure to define a built-in extension */ typedef struct extensions_definition_st { /* The defined type for the extension */ @@ -85,36 +78,31 @@ typedef struct extensions_definition_st { * Initialise extension before parsing. Always called for relevant contexts * even if extension not present */ - int (*init)(SSL_CONNECTION *s, unsigned int context); + int (*init)(SSL *s, unsigned int context); /* Parse extension sent from client to server */ - int (*parse_ctos)(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); + int (*parse_ctos)(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); /* Parse extension send from server to client */ - int (*parse_stoc)(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); + int (*parse_stoc)(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); /* Construct extension sent from server to client */ - EXT_RETURN (*construct_stoc)(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, + EXT_RETURN (*construct_stoc)(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); /* Construct extension sent from client to server */ - EXT_RETURN (*construct_ctos)(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, + EXT_RETURN (*construct_ctos)(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); /* * Finalise extension after parsing. Always called where an extensions was * initialised even if the extension was not present. |sent| is set to 1 if * the extension was seen, or 0 otherwise. */ - int (*final)(SSL_CONNECTION *s, unsigned int context, int sent); + int (*final)(SSL *s, unsigned int context, int sent); } EXTENSION_DEFINITION; /* * Definitions of all built-in extensions. NOTE: Changes in the number or order * of these extensions should be mirrored with equivalent changes to the * indexes ( TLSEXT_IDX_* ) defined in ssl_local.h. - * Extensions should be added to test/ext_internal_test.c as well, as that - * tests the ordering of the extensions. - * * Each extension has an initialiser, a client and * server side parser and a finaliser. The initialiser is called (if the * extension is relevant to the given context) even if we did not see the @@ -135,7 +123,7 @@ typedef struct extensions_definition_st { * NOTE: WebSphere Application Server 7+ cannot handle empty extensions at * the end, keep these extensions before signature_algorithm. */ -#define INVALID_EXTENSION { TLSEXT_TYPE_invalid, 0, NULL, NULL, NULL, NULL, NULL, NULL } +#define INVALID_EXTENSION { 0x10000, 0, NULL, NULL, NULL, NULL, NULL, NULL } static const EXTENSION_DEFINITION ext_defs[] = { { TLSEXT_TYPE_renegotiate, @@ -287,6 +275,19 @@ static const EXTENSION_DEFINITION ext_defs[] = { }, #else INVALID_EXTENSION, +#endif +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + { + TLSEXT_TYPE_delegated_credential, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST + | SSL_EXT_TLS1_3_ONLY | SSL_EXT_TLS1_3_CERTIFICATE, + NULL, + tls_parse_ctos_delegated_credential, tls_parse_stoc_delegated_credential, + tls_construct_stoc_delegated_credential, tls_construct_ctos_delegated_credential, + NULL + }, +#else + INVALID_EXTENSION, #endif { TLSEXT_TYPE_extended_master_secret, @@ -311,24 +312,6 @@ static const EXTENSION_DEFINITION ext_defs[] = { NULL, tls_construct_ctos_post_handshake_auth, NULL, }, - { - TLSEXT_TYPE_client_cert_type, - SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS - | SSL_EXT_TLS1_2_SERVER_HELLO, - init_client_cert_type, - tls_parse_ctos_client_cert_type, tls_parse_stoc_client_cert_type, - tls_construct_stoc_client_cert_type, tls_construct_ctos_client_cert_type, - NULL - }, - { - TLSEXT_TYPE_server_cert_type, - SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS - | SSL_EXT_TLS1_2_SERVER_HELLO, - init_server_cert_type, - tls_parse_ctos_server_cert_type, tls_parse_stoc_server_cert_type, - tls_construct_stoc_server_cert_type, tls_construct_ctos_server_cert_type, - NULL - }, { TLSEXT_TYPE_signature_algorithms, SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST, @@ -374,26 +357,6 @@ static const EXTENSION_DEFINITION ext_defs[] = { NULL, tls_parse_ctos_cookie, tls_parse_stoc_cookie, tls_construct_stoc_cookie, tls_construct_ctos_cookie, NULL }, - { - /* - * Special unsolicited ServerHello extension only used when - * SSL_OP_CRYPTOPRO_TLSEXT_BUG is set. We allow it in a ClientHello but - * ignore it. - */ - TLSEXT_TYPE_cryptopro_bug, - SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO - | SSL_EXT_TLS1_2_AND_BELOW_ONLY, - NULL, NULL, NULL, tls_construct_stoc_cryptopro_bug, NULL, NULL - }, - { - TLSEXT_TYPE_compress_certificate, - SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST - | SSL_EXT_TLS_IMPLEMENTATION_ONLY | SSL_EXT_TLS1_3_ONLY, - tls_init_compress_certificate, - tls_parse_compress_certificate, tls_parse_compress_certificate, - tls_construct_compress_certificate, tls_construct_compress_certificate, - NULL - }, { TLSEXT_TYPE_early_data, SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS @@ -411,6 +374,42 @@ static const EXTENSION_DEFINITION ext_defs[] = { tls_construct_certificate_authorities, tls_construct_certificate_authorities, NULL, }, +#ifndef OPENSSL_NO_QUIC + { + TLSEXT_TYPE_quic_transport_parameters_draft, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS + | SSL_EXT_TLS_IMPLEMENTATION_ONLY | SSL_EXT_TLS1_3_ONLY, + init_quic_transport_params, + tls_parse_ctos_quic_transport_params_draft, tls_parse_stoc_quic_transport_params_draft, + tls_construct_stoc_quic_transport_params_draft, tls_construct_ctos_quic_transport_params_draft, + final_quic_transport_params_draft, + }, + { + TLSEXT_TYPE_quic_transport_parameters, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS + | SSL_EXT_TLS_IMPLEMENTATION_ONLY | SSL_EXT_TLS1_3_ONLY, + init_quic_transport_params, + tls_parse_ctos_quic_transport_params, tls_parse_stoc_quic_transport_params, + tls_construct_stoc_quic_transport_params, tls_construct_ctos_quic_transport_params, + final_quic_transport_params, + }, +#else + INVALID_EXTENSION, + INVALID_EXTENSION, +#endif +#ifndef OPENSSL_NO_CERT_COMPRESSION + { + TLSEXT_TYPE_compress_certificate, + SSL_EXT_TLS1_3_ONLY | SSL_EXT_CLIENT_HELLO + | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST, + NULL, + tls_parse_compress_cert, tls_parse_compress_cert, + tls_construct_compress_cert, tls_construct_compress_cert, + NULL + }, +#else + INVALID_EXTENSION, +#endif { /* Must be immediately before pre_shared_key */ TLSEXT_TYPE_padding, @@ -429,26 +428,14 @@ static const EXTENSION_DEFINITION ext_defs[] = { } }; -/* Returns a TLSEXT_TYPE for the given index */ -unsigned int ossl_get_extension_type(size_t idx) -{ - size_t num_exts = OSSL_NELEM(ext_defs); - - if (idx >= num_exts) - return TLSEXT_TYPE_out_of_range; - - return ext_defs[idx].type; -} - /* Check whether an extension's context matches the current context */ -static int validate_context(SSL_CONNECTION *s, unsigned int extctx, - unsigned int thisctx) +static int validate_context(SSL *s, unsigned int extctx, unsigned int thisctx) { /* Check we're allowed to use this extension in this context */ if ((thisctx & extctx) == 0) return 0; - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { if ((extctx & SSL_EXT_TLS_ONLY) != 0) return 0; } else if ((extctx & SSL_EXT_DTLS_ONLY) != 0) { @@ -458,8 +445,7 @@ static int validate_context(SSL_CONNECTION *s, unsigned int extctx, return 1; } -int tls_validate_all_contexts(SSL_CONNECTION *s, unsigned int thisctx, - RAW_EXTENSION *exts) +int tls_validate_all_contexts(SSL *s, unsigned int thisctx, RAW_EXTENSION *exts) { size_t i, num_exts, builtin_num = OSSL_NELEM(ext_defs), offset; RAW_EXTENSION *thisext; @@ -503,9 +489,9 @@ int tls_validate_all_contexts(SSL_CONNECTION *s, unsigned int thisctx, * indicate the extension is not allowed. If returning 1 then |*found| is set to * the definition for the extension we found. */ -static int verify_extension(SSL_CONNECTION *s, unsigned int context, - unsigned int type, custom_ext_methods *meths, - RAW_EXTENSION *rawexlist, RAW_EXTENSION **found) +static int verify_extension(SSL *s, unsigned int context, unsigned int type, + custom_ext_methods *meths, RAW_EXTENSION *rawexlist, + RAW_EXTENSION **found) { size_t i; size_t builtin_num = OSSL_NELEM(ext_defs); @@ -551,8 +537,7 @@ static int verify_extension(SSL_CONNECTION *s, unsigned int context, * the extension is relevant for the current context |thisctx| or not. Returns * 1 if the extension is relevant for this context, and 0 otherwise */ -int extension_is_relevant(SSL_CONNECTION *s, unsigned int extctx, - unsigned int thisctx) +int extension_is_relevant(SSL *s, unsigned int extctx, unsigned int thisctx) { int is_tls13; @@ -563,9 +548,9 @@ int extension_is_relevant(SSL_CONNECTION *s, unsigned int extctx, if ((thisctx & SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST) != 0) is_tls13 = 1; else - is_tls13 = SSL_CONNECTION_IS_TLS13(s); + is_tls13 = SSL_IS_TLS13(s); - if ((SSL_CONNECTION_IS_DTLS(s) + if ((SSL_IS_DTLS(s) && (extctx & SSL_EXT_TLS_IMPLEMENTATION_ONLY) != 0) || (s->version == SSL3_VERSION && (extctx & SSL_EXT_SSL3_ALLOWED) == 0) @@ -602,8 +587,7 @@ int extension_is_relevant(SSL_CONNECTION *s, unsigned int extctx, * found, or an internal error occurred. We only check duplicates for * extensions that we know about. We ignore others. */ -int tls_collect_extensions(SSL_CONNECTION *s, PACKET *packet, - unsigned int context, +int tls_collect_extensions(SSL *s, PACKET *packet, unsigned int context, RAW_EXTENSION **res, size_t *len, int init) { PACKET extensions = *packet; @@ -625,7 +609,7 @@ int tls_collect_extensions(SSL_CONNECTION *s, PACKET *packet, num_exts = OSSL_NELEM(ext_defs) + (exts != NULL ? exts->meths_count : 0); raw_extensions = OPENSSL_zalloc(num_exts * sizeof(*raw_extensions)); if (raw_extensions == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } @@ -676,10 +660,6 @@ int tls_collect_extensions(SSL_CONNECTION *s, PACKET *packet, && type != TLSEXT_TYPE_renegotiate && type != TLSEXT_TYPE_signed_certificate_timestamp && (s->ext.extflags[idx] & SSL_EXT_FLAG_SENT) == 0 -#ifndef OPENSSL_NO_GOST - && !((context & SSL_EXT_TLS1_2_SERVER_HELLO) != 0 - && type == TLSEXT_TYPE_cryptopro_bug) -#endif ) { SSLfatal(s, SSL_AD_UNSUPPORTED_EXTENSION, SSL_R_UNSOLICITED_EXTENSION); @@ -691,8 +671,8 @@ int tls_collect_extensions(SSL_CONNECTION *s, PACKET *packet, thisex->type = type; thisex->received_order = i++; if (s->ext.debug_cb) - s->ext.debug_cb(SSL_CONNECTION_GET_SSL(s), !s->server, - thisex->type, PACKET_data(&thisex->data), + s->ext.debug_cb(s, !s->server, thisex->type, + PACKET_data(&thisex->data), PACKET_remaining(&thisex->data), s->ext.debug_arg); } @@ -734,11 +714,11 @@ int tls_collect_extensions(SSL_CONNECTION *s, PACKET *packet, * Certificate. Returns 1 on success or 0 on failure. If an extension is not * present this counted as success. */ -int tls_parse_extension(SSL_CONNECTION *s, TLSEXT_INDEX idx, int context, +int tls_parse_extension(SSL *s, TLSEXT_INDEX idx, int context, RAW_EXTENSION *exts, X509 *x, size_t chainidx) { RAW_EXTENSION *currext = &exts[idx]; - int (*parser)(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, X509 *x, + int (*parser)(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) = NULL; /* Skip if the extension is not present */ @@ -784,8 +764,7 @@ int tls_parse_extension(SSL_CONNECTION *s, TLSEXT_INDEX idx, int context, * working on a Certificate message then we also pass the Certificate |x| and * its position in the |chainidx|, with 0 being the first certificate. */ -int tls_parse_all_extensions(SSL_CONNECTION *s, int context, - RAW_EXTENSION *exts, X509 *x, +int tls_parse_all_extensions(SSL *s, int context, RAW_EXTENSION *exts, X509 *x, size_t chainidx, int fin) { size_t i, numexts = OSSL_NELEM(ext_defs); @@ -820,8 +799,8 @@ int tls_parse_all_extensions(SSL_CONNECTION *s, int context, return 1; } -int should_add_extension(SSL_CONNECTION *s, unsigned int extctx, - unsigned int thisctx, int max_version) +int should_add_extension(SSL *s, unsigned int extctx, unsigned int thisctx, + int max_version) { /* Skip if not relevant for our context */ if ((extctx & thisctx) == 0) @@ -831,7 +810,7 @@ int should_add_extension(SSL_CONNECTION *s, unsigned int extctx, if (!extension_is_relevant(s, extctx, thisctx) || ((extctx & SSL_EXT_TLS1_3_ONLY) != 0 && (thisctx & SSL_EXT_CLIENT_HELLO) != 0 - && (SSL_CONNECTION_IS_DTLS(s) || max_version < TLS1_3_VERSION))) + && (SSL_IS_DTLS(s) || max_version < TLS1_3_VERSION))) return 0; return 1; @@ -845,14 +824,12 @@ int should_add_extension(SSL_CONNECTION *s, unsigned int extctx, * 0 being the first in the chain). Returns 1 on success or 0 on failure. On a * failure construction stops at the first extension to fail to construct. */ -int tls_construct_extensions(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +int tls_construct_extensions(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { size_t i; int min_version, max_version = 0, reason; const EXTENSION_DEFINITION *thisexd; - int for_comp = (context & SSL_EXT_TLS1_3_CERTIFICATE_COMPRESSION) != 0; if (!WPACKET_start_sub_packet_u16(pkt) /* @@ -863,17 +840,15 @@ int tls_construct_extensions(SSL_CONNECTION *s, WPACKET *pkt, || ((context & (SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO)) != 0 && !WPACKET_set_flags(pkt, - WPACKET_FLAGS_ABANDON_ON_ZERO_LENGTH))) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + WPACKET_FLAGS_ABANDON_ON_ZERO_LENGTH))) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } if ((context & SSL_EXT_CLIENT_HELLO) != 0) { reason = ssl_get_min_max_version(s, &min_version, &max_version, NULL); if (reason != 0) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, reason); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, reason); return 0; } } @@ -889,8 +864,7 @@ int tls_construct_extensions(SSL_CONNECTION *s, WPACKET *pkt, } for (i = 0, thisexd = ext_defs; i < OSSL_NELEM(ext_defs); i++, thisexd++) { - EXT_RETURN (*construct)(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, + EXT_RETURN (*construct)(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); EXT_RETURN ret; @@ -917,8 +891,7 @@ int tls_construct_extensions(SSL_CONNECTION *s, WPACKET *pkt, } if (!WPACKET_close(pkt)) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } @@ -932,7 +905,7 @@ int tls_construct_extensions(SSL_CONNECTION *s, WPACKET *pkt, * otherwise. These functions return 1 on success or 0 on failure. */ -static int final_renegotiate(SSL_CONNECTION *s, unsigned int context, int sent) +static int final_renegotiate(SSL *s, unsigned int context, int sent) { if (!s->server) { /* @@ -972,7 +945,7 @@ static ossl_inline void ssl_tsan_decr(const SSL_CTX *ctx, } } -static int init_server_name(SSL_CONNECTION *s, unsigned int context) +static int init_server_name(SSL *s, unsigned int context) { if (s->server) { s->servername_done = 0; @@ -984,24 +957,22 @@ static int init_server_name(SSL_CONNECTION *s, unsigned int context) return 1; } -static int final_server_name(SSL_CONNECTION *s, unsigned int context, int sent) +static int final_server_name(SSL *s, unsigned int context, int sent) { int ret = SSL_TLSEXT_ERR_NOACK; int altmp = SSL_AD_UNRECOGNIZED_NAME; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - int was_ticket = (SSL_get_options(ssl) & SSL_OP_NO_TICKET) == 0; + int was_ticket = (SSL_get_options(s) & SSL_OP_NO_TICKET) == 0; - if (!ossl_assert(sctx != NULL) || !ossl_assert(s->session_ctx != NULL)) { + if (!ossl_assert(s->ctx != NULL) || !ossl_assert(s->session_ctx != NULL)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - if (sctx->ext.servername_cb != NULL) - ret = sctx->ext.servername_cb(ssl, &altmp, - sctx->ext.servername_arg); + if (s->ctx->ext.servername_cb != NULL) + ret = s->ctx->ext.servername_cb(s, &altmp, + s->ctx->ext.servername_arg); else if (s->session_ctx->ext.servername_cb != NULL) - ret = s->session_ctx->ext.servername_cb(ssl, &altmp, + ret = s->session_ctx->ext.servername_cb(s, &altmp, s->session_ctx->ext.servername_arg); /* @@ -1029,9 +1000,9 @@ static int final_server_name(SSL_CONNECTION *s, unsigned int context, int sent) * context, to avoid the confusing situation of having sess_accept_good * exceed sess_accept (zero) for the new context. */ - if (SSL_IS_FIRST_HANDSHAKE(s) && sctx != s->session_ctx + if (SSL_IS_FIRST_HANDSHAKE(s) && s->ctx != s->session_ctx && s->hello_retry_request == SSL_HRR_NONE) { - ssl_tsan_counter(sctx, &sctx->stats.sess_accept); + ssl_tsan_counter(s->ctx, &s->ctx->stats.sess_accept); ssl_tsan_decr(s->session_ctx, &s->session_ctx->stats.sess_accept); } @@ -1041,10 +1012,10 @@ static int final_server_name(SSL_CONNECTION *s, unsigned int context, int sent) * Also, if this is not a resumption, create a new session ID */ if (ret == SSL_TLSEXT_ERR_OK && s->ext.ticket_expected - && was_ticket && (SSL_get_options(ssl) & SSL_OP_NO_TICKET) != 0) { + && was_ticket && (SSL_get_options(s) & SSL_OP_NO_TICKET) != 0) { s->ext.ticket_expected = 0; if (!s->hit) { - SSL_SESSION* ss = SSL_get_session(ssl); + SSL_SESSION* ss = SSL_get_session(s); if (ss != NULL) { OPENSSL_free(ss->ext.tick); @@ -1070,7 +1041,7 @@ static int final_server_name(SSL_CONNECTION *s, unsigned int context, int sent) case SSL_TLSEXT_ERR_ALERT_WARNING: /* TLSv1.3 doesn't have warning alerts so we suppress this */ - if (!SSL_CONNECTION_IS_TLS13(s)) + if (!SSL_IS_TLS13(s)) ssl3_send_alert(s, SSL3_AL_WARNING, altmp); s->servername_done = 0; return 1; @@ -1084,8 +1055,7 @@ static int final_server_name(SSL_CONNECTION *s, unsigned int context, int sent) } } -static int final_ec_pt_formats(SSL_CONNECTION *s, unsigned int context, - int sent) +static int final_ec_pt_formats(SSL *s, unsigned int context, int sent) { unsigned long alg_k, alg_a; @@ -1123,7 +1093,7 @@ static int final_ec_pt_formats(SSL_CONNECTION *s, unsigned int context, return 1; } -static int init_session_ticket(SSL_CONNECTION *s, unsigned int context) +static int init_session_ticket(SSL *s, unsigned int context) { if (!s->server) s->ext.ticket_expected = 0; @@ -1132,7 +1102,7 @@ static int init_session_ticket(SSL_CONNECTION *s, unsigned int context) } #ifndef OPENSSL_NO_OCSP -static int init_status_request(SSL_CONNECTION *s, unsigned int context) +static int init_status_request(SSL *s, unsigned int context) { if (s->server) { s->ext.status_type = TLSEXT_STATUSTYPE_nothing; @@ -1151,7 +1121,7 @@ static int init_status_request(SSL_CONNECTION *s, unsigned int context) #endif #ifndef OPENSSL_NO_NEXTPROTONEG -static int init_npn(SSL_CONNECTION *s, unsigned int context) +static int init_npn(SSL *s, unsigned int context) { s->s3.npn_seen = 0; @@ -1159,7 +1129,7 @@ static int init_npn(SSL_CONNECTION *s, unsigned int context) } #endif -static int init_alpn(SSL_CONNECTION *s, unsigned int context) +static int init_alpn(SSL *s, unsigned int context) { OPENSSL_free(s->s3.alpn_selected); s->s3.alpn_selected = NULL; @@ -1172,12 +1142,12 @@ static int init_alpn(SSL_CONNECTION *s, unsigned int context) return 1; } -static int final_alpn(SSL_CONNECTION *s, unsigned int context, int sent) +static int final_alpn(SSL *s, unsigned int context, int sent) { if (!s->server && !sent && s->session->ext.alpn_selected != NULL) s->ext.early_data_ok = 0; - if (!s->server || !SSL_CONNECTION_IS_TLS13(s)) + if (!s->server || !SSL_IS_TLS13(s)) return 1; /* @@ -1192,7 +1162,7 @@ static int final_alpn(SSL_CONNECTION *s, unsigned int context, int sent) return tls_handle_alpn(s); } -static int init_sig_algs(SSL_CONNECTION *s, unsigned int context) +static int init_sig_algs(SSL *s, unsigned int context) { /* Clear any signature algorithms extension received */ OPENSSL_free(s->s3.tmp.peer_sigalgs); @@ -1202,8 +1172,7 @@ static int init_sig_algs(SSL_CONNECTION *s, unsigned int context) return 1; } -static int init_sig_algs_cert(SSL_CONNECTION *s, - ossl_unused unsigned int context) +static int init_sig_algs_cert(SSL *s, ossl_unused unsigned int context) { /* Clear any signature algorithms extension received */ OPENSSL_free(s->s3.tmp.peer_cert_sigalgs); @@ -1214,7 +1183,7 @@ static int init_sig_algs_cert(SSL_CONNECTION *s, } #ifndef OPENSSL_NO_SRP -static int init_srp(SSL_CONNECTION *s, unsigned int context) +static int init_srp(SSL *s, unsigned int context) { OPENSSL_free(s->srp_ctx.login); s->srp_ctx.login = NULL; @@ -1223,7 +1192,7 @@ static int init_srp(SSL_CONNECTION *s, unsigned int context) } #endif -static int init_ec_point_formats(SSL_CONNECTION *s, unsigned int context) +static int init_ec_point_formats(SSL *s, unsigned int context) { OPENSSL_free(s->ext.peer_ecpointformats); s->ext.peer_ecpointformats = NULL; @@ -1232,14 +1201,14 @@ static int init_ec_point_formats(SSL_CONNECTION *s, unsigned int context) return 1; } -static int init_etm(SSL_CONNECTION *s, unsigned int context) +static int init_etm(SSL *s, unsigned int context) { s->ext.use_etm = 0; return 1; } -static int init_ems(SSL_CONNECTION *s, unsigned int context) +static int init_ems(SSL *s, unsigned int context) { if (s->s3.flags & TLS1_FLAGS_RECEIVED_EXTMS) { s->s3.flags &= ~TLS1_FLAGS_RECEIVED_EXTMS; @@ -1249,7 +1218,7 @@ static int init_ems(SSL_CONNECTION *s, unsigned int context) return 1; } -static int final_ems(SSL_CONNECTION *s, unsigned int context, int sent) +static int final_ems(SSL *s, unsigned int context, int sent) { /* * Check extended master secret extension is not dropped on @@ -1275,15 +1244,14 @@ static int final_ems(SSL_CONNECTION *s, unsigned int context, int sent) return 1; } -static int init_certificate_authorities(SSL_CONNECTION *s, unsigned int context) +static int init_certificate_authorities(SSL *s, unsigned int context) { sk_X509_NAME_pop_free(s->s3.tmp.peer_ca_names, X509_NAME_free); s->s3.tmp.peer_ca_names = NULL; return 1; } -static EXT_RETURN tls_construct_certificate_authorities(SSL_CONNECTION *s, - WPACKET *pkt, +static EXT_RETURN tls_construct_certificate_authorities(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) @@ -1312,7 +1280,7 @@ static EXT_RETURN tls_construct_certificate_authorities(SSL_CONNECTION *s, return EXT_RETURN_SENT; } -static int tls_parse_certificate_authorities(SSL_CONNECTION *s, PACKET *pkt, +static int tls_parse_certificate_authorities(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1326,7 +1294,7 @@ static int tls_parse_certificate_authorities(SSL_CONNECTION *s, PACKET *pkt, } #ifndef OPENSSL_NO_SRTP -static int init_srtp(SSL_CONNECTION *s, unsigned int context) +static int init_srtp(SSL *s, unsigned int context) { if (s->server) s->srtp_profile = NULL; @@ -1335,9 +1303,9 @@ static int init_srtp(SSL_CONNECTION *s, unsigned int context) } #endif -static int final_sig_algs(SSL_CONNECTION *s, unsigned int context, int sent) +static int final_sig_algs(SSL *s, unsigned int context, int sent) { - if (!sent && SSL_CONNECTION_IS_TLS13(s) && !s->hit) { + if (!sent && SSL_IS_TLS13(s) && !s->hit) { SSLfatal(s, TLS13_AD_MISSING_EXTENSION, SSL_R_MISSING_SIGALGS_EXTENSION); return 0; @@ -1346,10 +1314,10 @@ static int final_sig_algs(SSL_CONNECTION *s, unsigned int context, int sent) return 1; } -static int final_key_share(SSL_CONNECTION *s, unsigned int context, int sent) +static int final_key_share(SSL *s, unsigned int context, int sent) { #if !defined(OPENSSL_NO_TLS1_3) - if (!SSL_CONNECTION_IS_TLS13(s)) + if (!SSL_IS_TLS13(s)) return 1; /* Nothing to do for key_share in an HRR */ @@ -1449,11 +1417,7 @@ static int final_key_share(SSL_CONNECTION *s, unsigned int context, int sent) group_id = pgroups[i]; if (check_in_list(s, group_id, clntgroups, clnt_num_groups, - 1) - && tls_group_allowed(s, group_id, - SSL_SECOP_CURVE_SUPPORTED) - && tls_valid_group(s, group_id, TLS1_3_VERSION, - TLS1_3_VERSION, 0, NULL)) + 1)) break; } @@ -1510,14 +1474,13 @@ static int final_key_share(SSL_CONNECTION *s, unsigned int context, int sent) return 1; } -static int init_psk_kex_modes(SSL_CONNECTION *s, unsigned int context) +static int init_psk_kex_modes(SSL *s, unsigned int context) { s->ext.psk_kex_mode = TLSEXT_KEX_MODE_FLAG_NONE; return 1; } -int tls_psk_do_binder(SSL_CONNECTION *s, const EVP_MD *md, - const unsigned char *msgstart, +int tls_psk_do_binder(SSL *s, const EVP_MD *md, const unsigned char *msgstart, size_t binderoffset, const unsigned char *binderin, unsigned char *binderout, SSL_SESSION *sess, int sign, int external) @@ -1527,16 +1490,18 @@ int tls_psk_do_binder(SSL_CONNECTION *s, const EVP_MD *md, unsigned char hash[EVP_MAX_MD_SIZE], binderkey[EVP_MAX_MD_SIZE]; unsigned char finishedkey[EVP_MAX_MD_SIZE], tmpbinder[EVP_MAX_MD_SIZE]; unsigned char *early_secret; - /* ASCII: "res binder", in hex for EBCDIC compatibility */ - static const unsigned char resumption_label[] = "\x72\x65\x73\x20\x62\x69\x6E\x64\x65\x72"; - /* ASCII: "ext binder", in hex for EBCDIC compatibility */ - static const unsigned char external_label[] = "\x65\x78\x74\x20\x62\x69\x6E\x64\x65\x72"; +#ifdef CHARSET_EBCDIC + static const unsigned char resumption_label[] = { 0x72, 0x65, 0x73, 0x20, 0x62, 0x69, 0x6E, 0x64, 0x65, 0x72, 0x00 }; + static const unsigned char external_label[] = { 0x65, 0x78, 0x74, 0x20, 0x62, 0x69, 0x6E, 0x64, 0x65, 0x72, 0x00 }; +#else + static const unsigned char resumption_label[] = "res binder"; + static const unsigned char external_label[] = "ext binder"; +#endif const unsigned char *label; size_t bindersize, labelsize, hashsize; int hashsizei = EVP_MD_get_size(md); int ret = -1; int usepskfored = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* Ensure cast to size_t is safe */ if (!ossl_assert(hashsizei >= 0)) { @@ -1656,8 +1621,8 @@ int tls_psk_do_binder(SSL_CONNECTION *s, const EVP_MD *md, goto err; } - mackey = EVP_PKEY_new_raw_private_key_ex(sctx->libctx, "HMAC", - sctx->propq, finishedkey, + mackey = EVP_PKEY_new_raw_private_key_ex(s->ctx->libctx, "HMAC", + s->ctx->propq, finishedkey, hashsize); if (mackey == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); @@ -1668,8 +1633,8 @@ int tls_psk_do_binder(SSL_CONNECTION *s, const EVP_MD *md, binderout = tmpbinder; bindersize = hashsize; - if (EVP_DigestSignInit_ex(mctx, NULL, EVP_MD_get0_name(md), sctx->libctx, - sctx->propq, mackey, NULL) <= 0 + if (EVP_DigestSignInit_ex(mctx, NULL, EVP_MD_get0_name(md), s->ctx->libctx, + s->ctx->propq, mackey, NULL) <= 0 || EVP_DigestSignUpdate(mctx, hash, hashsize) <= 0 || EVP_DigestSignFinal(mctx, binderout, &bindersize) <= 0 || bindersize != hashsize) { @@ -1695,7 +1660,7 @@ int tls_psk_do_binder(SSL_CONNECTION *s, const EVP_MD *md, return ret; } -static int final_early_data(SSL_CONNECTION *s, unsigned int context, int sent) +static int final_early_data(SSL *s, unsigned int context, int sent) { if (!sent) return 1; @@ -1722,7 +1687,7 @@ static int final_early_data(SSL_CONNECTION *s, unsigned int context, int sent) || !s->ext.early_data_ok || s->hello_retry_request != SSL_HRR_NONE || (s->allow_early_data_cb != NULL - && !s->allow_early_data_cb(SSL_CONNECTION_GET_SSL(s), + && !s->allow_early_data_cb(s, s->allow_early_data_cb_data))) { s->ext.early_data = SSL_EARLY_DATA_REJECTED; } else { @@ -1738,8 +1703,7 @@ static int final_early_data(SSL_CONNECTION *s, unsigned int context, int sent) return 1; } -static int final_maxfragmentlen(SSL_CONNECTION *s, unsigned int context, - int sent) +static int final_maxfragmentlen(SSL *s, unsigned int context, int sent) { /* * Session resumption on server-side with MFL extension active @@ -1751,18 +1715,19 @@ static int final_maxfragmentlen(SSL_CONNECTION *s, unsigned int context, return 0; } - if (s->session && USE_MAX_FRAGMENT_LENGTH_EXT(s->session)) { - s->rlayer.rrlmethod->set_max_frag_len(s->rlayer.rrl, - GET_MAX_FRAGMENT_LENGTH(s->session)); - s->rlayer.wrlmethod->set_max_frag_len(s->rlayer.wrl, - ssl_get_max_send_fragment(s)); - } + /* Current SSL buffer is lower than requested MFL */ + if (s->session && USE_MAX_FRAGMENT_LENGTH_EXT(s->session) + && s->max_send_fragment < GET_MAX_FRAGMENT_LENGTH(s->session)) + /* trigger a larger buffer reallocation */ + if (!ssl3_setup_buffers(s)) { + /* SSLfatal() already called */ + return 0; + } return 1; } -static int init_post_handshake_auth(SSL_CONNECTION *s, - ossl_unused unsigned int context) +static int init_post_handshake_auth(SSL *s, ossl_unused unsigned int context) { s->post_handshake_auth = SSL_PHA_NONE; @@ -1773,7 +1738,7 @@ static int init_post_handshake_auth(SSL_CONNECTION *s, * If clients offer "pre_shared_key" without a "psk_key_exchange_modes" * extension, servers MUST abort the handshake. */ -static int final_psk(SSL_CONNECTION *s, unsigned int context, int sent) +static int final_psk(SSL *s, unsigned int context, int sent) { if (s->server && sent && s->clienthello != NULL && !s->clienthello->pre_proc_exts[TLSEXT_IDX_psk_kex_modes].present) { @@ -1785,149 +1750,281 @@ static int final_psk(SSL_CONNECTION *s, unsigned int context, int sent) return 1; } -static int tls_init_compress_certificate(SSL_CONNECTION *sc, unsigned int context) +#ifndef OPENSSL_NO_QUIC +static int init_quic_transport_params(SSL *s, unsigned int context) { - memset(sc->ext.compress_certificate_from_peer, 0, - sizeof(sc->ext.compress_certificate_from_peer)); return 1; } -/* The order these are put into the packet imply a preference order: [brotli, zlib, zstd] */ -static EXT_RETURN tls_construct_compress_certificate(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) +static int final_quic_transport_params_draft(SSL *s, unsigned int context, + int sent) { -#ifndef OPENSSL_NO_COMP_ALG - int i; - - if (!ossl_comp_has_alg(0)) - return EXT_RETURN_NOT_SENT; + return 1; +} - /* Server: Don't attempt to compress a non-X509 (i.e. an RPK) */ - if (sc->server && sc->ext.server_cert_type != TLSEXT_cert_type_x509) { - sc->cert_comp_prefs[0] = TLSEXT_comp_cert_none; - return EXT_RETURN_NOT_SENT; +static int final_quic_transport_params(SSL *s, unsigned int context, int sent) +{ + /* called after final_quic_transport_params_draft */ + if (SSL_IS_QUIC(s)) { + if (s->ext.peer_quic_transport_params_len == 0 + && s->ext.peer_quic_transport_params_draft_len == 0) { + SSLfatal(s, SSL_AD_MISSING_EXTENSION, + SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION); + return 0; + } + /* if we got both, discard the one we can't use */ + if (s->ext.peer_quic_transport_params_len != 0 + && s->ext.peer_quic_transport_params_draft_len != 0) { + if (s->quic_transport_version == TLSEXT_TYPE_quic_transport_parameters_draft) { + OPENSSL_free(s->ext.peer_quic_transport_params); + s->ext.peer_quic_transport_params = NULL; + s->ext.peer_quic_transport_params_len = 0; + } else { + OPENSSL_free(s->ext.peer_quic_transport_params_draft); + s->ext.peer_quic_transport_params_draft = NULL; + s->ext.peer_quic_transport_params_draft_len = 0; + } + } } - /* Client: If we sent a client cert-type extension, don't indicate compression */ - if (!sc->server && sc->ext.client_cert_type_ctos) { - sc->cert_comp_prefs[0] = TLSEXT_comp_cert_none; + return 1; +} +#endif + +#ifndef OPENSSL_NO_CERT_COMPRESSION +EXT_RETURN tls_construct_compress_cert(SSL *s, WPACKET *pkt, + unsigned int context, + X509 *x, size_t chainidx) +{ + int i; + int first = 1; + + if (s->cert_comp_algs == NULL) { return EXT_RETURN_NOT_SENT; } - /* Do not indicate we support receiving compressed certificates */ - if ((sc->options & SSL_OP_NO_RX_CERTIFICATE_COMPRESSION) != 0) - return EXT_RETURN_NOT_SENT; + for (i = 0; i < sk_CERT_COMP_num(s->cert_comp_algs); ++i) { + const CERT_COMP *comp = sk_CERT_COMP_value(s->cert_comp_algs, i); - if (sc->cert_comp_prefs[0] == TLSEXT_comp_cert_none) - return EXT_RETURN_NOT_SENT; + if (comp->decompress == NULL) + continue; - if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_compress_certificate) - || !WPACKET_start_sub_packet_u16(pkt) - || !WPACKET_start_sub_packet_u8(pkt)) - goto err; + if (first && + (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_compress_certificate) + /* bytes of extension_data */ + || !WPACKET_start_sub_packet_u16(pkt) + /* bytes of algorithms */ + || !WPACKET_start_sub_packet_u8(pkt))) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } - for (i = 0; sc->cert_comp_prefs[i] != TLSEXT_comp_cert_none; i++) { - if (!WPACKET_put_bytes_u16(pkt, sc->cert_comp_prefs[i])) - goto err; + first = 0; + + if (!WPACKET_put_bytes_u16(pkt, comp->alg_id)) + return EXT_RETURN_FAIL; } - if (!WPACKET_close(pkt) || !WPACKET_close(pkt)) - goto err; - sc->ext.compress_certificate_sent = 1; - return EXT_RETURN_SENT; - err: - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return EXT_RETURN_FAIL; -#else - return EXT_RETURN_NOT_SENT; -#endif + if (!first && + (!WPACKET_close(pkt) + || !WPACKET_close(pkt))) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return (first == 1) ? EXT_RETURN_NOT_SENT : EXT_RETURN_SENT; } -#ifndef OPENSSL_NO_COMP_ALG -static int tls_comp_in_pref(SSL_CONNECTION *sc, int alg) +int tls_parse_compress_cert(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) { - int i; + unsigned int id; + CERT_COMP *comp = NULL; + PACKET cert_comp_alg_list; + size_t size, i; + int j, best, num_algs; + + if (s->cert_comp_algs == NULL) { + return 1; + } - /* ossl_comp_has_alg() considers 0 as "any" */ - if (alg == 0) + if (!PACKET_as_length_prefixed_1(pkt, &cert_comp_alg_list)) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); return 0; - /* Make sure algorithm is enabled */ - if (!ossl_comp_has_alg(alg)) + } + + size = PACKET_remaining(&cert_comp_alg_list); + + /* Each cert compression algorithm id is 2 bytes and we must have at least 1. */ + if (size == 0 || (size & 1) != 0) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); return 0; - /* If no preferences are set, it's ok */ - if (sc->cert_comp_prefs[0] == TLSEXT_comp_cert_none) - return 1; - /* Find the algorithm */ - for (i = 0; i < TLSEXT_comp_cert_limit; i++) - if (sc->cert_comp_prefs[i] == alg) - return 1; - return 0; + } + + size >>= 1; + + num_algs = sk_CERT_COMP_num(s->cert_comp_algs); + best = num_algs; + + for (i = 0; i < size && PACKET_get_net_2(&cert_comp_alg_list, &id); i++) { + for (j = 0; j < num_algs; j++) { + comp = sk_CERT_COMP_value(s->cert_comp_algs, j); + + if (comp->compress == NULL) + continue; + + if (id == comp->alg_id && j < best) { + best = j; + break; + } + } + } + + if (best < num_algs) { + comp = sk_CERT_COMP_value(s->cert_comp_algs, best); + s->cert_comp_compress_id = comp->alg_id; + } + + return 1; } #endif -int tls_parse_compress_certificate(SSL_CONNECTION *sc, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +int tls_parse_dc_from_extension(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) { -#ifndef OPENSSL_NO_COMP_ALG - PACKET supported_comp_algs; - unsigned int comp; - int already_set[TLSEXT_comp_cert_limit]; - int j = 0; - - /* If no algorithms are available, ignore the extension */ - if (!ossl_comp_has_alg(0)) + if (!s->enable_verify_peer_by_dc) return 1; + /* + * If the client receives a delegated credential without sending this extension, then the + * client MUST abort with an "unexpected_message" alert. + */ + if (!(s->delegated_credential_tag & DC_REQ_HAS_BEEN_SEND_TO_PEER)) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return 0; + } - /* Don't attempt to compress a non-X509 (i.e. an RPK) */ - if (sc->server && sc->ext.server_cert_type != TLSEXT_cert_type_x509) - return 1; - if (!sc->server && sc->ext.client_cert_type != TLSEXT_cert_type_x509) - return 1; + if (s->session == NULL) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return 0; + } - /* Ignore the extension and don't send compressed certificates */ - if ((sc->options & SSL_OP_NO_TX_CERTIFICATE_COMPRESSION) != 0) + if (PACKET_remaining(pkt) <= 0) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return 0; + } + + s->session->peer_dc = DC_new_from_raw_byte_ex(PACKET_data(pkt), + PACKET_remaining(pkt), + s->ctx->libctx, + s->ctx->propq); + if (s->session->peer_dc == NULL) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return 0; + } + + s->delegated_credential_tag |= DC_HAS_BEEN_USED_FOR_VERIFY_PEER; + return 1; +} + +int tls_process_dc_request(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + PACKET supported_sig_algs; + + if (!s->enable_sign_by_dc) return 1; - if (!PACKET_as_length_prefixed_1(pkt, &supported_comp_algs) - || PACKET_remaining(&supported_comp_algs) == 0) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + if (!PACKET_as_length_prefixed_2(pkt, &supported_sig_algs) + || PACKET_remaining(&supported_sig_algs) == 0) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + OPENSSL_free(s->s3.tmp.peer_dc_sigalgs); + s->s3.tmp.peer_dc_sigalgs = NULL; + s->s3.tmp.peer_dc_sigalgslen = 0; + + if (!tls1_save_u16(&supported_sig_algs, &s->s3.tmp.peer_dc_sigalgs, + &s->s3.tmp.peer_dc_sigalgslen)) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); return 0; } - memset(already_set, 0, sizeof(already_set)); - /* - * The preference array has real values, so take a look at each - * value coming in, and make sure it's in our preference list - * The array is 0 (i.e. "none") terminated - * The preference list only contains supported algorithms - */ - while (PACKET_get_net_2(&supported_comp_algs, &comp)) { - if (tls_comp_in_pref(sc, comp) && !already_set[comp]) { - sc->ext.compress_certificate_from_peer[j++] = comp; - already_set[comp] = 1; - } + if (!tls1_set_shared_dc_sigalgs(s)) { + SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_INTERNAL_ERROR); + return 0; } -#endif + return 1; } -static int init_server_cert_type(SSL_CONNECTION *sc, unsigned int context) +EXT_RETURN tls_construct_delegated_credential_raw(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) { - /* Only reset when parsing client hello */ - if (sc->server) { - sc->ext.server_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - sc->ext.server_cert_type = TLSEXT_cert_type_x509; + if (!s->enable_sign_by_dc) + return EXT_RETURN_NOT_SENT; + + if (s->delegated_credential_tag & DC_HAS_BEEN_USED_FOR_SIGN) { + DELEGATED_CREDENTIAL *dc; + unsigned char *dc_raw = NULL; + unsigned long dc_raw_len = 0; + + if (s->s3.tmp.dc == NULL || s->s3.tmp.dc->dc == NULL) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + dc = s->s3.tmp.dc->dc; + dc_raw = DC_get0_raw_byte(dc); + if (dc_raw == NULL) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + dc_raw_len = DC_get_raw_byte_len(dc); + if (dc_raw_len <= 0) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_delegated_credential) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_memcpy(pkt, dc_raw, dc_raw_len) + || !WPACKET_close(pkt)) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + return EXT_RETURN_SENT; } - return 1; + return EXT_RETURN_NOT_SENT; } -static int init_client_cert_type(SSL_CONNECTION *sc, unsigned int context) +EXT_RETURN tls_construct_delegated_credential_request(SSL *s, WPACKET *pkt, + unsigned int context, + X509 *x, size_t chainidx) { - /* Only reset when parsing client hello */ - if (sc->server) { - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - sc->ext.client_cert_type = TLSEXT_cert_type_x509; + size_t salglen; + const uint16_t *salg; + + if (!s->enable_verify_peer_by_dc) + return EXT_RETURN_NOT_SENT; + + salglen = tls12_get_psigalgs(s, 1, &salg); + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_delegated_credential) + /* Sub-packet for sig-algs extension */ + || !WPACKET_start_sub_packet_u16(pkt) + /* Sub-packet for the actual list */ + || !WPACKET_start_sub_packet_u16(pkt) + || !tls12_copy_sigalgs(s, pkt, salg, salglen) + || !WPACKET_close(pkt) + || !WPACKET_close(pkt)) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; } - return 1; + + s->delegated_credential_tag |= DC_REQ_HAS_BEEN_SEND_TO_PEER; + return EXT_RETURN_SENT; } +#endif + diff --git a/openssl/src/ssl/statem/extensions_clnt.c b/openssl/src/ssl/statem/extensions_clnt.c index 381a6c9d7..2bc61eb6f 100644 --- a/openssl/src/ssl/statem/extensions_clnt.c +++ b/openssl/src/ssl/statem/extensions_clnt.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,7 @@ #include "internal/cryptlib.h" #include "statem_local.h" -EXT_RETURN tls_construct_ctos_renegotiate(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_renegotiate(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -32,7 +32,7 @@ EXT_RETURN tls_construct_ctos_renegotiate(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_ctos_server_name(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_server_name(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -58,7 +58,7 @@ EXT_RETURN tls_construct_ctos_server_name(SSL_CONNECTION *s, WPACKET *pkt, } /* Push a Max Fragment Len extension into ClientHello */ -EXT_RETURN tls_construct_ctos_maxfragmentlen(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_maxfragmentlen(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -83,8 +83,7 @@ EXT_RETURN tls_construct_ctos_maxfragmentlen(SSL_CONNECTION *s, WPACKET *pkt, } #ifndef OPENSSL_NO_SRP -EXT_RETURN tls_construct_ctos_srp(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_srp(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { /* Add SRP username if there is one */ @@ -109,20 +108,19 @@ EXT_RETURN tls_construct_ctos_srp(SSL_CONNECTION *s, WPACKET *pkt, } #endif -static int use_ecc(SSL_CONNECTION *s, int min_version, int max_version) +static int use_ecc(SSL *s, int min_version, int max_version) { int i, end, ret = 0; unsigned long alg_k, alg_a; STACK_OF(SSL_CIPHER) *cipher_stack = NULL; const uint16_t *pgroups = NULL; size_t num_groups, j; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); /* See if we support any ECC ciphersuites */ if (s->version == SSL3_VERSION) return 0; - cipher_stack = SSL_get1_supported_ciphers(ssl); + cipher_stack = SSL_get1_supported_ciphers(s); end = sk_SSL_CIPHER_num(cipher_stack); for (i = 0; i < end; i++) { const SSL_CIPHER *c = sk_SSL_CIPHER_value(cipher_stack, i); @@ -153,7 +151,7 @@ static int use_ecc(SSL_CONNECTION *s, int min_version, int max_version) return 0; } -EXT_RETURN tls_construct_ctos_ec_pt_formats(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_ec_pt_formats(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -184,7 +182,7 @@ EXT_RETURN tls_construct_ctos_ec_pt_formats(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_ctos_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_supported_groups(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -203,7 +201,7 @@ EXT_RETURN tls_construct_ctos_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, * if we don't have EC support then we don't send this extension. */ if (!use_ecc(s, min_version, max_version) - && (SSL_CONNECTION_IS_DTLS(s) || max_version < TLS1_3_VERSION)) + && (SSL_IS_DTLS(s) || max_version < TLS1_3_VERSION)) return EXT_RETURN_NOT_SENT; /* @@ -219,6 +217,56 @@ EXT_RETURN tls_construct_ctos_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return EXT_RETURN_FAIL; } + +#ifndef OPENSSL_NO_SM2 + /* + * RFC 8998 requires that: + * For the key_share extension, a KeyShareEntry for the "curveSM2" group + * MUST be included. We re-order curveSM2 to the first supported group when + * enable_sm_tls13_strict so that the key_share extension will include a + * KeyShareEntry for the "curveSM2" group because only one KeyShareEntry is + * sent now. + */ + if (!SSL_IS_DTLS(s) && max_version >= TLS1_3_VERSION + && s->enable_sm_tls13_strict == 1) { + int sm2_idx = -1; + + for (i = 0; i < num_groups; i++) { + if (pgroups[i] == TLSEXT_curve_SM2) { + sm2_idx = i; + break; + } + } + + if (sm2_idx > 0) { + int *groups = OPENSSL_malloc(sizeof(int) * num_groups); + if (groups == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + for (i = 0; i < num_groups; i++) + groups[i] = tls1_group_id2nid(pgroups[i], 1); + + for (i = sm2_idx; i > 0; i--) + groups[i] = groups[i - 1]; + + groups[0] = NID_sm2; + + if (!tls1_set_groups(&s->ext.supportedgroups, + &s->ext.supportedgroups_len, + groups, num_groups)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + OPENSSL_free(groups); + return EXT_RETURN_FAIL; + } + + OPENSSL_free(groups); + tls1_get_supported_groups(s, &pgroups, &num_groups); + } + } +#endif + /* Copy group ID if supported */ for (i = 0; i < num_groups; i++) { uint16_t ctmp = pgroups[i]; @@ -253,7 +301,7 @@ EXT_RETURN tls_construct_ctos_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_ctos_session_ticket(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_session_ticket(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -294,7 +342,7 @@ EXT_RETURN tls_construct_ctos_session_ticket(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_ctos_sig_algs(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_sig_algs(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -321,7 +369,7 @@ EXT_RETURN tls_construct_ctos_sig_algs(SSL_CONNECTION *s, WPACKET *pkt, } #ifndef OPENSSL_NO_OCSP -EXT_RETURN tls_construct_ctos_status_request(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_status_request(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -386,12 +434,10 @@ EXT_RETURN tls_construct_ctos_status_request(SSL_CONNECTION *s, WPACKET *pkt, #endif #ifndef OPENSSL_NO_NEXTPROTONEG -EXT_RETURN tls_construct_ctos_npn(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_npn(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { - if (SSL_CONNECTION_GET_CTX(s)->ext.npn_select_cb == NULL - || !SSL_IS_FIRST_HANDSHAKE(s)) + if (s->ctx->ext.npn_select_cb == NULL || !SSL_IS_FIRST_HANDSHAKE(s)) return EXT_RETURN_NOT_SENT; /* @@ -408,8 +454,7 @@ EXT_RETURN tls_construct_ctos_npn(SSL_CONNECTION *s, WPACKET *pkt, } #endif -EXT_RETURN tls_construct_ctos_alpn(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_alpn(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { s->s3.alpn_sent = 0; @@ -433,12 +478,11 @@ EXT_RETURN tls_construct_ctos_alpn(SSL_CONNECTION *s, WPACKET *pkt, #ifndef OPENSSL_NO_SRTP -EXT_RETURN tls_construct_ctos_use_srtp(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_use_srtp(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - STACK_OF(SRTP_PROTECTION_PROFILE) *clnt = SSL_get_srtp_profiles(ssl); + STACK_OF(SRTP_PROTECTION_PROFILE) *clnt = SSL_get_srtp_profiles(s); int i, end; if (clnt == NULL) @@ -475,8 +519,7 @@ EXT_RETURN tls_construct_ctos_use_srtp(SSL_CONNECTION *s, WPACKET *pkt, } #endif -EXT_RETURN tls_construct_ctos_etm(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_etm(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (s->options & SSL_OP_NO_ENCRYPT_THEN_MAC) @@ -492,8 +535,7 @@ EXT_RETURN tls_construct_ctos_etm(SSL_CONNECTION *s, WPACKET *pkt, } #ifndef OPENSSL_NO_CT -EXT_RETURN tls_construct_ctos_sct(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_sct(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (s->ct_validation_callback == NULL) @@ -513,8 +555,21 @@ EXT_RETURN tls_construct_ctos_sct(SSL_CONNECTION *s, WPACKET *pkt, } #endif -EXT_RETURN tls_construct_ctos_ems(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +EXT_RETURN tls_construct_ctos_delegated_credential(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->statem.hand_state == TLS_ST_CW_CLNT_HELLO) + return tls_construct_delegated_credential_request(s, pkt, context, x, chainidx); + + if (s->statem.hand_state == TLS_ST_CW_CERT) + return tls_construct_delegated_credential_raw(s, pkt, context, x, chainidx); + + return EXT_RETURN_NOT_SENT; +} +#endif + +EXT_RETURN tls_construct_ctos_ems(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (s->options & SSL_OP_NO_EXTENDED_MASTER_SECRET) @@ -529,7 +584,7 @@ EXT_RETURN tls_construct_ctos_ems(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_ctos_supported_versions(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_supported_versions(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -572,7 +627,7 @@ EXT_RETURN tls_construct_ctos_supported_versions(SSL_CONNECTION *s, WPACKET *pkt /* * Construct a psk_kex_modes extension. */ -EXT_RETURN tls_construct_ctos_psk_kex_modes(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_psk_kex_modes(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -599,7 +654,7 @@ EXT_RETURN tls_construct_ctos_psk_kex_modes(SSL_CONNECTION *s, WPACKET *pkt, } #ifndef OPENSSL_NO_TLS1_3 -static int add_key_share(SSL_CONNECTION *s, WPACKET *pkt, unsigned int curve_id) +static int add_key_share(SSL *s, WPACKET *pkt, unsigned int curve_id) { unsigned char *encoded_point = NULL; EVP_PKEY *key_share_key = NULL; @@ -655,7 +710,7 @@ static int add_key_share(SSL_CONNECTION *s, WPACKET *pkt, unsigned int curve_id) } #endif -EXT_RETURN tls_construct_ctos_key_share(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_key_share(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -684,11 +739,8 @@ EXT_RETURN tls_construct_ctos_key_share(SSL_CONNECTION *s, WPACKET *pkt, curve_id = s->s3.group_id; } else { for (i = 0; i < num_groups; i++) { - if (!tls_group_allowed(s, pgroups[i], SSL_SECOP_CURVE_SUPPORTED)) - continue; - if (!tls_valid_group(s, pgroups[i], TLS1_3_VERSION, TLS1_3_VERSION, - 0, NULL)) + if (!tls_group_allowed(s, pgroups[i], SSL_SECOP_CURVE_SUPPORTED)) continue; curve_id = pgroups[i]; @@ -716,8 +768,7 @@ EXT_RETURN tls_construct_ctos_key_share(SSL_CONNECTION *s, WPACKET *pkt, #endif } -EXT_RETURN tls_construct_ctos_cookie(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_cookie(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { EXT_RETURN ret = EXT_RETURN_FAIL; @@ -745,7 +796,7 @@ EXT_RETURN tls_construct_ctos_cookie(SSL_CONNECTION *s, WPACKET *pkt, return ret; } -EXT_RETURN tls_construct_ctos_early_data(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_early_data(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -757,13 +808,12 @@ EXT_RETURN tls_construct_ctos_early_data(SSL_CONNECTION *s, WPACKET *pkt, SSL_SESSION *psksess = NULL; SSL_SESSION *edsess = NULL; const EVP_MD *handmd = NULL; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if (s->hello_retry_request == SSL_HRR_PENDING) handmd = ssl_handshake_md(s); if (s->psk_use_session_cb != NULL - && (!s->psk_use_session_cb(ssl, handmd, &id, &idlen, &psksess) + && (!s->psk_use_session_cb(s, handmd, &id, &idlen, &psksess) || (psksess != NULL && psksess->ssl_version != TLS1_3_VERSION))) { SSL_SESSION_free(psksess); @@ -777,8 +827,7 @@ EXT_RETURN tls_construct_ctos_early_data(SSL_CONNECTION *s, WPACKET *pkt, size_t psklen = 0; memset(identity, 0, sizeof(identity)); - psklen = s->psk_client_callback(ssl, NULL, - identity, sizeof(identity) - 1, + psklen = s->psk_client_callback(s, NULL, identity, sizeof(identity) - 1, psk, sizeof(psk)); if (psklen > PSK_MAX_PSK_LEN) { @@ -799,7 +848,7 @@ EXT_RETURN tls_construct_ctos_early_data(SSL_CONNECTION *s, WPACKET *pkt, * We found a PSK using an old style callback. We don't know * the digest so we default to SHA256 as per the TLSv1.3 spec */ - cipher = SSL_CIPHER_find(ssl, tls13_aes128gcmsha256_id); + cipher = SSL_CIPHER_find(s, tls13_aes128gcmsha256_id); if (cipher == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return EXT_RETURN_FAIL; @@ -916,7 +965,7 @@ EXT_RETURN tls_construct_ctos_early_data(SSL_CONNECTION *s, WPACKET *pkt, */ #define PSK_PRE_BINDER_OVERHEAD (2 + 2 + 2 + 2 + 4 + 2 + 1) -EXT_RETURN tls_construct_ctos_padding(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_padding(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -944,8 +993,7 @@ EXT_RETURN tls_construct_ctos_padding(SSL_CONNECTION *s, WPACKET *pkt, if (s->session->ssl_version == TLS1_3_VERSION && s->session->ext.ticklen != 0 && s->session->cipher != NULL) { - const EVP_MD *md = ssl_md(SSL_CONNECTION_GET_CTX(s), - s->session->cipher->algorithm2); + const EVP_MD *md = ssl_md(s->ctx, s->session->cipher->algorithm2); if (md != NULL) { /* @@ -986,18 +1034,15 @@ EXT_RETURN tls_construct_ctos_padding(SSL_CONNECTION *s, WPACKET *pkt, /* * Construct the pre_shared_key extension */ -EXT_RETURN tls_construct_ctos_psk(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_psk(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { #ifndef OPENSSL_NO_TLS1_3 - uint32_t agesec, agems = 0; + uint32_t now, agesec, agems = 0; size_t reshashsize = 0, pskhashsize = 0, binderoffset, msglen; unsigned char *resbinder = NULL, *pskbinder = NULL, *msgstart = NULL; const EVP_MD *handmd = NULL, *mdres = NULL, *mdpsk = NULL; int dores = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - OSSL_TIME t; s->ext.tick_identity = 0; @@ -1024,7 +1069,7 @@ EXT_RETURN tls_construct_ctos_psk(SSL_CONNECTION *s, WPACKET *pkt, SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return EXT_RETURN_FAIL; } - mdres = ssl_md(sctx, s->session->cipher->algorithm2); + mdres = ssl_md(s->ctx, s->session->cipher->algorithm2); if (mdres == NULL) { /* * Don't recognize this cipher so we can't use the session. @@ -1049,8 +1094,8 @@ EXT_RETURN tls_construct_ctos_psk(SSL_CONNECTION *s, WPACKET *pkt, * this in multiple places in the code, so portability shouldn't be an * issue. */ - t = ossl_time_subtract(ossl_time_now(), s->session->time); - agesec = (uint32_t)ossl_time2seconds(t); + now = (uint32_t)time(NULL); + agesec = now - (uint32_t)s->session->time; /* * We calculate the age in seconds but the server may work in ms. Due to * rounding errors we could overestimate the age by up to 1s. It is @@ -1097,7 +1142,7 @@ EXT_RETURN tls_construct_ctos_psk(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_NOT_SENT; if (s->psksession != NULL) { - mdpsk = ssl_md(sctx, s->psksession->cipher->algorithm2); + mdpsk = ssl_md(s->ctx, s->psksession->cipher->algorithm2); if (mdpsk == NULL) { /* * Don't recognize this cipher so we can't use the session. @@ -1187,7 +1232,7 @@ EXT_RETURN tls_construct_ctos_psk(SSL_CONNECTION *s, WPACKET *pkt, #endif } -EXT_RETURN tls_construct_ctos_post_handshake_auth(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_post_handshake_auth(SSL *s, WPACKET *pkt, ossl_unused unsigned int context, ossl_unused X509 *x, ossl_unused size_t chainidx) @@ -1212,12 +1257,52 @@ EXT_RETURN tls_construct_ctos_post_handshake_auth(SSL_CONNECTION *s, WPACKET *pk #endif } +#ifndef OPENSSL_NO_QUIC +EXT_RETURN tls_construct_ctos_quic_transport_params_draft(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (s->quic_transport_version == TLSEXT_TYPE_quic_transport_parameters + || s->ext.quic_transport_params == NULL + || s->ext.quic_transport_params_len == 0) { + return EXT_RETURN_NOT_SENT; + } + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_quic_transport_parameters_draft) + || !WPACKET_sub_memcpy_u16(pkt, s->ext.quic_transport_params, + s->ext.quic_transport_params_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +EXT_RETURN tls_construct_ctos_quic_transport_params(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (s->quic_transport_version == TLSEXT_TYPE_quic_transport_parameters_draft + || s->ext.quic_transport_params == NULL + || s->ext.quic_transport_params_len == 0) { + return EXT_RETURN_NOT_SENT; + } + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_quic_transport_parameters) + || !WPACKET_sub_memcpy_u16(pkt, s->ext.quic_transport_params, + s->ext.quic_transport_params_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} +#endif /* * Parse the server's renegotiation binding and abort if it's not right */ -int tls_parse_stoc_renegotiate(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_renegotiate(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { size_t expected_len = s->s3.previous_client_finished_len @@ -1271,8 +1356,7 @@ int tls_parse_stoc_renegotiate(SSL_CONNECTION *s, PACKET *pkt, } /* Parse the server's max fragment len extension packet */ -int tls_parse_stoc_maxfragmentlen(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_maxfragmentlen(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { unsigned int value; @@ -1310,8 +1394,7 @@ int tls_parse_stoc_maxfragmentlen(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_stoc_server_name(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_server_name(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (s->ext.hostname == NULL) { @@ -1339,8 +1422,7 @@ int tls_parse_stoc_server_name(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_stoc_ec_pt_formats(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_ec_pt_formats(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { size_t ecpointformats_len; @@ -1379,16 +1461,13 @@ int tls_parse_stoc_ec_pt_formats(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_stoc_session_ticket(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_session_ticket(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - if (s->ext.session_ticket_cb != NULL && - !s->ext.session_ticket_cb(ssl, PACKET_data(pkt), - PACKET_remaining(pkt), - s->ext.session_ticket_cb_arg)) { + !s->ext.session_ticket_cb(s, PACKET_data(pkt), + PACKET_remaining(pkt), + s->ext.session_ticket_cb_arg)) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_BAD_EXTENSION); return 0; } @@ -1408,8 +1487,7 @@ int tls_parse_stoc_session_ticket(SSL_CONNECTION *s, PACKET *pkt, } #ifndef OPENSSL_NO_OCSP -int tls_parse_stoc_status_request(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_status_request(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (context == SSL_EXT_TLS1_3_CERTIFICATE_REQUEST) { @@ -1425,12 +1503,12 @@ int tls_parse_stoc_status_request(SSL_CONNECTION *s, PACKET *pkt, SSLfatal(s, SSL_AD_UNSUPPORTED_EXTENSION, SSL_R_BAD_EXTENSION); return 0; } - if (!SSL_CONNECTION_IS_TLS13(s) && PACKET_remaining(pkt) > 0) { + if (!SSL_IS_TLS13(s) && PACKET_remaining(pkt) > 0) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); return 0; } - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* We only know how to handle this if it's for the first Certificate in * the chain. We ignore any other responses. */ @@ -1450,8 +1528,8 @@ int tls_parse_stoc_status_request(SSL_CONNECTION *s, PACKET *pkt, #ifndef OPENSSL_NO_CT -int tls_parse_stoc_sct(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_stoc_sct(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { if (context == SSL_EXT_TLS1_3_CERTIFICATE_REQUEST) { /* We ignore this if the server sends it in a CertificateRequest */ @@ -1475,7 +1553,7 @@ int tls_parse_stoc_sct(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, s->ext.scts = OPENSSL_malloc(size); if (s->ext.scts == NULL) { s->ext.scts_len = 0; - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } if (!PACKET_copy_bytes(pkt, s->ext.scts, size)) { @@ -1511,6 +1589,19 @@ int tls_parse_stoc_sct(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } #endif +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +int tls_parse_stoc_delegated_credential(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->statem.hand_state == TLS_ST_CR_CERT_REQ) + return tls_process_dc_request(s, pkt, context, x, chainidx); + + if (s->statem.hand_state == TLS_ST_CR_CERT) + return tls_parse_dc_from_extension(s, pkt, context, x, chainidx); + + return 0; +} +#endif #ifndef OPENSSL_NO_NEXTPROTONEG /* @@ -1518,7 +1609,7 @@ int tls_parse_stoc_sct(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, * elements of zero length are allowed and the set of elements must exactly * fill the length of the block. Returns 1 on success or 0 on failure. */ -static int ssl_next_proto_validate(SSL_CONNECTION *s, PACKET *pkt) +static int ssl_next_proto_validate(SSL *s, PACKET *pkt) { PACKET tmp_protocol; @@ -1533,20 +1624,19 @@ static int ssl_next_proto_validate(SSL_CONNECTION *s, PACKET *pkt) return 1; } -int tls_parse_stoc_npn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_stoc_npn(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { unsigned char *selected; unsigned char selected_len; PACKET tmppkt; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* Check if we are in a renegotiation. If so ignore this extension */ if (!SSL_IS_FIRST_HANDSHAKE(s)) return 1; /* We must have requested it. */ - if (sctx->ext.npn_select_cb == NULL) { + if (s->ctx->ext.npn_select_cb == NULL) { SSLfatal(s, SSL_AD_UNSUPPORTED_EXTENSION, SSL_R_BAD_EXTENSION); return 0; } @@ -1557,10 +1647,10 @@ int tls_parse_stoc_npn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, /* SSLfatal() already called */ return 0; } - if (sctx->ext.npn_select_cb(SSL_CONNECTION_GET_SSL(s), - &selected, &selected_len, - PACKET_data(pkt), PACKET_remaining(pkt), - sctx->ext.npn_select_cb_arg) != + if (s->ctx->ext.npn_select_cb(s, &selected, &selected_len, + PACKET_data(pkt), + PACKET_remaining(pkt), + s->ctx->ext.npn_select_cb_arg) != SSL_TLSEXT_ERR_OK) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_BAD_EXTENSION); return 0; @@ -1586,8 +1676,8 @@ int tls_parse_stoc_npn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } #endif -int tls_parse_stoc_alpn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_stoc_alpn(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { size_t len; @@ -1651,8 +1741,8 @@ int tls_parse_stoc_alpn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } #ifndef OPENSSL_NO_SRTP -int tls_parse_stoc_use_srtp(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx) +int tls_parse_stoc_use_srtp(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { unsigned int id, ct, mki; int i; @@ -1675,7 +1765,7 @@ int tls_parse_stoc_use_srtp(SSL_CONNECTION *s, PACKET *pkt, } /* Throw an error if the server gave us an unsolicited extension */ - clnt = SSL_get_srtp_profiles(SSL_CONNECTION_GET_SSL(s)); + clnt = SSL_get_srtp_profiles(s); if (clnt == NULL) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_NO_SRTP_PROFILES); return 0; @@ -1700,24 +1790,20 @@ int tls_parse_stoc_use_srtp(SSL_CONNECTION *s, PACKET *pkt, } #endif -int tls_parse_stoc_etm(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_stoc_etm(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { /* Ignore if inappropriate ciphersuite */ if (!(s->options & SSL_OP_NO_ENCRYPT_THEN_MAC) && s->s3.tmp.new_cipher->algorithm_mac != SSL_AEAD - && s->s3.tmp.new_cipher->algorithm_enc != SSL_RC4 - && s->s3.tmp.new_cipher->algorithm_enc != SSL_eGOST2814789CNT - && s->s3.tmp.new_cipher->algorithm_enc != SSL_eGOST2814789CNT12 - && s->s3.tmp.new_cipher->algorithm_enc != SSL_MAGMA - && s->s3.tmp.new_cipher->algorithm_enc != SSL_KUZNYECHIK) + && s->s3.tmp.new_cipher->algorithm_enc != SSL_RC4) s->ext.use_etm = 1; return 1; } -int tls_parse_stoc_ems(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_stoc_ems(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { if (s->options & SSL_OP_NO_EXTENDED_MASTER_SECRET) return 1; @@ -1728,8 +1814,7 @@ int tls_parse_stoc_ems(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, return 1; } -int tls_parse_stoc_supported_versions(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_supported_versions(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { unsigned int version; @@ -1756,16 +1841,11 @@ int tls_parse_stoc_supported_versions(SSL_CONNECTION *s, PACKET *pkt, /* We just set it here. We validate it in ssl_choose_client_version */ s->version = version; - if (!ssl_set_record_protocol_version(s, version)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } return 1; } -int tls_parse_stoc_key_share(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, +int tls_parse_stoc_key_share(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { #ifndef OPENSSL_NO_TLS1_3 @@ -1810,9 +1890,7 @@ int tls_parse_stoc_key_share(SSL_CONNECTION *s, PACKET *pkt, break; } if (i >= num_groups - || !tls_group_allowed(s, group_id, SSL_SECOP_CURVE_SUPPORTED) - || !tls_valid_group(s, group_id, TLS1_3_VERSION, TLS1_3_VERSION, - 0, NULL)) { + || !tls_group_allowed(s, group_id, SSL_SECOP_CURVE_SUPPORTED)) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); return 0; } @@ -1846,7 +1924,7 @@ int tls_parse_stoc_key_share(SSL_CONNECTION *s, PACKET *pkt, SSL_SESSION *new_sess; if ((new_sess = ssl_session_dup(s->session, 0)) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } SSL_SESSION_free(s->session); @@ -1854,8 +1932,7 @@ int tls_parse_stoc_key_share(SSL_CONNECTION *s, PACKET *pkt, s->session->kex_group = group_id; } - if ((ginf = tls1_group_id_lookup(SSL_CONNECTION_GET_CTX(s), - group_id)) == NULL) { + if ((ginf = tls1_group_id_lookup(s->ctx, group_id)) == NULL) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); return 0; } @@ -1904,8 +1981,8 @@ int tls_parse_stoc_key_share(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_stoc_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_stoc_cookie(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { PACKET cookie; @@ -1919,8 +1996,7 @@ int tls_parse_stoc_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, return 1; } -int tls_parse_stoc_early_data(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_early_data(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (context == SSL_EXT_TLS1_3_NEW_SESSION_TICKET) { @@ -1932,23 +2008,18 @@ int tls_parse_stoc_early_data(SSL_CONNECTION *s, PACKET *pkt, return 0; } - s->session->ext.max_early_data = max_early_data; - - if (SSL_IS_QUIC_HANDSHAKE(s) && max_early_data != 0xffffffff) { - /* - * QUIC allows missing max_early_data, or a max_early_data value - * of 0xffffffff. Missing max_early_data is stored in the session - * as 0. This is indistinguishable in OpenSSL from a present - * max_early_data value that was 0. In order that later checks for - * invalid max_early_data correctly treat as an error the case where - * max_early_data is present and it is 0, we store any invalid - * value in the same (non-zero) way. Otherwise we would have to - * introduce a new flag just for this. - */ - s->session->ext.max_early_data = 1; +#ifndef OPENSSL_NO_QUIC + /* + * QUIC server must send 0xFFFFFFFF or it's a PROTOCOL_VIOLATION + * per draft-ietf-quic-tls-27 S4.5 + */ + if (s->quic_method != NULL && max_early_data != 0xFFFFFFFF) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_INVALID_MAX_EARLY_DATA); return 0; } +#endif + + s->session->ext.max_early_data = max_early_data; return 1; } @@ -1974,8 +2045,7 @@ int tls_parse_stoc_early_data(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_stoc_psk(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, +int tls_parse_stoc_psk(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { #ifndef OPENSSL_NO_TLS1_3 @@ -2032,106 +2102,37 @@ int tls_parse_stoc_psk(SSL_CONNECTION *s, PACKET *pkt, return 1; } -EXT_RETURN tls_construct_ctos_client_cert_type(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) -{ - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - if (sc->client_cert_type == NULL) - return EXT_RETURN_NOT_SENT; - - if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_client_cert_type) - || !WPACKET_start_sub_packet_u16(pkt) - || !WPACKET_sub_memcpy_u8(pkt, sc->client_cert_type, sc->client_cert_type_len) - || !WPACKET_close(pkt)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return EXT_RETURN_FAIL; - } - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_GOOD; - return EXT_RETURN_SENT; -} - -int tls_parse_stoc_client_cert_type(SSL_CONNECTION *sc, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) +#ifndef OPENSSL_NO_QUIC +int tls_parse_stoc_quic_transport_params_draft(SSL *s, PACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) { - unsigned int type; + OPENSSL_free(s->ext.peer_quic_transport_params_draft); + s->ext.peer_quic_transport_params_draft = NULL; + s->ext.peer_quic_transport_params_draft_len = 0; - if (PACKET_remaining(pkt) != 1) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - if (!PACKET_get_1(pkt, &type)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - /* We did not send/ask for this */ - if (!ossl_assert(sc->ext.client_cert_type_ctos == OSSL_CERT_TYPE_CTOS_GOOD)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - /* We don't have this enabled */ - if (sc->client_cert_type == NULL) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - /* Given back a value we didn't configure */ - if (memchr(sc->client_cert_type, type, sc->client_cert_type_len) == NULL) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_VALUE); + if (!PACKET_memdup(pkt, + &s->ext.peer_quic_transport_params_draft, + &s->ext.peer_quic_transport_params_draft_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - sc->ext.client_cert_type = type; return 1; } -EXT_RETURN tls_construct_ctos_server_cert_type(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) -{ - sc->ext.server_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - if (sc->server_cert_type == NULL) - return EXT_RETURN_NOT_SENT; - - if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_server_cert_type) - || !WPACKET_start_sub_packet_u16(pkt) - || !WPACKET_sub_memcpy_u8(pkt, sc->server_cert_type, sc->server_cert_type_len) - || !WPACKET_close(pkt)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return EXT_RETURN_FAIL; - } - sc->ext.server_cert_type_ctos = OSSL_CERT_TYPE_CTOS_GOOD; - return EXT_RETURN_SENT; -} - -int tls_parse_stoc_server_cert_type(SSL_CONNECTION *sc, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_stoc_quic_transport_params(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) { - unsigned int type; + OPENSSL_free(s->ext.peer_quic_transport_params); + s->ext.peer_quic_transport_params = NULL; + s->ext.peer_quic_transport_params_len = 0; - if (PACKET_remaining(pkt) != 1) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - if (!PACKET_get_1(pkt, &type)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - /* We did not send/ask for this */ - if (!ossl_assert(sc->ext.server_cert_type_ctos == OSSL_CERT_TYPE_CTOS_GOOD)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - /* We don't have this enabled */ - if (sc->server_cert_type == NULL) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - /* Given back a value we didn't configure */ - if (memchr(sc->server_cert_type, type, sc->server_cert_type_len) == NULL) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_VALUE); + if (!PACKET_memdup(pkt, + &s->ext.peer_quic_transport_params, + &s->ext.peer_quic_transport_params_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - sc->ext.server_cert_type = type; return 1; } +#endif diff --git a/openssl/src/ssl/statem/extensions_cust.c b/openssl/src/ssl/statem/extensions_cust.c index fd840e891..401a4c5c7 100644 --- a/openssl/src/ssl/statem/extensions_cust.c +++ b/openssl/src/ssl/statem/extensions_cust.c @@ -1,5 +1,5 @@ /* - * Copyright 2014-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -110,12 +110,11 @@ void custom_ext_init(custom_ext_methods *exts) } /* Pass received custom extension data to the application for parsing. */ -int custom_ext_parse(SSL_CONNECTION *s, unsigned int context, - unsigned int ext_type, +int custom_ext_parse(SSL *s, unsigned int context, unsigned int ext_type, const unsigned char *ext_data, size_t ext_size, X509 *x, size_t chainidx) { - int al = 0; + int al; custom_ext_methods *exts = &s->cert->custext; custom_ext_method *meth; ENDPOINT role = ENDPOINT_BOTH; @@ -155,11 +154,11 @@ int custom_ext_parse(SSL_CONNECTION *s, unsigned int context, meth->ext_flags |= SSL_EXT_FLAG_RECEIVED; /* If no parse function set return success */ - if (meth->parse_cb == NULL) + if (!meth->parse_cb) return 1; - if (meth->parse_cb(SSL_CONNECTION_GET_SSL(s), ext_type, context, ext_data, - ext_size, x, chainidx, &al, meth->parse_arg) <= 0) { + if (meth->parse_cb(s, ext_type, context, ext_data, ext_size, x, chainidx, + &al, meth->parse_arg) <= 0) { SSLfatal(s, al, SSL_R_BAD_EXTENSION); return 0; } @@ -171,14 +170,13 @@ int custom_ext_parse(SSL_CONNECTION *s, unsigned int context, * Request custom extension data from the application and add to the return * buffer. */ -int custom_ext_add(SSL_CONNECTION *s, int context, WPACKET *pkt, X509 *x, - size_t chainidx, int maxversion) +int custom_ext_add(SSL *s, int context, WPACKET *pkt, X509 *x, size_t chainidx, + int maxversion) { custom_ext_methods *exts = &s->cert->custext; custom_ext_method *meth; size_t i; int al; - int for_comp = (context & SSL_EXT_TLS1_3_CERTIFICATE_COMPRESSION) != 0; for (i = 0; i < exts->meths_count; i++) { const unsigned char *out = NULL; @@ -193,7 +191,6 @@ int custom_ext_add(SSL_CONNECTION *s, int context, WPACKET *pkt, X509 *x, | SSL_EXT_TLS1_3_SERVER_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS | SSL_EXT_TLS1_3_CERTIFICATE - | SSL_EXT_TLS1_3_RAW_PUBLIC_KEY | SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST)) != 0) { /* Only send extensions present in ClientHello/CertificateRequest */ if (!(meth->ext_flags & SSL_EXT_FLAG_RECEIVED)) @@ -207,14 +204,12 @@ int custom_ext_add(SSL_CONNECTION *s, int context, WPACKET *pkt, X509 *x, continue; if (meth->add_cb != NULL) { - int cb_retval = meth->add_cb(SSL_CONNECTION_GET_SSL(s), - meth->ext_type, context, &out, + int cb_retval = meth->add_cb(s, meth->ext_type, context, &out, &outlen, x, chainidx, &al, meth->add_arg); if (cb_retval < 0) { - if (!for_comp) - SSLfatal(s, al, SSL_R_CALLBACK_FAILED); + SSLfatal(s, al, SSL_R_CALLBACK_FAILED); return 0; /* error */ } if (cb_retval == 0) @@ -225,11 +220,7 @@ int custom_ext_add(SSL_CONNECTION *s, int context, WPACKET *pkt, X509 *x, || !WPACKET_start_sub_packet_u16(pkt) || (outlen > 0 && !WPACKET_memcpy(pkt, out, outlen)) || !WPACKET_close(pkt)) { - if (meth->free_cb != NULL) - meth->free_cb(SSL_CONNECTION_GET_SSL(s), meth->ext_type, context, - out, meth->add_arg); - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } if ((context & SSL_EXT_CLIENT_HELLO) != 0) { @@ -237,11 +228,7 @@ int custom_ext_add(SSL_CONNECTION *s, int context, WPACKET *pkt, X509 *x, * We can't send duplicates: code logic should prevent this. */ if (!ossl_assert((meth->ext_flags & SSL_EXT_FLAG_SENT) == 0)) { - if (meth->free_cb != NULL) - meth->free_cb(SSL_CONNECTION_GET_SSL(s), meth->ext_type, - context, out, meth->add_arg); - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } /* @@ -252,8 +239,7 @@ int custom_ext_add(SSL_CONNECTION *s, int context, WPACKET *pkt, X509 *x, meth->ext_flags |= SSL_EXT_FLAG_SENT; } if (meth->free_cb != NULL) - meth->free_cb(SSL_CONNECTION_GET_SSL(s), meth->ext_type, context, - out, meth->add_arg); + meth->free_cb(s, meth->ext_type, context, out, meth->add_arg); } return 1; } @@ -342,8 +328,6 @@ void custom_exts_free(custom_ext_methods *exts) OPENSSL_free(meth->parse_arg); } OPENSSL_free(exts->meths); - exts->meths = NULL; - exts->meths_count = 0; } /* Return true if a client custom extension exists, false otherwise */ @@ -353,15 +337,16 @@ int SSL_CTX_has_client_custom_ext(const SSL_CTX *ctx, unsigned int ext_type) NULL) != NULL; } -int ossl_tls_add_custom_ext_intern(SSL_CTX *ctx, custom_ext_methods *exts, - ENDPOINT role, unsigned int ext_type, - unsigned int context, - SSL_custom_ext_add_cb_ex add_cb, - SSL_custom_ext_free_cb_ex free_cb, - void *add_arg, - SSL_custom_ext_parse_cb_ex parse_cb, - void *parse_arg) +static int add_custom_ext_intern(SSL_CTX *ctx, ENDPOINT role, + unsigned int ext_type, + unsigned int context, + SSL_custom_ext_add_cb_ex add_cb, + SSL_custom_ext_free_cb_ex free_cb, + void *add_arg, + SSL_custom_ext_parse_cb_ex parse_cb, + void *parse_arg) { + custom_ext_methods *exts = &ctx->cert->custext; custom_ext_method *meth, *tmp; /* @@ -371,9 +356,6 @@ int ossl_tls_add_custom_ext_intern(SSL_CTX *ctx, custom_ext_methods *exts, if (add_cb == NULL && free_cb != NULL) return 0; - if (exts == NULL) - exts = &ctx->cert->custext; - #ifndef OPENSSL_NO_CT /* * We don't want applications registering callbacks for SCT extensions @@ -382,7 +364,6 @@ int ossl_tls_add_custom_ext_intern(SSL_CTX *ctx, custom_ext_methods *exts, */ if (ext_type == TLSEXT_TYPE_signed_certificate_timestamp && (context & SSL_EXT_CLIENT_HELLO) != 0 - && ctx != NULL && SSL_CTX_ct_is_enabled(ctx)) return 0; #endif @@ -447,13 +428,13 @@ static int add_old_custom_ext(SSL_CTX *ctx, ENDPOINT role, parse_cb_wrap->parse_arg = parse_arg; parse_cb_wrap->parse_cb = parse_cb; - ret = ossl_tls_add_custom_ext_intern(ctx, NULL, role, ext_type, - context, - custom_ext_add_old_cb_wrap, - custom_ext_free_old_cb_wrap, - add_cb_wrap, - custom_ext_parse_old_cb_wrap, - parse_cb_wrap); + ret = add_custom_ext_intern(ctx, role, ext_type, + context, + custom_ext_add_old_cb_wrap, + custom_ext_free_old_cb_wrap, + add_cb_wrap, + custom_ext_parse_old_cb_wrap, + parse_cb_wrap); if (!ret) { OPENSSL_free(add_cb_wrap); @@ -499,9 +480,8 @@ int SSL_CTX_add_custom_ext(SSL_CTX *ctx, unsigned int ext_type, void *add_arg, SSL_custom_ext_parse_cb_ex parse_cb, void *parse_arg) { - return ossl_tls_add_custom_ext_intern(ctx, NULL, ENDPOINT_BOTH, ext_type, - context, add_cb, free_cb, add_arg, - parse_cb, parse_arg); + return add_custom_ext_intern(ctx, ENDPOINT_BOTH, ext_type, context, add_cb, + free_cb, add_arg, parse_cb, parse_arg); } int SSL_extension_supported(unsigned int ext_type) @@ -542,9 +522,6 @@ int SSL_extension_supported(unsigned int ext_type) case TLSEXT_TYPE_certificate_authorities: case TLSEXT_TYPE_psk: case TLSEXT_TYPE_post_handshake_auth: - case TLSEXT_TYPE_compress_certificate: - case TLSEXT_TYPE_client_cert_type: - case TLSEXT_TYPE_server_cert_type: return 1; default: return 0; diff --git a/openssl/src/ssl/statem/extensions_srvr.c b/openssl/src/ssl/statem/extensions_srvr.c index 21db977c8..a025c1313 100644 --- a/openssl/src/ssl/statem/extensions_srvr.c +++ b/openssl/src/ssl/statem/extensions_srvr.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,16 +12,16 @@ #include "statem_local.h" #include "internal/cryptlib.h" -#define COOKIE_STATE_FORMAT_VERSION 1 +#define COOKIE_STATE_FORMAT_VERSION 0 /* * 2 bytes for packet length, 2 bytes for format version, 2 bytes for * protocol version, 2 bytes for group id, 2 bytes for cipher id, 1 byte for - * key_share present flag, 8 bytes for timestamp, 2 bytes for the hashlen, + * key_share present flag, 4 bytes for timestamp, 2 bytes for the hashlen, * EVP_MAX_MD_SIZE for transcript hash, 1 byte for app cookie length, app cookie * length bytes, SHA256_DIGEST_LENGTH bytes for the HMAC of the whole thing. */ -#define MAX_COOKIE_SIZE (2 + 2 + 2 + 2 + 2 + 1 + 8 + 2 + EVP_MAX_MD_SIZE + 1 \ +#define MAX_COOKIE_SIZE (2 + 2 + 2 + 2 + 2 + 1 + 4 + 2 + EVP_MAX_MD_SIZE + 1 \ + SSL_COOKIE_LENGTH + SHA256_DIGEST_LENGTH) /* @@ -35,16 +35,18 @@ + SSL_MAX_SSL_SESSION_ID_LENGTH + 2 + 1 + 2 + 6 + 4 \ + MAX_COOKIE_SIZE) +#ifndef OPENSSL_NO_QUIC +static int quic_ticket_compatible(const SSL_SESSION *session, const SSL *s); +#endif + /* * Parse the client's renegotiation binding and abort if it's not right */ -int tls_parse_ctos_renegotiate(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_renegotiate(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { unsigned int ilen; const unsigned char *data; - int ok; /* Parse the length byte */ if (!PACKET_get_1(pkt, &ilen) @@ -59,16 +61,8 @@ int tls_parse_ctos_renegotiate(SSL_CONNECTION *s, PACKET *pkt, return 0; } - ok = memcmp(data, s->s3.previous_client_finished, - s->s3.previous_client_finished_len); -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (ok) { - if ((data[0] ^ s->s3.previous_client_finished[0]) != 0xFF) { - ok = 0; - } - } -#endif - if (ok) { + if (memcmp(data, s->s3.previous_client_finished, + s->s3.previous_client_finished_len)) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_RENEGOTIATION_MISMATCH); return 0; } @@ -101,8 +95,8 @@ int tls_parse_ctos_renegotiate(SSL_CONNECTION *s, PACKET *pkt, * extension. * - On session reconnect, the servername extension may be absent. */ -int tls_parse_ctos_server_name(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx) +int tls_parse_ctos_server_name(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) { unsigned int servname_type; PACKET sni, hostname; @@ -136,7 +130,7 @@ int tls_parse_ctos_server_name(SSL_CONNECTION *s, PACKET *pkt, * In TLSv1.2 and below the SNI is associated with the session. In TLSv1.3 * we always use the SNI value from the handshake. */ - if (!s->hit || SSL_CONNECTION_IS_TLS13(s)) { + if (!s->hit || SSL_IS_TLS13(s)) { if (PACKET_remaining(&hostname) > TLSEXT_MAXLEN_host_name) { SSLfatal(s, SSL_AD_UNRECOGNIZED_NAME, SSL_R_BAD_EXTENSION); return 0; @@ -173,8 +167,7 @@ int tls_parse_ctos_server_name(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_ctos_maxfragmentlen(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_maxfragmentlen(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { unsigned int value; @@ -211,8 +204,8 @@ int tls_parse_ctos_maxfragmentlen(SSL_CONNECTION *s, PACKET *pkt, } #ifndef OPENSSL_NO_SRP -int tls_parse_ctos_srp(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_ctos_srp(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { PACKET srp_I; @@ -231,8 +224,7 @@ int tls_parse_ctos_srp(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } #endif -int tls_parse_ctos_ec_pt_formats(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_ec_pt_formats(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { PACKET ec_point_format_list; @@ -255,14 +247,13 @@ int tls_parse_ctos_ec_pt_formats(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_ctos_session_ticket(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_session_ticket(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (s->ext.session_ticket_cb && - !s->ext.session_ticket_cb(SSL_CONNECTION_GET_SSL(s), - PACKET_data(pkt), PACKET_remaining(pkt), - s->ext.session_ticket_cb_arg)) { + !s->ext.session_ticket_cb(s, PACKET_data(pkt), + PACKET_remaining(pkt), + s->ext.session_ticket_cb_arg)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } @@ -270,7 +261,7 @@ int tls_parse_ctos_session_ticket(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_ctos_sig_algs_cert(SSL_CONNECTION *s, PACKET *pkt, +int tls_parse_ctos_sig_algs_cert(SSL *s, PACKET *pkt, ossl_unused unsigned int context, ossl_unused X509 *x, ossl_unused size_t chainidx) @@ -291,8 +282,8 @@ int tls_parse_ctos_sig_algs_cert(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_ctos_sig_algs(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx) +int tls_parse_ctos_sig_algs(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { PACKET supported_sig_algs; @@ -311,8 +302,7 @@ int tls_parse_ctos_sig_algs(SSL_CONNECTION *s, PACKET *pkt, } #ifndef OPENSSL_NO_OCSP -int tls_parse_ctos_status_request(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_status_request(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { PACKET responder_id_list, exts; @@ -351,7 +341,7 @@ int tls_parse_ctos_status_request(SSL_CONNECTION *s, PACKET *pkt, if (PACKET_remaining(&responder_id_list) > 0) { s->ext.ocsp.ids = sk_OCSP_RESPID_new_null(); if (s->ext.ocsp.ids == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } } else { @@ -416,8 +406,8 @@ int tls_parse_ctos_status_request(SSL_CONNECTION *s, PACKET *pkt, #endif #ifndef OPENSSL_NO_NEXTPROTONEG -int tls_parse_ctos_npn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_ctos_npn(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { /* * We shouldn't accept this extension on a @@ -434,8 +424,8 @@ int tls_parse_ctos_npn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, * Save the ALPN extension in a ClientHello.|pkt| holds the contents of the ALPN * extension, not including type and length. Returns: 1 on success, 0 on error. */ -int tls_parse_ctos_alpn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_ctos_alpn(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { PACKET protocol_list, save_protocol_list, protocol; @@ -471,17 +461,16 @@ int tls_parse_ctos_alpn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } #ifndef OPENSSL_NO_SRTP -int tls_parse_ctos_use_srtp(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx) +int tls_parse_ctos_use_srtp(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { STACK_OF(SRTP_PROTECTION_PROFILE) *srvr; unsigned int ct, mki_len, id; int i, srtp_pref; PACKET subpkt; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); /* Ignore this if we have no SRTP profiles */ - if (SSL_get_srtp_profiles(ssl) == NULL) + if (SSL_get_srtp_profiles(s) == NULL) return 1; /* Pull off the length of the cipher suite list and check it is even */ @@ -492,7 +481,7 @@ int tls_parse_ctos_use_srtp(SSL_CONNECTION *s, PACKET *pkt, return 0; } - srvr = SSL_get_srtp_profiles(ssl); + srvr = SSL_get_srtp_profiles(s); s->srtp_profile = NULL; /* Search all profiles for a match initially */ srtp_pref = sk_SRTP_PROTECTION_PROFILE_num(srvr); @@ -539,8 +528,8 @@ int tls_parse_ctos_use_srtp(SSL_CONNECTION *s, PACKET *pkt, } #endif -int tls_parse_ctos_etm(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_ctos_etm(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { if (!(s->options & SSL_OP_NO_ENCRYPT_THEN_MAC)) s->ext.use_etm = 1; @@ -552,8 +541,7 @@ int tls_parse_ctos_etm(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, * Process a psk_kex_modes extension received in the ClientHello. |pkt| contains * the raw PACKET data for the extension. Returns 1 on success or 0 on failure. */ -int tls_parse_ctos_psk_kex_modes(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_psk_kex_modes(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { #ifndef OPENSSL_NO_TLS1_3 @@ -573,21 +561,6 @@ int tls_parse_ctos_psk_kex_modes(SSL_CONNECTION *s, PACKET *pkt, && (s->options & SSL_OP_ALLOW_NO_DHE_KEX) != 0) s->ext.psk_kex_mode |= TLSEXT_KEX_MODE_FLAG_KE; } - - if (((s->ext.psk_kex_mode & TLSEXT_KEX_MODE_FLAG_KE) != 0) - && (s->options & SSL_OP_PREFER_NO_DHE_KEX) != 0) { - - /* - * If NO_DHE is supported and preferred, then we only remember this - * mode. DHE PSK will not be used for sure, because in any case where - * it would be supported (i.e. if a key share is present), NO_DHE would - * be supported as well. As the latter is preferred it would be - * chosen. By removing DHE PSK here, we don't have to deal with the - * SSL_OP_PREFER_NO_DHE_KEX option in any other place. - */ - s->ext.psk_kex_mode = TLSEXT_KEX_MODE_FLAG_KE; - } - #endif return 1; @@ -597,8 +570,8 @@ int tls_parse_ctos_psk_kex_modes(SSL_CONNECTION *s, PACKET *pkt, * Process a key_share extension received in the ClientHello. |pkt| contains * the raw PACKET data for the extension. Returns 1 on success or 0 on failure. */ -int tls_parse_ctos_key_share(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx) +int tls_parse_ctos_key_share(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { #ifndef OPENSSL_NO_TLS1_3 unsigned int group_id; @@ -679,28 +652,21 @@ int tls_parse_ctos_key_share(SSL_CONNECTION *s, PACKET *pkt, } /* Check if this share is for a group we can use */ - if (!check_in_list(s, group_id, srvrgroups, srvr_num_groups, 1) - || !tls_group_allowed(s, group_id, SSL_SECOP_CURVE_SUPPORTED) - /* - * We tolerate but ignore a group id that we don't think is - * suitable for TLSv1.3 - */ - || !tls_valid_group(s, group_id, TLS1_3_VERSION, TLS1_3_VERSION, - 0, NULL)) { + if (!check_in_list(s, group_id, srvrgroups, srvr_num_groups, 1)) { /* Share not suitable */ continue; } - s->s3.group_id = group_id; - /* Cache the selected group ID in the SSL_SESSION */ - s->session->kex_group = group_id; - if ((s->s3.peer_tmp = ssl_generate_param_group(s, group_id)) == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_UNABLE_TO_FIND_ECDH_PARAMETERS); return 0; } + s->s3.group_id = group_id; + /* Cache the selected group ID in the SSL_SESSION */ + s->session->kex_group = group_id; + if (tls13_set_encoded_pub_key(s->s3.peer_tmp, PACKET_data(&encoded_pt), PACKET_remaining(&encoded_pt)) <= 0) { @@ -715,8 +681,8 @@ int tls_parse_ctos_key_share(SSL_CONNECTION *s, PACKET *pkt, return 1; } -int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_ctos_cookie(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { #ifndef OPENSSL_NO_TLS1_3 unsigned int format, version, key_share, group_id; @@ -728,12 +694,10 @@ int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, unsigned char hmac[SHA256_DIGEST_LENGTH]; unsigned char hrr[MAX_HRR_SIZE]; size_t rawlen, hmaclen, hrrlen, ciphlen; - uint64_t tm, now; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); + unsigned long tm, now; /* Ignore any cookie if we're not set up to verify it */ - if (sctx->verify_stateless_cookie_cb == NULL + if (s->ctx->verify_stateless_cookie_cb == NULL || (s->s3.flags & TLS1_FLAGS_STATELESS) == 0) return 1; @@ -754,20 +718,20 @@ int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, /* Verify the HMAC of the cookie */ hctx = EVP_MD_CTX_create(); - pkey = EVP_PKEY_new_raw_private_key_ex(sctx->libctx, "HMAC", - sctx->propq, + pkey = EVP_PKEY_new_raw_private_key_ex(s->ctx->libctx, "HMAC", + s->ctx->propq, s->session_ctx->ext.cookie_hmac_key, sizeof(s->session_ctx->ext.cookie_hmac_key)); if (hctx == NULL || pkey == NULL) { EVP_MD_CTX_free(hctx); EVP_PKEY_free(pkey); - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } hmaclen = SHA256_DIGEST_LENGTH; - if (EVP_DigestSignInit_ex(hctx, NULL, "SHA2-256", sctx->libctx, - sctx->propq, pkey, NULL) <= 0 + if (EVP_DigestSignInit_ex(hctx, NULL, "SHA2-256", s->ctx->libctx, + s->ctx->propq, pkey, NULL) <= 0 || EVP_DigestSign(hctx, hmac, &hmaclen, data, rawlen - SHA256_DIGEST_LENGTH) <= 0 || hmaclen != SHA256_DIGEST_LENGTH) { @@ -831,7 +795,7 @@ int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } if (!PACKET_get_1(&cookie, &key_share) - || !PACKET_get_net_8(&cookie, &tm) + || !PACKET_get_net_4(&cookie, &tm) || !PACKET_get_length_prefixed_2(&cookie, &chhash) || !PACKET_get_length_prefixed_1(&cookie, &appcookie) || PACKET_remaining(&cookie) != SHA256_DIGEST_LENGTH) { @@ -840,16 +804,15 @@ int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } /* We tolerate a cookie age of up to 10 minutes (= 60 * 10 seconds) */ - now = time(NULL); + now = (unsigned long)time(NULL); if (tm > now || (now - tm) > 600) { /* Cookie is stale. Ignore it */ return 1; } /* Verify the app cookie */ - if (sctx->verify_stateless_cookie_cb(ssl, - PACKET_data(&appcookie), - PACKET_remaining(&appcookie)) == 0) { + if (s->ctx->verify_stateless_cookie_cb(s, PACKET_data(&appcookie), + PACKET_remaining(&appcookie)) == 0) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_COOKIE_MISMATCH); return 0; } @@ -869,8 +832,8 @@ int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, || !WPACKET_memcpy(&hrrpkt, hrrrandom, SSL3_RANDOM_SIZE) || !WPACKET_sub_memcpy_u8(&hrrpkt, s->tmp_session_id, s->tmp_session_id_len) - || !ssl->method->put_cipher_by_char(s->s3.tmp.new_cipher, &hrrpkt, - &ciphlen) + || !s->method->put_cipher_by_char(s->s3.tmp.new_cipher, &hrrpkt, + &ciphlen) || !WPACKET_put_bytes_u8(&hrrpkt, 0) || !WPACKET_start_sub_packet_u16(&hrrpkt)) { WPACKET_cleanup(&hrrpkt); @@ -917,7 +880,7 @@ int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } /* Act as if this ClientHello came after a HelloRetryRequest */ - s->hello_retry_request = SSL_HRR_PENDING; + s->hello_retry_request = 1; s->ext.cookieok = 1; #endif @@ -925,8 +888,7 @@ int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, return 1; } -int tls_parse_ctos_supported_groups(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_supported_groups(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { PACKET supported_groups_list; @@ -939,7 +901,7 @@ int tls_parse_ctos_supported_groups(SSL_CONNECTION *s, PACKET *pkt, return 0; } - if (!s->hit || SSL_CONNECTION_IS_TLS13(s)) { + if (!s->hit || SSL_IS_TLS13(s)) { OPENSSL_free(s->ext.peer_supportedgroups); s->ext.peer_supportedgroups = NULL; s->ext.peer_supportedgroups_len = 0; @@ -951,11 +913,55 @@ int tls_parse_ctos_supported_groups(SSL_CONNECTION *s, PACKET *pkt, } } +# ifndef OPENSSL_NO_STATUS + /* record client ellipitc_curves */ + if (s->status_param.ssl_status_enable) { + uint16_t *peer_supportedgroups = s->ext.peer_supportedgroups; + size_t peer_supportedgroups_len = s->ext.peer_supportedgroups_len; + + if (s->hit) { + peer_supportedgroups = NULL; + peer_supportedgroups_len = 0; + if (!tls1_save_u16(&supported_groups_list, &peer_supportedgroups, + &peer_supportedgroups_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + + s->status_param.type = SSL_CLIENT_ECC_CURVES; + if (s->status_callback((unsigned char *)peer_supportedgroups, + peer_supportedgroups_len * 2, + &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + return 0; + } + + if (s->hit) + OPENSSL_free(peer_supportedgroups); + + } +# endif + return 1; } -int tls_parse_ctos_ems(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +int tls_parse_ctos_delegated_credential(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->statem.hand_state == TLS_ST_SR_CLNT_HELLO) + return tls_process_dc_request(s, pkt, context, x, chainidx); + + if (s->statem.hand_state == TLS_ST_SR_CERT) + return tls_parse_dc_from_extension(s, pkt, context, x, chainidx); + + return 0; +} +#endif + +int tls_parse_ctos_ems(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { /* The extension must always be empty */ if (PACKET_remaining(pkt) != 0) { @@ -972,7 +978,7 @@ int tls_parse_ctos_ems(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, } -int tls_parse_ctos_early_data(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, +int tls_parse_ctos_early_data(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (PACKET_remaining(pkt) != 0) { @@ -988,7 +994,7 @@ int tls_parse_ctos_early_data(SSL_CONNECTION *s, PACKET *pkt, unsigned int conte return 1; } -static SSL_TICKET_STATUS tls_get_stateful_ticket(SSL_CONNECTION *s, PACKET *tick, +static SSL_TICKET_STATUS tls_get_stateful_ticket(SSL *s, PACKET *tick, SSL_SESSION **sess) { SSL_SESSION *tmpsess = NULL; @@ -1016,16 +1022,14 @@ static SSL_TICKET_STATUS tls_get_stateful_ticket(SSL_CONNECTION *s, PACKET *tick return SSL_TICKET_SUCCESS; } -int tls_parse_ctos_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx) +int tls_parse_ctos_psk(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) { PACKET identities, binders, binder; size_t binderoffset, hashsize; SSL_SESSION *sess = NULL; unsigned int id, i, ext = 0; const EVP_MD *md = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - SSL *ssl = SSL_CONNECTION_GET_SSL(s); /* * If we have no PSK kex mode that we recognise then we can't resume so @@ -1054,14 +1058,14 @@ int tls_parse_ctos_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, idlen = PACKET_remaining(&identity); if (s->psk_find_session_cb != NULL - && !s->psk_find_session_cb(ssl, PACKET_data(&identity), idlen, + && !s->psk_find_session_cb(s, PACKET_data(&identity), idlen, &sess)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_EXTENSION); return 0; } #ifndef OPENSSL_NO_PSK - if (sess == NULL + if(sess == NULL && s->psk_server_callback != NULL && idlen <= PSK_MAX_IDENTITY_LEN) { char *pskid = NULL; @@ -1072,7 +1076,7 @@ int tls_parse_ctos_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - pskdatalen = s->psk_server_callback(ssl, pskid, pskdata, + pskdatalen = s->psk_server_callback(s, pskid, pskdata, sizeof(pskdata)); OPENSSL_free(pskid); if (pskdatalen > PSK_MAX_PSK_LEN) { @@ -1086,7 +1090,7 @@ int tls_parse_ctos_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, * We found a PSK using an old style callback. We don't know * the digest so we default to SHA256 as per the TLSv1.3 spec */ - cipher = SSL_CIPHER_find(ssl, tls13_aes128gcmsha256_id); + cipher = SSL_CIPHER_find(s, tls13_aes128gcmsha256_id); if (cipher == NULL) { OPENSSL_cleanse(pskdata, pskdatalen); SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); @@ -1131,7 +1135,7 @@ int tls_parse_ctos_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, s->ext.early_data_ok = 1; s->ext.ticket_expected = 1; } else { - OSSL_TIME t, age, expire; + uint32_t ticket_age = 0, now, agesec, agems; int ret; /* @@ -1170,41 +1174,42 @@ int tls_parse_ctos_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, continue; } - age = ossl_time_subtract(ossl_ms2time(ticket_agel), - ossl_ms2time(sess->ext.tick_age_add)); - t = ossl_time_subtract(ossl_time_now(), sess->time); + ticket_age = (uint32_t)ticket_agel; + now = (uint32_t)time(NULL); + agesec = now - (uint32_t)sess->time; + agems = agesec * (uint32_t)1000; + ticket_age -= sess->ext.tick_age_add; /* - * Although internally we use OSS_TIME which has ns granularity, - * when SSL_SESSION structures are serialised/deserialised we use - * second granularity for the sess->time field. Therefore it could - * appear that the client's ticket age is longer than ours (our - * ticket age calculation should always be slightly longer than the - * client's due to the network latency). Therefore we add 1000ms to - * our age calculation to adjust for rounding errors. + * For simplicity we do our age calculations in seconds. If the + * client does it in ms then it could appear that their ticket age + * is longer than ours (our ticket age calculation should always be + * slightly longer than the client's due to the network latency). + * Therefore we add 1000ms to our age calculation to adjust for + * rounding errors. */ - expire = ossl_time_add(t, ossl_ms2time(1000)); - if (id == 0 - && ossl_time_compare(sess->timeout, t) >= 0 - && ossl_time_compare(age, expire) <= 0 - && ossl_time_compare(ossl_time_add(age, TICKET_AGE_ALLOWANCE), - expire) >= 0) { + && sess->timeout >= (long)agesec + && agems / (uint32_t)1000 == agesec + && ticket_age <= agems + 1000 + && ticket_age + TICKET_AGE_ALLOWANCE >= agems + 1000) { /* * Ticket age is within tolerance and not expired. We allow it * for early data */ s->ext.early_data_ok = 1; } +#ifndef OPENSSL_NO_QUIC + if (SSL_is_quic(s)) { + if (!quic_ticket_compatible(sess, s)) + s->ext.early_data_ok = 0; + } +#endif } - md = ssl_md(sctx, sess->cipher->algorithm2); - if (md == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } + md = ssl_md(s->ctx, sess->cipher->algorithm2); if (!EVP_MD_is_a(md, - EVP_MD_get0_name(ssl_md(sctx, + EVP_MD_get0_name(ssl_md(s->ctx, s->s3.tmp.new_cipher->algorithm2)))) { /* The ciphersuite is not compatible with this session. */ SSL_SESSION_free(sess); @@ -1255,7 +1260,7 @@ int tls_parse_ctos_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, return 0; } -int tls_parse_ctos_post_handshake_auth(SSL_CONNECTION *s, PACKET *pkt, +int tls_parse_ctos_post_handshake_auth(SSL *s, PACKET *pkt, ossl_unused unsigned int context, ossl_unused X509 *x, ossl_unused size_t chainidx) @@ -1271,10 +1276,62 @@ int tls_parse_ctos_post_handshake_auth(SSL_CONNECTION *s, PACKET *pkt, return 1; } +#ifndef OPENSSL_NO_QUIC +static int quic_ticket_compatible(const SSL_SESSION *session, const SSL *s) +{ + if (!session->is_quic) { + return 0; + } + + if (session->quic_early_data_context == NULL || + s->quic_early_data_context_len != + session->quic_early_data_context_len || + CRYPTO_memcmp(s->quic_early_data_context, + session->quic_early_data_context, + session->quic_early_data_context_len) != 0) { + return 0; + } + + return 1; +} + +int tls_parse_ctos_quic_transport_params_draft(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + OPENSSL_free(s->ext.peer_quic_transport_params_draft); + s->ext.peer_quic_transport_params_draft = NULL; + s->ext.peer_quic_transport_params_draft_len = 0; + + if (!PACKET_memdup(pkt, + &s->ext.peer_quic_transport_params_draft, + &s->ext.peer_quic_transport_params_draft_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + return 1; +} + +int tls_parse_ctos_quic_transport_params(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + OPENSSL_free(s->ext.peer_quic_transport_params); + s->ext.peer_quic_transport_params = NULL; + s->ext.peer_quic_transport_params_len = 0; + + if (!PACKET_memdup(pkt, + &s->ext.peer_quic_transport_params, + &s->ext.peer_quic_transport_params_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + return 1; +} +#endif + /* * Add the server's renegotiation binding */ -EXT_RETURN tls_construct_stoc_renegotiate(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_renegotiate(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1298,7 +1355,7 @@ EXT_RETURN tls_construct_stoc_renegotiate(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_server_name(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_server_name(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1309,7 +1366,7 @@ EXT_RETURN tls_construct_stoc_server_name(SSL_CONNECTION *s, WPACKET *pkt, * Prior to TLSv1.3 we ignore any SNI in the current handshake if resuming. * We just use the servername from the initial handshake. */ - if (s->hit && !SSL_CONNECTION_IS_TLS13(s)) + if (s->hit && !SSL_IS_TLS13(s)) return EXT_RETURN_NOT_SENT; if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_server_name) @@ -1322,7 +1379,7 @@ EXT_RETURN tls_construct_stoc_server_name(SSL_CONNECTION *s, WPACKET *pkt, } /* Add/include the server's max fragment len extension into ServerHello */ -EXT_RETURN tls_construct_stoc_maxfragmentlen(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_maxfragmentlen(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1344,7 +1401,7 @@ EXT_RETURN tls_construct_stoc_maxfragmentlen(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_ec_pt_formats(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_ec_pt_formats(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1370,7 +1427,7 @@ EXT_RETURN tls_construct_stoc_ec_pt_formats(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_supported_groups(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1390,7 +1447,7 @@ EXT_RETURN tls_construct_stoc_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, } /* Copy group ID if supported */ - version = SSL_version(SSL_CONNECTION_GET_SSL(s)); + version = SSL_version(s); for (i = 0; i < numgroups; i++) { uint16_t group = groups[i]; @@ -1430,7 +1487,7 @@ EXT_RETURN tls_construct_stoc_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_session_ticket(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_session_ticket(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1449,7 +1506,7 @@ EXT_RETURN tls_construct_stoc_session_ticket(SSL_CONNECTION *s, WPACKET *pkt, } #ifndef OPENSSL_NO_OCSP -EXT_RETURN tls_construct_stoc_status_request(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_status_request(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1460,7 +1517,7 @@ EXT_RETURN tls_construct_stoc_status_request(SSL_CONNECTION *s, WPACKET *pkt, if (!s->ext.status_expected) return EXT_RETURN_NOT_SENT; - if (SSL_CONNECTION_IS_TLS13(s) && chainidx != 0) + if (SSL_IS_TLS13(s) && chainidx != 0) return EXT_RETURN_NOT_SENT; if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_status_request) @@ -1474,7 +1531,7 @@ EXT_RETURN tls_construct_stoc_status_request(SSL_CONNECTION *s, WPACKET *pkt, * send back an empty extension, with the certificate status appearing as a * separate message */ - if (SSL_CONNECTION_IS_TLS13(s) && !tls_construct_cert_status_body(s, pkt)) { + if (SSL_IS_TLS13(s) && !tls_construct_cert_status_body(s, pkt)) { /* SSLfatal() already called */ return EXT_RETURN_FAIL; } @@ -1488,7 +1545,7 @@ EXT_RETURN tls_construct_stoc_status_request(SSL_CONNECTION *s, WPACKET *pkt, #endif #ifndef OPENSSL_NO_NEXTPROTONEG -EXT_RETURN tls_construct_stoc_next_proto_neg(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_next_proto_neg(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1496,14 +1553,13 @@ EXT_RETURN tls_construct_stoc_next_proto_neg(SSL_CONNECTION *s, WPACKET *pkt, unsigned int npalen; int ret; int npn_seen = s->s3.npn_seen; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); s->s3.npn_seen = 0; - if (!npn_seen || sctx->ext.npn_advertised_cb == NULL) + if (!npn_seen || s->ctx->ext.npn_advertised_cb == NULL) return EXT_RETURN_NOT_SENT; - ret = sctx->ext.npn_advertised_cb(SSL_CONNECTION_GET_SSL(s), &npa, &npalen, - sctx->ext.npn_advertised_cb_arg); + ret = s->ctx->ext.npn_advertised_cb(s, &npa, &npalen, + s->ctx->ext.npn_advertised_cb_arg); if (ret == SSL_TLSEXT_ERR_OK) { if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_next_proto_neg) || !WPACKET_sub_memcpy_u16(pkt, npa, npalen)) { @@ -1517,7 +1573,7 @@ EXT_RETURN tls_construct_stoc_next_proto_neg(SSL_CONNECTION *s, WPACKET *pkt, } #endif -EXT_RETURN tls_construct_stoc_alpn(SSL_CONNECTION *s, WPACKET *pkt, unsigned int context, +EXT_RETURN tls_construct_stoc_alpn(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (s->s3.alpn_selected == NULL) @@ -1539,7 +1595,7 @@ EXT_RETURN tls_construct_stoc_alpn(SSL_CONNECTION *s, WPACKET *pkt, unsigned int } #ifndef OPENSSL_NO_SRTP -EXT_RETURN tls_construct_stoc_use_srtp(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_use_srtp(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1560,8 +1616,7 @@ EXT_RETURN tls_construct_stoc_use_srtp(SSL_CONNECTION *s, WPACKET *pkt, } #endif -EXT_RETURN tls_construct_stoc_etm(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_stoc_etm(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (!s->ext.use_etm) @@ -1572,11 +1627,7 @@ EXT_RETURN tls_construct_stoc_etm(SSL_CONNECTION *s, WPACKET *pkt, * for other cases too. */ if (s->s3.tmp.new_cipher->algorithm_mac == SSL_AEAD - || s->s3.tmp.new_cipher->algorithm_enc == SSL_RC4 - || s->s3.tmp.new_cipher->algorithm_enc == SSL_eGOST2814789CNT - || s->s3.tmp.new_cipher->algorithm_enc == SSL_eGOST2814789CNT12 - || s->s3.tmp.new_cipher->algorithm_enc == SSL_MAGMA - || s->s3.tmp.new_cipher->algorithm_enc == SSL_KUZNYECHIK) { + || s->s3.tmp.new_cipher->algorithm_enc == SSL_RC4) { s->ext.use_etm = 0; return EXT_RETURN_NOT_SENT; } @@ -1590,8 +1641,24 @@ EXT_RETURN tls_construct_stoc_etm(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_ems(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +EXT_RETURN tls_construct_stoc_delegated_credential(SSL *s, WPACKET *pkt, + unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->statem.hand_state == TLS_ST_SW_CERT) + return tls_construct_delegated_credential_raw(s, pkt, context, x, + chainidx); + + if (s->statem.hand_state == TLS_ST_SW_CERT_REQ) + return tls_construct_delegated_credential_request(s, pkt, context, x, + chainidx); + + return EXT_RETURN_NOT_SENT; +} +#endif + +EXT_RETURN tls_construct_stoc_ems(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if ((s->s3.flags & TLS1_FLAGS_RECEIVED_EXTMS) == 0) @@ -1606,11 +1673,11 @@ EXT_RETURN tls_construct_stoc_ems(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_supported_versions(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_supported_versions(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { - if (!ossl_assert(SSL_CONNECTION_IS_TLS13(s))) { + if (!ossl_assert(SSL_IS_TLS13(s))) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return EXT_RETURN_FAIL; } @@ -1626,7 +1693,7 @@ EXT_RETURN tls_construct_stoc_supported_versions(SSL_CONNECTION *s, WPACKET *pkt return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_key_share(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_key_share(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { @@ -1660,13 +1727,10 @@ EXT_RETURN tls_construct_stoc_key_share(SSL_CONNECTION *s, WPACKET *pkt, } return EXT_RETURN_NOT_SENT; } - if (s->hit && (s->ext.psk_kex_mode & TLSEXT_KEX_MODE_FLAG_KE_DHE) == 0) { /* - * PSK ('hit') and explicitly not doing DHE. If the client sent the - * DHE option, we take it by default, except if non-DHE would be - * preferred by config, but this case would have been handled in - * tls_parse_ctos_psk_kex_modes(). + * PSK ('hit') and explicitly not doing DHE (if the client sent the + * DHE option we always take it); don't send key share. */ return EXT_RETURN_NOT_SENT; } @@ -1678,8 +1742,7 @@ EXT_RETURN tls_construct_stoc_key_share(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_FAIL; } - if ((ginf = tls1_group_id_lookup(SSL_CONNECTION_GET_CTX(s), - s->s3.group_id)) == NULL) { + if ((ginf = tls1_group_id_lookup(s->ctx, s->s3.group_id)) == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return EXT_RETURN_FAIL; } @@ -1688,7 +1751,7 @@ EXT_RETURN tls_construct_stoc_key_share(SSL_CONNECTION *s, WPACKET *pkt, /* Regular KEX */ skey = ssl_generate_pkey(s, ckey); if (skey == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return EXT_RETURN_FAIL; } @@ -1762,8 +1825,7 @@ EXT_RETURN tls_construct_stoc_key_share(SSL_CONNECTION *s, WPACKET *pkt, #endif } -EXT_RETURN tls_construct_stoc_cookie(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_stoc_cookie(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { #ifndef OPENSSL_NO_TLS1_3 @@ -1773,13 +1835,11 @@ EXT_RETURN tls_construct_stoc_cookie(SSL_CONNECTION *s, WPACKET *pkt, EVP_MD_CTX *hctx; EVP_PKEY *pkey; int ret = EXT_RETURN_FAIL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if ((s->s3.flags & TLS1_FLAGS_STATELESS) == 0) return EXT_RETURN_NOT_SENT; - if (sctx->gen_stateless_cookie_cb == NULL) { + if (s->ctx->gen_stateless_cookie_cb == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_NO_COOKIE_CALLBACK_SET); return EXT_RETURN_FAIL; } @@ -1792,11 +1852,11 @@ EXT_RETURN tls_construct_stoc_cookie(SSL_CONNECTION *s, WPACKET *pkt, || !WPACKET_put_bytes_u16(pkt, COOKIE_STATE_FORMAT_VERSION) || !WPACKET_put_bytes_u16(pkt, TLS1_3_VERSION) || !WPACKET_put_bytes_u16(pkt, s->s3.group_id) - || !ssl->method->put_cipher_by_char(s->s3.tmp.new_cipher, pkt, - &ciphlen) + || !s->method->put_cipher_by_char(s->s3.tmp.new_cipher, pkt, + &ciphlen) /* Is there a key_share extension present in this HRR? */ || !WPACKET_put_bytes_u8(pkt, s->s3.peer_tmp == NULL) - || !WPACKET_put_bytes_u64(pkt, time(NULL)) + || !WPACKET_put_bytes_u32(pkt, (unsigned int)time(NULL)) || !WPACKET_start_sub_packet_u16(pkt) || !WPACKET_reserve_bytes(pkt, EVP_MAX_MD_SIZE, &hashval1)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); @@ -1824,8 +1884,7 @@ EXT_RETURN tls_construct_stoc_cookie(SSL_CONNECTION *s, WPACKET *pkt, } /* Generate the application cookie */ - if (sctx->gen_stateless_cookie_cb(ssl, appcookie1, - &appcookielen) == 0) { + if (s->ctx->gen_stateless_cookie_cb(s, appcookie1, &appcookielen) == 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_COOKIE_GEN_CALLBACK_FAILURE); return EXT_RETURN_FAIL; } @@ -1848,17 +1907,17 @@ EXT_RETURN tls_construct_stoc_cookie(SSL_CONNECTION *s, WPACKET *pkt, /* HMAC the cookie */ hctx = EVP_MD_CTX_create(); - pkey = EVP_PKEY_new_raw_private_key_ex(sctx->libctx, "HMAC", - sctx->propq, + pkey = EVP_PKEY_new_raw_private_key_ex(s->ctx->libctx, "HMAC", + s->ctx->propq, s->session_ctx->ext.cookie_hmac_key, sizeof(s->session_ctx->ext.cookie_hmac_key)); if (hctx == NULL || pkey == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } - if (EVP_DigestSignInit_ex(hctx, NULL, "SHA2-256", sctx->libctx, - sctx->propq, pkey, NULL) <= 0 + if (EVP_DigestSignInit_ex(hctx, NULL, "SHA2-256", s->ctx->libctx, + s->ctx->propq, pkey, NULL) <= 0 || EVP_DigestSign(hctx, hmac, &hmaclen, cookie, totcookielen) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); @@ -1890,44 +1949,36 @@ EXT_RETURN tls_construct_stoc_cookie(SSL_CONNECTION *s, WPACKET *pkt, #endif } -EXT_RETURN tls_construct_stoc_cryptopro_bug(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, X509 *x, - size_t chainidx) -{ - const unsigned char cryptopro_ext[36] = { - 0xfd, 0xe8, /* 65000 */ - 0x00, 0x20, /* 32 bytes length */ - 0x30, 0x1e, 0x30, 0x08, 0x06, 0x06, 0x2a, 0x85, - 0x03, 0x02, 0x02, 0x09, 0x30, 0x08, 0x06, 0x06, - 0x2a, 0x85, 0x03, 0x02, 0x02, 0x16, 0x30, 0x08, - 0x06, 0x06, 0x2a, 0x85, 0x03, 0x02, 0x02, 0x17 - }; - - if (((s->s3.tmp.new_cipher->id & 0xFFFF) != 0x80 - && (s->s3.tmp.new_cipher->id & 0xFFFF) != 0x81) - || (SSL_get_options(SSL_CONNECTION_GET_SSL(s)) - & SSL_OP_CRYPTOPRO_TLSEXT_BUG) == 0) - return EXT_RETURN_NOT_SENT; - - if (!WPACKET_memcpy(pkt, cryptopro_ext, sizeof(cryptopro_ext))) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return EXT_RETURN_FAIL; - } - - return EXT_RETURN_SENT; -} - -EXT_RETURN tls_construct_stoc_early_data(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_early_data(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (context == SSL_EXT_TLS1_3_NEW_SESSION_TICKET) { - if (s->max_early_data == 0) + uint32_t max_early_data = s->max_early_data; + + if (max_early_data == 0) return EXT_RETURN_NOT_SENT; +#ifndef OPENSSL_NO_QUIC + if (s->quic_method != NULL) { + if (s->quic_early_data_context == NULL) { + /* if the server is to support 0-RTT in QUIC, + * SSL_set_quic_early_data_context must be called on the server + * with a non-empty context + */ + return EXT_RETURN_NOT_SENT; + } else { + /* QUIC does not use the max_early_data_size parameter and + * always sets it to a fixed value. See draft-ietf-quic-tls-22. + */ + max_early_data = 0xFFFFFFFF; + } + } +#endif + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_early_data) || !WPACKET_start_sub_packet_u16(pkt) - || !WPACKET_put_bytes_u32(pkt, s->max_early_data) + || !WPACKET_put_bytes_u32(pkt, max_early_data) || !WPACKET_close(pkt)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return EXT_RETURN_FAIL; @@ -1949,8 +2000,7 @@ EXT_RETURN tls_construct_stoc_early_data(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_psk(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_stoc_psk(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx) { if (!s->hit) @@ -1967,163 +2017,47 @@ EXT_RETURN tls_construct_stoc_psk(SSL_CONNECTION *s, WPACKET *pkt, return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_client_cert_type(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) +#ifndef OPENSSL_NO_QUIC +EXT_RETURN tls_construct_stoc_quic_transport_params_draft(SSL *s, WPACKET *pkt, + unsigned int context, + X509 *x, + size_t chainidx) { - if (sc->ext.client_cert_type_ctos == OSSL_CERT_TYPE_CTOS_ERROR - && (send_certificate_request(sc) - || sc->post_handshake_auth == SSL_PHA_EXT_RECEIVED)) { - /* Did not receive an acceptable cert type - and doing client auth */ - SSLfatal(sc, SSL_AD_UNSUPPORTED_CERTIFICATE, SSL_R_BAD_EXTENSION); - return EXT_RETURN_FAIL; - } - - if (sc->ext.client_cert_type == TLSEXT_cert_type_x509) { - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - return EXT_RETURN_NOT_SENT; - } - - /* - * Note: only supposed to send this if we are going to do a cert request, - * but TLSv1.3 could do a PHA request if the client supports it - */ - if ((!send_certificate_request(sc) && sc->post_handshake_auth != SSL_PHA_EXT_RECEIVED) - || sc->ext.client_cert_type_ctos != OSSL_CERT_TYPE_CTOS_GOOD - || sc->client_cert_type == NULL) { - /* if we don't send it, reset to TLSEXT_cert_type_x509 */ - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - sc->ext.client_cert_type = TLSEXT_cert_type_x509; + if (s->quic_transport_version == TLSEXT_TYPE_quic_transport_parameters + || s->ext.peer_quic_transport_params_draft_len == 0 + || s->ext.quic_transport_params == NULL + || s->ext.quic_transport_params_len == 0) { return EXT_RETURN_NOT_SENT; } - if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_client_cert_type) - || !WPACKET_start_sub_packet_u16(pkt) - || !WPACKET_put_bytes_u8(pkt, sc->ext.client_cert_type) - || !WPACKET_close(pkt)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_quic_transport_parameters_draft) + || !WPACKET_sub_memcpy_u16(pkt, s->ext.quic_transport_params, + s->ext.quic_transport_params_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return EXT_RETURN_FAIL; } - return EXT_RETURN_SENT; -} - -/* One of |pref|, |other| is configured and the values are sanitized */ -static int reconcile_cert_type(const unsigned char *pref, size_t pref_len, - const unsigned char *other, size_t other_len, - uint8_t *chosen_cert_type) -{ - size_t i; - - for (i = 0; i < pref_len; i++) { - if (memchr(other, pref[i], other_len) != NULL) { - *chosen_cert_type = pref[i]; - return OSSL_CERT_TYPE_CTOS_GOOD; - } - } - return OSSL_CERT_TYPE_CTOS_ERROR; -} -int tls_parse_ctos_client_cert_type(SSL_CONNECTION *sc, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) -{ - PACKET supported_cert_types; - const unsigned char *data; - size_t len; - - /* Ignore the extension */ - if (sc->client_cert_type == NULL) { - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - sc->ext.client_cert_type = TLSEXT_cert_type_x509; - return 1; - } - - if (!PACKET_as_length_prefixed_1(pkt, &supported_cert_types)) { - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_ERROR; - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - if ((len = PACKET_remaining(&supported_cert_types)) == 0) { - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_ERROR; - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - if (!PACKET_get_bytes(&supported_cert_types, &data, len)) { - sc->ext.client_cert_type_ctos = OSSL_CERT_TYPE_CTOS_ERROR; - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - /* client_cert_type: client (peer) has priority */ - sc->ext.client_cert_type_ctos = reconcile_cert_type(data, len, - sc->client_cert_type, sc->client_cert_type_len, - &sc->ext.client_cert_type); - - /* Ignore the error until sending - so we can check cert auth*/ - return 1; + return EXT_RETURN_SENT; } -EXT_RETURN tls_construct_stoc_server_cert_type(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) +EXT_RETURN tls_construct_stoc_quic_transport_params(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) { - if (sc->ext.server_cert_type == TLSEXT_cert_type_x509) { - sc->ext.server_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - return EXT_RETURN_NOT_SENT; - } - if (sc->ext.server_cert_type_ctos != OSSL_CERT_TYPE_CTOS_GOOD - || sc->server_cert_type == NULL) { - /* if we don't send it, reset to TLSEXT_cert_type_x509 */ - sc->ext.server_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - sc->ext.server_cert_type = TLSEXT_cert_type_x509; + if (s->quic_transport_version == TLSEXT_TYPE_quic_transport_parameters_draft + || s->ext.peer_quic_transport_params_len == 0 + || s->ext.quic_transport_params == NULL + || s->ext.quic_transport_params_len == 0) { return EXT_RETURN_NOT_SENT; } - if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_server_cert_type) - || !WPACKET_start_sub_packet_u16(pkt) - || !WPACKET_put_bytes_u8(pkt, sc->ext.server_cert_type) - || !WPACKET_close(pkt)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_quic_transport_parameters) + || !WPACKET_sub_memcpy_u16(pkt, s->ext.quic_transport_params, + s->ext.quic_transport_params_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return EXT_RETURN_FAIL; } - return EXT_RETURN_SENT; -} - -int tls_parse_ctos_server_cert_type(SSL_CONNECTION *sc, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx) -{ - PACKET supported_cert_types; - const unsigned char *data; - size_t len; - - /* Ignore the extension */ - if (sc->server_cert_type == NULL) { - sc->ext.server_cert_type_ctos = OSSL_CERT_TYPE_CTOS_NONE; - sc->ext.server_cert_type = TLSEXT_cert_type_x509; - return 1; - } - - if (!PACKET_as_length_prefixed_1(pkt, &supported_cert_types)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - - if ((len = PACKET_remaining(&supported_cert_types)) == 0) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - if (!PACKET_get_bytes(&supported_cert_types, &data, len)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); - return 0; - } - /* server_cert_type: server (this) has priority */ - sc->ext.server_cert_type_ctos = reconcile_cert_type(sc->server_cert_type, sc->server_cert_type_len, - data, len, - &sc->ext.server_cert_type); - if (sc->ext.server_cert_type_ctos == OSSL_CERT_TYPE_CTOS_GOOD) - return 1; - /* Did not receive an acceptable cert type */ - SSLfatal(sc, SSL_AD_UNSUPPORTED_CERTIFICATE, SSL_R_BAD_EXTENSION); - return 0; + return EXT_RETURN_SENT; } +#endif diff --git a/openssl/src/providers/implementations/kem/eckem.h b/openssl/src/ssl/statem/local.h similarity index 50% rename from openssl/src/providers/implementations/kem/eckem.h rename to openssl/src/ssl/statem/local.h index 2e46a0f2f..a0c02bcf1 100644 --- a/openssl/src/providers/implementations/kem/eckem.h +++ b/openssl/src/ssl/statem/local.h @@ -1,13 +1,12 @@ /* - * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt */ -#define KEM_MODE_UNDEFINED 0 -#define KEM_MODE_DHKEM 1 - -int ossl_eckem_modename2id(const char *name); +/* + * This header file is only used for the --symbol-prefix search export symbol. + */ diff --git a/openssl/src/ssl/statem/statem.c b/openssl/src/ssl/statem/statem.c index 921d7cfb1..c8a66984b 100644 --- a/openssl/src/ssl/statem/statem.c +++ b/openssl/src/ssl/statem/statem.c @@ -17,6 +17,9 @@ #include "../ssl_local.h" #include "statem_local.h" #include +#ifndef OPENSSL_NO_NTLS +# include "../statem_ntls/ntls_statem.h" +#endif /* * This file implements the SSL/TLS/DTLS state machines. @@ -62,49 +65,29 @@ typedef enum { SUB_STATE_END_HANDSHAKE } SUB_STATE_RETURN; -static int state_machine(SSL_CONNECTION *s, int server); -static void init_read_state_machine(SSL_CONNECTION *s); -static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s); -static void init_write_state_machine(SSL_CONNECTION *s); -static SUB_STATE_RETURN write_state_machine(SSL_CONNECTION *s); +static int state_machine(SSL *s, int server); +static void init_read_state_machine(SSL *s); +static SUB_STATE_RETURN read_state_machine(SSL *s); +static void init_write_state_machine(SSL *s); +static SUB_STATE_RETURN write_state_machine(SSL *s); OSSL_HANDSHAKE_STATE SSL_get_state(const SSL *ssl) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(ssl); - - if (sc == NULL) - return TLS_ST_BEFORE; - - return sc->statem.hand_state; + return ssl->statem.hand_state; } int SSL_in_init(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return sc->statem.in_init; + return s->statem.in_init; } int SSL_is_init_finished(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - return !(sc->statem.in_init) && (sc->statem.hand_state == TLS_ST_OK); + return !(s->statem.in_init) && (s->statem.hand_state == TLS_ST_OK); } int SSL_in_before(const SSL *s) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - /* * Historically being "in before" meant before anything had happened. In the * current code though we remain in the "before" state for a while after we @@ -112,43 +95,39 @@ int SSL_in_before(const SSL *s) * first message to arrive). There "in before" is taken to mean "in before" * and not started any handshake process yet. */ - return (sc->statem.hand_state == TLS_ST_BEFORE) - && (sc->statem.state == MSG_FLOW_UNINITED); -} - -OSSL_HANDSHAKE_STATE ossl_statem_get_state(SSL_CONNECTION *s) -{ - return s != NULL ? s->statem.hand_state : TLS_ST_BEFORE; + return (s->statem.hand_state == TLS_ST_BEFORE) + && (s->statem.state == MSG_FLOW_UNINITED); } /* * Clear the state machine state and reset back to MSG_FLOW_UNINITED */ -void ossl_statem_clear(SSL_CONNECTION *s) +void ossl_statem_clear(SSL *s) { s->statem.state = MSG_FLOW_UNINITED; s->statem.hand_state = TLS_ST_BEFORE; - ossl_statem_set_in_init(s, 1); + s->statem.in_init = 1; s->statem.no_cert_verify = 0; } /* * Set the state machine up ready for a renegotiation handshake */ -void ossl_statem_set_renegotiate(SSL_CONNECTION *s) +void ossl_statem_set_renegotiate(SSL *s) { - ossl_statem_set_in_init(s, 1); + s->statem.in_init = 1; s->statem.request_state = TLS_ST_SW_HELLO_REQ; } -void ossl_statem_send_fatal(SSL_CONNECTION *s, int al) +void ossl_statem_send_fatal(SSL *s, int al) { /* We shouldn't call SSLfatal() twice. Once is enough */ if (s->statem.in_init && s->statem.state == MSG_FLOW_ERROR) return; - ossl_statem_set_in_init(s, 1); + s->statem.in_init = 1; s->statem.state = MSG_FLOW_ERROR; - if (al != SSL_AD_NO_ALERT) + if (al != SSL_AD_NO_ALERT + && s->statem.enc_write_state != ENC_WRITE_STATE_INVALID) ssl3_send_alert(s, SSL3_AL_FATAL, al); } @@ -158,8 +137,7 @@ void ossl_statem_send_fatal(SSL_CONNECTION *s, int al) * into an error state and sends an alert if appropriate. * This is a permanent error for the current connection. */ -void ossl_statem_fatal(SSL_CONNECTION *s, int al, int reason, - const char *fmt, ...) +void ossl_statem_fatal(SSL *s, int al, int reason, const char *fmt, ...) { va_list args; @@ -189,7 +167,7 @@ void ossl_statem_fatal(SSL_CONNECTION *s, int al, int reason, * 1: Yes * 0: No */ -int ossl_statem_in_error(const SSL_CONNECTION *s) +int ossl_statem_in_error(const SSL *s) { if (s->statem.state == MSG_FLOW_ERROR) return 1; @@ -197,19 +175,17 @@ int ossl_statem_in_error(const SSL_CONNECTION *s) return 0; } -void ossl_statem_set_in_init(SSL_CONNECTION *s, int init) +void ossl_statem_set_in_init(SSL *s, int init) { s->statem.in_init = init; - if (s->rlayer.rrlmethod != NULL && s->rlayer.rrlmethod->set_in_init != NULL) - s->rlayer.rrlmethod->set_in_init(s->rlayer.rrl, init); } -int ossl_statem_get_in_handshake(SSL_CONNECTION *s) +int ossl_statem_get_in_handshake(SSL *s) { return s->statem.in_handshake; } -void ossl_statem_set_in_handshake(SSL_CONNECTION *s, int inhand) +void ossl_statem_set_in_handshake(SSL *s, int inhand) { if (inhand) s->statem.in_handshake++; @@ -218,7 +194,7 @@ void ossl_statem_set_in_handshake(SSL_CONNECTION *s, int inhand) } /* Are we in a sensible state to skip over unreadable early data? */ -int ossl_statem_skip_early_data(SSL_CONNECTION *s) +int ossl_statem_skip_early_data(SSL *s) { if (s->ext.early_data != SSL_EARLY_DATA_REJECTED) return 0; @@ -239,7 +215,7 @@ int ossl_statem_skip_early_data(SSL_CONNECTION *s) * attempting to read data (SSL_read*()), or -1 if we are in SSL_do_handshake() * or similar. */ -void ossl_statem_check_finish_init(SSL_CONNECTION *s, int sending) +void ossl_statem_check_finish_init(SSL *s, int sending) { if (sending == -1) { if (s->statem.hand_state == TLS_ST_PENDING_EARLY_DATA_END @@ -273,10 +249,10 @@ void ossl_statem_check_finish_init(SSL_CONNECTION *s, int sending) } } -void ossl_statem_set_hello_verify_done(SSL_CONNECTION *s) +void ossl_statem_set_hello_verify_done(SSL *s) { s->statem.state = MSG_FLOW_UNINITED; - ossl_statem_set_in_init(s, 1); + s->statem.in_init = 1; /* * This will get reset (briefly) back to TLS_ST_BEFORE when we enter * state_machine() because |state| is MSG_FLOW_UNINITED, but until then any @@ -289,34 +265,50 @@ void ossl_statem_set_hello_verify_done(SSL_CONNECTION *s) int ossl_statem_connect(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return -1; +#ifndef OPENSSL_NO_NTLS + int ret; - return state_machine(sc, 0); + if (s->enable_ntls == 1) { + ret = SSL_connection_is_ntls(s, 0); + if (ret == 0) + return state_machine(s, 0); + else if (ret == 1) + return state_machine_ntls(s, 0); + else + return -1; + } else +#endif + return state_machine(s, 0); } int ossl_statem_accept(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return -1; +#ifndef OPENSSL_NO_NTLS + int ret; - return state_machine(sc, 1); + if (s->enable_force_ntls == 1) + return state_machine_ntls(s, 1); + else if (s->enable_ntls == 1) { + ret = SSL_connection_is_ntls(s, 1); + if (ret == 0) + return state_machine(s, 1); + else if (ret == 1) + return state_machine_ntls(s, 1); + else + return ret; + } else +#endif + return state_machine(s, 1); } typedef void (*info_cb) (const SSL *, int, int); -static info_cb get_callback(SSL_CONNECTION *s) +static info_cb get_callback(SSL *s) { - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - if (s->info_callback != NULL) return s->info_callback; - else if (sctx->info_callback != NULL) - return sctx->info_callback; + else if (s->ctx->info_callback != NULL) + return s->ctx->info_callback; return NULL; } @@ -349,14 +341,13 @@ static info_cb get_callback(SSL_CONNECTION *s) * 1: Success * <=0: NBIO or error */ -static int state_machine(SSL_CONNECTION *s, int server) +static int state_machine(SSL *s, int server) { BUF_MEM *buf = NULL; void (*cb) (const SSL *ssl, int type, int val) = NULL; OSSL_STATEM *st = &s->statem; int ret = -1; int ssret; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if (st->state == MSG_FLOW_ERROR) { /* Shouldn't have been called if we're already in the error state */ @@ -369,21 +360,21 @@ static int state_machine(SSL_CONNECTION *s, int server) cb = get_callback(s); st->in_handshake++; - if (!SSL_in_init(ssl) || SSL_in_before(ssl)) { + if (!SSL_in_init(s) || SSL_in_before(s)) { /* * If we are stateless then we already called SSL_clear() - don't do * it again and clear the STATELESS flag itself. */ - if ((s->s3.flags & TLS1_FLAGS_STATELESS) == 0 && !SSL_clear(ssl)) + if ((s->s3.flags & TLS1_FLAGS_STATELESS) == 0 && !SSL_clear(s)) return -1; } #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(ssl))) { + if (SSL_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(s))) { /* * Notify SCTP BIO socket to enter handshake mode and prevent stream * identifier other than 0. */ - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE, st->in_handshake, NULL); } #endif @@ -398,8 +389,8 @@ static int state_machine(SSL_CONNECTION *s, int server) s->server = server; if (cb != NULL) { - if (SSL_IS_FIRST_HANDSHAKE(s) || !SSL_CONNECTION_IS_TLS13(s)) - cb(ssl, SSL_CB_HANDSHAKE_START, 1); + if (SSL_IS_FIRST_HANDSHAKE(s) || !SSL_IS_TLS13(s)) + cb(s, SSL_CB_HANDSHAKE_START, 1); } /* @@ -408,7 +399,7 @@ static int state_machine(SSL_CONNECTION *s, int server) * doomed to failure. */ - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { if ((s->version & 0xff00) != (DTLS1_VERSION & 0xff00) && (server || (s->version & 0xff00) != (DTLS1_BAD_VER & 0xff00))) { SSLfatal(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); @@ -439,6 +430,10 @@ static int state_machine(SSL_CONNECTION *s, int server) buf = NULL; } + if (!ssl3_setup_buffers(s)) { + SSLfatal(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); + goto end; + } s->init_num = 0; /* @@ -451,14 +446,14 @@ static int state_machine(SSL_CONNECTION *s, int server) * SCTP */ #ifndef OPENSSL_NO_SCTP - if (!SSL_CONNECTION_IS_DTLS(s) || !BIO_dgram_is_sctp(SSL_get_wbio(ssl))) + if (!SSL_IS_DTLS(s) || !BIO_dgram_is_sctp(SSL_get_wbio(s))) #endif if (!ssl_init_wbio_buffer(s)) { SSLfatal(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); goto end; } - if ((SSL_in_before(ssl)) + if ((SSL_in_before(s)) || s->renegotiate) { if (!tls_setup_handshake(s)) { /* SSLfatal() already called */ @@ -508,12 +503,12 @@ static int state_machine(SSL_CONNECTION *s, int server) st->in_handshake--; #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(ssl))) { + if (SSL_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(s))) { /* * Notify SCTP BIO socket to leave handshake mode and allow stream * identifier other than 0. */ - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE, st->in_handshake, NULL); } #endif @@ -521,9 +516,9 @@ static int state_machine(SSL_CONNECTION *s, int server) BUF_MEM_free(buf); if (cb != NULL) { if (server) - cb(ssl, SSL_CB_ACCEPT_EXIT, ret); + cb(s, SSL_CB_ACCEPT_EXIT, ret); else - cb(ssl, SSL_CB_CONNECT_EXIT, ret); + cb(s, SSL_CB_CONNECT_EXIT, ret); } return ret; } @@ -531,14 +526,14 @@ static int state_machine(SSL_CONNECTION *s, int server) /* * Initialise the MSG_FLOW_READING sub-state machine */ -static void init_read_state_machine(SSL_CONNECTION *s) +static void init_read_state_machine(SSL *s) { OSSL_STATEM *st = &s->statem; st->read_state = READ_STATE_HEADER; } -static int grow_init_buf(SSL_CONNECTION *s, size_t size) { +static int grow_init_buf(SSL *s, size_t size) { size_t msg_offset = (char *)s->init_msg - s->init_buf->data; @@ -579,18 +574,17 @@ static int grow_init_buf(SSL_CONNECTION *s, size_t size) { * control returns to the calling application. When this function is recalled we * will resume in the same state where we left off. */ -static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s) +static SUB_STATE_RETURN read_state_machine(SSL *s) { OSSL_STATEM *st = &s->statem; int ret, mt; size_t len = 0; - int (*transition) (SSL_CONNECTION *s, int mt); + int (*transition) (SSL *s, int mt); PACKET pkt; - MSG_PROCESS_RETURN(*process_message) (SSL_CONNECTION *s, PACKET *pkt); - WORK_STATE(*post_process_message) (SSL_CONNECTION *s, WORK_STATE wst); - size_t (*max_message_size) (SSL_CONNECTION *s); + MSG_PROCESS_RETURN(*process_message) (SSL *s, PACKET *pkt); + WORK_STATE(*post_process_message) (SSL *s, WORK_STATE wst); + size_t (*max_message_size) (SSL *s); void (*cb) (const SSL *ssl, int type, int val) = NULL; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); cb = get_callback(s); @@ -615,11 +609,15 @@ static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s) switch (st->read_state) { case READ_STATE_HEADER: /* Get the state the peer wants to move to */ - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* * In DTLS we get the whole message in one go - header and body */ ret = dtls_get_message(s, &mt); +#ifndef OPENSSL_NO_QUIC + } else if (SSL_IS_QUIC(s)) { + ret = quic_get_message(s, &mt, &len); +#endif } else { ret = tls_get_message_header(s, &mt); } @@ -632,9 +630,9 @@ static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s) if (cb != NULL) { /* Notify callback of an impending state change */ if (s->server) - cb(ssl, SSL_CB_ACCEPT_LOOP, 1); + cb(s, SSL_CB_ACCEPT_LOOP, 1); else - cb(ssl, SSL_CB_CONNECT_LOOP, 1); + cb(s, SSL_CB_CONNECT_LOOP, 1); } /* * Validate that we are allowed to move to the new state and move @@ -649,8 +647,8 @@ static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s) return SUB_STATE_ERROR; } - /* dtls_get_message already did this */ - if (!SSL_CONNECTION_IS_DTLS(s) + /* dtls_get_message/quic_get_message already did this */ + if (!SSL_IS_DTLS(s) && !SSL_IS_QUIC(s) && s->s3.tmp.message_size > 0 && !grow_init_buf(s, s->s3.tmp.message_size + SSL3_HM_HEADER_LENGTH)) { @@ -662,12 +660,15 @@ static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s) /* Fall through */ case READ_STATE_BODY: - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* * Actually we already have the body, but we give DTLS the * opportunity to do any further processing. */ ret = dtls_get_message_body(s, &len); + } else if (SSL_IS_QUIC(s)) { + /* We already got this above for QUIC */ + ret = 1; } else { ret = tls_get_message_body(s, &len); } @@ -692,7 +693,7 @@ static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s) return SUB_STATE_ERROR; case MSG_PROCESS_FINISHED_READING: - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { dtls1_stop_timer(s); } return SUB_STATE_FINISHED; @@ -724,7 +725,7 @@ static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s) break; case WORK_FINISHED_STOP: - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { dtls1_stop_timer(s); } return SUB_STATE_FINISHED; @@ -742,13 +743,13 @@ static SUB_STATE_RETURN read_state_machine(SSL_CONNECTION *s) /* * Send a previously constructed message to the peer. */ -static int statem_do_write(SSL_CONNECTION *s) +static int statem_do_write(SSL *s) { OSSL_STATEM *st = &s->statem; if (st->hand_state == TLS_ST_CW_CHANGE || st->hand_state == TLS_ST_SW_CHANGE) { - if (SSL_CONNECTION_IS_DTLS(s)) + if (SSL_IS_DTLS(s)) return dtls1_do_write(s, SSL3_RT_CHANGE_CIPHER_SPEC); else return ssl3_do_write(s, SSL3_RT_CHANGE_CIPHER_SPEC); @@ -760,7 +761,7 @@ static int statem_do_write(SSL_CONNECTION *s) /* * Initialise the MSG_FLOW_WRITING sub-state machine */ -static void init_write_state_machine(SSL_CONNECTION *s) +static void init_write_state_machine(SSL *s) { OSSL_STATEM *st = &s->statem; @@ -798,22 +799,20 @@ static void init_write_state_machine(SSL_CONNECTION *s) * message has been completed. As for WRITE_STATE_PRE_WORK this could also * result in an NBIO event. */ -static SUB_STATE_RETURN write_state_machine(SSL_CONNECTION *s) +static SUB_STATE_RETURN write_state_machine(SSL *s) { OSSL_STATEM *st = &s->statem; int ret; - WRITE_TRAN(*transition) (SSL_CONNECTION *s); - WORK_STATE(*pre_work) (SSL_CONNECTION *s, WORK_STATE wst); - WORK_STATE(*post_work) (SSL_CONNECTION *s, WORK_STATE wst); - int (*get_construct_message_f) (SSL_CONNECTION *s, - CON_FUNC_RETURN (**confunc) (SSL_CONNECTION *s, - WPACKET *pkt), + WRITE_TRAN(*transition) (SSL *s); + WORK_STATE(*pre_work) (SSL *s, WORK_STATE wst); + WORK_STATE(*post_work) (SSL *s, WORK_STATE wst); + int (*get_construct_message_f) (SSL *s, WPACKET *pkt, + int (**confunc) (SSL *s, WPACKET *pkt), int *mt); void (*cb) (const SSL *ssl, int type, int val) = NULL; - CON_FUNC_RETURN (*confunc) (SSL_CONNECTION *s, WPACKET *pkt); + int (*confunc) (SSL *s, WPACKET *pkt); int mt; WPACKET pkt; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); cb = get_callback(s); @@ -835,9 +834,9 @@ static SUB_STATE_RETURN write_state_machine(SSL_CONNECTION *s) if (cb != NULL) { /* Notify callback of an impending state change */ if (s->server) - cb(ssl, SSL_CB_ACCEPT_LOOP, 1); + cb(s, SSL_CB_ACCEPT_LOOP, 1); else - cb(ssl, SSL_CB_CONNECT_LOOP, 1); + cb(s, SSL_CB_CONNECT_LOOP, 1); } switch (transition(s)) { case WRITE_TRAN_CONTINUE: @@ -872,7 +871,7 @@ static SUB_STATE_RETURN write_state_machine(SSL_CONNECTION *s) case WORK_FINISHED_STOP: return SUB_STATE_END_HANDSHAKE; } - if (!get_construct_message_f(s, &confunc, &mt)) { + if (!get_construct_message_f(s, &pkt, &confunc, &mt)) { /* SSLfatal() already called */ return SUB_STATE_ERROR; } @@ -888,24 +887,10 @@ static SUB_STATE_RETURN write_state_machine(SSL_CONNECTION *s) SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return SUB_STATE_ERROR; } - if (confunc != NULL) { - CON_FUNC_RETURN tmpret; - - tmpret = confunc(s, &pkt); - if (tmpret == CON_FUNC_ERROR) { - WPACKET_cleanup(&pkt); - check_fatal(s); - return SUB_STATE_ERROR; - } else if (tmpret == CON_FUNC_DONT_SEND) { - /* - * The construction function decided not to construct the - * message after all and continue. Skip sending. - */ - WPACKET_cleanup(&pkt); - st->write_state = WRITE_STATE_POST_WORK; - st->write_state_work = WORK_MORE_A; - break; - } /* else success */ + if (confunc != NULL && !confunc(s, &pkt)) { + WPACKET_cleanup(&pkt); + check_fatal(s); + return SUB_STATE_ERROR; } if (!ssl_close_construct_packet(s, &pkt, mt) || !WPACKET_finish(&pkt)) { @@ -917,7 +902,7 @@ static SUB_STATE_RETURN write_state_machine(SSL_CONNECTION *s) /* Fall through */ case WRITE_STATE_SEND: - if (SSL_CONNECTION_IS_DTLS(s) && st->use_timer) { + if (SSL_IS_DTLS(s) && st->use_timer) { dtls1_start_timer(s); } ret = statem_do_write(s); @@ -957,9 +942,18 @@ static SUB_STATE_RETURN write_state_machine(SSL_CONNECTION *s) /* * Flush the write BIO */ -int statem_flush(SSL_CONNECTION *s) +int statem_flush(SSL *s) { s->rwstate = SSL_WRITING; +#ifndef OPENSSL_NO_QUIC + if (SSL_IS_QUIC(s)) { + if (!s->quic_method->flush_flight(s)) { + /* NOTE: BIO_flush() does not generate an error */ + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + return 0; + } + } else +#endif if (BIO_flush(s->wbio) <= 0) { return 0; } @@ -976,7 +970,7 @@ int statem_flush(SSL_CONNECTION *s) * 1: Yes (application data allowed) * 0: No (application data not allowed) */ -int ossl_statem_app_data_allowed(SSL_CONNECTION *s) +int ossl_statem_app_data_allowed(SSL *s) { OSSL_STATEM *st = &s->statem; @@ -1010,7 +1004,7 @@ int ossl_statem_app_data_allowed(SSL_CONNECTION *s) * This function returns 1 if TLS exporter is ready to export keying * material, or 0 if otherwise. */ -int ossl_statem_export_allowed(SSL_CONNECTION *s) +int ossl_statem_export_allowed(SSL *s) { return s->s3.previous_server_finished_len != 0 && s->statem.hand_state != TLS_ST_SW_FINISHED; @@ -1020,7 +1014,7 @@ int ossl_statem_export_allowed(SSL_CONNECTION *s) * Return 1 if early TLS exporter is ready to export keying material, * or 0 if otherwise. */ -int ossl_statem_export_early_allowed(SSL_CONNECTION *s) +int ossl_statem_export_early_allowed(SSL *s) { /* * The early exporter secret is only present on the server if we diff --git a/openssl/include/internal/statem.h b/openssl/src/ssl/statem/statem.h similarity index 67% rename from openssl/include/internal/statem.h rename to openssl/src/ssl/statem/statem.h index 136e65236..5db31b635 100644 --- a/openssl/include/internal/statem.h +++ b/openssl/src/ssl/statem/statem.h @@ -1,13 +1,11 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at * https://www.openssl.org/source/license.html */ -#ifndef OSSL_INTERNAL_STATEM_H -# define OSSL_INTERNAL_STATEM_H /***************************************************************************** * * @@ -74,18 +72,20 @@ typedef enum { } WRITE_STATE; typedef enum { - CON_FUNC_ERROR = 0, - CON_FUNC_SUCCESS, - CON_FUNC_DONT_SEND -} CON_FUNC_RETURN; + /* The enc_write_ctx can be used normally */ + ENC_WRITE_STATE_VALID, + /* The enc_write_ctx cannot be used */ + ENC_WRITE_STATE_INVALID, + /* Write alerts in plaintext, but otherwise use the enc_write_ctx */ + ENC_WRITE_STATE_WRITE_PLAIN_ALERTS +} ENC_WRITE_STATES; -typedef int (*ossl_statem_mutate_handshake_cb)(const unsigned char *msgin, - size_t inlen, - unsigned char **msgout, - size_t *outlen, - void *arg); - -typedef void (*ossl_statem_finish_mutate_handshake_cb)(void *arg); +typedef enum { + /* The enc_read_ctx can be used normally */ + ENC_READ_STATE_VALID, + /* We may receive encrypted or plaintext alerts */ + ENC_READ_STATE_ALLOW_PLAIN_ALERTS +} ENC_READ_STATES; /***************************************************************************** * * @@ -116,12 +116,8 @@ struct ossl_statem_st { /* Should we skip the CertificateVerify message? */ unsigned int no_cert_verify; int use_timer; - - /* Test harness message mutator callbacks */ - ossl_statem_mutate_handshake_cb mutate_handshake_cb; - ossl_statem_finish_mutate_handshake_cb finish_mutate_handshake_cb; - void *mutatearg; - unsigned int write_in_progress : 1; + ENC_WRITE_STATES enc_write_state; + ENC_READ_STATES enc_read_state; }; typedef struct ossl_statem_st OSSL_STATEM; @@ -132,16 +128,13 @@ typedef struct ossl_statem_st OSSL_STATEM; * * *****************************************************************************/ -typedef struct ssl_connection_st SSL_CONNECTION; - __owur int ossl_statem_accept(SSL *s); __owur int ossl_statem_connect(SSL *s); -OSSL_HANDSHAKE_STATE ossl_statem_get_state(SSL_CONNECTION *s); -void ossl_statem_clear(SSL_CONNECTION *s); -void ossl_statem_set_renegotiate(SSL_CONNECTION *s); -void ossl_statem_send_fatal(SSL_CONNECTION *s, int al); -void ossl_statem_fatal(SSL_CONNECTION *s, int al, int reason, - const char *fmt, ...); +void ossl_statem_clear(SSL *s); +void ossl_statem_set_renegotiate(SSL *s); +void ossl_statem_send_fatal(SSL *s, int al); +void ossl_statem_fatal(SSL *s, int al, int reason, const char *fmt, ...); +# define SSL_AD_NO_ALERT -1 # define SSLfatal_alert(s, al) ossl_statem_send_fatal((s), (al)) # define SSLfatal(s, al, r) SSLfatal_data((s), (al), (r), NULL) # define SSLfatal_data \ @@ -149,23 +142,16 @@ void ossl_statem_fatal(SSL_CONNECTION *s, int al, int reason, ERR_set_debug(OPENSSL_FILE, OPENSSL_LINE, OPENSSL_FUNC), \ ossl_statem_fatal) -int ossl_statem_in_error(const SSL_CONNECTION *s); -void ossl_statem_set_in_init(SSL_CONNECTION *s, int init); -int ossl_statem_get_in_handshake(SSL_CONNECTION *s); -void ossl_statem_set_in_handshake(SSL_CONNECTION *s, int inhand); -__owur int ossl_statem_skip_early_data(SSL_CONNECTION *s); -void ossl_statem_check_finish_init(SSL_CONNECTION *s, int send); -void ossl_statem_set_hello_verify_done(SSL_CONNECTION *s); -__owur int ossl_statem_app_data_allowed(SSL_CONNECTION *s); -__owur int ossl_statem_export_allowed(SSL_CONNECTION *s); -__owur int ossl_statem_export_early_allowed(SSL_CONNECTION *s); +int ossl_statem_in_error(const SSL *s); +void ossl_statem_set_in_init(SSL *s, int init); +int ossl_statem_get_in_handshake(SSL *s); +void ossl_statem_set_in_handshake(SSL *s, int inhand); +__owur int ossl_statem_skip_early_data(SSL *s); +void ossl_statem_check_finish_init(SSL *s, int send); +void ossl_statem_set_hello_verify_done(SSL *s); +__owur int ossl_statem_app_data_allowed(SSL *s); +__owur int ossl_statem_export_allowed(SSL *s); +__owur int ossl_statem_export_early_allowed(SSL *s); /* Flush the write BIO */ -int statem_flush(SSL_CONNECTION *s); - -int ossl_statem_set_mutator(SSL *s, - ossl_statem_mutate_handshake_cb mutate_handshake_cb, - ossl_statem_finish_mutate_handshake_cb finish_mutate_handshake_cb, - void *mutatearg); - -#endif +int statem_flush(SSL *s); diff --git a/openssl/src/ssl/statem/statem_clnt.c b/openssl/src/ssl/statem/statem_clnt.c index 7d8b14037..689a75e72 100644 --- a/openssl/src/ssl/statem/statem_clnt.c +++ b/openssl/src/ssl/statem/statem_clnt.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -23,26 +23,21 @@ #include #include #include +#include #include #include #include +#include #include "internal/cryptlib.h" -static MSG_PROCESS_RETURN tls_process_as_hello_retry_request(SSL_CONNECTION *s, - PACKET *pkt); -static MSG_PROCESS_RETURN tls_process_encrypted_extensions(SSL_CONNECTION *s, - PACKET *pkt); +static MSG_PROCESS_RETURN tls_process_as_hello_retry_request(SSL *s, PACKET *pkt); +static MSG_PROCESS_RETURN tls_process_encrypted_extensions(SSL *s, PACKET *pkt); -static ossl_inline int cert_req_allowed(SSL_CONNECTION *s); -static int key_exchange_expected(SSL_CONNECTION *s); -static int ssl_cipher_list_to_bytes(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *sk, +static ossl_inline int cert_req_allowed(SSL *s); +static int key_exchange_expected(SSL *s); +static int ssl_cipher_list_to_bytes(SSL *s, STACK_OF(SSL_CIPHER) *sk, WPACKET *pkt); -static ossl_inline int received_server_cert(SSL_CONNECTION *sc) -{ - return sc->session->peer_rpk != NULL || sc->session->peer != NULL; -} - /* * Is a CertificateRequest message allowed at the moment or not? * @@ -50,7 +45,7 @@ static ossl_inline int received_server_cert(SSL_CONNECTION *sc) * 1: Yes * 0: No */ -static ossl_inline int cert_req_allowed(SSL_CONNECTION *s) +static ossl_inline int cert_req_allowed(SSL *s) { /* TLS does not like anon-DH with client cert */ if ((s->version > SSL3_VERSION @@ -68,7 +63,7 @@ static ossl_inline int cert_req_allowed(SSL_CONNECTION *s) * 1: Yes * 0: No */ -static int key_exchange_expected(SSL_CONNECTION *s) +static int key_exchange_expected(SSL *s) { long alg_k = s->s3.tmp.new_cipher->algorithm_mkey; @@ -93,7 +88,7 @@ static int key_exchange_expected(SSL_CONNECTION *s) * Return values are 1 for success (transition allowed) and 0 on error * (transition not allowed) */ -static int ossl_statem_client13_read_transition(SSL_CONNECTION *s, int mt) +static int ossl_statem_client13_read_transition(SSL *s, int mt) { OSSL_STATEM *st = &s->statem; @@ -136,36 +131,29 @@ static int ossl_statem_client13_read_transition(SSL_CONNECTION *s, int mt) st->hand_state = TLS_ST_CR_CERT_REQ; return 1; } - if (mt == SSL3_MT_CERTIFICATE) { + if (mt == SSL3_MT_CERTIFICATE +#ifndef OPENSSL_NO_CERT_COMPRESSION + || mt == SSL3_MT_COMPRESSED_CERTIFICATE +#endif + ) { st->hand_state = TLS_ST_CR_CERT; return 1; } -#ifndef OPENSSL_NO_COMP_ALG - if (mt == SSL3_MT_COMPRESSED_CERTIFICATE - && s->ext.compress_certificate_sent) { - st->hand_state = TLS_ST_CR_COMP_CERT; - return 1; - } -#endif } break; case TLS_ST_CR_CERT_REQ: - if (mt == SSL3_MT_CERTIFICATE) { + if (mt == SSL3_MT_CERTIFICATE +#ifndef OPENSSL_NO_CERT_COMPRESSION + || mt == SSL3_MT_COMPRESSED_CERTIFICATE +#endif + ) { st->hand_state = TLS_ST_CR_CERT; return 1; } -#ifndef OPENSSL_NO_COMP_ALG - if (mt == SSL3_MT_COMPRESSED_CERTIFICATE - && s->ext.compress_certificate_sent) { - st->hand_state = TLS_ST_CR_COMP_CERT; - return 1; - } -#endif break; case TLS_ST_CR_CERT: - case TLS_ST_CR_COMP_CERT: if (mt == SSL3_MT_CERTIFICATE_VERIFY) { st->hand_state = TLS_ST_CR_CERT_VRFY; return 1; @@ -184,7 +172,7 @@ static int ossl_statem_client13_read_transition(SSL_CONNECTION *s, int mt) st->hand_state = TLS_ST_CR_SESSION_TICKET; return 1; } - if (mt == SSL3_MT_KEY_UPDATE && !SSL_IS_QUIC_HANDSHAKE(s)) { + if (mt == SSL3_MT_KEY_UPDATE) { st->hand_state = TLS_ST_CR_KEY_UPDATE; return 1; } @@ -193,8 +181,7 @@ static int ossl_statem_client13_read_transition(SSL_CONNECTION *s, int mt) /* Restore digest for PHA before adding message.*/ # error Internal DTLS version error #endif - if (!SSL_CONNECTION_IS_DTLS(s) - && s->post_handshake_auth == SSL_PHA_EXT_SENT) { + if (!SSL_IS_DTLS(s) && s->post_handshake_auth == SSL_PHA_EXT_SENT) { s->post_handshake_auth = SSL_PHA_REQUESTED; /* * In TLS, this is called before the message is added to the @@ -226,7 +213,7 @@ static int ossl_statem_client13_read_transition(SSL_CONNECTION *s, int mt) * Return values are 1 for success (transition allowed) and 0 on error * (transition not allowed) */ -int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) +int ossl_statem_client_read_transition(SSL *s, int mt) { OSSL_STATEM *st = &s->statem; int ske_expected; @@ -235,7 +222,7 @@ int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) * Note that after writing the first ClientHello we don't know what version * we are going to negotiate yet, so we don't take this branch until later. */ - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { if (!ossl_statem_client13_read_transition(s, mt)) goto err; return 1; @@ -251,7 +238,7 @@ int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) return 1; } - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { if (mt == DTLS1_MT_HELLO_VERIFY_REQUEST) { st->hand_state = DTLS_ST_CR_HELLO_VERIFY_REQUEST; return 1; @@ -283,8 +270,7 @@ int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) return 1; } } else { - if (SSL_CONNECTION_IS_DTLS(s) - && mt == DTLS1_MT_HELLO_VERIFY_REQUEST) { + if (SSL_IS_DTLS(s) && mt == DTLS1_MT_HELLO_VERIFY_REQUEST) { st->hand_state = DTLS_ST_CR_HELLO_VERIFY_REQUEST; return 1; } else if (s->version >= TLS1_VERSION @@ -302,7 +288,11 @@ int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) return 1; } else if (!(s->s3.tmp.new_cipher->algorithm_auth & (SSL_aNULL | SSL_aSRP | SSL_aPSK))) { - if (mt == SSL3_MT_CERTIFICATE) { + if (mt == SSL3_MT_CERTIFICATE +#ifndef OPENSSL_NO_CERT_COMPRESSION + || mt == SSL3_MT_COMPRESSED_CERTIFICATE +#endif + ) { st->hand_state = TLS_ST_CR_CERT; return 1; } @@ -329,7 +319,6 @@ int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) break; case TLS_ST_CR_CERT: - case TLS_ST_CR_COMP_CERT: /* * The CertificateStatus message is optional even if * |ext.status_expected| is set @@ -406,7 +395,7 @@ int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) err: /* No valid transition found */ - if (SSL_CONNECTION_IS_DTLS(s) && mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + if (SSL_IS_DTLS(s) && mt == SSL3_MT_CHANGE_CIPHER_SPEC) { BIO *rbio; /* @@ -415,7 +404,7 @@ int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) */ s->init_num = 0; s->rwstate = SSL_READING; - rbio = SSL_get_rbio(SSL_CONNECTION_GET_SSL(s)); + rbio = SSL_get_rbio(s); BIO_clear_retry_flags(rbio); BIO_set_retry_read(rbio); return 0; @@ -424,19 +413,12 @@ int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt) return 0; } -static int do_compressed_cert(SSL_CONNECTION *sc) -{ - /* If we negotiated RPK, we won't try to compress it */ - return sc->ext.client_cert_type == TLSEXT_cert_type_x509 - && sc->ext.compress_certificate_from_peer[0] != TLSEXT_comp_cert_none; -} - /* * ossl_statem_client13_write_transition() works out what handshake state to * move to next when the TLSv1.3 client is writing messages to be sent to the * server. */ -static WRITE_TRAN ossl_statem_client13_write_transition(SSL_CONNECTION *s) +static WRITE_TRAN ossl_statem_client13_write_transition(SSL *s) { OSSL_STATEM *st = &s->statem; @@ -453,10 +435,7 @@ static WRITE_TRAN ossl_statem_client13_write_transition(SSL_CONNECTION *s) case TLS_ST_CR_CERT_REQ: if (s->post_handshake_auth == SSL_PHA_REQUESTED) { - if (do_compressed_cert(s)) - st->hand_state = TLS_ST_CW_COMP_CERT; - else - st->hand_state = TLS_ST_CW_CERT; + st->hand_state = TLS_ST_CW_CERT; return WRITE_TRAN_CONTINUE; } /* @@ -478,14 +457,9 @@ static WRITE_TRAN ossl_statem_client13_write_transition(SSL_CONNECTION *s) else if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0 && s->hello_retry_request == SSL_HRR_NONE) st->hand_state = TLS_ST_CW_CHANGE; - else if (s->s3.tmp.cert_req == 0) - st->hand_state = TLS_ST_CW_FINISHED; - else if (do_compressed_cert(s)) - st->hand_state = TLS_ST_CW_COMP_CERT; else - st->hand_state = TLS_ST_CW_CERT; - - s->ts_msg_read = ossl_time_now(); + st->hand_state = (s->s3.tmp.cert_req != 0) ? TLS_ST_CW_CERT + : TLS_ST_CW_FINISHED; return WRITE_TRAN_CONTINUE; case TLS_ST_PENDING_EARLY_DATA_END: @@ -497,15 +471,10 @@ static WRITE_TRAN ossl_statem_client13_write_transition(SSL_CONNECTION *s) case TLS_ST_CW_END_OF_EARLY_DATA: case TLS_ST_CW_CHANGE: - if (s->s3.tmp.cert_req == 0) - st->hand_state = TLS_ST_CW_FINISHED; - else if (do_compressed_cert(s)) - st->hand_state = TLS_ST_CW_COMP_CERT; - else - st->hand_state = TLS_ST_CW_CERT; + st->hand_state = (s->s3.tmp.cert_req != 0) ? TLS_ST_CW_CERT + : TLS_ST_CW_FINISHED; return WRITE_TRAN_CONTINUE; - case TLS_ST_CW_COMP_CERT: case TLS_ST_CW_CERT: /* If a non-empty Certificate we also send CertificateVerify */ st->hand_state = (s->s3.tmp.cert_req == 1) ? TLS_ST_CW_CERT_VRFY @@ -538,7 +507,7 @@ static WRITE_TRAN ossl_statem_client13_write_transition(SSL_CONNECTION *s) * ossl_statem_client_write_transition() works out what handshake state to * move to next when the client is writing messages to be sent to the server. */ -WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s) +WRITE_TRAN ossl_statem_client_write_transition(SSL *s) { OSSL_STATEM *st = &s->statem; @@ -547,7 +516,7 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s) * version we are going to negotiate yet, so we don't take this branch until * later */ - if (SSL_CONNECTION_IS_TLS13(s)) + if (SSL_IS_TLS13(s)) return ossl_statem_client13_write_transition(s); switch (st->hand_state) { @@ -586,7 +555,6 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s) * No transition at the end of writing because we don't know what * we will be sent */ - s->ts_msg_write = ossl_time_now(); return WRITE_TRAN_FINISHED; case TLS_ST_CR_SRVR_HELLO: @@ -603,7 +571,6 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s) return WRITE_TRAN_CONTINUE; case TLS_ST_EARLY_DATA: - s->ts_msg_write = ossl_time_now(); return WRITE_TRAN_FINISHED; case DTLS_ST_CR_HELLO_VERIFY_REQUEST: @@ -611,7 +578,6 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s) return WRITE_TRAN_CONTINUE; case TLS_ST_CR_SRVR_DONE: - s->ts_msg_read = ossl_time_now(); if (s->s3.tmp.cert_req) st->hand_state = TLS_ST_CW_CERT; else @@ -656,7 +622,7 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s) #if defined(OPENSSL_NO_NEXTPROTONEG) st->hand_state = TLS_ST_CW_FINISHED; #else - if (!SSL_CONNECTION_IS_DTLS(s) && s->s3.npn_seen) + if (!SSL_IS_DTLS(s) && s->s3.npn_seen) st->hand_state = TLS_ST_CW_NEXT_PROTO; else st->hand_state = TLS_ST_CW_FINISHED; @@ -692,7 +658,7 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s) * If we can renegotiate now then do so, otherwise wait for a more * convenient time. */ - if (ssl3_renegotiate_check(SSL_CONNECTION_GET_SSL(s), 1)) { + if (ssl3_renegotiate_check(s, 1)) { if (!tls_setup_handshake(s)) { /* SSLfatal() already called */ return WRITE_TRAN_ERROR; @@ -709,7 +675,7 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s) * Perform any pre work that needs to be done prior to sending a message from * the client to the server. */ -WORK_STATE ossl_statem_client_pre_work(SSL_CONNECTION *s, WORK_STATE wst) +WORK_STATE ossl_statem_client_pre_work(SSL *s, WORK_STATE wst) { OSSL_STATEM *st = &s->statem; @@ -720,34 +686,17 @@ WORK_STATE ossl_statem_client_pre_work(SSL_CONNECTION *s, WORK_STATE wst) case TLS_ST_CW_CLNT_HELLO: s->shutdown = 0; - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* every DTLS ClientHello resets Finished MAC */ if (!ssl3_init_finished_mac(s)) { /* SSLfatal() already called */ return WORK_ERROR; } - } else if (s->ext.early_data == SSL_EARLY_DATA_REJECTED) { - /* - * This must be a second ClientHello after an HRR following an - * earlier rejected attempt to send early data. Since we were - * previously encrypting the early data we now need to reset the - * write record layer in order to write in plaintext again. - */ - if (!ssl_set_new_record_layer(s, - TLS_ANY_VERSION, - OSSL_RECORD_DIRECTION_WRITE, - OSSL_RECORD_PROTECTION_LEVEL_NONE, - NULL, 0, NULL, 0, NULL, 0, NULL, 0, - NULL, 0, NID_undef, NULL, NULL, - NULL)) { - /* SSLfatal already called */ - return WORK_ERROR; - } } break; case TLS_ST_CW_CHANGE: - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { if (s->hit) { /* * We're into the last flight so we don't retransmit these @@ -756,7 +705,7 @@ WORK_STATE ossl_statem_client_pre_work(SSL_CONNECTION *s, WORK_STATE wst) st->use_timer = 0; } #ifndef OPENSSL_NO_SCTP - if (BIO_dgram_is_sctp(SSL_get_wbio(SSL_CONNECTION_GET_SSL(s)))) { + if (BIO_dgram_is_sctp(SSL_get_wbio(s))) { /* Calls SSLfatal() as required */ return dtls_wait_for_dry(s); } @@ -790,10 +739,9 @@ WORK_STATE ossl_statem_client_pre_work(SSL_CONNECTION *s, WORK_STATE wst) * Perform any work that needs to be done after sending a message from the * client to the server. */ -WORK_STATE ossl_statem_client_post_work(SSL_CONNECTION *s, WORK_STATE wst) +WORK_STATE ossl_statem_client_post_work(SSL *s, WORK_STATE wst) { OSSL_STATEM *st = &s->statem; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); s->init_num = 0; @@ -822,12 +770,21 @@ WORK_STATE ossl_statem_client_post_work(SSL_CONNECTION *s, WORK_STATE wst) return WORK_MORE_A; } - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* Treat the next message as the first packet */ s->first_packet = 1; } break; + case TLS_ST_CW_END_OF_EARLY_DATA: + /* + * We set the enc_write_ctx back to NULL because we may end up writing + * in cleartext again if we get a HelloRetryRequest from the server. + */ + EVP_CIPHER_CTX_free(s->enc_write_ctx); + s->enc_write_ctx = NULL; + break; + case TLS_ST_CW_KEY_EXCH: if (tls_client_key_exchange_post_work(s) == 0) { /* SSLfatal() already called */ @@ -836,8 +793,7 @@ WORK_STATE ossl_statem_client_post_work(SSL_CONNECTION *s, WORK_STATE wst) break; case TLS_ST_CW_CHANGE: - if (SSL_CONNECTION_IS_TLS13(s) - || s->hello_retry_request == SSL_HRR_PENDING) + if (SSL_IS_TLS13(s) || s->hello_retry_request == SSL_HRR_PENDING) break; if (s->early_data_state == SSL_EARLY_DATA_CONNECTING && s->max_early_data > 0) { @@ -860,50 +816,54 @@ WORK_STATE ossl_statem_client_post_work(SSL_CONNECTION *s, WORK_STATE wst) else s->session->compress_meth = s->s3.tmp.new_compression->id; #endif - if (!ssl->method->ssl3_enc->setup_key_block(s)) { + if (!s->method->ssl3_enc->setup_key_block(s)) { /* SSLfatal() already called */ return WORK_ERROR; } - if (!ssl->method->ssl3_enc->change_cipher_state(s, + if (!s->method->ssl3_enc->change_cipher_state(s, SSL3_CHANGE_CIPHER_CLIENT_WRITE)) { /* SSLfatal() already called */ return WORK_ERROR; } + if (SSL_IS_DTLS(s)) { #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s) && s->hit) { - /* - * Change to new shared key of SCTP-Auth, will be ignored if - * no SCTP used. - */ - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY, - 0, NULL); - } + if (s->hit) { + /* + * Change to new shared key of SCTP-Auth, will be ignored if + * no SCTP used. + */ + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY, + 0, NULL); + } #endif + + dtls1_reset_seq_numbers(s, SSL3_CC_WRITE); + } break; case TLS_ST_CW_FINISHED: #ifndef OPENSSL_NO_SCTP - if (wst == WORK_MORE_A && SSL_CONNECTION_IS_DTLS(s) && s->hit == 0) { + if (wst == WORK_MORE_A && SSL_IS_DTLS(s) && s->hit == 0) { /* * Change to new shared key of SCTP-Auth, will be ignored if * no SCTP used. */ - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY, 0, NULL); } #endif if (statem_flush(s) != 1) return WORK_MORE_B; - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { if (!tls13_save_handshake_digest_for_pha(s)) { /* SSLfatal() already called */ return WORK_ERROR; } if (s->post_handshake_auth != SSL_PHA_REQUESTED) { - if (!ssl->method->ssl3_enc->change_cipher_state(s, + if (!s->method->ssl3_enc->change_cipher_state(s, SSL3_CC_APPLICATION | SSL3_CHANGE_CIPHER_CLIENT_WRITE)) { /* SSLfatal() already called */ return WORK_ERROR; @@ -933,7 +893,7 @@ WORK_STATE ossl_statem_client_post_work(SSL_CONNECTION *s, WORK_STATE wst) * 1: Success * 0: Error */ -int ossl_statem_client_construct_message(SSL_CONNECTION *s, +int ossl_statem_client_construct_message(SSL *s, WPACKET *pkt, confunc_f *confunc, int *mt) { OSSL_STATEM *st = &s->statem; @@ -945,7 +905,7 @@ int ossl_statem_client_construct_message(SSL_CONNECTION *s, return 0; case TLS_ST_CW_CHANGE: - if (SSL_CONNECTION_IS_DTLS(s)) + if (SSL_IS_DTLS(s)) *confunc = dtls_construct_change_cipher_spec; else *confunc = tls_construct_change_cipher_spec; @@ -958,6 +918,14 @@ int ossl_statem_client_construct_message(SSL_CONNECTION *s, break; case TLS_ST_CW_END_OF_EARLY_DATA: +#ifndef OPENSSL_NO_QUIC + /* QUIC does not send EndOfEarlyData, draft-ietf-quic-tls-24 S8.3 */ + if (s->quic_method != NULL) { + *confunc = NULL; + *mt = SSL3_MT_DUMMY; + break; + } +#endif *confunc = tls_construct_end_of_early_data; *mt = SSL3_MT_END_OF_EARLY_DATA; break; @@ -968,16 +936,18 @@ int ossl_statem_client_construct_message(SSL_CONNECTION *s, break; case TLS_ST_CW_CERT: +#ifndef OPENSSL_NO_CERT_COMPRESSION + if (s->cert_comp_compress_id) { + *confunc = tls_construct_client_compressed_certificate; + *mt = SSL3_MT_COMPRESSED_CERTIFICATE; + } else { +#endif *confunc = tls_construct_client_certificate; *mt = SSL3_MT_CERTIFICATE; - break; - -#ifndef OPENSSL_NO_COMP_ALG - case TLS_ST_CW_COMP_CERT: - *confunc = tls_construct_client_compressed_certificate; - *mt = SSL3_MT_COMPRESSED_CERTIFICATE; - break; +#ifndef OPENSSL_NO_CERT_COMPRESSION + } #endif + break; case TLS_ST_CW_KEY_EXCH: *confunc = tls_construct_client_key_exchange; @@ -1013,7 +983,7 @@ int ossl_statem_client_construct_message(SSL_CONNECTION *s, * Returns the maximum allowed length for the current message that we are * reading. Excludes the message header. */ -size_t ossl_statem_client_max_message_size(SSL_CONNECTION *s) +size_t ossl_statem_client_max_message_size(SSL *s) { OSSL_STATEM *st = &s->statem; @@ -1028,12 +998,11 @@ size_t ossl_statem_client_max_message_size(SSL_CONNECTION *s) case DTLS_ST_CR_HELLO_VERIFY_REQUEST: return HELLO_VERIFY_REQUEST_MAX_LENGTH; - case TLS_ST_CR_COMP_CERT: case TLS_ST_CR_CERT: return s->max_cert_list; case TLS_ST_CR_CERT_VRFY: - return CERTIFICATE_VERIFY_MAX_LENGTH; + return SSL3_RT_MAX_PLAIN_LENGTH; case TLS_ST_CR_CERT_STATUS: return SSL3_RT_MAX_PLAIN_LENGTH; @@ -1058,8 +1027,8 @@ size_t ossl_statem_client_max_message_size(SSL_CONNECTION *s) return CCS_MAX_LENGTH; case TLS_ST_CR_SESSION_TICKET: - return (SSL_CONNECTION_IS_TLS13(s)) ? SESSION_TICKET_MAX_LENGTH_TLS13 - : SESSION_TICKET_MAX_LENGTH_TLS12; + return (SSL_IS_TLS13(s)) ? SESSION_TICKET_MAX_LENGTH_TLS13 + : SESSION_TICKET_MAX_LENGTH_TLS12; case TLS_ST_CR_FINISHED: return FINISHED_MAX_LENGTH; @@ -1075,8 +1044,7 @@ size_t ossl_statem_client_max_message_size(SSL_CONNECTION *s) /* * Process a message that the client has received from the server. */ -MSG_PROCESS_RETURN ossl_statem_client_process_message(SSL_CONNECTION *s, - PACKET *pkt) +MSG_PROCESS_RETURN ossl_statem_client_process_message(SSL *s, PACKET *pkt) { OSSL_STATEM *st = &s->statem; @@ -1093,12 +1061,12 @@ MSG_PROCESS_RETURN ossl_statem_client_process_message(SSL_CONNECTION *s, return dtls_process_hello_verify(s, pkt); case TLS_ST_CR_CERT: - return tls_process_server_certificate(s, pkt); - -#ifndef OPENSSL_NO_COMP_ALG - case TLS_ST_CR_COMP_CERT: - return tls_process_server_compressed_certificate(s, pkt); +#ifndef OPENSSL_NO_CERT_COMPRESSION + if (s->s3.tmp.message_type == SSL3_MT_COMPRESSED_CERTIFICATE) + return tls_process_server_compressed_certificate(s, pkt); + else #endif + return tls_process_server_certificate(s, pkt); case TLS_ST_CR_CERT_VRFY: return tls_process_cert_verify(s, pkt); @@ -1139,8 +1107,7 @@ MSG_PROCESS_RETURN ossl_statem_client_process_message(SSL_CONNECTION *s, * Perform any further processing required following the receipt of a message * from the server */ -WORK_STATE ossl_statem_client_post_process_message(SSL_CONNECTION *s, - WORK_STATE wst) +WORK_STATE ossl_statem_client_post_process_message(SSL *s, WORK_STATE wst) { OSSL_STATEM *st = &s->statem; @@ -1151,7 +1118,6 @@ WORK_STATE ossl_statem_client_post_process_message(SSL_CONNECTION *s, return WORK_ERROR; case TLS_ST_CR_CERT: - case TLS_ST_CR_COMP_CERT: return tls_post_process_server_certificate(s, wst); case TLS_ST_CR_CERT_VRFY: @@ -1160,7 +1126,7 @@ WORK_STATE ossl_statem_client_post_process_message(SSL_CONNECTION *s, } } -CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_client_hello(SSL *s, WPACKET *pkt) { unsigned char *p; size_t sess_id_len; @@ -1170,13 +1136,12 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) #endif SSL_SESSION *sess = s->session; unsigned char *session_id; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* Work out what SSL/TLS/DTLS version to use */ protverr = ssl_set_client_hello_version(s); if (protverr != 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, protverr); - return CON_FUNC_ERROR; + return 0; } if (sess == NULL @@ -1185,7 +1150,7 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) if (s->hello_retry_request == SSL_HRR_NONE && !ssl_get_new_session(s, 0)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } } /* else use the pre-loaded session */ @@ -1196,7 +1161,7 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) * for DTLS if client_random is initialized, reuse it, we are * required to use same upon reply to HelloVerify */ - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { size_t idx; i = 1; for (idx = 0; idx < sizeof(s->s3.client_random); idx++) { @@ -1212,7 +1177,7 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) if (i && ssl_fill_hello_random(s, 0, p, sizeof(s->s3.client_random), DOWNGRADE_NONE) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /*- @@ -1251,7 +1216,7 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) if (!WPACKET_put_bytes_u16(pkt, s->client_version) || !WPACKET_memcpy(pkt, s->s3.client_random, SSL3_RANDOM_SIZE)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /* Session ID */ @@ -1263,10 +1228,10 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) s->tmp_session_id_len = sess_id_len; session_id = s->tmp_session_id; if (s->hello_retry_request == SSL_HRR_NONE - && RAND_bytes_ex(sctx->libctx, s->tmp_session_id, + && RAND_bytes_ex(s->ctx->libctx, s->tmp_session_id, sess_id_len, 0) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } } else { sess_id_len = 0; @@ -1284,51 +1249,49 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) sess_id_len)) || !WPACKET_close(pkt)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /* cookie stuff for DTLS */ - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { if (s->d1->cookie_len > sizeof(s->d1->cookie) || !WPACKET_sub_memcpy_u8(pkt, s->d1->cookie, s->d1->cookie_len)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } } /* Ciphers supported */ if (!WPACKET_start_sub_packet_u16(pkt)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } - if (!ssl_cipher_list_to_bytes(s, SSL_get_ciphers(SSL_CONNECTION_GET_SSL(s)), - pkt)) { + if (!ssl_cipher_list_to_bytes(s, SSL_get_ciphers(s), pkt)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } if (!WPACKET_close(pkt)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /* COMPRESSION */ if (!WPACKET_start_sub_packet_u8(pkt)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } #ifndef OPENSSL_NO_COMP if (ssl_allow_compression(s) - && sctx->comp_methods - && (SSL_CONNECTION_IS_DTLS(s) - || s->s3.tmp.max_ver < TLS1_3_VERSION)) { - int compnum = sk_SSL_COMP_num(sctx->comp_methods); + && s->ctx->comp_methods + && (SSL_IS_DTLS(s) || s->s3.tmp.max_ver < TLS1_3_VERSION)) { + int compnum = sk_SSL_COMP_num(s->ctx->comp_methods); for (i = 0; i < compnum; i++) { - comp = sk_SSL_COMP_value(sctx->comp_methods, i); + comp = sk_SSL_COMP_value(s->ctx->comp_methods, i); if (!WPACKET_put_bytes_u8(pkt, comp->id)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } } } @@ -1336,19 +1299,19 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt) /* Add the NULL method */ if (!WPACKET_put_bytes_u8(pkt, 0) || !WPACKET_close(pkt)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /* TLS extensions */ if (!tls_construct_extensions(s, pkt, SSL_EXT_CLIENT_HELLO, NULL, 0)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } - return CON_FUNC_SUCCESS; + return 1; } -MSG_PROCESS_RETURN dtls_process_hello_verify(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN dtls_process_hello_verify(SSL *s, PACKET *pkt) { size_t cookie_len; PACKET cookiepkt; @@ -1374,13 +1337,11 @@ MSG_PROCESS_RETURN dtls_process_hello_verify(SSL_CONNECTION *s, PACKET *pkt) return MSG_PROCESS_FINISHED_READING; } -static int set_client_ciphersuite(SSL_CONNECTION *s, - const unsigned char *cipherchars) +static int set_client_ciphersuite(SSL *s, const unsigned char *cipherchars) { STACK_OF(SSL_CIPHER) *sk; const SSL_CIPHER *c; int i; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); c = ssl_get_cipher_by_char(s, cipherchars, 0); if (c == NULL) { @@ -1405,7 +1366,7 @@ static int set_client_ciphersuite(SSL_CONNECTION *s, return 0; } - if (SSL_CONNECTION_IS_TLS13(s) && s->s3.tmp.new_cipher != NULL + if (SSL_IS_TLS13(s) && s->s3.tmp.new_cipher != NULL && s->s3.tmp.new_cipher->id != c->id) { /* ServerHello selected a different ciphersuite to that in the HRR */ SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_CIPHER_RETURNED); @@ -1420,19 +1381,13 @@ static int set_client_ciphersuite(SSL_CONNECTION *s, if (s->session->cipher != NULL) s->session->cipher_id = s->session->cipher->id; if (s->hit && (s->session->cipher_id != c->id)) { - if (SSL_CONNECTION_IS_TLS13(s)) { - const EVP_MD *md = ssl_md(sctx, c->algorithm2); - - if (!ossl_assert(s->session->cipher != NULL)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } + if (SSL_IS_TLS13(s)) { /* * In TLSv1.3 it is valid for the server to select a different * ciphersuite as long as the hash is the same. */ - if (md == NULL - || md != ssl_md(sctx, s->session->cipher->algorithm2)) { + if (ssl_md(s->ctx, c->algorithm2) + != ssl_md(s->ctx, s->session->cipher->algorithm2)) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_CIPHERSUITE_DIGEST_HAS_CHANGED); return 0; @@ -1452,7 +1407,7 @@ static int set_client_ciphersuite(SSL_CONNECTION *s, return 1; } -MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_server_hello(SSL *s, PACKET *pkt) { PACKET session_id, extpkt; size_t session_id_len; @@ -1462,7 +1417,6 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) unsigned int sversion; unsigned int context; RAW_EXTENSION *extensions = NULL; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); #ifndef OPENSSL_NO_COMP SSL_COMP *comp; #endif @@ -1482,11 +1436,6 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) goto err; } s->hello_retry_request = SSL_HRR_PENDING; - /* Tell the record layer that we know we're going to get TLSv1.3 */ - if (!ssl_set_record_protocol_version(s, s->version)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } hrr = 1; if (!PACKET_forward(pkt, SSL3_RANDOM_SIZE)) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); @@ -1545,7 +1494,7 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) } } - if (SSL_CONNECTION_IS_TLS13(s) || hrr) { + if (SSL_IS_TLS13(s) || hrr) { if (compression != 0) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_INVALID_COMPRESSION_ALGORITHM); @@ -1573,8 +1522,8 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) * Now we have chosen the version we need to check again that the extensions * are appropriate for this version. */ - context = SSL_CONNECTION_IS_TLS13(s) ? SSL_EXT_TLS1_3_SERVER_HELLO - : SSL_EXT_TLS1_2_SERVER_HELLO; + context = SSL_IS_TLS13(s) ? SSL_EXT_TLS1_3_SERVER_HELLO + : SSL_EXT_TLS1_2_SERVER_HELLO; if (!tls_validate_all_contexts(s, context, extensions)) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_EXTENSION); goto err; @@ -1582,7 +1531,7 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) s->hit = 0; - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* * In TLSv1.3 a ServerHello message signals a key change so the end of * the message must be on a record boundary. @@ -1621,9 +1570,8 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) * backwards compat reasons */ int master_key_length; - master_key_length = sizeof(s->session->master_key); - if (s->ext.session_secret_cb(ssl, s->session->master_key, + if (s->ext.session_secret_cb(s, s->session->master_key, &master_key_length, NULL, &pref_cipher, s->ext.session_secret_cb_arg) @@ -1675,7 +1623,7 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) * echo of what we originally sent in the ClientHello and should not be * used for resumption. */ - if (!SSL_CONNECTION_IS_TLS13(s)) { + if (!SSL_IS_TLS13(s)) { s->session->session_id_length = session_id_len; /* session_id_len could be 0 */ if (session_id_len > 0) @@ -1728,8 +1676,7 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_COMPRESSION_DISABLED); goto err; } else { - comp = ssl3_comp_find(SSL_CONNECTION_GET_CTX(s)->comp_methods, - compression); + comp = ssl3_comp_find(s->ctx->comp_methods, compression); } if (compression != 0 && comp == NULL) { @@ -1746,8 +1693,28 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) goto err; } +#ifndef OPENSSL_NO_SM2 + /* + * To use the cipher suites TLS_SM4_GCM_SM3 and TLS_SM4_CCM_SM3, + * RFC 8998 demand that: + * For the key_share extension, a KeyShareEntry with SM2-related + * values MUST be added. + */ + if (SSL_IS_TLS13(s) && s->enable_sm_tls13_strict == 1) { + const SSL_CIPHER *cipher = s->s3.tmp.new_cipher; + + if (cipher->id == TLS1_3_CK_SM4_GCM_SM3 + || cipher->id == TLS1_3_CK_SM4_CCM_SM3) { + if (s->s3.group_id != TLSEXT_curve_SM2) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + goto err; + } + } + } +#endif + #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s) && s->hit) { + if (SSL_IS_DTLS(s) && s->hit) { unsigned char sctpauthkey[64]; char labelbuffer[sizeof(DTLS1_SCTP_AUTH_LABEL)]; size_t labellen; @@ -1764,7 +1731,7 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) if (s->mode & SSL_MODE_DTLS_SCTP_LABEL_LENGTH_BUG) labellen += 1; - if (SSL_export_keying_material(ssl, sctpauthkey, + if (SSL_export_keying_material(s, sctpauthkey, sizeof(sctpauthkey), labelbuffer, labellen, NULL, 0, 0) <= 0) { @@ -1772,7 +1739,7 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) goto err; } - BIO_ctrl(SSL_get_wbio(ssl), + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY, sizeof(sctpauthkey), sctpauthkey); } @@ -1782,29 +1749,12 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) * In TLSv1.3 we have some post-processing to change cipher state, otherwise * we're done with this message */ - if (SSL_CONNECTION_IS_TLS13(s)) { - if (!ssl->method->ssl3_enc->setup_key_block(s) - || !ssl->method->ssl3_enc->change_cipher_state(s, - SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_READ)) { - /* SSLfatal() already called */ - goto err; - } - /* - * If we're not doing early-data and we're not going to send a dummy CCS - * (i.e. no middlebox compat mode) then we can change the write keys - * immediately. Otherwise we have to defer this until after all possible - * early data is written. We could just always defer until the last - * moment except QUIC needs it done at the same time as the read keys - * are changed. Since QUIC doesn't do TLS early data or need middlebox - * compat this doesn't cause a problem. - */ - if (s->early_data_state == SSL_EARLY_DATA_NONE - && (s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) == 0 - && !ssl->method->ssl3_enc->change_cipher_state(s, - SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_WRITE)) { - /* SSLfatal() already called */ - goto err; - } + if (SSL_IS_TLS13(s) + && (!s->method->ssl3_enc->setup_key_block(s) + || !s->method->ssl3_enc->change_cipher_state(s, + SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_READ))) { + /* SSLfatal() already called */ + goto err; } OPENSSL_free(extensions); @@ -1814,27 +1764,17 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt) return MSG_PROCESS_ERROR; } -static MSG_PROCESS_RETURN tls_process_as_hello_retry_request(SSL_CONNECTION *s, +static MSG_PROCESS_RETURN tls_process_as_hello_retry_request(SSL *s, PACKET *extpkt) { RAW_EXTENSION *extensions = NULL; /* - * If we were sending early_data then any alerts should not be sent using - * the old wrlmethod. + * If we were sending early_data then the enc_write_ctx is now invalid and + * should not be used. */ - if (s->early_data_state == SSL_EARLY_DATA_FINISHED_WRITING - && !ssl_set_new_record_layer(s, - TLS_ANY_VERSION, - OSSL_RECORD_DIRECTION_WRITE, - OSSL_RECORD_PROTECTION_LEVEL_NONE, - NULL, 0, NULL, 0, NULL, 0, NULL, 0, - NULL, 0, NID_undef, NULL, NULL, NULL)) { - /* SSLfatal already called */ - goto err; - } - /* We are definitely going to be using TLSv1.3 */ - s->rlayer.wrlmethod->set_protocol_version(s->rlayer.wrl, TLS1_3_VERSION); + EVP_CIPHER_CTX_free(s->enc_write_ctx); + s->enc_write_ctx = NULL; if (!tls_collect_extensions(s, extpkt, SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST, &extensions, NULL, 1) @@ -1883,106 +1823,21 @@ static MSG_PROCESS_RETURN tls_process_as_hello_retry_request(SSL_CONNECTION *s, return MSG_PROCESS_ERROR; } -MSG_PROCESS_RETURN tls_process_server_rpk(SSL_CONNECTION *sc, PACKET *pkt) -{ - EVP_PKEY *peer_rpk; - - if (!tls_process_rpk(sc, pkt, &peer_rpk)) { - /* SSLfatal() already called */ - return MSG_PROCESS_ERROR; - } - - if (peer_rpk == NULL) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_CERTIFICATE); - return MSG_PROCESS_ERROR; - } - - EVP_PKEY_free(sc->session->peer_rpk); - sc->session->peer_rpk = peer_rpk; - - return MSG_PROCESS_CONTINUE_PROCESSING; -} - -static WORK_STATE tls_post_process_server_rpk(SSL_CONNECTION *sc, - WORK_STATE wst) -{ - size_t certidx; - const SSL_CERT_LOOKUP *clu; - - if (sc->session->peer_rpk == NULL) { - SSLfatal(sc, SSL_AD_ILLEGAL_PARAMETER, - SSL_R_INVALID_RAW_PUBLIC_KEY); - return WORK_ERROR; - } - - if (sc->rwstate == SSL_RETRY_VERIFY) - sc->rwstate = SSL_NOTHING; - if (ssl_verify_rpk(sc, sc->session->peer_rpk) > 0 - && sc->rwstate == SSL_RETRY_VERIFY) - return WORK_MORE_A; - - if ((clu = ssl_cert_lookup_by_pkey(sc->session->peer_rpk, &certidx, - SSL_CONNECTION_GET_CTX(sc))) == NULL) { - SSLfatal(sc, SSL_AD_ILLEGAL_PARAMETER, SSL_R_UNKNOWN_CERTIFICATE_TYPE); - return WORK_ERROR; - } - - /* - * Check certificate type is consistent with ciphersuite. For TLS 1.3 - * skip check since TLS 1.3 ciphersuites can be used with any certificate - * type. - */ - if (!SSL_CONNECTION_IS_TLS13(sc)) { - if ((clu->amask & sc->s3.tmp.new_cipher->algorithm_auth) == 0) { - SSLfatal(sc, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_RPK_TYPE); - return WORK_ERROR; - } - } - - /* Ensure there is no peer/peer_chain */ - X509_free(sc->session->peer); - sc->session->peer = NULL; - sk_X509_pop_free(sc->session->peer_chain, X509_free); - sc->session->peer_chain = NULL; - sc->session->verify_result = sc->verify_result; - - /* Save the current hash state for when we receive the CertificateVerify */ - if (SSL_CONNECTION_IS_TLS13(sc) - && !ssl_handshake_hash(sc, sc->cert_verify_hash, - sizeof(sc->cert_verify_hash), - &sc->cert_verify_hash_len)) { - /* SSLfatal() already called */ - return WORK_ERROR; - } - - return WORK_FINISHED_CONTINUE; -} - /* prepare server cert verification by setting s->session->peer_chain from pkt */ -MSG_PROCESS_RETURN tls_process_server_certificate(SSL_CONNECTION *s, - PACKET *pkt) +MSG_PROCESS_RETURN tls_process_server_certificate(SSL *s, PACKET *pkt) { unsigned long cert_list_len, cert_len; X509 *x = NULL; const unsigned char *certstart, *certbytes; size_t chainidx; unsigned int context = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - - if (s->ext.server_cert_type == TLSEXT_cert_type_rpk) - return tls_process_server_rpk(s, pkt); - if (s->ext.server_cert_type != TLSEXT_cert_type_x509) { - SSLfatal(s, SSL_AD_UNSUPPORTED_CERTIFICATE, - SSL_R_UNKNOWN_CERTIFICATE_TYPE); - goto err; - } if ((s->session->peer_chain = sk_X509_new_null()) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } - if ((SSL_CONNECTION_IS_TLS13(s) && !PACKET_get_1(pkt, &context)) + if ((SSL_IS_TLS13(s) && !PACKET_get_1(pkt, &context)) || context != 0 || !PACKET_get_net_3(pkt, &cert_list_len) || PACKET_remaining(pkt) != cert_list_len @@ -1998,9 +1853,10 @@ MSG_PROCESS_RETURN tls_process_server_certificate(SSL_CONNECTION *s, } certstart = certbytes; - x = X509_new_ex(sctx->libctx, sctx->propq); + x = X509_new_ex(s->ctx->libctx, s->ctx->propq); if (x == NULL) { - SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_ASN1_LIB); + SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_MALLOC_FAILURE); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; } if (d2i_X509(&x, (const unsigned char **)&certbytes, @@ -2014,7 +1870,7 @@ MSG_PROCESS_RETURN tls_process_server_certificate(SSL_CONNECTION *s, goto err; } - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { RAW_EXTENSION *rawexts = NULL; PACKET extensions; @@ -2036,16 +1892,56 @@ MSG_PROCESS_RETURN tls_process_server_certificate(SSL_CONNECTION *s, } if (!sk_X509_push(s->session->peer_chain, x)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } x = NULL; } + +#ifndef OPENSSL_NO_SM2 + { + EVP_PKEY *pkey = NULL; + int n = sk_X509_num(s->session->peer_chain) - 1; + + x = sk_X509_value(s->session->peer_chain, 0); + pkey = X509_get0_pubkey(x); + + if (pkey != NULL && EVP_PKEY_is_sm2(pkey)) { + if (!EVP_PKEY_set_alias_type(pkey, EVP_PKEY_SM2)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + while (n >= 0) { + X509 *cert = sk_X509_value(s->session->peer_chain, n); + ASN1_OCTET_STRING *sm2_id; + sm2_id = ASN1_OCTET_STRING_new(); + + if (sm2_id == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!ASN1_OCTET_STRING_set(sm2_id, + (const unsigned char *)CERTVRIFY_SM2_ID, + CERTVRIFY_SM2_ID_LEN)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + ASN1_OCTET_STRING_free(sm2_id); + goto err; + } + + X509_set0_sm2_id(cert, sm2_id); + n--; + } + } + } +#endif + return MSG_PROCESS_CONTINUE_PROCESSING; err: X509_free(x); - OSSL_STACK_OF_X509_free(s->session->peer_chain); + sk_X509_pop_free(s->session->peer_chain, X509_free); s->session->peer_chain = NULL; return MSG_PROCESS_ERROR; } @@ -2055,8 +1951,7 @@ MSG_PROCESS_RETURN tls_process_server_certificate(SSL_CONNECTION *s, * On success set s->session->peer and s->session->verify_result. * Else the peer certificate verification callback may request retry. */ -WORK_STATE tls_post_process_server_certificate(SSL_CONNECTION *s, - WORK_STATE wst) +WORK_STATE tls_post_process_server_certificate(SSL *s, WORK_STATE wst) { X509 *x; EVP_PKEY *pkey = NULL; @@ -2064,9 +1959,6 @@ WORK_STATE tls_post_process_server_certificate(SSL_CONNECTION *s, size_t certidx; int i; - if (s->ext.server_cert_type == TLSEXT_cert_type_rpk) - return tls_post_process_server_rpk(s, wst); - if (s->rwstate == SSL_RETRY_VERIFY) s->rwstate = SSL_NOTHING; i = ssl_verify_cert_chain(s, s->session->peer_chain); @@ -2108,8 +2000,7 @@ WORK_STATE tls_post_process_server_certificate(SSL_CONNECTION *s, return WORK_ERROR; } - if ((clu = ssl_cert_lookup_by_pkey(pkey, &certidx, - SSL_CONNECTION_GET_CTX(s))) == NULL) { + if ((clu = ssl_cert_lookup_by_pkey(pkey, &certidx)) == NULL) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_UNKNOWN_CERTIFICATE_TYPE); return WORK_ERROR; } @@ -2118,23 +2009,76 @@ WORK_STATE tls_post_process_server_certificate(SSL_CONNECTION *s, * skip check since TLS 1.3 ciphersuites can be used with any certificate * type. */ - if (!SSL_CONNECTION_IS_TLS13(s)) { + if (!SSL_IS_TLS13(s)) { if ((clu->amask & s->s3.tmp.new_cipher->algorithm_auth) == 0) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_CERTIFICATE_TYPE); return WORK_ERROR; } } +#ifndef OPENSSL_NO_SM2 + /* + * RFC 8998 requires that + * The public key in the certificate MUST be a valid SM2 public key. + * The signature algorithm used by the CA to sign the current + * certificate MUST be "sm2sig_sm3". + * The certificate MUST be capable of signing; e.g., the digitalSignature + * bit of X.509's Key Usage extension is set. + */ + if (SSL_IS_TLS13(s) && s->enable_sm_tls13_strict == 1) { + const SSL_CIPHER *cipher = s->s3.tmp.new_cipher; + + if (cipher->id == TLS1_3_CK_SM4_GCM_SM3 + || cipher->id == TLS1_3_CK_SM4_CCM_SM3) { + if (EVP_PKEY_id(pkey) != EVP_PKEY_SM2) { + SSLfatal(s, SSL_AD_BAD_CERTIFICATE, + SSL_R_WRONG_CERTIFICATE_TYPE); + return WORK_ERROR; + } + + if (X509_get_signature_nid(x) != NID_SM2_with_SM3) { + SSLfatal(s, SSL_AD_BAD_CERTIFICATE, + SSL_R_BAD_CERTIFICATE_SIGNATURE_TYPE); + return WORK_ERROR; + } + + if ((X509_get_key_usage(x) & X509v3_KU_DIGITAL_SIGNATURE) == 0) { + SSLfatal(s, SSL_AD_BAD_CERTIFICATE, + SSL_R_BAD_CERTIFICATE_USAGE); + return WORK_ERROR; + } + } + } +#endif + X509_free(s->session->peer); X509_up_ref(x); s->session->peer = x; s->session->verify_result = s->verify_result; - /* Ensure there is no RPK */ - EVP_PKEY_free(s->session->peer_rpk); - s->session->peer_rpk = NULL; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + if (s->delegated_credential_tag & DC_HAS_BEEN_USED_FOR_VERIFY_PEER) { + if (!SSL_IS_TLS13(s)) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_UNKNOWN_CERTIFICATE_TYPE); + return WORK_ERROR; + } + if (!DC_check_valid(x, s->session->peer_dc)) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_CERTIFICATE_VERIFY_FAILED); + return WORK_ERROR; + } + + if (SSL_verify_delegated_credential_signature(x, s->session->peer_dc, + 1) <= 0) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_CERTIFICATE_VERIFY_FAILED); + return WORK_ERROR; + } + } +#endif /* Save the current hash state for when we receive the CertificateVerify */ - if (SSL_CONNECTION_IS_TLS13(s) + if (SSL_IS_TLS13(s) && !ssl_handshake_hash(s, s->cert_verify_hash, sizeof(s->cert_verify_hash), &s->cert_verify_hash_len)) { @@ -2144,22 +2088,7 @@ WORK_STATE tls_post_process_server_certificate(SSL_CONNECTION *s, return WORK_FINISHED_CONTINUE; } -#ifndef OPENSSL_NO_COMP_ALG -MSG_PROCESS_RETURN tls_process_server_compressed_certificate(SSL_CONNECTION *sc, PACKET *pkt) -{ - MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR; - PACKET tmppkt; - BUF_MEM *buf = BUF_MEM_new(); - - if (tls13_process_compressed_certificate(sc, pkt, &tmppkt, buf) != MSG_PROCESS_ERROR) - ret = tls_process_server_certificate(sc, &tmppkt); - - BUF_MEM_free(buf); - return ret; -} -#endif - -static int tls_process_ske_psk_preamble(SSL_CONNECTION *s, PACKET *pkt) +static int tls_process_ske_psk_preamble(SSL *s, PACKET *pkt) { #ifndef OPENSSL_NO_PSK PACKET psk_identity_hint; @@ -2198,7 +2127,7 @@ static int tls_process_ske_psk_preamble(SSL_CONNECTION *s, PACKET *pkt) #endif } -static int tls_process_ske_srp(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) +static int tls_process_ske_srp(SSL *s, PACKET *pkt, EVP_PKEY **pkey) { #ifndef OPENSSL_NO_SRP PACKET prime, generator, salt, server_pub; @@ -2234,7 +2163,7 @@ static int tls_process_ske_srp(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) /* We must check if there is a certificate */ if (s->s3.tmp.new_cipher->algorithm_auth & (SSL_aRSA | SSL_aDSS)) - *pkey = tls_get_peer_pkey(s); + *pkey = X509_get0_pubkey(s->session->peer); return 1; #else @@ -2243,7 +2172,7 @@ static int tls_process_ske_srp(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) #endif } -static int tls_process_ske_dhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) +static int tls_process_ske_dhe(SSL *s, PACKET *pkt, EVP_PKEY **pkey) { PACKET prime, generator, pub_key; EVP_PKEY *peer_tmp = NULL; @@ -2251,7 +2180,6 @@ static int tls_process_ske_dhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) EVP_PKEY_CTX *pctx = NULL; OSSL_PARAM *params = NULL; OSSL_PARAM_BLD *tmpl = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); int ret = 0; if (!PACKET_get_length_prefixed_2(pkt, &prime) @@ -2282,7 +2210,7 @@ static int tls_process_ske_dhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) goto err; } - pctx = EVP_PKEY_CTX_new_from_name(sctx->libctx, "DH", sctx->propq); + pctx = EVP_PKEY_CTX_new_from_name(s->ctx->libctx, "DH", s->ctx->propq); if (pctx == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; @@ -2294,7 +2222,7 @@ static int tls_process_ske_dhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) } EVP_PKEY_CTX_free(pctx); - pctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, peer_tmp, sctx->propq); + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, peer_tmp, s->ctx->propq); if (pctx == NULL /* * EVP_PKEY_param_check() will verify that the DH params are using @@ -2323,7 +2251,7 @@ static int tls_process_ske_dhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) * public keys. We should have a less ad-hoc way of doing this */ if (s->s3.tmp.new_cipher->algorithm_auth & (SSL_aRSA | SSL_aDSS)) - *pkey = tls_get_peer_pkey(s); + *pkey = X509_get0_pubkey(s->session->peer); /* else anonymous DH, so no certificate or pkey. */ ret = 1; @@ -2340,7 +2268,7 @@ static int tls_process_ske_dhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) return ret; } -static int tls_process_ske_ecdhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey) +static int tls_process_ske_ecdhe(SSL *s, PACKET *pkt, EVP_PKEY **pkey) { PACKET encoded_pt; unsigned int curve_type, curve_id; @@ -2388,9 +2316,9 @@ static int tls_process_ske_ecdhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey * and ECDSA. */ if (s->s3.tmp.new_cipher->algorithm_auth & SSL_aECDSA) - *pkey = tls_get_peer_pkey(s); + *pkey = X509_get0_pubkey(s->session->peer); else if (s->s3.tmp.new_cipher->algorithm_auth & SSL_aRSA) - *pkey = tls_get_peer_pkey(s); + *pkey = X509_get0_pubkey(s->session->peer); /* else anonymous ECDH, so no certificate or pkey. */ /* Cache the agreed upon group in the SSL_SESSION */ @@ -2398,14 +2326,13 @@ static int tls_process_ske_ecdhe(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **pkey return 1; } -MSG_PROCESS_RETURN tls_process_key_exchange(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_key_exchange(SSL *s, PACKET *pkt) { long alg_k; EVP_PKEY *pkey = NULL; EVP_MD_CTX *md_ctx = NULL; EVP_PKEY_CTX *pctx = NULL; PACKET save_param_start, signature; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); alg_k = s->s3.tmp.new_cipher->algorithm_mkey; @@ -2474,12 +2401,11 @@ MSG_PROCESS_RETURN tls_process_key_exchange(SSL_CONNECTION *s, PACKET *pkt) goto err; } } else if (!tls1_set_peer_legacy_sigalg(s, pkey)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, - SSL_R_LEGACY_SIGALG_DISALLOWED_OR_UNSUPPORTED); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } - if (!tls1_lookup_md(sctx, s->s3.tmp.peer_sigalg, &md)) { + if (!tls1_lookup_md(s->ctx, s->s3.tmp.peer_sigalg, &md)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_NO_SUITABLE_DIGEST_ALGORITHM); goto err; @@ -2496,13 +2422,13 @@ MSG_PROCESS_RETURN tls_process_key_exchange(SSL_CONNECTION *s, PACKET *pkt) md_ctx = EVP_MD_CTX_new(); if (md_ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } if (EVP_DigestVerifyInit_ex(md_ctx, &pctx, md == NULL ? NULL : EVP_MD_get0_name(md), - sctx->libctx, sctx->propq, pkey, + s->ctx->libctx, s->ctx->propq, pkey, NULL) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); goto err; @@ -2555,20 +2481,15 @@ MSG_PROCESS_RETURN tls_process_key_exchange(SSL_CONNECTION *s, PACKET *pkt) return MSG_PROCESS_ERROR; } -MSG_PROCESS_RETURN tls_process_certificate_request(SSL_CONNECTION *s, - PACKET *pkt) +MSG_PROCESS_RETURN tls_process_certificate_request(SSL *s, PACKET *pkt) { - /* Clear certificate validity flags */ - if (s->s3.tmp.valid_flags != NULL) - memset(s->s3.tmp.valid_flags, 0, s->ssl_pkey_num * sizeof(uint32_t)); - else - s->s3.tmp.valid_flags = OPENSSL_zalloc(s->ssl_pkey_num * sizeof(uint32_t)); + size_t i; - /* Give up for good if allocation didn't work */ - if (s->s3.tmp.valid_flags == NULL) - return 0; + /* Clear certificate validity flags */ + for (i = 0; i < SSL_PKEY_NUM; i++) + s->s3.tmp.valid_flags[i] = 0; - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { PACKET reqctx, extensions; RAW_EXTENSION *rawexts = NULL; @@ -2609,6 +2530,39 @@ MSG_PROCESS_RETURN tls_process_certificate_request(SSL_CONNECTION *s, return MSG_PROCESS_ERROR; } OPENSSL_free(rawexts); +#ifndef OPENSSL_NO_SM2 + /* + * RFC 8998 requires that + * if the server chooses TLS_SM4_GCM_SM3 or TLS_SM4_CCM_SM3, + * the only valid signature algorithm present in + * "signature_algorithms" extension MUST be "sm2sig_sm3". + */ + if (s->enable_sm_tls13_strict == 1) { + const SSL_CIPHER *cipher = s->s3.tmp.new_cipher; + + if (cipher->id == TLS1_3_CK_SM4_GCM_SM3 + || cipher->id == TLS1_3_CK_SM4_CCM_SM3) { + + if (s->s3.tmp.peer_sigalgslen > 0 + && (s->s3.tmp.peer_sigalgslen != 1 + || s->s3.tmp.peer_sigalgs[0] != TLSEXT_SIGALG_sm2sig_sm3)) + { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); + return MSG_PROCESS_ERROR; + } + + if (s->s3.tmp.peer_cert_sigalgslen > 0 + && (s->s3.tmp.peer_cert_sigalgslen != 1 + || s->s3.tmp.peer_cert_sigalgs[0] != TLSEXT_SIGALG_sm2sig_sm3)) + { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); + return MSG_PROCESS_ERROR; + } + } + } +#endif if (!tls1_process_sigalgs(s)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_LENGTH); return MSG_PROCESS_ERROR; @@ -2645,7 +2599,7 @@ MSG_PROCESS_RETURN tls_process_certificate_request(SSL_CONNECTION *s, return MSG_PROCESS_ERROR; } if (!tls1_process_sigalgs(s)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return MSG_PROCESS_ERROR; } } @@ -2673,15 +2627,13 @@ MSG_PROCESS_RETURN tls_process_certificate_request(SSL_CONNECTION *s, * SSL_get1_peer_certificate() returns something sensible in * client_cert_cb. */ - if (SSL_CONNECTION_IS_TLS13(s) - && s->post_handshake_auth != SSL_PHA_REQUESTED) + if (SSL_IS_TLS13(s) && s->post_handshake_auth != SSL_PHA_REQUESTED) return MSG_PROCESS_CONTINUE_READING; return MSG_PROCESS_CONTINUE_PROCESSING; } -MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, - PACKET *pkt) +MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL *s, PACKET *pkt) { unsigned int ticklen; unsigned long ticket_lifetime_hint, age_add = 0; @@ -2689,18 +2641,16 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, RAW_EXTENSION *exts = NULL; PACKET nonce; EVP_MD *sha256 = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); PACKET_null_init(&nonce); if (!PACKET_get_net_4(pkt, &ticket_lifetime_hint) - || (SSL_CONNECTION_IS_TLS13(s) + || (SSL_IS_TLS13(s) && (!PACKET_get_net_4(pkt, &age_add) || !PACKET_get_length_prefixed_1(pkt, &nonce))) || !PACKET_get_net_2(pkt, &ticklen) - || (SSL_CONNECTION_IS_TLS13(s) ? (ticklen == 0 - || PACKET_remaining(pkt) < ticklen) - : PACKET_remaining(pkt) != ticklen)) { + || (SSL_IS_TLS13(s) ? (ticklen == 0 || PACKET_remaining(pkt) < ticklen) + : PACKET_remaining(pkt) != ticklen)) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); goto err; } @@ -2721,7 +2671,7 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, * post-handshake and the session may have already gone into the session * cache. */ - if (SSL_CONNECTION_IS_TLS13(s) || s->session->session_id_length > 0) { + if (SSL_IS_TLS13(s) || s->session->session_id_length > 0) { SSL_SESSION *new_sess; /* @@ -2729,12 +2679,12 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, * one */ if ((new_sess = ssl_session_dup(s->session, 0)) == 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } if ((s->session_ctx->session_cache_mode & SSL_SESS_CACHE_CLIENT) != 0 - && !SSL_CONNECTION_IS_TLS13(s)) { + && !SSL_IS_TLS13(s)) { /* * In TLSv1.2 and below the arrival of a new tickets signals that * any old ticket we were using is now out of date, so we remove the @@ -2747,7 +2697,7 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, s->session = new_sess; } - s->session->time = ossl_time_now(); + s->session->time = time(NULL); ssl_session_calculate_timeout(s->session); OPENSSL_free(s->session->ext.tick); @@ -2756,7 +2706,7 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, s->session->ext.tick = OPENSSL_malloc(ticklen); if (s->session->ext.tick == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } if (!PACKET_copy_bytes(pkt, s->session->ext.tick, ticklen)) { @@ -2768,7 +2718,7 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, s->session->ext.tick_age_add = age_add; s->session->ext.ticklen = ticklen; - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { PACKET extpkt; if (!PACKET_as_length_prefixed_2(pkt, &extpkt) @@ -2799,7 +2749,7 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, * elsewhere in OpenSSL. The session ID is set to the SHA256 hash of the * ticket. */ - sha256 = EVP_MD_fetch(sctx->libctx, "SHA2-256", sctx->propq); + sha256 = EVP_MD_fetch(s->ctx->libctx, "SHA2-256", s->ctx->propq); if (sha256 == NULL) { /* Error is already recorded */ SSLfatal_alert(s, SSL_AD_INTERNAL_ERROR); @@ -2821,7 +2771,7 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, s->session->not_resumable = 0; /* This is a standalone message in TLSv1.3, so there is no more to read */ - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { const EVP_MD *md = ssl_handshake_md(s); int hashleni = EVP_MD_get_size(md); size_t hashlen; @@ -2862,7 +2812,7 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, * In TLSv1.3 this is called from the extensions code, otherwise it is used to * parse a separate message. Returns 1 on success or 0 on failure */ -int tls_process_cert_status_body(SSL_CONNECTION *s, PACKET *pkt) +int tls_process_cert_status_body(SSL *s, PACKET *pkt) { size_t resplen; unsigned int type; @@ -2880,7 +2830,7 @@ int tls_process_cert_status_body(SSL_CONNECTION *s, PACKET *pkt) s->ext.ocsp.resp = OPENSSL_malloc(resplen); if (s->ext.ocsp.resp == NULL) { s->ext.ocsp.resp_len = 0; - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } s->ext.ocsp.resp_len = resplen; @@ -2893,7 +2843,7 @@ int tls_process_cert_status_body(SSL_CONNECTION *s, PACKET *pkt) } -MSG_PROCESS_RETURN tls_process_cert_status(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_cert_status(SSL *s, PACKET *pkt) { if (!tls_process_cert_status_body(s, pkt)) { /* SSLfatal() already called */ @@ -2909,10 +2859,8 @@ MSG_PROCESS_RETURN tls_process_cert_status(SSL_CONNECTION *s, PACKET *pkt) * In <=TLS1.2 this is after the ServerDone message. Returns 1 on success or 0 * on failure. */ -int tls_process_initial_server_flight(SSL_CONNECTION *s) +int tls_process_initial_server_flight(SSL *s) { - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - /* * at this point we check that we have the required stuff from * the server @@ -2928,9 +2876,8 @@ int tls_process_initial_server_flight(SSL_CONNECTION *s) * message, or NULL and -1 otherwise */ if (s->ext.status_type != TLSEXT_STATUSTYPE_nothing - && sctx->ext.status_cb != NULL) { - int ret = sctx->ext.status_cb(SSL_CONNECTION_GET_SSL(s), - sctx->ext.status_arg); + && s->ctx->ext.status_cb != NULL) { + int ret = s->ctx->ext.status_cb(s, s->ctx->ext.status_arg); if (ret == 0) { SSLfatal(s, SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE, @@ -2956,7 +2903,7 @@ int tls_process_initial_server_flight(SSL_CONNECTION *s) return 1; } -MSG_PROCESS_RETURN tls_process_server_done(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_server_done(SSL *s, PACKET *pkt) { if (PACKET_remaining(pkt) > 0) { /* should contain no data */ @@ -2980,7 +2927,7 @@ MSG_PROCESS_RETURN tls_process_server_done(SSL_CONNECTION *s, PACKET *pkt) return MSG_PROCESS_FINISHED_READING; } -static int tls_construct_cke_psk_preamble(SSL_CONNECTION *s, WPACKET *pkt) +static int tls_construct_cke_psk_preamble(SSL *s, WPACKET *pkt) { #ifndef OPENSSL_NO_PSK int ret = 0; @@ -3003,8 +2950,7 @@ static int tls_construct_cke_psk_preamble(SSL_CONNECTION *s, WPACKET *pkt) memset(identity, 0, sizeof(identity)); - psklen = s->psk_client_callback(SSL_CONNECTION_GET_SSL(s), - s->session->psk_identity_hint, + psklen = s->psk_client_callback(s, s->session->psk_identity_hint, identity, sizeof(identity) - 1, psk, sizeof(psk)); @@ -3026,7 +2972,7 @@ static int tls_construct_cke_psk_preamble(SSL_CONNECTION *s, WPACKET *pkt) tmppsk = OPENSSL_memdup(psk, psklen); tmpidentity = OPENSSL_strdup(identity); if (tmppsk == NULL || tmpidentity == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -3058,7 +3004,7 @@ static int tls_construct_cke_psk_preamble(SSL_CONNECTION *s, WPACKET *pkt) #endif } -static int tls_construct_cke_rsa(SSL_CONNECTION *s, WPACKET *pkt) +static int tls_construct_cke_rsa(SSL *s, WPACKET *pkt) { unsigned char *encdata = NULL; EVP_PKEY *pkey = NULL; @@ -3066,9 +3012,8 @@ static int tls_construct_cke_rsa(SSL_CONNECTION *s, WPACKET *pkt) size_t enclen; unsigned char *pms = NULL; size_t pmslen = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - if (!received_server_cert(s)) { + if (s->session->peer == NULL) { /* * We should always have a server certificate with SSL_kRSA. */ @@ -3076,11 +3021,7 @@ static int tls_construct_cke_rsa(SSL_CONNECTION *s, WPACKET *pkt) return 0; } - if ((pkey = tls_get_peer_pkey(s)) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - + pkey = X509_get0_pubkey(s->session->peer); if (!EVP_PKEY_is_a(pkey, "RSA")) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; @@ -3089,14 +3030,14 @@ static int tls_construct_cke_rsa(SSL_CONNECTION *s, WPACKET *pkt) pmslen = SSL_MAX_MASTER_KEY_LENGTH; pms = OPENSSL_malloc(pmslen); if (pms == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } pms[0] = s->client_version >> 8; pms[1] = s->client_version & 0xff; - if (RAND_bytes_ex(sctx->libctx, pms + 2, pmslen - 2, 0) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_RAND_LIB); + if (RAND_bytes_ex(s->ctx->libctx, pms + 2, pmslen - 2, 0) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -3106,7 +3047,7 @@ static int tls_construct_cke_rsa(SSL_CONNECTION *s, WPACKET *pkt) goto err; } - pctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, pkey, sctx->propq); + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, pkey, s->ctx->propq); if (pctx == NULL || EVP_PKEY_encrypt_init(pctx) <= 0 || EVP_PKEY_encrypt(pctx, NULL, &enclen, pms, pmslen) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); @@ -3143,7 +3084,7 @@ static int tls_construct_cke_rsa(SSL_CONNECTION *s, WPACKET *pkt) return 0; } -static int tls_construct_cke_dhe(SSL_CONNECTION *s, WPACKET *pkt) +static int tls_construct_cke_dhe(SSL *s, WPACKET *pkt) { EVP_PKEY *ckey = NULL, *skey = NULL; unsigned char *keybytes = NULL; @@ -3206,7 +3147,7 @@ static int tls_construct_cke_dhe(SSL_CONNECTION *s, WPACKET *pkt) return ret; } -static int tls_construct_cke_ecdhe(SSL_CONNECTION *s, WPACKET *pkt) +static int tls_construct_cke_ecdhe(SSL *s, WPACKET *pkt) { unsigned char *encodedPoint = NULL; size_t encoded_pt_len = 0; @@ -3221,7 +3162,7 @@ static int tls_construct_cke_ecdhe(SSL_CONNECTION *s, WPACKET *pkt) ckey = ssl_generate_pkey(s, skey); if (ckey == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -3250,252 +3191,7 @@ static int tls_construct_cke_ecdhe(SSL_CONNECTION *s, WPACKET *pkt) return ret; } -static int tls_construct_cke_gost(SSL_CONNECTION *s, WPACKET *pkt) -{ -#ifndef OPENSSL_NO_GOST - /* GOST key exchange message creation */ - EVP_PKEY_CTX *pkey_ctx = NULL; - EVP_PKEY *pkey = NULL; - size_t msglen; - unsigned int md_len; - unsigned char shared_ukm[32], tmp[256]; - EVP_MD_CTX *ukm_hash = NULL; - int dgst_nid = NID_id_GostR3411_94; - unsigned char *pms = NULL; - size_t pmslen = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - - if ((s->s3.tmp.new_cipher->algorithm_auth & SSL_aGOST12) != 0) - dgst_nid = NID_id_GostR3411_2012_256; - - /* - * Get server certificate PKEY and create ctx from it - */ - if ((pkey = tls_get_peer_pkey(s)) == NULL) { - SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, - SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER); - return 0; - } - - pkey_ctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, - pkey, - sctx->propq); - if (pkey_ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - return 0; - } - /* - * If we have send a certificate, and certificate key - * parameters match those of server certificate, use - * certificate key for key exchange - */ - - /* Otherwise, generate ephemeral key pair */ - pmslen = 32; - pms = OPENSSL_malloc(pmslen); - if (pms == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); - goto err; - } - - if (EVP_PKEY_encrypt_init(pkey_ctx) <= 0 - /* Generate session key - */ - || RAND_bytes_ex(sctx->libctx, pms, pmslen, 0) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - }; - /* - * Compute shared IV and store it in algorithm-specific context - * data - */ - ukm_hash = EVP_MD_CTX_new(); - if (ukm_hash == NULL - || EVP_DigestInit(ukm_hash, EVP_get_digestbynid(dgst_nid)) <= 0 - || EVP_DigestUpdate(ukm_hash, s->s3.client_random, - SSL3_RANDOM_SIZE) <= 0 - || EVP_DigestUpdate(ukm_hash, s->s3.server_random, - SSL3_RANDOM_SIZE) <= 0 - || EVP_DigestFinal_ex(ukm_hash, shared_ukm, &md_len) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - EVP_MD_CTX_free(ukm_hash); - ukm_hash = NULL; - if (EVP_PKEY_CTX_ctrl(pkey_ctx, -1, EVP_PKEY_OP_ENCRYPT, - EVP_PKEY_CTRL_SET_IV, 8, shared_ukm) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_LIBRARY_BUG); - goto err; - } - /* Make GOST keytransport blob message */ - /* - * Encapsulate it into sequence - */ - msglen = 255; - if (EVP_PKEY_encrypt(pkey_ctx, tmp, &msglen, pms, pmslen) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_LIBRARY_BUG); - goto err; - } - - if (!WPACKET_put_bytes_u8(pkt, V_ASN1_SEQUENCE | V_ASN1_CONSTRUCTED) - || (msglen >= 0x80 && !WPACKET_put_bytes_u8(pkt, 0x81)) - || !WPACKET_sub_memcpy_u8(pkt, tmp, msglen)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - EVP_PKEY_CTX_free(pkey_ctx); - s->s3.tmp.pms = pms; - s->s3.tmp.pmslen = pmslen; - - return 1; - err: - EVP_PKEY_CTX_free(pkey_ctx); - OPENSSL_clear_free(pms, pmslen); - EVP_MD_CTX_free(ukm_hash); - return 0; -#else - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; -#endif -} - -#ifndef OPENSSL_NO_GOST -int ossl_gost18_cke_cipher_nid(const SSL_CONNECTION *s) -{ - if ((s->s3.tmp.new_cipher->algorithm_enc & SSL_MAGMA) != 0) - return NID_magma_ctr; - else if ((s->s3.tmp.new_cipher->algorithm_enc & SSL_KUZNYECHIK) != 0) - return NID_kuznyechik_ctr; - - return NID_undef; -} - -int ossl_gost_ukm(const SSL_CONNECTION *s, unsigned char *dgst_buf) -{ - EVP_MD_CTX *hash = NULL; - unsigned int md_len; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - const EVP_MD *md = ssl_evp_md_fetch(sctx->libctx, NID_id_GostR3411_2012_256, - sctx->propq); - - if (md == NULL) - return 0; - - if ((hash = EVP_MD_CTX_new()) == NULL - || EVP_DigestInit(hash, md) <= 0 - || EVP_DigestUpdate(hash, s->s3.client_random, SSL3_RANDOM_SIZE) <= 0 - || EVP_DigestUpdate(hash, s->s3.server_random, SSL3_RANDOM_SIZE) <= 0 - || EVP_DigestFinal_ex(hash, dgst_buf, &md_len) <= 0) { - EVP_MD_CTX_free(hash); - ssl_evp_md_free(md); - return 0; - } - - EVP_MD_CTX_free(hash); - ssl_evp_md_free(md); - return 1; -} -#endif - -static int tls_construct_cke_gost18(SSL_CONNECTION *s, WPACKET *pkt) -{ -#ifndef OPENSSL_NO_GOST - /* GOST 2018 key exchange message creation */ - unsigned char rnd_dgst[32]; - unsigned char *encdata = NULL; - EVP_PKEY_CTX *pkey_ctx = NULL; - EVP_PKEY *pkey; - unsigned char *pms = NULL; - size_t pmslen = 0; - size_t msglen; - int cipher_nid = ossl_gost18_cke_cipher_nid(s); - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - - if (cipher_nid == NID_undef) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (ossl_gost_ukm(s, rnd_dgst) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* Pre-master secret - random bytes */ - pmslen = 32; - pms = OPENSSL_malloc(pmslen); - if (pms == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); - goto err; - } - - if (RAND_bytes_ex(sctx->libctx, pms, pmslen, 0) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* Get server certificate PKEY and create ctx from it */ - if ((pkey = tls_get_peer_pkey(s)) == NULL) { - SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, - SSL_R_NO_GOST_CERTIFICATE_SENT_BY_PEER); - goto err; - } - - pkey_ctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, - pkey, - sctx->propq); - if (pkey_ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - goto err; - } - - if (EVP_PKEY_encrypt_init(pkey_ctx) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - }; - - /* Reuse EVP_PKEY_CTRL_SET_IV, make choice in engine code */ - if (EVP_PKEY_CTX_ctrl(pkey_ctx, -1, EVP_PKEY_OP_ENCRYPT, - EVP_PKEY_CTRL_SET_IV, 32, rnd_dgst) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_LIBRARY_BUG); - goto err; - } - - if (EVP_PKEY_CTX_ctrl(pkey_ctx, -1, EVP_PKEY_OP_ENCRYPT, - EVP_PKEY_CTRL_CIPHER, cipher_nid, NULL) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_LIBRARY_BUG); - goto err; - } - - if (EVP_PKEY_encrypt(pkey_ctx, NULL, &msglen, pms, pmslen) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - goto err; - } - - if (!WPACKET_allocate_bytes(pkt, msglen, &encdata) - || EVP_PKEY_encrypt(pkey_ctx, encdata, &msglen, pms, pmslen) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - goto err; - } - - EVP_PKEY_CTX_free(pkey_ctx); - pkey_ctx = NULL; - s->s3.tmp.pms = pms; - s->s3.tmp.pmslen = pmslen; - - return 1; - err: - EVP_PKEY_CTX_free(pkey_ctx); - OPENSSL_clear_free(pms, pmslen); - return 0; -#else - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; -#endif -} - -static int tls_construct_cke_srp(SSL_CONNECTION *s, WPACKET *pkt) +static int tls_construct_cke_srp(SSL *s, WPACKET *pkt) { #ifndef OPENSSL_NO_SRP unsigned char *abytes = NULL; @@ -3511,7 +3207,7 @@ static int tls_construct_cke_srp(SSL_CONNECTION *s, WPACKET *pkt) OPENSSL_free(s->session->srp_username); s->session->srp_username = OPENSSL_strdup(s->srp_ctx.login); if (s->session->srp_username == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } @@ -3522,8 +3218,7 @@ static int tls_construct_cke_srp(SSL_CONNECTION *s, WPACKET *pkt) #endif } -CON_FUNC_RETURN tls_construct_client_key_exchange(SSL_CONNECTION *s, - WPACKET *pkt) +int tls_construct_client_key_exchange(SSL *s, WPACKET *pkt) { unsigned long alg_k; @@ -3546,12 +3241,6 @@ CON_FUNC_RETURN tls_construct_client_key_exchange(SSL_CONNECTION *s, } else if (alg_k & (SSL_kECDHE | SSL_kECDHEPSK)) { if (!tls_construct_cke_ecdhe(s, pkt)) goto err; - } else if (alg_k & SSL_kGOST) { - if (!tls_construct_cke_gost(s, pkt)) - goto err; - } else if (alg_k & SSL_kGOST18) { - if (!tls_construct_cke_gost18(s, pkt)) - goto err; } else if (alg_k & SSL_kSRP) { if (!tls_construct_cke_srp(s, pkt)) goto err; @@ -3560,7 +3249,7 @@ CON_FUNC_RETURN tls_construct_client_key_exchange(SSL_CONNECTION *s, goto err; } - return CON_FUNC_SUCCESS; + return 1; err: OPENSSL_clear_free(s->s3.tmp.pms, s->s3.tmp.pmslen); s->s3.tmp.pms = NULL; @@ -3570,10 +3259,10 @@ CON_FUNC_RETURN tls_construct_client_key_exchange(SSL_CONNECTION *s, s->s3.tmp.psk = NULL; s->s3.tmp.psklen = 0; #endif - return CON_FUNC_ERROR; + return 0; } -int tls_client_key_exchange_post_work(SSL_CONNECTION *s) +int tls_client_key_exchange_post_work(SSL *s) { unsigned char *pms = NULL; size_t pmslen = 0; @@ -3593,7 +3282,7 @@ int tls_client_key_exchange_post_work(SSL_CONNECTION *s) #endif if (pms == NULL && !(s->s3.tmp.new_cipher->algorithm_mkey & SSL_kPSK)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_PASSED_INVALID_ARGUMENT); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } if (!ssl_generate_master_secret(s, pms, pmslen, 1)) { @@ -3607,11 +3296,10 @@ int tls_client_key_exchange_post_work(SSL_CONNECTION *s) pmslen = 0; #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { unsigned char sctpauthkey[64]; char labelbuffer[sizeof(DTLS1_SCTP_AUTH_LABEL)]; size_t labellen; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); /* * Add new shared key for SCTP-Auth, will be ignored if no SCTP @@ -3625,14 +3313,14 @@ int tls_client_key_exchange_post_work(SSL_CONNECTION *s) if (s->mode & SSL_MODE_DTLS_SCTP_LABEL_LENGTH_BUG) labellen += 1; - if (SSL_export_keying_material(ssl, sctpauthkey, + if (SSL_export_keying_material(s, sctpauthkey, sizeof(sctpauthkey), labelbuffer, labellen, NULL, 0, 0) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY, sizeof(sctpauthkey), sctpauthkey); } #endif @@ -3650,7 +3338,7 @@ int tls_client_key_exchange_post_work(SSL_CONNECTION *s) * cert exists, if we have a suitable digest for TLS 1.2 if static DH client * certificates can be used and optionally checks suitability for Suite B. */ -static int ssl3_check_client_certificate(SSL_CONNECTION *s) +static int ssl3_check_client_certificate(SSL *s) { /* If no suitable signature algorithm can't use certificate */ if (!tls_choose_sigalg(s, 0) || s->s3.tmp.sigalg == NULL) @@ -3665,17 +3353,16 @@ static int ssl3_check_client_certificate(SSL_CONNECTION *s) return 1; } -WORK_STATE tls_prepare_client_certificate(SSL_CONNECTION *s, WORK_STATE wst) +WORK_STATE tls_prepare_client_certificate(SSL *s, WORK_STATE wst) { X509 *x509 = NULL; EVP_PKEY *pkey = NULL; int i; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if (wst == WORK_MORE_A) { /* Let cert callback update client certificates if required */ if (s->cert->cert_cb) { - i = s->cert->cert_cb(ssl, s->cert->cert_cb_arg); + i = s->cert->cert_cb(s, s->cert->cert_cb_arg); if (i < 0) { s->rwstate = SSL_X509_LOOKUP; return WORK_MORE_A; @@ -3710,8 +3397,7 @@ WORK_STATE tls_prepare_client_certificate(SSL_CONNECTION *s, WORK_STATE wst) } s->rwstate = SSL_NOTHING; if ((i == 1) && (pkey != NULL) && (x509 != NULL)) { - if (!SSL_use_certificate(ssl, x509) - || !SSL_use_PrivateKey(ssl, pkey)) + if (!SSL_use_certificate(s, x509) || !SSL_use_PrivateKey(s, pkey)) i = 0; } else if (i == 1) { i = 0; @@ -3729,7 +3415,6 @@ WORK_STATE tls_prepare_client_certificate(SSL_CONNECTION *s, WORK_STATE wst) return WORK_FINISHED_CONTINUE; } else { s->s3.tmp.cert_req = 2; - s->ext.compress_certificate_from_peer[0] = TLSEXT_comp_cert_none; if (!ssl3_digest_cached_records(s, 0)) { /* SSLfatal() already called */ return WORK_ERROR; @@ -3737,10 +3422,6 @@ WORK_STATE tls_prepare_client_certificate(SSL_CONNECTION *s, WORK_STATE wst) } } - if (!SSL_CONNECTION_IS_TLS13(s) - || (s->options & SSL_OP_NO_TX_CERTIFICATE_COMPRESSION) != 0) - s->ext.compress_certificate_from_peer[0] = TLSEXT_comp_cert_none; - if (s->post_handshake_auth == SSL_PHA_REQUESTED) return WORK_FINISHED_STOP; return WORK_FINISHED_CONTINUE; @@ -3751,173 +3432,47 @@ WORK_STATE tls_prepare_client_certificate(SSL_CONNECTION *s, WORK_STATE wst) return WORK_ERROR; } -CON_FUNC_RETURN tls_construct_client_certificate(SSL_CONNECTION *s, - WPACKET *pkt) +int tls_construct_client_certificate(SSL *s, WPACKET *pkt) { - CERT_PKEY *cpk = NULL; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { if (s->pha_context == NULL) { /* no context available, add 0-length context */ if (!WPACKET_put_bytes_u8(pkt, 0)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } } else if (!WPACKET_sub_memcpy_u8(pkt, s->pha_context, s->pha_context_len)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } } - if (s->s3.tmp.cert_req != 2) - cpk = s->cert->key; - switch (s->ext.client_cert_type) { - case TLSEXT_cert_type_rpk: - if (!tls_output_rpk(s, pkt, cpk)) { - /* SSLfatal() already called */ - return CON_FUNC_ERROR; - } - break; - case TLSEXT_cert_type_x509: - if (!ssl3_output_cert_chain(s, pkt, cpk, 0)) { - /* SSLfatal() already called */ - return CON_FUNC_ERROR; - } - break; - default: - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + if (!ssl3_output_cert_chain(s, pkt, + (s->s3.tmp.cert_req == 2) ? NULL + : s->cert->key)) { + /* SSLfatal() already called */ + return 0; } - /* - * If we attempted to write early data or we're in middlebox compat mode - * then we deferred changing the handshake write keys to the last possible - * moment. We need to do it now. - */ - if (SSL_CONNECTION_IS_TLS13(s) + if (SSL_IS_TLS13(s) && SSL_IS_FIRST_HANDSHAKE(s) - && (s->early_data_state != SSL_EARLY_DATA_NONE - || (s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0) - && (!ssl->method->ssl3_enc->change_cipher_state(s, + && (!s->method->ssl3_enc->change_cipher_state(s, SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_WRITE))) { /* * This is a fatal error, which leaves enc_write_ctx in an inconsistent * state and thus ssl3_send_alert may crash. */ SSLfatal(s, SSL_AD_NO_ALERT, SSL_R_CANNOT_CHANGE_CIPHER); - return CON_FUNC_ERROR; - } - - return CON_FUNC_SUCCESS; -} - -#ifndef OPENSSL_NO_COMP_ALG -CON_FUNC_RETURN tls_construct_client_compressed_certificate(SSL_CONNECTION *sc, - WPACKET *pkt) -{ - SSL *ssl = SSL_CONNECTION_GET_SSL(sc); - WPACKET tmppkt; - BUF_MEM *buf = NULL; - size_t length; - size_t max_length; - COMP_METHOD *method; - COMP_CTX *comp = NULL; - int comp_len; - int ret = 0; - int alg = sc->ext.compress_certificate_from_peer[0]; - - /* Note that sc->s3.tmp.cert_req == 2 is checked in write transition */ - - if ((buf = BUF_MEM_new()) == NULL || !WPACKET_init(&tmppkt, buf)) - goto err; - - /* Use the |tmppkt| for the to-be-compressed data */ - if (sc->pha_context == NULL) { - /* no context available, add 0-length context */ - if (!WPACKET_put_bytes_u8(&tmppkt, 0)) - goto err; - } else if (!WPACKET_sub_memcpy_u8(&tmppkt, sc->pha_context, sc->pha_context_len)) - goto err; - - if (!ssl3_output_cert_chain(sc, &tmppkt, sc->cert->key, 0)) { - /* SSLfatal() already called */ - goto out; - } - - /* continue with the real |pkt| */ - if (!WPACKET_put_bytes_u16(pkt, alg) - || !WPACKET_get_total_written(&tmppkt, &length) - || !WPACKET_put_bytes_u24(pkt, length)) - goto err; - - switch (alg) { - case TLSEXT_comp_cert_zlib: - method = COMP_zlib_oneshot(); - break; - case TLSEXT_comp_cert_brotli: - method = COMP_brotli_oneshot(); - break; - case TLSEXT_comp_cert_zstd: - method = COMP_zstd_oneshot(); - break; - default: - goto err; - } - max_length = ossl_calculate_comp_expansion(alg, length); - - if ((comp = COMP_CTX_new(method)) == NULL - || !WPACKET_start_sub_packet_u24(pkt) - || !WPACKET_reserve_bytes(pkt, max_length, NULL)) - goto err; - - comp_len = COMP_compress_block(comp, WPACKET_get_curr(pkt), max_length, - (unsigned char *)buf->data, length); - if (comp_len <= 0) - goto err; - - if (!WPACKET_allocate_bytes(pkt, comp_len, NULL) - || !WPACKET_close(pkt)) - goto err; - - /* - * If we attempted to write early data or we're in middlebox compat mode - * then we deferred changing the handshake write keys to the last possible - * moment. We need to do it now. - */ - if (SSL_IS_FIRST_HANDSHAKE(sc) - && (sc->early_data_state != SSL_EARLY_DATA_NONE - || (sc->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0) - && (!ssl->method->ssl3_enc->change_cipher_state(sc, - SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_WRITE))) { - /* - * This is a fatal error, which leaves sc->enc_write_ctx in an - * inconsistent state and thus ssl3_send_alert may crash. - */ - SSLfatal(sc, SSL_AD_NO_ALERT, SSL_R_CANNOT_CHANGE_CIPHER); - goto out; + return 0; } - ret = 1; - goto out; - err: - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - out: - if (buf != NULL) { - /* If |buf| is NULL, then |tmppkt| could not have been initialized */ - WPACKET_cleanup(&tmppkt); - } - BUF_MEM_free(buf); - COMP_CTX_free(comp); - return ret; + return 1; } -#endif -int ssl3_check_cert_and_algorithm(SSL_CONNECTION *s) +int ssl3_check_cert_and_algorithm(SSL *s) { const SSL_CERT_LOOKUP *clu; size_t idx; long alg_k, alg_a; - EVP_PKEY *pkey; alg_k = s->s3.tmp.new_cipher->algorithm_mkey; alg_a = s->s3.tmp.new_cipher->algorithm_auth; @@ -3927,8 +3482,7 @@ int ssl3_check_cert_and_algorithm(SSL_CONNECTION *s) return 1; /* This is the passed certificate */ - pkey = tls_get_peer_pkey(s); - clu = ssl_cert_lookup_by_pkey(pkey, &idx, SSL_CONNECTION_GET_CTX(s)); + clu = ssl_cert_lookup_by_pkey(X509_get0_pubkey(s->session->peer), &idx); /* Check certificate is recognised and suitable for cipher */ if (clu == NULL || (alg_a & clu->amask) == 0) { @@ -3936,6 +3490,13 @@ int ssl3_check_cert_and_algorithm(SSL_CONNECTION *s) return 0; } + if (clu->amask & SSL_aECDSA) { + if (ssl_check_srvr_ecc_cert_and_alg(s->session->peer, s)) + return 1; + SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_BAD_ECC_CERT); + return 0; + } + if (alg_k & (SSL_kRSA | SSL_kRSAPSK) && idx != SSL_PKEY_RSA) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_MISSING_RSA_ENCRYPTING_CERT); @@ -3947,22 +3508,11 @@ int ssl3_check_cert_and_algorithm(SSL_CONNECTION *s) return 0; } - /* Early out to skip the checks below */ - if (s->session->peer_rpk != NULL) - return 1; - - if (clu->amask & SSL_aECDSA) { - if (ssl_check_srvr_ecc_cert_and_alg(s->session->peer, s)) - return 1; - SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_BAD_ECC_CERT); - return 0; - } - return 1; } #ifndef OPENSSL_NO_NEXTPROTONEG -CON_FUNC_RETURN tls_construct_next_proto(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_next_proto(SSL *s, WPACKET *pkt) { size_t len, padding_len; unsigned char *padding = NULL; @@ -3973,19 +3523,17 @@ CON_FUNC_RETURN tls_construct_next_proto(SSL_CONNECTION *s, WPACKET *pkt) if (!WPACKET_sub_memcpy_u8(pkt, s->ext.npn, len) || !WPACKET_sub_allocate_bytes_u8(pkt, padding_len, &padding)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } memset(padding, 0, padding_len); - return CON_FUNC_SUCCESS; + return 1; } #endif -MSG_PROCESS_RETURN tls_process_hello_req(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_hello_req(SSL *s, PACKET *pkt) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - if (PACKET_remaining(pkt) > 0) { /* should contain no data */ SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); @@ -4004,16 +3552,15 @@ MSG_PROCESS_RETURN tls_process_hello_req(SSL_CONNECTION *s, PACKET *pkt) * HelloRequest it will do a full handshake. Either behaviour is reasonable * but doing one for TLS and another for DTLS is odd. */ - if (SSL_CONNECTION_IS_DTLS(s)) - SSL_renegotiate(ssl); + if (SSL_IS_DTLS(s)) + SSL_renegotiate(s); else - SSL_renegotiate_abbreviated(ssl); + SSL_renegotiate_abbreviated(s); return MSG_PROCESS_FINISHED_READING; } -static MSG_PROCESS_RETURN tls_process_encrypted_extensions(SSL_CONNECTION *s, - PACKET *pkt) +static MSG_PROCESS_RETURN tls_process_encrypted_extensions(SSL *s, PACKET *pkt) { PACKET extensions; RAW_EXTENSION *rawexts = NULL; @@ -4041,32 +3588,26 @@ static MSG_PROCESS_RETURN tls_process_encrypted_extensions(SSL_CONNECTION *s, return MSG_PROCESS_ERROR; } -int ssl_do_client_cert_cb(SSL_CONNECTION *s, X509 **px509, EVP_PKEY **ppkey) +int ssl_do_client_cert_cb(SSL *s, X509 **px509, EVP_PKEY **ppkey) { int i = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - #ifndef OPENSSL_NO_ENGINE - if (sctx->client_cert_engine) { + if (s->ctx->client_cert_engine) { i = tls_engine_load_ssl_client_cert(s, px509, ppkey); if (i != 0) return i; } #endif - if (sctx->client_cert_cb) - i = sctx->client_cert_cb(SSL_CONNECTION_GET_SSL(s), px509, ppkey); + if (s->ctx->client_cert_cb) + i = s->ctx->client_cert_cb(s, px509, ppkey); return i; } -int ssl_cipher_list_to_bytes(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *sk, - WPACKET *pkt) +int ssl_cipher_list_to_bytes(SSL *s, STACK_OF(SSL_CIPHER) *sk, WPACKET *pkt) { int i; size_t totlen = 0, len, maxlen, maxverok = 0; - int empty_reneg_info_scsv = !s->renegotiate - && (SSL_CONNECTION_IS_DTLS(s) - || s->min_proto_version < TLS1_3_VERSION); - SSL *ssl = SSL_CONNECTION_GET_SSL(s); + int empty_reneg_info_scsv = !s->renegotiate; /* Set disabled masks for this session */ if (!ssl_set_client_disabled(s)) { @@ -4088,7 +3629,7 @@ int ssl_cipher_list_to_bytes(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *sk, * chop number of supported ciphers to keep it well below this if we * use TLS v1.2 */ - if (TLS1_get_version(ssl) >= TLS1_2_VERSION) + if (TLS1_get_version(s) >= TLS1_2_VERSION) maxlen = OPENSSL_MAX_TLS1_2_CIPHER_LENGTH & ~1; else #endif @@ -4108,19 +3649,22 @@ int ssl_cipher_list_to_bytes(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *sk, if (ssl_cipher_disabled(s, c, SSL_SECOP_CIPHER_SUPPORTED, 0)) continue; - if (!ssl->method->put_cipher_by_char(c, pkt, &len)) { + if (!s->method->put_cipher_by_char(c, pkt, &len)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } /* Sanity check that the maximum version we offer has ciphers enabled */ if (!maxverok) { - int minproto = SSL_CONNECTION_IS_DTLS(s) ? c->min_dtls : c->min_tls; - int maxproto = SSL_CONNECTION_IS_DTLS(s) ? c->max_dtls : c->max_tls; - - if (ssl_version_cmp(s, maxproto, s->s3.tmp.max_ver) >= 0 - && ssl_version_cmp(s, minproto, s->s3.tmp.max_ver) <= 0) - maxverok = 1; + if (SSL_IS_DTLS(s)) { + if (DTLS_VERSION_GE(c->max_dtls, s->s3.tmp.max_ver) + && DTLS_VERSION_LE(c->min_dtls, s->s3.tmp.max_ver)) + maxverok = 1; + } else { + if (c->max_tls >= s->s3.tmp.max_ver + && c->min_tls <= s->s3.tmp.max_ver) + maxverok = 1; + } } totlen += len; @@ -4139,19 +3683,19 @@ int ssl_cipher_list_to_bytes(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *sk, if (totlen != 0) { if (empty_reneg_info_scsv) { - static const SSL_CIPHER scsv = { + static SSL_CIPHER scsv = { 0, NULL, NULL, SSL3_CK_SCSV, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - if (!ssl->method->put_cipher_by_char(&scsv, pkt, &len)) { + if (!s->method->put_cipher_by_char(&scsv, pkt, &len)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } } if (s->mode & SSL_MODE_SEND_FALLBACK_SCSV) { - static const SSL_CIPHER scsv = { + static SSL_CIPHER scsv = { 0, NULL, NULL, SSL3_CK_FALLBACK_SCSV, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - if (!ssl->method->put_cipher_by_char(&scsv, pkt, &len)) { + if (!s->method->put_cipher_by_char(&scsv, pkt, &len)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } @@ -4161,14 +3705,29 @@ int ssl_cipher_list_to_bytes(SSL_CONNECTION *s, STACK_OF(SSL_CIPHER) *sk, return 1; } -CON_FUNC_RETURN tls_construct_end_of_early_data(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_end_of_early_data(SSL *s, WPACKET *pkt) { if (s->early_data_state != SSL_EARLY_DATA_WRITE_RETRY && s->early_data_state != SSL_EARLY_DATA_FINISHED_WRITING) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - return CON_FUNC_ERROR; + return 0; } s->early_data_state = SSL_EARLY_DATA_FINISHED_WRITING; - return CON_FUNC_SUCCESS; + return 1; } + +#ifndef OPENSSL_NO_CERT_COMPRESSION +int tls_construct_client_compressed_certificate(SSL *s, WPACKET *pkt) +{ + return tls_construct_compressed_certificate(s, pkt, + tls_construct_client_certificate); +} + +MSG_PROCESS_RETURN tls_process_server_compressed_certificate(SSL *s, + PACKET *pkt) +{ + return tls_process_compressed_certificate(s, pkt, + tls_process_server_certificate); +} +#endif diff --git a/openssl/src/ssl/statem/statem_dtls.c b/openssl/src/ssl/statem/statem_dtls.c index b37ac80a6..2838d51bd 100644 --- a/openssl/src/ssl/statem/statem_dtls.c +++ b/openssl/src/ssl/statem/statem_dtls.c @@ -1,5 +1,5 @@ /* - * Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2005-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -7,7 +7,6 @@ * https://www.openssl.org/source/license.html */ -#include #include #include #include @@ -39,22 +38,20 @@ if (is_complete) for (ii = (((msg_len) - 1) >> 3) - 1; ii >= 0 ; ii--) \ if (bitmask[ii] != 0xff) { is_complete = 0; break; } } -static const unsigned char bitmask_start_values[] = +static unsigned char bitmask_start_values[] = { 0xff, 0xfe, 0xfc, 0xf8, 0xf0, 0xe0, 0xc0, 0x80 }; -static const unsigned char bitmask_end_values[] = +static unsigned char bitmask_end_values[] = { 0xff, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f }; -static void dtls1_fix_message_header(SSL_CONNECTION *s, size_t frag_off, +static void dtls1_fix_message_header(SSL *s, size_t frag_off, size_t frag_len); -static unsigned char *dtls1_write_message_header(SSL_CONNECTION *s, - unsigned char *p); -static void dtls1_set_message_header_int(SSL_CONNECTION *s, unsigned char mt, +static unsigned char *dtls1_write_message_header(SSL *s, unsigned char *p); +static void dtls1_set_message_header_int(SSL *s, unsigned char mt, size_t len, unsigned short seq_num, size_t frag_off, size_t frag_len); -static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, - size_t *len); +static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len); static hm_fragment *dtls1_hm_fragment_new(size_t frag_len, int reassembly) { @@ -62,11 +59,14 @@ static hm_fragment *dtls1_hm_fragment_new(size_t frag_len, int reassembly) unsigned char *buf = NULL; unsigned char *bitmask = NULL; - if ((frag = OPENSSL_zalloc(sizeof(*frag))) == NULL) + if ((frag = OPENSSL_malloc(sizeof(*frag))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return NULL; + } if (frag_len) { if ((buf = OPENSSL_malloc(frag_len)) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); OPENSSL_free(frag); return NULL; } @@ -79,6 +79,7 @@ static hm_fragment *dtls1_hm_fragment_new(size_t frag_len, int reassembly) if (reassembly) { bitmask = OPENSSL_zalloc(RSMBLY_BITMASK_SIZE(frag_len)); if (bitmask == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); OPENSSL_free(buf); OPENSSL_free(frag); return NULL; @@ -94,7 +95,11 @@ void dtls1_hm_fragment_free(hm_fragment *frag) { if (!frag) return; - + if (frag->msg_header.is_ccs) { + EVP_CIPHER_CTX_free(frag->msg_header. + saved_retransmit_state.enc_write_ctx); + EVP_MD_CTX_free(frag->msg_header.saved_retransmit_state.write_hash); + } OPENSSL_free(frag->fragment); OPENSSL_free(frag->reassembly); OPENSSL_free(frag); @@ -104,14 +109,13 @@ void dtls1_hm_fragment_free(hm_fragment *frag) * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or * SSL3_RT_CHANGE_CIPHER_SPEC) */ -int dtls1_do_write(SSL_CONNECTION *s, uint8_t type) +int dtls1_do_write(SSL *s, int type) { int ret; size_t written; size_t curr_mtu; int retry = 1; - size_t len, frag_off, overhead, used_len; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); + size_t len, frag_off, mac_size, blocksize, used_len; if (!dtls1_query_mtu(s)) return -1; @@ -126,7 +130,21 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t type) return -1; } - overhead = s->rlayer.wrlmethod->get_max_record_overhead(s->rlayer.wrl); + if (s->write_hash) { + if (s->enc_write_ctx + && (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(s->enc_write_ctx)) & + EVP_CIPH_FLAG_AEAD_CIPHER) != 0) + mac_size = 0; + else + mac_size = EVP_MD_CTX_get_size(s->write_hash); + } else + mac_size = 0; + + if (s->enc_write_ctx && + (EVP_CIPHER_CTX_get_mode(s->enc_write_ctx) == EVP_CIPH_CBC_MODE)) + blocksize = 2 * EVP_CIPHER_CTX_get_block_size(s->enc_write_ctx); + else + blocksize = 0; frag_off = 0; s->rwstate = SSL_NOTHING; @@ -167,7 +185,8 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t type) } } - used_len = BIO_wpending(s->wbio) + overhead; + used_len = BIO_wpending(s->wbio) + DTLS1_RT_HEADER_LENGTH + + mac_size + blocksize; if (s->d1->mtu > used_len) curr_mtu = s->d1->mtu - used_len; else @@ -182,8 +201,9 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t type) s->rwstate = SSL_WRITING; return ret; } - if (s->d1->mtu > overhead + DTLS1_HM_HEADER_LENGTH) { - curr_mtu = s->d1->mtu - overhead; + used_len = DTLS1_RT_HEADER_LENGTH + mac_size + blocksize; + if (s->d1->mtu > used_len + DTLS1_HM_HEADER_LENGTH) { + curr_mtu = s->d1->mtu - used_len; } else { /* Shouldn't happen */ return -1; @@ -228,9 +248,9 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t type) * retransmit anything. continue as if everything is fine and * wait for an alert to handle the retransmit */ - if (retry && BIO_ctrl(SSL_get_wbio(ssl), + if (retry && BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_MTU_EXCEEDED, 0, NULL) > 0) { - if (!(SSL_get_options(ssl) & SSL_OP_NO_QUERY_MTU)) { + if (!(SSL_get_options(s) & SSL_OP_NO_QUERY_MTU)) { if (!dtls1_query_mtu(s)) return -1; /* Have one more go */ @@ -249,16 +269,6 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t type) if (!ossl_assert(len == written)) return -1; - /* - * We should not exceed the MTU size. If compression is in use - * then the max record overhead calculation is unreliable so we do - * not check in that case. We use assert rather than ossl_assert - * because in a production build, if this assert were ever to fail, - * then the best thing to do is probably carry on regardless. - */ - assert(s->s3.tmp.new_compression != NULL - || BIO_wpending(s->wbio) <= (int)s->d1->mtu); - if (type == SSL3_RT_HANDSHAKE && !s->d1->retransmitting) { /* * should not be done for 'Hello Request's, but in that case @@ -293,7 +303,7 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t type) if (written == s->init_num) { if (s->msg_callback) s->msg_callback(1, s->version, type, s->init_buf->data, - (size_t)(s->init_off + s->init_num), ssl, + (size_t)(s->init_off + s->init_num), s, s->msg_callback_arg); s->init_off = 0; /* done writing this message */ @@ -318,7 +328,7 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t type) return 0; } -int dtls_get_message(SSL_CONNECTION *s, int *mt) +int dtls_get_message(SSL *s, int *mt) { struct hm_header_st *msg_hdr; unsigned char *p; @@ -346,8 +356,7 @@ int dtls_get_message(SSL_CONNECTION *s, int *mt) if (*mt == SSL3_MT_CHANGE_CIPHER_SPEC) { if (s->msg_callback) { s->msg_callback(0, s->version, SSL3_RT_CHANGE_CIPHER_SPEC, - p, 1, SSL_CONNECTION_GET_SSL(s), - s->msg_callback_arg); + p, 1, s, s->msg_callback_arg); } /* * This isn't a real handshake message so skip the processing below. @@ -378,7 +387,7 @@ int dtls_get_message(SSL_CONNECTION *s, int *mt) * DTLS to do any further processing it wants at the same point that TLS would * be asked for the message body. */ -int dtls_get_message_body(SSL_CONNECTION *s, size_t *len) +int dtls_get_message_body(SSL *s, size_t *len) { unsigned char *msg = (unsigned char *)s->init_buf->data; size_t msg_len = s->init_num + DTLS1_HM_HEADER_LENGTH; @@ -407,7 +416,7 @@ int dtls_get_message_body(SSL_CONNECTION *s, size_t *len) if (s->msg_callback) s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, s->init_buf->data, s->init_num + DTLS1_HM_HEADER_LENGTH, - SSL_CONNECTION_GET_SSL(s), s->msg_callback_arg); + s, s->msg_callback_arg); end: *len = s->init_num; @@ -419,7 +428,7 @@ int dtls_get_message_body(SSL_CONNECTION *s, size_t *len) * permitted in a DTLS handshake message for |s|. The minimum is 16KB, but * may be greater if the maximum certificate list size requires it. */ -static size_t dtls1_max_handshake_message_len(const SSL_CONNECTION *s) +static size_t dtls1_max_handshake_message_len(const SSL *s) { size_t max_len = DTLS1_HM_HEADER_LENGTH + SSL3_RT_MAX_ENCRYPTED_LENGTH; if (max_len < s->max_cert_list) @@ -427,8 +436,7 @@ static size_t dtls1_max_handshake_message_len(const SSL_CONNECTION *s) return max_len; } -static int dtls1_preprocess_fragment(SSL_CONNECTION *s, - struct hm_header_st *msg_hdr) +static int dtls1_preprocess_fragment(SSL *s, struct hm_header_st *msg_hdr) { size_t frag_off, frag_len, msg_len; @@ -474,7 +482,7 @@ static int dtls1_preprocess_fragment(SSL_CONNECTION *s, * Returns 1 if there is a buffered fragment available, 0 if not, or -1 on a * fatal error. */ -static int dtls1_retrieve_buffered_fragment(SSL_CONNECTION *s, size_t *len) +static int dtls1_retrieve_buffered_fragment(SSL *s, size_t *len) { /*- * (0) check whether the desired fragment is available @@ -483,64 +491,23 @@ static int dtls1_retrieve_buffered_fragment(SSL_CONNECTION *s, size_t *len) * (2) update s->init_num */ pitem *item; - piterator iter; hm_fragment *frag; int ret; - int chretran = 0; - iter = pqueue_iterator(s->d1->buffered_messages); do { - item = pqueue_next(&iter); + item = pqueue_peek(s->d1->buffered_messages); if (item == NULL) return 0; frag = (hm_fragment *)item->data; if (frag->msg_header.seq < s->d1->handshake_read_seq) { - pitem *next; - hm_fragment *nextfrag; - - if (!s->server - || frag->msg_header.seq != 0 - || s->d1->handshake_read_seq != 1 - || s->statem.hand_state != DTLS_ST_SW_HELLO_VERIFY_REQUEST) { - /* - * This is a stale message that has been buffered so clear it. - * It is safe to pop this message from the queue even though - * we have an active iterator - */ - pqueue_pop(s->d1->buffered_messages); - dtls1_hm_fragment_free(frag); - pitem_free(item); - item = NULL; - frag = NULL; - } else { - /* - * We have fragments for a ClientHello without a cookie, - * even though we have sent a HelloVerifyRequest. It is possible - * that the HelloVerifyRequest got lost and this is a - * retransmission of the original ClientHello - */ - next = pqueue_next(&iter); - if (next != NULL) { - nextfrag = (hm_fragment *)next->data; - if (nextfrag->msg_header.seq == s->d1->handshake_read_seq) { - /* - * We have fragments for both a ClientHello without - * cookie and one with. Ditch the one without. - */ - pqueue_pop(s->d1->buffered_messages); - dtls1_hm_fragment_free(frag); - pitem_free(item); - item = next; - frag = nextfrag; - } else { - chretran = 1; - } - } else { - chretran = 1; - } - } + /* This is a stale message that has been buffered so clear it */ + pqueue_pop(s->d1->buffered_messages); + dtls1_hm_fragment_free(frag); + pitem_free(item); + item = NULL; + frag = NULL; } } while (item == NULL); @@ -548,7 +515,7 @@ static int dtls1_retrieve_buffered_fragment(SSL_CONNECTION *s, size_t *len) if (frag->reassembly != NULL) return 0; - if (s->d1->handshake_read_seq == frag->msg_header.seq || chretran) { + if (s->d1->handshake_read_seq == frag->msg_header.seq) { size_t frag_len = frag->msg_header.frag_len; pqueue_pop(s->d1->buffered_messages); @@ -566,16 +533,6 @@ static int dtls1_retrieve_buffered_fragment(SSL_CONNECTION *s, size_t *len) pitem_free(item); if (ret) { - if (chretran) { - /* - * We got a new ClientHello with a message sequence of 0. - * Reset the read/write sequences back to the beginning. - * We process it like this is the first time we've seen a - * ClientHello from the client. - */ - s->d1->handshake_read_seq = 0; - s->d1->next_handshake_write_seq = 0; - } *len = frag_len; return 1; } @@ -588,8 +545,8 @@ static int dtls1_retrieve_buffered_fragment(SSL_CONNECTION *s, size_t *len) } } -static int dtls1_reassemble_fragment(SSL_CONNECTION *s, - const struct hm_header_st *msg_hdr) +static int +dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr) { hm_fragment *frag = NULL; pitem *item = NULL; @@ -597,7 +554,6 @@ static int dtls1_reassemble_fragment(SSL_CONNECTION *s, unsigned char seq64be[8]; size_t frag_len = msg_hdr->frag_len; size_t readbytes; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if ((msg_hdr->frag_off + frag_len) > msg_hdr->msg_len || msg_hdr->msg_len > dtls1_max_handshake_message_len(s)) @@ -638,11 +594,11 @@ static int dtls1_reassemble_fragment(SSL_CONNECTION *s, unsigned char devnull[256]; while (frag_len) { - i = ssl->method->ssl_read_bytes(ssl, SSL3_RT_HANDSHAKE, NULL, - devnull, - frag_len > - sizeof(devnull) ? sizeof(devnull) : - frag_len, 0, &readbytes); + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL, + devnull, + frag_len > + sizeof(devnull) ? sizeof(devnull) : + frag_len, 0, &readbytes); if (i <= 0) goto err; frag_len -= readbytes; @@ -651,9 +607,9 @@ static int dtls1_reassemble_fragment(SSL_CONNECTION *s, } /* read the body of the fragment (header has already been read */ - i = ssl->method->ssl_read_bytes(ssl, SSL3_RT_HANDSHAKE, NULL, - frag->fragment + msg_hdr->frag_off, - frag_len, 0, &readbytes); + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL, + frag->fragment + msg_hdr->frag_off, + frag_len, 0, &readbytes); if (i <= 0 || readbytes != frag_len) i = -1; if (i <= 0) @@ -698,8 +654,8 @@ static int dtls1_reassemble_fragment(SSL_CONNECTION *s, return -1; } -static int dtls1_process_out_of_seq_message(SSL_CONNECTION *s, - const struct hm_header_st *msg_hdr) +static int +dtls1_process_out_of_seq_message(SSL *s, const struct hm_header_st *msg_hdr) { int i = -1; hm_fragment *frag = NULL; @@ -707,7 +663,6 @@ static int dtls1_process_out_of_seq_message(SSL_CONNECTION *s, unsigned char seq64be[8]; size_t frag_len = msg_hdr->frag_len; size_t readbytes; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if ((msg_hdr->frag_off + frag_len) > msg_hdr->msg_len) goto err; @@ -736,11 +691,11 @@ static int dtls1_process_out_of_seq_message(SSL_CONNECTION *s, unsigned char devnull[256]; while (frag_len) { - i = ssl->method->ssl_read_bytes(ssl, SSL3_RT_HANDSHAKE, NULL, - devnull, - frag_len > - sizeof(devnull) ? sizeof(devnull) : - frag_len, 0, &readbytes); + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL, + devnull, + frag_len > + sizeof(devnull) ? sizeof(devnull) : + frag_len, 0, &readbytes); if (i <= 0) goto err; frag_len -= readbytes; @@ -763,9 +718,9 @@ static int dtls1_process_out_of_seq_message(SSL_CONNECTION *s, /* * read the body of the fragment (header has already been read */ - i = ssl->method->ssl_read_bytes(ssl, SSL3_RT_HANDSHAKE, NULL, - frag->fragment, frag_len, 0, - &readbytes); + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL, + frag->fragment, frag_len, 0, + &readbytes); if (i<=0 || readbytes != frag_len) i = -1; if (i <= 0) @@ -797,22 +752,16 @@ static int dtls1_process_out_of_seq_message(SSL_CONNECTION *s, return 0; } -static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, - size_t *len) +static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len) { + unsigned char wire[DTLS1_HM_HEADER_LENGTH]; size_t mlen, frag_off, frag_len; - int i, ret; - uint8_t recvd_type; + int i, ret, recvd_type; struct hm_header_st msg_hdr; size_t readbytes; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - int chretran = 0; - unsigned char *p; *errtype = 0; - p = (unsigned char *)s->init_buf->data; - redo: /* see if we have the required fragment already */ ret = dtls1_retrieve_buffered_fragment(s, &frag_len); @@ -827,20 +776,21 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, } /* read handshake message header */ - i = ssl->method->ssl_read_bytes(ssl, SSL3_RT_HANDSHAKE, &recvd_type, p, - DTLS1_HM_HEADER_LENGTH, 0, &readbytes); + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, &recvd_type, wire, + DTLS1_HM_HEADER_LENGTH, 0, &readbytes); if (i <= 0) { /* nbio, or an error */ s->rwstate = SSL_READING; *len = 0; return 0; } if (recvd_type == SSL3_RT_CHANGE_CIPHER_SPEC) { - if (p[0] != SSL3_MT_CCS) { + if (wire[0] != SSL3_MT_CCS) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_BAD_CHANGE_CIPHER_SPEC); goto f_err; } + memcpy(s->init_buf->data, wire, readbytes); s->init_num = readbytes - 1; s->init_msg = s->init_buf->data + 1; s->s3.tmp.message_type = SSL3_MT_CHANGE_CIPHER_SPEC; @@ -856,7 +806,7 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, } /* parse the message fragment header */ - dtls1_get_message_header(p, &msg_hdr); + dtls1_get_message_header(wire, &msg_hdr); mlen = msg_hdr.msg_len; frag_off = msg_hdr.frag_off; @@ -866,7 +816,7 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, * We must have at least frag_len bytes left in the record to be read. * Fragments must not span records. */ - if (frag_len > s->rlayer.tlsrecs[s->rlayer.curr_rec].length) { + if (frag_len > RECORD_LAYER_get_rrec_length(&s->rlayer)) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_LENGTH); goto f_err; } @@ -878,20 +828,8 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, * although we're still expecting seq 0 (ClientHello) */ if (msg_hdr.seq != s->d1->handshake_read_seq) { - if (!s->server - || msg_hdr.seq != 0 - || s->d1->handshake_read_seq != 1 - || p[0] != SSL3_MT_CLIENT_HELLO - || s->statem.hand_state != DTLS_ST_SW_HELLO_VERIFY_REQUEST) { - *errtype = dtls1_process_out_of_seq_message(s, &msg_hdr); - return 0; - } - /* - * We received a ClientHello and sent back a HelloVerifyRequest. We - * now seem to have received a retransmitted initial ClientHello. That - * is allowed (possibly our HelloVerifyRequest got lost). - */ - chretran = 1; + *errtype = dtls1_process_out_of_seq_message(s, &msg_hdr); + return 0; } if (frag_len && frag_len < mlen) { @@ -902,16 +840,16 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, if (!s->server && s->d1->r_msg_hdr.frag_off == 0 && s->statem.hand_state != TLS_ST_OK - && p[0] == SSL3_MT_HELLO_REQUEST) { + && wire[0] == SSL3_MT_HELLO_REQUEST) { /* * The server may always send 'Hello Request' messages -- we are * doing a handshake anyway now, so ignore them if their format is * correct. Does not count for 'Finished' MAC. */ - if (p[1] == 0 && p[2] == 0 && p[3] == 0) { + if (wire[1] == 0 && wire[2] == 0 && wire[3] == 0) { if (s->msg_callback) s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, - p, DTLS1_HM_HEADER_LENGTH, ssl, + wire, DTLS1_HM_HEADER_LENGTH, s, s->msg_callback_arg); s->init_num = 0; @@ -929,10 +867,11 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, } if (frag_len > 0) { - p += DTLS1_HM_HEADER_LENGTH; + unsigned char *p = + (unsigned char *)s->init_buf->data + DTLS1_HM_HEADER_LENGTH; - i = ssl->method->ssl_read_bytes(ssl, SSL3_RT_HANDSHAKE, NULL, - &p[frag_off], frag_len, 0, &readbytes); + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL, + &p[frag_off], frag_len, 0, &readbytes); /* * This shouldn't ever fail due to NBIO because we already checked @@ -956,17 +895,6 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, goto f_err; } - if (chretran) { - /* - * We got a new ClientHello with a message sequence of 0. - * Reset the read/write sequences back to the beginning. - * We process it like this is the first time we've seen a ClientHello - * from the client. - */ - s->d1->handshake_read_seq = 0; - s->d1->next_handshake_write_seq = 0; - } - /* * Note that s->init_num is *not* used as current offset in * s->init_buf->data, but as a counter summing up fragments' lengths: as @@ -984,23 +912,25 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype, /*- * for these 2 messages, we need to + * ssl->enc_read_ctx re-init + * ssl->rlayer.read_sequence zero + * ssl->s3.read_mac_secret re-init * ssl->session->read_sym_enc assign * ssl->session->read_compression assign * ssl->session->read_hash assign */ -CON_FUNC_RETURN dtls_construct_change_cipher_spec(SSL_CONNECTION *s, - WPACKET *pkt) +int dtls_construct_change_cipher_spec(SSL *s, WPACKET *pkt) { if (s->version == DTLS1_BAD_VER) { s->d1->next_handshake_write_seq++; if (!WPACKET_put_bytes_u16(pkt, s->d1->handshake_write_seq)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } } - return CON_FUNC_SUCCESS; + return 1; } #ifndef OPENSSL_NO_SCTP @@ -1008,14 +938,13 @@ CON_FUNC_RETURN dtls_construct_change_cipher_spec(SSL_CONNECTION *s, * Wait for a dry event. Should only be called at a point in the handshake * where we are not expecting any data from the peer except an alert. */ -WORK_STATE dtls_wait_for_dry(SSL_CONNECTION *s) +WORK_STATE dtls_wait_for_dry(SSL *s) { int ret, errtype; size_t len; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); /* read app data until dry event */ - ret = BIO_dgram_sctp_wait_for_dry(SSL_get_wbio(ssl)); + ret = BIO_dgram_sctp_wait_for_dry(SSL_get_wbio(s)); if (ret < 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return WORK_ERROR; @@ -1036,18 +965,16 @@ WORK_STATE dtls_wait_for_dry(SSL_CONNECTION *s) s->s3.in_read_app_data = 2; s->rwstate = SSL_READING; - BIO_clear_retry_flags(SSL_get_rbio(ssl)); - BIO_set_retry_read(SSL_get_rbio(ssl)); + BIO_clear_retry_flags(SSL_get_rbio(s)); + BIO_set_retry_read(SSL_get_rbio(s)); return WORK_MORE_A; } return WORK_FINISHED_CONTINUE; } #endif -int dtls1_read_failed(SSL_CONNECTION *s, int code) +int dtls1_read_failed(SSL *s, int code) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - if (code > 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; @@ -1061,9 +988,9 @@ int dtls1_read_failed(SSL_CONNECTION *s, int code) return code; } /* done, no need to send a retransmit */ - if (!SSL_in_init(ssl)) + if (!SSL_in_init(s)) { - BIO_set_flags(SSL_get_rbio(ssl), BIO_FLAGS_READ); + BIO_set_flags(SSL_get_rbio(s), BIO_FLAGS_READ); return code; } @@ -1085,7 +1012,7 @@ int dtls1_get_queue_priority(unsigned short seq, int is_ccs) return seq * 2 - is_ccs; } -int dtls1_retransmit_buffered_messages(SSL_CONNECTION *s) +int dtls1_retransmit_buffered_messages(SSL *s) { pqueue *sent = s->d1->sent_messages; piterator iter; @@ -1107,7 +1034,7 @@ int dtls1_retransmit_buffered_messages(SSL_CONNECTION *s) return 1; } -int dtls1_buffer_message(SSL_CONNECTION *s, int is_ccs) +int dtls1_buffer_message(SSL *s, int is_ccs) { pitem *item; hm_fragment *frag; @@ -1151,9 +1078,12 @@ int dtls1_buffer_message(SSL_CONNECTION *s, int is_ccs) frag->msg_header.is_ccs = is_ccs; /* save current state */ - frag->msg_header.saved_retransmit_state.wrlmethod = s->rlayer.wrlmethod; - frag->msg_header.saved_retransmit_state.wrl = s->rlayer.wrl; - + frag->msg_header.saved_retransmit_state.enc_write_ctx = s->enc_write_ctx; + frag->msg_header.saved_retransmit_state.write_hash = s->write_hash; + frag->msg_header.saved_retransmit_state.compress = s->compress; + frag->msg_header.saved_retransmit_state.session = s->session; + frag->msg_header.saved_retransmit_state.epoch = + DTLS_RECORD_LAYER_get_w_epoch(&s->rlayer); memset(seq64be, 0, sizeof(seq64be)); seq64be[6] = @@ -1175,7 +1105,7 @@ int dtls1_buffer_message(SSL_CONNECTION *s, int is_ccs) return 1; } -int dtls1_retransmit_message(SSL_CONNECTION *s, unsigned short seq, int *found) +int dtls1_retransmit_message(SSL *s, unsigned short seq, int *found) { int ret; /* XDTLS: for now assuming that read/writes are blocking */ @@ -1215,27 +1145,32 @@ int dtls1_retransmit_message(SSL_CONNECTION *s, unsigned short seq, int *found) frag->msg_header.frag_len); /* save current state */ - saved_state.wrlmethod = s->rlayer.wrlmethod; - saved_state.wrl = s->rlayer.wrl; + saved_state.enc_write_ctx = s->enc_write_ctx; + saved_state.write_hash = s->write_hash; + saved_state.compress = s->compress; + saved_state.session = s->session; + saved_state.epoch = DTLS_RECORD_LAYER_get_w_epoch(&s->rlayer); s->d1->retransmitting = 1; /* restore state in which the message was originally sent */ - s->rlayer.wrlmethod = frag->msg_header.saved_retransmit_state.wrlmethod; - s->rlayer.wrl = frag->msg_header.saved_retransmit_state.wrl; - - /* - * The old wrl may be still pointing at an old BIO. Update it to what we're - * using now. - */ - s->rlayer.wrlmethod->set1_bio(s->rlayer.wrl, s->wbio); + s->enc_write_ctx = frag->msg_header.saved_retransmit_state.enc_write_ctx; + s->write_hash = frag->msg_header.saved_retransmit_state.write_hash; + s->compress = frag->msg_header.saved_retransmit_state.compress; + s->session = frag->msg_header.saved_retransmit_state.session; + DTLS_RECORD_LAYER_set_saved_w_epoch(&s->rlayer, + frag->msg_header. + saved_retransmit_state.epoch); ret = dtls1_do_write(s, frag->msg_header.is_ccs ? SSL3_RT_CHANGE_CIPHER_SPEC : SSL3_RT_HANDSHAKE); /* restore current state */ - s->rlayer.wrlmethod = saved_state.wrlmethod; - s->rlayer.wrl = saved_state.wrl; + s->enc_write_ctx = saved_state.enc_write_ctx; + s->write_hash = saved_state.write_hash; + s->compress = saved_state.compress; + s->session = saved_state.session; + DTLS_RECORD_LAYER_set_saved_w_epoch(&s->rlayer, saved_state.epoch); s->d1->retransmitting = 0; @@ -1243,7 +1178,7 @@ int dtls1_retransmit_message(SSL_CONNECTION *s, unsigned short seq, int *found) return ret; } -void dtls1_set_message_header(SSL_CONNECTION *s, +void dtls1_set_message_header(SSL *s, unsigned char mt, size_t len, size_t frag_off, size_t frag_len) { @@ -1258,7 +1193,7 @@ void dtls1_set_message_header(SSL_CONNECTION *s, /* don't actually do the writing, wait till the MTU has been retrieved */ static void -dtls1_set_message_header_int(SSL_CONNECTION *s, unsigned char mt, +dtls1_set_message_header_int(SSL *s, unsigned char mt, size_t len, unsigned short seq_num, size_t frag_off, size_t frag_len) { @@ -1272,7 +1207,7 @@ dtls1_set_message_header_int(SSL_CONNECTION *s, unsigned char mt, } static void -dtls1_fix_message_header(SSL_CONNECTION *s, size_t frag_off, size_t frag_len) +dtls1_fix_message_header(SSL *s, size_t frag_off, size_t frag_len) { struct hm_header_st *msg_hdr = &s->d1->w_msg_hdr; @@ -1280,8 +1215,7 @@ dtls1_fix_message_header(SSL_CONNECTION *s, size_t frag_off, size_t frag_len) msg_hdr->frag_len = frag_len; } -static unsigned char *dtls1_write_message_header(SSL_CONNECTION *s, - unsigned char *p) +static unsigned char *dtls1_write_message_header(SSL *s, unsigned char *p) { struct hm_header_st *msg_hdr = &s->d1->w_msg_hdr; @@ -1295,8 +1229,7 @@ static unsigned char *dtls1_write_message_header(SSL_CONNECTION *s, return p; } -void dtls1_get_message_header(const unsigned char *data, struct - hm_header_st *msg_hdr) +void dtls1_get_message_header(unsigned char *data, struct hm_header_st *msg_hdr) { memset(msg_hdr, 0, sizeof(*msg_hdr)); msg_hdr->type = *(data++); @@ -1307,7 +1240,7 @@ void dtls1_get_message_header(const unsigned char *data, struct n2l3(data, msg_hdr->frag_len); } -int dtls1_set_handshake_header(SSL_CONNECTION *s, WPACKET *pkt, int htype) +int dtls1_set_handshake_header(SSL *s, WPACKET *pkt, int htype) { unsigned char *header; @@ -1331,7 +1264,7 @@ int dtls1_set_handshake_header(SSL_CONNECTION *s, WPACKET *pkt, int htype) return 1; } -int dtls1_close_construct_packet(SSL_CONNECTION *s, WPACKET *pkt, int htype) +int dtls1_close_construct_packet(SSL *s, WPACKET *pkt, int htype) { size_t msglen; diff --git a/openssl/src/ssl/statem/statem_lib.c b/openssl/src/ssl/statem/statem_lib.c index 8932ac44c..cff0338cf 100644 --- a/openssl/src/ssl/statem/statem_lib.c +++ b/openssl/src/ssl/statem/statem_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -20,7 +20,6 @@ #include #include #include -#include /* * Map error codes to TLS/SSL alart types. @@ -37,62 +36,38 @@ const unsigned char hrrrandom[] = { 0x07, 0x9e, 0x09, 0xe2, 0xc8, 0xa8, 0x33, 0x9c }; -int ossl_statem_set_mutator(SSL *s, - ossl_statem_mutate_handshake_cb mutate_handshake_cb, - ossl_statem_finish_mutate_handshake_cb finish_mutate_handshake_cb, - void *mutatearg) -{ - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - sc->statem.mutate_handshake_cb = mutate_handshake_cb; - sc->statem.mutatearg = mutatearg; - sc->statem.finish_mutate_handshake_cb = finish_mutate_handshake_cb; - - return 1; -} - /* * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or * SSL3_RT_CHANGE_CIPHER_SPEC) */ -int ssl3_do_write(SSL_CONNECTION *s, uint8_t type) +int ssl3_do_write(SSL *s, int type) { int ret; size_t written = 0; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - - /* - * If we're running the test suite then we may need to mutate the message - * we've been asked to write. Does not happen in normal operation. - */ - if (s->statem.mutate_handshake_cb != NULL - && !s->statem.write_in_progress - && type == SSL3_RT_HANDSHAKE - && s->init_num >= SSL3_HM_HEADER_LENGTH) { - unsigned char *msg; - size_t msglen; - - if (!s->statem.mutate_handshake_cb((unsigned char *)s->init_buf->data, - s->init_num, - &msg, &msglen, - s->statem.mutatearg)) - return -1; - if (msglen < SSL3_HM_HEADER_LENGTH - || !BUF_MEM_grow(s->init_buf, msglen)) - return -1; - memcpy(s->init_buf->data, msg, msglen); - s->init_num = msglen; - s->init_msg = s->init_buf->data + SSL3_HM_HEADER_LENGTH; - s->statem.finish_mutate_handshake_cb(s->statem.mutatearg); - s->statem.write_in_progress = 1; - } - ret = ssl3_write_bytes(ssl, type, &s->init_buf->data[s->init_off], - s->init_num, &written); - if (ret <= 0) +#ifndef OPENSSL_NO_QUIC + if (SSL_IS_QUIC(s)) { + if (type == SSL3_RT_HANDSHAKE) { + ret = s->quic_method->add_handshake_data(s, s->quic_write_level, + (const uint8_t*)&s->init_buf->data[s->init_off], + s->init_num); + if (!ret) { + ret = -1; + /* QUIC can't sent anything out sice the above failed */ + ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); + } else { + written = s->init_num; + } + } else { + /* QUIC doesn't use ChangeCipherSpec */ + ret = -1; + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + } + } else +#endif + ret = ssl3_write_bytes(s, type, &s->init_buf->data[s->init_off], + s->init_num, &written); + if (ret < 0) return -1; if (type == SSL3_RT_HANDSHAKE) /* @@ -100,8 +75,7 @@ int ssl3_do_write(SSL_CONNECTION *s, uint8_t type) * ignore the result anyway * TLS1.3 KeyUpdate and NewSessionTicket do not need to be added */ - if (!SSL_CONNECTION_IS_TLS13(s) - || (s->statem.hand_state != TLS_ST_SW_SESSION_TICKET + if (!SSL_IS_TLS13(s) || (s->statem.hand_state != TLS_ST_SW_SESSION_TICKET && s->statem.hand_state != TLS_ST_CW_KEY_UPDATE && s->statem.hand_state != TLS_ST_SW_KEY_UPDATE)) if (!ssl3_finish_mac(s, @@ -109,10 +83,9 @@ int ssl3_do_write(SSL_CONNECTION *s, uint8_t type) written)) return -1; if (written == s->init_num) { - s->statem.write_in_progress = 0; if (s->msg_callback) s->msg_callback(1, s->version, type, s->init_buf->data, - (size_t)(s->init_off + s->init_num), ssl, + (size_t)(s->init_off + s->init_num), s, s->msg_callback_arg); return 1; } @@ -121,7 +94,7 @@ int ssl3_do_write(SSL_CONNECTION *s, uint8_t type) return 0; } -int tls_close_construct_packet(SSL_CONNECTION *s, WPACKET *pkt, int htype) +int tls_close_construct_packet(SSL *s, WPACKET *pkt, int htype) { size_t msglen; @@ -135,11 +108,9 @@ int tls_close_construct_packet(SSL_CONNECTION *s, WPACKET *pkt, int htype) return 1; } -int tls_setup_handshake(SSL_CONNECTION *s) +int tls_setup_handshake(SSL *s) { int ver_min, ver_max, ok; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (!ssl3_init_finished_mac(s)) { /* SSLfatal() already called */ @@ -155,13 +126,18 @@ int tls_setup_handshake(SSL_CONNECTION *s) } /* Sanity check that we have MD5-SHA1 if we need it */ - if (sctx->ssl_digest_methods[SSL_MD_MD5_SHA1_IDX] == NULL) { - int negotiated_minversion; - int md5sha1_needed_maxversion = SSL_CONNECTION_IS_DTLS(s) - ? DTLS1_VERSION : TLS1_1_VERSION; + if (s->ctx->ssl_digest_methods[SSL_MD_MD5_SHA1_IDX] == NULL) { + int md5sha1_needed = 0; /* We don't have MD5-SHA1 - do we need it? */ - if (ssl_version_cmp(s, ver_max, md5sha1_needed_maxversion) <= 0) { + if (SSL_IS_DTLS(s)) { + if (DTLS_VERSION_LE(ver_max, DTLS1_VERSION)) + md5sha1_needed = 1; + } else { + if (ver_max <= TLS1_1_VERSION) + md5sha1_needed = 1; + } + if (md5sha1_needed) { SSLfatal_data(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_NO_SUITABLE_DIGEST_ALGORITHM, "The max supported SSL/TLS version needs the" @@ -172,12 +148,14 @@ int tls_setup_handshake(SSL_CONNECTION *s) } ok = 1; - /* Don't allow TLSv1.1 or below to be negotiated */ - negotiated_minversion = SSL_CONNECTION_IS_DTLS(s) ? - DTLS1_2_VERSION : TLS1_2_VERSION; - if (ssl_version_cmp(s, ver_min, negotiated_minversion) < 0) - ok = SSL_set_min_proto_version(ssl, negotiated_minversion); + if (SSL_IS_DTLS(s)) { + if (DTLS_VERSION_LT(ver_min, DTLS1_2_VERSION)) + ok = SSL_set_min_proto_version(s, DTLS1_2_VERSION); + } else { + if (ver_min < TLS1_2_VERSION) + ok = SSL_set_min_proto_version(s, TLS1_2_VERSION); + } if (!ok) { /* Shouldn't happen */ SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, ERR_R_INTERNAL_ERROR); @@ -187,7 +165,7 @@ int tls_setup_handshake(SSL_CONNECTION *s) ok = 0; if (s->server) { - STACK_OF(SSL_CIPHER) *ciphers = SSL_get_ciphers(ssl); + STACK_OF(SSL_CIPHER) *ciphers = SSL_get_ciphers(s); int i; /* @@ -197,16 +175,16 @@ int tls_setup_handshake(SSL_CONNECTION *s) */ for (i = 0; i < sk_SSL_CIPHER_num(ciphers); i++) { const SSL_CIPHER *c = sk_SSL_CIPHER_value(ciphers, i); - int cipher_minprotover = SSL_CONNECTION_IS_DTLS(s) - ? c->min_dtls : c->min_tls; - int cipher_maxprotover = SSL_CONNECTION_IS_DTLS(s) - ? c->max_dtls : c->max_tls; - if (ssl_version_cmp(s, ver_max, cipher_minprotover) >= 0 - && ssl_version_cmp(s, ver_max, cipher_maxprotover) <= 0) { + if (SSL_IS_DTLS(s)) { + if (DTLS_VERSION_GE(ver_max, c->min_dtls) && + DTLS_VERSION_LE(ver_max, c->max_dtls)) + ok = 1; + } else if (ver_max >= c->min_tls && ver_max <= c->max_tls) { ok = 1; - break; } + if (ok) + break; } if (!ok) { SSLfatal_data(s, SSL_AD_HANDSHAKE_FAILURE, @@ -220,7 +198,7 @@ int tls_setup_handshake(SSL_CONNECTION *s) ssl_tsan_counter(s->session_ctx, &s->session_ctx->stats.sess_accept); } else { /* N.B. s->ctx may not equal s->session_ctx */ - ssl_tsan_counter(sctx, &sctx->stats.sess_accept_renegotiate); + ssl_tsan_counter(s->ctx, &s->ctx->stats.sess_accept_renegotiate); s->s3.tmp.cert_request = 0; } @@ -237,7 +215,7 @@ int tls_setup_handshake(SSL_CONNECTION *s) s->s3.tmp.cert_req = 0; - if (SSL_CONNECTION_IS_DTLS(s)) + if (SSL_IS_DTLS(s)) s->statem.use_timer = 1; } @@ -251,17 +229,23 @@ int tls_setup_handshake(SSL_CONNECTION *s) #define TLS13_TBS_START_SIZE 64 #define TLS13_TBS_PREAMBLE_SIZE (TLS13_TBS_START_SIZE + 33 + 1) -static int get_cert_verify_tbs_data(SSL_CONNECTION *s, unsigned char *tls13tbs, +static int get_cert_verify_tbs_data(SSL *s, unsigned char *tls13tbs, void **hdata, size_t *hdatalen) { - /* ASCII: "TLS 1.3, server CertificateVerify", in hex for EBCDIC compatibility */ - static const char servercontext[] = "\x54\x4c\x53\x20\x31\x2e\x33\x2c\x20\x73\x65\x72" - "\x76\x65\x72\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x56\x65\x72\x69\x66\x79"; - /* ASCII: "TLS 1.3, client CertificateVerify", in hex for EBCDIC compatibility */ - static const char clientcontext[] = "\x54\x4c\x53\x20\x31\x2e\x33\x2c\x20\x63\x6c\x69" - "\x65\x6e\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x56\x65\x72\x69\x66\x79"; - - if (SSL_CONNECTION_IS_TLS13(s)) { +#ifdef CHARSET_EBCDIC + static const char servercontext[] = { 0x54, 0x4c, 0x53, 0x20, 0x31, 0x2e, + 0x33, 0x2c, 0x20, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x20, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, + 0x69, 0x66, 0x79, 0x00 }; + static const char clientcontext[] = { 0x54, 0x4c, 0x53, 0x20, 0x31, 0x2e, + 0x33, 0x2c, 0x20, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, + 0x69, 0x66, 0x79, 0x00 }; +#else + static const char servercontext[] = "TLS 1.3, server CertificateVerify"; + static const char clientcontext[] = "TLS 1.3, client CertificateVerify"; +#endif + if (SSL_IS_TLS13(s)) { size_t hashlen; /* Set the first 64 bytes of to-be-signed data to octet 32 */ @@ -306,7 +290,7 @@ static int get_cert_verify_tbs_data(SSL_CONNECTION *s, unsigned char *tls13tbs, return 1; } -CON_FUNC_RETURN tls_construct_cert_verify(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_cert_verify(SSL *s, WPACKET *pkt) { EVP_PKEY *pkey = NULL; const EVP_MD *md = NULL; @@ -317,25 +301,55 @@ CON_FUNC_RETURN tls_construct_cert_verify(SSL_CONNECTION *s, WPACKET *pkt) unsigned char *sig = NULL; unsigned char tls13tbs[TLS13_TBS_PREAMBLE_SIZE + EVP_MAX_MD_SIZE]; const SIGALG_LOOKUP *lu = s->s3.tmp.sigalg; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (lu == NULL || s->s3.tmp.cert == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } + +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + if (s->delegated_credential_tag & DC_HAS_BEEN_USED_FOR_SIGN) { + if (s->s3.tmp.dc == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + pkey = s->s3.tmp.dc->privatekey; + } else +#endif pkey = s->s3.tmp.cert->privatekey; - if (pkey == NULL || !tls1_lookup_md(sctx, lu, &md)) { + if (pkey == NULL || !tls1_lookup_md(s->ctx, lu, &md)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } mctx = EVP_MD_CTX_new(); if (mctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } +#ifndef OPENSSL_NO_SM2 + if (EVP_PKEY_is_sm2(pkey)) { + if (pkey != NULL) { + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, pkey, + s->ctx->propq); + if (pctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (EVP_PKEY_CTX_set1_id(pctx, HANDSHAKE_SM2_ID, + HANDSHAKE_SM2_ID_LEN) != 1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + EVP_MD_CTX_set_pkey_ctx(mctx, pctx); + } + } +#endif + /* Get the data to be signed */ if (!get_cert_verify_tbs_data(s, tls13tbs, &hdata, &hdatalen)) { /* SSLfatal() already called */ @@ -349,7 +363,7 @@ CON_FUNC_RETURN tls_construct_cert_verify(SSL_CONNECTION *s, WPACKET *pkt) if (EVP_DigestSignInit_ex(mctx, &pctx, md == NULL ? NULL : EVP_MD_get0_name(md), - sctx->libctx, sctx->propq, pkey, + s->ctx->libctx, s->ctx->propq, pkey, NULL) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); goto err; @@ -400,17 +414,6 @@ CON_FUNC_RETURN tls_construct_cert_verify(SSL_CONNECTION *s, WPACKET *pkt) } } -#ifndef OPENSSL_NO_GOST - { - int pktype = lu->sig; - - if (pktype == NID_id_GostR3410_2001 - || pktype == NID_id_GostR3410_2012_256 - || pktype == NID_id_GostR3410_2012_512) - BUF_reverse(sig, NULL, siglen); - } -#endif - if (!WPACKET_sub_memcpy_u16(pkt, sig, siglen)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; @@ -424,43 +427,68 @@ CON_FUNC_RETURN tls_construct_cert_verify(SSL_CONNECTION *s, WPACKET *pkt) OPENSSL_free(sig); EVP_MD_CTX_free(mctx); - return CON_FUNC_SUCCESS; +#ifndef OPENSSL_NO_SM2 + /*other sig call EVP_PKEY_CTX_free there may cause segfault */ + if (pkey != NULL && EVP_PKEY_is_sm2(pkey)) + EVP_PKEY_CTX_free(pctx); +#endif + return 1; err: OPENSSL_free(sig); EVP_MD_CTX_free(mctx); - return CON_FUNC_ERROR; +#ifndef OPENSSL_NO_SM2 + if (pkey != NULL && EVP_PKEY_is_sm2(pkey)) + EVP_PKEY_CTX_free(pctx); +#endif + return 0; } -MSG_PROCESS_RETURN tls_process_cert_verify(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_cert_verify(SSL *s, PACKET *pkt) { EVP_PKEY *pkey = NULL; const unsigned char *data; -#ifndef OPENSSL_NO_GOST - unsigned char *gost_data = NULL; -#endif MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR; int j; unsigned int len; + X509 *peer; const EVP_MD *md = NULL; size_t hdatalen = 0; void *hdata; unsigned char tls13tbs[TLS13_TBS_PREAMBLE_SIZE + EVP_MAX_MD_SIZE]; EVP_MD_CTX *mctx = EVP_MD_CTX_new(); EVP_PKEY_CTX *pctx = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (mctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } - pkey = tls_get_peer_pkey(s); + peer = s->session->peer; + pkey = X509_get0_pubkey(peer); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + if (s->delegated_credential_tag & DC_HAS_BEEN_USED_FOR_VERIFY_PEER) { + if (s->session->peer_dc == NULL) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + goto err; + } + pkey = DC_get0_publickey(s->session->peer_dc); + } +#endif if (pkey == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } - if (ssl_cert_lookup_by_pkey(pkey, NULL, sctx) == NULL) { +#ifndef OPENSSL_NO_SM2 + if (EVP_PKEY_is_sm2(pkey)) { + if (!EVP_PKEY_set_alias_type(pkey, EVP_PKEY_SM2)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } +#endif + + if (ssl_cert_lookup_by_pkey(pkey, NULL) == NULL) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE); goto err; @@ -477,13 +505,29 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL_CONNECTION *s, PACKET *pkt) /* SSLfatal() already called */ goto err; } +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + /* + * Verify that dc expected_cert_verify_algorithm matches the scheme + * indicated in the server's CertificateVerify message. + */ + if (s->delegated_credential_tag & DC_HAS_BEEN_USED_FOR_VERIFY_PEER) { + if (s->session->peer_dc == NULL) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (DC_get_expected_cert_verify_algorithm(s->session->peer_dc) != sigalg) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, ERR_R_INTERNAL_ERROR); + goto err; + } + } +#endif } else if (!tls1_set_peer_legacy_sigalg(s, pkey)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, - SSL_R_LEGACY_SIGALG_DISALLOWED_OR_UNSUPPORTED); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } - if (!tls1_lookup_md(sctx, s->s3.tmp.peer_sigalg, &md)) { + if (!tls1_lookup_md(s->ctx, s->s3.tmp.peer_sigalg, &md)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } @@ -492,21 +536,6 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL_CONNECTION *s, PACKET *pkt) OSSL_TRACE1(TLS, "USING TLSv1.2 HASH %s\n", md == NULL ? "n/a" : EVP_MD_get0_name(md)); - /* Check for broken implementations of GOST ciphersuites */ - /* - * If key is GOST and len is exactly 64 or 128, it is signature without - * length field (CryptoPro implementations at least till TLS 1.2) - */ -#ifndef OPENSSL_NO_GOST - if (!SSL_USE_SIGALGS(s) - && ((PACKET_remaining(pkt) == 64 - && (EVP_PKEY_get_id(pkey) == NID_id_GostR3410_2001 - || EVP_PKEY_get_id(pkey) == NID_id_GostR3410_2012_256)) - || (PACKET_remaining(pkt) == 128 - && EVP_PKEY_get_id(pkey) == NID_id_GostR3410_2012_512))) { - len = PACKET_remaining(pkt); - } else -#endif if (!PACKET_get_net_2(pkt, &len)) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); goto err; @@ -525,26 +554,31 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL_CONNECTION *s, PACKET *pkt) OSSL_TRACE1(TLS, "Using client verify alg %s\n", md == NULL ? "n/a" : EVP_MD_get0_name(md)); +#ifndef OPENSSL_NO_SM2 + if (EVP_PKEY_is_sm2(pkey)) { + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, pkey, s->ctx->propq); + if (pctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (EVP_PKEY_CTX_set1_id(pctx, HANDSHAKE_SM2_ID, + HANDSHAKE_SM2_ID_LEN) != 1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + EVP_MD_CTX_set_pkey_ctx(mctx, pctx); + } +#endif + if (EVP_DigestVerifyInit_ex(mctx, &pctx, md == NULL ? NULL : EVP_MD_get0_name(md), - sctx->libctx, sctx->propq, pkey, + s->ctx->libctx, s->ctx->propq, pkey, NULL) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); goto err; } -#ifndef OPENSSL_NO_GOST - { - int pktype = EVP_PKEY_get_id(pkey); - if (pktype == NID_id_GostR3410_2001 - || pktype == NID_id_GostR3410_2012_256 - || pktype == NID_id_GostR3410_2012_512) { - if ((gost_data = OPENSSL_malloc(len)) == NULL) - goto err; - BUF_reverse(gost_data, data, len); - data = gost_data; - } - } -#endif if (SSL_USE_PSS(s)) { if (EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_PSS_PADDING) <= 0 @@ -568,11 +602,6 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL_CONNECTION *s, PACKET *pkt) } } else { j = EVP_DigestVerify(mctx, data, len, hdata, hdatalen); -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - /* Ignore bad signatures when fuzzing */ - if (SSL_IS_QUIC_HANDSHAKE(s)) - j = 1; -#endif if (j <= 0) { SSLfatal(s, SSL_AD_DECRYPT_ERROR, SSL_R_BAD_SIGNATURE); goto err; @@ -587,7 +616,7 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL_CONNECTION *s, PACKET *pkt) * want to make sure that SSL_get1_peer_certificate() will return the actual * server certificate from the client_cert_cb callback. */ - if (!s->server && SSL_CONNECTION_IS_TLS13(s) && s->s3.tmp.cert_req == 1) + if (!s->server && SSL_IS_TLS13(s) && s->s3.tmp.cert_req == 1) ret = MSG_PROCESS_CONTINUE_PROCESSING; else ret = MSG_PROCESS_CONTINUE_READING; @@ -595,72 +624,69 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL_CONNECTION *s, PACKET *pkt) BIO_free(s->s3.handshake_buffer); s->s3.handshake_buffer = NULL; EVP_MD_CTX_free(mctx); -#ifndef OPENSSL_NO_GOST - OPENSSL_free(gost_data); +#ifndef OPENSSL_NO_SM2 + /*other sig call EVP_PKEY_CTX_free there may cause segfault */ + if (pkey != NULL && EVP_PKEY_is_sm2(pkey)) + EVP_PKEY_CTX_free(pctx); #endif return ret; } -CON_FUNC_RETURN tls_construct_finished(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_finished(SSL *s, WPACKET *pkt) { size_t finish_md_len; const char *sender; size_t slen; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); /* This is a real handshake so make sure we clean it up at the end */ if (!s->server && s->post_handshake_auth != SSL_PHA_REQUESTED) s->statem.cleanuphand = 1; /* - * If we attempted to write early data or we're in middlebox compat mode - * then we deferred changing the handshake write keys to the last possible - * moment. If we didn't already do this when we sent the client certificate - * then we need to do it now. + * We only change the keys if we didn't already do this when we sent the + * client certificate */ - if (SSL_CONNECTION_IS_TLS13(s) + if (SSL_IS_TLS13(s) && !s->server - && (s->early_data_state != SSL_EARLY_DATA_NONE - || (s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0) && s->s3.tmp.cert_req == 0 - && (!ssl->method->ssl3_enc->change_cipher_state(s, + && (!s->method->ssl3_enc->change_cipher_state(s, SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_WRITE))) {; /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } if (s->server) { - sender = ssl->method->ssl3_enc->server_finished_label; - slen = ssl->method->ssl3_enc->server_finished_label_len; + sender = s->method->ssl3_enc->server_finished_label; + slen = s->method->ssl3_enc->server_finished_label_len; } else { - sender = ssl->method->ssl3_enc->client_finished_label; - slen = ssl->method->ssl3_enc->client_finished_label_len; + sender = s->method->ssl3_enc->client_finished_label; + slen = s->method->ssl3_enc->client_finished_label_len; } - finish_md_len = ssl->method->ssl3_enc->final_finish_mac(s, - sender, slen, - s->s3.tmp.finish_md); + finish_md_len = s->method->ssl3_enc->final_finish_mac(s, + sender, slen, + s->s3.tmp.finish_md); if (finish_md_len == 0) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } s->s3.tmp.finish_md_len = finish_md_len; if (!WPACKET_memcpy(pkt, s->s3.tmp.finish_md, finish_md_len)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /* * Log the master secret, if logging is enabled. We don't log it for * TLSv1.3: there's a different key schedule for that. */ - if (!SSL_CONNECTION_IS_TLS13(s) - && !ssl_log_secret(s, MASTER_SECRET_LABEL, s->session->master_key, - s->session->master_key_length)) { + if (!SSL_IS_TLS13(s) && !ssl_log_secret(s, MASTER_SECRET_LABEL, + s->session->master_key, + s->session->master_key_length)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } /* @@ -668,7 +694,7 @@ CON_FUNC_RETURN tls_construct_finished(SSL_CONNECTION *s, WPACKET *pkt) */ if (!ossl_assert(finish_md_len <= EVP_MAX_MD_SIZE)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } if (!s->server) { memcpy(s->s3.previous_client_finished, s->s3.tmp.finish_md, @@ -680,21 +706,28 @@ CON_FUNC_RETURN tls_construct_finished(SSL_CONNECTION *s, WPACKET *pkt) s->s3.previous_server_finished_len = finish_md_len; } - return CON_FUNC_SUCCESS; + return 1; } -CON_FUNC_RETURN tls_construct_key_update(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_key_update(SSL *s, WPACKET *pkt) { +#ifndef OPENSSL_NO_QUIC + if (SSL_IS_QUIC(s)) { + /* TLS KeyUpdate is not used for QUIC, so this is an error. */ + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } +#endif if (!WPACKET_put_bytes_u8(pkt, s->key_update)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } s->key_update = SSL_KEY_UPDATE_NONE; - return CON_FUNC_SUCCESS; + return 1; } -MSG_PROCESS_RETURN tls_process_key_update(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_key_update(SSL *s, PACKET *pkt) { unsigned int updatetype; @@ -707,6 +740,13 @@ MSG_PROCESS_RETURN tls_process_key_update(SSL_CONNECTION *s, PACKET *pkt) return MSG_PROCESS_ERROR; } +#ifndef OPENSSL_NO_QUIC + if (SSL_is_quic(s)) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_MESSAGE); + return MSG_PROCESS_ERROR; + } +#endif + if (!PACKET_get_1(pkt, &updatetype) || PACKET_remaining(pkt) != 0) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_KEY_UPDATE); @@ -743,23 +783,22 @@ MSG_PROCESS_RETURN tls_process_key_update(SSL_CONNECTION *s, PACKET *pkt) * ssl3_take_mac calculates the Finished MAC for the handshakes messages seen * to far. */ -int ssl3_take_mac(SSL_CONNECTION *s) +int ssl3_take_mac(SSL *s) { const char *sender; size_t slen; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if (!s->server) { - sender = ssl->method->ssl3_enc->server_finished_label; - slen = ssl->method->ssl3_enc->server_finished_label_len; + sender = s->method->ssl3_enc->server_finished_label; + slen = s->method->ssl3_enc->server_finished_label_len; } else { - sender = ssl->method->ssl3_enc->client_finished_label; - slen = ssl->method->ssl3_enc->client_finished_label_len; + sender = s->method->ssl3_enc->client_finished_label; + slen = s->method->ssl3_enc->client_finished_label_len; } s->s3.tmp.peer_finish_md_len = - ssl->method->ssl3_enc->final_finish_mac(s, sender, slen, - s->s3.tmp.peer_finish_md); + s->method->ssl3_enc->final_finish_mac(s, sender, slen, + s->s3.tmp.peer_finish_md); if (s->s3.tmp.peer_finish_md_len == 0) { /* SSLfatal() already called */ @@ -769,8 +808,7 @@ int ssl3_take_mac(SSL_CONNECTION *s) return 1; } -MSG_PROCESS_RETURN tls_process_change_cipher_spec(SSL_CONNECTION *s, - PACKET *pkt) +MSG_PROCESS_RETURN tls_process_change_cipher_spec(SSL *s, PACKET *pkt) { size_t remain; @@ -780,7 +818,7 @@ MSG_PROCESS_RETURN tls_process_change_cipher_spec(SSL_CONNECTION *s, * been consumed by ssl_get_message() so there should be no bytes left, * unless we're using DTLS1_BAD_VER, which has an extra 2 bytes */ - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { if ((s->version == DTLS1_BAD_VER && remain != DTLS1_CCS_HEADER_LENGTH + 1) || (s->version != DTLS1_BAD_VER @@ -807,7 +845,9 @@ MSG_PROCESS_RETURN tls_process_change_cipher_spec(SSL_CONNECTION *s, return MSG_PROCESS_ERROR; } - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { + dtls1_reset_seq_numbers(s, SSL3_CC_READ); + if (s->version == DTLS1_BAD_VER) s->d1->handshake_read_seq++; @@ -817,35 +857,29 @@ MSG_PROCESS_RETURN tls_process_change_cipher_spec(SSL_CONNECTION *s, * SCTP-Auth can be deleted when a CCS is sent. Will be ignored if no * SCTP is used */ - BIO_ctrl(SSL_get_wbio(SSL_CONNECTION_GET_SSL(s)), - BIO_CTRL_DGRAM_SCTP_AUTH_CCS_RCVD, 1, NULL); + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_AUTH_CCS_RCVD, 1, NULL); #endif } return MSG_PROCESS_CONTINUE_READING; } -MSG_PROCESS_RETURN tls_process_finished(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_finished(SSL *s, PACKET *pkt) { size_t md_len; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - int was_first = SSL_IS_FIRST_HANDSHAKE(s); - int ok; /* This is a real handshake so make sure we clean it up at the end */ if (s->server) { /* * To get this far we must have read encrypted data from the client. We - * no longer tolerate unencrypted alerts. This is ignored if less than - * TLSv1.3 + * no longer tolerate unencrypted alerts. This value is ignored if less + * than TLSv1.3 */ - if (s->rlayer.rrlmethod->set_plain_alerts != NULL) - s->rlayer.rrlmethod->set_plain_alerts(s->rlayer.rrl, 0); + s->statem.enc_read_state = ENC_READ_STATE_VALID; if (s->post_handshake_auth != SSL_PHA_REQUESTED) s->statem.cleanuphand = 1; - if (SSL_CONNECTION_IS_TLS13(s) - && !tls13_save_handshake_digest_for_pha(s)) { + if (SSL_IS_TLS13(s) && !tls13_save_handshake_digest_for_pha(s)) { /* SSLfatal() already called */ return MSG_PROCESS_ERROR; } @@ -855,14 +889,13 @@ MSG_PROCESS_RETURN tls_process_finished(SSL_CONNECTION *s, PACKET *pkt) * In TLSv1.3 a Finished message signals a key change so the end of the * message must be on a record boundary. */ - if (SSL_CONNECTION_IS_TLS13(s) - && RECORD_LAYER_processed_read_pending(&s->rlayer)) { + if (SSL_IS_TLS13(s) && RECORD_LAYER_processed_read_pending(&s->rlayer)) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_NOT_ON_RECORD_BOUNDARY); return MSG_PROCESS_ERROR; } /* If this occurs, we have missed a message */ - if (!SSL_CONNECTION_IS_TLS13(s) && !s->s3.change_cipher_spec) { + if (!SSL_IS_TLS13(s) && !s->s3.change_cipher_spec) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_GOT_A_FIN_BEFORE_A_CCS); return MSG_PROCESS_ERROR; } @@ -875,16 +908,8 @@ MSG_PROCESS_RETURN tls_process_finished(SSL_CONNECTION *s, PACKET *pkt) return MSG_PROCESS_ERROR; } - ok = CRYPTO_memcmp(PACKET_data(pkt), s->s3.tmp.peer_finish_md, - md_len); -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (ok != 0) { - if ((PACKET_data(pkt)[0] ^ s->s3.tmp.peer_finish_md[0]) != 0xFF) { - ok = 0; - } - } -#endif - if (ok != 0) { + if (CRYPTO_memcmp(PACKET_data(pkt), s->s3.tmp.peer_finish_md, + md_len) != 0) { SSLfatal(s, SSL_AD_DECRYPT_ERROR, SSL_R_DIGEST_CHECK_FAILED); return MSG_PROCESS_ERROR; } @@ -910,24 +935,24 @@ MSG_PROCESS_RETURN tls_process_finished(SSL_CONNECTION *s, PACKET *pkt) * In TLS1.3 we also have to change cipher state and do any final processing * of the initial server flight (if we are a client) */ - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { if (s->server) { if (s->post_handshake_auth != SSL_PHA_REQUESTED && - !ssl->method->ssl3_enc->change_cipher_state(s, - SSL3_CC_APPLICATION | SSL3_CHANGE_CIPHER_SERVER_READ)) { + !s->method->ssl3_enc->change_cipher_state(s, + SSL3_CC_APPLICATION | SSL3_CHANGE_CIPHER_SERVER_READ)) { /* SSLfatal() already called */ return MSG_PROCESS_ERROR; } } else { /* TLS 1.3 gets the secret size from the handshake md */ size_t dummy; - if (!ssl->method->ssl3_enc->generate_master_secret(s, + if (!s->method->ssl3_enc->generate_master_secret(s, s->master_secret, s->handshake_secret, 0, &dummy)) { /* SSLfatal() already called */ return MSG_PROCESS_ERROR; } - if (!ssl->method->ssl3_enc->change_cipher_state(s, + if (!s->method->ssl3_enc->change_cipher_state(s, SSL3_CC_APPLICATION | SSL3_CHANGE_CIPHER_CLIENT_READ)) { /* SSLfatal() already called */ return MSG_PROCESS_ERROR; @@ -939,50 +964,39 @@ MSG_PROCESS_RETURN tls_process_finished(SSL_CONNECTION *s, PACKET *pkt) } } - if (was_first - && !SSL_IS_FIRST_HANDSHAKE(s) - && s->rlayer.rrlmethod->set_first_handshake != NULL) - s->rlayer.rrlmethod->set_first_handshake(s->rlayer.rrl, 0); - return MSG_PROCESS_FINISHED_READING; } -CON_FUNC_RETURN tls_construct_change_cipher_spec(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_change_cipher_spec(SSL *s, WPACKET *pkt) { if (!WPACKET_put_bytes_u8(pkt, SSL3_MT_CCS)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } - return CON_FUNC_SUCCESS; + return 1; } /* Add a certificate to the WPACKET */ -static int ssl_add_cert_to_wpacket(SSL_CONNECTION *s, WPACKET *pkt, - X509 *x, int chain, int for_comp) +static int ssl_add_cert_to_wpacket(SSL *s, WPACKET *pkt, X509 *x, int chain) { int len; unsigned char *outbytes; - int context = SSL_EXT_TLS1_3_CERTIFICATE; - - if (for_comp) - context |= SSL_EXT_TLS1_3_CERTIFICATE_COMPRESSION; len = i2d_X509(x, NULL); if (len < 0) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_BUF_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_BUF_LIB); return 0; } if (!WPACKET_sub_allocate_bytes_u24(pkt, len, &outbytes) || i2d_X509(x, &outbytes) != len) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - if ((SSL_CONNECTION_IS_TLS13(s) || for_comp) - && !tls_construct_extensions(s, pkt, context, x, chain)) { + if (SSL_IS_TLS13(s) + && !tls_construct_extensions(s, pkt, SSL_EXT_TLS1_3_CERTIFICATE, x, + chain)) { /* SSLfatal() already called */ return 0; } @@ -991,14 +1005,13 @@ static int ssl_add_cert_to_wpacket(SSL_CONNECTION *s, WPACKET *pkt, } /* Add certificate chain to provided WPACKET */ -static int ssl_add_cert_chain(SSL_CONNECTION *s, WPACKET *pkt, CERT_PKEY *cpk, int for_comp) +static int ssl_add_cert_chain(SSL *s, WPACKET *pkt, CERT_PKEY *cpk) { int i, chain_count; X509 *x; STACK_OF(X509) *extra_certs; STACK_OF(X509) *chain = NULL; X509_STORE *chain_store; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (cpk == NULL || cpk->x509 == NULL) return 1; @@ -1011,28 +1024,26 @@ static int ssl_add_cert_chain(SSL_CONNECTION *s, WPACKET *pkt, CERT_PKEY *cpk, i if (cpk->chain != NULL) extra_certs = cpk->chain; else - extra_certs = sctx->extra_certs; + extra_certs = s->ctx->extra_certs; if ((s->mode & SSL_MODE_NO_AUTO_CHAIN) || extra_certs) chain_store = NULL; else if (s->cert->chain_store) chain_store = s->cert->chain_store; else - chain_store = sctx->cert_store; + chain_store = s->ctx->cert_store; if (chain_store != NULL) { - X509_STORE_CTX *xs_ctx = X509_STORE_CTX_new_ex(sctx->libctx, - sctx->propq); + X509_STORE_CTX *xs_ctx = X509_STORE_CTX_new_ex(s->ctx->libctx, + s->ctx->propq); if (xs_ctx == NULL) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_X509_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } if (!X509_STORE_CTX_init(xs_ctx, chain_store, x, NULL)) { X509_STORE_CTX_free(xs_ctx); - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_X509_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_X509_LIB); return 0; } /* @@ -1054,15 +1065,14 @@ static int ssl_add_cert_chain(SSL_CONNECTION *s, WPACKET *pkt, CERT_PKEY *cpk, i ERR_raise(ERR_LIB_SSL, SSL_R_CA_MD_TOO_WEAK); #endif X509_STORE_CTX_free(xs_ctx); - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, i); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, i); return 0; } chain_count = sk_X509_num(chain); for (i = 0; i < chain_count; i++) { x = sk_X509_value(chain, i); - if (!ssl_add_cert_to_wpacket(s, pkt, x, i, for_comp)) { + if (!ssl_add_cert_to_wpacket(s, pkt, x, i)) { /* SSLfatal() already called */ X509_STORE_CTX_free(xs_ctx); return 0; @@ -1072,17 +1082,16 @@ static int ssl_add_cert_chain(SSL_CONNECTION *s, WPACKET *pkt, CERT_PKEY *cpk, i } else { i = ssl_security_cert_chain(s, extra_certs, x, 0); if (i != 1) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, i); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, i); return 0; } - if (!ssl_add_cert_to_wpacket(s, pkt, x, 0, for_comp)) { + if (!ssl_add_cert_to_wpacket(s, pkt, x, 0)) { /* SSLfatal() already called */ return 0; } for (i = 0; i < sk_X509_num(extra_certs); i++) { x = sk_X509_value(extra_certs, i); - if (!ssl_add_cert_to_wpacket(s, pkt, x, i + 1, for_comp)) { + if (!ssl_add_cert_to_wpacket(s, pkt, x, i + 1)) { /* SSLfatal() already called */ return 0; } @@ -1091,306 +1100,18 @@ static int ssl_add_cert_chain(SSL_CONNECTION *s, WPACKET *pkt, CERT_PKEY *cpk, i return 1; } -EVP_PKEY* tls_get_peer_pkey(const SSL_CONNECTION *sc) -{ - if (sc->session->peer_rpk != NULL) - return sc->session->peer_rpk; - if (sc->session->peer != NULL) - return X509_get0_pubkey(sc->session->peer); - return NULL; -} - -int tls_process_rpk(SSL_CONNECTION *sc, PACKET *pkt, EVP_PKEY **peer_rpk) -{ - EVP_PKEY *pkey = NULL; - int ret = 0; - RAW_EXTENSION *rawexts = NULL; - PACKET extensions; - PACKET context; - unsigned long cert_len = 0, spki_len = 0; - const unsigned char *spki, *spkistart; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(sc); - - /*- - * ---------------------------- - * TLS 1.3 Certificate message: - * ---------------------------- - * https://datatracker.ietf.org/doc/html/rfc8446#section-4.4.2 - * - * enum { - * X509(0), - * RawPublicKey(2), - * (255) - * } CertificateType; - * - * struct { - * select (certificate_type) { - * case RawPublicKey: - * // From RFC 7250 ASN.1_subjectPublicKeyInfo - * opaque ASN1_subjectPublicKeyInfo<1..2^24-1>; - * - * case X509: - * opaque cert_data<1..2^24-1>; - * }; - * Extension extensions<0..2^16-1>; - * } CertificateEntry; - * - * struct { - * opaque certificate_request_context<0..2^8-1>; - * CertificateEntry certificate_list<0..2^24-1>; - * } Certificate; - * - * The client MUST send a Certificate message if and only if the server - * has requested client authentication via a CertificateRequest message - * (Section 4.3.2). If the server requests client authentication but no - * suitable certificate is available, the client MUST send a Certificate - * message containing no certificates (i.e., with the "certificate_list" - * field having length 0). - * - * ---------------------------- - * TLS 1.2 Certificate message: - * ---------------------------- - * https://datatracker.ietf.org/doc/html/rfc7250#section-3 - * - * opaque ASN.1Cert<1..2^24-1>; - * - * struct { - * select(certificate_type){ - * - * // certificate type defined in this document. - * case RawPublicKey: - * opaque ASN.1_subjectPublicKeyInfo<1..2^24-1>; - * - * // X.509 certificate defined in RFC 5246 - * case X.509: - * ASN.1Cert certificate_list<0..2^24-1>; - * - * // Additional certificate type based on - * // "TLS Certificate Types" subregistry - * }; - * } Certificate; - * - * ------------- - * Consequently: - * ------------- - * After the (TLS 1.3 only) context octet string (1 byte length + data) the - * Certificate message has a 3-byte length that is zero in the client to - * server message when the client has no RPK to send. In that case, there - * are no (TLS 1.3 only) per-certificate extensions either, because the - * [CertificateEntry] list is empty. - * - * In the server to client direction, or when the client had an RPK to send, - * the TLS 1.3 message just prepends the length of the RPK+extensions, - * while TLS <= 1.2 sends just the RPK (octet-string). - * - * The context must be zero-length in the server to client direction, and - * must match the value recorded in the certificate request in the client - * to server direction. - */ - if (SSL_CONNECTION_IS_TLS13(sc)) { - if (!PACKET_get_length_prefixed_1(pkt, &context)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_INVALID_CONTEXT); - goto err; - } - if (sc->server) { - if (sc->pha_context == NULL) { - if (PACKET_remaining(&context) != 0) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_INVALID_CONTEXT); - goto err; - } - } else { - if (!PACKET_equal(&context, sc->pha_context, sc->pha_context_len)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_INVALID_CONTEXT); - goto err; - } - } - } else { - if (PACKET_remaining(&context) != 0) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_INVALID_CONTEXT); - goto err; - } - } - } - - if (!PACKET_get_net_3(pkt, &cert_len) - || PACKET_remaining(pkt) != cert_len) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); - goto err; - } - - /* - * The list length may be zero when there is no RPK. In the case of TLS - * 1.2 this is actually the RPK length, which cannot be zero as specified, - * but that breaks the ability of the client to decline client auth. We - * overload the 0 RPK length to mean "no RPK". This interpretation is - * also used some other (reference?) implementations, but is not supported - * by the verbatim RFC7250 text. - */ - if (cert_len == 0) - return 1; - - if (SSL_CONNECTION_IS_TLS13(sc)) { - /* - * With TLS 1.3, a non-empty explicit-length RPK octet-string followed - * by a possibly empty extension block. - */ - if (!PACKET_get_net_3(pkt, &spki_len)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); - goto err; - } - if (spki_len == 0) { - /* empty RPK */ - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_EMPTY_RAW_PUBLIC_KEY); - goto err; - } - } else { - spki_len = cert_len; - } - - if (!PACKET_get_bytes(pkt, &spki, spki_len)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); - goto err; - } - spkistart = spki; - if ((pkey = d2i_PUBKEY_ex(NULL, &spki, spki_len, sctx->libctx, sctx->propq)) == NULL - || spki != (spkistart + spki_len)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); - goto err; - } - if (EVP_PKEY_missing_parameters(pkey)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, - SSL_R_UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS); - goto err; - } - - /* Process the Extensions block */ - if (SSL_CONNECTION_IS_TLS13(sc)) { - if (PACKET_remaining(pkt) != (cert_len - 3 - spki_len)) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_BAD_LENGTH); - goto err; - } - if (!PACKET_as_length_prefixed_2(pkt, &extensions) - || PACKET_remaining(pkt) != 0) { - SSLfatal(sc, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); - goto err; - } - if (!tls_collect_extensions(sc, &extensions, SSL_EXT_TLS1_3_RAW_PUBLIC_KEY, - &rawexts, NULL, 1)) { - /* SSLfatal already called */ - goto err; - } - /* chain index is always zero and fin always 1 for RPK */ - if (!tls_parse_all_extensions(sc, SSL_EXT_TLS1_3_RAW_PUBLIC_KEY, - rawexts, NULL, 0, 1)) { - /* SSLfatal already called */ - goto err; - } - } - ret = 1; - if (peer_rpk != NULL) { - *peer_rpk = pkey; - pkey = NULL; - } - - err: - OPENSSL_free(rawexts); - EVP_PKEY_free(pkey); - return ret; -} - -unsigned long tls_output_rpk(SSL_CONNECTION *sc, WPACKET *pkt, CERT_PKEY *cpk) -{ - int pdata_len = 0; - unsigned char *pdata = NULL; - X509_PUBKEY *xpk = NULL; - unsigned long ret = 0; - X509 *x509 = NULL; - - if (cpk != NULL && cpk->x509 != NULL) { - x509 = cpk->x509; - /* Get the RPK from the certificate */ - xpk = X509_get_X509_PUBKEY(cpk->x509); - if (xpk == NULL) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - pdata_len = i2d_X509_PUBKEY(xpk, &pdata); - } else if (cpk != NULL && cpk->privatekey != NULL) { - /* Get the RPK from the private key */ - pdata_len = i2d_PUBKEY(cpk->privatekey, &pdata); - } else { - /* The server RPK is not optional */ - if (sc->server) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - /* The client can send a zero length certificate list */ - if (!WPACKET_sub_memcpy_u24(pkt, pdata, pdata_len)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - return 1; - } - - if (pdata_len <= 0) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* - * TLSv1.2 is _just_ the raw public key - * TLSv1.3 includes extensions, so there's a length wrapper - */ - if (SSL_CONNECTION_IS_TLS13(sc)) { - if (!WPACKET_start_sub_packet_u24(pkt)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - } - - if (!WPACKET_sub_memcpy_u24(pkt, pdata, pdata_len)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - if (SSL_CONNECTION_IS_TLS13(sc)) { - /* - * Only send extensions relevant to raw public keys. Until such - * extensions are defined, this will be an empty set of extensions. - * |x509| may be NULL, which raw public-key extensions need to handle. - */ - if (!tls_construct_extensions(sc, pkt, SSL_EXT_TLS1_3_RAW_PUBLIC_KEY, - x509, 0)) { - /* SSLfatal() already called */ - goto err; - } - if (!WPACKET_close(pkt)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - } - - ret = 1; - err: - OPENSSL_free(pdata); - return ret; -} - -unsigned long ssl3_output_cert_chain(SSL_CONNECTION *s, WPACKET *pkt, - CERT_PKEY *cpk, int for_comp) +unsigned long ssl3_output_cert_chain(SSL *s, WPACKET *pkt, CERT_PKEY *cpk) { if (!WPACKET_start_sub_packet_u24(pkt)) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - if (!ssl_add_cert_chain(s, pkt, cpk, for_comp)) + if (!ssl_add_cert_chain(s, pkt, cpk)) return 0; if (!WPACKET_close(pkt)) { - if (!for_comp) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } @@ -1402,16 +1123,14 @@ unsigned long ssl3_output_cert_chain(SSL_CONNECTION *s, WPACKET *pkt, * in NBIO events. If |clearbufs| is set then init_buf and the wbio buffer is * freed up as well. */ -WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst, +WORK_STATE tls_finish_handshake(SSL *s, ossl_unused WORK_STATE wst, int clearbufs, int stop) { void (*cb) (const SSL *ssl, int type, int val) = NULL; int cleanuphand = s->statem.cleanuphand; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (clearbufs) { - if (!SSL_CONNECTION_IS_DTLS(s) + if (!SSL_IS_DTLS(s) #ifndef OPENSSL_NO_SCTP /* * RFC6083: SCTP provides a reliable and in-sequence transport service for DTLS @@ -1419,7 +1138,7 @@ WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst, * MUST NOT be used. * Hence the init_buf can be cleared when DTLS over SCTP as transport is used. */ - || BIO_dgram_is_sctp(SSL_get_wbio(ssl)) + || BIO_dgram_is_sctp(SSL_get_wbio(s)) #endif ) { /* @@ -1437,7 +1156,7 @@ WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst, s->init_num = 0; } - if (SSL_CONNECTION_IS_TLS13(s) && !s->server + if (SSL_IS_TLS13(s) && !s->server && s->post_handshake_auth == SSL_PHA_REQUESTED) s->post_handshake_auth = SSL_PHA_EXT_SENT; @@ -1459,14 +1178,14 @@ WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst, * In TLSv1.3 we update the cache as part of constructing the * NewSessionTicket */ - if (!SSL_CONNECTION_IS_TLS13(s)) + if (!SSL_IS_TLS13(s)) ssl_update_cache(s, SSL_SESS_CACHE_SERVER); /* N.B. s->ctx may not equal s->session_ctx */ - ssl_tsan_counter(sctx, &sctx->stats.sess_accept_good); + ssl_tsan_counter(s->ctx, &s->ctx->stats.sess_accept_good); s->handshake_func = ossl_statem_accept; } else { - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* * We encourage applications to only use TLSv1.3 tickets once, * so we remove this one from the cache. @@ -1490,7 +1209,7 @@ WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst, &s->session_ctx->stats.sess_connect_good); } - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* done with handshaking */ s->d1->handshake_read_seq = 0; s->d1->handshake_write_seq = 0; @@ -1501,17 +1220,17 @@ WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst, if (s->info_callback != NULL) cb = s->info_callback; - else if (sctx->info_callback != NULL) - cb = sctx->info_callback; + else if (s->ctx->info_callback != NULL) + cb = s->ctx->info_callback; /* The callback may expect us to not be in init at handshake done */ ossl_statem_set_in_init(s, 0); if (cb != NULL) { if (cleanuphand - || !SSL_CONNECTION_IS_TLS13(s) + || !SSL_IS_TLS13(s) || SSL_IS_FIRST_HANDSHAKE(s)) - cb(ssl, SSL_CB_HANDSHAKE_DONE, 1); + cb(s, SSL_CB_HANDSHAKE_DONE, 1); } if (!stop) { @@ -1523,23 +1242,22 @@ WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst, return WORK_FINISHED_STOP; } -int tls_get_message_header(SSL_CONNECTION *s, int *mt) +int tls_get_message_header(SSL *s, int *mt) { /* s->init_num < SSL3_HM_HEADER_LENGTH */ - int skip_message, i; - uint8_t recvd_type; + int skip_message, i, recvd_type; unsigned char *p; size_t l, readbytes; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); p = (unsigned char *)s->init_buf->data; do { while (s->init_num < SSL3_HM_HEADER_LENGTH) { - i = ssl->method->ssl_read_bytes(ssl, SSL3_RT_HANDSHAKE, &recvd_type, - &p[s->init_num], - SSL3_HM_HEADER_LENGTH - s->init_num, - 0, &readbytes); + /* QUIC: either create a special ssl_read_bytes... or if/else this */ + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, &recvd_type, + &p[s->init_num], + SSL3_HM_HEADER_LENGTH - s->init_num, + 0, &readbytes); if (i <= 0) { s->rwstate = SSL_READING; return 0; @@ -1594,7 +1312,7 @@ int tls_get_message_header(SSL_CONNECTION *s, int *mt) if (s->msg_callback) s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, - p, SSL3_HM_HEADER_LENGTH, ssl, + p, SSL3_HM_HEADER_LENGTH, s, s->msg_callback_arg); } } while (skip_message); @@ -1611,7 +1329,8 @@ int tls_get_message_header(SSL_CONNECTION *s, int *mt) * Total message size is the remaining record bytes to read * plus the SSL3_HM_HEADER_LENGTH bytes that we already read */ - l = s->rlayer.tlsrecs[0].length + SSL3_HM_HEADER_LENGTH; + l = RECORD_LAYER_get_rrec_length(&s->rlayer) + + SSL3_HM_HEADER_LENGTH; s->s3.tmp.message_size = l; s->init_msg = s->init_buf->data; @@ -1633,12 +1352,11 @@ int tls_get_message_header(SSL_CONNECTION *s, int *mt) return 1; } -int tls_get_message_body(SSL_CONNECTION *s, size_t *len) +int tls_get_message_body(SSL *s, size_t *len) { size_t n, readbytes; unsigned char *p; int i; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if (s->s3.tmp.message_type == SSL3_MT_CHANGE_CIPHER_SPEC) { /* We've already read everything in */ @@ -1649,8 +1367,8 @@ int tls_get_message_body(SSL_CONNECTION *s, size_t *len) p = s->init_msg; n = s->s3.tmp.message_size - s->init_num; while (n > 0) { - i = ssl->method->ssl_read_bytes(ssl, SSL3_RT_HANDSHAKE, NULL, - &p[s->init_num], n, 0, &readbytes); + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL, + &p[s->init_num], n, 0, &readbytes); if (i <= 0) { s->rwstate = SSL_READING; *len = 0; @@ -1680,7 +1398,7 @@ int tls_get_message_body(SSL_CONNECTION *s, size_t *len) } if (s->msg_callback) s->msg_callback(0, SSL2_VERSION, 0, s->init_buf->data, - (size_t)s->init_num, ssl, s->msg_callback_arg); + (size_t)s->init_num, s, s->msg_callback_arg); } else { /* * We defer feeding in the HRR until later. We'll do it as part of @@ -1690,9 +1408,8 @@ int tls_get_message_body(SSL_CONNECTION *s, size_t *len) */ #define SERVER_HELLO_RANDOM_OFFSET (SSL3_HM_HEADER_LENGTH + 2) /* KeyUpdate and NewSessionTicket do not need to be added */ - if (!SSL_CONNECTION_IS_TLS13(s) - || (s->s3.tmp.message_type != SSL3_MT_NEWSESSION_TICKET - && s->s3.tmp.message_type != SSL3_MT_KEY_UPDATE)) { + if (!SSL_IS_TLS13(s) || (s->s3.tmp.message_type != SSL3_MT_NEWSESSION_TICKET + && s->s3.tmp.message_type != SSL3_MT_KEY_UPDATE)) { if (s->s3.tmp.message_type != SSL3_MT_SERVER_HELLO || s->init_num < SERVER_HELLO_RANDOM_OFFSET + SSL3_RANDOM_SIZE || memcmp(hrrrandom, @@ -1708,7 +1425,7 @@ int tls_get_message_body(SSL_CONNECTION *s, size_t *len) } if (s->msg_callback) s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, s->init_buf->data, - (size_t)s->init_num + SSL3_HM_HEADER_LENGTH, ssl, + (size_t)s->init_num + SSL3_HM_HEADER_LENGTH, s, s->msg_callback_arg); } @@ -1772,30 +1489,22 @@ int ssl_x509err2alert(int x509err) return tp->alert; } -int ssl_allow_compression(SSL_CONNECTION *s) +int ssl_allow_compression(SSL *s) { if (s->options & SSL_OP_NO_COMPRESSION) return 0; return ssl_security(s, SSL_SECOP_COMPRESSION, 0, 0, NULL); } -/* - * SSL/TLS/DTLS version comparison - * - * Returns - * 0 if versiona is equal to versionb - * 1 if versiona is greater than versionb - * -1 if versiona is less than versionb - */ -int ssl_version_cmp(const SSL_CONNECTION *s, int versiona, int versionb) +static int version_cmp(const SSL *s, int a, int b) { - int dtls = SSL_CONNECTION_IS_DTLS(s); + int dtls = SSL_IS_DTLS(s); - if (versiona == versionb) + if (a == b) return 0; if (!dtls) - return versiona < versionb ? -1 : 1; - return DTLS_VERSION_LT(versiona, versionb) ? -1 : 1; + return a < b ? -1 : 1; + return DTLS_VERSION_LT(a, b) ? -1 : 1; } typedef struct { @@ -1867,17 +1576,17 @@ static const version_info dtls_version_table[] = { * * Returns 0 on success, or an SSL error reason on failure. */ -static int ssl_method_error(const SSL_CONNECTION *s, const SSL_METHOD *method) +static int ssl_method_error(const SSL *s, const SSL_METHOD *method) { int version = method->version; if ((s->min_proto_version != 0 && - ssl_version_cmp(s, version, s->min_proto_version) < 0) || + version_cmp(s, version, s->min_proto_version) < 0) || ssl_security(s, SSL_SECOP_VERSION, 0, version, NULL) == 0) return SSL_R_VERSION_TOO_LOW; if (s->max_proto_version != 0 && - ssl_version_cmp(s, version, s->max_proto_version) > 0) + version_cmp(s, version, s->max_proto_version) > 0) return SSL_R_VERSION_TOO_HIGH; if ((s->options & method->mask) != 0) @@ -1893,20 +1602,19 @@ static int ssl_method_error(const SSL_CONNECTION *s, const SSL_METHOD *method) * certificate type, or has PSK or a certificate callback configured, or has * a servername callback configure. Otherwise returns 0. */ -static int is_tls13_capable(const SSL_CONNECTION *s) +static int is_tls13_capable(const SSL *s) { - size_t i; + int i; int curve; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - if (!ossl_assert(sctx != NULL) || !ossl_assert(s->session_ctx != NULL)) + if (!ossl_assert(s->ctx != NULL) || !ossl_assert(s->session_ctx != NULL)) return 0; /* * A servername callback can change the available certs, so if a servername * cb is set then we just assume TLSv1.3 will be ok */ - if (sctx->ext.servername_cb != NULL + if (s->ctx->ext.servername_cb != NULL || s->session_ctx->ext.servername_cb != NULL) return 1; @@ -1918,18 +1626,20 @@ static int is_tls13_capable(const SSL_CONNECTION *s) if (s->psk_find_session_cb != NULL || s->cert->cert_cb != NULL) return 1; - /* All provider-based sig algs are required to support at least TLS1.3 */ - for (i = 0; i < s->ssl_pkey_num; i++) { + for (i = 0; i < SSL_PKEY_NUM; i++) { /* Skip over certs disallowed for TLSv1.3 */ switch (i) { case SSL_PKEY_DSA_SIGN: - case SSL_PKEY_GOST01: - case SSL_PKEY_GOST12_256: - case SSL_PKEY_GOST12_512: continue; default: break; } +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + if (s->enable_sign_by_dc + && ssl_has_dc(s, i)) { + /* nothing */ + } else +#endif if (!ssl_has_cert(s, i)) continue; if (i != SSL_PKEY_ECC) @@ -1939,7 +1649,13 @@ static int is_tls13_capable(const SSL_CONNECTION *s) * more restrictive so check that our sig algs are consistent with this * EC cert. See section 4.2.3 of RFC8446. */ - curve = ssl_get_EC_curve_nid(s->cert->pkeys[SSL_PKEY_ECC].privatekey); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + if (s->enable_sign_by_dc && ssl_has_dc(s, SSL_PKEY_ECC)) + curve = ssl_get_EC_curve_nid( + s->cert->dc_pkeys[SSL_PKEY_ECC].privatekey); + else +#endif + curve = ssl_get_EC_curve_nid(s->cert->pkeys[SSL_PKEY_ECC].privatekey); if (tls_check_sigalg_curve(s, curve)) return 1; } @@ -1956,16 +1672,15 @@ static int is_tls13_capable(const SSL_CONNECTION *s) * * Returns 1 when supported, otherwise 0 */ -int ssl_version_supported(const SSL_CONNECTION *s, int version, - const SSL_METHOD **meth) +int ssl_version_supported(const SSL *s, int version, const SSL_METHOD **meth) { const version_info *vent; const version_info *table; - switch (SSL_CONNECTION_GET_SSL(s)->method->version) { + switch (s->method->version) { default: /* Version should match method version for non-ANY method */ - return ssl_version_cmp(s, version, s->version) == 0; + return version_cmp(s, version, s->version) == 0; case TLS_ANY_VERSION: table = tls_version_table; break; @@ -1975,19 +1690,16 @@ int ssl_version_supported(const SSL_CONNECTION *s, int version, } for (vent = table; - vent->version != 0 && ssl_version_cmp(s, version, vent->version) <= 0; + vent->version != 0 && version_cmp(s, version, vent->version) <= 0; ++vent) { - const SSL_METHOD *(*thismeth)(void) = s->server ? vent->smeth - : vent->cmeth; - - if (thismeth != NULL - && ssl_version_cmp(s, version, vent->version) == 0 - && ssl_method_error(s, thismeth()) == 0 + if (vent->cmeth != NULL + && version_cmp(s, version, vent->version) == 0 + && ssl_method_error(s, vent->cmeth()) == 0 && (!s->server || version != TLS1_3_VERSION || is_tls13_capable(s))) { if (meth != NULL) - *meth = thismeth(); + *meth = vent->cmeth(); return 1; } } @@ -2003,27 +1715,26 @@ int ssl_version_supported(const SSL_CONNECTION *s, int version, * * Returns 1 when using the highest enabled version, 0 otherwise. */ -int ssl_check_version_downgrade(SSL_CONNECTION *s) +int ssl_check_version_downgrade(SSL *s) { const version_info *vent; const version_info *table; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); /* * Check that the current protocol is the highest enabled version - * (according to ssl->defltmethod, as version negotiation may have changed + * (according to s->ctx->method, as version negotiation may have changed * s->method). */ - if (s->version == ssl->defltmeth->version) + if (s->version == s->ctx->method->version) return 1; /* * Apparently we're using a version-flexible SSL_METHOD (not at its * highest protocol version). */ - if (ssl->defltmeth->version == TLS_method()->version) + if (s->ctx->method->version == TLS_method()->version) table = tls_version_table; - else if (ssl->defltmeth->version == DTLS_method()->version) + else if (s->ctx->method->version == DTLS_method()->version) table = dtls_version_table; else { /* Unexpected state; fail closed. */ @@ -2061,10 +1772,8 @@ int ssl_set_version_bound(int method_version, int version, int *bound) valid_tls = version >= SSL3_VERSION && version <= TLS_MAX_VERSION_INTERNAL; valid_dtls = - /* We support client side pre-standardisation version of DTLS */ - (version == DTLS1_BAD_VER) - || (DTLS_VERSION_LE(version, DTLS_MAX_VERSION_INTERNAL) - && DTLS_VERSION_GE(version, DTLS1_VERSION)); + DTLS_VERSION_LE(version, DTLS_MAX_VERSION_INTERNAL) && + DTLS_VERSION_GE(version, DTLS1_BAD_VER); if (!valid_tls && !valid_dtls) return 0; @@ -2100,12 +1809,12 @@ int ssl_set_version_bound(int method_version, int version, int *bound) return 1; } -static void check_for_downgrade(SSL_CONNECTION *s, int vers, DOWNGRADE *dgrd) +static void check_for_downgrade(SSL *s, int vers, DOWNGRADE *dgrd) { if (vers == TLS1_2_VERSION && ssl_version_supported(s, TLS1_3_VERSION, NULL)) { *dgrd = DOWNGRADE_TO_1_2; - } else if (!SSL_CONNECTION_IS_DTLS(s) + } else if (!SSL_IS_DTLS(s) && vers < TLS1_2_VERSION /* * We need to ensure that a server that disables TLSv1.2 @@ -2130,8 +1839,7 @@ static void check_for_downgrade(SSL_CONNECTION *s, int vers, DOWNGRADE *dgrd) * * Returns 0 on success or an SSL error reason number on failure. */ -int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, - DOWNGRADE *dgrd) +int ssl_choose_server_version(SSL *s, CLIENTHELLO_MSG *hello, DOWNGRADE *dgrd) { /*- * With version-flexible methods we have an initial state with: @@ -2142,8 +1850,7 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, * So we detect version-flexible methods via the method version, not the * handle version. */ - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - int server_version = ssl->method->version; + int server_version = s->method->version; int client_version = hello->legacy_version; const version_info *vent; const version_info *table; @@ -2154,8 +1861,8 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, switch (server_version) { default: - if (!SSL_CONNECTION_IS_TLS13(s)) { - if (ssl_version_cmp(s, client_version, s->version) < 0) + if (!SSL_IS_TLS13(s)) { + if (version_cmp(s, client_version, s->version) < 0) return SSL_R_WRONG_SSL_VERSION; *dgrd = DOWNGRADE_NONE; /* @@ -2186,7 +1893,7 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, if (!suppversions->present && s->hello_retry_request != SSL_HRR_NONE) return SSL_R_UNSUPPORTED_PROTOCOL; - if (suppversions->present && !SSL_CONNECTION_IS_DTLS(s)) { + if (suppversions->present && !SSL_IS_DTLS(s)) { unsigned int candidate_vers = 0; unsigned int best_vers = 0; const SSL_METHOD *best_method = NULL; @@ -2212,7 +1919,7 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, return SSL_R_BAD_LEGACY_VERSION; while (PACKET_get_net_2(&versionslist, &candidate_vers)) { - if (ssl_version_cmp(s, candidate_vers, best_vers) <= 0) + if (version_cmp(s, candidate_vers, best_vers) <= 0) continue; if (ssl_version_supported(s, candidate_vers, &best_method)) best_vers = candidate_vers; @@ -2234,10 +1941,7 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, } check_for_downgrade(s, best_vers, dgrd); s->version = best_vers; - ssl->method = best_method; - if (!ssl_set_record_protocol_version(s, best_vers)) - return ERR_R_INTERNAL_ERROR; - + s->method = best_method; return 0; } return SSL_R_UNSUPPORTED_PROTOCOL; @@ -2247,7 +1951,7 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, * If the supported versions extension isn't present, then the highest * version we can negotiate is TLSv1.2 */ - if (ssl_version_cmp(s, client_version, TLS1_3_VERSION) >= 0) + if (version_cmp(s, client_version, TLS1_3_VERSION) >= 0) client_version = TLS1_2_VERSION; /* @@ -2258,16 +1962,13 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, const SSL_METHOD *method; if (vent->smeth == NULL || - ssl_version_cmp(s, client_version, vent->version) < 0) + version_cmp(s, client_version, vent->version) < 0) continue; method = vent->smeth(); if (ssl_method_error(s, method) == 0) { check_for_downgrade(s, vent->version, dgrd); s->version = vent->version; - ssl->method = method; - if (!ssl_set_record_protocol_version(s, s->version)) - return ERR_R_INTERNAL_ERROR; - + s->method = method; return 0; } disabled = 1; @@ -2286,13 +1987,11 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello, * * Returns 1 on success or 0 on error. */ -int ssl_choose_client_version(SSL_CONNECTION *s, int version, - RAW_EXTENSION *extensions) +int ssl_choose_client_version(SSL *s, int version, RAW_EXTENSION *extensions) { const version_info *vent; const version_info *table; int ret, ver_min, ver_max, real_max, origv; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); origv = s->version; s->version = version; @@ -2313,9 +2012,9 @@ int ssl_choose_client_version(SSL_CONNECTION *s, int version, return 0; } - switch (ssl->method->version) { + switch (s->method->version) { default: - if (s->version != ssl->method->version) { + if (s->version != s->method->version) { s->version = origv; SSLfatal(s, SSL_AD_PROTOCOL_VERSION, SSL_R_WRONG_SSL_VERSION); return 0; @@ -2327,10 +2026,6 @@ int ssl_choose_client_version(SSL_CONNECTION *s, int version, * versions they don't want. If not, then easy to fix, just return * ssl_method_error(s, s->method) */ - if (!ssl_set_record_protocol_version(s, s->version)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } return 1; case TLS_ANY_VERSION: table = tls_version_table; @@ -2346,8 +2041,13 @@ int ssl_choose_client_version(SSL_CONNECTION *s, int version, SSLfatal(s, SSL_AD_PROTOCOL_VERSION, ret); return 0; } - if (ssl_version_cmp(s, s->version, ver_min) < 0 - || ssl_version_cmp(s, s->version, ver_max) > 0) { + if (SSL_IS_DTLS(s) ? DTLS_VERSION_LT(s->version, ver_min) + : s->version < ver_min) { + s->version = origv; + SSLfatal(s, SSL_AD_PROTOCOL_VERSION, SSL_R_UNSUPPORTED_PROTOCOL); + return 0; + } else if (SSL_IS_DTLS(s) ? DTLS_VERSION_GT(s->version, ver_max) + : s->version > ver_max) { s->version = origv; SSLfatal(s, SSL_AD_PROTOCOL_VERSION, SSL_R_UNSUPPORTED_PROTOCOL); return 0; @@ -2367,7 +2067,7 @@ int ssl_choose_client_version(SSL_CONNECTION *s, int version, SSL_R_INAPPROPRIATE_FALLBACK); return 0; } - } else if (!SSL_CONNECTION_IS_DTLS(s) + } else if (!SSL_IS_DTLS(s) && s->version < TLS1_2_VERSION && real_max > s->version) { if (memcmp(tls11downgrade, @@ -2385,11 +2085,7 @@ int ssl_choose_client_version(SSL_CONNECTION *s, int version, if (vent->cmeth == NULL || s->version != vent->version) continue; - ssl->method = vent->cmeth(); - if (!ssl_set_record_protocol_version(s, s->version)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } + s->method = vent->cmeth(); return 1; } @@ -2420,17 +2116,17 @@ int ssl_choose_client_version(SSL_CONNECTION *s, int version, * Returns 0 on success or an SSL error reason number on failure. On failure * min_version and max_version will also be set to 0. */ -int ssl_get_min_max_version(const SSL_CONNECTION *s, int *min_version, - int *max_version, int *real_max) +int ssl_get_min_max_version(const SSL *s, int *min_version, int *max_version, + int *real_max) { int version, tmp_real_max; int hole; + const SSL_METHOD *single = NULL; const SSL_METHOD *method; const version_info *table; const version_info *vent; - const SSL *ssl = SSL_CONNECTION_GET_SSL(s); - switch (ssl->method->version) { + switch (s->method->version) { default: /* * If this SSL handle is not from a version flexible method we don't @@ -2466,12 +2162,13 @@ int ssl_get_min_max_version(const SSL_CONNECTION *s, int *min_version, * the valid protocol entries) and we don't have a selected version yet. * * Whenever "hole == 1", and we hit an enabled method, its version becomes - * the selected version. We're no longer in a hole, so "hole" becomes 0. + * the selected version, and the method becomes a candidate "single" + * method. We're no longer in a hole, so "hole" becomes 0. * - * If "hole == 0" and we hit an enabled method, we support a contiguous - * range of at least two methods. If we hit a disabled method, - * then hole becomes true again, but nothing else changes yet, - * because all the remaining methods may be disabled too. + * If "hole == 0" and we hit an enabled method, then "single" is cleared, + * as we support a contiguous range of at least two methods. If we hit + * a disabled method, then hole becomes true again, but nothing else + * changes yet, because all the remaining methods may be disabled too. * If we again hit an enabled method after the new hole, it becomes * selected, as we start from scratch. */ @@ -2498,11 +2195,12 @@ int ssl_get_min_max_version(const SSL_CONNECTION *s, int *min_version, if (ssl_method_error(s, method) != 0) { hole = 1; } else if (!hole) { + single = NULL; *min_version = method->version; } else { if (real_max != NULL && tmp_real_max != 0) *real_max = tmp_real_max; - version = method->version; + version = (single = method)->version; *min_version = version; hole = 0; } @@ -2525,7 +2223,7 @@ int ssl_get_min_max_version(const SSL_CONNECTION *s, int *min_version, * * Returns 0 on success or an SSL error reason number on failure. */ -int ssl_set_client_hello_version(SSL_CONNECTION *s) +int ssl_set_client_hello_version(SSL *s) { int ver_min, ver_max, ret; @@ -2543,22 +2241,9 @@ int ssl_set_client_hello_version(SSL_CONNECTION *s) s->version = ver_max; - if (SSL_CONNECTION_IS_DTLS(s)) { - if (ver_max == DTLS1_BAD_VER) { - /* - * Even though this is technically before version negotiation, - * because we have asked for DTLS1_BAD_VER we will never negotiate - * anything else, and this has impacts on the record layer for when - * we read the ServerHello. So we need to tell the record layer - * about this immediately. - */ - if (!ssl_set_record_protocol_version(s, ver_max)) - return 0; - } - } else if (ver_max > TLS1_2_VERSION) { - /* TLS1.3 always uses TLS1.2 in the legacy_version field */ + /* TLS1.3 always uses TLS1.2 in the legacy_version field */ + if (!SSL_IS_DTLS(s) && ver_max > TLS1_2_VERSION) ver_max = TLS1_2_VERSION; - } s->client_version = ver_max; return 0; @@ -2570,7 +2255,7 @@ int ssl_set_client_hello_version(SSL_CONNECTION *s) * used. Returns 1 if the group is in the list (and allowed if |checkallow| is * 1) or 0 otherwise. */ -int check_in_list(SSL_CONNECTION *s, uint16_t group_id, const uint16_t *groups, +int check_in_list(SSL *s, uint16_t group_id, const uint16_t *groups, size_t num_groups, int checkallow) { size_t i; @@ -2592,8 +2277,7 @@ int check_in_list(SSL_CONNECTION *s, uint16_t group_id, const uint16_t *groups, } /* Replace ClientHello1 in the transcript hash with a synthetic message */ -int create_synthetic_message_hash(SSL_CONNECTION *s, - const unsigned char *hashval, +int create_synthetic_message_hash(SSL *s, const unsigned char *hashval, size_t hashlen, const unsigned char *hrr, size_t hrrlen) { @@ -2651,14 +2335,14 @@ static int ca_dn_cmp(const X509_NAME *const *a, const X509_NAME *const *b) return X509_NAME_cmp(*a, *b); } -int parse_ca_names(SSL_CONNECTION *s, PACKET *pkt) +int parse_ca_names(SSL *s, PACKET *pkt) { STACK_OF(X509_NAME) *ca_sk = sk_X509_NAME_new(ca_dn_cmp); X509_NAME *xn = NULL; PACKET cadns; if (ca_sk == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } /* get the CA RDNs */ @@ -2688,7 +2372,7 @@ int parse_ca_names(SSL_CONNECTION *s, PACKET *pkt) } if (!sk_X509_NAME_push(ca_sk, xn)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } xn = NULL; @@ -2705,25 +2389,23 @@ int parse_ca_names(SSL_CONNECTION *s, PACKET *pkt) return 0; } -const STACK_OF(X509_NAME) *get_ca_names(SSL_CONNECTION *s) +const STACK_OF(X509_NAME) *get_ca_names(SSL *s) { - const STACK_OF(X509_NAME) *ca_sk = NULL; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); + const STACK_OF(X509_NAME) *ca_sk = NULL;; if (s->server) { - ca_sk = SSL_get_client_CA_list(ssl); + ca_sk = SSL_get_client_CA_list(s); if (ca_sk != NULL && sk_X509_NAME_num(ca_sk) == 0) ca_sk = NULL; } if (ca_sk == NULL) - ca_sk = SSL_get0_CA_list(ssl); + ca_sk = SSL_get0_CA_list(s); return ca_sk; } -int construct_ca_names(SSL_CONNECTION *s, const STACK_OF(X509_NAME) *ca_sk, - WPACKET *pkt) +int construct_ca_names(SSL *s, const STACK_OF(X509_NAME) *ca_sk, WPACKET *pkt) { /* Start sub-packet for client CA list */ if (!WPACKET_start_sub_packet_u16(pkt)) { @@ -2759,14 +2441,14 @@ int construct_ca_names(SSL_CONNECTION *s, const STACK_OF(X509_NAME) *ca_sk, } /* Create a buffer containing data to be signed for server key exchange */ -size_t construct_key_exchange_tbs(SSL_CONNECTION *s, unsigned char **ptbs, +size_t construct_key_exchange_tbs(SSL *s, unsigned char **ptbs, const void *param, size_t paramlen) { size_t tbslen = 2 * SSL3_RANDOM_SIZE + paramlen; unsigned char *tbs = OPENSSL_malloc(tbslen); if (tbs == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } memcpy(tbs, s->s3.client_random, SSL3_RANDOM_SIZE); @@ -2782,7 +2464,7 @@ size_t construct_key_exchange_tbs(SSL_CONNECTION *s, unsigned char **ptbs, * Saves the current handshake digest for Post-Handshake Auth, * Done after ClientFinished is processed, done exactly once */ -int tls13_save_handshake_digest_for_pha(SSL_CONNECTION *s) +int tls13_save_handshake_digest_for_pha(SSL *s) { if (s->pha_dgst == NULL) { if (!ssl3_digest_cached_records(s, 1)) @@ -2809,7 +2491,7 @@ int tls13_save_handshake_digest_for_pha(SSL_CONNECTION *s) * Restores the Post-Handshake Auth handshake digest * Done just before sending/processing the Cert Request */ -int tls13_restore_handshake_digest_for_pha(SSL_CONNECTION *s) +int tls13_restore_handshake_digest_for_pha(SSL *s) { if (s->pha_dgst == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); @@ -2823,75 +2505,174 @@ int tls13_restore_handshake_digest_for_pha(SSL_CONNECTION *s) return 1; } -#ifndef OPENSSL_NO_COMP_ALG -MSG_PROCESS_RETURN tls13_process_compressed_certificate(SSL_CONNECTION *sc, - PACKET *pkt, - PACKET *tmppkt, - BUF_MEM *buf) +#ifndef OPENSSL_NO_CERT_COMPRESSION +int tls_construct_compressed_certificate(SSL *s, WPACKET *pkt, + confunc_f confunc) { - MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR; - int comp_alg; - COMP_METHOD *method = NULL; - COMP_CTX *comp = NULL; - size_t expected_length; - size_t comp_length; + SSL_cert_compress_cb_fn compress_fn = NULL; + unsigned char *compressed_msg = NULL; + size_t compressed_size; + size_t uncompressed_length; + WPACKET inpkt; + BUF_MEM *buf = NULL; int i; - int found = 0; - if (buf == NULL) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + if ((buf = BUF_MEM_new()) == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + + if (!BUF_MEM_grow(buf, SSL3_RT_MAX_PLAIN_LENGTH)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } - if (!PACKET_get_net_2(pkt, (unsigned int*)&comp_alg)) { - SSLfatal(sc, SSL_AD_BAD_CERTIFICATE, ERR_R_INTERNAL_ERROR); + + if (!WPACKET_init(&inpkt, buf)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } - /* If we have a prefs list, make sure the algorithm is in it */ - if (sc->cert_comp_prefs[0] != TLSEXT_comp_cert_none) { - for (i = 0; sc->cert_comp_prefs[i] != TLSEXT_comp_cert_none; i++) { - if (sc->cert_comp_prefs[i] == comp_alg) { - found = 1; - break; - } + + if (!confunc(s, &inpkt)) { + WPACKET_cleanup(&inpkt); + /* SSLfatal() already called */ + goto err; + } + + if (!WPACKET_get_length(&inpkt, &uncompressed_length) + || !WPACKET_finish(&inpkt)) { + WPACKET_cleanup(&inpkt); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + for (i = 0; i < sk_CERT_COMP_num(s->cert_comp_algs); i++) { + CERT_COMP *comp = sk_CERT_COMP_value(s->cert_comp_algs, i); + + if (comp->alg_id == s->cert_comp_compress_id + && comp->compress) { + compress_fn = comp->compress; + break; } - if (!found) { - SSLfatal(sc, SSL_AD_BAD_CERTIFICATE, SSL_R_BAD_COMPRESSION_ALGORITHM); - goto err; + } + + if (compress_fn == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* get compressed_size */ + if (!compress_fn(s, (const unsigned char *)buf->data, uncompressed_length, + NULL, &compressed_size)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + compressed_msg = OPENSSL_malloc(compressed_size); + if (compressed_msg == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!compress_fn(s, (const unsigned char *)buf->data, uncompressed_length, + compressed_msg, &compressed_size)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (!WPACKET_put_bytes_u16(pkt, s->cert_comp_compress_id) + || !WPACKET_put_bytes_u24(pkt, uncompressed_length) + || !WPACKET_sub_memcpy_u24(pkt, compressed_msg, compressed_size)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + BUF_MEM_free(buf); + OPENSSL_free(compressed_msg); + + return 1; +err: + BUF_MEM_free(buf); + + if (compressed_msg != NULL) + OPENSSL_free(compressed_msg); + + return 0; +} + +MSG_PROCESS_RETURN tls_process_compressed_certificate(SSL *s, + PACKET *pkt, + profunc_f profunc) +{ + SSL_cert_decompress_cb_fn decompress_fn = NULL; + MSG_PROCESS_RETURN ret; + unsigned char *out = NULL; + unsigned int alg_id; + size_t msg_len; + unsigned long uncompressed_len; + PACKET subpkt; + int i; + + if (s->cert_comp_algs == NULL) { + SSLfatal(s, SSL_AD_BAD_CERTIFICATE, SSL_R_BAD_PACKET); + goto err; + } + + if (!PACKET_get_net_2(pkt, &alg_id) + || !PACKET_get_net_3(pkt, &uncompressed_len) + || !PACKET_get_net_3_len(pkt, &msg_len) + || !PACKET_get_sub_packet(pkt, &subpkt, msg_len) + || PACKET_remaining(pkt) != 0) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_PACKET); + goto err; + } + + if (uncompressed_len > s->max_cert_list) { + SSLfatal(s, SSL_AD_BAD_CERTIFICATE, SSL_R_LENGTH_TOO_LONG); + goto err; + } + + for (i = 0; i < sk_CERT_COMP_num(s->cert_comp_algs); i++) { + CERT_COMP *comp = sk_CERT_COMP_value(s->cert_comp_algs, i); + + if ((comp->alg_id == alg_id) && comp->decompress) { + decompress_fn = comp->decompress; + break; } } - if (!ossl_comp_has_alg(comp_alg)) { - SSLfatal(sc, SSL_AD_BAD_CERTIFICATE, SSL_R_BAD_COMPRESSION_ALGORITHM); + + if (decompress_fn == NULL) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_PACKET); goto err; } - switch (comp_alg) { - case TLSEXT_comp_cert_zlib: - method = COMP_zlib_oneshot(); - break; - case TLSEXT_comp_cert_brotli: - method = COMP_brotli_oneshot(); - break; - case TLSEXT_comp_cert_zstd: - method = COMP_zstd_oneshot(); - break; - default: - SSLfatal(sc, SSL_AD_BAD_CERTIFICATE, SSL_R_BAD_COMPRESSION_ALGORITHM); + + s->cert_comp_decompress_id = alg_id; + + out = OPENSSL_malloc(uncompressed_len); + if (out == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } - if ((comp = COMP_CTX_new(method)) == NULL - || !PACKET_get_net_3_len(pkt, &expected_length) - || !PACKET_get_net_3_len(pkt, &comp_length) - || PACKET_remaining(pkt) != comp_length - || !BUF_MEM_grow(buf, expected_length) - || !PACKET_buf_init(tmppkt, (unsigned char *)buf->data, expected_length) - || COMP_expand_block(comp, (unsigned char *)buf->data, expected_length, - (unsigned char*)PACKET_data(pkt), comp_length) != (int)expected_length) { - SSLfatal(sc, SSL_AD_BAD_CERTIFICATE, SSL_R_BAD_DECOMPRESSION); + if (!decompress_fn(s, PACKET_data(&subpkt), msg_len, + out, uncompressed_len)) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_PACKET); goto err; } - ret = MSG_PROCESS_CONTINUE_PROCESSING; - err: - COMP_CTX_free(comp); + + if (!PACKET_buf_init(&subpkt, out, uncompressed_len)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + ret = profunc(s, &subpkt); + + OPENSSL_free(out); + return ret; +err: + if (out != NULL) + OPENSSL_free(out); + + return MSG_PROCESS_ERROR; } #endif diff --git a/openssl/src/ssl/statem/statem_local.h b/openssl/src/ssl/statem/statem_local.h index 04114b1e2..c152c4883 100644 --- a/openssl/src/ssl/statem/statem_local.h +++ b/openssl/src/ssl/statem/statem_local.h @@ -1,5 +1,5 @@ /* - * Copyright 2015-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2015-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -31,19 +31,22 @@ /* Max ServerHello size permitted by RFC 8446 */ #define SERVER_HELLO_MAX_LENGTH 65607 -/* Max CertificateVerify size permitted by RFC 8446 */ -#define CERTIFICATE_VERIFY_MAX_LENGTH 65539 - /* Max should actually be 36 but we are generous */ #define FINISHED_MAX_LENGTH 64 /* Dummy message type */ #define SSL3_MT_DUMMY -1 -/* Invalid extension ID for non-supported extensions */ -#define TLSEXT_TYPE_invalid 0x10000 -#define TLSEXT_TYPE_out_of_range 0x10001 -unsigned int ossl_get_extension_type(size_t idx); +#ifndef OPENSSL_NO_SM2 +/* + * standard handshake sm2-id and cert verify id is defined in RFC 8998 + */ +# define HANDSHAKE_SM2_ID "TLSv1.3+GM+Cipher+Suite" +# define HANDSHAKE_SM2_ID_LEN sizeof(HANDSHAKE_SM2_ID) - 1 +# define CERTVRIFY_SM2_ID "1234567812345678" +# define CERTVRIFY_SM2_ID_LEN sizeof(CERTVRIFY_SM2_ID) - 1 + +#endif extern const unsigned char hrrrandom[]; @@ -62,188 +65,115 @@ typedef enum { MSG_PROCESS_CONTINUE_READING } MSG_PROCESS_RETURN; -typedef CON_FUNC_RETURN (*confunc_f) (SSL_CONNECTION *s, WPACKET *pkt); +typedef int (*confunc_f) (SSL *s, WPACKET *pkt); +typedef MSG_PROCESS_RETURN (*profunc_f) (SSL *s, PACKET *pkt); -int ssl3_take_mac(SSL_CONNECTION *s); -int check_in_list(SSL_CONNECTION *s, uint16_t group_id, const uint16_t *groups, +int ssl3_take_mac(SSL *s); +int check_in_list(SSL *s, uint16_t group_id, const uint16_t *groups, size_t num_groups, int checkallow); -int create_synthetic_message_hash(SSL_CONNECTION *s, - const unsigned char *hashval, +int create_synthetic_message_hash(SSL *s, const unsigned char *hashval, size_t hashlen, const unsigned char *hrr, size_t hrrlen); -int parse_ca_names(SSL_CONNECTION *s, PACKET *pkt); -const STACK_OF(X509_NAME) *get_ca_names(SSL_CONNECTION *s); -int construct_ca_names(SSL_CONNECTION *s, const STACK_OF(X509_NAME) *ca_sk, - WPACKET *pkt); -size_t construct_key_exchange_tbs(SSL_CONNECTION *s, unsigned char **ptbs, +int parse_ca_names(SSL *s, PACKET *pkt); +const STACK_OF(X509_NAME) *get_ca_names(SSL *s); +int construct_ca_names(SSL *s, const STACK_OF(X509_NAME) *ca_sk, WPACKET *pkt); +size_t construct_key_exchange_tbs(SSL *s, unsigned char **ptbs, const void *param, size_t paramlen); /* * TLS/DTLS client state machine functions */ -int ossl_statem_client_read_transition(SSL_CONNECTION *s, int mt); -WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s); -WORK_STATE ossl_statem_client_pre_work(SSL_CONNECTION *s, WORK_STATE wst); -WORK_STATE ossl_statem_client_post_work(SSL_CONNECTION *s, WORK_STATE wst); -int ossl_statem_client_construct_message(SSL_CONNECTION *s, +int ossl_statem_client_read_transition(SSL *s, int mt); +WRITE_TRAN ossl_statem_client_write_transition(SSL *s); +WORK_STATE ossl_statem_client_pre_work(SSL *s, WORK_STATE wst); +WORK_STATE ossl_statem_client_post_work(SSL *s, WORK_STATE wst); +int ossl_statem_client_construct_message(SSL *s, WPACKET *pkt, confunc_f *confunc, int *mt); -size_t ossl_statem_client_max_message_size(SSL_CONNECTION *s); -MSG_PROCESS_RETURN ossl_statem_client_process_message(SSL_CONNECTION *s, - PACKET *pkt); -WORK_STATE ossl_statem_client_post_process_message(SSL_CONNECTION *s, - WORK_STATE wst); +size_t ossl_statem_client_max_message_size(SSL *s); +MSG_PROCESS_RETURN ossl_statem_client_process_message(SSL *s, PACKET *pkt); +WORK_STATE ossl_statem_client_post_process_message(SSL *s, WORK_STATE wst); /* * TLS/DTLS server state machine functions */ -int ossl_statem_server_read_transition(SSL_CONNECTION *s, int mt); -WRITE_TRAN ossl_statem_server_write_transition(SSL_CONNECTION *s); -WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst); -WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst); -int ossl_statem_server_construct_message(SSL_CONNECTION *s, +int ossl_statem_server_read_transition(SSL *s, int mt); +WRITE_TRAN ossl_statem_server_write_transition(SSL *s); +WORK_STATE ossl_statem_server_pre_work(SSL *s, WORK_STATE wst); +WORK_STATE ossl_statem_server_post_work(SSL *s, WORK_STATE wst); +int ossl_statem_server_construct_message(SSL *s, WPACKET *pkt, confunc_f *confunc,int *mt); -size_t ossl_statem_server_max_message_size(SSL_CONNECTION *s); -MSG_PROCESS_RETURN ossl_statem_server_process_message(SSL_CONNECTION *s, - PACKET *pkt); -WORK_STATE ossl_statem_server_post_process_message(SSL_CONNECTION *s, - WORK_STATE wst); +size_t ossl_statem_server_max_message_size(SSL *s); +MSG_PROCESS_RETURN ossl_statem_server_process_message(SSL *s, PACKET *pkt); +WORK_STATE ossl_statem_server_post_process_message(SSL *s, WORK_STATE wst); /* Functions for getting new message data */ -__owur int tls_get_message_header(SSL_CONNECTION *s, int *mt); -__owur int tls_get_message_body(SSL_CONNECTION *s, size_t *len); -__owur int dtls_get_message(SSL_CONNECTION *s, int *mt); -__owur int dtls_get_message_body(SSL_CONNECTION *s, size_t *len); +__owur int tls_get_message_header(SSL *s, int *mt); +__owur int tls_get_message_body(SSL *s, size_t *len); +__owur int dtls_get_message(SSL *s, int *mt); +__owur int dtls_get_message_body(SSL *s, size_t *len); +#ifndef OPENSSL_NO_QUIC +__owur int quic_get_message(SSL *s, int *mt, size_t *len); +#endif /* Message construction and processing functions */ -__owur int tls_process_initial_server_flight(SSL_CONNECTION *s); -__owur MSG_PROCESS_RETURN tls_process_change_cipher_spec(SSL_CONNECTION *s, - PACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_finished(SSL_CONNECTION *s, PACKET *pkt); -__owur CON_FUNC_RETURN tls_construct_change_cipher_spec(SSL_CONNECTION *s, - WPACKET *pkt); -__owur CON_FUNC_RETURN dtls_construct_change_cipher_spec(SSL_CONNECTION *s, - WPACKET *pkt); - -__owur CON_FUNC_RETURN tls_construct_finished(SSL_CONNECTION *s, WPACKET *pkt); -__owur CON_FUNC_RETURN tls_construct_key_update(SSL_CONNECTION *s, WPACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_key_update(SSL_CONNECTION *s, - PACKET *pkt); -__owur WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, WORK_STATE wst, - int clearbufs, int stop); -__owur WORK_STATE dtls_wait_for_dry(SSL_CONNECTION *s); - -#ifndef OPENSSL_NO_COMP_ALG -__owur MSG_PROCESS_RETURN tls13_process_compressed_certificate(SSL_CONNECTION *sc, - PACKET *pkt, - PACKET *tmppkt, - BUF_MEM *buf); -#endif +__owur int tls_process_initial_server_flight(SSL *s); +__owur MSG_PROCESS_RETURN tls_process_change_cipher_spec(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_finished(SSL *s, PACKET *pkt); +__owur int tls_construct_change_cipher_spec(SSL *s, WPACKET *pkt); +__owur int dtls_construct_change_cipher_spec(SSL *s, WPACKET *pkt); + +__owur int tls_construct_finished(SSL *s, WPACKET *pkt); +__owur int tls_construct_key_update(SSL *s, WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_key_update(SSL *s, PACKET *pkt); +__owur WORK_STATE tls_finish_handshake(SSL *s, WORK_STATE wst, int clearbufs, + int stop); +__owur WORK_STATE dtls_wait_for_dry(SSL *s); /* some client-only functions */ -__owur CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, - WPACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, - PACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_certificate_request(SSL_CONNECTION *s, - PACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL_CONNECTION *s, - PACKET *pkt); -__owur int tls_process_cert_status_body(SSL_CONNECTION *s, PACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_cert_status(SSL_CONNECTION *s, - PACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_server_done(SSL_CONNECTION *s, - PACKET *pkt); -__owur CON_FUNC_RETURN tls_construct_cert_verify(SSL_CONNECTION *s, - WPACKET *pkt); -__owur WORK_STATE tls_prepare_client_certificate(SSL_CONNECTION *s, - WORK_STATE wst); -__owur CON_FUNC_RETURN tls_construct_client_certificate(SSL_CONNECTION *s, - WPACKET *pkt); -#ifndef OPENSSL_NO_COMP_ALG -__owur CON_FUNC_RETURN tls_construct_client_compressed_certificate(SSL_CONNECTION *sc, - WPACKET *pkt); -#endif -__owur int ssl_do_client_cert_cb(SSL_CONNECTION *s, X509 **px509, - EVP_PKEY **ppkey); -__owur CON_FUNC_RETURN tls_construct_client_key_exchange(SSL_CONNECTION *s, - WPACKET *pkt); -__owur int tls_client_key_exchange_post_work(SSL_CONNECTION *s); -__owur int tls_construct_cert_status_body(SSL_CONNECTION *s, WPACKET *pkt); -__owur CON_FUNC_RETURN tls_construct_cert_status(SSL_CONNECTION *s, - WPACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_key_exchange(SSL_CONNECTION *s, - PACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_server_rpk(SSL_CONNECTION *sc, - PACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_client_rpk(SSL_CONNECTION *sc, - PACKET *pkt); -__owur unsigned long tls_output_rpk(SSL_CONNECTION *sc, WPACKET *pkt, - CERT_PKEY *cpk); -__owur int tls_process_rpk(SSL_CONNECTION *s, PACKET *pkt, EVP_PKEY **peer_rpk); -__owur MSG_PROCESS_RETURN tls_process_server_certificate(SSL_CONNECTION *s, - PACKET *pkt); -__owur WORK_STATE tls_post_process_server_certificate(SSL_CONNECTION *s, - WORK_STATE wst); -#ifndef OPENSSL_NO_COMP_ALG -__owur MSG_PROCESS_RETURN tls_process_server_compressed_certificate(SSL_CONNECTION *sc, - PACKET *pkt); -#endif -__owur int ssl3_check_cert_and_algorithm(SSL_CONNECTION *s); +__owur int tls_construct_client_hello(SSL *s, WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_server_hello(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_certificate_request(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL *s, PACKET *pkt); +__owur int tls_process_cert_status_body(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_cert_status(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_server_done(SSL *s, PACKET *pkt); +__owur int tls_construct_cert_verify(SSL *s, WPACKET *pkt); +__owur WORK_STATE tls_prepare_client_certificate(SSL *s, WORK_STATE wst); +__owur int tls_construct_client_certificate(SSL *s, WPACKET *pkt); +__owur int ssl_do_client_cert_cb(SSL *s, X509 **px509, EVP_PKEY **ppkey); +__owur int tls_construct_client_key_exchange(SSL *s, WPACKET *pkt); +__owur int tls_client_key_exchange_post_work(SSL *s); +__owur int tls_construct_cert_status_body(SSL *s, WPACKET *pkt); +__owur int tls_construct_cert_status(SSL *s, WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_key_exchange(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_server_certificate(SSL *s, PACKET *pkt); +__owur WORK_STATE tls_post_process_server_certificate(SSL *s, WORK_STATE wst); +__owur int ssl3_check_cert_and_algorithm(SSL *s); #ifndef OPENSSL_NO_NEXTPROTONEG -__owur CON_FUNC_RETURN tls_construct_next_proto(SSL_CONNECTION *s, WPACKET *pkt); +__owur int tls_construct_next_proto(SSL *s, WPACKET *pkt); #endif -__owur MSG_PROCESS_RETURN tls_process_hello_req(SSL_CONNECTION *s, PACKET *pkt); -__owur MSG_PROCESS_RETURN dtls_process_hello_verify(SSL_CONNECTION *s, PACKET *pkt); -__owur CON_FUNC_RETURN tls_construct_end_of_early_data(SSL_CONNECTION *s, - WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_hello_req(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN dtls_process_hello_verify(SSL *s, PACKET *pkt); +__owur int tls_construct_end_of_early_data(SSL *s, WPACKET *pkt); /* some server-only functions */ -__owur MSG_PROCESS_RETURN tls_process_client_hello(SSL_CONNECTION *s, - PACKET *pkt); -__owur WORK_STATE tls_post_process_client_hello(SSL_CONNECTION *s, - WORK_STATE wst); -__owur CON_FUNC_RETURN tls_construct_server_hello(SSL_CONNECTION *s, - WPACKET *pkt); -__owur CON_FUNC_RETURN dtls_construct_hello_verify_request(SSL_CONNECTION *s, - WPACKET *pkt); -__owur CON_FUNC_RETURN tls_construct_server_certificate(SSL_CONNECTION *s, - WPACKET *pkt); -#ifndef OPENSSL_NO_COMP_ALG -__owur CON_FUNC_RETURN tls_construct_server_compressed_certificate(SSL_CONNECTION *sc, - WPACKET *pkt); -#endif -__owur CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, - WPACKET *pkt); -__owur CON_FUNC_RETURN tls_construct_certificate_request(SSL_CONNECTION *s, - WPACKET *pkt); -__owur CON_FUNC_RETURN tls_construct_server_done(SSL_CONNECTION *s, - WPACKET *pkt); -__owur MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, - PACKET *pkt); -#ifndef OPENSSL_NO_COMP_ALG -__owur MSG_PROCESS_RETURN tls_process_client_compressed_certificate(SSL_CONNECTION *sc, - PACKET *pkt); -#endif -__owur MSG_PROCESS_RETURN tls_process_client_key_exchange(SSL_CONNECTION *s, - PACKET *pkt); -__owur WORK_STATE tls_post_process_client_key_exchange(SSL_CONNECTION *s, - WORK_STATE wst); -__owur MSG_PROCESS_RETURN tls_process_cert_verify(SSL_CONNECTION *s, - PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_client_hello(SSL *s, PACKET *pkt); +__owur WORK_STATE tls_post_process_client_hello(SSL *s, WORK_STATE wst); +__owur int tls_construct_server_hello(SSL *s, WPACKET *pkt); +__owur int dtls_construct_hello_verify_request(SSL *s, WPACKET *pkt); +__owur int tls_construct_server_certificate(SSL *s, WPACKET *pkt); +__owur int tls_construct_server_key_exchange(SSL *s, WPACKET *pkt); +__owur int tls_construct_certificate_request(SSL *s, WPACKET *pkt); +__owur int tls_construct_server_done(SSL *s, WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_client_certificate(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_client_key_exchange(SSL *s, PACKET *pkt); +__owur WORK_STATE tls_post_process_client_key_exchange(SSL *s, WORK_STATE wst); +__owur MSG_PROCESS_RETURN tls_process_cert_verify(SSL *s, PACKET *pkt); #ifndef OPENSSL_NO_NEXTPROTONEG -__owur MSG_PROCESS_RETURN tls_process_next_proto(SSL_CONNECTION *s, - PACKET *pkt); -#endif -__owur CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, - WPACKET *pkt); -MSG_PROCESS_RETURN tls_process_end_of_early_data(SSL_CONNECTION *s, - PACKET *pkt); - -#ifndef OPENSSL_NO_GOST -/* These functions are used in GOST18 CKE, both for client and server */ -int ossl_gost18_cke_cipher_nid(const SSL_CONNECTION *s); -int ossl_gost_ukm(const SSL_CONNECTION *s, unsigned char *dgst_buf); +__owur MSG_PROCESS_RETURN tls_process_next_proto(SSL *s, PACKET *pkt); #endif +__owur int tls_construct_new_session_ticket(SSL *s, WPACKET *pkt); +MSG_PROCESS_RETURN tls_process_end_of_early_data(SSL *s, PACKET *pkt); /* Extension processing */ @@ -253,320 +183,327 @@ typedef enum ext_return_en { EXT_RETURN_NOT_SENT } EXT_RETURN; -__owur int tls_validate_all_contexts(SSL_CONNECTION *s, unsigned int thisctx, +__owur int tls_validate_all_contexts(SSL *s, unsigned int thisctx, RAW_EXTENSION *exts); -__owur int extension_is_relevant(SSL_CONNECTION *s, unsigned int extctx, +__owur int extension_is_relevant(SSL *s, unsigned int extctx, unsigned int thisctx); -__owur int tls_collect_extensions(SSL_CONNECTION *s, PACKET *packet, - unsigned int context, +__owur int tls_collect_extensions(SSL *s, PACKET *packet, unsigned int context, RAW_EXTENSION **res, size_t *len, int init); -__owur int tls_parse_extension(SSL_CONNECTION *s, TLSEXT_INDEX idx, int context, +__owur int tls_parse_extension(SSL *s, TLSEXT_INDEX idx, int context, RAW_EXTENSION *exts, X509 *x, size_t chainidx); -__owur int tls_parse_all_extensions(SSL_CONNECTION *s, int context, - RAW_EXTENSION *exts, +__owur int tls_parse_all_extensions(SSL *s, int context, RAW_EXTENSION *exts, X509 *x, size_t chainidx, int fin); -__owur int should_add_extension(SSL_CONNECTION *s, unsigned int extctx, +__owur int should_add_extension(SSL *s, unsigned int extctx, unsigned int thisctx, int max_version); -__owur int tls_construct_extensions(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +__owur int tls_construct_extensions(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -__owur int tls_psk_do_binder(SSL_CONNECTION *s, const EVP_MD *md, +__owur int tls_psk_do_binder(SSL *s, const EVP_MD *md, const unsigned char *msgstart, size_t binderoffset, const unsigned char *binderin, unsigned char *binderout, SSL_SESSION *sess, int sign, int external); /* Server Extension processing */ -int tls_parse_ctos_renegotiate(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_renegotiate(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_ctos_server_name(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_server_name(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_ctos_maxfragmentlen(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_maxfragmentlen(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_SRP -int tls_parse_ctos_srp(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); +int tls_parse_ctos_srp(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #endif -int tls_parse_ctos_early_data(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_early_data(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_ctos_ec_pt_formats(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_ec_pt_formats(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_ctos_supported_groups(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_supported_groups(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidxl); -int tls_parse_ctos_session_ticket(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_session_ticket(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_ctos_sig_algs_cert(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_sig_algs_cert(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_ctos_sig_algs(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx); +int tls_parse_ctos_sig_algs(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #ifndef OPENSSL_NO_OCSP -int tls_parse_ctos_status_request(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_status_request(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #endif #ifndef OPENSSL_NO_NEXTPROTONEG -int tls_parse_ctos_npn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); +int tls_parse_ctos_npn(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #endif -int tls_parse_ctos_alpn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); +int tls_parse_ctos_alpn(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #ifndef OPENSSL_NO_SRTP -int tls_parse_ctos_use_srtp(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx); +int tls_parse_ctos_use_srtp(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#endif +int tls_parse_ctos_etm(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_key_share(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_cookie(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +int tls_parse_ctos_delegated_credential(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); #endif -int tls_parse_ctos_etm(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_ctos_key_share(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx); -int tls_parse_ctos_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_ctos_ems(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_ctos_psk_kex_modes(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_ems(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_psk_kex_modes(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_ctos_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_ctos_post_handshake_auth(SSL_CONNECTION *, PACKET *pkt, - unsigned int context, +int tls_parse_ctos_psk(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_post_handshake_auth(SSL *, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); +#ifndef OPENSSL_NO_QUIC +int tls_parse_ctos_quic_transport_params_draft(SSL *s, PACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); + +int tls_parse_ctos_quic_transport_params(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif -EXT_RETURN tls_construct_stoc_renegotiate(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_renegotiate(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_server_name(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_server_name(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_early_data(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_early_data(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_maxfragmentlen(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_maxfragmentlen(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_ec_pt_formats(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_ec_pt_formats(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_supported_groups(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_session_ticket(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_session_ticket(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_OCSP -EXT_RETURN tls_construct_stoc_status_request(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_status_request(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #endif #ifndef OPENSSL_NO_NEXTPROTONEG -EXT_RETURN tls_construct_stoc_next_proto_neg(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_next_proto_neg(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #endif -EXT_RETURN tls_construct_stoc_alpn(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_stoc_alpn(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_SRTP -EXT_RETURN tls_construct_stoc_use_srtp(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); +EXT_RETURN tls_construct_stoc_use_srtp(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); #endif -EXT_RETURN tls_construct_stoc_etm(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_stoc_etm(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_ems(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +EXT_RETURN tls_construct_stoc_delegated_credential(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif +EXT_RETURN tls_construct_stoc_ems(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_supported_versions(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_supported_versions(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_key_share(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_stoc_key_share(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_cookie(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_stoc_cookie(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -/* - * Not in public headers as this is not an official extension. Only used when - * SSL_OP_CRYPTOPRO_TLSEXT_BUG is set. - */ -#define TLSEXT_TYPE_cryptopro_bug 0xfde8 -EXT_RETURN tls_construct_stoc_cryptopro_bug(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, X509 *x, - size_t chainidx); -EXT_RETURN tls_construct_stoc_psk(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_stoc_psk(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); +#ifndef OPENSSL_NO_QUIC +EXT_RETURN tls_construct_stoc_quic_transport_params_draft(SSL *s, WPACKET *pkt, + unsigned int context, + X509 *x, + size_t chainidx); + +EXT_RETURN tls_construct_stoc_quic_transport_params(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +#endif /* Client Extension processing */ -EXT_RETURN tls_construct_ctos_renegotiate(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_server_name(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_maxfragmentlen(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_renegotiate(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_ctos_server_name(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_ctos_maxfragmentlen(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_SRP -EXT_RETURN tls_construct_ctos_srp(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, X509 *x, - size_t chainidx); +EXT_RETURN tls_construct_ctos_srp(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #endif -EXT_RETURN tls_construct_ctos_ec_pt_formats(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_ec_pt_formats(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_supported_groups(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_supported_groups(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_early_data(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_early_data(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_session_ticket(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_session_ticket(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_sig_algs(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_sig_algs(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_OCSP -EXT_RETURN tls_construct_ctos_status_request(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_status_request(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #endif #ifndef OPENSSL_NO_NEXTPROTONEG -EXT_RETURN tls_construct_ctos_npn(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_npn(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #endif -EXT_RETURN tls_construct_ctos_alpn(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_alpn(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_SRTP -EXT_RETURN tls_construct_ctos_use_srtp(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_use_srtp(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #endif -EXT_RETURN tls_construct_ctos_etm(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_etm(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_CT -EXT_RETURN tls_construct_ctos_sct(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_sct(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #endif -EXT_RETURN tls_construct_ctos_ems(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +EXT_RETURN tls_construct_ctos_delegated_credential(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif +EXT_RETURN tls_construct_ctos_ems(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_supported_versions(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_supported_versions(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_key_share(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_key_share(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_psk_kex_modes(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_psk_kex_modes(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_cookie(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_cookie(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_padding(SSL_CONNECTION *s, WPACKET *pkt, +EXT_RETURN tls_construct_ctos_padding(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_psk(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_psk(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_post_handshake_auth(SSL_CONNECTION *s, WPACKET *pkt, - unsigned int context, +EXT_RETURN tls_construct_ctos_post_handshake_auth(SSL *s, WPACKET *pkt, unsigned int context, X509 *x, size_t chainidx); +#ifndef OPENSSL_NO_QUIC +EXT_RETURN tls_construct_ctos_quic_transport_params_draft(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); + +EXT_RETURN tls_construct_ctos_quic_transport_params(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +#endif -int tls_parse_stoc_renegotiate(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_renegotiate(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_stoc_server_name(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_server_name(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_stoc_early_data(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_early_data(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_stoc_maxfragmentlen(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_maxfragmentlen(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_stoc_ec_pt_formats(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_ec_pt_formats(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_stoc_session_ticket(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_session_ticket(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #ifndef OPENSSL_NO_OCSP -int tls_parse_stoc_status_request(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_status_request(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); #endif #ifndef OPENSSL_NO_CT -int tls_parse_stoc_sct(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); +int tls_parse_stoc_sct(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #endif #ifndef OPENSSL_NO_NEXTPROTONEG -int tls_parse_stoc_npn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); +int tls_parse_stoc_npn(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #endif -int tls_parse_stoc_alpn(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); +int tls_parse_stoc_alpn(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #ifndef OPENSSL_NO_SRTP -int tls_parse_stoc_use_srtp(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx); +int tls_parse_stoc_use_srtp(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#endif +int tls_parse_stoc_etm(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +int tls_parse_stoc_delegated_credential(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); #endif -int tls_parse_stoc_etm(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_stoc_ems(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_stoc_supported_versions(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, +int tls_parse_stoc_ems(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_stoc_supported_versions(SSL *s, PACKET *pkt, unsigned int context, X509 *x, size_t chainidx); -int tls_parse_stoc_key_share(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, X509 *x, size_t chainidx); -int tls_parse_stoc_cookie(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_stoc_psk(SSL_CONNECTION *s, PACKET *pkt, unsigned int context, - X509 *x, size_t chainidx); - -int tls_handle_alpn(SSL_CONNECTION *s); - -int tls13_save_handshake_digest_for_pha(SSL_CONNECTION *s); -int tls13_restore_handshake_digest_for_pha(SSL_CONNECTION *s); - -__owur EVP_PKEY* tls_get_peer_pkey(const SSL_CONNECTION *sc); -/* RFC7250 */ -EXT_RETURN tls_construct_ctos_client_cert_type(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_client_cert_type(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_ctos_client_cert_type(SSL_CONNECTION *sc, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_stoc_client_cert_type(SSL_CONNECTION *sc, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -EXT_RETURN tls_construct_ctos_server_cert_type(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -EXT_RETURN tls_construct_stoc_server_cert_type(SSL_CONNECTION *sc, WPACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_ctos_server_cert_type(SSL_CONNECTION *sc, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); -int tls_parse_stoc_server_cert_type(SSL_CONNECTION *s, PACKET *pkt, - unsigned int context, - X509 *x, size_t chainidx); +int tls_parse_stoc_key_share(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_stoc_cookie(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_stoc_psk(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#ifndef OPENSSL_NO_QUIC +int tls_parse_stoc_quic_transport_params_draft(SSL *s, PACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); + +int tls_parse_stoc_quic_transport_params(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif + +int tls_handle_alpn(SSL *s); + +int tls13_save_handshake_digest_for_pha(SSL *s); +int tls13_restore_handshake_digest_for_pha(SSL *s); + +#ifndef OPENSSL_NO_CERT_COMPRESSION +__owur EXT_RETURN tls_construct_compress_cert(SSL *s, WPACKET *pkt, + unsigned int context, + X509 *x, size_t chainidx); +__owur int tls_parse_compress_cert(SSL *s, PACKET *pkt, + unsigned int context, + X509 *x, size_t chainidx); +__owur int tls_construct_compressed_certificate(SSL *s, WPACKET *pkt, + confunc_f confunc); +__owur int tls_construct_client_compressed_certificate(SSL *s, WPACKET *pkt); +__owur int tls_construct_server_compressed_certificate(SSL *s, WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_compressed_certificate(SSL *s, + PACKET *pkt, + profunc_f profunc); +__owur MSG_PROCESS_RETURN tls_process_client_compressed_certificate(SSL *s, + PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_server_compressed_certificate(SSL *s, + PACKET *pkt); +#endif + +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +int tls_parse_dc_from_extension(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_process_dc_request(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_delegated_credential_raw(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_delegated_credential_request(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif + diff --git a/openssl/src/ssl/statem/statem_quic.c b/openssl/src/ssl/statem/statem_quic.c new file mode 100644 index 000000000..b54345251 --- /dev/null +++ b/openssl/src/ssl/statem/statem_quic.c @@ -0,0 +1,117 @@ +/* + * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../ssl_local.h" +#include "statem_local.h" +#include "internal/cryptlib.h" + +#ifdef OPENSSL_NO_QUIC +NON_EMPTY_TRANSLATION_UNIT +#else + +int quic_get_message(SSL *s, int *mt, size_t *len) +{ + size_t l; + QUIC_DATA *qd = s->quic_input_data_head; + uint8_t *p; + + if (qd == NULL) { + s->rwstate = SSL_READING; + *mt = *len = 0; + return 0; + } + + if (!ossl_assert(qd->length >= SSL3_HM_HEADER_LENGTH)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_LENGTH); + *mt = *len = 0; + return 0; + } + + /* This is where we check for the proper level, not when data is given */ + if (qd->level != s->quic_read_level) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_WRONG_ENCRYPTION_LEVEL_RECEIVED); + *mt = *len = 0; + return 0; + } + + if (!BUF_MEM_grow_clean(s->init_buf, (int)qd->length)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_BUF_LIB); + *mt = *len = 0; + return 0; + } + + /* Copy buffered data */ + memcpy(s->init_buf->data, s->quic_buf->data + qd->start, qd->length); + s->init_buf->length = qd->length; + s->quic_input_data_head = qd->next; + if (s->quic_input_data_head == NULL) + s->quic_input_data_tail = NULL; + OPENSSL_free(qd); + + s->s3.tmp.message_type = *mt = *(s->init_buf->data); + p = (uint8_t*)s->init_buf->data + 1; + n2l3(p, l); + s->init_num = s->s3.tmp.message_size = *len = l; + s->init_msg = s->init_buf->data + SSL3_HM_HEADER_LENGTH; + + /* No CCS in QUIC/TLSv1.3? */ + if (*mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_CCS_RECEIVED_EARLY); + *len = 0; + return 0; + } + /* No KeyUpdate in QUIC */ + if (*mt == SSL3_MT_KEY_UPDATE) { + SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_MESSAGE); + *len = 0; + return 0; + } + + /* + * If receiving Finished, record MAC of prior handshake messages for + * Finished verification. + */ + if (*mt == SSL3_MT_FINISHED && !ssl3_take_mac(s)) { + /* SSLfatal() already called */ + *len = 0; + return 0; + } + + /* + * We defer feeding in the HRR until later. We'll do it as part of + * processing the message + * The TLsv1.3 handshake transcript stops at the ClientFinished + * message. + */ +#define SERVER_HELLO_RANDOM_OFFSET (SSL3_HM_HEADER_LENGTH + 2) + /* KeyUpdate and NewSessionTicket do not need to be added */ + if (s->s3.tmp.message_type != SSL3_MT_NEWSESSION_TICKET + && s->s3.tmp.message_type != SSL3_MT_KEY_UPDATE) { + if (s->s3.tmp.message_type != SSL3_MT_SERVER_HELLO + || s->init_num < SERVER_HELLO_RANDOM_OFFSET + SSL3_RANDOM_SIZE + || memcmp(hrrrandom, + s->init_buf->data + SERVER_HELLO_RANDOM_OFFSET, + SSL3_RANDOM_SIZE) != 0) { + if (!ssl3_finish_mac(s, (unsigned char *)s->init_buf->data, + s->init_num + SSL3_HM_HEADER_LENGTH)) { + /* SSLfatal() already called */ + *len = 0; + return 0; + } + } + } + if (s->msg_callback) + s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, s->init_buf->data, + (size_t)s->init_num + SSL3_HM_HEADER_LENGTH, s, + s->msg_callback_arg); + + return 1; +} + +#endif diff --git a/openssl/src/ssl/statem/statem_srvr.c b/openssl/src/ssl/statem/statem_srvr.c index 5ff479a2e..4422bffa9 100644 --- a/openssl/src/ssl/statem/statem_srvr.c +++ b/openssl/src/ssl/statem/statem_srvr.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved * Copyright 2005 Nokia. All rights reserved. * @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -26,31 +27,11 @@ #include #include #include -#include +#include #define TICKET_NONCE_SIZE 8 -typedef struct { - ASN1_TYPE *kxBlob; - ASN1_TYPE *opaqueBlob; -} GOST_KX_MESSAGE; - -DECLARE_ASN1_FUNCTIONS(GOST_KX_MESSAGE) - -ASN1_SEQUENCE(GOST_KX_MESSAGE) = { - ASN1_SIMPLE(GOST_KX_MESSAGE, kxBlob, ASN1_ANY), - ASN1_OPT(GOST_KX_MESSAGE, opaqueBlob, ASN1_ANY), -} ASN1_SEQUENCE_END(GOST_KX_MESSAGE) - -IMPLEMENT_ASN1_FUNCTIONS(GOST_KX_MESSAGE) - -static CON_FUNC_RETURN tls_construct_encrypted_extensions(SSL_CONNECTION *s, - WPACKET *pkt); - -static ossl_inline int received_client_cert(const SSL_CONNECTION *sc) -{ - return sc->session->peer_rpk != NULL || sc->session->peer != NULL; -} +static int tls_construct_encrypted_extensions(SSL *s, WPACKET *pkt); /* * ossl_statem_server13_read_transition() encapsulates the logic for the allowed @@ -61,7 +42,7 @@ static ossl_inline int received_client_cert(const SSL_CONNECTION *sc) * Return values are 1 for success (transition allowed) and 0 on error * (transition not allowed) */ -static int ossl_statem_server13_read_transition(SSL_CONNECTION *s, int mt) +static int ossl_statem_server13_read_transition(SSL *s, int mt) { OSSL_STATEM *st = &s->statem; @@ -81,7 +62,8 @@ static int ossl_statem_server13_read_transition(SSL_CONNECTION *s, int mt) return 1; } break; - } else if (s->ext.early_data == SSL_EARLY_DATA_ACCEPTED) { + } else if (s->ext.early_data == SSL_EARLY_DATA_ACCEPTED + && !SSL_IS_QUIC(s)) { if (mt == SSL3_MT_END_OF_EARLY_DATA) { st->hand_state = TLS_ST_SR_END_OF_EARLY_DATA; return 1; @@ -93,17 +75,14 @@ static int ossl_statem_server13_read_transition(SSL_CONNECTION *s, int mt) case TLS_ST_SR_END_OF_EARLY_DATA: case TLS_ST_SW_FINISHED: if (s->s3.tmp.cert_request) { - if (mt == SSL3_MT_CERTIFICATE) { + if (mt == SSL3_MT_CERTIFICATE +#ifndef OPENSSL_NO_CERT_COMPRESSION + || mt == SSL3_MT_COMPRESSED_CERTIFICATE +#endif + ) { st->hand_state = TLS_ST_SR_CERT; return 1; } -#ifndef OPENSSL_NO_COMP_ALG - if (mt == SSL3_MT_COMPRESSED_CERTIFICATE - && s->ext.compress_certificate_sent) { - st->hand_state = TLS_ST_SR_COMP_CERT; - return 1; - } -#endif } else { if (mt == SSL3_MT_FINISHED) { st->hand_state = TLS_ST_SR_FINISHED; @@ -112,9 +91,8 @@ static int ossl_statem_server13_read_transition(SSL_CONNECTION *s, int mt) } break; - case TLS_ST_SR_COMP_CERT: case TLS_ST_SR_CERT: - if (!received_client_cert(s)) { + if (s->session->peer == NULL) { if (mt == SSL3_MT_FINISHED) { st->hand_state = TLS_ST_SR_FINISHED; return 1; @@ -142,21 +120,16 @@ static int ossl_statem_server13_read_transition(SSL_CONNECTION *s, int mt) if (s->early_data_state == SSL_EARLY_DATA_READING) break; - if (s->post_handshake_auth == SSL_PHA_REQUESTED) { - if (mt == SSL3_MT_CERTIFICATE) { - st->hand_state = TLS_ST_SR_CERT; - return 1; - } -#ifndef OPENSSL_NO_COMP_ALG - if (mt == SSL3_MT_COMPRESSED_CERTIFICATE - && s->ext.compress_certificate_sent) { - st->hand_state = TLS_ST_SR_COMP_CERT; - return 1; - } + if ((mt == SSL3_MT_CERTIFICATE +#ifndef OPENSSL_NO_CERT_COMPRESSION + || mt == SSL3_MT_COMPRESSED_CERTIFICATE #endif + ) && s->post_handshake_auth == SSL_PHA_REQUESTED) { + st->hand_state = TLS_ST_SR_CERT; + return 1; } - if (mt == SSL3_MT_KEY_UPDATE && !SSL_IS_QUIC_HANDSHAKE(s)) { + if (mt == SSL3_MT_KEY_UPDATE) { st->hand_state = TLS_ST_SR_KEY_UPDATE; return 1; } @@ -176,11 +149,11 @@ static int ossl_statem_server13_read_transition(SSL_CONNECTION *s, int mt) * Return values are 1 for success (transition allowed) and 0 on error * (transition not allowed) */ -int ossl_statem_server_read_transition(SSL_CONNECTION *s, int mt) +int ossl_statem_server_read_transition(SSL *s, int mt) { OSSL_STATEM *st = &s->statem; - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { if (!ossl_statem_server13_read_transition(s, mt)) goto err; return 1; @@ -232,7 +205,11 @@ int ossl_statem_server_read_transition(SSL_CONNECTION *s, int mt) return 1; } } else if (s->s3.tmp.cert_request) { - if (mt == SSL3_MT_CERTIFICATE) { + if (mt == SSL3_MT_CERTIFICATE +#ifndef OPENSSL_NO_CERT_COMPRESSION + || mt == SSL3_MT_COMPRESSED_CERTIFICATE +#endif + ) { st->hand_state = TLS_ST_SR_CERT; return 1; } @@ -255,13 +232,12 @@ int ossl_statem_server_read_transition(SSL_CONNECTION *s, int mt) * the case of static DH). In that case |st->no_cert_verify| should be * set. */ - if (!received_client_cert(s) || st->no_cert_verify) { + if (s->session->peer == NULL || st->no_cert_verify) { if (mt == SSL3_MT_CHANGE_CIPHER_SPEC) { /* * For the ECDH ciphersuites when the client sends its ECDH * pub key in a certificate, the CertificateVerify message is - * not sent. Also for GOST ciphersuites when the client uses - * its key from the certificate for key exchange. + * not sent. */ st->hand_state = TLS_ST_SR_CHANGE; return 1; @@ -318,7 +294,7 @@ int ossl_statem_server_read_transition(SSL_CONNECTION *s, int mt) err: /* No valid transition found */ - if (SSL_CONNECTION_IS_DTLS(s) && mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + if (SSL_IS_DTLS(s) && mt == SSL3_MT_CHANGE_CIPHER_SPEC) { BIO *rbio; /* @@ -327,7 +303,7 @@ int ossl_statem_server_read_transition(SSL_CONNECTION *s, int mt) */ s->init_num = 0; s->rwstate = SSL_READING; - rbio = SSL_get_rbio(SSL_CONNECTION_GET_SSL(s)); + rbio = SSL_get_rbio(s); BIO_clear_retry_flags(rbio); BIO_set_retry_read(rbio); return 0; @@ -343,7 +319,7 @@ int ossl_statem_server_read_transition(SSL_CONNECTION *s, int mt) * 1: Yes * 0: No */ -static int send_server_key_exchange(SSL_CONNECTION *s) +static int send_server_key_exchange(SSL *s) { unsigned long alg_k = s->s3.tmp.new_cipher->algorithm_mkey; @@ -378,27 +354,6 @@ static int send_server_key_exchange(SSL_CONNECTION *s) return 0; } -/* - * Used to determine if we should send a CompressedCertificate message - * - * Returns the algorithm to use, TLSEXT_comp_cert_none means no compression - */ -static int get_compressed_certificate_alg(SSL_CONNECTION *sc) -{ -#ifndef OPENSSL_NO_COMP_ALG - int *alg = sc->ext.compress_certificate_from_peer; - - if (sc->s3.tmp.cert == NULL) - return TLSEXT_comp_cert_none; - - for (; *alg != TLSEXT_comp_cert_none; alg++) { - if (sc->s3.tmp.cert->comp_cert[*alg] != NULL) - return *alg; - } -#endif - return TLSEXT_comp_cert_none; -} - /* * Should we send a CertificateRequest message? * @@ -406,7 +361,7 @@ static int get_compressed_certificate_alg(SSL_CONNECTION *sc) * 1: Yes * 0: No */ -int send_certificate_request(SSL_CONNECTION *s) +int send_certificate_request(SSL *s) { if ( /* don't request cert unless asked for it: */ @@ -415,8 +370,7 @@ int send_certificate_request(SSL_CONNECTION *s) * don't request if post-handshake-only unless doing * post-handshake in TLSv1.3: */ - && (!SSL_CONNECTION_IS_TLS13(s) - || !(s->verify_mode & SSL_VERIFY_POST_HANDSHAKE) + && (!SSL_IS_TLS13(s) || !(s->verify_mode & SSL_VERIFY_POST_HANDSHAKE) || s->post_handshake_auth == SSL_PHA_REQUEST_PENDING) /* * if SSL_VERIFY_CLIENT_ONCE is set, don't request cert @@ -449,19 +403,12 @@ int send_certificate_request(SSL_CONNECTION *s) return 0; } -static int do_compressed_cert(SSL_CONNECTION *sc) -{ - /* If we negotiated RPK, we won't attempt to compress it */ - return sc->ext.server_cert_type == TLSEXT_cert_type_x509 - && get_compressed_certificate_alg(sc) != TLSEXT_comp_cert_none; -} - /* * ossl_statem_server13_write_transition() works out what handshake state to * move to next when a TLSv1.3 server is writing messages to be sent to the * client. */ -static WRITE_TRAN ossl_statem_server13_write_transition(SSL_CONNECTION *s) +static WRITE_TRAN ossl_statem_server13_write_transition(SSL *s) { OSSL_STATEM *st = &s->statem; @@ -518,8 +465,6 @@ static WRITE_TRAN ossl_statem_server13_write_transition(SSL_CONNECTION *s) st->hand_state = TLS_ST_SW_FINISHED; else if (send_certificate_request(s)) st->hand_state = TLS_ST_SW_CERT_REQ; - else if (do_compressed_cert(s)) - st->hand_state = TLS_ST_SW_COMP_CERT; else st->hand_state = TLS_ST_SW_CERT; @@ -529,14 +474,11 @@ static WRITE_TRAN ossl_statem_server13_write_transition(SSL_CONNECTION *s) if (s->post_handshake_auth == SSL_PHA_REQUEST_PENDING) { s->post_handshake_auth = SSL_PHA_REQUESTED; st->hand_state = TLS_ST_OK; - } else if (do_compressed_cert(s)) { - st->hand_state = TLS_ST_SW_COMP_CERT; } else { st->hand_state = TLS_ST_SW_CERT; } return WRITE_TRAN_CONTINUE; - case TLS_ST_SW_COMP_CERT: case TLS_ST_SW_CERT: st->hand_state = TLS_ST_SW_CERT_VRFY; return WRITE_TRAN_CONTINUE; @@ -547,14 +489,12 @@ static WRITE_TRAN ossl_statem_server13_write_transition(SSL_CONNECTION *s) case TLS_ST_SW_FINISHED: st->hand_state = TLS_ST_EARLY_DATA; - s->ts_msg_write = ossl_time_now(); return WRITE_TRAN_CONTINUE; case TLS_ST_EARLY_DATA: return WRITE_TRAN_FINISHED; case TLS_ST_SR_FINISHED: - s->ts_msg_read = ossl_time_now(); /* * Technically we have finished the handshake at this point, but we're * going to remain "in_init" for now and write out any session tickets @@ -600,7 +540,7 @@ static WRITE_TRAN ossl_statem_server13_write_transition(SSL_CONNECTION *s) * ossl_statem_server_write_transition() works out what handshake state to move * to next when the server is writing messages to be sent to the client. */ -WRITE_TRAN ossl_statem_server_write_transition(SSL_CONNECTION *s) +WRITE_TRAN ossl_statem_server_write_transition(SSL *s) { OSSL_STATEM *st = &s->statem; @@ -609,7 +549,7 @@ WRITE_TRAN ossl_statem_server_write_transition(SSL_CONNECTION *s) * to negotiate yet, so we don't take this branch until later */ - if (SSL_CONNECTION_IS_TLS13(s)) + if (SSL_IS_TLS13(s)) return ossl_statem_server13_write_transition(s); switch (st->hand_state) { @@ -641,8 +581,8 @@ WRITE_TRAN ossl_statem_server_write_transition(SSL_CONNECTION *s) return WRITE_TRAN_CONTINUE; case TLS_ST_SR_CLNT_HELLO: - if (SSL_CONNECTION_IS_DTLS(s) && !s->d1->cookie_verified - && (SSL_get_options(SSL_CONNECTION_GET_SSL(s)) & SSL_OP_COOKIE_EXCHANGE)) { + if (SSL_IS_DTLS(s) && !s->d1->cookie_verified + && (SSL_get_options(s) & SSL_OP_COOKIE_EXCHANGE)) { st->hand_state = DTLS_ST_SW_HELLO_VERIFY_REQUEST; } else if (s->renegotiate == 0 && !SSL_IS_FIRST_HANDSHAKE(s)) { /* We must have rejected the renegotiation */ @@ -704,11 +644,9 @@ WRITE_TRAN ossl_statem_server_write_transition(SSL_CONNECTION *s) return WRITE_TRAN_CONTINUE; case TLS_ST_SW_SRVR_DONE: - s->ts_msg_write = ossl_time_now(); return WRITE_TRAN_FINISHED; case TLS_ST_SR_FINISHED: - s->ts_msg_read = ossl_time_now(); if (s->hit) { st->hand_state = TLS_ST_OK; return WRITE_TRAN_CONTINUE; @@ -740,10 +678,9 @@ WRITE_TRAN ossl_statem_server_write_transition(SSL_CONNECTION *s) * Perform any pre work that needs to be done prior to sending a message from * the server to the client. */ -WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst) +WORK_STATE ossl_statem_server_pre_work(SSL *s, WORK_STATE wst) { OSSL_STATEM *st = &s->statem; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); switch (st->hand_state) { default: @@ -752,13 +689,13 @@ WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst) case TLS_ST_SW_HELLO_REQ: s->shutdown = 0; - if (SSL_CONNECTION_IS_DTLS(s)) + if (SSL_IS_DTLS(s)) dtls1_clear_sent_buffer(s); break; case DTLS_ST_SW_HELLO_VERIFY_REQUEST: s->shutdown = 0; - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { dtls1_clear_sent_buffer(s); /* We don't buffer this message so don't use the timer */ st->use_timer = 0; @@ -766,7 +703,7 @@ WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst) break; case TLS_ST_SW_SRVR_HELLO: - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* * Messages we write from now on should be buffered and * retransmitted if necessary, so we need to use the timer now @@ -777,7 +714,7 @@ WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst) case TLS_ST_SW_SRVR_DONE: #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(ssl))) { + if (SSL_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(s))) { /* Calls SSLfatal() as required */ return dtls_wait_for_dry(s); } @@ -785,7 +722,7 @@ WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst) return WORK_FINISHED_CONTINUE; case TLS_ST_SW_SESSION_TICKET: - if (SSL_CONNECTION_IS_TLS13(s) && s->sent_tickets == 0 + if (SSL_IS_TLS13(s) && s->sent_tickets == 0 && s->ext.extra_tickets_expected == 0) { /* * Actually this is the end of the handshake, but we're going @@ -796,7 +733,7 @@ WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst) */ return tls_finish_handshake(s, wst, 0, 0); } - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* * We're into the last flight. We don't retransmit the last flight * unless we need to, so we don't use the timer @@ -806,7 +743,7 @@ WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst) break; case TLS_ST_SW_CHANGE: - if (SSL_CONNECTION_IS_TLS13(s)) + if (SSL_IS_TLS13(s)) break; /* Writes to s->session are only safe for initial handshakes */ if (s->session->cipher == NULL) { @@ -815,11 +752,11 @@ WORK_STATE ossl_statem_server_pre_work(SSL_CONNECTION *s, WORK_STATE wst) SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return WORK_ERROR; } - if (!ssl->method->ssl3_enc->setup_key_block(s)) { + if (!s->method->ssl3_enc->setup_key_block(s)) { /* SSLfatal() already called */ return WORK_ERROR; } - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* * We're into the last flight. We don't retransmit the last flight * unless we need to, so we don't use the timer. This might have @@ -868,10 +805,9 @@ static ossl_inline int conn_is_closed(void) * Perform any work that needs to be done after sending a message from the * server to the client. */ -WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) +WORK_STATE ossl_statem_server_post_work(SSL *s, WORK_STATE wst) { OSSL_STATEM *st = &s->statem; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); s->init_num = 0; @@ -905,15 +841,14 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) break; case TLS_ST_SW_SRVR_HELLO: - if (SSL_CONNECTION_IS_TLS13(s) - && s->hello_retry_request == SSL_HRR_PENDING) { + if (SSL_IS_TLS13(s) && s->hello_retry_request == SSL_HRR_PENDING) { if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) == 0 && statem_flush(s) != 1) return WORK_MORE_A; break; } #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s) && s->hit) { + if (SSL_IS_DTLS(s) && s->hit) { unsigned char sctpauthkey[64]; char labelbuffer[sizeof(DTLS1_SCTP_AUTH_LABEL)]; size_t labellen; @@ -930,7 +865,7 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) if (s->mode & SSL_MODE_DTLS_SCTP_LABEL_LENGTH_BUG) labellen += 1; - if (SSL_export_keying_material(ssl, sctpauthkey, + if (SSL_export_keying_material(s, sctpauthkey, sizeof(sctpauthkey), labelbuffer, labellen, NULL, 0, 0) <= 0) { @@ -938,11 +873,11 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) return WORK_ERROR; } - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY, sizeof(sctpauthkey), sctpauthkey); } #endif - if (!SSL_CONNECTION_IS_TLS13(s) + if (!SSL_IS_TLS13(s) || ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0 && s->hello_retry_request != SSL_HRR_COMPLETE)) break; @@ -955,16 +890,16 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) break; } - if (SSL_CONNECTION_IS_TLS13(s)) { - if (!ssl->method->ssl3_enc->setup_key_block(s) - || !ssl->method->ssl3_enc->change_cipher_state(s, + if (SSL_IS_TLS13(s)) { + if (!s->method->ssl3_enc->setup_key_block(s) + || !s->method->ssl3_enc->change_cipher_state(s, SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_SERVER_WRITE)) { /* SSLfatal() already called */ return WORK_ERROR; } if (s->ext.early_data != SSL_EARLY_DATA_ACCEPTED - && !ssl->method->ssl3_enc->change_cipher_state(s, + && !s->method->ssl3_enc->change_cipher_state(s, SSL3_CC_HANDSHAKE |SSL3_CHANGE_CIPHER_SERVER_READ)) { /* SSLfatal() already called */ return WORK_ERROR; @@ -974,26 +909,29 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) * is an unencrypted alert, an encrypted alert, or an encrypted * handshake message. We temporarily tolerate unencrypted alerts. */ - if (s->rlayer.rrlmethod->set_plain_alerts != NULL) - s->rlayer.rrlmethod->set_plain_alerts(s->rlayer.rrl, 1); + s->statem.enc_read_state = ENC_READ_STATE_ALLOW_PLAIN_ALERTS; break; } #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s) && !s->hit) { + if (SSL_IS_DTLS(s) && !s->hit) { /* * Change to new shared key of SCTP-Auth, will be ignored if * no SCTP used. */ - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY, 0, NULL); } #endif - if (!ssl->method->ssl3_enc->change_cipher_state(s, - SSL3_CHANGE_CIPHER_SERVER_WRITE)) { + if (!s->method->ssl3_enc->change_cipher_state(s, + SSL3_CHANGE_CIPHER_SERVER_WRITE)) + { /* SSLfatal() already called */ return WORK_ERROR; } + + if (SSL_IS_DTLS(s)) + dtls1_reset_seq_numbers(s, SSL3_CC_WRITE); break; case TLS_ST_SW_SRVR_DONE: @@ -1005,25 +943,36 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) if (statem_flush(s) != 1) return WORK_MORE_A; #ifndef OPENSSL_NO_SCTP - if (SSL_CONNECTION_IS_DTLS(s) && s->hit) { + if (SSL_IS_DTLS(s) && s->hit) { /* * Change to new shared key of SCTP-Auth, will be ignored if * no SCTP used. */ - BIO_ctrl(SSL_get_wbio(ssl), BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY, 0, NULL); } #endif - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* TLS 1.3 gets the secret size from the handshake md */ size_t dummy; - if (!ssl->method->ssl3_enc->generate_master_secret(s, + if (!s->method->ssl3_enc->generate_master_secret(s, s->master_secret, s->handshake_secret, 0, &dummy) - || !ssl->method->ssl3_enc->change_cipher_state(s, + || !s->method->ssl3_enc->change_cipher_state(s, SSL3_CC_APPLICATION | SSL3_CHANGE_CIPHER_SERVER_WRITE)) /* SSLfatal() already called */ return WORK_ERROR; + +#ifndef OPENSSL_NO_QUIC + if (SSL_IS_QUIC(s) + && s->ext.early_data == SSL_EARLY_DATA_ACCEPTED) { + s->early_data_state = SSL_EARLY_DATA_FINISHED_READING; + if (!s->method->ssl3_enc->change_cipher_state( + s, SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_SERVER_READ)) + /* SSLfatal() already called */ + return WORK_ERROR; + } +#endif } break; @@ -1031,18 +980,6 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) if (s->post_handshake_auth == SSL_PHA_REQUEST_PENDING) { if (statem_flush(s) != 1) return WORK_MORE_A; - } else { - if (!SSL_CONNECTION_IS_TLS13(s) - || (s->options & SSL_OP_NO_TX_CERTIFICATE_COMPRESSION) != 0) - s->ext.compress_certificate_from_peer[0] = TLSEXT_comp_cert_none; - } - break; - - case TLS_ST_SW_ENCRYPTED_EXTENSIONS: - if (!s->hit && !send_certificate_request(s)) { - if (!SSL_CONNECTION_IS_TLS13(s) - || (s->options & SSL_OP_NO_TX_CERTIFICATE_COMPRESSION) != 0) - s->ext.compress_certificate_from_peer[0] = TLSEXT_comp_cert_none; } break; @@ -1057,8 +994,8 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) case TLS_ST_SW_SESSION_TICKET: clear_sys_error(); - if (SSL_CONNECTION_IS_TLS13(s) && statem_flush(s) != 1) { - if (SSL_get_error(ssl, 0) == SSL_ERROR_SYSCALL + if (SSL_IS_TLS13(s) && statem_flush(s) != 1) { + if (SSL_get_error(s, 0) == SSL_ERROR_SYSCALL && conn_is_closed()) { /* * We ignore connection closed errors in TLSv1.3 when sending a @@ -1087,7 +1024,7 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst) * 1: Success * 0: Error */ -int ossl_statem_server_construct_message(SSL_CONNECTION *s, +int ossl_statem_server_construct_message(SSL *s, WPACKET *pkt, confunc_f *confunc, int *mt) { OSSL_STATEM *st = &s->statem; @@ -1099,7 +1036,7 @@ int ossl_statem_server_construct_message(SSL_CONNECTION *s, return 0; case TLS_ST_SW_CHANGE: - if (SSL_CONNECTION_IS_DTLS(s)) + if (SSL_IS_DTLS(s)) *confunc = dtls_construct_change_cipher_spec; else *confunc = tls_construct_change_cipher_spec; @@ -1123,16 +1060,18 @@ int ossl_statem_server_construct_message(SSL_CONNECTION *s, break; case TLS_ST_SW_CERT: +#ifndef OPENSSL_NO_CERT_COMPRESSION + if (s->cert_comp_compress_id) { + *confunc = tls_construct_server_compressed_certificate; + *mt = SSL3_MT_COMPRESSED_CERTIFICATE; + } else { +#endif *confunc = tls_construct_server_certificate; *mt = SSL3_MT_CERTIFICATE; - break; - -#ifndef OPENSSL_NO_COMP_ALG - case TLS_ST_SW_COMP_CERT: - *confunc = tls_construct_server_compressed_certificate; - *mt = SSL3_MT_COMPRESSED_CERTIFICATE; - break; +#ifndef OPENSSL_NO_CERT_COMPRESSION + } #endif + break; case TLS_ST_SW_CERT_VRFY: *confunc = tls_construct_cert_verify; @@ -1213,7 +1152,7 @@ int ossl_statem_server_construct_message(SSL_CONNECTION *s, * Returns the maximum allowed length for the current message that we are * reading. Excludes the message header. */ -size_t ossl_statem_server_max_message_size(SSL_CONNECTION *s) +size_t ossl_statem_server_max_message_size(SSL *s) { OSSL_STATEM *st = &s->statem; @@ -1228,7 +1167,6 @@ size_t ossl_statem_server_max_message_size(SSL_CONNECTION *s) case TLS_ST_SR_END_OF_EARLY_DATA: return END_OF_EARLY_DATA_MAX_LENGTH; - case TLS_ST_SR_COMP_CERT: case TLS_ST_SR_CERT: return s->max_cert_list; @@ -1236,7 +1174,7 @@ size_t ossl_statem_server_max_message_size(SSL_CONNECTION *s) return CLIENT_KEY_EXCH_MAX_LENGTH; case TLS_ST_SR_CERT_VRFY: - return CERTIFICATE_VERIFY_MAX_LENGTH; + return SSL3_RT_MAX_PLAIN_LENGTH; #ifndef OPENSSL_NO_NEXTPROTONEG case TLS_ST_SR_NEXT_PROTO: @@ -1257,8 +1195,7 @@ size_t ossl_statem_server_max_message_size(SSL_CONNECTION *s) /* * Process a message that the server has received from the client. */ -MSG_PROCESS_RETURN ossl_statem_server_process_message(SSL_CONNECTION *s, - PACKET *pkt) +MSG_PROCESS_RETURN ossl_statem_server_process_message(SSL *s, PACKET *pkt) { OSSL_STATEM *st = &s->statem; @@ -1275,12 +1212,12 @@ MSG_PROCESS_RETURN ossl_statem_server_process_message(SSL_CONNECTION *s, return tls_process_end_of_early_data(s, pkt); case TLS_ST_SR_CERT: - return tls_process_client_certificate(s, pkt); - -#ifndef OPENSSL_NO_COMP_ALG - case TLS_ST_SR_COMP_CERT: - return tls_process_client_compressed_certificate(s, pkt); +#ifndef OPENSSL_NO_CERT_COMPRESSION + if (s->s3.tmp.message_type == SSL3_MT_COMPRESSED_CERTIFICATE) + return tls_process_client_compressed_certificate(s, pkt); + else #endif + return tls_process_client_certificate(s, pkt); case TLS_ST_SR_KEY_EXCH: return tls_process_client_key_exchange(s, pkt); @@ -1309,8 +1246,7 @@ MSG_PROCESS_RETURN ossl_statem_server_process_message(SSL_CONNECTION *s, * Perform any further processing required following the receipt of a message * from the client */ -WORK_STATE ossl_statem_server_post_process_message(SSL_CONNECTION *s, - WORK_STATE wst) +WORK_STATE ossl_statem_server_post_process_message(SSL *s, WORK_STATE wst) { OSSL_STATEM *st = &s->statem; @@ -1330,7 +1266,7 @@ WORK_STATE ossl_statem_server_post_process_message(SSL_CONNECTION *s, #ifndef OPENSSL_NO_SRP /* Returns 1 on success, 0 for retryable error, -1 for fatal error */ -static int ssl_check_srp_ext_ClientHello(SSL_CONNECTION *s) +static int ssl_check_srp_ext_ClientHello(SSL *s) { int ret; int al = SSL_AD_UNRECOGNIZED_NAME; @@ -1373,28 +1309,25 @@ int dtls_raw_hello_verify_request(WPACKET *pkt, unsigned char *cookie, return 1; } -CON_FUNC_RETURN dtls_construct_hello_verify_request(SSL_CONNECTION *s, - WPACKET *pkt) +int dtls_construct_hello_verify_request(SSL *s, WPACKET *pkt) { unsigned int cookie_leni; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - - if (sctx->app_gen_cookie_cb == NULL - || sctx->app_gen_cookie_cb(SSL_CONNECTION_GET_SSL(s), s->d1->cookie, - &cookie_leni) == 0 - || cookie_leni > DTLS1_COOKIE_LENGTH) { + if (s->ctx->app_gen_cookie_cb == NULL || + s->ctx->app_gen_cookie_cb(s, s->d1->cookie, + &cookie_leni) == 0 || + cookie_leni > DTLS1_COOKIE_LENGTH) { SSLfatal(s, SSL_AD_NO_ALERT, SSL_R_COOKIE_GEN_CALLBACK_FAILURE); - return CON_FUNC_ERROR; + return 0; } s->d1->cookie_len = cookie_leni; if (!dtls_raw_hello_verify_request(pkt, s->d1->cookie, - s->d1->cookie_len)) { + s->d1->cookie_len)) { SSLfatal(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } - return CON_FUNC_SUCCESS; + return 1; } /*- @@ -1411,8 +1344,7 @@ CON_FUNC_RETURN dtls_construct_hello_verify_request(SSL_CONNECTION *s, * Sadly we cannot differentiate 10.6, 10.7 and 10.8.4 (which work), from * 10.8..10.8.3 (which don't work). */ -static void ssl_check_for_safari(SSL_CONNECTION *s, - const CLIENTHELLO_MSG *hello) +static void ssl_check_for_safari(SSL *s, const CLIENTHELLO_MSG *hello) { static const unsigned char kSafariExtensionsBlock[] = { 0x00, 0x0a, /* elliptic_curves extension */ @@ -1453,9 +1385,8 @@ static void ssl_check_for_safari(SSL_CONNECTION *s, if (type != TLSEXT_TYPE_server_name) return; - ext_len = TLS1_get_client_version( - SSL_CONNECTION_GET_SSL(s)) >= TLS1_2_VERSION ? - sizeof(kSafariExtensionsBlock) : kSafariCommonExtensionsLength; + ext_len = TLS1_get_client_version(s) >= TLS1_2_VERSION ? + sizeof(kSafariExtensionsBlock) : kSafariCommonExtensionsLength; s->s3.is_probably_safari = PACKET_equal(&tmppkt, kSafariExtensionsBlock, ext_len); @@ -1465,7 +1396,7 @@ static void ssl_check_for_safari(SSL_CONNECTION *s, ((options & SSL_OP_NO_RENEGOTIATION) == 0 \ && (options & SSL_OP_ALLOW_CLIENT_RENEGOTIATION) != 0) -MSG_PROCESS_RETURN tls_process_client_hello(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_client_hello(SSL *s, PACKET *pkt) { /* |cookie| will only be initialized for DTLS. */ PACKET session_id, compression, extensions, cookie; @@ -1474,7 +1405,7 @@ MSG_PROCESS_RETURN tls_process_client_hello(SSL_CONNECTION *s, PACKET *pkt) /* Check if this is actually an unexpected renegotiation ClientHello */ if (s->renegotiate == 0 && !SSL_IS_FIRST_HANDSHAKE(s)) { - if (!ossl_assert(!SSL_CONNECTION_IS_TLS13(s))) { + if (!ossl_assert(!SSL_IS_TLS13(s))) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } @@ -1604,7 +1535,7 @@ MSG_PROCESS_RETURN tls_process_client_hello(SSL_CONNECTION *s, PACKET *pkt) goto err; } - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { if (!PACKET_get_length_prefixed_1(pkt, &cookie)) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); goto err; @@ -1620,7 +1551,7 @@ MSG_PROCESS_RETURN tls_process_client_hello(SSL_CONNECTION *s, PACKET *pkt) * just return since we do not want to allocate any memory yet. * So check cookie length... */ - if (SSL_get_options(SSL_CONNECTION_GET_SSL(s)) & SSL_OP_COOKIE_EXCHANGE) { + if (SSL_get_options(s) & SSL_OP_COOKIE_EXCHANGE) { if (clienthello->dtls_cookie_len == 0) { OPENSSL_free(clienthello); return MSG_PROCESS_FINISHED_READING; @@ -1650,6 +1581,30 @@ MSG_PROCESS_RETURN tls_process_client_hello(SSL_CONNECTION *s, PACKET *pkt) } } +#ifndef OPENSSL_NO_STATUS + if (s->status_param.ssl_status_enable) { + /* record client session_id */ + s->status_param.type = SSL_CLIENT_SESSION_ID; + if (s->status_callback(clienthello->session_id, + clienthello->session_id_len, + &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + + /* record client ciphers */ + int ciphersuite_size = 2; + s->status_param.type = clienthello->isv2 ? SSL_CLIENT_V2_CIPHER : SSL_CLIENT_CIPHER; + s->status_param.parg = &ciphersuite_size; + if (s->status_callback((unsigned char *)clienthello->ciphersuites.curr, + clienthello->ciphersuites.remaining, + &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + } +#endif + if (!PACKET_copy_all(&compression, clienthello->compressions, MAX_COMPRESSIONS_SIZE, &clienthello->compressions_len)) { @@ -1677,7 +1632,7 @@ MSG_PROCESS_RETURN tls_process_client_hello(SSL_CONNECTION *s, PACKET *pkt) return MSG_PROCESS_ERROR; } -static int tls_early_post_process_client_hello(SSL_CONNECTION *s) +static int tls_early_post_process_client_hello(SSL *s) { unsigned int j; int i, al = SSL_AD_INTERNAL_ERROR; @@ -1692,14 +1647,28 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) STACK_OF(SSL_CIPHER) *scsvs = NULL; CLIENTHELLO_MSG *clienthello = s->clienthello; DOWNGRADE dgrd = DOWNGRADE_NONE; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - SSL *ssl = SSL_CONNECTION_GET_SSL(s); +#ifndef OPENSSL_NO_SESSION_LOOKUP + if (SSL_want_sess_lookup(s)) + goto query_session_reentry; +#endif /* Finished parsing the ClientHello, now we can start processing it */ /* Give the ClientHello callback a crack at things */ - if (sctx->client_hello_cb != NULL) { + if (s->ctx->client_hello_cb != NULL) { + /* + * Support setting ocsp response message in clienthello callback + * function. Parse the ocsp status and set the relevant flags here, + * otherwise the ocsp API may fail in the client_hello_cb function. + * Because in client_hello_cb it is possible to check if the client + * sends a status_request message. + */ + if (!tls_parse_extension(s, TLSEXT_IDX_status_request, + SSL_EXT_CLIENT_HELLO, + clienthello->pre_proc_exts, NULL, 0)) + goto err; + /* A failure in the ClientHello callback terminates the connection. */ - switch (sctx->client_hello_cb(ssl, &al, sctx->client_hello_cb_arg)) { + switch (s->ctx->client_hello_cb(s, &al, s->ctx->client_hello_cb_arg)) { case SSL_CLIENT_HELLO_SUCCESS: break; case SSL_CLIENT_HELLO_RETRY: @@ -1731,9 +1700,18 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) /* SSLv3/TLS */ s->client_version = clienthello->legacy_version; } - - /* Choose the server SSL/TLS/DTLS version. */ - protverr = ssl_choose_server_version(s, clienthello, &dgrd); + /* + * Do SSL/TLS version negotiation if applicable. For DTLS we just check + * versions are potentially compatible. Version negotiation comes later. + */ + if (!SSL_IS_DTLS(s)) { + protverr = ssl_choose_server_version(s, clienthello, &dgrd); + } else if (s->method->version != DTLS_ANY_VERSION && + DTLS_VERSION_LT((int)clienthello->legacy_version, s->version)) { + protverr = SSL_R_VERSION_TOO_LOW; + } else { + protverr = 0; + } if (protverr) { if (SSL_IS_FIRST_HANDSHAKE(s)) { @@ -1745,17 +1723,16 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) } /* TLSv1.3 specifies that a ClientHello must end on a record boundary */ - if (SSL_CONNECTION_IS_TLS13(s) - && RECORD_LAYER_processed_read_pending(&s->rlayer)) { + if (SSL_IS_TLS13(s) && RECORD_LAYER_processed_read_pending(&s->rlayer)) { SSLfatal(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_NOT_ON_RECORD_BOUNDARY); goto err; } - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { /* Empty cookie was already handled above by returning early. */ - if (SSL_get_options(ssl) & SSL_OP_COOKIE_EXCHANGE) { - if (sctx->app_verify_cookie_cb != NULL) { - if (sctx->app_verify_cookie_cb(ssl, clienthello->dtls_cookie, + if (SSL_get_options(s) & SSL_OP_COOKIE_EXCHANGE) { + if (s->ctx->app_verify_cookie_cb != NULL) { + if (s->ctx->app_verify_cookie_cb(s, clienthello->dtls_cookie, clienthello->dtls_cookie_len) == 0) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_COOKIE_MISMATCH); @@ -1771,14 +1748,26 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) } s->d1->cookie_verified = 1; } + if (s->method->version == DTLS_ANY_VERSION) { + protverr = ssl_choose_server_version(s, clienthello, &dgrd); + if (protverr != 0) { + s->version = s->client_version; + SSLfatal(s, SSL_AD_PROTOCOL_VERSION, protverr); + goto err; + } + } } s->hit = 0; +#ifndef OPENSSL_NO_SESSION_REUSED_TYPE + s->session_reused_type = SSL_SESSION_REUSED_TYPE_NOCACHE; +#endif + if (!ssl_cache_cipherlist(s, &clienthello->ciphersuites, clienthello->isv2) || - !ossl_bytes_to_cipher_list(s, &clienthello->ciphersuites, &ciphers, - &scsvs, clienthello->isv2, 1)) { + !bytes_to_cipher_list(s, &clienthello->ciphersuites, &ciphers, &scsvs, + clienthello->isv2, 1)) { /* SSLfatal() already called */ goto err; } @@ -1786,7 +1775,7 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) s->s3.send_connection_binding = 0; /* Check what signalling cipher-suite values were received. */ if (scsvs != NULL) { - for (i = 0; i < sk_SSL_CIPHER_num(scsvs); i++) { + for(i = 0; i < sk_SSL_CIPHER_num(scsvs); i++) { c = sk_SSL_CIPHER_value(scsvs, i); if (SSL_CIPHER_get_id(c) == SSL3_CK_SCSV) { if (s->renegotiate) { @@ -1798,6 +1787,12 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) s->s3.send_connection_binding = 1; } else if (SSL_CIPHER_get_id(c) == SSL3_CK_FALLBACK_SCSV && !ssl_check_version_downgrade(s)) { +#ifndef OPENSSL_NO_SKIP_SCSV + /* XXX: we skip SCSV in case of bad client implementations */ + if (s->skip_scsv) + continue; +#endif + /* * This SCSV indicates that the client previously tried * a higher version. We should fail if the current version @@ -1813,9 +1808,9 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) } /* For TLSv1.3 we must select the ciphersuite *before* session resumption */ - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { const SSL_CIPHER *cipher = - ssl3_choose_cipher(s, ciphers, SSL_get_ciphers(ssl)); + ssl3_choose_cipher(s, ciphers, SSL_get_ciphers(s)); if (cipher == NULL) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_NO_SHARED_CIPHER); @@ -1832,6 +1827,24 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) goto err; } s->s3.tmp.new_cipher = cipher; +#ifndef OPENSSL_NO_SM2 + /* + * According to RFC 8998, if server chooses TLS_SM4_GCM_SM3 or + * TLS_SM4_CCM_SM3, the named group must be curveSM2. + */ + if (s->enable_sm_tls13_strict == 1 + && (cipher->id == TLS1_3_CK_SM4_GCM_SM3 + || cipher->id == TLS1_3_CK_SM4_CCM_SM3)) { + int sm2_group = NID_sm2; + + if (!tls1_set_groups(&s->ext.supportedgroups, + &s->ext.supportedgroups_len, + &sm2_group, 1)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } +#endif } /* We need to do this before getting the session */ @@ -1866,6 +1879,9 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) goto err; } } else { +#if !defined(OPENSSL_NO_SESSION_LOOKUP) + query_session_reentry: +#endif i = ssl_get_prev_session(s, clienthello); if (i == 1) { /* previous session */ @@ -1873,6 +1889,14 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) } else if (i == -1) { /* SSLfatal() already called */ goto err; +#ifndef OPENSSL_NO_SESSION_LOOKUP + } else if (i == -2) { + s->rwstate = SSL_SESS_LOOKUP; + s->statem.read_state_work = WORK_MORE_A; + s->clienthello->ciphers = ciphers; + sk_SSL_CIPHER_free(scsvs); + return i; +#endif } else { /* i == 0 */ if (!ssl_get_new_session(s, 1)) { @@ -1880,19 +1904,27 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) goto err; } } +#ifndef OPENSSL_NO_SESSION_LOOKUP + s->rwstate = SSL_NOTHING; +#endif } - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { memcpy(s->tmp_session_id, s->clienthello->session_id, s->clienthello->session_id_len); s->tmp_session_id_len = s->clienthello->session_id_len; } +#if !defined(OPENSSL_NO_SESSION_LOOKUP) + if (ciphers == NULL && s->clienthello->ciphers != NULL) + ciphers = s->clienthello->ciphers; +#endif + /* * If it is a hit, check that the cipher is in the list. In TLSv1.3 we check * ciphersuite compatibility with the session as part of resumption. */ - if (!SSL_CONNECTION_IS_TLS13(s) && s->hit) { + if (!SSL_IS_TLS13(s) && s->hit) { j = 0; id = s->session->cipher->id; @@ -1944,6 +1976,55 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) goto err; } +#ifndef OPENSSL_NO_SM2 + /* + * To use the cipher suites TLS_SM4_GCM_SM3 and TLS_SM4_CCM_SM3, + * RFC 8998 demand that: + * The certificate must be a SM2 cert; + * For the signature_algorithms extension, "sm2sig_sm3" MUST be included. + * For the signature_algorithms_cert extension (if present), "sm2sig_sm3" + * MUST be included. + * For the key_share extension, a KeyShareEntry for the "curveSM2" + * group MUST be included. + */ + if (SSL_IS_TLS13(s) && s->enable_sm_tls13_strict == 1) { + const SSL_CIPHER *cipher = s->s3.tmp.new_cipher; + + if (cipher->id == TLS1_3_CK_SM4_GCM_SM3 + || cipher->id == TLS1_3_CK_SM4_CCM_SM3) { + uint16_t *peer_sig = s->s3.tmp.peer_sigalgs; + size_t idx; + + for (idx = 0; idx < s->s3.tmp.peer_sigalgslen; idx++) { + if (peer_sig[idx] == TLSEXT_SIGALG_sm2sig_sm3) + break; + } + + if (s->s3.tmp.peer_sigalgslen > 0 + && idx >= s->s3.tmp.peer_sigalgslen) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_VALUE); + goto err; + } + + peer_sig = s->s3.tmp.peer_cert_sigalgs; + for (idx = 0; idx < s->s3.tmp.peer_cert_sigalgslen; idx++) { + if (peer_sig[idx] == TLSEXT_SIGALG_sm2sig_sm3) + break; + } + + if (s->s3.tmp.peer_cert_sigalgslen > 0 + && idx >= s->s3.tmp.peer_cert_sigalgslen) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_VALUE); + goto err; + } + + if (s->s3.group_id != TLSEXT_curve_SM2) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + goto err; + } + } + } +#endif /* * Check if we want to use external pre-shared secret for this handshake * for not reused session only. We need to generate server_random before @@ -1959,16 +2040,11 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) } } - if (!s->hit && !tls1_set_server_sigalgs(s)) { - /* SSLfatal() already called */ - goto err; - } - if (!s->hit && s->version >= TLS1_VERSION - && !SSL_CONNECTION_IS_TLS13(s) - && !SSL_CONNECTION_IS_DTLS(s) - && s->ext.session_secret_cb != NULL) { + && !SSL_IS_TLS13(s) + && !SSL_IS_DTLS(s) + && s->ext.session_secret_cb) { const SSL_CIPHER *pref_cipher = NULL; /* * s->session->master_key_length is a size_t, but this is an int for @@ -1977,7 +2053,7 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) int master_key_length; master_key_length = sizeof(s->session->master_key); - if (s->ext.session_secret_cb(ssl, s->session->master_key, + if (s->ext.session_secret_cb(s, s->session->master_key, &master_key_length, ciphers, &pref_cipher, s->ext.session_secret_cb_arg) @@ -1987,12 +2063,16 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) s->peer_ciphers = ciphers; s->session->verify_result = X509_V_OK; +#ifndef OPENSSL_NO_SESSION_REUSED_TYPE + s->session_reused_type = SSL_SESSION_REUSED_TYPE_CACHE; +#endif + ciphers = NULL; /* check if some cipher was preferred by call back */ if (pref_cipher == NULL) pref_cipher = ssl3_choose_cipher(s, s->peer_ciphers, - SSL_get_ciphers(ssl)); + SSL_get_ciphers(s)); if (pref_cipher == NULL) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_NO_SHARED_CIPHER); goto err; @@ -2012,7 +2092,7 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) * algorithms from the client, starting at q. */ s->s3.tmp.new_compression = NULL; - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* * We already checked above that the NULL compression method appears in * the list. Now we check there aren't any others (which is illegal in @@ -2037,8 +2117,8 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) goto err; } /* Look for resumed compression method */ - for (m = 0; m < sk_SSL_COMP_num(sctx->comp_methods); m++) { - comp = sk_SSL_COMP_value(sctx->comp_methods, m); + for (m = 0; m < sk_SSL_COMP_num(s->ctx->comp_methods); m++) { + comp = sk_SSL_COMP_value(s->ctx->comp_methods, m); if (comp_id == comp->id) { s->s3.tmp.new_compression = comp; break; @@ -2061,14 +2141,14 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) } } else if (s->hit) { comp = NULL; - } else if (ssl_allow_compression(s) && sctx->comp_methods) { + } else if (ssl_allow_compression(s) && s->ctx->comp_methods) { /* See if we have a match */ int m, nn, v, done = 0; unsigned int o; - nn = sk_SSL_COMP_num(sctx->comp_methods); + nn = sk_SSL_COMP_num(s->ctx->comp_methods); for (m = 0; m < nn; m++) { - comp = sk_SSL_COMP_value(sctx->comp_methods, m); + comp = sk_SSL_COMP_value(s->ctx->comp_methods, m); v = comp->id; for (o = 0; o < clienthello->compressions_len; o++) { if (v == clienthello->compressions[o]) { @@ -2099,7 +2179,7 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) * Given s->peer_ciphers and SSL_get_ciphers, we must pick a cipher */ - if (!s->hit || SSL_CONNECTION_IS_TLS13(s)) { + if (!s->hit || SSL_IS_TLS13(s)) { sk_SSL_CIPHER_free(s->peer_ciphers); s->peer_ciphers = ciphers; if (ciphers == NULL) { @@ -2115,6 +2195,10 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) #else s->session->compress_meth = (comp == NULL) ? 0 : comp->id; #endif + if (!tls1_set_server_sigalgs(s)) { + /* SSLfatal() already called */ + goto err; + } } sk_SSL_CIPHER_free(ciphers); @@ -2137,10 +2221,8 @@ static int tls_early_post_process_client_hello(SSL_CONNECTION *s) * Call the status request callback if needed. Upon success, returns 1. * Upon failure, returns 0. */ -static int tls_handle_status_request(SSL_CONNECTION *s) +static int tls_handle_status_request(SSL *s) { - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - s->ext.status_expected = 0; /* @@ -2149,8 +2231,8 @@ static int tls_handle_status_request(SSL_CONNECTION *s) * and must be called after the cipher has been chosen because this may * influence which certificate is sent */ - if (s->ext.status_type != TLSEXT_STATUSTYPE_nothing && sctx != NULL - && sctx->ext.status_cb != NULL) { + if (s->ext.status_type != TLSEXT_STATUSTYPE_nothing && s->ctx != NULL + && s->ctx->ext.status_cb != NULL) { int ret; /* If no certificate can't return certificate status */ @@ -2160,8 +2242,7 @@ static int tls_handle_status_request(SSL_CONNECTION *s) * et al can pick it up. */ s->cert->key = s->s3.tmp.cert; - ret = sctx->ext.status_cb(SSL_CONNECTION_GET_SSL(s), - sctx->ext.status_arg); + ret = s->ctx->ext.status_cb(s, s->ctx->ext.status_arg); switch (ret) { /* We don't want to send a status request response */ case SSL_TLSEXT_ERR_NOACK: @@ -2188,18 +2269,16 @@ static int tls_handle_status_request(SSL_CONNECTION *s) * Call the alpn_select callback if needed. Upon success, returns 1. * Upon failure, returns 0. */ -int tls_handle_alpn(SSL_CONNECTION *s) +int tls_handle_alpn(SSL *s) { const unsigned char *selected = NULL; unsigned char selected_len = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - if (sctx->ext.alpn_select_cb != NULL && s->s3.alpn_proposed != NULL) { - int r = sctx->ext.alpn_select_cb(SSL_CONNECTION_GET_SSL(s), - &selected, &selected_len, - s->s3.alpn_proposed, - (unsigned int)s->s3.alpn_proposed_len, - sctx->ext.alpn_select_cb_arg); + if (s->ctx->ext.alpn_select_cb != NULL && s->s3.alpn_proposed != NULL) { + int r = s->ctx->ext.alpn_select_cb(s, &selected, &selected_len, + s->s3.alpn_proposed, + (unsigned int)s->s3.alpn_proposed_len, + s->ctx->ext.alpn_select_cb_arg); if (r == SSL_TLSEXT_ERR_OK) { OPENSSL_free(s->s3.alpn_selected); @@ -2266,10 +2345,9 @@ int tls_handle_alpn(SSL_CONNECTION *s) return 1; } -WORK_STATE tls_post_process_client_hello(SSL_CONNECTION *s, WORK_STATE wst) +WORK_STATE tls_post_process_client_hello(SSL *s, WORK_STATE wst) { const SSL_CIPHER *cipher; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); if (wst == WORK_MORE_A) { int rv = tls_early_post_process_client_hello(s); @@ -2282,10 +2360,10 @@ WORK_STATE tls_post_process_client_hello(SSL_CONNECTION *s, WORK_STATE wst) wst = WORK_MORE_B; } if (wst == WORK_MORE_B) { - if (!s->hit || SSL_CONNECTION_IS_TLS13(s)) { + if (!s->hit || SSL_IS_TLS13(s)) { /* Let cert callback update server certificates if required */ if (!s->hit && s->cert->cert_cb != NULL) { - int rv = s->cert->cert_cb(ssl, s->cert->cert_cb_arg); + int rv = s->cert->cert_cb(s, s->cert->cert_cb_arg); if (rv == 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_CERT_CB_ERROR); goto err; @@ -2298,10 +2376,9 @@ WORK_STATE tls_post_process_client_hello(SSL_CONNECTION *s, WORK_STATE wst) } /* In TLSv1.3 we selected the ciphersuite before resumption */ - if (!SSL_CONNECTION_IS_TLS13(s)) { + if (!SSL_IS_TLS13(s)) { cipher = - ssl3_choose_cipher(s, s->peer_ciphers, - SSL_get_ciphers(ssl)); + ssl3_choose_cipher(s, s->peer_ciphers, SSL_get_ciphers(s)); if (cipher == NULL) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, @@ -2318,7 +2395,7 @@ WORK_STATE tls_post_process_client_hello(SSL_CONNECTION *s, WORK_STATE wst) /* check whether we should disable session resumption */ if (s->not_resumable_session_cb != NULL) s->session->not_resumable = - s->not_resumable_session_cb(ssl, + s->not_resumable_session_cb(s, ((s->s3.tmp.new_cipher->algorithm_mkey & (SSL_kDHE | SSL_kECDHE)) != 0)); if (s->session->not_resumable) @@ -2334,7 +2411,7 @@ WORK_STATE tls_post_process_client_hello(SSL_CONNECTION *s, WORK_STATE wst) * we now have the following setup. * client_random * cipher_list - our preferred list of ciphers - * ciphers - the client's preferred list of ciphers + * ciphers - the clients preferred list of ciphers * compression - basically ignored right now * ssl version is set - sslv3 * s->session - The ssl session has been setup. @@ -2356,7 +2433,7 @@ WORK_STATE tls_post_process_client_hello(SSL_CONNECTION *s, WORK_STATE wst) * we already did this because cipher negotiation happens earlier, and * we must handle ALPN before we decide whether to accept early_data. */ - if (!SSL_CONNECTION_IS_TLS13(s) && !tls_handle_alpn(s)) { + if (!SSL_IS_TLS13(s) && !tls_handle_alpn(s)) { /* SSLfatal() already called */ goto err; } @@ -2385,14 +2462,33 @@ WORK_STATE tls_post_process_client_hello(SSL_CONNECTION *s, WORK_STATE wst) return WORK_ERROR; } -CON_FUNC_RETURN tls_construct_server_hello(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_server_hello(SSL *s, WPACKET *pkt) { int compm; size_t sl, len; int version; unsigned char *session_id; - int usetls13 = SSL_CONNECTION_IS_TLS13(s) - || s->hello_retry_request == SSL_HRR_PENDING; + int usetls13 = SSL_IS_TLS13(s) || s->hello_retry_request == SSL_HRR_PENDING; + +#ifndef OPENSSL_NO_VERIFY_SNI + X509 *x509; + const char *servername; + + if (!s->hit && + (s->verify_mode & SSL_VERIFY_FAIL_IF_SNI_NOT_MATCH_CERT)) { + servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name); + + if (servername != NULL && (x509 = SSL_get_certificate(s)) != NULL + && X509_check_host(x509, servername, strlen(servername), + X509_CHECK_FLAG_MULTI_LABEL_WILDCARDS, + NULL) != 1) { + if (s->verify_mode & SSL_VERIFY_FAIL_IF_SNI_NOT_MATCH_CERT) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + } +#endif version = usetls13 ? TLS1_2_VERSION : s->version; if (!WPACKET_put_bytes_u16(pkt, version) @@ -2405,7 +2501,7 @@ CON_FUNC_RETURN tls_construct_server_hello(SSL_CONNECTION *s, WPACKET *pkt) ? hrrrandom : s->s3.server_random, SSL3_RANDOM_SIZE)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /*- @@ -2426,8 +2522,9 @@ CON_FUNC_RETURN tls_construct_server_hello(SSL_CONNECTION *s, WPACKET *pkt) * so the following won't overwrite an ID that we're supposed * to send back. */ - if (!(SSL_CONNECTION_GET_CTX(s)->session_cache_mode & SSL_SESS_CACHE_SERVER) - && !s->hit) + if (s->session->not_resumable || + (!(s->ctx->session_cache_mode & SSL_SESS_CACHE_SERVER) + && !s->hit)) s->session->session_id_length = 0; if (usetls13) { @@ -2440,7 +2537,7 @@ CON_FUNC_RETURN tls_construct_server_hello(SSL_CONNECTION *s, WPACKET *pkt) if (sl > sizeof(s->session->session_id)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /* set up the compression method */ @@ -2454,22 +2551,21 @@ CON_FUNC_RETURN tls_construct_server_hello(SSL_CONNECTION *s, WPACKET *pkt) #endif if (!WPACKET_sub_memcpy_u8(pkt, session_id, sl) - || !SSL_CONNECTION_GET_SSL(s)->method->put_cipher_by_char(s->s3.tmp.new_cipher, - pkt, &len) + || !s->method->put_cipher_by_char(s->s3.tmp.new_cipher, pkt, &len) || !WPACKET_put_bytes_u8(pkt, compm)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } if (!tls_construct_extensions(s, pkt, s->hello_retry_request == SSL_HRR_PENDING ? SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST - : (SSL_CONNECTION_IS_TLS13(s) + : (SSL_IS_TLS13(s) ? SSL_EXT_TLS1_3_SERVER_HELLO : SSL_EXT_TLS1_2_SERVER_HELLO), NULL, 0)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } if (s->hello_retry_request == SSL_HRR_PENDING) { @@ -2484,30 +2580,29 @@ CON_FUNC_RETURN tls_construct_server_hello(SSL_CONNECTION *s, WPACKET *pkt) */ if (!create_synthetic_message_hash(s, NULL, 0, NULL, 0)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } } else if (!(s->verify_mode & SSL_VERIFY_PEER) && !ssl3_digest_cached_records(s, 0)) { /* SSLfatal() already called */; - return CON_FUNC_ERROR; + return 0; } - return CON_FUNC_SUCCESS; + return 1; } -CON_FUNC_RETURN tls_construct_server_done(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_server_done(SSL *s, WPACKET *pkt) { if (!s->s3.tmp.cert_request) { if (!ssl3_digest_cached_records(s, 0)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } } - return CON_FUNC_SUCCESS; + return 1; } -CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, - WPACKET *pkt) +int tls_construct_server_key_exchange(SSL *s, WPACKET *pkt) { EVP_PKEY *pkdh = NULL; unsigned char *encodedPoint = NULL; @@ -2520,9 +2615,7 @@ CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, EVP_MD_CTX *md_ctx = EVP_MD_CTX_new(); EVP_PKEY_CTX *pctx = NULL; size_t paramlen, paramoffset; - int freer = 0; - CON_FUNC_RETURN ret = CON_FUNC_ERROR; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); + int freer = 0, ret = 0; if (!WPACKET_get_total_written(pkt, ¶moffset)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); @@ -2530,7 +2623,7 @@ CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, } if (md_ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -2558,8 +2651,7 @@ CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, } #if !defined(OPENSSL_NO_DEPRECATED_3_0) if ((pkdhp == NULL) && (s->cert->dh_tmp_cb != NULL)) { - pkdh = ssl_dh_to_pkey(s->cert->dh_tmp_cb(SSL_CONNECTION_GET_SSL(s), - 0, 1024)); + pkdh = ssl_dh_to_pkey(s->cert->dh_tmp_cb(s, 0, 1024)); if (pkdh == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; @@ -2742,6 +2834,19 @@ CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } + +#ifndef OPENSSL_NO_STATUS + /* record curve_id and pubkey */ + if (s->status_param.ssl_status_enable) { + s->status_param.type = SSL_SERVER_EXCHANGE_PUBKEY; + if (s->status_callback(WPACKET_get_curr(pkt) - encodedlen, + encodedlen, &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + } +#endif + OPENSSL_free(encodedPoint); encodedPoint = NULL; } @@ -2753,7 +2858,7 @@ CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, unsigned char *sigbytes1, *sigbytes2, *tbs; size_t siglen = 0, tbslen; - if (pkey == NULL || !tls1_lookup_md(sctx, lu, &md)) { + if (pkey == NULL || !tls1_lookup_md(s->ctx, lu, &md)) { /* Should never happen */ SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; @@ -2771,7 +2876,7 @@ CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, if (EVP_DigestSignInit_ex(md_ctx, &pctx, md == NULL ? NULL : EVP_MD_get0_name(md), - sctx->libctx, sctx->propq, pkey, + s->ctx->libctx, s->ctx->propq, pkey, NULL) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; @@ -2803,7 +2908,7 @@ CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, OPENSSL_free(tbs); } - ret = CON_FUNC_SUCCESS; + ret = 1; err: EVP_PKEY_free(pkdh); OPENSSL_free(encodedPoint); @@ -2817,10 +2922,9 @@ CON_FUNC_RETURN tls_construct_server_key_exchange(SSL_CONNECTION *s, return ret; } -CON_FUNC_RETURN tls_construct_certificate_request(SSL_CONNECTION *s, - WPACKET *pkt) +int tls_construct_certificate_request(SSL *s, WPACKET *pkt) { - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* Send random context when doing post-handshake auth */ if (s->post_handshake_auth == SSL_PHA_REQUEST_PENDING) { OPENSSL_free(s->pha_context); @@ -2828,24 +2932,24 @@ CON_FUNC_RETURN tls_construct_certificate_request(SSL_CONNECTION *s, if ((s->pha_context = OPENSSL_malloc(s->pha_context_len)) == NULL) { s->pha_context_len = 0; SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } - if (RAND_bytes_ex(SSL_CONNECTION_GET_CTX(s)->libctx, - s->pha_context, s->pha_context_len, 0) <= 0 + if (RAND_bytes_ex(s->ctx->libctx, s->pha_context, + s->pha_context_len, 0) <= 0 || !WPACKET_sub_memcpy_u8(pkt, s->pha_context, s->pha_context_len)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /* reset the handshake hash back to just after the ClientFinished */ if (!tls13_restore_handshake_digest_for_pha(s)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } } else { if (!WPACKET_put_bytes_u8(pkt, 0)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } } @@ -2853,7 +2957,7 @@ CON_FUNC_RETURN tls_construct_certificate_request(SSL_CONNECTION *s, SSL_EXT_TLS1_3_CERTIFICATE_REQUEST, NULL, 0)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } goto done; } @@ -2862,7 +2966,7 @@ CON_FUNC_RETURN tls_construct_certificate_request(SSL_CONNECTION *s, if (!WPACKET_start_sub_packet_u8(pkt) || !ssl3_get_req_cert_type(s, pkt) || !WPACKET_close(pkt)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } if (SSL_USE_SIGALGS(s)) { @@ -2874,22 +2978,22 @@ CON_FUNC_RETURN tls_construct_certificate_request(SSL_CONNECTION *s, || !tls12_copy_sigalgs(s, pkt, psigs, nl) || !WPACKET_close(pkt)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } } if (!construct_ca_names(s, get_ca_names(s), pkt)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } done: s->certreqs_sent++; s->s3.tmp.cert_request = 1; - return CON_FUNC_SUCCESS; + return 1; } -static int tls_process_cke_psk_preamble(SSL_CONNECTION *s, PACKET *pkt) +static int tls_process_cke_psk_preamble(SSL *s, PACKET *pkt) { #ifndef OPENSSL_NO_PSK unsigned char psk[PSK_MAX_PSK_LEN]; @@ -2914,8 +3018,7 @@ static int tls_process_cke_psk_preamble(SSL_CONNECTION *s, PACKET *pkt) return 0; } - psklen = s->psk_server_callback(SSL_CONNECTION_GET_SSL(s), - s->session->psk_identity, + psklen = s->psk_server_callback(s, s->session->psk_identity, psk, sizeof(psk)); if (psklen > PSK_MAX_PSK_LEN) { @@ -2935,7 +3038,7 @@ static int tls_process_cke_psk_preamble(SSL_CONNECTION *s, PACKET *pkt) if (s->s3.tmp.psk == NULL) { s->s3.tmp.psklen = 0; - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } @@ -2949,7 +3052,7 @@ static int tls_process_cke_psk_preamble(SSL_CONNECTION *s, PACKET *pkt) #endif } -static int tls_process_cke_rsa(SSL_CONNECTION *s, PACKET *pkt) +static int tls_process_cke_rsa(SSL *s, PACKET *pkt) { size_t outlen; PACKET enc_premaster; @@ -2958,7 +3061,6 @@ static int tls_process_cke_rsa(SSL_CONNECTION *s, PACKET *pkt) int ret = 0; EVP_PKEY_CTX *ctx = NULL; OSSL_PARAM params[3], *p = params; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); rsa = s->cert->pkeys[SSL_PKEY_RSA].privatekey; if (rsa == NULL) { @@ -2977,16 +3079,29 @@ static int tls_process_cke_rsa(SSL_CONNECTION *s, PACKET *pkt) } } +#ifndef OPENSSL_NO_STATUS + /* record encrypted client pms with RSA KeyExchange mode */ + if (s->status_param.ssl_status_enable) { + s->status_param.type = SSL_CLIENT_RSA_EXCHANGE; + if (s->status_callback((unsigned char *)enc_premaster.curr, + enc_premaster.remaining, + &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + return 0; + } + } +#endif + outlen = SSL_MAX_MASTER_KEY_LENGTH; rsa_decrypt = OPENSSL_malloc(outlen); if (rsa_decrypt == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } - ctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, rsa, sctx->propq); + ctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, rsa, s->ctx->propq); if (ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -2994,7 +3109,7 @@ static int tls_process_cke_rsa(SSL_CONNECTION *s, PACKET *pkt) * We must not leak whether a decryption failure occurs because of * Bleichenbacher's attack on PKCS #1 v1.5 RSA padding (see RFC 2246, * section 7.4.7.1). We use the special padding type - * RSA_PKCS1_WITH_TLS_PADDING to do that. It will automatically decrypt the + * RSA_PKCS1_WITH_TLS_PADDING to do that. It will automaticaly decrypt the * RSA, check the padding and check that the client version is as expected * in the premaster secret. If any of that fails then the function appears * to return successfully but with a random result. The call below could @@ -3034,7 +3149,8 @@ static int tls_process_cke_rsa(SSL_CONNECTION *s, PACKET *pkt) } /* Also cleanses rsa_decrypt (on success or failure) */ - if (!ssl_generate_master_secret(s, rsa_decrypt, outlen, 0)) { + if (!ssl_generate_master_secret(s, rsa_decrypt, + SSL_MAX_MASTER_KEY_LENGTH, 0)) { /* SSLfatal() already called */ goto err; } @@ -3046,7 +3162,7 @@ static int tls_process_cke_rsa(SSL_CONNECTION *s, PACKET *pkt) return ret; } -static int tls_process_cke_dhe(SSL_CONNECTION *s, PACKET *pkt) +static int tls_process_cke_dhe(SSL *s, PACKET *pkt) { EVP_PKEY *skey = NULL; unsigned int i; @@ -3084,6 +3200,21 @@ static int tls_process_cke_dhe(SSL_CONNECTION *s, PACKET *pkt) goto err; } +#ifndef OPENSSL_NO_STATUS + /* record client DH pubkey */ + if (s->status_param.ssl_status_enable) { + BIGNUM *pub_key = BN_bin2bn(data, i, NULL); + s->status_param.type = SSL_SERVER_DH_PUBKEY; + s->status_param.parg = pub_key; + if (s->status_callback((unsigned char *)data, + BN_num_bytes(pub_key), + &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + } +#endif + if (ssl_derive(s, skey, ckey, 1) == 0) { /* SSLfatal() already called */ goto err; @@ -3097,7 +3228,7 @@ static int tls_process_cke_dhe(SSL_CONNECTION *s, PACKET *pkt) return ret; } -static int tls_process_cke_ecdhe(SSL_CONNECTION *s, PACKET *pkt) +static int tls_process_cke_ecdhe(SSL *s, PACKET *pkt) { EVP_PKEY *skey = s->s3.tmp.pkey; EVP_PKEY *ckey = NULL; @@ -3137,6 +3268,21 @@ static int tls_process_cke_ecdhe(SSL_CONNECTION *s, PACKET *pkt) SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EC_LIB); goto err; } + +#ifndef OPENSSL_NO_STATUS + /* record client DH pubkey */ + if (s->status_param.ssl_status_enable) { + BIGNUM *pub_key = BN_bin2bn(data, i, NULL); + s->status_param.type = SSL_SERVER_DH_PUBKEY; + s->status_param.parg = pub_key; + if (s->status_callback((unsigned char *)data, + BN_num_bytes(pub_key), + &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + } +#endif } if (ssl_derive(s, skey, ckey, 1) == 0) { @@ -3153,7 +3299,7 @@ static int tls_process_cke_ecdhe(SSL_CONNECTION *s, PACKET *pkt) return ret; } -static int tls_process_cke_srp(SSL_CONNECTION *s, PACKET *pkt) +static int tls_process_cke_srp(SSL *s, PACKET *pkt) { #ifndef OPENSSL_NO_SRP unsigned int i; @@ -3175,7 +3321,7 @@ static int tls_process_cke_srp(SSL_CONNECTION *s, PACKET *pkt) OPENSSL_free(s->session->srp_username); s->session->srp_username = OPENSSL_strdup(s->srp_ctx.login); if (s->session->srp_username == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); return 0; } @@ -3192,189 +3338,7 @@ static int tls_process_cke_srp(SSL_CONNECTION *s, PACKET *pkt) #endif } -static int tls_process_cke_gost(SSL_CONNECTION *s, PACKET *pkt) -{ -#ifndef OPENSSL_NO_GOST - EVP_PKEY_CTX *pkey_ctx; - EVP_PKEY *client_pub_pkey = NULL, *pk = NULL; - unsigned char premaster_secret[32]; - const unsigned char *start; - size_t outlen = sizeof(premaster_secret), inlen; - unsigned long alg_a; - GOST_KX_MESSAGE *pKX = NULL; - const unsigned char *ptr; - int ret = 0; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - - /* Get our certificate private key */ - alg_a = s->s3.tmp.new_cipher->algorithm_auth; - if (alg_a & SSL_aGOST12) { - /* - * New GOST ciphersuites have SSL_aGOST01 bit too - */ - pk = s->cert->pkeys[SSL_PKEY_GOST12_512].privatekey; - if (pk == NULL) { - pk = s->cert->pkeys[SSL_PKEY_GOST12_256].privatekey; - } - if (pk == NULL) { - pk = s->cert->pkeys[SSL_PKEY_GOST01].privatekey; - } - } else if (alg_a & SSL_aGOST01) { - pk = s->cert->pkeys[SSL_PKEY_GOST01].privatekey; - } - - pkey_ctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, pk, sctx->propq); - if (pkey_ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - return 0; - } - if (EVP_PKEY_decrypt_init(pkey_ctx) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - /* - * If client certificate is present and is of the same type, maybe - * use it for key exchange. Don't mind errors from - * EVP_PKEY_derive_set_peer, because it is completely valid to use a - * client certificate for authorization only. - */ - client_pub_pkey = tls_get_peer_pkey(s); - if (client_pub_pkey) { - if (EVP_PKEY_derive_set_peer(pkey_ctx, client_pub_pkey) <= 0) - ERR_clear_error(); - } - - ptr = PACKET_data(pkt); - /* Some implementations provide extra data in the opaqueBlob - * We have nothing to do with this blob so we just skip it */ - pKX = d2i_GOST_KX_MESSAGE(NULL, &ptr, PACKET_remaining(pkt)); - if (pKX == NULL - || pKX->kxBlob == NULL - || ASN1_TYPE_get(pKX->kxBlob) != V_ASN1_SEQUENCE) { - SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_DECRYPTION_FAILED); - goto err; - } - - if (!PACKET_forward(pkt, ptr - PACKET_data(pkt))) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_DECRYPTION_FAILED); - goto err; - } - - if (PACKET_remaining(pkt) != 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_DECRYPTION_FAILED); - goto err; - } - - inlen = pKX->kxBlob->value.sequence->length; - start = pKX->kxBlob->value.sequence->data; - - if (EVP_PKEY_decrypt(pkey_ctx, premaster_secret, &outlen, start, - inlen) <= 0) { - SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_DECRYPTION_FAILED); - goto err; - } - /* Generate master secret */ - if (!ssl_generate_master_secret(s, premaster_secret, outlen, 0)) { - /* SSLfatal() already called */ - goto err; - } - /* Check if pubkey from client certificate was used */ - if (EVP_PKEY_CTX_ctrl(pkey_ctx, -1, -1, EVP_PKEY_CTRL_PEER_KEY, 2, - NULL) > 0) - s->statem.no_cert_verify = 1; - - ret = 1; - err: - EVP_PKEY_CTX_free(pkey_ctx); - GOST_KX_MESSAGE_free(pKX); - return ret; -#else - /* Should never happen */ - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; -#endif -} - -static int tls_process_cke_gost18(SSL_CONNECTION *s, PACKET *pkt) -{ -#ifndef OPENSSL_NO_GOST - unsigned char rnd_dgst[32]; - EVP_PKEY_CTX *pkey_ctx = NULL; - EVP_PKEY *pk = NULL; - unsigned char premaster_secret[32]; - const unsigned char *start = NULL; - size_t outlen = sizeof(premaster_secret), inlen = 0; - int ret = 0; - int cipher_nid = ossl_gost18_cke_cipher_nid(s); - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - - if (cipher_nid == NID_undef) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - - if (ossl_gost_ukm(s, rnd_dgst) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* Get our certificate private key */ - pk = s->cert->pkeys[SSL_PKEY_GOST12_512].privatekey != NULL ? - s->cert->pkeys[SSL_PKEY_GOST12_512].privatekey : - s->cert->pkeys[SSL_PKEY_GOST12_256].privatekey; - if (pk == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_HANDSHAKE_STATE); - goto err; - } - - pkey_ctx = EVP_PKEY_CTX_new_from_pkey(sctx->libctx, pk, sctx->propq); - if (pkey_ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - goto err; - } - if (EVP_PKEY_decrypt_init(pkey_ctx) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - - /* Reuse EVP_PKEY_CTRL_SET_IV, make choice in engine code depending on size */ - if (EVP_PKEY_CTX_ctrl(pkey_ctx, -1, EVP_PKEY_OP_DECRYPT, - EVP_PKEY_CTRL_SET_IV, 32, rnd_dgst) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_LIBRARY_BUG); - goto err; - } - - if (EVP_PKEY_CTX_ctrl(pkey_ctx, -1, EVP_PKEY_OP_DECRYPT, - EVP_PKEY_CTRL_CIPHER, cipher_nid, NULL) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_LIBRARY_BUG); - goto err; - } - inlen = PACKET_remaining(pkt); - start = PACKET_data(pkt); - - if (EVP_PKEY_decrypt(pkey_ctx, premaster_secret, &outlen, start, inlen) <= 0) { - SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_DECRYPTION_FAILED); - goto err; - } - /* Generate master secret */ - if (!ssl_generate_master_secret(s, premaster_secret, outlen, 0)) { - /* SSLfatal() already called */ - goto err; - } - ret = 1; - - err: - EVP_PKEY_CTX_free(pkey_ctx); - return ret; -#else - /* Should never happen */ - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; -#endif -} - -MSG_PROCESS_RETURN tls_process_client_key_exchange(SSL_CONNECTION *s, - PACKET *pkt) +MSG_PROCESS_RETURN tls_process_client_key_exchange(SSL *s, PACKET *pkt) { unsigned long alg_k; @@ -3417,16 +3381,6 @@ MSG_PROCESS_RETURN tls_process_client_key_exchange(SSL_CONNECTION *s, /* SSLfatal() already called */ goto err; } - } else if (alg_k & SSL_kGOST) { - if (!tls_process_cke_gost(s, pkt)) { - /* SSLfatal() already called */ - goto err; - } - } else if (alg_k & SSL_kGOST18) { - if (!tls_process_cke_gost18(s, pkt)) { - /* SSLfatal() already called */ - goto err; - } } else { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_UNKNOWN_CIPHER_TYPE); goto err; @@ -3442,12 +3396,11 @@ MSG_PROCESS_RETURN tls_process_client_key_exchange(SSL_CONNECTION *s, return MSG_PROCESS_ERROR; } -WORK_STATE tls_post_process_client_key_exchange(SSL_CONNECTION *s, - WORK_STATE wst) +WORK_STATE tls_post_process_client_key_exchange(SSL *s, WORK_STATE wst) { #ifndef OPENSSL_NO_SCTP if (wst == WORK_MORE_A) { - if (SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { unsigned char sctpauthkey[64]; char labelbuffer[sizeof(DTLS1_SCTP_AUTH_LABEL)]; size_t labellen; @@ -3463,8 +3416,7 @@ WORK_STATE tls_post_process_client_key_exchange(SSL_CONNECTION *s, if (s->mode & SSL_MODE_DTLS_SCTP_LABEL_LENGTH_BUG) labellen += 1; - if (SSL_export_keying_material(SSL_CONNECTION_GET_SSL(s), - sctpauthkey, + if (SSL_export_keying_material(s, sctpauthkey, sizeof(sctpauthkey), labelbuffer, labellen, NULL, 0, 0) <= 0) { @@ -3472,13 +3424,13 @@ WORK_STATE tls_post_process_client_key_exchange(SSL_CONNECTION *s, return WORK_ERROR; } - BIO_ctrl(s->wbio, BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY, + BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY, sizeof(sctpauthkey), sctpauthkey); } } #endif - if (s->statem.no_cert_verify || !received_client_cert(s)) { + if (s->statem.no_cert_verify || !s->session->peer) { /* * No certificate verify or no peer certificate so we no longer need * the handshake_buffer @@ -3506,93 +3458,7 @@ WORK_STATE tls_post_process_client_key_exchange(SSL_CONNECTION *s, return WORK_FINISHED_CONTINUE; } -MSG_PROCESS_RETURN tls_process_client_rpk(SSL_CONNECTION *sc, PACKET *pkt) -{ - MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR; - SSL_SESSION *new_sess = NULL; - EVP_PKEY *peer_rpk = NULL; - - if (!tls_process_rpk(sc, pkt, &peer_rpk)) { - /* SSLfatal already called */ - goto err; - } - - if (peer_rpk == NULL) { - if ((sc->verify_mode & SSL_VERIFY_FAIL_IF_NO_PEER_CERT) - && (sc->verify_mode & SSL_VERIFY_PEER)) { - SSLfatal(sc, SSL_AD_CERTIFICATE_REQUIRED, - SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE); - goto err; - } - } else { - if (ssl_verify_rpk(sc, peer_rpk) <= 0) { - SSLfatal(sc, ssl_x509err2alert(sc->verify_result), - SSL_R_CERTIFICATE_VERIFY_FAILED); - goto err; - } - } - - /* - * Sessions must be immutable once they go into the session cache. Otherwise - * we can get multi-thread problems. Therefore we don't "update" sessions, - * we replace them with a duplicate. Here, we need to do this every time - * a new RPK (or certificate) is received via post-handshake authentication, - * as the session may have already gone into the session cache. - */ - - if (sc->post_handshake_auth == SSL_PHA_REQUESTED) { - if ((new_sess = ssl_session_dup(sc->session, 0)) == NULL) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); - goto err; - } - - SSL_SESSION_free(sc->session); - sc->session = new_sess; - } - - /* Ensure there is no peer/peer_chain */ - X509_free(sc->session->peer); - sc->session->peer = NULL; - sk_X509_pop_free(sc->session->peer_chain, X509_free); - sc->session->peer_chain = NULL; - /* Save RPK */ - EVP_PKEY_free(sc->session->peer_rpk); - sc->session->peer_rpk = peer_rpk; - peer_rpk = NULL; - - sc->session->verify_result = sc->verify_result; - - /* - * Freeze the handshake buffer. For cert_verify_hash, - sizeof(sc->cert_verify_hash), - &sc->cert_verify_hash_len)) { - /* SSLfatal() already called */; - goto err; - } - - /* resend session tickets */ - sc->sent_tickets = 0; - } - - ret = MSG_PROCESS_CONTINUE_READING; - - err: - EVP_PKEY_free(peer_rpk); - return ret; -} - -MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, - PACKET *pkt) +MSG_PROCESS_RETURN tls_process_client_certificate(SSL *s, PACKET *pkt) { int i; MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR; @@ -3603,35 +3469,23 @@ MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, PACKET spkt, context; size_t chainidx; SSL_SESSION *new_sess = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* * To get this far we must have read encrypted data from the client. We no - * longer tolerate unencrypted alerts. This is ignored if less than TLSv1.3 + * longer tolerate unencrypted alerts. This value is ignored if less than + * TLSv1.3 */ - if (s->rlayer.rrlmethod->set_plain_alerts != NULL) - s->rlayer.rrlmethod->set_plain_alerts(s->rlayer.rrl, 0); - - if (s->ext.client_cert_type == TLSEXT_cert_type_rpk) - return tls_process_client_rpk(s, pkt); - - if (s->ext.client_cert_type != TLSEXT_cert_type_x509) { - SSLfatal(s, SSL_AD_UNSUPPORTED_CERTIFICATE, - SSL_R_UNKNOWN_CERTIFICATE_TYPE); - goto err; - } + s->statem.enc_read_state = ENC_READ_STATE_VALID; if ((sk = sk_X509_new_null()) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } - if (SSL_CONNECTION_IS_TLS13(s) - && (!PACKET_get_length_prefixed_1(pkt, &context) - || (s->pha_context == NULL && PACKET_remaining(&context) != 0) - || (s->pha_context != NULL - && !PACKET_equal(&context, s->pha_context, - s->pha_context_len)))) { + if (SSL_IS_TLS13(s) && (!PACKET_get_length_prefixed_1(pkt, &context) + || (s->pha_context == NULL && PACKET_remaining(&context) != 0) + || (s->pha_context != NULL && + !PACKET_equal(&context, s->pha_context, s->pha_context_len)))) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_INVALID_CONTEXT); goto err; } @@ -3650,9 +3504,9 @@ MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, } certstart = certbytes; - x = X509_new_ex(sctx->libctx, sctx->propq); + x = X509_new_ex(s->ctx->libctx, s->ctx->propq); if (x == NULL) { - SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_X509_LIB); + SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_MALLOC_FAILURE); goto err; } if (d2i_X509(&x, (const unsigned char **)&certbytes, l) == NULL) { @@ -3665,7 +3519,7 @@ MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, goto err; } - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { RAW_EXTENSION *rawexts = NULL; PACKET extensions; @@ -3686,7 +3540,7 @@ MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, } if (!sk_X509_push(sk, x)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } x = NULL; @@ -3725,6 +3579,31 @@ MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, SSL_R_UNKNOWN_CERTIFICATE_TYPE); goto err; } +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + X509 *peer_cert = sk_X509_value(sk, 0); + + if (s->delegated_credential_tag & DC_HAS_BEEN_USED_FOR_VERIFY_PEER) { + if (!SSL_IS_TLS13(s)) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_UNKNOWN_CERTIFICATE_TYPE); + goto err; + } + + if (!DC_check_valid(peer_cert, s->session->peer_dc)) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_CERTIFICATE_VERIFY_FAILED); + goto err; + } + + if (SSL_verify_delegated_credential_signature(peer_cert, + s->session->peer_dc, + 0) <= 0) { + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_CERTIFICATE_VERIFY_FAILED); + goto err; + } + } +#endif } /* @@ -3737,7 +3616,7 @@ MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, if (s->post_handshake_auth == SSL_PHA_REQUESTED) { if ((new_sess = ssl_session_dup(s->session, 0)) == 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -3749,18 +3628,15 @@ MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, s->session->peer = sk_X509_shift(sk); s->session->verify_result = s->verify_result; - OSSL_STACK_OF_X509_free(s->session->peer_chain); + sk_X509_pop_free(s->session->peer_chain, X509_free); s->session->peer_chain = sk; sk = NULL; - /* Ensure there is no RPK */ - EVP_PKEY_free(s->session->peer_rpk); - s->session->peer_rpk = NULL; /* * Freeze the handshake buffer. For cert_verify_hash, sizeof(s->cert_verify_hash), &s->cert_verify_hash_len)) { @@ -3787,93 +3663,39 @@ MSG_PROCESS_RETURN tls_process_client_certificate(SSL_CONNECTION *s, err: X509_free(x); - OSSL_STACK_OF_X509_free(sk); + sk_X509_pop_free(sk, X509_free); return ret; } -#ifndef OPENSSL_NO_COMP_ALG -MSG_PROCESS_RETURN tls_process_client_compressed_certificate(SSL_CONNECTION *sc, PACKET *pkt) -{ - MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR; - PACKET tmppkt; - BUF_MEM *buf = BUF_MEM_new(); - - if (tls13_process_compressed_certificate(sc, pkt, &tmppkt, buf) != MSG_PROCESS_ERROR) - ret = tls_process_client_certificate(sc, &tmppkt); - - BUF_MEM_free(buf); - return ret; -} -#endif - -CON_FUNC_RETURN tls_construct_server_certificate(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_server_certificate(SSL *s, WPACKET *pkt) { CERT_PKEY *cpk = s->s3.tmp.cert; if (cpk == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; + return 0; } /* * In TLSv1.3 the certificate chain is always preceded by a 0 length context * for the server Certificate message */ - if (SSL_CONNECTION_IS_TLS13(s) && !WPACKET_put_bytes_u8(pkt, 0)) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return CON_FUNC_ERROR; - } - switch (s->ext.server_cert_type) { - case TLSEXT_cert_type_rpk: - if (!tls_output_rpk(s, pkt, cpk)) { - /* SSLfatal() already called */ - return 0; - } - break; - case TLSEXT_cert_type_x509: - if (!ssl3_output_cert_chain(s, pkt, cpk, 0)) { - /* SSLfatal() already called */ - return 0; - } - break; - default: + if (SSL_IS_TLS13(s) && !WPACKET_put_bytes_u8(pkt, 0)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); return 0; } - - return CON_FUNC_SUCCESS; -} - -#ifndef OPENSSL_NO_COMP_ALG -CON_FUNC_RETURN tls_construct_server_compressed_certificate(SSL_CONNECTION *sc, WPACKET *pkt) -{ - int alg = get_compressed_certificate_alg(sc); - OSSL_COMP_CERT *cc = sc->s3.tmp.cert->comp_cert[alg]; - - if (!ossl_assert(cc != NULL)) { - SSLfatal(sc, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + if (!ssl3_output_cert_chain(s, pkt, cpk)) { + /* SSLfatal() already called */ return 0; } - /* - * Server can't compress on-demand - * Use pre-compressed certificate - */ - if (!WPACKET_put_bytes_u16(pkt, alg) - || !WPACKET_put_bytes_u24(pkt, cc->orig_len) - || !WPACKET_start_sub_packet_u24(pkt) - || !WPACKET_memcpy(pkt, cc->data, cc->len) - || !WPACKET_close(pkt)) - return 0; - sc->s3.tmp.cert->cert_comp_used++; return 1; } -#endif -static int create_ticket_prequel(SSL_CONNECTION *s, WPACKET *pkt, - uint32_t age_add, unsigned char *tick_nonce) +static int create_ticket_prequel(SSL *s, WPACKET *pkt, uint32_t age_add, + unsigned char *tick_nonce) { - uint32_t timeout = (uint32_t)ossl_time2seconds(s->session->timeout); + uint32_t timeout = (uint32_t)s->session->timeout; /* * Ticket lifetime hint: @@ -3884,9 +3706,8 @@ static int create_ticket_prequel(SSL_CONNECTION *s, WPACKET *pkt, */ #define ONE_WEEK_SEC (7 * 24 * 60 * 60) - if (SSL_CONNECTION_IS_TLS13(s)) { - if (ossl_time_compare(s->session->timeout, - ossl_seconds2time(ONE_WEEK_SEC)) > 0) + if (SSL_IS_TLS13(s)) { + if (s->session->timeout > ONE_WEEK_SEC) timeout = ONE_WEEK_SEC; } else if (s->hit) timeout = 0; @@ -3896,7 +3717,7 @@ static int create_ticket_prequel(SSL_CONNECTION *s, WPACKET *pkt, return 0; } - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { if (!WPACKET_put_bytes_u32(pkt, age_add) || !WPACKET_sub_memcpy_u8(pkt, tick_nonce, TICKET_NONCE_SIZE)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); @@ -3913,10 +3734,8 @@ static int create_ticket_prequel(SSL_CONNECTION *s, WPACKET *pkt, return 1; } -static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, - WPACKET *pkt, - uint32_t age_add, - unsigned char *tick_nonce) +static int construct_stateless_ticket(SSL *s, WPACKET *pkt, uint32_t age_add, + unsigned char *tick_nonce) { unsigned char *senc = NULL; EVP_CIPHER_CTX *ctx = NULL; @@ -3929,11 +3748,8 @@ static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, SSL_CTX *tctx = s->session_ctx; unsigned char iv[EVP_MAX_IV_LENGTH]; unsigned char key_name[TLSEXT_KEYNAME_LENGTH]; - int iv_len; - CON_FUNC_RETURN ok = CON_FUNC_ERROR; + int iv_len, ok = 0; size_t macoffset, macendoffset; - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* get session encoding length */ slen_full = i2d_SSL_SESSION(s->session, NULL); @@ -3947,18 +3763,14 @@ static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, } senc = OPENSSL_malloc(slen_full); if (senc == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - goto err; - } hctx = ssl_hmac_new(tctx); - if (hctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); + if (ctx == NULL || hctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -3972,8 +3784,7 @@ static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, * create a fresh copy (not shared with other threads) to clean up */ const_p = senc; - sess = d2i_SSL_SESSION_ex(NULL, &const_p, slen_full, sctx->libctx, - sctx->propq); + sess = d2i_SSL_SESSION(NULL, &const_p, slen_full); if (sess == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; @@ -4007,26 +3818,18 @@ static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, int ret = 0; if (tctx->ext.ticket_key_evp_cb != NULL) - ret = tctx->ext.ticket_key_evp_cb(ssl, key_name, iv, ctx, + ret = tctx->ext.ticket_key_evp_cb(s, key_name, iv, ctx, ssl_hmac_get0_EVP_MAC_CTX(hctx), 1); #ifndef OPENSSL_NO_DEPRECATED_3_0 else if (tctx->ext.ticket_key_cb != NULL) /* if 0 is returned, write an empty ticket */ - ret = tctx->ext.ticket_key_cb(ssl, key_name, iv, ctx, + ret = tctx->ext.ticket_key_cb(s, key_name, iv, ctx, ssl_hmac_get0_HMAC_CTX(hctx), 1); #endif if (ret == 0) { - /* - * In TLSv1.2 we construct a 0 length ticket. In TLSv1.3 a 0 - * length ticket is not allowed so we abort construction of the - * ticket - */ - if (SSL_CONNECTION_IS_TLS13(s)) { - ok = CON_FUNC_DONT_SEND; - goto err; - } + /* Put timeout and length */ if (!WPACKET_put_bytes_u32(pkt, 0) || !WPACKET_put_bytes_u16(pkt, 0)) { @@ -4036,20 +3839,16 @@ static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, OPENSSL_free(senc); EVP_CIPHER_CTX_free(ctx); ssl_hmac_free(hctx); - return CON_FUNC_SUCCESS; + return 1; } if (ret < 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_CALLBACK_FAILED); goto err; } iv_len = EVP_CIPHER_CTX_get_iv_length(ctx); - if (iv_len < 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } } else { - EVP_CIPHER *cipher = EVP_CIPHER_fetch(sctx->libctx, "AES-256-CBC", - sctx->propq); + EVP_CIPHER *cipher = EVP_CIPHER_fetch(s->ctx->libctx, "AES-256-CBC", + s->ctx->propq); if (cipher == NULL) { /* Error is already recorded */ @@ -4059,7 +3858,7 @@ static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, iv_len = EVP_CIPHER_get_iv_length(cipher); if (iv_len < 0 - || RAND_bytes_ex(sctx->libctx, iv, iv_len, 0) <= 0 + || RAND_bytes_ex(s->ctx->libctx, iv, iv_len, 0) <= 0 || !EVP_EncryptInit_ex(ctx, cipher, NULL, tctx->ext.secure->tick_aes_key, iv) || !ssl_hmac_init(hctx, tctx->ext.secure->tick_hmac_key, @@ -4113,7 +3912,7 @@ static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, goto err; } - ok = CON_FUNC_SUCCESS; + ok = 1; err: OPENSSL_free(senc); EVP_CIPHER_CTX_free(ctx); @@ -4121,8 +3920,7 @@ static CON_FUNC_RETURN construct_stateless_ticket(SSL_CONNECTION *s, return ok; } -static int construct_stateful_ticket(SSL_CONNECTION *s, WPACKET *pkt, - uint32_t age_add, +static int construct_stateful_ticket(SSL *s, WPACKET *pkt, uint32_t age_add, unsigned char *tick_nonce) { if (!create_ticket_prequel(s, pkt, age_add, tick_nonce)) { @@ -4140,21 +3938,7 @@ static int construct_stateful_ticket(SSL_CONNECTION *s, WPACKET *pkt, return 1; } -static void tls_update_ticket_counts(SSL_CONNECTION *s) -{ - /* - * Increment both |sent_tickets| and |next_ticket_nonce|. |sent_tickets| - * gets reset to 0 if we send more tickets following a post-handshake - * auth, but |next_ticket_nonce| does not. If we're sending extra - * tickets, decrement the count of pending extra tickets. - */ - s->sent_tickets++; - s->next_ticket_nonce++; - if (s->ext.extra_tickets_expected > 0) - s->ext.extra_tickets_expected--; -} - -CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_new_session_ticket(SSL *s, WPACKET *pkt) { SSL_CTX *tctx = s->session_ctx; unsigned char tick_nonce[TICKET_NONCE_SIZE]; @@ -4162,11 +3946,10 @@ CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt unsigned char age_add_c[sizeof(uint32_t)]; uint32_t age_add; } age_add_u; - CON_FUNC_RETURN ret = CON_FUNC_ERROR; age_add_u.age_add = 0; - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { size_t i, hashlen; uint64_t nonce; static const unsigned char nonce_label[] = "resumption"; @@ -4201,8 +3984,8 @@ CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt /* SSLfatal() already called */ goto err; } - if (RAND_bytes_ex(SSL_CONNECTION_GET_CTX(s)->libctx, - age_add_u.age_add_c, sizeof(age_add_u), 0) <= 0) { + if (RAND_bytes_ex(s->ctx->libctx, age_add_u.age_add_c, + sizeof(age_add_u), 0) <= 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } @@ -4226,7 +4009,7 @@ CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt } s->session->master_key_length = hashlen; - s->session->time = ossl_time_now(); + s->session->time = time(NULL); ssl_session_calculate_timeout(s->session); if (s->s3.alpn_selected != NULL) { OPENSSL_free(s->session->ext.alpn_selected); @@ -4234,7 +4017,7 @@ CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt OPENSSL_memdup(s->s3.alpn_selected, s->s3.alpn_selected_len); if (s->session->ext.alpn_selected == NULL) { s->session->ext.alpn_selected_len = 0; - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } s->session->ext.alpn_selected_len = s->s3.alpn_selected_len; @@ -4243,8 +4026,7 @@ CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt } if (tctx->generate_ticket_cb != NULL && - tctx->generate_ticket_cb(SSL_CONNECTION_GET_SSL(s), - tctx->ticket_cb_data) == 0) { + tctx->generate_ticket_cb(s, tctx->ticket_cb_data) == 0) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } @@ -4253,7 +4035,7 @@ CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt * SSL_OP_NO_TICKET is set - we are caching tickets anyway so there * is no point in using full stateless tickets. */ - if (SSL_CONNECTION_IS_TLS13(s) + if (SSL_IS_TLS13(s) && ((s->options & SSL_OP_NO_TICKET) != 0 || (s->max_early_data > 0 && (s->options & SSL_OP_NO_ANTI_REPLAY) == 0))) { @@ -4261,44 +4043,42 @@ CON_FUNC_RETURN tls_construct_new_session_ticket(SSL_CONNECTION *s, WPACKET *pkt /* SSLfatal() already called */ goto err; } - } else { - CON_FUNC_RETURN tmpret; - - tmpret = construct_stateless_ticket(s, pkt, age_add_u.age_add, - tick_nonce); - if (tmpret != CON_FUNC_SUCCESS) { - if (tmpret == CON_FUNC_DONT_SEND) { - /* Non-fatal. Abort construction but continue */ - ret = CON_FUNC_DONT_SEND; - /* We count this as a success so update the counts anwyay */ - tls_update_ticket_counts(s); - } - /* else SSLfatal() already called */ - goto err; - } + } else if (!construct_stateless_ticket(s, pkt, age_add_u.age_add, + tick_nonce)) { + /* SSLfatal() already called */ + goto err; } - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { if (!tls_construct_extensions(s, pkt, SSL_EXT_TLS1_3_NEW_SESSION_TICKET, NULL, 0)) { /* SSLfatal() already called */ goto err; } - tls_update_ticket_counts(s); + /* + * Increment both |sent_tickets| and |next_ticket_nonce|. |sent_tickets| + * gets reset to 0 if we send more tickets following a post-handshake + * auth, but |next_ticket_nonce| does not. If we're sending extra + * tickets, decrement the count of pending extra tickets. + */ + s->sent_tickets++; + s->next_ticket_nonce++; + if (s->ext.extra_tickets_expected > 0) + s->ext.extra_tickets_expected--; ssl_update_cache(s, SSL_SESS_CACHE_SERVER); } - ret = CON_FUNC_SUCCESS; + return 1; err: - return ret; + return 0; } /* * In TLSv1.3 this is called from the extensions code, otherwise it is used to * create a separate message. Returns 1 on success or 0 on failure. */ -int tls_construct_cert_status_body(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_cert_status_body(SSL *s, WPACKET *pkt) { if (!WPACKET_put_bytes_u8(pkt, s->ext.status_type) || !WPACKET_sub_memcpy_u24(pkt, s->ext.ocsp.resp, @@ -4310,14 +4090,14 @@ int tls_construct_cert_status_body(SSL_CONNECTION *s, WPACKET *pkt) return 1; } -CON_FUNC_RETURN tls_construct_cert_status(SSL_CONNECTION *s, WPACKET *pkt) +int tls_construct_cert_status(SSL *s, WPACKET *pkt) { if (!tls_construct_cert_status_body(s, pkt)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } - return CON_FUNC_SUCCESS; + return 1; } #ifndef OPENSSL_NO_NEXTPROTONEG @@ -4325,7 +4105,7 @@ CON_FUNC_RETURN tls_construct_cert_status(SSL_CONNECTION *s, WPACKET *pkt) * tls_process_next_proto reads a Next Protocol Negotiation handshake message. * It sets the next_proto member in s if found */ -MSG_PROCESS_RETURN tls_process_next_proto(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_next_proto(SSL *s, PACKET *pkt) { PACKET next_proto, padding; size_t next_proto_len; @@ -4356,19 +4136,18 @@ MSG_PROCESS_RETURN tls_process_next_proto(SSL_CONNECTION *s, PACKET *pkt) } #endif -static CON_FUNC_RETURN tls_construct_encrypted_extensions(SSL_CONNECTION *s, - WPACKET *pkt) +static int tls_construct_encrypted_extensions(SSL *s, WPACKET *pkt) { if (!tls_construct_extensions(s, pkt, SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS, NULL, 0)) { /* SSLfatal() already called */ - return CON_FUNC_ERROR; + return 0; } - return CON_FUNC_SUCCESS; + return 1; } -MSG_PROCESS_RETURN tls_process_end_of_early_data(SSL_CONNECTION *s, PACKET *pkt) +MSG_PROCESS_RETURN tls_process_end_of_early_data(SSL *s, PACKET *pkt) { if (PACKET_remaining(pkt) != 0) { SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); @@ -4391,7 +4170,7 @@ MSG_PROCESS_RETURN tls_process_end_of_early_data(SSL_CONNECTION *s, PACKET *pkt) } s->early_data_state = SSL_EARLY_DATA_FINISHED_READING; - if (!SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->change_cipher_state(s, + if (!s->method->ssl3_enc->change_cipher_state(s, SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_SERVER_READ)) { /* SSLfatal() already called */ return MSG_PROCESS_ERROR; @@ -4399,3 +4178,18 @@ MSG_PROCESS_RETURN tls_process_end_of_early_data(SSL_CONNECTION *s, PACKET *pkt) return MSG_PROCESS_CONTINUE_READING; } + +#ifndef OPENSSL_NO_CERT_COMPRESSION +int tls_construct_server_compressed_certificate(SSL *s, WPACKET *pkt) +{ + return tls_construct_compressed_certificate(s, pkt, + tls_construct_server_certificate); +} + +MSG_PROCESS_RETURN tls_process_client_compressed_certificate(SSL *s, + PACKET *pkt) +{ + return tls_process_compressed_certificate(s, pkt, + tls_process_client_certificate); +} +#endif diff --git a/openssl/src/ssl/statem_ntls/ntls_extensions.c b/openssl/src/ssl/statem_ntls/ntls_extensions.c new file mode 100644 index 000000000..75c3d76a1 --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_extensions.c @@ -0,0 +1,1423 @@ +/* + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + + +#include +#include "internal/nelem.h" +#include "internal/cryptlib.h" +#include "ntls_ssl_local.h" +#include "ntls_statem_local.h" +#include "internal/cryptlib.h" + +static int init_server_name(SSL *s, unsigned int context); +static int final_server_name(SSL *s, unsigned int context, int sent); +static int final_ec_pt_formats(SSL *s, unsigned int context, int sent); +static int init_session_ticket(SSL *s, unsigned int context); +#ifndef OPENSSL_NO_OCSP +static int init_status_request(SSL *s, unsigned int context); +#endif +#ifndef OPENSSL_NO_NEXTPROTONEG +static int init_npn(SSL *s, unsigned int context); +#endif +static int init_alpn(SSL *s, unsigned int context); +static int final_alpn(SSL *s, unsigned int context, int sent); +static int init_sig_algs_cert(SSL *s, unsigned int context); +static int init_sig_algs(SSL *s, unsigned int context); +static int init_certificate_authorities(SSL *s, unsigned int context); +static EXT_RETURN tls_construct_certificate_authorities(SSL *s, WPACKET *pkt, + unsigned int context, + X509 *x, + size_t chainidx); +static int tls_parse_certificate_authorities(SSL *s, PACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); + +static int init_ec_point_formats(SSL *s, unsigned int context); +static int init_etm(SSL *s, unsigned int context); +static int init_ems(SSL *s, unsigned int context); +static int final_ems(SSL *s, unsigned int context, int sent); +static int init_psk_kex_modes(SSL *s, unsigned int context); +static int final_key_share(SSL *s, unsigned int context, int sent); +#ifndef OPENSSL_NO_SRTP +static int init_srtp(SSL *s, unsigned int context); +#endif +static int final_sig_algs(SSL *s, unsigned int context, int sent); +static int final_early_data(SSL *s, unsigned int context, int sent); +static int final_maxfragmentlen(SSL *s, unsigned int context, int sent); +static int init_post_handshake_auth(SSL *s, unsigned int context); +static int final_psk(SSL *s, unsigned int context, int sent); + +/* Structure to define a built-in extension */ +typedef struct extensions_definition_st { + /* The defined type for the extension */ + unsigned int type; + /* + * The context that this extension applies to, e.g. what messages and + * protocol versions + */ + unsigned int context; + /* + * Initialise extension before parsing. Always called for relevant contexts + * even if extension not present + */ + int (*init)(SSL *s, unsigned int context); + /* Parse extension sent from client to server */ + int (*parse_ctos)(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); + /* Parse extension send from server to client */ + int (*parse_stoc)(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); + /* Construct extension sent from server to client */ + EXT_RETURN (*construct_stoc)(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); + /* Construct extension sent from client to server */ + EXT_RETURN (*construct_ctos)(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); + /* + * Finalise extension after parsing. Always called where an extensions was + * initialised even if the extension was not present. |sent| is set to 1 if + * the extension was seen, or 0 otherwise. + */ + int (*final)(SSL *s, unsigned int context, int sent); +} EXTENSION_DEFINITION; + +/* + * Definitions of all built-in extensions. NOTE: Changes in the number or order + * of these extensions should be mirrored with equivalent changes to the + * indexes ( TLSEXT_IDX_* ) defined in ssl_local.h. + * Each extension has an initialiser, a client and + * server side parser and a finaliser. The initialiser is called (if the + * extension is relevant to the given context) even if we did not see the + * extension in the message that we received. The parser functions are only + * called if we see the extension in the message. The finalisers are always + * called if the initialiser was called. + * There are also server and client side constructor functions which are always + * called during message construction if the extension is relevant for the + * given context. + * The initialisation, parsing, finalisation and construction functions are + * always called in the order defined in this list. Some extensions may depend + * on others having been processed first, so the order of this list is + * significant. + * The extension context is defined by a series of flags which specify which + * messages the extension is relevant to. These flags also specify whether the + * extension is relevant to a particular protocol or protocol version. + * + * TODO(TLS1.3): Make sure we have a test to check the consistency of these + * + * NOTE: WebSphere Application Server 7+ cannot handle empty extensions at + * the end, keep these extensions before signature_algorithm. + */ +#define INVALID_EXTENSION { 0x10000, 0, NULL, NULL, NULL, NULL, NULL, NULL } +static const EXTENSION_DEFINITION ext_defs[] = { + INVALID_EXTENSION, /* TLSEXT_IDX_renegotiate */ + { + TLSEXT_TYPE_server_name, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS, + init_server_name, + tls_parse_ctos_server_name_ntls, tls_parse_stoc_server_name_ntls, + tls_construct_stoc_server_name_ntls, tls_construct_ctos_server_name_ntls, + final_server_name + }, + { + TLSEXT_TYPE_max_fragment_length, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS, + NULL, tls_parse_ctos_maxfragmentlen_ntls, tls_parse_stoc_maxfragmentlen_ntls, + tls_construct_stoc_maxfragmentlen_ntls, tls_construct_ctos_maxfragmentlen_ntls, + final_maxfragmentlen + }, + INVALID_EXTENSION, /* TLSEXT_IDX_srp */ + { + TLSEXT_TYPE_ec_point_formats, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_2_AND_BELOW_ONLY, + init_ec_point_formats, tls_parse_ctos_ec_pt_formats_ntls, tls_parse_stoc_ec_pt_formats_ntls, + tls_construct_stoc_ec_pt_formats_ntls, tls_construct_ctos_ec_pt_formats_ntls, + final_ec_pt_formats + }, + { + /* + * "supported_groups" is spread across several specifications. + * It was originally specified as "elliptic_curves" in RFC 4492, + * and broadened to include named FFDH groups by RFC 7919. + * Both RFCs 4492 and 7919 do not include a provision for the server + * to indicate to the client the complete list of groups supported + * by the server, with the server instead just indicating the + * selected group for this connection in the ServerKeyExchange + * message. TLS 1.3 adds a scheme for the server to indicate + * to the client its list of supported groups in the + * EncryptedExtensions message, but none of the relevant + * specifications permit sending supported_groups in the ServerHello. + * Nonetheless (possibly due to the close proximity to the + * "ec_point_formats" extension, which is allowed in the ServerHello), + * there are several servers that send this extension in the + * ServerHello anyway. Up to and including the 1.1.0 release, + * we did not check for the presence of nonpermitted extensions, + * so to avoid a regression, we must permit this extension in the + * TLS 1.2 ServerHello as well. + * + * Note that there is no tls_parse_stoc_supported_groups function, + * so we do not perform any additional parsing, validation, or + * processing on the server's group list -- this is just a minimal + * change to preserve compatibility with these misbehaving servers. + */ + TLSEXT_TYPE_supported_groups, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS + | SSL_EXT_TLS1_2_SERVER_HELLO, + NULL, tls_parse_ctos_supported_groups_ntls, NULL, + tls_construct_stoc_supported_groups_ntls, + tls_construct_ctos_supported_groups_ntls, NULL + }, + { + TLSEXT_TYPE_session_ticket, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_2_AND_BELOW_ONLY, + init_session_ticket, tls_parse_ctos_session_ticket_ntls, + tls_parse_stoc_session_ticket_ntls, tls_construct_stoc_session_ticket_ntls, + tls_construct_ctos_session_ticket_ntls, NULL + }, +#ifndef OPENSSL_NO_OCSP + { + TLSEXT_TYPE_status_request, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_CERTIFICATE | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST, + init_status_request, tls_parse_ctos_status_request_ntls, + tls_parse_stoc_status_request_ntls, tls_construct_stoc_status_request_ntls, + tls_construct_ctos_status_request_ntls, NULL + }, +#else + INVALID_EXTENSION, +#endif +#ifndef OPENSSL_NO_NEXTPROTONEG + { + TLSEXT_TYPE_next_proto_neg, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_2_AND_BELOW_ONLY, + init_npn, tls_parse_ctos_npn_ntls, tls_parse_stoc_npn_ntls, + tls_construct_stoc_next_proto_neg_ntls, tls_construct_ctos_npn_ntls, NULL + }, +#else + INVALID_EXTENSION, +#endif + { + /* + * Must appear in this list after server_name so that finalisation + * happens after server_name callbacks + */ + TLSEXT_TYPE_application_layer_protocol_negotiation, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS, + init_alpn, tls_parse_ctos_alpn_ntls, tls_parse_stoc_alpn_ntls, + tls_construct_stoc_alpn_ntls, tls_construct_ctos_alpn_ntls, final_alpn + }, +#ifndef OPENSSL_NO_SRTP + { + TLSEXT_TYPE_use_srtp, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS | SSL_EXT_DTLS_ONLY, + init_srtp, tls_parse_ctos_use_srtp_ntls, tls_parse_stoc_use_srtp_ntls, + tls_construct_stoc_use_srtp_ntls, tls_construct_ctos_use_srtp_ntls, NULL + }, +#else + INVALID_EXTENSION, +#endif + { + TLSEXT_TYPE_encrypt_then_mac, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_2_AND_BELOW_ONLY, + init_etm, tls_parse_ctos_etm_ntls, tls_parse_stoc_etm_ntls, + tls_construct_stoc_etm_ntls, tls_construct_ctos_etm_ntls, NULL + }, +#ifndef OPENSSL_NO_CT + { + TLSEXT_TYPE_signed_certificate_timestamp, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_CERTIFICATE | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST, + NULL, + /* + * No server side support for this, but can be provided by a custom + * extension. This is an exception to the rule that custom extensions + * cannot override built in ones. + */ + NULL, tls_parse_stoc_sct_ntls, NULL, tls_construct_ctos_sct_ntls, NULL + }, +#else + INVALID_EXTENSION, +#endif + INVALID_EXTENSION, /* TLSEXT_IDX_delegated_credential */ + { + TLSEXT_TYPE_extended_master_secret, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_2_AND_BELOW_ONLY, + init_ems, tls_parse_ctos_ems_ntls, tls_parse_stoc_ems_ntls, + tls_construct_stoc_ems_ntls, tls_construct_ctos_ems_ntls, final_ems + }, + { + TLSEXT_TYPE_signature_algorithms_cert, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST, + init_sig_algs_cert, tls_parse_ctos_sig_algs_cert_ntls, + tls_parse_ctos_sig_algs_cert_ntls, + /* We do not generate signature_algorithms_cert at present. */ + NULL, NULL, NULL + }, + { + TLSEXT_TYPE_post_handshake_auth, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ONLY, + init_post_handshake_auth, + tls_parse_ctos_post_handshake_auth_ntls, NULL, + NULL, tls_construct_ctos_post_handshake_auth_ntls, + NULL, + }, + { + TLSEXT_TYPE_signature_algorithms, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST, + init_sig_algs, tls_parse_ctos_sig_algs_ntls, + tls_parse_ctos_sig_algs_ntls, tls_construct_ctos_sig_algs_ntls, + tls_construct_ctos_sig_algs_ntls, final_sig_algs + }, + { + TLSEXT_TYPE_supported_versions, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_SERVER_HELLO + | SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST | SSL_EXT_TLS_IMPLEMENTATION_ONLY, + NULL, + /* Processed inline as part of version selection */ + NULL, tls_parse_stoc_supported_versions_ntls, + tls_construct_stoc_supported_versions_ntls, + tls_construct_ctos_supported_versions_ntls, NULL + }, + { + TLSEXT_TYPE_psk_kex_modes, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS_IMPLEMENTATION_ONLY + | SSL_EXT_TLS1_3_ONLY, + init_psk_kex_modes, tls_parse_ctos_psk_kex_modes_ntls, NULL, NULL, + tls_construct_ctos_psk_kex_modes_ntls, NULL + }, + { + /* + * Must be in this list after supported_groups. We need that to have + * been parsed before we do this one. + */ + TLSEXT_TYPE_key_share, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_SERVER_HELLO + | SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST | SSL_EXT_TLS_IMPLEMENTATION_ONLY + | SSL_EXT_TLS1_3_ONLY, + NULL, tls_parse_ctos_key_share_ntls, tls_parse_stoc_key_share_ntls, + tls_construct_stoc_key_share_ntls, tls_construct_ctos_key_share_ntls, + final_key_share + }, + { + /* Must be after key_share */ + TLSEXT_TYPE_cookie, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST + | SSL_EXT_TLS_IMPLEMENTATION_ONLY | SSL_EXT_TLS1_3_ONLY, + NULL, tls_parse_ctos_cookie_ntls, tls_parse_stoc_cookie_ntls, + tls_construct_stoc_cookie_ntls, tls_construct_ctos_cookie_ntls, NULL + }, + { + TLSEXT_TYPE_early_data, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS + | SSL_EXT_TLS1_3_NEW_SESSION_TICKET | SSL_EXT_TLS1_3_ONLY, + NULL, tls_parse_ctos_early_data_ntls, tls_parse_stoc_early_data_ntls, + tls_construct_stoc_early_data_ntls, tls_construct_ctos_early_data_ntls, + final_early_data + }, + { + TLSEXT_TYPE_certificate_authorities, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST + | SSL_EXT_TLS1_3_ONLY, + init_certificate_authorities, + tls_parse_certificate_authorities, tls_parse_certificate_authorities, + tls_construct_certificate_authorities, + tls_construct_certificate_authorities, NULL, + }, + INVALID_EXTENSION, /* TLSEXT_TYPE_quic_transport_parameters_draft */ + INVALID_EXTENSION, /* TLSEXT_TYPE_quic_transport_parameters */ + INVALID_EXTENSION, /* TLSEXT_TYPE_compress_certificate */ + { + /* Must be immediately before pre_shared_key */ + TLSEXT_TYPE_padding, + SSL_EXT_CLIENT_HELLO, + NULL, + /* We send this, but don't read it */ + NULL, NULL, NULL, tls_construct_ctos_padding_ntls, NULL + }, + { + /* Required by the TLSv1.3 spec to always be the last extension */ + TLSEXT_TYPE_psk, + SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_SERVER_HELLO + | SSL_EXT_TLS_IMPLEMENTATION_ONLY | SSL_EXT_TLS1_3_ONLY, + NULL, tls_parse_ctos_psk_ntls, tls_parse_stoc_psk_ntls, tls_construct_stoc_psk_ntls, + tls_construct_ctos_psk_ntls, final_psk + } +}; + +/* Check whether an extension's context matches the current context */ +static int validate_context(SSL *s, unsigned int extctx, unsigned int thisctx) +{ + /* Check we're allowed to use this extension in this context */ + if ((thisctx & extctx) == 0) + return 0; + + if (SSL_IS_DTLS(s)) { + if ((extctx & SSL_EXT_TLS_ONLY) != 0) + return 0; + } else if ((extctx & SSL_EXT_DTLS_ONLY) != 0) { + return 0; + } + + return 1; +} + +int tls_validate_all_contexts_ntls(SSL *s, unsigned int thisctx, RAW_EXTENSION *exts) +{ + size_t i, num_exts, builtin_num = OSSL_NELEM(ext_defs), offset; + RAW_EXTENSION *thisext; + unsigned int context; + ENDPOINT role = ENDPOINT_BOTH; + + if ((thisctx & SSL_EXT_CLIENT_HELLO) != 0) + role = ENDPOINT_SERVER; + else if ((thisctx & SSL_EXT_TLS1_2_SERVER_HELLO) != 0) + role = ENDPOINT_CLIENT; + + /* Calculate the number of extensions in the extensions list */ + num_exts = builtin_num + s->cert->custext.meths_count; + + for (thisext = exts, i = 0; i < num_exts; i++, thisext++) { + if (!thisext->present) + continue; + + if (i < builtin_num) { + context = ext_defs[i].context; + } else { + custom_ext_method *meth = NULL; + + meth = custom_ext_find_ntls(&s->cert->custext, role, thisext->type, + &offset); + if (!ossl_assert(meth != NULL)) + return 0; + context = meth->context; + } + + if (!validate_context(s, context, thisctx)) + return 0; + } + + return 1; +} + +/* + * Verify whether we are allowed to use the extension |type| in the current + * |context|. Returns 1 to indicate the extension is allowed or unknown or 0 to + * indicate the extension is not allowed. If returning 1 then |*found| is set to + * the definition for the extension we found. + */ +static int verify_extension(SSL *s, unsigned int context, unsigned int type, + custom_ext_methods *meths, RAW_EXTENSION *rawexlist, + RAW_EXTENSION **found) +{ + size_t i; + size_t builtin_num = OSSL_NELEM(ext_defs); + const EXTENSION_DEFINITION *thisext; + + for (i = 0, thisext = ext_defs; i < builtin_num; i++, thisext++) { + if (type == thisext->type) { + if (!validate_context(s, thisext->context, context)) + return 0; + + *found = &rawexlist[i]; + return 1; + } + } + + /* Check the custom extensions */ + if (meths != NULL) { + size_t offset = 0; + ENDPOINT role = ENDPOINT_BOTH; + custom_ext_method *meth = NULL; + + if ((context & SSL_EXT_CLIENT_HELLO) != 0) + role = ENDPOINT_SERVER; + else if ((context & SSL_EXT_TLS1_2_SERVER_HELLO) != 0) + role = ENDPOINT_CLIENT; + + meth = custom_ext_find_ntls(meths, role, type, &offset); + if (meth != NULL) { + if (!validate_context(s, meth->context, context)) + return 0; + *found = &rawexlist[offset + builtin_num]; + return 1; + } + } + + /* Unknown extension. We allow it */ + *found = NULL; + return 1; +} + +/* + * Check whether the context defined for an extension |extctx| means whether + * the extension is relevant for the current context |thisctx| or not. Returns + * 1 if the extension is relevant for this context, and 0 otherwise + */ +int extension_is_relevant_ntls(SSL *s, unsigned int extctx, unsigned int thisctx) +{ + int is_tls13; + + /* + * For HRR we haven't selected the version yet but we know it will be + * TLSv1.3 + */ + if ((thisctx & SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST) != 0) + is_tls13 = 1; + else + is_tls13 = SSL_IS_TLS13(s); + + if ((SSL_IS_DTLS(s) + && (extctx & SSL_EXT_TLS_IMPLEMENTATION_ONLY) != 0) + || (s->version == SSL3_VERSION + && (extctx & SSL_EXT_SSL3_ALLOWED) == 0) + /* + * Note that SSL_IS_TLS13() means "TLS 1.3 has been negotiated", + * which is never true when generating the ClientHello. + * However, version negotiation *has* occurred by the time the + * ClientHello extensions are being parsed. + * Be careful to allow TLS 1.3-only extensions when generating + * the ClientHello. + */ + || (is_tls13 && (extctx & SSL_EXT_TLS1_2_AND_BELOW_ONLY) != 0) + || (!is_tls13 && (extctx & SSL_EXT_TLS1_3_ONLY) != 0 + && (thisctx & SSL_EXT_CLIENT_HELLO) == 0) + || (s->server && !is_tls13 && (extctx & SSL_EXT_TLS1_3_ONLY) != 0) + || (s->hit && (extctx & SSL_EXT_IGNORE_ON_RESUMPTION) != 0)) + return 0; + return 1; +} + +/* + * Gather a list of all the extensions from the data in |packet]. |context| + * tells us which message this extension is for. The raw extension data is + * stored in |*res| on success. We don't actually process the content of the + * extensions yet, except to check their types. This function also runs the + * initialiser functions for all known extensions if |init| is nonzero (whether + * we have collected them or not). If successful the caller is responsible for + * freeing the contents of |*res|. + * + * Per http://tools.ietf.org/html/rfc5246#section-7.4.1.4, there may not be + * more than one extension of the same type in a ClientHello or ServerHello. + * This function returns 1 if all extensions are unique and we have parsed their + * types, and 0 if the extensions contain duplicates, could not be successfully + * found, or an internal error occurred. We only check duplicates for + * extensions that we know about. We ignore others. + */ +int tls_collect_extensions_ntls(SSL *s, PACKET *packet, unsigned int context, + RAW_EXTENSION **res, size_t *len, int init) +{ + PACKET extensions = *packet; + size_t i = 0; + size_t num_exts; + custom_ext_methods *exts = &s->cert->custext; + RAW_EXTENSION *raw_extensions = NULL; + const EXTENSION_DEFINITION *thisexd; + + *res = NULL; + + /* + * Initialise server side custom extensions. Client side is done during + * construction of extensions for the ClientHello. + */ + if ((context & SSL_EXT_CLIENT_HELLO) != 0) + custom_ext_init_ntls(&s->cert->custext); + + num_exts = OSSL_NELEM(ext_defs) + (exts != NULL ? exts->meths_count : 0); + raw_extensions = OPENSSL_zalloc(num_exts * sizeof(*raw_extensions)); + if (raw_extensions == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + + i = 0; + while (PACKET_remaining(&extensions) > 0) { + unsigned int type, idx; + PACKET extension; + RAW_EXTENSION *thisex; + + if (!PACKET_get_net_2(&extensions, &type) || + !PACKET_get_length_prefixed_2(&extensions, &extension)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + goto err; + } + /* + * Verify this extension is allowed. We only check duplicates for + * extensions that we recognise. We also have a special case for the + * PSK extension, which must be the last one in the ClientHello. + */ + if (!verify_extension(s, context, type, exts, raw_extensions, &thisex) + || (thisex != NULL && thisex->present == 1) + || (type == TLSEXT_TYPE_psk + && (context & SSL_EXT_CLIENT_HELLO) != 0 + && PACKET_remaining(&extensions) != 0)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_EXTENSION); + goto err; + } + idx = thisex - raw_extensions; + /*- + * Check that we requested this extension (if appropriate). Requests can + * be sent in the ClientHello and CertificateRequest. Unsolicited + * extensions can be sent in the NewSessionTicket. We only do this for + * the built-in extensions. Custom extensions have a different but + * similar check elsewhere. + * Special cases: + * - The HRR cookie extension is unsolicited + * - The renegotiate extension is unsolicited (the client signals + * support via an SCSV) + * - The signed_certificate_timestamp extension can be provided by a + * custom extension or by the built-in version. We let the extension + * itself handle unsolicited response checks. + */ + if (idx < OSSL_NELEM(ext_defs) + && (context & (SSL_EXT_CLIENT_HELLO + | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST + | SSL_EXT_TLS1_3_NEW_SESSION_TICKET)) == 0 + && type != TLSEXT_TYPE_cookie + && type != TLSEXT_TYPE_renegotiate + && type != TLSEXT_TYPE_signed_certificate_timestamp + && (s->ext.extflags[idx] & SSL_EXT_FLAG_SENT) == 0 + ) { + SSLfatal_ntls(s, SSL_AD_UNSUPPORTED_EXTENSION, + SSL_R_UNSOLICITED_EXTENSION); + goto err; + } + if (thisex != NULL) { + thisex->data = extension; + thisex->present = 1; + thisex->type = type; + thisex->received_order = i++; + if (s->ext.debug_cb) + s->ext.debug_cb(s, !s->server, thisex->type, + PACKET_data(&thisex->data), + PACKET_remaining(&thisex->data), + s->ext.debug_arg); + } + } + + if (init) { + /* + * Initialise all known extensions relevant to this context, + * whether we have found them or not + */ + for (thisexd = ext_defs, i = 0; i < OSSL_NELEM(ext_defs); + i++, thisexd++) { + if (thisexd->init != NULL && (thisexd->context & context) != 0 + && extension_is_relevant_ntls(s, thisexd->context, context) + && !thisexd->init(s, context)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } + } + + *res = raw_extensions; + if (len != NULL) + *len = num_exts; + return 1; + + err: + OPENSSL_free(raw_extensions); + return 0; +} + +/* + * Runs the parser for a given extension with index |idx|. |exts| contains the + * list of all parsed extensions previously collected by + * tls_collect_extensions_ntls(). The parser is only run if it is applicable for the + * given |context| and the parser has not already been run. If this is for a + * Certificate message, then we also provide the parser with the relevant + * Certificate |x| and its position in the |chainidx| with 0 being the first + * Certificate. Returns 1 on success or 0 on failure. If an extension is not + * present this counted as success. + */ +int tls_parse_extension_ntls(SSL *s, TLSEXT_INDEX idx, int context, + RAW_EXTENSION *exts, X509 *x, size_t chainidx) +{ + RAW_EXTENSION *currext = &exts[idx]; + int (*parser)(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) = NULL; + + /* Skip if the extension is not present */ + if (!currext->present) + return 1; + + /* Skip if we've already parsed this extension */ + if (currext->parsed) + return 1; + + currext->parsed = 1; + + if (idx < OSSL_NELEM(ext_defs)) { + /* We are handling a built-in extension */ + const EXTENSION_DEFINITION *extdef = &ext_defs[idx]; + + /* Check if extension is defined for our protocol. If not, skip */ + if (!extension_is_relevant_ntls(s, extdef->context, context)) + return 1; + + parser = s->server ? extdef->parse_ctos : extdef->parse_stoc; + + if (parser != NULL) + return parser(s, &currext->data, context, x, chainidx); + + /* + * If the parser is NULL we fall through to the custom extension + * processing + */ + } + + /* Parse custom extensions */ + return custom_ext_parse_ntls(s, context, currext->type, + PACKET_data(&currext->data), + PACKET_remaining(&currext->data), + x, chainidx); +} + +/* + * Parse all remaining extensions that have not yet been parsed. Also calls the + * finalisation for all extensions at the end if |fin| is nonzero, whether we + * collected them or not. Returns 1 for success or 0 for failure. If we are + * working on a Certificate message then we also pass the Certificate |x| and + * its position in the |chainidx|, with 0 being the first certificate. + */ +int tls_parse_all_extensions_ntls(SSL *s, int context, RAW_EXTENSION *exts, X509 *x, + size_t chainidx, int fin) +{ + size_t i, numexts = OSSL_NELEM(ext_defs); + const EXTENSION_DEFINITION *thisexd; + + /* Calculate the number of extensions in the extensions list */ + numexts += s->cert->custext.meths_count; + + /* Parse each extension in turn */ + for (i = 0; i < numexts; i++) { + if (!tls_parse_extension_ntls(s, i, context, exts, x, chainidx)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + + if (fin) { + /* + * Finalise all known extensions relevant to this context, + * whether we have found them or not + */ + for (i = 0, thisexd = ext_defs; i < OSSL_NELEM(ext_defs); + i++, thisexd++) { + if (thisexd->final != NULL && (thisexd->context & context) != 0 + && !thisexd->final(s, context, exts[i].present)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + } + + return 1; +} + +int should_add_extension_ntls(SSL *s, unsigned int extctx, unsigned int thisctx, + int max_version) +{ + /* Skip if not relevant for our context */ + if ((extctx & thisctx) == 0) + return 0; + + /* Check if this extension is defined for our protocol. If not, skip */ + if (!extension_is_relevant_ntls(s, extctx, thisctx) + || ((extctx & SSL_EXT_TLS1_3_ONLY) != 0 + && (thisctx & SSL_EXT_CLIENT_HELLO) != 0 + && (SSL_IS_DTLS(s) || max_version < TLS1_3_VERSION))) + return 0; + + return 1; +} + +/* + * Construct all the extensions relevant to the current |context| and write + * them to |pkt|. If this is an extension for a Certificate in a Certificate + * message, then |x| will be set to the Certificate we are handling, and + * |chainidx| will indicate the position in the chainidx we are processing (with + * 0 being the first in the chain). Returns 1 on success or 0 on failure. On a + * failure construction stops at the first extension to fail to construct. + */ +int tls_construct_extensions_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + size_t i; + int min_version, max_version = 0, reason; + const EXTENSION_DEFINITION *thisexd; + + if (!WPACKET_start_sub_packet_u16(pkt) + /* + * If extensions are of zero length then we don't even add the + * extensions length bytes to a ClientHello/ServerHello + * (for non-TLSv1.3). + */ + || ((context & + (SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO)) != 0 + && !WPACKET_set_flags(pkt, + WPACKET_FLAGS_ABANDON_ON_ZERO_LENGTH))) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if ((context & SSL_EXT_CLIENT_HELLO) != 0) { + reason = ssl_get_min_max_version_ntls(s, &min_version, &max_version, NULL); + if (reason != 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, reason); + return 0; + } + } + + /* Add custom extensions first */ + if ((context & SSL_EXT_CLIENT_HELLO) != 0) { + /* On the server side with initialise during ClientHello parsing */ + custom_ext_init_ntls(&s->cert->custext); + } + if (!custom_ext_add_ntls(s, context, pkt, x, chainidx, max_version)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + for (i = 0, thisexd = ext_defs; i < OSSL_NELEM(ext_defs); i++, thisexd++) { + EXT_RETURN (*construct)(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); + EXT_RETURN ret; + + /* Skip if not relevant for our context */ + if (!should_add_extension_ntls(s, thisexd->context, context, max_version)) + continue; + + construct = s->server ? thisexd->construct_stoc + : thisexd->construct_ctos; + + if (construct == NULL) + continue; + + ret = construct(s, pkt, context, x, chainidx); + if (ret == EXT_RETURN_FAIL) { + /* SSLfatal_ntls() already called */ + return 0; + } + if (ret == EXT_RETURN_SENT + && (context & (SSL_EXT_CLIENT_HELLO + | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST + | SSL_EXT_TLS1_3_NEW_SESSION_TICKET)) != 0) + s->ext.extflags[i] |= SSL_EXT_FLAG_SENT; + } + + if (!WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +/* + * Built in extension finalisation and initialisation functions. All initialise + * or finalise the associated extension type for the given |context|. For + * finalisers |sent| is set to 1 if we saw the extension during parsing, and 0 + * otherwise. These functions return 1 on success or 0 on failure. + */ + +static ossl_inline void ssl_tsan_decr(const SSL_CTX *ctx, + TSAN_QUALIFIER int *stat) +{ + if (ssl_tsan_lock(ctx)) { + tsan_decr(stat); + ssl_tsan_unlock(ctx); + } +} + +static int init_server_name(SSL *s, unsigned int context) +{ + if (s->server) { + s->servername_done = 0; + + OPENSSL_free(s->ext.hostname); + s->ext.hostname = NULL; + } + + return 1; +} + +static int final_server_name(SSL *s, unsigned int context, int sent) +{ + int ret = SSL_TLSEXT_ERR_NOACK; + int altmp = SSL_AD_UNRECOGNIZED_NAME; + int was_ticket = (SSL_get_options(s) & SSL_OP_NO_TICKET) == 0; + + if (!ossl_assert(s->ctx != NULL) || !ossl_assert(s->session_ctx != NULL)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (s->ctx->ext.servername_cb != NULL) + ret = s->ctx->ext.servername_cb(s, &altmp, + s->ctx->ext.servername_arg); + else if (s->session_ctx->ext.servername_cb != NULL) + ret = s->session_ctx->ext.servername_cb(s, &altmp, + s->session_ctx->ext.servername_arg); + + /* + * For servers, propagate the SNI hostname from the temporary + * storage in the SSL to the persistent SSL_SESSION, now that we + * know we accepted it. + * Clients make this copy when parsing the server's response to + * the extension, which is when they find out that the negotiation + * was successful. + */ + if (s->server) { + if (sent && ret == SSL_TLSEXT_ERR_OK && !s->hit) { + /* Only store the hostname in the session if we accepted it. */ + OPENSSL_free(s->session->ext.hostname); + s->session->ext.hostname = OPENSSL_strdup(s->ext.hostname); + if (s->session->ext.hostname == NULL && s->ext.hostname != NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + } + } + } + + /* + * If we switched contexts (whether here or in the client_hello callback), + * move the sess_accept increment from the session_ctx to the new + * context, to avoid the confusing situation of having sess_accept_good + * exceed sess_accept (zero) for the new context. + */ + if (SSL_IS_FIRST_HANDSHAKE(s) && s->ctx != s->session_ctx + && s->hello_retry_request == SSL_HRR_NONE) { + ssl_tsan_counter(s->ctx, &s->ctx->stats.sess_accept); + ssl_tsan_decr(s->session_ctx, &s->session_ctx->stats.sess_accept); + } + + /* + * If we're expecting to send a ticket, and tickets were previously enabled, + * and now tickets are disabled, then turn off expected ticket. + * Also, if this is not a resumption, create a new session ID + */ + if (ret == SSL_TLSEXT_ERR_OK && s->ext.ticket_expected + && was_ticket && (SSL_get_options(s) & SSL_OP_NO_TICKET) != 0) { + s->ext.ticket_expected = 0; + if (!s->hit) { + SSL_SESSION* ss = SSL_get_session(s); + + if (ss != NULL) { + OPENSSL_free(ss->ext.tick); + ss->ext.tick = NULL; + ss->ext.ticklen = 0; + ss->ext.tick_lifetime_hint = 0; + ss->ext.tick_age_add = 0; + if (!ssl_generate_session_id(s, ss)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } else { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + } + + switch (ret) { + case SSL_TLSEXT_ERR_ALERT_FATAL: + SSLfatal_ntls(s, altmp, SSL_R_CALLBACK_FAILED); + return 0; + + case SSL_TLSEXT_ERR_ALERT_WARNING: + ssl3_send_alert(s, SSL3_AL_WARNING, altmp); + s->servername_done = 0; + return 1; + + case SSL_TLSEXT_ERR_NOACK: + s->servername_done = 0; + return 1; + + default: + return 1; + } +} + +static int final_ec_pt_formats(SSL *s, unsigned int context, int sent) +{ + /* Ignore ec_point_formats */ + return 1; +} + +static int init_session_ticket(SSL *s, unsigned int context) +{ + if (!s->server) + s->ext.ticket_expected = 0; + + return 1; +} + +#ifndef OPENSSL_NO_OCSP +static int init_status_request(SSL *s, unsigned int context) +{ + if (s->server) { + s->ext.status_type = TLSEXT_STATUSTYPE_nothing; + } else { + /* + * Ensure we get sensible values passed to tlsext_status_cb in the event + * that we don't receive a status message + */ + OPENSSL_free(s->ext.ocsp.resp); + s->ext.ocsp.resp = NULL; + s->ext.ocsp.resp_len = 0; + } + + return 1; +} +#endif + +#ifndef OPENSSL_NO_NEXTPROTONEG +static int init_npn(SSL *s, unsigned int context) +{ + s->s3.npn_seen = 0; + + return 1; +} +#endif + +static int init_alpn(SSL *s, unsigned int context) +{ + OPENSSL_free(s->s3.alpn_selected); + s->s3.alpn_selected = NULL; + s->s3.alpn_selected_len = 0; + if (s->server) { + OPENSSL_free(s->s3.alpn_proposed); + s->s3.alpn_proposed = NULL; + s->s3.alpn_proposed_len = 0; + } + return 1; +} + +static int final_alpn(SSL *s, unsigned int context, int sent) +{ + if (!s->server && !sent && s->session->ext.alpn_selected != NULL) + s->ext.early_data_ok = 0; + return 1; + + /* + * Call alpn_select callback if needed. Has to be done after SNI and + * cipher negotiation (HTTP/2 restricts permitted ciphers). In TLSv1.3 + * we also have to do this before we decide whether to accept early_data. + * In TLSv1.3 we've already negotiated our cipher so we do this call now. + * For < TLSv1.3 we defer it until after cipher negotiation. + * + * On failure SSLfatal_ntls() already called. + */ + return tls_handle_alpn_ntls(s); +} + +static int init_sig_algs(SSL *s, unsigned int context) +{ + /* Clear any signature algorithms extension received */ + OPENSSL_free(s->s3.tmp.peer_sigalgs); + s->s3.tmp.peer_sigalgs = NULL; + s->s3.tmp.peer_sigalgslen = 0; + + return 1; +} + +static int init_sig_algs_cert(SSL *s, ossl_unused unsigned int context) +{ + /* Clear any signature algorithms extension received */ + OPENSSL_free(s->s3.tmp.peer_cert_sigalgs); + s->s3.tmp.peer_cert_sigalgs = NULL; + s->s3.tmp.peer_cert_sigalgslen = 0; + + return 1; +} + +static int init_ec_point_formats(SSL *s, unsigned int context) +{ + OPENSSL_free(s->ext.peer_ecpointformats); + s->ext.peer_ecpointformats = NULL; + s->ext.peer_ecpointformats_len = 0; + + return 1; +} + +static int init_etm(SSL *s, unsigned int context) +{ + s->ext.use_etm = 0; + + return 1; +} + +static int init_ems(SSL *s, unsigned int context) +{ + if (s->s3.flags & TLS1_FLAGS_RECEIVED_EXTMS) { + s->s3.flags &= ~TLS1_FLAGS_RECEIVED_EXTMS; + s->s3.flags |= TLS1_FLAGS_REQUIRED_EXTMS; + } + + return 1; +} + +static int final_ems(SSL *s, unsigned int context, int sent) +{ + /* Ignore extended_master_secret */ + return 1; +} + +static int init_certificate_authorities(SSL *s, unsigned int context) +{ + sk_X509_NAME_pop_free(s->s3.tmp.peer_ca_names, X509_NAME_free); + s->s3.tmp.peer_ca_names = NULL; + return 1; +} + +static EXT_RETURN tls_construct_certificate_authorities(SSL *s, WPACKET *pkt, + unsigned int context, + X509 *x, + size_t chainidx) +{ + const STACK_OF(X509_NAME) *ca_sk = get_ca_names_ntls(s); + + if (ca_sk == NULL || sk_X509_NAME_num(ca_sk) == 0) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_certificate_authorities) + || !WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + if (!construct_ca_names_ntls(s, ca_sk, pkt)) { + /* SSLfatal_ntls() already called */ + return EXT_RETURN_FAIL; + } + + if (!WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +static int tls_parse_certificate_authorities(SSL *s, PACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (!parse_ca_names_ntls(s, pkt)) + return 0; + if (PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + return 1; +} + +#ifndef OPENSSL_NO_SRTP +static int init_srtp(SSL *s, unsigned int context) +{ + if (s->server) + s->srtp_profile = NULL; + + return 1; +} +#endif + +static int final_sig_algs(SSL *s, unsigned int context, int sent) +{ + return 1; +} + +static int final_key_share(SSL *s, unsigned int context, int sent) +{ + return 1; +} + +static int init_psk_kex_modes(SSL *s, unsigned int context) +{ + s->ext.psk_kex_mode = TLSEXT_KEX_MODE_FLAG_NONE; + return 1; +} + +int tls_psk_do_binder_ntls(SSL *s, const EVP_MD *md, const unsigned char *msgstart, + size_t binderoffset, const unsigned char *binderin, + unsigned char *binderout, SSL_SESSION *sess, int sign, + int external) +{ + EVP_PKEY *mackey = NULL; + EVP_MD_CTX *mctx = NULL; + unsigned char hash[EVP_MAX_MD_SIZE], binderkey[EVP_MAX_MD_SIZE]; + unsigned char finishedkey[EVP_MAX_MD_SIZE], tmpbinder[EVP_MAX_MD_SIZE]; + unsigned char *early_secret; +#ifdef CHARSET_EBCDIC + static const unsigned char resumption_label[] = { 0x72, 0x65, 0x73, 0x20, 0x62, 0x69, 0x6E, 0x64, 0x65, 0x72, 0x00 }; + static const unsigned char external_label[] = { 0x65, 0x78, 0x74, 0x20, 0x62, 0x69, 0x6E, 0x64, 0x65, 0x72, 0x00 }; +#else + static const unsigned char resumption_label[] = "res binder"; + static const unsigned char external_label[] = "ext binder"; +#endif + const unsigned char *label; + size_t bindersize, labelsize, hashsize; + int hashsizei = EVP_MD_get_size(md); + int ret = -1; + int usepskfored = 0; + + /* Ensure cast to size_t is safe */ + if (!ossl_assert(hashsizei >= 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + hashsize = (size_t)hashsizei; + + if (external + && s->early_data_state == SSL_EARLY_DATA_CONNECTING + && s->session->ext.max_early_data == 0 + && sess->ext.max_early_data > 0) + usepskfored = 1; + + if (external) { + label = external_label; + labelsize = sizeof(external_label) - 1; + } else { + label = resumption_label; + labelsize = sizeof(resumption_label) - 1; + } + + /* + * Generate the early_secret. On the server side we've selected a PSK to + * resume with (internal or external) so we always do this. On the client + * side we do this for a non-external (i.e. resumption) PSK or external PSK + * that will be used for early_data so that it is in place for sending early + * data. For client side external PSK not being used for early_data we + * generate it but store it away for later use. + */ + if (s->server || !external || usepskfored) + early_secret = (unsigned char *)s->early_secret; + else + early_secret = (unsigned char *)sess->early_secret; + + if (!tls13_generate_secret(s, md, NULL, sess->master_key, + sess->master_key_length, early_secret)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + /* + * Create the handshake hash for the binder key...the messages so far are + * empty! + */ + mctx = EVP_MD_CTX_new(); + if (mctx == NULL + || EVP_DigestInit_ex(mctx, md, NULL) <= 0 + || EVP_DigestFinal_ex(mctx, hash, NULL) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* Generate the binder key */ + if (!tls13_hkdf_expand(s, md, early_secret, label, labelsize, hash, + hashsize, binderkey, hashsize, 1)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + /* Generate the finished key */ + if (!tls13_derive_finishedkey(s, md, binderkey, finishedkey, hashsize)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + if (EVP_DigestInit_ex(mctx, md, NULL) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* + * Get a hash of the ClientHello up to the start of the binders. If we are + * following a HelloRetryRequest then this includes the hash of the first + * ClientHello and the HelloRetryRequest itself. + */ + if (s->hello_retry_request == SSL_HRR_PENDING) { + size_t hdatalen; + long hdatalen_l; + void *hdata; + + hdatalen = hdatalen_l = + BIO_get_mem_data(s->s3.handshake_buffer, &hdata); + if (hdatalen_l <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_HANDSHAKE_LENGTH); + goto err; + } + + /* + * For servers the handshake buffer data will include the second + * ClientHello - which we don't want - so we need to take that bit off. + */ + if (s->server) { + PACKET hashprefix, msg; + + /* Find how many bytes are left after the first two messages */ + if (!PACKET_buf_init(&hashprefix, hdata, hdatalen) + || !PACKET_forward(&hashprefix, 1) + || !PACKET_get_length_prefixed_3(&hashprefix, &msg) + || !PACKET_forward(&hashprefix, 1) + || !PACKET_get_length_prefixed_3(&hashprefix, &msg)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + hdatalen -= PACKET_remaining(&hashprefix); + } + + if (EVP_DigestUpdate(mctx, hdata, hdatalen) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } + + if (EVP_DigestUpdate(mctx, msgstart, binderoffset) <= 0 + || EVP_DigestFinal_ex(mctx, hash, NULL) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + mackey = EVP_PKEY_new_raw_private_key_ex(s->ctx->libctx, "HMAC", + s->ctx->propq, finishedkey, + hashsize); + if (mackey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (!sign) + binderout = tmpbinder; + + bindersize = hashsize; + if (EVP_DigestSignInit_ex(mctx, NULL, EVP_MD_get0_name(md), s->ctx->libctx, + s->ctx->propq, mackey, NULL) <= 0 + || EVP_DigestSignUpdate(mctx, hash, hashsize) <= 0 + || EVP_DigestSignFinal(mctx, binderout, &bindersize) <= 0 + || bindersize != hashsize) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (sign) { + ret = 1; + } else { + /* HMAC keys can't do EVP_DigestVerify* - use CRYPTO_memcmp instead */ + ret = (CRYPTO_memcmp(binderin, binderout, hashsize) == 0); + if (!ret) + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BINDER_DOES_NOT_VERIFY); + } + + err: + OPENSSL_cleanse(binderkey, sizeof(binderkey)); + OPENSSL_cleanse(finishedkey, sizeof(finishedkey)); + EVP_PKEY_free(mackey); + EVP_MD_CTX_free(mctx); + + return ret; +} + +static int final_early_data(SSL *s, unsigned int context, int sent) +{ + if (!sent) + return 1; + + if (!s->server) { + if (context == SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS + && sent + && !s->ext.early_data_ok) { + /* + * If we get here then the server accepted our early_data but we + * later realised that it shouldn't have done (e.g. inconsistent + * ALPN) + */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_EARLY_DATA); + return 0; + } + + return 1; + } + + if (s->max_early_data == 0 + || !s->hit + || s->early_data_state != SSL_EARLY_DATA_ACCEPTING + || !s->ext.early_data_ok + || s->hello_retry_request != SSL_HRR_NONE + || (s->allow_early_data_cb != NULL + && !s->allow_early_data_cb(s, + s->allow_early_data_cb_data))) { + s->ext.early_data = SSL_EARLY_DATA_REJECTED; + } else { + s->ext.early_data = SSL_EARLY_DATA_ACCEPTED; + + if (!tls13_change_cipher_state(s, + SSL3_CC_EARLY | SSL3_CHANGE_CIPHER_SERVER_READ)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + + return 1; +} + +static int final_maxfragmentlen(SSL *s, unsigned int context, int sent) +{ + /* + * Session resumption on server-side with MFL extension active + * BUT MFL extension packet was not resent (i.e. sent == 0) + */ + if (s->server && s->hit && USE_MAX_FRAGMENT_LENGTH_EXT(s->session) + && !sent ) { + SSLfatal_ntls(s, SSL_AD_MISSING_EXTENSION, SSL_R_BAD_EXTENSION); + return 0; + } + + /* Current SSL buffer is lower than requested MFL */ + if (s->session && USE_MAX_FRAGMENT_LENGTH_EXT(s->session) + && s->max_send_fragment < GET_MAX_FRAGMENT_LENGTH(s->session)) + /* trigger a larger buffer reallocation */ + if (!ssl3_setup_buffers(s)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + return 1; +} + +static int init_post_handshake_auth(SSL *s, ossl_unused unsigned int context) +{ + s->post_handshake_auth = SSL_PHA_NONE; + + return 1; +} + +/* + * If clients offer "pre_shared_key" without a "psk_key_exchange_modes" + * extension, servers MUST abort the handshake. + */ +static int final_psk(SSL *s, unsigned int context, int sent) +{ + if (s->server && sent && s->clienthello != NULL + && !s->clienthello->pre_proc_exts[TLSEXT_IDX_psk_kex_modes].present) { + SSLfatal(s, TLS13_AD_MISSING_EXTENSION, + SSL_R_MISSING_PSK_KEX_MODES_EXTENSION); + return 0; + } + + return 1; +} diff --git a/openssl/src/ssl/statem_ntls/ntls_extensions_clnt.c b/openssl/src/ssl/statem_ntls/ntls_extensions_clnt.c new file mode 100644 index 000000000..a8872eaf5 --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_extensions_clnt.c @@ -0,0 +1,1602 @@ +/* + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include "ntls_ssl_local.h" +#include "internal/cryptlib.h" +#include "ntls_statem_local.h" + +EXT_RETURN tls_construct_ctos_server_name_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (s->ext.hostname == NULL) + return EXT_RETURN_NOT_SENT; + + /* Add TLS extension servername to the Client Hello message */ + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_server_name) + /* Sub-packet for server_name extension */ + || !WPACKET_start_sub_packet_u16(pkt) + /* Sub-packet for servername list (always 1 hostname)*/ + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u8(pkt, TLSEXT_NAMETYPE_host_name) + || !WPACKET_sub_memcpy_u16(pkt, s->ext.hostname, + strlen(s->ext.hostname)) + || !WPACKET_close(pkt) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +/* Push a Max Fragment Len extension into ClientHello */ +EXT_RETURN tls_construct_ctos_maxfragmentlen_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (s->ext.max_fragment_len_mode == TLSEXT_max_fragment_length_DISABLED) + return EXT_RETURN_NOT_SENT; + + /* Add Max Fragment Length extension if client enabled it. */ + /*- + * 4 bytes for this extension type and extension length + * 1 byte for the Max Fragment Length code value. + */ + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_max_fragment_length) + /* Sub-packet for Max Fragment Length extension (1 byte) */ + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u8(pkt, s->ext.max_fragment_len_mode) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +EXT_RETURN tls_construct_ctos_ec_pt_formats_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + /* No ec_point_formats */ + return EXT_RETURN_NOT_SENT; +} + +EXT_RETURN tls_construct_ctos_supported_groups_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + /* No supported_groups */ + return EXT_RETURN_NOT_SENT; +} + +EXT_RETURN tls_construct_ctos_session_ticket_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + size_t ticklen; + + if (!tls_use_ticket(s)) + return EXT_RETURN_NOT_SENT; + + if (!s->new_session && s->session != NULL + && s->session->ext.tick != NULL + && s->session->ssl_version != TLS1_3_VERSION) { + ticklen = s->session->ext.ticklen; + } else if (s->session && s->ext.session_ticket != NULL + && s->ext.session_ticket->data != NULL) { + ticklen = s->ext.session_ticket->length; + s->session->ext.tick = OPENSSL_malloc(ticklen); + if (s->session->ext.tick == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + memcpy(s->session->ext.tick, + s->ext.session_ticket->data, ticklen); + s->session->ext.ticklen = ticklen; + } else { + ticklen = 0; + } + + if (ticklen == 0 && s->ext.session_ticket != NULL && + s->ext.session_ticket->data == NULL) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_session_ticket) + || !WPACKET_sub_memcpy_u16(pkt, s->session->ext.tick, ticklen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +EXT_RETURN tls_construct_ctos_sig_algs_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + /* No signature_algorithms */ + return EXT_RETURN_NOT_SENT; +} + +#ifndef OPENSSL_NO_OCSP +EXT_RETURN tls_construct_ctos_status_request_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + int i; + + /* This extension isn't defined for client Certificates */ + if (x != NULL) + return EXT_RETURN_NOT_SENT; + + if (s->ext.status_type != TLSEXT_STATUSTYPE_ocsp) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_status_request) + /* Sub-packet for status request extension */ + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u8(pkt, TLSEXT_STATUSTYPE_ocsp) + /* Sub-packet for the ids */ + || !WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + for (i = 0; i < sk_OCSP_RESPID_num(s->ext.ocsp.ids); i++) { + unsigned char *idbytes; + OCSP_RESPID *id = sk_OCSP_RESPID_value(s->ext.ocsp.ids, i); + int idlen = i2d_OCSP_RESPID(id, NULL); + + if (idlen <= 0 + /* Sub-packet for an individual id */ + || !WPACKET_sub_allocate_bytes_u16(pkt, idlen, &idbytes) + || i2d_OCSP_RESPID(id, &idbytes) != idlen) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + } + if (!WPACKET_close(pkt) + || !WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + if (s->ext.ocsp.exts) { + unsigned char *extbytes; + int extlen = i2d_X509_EXTENSIONS(s->ext.ocsp.exts, NULL); + + if (extlen < 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + if (!WPACKET_allocate_bytes(pkt, extlen, &extbytes) + || i2d_X509_EXTENSIONS(s->ext.ocsp.exts, &extbytes) + != extlen) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + } + if (!WPACKET_close(pkt) || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} +#endif + +#ifndef OPENSSL_NO_NEXTPROTONEG +EXT_RETURN tls_construct_ctos_npn_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->ctx->ext.npn_select_cb == NULL || !SSL_IS_FIRST_HANDSHAKE(s)) + return EXT_RETURN_NOT_SENT; + + /* + * The client advertises an empty extension to indicate its support + * for Next Protocol Negotiation + */ + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_next_proto_neg) + || !WPACKET_put_bytes_u16(pkt, 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} +#endif + +EXT_RETURN tls_construct_ctos_alpn_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + s->s3.alpn_sent = 0; + + if (s->ext.alpn == NULL || !SSL_IS_FIRST_HANDSHAKE(s)) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, + TLSEXT_TYPE_application_layer_protocol_negotiation) + /* Sub-packet ALPN extension */ + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_sub_memcpy_u16(pkt, s->ext.alpn, s->ext.alpn_len) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + s->s3.alpn_sent = 1; + + return EXT_RETURN_SENT; +} + + +#ifndef OPENSSL_NO_SRTP +EXT_RETURN tls_construct_ctos_use_srtp_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + STACK_OF(SRTP_PROTECTION_PROFILE) *clnt = SSL_get_srtp_profiles(s); + int i, end; + + if (clnt == NULL) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_use_srtp) + /* Sub-packet for SRTP extension */ + || !WPACKET_start_sub_packet_u16(pkt) + /* Sub-packet for the protection profile list */ + || !WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + end = sk_SRTP_PROTECTION_PROFILE_num(clnt); + for (i = 0; i < end; i++) { + const SRTP_PROTECTION_PROFILE *prof = + sk_SRTP_PROTECTION_PROFILE_value(clnt, i); + + if (prof == NULL || !WPACKET_put_bytes_u16(pkt, prof->id)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + } + if (!WPACKET_close(pkt) + /* Add an empty use_mki value */ + || !WPACKET_put_bytes_u8(pkt, 0) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} +#endif + +EXT_RETURN tls_construct_ctos_etm_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + /* No encrypt-then-MAC */ + return EXT_RETURN_NOT_SENT; +} + +#ifndef OPENSSL_NO_CT +EXT_RETURN tls_construct_ctos_sct_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->ct_validation_callback == NULL) + return EXT_RETURN_NOT_SENT; + + /* Not defined for client Certificates */ + if (x != NULL) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_signed_certificate_timestamp) + || !WPACKET_put_bytes_u16(pkt, 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} +#endif + +EXT_RETURN tls_construct_ctos_ems_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + /* No extended_master_secret */ + return EXT_RETURN_NOT_SENT; +} + +EXT_RETURN tls_construct_ctos_supported_versions_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + /* No supported_versions */ + return EXT_RETURN_NOT_SENT; +} + +/* + * Construct a psk_kex_modes extension. + */ +EXT_RETURN tls_construct_ctos_psk_kex_modes_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + int nodhe = s->options & SSL_OP_ALLOW_NO_DHE_KEX; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_psk_kex_modes) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_start_sub_packet_u8(pkt) + || !WPACKET_put_bytes_u8(pkt, TLSEXT_KEX_MODE_KE_DHE) + || (nodhe && !WPACKET_put_bytes_u8(pkt, TLSEXT_KEX_MODE_KE)) + || !WPACKET_close(pkt) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + s->ext.psk_kex_mode = TLSEXT_KEX_MODE_FLAG_KE_DHE; + if (nodhe) + s->ext.psk_kex_mode |= TLSEXT_KEX_MODE_FLAG_KE; +#endif + + return EXT_RETURN_SENT; +} + +#ifndef OPENSSL_NO_TLS1_3 +static int add_key_share(SSL *s, WPACKET *pkt, unsigned int curve_id) +{ + unsigned char *encoded_point = NULL; + EVP_PKEY *key_share_key = NULL; + size_t encodedlen; + + if (s->s3.tmp.pkey != NULL) { + if (!ossl_assert(s->hello_retry_request == SSL_HRR_PENDING)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + /* + * Could happen if we got an HRR that wasn't requesting a new key_share + */ + key_share_key = s->s3.tmp.pkey; + } else { + key_share_key = ssl_generate_pkey_group(s, curve_id); + if (key_share_key == NULL) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + + /* Encode the public key. */ + encodedlen = EVP_PKEY_get1_encoded_public_key(key_share_key, + &encoded_point); + if (encodedlen == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EC_LIB); + goto err; + } + + /* Create KeyShareEntry */ + if (!WPACKET_put_bytes_u16(pkt, curve_id) + || !WPACKET_sub_memcpy_u16(pkt, encoded_point, encodedlen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* + * When changing to send more than one key_share we're + * going to need to be able to save more than one EVP_PKEY. For now + * we reuse the existing tmp.pkey + */ + s->s3.tmp.pkey = key_share_key; + s->s3.group_id = curve_id; + OPENSSL_free(encoded_point); + + return 1; + err: + if (s->s3.tmp.pkey == NULL) + EVP_PKEY_free(key_share_key); + OPENSSL_free(encoded_point); + return 0; +} +#endif + +EXT_RETURN tls_construct_ctos_key_share_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + size_t i, num_groups = 0; + const uint16_t *pgroups = NULL; + uint16_t curve_id = 0; + + /* key_share extension */ + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_key_share) + /* Extension data sub-packet */ + || !WPACKET_start_sub_packet_u16(pkt) + /* KeyShare list sub-packet */ + || !WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + tls1_get_supported_groups(s, &pgroups, &num_groups); + + /* + * Make the number of key_shares sent configurable. For + * now, we just send one + */ + if (s->s3.group_id != 0) { + curve_id = s->s3.group_id; + } else { + for (i = 0; i < num_groups; i++) { + + if (!tls_group_allowed(s, pgroups[i], SSL_SECOP_CURVE_SUPPORTED)) + continue; + + curve_id = pgroups[i]; + break; + } + } + + if (curve_id == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_NO_SUITABLE_KEY_SHARE); + return EXT_RETURN_FAIL; + } + + if (!add_key_share(s, pkt, curve_id)) { + /* SSLfatal_ntls() already called */ + return EXT_RETURN_FAIL; + } + + if (!WPACKET_close(pkt) || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + return EXT_RETURN_SENT; +#else + return EXT_RETURN_NOT_SENT; +#endif +} + +EXT_RETURN tls_construct_ctos_cookie_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + EXT_RETURN ret = EXT_RETURN_FAIL; + + /* Should only be set if we've had an HRR */ + if (s->ext.tls13_cookie_len == 0) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_cookie) + /* Extension data sub-packet */ + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_sub_memcpy_u16(pkt, s->ext.tls13_cookie, + s->ext.tls13_cookie_len) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto end; + } + + ret = EXT_RETURN_SENT; + end: + OPENSSL_free(s->ext.tls13_cookie); + s->ext.tls13_cookie = NULL; + s->ext.tls13_cookie_len = 0; + + return ret; +} + +EXT_RETURN tls_construct_ctos_early_data_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ +#ifndef OPENSSL_NO_PSK + char identity[PSK_MAX_IDENTITY_LEN + 1]; +#endif /* OPENSSL_NO_PSK */ + const unsigned char *id = NULL; + size_t idlen = 0; + SSL_SESSION *psksess = NULL; + SSL_SESSION *edsess = NULL; + const EVP_MD *handmd = NULL; + + if (s->hello_retry_request == SSL_HRR_PENDING) + handmd = ssl_handshake_md(s); + + if (s->psk_use_session_cb != NULL + && (!s->psk_use_session_cb(s, handmd, &id, &idlen, &psksess) + || (psksess != NULL + && psksess->ssl_version != TLS1_3_VERSION))) { + SSL_SESSION_free(psksess); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_PSK); + return EXT_RETURN_FAIL; + } + +#ifndef OPENSSL_NO_PSK + if (psksess == NULL && s->psk_client_callback != NULL) { + unsigned char psk[PSK_MAX_PSK_LEN]; + size_t psklen = 0; + + memset(identity, 0, sizeof(identity)); + psklen = s->psk_client_callback(s, NULL, identity, sizeof(identity) - 1, + psk, sizeof(psk)); + + if (psklen > PSK_MAX_PSK_LEN) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } else if (psklen > 0) { + const unsigned char tls13_aes128gcmsha256_id[] = { 0x13, 0x01 }; + const SSL_CIPHER *cipher; + + idlen = strlen(identity); + if (idlen > PSK_MAX_IDENTITY_LEN) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + id = (unsigned char *)identity; + + /* + * We found a PSK using an old style callback. We don't know + * the digest so we default to SHA256 as per the TLSv1.3 spec + */ + cipher = SSL_CIPHER_find(s, tls13_aes128gcmsha256_id); + if (cipher == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + psksess = SSL_SESSION_new(); + if (psksess == NULL + || !SSL_SESSION_set1_master_key(psksess, psk, psklen) + || !SSL_SESSION_set_cipher(psksess, cipher) + || !SSL_SESSION_set_protocol_version(psksess, TLS1_3_VERSION)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + OPENSSL_cleanse(psk, psklen); + return EXT_RETURN_FAIL; + } + OPENSSL_cleanse(psk, psklen); + } + } +#endif /* OPENSSL_NO_PSK */ + + SSL_SESSION_free(s->psksession); + s->psksession = psksess; + if (psksess != NULL) { + OPENSSL_free(s->psksession_id); + s->psksession_id = OPENSSL_memdup(id, idlen); + if (s->psksession_id == NULL) { + s->psksession_id_len = 0; + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + s->psksession_id_len = idlen; + } + + if (s->early_data_state != SSL_EARLY_DATA_CONNECTING + || (s->session->ext.max_early_data == 0 + && (psksess == NULL || psksess->ext.max_early_data == 0))) { + s->max_early_data = 0; + return EXT_RETURN_NOT_SENT; + } + edsess = s->session->ext.max_early_data != 0 ? s->session : psksess; + s->max_early_data = edsess->ext.max_early_data; + + if (edsess->ext.hostname != NULL) { + if (s->ext.hostname == NULL + || (s->ext.hostname != NULL + && strcmp(s->ext.hostname, edsess->ext.hostname) != 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_INCONSISTENT_EARLY_DATA_SNI); + return EXT_RETURN_FAIL; + } + } + + if ((s->ext.alpn == NULL && edsess->ext.alpn_selected != NULL)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_INCONSISTENT_EARLY_DATA_ALPN); + return EXT_RETURN_FAIL; + } + + /* + * Verify that we are offering an ALPN protocol consistent with the early + * data. + */ + if (edsess->ext.alpn_selected != NULL) { + PACKET prots, alpnpkt; + int found = 0; + + if (!PACKET_buf_init(&prots, s->ext.alpn, s->ext.alpn_len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + while (PACKET_get_length_prefixed_1(&prots, &alpnpkt)) { + if (PACKET_equal(&alpnpkt, edsess->ext.alpn_selected, + edsess->ext.alpn_selected_len)) { + found = 1; + break; + } + } + if (!found) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_INCONSISTENT_EARLY_DATA_ALPN); + return EXT_RETURN_FAIL; + } + } + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_early_data) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + /* + * We set this to rejected here. Later, if the server acknowledges the + * extension, we set it to accepted. + */ + s->ext.early_data = SSL_EARLY_DATA_REJECTED; + s->ext.early_data_ok = 1; + + return EXT_RETURN_SENT; +} + +#define F5_WORKAROUND_MIN_MSG_LEN 0xff +#define F5_WORKAROUND_MAX_MSG_LEN 0x200 + +/* + * PSK pre binder overhead = + * 2 bytes for TLSEXT_TYPE_psk + * 2 bytes for extension length + * 2 bytes for identities list length + * 2 bytes for identity length + * 4 bytes for obfuscated_ticket_age + * 2 bytes for binder list length + * 1 byte for binder length + * The above excludes the number of bytes for the identity itself and the + * subsequent binder bytes + */ +#define PSK_PRE_BINDER_OVERHEAD (2 + 2 + 2 + 2 + 4 + 2 + 1) + +EXT_RETURN tls_construct_ctos_padding_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + unsigned char *padbytes; + size_t hlen; + + if ((s->options & SSL_OP_TLSEXT_PADDING) == 0) + return EXT_RETURN_NOT_SENT; + + /* + * Add padding to workaround bugs in F5 terminators. See RFC7685. + * This code calculates the length of all extensions added so far but + * excludes the PSK extension (because that MUST be written last). Therefore + * this extension MUST always appear second to last. + */ + if (!WPACKET_get_total_written(pkt, &hlen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + /* + * If we're going to send a PSK then that will be written out after this + * extension, so we need to calculate how long it is going to be. + */ + if (s->session->ssl_version == TLS1_3_VERSION + && s->session->ext.ticklen != 0 + && s->session->cipher != NULL) { + const EVP_MD *md = ssl_md(s->ctx, s->session->cipher->algorithm2); + + if (md != NULL) { + /* + * Add the fixed PSK overhead, the identity length and the binder + * length. + */ + hlen += PSK_PRE_BINDER_OVERHEAD + s->session->ext.ticklen + + EVP_MD_get_size(md); + } + } + + if (hlen > F5_WORKAROUND_MIN_MSG_LEN && hlen < F5_WORKAROUND_MAX_MSG_LEN) { + /* Calculate the amount of padding we need to add */ + hlen = F5_WORKAROUND_MAX_MSG_LEN - hlen; + + /* + * Take off the size of extension header itself (2 bytes for type and + * 2 bytes for length bytes), but ensure that the extension is at least + * 1 byte long so as not to have an empty extension last (WebSphere 7.x, + * 8.x are intolerant of that condition) + */ + if (hlen > 4) + hlen -= 4; + else + hlen = 1; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_padding) + || !WPACKET_sub_allocate_bytes_u16(pkt, hlen, &padbytes)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + memset(padbytes, 0, hlen); + } + + return EXT_RETURN_SENT; +} + +/* + * Construct the pre_shared_key extension + */ +EXT_RETURN tls_construct_ctos_psk_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + uint32_t now, agesec, agems = 0; + size_t reshashsize = 0, pskhashsize = 0, binderoffset, msglen; + unsigned char *resbinder = NULL, *pskbinder = NULL, *msgstart = NULL; + const EVP_MD *handmd = NULL, *mdres = NULL, *mdpsk = NULL; + int dores = 0; + + s->ext.tick_identity = 0; + + /* + * Note: At this stage of the code we only support adding a single + * resumption PSK. If we add support for multiple PSKs then the length + * calculations in the padding extension will need to be adjusted. + */ + + /* + * If this is an incompatible or new session then we have nothing to resume + * so don't add this extension. + */ + if (s->session->ssl_version != TLS1_3_VERSION + || (s->session->ext.ticklen == 0 && s->psksession == NULL)) + return EXT_RETURN_NOT_SENT; + + if (s->hello_retry_request == SSL_HRR_PENDING) + handmd = ssl_handshake_md(s); + + if (s->session->ext.ticklen != 0) { + /* Get the digest associated with the ciphersuite in the session */ + if (s->session->cipher == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + mdres = ssl_md(s->ctx, s->session->cipher->algorithm2); + if (mdres == NULL) { + /* + * Don't recognize this cipher so we can't use the session. + * Ignore it + */ + goto dopsksess; + } + + if (s->hello_retry_request == SSL_HRR_PENDING && mdres != handmd) { + /* + * Selected ciphersuite hash does not match the hash for the session + * so we can't use it. + */ + goto dopsksess; + } + + /* + * Technically the C standard just says time() returns a time_t and says + * nothing about the encoding of that type. In practice most + * implementations follow POSIX which holds it as an integral type in + * seconds since epoch. We've already made the assumption that we can do + * this in multiple places in the code, so portability shouldn't be an + * issue. + */ + now = (uint32_t)time(NULL); + agesec = now - (uint32_t)s->session->time; + /* + * We calculate the age in seconds but the server may work in ms. Due to + * rounding errors we could overestimate the age by up to 1s. It is + * better to underestimate it. Otherwise, if the RTT is very short, when + * the server calculates the age reported by the client it could be + * bigger than the age calculated on the server - which should never + * happen. + */ + if (agesec > 0) + agesec--; + + if (s->session->ext.tick_lifetime_hint < agesec) { + /* Ticket is too old. Ignore it. */ + goto dopsksess; + } + + /* + * Calculate age in ms. We're just doing it to nearest second. Should be + * good enough. + */ + agems = agesec * (uint32_t)1000; + + if (agesec != 0 && agems / (uint32_t)1000 != agesec) { + /* + * Overflow. Shouldn't happen unless this is a *really* old session. + * If so we just ignore it. + */ + goto dopsksess; + } + + /* + * Obfuscate the age. Overflow here is fine, this addition is supposed + * to be mod 2^32. + */ + agems += s->session->ext.tick_age_add; + + reshashsize = EVP_MD_get_size(mdres); + s->ext.tick_identity++; + dores = 1; + } + + dopsksess: + if (!dores && s->psksession == NULL) + return EXT_RETURN_NOT_SENT; + + if (s->psksession != NULL) { + mdpsk = ssl_md(s->ctx, s->psksession->cipher->algorithm2); + if (mdpsk == NULL) { + /* + * Don't recognize this cipher so we can't use the session. + * If this happens it's an application bug. + */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_PSK); + return EXT_RETURN_FAIL; + } + + if (s->hello_retry_request == SSL_HRR_PENDING && mdpsk != handmd) { + /* + * Selected ciphersuite hash does not match the hash for the PSK + * session. This is an application bug. + */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_PSK); + return EXT_RETURN_FAIL; + } + + pskhashsize = EVP_MD_get_size(mdpsk); + } + + /* Create the extension, but skip over the binder for now */ + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_psk) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + if (dores) { + if (!WPACKET_sub_memcpy_u16(pkt, s->session->ext.tick, + s->session->ext.ticklen) + || !WPACKET_put_bytes_u32(pkt, agems)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + } + + if (s->psksession != NULL) { + if (!WPACKET_sub_memcpy_u16(pkt, s->psksession_id, + s->psksession_id_len) + || !WPACKET_put_bytes_u32(pkt, 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + s->ext.tick_identity++; + } + + if (!WPACKET_close(pkt) + || !WPACKET_get_total_written(pkt, &binderoffset) + || !WPACKET_start_sub_packet_u16(pkt) + || (dores + && !WPACKET_sub_allocate_bytes_u8(pkt, reshashsize, &resbinder)) + || (s->psksession != NULL + && !WPACKET_sub_allocate_bytes_u8(pkt, pskhashsize, &pskbinder)) + || !WPACKET_close(pkt) + || !WPACKET_close(pkt) + || !WPACKET_get_total_written(pkt, &msglen) + /* + * We need to fill in all the sub-packet lengths now so we can + * calculate the HMAC of the message up to the binders + */ + || !WPACKET_fill_lengths(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + msgstart = WPACKET_get_curr(pkt) - msglen; + + if (dores + && tls_psk_do_binder_ntls(s, mdres, msgstart, binderoffset, NULL, + resbinder, s->session, 1, 0) != 1) { + /* SSLfatal_ntls() already called */ + return EXT_RETURN_FAIL; + } + + if (s->psksession != NULL + && tls_psk_do_binder_ntls(s, mdpsk, msgstart, binderoffset, NULL, + pskbinder, s->psksession, 1, 1) != 1) { + /* SSLfatal_ntls() already called */ + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +#else + return EXT_RETURN_NOT_SENT; +#endif +} + +EXT_RETURN tls_construct_ctos_post_handshake_auth_ntls(SSL *s, WPACKET *pkt, + ossl_unused unsigned int context, + ossl_unused X509 *x, + ossl_unused size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + if (!s->pha_enabled) + return EXT_RETURN_NOT_SENT; + + /* construct extension - 0 length, no contents */ + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_post_handshake_auth) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + s->post_handshake_auth = SSL_PHA_EXT_SENT; + + return EXT_RETURN_SENT; +#else + return EXT_RETURN_NOT_SENT; +#endif +} + +/* Parse the server's max fragment len extension packet */ +int tls_parse_stoc_maxfragmentlen_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + unsigned int value; + + if (PACKET_remaining(pkt) != 1 || !PACKET_get_1(pkt, &value)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + /* |value| should contains a valid max-fragment-length code. */ + if (!IS_MAX_FRAGMENT_LENGTH_EXT_VALID(value)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH); + return 0; + } + + /* Must be the same value as client-configured one who was sent to server */ + /*- + * RFC 6066: if a client receives a maximum fragment length negotiation + * response that differs from the length it requested, ... + * It must abort with SSL_AD_ILLEGAL_PARAMETER alert + */ + if (value != s->ext.max_fragment_len_mode) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH); + return 0; + } + + /* + * Maximum Fragment Length Negotiation succeeded. + * The negotiated Maximum Fragment Length is binding now. + */ + s->session->ext.max_fragment_len_mode = value; + + return 1; +} + +int tls_parse_stoc_server_name_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->ext.hostname == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (PACKET_remaining(pkt) > 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + if (!s->hit) { + if (s->session->ext.hostname != NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + s->session->ext.hostname = OPENSSL_strdup(s->ext.hostname); + if (s->session->ext.hostname == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + + return 1; +} + +int tls_parse_stoc_ec_pt_formats_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + /* Ignore ec_point_formats */ + return 1; +} + +int tls_parse_stoc_session_ticket_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->ext.session_ticket_cb != NULL && + !s->ext.session_ticket_cb(s, PACKET_data(pkt), + PACKET_remaining(pkt), + s->ext.session_ticket_cb_arg)) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_BAD_EXTENSION); + return 0; + } + + if (!tls_use_ticket(s)) { + SSLfatal_ntls(s, SSL_AD_UNSUPPORTED_EXTENSION, SSL_R_BAD_EXTENSION); + return 0; + } + if (PACKET_remaining(pkt) > 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + s->ext.ticket_expected = 1; + + return 1; +} + +#ifndef OPENSSL_NO_OCSP +int tls_parse_stoc_status_request_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (context == SSL_EXT_TLS1_3_CERTIFICATE_REQUEST) { + /* We ignore this if the server sends a CertificateRequest */ + return 1; + } + + /* + * MUST only be sent if we've requested a status + * request message. In TLS <= 1.2 it must also be empty. + */ + if (s->ext.status_type != TLSEXT_STATUSTYPE_ocsp) { + SSLfatal_ntls(s, SSL_AD_UNSUPPORTED_EXTENSION, SSL_R_BAD_EXTENSION); + return 0; + } + if (PACKET_remaining(pkt) > 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + /* Set flag to expect CertificateStatus message */ + s->ext.status_expected = 1; + + return 1; +} +#endif + + +#ifndef OPENSSL_NO_CT +int tls_parse_stoc_sct_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + if (context == SSL_EXT_TLS1_3_CERTIFICATE_REQUEST) { + /* We ignore this if the server sends it in a CertificateRequest */ + return 1; + } + + /* + * Only take it if we asked for it - i.e if there is no CT validation + * callback set, then a custom extension MAY be processing it, so we + * need to let control continue to flow to that. + */ + if (s->ct_validation_callback != NULL) { + size_t size = PACKET_remaining(pkt); + + /* Simply copy it off for later processing */ + OPENSSL_free(s->ext.scts); + s->ext.scts = NULL; + + s->ext.scts_len = (uint16_t)size; + if (size > 0) { + s->ext.scts = OPENSSL_malloc(size); + if (s->ext.scts == NULL) { + s->ext.scts_len = 0; + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + if (!PACKET_copy_bytes(pkt, s->ext.scts, size)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + } else { + ENDPOINT role = (context & SSL_EXT_TLS1_2_SERVER_HELLO) != 0 + ? ENDPOINT_CLIENT : ENDPOINT_BOTH; + + /* + * If we didn't ask for it then there must be a custom extension, + * otherwise this is unsolicited. + */ + if (custom_ext_find_ntls(&s->cert->custext, role, + TLSEXT_TYPE_signed_certificate_timestamp, + NULL) == NULL) { + SSLfatal_ntls(s, TLS1_AD_UNSUPPORTED_EXTENSION, SSL_R_BAD_EXTENSION); + return 0; + } + + if (!custom_ext_parse_ntls(s, context, + TLSEXT_TYPE_signed_certificate_timestamp, + PACKET_data(pkt), PACKET_remaining(pkt), + x, chainidx)) { + /* SSLfatal_ntls already called */ + return 0; + } + } + + return 1; +} +#endif + + +#ifndef OPENSSL_NO_NEXTPROTONEG +/* + * ssl_next_proto_validate validates a Next Protocol Negotiation block. No + * elements of zero length are allowed and the set of elements must exactly + * fill the length of the block. Returns 1 on success or 0 on failure. + */ +static int ssl_next_proto_validate(SSL *s, PACKET *pkt) +{ + PACKET tmp_protocol; + + while (PACKET_remaining(pkt)) { + if (!PACKET_get_length_prefixed_1(pkt, &tmp_protocol) + || PACKET_remaining(&tmp_protocol) == 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + } + + return 1; +} + +int tls_parse_stoc_npn_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + unsigned char *selected; + unsigned char selected_len; + PACKET tmppkt; + + /* Check if we are in a renegotiation. If so ignore this extension */ + if (!SSL_IS_FIRST_HANDSHAKE(s)) + return 1; + + /* We must have requested it. */ + if (s->ctx->ext.npn_select_cb == NULL) { + SSLfatal_ntls(s, SSL_AD_UNSUPPORTED_EXTENSION, SSL_R_BAD_EXTENSION); + return 0; + } + + /* The data must be valid */ + tmppkt = *pkt; + if (!ssl_next_proto_validate(s, &tmppkt)) { + /* SSLfatal_ntls() already called */ + return 0; + } + if (s->ctx->ext.npn_select_cb(s, &selected, &selected_len, + PACKET_data(pkt), + PACKET_remaining(pkt), + s->ctx->ext.npn_select_cb_arg) != + SSL_TLSEXT_ERR_OK) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_BAD_EXTENSION); + return 0; + } + + /* + * Could be non-NULL if server has sent multiple NPN extensions in + * a single Serverhello + */ + OPENSSL_free(s->ext.npn); + s->ext.npn = OPENSSL_malloc(selected_len); + if (s->ext.npn == NULL) { + s->ext.npn_len = 0; + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + memcpy(s->ext.npn, selected, selected_len); + s->ext.npn_len = selected_len; + s->s3.npn_seen = 1; + + return 1; +} +#endif + +int tls_parse_stoc_alpn_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + size_t len; + + /* We must have requested it. */ + if (!s->s3.alpn_sent) { + SSLfatal_ntls(s, SSL_AD_UNSUPPORTED_EXTENSION, SSL_R_BAD_EXTENSION); + return 0; + } + /*- + * The extension data consists of: + * uint16 list_length + * uint8 proto_length; + * uint8 proto[proto_length]; + */ + if (!PACKET_get_net_2_len(pkt, &len) + || PACKET_remaining(pkt) != len || !PACKET_get_1_len(pkt, &len) + || PACKET_remaining(pkt) != len) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + OPENSSL_free(s->s3.alpn_selected); + s->s3.alpn_selected = OPENSSL_malloc(len); + if (s->s3.alpn_selected == NULL) { + s->s3.alpn_selected_len = 0; + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if (!PACKET_copy_bytes(pkt, s->s3.alpn_selected, len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + s->s3.alpn_selected_len = len; + + if (s->session->ext.alpn_selected == NULL + || s->session->ext.alpn_selected_len != len + || memcmp(s->session->ext.alpn_selected, s->s3.alpn_selected, len) + != 0) { + /* ALPN not consistent with the old session so cannot use early_data */ + s->ext.early_data_ok = 0; + } + if (!s->hit) { + /* + * This is a new session and so alpn_selected should have been + * initialised to NULL. We should update it with the selected ALPN. + */ + if (!ossl_assert(s->session->ext.alpn_selected == NULL)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + s->session->ext.alpn_selected = + OPENSSL_memdup(s->s3.alpn_selected, s->s3.alpn_selected_len); + if (s->session->ext.alpn_selected == NULL) { + s->session->ext.alpn_selected_len = 0; + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + s->session->ext.alpn_selected_len = s->s3.alpn_selected_len; + } + + return 1; +} + +#ifndef OPENSSL_NO_SRTP +int tls_parse_stoc_use_srtp_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + unsigned int id, ct, mki; + int i; + STACK_OF(SRTP_PROTECTION_PROFILE) *clnt; + SRTP_PROTECTION_PROFILE *prof; + + if (!PACKET_get_net_2(pkt, &ct) || ct != 2 + || !PACKET_get_net_2(pkt, &id) + || !PACKET_get_1(pkt, &mki) + || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, + SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST); + return 0; + } + + if (mki != 0) { + /* Must be no MKI, since we never offer one */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_SRTP_MKI_VALUE); + return 0; + } + + /* Throw an error if the server gave us an unsolicited extension */ + clnt = SSL_get_srtp_profiles(s); + if (clnt == NULL) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_NO_SRTP_PROFILES); + return 0; + } + + /* + * Check to see if the server gave us something we support (and + * presumably offered) + */ + for (i = 0; i < sk_SRTP_PROTECTION_PROFILE_num(clnt); i++) { + prof = sk_SRTP_PROTECTION_PROFILE_value(clnt, i); + + if (prof->id == id) { + s->srtp_profile = prof; + return 1; + } + } + + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, + SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST); + return 0; +} +#endif + +int tls_parse_stoc_etm_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + /* Ignore encrypt-then-MAC */ + return 1; +} + +int tls_parse_stoc_ems_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + /* Ignore extended_master_secret */ + return 1; +} + +int tls_parse_stoc_supported_versions_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + /* Ignore supported_versions */ + return 1; +} + +int tls_parse_stoc_key_share_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + unsigned int group_id; + PACKET encoded_pt; + EVP_PKEY *ckey = s->s3.tmp.pkey, *skey = NULL; + const TLS_GROUP_INFO *ginf = NULL; + + /* Sanity check */ + if (ckey == NULL || s->s3.peer_tmp != NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!PACKET_get_net_2(pkt, &group_id)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + if ((context & SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST) != 0) { + const uint16_t *pgroups = NULL; + size_t i, num_groups; + + if (PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + /* + * It is an error if the HelloRetryRequest wants a key_share that we + * already sent in the first ClientHello + */ + if (group_id == s->s3.group_id) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + return 0; + } + + /* Validate the selected group is one we support */ + tls1_get_supported_groups(s, &pgroups, &num_groups); + for (i = 0; i < num_groups; i++) { + if (group_id == pgroups[i]) + break; + } + if (i >= num_groups + || !tls_group_allowed(s, group_id, SSL_SECOP_CURVE_SUPPORTED)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + return 0; + } + + s->s3.group_id = group_id; + EVP_PKEY_free(s->s3.tmp.pkey); + s->s3.tmp.pkey = NULL; + return 1; + } + + if (group_id != s->s3.group_id) { + /* + * This isn't for the group that we sent in the original + * key_share! + */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + return 0; + } + /* Retain this group in the SSL_SESSION */ + if (!s->hit) { + s->session->kex_group = group_id; + } else if (group_id != s->session->kex_group) { + /* + * If this is a resumption but changed what group was used, we need + * to record the new group in the session, but the session is not + * a new session and could be in use by other threads. So, make + * a copy of the session to record the new information so that it's + * useful for any sessions resumed from tickets issued on this + * connection. + */ + SSL_SESSION *new_sess; + + if ((new_sess = ssl_session_dup(s->session, 0)) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + SSL_SESSION_free(s->session); + s->session = new_sess; + s->session->kex_group = group_id; + } + + if ((ginf = tls1_group_id_lookup(s->ctx, group_id)) == NULL) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + return 0; + } + + if (!PACKET_as_length_prefixed_2(pkt, &encoded_pt) + || PACKET_remaining(&encoded_pt) == 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + if (!ginf->is_kem) { + /* Regular KEX */ + skey = EVP_PKEY_new(); + if (skey == NULL || EVP_PKEY_copy_parameters(skey, ckey) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_COPY_PARAMETERS_FAILED); + EVP_PKEY_free(skey); + return 0; + } + + if (tls13_set_encoded_pub_key(skey, PACKET_data(&encoded_pt), + PACKET_remaining(&encoded_pt)) <= 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_ECPOINT); + EVP_PKEY_free(skey); + return 0; + } + + if (ssl_derive(s, ckey, skey, 1) == 0) { + /* SSLfatal_ntls() already called */ + EVP_PKEY_free(skey); + return 0; + } + s->s3.peer_tmp = skey; + } else { + /* KEM Mode */ + const unsigned char *ct = PACKET_data(&encoded_pt); + size_t ctlen = PACKET_remaining(&encoded_pt); + + if (ssl_decapsulate(s, ckey, ct, ctlen, 1) == 0) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + s->s3.did_kex = 1; +#endif + + return 1; +} + +int tls_parse_stoc_cookie_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + PACKET cookie; + + if (!PACKET_as_length_prefixed_2(pkt, &cookie) + || !PACKET_memdup(&cookie, &s->ext.tls13_cookie, + &s->ext.tls13_cookie_len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + return 1; +} + +int tls_parse_stoc_early_data_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (context == SSL_EXT_TLS1_3_NEW_SESSION_TICKET) { + unsigned long max_early_data; + + if (!PACKET_get_net_4(pkt, &max_early_data) + || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_INVALID_MAX_EARLY_DATA); + return 0; + } + + s->session->ext.max_early_data = max_early_data; + + return 1; + } + + if (PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + if (!s->ext.early_data_ok + || !s->hit) { + /* + * If we get here then we didn't send early data, or we didn't resume + * using the first identity, or the SNI/ALPN is not consistent so the + * server should not be accepting it. + */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_EXTENSION); + return 0; + } + + s->ext.early_data = SSL_EARLY_DATA_ACCEPTED; + + return 1; +} + +int tls_parse_stoc_psk_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + unsigned int identity; + + if (!PACKET_get_net_2(pkt, &identity) || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + if (identity >= (unsigned int)s->ext.tick_identity) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_PSK_IDENTITY); + return 0; + } + + /* + * Session resumption tickets are always sent before PSK tickets. If the + * ticket index is 0 then it must be for a session resumption ticket if we + * sent two tickets, or if we didn't send a PSK ticket. + */ + if (identity == 0 && (s->psksession == NULL || s->ext.tick_identity == 2)) { + s->hit = 1; + SSL_SESSION_free(s->psksession); + s->psksession = NULL; + return 1; + } + + if (s->psksession == NULL) { + /* Should never happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* + * If we used the external PSK for sending early_data then s->early_secret + * is already set up, so don't overwrite it. Otherwise we copy the + * early_secret across that we generated earlier. + */ + if ((s->early_data_state != SSL_EARLY_DATA_WRITE_RETRY + && s->early_data_state != SSL_EARLY_DATA_FINISHED_WRITING) + || s->session->ext.max_early_data > 0 + || s->psksession->ext.max_early_data == 0) + memcpy(s->early_secret, s->psksession->early_secret, EVP_MAX_MD_SIZE); + + SSL_SESSION_free(s->session); + s->session = s->psksession; + s->psksession = NULL; + s->hit = 1; + /* Early data is only allowed if we used the first ticket */ + if (identity != 0) + s->ext.early_data_ok = 0; +#endif + + return 1; +} + diff --git a/openssl/src/ssl/statem_ntls/ntls_extensions_cust.c b/openssl/src/ssl/statem_ntls/ntls_extensions_cust.c new file mode 100644 index 000000000..038b2cb63 --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_extensions_cust.c @@ -0,0 +1,528 @@ +/* + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* Custom extension utility functions */ + +#include +#include "ntls_ssl_local.h" +#include "internal/cryptlib.h" +#include "ntls_statem_local.h" + +typedef struct { + void *add_arg; + custom_ext_add_cb add_cb; + custom_ext_free_cb free_cb; +} custom_ext_add_cb_wrap; + +typedef struct { + void *parse_arg; + custom_ext_parse_cb parse_cb; +} custom_ext_parse_cb_wrap; + +/* + * Provide thin wrapper callbacks which convert new style arguments to old style + */ +static int custom_ext_add_ntls_old_cb_wrap(SSL *s, unsigned int ext_type, + unsigned int context, + const unsigned char **out, + size_t *outlen, X509 *x, size_t chainidx, + int *al, void *add_arg) +{ + custom_ext_add_cb_wrap *add_cb_wrap = (custom_ext_add_cb_wrap *)add_arg; + + if (add_cb_wrap->add_cb == NULL) + return 1; + + return add_cb_wrap->add_cb(s, ext_type, out, outlen, al, + add_cb_wrap->add_arg); +} + +static void custom_ext_free_old_cb_wrap(SSL *s, unsigned int ext_type, + unsigned int context, + const unsigned char *out, void *add_arg) +{ + custom_ext_add_cb_wrap *add_cb_wrap = (custom_ext_add_cb_wrap *)add_arg; + + if (add_cb_wrap->free_cb == NULL) + return; + + add_cb_wrap->free_cb(s, ext_type, out, add_cb_wrap->add_arg); +} + +static int custom_ext_parse_ntls_old_cb_wrap(SSL *s, unsigned int ext_type, + unsigned int context, + const unsigned char *in, + size_t inlen, X509 *x, size_t chainidx, + int *al, void *parse_arg) +{ + custom_ext_parse_cb_wrap *parse_cb_wrap = + (custom_ext_parse_cb_wrap *)parse_arg; + + if (parse_cb_wrap->parse_cb == NULL) + return 1; + + return parse_cb_wrap->parse_cb(s, ext_type, in, inlen, al, + parse_cb_wrap->parse_arg); +} + +/* + * Find a custom extension from the list. The |role| param is there to + * support the legacy API where custom extensions for client and server could + * be set independently on the same SSL_CTX. It is set to ENDPOINT_SERVER if we + * are trying to find a method relevant to the server, ENDPOINT_CLIENT for the + * client, or ENDPOINT_BOTH for either + */ +custom_ext_method *custom_ext_find_ntls(const custom_ext_methods *exts, + ENDPOINT role, unsigned int ext_type, + size_t *idx) +{ + size_t i; + custom_ext_method *meth = exts->meths; + + for (i = 0; i < exts->meths_count; i++, meth++) { + if (ext_type == meth->ext_type + && (role == ENDPOINT_BOTH || role == meth->role + || meth->role == ENDPOINT_BOTH)) { + if (idx != NULL) + *idx = i; + return meth; + } + } + return NULL; +} + +/* + * Initialise custom extensions flags to indicate neither sent nor received. + */ +void custom_ext_init_ntls(custom_ext_methods *exts) +{ + size_t i; + custom_ext_method *meth = exts->meths; + + for (i = 0; i < exts->meths_count; i++, meth++) + meth->ext_flags = 0; +} + +/* Pass received custom extension data to the application for parsing. */ +int custom_ext_parse_ntls(SSL *s, unsigned int context, unsigned int ext_type, + const unsigned char *ext_data, size_t ext_size, X509 *x, + size_t chainidx) +{ + int al; + custom_ext_methods *exts = &s->cert->custext; + custom_ext_method *meth; + ENDPOINT role = ENDPOINT_BOTH; + + if ((context & (SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO)) != 0) + role = s->server ? ENDPOINT_SERVER : ENDPOINT_CLIENT; + + meth = custom_ext_find_ntls(exts, role, ext_type, NULL); + /* If not found return success */ + if (!meth) + return 1; + + /* Check if extension is defined for our protocol. If not, skip */ + if (!extension_is_relevant_ntls(s, meth->context, context)) + return 1; + + if ((context & (SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_SERVER_HELLO + | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS)) != 0) { + /* + * If it's ServerHello or EncryptedExtensions we can't have any + * extensions not sent in ClientHello. + */ + if ((meth->ext_flags & SSL_EXT_FLAG_SENT) == 0) { + SSLfatal_ntls(s, TLS1_AD_UNSUPPORTED_EXTENSION, SSL_R_BAD_EXTENSION); + return 0; + } + } + + /* + * Extensions received in the ClientHello or CertificateRequest are marked + * with the SSL_EXT_FLAG_RECEIVED. This is so we know to add the equivalent + * extensions in the response messages + */ + if ((context & (SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_3_CERTIFICATE_REQUEST)) + != 0) + meth->ext_flags |= SSL_EXT_FLAG_RECEIVED; + + /* If no parse function set return success */ + if (!meth->parse_cb) + return 1; + + if (meth->parse_cb(s, ext_type, context, ext_data, ext_size, x, chainidx, + &al, meth->parse_arg) <= 0) { + SSLfatal_ntls(s, al, SSL_R_BAD_EXTENSION); + return 0; + } + + return 1; +} + +/* + * Request custom extension data from the application and add to the return + * buffer. + */ +int custom_ext_add_ntls(SSL *s, int context, WPACKET *pkt, X509 *x, size_t chainidx, + int maxversion) +{ + custom_ext_methods *exts = &s->cert->custext; + custom_ext_method *meth; + size_t i; + int al; + + for (i = 0; i < exts->meths_count; i++) { + const unsigned char *out = NULL; + size_t outlen = 0; + + meth = exts->meths + i; + + if (!should_add_extension_ntls(s, meth->context, context, maxversion)) + continue; + + if ((context & (SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_SERVER_HELLO + | SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS + | SSL_EXT_TLS1_3_CERTIFICATE + | SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST)) != 0) { + /* Only send extensions present in ClientHello/CertificateRequest */ + if (!(meth->ext_flags & SSL_EXT_FLAG_RECEIVED)) + continue; + } + /* + * We skip it if the callback is absent - except for a ClientHello where + * we add an empty extension. + */ + if ((context & SSL_EXT_CLIENT_HELLO) == 0 && meth->add_cb == NULL) + continue; + + if (meth->add_cb != NULL) { + int cb_retval = meth->add_cb(s, meth->ext_type, context, &out, + &outlen, x, chainidx, &al, + meth->add_arg); + + if (cb_retval < 0) { + SSLfatal_ntls(s, al, SSL_R_CALLBACK_FAILED); + return 0; /* error */ + } + if (cb_retval == 0) + continue; /* skip this extension */ + } + + if (!WPACKET_put_bytes_u16(pkt, meth->ext_type) + || !WPACKET_start_sub_packet_u16(pkt) + || (outlen > 0 && !WPACKET_memcpy(pkt, out, outlen)) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if ((context & SSL_EXT_CLIENT_HELLO) != 0) { + /* + * We can't send duplicates: code logic should prevent this. + */ + if (!ossl_assert((meth->ext_flags & SSL_EXT_FLAG_SENT) == 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + /* + * Indicate extension has been sent: this is both a sanity check to + * ensure we don't send duplicate extensions and indicates that it + * is not an error if the extension is present in ServerHello. + */ + meth->ext_flags |= SSL_EXT_FLAG_SENT; + } + if (meth->free_cb != NULL) + meth->free_cb(s, meth->ext_type, context, out, meth->add_arg); + } + return 1; +} + +/* Copy the flags from src to dst for any extensions that exist in both */ +int custom_exts_copy_flags_ntls(custom_ext_methods *dst, + const custom_ext_methods *src) +{ + size_t i; + custom_ext_method *methsrc = src->meths; + + for (i = 0; i < src->meths_count; i++, methsrc++) { + custom_ext_method *methdst = custom_ext_find_ntls(dst, methsrc->role, + methsrc->ext_type, NULL); + + if (methdst == NULL) + continue; + + methdst->ext_flags = methsrc->ext_flags; + } + + return 1; +} + +/* Copy table of custom extensions */ +int custom_exts_copy_ntls(custom_ext_methods *dst, const custom_ext_methods *src) +{ + size_t i; + int err = 0; + + if (src->meths_count > 0) { + dst->meths = + OPENSSL_memdup(src->meths, + sizeof(*src->meths) * src->meths_count); + if (dst->meths == NULL) + return 0; + dst->meths_count = src->meths_count; + + for (i = 0; i < src->meths_count; i++) { + custom_ext_method *methsrc = src->meths + i; + custom_ext_method *methdst = dst->meths + i; + + if (methsrc->add_cb != custom_ext_add_ntls_old_cb_wrap) + continue; + + /* + * We have found an old style API wrapper. We need to copy the + * arguments too. + */ + + if (err) { + methdst->add_arg = NULL; + methdst->parse_arg = NULL; + continue; + } + + methdst->add_arg = OPENSSL_memdup(methsrc->add_arg, + sizeof(custom_ext_add_cb_wrap)); + methdst->parse_arg = OPENSSL_memdup(methsrc->parse_arg, + sizeof(custom_ext_parse_cb_wrap)); + + if (methdst->add_arg == NULL || methdst->parse_arg == NULL) + err = 1; + } + } + + if (err) { + custom_exts_free_ntls(dst); + return 0; + } + + return 1; +} + +void custom_exts_free_ntls(custom_ext_methods *exts) +{ + size_t i; + custom_ext_method *meth; + + for (i = 0, meth = exts->meths; i < exts->meths_count; i++, meth++) { + if (meth->add_cb != custom_ext_add_ntls_old_cb_wrap) + continue; + + /* Old style API wrapper. Need to free the arguments too */ + OPENSSL_free(meth->add_arg); + OPENSSL_free(meth->parse_arg); + } + OPENSSL_free(exts->meths); +} + +/* Return true if a client custom extension exists, false otherwise */ +int SSL_CTX_has_client_custom_ext_ntls(const SSL_CTX *ctx, unsigned int ext_type) +{ + return custom_ext_find_ntls(&ctx->cert->custext, ENDPOINT_CLIENT, ext_type, + NULL) != NULL; +} + +static int add_custom_ext_intern(SSL_CTX *ctx, ENDPOINT role, + unsigned int ext_type, + unsigned int context, + SSL_custom_ext_add_cb_ex add_cb, + SSL_custom_ext_free_cb_ex free_cb, + void *add_arg, + SSL_custom_ext_parse_cb_ex parse_cb, + void *parse_arg) +{ + custom_ext_methods *exts = &ctx->cert->custext; + custom_ext_method *meth, *tmp; + + /* + * Check application error: if add_cb is not set free_cb will never be + * called. + */ + if (add_cb == NULL && free_cb != NULL) + return 0; + +#ifndef OPENSSL_NO_CT + /* + * We don't want applications registering callbacks for SCT extensions + * whilst simultaneously using the built-in SCT validation features, as + * these two things may not play well together. + */ + if (ext_type == TLSEXT_TYPE_signed_certificate_timestamp + && (context & SSL_EXT_CLIENT_HELLO) != 0 + && SSL_CTX_ct_is_enabled(ctx)) + return 0; +#endif + + /* + * Don't add if extension supported internally, but make exception + * for extension types that previously were not supported, but now are. + */ + if (SSL_extension_supported_ntls(ext_type) + && ext_type != TLSEXT_TYPE_signed_certificate_timestamp) + return 0; + + /* Extension type must fit in 16 bits */ + if (ext_type > 0xffff) + return 0; + /* Search for duplicate */ + if (custom_ext_find_ntls(exts, role, ext_type, NULL)) + return 0; + tmp = OPENSSL_realloc(exts->meths, + (exts->meths_count + 1) * sizeof(custom_ext_method)); + if (tmp == NULL) + return 0; + + exts->meths = tmp; + meth = exts->meths + exts->meths_count; + memset(meth, 0, sizeof(*meth)); + meth->role = role; + meth->context = context; + meth->parse_cb = parse_cb; + meth->add_cb = add_cb; + meth->free_cb = free_cb; + meth->ext_type = ext_type; + meth->add_arg = add_arg; + meth->parse_arg = parse_arg; + exts->meths_count++; + return 1; +} + +static int add_old_custom_ext(SSL_CTX *ctx, ENDPOINT role, + unsigned int ext_type, + unsigned int context, + custom_ext_add_cb add_cb, + custom_ext_free_cb free_cb, + void *add_arg, + custom_ext_parse_cb parse_cb, void *parse_arg) +{ + custom_ext_add_cb_wrap *add_cb_wrap + = OPENSSL_malloc(sizeof(*add_cb_wrap)); + custom_ext_parse_cb_wrap *parse_cb_wrap + = OPENSSL_malloc(sizeof(*parse_cb_wrap)); + int ret; + + if (add_cb_wrap == NULL || parse_cb_wrap == NULL) { + OPENSSL_free(add_cb_wrap); + OPENSSL_free(parse_cb_wrap); + return 0; + } + + add_cb_wrap->add_arg = add_arg; + add_cb_wrap->add_cb = add_cb; + add_cb_wrap->free_cb = free_cb; + parse_cb_wrap->parse_arg = parse_arg; + parse_cb_wrap->parse_cb = parse_cb; + + ret = add_custom_ext_intern(ctx, role, ext_type, + context, + custom_ext_add_ntls_old_cb_wrap, + custom_ext_free_old_cb_wrap, + add_cb_wrap, + custom_ext_parse_ntls_old_cb_wrap, + parse_cb_wrap); + + if (!ret) { + OPENSSL_free(add_cb_wrap); + OPENSSL_free(parse_cb_wrap); + } + + return ret; +} + +/* Application level functions to add the old custom extension callbacks */ +int SSL_CTX_add_client_custom_ext_ntls(SSL_CTX *ctx, unsigned int ext_type, + custom_ext_add_cb add_cb, + custom_ext_free_cb free_cb, + void *add_arg, + custom_ext_parse_cb parse_cb, void *parse_arg) +{ + return add_old_custom_ext(ctx, ENDPOINT_CLIENT, ext_type, + SSL_EXT_TLS1_2_AND_BELOW_ONLY + | SSL_EXT_CLIENT_HELLO + | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_IGNORE_ON_RESUMPTION, + add_cb, free_cb, add_arg, parse_cb, parse_arg); +} + +int SSL_CTX_add_server_custom_ext_ntls(SSL_CTX *ctx, unsigned int ext_type, + custom_ext_add_cb add_cb, + custom_ext_free_cb free_cb, + void *add_arg, + custom_ext_parse_cb parse_cb, void *parse_arg) +{ + return add_old_custom_ext(ctx, ENDPOINT_SERVER, ext_type, + SSL_EXT_TLS1_2_AND_BELOW_ONLY + | SSL_EXT_CLIENT_HELLO + | SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_IGNORE_ON_RESUMPTION, + add_cb, free_cb, add_arg, parse_cb, parse_arg); +} + +int SSL_CTX_add_custom_ext_ntls(SSL_CTX *ctx, unsigned int ext_type, + unsigned int context, + SSL_custom_ext_add_cb_ex add_cb, + SSL_custom_ext_free_cb_ex free_cb, + void *add_arg, + SSL_custom_ext_parse_cb_ex parse_cb, void *parse_arg) +{ + return add_custom_ext_intern(ctx, ENDPOINT_BOTH, ext_type, context, add_cb, + free_cb, add_arg, parse_cb, parse_arg); +} + +int SSL_extension_supported_ntls(unsigned int ext_type) +{ + switch (ext_type) { + /* Internally supported extensions. */ + case TLSEXT_TYPE_application_layer_protocol_negotiation: + case TLSEXT_TYPE_ec_point_formats: + case TLSEXT_TYPE_supported_groups: + case TLSEXT_TYPE_key_share: +#ifndef OPENSSL_NO_NEXTPROTONEG + case TLSEXT_TYPE_next_proto_neg: +#endif + case TLSEXT_TYPE_padding: + case TLSEXT_TYPE_renegotiate: + case TLSEXT_TYPE_max_fragment_length: + case TLSEXT_TYPE_server_name: + case TLSEXT_TYPE_session_ticket: + case TLSEXT_TYPE_signature_algorithms: + +#ifndef OPENSSL_NO_OCSP + case TLSEXT_TYPE_status_request: +#endif +#ifndef OPENSSL_NO_CT + case TLSEXT_TYPE_signed_certificate_timestamp: +#endif +#ifndef OPENSSL_NO_SRTP + case TLSEXT_TYPE_use_srtp: +#endif + case TLSEXT_TYPE_encrypt_then_mac: + case TLSEXT_TYPE_supported_versions: + case TLSEXT_TYPE_extended_master_secret: + case TLSEXT_TYPE_psk_kex_modes: + case TLSEXT_TYPE_cookie: + case TLSEXT_TYPE_early_data: + case TLSEXT_TYPE_certificate_authorities: + case TLSEXT_TYPE_psk: + case TLSEXT_TYPE_post_handshake_auth: + return 1; + default: + return 0; + } +} + diff --git a/openssl/src/ssl/statem_ntls/ntls_extensions_srvr.c b/openssl/src/ssl/statem_ntls/ntls_extensions_srvr.c new file mode 100644 index 000000000..7d8aacde4 --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_extensions_srvr.c @@ -0,0 +1,1609 @@ +/* + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include "ntls_ssl_local.h" +#include "ntls_statem_local.h" +#include "internal/cryptlib.h" + +#define COOKIE_STATE_FORMAT_VERSION 0 + +/* + * 2 bytes for packet length, 2 bytes for format version, 2 bytes for + * protocol version, 2 bytes for group id, 2 bytes for cipher id, 1 byte for + * key_share present flag, 4 bytes for timestamp, 2 bytes for the hashlen, + * EVP_MAX_MD_SIZE for transcript hash, 1 byte for app cookie length, app cookie + * length bytes, SHA256_DIGEST_LENGTH bytes for the HMAC of the whole thing. + */ +#define MAX_COOKIE_SIZE (2 + 2 + 2 + 2 + 2 + 1 + 4 + 2 + EVP_MAX_MD_SIZE + 1 \ + + SSL_COOKIE_LENGTH + SHA256_DIGEST_LENGTH) + +/* + * Message header + 2 bytes for protocol version + number of random bytes + + * + 1 byte for legacy session id length + number of bytes in legacy session id + * + 2 bytes for ciphersuite + 1 byte for legacy compression + * + 2 bytes for extension block length + 6 bytes for key_share extension + * + 4 bytes for cookie extension header + the number of bytes in the cookie + */ +#define MAX_HRR_SIZE (SSL3_HM_HEADER_LENGTH + 2 + SSL3_RANDOM_SIZE + 1 \ + + SSL_MAX_SSL_SESSION_ID_LENGTH + 2 + 1 + 2 + 6 + 4 \ + + MAX_COOKIE_SIZE) + + +/*- + * The servername extension is treated as follows: + * + * - Only the hostname type is supported with a maximum length of 255. + * - The servername is rejected if too long or if it contains zeros, + * in which case an fatal alert is generated. + * - The servername field is maintained together with the session cache. + * - When a session is resumed, the servername call back invoked in order + * to allow the application to position itself to the right context. + * - The servername is acknowledged if it is new for a session or when + * it is identical to a previously used for the same session. + * Applications can control the behaviour. They can at any time + * set a 'desirable' servername for a new SSL object. This can be the + * case for example with HTTPS when a Host: header field is received and + * a renegotiation is requested. In this case, a possible servername + * presented in the new client hello is only acknowledged if it matches + * the value of the Host: field. + * - Applications must use SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION + * if they provide for changing an explicit servername context for the + * session, i.e. when the session has been established with a servername + * extension. + * - On session reconnect, the servername extension may be absent. + */ +int tls_parse_ctos_server_name_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + unsigned int servname_type; + PACKET sni, hostname; + + if (!PACKET_as_length_prefixed_2(pkt, &sni) + /* ServerNameList must be at least 1 byte long. */ + || PACKET_remaining(&sni) == 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + /* + * Although the intent was for server_name to be extensible, RFC 4366 + * was not clear about it; and so OpenSSL among other implementations, + * always and only allows a 'host_name' name types. + * RFC 6066 corrected the mistake but adding new name types + * is nevertheless no longer feasible, so act as if no other + * SNI types can exist, to simplify parsing. + * + * Also note that the RFC permits only one SNI value per type, + * i.e., we can only have a single hostname. + */ + if (!PACKET_get_1(&sni, &servname_type) + || servname_type != TLSEXT_NAMETYPE_host_name + || !PACKET_as_length_prefixed_2(&sni, &hostname)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + /* + * In TLSv1.2 and below the SNI is associated with the session. In TLSv1.3 + * we always use the SNI value from the handshake. + */ + if (!s->hit || SSL_IS_TLS13(s)) { + if (PACKET_remaining(&hostname) > TLSEXT_MAXLEN_host_name) { + SSLfatal_ntls(s, SSL_AD_UNRECOGNIZED_NAME, SSL_R_BAD_EXTENSION); + return 0; + } + + if (PACKET_contains_zero_byte(&hostname)) { + SSLfatal_ntls(s, SSL_AD_UNRECOGNIZED_NAME, SSL_R_BAD_EXTENSION); + return 0; + } + + /* + * Store the requested SNI in the SSL as temporary storage. + * If we accept it, it will get stored in the SSL_SESSION as well. + */ + OPENSSL_free(s->ext.hostname); + s->ext.hostname = NULL; + if (!PACKET_strndup(&hostname, &s->ext.hostname)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + s->servername_done = 1; + } else { + /* + * In TLSv1.2 and below we should check if the SNI is consistent between + * the initial handshake and the resumption. In TLSv1.3 SNI is not + * associated with the session. + */ + s->servername_done = (s->session->ext.hostname != NULL) + && PACKET_equal(&hostname, s->session->ext.hostname, + strlen(s->session->ext.hostname)); + } + + return 1; +} + +int tls_parse_ctos_maxfragmentlen_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + unsigned int value; + + if (PACKET_remaining(pkt) != 1 || !PACKET_get_1(pkt, &value)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + /* Received |value| should be a valid max-fragment-length code. */ + if (!IS_MAX_FRAGMENT_LENGTH_EXT_VALID(value)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH); + return 0; + } + + /* + * RFC 6066: The negotiated length applies for the duration of the session + * including session resumptions. + * We should receive the same code as in resumed session ! + */ + if (s->hit && s->session->ext.max_fragment_len_mode != value) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH); + return 0; + } + + /* + * Store it in session, so it'll become binding for us + * and we'll include it in a next Server Hello. + */ + s->session->ext.max_fragment_len_mode = value; + return 1; +} + +int tls_parse_ctos_ec_pt_formats_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + /* Ignore ec_point_formats */ + return 1; +} + +int tls_parse_ctos_session_ticket_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->ext.session_ticket_cb && + !s->ext.session_ticket_cb(s, PACKET_data(pkt), + PACKET_remaining(pkt), + s->ext.session_ticket_cb_arg)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +int tls_parse_ctos_sig_algs_cert_ntls(SSL *s, PACKET *pkt, + ossl_unused unsigned int context, + ossl_unused X509 *x, + ossl_unused size_t chainidx) +{ + /* Ignore signature_algorithms_cert */ + return 1; +} + +int tls_parse_ctos_sig_algs_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + /* Ignore signature_algorithms */ + return 1; +} + +#ifndef OPENSSL_NO_OCSP +int tls_parse_ctos_status_request_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + PACKET responder_id_list, exts; + + /* We ignore this in a resumption handshake */ + if (s->hit) + return 1; + + /* Not defined if we get one of these in a client Certificate */ + if (x != NULL) + return 1; + + if (!PACKET_get_1(pkt, (unsigned int *)&s->ext.status_type)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + if (s->ext.status_type != TLSEXT_STATUSTYPE_ocsp) { + /* + * We don't know what to do with any other type so ignore it. + */ + s->ext.status_type = TLSEXT_STATUSTYPE_nothing; + return 1; + } + + if (!PACKET_get_length_prefixed_2 (pkt, &responder_id_list)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + /* + * We remove any OCSP_RESPIDs from a previous handshake + * to prevent unbounded memory growth - CVE-2016-6304 + */ + sk_OCSP_RESPID_pop_free(s->ext.ocsp.ids, OCSP_RESPID_free); + if (PACKET_remaining(&responder_id_list) > 0) { + s->ext.ocsp.ids = sk_OCSP_RESPID_new_null(); + if (s->ext.ocsp.ids == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + } else { + s->ext.ocsp.ids = NULL; + } + + while (PACKET_remaining(&responder_id_list) > 0) { + OCSP_RESPID *id; + PACKET responder_id; + const unsigned char *id_data; + + if (!PACKET_get_length_prefixed_2(&responder_id_list, &responder_id) + || PACKET_remaining(&responder_id) == 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + id_data = PACKET_data(&responder_id); + id = d2i_OCSP_RESPID(NULL, &id_data, + (int)PACKET_remaining(&responder_id)); + if (id == NULL) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + if (id_data != PACKET_end(&responder_id)) { + OCSP_RESPID_free(id); + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + + return 0; + } + + if (!sk_OCSP_RESPID_push(s->ext.ocsp.ids, id)) { + OCSP_RESPID_free(id); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + + return 0; + } + } + + /* Read in request_extensions */ + if (!PACKET_as_length_prefixed_2(pkt, &exts)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + if (PACKET_remaining(&exts) > 0) { + const unsigned char *ext_data = PACKET_data(&exts); + + sk_X509_EXTENSION_pop_free(s->ext.ocsp.exts, + X509_EXTENSION_free); + s->ext.ocsp.exts = + d2i_X509_EXTENSIONS(NULL, &ext_data, (int)PACKET_remaining(&exts)); + if (s->ext.ocsp.exts == NULL || ext_data != PACKET_end(&exts)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + } + + return 1; +} +#endif + +#ifndef OPENSSL_NO_NEXTPROTONEG +int tls_parse_ctos_npn_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + /* + * We shouldn't accept this extension on a + * renegotiation. + */ + if (SSL_IS_FIRST_HANDSHAKE(s)) + s->s3.npn_seen = 1; + + return 1; +} +#endif + +/* + * Save the ALPN extension in a ClientHello.|pkt| holds the contents of the ALPN + * extension, not including type and length. Returns: 1 on success, 0 on error. + */ +int tls_parse_ctos_alpn_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + PACKET protocol_list, save_protocol_list, protocol; + + if (!SSL_IS_FIRST_HANDSHAKE(s)) + return 1; + + if (!PACKET_as_length_prefixed_2(pkt, &protocol_list) + || PACKET_remaining(&protocol_list) < 2) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + save_protocol_list = protocol_list; + do { + /* Protocol names can't be empty. */ + if (!PACKET_get_length_prefixed_1(&protocol_list, &protocol) + || PACKET_remaining(&protocol) == 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + } while (PACKET_remaining(&protocol_list) != 0); + + OPENSSL_free(s->s3.alpn_proposed); + s->s3.alpn_proposed = NULL; + s->s3.alpn_proposed_len = 0; + if (!PACKET_memdup(&save_protocol_list, + &s->s3.alpn_proposed, &s->s3.alpn_proposed_len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +#ifndef OPENSSL_NO_SRTP +int tls_parse_ctos_use_srtp_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + STACK_OF(SRTP_PROTECTION_PROFILE) *srvr; + unsigned int ct, mki_len, id; + int i, srtp_pref; + PACKET subpkt; + + /* Ignore this if we have no SRTP profiles */ + if (SSL_get_srtp_profiles(s) == NULL) + return 1; + + /* Pull off the length of the cipher suite list and check it is even */ + if (!PACKET_get_net_2(pkt, &ct) || (ct & 1) != 0 + || !PACKET_get_sub_packet(pkt, &subpkt, ct)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, + SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST); + return 0; + } + + srvr = SSL_get_srtp_profiles(s); + s->srtp_profile = NULL; + /* Search all profiles for a match initially */ + srtp_pref = sk_SRTP_PROTECTION_PROFILE_num(srvr); + + while (PACKET_remaining(&subpkt)) { + if (!PACKET_get_net_2(&subpkt, &id)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, + SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST); + return 0; + } + + /* + * Only look for match in profiles of higher preference than + * current match. + * If no profiles have been have been configured then this + * does nothing. + */ + for (i = 0; i < srtp_pref; i++) { + SRTP_PROTECTION_PROFILE *sprof = + sk_SRTP_PROTECTION_PROFILE_value(srvr, i); + + if (sprof->id == id) { + s->srtp_profile = sprof; + srtp_pref = i; + break; + } + } + } + + /* Now extract the MKI value as a sanity check, but discard it for now */ + if (!PACKET_get_1(pkt, &mki_len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, + SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST); + return 0; + } + + if (!PACKET_forward(pkt, mki_len) + || PACKET_remaining(pkt)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_SRTP_MKI_VALUE); + return 0; + } + + return 1; +} +#endif + +int tls_parse_ctos_etm_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + /* Ignore encrypt-then-MAC */ + return 1; +} + +/* + * Process a psk_kex_modes extension received in the ClientHello. |pkt| contains + * the raw PACKET data for the extension. Returns 1 on success or 0 on failure. + */ +int tls_parse_ctos_psk_kex_modes_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + PACKET psk_kex_modes; + unsigned int mode; + + if (!PACKET_as_length_prefixed_1(pkt, &psk_kex_modes) + || PACKET_remaining(&psk_kex_modes) == 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + while (PACKET_get_1(&psk_kex_modes, &mode)) { + if (mode == TLSEXT_KEX_MODE_KE_DHE) + s->ext.psk_kex_mode |= TLSEXT_KEX_MODE_FLAG_KE_DHE; + else if (mode == TLSEXT_KEX_MODE_KE + && (s->options & SSL_OP_ALLOW_NO_DHE_KEX) != 0) + s->ext.psk_kex_mode |= TLSEXT_KEX_MODE_FLAG_KE; + } +#endif + + return 1; +} + +/* + * Process a key_share extension received in the ClientHello. |pkt| contains + * the raw PACKET data for the extension. Returns 1 on success or 0 on failure. + */ +int tls_parse_ctos_key_share_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + unsigned int group_id; + PACKET key_share_list, encoded_pt; + const uint16_t *clntgroups, *srvrgroups; + size_t clnt_num_groups, srvr_num_groups; + int found = 0; + + if (s->hit && (s->ext.psk_kex_mode & TLSEXT_KEX_MODE_FLAG_KE_DHE) == 0) + return 1; + + /* Sanity check */ + if (s->s3.peer_tmp != NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!PACKET_as_length_prefixed_2(pkt, &key_share_list)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + /* Get our list of supported groups */ + tls1_get_supported_groups(s, &srvrgroups, &srvr_num_groups); + /* Get the clients list of supported groups. */ + tls1_get_peer_groups(s, &clntgroups, &clnt_num_groups); + if (clnt_num_groups == 0) { + /* + * This can only happen if the supported_groups extension was not sent, + * because we verify that the length is non-zero when we process that + * extension. + */ + SSLfatal_ntls(s, SSL_AD_MISSING_EXTENSION, + SSL_R_MISSING_SUPPORTED_GROUPS_EXTENSION); + return 0; + } + + if (s->s3.group_id != 0 && PACKET_remaining(&key_share_list) == 0) { + /* + * If we set a group_id already, then we must have sent an HRR + * requesting a new key_share. If we haven't got one then that is an + * error + */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + return 0; + } + + while (PACKET_remaining(&key_share_list) > 0) { + if (!PACKET_get_net_2(&key_share_list, &group_id) + || !PACKET_get_length_prefixed_2(&key_share_list, &encoded_pt) + || PACKET_remaining(&encoded_pt) == 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + /* + * If we already found a suitable key_share we loop through the + * rest to verify the structure, but don't process them. + */ + if (found) + continue; + + /* + * If we sent an HRR then the key_share sent back MUST be for the group + * we requested, and must be the only key_share sent. + */ + if (s->s3.group_id != 0 + && (group_id != s->s3.group_id + || PACKET_remaining(&key_share_list) != 0)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + return 0; + } + + /* Check if this share is in supported_groups sent from client */ + if (!check_in_list_ntls(s, group_id, clntgroups, clnt_num_groups, 0)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_KEY_SHARE); + return 0; + } + + /* Check if this share is for a group we can use */ + if (!check_in_list_ntls(s, group_id, srvrgroups, srvr_num_groups, 1)) { + /* Share not suitable */ + continue; + } + + if ((s->s3.peer_tmp = ssl_generate_param_group(s, group_id)) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_UNABLE_TO_FIND_ECDH_PARAMETERS); + return 0; + } + + s->s3.group_id = group_id; + /* Cache the selected group ID in the SSL_SESSION */ + s->session->kex_group = group_id; + + if (tls13_set_encoded_pub_key(s->s3.peer_tmp, + PACKET_data(&encoded_pt), + PACKET_remaining(&encoded_pt)) <= 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_ECPOINT); + return 0; + } + + found = 1; + } +#endif + + return 1; +} + +int tls_parse_ctos_cookie_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + unsigned int format, version, key_share, group_id; + EVP_MD_CTX *hctx; + EVP_PKEY *pkey; + PACKET cookie, raw, chhash, appcookie; + WPACKET hrrpkt; + const unsigned char *data, *mdin, *ciphdata; + unsigned char hmac[SHA256_DIGEST_LENGTH]; + unsigned char hrr[MAX_HRR_SIZE]; + size_t rawlen, hmaclen, hrrlen, ciphlen; + unsigned long tm, now; + + /* Ignore any cookie if we're not set up to verify it */ + if (s->ctx->verify_stateless_cookie_cb == NULL + || (s->s3.flags & TLS1_FLAGS_STATELESS) == 0) + return 1; + + if (!PACKET_as_length_prefixed_2(pkt, &cookie)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + raw = cookie; + data = PACKET_data(&raw); + rawlen = PACKET_remaining(&raw); + if (rawlen < SHA256_DIGEST_LENGTH + || !PACKET_forward(&raw, rawlen - SHA256_DIGEST_LENGTH)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + mdin = PACKET_data(&raw); + + /* Verify the HMAC of the cookie */ + hctx = EVP_MD_CTX_create(); + pkey = EVP_PKEY_new_raw_private_key_ex(s->ctx->libctx, "HMAC", + s->ctx->propq, + s->session_ctx->ext.cookie_hmac_key, + sizeof(s->session_ctx->ext.cookie_hmac_key)); + if (hctx == NULL || pkey == NULL) { + EVP_MD_CTX_free(hctx); + EVP_PKEY_free(pkey); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + + hmaclen = SHA256_DIGEST_LENGTH; + if (EVP_DigestSignInit_ex(hctx, NULL, "SHA2-256", s->ctx->libctx, + s->ctx->propq, pkey, NULL) <= 0 + || EVP_DigestSign(hctx, hmac, &hmaclen, data, + rawlen - SHA256_DIGEST_LENGTH) <= 0 + || hmaclen != SHA256_DIGEST_LENGTH) { + EVP_MD_CTX_free(hctx); + EVP_PKEY_free(pkey); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + EVP_MD_CTX_free(hctx); + EVP_PKEY_free(pkey); + + if (CRYPTO_memcmp(hmac, mdin, SHA256_DIGEST_LENGTH) != 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_COOKIE_MISMATCH); + return 0; + } + + if (!PACKET_get_net_2(&cookie, &format)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + /* Check the cookie format is something we recognise. Ignore it if not */ + if (format != COOKIE_STATE_FORMAT_VERSION) + return 1; + + /* + * The rest of these checks really shouldn't fail since we have verified the + * HMAC above. + */ + + /* Check the version number is sane */ + if (!PACKET_get_net_2(&cookie, &version)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + if (version != TLS1_3_VERSION) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_BAD_PROTOCOL_VERSION_NUMBER); + return 0; + } + + if (!PACKET_get_net_2(&cookie, &group_id)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + ciphdata = PACKET_data(&cookie); + if (!PACKET_forward(&cookie, 2)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + if (group_id != s->s3.group_id + || s->s3.tmp.new_cipher + != ssl_get_cipher_by_char(s, ciphdata, 0)) { + /* + * We chose a different cipher or group id this time around to what is + * in the cookie. Something must have changed. + */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_CIPHER); + return 0; + } + + if (!PACKET_get_1(&cookie, &key_share) + || !PACKET_get_net_4(&cookie, &tm) + || !PACKET_get_length_prefixed_2(&cookie, &chhash) + || !PACKET_get_length_prefixed_1(&cookie, &appcookie) + || PACKET_remaining(&cookie) != SHA256_DIGEST_LENGTH) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + /* We tolerate a cookie age of up to 10 minutes (= 60 * 10 seconds) */ + now = (unsigned long)time(NULL); + if (tm > now || (now - tm) > 600) { + /* Cookie is stale. Ignore it */ + return 1; + } + + /* Verify the app cookie */ + if (s->ctx->verify_stateless_cookie_cb(s, PACKET_data(&appcookie), + PACKET_remaining(&appcookie)) == 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_COOKIE_MISMATCH); + return 0; + } + + /* + * Reconstruct the HRR that we would have sent in response to the original + * ClientHello so we can add it to the transcript hash. + * Note: This won't work with custom HRR extensions + */ + if (!WPACKET_init_static_len(&hrrpkt, hrr, sizeof(hrr), 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if (!WPACKET_put_bytes_u8(&hrrpkt, SSL3_MT_SERVER_HELLO) + || !WPACKET_start_sub_packet_u24(&hrrpkt) + || !WPACKET_put_bytes_u16(&hrrpkt, TLS1_2_VERSION) + || !WPACKET_memcpy(&hrrpkt, hrrrandom_ntls, SSL3_RANDOM_SIZE) + || !WPACKET_sub_memcpy_u8(&hrrpkt, s->tmp_session_id, + s->tmp_session_id_len) + || !s->method->put_cipher_by_char(s->s3.tmp.new_cipher, &hrrpkt, + &ciphlen) + || !WPACKET_put_bytes_u8(&hrrpkt, 0) + || !WPACKET_start_sub_packet_u16(&hrrpkt)) { + WPACKET_cleanup(&hrrpkt); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if (!WPACKET_put_bytes_u16(&hrrpkt, TLSEXT_TYPE_supported_versions) + || !WPACKET_start_sub_packet_u16(&hrrpkt) + || !WPACKET_put_bytes_u16(&hrrpkt, s->version) + || !WPACKET_close(&hrrpkt)) { + WPACKET_cleanup(&hrrpkt); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if (key_share) { + if (!WPACKET_put_bytes_u16(&hrrpkt, TLSEXT_TYPE_key_share) + || !WPACKET_start_sub_packet_u16(&hrrpkt) + || !WPACKET_put_bytes_u16(&hrrpkt, s->s3.group_id) + || !WPACKET_close(&hrrpkt)) { + WPACKET_cleanup(&hrrpkt); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + if (!WPACKET_put_bytes_u16(&hrrpkt, TLSEXT_TYPE_cookie) + || !WPACKET_start_sub_packet_u16(&hrrpkt) + || !WPACKET_sub_memcpy_u16(&hrrpkt, data, rawlen) + || !WPACKET_close(&hrrpkt) /* cookie extension */ + || !WPACKET_close(&hrrpkt) /* extension block */ + || !WPACKET_close(&hrrpkt) /* message */ + || !WPACKET_get_total_written(&hrrpkt, &hrrlen) + || !WPACKET_finish(&hrrpkt)) { + WPACKET_cleanup(&hrrpkt); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* Reconstruct the transcript hash */ + if (!create_synthetic_message_hash_ntls(s, PACKET_data(&chhash), + PACKET_remaining(&chhash), hrr, + hrrlen)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + /* Act as if this ClientHello came after a HelloRetryRequest */ + s->hello_retry_request = 1; + + s->ext.cookieok = 1; +#endif + + return 1; +} + +int tls_parse_ctos_supported_groups_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + /* Ignore supported_groups */ + return 1; +} + +int tls_parse_ctos_ems_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + /* Ignore extended_master_secret */ + return 1; +} + + +int tls_parse_ctos_early_data_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + if (s->hello_retry_request != SSL_HRR_NONE) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_EXTENSION); + return 0; + } + + return 1; +} + +static SSL_TICKET_STATUS tls_get_stateful_ticket(SSL *s, PACKET *tick, + SSL_SESSION **sess) +{ + SSL_SESSION *tmpsess = NULL; + + s->ext.ticket_expected = 1; + + switch (PACKET_remaining(tick)) { + case 0: + return SSL_TICKET_EMPTY; + + case SSL_MAX_SSL_SESSION_ID_LENGTH: + break; + + default: + return SSL_TICKET_NO_DECRYPT; + } + + tmpsess = lookup_sess_in_cache(s, PACKET_data(tick), + SSL_MAX_SSL_SESSION_ID_LENGTH); + + if (tmpsess == NULL) + return SSL_TICKET_NO_DECRYPT; + + *sess = tmpsess; + return SSL_TICKET_SUCCESS; +} + +int tls_parse_ctos_psk_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx) +{ + PACKET identities, binders, binder; + size_t binderoffset, hashsize; + SSL_SESSION *sess = NULL; + unsigned int id, i, ext = 0; + const EVP_MD *md = NULL; + + /* + * If we have no PSK kex mode that we recognise then we can't resume so + * ignore this extension + */ + if ((s->ext.psk_kex_mode + & (TLSEXT_KEX_MODE_FLAG_KE | TLSEXT_KEX_MODE_FLAG_KE_DHE)) == 0) + return 1; + + if (!PACKET_get_length_prefixed_2(pkt, &identities)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + s->ext.ticket_expected = 0; + for (id = 0; PACKET_remaining(&identities) != 0; id++) { + PACKET identity; + unsigned long ticket_agel; + size_t idlen; + + if (!PACKET_get_length_prefixed_2(&identities, &identity) + || !PACKET_get_net_4(&identities, &ticket_agel)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + idlen = PACKET_remaining(&identity); + if (s->psk_find_session_cb != NULL + && !s->psk_find_session_cb(s, PACKET_data(&identity), idlen, + &sess)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + +#ifndef OPENSSL_NO_PSK + if(sess == NULL + && s->psk_server_callback != NULL + && idlen <= PSK_MAX_IDENTITY_LEN) { + char *pskid = NULL; + unsigned char pskdata[PSK_MAX_PSK_LEN]; + unsigned int pskdatalen; + + if (!PACKET_strndup(&identity, &pskid)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + pskdatalen = s->psk_server_callback(s, pskid, pskdata, + sizeof(pskdata)); + OPENSSL_free(pskid); + if (pskdatalen > PSK_MAX_PSK_LEN) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } else if (pskdatalen > 0) { + const SSL_CIPHER *cipher; + const unsigned char tls13_aes128gcmsha256_id[] = { 0x13, 0x01 }; + + /* + * We found a PSK using an old style callback. We don't know + * the digest so we default to SHA256 as per the TLSv1.3 spec + */ + cipher = SSL_CIPHER_find(s, tls13_aes128gcmsha256_id); + if (cipher == NULL) { + OPENSSL_cleanse(pskdata, pskdatalen); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + sess = SSL_SESSION_new(); + if (sess == NULL + || !SSL_SESSION_set1_master_key(sess, pskdata, + pskdatalen) + || !SSL_SESSION_set_cipher(sess, cipher) + || !SSL_SESSION_set_protocol_version(sess, + TLS1_3_VERSION)) { + OPENSSL_cleanse(pskdata, pskdatalen); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + OPENSSL_cleanse(pskdata, pskdatalen); + } + } +#endif /* OPENSSL_NO_PSK */ + + if (sess != NULL) { + /* We found a PSK */ + SSL_SESSION *sesstmp = ssl_session_dup(sess, 0); + + if (sesstmp == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + SSL_SESSION_free(sess); + sess = sesstmp; + + /* + * We've just been told to use this session for this context so + * make sure the sid_ctx matches up. + */ + memcpy(sess->sid_ctx, s->sid_ctx, s->sid_ctx_length); + sess->sid_ctx_length = s->sid_ctx_length; + ext = 1; + if (id == 0) + s->ext.early_data_ok = 1; + s->ext.ticket_expected = 1; + } else { + uint32_t ticket_age = 0, now, agesec, agems; + int ret; + + /* + * If we are using anti-replay protection then we behave as if + * SSL_OP_NO_TICKET is set - we are caching tickets anyway so there + * is no point in using full stateless tickets. + */ + if ((s->options & SSL_OP_NO_TICKET) != 0 + || (s->max_early_data > 0 + && (s->options & SSL_OP_NO_ANTI_REPLAY) == 0)) + ret = tls_get_stateful_ticket(s, &identity, &sess); + else + ret = tls_decrypt_ticket(s, PACKET_data(&identity), + PACKET_remaining(&identity), NULL, 0, + &sess); + + if (ret == SSL_TICKET_EMPTY) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + return 0; + } + + if (ret == SSL_TICKET_FATAL_ERR_MALLOC + || ret == SSL_TICKET_FATAL_ERR_OTHER) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if (ret == SSL_TICKET_NONE || ret == SSL_TICKET_NO_DECRYPT) + continue; + + /* Check for replay */ + if (s->max_early_data > 0 + && (s->options & SSL_OP_NO_ANTI_REPLAY) == 0 + && !SSL_CTX_remove_session(s->session_ctx, sess)) { + SSL_SESSION_free(sess); + sess = NULL; + continue; + } + + ticket_age = (uint32_t)ticket_agel; + now = (uint32_t)time(NULL); + agesec = now - (uint32_t)sess->time; + agems = agesec * (uint32_t)1000; + ticket_age -= sess->ext.tick_age_add; + + /* + * For simplicity we do our age calculations in seconds. If the + * client does it in ms then it could appear that their ticket age + * is longer than ours (our ticket age calculation should always be + * slightly longer than the client's due to the network latency). + * Therefore we add 1000ms to our age calculation to adjust for + * rounding errors. + */ + if (id == 0 + && sess->timeout >= (long)agesec + && agems / (uint32_t)1000 == agesec + && ticket_age <= agems + 1000 + && ticket_age + TICKET_AGE_ALLOWANCE >= agems + 1000) { + /* + * Ticket age is within tolerance and not expired. We allow it + * for early data + */ + s->ext.early_data_ok = 1; + } + } + + md = ssl_md(s->ctx, sess->cipher->algorithm2); + if (!EVP_MD_is_a(md, + EVP_MD_get0_name(ssl_md(s->ctx, + s->s3.tmp.new_cipher->algorithm2)))) { + /* The ciphersuite is not compatible with this session. */ + SSL_SESSION_free(sess); + sess = NULL; + s->ext.early_data_ok = 0; + s->ext.ticket_expected = 0; + continue; + } + break; + } + + if (sess == NULL) + return 1; + + binderoffset = PACKET_data(pkt) - (const unsigned char *)s->init_buf->data; + hashsize = EVP_MD_get_size(md); + + if (!PACKET_get_length_prefixed_2(pkt, &binders)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + goto err; + } + + for (i = 0; i <= id; i++) { + if (!PACKET_get_length_prefixed_1(&binders, &binder)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + goto err; + } + } + + if (PACKET_remaining(&binder) != hashsize) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_EXTENSION); + goto err; + } + if (tls_psk_do_binder_ntls(s, md, (const unsigned char *)s->init_buf->data, + binderoffset, PACKET_data(&binder), NULL, sess, 0, + ext) != 1) { + /* SSLfatal_ntls() already called */ + goto err; + } + + s->ext.tick_identity = id; + + SSL_SESSION_free(s->session); + s->session = sess; + return 1; +err: + SSL_SESSION_free(sess); + return 0; +} + +int tls_parse_ctos_post_handshake_auth_ntls(SSL *s, PACKET *pkt, + ossl_unused unsigned int context, + ossl_unused X509 *x, + ossl_unused size_t chainidx) +{ + if (PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, + SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR); + return 0; + } + + s->post_handshake_auth = SSL_PHA_EXT_RECEIVED; + + return 1; +} + +EXT_RETURN tls_construct_stoc_server_name_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (s->servername_done != 1) + return EXT_RETURN_NOT_SENT; + + /* + * Prior to TLSv1.3 we ignore any SNI in the current handshake if resuming. + * We just use the servername from the initial handshake. + */ + if (s->hit && !SSL_IS_TLS13(s)) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_server_name) + || !WPACKET_put_bytes_u16(pkt, 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +/* Add/include the server's max fragment len extension into ServerHello */ +EXT_RETURN tls_construct_stoc_maxfragmentlen_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (!USE_MAX_FRAGMENT_LENGTH_EXT(s->session)) + return EXT_RETURN_NOT_SENT; + + /*- + * 4 bytes for this extension type and extension length + * 1 byte for the Max Fragment Length code value. + */ + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_max_fragment_length) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u8(pkt, s->session->ext.max_fragment_len_mode) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +EXT_RETURN tls_construct_stoc_ec_pt_formats_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + /* No ec_point_formats */ + return EXT_RETURN_NOT_SENT; +} + +EXT_RETURN tls_construct_stoc_supported_groups_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + /* No supported_groups */ + return EXT_RETURN_NOT_SENT; +} + +EXT_RETURN tls_construct_stoc_session_ticket_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (!s->ext.ticket_expected || !tls_use_ticket(s)) { + s->ext.ticket_expected = 0; + return EXT_RETURN_NOT_SENT; + } + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_session_ticket) + || !WPACKET_put_bytes_u16(pkt, 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +#ifndef OPENSSL_NO_OCSP +EXT_RETURN tls_construct_stoc_status_request_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + /* We don't currently support this extension inside a CertificateRequest */ + if (context == SSL_EXT_TLS1_3_CERTIFICATE_REQUEST) + return EXT_RETURN_NOT_SENT; + + if (!s->ext.status_expected) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_status_request) + || !WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + /* + * In TLSv1.3 we include the certificate status itself. In <= TLSv1.2 we + * send back an empty extension, with the certificate status appearing as a + * separate message + */ + if (!WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} +#endif + +#ifndef OPENSSL_NO_NEXTPROTONEG +EXT_RETURN tls_construct_stoc_next_proto_neg_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + const unsigned char *npa; + unsigned int npalen; + int ret; + int npn_seen = s->s3.npn_seen; + + s->s3.npn_seen = 0; + if (!npn_seen || s->ctx->ext.npn_advertised_cb == NULL) + return EXT_RETURN_NOT_SENT; + + ret = s->ctx->ext.npn_advertised_cb(s, &npa, &npalen, + s->ctx->ext.npn_advertised_cb_arg); + if (ret == SSL_TLSEXT_ERR_OK) { + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_next_proto_neg) + || !WPACKET_sub_memcpy_u16(pkt, npa, npalen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + s->s3.npn_seen = 1; + } + + return EXT_RETURN_SENT; +} +#endif + +EXT_RETURN tls_construct_stoc_alpn_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (s->s3.alpn_selected == NULL) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, + TLSEXT_TYPE_application_layer_protocol_negotiation) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_sub_memcpy_u8(pkt, s->s3.alpn_selected, + s->s3.alpn_selected_len) + || !WPACKET_close(pkt) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +#ifndef OPENSSL_NO_SRTP +EXT_RETURN tls_construct_stoc_use_srtp_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (s->srtp_profile == NULL) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_use_srtp) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u16(pkt, 2) + || !WPACKET_put_bytes_u16(pkt, s->srtp_profile->id) + || !WPACKET_put_bytes_u8(pkt, 0) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} +#endif + +EXT_RETURN tls_construct_stoc_etm_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + /* No encrypt-then-MAC */ + return EXT_RETURN_NOT_SENT; +} + +EXT_RETURN tls_construct_stoc_ems_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + /* No extended_master_secret */ + return EXT_RETURN_NOT_SENT; +} + +EXT_RETURN tls_construct_stoc_supported_versions_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + /* No supported_versions */ + return EXT_RETURN_NOT_SENT; +} + +EXT_RETURN tls_construct_stoc_key_share_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + unsigned char *encodedPoint; + size_t encoded_pt_len = 0; + EVP_PKEY *ckey = s->s3.peer_tmp, *skey = NULL; + const TLS_GROUP_INFO *ginf = NULL; + + if (s->hello_retry_request == SSL_HRR_PENDING) { + if (ckey != NULL) { + /* Original key_share was acceptable so don't ask for another one */ + return EXT_RETURN_NOT_SENT; + } + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_key_share) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u16(pkt, s->s3.group_id) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; + } + + if (ckey == NULL) { + /* No key_share received from client - must be resuming */ + if (!s->hit || !tls13_generate_handshake_secret(s, NULL, 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + return EXT_RETURN_NOT_SENT; + } + if (s->hit && (s->ext.psk_kex_mode & TLSEXT_KEX_MODE_FLAG_KE_DHE) == 0) { + /* + * PSK ('hit') and explicitly not doing DHE (if the client sent the + * DHE option we always take it); don't send key share. + */ + return EXT_RETURN_NOT_SENT; + } + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_key_share) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u16(pkt, s->s3.group_id)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + if ((ginf = tls1_group_id_lookup(s->ctx, s->s3.group_id)) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + if (!ginf->is_kem) { + /* Regular KEX */ + skey = ssl_generate_pkey(s, ckey); + if (skey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return EXT_RETURN_FAIL; + } + + /* Generate encoding of server key */ + encoded_pt_len = EVP_PKEY_get1_encoded_public_key(skey, &encodedPoint); + if (encoded_pt_len == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EC_LIB); + EVP_PKEY_free(skey); + return EXT_RETURN_FAIL; + } + + if (!WPACKET_sub_memcpy_u16(pkt, encodedPoint, encoded_pt_len) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + EVP_PKEY_free(skey); + OPENSSL_free(encodedPoint); + return EXT_RETURN_FAIL; + } + OPENSSL_free(encodedPoint); + + /* + * This causes the crypto state to be updated based on the derived keys + */ + s->s3.tmp.pkey = skey; + if (ssl_derive(s, skey, ckey, 1) == 0) { + /* SSLfatal_ntls() already called */ + return EXT_RETURN_FAIL; + } + } else { + /* KEM mode */ + unsigned char *ct = NULL; + size_t ctlen = 0; + + /* + * This does not update the crypto state. + * + * The generated pms is stored in `s->s3.tmp.pms` to be later used via + * ssl_gensecret(). + */ + if (ssl_encapsulate(s, ckey, &ct, &ctlen, 0) == 0) { + /* SSLfatal_ntls() already called */ + return EXT_RETURN_FAIL; + } + + if (ctlen == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + OPENSSL_free(ct); + return EXT_RETURN_FAIL; + } + + if (!WPACKET_sub_memcpy_u16(pkt, ct, ctlen) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + OPENSSL_free(ct); + return EXT_RETURN_FAIL; + } + OPENSSL_free(ct); + + /* + * This causes the crypto state to be updated based on the generated pms + */ + if (ssl_gensecret(s, s->s3.tmp.pms, s->s3.tmp.pmslen) == 0) { + /* SSLfatal_ntls() already called */ + return EXT_RETURN_FAIL; + } + } + s->s3.did_kex = 1; + return EXT_RETURN_SENT; +#else + return EXT_RETURN_FAIL; +#endif +} + +EXT_RETURN tls_construct_stoc_cookie_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ +#ifndef OPENSSL_NO_TLS1_3 + unsigned char *hashval1, *hashval2, *appcookie1, *appcookie2, *cookie; + unsigned char *hmac, *hmac2; + size_t startlen, ciphlen, totcookielen, hashlen, hmaclen, appcookielen; + EVP_MD_CTX *hctx; + EVP_PKEY *pkey; + int ret = EXT_RETURN_FAIL; + + if ((s->s3.flags & TLS1_FLAGS_STATELESS) == 0) + return EXT_RETURN_NOT_SENT; + + if (s->ctx->gen_stateless_cookie_cb == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_NO_COOKIE_CALLBACK_SET); + return EXT_RETURN_FAIL; + } + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_cookie) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_get_total_written(pkt, &startlen) + || !WPACKET_reserve_bytes(pkt, MAX_COOKIE_SIZE, &cookie) + || !WPACKET_put_bytes_u16(pkt, COOKIE_STATE_FORMAT_VERSION) + || !WPACKET_put_bytes_u16(pkt, TLS1_3_VERSION) + || !WPACKET_put_bytes_u16(pkt, s->s3.group_id) + || !s->method->put_cipher_by_char(s->s3.tmp.new_cipher, pkt, + &ciphlen) + /* Is there a key_share extension present in this HRR? */ + || !WPACKET_put_bytes_u8(pkt, s->s3.peer_tmp == NULL) + || !WPACKET_put_bytes_u32(pkt, (unsigned int)time(NULL)) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_reserve_bytes(pkt, EVP_MAX_MD_SIZE, &hashval1)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + /* + * Get the hash of the initial ClientHello. ssl_handshake_hash() operates + * on raw buffers, so we first reserve sufficient bytes (above) and then + * subsequently allocate them (below) + */ + if (!ssl3_digest_cached_records(s, 0) + || !ssl_handshake_hash(s, hashval1, EVP_MAX_MD_SIZE, &hashlen)) { + /* SSLfatal_ntls() already called */ + return EXT_RETURN_FAIL; + } + + if (!WPACKET_allocate_bytes(pkt, hashlen, &hashval2) + || !ossl_assert(hashval1 == hashval2) + || !WPACKET_close(pkt) + || !WPACKET_start_sub_packet_u8(pkt) + || !WPACKET_reserve_bytes(pkt, SSL_COOKIE_LENGTH, &appcookie1)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + /* Generate the application cookie */ + if (s->ctx->gen_stateless_cookie_cb(s, appcookie1, &appcookielen) == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_COOKIE_GEN_CALLBACK_FAILURE); + return EXT_RETURN_FAIL; + } + + if (!WPACKET_allocate_bytes(pkt, appcookielen, &appcookie2) + || !ossl_assert(appcookie1 == appcookie2) + || !WPACKET_close(pkt) + || !WPACKET_get_total_written(pkt, &totcookielen) + || !WPACKET_reserve_bytes(pkt, SHA256_DIGEST_LENGTH, &hmac)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + hmaclen = SHA256_DIGEST_LENGTH; + + totcookielen -= startlen; + if (!ossl_assert(totcookielen <= MAX_COOKIE_SIZE - SHA256_DIGEST_LENGTH)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + /* HMAC the cookie */ + hctx = EVP_MD_CTX_create(); + pkey = EVP_PKEY_new_raw_private_key_ex(s->ctx->libctx, "HMAC", + s->ctx->propq, + s->session_ctx->ext.cookie_hmac_key, + sizeof(s->session_ctx->ext.cookie_hmac_key)); + if (hctx == NULL || pkey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (EVP_DigestSignInit_ex(hctx, NULL, "SHA2-256", s->ctx->libctx, + s->ctx->propq, pkey, NULL) <= 0 + || EVP_DigestSign(hctx, hmac, &hmaclen, cookie, + totcookielen) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (!ossl_assert(totcookielen + hmaclen <= MAX_COOKIE_SIZE)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (!WPACKET_allocate_bytes(pkt, hmaclen, &hmac2) + || !ossl_assert(hmac == hmac2) + || !ossl_assert(cookie == hmac - totcookielen) + || !WPACKET_close(pkt) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + ret = EXT_RETURN_SENT; + + err: + EVP_MD_CTX_free(hctx); + EVP_PKEY_free(pkey); + return ret; +#else + return EXT_RETURN_FAIL; +#endif +} + +EXT_RETURN tls_construct_stoc_early_data_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx) +{ + if (context == SSL_EXT_TLS1_3_NEW_SESSION_TICKET) { + if (s->max_early_data == 0) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_early_data) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u32(pkt, s->max_early_data) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; + } + + if (s->ext.early_data != SSL_EARLY_DATA_ACCEPTED) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_early_data) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + +EXT_RETURN tls_construct_stoc_psk_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx) +{ + if (!s->hit) + return EXT_RETURN_NOT_SENT; + + if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_psk) + || !WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_put_bytes_u16(pkt, s->ext.tick_identity) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return EXT_RETURN_FAIL; + } + + return EXT_RETURN_SENT; +} + diff --git a/openssl/src/ssl/statem_ntls/ntls_ssl_local.h b/openssl/src/ssl/statem_ntls/ntls_ssl_local.h new file mode 100644 index 000000000..3c9f179dc --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_ssl_local.h @@ -0,0 +1,77 @@ +/* + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://github.com/Tongsuo-Project/Tongsuo/blob/master/LICENSE.txt + */ + +#ifndef OSSL_NTLS_SSL_LOCAL_H +# define OSSL_NTLS_SSL_LOCAL_H + +# include "../ssl_local.h" +# include "ntls_statem.h" + +# define SSL_CLIENT_USE_SIGALGS_NTLS(s) \ + (SSL_CLIENT_USE_TLS1_2_CIPHERS(s) || (s->client_version == NTLS_VERSION)) + +/* + *optimize later + *This is the default ID for NTLS context + */ +# define SM2_DEFAULT_ID "1234567812345678" +# define SM2_DEFAULT_ID_LEN (sizeof(SM2_DEFAULT_ID) - 1) + +__owur int ssl_x509err2alert_ntls(int type); +__owur int ssl3_do_write_ntls(SSL *s, int type); +__owur unsigned long ssl3_output_cert_chain_ntls(SSL *s, WPACKET *pkt, + CERT_PKEY *a_cpk, + CERT_PKEY *k_cpk); +__owur int tls_close_construct_packet_ntls(SSL *s, WPACKET *pkt, int htype); +__owur int tls_setup_handshake_ntls(SSL *s); + +__owur int ssl_allow_compression_ntls(SSL *s); + +__owur int ssl_version_supported_ntls(const SSL *s, int version, + const SSL_METHOD **meth); + +__owur int ssl_set_client_hello_version_ntls(SSL *s); +__owur int ssl_check_version_downgrade_ntls(SSL *s); +__owur int ssl_set_version_bound_ntls(int method_version, int version, int *bound); +__owur int ssl_choose_server_version_ntls(SSL *s, CLIENTHELLO_MSG *hello, + DOWNGRADE *dgrd); +__owur int ssl_choose_client_version_ntls(SSL *s, int version, + RAW_EXTENSION *extensions); +__owur int ssl_get_min_max_version_ntls(const SSL *s, int *min_version, + int *max_version, int *real_max); + +__owur int ntls_alert_code(int code); +__owur int send_certificate_request_ntls(SSL *s); + +/* statem/extensions_cust.c */ + +custom_ext_method *custom_ext_find_ntls(const custom_ext_methods *exts, + ENDPOINT role, unsigned int ext_type, + size_t *idx); + +void custom_ext_init_ntls(custom_ext_methods *meths); + +__owur int custom_ext_parse_ntls(SSL *s, unsigned int context, unsigned int ext_type, + const unsigned char *ext_data, size_t ext_size, + X509 *x, size_t chainidx); +__owur int custom_ext_add_ntls(SSL *s, int context, WPACKET *pkt, X509 *x, + size_t chainidx, int maxversion); + +__owur int custom_exts_copy_ntls(custom_ext_methods *dst, + const custom_ext_methods *src); +__owur int custom_exts_copy_flags_ntls(custom_ext_methods *dst, + const custom_ext_methods *src); +void custom_exts_free_ntls(custom_ext_methods *exts); + +void ssl_comp_free_compression_methods_int(void); + +/* ssl_mcnf.c */ +void ssl_ctx_system_config(SSL_CTX *ctx); + +#endif diff --git a/openssl/src/ssl/statem_ntls/ntls_statem.c b/openssl/src/ssl/statem_ntls/ntls_statem.c new file mode 100644 index 000000000..4d8a06243 --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_statem.c @@ -0,0 +1,907 @@ +/* + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "internal/cryptlib.h" +#include +#include "ntls_ssl_local.h" +#include "ntls_statem_local.h" +#include + +/* + * This file implements the SSL/TLS state machines. + * + * There are two primary state machines: + * + * 1) Message flow state machine + * 2) Handshake state machine + * + * The Message flow state machine controls the reading and sending of messages + * including handling of non-blocking IO events, flushing of the underlying + * write BIO, handling unexpected messages, etc. It is itself broken into two + * separate sub-state machines which control reading and writing respectively. + * + * The Handshake state machine keeps track of the current SSL/TLS handshake + * state. Transitions of the handshake state are the result of events that + * occur within the Message flow state machine. + * + * Overall it looks like this: + * + * --------------------------------------------- ------------------- + * | | | | + * | Message flow state machine | | | + * | | | | + * | -------------------- -------------------- | Transition | Handshake state | + * | | MSG_FLOW_READING | | MSG_FLOW_WRITING | | Event | machine | + * | | sub-state | | sub-state | |----------->| | + * | | machine for | | machine for | | | | + * | | reading messages | | writing messages | | | | + * | -------------------- -------------------- | | | + * | | | | + * --------------------------------------------- ------------------- + * + */ + +/* Sub state machine return values */ +typedef enum { + /* Something bad happened or NBIO */ + SUB_STATE_ERROR, + /* Sub state finished go to the next sub state */ + SUB_STATE_FINISHED, + /* Sub state finished and handshake was completed */ + SUB_STATE_END_HANDSHAKE +} SUB_STATE_RETURN; + +static void init_read_state_machine_ntls(SSL *s); +static SUB_STATE_RETURN read_state_machine_ntls(SSL *s); +static void init_write_state_machine_ntls(SSL *s); +static SUB_STATE_RETURN write_state_machine_ntls(SSL *s); + +OSSL_HANDSHAKE_STATE SSL_get_state_ntls(const SSL *ssl) +{ + return ssl->statem.hand_state; +} + +int SSL_in_init_ntls(const SSL *s) +{ + return s->statem.in_init; +} + +int SSL_is_init_finished_ntls(const SSL *s) +{ + return !(s->statem.in_init) && (s->statem.hand_state == TLS_ST_OK); +} + +int SSL_in_before_ntls(const SSL *s) +{ + /* + * Historically being "in before" meant before anything had happened. In the + * current code though we remain in the "before" state for a while after we + * have started the handshake process (e.g. as a server waiting for the + * first message to arrive). There "in before" is taken to mean "in before" + * and not started any handshake process yet. + */ + return (s->statem.hand_state == TLS_ST_BEFORE) + && (s->statem.state == MSG_FLOW_UNINITED); +} + +/* + * Clear the state machine state and reset back to MSG_FLOW_UNINITED + */ +void ossl_statem_clear_ntls(SSL *s) +{ + s->statem.state = MSG_FLOW_UNINITED; + s->statem.hand_state = TLS_ST_BEFORE; + s->statem.in_init = 1; + s->statem.no_cert_verify = 0; +} + +/* + * Set the state machine up ready for a renegotiation handshake + */ +void ossl_statem_set_renegotiate_ntls(SSL *s) +{ + s->statem.in_init = 1; + s->statem.request_state = TLS_ST_SW_HELLO_REQ; +} + +void ossl_statem_send_fatal_ntls(SSL *s, int al) +{ + /* We shouldn't call SSLfatal() twice. Once is enough */ + if (s->statem.in_init && s->statem.state == MSG_FLOW_ERROR) + return; + s->statem.in_init = 1; + s->statem.state = MSG_FLOW_ERROR; + if (al != SSL_AD_NO_ALERT + && s->statem.enc_write_state != ENC_WRITE_STATE_INVALID) + ssl3_send_alert(s, SSL3_AL_FATAL, al); +} + +/* + * Error reporting building block that's used instead of ERR_set_error(). + * In addition to what ERR_set_error() does, this puts the state machine + * into an error state and sends an alert if appropriate. + * This is a permanent error for the current connection. + */ +void ossl_statem_fatal_ntls(SSL *s, int al, int reason, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + ERR_vset_error(ERR_LIB_SSL, reason, fmt, args); + va_end(args); + + ossl_statem_send_fatal_ntls(s, al); +} + +/* + * This macro should only be called if we are already expecting to be in + * a fatal error state. We verify that we are, and set it if not (this would + * indicate a bug). + */ +#define check_fatal(s) \ + do { \ + if (!ossl_assert((s)->statem.in_init \ + && (s)->statem.state == MSG_FLOW_ERROR)) \ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_MISSING_FATAL); \ + } while (0) + +/* + * Discover whether the current connection is in the error state. + * + * Valid return values are: + * 1: Yes + * 0: No + */ +int ossl_statem_in_error_ntls(const SSL *s) +{ + if (s->statem.state == MSG_FLOW_ERROR) + return 1; + + return 0; +} + +void ossl_statem_set_in_init_ntls(SSL *s, int init) +{ + s->statem.in_init = init; +} + +int ossl_statem_get_in_handshake_ntls(SSL *s) +{ + return s->statem.in_handshake; +} + +void ossl_statem_set_in_handshake_ntls(SSL *s, int inhand) +{ + if (inhand) + s->statem.in_handshake++; + else + s->statem.in_handshake--; +} + +/* Are we in a sensible state to skip over unreadable early data? */ +int ossl_statem_skip_early_data_ntls(SSL *s) +{ + if (s->ext.early_data != SSL_EARLY_DATA_REJECTED) + return 0; + + if (!s->server + || s->statem.hand_state != TLS_ST_EARLY_DATA + || s->hello_retry_request == SSL_HRR_COMPLETE) + return 0; + + return 1; +} + +/* + * Called when we are in SSL_read*(), SSL_write*(), or SSL_accept() + * /SSL_connect()/SSL_do_handshake(). Used to test whether we are in an early + * data state and whether we should attempt to move the handshake on if so. + * |sending| is 1 if we are attempting to send data (SSL_write*()), 0 if we are + * attempting to read data (SSL_read*()), or -1 if we are in SSL_do_handshake() + * or similar. + */ +void ossl_statem_check_finish_init_ntls(SSL *s, int sending) +{ + if (sending == -1) { + if (s->statem.hand_state == TLS_ST_PENDING_EARLY_DATA_END + || s->statem.hand_state == TLS_ST_EARLY_DATA) { + ossl_statem_set_in_init_ntls(s, 1); + if (s->early_data_state == SSL_EARLY_DATA_WRITE_RETRY) { + /* + * SSL_connect() or SSL_do_handshake() has been called directly. + * We don't allow any more writing of early data. + */ + s->early_data_state = SSL_EARLY_DATA_FINISHED_WRITING; + } + } + } else if (!s->server) { + if ((sending && (s->statem.hand_state == TLS_ST_PENDING_EARLY_DATA_END + || s->statem.hand_state == TLS_ST_EARLY_DATA) + && s->early_data_state != SSL_EARLY_DATA_WRITING) + || (!sending && s->statem.hand_state == TLS_ST_EARLY_DATA)) { + ossl_statem_set_in_init_ntls(s, 1); + /* + * SSL_write() has been called directly. We don't allow any more + * writing of early data. + */ + if (sending && s->early_data_state == SSL_EARLY_DATA_WRITE_RETRY) + s->early_data_state = SSL_EARLY_DATA_FINISHED_WRITING; + } + } else { + if (s->early_data_state == SSL_EARLY_DATA_FINISHED_READING + && s->statem.hand_state == TLS_ST_EARLY_DATA) + ossl_statem_set_in_init_ntls(s, 1); + } +} + +void ossl_statem_set_hello_verify_done_ntls(SSL *s) +{ + s->statem.state = MSG_FLOW_UNINITED; + s->statem.in_init = 1; + /* + * This will get reset (briefly) back to TLS_ST_BEFORE when we enter + * state_machine() because |state| is MSG_FLOW_UNINITED, but until then any + * calls to SSL_in_before_ntls() will return false. Also calls to + * SSL_state_string() and SSL_state_string_long() will return something + * sensible. + */ + s->statem.hand_state = TLS_ST_SR_CLNT_HELLO; +} + +int ossl_statem_connect_ntls(SSL *s) +{ + return state_machine_ntls(s, 0); +} + +int ossl_statem_accept_ntls(SSL *s) +{ + return state_machine_ntls(s, 1); +} + +typedef void (*info_cb) (const SSL *, int, int); + +static info_cb get_callback(SSL *s) +{ + if (s->info_callback != NULL) + return s->info_callback; + else if (s->ctx->info_callback != NULL) + return s->ctx->info_callback; + + return NULL; +} + +/* + * The main message flow state machine. We start in the MSG_FLOW_UNINITED or + * MSG_FLOW_FINISHED state and finish in MSG_FLOW_FINISHED. Valid states and + * transitions are as follows: + * + * MSG_FLOW_UNINITED MSG_FLOW_FINISHED + * | | + * +-----------------------+ + * v + * MSG_FLOW_WRITING <---> MSG_FLOW_READING + * | + * V + * MSG_FLOW_FINISHED + * | + * V + * [SUCCESS] + * + * We may exit at any point due to an error or NBIO event. If an NBIO event + * occurs then we restart at the point we left off when we are recalled. + * MSG_FLOW_WRITING and MSG_FLOW_READING have sub-state machines associated with them. + * + * In addition to the above there is also the MSG_FLOW_ERROR state. We can move + * into that state at any point in the event that an irrecoverable error occurs. + * + * Valid return values are: + * 1: Success + * <=0: NBIO or error + */ +int state_machine_ntls(SSL *s, int server) +{ + BUF_MEM *buf = NULL; + void (*cb) (const SSL *ssl, int type, int val) = NULL; + OSSL_STATEM *st = &s->statem; + int ret = -1; + int ssret; + + if (st->state == MSG_FLOW_ERROR) { + /* Shouldn't have been called if we're already in the error state */ + return -1; + } + + ERR_clear_error(); + clear_sys_error(); + + cb = get_callback(s); + + st->in_handshake++; + if (!SSL_in_init_ntls(s) || SSL_in_before_ntls(s)) { + /* + * If we are stateless then we already called SSL_clear() - don't do + * it again and clear the STATELESS flag itself. + */ + if ((s->s3.flags & TLS1_FLAGS_STATELESS) == 0 && !SSL_clear(s)) + return -1; + } + + /* Initialise state machine */ + if (st->state == MSG_FLOW_UNINITED + || st->state == MSG_FLOW_FINISHED) { + if (st->state == MSG_FLOW_UNINITED) { + st->hand_state = TLS_ST_BEFORE; + st->request_state = TLS_ST_BEFORE; + } + + s->server = server; + if (cb != NULL) { + cb(s, SSL_CB_HANDSHAKE_START, 1); + } + + if (!ssl_security(s, SSL_SECOP_VERSION, 0, s->version, NULL)) { + SSLfatal_ntls(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); + goto end; + } + + if (s->init_buf == NULL) { + if ((buf = BUF_MEM_new()) == NULL) { + SSLfatal_ntls(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); + goto end; + } + if (!BUF_MEM_grow(buf, SSL3_RT_MAX_PLAIN_LENGTH)) { + SSLfatal_ntls(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); + goto end; + } + s->init_buf = buf; + buf = NULL; + } + + if (!ssl3_setup_buffers(s)) { + SSLfatal_ntls(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); + goto end; + } + s->init_num = 0; + + /* + * Should have been reset by tls_process_finished_ntls, too. + */ + s->s3.change_cipher_spec = 0; + + /* + * Ok, we now need to push on a buffering BIO ...but not with + * SCTP + */ +#ifndef OPENSSL_NO_SCTP + if (!SSL_IS_DTLS(s) || !BIO_dgram_is_sctp(SSL_get_wbio(s))) +#endif + if (!ssl_init_wbio_buffer(s)) { + SSLfatal_ntls(s, SSL_AD_NO_ALERT, ERR_R_INTERNAL_ERROR); + goto end; + } + + if ((SSL_in_before_ntls(s)) + || s->renegotiate) { + if (!tls_setup_handshake_ntls(s)) { + /* SSLfatal_ntls() already called */ + goto end; + } + + if (SSL_IS_FIRST_HANDSHAKE(s)) + st->read_state_first_init = 1; + } + + st->state = MSG_FLOW_WRITING; + init_write_state_machine_ntls(s); + } + + while (st->state != MSG_FLOW_FINISHED) { + if (st->state == MSG_FLOW_READING) { + ssret = read_state_machine_ntls(s); + if (ssret == SUB_STATE_FINISHED) { + st->state = MSG_FLOW_WRITING; + init_write_state_machine_ntls(s); + } else { + /* NBIO or error */ + goto end; + } + } else if (st->state == MSG_FLOW_WRITING) { + ssret = write_state_machine_ntls(s); + if (ssret == SUB_STATE_FINISHED) { + st->state = MSG_FLOW_READING; + init_read_state_machine_ntls(s); + } else if (ssret == SUB_STATE_END_HANDSHAKE) { + st->state = MSG_FLOW_FINISHED; + } else { + /* NBIO or error */ + goto end; + } + } else { + /* Error */ + check_fatal(s); + ERR_raise(ERR_LIB_SSL, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + goto end; + } + } + + ret = 1; + + end: + st->in_handshake--; + + BUF_MEM_free(buf); + if (cb != NULL) { + if (server) + cb(s, SSL_CB_ACCEPT_EXIT, ret); + else + cb(s, SSL_CB_CONNECT_EXIT, ret); + } + return ret; +} + +/* + * Initialise the MSG_FLOW_READING sub-state machine + */ +static void init_read_state_machine_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + + st->read_state = READ_STATE_HEADER; +} + +static int grow_init_buf(SSL *s, size_t size) { + + size_t msg_offset = (char *)s->init_msg - s->init_buf->data; + + if (!BUF_MEM_grow_clean(s->init_buf, (int)size)) + return 0; + + if (size < msg_offset) + return 0; + + s->init_msg = s->init_buf->data + msg_offset; + + return 1; +} + +/* + * This function implements the sub-state machine when the message flow is in + * MSG_FLOW_READING. The valid sub-states and transitions are: + * + * READ_STATE_HEADER <--+<-------------+ + * | | | + * v | | + * READ_STATE_BODY -----+-->READ_STATE_POST_PROCESS + * | | + * +----------------------------+ + * v + * [SUB_STATE_FINISHED] + * + * READ_STATE_HEADER has the responsibility for reading in the message header + * and transitioning the state of the handshake state machine. + * + * READ_STATE_BODY reads in the rest of the message and then subsequently + * processes it. + * + * READ_STATE_POST_PROCESS is an optional step that may occur if some post + * processing activity performed on the message may block. + * + * Any of the above states could result in an NBIO event occurring in which case + * control returns to the calling application. When this function is recalled we + * will resume in the same state where we left off. + */ +static SUB_STATE_RETURN read_state_machine_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + int ret, mt; + size_t len = 0; + int (*transition) (SSL *s, int mt); + PACKET pkt; + MSG_PROCESS_RETURN(*process_message) (SSL *s, PACKET *pkt); + WORK_STATE(*post_process_message) (SSL *s, WORK_STATE wst); + size_t (*max_message_size) (SSL *s); + void (*cb) (const SSL *ssl, int type, int val) = NULL; + + cb = get_callback(s); + + if (s->server) { + transition = ossl_statem_server_read_transition_ntls; + process_message = ossl_statem_server_process_message_ntls; + max_message_size = ossl_statem_server_max_message_size_ntls; + post_process_message = ossl_statem_server_post_process_message_ntls; + } else { + transition = ossl_statem_client_read_transition_ntls; + process_message = ossl_statem_client_process_message_ntls; + max_message_size = ossl_statem_client_max_message_size_ntls; + post_process_message = ossl_statem_client_post_process_message_ntls; + } + + if (st->read_state_first_init) { + s->first_packet = 1; + st->read_state_first_init = 0; + } + + while (1) { + switch (st->read_state) { + case READ_STATE_HEADER: + /* Get the state the peer wants to move to */ + + ret = tls_get_message_header_ntls(s, &mt); + + if (ret == 0) { + /* Could be non-blocking IO */ + return SUB_STATE_ERROR; + } + + if (cb != NULL) { + /* Notify callback of an impending state change */ + if (s->server) + cb(s, SSL_CB_ACCEPT_LOOP, 1); + else + cb(s, SSL_CB_CONNECT_LOOP, 1); + } + /* + * Validate that we are allowed to move to the new state and move + * to that state if so + */ + if (!transition(s, mt)) + return SUB_STATE_ERROR; + + if (s->s3.tmp.message_size > max_message_size(s)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_EXCESSIVE_MESSAGE_SIZE); + return SUB_STATE_ERROR; + } + + if (s->s3.tmp.message_size > 0 + && !grow_init_buf(s, s->s3.tmp.message_size + + SSL3_HM_HEADER_LENGTH)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_BUF_LIB); + return SUB_STATE_ERROR; + } + + st->read_state = READ_STATE_BODY; + /* Fall through */ + + case READ_STATE_BODY: + ret = tls_get_message_body_ntls(s, &len); + if (ret == 0) { + /* Could be non-blocking IO */ + return SUB_STATE_ERROR; + } + + s->first_packet = 0; + if (!PACKET_buf_init(&pkt, s->init_msg, len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return SUB_STATE_ERROR; + } + + ret = process_message(s, &pkt); + + /* Discard the packet data */ + s->init_num = 0; + + switch (ret) { + case MSG_PROCESS_ERROR: + check_fatal(s); + return SUB_STATE_ERROR; + + case MSG_PROCESS_FINISHED_READING: + return SUB_STATE_FINISHED; + + case MSG_PROCESS_CONTINUE_PROCESSING: + st->read_state = READ_STATE_POST_PROCESS; + st->read_state_work = WORK_MORE_A; + break; + + default: + st->read_state = READ_STATE_HEADER; + break; + } + break; + + case READ_STATE_POST_PROCESS: + st->read_state_work = post_process_message(s, st->read_state_work); + switch (st->read_state_work) { + case WORK_ERROR: + check_fatal(s); + /* Fall through */ + case WORK_MORE_A: + case WORK_MORE_B: + case WORK_MORE_C: + return SUB_STATE_ERROR; + + case WORK_FINISHED_CONTINUE: + st->read_state = READ_STATE_HEADER; + break; + + case WORK_FINISHED_STOP: + return SUB_STATE_FINISHED; + } + break; + + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return SUB_STATE_ERROR; + } + } +} + +/* + * Send a previously constructed message to the peer. + */ +static int statem_do_write(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + + if (st->hand_state == TLS_ST_CW_CHANGE + || st->hand_state == TLS_ST_SW_CHANGE) { + return ssl3_do_write_ntls(s, SSL3_RT_CHANGE_CIPHER_SPEC); + } else { + return ssl_do_write(s); + } +} + +/* + * Initialise the MSG_FLOW_WRITING sub-state machine + */ +static void init_write_state_machine_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + + st->write_state = WRITE_STATE_TRANSITION; +} + +/* + * This function implements the sub-state machine when the message flow is in + * MSG_FLOW_WRITING. The valid sub-states and transitions are: + * + * +-> WRITE_STATE_TRANSITION ------> [SUB_STATE_FINISHED] + * | | + * | v + * | WRITE_STATE_PRE_WORK -----> [SUB_STATE_END_HANDSHAKE] + * | | + * | v + * | WRITE_STATE_SEND + * | | + * | v + * | WRITE_STATE_POST_WORK + * | | + * +-------------+ + * + * WRITE_STATE_TRANSITION transitions the state of the handshake state machine + + * WRITE_STATE_PRE_WORK performs any work necessary to prepare the later + * sending of the message. This could result in an NBIO event occurring in + * which case control returns to the calling application. When this function + * is recalled we will resume in the same state where we left off. + * + * WRITE_STATE_SEND sends the message and performs any work to be done after + * sending. + * + * WRITE_STATE_POST_WORK performs any work necessary after the sending of the + * message has been completed. As for WRITE_STATE_PRE_WORK this could also + * result in an NBIO event. + */ +static SUB_STATE_RETURN write_state_machine_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + int ret; + WRITE_TRAN(*transition) (SSL *s); + WORK_STATE(*pre_work) (SSL *s, WORK_STATE wst); + WORK_STATE(*post_work) (SSL *s, WORK_STATE wst); + int (*get_construct_message_f) (SSL *s, WPACKET *pkt, + int (**confunc) (SSL *s, WPACKET *pkt), + int *mt); + void (*cb) (const SSL *ssl, int type, int val) = NULL; + int (*confunc) (SSL *s, WPACKET *pkt); + int mt; + WPACKET pkt; + + cb = get_callback(s); + + if (s->server) { + transition = ossl_statem_server_write_transition_ntls; + pre_work = ossl_statem_server_pre_work_ntls; + post_work = ossl_statem_server_post_work_ntls; + get_construct_message_f = ossl_statem_server_construct_message_ntls; + } else { + transition = ossl_statem_client_write_transition_ntls; + pre_work = ossl_statem_client_pre_work_ntls; + post_work = ossl_statem_client_post_work_ntls; + get_construct_message_f = ossl_statem_client_construct_message_ntls; + } + + while (1) { + switch (st->write_state) { + case WRITE_STATE_TRANSITION: + if (cb != NULL) { + /* Notify callback of an impending state change */ + if (s->server) + cb(s, SSL_CB_ACCEPT_LOOP, 1); + else + cb(s, SSL_CB_CONNECT_LOOP, 1); + } + switch (transition(s)) { + case WRITE_TRAN_CONTINUE: + st->write_state = WRITE_STATE_PRE_WORK; + st->write_state_work = WORK_MORE_A; + break; + + case WRITE_TRAN_FINISHED: + return SUB_STATE_FINISHED; + break; + + case WRITE_TRAN_ERROR: + check_fatal(s); + return SUB_STATE_ERROR; + } + break; + + case WRITE_STATE_PRE_WORK: + switch (st->write_state_work = pre_work(s, st->write_state_work)) { + case WORK_ERROR: + check_fatal(s); + /* Fall through */ + case WORK_MORE_A: + case WORK_MORE_B: + case WORK_MORE_C: + return SUB_STATE_ERROR; + + case WORK_FINISHED_CONTINUE: + st->write_state = WRITE_STATE_SEND; + break; + + case WORK_FINISHED_STOP: + return SUB_STATE_END_HANDSHAKE; + } + if (!get_construct_message_f(s, &pkt, &confunc, &mt)) { + /* SSLfatal_ntls() already called */ + return SUB_STATE_ERROR; + } + if (mt == SSL3_MT_DUMMY) { + /* Skip construction and sending. This isn't a "real" state */ + st->write_state = WRITE_STATE_POST_WORK; + st->write_state_work = WORK_MORE_A; + break; + } + + if (!WPACKET_init(&pkt, s->init_buf) + || !ssl_set_handshake_header(s, &pkt, mt)) { + WPACKET_cleanup(&pkt); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return SUB_STATE_ERROR; + } + if (confunc != NULL && !confunc(s, &pkt)) { + WPACKET_cleanup(&pkt); + check_fatal(s); + return SUB_STATE_ERROR; + } + if (!ssl_close_construct_packet(s, &pkt, mt) + || !WPACKET_finish(&pkt)) { + WPACKET_cleanup(&pkt); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return SUB_STATE_ERROR; + } + + /* Fall through */ + + case WRITE_STATE_SEND: + ret = statem_do_write(s); + if (ret <= 0) { + return SUB_STATE_ERROR; + } + st->write_state = WRITE_STATE_POST_WORK; + st->write_state_work = WORK_MORE_A; + /* Fall through */ + + case WRITE_STATE_POST_WORK: + switch (st->write_state_work = post_work(s, st->write_state_work)) { + case WORK_ERROR: + check_fatal(s); + /* Fall through */ + case WORK_MORE_A: + case WORK_MORE_B: + case WORK_MORE_C: + return SUB_STATE_ERROR; + + case WORK_FINISHED_CONTINUE: + st->write_state = WRITE_STATE_TRANSITION; + break; + + case WORK_FINISHED_STOP: + return SUB_STATE_END_HANDSHAKE; + } + break; + + default: + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return SUB_STATE_ERROR; + } + } +} + +/* + * Flush the write BIO + */ +int statem_flush_ntls(SSL *s) +{ + s->rwstate = SSL_WRITING; + if (BIO_flush(s->wbio) <= 0) + return 0; + + s->rwstate = SSL_NOTHING; + + return 1; +} + +/* + * Called by the record layer to determine whether application data is + * allowed to be received in the current handshake state or not. + * + * Return values are: + * 1: Yes (application data allowed) + * 0: No (application data not allowed) + */ +int ossl_statem_app_data_allowed_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + + if (st->state == MSG_FLOW_UNINITED) + return 0; + + if (!s->s3.in_read_app_data || (s->s3.total_renegotiations == 0)) + return 0; + + if (s->server) { + /* + * If we're a server and we haven't got as far as writing our + * ServerHello yet then we allow app data + */ + if (st->hand_state == TLS_ST_BEFORE + || st->hand_state == TLS_ST_SR_CLNT_HELLO) + return 1; + } else { + /* + * If we're a client and we haven't read the ServerHello yet then we + * allow app data + */ + if (st->hand_state == TLS_ST_CW_CLNT_HELLO) + return 1; + } + + return 0; +} + +/* + * This function returns 1 if TLS exporter is ready to export keying + * material, or 0 if otherwise. + */ +int ossl_statem_export_allowed_ntls(SSL *s) +{ + return s->s3.previous_server_finished_len != 0 + && s->statem.hand_state != TLS_ST_SW_FINISHED; +} + +/* + * Return 1 if early TLS exporter is ready to export keying material, + * or 0 if otherwise. + */ +int ossl_statem_export_early_allowed_ntls(SSL *s) +{ + /* + * The early exporter secret is only present on the server if we + * have accepted early_data. It is present on the client as long + * as we have sent early_data. + */ + return s->ext.early_data == SSL_EARLY_DATA_ACCEPTED + || (!s->server && s->ext.early_data != SSL_EARLY_DATA_NOT_SENT); +} + diff --git a/openssl/src/ssl/statem_ntls/ntls_statem.h b/openssl/src/ssl/statem_ntls/ntls_statem.h new file mode 100644 index 000000000..748fd80fd --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_statem.h @@ -0,0 +1,160 @@ +/* + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/***************************************************************************** + * * + * These enums should be considered PRIVATE to the state machine. No * + * non-state machine code should need to use these * + * * + *****************************************************************************/ +/* + * Valid return codes used for functions performing work prior to or after + * sending or receiving a message + */ + +typedef enum { + /* Something went wrong */ + WORK_ERROR_NTLS, + /* We're done working and there shouldn't be anything else to do after */ + WORK_FINISHED_STOP_NTLS, + /* We're done working move onto the next thing */ + WORK_FINISHED_CONTINUE_NTLS, + /* We're working on phase A */ + WORK_MORE_A_NTLS, + /* We're working on phase B */ + WORK_MORE_B_NTLS, + /* We're working on phase C */ + WORK_MORE_C_NTLS +} WORK_STATE_NTLS; + +/* Write transition return codes */ +typedef enum { + /* Something went wrong */ + WRITE_TRAN_ERROR_NTLS, + /* A transition was successfully completed and we should continue */ + WRITE_TRAN_CONTINUE_NTLS, + /* There is no more write work to be done */ + WRITE_TRAN_FINISHED_NTLS +} WRITE_TRAN_NTLS; + +/* Message flow states */ +typedef enum { + /* No handshake in progress */ + MSG_FLOW_UNINITED_NTLS, + /* A permanent error with this connection */ + MSG_FLOW_ERROR_NTLS, + /* We are reading messages */ + MSG_FLOW_READING_NTLS, + /* We are writing messages */ + MSG_FLOW_WRITING_NTLS, + /* Handshake has finished */ + MSG_FLOW_FINISHED_NTLS +} MSG_FLOW_STATE_NTLS; + +/* Read states */ +typedef enum { + READ_STATE_HEADER_NTLS, + READ_STATE_BODY_NTLS, + READ_STATE_POST_PROCESS_NTLS +} READ_STATE_NTLS; + +/* Write states */ +typedef enum { + WRITE_STATE_TRANSITION_NTLS, + WRITE_STATE_PRE_WORK_NTLS, + WRITE_STATE_SEND_NTLS, + WRITE_STATE_POST_WORK_NTLS +} WRITE_STATE_NTLS; + +typedef enum { + /* The enc_write_ctx can be used normally */ + ENC_WRITE_STATE_VALID_NTLS, + /* The enc_write_ctx cannot be used */ + ENC_WRITE_STATE_INVALID_NTLS, + /* Write alerts in plaintext, but otherwise use the enc_write_ctx */ + ENC_WRITE_STATE_WRITE_PLAIN_ALERTS_NTLS +} ENC_WRITE_STATES_NTLS; + +typedef enum { + /* The enc_read_ctx can be used normally */ + ENC_READ_STATE_VALID_NTLS, + /* We may receive encrypted or plaintext alerts */ + ENC_READ_STATE_ALLOW_PLAIN_ALERTS_NTLS +} ENC_READ_STATES_NTLS; + +/***************************************************************************** + * * + * This structure should be considered "opaque" to anything outside of the * + * state machine. No non-state machine code should be accessing the members * + * of this structure. * + * * + *****************************************************************************/ + +struct ossl_statem_st_ntls { + MSG_FLOW_STATE_NTLS state; + WRITE_STATE_NTLS write_state; + WORK_STATE_NTLS write_state_work; + READ_STATE_NTLS read_state; + WORK_STATE_NTLS read_state_work; + OSSL_HANDSHAKE_STATE hand_state; + /* The handshake state requested by an API call (e.g. HelloRequest) */ + OSSL_HANDSHAKE_STATE request_state; + int in_init; + int read_state_first_init; + /* true when we are actually in SSL_accept() or SSL_connect() */ + int in_handshake; + /* + * True when are processing a "real" handshake that needs cleaning up (not + * just a HelloRequest or similar). + */ + int cleanuphand; + /* Should we skip the CertificateVerify message? */ + unsigned int no_cert_verify; + int use_timer; + ENC_WRITE_STATES_NTLS enc_write_state; + ENC_READ_STATES_NTLS enc_read_state; +}; +typedef struct ossl_statem_st_ntls OSSL_STATEM_NTLS; + +/***************************************************************************** + * * + * The following macros/functions represent the libssl internal API to the * + * state machine. Any libssl code may call these functions/macros * + * * + *****************************************************************************/ + +__owur int ossl_statem_accept_ntls(SSL *s); +__owur int ossl_statem_connect_ntls(SSL *s); +void ossl_statem_clear_ntls(SSL *s); +void ossl_statem_set_renegotiate_ntls(SSL *s); +void ossl_statem_send_fatal_ntls(SSL *s, int al); +void ossl_statem_fatal_ntls(SSL *s, int al, int reason, const char *fmt, ...); +# define SSL_AD_NO_ALERT -1 +# define SSLfatal_alert_ntls(s, al) ossl_statem_send_fatal_ntls((s), (al)) +# define SSLfatal_ntls(s, al, r) SSLfatal_data_ntls((s), (al), (r), NULL) +# define SSLfatal_data_ntls \ + (ERR_new(), \ + ERR_set_debug(OPENSSL_FILE, OPENSSL_LINE, OPENSSL_FUNC), \ + ossl_statem_fatal_ntls) + +int ossl_statem_in_error_ntls(const SSL *s); +void ossl_statem_set_in_init_ntls(SSL *s, int init); +int ossl_statem_get_in_handshake_ntls(SSL *s); +void ossl_statem_set_in_handshake_ntls(SSL *s, int inhand); +__owur int ossl_statem_skip_early_data_ntls(SSL *s); +void ossl_statem_check_finish_init_ntls(SSL *s, int send); +void ossl_statem_set_hello_verify_done_ntls(SSL *s); +__owur int ossl_statem_app_data_allowed_ntls(SSL *s); +__owur int ossl_statem_export_allowed_ntls(SSL *s); +__owur int ossl_statem_export_early_allowed_ntls(SSL *s); + +/* Flush the write BIO */ +int statem_flush_ntls(SSL *s); +int state_machine_ntls(SSL *s, int server); +int SSL_connection_is_ntls(SSL *s, int is_server); diff --git a/openssl/src/ssl/statem_ntls/ntls_statem_clnt.c b/openssl/src/ssl/statem_ntls/ntls_statem_clnt.c new file mode 100644 index 000000000..5a7faada9 --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_statem_clnt.c @@ -0,0 +1,2403 @@ +/* + * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include +#include +#include "ntls_ssl_local.h" +#include "ntls_statem_local.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal/cryptlib.h" +#include "internal/tlsgroups.h" + +static MSG_PROCESS_RETURN tls_process_as_hello_retry_request(SSL *s, PACKET *pkt); + +static ossl_inline int cert_req_allowed(SSL *s); +static int key_exchange_expected(SSL *s); +static int ssl_cipher_list_to_bytes(SSL *s, STACK_OF(SSL_CIPHER) *sk, + WPACKET *pkt); + +/* + * Is a CertificateRequest message allowed at the moment or not? + * + * Return values are: + * 1: Yes + * 0: No + */ +static ossl_inline int cert_req_allowed(SSL *s) +{ + /* TLS does not like anon-DH with client cert */ + if ((s->version > SSL3_VERSION + && (s->s3.tmp.new_cipher->algorithm_auth & SSL_aNULL)) + || (s->s3.tmp.new_cipher->algorithm_auth & (SSL_aSRP | SSL_aPSK))) + return 0; + + return 1; +} + +/* + * Should we expect the ServerKeyExchange message or not? + * + * Return values are: + * 1: Yes + * 0: No + */ +static int key_exchange_expected(SSL *s) +{ + return 1; +} + + +/* + * ossl_statem_client_read_transition_ntls() encapsulates the logic for the allowed + * handshake state transitions when the client is reading messages from the + * server. The message type that the server has sent is provided in |mt|. The + * current state is in |s->statem.hand_state|. + * + * Return values are 1 for success (transition allowed) and 0 on error + * (transition not allowed) + */ +int ossl_statem_client_read_transition_ntls(SSL *s, int mt) +{ + OSSL_STATEM *st = &s->statem; + int ske_expected; + + /* + * Note that after writing the first ClientHello we don't know what version + * we are going to negotiate yet, so we don't take this branch until later. + */ + + switch (st->hand_state) { + default: + break; + + case TLS_ST_CW_CLNT_HELLO: + if (mt == SSL3_MT_SERVER_HELLO) { + st->hand_state = TLS_ST_CR_SRVR_HELLO; + return 1; + } + + break; + + case TLS_ST_EARLY_DATA: + /* + * We've not actually selected TLSv1.3 yet, but we have sent early + * data. The only thing allowed now is a ServerHello or a + * HelloRetryRequest. + */ + if (mt == SSL3_MT_SERVER_HELLO) { + st->hand_state = TLS_ST_CR_SRVR_HELLO; + return 1; + } + break; + + case TLS_ST_CR_SRVR_HELLO: + if (s->hit) { + if (s->ext.ticket_expected) { + if (mt == SSL3_MT_NEWSESSION_TICKET) { + st->hand_state = TLS_ST_CR_SESSION_TICKET; + return 1; + } + } else if (mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + st->hand_state = TLS_ST_CR_CHANGE; + return 1; + } + } else { + if (s->version >= NTLS_VERSION + && s->ext.session_secret_cb != NULL + && s->session->ext.tick != NULL + && mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + /* + * Normally, we can tell if the server is resuming the session + * from the session ID. EAP-FAST (RFC 4851), however, relies on + * the next server message after the ServerHello to determine if + * the server is resuming. + */ + s->hit = 1; + st->hand_state = TLS_ST_CR_CHANGE; + return 1; + } else if (!(s->s3.tmp.new_cipher->algorithm_auth + & (SSL_aNULL | SSL_aSRP | SSL_aPSK))) { + if (mt == SSL3_MT_CERTIFICATE) { + st->hand_state = TLS_ST_CR_CERT; + return 1; + } + } else { + ske_expected = key_exchange_expected(s); + /* SKE is optional for some PSK ciphersuites */ + if (ske_expected + || ((s->s3.tmp.new_cipher->algorithm_mkey & SSL_PSK) + && mt == SSL3_MT_SERVER_KEY_EXCHANGE)) { + if (mt == SSL3_MT_SERVER_KEY_EXCHANGE) { + st->hand_state = TLS_ST_CR_KEY_EXCH; + return 1; + } + } else if (mt == SSL3_MT_CERTIFICATE_REQUEST + && cert_req_allowed(s)) { + st->hand_state = TLS_ST_CR_CERT_REQ; + return 1; + } else if (mt == SSL3_MT_SERVER_DONE) { + st->hand_state = TLS_ST_CR_SRVR_DONE; + return 1; + } + } + } + break; + + case TLS_ST_CR_CERT: + /* + * The CertificateStatus message is optional even if + * |ext.status_expected| is set + */ + if (s->ext.status_expected && mt == SSL3_MT_CERTIFICATE_STATUS) { + st->hand_state = TLS_ST_CR_CERT_STATUS; + return 1; + } + /* Fall through */ + + case TLS_ST_CR_CERT_STATUS: + ske_expected = key_exchange_expected(s); + /* SKE is optional for some PSK ciphersuites */ + if (ske_expected || ((s->s3.tmp.new_cipher->algorithm_mkey & SSL_PSK) + && mt == SSL3_MT_SERVER_KEY_EXCHANGE)) { + if (mt == SSL3_MT_SERVER_KEY_EXCHANGE) { + st->hand_state = TLS_ST_CR_KEY_EXCH; + return 1; + } + goto err; + } + /* Fall through */ + + case TLS_ST_CR_KEY_EXCH: + if (mt == SSL3_MT_CERTIFICATE_REQUEST) { + if (cert_req_allowed(s)) { + st->hand_state = TLS_ST_CR_CERT_REQ; + return 1; + } + goto err; + } + /* Fall through */ + + case TLS_ST_CR_CERT_REQ: + if (mt == SSL3_MT_SERVER_DONE) { + st->hand_state = TLS_ST_CR_SRVR_DONE; + return 1; + } + break; + + case TLS_ST_CW_FINISHED: + if (s->ext.ticket_expected) { + if (mt == SSL3_MT_NEWSESSION_TICKET) { + st->hand_state = TLS_ST_CR_SESSION_TICKET; + return 1; + } + } else if (mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + st->hand_state = TLS_ST_CR_CHANGE; + return 1; + } + break; + + case TLS_ST_CR_SESSION_TICKET: + if (mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + st->hand_state = TLS_ST_CR_CHANGE; + return 1; + } + break; + + case TLS_ST_CR_CHANGE: + if (mt == SSL3_MT_FINISHED) { + st->hand_state = TLS_ST_CR_FINISHED; + return 1; + } + break; + + case TLS_ST_OK: + if (mt == SSL3_MT_HELLO_REQUEST) { + st->hand_state = TLS_ST_CR_HELLO_REQ; + return 1; + } + break; + } + + err: + SSLfatal_ntls(s, SSL3_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_MESSAGE); + return 0; +} + +/* + * ossl_statem_client_write_transition_ntls() works out what handshake state to + * move to next when the client is writing messages to be sent to the server. + */ +WRITE_TRAN ossl_statem_client_write_transition_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + + /* + * Note that immediately before/after a ClientHello we don't know what + * version we are going to negotiate yet, so we don't take this branch until + * later + */ + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return WRITE_TRAN_ERROR; + + case TLS_ST_OK: + if (!s->renegotiate) { + /* + * We haven't requested a renegotiation ourselves so we must have + * received a message from the server. Better read it. + */ + return WRITE_TRAN_FINISHED; + } + /* Renegotiation */ + /* fall thru */ + case TLS_ST_BEFORE: + st->hand_state = TLS_ST_CW_CLNT_HELLO; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_CW_CLNT_HELLO: + if (s->early_data_state == SSL_EARLY_DATA_CONNECTING) { + /* + * We are assuming this is a TLSv1.3 connection, although we haven't + * actually selected a version yet. + */ + if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0) + st->hand_state = TLS_ST_CW_CHANGE; + else + st->hand_state = TLS_ST_EARLY_DATA; + return WRITE_TRAN_CONTINUE; + } + /* + * No transition at the end of writing because we don't know what + * we will be sent + */ + return WRITE_TRAN_FINISHED; + + case TLS_ST_CR_SRVR_HELLO: + /* + * We only get here in TLSv1.3. We just received an HRR, so issue a + * CCS unless middlebox compat mode is off, or we already issued one + * because we did early data. + */ + if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0 + && s->early_data_state != SSL_EARLY_DATA_FINISHED_WRITING) + st->hand_state = TLS_ST_CW_CHANGE; + else + st->hand_state = TLS_ST_CW_CLNT_HELLO; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_EARLY_DATA: + return WRITE_TRAN_FINISHED; + + case TLS_ST_CR_SRVR_DONE: + if (s->s3.tmp.cert_req) + st->hand_state = TLS_ST_CW_CERT; + else + st->hand_state = TLS_ST_CW_KEY_EXCH; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_CW_CERT: + st->hand_state = TLS_ST_CW_KEY_EXCH; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_CW_KEY_EXCH: + /* + * For TLS, cert_req is set to 2, so a cert chain of nothing is + * sent, but no verify packet is sent + */ + /* + * XXX: For now, we do not support client authentication in ECDH + * cipher suites with ECDH (rather than ECDSA) certificates. We + * need to skip the certificate verify message when client's + * ECDH public key is sent inside the client certificate. + */ + if (s->s3.tmp.cert_req == 1) { + st->hand_state = TLS_ST_CW_CERT_VRFY; + } else { + st->hand_state = TLS_ST_CW_CHANGE; + } + if (s->s3.flags & TLS1_FLAGS_SKIP_CERT_VERIFY) { + st->hand_state = TLS_ST_CW_CHANGE; + } + return WRITE_TRAN_CONTINUE; + + case TLS_ST_CW_CERT_VRFY: + st->hand_state = TLS_ST_CW_CHANGE; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_CW_CHANGE: + if (s->hello_retry_request == SSL_HRR_PENDING) { + st->hand_state = TLS_ST_CW_CLNT_HELLO; + } else if (s->early_data_state == SSL_EARLY_DATA_CONNECTING) { + st->hand_state = TLS_ST_EARLY_DATA; + } else { +#if defined(OPENSSL_NO_NEXTPROTONEG) + st->hand_state = TLS_ST_CW_FINISHED; +#else + if (s->s3.npn_seen) + st->hand_state = TLS_ST_CW_NEXT_PROTO; + else + st->hand_state = TLS_ST_CW_FINISHED; +#endif + } + return WRITE_TRAN_CONTINUE; + +#if !defined(OPENSSL_NO_NEXTPROTONEG) + case TLS_ST_CW_NEXT_PROTO: + st->hand_state = TLS_ST_CW_FINISHED; + return WRITE_TRAN_CONTINUE; +#endif + + case TLS_ST_CW_FINISHED: + if (s->hit) { + st->hand_state = TLS_ST_OK; + return WRITE_TRAN_CONTINUE; + } else { + return WRITE_TRAN_FINISHED; + } + + case TLS_ST_CR_FINISHED: + if (s->hit) { + st->hand_state = TLS_ST_CW_CHANGE; + return WRITE_TRAN_CONTINUE; + } else { + st->hand_state = TLS_ST_OK; + return WRITE_TRAN_CONTINUE; + } + + case TLS_ST_CR_HELLO_REQ: + /* + * If we can renegotiate now then do so, otherwise wait for a more + * convenient time. + */ + if (ssl3_renegotiate_check(s, 1)) { + if (!tls_setup_handshake_ntls(s)) { + /* SSLfatal_ntls() already called */ + return WRITE_TRAN_ERROR; + } + st->hand_state = TLS_ST_CW_CLNT_HELLO; + return WRITE_TRAN_CONTINUE; + } + st->hand_state = TLS_ST_OK; + return WRITE_TRAN_CONTINUE; + } +} + +/* + * Perform any pre work that needs to be done prior to sending a message from + * the client to the server. + */ +WORK_STATE ossl_statem_client_pre_work_ntls(SSL *s, WORK_STATE wst) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* No pre work to be done */ + break; + + case TLS_ST_CW_CLNT_HELLO: + s->shutdown = 0; + break; + + case TLS_ST_CW_CHANGE: + break; + + case TLS_ST_PENDING_EARLY_DATA_END: + /* + * If we've been called by SSL_do_handshake()/SSL_write(), or we did not + * attempt to write early data before calling SSL_read() then we press + * on with the handshake. Otherwise we pause here. + */ + if (s->early_data_state == SSL_EARLY_DATA_FINISHED_WRITING + || s->early_data_state == SSL_EARLY_DATA_NONE) + return WORK_FINISHED_CONTINUE; + /* Fall through */ + + case TLS_ST_EARLY_DATA: + return tls_finish_handshake_ntls(s, wst, 0, 1); + + case TLS_ST_OK: + /* Calls SSLfatal_ntls() as required */ + return tls_finish_handshake_ntls(s, wst, 1, 1); + } + + return WORK_FINISHED_CONTINUE; +} + +/* + * Perform any work that needs to be done after sending a message from the + * client to the server. + */ +WORK_STATE ossl_statem_client_post_work_ntls(SSL *s, WORK_STATE wst) +{ + OSSL_STATEM *st = &s->statem; + + s->init_num = 0; + + switch (st->hand_state) { + default: + /* No post work to be done */ + break; + + case TLS_ST_CW_CLNT_HELLO: + if (s->early_data_state == SSL_EARLY_DATA_CONNECTING + && s->max_early_data > 0) { + /* + * We haven't selected TLSv1.3 yet so we don't call the change + * cipher state function associated with the SSL_METHOD. Instead + * we call tls13_change_cipher_state() directly. + */ + if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) == 0) { + if (!tls13_change_cipher_state(s, + SSL3_CC_EARLY | SSL3_CHANGE_CIPHER_CLIENT_WRITE)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + } + /* else we're in compat mode so we delay flushing until after CCS */ + } else if (!statem_flush_ntls(s)) { + return WORK_MORE_A; + } + + break; + + case TLS_ST_CW_END_OF_EARLY_DATA: + /* + * We set the enc_write_ctx back to NULL because we may end up writing + * in cleartext again if we get a HelloRetryRequest from the server. + */ + EVP_CIPHER_CTX_free(s->enc_write_ctx); + s->enc_write_ctx = NULL; + break; + + case TLS_ST_CW_KEY_EXCH: + if (tls_client_key_exchange_post_work_ntls(s) == 0) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + break; + + case TLS_ST_CW_CHANGE: + if (s->hello_retry_request == SSL_HRR_PENDING) + break; + if (s->early_data_state == SSL_EARLY_DATA_CONNECTING + && s->max_early_data > 0) { + /* + * We haven't selected TLSv1.3 yet so we don't call the change + * cipher state function associated with the SSL_METHOD. Instead + * we call tls13_change_cipher_state() directly. + */ + if (!tls13_change_cipher_state(s, + SSL3_CC_EARLY | SSL3_CHANGE_CIPHER_CLIENT_WRITE)) + return WORK_ERROR; + break; + } + s->session->cipher = s->s3.tmp.new_cipher; +#ifdef OPENSSL_NO_COMP + s->session->compress_meth = 0; +#else + if (s->s3.tmp.new_compression == NULL) + s->session->compress_meth = 0; + else + s->session->compress_meth = s->s3.tmp.new_compression->id; +#endif + if (!s->method->ssl3_enc->setup_key_block(s)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + + if (!s->method->ssl3_enc->change_cipher_state(s, + SSL3_CHANGE_CIPHER_CLIENT_WRITE)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + + break; + + case TLS_ST_CW_FINISHED: + if (statem_flush_ntls(s) != 1) + return WORK_MORE_B; + break; + + case TLS_ST_CW_KEY_UPDATE: + if (statem_flush_ntls(s) != 1) + return WORK_MORE_A; + if (!tls13_update_key(s, 1)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + break; + } + + return WORK_FINISHED_CONTINUE; +} + +/* + * Get the message construction function and message type for sending from the + * client + * + * Valid return values are: + * 1: Success + * 0: Error + */ +int ossl_statem_client_construct_message_ntls(SSL *s, WPACKET *pkt, + confunc_f *confunc, int *mt) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_HANDSHAKE_STATE); + return 0; + + case TLS_ST_CW_CHANGE: + *confunc = tls_construct_change_cipher_spec_ntls; + *mt = SSL3_MT_CHANGE_CIPHER_SPEC; + break; + + case TLS_ST_CW_CLNT_HELLO: + *confunc = tls_construct_client_hello_ntls; + *mt = SSL3_MT_CLIENT_HELLO; + break; + + case TLS_ST_CW_END_OF_EARLY_DATA: + *confunc = tls_construct_end_of_early_data_ntls; + *mt = SSL3_MT_END_OF_EARLY_DATA; + break; + + case TLS_ST_PENDING_EARLY_DATA_END: + *confunc = NULL; + *mt = SSL3_MT_DUMMY; + break; + + case TLS_ST_CW_CERT: + *confunc = tls_construct_client_certificate_ntls; + *mt = SSL3_MT_CERTIFICATE; + break; + + case TLS_ST_CW_KEY_EXCH: + *confunc = tls_construct_client_key_exchange_ntls; + *mt = SSL3_MT_CLIENT_KEY_EXCHANGE; + break; + + case TLS_ST_CW_CERT_VRFY: + *confunc = tls_construct_cert_verify_ntls; + *mt = SSL3_MT_CERTIFICATE_VERIFY; + break; + +#if !defined(OPENSSL_NO_NEXTPROTONEG) + case TLS_ST_CW_NEXT_PROTO: + *confunc = tls_construct_next_proto_ntls; + *mt = SSL3_MT_NEXT_PROTO; + break; +#endif + case TLS_ST_CW_FINISHED: + *confunc = tls_construct_finished_ntls; + *mt = SSL3_MT_FINISHED; + break; + } + return 1; +} + +/* + * Returns the maximum allowed length for the current message that we are + * reading. Excludes the message header. + */ +size_t ossl_statem_client_max_message_size_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + return 0; + + case TLS_ST_CR_SRVR_HELLO: + return SERVER_HELLO_MAX_LENGTH; + + case TLS_ST_CR_CERT: + return s->max_cert_list; + + case TLS_ST_CR_CERT_STATUS: + return SSL3_RT_MAX_PLAIN_LENGTH; + + case TLS_ST_CR_KEY_EXCH: + return SERVER_KEY_EXCH_MAX_LENGTH; + + case TLS_ST_CR_CERT_REQ: + /* + * Set to s->max_cert_list for compatibility with previous releases. In + * practice these messages can get quite long if servers are configured + * to provide a long list of acceptable CAs + */ + return s->max_cert_list; + + case TLS_ST_CR_SRVR_DONE: + return SERVER_HELLO_DONE_MAX_LENGTH; + + case TLS_ST_CR_CHANGE: + return CCS_MAX_LENGTH; + + case TLS_ST_CR_SESSION_TICKET: + return SESSION_TICKET_MAX_LENGTH_TLS12; + + case TLS_ST_CR_FINISHED: + return FINISHED_MAX_LENGTH; + + case TLS_ST_CR_ENCRYPTED_EXTENSIONS: + return ENCRYPTED_EXTENSIONS_MAX_LENGTH; + + case TLS_ST_CR_KEY_UPDATE: + return KEY_UPDATE_MAX_LENGTH; + } +} + +/* + * Process a message that the client has received from the server. + */ +MSG_PROCESS_RETURN ossl_statem_client_process_message_ntls(SSL *s, PACKET *pkt) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return MSG_PROCESS_ERROR; + + case TLS_ST_CR_SRVR_HELLO: + return tls_process_server_hello_ntls(s, pkt); + + case TLS_ST_CR_CERT: + return tls_process_server_certificate_ntls(s, pkt); + + case TLS_ST_CR_CERT_STATUS: + return tls_process_cert_status_ntls(s, pkt); + + case TLS_ST_CR_KEY_EXCH: + return tls_process_key_exchange_ntls(s, pkt); + + case TLS_ST_CR_CERT_REQ: + return tls_process_certificate_request_ntls(s, pkt); + + case TLS_ST_CR_SRVR_DONE: + return tls_process_server_done_ntls(s, pkt); + + case TLS_ST_CR_CHANGE: + return tls_process_change_cipher_spec_ntls(s, pkt); + + case TLS_ST_CR_SESSION_TICKET: + return tls_process_new_session_ticket_ntls(s, pkt); + + case TLS_ST_CR_FINISHED: + return tls_process_finished_ntls(s, pkt); + + case TLS_ST_CR_HELLO_REQ: + return tls_process_hello_req_ntls(s, pkt); + } +} + +/* + * Perform any further processing required following the receipt of a message + * from the server + */ +WORK_STATE ossl_statem_client_post_process_message_ntls(SSL *s, WORK_STATE wst) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return WORK_ERROR; + + case TLS_ST_CR_CERT: + return tls_post_process_server_certificate_ntls(s, wst); + + case TLS_ST_CR_CERT_REQ: + return tls_prepare_client_certificate_ntls(s, wst); + } +} + +int tls_construct_client_hello_ntls(SSL *s, WPACKET *pkt) +{ + unsigned char *p; + size_t sess_id_len; + int i, protverr; + SSL_SESSION *sess = s->session; + unsigned char *session_id; + + /* Work out what SSL/TLS version to use */ + protverr = ssl_set_client_hello_version_ntls(s); + if (protverr != 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, protverr); + return 0; + } + + if (sess == NULL + || !ssl_version_supported_ntls(s, sess->ssl_version, NULL) + || !SSL_SESSION_is_resumable(sess)) { + if (s->hello_retry_request == SSL_HRR_NONE + && !ssl_get_new_session(s, 0)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + /* else use the pre-loaded session */ + + p = s->s3.client_random; + i = (s->hello_retry_request == SSL_HRR_NONE); + + if (i && ssl_fill_hello_random(s, 0, p, sizeof(s->s3.client_random), + DOWNGRADE_NONE) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /*- + * version indicates the negotiated version: for example from + * an SSLv2/v3 compatible client hello). The client_version + * field is the maximum version we permit and it is also + * used in RSA encrypted premaster secrets. Some servers can + * choke if we initially report a higher version then + * renegotiate to a lower one in the premaster secret. This + * didn't happen with TLS 1.0 as most servers supported it + * but it can with TLS 1.1 or later if the server only supports + * 1.0. + * + * Possible scenario with previous logic: + * 1. Client hello indicates TLS 1.2 + * 2. Server hello says TLS 1.0 + * 3. RSA encrypted premaster secret uses 1.2. + * 4. Handshake proceeds using TLS 1.0. + * 5. Server sends hello request to renegotiate. + * 6. Client hello indicates TLS v1.0 as we now + * know that is maximum server supports. + * 7. Server chokes on RSA encrypted premaster secret + * containing version 1.0. + * + * For interoperability it should be OK to always use the + * maximum version we support in client hello and then rely + * on the checking of version to ensure the servers isn't + * being inconsistent: for example initially negotiating with + * TLS 1.0 and renegotiating with TLS 1.2. We do this by using + * client_version in client hello and not resetting it to + * the negotiated version. + * + * For TLS 1.3 we always set the ClientHello version to 1.2 and rely on the + * supported_versions extension for the real supported versions. + */ + if (!WPACKET_put_bytes_u16(pkt, s->client_version) + || !WPACKET_memcpy(pkt, s->s3.client_random, SSL3_RANDOM_SIZE)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* Session ID */ + session_id = s->session->session_id; + if (s->new_session || s->session->ssl_version == TLS1_3_VERSION) { + if (s->version == TLS1_3_VERSION + && (s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0) { + sess_id_len = sizeof(s->tmp_session_id); + s->tmp_session_id_len = sess_id_len; + session_id = s->tmp_session_id; + if (s->hello_retry_request == SSL_HRR_NONE + && RAND_bytes_ex(s->ctx->libctx, s->tmp_session_id, + sess_id_len, 0) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } else { + sess_id_len = 0; + } + } else { + assert(s->session->session_id_length <= sizeof(s->session->session_id)); + sess_id_len = s->session->session_id_length; + if (s->version == TLS1_3_VERSION) { + s->tmp_session_id_len = sess_id_len; + memcpy(s->tmp_session_id, s->session->session_id, sess_id_len); + } + } + if (!WPACKET_start_sub_packet_u8(pkt) + || (sess_id_len != 0 && !WPACKET_memcpy(pkt, session_id, + sess_id_len)) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* Ciphers supported */ + if (!WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!ssl_cipher_list_to_bytes(s, SSL_get_ciphers(s), pkt)) { + /* SSLfatal_ntls() already called */ + return 0; + } + if (!WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* COMPRESSION */ + if (!WPACKET_start_sub_packet_u8(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* Add the NULL method */ + if (!WPACKET_put_bytes_u8(pkt, 0) || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* TLS extensions */ + if (!tls_construct_extensions_ntls(s, pkt, SSL_EXT_CLIENT_HELLO, NULL, 0)) { + /* SSLfatal() already called */ + return 0; + } + + return 1; +} + +static int set_client_ciphersuite_ntls(SSL *s, const unsigned char *cipherchars) +{ + STACK_OF(SSL_CIPHER) *sk; + const SSL_CIPHER *c; + int i; + + c = ssl_get_cipher_by_char(s, cipherchars, 0); + if (c == NULL) { + /* unknown cipher */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_UNKNOWN_CIPHER_RETURNED); + return 0; + } + /* + * If it is a disabled cipher we either didn't send it in client hello, + * or it's not allowed for the selected protocol. So we return an error. + */ + if (ssl_cipher_disabled(s, c, SSL_SECOP_CIPHER_CHECK, 1)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_CIPHER_RETURNED); + return 0; + } + + sk = ssl_get_ciphers_by_id(s); + i = sk_SSL_CIPHER_find(sk, c); + if (i < 0) { + /* we did not say we would use this cipher */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_CIPHER_RETURNED); + return 0; + } + + /* + * Depending on the session caching (internal/external), the cipher + * and/or cipher_id values may not be set. Make sure that cipher_id is + * set and use it for comparison. + */ + if (s->session->cipher != NULL) + s->session->cipher_id = s->session->cipher->id; + if (s->hit && (s->session->cipher_id != c->id)) { + /* + * Prior to TLSv1.3 resuming a session always meant using the same + * ciphersuite. + */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_OLD_SESSION_CIPHER_NOT_RETURNED); + return 0; + } + s->s3.tmp.new_cipher = c; + + return 1; +} + +MSG_PROCESS_RETURN tls_process_server_hello_ntls(SSL *s, PACKET *pkt) +{ + PACKET session_id, extpkt; + size_t session_id_len; + const unsigned char *cipherchars; + int hrr = 0; + unsigned int compression; + unsigned int sversion; + unsigned int context; + RAW_EXTENSION *extensions = NULL; + + if (!PACKET_get_net_2(pkt, &sversion)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + /* load the server random */ + if (s->version == TLS1_3_VERSION + && sversion == TLS1_2_VERSION + && PACKET_remaining(pkt) >= SSL3_RANDOM_SIZE + && memcmp(hrrrandom_ntls, PACKET_data(pkt), SSL3_RANDOM_SIZE) == 0) { + s->hello_retry_request = SSL_HRR_PENDING; + hrr = 1; + if (!PACKET_forward(pkt, SSL3_RANDOM_SIZE)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + } else { + if (!PACKET_copy_bytes(pkt, s->s3.server_random, SSL3_RANDOM_SIZE)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + } + + /* Get the session-id. */ + if (!PACKET_get_length_prefixed_1(pkt, &session_id)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + session_id_len = PACKET_remaining(&session_id); + if (session_id_len > sizeof(s->session->session_id) + || session_id_len > SSL3_SESSION_ID_SIZE) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_SSL3_SESSION_ID_TOO_LONG); + goto err; + } + + if (!PACKET_get_bytes(pkt, &cipherchars, TLS_CIPHER_LEN)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + if (!PACKET_get_1(pkt, &compression)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + /* TLS extensions */ + if (PACKET_remaining(pkt) == 0 && !hrr) { + PACKET_null_init(&extpkt); + } else if (!PACKET_as_length_prefixed_2(pkt, &extpkt) + || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_LENGTH); + goto err; + } + + if (!hrr) { + if (!tls_collect_extensions_ntls(s, &extpkt, + SSL_EXT_TLS1_2_SERVER_HELLO + | SSL_EXT_TLS1_3_SERVER_HELLO, + &extensions, NULL, 1)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + if (!ssl_choose_client_version_ntls(s, sversion, extensions)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } + + if (hrr) { + if (compression != 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_INVALID_COMPRESSION_ALGORITHM); + goto err; + } + + if (session_id_len != s->tmp_session_id_len + || memcmp(PACKET_data(&session_id), s->tmp_session_id, + session_id_len) != 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_INVALID_SESSION_ID); + goto err; + } + } + + if (hrr) { + if (!set_client_ciphersuite_ntls(s, cipherchars)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + return tls_process_as_hello_retry_request(s, &extpkt); + } + + /* + * Now we have chosen the version we need to check again that the extensions + * are appropriate for this version. + */ + context = SSL_EXT_TLS1_2_SERVER_HELLO; + if (!tls_validate_all_contexts_ntls(s, context, extensions)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_EXTENSION); + goto err; + } + + s->hit = 0; + + /* + * Check if we can resume the session based on external pre-shared + * secret. EAP-FAST (RFC 4851) supports two types of session resumption. + * Resumption based on server-side state works with session IDs. + * Resumption based on pre-shared Protected Access Credentials (PACs) + * works by overriding the SessionTicket extension at the application + * layer, and does not send a session ID. (We do not know whether + * EAP-FAST servers would honour the session ID.) Therefore, the session + * ID alone is not a reliable indicator of session resumption, so we + * first check if we can resume, and later peek at the next handshake + * message to see if the server wants to resume. + */ + if (s->version >= NTLS_VERSION + && s->ext.session_secret_cb != NULL && s->session->ext.tick) { + const SSL_CIPHER *pref_cipher = NULL; + /* + * s->session->master_key_length is a size_t, but this is an int for + * backwards compat reasons + */ + int master_key_length; + master_key_length = sizeof(s->session->master_key); + if (s->ext.session_secret_cb(s, s->session->master_key, + &master_key_length, + NULL, &pref_cipher, + s->ext.session_secret_cb_arg) + && master_key_length > 0) { + s->session->master_key_length = master_key_length; + s->session->cipher = pref_cipher ? + pref_cipher : ssl_get_cipher_by_char(s, cipherchars, 0); + } else { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } + + if (session_id_len != 0 + && session_id_len == s->session->session_id_length + && memcmp(PACKET_data(&session_id), s->session->session_id, + session_id_len) == 0) + s->hit = 1; + + if (s->hit) { + if (s->sid_ctx_length != s->session->sid_ctx_length + || memcmp(s->session->sid_ctx, s->sid_ctx, s->sid_ctx_length)) { + /* actually a client application bug */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT); + goto err; + } + } else { + /* + * If we were trying for session-id reuse but the server + * didn't resume, make a new SSL_SESSION. + * In the case of EAP-FAST and PAC, we do not send a session ID, + * so the PAC-based session secret is always preserved. It'll be + * overwritten if the server refuses resumption. + */ + if (s->session->session_id_length > 0) { + ssl_tsan_counter(s->session_ctx, &s->session_ctx->stats.sess_miss); + if (!ssl_get_new_session(s, 0)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } + + s->session->ssl_version = s->version; + /* + * In TLSv1.2 and below we save the session id we were sent so we can + * resume it later. In TLSv1.3 the session id we were sent is just an + * echo of what we originally sent in the ClientHello and should not be + * used for resumption. + */ + s->session->session_id_length = session_id_len; + /* session_id_len could be 0 */ + if (session_id_len > 0) + memcpy(s->session->session_id, PACKET_data(&session_id), + session_id_len); + } + + /* Session version and negotiated protocol version should match */ + if (s->version != s->session->ssl_version) { + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, + SSL_R_SSL_SESSION_VERSION_MISMATCH); + goto err; + } + /* + * Now that we know the version, update the check to see if it's an allowed + * version. + */ + s->s3.tmp.min_ver = s->version; + s->s3.tmp.max_ver = s->version; + + if (!set_client_ciphersuite_ntls(s, cipherchars)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + if (compression != 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM); + goto err; + } + /* + * If compression is disabled we'd better not try to resume a session + * using compression. + */ + if (s->session->compress_meth != 0) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_INCONSISTENT_COMPRESSION); + goto err; + } + + + if (!tls_parse_all_extensions_ntls(s, context, extensions, NULL, 0, 1)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + /* + * In TLSv1.3 we have some post-processing to change cipher state, otherwise + * we're done with this message + */ + OPENSSL_free(extensions); + return MSG_PROCESS_CONTINUE_READING; + err: + OPENSSL_free(extensions); + return MSG_PROCESS_ERROR; +} + +static MSG_PROCESS_RETURN tls_process_as_hello_retry_request(SSL *s, + PACKET *extpkt) +{ + RAW_EXTENSION *extensions = NULL; + + /* + * If we were sending early_data then the enc_write_ctx is now invalid and + * should not be used. + */ + EVP_CIPHER_CTX_free(s->enc_write_ctx); + s->enc_write_ctx = NULL; + + if (!tls_collect_extensions_ntls(s, extpkt, SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST, + &extensions, NULL, 1) + || !tls_parse_all_extensions_ntls(s, SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST, + extensions, NULL, 0, 1)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + OPENSSL_free(extensions); + extensions = NULL; + + if (s->ext.tls13_cookie_len == 0 && s->s3.tmp.pkey != NULL) { + /* + * We didn't receive a cookie or a new key_share so the next + * ClientHello will not change + */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_NO_CHANGE_FOLLOWING_HRR); + goto err; + } + + /* + * Re-initialise the Transcript Hash. We're going to prepopulate it with + * a synthetic message_hash in place of ClientHello1. + */ + if (!create_synthetic_message_hash_ntls(s, NULL, 0, NULL, 0)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + /* + * Add this message to the Transcript Hash. Normally this is done + * automatically prior to the message processing stage. However due to the + * need to create the synthetic message hash, we defer that step until now + * for HRR messages. + */ + if (!ssl3_finish_mac(s, (unsigned char *)s->init_buf->data, + s->init_num + SSL3_HM_HEADER_LENGTH)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + return MSG_PROCESS_FINISHED_READING; + err: + OPENSSL_free(extensions); + return MSG_PROCESS_ERROR; +} + +/* prepare server cert verification by setting s->session->peer_chain from pkt */ +MSG_PROCESS_RETURN tls_process_server_certificate_ntls(SSL *s, PACKET *pkt) +{ + unsigned long cert_list_len, cert_len; + X509 *x = NULL; + const unsigned char *certstart, *certbytes; + unsigned int context = 0; + + if ((s->session->peer_chain = sk_X509_new_null()) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (context != 0 + || !PACKET_get_net_3(pkt, &cert_list_len) + || PACKET_remaining(pkt) != cert_list_len + || PACKET_remaining(pkt) == 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + while (PACKET_remaining(pkt)) { + if (!PACKET_get_net_3(pkt, &cert_len) + || !PACKET_get_bytes(pkt, &certbytes, cert_len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_CERT_LENGTH_MISMATCH); + goto err; + } + + certstart = certbytes; + x = X509_new_ex(s->ctx->libctx, s->ctx->propq); + if (x == NULL) { + SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_MALLOC_FAILURE); + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + goto err; + } + if (d2i_X509(&x, (const unsigned char **)&certbytes, + cert_len) == NULL) { + SSLfatal_ntls(s, SSL_AD_BAD_CERTIFICATE, ERR_R_ASN1_LIB); + goto err; + } + + if (certbytes != (certstart + cert_len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_CERT_LENGTH_MISMATCH); + goto err; + } + + if (!sk_X509_push(s->session->peer_chain, x)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + x = NULL; + } + +# ifndef OPENSSL_NO_SM2 + { + EVP_PKEY *pkey = NULL; + int n = sk_X509_num(s->session->peer_chain) - 1; + + x = sk_X509_value(s->session->peer_chain, 0); + pkey = X509_get0_pubkey(x); + + if (pkey != NULL && EVP_PKEY_is_sm2(pkey)) { + if (!EVP_PKEY_set_alias_type(pkey, EVP_PKEY_SM2)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + while (n >= 0) { + X509 *cert = sk_X509_value(s->session->peer_chain, n); + ASN1_OCTET_STRING *sm2_id; + sm2_id = ASN1_OCTET_STRING_new(); + + if (sm2_id == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!ASN1_OCTET_STRING_set(sm2_id, + (const unsigned char *)CERTVRIFY_SM2_ID, + CERTVRIFY_SM2_ID_LEN)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + ASN1_OCTET_STRING_free(sm2_id); + goto err; + } + + X509_set0_sm2_id(cert, sm2_id); + n--; + } + } + } +# endif + + return MSG_PROCESS_CONTINUE_PROCESSING; + + err: + X509_free(x); + sk_X509_pop_free(s->session->peer_chain, X509_free); + s->session->peer_chain = NULL; + return MSG_PROCESS_ERROR; +} + +/* + * Verify the s->session->peer_chain and check server cert type. + * On success set s->session->peer and s->session->verify_result. + * Else the peer certificate verification callback may request retry. + */ +WORK_STATE tls_post_process_server_certificate_ntls(SSL *s, WORK_STATE wst) +{ + X509 *x = NULL; + EVP_PKEY *pkey = NULL; + STACK_OF(X509) *sk = s->session->peer_chain; + const SSL_CERT_LOOKUP *clu; + size_t certidx; + int i, j; + + if (s->rwstate == SSL_RETRY_VERIFY) + s->rwstate = SSL_NOTHING; + + if (sk_X509_num(sk) >= 2) { + for (j = 0; j < 2; j++) { + if (j == 0) + sk_X509_push(sk, sk_X509_shift(sk)); + if (j == 1) + sk_X509_unshift(sk, sk_X509_pop(sk)); + + i = ssl_verify_cert_chain(s, sk); + if (i > 0 && s->rwstate == SSL_RETRY_VERIFY) { + return WORK_MORE_A; + } + + /* + * The documented interface is that SSL_VERIFY_PEER should be set in order + * for client side verification of the server certificate to take place. + * However, historically the code has only checked that *any* flag is set + * to cause server verification to take place. Use of the other flags makes + * no sense in client mode. An attempt to clean up the semantics was + * reverted because at least one application *only* set + * SSL_VERIFY_FAIL_IF_NO_PEER_CERT. Prior to the clean up this still caused + * server verification to take place, after the clean up it silently did + * nothing. SSL_CTX_set_verify()/SSL_set_verify() cannot validate the flags + * sent to them because they are void functions. Therefore, we now use the + * (less clean) historic behaviour of performing validation if any flag is + * set. The *documented* interface remains the same. + */ + if (s->verify_mode != SSL_VERIFY_NONE && i <= 0) { + SSLfatal_ntls(s, ssl_x509err2alert_ntls(s->verify_result), + SSL_R_CERTIFICATE_VERIFY_FAILED); + return WORK_ERROR; + } + } + + ERR_clear_error(); /* but we keep s->verify_result */ + + /* + * Inconsistency alert: cert_chain does include the peer's certificate, + * which we don't include in statem_srvr.c + */ + x = sk_X509_value(sk, 0); + + pkey = X509_get0_pubkey(x); + + if (pkey == NULL || EVP_PKEY_missing_parameters(pkey)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS); + return WORK_ERROR; + } + + if ((clu = ssl_cert_lookup_by_pkey(pkey, &certidx)) == NULL) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_UNKNOWN_CERTIFICATE_TYPE); + return WORK_ERROR; + } + + /* + * Check certificate type is consistent with ciphersuite. For TLS 1.3 + * skip check since TLS 1.3 ciphersuites can be used with any certificate + * type. + */ + if ((clu->amask & s->s3.tmp.new_cipher->algorithm_auth) == 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_CERTIFICATE_TYPE); + return WORK_ERROR; + } + } else { + if (s->verify_mode != SSL_VERIFY_NONE) { + SSLfatal_ntls(s, ssl_x509err2alert_ntls(s->verify_result), + SSL_R_CERTIFICATE_VERIFY_FAILED); + return WORK_ERROR; + } + } + + X509_free(s->session->peer); + X509_up_ref(x); + s->session->peer = x; + s->session->verify_result = s->verify_result; + + return WORK_FINISHED_CONTINUE; +} + +static int tls_process_ske_sm2dhe_ntls(SSL *s, PACKET *pkt) +{ + PACKET encoded_pt; + unsigned int curve_type, curve_id; + + if (!PACKET_get_1(pkt, &curve_type) || !PACKET_get_net_2(pkt, &curve_id)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); + return 0; + } + + if ((s->s3.peer_tmp = + ssl_generate_param_group(s, OSSL_TLS_GROUP_ID_sm2)) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_UNABLE_TO_FIND_ECDH_PARAMETERS); + return 0; + } + + if (!PACKET_get_length_prefixed_1(pkt, &encoded_pt)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + if (EVP_PKEY_set1_encoded_public_key(s->s3.peer_tmp, + PACKET_data(&encoded_pt), + PACKET_remaining(&encoded_pt)) <= 0) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_ECPOINT); + return 0; + } + + /* Cache the agreed upon group in the SSL_SESSION */ + s->session->kex_group = curve_id; + return 1; +} + +MSG_PROCESS_RETURN tls_process_key_exchange_ntls(SSL *s, PACKET *pkt) +{ + long alg_k; + EVP_PKEY *pkey = NULL; + EVP_MD_CTX *md_ctx = NULL; + EVP_PKEY_CTX *pctx = NULL; + PACKET save_param_start, signature; + unsigned char *buf = NULL; + size_t buflen; + + alg_k = s->s3.tmp.new_cipher->algorithm_mkey; + + save_param_start = *pkt; + + EVP_PKEY_free(s->s3.peer_tmp); + s->s3.peer_tmp = NULL; + + if (alg_k & SSL_kSM2DHE) { + if (!tls_process_ske_sm2dhe_ntls(s, pkt)) { + /* SSLfatal_ntls already called */ + goto err; + } + } + + /* get peer signing pkey */ + pkey = X509_get0_pubkey(s->session->peer); + + /* if it was signed, check the signature */ + if (pkey != NULL) { + PACKET params; + const EVP_MD *md = NULL; + unsigned char *tbs; + size_t tbslen; + X509 *x509; + int rv; + + if (alg_k & SSL_kSM2DHE) { + /* + * |pkt| now points to the beginning of the signature, so the difference + * equals the length of the parameters. + */ + if (!PACKET_get_sub_packet(&save_param_start, ¶ms, + PACKET_remaining(&save_param_start) - + PACKET_remaining(pkt))) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } else if (alg_k & (SSL_kSM2 | SSL_kRSA)) { + /* get peer's encryption cert */ + x509 = sk_X509_value(s->session->peer_chain, 1); + if (x509 == NULL + || (buf = x509_to_asn1_ntls(x509, &buflen)) == NULL + || !PACKET_buf_init(¶ms, buf, buflen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + } else { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (!tls1_set_peer_legacy_sigalg(s, pkey)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (!tls1_lookup_md(s->ctx, s->s3.tmp.peer_sigalg, &md)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_NO_SUITABLE_DIGEST_ALGORITHM); + goto err; + } + + if (!PACKET_get_length_prefixed_2(pkt, &signature) + || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + md_ctx = EVP_MD_CTX_new(); + if (md_ctx == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (EVP_DigestVerifyInit_ex(md_ctx, &pctx, + md == NULL ? NULL : EVP_MD_get0_name(md), + s->ctx->libctx, s->ctx->propq, pkey, + NULL) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) { + if (EVP_PKEY_CTX_set1_id(pctx, SM2_DEFAULT_ID, + SM2_DEFAULT_ID_LEN) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + } + + tbslen = construct_key_exchange_tbs_ntls(s, &tbs, PACKET_data(¶ms), + PACKET_remaining(¶ms)); + if (tbslen == 0) { + /* SSLfatal_ntls() already called */ + goto err; + } + OPENSSL_free(buf); + buf = NULL; + + rv = EVP_DigestVerify(md_ctx, PACKET_data(&signature), + PACKET_remaining(&signature), tbs, tbslen); + OPENSSL_free(tbs); + if (rv <= 0) { + SSLfatal_ntls(s, SSL_AD_DECRYPT_ERROR, SSL_R_BAD_SIGNATURE); + goto err; + } + EVP_MD_CTX_free(md_ctx); + md_ctx = NULL; + } else { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_MISSING_SIGNING_CERT); + goto err; + } + + return MSG_PROCESS_CONTINUE_READING; + err: + OPENSSL_free(buf); + EVP_MD_CTX_free(md_ctx); + return MSG_PROCESS_ERROR; +} + +MSG_PROCESS_RETURN tls_process_certificate_request_ntls(SSL *s, PACKET *pkt) +{ + size_t i; + + /* Clear certificate validity flags */ + for (i = 0; i < SSL_PKEY_NUM; i++) + s->s3.tmp.valid_flags[i] = 0; + + { + PACKET ctypes; + + /* get the certificate types */ + if (!PACKET_get_length_prefixed_1(pkt, &ctypes)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return MSG_PROCESS_ERROR; + } + + if (!PACKET_memdup(&ctypes, &s->s3.tmp.ctype, &s->s3.tmp.ctype_len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return MSG_PROCESS_ERROR; + } + + if (SSL_USE_SIGALGS(s)) { + PACKET sigalgs; + + if (!PACKET_get_length_prefixed_2(pkt, &sigalgs)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return MSG_PROCESS_ERROR; + } + + /* + * Despite this being for certificates, preserve compatibility + * with pre-TLS 1.3 and use the regular sigalgs field. + */ + if (!tls1_save_sigalgs(s, &sigalgs, 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_SIGNATURE_ALGORITHMS_ERROR); + return MSG_PROCESS_ERROR; + } + if (!tls1_process_sigalgs(s)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return MSG_PROCESS_ERROR; + } + } + + /* get the CA RDNs */ + if (!parse_ca_names_ntls(s, pkt)) { + /* SSLfatal_ntls() already called */ + return MSG_PROCESS_ERROR; + } + } + + if (PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return MSG_PROCESS_ERROR; + } + + /* we should setup a certificate to return.... */ + s->s3.tmp.cert_req = 1; + + return MSG_PROCESS_CONTINUE_PROCESSING; +} + +MSG_PROCESS_RETURN tls_process_new_session_ticket_ntls(SSL *s, PACKET *pkt) +{ + unsigned int ticklen; + unsigned long ticket_lifetime_hint, age_add = 0; + unsigned int sess_len; + RAW_EXTENSION *exts = NULL; + PACKET nonce; + EVP_MD *sha256 = NULL; + + PACKET_null_init(&nonce); + + if (!PACKET_get_net_4(pkt, &ticket_lifetime_hint) + || !PACKET_get_net_2(pkt, &ticklen) + || (PACKET_remaining(pkt) != ticklen)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + /* + * Server is allowed to change its mind (in <=TLSv1.2) and send an empty + * ticket. We already checked this TLSv1.3 case above, so it should never + * be 0 here in that instance + */ + if (ticklen == 0) + return MSG_PROCESS_CONTINUE_READING; + + /* + * Sessions must be immutable once they go into the session cache. Otherwise + * we can get multi-thread problems. Therefore we don't "update" sessions, + * we replace them with a duplicate. In TLSv1.3 we need to do this every + * time a NewSessionTicket arrives because those messages arrive + * post-handshake and the session may have already gone into the session + * cache. + */ + if (s->session->session_id_length > 0) { + SSL_SESSION *new_sess; + + /* + * We reused an existing session, so we need to replace it with a new + * one + */ + if ((new_sess = ssl_session_dup(s->session, 0)) == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if ((s->session_ctx->session_cache_mode & SSL_SESS_CACHE_CLIENT) != 0) { + /* + * In TLSv1.2 and below the arrival of a new tickets signals that + * any old ticket we were using is now out of date, so we remove the + * old session from the cache. We carry on if this fails + */ + SSL_CTX_remove_session(s->session_ctx, s->session); + } + + SSL_SESSION_free(s->session); + s->session = new_sess; + } + + s->session->time = time(NULL); + ssl_session_calculate_timeout(s->session); + + OPENSSL_free(s->session->ext.tick); + s->session->ext.tick = NULL; + s->session->ext.ticklen = 0; + + s->session->ext.tick = OPENSSL_malloc(ticklen); + if (s->session->ext.tick == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + if (!PACKET_copy_bytes(pkt, s->session->ext.tick, ticklen)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + s->session->ext.tick_lifetime_hint = ticket_lifetime_hint; + s->session->ext.tick_age_add = age_add; + s->session->ext.ticklen = ticklen; + + /* + * There are two ways to detect a resumed ticket session. One is to set + * an appropriate session ID and then the server must return a match in + * ServerHello. This allows the normal client session ID matching to work + * and we know much earlier that the ticket has been accepted. The + * other way is to set zero length session ID when the ticket is + * presented and rely on the handshake to determine session resumption. + * We choose the former approach because this fits in with assumptions + * elsewhere in OpenSSL. The session ID is set to the SHA256 hash of the + * ticket. + */ + sha256 = EVP_MD_fetch(s->ctx->libctx, "SHA2-256", s->ctx->propq); + if (sha256 == NULL) { + /* Error is already recorded */ + SSLfatal_alert_ntls(s, SSL_AD_INTERNAL_ERROR); + goto err; + } + /* + * We use sess_len here because EVP_Digest expects an int + * but s->session->session_id_length is a size_t + */ + if (!EVP_Digest(s->session->ext.tick, ticklen, + s->session->session_id, &sess_len, + sha256, NULL)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + EVP_MD_free(sha256); + sha256 = NULL; + s->session->session_id_length = sess_len; + s->session->not_resumable = 0; + + return MSG_PROCESS_CONTINUE_READING; + err: + EVP_MD_free(sha256); + OPENSSL_free(exts); + return MSG_PROCESS_ERROR; +} + +/* + * In TLSv1.3 this is called from the extensions code, otherwise it is used to + * parse a separate message. Returns 1 on success or 0 on failure + */ +int tls_process_cert_status_body_ntls(SSL *s, PACKET *pkt) +{ + size_t resplen; + unsigned int type; + + if (!PACKET_get_1(pkt, &type) + || type != TLSEXT_STATUSTYPE_ocsp) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_UNSUPPORTED_STATUS_TYPE); + return 0; + } + if (!PACKET_get_net_3_len(pkt, &resplen) + || PACKET_remaining(pkt) != resplen) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + s->ext.ocsp.resp = OPENSSL_malloc(resplen); + if (s->ext.ocsp.resp == NULL) { + s->ext.ocsp.resp_len = 0; + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + s->ext.ocsp.resp_len = resplen; + if (!PACKET_copy_bytes(pkt, s->ext.ocsp.resp, resplen)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + + return 1; +} + + +MSG_PROCESS_RETURN tls_process_cert_status_ntls(SSL *s, PACKET *pkt) +{ + if (!tls_process_cert_status_body_ntls(s, pkt)) { + /* SSLfatal_ntls() already called */ + return MSG_PROCESS_ERROR; + } + + return MSG_PROCESS_CONTINUE_READING; +} + +/* + * Perform miscellaneous checks and processing after we have received the + * server's initial flight. In TLS1.3 this is after the Server Finished message. + * In <=TLS1.2 this is after the ServerDone message. Returns 1 on success or 0 + * on failure. + */ +int tls_process_initial_server_flight_ntls(SSL *s) +{ + /* + * at this point we check that we have the required stuff from + * the server + */ + if (!ssl3_check_cert_and_algorithm_ntls(s)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + /* + * Call the ocsp status callback if needed. The |ext.ocsp.resp| and + * |ext.ocsp.resp_len| values will be set if we actually received a status + * message, or NULL and -1 otherwise + */ + if (s->ext.status_type != TLSEXT_STATUSTYPE_nothing + && s->ctx->ext.status_cb != NULL) { + int ret = s->ctx->ext.status_cb(s, s->ctx->ext.status_arg); + + if (ret == 0) { + SSLfatal_ntls(s, SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE, + SSL_R_INVALID_STATUS_RESPONSE); + return 0; + } + if (ret < 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_OCSP_CALLBACK_FAILURE); + return 0; + } + } +#ifndef OPENSSL_NO_CT + if (s->ct_validation_callback != NULL) { + /* Note we validate the SCTs whether or not we abort on error */ + if (!ssl_validate_ct(s) && (s->verify_mode & SSL_VERIFY_PEER)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } +#endif + + return 1; +} + +MSG_PROCESS_RETURN tls_process_server_done_ntls(SSL *s, PACKET *pkt) +{ + if (PACKET_remaining(pkt) > 0) { + /* should contain no data */ + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return MSG_PROCESS_ERROR; + } + + if (!tls_process_initial_server_flight_ntls(s)) { + /* SSLfatal_ntls() already called */ + return MSG_PROCESS_ERROR; + } + + return MSG_PROCESS_FINISHED_READING; +} + +/* construct encrypted pre master secret for kRSA or kSM2 */ +static int tls_construct_cke_pms_ntls(SSL *s, WPACKET *pkt, unsigned long alg_k) +{ + unsigned char *encbytes1, *encbytes2; + EVP_PKEY *pkey = NULL; + EVP_PKEY_CTX *pctx = NULL; + size_t enclen; + unsigned char *pms = NULL; + size_t pmslen = 0; + X509 *x509; + + /* + * for client side, s->session->peer == s->session->peer_chain[0] is + * the server signing certificate. + * + * s->session->peer_chain[1] is the server encryption certificate + */ + if (s->session->peer_chain == NULL + || (x509 = sk_X509_value(s->session->peer_chain, 1)) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + pkey = X509_get0_pubkey(x509); + if (((alg_k & SSL_kRSA) && !EVP_PKEY_is_a(pkey, "RSA")) + || ((alg_k & SSL_kSM2) && !EVP_PKEY_is_a(pkey, "SM2"))) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + pmslen = SSL_MAX_MASTER_KEY_LENGTH; + pms = OPENSSL_malloc(pmslen); + if (pms == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + + pms[0] = s->client_version >> 8; + pms[1] = s->client_version & 0xff; + if (RAND_bytes_ex(s->ctx->libctx, pms + 2, pmslen - 2, 0) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, pkey, s->ctx->propq); + if (pctx == NULL || EVP_PKEY_encrypt_init(pctx) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + if ((alg_k & SSL_kRSA) + && EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_PADDING) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + if (EVP_PKEY_encrypt(pctx, NULL, &enclen, pms, pmslen) <= 0 + || !WPACKET_sub_reserve_bytes_u16(pkt, enclen, &encbytes1) + || EVP_PKEY_encrypt(pctx, encbytes1, &enclen, pms, pmslen) <= 0 + || !WPACKET_sub_allocate_bytes_u16(pkt, enclen, &encbytes2) + || encbytes1 != encbytes2) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_RSA_ENCRYPT); + goto err; + } + EVP_PKEY_CTX_free(pctx); + pctx = NULL; + /* Log the premaster secret, if logging is enabled. */ + if ((alg_k & SSL_kRSA) + && !ssl_log_rsa_client_key_exchange(s, encbytes1, enclen, pms, pmslen)) + { + /* SSLfatal() already called */ + goto err; + } + + s->s3.tmp.pms = pms; + s->s3.tmp.pmslen = pmslen; + + return 1; + err: + OPENSSL_clear_free(pms, pmslen); + EVP_PKEY_CTX_free(pctx); + + return 0; +} + +static int tls_construct_cke_sm2dhe_ntls(SSL *s, WPACKET *pkt) +{ + unsigned char *encodedPoint = NULL; + size_t encoded_pt_len = 0; + EVP_PKEY *ckey = NULL, *skey = NULL; + int ret = 0; + int curve_id; + + skey = s->s3.peer_tmp; + if (skey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + ckey = ssl_generate_pkey(s, skey); + if (ckey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (ssl_derive_ntls(s, ckey, skey, 0) == 0) { + /* SSLfatal_ntls() already called */ + goto err; + } + + /* Generate encoding of client key */ + encoded_pt_len = EVP_PKEY_get1_encoded_public_key(ckey, &encodedPoint); + if (encoded_pt_len == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EC_LIB); + goto err; + } + + curve_id = tls1_shared_group(s, -2); + + if (!WPACKET_put_bytes_u8(pkt, NAMED_CURVE_TYPE) + || !WPACKET_put_bytes_u8(pkt, 0) + || !WPACKET_put_bytes_u8(pkt, curve_id) + || !WPACKET_sub_memcpy_u8(pkt, encodedPoint, encoded_pt_len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + ret = 1; + err: + OPENSSL_free(encodedPoint); + EVP_PKEY_free(ckey); + return ret; +} + +int tls_construct_client_key_exchange_ntls(SSL *s, WPACKET *pkt) +{ + unsigned long alg_k; + + alg_k = s->s3.tmp.new_cipher->algorithm_mkey; + + if (alg_k & (SSL_kRSA | SSL_kSM2)) { + if (!tls_construct_cke_pms_ntls(s, pkt, alg_k)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } else if (alg_k & (SSL_kSM2DHE)) { + if (!tls_construct_cke_sm2dhe_ntls(s, pkt)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } else { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, ERR_R_INTERNAL_ERROR); + goto err; + } + + return 1; +err: + OPENSSL_clear_free(s->s3.tmp.pms, s->s3.tmp.pmslen); + s->s3.tmp.pms = NULL; + s->s3.tmp.pmslen = 0; + return 0; +} + +int tls_client_key_exchange_post_work_ntls(SSL *s) +{ + unsigned char *pms = NULL; + size_t pmslen = 0; + + pms = s->s3.tmp.pms; + pmslen = s->s3.tmp.pmslen; + + if (pms == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + if (!ssl_generate_master_secret(s, pms, pmslen, 1)) { + /* SSLfatal_ntls() already called */ + /* ssl_generate_master_secret frees the pms even on error */ + pms = NULL; + pmslen = 0; + goto err; + } + pms = NULL; + pmslen = 0; + + return 1; + err: + OPENSSL_clear_free(pms, pmslen); + s->s3.tmp.pms = NULL; + s->s3.tmp.pmslen = 0; + return 0; +} + +/* + * Check a certificate can be used for client authentication. Currently check + * cert exists, if we have a suitable digest for TLS 1.2 if static DH client + * certificates can be used and optionally checks suitability for Suite B. + */ +static int ssl3_check_client_certificate_ntls(SSL *s) +{ + /* If no suitable signature algorithm can't use certificate */ + if (!tls_choose_sigalg_ntls(s, 0) || s->s3.tmp.sigalg == NULL) + return 0; + + /* + * If strict mode check suitability of chain before using it. This also + * adjusts suite B digest if necessary. + */ + if (s->cert->cert_flags & SSL_CERT_FLAGS_CHECK_TLS_STRICT && + !tls1_check_chain(s, NULL, NULL, NULL, -2)) + return 0; + return 1; +} + +WORK_STATE tls_prepare_client_certificate_ntls(SSL *s, WORK_STATE wst) +{ + X509 *x509 = NULL; + EVP_PKEY *pkey = NULL; + int i; + + if (wst == WORK_MORE_A) { + /* Let cert callback update client certificates if required */ + if (s->cert->cert_cb) { + i = s->cert->cert_cb(s, s->cert->cert_cb_arg); + if (i < 0) { + s->rwstate = SSL_X509_LOOKUP; + return WORK_MORE_A; + } + if (i == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_CALLBACK_FAILED); + return WORK_ERROR; + } + s->rwstate = SSL_NOTHING; + } + if (ssl3_check_client_certificate_ntls(s)) { + if (s->post_handshake_auth == SSL_PHA_REQUESTED) { + return WORK_FINISHED_STOP; + } + return WORK_FINISHED_CONTINUE; + } + + /* Fall through to WORK_MORE_B */ + wst = WORK_MORE_B; + } + + /* We need to get a client cert */ + if (wst == WORK_MORE_B) { + /* + * If we get an error, we need to ssl->rwstate=SSL_X509_LOOKUP; + * return(-1); We then get retied later + */ + i = ssl_do_client_cert_cb_ntls(s, &x509, &pkey); + if (i < 0) { + s->rwstate = SSL_X509_LOOKUP; + return WORK_MORE_B; + } + s->rwstate = SSL_NOTHING; + if ((i == 1) && (pkey != NULL) && (x509 != NULL)) { + if (!SSL_use_certificate(s, x509) || !SSL_use_PrivateKey(s, pkey)) + i = 0; + } else if (i == 1) { + i = 0; + ERR_raise(ERR_LIB_SSL, SSL_R_BAD_DATA_RETURNED_BY_CALLBACK); + } + + X509_free(x509); + EVP_PKEY_free(pkey); + if (i && !ssl3_check_client_certificate_ntls(s)) + i = 0; + if (i == 0) { + s->s3.tmp.cert_req = 2; + if (!ssl3_digest_cached_records(s, 0)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + } + + if (s->post_handshake_auth == SSL_PHA_REQUESTED) + return WORK_FINISHED_STOP; + return WORK_FINISHED_CONTINUE; + } + + /* Shouldn't ever get here */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return WORK_ERROR; +} + +int tls_construct_client_certificate_ntls(SSL *s, WPACKET *pkt) +{ + if (!ssl3_output_cert_chain_ntls(s, pkt, + (s->s3.tmp.cert_req == 2) ? NULL : s->s3.tmp.sign_cert, + (s->s3.tmp.cert_req == 2) ? NULL : s->s3.tmp.enc_cert)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + return 1; +} + +int ssl3_check_cert_and_algorithm_ntls(SSL *s) +{ + const SSL_CERT_LOOKUP *clu; + size_t idx; + long alg_k, alg_a; + + alg_k = s->s3.tmp.new_cipher->algorithm_mkey; + alg_a = s->s3.tmp.new_cipher->algorithm_auth; + + /* we don't have a certificate */ + if (!(alg_a & SSL_aCERT)) + return 1; + + /* This is the passed certificate */ + clu = ssl_cert_lookup_by_pkey(X509_get0_pubkey(s->session->peer), &idx); + + /* Check certificate is recognised and suitable for cipher */ + if (clu == NULL || (alg_a & clu->amask) == 0) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_MISSING_SIGNING_CERT); + return 0; + } + + if (clu->amask & SSL_aECDSA) { + if (ssl_check_srvr_ecc_cert_and_alg(s->session->peer, s)) + return 1; + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_BAD_ECC_CERT); + return 0; + } + + if (alg_k & (SSL_kRSA | SSL_kRSAPSK) && idx != SSL_PKEY_RSA) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_MISSING_RSA_ENCRYPTING_CERT); + return 0; + } + + if ((alg_k & SSL_kDHE) && (s->s3.peer_tmp == NULL)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +#ifndef OPENSSL_NO_NEXTPROTONEG +int tls_construct_next_proto_ntls(SSL *s, WPACKET *pkt) +{ + size_t len, padding_len; + unsigned char *padding = NULL; + + len = s->ext.npn_len; + padding_len = 32 - ((len + 2) % 32); + + if (!WPACKET_sub_memcpy_u8(pkt, s->ext.npn, len) + || !WPACKET_sub_allocate_bytes_u8(pkt, padding_len, &padding)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + memset(padding, 0, padding_len); + + return 1; +} +#endif + +MSG_PROCESS_RETURN tls_process_hello_req_ntls(SSL *s, PACKET *pkt) +{ + if (PACKET_remaining(pkt) > 0) { + /* should contain no data */ + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return MSG_PROCESS_ERROR; + } + + if ((s->options & SSL_OP_NO_RENEGOTIATION)) { + ssl3_send_alert(s, SSL3_AL_WARNING, SSL_AD_NO_RENEGOTIATION); + return MSG_PROCESS_FINISHED_READING; + } + + /* + * This is a historical discrepancy (not in the RFC) maintained for + * compatibility reasons. If a TLS client receives a HelloRequest it will + * attempt an abbreviated handshake. However if a DTLS client receives a + * HelloRequest it will do a full handshake. Either behaviour is reasonable + * but doing one for TLS and another for DTLS is odd. + */ + SSL_renegotiate_abbreviated(s); + + return MSG_PROCESS_FINISHED_READING; +} + +int ssl_do_client_cert_cb_ntls(SSL *s, X509 **px509, EVP_PKEY **ppkey) +{ + int i = 0; +#ifndef OPENSSL_NO_ENGINE + if (s->ctx->client_cert_engine) { + i = tls_engine_load_ssl_client_cert(s, px509, ppkey); + if (i != 0) + return i; + } +#endif + if (s->ctx->client_cert_cb) + i = s->ctx->client_cert_cb(s, px509, ppkey); + return i; +} + +int ssl_cipher_list_to_bytes(SSL *s, STACK_OF(SSL_CIPHER) *sk, WPACKET *pkt) +{ + int i; + size_t totlen = 0, len, maxlen, maxverok = 0; + int empty_reneg_info_scsv = !s->renegotiate; + + /* Set disabled masks for this session */ + if (!ssl_set_client_disabled(s)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_NO_PROTOCOLS_AVAILABLE); + return 0; + } + + if (sk == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + +#ifdef OPENSSL_MAX_TLS1_2_CIPHER_LENGTH +# if OPENSSL_MAX_TLS1_2_CIPHER_LENGTH < 6 +# error Max cipher length too short +# endif + /* + * Some servers hang if client hello > 256 bytes as hack workaround + * chop number of supported ciphers to keep it well below this if we + * use TLS v1.2 + */ + if (TLS1_get_version(s) >= TLS1_2_VERSION) + maxlen = OPENSSL_MAX_TLS1_2_CIPHER_LENGTH & ~1; + else +#endif + /* Maximum length that can be stored in 2 bytes. Length must be even */ + maxlen = 0xfffe; + + if (empty_reneg_info_scsv) + maxlen -= 2; + if (s->mode & SSL_MODE_SEND_FALLBACK_SCSV) + maxlen -= 2; + + for (i = 0; i < sk_SSL_CIPHER_num(sk) && totlen < maxlen; i++) { + const SSL_CIPHER *c; + + c = sk_SSL_CIPHER_value(sk, i); + /* Skip disabled ciphers */ + if (ssl_cipher_disabled(s, c, SSL_SECOP_CIPHER_SUPPORTED, 0)) + continue; + + if (!s->method->put_cipher_by_char(c, pkt, &len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* Sanity check that the maximum version we offer has ciphers enabled */ + if (!maxverok) { + if (c->max_tls >= s->s3.tmp.max_ver + && c->min_tls <= s->s3.tmp.max_ver) + maxverok = 1; + + } + + totlen += len; + } + + if (totlen == 0 || !maxverok) { + const char *maxvertext = + !maxverok + ? "No ciphers enabled for max supported SSL/TLS version" + : NULL; + + SSLfatal_data_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_NO_CIPHERS_AVAILABLE, + maxvertext); + return 0; + } + + if (totlen != 0) { + if (empty_reneg_info_scsv) { + static SSL_CIPHER scsv = { + 0, NULL, NULL, SSL3_CK_SCSV, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (!s->method->put_cipher_by_char(&scsv, pkt, &len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + if (s->mode & SSL_MODE_SEND_FALLBACK_SCSV) { + static SSL_CIPHER scsv = { + 0, NULL, NULL, SSL3_CK_FALLBACK_SCSV, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (!s->method->put_cipher_by_char(&scsv, pkt, &len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + } + + return 1; +} + +int tls_construct_end_of_early_data_ntls(SSL *s, WPACKET *pkt) +{ + if (s->early_data_state != SSL_EARLY_DATA_WRITE_RETRY + && s->early_data_state != SSL_EARLY_DATA_FINISHED_WRITING) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return 0; + } + + s->early_data_state = SSL_EARLY_DATA_FINISHED_WRITING; + return 1; +} + diff --git a/openssl/src/ssl/statem_ntls/ntls_statem_lib.c b/openssl/src/ssl/statem_ntls/ntls_statem_lib.c new file mode 100644 index 000000000..b2ea565c2 --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_statem_lib.c @@ -0,0 +1,2122 @@ +/* + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include +#include +#include +#include "ntls_ssl_local.h" +#include "ntls_statem_local.h" +#include "internal/cryptlib.h" +#include +#include +#include +#include +#include +#include +#include "internal/sockets.h" + +static int ssl_add_cert_to_wpacket_ntls(SSL *s, WPACKET *pkt, X509 *x); +/* + * Map error codes to TLS/SSL alart types. + */ +typedef struct x509err2alert_st { + int x509err; + int alert; +} X509ERR2ALERT; + +/* Fixed value used in the ServerHello random field to identify an HRR */ +const unsigned char hrrrandom_ntls[] = { + 0xcf, 0x21, 0xad, 0x74, 0xe5, 0x9a, 0x61, 0x11, 0xbe, 0x1d, 0x8c, 0x02, + 0x1e, 0x65, 0xb8, 0x91, 0xc2, 0xa2, 0x11, 0x16, 0x7a, 0xbb, 0x8c, 0x5e, + 0x07, 0x9e, 0x09, 0xe2, 0xc8, 0xa8, 0x33, 0x9c +}; + +/* + * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or + * SSL3_RT_CHANGE_CIPHER_SPEC) + */ +int ssl3_do_write_ntls(SSL *s, int type) +{ + int ret; + size_t written = 0; + + ret = ssl3_write_bytes(s, type, &s->init_buf->data[s->init_off], + s->init_num, &written); + if (ret < 0) + return -1; + if (type == SSL3_RT_HANDSHAKE) + /* + * should not be done for 'Hello Request's, but in that case we'll + * ignore the result anyway + */ + if (!ssl3_finish_mac(s, + (unsigned char *)&s->init_buf->data[s->init_off], + written)) + return -1; + if (written == s->init_num) { + if (s->msg_callback) + s->msg_callback(1, s->version, type, s->init_buf->data, + (size_t)(s->init_off + s->init_num), s, + s->msg_callback_arg); + return 1; + } + s->init_off += written; + s->init_num -= written; + return 0; +} + +int tls_close_construct_packet_ntls(SSL *s, WPACKET *pkt, int htype) +{ + size_t msglen; + + if ((htype != SSL3_MT_CHANGE_CIPHER_SPEC && !WPACKET_close(pkt)) + || !WPACKET_get_length(pkt, &msglen) + || msglen > INT_MAX) + return 0; + s->init_num = (int)msglen; + s->init_off = 0; + + return 1; +} + +int tls_setup_handshake_ntls(SSL *s) +{ + int ver_min, ver_max, ok; + + if (!ssl3_init_finished_mac(s)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + /* Reset any extension flags */ + memset(s->ext.extflags, 0, sizeof(s->ext.extflags)); + + if (ssl_get_min_max_version(s, &ver_min, &ver_max, NULL) != 0) { + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, SSL_R_NO_PROTOCOLS_AVAILABLE); + return 0; + } + ok = 0; + if (s->server) { + STACK_OF(SSL_CIPHER) *ciphers = SSL_get_ciphers(s); + int i; + + /* + * Sanity check that the maximum version we accept has ciphers + * enabled. For clients we do this check during construction of the + * ClientHello. + */ + for (i = 0; i < sk_SSL_CIPHER_num(ciphers); i++) { + const SSL_CIPHER *c = sk_SSL_CIPHER_value(ciphers, i); + + if (ver_max >= c->min_tls && ver_max <= c->max_tls) { + ok = 1; + } + if (ok) + break; + } + if (!ok) { + SSLfatal_data_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_NO_CIPHERS_AVAILABLE, + "No ciphers enabled for max supported " + "SSL/TLS version"); + return 0; + } + if (SSL_IS_FIRST_HANDSHAKE(s)) { + /* N.B. s->session_ctx == s->ctx here */ + ssl_tsan_counter(s->session_ctx, &s->session_ctx->stats.sess_accept); + } else { + /* N.B. s->ctx may not equal s->session_ctx */ + ssl_tsan_counter(s->ctx, &s->ctx->stats.sess_accept_renegotiate); + + s->s3.tmp.cert_request = 0; + } + } else { + if (SSL_IS_FIRST_HANDSHAKE(s)) + ssl_tsan_counter(s->session_ctx, &s->session_ctx->stats.sess_connect); + else + ssl_tsan_counter(s->session_ctx, + &s->session_ctx->stats.sess_connect_renegotiate); + + /* mark client_random uninitialized */ + memset(s->s3.client_random, 0, sizeof(s->s3.client_random)); + s->hit = 0; + + s->s3.tmp.cert_req = 0; + + } + + return 1; +} + +/* + * Size of the to-be-signed TLS13 data, without the hash size itself: + * 64 bytes of value 32, 33 context bytes, 1 byte separator + */ +#define TLS13_TBS_START_SIZE 64 +#define TLS13_TBS_PREAMBLE_SIZE (TLS13_TBS_START_SIZE + 33 + 1) + +static int get_cert_verify_tbs_data_ntls(SSL *s, void **hdata, size_t *hdatalen) +{ + size_t retlen; + long retlen_l; + + retlen = retlen_l = BIO_get_mem_data(s->s3.handshake_buffer, hdata); + if (retlen_l <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + *hdatalen = retlen; + + return 1; +} + +int tls_construct_cert_verify_ntls(SSL *s, WPACKET *pkt) +{ + EVP_PKEY *pkey = NULL; + const EVP_MD *md = NULL; + EVP_MD_CTX *mctx = NULL; + EVP_MD_CTX *mctx2 = NULL; + EVP_PKEY_CTX *pctx = NULL; + size_t hdatalen = 0, siglen = 0; + void *hdata; + unsigned char *sig = NULL; + unsigned char out[EVP_MAX_MD_SIZE]; + size_t outlen = 0; + const SIGALG_LOOKUP *lu = s->s3.tmp.sigalg; + + if (lu == NULL || s->s3.tmp.sign_cert == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + pkey = s->s3.tmp.sign_cert->privatekey; + + if (pkey == NULL || !tls1_lookup_md(s->ctx, lu, &md)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + mctx = EVP_MD_CTX_new(); + mctx2 = EVP_MD_CTX_new(); + if (mctx == NULL || mctx2 == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + /* Get the data to be signed */ + if (!get_cert_verify_tbs_data_ntls(s, &hdata, &hdatalen)) { + /* SSLfatal() already called */ + goto err; + } + + if (!EVP_DigestInit_ex(mctx2, md, NULL) + || !EVP_DigestUpdate(mctx2, hdata, hdatalen) + || !EVP_DigestFinal(mctx2, out, (unsigned int *)&outlen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + if (EVP_DigestSignInit_ex(mctx, &pctx, + md == NULL ? NULL : EVP_MD_get0_name(md), + s->ctx->libctx, s->ctx->propq, pkey, + NULL) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) { + if (EVP_PKEY_CTX_set1_id(pctx, SM2_DEFAULT_ID, + SM2_DEFAULT_ID_LEN) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + } + + if (EVP_DigestSign(mctx, NULL, &siglen, out, outlen) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + sig = OPENSSL_malloc(siglen); + if (sig == NULL + || EVP_DigestSign(mctx, sig, &siglen, out, outlen) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + if (!WPACKET_sub_memcpy_u16(pkt, sig, siglen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* Digest cached records and discard handshake buffer */ + if (!ssl3_digest_cached_records(s, 0)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + OPENSSL_free(sig); + EVP_MD_CTX_free(mctx); + EVP_MD_CTX_free(mctx2); + return 1; +err: + OPENSSL_free(sig); + EVP_MD_CTX_free(mctx); + EVP_MD_CTX_free(mctx2); + return 0; +} + +MSG_PROCESS_RETURN tls_process_cert_verify_ntls(SSL *s, PACKET *pkt) +{ + EVP_PKEY *pkey = NULL; + const unsigned char *data; + MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR; + int j; + unsigned int len; + X509 *peer; + const EVP_MD *md = NULL; + size_t hdatalen = 0; + void *hdata; + EVP_MD_CTX *mctx = EVP_MD_CTX_new(); + EVP_MD_CTX *mctx2 = EVP_MD_CTX_new(); + EVP_PKEY_CTX *pctx = NULL; + unsigned char out[EVP_MAX_MD_SIZE]; + size_t outlen = 0; + + if (mctx == NULL || mctx2 == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + /* For NTLS server, s->session->peer stores the client signing certificate */ + peer = s->session->peer; + pkey = X509_get0_pubkey(peer); + if (pkey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (ssl_cert_lookup_by_pkey(pkey, NULL) == NULL) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE); + goto err; + } + + if (!tls1_set_peer_legacy_sigalg(s, pkey)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (!tls1_lookup_md(s->ctx, s->s3.tmp.peer_sigalg, &md)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (!PACKET_get_net_2(pkt, &len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + if (!PACKET_get_bytes(pkt, &data, len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + if (!get_cert_verify_tbs_data_ntls(s, &hdata, &hdatalen)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + OSSL_TRACE1(TLS, "Using client verify alg %s\n", + md == NULL ? "n/a" : EVP_MD_get0_name(md)); + +#ifndef OPENSSL_NO_SM2 + if (EVP_PKEY_is_sm2(pkey)) { + if (!EVP_PKEY_set_alias_type(pkey, EVP_PKEY_SM2)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (pkey != NULL) { + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, pkey, s->ctx->propq); + if (pctx == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (EVP_PKEY_CTX_set1_id(pctx, HANDSHAKE_SM2_ID, + HANDSHAKE_SM2_ID_LEN) != 1) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + EVP_MD_CTX_set_pkey_ctx(mctx, pctx); + } + + if (!EVP_PKEY_set_alias_type(pkey, EVP_PKEY_EC)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } +#endif + + if (EVP_DigestVerifyInit_ex(mctx, &pctx, + md == NULL ? NULL : EVP_MD_get0_name(md), + s->ctx->libctx, s->ctx->propq, pkey, + NULL) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + + if (!EVP_DigestInit_ex(mctx2, md, NULL) + || !EVP_DigestUpdate(mctx2, hdata, hdatalen) + || !EVP_DigestFinal(mctx2, out, (unsigned int *)&outlen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) { + if (EVP_PKEY_CTX_set1_id(pctx, SM2_DEFAULT_ID, SM2_DEFAULT_ID_LEN) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + } + + + j = EVP_DigestVerify(mctx, data, len, out, outlen); + if (j <= 0) { + SSLfatal_ntls(s, SSL_AD_DECRYPT_ERROR, SSL_R_BAD_SIGNATURE); + goto err; + } + + ret = MSG_PROCESS_CONTINUE_READING; + + err: + BIO_free(s->s3.handshake_buffer); + s->s3.handshake_buffer = NULL; + EVP_MD_CTX_free(mctx2); + EVP_MD_CTX_free(mctx); + +#ifndef OPENSSL_NO_SM2 + /*other sig call EVP_PKEY_CTX_free there may cause segfault */ + if (pkey != NULL && EVP_PKEY_is_sm2(pkey)) + EVP_PKEY_CTX_free(pctx); +#endif + return ret; +} + +int tls_construct_finished_ntls(SSL *s, WPACKET *pkt) +{ + size_t finish_md_len; + const char *sender; + size_t slen; + + /* This is a real handshake so make sure we clean it up at the end */ + if (!s->server && s->post_handshake_auth != SSL_PHA_REQUESTED) + s->statem.cleanuphand = 1; + + if (s->server) { + sender = s->method->ssl3_enc->server_finished_label; + slen = s->method->ssl3_enc->server_finished_label_len; + } else { + sender = s->method->ssl3_enc->client_finished_label; + slen = s->method->ssl3_enc->client_finished_label_len; + } + + finish_md_len = s->method->ssl3_enc->final_finish_mac(s, + sender, slen, + s->s3.tmp.finish_md); + if (finish_md_len == 0) { + /* SSLfatal_ntls() already called */ + return 0; + } + + s->s3.tmp.finish_md_len = finish_md_len; + + if (!WPACKET_memcpy(pkt, s->s3.tmp.finish_md, finish_md_len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* + * Log the master secret, if logging is enabled. We don't log it for + * TLSv1.3: there's a different key schedule for that. + */ + if (!ssl_log_secret(s, MASTER_SECRET_LABEL, + s->session->master_key, + s->session->master_key_length)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + /* + * Copy the finished so we can use it for renegotiation checks + */ + if (!ossl_assert(finish_md_len <= EVP_MAX_MD_SIZE)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if (!s->server) { + memcpy(s->s3.previous_client_finished, s->s3.tmp.finish_md, + finish_md_len); + s->s3.previous_client_finished_len = finish_md_len; + } else { + memcpy(s->s3.previous_server_finished, s->s3.tmp.finish_md, + finish_md_len); + s->s3.previous_server_finished_len = finish_md_len; + } + + return 1; +} + +/* + * ssl3_take_mac_ntls calculates the Finished MAC for the handshakes messages seen + * to far. + */ +int ssl3_take_mac_ntls(SSL *s) +{ + const char *sender; + size_t slen; + + if (!s->server) { + sender = s->method->ssl3_enc->server_finished_label; + slen = s->method->ssl3_enc->server_finished_label_len; + } else { + sender = s->method->ssl3_enc->client_finished_label; + slen = s->method->ssl3_enc->client_finished_label_len; + } + + s->s3.tmp.peer_finish_md_len = + s->method->ssl3_enc->final_finish_mac(s, sender, slen, + s->s3.tmp.peer_finish_md); + + if (s->s3.tmp.peer_finish_md_len == 0) { + /* SSLfatal_ntls() already called */ + return 0; + } + + return 1; +} + +MSG_PROCESS_RETURN tls_process_change_cipher_spec_ntls(SSL *s, PACKET *pkt) +{ + size_t remain; + + remain = PACKET_remaining(pkt); + /* + * 'Change Cipher Spec' is just a single byte, which should already have + * been consumed by ssl_get_message() so there should be no bytes left, + */ + if (remain != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_CHANGE_CIPHER_SPEC); + return MSG_PROCESS_ERROR; + } + + /* Check we have a cipher to change to */ + if (s->s3.tmp.new_cipher == NULL) { + SSLfatal_ntls(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_CCS_RECEIVED_EARLY); + return MSG_PROCESS_ERROR; + } + + s->s3.change_cipher_spec = 1; + if (!ssl3_do_change_cipher_spec(s)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return MSG_PROCESS_ERROR; + } + + return MSG_PROCESS_CONTINUE_READING; +} + +MSG_PROCESS_RETURN tls_process_finished_ntls(SSL *s, PACKET *pkt) +{ + size_t md_len; + + + /* This is a real handshake so make sure we clean it up at the end */ + if (s->server) { + /* + * To get this far we must have read encrypted data from the client. We + * no longer tolerate unencrypted alerts. This value is ignored if less + * than TLSv1.3 + */ + s->statem.enc_read_state = ENC_READ_STATE_VALID; + if (s->post_handshake_auth != SSL_PHA_REQUESTED) + s->statem.cleanuphand = 1; + } + + /* If this occurs, we have missed a message */ + if (!s->s3.change_cipher_spec) { + SSLfatal_ntls(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_GOT_A_FIN_BEFORE_A_CCS); + return MSG_PROCESS_ERROR; + } + s->s3.change_cipher_spec = 0; + + md_len = s->s3.tmp.peer_finish_md_len; + + if (md_len != PACKET_remaining(pkt)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_BAD_DIGEST_LENGTH); + return MSG_PROCESS_ERROR; + } + + if (CRYPTO_memcmp(PACKET_data(pkt), s->s3.tmp.peer_finish_md, + md_len) != 0) { + SSLfatal_ntls(s, SSL_AD_DECRYPT_ERROR, SSL_R_DIGEST_CHECK_FAILED); + return MSG_PROCESS_ERROR; + } + + /* + * Copy the finished so we can use it for renegotiation checks + */ + if (!ossl_assert(md_len <= EVP_MAX_MD_SIZE)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return MSG_PROCESS_ERROR; + } + if (s->server) { + memcpy(s->s3.previous_client_finished, s->s3.tmp.peer_finish_md, + md_len); + s->s3.previous_client_finished_len = md_len; + } else { + memcpy(s->s3.previous_server_finished, s->s3.tmp.peer_finish_md, + md_len); + s->s3.previous_server_finished_len = md_len; + } + + return MSG_PROCESS_FINISHED_READING; +} + +int tls_construct_change_cipher_spec_ntls(SSL *s, WPACKET *pkt) +{ + if (!WPACKET_put_bytes_u8(pkt, SSL3_MT_CCS)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +/* Add a certificate to the WPACKET */ +static int ssl_add_cert_to_wpacket_ntls(SSL *s, WPACKET *pkt, X509 *x) +{ + int len; + unsigned char *outbytes; + + len = i2d_X509(x, NULL); + if (len < 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_BUF_LIB); + return 0; + } + if (!WPACKET_sub_allocate_bytes_u24(pkt, len, &outbytes) + || i2d_X509(x, &outbytes) != len) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +/* Add certificate chain to provided WPACKET */ +static int ssl_add_cert_chain_ntls(SSL *s, WPACKET *pkt, + CERT_PKEY *a_cpk, CERT_PKEY *k_cpk) +{ + int i, chain_count; + X509 *x; + STACK_OF(X509) *extra_certs; + STACK_OF(X509) *chain = NULL; + X509_STORE *chain_store; + + if (a_cpk == NULL || a_cpk->x509 == NULL + || k_cpk == NULL || k_cpk->x509 == NULL) + return 1; + + if (a_cpk->chain != NULL) + extra_certs = a_cpk->chain; + else if (k_cpk->chain != NULL) + extra_certs = k_cpk->chain; + else + extra_certs = s->ctx->extra_certs; + + if ((s->mode & SSL_MODE_NO_AUTO_CHAIN) || extra_certs) + chain_store = NULL; + else if (s->cert->chain_store) + chain_store = s->cert->chain_store; + else + chain_store = s->ctx->cert_store; + + if (chain_store != NULL) { + X509_STORE_CTX *xs_ctx = X509_STORE_CTX_new_ex(s->ctx->libctx, + s->ctx->propq); + + if (xs_ctx == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + + if (!X509_STORE_CTX_init(xs_ctx, chain_store, + a_cpk->x509, NULL)) { + X509_STORE_CTX_free(xs_ctx); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_X509_LIB); + return 0; + } + /* + * It is valid for the chain not to be complete (because normally we + * don't include the root cert in the chain). Therefore we deliberately + * ignore the error return from this call. We're not actually verifying + * the cert - we're just building as much of the chain as we can + */ + (void)X509_verify_cert(xs_ctx); + /* Don't leave errors in the queue */ + ERR_clear_error(); + chain = X509_STORE_CTX_get0_chain(xs_ctx); + i = ssl_security_cert_chain(s, chain, NULL, 0); + if (i != 1) { +#if 0 + /* Dummy error calls so mkerr generates them */ + ERR_raise(ERR_LIB_SSL, SSL_R_EE_KEY_TOO_SMALL); + ERR_raise(ERR_LIB_SSL, SSL_R_CA_KEY_TOO_SMALL); + ERR_raise(ERR_LIB_SSL, SSL_R_CA_MD_TOO_WEAK); +#endif + X509_STORE_CTX_free(xs_ctx); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, i); + return 0; + } + + /* add sign certificate */ + if (!ssl_add_cert_to_wpacket_ntls(s, pkt, a_cpk->x509)) { + /* SSLfatal_ntls() already called */ + X509_STORE_CTX_free(xs_ctx); + return 0; + } + + /* add encryption certificate */ + if (!ssl_add_cert_to_wpacket_ntls(s, pkt, k_cpk->x509)) { + /* SSLfatal_ntls() already called */ + X509_STORE_CTX_free(xs_ctx); + return 0; + } + + chain_count = sk_X509_num(chain); + for (i = 1; i < chain_count; i++) { + x = sk_X509_value(chain, i); + if (!ssl_add_cert_to_wpacket_ntls(s, pkt, x)) { + /* SSLfatal_ntls() already called */ + X509_STORE_CTX_free(xs_ctx); + return 0; + } + } + X509_STORE_CTX_free(xs_ctx); + } else { + i = ssl_security_cert_chain(s, extra_certs, a_cpk->x509, 0); + if (i != 1) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, i); + return 0; + } + + /* add sign certificate */ + if (!ssl_add_cert_to_wpacket_ntls(s, pkt, a_cpk->x509)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + /* add encryption certificate */ + if (!ssl_add_cert_to_wpacket_ntls(s, pkt, k_cpk->x509)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + /* output the following chain */ + for (i = 0; i < sk_X509_num(extra_certs); i++) { + x = sk_X509_value(extra_certs, i); + if (!ssl_add_cert_to_wpacket_ntls(s, pkt, x)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + } + + return 1; +} + +unsigned long ssl3_output_cert_chain_ntls(SSL *s, WPACKET *pkt, + CERT_PKEY *a_cpk, + CERT_PKEY *k_cpk) +{ + if (!WPACKET_start_sub_packet_u24(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!ssl_add_cert_chain_ntls(s, pkt, a_cpk, k_cpk)) + return 0; + + if (!WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +/* + * Tidy up after the end of a handshake. In the case of SCTP this may result + * in NBIO events. If |clearbufs| is set then init_buf and the wbio buffer is + * freed up as well. + */ +WORK_STATE tls_finish_handshake_ntls(SSL *s, ossl_unused WORK_STATE wst, + int clearbufs, int stop) +{ + void (*cb) (const SSL *ssl, int type, int val) = NULL; + int cleanuphand = s->statem.cleanuphand; + + if (clearbufs) { + BUF_MEM_free(s->init_buf); + s->init_buf = NULL; + + if (!ssl_free_wbio_buffer(s)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return WORK_ERROR; + } + s->init_num = 0; + } + + /* + * Only set if there was a Finished message and this isn't after a TLSv1.3 + * post handshake exchange + */ + if (cleanuphand) { + /* skipped if we just sent a HelloRequest */ + s->renegotiate = 0; + s->new_session = 0; + s->statem.cleanuphand = 0; + s->ext.ticket_expected = 0; + + ssl3_cleanup_key_block(s); + + if (s->server) { + ssl_update_cache(s, SSL_SESS_CACHE_SERVER); + + /* N.B. s->ctx may not equal s->session_ctx */ + ssl_tsan_counter(s->ctx, &s->ctx->stats.sess_accept_good); + s->handshake_func = ossl_statem_accept_ntls; + } else { + ssl_update_cache(s, SSL_SESS_CACHE_CLIENT); + if (s->hit) + ssl_tsan_counter(s->session_ctx, + &s->session_ctx->stats.sess_hit); + + s->handshake_func = ossl_statem_connect_ntls; + ssl_tsan_counter(s->session_ctx, + &s->session_ctx->stats.sess_connect_good); + } + } + + if (s->info_callback != NULL) + cb = s->info_callback; + else if (s->ctx->info_callback != NULL) + cb = s->ctx->info_callback; + + /* The callback may expect us to not be in init at handshake done */ + ossl_statem_set_in_init_ntls(s, 0); + + if (cb != NULL) + cb(s, SSL_CB_HANDSHAKE_DONE, 1); + + if (!stop) { + /* If we've got more work to do we go back into init */ + ossl_statem_set_in_init_ntls(s, 1); + return WORK_FINISHED_CONTINUE; + } + + return WORK_FINISHED_STOP; +} + +int tls_get_message_header_ntls(SSL *s, int *mt) +{ + /* s->init_num < SSL3_HM_HEADER_LENGTH */ + int skip_message, i, recvd_type; + unsigned char *p; + size_t l, readbytes; + + p = (unsigned char *)s->init_buf->data; + + do { + while (s->init_num < SSL3_HM_HEADER_LENGTH) { + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, &recvd_type, + &p[s->init_num], + SSL3_HM_HEADER_LENGTH - s->init_num, + 0, &readbytes); + if (i <= 0) { + s->rwstate = SSL_READING; + return 0; + } + if (recvd_type == SSL3_RT_CHANGE_CIPHER_SPEC) { + /* + * A ChangeCipherSpec must be a single byte and may not occur + * in the middle of a handshake message. + */ + if (s->init_num != 0 || readbytes != 1 || p[0] != SSL3_MT_CCS) { + SSLfatal_ntls(s, SSL_AD_UNEXPECTED_MESSAGE, + SSL_R_BAD_CHANGE_CIPHER_SPEC); + return 0; + } + if (s->statem.hand_state == TLS_ST_BEFORE + && (s->s3.flags & TLS1_FLAGS_STATELESS) != 0) { + /* + * We are stateless and we received a CCS. Probably this is + * from a client between the first and second ClientHellos. + * We should ignore this, but return an error because we do + * not return success until we see the second ClientHello + * with a valid cookie. + */ + return 0; + } + s->s3.tmp.message_type = *mt = SSL3_MT_CHANGE_CIPHER_SPEC; + s->init_num = readbytes - 1; + s->init_msg = s->init_buf->data; + s->s3.tmp.message_size = readbytes; + return 1; + } else if (recvd_type != SSL3_RT_HANDSHAKE) { + SSLfatal_ntls(s, SSL_AD_UNEXPECTED_MESSAGE, + SSL_R_CCS_RECEIVED_EARLY); + return 0; + } + s->init_num += readbytes; + } + + skip_message = 0; + if (!s->server) + if (s->statem.hand_state != TLS_ST_OK + && p[0] == SSL3_MT_HELLO_REQUEST) + /* + * The server may always send 'Hello Request' messages -- + * we are doing a handshake anyway now, so ignore them if + * their format is correct. Does not count for 'Finished' + * MAC. + */ + if (p[1] == 0 && p[2] == 0 && p[3] == 0) { + s->init_num = 0; + skip_message = 1; + + if (s->msg_callback) + s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, + p, SSL3_HM_HEADER_LENGTH, s, + s->msg_callback_arg); + } + } while (skip_message); + /* s->init_num == SSL3_HM_HEADER_LENGTH */ + + *mt = *p; + s->s3.tmp.message_type = *(p++); + + if (RECORD_LAYER_is_sslv2_record(&s->rlayer)) { + /* + * Only happens with SSLv3+ in an SSLv2 backward compatible + * ClientHello + * + * Total message size is the remaining record bytes to read + * plus the SSL3_HM_HEADER_LENGTH bytes that we already read + */ + l = RECORD_LAYER_get_rrec_length(&s->rlayer) + + SSL3_HM_HEADER_LENGTH; + s->s3.tmp.message_size = l; + + s->init_msg = s->init_buf->data; + s->init_num = SSL3_HM_HEADER_LENGTH; + } else { + n2l3(p, l); + /* BUF_MEM_grow takes an 'int' parameter */ + if (l > (INT_MAX - SSL3_HM_HEADER_LENGTH)) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_EXCESSIVE_MESSAGE_SIZE); + return 0; + } + s->s3.tmp.message_size = l; + + s->init_msg = s->init_buf->data + SSL3_HM_HEADER_LENGTH; + s->init_num = 0; + } + + return 1; +} + +int tls_get_message_body_ntls(SSL *s, size_t *len) +{ + size_t n, readbytes; + unsigned char *p; + int i; + + if (s->s3.tmp.message_type == SSL3_MT_CHANGE_CIPHER_SPEC) { + /* We've already read everything in */ + *len = (unsigned long)s->init_num; + return 1; + } + + p = s->init_msg; + n = s->s3.tmp.message_size - s->init_num; + while (n > 0) { + i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL, + &p[s->init_num], n, 0, &readbytes); + if (i <= 0) { + s->rwstate = SSL_READING; + *len = 0; + return 0; + } + s->init_num += readbytes; + n -= readbytes; + } + + /* + * If receiving Finished, record MAC of prior handshake messages for + * Finished verification. + */ + if (*(s->init_buf->data) == SSL3_MT_FINISHED && !ssl3_take_mac_ntls(s)) { + /* SSLfatal_ntls() already called */ + *len = 0; + return 0; + } + + /* Feed this message into MAC computation. */ + if (RECORD_LAYER_is_sslv2_record(&s->rlayer)) { + if (!ssl3_finish_mac(s, (unsigned char *)s->init_buf->data, + s->init_num)) { + /* SSLfatal_ntls() already called */ + *len = 0; + return 0; + } + if (s->msg_callback) + s->msg_callback(0, SSL2_VERSION, 0, s->init_buf->data, + (size_t)s->init_num, s, s->msg_callback_arg); + } else { + /* + * We defer feeding in the HRR until later. We'll do it as part of + * processing the message + */ +#define SERVER_HELLO_RANDOM_OFFSET (SSL3_HM_HEADER_LENGTH + 2) + if (s->s3.tmp.message_type != SSL3_MT_SERVER_HELLO + || s->init_num < SERVER_HELLO_RANDOM_OFFSET + SSL3_RANDOM_SIZE + || memcmp(hrrrandom_ntls, + s->init_buf->data + SERVER_HELLO_RANDOM_OFFSET, + SSL3_RANDOM_SIZE) != 0) { + if (!ssl3_finish_mac(s, (unsigned char *)s->init_buf->data, + s->init_num + SSL3_HM_HEADER_LENGTH)) { + /* SSLfatal_ntls() already called */ + *len = 0; + return 0; + } + } + if (s->msg_callback) + s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, s->init_buf->data, + (size_t)s->init_num + SSL3_HM_HEADER_LENGTH, s, + s->msg_callback_arg); + } + + *len = s->init_num; + return 1; +} + +static const X509ERR2ALERT x509table[] = { + {X509_V_ERR_APPLICATION_VERIFICATION, SSL_AD_HANDSHAKE_FAILURE}, + {X509_V_ERR_CA_KEY_TOO_SMALL, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_EC_KEY_EXPLICIT_PARAMS, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_CA_MD_TOO_WEAK, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_CERT_CHAIN_TOO_LONG, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_CERT_HAS_EXPIRED, SSL_AD_CERTIFICATE_EXPIRED}, + {X509_V_ERR_CERT_NOT_YET_VALID, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_CERT_REJECTED, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_CERT_REVOKED, SSL_AD_CERTIFICATE_REVOKED}, + {X509_V_ERR_CERT_SIGNATURE_FAILURE, SSL_AD_DECRYPT_ERROR}, + {X509_V_ERR_CERT_UNTRUSTED, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_CRL_HAS_EXPIRED, SSL_AD_CERTIFICATE_EXPIRED}, + {X509_V_ERR_CRL_NOT_YET_VALID, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_CRL_SIGNATURE_FAILURE, SSL_AD_DECRYPT_ERROR}, + {X509_V_ERR_DANE_NO_MATCH, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_EE_KEY_TOO_SMALL, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_EMAIL_MISMATCH, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_HOSTNAME_MISMATCH, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_INVALID_CA, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_INVALID_CALL, SSL_AD_INTERNAL_ERROR}, + {X509_V_ERR_INVALID_PURPOSE, SSL_AD_UNSUPPORTED_CERTIFICATE}, + {X509_V_ERR_IP_ADDRESS_MISMATCH, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_OUT_OF_MEM, SSL_AD_INTERNAL_ERROR}, + {X509_V_ERR_PATH_LENGTH_EXCEEDED, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_STORE_LOOKUP, SSL_AD_INTERNAL_ERROR}, + {X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE, SSL_AD_BAD_CERTIFICATE}, + {X509_V_ERR_UNABLE_TO_GET_CRL, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_UNABLE_TO_GET_CRL_ISSUER, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE, SSL_AD_UNKNOWN_CA}, + {X509_V_ERR_UNSPECIFIED, SSL_AD_INTERNAL_ERROR}, + + /* Last entry; return this if we don't find the value above. */ + {X509_V_OK, SSL_AD_CERTIFICATE_UNKNOWN} +}; + +int ssl_x509err2alert_ntls(int x509err) +{ + const X509ERR2ALERT *tp; + + for (tp = x509table; tp->x509err != X509_V_OK; ++tp) + if (tp->x509err == x509err) + break; + return tp->alert; +} + +int ssl_allow_compression_ntls(SSL *s) +{ + if (s->options & SSL_OP_NO_COMPRESSION) + return 0; + return ssl_security(s, SSL_SECOP_COMPRESSION, 0, 0, NULL); +} + +static int version_cmp(const SSL *s, int a, int b) +{ + if (a == b) + return 0; + + return a < b ? -1 : 1; +} + +typedef struct { + int version; + const SSL_METHOD *(*cmeth) (void); + const SSL_METHOD *(*smeth) (void); +} version_info; + +#if TLS_MAX_VERSION_INTERNAL != TLS1_3_VERSION +# error Code needs update for TLS_method() support beyond TLS1_3_VERSION. +#endif + +/* Must be in order high to low */ +static const version_info tls_version_table[] = { +#ifndef OPENSSL_NO_TLS1_3 + {TLS1_3_VERSION, tlsv1_3_client_method, tlsv1_3_server_method}, +#else + {TLS1_3_VERSION, NULL, NULL}, +#endif +#ifndef OPENSSL_NO_TLS1_2 + {TLS1_2_VERSION, tlsv1_2_client_method, tlsv1_2_server_method}, +#else + {TLS1_2_VERSION, NULL, NULL}, +#endif +#ifndef OPENSSL_NO_TLS1_1 + {TLS1_1_VERSION, tlsv1_1_client_method, tlsv1_1_server_method}, +#else + {TLS1_1_VERSION, NULL, NULL}, +#endif +#ifndef OPENSSL_NO_TLS1 + {TLS1_VERSION, tlsv1_client_method, tlsv1_server_method}, +#else + {TLS1_VERSION, NULL, NULL}, +#endif +#ifndef OPENSSL_NO_NTLS + {NTLS_VERSION, ntls_client_method, ntls_server_method}, +#else + {NTLS_VERSION, NULL, NULL}, +#endif +#ifndef OPENSSL_NO_SSL3 + {SSL3_VERSION, sslv3_client_method, sslv3_server_method}, +#else + {SSL3_VERSION, NULL, NULL}, +#endif + {0, NULL, NULL}, +}; + + + + + +/* + * ssl_method_error - Check whether an SSL_METHOD is enabled. + * + * @s: The SSL handle for the candidate method + * @method: the intended method. + * + * Returns 0 on success, or an SSL error reason on failure. + */ +static int ssl_method_error(const SSL *s, const SSL_METHOD *method) +{ + int version = method->version; + + if ((s->min_proto_version != 0 && + version_cmp(s, version, s->min_proto_version) < 0) || + ssl_security(s, SSL_SECOP_VERSION, 0, version, NULL) == 0) + return SSL_R_VERSION_TOO_LOW; + + if (s->max_proto_version != 0 && + version_cmp(s, version, s->max_proto_version) > 0) + return SSL_R_VERSION_TOO_HIGH; + + if ((s->options & method->mask) != 0) + return SSL_R_UNSUPPORTED_PROTOCOL; + if ((method->flags & SSL_METHOD_NO_SUITEB) != 0 && tls1_suiteb(s)) + return SSL_R_AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE; + + return 0; +} + +/* + * ssl_version_supported_ntls - Check that the specified `version` is supported by + * `SSL *` instance + * + * @s: The SSL handle for the candidate method + * @version: Protocol version to test against + * + * Returns 1 when supported, otherwise 0 + */ +int ssl_version_supported_ntls(const SSL *s, int version, const SSL_METHOD **meth) +{ + const version_info *vent; + const version_info *table; + + switch (s->method->version) { + default: + /* Version should match method version for non-ANY method */ + return version_cmp(s, version, s->version) == 0; + case TLS_ANY_VERSION: + table = tls_version_table; + break; + } + + for (vent = table; + vent->version != 0 && version_cmp(s, version, vent->version) <= 0; + ++vent) { + if (vent->cmeth != NULL + && version_cmp(s, version, vent->version) == 0 + && ssl_method_error(s, vent->cmeth()) == 0 + && (!s->server + || version != TLS1_3_VERSION)) { + if (meth != NULL) + *meth = vent->cmeth(); + return 1; + } + } + return 0; +} + +/* + * ssl_check_version_downgrade_ntls - In response to RFC7507 SCSV version + * fallback indication from a client check whether we're using the highest + * supported protocol version. + * + * @s server SSL handle. + * + * Returns 1 when using the highest enabled version, 0 otherwise. + */ +int ssl_check_version_downgrade_ntls(SSL *s) +{ + const version_info *vent; + const version_info *table; + + /* + * Check that the current protocol is the highest enabled version + * (according to s->ctx->method, as version negotiation may have changed + * s->method). + */ + if (s->version == s->ctx->method->version) + return 1; + + /* + * Apparently we're using a version-flexible SSL_METHOD (not at its + * highest protocol version). + */ + if (s->ctx->method->version == TLS_method()->version) + table = tls_version_table; + else { + /* Unexpected state; fail closed. */ + return 0; + } + + for (vent = table; vent->version != 0; ++vent) { + if (vent->smeth != NULL && ssl_method_error(s, vent->smeth()) == 0) + return s->version == vent->version; + } + return 0; +} + +/* + * ssl_set_version_bound_ntls - set an upper or lower bound on the supported (D)TLS + * protocols, provided the initial (D)TLS method is version-flexible. This + * function sanity-checks the proposed value and makes sure the method is + * version-flexible, then sets the limit if all is well. + * + * @method_version: The version of the current SSL_METHOD. + * @version: the intended limit. + * @bound: pointer to limit to be updated. + * + * Returns 1 on success, 0 on failure. + */ +int ssl_set_version_bound_ntls(int method_version, int version, int *bound) +{ + int valid_tls; + + if (version == 0) { + *bound = version; + return 1; + } + + valid_tls = version >= NTLS_VERSION && version <= TLS_MAX_VERSION_INTERNAL; + + if (!valid_tls) + return 0; + + /*- + * Restrict TLS methods to TLS protocol versions. + * + * Note that for both lower-bounds we use explicit versions, not + * (D)TLS_MIN_VERSION. This is because we don't want to break user + * configurations. If the MIN (supported) version ever rises, the user's + * "floor" remains valid even if no longer available. We don't expect the + * MAX ceiling to ever get lower, so making that variable makes sense. + */ + switch (method_version) { + default: + break; + + case TLS_ANY_VERSION: + if (valid_tls) + *bound = version; + break; + } + return 1; +} + +static void check_for_downgrade(SSL *s, int vers, DOWNGRADE *dgrd) +{ + if (vers == TLS1_2_VERSION + && ssl_version_supported_ntls(s, TLS1_3_VERSION, NULL)) { + *dgrd = DOWNGRADE_TO_1_2; + } else if (vers < TLS1_2_VERSION + /* + * We need to ensure that a server that disables TLSv1.2 + * (creating a hole between TLSv1.3 and TLSv1.1) can still + * complete handshakes with clients that support TLSv1.2 and + * below. Therefore we do not enable the sentinel if TLSv1.3 is + * enabled and TLSv1.2 is not. + */ + && ssl_version_supported_ntls(s, TLS1_2_VERSION, NULL)) { + *dgrd = DOWNGRADE_TO_1_1; + } else { + *dgrd = DOWNGRADE_NONE; + } +} + +/* + * ssl_choose_server_version_ntls - Choose server (D)TLS version. Called when the + * client HELLO is received to select the final server protocol version and + * the version specific method. + * + * @s: server SSL handle. + * + * Returns 0 on success or an SSL error reason number on failure. + */ +int ssl_choose_server_version_ntls(SSL *s, CLIENTHELLO_MSG *hello, DOWNGRADE *dgrd) +{ + /*- + * With version-flexible methods we have an initial state with: + * + * s->method->version == (D)TLS_ANY_VERSION, + * s->version == (D)TLS_MAX_VERSION_INTERNAL. + * + * So we detect version-flexible methods via the method version, not the + * handle version. + */ + int server_version = s->method->version; + int client_version = hello->legacy_version; + const version_info *vent; + const version_info *table; + int disabled = 0; + + s->client_version = client_version; + + switch (server_version) { + default: + if (version_cmp(s, client_version, s->version) < 0) + return SSL_R_WRONG_SSL_VERSION; + *dgrd = DOWNGRADE_NONE; + /* + * If this SSL handle is not from a version flexible method we don't + * (and never did) check min/max FIPS or Suite B constraints. Hope + * that's OK. It is up to the caller to not choose fixed protocol + * versions they don't want. If not, then easy to fix, just return + * ssl_method_error(s, s->method) + */ + return 0; + /* + * Fall through if we are TLSv1.3 already (this means we must be after + * a HelloRetryRequest + */ + /* fall thru */ + case TLS_ANY_VERSION: + table = tls_version_table; + break; + } + + /* + * just use the version supplied in the ClientHello. + */ + for (vent = table; vent->version != 0; ++vent) { + const SSL_METHOD *method; + + if (vent->smeth == NULL || + version_cmp(s, client_version, vent->version) < 0) + continue; + method = vent->smeth(); + if (ssl_method_error(s, method) == 0) { + check_for_downgrade(s, vent->version, dgrd); + s->version = vent->version; + s->method = method; + return 0; + } + disabled = 1; + } + return disabled ? SSL_R_UNSUPPORTED_PROTOCOL : SSL_R_VERSION_TOO_LOW; +} + +/* + * ssl_choose_client_version_ntls - Choose client (D)TLS version. Called when the + * server HELLO is received to select the final client protocol version and + * the version specific method. + * + * @s: client SSL handle. + * @version: The proposed version from the server's HELLO. + * @extensions: The extensions received + * + * Returns 1 on success or 0 on error. + */ +int ssl_choose_client_version_ntls(SSL *s, int version, RAW_EXTENSION *extensions) +{ + const version_info *vent; + const version_info *table; + int ret, ver_min, ver_max, real_max, origv; + + origv = s->version; + s->version = version; + + if (s->hello_retry_request != SSL_HRR_NONE + && s->version != TLS1_3_VERSION) { + s->version = origv; + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, SSL_R_WRONG_SSL_VERSION); + return 0; + } + + switch (s->method->version) { + default: + if (s->version != s->method->version) { + s->version = origv; + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, SSL_R_WRONG_SSL_VERSION); + return 0; + } + /* + * If this SSL handle is not from a version flexible method we don't + * (and never did) check min/max, FIPS or Suite B constraints. Hope + * that's OK. It is up to the caller to not choose fixed protocol + * versions they don't want. If not, then easy to fix, just return + * ssl_method_error(s, s->method) + */ + return 1; + case TLS_ANY_VERSION: + table = tls_version_table; + break; + } + + ret = ssl_get_min_max_version_ntls(s, &ver_min, &ver_max, &real_max); + if (ret != 0) { + s->version = origv; + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, ret); + return 0; + } + if (s->version < ver_min) { + s->version = origv; + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, SSL_R_UNSUPPORTED_PROTOCOL); + return 0; + } else if (s->version > ver_max) { + s->version = origv; + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, SSL_R_UNSUPPORTED_PROTOCOL); + return 0; + } + + if ((s->mode & SSL_MODE_SEND_FALLBACK_SCSV) == 0) + real_max = ver_max; + + /* Check for downgrades */ + if (s->version == TLS1_2_VERSION && real_max > s->version) { + if (memcmp(tls12downgrade, + s->s3.server_random + SSL3_RANDOM_SIZE + - sizeof(tls12downgrade), + sizeof(tls12downgrade)) == 0) { + s->version = origv; + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_INAPPROPRIATE_FALLBACK); + return 0; + } + } else if (s->version < TLS1_2_VERSION + && real_max > s->version) { + if (memcmp(tls11downgrade, + s->s3.server_random + SSL3_RANDOM_SIZE + - sizeof(tls11downgrade), + sizeof(tls11downgrade)) == 0) { + s->version = origv; + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_INAPPROPRIATE_FALLBACK); + return 0; + } + } + + for (vent = table; vent->version != 0; ++vent) { + if (vent->cmeth == NULL || s->version != vent->version) + continue; + + s->method = vent->cmeth(); + return 1; + } + + s->version = origv; + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, SSL_R_UNSUPPORTED_PROTOCOL); + return 0; +} + +/* + * ssl_get_min_max_version_ntls - get minimum and maximum protocol version + * @s: The SSL connection + * @min_version: The minimum supported version + * @max_version: The maximum supported version + * @real_max: The highest version below the lowest compile time version hole + * where that hole lies above at least one run-time enabled + * protocol. + * + * Work out what version we should be using for the initial ClientHello if the + * version is initially (D)TLS_ANY_VERSION. We apply any explicit SSL_OP_NO_xxx + * options, the MinProtocol and MaxProtocol configuration commands, any Suite B + * constraints and any floor imposed by the security level here, + * so we don't advertise the wrong protocol version to only reject the outcome later. + * + * Computing the right floor matters. If, e.g., TLS 1.0 and 1.2 are enabled, + * TLS 1.1 is disabled, but the security level, Suite-B and/or MinProtocol + * only allow TLS 1.2, we want to advertise TLS1.2, *not* TLS1. + * + * Returns 0 on success or an SSL error reason number on failure. On failure + * min_version and max_version will also be set to 0. + */ +int ssl_get_min_max_version_ntls(const SSL *s, int *min_version, int *max_version, + int *real_max) +{ + int version, tmp_real_max; + int hole; + const SSL_METHOD *single = NULL; + const SSL_METHOD *method; + const version_info *table; + const version_info *vent; + + switch (s->method->version) { + default: + /* + * If this SSL handle is not from a version flexible method we don't + * (and never did) check min/max FIPS or Suite B constraints. Hope + * that's OK. It is up to the caller to not choose fixed protocol + * versions they don't want. If not, then easy to fix, just return + * ssl_method_error(s, s->method) + */ + *min_version = *max_version = s->version; + /* + * Providing a real_max only makes sense where we're using a version + * flexible method. + */ + if (!ossl_assert(real_max == NULL)) + return ERR_R_INTERNAL_ERROR; + return 0; + case TLS_ANY_VERSION: + table = tls_version_table; + break; + } + + /* + * SSL_OP_NO_X disables all protocols above X *if* there are some protocols + * below X enabled. This is required in order to maintain the "version + * capability" vector contiguous. Any versions with a NULL client method + * (protocol version client is disabled at compile-time) is also a "hole". + * + * Our initial state is hole == 1, version == 0. That is, versions above + * the first version in the method table are disabled (a "hole" above + * the valid protocol entries) and we don't have a selected version yet. + * + * Whenever "hole == 1", and we hit an enabled method, its version becomes + * the selected version, and the method becomes a candidate "single" + * method. We're no longer in a hole, so "hole" becomes 0. + * + * If "hole == 0" and we hit an enabled method, then "single" is cleared, + * as we support a contiguous range of at least two methods. If we hit + * a disabled method, then hole becomes true again, but nothing else + * changes yet, because all the remaining methods may be disabled too. + * If we again hit an enabled method after the new hole, it becomes + * selected, as we start from scratch. + */ + *min_version = version = 0; + hole = 1; + if (real_max != NULL) + *real_max = 0; + tmp_real_max = 0; + for (vent = table; vent->version != 0; ++vent) { + /* + * A table entry with a NULL client method is still a hole in the + * "version capability" vector. + */ + if (vent->cmeth == NULL) { + hole = 1; + tmp_real_max = 0; + continue; + } + method = vent->cmeth(); + + if (hole == 1 && tmp_real_max == 0) + tmp_real_max = vent->version; + + if (ssl_method_error(s, method) != 0) { + hole = 1; + } else if (!hole) { + single = NULL; + *min_version = method->version; + } else { + if (real_max != NULL && tmp_real_max != 0) + *real_max = tmp_real_max; + version = (single = method)->version; + *min_version = version; + hole = 0; + } + } + + *max_version = version; + + /* Fail if everything is disabled */ + if (version == 0) + return SSL_R_NO_PROTOCOLS_AVAILABLE; + + return 0; +} + +/* + * ssl_set_client_hello_version_ntls - Work out what version we should be using for + * the initial ClientHello.legacy_version field. + * + * @s: client SSL handle. + * + * Returns 0 on success or an SSL error reason number on failure. + */ +int ssl_set_client_hello_version_ntls(SSL *s) +{ + int ver_min, ver_max, ret; + + /* + * In a renegotiation we always send the same client_version that we sent + * last time, regardless of which version we eventually negotiated. + */ + if (!SSL_IS_FIRST_HANDSHAKE(s)) + return 0; + + ret = ssl_get_min_max_version_ntls(s, &ver_min, &ver_max, NULL); + + if (ret != 0) + return ret; + + s->version = ver_max; + + /* TLS1.3 always uses TLS1.2 in the legacy_version field */ + if (ver_max > TLS1_2_VERSION) + ver_max = TLS1_2_VERSION; + + s->client_version = ver_max; + return 0; +} + +/* + * Checks a list of |groups| to determine if the |group_id| is in it. If it is + * and |checkallow| is 1 then additionally check if the group is allowed to be + * used. Returns 1 if the group is in the list (and allowed if |checkallow| is + * 1) or 0 otherwise. + */ +int check_in_list_ntls(SSL *s, uint16_t group_id, const uint16_t *groups, + size_t num_groups, int checkallow) +{ + size_t i; + + if (groups == NULL || num_groups == 0) + return 0; + + for (i = 0; i < num_groups; i++) { + uint16_t group = groups[i]; + + if (group_id == group + && (!checkallow + || tls_group_allowed(s, group, SSL_SECOP_CURVE_CHECK))) { + return 1; + } + } + + return 0; +} + +/* Replace ClientHello1 in the transcript hash with a synthetic message */ +int create_synthetic_message_hash_ntls(SSL *s, const unsigned char *hashval, + size_t hashlen, const unsigned char *hrr, + size_t hrrlen) +{ + unsigned char hashvaltmp[EVP_MAX_MD_SIZE]; + unsigned char msghdr[SSL3_HM_HEADER_LENGTH]; + + memset(msghdr, 0, sizeof(msghdr)); + + if (hashval == NULL) { + hashval = hashvaltmp; + hashlen = 0; + /* Get the hash of the initial ClientHello */ + if (!ssl3_digest_cached_records(s, 0) + || !ssl_handshake_hash(s, hashvaltmp, sizeof(hashvaltmp), + &hashlen)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + + /* Reinitialise the transcript hash */ + if (!ssl3_init_finished_mac(s)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + /* Inject the synthetic message_hash message */ + msghdr[0] = SSL3_MT_MESSAGE_HASH; + msghdr[SSL3_HM_HEADER_LENGTH - 1] = (unsigned char)hashlen; + if (!ssl3_finish_mac(s, msghdr, SSL3_HM_HEADER_LENGTH) + || !ssl3_finish_mac(s, hashval, hashlen)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + /* + * Now re-inject the HRR and current message if appropriate (we just deleted + * it when we reinitialised the transcript hash above). Only necessary after + * receiving a ClientHello2 with a cookie. + */ + if (hrr != NULL + && (!ssl3_finish_mac(s, hrr, hrrlen) + || !ssl3_finish_mac(s, (unsigned char *)s->init_buf->data, + s->s3.tmp.message_size + + SSL3_HM_HEADER_LENGTH))) { + /* SSLfatal_ntls() already called */ + return 0; + } + + return 1; +} + +static int ca_dn_cmp(const X509_NAME *const *a, const X509_NAME *const *b) +{ + return X509_NAME_cmp(*a, *b); +} + +int parse_ca_names_ntls(SSL *s, PACKET *pkt) +{ + STACK_OF(X509_NAME) *ca_sk = sk_X509_NAME_new(ca_dn_cmp); + X509_NAME *xn = NULL; + PACKET cadns; + + if (ca_sk == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + /* get the CA RDNs */ + if (!PACKET_get_length_prefixed_2(pkt, &cadns)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + while (PACKET_remaining(&cadns)) { + const unsigned char *namestart, *namebytes; + unsigned int name_len; + + if (!PACKET_get_net_2(&cadns, &name_len) + || !PACKET_get_bytes(&cadns, &namebytes, name_len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + namestart = namebytes; + if ((xn = d2i_X509_NAME(NULL, &namebytes, name_len)) == NULL) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, ERR_R_ASN1_LIB); + goto err; + } + if (namebytes != (namestart + name_len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_CA_DN_LENGTH_MISMATCH); + goto err; + } + + if (!sk_X509_NAME_push(ca_sk, xn)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + xn = NULL; + } + + sk_X509_NAME_pop_free(s->s3.tmp.peer_ca_names, X509_NAME_free); + s->s3.tmp.peer_ca_names = ca_sk; + + return 1; + + err: + sk_X509_NAME_pop_free(ca_sk, X509_NAME_free); + X509_NAME_free(xn); + return 0; +} + +const STACK_OF(X509_NAME) *get_ca_names_ntls(SSL *s) +{ + const STACK_OF(X509_NAME) *ca_sk = NULL;; + + if (s->server) { + ca_sk = SSL_get_client_CA_list(s); + if (ca_sk != NULL && sk_X509_NAME_num(ca_sk) == 0) + ca_sk = NULL; + } + + if (ca_sk == NULL) + ca_sk = SSL_get0_CA_list(s); + + return ca_sk; +} + +int construct_ca_names_ntls(SSL *s, const STACK_OF(X509_NAME) *ca_sk, WPACKET *pkt) +{ + /* Start sub-packet for client CA list */ + if (!WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if ((ca_sk != NULL) && !(s->options & SSL_OP_DISABLE_TLSEXT_CA_NAMES)) { + int i; + + for (i = 0; i < sk_X509_NAME_num(ca_sk); i++) { + unsigned char *namebytes; + X509_NAME *name = sk_X509_NAME_value(ca_sk, i); + int namelen; + + if (name == NULL + || (namelen = i2d_X509_NAME(name, NULL)) < 0 + || !WPACKET_sub_allocate_bytes_u16(pkt, namelen, + &namebytes) + || i2d_X509_NAME(name, &namebytes) != namelen) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + } + + if (!WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +/* Create a buffer containing data to be signed for server key exchange */ +size_t construct_key_exchange_tbs_ntls(SSL *s, unsigned char **ptbs, + const void *param, size_t paramlen) +{ + size_t tbslen = 2 * SSL3_RANDOM_SIZE + paramlen; + unsigned char *tbs = OPENSSL_malloc(tbslen); + + if (tbs == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + memcpy(tbs, s->s3.client_random, SSL3_RANDOM_SIZE); + memcpy(tbs + SSL3_RANDOM_SIZE, s->s3.server_random, SSL3_RANDOM_SIZE); + + memcpy(tbs + SSL3_RANDOM_SIZE * 2, param, paramlen); + + *ptbs = tbs; + return tbslen; +} + +/* + * Saves the current handshake digest for Post-Handshake Auth, + * Done after ClientFinished is processed, done exactly once + */ +int tls13_save_handshake_digest_for_pha_ntls(SSL *s) +{ + if (s->pha_dgst == NULL) { + if (!ssl3_digest_cached_records(s, 1)) + /* SSLfatal_ntls() already called */ + return 0; + + s->pha_dgst = EVP_MD_CTX_new(); + if (s->pha_dgst == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if (!EVP_MD_CTX_copy_ex(s->pha_dgst, + s->s3.handshake_dgst)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + EVP_MD_CTX_free(s->pha_dgst); + s->pha_dgst = NULL; + return 0; + } + } + return 1; +} + +/* + * Restores the Post-Handshake Auth handshake digest + * Done just before sending/processing the Cert Request + */ +int tls13_restore_handshake_digest_for_pha_ntls(SSL *s) +{ + if (s->pha_dgst == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + if (!EVP_MD_CTX_copy_ex(s->s3.handshake_dgst, + s->pha_dgst)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + return 1; +} + +unsigned char *x509_to_asn1_ntls(const X509 *x, size_t *len) +{ + unsigned char *buf = NULL; + unsigned char *p; + size_t buflen; + int n; + + if (x == NULL || len == NULL) + return NULL; + + if ((n = i2d_X509(x, NULL)) <= 0) + return NULL; + + /* opaque ASN.1Cert<1..2^24-1> */ + buflen = n + 3; + + buf = OPENSSL_malloc(buflen); + if (buf == NULL) + return NULL; + + p = buf + 3; + if ((n = i2d_X509(x, &p)) <= 0) { + OPENSSL_free(buf); + return NULL; + } + + l2n3(n, buf); + buf -= 3; + + *len = n + 3; + + return buf; +} + +int ssl_derive_ntls(SSL *s, EVP_PKEY *privkey, EVP_PKEY *pubkey, int gensecret) +{ + int rv = 0; + int idx = 1; + X509 *peer_x509 = NULL; + EVP_PKEY *peer_cert_pub = NULL; + EVP_PKEY *cert_priv = NULL; + unsigned char *pms = NULL; + size_t pmslen = SSL_MAX_MASTER_KEY_LENGTH; + EVP_PKEY_CTX *pctx = NULL; + OSSL_PARAM params[8], *p = params; + + if (privkey == NULL || pubkey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* SM2 requires to use the private key in encryption certificate */ + cert_priv = s->cert->pkeys[SSL_PKEY_SM2_ENC].privatekey; + if (cert_priv == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* + * XXX: + * + * For NTLS server side, s->session->peer stores the client signing + * certificate and s->session->peer_chain is an one-item stack which + * stores the client encryption certificate. + * + * We need to get the client encryption certificate at this stage, + * so we use index 0 in peer_chain. + * + * For client side of NTLS, the peer is an reference of the first element + * of the two-item stack stored in s->session->peer_chain, which is the + * signing certificate of server. So we need to get the second certificate + * in this scenario for encryption usage. + */ + if (s->server) + idx = 0; + + if (s->session->peer_chain == NULL + || (peer_x509 = sk_X509_value(s->session->peer_chain, idx)) == NULL + || (peer_cert_pub = X509_get0_pubkey(peer_x509)) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + pms = OPENSSL_malloc(pmslen); + if (pms == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + pctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, privkey, s->ctx->propq); + + /* for NTLS, server is initiator(Z_A), client is responder(Z_B) */ + *p++ = OSSL_PARAM_construct_int(OSSL_EXCHANGE_PARAM_INITIATOR, + &s->server); + *p++ = OSSL_PARAM_construct_octet_string(OSSL_EXCHANGE_PARAM_SELF_ID, + SM2_DEFAULT_ID, + SM2_DEFAULT_ID_LEN); + *p++ = OSSL_PARAM_construct_octet_string(OSSL_EXCHANGE_PARAM_PEER_ID, + SM2_DEFAULT_ID, + SM2_DEFAULT_ID_LEN); + *p++ = OSSL_PARAM_construct_octet_ptr(OSSL_EXCHANGE_PARAM_SELF_ENC_KEY, + (void **)&cert_priv, + sizeof(cert_priv)); + *p++ = OSSL_PARAM_construct_octet_ptr(OSSL_EXCHANGE_PARAM_PEER_ENC_KEY, + (void **)&peer_cert_pub, + sizeof(peer_cert_pub)); + *p++ = OSSL_PARAM_construct_utf8_string(OSSL_EXCHANGE_PARAM_DIGEST, + "SM3", 0); + *p++ = OSSL_PARAM_construct_size_t(OSSL_EXCHANGE_PARAM_OUTLEN, &pmslen); + *p = OSSL_PARAM_construct_end(); + + if (EVP_PKEY_derive_init_ex(pctx, params) <= 0 + || EVP_PKEY_derive_set_peer(pctx, pubkey) <= 0 + || EVP_PKEY_derive(pctx, pms, &pmslen) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (gensecret) { + rv = ssl_gensecret(s, pms, pmslen); + } else { + /* Save premaster secret */ + s->s3.tmp.pms = pms; + s->s3.tmp.pmslen = pmslen; + pms = NULL; + rv = 1; + } + +err: + OPENSSL_clear_free(pms, pmslen); + EVP_PKEY_CTX_free(pctx); + return rv; +} + +int SSL_connection_is_ntls(SSL *s, int is_server) +{ + int ret = 0; + unsigned int version; + uint8_t *p, *data = NULL; + + /* + * For client, or sometimes ssl_version is fixed, + * we can easily determine if version is NTLS + */ + if (s->version == NTLS1_1_VERSION) + return 1; + + if (is_server) { + /* After receiving client hello and before choosing server version, + * get version from s->clienthello->legacy_version + */ + if (s->clienthello) + return s->clienthello->legacy_version == NTLS1_1_VERSION; + + if (s->preread_len >= sizeof(s->preread_buf)) { + p = &s->preread_buf[1]; + n2s(p, version); + return version == NTLS1_1_VERSION; + } + + /* + * For server, first flight has not set version, we + * have to get the server version from clientHello + */ + if (!SSL_IS_FIRST_HANDSHAKE(s) || !SSL_in_before(s)) + return 0; + + if (s->rbio == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_READ_BIO_NOT_SET); + return -1; + } + + data = s->preread_buf + s->preread_len; + + clear_sys_error(); + s->rwstate = SSL_READING; + ret = BIO_read(s->rbio, data, sizeof(s->preread_buf) - s->preread_len); + + if (ret <= 0 && !BIO_should_retry(s->rbio) && BIO_eof(s->rbio)) { + if (s->options & SSL_OP_IGNORE_UNEXPECTED_EOF) { + SSL_set_shutdown(s, SSL_RECEIVED_SHUTDOWN); + s->s3.warn_alert = SSL_AD_CLOSE_NOTIFY; + } else { + SSLfatal(s, SSL_AD_DECODE_ERROR, + SSL_R_UNEXPECTED_EOF_WHILE_READING); + } + + return -1; + } + + if (ret > 0) + s->preread_len += ret; + + if (s->preread_len >= sizeof(s->preread_buf)) { + BIO *bbio = BIO_new(BIO_f_buffer()); + if (bbio == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return -1; + } + + if (BIO_set_buffer_read_data(bbio, s->preread_buf, + sizeof(s->preread_buf)) + != 1) { + BIO_vfree(bbio); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + + s->rwstate = SSL_NOTHING; + s->rbio = BIO_push(bbio, s->rbio); + + p = &s->preread_buf[1]; + n2s(p, version); + return version == NTLS1_1_VERSION; + } + + return -1; + } + + return 0; +} diff --git a/openssl/src/ssl/statem_ntls/ntls_statem_local.h b/openssl/src/ssl/statem_ntls/ntls_statem_local.h new file mode 100644 index 000000000..c48ad6481 --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_statem_local.h @@ -0,0 +1,450 @@ +/* + * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2022 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/***************************************************************************** + * * + * The following definitions are PRIVATE to the state machine. They should * + * NOT be used outside of the state machine. * + * * + *****************************************************************************/ + +/* Max message length definitions */ + +/* The spec allows for a longer length than this, but we limit it */ +#define HELLO_VERIFY_REQUEST_MAX_LENGTH 258 +#define END_OF_EARLY_DATA_MAX_LENGTH 0 +#define HELLO_RETRY_REQUEST_MAX_LENGTH 20000 +#define ENCRYPTED_EXTENSIONS_MAX_LENGTH 20000 +#define SESSION_TICKET_MAX_LENGTH_TLS13 131338 +#define SESSION_TICKET_MAX_LENGTH_TLS12 65541 +#define SERVER_KEY_EXCH_MAX_LENGTH 102400 +#define SERVER_HELLO_DONE_MAX_LENGTH 0 +#define KEY_UPDATE_MAX_LENGTH 1 +#define CCS_MAX_LENGTH 1 + +/* Max ServerHello size permitted by RFC 8446 */ +#define SERVER_HELLO_MAX_LENGTH 65607 + +/* Max should actually be 36 but we are generous */ +#define FINISHED_MAX_LENGTH 64 + +/* Dummy message type */ +#define SSL3_MT_DUMMY -1 + +#ifndef OPENSSL_NO_SM2 +/* + *standard handshake sm2-id and cert verify id is defined + * in IETF RFC 8998 + */ +# define HANDSHAKE_SM2_ID "TLSv1.3+GM+Cipher+Suite" +# define HANDSHAKE_SM2_ID_LEN sizeof(HANDSHAKE_SM2_ID) - 1 +# define CERTVRIFY_SM2_ID "1234567812345678" +# define CERTVRIFY_SM2_ID_LEN sizeof(CERTVRIFY_SM2_ID) - 1 + +#endif + +extern const unsigned char hrrrandom_ntls[]; + +/* Message processing return codes */ +typedef enum { + /* Something bad happened */ + MSG_PROCESS_ERROR, + /* We've finished reading - swap to writing */ + MSG_PROCESS_FINISHED_READING, + /* + * We've completed the main processing of this message but there is some + * post processing to be done. + */ + MSG_PROCESS_CONTINUE_PROCESSING, + /* We've finished this message - read the next message */ + MSG_PROCESS_CONTINUE_READING +} MSG_PROCESS_RETURN; + +typedef int (*confunc_f) (SSL *s, WPACKET *pkt); + +int ssl3_take_mac_ntls(SSL *s); +int check_in_list_ntls(SSL *s, uint16_t group_id, const uint16_t *groups, + size_t num_groups, int checkallow); +int create_synthetic_message_hash_ntls(SSL *s, const unsigned char *hashval, + size_t hashlen, const unsigned char *hrr, + size_t hrrlen); +int parse_ca_names_ntls(SSL *s, PACKET *pkt); +const STACK_OF(X509_NAME) *get_ca_names_ntls(SSL *s); +int construct_ca_names_ntls(SSL *s, const STACK_OF(X509_NAME) *ca_sk, WPACKET *pkt); +size_t construct_key_exchange_tbs_ntls(SSL *s, unsigned char **ptbs, + const void *param, size_t paramlen); + +/* + * TLS/DTLS client state machine functions + */ +int ossl_statem_client_read_transition_ntls(SSL *s, int mt); +WRITE_TRAN ossl_statem_client_write_transition_ntls(SSL *s); +WORK_STATE ossl_statem_client_pre_work_ntls(SSL *s, WORK_STATE wst); +WORK_STATE ossl_statem_client_post_work_ntls(SSL *s, WORK_STATE wst); +int ossl_statem_client_construct_message_ntls(SSL *s, WPACKET *pkt, + confunc_f *confunc, int *mt); +size_t ossl_statem_client_max_message_size_ntls(SSL *s); +MSG_PROCESS_RETURN ossl_statem_client_process_message_ntls(SSL *s, PACKET *pkt); +WORK_STATE ossl_statem_client_post_process_message_ntls(SSL *s, WORK_STATE wst); + +/* + * TLS/DTLS server state machine functions + */ +int ossl_statem_server_read_transition_ntls(SSL *s, int mt); +WRITE_TRAN ossl_statem_server_write_transition_ntls(SSL *s); +WORK_STATE ossl_statem_server_pre_work_ntls(SSL *s, WORK_STATE wst); +WORK_STATE ossl_statem_server_post_work_ntls(SSL *s, WORK_STATE wst); +int ossl_statem_server_construct_message_ntls(SSL *s, WPACKET *pkt, + confunc_f *confunc,int *mt); +size_t ossl_statem_server_max_message_size_ntls(SSL *s); +MSG_PROCESS_RETURN ossl_statem_server_process_message_ntls(SSL *s, PACKET *pkt); +WORK_STATE ossl_statem_server_post_process_message_ntls(SSL *s, WORK_STATE wst); + +/* Functions for getting new message data */ +__owur int tls_get_message_header_ntls(SSL *s, int *mt); +__owur int tls_get_message_body_ntls(SSL *s, size_t *len); +__owur int dtls_get_message_ntls(SSL *s, int *mt, size_t *len); + +/* Message construction and processing functions */ +__owur int tls_process_initial_server_flight_ntls(SSL *s); +__owur MSG_PROCESS_RETURN tls_process_change_cipher_spec_ntls(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_finished_ntls(SSL *s, PACKET *pkt); +__owur int tls_construct_change_cipher_spec_ntls(SSL *s, WPACKET *pkt); + + +__owur int tls_construct_finished_ntls(SSL *s, WPACKET *pkt); +__owur WORK_STATE tls_finish_handshake_ntls(SSL *s, WORK_STATE wst, int clearbufs, + int stop); +__owur WORK_STATE dtls_wait_for_dry_ntls(SSL *s); + +/* some client-only functions */ +__owur int tls_construct_client_hello_ntls(SSL *s, WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_server_hello_ntls(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_certificate_request_ntls(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_new_session_ticket_ntls(SSL *s, PACKET *pkt); +__owur int tls_process_cert_status_body_ntls(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_cert_status_ntls(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_server_done_ntls(SSL *s, PACKET *pkt); +__owur int tls_construct_cert_verify_ntls(SSL *s, WPACKET *pkt); +__owur WORK_STATE tls_prepare_client_certificate_ntls(SSL *s, WORK_STATE wst); +__owur int tls_construct_client_certificate_ntls(SSL *s, WPACKET *pkt); +__owur int ssl_do_client_cert_cb_ntls(SSL *s, X509 **px509, EVP_PKEY **ppkey); +__owur int tls_construct_client_key_exchange_ntls(SSL *s, WPACKET *pkt); +__owur int tls_client_key_exchange_post_work_ntls(SSL *s); +__owur int tls_construct_cert_status_body_ntls(SSL *s, WPACKET *pkt); +__owur int tls_construct_cert_status_ntls(SSL *s, WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_key_exchange_ntls(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_server_certificate_ntls(SSL *s, PACKET *pkt); +__owur WORK_STATE tls_post_process_server_certificate_ntls(SSL *s, WORK_STATE wst); +__owur int ssl3_check_cert_and_algorithm_ntls(SSL *s); +#ifndef OPENSSL_NO_NEXTPROTONEG +__owur int tls_construct_next_proto_ntls(SSL *s, WPACKET *pkt); +#endif +__owur MSG_PROCESS_RETURN tls_process_hello_req_ntls(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN dtls_process_hello_verify_ntls(SSL *s, PACKET *pkt); +__owur int tls_construct_end_of_early_data_ntls(SSL *s, WPACKET *pkt); + +/* some server-only functions */ +__owur MSG_PROCESS_RETURN tls_process_client_hello_ntls(SSL *s, PACKET *pkt); +__owur WORK_STATE tls_post_process_client_hello_ntls(SSL *s, WORK_STATE wst); +__owur int tls_construct_server_hello_ntls(SSL *s, WPACKET *pkt); +__owur int dtls_construct_hello_verify_request_ntls(SSL *s, WPACKET *pkt); +__owur int tls_construct_server_certificate_ntls(SSL *s, WPACKET *pkt); +__owur int tls_construct_server_key_exchange_ntls(SSL *s, WPACKET *pkt); +__owur int tls_construct_certificate_request_ntls(SSL *s, WPACKET *pkt); +__owur int tls_construct_server_done_ntls(SSL *s, WPACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_client_certificate_ntls(SSL *s, PACKET *pkt); +__owur MSG_PROCESS_RETURN tls_process_client_key_exchange_ntls(SSL *s, PACKET *pkt); +__owur WORK_STATE tls_post_process_client_key_exchange_ntls(SSL *s, WORK_STATE wst); +__owur MSG_PROCESS_RETURN tls_process_cert_verify_ntls(SSL *s, PACKET *pkt); +#ifndef OPENSSL_NO_NEXTPROTONEG +__owur MSG_PROCESS_RETURN tls_process_next_proto_ntls(SSL *s, PACKET *pkt); +#endif +__owur int tls_construct_new_session_ticket_ntls(SSL *s, WPACKET *pkt); +MSG_PROCESS_RETURN tls_process_end_of_early_data_ntls(SSL *s, PACKET *pkt); + + +/* Extension processing */ + +typedef enum ext_return_en { + EXT_RETURN_FAIL, + EXT_RETURN_SENT, + EXT_RETURN_NOT_SENT +} EXT_RETURN; + +__owur int tls_validate_all_contexts_ntls(SSL *s, unsigned int thisctx, + RAW_EXTENSION *exts); +__owur int extension_is_relevant_ntls(SSL *s, unsigned int extctx, + unsigned int thisctx); +__owur int tls_collect_extensions_ntls(SSL *s, PACKET *packet, unsigned int context, + RAW_EXTENSION **res, size_t *len, int init); +__owur int tls_parse_extension_ntls(SSL *s, TLSEXT_INDEX idx, int context, + RAW_EXTENSION *exts, X509 *x, size_t chainidx); +__owur int tls_parse_all_extensions_ntls(SSL *s, int context, RAW_EXTENSION *exts, + X509 *x, size_t chainidx, int fin); +__owur int should_add_extension_ntls(SSL *s, unsigned int extctx, + unsigned int thisctx, int max_version); +__owur int tls_construct_extensions_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); + +__owur int tls_psk_do_binder_ntls(SSL *s, const EVP_MD *md, + const unsigned char *msgstart, + size_t binderoffset, const unsigned char *binderin, + unsigned char *binderout, + SSL_SESSION *sess, int sign, int external); + +/* Server Extension processing */ +int tls_parse_ctos_server_name_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_ctos_maxfragmentlen_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_ctos_early_data_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_ctos_ec_pt_formats_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_ctos_supported_groups_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidxl); +int tls_parse_ctos_session_ticket_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_ctos_sig_algs_cert_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_ctos_sig_algs_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#ifndef OPENSSL_NO_OCSP +int tls_parse_ctos_status_request_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif +#ifndef OPENSSL_NO_NEXTPROTONEG +int tls_parse_ctos_npn_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#endif +int tls_parse_ctos_alpn_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#ifndef OPENSSL_NO_SRTP +int tls_parse_ctos_use_srtp_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#endif +int tls_parse_ctos_etm_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_key_share_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_cookie_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_ems_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_psk_kex_modes_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_ctos_psk_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_ctos_post_handshake_auth_ntls(SSL *, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_stoc_server_name_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_stoc_early_data_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_stoc_maxfragmentlen_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_stoc_ec_pt_formats_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_stoc_supported_groups_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_stoc_session_ticket_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +#ifndef OPENSSL_NO_OCSP +EXT_RETURN tls_construct_stoc_status_request_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +#endif +#ifndef OPENSSL_NO_NEXTPROTONEG +EXT_RETURN tls_construct_stoc_next_proto_neg_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +#endif +EXT_RETURN tls_construct_stoc_alpn_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#ifndef OPENSSL_NO_SRTP +EXT_RETURN tls_construct_stoc_use_srtp_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif +EXT_RETURN tls_construct_stoc_etm_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_stoc_ems_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_stoc_supported_versions_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_stoc_key_share_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_stoc_cookie_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); + +EXT_RETURN tls_construct_stoc_psk_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); + +/* Client Extension processing */ +EXT_RETURN tls_construct_ctos_server_name_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_ctos_maxfragmentlen_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); + +EXT_RETURN tls_construct_ctos_ec_pt_formats_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_ctos_supported_groups_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_ctos_early_data_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_ctos_session_ticket_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_ctos_sig_algs_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +#ifndef OPENSSL_NO_OCSP +EXT_RETURN tls_construct_ctos_status_request_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +#endif +#ifndef OPENSSL_NO_NEXTPROTONEG +EXT_RETURN tls_construct_ctos_npn_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif +EXT_RETURN tls_construct_ctos_alpn_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#ifndef OPENSSL_NO_SRTP +EXT_RETURN tls_construct_ctos_use_srtp_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif +EXT_RETURN tls_construct_ctos_etm_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#ifndef OPENSSL_NO_CT +EXT_RETURN tls_construct_ctos_sct_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif +EXT_RETURN tls_construct_ctos_ems_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_ctos_supported_versions_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_ctos_key_share_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_ctos_psk_kex_modes_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_ctos_cookie_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_ctos_padding_ntls(SSL *s, WPACKET *pkt, + unsigned int context, X509 *x, + size_t chainidx); +EXT_RETURN tls_construct_ctos_psk_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +EXT_RETURN tls_construct_ctos_post_handshake_auth_ntls(SSL *s, WPACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); + +int tls_parse_stoc_server_name_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_stoc_early_data_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_stoc_maxfragmentlen_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_stoc_ec_pt_formats_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_stoc_session_ticket_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#ifndef OPENSSL_NO_OCSP +int tls_parse_stoc_status_request_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +#endif +#ifndef OPENSSL_NO_CT +int tls_parse_stoc_sct_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#endif +#ifndef OPENSSL_NO_NEXTPROTONEG +int tls_parse_stoc_npn_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#endif +int tls_parse_stoc_alpn_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#ifndef OPENSSL_NO_SRTP +int tls_parse_stoc_use_srtp_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +#endif +int tls_parse_stoc_etm_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_stoc_ems_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_stoc_supported_versions_ntls(SSL *s, PACKET *pkt, unsigned int context, + X509 *x, size_t chainidx); +int tls_parse_stoc_key_share_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_stoc_cookie_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); +int tls_parse_stoc_psk_ntls(SSL *s, PACKET *pkt, unsigned int context, X509 *x, + size_t chainidx); + +int tls_handle_alpn_ntls(SSL *s); + +int tls13_save_handshake_digest_for_pha_ntls(SSL *s); +int tls13_restore_handshake_digest_for_pha_ntls(SSL *s); + + + +/* NTLS stuffs */ + +/* common functions */ +int ssl_derive_ntls(SSL *s, EVP_PKEY *privkey, EVP_PKEY *pubkey, int gensecret); + +/* from statem.h move here */ + +/* function define in ssl.h */ + +__owur int SSL_CTX_has_client_custom_ext_ntls(const SSL_CTX *ctx, + unsigned int ext_type); + +__owur int SSL_CTX_add_client_custom_ext_ntls(SSL_CTX *ctx, + unsigned int ext_type, + custom_ext_add_cb add_cb, + custom_ext_free_cb free_cb, + void *add_arg, + custom_ext_parse_cb parse_cb, + void *parse_arg); + +__owur int SSL_CTX_add_server_custom_ext_ntls(SSL_CTX *ctx, + unsigned int ext_type, + custom_ext_add_cb add_cb, + custom_ext_free_cb free_cb, + void *add_arg, + custom_ext_parse_cb parse_cb, + void *parse_arg); + +__owur int SSL_CTX_add_custom_ext_ntls(SSL_CTX *ctx, unsigned int ext_type, + unsigned int context, + SSL_custom_ext_add_cb_ex add_cb, + SSL_custom_ext_free_cb_ex free_cb, + void *add_arg, + SSL_custom_ext_parse_cb_ex parse_cb, + void *parse_arg); + +__owur int SSL_extension_supported_ntls(unsigned int ext_type); +__owur OSSL_HANDSHAKE_STATE SSL_get_state_ntls(const SSL *ssl); +int SSL_in_init_ntls(const SSL *s); +int SSL_in_before_ntls(const SSL *s); +int SSL_is_init_finished_ntls(const SSL *s); +unsigned char *x509_to_asn1_ntls(const X509 *x, size_t *len); diff --git a/openssl/src/ssl/statem_ntls/ntls_statem_srvr.c b/openssl/src/ssl/statem_ntls/ntls_statem_srvr.c new file mode 100644 index 000000000..ea68337fc --- /dev/null +++ b/openssl/src/ssl/statem_ntls/ntls_statem_srvr.c @@ -0,0 +1,2573 @@ +/* + * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2023 The Tongsuo Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include +#include "ntls_ssl_local.h" +#include "ntls_statem_local.h" +#include "internal/constant_time.h" +#include "internal/cryptlib.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TICKET_NONCE_SIZE 8 + +/* + * ossl_statem_server_read_transition_ntls() encapsulates the logic for the allowed + * handshake state transitions when the server is reading messages from the + * client. The message type that the client has sent is provided in |mt|. The + * current state is in |s->statem.hand_state|. + * + * Return values are 1 for success (transition allowed) and 0 on error + * (transition not allowed) + */ +int ossl_statem_server_read_transition_ntls(SSL *s, int mt) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + break; + + case TLS_ST_BEFORE: + case TLS_ST_OK: + if (mt == SSL3_MT_CLIENT_HELLO) { + st->hand_state = TLS_ST_SR_CLNT_HELLO; + return 1; + } + break; + + case TLS_ST_SW_SRVR_DONE: + /* + * If we get a CKE message after a ServerDone then either + * 1) We didn't request a Certificate + * OR + * 2) If we did request one then + * a) We allow no Certificate to be returned + * AND + * b) We are running SSL3 (in TLS1.0+ the client must return a 0 + * list if we requested a certificate) + */ + if (mt == SSL3_MT_CLIENT_KEY_EXCHANGE) { + if (s->s3.tmp.cert_request) { + if (s->version == SSL3_VERSION) { + if ((s->verify_mode & SSL_VERIFY_PEER) + && (s->verify_mode & SSL_VERIFY_FAIL_IF_NO_PEER_CERT)) { + /* + * This isn't an unexpected message as such - we're just + * not going to accept it because we require a client + * cert. + */ + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE); + return 0; + } + st->hand_state = TLS_ST_SR_KEY_EXCH; + return 1; + } + } else { + st->hand_state = TLS_ST_SR_KEY_EXCH; + return 1; + } + } else if (s->s3.tmp.cert_request) { + if (mt == SSL3_MT_CERTIFICATE) { + st->hand_state = TLS_ST_SR_CERT; + return 1; + } + } + break; + + case TLS_ST_SR_CERT: + if (mt == SSL3_MT_CLIENT_KEY_EXCHANGE) { + st->hand_state = TLS_ST_SR_KEY_EXCH; + return 1; + } + break; + + case TLS_ST_SR_KEY_EXCH: + /* + * We should only process a CertificateVerify message if we have + * received a Certificate from the client. If so then |s->session->peer| + * will be non NULL. In some instances a CertificateVerify message is + * not required even if the peer has sent a Certificate (e.g. such as in + * the case of static DH). In that case |st->no_cert_verify| should be + * set. + */ + if (s->session->peer == NULL || st->no_cert_verify) { + if (mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + /* + * For the ECDH ciphersuites when the client sends its ECDH + * pub key in a certificate, the CertificateVerify message is + * not sent. + */ + st->hand_state = TLS_ST_SR_CHANGE; + return 1; + } + } else { + if (mt == SSL3_MT_CERTIFICATE_VERIFY) { + st->hand_state = TLS_ST_SR_CERT_VRFY; + return 1; + } + } + break; + + case TLS_ST_SR_CERT_VRFY: + if (mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + st->hand_state = TLS_ST_SR_CHANGE; + return 1; + } + break; + + case TLS_ST_SR_CHANGE: +#ifndef OPENSSL_NO_NEXTPROTONEG + if (s->s3.npn_seen) { + if (mt == SSL3_MT_NEXT_PROTO) { + st->hand_state = TLS_ST_SR_NEXT_PROTO; + return 1; + } + } else { +#endif + if (mt == SSL3_MT_FINISHED) { + st->hand_state = TLS_ST_SR_FINISHED; + return 1; + } +#ifndef OPENSSL_NO_NEXTPROTONEG + } +#endif + break; + +#ifndef OPENSSL_NO_NEXTPROTONEG + case TLS_ST_SR_NEXT_PROTO: + if (mt == SSL3_MT_FINISHED) { + st->hand_state = TLS_ST_SR_FINISHED; + return 1; + } + break; +#endif + + case TLS_ST_SW_FINISHED: + if (mt == SSL3_MT_CHANGE_CIPHER_SPEC) { + st->hand_state = TLS_ST_SR_CHANGE; + return 1; + } + break; + } + + /* No valid transition found */ + SSLfatal_ntls(s, SSL3_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_MESSAGE); + return 0; +} + +/* + * Should we send a ServerKeyExchange message? + * + * Valid return values are: + * 1: Yes + * 0: No + */ +static int send_server_key_exchange(SSL *s) +{ + return 1; +} + +/* + * Should we send a CertificateRequest message? + * + * Valid return values are: + * 1: Yes + * 0: No + */ +int send_certificate_request_ntls(SSL *s) +{ + if ( + /* don't request cert unless asked for it: */ + s->verify_mode & SSL_VERIFY_PEER + /* + * if SSL_VERIFY_CLIENT_ONCE is set, don't request cert + * a second time: + */ + && (s->certreqs_sent < 1 || + !(s->verify_mode & SSL_VERIFY_CLIENT_ONCE)) + /* + * never request cert in anonymous ciphersuites (see + * section "Certificate request" in SSL 3 drafts and in + * RFC 2246): + */ + && (!(s->s3.tmp.new_cipher->algorithm_auth & SSL_aNULL) + /* + * ... except when the application insists on + * verification (against the specs, but statem_clnt.c accepts + * this for SSL 3) + */ + || (s->verify_mode & SSL_VERIFY_FAIL_IF_NO_PEER_CERT)) + /* don't request certificate for SRP auth */ + && !(s->s3.tmp.new_cipher->algorithm_auth & SSL_aSRP) + /* + * With normal PSK Certificates and Certificate Requests + * are omitted + */ + && !(s->s3.tmp.new_cipher->algorithm_auth & SSL_aPSK)) { + return 1; + } + + return 0; +} + +/* + * ossl_statem_server_write_transition_ntls() works out what handshake state to move + * to next when the server is writing messages to be sent to the client. + */ +WRITE_TRAN ossl_statem_server_write_transition_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + + /* + * Note that before the ClientHello we don't know what version we are going + * to negotiate yet, so we don't take this branch until later + */ + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return WRITE_TRAN_ERROR; + + case TLS_ST_OK: + if (st->request_state == TLS_ST_SW_HELLO_REQ) { + /* We must be trying to renegotiate */ + st->hand_state = TLS_ST_SW_HELLO_REQ; + st->request_state = TLS_ST_BEFORE; + return WRITE_TRAN_CONTINUE; + } + /* Must be an incoming ClientHello */ + if (!tls_setup_handshake_ntls(s)) { + /* SSLfatal_ntls() already called */ + return WRITE_TRAN_ERROR; + } + /* Fall through */ + + case TLS_ST_BEFORE: + /* Just go straight to trying to read from the client */ + return WRITE_TRAN_FINISHED; + + case TLS_ST_SW_HELLO_REQ: + st->hand_state = TLS_ST_OK; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_SR_CLNT_HELLO: + if (s->renegotiate == 0 && !SSL_IS_FIRST_HANDSHAKE(s)) { + /* We must have rejected the renegotiation */ + st->hand_state = TLS_ST_OK; + return WRITE_TRAN_CONTINUE; + } else { + st->hand_state = TLS_ST_SW_SRVR_HELLO; + } + return WRITE_TRAN_CONTINUE; + + case TLS_ST_SW_SRVR_HELLO: + if (s->hit) { + if (s->ext.ticket_expected) + st->hand_state = TLS_ST_SW_SESSION_TICKET; + else + st->hand_state = TLS_ST_SW_CHANGE; + } else { + /* Check if it is anon DH or anon ECDH, */ + /* normal PSK or SRP */ + if (!(s->s3.tmp.new_cipher->algorithm_auth & + (SSL_aNULL | SSL_aSRP | SSL_aPSK))) { + st->hand_state = TLS_ST_SW_CERT; + } else if (send_server_key_exchange(s)) { + st->hand_state = TLS_ST_SW_KEY_EXCH; + } else if (send_certificate_request_ntls(s)) { + st->hand_state = TLS_ST_SW_CERT_REQ; + } else { + st->hand_state = TLS_ST_SW_SRVR_DONE; + } + } + return WRITE_TRAN_CONTINUE; + + case TLS_ST_SW_CERT: + if (s->ext.status_expected) { + st->hand_state = TLS_ST_SW_CERT_STATUS; + return WRITE_TRAN_CONTINUE; + } + /* Fall through */ + + case TLS_ST_SW_CERT_STATUS: + if (send_server_key_exchange(s)) { + st->hand_state = TLS_ST_SW_KEY_EXCH; + return WRITE_TRAN_CONTINUE; + } + /* Fall through */ + + case TLS_ST_SW_KEY_EXCH: + if (send_certificate_request_ntls(s)) { + st->hand_state = TLS_ST_SW_CERT_REQ; + return WRITE_TRAN_CONTINUE; + } + /* Fall through */ + + case TLS_ST_SW_CERT_REQ: + st->hand_state = TLS_ST_SW_SRVR_DONE; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_SW_SRVR_DONE: + return WRITE_TRAN_FINISHED; + + case TLS_ST_SR_FINISHED: + if (s->hit) { + st->hand_state = TLS_ST_OK; + return WRITE_TRAN_CONTINUE; + } else if (s->ext.ticket_expected) { + st->hand_state = TLS_ST_SW_SESSION_TICKET; + } else { + st->hand_state = TLS_ST_SW_CHANGE; + } + return WRITE_TRAN_CONTINUE; + + case TLS_ST_SW_SESSION_TICKET: + st->hand_state = TLS_ST_SW_CHANGE; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_SW_CHANGE: + st->hand_state = TLS_ST_SW_FINISHED; + return WRITE_TRAN_CONTINUE; + + case TLS_ST_SW_FINISHED: + if (s->hit) { + return WRITE_TRAN_FINISHED; + } + st->hand_state = TLS_ST_OK; + return WRITE_TRAN_CONTINUE; + } +} + +/* + * Perform any pre work that needs to be done prior to sending a message from + * the server to the client. + */ +WORK_STATE ossl_statem_server_pre_work_ntls(SSL *s, WORK_STATE wst) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* No pre work to be done */ + break; + + case TLS_ST_SW_HELLO_REQ: + s->shutdown = 0; + break; + + case TLS_ST_SW_SRVR_HELLO: + break; + + case TLS_ST_SW_SRVR_DONE: + return WORK_FINISHED_CONTINUE; + + case TLS_ST_SW_SESSION_TICKET: + break; + + case TLS_ST_SW_CHANGE: + /* Writes to s->session are only safe for initial handshakes */ + if (s->session->cipher == NULL) { + s->session->cipher = s->s3.tmp.new_cipher; + } else if (s->session->cipher != s->s3.tmp.new_cipher) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return WORK_ERROR; + } + if (!s->method->ssl3_enc->setup_key_block(s)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + return WORK_FINISHED_CONTINUE; + + case TLS_ST_EARLY_DATA: + if (s->early_data_state != SSL_EARLY_DATA_ACCEPTING + && (s->s3.flags & TLS1_FLAGS_STATELESS) == 0) + return WORK_FINISHED_CONTINUE; + /* Fall through */ + + case TLS_ST_OK: + /* Calls SSLfatal_ntls() as required */ + return tls_finish_handshake_ntls(s, wst, 1, 1); + } + + return WORK_FINISHED_CONTINUE; +} + +/* + * Perform any work that needs to be done after sending a message from the + * server to the client. + */ +WORK_STATE ossl_statem_server_post_work_ntls(SSL *s, WORK_STATE wst) +{ + OSSL_STATEM *st = &s->statem; + + s->init_num = 0; + + switch (st->hand_state) { + default: + /* No post work to be done */ + break; + + case TLS_ST_SW_HELLO_REQ: + if (statem_flush_ntls(s) != 1) + return WORK_MORE_A; + if (!ssl3_init_finished_mac(s)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + break; + + case TLS_ST_SW_SRVR_HELLO: + break; + + case TLS_ST_SW_CHANGE: + if (s->hello_retry_request == SSL_HRR_PENDING) { + if (!statem_flush_ntls(s)) + return WORK_MORE_A; + break; + } + + if (!s->method->ssl3_enc->change_cipher_state(s, + SSL3_CHANGE_CIPHER_SERVER_WRITE)) + { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + + break; + + case TLS_ST_SW_SRVR_DONE: + if (statem_flush_ntls(s) != 1) + return WORK_MORE_A; + break; + + case TLS_ST_SW_FINISHED: + if (statem_flush_ntls(s) != 1) + return WORK_MORE_A; + break; + + case TLS_ST_SW_CERT_REQ: + if (s->post_handshake_auth == SSL_PHA_REQUEST_PENDING) { + if (statem_flush_ntls(s) != 1) + return WORK_MORE_A; + } + break; + + case TLS_ST_SW_KEY_UPDATE: + if (statem_flush_ntls(s) != 1) + return WORK_MORE_A; + if (!tls13_update_key(s, 1)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + break; + + case TLS_ST_SW_SESSION_TICKET: + clear_sys_error(); + break; + } + + return WORK_FINISHED_CONTINUE; +} + +/* + * Get the message construction function and message type for sending from the + * server + * + * Valid return values are: + * 1: Success + * 0: Error + */ +int ossl_statem_server_construct_message_ntls(SSL *s, WPACKET *pkt, + confunc_f *confunc, int *mt) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_HANDSHAKE_STATE); + return 0; + + case TLS_ST_SW_CHANGE: + *confunc = tls_construct_change_cipher_spec_ntls; + *mt = SSL3_MT_CHANGE_CIPHER_SPEC; + break; + + case TLS_ST_SW_HELLO_REQ: + /* No construction function needed */ + *confunc = NULL; + *mt = SSL3_MT_HELLO_REQUEST; + break; + + case TLS_ST_SW_SRVR_HELLO: + *confunc = tls_construct_server_hello_ntls; + *mt = SSL3_MT_SERVER_HELLO; + break; + + case TLS_ST_SW_CERT: + *confunc = tls_construct_server_certificate_ntls; + *mt = SSL3_MT_CERTIFICATE; + break; + + case TLS_ST_SW_KEY_EXCH: + *confunc = tls_construct_server_key_exchange_ntls; + *mt = SSL3_MT_SERVER_KEY_EXCHANGE; + break; + + case TLS_ST_SW_CERT_REQ: + *confunc = tls_construct_certificate_request_ntls; + *mt = SSL3_MT_CERTIFICATE_REQUEST; + break; + + case TLS_ST_SW_SRVR_DONE: + *confunc = tls_construct_server_done_ntls; + *mt = SSL3_MT_SERVER_DONE; + break; + + case TLS_ST_SW_SESSION_TICKET: + *confunc = tls_construct_new_session_ticket_ntls; + *mt = SSL3_MT_NEWSESSION_TICKET; + break; + + case TLS_ST_SW_CERT_STATUS: + *confunc = tls_construct_cert_status_ntls; + *mt = SSL3_MT_CERTIFICATE_STATUS; + break; + + case TLS_ST_SW_FINISHED: + *confunc = tls_construct_finished_ntls; + *mt = SSL3_MT_FINISHED; + break; + + case TLS_ST_EARLY_DATA: + *confunc = NULL; + *mt = SSL3_MT_DUMMY; + break; + } + + return 1; +} + +/* + * Maximum size (excluding the Handshake header) of a ClientHello message, + * calculated as follows: + * + * 2 + # client_version + * 32 + # only valid length for random + * 1 + # length of session_id + * 32 + # maximum size for session_id + * 2 + # length of cipher suites + * 2^16-2 + # maximum length of cipher suites array + * 1 + # length of compression_methods + * 2^8-1 + # maximum length of compression methods + * 2 + # length of extensions + * 2^16-1 # maximum length of extensions + */ +#define CLIENT_HELLO_MAX_LENGTH 131396 + +#define CLIENT_KEY_EXCH_MAX_LENGTH 2048 +#define NEXT_PROTO_MAX_LENGTH 514 + +/* + * Returns the maximum allowed length for the current message that we are + * reading. Excludes the message header. + */ +size_t ossl_statem_server_max_message_size_ntls(SSL *s) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + return 0; + + case TLS_ST_SR_CLNT_HELLO: + return CLIENT_HELLO_MAX_LENGTH; + + case TLS_ST_SR_CERT: + return s->max_cert_list; + + case TLS_ST_SR_KEY_EXCH: + return CLIENT_KEY_EXCH_MAX_LENGTH; + + case TLS_ST_SR_CERT_VRFY: + return SSL3_RT_MAX_PLAIN_LENGTH; + +#ifndef OPENSSL_NO_NEXTPROTONEG + case TLS_ST_SR_NEXT_PROTO: + return NEXT_PROTO_MAX_LENGTH; +#endif + + case TLS_ST_SR_CHANGE: + return CCS_MAX_LENGTH; + + case TLS_ST_SR_FINISHED: + return FINISHED_MAX_LENGTH; + } +} + +/* + * Process a message that the server has received from the client. + */ +MSG_PROCESS_RETURN ossl_statem_server_process_message_ntls(SSL *s, PACKET *pkt) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return MSG_PROCESS_ERROR; + + case TLS_ST_SR_CLNT_HELLO: + return tls_process_client_hello_ntls(s, pkt); + + case TLS_ST_SR_END_OF_EARLY_DATA: + return tls_process_end_of_early_data_ntls(s, pkt); + + case TLS_ST_SR_CERT: + return tls_process_client_certificate_ntls(s, pkt); + + case TLS_ST_SR_KEY_EXCH: + return tls_process_client_key_exchange_ntls(s, pkt); + + case TLS_ST_SR_CERT_VRFY: + return tls_process_cert_verify_ntls(s, pkt); + +#ifndef OPENSSL_NO_NEXTPROTONEG + case TLS_ST_SR_NEXT_PROTO: + return tls_process_next_proto_ntls(s, pkt); +#endif + + case TLS_ST_SR_CHANGE: + return tls_process_change_cipher_spec_ntls(s, pkt); + + case TLS_ST_SR_FINISHED: + return tls_process_finished_ntls(s, pkt); + } +} + +/* + * Perform any further processing required following the receipt of a message + * from the client + */ +WORK_STATE ossl_statem_server_post_process_message_ntls(SSL *s, WORK_STATE wst) +{ + OSSL_STATEM *st = &s->statem; + + switch (st->hand_state) { + default: + /* Shouldn't happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return WORK_ERROR; + + case TLS_ST_SR_CLNT_HELLO: + return tls_post_process_client_hello_ntls(s, wst); + + case TLS_ST_SR_KEY_EXCH: + return tls_post_process_client_key_exchange_ntls(s, wst); + } +} + +/*- + * ssl_check_for_safari attempts to fingerprint Safari using OS X + * SecureTransport using the TLS extension block in |hello|. + * Safari, since 10.6, sends exactly these extensions, in this order: + * SNI, + * elliptic_curves + * ec_point_formats + * signature_algorithms (for TLSv1.2 only) + * + * We wish to fingerprint Safari because they broke ECDHE-ECDSA support in 10.8, + * but they advertise support. So enabling ECDHE-ECDSA ciphers breaks them. + * Sadly we cannot differentiate 10.6, 10.7 and 10.8.4 (which work), from + * 10.8..10.8.3 (which don't work). + */ +static void ssl_check_for_safari(SSL *s, const CLIENTHELLO_MSG *hello) +{ + static const unsigned char kSafariExtensionsBlock[] = { + 0x00, 0x0a, /* elliptic_curves extension */ + 0x00, 0x08, /* 8 bytes */ + 0x00, 0x06, /* 6 bytes of curve ids */ + 0x00, 0x17, /* P-256 */ + 0x00, 0x18, /* P-384 */ + 0x00, 0x19, /* P-521 */ + + 0x00, 0x0b, /* ec_point_formats */ + 0x00, 0x02, /* 2 bytes */ + 0x01, /* 1 point format */ + 0x00, /* uncompressed */ + /* The following is only present in TLS 1.2 */ + 0x00, 0x0d, /* signature_algorithms */ + 0x00, 0x0c, /* 12 bytes */ + 0x00, 0x0a, /* 10 bytes */ + 0x05, 0x01, /* SHA-384/RSA */ + 0x04, 0x01, /* SHA-256/RSA */ + 0x02, 0x01, /* SHA-1/RSA */ + 0x04, 0x03, /* SHA-256/ECDSA */ + 0x02, 0x03, /* SHA-1/ECDSA */ + }; + /* Length of the common prefix (first two extensions). */ + static const size_t kSafariCommonExtensionsLength = 18; + unsigned int type; + PACKET sni, tmppkt; + size_t ext_len; + + tmppkt = hello->extensions; + + if (!PACKET_forward(&tmppkt, 2) + || !PACKET_get_net_2(&tmppkt, &type) + || !PACKET_get_length_prefixed_2(&tmppkt, &sni)) { + return; + } + + if (type != TLSEXT_TYPE_server_name) + return; + + ext_len = TLS1_get_client_version(s) >= TLS1_2_VERSION ? + sizeof(kSafariExtensionsBlock) : kSafariCommonExtensionsLength; + + s->s3.is_probably_safari = PACKET_equal(&tmppkt, kSafariExtensionsBlock, + ext_len); +} + +#define RENEG_OPTIONS_OK(options) \ + ((options & SSL_OP_NO_RENEGOTIATION) == 0 \ + && (options & SSL_OP_ALLOW_CLIENT_RENEGOTIATION) != 0) + +MSG_PROCESS_RETURN tls_process_client_hello_ntls(SSL *s, PACKET *pkt) +{ + PACKET session_id, compression, extensions, cookie; + static const unsigned char null_compression = 0; + CLIENTHELLO_MSG *clienthello = NULL; + + /* Check if this is actually an unexpected renegotiation ClientHello */ + if (s->renegotiate == 0 && !SSL_IS_FIRST_HANDSHAKE(s)) { + if (!ossl_assert(!SSL_IS_TLS13(s))) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + if (!RENEG_OPTIONS_OK(s->options) + || (!s->s3.send_connection_binding + && (s->options + & SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION) == 0)) { + ssl3_send_alert(s, SSL3_AL_WARNING, SSL_AD_NO_RENEGOTIATION); + return MSG_PROCESS_FINISHED_READING; + } + s->renegotiate = 1; + s->new_session = 1; + } + + clienthello = OPENSSL_zalloc(sizeof(*clienthello)); + if (clienthello == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* + * First, parse the raw ClientHello data into the CLIENTHELLO_MSG structure. + */ + clienthello->isv2 = RECORD_LAYER_is_sslv2_record(&s->rlayer); + PACKET_null_init(&cookie); + + if (clienthello->isv2) { + unsigned int mt; + + if (!SSL_IS_FIRST_HANDSHAKE(s) + || s->hello_retry_request != SSL_HRR_NONE) { + SSLfatal_ntls(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_UNEXPECTED_MESSAGE); + goto err; + } + + /*- + * An SSLv3/TLSv1 backwards-compatible CLIENT-HELLO in an SSLv2 + * header is sent directly on the wire, not wrapped as a TLS + * record. Our record layer just processes the message length and passes + * the rest right through. Its format is: + * Byte Content + * 0-1 msg_length - decoded by the record layer + * 2 msg_type - s->init_msg points here + * 3-4 version + * 5-6 cipher_spec_length + * 7-8 session_id_length + * 9-10 challenge_length + * ... ... + */ + + if (!PACKET_get_1(pkt, &mt) + || mt != SSL2_MT_CLIENT_HELLO) { + /* + * Should never happen. We should have tested this in the record + * layer in order to have determined that this is a SSLv2 record + * in the first place + */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } + + if (!PACKET_get_net_2(pkt, &clienthello->legacy_version)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); + goto err; + } + + /* Parse the message and load client random. */ + if (clienthello->isv2) { + /* + * Handle an SSLv2 backwards compatible ClientHello + * Note, this is only for SSLv3+ using the backward compatible format. + * Real SSLv2 is not supported, and is rejected below. + */ + unsigned int ciphersuite_len, session_id_len, challenge_len; + PACKET challenge; + + if (!PACKET_get_net_2(pkt, &ciphersuite_len) + || !PACKET_get_net_2(pkt, &session_id_len) + || !PACKET_get_net_2(pkt, &challenge_len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_RECORD_LENGTH_MISMATCH); + goto err; + } + + if (session_id_len > SSL_MAX_SSL_SESSION_ID_LENGTH) { + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_LENGTH_MISMATCH); + goto err; + } + + if (!PACKET_get_sub_packet(pkt, &clienthello->ciphersuites, + ciphersuite_len) + || !PACKET_copy_bytes(pkt, clienthello->session_id, session_id_len) + || !PACKET_get_sub_packet(pkt, &challenge, challenge_len) + /* No extensions. */ + || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_RECORD_LENGTH_MISMATCH); + goto err; + } + clienthello->session_id_len = session_id_len; + + /* Load the client random and compression list. We use SSL3_RANDOM_SIZE + * here rather than sizeof(clienthello->random) because that is the limit + * for SSLv3 and it is fixed. It won't change even if + * sizeof(clienthello->random) does. + */ + challenge_len = challenge_len > SSL3_RANDOM_SIZE + ? SSL3_RANDOM_SIZE : challenge_len; + memset(clienthello->random, 0, SSL3_RANDOM_SIZE); + if (!PACKET_copy_bytes(&challenge, + clienthello->random + SSL3_RANDOM_SIZE - + challenge_len, challenge_len) + /* Advertise only null compression. */ + || !PACKET_buf_init(&compression, &null_compression, 1)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + PACKET_null_init(&clienthello->extensions); + } else { + /* Regular ClientHello. */ + if (!PACKET_copy_bytes(pkt, clienthello->random, SSL3_RANDOM_SIZE) + || !PACKET_get_length_prefixed_1(pkt, &session_id) + || !PACKET_copy_all(&session_id, clienthello->session_id, + SSL_MAX_SSL_SESSION_ID_LENGTH, + &clienthello->session_id_len)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + if (!PACKET_get_length_prefixed_2(pkt, &clienthello->ciphersuites)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + if (!PACKET_get_length_prefixed_1(pkt, &compression)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + /* Could be empty. */ + if (PACKET_remaining(pkt) == 0) { + PACKET_null_init(&clienthello->extensions); + } else { + if (!PACKET_get_length_prefixed_2(pkt, &clienthello->extensions) + || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + } + } + +#ifndef OPENSSL_NO_STATUS + if (s->status_param.ssl_status_enable) { + /* record client session_id */ + s->status_param.type = SSL_CLIENT_SESSION_ID; + if (s->status_callback(clienthello->session_id, + clienthello->session_id_len, + &s->status_param) == -1) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + + /* record client ciphers */ + int ciphersuite_size = 2; + s->status_param.type = clienthello->isv2 ? SSL_CLIENT_V2_CIPHER : SSL_CLIENT_CIPHER; + s->status_param.parg = &ciphersuite_size; + if (s->status_callback((unsigned char *)clienthello->ciphersuites.curr, + clienthello->ciphersuites.remaining, + &s->status_param) == -1) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + } +#endif + + if (!PACKET_copy_all(&compression, clienthello->compressions, + MAX_COMPRESSIONS_SIZE, + &clienthello->compressions_len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* Preserve the raw extensions PACKET for later use */ + extensions = clienthello->extensions; + if (!tls_collect_extensions_ntls(s, &extensions, SSL_EXT_CLIENT_HELLO, + &clienthello->pre_proc_exts, + &clienthello->pre_proc_exts_len, 1)) { + /* SSLfatal_ntls already been called */ + goto err; + } + s->clienthello = clienthello; + + return MSG_PROCESS_CONTINUE_PROCESSING; + + err: + if (clienthello != NULL) + OPENSSL_free(clienthello->pre_proc_exts); + OPENSSL_free(clienthello); + + return MSG_PROCESS_ERROR; +} + +static int tls_early_post_process_client_hello(SSL *s) +{ + unsigned int j; + int i, al = SSL_AD_INTERNAL_ERROR; + int protverr; + size_t loop; + unsigned long id; + + const SSL_CIPHER *c; + STACK_OF(SSL_CIPHER) *ciphers = NULL; + STACK_OF(SSL_CIPHER) *scsvs = NULL; + CLIENTHELLO_MSG *clienthello = s->clienthello; + DOWNGRADE dgrd = DOWNGRADE_NONE; + + /* Finished parsing the ClientHello, now we can start processing it */ + /* Give the ClientHello callback a crack at things */ + if (s->ctx->client_hello_cb != NULL) { + /* + * Support setting ocsp response message in clienthello callback + * function. Parse the ocsp status and set the relevant flags here, + * otherwise the ocsp API may fail in the client_hello_cb function. + * Because in client_hello_cb it is possible to check if the client + * sends a status_request message. + */ + if (!tls_parse_extension_ntls(s, TLSEXT_IDX_status_request, + SSL_EXT_CLIENT_HELLO, + clienthello->pre_proc_exts, NULL, 0)) + goto err; + + /* A failure in the ClientHello callback terminates the connection. */ + switch (s->ctx->client_hello_cb(s, &al, s->ctx->client_hello_cb_arg)) { + case SSL_CLIENT_HELLO_SUCCESS: + break; + case SSL_CLIENT_HELLO_RETRY: + s->rwstate = SSL_CLIENT_HELLO_CB; + return -1; + case SSL_CLIENT_HELLO_ERROR: + default: + SSLfatal_ntls(s, al, SSL_R_CALLBACK_FAILED); + goto err; + } + } + + /* Set up the client_random */ + memcpy(s->s3.client_random, clienthello->random, SSL3_RANDOM_SIZE); + + /* Choose the version */ + + if (clienthello->isv2) { + if (clienthello->legacy_version == NTLS_VERSION) { + /* do nothing */ + } else if (clienthello->legacy_version == SSL2_VERSION + || (clienthello->legacy_version & 0xff00) + != (SSL3_VERSION_MAJOR << 8)) { + /* + * This is real SSLv2 or something completely unknown. We don't + * support it. + */ + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, SSL_R_UNKNOWN_PROTOCOL); + goto err; + } + /* SSLv3/TLS */ + s->client_version = clienthello->legacy_version; + } + /* + * Do SSL/TLS version negotiation if applicable. Version negotiation comes later. + */ + protverr = ssl_choose_server_version_ntls(s, clienthello, &dgrd); + + if (protverr) { + if (SSL_IS_FIRST_HANDSHAKE(s)) { + /* like ssl3_get_record, send alert using remote version number */ + s->version = s->client_version = clienthello->legacy_version; + } + SSLfatal_ntls(s, SSL_AD_PROTOCOL_VERSION, protverr); + goto err; + } + + s->hit = 0; + + if (!ssl_cache_cipherlist(s, &clienthello->ciphersuites, + clienthello->isv2) || + !bytes_to_cipher_list(s, &clienthello->ciphersuites, &ciphers, &scsvs, + clienthello->isv2, 1)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + s->s3.send_connection_binding = 0; + /* Check what signalling cipher-suite values were received. */ + if (scsvs != NULL) { + for(i = 0; i < sk_SSL_CIPHER_num(scsvs); i++) { + c = sk_SSL_CIPHER_value(scsvs, i); + if (SSL_CIPHER_get_id(c) == SSL3_CK_SCSV) { + if (s->renegotiate) { + /* SCSV is fatal if renegotiating */ + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING); + goto err; + } + s->s3.send_connection_binding = 1; + } else if (SSL_CIPHER_get_id(c) == SSL3_CK_FALLBACK_SCSV && + !ssl_check_version_downgrade_ntls(s)) { + /* + * This SCSV indicates that the client previously tried + * a higher version. We should fail if the current version + * is an unexpected downgrade, as that indicates that the first + * connection may have been tampered with in order to trigger + * an insecure downgrade. + */ + SSLfatal_ntls(s, SSL_AD_INAPPROPRIATE_FALLBACK, + SSL_R_INAPPROPRIATE_FALLBACK); + goto err; + } + } + } + + /* + * We don't allow resumption in a backwards compatible ClientHello. + * In TLS1.1+, session_id MUST be empty. + * + * Versions before 0.9.7 always allow clients to resume sessions in + * renegotiation. 0.9.7 and later allow this by default, but optionally + * ignore resumption requests with flag + * SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION (it's a new flag rather + * than a change to default behavior so that applications relying on + * this for security won't even compile against older library versions). + * 1.0.1 and later also have a function SSL_renegotiate_abbreviated() to + * request renegotiation but not a new session (s->new_session remains + * unset): for servers, this essentially just means that the + * SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION setting will be + * ignored. + */ + if (clienthello->isv2 || + (s->new_session && + (s->options & SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION))) { + if (!ssl_get_new_session(s, 1)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } else { + i = ssl_get_prev_session(s, clienthello); + if (i == 1) { + /* previous session */ + s->hit = 1; + } else if (i == -1) { + /* SSLfatal_ntls() already called */ + goto err; + } else { + /* i == 0 */ + if (!ssl_get_new_session(s, 1)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } + } + + + + /* + * If it is a hit, check that the cipher is in the list. In TLSv1.3 we check + * ciphersuite compatibility with the session as part of resumption. + */ + if (s->hit) { + j = 0; + id = s->session->cipher->id; + + OSSL_TRACE_BEGIN(TLS_CIPHER) { + BIO_printf(trc_out, "client sent %d ciphers\n", + sk_SSL_CIPHER_num(ciphers)); + } + for (i = 0; i < sk_SSL_CIPHER_num(ciphers); i++) { + c = sk_SSL_CIPHER_value(ciphers, i); + if (trc_out != NULL) + BIO_printf(trc_out, "client [%2d of %2d]:%s\n", i, + sk_SSL_CIPHER_num(ciphers), SSL_CIPHER_get_name(c)); + if (c->id == id) { + j = 1; + break; + } + } + if (j == 0) { + /* + * we need to have the cipher in the cipher list if we are asked + * to reuse it + */ + SSLfatal_ntls(s, SSL_AD_ILLEGAL_PARAMETER, + SSL_R_REQUIRED_CIPHER_MISSING); + OSSL_TRACE_CANCEL(TLS_CIPHER); + goto err; + } + OSSL_TRACE_END(TLS_CIPHER); + } + + for (loop = 0; loop < clienthello->compressions_len; loop++) { + if (clienthello->compressions[loop] == 0) + break; + } + + if (loop >= clienthello->compressions_len) { + /* no compress */ + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_NO_COMPRESSION_SPECIFIED); + goto err; + } + + if (s->options & SSL_OP_SAFARI_ECDHE_ECDSA_BUG) + ssl_check_for_safari(s, clienthello); + + /* TLS extensions */ + if (!tls_parse_all_extensions_ntls(s, SSL_EXT_CLIENT_HELLO, + clienthello->pre_proc_exts, NULL, 0, 1)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + /* + * Check if we want to use external pre-shared secret for this handshake + * for not reused session only. We need to generate server_random before + * calling tls_session_secret_cb in order to allow SessionTicket + * processing to use it in key derivation. + */ + { + unsigned char *pos; + pos = s->s3.server_random; + if (ssl_fill_hello_random(s, 1, pos, SSL3_RANDOM_SIZE, dgrd) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } + + if (!s->hit + && s->version >= TLS1_VERSION + && s->ext.session_secret_cb) { + const SSL_CIPHER *pref_cipher = NULL; + /* + * s->session->master_key_length is a size_t, but this is an int for + * backwards compat reasons + */ + int master_key_length; + + master_key_length = sizeof(s->session->master_key); + if (s->ext.session_secret_cb(s, s->session->master_key, + &master_key_length, ciphers, + &pref_cipher, + s->ext.session_secret_cb_arg) + && master_key_length > 0) { + s->session->master_key_length = master_key_length; + s->hit = 1; + s->peer_ciphers = ciphers; + s->session->verify_result = X509_V_OK; + + ciphers = NULL; + + /* check if some cipher was preferred by call back */ + if (pref_cipher == NULL) + pref_cipher = ssl3_choose_cipher(s, s->peer_ciphers, + SSL_get_ciphers(s)); + if (pref_cipher == NULL) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_NO_SHARED_CIPHER); + goto err; + } + + s->session->cipher = pref_cipher; + sk_SSL_CIPHER_free(s->cipher_list); + s->cipher_list = sk_SSL_CIPHER_dup(s->peer_ciphers); + sk_SSL_CIPHER_free(s->cipher_list_by_id); + s->cipher_list_by_id = sk_SSL_CIPHER_dup(s->peer_ciphers); + } + } + + /* + * Worst case, we will use the NULL compression, but if we have other + * options, we will now look for them. We have complen-1 compression + * algorithms from the client, starting at q. + */ + s->s3.tmp.new_compression = NULL; + + /* + * If compression is disabled we'd better not try to resume a session + * using compression. + */ + if (s->session->compress_meth != 0) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_INCONSISTENT_COMPRESSION); + goto err; + } + + + /* + * Given s->peer_ciphers and SSL_get_ciphers, we must pick a cipher + */ + + if (!s->hit) { + sk_SSL_CIPHER_free(s->peer_ciphers); + s->peer_ciphers = ciphers; + if (ciphers == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + ciphers = NULL; + } + + if (!s->hit) { + s->session->compress_meth = 0; + } + + sk_SSL_CIPHER_free(ciphers); + sk_SSL_CIPHER_free(scsvs); + OPENSSL_free(clienthello->pre_proc_exts); + OPENSSL_free(s->clienthello); + s->clienthello = NULL; + return 1; + err: + sk_SSL_CIPHER_free(ciphers); + sk_SSL_CIPHER_free(scsvs); + OPENSSL_free(clienthello->pre_proc_exts); + OPENSSL_free(s->clienthello); + s->clienthello = NULL; + + return 0; +} + +/* + * Call the status request callback if needed. Upon success, returns 1. + * Upon failure, returns 0. + */ +static int tls_handle_status_request(SSL *s) +{ + s->ext.status_expected = 0; + + /* + * If status request then ask callback what to do. Note: this must be + * called after servername callbacks in case the certificate has changed, + * and must be called after the cipher has been chosen because this may + * influence which certificate is sent + */ + if (s->ext.status_type != TLSEXT_STATUSTYPE_nothing && s->ctx != NULL + && s->ctx->ext.status_cb != NULL) { + int ret; + + /* If no certificate can't return certificate status */ + if (s->s3.tmp.cert != NULL) { + /* + * Set current certificate to one we will use so SSL_get_certificate + * et al can pick it up. + */ + s->cert->key = s->s3.tmp.cert; + ret = s->ctx->ext.status_cb(s, s->ctx->ext.status_arg); + switch (ret) { + /* We don't want to send a status request response */ + case SSL_TLSEXT_ERR_NOACK: + s->ext.status_expected = 0; + break; + /* status request response should be sent */ + case SSL_TLSEXT_ERR_OK: + if (s->ext.ocsp.resp) + s->ext.status_expected = 1; + break; + /* something bad happened */ + case SSL_TLSEXT_ERR_ALERT_FATAL: + default: + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_CLIENTHELLO_TLSEXT); + return 0; + } + } + } + + return 1; +} + +/* + * Call the alpn_select callback if needed. Upon success, returns 1. + * Upon failure, returns 0. + */ +int tls_handle_alpn_ntls(SSL *s) +{ + const unsigned char *selected = NULL; + unsigned char selected_len = 0; + + if (s->ctx->ext.alpn_select_cb != NULL && s->s3.alpn_proposed != NULL) { + int r = s->ctx->ext.alpn_select_cb(s, &selected, &selected_len, + s->s3.alpn_proposed, + (unsigned int)s->s3.alpn_proposed_len, + s->ctx->ext.alpn_select_cb_arg); + + if (r == SSL_TLSEXT_ERR_OK) { + OPENSSL_free(s->s3.alpn_selected); + s->s3.alpn_selected = OPENSSL_memdup(selected, selected_len); + if (s->s3.alpn_selected == NULL) { + s->s3.alpn_selected_len = 0; + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + s->s3.alpn_selected_len = selected_len; +#ifndef OPENSSL_NO_NEXTPROTONEG + /* ALPN takes precedence over NPN. */ + s->s3.npn_seen = 0; +#endif + + /* Check ALPN is consistent with session */ + if (s->session->ext.alpn_selected == NULL + || selected_len != s->session->ext.alpn_selected_len + || memcmp(selected, s->session->ext.alpn_selected, + selected_len) != 0) { + /* Not consistent so can't be used for early_data */ + s->ext.early_data_ok = 0; + + if (!s->hit) { + /* + * This is a new session and so alpn_selected should have + * been initialised to NULL. We should update it with the + * selected ALPN. + */ + if (!ossl_assert(s->session->ext.alpn_selected == NULL)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + ERR_R_INTERNAL_ERROR); + return 0; + } + s->session->ext.alpn_selected = OPENSSL_memdup(selected, + selected_len); + if (s->session->ext.alpn_selected == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + ERR_R_INTERNAL_ERROR); + return 0; + } + s->session->ext.alpn_selected_len = selected_len; + } + } + + return 1; + } else if (r != SSL_TLSEXT_ERR_NOACK) { + SSLfatal_ntls(s, SSL_AD_NO_APPLICATION_PROTOCOL, + SSL_R_NO_APPLICATION_PROTOCOL); + return 0; + } + /* + * If r == SSL_TLSEXT_ERR_NOACK then behave as if no callback was + * present. + */ + } + + /* Check ALPN is consistent with session */ + if (s->session->ext.alpn_selected != NULL) { + /* Not consistent so can't be used for early_data */ + s->ext.early_data_ok = 0; + } + + return 1; +} + +WORK_STATE tls_post_process_client_hello_ntls(SSL *s, WORK_STATE wst) +{ + const SSL_CIPHER *cipher; + + if (wst == WORK_MORE_A) { + int rv = tls_early_post_process_client_hello(s); + if (rv == 0) { + /* SSLfatal_ntls() was already called */ + goto err; + } + if (rv < 0) + return WORK_MORE_A; + wst = WORK_MORE_B; + } + if (wst == WORK_MORE_B) { + if (!s->hit) { + /* Let cert callback update server certificates if required */ + if (!s->hit && s->cert->cert_cb != NULL) { + int rv = s->cert->cert_cb(s, s->cert->cert_cb_arg); + if (rv == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_CERT_CB_ERROR); + goto err; + } + if (rv < 0) { + s->rwstate = SSL_X509_LOOKUP; + return WORK_MORE_B; + } + s->rwstate = SSL_NOTHING; + } + + cipher = + ssl3_choose_cipher(s, s->peer_ciphers, SSL_get_ciphers(s)); + + if (cipher == NULL) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_NO_SHARED_CIPHER); + goto err; + } + s->s3.tmp.new_cipher = cipher; + + if (!s->hit) { + if (!tls_choose_sigalg_ntls(s, 1)) { + /* SSLfatal_ntls already called */ + goto err; + } + /* check whether we should disable session resumption */ + if (s->not_resumable_session_cb != NULL) + s->session->not_resumable = + s->not_resumable_session_cb(s, + ((s->s3.tmp.new_cipher->algorithm_mkey + & (SSL_kDHE | SSL_kECDHE)) != 0)); + if (s->session->not_resumable) + /* do not send a session ticket */ + s->ext.ticket_expected = 0; + } + } else { + /* Session-id reuse */ + s->s3.tmp.new_cipher = s->session->cipher; + } + + if (s->s3.tmp.new_cipher->algorithm_mkey & SSL_kSM2DHE) + s->verify_mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT + | SSL_VERIFY_CLIENT_ONCE; + + /*- + * we now have the following setup. + * client_random + * cipher_list - our preferred list of ciphers + * ciphers - the clients preferred list of ciphers + * compression - basically ignored right now + * ssl version is set - sslv3 + * s->session - The ssl session has been setup. + * s->hit - session reuse flag + * s->s3.tmp.new_cipher - the new cipher to use. + */ + + /* + * Call status_request callback if needed. Has to be done after the + * certificate callbacks etc above. + */ + if (!tls_handle_status_request(s)) { + /* SSLfatal_ntls() already called */ + goto err; + } + /* + * Call alpn_select callback if needed. Has to be done after SNI and + * cipher negotiation (HTTP/2 restricts permitted ciphers). In TLSv1.3 + * we already did this because cipher negotiation happens earlier, and + * we must handle ALPN before we decide whether to accept early_data. + */ + if (!tls_handle_alpn_ntls(s)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + wst = WORK_MORE_C; + } + + return WORK_FINISHED_STOP; + err: + return WORK_ERROR; +} + +int tls_construct_server_hello_ntls(SSL *s, WPACKET *pkt) +{ + int compm; + size_t sl, len; + int version; + unsigned char *session_id; + int usetls13 = s->hello_retry_request == SSL_HRR_PENDING; + + version = usetls13 ? TLS1_2_VERSION : s->version; + if (!WPACKET_put_bytes_u16(pkt, version) + /* + * Random stuff. Filling of the server_random takes place in + * tls_process_client_hello_ntls() + */ + || !WPACKET_memcpy(pkt, + s->hello_retry_request == SSL_HRR_PENDING + ? hrrrandom_ntls : s->s3.server_random, + SSL3_RANDOM_SIZE)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /*- + * There are several cases for the session ID to send + * back in the server hello: + * - For session reuse from the session cache, + * we send back the old session ID. + * - If stateless session reuse (using a session ticket) + * is successful, we send back the client's "session ID" + * (which doesn't actually identify the session). + * - If it is a new session, we send back the new + * session ID. + * - However, if we want the new session to be single-use, + * we send back a 0-length session ID. + * - In TLSv1.3 we echo back the session id sent to us by the client + * regardless + * s->hit is non-zero in either case of session reuse, + * so the following won't overwrite an ID that we're supposed + * to send back. + */ + if (s->session->not_resumable || + (!(s->ctx->session_cache_mode & SSL_SESS_CACHE_SERVER) + && !s->hit)) + s->session->session_id_length = 0; + + if (usetls13) { + sl = s->tmp_session_id_len; + session_id = s->tmp_session_id; + } else { + sl = s->session->session_id_length; + session_id = s->session->session_id; + } + + if (sl > sizeof(s->session->session_id)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* set up the compression method */ + compm = 0; + + if (!WPACKET_sub_memcpy_u8(pkt, session_id, sl) + || !s->method->put_cipher_by_char(s->s3.tmp.new_cipher, pkt, &len) + || !WPACKET_put_bytes_u8(pkt, compm)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!tls_construct_extensions_ntls(s, pkt, + s->hello_retry_request == SSL_HRR_PENDING + ? SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST + : SSL_EXT_TLS1_2_SERVER_HELLO, + NULL, 0)) { + /* SSLfatal() already called */ + return 0; + } + + if (s->hello_retry_request == SSL_HRR_PENDING) { + /* Ditch the session. We'll create a new one next time around */ + SSL_SESSION_free(s->session); + s->session = NULL; + s->hit = 0; + + /* + * Re-initialise the Transcript Hash. We're going to prepopulate it with + * a synthetic message_hash in place of ClientHello1. + */ + if (!create_synthetic_message_hash_ntls(s, NULL, 0, NULL, 0)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } else if (!(s->verify_mode & SSL_VERIFY_PEER) + && !ssl3_digest_cached_records(s, 0)) { + /* SSLfatal_ntls() already called */; + return 0; + } + + return 1; +} + +int tls_construct_server_done_ntls(SSL *s, WPACKET *pkt) +{ + if (!s->s3.tmp.cert_request) { + if (!ssl3_digest_cached_records(s, 0)) { + /* SSLfatal_ntls() already called */ + return 0; + } + } + return 1; +} + +int tls_construct_server_key_exchange_ntls(SSL *s, WPACKET *pkt) +{ + unsigned char *encodedPoint = NULL; + size_t encodedlen = 0; + int curve_id = 0; + const SIGALG_LOOKUP *lu = s->s3.tmp.sigalg; + unsigned long type; + EVP_MD_CTX *md_ctx = EVP_MD_CTX_new(); + EVP_PKEY_CTX *pctx = NULL; + size_t paramlen = 0, paramoffset; + int ret = 0; + + if (!WPACKET_get_total_written(pkt, ¶moffset)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + type = s->s3.tmp.new_cipher->algorithm_mkey; + + if (type & SSL_kSM2DHE) { + if (s->s3.tmp.pkey != NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* Get NID of appropriate shared curve */ + curve_id = tls1_shared_group(s, -2); + if (curve_id == 0) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_UNSUPPORTED_ELLIPTIC_CURVE); + goto err; + } + /* Cache the group used in the SSL_SESSION */ + s->session->kex_group = curve_id; + /* Generate a new key for this curve */ + s->s3.tmp.pkey = ssl_generate_pkey_group(s, curve_id); + if (s->s3.tmp.pkey == NULL) { + /* SSLfatal() already called */ + goto err; + } + + /* Encode the public key. */ + encodedlen = EVP_PKEY_get1_encoded_public_key(s->s3.tmp.pkey, + &encodedPoint); + if (encodedlen == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + /* + * We only support named (not generic) curves. In this situation, the + * ServerKeyExchange message has: [1 byte CurveType], [2 byte CurveName] + * [1 byte length of encoded point], followed by the actual encoded + * point itself + */ + if (!WPACKET_put_bytes_u8(pkt, NAMED_CURVE_TYPE) + || !WPACKET_put_bytes_u8(pkt, 0) + || !WPACKET_put_bytes_u8(pkt, curve_id) + || !WPACKET_sub_memcpy_u8(pkt, encodedPoint, encodedlen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + +#ifndef OPENSSL_NO_STATUS + /* record curve_id and pubkey */ + if (s->status_param.ssl_status_enable) { + s->status_param.type = SSL_SERVER_EXCHANGE_PUBKEY; + if (s->status_callback(WPACKET_get_curr(pkt) - encodedlen, + encodedlen, &s->status_param) == -1) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + } +#endif + + OPENSSL_free(encodedPoint); + encodedPoint = NULL; + + /* Get length of the parameters we have written above */ + if (!WPACKET_get_length(pkt, ¶mlen)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } + + if (lu != NULL) { + EVP_PKEY *pkey = NULL; + X509 *x509 = NULL; + const EVP_MD *md; + unsigned char *sigbytes1, *sigbytes2, *tbs; + size_t siglen = 0, tbslen; + unsigned char *buf = NULL; + size_t buflen = 0; + + /* get signing cert and pkey */ + if (type & SSL_kRSA) { + pkey = s->cert->pkeys[SSL_PKEY_RSA_SIGN].privatekey; + x509 = s->cert->pkeys[SSL_PKEY_RSA_ENC].x509; + } else if (type & SSL_kSM2) { + pkey = s->cert->pkeys[SSL_PKEY_SM2_SIGN].privatekey; + x509 = s->cert->pkeys[SSL_PKEY_SM2_ENC].x509; + } else if (type & SSL_kSM2DHE) { + pkey = s->cert->pkeys[SSL_PKEY_SM2_SIGN].privatekey; + } + + if (pkey == NULL || !tls1_lookup_md(s->ctx, lu, &md)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (EVP_DigestSignInit_ex(md_ctx, &pctx, + md == NULL ? NULL : EVP_MD_get0_name(md), + s->ctx->libctx, s->ctx->propq, pkey, + NULL) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + if (EVP_PKEY_is_a(pkey, "SM2")) { + if (EVP_PKEY_CTX_set1_id(pctx, SM2_DEFAULT_ID, + SM2_DEFAULT_ID_LEN) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; + } + } + if (type & (SSL_kRSA | SSL_kSM2)) { + if (x509 == NULL + || (buf = x509_to_asn1_ntls(x509, &buflen)) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + } + + tbslen = construct_key_exchange_tbs_ntls(s, &tbs, + (buf != NULL) + ? (const void *)buf + : (const void *)(s->init_buf->data + paramoffset), + (buf != NULL) ? buflen : paramlen); + if (tbslen == 0) { + /* SSLfatal_ntls() already called */ + OPENSSL_free(buf); + goto err; + } + + OPENSSL_free(buf); + + if (EVP_DigestSign(md_ctx, NULL, &siglen, tbs, tbslen) <=0 + || !WPACKET_sub_reserve_bytes_u16(pkt, siglen, &sigbytes1) + || EVP_DigestSign(md_ctx, sigbytes1, &siglen, tbs, tbslen) <= 0 + || !WPACKET_sub_allocate_bytes_u16(pkt, siglen, &sigbytes2) + || sigbytes1 != sigbytes2) { + OPENSSL_free(tbs); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + OPENSSL_free(tbs); + } + + ret = 1; +err: + OPENSSL_free(encodedPoint); + EVP_MD_CTX_free(md_ctx); + return ret; +} + +int tls_construct_certificate_request_ntls(SSL *s, WPACKET *pkt) +{ + /* get the list of acceptable cert types */ + if (!WPACKET_start_sub_packet_u8(pkt) + || !ssl3_get_req_cert_type(s, pkt) || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (SSL_USE_SIGALGS(s)) { + const uint16_t *psigs; + size_t nl = tls12_get_psigalgs(s, 1, &psigs); + + if (!WPACKET_start_sub_packet_u16(pkt) + || !WPACKET_set_flags(pkt, WPACKET_FLAGS_NON_ZERO_LENGTH) + || !tls12_copy_sigalgs(s, pkt, psigs, nl) + || !WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + } + + if (!construct_ca_names_ntls(s, get_ca_names_ntls(s), pkt)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + s->certreqs_sent++; + s->s3.tmp.cert_request = 1; + return 1; +} + +/* process ClientKeyExchange encrypted pre master secret for kRSA or kSM2 */ +static int tls_process_cke_pms_ntls(SSL *s, PACKET *pkt, unsigned long alg_k) +{ + size_t outlen; + PACKET enc_premaster; + EVP_PKEY *pkey = NULL; + unsigned char *pkey_decrypt = NULL; + int ret = 0; + EVP_PKEY_CTX *ctx = NULL; + OSSL_PARAM params[2], *p = params; + + if (alg_k & SSL_kRSA) + pkey = s->cert->pkeys[SSL_PKEY_RSA_ENC].privatekey; + else if (alg_k & SSL_kSM2) + pkey = s->cert->pkeys[SSL_PKEY_SM2_ENC].privatekey; + + if (pkey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_MISSING_ENC_CERTIFICATE); + return 0; + } + + if (!PACKET_get_length_prefixed_2(pkt, &enc_premaster) + || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return 0; + } + +# ifndef OPENSSL_NO_STATUS + /* record encrypted client pms with RSA/SM2 KeyExchange mode */ + if (s->status_param.ssl_status_enable) { + s->status_param.type = (alg_k & SSL_kRSA) ? SSL_CLIENT_RSA_EXCHANGE : + SSL_CLIENT_SM2_EXCHANGE; + if (s->status_callback((unsigned char *)enc_premaster.curr, + (size_t)enc_premaster.remaining, + &s->status_param) == -1) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + return 0; + } + } +# endif + + outlen = SSL_MAX_MASTER_KEY_LENGTH; + pkey_decrypt = OPENSSL_malloc(outlen); + if (pkey_decrypt == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + return 0; + } + + ctx = EVP_PKEY_CTX_new_from_pkey(s->ctx->libctx, pkey, s->ctx->propq); + if (ctx == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (EVP_PKEY_decrypt_init(ctx) <= 0) { + SSLfatal_ntls(s, SSL_AD_DECRYPT_ERROR, SSL_R_DECRYPTION_FAILED); + goto err; + } + + /* + * We must not leak whether a decryption failure occurs because of + * Bleichenbacher's attack on PKCS #1 v1.5 RSA padding (see RFC 2246, + * section 7.4.7.1). We use the special padding type + * RSA_PKCS1_WITH_TLS_PADDING to do that. It will automaticaly decrypt the + * RSA, check the padding and check that the client version is as expected + * in the premaster secret. If any of that fails then the function appears + * to return successfully but with a random result. The call below could + * still fail if the input is publicly invalid. + * See https://tools.ietf.org/html/rfc5246#section-7.4.7.1 + */ + if (alg_k & SSL_kRSA) { + if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_WITH_TLS_PADDING) + <= 0) { + SSLfatal_ntls(s, SSL_AD_DECRYPT_ERROR, SSL_R_DECRYPTION_FAILED); + goto err; + } + + *p++ = OSSL_PARAM_construct_uint( + OSSL_ASYM_CIPHER_PARAM_TLS_CLIENT_VERSION, + (unsigned int *)&s->client_version); + *p++ = OSSL_PARAM_construct_end(); + + if (!EVP_PKEY_CTX_set_params(ctx, params)) { + SSLfatal_ntls(s, SSL_AD_DECRYPT_ERROR, SSL_R_DECRYPTION_FAILED); + goto err; + } + } + + if (EVP_PKEY_decrypt(ctx, pkey_decrypt, &outlen, + PACKET_data(&enc_premaster), + PACKET_remaining(&enc_premaster)) <= 0) { + SSLfatal_ntls(s, SSL_AD_DECRYPT_ERROR, SSL_R_DECRYPTION_FAILED); + goto err; + } + + /* + * This test should never fail (otherwise we should have failed above) but + * we double check anyway. + */ + if (outlen != SSL_MAX_MASTER_KEY_LENGTH) { + OPENSSL_cleanse(pkey_decrypt, SSL_MAX_MASTER_KEY_LENGTH); + SSLfatal_ntls(s, SSL_AD_DECRYPT_ERROR, SSL_R_DECRYPTION_FAILED); + goto err; + } + + /* Also cleanses pkey_decrypt (on success or failure) */ + if (!ssl_generate_master_secret(s, pkey_decrypt, + SSL_MAX_MASTER_KEY_LENGTH, 0)) { + /* SSLfatal_ntls already called */ + goto err; + } + + ret = 1; + err: + OPENSSL_free(pkey_decrypt); + EVP_PKEY_CTX_free(ctx); + return ret; +} + +static int tls_process_cke_sm2dhe_ntls(SSL *s, PACKET *pkt) +{ + const unsigned char *ecparams; + EVP_PKEY *skey = s->s3.tmp.pkey; + EVP_PKEY *ckey = NULL; + int ret = 0; + + if (PACKET_remaining(pkt) == 0L) { + /* We don't support ECDH client auth */ + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_MISSING_TMP_ECDH_KEY); + goto err; + } else { + unsigned int i; + const unsigned char *data; + + /* + * Get client's public key from encoded point in the + * ClientKeyExchange message. + */ + + if (!PACKET_get_bytes(pkt, &ecparams, 3)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_TOO_SHORT); + goto err; + } + + /* Get encoded point length */ + if (!PACKET_get_1(pkt, &i) || !PACKET_get_bytes(pkt, &data, i) + || PACKET_remaining(pkt) != 0) { + SSLfatal(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + if (skey == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_MISSING_TMP_ECDH_KEY); + goto err; + } + + ckey = EVP_PKEY_new(); + if (ckey == NULL || EVP_PKEY_copy_parameters(ckey, skey) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, + SSL_R_COPY_PARAMETERS_FAILED); + goto err; + } + + if (EVP_PKEY_set1_encoded_public_key(ckey, data, i) <= 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_EC_LIB); + goto err; + } + +#ifndef OPENSSL_NO_STATUS + /* record client DH pubkey */ + if (s->status_param.ssl_status_enable) { + BIGNUM *pub_key = BN_bin2bn(data, i, NULL); + s->status_param.type = SSL_SERVER_DH_PUBKEY; + s->status_param.parg = pub_key; + if (s->status_callback((unsigned char *)data, + BN_num_bytes(pub_key), + &s->status_param) == -1) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_STATUS_CALLBACK_ERROR); + goto err; + } + } +#endif + } + + if (ssl_derive_ntls(s, skey, ckey, 1) == 0) { + /* SSLfatal_ntls() already called */ + goto err; + } + + ret = 1; + EVP_PKEY_free(s->s3.tmp.pkey); + s->s3.tmp.pkey = NULL; + err: + EVP_PKEY_free(ckey); + + return ret; +} + +MSG_PROCESS_RETURN tls_process_client_key_exchange_ntls(SSL *s, PACKET *pkt) +{ + unsigned long alg_k; + + alg_k = s->s3.tmp.new_cipher->algorithm_mkey; + + if (alg_k & (SSL_kRSA | SSL_kSM2)) { + if (!tls_process_cke_pms_ntls(s, pkt, alg_k)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } else if (alg_k & SSL_kSM2DHE) { + if (!tls_process_cke_sm2dhe_ntls(s, pkt)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } else { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_UNKNOWN_CIPHER_TYPE); + goto err; + } + + return MSG_PROCESS_CONTINUE_PROCESSING; + err: + return MSG_PROCESS_ERROR; +} + +WORK_STATE tls_post_process_client_key_exchange_ntls(SSL *s, WORK_STATE wst) +{ + if (s->statem.no_cert_verify || !s->session->peer) { + /* + * No certificate verify or no peer certificate so we no longer need + * the handshake_buffer + */ + if (!ssl3_digest_cached_records(s, 0)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + return WORK_FINISHED_CONTINUE; + } else { + if (!s->s3.handshake_buffer) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return WORK_ERROR; + } + /* + * For sigalgs freeze the handshake buffer. If we support + * extms we've done this already so this is a no-op + */ + if (!ssl3_digest_cached_records(s, 1)) { + /* SSLfatal_ntls() already called */ + return WORK_ERROR; + } + } + + return WORK_FINISHED_CONTINUE; +} + +MSG_PROCESS_RETURN tls_process_client_certificate_ntls(SSL *s, PACKET *pkt) +{ + int i, j; + MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR; + X509 *x = NULL; + unsigned long l; + const unsigned char *certstart, *certbytes; + STACK_OF(X509) *sk = NULL; + PACKET spkt; + SSL_SESSION *new_sess = NULL; + + /* + * To get this far we must have read encrypted data from the client. We no + * longer tolerate unencrypted alerts. This value is ignored if less than + * TLSv1.3 + */ + s->statem.enc_read_state = ENC_READ_STATE_VALID; + + if ((sk = sk_X509_new_null()) == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + if (!PACKET_get_length_prefixed_3(pkt, &spkt) + || PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + goto err; + } + + while (PACKET_remaining(&spkt) > 0) { + if (!PACKET_get_net_3(&spkt, &l) + || !PACKET_get_bytes(&spkt, &certbytes, l)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_CERT_LENGTH_MISMATCH); + goto err; + } + + certstart = certbytes; + x = X509_new_ex(s->ctx->libctx, s->ctx->propq); + if (x == NULL) { + SSLfatal(s, SSL_AD_DECODE_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + if (d2i_X509(&x, (const unsigned char **)&certbytes, l) == NULL) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, ERR_R_ASN1_LIB); + goto err; + } + + if (certbytes != (certstart + l)) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_CERT_LENGTH_MISMATCH); + goto err; + } + + if (!sk_X509_push(sk, x)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + x = NULL; + } + + if (sk_X509_num(sk) <= 0) { + /*for ECDHE-SM2, certificates are required */ + unsigned long alg_k = s->s3.tmp.new_cipher->algorithm_mkey; + if (alg_k & SSL_kSM2DHE) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_NO_CERTIFICATES_RETURNED); + goto err; + } + + /* Fail for TLS only if we required a certificate */ + else if ((s->verify_mode & SSL_VERIFY_PEER) && + (s->verify_mode & SSL_VERIFY_FAIL_IF_NO_PEER_CERT)) { + SSLfatal_ntls(s, SSL_AD_CERTIFICATE_REQUIRED, + SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE); + goto err; + } + /* No client certificate so digest cached records */ + if (s->s3.handshake_buffer && !ssl3_digest_cached_records(s, 0)) { + /* SSLfatal_ntls() already called */ + goto err; + } + } else if (sk_X509_num(sk) < 2) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_NO_CERTIFICATES_RETURNED); + goto err; + } else { + for (j = 0; j < 2; j++) { + EVP_PKEY *pkey; + + if (j == 0) + sk_X509_push(sk, sk_X509_shift(sk)); + if (j == 1) + sk_X509_unshift(sk, sk_X509_pop(sk)); + + i = ssl_verify_cert_chain(s, sk); + + if (i <= 0) { + SSLfatal_ntls(s, ssl_x509err2alert_ntls(s->verify_result), + SSL_R_CERTIFICATE_VERIFY_FAILED); + goto err; + } + pkey = X509_get0_pubkey(sk_X509_value(sk, 0)); + if (pkey == NULL) { + SSLfatal_ntls(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_UNKNOWN_CERTIFICATE_TYPE); + goto err; + } + } + } + + /* + * Sessions must be immutable once they go into the session cache. Otherwise + * we can get multi-thread problems. Therefore we don't "update" sessions, + * we replace them with a duplicate. Here, we need to do this every time + * a new certificate is received via post-handshake authentication, as the + * session may have already gone into the session cache. + */ + + if (s->post_handshake_auth == SSL_PHA_REQUESTED) { + if ((new_sess = ssl_session_dup(s->session, 0)) == 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + SSL_SESSION_free(s->session); + s->session = new_sess; + } + + X509_free(s->session->peer); + /* For NTLS server, s->session->peer stores client sign cert */ + s->session->peer = sk_X509_shift(sk); + s->session->verify_result = s->verify_result; + + sk_X509_pop_free(s->session->peer_chain, X509_free); + /* For NTLS server, s->session->peer_chain stores client encryption cert and extra chain certs */ + s->session->peer_chain = sk; + sk = NULL; + + ret = MSG_PROCESS_CONTINUE_READING; + + err: + X509_free(x); + sk_X509_pop_free(sk, X509_free); + return ret; +} + +int tls_construct_server_certificate_ntls(SSL *s, WPACKET *pkt) +{ + CERT_PKEY *a_cpk = s->s3.tmp.sign_cert; + CERT_PKEY *k_cpk = s->s3.tmp.enc_cert; + + if (a_cpk == NULL || k_cpk == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + if (!ssl3_output_cert_chain_ntls(s, pkt, a_cpk, k_cpk)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + return 1; +} + +static int create_ticket_prequel(SSL *s, WPACKET *pkt, uint32_t age_add, + unsigned char *tick_nonce) +{ + /* + * Ticket lifetime hint: For TLSv1.2 this is advisory only and we leave this + * unspecified for resumed session (for simplicity). + * In TLSv1.3 we reset the "time" field above, and always specify the + * timeout. + */ + if (!WPACKET_put_bytes_u32(pkt, + s->hit ? 0 : (uint32_t)s->session->timeout)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + /* Start the sub-packet for the actual ticket data */ + if (!WPACKET_start_sub_packet_u16(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +static int construct_stateless_ticket(SSL *s, WPACKET *pkt, uint32_t age_add, + unsigned char *tick_nonce) +{ + unsigned char *senc = NULL; + EVP_CIPHER_CTX *ctx = NULL; + SSL_HMAC *hctx = NULL; + unsigned char *p, *encdata1, *encdata2, *macdata1, *macdata2; + const unsigned char *const_p; + int len, slen_full, slen, lenfinal; + SSL_SESSION *sess; + size_t hlen; + SSL_CTX *tctx = s->session_ctx; + unsigned char iv[EVP_MAX_IV_LENGTH]; + unsigned char key_name[TLSEXT_KEYNAME_LENGTH]; + int iv_len, ok = 0; + size_t macoffset, macendoffset; + + /* get session encoding length */ + slen_full = i2d_SSL_SESSION(s->session, NULL); + /* + * Some length values are 16 bits, so forget it if session is too + * long + */ + if (slen_full == 0 || slen_full > 0xFF00) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + senc = OPENSSL_malloc(slen_full); + if (senc == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + ctx = EVP_CIPHER_CTX_new(); + hctx = ssl_hmac_new(tctx); + if (ctx == NULL || hctx == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + + p = senc; + if (!i2d_SSL_SESSION(s->session, &p)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* + * create a fresh copy (not shared with other threads) to clean up + */ + const_p = senc; + sess = d2i_SSL_SESSION(NULL, &const_p, slen_full); + if (sess == NULL) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + slen = i2d_SSL_SESSION(sess, NULL); + if (slen == 0 || slen > slen_full) { + /* shouldn't ever happen */ + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSL_SESSION_free(sess); + goto err; + } + p = senc; + if (!i2d_SSL_SESSION(sess, &p)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + SSL_SESSION_free(sess); + goto err; + } + SSL_SESSION_free(sess); + + /* + * Initialize HMAC and cipher contexts. If callback present it does + * all the work otherwise use generated values from parent ctx. + */ +#ifndef OPENSSL_NO_DEPRECATED_3_0 + if (tctx->ext.ticket_key_evp_cb != NULL || tctx->ext.ticket_key_cb != NULL) +#else + if (tctx->ext.ticket_key_evp_cb != NULL) +#endif + { + int ret = 0; + + if (tctx->ext.ticket_key_evp_cb != NULL) + ret = tctx->ext.ticket_key_evp_cb(s, key_name, iv, ctx, + ssl_hmac_get0_EVP_MAC_CTX(hctx), + 1); +#ifndef OPENSSL_NO_DEPRECATED_3_0 + else if (tctx->ext.ticket_key_cb != NULL) + /* if 0 is returned, write an empty ticket */ + ret = tctx->ext.ticket_key_cb(s, key_name, iv, ctx, + ssl_hmac_get0_HMAC_CTX(hctx), 1); +#endif + + if (ret == 0) { + + /* Put timeout and length */ + if (!WPACKET_put_bytes_u32(pkt, 0) + || !WPACKET_put_bytes_u16(pkt, 0)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + OPENSSL_free(senc); + EVP_CIPHER_CTX_free(ctx); + ssl_hmac_free(hctx); + return 1; + } + if (ret < 0) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, SSL_R_CALLBACK_FAILED); + goto err; + } + iv_len = EVP_CIPHER_CTX_get_iv_length(ctx); + } else { + EVP_CIPHER *cipher = EVP_CIPHER_fetch(s->ctx->libctx, "AES-256-CBC", + s->ctx->propq); + + if (cipher == NULL) { + /* Error is already recorded */ + SSLfatal_alert_ntls(s, SSL_AD_INTERNAL_ERROR); + goto err; + } + + iv_len = EVP_CIPHER_get_iv_length(cipher); + if (iv_len < 0 + || RAND_bytes_ex(s->ctx->libctx, iv, iv_len, 0) <= 0 + || !EVP_EncryptInit_ex(ctx, cipher, NULL, + tctx->ext.secure->tick_aes_key, iv) + || !ssl_hmac_init(hctx, tctx->ext.secure->tick_hmac_key, + sizeof(tctx->ext.secure->tick_hmac_key), + "SHA256")) { + EVP_CIPHER_free(cipher); + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + EVP_CIPHER_free(cipher); + memcpy(key_name, tctx->ext.tick_key_name, + sizeof(tctx->ext.tick_key_name)); + } + + if (!create_ticket_prequel(s, pkt, age_add, tick_nonce)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + if (!WPACKET_get_total_written(pkt, &macoffset) + /* Output key name */ + || !WPACKET_memcpy(pkt, key_name, sizeof(key_name)) + /* output IV */ + || !WPACKET_memcpy(pkt, iv, iv_len) + || !WPACKET_reserve_bytes(pkt, slen + EVP_MAX_BLOCK_LENGTH, + &encdata1) + /* Encrypt session data */ + || !EVP_EncryptUpdate(ctx, encdata1, &len, senc, slen) + || !WPACKET_allocate_bytes(pkt, len, &encdata2) + || encdata1 != encdata2 + || !EVP_EncryptFinal(ctx, encdata1 + len, &lenfinal) + || !WPACKET_allocate_bytes(pkt, lenfinal, &encdata2) + || encdata1 + len != encdata2 + || len + lenfinal > slen + EVP_MAX_BLOCK_LENGTH + || !WPACKET_get_total_written(pkt, &macendoffset) + || !ssl_hmac_update(hctx, + (unsigned char *)s->init_buf->data + macoffset, + macendoffset - macoffset) + || !WPACKET_reserve_bytes(pkt, EVP_MAX_MD_SIZE, &macdata1) + || !ssl_hmac_final(hctx, macdata1, &hlen, EVP_MAX_MD_SIZE) + || hlen > EVP_MAX_MD_SIZE + || !WPACKET_allocate_bytes(pkt, hlen, &macdata2) + || macdata1 != macdata2) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + /* Close the sub-packet created by create_ticket_prequel() */ + if (!WPACKET_close(pkt)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + + ok = 1; + err: + OPENSSL_free(senc); + EVP_CIPHER_CTX_free(ctx); + ssl_hmac_free(hctx); + return ok; +} + +int tls_construct_new_session_ticket_ntls(SSL *s, WPACKET *pkt) +{ + SSL_CTX *tctx = s->session_ctx; + unsigned char tick_nonce[TICKET_NONCE_SIZE]; + union { + unsigned char age_add_c[sizeof(uint32_t)]; + uint32_t age_add; + } age_add_u; + + age_add_u.age_add = 0; + + if (tctx->generate_ticket_cb != NULL && + tctx->generate_ticket_cb(s, tctx->ticket_cb_data) == 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + if (!construct_stateless_ticket(s, pkt, age_add_u.age_add, + tick_nonce)) { + /* SSLfatal_ntls() already called */ + goto err; + } + + return 1; + err: + return 0; +} + +/* + * In TLSv1.3 this is called from the extensions code, otherwise it is used to + * create a separate message. Returns 1 on success or 0 on failure. + */ +int tls_construct_cert_status_body_ntls(SSL *s, WPACKET *pkt) +{ + if (!WPACKET_put_bytes_u8(pkt, s->ext.status_type) + || !WPACKET_sub_memcpy_u24(pkt, s->ext.ocsp.resp, + s->ext.ocsp.resp_len)) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + +int tls_construct_cert_status_ntls(SSL *s, WPACKET *pkt) +{ + if (!tls_construct_cert_status_body_ntls(s, pkt)) { + /* SSLfatal_ntls() already called */ + return 0; + } + + return 1; +} + +#ifndef OPENSSL_NO_NEXTPROTONEG +/* + * tls_process_next_proto_ntls reads a Next Protocol Negotiation handshake message. + * It sets the next_proto member in s if found + */ +MSG_PROCESS_RETURN tls_process_next_proto_ntls(SSL *s, PACKET *pkt) +{ + PACKET next_proto, padding; + size_t next_proto_len; + + /*- + * The payload looks like: + * uint8 proto_len; + * uint8 proto[proto_len]; + * uint8 padding_len; + * uint8 padding[padding_len]; + */ + if (!PACKET_get_length_prefixed_1(pkt, &next_proto) + || !PACKET_get_length_prefixed_1(pkt, &padding) + || PACKET_remaining(pkt) > 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return MSG_PROCESS_ERROR; + } + + if (!PACKET_memdup(&next_proto, &s->ext.npn, &next_proto_len)) { + s->ext.npn_len = 0; + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return MSG_PROCESS_ERROR; + } + + s->ext.npn_len = (unsigned char)next_proto_len; + + return MSG_PROCESS_CONTINUE_READING; +} +#endif + +MSG_PROCESS_RETURN tls_process_end_of_early_data_ntls(SSL *s, PACKET *pkt) +{ + if (PACKET_remaining(pkt) != 0) { + SSLfatal_ntls(s, SSL_AD_DECODE_ERROR, SSL_R_LENGTH_MISMATCH); + return MSG_PROCESS_ERROR; + } + + if (s->early_data_state != SSL_EARLY_DATA_READING + && s->early_data_state != SSL_EARLY_DATA_READ_RETRY) { + SSLfatal_ntls(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return MSG_PROCESS_ERROR; + } + + /* + * EndOfEarlyData signals a key change so the end of the message must be on + * a record boundary. + */ + if (RECORD_LAYER_processed_read_pending(&s->rlayer)) { + SSLfatal_ntls(s, SSL_AD_UNEXPECTED_MESSAGE, SSL_R_NOT_ON_RECORD_BOUNDARY); + return MSG_PROCESS_ERROR; + } + + s->early_data_state = SSL_EARLY_DATA_FINISHED_READING; + if (!s->method->ssl3_enc->change_cipher_state(s, + SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_SERVER_READ)) { + /* SSLfatal_ntls() already called */ + return MSG_PROCESS_ERROR; + } + + return MSG_PROCESS_CONTINUE_READING; +} + diff --git a/openssl/src/ssl/t1_enc.c b/openssl/src/ssl/t1_enc.c index 2e9e24a8c..2b449e1bc 100644 --- a/openssl/src/ssl/t1_enc.c +++ b/openssl/src/ssl/t1_enc.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright 2005 Nokia. All rights reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use @@ -22,7 +22,7 @@ #include /* seed1 through seed5 are concatenated */ -static int tls1_PRF(SSL_CONNECTION *s, +static int tls1_PRF(SSL *s, const void *seed1, size_t seed1_len, const void *seed2, size_t seed2_len, const void *seed3, size_t seed3_len, @@ -45,9 +45,7 @@ static int tls1_PRF(SSL_CONNECTION *s, ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); return 0; } - kdf = EVP_KDF_fetch(SSL_CONNECTION_GET_CTX(s)->libctx, - OSSL_KDF_NAME_TLS1_PRF, - SSL_CONNECTION_GET_CTX(s)->propq); + kdf = EVP_KDF_fetch(s->ctx->libctx, OSSL_KDF_NAME_TLS1_PRF, s->ctx->propq); if (kdf == NULL) goto err; kctx = EVP_KDF_CTX_new(kdf); @@ -85,8 +83,7 @@ static int tls1_PRF(SSL_CONNECTION *s, return 0; } -static int tls1_generate_key_block(SSL_CONNECTION *s, unsigned char *km, - size_t num) +static int tls1_generate_key_block(SSL *s, unsigned char *km, size_t num) { int ret; @@ -101,6 +98,80 @@ static int tls1_generate_key_block(SSL_CONNECTION *s, unsigned char *km, return ret; } +#ifndef OPENSSL_NO_KTLS + /* + * Count the number of records that were not processed yet from record boundary. + * + * This function assumes that there are only fully formed records read in the + * record layer. If read_ahead is enabled, then this might be false and this + * function will fail. + */ +# ifndef OPENSSL_NO_KTLS_RX +static int count_unprocessed_records(SSL *s) +{ + SSL3_BUFFER *rbuf = RECORD_LAYER_get_rbuf(&s->rlayer); + PACKET pkt, subpkt; + int count = 0; + + if (!PACKET_buf_init(&pkt, rbuf->buf + rbuf->offset, rbuf->left)) + return -1; + + while (PACKET_remaining(&pkt) > 0) { + /* Skip record type and version */ + if (!PACKET_forward(&pkt, 3)) + return -1; + + /* Read until next record */ + if (!PACKET_get_length_prefixed_2(&pkt, &subpkt)) + return -1; + + count += 1; + } + + return count; +} +# endif +#endif + + +int tls_provider_set_tls_params(SSL *s, EVP_CIPHER_CTX *ctx, + const EVP_CIPHER *ciph, + const EVP_MD *md) +{ + /* + * Provided cipher, the TLS padding/MAC removal is performed provider + * side so we need to tell the ctx about our TLS version and mac size + */ + OSSL_PARAM params[3], *pprm = params; + size_t macsize = 0; + int imacsize = -1; + + if ((EVP_CIPHER_get_flags(ciph) & EVP_CIPH_FLAG_AEAD_CIPHER) == 0 + /* + * We look at s->ext.use_etm instead of SSL_READ_ETM() or + * SSL_WRITE_ETM() because this test applies to both reading + * and writing. + */ + && !s->ext.use_etm) + imacsize = EVP_MD_get_size(md); + if (imacsize >= 0) + macsize = (size_t)imacsize; + + *pprm++ = OSSL_PARAM_construct_int(OSSL_CIPHER_PARAM_TLS_VERSION, + &s->version); + *pprm++ = OSSL_PARAM_construct_size_t(OSSL_CIPHER_PARAM_TLS_MAC_SIZE, + &macsize); + *pprm = OSSL_PARAM_construct_end(); + + if (!EVP_CIPHER_CTX_set_params(ctx, params)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return 0; + } + + return 1; +} + + static int tls_iv_length_within_key_block(const EVP_CIPHER *c) { /* If GCM/CCM mode only part of IV comes from PRF */ @@ -112,45 +183,141 @@ static int tls_iv_length_within_key_block(const EVP_CIPHER *c) return EVP_CIPHER_get_iv_length(c); } -int tls1_change_cipher_state(SSL_CONNECTION *s, int which) +int tls1_change_cipher_state(SSL *s, int which) { unsigned char *p, *mac_secret; - unsigned char *key, *iv; + unsigned char *ms, *key, *iv; + EVP_CIPHER_CTX *dd; const EVP_CIPHER *c; - const SSL_COMP *comp = NULL; +#ifndef OPENSSL_NO_COMP + const SSL_COMP *comp; +#endif const EVP_MD *m; - int mac_type; - size_t mac_secret_size; + size_t *mac_secret_size; + EVP_MD_CTX *mac_ctx; + EVP_PKEY *mac_key; size_t n, i, j, k, cl; - int iivlen; - /* - * Taglen is only relevant for CCM ciphersuites. Other ciphersuites - * ignore this value so we can default it to 0. - */ - size_t taglen = 0; - int direction; + int reuse_dd = 0; +#ifndef OPENSSL_NO_KTLS + ktls_crypto_info_t crypto_info; + unsigned char *rec_seq; + void *rl_sequence; +# ifndef OPENSSL_NO_KTLS_RX + int count_unprocessed; + int bit; +# endif + BIO *bio; +#endif c = s->s3.tmp.new_sym_enc; m = s->s3.tmp.new_hash; - mac_type = s->s3.tmp.new_mac_pkey_type; #ifndef OPENSSL_NO_COMP comp = s->s3.tmp.new_compression; #endif + if (which & SSL3_CC_READ) { + if (s->ext.use_etm) + s->s3.flags |= TLS1_FLAGS_ENCRYPT_THEN_MAC_READ; + else + s->s3.flags &= ~TLS1_FLAGS_ENCRYPT_THEN_MAC_READ; + + if (s->enc_read_ctx != NULL) { + reuse_dd = 1; + } else if ((s->enc_read_ctx = EVP_CIPHER_CTX_new()) == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } else { + /* + * make sure it's initialised in case we exit later with an error + */ + EVP_CIPHER_CTX_reset(s->enc_read_ctx); + } + dd = s->enc_read_ctx; + mac_ctx = ssl_replace_hash(&s->read_hash, NULL); + if (mac_ctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } +#ifndef OPENSSL_NO_COMP + COMP_CTX_free(s->expand); + s->expand = NULL; + if (comp != NULL) { + s->expand = COMP_CTX_new(comp->method); + if (s->expand == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, + SSL_R_COMPRESSION_LIBRARY_ERROR); + goto err; + } + } +#endif + /* + * this is done by dtls1_reset_seq_numbers for DTLS + */ + if (!SSL_IS_DTLS(s)) + RECORD_LAYER_reset_read_sequence(&s->rlayer); + mac_secret = &(s->s3.read_mac_secret[0]); + mac_secret_size = &(s->s3.read_mac_secret_size); + } else { + s->statem.enc_write_state = ENC_WRITE_STATE_INVALID; + if (s->ext.use_etm) + s->s3.flags |= TLS1_FLAGS_ENCRYPT_THEN_MAC_WRITE; + else + s->s3.flags &= ~TLS1_FLAGS_ENCRYPT_THEN_MAC_WRITE; + + if (s->enc_write_ctx != NULL && !SSL_IS_DTLS(s)) { + reuse_dd = 1; + } else if ((s->enc_write_ctx = EVP_CIPHER_CTX_new()) == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + dd = s->enc_write_ctx; + if (SSL_IS_DTLS(s)) { + mac_ctx = EVP_MD_CTX_new(); + if (mac_ctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + s->write_hash = mac_ctx; + } else { + mac_ctx = ssl_replace_hash(&s->write_hash, NULL); + if (mac_ctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + } +#ifndef OPENSSL_NO_COMP + COMP_CTX_free(s->compress); + s->compress = NULL; + if (comp != NULL) { + s->compress = COMP_CTX_new(comp->method); + if (s->compress == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, + SSL_R_COMPRESSION_LIBRARY_ERROR); + goto err; + } + } +#endif + /* + * this is done by dtls1_reset_seq_numbers for DTLS + */ + if (!SSL_IS_DTLS(s)) + RECORD_LAYER_reset_write_sequence(&s->rlayer); + mac_secret = &(s->s3.write_mac_secret[0]); + mac_secret_size = &(s->s3.write_mac_secret_size); + } + + if (reuse_dd) + EVP_CIPHER_CTX_reset(dd); + p = s->s3.tmp.key_block; - i = mac_secret_size = s->s3.tmp.new_mac_secret_size; + i = *mac_secret_size = s->s3.tmp.new_mac_secret_size; cl = EVP_CIPHER_get_key_length(c); j = cl; - iivlen = tls_iv_length_within_key_block(c); - if (iivlen < 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - goto err; - } - k = iivlen; + k = tls_iv_length_within_key_block(c); if ((which == SSL3_CHANGE_CIPHER_CLIENT_WRITE) || (which == SSL3_CHANGE_CIPHER_SERVER_READ)) { - mac_secret = &(p[0]); + ms = &(p[0]); n = i + i; key = &(p[n]); n += j + j; @@ -158,7 +325,7 @@ int tls1_change_cipher_state(SSL_CONNECTION *s, int which) n += k + k; } else { n = i; - mac_secret = &(p[n]); + ms = &(p[n]); n += i + j; key = &(p[n]); n += j + k; @@ -171,75 +338,146 @@ int tls1_change_cipher_state(SSL_CONNECTION *s, int which) goto err; } - switch (EVP_CIPHER_get_mode(c)) { - case EVP_CIPH_GCM_MODE: - taglen = EVP_GCM_TLS_TAG_LEN; - break; - case EVP_CIPH_CCM_MODE: - if ((s->s3.tmp.new_cipher->algorithm_enc - & (SSL_AES128CCM8 | SSL_AES256CCM8)) != 0) + memcpy(mac_secret, ms, i); + + if (!(EVP_CIPHER_get_flags(c) & EVP_CIPH_FLAG_AEAD_CIPHER)) { + mac_key = EVP_PKEY_new_raw_private_key_ex(s->ctx->libctx, "HMAC", + s->ctx->propq, mac_secret, + *mac_secret_size); + + if (mac_key == NULL + || EVP_DigestSignInit_ex(mac_ctx, NULL, EVP_MD_get0_name(m), + s->ctx->libctx, s->ctx->propq, mac_key, + NULL) <= 0) { + EVP_PKEY_free(mac_key); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + EVP_PKEY_free(mac_key); + } + + OSSL_TRACE_BEGIN(TLS) { + BIO_printf(trc_out, "which = %04X, mac key:\n", which); + BIO_dump_indent(trc_out, ms, i, 4); + } OSSL_TRACE_END(TLS); + + if (EVP_CIPHER_get_mode(c) == EVP_CIPH_GCM_MODE) { + if (!EVP_CipherInit_ex(dd, c, NULL, key, NULL, (which & SSL3_CC_WRITE)) + || !EVP_CIPHER_CTX_ctrl(dd, EVP_CTRL_GCM_SET_IV_FIXED, (int)k, + iv)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } else if (EVP_CIPHER_get_mode(c) == EVP_CIPH_CCM_MODE) { + int taglen; + if (s->s3.tmp. + new_cipher->algorithm_enc & (SSL_AES128CCM8 | SSL_AES256CCM8)) taglen = EVP_CCM8_TLS_TAG_LEN; else taglen = EVP_CCM_TLS_TAG_LEN; - break; - default: - if (EVP_CIPHER_is_a(c, "CHACHA20-POLY1305")) { - taglen = EVP_CHACHAPOLY_TLS_TAG_LEN; - } else { - /* MAC secret size corresponds to the MAC output size */ - taglen = s->s3.tmp.new_mac_secret_size; + if (!EVP_CipherInit_ex(dd, c, NULL, NULL, NULL, (which & SSL3_CC_WRITE)) + || !EVP_CIPHER_CTX_ctrl(dd, EVP_CTRL_AEAD_SET_IVLEN, 12, NULL) + || !EVP_CIPHER_CTX_ctrl(dd, EVP_CTRL_AEAD_SET_TAG, taglen, NULL) + || !EVP_CIPHER_CTX_ctrl(dd, EVP_CTRL_CCM_SET_IV_FIXED, (int)k, iv) + || !EVP_CipherInit_ex(dd, NULL, NULL, key, NULL, -1)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + } else { + if (!EVP_CipherInit_ex(dd, c, NULL, key, iv, (which & SSL3_CC_WRITE))) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; } - break; + } + /* Needed for "composite" AEADs, such as RC4-HMAC-MD5 */ + if ((EVP_CIPHER_get_flags(c) & EVP_CIPH_FLAG_AEAD_CIPHER) + && *mac_secret_size + && !EVP_CIPHER_CTX_ctrl(dd, EVP_CTRL_AEAD_SET_MAC_KEY, + (int)*mac_secret_size, mac_secret)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + if (EVP_CIPHER_get0_provider(c) != NULL + && !tls_provider_set_tls_params(s, dd, c, m)) { + /* SSLfatal already called */ + goto err; } - if (which & SSL3_CC_READ) { - if (s->ext.use_etm) - s->s3.flags |= TLS1_FLAGS_ENCRYPT_THEN_MAC_READ; - else - s->s3.flags &= ~TLS1_FLAGS_ENCRYPT_THEN_MAC_READ; +#ifndef OPENSSL_NO_KTLS + if (s->compress || (s->options & SSL_OP_ENABLE_KTLS) == 0) + goto skip_ktls; - if (s->s3.tmp.new_cipher->algorithm2 & TLS1_STREAM_MAC) - s->mac_flags |= SSL_MAC_FLAG_READ_MAC_STREAM; - else - s->mac_flags &= ~SSL_MAC_FLAG_READ_MAC_STREAM; + /* ktls supports only the maximum fragment size */ + if (ssl_get_max_send_fragment(s) != SSL3_RT_MAX_PLAIN_LENGTH) + goto skip_ktls; - if (s->s3.tmp.new_cipher->algorithm2 & TLS1_TLSTREE) - s->mac_flags |= SSL_MAC_FLAG_READ_MAC_TLSTREE; - else - s->mac_flags &= ~SSL_MAC_FLAG_READ_MAC_TLSTREE; + /* check that cipher is supported */ + if (!ktls_check_supported_cipher(s, c, dd)) + goto skip_ktls; - direction = OSSL_RECORD_DIRECTION_READ; - } else { - if (s->ext.use_etm) - s->s3.flags |= TLS1_FLAGS_ENCRYPT_THEN_MAC_WRITE; - else - s->s3.flags &= ~TLS1_FLAGS_ENCRYPT_THEN_MAC_WRITE; + if (which & SSL3_CC_WRITE) + bio = s->wbio; + else + bio = s->rbio; - if (s->s3.tmp.new_cipher->algorithm2 & TLS1_STREAM_MAC) - s->mac_flags |= SSL_MAC_FLAG_WRITE_MAC_STREAM; - else - s->mac_flags &= ~SSL_MAC_FLAG_WRITE_MAC_STREAM; + if (!ossl_assert(bio != NULL)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } - if (s->s3.tmp.new_cipher->algorithm2 & TLS1_TLSTREE) - s->mac_flags |= SSL_MAC_FLAG_WRITE_MAC_TLSTREE; - else - s->mac_flags &= ~SSL_MAC_FLAG_WRITE_MAC_TLSTREE; + /* All future data will get encrypted by ktls. Flush the BIO or skip ktls */ + if (which & SSL3_CC_WRITE) { + if (BIO_flush(bio) <= 0) + goto skip_ktls; + } - direction = OSSL_RECORD_DIRECTION_WRITE; + /* ktls doesn't support renegotiation */ + if ((BIO_get_ktls_send(s->wbio) && (which & SSL3_CC_WRITE)) || + (BIO_get_ktls_recv(s->rbio) && (which & SSL3_CC_READ))) { + SSLfatal(s, SSL_AD_NO_RENEGOTIATION, ERR_R_INTERNAL_ERROR); + goto err; } - if (SSL_CONNECTION_IS_DTLS(s)) - dtls1_increment_epoch(s, which); + if (which & SSL3_CC_WRITE) + rl_sequence = RECORD_LAYER_get_write_sequence(&s->rlayer); + else + rl_sequence = RECORD_LAYER_get_read_sequence(&s->rlayer); - if (!ssl_set_new_record_layer(s, s->version, direction, - OSSL_RECORD_PROTECTION_LEVEL_APPLICATION, - NULL, 0, key, cl, iv, (size_t)k, mac_secret, - mac_secret_size, c, taglen, mac_type, - m, comp, NULL)) { - /* SSLfatal already called */ - goto err; + if (!ktls_configure_crypto(s, c, dd, rl_sequence, &crypto_info, &rec_seq, + iv, key, ms, *mac_secret_size)) + goto skip_ktls; + + if (which & SSL3_CC_READ) { +# ifndef OPENSSL_NO_KTLS_RX + count_unprocessed = count_unprocessed_records(s); + if (count_unprocessed < 0) + goto skip_ktls; + + /* increment the crypto_info record sequence */ + while (count_unprocessed) { + for (bit = 7; bit >= 0; bit--) { /* increment */ + ++rec_seq[bit]; + if (rec_seq[bit] != 0) + break; + } + count_unprocessed--; + } +# else + goto skip_ktls; +# endif + } + + /* ktls works with user provided buffers directly */ + if (BIO_set_ktls(bio, &crypto_info, which & SSL3_CC_WRITE)) { + if (which & SSL3_CC_WRITE) + ssl3_release_write_buffer(s); + SSL_set_options(s, SSL_OP_NO_RENEGOTIATION); } + skip_ktls: +#endif /* OPENSSL_NO_KTLS */ + s->statem.enc_write_state = ENC_WRITE_STATE_VALID; + OSSL_TRACE_BEGIN(TLS) { BIO_printf(trc_out, "which = %04X, key:\n", which); BIO_dump_indent(trc_out, key, EVP_CIPHER_get_key_length(c), 4); @@ -252,7 +490,7 @@ int tls1_change_cipher_state(SSL_CONNECTION *s, int which) return 0; } -int tls1_setup_key_block(SSL_CONNECTION *s) +int tls1_setup_key_block(SSL *s) { unsigned char *p; const EVP_CIPHER *c; @@ -261,14 +499,12 @@ int tls1_setup_key_block(SSL_CONNECTION *s) int mac_type = NID_undef; size_t num, mac_secret_size = 0; int ret = 0; - int ivlen; if (s->s3.tmp.key_block_length != 0) return 1; - if (!ssl_cipher_get_evp(SSL_CONNECTION_GET_CTX(s), s->session, &c, &hash, - &mac_type, &mac_secret_size, &comp, - s->ext.use_etm)) { + if (!ssl_cipher_get_evp(s->ctx, s->session, &c, &hash, &mac_type, + &mac_secret_size, &comp, s->ext.use_etm)) { /* Error is already recorded */ SSLfatal_alert(s, SSL_AD_INTERNAL_ERROR); return 0; @@ -280,18 +516,14 @@ int tls1_setup_key_block(SSL_CONNECTION *s) s->s3.tmp.new_hash = hash; s->s3.tmp.new_mac_pkey_type = mac_type; s->s3.tmp.new_mac_secret_size = mac_secret_size; - ivlen = tls_iv_length_within_key_block(c); - if (ivlen < 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - num = mac_secret_size + EVP_CIPHER_get_key_length(c) + ivlen; + num = mac_secret_size + EVP_CIPHER_get_key_length(c) + + tls_iv_length_within_key_block(c); num *= 2; ssl3_cleanup_key_block(s); if ((p = OPENSSL_malloc(num)) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -320,21 +552,35 @@ int tls1_setup_key_block(SSL_CONNECTION *s) BIO_dump_indent(trc_out, p, num, 4); } OSSL_TRACE_END(TLS); + if (!(s->options & SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) + && s->method->version <= TLS1_VERSION) { + /* + * enable vulnerability countermeasure for CBC ciphers with known-IV + * problem (http://www.openssl.org/~bodo/tls-cbc.txt) + */ + s->s3.need_empty_fragments = 1; + + if (s->session->cipher != NULL) { + if (s->session->cipher->algorithm_enc == SSL_eNULL) + s->s3.need_empty_fragments = 0; + + if (s->session->cipher->algorithm_enc == SSL_RC4) + s->s3.need_empty_fragments = 0; + } + } + ret = 1; err: return ret; } -size_t tls1_final_finish_mac(SSL_CONNECTION *s, const char *str, - size_t slen, unsigned char *out) +size_t tls1_final_finish_mac(SSL *s, const char *str, size_t slen, + unsigned char *out) { size_t hashlen; unsigned char hash[EVP_MAX_MD_SIZE]; size_t finished_size = TLS1_FINISH_MAC_LENGTH; - if (s->s3.tmp.new_cipher->algorithm_mkey & SSL_kGOST18) - finished_size = 32; - if (!ssl3_digest_cached_records(s, 0)) { /* SSLfatal() already called */ return 0; @@ -355,9 +601,8 @@ size_t tls1_final_finish_mac(SSL_CONNECTION *s, const char *str, return finished_size; } -int tls1_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, - unsigned char *p, size_t len, - size_t *secret_size) +int tls1_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p, + size_t len, size_t *secret_size) { if (s->session->flags & SSL_SESS_FLAG_EXTMS) { unsigned char hash[EVP_MAX_MD_SIZE * 2]; @@ -419,23 +664,14 @@ int tls1_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, return 1; } -int tls1_export_keying_material(SSL_CONNECTION *s, unsigned char *out, - size_t olen, const char *label, size_t llen, +int tls1_export_keying_material(SSL *s, unsigned char *out, size_t olen, + const char *label, size_t llen, const unsigned char *context, size_t contextlen, int use_context) { unsigned char *val = NULL; size_t vallen = 0, currentvalpos; - int rv = 0; - - /* - * RFC 5705 embeds context length as uint16; reject longer context - * before proceeding. - */ - if (contextlen > 0xffff) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - return 0; - } + int rv; /* * construct PRF arguments we construct the PRF argument ourself rather @@ -449,7 +685,7 @@ int tls1_export_keying_material(SSL_CONNECTION *s, unsigned char *out, val = OPENSSL_malloc(vallen); if (val == NULL) - goto ret; + goto err2; currentvalpos = 0; memcpy(val + currentvalpos, (unsigned char *)label, llen); currentvalpos += llen; @@ -501,6 +737,11 @@ int tls1_export_keying_material(SSL_CONNECTION *s, unsigned char *out, goto ret; err1: ERR_raise(ERR_LIB_SSL, SSL_R_TLS_ILLEGAL_EXPORTER_LABEL); + rv = 0; + goto ret; + err2: + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + rv = 0; ret: OPENSSL_clear_free(val, vallen); return rv; @@ -581,3 +822,81 @@ int tls1_alert_code(int code) return -1; } } + +#ifndef OPENSSL_NO_NTLS +int ntls_alert_code(int code) +{ + switch (code) { + case SSL_AD_CLOSE_NOTIFY: + return SSL3_AD_CLOSE_NOTIFY; + case SSL_AD_UNEXPECTED_MESSAGE: + return SSL3_AD_UNEXPECTED_MESSAGE; + case SSL_AD_BAD_RECORD_MAC: + return SSL3_AD_BAD_RECORD_MAC; + case SSL_AD_DECRYPTION_FAILED: + return TLS1_AD_DECRYPTION_FAILED; + case SSL_AD_RECORD_OVERFLOW: + return TLS1_AD_RECORD_OVERFLOW; + case SSL_AD_DECOMPRESSION_FAILURE: + return SSL3_AD_DECOMPRESSION_FAILURE; + case SSL_AD_HANDSHAKE_FAILURE: + return SSL3_AD_HANDSHAKE_FAILURE; + case SSL_AD_BAD_CERTIFICATE: + return SSL3_AD_BAD_CERTIFICATE; + case SSL_AD_UNSUPPORTED_CERTIFICATE: + return SSL3_AD_UNSUPPORTED_CERTIFICATE; + case SSL_AD_CERTIFICATE_REVOKED: + return SSL3_AD_CERTIFICATE_REVOKED; + case SSL_AD_CERTIFICATE_EXPIRED: + return SSL3_AD_CERTIFICATE_EXPIRED; + case SSL_AD_CERTIFICATE_UNKNOWN: + return SSL3_AD_CERTIFICATE_UNKNOWN; + case SSL_AD_ILLEGAL_PARAMETER: + return SSL3_AD_ILLEGAL_PARAMETER; + case SSL_AD_UNKNOWN_CA: + return TLS1_AD_UNKNOWN_CA; + case SSL_AD_ACCESS_DENIED: + return TLS1_AD_ACCESS_DENIED; + case SSL_AD_DECODE_ERROR: + return TLS1_AD_DECODE_ERROR; + case SSL_AD_DECRYPT_ERROR: + return TLS1_AD_DECRYPT_ERROR; + case SSL_AD_PROTOCOL_VERSION: + return TLS1_AD_PROTOCOL_VERSION; + case SSL_AD_INSUFFICIENT_SECURITY: + return TLS1_AD_INSUFFICIENT_SECURITY; + case SSL_AD_INTERNAL_ERROR: + return TLS1_AD_INTERNAL_ERROR; + case SSL_AD_USER_CANCELLED: + return TLS1_AD_USER_CANCELLED; + case SSL_AD_UNSUPPORTED_EXTENSION: + return TLS1_AD_UNSUPPORTED_EXTENSION; + case SSL_AD_CERTIFICATE_UNOBTAINABLE: + return TLS1_AD_CERTIFICATE_UNOBTAINABLE; + case SSL_AD_UNRECOGNIZED_NAME: + return TLS1_AD_UNRECOGNIZED_NAME; + case SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE: + return TLS1_AD_BAD_CERTIFICATE_STATUS_RESPONSE; + case SSL_AD_BAD_CERTIFICATE_HASH_VALUE: + return TLS1_AD_BAD_CERTIFICATE_HASH_VALUE; + case SSL_AD_NO_APPLICATION_PROTOCOL: + return TLS1_AD_NO_APPLICATION_PROTOCOL; + case SSL_AD_CERTIFICATE_REQUIRED: + return SSL_AD_HANDSHAKE_FAILURE; + case SSL_AD_UNSUPPORTED_SITE2SITE: + return NTLS_AD_UNSUPPORTED_SITE2SITE; + case SSL_AD_NO_AREA: + return NTLS_AD_NO_AREA; + case SSL_AD_UNSUPPORTED_AREATYPE: + return NTLS_AD_UNSUPPORTED_AREATYPE; + case SSL_AD_BAD_IBCPARAM: + return NTLS_AD_BAD_IBCPARAM; + case SSL_AD_UNSUPPORTED_IBCPARAM: + return NTLS_AD_UNSUPPORTED_IBCPARAM; + case SSL_AD_IDENTITY_NEED: + return NTLS_AD_IDENTITY_NEED; + default: + return -1; + } +} +#endif diff --git a/openssl/src/ssl/t1_lib.c b/openssl/src/ssl/t1_lib.c index e9aa0785d..882713410 100644 --- a/openssl/src/ssl/t1_lib.c +++ b/openssl/src/ssl/t1_lib.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2024 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -24,13 +24,17 @@ #include "internal/sizes.h" #include "internal/tlsgroups.h" #include "ssl_local.h" -#include "quic/quic_local.h" #include -static const SIGALG_LOOKUP *find_sig_alg(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pkey); -static int tls12_sigalg_allowed(const SSL_CONNECTION *s, int op, const SIGALG_LOOKUP *lu); +static const SIGALG_LOOKUP *find_sig_alg(SSL *s, X509 *x, EVP_PKEY *pkey); +static int tls12_sigalg_allowed(const SSL *s, int op, const SIGALG_LOOKUP *lu); +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +static const SIGALG_LOOKUP *find_dc_sig_alg(SSL *s); +#endif SSL3_ENC_METHOD const TLSv1_enc_data = { + tls1_enc, + tls1_mac, tls1_setup_key_block, tls1_generate_master_secret, tls1_change_cipher_state, @@ -46,6 +50,8 @@ SSL3_ENC_METHOD const TLSv1_enc_data = { }; SSL3_ENC_METHOD const TLSv1_1_enc_data = { + tls1_enc, + tls1_mac, tls1_setup_key_block, tls1_generate_master_secret, tls1_change_cipher_state, @@ -61,6 +67,8 @@ SSL3_ENC_METHOD const TLSv1_1_enc_data = { }; SSL3_ENC_METHOD const TLSv1_2_enc_data = { + tls1_enc, + tls1_mac, tls1_setup_key_block, tls1_generate_master_secret, tls1_change_cipher_state, @@ -77,6 +85,8 @@ SSL3_ENC_METHOD const TLSv1_2_enc_data = { }; SSL3_ENC_METHOD const TLSv1_3_enc_data = { + tls13_enc, + tls1_mac, tls13_setup_key_block, tls13_generate_master_secret, tls13_change_cipher_state, @@ -91,13 +101,32 @@ SSL3_ENC_METHOD const TLSv1_3_enc_data = { ssl3_handshake_write }; -OSSL_TIME tls1_default_timeout(void) +#ifndef OPENSSL_NO_NTLS +SSL3_ENC_METHOD const NTLS_enc_data = { + tls1_enc, + tls1_mac, + tls1_setup_key_block, + tls1_generate_master_secret, + tls1_change_cipher_state, + tls1_final_finish_mac, + TLS_MD_CLIENT_FINISH_CONST, TLS_MD_CLIENT_FINISH_CONST_SIZE, + TLS_MD_SERVER_FINISH_CONST, TLS_MD_SERVER_FINISH_CONST_SIZE, + ntls_alert_code, + tls1_export_keying_material, + SSL_ENC_FLAG_EXPLICIT_IV, + ssl3_set_handshake_header, + tls_close_construct_packet, + ssl3_handshake_write +}; +#endif + +long tls1_default_timeout(void) { /* * 2 hours, the 24 hours mentioned in the TLSv1 spec is way too long for * http, the cache would over fill */ - return ossl_seconds2time(60 * 60 * 2); + return (60 * 60 * 2); } int tls1_new(SSL *s) @@ -112,35 +141,25 @@ int tls1_new(SSL *s) void tls1_free(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return; - - OPENSSL_free(sc->ext.session_ticket); + OPENSSL_free(s->ext.session_ticket); ssl3_free(s); } int tls1_clear(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - if (!ssl3_clear(s)) return 0; if (s->method->version == TLS_ANY_VERSION) - sc->version = TLS_MAX_VERSION_INTERNAL; + s->version = TLS_MAX_VERSION_INTERNAL; else - sc->version = s->method->version; + s->version = s->method->version; return 1; } /* Legacy NID to group_id mapping. Only works for groups we know about */ -static const struct { +static struct { int nid; uint16_t group_id; } nid_to_group[] = { @@ -174,21 +193,12 @@ static const struct { {NID_brainpoolP512r1, OSSL_TLS_GROUP_ID_brainpoolP512r1}, {EVP_PKEY_X25519, OSSL_TLS_GROUP_ID_x25519}, {EVP_PKEY_X448, OSSL_TLS_GROUP_ID_x448}, - {NID_brainpoolP256r1tls13, OSSL_TLS_GROUP_ID_brainpoolP256r1_tls13}, - {NID_brainpoolP384r1tls13, OSSL_TLS_GROUP_ID_brainpoolP384r1_tls13}, - {NID_brainpoolP512r1tls13, OSSL_TLS_GROUP_ID_brainpoolP512r1_tls13}, - {NID_id_tc26_gost_3410_2012_256_paramSetA, OSSL_TLS_GROUP_ID_gc256A}, - {NID_id_tc26_gost_3410_2012_256_paramSetB, OSSL_TLS_GROUP_ID_gc256B}, - {NID_id_tc26_gost_3410_2012_256_paramSetC, OSSL_TLS_GROUP_ID_gc256C}, - {NID_id_tc26_gost_3410_2012_256_paramSetD, OSSL_TLS_GROUP_ID_gc256D}, - {NID_id_tc26_gost_3410_2012_512_paramSetA, OSSL_TLS_GROUP_ID_gc512A}, - {NID_id_tc26_gost_3410_2012_512_paramSetB, OSSL_TLS_GROUP_ID_gc512B}, - {NID_id_tc26_gost_3410_2012_512_paramSetC, OSSL_TLS_GROUP_ID_gc512C}, {NID_ffdhe2048, OSSL_TLS_GROUP_ID_ffdhe2048}, {NID_ffdhe3072, OSSL_TLS_GROUP_ID_ffdhe3072}, {NID_ffdhe4096, OSSL_TLS_GROUP_ID_ffdhe4096}, {NID_ffdhe6144, OSSL_TLS_GROUP_ID_ffdhe6144}, - {NID_ffdhe8192, OSSL_TLS_GROUP_ID_ffdhe8192} + {NID_ffdhe8192, OSSL_TLS_GROUP_ID_ffdhe8192}, + {NID_sm2, OSSL_TLS_GROUP_ID_sm2}, }; static const unsigned char ecformats_default[] = { @@ -199,31 +209,34 @@ static const unsigned char ecformats_default[] = { /* The default curves */ static const uint16_t supported_groups_default[] = { - OSSL_TLS_GROUP_ID_x25519, /* X25519 (29) */ - OSSL_TLS_GROUP_ID_secp256r1, /* secp256r1 (23) */ - OSSL_TLS_GROUP_ID_x448, /* X448 (30) */ - OSSL_TLS_GROUP_ID_secp521r1, /* secp521r1 (25) */ - OSSL_TLS_GROUP_ID_secp384r1, /* secp384r1 (24) */ - OSSL_TLS_GROUP_ID_gc256A, /* GC256A (34) */ - OSSL_TLS_GROUP_ID_gc256B, /* GC256B (35) */ - OSSL_TLS_GROUP_ID_gc256C, /* GC256C (36) */ - OSSL_TLS_GROUP_ID_gc256D, /* GC256D (37) */ - OSSL_TLS_GROUP_ID_gc512A, /* GC512A (38) */ - OSSL_TLS_GROUP_ID_gc512B, /* GC512B (39) */ - OSSL_TLS_GROUP_ID_gc512C, /* GC512C (40) */ - OSSL_TLS_GROUP_ID_ffdhe2048, /* ffdhe2048 (0x100) */ - OSSL_TLS_GROUP_ID_ffdhe3072, /* ffdhe3072 (0x101) */ - OSSL_TLS_GROUP_ID_ffdhe4096, /* ffdhe4096 (0x102) */ - OSSL_TLS_GROUP_ID_ffdhe6144, /* ffdhe6144 (0x103) */ - OSSL_TLS_GROUP_ID_ffdhe8192, /* ffdhe8192 (0x104) */ + 29, /* X25519 (29) */ + 23, /* secp256r1 (23) */ + 30, /* X448 (30) */ + 25, /* secp521r1 (25) */ + 24, /* secp384r1 (24) */ + 34, /* GC256A (34) */ + 35, /* GC256B (35) */ + 36, /* GC256C (36) */ + 37, /* GC256D (37) */ + 38, /* GC512A (38) */ + 39, /* GC512B (39) */ + 40, /* GC512C (40) */ +#ifndef OPENSSL_NO_SM2 + 41, /* curveSM2 (41) */ +#endif + 0x100, /* ffdhe2048 (0x100) */ + 0x101, /* ffdhe3072 (0x101) */ + 0x102, /* ffdhe4096 (0x102) */ + 0x103, /* ffdhe6144 (0x103) */ + 0x104, /* ffdhe8192 (0x104) */ }; static const uint16_t suiteb_curves[] = { - OSSL_TLS_GROUP_ID_secp256r1, - OSSL_TLS_GROUP_ID_secp384r1, + TLSEXT_curve_P_256, + TLSEXT_curve_P_384 }; -struct provider_ctx_data_st { +struct provider_group_data_st { SSL_CTX *ctx; OSSL_PROVIDER *provider; }; @@ -232,7 +245,7 @@ struct provider_ctx_data_st { static OSSL_CALLBACK add_provider_groups; static int add_provider_groups(const OSSL_PARAM params[], void *data) { - struct provider_ctx_data_st *pgd = data; + struct provider_group_data_st *pgd = data; SSL_CTX *ctx = pgd->ctx; OSSL_PROVIDER *provider = pgd->provider; const OSSL_PARAM *p; @@ -253,8 +266,10 @@ static int add_provider_groups(const OSSL_PARAM params[], void *data) (ctx->group_list_max_len + TLS_GROUP_LIST_MALLOC_BLOCK_SIZE) * sizeof(TLS_GROUP_INFO)); - if (tmp == NULL) + if (tmp == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } ctx->group_list = tmp; memset(tmp + ctx->group_list_max_len, 0, @@ -270,8 +285,10 @@ static int add_provider_groups(const OSSL_PARAM params[], void *data) goto err; } ginf->tlsname = OPENSSL_strdup(p->data); - if (ginf->tlsname == NULL) + if (ginf->tlsname == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; + } p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_GROUP_NAME_INTERNAL); if (p == NULL || p->data_type != OSSL_PARAM_UTF8_STRING) { @@ -279,8 +296,10 @@ static int add_provider_groups(const OSSL_PARAM params[], void *data) goto err; } ginf->realname = OPENSSL_strdup(p->data); - if (ginf->realname == NULL) + if (ginf->realname == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; + } p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_GROUP_ID); if (p == NULL || !OSSL_PARAM_get_uint(p, &gid) || gid > UINT16_MAX) { @@ -295,8 +314,10 @@ static int add_provider_groups(const OSSL_PARAM params[], void *data) goto err; } ginf->algorithm = OPENSSL_strdup(p->data); - if (ginf->algorithm == NULL) + if (ginf->algorithm == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); goto err; + } p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_GROUP_SECURITY_BITS); if (p == NULL || !OSSL_PARAM_get_uint(p, &ginf->secbits)) { @@ -341,7 +362,6 @@ static int add_provider_groups(const OSSL_PARAM params[], void *data) * it. */ ret = 1; - ERR_set_mark(); keymgmt = EVP_KEYMGMT_fetch(ctx->libctx, ginf->algorithm, ctx->propq); if (keymgmt != NULL) { /* @@ -363,20 +383,19 @@ static int add_provider_groups(const OSSL_PARAM params[], void *data) } EVP_KEYMGMT_free(keymgmt); } - ERR_pop_to_mark(); err: if (ginf != NULL) { OPENSSL_free(ginf->tlsname); OPENSSL_free(ginf->realname); OPENSSL_free(ginf->algorithm); - ginf->algorithm = ginf->tlsname = ginf->realname = NULL; + ginf->tlsname = ginf->realname = NULL; } return ret; } static int discover_provider_groups(OSSL_PROVIDER *provider, void *vctx) { - struct provider_ctx_data_st pgd; + struct provider_group_data_st pgd; pgd.ctx = vctx; pgd.provider = provider; @@ -407,8 +426,10 @@ int ssl_load_groups(SSL_CTX *ctx) ctx->ext.supported_groups_default = OPENSSL_malloc(sizeof(uint16_t) * num_deflt_grps); - if (ctx->ext.supported_groups_default == NULL) + if (ctx->ext.supported_groups_default == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(ctx->ext.supported_groups_default, tmp_supp_groups, @@ -418,320 +439,6 @@ int ssl_load_groups(SSL_CTX *ctx) return 1; } -#define TLS_SIGALG_LIST_MALLOC_BLOCK_SIZE 10 -static OSSL_CALLBACK add_provider_sigalgs; -static int add_provider_sigalgs(const OSSL_PARAM params[], void *data) -{ - struct provider_ctx_data_st *pgd = data; - SSL_CTX *ctx = pgd->ctx; - OSSL_PROVIDER *provider = pgd->provider; - const OSSL_PARAM *p; - TLS_SIGALG_INFO *sinf = NULL; - EVP_KEYMGMT *keymgmt; - const char *keytype; - unsigned int code_point = 0; - int ret = 0; - - if (ctx->sigalg_list_max_len == ctx->sigalg_list_len) { - TLS_SIGALG_INFO *tmp = NULL; - - if (ctx->sigalg_list_max_len == 0) - tmp = OPENSSL_malloc(sizeof(TLS_SIGALG_INFO) - * TLS_SIGALG_LIST_MALLOC_BLOCK_SIZE); - else - tmp = OPENSSL_realloc(ctx->sigalg_list, - (ctx->sigalg_list_max_len - + TLS_SIGALG_LIST_MALLOC_BLOCK_SIZE) - * sizeof(TLS_SIGALG_INFO)); - if (tmp == NULL) - return 0; - ctx->sigalg_list = tmp; - memset(tmp + ctx->sigalg_list_max_len, 0, - sizeof(TLS_SIGALG_INFO) * TLS_SIGALG_LIST_MALLOC_BLOCK_SIZE); - ctx->sigalg_list_max_len += TLS_SIGALG_LIST_MALLOC_BLOCK_SIZE; - } - - sinf = &ctx->sigalg_list[ctx->sigalg_list_len]; - - /* First, mandatory parameters */ - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_NAME); - if (p == NULL || p->data_type != OSSL_PARAM_UTF8_STRING) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - OPENSSL_free(sinf->sigalg_name); - sinf->sigalg_name = OPENSSL_strdup(p->data); - if (sinf->sigalg_name == NULL) - goto err; - - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_IANA_NAME); - if (p == NULL || p->data_type != OSSL_PARAM_UTF8_STRING) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - OPENSSL_free(sinf->name); - sinf->name = OPENSSL_strdup(p->data); - if (sinf->name == NULL) - goto err; - - p = OSSL_PARAM_locate_const(params, - OSSL_CAPABILITY_TLS_SIGALG_CODE_POINT); - if (p == NULL - || !OSSL_PARAM_get_uint(p, &code_point) - || code_point > UINT16_MAX) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - sinf->code_point = (uint16_t)code_point; - - p = OSSL_PARAM_locate_const(params, - OSSL_CAPABILITY_TLS_SIGALG_SECURITY_BITS); - if (p == NULL || !OSSL_PARAM_get_uint(p, &sinf->secbits)) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - - /* Now, optional parameters */ - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_OID); - if (p == NULL) { - sinf->sigalg_oid = NULL; - } else if (p->data_type != OSSL_PARAM_UTF8_STRING) { - goto err; - } else { - OPENSSL_free(sinf->sigalg_oid); - sinf->sigalg_oid = OPENSSL_strdup(p->data); - if (sinf->sigalg_oid == NULL) - goto err; - } - - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_SIG_NAME); - if (p == NULL) { - sinf->sig_name = NULL; - } else if (p->data_type != OSSL_PARAM_UTF8_STRING) { - goto err; - } else { - OPENSSL_free(sinf->sig_name); - sinf->sig_name = OPENSSL_strdup(p->data); - if (sinf->sig_name == NULL) - goto err; - } - - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_SIG_OID); - if (p == NULL) { - sinf->sig_oid = NULL; - } else if (p->data_type != OSSL_PARAM_UTF8_STRING) { - goto err; - } else { - OPENSSL_free(sinf->sig_oid); - sinf->sig_oid = OPENSSL_strdup(p->data); - if (sinf->sig_oid == NULL) - goto err; - } - - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_HASH_NAME); - if (p == NULL) { - sinf->hash_name = NULL; - } else if (p->data_type != OSSL_PARAM_UTF8_STRING) { - goto err; - } else { - OPENSSL_free(sinf->hash_name); - sinf->hash_name = OPENSSL_strdup(p->data); - if (sinf->hash_name == NULL) - goto err; - } - - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_HASH_OID); - if (p == NULL) { - sinf->hash_oid = NULL; - } else if (p->data_type != OSSL_PARAM_UTF8_STRING) { - goto err; - } else { - OPENSSL_free(sinf->hash_oid); - sinf->hash_oid = OPENSSL_strdup(p->data); - if (sinf->hash_oid == NULL) - goto err; - } - - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_KEYTYPE); - if (p == NULL) { - sinf->keytype = NULL; - } else if (p->data_type != OSSL_PARAM_UTF8_STRING) { - goto err; - } else { - OPENSSL_free(sinf->keytype); - sinf->keytype = OPENSSL_strdup(p->data); - if (sinf->keytype == NULL) - goto err; - } - - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_KEYTYPE_OID); - if (p == NULL) { - sinf->keytype_oid = NULL; - } else if (p->data_type != OSSL_PARAM_UTF8_STRING) { - goto err; - } else { - OPENSSL_free(sinf->keytype_oid); - sinf->keytype_oid = OPENSSL_strdup(p->data); - if (sinf->keytype_oid == NULL) - goto err; - } - - /* The remaining parameters below are mandatory again */ - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_MIN_TLS); - if (p == NULL || !OSSL_PARAM_get_int(p, &sinf->mintls)) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - if ((sinf->mintls != 0) && (sinf->mintls != -1) && - ((sinf->mintls < TLS1_3_VERSION))) { - /* ignore this sigalg as this OpenSSL doesn't know how to handle it */ - ret = 1; - goto err; - } - - p = OSSL_PARAM_locate_const(params, OSSL_CAPABILITY_TLS_SIGALG_MAX_TLS); - if (p == NULL || !OSSL_PARAM_get_int(p, &sinf->maxtls)) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - if ((sinf->maxtls != 0) && (sinf->maxtls != -1) && - ((sinf->maxtls < sinf->mintls))) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - if ((sinf->maxtls != 0) && (sinf->maxtls != -1) && - ((sinf->maxtls < TLS1_3_VERSION))) { - /* ignore this sigalg as this OpenSSL doesn't know how to handle it */ - ret = 1; - goto err; - } - - /* - * Now check that the algorithm is actually usable for our property query - * string. Regardless of the result we still return success because we have - * successfully processed this signature, even though we may decide not to - * use it. - */ - ret = 1; - ERR_set_mark(); - keytype = (sinf->keytype != NULL - ? sinf->keytype - : (sinf->sig_name != NULL - ? sinf->sig_name - : sinf->sigalg_name)); - keymgmt = EVP_KEYMGMT_fetch(ctx->libctx, keytype, ctx->propq); - if (keymgmt != NULL) { - /* - * We have successfully fetched the algorithm - however if the provider - * doesn't match this one then we ignore it. - * - * Note: We're cheating a little here. Technically if the same algorithm - * is available from more than one provider then it is undefined which - * implementation you will get back. Theoretically this could be - * different every time...we assume here that you'll always get the - * same one back if you repeat the exact same fetch. Is this a reasonable - * assumption to make (in which case perhaps we should document this - * behaviour)? - */ - if (EVP_KEYMGMT_get0_provider(keymgmt) == provider) { - /* - * We have a match - so we could use this signature; - * Check proper object registration first, though. - * Don't care about return value as this may have been - * done within providers or previous calls to - * add_provider_sigalgs. - */ - OBJ_create(sinf->sigalg_oid, sinf->sigalg_name, NULL); - /* sanity check: Without successful registration don't use alg */ - if ((OBJ_txt2nid(sinf->sigalg_name) == NID_undef) || - (OBJ_nid2obj(OBJ_txt2nid(sinf->sigalg_name)) == NULL)) { - ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT); - goto err; - } - if (sinf->sig_name != NULL) - OBJ_create(sinf->sig_oid, sinf->sig_name, NULL); - if (sinf->keytype != NULL) - OBJ_create(sinf->keytype_oid, sinf->keytype, NULL); - if (sinf->hash_name != NULL) - OBJ_create(sinf->hash_oid, sinf->hash_name, NULL); - OBJ_add_sigid(OBJ_txt2nid(sinf->sigalg_name), - (sinf->hash_name != NULL - ? OBJ_txt2nid(sinf->hash_name) - : NID_undef), - OBJ_txt2nid(keytype)); - ctx->sigalg_list_len++; - sinf = NULL; - } - EVP_KEYMGMT_free(keymgmt); - } - ERR_pop_to_mark(); - err: - if (sinf != NULL) { - OPENSSL_free(sinf->name); - sinf->name = NULL; - OPENSSL_free(sinf->sigalg_name); - sinf->sigalg_name = NULL; - OPENSSL_free(sinf->sigalg_oid); - sinf->sigalg_oid = NULL; - OPENSSL_free(sinf->sig_name); - sinf->sig_name = NULL; - OPENSSL_free(sinf->sig_oid); - sinf->sig_oid = NULL; - OPENSSL_free(sinf->hash_name); - sinf->hash_name = NULL; - OPENSSL_free(sinf->hash_oid); - sinf->hash_oid = NULL; - OPENSSL_free(sinf->keytype); - sinf->keytype = NULL; - OPENSSL_free(sinf->keytype_oid); - sinf->keytype_oid = NULL; - } - return ret; -} - -static int discover_provider_sigalgs(OSSL_PROVIDER *provider, void *vctx) -{ - struct provider_ctx_data_st pgd; - - pgd.ctx = vctx; - pgd.provider = provider; - OSSL_PROVIDER_get_capabilities(provider, "TLS-SIGALG", - add_provider_sigalgs, &pgd); - /* - * Always OK, even if provider doesn't support the capability: - * Reconsider testing retval when legacy sigalgs are also loaded this way. - */ - return 1; -} - -int ssl_load_sigalgs(SSL_CTX *ctx) -{ - size_t i; - SSL_CERT_LOOKUP lu; - - if (!OSSL_PROVIDER_do_all(ctx->libctx, discover_provider_sigalgs, ctx)) - return 0; - - /* now populate ctx->ssl_cert_info */ - if (ctx->sigalg_list_len > 0) { - OPENSSL_free(ctx->ssl_cert_info); - ctx->ssl_cert_info = OPENSSL_zalloc(sizeof(lu) * ctx->sigalg_list_len); - if (ctx->ssl_cert_info == NULL) - return 0; - for(i = 0; i < ctx->sigalg_list_len; i++) { - ctx->ssl_cert_info[i].nid = OBJ_txt2nid(ctx->sigalg_list[i].sigalg_name); - ctx->ssl_cert_info[i].amask = SSL_aANY; - } - } - - /* - * For now, leave it at this: legacy sigalgs stay in their own - * data structures until "legacy cleanup" occurs. - */ - - return 1; -} - static uint16_t tls1_group_name2id(SSL_CTX *ctx, const char *name) { size_t i; @@ -757,16 +464,6 @@ const TLS_GROUP_INFO *tls1_group_id_lookup(SSL_CTX *ctx, uint16_t group_id) return NULL; } -const char *tls1_group_id2name(SSL_CTX *ctx, uint16_t group_id) -{ - const TLS_GROUP_INFO *tls_group_info = tls1_group_id_lookup(ctx, group_id); - - if (tls_group_info == NULL) - return NULL; - - return tls_group_info->tlsname; -} - int tls1_group_id2nid(uint16_t group_id, int include_unknown) { size_t i; @@ -809,11 +506,9 @@ uint16_t tls1_nid2group_id(int nid) * Set *pgroups to the supported groups list and *pgroupslen to * the number of groups supported. */ -void tls1_get_supported_groups(SSL_CONNECTION *s, const uint16_t **pgroups, +void tls1_get_supported_groups(SSL *s, const uint16_t **pgroups, size_t *pgroupslen) { - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - /* For Suite B mode only include P-256, P-384 */ switch (tls1_suiteb(s)) { case SSL_CERT_FLAG_SUITEB_128_LOS: @@ -833,8 +528,8 @@ void tls1_get_supported_groups(SSL_CONNECTION *s, const uint16_t **pgroups, default: if (s->ext.supportedgroups == NULL) { - *pgroups = sctx->ext.supported_groups_default; - *pgroupslen = sctx->ext.supported_groups_default_len; + *pgroups = s->ctx->ext.supported_groups_default; + *pgroupslen = s->ctx->ext.supported_groups_default_len; } else { *pgroups = s->ext.supportedgroups; *pgroupslen = s->ext.supportedgroups_len; @@ -843,14 +538,11 @@ void tls1_get_supported_groups(SSL_CONNECTION *s, const uint16_t **pgroups, } } -int tls_valid_group(SSL_CONNECTION *s, uint16_t group_id, - int minversion, int maxversion, +int tls_valid_group(SSL *s, uint16_t group_id, int minversion, int maxversion, int isec, int *okfortls13) { - const TLS_GROUP_INFO *ginfo = tls1_group_id_lookup(SSL_CONNECTION_GET_CTX(s), - group_id); + const TLS_GROUP_INFO *ginfo = tls1_group_id_lookup(s->ctx, group_id); int ret; - int group_minversion, group_maxversion; if (okfortls13 != NULL) *okfortls13 = 0; @@ -858,22 +550,27 @@ int tls_valid_group(SSL_CONNECTION *s, uint16_t group_id, if (ginfo == NULL) return 0; - group_minversion = SSL_CONNECTION_IS_DTLS(s) ? ginfo->mindtls : ginfo->mintls; - group_maxversion = SSL_CONNECTION_IS_DTLS(s) ? ginfo->maxdtls : ginfo->maxtls; - - if (group_minversion < 0 || group_maxversion < 0) - return 0; - if (group_maxversion == 0) - ret = 1; - else - ret = (ssl_version_cmp(s, minversion, group_maxversion) <= 0); - if (group_minversion > 0) - ret &= (ssl_version_cmp(s, maxversion, group_minversion) >= 0); - - if (!SSL_CONNECTION_IS_DTLS(s)) { + if (SSL_IS_DTLS(s)) { + if (ginfo->mindtls < 0 || ginfo->maxdtls < 0) + return 0; + if (ginfo->maxdtls == 0) + ret = 1; + else + ret = DTLS_VERSION_LE(minversion, ginfo->maxdtls); + if (ginfo->mindtls > 0) + ret &= DTLS_VERSION_GE(maxversion, ginfo->mindtls); + } else { + if (ginfo->mintls < 0 || ginfo->maxtls < 0) + return 0; + if (ginfo->maxtls == 0) + ret = 1; + else + ret = (minversion <= ginfo->maxtls); + if (ginfo->mintls > 0) + ret &= (maxversion >= ginfo->mintls); if (ret && okfortls13 != NULL && maxversion == TLS1_3_VERSION) - *okfortls13 = (group_maxversion == 0) - || (group_maxversion >= TLS1_3_VERSION); + *okfortls13 = (ginfo->maxtls == 0) + || (ginfo->maxtls >= TLS1_3_VERSION); } ret &= !isec || strcmp(ginfo->algorithm, "EC") == 0 @@ -884,10 +581,9 @@ int tls_valid_group(SSL_CONNECTION *s, uint16_t group_id, } /* See if group is allowed by security callback */ -int tls_group_allowed(SSL_CONNECTION *s, uint16_t group, int op) +int tls_group_allowed(SSL *s, uint16_t group, int op) { - const TLS_GROUP_INFO *ginfo = tls1_group_id_lookup(SSL_CONNECTION_GET_CTX(s), - group); + const TLS_GROUP_INFO *ginfo = tls1_group_id_lookup(s->ctx, group); unsigned char gtmp[2]; if (ginfo == NULL) @@ -916,12 +612,16 @@ static int tls1_in_list(uint16_t id, const uint16_t *list, size_t listlen) * For nmatch == -2, return the id of the group to use for * a tmp key, or 0 if there is no match. */ -uint16_t tls1_shared_group(SSL_CONNECTION *s, int nmatch) +uint16_t tls1_shared_group(SSL *s, int nmatch) { const uint16_t *pref, *supp; size_t num_pref, num_supp, i; int k; - SSL_CTX *ctx = SSL_CONNECTION_GET_CTX(s); + +#ifndef OPENSSL_NO_NTLS + if (SSL_IS_NTLS(s)) + return TLSEXT_curve_SM2; +#endif /* Can't do anything on client side */ if (s->server == 0) @@ -935,9 +635,9 @@ uint16_t tls1_shared_group(SSL_CONNECTION *s, int nmatch) unsigned long cid = s->s3.tmp.new_cipher->id; if (cid == TLS1_CK_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) - return OSSL_TLS_GROUP_ID_secp256r1; + return TLSEXT_curve_P_256; if (cid == TLS1_CK_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) - return OSSL_TLS_GROUP_ID_secp384r1; + return TLSEXT_curve_P_384; /* Should never happen */ return 0; } @@ -958,27 +658,10 @@ uint16_t tls1_shared_group(SSL_CONNECTION *s, int nmatch) for (k = 0, i = 0; i < num_pref; i++) { uint16_t id = pref[i]; - const TLS_GROUP_INFO *inf; - int minversion, maxversion; if (!tls1_in_list(id, supp, num_supp) - || !tls_group_allowed(s, id, SSL_SECOP_CURVE_SHARED)) - continue; - inf = tls1_group_id_lookup(ctx, id); - if (!ossl_assert(inf != NULL)) - return 0; - - minversion = SSL_CONNECTION_IS_DTLS(s) - ? inf->mindtls : inf->mintls; - maxversion = SSL_CONNECTION_IS_DTLS(s) - ? inf->maxdtls : inf->maxtls; - if (maxversion == -1) - continue; - if ((minversion != 0 && ssl_version_cmp(s, s->version, minversion) < 0) - || (maxversion != 0 - && ssl_version_cmp(s, s->version, maxversion) > 0)) - continue; - + || !tls_group_allowed(s, id, SSL_SECOP_CURVE_SHARED)) + continue; if (nmatch == k) return id; k++; @@ -994,31 +677,30 @@ int tls1_set_groups(uint16_t **pext, size_t *pextlen, { uint16_t *glist; size_t i; - /* - * Bitmap of groups included to detect duplicates: two variables are added - * to detect duplicates as some values are more than 32. - */ - unsigned long *dup_list = NULL; - unsigned long dup_list_egrp = 0; - unsigned long dup_list_dhgrp = 0; + uint8_t bitmap[64] = { 0 }; if (ngroups == 0) { ERR_raise(ERR_LIB_SSL, SSL_R_BAD_LENGTH); return 0; } - if ((glist = OPENSSL_malloc(ngroups * sizeof(*glist))) == NULL) + if ((glist = OPENSSL_malloc(ngroups * sizeof(*glist))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } for (i = 0; i < ngroups; i++) { - unsigned long idmask; uint16_t id; id = tls1_nid2group_id(groups[i]); - if ((id & 0x00FF) >= (sizeof(unsigned long) * 8)) + if (ngroups == 1) { + glist[i] = id; + break; + } + if (id == 0 || id >= sizeof(bitmap) * 8) goto err; - idmask = 1L << (id & 0x00FF); - dup_list = (id < 0x100) ? &dup_list_egrp : &dup_list_dhgrp; - if (!id || ((*dup_list) & idmask)) + + if (bitmap[id / 8] & (1 << (id % 8))) goto err; - *dup_list |= idmask; + + bitmap[id / 8] |= 1 << (id % 8); glist[i] = id; } OPENSSL_free(*pext); @@ -1045,19 +727,12 @@ static int gid_cb(const char *elem, int len, void *arg) size_t i; uint16_t gid = 0; char etmp[GROUP_NAME_BUFFER_LENGTH]; - int ignore_unknown = 0; if (elem == NULL) return 0; - if (elem[0] == '?') { - ignore_unknown = 1; - ++elem; - --len; - } if (garg->gidcnt == garg->gidmax) { uint16_t *tmp = - OPENSSL_realloc(garg->gid_arr, - (garg->gidmax + GROUPLIST_INCREMENT) * sizeof(*garg->gid_arr)); + OPENSSL_realloc(garg->gid_arr, garg->gidmax + GROUPLIST_INCREMENT); if (tmp == NULL) return 0; garg->gidmax += GROUPLIST_INCREMENT; @@ -1069,15 +744,11 @@ static int gid_cb(const char *elem, int len, void *arg) etmp[len] = 0; gid = tls1_group_name2id(garg->ctx, etmp); - if (gid == 0) { - /* Unknown group - ignore, if ignore_unknown */ - return ignore_unknown; - } + if (gid == 0) + return 0; for (i = 0; i < garg->gidcnt; i++) - if (garg->gid_arr[i] == gid) { - /* Duplicate group - ignore */ - return 1; - } + if (garg->gid_arr[i] == gid) + return 0; garg->gid_arr[garg->gidcnt++] = gid; return 1; } @@ -1098,11 +769,6 @@ int tls1_set_groups_list(SSL_CTX *ctx, uint16_t **pext, size_t *pextlen, gcb.ctx = ctx; if (!CONF_parse_list(str, ':', 1, gid_cb, &gcb)) goto end; - if (gcb.gidcnt == 0) { - ERR_raise_data(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT, - "No valid groups in '%s'", str); - goto end; - } if (pext == NULL) { ret = 1; goto end; @@ -1115,7 +781,6 @@ int tls1_set_groups_list(SSL_CTX *ctx, uint16_t **pext, size_t *pextlen, tmparr = OPENSSL_memdup(gcb.gid_arr, gcb.gidcnt * sizeof(*tmparr)); if (tmparr == NULL) goto end; - OPENSSL_free(*pext); *pext = tmparr; *pextlen = gcb.gidcnt; ret = 1; @@ -1125,8 +790,7 @@ int tls1_set_groups_list(SSL_CTX *ctx, uint16_t **pext, size_t *pextlen, } /* Check a group id matches preferences */ -int tls1_check_group_id(SSL_CONNECTION *s, uint16_t group_id, - int check_own_groups) +int tls1_check_group_id(SSL *s, uint16_t group_id, int check_own_groups) { const uint16_t *groups; size_t groups_len; @@ -1139,10 +803,10 @@ int tls1_check_group_id(SSL_CONNECTION *s, uint16_t group_id, unsigned long cid = s->s3.tmp.new_cipher->id; if (cid == TLS1_CK_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) { - if (group_id != OSSL_TLS_GROUP_ID_secp256r1) + if (group_id != TLSEXT_curve_P_256) return 0; } else if (cid == TLS1_CK_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) { - if (group_id != OSSL_TLS_GROUP_ID_secp384r1) + if (group_id != TLSEXT_curve_P_384) return 0; } else { /* Should never happen */ @@ -1178,7 +842,7 @@ int tls1_check_group_id(SSL_CONNECTION *s, uint16_t group_id, return tls1_in_list(group_id, groups, groups_len); } -void tls1_get_formatlist(SSL_CONNECTION *s, const unsigned char **pformats, +void tls1_get_formatlist(SSL *s, const unsigned char **pformats, size_t *num_formats) { /* @@ -1198,7 +862,7 @@ void tls1_get_formatlist(SSL_CONNECTION *s, const unsigned char **pformats, } /* Check a key is compatible with compression extension */ -static int tls1_check_pkey_comp(SSL_CONNECTION *s, EVP_PKEY *pkey) +static int tls1_check_pkey_comp(SSL *s, EVP_PKEY *pkey) { unsigned char comp_id; size_t i; @@ -1215,7 +879,7 @@ static int tls1_check_pkey_comp(SSL_CONNECTION *s, EVP_PKEY *pkey) return 0; if (point_conv == POINT_CONVERSION_UNCOMPRESSED) { comp_id = TLSEXT_ECPOINTFORMAT_uncompressed; - } else if (SSL_CONNECTION_IS_TLS13(s)) { + } else if (SSL_IS_TLS13(s)) { /* * ec_point_formats extension is not used in TLSv1.3 so we ignore * this check. @@ -1259,7 +923,7 @@ static uint16_t tls1_get_group_id(EVP_PKEY *pkey) * Check cert parameters compatible with extensions: currently just checks EC * certificates have compatible curves and compression. */ -static int tls1_check_cert_param(SSL_CONNECTION *s, X509 *x, int check_ee_md) +static int tls1_check_cert_param(SSL *s, X509 *x, int check_ee_md) { uint16_t group_id; EVP_PKEY *pkey; @@ -1288,15 +952,15 @@ static int tls1_check_cert_param(SSL_CONNECTION *s, X509 *x, int check_ee_md) size_t i; /* Check to see we have necessary signing algorithm */ - if (group_id == OSSL_TLS_GROUP_ID_secp256r1) + if (group_id == TLSEXT_curve_P_256) check_md = NID_ecdsa_with_SHA256; - else if (group_id == OSSL_TLS_GROUP_ID_secp384r1) + else if (group_id == TLSEXT_curve_P_384) check_md = NID_ecdsa_with_SHA384; else return 0; /* Should never happen */ for (i = 0; i < s->shared_sigalgslen; i++) { if (check_md == s->shared_sigalgs[i]->sigandhash) - return 1; + return 1;; } return 0; } @@ -1313,7 +977,7 @@ static int tls1_check_cert_param(SSL_CONNECTION *s, X509 *x, int check_ee_md) * * Returns 0 when the cipher can't be used or 1 when it can. */ -int tls1_check_ec_tmp_key(SSL_CONNECTION *s, unsigned long cid) +int tls1_check_ec_tmp_key(SSL *s, unsigned long cid) { /* If not Suite B just need a shared group */ if (!tls1_suiteb(s)) @@ -1323,23 +987,37 @@ int tls1_check_ec_tmp_key(SSL_CONNECTION *s, unsigned long cid) * curves permitted. */ if (cid == TLS1_CK_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256) - return tls1_check_group_id(s, OSSL_TLS_GROUP_ID_secp256r1, 1); + return tls1_check_group_id(s, TLSEXT_curve_P_256, 1); if (cid == TLS1_CK_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) - return tls1_check_group_id(s, OSSL_TLS_GROUP_ID_secp384r1, 1); + return tls1_check_group_id(s, TLSEXT_curve_P_384, 1); return 0; } + +#ifndef OPENSSL_NO_NTLS +static const SIGALG_LOOKUP ntls_sm2_sigalg = { + "sm2sig_sm3", TLSEXT_SIGALG_sm2sig_sm3, + NID_sm3, SSL_MD_SM3_IDX, NID_sm2, SSL_PKEY_SM2_SIGN, + NID_SM2_with_SM3, NID_sm2 +}; + +static const SIGALG_LOOKUP ntls_rsa_sigalg = { + "rsa_pkcs1_sha256", TLSEXT_SIGALG_rsa_pkcs1_sha256, + NID_sha256, SSL_MD_SHA256_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN, + NID_sha256WithRSAEncryption, NID_undef +}; +#endif /* Default sigalg schemes */ static const uint16_t tls12_sigalgs[] = { TLSEXT_SIGALG_ecdsa_secp256r1_sha256, TLSEXT_SIGALG_ecdsa_secp384r1_sha384, TLSEXT_SIGALG_ecdsa_secp521r1_sha512, +#if (!defined OPENSSL_NO_SM2) && (!defined OPENSSL_NO_SM3) + TLSEXT_SIGALG_sm2sig_sm3, +#endif TLSEXT_SIGALG_ed25519, TLSEXT_SIGALG_ed448, - TLSEXT_SIGALG_ecdsa_brainpoolP256r1_sha256, - TLSEXT_SIGALG_ecdsa_brainpoolP384r1_sha384, - TLSEXT_SIGALG_ecdsa_brainpoolP512r1_sha512, TLSEXT_SIGALG_rsa_pss_pss_sha256, TLSEXT_SIGALG_rsa_pss_pss_sha384, @@ -1364,14 +1042,6 @@ static const uint16_t tls12_sigalgs[] = { TLSEXT_SIGALG_dsa_sha256, TLSEXT_SIGALG_dsa_sha384, TLSEXT_SIGALG_dsa_sha512, - -#ifndef OPENSSL_NO_GOST - TLSEXT_SIGALG_gostr34102012_256_intrinsic, - TLSEXT_SIGALG_gostr34102012_512_intrinsic, - TLSEXT_SIGALG_gostr34102012_256_gostr34112012_256, - TLSEXT_SIGALG_gostr34102012_512_gostr34112012_512, - TLSEXT_SIGALG_gostr34102001_gostr3411, -#endif }; @@ -1381,6 +1051,11 @@ static const uint16_t suiteb_sigalgs[] = { }; static const SIGALG_LOOKUP sigalg_lookup_tbl[] = { +#if (!defined OPENSSL_NO_SM2) && (!defined OPENSSL_NO_SM3) + {"sm2sig_sm3", TLSEXT_SIGALG_sm2sig_sm3, + NID_sm3, SSL_MD_SM3_IDX, EVP_PKEY_SM2, SSL_PKEY_SM2, + NID_SM2_with_SM3, NID_sm2, 1}, +#endif {"ecdsa_secp256r1_sha256", TLSEXT_SIGALG_ecdsa_secp256r1_sha256, NID_sha256, SSL_MD_SHA256_IDX, EVP_PKEY_EC, SSL_PKEY_ECC, NID_ecdsa_with_SHA256, NID_X9_62_prime256v1, 1}, @@ -1402,15 +1077,6 @@ static const SIGALG_LOOKUP sigalg_lookup_tbl[] = { {NULL, TLSEXT_SIGALG_ecdsa_sha1, NID_sha1, SSL_MD_SHA1_IDX, EVP_PKEY_EC, SSL_PKEY_ECC, NID_ecdsa_with_SHA1, NID_undef, 1}, - {"ecdsa_brainpoolP256r1_sha256", TLSEXT_SIGALG_ecdsa_brainpoolP256r1_sha256, - NID_sha256, SSL_MD_SHA256_IDX, EVP_PKEY_EC, SSL_PKEY_ECC, - NID_ecdsa_with_SHA256, NID_brainpoolP256r1, 1}, - {"ecdsa_brainpoolP384r1_sha384", TLSEXT_SIGALG_ecdsa_brainpoolP384r1_sha384, - NID_sha384, SSL_MD_SHA384_IDX, EVP_PKEY_EC, SSL_PKEY_ECC, - NID_ecdsa_with_SHA384, NID_brainpoolP384r1, 1}, - {"ecdsa_brainpoolP512r1_sha512", TLSEXT_SIGALG_ecdsa_brainpoolP512r1_sha512, - NID_sha512, SSL_MD_SHA512_IDX, EVP_PKEY_EC, SSL_PKEY_ECC, - NID_ecdsa_with_SHA512, NID_brainpoolP512r1, 1}, {"rsa_pss_rsae_sha256", TLSEXT_SIGALG_rsa_pss_rsae_sha256, NID_sha256, SSL_MD_SHA256_IDX, EVP_PKEY_RSA_PSS, SSL_PKEY_RSA, NID_undef, NID_undef, 1}, @@ -1459,28 +1125,6 @@ static const SIGALG_LOOKUP sigalg_lookup_tbl[] = { {NULL, TLSEXT_SIGALG_dsa_sha1, NID_sha1, SSL_MD_SHA1_IDX, EVP_PKEY_DSA, SSL_PKEY_DSA_SIGN, NID_dsaWithSHA1, NID_undef, 1}, -#ifndef OPENSSL_NO_GOST - {NULL, TLSEXT_SIGALG_gostr34102012_256_intrinsic, - NID_id_GostR3411_2012_256, SSL_MD_GOST12_256_IDX, - NID_id_GostR3410_2012_256, SSL_PKEY_GOST12_256, - NID_undef, NID_undef, 1}, - {NULL, TLSEXT_SIGALG_gostr34102012_512_intrinsic, - NID_id_GostR3411_2012_512, SSL_MD_GOST12_512_IDX, - NID_id_GostR3410_2012_512, SSL_PKEY_GOST12_512, - NID_undef, NID_undef, 1}, - {NULL, TLSEXT_SIGALG_gostr34102012_256_gostr34112012_256, - NID_id_GostR3411_2012_256, SSL_MD_GOST12_256_IDX, - NID_id_GostR3410_2012_256, SSL_PKEY_GOST12_256, - NID_undef, NID_undef, 1}, - {NULL, TLSEXT_SIGALG_gostr34102012_512_gostr34112012_512, - NID_id_GostR3411_2012_512, SSL_MD_GOST12_512_IDX, - NID_id_GostR3410_2012_512, SSL_PKEY_GOST12_512, - NID_undef, NID_undef, 1}, - {NULL, TLSEXT_SIGALG_gostr34102001_gostr3411, - NID_id_GostR3411_94, SSL_MD_GOST94_IDX, - NID_id_GostR3410_2001, SSL_PKEY_GOST01, - NID_undef, NID_undef, 1} -#endif }; /* Legacy sigalgs for TLS < 1.2 RSA TLS signatures */ static const SIGALG_LOOKUP legacy_rsa_sigalg = { @@ -1499,43 +1143,34 @@ static const uint16_t tls_default_sigalg[] = { 0, /* SSL_PKEY_RSA_PSS_SIGN */ TLSEXT_SIGALG_dsa_sha1, /* SSL_PKEY_DSA_SIGN */ TLSEXT_SIGALG_ecdsa_sha1, /* SSL_PKEY_ECC */ - TLSEXT_SIGALG_gostr34102001_gostr3411, /* SSL_PKEY_GOST01 */ - TLSEXT_SIGALG_gostr34102012_256_intrinsic, /* SSL_PKEY_GOST12_256 */ - TLSEXT_SIGALG_gostr34102012_512_intrinsic, /* SSL_PKEY_GOST12_512 */ 0, /* SSL_PKEY_ED25519 */ 0, /* SSL_PKEY_ED448 */ +#ifndef OPENSSL_NO_NTLS + 0, /* SSL_PKEY_SM2_SIGN */ + 0, /* SSL_PKEY_SM2_ENC */ + 0, /* SSL_PKEY_RSA_SIGN */ + 0, /* SSL_PKEY_RSA_ENC */ +#endif }; -int ssl_setup_sigalgs(SSL_CTX *ctx) +int ssl_setup_sig_algs(SSL_CTX *ctx) { - size_t i, cache_idx, sigalgs_len; + size_t i; const SIGALG_LOOKUP *lu; - SIGALG_LOOKUP *cache = NULL; - uint16_t *tls12_sigalgs_list = NULL; + SIGALG_LOOKUP *cache + = OPENSSL_malloc(sizeof(*lu) * OSSL_NELEM(sigalg_lookup_tbl)); EVP_PKEY *tmpkey = EVP_PKEY_new(); int ret = 0; - if (ctx == NULL) - goto err; - - sigalgs_len = OSSL_NELEM(sigalg_lookup_tbl) + ctx->sigalg_list_len; - - cache = OPENSSL_malloc(sizeof(const SIGALG_LOOKUP) * sigalgs_len); if (cache == NULL || tmpkey == NULL) goto err; - tls12_sigalgs_list = OPENSSL_malloc(sizeof(uint16_t) * sigalgs_len); - if (tls12_sigalgs_list == NULL) - goto err; - ERR_set_mark(); - /* First fill cache and tls12_sigalgs list from legacy algorithm list */ for (i = 0, lu = sigalg_lookup_tbl; i < OSSL_NELEM(sigalg_lookup_tbl); lu++, i++) { EVP_PKEY_CTX *pctx; cache[i] = *lu; - tls12_sigalgs_list[i] = tls12_sigalgs[i]; /* * Check hash is available. @@ -1561,48 +1196,26 @@ int ssl_setup_sigalgs(SSL_CTX *ctx) cache[i].enabled = 0; EVP_PKEY_CTX_free(pctx); } - - /* Now complete cache and tls12_sigalgs list with provider sig information */ - cache_idx = OSSL_NELEM(sigalg_lookup_tbl); - for (i = 0; i < ctx->sigalg_list_len; i++) { - TLS_SIGALG_INFO si = ctx->sigalg_list[i]; - cache[cache_idx].name = si.name; - cache[cache_idx].sigalg = si.code_point; - tls12_sigalgs_list[cache_idx] = si.code_point; - cache[cache_idx].hash = si.hash_name?OBJ_txt2nid(si.hash_name):NID_undef; - cache[cache_idx].hash_idx = ssl_get_md_idx(cache[cache_idx].hash); - cache[cache_idx].sig = OBJ_txt2nid(si.sigalg_name); - cache[cache_idx].sig_idx = i + SSL_PKEY_NUM; - cache[cache_idx].sigandhash = OBJ_txt2nid(si.sigalg_name); - cache[cache_idx].curve = NID_undef; - /* all provided sigalgs are enabled by load */ - cache[cache_idx].enabled = 1; - cache_idx++; - } ERR_pop_to_mark(); ctx->sigalg_lookup_cache = cache; - ctx->tls12_sigalgs = tls12_sigalgs_list; - ctx->tls12_sigalgs_len = sigalgs_len; cache = NULL; - tls12_sigalgs_list = NULL; ret = 1; err: OPENSSL_free(cache); - OPENSSL_free(tls12_sigalgs_list); EVP_PKEY_free(tmpkey); return ret; } /* Lookup TLS signature algorithm */ -static const SIGALG_LOOKUP *tls1_lookup_sigalg(const SSL_CONNECTION *s, - uint16_t sigalg) +static const SIGALG_LOOKUP *tls1_lookup_sigalg(const SSL *s, uint16_t sigalg) { size_t i; const SIGALG_LOOKUP *lu; - for (i = 0, lu = SSL_CONNECTION_GET_CTX(s)->sigalg_lookup_cache; - i < SSL_CONNECTION_GET_CTX(s)->tls12_sigalgs_len; + for (i = 0, lu = s->ctx->sigalg_lookup_cache; + /* cache should have the same number of elements as sigalg_lookup_tbl */ + i < OSSL_NELEM(sigalg_lookup_tbl); lu++, i++) { if (lu->sigalg == sigalg) { if (!lu->enabled) @@ -1616,7 +1229,6 @@ static const SIGALG_LOOKUP *tls1_lookup_sigalg(const SSL_CONNECTION *s, int tls1_lookup_md(SSL_CTX *ctx, const SIGALG_LOOKUP *lu, const EVP_MD **pmd) { const EVP_MD *md; - if (lu == NULL) return 0; /* lu->hash == NID_undef means no associated digest */ @@ -1661,17 +1273,15 @@ static int rsa_pss_check_min_key_size(SSL_CTX *ctx, const EVP_PKEY *pkey, * certificate type from |s| will be used. * Returns the signature algorithm to use, or NULL on error. */ -static const SIGALG_LOOKUP *tls1_get_legacy_sigalg(const SSL_CONNECTION *s, - int idx) +static const SIGALG_LOOKUP *tls1_get_legacy_sigalg(const SSL *s, int idx) { if (idx == -1) { if (s->server) { size_t i; /* Work out index corresponding to ciphersuite */ - for (i = 0; i < s->ssl_pkey_num; i++) { - const SSL_CERT_LOOKUP *clu - = ssl_cert_lookup_by_idx(i, SSL_CONNECTION_GET_CTX(s)); + for (i = 0; i < SSL_PKEY_NUM; i++) { + const SSL_CERT_LOOKUP *clu = ssl_cert_lookup_by_idx(i); if (clu == NULL) continue; @@ -1680,49 +1290,18 @@ static const SIGALG_LOOKUP *tls1_get_legacy_sigalg(const SSL_CONNECTION *s, break; } } - - /* - * Some GOST ciphersuites allow more than one signature algorithms - * */ - if (idx == SSL_PKEY_GOST01 && s->s3.tmp.new_cipher->algorithm_auth != SSL_aGOST01) { - int real_idx; - - for (real_idx = SSL_PKEY_GOST12_512; real_idx >= SSL_PKEY_GOST01; - real_idx--) { - if (s->cert->pkeys[real_idx].privatekey != NULL) { - idx = real_idx; - break; - } - } - } - /* - * As both SSL_PKEY_GOST12_512 and SSL_PKEY_GOST12_256 indices can be used - * with new (aGOST12-only) ciphersuites, we should find out which one is available really. - */ - else if (idx == SSL_PKEY_GOST12_256) { - int real_idx; - - for (real_idx = SSL_PKEY_GOST12_512; real_idx >= SSL_PKEY_GOST12_256; - real_idx--) { - if (s->cert->pkeys[real_idx].privatekey != NULL) { - idx = real_idx; - break; - } - } - } - } else { - idx = s->cert->key - s->cert->pkeys; - } - } - if (idx < 0 || idx >= (int)OSSL_NELEM(tls_default_sigalg)) - return NULL; - - if (SSL_USE_SIGALGS(s) || idx != SSL_PKEY_RSA) { - const SIGALG_LOOKUP *lu = tls1_lookup_sigalg(s, tls_default_sigalg[idx]); + } else { + idx = s->cert->key - s->cert->pkeys; + } + } + if (idx < 0 || idx >= (int)OSSL_NELEM(tls_default_sigalg)) + return NULL; + if (SSL_USE_SIGALGS(s) || idx != SSL_PKEY_RSA) { + const SIGALG_LOOKUP *lu = tls1_lookup_sigalg(s, tls_default_sigalg[idx]); if (lu == NULL) return NULL; - if (!tls1_lookup_md(SSL_CONNECTION_GET_CTX(s), lu, NULL)) + if (!tls1_lookup_md(s->ctx, lu, NULL)) return NULL; if (!tls12_sigalg_allowed(s, SSL_SECOP_SIGALG_SUPPORTED, lu)) return NULL; @@ -1733,12 +1312,24 @@ static const SIGALG_LOOKUP *tls1_get_legacy_sigalg(const SSL_CONNECTION *s, return &legacy_rsa_sigalg; } /* Set peer sigalg based key type */ -int tls1_set_peer_legacy_sigalg(SSL_CONNECTION *s, const EVP_PKEY *pkey) +int tls1_set_peer_legacy_sigalg(SSL *s, const EVP_PKEY *pkey) { size_t idx; const SIGALG_LOOKUP *lu; - if (ssl_cert_lookup_by_pkey(pkey, &idx, SSL_CONNECTION_GET_CTX(s)) == NULL) +#ifndef OPENSSL_NO_NTLS + if (SSL_IS_NTLS(s)) { + if (EVP_PKEY_is_a(pkey, "SM2")) + s->s3.tmp.peer_sigalg = &ntls_sm2_sigalg; + else if (EVP_PKEY_is_a(pkey, "RSA")) + s->s3.tmp.peer_sigalg = &ntls_rsa_sigalg; + else + return 0; + + return 1; + } +#endif + if (ssl_cert_lookup_by_pkey(pkey, &idx) == NULL) return 0; lu = tls1_get_legacy_sigalg(s, idx); if (lu == NULL) @@ -1747,7 +1338,7 @@ int tls1_set_peer_legacy_sigalg(SSL_CONNECTION *s, const EVP_PKEY *pkey) return 1; } -size_t tls12_get_psigalgs(SSL_CONNECTION *s, int sent, const uint16_t **psigs) +size_t tls12_get_psigalgs(SSL *s, int sent, const uint16_t **psigs) { /* * If Suite B mode use Suite B sigalgs only, ignore any other @@ -1778,8 +1369,8 @@ size_t tls12_get_psigalgs(SSL_CONNECTION *s, int sent, const uint16_t **psigs) *psigs = s->cert->conf_sigalgs; return s->cert->conf_sigalgslen; } else { - *psigs = SSL_CONNECTION_GET_CTX(s)->tls12_sigalgs; - return SSL_CONNECTION_GET_CTX(s)->tls12_sigalgs_len; + *psigs = tls12_sigalgs; + return OSSL_NELEM(tls12_sigalgs); } } @@ -1787,7 +1378,7 @@ size_t tls12_get_psigalgs(SSL_CONNECTION *s, int sent, const uint16_t **psigs) * Called by servers only. Checks that we have a sig alg that supports the * specified EC curve. */ -int tls_check_sigalg_curve(const SSL_CONNECTION *s, int curve) +int tls_check_sigalg_curve(const SSL *s, int curve) { const uint16_t *sigs; size_t siglen, i; @@ -1796,8 +1387,8 @@ int tls_check_sigalg_curve(const SSL_CONNECTION *s, int curve) sigs = s->cert->conf_sigalgs; siglen = s->cert->conf_sigalgslen; } else { - sigs = SSL_CONNECTION_GET_CTX(s)->tls12_sigalgs; - siglen = SSL_CONNECTION_GET_CTX(s)->tls12_sigalgs_len; + sigs = tls12_sigalgs; + siglen = OSSL_NELEM(tls12_sigalgs); } for (i = 0; i < siglen; i++) { @@ -1854,14 +1445,6 @@ static int sigalg_security_bits(SSL_CTX *ctx, const SIGALG_LOOKUP *lu) else if (lu->sigalg == TLSEXT_SIGALG_ed448) secbits = 224; } - /* - * For provider-based sigalgs we have secbits information available - * in the (provider-loaded) sigalg_list structure - */ - if ((secbits == 0) && (lu->sig_idx >= SSL_PKEY_NUM) - && ((lu->sig_idx - SSL_PKEY_NUM) < (int)ctx->sigalg_list_len)) { - secbits = ctx->sigalg_list[lu->sig_idx - SSL_PKEY_NUM].secbits; - } return secbits; } @@ -1870,7 +1453,7 @@ static int sigalg_security_bits(SSL_CTX *ctx, const SIGALG_LOOKUP *lu) * algorithms and if so set relevant digest and signature scheme in * s. */ -int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t sig, EVP_PKEY *pkey) +int tls12_check_peer_sigalg(SSL *s, uint16_t sig, EVP_PKEY *pkey) { const uint16_t *sent_sigs; const EVP_MD *md = NULL; @@ -1881,8 +1464,29 @@ int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t sig, EVP_PKEY *pkey) int secbits = 0; pkeyid = EVP_PKEY_get_id(pkey); + /* Should never happen */ + if (pkeyid == -1) + return -1; + if (SSL_IS_TLS13(s)) { +#ifndef OPENSSL_NO_SM2 + /* + * RFC 8998 requires that if TLS_SM4_GCM_SM3 or TLS_SM4_CCM_SM3 was + * choosen, the only valid signature algorithm MUST be "sm2sig_sm3". + */ + if (s->enable_sm_tls13_strict == 1) { + const SSL_CIPHER *cipher = s->s3.tmp.new_cipher; + + if (cipher != NULL && (cipher->id == TLS1_3_CK_SM4_GCM_SM3 + || cipher->id == TLS1_3_CK_SM4_CCM_SM3)) { + if (sig != TLSEXT_SIGALG_sm2sig_sm3) { + SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_WRONG_SIGNATURE_TYPE); + return 0; + } + } + } +#endif - if (SSL_CONNECTION_IS_TLS13(s)) { /* Disallow DSA for TLS 1.3 */ if (pkeyid == EVP_PKEY_DSA) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_SIGNATURE_TYPE); @@ -1893,30 +1497,19 @@ int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t sig, EVP_PKEY *pkey) pkeyid = EVP_PKEY_RSA_PSS; } lu = tls1_lookup_sigalg(s, sig); - /* if this sigalg is loaded, set so far unknown pkeyid to its sig NID */ - if ((pkeyid == EVP_PKEY_KEYMGMT) && (lu != NULL)) - pkeyid = lu->sig; - - /* Should never happen */ - if (pkeyid == -1) - return -1; - /* * Check sigalgs is known. Disallow SHA1/SHA224 with TLS 1.3. Check key type * is consistent with signature: RSA keys can be used for RSA-PSS */ if (lu == NULL - || (SSL_CONNECTION_IS_TLS13(s) - && (lu->hash == NID_sha1 || lu->hash == NID_sha224)) - || (pkeyid != lu->sig + || (SSL_IS_TLS13(s) && (lu->hash == NID_sha1 || lu->hash == NID_sha224)) + || (pkeyid != lu->sig && lu->sig != EVP_PKEY_SM2 && (lu->sig != EVP_PKEY_RSA_PSS || pkeyid != EVP_PKEY_RSA))) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_SIGNATURE_TYPE); return 0; } /* Check the sigalg is consistent with the key OID */ - if (!ssl_cert_lookup_by_nid( - (pkeyid == EVP_PKEY_RSA_PSS) ? EVP_PKEY_get_id(pkey) : pkeyid, - &cidx, SSL_CONNECTION_GET_CTX(s)) + if (!ssl_cert_lookup_by_nid(EVP_PKEY_get_id(pkey), &cidx) || lu->sig_idx != (int)cidx) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_SIGNATURE_TYPE); return 0; @@ -1932,7 +1525,7 @@ int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t sig, EVP_PKEY *pkey) } /* For TLS 1.3 or Suite B check curve matches signature algorithm */ - if (SSL_CONNECTION_IS_TLS13(s) || tls1_suiteb(s)) { + if (SSL_IS_TLS13(s) || tls1_suiteb(s)) { int curve = ssl_get_EC_curve_nid(pkey); if (lu->curve != NID_undef && curve != lu->curve) { @@ -1940,7 +1533,7 @@ int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t sig, EVP_PKEY *pkey) return 0; } } - if (!SSL_CONNECTION_IS_TLS13(s)) { + if (!SSL_IS_TLS13(s)) { /* Check curve matches extensions */ if (!tls1_check_group_id(s, tls1_get_group_id(pkey), 1)) { SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_WRONG_CURVE); @@ -1973,7 +1566,7 @@ int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t sig, EVP_PKEY *pkey) SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_WRONG_SIGNATURE_TYPE); return 0; } - if (!tls1_lookup_md(SSL_CONNECTION_GET_CTX(s), lu, &md)) { + if (!tls1_lookup_md(s->ctx, lu, &md)) { SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_R_UNKNOWN_DIGEST); return 0; } @@ -1983,7 +1576,7 @@ int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t sig, EVP_PKEY *pkey) */ sigalgstr[0] = (sig >> 8) & 0xff; sigalgstr[1] = sig & 0xff; - secbits = sigalg_security_bits(SSL_CONNECTION_GET_CTX(s), lu); + secbits = sigalg_security_bits(s->ctx, lu); if (secbits == 0 || !ssl_security(s, SSL_SECOP_SIGALG_CHECK, secbits, md != NULL ? EVP_MD_get_type(md) : NID_undef, @@ -1998,27 +1591,17 @@ int tls12_check_peer_sigalg(SSL_CONNECTION *s, uint16_t sig, EVP_PKEY *pkey) int SSL_get_peer_signature_type_nid(const SSL *s, int *pnid) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->s3.tmp.peer_sigalg == NULL) + if (s->s3.tmp.peer_sigalg == NULL) return 0; - *pnid = sc->s3.tmp.peer_sigalg->sig; + *pnid = s->s3.tmp.peer_sigalg->sig; return 1; } int SSL_get_signature_type_nid(const SSL *s, int *pnid) { - const SSL_CONNECTION *sc = SSL_CONNECTION_FROM_CONST_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->s3.tmp.sigalg == NULL) + if (s->s3.tmp.sigalg == NULL) return 0; - *pnid = sc->s3.tmp.sigalg->sig; + *pnid = s->s3.tmp.sigalg->sig; return 1; } @@ -2032,7 +1615,7 @@ int SSL_get_signature_type_nid(const SSL *s, int *pnid) * * Call ssl_cipher_disabled() to check that it's enabled or not. */ -int ssl_set_client_disabled(SSL_CONNECTION *s) +int ssl_set_client_disabled(SSL *s) { s->s3.tmp.mask_a = 0; s->s3.tmp.mask_k = 0; @@ -2065,53 +1648,51 @@ int ssl_set_client_disabled(SSL_CONNECTION *s) * * Returns 1 when it's disabled, 0 when enabled. */ -int ssl_cipher_disabled(const SSL_CONNECTION *s, const SSL_CIPHER *c, - int op, int ecdhe) +int ssl_cipher_disabled(const SSL *s, const SSL_CIPHER *c, int op, int ecdhe) { - int minversion = SSL_CONNECTION_IS_DTLS(s) ? c->min_dtls : c->min_tls; - int maxversion = SSL_CONNECTION_IS_DTLS(s) ? c->max_dtls : c->max_tls; - if (c->algorithm_mkey & s->s3.tmp.mask_k || c->algorithm_auth & s->s3.tmp.mask_a) return 1; if (s->s3.tmp.max_ver == 0) return 1; + if (!SSL_IS_DTLS(s)) { + int min_tls = c->min_tls; - if (SSL_IS_QUIC_HANDSHAKE(s)) - /* For QUIC, only allow these ciphersuites. */ - switch (SSL_CIPHER_get_id(c)) { - case TLS1_3_CK_AES_128_GCM_SHA256: - case TLS1_3_CK_AES_256_GCM_SHA384: - case TLS1_3_CK_CHACHA20_POLY1305_SHA256: - break; - default: - return 1; - } +#ifndef OPENSSL_NO_NTLS + /* + * NTLS cipher can only use in NTLS + */ + if (min_tls == NTLS_VERSION) + if (s->s3.tmp.max_ver != NTLS_VERSION) + return 1; +#endif - /* - * For historical reasons we will allow ECHDE to be selected by a server - * in SSLv3 if we are a client - */ - if (minversion == TLS1_VERSION - && ecdhe - && (c->algorithm_mkey & (SSL_kECDHE | SSL_kECDHEPSK)) != 0) - minversion = SSL3_VERSION; + /* + * For historical reasons we will allow ECHDE to be selected by a server + * in SSLv3 if we are a client + */ + if (min_tls == TLS1_VERSION && ecdhe + && (c->algorithm_mkey & (SSL_kECDHE | SSL_kECDHEPSK)) != 0) + min_tls = SSL3_VERSION; - if (ssl_version_cmp(s, minversion, s->s3.tmp.max_ver) > 0 - || ssl_version_cmp(s, maxversion, s->s3.tmp.min_ver) < 0) + if ((min_tls > s->s3.tmp.max_ver) || (c->max_tls < s->s3.tmp.min_ver)) + return 1; + } + if (SSL_IS_DTLS(s) && (DTLS_VERSION_GT(c->min_dtls, s->s3.tmp.max_ver) + || DTLS_VERSION_LT(c->max_dtls, s->s3.tmp.min_ver))) return 1; return !ssl_security(s, op, c->strength_bits, 0, (void *)c); } -int tls_use_ticket(SSL_CONNECTION *s) +int tls_use_ticket(SSL *s) { if ((s->options & SSL_OP_NO_TICKET)) return 0; return ssl_security(s, SSL_SECOP_TICKET, 0, 0, NULL); } -int tls1_set_server_sigalgs(SSL_CONNECTION *s) +int tls1_set_server_sigalgs(SSL *s) { size_t i; @@ -2119,14 +1700,9 @@ int tls1_set_server_sigalgs(SSL_CONNECTION *s) OPENSSL_free(s->shared_sigalgs); s->shared_sigalgs = NULL; s->shared_sigalgslen = 0; - /* Clear certificate validity flags */ - if (s->s3.tmp.valid_flags) - memset(s->s3.tmp.valid_flags, 0, s->ssl_pkey_num * sizeof(uint32_t)); - else - s->s3.tmp.valid_flags = OPENSSL_zalloc(s->ssl_pkey_num * sizeof(uint32_t)); - if (s->s3.tmp.valid_flags == NULL) - return 0; + for (i = 0; i < SSL_PKEY_NUM; i++) + s->s3.tmp.valid_flags[i] = 0; /* * If peer sent no signature algorithms check to see if we support * the default algorithm for each certificate type @@ -2136,7 +1712,7 @@ int tls1_set_server_sigalgs(SSL_CONNECTION *s) const uint16_t *sent_sigs; size_t sent_sigslen = tls12_get_psigalgs(s, 1, &sent_sigs); - for (i = 0; i < s->ssl_pkey_num; i++) { + for (i = 0; i < SSL_PKEY_NUM; i++) { const SIGALG_LOOKUP *lu = tls1_get_legacy_sigalg(s, i); size_t j; @@ -2173,8 +1749,7 @@ int tls1_set_server_sigalgs(SSL_CONNECTION *s) * ret: (output) on return, if a ticket was decrypted, then this is set to * point to the resulting session. */ -SSL_TICKET_STATUS tls_get_ticket_from_client(SSL_CONNECTION *s, - CLIENTHELLO_MSG *hello, +SSL_TICKET_STATUS tls_get_ticket_from_client(SSL *s, CLIENTHELLO_MSG *hello, SSL_SESSION **ret) { size_t size; @@ -2188,7 +1763,16 @@ SSL_TICKET_STATUS tls_get_ticket_from_client(SSL_CONNECTION *s, * (e.g. TLSv1.3) behave as if no ticket present to permit stateful * resumption. */ - if (s->version <= SSL3_VERSION || !tls_use_ticket(s)) + if ( +#ifndef OPENSSL_NO_NTLS + ( +#endif + s->version <= SSL3_VERSION +#ifndef OPENSSL_NO_NTLS + && !SSL_IS_NTLS(s) + ) +#endif + || !tls_use_ticket(s)) return SSL_TICKET_NONE; ticketext = &hello->pre_proc_exts[TLSEXT_IDX_session_ticket]; @@ -2224,23 +1808,20 @@ SSL_TICKET_STATUS tls_get_ticket_from_client(SSL_CONNECTION *s, * psess: (output) on return, if a ticket was decrypted, then this is set to * point to the resulting session. */ -SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, - const unsigned char *etick, - size_t eticklen, - const unsigned char *sess_id, +SSL_TICKET_STATUS tls_decrypt_ticket(SSL *s, const unsigned char *etick, + size_t eticklen, const unsigned char *sess_id, size_t sesslen, SSL_SESSION **psess) { SSL_SESSION *sess = NULL; unsigned char *sdec; const unsigned char *p; - int slen, ivlen, renew_ticket = 0, declen; + int slen, renew_ticket = 0, declen; SSL_TICKET_STATUS ret = SSL_TICKET_FATAL_ERR_OTHER; size_t mlen; unsigned char tick_hmac[EVP_MAX_MD_SIZE]; SSL_HMAC *hctx = NULL; EVP_CIPHER_CTX *ctx = NULL; SSL_CTX *tctx = s->session_ctx; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (eticklen == 0) { /* @@ -2250,7 +1831,7 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, ret = SSL_TICKET_EMPTY; goto end; } - if (!SSL_CONNECTION_IS_TLS13(s) && s->ext.session_secret_cb) { + if (!SSL_IS_TLS13(s) && s->ext.session_secret_cb) { /* * Indicate that the ticket couldn't be decrypted rather than * generating the session from ticket now, trigger @@ -2288,7 +1869,7 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, int rv = 0; if (tctx->ext.ticket_key_evp_cb != NULL) - rv = tctx->ext.ticket_key_evp_cb(SSL_CONNECTION_GET_SSL(s), nctick, + rv = tctx->ext.ticket_key_evp_cb(s, nctick, nctick + TLSEXT_KEYNAME_LENGTH, ctx, ssl_hmac_get0_EVP_MAC_CTX(hctx), @@ -2296,7 +1877,7 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, #ifndef OPENSSL_NO_DEPRECATED_3_0 else if (tctx->ext.ticket_key_cb != NULL) /* if 0 is returned, write an empty ticket */ - rv = tctx->ext.ticket_key_cb(SSL_CONNECTION_GET_SSL(s), nctick, + rv = tctx->ext.ticket_key_cb(s, nctick, nctick + TLSEXT_KEYNAME_LENGTH, ctx, ssl_hmac_get0_HMAC_CTX(hctx), 0); #endif @@ -2320,8 +1901,8 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, goto end; } - aes256cbc = EVP_CIPHER_fetch(sctx->libctx, "AES-256-CBC", - sctx->propq); + aes256cbc = EVP_CIPHER_fetch(s->ctx->libctx, "AES-256-CBC", + s->ctx->propq); if (aes256cbc == NULL || ssl_hmac_init(hctx, tctx->ext.secure->tick_hmac_key, sizeof(tctx->ext.secure->tick_hmac_key), @@ -2334,7 +1915,7 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, goto end; } EVP_CIPHER_free(aes256cbc); - if (SSL_CONNECTION_IS_TLS13(s)) + if (SSL_IS_TLS13(s)) renew_ticket = 1; } /* @@ -2347,14 +1928,9 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, goto end; } - ivlen = EVP_CIPHER_CTX_get_iv_length(ctx); - if (ivlen < 0) { - ret = SSL_TICKET_FATAL_ERR_OTHER; - goto end; - } - /* Sanity check ticket length: must exceed keyname + IV + HMAC */ - if (eticklen <= TLSEXT_KEYNAME_LENGTH + ivlen + mlen) { + if (eticklen <= + TLSEXT_KEYNAME_LENGTH + EVP_CIPHER_CTX_get_iv_length(ctx) + mlen) { ret = SSL_TICKET_NO_DECRYPT; goto end; } @@ -2372,8 +1948,8 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, } /* Attempt to decrypt session data */ /* Move p after IV to start of encrypted ticket, update length */ - p = etick + TLSEXT_KEYNAME_LENGTH + ivlen; - eticklen -= TLSEXT_KEYNAME_LENGTH + ivlen; + p = etick + TLSEXT_KEYNAME_LENGTH + EVP_CIPHER_CTX_get_iv_length(ctx); + eticklen -= TLSEXT_KEYNAME_LENGTH + EVP_CIPHER_CTX_get_iv_length(ctx); sdec = OPENSSL_malloc(eticklen); if (sdec == NULL || EVP_DecryptUpdate(ctx, sdec, &slen, p, (int)eticklen) <= 0) { @@ -2389,7 +1965,7 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, slen += declen; p = sdec; - sess = d2i_SSL_SESSION_ex(NULL, &p, slen, sctx->libctx, sctx->propq); + sess = d2i_SSL_SESSION(NULL, &p, slen); slen -= p - sdec; OPENSSL_free(sdec); if (sess) { @@ -2441,8 +2017,7 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, if (keyname_len > TLSEXT_KEYNAME_LENGTH) keyname_len = TLSEXT_KEYNAME_LENGTH; - retcb = s->session_ctx->decrypt_ticket_cb(SSL_CONNECTION_GET_SSL(s), - sess, etick, keyname_len, + retcb = s->session_ctx->decrypt_ticket_cb(s, sess, etick, keyname_len, ret, s->session_ctx->ticket_cb_data); switch (retcb) { @@ -2480,7 +2055,7 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, } } - if (s->ext.session_secret_cb == NULL || SSL_CONNECTION_IS_TLS13(s)) { + if (s->ext.session_secret_cb == NULL || SSL_IS_TLS13(s)) { switch (ret) { case SSL_TICKET_NO_DECRYPT: case SSL_TICKET_SUCCESS_RENEW: @@ -2495,8 +2070,7 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, } /* Check to see if a signature algorithm is allowed */ -static int tls12_sigalg_allowed(const SSL_CONNECTION *s, int op, - const SIGALG_LOOKUP *lu) +static int tls12_sigalg_allowed(const SSL *s, int op, const SIGALG_LOOKUP *lu) { unsigned char sigalgstr[2]; int secbits; @@ -2504,64 +2078,24 @@ static int tls12_sigalg_allowed(const SSL_CONNECTION *s, int op, if (lu == NULL || !lu->enabled) return 0; /* DSA is not allowed in TLS 1.3 */ - if (SSL_CONNECTION_IS_TLS13(s) && lu->sig == EVP_PKEY_DSA) + if (SSL_IS_TLS13(s) && lu->sig == EVP_PKEY_DSA) return 0; /* * At some point we should fully axe DSA/etc. in ClientHello as per TLS 1.3 * spec */ - if (!s->server && !SSL_CONNECTION_IS_DTLS(s) - && s->s3.tmp.min_ver >= TLS1_3_VERSION + if (!s->server && !SSL_IS_DTLS(s) && s->s3.tmp.min_ver >= TLS1_3_VERSION && (lu->sig == EVP_PKEY_DSA || lu->hash_idx == SSL_MD_SHA1_IDX || lu->hash_idx == SSL_MD_MD5_IDX || lu->hash_idx == SSL_MD_SHA224_IDX)) return 0; /* See if public key algorithm allowed */ - if (ssl_cert_is_disabled(SSL_CONNECTION_GET_CTX(s), lu->sig_idx)) + if (ssl_cert_is_disabled(s->ctx, lu->sig_idx)) return 0; - if (lu->sig == NID_id_GostR3410_2012_256 - || lu->sig == NID_id_GostR3410_2012_512 - || lu->sig == NID_id_GostR3410_2001) { - /* We never allow GOST sig algs on the server with TLSv1.3 */ - if (s->server && SSL_CONNECTION_IS_TLS13(s)) - return 0; - if (!s->server - && SSL_CONNECTION_GET_SSL(s)->method->version == TLS_ANY_VERSION - && s->s3.tmp.max_ver >= TLS1_3_VERSION) { - int i, num; - STACK_OF(SSL_CIPHER) *sk; - - /* - * We're a client that could negotiate TLSv1.3. We only allow GOST - * sig algs if we could negotiate TLSv1.2 or below and we have GOST - * ciphersuites enabled. - */ - - if (s->s3.tmp.min_ver >= TLS1_3_VERSION) - return 0; - - sk = SSL_get_ciphers(SSL_CONNECTION_GET_SSL(s)); - num = sk != NULL ? sk_SSL_CIPHER_num(sk) : 0; - for (i = 0; i < num; i++) { - const SSL_CIPHER *c; - - c = sk_SSL_CIPHER_value(sk, i); - /* Skip disabled ciphers */ - if (ssl_cipher_disabled(s, c, SSL_SECOP_CIPHER_SUPPORTED, 0)) - continue; - - if ((c->algorithm_mkey & (SSL_kGOST | SSL_kGOST18)) != 0) - break; - } - if (i == num) - return 0; - } - } - /* Finally see if security callback allows it */ - secbits = sigalg_security_bits(SSL_CONNECTION_GET_CTX(s), lu); + secbits = sigalg_security_bits(s->ctx, lu); sigalgstr[0] = (lu->sigalg >> 8) & 0xff; sigalgstr[1] = lu->sigalg & 0xff; return ssl_security(s, op, secbits, lu->hash, (void *)sigalgstr); @@ -2573,7 +2107,7 @@ static int tls12_sigalg_allowed(const SSL_CONNECTION *s, int op, * disabled. */ -void ssl_set_sig_mask(uint32_t *pmask_a, SSL_CONNECTION *s, int op) +void ssl_set_sig_mask(uint32_t *pmask_a, SSL *s, int op) { const uint16_t *sigalgs; size_t i, sigalgslen; @@ -2590,8 +2124,7 @@ void ssl_set_sig_mask(uint32_t *pmask_a, SSL_CONNECTION *s, int op) if (lu == NULL) continue; - clu = ssl_cert_lookup_by_idx(lu->sig_idx, - SSL_CONNECTION_GET_CTX(s)); + clu = ssl_cert_lookup_by_idx(lu->sig_idx); if (clu == NULL) continue; @@ -2603,7 +2136,7 @@ void ssl_set_sig_mask(uint32_t *pmask_a, SSL_CONNECTION *s, int op) *pmask_a |= disabled_mask; } -int tls12_copy_sigalgs(SSL_CONNECTION *s, WPACKET *pkt, +int tls12_copy_sigalgs(SSL *s, WPACKET *pkt, const uint16_t *psig, size_t psiglen) { size_t i; @@ -2615,13 +2148,32 @@ int tls12_copy_sigalgs(SSL_CONNECTION *s, WPACKET *pkt, if (lu == NULL || !tls12_sigalg_allowed(s, SSL_SECOP_SIGALG_SUPPORTED, lu)) continue; + +#ifndef OPENSSL_NO_SM2 + /* + * RFC 8998 requires that + * if the server chooses TLS_SM4_GCM_SM3 or TLS_SM4_CCM_SM3, + * the only valid signature algorithm present in + * "signature_algorithms" extension MUST be "sm2sig_sm3". + */ + if (SSL_IS_TLS13(s) && s->enable_sm_tls13_strict == 1 && s->server) { + const SSL_CIPHER *cipher = s->s3.tmp.new_cipher; + + if (cipher != NULL && + (cipher->id == TLS1_3_CK_SM4_GCM_SM3 + || cipher->id == TLS1_3_CK_SM4_CCM_SM3)) { + if (lu->sigalg != TLSEXT_SIGALG_sm2sig_sm3) + continue; + } + } +#endif if (!WPACKET_put_bytes_u16(pkt, *psig)) return 0; /* * If TLS 1.3 must have at least one valid TLS 1.3 message * signing algorithm: i.e. neither RSA nor SHA1/SHA224 */ - if (rv == 0 && (!SSL_CONNECTION_IS_TLS13(s) + if (rv == 0 && (!SSL_IS_TLS13(s) || (lu->sig != EVP_PKEY_RSA && lu->hash != NID_sha1 && lu->hash != NID_sha224))) @@ -2633,8 +2185,7 @@ int tls12_copy_sigalgs(SSL_CONNECTION *s, WPACKET *pkt, } /* Given preference and allowed sigalgs set shared sigalgs */ -static size_t tls12_shared_sigalgs(SSL_CONNECTION *s, - const SIGALG_LOOKUP **shsig, +static size_t tls12_shared_sigalgs(SSL *s, const SIGALG_LOOKUP **shsig, const uint16_t *pref, size_t preflen, const uint16_t *allow, size_t allowlen) { @@ -2660,7 +2211,7 @@ static size_t tls12_shared_sigalgs(SSL_CONNECTION *s, } /* Set shared signature algorithms for SSL structures */ -static int tls1_set_shared_sigalgs(SSL_CONNECTION *s) +static int tls1_set_shared_sigalgs(SSL *s) { const uint16_t *pref, *allow, *conf; size_t preflen, allowlen, conflen; @@ -2694,8 +2245,10 @@ static int tls1_set_shared_sigalgs(SSL_CONNECTION *s) } nmatch = tls12_shared_sigalgs(s, NULL, pref, preflen, allow, allowlen); if (nmatch) { - if ((salgs = OPENSSL_malloc(nmatch * sizeof(*salgs))) == NULL) + if ((salgs = OPENSSL_malloc(nmatch * sizeof(*salgs))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } nmatch = tls12_shared_sigalgs(s, salgs, pref, preflen, allow, allowlen); } else { salgs = NULL; @@ -2719,8 +2272,10 @@ int tls1_save_u16(PACKET *pkt, uint16_t **pdest, size_t *pdestlen) size >>= 1; - if ((buf = OPENSSL_malloc(size * sizeof(*buf))) == NULL) + if ((buf = OPENSSL_malloc(size * sizeof(*buf))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } for (i = 0; i < size && PACKET_get_net_2(pkt, &stmp); i++) buf[i] = stmp; @@ -2736,7 +2291,7 @@ int tls1_save_u16(PACKET *pkt, uint16_t **pdest, size_t *pdestlen) return 1; } -int tls1_save_sigalgs(SSL_CONNECTION *s, PACKET *pkt, int cert) +int tls1_save_sigalgs(SSL *s, PACKET *pkt, int cert) { /* Extension ignored for inappropriate versions */ if (!SSL_USE_SIGALGS(s)) @@ -2756,7 +2311,7 @@ int tls1_save_sigalgs(SSL_CONNECTION *s, PACKET *pkt, int cert) /* Set preferred digest for each key type */ -int tls1_process_sigalgs(SSL_CONNECTION *s) +int tls1_process_sigalgs(SSL *s) { size_t i; uint32_t *pvalid = s->s3.tmp.valid_flags; @@ -2764,7 +2319,7 @@ int tls1_process_sigalgs(SSL_CONNECTION *s) if (!tls1_set_shared_sigalgs(s)) return 0; - for (i = 0; i < s->ssl_pkey_num; i++) + for (i = 0; i < SSL_PKEY_NUM; i++) pvalid[i] = 0; for (i = 0; i < s->shared_sigalgslen; i++) { @@ -2772,11 +2327,10 @@ int tls1_process_sigalgs(SSL_CONNECTION *s) int idx = sigptr->sig_idx; /* Ignore PKCS1 based sig algs in TLSv1.3 */ - if (SSL_CONNECTION_IS_TLS13(s) && sigptr->sig == EVP_PKEY_RSA) + if (SSL_IS_TLS13(s) && sigptr->sig == EVP_PKEY_RSA) continue; /* If not disabled indicate we can explicitly sign */ - if (pvalid[idx] == 0 - && !ssl_cert_is_disabled(SSL_CONNECTION_GET_CTX(s), idx)) + if (pvalid[idx] == 0 && !ssl_cert_is_disabled(s->ctx, idx)) pvalid[idx] = CERT_PKEY_EXPLICIT_SIGN | CERT_PKEY_SIGN; } return 1; @@ -2786,16 +2340,8 @@ int SSL_get_sigalgs(SSL *s, int idx, int *psign, int *phash, int *psignhash, unsigned char *rsig, unsigned char *rhash) { - uint16_t *psig; - size_t numsigalgs; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - psig = sc->s3.tmp.peer_sigalgs; - numsigalgs = sc->s3.tmp.peer_sigalgslen; - + uint16_t *psig = s->s3.tmp.peer_sigalgs; + size_t numsigalgs = s->s3.tmp.peer_sigalgslen; if (psig == NULL || numsigalgs > INT_MAX) return 0; if (idx >= 0) { @@ -2808,7 +2354,7 @@ int SSL_get_sigalgs(SSL *s, int idx, *rhash = (unsigned char)((*psig >> 8) & 0xff); if (rsig != NULL) *rsig = (unsigned char)(*psig & 0xff); - lu = tls1_lookup_sigalg(sc, *psig); + lu = tls1_lookup_sigalg(s, *psig); if (psign != NULL) *psign = lu != NULL ? lu->sig : NID_undef; if (phash != NULL) @@ -2824,17 +2370,12 @@ int SSL_get_shared_sigalgs(SSL *s, int idx, unsigned char *rsig, unsigned char *rhash) { const SIGALG_LOOKUP *shsigalgs; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - if (sc->shared_sigalgs == NULL + if (s->shared_sigalgs == NULL || idx < 0 - || idx >= (int)sc->shared_sigalgslen - || sc->shared_sigalgslen > INT_MAX) + || idx >= (int)s->shared_sigalgslen + || s->shared_sigalgslen > INT_MAX) return 0; - shsigalgs = sc->shared_sigalgs[idx]; + shsigalgs = s->shared_sigalgs[idx]; if (phash != NULL) *phash = shsigalgs->hash; if (psign != NULL) @@ -2845,7 +2386,7 @@ int SSL_get_shared_sigalgs(SSL *s, int idx, *rsig = (unsigned char)(shsigalgs->sigalg & 0xff); if (rhash != NULL) *rhash = (unsigned char)((shsigalgs->sigalg >> 8) & 0xff); - return (int)sc->shared_sigalgslen; + return (int)s->shared_sigalgslen; } /* Maximum possible number of unique entries in sigalgs array */ @@ -2855,7 +2396,6 @@ typedef struct { size_t sigalgcnt; /* TLSEXT_SIGALG_XXX values */ uint16_t sigalgs[TLS_MAX_SIGALGCNT]; - SSL_CTX *ctx; } sig_cb_st; static void get_sigorhash(int *psig, int *phash, const char *str) @@ -2880,19 +2420,12 @@ static void get_sigorhash(int *psig, int *phash, const char *str) static int sig_cb(const char *elem, int len, void *arg) { sig_cb_st *sarg = arg; - size_t i = 0; + size_t i; const SIGALG_LOOKUP *s; char etmp[TLS_MAX_SIGSTRING_LEN], *p; int sig_alg = NID_undef, hash_alg = NID_undef; - int ignore_unknown = 0; - if (elem == NULL) return 0; - if (elem[0] == '?') { - ignore_unknown = 1; - ++elem; - --len; - } if (sarg->sigalgcnt == TLS_MAX_SIGALGCNT) return 0; if (len > (int)(sizeof(etmp) - 1)) @@ -2910,33 +2443,15 @@ static int sig_cb(const char *elem, int len, void *arg) * in the table. */ if (p == NULL) { - /* Load provider sigalgs */ - if (sarg->ctx != NULL) { - /* Check if a provider supports the sigalg */ - for (i = 0; i < sarg->ctx->sigalg_list_len; i++) { - if (sarg->ctx->sigalg_list[i].sigalg_name != NULL - && strcmp(etmp, - sarg->ctx->sigalg_list[i].sigalg_name) == 0) { - sarg->sigalgs[sarg->sigalgcnt++] = - sarg->ctx->sigalg_list[i].code_point; - break; - } - } - } - /* Check the built-in sigalgs */ - if (sarg->ctx == NULL || i == sarg->ctx->sigalg_list_len) { - for (i = 0, s = sigalg_lookup_tbl; - i < OSSL_NELEM(sigalg_lookup_tbl); i++, s++) { - if (s->name != NULL && strcmp(etmp, s->name) == 0) { - sarg->sigalgs[sarg->sigalgcnt++] = s->sigalg; - break; - } - } - if (i == OSSL_NELEM(sigalg_lookup_tbl)) { - /* Ignore unknown algorithms if ignore_unknown */ - return ignore_unknown; + for (i = 0, s = sigalg_lookup_tbl; i < OSSL_NELEM(sigalg_lookup_tbl); + i++, s++) { + if (s->name != NULL && strcmp(etmp, s->name) == 0) { + sarg->sigalgs[sarg->sigalgcnt++] = s->sigalg; + break; } } + if (i == OSSL_NELEM(sigalg_lookup_tbl)) + return 0; } else { *p = 0; p++; @@ -2944,10 +2459,8 @@ static int sig_cb(const char *elem, int len, void *arg) return 0; get_sigorhash(&sig_alg, &hash_alg, etmp); get_sigorhash(&sig_alg, &hash_alg, p); - if (sig_alg == NID_undef || hash_alg == NID_undef) { - /* Ignore unknown algorithms if ignore_unknown */ - return ignore_unknown; - } + if (sig_alg == NID_undef || hash_alg == NID_undef) + return 0; for (i = 0, s = sigalg_lookup_tbl; i < OSSL_NELEM(sigalg_lookup_tbl); i++, s++) { if (s->hash == hash_alg && s->sig == sig_alg) { @@ -2955,17 +2468,15 @@ static int sig_cb(const char *elem, int len, void *arg) break; } } - if (i == OSSL_NELEM(sigalg_lookup_tbl)) { - /* Ignore unknown algorithms if ignore_unknown */ - return ignore_unknown; - } + if (i == OSSL_NELEM(sigalg_lookup_tbl)) + return 0; } - /* Ignore duplicates */ + /* Reject duplicates */ for (i = 0; i < sarg->sigalgcnt - 1; i++) { if (sarg->sigalgs[i] == sarg->sigalgs[sarg->sigalgcnt - 1]) { sarg->sigalgcnt--; - return 1; + return 0; } } return 1; @@ -2975,21 +2486,12 @@ static int sig_cb(const char *elem, int len, void *arg) * Set supported signature algorithms based on a colon separated list of the * form sig+hash e.g. RSA+SHA512:DSA+SHA512 */ -int tls1_set_sigalgs_list(SSL_CTX *ctx, CERT *c, const char *str, int client) +int tls1_set_sigalgs_list(CERT *c, const char *str, int client) { sig_cb_st sig; sig.sigalgcnt = 0; - - if (ctx != NULL && ssl_load_sigalgs(ctx)) { - sig.ctx = ctx; - } if (!CONF_parse_list(str, ':', 1, sig_cb, &sig)) return 0; - if (sig.sigalgcnt == 0) { - ERR_raise_data(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT, - "No valid signature algorithms in '%s'", str); - return 0; - } if (c == NULL) return 1; return tls1_set_raw_sigalgs(c, sig.sigalgs, sig.sigalgcnt, client); @@ -3000,8 +2502,10 @@ int tls1_set_raw_sigalgs(CERT *c, const uint16_t *psigs, size_t salglen, { uint16_t *sigalgs; - if ((sigalgs = OPENSSL_malloc(salglen * sizeof(*sigalgs))) == NULL) + if ((sigalgs = OPENSSL_malloc(salglen * sizeof(*sigalgs))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } memcpy(sigalgs, psigs, salglen * sizeof(*sigalgs)); if (client) { @@ -3024,8 +2528,10 @@ int tls1_set_sigalgs(CERT *c, const int *psig_nids, size_t salglen, int client) if (salglen & 1) return 0; - if ((sigalgs = OPENSSL_malloc((salglen / 2) * sizeof(*sigalgs))) == NULL) + if ((sigalgs = OPENSSL_malloc((salglen / 2) * sizeof(*sigalgs))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); return 0; + } for (i = 0, sptr = sigalgs; i < salglen; i += 2) { size_t j; const SIGALG_LOOKUP *curr; @@ -3061,20 +2567,19 @@ int tls1_set_sigalgs(CERT *c, const int *psig_nids, size_t salglen, int client) return 0; } -static int tls1_check_sig_alg(SSL_CONNECTION *s, X509 *x, int default_nid) +static int tls1_check_sig_alg(SSL *s, X509 *x, int default_nid) { int sig_nid, use_pc_sigalgs = 0; size_t i; const SIGALG_LOOKUP *sigalg; size_t sigalgslen; - if (default_nid == -1) return 1; sig_nid = X509_get_signature_nid(x); if (default_nid) return sig_nid == default_nid ? 1 : 0; - if (SSL_CONNECTION_IS_TLS13(s) && s->s3.tmp.peer_cert_sigalgs != NULL) { + if (SSL_IS_TLS13(s) && s->s3.tmp.peer_cert_sigalgs != NULL) { /* * If we're in TLSv1.3 then we only get here if we're checking the * chain. If the peer has specified peer_cert_sigalgs then we use them @@ -3124,8 +2629,8 @@ static int ssl_check_ca_name(STACK_OF(X509_NAME) *names, X509 *x) (CERT_PKEY_VALID_FLAGS|CERT_PKEY_CA_SIGNATURE|CERT_PKEY_CA_PARAM \ | CERT_PKEY_ISSUER_NAME|CERT_PKEY_CERT_TYPE) -int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, - STACK_OF(X509) *chain, int idx) +int tls1_check_chain(SSL *s, X509 *x, EVP_PKEY *pk, STACK_OF(X509) *chain, + int idx) { int i; int rv = 0; @@ -3134,16 +2639,9 @@ int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, CERT *c = s->cert; uint32_t *pvalid; unsigned int suiteb_flags = tls1_suiteb(s); - - /* - * Meaning of idx: - * idx == -1 means SSL_check_chain() invocation - * idx == -2 means checking client certificate chains - * idx >= 0 means checking SSL_PKEY index - * - * For RPK, where there may be no cert, we ignore -1 - */ + /* idx == -1 means checking server chains */ if (idx != -1) { + /* idx == -2 means checking client certificate chains */ if (idx == -2) { cpk = c->key; idx = (int)(cpk - c->pkeys); @@ -3154,23 +2652,16 @@ int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, pk = cpk->privatekey; chain = cpk->chain; strict_mode = c->cert_flags & SSL_CERT_FLAGS_CHECK_TLS_STRICT; - if (tls12_rpk_and_privkey(s, idx)) { - if (EVP_PKEY_is_a(pk, "EC") && !tls1_check_pkey_comp(s, pk)) - return 0; - *pvalid = rv = CERT_PKEY_RPK; - return rv; - } /* If no cert or key, forget it */ - if (x == NULL || pk == NULL) + if (!x || !pk) goto end; } else { size_t certidx; - if (x == NULL || pk == NULL) + if (!x || !pk) return 0; - if (ssl_cert_lookup_by_pkey(pk, &certidx, - SSL_CONNECTION_GET_CTX(s)) == NULL) + if (ssl_cert_lookup_by_pkey(pk, &certidx) == NULL) return 0; idx = certidx; pvalid = s->s3.tmp.valid_flags + idx; @@ -3197,11 +2688,9 @@ int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, * Check all signature algorithms are consistent with signature * algorithms extension if TLS 1.2 or later and strict mode. */ - if (TLS1_get_version(SSL_CONNECTION_GET_SSL(s)) >= TLS1_2_VERSION - && strict_mode) { + if (TLS1_get_version(s) >= TLS1_2_VERSION && strict_mode) { int default_nid; int rsign = 0; - if (s->s3.tmp.peer_cert_sigalgs != NULL || s->s3.tmp.peer_sigalgs != NULL) { default_nid = 0; @@ -3223,21 +2712,6 @@ int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, default_nid = NID_ecdsa_with_SHA1; break; - case SSL_PKEY_GOST01: - rsign = NID_id_GostR3410_2001; - default_nid = NID_id_GostR3411_94_with_GostR3410_2001; - break; - - case SSL_PKEY_GOST12_256: - rsign = NID_id_GostR3410_2012_256; - default_nid = NID_id_tc26_signwithdigest_gost3410_2012_256; - break; - - case SSL_PKEY_GOST12_512: - rsign = NID_id_GostR3410_2012_512; - default_nid = NID_id_tc26_signwithdigest_gost3410_2012_512; - break; - default: default_nid = -1; break; @@ -3264,7 +2738,7 @@ int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, } } /* Check signature algorithm of each cert in chain */ - if (SSL_CONNECTION_IS_TLS13(s)) { + if (SSL_IS_TLS13(s)) { /* * We only get here if the application has called SSL_check_chain(), * so check_flags is always set. @@ -3341,20 +2815,22 @@ int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, ca_dn = s->s3.tmp.peer_ca_names; - if (ca_dn == NULL - || sk_X509_NAME_num(ca_dn) == 0 - || ssl_check_ca_name(ca_dn, x)) + if (!sk_X509_NAME_num(ca_dn)) rv |= CERT_PKEY_ISSUER_NAME; - else + + if (!(rv & CERT_PKEY_ISSUER_NAME)) { + if (ssl_check_ca_name(ca_dn, x)) + rv |= CERT_PKEY_ISSUER_NAME; + } + if (!(rv & CERT_PKEY_ISSUER_NAME)) { for (i = 0; i < sk_X509_num(chain); i++) { X509 *xtmp = sk_X509_value(chain, i); - if (ssl_check_ca_name(ca_dn, xtmp)) { rv |= CERT_PKEY_ISSUER_NAME; break; } } - + } if (!check_flags && !(rv & CERT_PKEY_ISSUER_NAME)) goto end; } else @@ -3365,7 +2841,7 @@ int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, end: - if (TLS1_get_version(SSL_CONNECTION_GET_SSL(s)) >= TLS1_2_VERSION) + if (TLS1_get_version(s) >= TLS1_2_VERSION) rv |= *pvalid & (CERT_PKEY_EXPLICIT_SIGN | CERT_PKEY_SIGN); else rv |= CERT_PKEY_SIGN | CERT_PKEY_EXPLICIT_SIGN; @@ -3387,31 +2863,30 @@ int tls1_check_chain(SSL_CONNECTION *s, X509 *x, EVP_PKEY *pk, } /* Set validity of certificates in an SSL structure */ -void tls1_set_cert_validity(SSL_CONNECTION *s) +void tls1_set_cert_validity(SSL *s) { tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA); tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA_PSS_SIGN); tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_DSA_SIGN); tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_ECC); - tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_GOST01); - tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_GOST12_256); - tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_GOST12_512); tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_ED25519); tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_ED448); + tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_SM2); +#ifndef OPENSSL_NO_NTLS + tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_SM2_SIGN); + tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_SM2_ENC); + tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA_SIGN); + tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA_ENC); +#endif } /* User level utility function to check a chain is suitable */ int SSL_check_chain(SSL *s, X509 *x, EVP_PKEY *pk, STACK_OF(X509) *chain) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - return tls1_check_chain(sc, x, pk, chain, -1); + return tls1_check_chain(s, x, pk, chain, -1); } -EVP_PKEY *ssl_get_auto_dh(SSL_CONNECTION *s) +EVP_PKEY *ssl_get_auto_dh(SSL *s) { EVP_PKEY *dhp = NULL; BIGNUM *p; @@ -3419,7 +2894,6 @@ EVP_PKEY *ssl_get_auto_dh(SSL_CONNECTION *s) EVP_PKEY_CTX *pctx = NULL; OSSL_PARAM_BLD *tmpl = NULL; OSSL_PARAM *params = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (s->cert->dh_tmp_auto != 2) { if (s->s3.tmp.new_cipher->algorithm_auth & (SSL_aNULL | SSL_aPSK)) { @@ -3435,8 +2909,7 @@ EVP_PKEY *ssl_get_auto_dh(SSL_CONNECTION *s) } /* Do not pick a prime that is too weak for the current security level */ - sec_level_bits = ssl_get_security_level_bits(SSL_CONNECTION_GET_SSL(s), - NULL, NULL); + sec_level_bits = ssl_get_security_level_bits(s, NULL, NULL); if (dh_secbits < sec_level_bits) dh_secbits = sec_level_bits; @@ -3453,7 +2926,7 @@ EVP_PKEY *ssl_get_auto_dh(SSL_CONNECTION *s) if (p == NULL) goto err; - pctx = EVP_PKEY_CTX_new_from_name(sctx->libctx, "DH", sctx->propq); + pctx = EVP_PKEY_CTX_new_from_name(s->ctx->libctx, "DH", s->ctx->propq); if (pctx == NULL || EVP_PKEY_fromdata_init(pctx) != 1) goto err; @@ -3477,12 +2950,10 @@ EVP_PKEY *ssl_get_auto_dh(SSL_CONNECTION *s) return dhp; } -static int ssl_security_cert_key(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x, - int op) +static int ssl_security_cert_key(SSL *s, SSL_CTX *ctx, X509 *x, int op) { int secbits = -1; EVP_PKEY *pkey = X509_get0_pubkey(x); - if (pkey) { /* * If no parameters this will return -1 and fail using the default @@ -3492,18 +2963,16 @@ static int ssl_security_cert_key(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x, */ secbits = EVP_PKEY_get_security_bits(pkey); } - if (s != NULL) + if (s) return ssl_security(s, op, secbits, 0, x); else return ssl_ctx_security(ctx, op, secbits, 0, x); } -static int ssl_security_cert_sig(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x, - int op) +static int ssl_security_cert_sig(SSL *s, SSL_CTX *ctx, X509 *x, int op) { /* Lookup signature algorithm digest */ int secbits, nid, pknid; - /* Don't check signature if self signed */ if ((X509_get_extension_flags(x) & EXFLAG_SS) != 0) return 1; @@ -3512,14 +2981,13 @@ static int ssl_security_cert_sig(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x, /* If digest NID not defined use signature NID */ if (nid == NID_undef) nid = pknid; - if (s != NULL) + if (s) return ssl_security(s, op, secbits, nid, x); else return ssl_ctx_security(ctx, op, secbits, nid, x); } -int ssl_security_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x, int vfy, - int is_ee) +int ssl_security_cert(SSL *s, SSL_CTX *ctx, X509 *x, int vfy, int is_ee) { if (vfy) vfy = SSL_SECOP_PEER; @@ -3541,15 +3009,11 @@ int ssl_security_cert(SSL_CONNECTION *s, SSL_CTX *ctx, X509 *x, int vfy, * one to the peer. Return values: 1 if ok otherwise error code to use */ -int ssl_security_cert_chain(SSL_CONNECTION *s, STACK_OF(X509) *sk, - X509 *x, int vfy) +int ssl_security_cert_chain(SSL *s, STACK_OF(X509) *sk, X509 *x, int vfy) { int rv, start_idx, i; - if (x == NULL) { x = sk_X509_value(sk, 0); - if (x == NULL) - return ERR_R_INTERNAL_ERROR; start_idx = 1; } else start_idx = 0; @@ -3572,12 +3036,10 @@ int ssl_security_cert_chain(SSL_CONNECTION *s, STACK_OF(X509) *sk, * with the signature algorithm "lu" and return index of certificate. */ -static int tls12_get_cert_sigalg_idx(const SSL_CONNECTION *s, - const SIGALG_LOOKUP *lu) +static int tls12_get_cert_sigalg_idx(const SSL *s, const SIGALG_LOOKUP *lu) { int sig_idx = lu->sig_idx; - const SSL_CERT_LOOKUP *clu = ssl_cert_lookup_by_idx(sig_idx, - SSL_CONNECTION_GET_CTX(s)); + const SSL_CERT_LOOKUP *clu = ssl_cert_lookup_by_idx(sig_idx); /* If not recognised or not supported by cipher mask it is not suitable */ if (clu == NULL @@ -3586,10 +3048,6 @@ static int tls12_get_cert_sigalg_idx(const SSL_CONNECTION *s, && (s->s3.tmp.new_cipher->algorithm_mkey & SSL_kRSA) != 0)) return -1; - /* If doing RPK, the CERT_PKEY won't be "valid" */ - if (tls12_rpk_and_privkey(s, sig_idx)) - return s->s3.tmp.valid_flags[sig_idx] & CERT_PKEY_RPK ? sig_idx : -1; - return s->s3.tmp.valid_flags[sig_idx] & CERT_PKEY_VALID ? sig_idx : -1; } @@ -3599,14 +3057,13 @@ static int tls12_get_cert_sigalg_idx(const SSL_CONNECTION *s, * the key. * Returns true if the cert is usable and false otherwise. */ -static int check_cert_usable(SSL_CONNECTION *s, const SIGALG_LOOKUP *sig, - X509 *x, EVP_PKEY *pkey) +static int check_cert_usable(SSL *s, const SIGALG_LOOKUP *sig, X509 *x, + EVP_PKEY *pkey) { const SIGALG_LOOKUP *lu; int mdnid, pknid, supported; size_t i; const char *mdname = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* * If the given EVP_PKEY cannot support signing with this digest, @@ -3614,9 +3071,9 @@ static int check_cert_usable(SSL_CONNECTION *s, const SIGALG_LOOKUP *sig, */ if (sig->hash != NID_undef) mdname = OBJ_nid2sn(sig->hash); - supported = EVP_PKEY_digestsign_supports_digest(pkey, sctx->libctx, + supported = EVP_PKEY_digestsign_supports_digest(pkey, s->ctx->libctx, mdname, - sctx->propq); + s->ctx->propq); if (supported <= 0) return 0; @@ -3658,7 +3115,7 @@ static int check_cert_usable(SSL_CONNECTION *s, const SIGALG_LOOKUP *sig, * the signature_algorithm_cert restrictions sent by the peer (if any). * Returns false if no usable certificate is found. */ -static int has_usable_cert(SSL_CONNECTION *s, const SIGALG_LOOKUP *sig, int idx) +static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx) { /* TLS 1.2 callers can override sig->sig_idx, but not TLS 1.3 callers. */ if (idx == -1) @@ -3674,12 +3131,12 @@ static int has_usable_cert(SSL_CONNECTION *s, const SIGALG_LOOKUP *sig, int idx) * Returns true if the supplied cert |x| and key |pkey| is usable with the * specified signature scheme |sig|, or false otherwise. */ -static int is_cert_usable(SSL_CONNECTION *s, const SIGALG_LOOKUP *sig, X509 *x, +static int is_cert_usable(SSL *s, const SIGALG_LOOKUP *sig, X509 *x, EVP_PKEY *pkey) { size_t idx; - if (ssl_cert_lookup_by_pkey(pkey, &idx, SSL_CONNECTION_GET_CTX(s)) == NULL) + if (ssl_cert_lookup_by_pkey(pkey, &idx) == NULL) return 0; /* Check the key is consistent with the sig alg */ @@ -3694,14 +3151,12 @@ static int is_cert_usable(SSL_CONNECTION *s, const SIGALG_LOOKUP *sig, X509 *x, * |pkey|. |x| and |pkey| may be NULL in which case we additionally look at our * available certs/keys to find one that works. */ -static const SIGALG_LOOKUP *find_sig_alg(SSL_CONNECTION *s, X509 *x, - EVP_PKEY *pkey) +static const SIGALG_LOOKUP *find_sig_alg(SSL *s, X509 *x, EVP_PKEY *pkey) { const SIGALG_LOOKUP *lu = NULL; size_t i; int curve = -1; EVP_PKEY *tmppkey; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* Look for a shared sigalgs matching possible certificates */ for (i = 0; i < s->shared_sigalgslen; i++) { @@ -3714,23 +3169,42 @@ static const SIGALG_LOOKUP *find_sig_alg(SSL_CONNECTION *s, X509 *x, || lu->sig == EVP_PKEY_RSA) continue; /* Check that we have a cert, and signature_algorithms_cert */ - if (!tls1_lookup_md(sctx, lu, NULL)) + if (!tls1_lookup_md(s->ctx, lu, NULL)) continue; if ((pkey == NULL && !has_usable_cert(s, lu, -1)) || (pkey != NULL && !is_cert_usable(s, lu, x, pkey))) continue; +#ifndef OPENSSL_NO_SM2 + /* + * RFC 8998 requires that + * if the server chooses TLS_SM4_GCM_SM3 or TLS_SM4_CCM_SM3, + * the only valid signature algorithm present in + * "signature_algorithms" extension MUST be "sm2sig_sm3". + */ + if (SSL_IS_TLS13(s) && s->enable_sm_tls13_strict == 1 && s->server) { + const SSL_CIPHER *cipher = s->s3.tmp.new_cipher; + + if (cipher != NULL && + (cipher->id == TLS1_3_CK_SM4_GCM_SM3 + || cipher->id == TLS1_3_CK_SM4_CCM_SM3)) { + if (lu->sigalg != TLSEXT_SIGALG_sm2sig_sm3) + continue; + } + } +#endif + tmppkey = (pkey != NULL) ? pkey : s->cert->pkeys[lu->sig_idx].privatekey; - if (lu->sig == EVP_PKEY_EC) { + if (lu->sig == EVP_PKEY_EC || lu->sig == EVP_PKEY_SM2) { if (curve == -1) curve = ssl_get_EC_curve_nid(tmppkey); if (lu->curve != NID_undef && curve != lu->curve) continue; } else if (lu->sig == EVP_PKEY_RSA_PSS) { /* validate that key is large enough for the signature algorithm */ - if (!rsa_pss_check_min_key_size(sctx, tmppkey, lu)) + if (!rsa_pss_check_min_key_size(s->ctx, tmppkey, lu)) continue; } break; @@ -3742,6 +3216,47 @@ static const SIGALG_LOOKUP *find_sig_alg(SSL_CONNECTION *s, X509 *x, return lu; } +#ifndef OPENSSL_NO_NTLS +int tls_choose_sigalg_ntls(SSL *s, int fatalerrs) +{ + const SIGALG_LOOKUP *lu = NULL; + uint32_t kalg; + + s->s3.tmp.cert = NULL; + s->s3.tmp.sign_cert = NULL; + s->s3.tmp.enc_cert = NULL; + s->s3.tmp.sigalg = NULL; + + /* If ciphersuite doesn't require a cert nothing to do */ + if (!(s->s3.tmp.new_cipher->algorithm_auth & SSL_aCERT)) + return 1; + + if (!s->server && !ssl_has_cert(s, s->cert->key - s->cert->pkeys)) + return 1; + + kalg = s->s3.tmp.new_cipher->algorithm_mkey; + + if (kalg & (SSL_kSM2 | SSL_kSM2DHE)) { + lu = &ntls_sm2_sigalg; + } else if (kalg & SSL_kRSA) { + lu = &ntls_rsa_sigalg; + } else { + if (!fatalerrs) + return 1; + SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_UNKNOWN_KEY_EXCHANGE_TYPE); + return 0; + } + + s->s3.tmp.sign_cert = &s->cert->pkeys[lu->sig_idx]; + s->s3.tmp.enc_cert = &s->cert->pkeys[lu->sig_idx + 1]; + + s->cert->key = s->s3.tmp.cert = s->s3.tmp.sign_cert; + s->s3.tmp.sigalg = lu; + + return 1; +} +#endif + /* * Choose an appropriate signature algorithm based on available certificates * Sets chosen certificate and signature algorithm. @@ -3753,23 +3268,36 @@ static const SIGALG_LOOKUP *find_sig_alg(SSL_CONNECTION *s, X509 *x, * a fatal error: we will either try another certificate or not present one * to the server. In this case no error is set. */ -int tls_choose_sigalg(SSL_CONNECTION *s, int fatalerrs) +int tls_choose_sigalg(SSL *s, int fatalerrs) { +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + const SIGALG_LOOKUP *dc_lu = NULL; +#endif const SIGALG_LOOKUP *lu = NULL; int sig_idx = -1; s->s3.tmp.cert = NULL; s->s3.tmp.sigalg = NULL; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + s->s3.tmp.dc = NULL; +#endif - if (SSL_CONNECTION_IS_TLS13(s)) { - lu = find_sig_alg(s, NULL, NULL); - if (lu == NULL) { - if (!fatalerrs) - return 1; - SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, - SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); - return 0; + if (SSL_IS_TLS13(s)) { +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + dc_lu = find_dc_sig_alg(s); + if (dc_lu == NULL) { +#endif + lu = find_sig_alg(s, NULL, NULL); + if (lu == NULL) { + if (!fatalerrs) + return 1; + SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); + return 0; + } +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL } +#endif } else { /* If ciphersuite doesn't require a cert nothing to do */ if (!(s->s3.tmp.new_cipher->algorithm_auth & SSL_aCERT)) @@ -3781,7 +3309,6 @@ int tls_choose_sigalg(SSL_CONNECTION *s, int fatalerrs) size_t i; if (s->s3.tmp.peer_sigalgs != NULL) { int curve = -1; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* For Suite B need to match signature algorithm to curve */ if (tls1_suiteb(s)) @@ -3812,33 +3339,12 @@ int tls_choose_sigalg(SSL_CONNECTION *s, int fatalerrs) /* validate that key is large enough for the signature algorithm */ EVP_PKEY *pkey = s->cert->pkeys[sig_idx].privatekey; - if (!rsa_pss_check_min_key_size(sctx, pkey, lu)) + if (!rsa_pss_check_min_key_size(s->ctx, pkey, lu)) continue; } if (curve == -1 || lu->curve == curve) break; } -#ifndef OPENSSL_NO_GOST - /* - * Some Windows-based implementations do not send GOST algorithms indication - * in supported_algorithms extension, so when we have GOST-based ciphersuite, - * we have to assume GOST support. - */ - if (i == s->shared_sigalgslen - && (s->s3.tmp.new_cipher->algorithm_auth - & (SSL_aGOST01 | SSL_aGOST12)) != 0) { - if ((lu = tls1_get_legacy_sigalg(s, -1)) == NULL) { - if (!fatalerrs) - return 1; - SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, - SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); - return 0; - } else { - i = 0; - sig_idx = lu->sig_idx; - } - } -#endif if (i == s->shared_sigalgslen) { if (!fatalerrs) return 1; @@ -3886,11 +3392,37 @@ int tls_choose_sigalg(SSL_CONNECTION *s, int fatalerrs) } } } - if (sig_idx == -1) - sig_idx = lu->sig_idx; - s->s3.tmp.cert = &s->cert->pkeys[sig_idx]; - s->cert->key = s->s3.tmp.cert; - s->s3.tmp.sigalg = lu; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + if (dc_lu) { + sig_idx = dc_lu->sig_idx; + s->s3.tmp.dc = &s->cert->dc_pkeys[sig_idx]; + + lu = ssl_sigalg_lookup(DC_get_signature_sign_algorithm( + s->s3.tmp.dc->dc)); + if (lu == NULL) { + if (!fatalerrs) + return 1; + SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, + SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM); + return 0; + } + + s->s3.tmp.cert = &s->cert->pkeys[lu->sig_idx]; + s->cert->key = s->s3.tmp.cert; + s->s3.tmp.sigalg = dc_lu; + + s->delegated_credential_tag |= DC_HAS_BEEN_USED_FOR_SIGN; + } else { +#endif + if (sig_idx == -1) + sig_idx = lu->sig_idx; + s->s3.tmp.cert = &s->cert->pkeys[sig_idx]; + s->cert->key = s->s3.tmp.cert; + s->s3.tmp.sigalg = lu; +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL + } +#endif + return 1; } @@ -3908,19 +3440,13 @@ int SSL_CTX_set_tlsext_max_fragment_length(SSL_CTX *ctx, uint8_t mode) int SSL_set_tlsext_max_fragment_length(SSL *ssl, uint8_t mode) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); - - if (sc == NULL - || (IS_QUIC(ssl) && mode != TLSEXT_max_fragment_length_DISABLED)) - return 0; - if (mode != TLSEXT_max_fragment_length_DISABLED && !IS_MAX_FRAGMENT_LENGTH_EXT_VALID(mode)) { ERR_raise(ERR_LIB_SSL, SSL_R_SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH); return 0; } - sc->ext.max_fragment_len_mode = mode; + ssl->ext.max_fragment_len_mode = mode; return 1; } @@ -4054,3 +3580,211 @@ __owur int tls13_set_encoded_pub_key(EVP_PKEY *pkey, return EVP_PKEY_set1_encoded_public_key(pkey, enckey, enckeylen); } + +#ifndef OPENSSL_NO_DELEGATED_CREDENTIAL +static int check_dc_usable(SSL *s, const SIGALG_LOOKUP *sig, + DELEGATED_CREDENTIAL *dc, EVP_PKEY *pkey) +{ + int default_mdnid; + size_t i; + unsigned int dc_expect_sig_alg; + + /* If the EVP_PKEY reports a mandatory digest, allow nothing else. */ + ERR_set_mark(); + if (EVP_PKEY_get_default_digest_nid(pkey, &default_mdnid) == 2 && + sig->hash != default_mdnid) + return 0; + + /* If it didn't report a mandatory NID, for whatever reasons, + * just clear the error and allow all hashes to be used. */ + ERR_pop_to_mark(); + + dc_expect_sig_alg = DC_get_expected_cert_verify_algorithm(dc); + + for (i = 0; i < s->s3.tmp.peer_dc_sigalgslen; i++) + if (dc_expect_sig_alg == s->s3.tmp.peer_dc_sigalgs[i]) + return 1; + + return 0; +} + +static int has_usable_dc(SSL *s, const SIGALG_LOOKUP *sig, int idx) +{ + if (idx == -1) + idx = sig->sig_idx; + + if (!ssl_has_dc(s, idx)) + return 0; + + return check_dc_usable(s, sig, s->cert->dc_pkeys[idx].dc, + s->cert->dc_pkeys[idx].privatekey); +} + +/* refer to find_sig_alg() */ +static const SIGALG_LOOKUP *find_dc_sig_alg(SSL *s) +{ + const SIGALG_LOOKUP *lu = NULL; + size_t i; + int curve = -1; + EVP_PKEY *tmppkey; + + if (!s->enable_sign_by_dc || !SSL_IS_TLS13(s)) + return NULL; + + /* Look for a shared sigalgs matching possible certificates */ + for (i = 0; i < s->shared_dc_sigalgslen; i++) { + lu = s->shared_dc_sigalgs[i]; + + /* Skip SHA1, SHA224, DSA and RSA if not PSS */ + if (lu->hash == NID_sha1 + || lu->hash == NID_sha224 + || lu->sig == EVP_PKEY_DSA + || lu->sig == EVP_PKEY_RSA) + continue; + /* Check that we have a cert, and signature_algorithms_cert */ + if (!tls1_lookup_md(s->ctx, lu, NULL)) + continue; + + if (!has_usable_dc(s, lu, -1)) + continue; + + tmppkey = s->cert->dc_pkeys[lu->sig_idx].privatekey; + + if (lu->sig == EVP_PKEY_EC || lu->sig == EVP_PKEY_SM2) { + if (curve == -1) + curve = ssl_get_EC_curve_nid(tmppkey); + if (lu->curve != NID_undef && curve != lu->curve) + continue; + } else if (lu->sig == EVP_PKEY_RSA_PSS) { + /* validate that key is large enough for the signature algorithm */ + if (!rsa_pss_check_min_key_size(s->ctx, tmppkey, lu)) + continue; + } + break; + } + + if (i == s->shared_dc_sigalgslen) + return NULL; + + return lu; +} + +/* refer to tls1_set_shared_sigalgs */ +int tls1_set_shared_dc_sigalgs(SSL *s) +{ + const uint16_t *pref, *allow, *conf; + size_t preflen, allowlen, conflen; + size_t nmatch; + const SIGALG_LOOKUP **salgs = NULL; + CERT *c = s->cert; + unsigned int is_suiteb = tls1_suiteb(s); + + OPENSSL_free(s->shared_dc_sigalgs); + s->shared_dc_sigalgs = NULL; + s->shared_dc_sigalgslen = 0; + /* If client use client signature algorithms if not NULL */ + if (!s->server && c->client_sigalgs && !is_suiteb) { + conf = c->client_sigalgs; + conflen = c->client_sigalgslen; + } else if (c->conf_sigalgs && !is_suiteb) { + conf = c->conf_sigalgs; + conflen = c->conf_sigalgslen; + } else + conflen = tls12_get_psigalgs(s, 0, &conf); + if (s->options & SSL_OP_CIPHER_SERVER_PREFERENCE || is_suiteb) { + pref = conf; + preflen = conflen; + allow = s->s3.tmp.peer_dc_sigalgs; + allowlen = s->s3.tmp.peer_dc_sigalgslen; + } else { + allow = conf; + allowlen = conflen; + pref = s->s3.tmp.peer_dc_sigalgs; + preflen = s->s3.tmp.peer_dc_sigalgslen; + } + + nmatch = tls12_shared_sigalgs(s, NULL, pref, preflen, allow, allowlen); + if (nmatch) { + if ((salgs = OPENSSL_malloc(nmatch * sizeof(*salgs))) == NULL) { + ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE); + return 0; + } + nmatch = tls12_shared_sigalgs(s, salgs, pref, preflen, allow, allowlen); + } else { + salgs = NULL; + } + + s->shared_dc_sigalgs = salgs; + s->shared_dc_sigalgslen = nmatch; + return 1; +} +#endif + +const SIGALG_LOOKUP *ssl_sigalg_lookup_by_pkey_and_hash(EVP_PKEY *pkey, + int hash, + int is_tls13) +{ + int md_nid = hash; + int def_nid, def_ret; + int curve = NID_undef; + size_t i, sig_idx; + const SIGALG_LOOKUP *lu = NULL; + + if (ssl_cert_lookup_by_pkey(pkey, &sig_idx) == NULL) { + ERR_raise(ERR_LIB_SSL, SSL_R_UNKNOWN_CERTIFICATE_TYPE); + return 0; + } + + def_ret = EVP_PKEY_get_default_digest_nid(pkey, &def_nid); + if (def_ret == 2) { + md_nid = def_nid; + } else if (def_ret == 1) { + if (hash == NID_undef) + md_nid = def_nid; + } + + if (EVP_PKEY_is_a(pkey, "EC") || EVP_PKEY_is_a(pkey, "SM2")) + curve = ssl_get_EC_curve_nid(pkey); + + for (i = 0, lu = sigalg_lookup_tbl; i < OSSL_NELEM(sigalg_lookup_tbl); + i++, lu++) { + if (is_tls13) { + /* Skip SHA1, SHA224, DSA and RSA if not PSS */ + if (lu->hash == NID_sha1 + || lu->hash == NID_sha224 + || lu->sig == EVP_PKEY_DSA + || lu->sig == EVP_PKEY_RSA) + continue; + } + + if (lu->sig_idx != (int)sig_idx) + continue; + + if (lu->hash != NID_undef && lu->hash != md_nid) + continue; + + if (lu->curve != NID_undef && lu->curve != curve) + continue; + + return lu; + } + + return NULL; +} + +const SIGALG_LOOKUP *ssl_sigalg_lookup(uint16_t sigalg) +{ + size_t i; + const SIGALG_LOOKUP *lu; + + for (i = 0, lu = sigalg_lookup_tbl; i < OSSL_NELEM(sigalg_lookup_tbl); + i++, lu++) { + if (lu->sigalg == sigalg) { + if (!lu->enabled) + return NULL; + return lu; + } + } + + return NULL; +} diff --git a/openssl/src/ssl/t1_trce.c b/openssl/src/ssl/t1_trce.c index 29dce65e4..83e6958cf 100644 --- a/openssl/src/ssl/t1_trce.c +++ b/openssl/src/ssl/t1_trce.c @@ -1,5 +1,5 @@ /* - * Copyright 2012-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2012-2022 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -12,7 +12,6 @@ #ifndef OPENSSL_NO_SSL_TRACE /* Packet trace support for OpenSSL */ -#include "internal/nelem.h" typedef struct { int num; @@ -68,7 +67,10 @@ static const ssl_trace_tbl ssl_version_tbl[] = { {TLS1_3_VERSION, "TLS 1.3"}, {DTLS1_VERSION, "DTLS 1.0"}, {DTLS1_2_VERSION, "DTLS 1.2"}, - {DTLS1_BAD_VER, "DTLS 1.0 (bad)"} + {DTLS1_BAD_VER, "DTLS 1.0 (bad)"}, +#ifndef OPENSSL_NO_NTLS + {NTLS_VERSION, "NTLS"}, +#endif }; static const ssl_trace_tbl ssl_content_tbl[] = { @@ -113,8 +115,6 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0x0003, "TLS_RSA_EXPORT_WITH_RC4_40_MD5"}, {0x0004, "TLS_RSA_WITH_RC4_128_MD5"}, {0x0005, "TLS_RSA_WITH_RC4_128_SHA"}, - {0x0006, "TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5"}, - {0x0007, "TLS_RSA_WITH_IDEA_CBC_SHA"}, {0x0008, "TLS_RSA_EXPORT_WITH_DES40_CBC_SHA"}, {0x0009, "TLS_RSA_WITH_DES_CBC_SHA"}, {0x000A, "TLS_RSA_WITH_3DES_EDE_CBC_SHA"}, @@ -139,16 +139,12 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0x001E, "SSL_FORTEZZA_KEA_WITH_RC4_128_SHA"}, {0x001F, "TLS_KRB5_WITH_3DES_EDE_CBC_SHA"}, {0x0020, "TLS_KRB5_WITH_RC4_128_SHA"}, - {0x0021, "TLS_KRB5_WITH_IDEA_CBC_SHA"}, {0x0022, "TLS_KRB5_WITH_DES_CBC_MD5"}, {0x0023, "TLS_KRB5_WITH_3DES_EDE_CBC_MD5"}, {0x0024, "TLS_KRB5_WITH_RC4_128_MD5"}, - {0x0025, "TLS_KRB5_WITH_IDEA_CBC_MD5"}, {0x0026, "TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA"}, - {0x0027, "TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA"}, {0x0028, "TLS_KRB5_EXPORT_WITH_RC4_40_SHA"}, {0x0029, "TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5"}, - {0x002A, "TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5"}, {0x002B, "TLS_KRB5_EXPORT_WITH_RC4_40_MD5"}, {0x002C, "TLS_PSK_WITH_NULL_SHA"}, {0x002D, "TLS_DHE_PSK_WITH_NULL_SHA"}, @@ -171,12 +167,6 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0x003E, "TLS_DH_DSS_WITH_AES_128_CBC_SHA256"}, {0x003F, "TLS_DH_RSA_WITH_AES_128_CBC_SHA256"}, {0x0040, "TLS_DHE_DSS_WITH_AES_128_CBC_SHA256"}, - {0x0041, "TLS_RSA_WITH_CAMELLIA_128_CBC_SHA"}, - {0x0042, "TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA"}, - {0x0043, "TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA"}, - {0x0044, "TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA"}, - {0x0045, "TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA"}, - {0x0046, "TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA"}, {0x0067, "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256"}, {0x0068, "TLS_DH_DSS_WITH_AES_256_CBC_SHA256"}, {0x0069, "TLS_DH_RSA_WITH_AES_256_CBC_SHA256"}, @@ -184,14 +174,6 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0x006B, "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"}, {0x006C, "TLS_DH_anon_WITH_AES_128_CBC_SHA256"}, {0x006D, "TLS_DH_anon_WITH_AES_256_CBC_SHA256"}, - {0x0081, "TLS_GOSTR341001_WITH_28147_CNT_IMIT"}, - {0x0083, "TLS_GOSTR341001_WITH_NULL_GOSTR3411"}, - {0x0084, "TLS_RSA_WITH_CAMELLIA_256_CBC_SHA"}, - {0x0085, "TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA"}, - {0x0086, "TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA"}, - {0x0087, "TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA"}, - {0x0088, "TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA"}, - {0x0089, "TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA"}, {0x008A, "TLS_PSK_WITH_RC4_128_SHA"}, {0x008B, "TLS_PSK_WITH_3DES_EDE_CBC_SHA"}, {0x008C, "TLS_PSK_WITH_AES_128_CBC_SHA"}, @@ -204,12 +186,6 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0x0093, "TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA"}, {0x0094, "TLS_RSA_PSK_WITH_AES_128_CBC_SHA"}, {0x0095, "TLS_RSA_PSK_WITH_AES_256_CBC_SHA"}, - {0x0096, "TLS_RSA_WITH_SEED_CBC_SHA"}, - {0x0097, "TLS_DH_DSS_WITH_SEED_CBC_SHA"}, - {0x0098, "TLS_DH_RSA_WITH_SEED_CBC_SHA"}, - {0x0099, "TLS_DHE_DSS_WITH_SEED_CBC_SHA"}, - {0x009A, "TLS_DHE_RSA_WITH_SEED_CBC_SHA"}, - {0x009B, "TLS_DH_anon_WITH_SEED_CBC_SHA"}, {0x009C, "TLS_RSA_WITH_AES_128_GCM_SHA256"}, {0x009D, "TLS_RSA_WITH_AES_256_GCM_SHA384"}, {0x009E, "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"}, @@ -240,18 +216,6 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0x00B7, "TLS_RSA_PSK_WITH_AES_256_CBC_SHA384"}, {0x00B8, "TLS_RSA_PSK_WITH_NULL_SHA256"}, {0x00B9, "TLS_RSA_PSK_WITH_NULL_SHA384"}, - {0x00BA, "TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256"}, - {0x00BB, "TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256"}, - {0x00BC, "TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256"}, - {0x00BD, "TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256"}, - {0x00BE, "TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256"}, - {0x00BF, "TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256"}, - {0x00C0, "TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256"}, - {0x00C1, "TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256"}, - {0x00C2, "TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256"}, - {0x00C3, "TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256"}, - {0x00C4, "TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256"}, - {0x00C5, "TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256"}, {0x00FF, "TLS_EMPTY_RENEGOTIATION_INFO_SCSV"}, {0x5600, "TLS_FALLBACK_SCSV"}, {0xC001, "TLS_ECDH_ECDSA_WITH_NULL_SHA"}, @@ -313,102 +277,6 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0xC039, "TLS_ECDHE_PSK_WITH_NULL_SHA"}, {0xC03A, "TLS_ECDHE_PSK_WITH_NULL_SHA256"}, {0xC03B, "TLS_ECDHE_PSK_WITH_NULL_SHA384"}, - {0xC03C, "TLS_RSA_WITH_ARIA_128_CBC_SHA256"}, - {0xC03D, "TLS_RSA_WITH_ARIA_256_CBC_SHA384"}, - {0xC03E, "TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256"}, - {0xC03F, "TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384"}, - {0xC040, "TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256"}, - {0xC041, "TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384"}, - {0xC042, "TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256"}, - {0xC043, "TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384"}, - {0xC044, "TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256"}, - {0xC045, "TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384"}, - {0xC046, "TLS_DH_anon_WITH_ARIA_128_CBC_SHA256"}, - {0xC047, "TLS_DH_anon_WITH_ARIA_256_CBC_SHA384"}, - {0xC048, "TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256"}, - {0xC049, "TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384"}, - {0xC04A, "TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256"}, - {0xC04B, "TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384"}, - {0xC04C, "TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256"}, - {0xC04D, "TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384"}, - {0xC04E, "TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256"}, - {0xC04F, "TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384"}, - {0xC050, "TLS_RSA_WITH_ARIA_128_GCM_SHA256"}, - {0xC051, "TLS_RSA_WITH_ARIA_256_GCM_SHA384"}, - {0xC052, "TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256"}, - {0xC053, "TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384"}, - {0xC054, "TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256"}, - {0xC055, "TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384"}, - {0xC056, "TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256"}, - {0xC057, "TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384"}, - {0xC058, "TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256"}, - {0xC059, "TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384"}, - {0xC05A, "TLS_DH_anon_WITH_ARIA_128_GCM_SHA256"}, - {0xC05B, "TLS_DH_anon_WITH_ARIA_256_GCM_SHA384"}, - {0xC05C, "TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256"}, - {0xC05D, "TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384"}, - {0xC05E, "TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256"}, - {0xC05F, "TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384"}, - {0xC060, "TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256"}, - {0xC061, "TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384"}, - {0xC062, "TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256"}, - {0xC063, "TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384"}, - {0xC064, "TLS_PSK_WITH_ARIA_128_CBC_SHA256"}, - {0xC065, "TLS_PSK_WITH_ARIA_256_CBC_SHA384"}, - {0xC066, "TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256"}, - {0xC067, "TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384"}, - {0xC068, "TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256"}, - {0xC069, "TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384"}, - {0xC06A, "TLS_PSK_WITH_ARIA_128_GCM_SHA256"}, - {0xC06B, "TLS_PSK_WITH_ARIA_256_GCM_SHA384"}, - {0xC06C, "TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256"}, - {0xC06D, "TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384"}, - {0xC06E, "TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256"}, - {0xC06F, "TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384"}, - {0xC070, "TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256"}, - {0xC071, "TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384"}, - {0xC072, "TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256"}, - {0xC073, "TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384"}, - {0xC074, "TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256"}, - {0xC075, "TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384"}, - {0xC076, "TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256"}, - {0xC077, "TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384"}, - {0xC078, "TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256"}, - {0xC079, "TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384"}, - {0xC07A, "TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC07B, "TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC07C, "TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC07D, "TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC07E, "TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC07F, "TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC080, "TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC081, "TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC082, "TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC083, "TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC084, "TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC085, "TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC086, "TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC087, "TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC088, "TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC089, "TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC08A, "TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC08B, "TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC08C, "TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC08D, "TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC08E, "TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC08F, "TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC090, "TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC091, "TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC092, "TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256"}, - {0xC093, "TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384"}, - {0xC094, "TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256"}, - {0xC095, "TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384"}, - {0xC096, "TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256"}, - {0xC097, "TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384"}, - {0xC098, "TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256"}, - {0xC099, "TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384"}, - {0xC09A, "TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256"}, - {0xC09B, "TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384"}, {0xC09C, "TLS_RSA_WITH_AES_128_CCM"}, {0xC09D, "TLS_RSA_WITH_AES_256_CCM"}, {0xC09E, "TLS_DHE_RSA_WITH_AES_128_CCM"}, @@ -429,7 +297,6 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0xC0AD, "TLS_ECDHE_ECDSA_WITH_AES_256_CCM"}, {0xC0AE, "TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8"}, {0xC0AF, "TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8"}, - {0xC102, "IANA-GOST2012-GOST8912-GOST8912"}, {0xCCA8, "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"}, {0xCCA9, "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"}, {0xCCAA, "TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256"}, @@ -444,11 +311,18 @@ static const ssl_trace_tbl ssl_ciphers_tbl[] = { {0x1305, "TLS_AES_128_CCM_8_SHA256"}, {0xFEFE, "SSL_RSA_FIPS_WITH_DES_CBC_SHA"}, {0xFEFF, "SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA"}, - {0xFF85, "LEGACY-GOST2012-GOST8912-GOST8912"}, - {0xFF87, "GOST2012-NULL-GOST12"}, - {0xC100, "GOST2012-KUZNYECHIK-KUZNYECHIKOMAC"}, - {0xC101, "GOST2012-MAGMA-MAGMAOMAC"}, - {0xC102, "GOST2012-GOST8912-IANA"}, + {0x00C6, "TLS_SM4_GCM_SM3"}, + {0x00C7, "TLS_SM4_CCM_SM3"}, +#ifndef OPENSSL_NO_NTLS + {0xE011, "ECDHE_SM4_CBC_SM3"}, + {0xE051, "ECDHE_SM4_GCM_SM3"}, + {0xE013, "ECC_SM4_CBC_SM3"}, + {0xE053, "ECC_SM4_GCM_SM3"}, + {0xE019, "RSA_SM4_CBC_SM3"}, + {0xE059, "RSA_SM4_GCM_SM3"}, + {0xE01C, "RSA_SM4_CBC_SHA256"}, + {0xE05A, "RSA_SM4_GCM_SHA256"}, +#endif }; /* Compression methods */ @@ -477,8 +351,6 @@ static const ssl_trace_tbl ssl_exts_tbl[] = { {TLSEXT_TYPE_application_layer_protocol_negotiation, "application_layer_protocol_negotiation"}, {TLSEXT_TYPE_signed_certificate_timestamp, "signed_certificate_timestamps"}, - {TLSEXT_TYPE_client_cert_type, "client_cert_type"}, - {TLSEXT_TYPE_server_cert_type, "server_cert_type"}, {TLSEXT_TYPE_padding, "padding"}, {TLSEXT_TYPE_encrypt_then_mac, "encrypt_then_mac"}, {TLSEXT_TYPE_extended_master_secret, "extended_master_secret"}, @@ -530,9 +402,6 @@ static const ssl_trace_tbl ssl_groups_tbl[] = { {28, "brainpoolP512r1"}, {29, "ecdh_x25519"}, {30, "ecdh_x448"}, - {31, "brainpoolP256r1tls13"}, - {32, "brainpoolP384r1tls13"}, - {33, "brainpoolP512r1tls13"}, {34, "GC256A"}, {35, "GC256B"}, {36, "GC256C"}, @@ -545,10 +414,9 @@ static const ssl_trace_tbl ssl_groups_tbl[] = { {258, "ffdhe4096"}, {259, "ffdhe6144"}, {260, "ffdhe8192"}, - {25497, "X25519Kyber768Draft00"}, - {25498, "SecP256r1Kyber768Draft00"}, {0xFF01, "arbitrary_explicit_prime_curves"}, - {0xFF02, "arbitrary_explicit_char2_curves"} + {0xFF02, "arbitrary_explicit_char2_curves"}, + {41, "curveSM2"}, }; static const ssl_trace_tbl ssl_point_tbl[] = { @@ -589,14 +457,7 @@ static const ssl_trace_tbl ssl_sigalg_tbl[] = { {TLSEXT_SIGALG_dsa_sha512, "dsa_sha512"}, {TLSEXT_SIGALG_dsa_sha224, "dsa_sha224"}, {TLSEXT_SIGALG_dsa_sha1, "dsa_sha1"}, - {TLSEXT_SIGALG_gostr34102012_256_intrinsic, "gost2012_256"}, - {TLSEXT_SIGALG_gostr34102012_512_intrinsic, "gost2012_512"}, - {TLSEXT_SIGALG_gostr34102012_256_gostr34112012_256, "gost2012_256"}, - {TLSEXT_SIGALG_gostr34102012_512_gostr34112012_512, "gost2012_512"}, - {TLSEXT_SIGALG_gostr34102001_gostr3411, "gost2001_gost94"}, - {TLSEXT_SIGALG_ecdsa_brainpoolP256r1_sha256, "ecdsa_brainpoolP256r1_sha256"}, - {TLSEXT_SIGALG_ecdsa_brainpoolP384r1_sha384, "ecdsa_brainpoolP384r1_sha384"}, - {TLSEXT_SIGALG_ecdsa_brainpoolP512r1_sha512, "ecdsa_brainpoolP512r1_sha512"}, + {TLSEXT_SIGALG_sm2sig_sm3, "sm2sig_sm3"}, }; static const ssl_trace_tbl ssl_ctype_tbl[] = { @@ -610,8 +471,6 @@ static const ssl_trace_tbl ssl_ctype_tbl[] = { {64, "ecdsa_sign"}, {65, "rsa_fixed_ecdh"}, {66, "ecdsa_fixed_ecdh"}, - {67, "gost_sign256"}, - {68, "gost_sign512"}, }; static const ssl_trace_tbl ssl_psk_kex_modes_tbl[] = { @@ -624,23 +483,10 @@ static const ssl_trace_tbl ssl_key_update_tbl[] = { {SSL_KEY_UPDATE_REQUESTED, "update_requested"} }; -static const ssl_trace_tbl ssl_comp_cert_tbl[] = { - {TLSEXT_comp_cert_none, "none"}, - {TLSEXT_comp_cert_zlib, "zlib"}, - {TLSEXT_comp_cert_brotli, "brotli"}, - {TLSEXT_comp_cert_zstd, "zstd"} -}; - -/* - * "pgp" and "1609dot2" are defined in RFC7250, - * although OpenSSL doesn't support them, it can - * at least report them in traces - */ -static const ssl_trace_tbl ssl_cert_type_tbl[] = { - {TLSEXT_cert_type_x509, "x509"}, - {TLSEXT_cert_type_pgp, "pgp"}, - {TLSEXT_cert_type_rpk, "rpk"}, - {TLSEXT_cert_type_1609dot2, "1609dot2"} +static const ssl_trace_tbl ssl_cert_compression_tbl[] = { + {0, "zlib"}, + {1, "brotli"}, + {2, "zstd"} }; static void ssl_print_hex(BIO *bio, int indent, const char *name, @@ -717,12 +563,12 @@ static int ssl_print_random(BIO *bio, int indent, return 1; } -static int ssl_print_signature(BIO *bio, int indent, const SSL_CONNECTION *sc, +static int ssl_print_signature(BIO *bio, int indent, const SSL *ssl, const unsigned char **pmsg, size_t *pmsglen) { if (*pmsglen < 2) return 0; - if (SSL_USE_SIGALGS(sc)) { + if (SSL_USE_SIGALGS(ssl)) { const unsigned char *p = *pmsg; unsigned int sigalg = (p[0] << 8) | p[1]; @@ -747,14 +593,6 @@ static int ssl_print_extension(BIO *bio, int indent, int server, BIO_printf(bio, "extension_type=%s(%d), length=%d\n", ssl_trace_str(extype, ssl_exts_tbl), extype, (int)extlen); switch (extype) { - case TLSEXT_TYPE_compress_certificate: - if (extlen < 1) - return 0; - xlen = ext[0]; - if (extlen != xlen + 1) - return 0; - return ssl_trace_list(bio, indent + 2, ext + 1, xlen, 2, ssl_comp_cert_tbl); - case TLSEXT_TYPE_max_fragment_length: if (extlen < 1) return 0; @@ -923,23 +761,16 @@ static int ssl_print_extension(BIO *bio, int indent, int server, | ((unsigned int)ext[2] << 8) | (unsigned int)ext[3]; BIO_indent(bio, indent + 2, 80); - BIO_printf(bio, "max_early_data=%u\n", (unsigned int)max_early_data); + BIO_printf(bio, "max_early_data=%u\n", max_early_data); break; - - case TLSEXT_TYPE_server_cert_type: - case TLSEXT_TYPE_client_cert_type: - if (server) { - if (extlen != 1) - return 0; - return ssl_trace_list(bio, indent + 2, ext, 1, 1, ssl_cert_type_tbl); - } + case TLSEXT_TYPE_compress_certificate: if (extlen < 1) return 0; xlen = ext[0]; if (extlen != xlen + 1) return 0; - return ssl_trace_list(bio, indent + 2, ext + 1, xlen, 1, ssl_cert_type_tbl); - + return ssl_trace_list(bio, indent + 2, ext + 1, xlen, 2, + ssl_cert_compression_tbl); default: BIO_dump_indent(bio, (const char *)ext, extlen, indent + 2); } @@ -999,7 +830,7 @@ static int ssl_print_extensions(BIO *bio, int indent, int server, return 1; } -static int ssl_print_client_hello(BIO *bio, const SSL_CONNECTION *sc, int indent, +static int ssl_print_client_hello(BIO *bio, const SSL *ssl, int indent, const unsigned char *msg, size_t msglen) { size_t len; @@ -1011,7 +842,7 @@ static int ssl_print_client_hello(BIO *bio, const SSL_CONNECTION *sc, int indent return 0; if (!ssl_print_hexbuf(bio, indent, "session_id", 1, &msg, &msglen)) return 0; - if (SSL_CONNECTION_IS_DTLS(sc)) { + if (SSL_IS_DTLS(ssl)) { if (!ssl_print_hexbuf(bio, indent, "cookie", 1, &msg, &msglen)) return 0; } @@ -1102,9 +933,9 @@ static int ssl_print_server_hello(BIO *bio, int indent, return 1; } -static int ssl_get_keyex(const char **pname, const SSL_CONNECTION *sc) +static int ssl_get_keyex(const char **pname, const SSL *ssl) { - unsigned long alg_k = sc->s3.tmp.new_cipher->algorithm_mkey; + unsigned long alg_k = ssl->s3.tmp.new_cipher->algorithm_mkey; if (alg_k & SSL_kRSA) { *pname = "rsa"; @@ -1138,23 +969,23 @@ static int ssl_get_keyex(const char **pname, const SSL_CONNECTION *sc) *pname = "SRP"; return SSL_kSRP; } - if (alg_k & SSL_kGOST) { - *pname = "GOST"; - return SSL_kGOST; + if (alg_k & SSL_kSM2) { + *pname = "SM2"; + return SSL_kSM2; } - if (alg_k & SSL_kGOST18) { - *pname = "GOST18"; - return SSL_kGOST18; + if (alg_k & SSL_kSM2DHE) { + *pname = "SM2DHE"; + return SSL_kSM2DHE; } *pname = "UNKNOWN"; return 0; } -static int ssl_print_client_keyex(BIO *bio, int indent, const SSL_CONNECTION *sc, +static int ssl_print_client_keyex(BIO *bio, int indent, const SSL *ssl, const unsigned char *msg, size_t msglen) { const char *algname; - int id = ssl_get_keyex(&algname, sc); + int id = ssl_get_keyex(&algname, ssl); BIO_indent(bio, indent, 80); BIO_printf(bio, "KeyExchangeAlgorithm=%s\n", algname); @@ -1167,7 +998,7 @@ static int ssl_print_client_keyex(BIO *bio, int indent, const SSL_CONNECTION *sc case SSL_kRSA: case SSL_kRSAPSK: - if (TLS1_get_version(SSL_CONNECTION_GET_SSL(sc)) == SSL3_VERSION) { + if (TLS1_get_version(ssl) == SSL3_VERSION) { ssl_print_hex(bio, indent + 2, "EncryptedPreMasterSecret", msg, msglen); } else { @@ -1188,25 +1019,29 @@ static int ssl_print_client_keyex(BIO *bio, int indent, const SSL_CONNECTION *sc if (!ssl_print_hexbuf(bio, indent + 2, "ecdh_Yc", 1, &msg, &msglen)) return 0; break; - case SSL_kGOST: - ssl_print_hex(bio, indent + 2, "GostKeyTransportBlob", msg, msglen); - msglen = 0; + case SSL_kSM2: + if (!ssl_print_hexbuf(bio, indent + 2, + "EncryptedPreMasterSecret", 2, &msg, &msglen)) + return 0; break; - case SSL_kGOST18: - ssl_print_hex(bio, indent + 2, - "GOST-wrapped PreMasterSecret", msg, msglen); - msglen = 0; + case SSL_kSM2DHE: + ssl_print_hex(bio, indent + 2, "ECParameters", msg, 3); + msg += 3; + msglen -= 3; + if (!ssl_print_hexbuf(bio, indent + 2, + "sm2dh_Yc", 1, &msg, &msglen)) + return 0; break; } return !msglen; } -static int ssl_print_server_keyex(BIO *bio, int indent, const SSL_CONNECTION *sc, +static int ssl_print_server_keyex(BIO *bio, int indent, const SSL *ssl, const unsigned char *msg, size_t msglen) { const char *algname; - int id = ssl_get_keyex(&algname, sc); + int id = ssl_get_keyex(&algname, ssl); BIO_indent(bio, indent, 80); BIO_printf(bio, "KeyExchangeAlgorithm=%s\n", algname); @@ -1237,6 +1072,7 @@ static int ssl_print_server_keyex(BIO *bio, int indent, const SSL_CONNECTION *sc case SSL_kECDHE: case SSL_kECDHEPSK: + case SSL_kSM2DHE: if (msglen < 1) return 0; BIO_indent(bio, indent + 2, 80); @@ -1266,18 +1102,17 @@ static int ssl_print_server_keyex(BIO *bio, int indent, const SSL_CONNECTION *sc break; } if (!(id & SSL_PSK)) - ssl_print_signature(bio, indent, sc, &msg, &msglen); + ssl_print_signature(bio, indent, ssl, &msg, &msglen); return !msglen; } -static int ssl_print_certificate(BIO *bio, const SSL_CONNECTION *sc, int indent, +static int ssl_print_certificate(BIO *bio, int indent, const unsigned char **pmsg, size_t *pmsglen) { size_t msglen = *pmsglen; size_t clen; X509 *x; const unsigned char *p = *pmsg, *q; - SSL_CTX *ctx = SSL_CONNECTION_GET_CTX(sc); if (msglen < 3) return 0; @@ -1287,12 +1122,8 @@ static int ssl_print_certificate(BIO *bio, const SSL_CONNECTION *sc, int indent, q = p + 3; BIO_indent(bio, indent, 80); BIO_printf(bio, "ASN.1Cert, length=%d", (int)clen); - x = X509_new_ex(ctx->libctx, ctx->propq); - if (x != NULL && d2i_X509(&x, &q, clen) == NULL) { - X509_free(x); - x = NULL; - } - if (x == NULL) + x = d2i_X509(NULL, &q, clen); + if (!x) BIO_puts(bio, "\n"); else { BIO_puts(bio, "\n------details-----\n"); @@ -1310,43 +1141,63 @@ static int ssl_print_certificate(BIO *bio, const SSL_CONNECTION *sc, int indent, return 1; } -static int ssl_print_raw_public_key(BIO *bio, const SSL *ssl, int server, - int indent, const unsigned char **pmsg, - size_t *pmsglen) +static int ssl_print_compressed_certificate(BIO *bio, const SSL *ssl, + int server, int indent, + const unsigned char *msg, + size_t msglen) { - EVP_PKEY *pkey; - size_t clen; - const unsigned char *msg = *pmsg; - size_t msglen = *pmsglen; + uint16_t algorithm; + unsigned int uncompressed_length; + uint32_t clen; - if (msglen < 3) + if (msglen < 2) return 0; - clen = (msg[0] << 16) | (msg[1] << 8) | msg[2]; - if (msglen < clen + 3) + + algorithm = (msg[0] << 8) | msg[1]; + + BIO_indent(bio, indent, 80); + BIO_printf(bio, "algorithm: %u\n", algorithm); + + msg += 2; + msglen -= 2; + + if (msglen < 3) return 0; + uncompressed_length = (msg[0] << 16) | (msg[1] << 8) | msg[2]; + + BIO_indent(bio, indent, 80); + BIO_printf(bio, "uncompressed_length: %u\n", uncompressed_length); + msg += 3; + msglen -= 3; + + if (msglen < 3) + return 0; + + clen = (msg[0] << 16) | (msg[1] << 8) | msg[2]; BIO_indent(bio, indent, 80); - BIO_printf(bio, "raw_public_key, length=%d\n", (int)clen); + BIO_printf(bio, "compressed_certificate_message, length=%u\n", clen); - pkey = d2i_PUBKEY_ex(NULL, &msg, clen, ssl->ctx->libctx, ssl->ctx->propq); - if (pkey == NULL) + msg += 3; + msglen -= 3; + + if (msglen != clen) return 0; - EVP_PKEY_print_public(bio, pkey, indent + 2, NULL); - EVP_PKEY_free(pkey); - *pmsg += clen + 3; - *pmsglen -= clen + 3; + + ssl_print_hex(bio, indent + 2, "value", msg, msglen); + return 1; } -static int ssl_print_certificates(BIO *bio, const SSL_CONNECTION *sc, int server, +static int ssl_print_certificates(BIO *bio, const SSL *ssl, int server, int indent, const unsigned char *msg, size_t msglen) { size_t clen; - if (SSL_CONNECTION_IS_TLS13(sc) + if (SSL_IS_TLS13(ssl) && !ssl_print_hexbuf(bio, indent, "context", 1, &msg, &msglen)) return 0; @@ -1356,22 +1207,12 @@ static int ssl_print_certificates(BIO *bio, const SSL_CONNECTION *sc, int server if (msglen != clen + 3) return 0; msg += 3; - if ((server && sc->ext.server_cert_type == TLSEXT_cert_type_rpk) - || (!server && sc->ext.client_cert_type == TLSEXT_cert_type_rpk)) { - if (!ssl_print_raw_public_key(bio, &sc->ssl, server, indent, &msg, &clen)) - return 0; - if (SSL_CONNECTION_IS_TLS13(sc) - && !ssl_print_extensions(bio, indent + 2, server, - SSL3_MT_CERTIFICATE, &msg, &clen)) - return 0; - return 1; - } BIO_indent(bio, indent, 80); BIO_printf(bio, "certificate_list, length=%d\n", (int)clen); while (clen > 0) { - if (!ssl_print_certificate(bio, sc, indent + 2, &msg, &clen)) + if (!ssl_print_certificate(bio, indent + 2, &msg, &clen)) return 0; - if (SSL_CONNECTION_IS_TLS13(sc) + if (SSL_IS_TLS13(ssl) && !ssl_print_extensions(bio, indent + 2, server, SSL3_MT_CERTIFICATE, &msg, &clen)) return 0; @@ -1380,84 +1221,13 @@ static int ssl_print_certificates(BIO *bio, const SSL_CONNECTION *sc, int server return 1; } -static int ssl_print_compressed_certificates(BIO *bio, const SSL_CONNECTION *sc, - int server, int indent, - const unsigned char *msg, - size_t msglen) -{ - size_t uclen; - size_t clen; - unsigned int alg; - int ret = 1; -#ifndef OPENSSL_NO_COMP_ALG - COMP_METHOD *method; - COMP_CTX *comp = NULL; - unsigned char* ucdata = NULL; -#endif - - if (msglen < 8) - return 0; - - alg = (msg[0] << 8) | msg[1]; - uclen = (msg[2] << 16) | (msg[3] << 8) | msg[4]; - clen = (msg[5] << 16) | (msg[6] << 8) | msg[7]; - if (msglen != clen + 8) - return 0; - - msg += 8; - BIO_indent(bio, indent, 80); - BIO_printf(bio, "Compression type=%s (0x%04x)\n", ssl_trace_str(alg, ssl_comp_cert_tbl), alg); - BIO_indent(bio, indent, 80); - BIO_printf(bio, "Uncompressed length=%d\n", (int)uclen); - BIO_indent(bio, indent, 80); - if (clen > 0) - BIO_printf(bio, "Compressed length=%d, Ratio=%f:1\n", (int)clen, (float)uclen / (float)clen); - else - BIO_printf(bio, "Compressed length=%d, Ratio=unknown\n", (int)clen); - - BIO_dump_indent(bio, (const char *)msg, clen, indent); - -#ifndef OPENSSL_NO_COMP_ALG - if (!ossl_comp_has_alg(alg)) - return 0; - - /* Check against certificate maximum size (coverity) */ - if (uclen == 0 || uclen > 0xFFFFFF || (ucdata = OPENSSL_malloc(uclen)) == NULL) - return 0; - - switch (alg) { - case TLSEXT_comp_cert_zlib: - method = COMP_zlib(); - break; - case TLSEXT_comp_cert_brotli: - method = COMP_brotli_oneshot(); - break; - case TLSEXT_comp_cert_zstd: - method = COMP_zstd_oneshot(); - break; - default: - goto err; - } - - if ((comp = COMP_CTX_new(method)) == NULL - || COMP_expand_block(comp, ucdata, uclen, (unsigned char*)msg, clen) != (int)uclen) - goto err; - - ret = ssl_print_certificates(bio, sc, server, indent, ucdata, uclen); - err: - COMP_CTX_free(comp); - OPENSSL_free(ucdata); -#endif - return ret; -} - -static int ssl_print_cert_request(BIO *bio, int indent, const SSL_CONNECTION *sc, +static int ssl_print_cert_request(BIO *bio, int indent, const SSL *ssl, const unsigned char *msg, size_t msglen) { size_t xlen; unsigned int sigalg; - if (SSL_CONNECTION_IS_TLS13(sc)) { + if (SSL_IS_TLS13(ssl)) { if (!ssl_print_hexbuf(bio, indent, "request_context", 1, &msg, &msglen)) return 0; if (!ssl_print_extensions(bio, indent, 1, @@ -1478,7 +1248,7 @@ static int ssl_print_cert_request(BIO *bio, int indent, const SSL_CONNECTION *sc msg += xlen; msglen -= xlen + 1; } - if (SSL_USE_SIGALGS(sc)) { + if (SSL_USE_SIGALGS(ssl)) { if (msglen < 2) return 0; xlen = (msg[0] << 8) | msg[1]; @@ -1532,7 +1302,7 @@ static int ssl_print_cert_request(BIO *bio, int indent, const SSL_CONNECTION *sc xlen -= dlen + 2; msg += dlen; } - if (SSL_CONNECTION_IS_TLS13(sc)) { + if (SSL_IS_TLS13(ssl)) { if (!ssl_print_hexbuf(bio, indent, "request_extensions", 2, &msg, &msglen)) return 0; @@ -1540,7 +1310,7 @@ static int ssl_print_cert_request(BIO *bio, int indent, const SSL_CONNECTION *sc return msglen == 0; } -static int ssl_print_ticket(BIO *bio, int indent, const SSL_CONNECTION *sc, +static int ssl_print_ticket(BIO *bio, int indent, const SSL *ssl, const unsigned char *msg, size_t msglen) { unsigned int tick_life; @@ -1560,7 +1330,7 @@ static int ssl_print_ticket(BIO *bio, int indent, const SSL_CONNECTION *sc, msg += 4; BIO_indent(bio, indent + 2, 80); BIO_printf(bio, "ticket_lifetime_hint=%u\n", tick_life); - if (SSL_CONNECTION_IS_TLS13(sc)) { + if (SSL_IS_TLS13(ssl)) { unsigned int ticket_age_add; if (msglen < 4) @@ -1580,7 +1350,7 @@ static int ssl_print_ticket(BIO *bio, int indent, const SSL_CONNECTION *sc, } if (!ssl_print_hexbuf(bio, indent + 2, "ticket", 2, &msg, &msglen)) return 0; - if (SSL_CONNECTION_IS_TLS13(sc) + if (SSL_IS_TLS13(ssl) && !ssl_print_extensions(bio, indent + 2, 0, SSL3_MT_NEWSESSION_TICKET, &msg, &msglen)) return 0; @@ -1589,7 +1359,7 @@ static int ssl_print_ticket(BIO *bio, int indent, const SSL_CONNECTION *sc, return 1; } -static int ssl_print_handshake(BIO *bio, const SSL_CONNECTION *sc, int server, +static int ssl_print_handshake(BIO *bio, const SSL *ssl, int server, const unsigned char *msg, size_t msglen, int indent) { @@ -1605,7 +1375,7 @@ static int ssl_print_handshake(BIO *bio, const SSL_CONNECTION *sc, int server, ssl_trace_str(htype, ssl_handshake_tbl), (int)hlen); msg += 4; msglen -= 4; - if (SSL_CONNECTION_IS_DTLS(sc)) { + if (SSL_IS_DTLS(ssl)) { if (msglen < 8) return 0; BIO_indent(bio, indent, 80); @@ -1621,7 +1391,7 @@ static int ssl_print_handshake(BIO *bio, const SSL_CONNECTION *sc, int server, return 0; switch (htype) { case SSL3_MT_CLIENT_HELLO: - if (!ssl_print_client_hello(bio, sc, indent + 2, msg, msglen)) + if (!ssl_print_client_hello(bio, ssl, indent + 2, msg, msglen)) return 0; break; @@ -1636,32 +1406,33 @@ static int ssl_print_handshake(BIO *bio, const SSL_CONNECTION *sc, int server, break; case SSL3_MT_SERVER_KEY_EXCHANGE: - if (!ssl_print_server_keyex(bio, indent + 2, sc, msg, msglen)) + if (!ssl_print_server_keyex(bio, indent + 2, ssl, msg, msglen)) return 0; break; case SSL3_MT_CLIENT_KEY_EXCHANGE: - if (!ssl_print_client_keyex(bio, indent + 2, sc, msg, msglen)) + if (!ssl_print_client_keyex(bio, indent + 2, ssl, msg, msglen)) return 0; break; case SSL3_MT_CERTIFICATE: - if (!ssl_print_certificates(bio, sc, server, indent + 2, msg, msglen)) + if (!ssl_print_certificates(bio, ssl, server, indent + 2, msg, msglen)) return 0; break; case SSL3_MT_COMPRESSED_CERTIFICATE: - if (!ssl_print_compressed_certificates(bio, sc, server, indent + 2, msg, msglen)) + if (!ssl_print_compressed_certificate(bio, ssl, server, indent + 2, + msg, msglen)) return 0; break; case SSL3_MT_CERTIFICATE_VERIFY: - if (!ssl_print_signature(bio, indent + 2, sc, &msg, &msglen)) + if (!ssl_print_signature(bio, indent + 2, ssl, &msg, &msglen)) return 0; break; case SSL3_MT_CERTIFICATE_REQUEST: - if (!ssl_print_cert_request(bio, indent + 2, sc, msg, msglen)) + if (!ssl_print_cert_request(bio, indent + 2, ssl, msg, msglen)) return 0; break; @@ -1675,7 +1446,7 @@ static int ssl_print_handshake(BIO *bio, const SSL_CONNECTION *sc, int server, break; case SSL3_MT_NEWSESSION_TICKET: - if (!ssl_print_ticket(bio, indent + 2, sc, msg, msglen)) + if (!ssl_print_ticket(bio, indent + 2, ssl, msg, msglen)) return 0; break; @@ -1708,23 +1479,6 @@ void SSL_trace(int write_p, int version, int content_type, { const unsigned char *msg = buf; BIO *bio = arg; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(ssl); -#ifndef OPENSSL_NO_QUIC - QUIC_CONNECTION *qc = QUIC_CONNECTION_FROM_SSL(ssl); - - if (qc != NULL) { - if (ossl_quic_trace(write_p, version, content_type, buf, msglen, ssl, - arg)) - return; - /* - * Otherwise ossl_quic_trace didn't handle this content_type so we - * fallback to standard TLS handling - */ - } -#endif - - if (sc == NULL) - return; switch (content_type) { case SSL3_RT_HEADER: @@ -1732,7 +1486,7 @@ void SSL_trace(int write_p, int version, int content_type, int hvers; /* avoid overlapping with length at the end of buffer */ - if (msglen < (size_t)(SSL_CONNECTION_IS_DTLS(sc) ? + if (msglen < (size_t)(SSL_IS_DTLS(ssl) ? DTLS1_RT_HEADER_LENGTH : SSL3_RT_HEADER_LENGTH)) { BIO_puts(bio, write_p ? "Sent" : "Received"); ssl_print_hex(bio, 0, " too short message", msg, msglen); @@ -1740,9 +1494,9 @@ void SSL_trace(int write_p, int version, int content_type, } hvers = msg[1] << 8 | msg[2]; BIO_puts(bio, write_p ? "Sent" : "Received"); - BIO_printf(bio, " TLS Record\nHeader:\n Version = %s (0x%x)\n", + BIO_printf(bio, " Record\nHeader:\n Version = %s (0x%x)\n", ssl_trace_str(hvers, ssl_version_tbl), hvers); - if (SSL_CONNECTION_IS_DTLS(sc)) { + if (SSL_IS_DTLS(ssl)) { BIO_printf(bio, " epoch=%d, sequence_number=%04x%04x%04x\n", (msg[3] << 8 | msg[4]), @@ -1762,7 +1516,7 @@ void SSL_trace(int write_p, int version, int content_type, break; case SSL3_RT_HANDSHAKE: - if (!ssl_print_handshake(bio, sc, sc->server ? write_p : !write_p, + if (!ssl_print_handshake(bio, ssl, ssl->server ? write_p : !write_p, msg, msglen, 4)) BIO_printf(bio, "Message length parse error!\n"); break; diff --git a/openssl/src/ssl/tls13_enc.c b/openssl/src/ssl/tls13_enc.c index 772a6fc17..53145238c 100644 --- a/openssl/src/ssl/tls13_enc.c +++ b/openssl/src/ssl/tls13_enc.c @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -18,8 +18,11 @@ #define TLS13_MAX_LABEL_LEN 249 -/* ASCII: "tls13 ", in hex for EBCDIC compatibility */ -static const unsigned char label_prefix[] = "\x74\x6C\x73\x31\x33\x20"; +#ifdef CHARSET_EBCDIC +static const unsigned char label_prefix[] = { 0x74, 0x6C, 0x73, 0x31, 0x33, 0x20, 0x00 }; +#else +static const unsigned char label_prefix[] = "tls13 "; +#endif /* * Given a |secret|; a |label| of length |labellen|; and |data| of length @@ -27,16 +30,14 @@ static const unsigned char label_prefix[] = "\x74\x6C\x73\x31\x33\x20"; * secret |outlen| bytes long and store it in the location pointed to be |out|. * The |data| value may be zero length. Any errors will be treated as fatal if * |fatal| is set. Returns 1 on success 0 on failure. - * If |raise_error| is set, ERR_raise is called on failure. */ -int tls13_hkdf_expand_ex(OSSL_LIB_CTX *libctx, const char *propq, - const EVP_MD *md, - const unsigned char *secret, - const unsigned char *label, size_t labellen, - const unsigned char *data, size_t datalen, - unsigned char *out, size_t outlen, int raise_error) +int tls13_hkdf_expand(SSL *s, const EVP_MD *md, const unsigned char *secret, + const unsigned char *label, size_t labellen, + const unsigned char *data, size_t datalen, + unsigned char *out, size_t outlen, int fatal) { - EVP_KDF *kdf = EVP_KDF_fetch(libctx, OSSL_KDF_NAME_TLS1_3_KDF, propq); + EVP_KDF *kdf = EVP_KDF_fetch(s->ctx->libctx, OSSL_KDF_NAME_TLS1_3_KDF, + s->ctx->propq); EVP_KDF_CTX *kctx; OSSL_PARAM params[7], *p = params; int mode = EVP_PKEY_HKDEF_MODE_EXPAND_ONLY; @@ -50,20 +51,24 @@ int tls13_hkdf_expand_ex(OSSL_LIB_CTX *libctx, const char *propq, return 0; if (labellen > TLS13_MAX_LABEL_LEN) { - if (raise_error) + if (fatal) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + } else { /* * Probably we have been called from SSL_export_keying_material(), * or SSL_export_keying_material_early(). */ ERR_raise(ERR_LIB_SSL, SSL_R_TLS_ILLEGAL_EXPORTER_LABEL); - + } EVP_KDF_CTX_free(kctx); return 0; } if ((ret = EVP_MD_get_size(md)) <= 0) { EVP_KDF_CTX_free(kctx); - if (raise_error) + if (fatal) + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + else ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); return 0; } @@ -89,41 +94,27 @@ int tls13_hkdf_expand_ex(OSSL_LIB_CTX *libctx, const char *propq, EVP_KDF_CTX_free(kctx); if (ret != 0) { - if (raise_error) + if (fatal) + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + else ERR_raise(ERR_LIB_SSL, ERR_R_INTERNAL_ERROR); } return ret == 0; } -int tls13_hkdf_expand(SSL_CONNECTION *s, const EVP_MD *md, - const unsigned char *secret, - const unsigned char *label, size_t labellen, - const unsigned char *data, size_t datalen, - unsigned char *out, size_t outlen, int fatal) -{ - int ret; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - - ret = tls13_hkdf_expand_ex(sctx->libctx, sctx->propq, md, - secret, label, labellen, data, datalen, - out, outlen, !fatal); - if (ret == 0 && fatal) - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - - return ret; -} - /* * Given a |secret| generate a |key| of length |keylen| bytes. Returns 1 on * success 0 on failure. */ -int tls13_derive_key(SSL_CONNECTION *s, const EVP_MD *md, - const unsigned char *secret, +int tls13_derive_key(SSL *s, const EVP_MD *md, const unsigned char *secret, unsigned char *key, size_t keylen) { - /* ASCII: "key", in hex for EBCDIC compatibility */ - static const unsigned char keylabel[] = "\x6B\x65\x79"; +#ifdef CHARSET_EBCDIC + static const unsigned char keylabel[] ={ 0x6B, 0x65, 0x79, 0x00 }; +#else + static const unsigned char keylabel[] = "key"; +#endif return tls13_hkdf_expand(s, md, secret, keylabel, sizeof(keylabel) - 1, NULL, 0, key, keylen, 1); @@ -133,23 +124,28 @@ int tls13_derive_key(SSL_CONNECTION *s, const EVP_MD *md, * Given a |secret| generate an |iv| of length |ivlen| bytes. Returns 1 on * success 0 on failure. */ -int tls13_derive_iv(SSL_CONNECTION *s, const EVP_MD *md, - const unsigned char *secret, +int tls13_derive_iv(SSL *s, const EVP_MD *md, const unsigned char *secret, unsigned char *iv, size_t ivlen) { - /* ASCII: "iv", in hex for EBCDIC compatibility */ - static const unsigned char ivlabel[] = "\x69\x76"; +#ifdef CHARSET_EBCDIC + static const unsigned char ivlabel[] = { 0x69, 0x76, 0x00 }; +#else + static const unsigned char ivlabel[] = "iv"; +#endif return tls13_hkdf_expand(s, md, secret, ivlabel, sizeof(ivlabel) - 1, NULL, 0, iv, ivlen, 1); } -int tls13_derive_finishedkey(SSL_CONNECTION *s, const EVP_MD *md, +int tls13_derive_finishedkey(SSL *s, const EVP_MD *md, const unsigned char *secret, unsigned char *fin, size_t finlen) { - /* ASCII: "finished", in hex for EBCDIC compatibility */ - static const unsigned char finishedlabel[] = "\x66\x69\x6E\x69\x73\x68\x65\x64"; +#ifdef CHARSET_EBCDIC + static const unsigned char finishedlabel[] = { 0x66, 0x69, 0x6E, 0x69, 0x73, 0x68, 0x65, 0x64, 0x00 }; +#else + static const unsigned char finishedlabel[] = "finished"; +#endif return tls13_hkdf_expand(s, md, secret, finishedlabel, sizeof(finishedlabel) - 1, NULL, 0, fin, finlen, 1); @@ -160,7 +156,7 @@ int tls13_derive_finishedkey(SSL_CONNECTION *s, const EVP_MD *md, * length |insecretlen|, generate a new secret and store it in the location * pointed to by |outsecret|. Returns 1 on success 0 on failure. */ -int tls13_generate_secret(SSL_CONNECTION *s, const EVP_MD *md, +int tls13_generate_secret(SSL *s, const EVP_MD *md, const unsigned char *prevsecret, const unsigned char *insecret, size_t insecretlen, @@ -174,11 +170,13 @@ int tls13_generate_secret(SSL_CONNECTION *s, const EVP_MD *md, OSSL_PARAM params[7], *p = params; int mode = EVP_PKEY_HKDEF_MODE_EXTRACT_ONLY; const char *mdname = EVP_MD_get0_name(md); - /* ASCII: "derived", in hex for EBCDIC compatibility */ - static const char derived_secret_label[] = "\x64\x65\x72\x69\x76\x65\x64"; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); +#ifdef CHARSET_EBCDIC + static const char derived_secret_label[] = { 0x64, 0x65, 0x72, 0x69, 0x76, 0x65, 0x64, 0x00 }; +#else + static const char derived_secret_label[] = "derived"; +#endif - kdf = EVP_KDF_fetch(sctx->libctx, OSSL_KDF_NAME_TLS1_3_KDF, sctx->propq); + kdf = EVP_KDF_fetch(s->ctx->libctx, OSSL_KDF_NAME_TLS1_3_KDF, s->ctx->propq); kctx = EVP_KDF_CTX_new(kdf); EVP_KDF_free(kdf); if (kctx == NULL) { @@ -227,9 +225,8 @@ int tls13_generate_secret(SSL_CONNECTION *s, const EVP_MD *md, * handshake secret. This requires the early secret to already have been * generated. Returns 1 on success 0 on failure. */ -int tls13_generate_handshake_secret(SSL_CONNECTION *s, - const unsigned char *insecret, - size_t insecretlen) +int tls13_generate_handshake_secret(SSL *s, const unsigned char *insecret, + size_t insecretlen) { /* Calls SSLfatal() if required */ return tls13_generate_secret(s, ssl_handshake_md(s), s->early_secret, @@ -242,7 +239,7 @@ int tls13_generate_handshake_secret(SSL_CONNECTION *s, * secret and store its length in |*secret_size|. Returns 1 on success 0 on * failure. */ -int tls13_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, +int tls13_generate_master_secret(SSL *s, unsigned char *out, unsigned char *prev, size_t prevlen, size_t *secret_size) { @@ -257,25 +254,20 @@ int tls13_generate_master_secret(SSL_CONNECTION *s, unsigned char *out, * Generates the mac for the Finished message. Returns the length of the MAC or * 0 on error. */ -size_t tls13_final_finish_mac(SSL_CONNECTION *s, const char *str, size_t slen, +size_t tls13_final_finish_mac(SSL *s, const char *str, size_t slen, unsigned char *out) { - const EVP_MD *md = ssl_handshake_md(s); - const char *mdname = EVP_MD_get0_name(md); + const char *mdname = EVP_MD_get0_name(ssl_handshake_md(s)); unsigned char hash[EVP_MAX_MD_SIZE]; unsigned char finsecret[EVP_MAX_MD_SIZE]; unsigned char *key = NULL; size_t len = 0, hashlen; OSSL_PARAM params[2], *p = params; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - - if (md == NULL) - return 0; /* Safe to cast away const here since we're not "getting" any data */ - if (sctx->propq != NULL) + if (s->ctx->propq != NULL) *p++ = OSSL_PARAM_construct_utf8_string(OSSL_ALG_PARAM_PROPERTIES, - (char *)sctx->propq, + (char *)s->ctx->propq, 0); *p = OSSL_PARAM_construct_end(); @@ -284,19 +276,19 @@ size_t tls13_final_finish_mac(SSL_CONNECTION *s, const char *str, size_t slen, goto err; } - if (str == SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->server_finished_label) { + if (str == s->method->ssl3_enc->server_finished_label) { key = s->server_finished_secret; } else if (SSL_IS_FIRST_HANDSHAKE(s)) { key = s->client_finished_secret; } else { - if (!tls13_derive_finishedkey(s, md, + if (!tls13_derive_finishedkey(s, ssl_handshake_md(s), s->client_app_traffic_secret, finsecret, hashlen)) goto err; key = finsecret; } - if (!EVP_Q_mac(sctx->libctx, "HMAC", sctx->propq, mdname, + if (!EVP_Q_mac(s->ctx->libctx, "HMAC", s->ctx->propq, mdname, params, key, hashlen, hash, hashlen, /* outsize as per sizeof(peer_finish_md) */ out, EVP_MAX_MD_SIZE * 2, &len)) { @@ -313,14 +305,14 @@ size_t tls13_final_finish_mac(SSL_CONNECTION *s, const char *str, size_t slen, * There isn't really a key block in TLSv1.3, but we still need this function * for initialising the cipher and hash. Returns 1 on success or 0 on failure. */ -int tls13_setup_key_block(SSL_CONNECTION *s) +int tls13_setup_key_block(SSL *s) { const EVP_CIPHER *c; const EVP_MD *hash; s->session->cipher = s->s3.tmp.new_cipher; - if (!ssl_cipher_get_evp(SSL_CONNECTION_GET_CTX(s), s->session, &c, &hash, - NULL, NULL, NULL, 0)) { + if (!ssl_cipher_get_evp(s->ctx, s->session, &c, &hash, NULL, NULL, NULL, + 0)) { /* Error is already recorded */ SSLfatal_alert(s, SSL_AD_INTERNAL_ERROR); return 0; @@ -334,19 +326,18 @@ int tls13_setup_key_block(SSL_CONNECTION *s) return 1; } -static int derive_secret_key_and_iv(SSL_CONNECTION *s, const EVP_MD *md, +static int derive_secret_key_and_iv(SSL *s, int sending, const EVP_MD *md, const EVP_CIPHER *ciph, const unsigned char *insecret, const unsigned char *hash, const unsigned char *label, size_t labellen, unsigned char *secret, - unsigned char *key, size_t *keylen, - unsigned char *iv, size_t *ivlen, - size_t *taglen) + unsigned char *key, unsigned char *iv, + EVP_CIPHER_CTX *ciph_ctx) { + size_t ivlen, keylen, taglen; int hashleni = EVP_MD_get_size(md); size_t hashlen; - int mode; /* Ensure cast to size_t is safe */ if (!ossl_assert(hashleni >= 0)) { @@ -361,13 +352,11 @@ static int derive_secret_key_and_iv(SSL_CONNECTION *s, const EVP_MD *md, return 0; } - *keylen = EVP_CIPHER_get_key_length(ciph); - - mode = EVP_CIPHER_get_mode(ciph); - if (mode == EVP_CIPH_CCM_MODE) { + keylen = EVP_CIPHER_get_key_length(ciph); + if (EVP_CIPHER_get_mode(ciph) == EVP_CIPH_CCM_MODE) { uint32_t algenc; - *ivlen = EVP_CCM_TLS_IV_LEN; + ivlen = EVP_CCM_TLS_IV_LEN; if (s->s3.tmp.new_cipher != NULL) { algenc = s->s3.tmp.new_cipher->algorithm_enc; } else if (s->session->cipher != NULL) { @@ -381,54 +370,231 @@ static int derive_secret_key_and_iv(SSL_CONNECTION *s, const EVP_MD *md, return 0; } if (algenc & (SSL_AES128CCM8 | SSL_AES256CCM8)) - *taglen = EVP_CCM8_TLS_TAG_LEN; + taglen = EVP_CCM8_TLS_TAG_LEN; else - *taglen = EVP_CCM_TLS_TAG_LEN; + taglen = EVP_CCM_TLS_TAG_LEN; } else { - int iivlen; + ivlen = EVP_CIPHER_get_iv_length(ciph); + taglen = 0; + } - if (mode == EVP_CIPH_GCM_MODE) { - *taglen = EVP_GCM_TLS_TAG_LEN; - } else { - /* CHACHA20P-POLY1305 */ - *taglen = EVP_CHACHAPOLY_TLS_TAG_LEN; + if (!tls13_derive_key(s, md, secret, key, keylen) + || !tls13_derive_iv(s, md, secret, iv, ivlen)) { + /* SSLfatal() already called */ + return 0; + } + + if (EVP_CipherInit_ex(ciph_ctx, ciph, NULL, NULL, NULL, sending) <= 0 + || !EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_AEAD_SET_IVLEN, ivlen, NULL) + || (taglen != 0 && !EVP_CIPHER_CTX_ctrl(ciph_ctx, EVP_CTRL_AEAD_SET_TAG, + taglen, NULL)) + || EVP_CipherInit_ex(ciph_ctx, NULL, NULL, key, NULL, -1) <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + return 0; + } + + return 1; +} + +#ifdef CHARSET_EBCDIC +static const unsigned char client_early_traffic[] = {0x63, 0x20, 0x65, 0x20, /*traffic*/0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x00}; +static const unsigned char client_handshake_traffic[] = {0x63, 0x20, 0x68, 0x73, 0x20, /*traffic*/0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x00}; +static const unsigned char client_application_traffic[] = {0x63, 0x20, 0x61, 0x70, 0x20, /*traffic*/0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x00}; +static const unsigned char server_handshake_traffic[] = {0x73, 0x20, 0x68, 0x73, 0x20, /*traffic*/0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x00}; +static const unsigned char server_application_traffic[] = {0x73, 0x20, 0x61, 0x70, 0x20, /*traffic*/0x74, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x00}; +static const unsigned char exporter_master_secret[] = {0x65, 0x78, 0x70, 0x20, /* master*/ 0x6D, 0x61, 0x73, 0x74, 0x65, 0x72, 0x00}; +static const unsigned char resumption_master_secret[] = {0x72, 0x65, 0x73, 0x20, /* master*/ 0x6D, 0x61, 0x73, 0x74, 0x65, 0x72, 0x00}; +static const unsigned char early_exporter_master_secret[] = {0x65, 0x20, 0x65, 0x78, 0x70, 0x20, /* master*/ 0x6D, 0x61, 0x73, 0x74, 0x65, 0x72, 0x00}; +#else +static const unsigned char client_early_traffic[] = "c e traffic"; +static const unsigned char client_handshake_traffic[] = "c hs traffic"; +static const unsigned char client_application_traffic[] = "c ap traffic"; +static const unsigned char server_handshake_traffic[] = "s hs traffic"; +static const unsigned char server_application_traffic[] = "s ap traffic"; +static const unsigned char exporter_master_secret[] = "exp master"; +static const unsigned char resumption_master_secret[] = "res master"; +static const unsigned char early_exporter_master_secret[] = "e exp master"; +#endif + +#ifndef OPENSSL_NO_QUIC +static int quic_change_cipher_state(SSL *s, int which) +{ + unsigned char hash[EVP_MAX_MD_SIZE]; + size_t hashlen = 0; + int hashleni; + int ret = 0; + const EVP_MD *md = NULL; + OSSL_ENCRYPTION_LEVEL level = ssl_encryption_initial; + int is_handshake = ((which & SSL3_CC_HANDSHAKE) == SSL3_CC_HANDSHAKE); + int is_client_read = ((which & SSL3_CHANGE_CIPHER_CLIENT_READ) == SSL3_CHANGE_CIPHER_CLIENT_READ); + int is_server_write = ((which & SSL3_CHANGE_CIPHER_SERVER_WRITE) == SSL3_CHANGE_CIPHER_SERVER_WRITE); + int is_early = (which & SSL3_CC_EARLY); + + if (is_early) { + EVP_MD_CTX *mdctx = NULL; + long handlen; + void *hdata; + unsigned int hashlenui; + const SSL_CIPHER *sslcipher = SSL_SESSION_get0_cipher(s->session); + + handlen = BIO_get_mem_data(s->s3.handshake_buffer, &hdata); + if (handlen <= 0) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_HANDSHAKE_LENGTH); + goto err; + } + + if (s->early_data_state == SSL_EARLY_DATA_CONNECTING + && s->max_early_data > 0 && s->session->ext.max_early_data == 0) { + /* + * If we are attempting to send early data, and we've decided to + * actually do it but max_early_data in s->session is 0 then we + * must be using an external PSK. + */ + if (!ossl_assert(s->psksession != NULL + && s->max_early_data + == s->psksession->ext.max_early_data)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + goto err; + } + sslcipher = SSL_SESSION_get0_cipher(s->psksession); + } + if (sslcipher == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_BAD_PSK); + goto err; + } + + /* + * We need to calculate the handshake digest using the digest from + * the session. We haven't yet selected our ciphersuite so we can't + * use ssl_handshake_md(). + */ + mdctx = EVP_MD_CTX_new(); + if (mdctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + md = ssl_md(s->ctx, sslcipher->algorithm2); + if (md == NULL || !EVP_DigestInit_ex(mdctx, md, NULL) + || !EVP_DigestUpdate(mdctx, hdata, handlen) + || !EVP_DigestFinal_ex(mdctx, hash, &hashlenui)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + EVP_MD_CTX_free(mdctx); + goto err; } - iivlen = EVP_CIPHER_get_iv_length(ciph); - if (iivlen < 0) { + hashlen = hashlenui; + EVP_MD_CTX_free(mdctx); + } else { + md = ssl_handshake_md(s); + if (md == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); - return 0; + goto err; + } + + if (!ssl3_digest_cached_records(s, 1) + || !ssl_handshake_hash(s, hash, sizeof(hash), &hashlen)) { + /* SSLfatal() already called */; + goto err; + } + + /* Ensure cast to size_t is safe */ + hashleni = EVP_MD_size(md); + if (!ossl_assert(hashleni >= 0)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + goto err; } - *ivlen = iivlen; + hashlen = (size_t)hashleni; } - if (!tls13_derive_key(s, md, secret, key, *keylen) - || !tls13_derive_iv(s, md, secret, iv, *ivlen)) { - /* SSLfatal() already called */ - return 0; + if (is_client_read || is_server_write) { + if (is_handshake) { + level = ssl_encryption_handshake; + + if (!tls13_hkdf_expand(s, md, s->handshake_secret, client_handshake_traffic, + sizeof(client_handshake_traffic)-1, hash, hashlen, + s->client_hand_traffic_secret, hashlen, 1) + || !ssl_log_secret(s, CLIENT_HANDSHAKE_LABEL, s->client_hand_traffic_secret, hashlen) + || !tls13_derive_finishedkey(s, md, s->client_hand_traffic_secret, + s->client_finished_secret, hashlen) + || !tls13_hkdf_expand(s, md, s->handshake_secret, server_handshake_traffic, + sizeof(server_handshake_traffic)-1, hash, hashlen, + s->server_hand_traffic_secret, hashlen, 1) + || !ssl_log_secret(s, SERVER_HANDSHAKE_LABEL, s->server_hand_traffic_secret, hashlen) + || !tls13_derive_finishedkey(s, md, s->server_hand_traffic_secret, + s->server_finished_secret, hashlen)) { + /* SSLfatal() already called */ + goto err; + } + } else { + level = ssl_encryption_application; + + if (!tls13_hkdf_expand(s, md, s->master_secret, client_application_traffic, + sizeof(client_application_traffic)-1, hash, hashlen, + s->client_app_traffic_secret, hashlen, 1) + || !ssl_log_secret(s, CLIENT_APPLICATION_LABEL, s->client_app_traffic_secret, hashlen) + || !tls13_hkdf_expand(s, md, s->master_secret, server_application_traffic, + sizeof(server_application_traffic)-1, hash, hashlen, + s->server_app_traffic_secret, hashlen, 1) + || !ssl_log_secret(s, SERVER_APPLICATION_LABEL, s->server_app_traffic_secret, hashlen)) { + /* SSLfatal() already called */ + goto err; + } + } + if (!quic_set_encryption_secrets(s, level)) { + /* SSLfatal() already called */ + goto err; + } + if (s->server) + s->quic_write_level = level; + else + s->quic_read_level = level; + } else { + /* is_client_write || is_server_read */ + + if (is_early) { + level = ssl_encryption_early_data; + + if (!tls13_hkdf_expand(s, md, s->early_secret, client_early_traffic, + sizeof(client_early_traffic)-1, hash, hashlen, + s->client_early_traffic_secret, hashlen, 1) + || !ssl_log_secret(s, CLIENT_EARLY_LABEL, s->client_early_traffic_secret, hashlen) + || !quic_set_encryption_secrets(s, level)) { + /* SSLfatal() already called */ + goto err; + } + } else if (is_handshake) { + level = ssl_encryption_handshake; + } else { + level = ssl_encryption_application; + /* + * We also create the resumption master secret, but this time use the + * hash for the whole handshake including the Client Finished + */ + if (!tls13_hkdf_expand(s, md, s->master_secret, resumption_master_secret, + sizeof(resumption_master_secret)-1, hash, hashlen, + s->resumption_master_secret, hashlen, 1)) { + /* SSLfatal() already called */ + goto err; + } + } + + if (level != ssl_encryption_early_data) { + if (s->server) + s->quic_read_level = level; + else { + s->quic_write_level = level; + } + } } - return 1; + ret = 1; + err: + return ret; } +#endif /* OPENSSL_NO_QUIC */ -int tls13_change_cipher_state(SSL_CONNECTION *s, int which) +int tls13_change_cipher_state(SSL *s, int which) { - /* ASCII: "c e traffic", in hex for EBCDIC compatibility */ - static const unsigned char client_early_traffic[] = "\x63\x20\x65\x20\x74\x72\x61\x66\x66\x69\x63"; - /* ASCII: "c hs traffic", in hex for EBCDIC compatibility */ - static const unsigned char client_handshake_traffic[] = "\x63\x20\x68\x73\x20\x74\x72\x61\x66\x66\x69\x63"; - /* ASCII: "c ap traffic", in hex for EBCDIC compatibility */ - static const unsigned char client_application_traffic[] = "\x63\x20\x61\x70\x20\x74\x72\x61\x66\x66\x69\x63"; - /* ASCII: "s hs traffic", in hex for EBCDIC compatibility */ - static const unsigned char server_handshake_traffic[] = "\x73\x20\x68\x73\x20\x74\x72\x61\x66\x66\x69\x63"; - /* ASCII: "s ap traffic", in hex for EBCDIC compatibility */ - static const unsigned char server_application_traffic[] = "\x73\x20\x61\x70\x20\x74\x72\x61\x66\x66\x69\x63"; - /* ASCII: "exp master", in hex for EBCDIC compatibility */ - static const unsigned char exporter_master_secret[] = "\x65\x78\x70\x20\x6D\x61\x73\x74\x65\x72"; - /* ASCII: "res master", in hex for EBCDIC compatibility */ - static const unsigned char resumption_master_secret[] = "\x72\x65\x73\x20\x6D\x61\x73\x74\x65\x72"; - /* ASCII: "e exp master", in hex for EBCDIC compatibility */ - static const unsigned char early_exporter_master_secret[] = "\x65\x20\x65\x78\x70\x20\x6D\x61\x73\x74\x65\x72"; - unsigned char iv[EVP_MAX_IV_LENGTH]; + unsigned char *iv; unsigned char key[EVP_MAX_KEY_LENGTH]; unsigned char secret[EVP_MAX_MD_SIZE]; unsigned char hashval[EVP_MAX_MD_SIZE]; @@ -436,17 +602,53 @@ int tls13_change_cipher_state(SSL_CONNECTION *s, int which) unsigned char *insecret; unsigned char *finsecret = NULL; const char *log_label = NULL; + EVP_CIPHER_CTX *ciph_ctx; size_t finsecretlen = 0; const unsigned char *label; size_t labellen, hashlen = 0; int ret = 0; const EVP_MD *md = NULL; const EVP_CIPHER *cipher = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); - size_t keylen, ivlen, taglen; - int level; - int direction = (which & SSL3_CC_READ) != 0 ? OSSL_RECORD_DIRECTION_READ - : OSSL_RECORD_DIRECTION_WRITE; +#if !defined(OPENSSL_NO_KTLS) && defined(OPENSSL_KTLS_TLS13) + ktls_crypto_info_t crypto_info; + BIO *bio; +#endif + +#ifndef OPENSSL_NO_QUIC + if (SSL_IS_QUIC(s)) + return quic_change_cipher_state(s, which); +#endif + + if (which & SSL3_CC_READ) { + if (s->enc_read_ctx != NULL) { + EVP_CIPHER_CTX_reset(s->enc_read_ctx); + } else { + s->enc_read_ctx = EVP_CIPHER_CTX_new(); + if (s->enc_read_ctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + } + ciph_ctx = s->enc_read_ctx; + iv = s->read_iv; + + RECORD_LAYER_reset_read_sequence(&s->rlayer); + } else { + s->statem.enc_write_state = ENC_WRITE_STATE_INVALID; + if (s->enc_write_ctx != NULL) { + EVP_CIPHER_CTX_reset(s->enc_write_ctx); + } else { + s->enc_write_ctx = EVP_CIPHER_CTX_new(); + if (s->enc_write_ctx == NULL) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + goto err; + } + } + ciph_ctx = s->enc_write_ctx; + iv = s->write_iv; + + RECORD_LAYER_reset_write_sequence(&s->rlayer); + } if (((which & SSL3_CC_CLIENT) && (which & SSL3_CC_WRITE)) || ((which & SSL3_CC_SERVER) && (which & SSL3_CC_READ))) { @@ -496,7 +698,7 @@ int tls13_change_cipher_state(SSL_CONNECTION *s, int which) */ mdctx = EVP_MD_CTX_new(); if (mdctx == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_EVP_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } @@ -504,14 +706,14 @@ int tls13_change_cipher_state(SSL_CONNECTION *s, int which) * This ups the ref count on cipher so we better make sure we free * it again */ - if (!ssl_cipher_get_evp_cipher(sctx, sslcipher, &cipher)) { + if (!ssl_cipher_get_evp_cipher(s->ctx, sslcipher, &cipher)) { /* Error is already recorded */ SSLfatal_alert(s, SSL_AD_INTERNAL_ERROR); EVP_MD_CTX_free(mdctx); goto err; } - md = ssl_md(sctx, sslcipher->algorithm2); + md = ssl_md(s->ctx, sslcipher->algorithm2); if (md == NULL || !EVP_DigestInit_ex(mdctx, md, NULL) || !EVP_DigestUpdate(mdctx, hdata, handlen) || !EVP_DigestFinal_ex(mdctx, hashval, &hashlenui)) { @@ -620,12 +822,12 @@ int tls13_change_cipher_state(SSL_CONNECTION *s, int which) } /* check whether cipher is known */ - if (!ossl_assert(cipher != NULL)) + if(!ossl_assert(cipher != NULL)) goto err; - if (!derive_secret_key_and_iv(s, md, cipher, + if (!derive_secret_key_and_iv(s, which & SSL3_CC_WRITE, md, cipher, insecret, hash, label, labellen, secret, key, - &keylen, iv, &ivlen, &taglen)) { + iv, ciph_ctx)) { /* SSLfatal() already called */ goto err; } @@ -662,28 +864,52 @@ int tls13_change_cipher_state(SSL_CONNECTION *s, int which) goto err; } - if ((which & SSL3_CC_WRITE) != 0) { - if (!s->server && label == client_early_traffic) - s->rlayer.wrlmethod->set_plain_alerts(s->rlayer.wrl, 1); - else - s->rlayer.wrlmethod->set_plain_alerts(s->rlayer.wrl, 0); - } + if (!s->server && label == client_early_traffic) + s->statem.enc_write_state = ENC_WRITE_STATE_WRITE_PLAIN_ALERTS; + else + s->statem.enc_write_state = ENC_WRITE_STATE_VALID; +#ifndef OPENSSL_NO_KTLS +# if defined(OPENSSL_KTLS_TLS13) + if (!(which & SSL3_CC_WRITE) + || !(which & SSL3_CC_APPLICATION) + || (s->options & SSL_OP_ENABLE_KTLS) == 0) + goto skip_ktls; + + /* ktls supports only the maximum fragment size */ + if (ssl_get_max_send_fragment(s) != SSL3_RT_MAX_PLAIN_LENGTH) + goto skip_ktls; - level = (which & SSL3_CC_EARLY) != 0 - ? OSSL_RECORD_PROTECTION_LEVEL_EARLY - : ((which &SSL3_CC_HANDSHAKE) != 0 - ? OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE - : OSSL_RECORD_PROTECTION_LEVEL_APPLICATION); - - if (!ssl_set_new_record_layer(s, s->version, - direction, - level, secret, hashlen, key, keylen, iv, - ivlen, NULL, 0, cipher, taglen, NID_undef, - NULL, NULL, md)) { - /* SSLfatal already called */ + /* ktls does not support record padding */ + if (s->record_padding_cb != NULL) + goto skip_ktls; + + /* check that cipher is supported */ + if (!ktls_check_supported_cipher(s, cipher, ciph_ctx)) + goto skip_ktls; + + bio = s->wbio; + + if (!ossl_assert(bio != NULL)) { + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } + /* All future data will get encrypted by ktls. Flush the BIO or skip ktls */ + if (BIO_flush(bio) <= 0) + goto skip_ktls; + + /* configure kernel crypto structure */ + if (!ktls_configure_crypto(s, cipher, ciph_ctx, + RECORD_LAYER_get_write_sequence(&s->rlayer), + &crypto_info, NULL, iv, key, NULL, 0)) + goto skip_ktls; + + /* ktls works with user provided buffers directly */ + if (BIO_set_ktls(bio, &crypto_info, which & SSL3_CC_WRITE)) + ssl3_release_write_buffer(s); +skip_ktls: +# endif +#endif ret = 1; err: if ((which & SSL3_CC_EARLY) != 0) { @@ -695,60 +921,49 @@ int tls13_change_cipher_state(SSL_CONNECTION *s, int which) return ret; } -int tls13_update_key(SSL_CONNECTION *s, int sending) +int tls13_update_key(SSL *s, int sending) { - /* ASCII: "traffic upd", in hex for EBCDIC compatibility */ - static const unsigned char application_traffic[] = "\x74\x72\x61\x66\x66\x69\x63\x20\x75\x70\x64"; +#ifdef CHARSET_EBCDIC + static const unsigned char application_traffic[] = { 0x74, 0x72 ,0x61 ,0x66 ,0x66 ,0x69 ,0x63 ,0x20 ,0x75 ,0x70 ,0x64, 0x00}; +#else + static const unsigned char application_traffic[] = "traffic upd"; +#endif const EVP_MD *md = ssl_handshake_md(s); - size_t hashlen; + size_t hashlen = EVP_MD_get_size(md); unsigned char key[EVP_MAX_KEY_LENGTH]; - unsigned char *insecret; + unsigned char *insecret, *iv; unsigned char secret[EVP_MAX_MD_SIZE]; - char *log_label; - size_t keylen, ivlen, taglen; - int ret = 0, l; - int direction = sending ? OSSL_RECORD_DIRECTION_WRITE - : OSSL_RECORD_DIRECTION_READ; - unsigned char iv[EVP_MAX_IV_LENGTH]; - - if ((l = EVP_MD_get_size(md)) <= 0) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); - return 0; - } - hashlen = (size_t)l; + EVP_CIPHER_CTX *ciph_ctx; + int ret = 0; if (s->server == sending) insecret = s->server_app_traffic_secret; else insecret = s->client_app_traffic_secret; - if (!derive_secret_key_and_iv(s, md, + if (sending) { + s->statem.enc_write_state = ENC_WRITE_STATE_INVALID; + iv = s->write_iv; + ciph_ctx = s->enc_write_ctx; + RECORD_LAYER_reset_write_sequence(&s->rlayer); + } else { + iv = s->read_iv; + ciph_ctx = s->enc_read_ctx; + RECORD_LAYER_reset_read_sequence(&s->rlayer); + } + + if (!derive_secret_key_and_iv(s, sending, ssl_handshake_md(s), s->s3.tmp.new_sym_enc, insecret, NULL, application_traffic, sizeof(application_traffic) - 1, secret, key, - &keylen, iv, &ivlen, &taglen)) { + iv, ciph_ctx)) { /* SSLfatal() already called */ goto err; } memcpy(insecret, secret, hashlen); - if (!ssl_set_new_record_layer(s, s->version, - direction, - OSSL_RECORD_PROTECTION_LEVEL_APPLICATION, - insecret, hashlen, key, keylen, iv, ivlen, NULL, 0, - s->s3.tmp.new_sym_enc, taglen, NID_undef, NULL, - NULL, md)) { - /* SSLfatal already called */ - goto err; - } - - /* Call Key log on successful traffic secret update */ - log_label = s->server == sending ? SERVER_APPLICATION_N_LABEL : CLIENT_APPLICATION_N_LABEL; - if (!ssl_log_secret(s, log_label, secret, hashlen)) { - /* SSLfatal() already called */ - goto err; - } + s->statem.enc_write_state = ENC_WRITE_STATE_VALID; ret = 1; err: OPENSSL_cleanse(key, sizeof(key)); @@ -765,22 +980,24 @@ int tls13_alert_code(int code) return tls1_alert_code(code); } -int tls13_export_keying_material(SSL_CONNECTION *s, - unsigned char *out, size_t olen, +int tls13_export_keying_material(SSL *s, unsigned char *out, size_t olen, const char *label, size_t llen, const unsigned char *context, size_t contextlen, int use_context) { unsigned char exportsecret[EVP_MAX_MD_SIZE]; - /* ASCII: "exporter", in hex for EBCDIC compatibility */ - static const unsigned char exporterlabel[] = "\x65\x78\x70\x6F\x72\x74\x65\x72"; +#ifdef CHARSET_EBCDIC + static const unsigned char exporterlabel[] = {0x65, 0x78, 0x70, 0x6F, 0x72, 0x74, 0x65, 0x72, 0x00}; +#else + static const unsigned char exporterlabel[] = "exporter"; +#endif unsigned char hash[EVP_MAX_MD_SIZE], data[EVP_MAX_MD_SIZE]; const EVP_MD *md = ssl_handshake_md(s); EVP_MD_CTX *ctx = EVP_MD_CTX_new(); unsigned int hashsize, datalen; int ret = 0; - if (ctx == NULL || md == NULL || !ossl_statem_export_allowed(s)) + if (ctx == NULL || !ossl_statem_export_allowed(s)) goto err; if (!use_context) @@ -805,14 +1022,16 @@ int tls13_export_keying_material(SSL_CONNECTION *s, return ret; } -int tls13_export_keying_material_early(SSL_CONNECTION *s, - unsigned char *out, size_t olen, +int tls13_export_keying_material_early(SSL *s, unsigned char *out, size_t olen, const char *label, size_t llen, const unsigned char *context, size_t contextlen) { - /* ASCII: "exporter", in hex for EBCDIC compatibility */ - static const unsigned char exporterlabel[] = "\x65\x78\x70\x6F\x72\x74\x65\x72"; +#ifdef CHARSET_EBCDIC + static const unsigned char exporterlabel[] = {0x65, 0x78, 0x70, 0x6F, 0x72, 0x74, 0x65, 0x72, 0x00}; +#else + static const unsigned char exporterlabel[] = "exporter"; +#endif unsigned char exportsecret[EVP_MAX_MD_SIZE]; unsigned char hash[EVP_MAX_MD_SIZE], data[EVP_MAX_MD_SIZE]; const EVP_MD *md; @@ -830,7 +1049,7 @@ int tls13_export_keying_material_early(SSL_CONNECTION *s, else sslcipher = SSL_SESSION_get0_cipher(s->session); - md = ssl_md(SSL_CONNECTION_GET_CTX(s), sslcipher->algorithm2); + md = ssl_md(s->ctx, sslcipher->algorithm2); /* * Calculate the hash value and store it in |data|. The reason why @@ -847,8 +1066,7 @@ int tls13_export_keying_material_early(SSL_CONNECTION *s, * * Here Transcript-Hash is the cipher suite hash algorithm. */ - if (md == NULL - || EVP_DigestInit_ex(ctx, md, NULL) <= 0 + if (EVP_DigestInit_ex(ctx, md, NULL) <= 0 || EVP_DigestUpdate(ctx, context, contextlen) <= 0 || EVP_DigestFinal_ex(ctx, hash, &hashsize) <= 0 || EVP_DigestInit_ex(ctx, md, NULL) <= 0 diff --git a/openssl/src/ssl/tls_depr.c b/openssl/src/ssl/tls_depr.c index 85ed9f25f..1761ba1d8 100644 --- a/openssl/src/ssl/tls_depr.c +++ b/openssl/src/ssl/tls_depr.c @@ -64,14 +64,10 @@ const EVP_MD *tls_get_digest_from_engine(int nid) } #ifndef OPENSSL_NO_ENGINE -int tls_engine_load_ssl_client_cert(SSL_CONNECTION *s, X509 **px509, - EVP_PKEY **ppkey) +int tls_engine_load_ssl_client_cert(SSL *s, X509 **px509, EVP_PKEY **ppkey) { - SSL *ssl = SSL_CONNECTION_GET_SSL(s); - - return ENGINE_load_ssl_client_cert(SSL_CONNECTION_GET_CTX(s)->client_cert_engine, - ssl, - SSL_get_client_CA_list(ssl), + return ENGINE_load_ssl_client_cert(s->ctx->client_cert_engine, s, + SSL_get_client_CA_list(s), px509, ppkey, NULL, NULL, NULL); } #endif diff --git a/openssl/src/ssl/tls_srp.c b/openssl/src/ssl/tls_srp.c index 80c70bbaa..0ce3290dc 100644 --- a/openssl/src/ssl/tls_srp.c +++ b/openssl/src/ssl/tls_srp.c @@ -57,7 +57,7 @@ int SSL_CTX_SRP_CTX_free(SSL_CTX *ctx) * The public API SSL_SRP_CTX_free() is deprecated so we use * ssl_srp_ctx_free_intern() internally. */ -int ssl_srp_ctx_free_intern(SSL_CONNECTION *s) +int ssl_srp_ctx_free_intern(SSL *s) { if (s == NULL) return 0; @@ -78,21 +78,18 @@ int ssl_srp_ctx_free_intern(SSL_CONNECTION *s) int SSL_SRP_CTX_free(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - /* the call works with NULL sc */ - return ssl_srp_ctx_free_intern(sc); + return ssl_srp_ctx_free_intern(s); } /* * The public API SSL_SRP_CTX_init() is deprecated so we use * ssl_srp_ctx_init_intern() internally. */ -int ssl_srp_ctx_init_intern(SSL_CONNECTION *s) +int ssl_srp_ctx_init_intern(SSL *s) { SSL_CTX *ctx; - if (s == NULL || (ctx = SSL_CONNECTION_GET_CTX(s)) == NULL) + if ((s == NULL) || ((ctx = s->ctx) == NULL)) return 0; memset(&s->srp_ctx, 0, sizeof(s->srp_ctx)); @@ -159,10 +156,7 @@ int ssl_srp_ctx_init_intern(SSL_CONNECTION *s) int SSL_SRP_CTX_init(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - /* the call works with NULL sc */ - return ssl_srp_ctx_init_intern(sc); + return ssl_srp_ctx_init_intern(s); } /* @@ -190,17 +184,15 @@ int SSL_CTX_SRP_CTX_init(SSL_CTX *ctx) * The public API SSL_srp_server_param_with_username() is deprecated so we use * ssl_srp_server_param_with_username_intern() internally. */ -int ssl_srp_server_param_with_username_intern(SSL_CONNECTION *s, int *ad) +int ssl_srp_server_param_with_username_intern(SSL *s, int *ad) { unsigned char b[SSL_MAX_MASTER_KEY_LENGTH]; int al; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); *ad = SSL_AD_UNKNOWN_PSK_IDENTITY; if ((s->srp_ctx.TLS_ext_srp_username_callback != NULL) && ((al = - s->srp_ctx.TLS_ext_srp_username_callback(SSL_CONNECTION_GET_SSL(s), - ad, + s->srp_ctx.TLS_ext_srp_username_callback(s, ad, s->srp_ctx.SRP_cb_arg)) != SSL_ERROR_NONE)) return al; @@ -211,8 +203,7 @@ int ssl_srp_server_param_with_username_intern(SSL_CONNECTION *s, int *ad) (s->srp_ctx.s == NULL) || (s->srp_ctx.v == NULL)) return SSL3_AL_FATAL; - if (RAND_priv_bytes_ex(SSL_CONNECTION_GET_CTX(s)->libctx, b, sizeof(b), - 0) <= 0) + if (RAND_priv_bytes_ex(s->ctx->libctx, b, sizeof(b), 0) <= 0) return SSL3_AL_FATAL; s->srp_ctx.b = BN_bin2bn(b, sizeof(b), NULL); OPENSSL_cleanse(b, sizeof(b)); @@ -221,18 +212,13 @@ int ssl_srp_server_param_with_username_intern(SSL_CONNECTION *s, int *ad) return ((s->srp_ctx.B = SRP_Calc_B_ex(s->srp_ctx.b, s->srp_ctx.N, s->srp_ctx.g, - s->srp_ctx.v, sctx->libctx, sctx->propq)) != + s->srp_ctx.v, s->ctx->libctx, s->ctx->propq)) != NULL) ? SSL_ERROR_NONE : SSL3_AL_FATAL; } int SSL_srp_server_param_with_username(SSL *s, int *ad) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return SSL3_AL_FATAL; - - return ssl_srp_server_param_with_username_intern(sc, ad); + return ssl_srp_server_param_with_username_intern(s, ad); } /* @@ -242,23 +228,17 @@ int SSL_srp_server_param_with_username(SSL *s, int *ad) int SSL_set_srp_server_param_pw(SSL *s, const char *user, const char *pass, const char *grp) { - SRP_gN *GN; - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return -1; - - GN = SRP_get_default_gN(grp); + SRP_gN *GN = SRP_get_default_gN(grp); if (GN == NULL) return -1; - sc->srp_ctx.N = BN_dup(GN->N); - sc->srp_ctx.g = BN_dup(GN->g); - BN_clear_free(sc->srp_ctx.v); - sc->srp_ctx.v = NULL; - BN_clear_free(sc->srp_ctx.s); - sc->srp_ctx.s = NULL; - if (!SRP_create_verifier_BN_ex(user, pass, &sc->srp_ctx.s, &sc->srp_ctx.v, - sc->srp_ctx.N, sc->srp_ctx.g, s->ctx->libctx, + s->srp_ctx.N = BN_dup(GN->N); + s->srp_ctx.g = BN_dup(GN->g); + BN_clear_free(s->srp_ctx.v); + s->srp_ctx.v = NULL; + BN_clear_free(s->srp_ctx.s); + s->srp_ctx.s = NULL; + if (!SRP_create_verifier_BN_ex(user, pass, &s->srp_ctx.s, &s->srp_ctx.v, + GN->N, GN->g, s->ctx->libctx, s->ctx->propq)) return -1; @@ -268,72 +248,66 @@ int SSL_set_srp_server_param_pw(SSL *s, const char *user, const char *pass, int SSL_set_srp_server_param(SSL *s, const BIGNUM *N, const BIGNUM *g, BIGNUM *sa, BIGNUM *v, char *info) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return -1; - if (N != NULL) { - if (sc->srp_ctx.N != NULL) { - if (!BN_copy(sc->srp_ctx.N, N)) { - BN_free(sc->srp_ctx.N); - sc->srp_ctx.N = NULL; + if (s->srp_ctx.N != NULL) { + if (!BN_copy(s->srp_ctx.N, N)) { + BN_free(s->srp_ctx.N); + s->srp_ctx.N = NULL; } } else - sc->srp_ctx.N = BN_dup(N); + s->srp_ctx.N = BN_dup(N); } if (g != NULL) { - if (sc->srp_ctx.g != NULL) { - if (!BN_copy(sc->srp_ctx.g, g)) { - BN_free(sc->srp_ctx.g); - sc->srp_ctx.g = NULL; + if (s->srp_ctx.g != NULL) { + if (!BN_copy(s->srp_ctx.g, g)) { + BN_free(s->srp_ctx.g); + s->srp_ctx.g = NULL; } } else - sc->srp_ctx.g = BN_dup(g); + s->srp_ctx.g = BN_dup(g); } if (sa != NULL) { - if (sc->srp_ctx.s != NULL) { - if (!BN_copy(sc->srp_ctx.s, sa)) { - BN_free(sc->srp_ctx.s); - sc->srp_ctx.s = NULL; + if (s->srp_ctx.s != NULL) { + if (!BN_copy(s->srp_ctx.s, sa)) { + BN_free(s->srp_ctx.s); + s->srp_ctx.s = NULL; } } else - sc->srp_ctx.s = BN_dup(sa); + s->srp_ctx.s = BN_dup(sa); } if (v != NULL) { - if (sc->srp_ctx.v != NULL) { - if (!BN_copy(sc->srp_ctx.v, v)) { - BN_free(sc->srp_ctx.v); - sc->srp_ctx.v = NULL; + if (s->srp_ctx.v != NULL) { + if (!BN_copy(s->srp_ctx.v, v)) { + BN_free(s->srp_ctx.v); + s->srp_ctx.v = NULL; } } else - sc->srp_ctx.v = BN_dup(v); + s->srp_ctx.v = BN_dup(v); } if (info != NULL) { - if (sc->srp_ctx.info) - OPENSSL_free(sc->srp_ctx.info); - if ((sc->srp_ctx.info = OPENSSL_strdup(info)) == NULL) + if (s->srp_ctx.info) + OPENSSL_free(s->srp_ctx.info); + if ((s->srp_ctx.info = OPENSSL_strdup(info)) == NULL) return -1; } - if (!(sc->srp_ctx.N) || - !(sc->srp_ctx.g) || !(sc->srp_ctx.s) || !(sc->srp_ctx.v)) + if (!(s->srp_ctx.N) || + !(s->srp_ctx.g) || !(s->srp_ctx.s) || !(s->srp_ctx.v)) return -1; return 1; } -int srp_generate_server_master_secret(SSL_CONNECTION *s) +int srp_generate_server_master_secret(SSL *s) { BIGNUM *K = NULL, *u = NULL; - int ret = 0, tmp_len = 0; + int ret = -1, tmp_len = 0; unsigned char *tmp = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); if (!SRP_Verify_A_mod_N(s->srp_ctx.A, s->srp_ctx.N)) goto err; if ((u = SRP_Calc_u_ex(s->srp_ctx.A, s->srp_ctx.B, s->srp_ctx.N, - sctx->libctx, sctx->propq)) == NULL) + s->ctx->libctx, s->ctx->propq)) == NULL) goto err; if ((K = SRP_Calc_server_key(s->srp_ctx.A, s->srp_ctx.v, u, s->srp_ctx.b, s->srp_ctx.N)) == NULL) @@ -341,7 +315,7 @@ int srp_generate_server_master_secret(SSL_CONNECTION *s) tmp_len = BN_num_bytes(K); if ((tmp = OPENSSL_malloc(tmp_len)) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } BN_bn2bin(K, tmp); @@ -354,45 +328,44 @@ int srp_generate_server_master_secret(SSL_CONNECTION *s) } /* client side */ -int srp_generate_client_master_secret(SSL_CONNECTION *s) +int srp_generate_client_master_secret(SSL *s) { BIGNUM *x = NULL, *u = NULL, *K = NULL; - int ret = 0, tmp_len = 0; + int ret = -1, tmp_len = 0; char *passwd = NULL; unsigned char *tmp = NULL; - SSL_CTX *sctx = SSL_CONNECTION_GET_CTX(s); /* * Checks if b % n == 0 */ if (SRP_Verify_B_mod_N(s->srp_ctx.B, s->srp_ctx.N) == 0 || (u = SRP_Calc_u_ex(s->srp_ctx.A, s->srp_ctx.B, s->srp_ctx.N, - sctx->libctx, sctx->propq)) + s->ctx->libctx, s->ctx->propq)) == NULL || s->srp_ctx.SRP_give_srp_client_pwd_callback == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } - if ((passwd = s->srp_ctx.SRP_give_srp_client_pwd_callback(SSL_CONNECTION_GET_SSL(s), - s->srp_ctx.SRP_cb_arg)) + if ((passwd = s->srp_ctx.SRP_give_srp_client_pwd_callback(s, + s->srp_ctx.SRP_cb_arg)) == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_R_CALLBACK_FAILED); goto err; } if ((x = SRP_Calc_x_ex(s->srp_ctx.s, s->srp_ctx.login, passwd, - sctx->libctx, sctx->propq)) == NULL + s->ctx->libctx, s->ctx->propq)) == NULL || (K = SRP_Calc_client_key_ex(s->srp_ctx.N, s->srp_ctx.B, s->srp_ctx.g, x, s->srp_ctx.a, u, - sctx->libctx, - sctx->propq)) == NULL) { + s->ctx->libctx, + s->ctx->propq)) == NULL) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); goto err; } tmp_len = BN_num_bytes(K); if ((tmp = OPENSSL_malloc(tmp_len)) == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_CRYPTO_LIB); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); goto err; } BN_bn2bin(K, tmp); @@ -407,7 +380,7 @@ int srp_generate_client_master_secret(SSL_CONNECTION *s) return ret; } -int srp_verify_server_param(SSL_CONNECTION *s) +int srp_verify_server_param(SSL *s) { SRP_CTX *srp = &s->srp_ctx; /* @@ -426,8 +399,7 @@ int srp_verify_server_param(SSL_CONNECTION *s) } if (srp->SRP_verify_param_callback) { - if (srp->SRP_verify_param_callback(SSL_CONNECTION_GET_SSL(s), - srp->SRP_cb_arg) <= 0) { + if (srp->SRP_verify_param_callback(s, srp->SRP_cb_arg) <= 0) { SSLfatal(s, SSL_AD_INSUFFICIENT_SECURITY, SSL_R_CALLBACK_FAILED); return 0; } @@ -444,12 +416,11 @@ int srp_verify_server_param(SSL_CONNECTION *s) * The public API SRP_Calc_A_param() is deprecated so we use * ssl_srp_calc_a_param_intern() internally. */ -int ssl_srp_calc_a_param_intern(SSL_CONNECTION *s) +int ssl_srp_calc_a_param_intern(SSL *s) { unsigned char rnd[SSL_MAX_MASTER_KEY_LENGTH]; - if (RAND_priv_bytes_ex(SSL_CONNECTION_GET_CTX(s)->libctx, - rnd, sizeof(rnd), 0) <= 0) + if (RAND_priv_bytes_ex(s->ctx->libctx, rnd, sizeof(rnd), 0) <= 0) return 0; s->srp_ctx.a = BN_bin2bn(rnd, sizeof(rnd), s->srp_ctx.a); OPENSSL_cleanse(rnd, sizeof(rnd)); @@ -462,59 +433,34 @@ int ssl_srp_calc_a_param_intern(SSL_CONNECTION *s) int SRP_Calc_A_param(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return 0; - - return ssl_srp_calc_a_param_intern(sc); + return ssl_srp_calc_a_param_intern(s); } BIGNUM *SSL_get_srp_g(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - if (sc->srp_ctx.g != NULL) - return sc->srp_ctx.g; + if (s->srp_ctx.g != NULL) + return s->srp_ctx.g; return s->ctx->srp_ctx.g; } BIGNUM *SSL_get_srp_N(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - if (sc->srp_ctx.N != NULL) - return sc->srp_ctx.N; + if (s->srp_ctx.N != NULL) + return s->srp_ctx.N; return s->ctx->srp_ctx.N; } char *SSL_get_srp_username(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - if (sc->srp_ctx.login != NULL) - return sc->srp_ctx.login; + if (s->srp_ctx.login != NULL) + return s->srp_ctx.login; return s->ctx->srp_ctx.login; } char *SSL_get_srp_userinfo(SSL *s) { - SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL(s); - - if (sc == NULL) - return NULL; - - if (sc->srp_ctx.info != NULL) - return sc->srp_ctx.info; + if (s->srp_ctx.info != NULL) + return s->srp_ctx.info; return s->ctx->srp_ctx.info; } diff --git a/openssl/tools/gen_config.json b/openssl/tools/gen_config.json index 3703de5cc..df6a927ed 100644 --- a/openssl/tools/gen_config.json +++ b/openssl/tools/gen_config.json @@ -2,8 +2,5 @@ "-fPIC", "no-shared", "no-module", - "no-md4", - "no-mdc2", - "no-whirlpool", "-w" ] \ No newline at end of file diff --git a/openssl/tools/gen_linux.js b/openssl/tools/gen_linux.js index 344fb027c..804f695a6 100644 --- a/openssl/tools/gen_linux.js +++ b/openssl/tools/gen_linux.js @@ -33,7 +33,7 @@ const linux_config = { "prefix": "/usr/bin/riscv64-linux-gnu-" }, "loong64": { - "arch": "linux64-loongarch64", + "arch": "linux-generic64", "prefix": "/usr/cross-tools/bin/loongarch64-unknown-linux-gnu-" }, };